From bb94918905c238fd194f356090c4c0a2dca5b445 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 12 May 2021 11:36:19 -0400 Subject: [PATCH 0001/1042] storyteller-web "monolith", base dependencies --- Cargo.lock | 2802 +++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 46 + README.md | 4 + src/main.rs | 3 + 4 files changed, 2855 insertions(+) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 src/main.rs diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000..f9fb627a16 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2802 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "actix-codec" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78d1833b3838dbe990df0f1f87baf640cf6146e898166afe401839d1b001e570" +dependencies = [ + "bitflags", + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project 0.4.28", + "tokio 0.2.25", + "tokio-util", +] + +[[package]] +name = "actix-connect" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" +dependencies = [ + "actix-codec", + "actix-rt 1.1.1", + "actix-service", + "actix-utils 2.0.0", + "derive_more", + "either", + "futures-util", + "http", + "log", + "trust-dns-proto", + "trust-dns-resolver", +] + +[[package]] +name = "actix-cors" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36b133d8026a9f209a9aeeeacd028e7451bcca975f592881b305d37983f303d7" +dependencies = [ + "actix-web", + "derive_more", + "futures-util", + "log", + "once_cell", + "tinyvec", +] + +[[package]] +name = "actix-files" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51e8a9146c12fce92a6e4c24b8c4d9b05268130bfd8d61bc587e822c32ce689" +dependencies = [ + "actix-service", + "actix-web", + "bitflags", + "bytes 0.5.6", + "derive_more", + "futures-core", + "futures-util", + "log", + "mime", + "mime_guess", + "percent-encoding 2.1.0", + "v_htmlescape", +] + +[[package]] +name = "actix-http" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "452299e87817ae5673910e53c243484ca38be3828db819b6011736fc6982e874" +dependencies = [ + "actix-codec", + "actix-connect", + "actix-rt 1.1.1", + "actix-service", + "actix-threadpool", + "actix-utils 2.0.0", + "base64", + "bitflags", + "brotli2", + "bytes 0.5.6", + "cookie", + "copyless", + "derive_more", + "either", + "encoding_rs", + "flate2", + "futures-channel", + "futures-core", + "futures-util", + "fxhash", + "h2", + "http", + "httparse", + "indexmap", + "itoa", + "language-tags", + "lazy_static", + "log", + "mime", + "percent-encoding 2.1.0", + "pin-project 1.0.7", + "rand 0.7.3", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "sha-1", + "slab", + "time 0.2.26", +] + +[[package]] +name = "actix-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-macros" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-router" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" +dependencies = [ + "bytestring", + "http", + "log", + "regex", + "serde", +] + +[[package]] +name = "actix-rt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" +dependencies = [ + "actix-macros 0.1.3", + "actix-threadpool", + "copyless", + "futures-channel", + "futures-util", + "smallvec", + "tokio 0.2.25", +] + +[[package]] +name = "actix-rt" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d7cd957c9ed92288a7c3c96af81fa5291f65247a76a34dac7b6af74e52ba0" +dependencies = [ + "actix-macros 0.2.0", + "futures-core", + "tokio 1.5.0", +] + +[[package]] +name = "actix-server" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45407e6e672ca24784baa667c5d32ef109ccdd8d5e0b5ebb9ef8a67f4dfb708e" +dependencies = [ + "actix-codec", + "actix-rt 1.1.1", + "actix-service", + "actix-utils 2.0.0", + "futures-channel", + "futures-util", + "log", + "mio 0.6.23", + "mio-uds", + "num_cpus", + "slab", + "socket2", +] + +[[package]] +name = "actix-service" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0052435d581b5be835d11f4eb3bce417c8af18d87ddf8ace99f8e67e595882bb" +dependencies = [ + "futures-util", + "pin-project 0.4.28", +] + +[[package]] +name = "actix-testing" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47239ca38799ab74ee6a8a94d1ce857014b2ac36f242f70f3f75a66f691e791c" +dependencies = [ + "actix-macros 0.1.3", + "actix-rt 1.1.1", + "actix-server", + "actix-service", + "log", + "socket2", +] + +[[package]] +name = "actix-threadpool" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d209f04d002854b9afd3743032a27b066158817965bf5d036824d19ac2cc0e30" +dependencies = [ + "derive_more", + "futures-channel", + "lazy_static", + "log", + "num_cpus", + "parking_lot", + "threadpool", +] + +[[package]] +name = "actix-tls" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24789b7d7361cf5503a504ebe1c10806896f61e96eca9a7350e23001aca715fb" +dependencies = [ + "actix-codec", + "actix-service", + "actix-utils 2.0.0", + "futures-util", +] + +[[package]] +name = "actix-utils" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" +dependencies = [ + "actix-codec", + "actix-rt 1.1.1", + "actix-service", + "bitflags", + "bytes 0.5.6", + "either", + "futures-channel", + "futures-sink", + "futures-util", + "log", + "pin-project 0.4.28", + "slab", +] + +[[package]] +name = "actix-utils" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e491cbaac2e7fc788dfff99ff48ef317e23b3cf63dbaf7aaab6418f40f92aa94" +dependencies = [ + "local-waker", + "pin-project-lite 0.2.6", +] + +[[package]] +name = "actix-web" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e641d4a172e7faa0862241a20ff4f1f5ab0ab7c279f00c2d4587b77483477b86" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros 0.1.3", + "actix-router", + "actix-rt 1.1.1", + "actix-server", + "actix-service", + "actix-testing", + "actix-threadpool", + "actix-tls", + "actix-utils 2.0.0", + "actix-web-codegen", + "awc", + "bytes 0.5.6", + "derive_more", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "fxhash", + "log", + "mime", + "pin-project 1.0.7", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "socket2", + "time 0.2.26", + "tinyvec", + "url 2.2.2", +] + +[[package]] +name = "actix-web-codegen" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad26f77093333e0e7c6ffe54ebe3582d908a104e448723eec6d43d08b07143fb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" + +[[package]] +name = "ahash" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" +dependencies = [ + "getrandom 0.2.2", + "once_cell", + "version_check 0.9.3", +] + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "async-trait" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atoi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "awc" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b381e490e7b0cfc37ebc54079b0413d8093ef43d14a4e4747083f7fa47a9e691" +dependencies = [ + "actix-codec", + "actix-http", + "actix-rt 1.1.1", + "actix-service", + "base64", + "bytes 0.5.6", + "cfg-if 1.0.0", + "derive_more", + "futures-core", + "log", + "mime", + "percent-encoding 2.1.0", + "rand 0.7.3", + "serde", + "serde_json", + "serde_urlencoded", +] + +[[package]] +name = "base-x" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bitvec" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8942c8d352ae1838c9dda0b0ca2ab657696ef2232a20147cf1b30ae1a9cb4321" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" +dependencies = [ + "brotli-sys", + "libc", +] + +[[package]] +name = "buf-min" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa17aa1cf56bdd6bb30518767d00e58019d326f3f05d8c3e0730b549d332ea83" +dependencies = [ + "bytes 0.5.6", +] + +[[package]] +name = "build_const" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" + +[[package]] +name = "bumpalo" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "bytestring" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" +dependencies = [ + "bytes 1.0.1", +] + +[[package]] +name = "cc" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time 0.1.44", + "winapi 0.3.9", +] + +[[package]] +name = "const_fn" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec" + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "cookie" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" +dependencies = [ + "percent-encoding 2.1.0", + "time 0.2.26", + "version_check 0.9.3", +] + +[[package]] +name = "copyless" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" + +[[package]] +name = "cpufeatures" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec1028182c380cc45a2e2c5ec841134f2dfd0f8f5f0a5bcd68004f81b5efdf4" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +dependencies = [ + "build_const", +] + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f6cb3c7f5b8e51bc3ebb73a2327ad4abdbd119dc13223f14f961d2f38486756" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "derive_more" +version = "0.99.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b1b72f1263f214c0f823371768776c4f5841b942c9883aa8e5ec584fd0ba6" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "diesel" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "047bfc4d5c3bd2ef6ca6f981941046113524b9a9f9a7cbdfdd7ff40f58e6f542" +dependencies = [ + "byteorder", + "diesel_derives", + "mysqlclient-sys", + "url 1.7.2", +] + +[[package]] +name = "diesel_derives" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + +[[package]] +name = "easyenv" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852054bee794f364fbe890bd947cbf678c35226b81ede17720d6a17fc687a0d4" +dependencies = [ + "dotenv", + "env_logger", + "log", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "enum-as-inner" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "flate2" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding 2.1.0", +] + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "funty" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" + +[[package]] +name = "futures" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" + +[[package]] +name = "futures-executor" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" + +[[package]] +name = "futures-macro" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" +dependencies = [ + "autocfg 1.0.1", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" + +[[package]] +name = "futures-task" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" + +[[package]] +name = "futures-util" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" +dependencies = [ + "autocfg 1.0.1", + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite 0.2.6", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check 0.9.3", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", +] + +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 0.2.25", + "tokio-util", + "tracing", + "tracing-futures", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash 0.4.7", +] + +[[package]] +name = "hashlink" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi 0.3.9", +] + +[[package]] +name = "http" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" +dependencies = [ + "bytes 1.0.1", + "fnv", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "idna" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" +dependencies = [ + "autocfg 1.0.1", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "ipconfig" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +dependencies = [ + "socket2", + "widestring", + "winapi 0.3.9", + "winreg", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "js-sys" +version = "0.3.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin", +] + +[[package]] +name = "lexical-core" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" +dependencies = [ + "arrayvec", + "bitflags", + "cfg-if 1.0.0", + "ryu", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" + +[[package]] +name = "libm" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + +[[package]] +name = "local-waker" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f9a2d3e27ce99ce2c3aad0b09b1a7b916293ea9b2bf624c13fe646fadd8da4" + +[[package]] +name = "lock_api" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "memchr" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg 1.0.1", +] + +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.2", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956" +dependencies = [ + "libc", + "log", + "miow 0.3.7", + "ntapi", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio 0.6.23", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "mysqlclient-sys" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9637d93448044078aaafea7419aed69d301b4a12bcc4aa0ae856eb169bef85" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "net2" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nom" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +dependencies = [ + "memchr", + "version_check 0.1.5", +] + +[[package]] +name = "nom" +version = "6.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7413f999671bd4745a7b624bd370a569fb6bc574b23c83a3c5ed2e453f3d5e2" +dependencies = [ + "bitvec", + "funty", + "lexical-core", + "memchr", + "version_check 0.9.3", +] + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "num-bigint" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d0a3d5e207573f948a9e5376662aa743a2ea13f7c50a554d7af443a73fbfeba" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e0d047c1062aa51e256408c560894e5251f08925980e53cf1aa5bd00eec6512" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4547ee5541c18742396ae2c895d0717d0f886d8823b8399cdaf7b07d63ad0480" +dependencies = [ + "autocfg 0.1.7", + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.3", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg 1.0.1", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg 1.0.1", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "pem" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb" +dependencies = [ + "base64", + "once_cell", + "regex", +] + +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918192b5c59119d51e0cd221f4d49dde9112824ba717369e903c97d076083d0f" +dependencies = [ + "pin-project-internal 0.4.28", +] + +[[package]] +name = "pin-project" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" +dependencies = [ + "pin-project-internal 1.0.7", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be26700300be6d9d23264c73211d8190e755b6b5ca7a1b28230025511b52a5e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" + +[[package]] +name = "pin-project-lite" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + +[[package]] +name = "proc-macro2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + +[[package]] +name = "rsa" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ef841a26fc5d040ced0417c6c6a64ee851f42489df11cdf0218e545b6f8d28" +dependencies = [ + "byteorder", + "digest", + "lazy_static", + "num-bigint-dig", + "num-integer", + "num-iter", + "num-traits", + "pem", + "rand 0.8.3", + "simple_asn1", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha-1" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpufeatures", + "digest", + "opaque-debug", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "sha2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpufeatures", + "digest", + "opaque-debug", +] + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "simple_asn1" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e0e9076e5242ff5a58e854cb478ea9caebce01088f86d3d9c6ad336b7655263" +dependencies = [ + "chrono", + "num-bigint 0.4.0", + "num-traits", + "thiserror", +] + +[[package]] +name = "slab" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "sqlformat" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d86e3c77ff882a828346ba401a7ef4b8e440df804491c6064fe8295765de71c" +dependencies = [ + "lazy_static", + "maplit", + "nom 6.1.2", + "regex", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d582b9bc04ec6c03084196efc42c2226b018e9941f03ee62bd88921d500917c0" +dependencies = [ + "sqlx-core", + "sqlx-macros", +] + +[[package]] +name = "sqlx-core" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de52d1d473cebb2abb79c886ef6a8023e965e34c0676a99cfeac2cc7f0fde4c1" +dependencies = [ + "ahash 0.7.2", + "atoi", + "base64", + "bitflags", + "byteorder", + "bytes 1.0.1", + "crc", + "crossbeam-channel", + "crossbeam-queue", + "crossbeam-utils", + "digest", + "either", + "futures-channel", + "futures-core", + "futures-util", + "generic-array", + "hashlink", + "hex", + "itoa", + "libc", + "log", + "memchr", + "num-bigint 0.3.2", + "once_cell", + "parking_lot", + "percent-encoding 2.1.0", + "rand 0.8.3", + "rsa", + "rustls", + "sha-1", + "sha2", + "smallvec", + "sqlformat", + "sqlx-rt", + "stringprep", + "thiserror", + "tokio-stream", + "url 2.2.2", + "webpki", + "webpki-roots", + "whoami", +] + +[[package]] +name = "sqlx-macros" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a40f0be97e704d3fbf059e7e3333c3735639146a72d586c5534c70e79da88a4" +dependencies = [ + "dotenv", + "either", + "futures", + "heck", + "proc-macro2", + "quote", + "sha2", + "sqlx-core", + "sqlx-rt", + "syn", + "url 2.2.2", +] + +[[package]] +name = "sqlx-rt" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ae97ab05063ed515cdc23d90253213aa24dda0a288c5ec079af3d10f9771bc" +dependencies = [ + "once_cell", + "tokio 1.5.0", + "tokio-rustls", +] + +[[package]] +name = "standback" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" +dependencies = [ + "version_check 0.9.3", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "storyteller-web" +version = "0.0.1" +dependencies = [ + "actix-cors", + "actix-files", + "actix-http", + "actix-rt 2.2.0", + "actix-utils 3.0.0", + "actix-web", + "anyhow", + "base64", + "chrono", + "diesel", + "easyenv", + "futures", + "hostname", + "log", + "rand 0.8.3", + "serde", + "serde_derive", + "serde_json", + "sqlx", + "toml", +] + +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "subtle" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" + +[[package]] +name = "syn" +version = "1.0.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", +] + +[[package]] +name = "time" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08a8cbfbf47955132d0202d1662f49b2423ae35862aee471f3ba4b133358f372" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check 0.9.3", + "winapi 0.3.9", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio 0.6.23", + "mio-uds", + "pin-project-lite 0.1.12", + "signal-hook-registry", + "slab", + "winapi 0.3.9", +] + +[[package]] +name = "tokio" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" +dependencies = [ + "autocfg 1.0.1", + "bytes 1.0.1", + "libc", + "memchr", + "mio 0.7.11", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite 0.2.6", + "signal-hook-registry", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio 1.5.0", + "webpki", +] + +[[package]] +name = "tokio-stream" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e177a5d8c3bf36de9ebe6d58537d8879e964332f93fb3339e43f618c81361af0" +dependencies = [ + "futures-core", + "pin-project-lite 0.2.6", + "tokio 1.5.0", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.1.12", + "tokio 0.2.25", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + +[[package]] +name = "tracing" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" +dependencies = [ + "cfg-if 1.0.0", + "log", + "pin-project-lite 0.2.6", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project 1.0.7", + "tracing", +] + +[[package]] +name = "trust-dns-proto" +version = "0.19.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cad71a0c0d68ab9941d2fb6e82f8fb2e86d9945b94e1661dd0aaea2b88215a9" +dependencies = [ + "async-trait", + "cfg-if 1.0.0", + "enum-as-inner", + "futures", + "idna 0.2.3", + "lazy_static", + "log", + "rand 0.7.3", + "smallvec", + "thiserror", + "tokio 0.2.25", + "url 2.2.2", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.19.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710f593b371175db53a26d0b38ed2978fafb9e9e8d3868b1acd753ea18df0ceb" +dependencies = [ + "cfg-if 0.1.10", + "futures", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "resolv-conf", + "smallvec", + "thiserror", + "tokio 0.2.25", + "trust-dns-proto", +] + +[[package]] +name = "typenum" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check 0.9.3", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" +dependencies = [ + "idna 0.1.5", + "matches", + "percent-encoding 1.0.1", +] + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna 0.2.3", + "matches", + "percent-encoding 2.1.0", +] + +[[package]] +name = "v_escape" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3e0ab5fab1db278a9413d2ea794cb66f471f898c5b020c3c394f6447625d9d4" +dependencies = [ + "buf-min", + "v_escape_derive", +] + +[[package]] +name = "v_escape_derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c860ad1273f4eee7006cee05db20c9e60e5d24cba024a32e1094aa8e574f3668" +dependencies = [ + "nom 4.2.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "v_htmlescape" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f9a8af610ad6f7fc9989c9d2590d9764bc61f294884e9ee93baa58795174572" +dependencies = [ + "cfg-if 1.0.0", + "v_escape", +] + +[[package]] +name = "vcpkg" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" + +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f" + +[[package]] +name = "web-sys" +version = "0.3.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" +dependencies = [ + "webpki", +] + +[[package]] +name = "whoami" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4abacf325c958dfeaf1046931d37f2a901b6dfe0968ee965a29e94c6766b2af6" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "widestring" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" + +[[package]] +name = "zeroize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c1e130bebaeab2f23886bf9acbaca14b092408c452543c857f66399cd6dab1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000..02095871e7 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "storyteller-web" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[[bin]] +name = "storyteller-web" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.40" +base64 = "0.13.0" +chrono = { version = "0.4.19", features = ["serde"] } +easyenv = "0.0.5" +futures = "0.3.15" +hostname = "0.3.1" +log = "0.4.14" +rand = "0.8.3" + +# Database +sqlx = { version = "0.5.0", features = ["mysql", "runtime-tokio-rustls"] } +#diesel = { version = "1.4.4", features = ["mysql", "r2d2", "chrono"] } +#diesel-derive-enum = { version = "1.0.0", features = ["mysql"] } + +# Web +actix-cors = "0.5.4" +actix-files= "0.5" +actix-http = "2.2.0" +actix-rt = "2.2.0" +actix-utils = "3.0.0" +actix-web = "3.3.2" + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" +toml = "0.5.8" + +[build-dependencies] +# NB: Only using diesel for migrations support +diesel = { version = "1.4.6", features = ["mysql"] } diff --git a/README.md b/README.md new file mode 100644 index 0000000000..bb31e4acee --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +storyteller-web +=============== +This is our main user account monolith that we'll also bake other pieces into. + diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000000..b0d48774b4 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,3 @@ + +pub fn main() { +} \ No newline at end of file From 9ee55b6a484ff95c0684874284a8de5fc487a496 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 12 May 2021 11:37:03 -0400 Subject: [PATCH 0002/1042] gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..aee3b3b46c --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*~ +.env +.idea/ +secrets.toml +secrets.yaml +target/ From 2cd8f2875762abcdbe46f6e3ac58657ac6daeab3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 14 May 2021 18:12:31 -0400 Subject: [PATCH 0003/1042] Actix setup, ServerState arc --- src/main.rs | 122 +++++++++++++++++++++++++++++++++++++++++++- src/server_state.rs | 15 ++++++ 2 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 src/server_state.rs diff --git a/src/main.rs b/src/main.rs index b0d48774b4..9cfb3c23a0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,121 @@ +pub mod server_state; -pub fn main() { -} \ No newline at end of file +use actix_cors::Cors; +use actix_http::http; +use log::{info}; +use actix_web::middleware::{Logger, DefaultHeaders}; +use actix_web::{HttpServer, web, HttpResponse, App}; +use std::sync::Arc; +use crate::server_state::{ServerState, EnvConfig}; + +const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; + +pub type AnyhowResult = anyhow::Result; + +pub fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + let server_hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or("storyteller-web-unknown".to_string()); + + let server_state = ServerState { + env_config: EnvConfig { + num_workers: 4, + bind_address: DEFAULT_BIND_ADDRESS.to_string(), + }, + hostname: server_hostname, + }; + + serve(server_state)?; + Ok(()) +} + +#[actix_web::main] +pub async fn serve(server_state: ServerState) -> std::io::Result<()> +{ + let bind_address = server_state.env_config.bind_address.clone(); + let num_workers = server_state.env_config.num_workers.clone(); + let hostname = server_state.hostname.clone(); + + let server_state_arc = web::Data::new(Arc::new(server_state)); + + info!("Starting HTTP service."); + + let log_format = "[%{HOSTNAME}e] %a \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T"; + + HttpServer::new(move || { + let mut app = App::new() + .wrap(Cors::default() + .allowed_origin("http://api.vo.codes") + .allowed_origin("http://jungle.horse") + .allowed_origin("http://localhost:12345") + .allowed_origin("http://localhost:3000") + .allowed_origin("http://localhost:5555") + .allowed_origin("http://localhost:7000") + .allowed_origin("http://localhost:8000") + .allowed_origin("http://localhost:8080") + .allowed_origin("http://mumble.stream") + .allowed_origin("http://trumped.com") + .allowed_origin("http://vo.codes") + .allowed_origin("http://vocodes.com") + .allowed_origin("https://jungle.horse") + .allowed_origin("https://mumble.stream") + .allowed_origin("https://trumped.com") + .allowed_origin("https://vo.codes") + .allowed_origin("https://vocodes.com") + .allowed_methods(vec!["GET", "POST", "OPTIONS"]) + .allowed_headers(vec![ + http::header::ACCEPT, + http::header::ACCESS_CONTROL_ALLOW_ORIGIN, // Tabulator Ajax + http::header::CONTENT_TYPE, + http::header::HeaderName::from_static("x-requested-with") // Tabulator Ajax sends + ]) + .max_age(3600)) + .wrap(Logger::new(&log_format) + .exclude("/liveness") + .exclude("/readiness")) + .wrap(DefaultHeaders::new().header("X-Backend-Hostname", &hostname)) + .service( + web::resource("/test") + .route(web::get().to(|| HttpResponse::Ok())) + .route(web::head().to(|| HttpResponse::Ok())) + ); + // Admin UI & old frontend + //.service(Files::new("/frontend", admin_asset_directory.clone()) + // .index_file("FAKE_INDEX.HTML")) + //.service(Files::new("/adminui", admin_asset_directory.clone()) + // .index_file("index.html")) + // Early access static path for assets + //.service(Files::new("/static", static_asset_directory.clone()) + // .index_file("FAKE_INDEX.HTML")) + /*.service( + web::resource("/login") + .route(web::post().to(login_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/logout") + .route(web::post().to(logout_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(get_liveness) + .service(get_models) + .service(get_readiness) + .service(get_root) + .service(get_sentences) + .service(get_service_settings) + .service(get_speakers) + .service(get_early_access_speakers) + .service(get_dynamic_early_access_speakers) + .service(get_words);*/ + + app.app_data(server_state_arc.clone()) + }) + .bind(bind_address)? + .workers(num_workers) + .run() + .await +} diff --git a/src/server_state.rs b/src/server_state.rs new file mode 100644 index 0000000000..9210cb51c3 --- /dev/null +++ b/src/server_state.rs @@ -0,0 +1,15 @@ + +/// State that is injected into every endpoint. +#[derive(Clone)] +pub struct ServerState { + pub env_config: EnvConfig, + + pub hostname: String, +} + +#[derive(Clone)] +pub struct EnvConfig { + // Number of thread workers. + pub num_workers: usize, + pub bind_address: String, +} From 322895c0501cd6f40acfe773da8f9d5b52825d00 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 03:45:23 -0400 Subject: [PATCH 0004/1042] Migrations WIP --- migrations/2021-05-14-221408_initial/down.sql | 1 + migrations/2021-05-14-221408_initial/up.sql | 19 +++ migrations/2021-05-15-071534_users/down.sql | 6 + migrations/2021-05-15-071534_users/up.sql | 112 ++++++++++++++++++ migrations/2021-05-15-071545_jobs/down.sql | 5 + migrations/2021-05-15-071545_jobs/up.sql | 83 +++++++++++++ migrations/2021-05-15-071601_assets/down.sql | 2 + migrations/2021-05-15-071601_assets/up.sql | 60 ++++++++++ migrations/2021-05-15-071612_results/down.sql | 3 + migrations/2021-05-15-071612_results/up.sql | 3 + migrations/2021-05-15-071616_misc/down.sql | 5 + migrations/2021-05-15-071616_misc/up.sql | 30 +++++ 12 files changed, 329 insertions(+) create mode 100644 migrations/2021-05-14-221408_initial/down.sql create mode 100644 migrations/2021-05-14-221408_initial/up.sql create mode 100644 migrations/2021-05-15-071534_users/down.sql create mode 100644 migrations/2021-05-15-071534_users/up.sql create mode 100644 migrations/2021-05-15-071545_jobs/down.sql create mode 100644 migrations/2021-05-15-071545_jobs/up.sql create mode 100644 migrations/2021-05-15-071601_assets/down.sql create mode 100644 migrations/2021-05-15-071601_assets/up.sql create mode 100644 migrations/2021-05-15-071612_results/down.sql create mode 100644 migrations/2021-05-15-071612_results/up.sql create mode 100644 migrations/2021-05-15-071616_misc/down.sql create mode 100644 migrations/2021-05-15-071616_misc/up.sql diff --git a/migrations/2021-05-14-221408_initial/down.sql b/migrations/2021-05-14-221408_initial/down.sql new file mode 100644 index 0000000000..291a97c5ce --- /dev/null +++ b/migrations/2021-05-14-221408_initial/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` \ No newline at end of file diff --git a/migrations/2021-05-14-221408_initial/up.sql b/migrations/2021-05-14-221408_initial/up.sql new file mode 100644 index 0000000000..a26758e3f7 --- /dev/null +++ b/migrations/2021-05-14-221408_initial/up.sql @@ -0,0 +1,19 @@ +-- noinspection SqlNoDataSourceInspectionForFile + +-- ============================== USERS ============================== -- + + + +-- ============================== UPLOADS ============================== -- + +-- ============================== MODELS ============================== -- + +-- ============================== INFERENCE ============================== -- + +-- ============================== WAV2LIP TEMPLATES ============================== -- + + +-- ============================== RENDERED SENTENCES ============================== -- + +-- ============================== LOYALTY ============================== -- + diff --git a/migrations/2021-05-15-071534_users/down.sql b/migrations/2021-05-15-071534_users/down.sql new file mode 100644 index 0000000000..5b1ba4f03c --- /dev/null +++ b/migrations/2021-05-15-071534_users/down.sql @@ -0,0 +1,6 @@ +# noinspection SqlNoDataSourceInspectionForFile +# noinspection SqlResolveForFile + +drop table users; +drop table user_roles; +drop table user_sessions; diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql new file mode 100644 index 0000000000..12f01a4269 --- /dev/null +++ b/migrations/2021-05-15-071534_users/up.sql @@ -0,0 +1,112 @@ +# noinspection SqlNoDataSourceInspectionForFile +# noinspection SqlResolveForFile + +CREATE TABLE users ( + -- Not used for anything except replication. + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + -- Effective "primary key" + token CHAR(16) NOT NULL UNIQUE, + + -- Username is a lookup key; display_name allows the user to add custom case. + username VARCHAR(20) NOT NULL UNIQUE, + display_name VARCHAR(20) NOT NULL, + + email_address VARCHAR(255) NOT NULL UNIQUE, + + -- The role assigned to the user confers permissions. + user_role_slug VARCHAR(16) NOT NULL, + + -- Bcrypt password hash. Granted, there are newer methods: + -- https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html + -- Passwords may be a max of 64 characters per BCrypt. + password_hash BINARY(60) NOT NULL, + + -- Incremented with every update to the password. + password_version INT NOT NULL DEFAULT 0, + + -- For abuse tracking. + ip_address_creation VARCHAR(20) NOT NULL, + ip_address_last_login VARCHAR(20) NOT NULL, + + -- For tracking stats. + -- The "cached" values are updated by a background job. + cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0, + cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0, + + -- Settings + -- DO NOT REORDER. + dark_mode ENUM( + 'light-mode', + 'dark-mode', + 'use-clock' + ) NOT NULL DEFAULT 'light-mode', + + -- An uploaded avatar. Public hash in our bucket. + avatar_public_bucket_hash CHAR(32) NOT NULL UNIQUE, + + -- If the user doesn't want to use gravatar and doesn't have an uploaded avatar. + disable_gravatar BOOLEAN NOT NULL DEFAULT false, + + -- Hide results from others (ie. won't show up in the wall) + -- Moderators will still see them. + -- If the URLs are shared, they'll be visible. + hide_results BOOLEAN NOT NULL DEFAULT false, + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If the user is deleted (banned, etc.), we set this. + deleted_at TIMESTAMP DEFAULT NULL + +) ENGINE=INNODB; + +CREATE TABLE user_roles ( + -- Not used for anything except replication. + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + -- Effective "primary key" + slug CHAR(16) NOT NULL UNIQUE, + + name VARCHAR(255) NOT NULL, + + can_ban_users BOOLEAN NOT NULL DEFAULT FALSE, + can_edit_other_users_data BOOLEAN NOT NULL DEFAULT FALSE, + + can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE, + can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, + + can_use_tts BOOLEAN NOT NULL DEFAULT FALSE, + can_use_w2l BOOLEAN NOT NULL DEFAULT FALSE, + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + +) ENGINE=INNODB; + +CREATE TABLE user_sessions ( + -- Not used for anything except replication. + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + -- Session entropy + token CHAR(32) NOT NULL, + + -- Foreign key to user + user_token CHAR(16) NOT NULL, + + -- Track each session's creation IP. + ip_address_creation VARCHAR(20) NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- deletion = session termination + deleted_at TIMESTAMP DEFAULT NULL + +) ENGINE=INNODB; diff --git a/migrations/2021-05-15-071545_jobs/down.sql b/migrations/2021-05-15-071545_jobs/down.sql new file mode 100644 index 0000000000..117aee63d0 --- /dev/null +++ b/migrations/2021-05-15-071545_jobs/down.sql @@ -0,0 +1,5 @@ +# noinspection SqlNoDataSourceInspectionForFile +# noinspection SqlResolveForFile + +drop table tts_model_upload_jobs; +drop table w2l_template_upload_jobs; diff --git a/migrations/2021-05-15-071545_jobs/up.sql b/migrations/2021-05-15-071545_jobs/up.sql new file mode 100644 index 0000000000..0588d4cb72 --- /dev/null +++ b/migrations/2021-05-15-071545_jobs/up.sql @@ -0,0 +1,83 @@ +# noinspection SqlNoDataSourceInspectionForFile +# noinspection SqlResolveForFile + +-- TTS MODELS +CREATE TABLE tts_model_upload_jobs ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + -- Foreign key to user + creator_user_token CHAR(16) NOT NULL, + + -- Jobs begin as "pending", then transition to other states. + -- Pending -> Started -> Complete + -- \-> Failed -> Started -> { Complete, Failed, Dead } + status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending', + + -- We can track this against a "max_attempt_count" + attempt_count INT(3) NOT NULL DEFAULT 0, + + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + tts_model_type ENUM( + 'not-set', + 'tacotron2', + 'glowtts', + 'glowtts-vocodes' + ) NOT NULL DEFAULT 'not-set', + + -- If we need to download the file from Google Drive. + google_drive_url VARCHAR(512) DEFAULT NULL, + + -- If there is a failure, tell the user why. + failure_reason VARCHAR(512) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- Failed jobs will set a next attempt time. + -- Subsequent tries can increase the timeout. + -- Failures because of permissions require human intervention => [retry_at=null]. + -- Failures because of invalid files are dead => [status=dead]. + retry_at TIMESTAMP DEFAULT NULL + +) ENGINE=INNODB; + +-- WAV2LIP TEMPLATES +CREATE TABLE w2l_template_upload_jobs ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + -- Foreign key to user + creator_user_token CHAR(16) NOT NULL, + + -- Jobs begin as "pending", then transition to other states. + -- Pending -> Started -> Complete + -- \-> Failed -> Started -> { Complete, Failed, Dead } + status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending', + + -- We can track this against a "max_attempt_count" + attempt_count INT(3) NOT NULL DEFAULT 0, + + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + template_type ENUM( + 'video', + 'image' + ) NOT NULL, + + -- If we need to download the file from Google Drive. + google_drive_url VARCHAR(512) DEFAULT NULL, + + -- If there is a failure, tell the user why. + failure_reason VARCHAR(512) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- Failed jobs will set a next attempt time. + -- Subsequent tries can increase the timeout. + -- Failures because of permissions require human intervention => [retry_at=null]. + -- Failures because of invalid files are dead => [status=dead]. + retry_at TIMESTAMP DEFAULT NULL + +) ENGINE=INNODB; + diff --git a/migrations/2021-05-15-071601_assets/down.sql b/migrations/2021-05-15-071601_assets/down.sql new file mode 100644 index 0000000000..290e861cb1 --- /dev/null +++ b/migrations/2021-05-15-071601_assets/down.sql @@ -0,0 +1,2 @@ +# noinspection SqlNoDataSourceInspectionForFile +# noinspection SqlResolveForFile diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql new file mode 100644 index 0000000000..d93259f562 --- /dev/null +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -0,0 +1,60 @@ +# noinspection SqlNoDataSourceInspectionForFile +# noinspection SqlResolveForFile + +CREATE TABLE w2l_templates ( + -- Not used for anything except replication. + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + -- Effective "primary key" (PUBLIC) + token CHAR(16) NOT NULL UNIQUE, + + template_type ENUM( + 'image', + 'video' + ) NOT NULL, + + -- The title of the template. + title CHAR(255) NOT NULL UNIQUE, + + -- Users can upload their own private templates. + -- They can choose to make them public later. + is_private BOOLEAN NOT NULL DEFAULT FALSE, + + -- The description of the template. + description CHAR(512) NOT NULL UNIQUE, + + -- The person that created the template. + creator_user_token CHAR(16) NOT NULL, + + -- For abuse tracking. + creator_ip_address CHAR(16) NOT NULL, + + -- The filename that was used at upload time. + original_filename CHAR(255) NOT NULL UNIQUE, + + -- The original source image/video + private_bucket_hash_original CHAR(32) NOT NULL UNIQUE, + + -- The "precomputed" faces. + private_bucket_hash_precomputed CHAR(32) NOT NULL UNIQUE, + + -- For the thumbnail we show. + public_bucket_hash CHAR(32) NOT NULL UNIQUE, + + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + tts_model_type ENUM( + 'not-set', + 'tacotron2', + 'glowtts', + 'glowtts-vocodes' + ) NOT NULL DEFAULT 'not-set', + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed. + deleted_at TIMESTAMP DEFAULT NULL + +) ENGINE=INNODB; + diff --git a/migrations/2021-05-15-071612_results/down.sql b/migrations/2021-05-15-071612_results/down.sql new file mode 100644 index 0000000000..c08437d48b --- /dev/null +++ b/migrations/2021-05-15-071612_results/down.sql @@ -0,0 +1,3 @@ +# noinspection SqlNoDataSourceInspectionForFile +# noinspection SqlResolveForFile + diff --git a/migrations/2021-05-15-071612_results/up.sql b/migrations/2021-05-15-071612_results/up.sql new file mode 100644 index 0000000000..c08437d48b --- /dev/null +++ b/migrations/2021-05-15-071612_results/up.sql @@ -0,0 +1,3 @@ +# noinspection SqlNoDataSourceInspectionForFile +# noinspection SqlResolveForFile + diff --git a/migrations/2021-05-15-071616_misc/down.sql b/migrations/2021-05-15-071616_misc/down.sql new file mode 100644 index 0000000000..23db61a467 --- /dev/null +++ b/migrations/2021-05-15-071616_misc/down.sql @@ -0,0 +1,5 @@ +# noinspection SqlResolveForFile +# noinspection SqlNoDataSourceInspectionForFile + +drop table badges; +drop table user_badges; diff --git a/migrations/2021-05-15-071616_misc/up.sql b/migrations/2021-05-15-071616_misc/up.sql new file mode 100644 index 0000000000..f4faaafe42 --- /dev/null +++ b/migrations/2021-05-15-071616_misc/up.sql @@ -0,0 +1,30 @@ +# noinspection SqlNoDataSourceInspectionForFile +# noinspection SqlResolveForFile + +-- Badges are rewards for loyalty +CREATE TABLE badges ( + -- Not used for anything except replication. + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + -- Effective "primary key" + token CHAR(16) NOT NULL UNIQUE, + + -- Effective "primary key" + slug CHAR(16) NOT NULL UNIQUE, + + -- Description + title VARCHAR(255) NOT NULL, + description VARCHAR(512) NOT NULL, + image_url VARCHAR(512) NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + +) ENGINE=INNODB; + +-- Join table +CREATE TABLE user_badges ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + user_token CHAR(16) NOT NULL UNIQUE, + badge_token CHAR(16) NOT NULL UNIQUE +) ENGINE=INNODB; From b2cf2dc3ebfae2a6ce7af0403d67b788f4592fe3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 03:50:27 -0400 Subject: [PATCH 0005/1042] model asset tables --- migrations/.gitkeep | 0 migrations/2021-05-14-221408_initial/down.sql | 1 - migrations/2021-05-14-221408_initial/up.sql | 19 ----- migrations/2021-05-15-071601_assets/down.sql | 3 + migrations/2021-05-15-071601_assets/up.sql | 71 +++++++++++++++---- 5 files changed, 60 insertions(+), 34 deletions(-) create mode 100644 migrations/.gitkeep delete mode 100644 migrations/2021-05-14-221408_initial/down.sql delete mode 100644 migrations/2021-05-14-221408_initial/up.sql diff --git a/migrations/.gitkeep b/migrations/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/migrations/2021-05-14-221408_initial/down.sql b/migrations/2021-05-14-221408_initial/down.sql deleted file mode 100644 index 291a97c5ce..0000000000 --- a/migrations/2021-05-14-221408_initial/down.sql +++ /dev/null @@ -1 +0,0 @@ --- This file should undo anything in `up.sql` \ No newline at end of file diff --git a/migrations/2021-05-14-221408_initial/up.sql b/migrations/2021-05-14-221408_initial/up.sql deleted file mode 100644 index a26758e3f7..0000000000 --- a/migrations/2021-05-14-221408_initial/up.sql +++ /dev/null @@ -1,19 +0,0 @@ --- noinspection SqlNoDataSourceInspectionForFile - --- ============================== USERS ============================== -- - - - --- ============================== UPLOADS ============================== -- - --- ============================== MODELS ============================== -- - --- ============================== INFERENCE ============================== -- - --- ============================== WAV2LIP TEMPLATES ============================== -- - - --- ============================== RENDERED SENTENCES ============================== -- - --- ============================== LOYALTY ============================== -- - diff --git a/migrations/2021-05-15-071601_assets/down.sql b/migrations/2021-05-15-071601_assets/down.sql index 290e861cb1..8a9a8d2f98 100644 --- a/migrations/2021-05-15-071601_assets/down.sql +++ b/migrations/2021-05-15-071601_assets/down.sql @@ -1,2 +1,5 @@ # noinspection SqlNoDataSourceInspectionForFile # noinspection SqlResolveForFile + +drop table tts_models; +drop table w2l_templates; diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index d93259f562..31fee16e9c 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -1,24 +1,28 @@ # noinspection SqlNoDataSourceInspectionForFile # noinspection SqlResolveForFile -CREATE TABLE w2l_templates ( +CREATE TABLE tts_models ( -- Not used for anything except replication. id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, -- Effective "primary key" (PUBLIC) token CHAR(16) NOT NULL UNIQUE, - template_type ENUM( - 'image', - 'video' - ) NOT NULL, + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + tts_model_type ENUM( + 'not-set', + 'tacotron2', + 'glowtts', + 'glowtts-vocodes' + ) NOT NULL DEFAULT 'not-set', -- The title of the template. title CHAR(255) NOT NULL UNIQUE, -- Users can upload their own private templates. -- They can choose to make them public later. - is_private BOOLEAN NOT NULL DEFAULT FALSE, + is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, -- The description of the template. description CHAR(512) NOT NULL UNIQUE, @@ -41,14 +45,53 @@ CREATE TABLE w2l_templates ( -- For the thumbnail we show. public_bucket_hash CHAR(32) NOT NULL UNIQUE, - -- NB: DO NOT SORT! - -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. - tts_model_type ENUM( - 'not-set', - 'tacotron2', - 'glowtts', - 'glowtts-vocodes' - ) NOT NULL DEFAULT 'not-set', + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed. + deleted_at TIMESTAMP DEFAULT NULL + +) ENGINE=INNODB; + +CREATE TABLE w2l_templates ( + -- Not used for anything except replication. + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + -- Effective "primary key" (PUBLIC) + token CHAR(16) NOT NULL UNIQUE, + + template_type ENUM( + 'image', + 'video' + ) NOT NULL, + + -- The title of the template. + title CHAR(255) NOT NULL UNIQUE, + + -- Users can upload their own private templates. + -- They can choose to make them public later. + is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + + -- The description of the template. + description CHAR(512) NOT NULL UNIQUE, + + -- The person that created the template. + creator_user_token CHAR(16) NOT NULL, + + -- For abuse tracking. + creator_ip_address CHAR(16) NOT NULL, + + -- The filename that was used at upload time. + original_filename CHAR(255) NOT NULL UNIQUE, + + -- The original source image/video + private_bucket_hash_original CHAR(32) NOT NULL UNIQUE, + + -- The "precomputed" faces. + private_bucket_hash_precomputed CHAR(32) NOT NULL UNIQUE, + + -- For the thumbnail we show. + public_bucket_hash CHAR(32) NOT NULL UNIQUE, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, From 5a94e18779e907f3481df63131f76e26c924dbf1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 04:29:07 -0400 Subject: [PATCH 0006/1042] more schema work --- migrations/2021-05-15-071534_users/up.sql | 32 ++++++- migrations/2021-05-15-071545_jobs/up.sql | 16 +++- migrations/2021-05-15-071601_assets/up.sql | 101 +++++++++++---------- 3 files changed, 96 insertions(+), 53 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 12f01a4269..4c782879ba 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -8,12 +8,13 @@ CREATE TABLE users ( -- Effective "primary key" token CHAR(16) NOT NULL UNIQUE, + email_address VARCHAR(255) NOT NULL UNIQUE, + email_confirmed BOOLEAN NOT NULL DEFAULT false, + -- Username is a lookup key; display_name allows the user to add custom case. username VARCHAR(20) NOT NULL UNIQUE, display_name VARCHAR(20) NOT NULL, - email_address VARCHAR(255) NOT NULL UNIQUE, - -- The role assigned to the user confers permissions. user_role_slug VARCHAR(16) NOT NULL, @@ -110,3 +111,30 @@ CREATE TABLE user_sessions ( deleted_at TIMESTAMP DEFAULT NULL ) ENGINE=INNODB; + +-- We only allow the most recent record for any given user to be redeemed. +CREATE TABLE email_verifications ( + -- Not used for anything except replication. + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + verification_type ENUM( + 'email-verification', + 'password-reset' + ) NOT NULL, + + -- The redemption secret + verification_code VARCHAR(32) NOT NULL, + + -- Whether the attempt worked + successful BOOLEAN NOT NULL DEFAULT false, + + -- Foreign key to user + user_token CHAR(16) NOT NULL, + + -- Cannot be redeemed after this date + expires_at TIMESTAMP NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + +) ENGINE=INNODB; diff --git a/migrations/2021-05-15-071545_jobs/up.sql b/migrations/2021-05-15-071545_jobs/up.sql index 0588d4cb72..591cf6a83a 100644 --- a/migrations/2021-05-15-071545_jobs/up.sql +++ b/migrations/2021-05-15-071545_jobs/up.sql @@ -8,6 +8,16 @@ CREATE TABLE tts_model_upload_jobs ( -- Foreign key to user creator_user_token CHAR(16) NOT NULL, + -- Users can upload their own private models. + -- They can choose to make them public later. + is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + + -- The name of the voice + voice_name VARCHAR(255) NOT NULL, + + -- The speaker (in the case of cartoon characters) + voice_actor_name VARCHAR(255) DEFAULT NULL, + -- Jobs begin as "pending", then transition to other states. -- Pending -> Started -> Complete -- \-> Failed -> Started -> { Complete, Failed, Dead } @@ -16,7 +26,7 @@ CREATE TABLE tts_model_upload_jobs ( -- We can track this against a "max_attempt_count" attempt_count INT(3) NOT NULL DEFAULT 0, - -- NB: DO NOT SORT! + -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. tts_model_type ENUM( 'not-set', @@ -49,6 +59,10 @@ CREATE TABLE w2l_template_upload_jobs ( -- Foreign key to user creator_user_token CHAR(16) NOT NULL, + -- Users can upload their own private templates. + -- They can choose to make them public later. + is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + -- Jobs begin as "pending", then transition to other states. -- Pending -> Started -> Complete -- \-> Failed -> Started -> { Complete, Failed, Dead } diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 31fee16e9c..7f0825c39c 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -8,7 +8,7 @@ CREATE TABLE tts_models ( -- Effective "primary key" (PUBLIC) token CHAR(16) NOT NULL UNIQUE, - -- NB: DO NOT SORT! + -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. tts_model_type ENUM( 'not-set', @@ -17,6 +17,56 @@ CREATE TABLE tts_models ( 'glowtts-vocodes' ) NOT NULL DEFAULT 'not-set', + -- The name of the voice + voice_name VARCHAR(255) NOT NULL, + + -- The speaker (in the case of cartoon characters) + voice_actor_name VARCHAR(255) DEFAULT NULL, + + -- Users can upload their own private models. + -- They can choose to make them public later. + is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + + -- The description of the template. + description CHAR(512) NOT NULL UNIQUE, + + -- The person that created the template. + creator_user_token CHAR(16) NOT NULL, + + -- For abuse tracking. + creator_ip_address CHAR(16) NOT NULL, + + -- The filename that was used at upload time. + original_filename CHAR(255) NOT NULL UNIQUE, + + -- The pytorch model + -- For now, this will be a hash of the file contents. + private_bucket_hash CHAR(32) NOT NULL UNIQUE, + + -- For the thumbnail we show. + public_bucket_hash CHAR(32) NOT NULL UNIQUE, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed. + deleted_at TIMESTAMP DEFAULT NULL + +) ENGINE=INNODB; + +CREATE TABLE w2l_templates ( + -- Not used for anything except replication. + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + -- Effective "primary key" (PUBLIC) + token CHAR(16) NOT NULL UNIQUE, + + template_type ENUM( + 'not-set', + 'image', + 'video' + ) NOT NULL DEFAULT 'not-set', + -- The title of the template. title CHAR(255) NOT NULL UNIQUE, @@ -52,52 +102,3 @@ CREATE TABLE tts_models ( deleted_at TIMESTAMP DEFAULT NULL ) ENGINE=INNODB; - -CREATE TABLE w2l_templates ( - -- Not used for anything except replication. - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, - - -- Effective "primary key" (PUBLIC) - token CHAR(16) NOT NULL UNIQUE, - - template_type ENUM( - 'image', - 'video' - ) NOT NULL, - - -- The title of the template. - title CHAR(255) NOT NULL UNIQUE, - - -- Users can upload their own private templates. - -- They can choose to make them public later. - is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, - - -- The description of the template. - description CHAR(512) NOT NULL UNIQUE, - - -- The person that created the template. - creator_user_token CHAR(16) NOT NULL, - - -- For abuse tracking. - creator_ip_address CHAR(16) NOT NULL, - - -- The filename that was used at upload time. - original_filename CHAR(255) NOT NULL UNIQUE, - - -- The original source image/video - private_bucket_hash_original CHAR(32) NOT NULL UNIQUE, - - -- The "precomputed" faces. - private_bucket_hash_precomputed CHAR(32) NOT NULL UNIQUE, - - -- For the thumbnail we show. - public_bucket_hash CHAR(32) NOT NULL UNIQUE, - - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - - -- If this is removed. - deleted_at TIMESTAMP DEFAULT NULL - -) ENGINE=INNODB; - From ea2261c07a5ebab280bfdb02bcc704ed58532fab Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 04:48:41 -0400 Subject: [PATCH 0007/1042] more schema --- migrations/2021-05-15-071545_jobs/up.sql | 6 +++++ migrations/2021-05-15-071601_assets/up.sql | 26 ++++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/migrations/2021-05-15-071545_jobs/up.sql b/migrations/2021-05-15-071545_jobs/up.sql index 591cf6a83a..0cf56b8362 100644 --- a/migrations/2021-05-15-071545_jobs/up.sql +++ b/migrations/2021-05-15-071545_jobs/up.sql @@ -8,6 +8,9 @@ CREATE TABLE tts_model_upload_jobs ( -- Foreign key to user creator_user_token CHAR(16) NOT NULL, + -- For abuse tracking. + creator_ip_address CHAR(16) NOT NULL, + -- Users can upload their own private models. -- They can choose to make them public later. is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, @@ -59,6 +62,9 @@ CREATE TABLE w2l_template_upload_jobs ( -- Foreign key to user creator_user_token CHAR(16) NOT NULL, + -- For abuse tracking. + creator_ip_address CHAR(16) NOT NULL, + -- Users can upload their own private templates. -- They can choose to make them public later. is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 7f0825c39c..1b09d6b22a 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -8,6 +8,20 @@ CREATE TABLE tts_models ( -- Effective "primary key" (PUBLIC) token CHAR(16) NOT NULL UNIQUE, + -- Optional Pointer to a newer version of the voice + -- If there's a newer version, we can disable this one. + updated_model_token CHAR(16) DEFAULT NULL, + + -- We an disable a voice for a variety of reasons + -- In this case, the original author disables it. + user_disabled BOOLEAN NOT NULL DEFAULT FALSE, + -- In this case, a moderator author disables it. + mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, + + -- A combination of ['username' + 'voice-name'] + -- There can be public aliases for voices. + slug CHAR(64) NOT NULL UNIQUE, + -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. tts_model_type ENUM( @@ -20,6 +34,9 @@ CREATE TABLE tts_models ( -- The name of the voice voice_name VARCHAR(255) NOT NULL, + -- If the voice is "happy" or a singer "a-capella", etc. + voice_characteristic VARCHAR(255) DEFAULT NULL, + -- The speaker (in the case of cartoon characters) voice_actor_name VARCHAR(255) DEFAULT NULL, @@ -46,10 +63,15 @@ CREATE TABLE tts_models ( -- For the thumbnail we show. public_bucket_hash CHAR(32) NOT NULL UNIQUE, + -- Calculated average, on a scale of 0-100 + -- Null with zero ratings. + calculated_average_score INT(3) DEFAULT NULL, + calculated_score_count INT(10) NOT NULL DEFAULT 0, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - -- If this is removed. + -- If this is removed by a mod. deleted_at TIMESTAMP DEFAULT NULL ) ENGINE=INNODB; @@ -98,7 +120,7 @@ CREATE TABLE w2l_templates ( created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - -- If this is removed. + -- If this is removed by a mod. deleted_at TIMESTAMP DEFAULT NULL ) ENGINE=INNODB; From a733a2da0bcdb02ab9bb828eb89379188215633d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 05:04:05 -0400 Subject: [PATCH 0008/1042] voices table --- migrations/2021-05-15-071601_assets/down.sql | 1 + migrations/2021-05-15-071601_assets/up.sql | 50 +++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/down.sql b/migrations/2021-05-15-071601_assets/down.sql index 8a9a8d2f98..e1d28ed709 100644 --- a/migrations/2021-05-15-071601_assets/down.sql +++ b/migrations/2021-05-15-071601_assets/down.sql @@ -3,3 +3,4 @@ drop table tts_models; drop table w2l_templates; +drop table voices; \ No newline at end of file diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 1b09d6b22a..ddb971f8d0 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -19,7 +19,7 @@ CREATE TABLE tts_models ( mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, -- A combination of ['username' + 'voice-name'] - -- There can be public aliases for voices. + -- There can be public aliases for voices, eg. a voice's default model. slug CHAR(64) NOT NULL UNIQUE, -- NB: DO NOT CHANGE ORDER; APPEND ONLY! @@ -31,13 +31,19 @@ CREATE TABLE tts_models ( 'glowtts-vocodes' ) NOT NULL DEFAULT 'not-set', - -- The name of the voice + -- Can be linked to a well-known voice + voice_token CHAR(16) DEFAULT NULL, + + -- The name of the voice. + -- If voice_token is set, it's authoritative. voice_name VARCHAR(255) NOT NULL, -- If the voice is "happy" or a singer "a-capella", etc. + -- If voice_token is set, it's authoritative. voice_characteristic VARCHAR(255) DEFAULT NULL, -- The speaker (in the case of cartoon characters) + -- If voice_token is set, it's authoritative. voice_actor_name VARCHAR(255) DEFAULT NULL, -- Users can upload their own private models. @@ -124,3 +130,43 @@ CREATE TABLE w2l_templates ( deleted_at TIMESTAMP DEFAULT NULL ) ENGINE=INNODB; + +CREATE TABLE voices ( + -- Not used for anything except replication. + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + -- Effective "primary key" (PUBLIC) + token CHAR(16) NOT NULL UNIQUE, + + -- The URL we access the voice at. + slug CHAR(16) NOT NULL UNIQUE, + + -- We can assign an exemplary model to the voice + default_model_token CHAR(16) DEFAULT NULL, + + -- If a moderator author disables it. + -- This should prevent the voice from showing up in lists. + mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, + + -- The name of the voice + voice_name VARCHAR(255) NOT NULL, + + -- If the voice is "happy" or a singer "a-capella", etc. + voice_characteristic VARCHAR(255) DEFAULT NULL, + + -- The speaker (in the case of cartoon characters) + voice_actor_name VARCHAR(255) DEFAULT NULL, + + -- The 800x600 image + image_banner_public_bucket_hash CHAR(32) DEFAULT NULL, + + -- The square avatar image + image_square_public_bucket_hash CHAR(32) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod. + deleted_at TIMESTAMP DEFAULT NULL + +) ENGINE=INNODB; From 9b8f0ce0289a824226070dcfab5ae8464a710616 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 16:43:38 -0400 Subject: [PATCH 0009/1042] all migrations run --- migrations/2021-05-15-071534_users/down.sql | 3 ++- migrations/2021-05-15-071534_users/up.sql | 20 ++++++++++++++++---- migrations/2021-05-15-071545_jobs/up.sql | 4 ++-- migrations/2021-05-15-071601_assets/up.sql | 20 +++++++++++++------- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/migrations/2021-05-15-071534_users/down.sql b/migrations/2021-05-15-071534_users/down.sql index 5b1ba4f03c..7d31cdce5e 100644 --- a/migrations/2021-05-15-071534_users/down.sql +++ b/migrations/2021-05-15-071534_users/down.sql @@ -1,6 +1,7 @@ # noinspection SqlNoDataSourceInspectionForFile # noinspection SqlResolveForFile -drop table users; +drop table email_verifications; drop table user_roles; drop table user_sessions; +drop table users; diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 4c782879ba..520ef6b383 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -15,6 +15,12 @@ CREATE TABLE users ( username VARCHAR(20) NOT NULL UNIQUE, display_name VARCHAR(20) NOT NULL, + -- The profile of the user in markdown (user editable). + profile_markdown TEXT NOT NULL, + + -- Generated HTML (not user-editable). + profile_rendered_html TEXT NOT NULL, + -- The role assigned to the user confers permissions. user_role_slug VARCHAR(16) NOT NULL, @@ -26,9 +32,14 @@ CREATE TABLE users ( -- Incremented with every update to the password. password_version INT NOT NULL DEFAULT 0, + -- Different than deleted. + -- Users still show up, but can't do anything. + banned BOOLEAN NOT NULL DEFAULT false, + -- For abuse tracking. ip_address_creation VARCHAR(20) NOT NULL, ip_address_last_login VARCHAR(20) NOT NULL, + ip_address_last_update VARCHAR(20) NOT NULL, -- For tracking stats. -- The "cached" values are updated by a background job. @@ -52,7 +63,7 @@ CREATE TABLE users ( -- Hide results from others (ie. won't show up in the wall) -- Moderators will still see them. -- If the URLs are shared, they'll be visible. - hide_results BOOLEAN NOT NULL DEFAULT false, + hide_results_preference BOOLEAN NOT NULL DEFAULT false, -- Incremented with every update. version INT NOT NULL DEFAULT 0, @@ -60,8 +71,9 @@ CREATE TABLE users ( created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - -- If the user is deleted (banned, etc.), we set this. - deleted_at TIMESTAMP DEFAULT NULL + -- If the user is deleted, we set this. + -- This is different than banned. These users won't show up at all. + deleted_at TIMESTAMP NULL ) ENGINE=INNODB; @@ -108,7 +120,7 @@ CREATE TABLE user_sessions ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- deletion = session termination - deleted_at TIMESTAMP DEFAULT NULL + deleted_at TIMESTAMP NULL ) ENGINE=INNODB; diff --git a/migrations/2021-05-15-071545_jobs/up.sql b/migrations/2021-05-15-071545_jobs/up.sql index 0cf56b8362..7b1d3ee8f9 100644 --- a/migrations/2021-05-15-071545_jobs/up.sql +++ b/migrations/2021-05-15-071545_jobs/up.sql @@ -51,7 +51,7 @@ CREATE TABLE tts_model_upload_jobs ( -- Subsequent tries can increase the timeout. -- Failures because of permissions require human intervention => [retry_at=null]. -- Failures because of invalid files are dead => [status=dead]. - retry_at TIMESTAMP DEFAULT NULL + retry_at TIMESTAMP NULL ) ENGINE=INNODB; @@ -97,7 +97,7 @@ CREATE TABLE w2l_template_upload_jobs ( -- Subsequent tries can increase the timeout. -- Failures because of permissions require human intervention => [retry_at=null]. -- Failures because of invalid files are dead => [status=dead]. - retry_at TIMESTAMP DEFAULT NULL + retry_at TIMESTAMP NULL ) ENGINE=INNODB; diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index ddb971f8d0..18ef47fbd6 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -50,8 +50,11 @@ CREATE TABLE tts_models ( -- They can choose to make them public later. is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, - -- The description of the template. - description CHAR(512) NOT NULL UNIQUE, + -- The description of the model in markdown. + description_markdown TEXT NOT NULL, + + -- Generated HTML (not user-editable). + description_rendered_html TEXT NOT NULL, -- The person that created the template. creator_user_token CHAR(16) NOT NULL, @@ -78,7 +81,7 @@ CREATE TABLE tts_models ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- If this is removed by a mod. - deleted_at TIMESTAMP DEFAULT NULL + deleted_at TIMESTAMP NULL ) ENGINE=INNODB; @@ -102,8 +105,11 @@ CREATE TABLE w2l_templates ( -- They can choose to make them public later. is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, - -- The description of the template. - description CHAR(512) NOT NULL UNIQUE, + -- The description of the template in markdown. + description_markdown TEXT NOT NULL, + + -- Generated HTML (not user-editable). + description_rendered_html TEXT NOT NULL, -- The person that created the template. creator_user_token CHAR(16) NOT NULL, @@ -127,7 +133,7 @@ CREATE TABLE w2l_templates ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- If this is removed by a mod. - deleted_at TIMESTAMP DEFAULT NULL + deleted_at TIMESTAMP NULL ) ENGINE=INNODB; @@ -167,6 +173,6 @@ CREATE TABLE voices ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- If this is removed by a mod. - deleted_at TIMESTAMP DEFAULT NULL + deleted_at TIMESTAMP NULL ) ENGINE=INNODB; From b8d256d1374f1c1e6d3b914fdfd0f7bf3da830fc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 16:45:47 -0400 Subject: [PATCH 0010/1042] update readme --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index bb31e4acee..ead1735859 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,20 @@ storyteller-web =============== This is our main user account monolith that we'll also bake other pieces into. +Local development +----------------- + +### Database migrations + +To reset the entire database (drop, migrate), run: + +```diesel database reset``` + +To migrate at the current step and beyond: + +```diesel migration run``` + +To undo migrations at the current step: + +```diesel migration redo``` + From 783c61d1b4597664ffa7629887ff2144f4550343 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 16:54:51 -0400 Subject: [PATCH 0011/1042] correct collations --- README.md | 8 ++++++++ migrations/2021-05-15-071534_users/up.sql | 8 ++++---- migrations/2021-05-15-071545_jobs/up.sql | 4 ++-- migrations/2021-05-15-071601_assets/up.sql | 6 +++--- migrations/2021-05-15-071616_misc/up.sql | 5 +++-- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ead1735859..bb34caa32a 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,11 @@ To undo migrations at the current step: ```diesel migration redo``` +TODO +---- + +* Make all "token" types correct +* Use correct collation for all tables +* Install indexes on all foreign keys +* Make sure unique indexes are correct + diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 520ef6b383..d090fbffb8 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -75,7 +75,7 @@ CREATE TABLE users ( -- This is different than banned. These users won't show up at all. deleted_at TIMESTAMP NULL -) ENGINE=INNODB; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; CREATE TABLE user_roles ( -- Not used for anything except replication. @@ -101,7 +101,7 @@ CREATE TABLE user_roles ( created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) ENGINE=INNODB; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; CREATE TABLE user_sessions ( -- Not used for anything except replication. @@ -122,7 +122,7 @@ CREATE TABLE user_sessions ( -- deletion = session termination deleted_at TIMESTAMP NULL -) ENGINE=INNODB; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; -- We only allow the most recent record for any given user to be redeemed. CREATE TABLE email_verifications ( @@ -149,4 +149,4 @@ CREATE TABLE email_verifications ( created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) ENGINE=INNODB; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-071545_jobs/up.sql b/migrations/2021-05-15-071545_jobs/up.sql index 7b1d3ee8f9..9385a753fb 100644 --- a/migrations/2021-05-15-071545_jobs/up.sql +++ b/migrations/2021-05-15-071545_jobs/up.sql @@ -53,7 +53,7 @@ CREATE TABLE tts_model_upload_jobs ( -- Failures because of invalid files are dead => [status=dead]. retry_at TIMESTAMP NULL -) ENGINE=INNODB; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; -- WAV2LIP TEMPLATES CREATE TABLE w2l_template_upload_jobs ( @@ -99,5 +99,5 @@ CREATE TABLE w2l_template_upload_jobs ( -- Failures because of invalid files are dead => [status=dead]. retry_at TIMESTAMP NULL -) ENGINE=INNODB; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 18ef47fbd6..37a26e6cc6 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -83,7 +83,7 @@ CREATE TABLE tts_models ( -- If this is removed by a mod. deleted_at TIMESTAMP NULL -) ENGINE=INNODB; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; CREATE TABLE w2l_templates ( -- Not used for anything except replication. @@ -135,7 +135,7 @@ CREATE TABLE w2l_templates ( -- If this is removed by a mod. deleted_at TIMESTAMP NULL -) ENGINE=INNODB; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; CREATE TABLE voices ( -- Not used for anything except replication. @@ -175,4 +175,4 @@ CREATE TABLE voices ( -- If this is removed by a mod. deleted_at TIMESTAMP NULL -) ENGINE=INNODB; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-071616_misc/up.sql b/migrations/2021-05-15-071616_misc/up.sql index f4faaafe42..11a0560359 100644 --- a/migrations/2021-05-15-071616_misc/up.sql +++ b/migrations/2021-05-15-071616_misc/up.sql @@ -20,11 +20,12 @@ CREATE TABLE badges ( created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) ENGINE=INNODB; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; -- Join table CREATE TABLE user_badges ( id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, user_token CHAR(16) NOT NULL UNIQUE, badge_token CHAR(16) NOT NULL UNIQUE -) ENGINE=INNODB; + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 7514f162caa31223be4a23353c1ea59c3b0598b3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 17:01:21 -0400 Subject: [PATCH 0012/1042] change primary key definitions --- migrations/2021-05-15-071534_users/up.sql | 28 +++++++++++++++------- migrations/2021-05-15-071545_jobs/up.sql | 14 +++++++---- migrations/2021-05-15-071601_assets/up.sql | 21 +++++++++++----- migrations/2021-05-15-071616_misc/up.sql | 15 ++++++++---- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index d090fbffb8..71e4fb523d 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -3,7 +3,7 @@ CREATE TABLE users ( -- Not used for anything except replication. - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" token CHAR(16) NOT NULL UNIQUE, @@ -73,13 +73,16 @@ CREATE TABLE users ( -- If the user is deleted, we set this. -- This is different than banned. These users won't show up at all. - deleted_at TIMESTAMP NULL + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; CREATE TABLE user_roles ( -- Not used for anything except replication. - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" slug CHAR(16) NOT NULL UNIQUE, @@ -99,13 +102,16 @@ CREATE TABLE user_roles ( version INT NOT NULL DEFAULT 0, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- INDICES -- + PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; CREATE TABLE user_sessions ( -- Not used for anything except replication. - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Session entropy token CHAR(32) NOT NULL, @@ -120,14 +126,17 @@ CREATE TABLE user_sessions ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- deletion = session termination - deleted_at TIMESTAMP NULL + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; -- We only allow the most recent record for any given user to be redeemed. CREATE TABLE email_verifications ( -- Not used for anything except replication. - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + id BIGINT(20) NOT NULL AUTO_INCREMENT, verification_type ENUM( 'email-verification', @@ -147,6 +156,9 @@ CREATE TABLE email_verifications ( expires_at TIMESTAMP NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- INDICES -- + PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-071545_jobs/up.sql b/migrations/2021-05-15-071545_jobs/up.sql index 9385a753fb..19108697ab 100644 --- a/migrations/2021-05-15-071545_jobs/up.sql +++ b/migrations/2021-05-15-071545_jobs/up.sql @@ -3,7 +3,7 @@ -- TTS MODELS CREATE TABLE tts_model_upload_jobs ( - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Foreign key to user creator_user_token CHAR(16) NOT NULL, @@ -51,13 +51,16 @@ CREATE TABLE tts_model_upload_jobs ( -- Subsequent tries can increase the timeout. -- Failures because of permissions require human intervention => [retry_at=null]. -- Failures because of invalid files are dead => [status=dead]. - retry_at TIMESTAMP NULL + retry_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; -- WAV2LIP TEMPLATES CREATE TABLE w2l_template_upload_jobs ( - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Foreign key to user creator_user_token CHAR(16) NOT NULL, @@ -97,7 +100,10 @@ CREATE TABLE w2l_template_upload_jobs ( -- Subsequent tries can increase the timeout. -- Failures because of permissions require human intervention => [retry_at=null]. -- Failures because of invalid files are dead => [status=dead]. - retry_at TIMESTAMP NULL + retry_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 37a26e6cc6..3e7ac2b5fd 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -3,7 +3,7 @@ CREATE TABLE tts_models ( -- Not used for anything except replication. - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) token CHAR(16) NOT NULL UNIQUE, @@ -81,13 +81,16 @@ CREATE TABLE tts_models ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- If this is removed by a mod. - deleted_at TIMESTAMP NULL + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; CREATE TABLE w2l_templates ( -- Not used for anything except replication. - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) token CHAR(16) NOT NULL UNIQUE, @@ -133,13 +136,16 @@ CREATE TABLE w2l_templates ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- If this is removed by a mod. - deleted_at TIMESTAMP NULL + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; CREATE TABLE voices ( -- Not used for anything except replication. - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) token CHAR(16) NOT NULL UNIQUE, @@ -173,6 +179,9 @@ CREATE TABLE voices ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- If this is removed by a mod. - deleted_at TIMESTAMP NULL + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-071616_misc/up.sql b/migrations/2021-05-15-071616_misc/up.sql index 11a0560359..0e768b8c29 100644 --- a/migrations/2021-05-15-071616_misc/up.sql +++ b/migrations/2021-05-15-071616_misc/up.sql @@ -4,7 +4,7 @@ -- Badges are rewards for loyalty CREATE TABLE badges ( -- Not used for anything except replication. - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" token CHAR(16) NOT NULL UNIQUE, @@ -18,14 +18,21 @@ CREATE TABLE badges ( image_url VARCHAR(512) NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- INDICES -- + PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; -- Join table CREATE TABLE user_badges ( - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + id BIGINT(20) NOT NULL AUTO_INCREMENT, + user_token CHAR(16) NOT NULL UNIQUE, - badge_token CHAR(16) NOT NULL UNIQUE + badge_token CHAR(16) NOT NULL UNIQUE, + + -- INDICES -- + PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From cb505f1cb0df99de28d21181ffde1b157e534e49 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 17:22:17 -0400 Subject: [PATCH 0013/1042] pk/fk/index defs --- migrations/2021-05-15-071534_users/up.sql | 23 ++++++++++++++++------- migrations/2021-05-15-071545_jobs/up.sql | 10 ++++++++-- migrations/2021-05-15-071616_misc/up.sql | 8 ++++++-- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 71e4fb523d..fe0ab57224 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -6,9 +6,9 @@ CREATE TABLE users ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" - token CHAR(16) NOT NULL UNIQUE, + token CHAR(16) NOT NULL, - email_address VARCHAR(255) NOT NULL UNIQUE, + email_address VARCHAR(255) NOT NULL, email_confirmed BOOLEAN NOT NULL DEFAULT false, -- Username is a lookup key; display_name allows the user to add custom case. @@ -76,7 +76,10 @@ CREATE TABLE users ( deleted_at TIMESTAMP NULL, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE KEY (token), + UNIQUE KEY (email_address), + KEY fk_user_role_slug (user_role_slug) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -85,7 +88,7 @@ CREATE TABLE user_roles ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" - slug CHAR(16) NOT NULL UNIQUE, + slug CHAR(16) NOT NULL, name VARCHAR(255) NOT NULL, @@ -105,7 +108,8 @@ CREATE TABLE user_roles ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE KEY (slug) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -129,7 +133,10 @@ CREATE TABLE user_sessions ( deleted_at TIMESTAMP NULL, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY fk_user_token (user_token), + KEY index_ip_address_creation (ip_address_creation) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -159,6 +166,8 @@ CREATE TABLE email_verifications ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + KEY fk_user_token (user_token), + KEY index_verification_type (verification_type) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-071545_jobs/up.sql b/migrations/2021-05-15-071545_jobs/up.sql index 19108697ab..858cc2aaf9 100644 --- a/migrations/2021-05-15-071545_jobs/up.sql +++ b/migrations/2021-05-15-071545_jobs/up.sql @@ -54,7 +54,10 @@ CREATE TABLE tts_model_upload_jobs ( retry_at TIMESTAMP NULL, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + KEY fk_creator_user_token (creator_user_token), + KEY index_status (status), + KEY index_creator_ip_address (creator_ip_address) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -103,7 +106,10 @@ CREATE TABLE w2l_template_upload_jobs ( retry_at TIMESTAMP NULL, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + KEY fk_creator_user_token (creator_user_token), + KEY index_status (status), + KEY index_creator_ip_address (creator_ip_address) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-071616_misc/up.sql b/migrations/2021-05-15-071616_misc/up.sql index 0e768b8c29..fc84e8ec31 100644 --- a/migrations/2021-05-15-071616_misc/up.sql +++ b/migrations/2021-05-15-071616_misc/up.sql @@ -21,7 +21,9 @@ CREATE TABLE badges ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE KEY (token), + UNIQUE KEY (slug) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -33,6 +35,8 @@ CREATE TABLE user_badges ( badge_token CHAR(16) NOT NULL UNIQUE, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + KEY fk_user_token (user_token), + KEY fk_badge_token (badge_token) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 72e9c17d50dae695571fa090155b018aef2278a7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 17:25:55 -0400 Subject: [PATCH 0014/1042] social media profiles --- migrations/2021-05-15-071534_users/up.sql | 6 ++++++ migrations/2021-05-15-071616_misc/up.sql | 12 ++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index fe0ab57224..7522881df8 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -65,6 +65,12 @@ CREATE TABLE users ( -- If the URLs are shared, they'll be visible. hide_results_preference BOOLEAN NOT NULL DEFAULT false, + -- Social media usernames + -- These are not confirmed. We'll need to build an OAuth system to handle that. + discord_username VARCHAR(36) DEFAULT NULL, + twitter_username VARCHAR(36) DEFAULT NULL, + twitch_username VARCHAR(36) DEFAULT NULL, + -- Incremented with every update. version INT NOT NULL DEFAULT 0, diff --git a/migrations/2021-05-15-071616_misc/up.sql b/migrations/2021-05-15-071616_misc/up.sql index fc84e8ec31..65b1f0ebf2 100644 --- a/migrations/2021-05-15-071616_misc/up.sql +++ b/migrations/2021-05-15-071616_misc/up.sql @@ -7,10 +7,7 @@ CREATE TABLE badges ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" - token CHAR(16) NOT NULL UNIQUE, - - -- Effective "primary key" - slug CHAR(16) NOT NULL UNIQUE, + slug CHAR(16) NOT NULL, -- Description title VARCHAR(255) NOT NULL, @@ -22,7 +19,6 @@ CREATE TABLE badges ( -- INDICES -- PRIMARY KEY (id), - UNIQUE KEY (token), UNIQUE KEY (slug) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -31,12 +27,12 @@ CREATE TABLE badges ( CREATE TABLE user_badges ( id BIGINT(20) NOT NULL AUTO_INCREMENT, - user_token CHAR(16) NOT NULL UNIQUE, - badge_token CHAR(16) NOT NULL UNIQUE, + user_token CHAR(16) NOT NULL, + badge_slug CHAR(16) NOT NULL, -- INDICES -- PRIMARY KEY (id), KEY fk_user_token (user_token), - KEY fk_badge_token (badge_token) + KEY fk_badge_slug (badge_slug) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 3e80c8eba3ec0ccefe684728e0f0efe3fe14d0ec Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 17:31:34 -0400 Subject: [PATCH 0015/1042] more types of tables --- .../down.sql | 3 +-- .../up.sql | 3 +-- migrations/2021-05-15-212918_inference_results/down.sql | 2 ++ migrations/2021-05-15-212918_inference_results/up.sql | 2 ++ .../down.sql | 0 .../{2021-05-15-071616_misc => 2021-05-15-212923_misc}/up.sql | 0 6 files changed, 6 insertions(+), 4 deletions(-) rename migrations/{2021-05-15-071612_results => 2021-05-15-212913_inference_jobs}/down.sql (98%) rename migrations/{2021-05-15-071612_results => 2021-05-15-212913_inference_jobs}/up.sql (98%) create mode 100644 migrations/2021-05-15-212918_inference_results/down.sql create mode 100644 migrations/2021-05-15-212918_inference_results/up.sql rename migrations/{2021-05-15-071616_misc => 2021-05-15-212923_misc}/down.sql (100%) rename migrations/{2021-05-15-071616_misc => 2021-05-15-212923_misc}/up.sql (100%) diff --git a/migrations/2021-05-15-071612_results/down.sql b/migrations/2021-05-15-212913_inference_jobs/down.sql similarity index 98% rename from migrations/2021-05-15-071612_results/down.sql rename to migrations/2021-05-15-212913_inference_jobs/down.sql index c08437d48b..e9ead36df2 100644 --- a/migrations/2021-05-15-071612_results/down.sql +++ b/migrations/2021-05-15-212913_inference_jobs/down.sql @@ -1,3 +1,2 @@ -# noinspection SqlNoDataSourceInspectionForFile # noinspection SqlResolveForFile - +# noinspection SqlNoDataSourceInspectionForFile diff --git a/migrations/2021-05-15-071612_results/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql similarity index 98% rename from migrations/2021-05-15-071612_results/up.sql rename to migrations/2021-05-15-212913_inference_jobs/up.sql index c08437d48b..e9ead36df2 100644 --- a/migrations/2021-05-15-071612_results/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -1,3 +1,2 @@ -# noinspection SqlNoDataSourceInspectionForFile # noinspection SqlResolveForFile - +# noinspection SqlNoDataSourceInspectionForFile diff --git a/migrations/2021-05-15-212918_inference_results/down.sql b/migrations/2021-05-15-212918_inference_results/down.sql new file mode 100644 index 0000000000..e9ead36df2 --- /dev/null +++ b/migrations/2021-05-15-212918_inference_results/down.sql @@ -0,0 +1,2 @@ +# noinspection SqlResolveForFile +# noinspection SqlNoDataSourceInspectionForFile diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql new file mode 100644 index 0000000000..fc7d621a59 --- /dev/null +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -0,0 +1,2 @@ +# noinspection SqlResolveForFile +# noinspection SqlNoDataSourceInspectionForFile \ No newline at end of file diff --git a/migrations/2021-05-15-071616_misc/down.sql b/migrations/2021-05-15-212923_misc/down.sql similarity index 100% rename from migrations/2021-05-15-071616_misc/down.sql rename to migrations/2021-05-15-212923_misc/down.sql diff --git a/migrations/2021-05-15-071616_misc/up.sql b/migrations/2021-05-15-212923_misc/up.sql similarity index 100% rename from migrations/2021-05-15-071616_misc/up.sql rename to migrations/2021-05-15-212923_misc/up.sql From b6abf17096b49b6fc4ec73aa0c533ad548ed678d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 17:32:21 -0400 Subject: [PATCH 0016/1042] rename migration --- .../down.sql | 0 .../up.sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename migrations/{2021-05-15-071545_jobs => 2021-05-15-071545_asset_upload_jobs}/down.sql (100%) rename migrations/{2021-05-15-071545_jobs => 2021-05-15-071545_asset_upload_jobs}/up.sql (100%) diff --git a/migrations/2021-05-15-071545_jobs/down.sql b/migrations/2021-05-15-071545_asset_upload_jobs/down.sql similarity index 100% rename from migrations/2021-05-15-071545_jobs/down.sql rename to migrations/2021-05-15-071545_asset_upload_jobs/down.sql diff --git a/migrations/2021-05-15-071545_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql similarity index 100% rename from migrations/2021-05-15-071545_jobs/up.sql rename to migrations/2021-05-15-071545_asset_upload_jobs/up.sql From 6edbec3cb89ac545c9a9e6c6651c64bdb5b4428b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 17:46:40 -0400 Subject: [PATCH 0017/1042] inference jobs --- migrations/2021-05-15-071601_assets/up.sql | 6 +- .../2021-05-15-212913_inference_jobs/down.sql | 3 + .../2021-05-15-212913_inference_jobs/up.sql | 106 ++++++++++++++++++ 3 files changed, 113 insertions(+), 2 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 3e7ac2b5fd..6c2970602c 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -84,7 +84,8 @@ CREATE TABLE tts_models ( deleted_at TIMESTAMP NULL, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE KEY (token) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -139,7 +140,8 @@ CREATE TABLE w2l_templates ( deleted_at TIMESTAMP NULL, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE KEY (token) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-212913_inference_jobs/down.sql b/migrations/2021-05-15-212913_inference_jobs/down.sql index e9ead36df2..cad1d8e8ff 100644 --- a/migrations/2021-05-15-212913_inference_jobs/down.sql +++ b/migrations/2021-05-15-212913_inference_jobs/down.sql @@ -1,2 +1,5 @@ # noinspection SqlResolveForFile # noinspection SqlNoDataSourceInspectionForFile + +drop table tts_inference_jobs; +drop table w2l_inference_jobs; diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index e9ead36df2..539460164a 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -1,2 +1,108 @@ # noinspection SqlResolveForFile # noinspection SqlNoDataSourceInspectionForFile +CREATE TABLE tts_inference_jobs ( + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Foreign key to user + -- If no user is logged in, this is null. + maybe_creator_user_token CHAR(16) DEFAULT NULL, + + -- For abuse tracking. + creator_ip_address CHAR(16) NOT NULL, + + -- Users can upload their own private models. + -- They can choose to make them public later. + is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + + -- The model to use. + -- This also determines which architecture we're using. + model_token CHAR(16) NOT NULL, + + -- The raw, unprocessed user input. + inference_text TEXT NOT NULL, + + -- Jobs begin as "pending", then transition to other states. + -- Pending -> Started -> Complete + -- \-> Failed -> Started -> { Complete, Failed, Dead } + status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending', + + -- We can track this against a "max_attempt_count" + attempt_count INT(3) NOT NULL DEFAULT 0, + + -- If there is a failure, tell the user why. + failure_reason VARCHAR(512) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- Failed jobs will set a next attempt time. + -- Subsequent tries can increase the timeout. + -- Failures because of permissions require human intervention => [retry_at=null]. + -- Failures because of invalid files are dead => [status=dead]. + retry_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY index_status (status), + KEY index_creator_ip_address (creator_ip_address) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +CREATE TABLE w2l_inference_jobs ( + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Foreign key to user + -- If no user is logged in, this is null. + maybe_creator_user_token CHAR(16) DEFAULT NULL, + + -- For abuse tracking. + creator_ip_address CHAR(16) NOT NULL, + + -- Users can upload their own private models. + -- They can choose to make them public later. + is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + + -- The W2L template to use + -- Can be an image or video. + -- This is null if we're using a custom uploaded image. + w2l_template_token CHAR(16) DEFAULT NULL, + + -- If we're using TTS results, this will be present + tts_inference_result_token CHAR(16) DEFAULT NULL, + + -- If we're using custom uploaded audio, this will be present. + audio_bucket_location CHAR(16) DEFAULT NULL, + + -- If we're using a custom uploaded image, this will be present. + image_bucket_location CHAR(16) DEFAULT NULL, + + -- Jobs begin as "pending", then transition to other states. + -- Pending -> Started -> Complete + -- \-> Failed -> Started -> { Complete, Failed, Dead } + status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending', + + -- We can track this against a "max_attempt_count" + attempt_count INT(3) NOT NULL DEFAULT 0, + + -- If there is a failure, tell the user why. + failure_reason VARCHAR(512) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- Failed jobs will set a next attempt time. + -- Subsequent tries can increase the timeout. + -- Failures because of permissions require human intervention => [retry_at=null]. + -- Failures because of invalid files are dead => [status=dead]. + retry_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_w2l_template_token (w2l_template_token), + KEY fk_tts_inference_result_token (tts_inference_result_token), + KEY index_status (status), + KEY index_creator_ip_address (creator_ip_address) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From d1b7fef17ca6fdc0a1faea61337497e27fdaffe2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 15 May 2021 18:01:32 -0400 Subject: [PATCH 0018/1042] inference results --- migrations/2021-05-15-071601_assets/up.sql | 4 +- .../2021-05-15-212913_inference_jobs/up.sql | 12 +-- .../down.sql | 3 + .../up.sql | 87 ++++++++++++++++++- 4 files changed, 97 insertions(+), 9 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 6c2970602c..b052006ac3 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -94,7 +94,7 @@ CREATE TABLE w2l_templates ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) - token CHAR(16) NOT NULL UNIQUE, + token CHAR(16) NOT NULL, template_type ENUM( 'not-set', @@ -103,7 +103,7 @@ CREATE TABLE w2l_templates ( ) NOT NULL DEFAULT 'not-set', -- The title of the template. - title CHAR(255) NOT NULL UNIQUE, + title CHAR(255) NOT NULL, -- Users can upload their own private templates. -- They can choose to make them public later. diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index 539460164a..799b64e761 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -66,16 +66,16 @@ CREATE TABLE w2l_inference_jobs ( -- The W2L template to use -- Can be an image or video. -- This is null if we're using a custom uploaded image. - w2l_template_token CHAR(16) DEFAULT NULL, + maybe_w2l_template_token CHAR(16) DEFAULT NULL, -- If we're using TTS results, this will be present - tts_inference_result_token CHAR(16) DEFAULT NULL, + maybe_tts_inference_result_token CHAR(16) DEFAULT NULL, -- If we're using custom uploaded audio, this will be present. - audio_bucket_location CHAR(16) DEFAULT NULL, + maybe_audio_bucket_location CHAR(16) DEFAULT NULL, -- If we're using a custom uploaded image, this will be present. - image_bucket_location CHAR(16) DEFAULT NULL, + maybe_image_bucket_location CHAR(16) DEFAULT NULL, -- Jobs begin as "pending", then transition to other states. -- Pending -> Started -> Complete @@ -100,8 +100,8 @@ CREATE TABLE w2l_inference_jobs ( -- INDICES -- PRIMARY KEY (id), KEY fk_maybe_creator_user_token (maybe_creator_user_token), - KEY fk_w2l_template_token (w2l_template_token), - KEY fk_tts_inference_result_token (tts_inference_result_token), + KEY fk_maybe_w2l_template_token (maybe_w2l_template_token), + KEY fk_maybe_tts_inference_result_token (maybe_tts_inference_result_token), KEY index_status (status), KEY index_creator_ip_address (creator_ip_address) diff --git a/migrations/2021-05-15-212918_inference_results/down.sql b/migrations/2021-05-15-212918_inference_results/down.sql index e9ead36df2..261d58e669 100644 --- a/migrations/2021-05-15-212918_inference_results/down.sql +++ b/migrations/2021-05-15-212918_inference_results/down.sql @@ -1,2 +1,5 @@ # noinspection SqlResolveForFile # noinspection SqlNoDataSourceInspectionForFile + +drop table tts_results; +drop table w2l_results; diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index fc7d621a59..19270b58e8 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -1,2 +1,87 @@ # noinspection SqlResolveForFile -# noinspection SqlNoDataSourceInspectionForFile \ No newline at end of file +# noinspection SqlNoDataSourceInspectionForFile + +CREATE TABLE tts_results ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + token CHAR(32) NOT NULL, + + -- The TTS model that was used + model_token CHAR(16) NOT NULL, + + -- The original raw, unprocessed user input. + inference_text TEXT NOT NULL, + + -- Users can upload their own private templates. + -- They can choose to make them public later. + is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + + -- The person that created the template. + -- If the user wasn't logged in, this is null + maybe_creator_user_token CHAR(16) DEFAULT NULL, + + -- For abuse tracking. + creator_ip_address CHAR(16) NOT NULL, + + -- Where the wav, spectrogram, and etc. are located. + public_bucket_hash CHAR(32) NOT NULL UNIQUE, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod or the creator. + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY fk_model_token (model_token), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY index_creator_ip_address (creator_ip_address) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +CREATE TABLE w2l_results ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + token CHAR(32) NOT NULL, + + -- The w2l template that was used (if set) + maybe_w2l_template_token CHAR(16) NOT NULL, + + -- The inference result, if we're using them. + maybe_tts_inference_result_token CHAR(16) DEFAULT NULL, + + -- Users can upload their own private templates. + -- They can choose to make them public later. + is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + + -- The person that created the template. + -- If the user wasn't logged in, this is null + maybe_creator_user_token CHAR(16) DEFAULT NULL, + + -- For abuse tracking. + creator_ip_address CHAR(16) NOT NULL, + + -- Where the wav, spectrogram, and etc. are located. + public_bucket_hash CHAR(32) NOT NULL UNIQUE, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod or the creator. + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_maybe_w2l_template_token (maybe_w2l_template_token), + KEY fk_maybe_tts_inference_result_token (maybe_tts_inference_result_token), + KEY index_creator_ip_address (creator_ip_address) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 63a63fa28db59f6a564d9dd9f79492847bad1916 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 16 May 2021 10:16:48 -0400 Subject: [PATCH 0019/1042] widen ip address columns --- migrations/2021-05-15-071534_users/up.sql | 10 ++++++---- migrations/2021-05-15-071545_asset_upload_jobs/up.sql | 6 ++++-- migrations/2021-05-15-071601_assets/up.sql | 6 ++++-- migrations/2021-05-15-212913_inference_jobs/up.sql | 6 ++++-- migrations/2021-05-15-212918_inference_results/up.sql | 6 ++++-- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 7522881df8..57834da87d 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -37,9 +37,10 @@ CREATE TABLE users ( banned BOOLEAN NOT NULL DEFAULT false, -- For abuse tracking. - ip_address_creation VARCHAR(20) NOT NULL, - ip_address_last_login VARCHAR(20) NOT NULL, - ip_address_last_update VARCHAR(20) NOT NULL, + -- Wide enough for IPv4/6 + ip_address_creation VARCHAR(40) NOT NULL, + ip_address_last_login VARCHAR(40) NOT NULL, + ip_address_last_update VARCHAR(40) NOT NULL, -- For tracking stats. -- The "cached" values are updated by a background job. @@ -130,7 +131,8 @@ CREATE TABLE user_sessions ( user_token CHAR(16) NOT NULL, -- Track each session's creation IP. - ip_address_creation VARCHAR(20) NOT NULL, + -- Wide enough for IPv4/6 + ip_address_creation VARCHAR(40) NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index 858cc2aaf9..5c12653abb 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -9,7 +9,8 @@ CREATE TABLE tts_model_upload_jobs ( creator_user_token CHAR(16) NOT NULL, -- For abuse tracking. - creator_ip_address CHAR(16) NOT NULL, + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, -- Users can upload their own private models. -- They can choose to make them public later. @@ -69,7 +70,8 @@ CREATE TABLE w2l_template_upload_jobs ( creator_user_token CHAR(16) NOT NULL, -- For abuse tracking. - creator_ip_address CHAR(16) NOT NULL, + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, -- Users can upload their own private templates. -- They can choose to make them public later. diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index b052006ac3..02683c6ba2 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -60,7 +60,8 @@ CREATE TABLE tts_models ( creator_user_token CHAR(16) NOT NULL, -- For abuse tracking. - creator_ip_address CHAR(16) NOT NULL, + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, -- The filename that was used at upload time. original_filename CHAR(255) NOT NULL UNIQUE, @@ -119,7 +120,8 @@ CREATE TABLE w2l_templates ( creator_user_token CHAR(16) NOT NULL, -- For abuse tracking. - creator_ip_address CHAR(16) NOT NULL, + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, -- The filename that was used at upload time. original_filename CHAR(255) NOT NULL UNIQUE, diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index 799b64e761..f4a2a080bf 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -8,7 +8,8 @@ CREATE TABLE tts_inference_jobs ( maybe_creator_user_token CHAR(16) DEFAULT NULL, -- For abuse tracking. - creator_ip_address CHAR(16) NOT NULL, + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, -- Users can upload their own private models. -- They can choose to make them public later. @@ -57,7 +58,8 @@ CREATE TABLE w2l_inference_jobs ( maybe_creator_user_token CHAR(16) DEFAULT NULL, -- For abuse tracking. - creator_ip_address CHAR(16) NOT NULL, + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, -- Users can upload their own private models. -- They can choose to make them public later. diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 19270b58e8..4d7f48d6a2 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -23,7 +23,8 @@ CREATE TABLE tts_results ( maybe_creator_user_token CHAR(16) DEFAULT NULL, -- For abuse tracking. - creator_ip_address CHAR(16) NOT NULL, + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, -- Where the wav, spectrogram, and etc. are located. public_bucket_hash CHAR(32) NOT NULL UNIQUE, @@ -65,7 +66,8 @@ CREATE TABLE w2l_results ( maybe_creator_user_token CHAR(16) DEFAULT NULL, -- For abuse tracking. - creator_ip_address CHAR(16) NOT NULL, + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, -- Where the wav, spectrogram, and etc. are located. public_bucket_hash CHAR(32) NOT NULL UNIQUE, From 938f6d15fa623910df0ca437e3c5c614efb1b37f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 16 May 2021 10:35:54 -0400 Subject: [PATCH 0020/1042] widen tokens --- migrations/2021-05-15-071534_users/up.sql | 8 ++++---- .../2021-05-15-071545_asset_upload_jobs/up.sql | 4 ++-- migrations/2021-05-15-071601_assets/up.sql | 16 ++++++++-------- .../2021-05-15-212913_inference_jobs/up.sql | 8 ++++---- .../2021-05-15-212918_inference_results/up.sql | 14 +++++++------- migrations/2021-05-15-212923_misc/up.sql | 6 +++--- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 57834da87d..0075446109 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -6,7 +6,7 @@ CREATE TABLE users ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" - token CHAR(16) NOT NULL, + token VARCHAR(32) NOT NULL, email_address VARCHAR(255) NOT NULL, email_confirmed BOOLEAN NOT NULL DEFAULT false, @@ -125,10 +125,10 @@ CREATE TABLE user_sessions ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Session entropy - token CHAR(32) NOT NULL, + token VARCHAR(32) NOT NULL, -- Foreign key to user - user_token CHAR(16) NOT NULL, + user_token VARCHAR(32) NOT NULL, -- Track each session's creation IP. -- Wide enough for IPv4/6 @@ -165,7 +165,7 @@ CREATE TABLE email_verifications ( successful BOOLEAN NOT NULL DEFAULT false, -- Foreign key to user - user_token CHAR(16) NOT NULL, + user_token VARCHAR(32) NOT NULL, -- Cannot be redeemed after this date expires_at TIMESTAMP NOT NULL, diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index 5c12653abb..e33886c818 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -6,7 +6,7 @@ CREATE TABLE tts_model_upload_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Foreign key to user - creator_user_token CHAR(16) NOT NULL, + creator_user_token VARCHAR(32) NOT NULL, -- For abuse tracking. -- Wide enough for IPv4/6 @@ -67,7 +67,7 @@ CREATE TABLE w2l_template_upload_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Foreign key to user - creator_user_token CHAR(16) NOT NULL, + creator_user_token VARCHAR(32) NOT NULL, -- For abuse tracking. -- Wide enough for IPv4/6 diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 02683c6ba2..95971c942f 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -6,11 +6,11 @@ CREATE TABLE tts_models ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) - token CHAR(16) NOT NULL UNIQUE, + token VARCHAR(32) NOT NULL UNIQUE, -- Optional Pointer to a newer version of the voice -- If there's a newer version, we can disable this one. - updated_model_token CHAR(16) DEFAULT NULL, + updated_model_token VARCHAR(32) DEFAULT NULL, -- We an disable a voice for a variety of reasons -- In this case, the original author disables it. @@ -32,7 +32,7 @@ CREATE TABLE tts_models ( ) NOT NULL DEFAULT 'not-set', -- Can be linked to a well-known voice - voice_token CHAR(16) DEFAULT NULL, + voice_token VARCHAR(32) DEFAULT NULL, -- The name of the voice. -- If voice_token is set, it's authoritative. @@ -57,7 +57,7 @@ CREATE TABLE tts_models ( description_rendered_html TEXT NOT NULL, -- The person that created the template. - creator_user_token CHAR(16) NOT NULL, + creator_user_token VARCHAR(32) NOT NULL, -- For abuse tracking. -- Wide enough for IPv4/6 @@ -95,7 +95,7 @@ CREATE TABLE w2l_templates ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) - token CHAR(16) NOT NULL, + token VARCHAR(32) NOT NULL, template_type ENUM( 'not-set', @@ -117,7 +117,7 @@ CREATE TABLE w2l_templates ( description_rendered_html TEXT NOT NULL, -- The person that created the template. - creator_user_token CHAR(16) NOT NULL, + creator_user_token VARCHAR(32) NOT NULL, -- For abuse tracking. -- Wide enough for IPv4/6 @@ -152,13 +152,13 @@ CREATE TABLE voices ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) - token CHAR(16) NOT NULL UNIQUE, + token VARCHAR(32) NOT NULL UNIQUE, -- The URL we access the voice at. slug CHAR(16) NOT NULL UNIQUE, -- We can assign an exemplary model to the voice - default_model_token CHAR(16) DEFAULT NULL, + default_model_token VARCHAR(32) DEFAULT NULL, -- If a moderator author disables it. -- This should prevent the voice from showing up in lists. diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index f4a2a080bf..defbdeba69 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -17,7 +17,7 @@ CREATE TABLE tts_inference_jobs ( -- The model to use. -- This also determines which architecture we're using. - model_token CHAR(16) NOT NULL, + model_token VARCHAR(32) NOT NULL, -- The raw, unprocessed user input. inference_text TEXT NOT NULL, @@ -55,7 +55,7 @@ CREATE TABLE w2l_inference_jobs ( -- Foreign key to user -- If no user is logged in, this is null. - maybe_creator_user_token CHAR(16) DEFAULT NULL, + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, -- For abuse tracking. -- Wide enough for IPv4/6 @@ -68,10 +68,10 @@ CREATE TABLE w2l_inference_jobs ( -- The W2L template to use -- Can be an image or video. -- This is null if we're using a custom uploaded image. - maybe_w2l_template_token CHAR(16) DEFAULT NULL, + maybe_w2l_template_token VARCHAR(32) DEFAULT NULL, -- If we're using TTS results, this will be present - maybe_tts_inference_result_token CHAR(16) DEFAULT NULL, + maybe_tts_inference_result_token VARCHAR(32) DEFAULT NULL, -- If we're using custom uploaded audio, this will be present. maybe_audio_bucket_location CHAR(16) DEFAULT NULL, diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 4d7f48d6a2..9cd59a409f 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -6,10 +6,10 @@ CREATE TABLE tts_results ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) - token CHAR(32) NOT NULL, + token VARCHAR(32) NOT NULL, -- The TTS model that was used - model_token CHAR(16) NOT NULL, + model_token VARCHAR(32) NOT NULL, -- The original raw, unprocessed user input. inference_text TEXT NOT NULL, @@ -20,7 +20,7 @@ CREATE TABLE tts_results ( -- The person that created the template. -- If the user wasn't logged in, this is null - maybe_creator_user_token CHAR(16) DEFAULT NULL, + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, -- For abuse tracking. -- Wide enough for IPv4/6 @@ -49,13 +49,13 @@ CREATE TABLE w2l_results ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) - token CHAR(32) NOT NULL, + token VARCHAR(32) NOT NULL, -- The w2l template that was used (if set) - maybe_w2l_template_token CHAR(16) NOT NULL, + maybe_w2l_template_token VARCHAR(32) NOT NULL, -- The inference result, if we're using them. - maybe_tts_inference_result_token CHAR(16) DEFAULT NULL, + maybe_tts_inference_result_token VARCHAR(32) DEFAULT NULL, -- Users can upload their own private templates. -- They can choose to make them public later. @@ -63,7 +63,7 @@ CREATE TABLE w2l_results ( -- The person that created the template. -- If the user wasn't logged in, this is null - maybe_creator_user_token CHAR(16) DEFAULT NULL, + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, -- For abuse tracking. -- Wide enough for IPv4/6 diff --git a/migrations/2021-05-15-212923_misc/up.sql b/migrations/2021-05-15-212923_misc/up.sql index 65b1f0ebf2..2bd06f5d62 100644 --- a/migrations/2021-05-15-212923_misc/up.sql +++ b/migrations/2021-05-15-212923_misc/up.sql @@ -7,7 +7,7 @@ CREATE TABLE badges ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" - slug CHAR(16) NOT NULL, + slug VARCHAR(32) NOT NULL, -- Description title VARCHAR(255) NOT NULL, @@ -27,8 +27,8 @@ CREATE TABLE badges ( CREATE TABLE user_badges ( id BIGINT(20) NOT NULL AUTO_INCREMENT, - user_token CHAR(16) NOT NULL, - badge_slug CHAR(16) NOT NULL, + user_token VARCHAR(32) NOT NULL, + badge_slug VARCHAR(32) NOT NULL, -- INDICES -- PRIMARY KEY (id), From 7d33ecd1dfe05b3a269894c029755d43c0518ceb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 16 May 2021 10:50:24 -0400 Subject: [PATCH 0021/1042] indices, etc --- migrations/2021-05-15-071601_assets/up.sql | 55 +++++++++++-------- .../2021-05-15-212913_inference_jobs/up.sql | 37 +++++++------ .../up.sql | 4 +- 3 files changed, 54 insertions(+), 42 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 95971c942f..c208d5c225 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -6,7 +6,13 @@ CREATE TABLE tts_models ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) - token VARCHAR(32) NOT NULL UNIQUE, + token VARCHAR(32) NOT NULL, + + -- A combination of ['username' + 'voice-name'] + -- There can be public aliases for voices, eg. a voice's default model. + -- A user can change this. + -- As such, these should not be foreign keys. + updatable_slug VARCHAR(64) NOT NULL, -- Optional Pointer to a newer version of the voice -- If there's a newer version, we can disable this one. @@ -18,10 +24,6 @@ CREATE TABLE tts_models ( -- In this case, a moderator author disables it. mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, - -- A combination of ['username' + 'voice-name'] - -- There can be public aliases for voices, eg. a voice's default model. - slug CHAR(64) NOT NULL UNIQUE, - -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. tts_model_type ENUM( @@ -64,14 +66,11 @@ CREATE TABLE tts_models ( creator_ip_address VARCHAR(40) NOT NULL, -- The filename that was used at upload time. - original_filename CHAR(255) NOT NULL UNIQUE, + original_filename CHAR(255) NOT NULL, -- The pytorch model -- For now, this will be a hash of the file contents. - private_bucket_hash CHAR(32) NOT NULL UNIQUE, - - -- For the thumbnail we show. - public_bucket_hash CHAR(32) NOT NULL UNIQUE, + private_bucket_hash CHAR(32) NOT NULL, -- Calculated average, on a scale of 0-100 -- Null with zero ratings. @@ -86,7 +85,9 @@ CREATE TABLE tts_models ( -- INDICES -- PRIMARY KEY (id), - UNIQUE KEY (token) + UNIQUE KEY (token), + UNIQUE KEY (updatable_slug), + KEY index_private_bucket_hash (private_bucket_hash) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -97,6 +98,12 @@ CREATE TABLE w2l_templates ( -- Effective "primary key" (PUBLIC) token VARCHAR(32) NOT NULL, + -- A combination of ['username' + 'template-name'] + -- There can be public aliases for voices, eg. a voice's default model. + -- A user can change this. + -- As such, these should not be foreign keys. + updatable_slug VARCHAR(64) NOT NULL, + template_type ENUM( 'not-set', 'image', @@ -124,16 +131,13 @@ CREATE TABLE w2l_templates ( creator_ip_address VARCHAR(40) NOT NULL, -- The filename that was used at upload time. - original_filename CHAR(255) NOT NULL UNIQUE, + original_filename CHAR(255) NOT NULL, - -- The original source image/video - private_bucket_hash_original CHAR(32) NOT NULL UNIQUE, - - -- The "precomputed" faces. - private_bucket_hash_precomputed CHAR(32) NOT NULL UNIQUE, + -- The original source image/video and the "precomputed" faces + private_bucket_hash CHAR(32) NOT NULL, -- For the thumbnail we show. - public_bucket_hash CHAR(32) NOT NULL UNIQUE, + public_bucket_hash CHAR(32) NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -143,7 +147,10 @@ CREATE TABLE w2l_templates ( -- INDICES -- PRIMARY KEY (id), - UNIQUE KEY (token) + UNIQUE KEY (token), + UNIQUE KEY (updatable_slug), + KEY index_private_bucket_hash (private_bucket_hash), + KEY index_public_bucket_hash (public_bucket_hash) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -152,10 +159,12 @@ CREATE TABLE voices ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) - token VARCHAR(32) NOT NULL UNIQUE, + token VARCHAR(32) NOT NULL, -- The URL we access the voice at. - slug CHAR(16) NOT NULL UNIQUE, + -- These should be stable, but could be changed if necessary. + -- As such, these should not be foreign keys. + updatable_slug VARCHAR(64) NOT NULL, -- We can assign an exemplary model to the voice default_model_token VARCHAR(32) DEFAULT NULL, @@ -186,6 +195,8 @@ CREATE TABLE voices ( deleted_at TIMESTAMP NULL, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE KEY (token), + UNIQUE KEY (updatable_slug) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index defbdeba69..901ba4be2d 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -3,6 +3,10 @@ CREATE TABLE tts_inference_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, + -- The model to use. + -- This also determines which architecture we're using. + model_token VARCHAR(32) NOT NULL, + -- Foreign key to user -- If no user is logged in, this is null. maybe_creator_user_token CHAR(16) DEFAULT NULL, @@ -15,10 +19,6 @@ CREATE TABLE tts_inference_jobs ( -- They can choose to make them public later. is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, - -- The model to use. - -- This also determines which architecture we're using. - model_token VARCHAR(32) NOT NULL, - -- The raw, unprocessed user input. inference_text TEXT NOT NULL, @@ -44,6 +44,7 @@ CREATE TABLE tts_inference_jobs ( -- INDICES -- PRIMARY KEY (id), + KEY fk_model_token (model_token), KEY fk_maybe_creator_user_token (maybe_creator_user_token), KEY index_status (status), KEY index_creator_ip_address (creator_ip_address) @@ -53,18 +54,6 @@ CREATE TABLE tts_inference_jobs ( CREATE TABLE w2l_inference_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, - -- Foreign key to user - -- If no user is logged in, this is null. - maybe_creator_user_token VARCHAR(32) DEFAULT NULL, - - -- For abuse tracking. - -- Wide enough for IPv4/6 - creator_ip_address VARCHAR(40) NOT NULL, - - -- Users can upload their own private models. - -- They can choose to make them public later. - is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, - -- The W2L template to use -- Can be an image or video. -- This is null if we're using a custom uploaded image. @@ -79,7 +68,19 @@ CREATE TABLE w2l_inference_jobs ( -- If we're using a custom uploaded image, this will be present. maybe_image_bucket_location CHAR(16) DEFAULT NULL, - -- Jobs begin as "pending", then transition to other states. + -- Foreign key to user + -- If no user is logged in, this is null. + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- Users can upload their own private models. + -- They can choose to make them public later. + is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + + -- Jobs begin as "pending", then transition to other states. -- Pending -> Started -> Complete -- \-> Failed -> Started -> { Complete, Failed, Dead } status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending', @@ -101,9 +102,9 @@ CREATE TABLE w2l_inference_jobs ( -- INDICES -- PRIMARY KEY (id), - KEY fk_maybe_creator_user_token (maybe_creator_user_token), KEY fk_maybe_w2l_template_token (maybe_w2l_template_token), KEY fk_maybe_tts_inference_result_token (maybe_tts_inference_result_token), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), KEY index_status (status), KEY index_creator_ip_address (creator_ip_address) diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 9cd59a409f..93eb811fe8 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -27,7 +27,7 @@ CREATE TABLE tts_results ( creator_ip_address VARCHAR(40) NOT NULL, -- Where the wav, spectrogram, and etc. are located. - public_bucket_hash CHAR(32) NOT NULL UNIQUE, + public_bucket_hash CHAR(32) NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -70,7 +70,7 @@ CREATE TABLE w2l_results ( creator_ip_address VARCHAR(40) NOT NULL, -- Where the wav, spectrogram, and etc. are located. - public_bucket_hash CHAR(32) NOT NULL UNIQUE, + public_bucket_hash CHAR(32) NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, From 0cc74fe266d5f5938b56ee917f1ba7cecfeda935 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 16 May 2021 11:45:54 -0400 Subject: [PATCH 0022/1042] This configuration works for Actix + Sqlx (older Sqlx) --- Cargo.lock | 975 ++++++++++++++++++++++++++++++++-------------------- Cargo.toml | 4 +- README.md | 3 + src/main.rs | 45 ++- 4 files changed, 644 insertions(+), 383 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9fb627a16..2494fee7a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,7 +83,7 @@ dependencies = [ "actix-service", "actix-threadpool", "actix-utils 2.0.0", - "base64", + "base64 0.13.0", "bitflags", "brotli2", "bytes 0.5.6", @@ -113,7 +113,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sha-1", + "sha-1 0.9.6", "slab", "time 0.2.26", ] @@ -194,7 +194,7 @@ dependencies = [ "mio-uds", "num_cpus", "slab", - "socket2", + "socket2 0.3.19", ] [[package]] @@ -218,7 +218,7 @@ dependencies = [ "actix-server", "actix-service", "log", - "socket2", + "socket2 0.3.19", ] [[package]] @@ -311,7 +311,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "socket2", + "socket2 0.3.19", "time 0.2.26", "tinyvec", "url 2.2.2", @@ -334,23 +334,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" - -[[package]] -name = "ahash" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" -dependencies = [ - "getrandom 0.2.2", - "once_cell", - "version_check 0.9.3", -] - [[package]] name = "aho-corasick" version = "0.7.18" @@ -372,6 +355,169 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "async-channel" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9586ec52317f36de58453159d48351bc244bc24ced3effc1fce22f3d48664af6" +dependencies = [ + "async-channel", + "async-executor", + "async-io", + "async-mutex", + "blocking", + "futures-lite", + "num_cpus", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bbfd5cf2794b1e908ea8457e6c45f8f8f1f6ec5f74617bf4662623f47503c3b" +dependencies = [ + "concurrent-queue", + "fastrand", + "futures-lite", + "libc", + "log", + "once_cell", + "parking", + "polling", + "slab", + "socket2 0.4.0", + "waker-fn", + "winapi 0.3.9", +] + +[[package]] +name = "async-lock" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6a8ea61bf9947a1007c5cada31e647dbc77b103c679858150003ba697ea798b" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-mutex" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-native-tls" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9e7a929bd34c68a82d58a4de7f86fffdaf97fb2af850162a7bb19dd7269b33" +dependencies = [ + "async-std", + "native-tls", + "thiserror", + "url 2.2.2", +] + +[[package]] +name = "async-process" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f38756dd9ac84671c428afbf7c9f7495feff9ec5b0710f17100098e5b354ac" +dependencies = [ + "async-io", + "blocking", + "cfg-if 1.0.0", + "event-listener", + "futures-lite", + "libc", + "once_cell", + "signal-hook", + "winapi 0.3.9", +] + +[[package]] +name = "async-std" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f06685bad74e0570f5213741bea82158279a4103d988e57bfada11ad230341" +dependencies = [ + "async-channel", + "async-global-executor", + "async-io", + "async-lock", + "async-process", + "crossbeam-utils 0.8.4", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "num_cpus", + "once_cell", + "pin-project-lite 0.2.6", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-stream" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22068c0c19514942eefcfd4daf8976ef1aad84e61539f95cd200c35202f80af5" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f9db3b38af870bf7e5cc649167533b493928e50744e2c30ae350230b414670" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-task" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" + [[package]] name = "async-trait" version = "0.1.50" @@ -384,13 +530,10 @@ dependencies = [ ] [[package]] -name = "atoi" -version = "0.4.0" +name = "atomic-waker" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5" -dependencies = [ - "num-traits", -] +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" [[package]] name = "atty" @@ -403,12 +546,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "autocfg" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" - [[package]] name = "autocfg" version = "1.0.1" @@ -425,7 +562,7 @@ dependencies = [ "actix-http", "actix-rt 1.1.1", "actix-service", - "base64", + "base64 0.13.0", "bytes 0.5.6", "cfg-if 1.0.0", "derive_more", @@ -445,6 +582,12 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + [[package]] name = "base64" version = "0.13.0" @@ -469,13 +612,48 @@ dependencies = [ "wyz", ] +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array 0.12.4", +] + [[package]] name = "block-buffer" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array", + "generic-array 0.14.4", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "blocking" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", ] [[package]] @@ -507,18 +685,18 @@ dependencies = [ "bytes 0.5.6", ] -[[package]] -name = "build_const" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" - [[package]] name = "bumpalo" version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + [[package]] name = "byteorder" version = "1.4.3" @@ -546,6 +724,12 @@ dependencies = [ "bytes 1.0.1", ] +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + [[package]] name = "cc" version = "1.0.67" @@ -578,6 +762,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + [[package]] name = "const_fn" version = "0.4.7" @@ -607,6 +800,22 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + [[package]] name = "cpufeatures" version = "0.1.1" @@ -616,15 +825,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" -dependencies = [ - "build_const", -] - [[package]] name = "crc32fast" version = "1.2.1" @@ -635,23 +835,25 @@ dependencies = [ ] [[package]] -name = "crossbeam-channel" -version = "0.5.1" +name = "crossbeam-queue" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] -name = "crossbeam-queue" -version = "0.3.1" +name = "crossbeam-utils" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6cb3c7f5b8e51bc3ebb73a2327ad4abdbd119dc13223f14f961d2f38486756" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", + "autocfg", + "cfg-if 0.1.10", + "lazy_static", ] [[package]] @@ -660,11 +862,21 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" dependencies = [ - "autocfg 1.0.1", + "autocfg", "cfg-if 1.0.0", "lazy_static", ] +[[package]] +name = "ctor" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "derive_more" version = "0.99.13" @@ -700,13 +912,22 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.4", +] + [[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array", + "generic-array 0.14.4", ] [[package]] @@ -772,6 +993,27 @@ dependencies = [ "termcolor", ] +[[package]] +name = "event-listener" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "fastrand" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77b705829d1e87f762c2df6da140b26af5839e1033aa84aa5f56bb688e4e1bdb" +dependencies = [ + "instant", +] + [[package]] name = "flate2" version = "1.0.20" @@ -790,6 +1032,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -870,13 +1127,28 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" +[[package]] +name = "futures-lite" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4481d0cd0de1d204a4fa55e7d45f07b1d958abcb06714b3446438e2eff695fb" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite 0.2.6", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" dependencies = [ - "autocfg 1.0.1", + "autocfg", "proc-macro-hack", "proc-macro2", "quote", @@ -901,7 +1173,7 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" dependencies = [ - "autocfg 1.0.1", + "autocfg", "futures-channel", "futures-core", "futures-io", @@ -925,6 +1197,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + [[package]] name = "generic-array" version = "0.14.4" @@ -957,6 +1238,19 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] +[[package]] +name = "gloo-timers" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "h2" version = "0.2.7" @@ -982,18 +1276,6 @@ name = "hashbrown" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" -dependencies = [ - "ahash 0.4.7", -] - -[[package]] -name = "hashlink" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" -dependencies = [ - "hashbrown", -] [[package]] name = "heck" @@ -1084,7 +1366,7 @@ version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ - "autocfg 1.0.1", + "autocfg", "hashbrown", ] @@ -1112,7 +1394,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" dependencies = [ - "socket2", + "socket2 0.3.19", "widestring", "winapi 0.3.9", "winreg", @@ -1143,6 +1425,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "language-tags" version = "0.2.2" @@ -1154,9 +1445,6 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -dependencies = [ - "spin", -] [[package]] name = "lexical-core" @@ -1177,12 +1465,6 @@ version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" -[[package]] -name = "libm" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" - [[package]] name = "linked-hash-map" version = "0.5.4" @@ -1211,6 +1493,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ "cfg-if 1.0.0", + "value-bag", ] [[package]] @@ -1240,6 +1523,12 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + [[package]] name = "memchr" version = "2.4.0" @@ -1269,7 +1558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ "adler", - "autocfg 1.0.1", + "autocfg", ] [[package]] @@ -1346,6 +1635,24 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "net2" version = "0.2.37" @@ -1391,62 +1698,22 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.3.2" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d0a3d5e207573f948a9e5376662aa743a2ea13f7c50a554d7af443a73fbfeba" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-integer", "num-traits", ] -[[package]] -name = "num-bigint" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d047c1062aa51e256408c560894e5251f08925980e53cf1aa5bd00eec6512" -dependencies = [ - "autocfg 1.0.1", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-bigint-dig" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4547ee5541c18742396ae2c895d0717d0f886d8823b8399cdaf7b07d63ad0480" -dependencies = [ - "autocfg 0.1.7", - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand 0.8.3", - "smallvec", - "zeroize", -] - [[package]] name = "num-integer" version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ - "autocfg 1.0.1", - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" -dependencies = [ - "autocfg 1.0.1", - "num-integer", + "autocfg", "num-traits", ] @@ -1456,8 +1723,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ - "autocfg 1.0.1", - "libm", + "autocfg", ] [[package]] @@ -1476,12 +1742,57 @@ version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + [[package]] name = "opaque-debug" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.11.1" @@ -1507,17 +1818,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "pem" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb" -dependencies = [ - "base64", - "once_cell", - "regex", -] - [[package]] name = "percent-encoding" version = "1.0.1" @@ -1594,6 +1894,19 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +[[package]] +name = "polling" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc12d774e799ee9ebae13f4076ca003b40d18a11ac0f3641e6f899618580b7b" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "log", + "wepoll-sys", + "winapi 0.3.9", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1749,6 +2062,15 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -1759,41 +2081,6 @@ dependencies = [ "quick-error", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted", - "web-sys", - "winapi 0.3.9", -] - -[[package]] -name = "rsa" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ef841a26fc5d040ced0417c6c6a64ee851f42489df11cdf0218e545b6f8d28" -dependencies = [ - "byteorder", - "digest", - "lazy_static", - "num-bigint-dig", - "num-integer", - "num-iter", - "num-traits", - "pem", - "rand 0.8.3", - "simple_asn1", - "subtle", - "zeroize", -] - [[package]] name = "rustc_version" version = "0.2.3" @@ -1803,25 +2090,22 @@ dependencies = [ "semver", ] -[[package]] -name = "rustls" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" -dependencies = [ - "base64", - "log", - "ring", - "sct", - "webpki", -] - [[package]] name = "ryu" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -1829,13 +2113,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] -name = "sct" -version = "0.6.1" +name = "security-framework" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" dependencies = [ - "ring", - "untrusted", + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] @@ -1896,17 +2193,29 @@ dependencies = [ "serde", ] +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + [[package]] name = "sha-1" version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if 1.0.0", "cpufeatures", - "digest", - "opaque-debug", + "digest 0.9.0", + "opaque-debug 0.3.0", ] [[package]] @@ -1917,15 +2226,24 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.9.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" +checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" dependencies = [ - "block-buffer", - "cfg-if 1.0.0", - "cpufeatures", - "digest", - "opaque-debug", + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + +[[package]] +name = "signal-hook" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef33d6d0cd06e0840fba9985aab098c147e67e05cee14d412d3345ed14ff30ac" +dependencies = [ + "libc", + "signal-hook-registry", ] [[package]] @@ -1937,18 +2255,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simple_asn1" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0e9076e5242ff5a58e854cb478ea9caebce01088f86d3d9c6ad336b7655263" -dependencies = [ - "chrono", - "num-bigint 0.4.0", - "num-traits", - "thiserror", -] - [[package]] name = "slab" version = "0.4.3" @@ -1973,10 +2279,14 @@ dependencies = [ ] [[package]] -name = "spin" -version = "0.5.2" +name = "socket2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +dependencies = [ + "libc", + "winapi 0.3.9", +] [[package]] name = "sqlformat" @@ -1993,9 +2303,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.5.2" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d582b9bc04ec6c03084196efc42c2226b018e9941f03ee62bd88921d500917c0" +checksum = "8974cacd80085fbe49e778708d660dec6fb351604dc34c3905b26efb2803b038" dependencies = [ "sqlx-core", "sqlx-macros", @@ -2003,83 +2313,53 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.5.2" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de52d1d473cebb2abb79c886ef6a8023e965e34c0676a99cfeac2cc7f0fde4c1" +checksum = "88ac5a436f941c42eac509471a730df5c3c58e1450e68cd39afedbd948206273" dependencies = [ - "ahash 0.7.2", - "atoi", - "base64", + "async-native-tls", + "async-std", + "async-stream", + "base64 0.12.3", "bitflags", "byteorder", - "bytes 1.0.1", - "crc", - "crossbeam-channel", "crossbeam-queue", - "crossbeam-utils", - "digest", - "either", + "crossbeam-utils 0.7.2", + "digest 0.8.1", "futures-channel", "futures-core", "futures-util", - "generic-array", - "hashlink", + "generic-array 0.12.4", "hex", - "itoa", "libc", "log", "memchr", - "num-bigint 0.3.2", - "once_cell", - "parking_lot", + "num-bigint", "percent-encoding 2.1.0", - "rand 0.8.3", - "rsa", - "rustls", - "sha-1", + "rand 0.7.3", + "sha-1 0.8.2", "sha2", - "smallvec", "sqlformat", - "sqlx-rt", - "stringprep", - "thiserror", - "tokio-stream", "url 2.2.2", - "webpki", - "webpki-roots", - "whoami", ] [[package]] name = "sqlx-macros" -version = "0.5.2" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a40f0be97e704d3fbf059e7e3333c3735639146a72d586c5534c70e79da88a4" +checksum = "de2ae78b783af5922d811b14665a5a3755e531c3087bb805cf24cf71f15e6780" dependencies = [ + "async-std", "dotenv", - "either", "futures", "heck", "proc-macro2", "quote", - "sha2", "sqlx-core", - "sqlx-rt", "syn", "url 2.2.2", ] -[[package]] -name = "sqlx-rt" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6ae97ab05063ed515cdc23d90253213aa24dda0a288c5ec079af3d10f9771bc" -dependencies = [ - "once_cell", - "tokio 1.5.0", - "tokio-rustls", -] - [[package]] name = "standback" version = "0.2.17" @@ -2155,7 +2435,7 @@ dependencies = [ "actix-utils 3.0.0", "actix-web", "anyhow", - "base64", + "base64 0.13.0", "chrono", "diesel", "easyenv", @@ -2170,22 +2450,6 @@ dependencies = [ "toml", ] -[[package]] -name = "stringprep" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "subtle" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" - [[package]] name = "syn" version = "1.0.72" @@ -2197,24 +2461,26 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "synstructure" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", -] - [[package]] name = "tap" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand 0.8.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.9", +] + [[package]] name = "termcolor" version = "1.1.2" @@ -2343,12 +2609,9 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" dependencies = [ - "autocfg 1.0.1", - "bytes 1.0.1", + "autocfg", "libc", - "memchr", "mio 0.7.11", - "num_cpus", "once_cell", "parking_lot", "pin-project-lite 0.2.6", @@ -2356,28 +2619,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "tokio-rustls" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" -dependencies = [ - "rustls", - "tokio 1.5.0", - "webpki", -] - -[[package]] -name = "tokio-stream" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e177a5d8c3bf36de9ebe6d58537d8879e964332f93fb3339e43f618c81361af0" -dependencies = [ - "futures-core", - "pin-project-lite 0.2.6", - "tokio 1.5.0", -] - [[package]] name = "tokio-util" version = "0.3.1" @@ -2522,12 +2763,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "url" version = "1.7.2" @@ -2583,6 +2818,15 @@ dependencies = [ "v_escape", ] +[[package]] +name = "value-bag" +version = "1.0.0-alpha.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b676010e055c99033117c2343b33a40a30b91fecd6c49055ac9cd2d6c305ab1" +dependencies = [ + "ctor", +] + [[package]] name = "vcpkg" version = "0.2.12" @@ -2601,6 +2845,12 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -2638,6 +2888,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.74" @@ -2678,32 +2940,12 @@ dependencies = [ ] [[package]] -name = "webpki" -version = "0.21.4" +name = "wepoll-sys" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff" dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "webpki-roots" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" -dependencies = [ - "webpki", -] - -[[package]] -name = "whoami" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4abacf325c958dfeaf1046931d37f2a901b6dfe0968ee965a29e94c6766b2af6" -dependencies = [ - "wasm-bindgen", - "web-sys", + "cc", ] [[package]] @@ -2779,24 +3021,3 @@ name = "wyz" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" - -[[package]] -name = "zeroize" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2c1e130bebaeab2f23886bf9acbaca14b092408c452543c857f66399cd6dab1" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] diff --git a/Cargo.toml b/Cargo.toml index 02095871e7..dea21647af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,9 @@ log = "0.4.14" rand = "0.8.3" # Database -sqlx = { version = "0.5.0", features = ["mysql", "runtime-tokio-rustls"] } +#sqlx = { version = "0.5.0", features = ["mysql", "runtime-tokio-rustls", "chrono"] } +sqlx = { version = "0.3", features = [ "mysql" ] } +#sqlx = { version = "0.5.0", features = ["mysql", "runtime-actix-rustls", "chrono"] } #diesel = { version = "1.4.4", features = ["mysql", "r2d2", "chrono"] } #diesel-derive-enum = { version = "1.0.0", features = ["mysql"] } diff --git a/README.md b/README.md index bb34caa32a..81578f0776 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,6 @@ TODO * Install indexes on all foreign keys * Make sure unique indexes are correct + +* Tests: https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr + diff --git a/src/main.rs b/src/main.rs index 9cfb3c23a0..2f797874cc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,13 +7,21 @@ use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; use std::sync::Arc; use crate::server_state::{ServerState, EnvConfig}; +use sqlx::MySqlPool; +//use sqlx::mysql::MySqlPoolOptions; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; pub type AnyhowResult = anyhow::Result; -pub fn main() -> AnyhowResult<()> { +//pub async fn main() -> AnyhowResult<()> { +//#[tokio::main] +//#[async_std::main] +//#[actix_web::main] +//pub async fn main() -> std::io::Result<()> { +#[actix_web::main] +async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); let server_hostname = hostname::get() @@ -29,17 +37,42 @@ pub fn main() -> AnyhowResult<()> { hostname: server_hostname, }; - serve(server_state)?; + serve(server_state) + .await?; Ok(()) } -#[actix_web::main] -pub async fn serve(server_state: ServerState) -> std::io::Result<()> +//#[actix_web::main] +pub async fn serve(server_state: ServerState) -> AnyhowResult<()> { let bind_address = server_state.env_config.bind_address.clone(); let num_workers = server_state.env_config.num_workers.clone(); let hostname = server_state.hostname.clone(); + let db_connection_string = "mysql://root:root@localhost/storyteller"; + + //let pool = MySqlPool::connect(db_connection_string).await?; + /*let pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(db_connection_string) + .await?;*/ + let pool = MySqlPool::new(db_connection_string).await?; + + /*let row: (i64,) = sqlx::query_as("SELECT $1") + .bind(150_i64) + .fetch_one(&pool).await?;*/ + + let recs = sqlx::query!( + r#" + SELECT id, username + FROM users + ORDER BY id + "# + ) + .fetch_all(&pool) + .await?; + + let server_state_arc = web::Data::new(Arc::new(server_state)); info!("Starting HTTP service."); @@ -117,5 +150,7 @@ pub async fn serve(server_state: ServerState) -> std::io::Result<()> .bind(bind_address)? .workers(num_workers) .run() - .await + .await?; + + Ok(()) } From 32a9949559d37c3d088ca8bda0c0cd350f1727ea Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 16 May 2021 11:57:29 -0400 Subject: [PATCH 0023/1042] this version of actix/sqlx works, but uses diff runtime? --- Cargo.lock | 672 +++++++++++++++++++++++++++++----------------------- Cargo.toml | 9 +- README.md | 7 +- src/main.rs | 23 +- 4 files changed, 398 insertions(+), 313 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2494fee7a3..2fb3d56680 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,7 +83,7 @@ dependencies = [ "actix-service", "actix-threadpool", "actix-utils 2.0.0", - "base64 0.13.0", + "base64", "bitflags", "brotli2", "bytes 0.5.6", @@ -113,7 +113,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sha-1 0.9.6", + "sha-1", "slab", "time 0.2.26", ] @@ -334,6 +334,23 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" + +[[package]] +name = "ahash" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" +dependencies = [ + "getrandom 0.2.2", + "once_cell", + "version_check 0.9.3", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -434,18 +451,6 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-native-tls" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9e7a929bd34c68a82d58a4de7f86fffdaf97fb2af850162a7bb19dd7269b33" -dependencies = [ - "async-std", - "native-tls", - "thiserror", - "url 2.2.2", -] - [[package]] name = "async-process" version = "1.1.0" @@ -463,6 +468,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "async-rustls" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c86f33abd5a4f3e2d6d9251a9e0c6a7e52eb1113caf893dae8429bf4a53f378" +dependencies = [ + "futures-lite", + "rustls", + "webpki", +] + [[package]] name = "async-std" version = "1.9.0" @@ -474,7 +490,7 @@ dependencies = [ "async-io", "async-lock", "async-process", - "crossbeam-utils 0.8.4", + "crossbeam-utils", "futures-channel", "futures-core", "futures-io", @@ -491,27 +507,6 @@ dependencies = [ "wasm-bindgen-futures", ] -[[package]] -name = "async-stream" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22068c0c19514942eefcfd4daf8976ef1aad84e61539f95cd200c35202f80af5" -dependencies = [ - "async-stream-impl", - "futures-core", -] - -[[package]] -name = "async-stream-impl" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f9db3b38af870bf7e5cc649167533b493928e50744e2c30ae350230b414670" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "async-task" version = "4.0.3" @@ -529,6 +524,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atoi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.0.0" @@ -546,6 +550,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + [[package]] name = "autocfg" version = "1.0.1" @@ -562,7 +572,7 @@ dependencies = [ "actix-http", "actix-rt 1.1.1", "actix-service", - "base64 0.13.0", + "base64", "bytes 0.5.6", "cfg-if 1.0.0", "derive_more", @@ -582,12 +592,6 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" -[[package]] -name = "base64" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" - [[package]] name = "base64" version = "0.13.0" @@ -612,34 +616,13 @@ dependencies = [ "wyz", ] -[[package]] -name = "block-buffer" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -dependencies = [ - "block-padding", - "byte-tools", - "byteorder", - "generic-array 0.12.4", -] - [[package]] name = "block-buffer" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array 0.14.4", -] - -[[package]] -name = "block-padding" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -dependencies = [ - "byte-tools", + "generic-array", ] [[package]] @@ -685,18 +668,18 @@ dependencies = [ "bytes 0.5.6", ] +[[package]] +name = "build_const" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" + [[package]] name = "bumpalo" version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" - [[package]] name = "byteorder" version = "1.4.3" @@ -800,22 +783,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" -[[package]] -name = "core-foundation" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" - [[package]] name = "cpufeatures" version = "0.1.1" @@ -825,6 +792,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +dependencies = [ + "build_const", +] + [[package]] name = "crc32fast" version = "1.2.1" @@ -835,25 +811,23 @@ dependencies = [ ] [[package]] -name = "crossbeam-queue" -version = "0.2.3" +name = "crossbeam-channel" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "maybe-uninit", + "cfg-if 1.0.0", + "crossbeam-utils", ] [[package]] -name = "crossbeam-utils" -version = "0.7.2" +name = "crossbeam-queue" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +checksum = "0f6cb3c7f5b8e51bc3ebb73a2327ad4abdbd119dc13223f14f961d2f38486756" dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", + "cfg-if 1.0.0", + "crossbeam-utils", ] [[package]] @@ -862,7 +836,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" dependencies = [ - "autocfg", + "autocfg 1.0.1", "cfg-if 1.0.0", "lazy_static", ] @@ -912,22 +886,13 @@ dependencies = [ "syn", ] -[[package]] -name = "digest" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -dependencies = [ - "generic-array 0.12.4", -] - [[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.4", + "generic-array", ] [[package]] @@ -999,12 +964,6 @@ version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" - [[package]] name = "fastrand" version = "1.4.1" @@ -1032,21 +991,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.0.1" @@ -1148,7 +1092,7 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" dependencies = [ - "autocfg", + "autocfg 1.0.1", "proc-macro-hack", "proc-macro2", "quote", @@ -1173,7 +1117,7 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" dependencies = [ - "autocfg", + "autocfg 1.0.1", "futures-channel", "futures-core", "futures-io", @@ -1197,15 +1141,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "generic-array" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" -dependencies = [ - "typenum", -] - [[package]] name = "generic-array" version = "0.14.4" @@ -1276,6 +1211,18 @@ name = "hashbrown" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash 0.4.7", +] + +[[package]] +name = "hashlink" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" +dependencies = [ + "hashbrown", +] [[package]] name = "heck" @@ -1366,7 +1313,7 @@ version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ - "autocfg", + "autocfg 1.0.1", "hashbrown", ] @@ -1445,6 +1392,9 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin", +] [[package]] name = "lexical-core" @@ -1465,6 +1415,12 @@ version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" +[[package]] +name = "libm" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" + [[package]] name = "linked-hash-map" version = "0.5.4" @@ -1523,12 +1479,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "memchr" version = "2.4.0" @@ -1558,7 +1508,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ "adler", - "autocfg", + "autocfg 1.0.1", ] [[package]] @@ -1635,24 +1585,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "native-tls" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "net2" version = "0.2.37" @@ -1698,22 +1630,62 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.2.6" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +checksum = "7d0a3d5e207573f948a9e5376662aa743a2ea13f7c50a554d7af443a73fbfeba" dependencies = [ - "autocfg", + "autocfg 1.0.1", "num-integer", "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e0d047c1062aa51e256408c560894e5251f08925980e53cf1aa5bd00eec6512" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4547ee5541c18742396ae2c895d0717d0f886d8823b8399cdaf7b07d63ad0480" +dependencies = [ + "autocfg 0.1.7", + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.3", + "smallvec", + "zeroize", +] + [[package]] name = "num-integer" version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ - "autocfg", + "autocfg 1.0.1", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg 1.0.1", + "num-integer", "num-traits", ] @@ -1723,7 +1695,8 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ - "autocfg", + "autocfg 1.0.1", + "libm", ] [[package]] @@ -1742,51 +1715,12 @@ version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - [[package]] name = "opaque-debug" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" -[[package]] -name = "openssl" -version = "0.10.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8" -dependencies = [ - "bitflags", - "cfg-if 1.0.0", - "foreign-types", - "libc", - "once_cell", - "openssl-sys", -] - -[[package]] -name = "openssl-probe" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" - -[[package]] -name = "openssl-sys" -version = "0.9.63" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98" -dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "parking" version = "2.0.0" @@ -1818,6 +1752,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "pem" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb" +dependencies = [ + "base64", + "once_cell", + "regex", +] + [[package]] name = "percent-encoding" version = "1.0.1" @@ -2062,15 +2007,6 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi 0.3.9", -] - [[package]] name = "resolv-conf" version = "0.7.0" @@ -2081,6 +2017,41 @@ dependencies = [ "quick-error", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + +[[package]] +name = "rsa" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ef841a26fc5d040ced0417c6c6a64ee851f42489df11cdf0218e545b6f8d28" +dependencies = [ + "byteorder", + "digest", + "lazy_static", + "num-bigint-dig", + "num-integer", + "num-iter", + "num-traits", + "pem", + "rand 0.8.3", + "simple_asn1", + "subtle", + "zeroize", +] + [[package]] name = "rustc_version" version = "0.2.3" @@ -2090,22 +2061,25 @@ dependencies = [ "semver", ] +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64", + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "ryu" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" -[[package]] -name = "schannel" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" -dependencies = [ - "lazy_static", - "winapi 0.3.9", -] - [[package]] name = "scopeguard" version = "1.1.0" @@ -2113,26 +2087,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] -name = "security-framework" -version = "2.2.0" +name = "sct" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" -dependencies = [ - "core-foundation-sys", - "libc", + "ring", + "untrusted", ] [[package]] @@ -2193,29 +2154,17 @@ dependencies = [ "serde", ] -[[package]] -name = "sha-1" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" -dependencies = [ - "block-buffer 0.7.3", - "digest 0.8.1", - "fake-simd", - "opaque-debug 0.2.3", -] - [[package]] name = "sha-1" version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c4cfa741c5832d0ef7fab46cabed29c2aae926db0b11bb2069edd8db5e64e16" dependencies = [ - "block-buffer 0.9.0", + "block-buffer", "cfg-if 1.0.0", "cpufeatures", - "digest 0.9.0", - "opaque-debug 0.3.0", + "digest", + "opaque-debug", ] [[package]] @@ -2226,14 +2175,15 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.8.2" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" +checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" dependencies = [ - "block-buffer 0.7.3", - "digest 0.8.1", - "fake-simd", - "opaque-debug 0.2.3", + "block-buffer", + "cfg-if 1.0.0", + "cpufeatures", + "digest", + "opaque-debug", ] [[package]] @@ -2255,6 +2205,18 @@ dependencies = [ "libc", ] +[[package]] +name = "simple_asn1" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e0e9076e5242ff5a58e854cb478ea9caebce01088f86d3d9c6ad336b7655263" +dependencies = [ + "chrono", + "num-bigint 0.4.0", + "num-traits", + "thiserror", +] + [[package]] name = "slab" version = "0.4.3" @@ -2288,6 +2250,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "sqlformat" version = "0.1.6" @@ -2303,9 +2271,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.3.5" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8974cacd80085fbe49e778708d660dec6fb351604dc34c3905b26efb2803b038" +checksum = "d582b9bc04ec6c03084196efc42c2226b018e9941f03ee62bd88921d500917c0" dependencies = [ "sqlx-core", "sqlx-macros", @@ -2313,53 +2281,81 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.3.5" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ac5a436f941c42eac509471a730df5c3c58e1450e68cd39afedbd948206273" +checksum = "de52d1d473cebb2abb79c886ef6a8023e965e34c0676a99cfeac2cc7f0fde4c1" dependencies = [ - "async-native-tls", - "async-std", - "async-stream", - "base64 0.12.3", + "ahash 0.7.2", + "atoi", + "base64", "bitflags", "byteorder", + "bytes 1.0.1", + "crc", + "crossbeam-channel", "crossbeam-queue", - "crossbeam-utils 0.7.2", - "digest 0.8.1", + "crossbeam-utils", + "digest", + "either", "futures-channel", "futures-core", "futures-util", - "generic-array 0.12.4", + "generic-array", + "hashlink", "hex", + "itoa", "libc", "log", "memchr", - "num-bigint", + "num-bigint 0.3.2", + "once_cell", + "parking_lot", "percent-encoding 2.1.0", - "rand 0.7.3", - "sha-1 0.8.2", + "rand 0.8.3", + "rsa", + "rustls", + "sha-1", "sha2", + "smallvec", "sqlformat", + "sqlx-rt", + "stringprep", + "thiserror", "url 2.2.2", + "webpki", + "webpki-roots", + "whoami", ] [[package]] name = "sqlx-macros" -version = "0.3.5" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de2ae78b783af5922d811b14665a5a3755e531c3087bb805cf24cf71f15e6780" +checksum = "1a40f0be97e704d3fbf059e7e3333c3735639146a72d586c5534c70e79da88a4" dependencies = [ - "async-std", "dotenv", + "either", "futures", "heck", "proc-macro2", "quote", + "sha2", "sqlx-core", + "sqlx-rt", "syn", "url 2.2.2", ] +[[package]] +name = "sqlx-rt" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ae97ab05063ed515cdc23d90253213aa24dda0a288c5ec079af3d10f9771bc" +dependencies = [ + "async-rustls", + "async-std", +] + [[package]] name = "standback" version = "0.2.17" @@ -2435,7 +2431,7 @@ dependencies = [ "actix-utils 3.0.0", "actix-web", "anyhow", - "base64 0.13.0", + "base64", "chrono", "diesel", "easyenv", @@ -2450,6 +2446,22 @@ dependencies = [ "toml", ] +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "subtle" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" + [[package]] name = "syn" version = "1.0.72" @@ -2461,26 +2473,24 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + [[package]] name = "tap" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "tempfile" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "rand 0.8.3", - "redox_syscall", - "remove_dir_all", - "winapi 0.3.9", -] - [[package]] name = "termcolor" version = "1.1.2" @@ -2609,7 +2619,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" dependencies = [ - "autocfg", + "autocfg 1.0.1", "libc", "mio 0.7.11", "once_cell", @@ -2763,6 +2773,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "1.7.2" @@ -2939,6 +2955,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" +dependencies = [ + "webpki", +] + [[package]] name = "wepoll-sys" version = "3.0.1" @@ -2948,6 +2983,16 @@ dependencies = [ "cc", ] +[[package]] +name = "whoami" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4abacf325c958dfeaf1046931d37f2a901b6dfe0968ee965a29e94c6766b2af6" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "widestring" version = "0.4.3" @@ -3021,3 +3066,24 @@ name = "wyz" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" + +[[package]] +name = "zeroize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c1e130bebaeab2f23886bf9acbaca14b092408c452543c857f66399cd6dab1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] diff --git a/Cargo.toml b/Cargo.toml index dea21647af..e91779a172 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,14 @@ rand = "0.8.3" # Database #sqlx = { version = "0.5.0", features = ["mysql", "runtime-tokio-rustls", "chrono"] } -sqlx = { version = "0.3", features = [ "mysql" ] } + +#error: one of the features ['runtime-actix-native-tls', 'runtime-async-std-native-tls', 'runtime-tokio-native-tls', 'runtime-actix-rustls', 'runtime-async-std-rustls', 'runtime-tokio-rustls'] must be enabled + +#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls" ] } +sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-rustls" ] } + + + #sqlx = { version = "0.5.0", features = ["mysql", "runtime-actix-rustls", "chrono"] } #diesel = { version = "1.4.4", features = ["mysql", "r2d2", "chrono"] } #diesel-derive-enum = { version = "1.0.0", features = ["mysql"] } diff --git a/README.md b/README.md index 81578f0776..b3ad3b77bc 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,11 @@ TODO * Install indexes on all foreign keys * Make sure unique indexes are correct +Notes / TODOs: -* Tests: https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr +* Examples for good Actix+Sqlx Tests: + https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr + +* Actix/sqlx runtime compat: + https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734 diff --git a/src/main.rs b/src/main.rs index 2f797874cc..e8c40fa490 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,7 @@ use actix_web::{HttpServer, web, HttpResponse, App}; use std::sync::Arc; use crate::server_state::{ServerState, EnvConfig}; use sqlx::MySqlPool; -//use sqlx::mysql::MySqlPoolOptions; +use sqlx::mysql::MySqlPoolOptions; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -52,16 +52,22 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> let db_connection_string = "mysql://root:root@localhost/storyteller"; //let pool = MySqlPool::connect(db_connection_string).await?; - /*let pool = MySqlPoolOptions::new() + + + + let pool = MySqlPoolOptions::new() .max_connections(5) .connect(db_connection_string) - .await?;*/ + .await?; + let row: (i64,) = sqlx::query_as("SELECT 1") + //.bind(150_i64) + .fetch_one(&pool).await?; + + println!("Result: {:?}", row); + + /* + // Working example in 0.3 sqlx let pool = MySqlPool::new(db_connection_string).await?; - - /*let row: (i64,) = sqlx::query_as("SELECT $1") - .bind(150_i64) - .fetch_one(&pool).await?;*/ - let recs = sqlx::query!( r#" SELECT id, username @@ -71,6 +77,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> ) .fetch_all(&pool) .await?; + */ let server_state_arc = web::Data::new(Arc::new(server_state)); From 2ad5328c9877c4cbd237ee4f9854e5dac8ce2c9c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 16 May 2021 12:07:30 -0400 Subject: [PATCH 0024/1042] this inserts into the database! --- migrations/2021-05-15-071534_users/up.sql | 9 +++--- src/main.rs | 38 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 0075446109..2f295ebd9d 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -8,13 +8,13 @@ CREATE TABLE users ( -- Effective "primary key" token VARCHAR(32) NOT NULL, + -- Username is a lookup key; display_name allows the user to add custom case. + username VARCHAR(20) NOT NULL, + display_name VARCHAR(20) NOT NULL, + email_address VARCHAR(255) NOT NULL, email_confirmed BOOLEAN NOT NULL DEFAULT false, - -- Username is a lookup key; display_name allows the user to add custom case. - username VARCHAR(20) NOT NULL UNIQUE, - display_name VARCHAR(20) NOT NULL, - -- The profile of the user in markdown (user editable). profile_markdown TEXT NOT NULL, @@ -85,6 +85,7 @@ CREATE TABLE users ( -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (token), + UNIQUE KEY (username), UNIQUE KEY (email_address), KEY fk_user_role_slug (user_role_slug) diff --git a/src/main.rs b/src/main.rs index e8c40fa490..441af8f0ca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -79,6 +79,10 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .await?; */ + //create_user(&pool).await?; + let id = create_badge(&pool, "slug", "this is title", "decr", "img").await?; + println!("Created id: {}", id); + let server_state_arc = web::Data::new(Arc::new(server_state)); @@ -161,3 +165,37 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> Ok(()) } + +pub async fn create_badge(pool: &MySqlPool, slug: &str, title: &str, description: &str, image_url: &str) -> AnyhowResult { + // Insert the task, then obtain the ID of this row + let todo_id = sqlx::query!( + r#" +INSERT INTO badges ( slug, title, description, image_url ) +VALUES ( ?, ?, ?, ? ) + "#, + slug, + title, + description, + image_url, + ) + .execute(pool) + .await? + .last_insert_id(); + + Ok(todo_id) +} + +pub async fn create_user(pool: &MySqlPool) -> AnyhowResult<()> { + let mut tx = pool.begin().await?; + let todo = sqlx::query("INSERT INTO badges (slug, title, description, image_url) VALUES ($1, $2, $3, $4)") + .bind("foo") + .bind("bar") + .bind("baz") + .bind("bin") + .fetch_one(&mut tx) + .await?; + + tx.commit().await?; + + Ok(()) +} From cb3aa648e5f531b04042d8bdde6dbd0745ff1adc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 16 May 2021 12:15:44 -0400 Subject: [PATCH 0025/1042] modules for model queries --- src/main.rs | 40 ++++++++++++++++++++-------------------- src/queries/badges.rs | 29 +++++++++++++++++++++++++++++ src/queries/mod.rs | 2 ++ src/queries/users.rs | 0 4 files changed, 51 insertions(+), 20 deletions(-) create mode 100644 src/queries/badges.rs create mode 100644 src/queries/mod.rs create mode 100644 src/queries/users.rs diff --git a/src/main.rs b/src/main.rs index 441af8f0ca..854fce23f8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ pub mod server_state; +pub mod queries; use actix_cors::Cors; use actix_http::http; @@ -9,6 +10,7 @@ use std::sync::Arc; use crate::server_state::{ServerState, EnvConfig}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; +use crate::queries::badges::NewBadge; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -80,9 +82,26 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> */ //create_user(&pool).await?; - let id = create_badge(&pool, "slug", "this is title", "decr", "img").await?; + + let badge = NewBadge { + slug: "slug2".to_string(), + title: "title".to_string(), + description: "description".to_string(), + image_url: "url".to_string() + }; + + let id = badge.insert(&pool).await?; println!("Created id: {}", id); + let badge = NewBadge { + slug: "slug3".to_string(), + title: "title".to_string(), + description: "description".to_string(), + image_url: "url".to_string() + }; + + let id = badge.insert(&pool).await?; + println!("Created id: {}", id); let server_state_arc = web::Data::new(Arc::new(server_state)); @@ -166,25 +185,6 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> Ok(()) } -pub async fn create_badge(pool: &MySqlPool, slug: &str, title: &str, description: &str, image_url: &str) -> AnyhowResult { - // Insert the task, then obtain the ID of this row - let todo_id = sqlx::query!( - r#" -INSERT INTO badges ( slug, title, description, image_url ) -VALUES ( ?, ?, ?, ? ) - "#, - slug, - title, - description, - image_url, - ) - .execute(pool) - .await? - .last_insert_id(); - - Ok(todo_id) -} - pub async fn create_user(pool: &MySqlPool) -> AnyhowResult<()> { let mut tx = pool.begin().await?; let todo = sqlx::query("INSERT INTO badges (slug, title, description, image_url) VALUES ($1, $2, $3, $4)") diff --git a/src/queries/badges.rs b/src/queries/badges.rs new file mode 100644 index 0000000000..9425f0069e --- /dev/null +++ b/src/queries/badges.rs @@ -0,0 +1,29 @@ +use sqlx::MySqlPool; +use crate::AnyhowResult; + +pub struct NewBadge { + pub slug: String, + pub title: String, + pub description: String, + pub image_url: String, +} + +impl NewBadge { + pub async fn insert(&self, pool: &MySqlPool) -> AnyhowResult { + let record_id = sqlx::query!( + r#" +INSERT INTO badges ( slug, title, description, image_url ) +VALUES ( ?, ?, ?, ? ) + "#, + self.slug, + self.title, + self.description, + self.image_url, + ) + .execute(pool) + .await? + .last_insert_id(); + + Ok(record_id) + } +} diff --git a/src/queries/mod.rs b/src/queries/mod.rs new file mode 100644 index 0000000000..2e6f0082b7 --- /dev/null +++ b/src/queries/mod.rs @@ -0,0 +1,2 @@ +pub mod users; +pub mod badges; \ No newline at end of file diff --git a/src/queries/users.rs b/src/queries/users.rs new file mode 100644 index 0000000000..e69de29bb2 From 2559b735507d9fd1ee9f59f34a4a47b31bf955a5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 16 May 2021 20:37:57 -0400 Subject: [PATCH 0026/1042] singluar endpoint --- Cargo.toml | 1 - README.md | 20 ++++++++ src/endpoints/mod.rs | 1 + src/endpoints/users/create_account.rs | 40 ++++++++++++++++ src/endpoints/users/login.rs | 0 src/endpoints/users/logout.rs | 0 src/endpoints/users/mod.rs | 3 ++ src/main.rs | 69 +++++++-------------------- src/server_state.rs | 3 ++ 9 files changed, 84 insertions(+), 53 deletions(-) create mode 100644 src/endpoints/mod.rs create mode 100644 src/endpoints/users/create_account.rs create mode 100644 src/endpoints/users/login.rs create mode 100644 src/endpoints/users/logout.rs create mode 100644 src/endpoints/users/mod.rs diff --git a/Cargo.toml b/Cargo.toml index e91779a172..1ef8cd903e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,6 @@ rand = "0.8.3" sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-rustls" ] } - #sqlx = { version = "0.5.0", features = ["mysql", "runtime-actix-rustls", "chrono"] } #diesel = { version = "1.4.4", features = ["mysql", "r2d2", "chrono"] } #diesel-derive-enum = { version = "1.0.0", features = ["mysql"] } diff --git a/README.md b/README.md index b3ad3b77bc..417c2a92ed 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,26 @@ To undo migrations at the current step: ```diesel migration redo``` +Actix notes +----------- + +json request +``` +async fn handler(request: web::Json) -> impl Responder { "whatever" } +``` + +form-multipart request +``` +async fn handler(request: web::Form) -> impl Responder { "whatever" } +``` + +route parameters +``` +#[get("/{name}")] +async fn hello(name: web::Path) -> impl Responder { format!("Hi {}", name) } +``` + + TODO ---- diff --git a/src/endpoints/mod.rs b/src/endpoints/mod.rs new file mode 100644 index 0000000000..e3146d353b --- /dev/null +++ b/src/endpoints/mod.rs @@ -0,0 +1 @@ +pub mod users; \ No newline at end of file diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs new file mode 100644 index 0000000000..46ef88f2e1 --- /dev/null +++ b/src/endpoints/users/create_account.rs @@ -0,0 +1,40 @@ +use actix_web::{Responder, web}; + +#[derive(Deserialize)] +pub struct CreateAccountRequest { + pub username: String, + pub password: String, + pub password_verification: String, +} + +#[derive(Serialize)] +pub struct CreateAccountResponse { + pub success: bool, + +} + +//request: web::Json +pub async fn create_account_handler() -> impl Responder { + + /*let record_id = sqlx::query!( + r#" +INSERT INTO badges ( slug, title, description, image_url ) +VALUES ( ?, ?, ?, ? ) + "#, + self.slug, + self.title, + self.description, + self.image_url, + ) + .execute(pool) + .await? + .last_insert_id(); + + Ok(record_id)*/ + + + + web::Json(CreateAccountResponse { + success: true, + }) +} \ No newline at end of file diff --git a/src/endpoints/users/login.rs b/src/endpoints/users/login.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/endpoints/users/logout.rs b/src/endpoints/users/logout.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/endpoints/users/mod.rs b/src/endpoints/users/mod.rs new file mode 100644 index 0000000000..07c35b615a --- /dev/null +++ b/src/endpoints/users/mod.rs @@ -0,0 +1,3 @@ +pub mod login; +pub mod logout; +pub mod create_account; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 854fce23f8..fa2bb7de2d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,7 @@ +#[macro_use] extern crate serde_derive; + pub mod server_state; +pub mod endpoints; pub mod queries; use actix_cors::Cors; @@ -11,6 +14,7 @@ use crate::server_state::{ServerState, EnvConfig}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use crate::queries::badges::NewBadge; +use crate::endpoints::users::create_account::create_account_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -31,12 +35,20 @@ async fn main() -> AnyhowResult<()> { .and_then(|h| h.into_string().ok()) .unwrap_or("storyteller-web-unknown".to_string()); + let db_connection_string = "mysql://root:root@localhost/storyteller"; + + let pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(db_connection_string) + .await?; + let server_state = ServerState { env_config: EnvConfig { num_workers: 4, bind_address: DEFAULT_BIND_ADDRESS.to_string(), }, hostname: server_hostname, + mysql_pool: pool, }; serve(server_state) @@ -51,58 +63,6 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> let num_workers = server_state.env_config.num_workers.clone(); let hostname = server_state.hostname.clone(); - let db_connection_string = "mysql://root:root@localhost/storyteller"; - - //let pool = MySqlPool::connect(db_connection_string).await?; - - - - let pool = MySqlPoolOptions::new() - .max_connections(5) - .connect(db_connection_string) - .await?; - let row: (i64,) = sqlx::query_as("SELECT 1") - //.bind(150_i64) - .fetch_one(&pool).await?; - - println!("Result: {:?}", row); - - /* - // Working example in 0.3 sqlx - let pool = MySqlPool::new(db_connection_string).await?; - let recs = sqlx::query!( - r#" - SELECT id, username - FROM users - ORDER BY id - "# - ) - .fetch_all(&pool) - .await?; - */ - - //create_user(&pool).await?; - - let badge = NewBadge { - slug: "slug2".to_string(), - title: "title".to_string(), - description: "description".to_string(), - image_url: "url".to_string() - }; - - let id = badge.insert(&pool).await?; - println!("Created id: {}", id); - - let badge = NewBadge { - slug: "slug3".to_string(), - title: "title".to_string(), - description: "description".to_string(), - image_url: "url".to_string() - }; - - let id = badge.insert(&pool).await?; - println!("Created id: {}", id); - let server_state_arc = web::Data::new(Arc::new(server_state)); info!("Starting HTTP service."); @@ -141,6 +101,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .exclude("/liveness") .exclude("/readiness")) .wrap(DefaultHeaders::new().header("X-Backend-Hostname", &hostname)) + .service( + web::resource("/create_account") + .route(web::post().to(create_account_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/test") .route(web::get().to(|| HttpResponse::Ok())) diff --git a/src/server_state.rs b/src/server_state.rs index 9210cb51c3..753b35066d 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,3 +1,4 @@ +use sqlx::MySqlPool; /// State that is injected into every endpoint. #[derive(Clone)] @@ -5,6 +6,8 @@ pub struct ServerState { pub env_config: EnvConfig, pub hostname: String, + + pub mysql_pool: MySqlPool, } #[derive(Clone)] From d4a4c1bc96b0151696815674772a763acbbc64c4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 16 May 2021 21:12:19 -0400 Subject: [PATCH 0027/1042] more example code --- Cargo.lock | 5 +-- Cargo.toml | 1 + src/endpoints/users/create_account.rs | 51 ++++++++++++++++++++++++--- src/main.rs | 27 ++++++++++++-- 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2fb3d56680..de9b075947 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -853,9 +853,9 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.13" +version = "0.99.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b1b72f1263f214c0f823371768776c4f5841b942c9883aa8e5ec584fd0ba6" +checksum = "5cc7b9cef1e351660e5443924e4f43ab25fbbed3e9a5f052df3677deb4d6b320" dependencies = [ "convert_case", "proc-macro2", @@ -2433,6 +2433,7 @@ dependencies = [ "anyhow", "base64", "chrono", + "derive_more", "diesel", "easyenv", "futures", diff --git a/Cargo.toml b/Cargo.toml index 1ef8cd903e..e8bda15363 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" anyhow = "1.0.40" base64 = "0.13.0" chrono = { version = "0.4.19", features = ["serde"] } +derive_more = "0.99.14" easyenv = "0.0.5" futures = "0.3.15" hostname = "0.3.1" diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index 46ef88f2e1..06c9654c7c 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -1,4 +1,9 @@ -use actix_web::{Responder, web}; +use actix_web::{Responder, web, HttpResponse}; +use derive_more::{Display, Error}; +use actix_web::dev::HttpResponseBuilder; +use actix_web::http::StatusCode; +use actix_http::http::header; +use actix_http::Error; #[derive(Deserialize)] pub struct CreateAccountRequest { @@ -13,8 +18,36 @@ pub struct CreateAccountResponse { } +#[derive(Debug, Display, Error)] +enum CreateAccountResponseError { + #[display(fmt = "internal error")] + InternalError, + + #[display(fmt = "bad request")] + BadClientData, + + #[display(fmt = "timeout")] + Timeout, +} + +impl actix_web::error::ResponseError for CreateAccountResponseError { + fn status_code(&self) -> StatusCode { + match *self { + CreateAccountResponseError::InternalError => StatusCode::INTERNAL_SERVER_ERROR, + CreateAccountResponseError::BadClientData => StatusCode::BAD_REQUEST, + CreateAccountResponseError::Timeout => StatusCode::GATEWAY_TIMEOUT, + } + } + + fn error_response(&self) -> HttpResponse { + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .body(self.to_string()) + } +} + //request: web::Json -pub async fn create_account_handler() -> impl Responder { +pub async fn create_account_handler() -> Result { /*let record_id = sqlx::query!( r#" @@ -32,9 +65,19 @@ VALUES ( ?, ?, ?, ? ) Ok(record_id)*/ + /*if true { + return Err(CreateAccountResponse { + success: false, + }); + }*/ + /*Ok(web::Json())*/ - web::Json(CreateAccountResponse { + let response = CreateAccountResponse { success: true, - }) + }; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body("foo")) } \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index fa2bb7de2d..fb937d1ad2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -107,9 +107,30 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) .service( - web::resource("/test") - .route(web::get().to(|| HttpResponse::Ok())) - .route(web::head().to(|| HttpResponse::Ok())) + web::scope("/foo") + .service( + web::resource("/bar") + .route(web::get().to(|| HttpResponse::Ok())) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/bin") + .route(web::get().to(|| HttpResponse::Ok())) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/a") + .service( + web::resource("/b") + .route(web::get().to(|| HttpResponse::Ok())) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/c") + .route(web::get().to(|| HttpResponse::Ok())) + .route(web::head().to(|| HttpResponse::Ok())) + ) ); // Admin UI & old frontend //.service(Files::new("/frontend", admin_asset_directory.clone()) From e93a390a078ba6d56feab18b25e1e7a74e2f6bed Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 16 May 2021 21:44:07 -0400 Subject: [PATCH 0028/1042] different shape --- src/endpoints/users/create_account.rs | 57 +++++++++++++++++++++------ 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index 06c9654c7c..6f36ac9270 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -1,15 +1,17 @@ -use actix_web::{Responder, web, HttpResponse}; +use actix_web::{Responder, web, HttpResponse, error}; use derive_more::{Display, Error}; use actix_web::dev::HttpResponseBuilder; use actix_web::http::StatusCode; use actix_http::http::header; use actix_http::Error; +use actix_web::error::ResponseError; +use crate::endpoints::users::create_account::CreateAccountError::BadInput; #[derive(Deserialize)] pub struct CreateAccountRequest { pub username: String, pub password: String, - pub password_verification: String, + pub password_confirmation: String, } #[derive(Serialize)] @@ -19,7 +21,7 @@ pub struct CreateAccountResponse { } #[derive(Debug, Display, Error)] -enum CreateAccountResponseError { +pub enum CreateAccountResponseError { #[display(fmt = "internal error")] InternalError, @@ -30,7 +32,28 @@ enum CreateAccountResponseError { Timeout, } -impl actix_web::error::ResponseError for CreateAccountResponseError { + +#[derive(Debug, Display)] +pub enum CreateAccountError { + BadInput(String), +} + +impl ResponseError for CreateAccountError { + fn status_code(&self) -> StatusCode { + match *self { + CreateAccountError::BadInput(_) => StatusCode::BAD_REQUEST, + } + } + + fn error_response(&self) -> HttpResponse { + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .body("bad") + } +} + + +impl ResponseError for CreateAccountResponseError { fn status_code(&self) -> StatusCode { match *self { CreateAccountResponseError::InternalError => StatusCode::INTERNAL_SERVER_ERROR, @@ -46,8 +69,13 @@ impl actix_web::error::ResponseError for CreateAccountResponseError { } } -//request: web::Json -pub async fn create_account_handler() -> Result { + +pub async fn create_account_handler(request: web::Json) + -> Result +{ + if request.username.len() < 3 { + return Err(CreateAccountError::BadInput("username is too short".to_string())); + } /*let record_id = sqlx::query!( r#" @@ -65,19 +93,24 @@ VALUES ( ?, ?, ?, ? ) Ok(record_id)*/ - /*if true { - return Err(CreateAccountResponse { - success: false, - }); - }*/ /*Ok(web::Json())*/ + if true { + //return Err(error::ErrorBadRequest(CreateAccountResponse { success: false } )); + //return Err(CreateAccountResponseError::UsernameExists("foo".to_string())); + } + + // .map_err(error::ErrorInternalServerError)?; + // error::ErrorBadRequest(msg) let response = CreateAccountResponse { success: true, }; + let body = serde_json::to_string(&response) + .map_err(|e| BadInput("".to_string()))?; + Ok(HttpResponse::Ok() .content_type("application/json") - .body("foo")) + .body(body)) } \ No newline at end of file From d0a5959e3ce2d48f0309ce87b4688164148044fd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 00:03:45 -0400 Subject: [PATCH 0029/1042] wip --- src/endpoints/users/create_account.rs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index 6f36ac9270..3ffd9e16b2 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -20,6 +20,12 @@ pub struct CreateAccountResponse { } +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + #[derive(Debug, Display, Error)] pub enum CreateAccountResponseError { #[display(fmt = "internal error")] @@ -46,13 +52,22 @@ impl ResponseError for CreateAccountError { } fn error_response(&self) -> HttpResponse { + let response = ErrorResponse { + success: false, + error_reason: "there was a problem".to_string() + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "text/html; charset=utf-8") - .body("bad") + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) } } - impl ResponseError for CreateAccountResponseError { fn status_code(&self) -> StatusCode { match *self { From 47ead0febfb6b28ef698ef91dbc72a0986a81f08 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 02:00:31 -0400 Subject: [PATCH 0030/1042] random tokens --- migrations/2021-05-15-071534_users/up.sql | 2 +- src/endpoints/users/create_account.rs | 124 +++++++++++++--------- src/main.rs | 1 + src/util/mod.rs | 1 + src/util/random.rs | 30 ++++++ 5 files changed, 105 insertions(+), 53 deletions(-) create mode 100644 src/util/mod.rs create mode 100644 src/util/random.rs diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 2f295ebd9d..bd9c6d9a07 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -56,7 +56,7 @@ CREATE TABLE users ( ) NOT NULL DEFAULT 'light-mode', -- An uploaded avatar. Public hash in our bucket. - avatar_public_bucket_hash CHAR(32) NOT NULL UNIQUE, + avatar_public_bucket_hash CHAR(32) NOT NULL, -- If the user doesn't want to use gravatar and doesn't have an uploaded avatar. disable_gravatar BOOLEAN NOT NULL DEFAULT false, diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index 3ffd9e16b2..d85f853bb7 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -1,23 +1,27 @@ -use actix_web::{Responder, web, HttpResponse, error}; -use derive_more::{Display, Error}; -use actix_web::dev::HttpResponseBuilder; -use actix_web::http::StatusCode; -use actix_http::http::header; use actix_http::Error; +use actix_http::http::header; +use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error}; use crate::endpoints::users::create_account::CreateAccountError::BadInput; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use crate::server_state::ServerState; + +const NEW_USER_ROLE: &'static str = "new-user"; #[derive(Deserialize)] pub struct CreateAccountRequest { pub username: String, pub password: String, pub password_confirmation: String, + pub email_address: String, } #[derive(Serialize)] -pub struct CreateAccountResponse { +pub struct CreateAccountSuccessResponse { pub success: bool, - } #[derive(Serialize)] @@ -26,35 +30,29 @@ pub struct ErrorResponse { pub error_reason: String, } -#[derive(Debug, Display, Error)] -pub enum CreateAccountResponseError { - #[display(fmt = "internal error")] - InternalError, - - #[display(fmt = "bad request")] - BadClientData, - - #[display(fmt = "timeout")] - Timeout, -} - - #[derive(Debug, Display)] pub enum CreateAccountError { BadInput(String), + ServerError, } impl ResponseError for CreateAccountError { fn status_code(&self) -> StatusCode { match *self { CreateAccountError::BadInput(_) => StatusCode::BAD_REQUEST, + CreateAccountError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { + let error_reason = match self { + BadInput(reason) => reason.to_string(), + CreateAccountError::ServerError => "server error".to_string(), + }; + let response = ErrorResponse { success: false, - error_reason: "there was a problem".to_string() + error_reason, }; let body = match serde_json::to_string(&response) { @@ -68,47 +66,69 @@ impl ResponseError for CreateAccountError { } } -impl ResponseError for CreateAccountResponseError { - fn status_code(&self) -> StatusCode { - match *self { - CreateAccountResponseError::InternalError => StatusCode::INTERNAL_SERVER_ERROR, - CreateAccountResponseError::BadClientData => StatusCode::BAD_REQUEST, - CreateAccountResponseError::Timeout => StatusCode::GATEWAY_TIMEOUT, - } - } - - fn error_response(&self) -> HttpResponse { - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "text/html; charset=utf-8") - .body(self.to_string()) - } -} - - -pub async fn create_account_handler(request: web::Json) - -> Result +pub async fn create_account_handler( + request: web::Json, + server_state: web::Data) -> Result { if request.username.len() < 3 { return Err(CreateAccountError::BadInput("username is too short".to_string())); } - /*let record_id = sqlx::query!( + if request.password.len() < 6 { + return Err(CreateAccountError::BadInput("password is too short".to_string())); + } + + if request.password != request.password_confirmation { + return Err(CreateAccountError::BadInput("passwords do not match".to_string())); + } + + if !request.email_address.contains("@") { + return Err(CreateAccountError::BadInput("invalid email address".to_string())); + } + + let token = "token"; + let password_hash = "temp"; + let profile_markdown = ""; + let profile_rendered_html = ""; + let ip_address = "1.1.1.1"; + + let record_id = sqlx::query!( r#" -INSERT INTO badges ( slug, title, description, image_url ) -VALUES ( ?, ?, ?, ? ) +INSERT INTO users ( + token, + username, + display_name, + email_address, + profile_markdown, + profile_rendered_html, + user_role_slug, + password_hash, + ip_address_creation, + ip_address_last_login, + ip_address_last_update +) +VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) "#, - self.slug, - self.title, - self.description, - self.image_url, + token.to_string(), + request.username.to_string(), + request.username.to_string(), + request.email_address.to_string(), + profile_markdown, + profile_rendered_html, + NEW_USER_ROLE, + password_hash, + ip_address.to_string(), + ip_address.to_string(), + ip_address.to_string(), ) - .execute(pool) - .await? + .execute(&server_state.mysql_pool) + .await + .map_err(|_| CreateAccountError::ServerError)? .last_insert_id(); - Ok(record_id)*/ - + info!("new user id: {}", record_id); + //Ok(record_id) /*Ok(web::Json())*/ if true { @@ -118,7 +138,7 @@ VALUES ( ?, ?, ?, ? ) // .map_err(error::ErrorInternalServerError)?; // error::ErrorBadRequest(msg) - let response = CreateAccountResponse { + let response = CreateAccountSuccessResponse { success: true, }; diff --git a/src/main.rs b/src/main.rs index fb937d1ad2..a8e0862418 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ pub mod server_state; pub mod endpoints; pub mod queries; +pub mod util; use actix_cors::Cors; use actix_http::http; diff --git a/src/util/mod.rs b/src/util/mod.rs new file mode 100644 index 0000000000..b9f20e6fd8 --- /dev/null +++ b/src/util/mod.rs @@ -0,0 +1 @@ +pub mod random; \ No newline at end of file diff --git a/src/util/random.rs b/src/util/random.rs new file mode 100644 index 0000000000..79d448a61e --- /dev/null +++ b/src/util/random.rs @@ -0,0 +1,30 @@ +use rand::Rng; + +// Crockford characters +const CROCKFORD_CHARSET: &[u8] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + +pub fn random_token(length: usize) -> String { + let mut rng = rand::thread_rng(); + + let token: String = (0..length) + .map(|_| { + let idx = rng.gen_range(0..CROCKFORD_CHARSET.len()); + CROCKFORD_CHARSET[idx] as char + }) + .collect(); + + token +} + +#[cfg(test)] +mod tests { + use crate::util::random::random_token; + + #[test] + fn random_token_length() { + assert_eq!(random_token(0), "".to_string()); + assert_eq!(random_token(1).len(), 1); + assert_eq!(random_token(10).len(), 10); + assert_eq!(random_token(32).len(), 32); + } +} From 03eee526a0795e7a8a765c82ae1e1d61ee1b8102 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 02:28:25 -0400 Subject: [PATCH 0031/1042] signup wip; db writes --- migrations/2021-05-15-071534_users/up.sql | 2 +- src/endpoints/users/create_account.rs | 13 +++-- src/main.rs | 70 +++++++++++------------ 3 files changed, 45 insertions(+), 40 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index bd9c6d9a07..e2642b2e44 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -56,7 +56,7 @@ CREATE TABLE users ( ) NOT NULL DEFAULT 'light-mode', -- An uploaded avatar. Public hash in our bucket. - avatar_public_bucket_hash CHAR(32) NOT NULL, + avatar_public_bucket_hash CHAR(32) DEFAULT NULL, -- If the user doesn't want to use gravatar and doesn't have an uploaded avatar. disable_gravatar BOOLEAN NOT NULL DEFAULT false, diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index d85f853bb7..d76d7e648a 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -5,9 +5,11 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error}; use crate::endpoints::users::create_account::CreateAccountError::BadInput; +use crate::server_state::ServerState; +use crate::util::random::random_token; use derive_more::{Display, Error}; use log::{info, warn, log}; -use crate::server_state::ServerState; +use std::sync::Arc; const NEW_USER_ROLE: &'static str = "new-user"; @@ -68,7 +70,7 @@ impl ResponseError for CreateAccountError { pub async fn create_account_handler( request: web::Json, - server_state: web::Data) -> Result + server_state: web::Data>) -> Result { if request.username.len() < 3 { return Err(CreateAccountError::BadInput("username is too short".to_string())); @@ -86,7 +88,7 @@ pub async fn create_account_handler( return Err(CreateAccountError::BadInput("invalid email address".to_string())); } - let token = "token"; + let token = random_token(32); let password_hash = "temp"; let profile_markdown = ""; let profile_rendered_html = ""; @@ -123,7 +125,10 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) ) .execute(&server_state.mysql_pool) .await - .map_err(|_| CreateAccountError::ServerError)? + .map_err(|e| { + warn!("Query error: {:?}", e); + return CreateAccountError::ServerError; + })? .last_insert_id(); info!("new user id: {}", record_id); diff --git a/src/main.rs b/src/main.rs index a8e0862418..1dad1560d2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -71,7 +71,8 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> let log_format = "[%{HOSTNAME}e] %a \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T"; HttpServer::new(move || { - let mut app = App::new() + App::new() + .app_data(server_state_arc.clone()) .wrap(Cors::default() .allowed_origin("http://api.vo.codes") .allowed_origin("http://jungle.horse") @@ -132,42 +133,41 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(|| HttpResponse::Ok())) .route(web::head().to(|| HttpResponse::Ok())) ) - ); - // Admin UI & old frontend - //.service(Files::new("/frontend", admin_asset_directory.clone()) - // .index_file("FAKE_INDEX.HTML")) - //.service(Files::new("/adminui", admin_asset_directory.clone()) - // .index_file("index.html")) - // Early access static path for assets - //.service(Files::new("/static", static_asset_directory.clone()) - // .index_file("FAKE_INDEX.HTML")) - /*.service( - web::resource("/login") - .route(web::post().to(login_handler)) - .route(web::head().to(|| HttpResponse::Ok())) ) - .service( - web::resource("/logout") - .route(web::post().to(logout_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service(get_liveness) - .service(get_models) - .service(get_readiness) - .service(get_root) - .service(get_sentences) - .service(get_service_settings) - .service(get_speakers) - .service(get_early_access_speakers) - .service(get_dynamic_early_access_speakers) - .service(get_words);*/ - - app.app_data(server_state_arc.clone()) }) - .bind(bind_address)? - .workers(num_workers) - .run() - .await?; + .bind(bind_address)? + .workers(num_workers) + .run() + .await?; + + // Admin UI & old frontend + //.service(Files::new("/frontend", admin_asset_directory.clone()) + // .index_file("FAKE_INDEX.HTML")) + //.service(Files::new("/adminui", admin_asset_directory.clone()) + // .index_file("index.html")) + // Early access static path for assets + //.service(Files::new("/static", static_asset_directory.clone()) + // .index_file("FAKE_INDEX.HTML")) + /*.service( + web::resource("/login") + .route(web::post().to(login_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/logout") + .route(web::post().to(logout_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(get_liveness) + .service(get_models) + .service(get_readiness) + .service(get_root) + .service(get_sentences) + .service(get_service_settings) + .service(get_speakers) + .service(get_early_access_speakers) + .service(get_dynamic_early_access_speakers) + .service(get_words);*/ Ok(()) } From 215df96f03f33341fcc3e10b6932ec447b331436 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 15:45:24 -0400 Subject: [PATCH 0032/1042] ip address, duplicate record handling --- Cargo.lock | 51 +++++++++++++++--- Cargo.toml | 1 + src/endpoints/users/create_account.rs | 78 +++++++++++++++++++-------- src/util/ip_address.rs | 32 +++++++++++ src/util/mod.rs | 3 +- 5 files changed, 136 insertions(+), 29 deletions(-) create mode 100644 src/util/ip_address.rs diff --git a/Cargo.lock b/Cargo.lock index de9b075947..2d717cf393 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,7 +83,7 @@ dependencies = [ "actix-service", "actix-threadpool", "actix-utils 2.0.0", - "base64", + "base64 0.13.0", "bitflags", "brotli2", "bytes 0.5.6", @@ -572,7 +572,7 @@ dependencies = [ "actix-http", "actix-rt 1.1.1", "actix-service", - "base64", + "base64 0.13.0", "bytes 0.5.6", "cfg-if 1.0.0", "derive_more", @@ -592,12 +592,30 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + [[package]] name = "base64" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bcrypt" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2cab630912253fb9dc92c0e2fabd0a7b51f5a5a4007177cfa31e517015b7204" +dependencies = [ + "base64 0.12.3", + "blowfish", + "byteorder", + "getrandom 0.1.16", +] + [[package]] name = "bitflags" version = "1.2.1" @@ -625,6 +643,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-cipher" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f337a3e6da609650eb74e02bc9fac7b735049f7623ab12f2e4c719316fcc7e80" +dependencies = [ + "generic-array", +] + [[package]] name = "blocking" version = "1.0.2" @@ -639,6 +666,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "blowfish" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f06850ba969bc59388b2cc0a4f186fc6d9d37208863b15b84ae3866ac90ac06" +dependencies = [ + "block-cipher", + "byteorder", + "opaque-debug", +] + [[package]] name = "brotli-sys" version = "0.3.2" @@ -1758,7 +1796,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb" dependencies = [ - "base64", + "base64 0.13.0", "once_cell", "regex", ] @@ -2067,7 +2105,7 @@ version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ - "base64", + "base64 0.13.0", "log", "ring", "sct", @@ -2287,7 +2325,7 @@ checksum = "de52d1d473cebb2abb79c886ef6a8023e965e34c0676a99cfeac2cc7f0fde4c1" dependencies = [ "ahash 0.7.2", "atoi", - "base64", + "base64 0.13.0", "bitflags", "byteorder", "bytes 1.0.1", @@ -2431,7 +2469,8 @@ dependencies = [ "actix-utils 3.0.0", "actix-web", "anyhow", - "base64", + "base64 0.13.0", + "bcrypt", "chrono", "derive_more", "diesel", diff --git a/Cargo.toml b/Cargo.toml index e8bda15363..02e38a2ac3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" [dependencies] anyhow = "1.0.40" base64 = "0.13.0" +bcrypt = "0.8" chrono = { version = "0.4.19", features = ["serde"] } derive_more = "0.99.14" easyenv = "0.0.5" diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index d76d7e648a..43803f3db6 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -3,13 +3,17 @@ use actix_http::http::header; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; -use actix_web::{Responder, web, HttpResponse, error}; -use crate::endpoints::users::create_account::CreateAccountError::BadInput; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::server_state::ServerState; use crate::util::random::random_token; use derive_more::{Display, Error}; use log::{info, warn, log}; use std::sync::Arc; +use crate::util::ip_address::get_request_ip; +use sqlx::mysql::MySqlDatabaseError; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; const NEW_USER_ROLE: &'static str = "new-user"; @@ -35,6 +39,8 @@ pub struct ErrorResponse { #[derive(Debug, Display)] pub enum CreateAccountError { BadInput(String), + UsernameTaken, + EmailTaken, ServerError, } @@ -42,6 +48,8 @@ impl ResponseError for CreateAccountError { fn status_code(&self) -> StatusCode { match *self { CreateAccountError::BadInput(_) => StatusCode::BAD_REQUEST, + CreateAccountError::UsernameTaken => StatusCode::BAD_REQUEST, + CreateAccountError::EmailTaken => StatusCode::BAD_REQUEST, CreateAccountError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } } @@ -49,6 +57,8 @@ impl ResponseError for CreateAccountError { fn error_response(&self) -> HttpResponse { let error_reason = match self { BadInput(reason) => reason.to_string(), + CreateAccountError::UsernameTaken => "username already taken".to_string(), + CreateAccountError::EmailTaken => "email already taken".to_string(), CreateAccountError::ServerError => "server error".to_string(), }; @@ -69,6 +79,7 @@ impl ResponseError for CreateAccountError { } pub async fn create_account_handler( + http_request: HttpRequest, request: web::Json, server_state: web::Data>) -> Result { @@ -88,13 +99,22 @@ pub async fn create_account_handler( return Err(CreateAccountError::BadInput("invalid email address".to_string())); } - let token = random_token(32); - let password_hash = "temp"; + //let token = random_token(32); + let token = random_token(15); + + let password_hash = match bcrypt::hash(&request.password, bcrypt::DEFAULT_COST) { + Ok(hash) => hash, + Err(err) => { + warn!("Bcrypt error: {:?}", err); + return Err(ServerError); + } + }; + let profile_markdown = ""; let profile_rendered_html = ""; - let ip_address = "1.1.1.1"; + let ip_address = get_request_ip(&http_request); - let record_id = sqlx::query!( + let result = sqlx::query!( r#" INSERT INTO users ( token, @@ -124,25 +144,39 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) ip_address.to_string(), ) .execute(&server_state.mysql_pool) - .await - .map_err(|e| { - warn!("Query error: {:?}", e); - return CreateAccountError::ServerError; - })? - .last_insert_id(); + .await; + + let record_id = match result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + warn!("Query error: {:?}", err); + + // NB: SQLSTATE[23000]: Integrity constraint violation + // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access) + match err { + Database(err) => { + let maybe_code = err.code().map(|c| c.into_owned()); + match maybe_code.as_deref() { + Some("23000") => { + if err.message().contains("username") { + return Err(UsernameTaken); + } else if err.message().contains("email_address") { + return Err(EmailTaken); + } + } + _ => {}, + } + }, + _ => {}, + } + return Err(ServerError); + } + }; info!("new user id: {}", record_id); - //Ok(record_id) - /*Ok(web::Json())*/ - - if true { - //return Err(error::ErrorBadRequest(CreateAccountResponse { success: false } )); - //return Err(CreateAccountResponseError::UsernameExists("foo".to_string())); - } - - // .map_err(error::ErrorInternalServerError)?; - // error::ErrorBadRequest(msg) let response = CreateAccountSuccessResponse { success: true, }; diff --git a/src/util/ip_address.rs b/src/util/ip_address.rs new file mode 100644 index 0000000000..bb500e129b --- /dev/null +++ b/src/util/ip_address.rs @@ -0,0 +1,32 @@ +use actix_web::HttpRequest; +use actix_web::http::HeaderName; +use log::{info}; + +pub fn get_request_ip(request: &HttpRequest) -> String { + match request.headers().get(HeaderName::from_static("x-voder-proxy-for")) { + Some(ip_address) => { + // Unfortunately the upstream Rust proxy is replacing the `forwarded` and `x-forwarded-for` + // headers, so we populate this custom header as a workaround. + info!("Proxied IP address: {:?}", ip_address); + ip_address.to_str() + .unwrap_or("") + .to_string() + }, + None => { + // If we're running without the upstream Rust proxy, we can grab 'x-forarded-for', which is + // populated by the DigitalOcean load balancer. + let ip_address_and_port = request.connection_info() + .remote_addr() + .unwrap_or("") + .to_string(); + let ip_address = ip_address_and_port.split(":") + .collect::>() + .get(0) + .copied() + .unwrap_or("") + .to_string(); + info!("Forwarded IP address: {}", &ip_address); + ip_address + }, + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index b9f20e6fd8..e388ba0dac 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1 +1,2 @@ -pub mod random; \ No newline at end of file +pub mod random; +pub mod ip_address; \ No newline at end of file From 5076536e6d557b25a011c75de7982b88b37db205 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 16:00:32 -0400 Subject: [PATCH 0033/1042] username validations --- Cargo.lock | 2 ++ Cargo.toml | 2 ++ src/endpoints/users/create_account.rs | 17 ++++++++++++----- src/main.rs | 2 ++ src/validations/mod.rs | 1 + src/validations/username.rs | 24 ++++++++++++++++++++++++ 6 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 src/validations/mod.rs create mode 100644 src/validations/username.rs diff --git a/Cargo.lock b/Cargo.lock index 2d717cf393..9ea94c585b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2477,8 +2477,10 @@ dependencies = [ "easyenv", "futures", "hostname", + "lazy_static", "log", "rand 0.8.3", + "regex", "serde", "serde_derive", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 02e38a2ac3..d59ba75c15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,8 +21,10 @@ derive_more = "0.99.14" easyenv = "0.0.5" futures = "0.3.15" hostname = "0.3.1" +lazy_static = "1.4.0" log = "0.4.14" rand = "0.8.3" +regex = "1.5.4" # Database #sqlx = { version = "0.5.0", features = ["mysql", "runtime-tokio-rustls", "chrono"] } diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index 43803f3db6..4a66b4d585 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -6,14 +6,16 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::server_state::ServerState; +use crate::util::ip_address::get_request_ip; use crate::util::random::random_token; use derive_more::{Display, Error}; use log::{info, warn, log}; -use std::sync::Arc; -use crate::util::ip_address::get_request_ip; -use sqlx::mysql::MySqlDatabaseError; +use regex::Regex; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use crate::validations::username::validate_username; const NEW_USER_ROLE: &'static str = "new-user"; @@ -83,8 +85,8 @@ pub async fn create_account_handler( request: web::Json, server_state: web::Data>) -> Result { - if request.username.len() < 3 { - return Err(CreateAccountError::BadInput("username is too short".to_string())); + if let Err(reason) = validate_username(&request.username) { + return Err(CreateAccountError::BadInput(reason)); } if request.password.len() < 6 { @@ -177,6 +179,11 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) info!("new user id: {}", record_id); + + + + + let response = CreateAccountSuccessResponse { success: true, }; diff --git a/src/main.rs b/src/main.rs index 1dad1560d2..7397584805 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,11 @@ +#[macro_use] extern crate lazy_static; #[macro_use] extern crate serde_derive; pub mod server_state; pub mod endpoints; pub mod queries; pub mod util; +pub mod validations; use actix_cors::Cors; use actix_http::http; diff --git a/src/validations/mod.rs b/src/validations/mod.rs new file mode 100644 index 0000000000..2c07d9fd86 --- /dev/null +++ b/src/validations/mod.rs @@ -0,0 +1 @@ +pub mod username; \ No newline at end of file diff --git a/src/validations/username.rs b/src/validations/username.rs new file mode 100644 index 0000000000..b03dcbc984 --- /dev/null +++ b/src/validations/username.rs @@ -0,0 +1,24 @@ +use crate::AnyhowResult; +use regex::Regex; + +pub fn validate_username(potential_username: &str) -> Result<(), String> { + lazy_static! { + static ref USERNAME_REGEX: Regex = { + Regex::new(r"^\w{3,16}$").expect("should be valid regex") + }; + } + + if potential_username.len() < 3 { + return Err("username is too short".to_string()); + } + + if potential_username.len() > 16 { + return Err("username is too long".to_string()); + } + + if !USERNAME_REGEX.is_match(potential_username) { + return Err("invalid username".to_string()); + } + + Ok(()) +} \ No newline at end of file From d2a75a29ecb344e9da99b6c38ba10cc5b981031a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 16:41:53 -0400 Subject: [PATCH 0034/1042] pw validations, session expiry +1 year, etc --- migrations/2021-05-15-071534_users/up.sql | 4 ++ src/endpoints/users/create_account.rs | 64 +++++++++++++++++------ src/validations/mod.rs | 3 +- src/validations/passwords.rs | 12 +++++ src/validations/username.rs | 34 +++++++++--- 5 files changed, 95 insertions(+), 22 deletions(-) create mode 100644 src/validations/passwords.rs diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index e2642b2e44..4bfd3d80be 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -138,6 +138,10 @@ CREATE TABLE user_sessions ( created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + -- Session termination time. + -- This must be set by the server code, or the session is invalid + expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + -- deletion = session termination deleted_at TIMESTAMP NULL, diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index 4a66b4d585..be5850b979 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -16,6 +16,8 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::validations::username::validate_username; +use crate::validations::passwords::validate_passwords; +use actix_web::cookie::Cookie; const NEW_USER_ROLE: &'static str = "new-user"; @@ -89,20 +91,15 @@ pub async fn create_account_handler( return Err(CreateAccountError::BadInput(reason)); } - if request.password.len() < 6 { - return Err(CreateAccountError::BadInput("password is too short".to_string())); - } - - if request.password != request.password_confirmation { - return Err(CreateAccountError::BadInput("passwords do not match".to_string())); + if let Err(reason) = validate_passwords(&request.password, &request.password_confirmation) { + return Err(CreateAccountError::BadInput(reason)); } if !request.email_address.contains("@") { return Err(CreateAccountError::BadInput("invalid email address".to_string())); } - //let token = random_token(32); - let token = random_token(15); + let user_token = random_token(15); let password_hash = match bcrypt::hash(&request.password, bcrypt::DEFAULT_COST) { Ok(hash) => hash, @@ -112,11 +109,14 @@ pub async fn create_account_handler( } }; + let username = request.username.to_lowercase(); + let display_name = request.username.to_string(); + let profile_markdown = ""; let profile_rendered_html = ""; let ip_address = get_request_ip(&http_request); - let result = sqlx::query!( + let query_result = sqlx::query!( r#" INSERT INTO users ( token, @@ -133,9 +133,9 @@ INSERT INTO users ( ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) "#, - token.to_string(), - request.username.to_string(), - request.username.to_string(), + user_token.to_string(), + username, + display_name, request.email_address.to_string(), profile_markdown, profile_rendered_html, @@ -148,12 +148,12 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) .execute(&server_state.mysql_pool) .await; - let record_id = match result { + let record_id = match query_result { Ok(res) => { res.last_insert_id() }, Err(err) => { - warn!("Query error: {:?}", err); + warn!("New user creation DB error: {:?}", err); // NB: SQLSTATE[23000]: Integrity constraint violation // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access) @@ -179,10 +179,43 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) info!("new user id: {}", record_id); + let session_token = random_token(32); + let query_result = sqlx::query!( + r#" +INSERT INTO user_sessions ( + token, + user_token, + ip_address_creation, + expires_at +) +VALUES ( ?, ?, ?, NOW() + interval 1 year ) + "#, + session_token.to_string(), + user_token.to_string(), + ip_address.to_string(), + ) + .execute(&server_state.mysql_pool) + .await; + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + warn!("New user session creation DB error: {:?}", err); + return Err(ServerError); + } + }; + info!("new user session created"); + let session_cookie = Cookie::build("session", session_token) + .domain(".vo.codes") + .path("/") + .secure(true) // HTTPS-only + .http_only(true) // Not exposed to Javascript + .finish(); let response = CreateAccountSuccessResponse { success: true, @@ -192,6 +225,7 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) .map_err(|e| BadInput("".to_string()))?; Ok(HttpResponse::Ok() + .cookie(session_cookie) .content_type("application/json") .body(body)) -} \ No newline at end of file +} diff --git a/src/validations/mod.rs b/src/validations/mod.rs index 2c07d9fd86..f02a6b769a 100644 --- a/src/validations/mod.rs +++ b/src/validations/mod.rs @@ -1 +1,2 @@ -pub mod username; \ No newline at end of file +pub mod username; +pub mod passwords; \ No newline at end of file diff --git a/src/validations/passwords.rs b/src/validations/passwords.rs new file mode 100644 index 0000000000..b86477f2cc --- /dev/null +++ b/src/validations/passwords.rs @@ -0,0 +1,12 @@ + +pub fn validate_passwords(password: &str, password_confirmation: &str) -> Result<(), String> { + if password.len() < 6 { + return Err("password is too short".to_string()); + } + + if password != password_confirmation { + return Err("passwords do not match".to_string()); + } + + Ok(()) +} diff --git a/src/validations/username.rs b/src/validations/username.rs index b03dcbc984..b16c2f4fc5 100644 --- a/src/validations/username.rs +++ b/src/validations/username.rs @@ -1,24 +1,46 @@ use crate::AnyhowResult; use regex::Regex; -pub fn validate_username(potential_username: &str) -> Result<(), String> { +pub fn validate_username(username: &str) -> Result<(), String> { lazy_static! { static ref USERNAME_REGEX: Regex = { - Regex::new(r"^\w{3,16}$").expect("should be valid regex") + Regex::new(r"^[A-Za-z0-9]{3,16}$").expect("should be valid regex") }; } - if potential_username.len() < 3 { + if username.len() < 3 { return Err("username is too short".to_string()); } - if potential_username.len() > 16 { + if username.len() > 16 { return Err("username is too long".to_string()); } - if !USERNAME_REGEX.is_match(potential_username) { + if !USERNAME_REGEX.is_match(username) { return Err("invalid username".to_string()); } Ok(()) -} \ No newline at end of file +} + +/*pub fn validate_display_name(display_name: &str) -> Result<(), String> { + lazy_static! { + static ref DISPLAY_NAME_REGEX: Regex = { + Regex::new(r"^\w{3,16}$").expect("should be valid regex") + }; + } + + if username.len() < 3 { + return Err("display name is too short".to_string()); + } + + if username.len() > 16 { + return Err("display name is too long".to_string()); + } + + if !DISPLAY_NAME_REGEX.is_match(username) { + return Err("invalid display name".to_string()); + } + + Ok(()) +}*/ From 9893737a5747f09568874e4909d6bb4314b4e4cd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 20:31:55 -0400 Subject: [PATCH 0035/1042] hmac jwts --- Cargo.lock | 38 ++++++ Cargo.toml | 7 +- src/endpoints/users/create_account.rs | 12 +- src/endpoints/users/login.rs | 176 ++++++++++++++++++++++++++ src/endpoints/users/mod.rs | 4 +- src/main.rs | 17 ++- src/server_state.rs | 4 + src/util/cookies.rs | 46 +++++++ src/util/mod.rs | 5 +- src/util/{random.rs => tokens.rs} | 2 +- 10 files changed, 292 insertions(+), 19 deletions(-) create mode 100644 src/util/cookies.rs rename src/util/{random.rs => tokens.rs} (93%) diff --git a/Cargo.lock b/Cargo.lock index 9ea94c585b..898977a4e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -879,6 +879,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crypto-mac" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "ctor" version = "0.1.20" @@ -1286,6 +1296,16 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +dependencies = [ + "crypto-mac", + "digest", +] + [[package]] name = "hostname" version = "0.3.1" @@ -1400,6 +1420,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jwt" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e46349d67dc03bdbdb28da0337a355a53ca1d5156452722c36fe21d0e6389b" +dependencies = [ + "base64 0.13.0", + "crypto-mac", + "digest", + "hmac", + "serde", + "serde_json", + "sha2", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -2476,7 +2511,9 @@ dependencies = [ "diesel", "easyenv", "futures", + "hmac", "hostname", + "jwt", "lazy_static", "log", "rand 0.8.3", @@ -2484,6 +2521,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "sha2", "sqlx", "toml", ] diff --git a/Cargo.toml b/Cargo.toml index d59ba75c15..b9142b021c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ path = "src/main.rs" [dependencies] anyhow = "1.0.40" base64 = "0.13.0" -bcrypt = "0.8" chrono = { version = "0.4.19", features = ["serde"] } derive_more = "0.99.14" easyenv = "0.0.5" @@ -46,6 +45,12 @@ actix-http = "2.2.0" actix-rt = "2.2.0" actix-utils = "3.0.0" actix-web = "3.3.2" +jwt = "0.13.0" + +# Crypto +bcrypt = "0.8" +hmac = "0.10.0" +sha2 = "0.9.5" # Serialization serde = "1.0.125" diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index be5850b979..9c9d728f3b 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -7,7 +7,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; -use crate::util::random::random_token; +use crate::util::tokens::random_token; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; @@ -210,12 +210,10 @@ VALUES ( ?, ?, ?, NOW() + interval 1 year ) info!("new user session created"); - let session_cookie = Cookie::build("session", session_token) - .domain(".vo.codes") - .path("/") - .secure(true) // HTTPS-only - .http_only(true) // Not exposed to Javascript - .finish(); + let session_cookie = match server_state.cookie_manager.create_cookie(&session_token) { + Ok(cookie) => cookie, + Err(_) => return Err(ServerError), + }; let response = CreateAccountSuccessResponse { success: true, diff --git a/src/endpoints/users/login.rs b/src/endpoints/users/login.rs index e69de29bb2..6ef9ddb9d5 100644 --- a/src/endpoints/users/login.rs +++ b/src/endpoints/users/login.rs @@ -0,0 +1,176 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::server_state::ServerState; +use crate::util::ip_address::get_request_ip; +use crate::util::tokens::random_token; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use crate::validations::username::validate_username; +use crate::validations::passwords::validate_passwords; +use actix_web::cookie::Cookie; + +#[derive(Deserialize)] +pub struct LoginRequest { + pub username_or_email: String, + pub password: String, +} + +#[derive(Serialize)] +pub struct LoginSuccessResponse { + pub success: bool, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum CreateAccountError { + BadInput(String), + UsernameTaken, + EmailTaken, + ServerError, +} + +impl ResponseError for CreateAccountError { + fn status_code(&self) -> StatusCode { + match *self { + CreateAccountError::BadInput(_) => StatusCode::BAD_REQUEST, + CreateAccountError::UsernameTaken => StatusCode::BAD_REQUEST, + CreateAccountError::EmailTaken => StatusCode::BAD_REQUEST, + CreateAccountError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + BadInput(reason) => reason.to_string(), + CreateAccountError::UsernameTaken => "username already taken".to_string(), + CreateAccountError::EmailTaken => "email already taken".to_string(), + CreateAccountError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn login_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let try_password_hash = match bcrypt::hash(&request.password, bcrypt::DEFAULT_COST) { + Ok(hash) => hash, + Err(err) => { + warn!("Bcrypt error: {:?}", err); + return Err(ServerError); + } + }; + + let query_result = sqlx::query!( + r#" +INSERT INTO users ( + token, + username, + display_name, + email_address, + profile_markdown, + profile_rendered_html, + user_role_slug, + password_hash, + ip_address_creation, + ip_address_last_login, + ip_address_last_update +) +VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) + "#, + user_token.to_string(), + username, + display_name, + request.email_address.to_string(), + profile_markdown, + profile_rendered_html, + NEW_USER_ROLE, + password_hash, + ip_address.to_string(), + ip_address.to_string(), + ip_address.to_string(), + ) + .execute(&server_state.mysql_pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + warn!("New user creation DB error: {:?}", err); + + // NB: SQLSTATE[23000]: Integrity constraint violation + // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access) + match err { + Database(err) => { + let maybe_code = err.code().map(|c| c.into_owned()); + match maybe_code.as_deref() { + Some("23000") => { + if err.message().contains("username") { + return Err(UsernameTaken); + } else if err.message().contains("email_address") { + return Err(EmailTaken); + } + } + _ => {}, + } + }, + _ => {}, + } + return Err(ServerError); + } + }; + + + info!("login session created"); + + let session_cookie = Cookie::build("session", session_token) + .domain(".vo.codes") + .path("/") + .secure(true) // HTTPS-only + .http_only(true) // Not exposed to Javascript + .finish(); + + let response = CreateAccountSuccessResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| BadInput("".to_string()))?; + + Ok(HttpResponse::Ok() + .cookie(session_cookie) + .content_type("application/json") + .body(body)) +} diff --git a/src/endpoints/users/mod.rs b/src/endpoints/users/mod.rs index 07c35b615a..5714e6443f 100644 --- a/src/endpoints/users/mod.rs +++ b/src/endpoints/users/mod.rs @@ -1,3 +1,3 @@ -pub mod login; -pub mod logout; +//pub mod login; +//pub mod logout; pub mod create_account; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 7397584805..60ca3e5d01 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,17 +18,14 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use crate::queries::badges::NewBadge; use crate::endpoints::users::create_account::create_account_handler; +//use crate::endpoints::users::login::login_handler; +use crate::util::cookies::CookieManager; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; pub type AnyhowResult = anyhow::Result; -//pub async fn main() -> AnyhowResult<()> { -//#[tokio::main] -//#[async_std::main] -//#[actix_web::main] -//pub async fn main() -> std::io::Result<()> { #[actix_web::main] async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); @@ -45,6 +42,9 @@ async fn main() -> AnyhowResult<()> { .connect(db_connection_string) .await?; + let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); + let cookie_manager = CookieManager::new(".vo.codes", &hmac_secret); + let server_state = ServerState { env_config: EnvConfig { num_workers: 4, @@ -52,6 +52,7 @@ async fn main() -> AnyhowResult<()> { }, hostname: server_hostname, mysql_pool: pool, + cookie_manager, }; serve(server_state) @@ -59,7 +60,6 @@ async fn main() -> AnyhowResult<()> { Ok(()) } -//#[actix_web::main] pub async fn serve(server_state: ServerState) -> AnyhowResult<()> { let bind_address = server_state.env_config.bind_address.clone(); @@ -110,6 +110,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::post().to(create_account_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + /*.service( + web::resource("/login") + .route(web::post().to(login_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + )*/ .service( web::scope("/foo") .service( diff --git a/src/server_state.rs b/src/server_state.rs index 753b35066d..88c4d60891 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,13 +1,17 @@ use sqlx::MySqlPool; +use crate::util::cookies::CookieManager; /// State that is injected into every endpoint. #[derive(Clone)] pub struct ServerState { + /// Configuration from ENV vars. pub env_config: EnvConfig, pub hostname: String, pub mysql_pool: MySqlPool, + + pub cookie_manager: CookieManager, } #[derive(Clone)] diff --git a/src/util/cookies.rs b/src/util/cookies.rs new file mode 100644 index 0000000000..827b9803ea --- /dev/null +++ b/src/util/cookies.rs @@ -0,0 +1,46 @@ +use actix_web::cookie::Cookie; +use anyhow::anyhow; +use crate::AnyhowResult; +use hmac::Hmac; +use hmac::NewMac; +use std::collections::BTreeMap; +use sha2::Sha256; +use jwt::SignWithKey; + +const COOKIE_VERSION : u32 = 1; + +#[derive(Clone)] +pub struct CookieManager { + cookie_domain: String, + hmac_secret: String, +} + +impl CookieManager { + pub fn new(cookie_domain: &str, hmac_secret: &str) -> Self { + Self { + cookie_domain: cookie_domain.to_string(), + hmac_secret: hmac_secret.to_string(), + } + } + + pub fn create_cookie(&self, session_token: &str) -> AnyhowResult { + let key: Hmac = Hmac::new_varkey(self.hmac_secret.as_bytes()) + .map_err(|e| anyhow!("invalid hmac: {:?}", e))?; + + let cookie_version = COOKIE_VERSION.to_string(); + + let mut claims = BTreeMap::new(); + claims.insert("session_token", session_token); + claims.insert("cookie_version", &cookie_version); + + let jwt_string = claims.sign_with_key(&key)?; + + Ok(Cookie::build("session", jwt_string) + .domain(&self.cookie_domain) + .path("/") + .secure(true) // HTTPS-only + .http_only(true) // Not exposed to Javascript + .finish()) + } +} + diff --git a/src/util/mod.rs b/src/util/mod.rs index e388ba0dac..d169123365 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,2 +1,3 @@ -pub mod random; -pub mod ip_address; \ No newline at end of file +pub mod cookies; +pub mod ip_address; +pub mod tokens; \ No newline at end of file diff --git a/src/util/random.rs b/src/util/tokens.rs similarity index 93% rename from src/util/random.rs rename to src/util/tokens.rs index 79d448a61e..7551e50fcd 100644 --- a/src/util/random.rs +++ b/src/util/tokens.rs @@ -18,7 +18,7 @@ pub fn random_token(length: usize) -> String { #[cfg(test)] mod tests { - use crate::util::random::random_token; + use crate::util::tokens::random_token; #[test] fn random_token_length() { From 53c1ed538149713f634f3917e97007791bb20b72 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 20:33:08 -0400 Subject: [PATCH 0036/1042] ignore diesel codegen --- .gitignore | 1 + diesel.toml | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 diesel.toml diff --git a/.gitignore b/.gitignore index aee3b3b46c..7de72bfe0f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *~ .env .idea/ +_ignored_schema.rs secrets.toml secrets.yaml target/ diff --git a/diesel.toml b/diesel.toml new file mode 100644 index 0000000000..0588bd6b16 --- /dev/null +++ b/diesel.toml @@ -0,0 +1,5 @@ +# For documentation on how to configure this file, +# see diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "_ignored_schema.rs" From 0c9751bea5b4173b6b2825dd025d2f1c9d15d4cc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 20:39:09 -0400 Subject: [PATCH 0037/1042] test scripts in lieu of actual testing --- debug/curl/create_account_1.sh | 7 +++++++ debug/curl/create_account_2.sh | 7 +++++++ debug/mysql/last_session.sh | 7 +++++++ debug/mysql/last_user.sh | 7 +++++++ 4 files changed, 28 insertions(+) create mode 100755 debug/curl/create_account_1.sh create mode 100755 debug/curl/create_account_2.sh create mode 100755 debug/mysql/last_session.sh create mode 100755 debug/mysql/last_user.sh diff --git a/debug/curl/create_account_1.sh b/debug/curl/create_account_1.sh new file mode 100755 index 0000000000..85a4d0cb09 --- /dev/null +++ b/debug/curl/create_account_1.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# LOCALHOST ONLY + +curl -vvv \ + -H "content-type: application/json" \ + --data '{"username": "echelon", "password": "testing", "password_confirmation": "testing", "email_address": "echelon@gmail.com"}' \ + http://localhost:12345/create_account diff --git a/debug/curl/create_account_2.sh b/debug/curl/create_account_2.sh new file mode 100755 index 0000000000..457b415272 --- /dev/null +++ b/debug/curl/create_account_2.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# LOCALHOST ONLY + +curl -vvv \ + -H "content-type: application/json" \ + --data '{"username": "ernest", "password": "knowwhatimean", "password_confirmation": "knowwhatimean", "email_address": "vern@knowwhatimean.com"}' \ + http://localhost:12345/create_account diff --git a/debug/mysql/last_session.sh b/debug/mysql/last_session.sh new file mode 100755 index 0000000000..149a552384 --- /dev/null +++ b/debug/mysql/last_session.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u root \ + -proot \ + -h localhost \ + -D storyteller \ + -e "select * from user_sessions\G" diff --git a/debug/mysql/last_user.sh b/debug/mysql/last_user.sh new file mode 100755 index 0000000000..a7c46a877d --- /dev/null +++ b/debug/mysql/last_user.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u root \ + -proot \ + -h localhost \ + -D storyteller \ + -e "select * from users\G" From 245ef66969de5ec83dc8d02c1d229494b618cf95 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 22:08:48 -0400 Subject: [PATCH 0038/1042] working login --- src/{queries => common_queries}/badges.rs | 0 src/common_queries/mod.rs | 3 + src/common_queries/sessions.rs | 41 +++++ src/{queries => common_queries}/users.rs | 0 src/endpoints/users/create_account.rs | 5 +- src/endpoints/users/login.rs | 188 ++++++++++++---------- src/endpoints/users/mod.rs | 3 +- src/main.rs | 11 +- src/queries/mod.rs | 2 - 9 files changed, 159 insertions(+), 94 deletions(-) rename src/{queries => common_queries}/badges.rs (100%) create mode 100644 src/common_queries/mod.rs create mode 100644 src/common_queries/sessions.rs rename src/{queries => common_queries}/users.rs (100%) delete mode 100644 src/queries/mod.rs diff --git a/src/queries/badges.rs b/src/common_queries/badges.rs similarity index 100% rename from src/queries/badges.rs rename to src/common_queries/badges.rs diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs new file mode 100644 index 0000000000..ca47e7283b --- /dev/null +++ b/src/common_queries/mod.rs @@ -0,0 +1,3 @@ +pub mod users; +pub mod badges; +pub mod sessions; \ No newline at end of file diff --git a/src/common_queries/sessions.rs b/src/common_queries/sessions.rs new file mode 100644 index 0000000000..e0260af3e2 --- /dev/null +++ b/src/common_queries/sessions.rs @@ -0,0 +1,41 @@ +use anyhow::anyhow; +use log::{info, warn}; +use crate::AnyhowResult; +use crate::util::tokens::random_token; +use sqlx::MySqlPool; + +pub async fn create_session_for_user(user_token: &str, ip_address: &str, mysql_pool: &MySqlPool) + -> AnyhowResult +{ + let session_token = random_token(32); + + let query_result = sqlx::query!( + r#" +INSERT INTO user_sessions ( + token, + user_token, + ip_address_creation, + expires_at +) +VALUES ( ?, ?, ?, NOW() + interval 1 year ) + "#, + session_token.to_string(), + user_token.to_string(), + ip_address.to_string(), + ) + .execute(mysql_pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + return Err(anyhow!("session creation DB error: {:?}", err)); + } + }; + + info!("Created session id: {}", record_id); + + Ok(session_token) +} diff --git a/src/queries/users.rs b/src/common_queries/users.rs similarity index 100% rename from src/queries/users.rs rename to src/common_queries/users.rs diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index 9c9d728f3b..cd402b5436 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -112,8 +112,11 @@ pub async fn create_account_handler( let username = request.username.to_lowercase(); let display_name = request.username.to_string(); + let email_address = request.email_address.to_lowercase(); + let profile_markdown = ""; let profile_rendered_html = ""; + let ip_address = get_request_ip(&http_request); let query_result = sqlx::query!( @@ -136,7 +139,7 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) user_token.to_string(), username, display_name, - request.email_address.to_string(), + email_address, profile_markdown, profile_rendered_html, NEW_USER_ROLE, diff --git a/src/endpoints/users/login.rs b/src/endpoints/users/login.rs index 6ef9ddb9d5..c7a7917e78 100644 --- a/src/endpoints/users/login.rs +++ b/src/endpoints/users/login.rs @@ -18,6 +18,9 @@ use std::sync::Arc; use crate::validations::username::validate_username; use crate::validations::passwords::validate_passwords; use actix_web::cookie::Cookie; +use sqlx::MySqlPool; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; #[derive(Deserialize)] pub struct LoginRequest { @@ -37,29 +40,23 @@ pub struct ErrorResponse { } #[derive(Debug, Display)] -pub enum CreateAccountError { - BadInput(String), - UsernameTaken, - EmailTaken, +pub enum LoginError { + WrongCredentials, ServerError, } -impl ResponseError for CreateAccountError { +impl ResponseError for LoginError { fn status_code(&self) -> StatusCode { match *self { - CreateAccountError::BadInput(_) => StatusCode::BAD_REQUEST, - CreateAccountError::UsernameTaken => StatusCode::BAD_REQUEST, - CreateAccountError::EmailTaken => StatusCode::BAD_REQUEST, - CreateAccountError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + LoginError::WrongCredentials => StatusCode::BAD_REQUEST, + LoginError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { - BadInput(reason) => reason.to_string(), - CreateAccountError::UsernameTaken => "username already taken".to_string(), - CreateAccountError::EmailTaken => "email already taken".to_string(), - CreateAccountError::ServerError => "server error".to_string(), + LoginError::WrongCredentials => "wrong credentials".to_string(), + LoginError::ServerError => "server error".to_string(), }; let response = ErrorResponse { @@ -81,96 +78,119 @@ impl ResponseError for CreateAccountError { pub async fn login_handler( http_request: HttpRequest, request: web::Json, - server_state: web::Data>) -> Result + server_state: web::Data>) -> Result { - let try_password_hash = match bcrypt::hash(&request.password, bcrypt::DEFAULT_COST) { + let check_username_or_email = request.username_or_email.to_lowercase(); + + let maybe_user = if check_username_or_email.contains("@") { + lookup_by_email(&check_username_or_email, &server_state.mysql_pool).await + } else { + lookup_by_username(&check_username_or_email, &server_state.mysql_pool).await + }; + + let user = match maybe_user { + Ok(user) => user, + Err(e) => { + warn!("Login lookup error: {:?}", e); + return Err(LoginError::WrongCredentials); // TODO: This isn't necessarily user error. + } + }; + + info!("login user found"); + + let actual_hash = match String::from_utf8(user.password_hash.clone()) { Ok(hash) => hash, - Err(err) => { - warn!("Bcrypt error: {:?}", err); - return Err(ServerError); + Err(e) => { + warn!("Login hash hydration error: {:?}", e); + return Err(LoginError::ServerError); } }; - let query_result = sqlx::query!( - r#" -INSERT INTO users ( - token, - username, - display_name, - email_address, - profile_markdown, - profile_rendered_html, - user_role_slug, - password_hash, - ip_address_creation, - ip_address_last_login, - ip_address_last_update -) -VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) - "#, - user_token.to_string(), - username, - display_name, - request.email_address.to_string(), - profile_markdown, - profile_rendered_html, - NEW_USER_ROLE, - password_hash, - ip_address.to_string(), - ip_address.to_string(), - ip_address.to_string(), - ) - .execute(&server_state.mysql_pool) - .await; - - let record_id = match query_result { - Ok(res) => { - res.last_insert_id() - }, - Err(err) => { - warn!("New user creation DB error: {:?}", err); - - // NB: SQLSTATE[23000]: Integrity constraint violation - // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access) - match err { - Database(err) => { - let maybe_code = err.code().map(|c| c.into_owned()); - match maybe_code.as_deref() { - Some("23000") => { - if err.message().contains("username") { - return Err(UsernameTaken); - } else if err.message().contains("email_address") { - return Err(EmailTaken); - } - } - _ => {}, - } - }, - _ => {}, + let is_valid = match bcrypt::verify(&request.password, &actual_hash) { + Ok(is_valid) => { + if !is_valid { + info!("invalid credentials"); + return Err(LoginError::WrongCredentials); } - return Err(ServerError); + }, + Err(e) => { + warn!("Login hash comparison error: {:?}", e); + return Err(LoginError::ServerError); } }; + let ip_address = get_request_ip(&http_request); - info!("login session created"); + let create_session_result = + create_session_for_user(&user.token, &ip_address, &server_state.mysql_pool).await; - let session_cookie = Cookie::build("session", session_token) - .domain(".vo.codes") - .path("/") - .secure(true) // HTTPS-only - .http_only(true) // Not exposed to Javascript - .finish(); + let session_token = match create_session_result { + Ok(token) => token, + Err(e) => { + warn!("login create session error : {:?}", e); + return Err(LoginError::ServerError); + } + }; - let response = CreateAccountSuccessResponse { + let session_cookie = match server_state.cookie_manager.create_cookie(&session_token) { + Ok(cookie) => cookie, + Err(_) => return Err(LoginError::ServerError), + }; + + let response = LoginSuccessResponse { success: true, }; let body = serde_json::to_string(&response) - .map_err(|e| BadInput("".to_string()))?; + .map_err(|e| LoginError::ServerError)?; Ok(HttpResponse::Ok() .cookie(session_cookie) .content_type("application/json") .body(body)) } + +#[derive(Debug)] +pub struct UserRecordForLogin { + token: String, + username: String, + email_address: String, + password_hash: Vec, +} + +async fn lookup_by_username(username: &str, pool: &MySqlPool) -> AnyhowResult +{ + // NB: Lookup failure is Err(RowNotFound). + let record = sqlx::query_as!( + UserRecordForLogin, + r#" +SELECT token, username, email_address, password_hash +FROM users +WHERE username = ? +LIMIT 1 + "#, + username.to_string(), + ) + .fetch_one(pool) + .await?; + + Ok(record) +} + +async fn lookup_by_email(email: &str, pool: &MySqlPool) -> AnyhowResult { + // NB: Lookup failure is Err(RowNotFound). + let record = sqlx::query_as!( + UserRecordForLogin, + r#" +SELECT token, username, email_address, password_hash +FROM users +WHERE email_address = ? +LIMIT 1 + "#, + email.to_string(), + ) + .fetch_one(pool) + .await?; + + Ok(record) +} diff --git a/src/endpoints/users/mod.rs b/src/endpoints/users/mod.rs index 5714e6443f..3eead82462 100644 --- a/src/endpoints/users/mod.rs +++ b/src/endpoints/users/mod.rs @@ -1,3 +1,4 @@ //pub mod login; //pub mod logout; -pub mod create_account; \ No newline at end of file +pub mod create_account; +pub mod login; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 60ca3e5d01..3eef84a522 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,9 @@ #[macro_use] extern crate lazy_static; #[macro_use] extern crate serde_derive; -pub mod server_state; +pub mod common_queries; pub mod endpoints; -pub mod queries; +pub mod server_state; pub mod util; pub mod validations; @@ -16,9 +16,8 @@ use std::sync::Arc; use crate::server_state::{ServerState, EnvConfig}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; -use crate::queries::badges::NewBadge; use crate::endpoints::users::create_account::create_account_handler; -//use crate::endpoints::users::login::login_handler; +use crate::endpoints::users::login::login_handler; use crate::util::cookies::CookieManager; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -110,11 +109,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::post().to(create_account_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - /*.service( + .service( web::resource("/login") .route(web::post().to(login_handler)) .route(web::head().to(|| HttpResponse::Ok())) - )*/ + ) .service( web::scope("/foo") .service( diff --git a/src/queries/mod.rs b/src/queries/mod.rs deleted file mode 100644 index 2e6f0082b7..0000000000 --- a/src/queries/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod users; -pub mod badges; \ No newline at end of file From aa740db35b2c2162534413245fa95c92e6fb7aad Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 22:09:01 -0400 Subject: [PATCH 0039/1042] more "test" scripts --- debug/curl/login_bad_password.sh | 7 +++++++ debug/curl/login_bad_username.sh | 7 +++++++ debug/curl/login_email.sh | 7 +++++++ debug/curl/login_username.sh | 7 +++++++ 4 files changed, 28 insertions(+) create mode 100755 debug/curl/login_bad_password.sh create mode 100755 debug/curl/login_bad_username.sh create mode 100755 debug/curl/login_email.sh create mode 100755 debug/curl/login_username.sh diff --git a/debug/curl/login_bad_password.sh b/debug/curl/login_bad_password.sh new file mode 100755 index 0000000000..8da6754518 --- /dev/null +++ b/debug/curl/login_bad_password.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# LOCALHOST ONLY + +curl -vvv \ + -H "content-type: application/json" \ + --data '{"username_or_email": "echelon", "password": "BAD_PASSWORD"}' \ + http://localhost:12345/login diff --git a/debug/curl/login_bad_username.sh b/debug/curl/login_bad_username.sh new file mode 100755 index 0000000000..85b70e89f6 --- /dev/null +++ b/debug/curl/login_bad_username.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# LOCALHOST ONLY + +curl -vvv \ + -H "content-type: application/json" \ + --data '{"username_or_email": "nobody", "password": "testing"}' \ + http://localhost:12345/login diff --git a/debug/curl/login_email.sh b/debug/curl/login_email.sh new file mode 100755 index 0000000000..977f07023a --- /dev/null +++ b/debug/curl/login_email.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# LOCALHOST ONLY + +curl -vvv \ + -H "content-type: application/json" \ + --data '{"username_or_email": "echelon@gmail.com", "password": "testing"}' \ + http://localhost:12345/login diff --git a/debug/curl/login_username.sh b/debug/curl/login_username.sh new file mode 100755 index 0000000000..95b72e8857 --- /dev/null +++ b/debug/curl/login_username.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# LOCALHOST ONLY + +curl -vvv \ + -H "content-type: application/json" \ + --data '{"username_or_email": "echelon", "password": "testing"}' \ + http://localhost:12345/login From 80966cd876f22e00934bae550399a31f8f0a04b4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 22:48:49 -0400 Subject: [PATCH 0040/1042] reuse session creation --- src/endpoints/users/create_account.rs | 32 +++++++-------------------- src/endpoints/users/login.rs | 2 ++ src/endpoints/users/mod.rs | 5 ++--- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index cd402b5436..c86fb837df 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use crate::validations::username::validate_username; use crate::validations::passwords::validate_passwords; use actix_web::cookie::Cookie; +use crate::common_queries::sessions::create_session_for_user; const NEW_USER_ROLE: &'static str = "new-user"; @@ -182,32 +183,15 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) info!("new user id: {}", record_id); - let session_token = random_token(32); - let query_result = sqlx::query!( - r#" -INSERT INTO user_sessions ( - token, - user_token, - ip_address_creation, - expires_at -) -VALUES ( ?, ?, ?, NOW() + interval 1 year ) - "#, - session_token.to_string(), - user_token.to_string(), - ip_address.to_string(), - ) - .execute(&server_state.mysql_pool) - .await; + let create_session_result = + create_session_for_user(&user_token, &ip_address, &server_state.mysql_pool).await; - let record_id = match query_result { - Ok(res) => { - res.last_insert_id() - }, - Err(err) => { - warn!("New user session creation DB error: {:?}", err); - return Err(ServerError); + let session_token = match create_session_result { + Ok(token) => token, + Err(e) => { + warn!("create account session creation error : {:?}", e); + return Err(CreateAccountError::ServerError); } }; diff --git a/src/endpoints/users/login.rs b/src/endpoints/users/login.rs index c7a7917e78..fc5e82b659 100644 --- a/src/endpoints/users/login.rs +++ b/src/endpoints/users/login.rs @@ -132,6 +132,8 @@ pub async fn login_handler( } }; + info!("login session created"); + let session_cookie = match server_state.cookie_manager.create_cookie(&session_token) { Ok(cookie) => cookie, Err(_) => return Err(LoginError::ServerError), diff --git a/src/endpoints/users/mod.rs b/src/endpoints/users/mod.rs index 3eead82462..019ccd2199 100644 --- a/src/endpoints/users/mod.rs +++ b/src/endpoints/users/mod.rs @@ -1,4 +1,3 @@ -//pub mod login; -//pub mod logout; pub mod create_account; -pub mod login; \ No newline at end of file +pub mod login; +pub mod logout; From 758314093b4c4374e0ba8348475444e79e032647 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 23:36:57 -0400 Subject: [PATCH 0041/1042] logout endpoint and "test" --- Cargo.lock | 1 + Cargo.toml | 1 + debug/curl/logout_roundtrip.py | 36 ++++++++++ src/endpoints/users/logout.rs | 118 +++++++++++++++++++++++++++++++++ src/main.rs | 37 +++-------- src/util/cookies.rs | 30 ++++++++- 6 files changed, 193 insertions(+), 30 deletions(-) create mode 100755 debug/curl/logout_roundtrip.py diff --git a/Cargo.lock b/Cargo.lock index 898977a4e0..beaca6a88c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2523,6 +2523,7 @@ dependencies = [ "serde_json", "sha2", "sqlx", + "time 0.2.26", "toml", ] diff --git a/Cargo.toml b/Cargo.toml index b9142b021c..e116ffdb9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ lazy_static = "1.4.0" log = "0.4.14" rand = "0.8.3" regex = "1.5.4" +time = "0.2.26" # Database #sqlx = { version = "0.5.0", features = ["mysql", "runtime-tokio-rustls", "chrono"] } diff --git a/debug/curl/logout_roundtrip.py b/debug/curl/logout_roundtrip.py new file mode 100755 index 0000000000..5e347d6415 --- /dev/null +++ b/debug/curl/logout_roundtrip.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import requests +import re + +login_url = 'http://localhost:12345/login' +logout_url = 'http://localhost:12345/logout' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print('Login status: {}'.format(r.status_code)) + +print("==============") +print(r) + +print("==============") +print(r.headers) + +# May not be set due to cookie domain: +#r.cookies['session'] + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + +cookies = { 'session': session_cookie } + +r = requests.post(logout_url, cookies=cookies) + +print('Logout status: {}'.format(r.status_code)) + diff --git a/src/endpoints/users/logout.rs b/src/endpoints/users/logout.rs index e69de29bb2..3cd805e19b 100644 --- a/src/endpoints/users/logout.rs +++ b/src/endpoints/users/logout.rs @@ -0,0 +1,118 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::server_state::ServerState; +use crate::util::ip_address::get_request_ip; +use crate::util::tokens::random_token; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use crate::validations::username::validate_username; +use crate::validations::passwords::validate_passwords; +use actix_web::cookie::Cookie; +use sqlx::MySqlPool; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::endpoints::users::login::LoginSuccessResponse; + +#[derive(Serialize)] +pub struct LogoutSuccessResponse { + pub success: bool, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum LogoutError { + ServerError, +} + +impl ResponseError for LogoutError { + fn status_code(&self) -> StatusCode { + match *self { + LogoutError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + LogoutError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn logout_handler( + http_request: HttpRequest, + server_state: web::Data>) -> Result +{ + let delete_cookie = match http_request.cookie("session") { + Some(cookie) => { + match server_state.cookie_manager.decode_session_token(&cookie) { + Err(e) => { + warn!("Session cookie decode error: {:?}", e); + }, + Ok(session_token) => { + let _r = delete_session(&session_token, &server_state.mysql_pool).await; + } + } + + cookie // delete this cookie + }, + None => { + server_state.cookie_manager.delete_cookie() + } + }; + + let response = LogoutSuccessResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| LogoutError::ServerError)?; + + Ok(HttpResponse::Ok() + .del_cookie(&delete_cookie) + .content_type("application/json") + .body(body)) +} + +async fn delete_session(session_token: &str, mysql_pool: &MySqlPool) -> AnyhowResult<()> { + let query_result = sqlx::query!( + r#" +UPDATE user_sessions +SET deleted_at = CURRENT_TIMESTAMP() +WHERE token = ? + "#, + session_token.to_string(), + ) + .execute(mysql_pool) + .await; + + Ok(()) +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 3eef84a522..cc94c40acf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,6 +19,7 @@ use sqlx::mysql::MySqlPoolOptions; use crate::endpoints::users::create_account::create_account_handler; use crate::endpoints::users::login::login_handler; use crate::util::cookies::CookieManager; +use crate::endpoints::users::logout::logout_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -103,7 +104,9 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .wrap(Logger::new(&log_format) .exclude("/liveness") .exclude("/readiness")) - .wrap(DefaultHeaders::new().header("X-Backend-Hostname", &hostname)) + .wrap(DefaultHeaders::new() + .header("X-Backend-Hostname", &hostname) + .header("X-Build-Sha", "")) .service( web::resource("/create_account") .route(web::post().to(create_account_handler)) @@ -114,6 +117,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::post().to(login_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/logout") + .route(web::post().to(logout_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::scope("/foo") .service( @@ -154,17 +162,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> // Early access static path for assets //.service(Files::new("/static", static_asset_directory.clone()) // .index_file("FAKE_INDEX.HTML")) - /*.service( - web::resource("/login") - .route(web::post().to(login_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/logout") - .route(web::post().to(logout_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service(get_liveness) + /*.service(get_liveness) .service(get_models) .service(get_readiness) .service(get_root) @@ -177,18 +175,3 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> Ok(()) } - -pub async fn create_user(pool: &MySqlPool) -> AnyhowResult<()> { - let mut tx = pool.begin().await?; - let todo = sqlx::query("INSERT INTO badges (slug, title, description, image_url) VALUES ($1, $2, $3, $4)") - .bind("foo") - .bind("bar") - .bind("baz") - .bind("bin") - .fetch_one(&mut tx) - .await?; - - tx.commit().await?; - - Ok(()) -} diff --git a/src/util/cookies.rs b/src/util/cookies.rs index 827b9803ea..905d5d2761 100644 --- a/src/util/cookies.rs +++ b/src/util/cookies.rs @@ -3,12 +3,17 @@ use anyhow::anyhow; use crate::AnyhowResult; use hmac::Hmac; use hmac::NewMac; -use std::collections::BTreeMap; -use sha2::Sha256; use jwt::SignWithKey; +use jwt::VerifyWithKey; +use sha2::Sha256; +use std::collections::BTreeMap; +use std::ops::Sub; +use time::OffsetDateTime; const COOKIE_VERSION : u32 = 1; +const SESSION_COOKIE_NAME : &'static str = "session"; + #[derive(Clone)] pub struct CookieManager { cookie_domain: String, @@ -35,12 +40,31 @@ impl CookieManager { let jwt_string = claims.sign_with_key(&key)?; - Ok(Cookie::build("session", jwt_string) + Ok(Cookie::build(SESSION_COOKIE_NAME, jwt_string) .domain(&self.cookie_domain) .path("/") .secure(true) // HTTPS-only .http_only(true) // Not exposed to Javascript .finish()) } + + pub fn delete_cookie(&self) -> Cookie { + Cookie::build(SESSION_COOKIE_NAME, "DELETED") + .expires(OffsetDateTime::unix_epoch()) + .finish() + } + + pub fn decode_session_token(&self, session_cookie: &Cookie) -> AnyhowResult { + let key: Hmac = Hmac::new_varkey(self.hmac_secret.as_bytes()) + .map_err(|e| anyhow!("invalid hmac: {:?}", e))?; + + let cookie_contents = session_cookie.value().to_string(); + + let claims: BTreeMap = cookie_contents.verify_with_key(&key)?; + + let session_token = claims["session_token"].clone(); + + Ok(session_token) + } } From b7659d5b5d2589cb3b6a9bb783f1f53063fbc26f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 17 May 2021 23:42:35 -0400 Subject: [PATCH 0042/1042] better script output --- debug/curl/logout_roundtrip.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/debug/curl/logout_roundtrip.py b/debug/curl/logout_roundtrip.py index 5e347d6415..348445541f 100755 --- a/debug/curl/logout_roundtrip.py +++ b/debug/curl/logout_roundtrip.py @@ -13,15 +13,13 @@ payload = { r = requests.post(login_url, json=payload) -print('Login status: {}'.format(r.status_code)) +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) -print("==============") -print(r) - -print("==============") -print(r.headers) - -# May not be set due to cookie domain: +# NB: May not be set due to cookie domain: #r.cookies['session'] raw_set_cookie_header = r.headers['set-cookie'] @@ -32,5 +30,9 @@ cookies = { 'session': session_cookie } r = requests.post(logout_url, cookies=cookies) -print('Logout status: {}'.format(r.status_code)) +print("===== Logout =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) From 57e3fc4ebe3c85b1c9573a1fdc150f84406b1bed Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 May 2021 00:37:41 -0400 Subject: [PATCH 0043/1042] session checker, etc. etc. --- src/endpoints/users/logout.rs | 1 + src/endpoints/users/mod.rs | 1 + src/endpoints/users/session_info.rs | 0 src/main.rs | 20 +++++++-- src/server_state.rs | 3 ++ src/util/cookies.rs | 19 +++++++++ src/util/mod.rs | 3 +- src/util/session_checker.rs | 66 +++++++++++++++++++++++++++++ 8 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 src/endpoints/users/session_info.rs create mode 100644 src/util/session_checker.rs diff --git a/src/endpoints/users/logout.rs b/src/endpoints/users/logout.rs index 3cd805e19b..30eaf12ff2 100644 --- a/src/endpoints/users/logout.rs +++ b/src/endpoints/users/logout.rs @@ -108,6 +108,7 @@ async fn delete_session(session_token: &str, mysql_pool: &MySqlPool) -> AnyhowRe UPDATE user_sessions SET deleted_at = CURRENT_TIMESTAMP() WHERE token = ? +AND deleted_at IS NULL "#, session_token.to_string(), ) diff --git a/src/endpoints/users/mod.rs b/src/endpoints/users/mod.rs index 019ccd2199..09256cb3e6 100644 --- a/src/endpoints/users/mod.rs +++ b/src/endpoints/users/mod.rs @@ -1,3 +1,4 @@ pub mod create_account; pub mod login; pub mod logout; +pub mod session_info; diff --git a/src/endpoints/users/session_info.rs b/src/endpoints/users/session_info.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/main.rs b/src/main.rs index cc94c40acf..9145213b27 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,9 @@ +#![allow(dead_code)] +#![allow(unused_mut)] +#![allow(unused_imports)] +#![warn(unused_must_use)] +//#![allow(warnings)] + #[macro_use] extern crate lazy_static; #[macro_use] extern crate serde_derive; @@ -20,6 +26,7 @@ use crate::endpoints::users::create_account::create_account_handler; use crate::endpoints::users::login::login_handler; use crate::util::cookies::CookieManager; use crate::endpoints::users::logout::logout_handler; +use crate::util::session_checker::SessionChecker; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -35,15 +42,21 @@ async fn main() -> AnyhowResult<()> { .and_then(|h| h.into_string().ok()) .unwrap_or("storyteller-web-unknown".to_string()); - let db_connection_string = "mysql://root:root@localhost/storyteller"; + let db_connection_string = + easyenv::get_env_string_or_default( + "MYSQL_URL", + "mysql://root:root@localhost/storyteller"); let pool = MySqlPoolOptions::new() .max_connections(5) - .connect(db_connection_string) + .connect(&db_connection_string) .await?; let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); - let cookie_manager = CookieManager::new(".vo.codes", &hmac_secret); + let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); + + let cookie_manager = CookieManager::new(&cookie_domain, &hmac_secret); + let session_checker = SessionChecker::new(&cookie_manager); let server_state = ServerState { env_config: EnvConfig { @@ -53,6 +66,7 @@ async fn main() -> AnyhowResult<()> { hostname: server_hostname, mysql_pool: pool, cookie_manager, + session_checker, }; serve(server_state) diff --git a/src/server_state.rs b/src/server_state.rs index 88c4d60891..6069821760 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,5 +1,6 @@ use sqlx::MySqlPool; use crate::util::cookies::CookieManager; +use crate::util::session_checker::SessionChecker; /// State that is injected into every endpoint. #[derive(Clone)] @@ -12,6 +13,8 @@ pub struct ServerState { pub mysql_pool: MySqlPool, pub cookie_manager: CookieManager, + + pub session_checker: SessionChecker, } #[derive(Clone)] diff --git a/src/util/cookies.rs b/src/util/cookies.rs index 905d5d2761..adc0c14bc7 100644 --- a/src/util/cookies.rs +++ b/src/util/cookies.rs @@ -1,10 +1,12 @@ use actix_web::cookie::Cookie; +use actix_web::{HttpRequest, HttpMessage}; use anyhow::anyhow; use crate::AnyhowResult; use hmac::Hmac; use hmac::NewMac; use jwt::SignWithKey; use jwt::VerifyWithKey; +use log::{info, warn}; use sha2::Sha256; use std::collections::BTreeMap; use std::ops::Sub; @@ -66,5 +68,22 @@ impl CookieManager { Ok(session_token) } + + pub fn decode_session_token_from_request(&self, request: &HttpRequest) + -> AnyhowResult> + { + let cookie = match request.cookie(SESSION_COOKIE_NAME) { + None => return Ok(None), + Some(cookie) => cookie, + }; + + match self.decode_session_token(&cookie) { + Err(e) => { + warn!("Session cookie decode error: {:?}", e); + Err(anyhow!("Could not decode session cookie: {:?}", e)) + }, + Ok(payload) => Ok(Some(payload)), + } + } } diff --git a/src/util/mod.rs b/src/util/mod.rs index d169123365..9f8449793c 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,3 +1,4 @@ pub mod cookies; pub mod ip_address; -pub mod tokens; \ No newline at end of file +pub mod tokens; +pub mod session_checker; \ No newline at end of file diff --git a/src/util/session_checker.rs b/src/util/session_checker.rs new file mode 100644 index 0000000000..1ff6c6f1ff --- /dev/null +++ b/src/util/session_checker.rs @@ -0,0 +1,66 @@ +use crate::util::cookies::CookieManager; +use sqlx::MySqlPool; +use crate::AnyhowResult; +use actix_web::HttpRequest; + +#[derive(Clone)] +pub struct SessionChecker { + cookie_manager: CookieManager, +} + +pub struct SessionRecord { + session_token: String, + user_token: String, +} + +pub struct SessionUserRecord { + pub token: String, + pub username: String, + pub display_name: String, + pub email_address: String, + pub banned: bool, +} + +impl SessionChecker { + + pub fn new(cookie_manager: &CookieManager) -> Self { + Self { + cookie_manager: cookie_manager.clone(), + } + } + + pub async fn maybe_get_session(&self, request: &HttpRequest, pool: &MySqlPool) + -> AnyhowResult> + { + let session_token = match self.cookie_manager.decode_session_token_from_request(request)? { + None => return Ok(None), + Some(session_token) => session_token, + }; + + // NB: Lookup failure is Err(RowNotFound). + let session_record = sqlx::query_as!( + SessionRecord, + r#" +SELECT + token as session_token, + user_token +FROM user_sessions +WHERE token = ? +AND deleted_at IS NULL + "#, + session_token.to_string(), + ) + .fetch_one(pool) + .await?; // TODO: This will return error if it doesn't exist + + Ok(Some(session_record)) + } + + /*pub async fn maybe_get_user(request: &HttpRequest, pool: &MySqlPool) + -> AnyhowResult + { + Ok(SessionUserRecord { + + }) + }*/ +} \ No newline at end of file From 03b4e92d3a352d7e79beccaf8abb792456870afb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 May 2021 00:57:53 -0400 Subject: [PATCH 0044/1042] session info handler --- debug/curl/session_info.py | 41 ++++++++++ src/endpoints/users/session_info.rs | 114 ++++++++++++++++++++++++++++ src/main.rs | 6 ++ src/util/session_checker.rs | 35 +++++++-- 4 files changed, 188 insertions(+), 8 deletions(-) create mode 100755 debug/curl/session_info.py diff --git a/debug/curl/session_info.py b/debug/curl/session_info.py new file mode 100755 index 0000000000..7326a74da1 --- /dev/null +++ b/debug/curl/session_info.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import requests +import re + +login_url = 'http://localhost:12345/login' +session_info_url = 'http://localhost:12345/session' + + +r = requests.get(session_info_url) + +print("===== Session Info (logged out) =====") +print(r.content) + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +# NB: May not be set due to cookie domain: +#r.cookies['session'] + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + +cookies = { 'session': session_cookie } + +r = requests.get(session_info_url, cookies=cookies) + +print("===== Session Info (logged in) =====") +print(r.content) + diff --git a/src/endpoints/users/session_info.rs b/src/endpoints/users/session_info.rs index e69de29bb2..1d632cc38a 100644 --- a/src/endpoints/users/session_info.rs +++ b/src/endpoints/users/session_info.rs @@ -0,0 +1,114 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::server_state::ServerState; +use crate::util::ip_address::get_request_ip; +use crate::util::tokens::random_token; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use crate::validations::username::validate_username; +use crate::validations::passwords::validate_passwords; +use actix_web::cookie::Cookie; +use sqlx::MySqlPool; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::endpoints::users::login::LoginSuccessResponse; +use crate::util::session_checker::SessionRecord; + +#[derive(Serialize)] +pub struct UserInfo { + pub user_token: String, +} + +#[derive(Serialize)] +pub struct SessionInfoSuccessResponse { + pub success: bool, + pub logged_in: bool, + pub user: Option, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum SessionInfoError { + ServerError, +} + +impl ResponseError for SessionInfoError { + fn status_code(&self) -> StatusCode { + match *self { + SessionInfoError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + SessionInfoError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn session_info_handler( + http_request: HttpRequest, + server_state: web::Data>) -> Result +{ + let maybe_session = server_state.session_checker.maybe_get_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + SessionInfoError::ServerError + })?; + + let mut logged_in = false; + let mut user_info = None; + + match maybe_session { + None => {} + Some(session_data) => { + logged_in = true; + user_info = Some(UserInfo { + user_token: session_data.user_token.clone(), + }); + } + } + + let response = SessionInfoSuccessResponse { + success: true, + logged_in, + user: user_info, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| SessionInfoError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/main.rs b/src/main.rs index 9145213b27..84432345c0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,6 +27,7 @@ use crate::endpoints::users::login::login_handler; use crate::util::cookies::CookieManager; use crate::endpoints::users::logout::logout_handler; use crate::util::session_checker::SessionChecker; +use crate::endpoints::users::session_info::session_info_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -136,6 +137,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::post().to(logout_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/session") + .route(web::get().to(session_info_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::scope("/foo") .service( diff --git a/src/util/session_checker.rs b/src/util/session_checker.rs index 1ff6c6f1ff..92739e40c0 100644 --- a/src/util/session_checker.rs +++ b/src/util/session_checker.rs @@ -1,7 +1,10 @@ -use crate::util::cookies::CookieManager; -use sqlx::MySqlPool; -use crate::AnyhowResult; use actix_web::HttpRequest; +use anyhow::anyhow; +use crate::AnyhowResult; +use crate::util::cookies::CookieManager; +use log::{info, warn}; +use sqlx::MySqlPool; +use sqlx::error::Error::RowNotFound; #[derive(Clone)] pub struct SessionChecker { @@ -9,8 +12,8 @@ pub struct SessionChecker { } pub struct SessionRecord { - session_token: String, - user_token: String, + pub session_token: String, + pub user_token: String, } pub struct SessionUserRecord { @@ -38,7 +41,7 @@ impl SessionChecker { }; // NB: Lookup failure is Err(RowNotFound). - let session_record = sqlx::query_as!( + let maybe_session_record = sqlx::query_as!( SessionRecord, r#" SELECT @@ -51,9 +54,25 @@ AND deleted_at IS NULL session_token.to_string(), ) .fetch_one(pool) - .await?; // TODO: This will return error if it doesn't exist + .await; // TODO: This will return error if it doesn't exist - Ok(Some(session_record)) + match maybe_session_record { + Ok(session_record) => { + Ok(Some(session_record)) + }, + Err(err) => { + match err { + RowNotFound => { + warn!("Valid cookie; invalid session: {}", session_token); + Ok(None) + }, + _ => { + warn!("Session query error: {:?}", err); + Err(anyhow!("session query error: {:?}", err)) + } + } + } + } } /*pub async fn maybe_get_user(request: &HttpRequest, pool: &MySqlPool) From c264f09ee2af626f2844661e2a0dfa58e5939ab6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 May 2021 01:52:52 -0400 Subject: [PATCH 0045/1042] improve session endpoint --- src/endpoints/users/session_info.rs | 10 ++- src/util/session_checker.rs | 97 ++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 9 deletions(-) diff --git a/src/endpoints/users/session_info.rs b/src/endpoints/users/session_info.rs index 1d632cc38a..a24d757a4a 100644 --- a/src/endpoints/users/session_info.rs +++ b/src/endpoints/users/session_info.rs @@ -27,6 +27,8 @@ use crate::util::session_checker::SessionRecord; #[derive(Serialize)] pub struct UserInfo { pub user_token: String, + pub username: String, + pub display_name: String, } #[derive(Serialize)] @@ -79,7 +81,9 @@ pub async fn session_info_handler( http_request: HttpRequest, server_state: web::Data>) -> Result { - let maybe_session = server_state.session_checker.maybe_get_session(&http_request, &server_state.mysql_pool) + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) .await .map_err(|e| { warn!("Session checker error: {:?}", e); @@ -89,12 +93,14 @@ pub async fn session_info_handler( let mut logged_in = false; let mut user_info = None; - match maybe_session { + match maybe_user_session { None => {} Some(session_data) => { logged_in = true; user_info = Some(UserInfo { user_token: session_data.user_token.clone(), + username: session_data.username.to_string(), + display_name: session_data.display_name.to_string() }); } } diff --git a/src/util/session_checker.rs b/src/util/session_checker.rs index 92739e40c0..e6e41b6cbe 100644 --- a/src/util/session_checker.rs +++ b/src/util/session_checker.rs @@ -17,11 +17,22 @@ pub struct SessionRecord { } pub struct SessionUserRecord { - pub token: String, + pub user_token: String, pub username: String, pub display_name: String, pub email_address: String, - pub banned: bool, + pub email_confirmed: i8, + pub profile_markdown: String, + pub profile_rendered_html: String, + pub user_role_slug: String, + pub banned: i8, + pub dark_mode: String, + pub avatar_public_bucket_hash: Option, + pub disable_gravatar: i8, + pub hide_results_preference: i8, + pub discord_username: Option, + pub twitch_username: Option, + pub twitter_username: Option, } impl SessionChecker { @@ -75,11 +86,83 @@ AND deleted_at IS NULL } } - /*pub async fn maybe_get_user(request: &HttpRequest, pool: &MySqlPool) - -> AnyhowResult + pub async fn maybe_get_user_session(&self, request: &HttpRequest, pool: &MySqlPool) + -> AnyhowResult> { - Ok(SessionUserRecord { - }) - }*/ + let session_token = match self.cookie_manager.decode_session_token_from_request(request)? { + None => return Ok(None), + Some(session_token) => session_token, + }; + + /* + + pub token: String, + pub username: String, + pub display_name: String, + pub email_address: String, + pub email_confirmed: bool, + pub profile_markdown: String, + pub profile_rendered_html: String, + pub user_role_slug: String, + pub banned: bool, + pub dark_mode: String, + pub avatar_public_bucket_hash: Option, + pub disable_gravatar: bool, + pub hide_results_preference: bool, + pub discord_username: Option, + pub twitch_username: Option, + pub twitter_username: Option, + */ + // NB: Lookup failure is Err(RowNotFound). + let maybe_user_record = sqlx::query_as!( + SessionUserRecord, + r#" +SELECT + users.token as user_token, + username, + display_name, + email_address, + email_confirmed, + profile_markdown, + profile_rendered_html, + user_role_slug, + banned, + dark_mode, + avatar_public_bucket_hash, + disable_gravatar, + hide_results_preference, + discord_username, + twitch_username, + twitter_username +FROM users +LEFT OUTER JOIN user_sessions +ON users.token = user_sessions.user_token +WHERE user_sessions.token = ? +AND user_sessions.deleted_at IS NULL +AND users.deleted_at IS NULL + "#, + session_token.to_string(), + ) + .fetch_one(pool) + .await; // TODO: This will return error if it doesn't exist + + match maybe_user_record { + Ok(user_record) => { + Ok(Some(user_record)) + }, + Err(err) => { + match err { + RowNotFound => { + warn!("Valid cookie; invalid session: {}", session_token); + Ok(None) + }, + _ => { + warn!("Session query error: {:?}", err); + Err(anyhow!("session query error: {:?}", err)) + } + } + } + } + } } \ No newline at end of file From 0f68ce115b38d33a6d96e7ffbf7135ef696c32f0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 May 2021 06:16:39 -0400 Subject: [PATCH 0046/1042] session state persists; using nginx as local dev proxy --- src/main.rs | 6 ++++++ src/util/cookies.rs | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 84432345c0..276b38eddb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -91,8 +91,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> App::new() .app_data(server_state_arc.clone()) .wrap(Cors::default() + .allowed_origin("http://api.jungle.horse") .allowed_origin("http://api.vo.codes") .allowed_origin("http://jungle.horse") + .allowed_origin("http://jungle.horse:12345") + .allowed_origin("http://jungle.horse:7000") .allowed_origin("http://localhost:12345") .allowed_origin("http://localhost:3000") .allowed_origin("http://localhost:5555") @@ -104,15 +107,18 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .allowed_origin("http://vo.codes") .allowed_origin("http://vocodes.com") .allowed_origin("https://jungle.horse") + .allowed_origin("https://api.jungle.horse") .allowed_origin("https://mumble.stream") .allowed_origin("https://trumped.com") .allowed_origin("https://vo.codes") .allowed_origin("https://vocodes.com") .allowed_methods(vec!["GET", "POST", "OPTIONS"]) + .supports_credentials() .allowed_headers(vec![ http::header::ACCEPT, http::header::ACCESS_CONTROL_ALLOW_ORIGIN, // Tabulator Ajax http::header::CONTENT_TYPE, + http::header::ACCESS_CONTROL_ALLOW_CREDENTIALS, // https://stackoverflow.com/a/46412839 http::header::HeaderName::from_static("x-requested-with") // Tabulator Ajax sends ]) .max_age(3600)) diff --git a/src/util/cookies.rs b/src/util/cookies.rs index adc0c14bc7..966c7b895b 100644 --- a/src/util/cookies.rs +++ b/src/util/cookies.rs @@ -1,4 +1,4 @@ -use actix_web::cookie::Cookie; +use actix_web::cookie::{Cookie, SameSite}; use actix_web::{HttpRequest, HttpMessage}; use anyhow::anyhow; use crate::AnyhowResult; @@ -11,6 +11,7 @@ use sha2::Sha256; use std::collections::BTreeMap; use std::ops::Sub; use time::OffsetDateTime; +use log::LevelFilter::Off; const COOKIE_VERSION : u32 = 1; @@ -42,11 +43,17 @@ impl CookieManager { let jwt_string = claims.sign_with_key(&key)?; + let make_secure = !self.cookie_domain.to_lowercase().contains("jungle.horse") + && !self.cookie_domain.to_lowercase().contains("localhost"); + Ok(Cookie::build(SESSION_COOKIE_NAME, jwt_string) - .domain(&self.cookie_domain) - .path("/") - .secure(true) // HTTPS-only - .http_only(true) // Not exposed to Javascript + //.domain(&self.cookie_domain) + .secure(make_secure) // HTTPS-only + //.path("/") + //.http_only(true) // Not exposed to Javascript + //.expires(OffsetDateTime::now_utc() + time::Duration::days(365)) + .permanent() + //.same_site(SameSite::Lax) .finish()) } From d74cc614b84029e0a783dd30e3f10e66aae6403c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 May 2021 23:50:48 -0400 Subject: [PATCH 0047/1042] cleanup --- src/main.rs | 45 --------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/src/main.rs b/src/main.rs index 276b38eddb..9ecafa2868 100644 --- a/src/main.rs +++ b/src/main.rs @@ -148,56 +148,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(session_info_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - .service( - web::scope("/foo") - .service( - web::resource("/bar") - .route(web::get().to(|| HttpResponse::Ok())) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/bin") - .route(web::get().to(|| HttpResponse::Ok())) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/a") - .service( - web::resource("/b") - .route(web::get().to(|| HttpResponse::Ok())) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/c") - .route(web::get().to(|| HttpResponse::Ok())) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) }) .bind(bind_address)? .workers(num_workers) .run() .await?; - // Admin UI & old frontend - //.service(Files::new("/frontend", admin_asset_directory.clone()) - // .index_file("FAKE_INDEX.HTML")) - //.service(Files::new("/adminui", admin_asset_directory.clone()) - // .index_file("index.html")) - // Early access static path for assets - //.service(Files::new("/static", static_asset_directory.clone()) - // .index_file("FAKE_INDEX.HTML")) - /*.service(get_liveness) - .service(get_models) - .service(get_readiness) - .service(get_root) - .service(get_sentences) - .service(get_service_settings) - .service(get_speakers) - .service(get_early_access_speakers) - .service(get_dynamic_early_access_speakers) - .service(get_words);*/ - Ok(()) } From 06d597ebc5af2635521abc100cabca244d9ebe1b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 May 2021 23:58:31 -0400 Subject: [PATCH 0048/1042] .env for local dev and sqlx builds --- .env | 1 + 1 file changed, 1 insertion(+) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000000..070ce0d054 --- /dev/null +++ b/.env @@ -0,0 +1 @@ +DATABASE_URL=mysql://root:root@localhost/storyteller From 498ff80a5c86db5805111d29eb835ac94a06086e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 19 May 2021 00:15:35 -0400 Subject: [PATCH 0049/1042] Cache SQLX database info so it'll build without DB --- .env | 5 + Cargo.lock | 9 ++ Cargo.toml | 2 +- sqlx-data.json | 429 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 444 insertions(+), 1 deletion(-) create mode 100644 sqlx-data.json diff --git a/.env b/.env index 070ce0d054..9793767d64 100644 --- a/.env +++ b/.env @@ -1 +1,6 @@ DATABASE_URL=mysql://root:root@localhost/storyteller + +# SQLX normally requires a live database to reflect upon at build time. +# This setting turns that on and requires a `sql-data.json` cache file instead. +# See https://github.com/launchbadge/sqlx/tree/master/sqlx-cli#readme +SQLX_OFFLINE=true diff --git a/Cargo.lock b/Cargo.lock index beaca6a88c..c80ead4d0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -971,6 +971,9 @@ name = "either" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +dependencies = [ + "serde", +] [[package]] name = "encoding_rs" @@ -2210,6 +2213,7 @@ version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" dependencies = [ + "indexmap", "itoa", "ryu", "serde", @@ -2387,6 +2391,7 @@ dependencies = [ "rand 0.8.3", "rsa", "rustls", + "serde", "sha-1", "sha2", "smallvec", @@ -2410,8 +2415,12 @@ dependencies = [ "either", "futures", "heck", + "hex", + "once_cell", "proc-macro2", "quote", + "serde", + "serde_json", "sha2", "sqlx-core", "sqlx-rt", diff --git a/Cargo.toml b/Cargo.toml index e116ffdb9f..ee2c5d2e59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ time = "0.2.26" #error: one of the features ['runtime-actix-native-tls', 'runtime-async-std-native-tls', 'runtime-tokio-native-tls', 'runtime-actix-rustls', 'runtime-async-std-rustls', 'runtime-tokio-rustls'] must be enabled #sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls" ] } -sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-rustls" ] } +sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-rustls", "offline" ] } #sqlx = { version = "0.5.0", features = ["mysql", "runtime-actix-rustls", "chrono"] } diff --git a/sqlx-data.json b/sqlx-data.json new file mode 100644 index 0000000000..6134b4cce3 --- /dev/null +++ b/sqlx-data.json @@ -0,0 +1,429 @@ +{ + "db": "MySQL", + "26fbb143f69b900a722ce49e14942c6eb14b53dad85aa551f13e742dd6a21573": { + "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 11 + }, + "nullable": [] + } + }, + "345cbabbe69652c89ccd384ed7db44f70c53791488cf63721f9563291b45db3e": { + "query": "\nINSERT INTO user_sessions (\n token,\n user_token,\n ip_address_creation,\n expires_at\n)\nVALUES ( ?, ?, ?, NOW() + interval 1 year )\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "53d77ee58e9b1f7fb900914895c25bcccb9297378ff018c1e553b187d1956e92": { + "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE username = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "password_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 63, + "max_size": 60 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false + ] + } + }, + "86f9a76cc97b471a23fff5b9ea78f7fc7ecf7603215f5db5a1ca4e063568b054": { + "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE token = ?\nAND deleted_at IS NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "8d61bb77449a78044135a3d6e5009a3a27b4671b7e40501d318b05ae401f02a8": { + "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE email_address = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "password_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 63, + "max_size": 60 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false + ] + } + }, + "95a8d3bea2906e3c3607989e329fe925dea7e8307a4924d9b3d52d24193d77c4": { + "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_address,\n email_confirmed,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n banned,\n dark_mode,\n avatar_public_bucket_hash,\n disable_gravatar,\n hide_results_preference,\n discord_username,\n twitch_username,\n twitter_username\nFROM users\nLEFT OUTER JOIN user_sessions\nON users.token = user_sessions.user_token\nWHERE user_sessions.token = ?\nAND user_sessions.deleted_at IS NULL\nAND users.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "email_confirmed", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 5, + "name": "profile_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 6, + "name": "profile_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 7, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 8, + "name": "banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "dark_mode", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 40 + } + }, + { + "ordinal": 10, + "name": "avatar_public_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "hide_results_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "discord_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 14, + "name": "twitch_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 15, + "name": "twitter_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false, + true, + true, + true + ] + } + }, + "ca5efc3be0f09649df2add5270c06544133288a95ffcb9a0589efd4fe345efe7": { + "query": "\nINSERT INTO badges ( slug, title, description, image_url )\nVALUES ( ?, ?, ?, ? )\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 4 + }, + "nullable": [] + } + }, + "e73192a3e763a287d5737821443c5a81649f17e39978497b99c52e1e896a75f6": { + "query": "\nSELECT\n token as session_token,\n user_token\nFROM user_sessions\nWHERE token = ?\nAND deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "session_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + } +} \ No newline at end of file From 29f815578ff8a4ac1e7cf271a78522977e2f7ce8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 19 May 2021 00:16:26 -0400 Subject: [PATCH 0050/1042] update readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 417c2a92ed..d1cdaf6345 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,10 @@ To undo migrations at the current step: ```diesel migration redo``` +To save sqlx database state to cache file (necessary for builds): + +```SQLX_OFFLINE=true cargo sqlx prepare``` + Actix notes ----------- From c1fc3728c349e934bc8764762bce792f102fdb63 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 19 May 2021 00:24:32 -0400 Subject: [PATCH 0051/1042] Add docker file --- .github/workflows/docker-publish.yml | 65 ++++++++++++++++++++++++++++ Dockerfile | 49 +++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 Dockerfile diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000000..95e3e4a398 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,65 @@ +# Adapted from +# https://github.com/actions/starter-workflows/ci/docker-publish.yml +# If using base images, don't forget to add READ access, eg. from a package settings page like: +# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access +name: Publish Docker Container to GHCR +on: + push: + branches: + # NB: Default-branch doesn't work, despite Github's documentation. + #- $default-branch + - master +env: + IMAGE_NAME: storyteller-web + +jobs: + # Push image to GitHub Packages. + # See also https://docs.docker.com/docker-hub/builds/ + push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + permissions: + packages: write + contents: read + + steps: + - uses: actions/checkout@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + #username: ${{ github.repository_owner }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build image + run: docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" + + - name: Push image to GitHub Container Registry + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "$default-branch" ] && VERSION=latest + + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + + SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12) + + docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA + docker push $IMAGE_ID:$SHORT_SHA + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..4bff782ea3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,49 @@ +# Custom base image +# Make sure to add this repository so it has read acces to the base image: +# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access +FROM ghcr.io/storytold/docker-base-images-rust-ssl:latest as build +WORKDIR /tmp + +COPY Cargo.lock . +COPY Cargo.toml . +COPY src/ ./src +COPY protos/ ./protos + +RUN $HOME/.cargo/bin/cargo fetch + +#RUN SQLX_OFFLINE=true LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} $HOME/.cargo/bin/cargo build +RUN SQLX_OFFLINE=true LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} $HOME/.cargo/bin/cargo build --release + +# Final image +FROM ubuntu:xenial +WORKDIR / + +# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md +LABEL org.opencontainers.image.authors='bt@brand.io, echelon@gmail.com' +LABEL org.opencontainers.image.url='https://github.com/storytold/storyteller-web' +LABEL org.opencontainers.image.documentation='https://github.com/storytold/storyteller-web' +LABEL org.opencontainers.image.source='https://github.com/storytold/storyteller-web' + +# NB: Comment this out for non-debug images +# TODO: Figure out how this is done elsewhere with just the static binaries +RUN apt-get update \ + && apt-get install -y \ + curl \ + wget + +#COPY --from=build /tmp/target/debug/storyteller-web / +COPY --from=build /tmp/target/release/storyteller-web / + +# SSL certs are required for crypto +COPY --from=build /etc/ssl /etc/ssl + +# Required dynamically linked libraries +COPY --from=build /lib/x86_64-linux-gnu/libssl.* /lib/x86_64-linux-gnu/ +COPY --from=build /lib/x86_64-linux-gnu/libcrypto.* /lib/x86_64-linux-gnu/ + +# Make sure all the links resolve +RUN ldd storyteller-web + +EXPOSE 8080 +CMD LD_LIBRARY_PATH=/usr/lib /storyteller-web + From 7ad7437ef5e2f765ac8de88a1b92b606d2ea2c85 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 19 May 2021 00:31:50 -0400 Subject: [PATCH 0052/1042] bump --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d1cdaf6345..5aab8fe1fc 100644 --- a/README.md +++ b/README.md @@ -59,3 +59,4 @@ Notes / TODOs: * Actix/sqlx runtime compat: https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734 + From e0150a844ad38b47a3e0271cf21e0e1a16c93b26 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 19 May 2021 00:36:41 -0400 Subject: [PATCH 0053/1042] update readme with instructions --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 5aab8fe1fc..751f8fadf2 100644 --- a/README.md +++ b/README.md @@ -60,3 +60,10 @@ Notes / TODOs: https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734 +Docker builds +------------- + +The repository needs to be given read access to the base docker image: + +https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings + From 122f431bde51a9a93aa91bb73b32c9ab2cef6a29 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 19 May 2021 00:41:14 -0400 Subject: [PATCH 0054/1042] fixup dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4bff782ea3..afea4a1c23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,9 @@ WORKDIR /tmp COPY Cargo.lock . COPY Cargo.toml . +COPY sqlx-data.json . COPY src/ ./src -COPY protos/ ./protos +COPY migrations/ ./migrations RUN $HOME/.cargo/bin/cargo fetch From 15ed33d1c48f3f5f49350daf6d6a8963fbca9e7c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 19 May 2021 00:51:45 -0400 Subject: [PATCH 0055/1042] bump --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 751f8fadf2..1d0913bb65 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,4 @@ The repository needs to be given read access to the base docker image: https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings + From bdd196308b84c82850cad3f1b56110ab5bdfd85f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 19 May 2021 01:50:25 -0400 Subject: [PATCH 0056/1042] more logging --- .gitignore | 1 + src/main.rs | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/.gitignore b/.gitignore index 7de72bfe0f..d690ce6512 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*secret*sh *~ .env .idea/ diff --git a/src/main.rs b/src/main.rs index 9ecafa2868..3f19ba2024 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,11 +38,17 @@ pub type AnyhowResult = anyhow::Result; async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + info!("Obtaining hostname..."); + let server_hostname = hostname::get() .ok() .and_then(|h| h.into_string().ok()) .unwrap_or("storyteller-web-unknown".to_string()); + info!("Hostname: {}", &server_hostname); + + info!("Connecting to database..."); + let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", @@ -53,6 +59,8 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; + info!("Reading env vars and setting up utils..."); + let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); From 74c67ee1325fab5d265ab895a78e8cb33f336805 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 19 May 2021 20:29:57 -0400 Subject: [PATCH 0057/1042] Use nativetls (production issue?) and add nginx docs --- Cargo.lock | 220 +++++++++++++++++++++++++++++++++++------------------ Cargo.toml | 3 +- Dockerfile | 1 + README.md | 27 +++++++ 4 files changed, 176 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c80ead4d0c..b79981f40f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -451,6 +451,18 @@ dependencies = [ "event-listener", ] +[[package]] +name = "async-native-tls" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9e7a929bd34c68a82d58a4de7f86fffdaf97fb2af850162a7bb19dd7269b33" +dependencies = [ + "async-std", + "native-tls", + "thiserror", + "url 2.2.2", +] + [[package]] name = "async-process" version = "1.1.0" @@ -468,17 +480,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "async-rustls" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c86f33abd5a4f3e2d6d9251a9e0c6a7e52eb1113caf893dae8429bf4a53f378" -dependencies = [ - "futures-lite", - "rustls", - "webpki", -] - [[package]] name = "async-std" version = "1.9.0" @@ -821,6 +822,22 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + [[package]] name = "cpufeatures" version = "0.1.1" @@ -1042,6 +1059,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -1661,6 +1693,24 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "net2" version = "0.2.37" @@ -1797,6 +1847,39 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98" +dependencies = [ + "autocfg 1.0.1", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking" version = "2.0.0" @@ -2083,6 +2166,15 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -2093,21 +2185,6 @@ dependencies = [ "quick-error", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted", - "web-sys", - "winapi 0.3.9", -] - [[package]] name = "rsa" version = "0.4.0" @@ -2137,25 +2214,22 @@ dependencies = [ "semver", ] -[[package]] -name = "rustls" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" -dependencies = [ - "base64 0.13.0", - "log", - "ring", - "sct", - "webpki", -] - [[package]] name = "ryu" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -2163,13 +2237,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] -name = "sct" -version = "0.6.1" +name = "security-framework" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" dependencies = [ - "ring", - "untrusted", + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] @@ -2390,7 +2477,6 @@ dependencies = [ "percent-encoding 2.1.0", "rand 0.8.3", "rsa", - "rustls", "serde", "sha-1", "sha2", @@ -2400,8 +2486,6 @@ dependencies = [ "stringprep", "thiserror", "url 2.2.2", - "webpki", - "webpki-roots", "whoami", ] @@ -2434,8 +2518,9 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6ae97ab05063ed515cdc23d90253213aa24dda0a288c5ec079af3d10f9771bc" dependencies = [ - "async-rustls", + "async-native-tls", "async-std", + "native-tls", ] [[package]] @@ -2581,6 +2666,20 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand 0.8.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.9", +] + [[package]] name = "termcolor" version = "1.1.2" @@ -2863,12 +2962,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "url" version = "1.7.2" @@ -3045,25 +3138,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "webpki-roots" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" -dependencies = [ - "webpki", -] - [[package]] name = "wepoll-sys" version = "3.0.1" diff --git a/Cargo.toml b/Cargo.toml index ee2c5d2e59..ea91fb8ddf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,8 +32,7 @@ time = "0.2.26" #error: one of the features ['runtime-actix-native-tls', 'runtime-async-std-native-tls', 'runtime-tokio-native-tls', 'runtime-actix-rustls', 'runtime-async-std-rustls', 'runtime-tokio-rustls'] must be enabled #sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls" ] } -sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-rustls", "offline" ] } - +sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline" ] } #sqlx = { version = "0.5.0", features = ["mysql", "runtime-actix-rustls", "chrono"] } #diesel = { version = "1.4.4", features = ["mysql", "r2d2", "chrono"] } diff --git a/Dockerfile b/Dockerfile index afea4a1c23..224a785a73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ LABEL org.opencontainers.image.source='https://github.com/storytold/storyteller- # NB: Comment this out for non-debug images # TODO: Figure out how this is done elsewhere with just the static binaries +# Others: `mysql-client` RUN apt-get update \ && apt-get install -y \ curl \ diff --git a/README.md b/README.md index 1d0913bb65..f675e106fd 100644 --- a/README.md +++ b/README.md @@ -67,4 +67,31 @@ The repository needs to be given read access to the base docker image: https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings +Local Nginx Proxy +----------------- +Set up a local nginx to proxy to the frontend and backend so cookie issues aren't annoying + +Edit, `/etc/nginx/sites-enabled/storyteller` + +``` +server { + listen 80; + server_name api.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:12345; + proxy_redirect off; + } +} +server { + listen 80; + server_name jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:7000; + proxy_redirect off; + } +} + +``` From 793fd30c790427bf4321967f6617ed3b7b4c8dc2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 19 May 2021 20:49:57 -0400 Subject: [PATCH 0058/1042] document production migrations --- .gitignore | 1 + Cargo.toml | 14 ++++---------- README.md | 24 ++++++++++++++++++++---- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index d690ce6512..e726beb732 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .env .idea/ _ignored_schema.rs +production_database_secret_connect.sh secrets.toml secrets.yaml target/ diff --git a/Cargo.toml b/Cargo.toml index ea91fb8ddf..28beac36a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,16 +27,10 @@ regex = "1.5.4" time = "0.2.26" # Database -#sqlx = { version = "0.5.0", features = ["mysql", "runtime-tokio-rustls", "chrono"] } - -#error: one of the features ['runtime-actix-native-tls', 'runtime-async-std-native-tls', 'runtime-tokio-native-tls', 'runtime-actix-rustls', 'runtime-async-std-rustls', 'runtime-tokio-rustls'] must be enabled - -#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls" ] } -sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline" ] } - -#sqlx = { version = "0.5.0", features = ["mysql", "runtime-actix-rustls", "chrono"] } -#diesel = { version = "1.4.4", features = ["mysql", "r2d2", "chrono"] } -#diesel-derive-enum = { version = "1.0.0", features = ["mysql"] } +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } # Web actix-cors = "0.5.4" diff --git a/README.md b/README.md index f675e106fd..e4697c34b1 100644 --- a/README.md +++ b/README.md @@ -9,19 +9,35 @@ Local development To reset the entire database (drop, migrate), run: -```diesel database reset``` +``` +diesel database reset +``` To migrate at the current step and beyond: -```diesel migration run``` +``` +diesel migration run +``` To undo migrations at the current step: -```diesel migration redo``` +``` +diesel migration redo +``` To save sqlx database state to cache file (necessary for builds): -```SQLX_OFFLINE=true cargo sqlx prepare``` +``` +SQLX_OFFLINE=true cargo sqlx prepare +``` + +Production +---------- + +### Database migrations + +1. Set `DATABASE_URL` in `.env` +2. Run `diesel migration run` Actix notes ----------- From 1979d604b05db7c12f1608fa73d9d9d55a35916e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 00:07:40 -0400 Subject: [PATCH 0059/1042] index page --- Cargo.lock | 1 + src/endpoints/mod.rs | 1 + src/endpoints/root_index.rs | 11 +++++++++++ src/main.rs | 2 ++ 4 files changed, 15 insertions(+) create mode 100644 src/endpoints/root_index.rs diff --git a/Cargo.lock b/Cargo.lock index b79981f40f..1fa0885f1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2455,6 +2455,7 @@ dependencies = [ "bitflags", "byteorder", "bytes 1.0.1", + "chrono", "crc", "crossbeam-channel", "crossbeam-queue", diff --git a/src/endpoints/mod.rs b/src/endpoints/mod.rs index e3146d353b..daae90cd5b 100644 --- a/src/endpoints/mod.rs +++ b/src/endpoints/mod.rs @@ -1 +1,2 @@ +pub mod root_index; pub mod users; \ No newline at end of file diff --git a/src/endpoints/root_index.rs b/src/endpoints/root_index.rs new file mode 100644 index 0000000000..aa1c45e309 --- /dev/null +++ b/src/endpoints/root_index.rs @@ -0,0 +1,11 @@ +use actix_http::http::StatusCode; +use actix_web::{HttpResponse, HttpRequest, Responder, get}; +use log::info; + +#[get("/")] +pub async fn get_root_index() -> impl Responder { + info!("GET /"); + HttpResponse::build(StatusCode::OK) + .content_type("text/html; charset=utf-8") + .body("

hello friend

") +} diff --git a/src/main.rs b/src/main.rs index 3f19ba2024..0fe338815c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,6 +28,7 @@ use crate::util::cookies::CookieManager; use crate::endpoints::users::logout::logout_handler; use crate::util::session_checker::SessionChecker; use crate::endpoints::users::session_info::session_info_handler; +use crate::endpoints::root_index::get_root_index; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -156,6 +157,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(session_info_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(get_root_index) }) .bind(bind_address)? .workers(num_workers) From 6a04f03f32fa7a99018d228b593e46bb21cc0cae Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 00:14:40 -0400 Subject: [PATCH 0060/1042] 404 route --- src/endpoints/default_route_404.rs | 10 ++++++++++ src/endpoints/mod.rs | 1 + src/main.rs | 2 ++ 3 files changed, 13 insertions(+) create mode 100644 src/endpoints/default_route_404.rs diff --git a/src/endpoints/default_route_404.rs b/src/endpoints/default_route_404.rs new file mode 100644 index 0000000000..ad5f379a43 --- /dev/null +++ b/src/endpoints/default_route_404.rs @@ -0,0 +1,10 @@ +use actix_http::http::StatusCode; +use actix_web::{HttpResponse, HttpRequest, Responder, get}; +use log::warn; + +pub async fn default_route_404() -> impl Responder { + warn!("404 not found"); + HttpResponse::NotFound() + .content_type("text/html; charset=utf-8") + .body("

nope

") +} diff --git a/src/endpoints/mod.rs b/src/endpoints/mod.rs index daae90cd5b..d21a0a3a4b 100644 --- a/src/endpoints/mod.rs +++ b/src/endpoints/mod.rs @@ -1,2 +1,3 @@ +pub mod default_route_404; pub mod root_index; pub mod users; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 0fe338815c..4d5b87d10a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,6 +29,7 @@ use crate::endpoints::users::logout::logout_handler; use crate::util::session_checker::SessionChecker; use crate::endpoints::users::session_info::session_info_handler; use crate::endpoints::root_index::get_root_index; +use crate::endpoints::default_route_404::default_route_404; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -158,6 +159,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) .service(get_root_index) + .default_service( web::route().to(default_route_404)) }) .bind(bind_address)? .workers(num_workers) From 5e434498ac3e713cc7484119fa15f40565cbeb09 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 00:32:05 -0400 Subject: [PATCH 0061/1042] configurable bind address --- src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 4d5b87d10a..f21c9aec35 100644 --- a/src/main.rs +++ b/src/main.rs @@ -63,6 +63,7 @@ async fn main() -> AnyhowResult<()> { info!("Reading env vars and setting up utils..."); + let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); @@ -72,7 +73,7 @@ async fn main() -> AnyhowResult<()> { let server_state = ServerState { env_config: EnvConfig { num_workers: 4, - bind_address: DEFAULT_BIND_ADDRESS.to_string(), + bind_address, }, hostname: server_hostname, mysql_pool: pool, From bedfe75355ddab5d3c0e70c73159b89b101c7d25 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 02:35:34 -0400 Subject: [PATCH 0062/1042] enable-alpha cookie endpoint --- .env | 3 ++ src/endpoints/misc/enable_alpha.rs | 53 ++++++++++++++++++++++++++++++ src/endpoints/misc/mod.rs | 1 + src/endpoints/mod.rs | 1 + src/endpoints/root_index.rs | 4 +-- src/main.rs | 10 +++++- src/server_state.rs | 3 ++ 7 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 src/endpoints/misc/enable_alpha.rs create mode 100644 src/endpoints/misc/mod.rs diff --git a/.env b/.env index 9793767d64..17a2fc83df 100644 --- a/.env +++ b/.env @@ -4,3 +4,6 @@ DATABASE_URL=mysql://root:root@localhost/storyteller # This setting turns that on and requires a `sql-data.json` cache file instead. # See https://github.com/launchbadge/sqlx/tree/master/sqlx-cli#readme SQLX_OFFLINE=true + +# We can't serve HTTPS locally without self-signed certificate nonsense. +COOKIE_SECURE=false diff --git a/src/endpoints/misc/enable_alpha.rs b/src/endpoints/misc/enable_alpha.rs new file mode 100644 index 0000000000..7b5c11cb97 --- /dev/null +++ b/src/endpoints/misc/enable_alpha.rs @@ -0,0 +1,53 @@ +use actix_http::http::StatusCode; +use actix_web::{HttpResponse, HttpRequest, Responder, get, web, HttpMessage}; +use log::info; +use actix_http::cookie::Cookie; +use crate::server_state::ServerState; +use std::sync::Arc; +use actix_web::web::Query; + +const ALPHA_COOKIE_NAME : &'static str = "enable-alpha"; +const CONTENT_TYPE : &'static str = "text/html; charset=utf-8"; + +#[derive(Deserialize)] +struct QueryFields { + enable: Option, +} + +#[get("/alpha")] +pub async fn enable_alpha(http_request: HttpRequest, + query: Query, + server_state: web::Data>) -> impl Responder +{ + info!("GET /alpha"); + + let cookie = Cookie::build(ALPHA_COOKIE_NAME, "true") + .domain(&server_state.env_config.cookie_domain) + .secure(server_state.env_config.cookie_secure) // HTTPS-only + .http_only(false) // This is meant to be exposed to Javascript! + .permanent() + .finish(); + + match query.enable { + None => { + let cookie_exists = http_request.cookie(ALPHA_COOKIE_NAME).is_some(); + + HttpResponse::build(StatusCode::OK) + .content_type(CONTENT_TYPE) + .body(format!("

use ?enable={{true, false}}; current `{}` cookie state = {}

", + ALPHA_COOKIE_NAME, cookie_exists)) + } + Some(true) => { + HttpResponse::build(StatusCode::OK) + .content_type(CONTENT_TYPE) + .cookie(cookie) + .body(format!("

?enable=true; setting `{}` cookie

", ALPHA_COOKIE_NAME)) + } + Some(false) => { + HttpResponse::build(StatusCode::OK) + .content_type(CONTENT_TYPE) + .del_cookie(&cookie) + .body(format!("

?enable=false; unsetting `{}` mode cookie

", ALPHA_COOKIE_NAME)) + } + } +} diff --git a/src/endpoints/misc/mod.rs b/src/endpoints/misc/mod.rs new file mode 100644 index 0000000000..c43dd66c3d --- /dev/null +++ b/src/endpoints/misc/mod.rs @@ -0,0 +1 @@ +pub mod enable_alpha; \ No newline at end of file diff --git a/src/endpoints/mod.rs b/src/endpoints/mod.rs index d21a0a3a4b..a3cbbff247 100644 --- a/src/endpoints/mod.rs +++ b/src/endpoints/mod.rs @@ -1,3 +1,4 @@ pub mod default_route_404; +pub mod misc; pub mod root_index; pub mod users; \ No newline at end of file diff --git a/src/endpoints/root_index.rs b/src/endpoints/root_index.rs index aa1c45e309..3f49e31b9f 100644 --- a/src/endpoints/root_index.rs +++ b/src/endpoints/root_index.rs @@ -1,10 +1,10 @@ use actix_http::http::StatusCode; use actix_web::{HttpResponse, HttpRequest, Responder, get}; -use log::info; +use log::debug; #[get("/")] pub async fn get_root_index() -> impl Responder { - info!("GET /"); + debug!("GET /"); // NB: Google load balancer hits this a lot, and it spams. HttpResponse::build(StatusCode::OK) .content_type("text/html; charset=utf-8") .body("

hello friend

") diff --git a/src/main.rs b/src/main.rs index f21c9aec35..bf3c09f02a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,6 +30,7 @@ use crate::util::session_checker::SessionChecker; use crate::endpoints::users::session_info::session_info_handler; use crate::endpoints::root_index::get_root_index; use crate::endpoints::default_route_404::default_route_404; +use crate::endpoints::misc::enable_alpha::enable_alpha; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -64,16 +65,22 @@ async fn main() -> AnyhowResult<()> { info!("Reading env vars and setting up utils..."); let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); + let num_workers = easyenv::get_env_num("NUM_WORKERS", 4)?; let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); + let cookie_secure = easyenv::get_env_bool_or_default("COOKIE_SECURE", true); + let cookie_http_only = easyenv::get_env_bool_or_default("COOKIE_HTTP_ONLY", true); let cookie_manager = CookieManager::new(&cookie_domain, &hmac_secret); let session_checker = SessionChecker::new(&cookie_manager); let server_state = ServerState { env_config: EnvConfig { - num_workers: 4, + num_workers, bind_address, + cookie_domain, + cookie_secure, + cookie_http_only, }, hostname: server_hostname, mysql_pool: pool, @@ -160,6 +167,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) .service(get_root_index) + .service(enable_alpha) .default_service( web::route().to(default_route_404)) }) .bind(bind_address)? diff --git a/src/server_state.rs b/src/server_state.rs index 6069821760..7cc0801642 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -22,4 +22,7 @@ pub struct EnvConfig { // Number of thread workers. pub num_workers: usize, pub bind_address: String, + pub cookie_domain: String, + pub cookie_secure: bool, + pub cookie_http_only: bool, } From 61c583753e879fcf5c30ce77634c205852865112 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 18:36:14 -0400 Subject: [PATCH 0063/1042] schema updates --- .../up.sql | 93 ++++++++++++++----- migrations/2021-05-15-071601_assets/up.sql | 65 ++++++++----- 2 files changed, 112 insertions(+), 46 deletions(-) diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index e33886c818..5c096a260f 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -14,33 +14,55 @@ CREATE TABLE tts_model_upload_jobs ( -- Users can upload their own private models. -- They can choose to make them public later. + -- NB: Not sure I'm going to use this at first (or at all) is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, -- The name of the voice voice_name VARCHAR(255) NOT NULL, - - -- The speaker (in the case of cartoon characters) + -- The voice actor (in the case of cartoon characters) voice_actor_name VARCHAR(255) DEFAULT NULL, - -- Jobs begin as "pending", then transition to other states. - -- Pending -> Started -> Complete - -- \-> Failed -> Started -> { Complete, Failed, Dead } - status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending', - - -- We can track this against a "max_attempt_count" - attempt_count INT(3) NOT NULL DEFAULT 0, - -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. tts_model_type ENUM( - 'not-set', - 'tacotron2', - 'glowtts', - 'glowtts-vocodes' + 'not-set', + 'tacotron2', + 'glowtts', + 'glowtts-vocodes' ) NOT NULL DEFAULT 'not-set', -- If we need to download the file from Google Drive. - google_drive_url VARCHAR(512) DEFAULT NULL, + download_url VARCHAR(512) DEFAULT NULL, + + -- NB: DO NOT SORT! + download_url_type ENUM( + 'not-set', + 'google-drive', + 'web' + ) NOT NULL DEFAULT 'not-set', + + -- Jobs begin as "pending", then transition to other states. + -- + -- * Pending = job is ready to go + -- * Started = job is running + -- * Complete_Success = job is done (success) + -- * Complete_Failure = job is done (failure) + -- * Attempt_Failed = job failed but may retry. + -- * Dead = job failed permanently. + -- + -- Pending -> Started -> Complete_Success + -- |-> Complete_Failure + -- \-> Attempt_Failed -> Started -> { Complete, Failed, Dead } + status ENUM( + 'pending', + 'started', + 'complete_success', + 'complete_failure', + 'attempt_failed', + 'dead') NOT NULL DEFAULT 'pending', + + -- We can track this against a "max_attempt_count" + attempt_count INT(3) NOT NULL DEFAULT 0, -- If there is a failure, tell the user why. failure_reason VARCHAR(512) DEFAULT NULL, @@ -75,16 +97,9 @@ CREATE TABLE w2l_template_upload_jobs ( -- Users can upload their own private templates. -- They can choose to make them public later. + -- NB: Not sure I'm going to use this at first (or at all); would be separate from mod approval. is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, - -- Jobs begin as "pending", then transition to other states. - -- Pending -> Started -> Complete - -- \-> Failed -> Started -> { Complete, Failed, Dead } - status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending', - - -- We can track this against a "max_attempt_count" - attempt_count INT(3) NOT NULL DEFAULT 0, - -- NB: DO NOT SORT! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. template_type ENUM( @@ -93,7 +108,37 @@ CREATE TABLE w2l_template_upload_jobs ( ) NOT NULL, -- If we need to download the file from Google Drive. - google_drive_url VARCHAR(512) DEFAULT NULL, + download_url VARCHAR(512) DEFAULT NULL, + + -- NB: DO NOT SORT! + download_url_type ENUM( + 'not-set', + 'google-drive', + 'web' + ) NOT NULL DEFAULT 'not-set', + + -- Jobs begin as "pending", then transition to other states. + -- + -- * Pending = job is ready to go + -- * Started = job is running + -- * Complete_Success = job is done (success) + -- * Complete_Failure = job is done (failure) + -- * Attempt_Failed = job failed but may retry. + -- * Dead = job failed permanently. + -- + -- Pending -> Started -> Complete_Success + -- |-> Complete_Failure + -- \-> Attempt_Failed -> Started -> { Complete, Failed, Dead } + status ENUM( + 'pending', + 'started', + 'complete_success', + 'complete_failure', + 'attempt_failed', + 'dead') NOT NULL DEFAULT 'pending', + + -- We can track this against a "max_attempt_count" + attempt_count INT(3) NOT NULL DEFAULT 0, -- If there is a failure, tell the user why. failure_reason VARCHAR(512) DEFAULT NULL, diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index c208d5c225..181c2b5430 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -8,7 +8,14 @@ CREATE TABLE tts_models ( -- Effective "primary key" (PUBLIC) token VARCHAR(32) NOT NULL, - -- A combination of ['username' + 'voice-name'] + -- The person that created the template. + creator_user_token VARCHAR(32) NOT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- We can set as a combination of ['username' + 'voice-name'] -- There can be public aliases for voices, eg. a voice's default model. -- A user can change this. -- As such, these should not be foreign keys. @@ -16,13 +23,28 @@ CREATE TABLE tts_models ( -- Optional Pointer to a newer version of the voice -- If there's a newer version, we can disable this one. - updated_model_token VARCHAR(32) DEFAULT NULL, + maybe_updated_model_token VARCHAR(32) DEFAULT NULL, - -- We an disable a voice for a variety of reasons - -- In this case, the original author disables it. - user_disabled BOOLEAN NOT NULL DEFAULT FALSE, - -- In this case, a moderator author disables it. - mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, + -- (Maybe) users can upload their own private models. + -- They can choose to make them public later. + is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- In this case, a moderator disables it. + is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, + + -- If a moderator has comments. + maybe_mod_comments VARCHAR(255) DEFAULT NULL, + -- The last moderator that made changes. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. @@ -34,10 +56,10 @@ CREATE TABLE tts_models ( ) NOT NULL DEFAULT 'not-set', -- Can be linked to a well-known voice - voice_token VARCHAR(32) DEFAULT NULL, + maybe_voice_token VARCHAR(32) DEFAULT NULL, -- The name of the voice. - -- If voice_token is set, it's authoritative. + -- If voice_token is set, then it's authoritative instead. voice_name VARCHAR(255) NOT NULL, -- If the voice is "happy" or a singer "a-capella", etc. @@ -48,46 +70,45 @@ CREATE TABLE tts_models ( -- If voice_token is set, it's authoritative. voice_actor_name VARCHAR(255) DEFAULT NULL, - -- Users can upload their own private models. - -- They can choose to make them public later. - is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, - -- The description of the model in markdown. description_markdown TEXT NOT NULL, -- Generated HTML (not user-editable). description_rendered_html TEXT NOT NULL, - -- The person that created the template. - creator_user_token VARCHAR(32) NOT NULL, - - -- For abuse tracking. - -- Wide enough for IPv4/6 - creator_ip_address VARCHAR(40) NOT NULL, - -- The filename that was used at upload time. original_filename CHAR(255) NOT NULL, -- The pytorch model -- For now, this will be a hash of the file contents. + -- NB: NOT UNIQUE! We can allow duplicate uploads. private_bucket_hash CHAR(32) NOT NULL, -- Calculated average, on a scale of 0-100 -- Null with zero ratings. calculated_average_score INT(3) DEFAULT NULL, - calculated_score_count INT(10) NOT NULL DEFAULT 0, + -- Other metrics + calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0, + calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- If this is removed by a mod. + -- It completely disappears from the system. deleted_at TIMESTAMP NULL, -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (token), UNIQUE KEY (updatable_slug), - KEY index_private_bucket_hash (private_bucket_hash) + KEY fk_maybe_updated_model_token (maybe_updated_model_token), + KEY fk_creator_user_token (creator_user_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), + KEY fk_maybe_voice_token (maybe_voice_token), + KEY index_creator_ip_address (creator_ip_address), + KEY index_private_bucket_hash (private_bucket_hash), + KEY index_maybe_mod_user_token (maybe_mod_user_token) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From bcc83df3bf586d884fbde54f937dbf9abb247c49 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 19:24:14 -0400 Subject: [PATCH 0064/1042] model refinements --- migrations/2021-05-15-071601_assets/down.sql | 3 +- migrations/2021-05-15-071601_assets/up.sql | 124 +++++++--------- migrations/2021-05-15-071602_voices/down.sql | 5 + migrations/2021-05-15-071602_voices/up.sql | 140 +++++++++++++++++++ 4 files changed, 199 insertions(+), 73 deletions(-) create mode 100644 migrations/2021-05-15-071602_voices/down.sql create mode 100644 migrations/2021-05-15-071602_voices/up.sql diff --git a/migrations/2021-05-15-071601_assets/down.sql b/migrations/2021-05-15-071601_assets/down.sql index e1d28ed709..872ae266d7 100644 --- a/migrations/2021-05-15-071601_assets/down.sql +++ b/migrations/2021-05-15-071601_assets/down.sql @@ -2,5 +2,4 @@ # noinspection SqlResolveForFile drop table tts_models; -drop table w2l_templates; -drop table voices; \ No newline at end of file +drop table w2l_templates; \ No newline at end of file diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 181c2b5430..9f7348fedf 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -25,10 +25,6 @@ CREATE TABLE tts_models ( -- If there's a newer version, we can disable this one. maybe_updated_model_token VARCHAR(32) DEFAULT NULL, - -- (Maybe) users can upload their own private models. - -- They can choose to make them public later. - is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, - -- (THIS MIGHT NOT BE USED) -- NB: DO NOT SORT! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. @@ -38,14 +34,6 @@ CREATE TABLE tts_models ( 'private' ) NOT NULL DEFAULT 'public', - -- In this case, a moderator disables it. - is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, - - -- If a moderator has comments. - maybe_mod_comments VARCHAR(255) DEFAULT NULL, - -- The last moderator that made changes. - maybe_mod_user_token VARCHAR(32) DEFAULT NULL, - -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. tts_model_type ENUM( @@ -56,20 +44,23 @@ CREATE TABLE tts_models ( ) NOT NULL DEFAULT 'not-set', -- Can be linked to a well-known voice - maybe_voice_token VARCHAR(32) DEFAULT NULL, + maybe_subject_token VARCHAR(32) DEFAULT NULL, + maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, -- The name of the voice. -- If voice_token is set, then it's authoritative instead. voice_name VARCHAR(255) NOT NULL, - -- If the voice is "happy" or a singer "a-capella", etc. - -- If voice_token is set, it's authoritative. - voice_characteristic VARCHAR(255) DEFAULT NULL, - -- The speaker (in the case of cartoon characters) -- If voice_token is set, it's authoritative. voice_actor_name VARCHAR(255) DEFAULT NULL, + -- If the voice is "happy" or a singer "a-capella", etc. + voice_characteristic VARCHAR(255) DEFAULT NULL, + + -- Eg. ("poor quality", "trained longer", etc.) + extra_model_disambiguation VARCHAR(255) DEFAULT NULL, + -- The description of the model in markdown. description_markdown TEXT NOT NULL, @@ -77,7 +68,7 @@ CREATE TABLE tts_models ( description_rendered_html TEXT NOT NULL, -- The filename that was used at upload time. - original_filename CHAR(255) NOT NULL, + original_filename VARCHAR(255) NOT NULL, -- The pytorch model -- For now, this will be a hash of the file contents. @@ -87,10 +78,19 @@ CREATE TABLE tts_models ( -- Calculated average, on a scale of 0-100 -- Null with zero ratings. calculated_average_score INT(3) DEFAULT NULL, + -- Other metrics calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0, calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0, + -- In this case, a moderator disables it. + is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, + + -- If a moderator has comments. + maybe_mod_comments VARCHAR(255) DEFAULT NULL, + -- The last moderator that made changes. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -105,10 +105,12 @@ CREATE TABLE tts_models ( KEY fk_maybe_updated_model_token (maybe_updated_model_token), KEY fk_creator_user_token (creator_user_token), KEY fk_maybe_mod_user_token (maybe_mod_user_token), - KEY fk_maybe_voice_token (maybe_voice_token), + KEY fk_maybe_subject_token (maybe_subject_token), + KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), KEY index_creator_ip_address (creator_ip_address), + KEY index_creator_set_visibility (creator_set_visibility), KEY index_private_bucket_hash (private_bucket_hash), - KEY index_maybe_mod_user_token (maybe_mod_user_token) + KEY index_is_mod_disabled (is_mod_disabled) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -131,12 +133,21 @@ CREATE TABLE w2l_templates ( 'video' ) NOT NULL DEFAULT 'not-set', - -- The title of the template. + -- Can be linked to a well-known voice + maybe_subject_token VARCHAR(32) DEFAULT NULL, + maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, + + -- The title of the template. title CHAR(255) NOT NULL, - -- Users can upload their own private templates. - -- They can choose to make them public later. - is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', -- The description of the template in markdown. description_markdown TEXT NOT NULL, @@ -160,6 +171,15 @@ CREATE TABLE w2l_templates ( -- For the thumbnail we show. public_bucket_hash CHAR(32) NOT NULL, + -- In this case, a moderator disables it. + is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, + + -- If a moderator has comments. + maybe_mod_comments VARCHAR(255) DEFAULT NULL, + + -- The last moderator that made changes. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -170,54 +190,16 @@ CREATE TABLE w2l_templates ( PRIMARY KEY (id), UNIQUE KEY (token), UNIQUE KEY (updatable_slug), + KEY fk_creator_user_token (creator_user_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), + KEY fk_maybe_subject_token (maybe_subject_token), + KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), + KEY index_template_type (template_type), + KEY index_creator_ip_address (creator_ip_address), KEY index_private_bucket_hash (private_bucket_hash), - KEY index_public_bucket_hash (public_bucket_hash) + KEY index_public_bucket_hash (public_bucket_hash), + KEY index_creator_set_visibility (creator_set_visibility), + KEY index_is_mod_disabled (is_mod_disabled) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; -CREATE TABLE voices ( - -- Not used for anything except replication. - id BIGINT(20) NOT NULL AUTO_INCREMENT, - - -- Effective "primary key" (PUBLIC) - token VARCHAR(32) NOT NULL, - - -- The URL we access the voice at. - -- These should be stable, but could be changed if necessary. - -- As such, these should not be foreign keys. - updatable_slug VARCHAR(64) NOT NULL, - - -- We can assign an exemplary model to the voice - default_model_token VARCHAR(32) DEFAULT NULL, - - -- If a moderator author disables it. - -- This should prevent the voice from showing up in lists. - mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, - - -- The name of the voice - voice_name VARCHAR(255) NOT NULL, - - -- If the voice is "happy" or a singer "a-capella", etc. - voice_characteristic VARCHAR(255) DEFAULT NULL, - - -- The speaker (in the case of cartoon characters) - voice_actor_name VARCHAR(255) DEFAULT NULL, - - -- The 800x600 image - image_banner_public_bucket_hash CHAR(32) DEFAULT NULL, - - -- The square avatar image - image_square_public_bucket_hash CHAR(32) DEFAULT NULL, - - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - - -- If this is removed by a mod. - deleted_at TIMESTAMP NULL, - - -- INDICES -- - PRIMARY KEY (id), - UNIQUE KEY (token), - UNIQUE KEY (updatable_slug) - -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-071602_voices/down.sql b/migrations/2021-05-15-071602_voices/down.sql new file mode 100644 index 0000000000..199e0c7583 --- /dev/null +++ b/migrations/2021-05-15-071602_voices/down.sql @@ -0,0 +1,5 @@ +# noinspection SqlNoDataSourceInspectionForFile +# noinspection SqlResolveForFile + +drop table subjects; +drop table voices; diff --git a/migrations/2021-05-15-071602_voices/up.sql b/migrations/2021-05-15-071602_voices/up.sql new file mode 100644 index 0000000000..1d5eed5fc4 --- /dev/null +++ b/migrations/2021-05-15-071602_voices/up.sql @@ -0,0 +1,140 @@ +# noinspection SqlNoDataSourceInspectionForFile +# noinspection SqlResolveForFile + +-- These can be people, cartoon characters, fictional characters, etc. +-- Subjects may be "played" by different entities. +CREATE TABLE subjects ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- The name of the subject + -- Might have parens to disambiguate, eg. "JFK (Clone High)" + subject_name VARCHAR(255) NOT NULL, + + -- The URL we access the voice at. + -- These should be stable, but could be changed if necessary. + -- As such, these should not be foreign keys. + updatable_slug VARCHAR(64) NOT NULL, + + -- The type of subject + -- For eg. "JFK (Clone High)", that would be a separate subject. + type ENUM( + 'not-set', + 'real-person', + 'fictional-character' + ) NOT NULL DEFAULT 'not-set', + + -- The description of the model in markdown. + description_markdown TEXT NOT NULL, + + -- Generated HTML (not user-editable). + description_rendered_html TEXT NOT NULL, + + -- The person that created the template. + creator_user_token VARCHAR(32) NOT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- The person that created the template. + -- More than just mods may update. + maybe_updater_user_token VARCHAR(32) NOT NULL, + + -- More than just mods may update. + -- For abuse tracking. + -- Wide enough for IPv4/6 + maybe_updater_ip_address VARCHAR(40) NOT NULL, + + -- We can assign an exemplary model to the subject + maybe_default_tts_model_token VARCHAR(32) DEFAULT NULL, + maybe_default_w2l_template_token VARCHAR(32) DEFAULT NULL, + + -- The 800x600 image + maybe_image_banner_public_bucket_hash CHAR(32) DEFAULT NULL, + + -- The square avatar image + maybe_image_square_public_bucket_hash CHAR(32) DEFAULT NULL, + + -- If a moderator author disables it. + -- This should prevent the voice from showing up in lists. + -- Eg. if someone creates "Hitler", we'd want to prevent it from showing up. + is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, + + -- If a moderator has comments. + maybe_mod_comments VARCHAR(255) DEFAULT NULL, + -- The last moderator that made changes. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod. + -- It completely disappears from the system. + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + UNIQUE KEY (updatable_slug), + KEY fk_creator_user_token (creator_user_token), + KEY fk_maybe_updater_user_token (maybe_updater_user_token), + KEY fk_maybe_default_tts_model_token (maybe_default_tts_model_token), + KEY fk_maybe_default_w2l_template_token (maybe_default_w2l_template_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), + KEY index_type (type), + KEY index_creator_ip_address (creator_ip_address), + KEY index_maybe_updater_ip_address (maybe_updater_ip_address), + KEY index_is_mod_disabled (is_mod_disabled) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +CREATE TABLE voices ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- The URL we access the voice at. + -- These should be stable, but could be changed if necessary. + -- As such, these should not be foreign keys. + updatable_slug VARCHAR(64) NOT NULL, + + -- We can assign an exemplary model to the voice + default_model_token VARCHAR(32) DEFAULT NULL, + + -- If a moderator author disables it. + -- This should prevent the voice from showing up in lists. + mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, + + -- The name of the voice + voice_name VARCHAR(255) NOT NULL, + + -- If the voice is "happy" or a singer "a-capella", etc. + voice_characteristic VARCHAR(255) DEFAULT NULL, + + -- The speaker (in the case of cartoon characters) + voice_actor_name VARCHAR(255) DEFAULT NULL, + + -- The 800x600 image + image_banner_public_bucket_hash CHAR(32) DEFAULT NULL, + + -- The square avatar image + image_square_public_bucket_hash CHAR(32) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod. + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + UNIQUE KEY (updatable_slug) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 9cb3122723a872b24130daeeffefd2570046a7e9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 19:29:24 -0400 Subject: [PATCH 0065/1042] model updates --- migrations/2021-05-15-071601_assets/up.sql | 31 ++++--- migrations/2021-05-15-071602_voices/down.sql | 3 +- migrations/2021-05-15-071602_voices/up.sql | 92 ++++++++++---------- 3 files changed, 66 insertions(+), 60 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 9f7348fedf..88a021b73d 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -25,15 +25,6 @@ CREATE TABLE tts_models ( -- If there's a newer version, we can disable this one. maybe_updated_model_token VARCHAR(32) DEFAULT NULL, - -- (THIS MIGHT NOT BE USED) - -- NB: DO NOT SORT! - -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. - creator_set_visibility ENUM( - 'public', - 'hidden', - 'private' - ) NOT NULL DEFAULT 'public', - -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. tts_model_type ENUM( @@ -47,6 +38,15 @@ CREATE TABLE tts_models ( maybe_subject_token VARCHAR(32) DEFAULT NULL, maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + -- The name of the voice. -- If voice_token is set, then it's authoritative instead. voice_name VARCHAR(255) NOT NULL, @@ -127,6 +127,10 @@ CREATE TABLE w2l_templates ( -- As such, these should not be foreign keys. updatable_slug VARCHAR(64) NOT NULL, + -- Optional Pointer to a newer version of the template + -- If there's a newer version, we can disable this one. + maybe_updated_template_token VARCHAR(32) DEFAULT NULL, + template_type ENUM( 'not-set', 'image', @@ -137,9 +141,6 @@ CREATE TABLE w2l_templates ( maybe_subject_token VARCHAR(32) DEFAULT NULL, maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, - -- The title of the template. - title CHAR(255) NOT NULL, - -- (THIS MIGHT NOT BE USED) -- NB: DO NOT SORT! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. @@ -149,6 +150,9 @@ CREATE TABLE w2l_templates ( 'private' ) NOT NULL DEFAULT 'public', + -- The title of the template. + title CHAR(255) NOT NULL, + -- The description of the template in markdown. description_markdown TEXT NOT NULL, @@ -190,15 +194,16 @@ CREATE TABLE w2l_templates ( PRIMARY KEY (id), UNIQUE KEY (token), UNIQUE KEY (updatable_slug), + KEY fk_maybe_updated_template_token (maybe_updated_template_token), KEY fk_creator_user_token (creator_user_token), KEY fk_maybe_mod_user_token (maybe_mod_user_token), KEY fk_maybe_subject_token (maybe_subject_token), KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), KEY index_template_type (template_type), KEY index_creator_ip_address (creator_ip_address), + KEY index_creator_set_visibility (creator_set_visibility), KEY index_private_bucket_hash (private_bucket_hash), KEY index_public_bucket_hash (public_bucket_hash), - KEY index_creator_set_visibility (creator_set_visibility), KEY index_is_mod_disabled (is_mod_disabled) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-071602_voices/down.sql b/migrations/2021-05-15-071602_voices/down.sql index 199e0c7583..96d002ce10 100644 --- a/migrations/2021-05-15-071602_voices/down.sql +++ b/migrations/2021-05-15-071602_voices/down.sql @@ -2,4 +2,5 @@ # noinspection SqlResolveForFile drop table subjects; -drop table voices; +--drop table voices; + diff --git a/migrations/2021-05-15-071602_voices/up.sql b/migrations/2021-05-15-071602_voices/up.sql index 1d5eed5fc4..42b32bb82c 100644 --- a/migrations/2021-05-15-071602_voices/up.sql +++ b/migrations/2021-05-15-071602_voices/up.sql @@ -92,49 +92,49 @@ CREATE TABLE subjects ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; -CREATE TABLE voices ( - -- Not used for anything except replication. - id BIGINT(20) NOT NULL AUTO_INCREMENT, - - -- Effective "primary key" (PUBLIC) - token VARCHAR(32) NOT NULL, - - -- The URL we access the voice at. - -- These should be stable, but could be changed if necessary. - -- As such, these should not be foreign keys. - updatable_slug VARCHAR(64) NOT NULL, - - -- We can assign an exemplary model to the voice - default_model_token VARCHAR(32) DEFAULT NULL, - - -- If a moderator author disables it. - -- This should prevent the voice from showing up in lists. - mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, - - -- The name of the voice - voice_name VARCHAR(255) NOT NULL, - - -- If the voice is "happy" or a singer "a-capella", etc. - voice_characteristic VARCHAR(255) DEFAULT NULL, - - -- The speaker (in the case of cartoon characters) - voice_actor_name VARCHAR(255) DEFAULT NULL, - - -- The 800x600 image - image_banner_public_bucket_hash CHAR(32) DEFAULT NULL, - - -- The square avatar image - image_square_public_bucket_hash CHAR(32) DEFAULT NULL, - - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - - -- If this is removed by a mod. - deleted_at TIMESTAMP NULL, - - -- INDICES -- - PRIMARY KEY (id), - UNIQUE KEY (token), - UNIQUE KEY (updatable_slug) - -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +-- CREATE TABLE voices ( +-- -- Not used for anything except replication. +-- id BIGINT(20) NOT NULL AUTO_INCREMENT, +-- +-- -- Effective "primary key" (PUBLIC) +-- token VARCHAR(32) NOT NULL, +-- +-- -- The URL we access the voice at. +-- -- These should be stable, but could be changed if necessary. +-- -- As such, these should not be foreign keys. +-- updatable_slug VARCHAR(64) NOT NULL, +-- +-- -- We can assign an exemplary model to the voice +-- default_model_token VARCHAR(32) DEFAULT NULL, +-- +-- -- If a moderator author disables it. +-- -- This should prevent the voice from showing up in lists. +-- mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, +-- +-- -- The name of the voice +-- voice_name VARCHAR(255) NOT NULL, +-- +-- -- If the voice is "happy" or a singer "a-capella", etc. +-- voice_characteristic VARCHAR(255) DEFAULT NULL, +-- +-- -- The speaker (in the case of cartoon characters) +-- voice_actor_name VARCHAR(255) DEFAULT NULL, +-- +-- -- The 800x600 image +-- image_banner_public_bucket_hash CHAR(32) DEFAULT NULL, +-- +-- -- The square avatar image +-- image_square_public_bucket_hash CHAR(32) DEFAULT NULL, +-- +-- created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, +-- updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, +-- +-- -- If this is removed by a mod. +-- deleted_at TIMESTAMP NULL, +-- +-- -- INDICES -- +-- PRIMARY KEY (id), +-- UNIQUE KEY (token), +-- UNIQUE KEY (updatable_slug) +-- +-- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 173102f1d892511579e561444982beef5bc8696f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 19:52:15 -0400 Subject: [PATCH 0066/1042] more model refinements --- .../up.sql | 19 ++- migrations/2021-05-15-071601_assets/up.sql | 128 ++++++++++-------- 2 files changed, 87 insertions(+), 60 deletions(-) diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index 5c096a260f..b5f6a2dd6b 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -12,13 +12,18 @@ CREATE TABLE tts_model_upload_jobs ( -- Wide enough for IPv4/6 creator_ip_address VARCHAR(40) NOT NULL, - -- Users can upload their own private models. - -- They can choose to make them public later. - -- NB: Not sure I'm going to use this at first (or at all) - is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', -- The name of the voice voice_name VARCHAR(255) NOT NULL, + -- The voice actor (in the case of cartoon characters) voice_actor_name VARCHAR(255) DEFAULT NULL, @@ -31,6 +36,10 @@ CREATE TABLE tts_model_upload_jobs ( 'glowtts-vocodes' ) NOT NULL DEFAULT 'not-set', + -- Can be linked to a well-known voice + maybe_subject_token VARCHAR(32) DEFAULT NULL, + maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, + -- If we need to download the file from Google Drive. download_url VARCHAR(512) DEFAULT NULL, @@ -79,6 +88,8 @@ CREATE TABLE tts_model_upload_jobs ( -- INDICES -- PRIMARY KEY (id), KEY fk_creator_user_token (creator_user_token), + KEY fk_maybe_subject_token (maybe_subject_token), + KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), KEY index_status (status), KEY index_creator_ip_address (creator_ip_address) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 88a021b73d..40ac134b55 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -8,12 +8,18 @@ CREATE TABLE tts_models ( -- Effective "primary key" (PUBLIC) token VARCHAR(32) NOT NULL, - -- The person that created the template. - creator_user_token VARCHAR(32) NOT NULL, + -- NB: DO NOT CHANGE ORDER; APPEND ONLY! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + tts_model_type ENUM( + 'not-set', + 'tacotron2', + 'glowtts', + 'glowtts-vocodes' + ) NOT NULL DEFAULT 'not-set', - -- For abuse tracking. - -- Wide enough for IPv4/6 - creator_ip_address VARCHAR(40) NOT NULL, + -- Optional Pointer to a newer version of the voice + -- If there's a newer version, we can disable this one. + maybe_updated_model_token VARCHAR(32) DEFAULT NULL, -- We can set as a combination of ['username' + 'voice-name'] -- There can be public aliases for voices, eg. a voice's default model. @@ -21,23 +27,39 @@ CREATE TABLE tts_models ( -- As such, these should not be foreign keys. updatable_slug VARCHAR(64) NOT NULL, - -- Optional Pointer to a newer version of the voice - -- If there's a newer version, we can disable this one. - maybe_updated_model_token VARCHAR(32) DEFAULT NULL, - - -- NB: DO NOT CHANGE ORDER; APPEND ONLY! - -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. - tts_model_type ENUM( - 'not-set', - 'tacotron2', - 'glowtts', - 'glowtts-vocodes' - ) NOT NULL DEFAULT 'not-set', - -- Can be linked to a well-known voice maybe_subject_token VARCHAR(32) DEFAULT NULL, maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, + -- ========== UNSTRUCTURED METADATA ========== + + -- The "name" of the voice model, which might be complicated. + -- If maybe_subject_token (etc.) is set, then it's authoritative instead. + title VARCHAR(255) NOT NULL, + + -- If the voice is "happy" or a singer "a-capella", etc. + -- This is used with "maybe_subject_token" etc. + maybe_voice_characteristic VARCHAR(255) DEFAULT NULL, + + -- Eg. ("poor quality", "trained longer", etc.) + -- This is used with "maybe_subject_token" etc. + maybe_extra_model_disambiguation VARCHAR(255) DEFAULT NULL, + + -- The description of the model in markdown. + description_markdown TEXT NOT NULL, + + -- Generated HTML (not user-editable). + description_rendered_html TEXT NOT NULL, + + -- ========== CREATOR DETAILS ========== + + -- The person that created the template. + creator_user_token VARCHAR(32) NOT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + -- (THIS MIGHT NOT BE USED) -- NB: DO NOT SORT! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. @@ -47,26 +69,6 @@ CREATE TABLE tts_models ( 'private' ) NOT NULL DEFAULT 'public', - -- The name of the voice. - -- If voice_token is set, then it's authoritative instead. - voice_name VARCHAR(255) NOT NULL, - - -- The speaker (in the case of cartoon characters) - -- If voice_token is set, it's authoritative. - voice_actor_name VARCHAR(255) DEFAULT NULL, - - -- If the voice is "happy" or a singer "a-capella", etc. - voice_characteristic VARCHAR(255) DEFAULT NULL, - - -- Eg. ("poor quality", "trained longer", etc.) - extra_model_disambiguation VARCHAR(255) DEFAULT NULL, - - -- The description of the model in markdown. - description_markdown TEXT NOT NULL, - - -- Generated HTML (not user-editable). - description_rendered_html TEXT NOT NULL, - -- The filename that was used at upload time. original_filename VARCHAR(255) NOT NULL, @@ -121,34 +123,29 @@ CREATE TABLE w2l_templates ( -- Effective "primary key" (PUBLIC) token VARCHAR(32) NOT NULL, - -- A combination of ['username' + 'template-name'] - -- There can be public aliases for voices, eg. a voice's default model. - -- A user can change this. - -- As such, these should not be foreign keys. - updatable_slug VARCHAR(64) NOT NULL, - - -- Optional Pointer to a newer version of the template - -- If there's a newer version, we can disable this one. - maybe_updated_template_token VARCHAR(32) DEFAULT NULL, - + -- NB: DO NOT CHANGE ORDER; APPEND ONLY! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. template_type ENUM( 'not-set', 'image', 'video' ) NOT NULL DEFAULT 'not-set', + -- Optional Pointer to a newer version of the template + -- If there's a newer version, we can disable this one. + maybe_updated_template_token VARCHAR(32) DEFAULT NULL, + + -- A combination of ['username' + 'template-name'] + -- There can be public aliases for voices, eg. a voice's default model. + -- A user can change this. + -- As such, these should not be foreign keys. + updatable_slug VARCHAR(64) NOT NULL, + -- Can be linked to a well-known voice maybe_subject_token VARCHAR(32) DEFAULT NULL, maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, - -- (THIS MIGHT NOT BE USED) - -- NB: DO NOT SORT! - -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. - creator_set_visibility ENUM( - 'public', - 'hidden', - 'private' - ) NOT NULL DEFAULT 'public', + -- ========== UNSTRUCTURED METADATA ========== -- The title of the template. title CHAR(255) NOT NULL, @@ -159,6 +156,8 @@ CREATE TABLE w2l_templates ( -- Generated HTML (not user-editable). description_rendered_html TEXT NOT NULL, + -- ========== CREATOR DETAILS ========== + -- The person that created the template. creator_user_token VARCHAR(32) NOT NULL, @@ -166,6 +165,15 @@ CREATE TABLE w2l_templates ( -- Wide enough for IPv4/6 creator_ip_address VARCHAR(40) NOT NULL, + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + -- The filename that was used at upload time. original_filename CHAR(255) NOT NULL, @@ -175,6 +183,14 @@ CREATE TABLE w2l_templates ( -- For the thumbnail we show. public_bucket_hash CHAR(32) NOT NULL, + -- Calculated average, on a scale of 0-100 + -- Null with zero ratings. + calculated_average_score INT(3) DEFAULT NULL, + + -- Other metrics + calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0, + calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0, + -- In this case, a moderator disables it. is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, From 23cdeaddf620347b6030b75e50c3f12bd0c5f8b4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 20:12:17 -0400 Subject: [PATCH 0067/1042] model refinements --- .../up.sql | 35 +++++++++++++------ migrations/2021-05-15-071601_assets/up.sql | 5 +++ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index b5f6a2dd6b..5e603b0014 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -22,10 +22,9 @@ CREATE TABLE tts_model_upload_jobs ( ) NOT NULL DEFAULT 'public', -- The name of the voice - voice_name VARCHAR(255) NOT NULL, - - -- The voice actor (in the case of cartoon characters) - voice_actor_name VARCHAR(255) DEFAULT NULL, + -- The "name" of the voice model, which might be complicated. + -- If maybe_subject_token (etc.) is set, then it's authoritative instead. + title VARCHAR(255) NOT NULL, -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. @@ -48,7 +47,7 @@ CREATE TABLE tts_model_upload_jobs ( 'not-set', 'google-drive', 'web' - ) NOT NULL DEFAULT 'not-set', + ) NOT NULL DEFAULT 'not-set', -- Jobs begin as "pending", then transition to other states. -- @@ -106,17 +105,29 @@ CREATE TABLE w2l_template_upload_jobs ( -- Wide enough for IPv4/6 creator_ip_address VARCHAR(40) NOT NULL, - -- Users can upload their own private templates. - -- They can choose to make them public later. - -- NB: Not sure I'm going to use this at first (or at all); would be separate from mod approval. - is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- The title of the template + title VARCHAR(255) NOT NULL, -- NB: DO NOT SORT! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. template_type ENUM( + 'not-set', 'video', 'image' - ) NOT NULL, + ) NOT NULL DEFAULT 'not-set', + + -- Can be linked to a well-known subject + maybe_subject_token VARCHAR(32) DEFAULT NULL, + maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, -- If we need to download the file from Google Drive. download_url VARCHAR(512) DEFAULT NULL, @@ -126,7 +137,7 @@ CREATE TABLE w2l_template_upload_jobs ( 'not-set', 'google-drive', 'web' - ) NOT NULL DEFAULT 'not-set', + ) NOT NULL DEFAULT 'not-set', -- Jobs begin as "pending", then transition to other states. -- @@ -166,6 +177,8 @@ CREATE TABLE w2l_template_upload_jobs ( -- INDICES -- PRIMARY KEY (id), KEY fk_creator_user_token (creator_user_token), + KEY fk_maybe_subject_token (maybe_subject_token), + KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), KEY index_status (status), KEY index_creator_ip_address (creator_ip_address) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 40ac134b55..eb4e200277 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -131,6 +131,9 @@ CREATE TABLE w2l_templates ( 'video' ) NOT NULL DEFAULT 'not-set', + -- Mods have to approve of w2l templates. + is_mod_approved BOOLEAN NOT NULL DEFAULT FALSE, + -- Optional Pointer to a newer version of the template -- If there's a newer version, we can disable this one. maybe_updated_template_token VARCHAR(32) DEFAULT NULL, @@ -192,6 +195,7 @@ CREATE TABLE w2l_templates ( calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0, -- In this case, a moderator disables it. + -- This also disables it for the creator. is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, -- If a moderator has comments. @@ -204,6 +208,7 @@ CREATE TABLE w2l_templates ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- If this is removed by a mod. + -- It shows up nowhere if deleted. deleted_at TIMESTAMP NULL, -- INDICES -- From b4ce8dfaf7e175ff0754f8b66753e4069297a52e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 20:19:59 -0400 Subject: [PATCH 0068/1042] model refinements --- .../2021-05-15-212913_inference_jobs/up.sql | 40 ++++++++++++++----- .../up.sql | 30 +++++++++++--- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index 901ba4be2d..5cf19f515a 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -3,10 +3,17 @@ CREATE TABLE tts_inference_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, + -- ========== INFERENCE DETAILS ========== + -- The model to use. -- This also determines which architecture we're using. model_token VARCHAR(32) NOT NULL, + -- The raw, unprocessed user input. + inference_text TEXT NOT NULL, + + -- ========== CREATOR DETAILS ========== + -- Foreign key to user -- If no user is logged in, this is null. maybe_creator_user_token CHAR(16) DEFAULT NULL, @@ -15,12 +22,16 @@ CREATE TABLE tts_inference_jobs ( -- Wide enough for IPv4/6 creator_ip_address VARCHAR(40) NOT NULL, - -- Users can upload their own private models. - -- They can choose to make them public later. - is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', - -- The raw, unprocessed user input. - inference_text TEXT NOT NULL, + -- ========== JOB SYSTEM DETAILS ========== -- Jobs begin as "pending", then transition to other states. -- Pending -> Started -> Complete @@ -54,6 +65,8 @@ CREATE TABLE tts_inference_jobs ( CREATE TABLE w2l_inference_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, + -- ========== INFERENCE DETAILS ========== + -- The W2L template to use -- Can be an image or video. -- This is null if we're using a custom uploaded image. @@ -68,6 +81,8 @@ CREATE TABLE w2l_inference_jobs ( -- If we're using a custom uploaded image, this will be present. maybe_image_bucket_location CHAR(16) DEFAULT NULL, + -- ========== CREATOR DETAILS ========== + -- Foreign key to user -- If no user is logged in, this is null. maybe_creator_user_token VARCHAR(32) DEFAULT NULL, @@ -76,11 +91,18 @@ CREATE TABLE w2l_inference_jobs ( -- Wide enough for IPv4/6 creator_ip_address VARCHAR(40) NOT NULL, - -- Users can upload their own private models. - -- They can choose to make them public later. - is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', - -- Jobs begin as "pending", then transition to other states. + -- ========== JOB SYSTEM DETAILS ========== + + -- Jobs begin as "pending", then transition to other states. -- Pending -> Started -> Complete -- \-> Failed -> Started -> { Complete, Failed, Dead } status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending', diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 93eb811fe8..9a576aff85 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -8,15 +8,15 @@ CREATE TABLE tts_results ( -- Effective "primary key" (PUBLIC) token VARCHAR(32) NOT NULL, + -- ========== INFERENCE DETAILS ========== + -- The TTS model that was used model_token VARCHAR(32) NOT NULL, -- The original raw, unprocessed user input. inference_text TEXT NOT NULL, - -- Users can upload their own private templates. - -- They can choose to make them public later. - is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + -- ========== CREATOR DETAILS ========== -- The person that created the template. -- If the user wasn't logged in, this is null @@ -26,6 +26,15 @@ CREATE TABLE tts_results ( -- Wide enough for IPv4/6 creator_ip_address VARCHAR(40) NOT NULL, + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + -- Where the wav, spectrogram, and etc. are located. public_bucket_hash CHAR(32) NOT NULL, @@ -51,15 +60,15 @@ CREATE TABLE w2l_results ( -- Effective "primary key" (PUBLIC) token VARCHAR(32) NOT NULL, + -- ========== INFERENCE DETAILS ========== + -- The w2l template that was used (if set) maybe_w2l_template_token VARCHAR(32) NOT NULL, -- The inference result, if we're using them. maybe_tts_inference_result_token VARCHAR(32) DEFAULT NULL, - -- Users can upload their own private templates. - -- They can choose to make them public later. - is_private_for_creator BOOLEAN NOT NULL DEFAULT FALSE, + -- ========== CREATOR DETAILS ========== -- The person that created the template. -- If the user wasn't logged in, this is null @@ -69,6 +78,15 @@ CREATE TABLE w2l_results ( -- Wide enough for IPv4/6 creator_ip_address VARCHAR(40) NOT NULL, + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + -- Where the wav, spectrogram, and etc. are located. public_bucket_hash CHAR(32) NOT NULL, From f1ac70f6fe686cd57206b400810c79192ad93bf4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 20:22:47 -0400 Subject: [PATCH 0069/1042] model refinements --- migrations/2021-05-15-071601_assets/up.sql | 2 ++ .../up.sql | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index eb4e200277..4c294b2ba2 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -194,6 +194,8 @@ CREATE TABLE w2l_templates ( calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0, calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0, + -- ========== MODERATION DETAILS ========== + -- In this case, a moderator disables it. -- This also disables it for the creator. is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 9a576aff85..80316bdc1b 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -38,6 +38,17 @@ CREATE TABLE tts_results ( -- Where the wav, spectrogram, and etc. are located. public_bucket_hash CHAR(32) NOT NULL, + -- ========== MODERATION DETAILS ========== + + -- In this case, a moderator hides it from public. + is_mod_hidden_from_public BOOLEAN NOT NULL DEFAULT FALSE, + + -- If a moderator has comments. + maybe_mod_comments VARCHAR(255) DEFAULT NULL, + + -- The last moderator that made changes. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -49,6 +60,7 @@ CREATE TABLE tts_results ( UNIQUE KEY (token), KEY fk_model_token (model_token), KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), KEY index_creator_ip_address (creator_ip_address) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -90,6 +102,17 @@ CREATE TABLE w2l_results ( -- Where the wav, spectrogram, and etc. are located. public_bucket_hash CHAR(32) NOT NULL, + -- ========== MODERATION DETAILS ========== + + -- In this case, a moderator hides it from public. + is_mod_hidden_from_public BOOLEAN NOT NULL DEFAULT FALSE, + + -- If a moderator has comments. + maybe_mod_comments VARCHAR(255) DEFAULT NULL, + + -- The last moderator that made changes. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -102,6 +125,7 @@ CREATE TABLE w2l_results ( KEY fk_maybe_creator_user_token (maybe_creator_user_token), KEY fk_maybe_w2l_template_token (maybe_w2l_template_token), KEY fk_maybe_tts_inference_result_token (maybe_tts_inference_result_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), KEY index_creator_ip_address (creator_ip_address) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 40c17a6337ea107b9dbb3ad6db5e5fd1671899d1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 21:53:30 -0400 Subject: [PATCH 0070/1042] create upload tts job --- .env | 3 +- debug/curl/upload_tts.py | 43 ++++ debug/mysql/last_tts_upload.sh | 7 + .../up.sql | 10 +- src/database_helpers/enums.rs | 33 +++ src/database_helpers/mod.rs | 1 + src/endpoints/mod.rs | 1 + src/endpoints/uploads/mod.rs | 2 + src/endpoints/uploads/upload_tts_model.rs | 192 ++++++++++++++++++ src/endpoints/uploads/upload_w2l_template.rs | 0 src/main.rs | 12 ++ src/util/session_checker.rs | 19 -- src/validations/mod.rs | 5 +- src/validations/model_uploads.rs | 14 ++ 14 files changed, 314 insertions(+), 28 deletions(-) create mode 100755 debug/curl/upload_tts.py create mode 100755 debug/mysql/last_tts_upload.sh create mode 100644 src/database_helpers/enums.rs create mode 100644 src/database_helpers/mod.rs create mode 100644 src/endpoints/uploads/mod.rs create mode 100644 src/endpoints/uploads/upload_tts_model.rs create mode 100644 src/endpoints/uploads/upload_w2l_template.rs create mode 100644 src/validations/model_uploads.rs diff --git a/.env b/.env index 17a2fc83df..09610ddca7 100644 --- a/.env +++ b/.env @@ -3,7 +3,8 @@ DATABASE_URL=mysql://root:root@localhost/storyteller # SQLX normally requires a live database to reflect upon at build time. # This setting turns that on and requires a `sql-data.json` cache file instead. # See https://github.com/launchbadge/sqlx/tree/master/sqlx-cli#readme -SQLX_OFFLINE=true +#SQLX_OFFLINE=true # We can't serve HTTPS locally without self-signed certificate nonsense. COOKIE_SECURE=false +COOKIE_DOMAIN=.jungle.horse diff --git a/debug/curl/upload_tts.py b/debug/curl/upload_tts.py new file mode 100755 index 0000000000..b5bf458be1 --- /dev/null +++ b/debug/curl/upload_tts.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import requests +import re + +login_url = 'http://localhost:12345/login' +upload_url = 'http://localhost:12345/upload_tts' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +# NB: May not be set due to cookie domain: +#r.cookies['session'] + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + + +print("===== Upload TTS=====") + +cookies = { 'session': session_cookie } + +payload = { + 'title': 'this is a new text to speech model', + 'download_url': 'https://website.com', +} + +r = requests.post(upload_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + diff --git a/debug/mysql/last_tts_upload.sh b/debug/mysql/last_tts_upload.sh new file mode 100755 index 0000000000..1772026b04 --- /dev/null +++ b/debug/mysql/last_tts_upload.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u root \ + -proot \ + -h localhost \ + -D storyteller \ + -e "select * from tts_model_upload_jobs\G" diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index 5e603b0014..eeeed7f44f 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -44,10 +44,9 @@ CREATE TABLE tts_model_upload_jobs ( -- NB: DO NOT SORT! download_url_type ENUM( - 'not-set', 'google-drive', 'web' - ) NOT NULL DEFAULT 'not-set', + ) NOT NULL DEFAULT 'web', -- Jobs begin as "pending", then transition to other states. -- @@ -120,10 +119,10 @@ CREATE TABLE w2l_template_upload_jobs ( -- NB: DO NOT SORT! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. template_type ENUM( - 'not-set', + 'unknown', 'video', 'image' - ) NOT NULL DEFAULT 'not-set', + ) NOT NULL DEFAULT 'unknown', -- Can be linked to a well-known subject maybe_subject_token VARCHAR(32) DEFAULT NULL, @@ -134,10 +133,9 @@ CREATE TABLE w2l_template_upload_jobs ( -- NB: DO NOT SORT! download_url_type ENUM( - 'not-set', 'google-drive', 'web' - ) NOT NULL DEFAULT 'not-set', + ) NOT NULL DEFAULT 'web', -- Jobs begin as "pending", then transition to other states. -- diff --git a/src/database_helpers/enums.rs b/src/database_helpers/enums.rs new file mode 100644 index 0000000000..87d66c66c6 --- /dev/null +++ b/src/database_helpers/enums.rs @@ -0,0 +1,33 @@ +//! Common enums in the database. +//! These may be used in one or more tables. +//! This defines the expected serializations. + +#![allow(non_camel_case_types)] + +#[derive(Deserialize)] +pub enum CreatorSetVisibility { + /// public + Public, + /// hidden + Hidden, + /// private + Private, +} + +#[derive(Deserialize)] +pub enum TtsModelType { + /// tacotron2 + Tacotron2, + /// glowtts + GlowTts, + /// glowtts-vocodes + GlowTts_Vocodes, +} + +#[derive(Deserialize)] +pub enum DownloadUrlType { + /// google-drive + Google_Drive, + /// web + Web, +} diff --git a/src/database_helpers/mod.rs b/src/database_helpers/mod.rs new file mode 100644 index 0000000000..35b3598e87 --- /dev/null +++ b/src/database_helpers/mod.rs @@ -0,0 +1 @@ +pub mod enums; \ No newline at end of file diff --git a/src/endpoints/mod.rs b/src/endpoints/mod.rs index a3cbbff247..fde1e2b7c0 100644 --- a/src/endpoints/mod.rs +++ b/src/endpoints/mod.rs @@ -1,4 +1,5 @@ pub mod default_route_404; pub mod misc; pub mod root_index; +pub mod uploads; pub mod users; \ No newline at end of file diff --git a/src/endpoints/uploads/mod.rs b/src/endpoints/uploads/mod.rs new file mode 100644 index 0000000000..1212193bbc --- /dev/null +++ b/src/endpoints/uploads/mod.rs @@ -0,0 +1,2 @@ +pub mod upload_tts_model; +pub mod upload_w2l_template; diff --git a/src/endpoints/uploads/upload_tts_model.rs b/src/endpoints/uploads/upload_tts_model.rs new file mode 100644 index 0000000000..a07ad99848 --- /dev/null +++ b/src/endpoints/uploads/upload_tts_model.rs @@ -0,0 +1,192 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::server_state::ServerState; +use crate::util::ip_address::get_request_ip; +use crate::util::tokens::random_token; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use crate::validations::username::validate_username; +use crate::validations::passwords::validate_passwords; +use actix_web::cookie::Cookie; +use crate::database_helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; +use crate::validations::model_uploads::validate_model_title; + +const NEW_USER_ROLE: &'static str = "new-user"; + +#[derive(Deserialize)] +pub struct UploadTtsModelRequest { + title: String, + download_url: String, + download_url_type: Option, + tts_model_type: Option, + creator_set_visibility: Option, + maybe_subject_token: Option, + maybe_actor_subject_token: Option, +} + +#[derive(Serialize)] +pub struct UploadTtsModelSuccessResponse { + pub success: bool, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum UploadTtsModelError { + BadInput(String), + MustBeLoggedIn, + ServerError, +} + +impl ResponseError for UploadTtsModelError { + fn status_code(&self) -> StatusCode { + match *self { + UploadTtsModelError::BadInput(_) => StatusCode::BAD_REQUEST, + UploadTtsModelError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, + UploadTtsModelError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + UploadTtsModelError::BadInput(reason) => reason.to_string(), + UploadTtsModelError::MustBeLoggedIn => "user must be logged in".to_string(), + UploadTtsModelError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn upload_tts_model_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + UploadTtsModelError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(UploadTtsModelError::MustBeLoggedIn); + } + }; + + if let Err(reason) = validate_model_title(&request.title) { + return Err(UploadTtsModelError::BadInput(reason)); + } + + let ip_address = get_request_ip(&http_request); + + let title = request.title.to_string(); + let download_url = request.download_url.to_string(); + + let tts_model_type = "tacotron2".to_string(); + let download_url_type = "google-drive".to_string(); + let creator_set_visibility = "public".to_string(); + let maybe_subject_token : Option = None; + let maybe_actor_subject_token : Option = None; + + let query_result = sqlx::query!( + r#" +INSERT INTO tts_model_upload_jobs +SET + creator_user_token = ?, + creator_ip_address = ?, + creator_set_visibility = ?, + title = ?, + tts_model_type = ?, + maybe_subject_token = ?, + maybe_actor_subject_token = ?, + download_url = ?, + download_url_type = ?, + status = "pending" + "#, + user_session.user_token.to_string(), + ip_address.to_string(), + creator_set_visibility.to_string(), + title.to_string(), + tts_model_type, + maybe_subject_token, + maybe_actor_subject_token, + download_url, + download_url_type + ) + .execute(&server_state.mysql_pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + warn!("New tts model upload creation DB error: {:?}", err); + + // NB: SQLSTATE[23000]: Integrity constraint violation + // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access) + match err { + Database(err) => { + let maybe_code = err.code().map(|c| c.into_owned()); + /*match maybe_code.as_deref() { + Some("23000") => { + if err.message().contains("username") { + return Err(UsernameTaken); + } else if err.message().contains("email_address") { + return Err(EmailTaken); + } + } + _ => {}, + }*/ + }, + _ => {}, + } + return Err(UploadTtsModelError::ServerError); + } + }; + + info!("new model upload job id: {}", record_id); + + let response = UploadTtsModelSuccessResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| UploadTtsModelError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/endpoints/uploads/upload_w2l_template.rs b/src/endpoints/uploads/upload_w2l_template.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/main.rs b/src/main.rs index bf3c09f02a..3d3dcb7058 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ #[macro_use] extern crate serde_derive; pub mod common_queries; +pub mod database_helpers; pub mod endpoints; pub mod server_state; pub mod util; @@ -31,6 +32,7 @@ use crate::endpoints::users::session_info::session_info_handler; use crate::endpoints::root_index::get_root_index; use crate::endpoints::default_route_404::default_route_404; use crate::endpoints::misc::enable_alpha::enable_alpha; +use crate::endpoints::uploads::upload_tts_model::upload_tts_model_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -166,6 +168,16 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(session_info_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/upload_tts") + .route(web::post().to(upload_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + /*.service( + web::resource("/upload_w2l") + .route(web::post().to()) + .route(web::head().to(|| HttpResponse::Ok())) + )*/ .service(get_root_index) .service(enable_alpha) .default_service( web::route().to(default_route_404)) diff --git a/src/util/session_checker.rs b/src/util/session_checker.rs index e6e41b6cbe..1a7d9b2448 100644 --- a/src/util/session_checker.rs +++ b/src/util/session_checker.rs @@ -95,25 +95,6 @@ AND deleted_at IS NULL Some(session_token) => session_token, }; - /* - - pub token: String, - pub username: String, - pub display_name: String, - pub email_address: String, - pub email_confirmed: bool, - pub profile_markdown: String, - pub profile_rendered_html: String, - pub user_role_slug: String, - pub banned: bool, - pub dark_mode: String, - pub avatar_public_bucket_hash: Option, - pub disable_gravatar: bool, - pub hide_results_preference: bool, - pub discord_username: Option, - pub twitch_username: Option, - pub twitter_username: Option, - */ // NB: Lookup failure is Err(RowNotFound). let maybe_user_record = sqlx::query_as!( SessionUserRecord, diff --git a/src/validations/mod.rs b/src/validations/mod.rs index f02a6b769a..ef3b154a7c 100644 --- a/src/validations/mod.rs +++ b/src/validations/mod.rs @@ -1,2 +1,3 @@ -pub mod username; -pub mod passwords; \ No newline at end of file +pub mod model_uploads; +pub mod passwords; +pub mod username; \ No newline at end of file diff --git a/src/validations/model_uploads.rs b/src/validations/model_uploads.rs new file mode 100644 index 0000000000..c324f5ebc0 --- /dev/null +++ b/src/validations/model_uploads.rs @@ -0,0 +1,14 @@ +use crate::AnyhowResult; +use regex::Regex; + +pub fn validate_model_title(title: &str) -> Result<(), String> { + if title.len() < 3 { + return Err("title is too short".to_string()); + } + + if title.len() > 200 { + return Err("title is too long".to_string()); + } + + Ok(()) +} From 4748cf6fd0f547a26d0de46d4790c0cb947e53f5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 22:15:50 -0400 Subject: [PATCH 0071/1042] upload w2l job handler --- debug/curl/upload_w2l.py | 43 +++++ debug/mysql/last_w2l_upload.sh | 7 + src/database_helpers/enums.rs | 10 + src/endpoints/uploads/upload_w2l_template.rs | 192 +++++++++++++++++++ src/main.rs | 7 +- 5 files changed, 256 insertions(+), 3 deletions(-) create mode 100755 debug/curl/upload_w2l.py create mode 100755 debug/mysql/last_w2l_upload.sh diff --git a/debug/curl/upload_w2l.py b/debug/curl/upload_w2l.py new file mode 100755 index 0000000000..7f269c53a5 --- /dev/null +++ b/debug/curl/upload_w2l.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import requests +import re + +login_url = 'http://localhost:12345/login' +upload_url = 'http://localhost:12345/upload_w2l' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +# NB: May not be set due to cookie domain: +#r.cookies['session'] + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + + +print("===== Upload W2L =====") + +cookies = { 'session': session_cookie } + +payload = { + 'title': 'this is a new text to W2L template', + 'download_url': 'https://video.com', +} + +r = requests.post(upload_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + diff --git a/debug/mysql/last_w2l_upload.sh b/debug/mysql/last_w2l_upload.sh new file mode 100755 index 0000000000..3d7a20448c --- /dev/null +++ b/debug/mysql/last_w2l_upload.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u root \ + -proot \ + -h localhost \ + -D storyteller \ + -e "select * from w2l_template_upload_jobs\G" diff --git a/src/database_helpers/enums.rs b/src/database_helpers/enums.rs index 87d66c66c6..876d9dc2d3 100644 --- a/src/database_helpers/enums.rs +++ b/src/database_helpers/enums.rs @@ -31,3 +31,13 @@ pub enum DownloadUrlType { /// web Web, } + +#[derive(Deserialize)] +pub enum W2lTemplateType { + /// unknown + Unknown, + /// video + Video, + /// image + Image, +} diff --git a/src/endpoints/uploads/upload_w2l_template.rs b/src/endpoints/uploads/upload_w2l_template.rs index e69de29bb2..6feac2c310 100644 --- a/src/endpoints/uploads/upload_w2l_template.rs +++ b/src/endpoints/uploads/upload_w2l_template.rs @@ -0,0 +1,192 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::server_state::ServerState; +use crate::util::ip_address::get_request_ip; +use crate::util::tokens::random_token; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use crate::validations::username::validate_username; +use crate::validations::passwords::validate_passwords; +use actix_web::cookie::Cookie; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::validations::model_uploads::validate_model_title; + +const NEW_USER_ROLE: &'static str = "new-user"; + +#[derive(Deserialize)] +pub struct UploadW2lTemplateRequest { + title: String, + download_url: String, + download_url_type: Option, + template_type: Option, + creator_set_visibility: Option, + maybe_subject_token: Option, + maybe_actor_subject_token: Option, +} + +#[derive(Serialize)] +pub struct UploadW2lTemplateSuccessResponse { + pub success: bool, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum UploadW2lTemplateError { + BadInput(String), + MustBeLoggedIn, + ServerError, +} + +impl ResponseError for UploadW2lTemplateError { + fn status_code(&self) -> StatusCode { + match *self { + UploadW2lTemplateError::BadInput(_) => StatusCode::BAD_REQUEST, + UploadW2lTemplateError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, + UploadW2lTemplateError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + UploadW2lTemplateError::BadInput(reason) => reason.to_string(), + UploadW2lTemplateError::MustBeLoggedIn => "user must be logged in".to_string(), + UploadW2lTemplateError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn upload_w2l_template_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + UploadW2lTemplateError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(UploadW2lTemplateError::MustBeLoggedIn); + } + }; + + if let Err(reason) = validate_model_title(&request.title) { + return Err(UploadW2lTemplateError::BadInput(reason)); + } + + let ip_address = get_request_ip(&http_request); + + let title = request.title.to_string(); + let download_url = request.download_url.to_string(); + + let template_type = "unknown".to_string(); + let download_url_type = "google-drive".to_string(); + let creator_set_visibility = "public".to_string(); + let maybe_subject_token : Option = None; + let maybe_actor_subject_token : Option = None; + + let query_result = sqlx::query!( + r#" +INSERT INTO w2l_template_upload_jobs +SET + creator_user_token = ?, + creator_ip_address = ?, + creator_set_visibility = ?, + title = ?, + template_type = ?, + maybe_subject_token = ?, + maybe_actor_subject_token = ?, + download_url = ?, + download_url_type = ?, + status = "pending" + "#, + user_session.user_token.to_string(), + ip_address.to_string(), + creator_set_visibility.to_string(), + title.to_string(), + template_type, + maybe_subject_token, + maybe_actor_subject_token, + download_url, + download_url_type + ) + .execute(&server_state.mysql_pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + warn!("New w2l template upload creation DB error: {:?}", err); + + // NB: SQLSTATE[23000]: Integrity constraint violation + // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access) + match err { + Database(err) => { + let maybe_code = err.code().map(|c| c.into_owned()); + /*match maybe_code.as_deref() { + Some("23000") => { + if err.message().contains("username") { + return Err(UsernameTaken); + } else if err.message().contains("email_address") { + return Err(EmailTaken); + } + } + _ => {}, + }*/ + }, + _ => {}, + } + return Err(UploadW2lTemplateError::ServerError); + } + }; + + info!("new w2l template upload job id: {}", record_id); + + let response = UploadW2lTemplateSuccessResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| UploadW2lTemplateError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/main.rs b/src/main.rs index 3d3dcb7058..01a3e0d584 100644 --- a/src/main.rs +++ b/src/main.rs @@ -33,6 +33,7 @@ use crate::endpoints::root_index::get_root_index; use crate::endpoints::default_route_404::default_route_404; use crate::endpoints::misc::enable_alpha::enable_alpha; use crate::endpoints::uploads::upload_tts_model::upload_tts_model_handler; +use crate::endpoints::uploads::upload_w2l_template::upload_w2l_template_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -173,11 +174,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::post().to(upload_tts_model_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - /*.service( + .service( web::resource("/upload_w2l") - .route(web::post().to()) + .route(web::post().to(upload_w2l_template_handler)) .route(web::head().to(|| HttpResponse::Ok())) - )*/ + ) .service(get_root_index) .service(enable_alpha) .default_service( web::route().to(default_route_404)) From 8c6a2c84e3e70971958671c4788d49ad8a400502 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 23:29:28 -0400 Subject: [PATCH 0072/1042] create w2l inference jobs (unchecked) --- src/endpoints/enqueue_inference/infer_tts.rs | 0 src/endpoints/enqueue_inference/infer_w2l.rs | 184 ++++++++++++++++++ src/endpoints/enqueue_inference/mod.rs | 2 + .../{uploads => enqueue_uploads}/mod.rs | 0 .../upload_tts_model.rs | 0 .../upload_w2l_template.rs | 0 src/endpoints/mod.rs | 3 +- src/main.rs | 39 ++-- 8 files changed, 212 insertions(+), 16 deletions(-) create mode 100644 src/endpoints/enqueue_inference/infer_tts.rs create mode 100644 src/endpoints/enqueue_inference/infer_w2l.rs create mode 100644 src/endpoints/enqueue_inference/mod.rs rename src/endpoints/{uploads => enqueue_uploads}/mod.rs (100%) rename src/endpoints/{uploads => enqueue_uploads}/upload_tts_model.rs (100%) rename src/endpoints/{uploads => enqueue_uploads}/upload_w2l_template.rs (100%) diff --git a/src/endpoints/enqueue_inference/infer_tts.rs b/src/endpoints/enqueue_inference/infer_tts.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/endpoints/enqueue_inference/infer_w2l.rs b/src/endpoints/enqueue_inference/infer_w2l.rs new file mode 100644 index 0000000000..a1250d6fa0 --- /dev/null +++ b/src/endpoints/enqueue_inference/infer_w2l.rs @@ -0,0 +1,184 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::server_state::ServerState; +use crate::util::ip_address::get_request_ip; +use crate::util::tokens::random_token; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use crate::validations::username::validate_username; +use crate::validations::passwords::validate_passwords; +use actix_web::cookie::Cookie; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::validations::model_uploads::validate_model_title; + +const NEW_USER_ROLE: &'static str = "new-user"; + +#[derive(Deserialize)] +pub struct InferW2lRequest { + w2l_template_token: Option, + tts_inference_result_token: Option, + creator_set_visibility: Option, +} + +#[derive(Serialize)] +pub struct InferW2lSuccessResponse { + pub success: bool, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum InferW2lError { + BadInput(String), + MustBeLoggedIn, + ServerError, +} + +impl ResponseError for InferW2lError { + fn status_code(&self) -> StatusCode { + match *self { + InferW2lError::BadInput(_) => StatusCode::BAD_REQUEST, + InferW2lError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, + InferW2lError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + InferW2lError::BadInput(reason) => reason.to_string(), + InferW2lError::MustBeLoggedIn => "user must be logged in".to_string(), + InferW2lError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn infer_w2l_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + InferW2lError::ServerError + })?; + + let mut maybe_user_token : Option = maybe_user_session + .as_ref() + .map(|user_session| user_session.user_token.to_string()); + + let w2l_template_token = match &request.w2l_template_token { + None => { + // TODO: Allow image uploads. + return Err(InferW2lError::BadInput("w2l token is required".to_string())); + }, + Some(t) => { + t.to_string() + }, + }; + + let tts_inference_result_token = match &request.tts_inference_result_token { + None => { + // TODO: Allow audio uploads. + return Err(InferW2lError::BadInput("tts token is required".to_string())); + }, + Some(t) => t.to_string(), + }; + + let ip_address = get_request_ip(&http_request); + let creator_set_visibility = "public".to_string(); + + let query_result = sqlx::query!( + r#" +INSERT INTO w2l_inference_jobs +SET + maybe_w2l_template_token = ?, + maybe_tts_inference_result_token = ?, + maybe_audio_bucket_location = NULL, + maybe_image_bucket_location = NULL, + maybe_creator_user_token = ?, + creator_ip_address = ?, + creator_set_visibility = ?, + status = "pending" + "#, + w2l_template_token, + tts_inference_result_token, + maybe_user_token, + ip_address, + creator_set_visibility + ) + .execute(&server_state.mysql_pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + warn!("New w2l inference job creation DB error: {:?}", err); + + // NB: SQLSTATE[23000]: Integrity constraint violation + // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access) + match err { + Database(err) => { + let maybe_code = err.code().map(|c| c.into_owned()); + /*match maybe_code.as_deref() { + Some("23000") => { + if err.message().contains("username") { + return Err(UsernameTaken); + } else if err.message().contains("email_address") { + return Err(EmailTaken); + } + } + _ => {}, + }*/ + }, + _ => {}, + } + return Err(InferW2lError::ServerError); + } + }; + + info!("new w2l inference job id: {}", record_id); + + let response = InferW2lSuccessResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| InferW2lError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/endpoints/enqueue_inference/mod.rs b/src/endpoints/enqueue_inference/mod.rs new file mode 100644 index 0000000000..2b7f9d8025 --- /dev/null +++ b/src/endpoints/enqueue_inference/mod.rs @@ -0,0 +1,2 @@ +pub mod infer_tts; +pub mod infer_w2l; diff --git a/src/endpoints/uploads/mod.rs b/src/endpoints/enqueue_uploads/mod.rs similarity index 100% rename from src/endpoints/uploads/mod.rs rename to src/endpoints/enqueue_uploads/mod.rs diff --git a/src/endpoints/uploads/upload_tts_model.rs b/src/endpoints/enqueue_uploads/upload_tts_model.rs similarity index 100% rename from src/endpoints/uploads/upload_tts_model.rs rename to src/endpoints/enqueue_uploads/upload_tts_model.rs diff --git a/src/endpoints/uploads/upload_w2l_template.rs b/src/endpoints/enqueue_uploads/upload_w2l_template.rs similarity index 100% rename from src/endpoints/uploads/upload_w2l_template.rs rename to src/endpoints/enqueue_uploads/upload_w2l_template.rs diff --git a/src/endpoints/mod.rs b/src/endpoints/mod.rs index fde1e2b7c0..970c7a3129 100644 --- a/src/endpoints/mod.rs +++ b/src/endpoints/mod.rs @@ -1,5 +1,6 @@ pub mod default_route_404; +pub mod enqueue_inference; +pub mod enqueue_uploads; pub mod misc; pub mod root_index; -pub mod uploads; pub mod users; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 01a3e0d584..011bbb1417 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,24 +16,25 @@ pub mod validations; use actix_cors::Cors; use actix_http::http; -use log::{info}; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; -use std::sync::Arc; -use crate::server_state::{ServerState, EnvConfig}; -use sqlx::MySqlPool; -use sqlx::mysql::MySqlPoolOptions; +use crate::endpoints::default_route_404::default_route_404; +use crate::endpoints::enqueue_uploads::upload_tts_model::upload_tts_model_handler; +use crate::endpoints::enqueue_uploads::upload_w2l_template::upload_w2l_template_handler; +use crate::endpoints::misc::enable_alpha::enable_alpha; +use crate::endpoints::root_index::get_root_index; use crate::endpoints::users::create_account::create_account_handler; use crate::endpoints::users::login::login_handler; -use crate::util::cookies::CookieManager; use crate::endpoints::users::logout::logout_handler; -use crate::util::session_checker::SessionChecker; use crate::endpoints::users::session_info::session_info_handler; -use crate::endpoints::root_index::get_root_index; -use crate::endpoints::default_route_404::default_route_404; -use crate::endpoints::misc::enable_alpha::enable_alpha; -use crate::endpoints::uploads::upload_tts_model::upload_tts_model_handler; -use crate::endpoints::uploads::upload_w2l_template::upload_w2l_template_handler; +use crate::server_state::{ServerState, EnvConfig}; +use crate::util::cookies::CookieManager; +use crate::util::session_checker::SessionChecker; +use log::{info}; +use sqlx::MySqlPool; +use sqlx::mysql::MySqlPoolOptions; +use std::sync::Arc; +use crate::endpoints::enqueue_inference::infer_w2l::infer_w2l_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -175,9 +176,17 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) .service( - web::resource("/upload_w2l") - .route(web::post().to(upload_w2l_template_handler)) - .route(web::head().to(|| HttpResponse::Ok())) + web::scope("/w2l") + .service( + web::resource("/upload") + .route(web::post().to(upload_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/inference") + .route(web::post().to(infer_w2l_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service(get_root_index) .service(enable_alpha) From f4437cfe0ab717e66d6b4310d1a984ade65793b7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 23:33:18 -0400 Subject: [PATCH 0073/1042] debug scripts --- debug/curl/infer_w2l.py | 43 +++++++++++++++++++++++++++ debug/curl/upload_w2l.py | 2 +- debug/mysql/last_w2l_inference_job.sh | 7 +++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100755 debug/curl/infer_w2l.py create mode 100755 debug/mysql/last_w2l_inference_job.sh diff --git a/debug/curl/infer_w2l.py b/debug/curl/infer_w2l.py new file mode 100755 index 0000000000..3fcf417e1d --- /dev/null +++ b/debug/curl/infer_w2l.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import requests +import re + +login_url = 'http://localhost:12345/login' +inference_url = 'http://localhost:12345/w2l/inference' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +# NB: May not be set due to cookie domain: +#r.cookies['session'] + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + + +print("===== Infer W2L =====") + +cookies = { 'session': session_cookie } + +payload = { + 'w2l_template_token': 'W2L_TOKEN', + 'tts_inference_result_token': 'TTS_TOKEN', +} + +r = requests.post(inference_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + diff --git a/debug/curl/upload_w2l.py b/debug/curl/upload_w2l.py index 7f269c53a5..9479de1327 100755 --- a/debug/curl/upload_w2l.py +++ b/debug/curl/upload_w2l.py @@ -4,7 +4,7 @@ import requests import re login_url = 'http://localhost:12345/login' -upload_url = 'http://localhost:12345/upload_w2l' +upload_url = 'http://localhost:12345/w2l/upload' payload = { 'username_or_email': 'echelon', diff --git a/debug/mysql/last_w2l_inference_job.sh b/debug/mysql/last_w2l_inference_job.sh new file mode 100755 index 0000000000..9b53b29216 --- /dev/null +++ b/debug/mysql/last_w2l_inference_job.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u root \ + -proot \ + -h localhost \ + -D storyteller \ + -e "select * from w2l_inference_jobs\G" From 235d86ca2b25024364af4754e0532f25988c1eb7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 May 2021 23:42:54 -0400 Subject: [PATCH 0074/1042] refactor how endpoints are grouped --- debug/curl/upload_tts.py | 2 +- src/endpoints/enqueue_inference/mod.rs | 2 -- src/endpoints/enqueue_uploads/mod.rs | 2 -- src/endpoints/mod.rs | 6 +++--- .../infer_tts.rs => tts/enqueue_infer_tts.rs} | 0 .../enqueue_upload_tts_model.rs} | 0 src/endpoints/tts/mod.rs | 2 ++ .../infer_w2l.rs => w2l/enqueue_infer_w2l.rs} | 6 +++++- .../enqueue_upload_w2l_template.rs} | 0 src/endpoints/w2l/mod.rs | 2 ++ src/main.rs | 20 +++++++++++++------ 11 files changed, 27 insertions(+), 15 deletions(-) delete mode 100644 src/endpoints/enqueue_inference/mod.rs delete mode 100644 src/endpoints/enqueue_uploads/mod.rs rename src/endpoints/{enqueue_inference/infer_tts.rs => tts/enqueue_infer_tts.rs} (100%) rename src/endpoints/{enqueue_uploads/upload_tts_model.rs => tts/enqueue_upload_tts_model.rs} (100%) create mode 100644 src/endpoints/tts/mod.rs rename src/endpoints/{enqueue_inference/infer_w2l.rs => w2l/enqueue_infer_w2l.rs} (97%) rename src/endpoints/{enqueue_uploads/upload_w2l_template.rs => w2l/enqueue_upload_w2l_template.rs} (100%) create mode 100644 src/endpoints/w2l/mod.rs diff --git a/debug/curl/upload_tts.py b/debug/curl/upload_tts.py index b5bf458be1..5a696c86bf 100755 --- a/debug/curl/upload_tts.py +++ b/debug/curl/upload_tts.py @@ -4,7 +4,7 @@ import requests import re login_url = 'http://localhost:12345/login' -upload_url = 'http://localhost:12345/upload_tts' +upload_url = 'http://localhost:12345/tts/upload' payload = { 'username_or_email': 'echelon', diff --git a/src/endpoints/enqueue_inference/mod.rs b/src/endpoints/enqueue_inference/mod.rs deleted file mode 100644 index 2b7f9d8025..0000000000 --- a/src/endpoints/enqueue_inference/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod infer_tts; -pub mod infer_w2l; diff --git a/src/endpoints/enqueue_uploads/mod.rs b/src/endpoints/enqueue_uploads/mod.rs deleted file mode 100644 index 1212193bbc..0000000000 --- a/src/endpoints/enqueue_uploads/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod upload_tts_model; -pub mod upload_w2l_template; diff --git a/src/endpoints/mod.rs b/src/endpoints/mod.rs index 970c7a3129..60bd52fecb 100644 --- a/src/endpoints/mod.rs +++ b/src/endpoints/mod.rs @@ -1,6 +1,6 @@ pub mod default_route_404; -pub mod enqueue_inference; -pub mod enqueue_uploads; pub mod misc; pub mod root_index; -pub mod users; \ No newline at end of file +pub mod tts; +pub mod users; +pub mod w2l; \ No newline at end of file diff --git a/src/endpoints/enqueue_inference/infer_tts.rs b/src/endpoints/tts/enqueue_infer_tts.rs similarity index 100% rename from src/endpoints/enqueue_inference/infer_tts.rs rename to src/endpoints/tts/enqueue_infer_tts.rs diff --git a/src/endpoints/enqueue_uploads/upload_tts_model.rs b/src/endpoints/tts/enqueue_upload_tts_model.rs similarity index 100% rename from src/endpoints/enqueue_uploads/upload_tts_model.rs rename to src/endpoints/tts/enqueue_upload_tts_model.rs diff --git a/src/endpoints/tts/mod.rs b/src/endpoints/tts/mod.rs new file mode 100644 index 0000000000..371b17ad4d --- /dev/null +++ b/src/endpoints/tts/mod.rs @@ -0,0 +1,2 @@ +pub mod enqueue_infer_tts; +pub mod enqueue_upload_tts_model; \ No newline at end of file diff --git a/src/endpoints/enqueue_inference/infer_w2l.rs b/src/endpoints/w2l/enqueue_infer_w2l.rs similarity index 97% rename from src/endpoints/enqueue_inference/infer_w2l.rs rename to src/endpoints/w2l/enqueue_infer_w2l.rs index a1250d6fa0..1a4e549a25 100644 --- a/src/endpoints/enqueue_inference/infer_w2l.rs +++ b/src/endpoints/w2l/enqueue_infer_w2l.rs @@ -103,6 +103,7 @@ pub async fn infer_w2l_handler( return Err(InferW2lError::BadInput("w2l token is required".to_string())); }, Some(t) => { + // TODO: CHECK DATABASE! t.to_string() }, }; @@ -112,7 +113,10 @@ pub async fn infer_w2l_handler( // TODO: Allow audio uploads. return Err(InferW2lError::BadInput("tts token is required".to_string())); }, - Some(t) => t.to_string(), + Some(t) => { + // TODO: CHECK DATABASE! + t.to_string() + }, }; let ip_address = get_request_ip(&http_request); diff --git a/src/endpoints/enqueue_uploads/upload_w2l_template.rs b/src/endpoints/w2l/enqueue_upload_w2l_template.rs similarity index 100% rename from src/endpoints/enqueue_uploads/upload_w2l_template.rs rename to src/endpoints/w2l/enqueue_upload_w2l_template.rs diff --git a/src/endpoints/w2l/mod.rs b/src/endpoints/w2l/mod.rs new file mode 100644 index 0000000000..d34936265e --- /dev/null +++ b/src/endpoints/w2l/mod.rs @@ -0,0 +1,2 @@ +pub mod enqueue_infer_w2l; +pub mod enqueue_upload_w2l_template; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 011bbb1417..c8b808c009 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,14 +19,14 @@ use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; use crate::endpoints::default_route_404::default_route_404; -use crate::endpoints::enqueue_uploads::upload_tts_model::upload_tts_model_handler; -use crate::endpoints::enqueue_uploads::upload_w2l_template::upload_w2l_template_handler; use crate::endpoints::misc::enable_alpha::enable_alpha; use crate::endpoints::root_index::get_root_index; use crate::endpoints::users::create_account::create_account_handler; use crate::endpoints::users::login::login_handler; use crate::endpoints::users::logout::logout_handler; use crate::endpoints::users::session_info::session_info_handler; +use crate::endpoints::w2l::enqueue_infer_w2l::infer_w2l_handler; +use crate::endpoints::w2l::enqueue_upload_w2l_template::upload_w2l_template_handler; use crate::server_state::{ServerState, EnvConfig}; use crate::util::cookies::CookieManager; use crate::util::session_checker::SessionChecker; @@ -34,7 +34,7 @@ use log::{info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; -use crate::endpoints::enqueue_inference::infer_w2l::infer_w2l_handler; +use crate::endpoints::tts::enqueue_upload_tts_model::upload_tts_model_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -171,9 +171,17 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) .service( - web::resource("/upload_tts") - .route(web::post().to(upload_tts_model_handler)) - .route(web::head().to(|| HttpResponse::Ok())) + web::scope("/tts") + .service( + web::resource("/upload") + .route(web::post().to(upload_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + //.service( + // web::resource("/inference") + // .route(web::post().to()) + // .route(web::head().to(|| HttpResponse::Ok())) + //) ) .service( web::scope("/w2l") From fa546eac0504549d6740f33f371e76976b8abb3d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 21 May 2021 00:00:35 -0400 Subject: [PATCH 0075/1042] tts inference enqueue job + test scripts --- debug/curl/infer_tts.py | 43 ++++++ debug/mysql/last_tts_inference_job.sh | 7 + src/endpoints/tts/enqueue_infer_tts.rs | 182 +++++++++++++++++++++++++ src/endpoints/w2l/enqueue_infer_w2l.rs | 3 - src/main.rs | 11 +- 5 files changed, 238 insertions(+), 8 deletions(-) create mode 100755 debug/curl/infer_tts.py create mode 100755 debug/mysql/last_tts_inference_job.sh diff --git a/debug/curl/infer_tts.py b/debug/curl/infer_tts.py new file mode 100755 index 0000000000..84d660ba23 --- /dev/null +++ b/debug/curl/infer_tts.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import requests +import re + +login_url = 'http://localhost:12345/login' +inference_url = 'http://localhost:12345/tts/inference' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +# NB: May not be set due to cookie domain: +#r.cookies['session'] + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + + +print("===== Infer TTS =====") + +cookies = { 'session': session_cookie } + +payload = { + 'tts_model_token': 'TTS_TOKEN', + 'inference_text': 'this is the text to generate', +} + +r = requests.post(inference_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + diff --git a/debug/mysql/last_tts_inference_job.sh b/debug/mysql/last_tts_inference_job.sh new file mode 100755 index 0000000000..4278bdc435 --- /dev/null +++ b/debug/mysql/last_tts_inference_job.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u root \ + -proot \ + -h localhost \ + -D storyteller \ + -e "select * from tts_inference_jobs\G" diff --git a/src/endpoints/tts/enqueue_infer_tts.rs b/src/endpoints/tts/enqueue_infer_tts.rs index e69de29bb2..ff3ae002a5 100644 --- a/src/endpoints/tts/enqueue_infer_tts.rs +++ b/src/endpoints/tts/enqueue_infer_tts.rs @@ -0,0 +1,182 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::server_state::ServerState; +use crate::util::ip_address::get_request_ip; +use crate::util::tokens::random_token; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +const NEW_USER_ROLE: &'static str = "new-user"; + +#[derive(Deserialize)] +pub struct InferTtsRequest { + tts_model_token: String, + inference_text: String, + creator_set_visibility: Option, +} + +#[derive(Serialize)] +pub struct InferTtsSuccessResponse { + pub success: bool, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum InferTtsError { + BadInput(String), + ServerError, +} + +impl ResponseError for InferTtsError { + fn status_code(&self) -> StatusCode { + match *self { + InferTtsError::BadInput(_) => StatusCode::BAD_REQUEST, + InferTtsError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + InferTtsError::BadInput(reason) => reason.to_string(), + InferTtsError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn infer_tts_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + InferTtsError::ServerError + })?; + + let mut maybe_user_token : Option = maybe_user_session + .as_ref() + .map(|user_session| user_session.user_token.to_string()); + + let inference_text = &request.inference_text.trim().to_string(); + + if let Err(reason) = validate_inference_text(&inference_text) { + return Err(InferTtsError::BadInput(reason)); + } + + // TODO: CHECK DATABASE! + let model_token = request.tts_model_token.to_string(); + + let ip_address = get_request_ip(&http_request); + let creator_set_visibility = "public".to_string(); + + let query_result = sqlx::query!( + r#" +INSERT INTO tts_inference_jobs +SET + model_token = ?, + inference_text = ?, + maybe_creator_user_token = ?, + creator_ip_address = ?, + creator_set_visibility = ?, + status = "pending" + "#, + model_token, + inference_text, + maybe_user_token, + ip_address, + creator_set_visibility + ) + .execute(&server_state.mysql_pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + warn!("New w2l inference job creation DB error: {:?}", err); + + // NB: SQLSTATE[23000]: Integrity constraint violation + // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access) + match err { + Database(err) => { + let maybe_code = err.code().map(|c| c.into_owned()); + /*match maybe_code.as_deref() { + Some("23000") => { + if err.message().contains("username") { + return Err(UsernameTaken); + } else if err.message().contains("email_address") { + return Err(EmailTaken); + } + } + _ => {}, + }*/ + }, + _ => {}, + } + return Err(InferTtsError::ServerError); + } + }; + + info!("new w2l inference job id: {}", record_id); + + let response = InferTtsSuccessResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| InferTtsError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} + +pub fn validate_inference_text(text: &str) -> Result<(), String> { + if text.len() < 3 { + return Err("text is too short".to_string()); + } + + if text.len() > 1024 { + return Err("text is too long".to_string()); + } + + Ok(()) +} diff --git a/src/endpoints/w2l/enqueue_infer_w2l.rs b/src/endpoints/w2l/enqueue_infer_w2l.rs index 1a4e549a25..703d5365e6 100644 --- a/src/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/endpoints/w2l/enqueue_infer_w2l.rs @@ -43,7 +43,6 @@ pub struct ErrorResponse { #[derive(Debug, Display)] pub enum InferW2lError { BadInput(String), - MustBeLoggedIn, ServerError, } @@ -51,7 +50,6 @@ impl ResponseError for InferW2lError { fn status_code(&self) -> StatusCode { match *self { InferW2lError::BadInput(_) => StatusCode::BAD_REQUEST, - InferW2lError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, InferW2lError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } } @@ -59,7 +57,6 @@ impl ResponseError for InferW2lError { fn error_response(&self) -> HttpResponse { let error_reason = match self { InferW2lError::BadInput(reason) => reason.to_string(), - InferW2lError::MustBeLoggedIn => "user must be logged in".to_string(), InferW2lError::ServerError => "server error".to_string(), }; diff --git a/src/main.rs b/src/main.rs index c8b808c009..19208f2157 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,6 +35,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use crate::endpoints::tts::enqueue_upload_tts_model::upload_tts_model_handler; +use crate::endpoints::tts::enqueue_infer_tts::infer_tts_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -177,11 +178,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::post().to(upload_tts_model_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - //.service( - // web::resource("/inference") - // .route(web::post().to()) - // .route(web::head().to(|| HttpResponse::Ok())) - //) + .service( + web::resource("/inference") + .route(web::post().to(infer_tts_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service( web::scope("/w2l") From ddf445701d63e9541c26270fef89450a14a87002 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 21 May 2021 00:03:18 -0400 Subject: [PATCH 0076/1042] refactor filenames --- src/common_queries/sessions.rs | 2 +- src/endpoints/tts/enqueue_infer_tts.rs | 2 +- src/endpoints/tts/enqueue_upload_tts_model.rs | 2 +- src/endpoints/users/create_account.rs | 2 +- src/endpoints/users/login.rs | 2 +- src/endpoints/users/logout.rs | 2 +- src/endpoints/users/session_info.rs | 2 +- src/endpoints/w2l/enqueue_infer_w2l.rs | 2 +- src/endpoints/w2l/enqueue_upload_w2l_template.rs | 2 +- src/main.rs | 2 +- src/server_state.rs | 2 +- src/util/{cookies.rs => cookie_manager.rs} | 0 src/util/mod.rs | 4 ++-- src/util/{tokens.rs => random_token.rs} | 2 +- src/util/session_checker.rs | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) rename src/util/{cookies.rs => cookie_manager.rs} (100%) rename src/util/{tokens.rs => random_token.rs} (93%) diff --git a/src/common_queries/sessions.rs b/src/common_queries/sessions.rs index e0260af3e2..1c88e372c7 100644 --- a/src/common_queries/sessions.rs +++ b/src/common_queries/sessions.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use log::{info, warn}; use crate::AnyhowResult; -use crate::util::tokens::random_token; +use crate::util::random_token::random_token; use sqlx::MySqlPool; pub async fn create_session_for_user(user_token: &str, ip_address: &str, mysql_pool: &MySqlPool) diff --git a/src/endpoints/tts/enqueue_infer_tts.rs b/src/endpoints/tts/enqueue_infer_tts.rs index ff3ae002a5..56d1d307fd 100644 --- a/src/endpoints/tts/enqueue_infer_tts.rs +++ b/src/endpoints/tts/enqueue_infer_tts.rs @@ -8,7 +8,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; -use crate::util::tokens::random_token; +use crate::util::random_token::random_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/src/endpoints/tts/enqueue_upload_tts_model.rs b/src/endpoints/tts/enqueue_upload_tts_model.rs index a07ad99848..de0068a435 100644 --- a/src/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/endpoints/tts/enqueue_upload_tts_model.rs @@ -6,7 +6,7 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; -use crate::util::tokens::random_token; +use crate::util::random_token::random_token; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/src/endpoints/users/create_account.rs b/src/endpoints/users/create_account.rs index c86fb837df..3f19d01728 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/endpoints/users/create_account.rs @@ -7,7 +7,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; -use crate::util::tokens::random_token; +use crate::util::random_token::random_token; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/src/endpoints/users/login.rs b/src/endpoints/users/login.rs index fc5e82b659..f3db830372 100644 --- a/src/endpoints/users/login.rs +++ b/src/endpoints/users/login.rs @@ -7,7 +7,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; -use crate::util::tokens::random_token; +use crate::util::random_token::random_token; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/src/endpoints/users/logout.rs b/src/endpoints/users/logout.rs index 30eaf12ff2..18dbf3b073 100644 --- a/src/endpoints/users/logout.rs +++ b/src/endpoints/users/logout.rs @@ -7,7 +7,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; -use crate::util::tokens::random_token; +use crate::util::random_token::random_token; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/src/endpoints/users/session_info.rs b/src/endpoints/users/session_info.rs index a24d757a4a..70a14d7942 100644 --- a/src/endpoints/users/session_info.rs +++ b/src/endpoints/users/session_info.rs @@ -7,7 +7,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; -use crate::util::tokens::random_token; +use crate::util::random_token::random_token; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/src/endpoints/w2l/enqueue_infer_w2l.rs b/src/endpoints/w2l/enqueue_infer_w2l.rs index 703d5365e6..ffc7640377 100644 --- a/src/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/endpoints/w2l/enqueue_infer_w2l.rs @@ -6,7 +6,7 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; -use crate::util::tokens::random_token; +use crate::util::random_token::random_token; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/src/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/endpoints/w2l/enqueue_upload_w2l_template.rs index 6feac2c310..9872755af6 100644 --- a/src/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -6,7 +6,7 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; -use crate::util::tokens::random_token; +use crate::util::random_token::random_token; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/src/main.rs b/src/main.rs index 19208f2157..f5a2afb2d8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,7 +28,7 @@ use crate::endpoints::users::session_info::session_info_handler; use crate::endpoints::w2l::enqueue_infer_w2l::infer_w2l_handler; use crate::endpoints::w2l::enqueue_upload_w2l_template::upload_w2l_template_handler; use crate::server_state::{ServerState, EnvConfig}; -use crate::util::cookies::CookieManager; +use crate::util::cookie_manager::CookieManager; use crate::util::session_checker::SessionChecker; use log::{info}; use sqlx::MySqlPool; diff --git a/src/server_state.rs b/src/server_state.rs index 7cc0801642..222dc8c2f7 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,5 +1,5 @@ use sqlx::MySqlPool; -use crate::util::cookies::CookieManager; +use crate::util::cookie_manager::CookieManager; use crate::util::session_checker::SessionChecker; /// State that is injected into every endpoint. diff --git a/src/util/cookies.rs b/src/util/cookie_manager.rs similarity index 100% rename from src/util/cookies.rs rename to src/util/cookie_manager.rs diff --git a/src/util/mod.rs b/src/util/mod.rs index 9f8449793c..ca64d2a562 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,4 +1,4 @@ -pub mod cookies; +pub mod cookie_manager; pub mod ip_address; -pub mod tokens; +pub mod random_token; pub mod session_checker; \ No newline at end of file diff --git a/src/util/tokens.rs b/src/util/random_token.rs similarity index 93% rename from src/util/tokens.rs rename to src/util/random_token.rs index 7551e50fcd..9805463674 100644 --- a/src/util/tokens.rs +++ b/src/util/random_token.rs @@ -18,7 +18,7 @@ pub fn random_token(length: usize) -> String { #[cfg(test)] mod tests { - use crate::util::tokens::random_token; + use crate::util::random_token::random_token; #[test] fn random_token_length() { diff --git a/src/util/session_checker.rs b/src/util/session_checker.rs index 1a7d9b2448..00b7471eeb 100644 --- a/src/util/session_checker.rs +++ b/src/util/session_checker.rs @@ -1,7 +1,7 @@ use actix_web::HttpRequest; use anyhow::anyhow; use crate::AnyhowResult; -use crate::util::cookies::CookieManager; +use crate::util::cookie_manager::CookieManager; use log::{info, warn}; use sqlx::MySqlPool; use sqlx::error::Error::RowNotFound; From e154faaf32dae51b16f9aa398f2b67cac5adf296 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 21 May 2021 05:12:53 -0400 Subject: [PATCH 0077/1042] add idempotency tokens to upload endpoints --- debug/curl/upload_tts.py | 2 ++ debug/curl/upload_w2l.py | 4 +++- .../2021-05-15-071545_asset_upload_jobs/up.sql | 10 +++++++++- src/endpoints/tts/enqueue_upload_tts_model.rs | 16 ++++++++++++++++ src/endpoints/w2l/enqueue_upload_w2l_template.rs | 16 ++++++++++++++++ 5 files changed, 46 insertions(+), 2 deletions(-) diff --git a/debug/curl/upload_tts.py b/debug/curl/upload_tts.py index 5a696c86bf..f8da0109fd 100755 --- a/debug/curl/upload_tts.py +++ b/debug/curl/upload_tts.py @@ -2,6 +2,7 @@ import requests import re +import uuid login_url = 'http://localhost:12345/login' upload_url = 'http://localhost:12345/tts/upload' @@ -32,6 +33,7 @@ print("===== Upload TTS=====") cookies = { 'session': session_cookie } payload = { + 'idempotency_token': str(uuid.uuid4()), 'title': 'this is a new text to speech model', 'download_url': 'https://website.com', } diff --git a/debug/curl/upload_w2l.py b/debug/curl/upload_w2l.py index 9479de1327..37b6e02d55 100755 --- a/debug/curl/upload_w2l.py +++ b/debug/curl/upload_w2l.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 -import requests import re +import requests +import uuid login_url = 'http://localhost:12345/login' upload_url = 'http://localhost:12345/w2l/upload' @@ -32,6 +33,7 @@ print("===== Upload W2L =====") cookies = { 'session': session_cookie } payload = { + 'idempotency_token': str(uuid.uuid4()), 'title': 'this is a new text to W2L template', 'download_url': 'https://video.com', } diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index eeeed7f44f..63da573ff0 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -5,6 +5,9 @@ CREATE TABLE tts_model_upload_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, + -- Idempotency token + uuid_idempotency_token VARCHAR(36) NOT NULL, + -- Foreign key to user creator_user_token VARCHAR(32) NOT NULL, @@ -85,6 +88,7 @@ CREATE TABLE tts_model_upload_jobs ( -- INDICES -- PRIMARY KEY (id), + UNIQUE KEY (uuid_idempotency_token), KEY fk_creator_user_token (creator_user_token), KEY fk_maybe_subject_token (maybe_subject_token), KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), @@ -97,7 +101,10 @@ CREATE TABLE tts_model_upload_jobs ( CREATE TABLE w2l_template_upload_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, - -- Foreign key to user + -- Idempotency token + uuid_idempotency_token VARCHAR(36) NOT NULL, + + -- Foreign key to user creator_user_token VARCHAR(32) NOT NULL, -- For abuse tracking. @@ -174,6 +181,7 @@ CREATE TABLE w2l_template_upload_jobs ( -- INDICES -- PRIMARY KEY (id), + UNIQUE KEY (uuid_idempotency_token), KEY fk_creator_user_token (creator_user_token), KEY fk_maybe_subject_token (maybe_subject_token), KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), diff --git a/src/endpoints/tts/enqueue_upload_tts_model.rs b/src/endpoints/tts/enqueue_upload_tts_model.rs index de0068a435..3775dddebb 100644 --- a/src/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/endpoints/tts/enqueue_upload_tts_model.rs @@ -24,6 +24,7 @@ const NEW_USER_ROLE: &'static str = "new-user"; #[derive(Deserialize)] pub struct UploadTtsModelRequest { + idempotency_token: String, title: String, download_url: String, download_url_type: Option, @@ -105,12 +106,17 @@ pub async fn upload_tts_model_handler( } }; + if let Err(reason) = validate_idempotency_token(&request.idempotency_token) { + return Err(UploadTtsModelError::BadInput(reason)); + } + if let Err(reason) = validate_model_title(&request.title) { return Err(UploadTtsModelError::BadInput(reason)); } let ip_address = get_request_ip(&http_request); + let uuid = request.idempotency_token.to_string(); let title = request.title.to_string(); let download_url = request.download_url.to_string(); @@ -124,6 +130,7 @@ pub async fn upload_tts_model_handler( r#" INSERT INTO tts_model_upload_jobs SET + uuid_idempotency_token = ?, creator_user_token = ?, creator_ip_address = ?, creator_set_visibility = ?, @@ -135,6 +142,7 @@ SET download_url_type = ?, status = "pending" "#, + uuid.to_string(), user_session.user_token.to_string(), ip_address.to_string(), creator_set_visibility.to_string(), @@ -190,3 +198,11 @@ SET .content_type("application/json") .body(body)) } + +fn validate_idempotency_token(token: &str) -> Result<(), String> { + if token.len() != 36 { + return Err("idempotency token should be 36 characters".to_string()); + } + + Ok(()) +} diff --git a/src/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/endpoints/w2l/enqueue_upload_w2l_template.rs index 9872755af6..9b57b2533e 100644 --- a/src/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -24,6 +24,7 @@ const NEW_USER_ROLE: &'static str = "new-user"; #[derive(Deserialize)] pub struct UploadW2lTemplateRequest { + idempotency_token: String, title: String, download_url: String, download_url_type: Option, @@ -105,12 +106,17 @@ pub async fn upload_w2l_template_handler( } }; + if let Err(reason) = validate_idempotency_token(&request.idempotency_token) { + return Err(UploadW2lTemplateError::BadInput(reason)); + } + if let Err(reason) = validate_model_title(&request.title) { return Err(UploadW2lTemplateError::BadInput(reason)); } let ip_address = get_request_ip(&http_request); + let uuid = request.idempotency_token.to_string(); let title = request.title.to_string(); let download_url = request.download_url.to_string(); @@ -124,6 +130,7 @@ pub async fn upload_w2l_template_handler( r#" INSERT INTO w2l_template_upload_jobs SET + uuid_idempotency_token = ?, creator_user_token = ?, creator_ip_address = ?, creator_set_visibility = ?, @@ -135,6 +142,7 @@ SET download_url_type = ?, status = "pending" "#, + uuid.to_string(), user_session.user_token.to_string(), ip_address.to_string(), creator_set_visibility.to_string(), @@ -190,3 +198,11 @@ SET .content_type("application/json") .body(body)) } + +fn validate_idempotency_token(token: &str) -> Result<(), String> { + if token.len() != 36 { + return Err("idempotency token should be 36 characters".to_string()); + } + + Ok(()) +} From 5bd14c11278d49aba1d925b3c512a5a168569e92 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 01:58:59 -0400 Subject: [PATCH 0078/1042] 4 jobs + docker builds of each --- Cargo.toml | 16 ++++++++++++++++ Dockerfile | 38 +++++++++++++++++++++++++++++++++++--- src/tts_download_job.rs | 7 +++++++ src/tts_inference_job.rs | 7 +++++++ src/util/anyhow_result.rs | 2 ++ src/util/mod.rs | 1 + src/w2l_download_job.rs | 7 +++++++ src/w2l_inference_job.rs | 7 +++++++ 8 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 src/tts_download_job.rs create mode 100644 src/tts_inference_job.rs create mode 100644 src/util/anyhow_result.rs create mode 100644 src/w2l_download_job.rs create mode 100644 src/w2l_inference_job.rs diff --git a/Cargo.toml b/Cargo.toml index 28beac36a1..08021637d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,22 @@ publish = false name = "storyteller-web" path = "src/main.rs" +[[bin]] +name = "tts-download-job" +path = "src/tts_download_job.rs" + +[[bin]] +name = "tts-inference-job" +path = "src/tts_inference_job.rs" + +[[bin]] +name = "w2l-download-job" +path = "src/w2l_download_job.rs" + +[[bin]] +name = "w2l-inference-job" +path = "src/w2l_inference_job.rs" + [dependencies] anyhow = "1.0.40" base64 = "0.13.0" diff --git a/Dockerfile b/Dockerfile index 224a785a73..ac44a4658b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,36 @@ COPY migrations/ ./migrations RUN $HOME/.cargo/bin/cargo fetch -#RUN SQLX_OFFLINE=true LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} $HOME/.cargo/bin/cargo build -RUN SQLX_OFFLINE=true LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} $HOME/.cargo/bin/cargo build --release +# Build all the binaries. +RUN SQLX_OFFLINE=true \ + LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ + $HOME/.cargo/bin/cargo build \ + --release \ + --bin storyteller-web + +RUN SQLX_OFFLINE=true \ + LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ + $HOME/.cargo/bin/cargo build \ + --release \ + --bin tts-download-job + +RUN SQLX_OFFLINE=true \ + LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ + $HOME/.cargo/bin/cargo build \ + --release \ + --bin w2l-download-job + +RUN SQLX_OFFLINE=true \ + LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ + $HOME/.cargo/bin/cargo build \ + --release \ + --bin tts-inference-job + +RUN SQLX_OFFLINE=true \ + LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ + $HOME/.cargo/bin/cargo build \ + --release \ + --bin w2l-inference-job # Final image FROM ubuntu:xenial @@ -33,8 +61,12 @@ RUN apt-get update \ curl \ wget -#COPY --from=build /tmp/target/debug/storyteller-web / +# Copy all the binaries. COPY --from=build /tmp/target/release/storyteller-web / +COPY --from=build /tmp/target/release/tts-download-job / +COPY --from=build /tmp/target/release/tts-inference-job / +COPY --from=build /tmp/target/release/w2l-download-job / +COPY --from=build /tmp/target/release/w2l-inference-job / # SSL certs are required for crypto COPY --from=build /etc/ssl /etc/ssl diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs new file mode 100644 index 0000000000..559e546bd4 --- /dev/null +++ b/src/tts_download_job.rs @@ -0,0 +1,7 @@ +pub mod util; + +use crate::util::anyhow_result::AnyhowResult; + +fn main() -> AnyhowResult<()> { + Ok(()) +} diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs new file mode 100644 index 0000000000..559e546bd4 --- /dev/null +++ b/src/tts_inference_job.rs @@ -0,0 +1,7 @@ +pub mod util; + +use crate::util::anyhow_result::AnyhowResult; + +fn main() -> AnyhowResult<()> { + Ok(()) +} diff --git a/src/util/anyhow_result.rs b/src/util/anyhow_result.rs new file mode 100644 index 0000000000..7784374637 --- /dev/null +++ b/src/util/anyhow_result.rs @@ -0,0 +1,2 @@ +/// Easier to import than anyhow::Result. +pub type AnyhowResult = anyhow::Result; diff --git a/src/util/mod.rs b/src/util/mod.rs index ca64d2a562..2b2f0d0bd0 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,3 +1,4 @@ +pub mod anyhow_result; pub mod cookie_manager; pub mod ip_address; pub mod random_token; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs new file mode 100644 index 0000000000..559e546bd4 --- /dev/null +++ b/src/w2l_download_job.rs @@ -0,0 +1,7 @@ +pub mod util; + +use crate::util::anyhow_result::AnyhowResult; + +fn main() -> AnyhowResult<()> { + Ok(()) +} diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs new file mode 100644 index 0000000000..559e546bd4 --- /dev/null +++ b/src/w2l_inference_job.rs @@ -0,0 +1,7 @@ +pub mod util; + +use crate::util::anyhow_result::AnyhowResult; + +fn main() -> AnyhowResult<()> { + Ok(()) +} From bc1ccbc1769c2e3a0c471c53dd36b9bfdde91dec Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 02:36:10 -0400 Subject: [PATCH 0079/1042] job can query results! --- Cargo.lock | 565 +++++++++++++++++++++++++++++++++++++++- Cargo.toml | 9 + src/tts_download_job.rs | 128 ++++++++- 3 files changed, 697 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1fa0885f1c..0a85409874 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,7 +15,7 @@ dependencies = [ "log", "pin-project 0.4.28", "tokio 0.2.25", - "tokio-util", + "tokio-util 0.3.1", ] [[package]] @@ -97,7 +97,7 @@ dependencies = [ "futures-core", "futures-util", "fxhash", - "h2", + "h2 0.2.7", "http", "httparse", "indexmap", @@ -372,6 +372,16 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "async-attributes" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "async-channel" version = "1.6.1" @@ -486,6 +496,7 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9f06685bad74e0570f5213741bea82158279a4103d988e57bfada11ad230341" dependencies = [ + "async-attributes", "async-channel", "async-global-executor", "async-io", @@ -540,6 +551,22 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" +[[package]] +name = "attohttpc" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb8867f378f33f78a811a8eb9bf108ad99430d7aad43315dd9319c827ef6247" +dependencies = [ + "http", + "log", + "native-tls", + "openssl", + "serde", + "serde_json", + "url 2.2.2", + "wildmatch", +] + [[package]] name = "atty" version = "0.2.14" @@ -587,6 +614,31 @@ dependencies = [ "serde_urlencoded", ] +[[package]] +name = "aws-creds" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1331d069460a674d42bd27c12b47ce578f789954c7bd7f239fd030771eca6616" +dependencies = [ + "anyhow", + "attohttpc", + "dirs", + "rust-ini", + "serde", + "serde-xml-rs", + "serde_derive", + "url 2.2.2", +] + +[[package]] +name = "aws-region" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2884b8f2aaeb4a4bf80b219b4fe1d340139ca9331679c57e0fd4a24f571a78bd" +dependencies = [ + "anyhow", +] + [[package]] name = "base-x" version = "0.2.8" @@ -906,6 +958,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct", +] + [[package]] name = "ctor" version = "0.1.20" @@ -960,12 +1021,41 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dirs" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + [[package]] name = "discard" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "dlv-list" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68df3f2b690c1b86e65ef7830956aededf3cb0a16f898f79b9a6f421a7b6211b" +dependencies = [ + "rand 0.8.3", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -1269,6 +1359,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "google-drive" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fe473f3853a7668300abb646ffb01b6eb7b97548f24575f73079b03e5c73b2" +dependencies = [ + "bytes 1.0.1", + "reqwest", + "serde", + "yup-oauth2", +] + [[package]] name = "h2" version = "0.2.7" @@ -1284,11 +1386,30 @@ dependencies = [ "indexmap", "slab", "tokio 0.2.25", - "tokio-util", + "tokio-util 0.3.1", "tracing", "tracing-futures", ] +[[package]] +name = "h2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" +dependencies = [ + "bytes 1.0.1", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 1.5.0", + "tokio-util 0.6.7", + "tracing", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -1363,12 +1484,29 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" +dependencies = [ + "bytes 1.0.1", + "http", + "pin-project-lite 0.2.6", +] + [[package]] name = "httparse" version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" +[[package]] +name = "httpdate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9" + [[package]] name = "humantime" version = "1.3.0" @@ -1378,6 +1516,60 @@ dependencies = [ "quick-error", ] +[[package]] +name = "hyper" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54" +dependencies = [ + "bytes 1.0.1", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.3", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.7", + "socket2 0.4.0", + "tokio 1.5.0", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio 1.5.0", + "tokio-rustls", + "webpki", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes 1.0.1", + "hyper", + "native-tls", + "tokio 1.5.0", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.1.5" @@ -1437,9 +1629,15 @@ dependencies = [ "socket2 0.3.19", "widestring", "winapi 0.3.9", - "winreg", + "winreg 0.6.2", ] +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + [[package]] name = "itoa" version = "0.4.7" @@ -1587,6 +1785,34 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "maybe-async" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3919783e441e38b0512fae9eebd87c445c42e39508231fba2a8b9fd7ba88e999" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "md-5" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +dependencies = [ + "block-buffer", + "digest", + "opaque-debug", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.4.0" @@ -1609,6 +1835,15 @@ dependencies = [ "unicase", ] +[[package]] +name = "minidom" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e" +dependencies = [ + "quick-xml", +] + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -1880,6 +2115,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-multimap" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485" +dependencies = [ + "dlv-list", + "hashbrown", +] + [[package]] name = "parking" version = "2.0.0" @@ -2044,6 +2289,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.9" @@ -2149,6 +2403,16 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom 0.2.2", + "redox_syscall", +] + [[package]] name = "regex" version = "1.5.4" @@ -2175,6 +2439,41 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "reqwest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" +dependencies = [ + "base64 0.13.0", + "bytes 1.0.1", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding 2.1.0", + "pin-project-lite 0.2.6", + "serde", + "serde_json", + "serde_urlencoded", + "tokio 1.5.0", + "tokio-native-tls", + "url 2.2.2", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.7.0", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -2185,6 +2484,21 @@ dependencies = [ "quick-error", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "rsa" version = "0.4.0" @@ -2205,6 +2519,48 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rust-ini" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55b134767a87e0b086f73a4ce569ac9ce7d202f39c8eab6caa266e2617e73ac6" +dependencies = [ + "cfg-if 0.1.10", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.27.0-rc4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c93272c1d654d492f8ab30b94cd43d98f2700b1db55b2576aff7712ce40e3ef" +dependencies = [ + "anyhow", + "async-trait", + "aws-creds", + "aws-region", + "base64 0.13.0", + "cfg-if 1.0.0", + "chrono", + "futures", + "hex", + "hmac", + "http", + "log", + "maybe-async", + "md5", + "minidom", + "percent-encoding 2.1.0", + "reqwest", + "serde", + "serde-xml-rs", + "serde_derive", + "sha2", + "tokio 1.5.0", + "tokio-stream", + "url 2.2.2", +] + [[package]] name = "rustc_version" version = "0.2.3" @@ -2214,6 +2570,31 @@ dependencies = [ "semver", ] +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls", + "schannel", + "security-framework", +] + [[package]] name = "ryu" version = "1.0.5" @@ -2236,6 +2617,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "2.2.0" @@ -2283,6 +2680,18 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-xml-rs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0bf1ba0696ccf0872866277143ff1fd14d22eec235d2b23702f95e6660f7dfa" +dependencies = [ + "log", + "serde", + "thiserror", + "xml-rs", +] + [[package]] name = "serde_derive" version = "1.0.125" @@ -2599,6 +3008,7 @@ dependencies = [ "actix-utils 3.0.0", "actix-web", "anyhow", + "async-std", "base64 0.13.0", "bcrypt", "chrono", @@ -2606,13 +3016,17 @@ dependencies = [ "diesel", "easyenv", "futures", + "google-drive", + "hex", "hmac", "hostname", "jwt", "lazy_static", "log", + "md-5", "rand 0.8.3", "regex", + "rust-s3", "serde", "serde_derive", "serde_json", @@ -2810,15 +3224,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" dependencies = [ "autocfg 1.0.1", + "bytes 1.0.1", "libc", + "memchr", "mio 0.7.11", "once_cell", "parking_lot", "pin-project-lite 0.2.6", "signal-hook-registry", + "tokio-macros", "winapi 0.3.9", ] +[[package]] +name = "tokio-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c49e3df43841dafb86046472506755d8501c5615673955f6aa17181125d13c37" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio 1.5.0", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio 1.5.0", + "webpki", +] + +[[package]] +name = "tokio-stream" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8864d706fdb3cc0843a49647ac892720dac98a6eeb818b77190592cf4994066" +dependencies = [ + "futures-core", + "pin-project-lite 0.2.6", + "tokio 1.5.0", +] + [[package]] name = "tokio-util" version = "0.3.1" @@ -2833,6 +3293,20 @@ dependencies = [ "tokio 0.2.25", ] +[[package]] +name = "tokio-util" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +dependencies = [ + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.6", + "tokio 1.5.0", +] + [[package]] name = "toml" version = "0.5.8" @@ -2842,6 +3316,12 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + [[package]] name = "tracing" version = "0.1.26" @@ -2912,6 +3392,12 @@ dependencies = [ "trust-dns-proto", ] +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.13.0" @@ -2963,6 +3449,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "1.7.2" @@ -3051,6 +3543,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -3070,6 +3572,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if 1.0.0", + "serde", + "serde_json", "wasm-bindgen-macro", ] @@ -3139,6 +3643,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "wepoll-sys" version = "3.0.1" @@ -3164,6 +3678,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" +[[package]] +name = "wildmatch" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f44b95f62d34113cf558c93511ac93027e03e9c29a60dd0fd70e6e025c7270a" + [[package]] name = "winapi" version = "0.2.8" @@ -3216,6 +3736,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "ws2_32-sys" version = "0.2.1" @@ -3232,6 +3761,34 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + +[[package]] +name = "yup-oauth2" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2573621fa28865489bdf556cdb8703604f4e8498612e3041a212ac20e59c4aeb" +dependencies = [ + "base64 0.13.0", + "chrono", + "futures", + "http", + "hyper", + "hyper-rustls", + "log", + "percent-encoding 2.1.0", + "rustls", + "seahash", + "serde", + "serde_json", + "tokio 1.5.0", + "url 2.2.2", +] + [[package]] name = "zeroize" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index 08021637d7..7fa8119421 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,10 +8,12 @@ authors = [ ] publish = false +# Main server monolith [[bin]] name = "storyteller-web" path = "src/main.rs" +# Jobs for managing downloads and inference [[bin]] name = "tts-download-job" path = "src/tts_download_job.rs" @@ -30,11 +32,13 @@ path = "src/w2l_inference_job.rs" [dependencies] anyhow = "1.0.40" +async-std = { version = "1.6", features = [ "attributes" ] } base64 = "0.13.0" chrono = { version = "0.4.19", features = ["serde"] } derive_more = "0.99.14" easyenv = "0.0.5" futures = "0.3.15" +hex = "0.4.3" hostname = "0.3.1" lazy_static = "1.4.0" log = "0.4.14" @@ -48,6 +52,10 @@ time = "0.2.26" # https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } +# Files +google-drive = "0.1.18" +rust-s3 = "0.27.0-rc4" + # Web actix-cors = "0.5.4" actix-files= "0.5" @@ -60,6 +68,7 @@ jwt = "0.13.0" # Crypto bcrypt = "0.8" hmac = "0.10.0" +md-5 = "0.9.1" sha2 = "0.9.5" # Serialization diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 559e546bd4..2a248a06a3 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -1,7 +1,133 @@ +#![allow(dead_code)] +#![allow(unused_mut)] +#![allow(unused_imports)] +#![warn(unused_must_use)] +//#![allow(warnings)] + pub mod util; use crate::util::anyhow_result::AnyhowResult; +use std::time::Duration; +use log::info; +use chrono::Utc; +use sqlx::mysql::MySqlPoolOptions; +use sqlx::MySqlPool; + +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; + +#[async_std::main] +async fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + info!("Obtaining hostname..."); + + let server_hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or("storyteller-web-unknown".to_string()); + + info!("Hostname: {}", &server_hostname); + + let db_connection_string = + easyenv::get_env_string_or_default( + "MYSQL_URL", + "mysql://root:root@localhost/storyteller"); + + info!("Connecting to database..."); + + let pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(&db_connection_string) + .await?; + + + loop { + query_and_download(&pool).await; + std::thread::sleep(Duration::from_millis(500)); + } -fn main() -> AnyhowResult<()> { Ok(()) } + +/// table: tts_model_upload_jobs +#[derive(Debug)] +pub struct JobRecord { + pub id: i64, + pub uuid_idempotency_token: String, + pub creator_user_token: String, + pub creator_ip_address: String, + pub creator_set_visibility: String, // TODO + pub title: String, + pub tts_model_type: String, // TODO + pub maybe_subject_token: Option, + pub maybe_actor_subject_token: Option, + pub download_url: Option, + pub download_url_type: String, // TODO + pub status: String, // TODO + pub attempt_count: i32, + pub failure_reason: Option, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub retry_at: Option>, +} + +pub async fn query_and_download(pool: &MySqlPool) { + + let job_records = sqlx::query_as!( + JobRecord, + r#" +SELECT * +FROM tts_model_upload_jobs +WHERE + ( + status IN ("pending", "attempt_failed") + ) + AND + ( + retry_at IS NULL + OR + retry_at < CURRENT_TIMESTAMP + ) + LIMIT 20 + "#, + ) + .fetch_all(pool) + .await; // TODO: This will return error if it doesn't exist + + info!("Job records..."); + for job_record in job_records { + info!("job record: {:?}", job_record); + } +/* + + let query_result = sqlx::query!( + r#" +INSERT INTO w2l_template_upload_jobs +SET + uuid_idempotency_token = ?, + creator_user_token = ?, + creator_ip_address = ?, + creator_set_visibility = ?, + title = ?, + template_type = ?, + maybe_subject_token = ?, + maybe_actor_subject_token = ?, + download_url = ?, + download_url_type = ?, + status = "pending" + "#, + uuid.to_string(), + user_session.user_token.to_string(), + ip_address.to_string(), + creator_set_visibility.to_string(), + title.to_string(), + template_type, + maybe_subject_token, + maybe_actor_subject_token, + download_url, + download_url_type + ) + .execute(&server_state.mysql_pool) + .await; + */ +} \ No newline at end of file From 4a3a39e5e11355a8a5561c323af65926f3b776f0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 02:47:42 -0400 Subject: [PATCH 0080/1042] Bake in job-related queries to build --- sqlx-data.json | 273 ++++++++++++++++++++++++++++++++++++++++ src/job/job_queries.rs | 54 ++++++++ src/job/mod.rs | 1 + src/main.rs | 3 + src/tts_download_job.rs | 99 ++------------- 5 files changed, 342 insertions(+), 88 deletions(-) create mode 100644 src/job/job_queries.rs create mode 100644 src/job/mod.rs diff --git a/sqlx-data.json b/sqlx-data.json index 6134b4cce3..93af734c4e 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1,5 +1,15 @@ { "db": "MySQL", + "10c82778c4228a4f82ac310371b64171b13d4e6089232bd8c536f1df2614b4c4": { + "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, "26fbb143f69b900a722ce49e14942c6eb14b53dad85aa551f13e742dd6a21573": { "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", "describe": { @@ -84,6 +94,249 @@ ] } }, + "7c7df8a39f420dd88ec1575ca6ef11344e58777dacb87120327325d00fd48f24": { + "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT 20\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 4, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 7, + "name": "maybe_subject_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "maybe_actor_subject_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "download_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 10, + "name": "download_url_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 48 + } + }, + { + "ordinal": 11, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 12, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 13, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 14, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 15, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 16, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false, + false, + true, + false, + false, + true + ] + } + }, + "7e05d44d9301158232b9fa5789e22bc0b2193f7c619368feec791e710c5bc3e7": { + "query": "\nINSERT INTO w2l_inference_jobs\nSET\n maybe_w2l_template_token = ?,\n maybe_tts_inference_result_token = ?,\n maybe_audio_bucket_location = NULL,\n maybe_image_bucket_location = NULL,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + } + }, "86f9a76cc97b471a23fff5b9ea78f7fc7ecf7603215f5db5a1ca4e063568b054": { "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE token = ?\nAND deleted_at IS NULL\n ", "describe": { @@ -94,6 +347,26 @@ "nullable": [] } }, + "87882bf91555f47924b49656e5f4fdf8741d00c0543c56a06fc23cdd5b745303": { + "query": "\nINSERT INTO tts_inference_jobs\nSET\n model_token = ?,\n inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + } + }, + "8bda35672e05e17329659fcb56211978c727cb34210bf91173fbcb4b9389f615": { + "query": "\nINSERT INTO tts_model_upload_jobs\nSET\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, "8d61bb77449a78044135a3d6e5009a3a27b4671b7e40501d318b05ae401f02a8": { "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE email_address = ?\nLIMIT 1\n ", "describe": { diff --git a/src/job/job_queries.rs b/src/job/job_queries.rs new file mode 100644 index 0000000000..964d0ecce8 --- /dev/null +++ b/src/job/job_queries.rs @@ -0,0 +1,54 @@ +//! NB: This seems required for sqlx to generate the cached queries. +//! Sqlx's prepare needs a *single* binary to work against, so we need to +//! include these in the main binary to generate all the queries. + +use chrono::Utc; +use crate::util::anyhow_result::AnyhowResult; +use sqlx::MySqlPool; + +/// table: tts_model_upload_jobs +#[derive(Debug)] +pub struct TtsUploadJobRecord { + pub id: i64, + pub uuid_idempotency_token: String, + pub creator_user_token: String, + pub creator_ip_address: String, + pub creator_set_visibility: String, // TODO + pub title: String, + pub tts_model_type: String, // TODO + pub maybe_subject_token: Option, + pub maybe_actor_subject_token: Option, + pub download_url: Option, + pub download_url_type: String, // TODO + pub status: String, // TODO + pub attempt_count: i32, + pub failure_reason: Option, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub retry_at: Option>, +} + +pub async fn query_tts_upload_job_records(pool: &MySqlPool) -> AnyhowResult> { + let job_records = sqlx::query_as!( + TtsUploadJobRecord, + r#" +SELECT * +FROM tts_model_upload_jobs +WHERE + ( + status IN ("pending", "attempt_failed") + ) + AND + ( + retry_at IS NULL + OR + retry_at < CURRENT_TIMESTAMP + ) + LIMIT 20 + "#, + ) + .fetch_all(pool) + .await?; + + Ok(job_records) +} diff --git a/src/job/mod.rs b/src/job/mod.rs new file mode 100644 index 0000000000..074f5ee88b --- /dev/null +++ b/src/job/mod.rs @@ -0,0 +1 @@ +pub mod job_queries; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index f5a2afb2d8..16196aeff9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,6 +14,9 @@ pub mod server_state; pub mod util; pub mod validations; +// NB: This is included so sqlx can generate all the queries. +mod job; + use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 2a248a06a3..37e64c8ccf 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -4,14 +4,17 @@ #![warn(unused_must_use)] //#![allow(warnings)] +pub mod job; pub mod util; -use crate::util::anyhow_result::AnyhowResult; -use std::time::Duration; -use log::info; use chrono::Utc; -use sqlx::mysql::MySqlPoolOptions; +use crate::job::job_queries::TtsUploadJobRecord; +use crate::job::job_queries::query_tts_upload_job_records; +use crate::util::anyhow_result::AnyhowResult; +use log::info; use sqlx::MySqlPool; +use sqlx::mysql::MySqlPoolOptions; +use std::time::Duration; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; @@ -40,94 +43,14 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; - loop { - query_and_download(&pool).await; + let jobs = query_tts_upload_job_records(&pool).await; + for job in jobs { + info!("Job: {:?}", job); + } std::thread::sleep(Duration::from_millis(500)); } Ok(()) } -/// table: tts_model_upload_jobs -#[derive(Debug)] -pub struct JobRecord { - pub id: i64, - pub uuid_idempotency_token: String, - pub creator_user_token: String, - pub creator_ip_address: String, - pub creator_set_visibility: String, // TODO - pub title: String, - pub tts_model_type: String, // TODO - pub maybe_subject_token: Option, - pub maybe_actor_subject_token: Option, - pub download_url: Option, - pub download_url_type: String, // TODO - pub status: String, // TODO - pub attempt_count: i32, - pub failure_reason: Option, - pub created_at: chrono::DateTime, - pub updated_at: chrono::DateTime, - pub retry_at: Option>, -} - -pub async fn query_and_download(pool: &MySqlPool) { - - let job_records = sqlx::query_as!( - JobRecord, - r#" -SELECT * -FROM tts_model_upload_jobs -WHERE - ( - status IN ("pending", "attempt_failed") - ) - AND - ( - retry_at IS NULL - OR - retry_at < CURRENT_TIMESTAMP - ) - LIMIT 20 - "#, - ) - .fetch_all(pool) - .await; // TODO: This will return error if it doesn't exist - - info!("Job records..."); - for job_record in job_records { - info!("job record: {:?}", job_record); - } -/* - - let query_result = sqlx::query!( - r#" -INSERT INTO w2l_template_upload_jobs -SET - uuid_idempotency_token = ?, - creator_user_token = ?, - creator_ip_address = ?, - creator_set_visibility = ?, - title = ?, - template_type = ?, - maybe_subject_token = ?, - maybe_actor_subject_token = ?, - download_url = ?, - download_url_type = ?, - status = "pending" - "#, - uuid.to_string(), - user_session.user_token.to_string(), - ip_address.to_string(), - creator_set_visibility.to_string(), - title.to_string(), - template_type, - maybe_subject_token, - maybe_actor_subject_token, - download_url, - download_url_type - ) - .execute(&server_state.mysql_pool) - .await; - */ -} \ No newline at end of file From b93f8a8099ee7847b580b11528c3ab69bf08ab92 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 03:49:18 -0400 Subject: [PATCH 0081/1042] progress --- README.md | 8 +++++ src/job/job_queries.rs | 34 +++++++++++++++++++-- src/tts_download_job.rs | 67 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 103 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e4697c34b1..e8ccd9d054 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,14 @@ To save sqlx database state to cache file (necessary for builds): SQLX_OFFLINE=true cargo sqlx prepare ``` +Now that we have multiple binaries, it's required to include all the queries in the main +binary so we can generate the cached queries as a single target. That's then executed +with: + +``` +SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web +``` + Production ---------- diff --git a/src/job/job_queries.rs b/src/job/job_queries.rs index 964d0ecce8..28fc23dc05 100644 --- a/src/job/job_queries.rs +++ b/src/job/job_queries.rs @@ -28,7 +28,9 @@ pub struct TtsUploadJobRecord { pub retry_at: Option>, } -pub async fn query_tts_upload_job_records(pool: &MySqlPool) -> AnyhowResult> { +pub async fn query_tts_upload_job_records(pool: &MySqlPool, num_records: u32) + -> AnyhowResult> +{ let job_records = sqlx::query_as!( TtsUploadJobRecord, r#" @@ -44,11 +46,39 @@ WHERE OR retry_at < CURRENT_TIMESTAMP ) - LIMIT 20 + LIMIT ? "#, + num_records, ) .fetch_all(pool) .await?; Ok(job_records) } + +pub async fn mark_tts_upload_job_failure(pool: &MySqlPool, + job: &TtsUploadJobRecord, + failure_reason: &str) + -> AnyhowResult<()> +{ + // statuses: "attempt_failed", "complete_failure", "dead" + let status = "attempt_failed"; + + let query_result = sqlx::query!( + r#" +UPDATE tts_model_upload_jobs +SET + status = ?, + failure_reason = ?, + retry_at = NOW() + interval 2 minute +WHERE id = ? + "#, + status, + failure_reason.to_string(), + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} \ No newline at end of file diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 37e64c8ccf..9376f5f7b2 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -10,8 +10,9 @@ pub mod util; use chrono::Utc; use crate::job::job_queries::TtsUploadJobRecord; use crate::job::job_queries::query_tts_upload_job_records; +use crate::job::job_queries::mark_tts_upload_job_failure; use crate::util::anyhow_result::AnyhowResult; -use log::info; +use log::{warn, info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::time::Duration; @@ -43,13 +44,71 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; + main_loop(pool); + + Ok(()) +} + +const START_TIMEOUT_MILLIS : u64 = 500; +const INCREASE_TIMEOUT_MILLIS : u64 = 1000; + +async fn main_loop(pool: MySqlPool) { + let mut timeout_millis = START_TIMEOUT_MILLIS; + loop { - let jobs = query_tts_upload_job_records(&pool).await; - for job in jobs { - info!("Job: {:?}", job); + let num_records = 1; + let query_result = query_tts_upload_job_records(&pool, num_records).await; + + let jobs = match query_result { + Ok(jobs) => jobs, + Err(e) => { + warn!("Error querying jobs: {:?}", e); + std::thread::sleep(Duration::from_millis(timeout_millis)); + timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; + } + }; + + let result = process_jobs(&pool, jobs).await; + + match result { + Ok(_) => {}, + Err(e) => { + warn!("Error querying jobs: {:?}", e); + std::thread::sleep(Duration::from_millis(timeout_millis)); + timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; + } } + + timeout_millis = START_TIMEOUT_MILLIS; // reset + std::thread::sleep(Duration::from_millis(500)); } +} + +async fn process_jobs(pool: &MySqlPool, jobs: Vec) -> AnyhowResult<()> { + for job in jobs.into_iter() { + let result = process_job(&job).await; + match result { + Ok(_) => {}, + Err(e) => { + warn!("Failure to process job: {:?}", e); + let failure_reason = ""; + let _r = mark_tts_upload_job_failure(pool, &job, failure_reason).await; + } + } + } + + Ok(()) +} + +async fn process_job(job: &TtsUploadJobRecord) -> AnyhowResult<()> { + // TODO: 1. Mark processing. + // TODO: 2. Download. + // TODO: 3. Upload. + // TODO: 4. Save record. + // TODO: 5. Mark job done. Ok(()) } From 232f66eb69b93504894c331aa2553fbcc6fa1a63 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 03:57:07 -0400 Subject: [PATCH 0082/1042] runs jobs to exhaustion --- src/job/job_queries.rs | 26 +++++++++++++++++++++++++- src/tts_download_job.rs | 19 +++++++++++++++---- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/job/job_queries.rs b/src/job/job_queries.rs index 28fc23dc05..d6c5581799 100644 --- a/src/job/job_queries.rs +++ b/src/job/job_queries.rs @@ -81,4 +81,28 @@ WHERE id = ? .await?; Ok(()) -} \ No newline at end of file +} +pub async fn mark_tts_upload_job_done(pool: &MySqlPool, + job: &TtsUploadJobRecord, + success: bool) + -> AnyhowResult<()> +{ + let status = if success { "complete_success" } else { "complete_failure" }; + + let query_result = sqlx::query!( + r#" +UPDATE tts_model_upload_jobs +SET + status = ?, + failure_reason = NULL, + retry_at = NULL +WHERE id = ? + "#, + status, + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 9376f5f7b2..566648c87e 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -1,3 +1,4 @@ +#![deny(unused_must_use)] #![allow(dead_code)] #![allow(unused_mut)] #![allow(unused_imports)] @@ -9,8 +10,9 @@ pub mod util; use chrono::Utc; use crate::job::job_queries::TtsUploadJobRecord; -use crate::job::job_queries::query_tts_upload_job_records; +use crate::job::job_queries::mark_tts_upload_job_done; use crate::job::job_queries::mark_tts_upload_job_failure; +use crate::job::job_queries::query_tts_upload_job_records; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; @@ -44,7 +46,7 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; - main_loop(pool); + main_loop(pool).await; Ok(()) } @@ -69,6 +71,12 @@ async fn main_loop(pool: MySqlPool) { } }; + if jobs.is_empty() { + info!("No jobs!"); + std::thread::sleep(Duration::from_millis(1500)); + continue; + } + let result = process_jobs(&pool, jobs).await; match result { @@ -89,7 +97,7 @@ async fn main_loop(pool: MySqlPool) { async fn process_jobs(pool: &MySqlPool, jobs: Vec) -> AnyhowResult<()> { for job in jobs.into_iter() { - let result = process_job(&job).await; + let result = process_job(pool, &job).await; match result { Ok(_) => {}, Err(e) => { @@ -103,13 +111,16 @@ async fn process_jobs(pool: &MySqlPool, jobs: Vec) -> Anyhow Ok(()) } -async fn process_job(job: &TtsUploadJobRecord) -> AnyhowResult<()> { +async fn process_job(pool: &MySqlPool, job: &TtsUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. // TODO: 3. Upload. // TODO: 4. Save record. // TODO: 5. Mark job done. + info!("Job done: {}", job.id); + mark_tts_upload_job_done(pool, job, true).await?; + Ok(()) } From a235aa9f888719fb5f96c150129b96d70b1c764f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 04:14:39 -0400 Subject: [PATCH 0083/1042] insert tts model --- debug/mysql/last_tts_model.sh | 7 +++++ src/job/job_queries.rs | 48 +++++++++++++++++++++++++++++++++++ src/tts_download_job.rs | 11 +++++++- 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100755 debug/mysql/last_tts_model.sh diff --git a/debug/mysql/last_tts_model.sh b/debug/mysql/last_tts_model.sh new file mode 100755 index 0000000000..566dcfa713 --- /dev/null +++ b/debug/mysql/last_tts_model.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u root \ + -proot \ + -h localhost \ + -D storyteller \ + -e "select * from tts_models\G" diff --git a/src/job/job_queries.rs b/src/job/job_queries.rs index d6c5581799..24b380949a 100644 --- a/src/job/job_queries.rs +++ b/src/job/job_queries.rs @@ -2,8 +2,10 @@ //! Sqlx's prepare needs a *single* binary to work against, so we need to //! include these in the main binary to generate all the queries. +use anyhow::anyhow; use chrono::Utc; use crate::util::anyhow_result::AnyhowResult; +use crate::util::random_token::random_token; use sqlx::MySqlPool; /// table: tts_model_upload_jobs @@ -106,3 +108,49 @@ WHERE id = ? Ok(()) } + +pub async fn insert_tts_model(pool: &MySqlPool, + job: &TtsUploadJobRecord, + private_bucket_hash: &str) + -> AnyhowResult +{ + let model_token = random_token(32); + let updatable_slug = model_token.clone(); + + let query_result = sqlx::query!( + r#" +INSERT INTO tts_models +SET + token = ?, + tts_model_type = "tacotron2", + updatable_slug = ?, + title = ?, + description_markdown = '', + description_rendered_html = '', + creator_user_token = ?, + creator_ip_address = ?, + original_filename = "TODO", + private_bucket_hash = ? + "#, + model_token, + updatable_slug, + job.title.to_string(), + job.creator_user_token.clone(), + job.creator_ip_address.clone(), + private_bucket_hash.to_string(), + ) + .execute(pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + Ok(record_id) +} \ No newline at end of file diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 566648c87e..d3c0f8fb2d 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -10,10 +10,12 @@ pub mod util; use chrono::Utc; use crate::job::job_queries::TtsUploadJobRecord; +use crate::job::job_queries::insert_tts_model; use crate::job::job_queries::mark_tts_upload_job_done; use crate::job::job_queries::mark_tts_upload_job_failure; use crate::job::job_queries::query_tts_upload_job_records; use crate::util::anyhow_result::AnyhowResult; +use crate::util::random_token::random_token; use log::{warn, info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; @@ -116,7 +118,14 @@ async fn process_job(pool: &MySqlPool, job: &TtsUploadJobRecord) -> AnyhowResult // TODO: 2. Download. // TODO: 3. Upload. // TODO: 4. Save record. - // TODO: 5. Mark job done. + // TODO: 5. Mark job done. (DONE) + + let private_bucket_hash = random_token(32); // TODO: Use sha2/sha256 instead. + + info!("Saving model record..."); + let id = insert_tts_model(pool, job, &private_bucket_hash).await?; + + info!("Saved model record: {}", id); info!("Job done: {}", job.id); mark_tts_upload_job_done(pool, job, true).await?; From 09a6aa286d1e3389300de388b790bd27fb557ed7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 04:28:29 -0400 Subject: [PATCH 0084/1042] filesystem tempdir --- src/tts_download_job.rs | 44 ++++++++++++++++++++++++++++++----------- src/util/filesystem.rs | 13 ++++++++++++ src/util/mod.rs | 1 + 3 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 src/util/filesystem.rs diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index d3c0f8fb2d..df6a40c9ce 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -15,14 +15,21 @@ use crate::job::job_queries::mark_tts_upload_job_done; use crate::job::job_queries::mark_tts_upload_job_failure; use crate::job::job_queries::query_tts_upload_job_records; use crate::util::anyhow_result::AnyhowResult; +use crate::util::filesystem::check_directory_exists; use crate::util::random_token::random_token; use log::{warn, info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; +use std::path::PathBuf; use std::time::Duration; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; +struct Downloader { + pub download_temp_directory: PathBuf, + pub mysql_pool: MySqlPool, +} + #[async_std::main] async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); @@ -36,6 +43,14 @@ async fn main() -> AnyhowResult<()> { info!("Hostname: {}", &server_hostname); + let temp_directory = easyenv::get_env_string_or_default( + "DOWNLOAD_TEMP_DIR", + "/tmp"); + + let temp_directory = PathBuf::from(temp_directory); + + check_directory_exists(&temp_directory)?; + let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", @@ -43,12 +58,17 @@ async fn main() -> AnyhowResult<()> { info!("Connecting to database..."); - let pool = MySqlPoolOptions::new() + let mysql_pool = MySqlPoolOptions::new() .max_connections(5) .connect(&db_connection_string) .await?; - main_loop(pool).await; + let downloader = Downloader { + download_temp_directory: temp_directory, + mysql_pool, + }; + + main_loop(downloader).await; Ok(()) } @@ -56,12 +76,12 @@ async fn main() -> AnyhowResult<()> { const START_TIMEOUT_MILLIS : u64 = 500; const INCREASE_TIMEOUT_MILLIS : u64 = 1000; -async fn main_loop(pool: MySqlPool) { +async fn main_loop(downloader: Downloader) { let mut timeout_millis = START_TIMEOUT_MILLIS; loop { let num_records = 1; - let query_result = query_tts_upload_job_records(&pool, num_records).await; + let query_result = query_tts_upload_job_records(&downloader.mysql_pool, num_records).await; let jobs = match query_result { Ok(jobs) => jobs, @@ -79,7 +99,7 @@ async fn main_loop(pool: MySqlPool) { continue; } - let result = process_jobs(&pool, jobs).await; + let result = process_jobs(&downloader, jobs).await; match result { Ok(_) => {}, @@ -97,15 +117,15 @@ async fn main_loop(pool: MySqlPool) { } } -async fn process_jobs(pool: &MySqlPool, jobs: Vec) -> AnyhowResult<()> { +async fn process_jobs(downloader: &Downloader, jobs: Vec) -> AnyhowResult<()> { for job in jobs.into_iter() { - let result = process_job(pool, &job).await; + let result = process_job(downloader, &job).await; match result { Ok(_) => {}, Err(e) => { warn!("Failure to process job: {:?}", e); let failure_reason = ""; - let _r = mark_tts_upload_job_failure(pool, &job, failure_reason).await; + let _r = mark_tts_upload_job_failure(&downloader.mysql_pool, &job, failure_reason).await; } } } @@ -113,22 +133,22 @@ async fn process_jobs(pool: &MySqlPool, jobs: Vec) -> Anyhow Ok(()) } -async fn process_job(pool: &MySqlPool, job: &TtsUploadJobRecord) -> AnyhowResult<()> { +async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. // TODO: 3. Upload. - // TODO: 4. Save record. + // TODO: 4. Save record. (DONE) // TODO: 5. Mark job done. (DONE) let private_bucket_hash = random_token(32); // TODO: Use sha2/sha256 instead. info!("Saving model record..."); - let id = insert_tts_model(pool, job, &private_bucket_hash).await?; + let id = insert_tts_model(&downloader.mysql_pool, job, &private_bucket_hash).await?; info!("Saved model record: {}", id); info!("Job done: {}", job.id); - mark_tts_upload_job_done(pool, job, true).await?; + mark_tts_upload_job_done(&downloader.mysql_pool, job, true).await?; Ok(()) } diff --git a/src/util/filesystem.rs b/src/util/filesystem.rs new file mode 100644 index 0000000000..5a1c8e3d1c --- /dev/null +++ b/src/util/filesystem.rs @@ -0,0 +1,13 @@ +use anyhow::bail; +use crate::util::anyhow_result::AnyhowResult; +use std::path::PathBuf; + +pub fn check_directory_exists(path: &PathBuf) -> AnyhowResult<()> { + if !path.exists() { + bail!("Path doesn't exist: {:?}", path); + } + if !path.is_dir() { + bail!("Path isn't a directory: {:?}", path); + } + Ok(()) +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 2b2f0d0bd0..3dda00a986 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,5 +1,6 @@ pub mod anyhow_result; pub mod cookie_manager; +pub mod filesystem; pub mod ip_address; pub mod random_token; pub mod session_checker; \ No newline at end of file From fbf5b2513a24b75b75aba219796730e8f5733fff Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 05:01:13 -0400 Subject: [PATCH 0085/1042] ugh, this requires tokio runtime and hyper is inflexible --- Cargo.lock | 134 +++++++++++++++++++++++++++------------- Cargo.toml | 4 ++ src/tts_download_job.rs | 37 ++++++++++- 3 files changed, 130 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a85409874..43bfa47960 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,7 +12,7 @@ dependencies = [ "bytes 0.5.6", "futures-core", "futures-sink", - "log", + "log 0.4.14", "pin-project 0.4.28", "tokio 0.2.25", "tokio-util 0.3.1", @@ -32,7 +32,7 @@ dependencies = [ "either", "futures-util", "http", - "log", + "log 0.4.14", "trust-dns-proto", "trust-dns-resolver", ] @@ -46,7 +46,7 @@ dependencies = [ "actix-web", "derive_more", "futures-util", - "log", + "log 0.4.14", "once_cell", "tinyvec", ] @@ -64,8 +64,8 @@ dependencies = [ "derive_more", "futures-core", "futures-util", - "log", - "mime", + "log 0.4.14", + "mime 0.3.16", "mime_guess", "percent-encoding 2.1.0", "v_htmlescape", @@ -104,8 +104,8 @@ dependencies = [ "itoa", "language-tags", "lazy_static", - "log", - "mime", + "log 0.4.14", + "mime 0.3.16", "percent-encoding 2.1.0", "pin-project 1.0.7", "rand 0.7.3", @@ -146,7 +146,7 @@ checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" dependencies = [ "bytestring", "http", - "log", + "log 0.4.14", "regex", "serde", ] @@ -189,7 +189,7 @@ dependencies = [ "actix-utils 2.0.0", "futures-channel", "futures-util", - "log", + "log 0.4.14", "mio 0.6.23", "mio-uds", "num_cpus", @@ -217,7 +217,7 @@ dependencies = [ "actix-rt 1.1.1", "actix-server", "actix-service", - "log", + "log 0.4.14", "socket2 0.3.19", ] @@ -230,7 +230,7 @@ dependencies = [ "derive_more", "futures-channel", "lazy_static", - "log", + "log 0.4.14", "num_cpus", "parking_lot", "threadpool", @@ -263,7 +263,7 @@ dependencies = [ "futures-channel", "futures-sink", "futures-util", - "log", + "log 0.4.14", "pin-project 0.4.28", "slab", ] @@ -304,8 +304,8 @@ dependencies = [ "futures-core", "futures-util", "fxhash", - "log", - "mime", + "log 0.4.14", + "mime 0.3.16", "pin-project 1.0.7", "regex", "serde", @@ -433,7 +433,7 @@ dependencies = [ "fastrand", "futures-lite", "libc", - "log", + "log 0.4.14", "once_cell", "parking", "polling", @@ -509,7 +509,7 @@ dependencies = [ "futures-lite", "gloo-timers", "kv-log-macro", - "log", + "log 0.4.14", "memchr", "num_cpus", "once_cell", @@ -558,7 +558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb8867f378f33f78a811a8eb9bf108ad99430d7aad43315dd9319c827ef6247" dependencies = [ "http", - "log", + "log 0.4.14", "native-tls", "openssl", "serde", @@ -605,8 +605,8 @@ dependencies = [ "cfg-if 1.0.0", "derive_more", "futures-core", - "log", - "mime", + "log 0.4.14", + "mime 0.3.16", "percent-encoding 2.1.0", "rand 0.7.3", "serde", @@ -1070,7 +1070,7 @@ checksum = "852054bee794f364fbe890bd947cbf678c35226b81ede17720d6a17fc687a0d4" dependencies = [ "dotenv", "env_logger", - "log", + "log 0.4.14", ] [[package]] @@ -1111,7 +1111,7 @@ checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", "humantime", - "log", + "log 0.4.14", "regex", "termcolor", ] @@ -1371,6 +1371,23 @@ dependencies = [ "yup-oauth2", ] +[[package]] +name = "google-drive3" +version = "2.0.5+20210322" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f37f915aa1329a2d0c97abfbbbd823dc3436c052b52f1760677ddb1f3d414896" +dependencies = [ + "hyper", + "hyper-rustls", + "itertools", + "mime 0.2.6", + "serde", + "serde_derive", + "serde_json", + "url 1.7.2", + "yup-oauth2", +] + [[package]] name = "h2" version = "0.2.7" @@ -1549,7 +1566,7 @@ dependencies = [ "ct-logs", "futures-util", "hyper", - "log", + "log 0.4.14", "rustls", "rustls-native-certs", "tokio 1.5.0", @@ -1638,6 +1655,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.7" @@ -1684,7 +1710,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" dependencies = [ - "log", + "log 0.4.14", ] [[package]] @@ -1748,6 +1774,15 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +dependencies = [ + "log 0.4.14", +] + [[package]] name = "log" version = "0.4.14" @@ -1819,6 +1854,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" +[[package]] +name = "mime" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" +dependencies = [ + "log 0.3.9", +] + [[package]] name = "mime" version = "0.3.16" @@ -1831,7 +1875,7 @@ version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" dependencies = [ - "mime", + "mime 0.3.16", "unicase", ] @@ -1866,7 +1910,7 @@ dependencies = [ "iovec", "kernel32-sys", "libc", - "log", + "log 0.4.14", "miow 0.2.2", "net2", "slab", @@ -1880,7 +1924,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956" dependencies = [ "libc", - "log", + "log 0.4.14", "miow 0.3.7", "ntapi", "winapi 0.3.9", @@ -1936,7 +1980,7 @@ checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" dependencies = [ "lazy_static", "libc", - "log", + "log 0.4.14", "openssl", "openssl-probe", "openssl-sys", @@ -2251,7 +2295,7 @@ checksum = "4fc12d774e799ee9ebae13f4076ca003b40d18a11ac0f3641e6f899618580b7b" dependencies = [ "cfg-if 1.0.0", "libc", - "log", + "log 0.4.14", "wepoll-sys", "winapi 0.3.9", ] @@ -2457,8 +2501,8 @@ dependencies = [ "ipnet", "js-sys", "lazy_static", - "log", - "mime", + "log 0.4.14", + "mime 0.3.16", "native-tls", "percent-encoding 2.1.0", "pin-project-lite 0.2.6", @@ -2546,7 +2590,7 @@ dependencies = [ "hex", "hmac", "http", - "log", + "log 0.4.14", "maybe-async", "md5", "minidom", @@ -2577,7 +2621,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ "base64 0.13.0", - "log", + "log 0.4.14", "ring", "sct", "webpki", @@ -2686,7 +2730,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0bf1ba0696ccf0872866277143ff1fd14d22eec235d2b23702f95e6660f7dfa" dependencies = [ - "log", + "log 0.4.14", "serde", "thiserror", "xml-rs", @@ -2879,7 +2923,7 @@ dependencies = [ "hex", "itoa", "libc", - "log", + "log 0.4.14", "memchr", "num-bigint 0.3.2", "once_cell", @@ -3017,12 +3061,15 @@ dependencies = [ "easyenv", "futures", "google-drive", + "google-drive3", "hex", "hmac", "hostname", + "hyper", + "hyper-rustls", "jwt", "lazy_static", - "log", + "log 0.4.14", "md-5", "rand 0.8.3", "regex", @@ -3034,6 +3081,7 @@ dependencies = [ "sqlx", "time 0.2.26", "toml", + "yup-oauth2", ] [[package]] @@ -3288,7 +3336,7 @@ dependencies = [ "bytes 0.5.6", "futures-core", "futures-sink", - "log", + "log 0.4.14", "pin-project-lite 0.1.12", "tokio 0.2.25", ] @@ -3302,7 +3350,7 @@ dependencies = [ "bytes 1.0.1", "futures-core", "futures-sink", - "log", + "log 0.4.14", "pin-project-lite 0.2.6", "tokio 1.5.0", ] @@ -3329,7 +3377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" dependencies = [ "cfg-if 1.0.0", - "log", + "log 0.4.14", "pin-project-lite 0.2.6", "tracing-core", ] @@ -3365,7 +3413,7 @@ dependencies = [ "futures", "idna 0.2.3", "lazy_static", - "log", + "log 0.4.14", "rand 0.7.3", "smallvec", "thiserror", @@ -3383,7 +3431,7 @@ dependencies = [ "futures", "ipconfig", "lazy_static", - "log", + "log 0.4.14", "lru-cache", "resolv-conf", "smallvec", @@ -3549,7 +3597,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log", + "log 0.4.14", "try-lock", ] @@ -3585,7 +3633,7 @@ checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900" dependencies = [ "bumpalo", "lazy_static", - "log", + "log 0.4.14", "proc-macro2", "quote", "syn", @@ -3779,7 +3827,7 @@ dependencies = [ "http", "hyper", "hyper-rustls", - "log", + "log 0.4.14", "percent-encoding 2.1.0", "rustls", "seahash", diff --git a/Cargo.toml b/Cargo.toml index 7fa8119421..8c55dcaf38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,11 @@ sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", # Files google-drive = "0.1.18" +google-drive3 = "2.0.5+20210322" +hyper = { version = "0.14.7", features = [""] } +hyper-rustls = "0.22.1" rust-s3 = "0.27.0-rc4" +yup-oauth2 = "5.1.0" # for google-drive3 # Web actix-cors = "0.5.4" diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index df6a40c9ce..2c6ecd9b5f 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -17,6 +17,7 @@ use crate::job::job_queries::query_tts_upload_job_records; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::random_token::random_token; +use google_drive3::DriveHub; use log::{warn, info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; @@ -24,9 +25,11 @@ use std::path::PathBuf; use std::time::Duration; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; +const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Downloader { pub download_temp_directory: PathBuf, + pub gdrive_client: DriveHub, pub mysql_pool: MySqlPool, } @@ -39,18 +42,20 @@ async fn main() -> AnyhowResult<()> { let server_hostname = hostname::get() .ok() .and_then(|h| h.into_string().ok()) - .unwrap_or("storyteller-web-unknown".to_string()); + .unwrap_or("tts-download-job".to_string()); info!("Hostname: {}", &server_hostname); let temp_directory = easyenv::get_env_string_or_default( "DOWNLOAD_TEMP_DIR", - "/tmp"); + DEFAULT_TEMP_DIR); let temp_directory = PathBuf::from(temp_directory); check_directory_exists(&temp_directory)?; + let gdrive_client = build_gdrive_client().await?; + let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", @@ -66,6 +71,7 @@ async fn main() -> AnyhowResult<()> { let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, + gdrive_client, }; main_loop(downloader).await; @@ -73,6 +79,21 @@ async fn main() -> AnyhowResult<()> { Ok(()) } +async fn build_gdrive_client() -> AnyhowResult { + // https://docs.rs/google-drive3/2.0.5+20210322/google_drive3/api/struct.FileGetCall.html + let secret: yup_oauth2::ApplicationSecret = Default::default(); + + let auth = yup_oauth2::InstalledFlowAuthenticator::builder( + secret, + yup_oauth2::InstalledFlowReturnMethod::HTTPRedirect) + .build() + .await?; + + Ok(DriveHub::new( + hyper::Client::builder() + .build(hyper_rustls::HttpsConnector::with_native_roots()), auth)) +} + const START_TIMEOUT_MILLIS : u64 = 500; const INCREASE_TIMEOUT_MILLIS : u64 = 1000; @@ -140,6 +161,18 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho // TODO: 4. Save record. (DONE) // TODO: 5. Mark job done. (DONE) + let result = downloader.gdrive_client + .files() + .get("fileId") + .supports_team_drives(true) + .supports_all_drives(false) + .include_permissions_for_view("accusam") + .acknowledge_abuse(true) + .doit() + .await; + + info!("Download result: {:?}", result); + let private_bucket_hash = random_token(32); // TODO: Use sha2/sha256 instead. info!("Saving model record..."); From b165996f9d073bab8ab6d90539f9a4c89d5588ba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 07:17:39 -0400 Subject: [PATCH 0086/1042] use python 'gdown' to download files --- scripts/download_gdrive.py | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 scripts/download_gdrive.py diff --git a/scripts/download_gdrive.py b/scripts/download_gdrive.py new file mode 100755 index 0000000000..1a12946f7f --- /dev/null +++ b/scripts/download_gdrive.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import argparse +import gdown +import re + +parser = argparse.ArgumentParser(description='Download Google Drive files') + +parser.add_argument('--url', type=str, help='the google drive link') +parser.add_argument('--output_filename', type=str, help='the output filename') + +# This URL format gets an OAuth screen!! UGH!! +# 'https://drive.google.com/file/d/15-tkgblTZpa0ifmvvyr_kgWy2cguTpAe/view?usp=sharing' +# Fixed URL: +# 'https://drive.google.com/uc?id=15-tkgblTZpa0ifmvvyr_kgWy2cguTpAe' + +def normalize_url(url): + """ + URLs must be in a certain form, or they present the user with an OAuth wall. + This attempts to fix URLs in the wrong format. + """ + matches = re.match(r"^https:\/\/drive\.google\.com\/file\/d\/([^\/]+)\/", url) + match_groups = matches.groups() + + if not match_groups or not len(match_groups) == 1: + return url + + document_token = match_groups[0] + return 'https://drive.google.com/uc?id={}'.format(document_token) + + +args = parser.parse_args() + +print('URL: {}'.format(args.url)) + +url = normalize_url(args.url) +print('Normalized URL: {}'.format(url)) + +gdown.download(url, args.output_filename, quiet=False) + From 11c4e78ea89e61b610d5982d6f23f263678777c7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 07:20:21 -0400 Subject: [PATCH 0087/1042] hack to install gdown and scripts in dockerfile (TODO: use venv) --- Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Dockerfile b/Dockerfile index ac44a4658b..a1d9a9ab54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,9 @@ RUN SQLX_OFFLINE=true \ FROM ubuntu:xenial WORKDIR / + +COPY scripts/ ./scripts + # See: https://github.com/opencontainers/image-spec/blob/master/annotations.md LABEL org.opencontainers.image.authors='bt@brand.io, echelon@gmail.com' LABEL org.opencontainers.image.url='https://github.com/storytold/storyteller-web' @@ -59,8 +62,13 @@ LABEL org.opencontainers.image.source='https://github.com/storytold/storyteller- RUN apt-get update \ && apt-get install -y \ curl \ + python3 \ + python3-pip \ wget +# TODO: Use venv to do this instead. +RUN pip install gdown + # Copy all the binaries. COPY --from=build /tmp/target/release/storyteller-web / COPY --from=build /tmp/target/release/tts-download-job / From c0767a1ba18847762b0c693389ab026a80900520 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 07:26:37 -0400 Subject: [PATCH 0088/1042] fixup compile/remove google drive stuff --- Cargo.lock | 284 ++++++---------------------------------- Cargo.toml | 5 - src/tts_download_job.rs | 32 ----- 3 files changed, 40 insertions(+), 281 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43bfa47960..a29dde4cf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,7 +12,7 @@ dependencies = [ "bytes 0.5.6", "futures-core", "futures-sink", - "log 0.4.14", + "log", "pin-project 0.4.28", "tokio 0.2.25", "tokio-util 0.3.1", @@ -32,7 +32,7 @@ dependencies = [ "either", "futures-util", "http", - "log 0.4.14", + "log", "trust-dns-proto", "trust-dns-resolver", ] @@ -46,7 +46,7 @@ dependencies = [ "actix-web", "derive_more", "futures-util", - "log 0.4.14", + "log", "once_cell", "tinyvec", ] @@ -64,8 +64,8 @@ dependencies = [ "derive_more", "futures-core", "futures-util", - "log 0.4.14", - "mime 0.3.16", + "log", + "mime", "mime_guess", "percent-encoding 2.1.0", "v_htmlescape", @@ -104,8 +104,8 @@ dependencies = [ "itoa", "language-tags", "lazy_static", - "log 0.4.14", - "mime 0.3.16", + "log", + "mime", "percent-encoding 2.1.0", "pin-project 1.0.7", "rand 0.7.3", @@ -146,7 +146,7 @@ checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" dependencies = [ "bytestring", "http", - "log 0.4.14", + "log", "regex", "serde", ] @@ -189,7 +189,7 @@ dependencies = [ "actix-utils 2.0.0", "futures-channel", "futures-util", - "log 0.4.14", + "log", "mio 0.6.23", "mio-uds", "num_cpus", @@ -217,7 +217,7 @@ dependencies = [ "actix-rt 1.1.1", "actix-server", "actix-service", - "log 0.4.14", + "log", "socket2 0.3.19", ] @@ -230,7 +230,7 @@ dependencies = [ "derive_more", "futures-channel", "lazy_static", - "log 0.4.14", + "log", "num_cpus", "parking_lot", "threadpool", @@ -263,7 +263,7 @@ dependencies = [ "futures-channel", "futures-sink", "futures-util", - "log 0.4.14", + "log", "pin-project 0.4.28", "slab", ] @@ -304,8 +304,8 @@ dependencies = [ "futures-core", "futures-util", "fxhash", - "log 0.4.14", - "mime 0.3.16", + "log", + "mime", "pin-project 1.0.7", "regex", "serde", @@ -433,7 +433,7 @@ dependencies = [ "fastrand", "futures-lite", "libc", - "log 0.4.14", + "log", "once_cell", "parking", "polling", @@ -509,7 +509,7 @@ dependencies = [ "futures-lite", "gloo-timers", "kv-log-macro", - "log 0.4.14", + "log", "memchr", "num_cpus", "once_cell", @@ -558,7 +558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb8867f378f33f78a811a8eb9bf108ad99430d7aad43315dd9319c827ef6247" dependencies = [ "http", - "log 0.4.14", + "log", "native-tls", "openssl", "serde", @@ -605,8 +605,8 @@ dependencies = [ "cfg-if 1.0.0", "derive_more", "futures-core", - "log 0.4.14", - "mime 0.3.16", + "log", + "mime", "percent-encoding 2.1.0", "rand 0.7.3", "serde", @@ -958,15 +958,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ct-logs" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" -dependencies = [ - "sct", -] - [[package]] name = "ctor" version = "0.1.20" @@ -1070,7 +1061,7 @@ checksum = "852054bee794f364fbe890bd947cbf678c35226b81ede17720d6a17fc687a0d4" dependencies = [ "dotenv", "env_logger", - "log 0.4.14", + "log", ] [[package]] @@ -1111,7 +1102,7 @@ checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", "humantime", - "log 0.4.14", + "log", "regex", "termcolor", ] @@ -1359,35 +1350,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "google-drive" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fe473f3853a7668300abb646ffb01b6eb7b97548f24575f73079b03e5c73b2" -dependencies = [ - "bytes 1.0.1", - "reqwest", - "serde", - "yup-oauth2", -] - -[[package]] -name = "google-drive3" -version = "2.0.5+20210322" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f37f915aa1329a2d0c97abfbbbd823dc3436c052b52f1760677ddb1f3d414896" -dependencies = [ - "hyper", - "hyper-rustls", - "itertools", - "mime 0.2.6", - "serde", - "serde_derive", - "serde_json", - "url 1.7.2", - "yup-oauth2", -] - [[package]] name = "h2" version = "0.2.7" @@ -1557,23 +1519,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" -dependencies = [ - "ct-logs", - "futures-util", - "hyper", - "log 0.4.14", - "rustls", - "rustls-native-certs", - "tokio 1.5.0", - "tokio-rustls", - "webpki", -] - [[package]] name = "hyper-tls" version = "0.5.0" @@ -1655,15 +1600,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" -[[package]] -name = "itertools" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "0.4.7" @@ -1710,7 +1646,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" dependencies = [ - "log 0.4.14", + "log", ] [[package]] @@ -1774,15 +1710,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "log" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" -dependencies = [ - "log 0.4.14", -] - [[package]] name = "log" version = "0.4.14" @@ -1854,15 +1781,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" -[[package]] -name = "mime" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" -dependencies = [ - "log 0.3.9", -] - [[package]] name = "mime" version = "0.3.16" @@ -1875,7 +1793,7 @@ version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" dependencies = [ - "mime 0.3.16", + "mime", "unicase", ] @@ -1910,7 +1828,7 @@ dependencies = [ "iovec", "kernel32-sys", "libc", - "log 0.4.14", + "log", "miow 0.2.2", "net2", "slab", @@ -1924,7 +1842,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956" dependencies = [ "libc", - "log 0.4.14", + "log", "miow 0.3.7", "ntapi", "winapi 0.3.9", @@ -1980,7 +1898,7 @@ checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" dependencies = [ "lazy_static", "libc", - "log 0.4.14", + "log", "openssl", "openssl-probe", "openssl-sys", @@ -2295,7 +2213,7 @@ checksum = "4fc12d774e799ee9ebae13f4076ca003b40d18a11ac0f3641e6f899618580b7b" dependencies = [ "cfg-if 1.0.0", "libc", - "log 0.4.14", + "log", "wepoll-sys", "winapi 0.3.9", ] @@ -2501,8 +2419,8 @@ dependencies = [ "ipnet", "js-sys", "lazy_static", - "log 0.4.14", - "mime 0.3.16", + "log", + "mime", "native-tls", "percent-encoding 2.1.0", "pin-project-lite 0.2.6", @@ -2528,21 +2446,6 @@ dependencies = [ "quick-error", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted", - "web-sys", - "winapi 0.3.9", -] - [[package]] name = "rsa" version = "0.4.0" @@ -2590,7 +2493,7 @@ dependencies = [ "hex", "hmac", "http", - "log 0.4.14", + "log", "maybe-async", "md5", "minidom", @@ -2614,31 +2517,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustls" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" -dependencies = [ - "base64 0.13.0", - "log 0.4.14", - "ring", - "sct", - "webpki", -] - -[[package]] -name = "rustls-native-certs" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" -dependencies = [ - "openssl-probe", - "rustls", - "schannel", - "security-framework", -] - [[package]] name = "ryu" version = "1.0.5" @@ -2661,22 +2539,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "sct" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - [[package]] name = "security-framework" version = "2.2.0" @@ -2730,7 +2592,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0bf1ba0696ccf0872866277143ff1fd14d22eec235d2b23702f95e6660f7dfa" dependencies = [ - "log 0.4.14", + "log", "serde", "thiserror", "xml-rs", @@ -2923,7 +2785,7 @@ dependencies = [ "hex", "itoa", "libc", - "log 0.4.14", + "log", "memchr", "num-bigint 0.3.2", "once_cell", @@ -3060,16 +2922,12 @@ dependencies = [ "diesel", "easyenv", "futures", - "google-drive", - "google-drive3", "hex", "hmac", "hostname", - "hyper", - "hyper-rustls", "jwt", "lazy_static", - "log 0.4.14", + "log", "md-5", "rand 0.8.3", "regex", @@ -3081,7 +2939,6 @@ dependencies = [ "sqlx", "time 0.2.26", "toml", - "yup-oauth2", ] [[package]] @@ -3280,21 +3137,9 @@ dependencies = [ "parking_lot", "pin-project-lite 0.2.6", "signal-hook-registry", - "tokio-macros", "winapi 0.3.9", ] -[[package]] -name = "tokio-macros" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c49e3df43841dafb86046472506755d8501c5615673955f6aa17181125d13c37" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tokio-native-tls" version = "0.3.0" @@ -3305,17 +3150,6 @@ dependencies = [ "tokio 1.5.0", ] -[[package]] -name = "tokio-rustls" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" -dependencies = [ - "rustls", - "tokio 1.5.0", - "webpki", -] - [[package]] name = "tokio-stream" version = "0.1.6" @@ -3336,7 +3170,7 @@ dependencies = [ "bytes 0.5.6", "futures-core", "futures-sink", - "log 0.4.14", + "log", "pin-project-lite 0.1.12", "tokio 0.2.25", ] @@ -3350,7 +3184,7 @@ dependencies = [ "bytes 1.0.1", "futures-core", "futures-sink", - "log 0.4.14", + "log", "pin-project-lite 0.2.6", "tokio 1.5.0", ] @@ -3377,7 +3211,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" dependencies = [ "cfg-if 1.0.0", - "log 0.4.14", + "log", "pin-project-lite 0.2.6", "tracing-core", ] @@ -3413,7 +3247,7 @@ dependencies = [ "futures", "idna 0.2.3", "lazy_static", - "log 0.4.14", + "log", "rand 0.7.3", "smallvec", "thiserror", @@ -3431,7 +3265,7 @@ dependencies = [ "futures", "ipconfig", "lazy_static", - "log 0.4.14", + "log", "lru-cache", "resolv-conf", "smallvec", @@ -3497,12 +3331,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "url" version = "1.7.2" @@ -3597,7 +3425,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log 0.4.14", + "log", "try-lock", ] @@ -3633,7 +3461,7 @@ checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900" dependencies = [ "bumpalo", "lazy_static", - "log 0.4.14", + "log", "proc-macro2", "quote", "syn", @@ -3691,16 +3519,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "wepoll-sys" version = "3.0.1" @@ -3815,28 +3633,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" -[[package]] -name = "yup-oauth2" -version = "5.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2573621fa28865489bdf556cdb8703604f4e8498612e3041a212ac20e59c4aeb" -dependencies = [ - "base64 0.13.0", - "chrono", - "futures", - "http", - "hyper", - "hyper-rustls", - "log 0.4.14", - "percent-encoding 2.1.0", - "rustls", - "seahash", - "serde", - "serde_json", - "tokio 1.5.0", - "url 2.2.2", -] - [[package]] name = "zeroize" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index 8c55dcaf38..83797f8262 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,12 +53,7 @@ time = "0.2.26" sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } # Files -google-drive = "0.1.18" -google-drive3 = "2.0.5+20210322" -hyper = { version = "0.14.7", features = [""] } -hyper-rustls = "0.22.1" rust-s3 = "0.27.0-rc4" -yup-oauth2 = "5.1.0" # for google-drive3 # Web actix-cors = "0.5.4" diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 2c6ecd9b5f..8774c13443 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -17,7 +17,6 @@ use crate::job::job_queries::query_tts_upload_job_records; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::random_token::random_token; -use google_drive3::DriveHub; use log::{warn, info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; @@ -29,7 +28,6 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Downloader { pub download_temp_directory: PathBuf, - pub gdrive_client: DriveHub, pub mysql_pool: MySqlPool, } @@ -54,8 +52,6 @@ async fn main() -> AnyhowResult<()> { check_directory_exists(&temp_directory)?; - let gdrive_client = build_gdrive_client().await?; - let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", @@ -71,7 +67,6 @@ async fn main() -> AnyhowResult<()> { let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, - gdrive_client, }; main_loop(downloader).await; @@ -79,21 +74,6 @@ async fn main() -> AnyhowResult<()> { Ok(()) } -async fn build_gdrive_client() -> AnyhowResult { - // https://docs.rs/google-drive3/2.0.5+20210322/google_drive3/api/struct.FileGetCall.html - let secret: yup_oauth2::ApplicationSecret = Default::default(); - - let auth = yup_oauth2::InstalledFlowAuthenticator::builder( - secret, - yup_oauth2::InstalledFlowReturnMethod::HTTPRedirect) - .build() - .await?; - - Ok(DriveHub::new( - hyper::Client::builder() - .build(hyper_rustls::HttpsConnector::with_native_roots()), auth)) -} - const START_TIMEOUT_MILLIS : u64 = 500; const INCREASE_TIMEOUT_MILLIS : u64 = 1000; @@ -161,18 +141,6 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho // TODO: 4. Save record. (DONE) // TODO: 5. Mark job done. (DONE) - let result = downloader.gdrive_client - .files() - .get("fileId") - .supports_team_drives(true) - .supports_all_drives(false) - .include_permissions_for_view("accusam") - .acknowledge_abuse(true) - .doit() - .await; - - info!("Download result: {:?}", result); - let private_bucket_hash = random_token(32); // TODO: Use sha2/sha256 instead. info!("Saving model record..."); From 878acbad766d8f6ca50855c1f0edf854de1ba6a1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 07:37:20 -0400 Subject: [PATCH 0089/1042] forgot these queries --- sqlx-data.json | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 93af734c4e..322fa1850e 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -94,8 +94,8 @@ ] } }, - "7c7df8a39f420dd88ec1575ca6ef11344e58777dacb87120327325d00fd48f24": { - "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT 20\n ", + "7ab54466ca3021d90e0feb234acba56bf53d415dde69bc1c35120ed0c8dd1858": { + "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { "columns": [ { @@ -304,7 +304,7 @@ } ], "parameters": { - "Right": 0 + "Right": 1 }, "nullable": [ false, @@ -651,6 +651,36 @@ ] } }, + "b4fa3ec3e2f65f217ef8a08f54f1366b537d0e99497cd7e2882e8e5a96c4c4ab": { + "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_filename = \"TODO\",\n private_bucket_hash = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + } + }, + "bf489cf419d342d144f072c310d6d70c9073b092b5721f489eaf2ffc8ad3915e": { + "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "c11318a1a944cd74f1210b211e19dfaa12652bad2a5d79b010f6efa62f3fa8a9": { + "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "ca5efc3be0f09649df2add5270c06544133288a95ffcb9a0589efd4fe345efe7": { "query": "\nINSERT INTO badges ( slug, title, description, image_url )\nVALUES ( ?, ?, ?, ? )\n ", "describe": { From 2695567ef6f6cad8f0f72e935062e4a6ce987706 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 07:52:43 -0400 Subject: [PATCH 0090/1042] closer --- src/tts_download_job.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 8774c13443..e4f07ffa67 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -22,6 +22,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::path::PathBuf; use std::time::Duration; +use std::process::Command; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; @@ -29,6 +30,8 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Downloader { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, + // Command to run + pub download_script: String, } #[async_std::main] @@ -48,6 +51,10 @@ async fn main() -> AnyhowResult<()> { "DOWNLOAD_TEMP_DIR", DEFAULT_TEMP_DIR); + let download_script = easyenv::get_env_string_or_default( + "DOWNLOAD_SCRIPT", + "./scripts/download_gdrive.py"); + let temp_directory = PathBuf::from(temp_directory); check_directory_exists(&temp_directory)?; @@ -67,6 +74,7 @@ async fn main() -> AnyhowResult<()> { let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, + download_script, }; main_loop(downloader).await; @@ -134,6 +142,26 @@ async fn process_jobs(downloader: &Downloader, jobs: Vec) -> Ok(()) } +async fn call_script(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { + + let url = job.download_url.as_ref().map(|c| c.to_string()).unwrap_or("".to_string()); + let command = format!("{} --url {} --output_file {}", + downloader.download_script, + &url, + "filename.txt"); + + info!("Running command: {}", command); + + let result = Command::new("sh") + .arg("-c") + .arg(command) + .output()?; + + info!("Downloader Result: {:?}", result); + + Ok(()) +} + async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. @@ -143,6 +171,9 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho let private_bucket_hash = random_token(32); // TODO: Use sha2/sha256 instead. + info!("Calling downloader..."); + call_script(downloader, job).await?; + info!("Saving model record..."); let id = insert_tts_model(&downloader.mysql_pool, job, &private_bucket_hash).await?; From 9a9812f379bd330de3a360ce2f722944f580bfac Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 07:59:45 -0400 Subject: [PATCH 0091/1042] downloader works --- debug/curl/upload_tts_gdrive.py | 45 +++++++++++++++++++++++++++++++++ scripts/download_gdrive.py | 4 +++ src/tts_download_job.rs | 8 +++++- 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100755 debug/curl/upload_tts_gdrive.py diff --git a/debug/curl/upload_tts_gdrive.py b/debug/curl/upload_tts_gdrive.py new file mode 100755 index 0000000000..de74ed3902 --- /dev/null +++ b/debug/curl/upload_tts_gdrive.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +import requests +import re +import uuid + +login_url = 'http://localhost:12345/login' +upload_url = 'http://localhost:12345/tts/upload' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +# NB: May not be set due to cookie domain: +#r.cookies['session'] + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + + +print("===== Upload TTS=====") + +cookies = { 'session': session_cookie } + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'title': 'this is a new text to speech model', + 'download_url': 'https://drive.google.com/file/d/15-tkgblTZpa0ifmvvyr_kgWy2cguTpAe/view?usp=sharing', +} + +r = requests.post(upload_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + diff --git a/scripts/download_gdrive.py b/scripts/download_gdrive.py index 1a12946f7f..0651a6cd9f 100755 --- a/scripts/download_gdrive.py +++ b/scripts/download_gdrive.py @@ -20,6 +20,10 @@ def normalize_url(url): This attempts to fix URLs in the wrong format. """ matches = re.match(r"^https:\/\/drive\.google\.com\/file\/d\/([^\/]+)\/", url) + + if not matches: + return url + match_groups = matches.groups() if not match_groups or not len(match_groups) == 1: diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index e4f07ffa67..b7e6751d44 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -8,6 +8,7 @@ pub mod job; pub mod util; +use anyhow::anyhow; use chrono::Utc; use crate::job::job_queries::TtsUploadJobRecord; use crate::job::job_queries::insert_tts_model; @@ -21,8 +22,8 @@ use log::{warn, info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::path::PathBuf; -use std::time::Duration; use std::process::Command; +use std::time::Duration; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; @@ -159,6 +160,11 @@ async fn call_script(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("Downloader Result: {:?}", result); + if !result.status.success() { + let reason = String::from_utf8(result.stderr).unwrap_or("UNKNOWN".to_string()); + return Err(anyhow!("Failure to execute command: {:?}", reason)) + } + Ok(()) } From 0f366884f01e077650ff58470ec72e1f082dcd5d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 08:04:43 -0400 Subject: [PATCH 0092/1042] fix dockerfile ? --- Dockerfile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a1d9a9ab54..5292d5080b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,8 @@ RUN SQLX_OFFLINE=true \ # Final image FROM ubuntu:xenial +#RUN mkdir /storyteller +#WORKDIR /storyteller WORKDIR / @@ -62,12 +64,21 @@ LABEL org.opencontainers.image.source='https://github.com/storytold/storyteller- RUN apt-get update \ && apt-get install -y \ curl \ - python3 \ python3-pip \ + python3.7 \ + python3.7-dev \ + python3.7-venv \ wget +RUN python3.7 --version +#RUN python3.7 -m venv python +#RUN . python/bin/activate \ +# && pip install --upgrade pip \ +# && pip install -r requirements.txt + + # TODO: Use venv to do this instead. -RUN pip install gdown +RUN pip3 install gdown # Copy all the binaries. COPY --from=build /tmp/target/release/storyteller-web / From 44d1d5bf5ed87b4b01d826ae7aac798e744c97b2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 08:18:17 -0400 Subject: [PATCH 0093/1042] update dockerfile --- Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5292d5080b..20e3aefc53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,12 +65,10 @@ RUN apt-get update \ && apt-get install -y \ curl \ python3-pip \ - python3.7 \ - python3.7-dev \ - python3.7-venv \ + python3 \ wget -RUN python3.7 --version +RUN python3 --version #RUN python3.7 -m venv python #RUN . python/bin/activate \ # && pip install --upgrade pip \ From bb2c189c5644a7011c4a1c0d1e1b1be56f63882b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 08:54:21 -0400 Subject: [PATCH 0094/1042] file hashing --- Cargo.lock | 83 +++++++++++++++++++ Cargo.toml | 3 + migrations/2021-05-15-071601_assets/up.sql | 6 +- .../up.sql | 4 +- src/tts_download_job.rs | 65 +++++++++++++-- 5 files changed, 149 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a29dde4cf4..5fcd5bd984 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -968,6 +968,12 @@ dependencies = [ "syn", ] +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "derive_more" version = "0.99.14" @@ -1165,6 +1171,12 @@ dependencies = [ "percent-encoding 2.1.0", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "fuchsia-zircon" version = "0.3.3" @@ -2275,6 +2287,19 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi 0.3.9", +] + [[package]] name = "rand" version = "0.7.3" @@ -2320,6 +2345,21 @@ dependencies = [ "rand_core 0.6.2", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.5.1" @@ -2356,6 +2396,15 @@ dependencies = [ "rand_core 0.6.2", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.8" @@ -2446,6 +2495,21 @@ dependencies = [ "quick-error", ] +[[package]] +name = "ring" +version = "0.17.0-alpha.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ae9a4d2975bdd8254d7bcdd2261be62b42b8920f903ec682b08cdc11b87af2" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "rsa" version = "0.4.0" @@ -2918,6 +2982,7 @@ dependencies = [ "base64 0.13.0", "bcrypt", "chrono", + "data-encoding", "derive_more", "diesel", "easyenv", @@ -2931,12 +2996,14 @@ dependencies = [ "md-5", "rand 0.8.3", "regex", + "ring", "rust-s3", "serde", "serde_derive", "serde_json", "sha2", "sqlx", + "tempdir", "time 0.2.26", "toml", ] @@ -2986,6 +3053,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.2.0" @@ -3331,6 +3408,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "1.7.2" diff --git a/Cargo.toml b/Cargo.toml index 83797f8262..a12bd83942 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,10 @@ time = "0.2.26" sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } # Files +data-encoding = "2.3.2" +ring = "0.17.0-alpha.10" rust-s3 = "0.27.0-rc4" +tempdir = "0.3.7" # Web actix-cors = "0.5.4" diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 4c294b2ba2..967c91b5a1 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -75,7 +75,7 @@ CREATE TABLE tts_models ( -- The pytorch model -- For now, this will be a hash of the file contents. -- NB: NOT UNIQUE! We can allow duplicate uploads. - private_bucket_hash CHAR(32) NOT NULL, + private_bucket_hash CHAR(64) NOT NULL, -- Calculated average, on a scale of 0-100 -- Null with zero ratings. @@ -181,10 +181,10 @@ CREATE TABLE w2l_templates ( original_filename CHAR(255) NOT NULL, -- The original source image/video and the "precomputed" faces - private_bucket_hash CHAR(32) NOT NULL, + private_bucket_hash CHAR(64) NOT NULL, -- For the thumbnail we show. - public_bucket_hash CHAR(32) NOT NULL, + public_bucket_hash CHAR(64) NOT NULL, -- Calculated average, on a scale of 0-100 -- Null with zero ratings. diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 80316bdc1b..c5c0dab260 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -36,7 +36,7 @@ CREATE TABLE tts_results ( ) NOT NULL DEFAULT 'public', -- Where the wav, spectrogram, and etc. are located. - public_bucket_hash CHAR(32) NOT NULL, + public_bucket_hash CHAR(64) NOT NULL, -- ========== MODERATION DETAILS ========== @@ -100,7 +100,7 @@ CREATE TABLE w2l_results ( ) NOT NULL DEFAULT 'public', -- Where the wav, spectrogram, and etc. are located. - public_bucket_hash CHAR(32) NOT NULL, + public_bucket_hash CHAR(64) NOT NULL, -- ========== MODERATION DETAILS ========== diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index b7e6751d44..6e215d4623 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -24,6 +24,11 @@ use sqlx::mysql::MySqlPoolOptions; use std::path::PathBuf; use std::process::Command; use std::time::Duration; +use tempdir::TempDir; +use std::fs::File; +use std::io::{BufReader, Read}; +use ring::digest::{Context, Digest, SHA256}; +use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; @@ -143,13 +148,29 @@ async fn process_jobs(downloader: &Downloader, jobs: Vec) -> Ok(()) } -async fn call_script(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { +async fn download_file(downloader: &Downloader, + job: &TtsUploadJobRecord, + temp_dir: &TempDir) -> AnyhowResult +{ + + let temp_dir_path = temp_dir.path() + .to_str() + .unwrap_or("/tmp") + .to_string(); + + let filename = random_token(10); + let filename = format!("{}/{}.bin", temp_dir_path, filename); + + info!("Downloading to: {}", filename); + + let url = job.download_url.as_ref() + .map(|c| c.to_string()) + .unwrap_or("".to_string()); - let url = job.download_url.as_ref().map(|c| c.to_string()).unwrap_or("".to_string()); let command = format!("{} --url {} --output_file {}", downloader.download_script, &url, - "filename.txt"); + &filename); info!("Running command: {}", command); @@ -165,20 +186,50 @@ async fn call_script(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho return Err(anyhow!("Failure to execute command: {:?}", reason)) } - Ok(()) + Ok(filename) +} + + +fn sha256_digest(mut reader: R) -> AnyhowResult { + let mut context = Context::new(&SHA256); + let mut buffer = [0; 1024]; + + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + context.update(&buffer[..count]); + } + + Ok(context.finish()) +} + +fn get_file_hash(filename: &str) -> AnyhowResult { + let input = File::open(filename)?; + let reader = BufReader::new(input); + let digest = sha256_digest(reader)?; + + let hash = HEXLOWER_PERMISSIVE.encode(digest.as_ref()); + Ok(hash) } async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. - // TODO: 2. Download. + // TODO: 2. Download. (DONE) // TODO: 3. Upload. // TODO: 4. Save record. (DONE) // TODO: 5. Mark job done. (DONE) - let private_bucket_hash = random_token(32); // TODO: Use sha2/sha256 instead. + let temp_dir = format!("temp_{}", job.id); + let temp_dir = TempDir::new(&temp_dir)?; info!("Calling downloader..."); - call_script(downloader, job).await?; + let download_filename = download_file(downloader, job, &temp_dir).await?; + + let private_bucket_hash = get_file_hash(&download_filename)?; + + info!("File hash: {}", private_bucket_hash); info!("Saving model record..."); let id = insert_tts_model(&downloader.mysql_pool, job, &private_bucket_hash).await?; From a484207eb66d7f76a1cf6396bcb18d856e7f9c9f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 09:02:02 -0400 Subject: [PATCH 0095/1042] copy url --- migrations/2021-05-15-071601_assets/up.sql | 10 ++++++++-- sqlx-data.json | 20 ++++++++++---------- src/job/job_queries.rs | 3 ++- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 967c91b5a1..34596b98a8 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -70,7 +70,10 @@ CREATE TABLE tts_models ( ) NOT NULL DEFAULT 'public', -- The filename that was used at upload time. - original_filename VARCHAR(255) NOT NULL, + original_filename VARCHAR(255) DEFAULT NULL, + + -- Where the file was originally downloaded (if it was downloaded) + original_download_url VARCHAR(512) DEFAULT NULL, -- The pytorch model -- For now, this will be a hash of the file contents. @@ -178,7 +181,10 @@ CREATE TABLE w2l_templates ( ) NOT NULL DEFAULT 'public', -- The filename that was used at upload time. - original_filename CHAR(255) NOT NULL, + original_filename CHAR(255) DEFAULT NULL, + + -- Where the file was originally downloaded (if it was downloaded) + original_download_url VARCHAR(512) DEFAULT NULL, -- The original source image/video and the "precomputed" faces private_bucket_hash CHAR(64) NOT NULL, diff --git a/sqlx-data.json b/sqlx-data.json index 322fa1850e..6eb95a1a71 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -651,16 +651,6 @@ ] } }, - "b4fa3ec3e2f65f217ef8a08f54f1366b537d0e99497cd7e2882e8e5a96c4c4ab": { - "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_filename = \"TODO\",\n private_bucket_hash = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - } - }, "bf489cf419d342d144f072c310d6d70c9073b092b5721f489eaf2ffc8ad3915e": { "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -691,6 +681,16 @@ "nullable": [] } }, + "d1c8e0f9c5600fd7c296558cdb3525a2678b3e775f9e2bee0f1da979d4a82d70": { + "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + } + }, "e73192a3e763a287d5737821443c5a81649f17e39978497b99c52e1e896a75f6": { "query": "\nSELECT\n token as session_token,\n user_token\nFROM user_sessions\nWHERE token = ?\nAND deleted_at IS NULL\n ", "describe": { diff --git a/src/job/job_queries.rs b/src/job/job_queries.rs index 24b380949a..91cb43449c 100644 --- a/src/job/job_queries.rs +++ b/src/job/job_queries.rs @@ -129,7 +129,7 @@ SET description_rendered_html = '', creator_user_token = ?, creator_ip_address = ?, - original_filename = "TODO", + original_download_url = ?, private_bucket_hash = ? "#, model_token, @@ -137,6 +137,7 @@ SET job.title.to_string(), job.creator_user_token.clone(), job.creator_ip_address.clone(), + job.download_url.clone(), private_bucket_hash.to_string(), ) .execute(pool) From bf3c0c19daf8d4647fc09baa3f7fc268886357be Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 May 2021 02:31:31 -0400 Subject: [PATCH 0096/1042] bucket client (but needs tokio runtime!) --- .env | 4 + .gitignore | 1 + Cargo.lock | 238 +++++++++++++++++--------------------- Cargo.toml | 4 +- src/tts_download_job.rs | 39 ++++++- src/util/bucket_client.rs | 106 +++++++++++++++++ src/util/mod.rs | 1 + 7 files changed, 261 insertions(+), 132 deletions(-) create mode 100644 src/util/bucket_client.rs diff --git a/.env b/.env index 09610ddca7..039a586908 100644 --- a/.env +++ b/.env @@ -8,3 +8,7 @@ DATABASE_URL=mysql://root:root@localhost/storyteller # We can't serve HTTPS locally without self-signed certificate nonsense. COOKIE_SECURE=false COOKIE_DOMAIN=.jungle.horse + +# Bucket config (non-secret) +REGION_NAME=us-east1 +BUCKET_NAME=dev-vocodes-private-uploads diff --git a/.gitignore b/.gitignore index e726beb732..e68a75a505 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *secret*sh *~ .env +.env-secrets .idea/ _ignored_schema.rs production_database_secret_connect.sh diff --git a/Cargo.lock b/Cargo.lock index 5fcd5bd984..bb0df3ab05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,7 +15,7 @@ dependencies = [ "log", "pin-project 0.4.28", "tokio 0.2.25", - "tokio-util 0.3.1", + "tokio-util", ] [[package]] @@ -97,7 +97,7 @@ dependencies = [ "futures-core", "futures-util", "fxhash", - "h2 0.2.7", + "h2", "http", "httparse", "indexmap", @@ -334,6 +334,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" + [[package]] name = "ahash" version = "0.4.7" @@ -553,9 +559,9 @@ checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" [[package]] name = "attohttpc" -version = "0.16.3" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb8867f378f33f78a811a8eb9bf108ad99430d7aad43315dd9319c827ef6247" +checksum = "fe174d1b67f7b2bafed829c09db039301eb5841f66e43be2cf60b326e7f8e2cc" dependencies = [ "http", "log", @@ -564,7 +570,6 @@ dependencies = [ "serde", "serde_json", "url 2.2.2", - "wildmatch", ] [[package]] @@ -616,27 +621,27 @@ dependencies = [ [[package]] name = "aws-creds" -version = "0.26.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1331d069460a674d42bd27c12b47ce578f789954c7bd7f239fd030771eca6616" +checksum = "ad53a54cb2c99990e96eefacde6f143dc6d471ab70809d26d360292be421d490" dependencies = [ - "anyhow", "attohttpc", "dirs", "rust-ini", "serde", "serde-xml-rs", "serde_derive", + "simpl", "url 2.2.2", ] [[package]] name = "aws-region" -version = "0.23.2" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2884b8f2aaeb4a4bf80b219b4fe1d340139ca9331679c57e0fd4a24f571a78bd" +checksum = "f610af4a396f07592014dc3410f6ad78fab931852a99bb6cfdc1ad04b9329b80" dependencies = [ - "anyhow", + "simpl", ] [[package]] @@ -948,6 +953,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crypto-mac" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58bcd97a54c7ca5ce2f6eb16f6bede5b0ab5f0055fedc17d2f0b4466e21671ca" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "crypto-mac" version = "0.10.0" @@ -1377,28 +1392,19 @@ dependencies = [ "indexmap", "slab", "tokio 0.2.25", - "tokio-util 0.3.1", + "tokio-util", "tracing", "tracing-futures", ] [[package]] -name = "h2" -version = "0.3.3" +name = "hashbrown" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" +checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" dependencies = [ - "bytes 1.0.1", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio 1.5.0", - "tokio-util 0.6.7", - "tracing", + "ahash 0.3.8", + "autocfg 1.0.1", ] [[package]] @@ -1416,7 +1422,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" dependencies = [ - "hashbrown", + "hashbrown 0.9.1", ] [[package]] @@ -1443,13 +1449,23 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deae6d9dbb35ec2c502d62b8f7b1c000a0822c3b0794ba36b3149c0a1c840dff" +dependencies = [ + "crypto-mac 0.9.1", + "digest", +] + [[package]] name = "hmac" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ - "crypto-mac", + "crypto-mac 0.10.0", "digest", ] @@ -1477,13 +1493,12 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" dependencies = [ - "bytes 1.0.1", + "bytes 0.5.6", "http", - "pin-project-lite 0.2.6", ] [[package]] @@ -1494,9 +1509,9 @@ checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" [[package]] name = "httpdate" -version = "1.0.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" [[package]] name = "humantime" @@ -1509,23 +1524,23 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.7" +version = "0.13.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54" +checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" dependencies = [ - "bytes 1.0.1", + "bytes 0.5.6", "futures-channel", "futures-core", "futures-util", - "h2 0.3.3", + "h2", "http", "http-body", "httparse", "httpdate", "itoa", "pin-project 1.0.7", - "socket2 0.4.0", - "tokio 1.5.0", + "socket2 0.3.19", + "tokio 0.2.25", "tower-service", "tracing", "want", @@ -1533,15 +1548,15 @@ dependencies = [ [[package]] name = "hyper-tls" -version = "0.5.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" dependencies = [ - "bytes 1.0.1", + "bytes 0.5.6", "hyper", "native-tls", - "tokio 1.5.0", - "tokio-native-tls", + "tokio 0.2.25", + "tokio-tls", ] [[package]] @@ -1573,7 +1588,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg 1.0.1", - "hashbrown", + "hashbrown 0.9.1", ] [[package]] @@ -1634,9 +1649,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86e46349d67dc03bdbdb28da0337a355a53ca1d5156452722c36fe21d0e6389b" dependencies = [ "base64 0.13.0", - "crypto-mac", + "crypto-mac 0.10.0", "digest", - "hmac", + "hmac 0.10.1", "serde", "serde_json", "sha2", @@ -1759,17 +1774,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-async" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3919783e441e38b0512fae9eebd87c445c42e39508231fba2a8b9fd7ba88e999" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "md-5" version = "0.9.1" @@ -1809,15 +1813,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "minidom" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e" -dependencies = [ - "quick-xml", -] - [[package]] name = "miniz_oxide" version = "0.4.4" @@ -2091,12 +2086,12 @@ dependencies = [ [[package]] name = "ordered-multimap" -version = "0.3.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485" +checksum = "e88f947c6799d5eff50e6cf8a2365c17ac4aa8f8f43aceeedc29b616d872a358" dependencies = [ "dlv-list", - "hashbrown", + "hashbrown 0.7.2", ] [[package]] @@ -2263,15 +2258,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-xml" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" -dependencies = [ - "memchr", -] - [[package]] name = "quote" version = "1.0.9" @@ -2452,12 +2438,12 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.3" +version = "0.10.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" +checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" dependencies = [ "base64 0.13.0", - "bytes 1.0.1", + "bytes 0.5.6", "encoding_rs", "futures-core", "futures-util", @@ -2470,14 +2456,15 @@ dependencies = [ "lazy_static", "log", "mime", + "mime_guess", "native-tls", "percent-encoding 2.1.0", "pin-project-lite 0.2.6", "serde", "serde_json", "serde_urlencoded", - "tokio 1.5.0", - "tokio-native-tls", + "tokio 0.2.25", + "tokio-tls", "url 2.2.2", "wasm-bindgen", "wasm-bindgen-futures", @@ -2532,9 +2519,9 @@ dependencies = [ [[package]] name = "rust-ini" -version = "0.16.1" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55b134767a87e0b086f73a4ce569ac9ce7d202f39c8eab6caa266e2617e73ac6" +checksum = "7a3679dd538c876a7b606f3bb951c8a20fc281a0ff7795f59f7cb490e3f979e1" dependencies = [ "cfg-if 0.1.10", "ordered-multimap", @@ -2542,12 +2529,11 @@ dependencies = [ [[package]] name = "rust-s3" -version = "0.27.0-rc4" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c93272c1d654d492f8ab30b94cd43d98f2700b1db55b2576aff7712ce40e3ef" +checksum = "a04bdd0f5118d06ef0e2daa658a9e5135282bcecfd6c46b11fddf47b1d5d736d" dependencies = [ - "anyhow", - "async-trait", + "async-std", "aws-creds", "aws-region", "base64 0.13.0", @@ -2555,21 +2541,20 @@ dependencies = [ "chrono", "futures", "hex", - "hmac", + "hmac 0.9.0", "http", "log", - "maybe-async", "md5", - "minidom", "percent-encoding 2.1.0", "reqwest", "serde", "serde-xml-rs", "serde_derive", "sha2", - "tokio 1.5.0", - "tokio-stream", + "simpl", + "tokio 0.2.25", "url 2.2.2", + "uuid", ] [[package]] @@ -2748,6 +2733,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simpl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a30f10c911c0355f80f1c2faa8096efc4a58cdf8590b954d5b395efa071c711" + [[package]] name = "simple_asn1" version = "0.5.2" @@ -2985,10 +2976,11 @@ dependencies = [ "data-encoding", "derive_more", "diesel", + "dotenv", "easyenv", "futures", "hex", - "hmac", + "hmac 0.10.1", "hostname", "jwt", "lazy_static", @@ -3186,6 +3178,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" dependencies = [ "bytes 0.5.6", + "fnv", "futures-core", "iovec", "lazy_static", @@ -3196,6 +3189,7 @@ dependencies = [ "pin-project-lite 0.1.12", "signal-hook-registry", "slab", + "tokio-macros", "winapi 0.3.9", ] @@ -3206,9 +3200,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" dependencies = [ "autocfg 1.0.1", - "bytes 1.0.1", "libc", - "memchr", "mio 0.7.11", "once_cell", "parking_lot", @@ -3218,24 +3210,24 @@ dependencies = [ ] [[package]] -name = "tokio-native-tls" -version = "0.3.0" +name = "tokio-macros" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" dependencies = [ - "native-tls", - "tokio 1.5.0", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "tokio-stream" -version = "0.1.6" +name = "tokio-tls" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8864d706fdb3cc0843a49647ac892720dac98a6eeb818b77190592cf4994066" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" dependencies = [ - "futures-core", - "pin-project-lite 0.2.6", - "tokio 1.5.0", + "native-tls", + "tokio 0.2.25", ] [[package]] @@ -3252,20 +3244,6 @@ dependencies = [ "tokio 0.2.25", ] -[[package]] -name = "tokio-util" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" -dependencies = [ - "bytes 1.0.1", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.2.6", - "tokio 1.5.0", -] - [[package]] name = "toml" version = "0.5.8" @@ -3437,6 +3415,12 @@ dependencies = [ "percent-encoding 2.1.0", ] +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" + [[package]] name = "v_escape" version = "0.15.0" @@ -3627,12 +3611,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" -[[package]] -name = "wildmatch" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f44b95f62d34113cf558c93511ac93027e03e9c29a60dd0fd70e6e025c7270a" - [[package]] name = "winapi" version = "0.2.8" diff --git a/Cargo.toml b/Cargo.toml index a12bd83942..7539f7ec29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ async-std = { version = "1.6", features = [ "attributes" ] } base64 = "0.13.0" chrono = { version = "0.4.19", features = ["serde"] } derive_more = "0.99.14" +dotenv = "0.15.0" easyenv = "0.0.5" futures = "0.3.15" hex = "0.4.3" @@ -55,7 +56,8 @@ sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", # Files data-encoding = "2.3.2" ring = "0.17.0-alpha.10" -rust-s3 = "0.27.0-rc4" +#rust-s3 = "0.27.0-rc4" +rust-s3 = "0.26.3" tempdir = "0.3.7" # Web diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 6e215d4623..f06a4b89cf 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -10,6 +10,7 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; +use crate::util::bucket_client::BucketClient; use crate::job::job_queries::TtsUploadJobRecord; use crate::job::job_queries::insert_tts_model; use crate::job::job_queries::mark_tts_upload_job_done; @@ -21,7 +22,7 @@ use crate::util::random_token::random_token; use log::{warn, info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; -use std::path::PathBuf; +use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; use tempdir::TempDir; @@ -30,12 +31,19 @@ use std::io::{BufReader, Read}; use ring::digest::{Context, Digest, SHA256}; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; +// Buckets +const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; +const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; +const ENV_REGION_NAME : &'static str = "REGION_NAME"; +const ENV_BUCKET_NAME : &'static str = "BUCKET_NAME"; + const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Downloader { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, + pub bucket_client: BucketClient, // Command to run pub download_script: String, } @@ -44,6 +52,8 @@ struct Downloader { async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + let _ = dotenv::from_filename(".env-secrets").ok(); + info!("Obtaining hostname..."); let server_hostname = hostname::get() @@ -53,6 +63,19 @@ async fn main() -> AnyhowResult<()> { info!("Hostname: {}", &server_hostname); + // Bucket stuff + let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; + let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; + let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; + let bucket_name = easyenv::get_env_string_required(ENV_BUCKET_NAME)?; + + let bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &bucket_name, + )?; + let temp_directory = easyenv::get_env_string_or_default( "DOWNLOAD_TEMP_DIR", DEFAULT_TEMP_DIR); @@ -80,6 +103,7 @@ async fn main() -> AnyhowResult<()> { let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, + bucket_client, download_script, }; @@ -231,6 +255,19 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("File hash: {}", private_bucket_hash); + let object_name = format!( + "user_uploaded_tts_models/{}/{}/{}/{}.bin", + &private_bucket_hash[0..1], + &private_bucket_hash[1..2], + &private_bucket_hash[2..3], + &private_bucket_hash, + ); + + info!("Destination bucket path: {}", object_name); + + let file_path = PathBuf::from(download_filename); + downloader.bucket_client.upload_filename(&object_name, &file_path).await?; + info!("Saving model record..."); let id = insert_tts_model(&downloader.mysql_pool, job, &private_bucket_hash).await?; diff --git a/src/util/bucket_client.rs b/src/util/bucket_client.rs new file mode 100644 index 0000000000..1da9ec95b2 --- /dev/null +++ b/src/util/bucket_client.rs @@ -0,0 +1,106 @@ +use anyhow::bail; +use log::info; +use s3::bucket::Bucket; +use s3::creds::Credentials; +use s3::region::Region; +use std::path::{PathBuf, Path}; +use std::str::FromStr; +use std::fs::File; +use std::io::Read; + +#[derive(Clone)] +pub struct BucketClient { + bucket: Bucket, +} + +impl BucketClient { + + pub fn create( + access_key: &str, + secret_key: &str, + region_name: &str, + bucket_name: &str) -> anyhow::Result + { + let credentials = Credentials::new( + Some(&access_key), + Some(&secret_key), + None, + None, + None, + )?; + // NB: The GCS buckets aren't supported by default. + let region = Region::Custom { + region: region_name.to_owned(), + endpoint: "https://storage.googleapis.com".to_owned(), + }; + let bucket = Bucket::new(&bucket_name, region, credentials)?; + + Ok(Self { + bucket, + }) + } + + pub async fn upload_file(&self, object_name: &str, bytes: &[u8]) -> anyhow::Result<()> { + info!("Filename for bucket: {}", object_name); + + let (_, code) = self.bucket.put_object(object_name, bytes).await?; + + info!("upload code: {}", code); + + Ok(()) + } + + pub fn upload_file_blocking(&self, object_name: &str, bytes: &[u8]) -> anyhow::Result<()> { + info!("Filename for bucket: {}", object_name); + + let (_, code) = self.bucket.put_object_blocking(object_name, bytes)?; + + info!("upload code: {}", code); + + Ok(()) + } + + pub async fn upload_filename(&self, object_name: &str, filename: &Path) -> anyhow::Result<()> { + // TODO: does a newer version of this crate handle streaming/buffering file contents? + let mut file = File::open(filename)?; + let mut buffer : Vec = Vec::new(); + file.read_to_end(&mut buffer)?; + + self.upload_file(object_name, &buffer).await + } + + pub fn upload_filename_blocking(&self, object_name: &str, filename: &Path) -> anyhow::Result<()> { + // TODO: does a newer version of this crate handle streaming/buffering file contents? + let mut file = File::open(filename)?; + let mut buffer : Vec = Vec::new(); + file.read_to_end(&mut buffer)?; + + self.upload_file_blocking(object_name, &buffer) + } + + pub async fn download_file(&self, path: &str) -> anyhow::Result> { + info!("downloading from bucket: {}", path); + let (bytes, code) = self.bucket.get_object(path).await?; + + match code { + 404 => bail!("File not found in bucket: {}", path), + _ => {}, + } + + info!("download code: {}", code); + Ok(bytes) + } + + pub fn download_file_blocking(&self, path: &str) -> anyhow::Result> { + info!("downloading from bucket: {}", path); + let (bytes, code) = self.bucket.get_object_blocking(path)?; + + match code { + 404 => bail!("File not found in bucket: {}", path), + _ => {}, + } + + info!("download code: {}", code); + Ok(bytes) + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 3dda00a986..61b5049457 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,4 +1,5 @@ pub mod anyhow_result; +pub mod bucket_client; pub mod cookie_manager; pub mod filesystem; pub mod ip_address; From 4d9fb180666d378fbb8bc4d6dd0b593c475bb8aa Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 May 2021 02:52:26 -0400 Subject: [PATCH 0097/1042] bucket uploads with correct runtime (but getting 403 forbid) --- Cargo.lock | 405 ++++++++++++++++++++++++++------------ Cargo.toml | 3 +- src/util/bucket_client.rs | 61 +++--- 3 files changed, 311 insertions(+), 158 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bb0df3ab05..c2e4f97154 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -335,10 +335,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] -name = "ahash" -version = "0.3.8" +name = "aead" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" +checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" +dependencies = [ + "generic-array", +] + +[[package]] +name = "aes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" +dependencies = [ + "aes-soft", + "aesni", + "cipher", +] + +[[package]] +name = "aes-gcm" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aes-soft" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" +dependencies = [ + "cipher", + "opaque-debug", +] + +[[package]] +name = "aesni" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" +dependencies = [ + "cipher", + "opaque-debug", +] [[package]] name = "ahash" @@ -427,6 +475,7 @@ dependencies = [ "futures-lite", "num_cpus", "once_cell", + "tokio 0.2.25", ] [[package]] @@ -559,17 +608,16 @@ checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" [[package]] name = "attohttpc" -version = "0.15.0" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe174d1b67f7b2bafed829c09db039301eb5841f66e43be2cf60b326e7f8e2cc" +checksum = "fdb8867f378f33f78a811a8eb9bf108ad99430d7aad43315dd9319c827ef6247" dependencies = [ "http", "log", - "native-tls", - "openssl", "serde", "serde_json", "url 2.2.2", + "wildmatch", ] [[package]] @@ -621,27 +669,27 @@ dependencies = [ [[package]] name = "aws-creds" -version = "0.24.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad53a54cb2c99990e96eefacde6f143dc6d471ab70809d26d360292be421d490" +checksum = "1331d069460a674d42bd27c12b47ce578f789954c7bd7f239fd030771eca6616" dependencies = [ + "anyhow", "attohttpc", "dirs", "rust-ini", "serde", "serde-xml-rs", "serde_derive", - "simpl", "url 2.2.2", ] [[package]] name = "aws-region" -version = "0.22.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f610af4a396f07592014dc3410f6ad78fab931852a99bb6cfdc1ad04b9329b80" +checksum = "2884b8f2aaeb4a4bf80b219b4fe1d340139ca9331679c57e0fd4a24f571a78bd" dependencies = [ - "simpl", + "anyhow", ] [[package]] @@ -841,6 +889,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "cipher" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" +dependencies = [ + "generic-array", +] + [[package]] name = "concurrent-queue" version = "1.2.2" @@ -868,7 +925,13 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" dependencies = [ + "aes-gcm", + "base64 0.13.0", + "hkdf", + "hmac", "percent-encoding 2.1.0", + "rand 0.8.3", + "sha2", "time 0.2.26", "version_check 0.9.3", ] @@ -904,6 +967,12 @@ dependencies = [ "libc", ] +[[package]] +name = "cpuid-bool" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" + [[package]] name = "crc" version = "1.8.1" @@ -953,16 +1022,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "crypto-mac" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bcd97a54c7ca5ce2f6eb16f6bede5b0ab5f0055fedc17d2f0b4466e21671ca" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "crypto-mac" version = "0.10.0" @@ -983,6 +1042,25 @@ dependencies = [ "syn", ] +[[package]] +name = "ctr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" +dependencies = [ + "cipher", +] + +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", +] + [[package]] name = "data-encoding" version = "2.3.2" @@ -1364,6 +1442,16 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] +[[package]] +name = "ghash" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gloo-timers" version = "0.2.1" @@ -1397,16 +1485,6 @@ dependencies = [ "tracing-futures", ] -[[package]] -name = "hashbrown" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" -dependencies = [ - "ahash 0.3.8", - "autocfg 1.0.1", -] - [[package]] name = "hashbrown" version = "0.9.1" @@ -1422,7 +1500,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" dependencies = [ - "hashbrown 0.9.1", + "hashbrown", ] [[package]] @@ -1450,13 +1528,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] -name = "hmac" -version = "0.9.0" +name = "hkdf" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deae6d9dbb35ec2c502d62b8f7b1c000a0822c3b0794ba36b3149c0a1c840dff" +checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" dependencies = [ - "crypto-mac 0.9.1", "digest", + "hmac", ] [[package]] @@ -1465,7 +1543,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ - "crypto-mac 0.10.0", + "crypto-mac", "digest", ] @@ -1501,6 +1579,46 @@ dependencies = [ "http", ] +[[package]] +name = "http-client" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce318d86a47d18d1db645c979214f809a6cd625202ad334ef75ca813b30dac80" +dependencies = [ + "async-trait", + "cfg-if 1.0.0", + "dashmap", + "futures-util", + "http-types", + "hyper", + "hyper-tls", + "log", + "tokio 0.2.25", +] + +[[package]] +name = "http-types" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad077d89137cd3debdce53c66714dc536525ef43fe075d41ddc0a8ac11f85957" +dependencies = [ + "anyhow", + "async-channel", + "async-std", + "base64 0.13.0", + "cookie", + "futures-lite", + "http", + "infer", + "pin-project-lite 0.2.6", + "rand 0.7.3", + "serde", + "serde_json", + "serde_qs", + "serde_urlencoded", + "url 2.2.2", +] + [[package]] name = "httparse" version = "1.4.1" @@ -1588,9 +1706,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg 1.0.1", - "hashbrown 0.9.1", + "hashbrown", ] +[[package]] +name = "infer" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" + [[package]] name = "instant" version = "0.1.9" @@ -1618,15 +1742,9 @@ dependencies = [ "socket2 0.3.19", "widestring", "winapi 0.3.9", - "winreg 0.6.2", + "winreg", ] -[[package]] -name = "ipnet" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" - [[package]] name = "itoa" version = "0.4.7" @@ -1649,9 +1767,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86e46349d67dc03bdbdb28da0337a355a53ca1d5156452722c36fe21d0e6389b" dependencies = [ "base64 0.13.0", - "crypto-mac 0.10.0", + "crypto-mac", "digest", - "hmac 0.10.1", + "hmac", "serde", "serde_json", "sha2", @@ -1774,6 +1892,17 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "maybe-async" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3919783e441e38b0512fae9eebd87c445c42e39508231fba2a8b9fd7ba88e999" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "md-5" version = "0.9.1" @@ -1813,6 +1942,15 @@ dependencies = [ "unicase", ] +[[package]] +name = "minidom" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e" +dependencies = [ + "quick-xml", +] + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -2086,12 +2224,12 @@ dependencies = [ [[package]] name = "ordered-multimap" -version = "0.2.4" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88f947c6799d5eff50e6cf8a2365c17ac4aa8f8f43aceeedc29b616d872a358" +checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485" dependencies = [ "dlv-list", - "hashbrown 0.7.2", + "hashbrown", ] [[package]] @@ -2225,6 +2363,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "polyval" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" +dependencies = [ + "cpuid-bool", + "opaque-debug", + "universal-hash", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -2258,6 +2407,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.9" @@ -2436,42 +2594,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "reqwest" -version = "0.10.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" -dependencies = [ - "base64 0.13.0", - "bytes 0.5.6", - "encoding_rs", - "futures-core", - "futures-util", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "lazy_static", - "log", - "mime", - "mime_guess", - "native-tls", - "percent-encoding 2.1.0", - "pin-project-lite 0.2.6", - "serde", - "serde_json", - "serde_urlencoded", - "tokio 0.2.25", - "tokio-tls", - "url 2.2.2", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg 0.7.0", -] - [[package]] name = "resolv-conf" version = "0.7.0" @@ -2519,9 +2641,9 @@ dependencies = [ [[package]] name = "rust-ini" -version = "0.15.3" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a3679dd538c876a7b606f3bb951c8a20fc281a0ff7795f59f7cb490e3f979e1" +checksum = "55b134767a87e0b086f73a4ce569ac9ce7d202f39c8eab6caa266e2617e73ac6" dependencies = [ "cfg-if 0.1.10", "ordered-multimap", @@ -2529,11 +2651,13 @@ dependencies = [ [[package]] name = "rust-s3" -version = "0.26.4" +version = "0.27.0-rc4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a04bdd0f5118d06ef0e2daa658a9e5135282bcecfd6c46b11fddf47b1d5d736d" +checksum = "5c93272c1d654d492f8ab30b94cd43d98f2700b1db55b2576aff7712ce40e3ef" dependencies = [ + "anyhow", "async-std", + "async-trait", "aws-creds", "aws-region", "base64 0.13.0", @@ -2541,20 +2665,20 @@ dependencies = [ "chrono", "futures", "hex", - "hmac 0.9.0", + "hmac", "http", "log", + "maybe-async", "md5", + "minidom", "percent-encoding 2.1.0", - "reqwest", "serde", "serde-xml-rs", "serde_derive", "sha2", - "simpl", - "tokio 0.2.25", + "surf", + "tokio-stream", "url 2.2.2", - "uuid", ] [[package]] @@ -2670,6 +2794,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_qs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5af82de3c6549b001bec34961ff2d6a54339a87bab37ce901b693401f27de6cb" +dependencies = [ + "data-encoding", + "percent-encoding 2.1.0", + "serde", + "thiserror", +] + [[package]] name = "serde_urlencoded" version = "0.7.0" @@ -2733,12 +2869,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simpl" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a30f10c911c0355f80f1c2faa8096efc4a58cdf8590b954d5b395efa071c711" - [[package]] name = "simple_asn1" version = "0.5.2" @@ -2980,7 +3110,7 @@ dependencies = [ "easyenv", "futures", "hex", - "hmac 0.10.1", + "hmac", "hostname", "jwt", "lazy_static", @@ -3016,6 +3146,26 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" +[[package]] +name = "surf" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a154d33ca6b5e1fe6fd1c760e5a5cc1202425f6cca2e13229f16a69009f6328" +dependencies = [ + "async-std", + "async-trait", + "cfg-if 1.0.0", + "futures-util", + "http-client", + "http-types", + "log", + "mime_guess", + "once_cell", + "pin-project-lite 0.2.6", + "serde", + "serde_json", +] + [[package]] name = "syn" version = "1.0.72" @@ -3189,7 +3339,6 @@ dependencies = [ "pin-project-lite 0.1.12", "signal-hook-registry", "slab", - "tokio-macros", "winapi 0.3.9", ] @@ -3210,14 +3359,14 @@ dependencies = [ ] [[package]] -name = "tokio-macros" -version = "0.2.6" +name = "tokio-stream" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" +checksum = "f8864d706fdb3cc0843a49647ac892720dac98a6eeb818b77190592cf4994066" dependencies = [ - "proc-macro2", - "quote", - "syn", + "futures-core", + "pin-project-lite 0.2.6", + "tokio 1.5.0", ] [[package]] @@ -3386,6 +3535,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "universal-hash" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -3413,14 +3572,9 @@ dependencies = [ "idna 0.2.3", "matches", "percent-encoding 2.1.0", + "serde", ] -[[package]] -name = "uuid" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" - [[package]] name = "v_escape" version = "0.15.0" @@ -3515,8 +3669,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if 1.0.0", - "serde", - "serde_json", "wasm-bindgen-macro", ] @@ -3611,6 +3763,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" +[[package]] +name = "wildmatch" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f44b95f62d34113cf558c93511ac93027e03e9c29a60dd0fd70e6e025c7270a" + [[package]] name = "winapi" version = "0.2.8" @@ -3663,15 +3821,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "winreg" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" -dependencies = [ - "winapi 0.3.9", -] - [[package]] name = "ws2_32-sys" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 7539f7ec29..c51d8cb97e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,8 @@ sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", data-encoding = "2.3.2" ring = "0.17.0-alpha.10" #rust-s3 = "0.27.0-rc4" -rust-s3 = "0.26.3" +rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } + tempdir = "0.3.7" # Web diff --git a/src/util/bucket_client.rs b/src/util/bucket_client.rs index 1da9ec95b2..c910aa8b8c 100644 --- a/src/util/bucket_client.rs +++ b/src/util/bucket_client.rs @@ -50,15 +50,16 @@ impl BucketClient { Ok(()) } - pub fn upload_file_blocking(&self, object_name: &str, bytes: &[u8]) -> anyhow::Result<()> { - info!("Filename for bucket: {}", object_name); - - let (_, code) = self.bucket.put_object_blocking(object_name, bytes)?; - - info!("upload code: {}", code); - - Ok(()) - } + // NB: New version has blocking client rather than blocking calls. + // pub fn upload_file_blocking(&self, object_name: &str, bytes: &[u8]) -> anyhow::Result<()> { + // info!("Filename for bucket: {}", object_name); + // + // let (_, code) = self.bucket.put_object_blocking(object_name, bytes)?; + // + // info!("upload code: {}", code); + // + // Ok(()) + // } pub async fn upload_filename(&self, object_name: &str, filename: &Path) -> anyhow::Result<()> { // TODO: does a newer version of this crate handle streaming/buffering file contents? @@ -69,14 +70,15 @@ impl BucketClient { self.upload_file(object_name, &buffer).await } - pub fn upload_filename_blocking(&self, object_name: &str, filename: &Path) -> anyhow::Result<()> { - // TODO: does a newer version of this crate handle streaming/buffering file contents? - let mut file = File::open(filename)?; - let mut buffer : Vec = Vec::new(); - file.read_to_end(&mut buffer)?; - - self.upload_file_blocking(object_name, &buffer) - } + // NB: New version has blocking client rather than blocking calls. + // pub fn upload_filename_blocking(&self, object_name: &str, filename: &Path) -> anyhow::Result<()> { + // // TODO: does a newer version of this crate handle streaming/buffering file contents? + // let mut file = File::open(filename)?; + // let mut buffer : Vec = Vec::new(); + // file.read_to_end(&mut buffer)?; + // + // self.upload_file_blocking(object_name, &buffer) + // } pub async fn download_file(&self, path: &str) -> anyhow::Result> { info!("downloading from bucket: {}", path); @@ -91,16 +93,17 @@ impl BucketClient { Ok(bytes) } - pub fn download_file_blocking(&self, path: &str) -> anyhow::Result> { - info!("downloading from bucket: {}", path); - let (bytes, code) = self.bucket.get_object_blocking(path)?; - - match code { - 404 => bail!("File not found in bucket: {}", path), - _ => {}, - } - - info!("download code: {}", code); - Ok(bytes) - } + // NB: New version has blocking client rather than blocking calls. + // pub fn download_file_blocking(&self, path: &str) -> anyhow::Result> { + // info!("downloading from bucket: {}", path); + // let (bytes, code) = self.bucket.get_object_blocking(path)?; + // + // match code { + // 404 => bail!("File not found in bucket: {}", path), + // _ => {}, + // } + // + // info!("download code: {}", code); + // Ok(bytes) + // } } From 68aeded077069cfa8d5e23d956f607c9d811b73b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 May 2021 03:40:57 -0400 Subject: [PATCH 0098/1042] This upload worked (filename causes failure!) --- Cargo.lock | 35 ++--------------------------------- Cargo.toml | 2 +- src/tts_download_job.rs | 4 ++++ src/util/bucket_client.rs | 6 +++++- 4 files changed, 12 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c2e4f97154..a0d3176d09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1942,15 +1942,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "minidom" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e" -dependencies = [ - "quick-xml", -] - [[package]] name = "miniz_oxide" version = "0.4.4" @@ -2407,15 +2398,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-xml" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" -dependencies = [ - "memchr", -] - [[package]] name = "quote" version = "1.0.9" @@ -2651,9 +2633,9 @@ dependencies = [ [[package]] name = "rust-s3" -version = "0.27.0-rc4" +version = "0.27.0-beta5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c93272c1d654d492f8ab30b94cd43d98f2700b1db55b2576aff7712ce40e3ef" +checksum = "caa667e2c2460cad8e10d8d20ae32b2d8ed6e965e22418d6a984a594871013d2" dependencies = [ "anyhow", "async-std", @@ -2670,14 +2652,12 @@ dependencies = [ "log", "maybe-async", "md5", - "minidom", "percent-encoding 2.1.0", "serde", "serde-xml-rs", "serde_derive", "sha2", "surf", - "tokio-stream", "url 2.2.2", ] @@ -3358,17 +3338,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "tokio-stream" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8864d706fdb3cc0843a49647ac892720dac98a6eeb818b77190592cf4994066" -dependencies = [ - "futures-core", - "pin-project-lite 0.2.6", - "tokio 1.5.0", -] - [[package]] name = "tokio-tls" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index c51d8cb97e..9a6fc5b4f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", data-encoding = "2.3.2" ring = "0.17.0-alpha.10" #rust-s3 = "0.27.0-rc4" -rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } +rust-s3 = { version = "=0.27.0-beta5", features = ["with-async-std"], default-features = false } tempdir = "0.3.7" diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index f06a4b89cf..c16146587e 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -262,6 +262,10 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho &private_bucket_hash[2..3], &private_bucket_hash, ); + let object_name = format!( + "{}.bin", + &private_bucket_hash, + ); info!("Destination bucket path: {}", object_name); diff --git a/src/util/bucket_client.rs b/src/util/bucket_client.rs index c910aa8b8c..87b709dd0e 100644 --- a/src/util/bucket_client.rs +++ b/src/util/bucket_client.rs @@ -43,10 +43,14 @@ impl BucketClient { pub async fn upload_file(&self, object_name: &str, bytes: &[u8]) -> anyhow::Result<()> { info!("Filename for bucket: {}", object_name); - let (_, code) = self.bucket.put_object(object_name, bytes).await?; + let (body_bytes, code) = self.bucket.put_object(object_name, bytes).await?; info!("upload code: {}", code); + let body = String::from_utf8(body_bytes)?; + + info!("upload body: {}", body); + Ok(()) } From e9ad9bbfe04b8a525b1d10f5caa2e1a42b93ea00 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 May 2021 04:39:55 -0400 Subject: [PATCH 0099/1042] switch back to tokio, this works onlyfor non-dirs --- Cargo.lock | 723 +++++++++----------------------------- Cargo.toml | 13 +- src/tts_download_job.rs | 22 +- src/util/bucket_client.rs | 19 +- 4 files changed, 207 insertions(+), 570 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a0d3176d09..b2279ce59e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,7 +15,7 @@ dependencies = [ "log", "pin-project 0.4.28", "tokio 0.2.25", - "tokio-util", + "tokio-util 0.3.1", ] [[package]] @@ -97,7 +97,7 @@ dependencies = [ "futures-core", "futures-util", "fxhash", - "h2", + "h2 0.2.7", "http", "httparse", "indexmap", @@ -174,7 +174,7 @@ checksum = "bc7d7cd957c9ed92288a7c3c96af81fa5291f65247a76a34dac7b6af74e52ba0" dependencies = [ "actix-macros 0.2.0", "futures-core", - "tokio 1.5.0", + "tokio 1.6.0", ] [[package]] @@ -334,60 +334,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aead" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" -dependencies = [ - "generic-array", -] - -[[package]] -name = "aes" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" -dependencies = [ - "aes-soft", - "aesni", - "cipher", -] - -[[package]] -name = "aes-gcm" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" -dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", - "subtle", -] - -[[package]] -name = "aes-soft" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" -dependencies = [ - "cipher", - "opaque-debug", -] - -[[package]] -name = "aesni" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" -dependencies = [ - "cipher", - "opaque-debug", -] - [[package]] name = "ahash" version = "0.4.7" @@ -426,160 +372,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" -[[package]] -name = "async-attributes" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" -dependencies = [ - "quote", - "syn", -] - -[[package]] -name = "async-channel" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - -[[package]] -name = "async-executor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" -dependencies = [ - "async-task", - "concurrent-queue", - "fastrand", - "futures-lite", - "once_cell", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9586ec52317f36de58453159d48351bc244bc24ced3effc1fce22f3d48664af6" -dependencies = [ - "async-channel", - "async-executor", - "async-io", - "async-mutex", - "blocking", - "futures-lite", - "num_cpus", - "once_cell", - "tokio 0.2.25", -] - -[[package]] -name = "async-io" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bbfd5cf2794b1e908ea8457e6c45f8f8f1f6ec5f74617bf4662623f47503c3b" -dependencies = [ - "concurrent-queue", - "fastrand", - "futures-lite", - "libc", - "log", - "once_cell", - "parking", - "polling", - "slab", - "socket2 0.4.0", - "waker-fn", - "winapi 0.3.9", -] - -[[package]] -name = "async-lock" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6a8ea61bf9947a1007c5cada31e647dbc77b103c679858150003ba697ea798b" -dependencies = [ - "event-listener", -] - -[[package]] -name = "async-mutex" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" -dependencies = [ - "event-listener", -] - -[[package]] -name = "async-native-tls" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9e7a929bd34c68a82d58a4de7f86fffdaf97fb2af850162a7bb19dd7269b33" -dependencies = [ - "async-std", - "native-tls", - "thiserror", - "url 2.2.2", -] - -[[package]] -name = "async-process" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f38756dd9ac84671c428afbf7c9f7495feff9ec5b0710f17100098e5b354ac" -dependencies = [ - "async-io", - "blocking", - "cfg-if 1.0.0", - "event-listener", - "futures-lite", - "libc", - "once_cell", - "signal-hook", - "winapi 0.3.9", -] - -[[package]] -name = "async-std" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f06685bad74e0570f5213741bea82158279a4103d988e57bfada11ad230341" -dependencies = [ - "async-attributes", - "async-channel", - "async-global-executor", - "async-io", - "async-lock", - "async-process", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "num_cpus", - "once_cell", - "pin-project-lite 0.2.6", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - -[[package]] -name = "async-task" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" - [[package]] name = "async-trait" version = "0.1.50" @@ -600,12 +392,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "atomic-waker" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" - [[package]] name = "attohttpc" version = "0.16.3" @@ -614,6 +400,8 @@ checksum = "fdb8867f378f33f78a811a8eb9bf108ad99430d7aad43315dd9319c827ef6247" dependencies = [ "http", "log", + "native-tls", + "openssl", "serde", "serde_json", "url 2.2.2", @@ -758,20 +546,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "blocking" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" -dependencies = [ - "async-channel", - "async-task", - "atomic-waker", - "fastrand", - "futures-lite", - "once_cell", -] - [[package]] name = "blowfish" version = "0.6.0" @@ -851,12 +625,6 @@ dependencies = [ "bytes 1.0.1", ] -[[package]] -name = "cache-padded" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" - [[package]] name = "cc" version = "1.0.67" @@ -889,24 +657,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "cipher" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" -dependencies = [ - "generic-array", -] - -[[package]] -name = "concurrent-queue" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" -dependencies = [ - "cache-padded", -] - [[package]] name = "const_fn" version = "0.4.7" @@ -925,13 +675,7 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" dependencies = [ - "aes-gcm", - "base64 0.13.0", - "hkdf", - "hmac", "percent-encoding 2.1.0", - "rand 0.8.3", - "sha2", "time 0.2.26", "version_check 0.9.3", ] @@ -967,12 +711,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cpuid-bool" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" - [[package]] name = "crc" version = "1.8.1" @@ -1032,35 +770,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ctor" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" -dependencies = [ - "quote", - "syn", -] - -[[package]] -name = "ctr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" -dependencies = [ - "cipher", -] - -[[package]] -name = "dashmap" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" -dependencies = [ - "cfg-if 1.0.0", - "num_cpus", -] - [[package]] name = "data-encoding" version = "2.3.2" @@ -1206,21 +915,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "event-listener" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" - -[[package]] -name = "fastrand" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b705829d1e87f762c2df6da140b26af5839e1033aa84aa5f56bb688e4e1bdb" -dependencies = [ - "instant", -] - [[package]] name = "flate2" version = "1.0.20" @@ -1340,21 +1034,6 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" -[[package]] -name = "futures-lite" -version = "1.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4481d0cd0de1d204a4fa55e7d45f07b1d958abcb06714b3446438e2eff695fb" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite 0.2.6", - "waker-fn", -] - [[package]] name = "futures-macro" version = "0.3.15" @@ -1442,29 +1121,6 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] -[[package]] -name = "ghash" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" -dependencies = [ - "opaque-debug", - "polyval", -] - -[[package]] -name = "gloo-timers" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "h2" version = "0.2.7" @@ -1480,11 +1136,30 @@ dependencies = [ "indexmap", "slab", "tokio 0.2.25", - "tokio-util", + "tokio-util 0.3.1", "tracing", "tracing-futures", ] +[[package]] +name = "h2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" +dependencies = [ + "bytes 1.0.1", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 1.6.0", + "tokio-util 0.6.7", + "tracing", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -1527,16 +1202,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hkdf" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" -dependencies = [ - "digest", - "hmac", -] - [[package]] name = "hmac" version = "0.10.1" @@ -1571,52 +1236,13 @@ dependencies = [ [[package]] name = "http-body" -version = "0.3.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "http", -] - -[[package]] -name = "http-client" -version = "6.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce318d86a47d18d1db645c979214f809a6cd625202ad334ef75ca813b30dac80" -dependencies = [ - "async-trait", - "cfg-if 1.0.0", - "dashmap", - "futures-util", - "http-types", - "hyper", - "hyper-tls", - "log", - "tokio 0.2.25", -] - -[[package]] -name = "http-types" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad077d89137cd3debdce53c66714dc536525ef43fe075d41ddc0a8ac11f85957" -dependencies = [ - "anyhow", - "async-channel", - "async-std", - "base64 0.13.0", - "cookie", - "futures-lite", - "http", - "infer", "pin-project-lite 0.2.6", - "rand 0.7.3", - "serde", - "serde_json", - "serde_qs", - "serde_urlencoded", - "url 2.2.2", ] [[package]] @@ -1627,9 +1253,9 @@ checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" [[package]] name = "httpdate" -version = "0.3.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" +checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9" [[package]] name = "humantime" @@ -1642,23 +1268,23 @@ dependencies = [ [[package]] name = "hyper" -version = "0.13.10" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" +checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.3", "http", "http-body", "httparse", "httpdate", "itoa", "pin-project 1.0.7", - "socket2 0.3.19", - "tokio 0.2.25", + "socket2 0.4.0", + "tokio 1.6.0", "tower-service", "tracing", "want", @@ -1666,15 +1292,15 @@ dependencies = [ [[package]] name = "hyper-tls" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "hyper", "native-tls", - "tokio 0.2.25", - "tokio-tls", + "tokio 1.6.0", + "tokio-native-tls", ] [[package]] @@ -1709,12 +1335,6 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "infer" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" - [[package]] name = "instant" version = "0.1.9" @@ -1742,9 +1362,15 @@ dependencies = [ "socket2 0.3.19", "widestring", "winapi 0.3.9", - "winreg", + "winreg 0.6.2", ] +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + [[package]] name = "itoa" version = "0.4.7" @@ -1785,15 +1411,6 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - [[package]] name = "language-tags" version = "0.2.2" @@ -1862,7 +1479,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ "cfg-if 1.0.0", - "value-bag", ] [[package]] @@ -1942,6 +1558,15 @@ dependencies = [ "unicase", ] +[[package]] +name = "minidom" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e" +dependencies = [ + "quick-xml", +] + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -2223,12 +1848,6 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "parking" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" - [[package]] name = "parking_lot" version = "0.11.1" @@ -2341,30 +1960,6 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" -[[package]] -name = "polling" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fc12d774e799ee9ebae13f4076ca003b40d18a11ac0f3641e6f899618580b7b" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "log", - "wepoll-sys", - "winapi 0.3.9", -] - -[[package]] -name = "polyval" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" -dependencies = [ - "cpuid-bool", - "opaque-debug", - "universal-hash", -] - [[package]] name = "ppv-lite86" version = "0.2.10" @@ -2398,6 +1993,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.9" @@ -2576,6 +2180,41 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "reqwest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" +dependencies = [ + "base64 0.13.0", + "bytes 1.0.1", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding 2.1.0", + "pin-project-lite 0.2.6", + "serde", + "serde_json", + "serde_urlencoded", + "tokio 1.6.0", + "tokio-native-tls", + "url 2.2.2", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.7.0", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -2633,12 +2272,11 @@ dependencies = [ [[package]] name = "rust-s3" -version = "0.27.0-beta5" +version = "0.27.0-rc4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa667e2c2460cad8e10d8d20ae32b2d8ed6e965e22418d6a984a594871013d2" +checksum = "5c93272c1d654d492f8ab30b94cd43d98f2700b1db55b2576aff7712ce40e3ef" dependencies = [ "anyhow", - "async-std", "async-trait", "aws-creds", "aws-region", @@ -2652,12 +2290,15 @@ dependencies = [ "log", "maybe-async", "md5", + "minidom", "percent-encoding 2.1.0", + "reqwest", "serde", "serde-xml-rs", "serde_derive", "sha2", - "surf", + "tokio 1.6.0", + "tokio-stream", "url 2.2.2", ] @@ -2774,18 +2415,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_qs" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5af82de3c6549b001bec34961ff2d6a54339a87bab37ce901b693401f27de6cb" -dependencies = [ - "data-encoding", - "percent-encoding 2.1.0", - "serde", - "thiserror", -] - [[package]] name = "serde_urlencoded" version = "0.7.0" @@ -2830,16 +2459,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "signal-hook" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef33d6d0cd06e0840fba9985aab098c147e67e05cee14d412d3345ed14ff30ac" -dependencies = [ - "libc", - "signal-hook-registry", -] - [[package]] name = "signal-hook-registry" version = "1.3.0" @@ -2966,6 +2585,7 @@ dependencies = [ "sqlx-rt", "stringprep", "thiserror", + "tokio-stream", "url 2.2.2", "whoami", ] @@ -2999,9 +2619,11 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6ae97ab05063ed515cdc23d90253213aa24dda0a288c5ec079af3d10f9771bc" dependencies = [ - "async-native-tls", - "async-std", + "actix-rt 2.2.0", "native-tls", + "once_cell", + "tokio 1.6.0", + "tokio-native-tls", ] [[package]] @@ -3079,7 +2701,6 @@ dependencies = [ "actix-utils 3.0.0", "actix-web", "anyhow", - "async-std", "base64 0.13.0", "bcrypt", "chrono", @@ -3107,6 +2728,7 @@ dependencies = [ "sqlx", "tempdir", "time 0.2.26", + "tokio 1.6.0", "toml", ] @@ -3126,26 +2748,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" -[[package]] -name = "surf" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a154d33ca6b5e1fe6fd1c760e5a5cc1202425f6cca2e13229f16a69009f6328" -dependencies = [ - "async-std", - "async-trait", - "cfg-if 1.0.0", - "futures-util", - "http-client", - "http-types", - "log", - "mime_guess", - "once_cell", - "pin-project-lite 0.2.6", - "serde", - "serde_json", -] - [[package]] name = "syn" version = "1.0.72" @@ -3308,7 +2910,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" dependencies = [ "bytes 0.5.6", - "fnv", "futures-core", "iovec", "lazy_static", @@ -3324,28 +2925,54 @@ dependencies = [ [[package]] name = "tokio" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" +checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37" dependencies = [ "autocfg 1.0.1", + "bytes 1.0.1", "libc", + "memchr", "mio 0.7.11", + "num_cpus", "once_cell", "parking_lot", "pin-project-lite 0.2.6", "signal-hook-registry", + "tokio-macros", "winapi 0.3.9", ] [[package]] -name = "tokio-tls" -version = "0.3.1" +name = "tokio-macros" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" +checksum = "c49e3df43841dafb86046472506755d8501c5615673955f6aa17181125d13c37" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 0.2.25", + "tokio 1.6.0", +] + +[[package]] +name = "tokio-stream" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8864d706fdb3cc0843a49647ac892720dac98a6eeb818b77190592cf4994066" +dependencies = [ + "futures-core", + "pin-project-lite 0.2.6", + "tokio 1.6.0", ] [[package]] @@ -3362,6 +2989,20 @@ dependencies = [ "tokio 0.2.25", ] +[[package]] +name = "tokio-util" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +dependencies = [ + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.6", + "tokio 1.6.0", +] + [[package]] name = "toml" version = "0.5.8" @@ -3504,16 +3145,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "universal-hash" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "untrusted" version = "0.7.1" @@ -3541,7 +3172,6 @@ dependencies = [ "idna 0.2.3", "matches", "percent-encoding 2.1.0", - "serde", ] [[package]] @@ -3576,15 +3206,6 @@ dependencies = [ "v_escape", ] -[[package]] -name = "value-bag" -version = "1.0.0-alpha.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b676010e055c99033117c2343b33a40a30b91fecd6c49055ac9cd2d6c305ab1" -dependencies = [ - "ctor", -] - [[package]] name = "vcpkg" version = "0.2.12" @@ -3603,12 +3224,6 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" -[[package]] -name = "waker-fn" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" - [[package]] name = "want" version = "0.3.0" @@ -3638,6 +3253,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if 1.0.0", + "serde", + "serde_json", "wasm-bindgen-macro", ] @@ -3707,15 +3324,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "wepoll-sys" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff" -dependencies = [ - "cc", -] - [[package]] name = "whoami" version = "1.1.2" @@ -3790,6 +3398,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "ws2_32-sys" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 9a6fc5b4f1..c2cf117467 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,6 @@ path = "src/w2l_inference_job.rs" [dependencies] anyhow = "1.0.40" -async-std = { version = "1.6", features = [ "attributes" ] } base64 = "0.13.0" chrono = { version = "0.4.19", features = ["serde"] } derive_more = "0.99.14" @@ -47,17 +46,23 @@ rand = "0.8.3" regex = "1.5.4" time = "0.2.26" +# Runtime +#async-std = { version = "1.6", features = [ "attributes" ] } +tokio = { version = "1.6.0", features = ["macros"] } + # Database # "offline" supports typesafe builds without a live database # "rustls" doesn't appear to work with IP addresses, per: # https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 -sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } +#sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } +sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } # Files data-encoding = "2.3.2" ring = "0.17.0-alpha.10" -#rust-s3 = "0.27.0-rc4" -rust-s3 = { version = "=0.27.0-beta5", features = ["with-async-std"], default-features = false } +rust-s3 = "0.27.0-rc4" +#rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } +#rust-s3 = { version = "0.26" } tempdir = "0.3.7" diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index c16146587e..1bdb406243 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -48,7 +48,7 @@ struct Downloader { pub download_script: String, } -#[async_std::main] +#[tokio::main] async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); @@ -255,16 +255,26 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("File hash: {}", private_bucket_hash); - let object_name = format!( - "user_uploaded_tts_models/{}/{}/{}/{}.bin", + /*let object_name = format!( + "/user_uploaded_tts_models/{}/{}/{}/{}.bin", &private_bucket_hash[0..1], &private_bucket_hash[1..2], &private_bucket_hash[2..3], &private_bucket_hash, - ); + );*/ + + //"/dev-vocodes-private-uploads/user_uploaded_tts_models/foobarbaz.bin", + //"/dev-vocodes-private-uploads/foobarbaz.bin", + //"/user_uploaded_tts_models/foobarbaz", + + // [2021-05-23T08:38:48Z DEBUG hyper::proto::h1::conn] incoming body is empty + // [2021-05-23T08:38:48Z DEBUG hyper::client::pool] pooling idle connection + // for ("https", dev-vocodes-private-uploads.storage.googleapis.com) + // [2021-05-23T08:38:48Z DEBUG reqwest::async_impl::client] response '200 OK' + // for https://dev-vocodes-private-uploads.storage.googleapis.com/95497399bf361e032cee4e180ee605b9dd5e1baa5ff31faefa42119fe19a6664 + // [2021-05-23T08:38:48Z INFO tts_download_job::util::bucket_client] upload code: 200 let object_name = format!( - "{}.bin", - &private_bucket_hash, + "/{}", private_bucket_hash ); info!("Destination bucket path: {}", object_name); diff --git a/src/util/bucket_client.rs b/src/util/bucket_client.rs index 87b709dd0e..5728ad557e 100644 --- a/src/util/bucket_client.rs +++ b/src/util/bucket_client.rs @@ -1,12 +1,13 @@ use anyhow::bail; use log::info; +use log::warn; use s3::bucket::Bucket; use s3::creds::Credentials; use s3::region::Region; -use std::path::{PathBuf, Path}; -use std::str::FromStr; use std::fs::File; use std::io::Read; +use std::path::{PathBuf, Path}; +use std::str::FromStr; #[derive(Clone)] pub struct BucketClient { @@ -33,7 +34,10 @@ impl BucketClient { region: region_name.to_owned(), endpoint: "https://storage.googleapis.com".to_owned(), }; - let bucket = Bucket::new(&bucket_name, region, credentials)?; + let mut bucket = Bucket::new(&bucket_name, region, credentials)?; + + //bucket.set_path_style(); + //bucket.set_subdomain_style(); Ok(Self { bucket, @@ -43,13 +47,14 @@ impl BucketClient { pub async fn upload_file(&self, object_name: &str, bytes: &[u8]) -> anyhow::Result<()> { info!("Filename for bucket: {}", object_name); - let (body_bytes, code) = self.bucket.put_object(object_name, bytes).await?; + let (body_bytes, code) = self.bucket.put_object_with_content_type(object_name, bytes, "application/zip").await?; info!("upload code: {}", code); - let body = String::from_utf8(body_bytes)?; - - info!("upload body: {}", body); + if code != 200 { + let body = String::from_utf8(body_bytes)?; + warn!("upload body: {}", body); + } Ok(()) } From 73849e825fe0d6f3c7d39f50f448a82b4c478fc2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 May 2021 12:01:15 -0400 Subject: [PATCH 0100/1042] still not working in subdirs... --- .env | 1 + src/tts_download_job.rs | 4 +++- src/util/bucket_client.rs | 28 ++++++++++++++++++++++------ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 039a586908..2651903499 100644 --- a/.env +++ b/.env @@ -12,3 +12,4 @@ COOKIE_DOMAIN=.jungle.horse # Bucket config (non-secret) REGION_NAME=us-east1 BUCKET_NAME=dev-vocodes-private-uploads +#BUCKET_NAME=vocodes-audio-uploads diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 1bdb406243..965ff0f0c7 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -274,7 +274,9 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho // for https://dev-vocodes-private-uploads.storage.googleapis.com/95497399bf361e032cee4e180ee605b9dd5e1baa5ff31faefa42119fe19a6664 // [2021-05-23T08:38:48Z INFO tts_download_job::util::bucket_client] upload code: 200 let object_name = format!( - "/{}", private_bucket_hash + //"uploads/{}", private_bucket_hash + //"/uploads/{}", private_bucket_hash + "/dev-vocodes-private-uploads/foobarbaz_test.bin", ); info!("Destination bucket path: {}", object_name); diff --git a/src/util/bucket_client.rs b/src/util/bucket_client.rs index 5728ad557e..f9fc97a609 100644 --- a/src/util/bucket_client.rs +++ b/src/util/bucket_client.rs @@ -4,10 +4,12 @@ use log::warn; use s3::bucket::Bucket; use s3::creds::Credentials; use s3::region::Region; -use std::fs::File; use std::io::Read; use std::path::{PathBuf, Path}; use std::str::FromStr; +use tokio::io::{AsyncBufRead, BufReader, AsyncReadExt}; +use tokio::io::AsyncBufReadExt; +use tokio::fs::File; #[derive(Clone)] pub struct BucketClient { @@ -36,8 +38,8 @@ impl BucketClient { }; let mut bucket = Bucket::new(&bucket_name, region, credentials)?; - //bucket.set_path_style(); - //bucket.set_subdomain_style(); + bucket.set_path_style(); + bucket.set_subdomain_style(); Ok(Self { bucket, @@ -47,7 +49,7 @@ impl BucketClient { pub async fn upload_file(&self, object_name: &str, bytes: &[u8]) -> anyhow::Result<()> { info!("Filename for bucket: {}", object_name); - let (body_bytes, code) = self.bucket.put_object_with_content_type(object_name, bytes, "application/zip").await?; + let (body_bytes, code) = self.bucket.put_object(object_name, bytes).await?; info!("upload code: {}", code); @@ -71,10 +73,24 @@ impl BucketClient { // } pub async fn upload_filename(&self, object_name: &str, filename: &Path) -> anyhow::Result<()> { + /*let mut file = File::open(filename).await?; + let mut reader = BufReader::new(file); + + info!("Uploading..."); + + let code = self.bucket.put_object_stream(&mut reader, object_name).await?; + + info!("upload code: {}", code); + + Ok(()) + */ + // TODO: does a newer version of this crate handle streaming/buffering file contents? - let mut file = File::open(filename)?; + let mut file = File::open(filename).await?; let mut buffer : Vec = Vec::new(); - file.read_to_end(&mut buffer)?; + file.read_to_end(&mut buffer).await?; + + info!("Uploading..."); self.upload_file(object_name, &buffer).await } From 50de9cfbf2b16b0c90af63dc3813171987d75ebb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 May 2021 13:26:25 -0400 Subject: [PATCH 0101/1042] file bug report --- src/tts_download_job.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 965ff0f0c7..de7ab470aa 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -276,7 +276,7 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho let object_name = format!( //"uploads/{}", private_bucket_hash //"/uploads/{}", private_bucket_hash - "/dev-vocodes-private-uploads/foobarbaz_test.bin", + "/uploads/foobarbaz_test.bin", ); info!("Destination bucket path: {}", object_name); From c9978cecda13965361db0e112749bd219f01d924 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 May 2021 13:33:01 -0400 Subject: [PATCH 0102/1042] TOKIO DOWNGRADE 1.0 -> 0.2; this fixes Bucket/S3 error --- Cargo.lock | 637 ++++++++++++++++++++++++++++++++++++++--------------- Cargo.toml | 16 +- 2 files changed, 473 insertions(+), 180 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2279ce59e..99739046ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,7 +15,7 @@ dependencies = [ "log", "pin-project 0.4.28", "tokio 0.2.25", - "tokio-util 0.3.1", + "tokio-util", ] [[package]] @@ -97,7 +97,7 @@ dependencies = [ "futures-core", "futures-util", "fxhash", - "h2 0.2.7", + "h2", "http", "httparse", "indexmap", @@ -334,6 +334,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" + [[package]] name = "ahash" version = "0.4.7" @@ -342,9 +348,9 @@ checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" [[package]] name = "ahash" -version = "0.7.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" +checksum = "796540673305a66d127804eef19ad696f1f204b8c1025aaca4958c17eab32877" dependencies = [ "getrandom 0.2.2", "once_cell", @@ -372,6 +378,118 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "async-channel" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9586ec52317f36de58453159d48351bc244bc24ced3effc1fce22f3d48664af6" +dependencies = [ + "async-channel", + "async-executor", + "async-io", + "async-mutex", + "blocking", + "futures-lite", + "num_cpus", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bbfd5cf2794b1e908ea8457e6c45f8f8f1f6ec5f74617bf4662623f47503c3b" +dependencies = [ + "concurrent-queue", + "fastrand", + "futures-lite", + "libc", + "log", + "once_cell", + "parking", + "polling", + "slab", + "socket2 0.4.0", + "waker-fn", + "winapi 0.3.9", +] + +[[package]] +name = "async-lock" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6a8ea61bf9947a1007c5cada31e647dbc77b103c679858150003ba697ea798b" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-mutex" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-std" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f06685bad74e0570f5213741bea82158279a4103d988e57bfada11ad230341" +dependencies = [ + "async-channel", + "async-global-executor", + "async-io", + "async-lock", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "num_cpus", + "once_cell", + "pin-project-lite 0.2.6", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" + [[package]] name = "async-trait" version = "0.1.50" @@ -393,10 +511,16 @@ dependencies = [ ] [[package]] -name = "attohttpc" -version = "0.16.3" +name = "atomic-waker" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb8867f378f33f78a811a8eb9bf108ad99430d7aad43315dd9319c827ef6247" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + +[[package]] +name = "attohttpc" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe174d1b67f7b2bafed829c09db039301eb5841f66e43be2cf60b326e7f8e2cc" dependencies = [ "http", "log", @@ -405,7 +529,6 @@ dependencies = [ "serde", "serde_json", "url 2.2.2", - "wildmatch", ] [[package]] @@ -457,27 +580,27 @@ dependencies = [ [[package]] name = "aws-creds" -version = "0.26.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1331d069460a674d42bd27c12b47ce578f789954c7bd7f239fd030771eca6616" +checksum = "ad53a54cb2c99990e96eefacde6f143dc6d471ab70809d26d360292be421d490" dependencies = [ - "anyhow", "attohttpc", "dirs", "rust-ini", "serde", "serde-xml-rs", "serde_derive", + "simpl", "url 2.2.2", ] [[package]] name = "aws-region" -version = "0.23.2" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2884b8f2aaeb4a4bf80b219b4fe1d340139ca9331679c57e0fd4a24f571a78bd" +checksum = "f610af4a396f07592014dc3410f6ad78fab931852a99bb6cfdc1ad04b9329b80" dependencies = [ - "anyhow", + "simpl", ] [[package]] @@ -546,6 +669,20 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blocking" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", +] + [[package]] name = "blowfish" version = "0.6.0" @@ -625,6 +762,34 @@ dependencies = [ "bytes 1.0.1", ] +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + +[[package]] +name = "cargo-platform" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0226944a63d1bf35a3b5f948dd7c59e263db83695c9e8bffc4037de02e30f1d7" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714a157da7991e23d90686b9524b9e12e0407a108647f52e9328f4b3d51ac7f" +dependencies = [ + "cargo-platform", + "semver 0.11.0", + "semver-parser 0.10.2", + "serde", + "serde_json", +] + [[package]] name = "cc" version = "1.0.67" @@ -657,6 +822,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + [[package]] name = "const_fn" version = "0.4.7" @@ -760,6 +934,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crypto-mac" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58bcd97a54c7ca5ce2f6eb16f6bede5b0ab5f0055fedc17d2f0b4466e21671ca" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "crypto-mac" version = "0.10.0" @@ -770,6 +954,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "ctor" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "data-encoding" version = "2.3.2" @@ -915,6 +1109,21 @@ dependencies = [ "termcolor", ] +[[package]] +name = "event-listener" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" + +[[package]] +name = "fastrand" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77b705829d1e87f762c2df6da140b26af5839e1033aa84aa5f56bb688e4e1bdb" +dependencies = [ + "instant", +] + [[package]] name = "flate2" version = "1.0.20" @@ -1034,6 +1243,21 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" +[[package]] +name = "futures-lite" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4481d0cd0de1d204a4fa55e7d45f07b1d958abcb06714b3446438e2eff695fb" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite 0.2.6", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.15" @@ -1121,6 +1345,19 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] +[[package]] +name = "gloo-timers" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "h2" version = "0.2.7" @@ -1136,28 +1373,19 @@ dependencies = [ "indexmap", "slab", "tokio 0.2.25", - "tokio-util 0.3.1", + "tokio-util", "tracing", "tracing-futures", ] [[package]] -name = "h2" -version = "0.3.3" +name = "hashbrown" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" +checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" dependencies = [ - "bytes 1.0.1", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio 1.6.0", - "tokio-util 0.6.7", - "tracing", + "ahash 0.3.8", + "autocfg 1.0.1", ] [[package]] @@ -1175,7 +1403,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" dependencies = [ - "hashbrown", + "hashbrown 0.9.1", ] [[package]] @@ -1202,13 +1430,23 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deae6d9dbb35ec2c502d62b8f7b1c000a0822c3b0794ba36b3149c0a1c840dff" +dependencies = [ + "crypto-mac 0.9.1", + "digest", +] + [[package]] name = "hmac" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ - "crypto-mac", + "crypto-mac 0.10.0", "digest", ] @@ -1236,13 +1474,12 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" dependencies = [ - "bytes 1.0.1", + "bytes 0.5.6", "http", - "pin-project-lite 0.2.6", ] [[package]] @@ -1253,9 +1490,9 @@ checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" [[package]] name = "httpdate" -version = "1.0.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" [[package]] name = "humantime" @@ -1268,23 +1505,23 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.7" +version = "0.13.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54" +checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" dependencies = [ - "bytes 1.0.1", + "bytes 0.5.6", "futures-channel", "futures-core", "futures-util", - "h2 0.3.3", + "h2", "http", "http-body", "httparse", "httpdate", "itoa", "pin-project 1.0.7", - "socket2 0.4.0", - "tokio 1.6.0", + "socket2 0.3.19", + "tokio 0.2.25", "tower-service", "tracing", "want", @@ -1292,15 +1529,15 @@ dependencies = [ [[package]] name = "hyper-tls" -version = "0.5.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" dependencies = [ - "bytes 1.0.1", + "bytes 0.5.6", "hyper", "native-tls", - "tokio 1.6.0", - "tokio-native-tls", + "tokio 0.2.25", + "tokio-tls", ] [[package]] @@ -1332,7 +1569,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg 1.0.1", - "hashbrown", + "hashbrown 0.9.1", ] [[package]] @@ -1393,9 +1630,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86e46349d67dc03bdbdb28da0337a355a53ca1d5156452722c36fe21d0e6389b" dependencies = [ "base64 0.13.0", - "crypto-mac", + "crypto-mac 0.10.0", "digest", - "hmac", + "hmac 0.10.1", "serde", "serde_json", "sha2", @@ -1411,6 +1648,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "language-tags" version = "0.2.2" @@ -1479,6 +1725,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ "cfg-if 1.0.0", + "value-bag", ] [[package]] @@ -1508,17 +1755,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-async" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3919783e441e38b0512fae9eebd87c445c42e39508231fba2a8b9fd7ba88e999" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "md-5" version = "0.9.1" @@ -1558,15 +1794,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "minidom" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e" -dependencies = [ - "quick-xml", -] - [[package]] name = "miniz_oxide" version = "0.4.4" @@ -1712,6 +1939,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.3.2" @@ -1723,22 +1961,11 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-bigint" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d047c1062aa51e256408c560894e5251f08925980e53cf1aa5bd00eec6512" -dependencies = [ - "autocfg 1.0.1", - "num-integer", - "num-traits", -] - [[package]] name = "num-bigint-dig" -version = "0.7.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4547ee5541c18742396ae2c895d0717d0f886d8823b8399cdaf7b07d63ad0480" +checksum = "5d51546d704f52ef14b3c962b5776e53d5b862e5790e40a350d366c209bd7f7a" dependencies = [ "autocfg 0.1.7", "byteorder", @@ -1747,7 +1974,8 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.3", + "rand 0.7.3", + "serde", "smallvec", "zeroize", ] @@ -1780,7 +2008,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ "autocfg 1.0.1", - "libm", ] [[package]] @@ -1840,14 +2067,20 @@ dependencies = [ [[package]] name = "ordered-multimap" -version = "0.3.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485" +checksum = "e88f947c6799d5eff50e6cf8a2365c17ac4aa8f8f43aceeedc29b616d872a358" dependencies = [ "dlv-list", - "hashbrown", + "hashbrown 0.7.2", ] +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.11.1" @@ -1896,6 +2129,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + [[package]] name = "pin-project" version = "0.4.28" @@ -1960,6 +2202,19 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +[[package]] +name = "polling" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc12d774e799ee9ebae13f4076ca003b40d18a11ac0f3641e6f899618580b7b" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "log", + "wepoll-sys", + "winapi 0.3.9", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1993,15 +2248,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-xml" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" -dependencies = [ - "memchr", -] - [[package]] name = "quote" version = "1.0.9" @@ -2182,12 +2428,12 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.3" +version = "0.10.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" +checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" dependencies = [ "base64 0.13.0", - "bytes 1.0.1", + "bytes 0.5.6", "encoding_rs", "futures-core", "futures-util", @@ -2200,14 +2446,15 @@ dependencies = [ "lazy_static", "log", "mime", + "mime_guess", "native-tls", "percent-encoding 2.1.0", "pin-project-lite 0.2.6", "serde", "serde_json", "serde_urlencoded", - "tokio 1.6.0", - "tokio-native-tls", + "tokio 0.2.25", + "tokio-tls", "url 2.2.2", "wasm-bindgen", "wasm-bindgen-futures", @@ -2242,9 +2489,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.4.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ef841a26fc5d040ced0417c6c6a64ee851f42489df11cdf0218e545b6f8d28" +checksum = "3648b669b10afeab18972c105e284a7b953a669b0be3514c27f9b17acab2f9cd" dependencies = [ "byteorder", "digest", @@ -2254,17 +2501,19 @@ dependencies = [ "num-iter", "num-traits", "pem", - "rand 0.8.3", + "rand 0.7.3", + "sha2", "simple_asn1", "subtle", + "thiserror", "zeroize", ] [[package]] name = "rust-ini" -version = "0.16.1" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55b134767a87e0b086f73a4ce569ac9ce7d202f39c8eab6caa266e2617e73ac6" +checksum = "7a3679dd538c876a7b606f3bb951c8a20fc281a0ff7795f59f7cb490e3f979e1" dependencies = [ "cfg-if 0.1.10", "ordered-multimap", @@ -2272,12 +2521,11 @@ dependencies = [ [[package]] name = "rust-s3" -version = "0.27.0-rc4" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c93272c1d654d492f8ab30b94cd43d98f2700b1db55b2576aff7712ce40e3ef" +checksum = "a04bdd0f5118d06ef0e2daa658a9e5135282bcecfd6c46b11fddf47b1d5d736d" dependencies = [ - "anyhow", - "async-trait", + "async-std", "aws-creds", "aws-region", "base64 0.13.0", @@ -2285,21 +2533,20 @@ dependencies = [ "chrono", "futures", "hex", - "hmac", + "hmac 0.9.0", "http", "log", - "maybe-async", "md5", - "minidom", "percent-encoding 2.1.0", "reqwest", "serde", "serde-xml-rs", "serde_derive", "sha2", - "tokio 1.6.0", - "tokio-stream", + "simpl", + "tokio 0.2.25", "url 2.2.2", + "uuid", ] [[package]] @@ -2308,7 +2555,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", ] [[package]] @@ -2362,7 +2609,17 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.2", + "serde", ] [[package]] @@ -2371,6 +2628,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + [[package]] name = "serde" version = "1.0.125" @@ -2469,15 +2735,20 @@ dependencies = [ ] [[package]] -name = "simple_asn1" -version = "0.5.2" +name = "simpl" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0e9076e5242ff5a58e854cb478ea9caebce01088f86d3d9c6ad336b7655263" +checksum = "2a30f10c911c0355f80f1c2faa8096efc4a58cdf8590b954d5b395efa071c711" + +[[package]] +name = "simple_asn1" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692ca13de57ce0613a363c8c2f1de925adebc81b04c923ac60c5488bb44abe4b" dependencies = [ "chrono", - "num-bigint 0.4.0", + "num-bigint 0.2.6", "num-traits", - "thiserror", ] [[package]] @@ -2534,9 +2805,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.5.2" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d582b9bc04ec6c03084196efc42c2226b018e9941f03ee62bd88921d500917c0" +checksum = "e1a98f9bf17b690f026b6fec565293a995b46dfbd6293debcb654dcffd2d1b34" dependencies = [ "sqlx-core", "sqlx-macros", @@ -2544,16 +2815,16 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.5.2" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de52d1d473cebb2abb79c886ef6a8023e965e34c0676a99cfeac2cc7f0fde4c1" +checksum = "36bb6a2ca3345a86493bc3b71eabc2c6c16a8bb1aa476cf5303bee27f67627d7" dependencies = [ - "ahash 0.7.2", + "ahash 0.6.3", "atoi", "base64 0.13.0", "bitflags", "byteorder", - "bytes 1.0.1", + "bytes 0.5.6", "chrono", "crc", "crossbeam-channel", @@ -2575,7 +2846,7 @@ dependencies = [ "once_cell", "parking_lot", "percent-encoding 2.1.0", - "rand 0.8.3", + "rand 0.7.3", "rsa", "serde", "sha-1", @@ -2585,23 +2856,23 @@ dependencies = [ "sqlx-rt", "stringprep", "thiserror", - "tokio-stream", "url 2.2.2", "whoami", ] [[package]] name = "sqlx-macros" -version = "0.5.2" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a40f0be97e704d3fbf059e7e3333c3735639146a72d586c5534c70e79da88a4" +checksum = "2b5ada8b3b565331275ce913368565a273a74faf2a34da58c4dc010ce3286844" dependencies = [ + "cargo_metadata", "dotenv", "either", "futures", "heck", "hex", - "once_cell", + "lazy_static", "proc-macro2", "quote", "serde", @@ -2615,14 +2886,13 @@ dependencies = [ [[package]] name = "sqlx-rt" -version = "0.5.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6ae97ab05063ed515cdc23d90253213aa24dda0a288c5ec079af3d10f9771bc" +checksum = "63fc5454c9dd7aaea3a0eeeb65ca40d06d0d8e7413a8184f7c3a3ffa5056190b" dependencies = [ - "actix-rt 2.2.0", "native-tls", "once_cell", - "tokio 1.6.0", + "tokio 0.2.25", "tokio-native-tls", ] @@ -2711,7 +2981,7 @@ dependencies = [ "easyenv", "futures", "hex", - "hmac", + "hmac 0.10.1", "hostname", "jwt", "lazy_static", @@ -2728,7 +2998,7 @@ dependencies = [ "sqlx", "tempdir", "time 0.2.26", - "tokio 1.6.0", + "tokio 0.2.25", "toml", ] @@ -2910,6 +3180,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" dependencies = [ "bytes 0.5.6", + "fnv", "futures-core", "iovec", "lazy_static", @@ -2917,9 +3188,11 @@ dependencies = [ "memchr", "mio 0.6.23", "mio-uds", + "num_cpus", "pin-project-lite 0.1.12", "signal-hook-registry", "slab", + "tokio-macros", "winapi 0.3.9", ] @@ -2930,24 +3203,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37" dependencies = [ "autocfg 1.0.1", - "bytes 1.0.1", "libc", - "memchr", "mio 0.7.11", - "num_cpus", "once_cell", "parking_lot", "pin-project-lite 0.2.6", "signal-hook-registry", - "tokio-macros", "winapi 0.3.9", ] [[package]] name = "tokio-macros" -version = "1.2.0" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c49e3df43841dafb86046472506755d8501c5615673955f6aa17181125d13c37" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" dependencies = [ "proc-macro2", "quote", @@ -2956,23 +3225,22 @@ dependencies = [ [[package]] name = "tokio-native-tls" -version = "0.3.0" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "cd608593a919a8e05a7d1fc6df885e40f6a88d3a70a3a7eff23ff27964eda069" dependencies = [ "native-tls", - "tokio 1.6.0", + "tokio 0.2.25", ] [[package]] -name = "tokio-stream" -version = "0.1.6" +name = "tokio-tls" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8864d706fdb3cc0843a49647ac892720dac98a6eeb818b77190592cf4994066" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" dependencies = [ - "futures-core", - "pin-project-lite 0.2.6", - "tokio 1.6.0", + "native-tls", + "tokio 0.2.25", ] [[package]] @@ -2989,20 +3257,6 @@ dependencies = [ "tokio 0.2.25", ] -[[package]] -name = "tokio-util" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" -dependencies = [ - "bytes 1.0.1", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.2.6", - "tokio 1.6.0", -] - [[package]] name = "toml" version = "0.5.8" @@ -3100,6 +3354,12 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + [[package]] name = "unicase" version = "2.6.0" @@ -3174,6 +3434,12 @@ dependencies = [ "percent-encoding 2.1.0", ] +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" + [[package]] name = "v_escape" version = "0.15.0" @@ -3206,6 +3472,16 @@ dependencies = [ "v_escape", ] +[[package]] +name = "value-bag" +version = "1.0.0-alpha.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd320e1520f94261153e96f7534476ad869c14022aee1e59af7c778075d840ae" +dependencies = [ + "ctor", + "version_check 0.9.3", +] + [[package]] name = "vcpkg" version = "0.2.12" @@ -3224,6 +3500,12 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "want" version = "0.3.0" @@ -3324,6 +3606,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wepoll-sys" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff" +dependencies = [ + "cc", +] + [[package]] name = "whoami" version = "1.1.2" @@ -3340,12 +3631,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" -[[package]] -name = "wildmatch" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f44b95f62d34113cf558c93511ac93027e03e9c29a60dd0fd70e6e025c7270a" - [[package]] name = "winapi" version = "0.2.8" diff --git a/Cargo.toml b/Cargo.toml index c2cf117467..0897048b18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,22 +47,30 @@ regex = "1.5.4" time = "0.2.26" # Runtime +# NB(1): We switched to Tokio from `async-std` for broader ecosystem support. #async-std = { version = "1.6", features = [ "attributes" ] } -tokio = { version = "1.6.0", features = ["macros"] } +# NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: +# https://github.com/durch/rust-s3/issues/198 +#tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "0.2.25", features = ["macros"] } # Database # "offline" supports typesafe builds without a live database # "rustls" doesn't appear to work with IP addresses, per: # https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 #sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } -sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } +# NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade +#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } +sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } # Files data-encoding = "2.3.2" ring = "0.17.0-alpha.10" -rust-s3 = "0.27.0-rc4" +# NB: This is the source of all of our problems. `rust-s3` can't upload to object names with directories +# in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2 +#rust-s3 = "0.27.0-rc4" #rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } -#rust-s3 = { version = "0.26" } +rust-s3 = { version = "0.26" } tempdir = "0.3.7" From a7e47951e69d8e925d2fa06ca213a7c4bbc93f59 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 May 2021 13:35:50 -0400 Subject: [PATCH 0103/1042] working path --- src/tts_download_job.rs | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index de7ab470aa..5cbd7659c8 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -255,28 +255,12 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("File hash: {}", private_bucket_hash); - /*let object_name = format!( + let object_name = format!( "/user_uploaded_tts_models/{}/{}/{}/{}.bin", &private_bucket_hash[0..1], &private_bucket_hash[1..2], &private_bucket_hash[2..3], &private_bucket_hash, - );*/ - - //"/dev-vocodes-private-uploads/user_uploaded_tts_models/foobarbaz.bin", - //"/dev-vocodes-private-uploads/foobarbaz.bin", - //"/user_uploaded_tts_models/foobarbaz", - - // [2021-05-23T08:38:48Z DEBUG hyper::proto::h1::conn] incoming body is empty - // [2021-05-23T08:38:48Z DEBUG hyper::client::pool] pooling idle connection - // for ("https", dev-vocodes-private-uploads.storage.googleapis.com) - // [2021-05-23T08:38:48Z DEBUG reqwest::async_impl::client] response '200 OK' - // for https://dev-vocodes-private-uploads.storage.googleapis.com/95497399bf361e032cee4e180ee605b9dd5e1baa5ff31faefa42119fe19a6664 - // [2021-05-23T08:38:48Z INFO tts_download_job::util::bucket_client] upload code: 200 - let object_name = format!( - //"uploads/{}", private_bucket_hash - //"/uploads/{}", private_bucket_hash - "/uploads/foobarbaz_test.bin", ); info!("Destination bucket path: {}", object_name); From 366bba97cbb3b5b3a47093b0db964ed0bd0d7d4e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 May 2021 13:43:50 -0400 Subject: [PATCH 0104/1042] store object name --- migrations/2021-05-15-071601_assets/up.sql | 6 ++++++ src/job/job_queries.rs | 7 +++++-- src/tts_download_job.rs | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 34596b98a8..c680f0cd16 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -79,6 +79,8 @@ CREATE TABLE tts_models ( -- For now, this will be a hash of the file contents. -- NB: NOT UNIQUE! We can allow duplicate uploads. private_bucket_hash CHAR(64) NOT NULL, + -- The "full url" version of the path + private_bucket_object_name VARCHAR(255) NOT NULL, -- Calculated average, on a scale of 0-100 -- Null with zero ratings. @@ -188,9 +190,13 @@ CREATE TABLE w2l_templates ( -- The original source image/video and the "precomputed" faces private_bucket_hash CHAR(64) NOT NULL, + -- The "full url" version of the path + private_bucket_object_name VARCHAR(255) NOT NULL, -- For the thumbnail we show. public_bucket_hash CHAR(64) NOT NULL, + -- The "full url" version of the path + public_bucket_object_name VARCHAR(255) NOT NULL, -- Calculated average, on a scale of 0-100 -- Null with zero ratings. diff --git a/src/job/job_queries.rs b/src/job/job_queries.rs index 91cb43449c..78e6fa2203 100644 --- a/src/job/job_queries.rs +++ b/src/job/job_queries.rs @@ -111,7 +111,8 @@ WHERE id = ? pub async fn insert_tts_model(pool: &MySqlPool, job: &TtsUploadJobRecord, - private_bucket_hash: &str) + private_bucket_hash: &str, + private_bucket_object_name: &str) -> AnyhowResult { let model_token = random_token(32); @@ -130,7 +131,8 @@ SET creator_user_token = ?, creator_ip_address = ?, original_download_url = ?, - private_bucket_hash = ? + private_bucket_hash = ?, + private_bucket_object_name = ? "#, model_token, updatable_slug, @@ -139,6 +141,7 @@ SET job.creator_ip_address.clone(), job.download_url.clone(), private_bucket_hash.to_string(), + private_bucket_object_name.to_string() ) .execute(pool) .await; diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 5cbd7659c8..61d7522662 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -269,7 +269,7 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho downloader.bucket_client.upload_filename(&object_name, &file_path).await?; info!("Saving model record..."); - let id = insert_tts_model(&downloader.mysql_pool, job, &private_bucket_hash).await?; + let id = insert_tts_model(&downloader.mysql_pool, job, &private_bucket_hash, &object_name).await?; info!("Saved model record: {}", id); From 5d132c8277c8def1a881be0e225ea38a111e5a6a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 00:42:06 -0400 Subject: [PATCH 0105/1042] add more bucket stuff, w2l/tts upload jobs --- src/{util => buckets}/bucket_client.rs | 21 +- src/buckets/bucket_paths.rs | 51 +++++ src/buckets/mod.rs | 2 + src/main.rs | 1 + src/tts_download_job.rs | 25 ++- src/util/mod.rs | 1 - src/w2l_download_job.rs | 284 ++++++++++++++++++++++++- 7 files changed, 373 insertions(+), 12 deletions(-) rename src/{util => buckets}/bucket_client.rs (83%) create mode 100644 src/buckets/bucket_paths.rs create mode 100644 src/buckets/mod.rs diff --git a/src/util/bucket_client.rs b/src/buckets/bucket_client.rs similarity index 83% rename from src/util/bucket_client.rs rename to src/buckets/bucket_client.rs index f9fc97a609..f7915b7e12 100644 --- a/src/util/bucket_client.rs +++ b/src/buckets/bucket_client.rs @@ -14,6 +14,9 @@ use tokio::fs::File; #[derive(Clone)] pub struct BucketClient { bucket: Bucket, + + /// If set, put all files under this root path. + optional_bucket_root: Option, } impl BucketClient { @@ -22,7 +25,8 @@ impl BucketClient { access_key: &str, secret_key: &str, region_name: &str, - bucket_name: &str) -> anyhow::Result + bucket_name: &str, + optional_bucket_root: Option<&str>) -> anyhow::Result { let credentials = Credentials::new( Some(&access_key), @@ -41,15 +45,28 @@ impl BucketClient { bucket.set_path_style(); bucket.set_subdomain_style(); + let optional_bucket_root = optional_bucket_root.map(|s| s.to_string()); + Ok(Self { bucket, + optional_bucket_root, }) } + fn get_rooted_object_name(&self, object_name: &str) -> String { + match &self.optional_bucket_root { + None => object_name.to_string(), + Some(root) => format!("{}/{}", root, object_name), + } + } + pub async fn upload_file(&self, object_name: &str, bytes: &[u8]) -> anyhow::Result<()> { info!("Filename for bucket: {}", object_name); - let (body_bytes, code) = self.bucket.put_object(object_name, bytes).await?; + let object_name = self.get_rooted_object_name(object_name); + info!("Rooted filename for bucket: {}", object_name); + + let (body_bytes, code) = self.bucket.put_object(&object_name, bytes).await?; info!("upload code: {}", code); diff --git a/src/buckets/bucket_paths.rs b/src/buckets/bucket_paths.rs new file mode 100644 index 0000000000..52c1c1fad0 --- /dev/null +++ b/src/buckets/bucket_paths.rs @@ -0,0 +1,51 @@ +use anyhow::anyhow; +use anyhow::bail; +use crate::util::anyhow_result::AnyhowResult; +use std::path::PathBuf; + +pub fn hash_to_bucket_path(file_hash: &str, + optional_root_directory: Option<&str>) + -> AnyhowResult +{ + if file_hash.len() < 4 { + bail!("File length is too short"); + } + + let optional_directory = optional_root_directory + .map(|root| PathBuf::from(root.to_string())) + .unwrap_or(PathBuf::from("/")); + + let file_path = format!( + "{}/{}/{}/{}", + &file_hash[0..1], + &file_hash[1..2], + &file_hash[2..3], + &file_hash, + ); + + let maybe_path = optional_directory + .join(file_path) + .to_str() + .map(|s| s.to_string()); + + maybe_path.ok_or(anyhow!("Path could not be converted")) +} + +#[cfg(test)] +mod tests { + use crate::buckets::bucket_paths::hash_to_bucket_path; + + #[test] + fn test_bucket_without_root() { + assert_eq!( + hash_to_bucket_path("abcdefg", None).unwrap(), + "/a/b/c/abcdefg".to_string()); + } + + #[test] + fn test_bucket_with_root() { + assert_eq!( + hash_to_bucket_path("abcdefg", Some("/root")).unwrap(), + "/root/a/b/c/abcdefg".to_string()); + } +} diff --git a/src/buckets/mod.rs b/src/buckets/mod.rs new file mode 100644 index 0000000000..90dd7c3a20 --- /dev/null +++ b/src/buckets/mod.rs @@ -0,0 +1,2 @@ +pub mod bucket_client; +pub mod bucket_paths; diff --git a/src/main.rs b/src/main.rs index 16196aeff9..d7de84cbd8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ #[macro_use] extern crate lazy_static; #[macro_use] extern crate serde_derive; +pub mod buckets; pub mod common_queries; pub mod database_helpers; pub mod endpoints; diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 61d7522662..6f3d774fa5 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -5,12 +5,13 @@ #![warn(unused_must_use)] //#![allow(warnings)] +pub mod buckets; pub mod job; pub mod util; use anyhow::anyhow; use chrono::Utc; -use crate::util::bucket_client::BucketClient; +use crate::buckets::bucket_client::BucketClient; use crate::job::job_queries::TtsUploadJobRecord; use crate::job::job_queries::insert_tts_model; use crate::job::job_queries::mark_tts_upload_job_done; @@ -19,23 +20,24 @@ use crate::job::job_queries::query_tts_upload_job_records; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::random_token::random_token; +use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; +use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; +use std::fs::File; +use std::io::{BufReader, Read}; use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; use tempdir::TempDir; -use std::fs::File; -use std::io::{BufReader, Read}; -use ring::digest::{Context, Digest, SHA256}; -use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; // Buckets const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; const ENV_REGION_NAME : &'static str = "REGION_NAME"; -const ENV_BUCKET_NAME : &'static str = "BUCKET_NAME"; +const ENV_BUCKET_NAME : &'static str = "TTS_DOWNLOAD_BUCKET_NAME"; +const ENV_BUCKET_ROOT : &'static str = "TTS_DOWNLOAD_BUCKET_ROOT"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; @@ -68,12 +70,14 @@ async fn main() -> AnyhowResult<()> { let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; let bucket_name = easyenv::get_env_string_required(ENV_BUCKET_NAME)?; + let bucket_root = easyenv::get_env_string_required(ENV_BUCKET_ROOT)?; let bucket_client = BucketClient::create( &access_key, &secret_key, ®ion_name, &bucket_name, + Some(&bucket_root), )?; let temp_directory = easyenv::get_env_string_or_default( @@ -255,6 +259,7 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("File hash: {}", private_bucket_hash); + // NB: /.../a/b/c/d/abcdefg.bin let object_name = format!( "/user_uploaded_tts_models/{}/{}/{}/{}.bin", &private_bucket_hash[0..1], @@ -269,7 +274,12 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho downloader.bucket_client.upload_filename(&object_name, &file_path).await?; info!("Saving model record..."); - let id = insert_tts_model(&downloader.mysql_pool, job, &private_bucket_hash, &object_name).await?; + let id = insert_tts_model( + &downloader.mysql_pool, + job, + &private_bucket_hash, + &object_name) + .await?; info!("Saved model record: {}", id); @@ -278,4 +288,3 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho Ok(()) } - diff --git a/src/util/mod.rs b/src/util/mod.rs index 61b5049457..3dda00a986 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,5 +1,4 @@ pub mod anyhow_result; -pub mod bucket_client; pub mod cookie_manager; pub mod filesystem; pub mod ip_address; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 559e546bd4..4974baaf06 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -1,7 +1,289 @@ +#![deny(unused_must_use)] +#![allow(dead_code)] +#![allow(unused_mut)] +#![allow(unused_imports)] +#![warn(unused_must_use)] +//#![allow(warnings)] + +pub mod buckets; +pub mod job; pub mod util; +use anyhow::anyhow; +use chrono::Utc; +use crate::buckets::bucket_client::BucketClient; +use crate::job::job_queries::TtsUploadJobRecord; +use crate::job::job_queries::insert_tts_model; +use crate::job::job_queries::mark_tts_upload_job_done; +use crate::job::job_queries::mark_tts_upload_job_failure; +use crate::job::job_queries::query_tts_upload_job_records; use crate::util::anyhow_result::AnyhowResult; +use crate::util::filesystem::check_directory_exists; +use crate::util::random_token::random_token; +use log::{warn, info}; +use sqlx::MySqlPool; +use sqlx::mysql::MySqlPoolOptions; +use std::path::{PathBuf, Path}; +use std::process::Command; +use std::time::Duration; +use tempdir::TempDir; +use std::fs::File; +use std::io::{BufReader, Read}; +use ring::digest::{Context, Digest, SHA256}; +use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; + +// Buckets +const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; +const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; +const ENV_REGION_NAME : &'static str = "REGION_NAME"; +const ENV_BUCKET_NAME : &'static str = "W2L_DOWNLOAD_BUCKET_NAME"; +const ENV_BUCKET_PATH: &'static str = "W2L_DOWNLOAD_BUCKET_PATH"; + +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; +const DEFAULT_TEMP_DIR: &'static str = "/tmp"; + +struct Downloader { + pub download_temp_directory: PathBuf, + pub mysql_pool: MySqlPool, + pub bucket_client: BucketClient, + // Command to run + pub download_script: String, +} + +#[tokio::main] +async fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + let _ = dotenv::from_filename(".env-secrets").ok(); + + info!("Obtaining hostname..."); + + let server_hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or("tts-download-job".to_string()); + + info!("Hostname: {}", &server_hostname); + + // Bucket stuff + let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; + let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; + let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; + let bucket_name = easyenv::get_env_string_required(ENV_BUCKET_NAME)?; + + let bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &bucket_name, + None, + )?; + + let temp_directory = easyenv::get_env_string_or_default( + "DOWNLOAD_TEMP_DIR", + DEFAULT_TEMP_DIR); + + let download_script = easyenv::get_env_string_or_default( + "DOWNLOAD_SCRIPT", + "./scripts/download_gdrive.py"); + + let temp_directory = PathBuf::from(temp_directory); + + check_directory_exists(&temp_directory)?; + + let db_connection_string = + easyenv::get_env_string_or_default( + "MYSQL_URL", + "mysql://root:root@localhost/storyteller"); + + info!("Connecting to database..."); + + let mysql_pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(&db_connection_string) + .await?; + + let downloader = Downloader { + download_temp_directory: temp_directory, + mysql_pool, + bucket_client, + download_script, + }; + + main_loop(downloader).await; + + Ok(()) +} + +const START_TIMEOUT_MILLIS : u64 = 500; +const INCREASE_TIMEOUT_MILLIS : u64 = 1000; + +async fn main_loop(downloader: Downloader) { + let mut timeout_millis = START_TIMEOUT_MILLIS; + + loop { + let num_records = 1; + let query_result = query_tts_upload_job_records(&downloader.mysql_pool, num_records).await; + + let jobs = match query_result { + Ok(jobs) => jobs, + Err(e) => { + warn!("Error querying jobs: {:?}", e); + std::thread::sleep(Duration::from_millis(timeout_millis)); + timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; + } + }; + + if jobs.is_empty() { + info!("No jobs!"); + std::thread::sleep(Duration::from_millis(1500)); + continue; + } + + let result = process_jobs(&downloader, jobs).await; + + match result { + Ok(_) => {}, + Err(e) => { + warn!("Error querying jobs: {:?}", e); + std::thread::sleep(Duration::from_millis(timeout_millis)); + timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; + } + } + + timeout_millis = START_TIMEOUT_MILLIS; // reset + + std::thread::sleep(Duration::from_millis(500)); + } +} + +async fn process_jobs(downloader: &Downloader, jobs: Vec) -> AnyhowResult<()> { + for job in jobs.into_iter() { + let result = process_job(downloader, &job).await; + match result { + Ok(_) => {}, + Err(e) => { + warn!("Failure to process job: {:?}", e); + let failure_reason = ""; + let _r = mark_tts_upload_job_failure(&downloader.mysql_pool, &job, failure_reason).await; + } + } + } + + Ok(()) +} + +async fn download_file(downloader: &Downloader, + job: &TtsUploadJobRecord, + temp_dir: &TempDir) -> AnyhowResult +{ + + let temp_dir_path = temp_dir.path() + .to_str() + .unwrap_or("/tmp") + .to_string(); + + let filename = random_token(10); + let filename = format!("{}/{}.bin", temp_dir_path, filename); + + info!("Downloading to: {}", filename); + + let url = job.download_url.as_ref() + .map(|c| c.to_string()) + .unwrap_or("".to_string()); + + let command = format!("{} --url {} --output_file {}", + downloader.download_script, + &url, + &filename); + + info!("Running command: {}", command); + + let result = Command::new("sh") + .arg("-c") + .arg(command) + .output()?; + + info!("Downloader Result: {:?}", result); + + if !result.status.success() { + let reason = String::from_utf8(result.stderr).unwrap_or("UNKNOWN".to_string()); + return Err(anyhow!("Failure to execute command: {:?}", reason)) + } + + Ok(filename) +} + + +fn sha256_digest(mut reader: R) -> AnyhowResult { + let mut context = Context::new(&SHA256); + let mut buffer = [0; 1024]; + + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + context.update(&buffer[..count]); + } + + Ok(context.finish()) +} + +fn get_file_hash(filename: &str) -> AnyhowResult { + let input = File::open(filename)?; + let reader = BufReader::new(input); + let digest = sha256_digest(reader)?; + + let hash = HEXLOWER_PERMISSIVE.encode(digest.as_ref()); + Ok(hash) +} + +async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { + // TODO: 1. Mark processing. + // TODO: 2. Download. (DONE) + // TODO: 3. Upload. + // TODO: 4. Save record. (DONE) + // TODO: 5. Mark job done. (DONE) + + let temp_dir = format!("temp_{}", job.id); + let temp_dir = TempDir::new(&temp_dir)?; + + info!("Calling downloader..."); + let download_filename = download_file(downloader, job, &temp_dir).await?; + + let private_bucket_hash = get_file_hash(&download_filename)?; + + info!("File hash: {}", private_bucket_hash); + + // NB: /.../a/b/c/d/abcdefg.bin + let object_name = format!( + "/user_uploaded_tts_models/{}/{}/{}/{}.bin", + &private_bucket_hash[0..1], + &private_bucket_hash[1..2], + &private_bucket_hash[2..3], + &private_bucket_hash, + ); + + info!("Destination bucket path: {}", object_name); + + let file_path = PathBuf::from(download_filename); + downloader.bucket_client.upload_filename(&object_name, &file_path).await?; + + info!("Saving model record..."); + let id = insert_tts_model( + &downloader.mysql_pool, + job, + &private_bucket_hash, + &object_name) + .await?; + + info!("Saved model record: {}", id); + + info!("Job done: {}", job.id); + mark_tts_upload_job_done(&downloader.mysql_pool, job, true).await?; -fn main() -> AnyhowResult<()> { Ok(()) } From 9aae3faab351b16f5c07445dbaeb34b0e0935be6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 00:54:04 -0400 Subject: [PATCH 0106/1042] more common pieces --- .env | 4 ++-- src/buckets/file_hashing.rs | 29 +++++++++++++++++++++++++++ src/buckets/mod.rs | 1 + src/tts_download_job.rs | 40 ++++++++----------------------------- src/w2l_download_job.rs | 34 +++++-------------------------- 5 files changed, 45 insertions(+), 63 deletions(-) create mode 100644 src/buckets/file_hashing.rs diff --git a/.env b/.env index 2651903499..4730549117 100644 --- a/.env +++ b/.env @@ -11,5 +11,5 @@ COOKIE_DOMAIN=.jungle.horse # Bucket config (non-secret) REGION_NAME=us-east1 -BUCKET_NAME=dev-vocodes-private-uploads -#BUCKET_NAME=vocodes-audio-uploads +TTS_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads +TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models diff --git a/src/buckets/file_hashing.rs b/src/buckets/file_hashing.rs new file mode 100644 index 0000000000..13fefee0a5 --- /dev/null +++ b/src/buckets/file_hashing.rs @@ -0,0 +1,29 @@ +use crate::util::anyhow_result::AnyhowResult; +use data_encoding::HEXLOWER_PERMISSIVE; +use ring::digest::{Context, Digest, SHA256}; +use std::fs::File; +use std::io::{Read, BufReader}; + +pub fn get_file_hash(filename: &str) -> AnyhowResult { + let input = File::open(filename)?; + let reader = BufReader::new(input); + let digest = sha256_digest(reader)?; + + let hash = HEXLOWER_PERMISSIVE.encode(digest.as_ref()); + Ok(hash) +} + +fn sha256_digest(mut reader: R) -> AnyhowResult { + let mut context = Context::new(&SHA256); + let mut buffer = [0; 1024]; + + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + context.update(&buffer[..count]); + } + + Ok(context.finish()) +} diff --git a/src/buckets/mod.rs b/src/buckets/mod.rs index 90dd7c3a20..477fc55a78 100644 --- a/src/buckets/mod.rs +++ b/src/buckets/mod.rs @@ -1,2 +1,3 @@ pub mod bucket_client; pub mod bucket_paths; +pub mod file_hashing; diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 6f3d774fa5..149fb8d7c0 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -12,6 +12,8 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; use crate::buckets::bucket_client::BucketClient; +use crate::buckets::bucket_paths::hash_to_bucket_path; +use crate::buckets::file_hashing::get_file_hash; use crate::job::job_queries::TtsUploadJobRecord; use crate::job::job_queries::insert_tts_model; use crate::job::job_queries::mark_tts_upload_job_done; @@ -48,6 +50,8 @@ struct Downloader { pub bucket_client: BucketClient, // Command to run pub download_script: String, + // Root to store TTS results + pub bucket_root_tts_model_uploads: String, } #[tokio::main] @@ -77,7 +81,7 @@ async fn main() -> AnyhowResult<()> { &secret_key, ®ion_name, &bucket_name, - Some(&bucket_root), + None, )?; let temp_directory = easyenv::get_env_string_or_default( @@ -109,6 +113,7 @@ async fn main() -> AnyhowResult<()> { mysql_pool, bucket_client, download_script, + bucket_root_tts_model_uploads: bucket_root.to_string(), }; main_loop(downloader).await; @@ -217,31 +222,6 @@ async fn download_file(downloader: &Downloader, Ok(filename) } - -fn sha256_digest(mut reader: R) -> AnyhowResult { - let mut context = Context::new(&SHA256); - let mut buffer = [0; 1024]; - - loop { - let count = reader.read(&mut buffer)?; - if count == 0 { - break; - } - context.update(&buffer[..count]); - } - - Ok(context.finish()) -} - -fn get_file_hash(filename: &str) -> AnyhowResult { - let input = File::open(filename)?; - let reader = BufReader::new(input); - let digest = sha256_digest(reader)?; - - let hash = HEXLOWER_PERMISSIVE.encode(digest.as_ref()); - Ok(hash) -} - async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. (DONE) @@ -260,13 +240,9 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("File hash: {}", private_bucket_hash); // NB: /.../a/b/c/d/abcdefg.bin - let object_name = format!( - "/user_uploaded_tts_models/{}/{}/{}/{}.bin", - &private_bucket_hash[0..1], - &private_bucket_hash[1..2], - &private_bucket_hash[2..3], + let object_name = hash_to_bucket_path( &private_bucket_hash, - ); + Some(&downloader.bucket_root_tts_model_uploads))?; info!("Destination bucket path: {}", object_name); diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 4974baaf06..26b98b23f6 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -12,6 +12,7 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; use crate::buckets::bucket_client::BucketClient; +use crate::buckets::file_hashing::get_file_hash; use crate::job::job_queries::TtsUploadJobRecord; use crate::job::job_queries::insert_tts_model; use crate::job::job_queries::mark_tts_upload_job_done; @@ -20,17 +21,17 @@ use crate::job::job_queries::query_tts_upload_job_records; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::random_token::random_token; +use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; +use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; +use std::fs::File; +use std::io::{BufReader, Read}; use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; use tempdir::TempDir; -use std::fs::File; -use std::io::{BufReader, Read}; -use ring::digest::{Context, Digest, SHA256}; -use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; // Buckets const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -216,31 +217,6 @@ async fn download_file(downloader: &Downloader, Ok(filename) } - -fn sha256_digest(mut reader: R) -> AnyhowResult { - let mut context = Context::new(&SHA256); - let mut buffer = [0; 1024]; - - loop { - let count = reader.read(&mut buffer)?; - if count == 0 { - break; - } - context.update(&buffer[..count]); - } - - Ok(context.finish()) -} - -fn get_file_hash(filename: &str) -> AnyhowResult { - let input = File::open(filename)?; - let reader = BufReader::new(input); - let digest = sha256_digest(reader)?; - - let hash = HEXLOWER_PERMISSIVE.encode(digest.as_ref()); - Ok(hash) -} - async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. (DONE) From ab946489cf9669e276ac9c0a8d9e825fc32d1a31 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 01:18:56 -0400 Subject: [PATCH 0107/1042] upload w2l (minus face dtection and fancy things) --- .env | 4 + debug/curl/upload_w2l_image_gdrive.py | 45 +++++++ src/job/mod.rs | 3 +- src/job/w2l_download_job_queries.rs | 162 ++++++++++++++++++++++++++ src/tts_download_job.rs | 2 +- src/w2l_download_job.rs | 61 ++++++---- 6 files changed, 252 insertions(+), 25 deletions(-) create mode 100755 debug/curl/upload_w2l_image_gdrive.py create mode 100644 src/job/w2l_download_job_queries.rs diff --git a/.env b/.env index 4730549117..b978eb5bf5 100644 --- a/.env +++ b/.env @@ -11,5 +11,9 @@ COOKIE_DOMAIN=.jungle.horse # Bucket config (non-secret) REGION_NAME=us-east1 + TTS_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models + +W2L_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads +W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates diff --git a/debug/curl/upload_w2l_image_gdrive.py b/debug/curl/upload_w2l_image_gdrive.py new file mode 100755 index 0000000000..aaa9ec65fb --- /dev/null +++ b/debug/curl/upload_w2l_image_gdrive.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +import requests +import re +import uuid + +login_url = 'http://localhost:12345/login' +upload_url = 'http://localhost:12345/w2l/upload' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +# NB: May not be set due to cookie domain: +#r.cookies['session'] + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + + +print("===== Upload W2L =====") + +cookies = { 'session': session_cookie } + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'title': 'this is a new image template', + 'download_url': 'https://drive.google.com/file/d/15-tkgblTZpa0ifmvvyr_kgWy2cguTpAe/view?usp=sharing', +} + +r = requests.post(upload_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + diff --git a/src/job/mod.rs b/src/job/mod.rs index 074f5ee88b..3f5f53b55f 100644 --- a/src/job/mod.rs +++ b/src/job/mod.rs @@ -1 +1,2 @@ -pub mod job_queries; \ No newline at end of file +pub mod job_queries; +pub mod w2l_download_job_queries; \ No newline at end of file diff --git a/src/job/w2l_download_job_queries.rs b/src/job/w2l_download_job_queries.rs new file mode 100644 index 0000000000..7acfe7a262 --- /dev/null +++ b/src/job/w2l_download_job_queries.rs @@ -0,0 +1,162 @@ +//! NB: This seems required for sqlx to generate the cached queries. +//! Sqlx's prepare needs a *single* binary to work against, so we need to +//! include these in the main binary to generate all the queries. + +use anyhow::anyhow; +use chrono::Utc; +use crate::util::anyhow_result::AnyhowResult; +use crate::util::random_token::random_token; +use sqlx::MySqlPool; + +/// table: w2l_template_upload_jobs +#[derive(Debug)] +pub struct W2lTemplateUploadJobRecord { + pub id: i64, + pub uuid_idempotency_token: String, + pub creator_user_token: String, + pub creator_ip_address: String, + pub creator_set_visibility: String, // TODO + pub title: String, + pub template_type: String, // TODO + pub maybe_subject_token: Option, + pub maybe_actor_subject_token: Option, + pub download_url: Option, + pub download_url_type: String, // TODO + pub status: String, // TODO + pub attempt_count: i32, + pub failure_reason: Option, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub retry_at: Option>, +} + +pub async fn query_w2l_template_upload_job_records(pool: &MySqlPool, num_records: u32) + -> AnyhowResult> +{ + let job_records = sqlx::query_as!( + W2lTemplateUploadJobRecord, + r#" +SELECT * +FROM w2l_template_upload_jobs +WHERE + ( + status IN ("pending", "attempt_failed") + ) + AND + ( + retry_at IS NULL + OR + retry_at < CURRENT_TIMESTAMP + ) + LIMIT ? + "#, + num_records, + ) + .fetch_all(pool) + .await?; + + Ok(job_records) +} + +pub async fn mark_w2l_template_upload_job_failure(pool: &MySqlPool, + job: &W2lTemplateUploadJobRecord, + failure_reason: &str) + -> AnyhowResult<()> +{ + // statuses: "attempt_failed", "complete_failure", "dead" + let status = "attempt_failed"; + + let query_result = sqlx::query!( + r#" +UPDATE w2l_template_upload_jobs +SET + status = ?, + failure_reason = ?, + retry_at = NOW() + interval 2 minute +WHERE id = ? + "#, + status, + failure_reason.to_string(), + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} +pub async fn mark_w2l_template_upload_job_done(pool: &MySqlPool, + job: &W2lTemplateUploadJobRecord, + success: bool) + -> AnyhowResult<()> +{ + let status = if success { "complete_success" } else { "complete_failure" }; + + let query_result = sqlx::query!( + r#" +UPDATE w2l_template_upload_jobs +SET + status = ?, + failure_reason = NULL, + retry_at = NULL +WHERE id = ? + "#, + status, + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} + +pub async fn insert_w2l_template(pool: &MySqlPool, + template_type: &str, // TODO: ENUM! + job: &W2lTemplateUploadJobRecord, + private_bucket_hash: &str, + private_bucket_object_name: &str) + -> AnyhowResult +{ + let model_token = random_token(32); + let updatable_slug = model_token.clone(); + + let query_result = sqlx::query!( + r#" +INSERT INTO w2l_templates +SET + token = ?, + template_type = ?, + updatable_slug = ?, + title = ?, + description_markdown = '', + description_rendered_html = '', + creator_user_token = ?, + creator_ip_address = ?, + original_download_url = ?, + private_bucket_hash = ?, + private_bucket_object_name = ? + "#, + model_token, + template_type, + updatable_slug, + job.title.to_string(), + job.creator_user_token.clone(), + job.creator_ip_address.clone(), + job.download_url.clone(), + private_bucket_hash.to_string(), + private_bucket_object_name.to_string() + ) + .execute(pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + Ok(record_id) +} diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 149fb8d7c0..34c68caf0e 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -225,7 +225,7 @@ async fn download_file(downloader: &Downloader, async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. (DONE) - // TODO: 3. Upload. + // TODO: 3. Upload. (DONE) // TODO: 4. Save record. (DONE) // TODO: 5. Mark job done. (DONE) diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 26b98b23f6..6f658f5c2b 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -12,12 +12,13 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; use crate::buckets::bucket_client::BucketClient; +use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::buckets::file_hashing::get_file_hash; -use crate::job::job_queries::TtsUploadJobRecord; -use crate::job::job_queries::insert_tts_model; -use crate::job::job_queries::mark_tts_upload_job_done; -use crate::job::job_queries::mark_tts_upload_job_failure; -use crate::job::job_queries::query_tts_upload_job_records; +use crate::job::w2l_download_job_queries::W2lTemplateUploadJobRecord; +use crate::job::w2l_download_job_queries::insert_w2l_template; +use crate::job::w2l_download_job_queries::mark_w2l_template_upload_job_done; +use crate::job::w2l_download_job_queries::mark_w2l_template_upload_job_failure; +use crate::job::w2l_download_job_queries::query_w2l_template_upload_job_records; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::random_token::random_token; @@ -38,7 +39,7 @@ const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; const ENV_REGION_NAME : &'static str = "REGION_NAME"; const ENV_BUCKET_NAME : &'static str = "W2L_DOWNLOAD_BUCKET_NAME"; -const ENV_BUCKET_PATH: &'static str = "W2L_DOWNLOAD_BUCKET_PATH"; +const ENV_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; @@ -49,6 +50,8 @@ struct Downloader { pub bucket_client: BucketClient, // Command to run pub download_script: String, + // Root to store W2L templates + pub bucket_root_w2l_template_uploads: String, } #[tokio::main] @@ -62,7 +65,7 @@ async fn main() -> AnyhowResult<()> { let server_hostname = hostname::get() .ok() .and_then(|h| h.into_string().ok()) - .unwrap_or("tts-download-job".to_string()); + .unwrap_or("w2l-download-job".to_string()); info!("Hostname: {}", &server_hostname); @@ -71,6 +74,7 @@ async fn main() -> AnyhowResult<()> { let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; let bucket_name = easyenv::get_env_string_required(ENV_BUCKET_NAME)?; + let bucket_root = easyenv::get_env_string_required(ENV_BUCKET_ROOT)?; let bucket_client = BucketClient::create( &access_key, @@ -109,6 +113,7 @@ async fn main() -> AnyhowResult<()> { mysql_pool, bucket_client, download_script, + bucket_root_w2l_template_uploads: bucket_root.to_string(), }; main_loop(downloader).await; @@ -124,7 +129,11 @@ async fn main_loop(downloader: Downloader) { loop { let num_records = 1; - let query_result = query_tts_upload_job_records(&downloader.mysql_pool, num_records).await; + + let query_result = query_w2l_template_upload_job_records( + &downloader.mysql_pool, + num_records) + .await; let jobs = match query_result { Ok(jobs) => jobs, @@ -160,7 +169,7 @@ async fn main_loop(downloader: Downloader) { } } -async fn process_jobs(downloader: &Downloader, jobs: Vec) -> AnyhowResult<()> { +async fn process_jobs(downloader: &Downloader, jobs: Vec) -> AnyhowResult<()> { for job in jobs.into_iter() { let result = process_job(downloader, &job).await; match result { @@ -168,7 +177,11 @@ async fn process_jobs(downloader: &Downloader, jobs: Vec) -> Err(e) => { warn!("Failure to process job: {:?}", e); let failure_reason = ""; - let _r = mark_tts_upload_job_failure(&downloader.mysql_pool, &job, failure_reason).await; + let _r = mark_w2l_template_upload_job_failure( + &downloader.mysql_pool, + &job, + failure_reason) + .await; } } } @@ -177,7 +190,7 @@ async fn process_jobs(downloader: &Downloader, jobs: Vec) -> } async fn download_file(downloader: &Downloader, - job: &TtsUploadJobRecord, + job: &W2lTemplateUploadJobRecord, temp_dir: &TempDir) -> AnyhowResult { @@ -217,12 +230,14 @@ async fn download_file(downloader: &Downloader, Ok(filename) } -async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { +async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. (DONE) - // TODO: 3. Upload. - // TODO: 4. Save record. (DONE) - // TODO: 5. Mark job done. (DONE) + // TODO: 3. Process template with face detection + // TODO: 4. Take a screenshot/gif + // TODO: 5. Upload all (partially done). + // TODO: 6. Save record. (DONE) + // TODO: 7. Mark job done. (DONE) let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; @@ -235,22 +250,22 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("File hash: {}", private_bucket_hash); // NB: /.../a/b/c/d/abcdefg.bin - let object_name = format!( - "/user_uploaded_tts_models/{}/{}/{}/{}.bin", - &private_bucket_hash[0..1], - &private_bucket_hash[1..2], - &private_bucket_hash[2..3], + let object_name = hash_to_bucket_path( &private_bucket_hash, - ); + Some(&downloader.bucket_root_w2l_template_uploads))?; info!("Destination bucket path: {}", object_name); let file_path = PathBuf::from(download_filename); downloader.bucket_client.upload_filename(&object_name, &file_path).await?; + // TODO: + let template_type = "image"; + info!("Saving model record..."); - let id = insert_tts_model( + let id = insert_w2l_template( &downloader.mysql_pool, + template_type, job, &private_bucket_hash, &object_name) @@ -259,7 +274,7 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("Saved model record: {}", id); info!("Job done: {}", job.id); - mark_tts_upload_job_done(&downloader.mysql_pool, job, true).await?; + mark_w2l_template_upload_job_done(&downloader.mysql_pool, job, true).await?; Ok(()) } From a4f3a06f3c1bfeb9e53ab14f21efa977bdfc64ba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 01:59:27 -0400 Subject: [PATCH 0108/1042] refactoring --- src/job/mod.rs | 2 -- src/job_queries/mod.rs | 2 ++ .../tts_download_job_queries.rs} | 0 src/{job => job_queries}/w2l_download_job_queries.rs | 0 src/main.rs | 2 +- src/tts_download_job.rs | 12 ++++++------ src/w2l_download_job.rs | 12 ++++++------ 7 files changed, 15 insertions(+), 15 deletions(-) delete mode 100644 src/job/mod.rs create mode 100644 src/job_queries/mod.rs rename src/{job/job_queries.rs => job_queries/tts_download_job_queries.rs} (100%) rename src/{job => job_queries}/w2l_download_job_queries.rs (100%) diff --git a/src/job/mod.rs b/src/job/mod.rs deleted file mode 100644 index 3f5f53b55f..0000000000 --- a/src/job/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod job_queries; -pub mod w2l_download_job_queries; \ No newline at end of file diff --git a/src/job_queries/mod.rs b/src/job_queries/mod.rs new file mode 100644 index 0000000000..de78fe1474 --- /dev/null +++ b/src/job_queries/mod.rs @@ -0,0 +1,2 @@ +pub mod tts_download_job_queries; +pub mod w2l_download_job_queries; \ No newline at end of file diff --git a/src/job/job_queries.rs b/src/job_queries/tts_download_job_queries.rs similarity index 100% rename from src/job/job_queries.rs rename to src/job_queries/tts_download_job_queries.rs diff --git a/src/job/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs similarity index 100% rename from src/job/w2l_download_job_queries.rs rename to src/job_queries/w2l_download_job_queries.rs diff --git a/src/main.rs b/src/main.rs index d7de84cbd8..3cc36d9acb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,7 @@ pub mod util; pub mod validations; // NB: This is included so sqlx can generate all the queries. -mod job; +mod job_queries; use actix_cors::Cors; use actix_http::http; diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 34c68caf0e..c3acc29e90 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -6,7 +6,7 @@ //#![allow(warnings)] pub mod buckets; -pub mod job; +pub mod job_queries; pub mod util; use anyhow::anyhow; @@ -14,11 +14,11 @@ use chrono::Utc; use crate::buckets::bucket_client::BucketClient; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::buckets::file_hashing::get_file_hash; -use crate::job::job_queries::TtsUploadJobRecord; -use crate::job::job_queries::insert_tts_model; -use crate::job::job_queries::mark_tts_upload_job_done; -use crate::job::job_queries::mark_tts_upload_job_failure; -use crate::job::job_queries::query_tts_upload_job_records; +use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; +use crate::job_queries::tts_download_job_queries::insert_tts_model; +use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_done; +use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_failure; +use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::random_token::random_token; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 6f658f5c2b..80810ba41d 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -6,7 +6,7 @@ //#![allow(warnings)] pub mod buckets; -pub mod job; +pub mod job_queries; pub mod util; use anyhow::anyhow; @@ -14,11 +14,11 @@ use chrono::Utc; use crate::buckets::bucket_client::BucketClient; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::buckets::file_hashing::get_file_hash; -use crate::job::w2l_download_job_queries::W2lTemplateUploadJobRecord; -use crate::job::w2l_download_job_queries::insert_w2l_template; -use crate::job::w2l_download_job_queries::mark_w2l_template_upload_job_done; -use crate::job::w2l_download_job_queries::mark_w2l_template_upload_job_failure; -use crate::job::w2l_download_job_queries::query_w2l_template_upload_job_records; +use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; +use crate::job_queries::w2l_download_job_queries::insert_w2l_template; +use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_done; +use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_failure; +use crate::job_queries::w2l_download_job_queries::query_w2l_template_upload_job_records; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::random_token::random_token; From b958b8badf5a8c24db057882e8e94655a4707ffc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 02:05:47 -0400 Subject: [PATCH 0109/1042] refactoring --- .../endpoints/default_route_404.rs | 0 .../endpoints/misc/enable_alpha.rs | 10 ++++---- src/{ => http_server}/endpoints/misc/mod.rs | 0 src/{ => http_server}/endpoints/mod.rs | 0 src/{ => http_server}/endpoints/root_index.rs | 0 .../endpoints/tts/enqueue_infer_tts.rs | 0 .../endpoints/tts/enqueue_upload_tts_model.rs | 10 ++++---- src/{ => http_server}/endpoints/tts/mod.rs | 0 .../endpoints/users/create_account.rs | 10 ++++---- .../endpoints/users/login.rs | 14 +++++------ .../endpoints/users/logout.rs | 16 ++++++------- src/{ => http_server}/endpoints/users/mod.rs | 0 .../endpoints/users/session_info.rs | 18 +++++++------- .../endpoints/w2l/enqueue_infer_w2l.rs | 10 ++++---- .../w2l/enqueue_upload_w2l_template.rs | 10 ++++---- src/{ => http_server}/endpoints/w2l/mod.rs | 0 src/http_server/mod.rs | 1 + src/main.rs | 24 +++++++++---------- 18 files changed, 62 insertions(+), 61 deletions(-) rename src/{ => http_server}/endpoints/default_route_404.rs (100%) rename src/{ => http_server}/endpoints/misc/enable_alpha.rs (100%) rename src/{ => http_server}/endpoints/misc/mod.rs (100%) rename src/{ => http_server}/endpoints/mod.rs (100%) rename src/{ => http_server}/endpoints/root_index.rs (100%) rename src/{ => http_server}/endpoints/tts/enqueue_infer_tts.rs (100%) rename src/{ => http_server}/endpoints/tts/enqueue_upload_tts_model.rs (100%) rename src/{ => http_server}/endpoints/tts/mod.rs (100%) rename src/{ => http_server}/endpoints/users/create_account.rs (97%) rename src/{ => http_server}/endpoints/users/login.rs (97%) rename src/{ => http_server}/endpoints/users/logout.rs (93%) rename src/{ => http_server}/endpoints/users/mod.rs (100%) rename src/{ => http_server}/endpoints/users/session_info.rs (93%) rename src/{ => http_server}/endpoints/w2l/enqueue_infer_w2l.rs (100%) rename src/{ => http_server}/endpoints/w2l/enqueue_upload_w2l_template.rs (100%) rename src/{ => http_server}/endpoints/w2l/mod.rs (100%) create mode 100644 src/http_server/mod.rs diff --git a/src/endpoints/default_route_404.rs b/src/http_server/endpoints/default_route_404.rs similarity index 100% rename from src/endpoints/default_route_404.rs rename to src/http_server/endpoints/default_route_404.rs diff --git a/src/endpoints/misc/enable_alpha.rs b/src/http_server/endpoints/misc/enable_alpha.rs similarity index 100% rename from src/endpoints/misc/enable_alpha.rs rename to src/http_server/endpoints/misc/enable_alpha.rs index 7b5c11cb97..c062dbde56 100644 --- a/src/endpoints/misc/enable_alpha.rs +++ b/src/http_server/endpoints/misc/enable_alpha.rs @@ -1,10 +1,10 @@ -use actix_http::http::StatusCode; -use actix_web::{HttpResponse, HttpRequest, Responder, get, web, HttpMessage}; -use log::info; use actix_http::cookie::Cookie; -use crate::server_state::ServerState; -use std::sync::Arc; +use actix_http::http::StatusCode; use actix_web::web::Query; +use actix_web::{HttpResponse, HttpRequest, Responder, get, web, HttpMessage}; +use crate::server_state::ServerState; +use log::info; +use std::sync::Arc; const ALPHA_COOKIE_NAME : &'static str = "enable-alpha"; const CONTENT_TYPE : &'static str = "text/html; charset=utf-8"; diff --git a/src/endpoints/misc/mod.rs b/src/http_server/endpoints/misc/mod.rs similarity index 100% rename from src/endpoints/misc/mod.rs rename to src/http_server/endpoints/misc/mod.rs diff --git a/src/endpoints/mod.rs b/src/http_server/endpoints/mod.rs similarity index 100% rename from src/endpoints/mod.rs rename to src/http_server/endpoints/mod.rs diff --git a/src/endpoints/root_index.rs b/src/http_server/endpoints/root_index.rs similarity index 100% rename from src/endpoints/root_index.rs rename to src/http_server/endpoints/root_index.rs diff --git a/src/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs similarity index 100% rename from src/endpoints/tts/enqueue_infer_tts.rs rename to src/http_server/endpoints/tts/enqueue_infer_tts.rs diff --git a/src/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs similarity index 100% rename from src/endpoints/tts/enqueue_upload_tts_model.rs rename to src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index 3775dddebb..fc51a1faf3 100644 --- a/src/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -1,12 +1,17 @@ use actix_http::Error; use actix_http::http::header; +use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database_helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; @@ -14,11 +19,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::validations::username::validate_username; -use crate::validations::passwords::validate_passwords; -use actix_web::cookie::Cookie; -use crate::database_helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; -use crate::validations::model_uploads::validate_model_title; const NEW_USER_ROLE: &'static str = "new-user"; diff --git a/src/endpoints/tts/mod.rs b/src/http_server/endpoints/tts/mod.rs similarity index 100% rename from src/endpoints/tts/mod.rs rename to src/http_server/endpoints/tts/mod.rs diff --git a/src/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs similarity index 97% rename from src/endpoints/users/create_account.rs rename to src/http_server/endpoints/users/create_account.rs index 3f19d01728..7277c4889d 100644 --- a/src/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -1,13 +1,17 @@ use actix_http::Error; use actix_http::http::header; +use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; @@ -15,10 +19,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::validations::username::validate_username; -use crate::validations::passwords::validate_passwords; -use actix_web::cookie::Cookie; -use crate::common_queries::sessions::create_session_for_user; const NEW_USER_ROLE: &'static str = "new-user"; diff --git a/src/endpoints/users/login.rs b/src/http_server/endpoints/users/login.rs similarity index 97% rename from src/endpoints/users/login.rs rename to src/http_server/endpoints/users/login.rs index f3db830372..47e31c310f 100644 --- a/src/endpoints/users/login.rs +++ b/src/http_server/endpoints/users/login.rs @@ -1,26 +1,26 @@ use actix_http::Error; use actix_http::http::header; +use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; +use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::validations::username::validate_username; -use crate::validations::passwords::validate_passwords; -use actix_web::cookie::Cookie; -use sqlx::MySqlPool; -use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; #[derive(Deserialize)] pub struct LoginRequest { diff --git a/src/endpoints/users/logout.rs b/src/http_server/endpoints/users/logout.rs similarity index 93% rename from src/endpoints/users/logout.rs rename to src/http_server/endpoints/users/logout.rs index 18dbf3b073..78b07cd428 100644 --- a/src/endpoints/users/logout.rs +++ b/src/http_server/endpoints/users/logout.rs @@ -1,27 +1,27 @@ use actix_http::Error; use actix_http::http::header; +use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; -use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; +use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::validations::username::validate_username; -use crate::validations::passwords::validate_passwords; -use actix_web::cookie::Cookie; -use sqlx::MySqlPool; -use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; -use crate::endpoints::users::login::LoginSuccessResponse; #[derive(Serialize)] pub struct LogoutSuccessResponse { diff --git a/src/endpoints/users/mod.rs b/src/http_server/endpoints/users/mod.rs similarity index 100% rename from src/endpoints/users/mod.rs rename to src/http_server/endpoints/users/mod.rs diff --git a/src/endpoints/users/session_info.rs b/src/http_server/endpoints/users/session_info.rs similarity index 93% rename from src/endpoints/users/session_info.rs rename to src/http_server/endpoints/users/session_info.rs index 70a14d7942..c93e8cc04e 100644 --- a/src/endpoints/users/session_info.rs +++ b/src/http_server/endpoints/users/session_info.rs @@ -1,28 +1,28 @@ use actix_http::Error; use actix_http::http::header; +use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; -use crate::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; +use crate::util::session_checker::SessionRecord; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; +use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::validations::username::validate_username; -use crate::validations::passwords::validate_passwords; -use actix_web::cookie::Cookie; -use sqlx::MySqlPool; -use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; -use crate::endpoints::users::login::LoginSuccessResponse; -use crate::util::session_checker::SessionRecord; #[derive(Serialize)] pub struct UserInfo { diff --git a/src/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs similarity index 100% rename from src/endpoints/w2l/enqueue_infer_w2l.rs rename to src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index ffc7640377..e64c077aa3 100644 --- a/src/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -1,12 +1,17 @@ use actix_http::Error; use actix_http::http::header; +use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; @@ -14,11 +19,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::validations::username::validate_username; -use crate::validations::passwords::validate_passwords; -use actix_web::cookie::Cookie; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; -use crate::validations::model_uploads::validate_model_title; const NEW_USER_ROLE: &'static str = "new-user"; diff --git a/src/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs similarity index 100% rename from src/endpoints/w2l/enqueue_upload_w2l_template.rs rename to src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index 9b57b2533e..e6589b4848 100644 --- a/src/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -1,12 +1,17 @@ use actix_http::Error; use actix_http::http::header; +use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::server_state::ServerState; use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; @@ -14,11 +19,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::validations::username::validate_username; -use crate::validations::passwords::validate_passwords; -use actix_web::cookie::Cookie; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; -use crate::validations::model_uploads::validate_model_title; const NEW_USER_ROLE: &'static str = "new-user"; diff --git a/src/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs similarity index 100% rename from src/endpoints/w2l/mod.rs rename to src/http_server/endpoints/w2l/mod.rs diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs new file mode 100644 index 0000000000..c4b360f4b8 --- /dev/null +++ b/src/http_server/mod.rs @@ -0,0 +1 @@ +pub mod endpoints; diff --git a/src/main.rs b/src/main.rs index 3cc36d9acb..73781218b4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,7 @@ pub mod buckets; pub mod common_queries; pub mod database_helpers; -pub mod endpoints; +pub mod http_server; pub mod server_state; pub mod util; pub mod validations; @@ -22,15 +22,17 @@ use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; -use crate::endpoints::default_route_404::default_route_404; -use crate::endpoints::misc::enable_alpha::enable_alpha; -use crate::endpoints::root_index::get_root_index; -use crate::endpoints::users::create_account::create_account_handler; -use crate::endpoints::users::login::login_handler; -use crate::endpoints::users::logout::logout_handler; -use crate::endpoints::users::session_info::session_info_handler; -use crate::endpoints::w2l::enqueue_infer_w2l::infer_w2l_handler; -use crate::endpoints::w2l::enqueue_upload_w2l_template::upload_w2l_template_handler; +use crate::http_server::endpoints::default_route_404::default_route_404; +use crate::http_server::endpoints::misc::enable_alpha::enable_alpha; +use crate::http_server::endpoints::root_index::get_root_index; +use crate::http_server::endpoints::tts::enqueue_infer_tts::infer_tts_handler; +use crate::http_server::endpoints::tts::enqueue_upload_tts_model::upload_tts_model_handler; +use crate::http_server::endpoints::users::create_account::create_account_handler; +use crate::http_server::endpoints::users::login::login_handler; +use crate::http_server::endpoints::users::logout::logout_handler; +use crate::http_server::endpoints::users::session_info::session_info_handler; +use crate::http_server::endpoints::w2l::enqueue_infer_w2l::infer_w2l_handler; +use crate::http_server::endpoints::w2l::enqueue_upload_w2l_template::upload_w2l_template_handler; use crate::server_state::{ServerState, EnvConfig}; use crate::util::cookie_manager::CookieManager; use crate::util::session_checker::SessionChecker; @@ -38,8 +40,6 @@ use log::{info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; -use crate::endpoints::tts::enqueue_upload_tts_model::upload_tts_model_handler; -use crate::endpoints::tts::enqueue_infer_tts::infer_tts_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; From 7e15d675b64bff93164e7396f8f09bc3c605f7a2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 02:14:11 -0400 Subject: [PATCH 0110/1042] refactor more --- src/http_server/endpoints/tts/enqueue_infer_tts.rs | 2 +- src/http_server/endpoints/tts/enqueue_upload_tts_model.rs | 2 +- src/http_server/endpoints/users/create_account.rs | 2 +- src/http_server/endpoints/users/login.rs | 2 +- src/http_server/endpoints/users/logout.rs | 2 +- src/http_server/endpoints/users/session_info.rs | 4 ++-- src/http_server/endpoints/w2l/enqueue_infer_w2l.rs | 2 +- src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs | 2 +- src/http_server/mod.rs | 1 + src/{util => http_server/web_utils}/cookie_manager.rs | 2 +- src/{util => http_server/web_utils}/ip_address.rs | 0 src/http_server/web_utils/mod.rs | 3 +++ src/{util => http_server/web_utils}/session_checker.rs | 2 +- src/main.rs | 4 ++-- src/server_state.rs | 4 ++-- src/util/mod.rs | 5 +---- 16 files changed, 20 insertions(+), 19 deletions(-) rename src/{util => http_server/web_utils}/cookie_manager.rs (100%) rename src/{util => http_server/web_utils}/ip_address.rs (100%) create mode 100644 src/http_server/web_utils/mod.rs rename src/{util => http_server/web_utils}/session_checker.rs (98%) diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 56d1d307fd..45f572857a 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -6,8 +6,8 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index fc51a1faf3..7b35d976de 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -6,8 +6,8 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; +use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index 7277c4889d..05eaf5a622 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -7,8 +7,8 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/src/http_server/endpoints/users/login.rs b/src/http_server/endpoints/users/login.rs index 47e31c310f..c46919975b 100644 --- a/src/http_server/endpoints/users/login.rs +++ b/src/http_server/endpoints/users/login.rs @@ -8,8 +8,8 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::AnyhowResult; use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/src/http_server/endpoints/users/logout.rs b/src/http_server/endpoints/users/logout.rs index 78b07cd428..4b1b172c88 100644 --- a/src/http_server/endpoints/users/logout.rs +++ b/src/http_server/endpoints/users/logout.rs @@ -9,8 +9,8 @@ use crate::AnyhowResult; use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/src/http_server/endpoints/users/session_info.rs b/src/http_server/endpoints/users/session_info.rs index c93e8cc04e..22078b4b9d 100644 --- a/src/http_server/endpoints/users/session_info.rs +++ b/src/http_server/endpoints/users/session_info.rs @@ -9,10 +9,10 @@ use crate::AnyhowResult; use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; -use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; -use crate::util::session_checker::SessionRecord; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index e64c077aa3..58fbf961be 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -6,8 +6,8 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index e6589b4848..b2b39198a2 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -6,8 +6,8 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::ip_address::get_request_ip; use crate::util::random_token::random_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs index c4b360f4b8..9b6c9f4477 100644 --- a/src/http_server/mod.rs +++ b/src/http_server/mod.rs @@ -1 +1,2 @@ pub mod endpoints; +pub mod web_utils; diff --git a/src/util/cookie_manager.rs b/src/http_server/web_utils/cookie_manager.rs similarity index 100% rename from src/util/cookie_manager.rs rename to src/http_server/web_utils/cookie_manager.rs index 966c7b895b..c2de649aff 100644 --- a/src/util/cookie_manager.rs +++ b/src/http_server/web_utils/cookie_manager.rs @@ -6,12 +6,12 @@ use hmac::Hmac; use hmac::NewMac; use jwt::SignWithKey; use jwt::VerifyWithKey; +use log::LevelFilter::Off; use log::{info, warn}; use sha2::Sha256; use std::collections::BTreeMap; use std::ops::Sub; use time::OffsetDateTime; -use log::LevelFilter::Off; const COOKIE_VERSION : u32 = 1; diff --git a/src/util/ip_address.rs b/src/http_server/web_utils/ip_address.rs similarity index 100% rename from src/util/ip_address.rs rename to src/http_server/web_utils/ip_address.rs diff --git a/src/http_server/web_utils/mod.rs b/src/http_server/web_utils/mod.rs new file mode 100644 index 0000000000..845d875119 --- /dev/null +++ b/src/http_server/web_utils/mod.rs @@ -0,0 +1,3 @@ +pub mod cookie_manager; +pub mod ip_address; +pub mod session_checker; \ No newline at end of file diff --git a/src/util/session_checker.rs b/src/http_server/web_utils/session_checker.rs similarity index 98% rename from src/util/session_checker.rs rename to src/http_server/web_utils/session_checker.rs index 00b7471eeb..68a491b0aa 100644 --- a/src/util/session_checker.rs +++ b/src/http_server/web_utils/session_checker.rs @@ -1,7 +1,7 @@ use actix_web::HttpRequest; use anyhow::anyhow; use crate::AnyhowResult; -use crate::util::cookie_manager::CookieManager; +use crate::http_server::web_utils::cookie_manager::CookieManager; use log::{info, warn}; use sqlx::MySqlPool; use sqlx::error::Error::RowNotFound; diff --git a/src/main.rs b/src/main.rs index 73781218b4..fca1ed55a2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -33,9 +33,9 @@ use crate::http_server::endpoints::users::logout::logout_handler; use crate::http_server::endpoints::users::session_info::session_info_handler; use crate::http_server::endpoints::w2l::enqueue_infer_w2l::infer_w2l_handler; use crate::http_server::endpoints::w2l::enqueue_upload_w2l_template::upload_w2l_template_handler; +use crate::http_server::web_utils::cookie_manager::CookieManager; +use crate::http_server::web_utils::session_checker::SessionChecker; use crate::server_state::{ServerState, EnvConfig}; -use crate::util::cookie_manager::CookieManager; -use crate::util::session_checker::SessionChecker; use log::{info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; diff --git a/src/server_state.rs b/src/server_state.rs index 222dc8c2f7..0a8f4f01c4 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,6 +1,6 @@ +use crate::http_server::web_utils::cookie_manager::CookieManager; +use crate::http_server::web_utils::session_checker::SessionChecker; use sqlx::MySqlPool; -use crate::util::cookie_manager::CookieManager; -use crate::util::session_checker::SessionChecker; /// State that is injected into every endpoint. #[derive(Clone)] diff --git a/src/util/mod.rs b/src/util/mod.rs index 3dda00a986..5bdef0899a 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,6 +1,3 @@ pub mod anyhow_result; -pub mod cookie_manager; pub mod filesystem; -pub mod ip_address; -pub mod random_token; -pub mod session_checker; \ No newline at end of file +pub mod random_token; \ No newline at end of file From 281283ed8bf5c5b3ef53e0a8f0332cb9461c14d5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 02:32:28 -0400 Subject: [PATCH 0111/1042] extract gdown command wrapper --- .../google_drive_download_command.rs | 66 +++++++++++++++++++ src/script_execution/mod.rs | 1 + src/tts_download_job.rs | 54 ++++----------- src/w2l_download_job.rs | 1 + 4 files changed, 80 insertions(+), 42 deletions(-) create mode 100644 src/script_execution/google_drive_download_command.rs create mode 100644 src/script_execution/mod.rs diff --git a/src/script_execution/google_drive_download_command.rs b/src/script_execution/google_drive_download_command.rs new file mode 100644 index 0000000000..f3ad11abc0 --- /dev/null +++ b/src/script_execution/google_drive_download_command.rs @@ -0,0 +1,66 @@ +use anyhow::anyhow; +use crate::util::anyhow_result::AnyhowResult; +use crate::util::random_token::random_token; +use log::{info, warn}; +use std::io::{BufReader, Read}; +use std::path::{PathBuf, Path}; +use std::process::Command; +use tempdir::TempDir; + +/// This is a Python script that uses the `gdown` package to download from Google Drive. +/// We're using this because it's a hack that gets around OAuth gateways. All the Rust +/// crates require OAuth permissions. Ugh. +/// +/// Filename: `download_gdrive.py` +/// Arguments: +/// --url (google drive url) +/// --output_file (local download filename) +pub struct GoogleDriveDownloadCommand { + command: String, +} + +impl GoogleDriveDownloadCommand { + pub fn new(command: &str) -> Self { + Self { + command: command.to_string(), + } + } + + /// Download file from Google Drive into the `TempDir`. + /// Return the local filesystem filename. + pub async fn download_file(&self, + download_url: &str, + temp_dir: &TempDir) -> AnyhowResult + { + let temp_dir_path = temp_dir.path() + .to_str() + .unwrap_or("/tmp") + .to_string(); + + let temp_filename = random_token(10); + let temp_filename = format!("{}/{}.bin", temp_dir_path, temp_filename); + + info!("Downloading {} to: {}", download_url, temp_filename); + + let command = format!("{} --url {} --output_file {}", + &self.command, + &download_url, + &temp_filename); + + info!("Running command: {}", command); + + let result = Command::new("sh") + .arg("-c") + .arg(command) + .output()?; + + info!("Downloader Result: {:?}", result); + + if !result.status.success() { + let reason = String::from_utf8(result.stderr).unwrap_or("UNKNOWN".to_string()); + return Err(anyhow!("Failure to execute command: {:?}", reason)) + } + + Ok(temp_filename) + } +} diff --git a/src/script_execution/mod.rs b/src/script_execution/mod.rs new file mode 100644 index 0000000000..49f2e9d2cf --- /dev/null +++ b/src/script_execution/mod.rs @@ -0,0 +1 @@ +pub mod google_drive_download_command; diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index c3acc29e90..d0b8b9efb3 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -7,6 +7,7 @@ pub mod buckets; pub mod job_queries; +pub mod script_execution; pub mod util; use anyhow::anyhow; @@ -19,6 +20,7 @@ use crate::job_queries::tts_download_job_queries::insert_tts_model; use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_done; use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_failure; use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; +use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::random_token::random_token; @@ -48,6 +50,7 @@ struct Downloader { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, pub bucket_client: BucketClient, + pub google_drive_downloader: GoogleDriveDownloadCommand, // Command to run pub download_script: String, // Root to store TTS results @@ -92,6 +95,8 @@ async fn main() -> AnyhowResult<()> { "DOWNLOAD_SCRIPT", "./scripts/download_gdrive.py"); + let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); + let temp_directory = PathBuf::from(temp_directory); check_directory_exists(&temp_directory)?; @@ -113,6 +118,7 @@ async fn main() -> AnyhowResult<()> { mysql_pool, bucket_client, download_script, + google_drive_downloader, bucket_root_tts_model_uploads: bucket_root.to_string(), }; @@ -181,47 +187,6 @@ async fn process_jobs(downloader: &Downloader, jobs: Vec) -> Ok(()) } -async fn download_file(downloader: &Downloader, - job: &TtsUploadJobRecord, - temp_dir: &TempDir) -> AnyhowResult -{ - - let temp_dir_path = temp_dir.path() - .to_str() - .unwrap_or("/tmp") - .to_string(); - - let filename = random_token(10); - let filename = format!("{}/{}.bin", temp_dir_path, filename); - - info!("Downloading to: {}", filename); - - let url = job.download_url.as_ref() - .map(|c| c.to_string()) - .unwrap_or("".to_string()); - - let command = format!("{} --url {} --output_file {}", - downloader.download_script, - &url, - &filename); - - info!("Running command: {}", command); - - let result = Command::new("sh") - .arg("-c") - .arg(command) - .output()?; - - info!("Downloader Result: {:?}", result); - - if !result.status.success() { - let reason = String::from_utf8(result.stderr).unwrap_or("UNKNOWN".to_string()); - return Err(anyhow!("Failure to execute command: {:?}", reason)) - } - - Ok(filename) -} - async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. (DONE) @@ -232,8 +197,13 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; + let download_url = job.download_url.as_ref() + .map(|c| c.to_string()) + .unwrap_or("".to_string()); + info!("Calling downloader..."); - let download_filename = download_file(downloader, job, &temp_dir).await?; + let download_filename = downloader.google_drive_downloader + .download_file(&download_url, &temp_dir).await?; let private_bucket_hash = get_file_hash(&download_filename)?; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 80810ba41d..d4ef0d58c9 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -7,6 +7,7 @@ pub mod buckets; pub mod job_queries; +pub mod script_execution; pub mod util; use anyhow::anyhow; From 0378735ac39ec0d1e5d685a1d6c176e8f3883ec2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 02:35:33 -0400 Subject: [PATCH 0112/1042] refactor to use shared script wrapper --- src/w2l_download_job.rs | 53 +++++++++-------------------------------- 1 file changed, 11 insertions(+), 42 deletions(-) diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index d4ef0d58c9..39f136e204 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -20,6 +20,7 @@ use crate::job_queries::w2l_download_job_queries::insert_w2l_template; use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_done; use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_failure; use crate::job_queries::w2l_download_job_queries::query_w2l_template_upload_job_records; +use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::random_token::random_token; @@ -49,6 +50,7 @@ struct Downloader { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, pub bucket_client: BucketClient, + pub google_drive_downloader: GoogleDriveDownloadCommand, // Command to run pub download_script: String, // Root to store W2L templates @@ -93,6 +95,8 @@ async fn main() -> AnyhowResult<()> { "DOWNLOAD_SCRIPT", "./scripts/download_gdrive.py"); + let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); + let temp_directory = PathBuf::from(temp_directory); check_directory_exists(&temp_directory)?; @@ -114,6 +118,7 @@ async fn main() -> AnyhowResult<()> { mysql_pool, bucket_client, download_script, + google_drive_downloader, bucket_root_w2l_template_uploads: bucket_root.to_string(), }; @@ -190,47 +195,6 @@ async fn process_jobs(downloader: &Downloader, jobs: Vec AnyhowResult -{ - - let temp_dir_path = temp_dir.path() - .to_str() - .unwrap_or("/tmp") - .to_string(); - - let filename = random_token(10); - let filename = format!("{}/{}.bin", temp_dir_path, filename); - - info!("Downloading to: {}", filename); - - let url = job.download_url.as_ref() - .map(|c| c.to_string()) - .unwrap_or("".to_string()); - - let command = format!("{} --url {} --output_file {}", - downloader.download_script, - &url, - &filename); - - info!("Running command: {}", command); - - let result = Command::new("sh") - .arg("-c") - .arg(command) - .output()?; - - info!("Downloader Result: {:?}", result); - - if !result.status.success() { - let reason = String::from_utf8(result.stderr).unwrap_or("UNKNOWN".to_string()); - return Err(anyhow!("Failure to execute command: {:?}", reason)) - } - - Ok(filename) -} - async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. (DONE) @@ -243,8 +207,13 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; + let download_url = job.download_url.as_ref() + .map(|c| c.to_string()) + .unwrap_or("".to_string()); + info!("Calling downloader..."); - let download_filename = download_file(downloader, job, &temp_dir).await?; + let download_filename = downloader.google_drive_downloader + .download_file(&download_url, &temp_dir).await?; let private_bucket_hash = get_file_hash(&download_filename)?; From 0f4d4e9cd49f89aae3acd7e67e8ce72dae80a5bb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 03:07:08 -0400 Subject: [PATCH 0113/1042] add w2l command (not yet in use) --- Cargo.lock | 11 ++ Cargo.toml | 1 + src/script_execution/mod.rs | 1 + .../wav2lip_preprocess_command.rs | 154 ++++++++++++++++++ 4 files changed, 167 insertions(+) create mode 100644 src/script_execution/wav2lip_preprocess_command.rs diff --git a/Cargo.lock b/Cargo.lock index 99739046ab..3b69872523 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2996,6 +2996,7 @@ dependencies = [ "serde_json", "sha2", "sqlx", + "subprocess", "tempdir", "time 0.2.26", "tokio 0.2.25", @@ -3012,6 +3013,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "subprocess" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "334b801f9ca2529ba9605109f1d2835f15aff94cd6cfe5afe6ce8cf71e5fd3c4" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "subtle" version = "2.4.0" diff --git a/Cargo.toml b/Cargo.toml index 0897048b18..b303005d4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ ring = "0.17.0-alpha.10" #rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } rust-s3 = { version = "0.26" } +subprocess = "0.2.6" tempdir = "0.3.7" # Web diff --git a/src/script_execution/mod.rs b/src/script_execution/mod.rs index 49f2e9d2cf..41465de59d 100644 --- a/src/script_execution/mod.rs +++ b/src/script_execution/mod.rs @@ -1 +1,2 @@ pub mod google_drive_download_command; +pub mod wav2lip_preprocess_command; diff --git a/src/script_execution/wav2lip_preprocess_command.rs b/src/script_execution/wav2lip_preprocess_command.rs new file mode 100644 index 0000000000..1ab44b1d9c --- /dev/null +++ b/src/script_execution/wav2lip_preprocess_command.rs @@ -0,0 +1,154 @@ +use crate::AnyhowResult; +use log::{info,warn}; +use std::process::{Command, Stdio}; +use subprocess::{Popen, PopenConfig, Redirection}; +use std::fs::OpenOptions; + +/// This command is used to preprocess the face detection frames from user-submitted video. +/// This should only ever need to run once. The frames can then be uploaded to Buckets and saved. +#[derive(Clone)] +pub struct Wav2LipPreprocessClient { + w2l_directory: String, + script_name: String, + checkpoint_path: String, + temp_directory: String, + audio_start_pad_millis: Option, + audio_end_pad_millis: Option, + end_bump_filename: Option, +} + +impl Wav2LipPreprocessClient { + pub fn new( + w2l_directory: &str, + script_name: &str, + checkpoint_path: &str, + temp_directory: &str, + audio_start_pad_millis: Option, + audio_end_pad_millis: Option, + end_bump_filename: Option, + ) -> Self { + Self { + w2l_directory: w2l_directory.to_string(), + script_name: script_name.to_string(), + checkpoint_path: checkpoint_path.to_string(), + temp_directory: temp_directory.to_string(), + audio_start_pad_millis, + audio_end_pad_millis, + end_bump_filename, + } + } + + pub fn execute(&self, + audio_filename: &str, + image_or_video_filename: &str, + output_video_filename: &str, + is_image: bool, + disable_end_bump: bool, + spawn_process: bool) -> AnyhowResult<()> + { + let mut command = String::new(); + + command.push_str("echo 'test'"); + command.push_str(" && "); + command.push_str(&format!("cd {}", self.w2l_directory)); + command.push_str(" && "); + command.push_str("source python/bin/activate"); + command.push_str(" && "); + command.push_str("python "); + command.push_str(&self.script_name); + command.push_str(" --checkpoint_path "); + command.push_str(&self.checkpoint_path); + command.push_str(" --face "); + command.push_str(image_or_video_filename); + command.push_str(" --audio "); + command.push_str(audio_filename); + command.push_str(" --outfile "); + command.push_str(output_video_filename); + + if let Some(millis) = self.audio_start_pad_millis { + command.push_str(" --audio_start_pad_millis "); + command.push_str(&millis.to_string()); + } + + if let Some(millis) = self.audio_end_pad_millis { + command.push_str(" --audio_end_pad_millis "); + command.push_str(&millis.to_string()); + } + + if !disable_end_bump { + if let Some(end_bump_filename) = self.end_bump_filename.as_ref() { + command.push_str(" --end_bump_file "); + command.push_str(end_bump_filename); + } + } + + if is_image { + command.push_str(" --is_image "); + } + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + if spawn_process { + // NB: This forks and returns immediately. + //let _child_pid = command_builder.spawn()?; + + let stdout_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/stdout.txt")?; + + let stderr_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/stderr.txt")?; + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + stdout: Redirection::File(stdout_file), + stderr: Redirection::File(stderr_file), + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + p.detach(); + + } else { + // NB: This is a blocking call. + /*let output = command_builder.output()?; + + info!("Output status: {}", output.status); + info!("Stdout: {:?}", String::from_utf8(output.stdout)); + error!("Stderr: {:?}", String::from_utf8(output.stderr)); + + if !output.status.success() { + bail!("Bad error code: {:?}", output.status); + }*/ + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + let exit_status = p.wait()?; + + info!("Exit status: {:?}", exit_status); + } + + Ok(()) + } +} From 4993707316dbcdee387c8a042c93e87a23ad23e2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 03:12:48 -0400 Subject: [PATCH 0114/1042] remove args that won't be used --- .../wav2lip_preprocess_command.rs | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/src/script_execution/wav2lip_preprocess_command.rs b/src/script_execution/wav2lip_preprocess_command.rs index 1ab44b1d9c..6c1d704040 100644 --- a/src/script_execution/wav2lip_preprocess_command.rs +++ b/src/script_execution/wav2lip_preprocess_command.rs @@ -11,10 +11,6 @@ pub struct Wav2LipPreprocessClient { w2l_directory: String, script_name: String, checkpoint_path: String, - temp_directory: String, - audio_start_pad_millis: Option, - audio_end_pad_millis: Option, - end_bump_filename: Option, } impl Wav2LipPreprocessClient { @@ -22,19 +18,11 @@ impl Wav2LipPreprocessClient { w2l_directory: &str, script_name: &str, checkpoint_path: &str, - temp_directory: &str, - audio_start_pad_millis: Option, - audio_end_pad_millis: Option, - end_bump_filename: Option, ) -> Self { Self { w2l_directory: w2l_directory.to_string(), script_name: script_name.to_string(), checkpoint_path: checkpoint_path.to_string(), - temp_directory: temp_directory.to_string(), - audio_start_pad_millis, - audio_end_pad_millis, - end_bump_filename, } } @@ -65,23 +53,6 @@ impl Wav2LipPreprocessClient { command.push_str(" --outfile "); command.push_str(output_video_filename); - if let Some(millis) = self.audio_start_pad_millis { - command.push_str(" --audio_start_pad_millis "); - command.push_str(&millis.to_string()); - } - - if let Some(millis) = self.audio_end_pad_millis { - command.push_str(" --audio_end_pad_millis "); - command.push_str(&millis.to_string()); - } - - if !disable_end_bump { - if let Some(end_bump_filename) = self.end_bump_filename.as_ref() { - command.push_str(" --end_bump_file "); - command.push_str(end_bump_filename); - } - } - if is_image { command.push_str(" --is_image "); } From effe301d23dac229e9c7f7426ef2ba8f338464fc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 03:17:53 -0400 Subject: [PATCH 0115/1042] Add adapted Wav2Lip from https://github.com/ml-applications/wav2lip-edge --- .gitignore | 1 + models/Wav2Lip/.gitignore | 16 + models/Wav2Lip/README.md | 145 ++++++ models/Wav2Lip/audio.py | 136 ++++++ models/Wav2Lip/color_syncnet_train.py | 279 +++++++++++ .../evaluation/gen_videos_from_filelist.py | 238 +++++++++ .../evaluation/real_videos_inference.py | 305 ++++++++++++ .../scores_LSE/SyncNetInstance_calc_scores.py | 210 ++++++++ .../scores_LSE/calculate_scores_LRS.py | 53 ++ .../calculate_scores_real_videos.py | 45 ++ .../calculate_scores_real_videos.sh | 7 + models/Wav2Lip/face_detection/__init__.py | 7 + models/Wav2Lip/face_detection/api.py | 79 +++ .../face_detection/detection/__init__.py | 1 + .../Wav2Lip/face_detection/detection/core.py | 130 +++++ .../face_detection/detection/sfd/__init__.py | 1 + .../face_detection/detection/sfd/bbox.py | 129 +++++ .../face_detection/detection/sfd/detect.py | 112 +++++ .../face_detection/detection/sfd/net_s3fd.py | 129 +++++ .../detection/sfd/sfd_detector.py | 59 +++ models/Wav2Lip/face_detection/models.py | 261 ++++++++++ models/Wav2Lip/face_detection/utils.py | 313 ++++++++++++ models/Wav2Lip/hparams.py | 86 ++++ models/Wav2Lip/hq_wav2lip_train.py | 443 +++++++++++++++++ models/Wav2Lip/inference.py | 286 +++++++++++ models/Wav2Lip/inference_custom.py | 458 ++++++++++++++++++ models/Wav2Lip/models/__init__.py | 2 + models/Wav2Lip/models/conv.py | 44 ++ models/Wav2Lip/models/syncnet.py | 66 +++ models/Wav2Lip/models/wav2lip.py | 184 +++++++ models/Wav2Lip/preprocess.py | 113 +++++ models/Wav2Lip/requirements.txt | 9 + models/Wav2Lip/wav2lip_train.py | 374 ++++++++++++++ 33 files changed, 4721 insertions(+) create mode 100644 models/Wav2Lip/.gitignore create mode 100644 models/Wav2Lip/README.md create mode 100644 models/Wav2Lip/audio.py create mode 100644 models/Wav2Lip/color_syncnet_train.py create mode 100644 models/Wav2Lip/evaluation/gen_videos_from_filelist.py create mode 100644 models/Wav2Lip/evaluation/real_videos_inference.py create mode 100644 models/Wav2Lip/evaluation/scores_LSE/SyncNetInstance_calc_scores.py create mode 100644 models/Wav2Lip/evaluation/scores_LSE/calculate_scores_LRS.py create mode 100755 models/Wav2Lip/evaluation/scores_LSE/calculate_scores_real_videos.py create mode 100644 models/Wav2Lip/evaluation/scores_LSE/calculate_scores_real_videos.sh create mode 100644 models/Wav2Lip/face_detection/__init__.py create mode 100644 models/Wav2Lip/face_detection/api.py create mode 100644 models/Wav2Lip/face_detection/detection/__init__.py create mode 100644 models/Wav2Lip/face_detection/detection/core.py create mode 100644 models/Wav2Lip/face_detection/detection/sfd/__init__.py create mode 100644 models/Wav2Lip/face_detection/detection/sfd/bbox.py create mode 100644 models/Wav2Lip/face_detection/detection/sfd/detect.py create mode 100644 models/Wav2Lip/face_detection/detection/sfd/net_s3fd.py create mode 100644 models/Wav2Lip/face_detection/detection/sfd/sfd_detector.py create mode 100644 models/Wav2Lip/face_detection/models.py create mode 100644 models/Wav2Lip/face_detection/utils.py create mode 100644 models/Wav2Lip/hparams.py create mode 100644 models/Wav2Lip/hq_wav2lip_train.py create mode 100644 models/Wav2Lip/inference.py create mode 100644 models/Wav2Lip/inference_custom.py create mode 100644 models/Wav2Lip/models/__init__.py create mode 100644 models/Wav2Lip/models/conv.py create mode 100644 models/Wav2Lip/models/syncnet.py create mode 100644 models/Wav2Lip/models/wav2lip.py create mode 100644 models/Wav2Lip/preprocess.py create mode 100644 models/Wav2Lip/requirements.txt create mode 100644 models/Wav2Lip/wav2lip_train.py diff --git a/.gitignore b/.gitignore index e68a75a505..dc88b1914f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .env .env-secrets .idea/ +__pycache__/ _ignored_schema.rs production_database_secret_connect.sh secrets.toml diff --git a/models/Wav2Lip/.gitignore b/models/Wav2Lip/.gitignore new file mode 100644 index 0000000000..56f02807a5 --- /dev/null +++ b/models/Wav2Lip/.gitignore @@ -0,0 +1,16 @@ +*.pkl +*.jpg +*.mp4 +*.pth +*.pyc +__pycache__ +*.h5 +*.avi +*.wav +filelists/*.txt +evaluation/test_filelists/lr*.txt +*.pyc +*.mkv +*.gif +*.webm +*.mp3 diff --git a/models/Wav2Lip/README.md b/models/Wav2Lip/README.md new file mode 100644 index 0000000000..19c43c453c --- /dev/null +++ b/models/Wav2Lip/README.md @@ -0,0 +1,145 @@ +# **Wav2Lip**: *Accurately Lip-syncing Videos In The Wild* + +This code is part of the paper: _A Lip Sync Expert Is All You Need for Speech to Lip Generation In the Wild_ published at ACM Multimedia 2020. + +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/a-lip-sync-expert-is-all-you-need-for-speech/lip-sync-on-lrs2)](https://paperswithcode.com/sota/lip-sync-on-lrs2?p=a-lip-sync-expert-is-all-you-need-for-speech) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/a-lip-sync-expert-is-all-you-need-for-speech/lip-sync-on-lrs3)](https://paperswithcode.com/sota/lip-sync-on-lrs3?p=a-lip-sync-expert-is-all-you-need-for-speech) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/a-lip-sync-expert-is-all-you-need-for-speech/lip-sync-on-lrw)](https://paperswithcode.com/sota/lip-sync-on-lrw?p=a-lip-sync-expert-is-all-you-need-for-speech) + +[[Paper]](http://arxiv.org/abs/2008.10010) | [[Project Page]](http://cvit.iiit.ac.in/research/projects/cvit-projects/a-lip-sync-expert-is-all-you-need-for-speech-to-lip-generation-in-the-wild/) | [[Demo Video]](https://youtu.be/0fXaDCZNOJc) | [[Interactive Demo]](https://bhaasha.iiit.ac.in/lipsync) | [[Colab Notebook]](https://colab.research.google.com/drive/1tZpDWXz49W6wDcTprANRGLo2D_EbD5J8?usp=sharing) | [[ReSyncED] (coming soon)](#) + + + +---------- +**Highlights** +---------- + - Lip-sync videos to any target speech with high accuracy. Try our [interactive demo](https://bhaasha.iiit.ac.in/lipsync). + - Works for any identity, voice, and language. Also works for CGI faces and synthetic voices. + - Complete training code, inference code, and pretrained models are available. + - Or, quick-start with the Google Colab Notebook: [Link](https://colab.research.google.com/drive/1tZpDWXz49W6wDcTprANRGLo2D_EbD5J8?usp=sharing). Checkpoints and samples are available in a Google Drive [folder](https://drive.google.com/drive/folders/1I-0dNLfFOSFwrfqjNa-SXuwaURHE5K4k?usp=sharing) as well. + - Several new, reliable evaluation benchmarks and metrics [[`evaluation/` folder of this repo]](https://github.com/Rudrabha/Wav2Lip/tree/master/evaluation) released. + - Code to calculate metrics reported in the paper is also made available. + +-------- +**Disclaimer** +-------- +All results from this open-source code or our [demo website](https://bhaasha.iiit.ac.in/lipsync) should only be used for research/academic/personal purposes only. As the models are trained on the LRS2 dataset, any form of commercial use is strictly prohibhited. Please contact us for all further queries. + +Prerequisites +------------- +- `Python 3.5.2` (code has been tested with this version at our end, but several other users say that `3.6+` is the one that works instead.) +- ffmpeg: `sudo apt-get install ffmpeg` +- Install necessary packages using `pip install -r requirements.txt` +- Face detection [pre-trained model](https://www.adrianbulat.com/downloads/python-fan/s3fd-619a316812.pth) should be downloaded to `face_detection/detection/sfd/s3fd.pth`. Alternative [link](https://iiitaphyd-my.sharepoint.com/:u:/g/personal/prajwal_k_research_iiit_ac_in/EZsy6qWuivtDnANIG73iHjIBjMSoojcIV0NULXV-yiuiIg?e=qTasa8) if the above does not work. + +Getting the weights +---------- +| Model | Description | Link to the model | +| :-------------: | :---------------: | :---------------: | +| Wav2Lip | Highly accurate lip-sync | [Link](https://iiitaphyd-my.sharepoint.com/:u:/g/personal/radrabha_m_research_iiit_ac_in/Eb3LEzbfuKlJiR600lQWRxgBIY27JZg80f7V9jtMfbNDaQ?e=TBFBVW) | +| Wav2Lip + GAN | Slightly inferior lip-sync, but better visual quality | [Link](https://iiitaphyd-my.sharepoint.com/:u:/g/personal/radrabha_m_research_iiit_ac_in/EdjI7bZlgApMqsVoEUUXpLsBxqXbn5z8VTmoxp55YNDcIA?e=n9ljGW) | +| Expert Discriminator | Weights of the expert discriminator | [Link](https://iiitaphyd-my.sharepoint.com/:u:/g/personal/radrabha_m_research_iiit_ac_in/EQRvmiZg-HRAjvI6zqN9eTEBP74KefynCwPWVmF57l-AYA?e=ZRPHKP) | + +Lip-syncing videos using the pre-trained models (Inference) +------- +You can lip-sync any video to any audio: +```bash +python inference.py --checkpoint_path --face --audio +``` +The result is saved (by default) in `results/result_voice.mp4`. You can specify it as an argument, similar to several other available options. The audio source can be any file supported by `FFMPEG` containing audio data: `*.wav`, `*.mp3` or even a video file, from which the code will automatically extract the audio. + +##### Tips for better results: +- Experiment with the `--pads` argument to adjust the detected face bounding box. Often leads to improved results. You might need to increase the bottom padding to include the chin region. E.g. `--pads 0 20 0 0`. +- If you see the mouth position dislocated or some weird artifacts such as two mouths, then it can be because of over-smoothing the face detections. Use the `--nosmooth` argument and give another try. +- Experiment with the `--resize_factor` argument, to get a lower resolution video. Why? The models are trained on faces which were at a lower resolution. You might get better, visually pleasing results for 720p videos than for 1080p videos (in many cases, the latter works well too). +- The Wav2Lip model without GAN usually needs more experimenting with the above two to get the most ideal results, and sometimes, can give you a better result as well. + +Preparing LRS2 for training +---------- +Our models are trained on LRS2. See [here](#training-on-datasets-other-than-lrs2) for a few suggestions regarding training on other datasets. +##### LRS2 dataset folder structure + +``` +data_root (mvlrs_v1) +├── main, pretrain (we use only main folder in this work) +| ├── list of folders +| │ ├── five-digit numbered video IDs ending with (.mp4) +``` + +Place the LRS2 filelists (train, val, test) `.txt` files in the `filelists/` folder. + +##### Preprocess the dataset for fast training + +```bash +python preprocess.py --data_root data_root/main --preprocessed_root lrs2_preprocessed/ +``` +Additional options like `batch_size` and number of GPUs to use in parallel to use can also be set. + +##### Preprocessed LRS2 folder structure +``` +preprocessed_root (lrs2_preprocessed) +├── list of folders +| ├── Folders with five-digit numbered video IDs +| │ ├── *.jpg +| │ ├── audio.wav +``` + +Train! +---------- +There are two major steps: (i) Train the expert lip-sync discriminator, (ii) Train the Wav2Lip model(s). + +##### Training the expert discriminator +You can download [the pre-trained weights](#getting-the-weights) if you want to skip this step. To train it: +```bash +python color_syncnet_train.py --data_root lrs2_preprocessed/ --checkpoint_dir +``` +##### Training the Wav2Lip models +You can either train the model without the additional visual quality disriminator (< 1 day of training) or use the discriminator (~2 days). For the former, run: +```bash +python wav2lip_train.py --data_root lrs2_preprocessed/ --checkpoint_dir --syncnet_checkpoint_path +``` + +To train with the visual quality discriminator, you should run `hq_wav2lip_train.py` instead. The arguments for both the files are similar. In both the cases, you can resume training as well. Look at `python wav2lip_train.py --help` for more details. You can also set additional less commonly-used hyper-parameters at the bottom of the `hparams.py` file. + +Training on datasets other than LRS2 +------------------------------------ +Training on other datasets might require modifications to the code. Please read the following before you raise an issue: + +- You might not get good results by training/fine-tuning on a few minutes of a single speaker. This is a separate research problem, to which we do not have a solution yet. Thus, we would most likely not be able to resolve your issue. +- You must train the expert discriminator for your own dataset before training Wav2Lip. +- If it is your own dataset downloaded from the web, in most cases, needs to be sync-corrected. +- Be mindful of the FPS of the videos of your dataset. Changes to FPS would need significant code changes. +- The expert discriminator's eval loss should go down to ~0.25 and the Wav2Lip eval sync loss should go down to ~0.2 to get good results. + +When raising an issue on this topic, please let us know that you are aware of all these points. + +Evaluation +---------- +Will be updated. + +License and Citation +---------- +The software can only be used for personal/research/non-commercial purposes. Please cite the following paper if you have use this code: +``` +@inproceedings{10.1145/3394171.3413532, +author = {Prajwal, K R and Mukhopadhyay, Rudrabha and Namboodiri, Vinay P. and Jawahar, C.V.}, +title = {A Lip Sync Expert Is All You Need for Speech to Lip Generation In the Wild}, +year = {2020}, +isbn = {9781450379885}, +publisher = {Association for Computing Machinery}, +address = {New York, NY, USA}, +url = {https://doi.org/10.1145/3394171.3413532}, +doi = {10.1145/3394171.3413532}, +booktitle = {Proceedings of the 28th ACM International Conference on Multimedia}, +pages = {484–492}, +numpages = {9}, +keywords = {lip sync, talking face generation, video generation}, +location = {Seattle, WA, USA}, +series = {MM '20} +} +``` + + +Acknowledgements +---------- +Parts of the code structure is inspired by this [TTS repository](https://github.com/r9y9/deepvoice3_pytorch). We thank the author for this wonderful code. The code for Face Detection has been taken from the [face_alignment](https://github.com/1adrianb/face-alignment) repository. We thank the authors for releasing their code and models. diff --git a/models/Wav2Lip/audio.py b/models/Wav2Lip/audio.py new file mode 100644 index 0000000000..1c5c08332d --- /dev/null +++ b/models/Wav2Lip/audio.py @@ -0,0 +1,136 @@ +import librosa +import librosa.filters +import numpy as np +import tensorflow as tf +from scipy import signal +from scipy.io import wavfile +from hparams import hparams as hp + +def load_wav(path, sr): + return librosa.core.load(path, sr=sr)[0] + +def save_wav(wav, path, sr): + wav *= 32767 / max(0.01, np.max(np.abs(wav))) + #proposed by @dsmiller + wavfile.write(path, sr, wav.astype(np.int16)) + +def save_wavenet_wav(wav, path, sr): + librosa.output.write_wav(path, wav, sr=sr) + +def preemphasis(wav, k, preemphasize=True): + if preemphasize: + return signal.lfilter([1, -k], [1], wav) + return wav + +def inv_preemphasis(wav, k, inv_preemphasize=True): + if inv_preemphasize: + return signal.lfilter([1], [1, -k], wav) + return wav + +def get_hop_size(): + hop_size = hp.hop_size + if hop_size is None: + assert hp.frame_shift_ms is not None + hop_size = int(hp.frame_shift_ms / 1000 * hp.sample_rate) + return hop_size + +def linearspectrogram(wav): + D = _stft(preemphasis(wav, hp.preemphasis, hp.preemphasize)) + S = _amp_to_db(np.abs(D)) - hp.ref_level_db + + if hp.signal_normalization: + return _normalize(S) + return S + +def melspectrogram(wav): + D = _stft(preemphasis(wav, hp.preemphasis, hp.preemphasize)) + S = _amp_to_db(_linear_to_mel(np.abs(D))) - hp.ref_level_db + + if hp.signal_normalization: + return _normalize(S) + return S + +def _lws_processor(): + import lws + return lws.lws(hp.n_fft, get_hop_size(), fftsize=hp.win_size, mode="speech") + +def _stft(y): + if hp.use_lws: + return _lws_processor(hp).stft(y).T + else: + return librosa.stft(y=y, n_fft=hp.n_fft, hop_length=get_hop_size(), win_length=hp.win_size) + +########################################################## +#Those are only correct when using lws!!! (This was messing with Wavenet quality for a long time!) +def num_frames(length, fsize, fshift): + """Compute number of time frames of spectrogram + """ + pad = (fsize - fshift) + if length % fshift == 0: + M = (length + pad * 2 - fsize) // fshift + 1 + else: + M = (length + pad * 2 - fsize) // fshift + 2 + return M + + +def pad_lr(x, fsize, fshift): + """Compute left and right padding + """ + M = num_frames(len(x), fsize, fshift) + pad = (fsize - fshift) + T = len(x) + 2 * pad + r = (M - 1) * fshift + fsize - T + return pad, pad + r +########################################################## +#Librosa correct padding +def librosa_pad_lr(x, fsize, fshift): + return 0, (x.shape[0] // fshift + 1) * fshift - x.shape[0] + +# Conversions +_mel_basis = None + +def _linear_to_mel(spectogram): + global _mel_basis + if _mel_basis is None: + _mel_basis = _build_mel_basis() + return np.dot(_mel_basis, spectogram) + +def _build_mel_basis(): + assert hp.fmax <= hp.sample_rate // 2 + return librosa.filters.mel(hp.sample_rate, hp.n_fft, n_mels=hp.num_mels, + fmin=hp.fmin, fmax=hp.fmax) + +def _amp_to_db(x): + min_level = np.exp(hp.min_level_db / 20 * np.log(10)) + return 20 * np.log10(np.maximum(min_level, x)) + +def _db_to_amp(x): + return np.power(10.0, (x) * 0.05) + +def _normalize(S): + if hp.allow_clipping_in_normalization: + if hp.symmetric_mels: + return np.clip((2 * hp.max_abs_value) * ((S - hp.min_level_db) / (-hp.min_level_db)) - hp.max_abs_value, + -hp.max_abs_value, hp.max_abs_value) + else: + return np.clip(hp.max_abs_value * ((S - hp.min_level_db) / (-hp.min_level_db)), 0, hp.max_abs_value) + + assert S.max() <= 0 and S.min() - hp.min_level_db >= 0 + if hp.symmetric_mels: + return (2 * hp.max_abs_value) * ((S - hp.min_level_db) / (-hp.min_level_db)) - hp.max_abs_value + else: + return hp.max_abs_value * ((S - hp.min_level_db) / (-hp.min_level_db)) + +def _denormalize(D): + if hp.allow_clipping_in_normalization: + if hp.symmetric_mels: + return (((np.clip(D, -hp.max_abs_value, + hp.max_abs_value) + hp.max_abs_value) * -hp.min_level_db / (2 * hp.max_abs_value)) + + hp.min_level_db) + else: + return ((np.clip(D, 0, hp.max_abs_value) * -hp.min_level_db / hp.max_abs_value) + hp.min_level_db) + + if hp.symmetric_mels: + return (((D + hp.max_abs_value) * -hp.min_level_db / (2 * hp.max_abs_value)) + hp.min_level_db) + else: + return ((D * -hp.min_level_db / hp.max_abs_value) + hp.min_level_db) diff --git a/models/Wav2Lip/color_syncnet_train.py b/models/Wav2Lip/color_syncnet_train.py new file mode 100644 index 0000000000..afa0054438 --- /dev/null +++ b/models/Wav2Lip/color_syncnet_train.py @@ -0,0 +1,279 @@ +from os.path import dirname, join, basename, isfile +from tqdm import tqdm + +from models import SyncNet_color as SyncNet +import audio + +import torch +from torch import nn +from torch import optim +import torch.backends.cudnn as cudnn +from torch.utils import data as data_utils +import numpy as np + +from glob import glob + +import os, random, cv2, argparse +from hparams import hparams, get_image_list + +parser = argparse.ArgumentParser(description='Code to train the expert lip-sync discriminator') + +parser.add_argument("--data_root", help="Root folder of the preprocessed LRS2 dataset", required=True) + +parser.add_argument('--checkpoint_dir', help='Save checkpoints to this directory', required=True, type=str) +parser.add_argument('--checkpoint_path', help='Resumed from this checkpoint', default=None, type=str) + +args = parser.parse_args() + + +global_step = 0 +global_epoch = 0 +use_cuda = torch.cuda.is_available() +print('use_cuda: {}'.format(use_cuda)) + +syncnet_T = 5 +syncnet_mel_step_size = 16 + +class Dataset(object): + def __init__(self, split): + self.all_videos = get_image_list(args.data_root, split) + + def get_frame_id(self, frame): + return int(basename(frame).split('.')[0]) + + def get_window(self, start_frame): + start_id = self.get_frame_id(start_frame) + vidname = dirname(start_frame) + + window_fnames = [] + for frame_id in range(start_id, start_id + syncnet_T): + frame = join(vidname, '{}.jpg'.format(frame_id)) + if not isfile(frame): + return None + window_fnames.append(frame) + return window_fnames + + def crop_audio_window(self, spec, start_frame): + # num_frames = (T x hop_size * fps) / sample_rate + start_frame_num = self.get_frame_id(start_frame) + start_idx = int(80. * (start_frame_num / float(hparams.fps))) + + end_idx = start_idx + syncnet_mel_step_size + + return spec[start_idx : end_idx, :] + + + def __len__(self): + return len(self.all_videos) + + def __getitem__(self, idx): + while 1: + idx = random.randint(0, len(self.all_videos) - 1) + vidname = self.all_videos[idx] + + img_names = list(glob(join(vidname, '*.jpg'))) + if len(img_names) <= 3 * syncnet_T: + continue + img_name = random.choice(img_names) + wrong_img_name = random.choice(img_names) + while wrong_img_name == img_name: + wrong_img_name = random.choice(img_names) + + if random.choice([True, False]): + y = torch.ones(1).float() + chosen = img_name + else: + y = torch.zeros(1).float() + chosen = wrong_img_name + + window_fnames = self.get_window(chosen) + if window_fnames is None: + continue + + window = [] + all_read = True + for fname in window_fnames: + img = cv2.imread(fname) + if img is None: + all_read = False + break + try: + img = cv2.resize(img, (hparams.img_size, hparams.img_size)) + except Exception as e: + all_read = False + break + + window.append(img) + + if not all_read: continue + + try: + wavpath = join(vidname, "audio.wav") + wav = audio.load_wav(wavpath, hparams.sample_rate) + + orig_mel = audio.melspectrogram(wav).T + except Exception as e: + continue + + mel = self.crop_audio_window(orig_mel.copy(), img_name) + + if (mel.shape[0] != syncnet_mel_step_size): + continue + + # H x W x 3 * T + x = np.concatenate(window, axis=2) / 255. + x = x.transpose(2, 0, 1) + x = x[:, x.shape[1]//2:] + + x = torch.FloatTensor(x) + mel = torch.FloatTensor(mel.T).unsqueeze(0) + + return x, mel, y + +logloss = nn.BCELoss() +def cosine_loss(a, v, y): + d = nn.functional.cosine_similarity(a, v) + loss = logloss(d.unsqueeze(1), y) + + return loss + +def train(device, model, train_data_loader, test_data_loader, optimizer, + checkpoint_dir=None, checkpoint_interval=None, nepochs=None): + + global global_step, global_epoch + resumed_step = global_step + + while global_epoch < nepochs: + running_loss = 0. + prog_bar = tqdm(enumerate(train_data_loader)) + for step, (x, mel, y) in prog_bar: + model.train() + optimizer.zero_grad() + + # Transform data to CUDA device + x = x.to(device) + + mel = mel.to(device) + + a, v = model(mel, x) + y = y.to(device) + + loss = cosine_loss(a, v, y) + loss.backward() + optimizer.step() + + global_step += 1 + cur_session_steps = global_step - resumed_step + running_loss += loss.item() + + if global_step == 1 or global_step % checkpoint_interval == 0: + save_checkpoint( + model, optimizer, global_step, checkpoint_dir, global_epoch) + + if global_step % hparams.syncnet_eval_interval == 0: + with torch.no_grad(): + eval_model(test_data_loader, global_step, device, model, checkpoint_dir) + + prog_bar.set_description('Loss: {}'.format(running_loss / (step + 1))) + + global_epoch += 1 + +def eval_model(test_data_loader, global_step, device, model, checkpoint_dir): + eval_steps = 1400 + print('Evaluating for {} steps'.format(eval_steps)) + losses = [] + while 1: + for step, (x, mel, y) in enumerate(test_data_loader): + + model.eval() + + # Transform data to CUDA device + x = x.to(device) + + mel = mel.to(device) + + a, v = model(mel, x) + y = y.to(device) + + loss = cosine_loss(a, v, y) + losses.append(loss.item()) + + if step > eval_steps: break + + averaged_loss = sum(losses) / len(losses) + print(averaged_loss) + + return + +def save_checkpoint(model, optimizer, step, checkpoint_dir, epoch): + + checkpoint_path = join( + checkpoint_dir, "checkpoint_step{:09d}.pth".format(global_step)) + optimizer_state = optimizer.state_dict() if hparams.save_optimizer_state else None + torch.save({ + "state_dict": model.state_dict(), + "optimizer": optimizer_state, + "global_step": step, + "global_epoch": epoch, + }, checkpoint_path) + print("Saved checkpoint:", checkpoint_path) + +def _load(checkpoint_path): + if use_cuda: + checkpoint = torch.load(checkpoint_path) + else: + checkpoint = torch.load(checkpoint_path, + map_location=lambda storage, loc: storage) + return checkpoint + +def load_checkpoint(path, model, optimizer, reset_optimizer=False): + global global_step + global global_epoch + + print("Load checkpoint from: {}".format(path)) + checkpoint = _load(path) + model.load_state_dict(checkpoint["state_dict"]) + if not reset_optimizer: + optimizer_state = checkpoint["optimizer"] + if optimizer_state is not None: + print("Load optimizer state from {}".format(path)) + optimizer.load_state_dict(checkpoint["optimizer"]) + global_step = checkpoint["global_step"] + global_epoch = checkpoint["global_epoch"] + + return model + +if __name__ == "__main__": + checkpoint_dir = args.checkpoint_dir + checkpoint_path = args.checkpoint_path + + if not os.path.exists(checkpoint_dir): os.mkdir(checkpoint_dir) + + # Dataset and Dataloader setup + train_dataset = Dataset('train') + test_dataset = Dataset('val') + + train_data_loader = data_utils.DataLoader( + train_dataset, batch_size=hparams.syncnet_batch_size, shuffle=True, + num_workers=hparams.num_workers) + + test_data_loader = data_utils.DataLoader( + test_dataset, batch_size=hparams.syncnet_batch_size, + num_workers=8) + + device = torch.device("cuda" if use_cuda else "cpu") + + # Model + model = SyncNet().to(device) + print('total trainable params {}'.format(sum(p.numel() for p in model.parameters() if p.requires_grad))) + + optimizer = optim.Adam([p for p in model.parameters() if p.requires_grad], + lr=hparams.syncnet_lr) + + if checkpoint_path is not None: + load_checkpoint(checkpoint_path, model, optimizer, reset_optimizer=False) + + train(device, model, train_data_loader, test_data_loader, optimizer, + checkpoint_dir=checkpoint_dir, + checkpoint_interval=hparams.syncnet_checkpoint_interval, + nepochs=hparams.nepochs) diff --git a/models/Wav2Lip/evaluation/gen_videos_from_filelist.py b/models/Wav2Lip/evaluation/gen_videos_from_filelist.py new file mode 100644 index 0000000000..bd666b9325 --- /dev/null +++ b/models/Wav2Lip/evaluation/gen_videos_from_filelist.py @@ -0,0 +1,238 @@ +from os import listdir, path +import numpy as np +import scipy, cv2, os, sys, argparse +import dlib, json, subprocess +from tqdm import tqdm +from glob import glob +import torch + +sys.path.append('../') +import audio +import face_detection +from models import Wav2Lip + +parser = argparse.ArgumentParser(description='Code to generate results for test filelists') + +parser.add_argument('--filelist', type=str, + help='Filepath of filelist file to read', required=True) +parser.add_argument('--results_dir', type=str, help='Folder to save all results into', + required=True) +parser.add_argument('--data_root', type=str, required=True) +parser.add_argument('--checkpoint_path', type=str, + help='Name of saved checkpoint to load weights from', required=True) + +parser.add_argument('--pads', nargs='+', type=int, default=[0, 0, 0, 0], + help='Padding (top, bottom, left, right)') +parser.add_argument('--face_det_batch_size', type=int, + help='Single GPU batch size for face detection', default=64) +parser.add_argument('--wav2lip_batch_size', type=int, help='Batch size for Wav2Lip', default=128) + +# parser.add_argument('--resize_factor', default=1, type=int) + +args = parser.parse_args() +args.img_size = 96 + +def get_smoothened_boxes(boxes, T): + for i in range(len(boxes)): + if i + T > len(boxes): + window = boxes[len(boxes) - T:] + else: + window = boxes[i : i + T] + boxes[i] = np.mean(window, axis=0) + return boxes + +def face_detect(images): + batch_size = args.face_det_batch_size + + while 1: + predictions = [] + try: + for i in range(0, len(images), batch_size): + predictions.extend(detector.get_detections_for_batch(np.array(images[i:i + batch_size]))) + except RuntimeError: + if batch_size == 1: + raise RuntimeError('Image too big to run face detection on GPU') + batch_size //= 2 + args.face_det_batch_size = batch_size + print('Recovering from OOM error; New batch size: {}'.format(batch_size)) + continue + break + + results = [] + pady1, pady2, padx1, padx2 = args.pads + for rect, image in zip(predictions, images): + if rect is None: + raise ValueError('Face not detected!') + + y1 = max(0, rect[1] - pady1) + y2 = min(image.shape[0], rect[3] + pady2) + x1 = max(0, rect[0] - padx1) + x2 = min(image.shape[1], rect[2] + padx2) + + results.append([x1, y1, x2, y2]) + + boxes = get_smoothened_boxes(np.array(results), T=5) + results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2), True] for image, (x1, y1, x2, y2) in zip(images, boxes)] + + return results + +def datagen(frames, face_det_results, mels): + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + for i, m in enumerate(mels): + if i >= len(frames): raise ValueError('Equal or less lengths only') + + frame_to_save = frames[i].copy() + face, coords, valid_frame = face_det_results[i].copy() + if not valid_frame: + continue + + face = cv2.resize(face, (args.img_size, args.img_size)) + + img_batch.append(face) + mel_batch.append(m) + frame_batch.append(frame_to_save) + coords_batch.append(coords) + + if len(img_batch) >= args.wav2lip_batch_size: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + if len(img_batch) > 0: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + +fps = 25 +mel_step_size = 16 +mel_idx_multiplier = 80./fps +device = 'cuda' if torch.cuda.is_available() else 'cpu' +print('Using {} for inference.'.format(device)) + +detector = face_detection.FaceAlignment(face_detection.LandmarksType._2D, + flip_input=False, device=device) + +def _load(checkpoint_path): + if device == 'cuda': + checkpoint = torch.load(checkpoint_path) + else: + checkpoint = torch.load(checkpoint_path, + map_location=lambda storage, loc: storage) + return checkpoint + +def load_model(path): + model = Wav2Lip() + print("Load checkpoint from: {}".format(path)) + checkpoint = _load(path) + s = checkpoint["state_dict"] + new_s = {} + for k, v in s.items(): + new_s[k.replace('module.', '')] = v + model.load_state_dict(new_s) + + model = model.to(device) + return model.eval() + +model = load_model(args.checkpoint_path) + +def main(): + assert args.data_root is not None + data_root = args.data_root + + if not os.path.isdir(args.results_dir): os.makedirs(args.results_dir) + + with open(args.filelist, 'r') as filelist: + lines = filelist.readlines() + + for idx, line in enumerate(tqdm(lines)): + audio_src, video = line.strip().split() + + audio_src = os.path.join(data_root, audio_src) + '.mp4' + video = os.path.join(data_root, video) + '.mp4' + + command = 'ffmpeg -loglevel panic -y -i {} -strict -2 {}'.format(audio_src, '../temp/temp.wav') + subprocess.call(command, shell=True) + temp_audio = '../temp/temp.wav' + + wav = audio.load_wav(temp_audio, 16000) + mel = audio.melspectrogram(wav) + if np.isnan(mel.reshape(-1)).sum() > 0: + continue + + mel_chunks = [] + i = 0 + while 1: + start_idx = int(i * mel_idx_multiplier) + if start_idx + mel_step_size > len(mel[0]): + break + mel_chunks.append(mel[:, start_idx : start_idx + mel_step_size]) + i += 1 + + video_stream = cv2.VideoCapture(video) + + full_frames = [] + while 1: + still_reading, frame = video_stream.read() + if not still_reading or len(full_frames) > len(mel_chunks): + video_stream.release() + break + full_frames.append(frame) + + if len(full_frames) < len(mel_chunks): + continue + + full_frames = full_frames[:len(mel_chunks)] + + try: + face_det_results = face_detect(full_frames.copy()) + except ValueError as e: + continue + + batch_size = args.wav2lip_batch_size + gen = datagen(full_frames.copy(), face_det_results, mel_chunks) + + for i, (img_batch, mel_batch, frames, coords) in enumerate(gen): + if i == 0: + frame_h, frame_w = full_frames[0].shape[:-1] + out = cv2.VideoWriter('../temp/result.avi', + cv2.VideoWriter_fourcc(*'DIVX'), fps, (frame_w, frame_h)) + + img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device) + mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device) + + with torch.no_grad(): + pred = model(mel_batch, img_batch) + + + pred = pred.cpu().numpy().transpose(0, 2, 3, 1) * 255. + + for pl, f, c in zip(pred, frames, coords): + y1, y2, x1, x2 = c + pl = cv2.resize(pl.astype(np.uint8), (x2 - x1, y2 - y1)) + f[y1:y2, x1:x2] = pl + out.write(f) + + out.release() + + vid = os.path.join(args.results_dir, '{}.mp4'.format(idx)) + + command = 'ffmpeg -loglevel panic -y -i {} -i {} -strict -2 -q:v 1 {}'.format(temp_audio, + '../temp/result.avi', vid) + subprocess.call(command, shell=True) + +if __name__ == '__main__': + main() diff --git a/models/Wav2Lip/evaluation/real_videos_inference.py b/models/Wav2Lip/evaluation/real_videos_inference.py new file mode 100644 index 0000000000..8c9fb15ef3 --- /dev/null +++ b/models/Wav2Lip/evaluation/real_videos_inference.py @@ -0,0 +1,305 @@ +from os import listdir, path +import numpy as np +import scipy, cv2, os, sys, argparse +import dlib, json, subprocess +from tqdm import tqdm +from glob import glob +import torch + +sys.path.append('../') +import audio +import face_detection +from models import Wav2Lip + +parser = argparse.ArgumentParser(description='Code to generate results on ReSyncED evaluation set') + +parser.add_argument('--mode', type=str, + help='random | dubbed | tts', required=True) + +parser.add_argument('--filelist', type=str, + help='Filepath of filelist file to read', default=None) + +parser.add_argument('--results_dir', type=str, help='Folder to save all results into', + required=True) +parser.add_argument('--data_root', type=str, required=True) +parser.add_argument('--checkpoint_path', type=str, + help='Name of saved checkpoint to load weights from', required=True) +parser.add_argument('--pads', nargs='+', type=int, default=[0, 10, 0, 0], + help='Padding (top, bottom, left, right)') + +parser.add_argument('--face_det_batch_size', type=int, + help='Single GPU batch size for face detection', default=16) + +parser.add_argument('--wav2lip_batch_size', type=int, help='Batch size for Wav2Lip', default=128) +parser.add_argument('--face_res', help='Approximate resolution of the face at which to test', default=180) +parser.add_argument('--min_frame_res', help='Do not downsample further below this frame resolution', default=480) +parser.add_argument('--max_frame_res', help='Downsample to at least this frame resolution', default=720) +# parser.add_argument('--resize_factor', default=1, type=int) + +args = parser.parse_args() +args.img_size = 96 + +def get_smoothened_boxes(boxes, T): + for i in range(len(boxes)): + if i + T > len(boxes): + window = boxes[len(boxes) - T:] + else: + window = boxes[i : i + T] + boxes[i] = np.mean(window, axis=0) + return boxes + +def rescale_frames(images): + rect = detector.get_detections_for_batch(np.array([images[0]]))[0] + if rect is None: + raise ValueError('Face not detected!') + h, w = images[0].shape[:-1] + + x1, y1, x2, y2 = rect + + face_size = max(np.abs(y1 - y2), np.abs(x1 - x2)) + + diff = np.abs(face_size - args.face_res) + for factor in range(2, 16): + downsampled_res = face_size // factor + if min(h//factor, w//factor) < args.min_frame_res: break + if np.abs(downsampled_res - args.face_res) >= diff: break + + factor -= 1 + if factor == 1: return images + + return [cv2.resize(im, (im.shape[1]//(factor), im.shape[0]//(factor))) for im in images] + + +def face_detect(images): + batch_size = args.face_det_batch_size + images = rescale_frames(images) + + while 1: + predictions = [] + try: + for i in range(0, len(images), batch_size): + predictions.extend(detector.get_detections_for_batch(np.array(images[i:i + batch_size]))) + except RuntimeError: + if batch_size == 1: + raise RuntimeError('Image too big to run face detection on GPU') + batch_size //= 2 + print('Recovering from OOM error; New batch size: {}'.format(batch_size)) + continue + break + + results = [] + pady1, pady2, padx1, padx2 = args.pads + for rect, image in zip(predictions, images): + if rect is None: + raise ValueError('Face not detected!') + + y1 = max(0, rect[1] - pady1) + y2 = min(image.shape[0], rect[3] + pady2) + x1 = max(0, rect[0] - padx1) + x2 = min(image.shape[1], rect[2] + padx2) + + results.append([x1, y1, x2, y2]) + + boxes = get_smoothened_boxes(np.array(results), T=5) + results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2), True] for image, (x1, y1, x2, y2) in zip(images, boxes)] + + return results, images + +def datagen(frames, face_det_results, mels): + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + for i, m in enumerate(mels): + if i >= len(frames): raise ValueError('Equal or less lengths only') + + frame_to_save = frames[i].copy() + face, coords, valid_frame = face_det_results[i].copy() + if not valid_frame: + continue + + face = cv2.resize(face, (args.img_size, args.img_size)) + + img_batch.append(face) + mel_batch.append(m) + frame_batch.append(frame_to_save) + coords_batch.append(coords) + + if len(img_batch) >= args.wav2lip_batch_size: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + if len(img_batch) > 0: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + +def increase_frames(frames, l): + ## evenly duplicating frames to increase length of video + while len(frames) < l: + dup_every = float(l) / len(frames) + + final_frames = [] + next_duplicate = 0. + + for i, f in enumerate(frames): + final_frames.append(f) + + if int(np.ceil(next_duplicate)) == i: + final_frames.append(f) + + next_duplicate += dup_every + + frames = final_frames + + return frames[:l] + +mel_step_size = 16 +device = 'cuda' if torch.cuda.is_available() else 'cpu' +print('Using {} for inference.'.format(device)) + +detector = face_detection.FaceAlignment(face_detection.LandmarksType._2D, + flip_input=False, device=device) + +def _load(checkpoint_path): + if device == 'cuda': + checkpoint = torch.load(checkpoint_path) + else: + checkpoint = torch.load(checkpoint_path, + map_location=lambda storage, loc: storage) + return checkpoint + +def load_model(path): + model = Wav2Lip() + print("Load checkpoint from: {}".format(path)) + checkpoint = _load(path) + s = checkpoint["state_dict"] + new_s = {} + for k, v in s.items(): + new_s[k.replace('module.', '')] = v + model.load_state_dict(new_s) + + model = model.to(device) + return model.eval() + +model = load_model(args.checkpoint_path) + +def main(): + if not os.path.isdir(args.results_dir): os.makedirs(args.results_dir) + + if args.mode == 'dubbed': + files = listdir(args.data_root) + lines = ['{} {}'.format(f, f) for f in files] + + else: + assert args.filelist is not None + with open(args.filelist, 'r') as filelist: + lines = filelist.readlines() + + for idx, line in enumerate(tqdm(lines)): + video, audio_src = line.strip().split() + + audio_src = os.path.join(args.data_root, audio_src) + video = os.path.join(args.data_root, video) + + command = 'ffmpeg -loglevel panic -y -i {} -strict -2 {}'.format(audio_src, '../temp/temp.wav') + subprocess.call(command, shell=True) + temp_audio = '../temp/temp.wav' + + wav = audio.load_wav(temp_audio, 16000) + mel = audio.melspectrogram(wav) + + if np.isnan(mel.reshape(-1)).sum() > 0: + raise ValueError('Mel contains nan!') + + video_stream = cv2.VideoCapture(video) + + fps = video_stream.get(cv2.CAP_PROP_FPS) + mel_idx_multiplier = 80./fps + + full_frames = [] + while 1: + still_reading, frame = video_stream.read() + if not still_reading: + video_stream.release() + break + + if min(frame.shape[:-1]) > args.max_frame_res: + h, w = frame.shape[:-1] + scale_factor = min(h, w) / float(args.max_frame_res) + h = int(h/scale_factor) + w = int(w/scale_factor) + + frame = cv2.resize(frame, (w, h)) + full_frames.append(frame) + + mel_chunks = [] + i = 0 + while 1: + start_idx = int(i * mel_idx_multiplier) + if start_idx + mel_step_size > len(mel[0]): + break + mel_chunks.append(mel[:, start_idx : start_idx + mel_step_size]) + i += 1 + + if len(full_frames) < len(mel_chunks): + if args.mode == 'tts': + full_frames = increase_frames(full_frames, len(mel_chunks)) + else: + raise ValueError('#Frames, audio length mismatch') + + else: + full_frames = full_frames[:len(mel_chunks)] + + try: + face_det_results, full_frames = face_detect(full_frames.copy()) + except ValueError as e: + continue + + batch_size = args.wav2lip_batch_size + gen = datagen(full_frames.copy(), face_det_results, mel_chunks) + + for i, (img_batch, mel_batch, frames, coords) in enumerate(gen): + if i == 0: + frame_h, frame_w = full_frames[0].shape[:-1] + + out = cv2.VideoWriter('../temp/result.avi', + cv2.VideoWriter_fourcc(*'DIVX'), fps, (frame_w, frame_h)) + + img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device) + mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device) + + with torch.no_grad(): + pred = model(mel_batch, img_batch) + + + pred = pred.cpu().numpy().transpose(0, 2, 3, 1) * 255. + + for pl, f, c in zip(pred, frames, coords): + y1, y2, x1, x2 = c + pl = cv2.resize(pl.astype(np.uint8), (x2 - x1, y2 - y1)) + f[y1:y2, x1:x2] = pl + out.write(f) + + out.release() + + vid = os.path.join(args.results_dir, '{}.mp4'.format(idx)) + command = 'ffmpeg -loglevel panic -y -i {} -i {} -strict -2 -q:v 1 {}'.format('../temp/temp.wav', + '../temp/result.avi', vid) + subprocess.call(command, shell=True) + + +if __name__ == '__main__': + main() diff --git a/models/Wav2Lip/evaluation/scores_LSE/SyncNetInstance_calc_scores.py b/models/Wav2Lip/evaluation/scores_LSE/SyncNetInstance_calc_scores.py new file mode 100644 index 0000000000..64906e257b --- /dev/null +++ b/models/Wav2Lip/evaluation/scores_LSE/SyncNetInstance_calc_scores.py @@ -0,0 +1,210 @@ +#!/usr/bin/python +#-*- coding: utf-8 -*- +# Video 25 FPS, Audio 16000HZ + +import torch +import numpy +import time, pdb, argparse, subprocess, os, math, glob +import cv2 +import python_speech_features + +from scipy import signal +from scipy.io import wavfile +from SyncNetModel import * +from shutil import rmtree + + +# ==================== Get OFFSET ==================== + +def calc_pdist(feat1, feat2, vshift=10): + + win_size = vshift*2+1 + + feat2p = torch.nn.functional.pad(feat2,(0,0,vshift,vshift)) + + dists = [] + + for i in range(0,len(feat1)): + + dists.append(torch.nn.functional.pairwise_distance(feat1[[i],:].repeat(win_size, 1), feat2p[i:i+win_size,:])) + + return dists + +# ==================== MAIN DEF ==================== + +class SyncNetInstance(torch.nn.Module): + + def __init__(self, dropout = 0, num_layers_in_fc_layers = 1024): + super(SyncNetInstance, self).__init__(); + + self.__S__ = S(num_layers_in_fc_layers = num_layers_in_fc_layers).cuda(); + + def evaluate(self, opt, videofile): + + self.__S__.eval(); + + # ========== ========== + # Convert files + # ========== ========== + + if os.path.exists(os.path.join(opt.tmp_dir,opt.reference)): + rmtree(os.path.join(opt.tmp_dir,opt.reference)) + + os.makedirs(os.path.join(opt.tmp_dir,opt.reference)) + + command = ("ffmpeg -loglevel error -y -i %s -threads 1 -f image2 %s" % (videofile,os.path.join(opt.tmp_dir,opt.reference,'%06d.jpg'))) + output = subprocess.call(command, shell=True, stdout=None) + + command = ("ffmpeg -loglevel error -y -i %s -async 1 -ac 1 -vn -acodec pcm_s16le -ar 16000 %s" % (videofile,os.path.join(opt.tmp_dir,opt.reference,'audio.wav'))) + output = subprocess.call(command, shell=True, stdout=None) + + # ========== ========== + # Load video + # ========== ========== + + images = [] + + flist = glob.glob(os.path.join(opt.tmp_dir,opt.reference,'*.jpg')) + flist.sort() + + for fname in flist: + img_input = cv2.imread(fname) + img_input = cv2.resize(img_input, (224,224)) #HARD CODED, CHANGE BEFORE RELEASE + images.append(img_input) + + im = numpy.stack(images,axis=3) + im = numpy.expand_dims(im,axis=0) + im = numpy.transpose(im,(0,3,4,1,2)) + + imtv = torch.autograd.Variable(torch.from_numpy(im.astype(float)).float()) + + # ========== ========== + # Load audio + # ========== ========== + + sample_rate, audio = wavfile.read(os.path.join(opt.tmp_dir,opt.reference,'audio.wav')) + mfcc = zip(*python_speech_features.mfcc(audio,sample_rate)) + mfcc = numpy.stack([numpy.array(i) for i in mfcc]) + + cc = numpy.expand_dims(numpy.expand_dims(mfcc,axis=0),axis=0) + cct = torch.autograd.Variable(torch.from_numpy(cc.astype(float)).float()) + + # ========== ========== + # Check audio and video input length + # ========== ========== + + #if (float(len(audio))/16000) != (float(len(images))/25) : + # print("WARNING: Audio (%.4fs) and video (%.4fs) lengths are different."%(float(len(audio))/16000,float(len(images))/25)) + + min_length = min(len(images),math.floor(len(audio)/640)) + + # ========== ========== + # Generate video and audio feats + # ========== ========== + + lastframe = min_length-5 + im_feat = [] + cc_feat = [] + + tS = time.time() + for i in range(0,lastframe,opt.batch_size): + + im_batch = [ imtv[:,:,vframe:vframe+5,:,:] for vframe in range(i,min(lastframe,i+opt.batch_size)) ] + im_in = torch.cat(im_batch,0) + im_out = self.__S__.forward_lip(im_in.cuda()); + im_feat.append(im_out.data.cpu()) + + cc_batch = [ cct[:,:,:,vframe*4:vframe*4+20] for vframe in range(i,min(lastframe,i+opt.batch_size)) ] + cc_in = torch.cat(cc_batch,0) + cc_out = self.__S__.forward_aud(cc_in.cuda()) + cc_feat.append(cc_out.data.cpu()) + + im_feat = torch.cat(im_feat,0) + cc_feat = torch.cat(cc_feat,0) + + # ========== ========== + # Compute offset + # ========== ========== + + #print('Compute time %.3f sec.' % (time.time()-tS)) + + dists = calc_pdist(im_feat,cc_feat,vshift=opt.vshift) + mdist = torch.mean(torch.stack(dists,1),1) + + minval, minidx = torch.min(mdist,0) + + offset = opt.vshift-minidx + conf = torch.median(mdist) - minval + + fdist = numpy.stack([dist[minidx].numpy() for dist in dists]) + # fdist = numpy.pad(fdist, (3,3), 'constant', constant_values=15) + fconf = torch.median(mdist).numpy() - fdist + fconfm = signal.medfilt(fconf,kernel_size=9) + + numpy.set_printoptions(formatter={'float': '{: 0.3f}'.format}) + #print('Framewise conf: ') + #print(fconfm) + #print('AV offset: \t%d \nMin dist: \t%.3f\nConfidence: \t%.3f' % (offset,minval,conf)) + + dists_npy = numpy.array([ dist.numpy() for dist in dists ]) + return offset.numpy(), conf.numpy(), minval.numpy() + + def extract_feature(self, opt, videofile): + + self.__S__.eval(); + + # ========== ========== + # Load video + # ========== ========== + cap = cv2.VideoCapture(videofile) + + frame_num = 1; + images = [] + while frame_num: + frame_num += 1 + ret, image = cap.read() + if ret == 0: + break + + images.append(image) + + im = numpy.stack(images,axis=3) + im = numpy.expand_dims(im,axis=0) + im = numpy.transpose(im,(0,3,4,1,2)) + + imtv = torch.autograd.Variable(torch.from_numpy(im.astype(float)).float()) + + # ========== ========== + # Generate video feats + # ========== ========== + + lastframe = len(images)-4 + im_feat = [] + + tS = time.time() + for i in range(0,lastframe,opt.batch_size): + + im_batch = [ imtv[:,:,vframe:vframe+5,:,:] for vframe in range(i,min(lastframe,i+opt.batch_size)) ] + im_in = torch.cat(im_batch,0) + im_out = self.__S__.forward_lipfeat(im_in.cuda()); + im_feat.append(im_out.data.cpu()) + + im_feat = torch.cat(im_feat,0) + + # ========== ========== + # Compute offset + # ========== ========== + + print('Compute time %.3f sec.' % (time.time()-tS)) + + return im_feat + + + def loadParameters(self, path): + loaded_state = torch.load(path, map_location=lambda storage, loc: storage); + + self_state = self.__S__.state_dict(); + + for name, param in loaded_state.items(): + + self_state[name].copy_(param); diff --git a/models/Wav2Lip/evaluation/scores_LSE/calculate_scores_LRS.py b/models/Wav2Lip/evaluation/scores_LSE/calculate_scores_LRS.py new file mode 100644 index 0000000000..eda02b8fbb --- /dev/null +++ b/models/Wav2Lip/evaluation/scores_LSE/calculate_scores_LRS.py @@ -0,0 +1,53 @@ +#!/usr/bin/python +#-*- coding: utf-8 -*- + +import time, pdb, argparse, subprocess +import glob +import os +from tqdm import tqdm + +from SyncNetInstance_calc_scores import * + +# ==================== LOAD PARAMS ==================== + + +parser = argparse.ArgumentParser(description = "SyncNet"); + +parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help=''); +parser.add_argument('--batch_size', type=int, default='20', help=''); +parser.add_argument('--vshift', type=int, default='15', help=''); +parser.add_argument('--data_root', type=str, required=True, help=''); +parser.add_argument('--tmp_dir', type=str, default="data/work/pytmp", help=''); +parser.add_argument('--reference', type=str, default="demo", help=''); + +opt = parser.parse_args(); + + +# ==================== RUN EVALUATION ==================== + +s = SyncNetInstance(); + +s.loadParameters(opt.initial_model); +#print("Model %s loaded."%opt.initial_model); +path = os.path.join(opt.data_root, "*.mp4") + +all_videos = glob.glob(path) + +prog_bar = tqdm(range(len(all_videos))) +avg_confidence = 0. +avg_min_distance = 0. + + +for videofile_idx in prog_bar: + videofile = all_videos[videofile_idx] + offset, confidence, min_distance = s.evaluate(opt, videofile=videofile) + avg_confidence += confidence + avg_min_distance += min_distance + prog_bar.set_description('Avg Confidence: {}, Avg Minimum Dist: {}'.format(round(avg_confidence / (videofile_idx + 1), 3), round(avg_min_distance / (videofile_idx + 1), 3))) + prog_bar.refresh() + +print ('Average Confidence: {}'.format(avg_confidence/len(all_videos))) +print ('Average Minimum Distance: {}'.format(avg_min_distance/len(all_videos))) + + + diff --git a/models/Wav2Lip/evaluation/scores_LSE/calculate_scores_real_videos.py b/models/Wav2Lip/evaluation/scores_LSE/calculate_scores_real_videos.py new file mode 100755 index 0000000000..0962258465 --- /dev/null +++ b/models/Wav2Lip/evaluation/scores_LSE/calculate_scores_real_videos.py @@ -0,0 +1,45 @@ +#!/usr/bin/python +#-*- coding: utf-8 -*- + +import time, pdb, argparse, subprocess, pickle, os, gzip, glob + +from SyncNetInstance_calc_scores import * + +# ==================== PARSE ARGUMENT ==================== + +parser = argparse.ArgumentParser(description = "SyncNet"); +parser.add_argument('--initial_model', type=str, default="data/syncnet_v2.model", help=''); +parser.add_argument('--batch_size', type=int, default='20', help=''); +parser.add_argument('--vshift', type=int, default='15', help=''); +parser.add_argument('--data_dir', type=str, default='data/work', help=''); +parser.add_argument('--videofile', type=str, default='', help=''); +parser.add_argument('--reference', type=str, default='', help=''); +opt = parser.parse_args(); + +setattr(opt,'avi_dir',os.path.join(opt.data_dir,'pyavi')) +setattr(opt,'tmp_dir',os.path.join(opt.data_dir,'pytmp')) +setattr(opt,'work_dir',os.path.join(opt.data_dir,'pywork')) +setattr(opt,'crop_dir',os.path.join(opt.data_dir,'pycrop')) + + +# ==================== LOAD MODEL AND FILE LIST ==================== + +s = SyncNetInstance(); + +s.loadParameters(opt.initial_model); +#print("Model %s loaded."%opt.initial_model); + +flist = glob.glob(os.path.join(opt.crop_dir,opt.reference,'0*.avi')) +flist.sort() + +# ==================== GET OFFSETS ==================== + +dists = [] +for idx, fname in enumerate(flist): + offset, conf, dist = s.evaluate(opt,videofile=fname) + print (str(dist)+" "+str(conf)) + +# ==================== PRINT RESULTS TO FILE ==================== + +#with open(os.path.join(opt.work_dir,opt.reference,'activesd.pckl'), 'wb') as fil: +# pickle.dump(dists, fil) diff --git a/models/Wav2Lip/evaluation/scores_LSE/calculate_scores_real_videos.sh b/models/Wav2Lip/evaluation/scores_LSE/calculate_scores_real_videos.sh new file mode 100644 index 0000000000..bd02c95405 --- /dev/null +++ b/models/Wav2Lip/evaluation/scores_LSE/calculate_scores_real_videos.sh @@ -0,0 +1,7 @@ +rm all_scores.txt +yourfilenames=`ls $1` +for eachfile in $yourfilenames +do + python run_pipeline.py --videofile $eachfile --reference wav2lip --data_dir tmp_dir + python calculate_scores_real_videos.py --videofile $eachfile --reference wav2lip --data_dir /ssd_scratch/cvit/prajwalkr_rudra/tmp_dir >> all_scores.txt +done diff --git a/models/Wav2Lip/face_detection/__init__.py b/models/Wav2Lip/face_detection/__init__.py new file mode 100644 index 0000000000..4bae29fd5f --- /dev/null +++ b/models/Wav2Lip/face_detection/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +__author__ = """Adrian Bulat""" +__email__ = 'adrian.bulat@nottingham.ac.uk' +__version__ = '1.0.1' + +from .api import FaceAlignment, LandmarksType, NetworkSize diff --git a/models/Wav2Lip/face_detection/api.py b/models/Wav2Lip/face_detection/api.py new file mode 100644 index 0000000000..cb02d5252d --- /dev/null +++ b/models/Wav2Lip/face_detection/api.py @@ -0,0 +1,79 @@ +from __future__ import print_function +import os +import torch +from torch.utils.model_zoo import load_url +from enum import Enum +import numpy as np +import cv2 +try: + import urllib.request as request_file +except BaseException: + import urllib as request_file + +from .models import FAN, ResNetDepth +from .utils import * + + +class LandmarksType(Enum): + """Enum class defining the type of landmarks to detect. + + ``_2D`` - the detected points ``(x,y)`` are detected in a 2D space and follow the visible contour of the face + ``_2halfD`` - this points represent the projection of the 3D points into 3D + ``_3D`` - detect the points ``(x,y,z)``` in a 3D space + + """ + _2D = 1 + _2halfD = 2 + _3D = 3 + + +class NetworkSize(Enum): + # TINY = 1 + # SMALL = 2 + # MEDIUM = 3 + LARGE = 4 + + def __new__(cls, value): + member = object.__new__(cls) + member._value_ = value + return member + + def __int__(self): + return self.value + +ROOT = os.path.dirname(os.path.abspath(__file__)) + +class FaceAlignment: + def __init__(self, landmarks_type, network_size=NetworkSize.LARGE, + device='cuda', flip_input=False, face_detector='sfd', verbose=False): + self.device = device + self.flip_input = flip_input + self.landmarks_type = landmarks_type + self.verbose = verbose + + network_size = int(network_size) + + if 'cuda' in device: + torch.backends.cudnn.benchmark = True + + # Get the face detector + face_detector_module = __import__('face_detection.detection.' + face_detector, + globals(), locals(), [face_detector], 0) + self.face_detector = face_detector_module.FaceDetector(device=device, verbose=verbose) + + def get_detections_for_batch(self, images): + images = images[..., ::-1] + detected_faces = self.face_detector.detect_from_batch(images.copy()) + results = [] + + for i, d in enumerate(detected_faces): + if len(d) == 0: + results.append(None) + continue + d = d[0] + d = np.clip(d, 0, None) + + x1, y1, x2, y2 = map(int, d[:-1]) + results.append((x1, y1, x2, y2)) + + return results \ No newline at end of file diff --git a/models/Wav2Lip/face_detection/detection/__init__.py b/models/Wav2Lip/face_detection/detection/__init__.py new file mode 100644 index 0000000000..1a6b0402da --- /dev/null +++ b/models/Wav2Lip/face_detection/detection/__init__.py @@ -0,0 +1 @@ +from .core import FaceDetector \ No newline at end of file diff --git a/models/Wav2Lip/face_detection/detection/core.py b/models/Wav2Lip/face_detection/detection/core.py new file mode 100644 index 0000000000..0f8275e8e5 --- /dev/null +++ b/models/Wav2Lip/face_detection/detection/core.py @@ -0,0 +1,130 @@ +import logging +import glob +from tqdm import tqdm +import numpy as np +import torch +import cv2 + + +class FaceDetector(object): + """An abstract class representing a face detector. + + Any other face detection implementation must subclass it. All subclasses + must implement ``detect_from_image``, that return a list of detected + bounding boxes. Optionally, for speed considerations detect from path is + recommended. + """ + + def __init__(self, device, verbose): + self.device = device + self.verbose = verbose + + if verbose: + if 'cpu' in device: + logger = logging.getLogger(__name__) + logger.warning("Detection running on CPU, this may be potentially slow.") + + if 'cpu' not in device and 'cuda' not in device: + if verbose: + logger.error("Expected values for device are: {cpu, cuda} but got: %s", device) + raise ValueError + + def detect_from_image(self, tensor_or_path): + """Detects faces in a given image. + + This function detects the faces present in a provided BGR(usually) + image. The input can be either the image itself or the path to it. + + Arguments: + tensor_or_path {numpy.ndarray, torch.tensor or string} -- the path + to an image or the image itself. + + Example:: + + >>> path_to_image = 'data/image_01.jpg' + ... detected_faces = detect_from_image(path_to_image) + [A list of bounding boxes (x1, y1, x2, y2)] + >>> image = cv2.imread(path_to_image) + ... detected_faces = detect_from_image(image) + [A list of bounding boxes (x1, y1, x2, y2)] + + """ + raise NotImplementedError + + def detect_from_directory(self, path, extensions=['.jpg', '.png'], recursive=False, show_progress_bar=True): + """Detects faces from all the images present in a given directory. + + Arguments: + path {string} -- a string containing a path that points to the folder containing the images + + Keyword Arguments: + extensions {list} -- list of string containing the extensions to be + consider in the following format: ``.extension_name`` (default: + {['.jpg', '.png']}) recursive {bool} -- option wherever to scan the + folder recursively (default: {False}) show_progress_bar {bool} -- + display a progressbar (default: {True}) + + Example: + >>> directory = 'data' + ... detected_faces = detect_from_directory(directory) + {A dictionary of [lists containing bounding boxes(x1, y1, x2, y2)]} + + """ + if self.verbose: + logger = logging.getLogger(__name__) + + if len(extensions) == 0: + if self.verbose: + logger.error("Expected at list one extension, but none was received.") + raise ValueError + + if self.verbose: + logger.info("Constructing the list of images.") + additional_pattern = '/**/*' if recursive else '/*' + files = [] + for extension in extensions: + files.extend(glob.glob(path + additional_pattern + extension, recursive=recursive)) + + if self.verbose: + logger.info("Finished searching for images. %s images found", len(files)) + logger.info("Preparing to run the detection.") + + predictions = {} + for image_path in tqdm(files, disable=not show_progress_bar): + if self.verbose: + logger.info("Running the face detector on image: %s", image_path) + predictions[image_path] = self.detect_from_image(image_path) + + if self.verbose: + logger.info("The detector was successfully run on all %s images", len(files)) + + return predictions + + @property + def reference_scale(self): + raise NotImplementedError + + @property + def reference_x_shift(self): + raise NotImplementedError + + @property + def reference_y_shift(self): + raise NotImplementedError + + @staticmethod + def tensor_or_path_to_ndarray(tensor_or_path, rgb=True): + """Convert path (represented as a string) or torch.tensor to a numpy.ndarray + + Arguments: + tensor_or_path {numpy.ndarray, torch.tensor or string} -- path to the image, or the image itself + """ + if isinstance(tensor_or_path, str): + return cv2.imread(tensor_or_path) if not rgb else cv2.imread(tensor_or_path)[..., ::-1] + elif torch.is_tensor(tensor_or_path): + # Call cpu in case its coming from cuda + return tensor_or_path.cpu().numpy()[..., ::-1].copy() if not rgb else tensor_or_path.cpu().numpy() + elif isinstance(tensor_or_path, np.ndarray): + return tensor_or_path[..., ::-1].copy() if not rgb else tensor_or_path + else: + raise TypeError diff --git a/models/Wav2Lip/face_detection/detection/sfd/__init__.py b/models/Wav2Lip/face_detection/detection/sfd/__init__.py new file mode 100644 index 0000000000..5a63ecd456 --- /dev/null +++ b/models/Wav2Lip/face_detection/detection/sfd/__init__.py @@ -0,0 +1 @@ +from .sfd_detector import SFDDetector as FaceDetector \ No newline at end of file diff --git a/models/Wav2Lip/face_detection/detection/sfd/bbox.py b/models/Wav2Lip/face_detection/detection/sfd/bbox.py new file mode 100644 index 0000000000..4bd7222e5e --- /dev/null +++ b/models/Wav2Lip/face_detection/detection/sfd/bbox.py @@ -0,0 +1,129 @@ +from __future__ import print_function +import os +import sys +import cv2 +import random +import datetime +import time +import math +import argparse +import numpy as np +import torch + +try: + from iou import IOU +except BaseException: + # IOU cython speedup 10x + def IOU(ax1, ay1, ax2, ay2, bx1, by1, bx2, by2): + sa = abs((ax2 - ax1) * (ay2 - ay1)) + sb = abs((bx2 - bx1) * (by2 - by1)) + x1, y1 = max(ax1, bx1), max(ay1, by1) + x2, y2 = min(ax2, bx2), min(ay2, by2) + w = x2 - x1 + h = y2 - y1 + if w < 0 or h < 0: + return 0.0 + else: + return 1.0 * w * h / (sa + sb - w * h) + + +def bboxlog(x1, y1, x2, y2, axc, ayc, aww, ahh): + xc, yc, ww, hh = (x2 + x1) / 2, (y2 + y1) / 2, x2 - x1, y2 - y1 + dx, dy = (xc - axc) / aww, (yc - ayc) / ahh + dw, dh = math.log(ww / aww), math.log(hh / ahh) + return dx, dy, dw, dh + + +def bboxloginv(dx, dy, dw, dh, axc, ayc, aww, ahh): + xc, yc = dx * aww + axc, dy * ahh + ayc + ww, hh = math.exp(dw) * aww, math.exp(dh) * ahh + x1, x2, y1, y2 = xc - ww / 2, xc + ww / 2, yc - hh / 2, yc + hh / 2 + return x1, y1, x2, y2 + + +def nms(dets, thresh): + if 0 == len(dets): + return [] + x1, y1, x2, y2, scores = dets[:, 0], dets[:, 1], dets[:, 2], dets[:, 3], dets[:, 4] + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1, yy1 = np.maximum(x1[i], x1[order[1:]]), np.maximum(y1[i], y1[order[1:]]) + xx2, yy2 = np.minimum(x2[i], x2[order[1:]]), np.minimum(y2[i], y2[order[1:]]) + + w, h = np.maximum(0.0, xx2 - xx1 + 1), np.maximum(0.0, yy2 - yy1 + 1) + ovr = w * h / (areas[i] + areas[order[1:]] - w * h) + + inds = np.where(ovr <= thresh)[0] + order = order[inds + 1] + + return keep + + +def encode(matched, priors, variances): + """Encode the variances from the priorbox layers into the ground truth boxes + we have matched (based on jaccard overlap) with the prior boxes. + Args: + matched: (tensor) Coords of ground truth for each prior in point-form + Shape: [num_priors, 4]. + priors: (tensor) Prior boxes in center-offset form + Shape: [num_priors,4]. + variances: (list[float]) Variances of priorboxes + Return: + encoded boxes (tensor), Shape: [num_priors, 4] + """ + + # dist b/t match center and prior's center + g_cxcy = (matched[:, :2] + matched[:, 2:]) / 2 - priors[:, :2] + # encode variance + g_cxcy /= (variances[0] * priors[:, 2:]) + # match wh / prior wh + g_wh = (matched[:, 2:] - matched[:, :2]) / priors[:, 2:] + g_wh = torch.log(g_wh) / variances[1] + # return target for smooth_l1_loss + return torch.cat([g_cxcy, g_wh], 1) # [num_priors,4] + + +def decode(loc, priors, variances): + """Decode locations from predictions using priors to undo + the encoding we did for offset regression at train time. + Args: + loc (tensor): location predictions for loc layers, + Shape: [num_priors,4] + priors (tensor): Prior boxes in center-offset form. + Shape: [num_priors,4]. + variances: (list[float]) Variances of priorboxes + Return: + decoded bounding box predictions + """ + + boxes = torch.cat(( + priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:], + priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1) + boxes[:, :2] -= boxes[:, 2:] / 2 + boxes[:, 2:] += boxes[:, :2] + return boxes + +def batch_decode(loc, priors, variances): + """Decode locations from predictions using priors to undo + the encoding we did for offset regression at train time. + Args: + loc (tensor): location predictions for loc layers, + Shape: [num_priors,4] + priors (tensor): Prior boxes in center-offset form. + Shape: [num_priors,4]. + variances: (list[float]) Variances of priorboxes + Return: + decoded bounding box predictions + """ + + boxes = torch.cat(( + priors[:, :, :2] + loc[:, :, :2] * variances[0] * priors[:, :, 2:], + priors[:, :, 2:] * torch.exp(loc[:, :, 2:] * variances[1])), 2) + boxes[:, :, :2] -= boxes[:, :, 2:] / 2 + boxes[:, :, 2:] += boxes[:, :, :2] + return boxes diff --git a/models/Wav2Lip/face_detection/detection/sfd/detect.py b/models/Wav2Lip/face_detection/detection/sfd/detect.py new file mode 100644 index 0000000000..efef6273ad --- /dev/null +++ b/models/Wav2Lip/face_detection/detection/sfd/detect.py @@ -0,0 +1,112 @@ +import torch +import torch.nn.functional as F + +import os +import sys +import cv2 +import random +import datetime +import math +import argparse +import numpy as np + +import scipy.io as sio +import zipfile +from .net_s3fd import s3fd +from .bbox import * + + +def detect(net, img, device): + img = img - np.array([104, 117, 123]) + img = img.transpose(2, 0, 1) + img = img.reshape((1,) + img.shape) + + if 'cuda' in device: + torch.backends.cudnn.benchmark = True + + img = torch.from_numpy(img).float().to(device) + BB, CC, HH, WW = img.size() + with torch.no_grad(): + olist = net(img) + + bboxlist = [] + for i in range(len(olist) // 2): + olist[i * 2] = F.softmax(olist[i * 2], dim=1) + olist = [oelem.data.cpu() for oelem in olist] + for i in range(len(olist) // 2): + ocls, oreg = olist[i * 2], olist[i * 2 + 1] + FB, FC, FH, FW = ocls.size() # feature map size + stride = 2**(i + 2) # 4,8,16,32,64,128 + anchor = stride * 4 + poss = zip(*np.where(ocls[:, 1, :, :] > 0.05)) + for Iindex, hindex, windex in poss: + axc, ayc = stride / 2 + windex * stride, stride / 2 + hindex * stride + score = ocls[0, 1, hindex, windex] + loc = oreg[0, :, hindex, windex].contiguous().view(1, 4) + priors = torch.Tensor([[axc / 1.0, ayc / 1.0, stride * 4 / 1.0, stride * 4 / 1.0]]) + variances = [0.1, 0.2] + box = decode(loc, priors, variances) + x1, y1, x2, y2 = box[0] * 1.0 + # cv2.rectangle(imgshow,(int(x1),int(y1)),(int(x2),int(y2)),(0,0,255),1) + bboxlist.append([x1, y1, x2, y2, score]) + bboxlist = np.array(bboxlist) + if 0 == len(bboxlist): + bboxlist = np.zeros((1, 5)) + + return bboxlist + +def batch_detect(net, imgs, device): + imgs = imgs - np.array([104, 117, 123]) + imgs = imgs.transpose(0, 3, 1, 2) + + if 'cuda' in device: + torch.backends.cudnn.benchmark = True + + imgs = torch.from_numpy(imgs).float().to(device) + BB, CC, HH, WW = imgs.size() + with torch.no_grad(): + olist = net(imgs) + + bboxlist = [] + for i in range(len(olist) // 2): + olist[i * 2] = F.softmax(olist[i * 2], dim=1) + olist = [oelem.data.cpu() for oelem in olist] + for i in range(len(olist) // 2): + ocls, oreg = olist[i * 2], olist[i * 2 + 1] + FB, FC, FH, FW = ocls.size() # feature map size + stride = 2**(i + 2) # 4,8,16,32,64,128 + anchor = stride * 4 + poss = zip(*np.where(ocls[:, 1, :, :] > 0.05)) + for Iindex, hindex, windex in poss: + axc, ayc = stride / 2 + windex * stride, stride / 2 + hindex * stride + score = ocls[:, 1, hindex, windex] + loc = oreg[:, :, hindex, windex].contiguous().view(BB, 1, 4) + priors = torch.Tensor([[axc / 1.0, ayc / 1.0, stride * 4 / 1.0, stride * 4 / 1.0]]).view(1, 1, 4) + variances = [0.1, 0.2] + box = batch_decode(loc, priors, variances) + box = box[:, 0] * 1.0 + # cv2.rectangle(imgshow,(int(x1),int(y1)),(int(x2),int(y2)),(0,0,255),1) + bboxlist.append(torch.cat([box, score.unsqueeze(1)], 1).cpu().numpy()) + bboxlist = np.array(bboxlist) + if 0 == len(bboxlist): + bboxlist = np.zeros((1, BB, 5)) + + return bboxlist + +def flip_detect(net, img, device): + img = cv2.flip(img, 1) + b = detect(net, img, device) + + bboxlist = np.zeros(b.shape) + bboxlist[:, 0] = img.shape[1] - b[:, 2] + bboxlist[:, 1] = b[:, 1] + bboxlist[:, 2] = img.shape[1] - b[:, 0] + bboxlist[:, 3] = b[:, 3] + bboxlist[:, 4] = b[:, 4] + return bboxlist + + +def pts_to_bb(pts): + min_x, min_y = np.min(pts, axis=0) + max_x, max_y = np.max(pts, axis=0) + return np.array([min_x, min_y, max_x, max_y]) diff --git a/models/Wav2Lip/face_detection/detection/sfd/net_s3fd.py b/models/Wav2Lip/face_detection/detection/sfd/net_s3fd.py new file mode 100644 index 0000000000..fc64313c27 --- /dev/null +++ b/models/Wav2Lip/face_detection/detection/sfd/net_s3fd.py @@ -0,0 +1,129 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class L2Norm(nn.Module): + def __init__(self, n_channels, scale=1.0): + super(L2Norm, self).__init__() + self.n_channels = n_channels + self.scale = scale + self.eps = 1e-10 + self.weight = nn.Parameter(torch.Tensor(self.n_channels)) + self.weight.data *= 0.0 + self.weight.data += self.scale + + def forward(self, x): + norm = x.pow(2).sum(dim=1, keepdim=True).sqrt() + self.eps + x = x / norm * self.weight.view(1, -1, 1, 1) + return x + + +class s3fd(nn.Module): + def __init__(self): + super(s3fd, self).__init__() + self.conv1_1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1) + self.conv1_2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) + + self.conv2_1 = nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1) + self.conv2_2 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1) + + self.conv3_1 = nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1) + self.conv3_2 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1) + self.conv3_3 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1) + + self.conv4_1 = nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=1) + self.conv4_2 = nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1) + self.conv4_3 = nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1) + + self.conv5_1 = nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1) + self.conv5_2 = nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1) + self.conv5_3 = nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1) + + self.fc6 = nn.Conv2d(512, 1024, kernel_size=3, stride=1, padding=3) + self.fc7 = nn.Conv2d(1024, 1024, kernel_size=1, stride=1, padding=0) + + self.conv6_1 = nn.Conv2d(1024, 256, kernel_size=1, stride=1, padding=0) + self.conv6_2 = nn.Conv2d(256, 512, kernel_size=3, stride=2, padding=1) + + self.conv7_1 = nn.Conv2d(512, 128, kernel_size=1, stride=1, padding=0) + self.conv7_2 = nn.Conv2d(128, 256, kernel_size=3, stride=2, padding=1) + + self.conv3_3_norm = L2Norm(256, scale=10) + self.conv4_3_norm = L2Norm(512, scale=8) + self.conv5_3_norm = L2Norm(512, scale=5) + + self.conv3_3_norm_mbox_conf = nn.Conv2d(256, 4, kernel_size=3, stride=1, padding=1) + self.conv3_3_norm_mbox_loc = nn.Conv2d(256, 4, kernel_size=3, stride=1, padding=1) + self.conv4_3_norm_mbox_conf = nn.Conv2d(512, 2, kernel_size=3, stride=1, padding=1) + self.conv4_3_norm_mbox_loc = nn.Conv2d(512, 4, kernel_size=3, stride=1, padding=1) + self.conv5_3_norm_mbox_conf = nn.Conv2d(512, 2, kernel_size=3, stride=1, padding=1) + self.conv5_3_norm_mbox_loc = nn.Conv2d(512, 4, kernel_size=3, stride=1, padding=1) + + self.fc7_mbox_conf = nn.Conv2d(1024, 2, kernel_size=3, stride=1, padding=1) + self.fc7_mbox_loc = nn.Conv2d(1024, 4, kernel_size=3, stride=1, padding=1) + self.conv6_2_mbox_conf = nn.Conv2d(512, 2, kernel_size=3, stride=1, padding=1) + self.conv6_2_mbox_loc = nn.Conv2d(512, 4, kernel_size=3, stride=1, padding=1) + self.conv7_2_mbox_conf = nn.Conv2d(256, 2, kernel_size=3, stride=1, padding=1) + self.conv7_2_mbox_loc = nn.Conv2d(256, 4, kernel_size=3, stride=1, padding=1) + + def forward(self, x): + h = F.relu(self.conv1_1(x)) + h = F.relu(self.conv1_2(h)) + h = F.max_pool2d(h, 2, 2) + + h = F.relu(self.conv2_1(h)) + h = F.relu(self.conv2_2(h)) + h = F.max_pool2d(h, 2, 2) + + h = F.relu(self.conv3_1(h)) + h = F.relu(self.conv3_2(h)) + h = F.relu(self.conv3_3(h)) + f3_3 = h + h = F.max_pool2d(h, 2, 2) + + h = F.relu(self.conv4_1(h)) + h = F.relu(self.conv4_2(h)) + h = F.relu(self.conv4_3(h)) + f4_3 = h + h = F.max_pool2d(h, 2, 2) + + h = F.relu(self.conv5_1(h)) + h = F.relu(self.conv5_2(h)) + h = F.relu(self.conv5_3(h)) + f5_3 = h + h = F.max_pool2d(h, 2, 2) + + h = F.relu(self.fc6(h)) + h = F.relu(self.fc7(h)) + ffc7 = h + h = F.relu(self.conv6_1(h)) + h = F.relu(self.conv6_2(h)) + f6_2 = h + h = F.relu(self.conv7_1(h)) + h = F.relu(self.conv7_2(h)) + f7_2 = h + + f3_3 = self.conv3_3_norm(f3_3) + f4_3 = self.conv4_3_norm(f4_3) + f5_3 = self.conv5_3_norm(f5_3) + + cls1 = self.conv3_3_norm_mbox_conf(f3_3) + reg1 = self.conv3_3_norm_mbox_loc(f3_3) + cls2 = self.conv4_3_norm_mbox_conf(f4_3) + reg2 = self.conv4_3_norm_mbox_loc(f4_3) + cls3 = self.conv5_3_norm_mbox_conf(f5_3) + reg3 = self.conv5_3_norm_mbox_loc(f5_3) + cls4 = self.fc7_mbox_conf(ffc7) + reg4 = self.fc7_mbox_loc(ffc7) + cls5 = self.conv6_2_mbox_conf(f6_2) + reg5 = self.conv6_2_mbox_loc(f6_2) + cls6 = self.conv7_2_mbox_conf(f7_2) + reg6 = self.conv7_2_mbox_loc(f7_2) + + # max-out background label + chunk = torch.chunk(cls1, 4, 1) + bmax = torch.max(torch.max(chunk[0], chunk[1]), chunk[2]) + cls1 = torch.cat([bmax, chunk[3]], dim=1) + + return [cls1, reg1, cls2, reg2, cls3, reg3, cls4, reg4, cls5, reg5, cls6, reg6] diff --git a/models/Wav2Lip/face_detection/detection/sfd/sfd_detector.py b/models/Wav2Lip/face_detection/detection/sfd/sfd_detector.py new file mode 100644 index 0000000000..8fbce15253 --- /dev/null +++ b/models/Wav2Lip/face_detection/detection/sfd/sfd_detector.py @@ -0,0 +1,59 @@ +import os +import cv2 +from torch.utils.model_zoo import load_url + +from ..core import FaceDetector + +from .net_s3fd import s3fd +from .bbox import * +from .detect import * + +models_urls = { + 's3fd': 'https://www.adrianbulat.com/downloads/python-fan/s3fd-619a316812.pth', +} + + +class SFDDetector(FaceDetector): + def __init__(self, device, path_to_detector=os.path.join(os.path.dirname(os.path.abspath(__file__)), 's3fd.pth'), verbose=False): + super(SFDDetector, self).__init__(device, verbose) + + # Initialise the face detector + if not os.path.isfile(path_to_detector): + model_weights = load_url(models_urls['s3fd']) + else: + model_weights = torch.load(path_to_detector) + + self.face_detector = s3fd() + self.face_detector.load_state_dict(model_weights) + self.face_detector.to(device) + self.face_detector.eval() + + def detect_from_image(self, tensor_or_path): + image = self.tensor_or_path_to_ndarray(tensor_or_path) + + bboxlist = detect(self.face_detector, image, device=self.device) + keep = nms(bboxlist, 0.3) + bboxlist = bboxlist[keep, :] + bboxlist = [x for x in bboxlist if x[-1] > 0.5] + + return bboxlist + + def detect_from_batch(self, images): + bboxlists = batch_detect(self.face_detector, images, device=self.device) + keeps = [nms(bboxlists[:, i, :], 0.3) for i in range(bboxlists.shape[1])] + bboxlists = [bboxlists[keep, i, :] for i, keep in enumerate(keeps)] + bboxlists = [[x for x in bboxlist if x[-1] > 0.5] for bboxlist in bboxlists] + + return bboxlists + + @property + def reference_scale(self): + return 195 + + @property + def reference_x_shift(self): + return 0 + + @property + def reference_y_shift(self): + return 0 diff --git a/models/Wav2Lip/face_detection/models.py b/models/Wav2Lip/face_detection/models.py new file mode 100644 index 0000000000..ee2dde32bd --- /dev/null +++ b/models/Wav2Lip/face_detection/models.py @@ -0,0 +1,261 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import math + + +def conv3x3(in_planes, out_planes, strd=1, padding=1, bias=False): + "3x3 convolution with padding" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, + stride=strd, padding=padding, bias=bias) + + +class ConvBlock(nn.Module): + def __init__(self, in_planes, out_planes): + super(ConvBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.conv1 = conv3x3(in_planes, int(out_planes / 2)) + self.bn2 = nn.BatchNorm2d(int(out_planes / 2)) + self.conv2 = conv3x3(int(out_planes / 2), int(out_planes / 4)) + self.bn3 = nn.BatchNorm2d(int(out_planes / 4)) + self.conv3 = conv3x3(int(out_planes / 4), int(out_planes / 4)) + + if in_planes != out_planes: + self.downsample = nn.Sequential( + nn.BatchNorm2d(in_planes), + nn.ReLU(True), + nn.Conv2d(in_planes, out_planes, + kernel_size=1, stride=1, bias=False), + ) + else: + self.downsample = None + + def forward(self, x): + residual = x + + out1 = self.bn1(x) + out1 = F.relu(out1, True) + out1 = self.conv1(out1) + + out2 = self.bn2(out1) + out2 = F.relu(out2, True) + out2 = self.conv2(out2) + + out3 = self.bn3(out2) + out3 = F.relu(out3, True) + out3 = self.conv3(out3) + + out3 = torch.cat((out1, out2, out3), 1) + + if self.downsample is not None: + residual = self.downsample(residual) + + out3 += residual + + return out3 + + +class Bottleneck(nn.Module): + + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * 4) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class HourGlass(nn.Module): + def __init__(self, num_modules, depth, num_features): + super(HourGlass, self).__init__() + self.num_modules = num_modules + self.depth = depth + self.features = num_features + + self._generate_network(self.depth) + + def _generate_network(self, level): + self.add_module('b1_' + str(level), ConvBlock(self.features, self.features)) + + self.add_module('b2_' + str(level), ConvBlock(self.features, self.features)) + + if level > 1: + self._generate_network(level - 1) + else: + self.add_module('b2_plus_' + str(level), ConvBlock(self.features, self.features)) + + self.add_module('b3_' + str(level), ConvBlock(self.features, self.features)) + + def _forward(self, level, inp): + # Upper branch + up1 = inp + up1 = self._modules['b1_' + str(level)](up1) + + # Lower branch + low1 = F.avg_pool2d(inp, 2, stride=2) + low1 = self._modules['b2_' + str(level)](low1) + + if level > 1: + low2 = self._forward(level - 1, low1) + else: + low2 = low1 + low2 = self._modules['b2_plus_' + str(level)](low2) + + low3 = low2 + low3 = self._modules['b3_' + str(level)](low3) + + up2 = F.interpolate(low3, scale_factor=2, mode='nearest') + + return up1 + up2 + + def forward(self, x): + return self._forward(self.depth, x) + + +class FAN(nn.Module): + + def __init__(self, num_modules=1): + super(FAN, self).__init__() + self.num_modules = num_modules + + # Base part + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3) + self.bn1 = nn.BatchNorm2d(64) + self.conv2 = ConvBlock(64, 128) + self.conv3 = ConvBlock(128, 128) + self.conv4 = ConvBlock(128, 256) + + # Stacking part + for hg_module in range(self.num_modules): + self.add_module('m' + str(hg_module), HourGlass(1, 4, 256)) + self.add_module('top_m_' + str(hg_module), ConvBlock(256, 256)) + self.add_module('conv_last' + str(hg_module), + nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0)) + self.add_module('bn_end' + str(hg_module), nn.BatchNorm2d(256)) + self.add_module('l' + str(hg_module), nn.Conv2d(256, + 68, kernel_size=1, stride=1, padding=0)) + + if hg_module < self.num_modules - 1: + self.add_module( + 'bl' + str(hg_module), nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0)) + self.add_module('al' + str(hg_module), nn.Conv2d(68, + 256, kernel_size=1, stride=1, padding=0)) + + def forward(self, x): + x = F.relu(self.bn1(self.conv1(x)), True) + x = F.avg_pool2d(self.conv2(x), 2, stride=2) + x = self.conv3(x) + x = self.conv4(x) + + previous = x + + outputs = [] + for i in range(self.num_modules): + hg = self._modules['m' + str(i)](previous) + + ll = hg + ll = self._modules['top_m_' + str(i)](ll) + + ll = F.relu(self._modules['bn_end' + str(i)] + (self._modules['conv_last' + str(i)](ll)), True) + + # Predict heatmaps + tmp_out = self._modules['l' + str(i)](ll) + outputs.append(tmp_out) + + if i < self.num_modules - 1: + ll = self._modules['bl' + str(i)](ll) + tmp_out_ = self._modules['al' + str(i)](tmp_out) + previous = previous + ll + tmp_out_ + + return outputs + + +class ResNetDepth(nn.Module): + + def __init__(self, block=Bottleneck, layers=[3, 8, 36, 3], num_classes=68): + self.inplanes = 64 + super(ResNetDepth, self).__init__() + self.conv1 = nn.Conv2d(3 + 68, 64, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + self.avgpool = nn.AvgPool2d(7) + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes * block.expansion, + kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + + return x diff --git a/models/Wav2Lip/face_detection/utils.py b/models/Wav2Lip/face_detection/utils.py new file mode 100644 index 0000000000..3dc4cf3e32 --- /dev/null +++ b/models/Wav2Lip/face_detection/utils.py @@ -0,0 +1,313 @@ +from __future__ import print_function +import os +import sys +import time +import torch +import math +import numpy as np +import cv2 + + +def _gaussian( + size=3, sigma=0.25, amplitude=1, normalize=False, width=None, + height=None, sigma_horz=None, sigma_vert=None, mean_horz=0.5, + mean_vert=0.5): + # handle some defaults + if width is None: + width = size + if height is None: + height = size + if sigma_horz is None: + sigma_horz = sigma + if sigma_vert is None: + sigma_vert = sigma + center_x = mean_horz * width + 0.5 + center_y = mean_vert * height + 0.5 + gauss = np.empty((height, width), dtype=np.float32) + # generate kernel + for i in range(height): + for j in range(width): + gauss[i][j] = amplitude * math.exp(-(math.pow((j + 1 - center_x) / ( + sigma_horz * width), 2) / 2.0 + math.pow((i + 1 - center_y) / (sigma_vert * height), 2) / 2.0)) + if normalize: + gauss = gauss / np.sum(gauss) + return gauss + + +def draw_gaussian(image, point, sigma): + # Check if the gaussian is inside + ul = [math.floor(point[0] - 3 * sigma), math.floor(point[1] - 3 * sigma)] + br = [math.floor(point[0] + 3 * sigma), math.floor(point[1] + 3 * sigma)] + if (ul[0] > image.shape[1] or ul[1] > image.shape[0] or br[0] < 1 or br[1] < 1): + return image + size = 6 * sigma + 1 + g = _gaussian(size) + g_x = [int(max(1, -ul[0])), int(min(br[0], image.shape[1])) - int(max(1, ul[0])) + int(max(1, -ul[0]))] + g_y = [int(max(1, -ul[1])), int(min(br[1], image.shape[0])) - int(max(1, ul[1])) + int(max(1, -ul[1]))] + img_x = [int(max(1, ul[0])), int(min(br[0], image.shape[1]))] + img_y = [int(max(1, ul[1])), int(min(br[1], image.shape[0]))] + assert (g_x[0] > 0 and g_y[1] > 0) + image[img_y[0] - 1:img_y[1], img_x[0] - 1:img_x[1] + ] = image[img_y[0] - 1:img_y[1], img_x[0] - 1:img_x[1]] + g[g_y[0] - 1:g_y[1], g_x[0] - 1:g_x[1]] + image[image > 1] = 1 + return image + + +def transform(point, center, scale, resolution, invert=False): + """Generate and affine transformation matrix. + + Given a set of points, a center, a scale and a targer resolution, the + function generates and affine transformation matrix. If invert is ``True`` + it will produce the inverse transformation. + + Arguments: + point {torch.tensor} -- the input 2D point + center {torch.tensor or numpy.array} -- the center around which to perform the transformations + scale {float} -- the scale of the face/object + resolution {float} -- the output resolution + + Keyword Arguments: + invert {bool} -- define wherever the function should produce the direct or the + inverse transformation matrix (default: {False}) + """ + _pt = torch.ones(3) + _pt[0] = point[0] + _pt[1] = point[1] + + h = 200.0 * scale + t = torch.eye(3) + t[0, 0] = resolution / h + t[1, 1] = resolution / h + t[0, 2] = resolution * (-center[0] / h + 0.5) + t[1, 2] = resolution * (-center[1] / h + 0.5) + + if invert: + t = torch.inverse(t) + + new_point = (torch.matmul(t, _pt))[0:2] + + return new_point.int() + + +def crop(image, center, scale, resolution=256.0): + """Center crops an image or set of heatmaps + + Arguments: + image {numpy.array} -- an rgb image + center {numpy.array} -- the center of the object, usually the same as of the bounding box + scale {float} -- scale of the face + + Keyword Arguments: + resolution {float} -- the size of the output cropped image (default: {256.0}) + + Returns: + [type] -- [description] + """ # Crop around the center point + """ Crops the image around the center. Input is expected to be an np.ndarray """ + ul = transform([1, 1], center, scale, resolution, True) + br = transform([resolution, resolution], center, scale, resolution, True) + # pad = math.ceil(torch.norm((ul - br).float()) / 2.0 - (br[0] - ul[0]) / 2.0) + if image.ndim > 2: + newDim = np.array([br[1] - ul[1], br[0] - ul[0], + image.shape[2]], dtype=np.int32) + newImg = np.zeros(newDim, dtype=np.uint8) + else: + newDim = np.array([br[1] - ul[1], br[0] - ul[0]], dtype=np.int) + newImg = np.zeros(newDim, dtype=np.uint8) + ht = image.shape[0] + wd = image.shape[1] + newX = np.array( + [max(1, -ul[0] + 1), min(br[0], wd) - ul[0]], dtype=np.int32) + newY = np.array( + [max(1, -ul[1] + 1), min(br[1], ht) - ul[1]], dtype=np.int32) + oldX = np.array([max(1, ul[0] + 1), min(br[0], wd)], dtype=np.int32) + oldY = np.array([max(1, ul[1] + 1), min(br[1], ht)], dtype=np.int32) + newImg[newY[0] - 1:newY[1], newX[0] - 1:newX[1] + ] = image[oldY[0] - 1:oldY[1], oldX[0] - 1:oldX[1], :] + newImg = cv2.resize(newImg, dsize=(int(resolution), int(resolution)), + interpolation=cv2.INTER_LINEAR) + return newImg + + +def get_preds_fromhm(hm, center=None, scale=None): + """Obtain (x,y) coordinates given a set of N heatmaps. If the center + and the scale is provided the function will return the points also in + the original coordinate frame. + + Arguments: + hm {torch.tensor} -- the predicted heatmaps, of shape [B, N, W, H] + + Keyword Arguments: + center {torch.tensor} -- the center of the bounding box (default: {None}) + scale {float} -- face scale (default: {None}) + """ + max, idx = torch.max( + hm.view(hm.size(0), hm.size(1), hm.size(2) * hm.size(3)), 2) + idx += 1 + preds = idx.view(idx.size(0), idx.size(1), 1).repeat(1, 1, 2).float() + preds[..., 0].apply_(lambda x: (x - 1) % hm.size(3) + 1) + preds[..., 1].add_(-1).div_(hm.size(2)).floor_().add_(1) + + for i in range(preds.size(0)): + for j in range(preds.size(1)): + hm_ = hm[i, j, :] + pX, pY = int(preds[i, j, 0]) - 1, int(preds[i, j, 1]) - 1 + if pX > 0 and pX < 63 and pY > 0 and pY < 63: + diff = torch.FloatTensor( + [hm_[pY, pX + 1] - hm_[pY, pX - 1], + hm_[pY + 1, pX] - hm_[pY - 1, pX]]) + preds[i, j].add_(diff.sign_().mul_(.25)) + + preds.add_(-.5) + + preds_orig = torch.zeros(preds.size()) + if center is not None and scale is not None: + for i in range(hm.size(0)): + for j in range(hm.size(1)): + preds_orig[i, j] = transform( + preds[i, j], center, scale, hm.size(2), True) + + return preds, preds_orig + +def get_preds_fromhm_batch(hm, centers=None, scales=None): + """Obtain (x,y) coordinates given a set of N heatmaps. If the centers + and the scales is provided the function will return the points also in + the original coordinate frame. + + Arguments: + hm {torch.tensor} -- the predicted heatmaps, of shape [B, N, W, H] + + Keyword Arguments: + centers {torch.tensor} -- the centers of the bounding box (default: {None}) + scales {float} -- face scales (default: {None}) + """ + max, idx = torch.max( + hm.view(hm.size(0), hm.size(1), hm.size(2) * hm.size(3)), 2) + idx += 1 + preds = idx.view(idx.size(0), idx.size(1), 1).repeat(1, 1, 2).float() + preds[..., 0].apply_(lambda x: (x - 1) % hm.size(3) + 1) + preds[..., 1].add_(-1).div_(hm.size(2)).floor_().add_(1) + + for i in range(preds.size(0)): + for j in range(preds.size(1)): + hm_ = hm[i, j, :] + pX, pY = int(preds[i, j, 0]) - 1, int(preds[i, j, 1]) - 1 + if pX > 0 and pX < 63 and pY > 0 and pY < 63: + diff = torch.FloatTensor( + [hm_[pY, pX + 1] - hm_[pY, pX - 1], + hm_[pY + 1, pX] - hm_[pY - 1, pX]]) + preds[i, j].add_(diff.sign_().mul_(.25)) + + preds.add_(-.5) + + preds_orig = torch.zeros(preds.size()) + if centers is not None and scales is not None: + for i in range(hm.size(0)): + for j in range(hm.size(1)): + preds_orig[i, j] = transform( + preds[i, j], centers[i], scales[i], hm.size(2), True) + + return preds, preds_orig + +def shuffle_lr(parts, pairs=None): + """Shuffle the points left-right according to the axis of symmetry + of the object. + + Arguments: + parts {torch.tensor} -- a 3D or 4D object containing the + heatmaps. + + Keyword Arguments: + pairs {list of integers} -- [order of the flipped points] (default: {None}) + """ + if pairs is None: + pairs = [16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 27, 28, 29, 30, 35, + 34, 33, 32, 31, 45, 44, 43, 42, 47, 46, 39, 38, 37, 36, 41, + 40, 54, 53, 52, 51, 50, 49, 48, 59, 58, 57, 56, 55, 64, 63, + 62, 61, 60, 67, 66, 65] + if parts.ndimension() == 3: + parts = parts[pairs, ...] + else: + parts = parts[:, pairs, ...] + + return parts + + +def flip(tensor, is_label=False): + """Flip an image or a set of heatmaps left-right + + Arguments: + tensor {numpy.array or torch.tensor} -- [the input image or heatmaps] + + Keyword Arguments: + is_label {bool} -- [denote wherever the input is an image or a set of heatmaps ] (default: {False}) + """ + if not torch.is_tensor(tensor): + tensor = torch.from_numpy(tensor) + + if is_label: + tensor = shuffle_lr(tensor).flip(tensor.ndimension() - 1) + else: + tensor = tensor.flip(tensor.ndimension() - 1) + + return tensor + +# From pyzolib/paths.py (https://bitbucket.org/pyzo/pyzolib/src/tip/paths.py) + + +def appdata_dir(appname=None, roaming=False): + """ appdata_dir(appname=None, roaming=False) + + Get the path to the application directory, where applications are allowed + to write user specific files (e.g. configurations). For non-user specific + data, consider using common_appdata_dir(). + If appname is given, a subdir is appended (and created if necessary). + If roaming is True, will prefer a roaming directory (Windows Vista/7). + """ + + # Define default user directory + userDir = os.getenv('FACEALIGNMENT_USERDIR', None) + if userDir is None: + userDir = os.path.expanduser('~') + if not os.path.isdir(userDir): # pragma: no cover + userDir = '/var/tmp' # issue #54 + + # Get system app data dir + path = None + if sys.platform.startswith('win'): + path1, path2 = os.getenv('LOCALAPPDATA'), os.getenv('APPDATA') + path = (path2 or path1) if roaming else (path1 or path2) + elif sys.platform.startswith('darwin'): + path = os.path.join(userDir, 'Library', 'Application Support') + # On Linux and as fallback + if not (path and os.path.isdir(path)): + path = userDir + + # Maybe we should store things local to the executable (in case of a + # portable distro or a frozen application that wants to be portable) + prefix = sys.prefix + if getattr(sys, 'frozen', None): + prefix = os.path.abspath(os.path.dirname(sys.executable)) + for reldir in ('settings', '../settings'): + localpath = os.path.abspath(os.path.join(prefix, reldir)) + if os.path.isdir(localpath): # pragma: no cover + try: + open(os.path.join(localpath, 'test.write'), 'wb').close() + os.remove(os.path.join(localpath, 'test.write')) + except IOError: + pass # We cannot write in this directory + else: + path = localpath + break + + # Get path specific for this app + if appname: + if path == userDir: + appname = '.' + appname.lstrip('.') # Make it a hidden directory + path = os.path.join(path, appname) + if not os.path.isdir(path): # pragma: no cover + os.mkdir(path) + + # Done + return path diff --git a/models/Wav2Lip/hparams.py b/models/Wav2Lip/hparams.py new file mode 100644 index 0000000000..cc2a0d39c0 --- /dev/null +++ b/models/Wav2Lip/hparams.py @@ -0,0 +1,86 @@ +from tensorflow.contrib.training import HParams +from glob import glob +import os + +def get_image_list(data_root, split): + filelist = [] + + with open('filelists/{}.txt'.format(split)) as f: + for line in f: + line = line.strip() + if ' ' in line: line = line.split()[0] + filelist.append(os.path.join(data_root, line)) + + return filelist + +# Default hyperparameters +hparams = HParams( + num_mels=80, # Number of mel-spectrogram channels and local conditioning dimensionality + # network + rescale=True, # Whether to rescale audio prior to preprocessing + rescaling_max=0.9, # Rescaling value + + # Use LWS (https://github.com/Jonathan-LeRoux/lws) for STFT and phase reconstruction + # It"s preferred to set True to use with https://github.com/r9y9/wavenet_vocoder + # Does not work if n_ffit is not multiple of hop_size!! + use_lws=False, + + n_fft=800, # Extra window size is filled with 0 paddings to match this parameter + hop_size=200, # For 16000Hz, 200 = 12.5 ms (0.0125 * sample_rate) + win_size=800, # For 16000Hz, 800 = 50 ms (If None, win_size = n_fft) (0.05 * sample_rate) + sample_rate=16000, # 16000Hz (corresponding to librispeech) (sox --i ) + + frame_shift_ms=None, # Can replace hop_size parameter. (Recommended: 12.5) + + # Mel and Linear spectrograms normalization/scaling and clipping + signal_normalization=True, + # Whether to normalize mel spectrograms to some predefined range (following below parameters) + allow_clipping_in_normalization=True, # Only relevant if mel_normalization = True + symmetric_mels=True, + # Whether to scale the data to be symmetric around 0. (Also multiplies the output range by 2, + # faster and cleaner convergence) + max_abs_value=4., + # max absolute value of data. If symmetric, data will be [-max, max] else [0, max] (Must not + # be too big to avoid gradient explosion, + # not too small for fast convergence) + # Contribution by @begeekmyfriend + # Spectrogram Pre-Emphasis (Lfilter: Reduce spectrogram noise and helps model certitude + # levels. Also allows for better G&L phase reconstruction) + preemphasize=True, # whether to apply filter + preemphasis=0.97, # filter coefficient. + + # Limits + min_level_db=-100, + ref_level_db=20, + fmin=55, + # Set this to 55 if your speaker is male! if female, 95 should help taking off noise. (To + # test depending on dataset. Pitch info: male~[65, 260], female~[100, 525]) + fmax=7600, # To be increased/reduced depending on data. + + ###################### Our training parameters ################################# + img_size=96, + fps=25, + + batch_size=16, + initial_learning_rate=1e-4, + nepochs=200000000000000000, ### ctrl + c, stop whenever eval loss is consistently greater than train loss for ~10 epochs + num_workers=16, + checkpoint_interval=3000, + eval_interval=3000, + save_optimizer_state=True, + + syncnet_wt=0.0, # is initially zero, will be set automatically to 0.03 later. Leads to faster convergence. + syncnet_batch_size=64, + syncnet_lr=1e-4, + syncnet_eval_interval=10000, + syncnet_checkpoint_interval=10000, + + disc_wt=0.07, + disc_initial_learning_rate=1e-4, +) + + +def hparams_debug_string(): + values = hparams.values() + hp = [" %s: %s" % (name, values[name]) for name in sorted(values) if name != "sentences"] + return "Hyperparameters:\n" + "\n".join(hp) diff --git a/models/Wav2Lip/hq_wav2lip_train.py b/models/Wav2Lip/hq_wav2lip_train.py new file mode 100644 index 0000000000..c384ad9d80 --- /dev/null +++ b/models/Wav2Lip/hq_wav2lip_train.py @@ -0,0 +1,443 @@ +from os.path import dirname, join, basename, isfile +from tqdm import tqdm + +from models import SyncNet_color as SyncNet +from models import Wav2Lip, Wav2Lip_disc_qual +import audio + +import torch +from torch import nn +from torch.nn import functional as F +from torch import optim +import torch.backends.cudnn as cudnn +from torch.utils import data as data_utils +import numpy as np + +from glob import glob + +import os, random, cv2, argparse +from hparams import hparams, get_image_list + +parser = argparse.ArgumentParser(description='Code to train the Wav2Lip model WITH the visual quality discriminator') + +parser.add_argument("--data_root", help="Root folder of the preprocessed LRS2 dataset", required=True, type=str) + +parser.add_argument('--checkpoint_dir', help='Save checkpoints to this directory', required=True, type=str) +parser.add_argument('--syncnet_checkpoint_path', help='Load the pre-trained Expert discriminator', required=True, type=str) + +parser.add_argument('--checkpoint_path', help='Resume generator from this checkpoint', default=None, type=str) +parser.add_argument('--disc_checkpoint_path', help='Resume quality disc from this checkpoint', default=None, type=str) + +args = parser.parse_args() + + +global_step = 0 +global_epoch = 0 +use_cuda = torch.cuda.is_available() +print('use_cuda: {}'.format(use_cuda)) + +syncnet_T = 5 +syncnet_mel_step_size = 16 + +class Dataset(object): + def __init__(self, split): + self.all_videos = get_image_list(args.data_root, split) + + def get_frame_id(self, frame): + return int(basename(frame).split('.')[0]) + + def get_window(self, start_frame): + start_id = self.get_frame_id(start_frame) + vidname = dirname(start_frame) + + window_fnames = [] + for frame_id in range(start_id, start_id + syncnet_T): + frame = join(vidname, '{}.jpg'.format(frame_id)) + if not isfile(frame): + return None + window_fnames.append(frame) + return window_fnames + + def read_window(self, window_fnames): + if window_fnames is None: return None + window = [] + for fname in window_fnames: + img = cv2.imread(fname) + if img is None: + return None + try: + img = cv2.resize(img, (hparams.img_size, hparams.img_size)) + except Exception as e: + return None + + window.append(img) + + return window + + def crop_audio_window(self, spec, start_frame): + if type(start_frame) == int: + start_frame_num = start_frame + else: + start_frame_num = self.get_frame_id(start_frame) + start_idx = int(80. * (start_frame_num / float(hparams.fps))) + + end_idx = start_idx + syncnet_mel_step_size + + return spec[start_idx : end_idx, :] + + def get_segmented_mels(self, spec, start_frame): + mels = [] + assert syncnet_T == 5 + start_frame_num = self.get_frame_id(start_frame) + 1 # 0-indexing ---> 1-indexing + if start_frame_num - 2 < 0: return None + for i in range(start_frame_num, start_frame_num + syncnet_T): + m = self.crop_audio_window(spec, i - 2) + if m.shape[0] != syncnet_mel_step_size: + return None + mels.append(m.T) + + mels = np.asarray(mels) + + return mels + + def prepare_window(self, window): + # 3 x T x H x W + x = np.asarray(window) / 255. + x = np.transpose(x, (3, 0, 1, 2)) + + return x + + def __len__(self): + return len(self.all_videos) + + def __getitem__(self, idx): + while 1: + idx = random.randint(0, len(self.all_videos) - 1) + vidname = self.all_videos[idx] + img_names = list(glob(join(vidname, '*.jpg'))) + if len(img_names) <= 3 * syncnet_T: + continue + + img_name = random.choice(img_names) + wrong_img_name = random.choice(img_names) + while wrong_img_name == img_name: + wrong_img_name = random.choice(img_names) + + window_fnames = self.get_window(img_name) + wrong_window_fnames = self.get_window(wrong_img_name) + if window_fnames is None or wrong_window_fnames is None: + continue + + window = self.read_window(window_fnames) + if window is None: + continue + + wrong_window = self.read_window(wrong_window_fnames) + if wrong_window is None: + continue + + try: + wavpath = join(vidname, "audio.wav") + wav = audio.load_wav(wavpath, hparams.sample_rate) + + orig_mel = audio.melspectrogram(wav).T + except Exception as e: + continue + + mel = self.crop_audio_window(orig_mel.copy(), img_name) + + if (mel.shape[0] != syncnet_mel_step_size): + continue + + indiv_mels = self.get_segmented_mels(orig_mel.copy(), img_name) + if indiv_mels is None: continue + + window = self.prepare_window(window) + y = window.copy() + window[:, :, window.shape[2]//2:] = 0. + + wrong_window = self.prepare_window(wrong_window) + x = np.concatenate([window, wrong_window], axis=0) + + x = torch.FloatTensor(x) + mel = torch.FloatTensor(mel.T).unsqueeze(0) + indiv_mels = torch.FloatTensor(indiv_mels).unsqueeze(1) + y = torch.FloatTensor(y) + return x, indiv_mels, mel, y + +def save_sample_images(x, g, gt, global_step, checkpoint_dir): + x = (x.detach().cpu().numpy().transpose(0, 2, 3, 4, 1) * 255.).astype(np.uint8) + g = (g.detach().cpu().numpy().transpose(0, 2, 3, 4, 1) * 255.).astype(np.uint8) + gt = (gt.detach().cpu().numpy().transpose(0, 2, 3, 4, 1) * 255.).astype(np.uint8) + + refs, inps = x[..., 3:], x[..., :3] + folder = join(checkpoint_dir, "samples_step{:09d}".format(global_step)) + if not os.path.exists(folder): os.mkdir(folder) + collage = np.concatenate((refs, inps, g, gt), axis=-2) + for batch_idx, c in enumerate(collage): + for t in range(len(c)): + cv2.imwrite('{}/{}_{}.jpg'.format(folder, batch_idx, t), c[t]) + +logloss = nn.BCELoss() +def cosine_loss(a, v, y): + d = nn.functional.cosine_similarity(a, v) + loss = logloss(d.unsqueeze(1), y) + + return loss + +device = torch.device("cuda" if use_cuda else "cpu") +syncnet = SyncNet().to(device) +for p in syncnet.parameters(): + p.requires_grad = False + +recon_loss = nn.L1Loss() +def get_sync_loss(mel, g): + g = g[:, :, :, g.size(3)//2:] + g = torch.cat([g[:, :, i] for i in range(syncnet_T)], dim=1) + # B, 3 * T, H//2, W + a, v = syncnet(mel, g) + y = torch.ones(g.size(0), 1).float().to(device) + return cosine_loss(a, v, y) + +def train(device, model, disc, train_data_loader, test_data_loader, optimizer, disc_optimizer, + checkpoint_dir=None, checkpoint_interval=None, nepochs=None): + global global_step, global_epoch + resumed_step = global_step + + while global_epoch < nepochs: + print('Starting Epoch: {}'.format(global_epoch)) + running_sync_loss, running_l1_loss, disc_loss, running_perceptual_loss = 0., 0., 0., 0. + running_disc_real_loss, running_disc_fake_loss = 0., 0. + prog_bar = tqdm(enumerate(train_data_loader)) + for step, (x, indiv_mels, mel, gt) in prog_bar: + disc.train() + model.train() + + x = x.to(device) + mel = mel.to(device) + indiv_mels = indiv_mels.to(device) + gt = gt.to(device) + + ### Train generator now. Remove ALL grads. + optimizer.zero_grad() + disc_optimizer.zero_grad() + + g = model(indiv_mels, x) + + if hparams.syncnet_wt > 0.: + sync_loss = get_sync_loss(mel, g) + else: + sync_loss = 0. + + if hparams.disc_wt > 0.: + perceptual_loss = disc.perceptual_forward(g) + else: + perceptual_loss = 0. + + l1loss = recon_loss(g, gt) + + loss = hparams.syncnet_wt * sync_loss + hparams.disc_wt * perceptual_loss + \ + (1. - hparams.syncnet_wt - hparams.disc_wt) * l1loss + + loss.backward() + optimizer.step() + + ### Remove all gradients before Training disc + disc_optimizer.zero_grad() + + pred = disc(gt) + disc_real_loss = F.binary_cross_entropy(pred, torch.ones((len(pred), 1)).to(device)) + disc_real_loss.backward() + + pred = disc(g.detach()) + disc_fake_loss = F.binary_cross_entropy(pred, torch.zeros((len(pred), 1)).to(device)) + disc_fake_loss.backward() + + disc_optimizer.step() + + running_disc_real_loss += disc_real_loss.item() + running_disc_fake_loss += disc_fake_loss.item() + + if global_step % checkpoint_interval == 0: + save_sample_images(x, g, gt, global_step, checkpoint_dir) + + # Logs + global_step += 1 + cur_session_steps = global_step - resumed_step + + running_l1_loss += l1loss.item() + if hparams.syncnet_wt > 0.: + running_sync_loss += sync_loss.item() + else: + running_sync_loss += 0. + + if hparams.disc_wt > 0.: + running_perceptual_loss += perceptual_loss.item() + else: + running_perceptual_loss += 0. + + if global_step == 1 or global_step % checkpoint_interval == 0: + save_checkpoint( + model, optimizer, global_step, checkpoint_dir, global_epoch) + save_checkpoint(disc, disc_optimizer, global_step, checkpoint_dir, global_epoch, prefix='disc_') + + + if global_step % hparams.eval_interval == 0: + with torch.no_grad(): + average_sync_loss = eval_model(test_data_loader, global_step, device, model, disc) + + if average_sync_loss < .75: + hparams.set_hparam('syncnet_wt', 0.03) + + prog_bar.set_description('L1: {}, Sync: {}, Percep: {} | Fake: {}, Real: {}'.format(running_l1_loss / (step + 1), + running_sync_loss / (step + 1), + running_perceptual_loss / (step + 1), + running_disc_fake_loss / (step + 1), + running_disc_real_loss / (step + 1))) + + global_epoch += 1 + +def eval_model(test_data_loader, global_step, device, model, disc): + eval_steps = 300 + print('Evaluating for {} steps'.format(eval_steps)) + running_sync_loss, running_l1_loss, running_disc_real_loss, running_disc_fake_loss, running_perceptual_loss = [], [], [], [], [] + while 1: + for step, (x, indiv_mels, mel, gt) in enumerate((test_data_loader)): + model.eval() + disc.eval() + + x = x.to(device) + mel = mel.to(device) + indiv_mels = indiv_mels.to(device) + gt = gt.to(device) + + pred = disc(gt) + disc_real_loss = F.binary_cross_entropy(pred, torch.ones((len(pred), 1)).to(device)) + + g = model(indiv_mels, x) + pred = disc(g) + disc_fake_loss = F.binary_cross_entropy(pred, torch.zeros((len(pred), 1)).to(device)) + + running_disc_real_loss.append(disc_real_loss.item()) + running_disc_fake_loss.append(disc_fake_loss.item()) + + sync_loss = get_sync_loss(mel, g) + + if hparams.disc_wt > 0.: + perceptual_loss = disc.perceptual_forward(g) + else: + perceptual_loss = 0. + + l1loss = recon_loss(g, gt) + + loss = hparams.syncnet_wt * sync_loss + hparams.disc_wt * perceptual_loss + \ + (1. - hparams.syncnet_wt - hparams.disc_wt) * l1loss + + running_l1_loss.append(l1loss.item()) + running_sync_loss.append(sync_loss.item()) + + if hparams.disc_wt > 0.: + running_perceptual_loss.append(perceptual_loss.item()) + else: + running_perceptual_loss.append(0.) + + if step > eval_steps: break + + print('L1: {}, Sync: {}, Percep: {} | Fake: {}, Real: {}'.format(sum(running_l1_loss) / len(running_l1_loss), + sum(running_sync_loss) / len(running_sync_loss), + sum(running_perceptual_loss) / len(running_perceptual_loss), + sum(running_disc_fake_loss) / len(running_disc_fake_loss), + sum(running_disc_real_loss) / len(running_disc_real_loss))) + return sum(running_sync_loss) / len(running_sync_loss) + + +def save_checkpoint(model, optimizer, step, checkpoint_dir, epoch, prefix=''): + checkpoint_path = join( + checkpoint_dir, "{}checkpoint_step{:09d}.pth".format(prefix, global_step)) + optimizer_state = optimizer.state_dict() if hparams.save_optimizer_state else None + torch.save({ + "state_dict": model.state_dict(), + "optimizer": optimizer_state, + "global_step": step, + "global_epoch": epoch, + }, checkpoint_path) + print("Saved checkpoint:", checkpoint_path) + +def _load(checkpoint_path): + if use_cuda: + checkpoint = torch.load(checkpoint_path) + else: + checkpoint = torch.load(checkpoint_path, + map_location=lambda storage, loc: storage) + return checkpoint + + +def load_checkpoint(path, model, optimizer, reset_optimizer=False, overwrite_global_states=True): + global global_step + global global_epoch + + print("Load checkpoint from: {}".format(path)) + checkpoint = _load(path) + s = checkpoint["state_dict"] + new_s = {} + for k, v in s.items(): + new_s[k.replace('module.', '')] = v + model.load_state_dict(new_s) + if not reset_optimizer: + optimizer_state = checkpoint["optimizer"] + if optimizer_state is not None: + print("Load optimizer state from {}".format(path)) + optimizer.load_state_dict(checkpoint["optimizer"]) + if overwrite_global_states: + global_step = checkpoint["global_step"] + global_epoch = checkpoint["global_epoch"] + + return model + +if __name__ == "__main__": + checkpoint_dir = args.checkpoint_dir + + # Dataset and Dataloader setup + train_dataset = Dataset('train') + test_dataset = Dataset('val') + + train_data_loader = data_utils.DataLoader( + train_dataset, batch_size=hparams.batch_size, shuffle=True, + num_workers=hparams.num_workers) + + test_data_loader = data_utils.DataLoader( + test_dataset, batch_size=hparams.batch_size, + num_workers=4) + + device = torch.device("cuda" if use_cuda else "cpu") + + # Model + model = Wav2Lip().to(device) + disc = Wav2Lip_disc_qual().to(device) + + print('total trainable params {}'.format(sum(p.numel() for p in model.parameters() if p.requires_grad))) + print('total DISC trainable params {}'.format(sum(p.numel() for p in disc.parameters() if p.requires_grad))) + + optimizer = optim.Adam([p for p in model.parameters() if p.requires_grad], + lr=hparams.initial_learning_rate, betas=(0.5, 0.999)) + disc_optimizer = optim.Adam([p for p in disc.parameters() if p.requires_grad], + lr=hparams.disc_initial_learning_rate, betas=(0.5, 0.999)) + + if args.checkpoint_path is not None: + load_checkpoint(args.checkpoint_path, model, optimizer, reset_optimizer=False) + + if args.disc_checkpoint_path is not None: + load_checkpoint(args.disc_checkpoint_path, disc, disc_optimizer, + reset_optimizer=False, overwrite_global_states=False) + + load_checkpoint(args.syncnet_checkpoint_path, syncnet, None, reset_optimizer=True, + overwrite_global_states=False) + + if not os.path.exists(checkpoint_dir): + os.mkdir(checkpoint_dir) + + # Train! + train(device, model, disc, train_data_loader, test_data_loader, optimizer, disc_optimizer, + checkpoint_dir=checkpoint_dir, + checkpoint_interval=hparams.checkpoint_interval, + nepochs=hparams.nepochs) diff --git a/models/Wav2Lip/inference.py b/models/Wav2Lip/inference.py new file mode 100644 index 0000000000..b47f4af24f --- /dev/null +++ b/models/Wav2Lip/inference.py @@ -0,0 +1,286 @@ +from os import listdir, path +import numpy as np +import scipy, cv2, os, sys, argparse, audio +import json, subprocess, random, string +from tqdm import tqdm +from glob import glob +import torch, face_detection +from models import Wav2Lip + +for k, v in os.environ.items(): + print('{}={}'.format(k, v)) + +print('PyTorch version', torch.__version__) +print('CUDA Available?', torch.cuda.is_available()) +print('CUDA Device count', torch.cuda.device_count()) + +parser = argparse.ArgumentParser(description='Inference code to lip-sync videos in the wild using Wav2Lip models') + +parser.add_argument('--checkpoint_path', type=str, + help='Name of saved checkpoint to load weights from', required=True) + +parser.add_argument('--face', type=str, + help='Filepath of video/image that contains faces to use', required=True) +parser.add_argument('--audio', type=str, + help='Filepath of video/audio file to use as raw audio source', required=True) +parser.add_argument('--outfile', type=str, help='Video path to save result. See default for an e.g.', + default='results/result_voice.mp4') + +parser.add_argument('--static', type=bool, + help='If True, then use only first video frame for inference', default=False) +parser.add_argument('--fps', type=float, help='Can be specified only if input is a static image (default: 25)', + default=25., required=False) + +parser.add_argument('--pads', nargs='+', type=int, default=[0, 10, 0, 0], + help='Padding (top, bottom, left, right). Please adjust to include chin at least') + +parser.add_argument('--face_det_batch_size', type=int, + help='Batch size for face detection', default=16) +parser.add_argument('--wav2lip_batch_size', type=int, help='Batch size for Wav2Lip model(s)', default=128) + +parser.add_argument('--resize_factor', default=1, type=int, + help='Reduce the resolution by this factor. Sometimes, best results are obtained at 480p or 720p') + +parser.add_argument('--crop', nargs='+', type=int, default=[0, -1, 0, -1], + help='Crop video to a smaller region (top, bottom, left, right). Applied after resize_factor and rotate arg. ' + 'Useful if multiple face present. -1 implies the value will be auto-inferred based on height, width') + +parser.add_argument('--box', nargs='+', type=int, default=[-1, -1, -1, -1], + help='Specify a constant bounding box for the face. Use only as a last resort if the face is not detected.' + 'Also, might work only if the face is not moving around much. Syntax: (top, bottom, left, right).') + +parser.add_argument('--rotate', default=False, action='store_true', + help='Sometimes videos taken from a phone can be flipped 90deg. If true, will flip video right by 90deg.' + 'Use if you get a flipped result, despite feeding a normal looking video') + +parser.add_argument('--nosmooth', default=False, action='store_true', + help='Prevent smoothing face detections over a short temporal window') + +args = parser.parse_args() +args.img_size = 96 + +if os.path.isfile(args.face) and args.face.split('.')[1] in ['jpg', 'png', 'jpeg']: + args.static = True + +def get_smoothened_boxes(boxes, T): + for i in range(len(boxes)): + if i + T > len(boxes): + window = boxes[len(boxes) - T:] + else: + window = boxes[i : i + T] + boxes[i] = np.mean(window, axis=0) + return boxes + +def face_detect(images): + detector = face_detection.FaceAlignment(face_detection.LandmarksType._2D, + flip_input=False, device=device) + + batch_size = args.face_det_batch_size + + while 1: + predictions = [] + try: + for i in tqdm(range(0, len(images), batch_size)): + predictions.extend(detector.get_detections_for_batch(np.array(images[i:i + batch_size]))) + except RuntimeError: + if batch_size == 1: + raise RuntimeError('Image too big to run face detection on GPU. Please use the --resize_factor argument') + batch_size //= 2 + print('Recovering from OOM error; New batch size: {}'.format(batch_size)) + continue + break + + results = [] + pady1, pady2, padx1, padx2 = args.pads + for rect, image in zip(predictions, images): + if rect is None: + cv2.imwrite('temp/faulty_frame.jpg', image) # check this frame where the face was not detected. + raise ValueError('Face not detected! Ensure the video contains a face in all the frames.') + + y1 = max(0, rect[1] - pady1) + y2 = min(image.shape[0], rect[3] + pady2) + x1 = max(0, rect[0] - padx1) + x2 = min(image.shape[1], rect[2] + padx2) + + results.append([x1, y1, x2, y2]) + + boxes = np.array(results) + if not args.nosmooth: boxes = get_smoothened_boxes(boxes, T=5) + results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2)] for image, (x1, y1, x2, y2) in zip(images, boxes)] + + del detector + return results + +def datagen(frames, mels): + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + if args.box[0] == -1: + if not args.static: + face_det_results = face_detect(frames) # BGR2RGB for CNN face detection + else: + face_det_results = face_detect([frames[0]]) + else: + print('Using the specified bounding box instead of face detection...') + y1, y2, x1, x2 = args.box + face_det_results = [[f[y1: y2, x1:x2], (y1, y2, x1, x2)] for f in frames] + + for i, m in enumerate(mels): + idx = 0 if args.static else i%len(frames) + frame_to_save = frames[idx].copy() + face, coords = face_det_results[idx].copy() + + face = cv2.resize(face, (args.img_size, args.img_size)) + + img_batch.append(face) + mel_batch.append(m) + frame_batch.append(frame_to_save) + coords_batch.append(coords) + + if len(img_batch) >= args.wav2lip_batch_size: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + if len(img_batch) > 0: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + +mel_step_size = 16 +device = 'cuda' if torch.cuda.is_available() else 'cpu' +print('Using {} for inference.'.format(device)) + +def _load(checkpoint_path): + if device == 'cuda': + checkpoint = torch.load(checkpoint_path) + else: + checkpoint = torch.load(checkpoint_path, + map_location=lambda storage, loc: storage) + return checkpoint + +def load_model(path): + model = Wav2Lip() + print("Load checkpoint from: {}".format(path)) + checkpoint = _load(path) + s = checkpoint["state_dict"] + new_s = {} + for k, v in s.items(): + new_s[k.replace('module.', '')] = v + model.load_state_dict(new_s) + + model = model.to(device) + return model.eval() + +def main(): + if not os.path.isfile(args.face): + raise ValueError('--face argument must be a valid path to video/image file') + + elif args.face.split('.')[1] in ['jpg', 'png', 'jpeg']: + full_frames = [cv2.imread(args.face)] + fps = args.fps + + else: + video_stream = cv2.VideoCapture(args.face) + fps = video_stream.get(cv2.CAP_PROP_FPS) + + print('Reading video frames...') + + full_frames = [] + while 1: + still_reading, frame = video_stream.read() + if not still_reading: + video_stream.release() + break + if args.resize_factor > 1: + frame = cv2.resize(frame, (frame.shape[1]//args.resize_factor, frame.shape[0]//args.resize_factor)) + + if args.rotate: + frame = cv2.rotate(frame, cv2.cv2.ROTATE_90_CLOCKWISE) + + y1, y2, x1, x2 = args.crop + if x2 == -1: x2 = frame.shape[1] + if y2 == -1: y2 = frame.shape[0] + + frame = frame[y1:y2, x1:x2] + + full_frames.append(frame) + + print ("Number of frames available for inference: "+str(len(full_frames))) + + if not args.audio.endswith('.wav'): + print('Extracting raw audio...') + command = 'ffmpeg -y -i {} -strict -2 {}'.format(args.audio, 'temp/temp.wav') + + subprocess.call(command, shell=True) + args.audio = 'temp/temp.wav' + + wav = audio.load_wav(args.audio, 16000) + mel = audio.melspectrogram(wav) + print(mel.shape) + + if np.isnan(mel.reshape(-1)).sum() > 0: + raise ValueError('Mel contains nan! Using a TTS voice? Add a small epsilon noise to the wav file and try again') + + mel_chunks = [] + mel_idx_multiplier = 80./fps + i = 0 + while 1: + start_idx = int(i * mel_idx_multiplier) + if start_idx + mel_step_size > len(mel[0]): + mel_chunks.append(mel[:, len(mel[0]) - mel_step_size:]) + break + mel_chunks.append(mel[:, start_idx : start_idx + mel_step_size]) + i += 1 + + print("Length of mel chunks: {}".format(len(mel_chunks))) + + full_frames = full_frames[:len(mel_chunks)] + + batch_size = args.wav2lip_batch_size + gen = datagen(full_frames.copy(), mel_chunks) + + for i, (img_batch, mel_batch, frames, coords) in enumerate(tqdm(gen, + total=int(np.ceil(float(len(mel_chunks))/batch_size)))): + if i == 0: + model = load_model(args.checkpoint_path) + print ("Model loaded") + + frame_h, frame_w = full_frames[0].shape[:-1] + out = cv2.VideoWriter('temp/result.avi', + cv2.VideoWriter_fourcc(*'DIVX'), fps, (frame_w, frame_h)) + + img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device) + mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device) + + with torch.no_grad(): + pred = model(mel_batch, img_batch) + + pred = pred.cpu().numpy().transpose(0, 2, 3, 1) * 255. + + for p, f, c in zip(pred, frames, coords): + y1, y2, x1, x2 = c + p = cv2.resize(p.astype(np.uint8), (x2 - x1, y2 - y1)) + + f[y1:y2, x1:x2] = p + out.write(f) + + out.release() + + command = 'ffmpeg -y -i {} -i {} -strict -2 -q:v 1 {}'.format(args.audio, 'temp/result.avi', args.outfile) + subprocess.call(command, shell=True) + +if __name__ == '__main__': + main() diff --git a/models/Wav2Lip/inference_custom.py b/models/Wav2Lip/inference_custom.py new file mode 100644 index 0000000000..7fd4a2032d --- /dev/null +++ b/models/Wav2Lip/inference_custom.py @@ -0,0 +1,458 @@ +from os import listdir, path +import numpy as np +import scipy, cv2, os, sys, argparse, audio +import json, subprocess, random, string +from tqdm import tqdm +from glob import glob +import torch, face_detection +from models import Wav2Lip +import tempfile +import shutil +import datetime +import pickle +from PIL import Image + +if 'PRINT_ENV' in os.environ and os.environ['PRINT_ENV']: + for k, v in os.environ.items(): + print('{}={}'.format(k, v)) + +print('========================================') +print('Python interpreter', sys.executable) +print('PyTorch version', torch.__version__) +print('CUDA Available?', torch.cuda.is_available()) +print('CUDA Device count', torch.cuda.device_count()) +print('========================================', flush=True) + +parser = argparse.ArgumentParser(description='Inference code to lip-sync videos in the wild using Wav2Lip models') + +parser.add_argument('--checkpoint_path', type=str, + help='Name of saved checkpoint to load weights from', required=True) + +parser.add_argument('--face', type=str, + help='Filepath of video/image that contains faces to use', required=True) +parser.add_argument('--audio', type=str, + help='Filepath of video/audio file to use as raw audio source', required=True) +parser.add_argument('--outfile', type=str, help='Video path to save result. See default for an e.g.', + default='results/result_voice.mp4') + +parser.add_argument('--static', type=bool, + help='If True, then use only first video frame for inference', default=False) + +# NB: I'm adjusting the FPS to match that of the end bump I made. The old default FPS was "25." (float 25) +parser.add_argument('--fps', type=float, help='Can be specified only if input is a static image (default: 29.97)', + default=29.97, required=False) + +parser.add_argument('--pads', nargs='+', type=int, default=[0, 10, 0, 0], + help='Padding (top, bottom, left, right). Please adjust to include chin at least') + +parser.add_argument('--face_det_batch_size', type=int, + help='Batch size for face detection', default=16) +parser.add_argument('--wav2lip_batch_size', type=int, help='Batch size for Wav2Lip model(s)', default=128) + +parser.add_argument('--resize_factor', default=1, type=int, + help='Reduce the resolution by this factor. Sometimes, best results are obtained at 480p or 720p') + +parser.add_argument('--crop', nargs='+', type=int, default=[0, -1, 0, -1], + help='Crop video to a smaller region (top, bottom, left, right). Applied after resize_factor and rotate arg. ' + 'Useful if multiple face present. -1 implies the value will be auto-inferred based on height, width') + +parser.add_argument('--box', nargs='+', type=int, default=[-1, -1, -1, -1], + help='Specify a constant bounding box for the face. Use only as a last resort if the face is not detected.' + 'Also, might work only if the face is not moving around much. Syntax: (top, bottom, left, right).') + +parser.add_argument('--rotate', default=False, action='store_true', + help='Sometimes videos taken from a phone can be flipped 90deg. If true, will flip video right by 90deg.' + 'Use if you get a flipped result, despite feeding a normal looking video') + +parser.add_argument('--nosmooth', default=False, action='store_true', + help='Prevent smoothing face detections over a short temporal window') + +parser.add_argument('--audio_start_pad_millis', default=0, type=int, + help='Seconds to pad the start of the audio') + +parser.add_argument('--audio_end_pad_millis', default=0, type=int, + help='Seconds to pad the end of the audio') + +parser.add_argument('--end_bump_file', type=str, + help='Video file to concatenate at the end') + +# This is a little more deliberate than "static" and causes us to pick an FPS, etc. +# This is useful when the filename doesn't have an extension. +parser.add_argument('--is_image', default=False, action='store_true', + help='Denote that the input is a single-frame image, not a video') + +# Purely for debugging on the host machine: +parser.add_argument('--preserve_tempdir', default=False, action='store_true', + help='Keep the tempdir arround for debugging') + +args = parser.parse_args() +args.img_size = 96 + +if os.path.isfile(args.face) and os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg']: + args.static = True +elif args.is_image: + args.static = True + +def get_smoothened_boxes(boxes, T): + for i in range(len(boxes)): + if i + T > len(boxes): + window = boxes[len(boxes) - T:] + else: + window = boxes[i : i + T] + boxes[i] = np.mean(window, axis=0) + return boxes + +def face_detect(images): + detector = face_detection.FaceAlignment(face_detection.LandmarksType._2D, + flip_input=False, device=device) + + batch_size = args.face_det_batch_size + + while 1: + predictions = [] + try: + for i in tqdm(range(0, len(images), batch_size)): + predictions.extend(detector.get_detections_for_batch(np.array(images[i:i + batch_size]))) + except RuntimeError: + if batch_size == 1: + raise RuntimeError('Image too big to run face detection on GPU. Please use the --resize_factor argument') + batch_size //= 2 + print('Recovering from OOM error; New batch size: {}'.format(batch_size)) + continue + break + + results = [] + pady1, pady2, padx1, padx2 = args.pads + for rect, image in zip(predictions, images): + if rect is None: + cv2.imwrite('temp/faulty_frame.jpg', image) # check this frame where the face was not detected. + raise ValueError('Face not detected! Ensure the video contains a face in all the frames.') + + y1 = max(0, rect[1] - pady1) + y2 = min(image.shape[0], rect[3] + pady2) + x1 = max(0, rect[0] - padx1) + x2 = min(image.shape[1], rect[2] + padx2) + + results.append([x1, y1, x2, y2]) + + boxes = np.array(results) + if not args.nosmooth: boxes = get_smoothened_boxes(boxes, T=5) + results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2)] for image, (x1, y1, x2, y2) in zip(images, boxes)] + + del detector + return results + +def detect_faces_in_frames(frames, video_faces_pickle_file): + t0 = datetime.datetime.now() + if args.box[0] == -1: + if not args.static: + face_det_results = face_detect(frames) # BGR2RGB for CNN face detection + else: + face_det_results = face_detect([frames[0]]) + else: + print('Using the specified bounding box instead of face detection...') + y1, y2, x1, x2 = args.box + face_det_results = [[f[y1: y2, x1:x2], (y1, y2, x1, x2)] for f in frames] + t1 = datetime.datetime.now() + + d = t1 - t0 + print('Total seconds to detect faces in {} frames: {}'.format(len(frames), d.total_seconds())) + + with open(video_faces_pickle_file, 'wb') as f: + print('Saving pickle file: {}'.format(video_faces_pickle_file)) + pickle.dump(face_det_results, f) + + return face_det_results + +#def datagen(frames, mels): +def datagen(frames, face_det_results, mels): + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + #face_det_results = detect_faces_in_frames(frames) + + for i, m in enumerate(mels): + idx = 0 if args.static else i%len(frames) + frame_to_save = frames[idx].copy() + face, coords = face_det_results[idx].copy() + + face = cv2.resize(face, (args.img_size, args.img_size)) + + img_batch.append(face) + mel_batch.append(m) + frame_batch.append(frame_to_save) + coords_batch.append(coords) + + if len(img_batch) >= args.wav2lip_batch_size: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + if len(img_batch) > 0: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + +mel_step_size = 16 +device = 'cuda' if torch.cuda.is_available() else 'cpu' +print('Using {} for inference.'.format(device)) + +def _load(checkpoint_path): + if device == 'cuda': + checkpoint = torch.load(checkpoint_path) + else: + checkpoint = torch.load(checkpoint_path, + map_location=lambda storage, loc: storage) + return checkpoint + +def load_model(path): + model = Wav2Lip() + print("Load checkpoint from: {}".format(path), flush=True) + checkpoint = _load(path) + s = checkpoint["state_dict"] + new_s = {} + for k, v in s.items(): + new_s[k.replace('module.', '')] = v + model.load_state_dict(new_s) + + model = model.to(device) + return model.eval() + +def maybe_pad_audio_file(tempdir, args): + if not args.audio_start_pad_millis and not args.audio_end_pad_millis: + return + + print('Padding audio with {} millis at start, {} millis at end.'.format( + args.audio_start_pad_millis, args.audio_end_pad_millis), flush=True) + + padded_wav_filename = os.path.join(tempdir, 'padded_audio.wav') + # ffmpeg padding: https://superuser.com/a/579110 + # Structure of filters: https://stackoverflow.com/a/55463101 + command = ' '.join([ + 'ffmpeg', + '-i {}'.format(args.audio), + '-filter_complex "[0] adelay={}ms:all=true [a] ; [a] apad=pad_dur={}ms"'.format(args.audio_start_pad_millis, args.audio_end_pad_millis), + '{}'.format(padded_wav_filename) + ]) + print('command:', command, flush=True) + subprocess.call(command, shell=True) + args.audio = padded_wav_filename + + +def maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h): + if not args.end_bump_file: + print('No end bump file.', flush=True) + return + + print('Adding end bump file: {}'.format(args.end_bump_file), flush=True) + + bumpfile = args.end_bump_file + + # May need to resize our bump file... + # After many attempts to get this into the same ffmpeg pipeline, + # I'm giving up and just adding another step. + # + # NOTE: The FPS of the bump needs to match the video. Especially + # important for videos generated from still photos, because that defaults + # to the odd choice of "25 fps". + if frame_w is not 1920 or frame_h is not 1080: + print('Need to resize bump.', flush=True) + bumpfile = os.path.join(tempdir, 'end_bump_resized.mp4') + + # Scale / aspect ratio / sar / resizing / padding: + # https://stackoverflow.com/a/51946719 + command = ' '.join([ + 'ffmpeg', + '-i {}'.format(args.end_bump_file), + '-vf "scale={}:{}:force_original_aspect_ratio=decrease,pad={}:{}:(ow-iw)/2:(oh-ih)/2,setsar=1"'.format(frame_w, frame_h, frame_w, frame_h), + '{}'.format(bumpfile) + ]) + print('command:', command, flush=True) + failure_code = subprocess.call(command, shell=True) + + if failure_code: + print('Failed to resize end bump') + return + + padded_wav_filename = os.path.join(tempdir, 'output_with_end_bump.mp4') + # ffmpeg video concatenation: https://stackoverflow.com/a/11175851 + # Structure of filters: https://stackoverflow.com/a/55463101 + # More info on filters: https://stackoverflow.com/a/22958746 + command = ' '.join([ + 'ffmpeg', + '-i {}'.format(args.outfile), + '-i {}'.format(bumpfile), + '-filter_complex "concat=n=2:v=1:a=1"', + '{}'.format(padded_wav_filename) + ]) + print('command:', command, flush=True) + failure_code = subprocess.call(command, shell=True) + + if failure_code: + print('Failed to concatenate end bump') + else: + print('Done concatenating end bump; renaming file...', flush=True) + os.rename(padded_wav_filename, args.outfile) + + +def main(tempdir): + video_faces_pickle_file = args.face + '.faces' + print('Video faces pickle file: {}'.format(video_faces_pickle_file), flush=True) + + frame_w = 0 + frame_h = 0 + + if not os.path.isfile(args.face): + raise ValueError('--face argument must be a valid path to video/image file') + + elif os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg'] \ + or args.is_image: + full_frames = [cv2.imread(args.face)] + fps = args.fps + frame_h, frame_w = full_frames[0].shape[:-1] + + else: + video_stream = cv2.VideoCapture(args.face) + fps = video_stream.get(cv2.CAP_PROP_FPS) + + print('Reading video frames...') + + full_frames = [] + while 1: + still_reading, frame = video_stream.read() + if not still_reading: + video_stream.release() + break + if args.resize_factor > 1: + frame = cv2.resize(frame, (frame.shape[1]//args.resize_factor, frame.shape[0]//args.resize_factor)) + + if args.rotate: + frame = cv2.rotate(frame, cv2.cv2.ROTATE_90_CLOCKWISE) + + y1, y2, x1, x2 = args.crop + if x2 == -1: x2 = frame.shape[1] + if y2 == -1: y2 = frame.shape[0] + + frame = frame[y1:y2, x1:x2] + + full_frames.append(frame) + + frame_h, frame_w = full_frames[0].shape[:-1] + + print("Number of frames available for inference: "+str(len(full_frames)), flush=True) + print("Frame dimensions: {}x{}".format(frame_w, frame_h), flush=True) + + if not args.audio.endswith('.wav'): + print('Extracting raw audio...') + temp_wav_filename = os.path.join(tempdir, 'temp.wav') # previously 'temp/temp.wav' + command = 'ffmpeg -y -i {} -strict -2 {}'.format(args.audio, temp_wav_filename) + + print('command:', command, flush=True) + subprocess.call(command, shell=True) + args.audio = temp_wav_filename + + maybe_pad_audio_file(tempdir, args) + + wav = audio.load_wav(args.audio, 16000) + mel = audio.melspectrogram(wav) + print(mel.shape) + + if np.isnan(mel.reshape(-1)).sum() > 0: + raise ValueError('Mel contains nan! Using a TTS voice? Add a small epsilon noise to the wav file and try again') + + mel_chunks = [] + mel_idx_multiplier = 80./fps + i = 0 + while 1: + start_idx = int(i * mel_idx_multiplier) + if start_idx + mel_step_size > len(mel[0]): + mel_chunks.append(mel[:, len(mel[0]) - mel_step_size:]) + break + mel_chunks.append(mel[:, start_idx : start_idx + mel_step_size]) + i += 1 + + print("Length of mel chunks: {}".format(len(mel_chunks)), flush=True) + + if len(mel_chunks) > 2000: + # TODO: Catch that the file is too long earlier on. + raise Exception("Too many mel chunks: {}".format(len(mel_chunks))) + + full_frames = full_frames[:len(mel_chunks)] + + # TODO: Do this offline. + print('Face detection time...', flush=True) + if not os.path.isfile(video_faces_pickle_file): + print('Detecting faces...', flush=True) + face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) + else: + with open(video_faces_pickle_file, 'rb') as f: + face_det_results = pickle.load(f) + if len(face_det_results) < len(full_frames): + print("Face detect results have {} frames, but we need {}".format(len(face_det_results), len(full_frames))) + print('Detecting faces again...', flush=True) + face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) + else: + print("We don't need to find faces! Woo!", flush=True) + + face_det_results = face_det_results[:len(mel_chunks)] + + batch_size = args.wav2lip_batch_size + #gen = datagen(full_frames.copy(), mel_chunks) + gen = datagen(full_frames.copy(), face_det_results.copy(), mel_chunks) + + output_video_filename = os.path.join(tempdir, 'result.avi') # previously 'temp/result.avi' + + for i, (img_batch, mel_batch, frames, coords) in enumerate(tqdm(gen, + total=int(np.ceil(float(len(mel_chunks))/batch_size)))): + if i == 0: + model = load_model(args.checkpoint_path) + print ("Model loaded") + + #frame_h, frame_w = full_frames[0].shape[:-1] + out = cv2.VideoWriter(output_video_filename, + cv2.VideoWriter_fourcc(*'DIVX'), fps, (frame_w, frame_h)) + + img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device) + mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device) + + with torch.no_grad(): + pred = model(mel_batch, img_batch) + + pred = pred.cpu().numpy().transpose(0, 2, 3, 1) * 255. + + for p, f, c in zip(pred, frames, coords): + y1, y2, x1, x2 = c + p = cv2.resize(p.astype(np.uint8), (x2 - x1, y2 - y1)) + + f[y1:y2, x1:x2] = p + out.write(f) + + out.release() + + command = 'ffmpeg -y -i {} -i {} -strict -2 -q:v 1 {}'.format(args.audio, output_video_filename, args.outfile) + print('command:', command, flush=True) + subprocess.call(command, shell=True) + + maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h) + +if __name__ == '__main__': + tempdir = tempfile.mkdtemp() + print('Using Python tempdir: {}'.format(tempdir), flush=True) + try: + main(tempdir) + finally: + if not args.preserve_tempdir: + shutil.rmtree(tempdir) + diff --git a/models/Wav2Lip/models/__init__.py b/models/Wav2Lip/models/__init__.py new file mode 100644 index 0000000000..4374370494 --- /dev/null +++ b/models/Wav2Lip/models/__init__.py @@ -0,0 +1,2 @@ +from .wav2lip import Wav2Lip, Wav2Lip_disc_qual +from .syncnet import SyncNet_color \ No newline at end of file diff --git a/models/Wav2Lip/models/conv.py b/models/Wav2Lip/models/conv.py new file mode 100644 index 0000000000..ed83da00cb --- /dev/null +++ b/models/Wav2Lip/models/conv.py @@ -0,0 +1,44 @@ +import torch +from torch import nn +from torch.nn import functional as F + +class Conv2d(nn.Module): + def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, *args, **kwargs): + super().__init__(*args, **kwargs) + self.conv_block = nn.Sequential( + nn.Conv2d(cin, cout, kernel_size, stride, padding), + nn.BatchNorm2d(cout) + ) + self.act = nn.ReLU() + self.residual = residual + + def forward(self, x): + out = self.conv_block(x) + if self.residual: + out += x + return self.act(out) + +class nonorm_Conv2d(nn.Module): + def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, *args, **kwargs): + super().__init__(*args, **kwargs) + self.conv_block = nn.Sequential( + nn.Conv2d(cin, cout, kernel_size, stride, padding), + ) + self.act = nn.LeakyReLU(0.01, inplace=True) + + def forward(self, x): + out = self.conv_block(x) + return self.act(out) + +class Conv2dTranspose(nn.Module): + def __init__(self, cin, cout, kernel_size, stride, padding, output_padding=0, *args, **kwargs): + super().__init__(*args, **kwargs) + self.conv_block = nn.Sequential( + nn.ConvTranspose2d(cin, cout, kernel_size, stride, padding, output_padding), + nn.BatchNorm2d(cout) + ) + self.act = nn.ReLU() + + def forward(self, x): + out = self.conv_block(x) + return self.act(out) diff --git a/models/Wav2Lip/models/syncnet.py b/models/Wav2Lip/models/syncnet.py new file mode 100644 index 0000000000..e773cdca67 --- /dev/null +++ b/models/Wav2Lip/models/syncnet.py @@ -0,0 +1,66 @@ +import torch +from torch import nn +from torch.nn import functional as F + +from .conv import Conv2d + +class SyncNet_color(nn.Module): + def __init__(self): + super(SyncNet_color, self).__init__() + + self.face_encoder = nn.Sequential( + Conv2d(15, 32, kernel_size=(7, 7), stride=1, padding=3), + + Conv2d(32, 64, kernel_size=5, stride=(1, 2), padding=1), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(64, 128, kernel_size=3, stride=2, padding=1), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(128, 256, kernel_size=3, stride=2, padding=1), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(256, 512, kernel_size=3, stride=2, padding=1), + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(512, 512, kernel_size=3, stride=2, padding=1), + Conv2d(512, 512, kernel_size=3, stride=1, padding=0), + Conv2d(512, 512, kernel_size=1, stride=1, padding=0),) + + self.audio_encoder = nn.Sequential( + Conv2d(1, 32, kernel_size=3, stride=1, padding=1), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(32, 64, kernel_size=3, stride=(3, 1), padding=1), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(64, 128, kernel_size=3, stride=3, padding=1), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(128, 256, kernel_size=3, stride=(3, 2), padding=1), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(256, 512, kernel_size=3, stride=1, padding=0), + Conv2d(512, 512, kernel_size=1, stride=1, padding=0),) + + def forward(self, audio_sequences, face_sequences): # audio_sequences := (B, dim, T) + face_embedding = self.face_encoder(face_sequences) + audio_embedding = self.audio_encoder(audio_sequences) + + audio_embedding = audio_embedding.view(audio_embedding.size(0), -1) + face_embedding = face_embedding.view(face_embedding.size(0), -1) + + audio_embedding = F.normalize(audio_embedding, p=2, dim=1) + face_embedding = F.normalize(face_embedding, p=2, dim=1) + + + return audio_embedding, face_embedding diff --git a/models/Wav2Lip/models/wav2lip.py b/models/Wav2Lip/models/wav2lip.py new file mode 100644 index 0000000000..ae5d691916 --- /dev/null +++ b/models/Wav2Lip/models/wav2lip.py @@ -0,0 +1,184 @@ +import torch +from torch import nn +from torch.nn import functional as F +import math + +from .conv import Conv2dTranspose, Conv2d, nonorm_Conv2d + +class Wav2Lip(nn.Module): + def __init__(self): + super(Wav2Lip, self).__init__() + + self.face_encoder_blocks = nn.ModuleList([ + nn.Sequential(Conv2d(6, 16, kernel_size=7, stride=1, padding=3)), # 96,96 + + nn.Sequential(Conv2d(16, 32, kernel_size=3, stride=2, padding=1), # 48,48 + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True)), + + nn.Sequential(Conv2d(32, 64, kernel_size=3, stride=2, padding=1), # 24,24 + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True)), + + nn.Sequential(Conv2d(64, 128, kernel_size=3, stride=2, padding=1), # 12,12 + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True)), + + nn.Sequential(Conv2d(128, 256, kernel_size=3, stride=2, padding=1), # 6,6 + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True)), + + nn.Sequential(Conv2d(256, 512, kernel_size=3, stride=2, padding=1), # 3,3 + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True),), + + nn.Sequential(Conv2d(512, 512, kernel_size=3, stride=1, padding=0), # 1, 1 + Conv2d(512, 512, kernel_size=1, stride=1, padding=0)),]) + + self.audio_encoder = nn.Sequential( + Conv2d(1, 32, kernel_size=3, stride=1, padding=1), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(32, 64, kernel_size=3, stride=(3, 1), padding=1), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(64, 128, kernel_size=3, stride=3, padding=1), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(128, 256, kernel_size=3, stride=(3, 2), padding=1), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(256, 512, kernel_size=3, stride=1, padding=0), + Conv2d(512, 512, kernel_size=1, stride=1, padding=0),) + + self.face_decoder_blocks = nn.ModuleList([ + nn.Sequential(Conv2d(512, 512, kernel_size=1, stride=1, padding=0),), + + nn.Sequential(Conv2dTranspose(1024, 512, kernel_size=3, stride=1, padding=0), # 3,3 + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True),), + + nn.Sequential(Conv2dTranspose(1024, 512, kernel_size=3, stride=2, padding=1, output_padding=1), + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True),), # 6, 6 + + nn.Sequential(Conv2dTranspose(768, 384, kernel_size=3, stride=2, padding=1, output_padding=1), + Conv2d(384, 384, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(384, 384, kernel_size=3, stride=1, padding=1, residual=True),), # 12, 12 + + nn.Sequential(Conv2dTranspose(512, 256, kernel_size=3, stride=2, padding=1, output_padding=1), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True),), # 24, 24 + + nn.Sequential(Conv2dTranspose(320, 128, kernel_size=3, stride=2, padding=1, output_padding=1), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True),), # 48, 48 + + nn.Sequential(Conv2dTranspose(160, 64, kernel_size=3, stride=2, padding=1, output_padding=1), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True),),]) # 96,96 + + self.output_block = nn.Sequential(Conv2d(80, 32, kernel_size=3, stride=1, padding=1), + nn.Conv2d(32, 3, kernel_size=1, stride=1, padding=0), + nn.Sigmoid()) + + def forward(self, audio_sequences, face_sequences): + # audio_sequences = (B, T, 1, 80, 16) + B = audio_sequences.size(0) + + input_dim_size = len(face_sequences.size()) + if input_dim_size > 4: + audio_sequences = torch.cat([audio_sequences[:, i] for i in range(audio_sequences.size(1))], dim=0) + face_sequences = torch.cat([face_sequences[:, :, i] for i in range(face_sequences.size(2))], dim=0) + + audio_embedding = self.audio_encoder(audio_sequences) # B, 512, 1, 1 + + feats = [] + x = face_sequences + for f in self.face_encoder_blocks: + x = f(x) + feats.append(x) + + x = audio_embedding + for f in self.face_decoder_blocks: + x = f(x) + try: + x = torch.cat((x, feats[-1]), dim=1) + except Exception as e: + print(x.size()) + print(feats[-1].size()) + raise e + + feats.pop() + + x = self.output_block(x) + + if input_dim_size > 4: + x = torch.split(x, B, dim=0) # [(B, C, H, W)] + outputs = torch.stack(x, dim=2) # (B, C, T, H, W) + + else: + outputs = x + + return outputs + +class Wav2Lip_disc_qual(nn.Module): + def __init__(self): + super(Wav2Lip_disc_qual, self).__init__() + + self.face_encoder_blocks = nn.ModuleList([ + nn.Sequential(nonorm_Conv2d(3, 32, kernel_size=7, stride=1, padding=3)), # 48,96 + + nn.Sequential(nonorm_Conv2d(32, 64, kernel_size=5, stride=(1, 2), padding=2), # 48,48 + nonorm_Conv2d(64, 64, kernel_size=5, stride=1, padding=2)), + + nn.Sequential(nonorm_Conv2d(64, 128, kernel_size=5, stride=2, padding=2), # 24,24 + nonorm_Conv2d(128, 128, kernel_size=5, stride=1, padding=2)), + + nn.Sequential(nonorm_Conv2d(128, 256, kernel_size=5, stride=2, padding=2), # 12,12 + nonorm_Conv2d(256, 256, kernel_size=5, stride=1, padding=2)), + + nn.Sequential(nonorm_Conv2d(256, 512, kernel_size=3, stride=2, padding=1), # 6,6 + nonorm_Conv2d(512, 512, kernel_size=3, stride=1, padding=1)), + + nn.Sequential(nonorm_Conv2d(512, 512, kernel_size=3, stride=2, padding=1), # 3,3 + nonorm_Conv2d(512, 512, kernel_size=3, stride=1, padding=1),), + + nn.Sequential(nonorm_Conv2d(512, 512, kernel_size=3, stride=1, padding=0), # 1, 1 + nonorm_Conv2d(512, 512, kernel_size=1, stride=1, padding=0)),]) + + self.binary_pred = nn.Sequential(nn.Conv2d(512, 1, kernel_size=1, stride=1, padding=0), nn.Sigmoid()) + self.label_noise = .0 + + def get_lower_half(self, face_sequences): + return face_sequences[:, :, face_sequences.size(2)//2:] + + def to_2d(self, face_sequences): + B = face_sequences.size(0) + face_sequences = torch.cat([face_sequences[:, :, i] for i in range(face_sequences.size(2))], dim=0) + return face_sequences + + def perceptual_forward(self, false_face_sequences): + false_face_sequences = self.to_2d(false_face_sequences) + false_face_sequences = self.get_lower_half(false_face_sequences) + + false_feats = false_face_sequences + for f in self.face_encoder_blocks: + false_feats = f(false_feats) + + false_pred_loss = F.binary_cross_entropy(self.binary_pred(false_feats).view(len(false_feats), -1), + torch.ones((len(false_feats), 1)).cuda()) + + return false_pred_loss + + def forward(self, face_sequences): + face_sequences = self.to_2d(face_sequences) + face_sequences = self.get_lower_half(face_sequences) + + x = face_sequences + for f in self.face_encoder_blocks: + x = f(x) + + return self.binary_pred(x).view(len(x), -1) diff --git a/models/Wav2Lip/preprocess.py b/models/Wav2Lip/preprocess.py new file mode 100644 index 0000000000..5322012ac6 --- /dev/null +++ b/models/Wav2Lip/preprocess.py @@ -0,0 +1,113 @@ +import sys + +if sys.version_info[0] < 3 and sys.version_info[1] < 2: + raise Exception("Must be using >= Python 3.2") + +from os import listdir, path + +if not path.isfile('face_detection/detection/sfd/s3fd.pth'): + raise FileNotFoundError('Save the s3fd model to face_detection/detection/sfd/s3fd.pth \ + before running this script!') + +import multiprocessing as mp +from concurrent.futures import ThreadPoolExecutor, as_completed +import numpy as np +import argparse, os, cv2, traceback, subprocess +from tqdm import tqdm +from glob import glob +import audio +from hparams import hparams as hp + +import face_detection + +parser = argparse.ArgumentParser() + +parser.add_argument('--ngpu', help='Number of GPUs across which to run in parallel', default=1, type=int) +parser.add_argument('--batch_size', help='Single GPU Face detection batch size', default=32, type=int) +parser.add_argument("--data_root", help="Root folder of the LRS2 dataset", required=True) +parser.add_argument("--preprocessed_root", help="Root folder of the preprocessed dataset", required=True) + +args = parser.parse_args() + +fa = [face_detection.FaceAlignment(face_detection.LandmarksType._2D, flip_input=False, + device='cuda:{}'.format(id)) for id in range(args.ngpu)] + +template = 'ffmpeg -loglevel panic -y -i {} -strict -2 {}' +# template2 = 'ffmpeg -hide_banner -loglevel panic -threads 1 -y -i {} -async 1 -ac 1 -vn -acodec pcm_s16le -ar 16000 {}' + +def process_video_file(vfile, args, gpu_id): + video_stream = cv2.VideoCapture(vfile) + + frames = [] + while 1: + still_reading, frame = video_stream.read() + if not still_reading: + video_stream.release() + break + frames.append(frame) + + vidname = os.path.basename(vfile).split('.')[0] + dirname = vfile.split('/')[-2] + + fulldir = path.join(args.preprocessed_root, dirname, vidname) + os.makedirs(fulldir, exist_ok=True) + + batches = [frames[i:i + args.batch_size] for i in range(0, len(frames), args.batch_size)] + + i = -1 + for fb in batches: + preds = fa[gpu_id].get_detections_for_batch(np.asarray(fb)) + + for j, f in enumerate(preds): + i += 1 + if f is None: + continue + + x1, y1, x2, y2 = f + cv2.imwrite(path.join(fulldir, '{}.jpg'.format(i)), fb[j][y1:y2, x1:x2]) + +def process_audio_file(vfile, args): + vidname = os.path.basename(vfile).split('.')[0] + dirname = vfile.split('/')[-2] + + fulldir = path.join(args.preprocessed_root, dirname, vidname) + os.makedirs(fulldir, exist_ok=True) + + wavpath = path.join(fulldir, 'audio.wav') + + command = template.format(vfile, wavpath) + subprocess.call(command, shell=True) + + +def mp_handler(job): + vfile, args, gpu_id = job + try: + process_video_file(vfile, args, gpu_id) + except KeyboardInterrupt: + exit(0) + except: + traceback.print_exc() + +def main(args): + print('Started processing for {} with {} GPUs'.format(args.data_root, args.ngpu)) + + filelist = glob(path.join(args.data_root, '*/*.mp4')) + + jobs = [(vfile, args, i%args.ngpu) for i, vfile in enumerate(filelist)] + p = ThreadPoolExecutor(args.ngpu) + futures = [p.submit(mp_handler, j) for j in jobs] + _ = [r.result() for r in tqdm(as_completed(futures), total=len(futures))] + + print('Dumping audios...') + + for vfile in tqdm(filelist): + try: + process_audio_file(vfile, args) + except KeyboardInterrupt: + exit(0) + except: + traceback.print_exc() + continue + +if __name__ == '__main__': + main(args) \ No newline at end of file diff --git a/models/Wav2Lip/requirements.txt b/models/Wav2Lip/requirements.txt new file mode 100644 index 0000000000..c9cf4ce923 --- /dev/null +++ b/models/Wav2Lip/requirements.txt @@ -0,0 +1,9 @@ +librosa==0.7.0 +numpy==1.17.1 +opencv-contrib-python==4.2.0.34 +opencv-python==4.1.0.25 +tensorflow==1.12.0 +torch==1.1.0 +torchvision==0.3.0 +tqdm==4.45.0 +numba==0.48 diff --git a/models/Wav2Lip/wav2lip_train.py b/models/Wav2Lip/wav2lip_train.py new file mode 100644 index 0000000000..6e0811808a --- /dev/null +++ b/models/Wav2Lip/wav2lip_train.py @@ -0,0 +1,374 @@ +from os.path import dirname, join, basename, isfile +from tqdm import tqdm + +from models import SyncNet_color as SyncNet +from models import Wav2Lip as Wav2Lip +import audio + +import torch +from torch import nn +from torch import optim +import torch.backends.cudnn as cudnn +from torch.utils import data as data_utils +import numpy as np + +from glob import glob + +import os, random, cv2, argparse +from hparams import hparams, get_image_list + +parser = argparse.ArgumentParser(description='Code to train the Wav2Lip model without the visual quality discriminator') + +parser.add_argument("--data_root", help="Root folder of the preprocessed LRS2 dataset", required=True, type=str) + +parser.add_argument('--checkpoint_dir', help='Save checkpoints to this directory', required=True, type=str) +parser.add_argument('--syncnet_checkpoint_path', help='Load the pre-trained Expert discriminator', required=True, type=str) + +parser.add_argument('--checkpoint_path', help='Resume from this checkpoint', default=None, type=str) + +args = parser.parse_args() + + +global_step = 0 +global_epoch = 0 +use_cuda = torch.cuda.is_available() +print('use_cuda: {}'.format(use_cuda)) + +syncnet_T = 5 +syncnet_mel_step_size = 16 + +class Dataset(object): + def __init__(self, split): + self.all_videos = get_image_list(args.data_root, split) + + def get_frame_id(self, frame): + return int(basename(frame).split('.')[0]) + + def get_window(self, start_frame): + start_id = self.get_frame_id(start_frame) + vidname = dirname(start_frame) + + window_fnames = [] + for frame_id in range(start_id, start_id + syncnet_T): + frame = join(vidname, '{}.jpg'.format(frame_id)) + if not isfile(frame): + return None + window_fnames.append(frame) + return window_fnames + + def read_window(self, window_fnames): + if window_fnames is None: return None + window = [] + for fname in window_fnames: + img = cv2.imread(fname) + if img is None: + return None + try: + img = cv2.resize(img, (hparams.img_size, hparams.img_size)) + except Exception as e: + return None + + window.append(img) + + return window + + def crop_audio_window(self, spec, start_frame): + if type(start_frame) == int: + start_frame_num = start_frame + else: + start_frame_num = self.get_frame_id(start_frame) # 0-indexing ---> 1-indexing + start_idx = int(80. * (start_frame_num / float(hparams.fps))) + + end_idx = start_idx + syncnet_mel_step_size + + return spec[start_idx : end_idx, :] + + def get_segmented_mels(self, spec, start_frame): + mels = [] + assert syncnet_T == 5 + start_frame_num = self.get_frame_id(start_frame) + 1 # 0-indexing ---> 1-indexing + if start_frame_num - 2 < 0: return None + for i in range(start_frame_num, start_frame_num + syncnet_T): + m = self.crop_audio_window(spec, i - 2) + if m.shape[0] != syncnet_mel_step_size: + return None + mels.append(m.T) + + mels = np.asarray(mels) + + return mels + + def prepare_window(self, window): + # 3 x T x H x W + x = np.asarray(window) / 255. + x = np.transpose(x, (3, 0, 1, 2)) + + return x + + def __len__(self): + return len(self.all_videos) + + def __getitem__(self, idx): + while 1: + idx = random.randint(0, len(self.all_videos) - 1) + vidname = self.all_videos[idx] + img_names = list(glob(join(vidname, '*.jpg'))) + if len(img_names) <= 3 * syncnet_T: + continue + + img_name = random.choice(img_names) + wrong_img_name = random.choice(img_names) + while wrong_img_name == img_name: + wrong_img_name = random.choice(img_names) + + window_fnames = self.get_window(img_name) + wrong_window_fnames = self.get_window(wrong_img_name) + if window_fnames is None or wrong_window_fnames is None: + continue + + window = self.read_window(window_fnames) + if window is None: + continue + + wrong_window = self.read_window(wrong_window_fnames) + if wrong_window is None: + continue + + try: + wavpath = join(vidname, "audio.wav") + wav = audio.load_wav(wavpath, hparams.sample_rate) + + orig_mel = audio.melspectrogram(wav).T + except Exception as e: + continue + + mel = self.crop_audio_window(orig_mel.copy(), img_name) + + if (mel.shape[0] != syncnet_mel_step_size): + continue + + indiv_mels = self.get_segmented_mels(orig_mel.copy(), img_name) + if indiv_mels is None: continue + + window = self.prepare_window(window) + y = window.copy() + window[:, :, window.shape[2]//2:] = 0. + + wrong_window = self.prepare_window(wrong_window) + x = np.concatenate([window, wrong_window], axis=0) + + x = torch.FloatTensor(x) + mel = torch.FloatTensor(mel.T).unsqueeze(0) + indiv_mels = torch.FloatTensor(indiv_mels).unsqueeze(1) + y = torch.FloatTensor(y) + return x, indiv_mels, mel, y + +def save_sample_images(x, g, gt, global_step, checkpoint_dir): + x = (x.detach().cpu().numpy().transpose(0, 2, 3, 4, 1) * 255.).astype(np.uint8) + g = (g.detach().cpu().numpy().transpose(0, 2, 3, 4, 1) * 255.).astype(np.uint8) + gt = (gt.detach().cpu().numpy().transpose(0, 2, 3, 4, 1) * 255.).astype(np.uint8) + + refs, inps = x[..., 3:], x[..., :3] + folder = join(checkpoint_dir, "samples_step{:09d}".format(global_step)) + if not os.path.exists(folder): os.mkdir(folder) + collage = np.concatenate((refs, inps, g, gt), axis=-2) + for batch_idx, c in enumerate(collage): + for t in range(len(c)): + cv2.imwrite('{}/{}_{}.jpg'.format(folder, batch_idx, t), c[t]) + +logloss = nn.BCELoss() +def cosine_loss(a, v, y): + d = nn.functional.cosine_similarity(a, v) + loss = logloss(d.unsqueeze(1), y) + + return loss + +device = torch.device("cuda" if use_cuda else "cpu") +syncnet = SyncNet().to(device) +for p in syncnet.parameters(): + p.requires_grad = False + +recon_loss = nn.L1Loss() +def get_sync_loss(mel, g): + g = g[:, :, :, g.size(3)//2:] + g = torch.cat([g[:, :, i] for i in range(syncnet_T)], dim=1) + # B, 3 * T, H//2, W + a, v = syncnet(mel, g) + y = torch.ones(g.size(0), 1).float().to(device) + return cosine_loss(a, v, y) + +def train(device, model, train_data_loader, test_data_loader, optimizer, + checkpoint_dir=None, checkpoint_interval=None, nepochs=None): + + global global_step, global_epoch + resumed_step = global_step + + while global_epoch < nepochs: + print('Starting Epoch: {}'.format(global_epoch)) + running_sync_loss, running_l1_loss = 0., 0. + prog_bar = tqdm(enumerate(train_data_loader)) + for step, (x, indiv_mels, mel, gt) in prog_bar: + model.train() + optimizer.zero_grad() + + # Move data to CUDA device + x = x.to(device) + mel = mel.to(device) + indiv_mels = indiv_mels.to(device) + gt = gt.to(device) + + g = model(indiv_mels, x) + + if hparams.syncnet_wt > 0.: + sync_loss = get_sync_loss(mel, g) + else: + sync_loss = 0. + + l1loss = recon_loss(g, gt) + + loss = hparams.syncnet_wt * sync_loss + (1 - hparams.syncnet_wt) * l1loss + loss.backward() + optimizer.step() + + if global_step % checkpoint_interval == 0: + save_sample_images(x, g, gt, global_step, checkpoint_dir) + + global_step += 1 + cur_session_steps = global_step - resumed_step + + running_l1_loss += l1loss.item() + if hparams.syncnet_wt > 0.: + running_sync_loss += sync_loss.item() + else: + running_sync_loss += 0. + + if global_step == 1 or global_step % checkpoint_interval == 0: + save_checkpoint( + model, optimizer, global_step, checkpoint_dir, global_epoch) + + if global_step == 1 or global_step % hparams.eval_interval == 0: + with torch.no_grad(): + average_sync_loss = eval_model(test_data_loader, global_step, device, model, checkpoint_dir) + + if average_sync_loss < .75: + hparams.set_hparam('syncnet_wt', 0.01) # without image GAN a lesser weight is sufficient + + prog_bar.set_description('L1: {}, Sync Loss: {}'.format(running_l1_loss / (step + 1), + running_sync_loss / (step + 1))) + + global_epoch += 1 + + +def eval_model(test_data_loader, global_step, device, model, checkpoint_dir): + eval_steps = 700 + print('Evaluating for {} steps'.format(eval_steps)) + sync_losses, recon_losses = [], [] + step = 0 + while 1: + for x, indiv_mels, mel, gt in test_data_loader: + step += 1 + model.eval() + + # Move data to CUDA device + x = x.to(device) + gt = gt.to(device) + indiv_mels = indiv_mels.to(device) + mel = mel.to(device) + + g = model(indiv_mels, x) + + sync_loss = get_sync_loss(mel, g) + l1loss = recon_loss(g, gt) + + sync_losses.append(sync_loss.item()) + recon_losses.append(l1loss.item()) + + if step > eval_steps: + averaged_sync_loss = sum(sync_losses) / len(sync_losses) + averaged_recon_loss = sum(recon_losses) / len(recon_losses) + + print('L1: {}, Sync loss: {}'.format(averaged_recon_loss, averaged_sync_loss)) + + return averaged_sync_loss + +def save_checkpoint(model, optimizer, step, checkpoint_dir, epoch): + + checkpoint_path = join( + checkpoint_dir, "checkpoint_step{:09d}.pth".format(global_step)) + optimizer_state = optimizer.state_dict() if hparams.save_optimizer_state else None + torch.save({ + "state_dict": model.state_dict(), + "optimizer": optimizer_state, + "global_step": step, + "global_epoch": epoch, + }, checkpoint_path) + print("Saved checkpoint:", checkpoint_path) + + +def _load(checkpoint_path): + if use_cuda: + checkpoint = torch.load(checkpoint_path) + else: + checkpoint = torch.load(checkpoint_path, + map_location=lambda storage, loc: storage) + return checkpoint + +def load_checkpoint(path, model, optimizer, reset_optimizer=False, overwrite_global_states=True): + global global_step + global global_epoch + + print("Load checkpoint from: {}".format(path)) + checkpoint = _load(path) + s = checkpoint["state_dict"] + new_s = {} + for k, v in s.items(): + new_s[k.replace('module.', '')] = v + model.load_state_dict(new_s) + if not reset_optimizer: + optimizer_state = checkpoint["optimizer"] + if optimizer_state is not None: + print("Load optimizer state from {}".format(path)) + optimizer.load_state_dict(checkpoint["optimizer"]) + if overwrite_global_states: + global_step = checkpoint["global_step"] + global_epoch = checkpoint["global_epoch"] + + return model + +if __name__ == "__main__": + checkpoint_dir = args.checkpoint_dir + + # Dataset and Dataloader setup + train_dataset = Dataset('train') + test_dataset = Dataset('val') + + train_data_loader = data_utils.DataLoader( + train_dataset, batch_size=hparams.batch_size, shuffle=True, + num_workers=hparams.num_workers) + + test_data_loader = data_utils.DataLoader( + test_dataset, batch_size=hparams.batch_size, + num_workers=4) + + device = torch.device("cuda" if use_cuda else "cpu") + + # Model + model = Wav2Lip().to(device) + print('total trainable params {}'.format(sum(p.numel() for p in model.parameters() if p.requires_grad))) + + optimizer = optim.Adam([p for p in model.parameters() if p.requires_grad], + lr=hparams.initial_learning_rate) + + if args.checkpoint_path is not None: + load_checkpoint(args.checkpoint_path, model, optimizer, reset_optimizer=False) + + load_checkpoint(args.syncnet_checkpoint_path, syncnet, None, reset_optimizer=True, overwrite_global_states=False) + + if not os.path.exists(checkpoint_dir): + os.mkdir(checkpoint_dir) + + # Train! + train(device, model, train_data_loader, test_data_loader, optimizer, + checkpoint_dir=checkpoint_dir, + checkpoint_interval=hparams.checkpoint_interval, + nepochs=hparams.nepochs) From a8f969dc55a80e3c7aff2363bbf457f0182789df Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 03:19:33 -0400 Subject: [PATCH 0116/1042] add READMEs for w2l --- models/Wav2Lip/checkpoints/README.md | 1 + models/Wav2Lip/evaluation/README.md | 59 +++++++ .../evaluation/test_filelists/README.md | 1 + .../test_filelists/ReSyncED/random_pairs.txt | 160 ++++++++++++++++++ .../test_filelists/ReSyncED/tts_pairs.txt | 18 ++ models/Wav2Lip/face_detection/README.md | 1 + models/Wav2Lip/filelists/README.md | 1 + models/Wav2Lip/results/README.md | 1 + models/Wav2Lip/temp/README.md | 1 + 9 files changed, 243 insertions(+) create mode 100644 models/Wav2Lip/checkpoints/README.md create mode 100644 models/Wav2Lip/evaluation/README.md create mode 100644 models/Wav2Lip/evaluation/test_filelists/README.md create mode 100644 models/Wav2Lip/evaluation/test_filelists/ReSyncED/random_pairs.txt create mode 100644 models/Wav2Lip/evaluation/test_filelists/ReSyncED/tts_pairs.txt create mode 100644 models/Wav2Lip/face_detection/README.md create mode 100644 models/Wav2Lip/filelists/README.md create mode 100644 models/Wav2Lip/results/README.md create mode 100644 models/Wav2Lip/temp/README.md diff --git a/models/Wav2Lip/checkpoints/README.md b/models/Wav2Lip/checkpoints/README.md new file mode 100644 index 0000000000..80258ec8fb --- /dev/null +++ b/models/Wav2Lip/checkpoints/README.md @@ -0,0 +1 @@ +Place all your checkpoints (.pth files) here. \ No newline at end of file diff --git a/models/Wav2Lip/evaluation/README.md b/models/Wav2Lip/evaluation/README.md new file mode 100644 index 0000000000..613e3a3b59 --- /dev/null +++ b/models/Wav2Lip/evaluation/README.md @@ -0,0 +1,59 @@ +# Evaluation of Lip-sync using LSE-D and LSE-C metric. + +We use the pre-trained syncnet model available in this [repository](https://github.com/joonson/syncnet_python). + +### Steps to set-up the evaluation repository for LSE-D and LSE-C metric: +* Clone the SyncNet repository. +``` +git clone https://github.com/joonson/syncnet_python.git +``` +* Follow the procedure given in the above linked [repository](https://github.com/joonson/syncnet_python) to download the pretrained models and set up the dependencies. + * **Note: Please install a separate virtual environment for the evaluation scripts. The versions used by Wav2Lip and the publicly released code of SyncNet is different and can cause version mis-match issues. To avoid this, we suggest the users to install a separate virtual environment for the evaluation scripts** +``` +cd syncnet_python +pip install -r requirements.txt +sh download_model.sh +``` +* The above step should ensure that all the dependencies required by the repository is installed and the pre-trained models are downloaded. + +### Running the evaluation scripts: +* Copy our evaluation scripts given in this folder to the cloned repository. +``` + cd Wav2Lip/evaluation/scores_LSE/ + cp *.py syncnet_python/ + cp *.sh syncnet_python/ +``` +**Note: We will release the test filelists for LRW, LRS2 and LRS3 shortly once we receive permission from the dataset creators. We will also release the Real World Dataset we have collected shortly.** + +* Our evaluation technique does not require ground-truth of any sorts. Given lip-synced videos we can directly calculate the scores from only the generated videos. Please store the generated videos (from our test sets or your own generated videos) in the following folder structure. +``` +video data root (Folder containing all videos) +├── All .mp4 files +``` +* Change the folder back to the cloned repository. +``` +cd syncnet_python +``` +* To run evaluation on the LRW, LRS2 and LRS3 test files, please run the following command: +``` +python calculate_scores_LRS.py --data_root /path/to/video/data/root --tmp_dir tmp_dir/ +``` + +* To run evaluation on the ReSynced dataset or your own generated videos, please run the following command: +``` +sh calculate_scores_real_videos.sh /path/to/video/data/root +``` +* The generated scores will be present in the all_scores.txt generated in the ```syncnet_python/``` folder + +# Evaluation of image quality using FID metric. +We will update the scripts for the FID metric shortly + + +# Opening issues related to evaluation scripts +* Please open the issues with the "Evaluation" label if you face any issues in the evaluation scripts. + +# Acknowledgements +Our evaluation pipeline in based on two existing repositories. LSE metrics are based on the [syncnet_python](https://github.com/joonson/syncnet_python) repository and the FID score is based on [pytorch-fid](https://github.com/mseitzer/pytorch-fid) repository. We thank the authors of both the repositories for releasing their wonderful code. + + + diff --git a/models/Wav2Lip/evaluation/test_filelists/README.md b/models/Wav2Lip/evaluation/test_filelists/README.md new file mode 100644 index 0000000000..f8033e6978 --- /dev/null +++ b/models/Wav2Lip/evaluation/test_filelists/README.md @@ -0,0 +1 @@ +Place LRS2, LRW, and LRS3 (and any other) test set filelists here. \ No newline at end of file diff --git a/models/Wav2Lip/evaluation/test_filelists/ReSyncED/random_pairs.txt b/models/Wav2Lip/evaluation/test_filelists/ReSyncED/random_pairs.txt new file mode 100644 index 0000000000..ffe2c40e11 --- /dev/null +++ b/models/Wav2Lip/evaluation/test_filelists/ReSyncED/random_pairs.txt @@ -0,0 +1,160 @@ +sachin.mp4 emma_cropped.mp4 +sachin.mp4 mourinho.mp4 +sachin.mp4 elon.mp4 +sachin.mp4 messi2.mp4 +sachin.mp4 cr1.mp4 +sachin.mp4 sachin.mp4 +sachin.mp4 sg.mp4 +sachin.mp4 fergi.mp4 +sachin.mp4 spanish_lec1.mp4 +sachin.mp4 bush_small.mp4 +sachin.mp4 macca_cut.mp4 +sachin.mp4 ca_cropped.mp4 +sachin.mp4 lecun.mp4 +sachin.mp4 spanish_lec0.mp4 +srk.mp4 emma_cropped.mp4 +srk.mp4 mourinho.mp4 +srk.mp4 elon.mp4 +srk.mp4 messi2.mp4 +srk.mp4 cr1.mp4 +srk.mp4 srk.mp4 +srk.mp4 sachin.mp4 +srk.mp4 sg.mp4 +srk.mp4 fergi.mp4 +srk.mp4 spanish_lec1.mp4 +srk.mp4 bush_small.mp4 +srk.mp4 macca_cut.mp4 +srk.mp4 ca_cropped.mp4 +srk.mp4 guardiola.mp4 +srk.mp4 lecun.mp4 +srk.mp4 spanish_lec0.mp4 +cr1.mp4 emma_cropped.mp4 +cr1.mp4 elon.mp4 +cr1.mp4 messi2.mp4 +cr1.mp4 cr1.mp4 +cr1.mp4 spanish_lec1.mp4 +cr1.mp4 bush_small.mp4 +cr1.mp4 macca_cut.mp4 +cr1.mp4 ca_cropped.mp4 +cr1.mp4 lecun.mp4 +cr1.mp4 spanish_lec0.mp4 +macca_cut.mp4 emma_cropped.mp4 +macca_cut.mp4 elon.mp4 +macca_cut.mp4 messi2.mp4 +macca_cut.mp4 spanish_lec1.mp4 +macca_cut.mp4 macca_cut.mp4 +macca_cut.mp4 ca_cropped.mp4 +macca_cut.mp4 spanish_lec0.mp4 +lecun.mp4 emma_cropped.mp4 +lecun.mp4 elon.mp4 +lecun.mp4 messi2.mp4 +lecun.mp4 spanish_lec1.mp4 +lecun.mp4 macca_cut.mp4 +lecun.mp4 ca_cropped.mp4 +lecun.mp4 lecun.mp4 +lecun.mp4 spanish_lec0.mp4 +messi2.mp4 emma_cropped.mp4 +messi2.mp4 elon.mp4 +messi2.mp4 messi2.mp4 +messi2.mp4 spanish_lec1.mp4 +messi2.mp4 macca_cut.mp4 +messi2.mp4 ca_cropped.mp4 +messi2.mp4 spanish_lec0.mp4 +ca_cropped.mp4 emma_cropped.mp4 +ca_cropped.mp4 elon.mp4 +ca_cropped.mp4 spanish_lec1.mp4 +ca_cropped.mp4 ca_cropped.mp4 +ca_cropped.mp4 spanish_lec0.mp4 +spanish_lec1.mp4 spanish_lec1.mp4 +spanish_lec1.mp4 spanish_lec0.mp4 +elon.mp4 elon.mp4 +elon.mp4 spanish_lec1.mp4 +elon.mp4 spanish_lec0.mp4 +guardiola.mp4 emma_cropped.mp4 +guardiola.mp4 mourinho.mp4 +guardiola.mp4 elon.mp4 +guardiola.mp4 messi2.mp4 +guardiola.mp4 cr1.mp4 +guardiola.mp4 sachin.mp4 +guardiola.mp4 sg.mp4 +guardiola.mp4 fergi.mp4 +guardiola.mp4 spanish_lec1.mp4 +guardiola.mp4 bush_small.mp4 +guardiola.mp4 macca_cut.mp4 +guardiola.mp4 ca_cropped.mp4 +guardiola.mp4 guardiola.mp4 +guardiola.mp4 lecun.mp4 +guardiola.mp4 spanish_lec0.mp4 +fergi.mp4 emma_cropped.mp4 +fergi.mp4 mourinho.mp4 +fergi.mp4 elon.mp4 +fergi.mp4 messi2.mp4 +fergi.mp4 cr1.mp4 +fergi.mp4 sachin.mp4 +fergi.mp4 sg.mp4 +fergi.mp4 fergi.mp4 +fergi.mp4 spanish_lec1.mp4 +fergi.mp4 bush_small.mp4 +fergi.mp4 macca_cut.mp4 +fergi.mp4 ca_cropped.mp4 +fergi.mp4 lecun.mp4 +fergi.mp4 spanish_lec0.mp4 +spanish.mp4 emma_cropped.mp4 +spanish.mp4 spanish.mp4 +spanish.mp4 mourinho.mp4 +spanish.mp4 elon.mp4 +spanish.mp4 messi2.mp4 +spanish.mp4 cr1.mp4 +spanish.mp4 srk.mp4 +spanish.mp4 sachin.mp4 +spanish.mp4 sg.mp4 +spanish.mp4 fergi.mp4 +spanish.mp4 spanish_lec1.mp4 +spanish.mp4 bush_small.mp4 +spanish.mp4 macca_cut.mp4 +spanish.mp4 ca_cropped.mp4 +spanish.mp4 guardiola.mp4 +spanish.mp4 lecun.mp4 +spanish.mp4 spanish_lec0.mp4 +bush_small.mp4 emma_cropped.mp4 +bush_small.mp4 elon.mp4 +bush_small.mp4 messi2.mp4 +bush_small.mp4 spanish_lec1.mp4 +bush_small.mp4 bush_small.mp4 +bush_small.mp4 macca_cut.mp4 +bush_small.mp4 ca_cropped.mp4 +bush_small.mp4 lecun.mp4 +bush_small.mp4 spanish_lec0.mp4 +emma_cropped.mp4 emma_cropped.mp4 +emma_cropped.mp4 elon.mp4 +emma_cropped.mp4 spanish_lec1.mp4 +emma_cropped.mp4 spanish_lec0.mp4 +sg.mp4 emma_cropped.mp4 +sg.mp4 mourinho.mp4 +sg.mp4 elon.mp4 +sg.mp4 messi2.mp4 +sg.mp4 cr1.mp4 +sg.mp4 sachin.mp4 +sg.mp4 sg.mp4 +sg.mp4 fergi.mp4 +sg.mp4 spanish_lec1.mp4 +sg.mp4 bush_small.mp4 +sg.mp4 macca_cut.mp4 +sg.mp4 ca_cropped.mp4 +sg.mp4 lecun.mp4 +sg.mp4 spanish_lec0.mp4 +spanish_lec0.mp4 spanish_lec0.mp4 +mourinho.mp4 emma_cropped.mp4 +mourinho.mp4 mourinho.mp4 +mourinho.mp4 elon.mp4 +mourinho.mp4 messi2.mp4 +mourinho.mp4 cr1.mp4 +mourinho.mp4 sachin.mp4 +mourinho.mp4 sg.mp4 +mourinho.mp4 fergi.mp4 +mourinho.mp4 spanish_lec1.mp4 +mourinho.mp4 bush_small.mp4 +mourinho.mp4 macca_cut.mp4 +mourinho.mp4 ca_cropped.mp4 +mourinho.mp4 lecun.mp4 +mourinho.mp4 spanish_lec0.mp4 diff --git a/models/Wav2Lip/evaluation/test_filelists/ReSyncED/tts_pairs.txt b/models/Wav2Lip/evaluation/test_filelists/ReSyncED/tts_pairs.txt new file mode 100644 index 0000000000..b7dc1a8c0b --- /dev/null +++ b/models/Wav2Lip/evaluation/test_filelists/ReSyncED/tts_pairs.txt @@ -0,0 +1,18 @@ +adam_1.mp4 andreng_optimization.wav +agad_2.mp4 agad_2.wav +agad_1.mp4 agad_1.wav +agad_3.mp4 agad_3.wav +rms_prop_1.mp4 rms_prop_tts.wav +tf_1.mp4 tf_1.wav +tf_2.mp4 tf_2.wav +andrew_ng_ai_business.mp4 andrewng_business_tts.wav +covid_autopsy_1.mp4 autopsy_tts.wav +news_1.mp4 news_tts.wav +andrew_ng_fund_1.mp4 andrewng_ai_fund.wav +covid_treatments_1.mp4 covid_tts.wav +pytorch_v_tf.mp4 pytorch_vs_tf_eng.wav +pytorch_1.mp4 pytorch.wav +pkb_1.mp4 pkb_1.wav +ss_1.mp4 ss_1.wav +carlsen_1.mp4 carlsen_eng.wav +french.mp4 french.wav \ No newline at end of file diff --git a/models/Wav2Lip/face_detection/README.md b/models/Wav2Lip/face_detection/README.md new file mode 100644 index 0000000000..c073376e4e --- /dev/null +++ b/models/Wav2Lip/face_detection/README.md @@ -0,0 +1 @@ +The code for Face Detection in this folder has been taken from the wonderful [face_alignment](https://github.com/1adrianb/face-alignment) repository. This has been modified to take batches of faces at a time. \ No newline at end of file diff --git a/models/Wav2Lip/filelists/README.md b/models/Wav2Lip/filelists/README.md new file mode 100644 index 0000000000..e7d7e7bb3b --- /dev/null +++ b/models/Wav2Lip/filelists/README.md @@ -0,0 +1 @@ +Place LRS2 (and any other) filelists here for training. \ No newline at end of file diff --git a/models/Wav2Lip/results/README.md b/models/Wav2Lip/results/README.md new file mode 100644 index 0000000000..b1bbfd53fd --- /dev/null +++ b/models/Wav2Lip/results/README.md @@ -0,0 +1 @@ +Generated results will be placed in this folder by default. \ No newline at end of file diff --git a/models/Wav2Lip/temp/README.md b/models/Wav2Lip/temp/README.md new file mode 100644 index 0000000000..04c9104993 --- /dev/null +++ b/models/Wav2Lip/temp/README.md @@ -0,0 +1 @@ +Temporary files at the time of inference/testing will be saved here. You can ignore them. \ No newline at end of file From b2dcdbbaeaf67e97eeb735fd728058a52bb15a94 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 03:21:03 -0400 Subject: [PATCH 0117/1042] add codegen cache for new queries --- codegen_database.sh | 2 + sqlx-data.json | 269 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 268 insertions(+), 3 deletions(-) create mode 100755 codegen_database.sh diff --git a/codegen_database.sh b/codegen_database.sh new file mode 100755 index 0000000000..cd78945620 --- /dev/null +++ b/codegen_database.sh @@ -0,0 +1,2 @@ +#!/bin/bash +SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web diff --git a/sqlx-data.json b/sqlx-data.json index 6eb95a1a71..df9df95c4e 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -10,6 +10,16 @@ "nullable": [] } }, + "1eeb7a88f5f8c2961694622b739e0b0689d89dff8a5f6b79d5a57c5b807a23f8": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "26fbb143f69b900a722ce49e14942c6eb14b53dad85aa551f13e742dd6a21573": { "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", "describe": { @@ -20,6 +30,16 @@ "nullable": [] } }, + "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, "345cbabbe69652c89ccd384ed7db44f70c53791488cf63721f9563291b45db3e": { "query": "\nINSERT INTO user_sessions (\n token,\n user_token,\n ip_address_creation,\n expires_at\n)\nVALUES ( ?, ?, ?, NOW() + interval 1 year )\n ", "describe": { @@ -651,6 +671,249 @@ ] } }, + "b7535ecf2e63b7f02b5b96f833fd8cd28285b24473df6f89795ff360c1723215": { + "query": "\nSELECT *\nFROM w2l_template_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 4, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 7, + "name": "maybe_subject_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "maybe_actor_subject_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "download_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 10, + "name": "download_url_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 48 + } + }, + { + "ordinal": 11, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 12, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 13, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 14, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 15, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 16, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false, + false, + true, + false, + false, + true + ] + } + }, + "b8e0f7961f7b339e4a81b7fd47f99d0cd07567b33f0d093b449586154dc16235": { + "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 8 + }, + "nullable": [] + } + }, "bf489cf419d342d144f072c310d6d70c9073b092b5721f489eaf2ffc8ad3915e": { "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -681,12 +944,12 @@ "nullable": [] } }, - "d1c8e0f9c5600fd7c296558cdb3525a2678b3e775f9e2bee0f1da979d4a82d70": { - "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?\n ", + "e228a3f6766d19b50fa0e01ef2e085bdcb57813cf30e88948f0ec2592e27dbab": { + "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?\n ", "describe": { "columns": [], "parameters": { - "Right": 7 + "Right": 9 }, "nullable": [] } From 7622c9b0e39f39e68c7b4ebe4045478a0a5fc632 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 03:21:31 -0400 Subject: [PATCH 0118/1042] update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index dc88b1914f..b768de628d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .idea/ __pycache__/ _ignored_schema.rs +models/Wav2Lip/python/ production_database_secret_connect.sh secrets.toml secrets.yaml From f77563e03e647f31a36115eff67bb44a5d8a7a38 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 03:40:17 -0400 Subject: [PATCH 0119/1042] major dockerfile overhaul --- Dockerfile | 137 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 127 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 20e3aefc53..c9b5eaaf50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,67 @@ +# ==================== Python Build Base ==================== + +FROM nvidia/cuda:10.1-devel-ubuntu18.04 as pybuild-base + +# NB: https://github.com/NVIDIA/nvidia-docker/issues/864#issuecomment-439848887 +# NB: We do not install ffmpeg, since the version is 3.8.* series and we need 4.2.* +RUN apt-get update \ + && apt-get install -y \ + build-essential \ + curl \ + g++-7 \ + gcc-7 \ + git \ + htop \ + libc++-7-dev \ + libffi-dev \ + libgcc-7-dev \ + libsm6 \ + libsndfile1 \ + libssl-dev \ + libxext6 \ + libxrender-dev \ + netcat \ + python-dev \ + python3-pip \ + python3.6 \ + python3.6-dev \ + python3.6-venv\ + redis-tools \ + silversearcher-ag \ + sox \ + tmux \ + vim \ + wget + +# We need ffmpeg version >= 4.2 (https://superuser.com/a/579110) +# http://ubuntuhandbook.org/index.php/2020/06/install-ffmpeg-4-3-via-ppa-ubuntu-18-04-16-04/ + +RUN apt-get install -y software-properties-common \ + && add-apt-repository ppa:jonathonf/ffmpeg-4 \ + && apt-get update \ + && apt-get install -y ffmpeg + +RUN ffmpeg -version + +# ==================== Python Build Step 2: Wav2Lip Requirements ==================== + +FROM pybuild-base as pybuild-requirements + +COPY models/Wav2Lip ./models/Wav2Lip +WORKDIR models/Wav2Lip + +RUN python3.6 --version +RUN python3.6 -m venv python +RUN . python/bin/activate \ + && pip install --upgrade pip \ + && pip install -r requirements.txt + +# ==================== Rust Build Base ==================== + # Custom base image # Make sure to add this repository so it has read acces to the base image: # https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access -FROM ghcr.io/storytold/docker-base-images-rust-ssl:latest as build +FROM ghcr.io/storytold/docker-base-images-rust-ssl:latest as rust-build WORKDIR /tmp COPY Cargo.lock . @@ -44,7 +104,8 @@ RUN SQLX_OFFLINE=true \ --bin w2l-inference-job # Final image -FROM ubuntu:xenial +# FROM ubuntu:xenial +FROM pybuild-requirements as final #RUN mkdir /storyteller #WORKDIR /storyteller WORKDIR / @@ -79,22 +140,78 @@ RUN python3 --version RUN pip3 install gdown # Copy all the binaries. -COPY --from=build /tmp/target/release/storyteller-web / -COPY --from=build /tmp/target/release/tts-download-job / -COPY --from=build /tmp/target/release/tts-inference-job / -COPY --from=build /tmp/target/release/w2l-download-job / -COPY --from=build /tmp/target/release/w2l-inference-job / +COPY --from=rust-build /tmp/target/release/storyteller-web / +COPY --from=rust-build /tmp/target/release/tts-download-job / +COPY --from=rust-build /tmp/target/release/tts-inference-job / +COPY --from=rust-build /tmp/target/release/w2l-download-job / +COPY --from=rust-build /tmp/target/release/w2l-inference-job / # SSL certs are required for crypto -COPY --from=build /etc/ssl /etc/ssl +COPY --from=rust-build /etc/ssl /etc/ssl # Required dynamically linked libraries -COPY --from=build /lib/x86_64-linux-gnu/libssl.* /lib/x86_64-linux-gnu/ -COPY --from=build /lib/x86_64-linux-gnu/libcrypto.* /lib/x86_64-linux-gnu/ +COPY --from=rust-build /lib/x86_64-linux-gnu/libssl.* /lib/x86_64-linux-gnu/ +COPY --from=rust-build /lib/x86_64-linux-gnu/libcrypto.* /lib/x86_64-linux-gnu/ # Make sure all the links resolve RUN ldd storyteller-web +# Without a .env file, Rust crashes "mysteriously" (ugh) +RUN touch .env +RUN touch .env-secrets + EXPOSE 8080 CMD LD_LIBRARY_PATH=/usr/lib /storyteller-web +# =================================================================================================== +# WAV2LIP WORKER + +# ==================== Rust Build Base ==================== + +# #FROM ubuntu:xenial as rustbuild-base +# WORKDIR /tmp +# RUN apt-get update \ +# && apt-get install -y \ +# build-essential \ +# curl \ +# libssl-dev \ +# libssl1.0.0 \ +# pkg-config \ +# unzip \ +# wget +# RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ +# | sh -s -- --default-toolchain stable -y + + + +# ==================== Rust Build Binary ==================== + +# FROM rustbuild-base as rustbuild-binary +# +# COPY Cargo.lock . +# COPY Cargo.toml . +# COPY w2l_server ./w2l_server +# COPY w2l_shared ./w2l_shared +# COPY w2l_worker ./w2l_worker +# +# RUN $HOME/.cargo/bin/cargo fetch +# RUN LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} $HOME/.cargo/bin/cargo build --release --bin wav2lip_worker + +# ==================== Final Image ==================== + +# FROM pybuild-requirements as final +# WORKDIR / +# +# COPY --from=rustbuild-binary /tmp/target/release/wav2lip_worker / +# COPY --from=rustbuild-binary /etc/ssl /etc/ssl +# COPY --from=rustbuild-binary /lib/x86_64-linux-gnu/libssl.* /lib/x86_64-linux-gnu/ +# COPY --from=rustbuild-binary /lib/x86_64-linux-gnu/libcrypto.* /lib/x86_64-linux-gnu/ +# +# # Without a .env file, Rust crashes "mysteriously" (ugh) +# RUN touch .env +# +# RUN ldd wav2lip_worker +# +# EXPOSE 8080 +# CMD LD_LIBRARY_PATH=/usr/lib /wav2lip_worker + From 2727afd6446b94b32f0457cfe60d136009f0632a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 15:00:10 -0400 Subject: [PATCH 0120/1042] Getting ready to process W2L uploads... --- ...ference_custom.py => vocodes_inference.py} | 0 models/Wav2Lip/vocodes_process_upload.py | 480 ++++++++++++++++++ src/script_execution/mod.rs | 3 +- .../wav2lip_inference_command.rs | 125 +++++ ...d.rs => wav2lip_process_upload_command.rs} | 0 5 files changed, 607 insertions(+), 1 deletion(-) rename models/Wav2Lip/{inference_custom.py => vocodes_inference.py} (100%) create mode 100644 models/Wav2Lip/vocodes_process_upload.py create mode 100644 src/script_execution/wav2lip_inference_command.rs rename src/script_execution/{wav2lip_preprocess_command.rs => wav2lip_process_upload_command.rs} (100%) diff --git a/models/Wav2Lip/inference_custom.py b/models/Wav2Lip/vocodes_inference.py similarity index 100% rename from models/Wav2Lip/inference_custom.py rename to models/Wav2Lip/vocodes_inference.py diff --git a/models/Wav2Lip/vocodes_process_upload.py b/models/Wav2Lip/vocodes_process_upload.py new file mode 100644 index 0000000000..5d0349094f --- /dev/null +++ b/models/Wav2Lip/vocodes_process_upload.py @@ -0,0 +1,480 @@ +from os import listdir, path +import numpy as np +import scipy, cv2, os, sys, argparse, audio +import json, subprocess, random, string +from tqdm import tqdm +from glob import glob +import torch, face_detection +from models import Wav2Lip +import tempfile +import shutil +import datetime +import pickle +from PIL import Image + +""" +Current worker filesystem architecture: + +/tmp +/tmp/models +/tmp/models/wav2lip_gan.pth +/tmp/.tmppd1Kvq +/tmp/.tmppd1Kvq/input_audio_file +/tmp/templates_images +/tmp/templates_images/elon-musk.jpg +/tmp/templates_videos +/tmp/templates_videos/ben-shapiro-baby.mp4 +/tmp/templates_videos/ben-shapiro-baby.mp4.faces +/tmp/templates_videos/dr-phil-bubble.mp4 +/tmp/tmp7um0yxjo +/tmp/tmp7um0yxjo/padded_audio.wav +/tmp/tmp7um0yxjo/temp.wav +/tmp/assets +/tmp/assets/vocodes-short-end-bump.mp4 + +""" + +if 'PRINT_ENV' in os.environ and os.environ['PRINT_ENV']: + for k, v in os.environ.items(): + print('{}={}'.format(k, v)) + +print('========================================') +print('Python interpreter', sys.executable) +print('PyTorch version', torch.__version__) +print('CUDA Available?', torch.cuda.is_available()) +print('CUDA Device count', torch.cuda.device_count()) +print('========================================', flush=True) + +parser = argparse.ArgumentParser(description='Inference code to lip-sync videos in the wild using Wav2Lip models') + +parser.add_argument('--checkpoint_path', type=str, + help='Name of saved checkpoint to load weights from', required=True) + +parser.add_argument('--face', type=str, + help='Filepath of video/image that contains faces to use', required=True) +parser.add_argument('--audio', type=str, + help='Filepath of video/audio file to use as raw audio source', required=True) +parser.add_argument('--outfile', type=str, help='Video path to save result. See default for an e.g.', + default='results/result_voice.mp4') + +parser.add_argument('--static', type=bool, + help='If True, then use only first video frame for inference', default=False) + +# NB: I'm adjusting the FPS to match that of the end bump I made. The old default FPS was "25." (float 25) +parser.add_argument('--fps', type=float, help='Can be specified only if input is a static image (default: 29.97)', + default=29.97, required=False) + +parser.add_argument('--pads', nargs='+', type=int, default=[0, 10, 0, 0], + help='Padding (top, bottom, left, right). Please adjust to include chin at least') + +parser.add_argument('--face_det_batch_size', type=int, + help='Batch size for face detection', default=16) +parser.add_argument('--wav2lip_batch_size', type=int, help='Batch size for Wav2Lip model(s)', default=128) + +parser.add_argument('--resize_factor', default=1, type=int, + help='Reduce the resolution by this factor. Sometimes, best results are obtained at 480p or 720p') + +parser.add_argument('--crop', nargs='+', type=int, default=[0, -1, 0, -1], + help='Crop video to a smaller region (top, bottom, left, right). Applied after resize_factor and rotate arg. ' + 'Useful if multiple face present. -1 implies the value will be auto-inferred based on height, width') + +parser.add_argument('--box', nargs='+', type=int, default=[-1, -1, -1, -1], + help='Specify a constant bounding box for the face. Use only as a last resort if the face is not detected.' + 'Also, might work only if the face is not moving around much. Syntax: (top, bottom, left, right).') + +parser.add_argument('--rotate', default=False, action='store_true', + help='Sometimes videos taken from a phone can be flipped 90deg. If true, will flip video right by 90deg.' + 'Use if you get a flipped result, despite feeding a normal looking video') + +parser.add_argument('--nosmooth', default=False, action='store_true', + help='Prevent smoothing face detections over a short temporal window') + +parser.add_argument('--audio_start_pad_millis', default=0, type=int, + help='Seconds to pad the start of the audio') + +parser.add_argument('--audio_end_pad_millis', default=0, type=int, + help='Seconds to pad the end of the audio') + +parser.add_argument('--end_bump_file', type=str, + help='Video file to concatenate at the end') + +# This is a little more deliberate than "static" and causes us to pick an FPS, etc. +# This is useful when the filename doesn't have an extension. +parser.add_argument('--is_image', default=False, action='store_true', + help='Denote that the input is a single-frame image, not a video') + +# Purely for debugging on the host machine: +parser.add_argument('--preserve_tempdir', default=False, action='store_true', + help='Keep the tempdir arround for debugging') + +args = parser.parse_args() +args.img_size = 96 + +if os.path.isfile(args.face) and os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg']: + args.static = True +elif args.is_image: + args.static = True + +def get_smoothened_boxes(boxes, T): + for i in range(len(boxes)): + if i + T > len(boxes): + window = boxes[len(boxes) - T:] + else: + window = boxes[i : i + T] + boxes[i] = np.mean(window, axis=0) + return boxes + +def face_detect(images): + detector = face_detection.FaceAlignment(face_detection.LandmarksType._2D, + flip_input=False, device=device) + + batch_size = args.face_det_batch_size + + while 1: + predictions = [] + try: + for i in tqdm(range(0, len(images), batch_size)): + predictions.extend(detector.get_detections_for_batch(np.array(images[i:i + batch_size]))) + except RuntimeError: + if batch_size == 1: + raise RuntimeError('Image too big to run face detection on GPU. Please use the --resize_factor argument') + batch_size //= 2 + print('Recovering from OOM error; New batch size: {}'.format(batch_size)) + continue + break + + results = [] + pady1, pady2, padx1, padx2 = args.pads + for rect, image in zip(predictions, images): + if rect is None: + cv2.imwrite('temp/faulty_frame.jpg', image) # check this frame where the face was not detected. + raise ValueError('Face not detected! Ensure the video contains a face in all the frames.') + + y1 = max(0, rect[1] - pady1) + y2 = min(image.shape[0], rect[3] + pady2) + x1 = max(0, rect[0] - padx1) + x2 = min(image.shape[1], rect[2] + padx2) + + results.append([x1, y1, x2, y2]) + + boxes = np.array(results) + if not args.nosmooth: boxes = get_smoothened_boxes(boxes, T=5) + results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2)] for image, (x1, y1, x2, y2) in zip(images, boxes)] + + del detector + return results + +def detect_faces_in_frames(frames, video_faces_pickle_file): + t0 = datetime.datetime.now() + if args.box[0] == -1: + if not args.static: + face_det_results = face_detect(frames) # BGR2RGB for CNN face detection + else: + face_det_results = face_detect([frames[0]]) + else: + print('Using the specified bounding box instead of face detection...') + y1, y2, x1, x2 = args.box + face_det_results = [[f[y1: y2, x1:x2], (y1, y2, x1, x2)] for f in frames] + t1 = datetime.datetime.now() + + d = t1 - t0 + print('Total seconds to detect faces in {} frames: {}'.format(len(frames), d.total_seconds())) + + with open(video_faces_pickle_file, 'wb') as f: + print('Saving pickle file: {}'.format(video_faces_pickle_file)) + pickle.dump(face_det_results, f) + + return face_det_results + +#def datagen(frames, mels): +def datagen(frames, face_det_results, mels): + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + #face_det_results = detect_faces_in_frames(frames) + + for i, m in enumerate(mels): + idx = 0 if args.static else i%len(frames) + frame_to_save = frames[idx].copy() + face, coords = face_det_results[idx].copy() + + face = cv2.resize(face, (args.img_size, args.img_size)) + + img_batch.append(face) + mel_batch.append(m) + frame_batch.append(frame_to_save) + coords_batch.append(coords) + + if len(img_batch) >= args.wav2lip_batch_size: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + if len(img_batch) > 0: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + +mel_step_size = 16 +device = 'cuda' if torch.cuda.is_available() else 'cpu' +print('Using {} for inference.'.format(device)) + +def _load(checkpoint_path): + if device == 'cuda': + checkpoint = torch.load(checkpoint_path) + else: + checkpoint = torch.load(checkpoint_path, + map_location=lambda storage, loc: storage) + return checkpoint + +def load_model(path): + model = Wav2Lip() + print("Load checkpoint from: {}".format(path), flush=True) + checkpoint = _load(path) + s = checkpoint["state_dict"] + new_s = {} + for k, v in s.items(): + new_s[k.replace('module.', '')] = v + model.load_state_dict(new_s) + + model = model.to(device) + return model.eval() + +def maybe_pad_audio_file(tempdir, args): + if not args.audio_start_pad_millis and not args.audio_end_pad_millis: + return + + print('Padding audio with {} millis at start, {} millis at end.'.format( + args.audio_start_pad_millis, args.audio_end_pad_millis), flush=True) + + padded_wav_filename = os.path.join(tempdir, 'padded_audio.wav') + # ffmpeg padding: https://superuser.com/a/579110 + # Structure of filters: https://stackoverflow.com/a/55463101 + command = ' '.join([ + 'ffmpeg', + '-i {}'.format(args.audio), + '-filter_complex "[0] adelay={}ms:all=true [a] ; [a] apad=pad_dur={}ms"'.format(args.audio_start_pad_millis, args.audio_end_pad_millis), + '{}'.format(padded_wav_filename) + ]) + print('command:', command, flush=True) + subprocess.call(command, shell=True) + args.audio = padded_wav_filename + + +def maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h): + if not args.end_bump_file: + print('No end bump file.', flush=True) + return + + print('Adding end bump file: {}'.format(args.end_bump_file), flush=True) + + bumpfile = args.end_bump_file + + # May need to resize our bump file... + # After many attempts to get this into the same ffmpeg pipeline, + # I'm giving up and just adding another step. + # + # NOTE: The FPS of the bump needs to match the video. Especially + # important for videos generated from still photos, because that defaults + # to the odd choice of "25 fps". + if frame_w is not 1920 or frame_h is not 1080: + print('Need to resize bump.', flush=True) + bumpfile = os.path.join(tempdir, 'end_bump_resized.mp4') + + # Scale / aspect ratio / sar / resizing / padding: + # https://stackoverflow.com/a/51946719 + command = ' '.join([ + 'ffmpeg', + '-i {}'.format(args.end_bump_file), + '-vf "scale={}:{}:force_original_aspect_ratio=decrease,pad={}:{}:(ow-iw)/2:(oh-ih)/2,setsar=1"'.format(frame_w, frame_h, frame_w, frame_h), + '{}'.format(bumpfile) + ]) + print('command:', command, flush=True) + failure_code = subprocess.call(command, shell=True) + + if failure_code: + print('Failed to resize end bump') + return + + padded_wav_filename = os.path.join(tempdir, 'output_with_end_bump.mp4') + # ffmpeg video concatenation: https://stackoverflow.com/a/11175851 + # Structure of filters: https://stackoverflow.com/a/55463101 + # More info on filters: https://stackoverflow.com/a/22958746 + command = ' '.join([ + 'ffmpeg', + '-i {}'.format(args.outfile), + '-i {}'.format(bumpfile), + '-filter_complex "concat=n=2:v=1:a=1"', + '{}'.format(padded_wav_filename) + ]) + print('command:', command, flush=True) + failure_code = subprocess.call(command, shell=True) + + if failure_code: + print('Failed to concatenate end bump') + else: + print('Done concatenating end bump; renaming file...', flush=True) + os.rename(padded_wav_filename, args.outfile) + + +def main(tempdir): + video_faces_pickle_file = args.face + '.faces' + print('Video faces pickle file: {}'.format(video_faces_pickle_file), flush=True) + + frame_w = 0 + frame_h = 0 + + if not os.path.isfile(args.face): + raise ValueError('--face argument must be a valid path to video/image file') + + elif os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg'] \ + or args.is_image: + full_frames = [cv2.imread(args.face)] + fps = args.fps + frame_h, frame_w = full_frames[0].shape[:-1] + + else: + video_stream = cv2.VideoCapture(args.face) + fps = video_stream.get(cv2.CAP_PROP_FPS) + + print('Reading video frames...') + + full_frames = [] + while 1: + still_reading, frame = video_stream.read() + if not still_reading: + video_stream.release() + break + if args.resize_factor > 1: + frame = cv2.resize(frame, (frame.shape[1]//args.resize_factor, frame.shape[0]//args.resize_factor)) + + if args.rotate: + frame = cv2.rotate(frame, cv2.cv2.ROTATE_90_CLOCKWISE) + + y1, y2, x1, x2 = args.crop + if x2 == -1: x2 = frame.shape[1] + if y2 == -1: y2 = frame.shape[0] + + frame = frame[y1:y2, x1:x2] + + full_frames.append(frame) + + frame_h, frame_w = full_frames[0].shape[:-1] + + print("Number of frames available for inference: "+str(len(full_frames)), flush=True) + print("Frame dimensions: {}x{}".format(frame_w, frame_h), flush=True) + + if not args.audio.endswith('.wav'): + print('Extracting raw audio...') + temp_wav_filename = os.path.join(tempdir, 'temp.wav') # previously 'temp/temp.wav' + command = 'ffmpeg -y -i {} -strict -2 {}'.format(args.audio, temp_wav_filename) + + print('command:', command, flush=True) + subprocess.call(command, shell=True) + args.audio = temp_wav_filename + + maybe_pad_audio_file(tempdir, args) + + wav = audio.load_wav(args.audio, 16000) + mel = audio.melspectrogram(wav) + print(mel.shape) + + if np.isnan(mel.reshape(-1)).sum() > 0: + raise ValueError('Mel contains nan! Using a TTS voice? Add a small epsilon noise to the wav file and try again') + + mel_chunks = [] + mel_idx_multiplier = 80./fps + i = 0 + while 1: + start_idx = int(i * mel_idx_multiplier) + if start_idx + mel_step_size > len(mel[0]): + mel_chunks.append(mel[:, len(mel[0]) - mel_step_size:]) + break + mel_chunks.append(mel[:, start_idx : start_idx + mel_step_size]) + i += 1 + + print("Length of mel chunks: {}".format(len(mel_chunks)), flush=True) + + if len(mel_chunks) > 2000: + # TODO: Catch that the file is too long earlier on. + raise Exception("Too many mel chunks: {}".format(len(mel_chunks))) + + full_frames = full_frames[:len(mel_chunks)] + + # TODO: Do this offline. + print('Face detection time...', flush=True) + if not os.path.isfile(video_faces_pickle_file): + print('Detecting faces...', flush=True) + face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) + else: + with open(video_faces_pickle_file, 'rb') as f: + face_det_results = pickle.load(f) + if len(face_det_results) < len(full_frames): + print("Face detect results have {} frames, but we need {}".format(len(face_det_results), len(full_frames))) + print('Detecting faces again...', flush=True) + face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) + else: + print("We don't need to find faces! Woo!", flush=True) + + face_det_results = face_det_results[:len(mel_chunks)] + + batch_size = args.wav2lip_batch_size + #gen = datagen(full_frames.copy(), mel_chunks) + gen = datagen(full_frames.copy(), face_det_results.copy(), mel_chunks) + + output_video_filename = os.path.join(tempdir, 'result.avi') # previously 'temp/result.avi' + + for i, (img_batch, mel_batch, frames, coords) in enumerate(tqdm(gen, + total=int(np.ceil(float(len(mel_chunks))/batch_size)))): + if i == 0: + model = load_model(args.checkpoint_path) + print ("Model loaded") + + #frame_h, frame_w = full_frames[0].shape[:-1] + out = cv2.VideoWriter(output_video_filename, + cv2.VideoWriter_fourcc(*'DIVX'), fps, (frame_w, frame_h)) + + img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device) + mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device) + + with torch.no_grad(): + pred = model(mel_batch, img_batch) + + pred = pred.cpu().numpy().transpose(0, 2, 3, 1) * 255. + + for p, f, c in zip(pred, frames, coords): + y1, y2, x1, x2 = c + p = cv2.resize(p.astype(np.uint8), (x2 - x1, y2 - y1)) + + f[y1:y2, x1:x2] = p + out.write(f) + + out.release() + + command = 'ffmpeg -y -i {} -i {} -strict -2 -q:v 1 {}'.format(args.audio, output_video_filename, args.outfile) + print('command:', command, flush=True) + subprocess.call(command, shell=True) + + maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h) + +if __name__ == '__main__': + tempdir = tempfile.mkdtemp() + print('Using Python tempdir: {}'.format(tempdir), flush=True) + try: + main(tempdir) + finally: + if not args.preserve_tempdir: + shutil.rmtree(tempdir) + diff --git a/src/script_execution/mod.rs b/src/script_execution/mod.rs index 41465de59d..23c8b87876 100644 --- a/src/script_execution/mod.rs +++ b/src/script_execution/mod.rs @@ -1,2 +1,3 @@ pub mod google_drive_download_command; -pub mod wav2lip_preprocess_command; +pub mod wav2lip_inference_command; +pub mod wav2lip_process_upload_command; diff --git a/src/script_execution/wav2lip_inference_command.rs b/src/script_execution/wav2lip_inference_command.rs new file mode 100644 index 0000000000..cb26a5be61 --- /dev/null +++ b/src/script_execution/wav2lip_inference_command.rs @@ -0,0 +1,125 @@ +use crate::AnyhowResult; +use log::{info,warn}; +use std::process::{Command, Stdio}; +use subprocess::{Popen, PopenConfig, Redirection}; +use std::fs::OpenOptions; + +/// This command is used to preprocess the face detection frames from user-submitted video. +/// This should only ever need to run once. The frames can then be uploaded to Buckets and saved. +#[derive(Clone)] +pub struct Wav2LipInferenceCommand { + w2l_directory: String, + script_name: String, + checkpoint_path: String, +} + +impl Wav2LipInferenceCommand { + pub fn new( + w2l_directory: &str, + script_name: &str, + checkpoint_path: &str, + ) -> Self { + Self { + w2l_directory: w2l_directory.to_string(), + script_name: script_name.to_string(), + checkpoint_path: checkpoint_path.to_string(), + } + } + + pub fn execute(&self, + audio_filename: &str, + image_or_video_filename: &str, + output_video_filename: &str, + is_image: bool, + disable_end_bump: bool, + spawn_process: bool) -> AnyhowResult<()> + { + let mut command = String::new(); + + command.push_str("echo 'test'"); + command.push_str(" && "); + command.push_str(&format!("cd {}", self.w2l_directory)); + command.push_str(" && "); + command.push_str("source python/bin/activate"); + command.push_str(" && "); + command.push_str("python "); + command.push_str(&self.script_name); + command.push_str(" --checkpoint_path "); + command.push_str(&self.checkpoint_path); + command.push_str(" --face "); + command.push_str(image_or_video_filename); + command.push_str(" --audio "); + command.push_str(audio_filename); + command.push_str(" --outfile "); + command.push_str(output_video_filename); + + if is_image { + command.push_str(" --is_image "); + } + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + if spawn_process { + // NB: This forks and returns immediately. + //let _child_pid = command_builder.spawn()?; + + let stdout_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/stdout.txt")?; + + let stderr_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/stderr.txt")?; + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + stdout: Redirection::File(stdout_file), + stderr: Redirection::File(stderr_file), + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + p.detach(); + + } else { + // NB: This is a blocking call. + /*let output = command_builder.output()?; + + info!("Output status: {}", output.status); + info!("Stdout: {:?}", String::from_utf8(output.stdout)); + error!("Stderr: {:?}", String::from_utf8(output.stderr)); + + if !output.status.success() { + bail!("Bad error code: {:?}", output.status); + }*/ + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + let exit_status = p.wait()?; + + info!("Exit status: {:?}", exit_status); + } + + Ok(()) + } +} diff --git a/src/script_execution/wav2lip_preprocess_command.rs b/src/script_execution/wav2lip_process_upload_command.rs similarity index 100% rename from src/script_execution/wav2lip_preprocess_command.rs rename to src/script_execution/wav2lip_process_upload_command.rs From 066cd01dd9e2c96486679d9f62b726dd942e672e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 15:11:40 -0400 Subject: [PATCH 0121/1042] remove unnecessary steps from preprocessing --- models/Wav2Lip/vocodes_process_upload.py | 184 ++---------------- .../wav2lip_process_upload_command.rs | 8 +- 2 files changed, 19 insertions(+), 173 deletions(-) diff --git a/models/Wav2Lip/vocodes_process_upload.py b/models/Wav2Lip/vocodes_process_upload.py index 5d0349094f..cd1cda16b5 100644 --- a/models/Wav2Lip/vocodes_process_upload.py +++ b/models/Wav2Lip/vocodes_process_upload.py @@ -52,10 +52,14 @@ parser.add_argument('--checkpoint_path', type=str, parser.add_argument('--face', type=str, help='Filepath of video/image that contains faces to use', required=True) -parser.add_argument('--audio', type=str, - help='Filepath of video/audio file to use as raw audio source', required=True) -parser.add_argument('--outfile', type=str, help='Video path to save result. See default for an e.g.', - default='results/result_voice.mp4') + +# NB: Not needed for processing the upload file for faces: +# +# parser.add_argument('--audio', type=str, +# help='Filepath of video/audio file to use as raw audio source', required=True) +# parser.add_argument('--outfile', type=str, help='Video path to save result. See default for an e.g.', +# default='results/result_voice.mp4') +# parser.add_argument('--static', type=bool, help='If True, then use only first video frame for inference', default=False) @@ -252,85 +256,9 @@ def load_model(path): model = model.to(device) return model.eval() -def maybe_pad_audio_file(tempdir, args): - if not args.audio_start_pad_millis and not args.audio_end_pad_millis: - return - - print('Padding audio with {} millis at start, {} millis at end.'.format( - args.audio_start_pad_millis, args.audio_end_pad_millis), flush=True) - - padded_wav_filename = os.path.join(tempdir, 'padded_audio.wav') - # ffmpeg padding: https://superuser.com/a/579110 - # Structure of filters: https://stackoverflow.com/a/55463101 - command = ' '.join([ - 'ffmpeg', - '-i {}'.format(args.audio), - '-filter_complex "[0] adelay={}ms:all=true [a] ; [a] apad=pad_dur={}ms"'.format(args.audio_start_pad_millis, args.audio_end_pad_millis), - '{}'.format(padded_wav_filename) - ]) - print('command:', command, flush=True) - subprocess.call(command, shell=True) - args.audio = padded_wav_filename - - -def maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h): - if not args.end_bump_file: - print('No end bump file.', flush=True) - return - - print('Adding end bump file: {}'.format(args.end_bump_file), flush=True) - - bumpfile = args.end_bump_file - - # May need to resize our bump file... - # After many attempts to get this into the same ffmpeg pipeline, - # I'm giving up and just adding another step. - # - # NOTE: The FPS of the bump needs to match the video. Especially - # important for videos generated from still photos, because that defaults - # to the odd choice of "25 fps". - if frame_w is not 1920 or frame_h is not 1080: - print('Need to resize bump.', flush=True) - bumpfile = os.path.join(tempdir, 'end_bump_resized.mp4') - - # Scale / aspect ratio / sar / resizing / padding: - # https://stackoverflow.com/a/51946719 - command = ' '.join([ - 'ffmpeg', - '-i {}'.format(args.end_bump_file), - '-vf "scale={}:{}:force_original_aspect_ratio=decrease,pad={}:{}:(ow-iw)/2:(oh-ih)/2,setsar=1"'.format(frame_w, frame_h, frame_w, frame_h), - '{}'.format(bumpfile) - ]) - print('command:', command, flush=True) - failure_code = subprocess.call(command, shell=True) - - if failure_code: - print('Failed to resize end bump') - return - - padded_wav_filename = os.path.join(tempdir, 'output_with_end_bump.mp4') - # ffmpeg video concatenation: https://stackoverflow.com/a/11175851 - # Structure of filters: https://stackoverflow.com/a/55463101 - # More info on filters: https://stackoverflow.com/a/22958746 - command = ' '.join([ - 'ffmpeg', - '-i {}'.format(args.outfile), - '-i {}'.format(bumpfile), - '-filter_complex "concat=n=2:v=1:a=1"', - '{}'.format(padded_wav_filename) - ]) - print('command:', command, flush=True) - failure_code = subprocess.call(command, shell=True) - - if failure_code: - print('Failed to concatenate end bump') - else: - print('Done concatenating end bump; renaming file...', flush=True) - os.rename(padded_wav_filename, args.outfile) - def main(tempdir): - video_faces_pickle_file = args.face + '.faces' + video_faces_pickle_file = args.face + '.faces.pickle' print('Video faces pickle file: {}'.format(video_faces_pickle_file), flush=True) frame_w = 0 @@ -376,98 +304,16 @@ def main(tempdir): print("Number of frames available for inference: "+str(len(full_frames)), flush=True) print("Frame dimensions: {}x{}".format(frame_w, frame_h), flush=True) - if not args.audio.endswith('.wav'): - print('Extracting raw audio...') - temp_wav_filename = os.path.join(tempdir, 'temp.wav') # previously 'temp/temp.wav' - command = 'ffmpeg -y -i {} -strict -2 {}'.format(args.audio, temp_wav_filename) + # NB: Instead of truncating, let's detect the entire file. + # full_frames = full_frames[:len(mel_chunks)] - print('command:', command, flush=True) - subprocess.call(command, shell=True) - args.audio = temp_wav_filename - - maybe_pad_audio_file(tempdir, args) - - wav = audio.load_wav(args.audio, 16000) - mel = audio.melspectrogram(wav) - print(mel.shape) - - if np.isnan(mel.reshape(-1)).sum() > 0: - raise ValueError('Mel contains nan! Using a TTS voice? Add a small epsilon noise to the wav file and try again') - - mel_chunks = [] - mel_idx_multiplier = 80./fps - i = 0 - while 1: - start_idx = int(i * mel_idx_multiplier) - if start_idx + mel_step_size > len(mel[0]): - mel_chunks.append(mel[:, len(mel[0]) - mel_step_size:]) - break - mel_chunks.append(mel[:, start_idx : start_idx + mel_step_size]) - i += 1 - - print("Length of mel chunks: {}".format(len(mel_chunks)), flush=True) - - if len(mel_chunks) > 2000: - # TODO: Catch that the file is too long earlier on. - raise Exception("Too many mel chunks: {}".format(len(mel_chunks))) - - full_frames = full_frames[:len(mel_chunks)] - - # TODO: Do this offline. print('Face detection time...', flush=True) - if not os.path.isfile(video_faces_pickle_file): - print('Detecting faces...', flush=True) - face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) - else: - with open(video_faces_pickle_file, 'rb') as f: - face_det_results = pickle.load(f) - if len(face_det_results) < len(full_frames): - print("Face detect results have {} frames, but we need {}".format(len(face_det_results), len(full_frames))) - print('Detecting faces again...', flush=True) - face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) - else: - print("We don't need to find faces! Woo!", flush=True) + print('Number of frames to detect faces in: {}'.format(len(full_frames)), flush=True) - face_det_results = face_det_results[:len(mel_chunks)] + print('Detecting faces...', flush=True) + _face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) - batch_size = args.wav2lip_batch_size - #gen = datagen(full_frames.copy(), mel_chunks) - gen = datagen(full_frames.copy(), face_det_results.copy(), mel_chunks) - - output_video_filename = os.path.join(tempdir, 'result.avi') # previously 'temp/result.avi' - - for i, (img_batch, mel_batch, frames, coords) in enumerate(tqdm(gen, - total=int(np.ceil(float(len(mel_chunks))/batch_size)))): - if i == 0: - model = load_model(args.checkpoint_path) - print ("Model loaded") - - #frame_h, frame_w = full_frames[0].shape[:-1] - out = cv2.VideoWriter(output_video_filename, - cv2.VideoWriter_fourcc(*'DIVX'), fps, (frame_w, frame_h)) - - img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device) - mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device) - - with torch.no_grad(): - pred = model(mel_batch, img_batch) - - pred = pred.cpu().numpy().transpose(0, 2, 3, 1) * 255. - - for p, f, c in zip(pred, frames, coords): - y1, y2, x1, x2 = c - p = cv2.resize(p.astype(np.uint8), (x2 - x1, y2 - y1)) - - f[y1:y2, x1:x2] = p - out.write(f) - - out.release() - - command = 'ffmpeg -y -i {} -i {} -strict -2 -q:v 1 {}'.format(args.audio, output_video_filename, args.outfile) - print('command:', command, flush=True) - subprocess.call(command, shell=True) - - maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h) + print('Done detecting faces!', flush=True) if __name__ == '__main__': tempdir = tempfile.mkdtemp() diff --git a/src/script_execution/wav2lip_process_upload_command.rs b/src/script_execution/wav2lip_process_upload_command.rs index 6c1d704040..73df5307d9 100644 --- a/src/script_execution/wav2lip_process_upload_command.rs +++ b/src/script_execution/wav2lip_process_upload_command.rs @@ -48,10 +48,10 @@ impl Wav2LipPreprocessClient { command.push_str(&self.checkpoint_path); command.push_str(" --face "); command.push_str(image_or_video_filename); - command.push_str(" --audio "); - command.push_str(audio_filename); - command.push_str(" --outfile "); - command.push_str(output_video_filename); + //command.push_str(" --audio "); + //command.push_str(audio_filename); + //command.push_str(" --outfile "); + //command.push_str(output_video_filename); if is_image { command.push_str(" --is_image "); From 6a07f5f91d02c7276bb7ee89e6c7d05d20173ec1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 19:22:40 -0400 Subject: [PATCH 0122/1042] Preprocess W2L uploads (but glitchy) --- .env | 5 ++ debug/curl/upload_w2l_image_gdrive.py | 7 ++- migrations/2021-05-15-071601_assets/up.sql | 2 + models/Wav2Lip/vocodes_process_upload.py | 20 ++++--- src/job_queries/w2l_download_job_queries.rs | 9 ++- .../wav2lip_process_upload_command.rs | 16 ++--- src/w2l_download_job.rs | 60 +++++++++++++++++-- 7 files changed, 91 insertions(+), 28 deletions(-) diff --git a/.env b/.env index b978eb5bf5..4f9feebeb5 100644 --- a/.env +++ b/.env @@ -17,3 +17,8 @@ TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models W2L_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates + +W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip +#W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/s3fd-619a316812.pth +W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/wav2lip_gan.pth +W2L_SCRIPT_NAME=vocodes_process_upload.py \ No newline at end of file diff --git a/debug/curl/upload_w2l_image_gdrive.py b/debug/curl/upload_w2l_image_gdrive.py index aaa9ec65fb..aadb14da89 100755 --- a/debug/curl/upload_w2l_image_gdrive.py +++ b/debug/curl/upload_w2l_image_gdrive.py @@ -32,10 +32,15 @@ print("===== Upload W2L =====") cookies = { 'session': session_cookie } +# mario.zip +#download_url = 'https://drive.google.com/file/d/15-tkgblTZpa0ifmvvyr_kgWy2cguTpAe/view?usp=sharing' +# elon_integration_test.jpg +download_url = 'https://drive.google.com/file/d/1yoBSb6nwFUZVI4CD-nM2BSofGdBjC8Nn/view?usp=sharing' + payload = { 'idempotency_token': str(uuid.uuid4()), 'title': 'this is a new image template', - 'download_url': 'https://drive.google.com/file/d/15-tkgblTZpa0ifmvvyr_kgWy2cguTpAe/view?usp=sharing', + 'download_url': download_url, } r = requests.post(upload_url, cookies=cookies, json=payload) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index c680f0cd16..0f1fcf6241 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -192,6 +192,8 @@ CREATE TABLE w2l_templates ( private_bucket_hash CHAR(64) NOT NULL, -- The "full url" version of the path private_bucket_object_name VARCHAR(255) NOT NULL, + -- The "full url" of the cached faces object + private_bucket_cached_faces_object_name VARCHAR(255) NOT NULL, -- For the thumbnail we show. public_bucket_hash CHAR(64) NOT NULL, diff --git a/models/Wav2Lip/vocodes_process_upload.py b/models/Wav2Lip/vocodes_process_upload.py index cd1cda16b5..848f51e11b 100644 --- a/models/Wav2Lip/vocodes_process_upload.py +++ b/models/Wav2Lip/vocodes_process_upload.py @@ -50,9 +50,12 @@ parser = argparse.ArgumentParser(description='Inference code to lip-sync videos parser.add_argument('--checkpoint_path', type=str, help='Name of saved checkpoint to load weights from', required=True) -parser.add_argument('--face', type=str, +parser.add_argument('--image_or_video_filename', type=str, help='Filepath of video/image that contains faces to use', required=True) +parser.add_argument('--output_cached_faces_filename', type=str, + help='Output filename for the cached faces file', required=True) + # NB: Not needed for processing the upload file for faces: # # parser.add_argument('--audio', type=str, @@ -114,7 +117,8 @@ parser.add_argument('--preserve_tempdir', default=False, action='store_true', args = parser.parse_args() args.img_size = 96 -if os.path.isfile(args.face) and os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg']: +if os.path.isfile(args.image_or_video_filename) \ + and os.path.splitext(args.image_or_video_filename)[1] in ['.jpg', '.png', '.jpeg']: args.static = True elif args.is_image: args.static = True @@ -258,23 +262,23 @@ def load_model(path): def main(tempdir): - video_faces_pickle_file = args.face + '.faces.pickle' + video_faces_pickle_file = args.output_cached_faces_filename print('Video faces pickle file: {}'.format(video_faces_pickle_file), flush=True) frame_w = 0 frame_h = 0 - if not os.path.isfile(args.face): - raise ValueError('--face argument must be a valid path to video/image file') + if not os.path.isfile(args.image_or_video_filename): + raise ValueError('--image_or_video_filename argument must be a valid path to video/image file') - elif os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg'] \ + elif os.path.splitext(args.image_or_video_filename)[1] in ['.jpg', '.png', '.jpeg'] \ or args.is_image: - full_frames = [cv2.imread(args.face)] + full_frames = [cv2.imread(args.image_or_video_filename)] fps = args.fps frame_h, frame_w = full_frames[0].shape[:-1] else: - video_stream = cv2.VideoCapture(args.face) + video_stream = cv2.VideoCapture(args.image_or_video_filename) fps = video_stream.get(cv2.CAP_PROP_FPS) print('Reading video frames...') diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 7acfe7a262..0141f95b3d 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -113,7 +113,8 @@ pub async fn insert_w2l_template(pool: &MySqlPool, template_type: &str, // TODO: ENUM! job: &W2lTemplateUploadJobRecord, private_bucket_hash: &str, - private_bucket_object_name: &str) + private_bucket_object_name: &str, + private_bucket_cached_faces_object_name: &str) -> AnyhowResult { let model_token = random_token(32); @@ -133,7 +134,8 @@ SET creator_ip_address = ?, original_download_url = ?, private_bucket_hash = ?, - private_bucket_object_name = ? + private_bucket_object_name = ?, + private_bucket_cached_faces_object_name = ? "#, model_token, template_type, @@ -143,7 +145,8 @@ SET job.creator_ip_address.clone(), job.download_url.clone(), private_bucket_hash.to_string(), - private_bucket_object_name.to_string() + private_bucket_object_name.to_string(), + private_bucket_cached_faces_object_name.to_string() ) .execute(pool) .await; diff --git a/src/script_execution/wav2lip_process_upload_command.rs b/src/script_execution/wav2lip_process_upload_command.rs index 73df5307d9..d5eec7eba7 100644 --- a/src/script_execution/wav2lip_process_upload_command.rs +++ b/src/script_execution/wav2lip_process_upload_command.rs @@ -27,11 +27,9 @@ impl Wav2LipPreprocessClient { } pub fn execute(&self, - audio_filename: &str, image_or_video_filename: &str, - output_video_filename: &str, + output_cached_faces_filename: &str, is_image: bool, - disable_end_bump: bool, spawn_process: bool) -> AnyhowResult<()> { let mut command = String::new(); @@ -46,12 +44,10 @@ impl Wav2LipPreprocessClient { command.push_str(&self.script_name); command.push_str(" --checkpoint_path "); command.push_str(&self.checkpoint_path); - command.push_str(" --face "); + command.push_str(" --image_or_video_filename "); command.push_str(image_or_video_filename); - //command.push_str(" --audio "); - //command.push_str(audio_filename); - //command.push_str(" --outfile "); - //command.push_str(output_video_filename); + command.push_str(" --output_cached_faces_filename"); + command.push_str(output_cached_faces_filename); if is_image { command.push_str(" --is_image "); @@ -74,14 +70,14 @@ impl Wav2LipPreprocessClient { .write(true) .create(true) .truncate(true) - .open("/tmp/stdout.txt")?; + .open("/tmp/wav2lip_upload_stdout.txt")?; let stderr_file = OpenOptions::new() .read(true) .write(true) .create(true) .truncate(true) - .open("/tmp/stderr.txt")?; + .open("/tmp/wav2lip_upload_stderr.txt")?; let mut p = Popen::create(&command_parts, PopenConfig { //stdout: Redirection::Pipe, diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 39f136e204..a7f06e0ee2 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -35,6 +35,7 @@ use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; use tempdir::TempDir; +use crate::script_execution::wav2lip_process_upload_command::Wav2LipPreprocessClient; // Buckets const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -43,6 +44,11 @@ const ENV_REGION_NAME : &'static str = "REGION_NAME"; const ENV_BUCKET_NAME : &'static str = "W2L_DOWNLOAD_BUCKET_NAME"; const ENV_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT"; +// Python code +const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; +const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; +const ENV_SCRIPT_NAME : &'static str = "W2L_SCRIPT_NAME"; + const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; @@ -51,6 +57,7 @@ struct Downloader { pub mysql_pool: MySqlPool, pub bucket_client: BucketClient, pub google_drive_downloader: GoogleDriveDownloadCommand, + pub w2l_processor: Wav2LipPreprocessClient, // Command to run pub download_script: String, // Root to store W2L templates @@ -87,6 +94,16 @@ async fn main() -> AnyhowResult<()> { None, )?; + let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?; + let py_script_name = easyenv::get_env_string_required(ENV_SCRIPT_NAME)?; + let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?; + + let w2l_preprecess_command = Wav2LipPreprocessClient::new( + &py_code_directory, + &py_script_name, + &py_model_checkpoint, + ); + let temp_directory = easyenv::get_env_string_or_default( "DOWNLOAD_TEMP_DIR", DEFAULT_TEMP_DIR); @@ -119,6 +136,7 @@ async fn main() -> AnyhowResult<()> { bucket_client, download_script, google_drive_downloader, + w2l_processor: w2l_preprecess_command, bucket_root_w2l_template_uploads: bucket_root.to_string(), }; @@ -211,23 +229,52 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) .map(|c| c.to_string()) .unwrap_or("".to_string()); + // ==================== DOWNLOAD FILE ==================== // + info!("Calling downloader..."); let download_filename = downloader.google_drive_downloader .download_file(&download_url, &temp_dir).await?; + let file_path = PathBuf::from(&download_filename); + + // ==================== PROCESS FACES ==================== // + + let cached_faces_filename = format!("{}_detected_faces.pickle", &download_filename); + let is_image = false; // TODO: Don't always treat as video. + let spawn_process = true; + + downloader.w2l_processor.execute( + &download_filename, + &cached_faces_filename, + is_image, + spawn_process)?; + + + // ==================== UPLOAD TO BUCKET ==================== // + let private_bucket_hash = get_file_hash(&download_filename)?; info!("File hash: {}", private_bucket_hash); - // NB: /.../a/b/c/d/abcdefg.bin - let object_name = hash_to_bucket_path( + // Full path to video/image + let full_object_path = hash_to_bucket_path( &private_bucket_hash, Some(&downloader.bucket_root_w2l_template_uploads))?; - info!("Destination bucket path: {}", object_name); + info!("Image/video destination bucket path: {}", full_object_path); - let file_path = PathBuf::from(download_filename); - downloader.bucket_client.upload_filename(&object_name, &file_path).await?; + // Full path to cached faces + let full_object_path_cached_faces = format!("{}_detected_faces.pickle", full_object_path); + + info!("Cached faces destination bucket path: {}", full_object_path_cached_faces); + + info!("Uploading image/video..."); + downloader.bucket_client.upload_filename(&full_object_path, &file_path).await?; + + let cached_faces_path = PathBuf::from(&cached_faces_filename); + + info!("Uploading cached faces..."); + downloader.bucket_client.upload_filename(&full_object_path_cached_faces, &cached_faces_path).await?; // TODO: let template_type = "image"; @@ -238,7 +285,8 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) template_type, job, &private_bucket_hash, - &object_name) + &full_object_path, + &full_object_path_cached_faces) .await?; info!("Saved model record: {}", id); From 46b613fef20e33719666516a3fa4f27d1fd559d4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 19:37:04 -0400 Subject: [PATCH 0123/1042] uploading image template processed faces works! --- src/job_queries/w2l_download_job_queries.rs | 4 +++- src/script_execution/wav2lip_process_upload_command.rs | 2 +- src/w2l_download_job.rs | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 0141f95b3d..16d27409ea 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -135,7 +135,9 @@ SET original_download_url = ?, private_bucket_hash = ?, private_bucket_object_name = ?, - private_bucket_cached_faces_object_name = ? + private_bucket_cached_faces_object_name = ?, + public_bucket_hash = 'TODO', + public_bucket_object_name = 'TODO' "#, model_token, template_type, diff --git a/src/script_execution/wav2lip_process_upload_command.rs b/src/script_execution/wav2lip_process_upload_command.rs index d5eec7eba7..ae2268b5dc 100644 --- a/src/script_execution/wav2lip_process_upload_command.rs +++ b/src/script_execution/wav2lip_process_upload_command.rs @@ -46,7 +46,7 @@ impl Wav2LipPreprocessClient { command.push_str(&self.checkpoint_path); command.push_str(" --image_or_video_filename "); command.push_str(image_or_video_filename); - command.push_str(" --output_cached_faces_filename"); + command.push_str(" --output_cached_faces_filename "); command.push_str(output_cached_faces_filename); if is_image { diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index a7f06e0ee2..0efc0c4af4 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -241,7 +241,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) let cached_faces_filename = format!("{}_detected_faces.pickle", &download_filename); let is_image = false; // TODO: Don't always treat as video. - let spawn_process = true; + let spawn_process = false; downloader.w2l_processor.execute( &download_filename, From 5ea339815198c99a81cca6f7c47745907875322b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 19:37:32 -0400 Subject: [PATCH 0124/1042] query to test --- debug/mysql/last_w2l_template.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 debug/mysql/last_w2l_template.sh diff --git a/debug/mysql/last_w2l_template.sh b/debug/mysql/last_w2l_template.sh new file mode 100755 index 0000000000..957943c746 --- /dev/null +++ b/debug/mysql/last_w2l_template.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u root \ + -proot \ + -h localhost \ + -D storyteller \ + -e "select * from w2l_templates\G" From 9cdd6aa4b3de313317adfe526db61f41477aed82 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 20:10:04 -0400 Subject: [PATCH 0125/1042] check that both files exist before upload --- src/util/filesystem.rs | 10 ++++++++++ src/w2l_download_job.rs | 22 +++++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/util/filesystem.rs b/src/util/filesystem.rs index 5a1c8e3d1c..1ad312633e 100644 --- a/src/util/filesystem.rs +++ b/src/util/filesystem.rs @@ -11,3 +11,13 @@ pub fn check_directory_exists(path: &PathBuf) -> AnyhowResult<()> { } Ok(()) } + +pub fn check_file_exists(path: &PathBuf) -> AnyhowResult<()> { + if !path.exists() { + bail!("Path doesn't exist: {:?}", path); + } + if !path.is_file() { + bail!("Path isn't a file: {:?}", path); + } + Ok(()) +} diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 0efc0c4af4..ca3e279b0e 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -23,6 +23,7 @@ use crate::job_queries::w2l_download_job_queries::query_w2l_template_upload_job_ use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; +use crate::util::filesystem::check_file_exists; use crate::util::random_token::random_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; @@ -235,8 +236,6 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) let download_filename = downloader.google_drive_downloader .download_file(&download_url, &temp_dir).await?; - let file_path = PathBuf::from(&download_filename); - // ==================== PROCESS FACES ==================== // let cached_faces_filename = format!("{}_detected_faces.pickle", &download_filename); @@ -249,9 +248,16 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) is_image, spawn_process)?; - // ==================== UPLOAD TO BUCKET ==================== // + let video_or_image_path = PathBuf::from(&download_filename); + let cached_faces_path = &PathBuf::from(&cached_faces_filename); + + info!("Checking that both files exist (original source + cached faces) ..."); + + check_file_exists(&video_or_image_path)?; + check_file_exists(&cached_faces_path)?; + let private_bucket_hash = get_file_hash(&download_filename)?; info!("File hash: {}", private_bucket_hash); @@ -269,12 +275,14 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) info!("Cached faces destination bucket path: {}", full_object_path_cached_faces); info!("Uploading image/video..."); - downloader.bucket_client.upload_filename(&full_object_path, &file_path).await?; - - let cached_faces_path = PathBuf::from(&cached_faces_filename); + downloader.bucket_client.upload_filename( + &full_object_path, + &video_or_image_path).await?; info!("Uploading cached faces..."); - downloader.bucket_client.upload_filename(&full_object_path_cached_faces, &cached_faces_path).await?; + downloader.bucket_client.upload_filename( + &full_object_path_cached_faces, + &cached_faces_path).await?; // TODO: let template_type = "image"; From 003f35b0ba631dc7046489ca26f3a087ef3ba492 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 20:21:49 -0400 Subject: [PATCH 0126/1042] stop logspam --- src/w2l_download_job.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index ca3e279b0e..008e400955 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -50,7 +50,8 @@ const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; const ENV_SCRIPT_NAME : &'static str = "W2L_SCRIPT_NAME"; -const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; +// NB: sqlx::query is spammy and logs all queries as "info"-level +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Downloader { @@ -297,9 +298,9 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) &full_object_path_cached_faces) .await?; - info!("Saved model record: {}", id); + info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}", + job.id, id); - info!("Job done: {}", job.id); mark_w2l_template_upload_job_done(&downloader.mysql_pool, job, true).await?; Ok(()) From fa4a43c8916097d6b4ca3a0673bedfbab5b8602a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 23:39:04 -0400 Subject: [PATCH 0127/1042] notes --- src/w2l_download_job.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 008e400955..2b8521f6e9 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -218,11 +218,12 @@ async fn process_jobs(downloader: &Downloader, jobs: Vec AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. (DONE) - // TODO: 3. Process template with face detection - // TODO: 4. Take a screenshot/gif - // TODO: 5. Upload all (partially done). - // TODO: 6. Save record. (DONE) - // TODO: 7. Mark job done. (DONE) + // TODO: 3. Process template with face detection (DONE) + // TODO: 4. Determine if picture or video + // TODO: 5. Take a screenshot/gif + // TODO: 6. Upload all (partially done). + // TODO: 7. Save record. (DONE) + // TODO: 8. Mark job done. (DONE) let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; From a5349405098153a030a3a590962d8faa804b57d6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 May 2021 23:41:02 -0400 Subject: [PATCH 0128/1042] test with video --- debug/curl/upload_w2l_video_gdrive.py | 55 +++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100755 debug/curl/upload_w2l_video_gdrive.py diff --git a/debug/curl/upload_w2l_video_gdrive.py b/debug/curl/upload_w2l_video_gdrive.py new file mode 100755 index 0000000000..be64549005 --- /dev/null +++ b/debug/curl/upload_w2l_video_gdrive.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import requests +import re +import uuid + +login_url = 'http://localhost:12345/login' +upload_url = 'http://localhost:12345/w2l/upload' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +# NB: May not be set due to cookie domain: +#r.cookies['session'] + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + + +print("===== Upload W2L =====") + +cookies = { 'session': session_cookie } + +# mario.zip +#download_url = 'https://drive.google.com/file/d/15-tkgblTZpa0ifmvvyr_kgWy2cguTpAe/view?usp=sharing' +# elon_integration_test.jpg +#download_url = 'https://drive.google.com/file/d/1yoBSb6nwFUZVI4CD-nM2BSofGdBjC8Nn/view?usp=sharing' +# bill-integration-test.mp4 +# NB: This freezes up my poor laptop: +###download_url = 'https://drive.google.com/file/d/14yO_60lc1FJCYhDGaeiQabgGvYLa86WI/THIS_IS_TOO_BIGview?usp=sharing' +# bill-integration-small.mp4 (small enough to run on my laptop) +download_url = 'https://drive.google.com/file/d/1ysBAdRjlSmfWQPFjGlMJpv_zcGmVw7RC/view?usp=sharing' + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'title': 'this is a new image template', + 'download_url': download_url, +} + +r = requests.post(upload_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + From aacc6e5fb9c0a915dc2f271ad1d6fdf0c2f1d22d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 00:54:49 -0400 Subject: [PATCH 0129/1042] Read file metadata and populate DB --- migrations/2021-05-15-071601_assets/up.sql | 6 +++ models/Wav2Lip/vocodes_process_upload.py | 24 +++++++++++ src/job_queries/w2l_download_job_queries.rs | 18 ++++++-- .../wav2lip_process_upload_command.rs | 3 ++ src/w2l_download_job.rs | 42 +++++++++++++++++-- 5 files changed, 86 insertions(+), 7 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 0f1fcf6241..2628966066 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -185,6 +185,12 @@ CREATE TABLE w2l_templates ( -- The filename that was used at upload time. original_filename CHAR(255) DEFAULT NULL, + -- Dimensions and other characteristics + frame_width INT(10) NOT NULL DEFAULT 0, + frame_height INT(10) NOT NULL DEFAULT 0, + frame_count INT(10) NOT NULL DEFAULT 0, + fps FLOAT NOT NULL DEFAULT 0.0, + -- Where the file was originally downloaded (if it was downloaded) original_download_url VARCHAR(512) DEFAULT NULL, diff --git a/models/Wav2Lip/vocodes_process_upload.py b/models/Wav2Lip/vocodes_process_upload.py index 848f51e11b..4359c1971e 100644 --- a/models/Wav2Lip/vocodes_process_upload.py +++ b/models/Wav2Lip/vocodes_process_upload.py @@ -56,6 +56,9 @@ parser.add_argument('--image_or_video_filename', type=str, parser.add_argument('--output_cached_faces_filename', type=str, help='Output filename for the cached faces file', required=True) +parser.add_argument('--output_metadata_filename', type=str, + help='Output filename for the JSON containing width, height, etc.', required=True) + # NB: Not needed for processing the upload file for faces: # # parser.add_argument('--audio', type=str, @@ -265,8 +268,10 @@ def main(tempdir): video_faces_pickle_file = args.output_cached_faces_filename print('Video faces pickle file: {}'.format(video_faces_pickle_file), flush=True) + is_video = False frame_w = 0 frame_h = 0 + fps = 0 if not os.path.isfile(args.image_or_video_filename): raise ValueError('--image_or_video_filename argument must be a valid path to video/image file') @@ -276,6 +281,7 @@ def main(tempdir): full_frames = [cv2.imread(args.image_or_video_filename)] fps = args.fps frame_h, frame_w = full_frames[0].shape[:-1] + is_video = False else: video_stream = cv2.VideoCapture(args.image_or_video_filename) @@ -305,6 +311,11 @@ def main(tempdir): frame_h, frame_w = full_frames[0].shape[:-1] + if len(full_frames) < 2: + is_video = False + else: + is_video = True + print("Number of frames available for inference: "+str(len(full_frames)), flush=True) print("Frame dimensions: {}x{}".format(frame_w, frame_h), flush=True) @@ -319,6 +330,19 @@ def main(tempdir): print('Done detecting faces!', flush=True) + metadata = { + 'is_video': is_video, + 'width': frame_w, + 'height': frame_h, + 'num_frames': len(full_frames), + } + + if is_video: + metadata['fps'] = fps + + with open(args.output_metadata_filename, 'w') as json_file: + json.dump(metadata, json_file) + if __name__ == '__main__': tempdir = tempfile.mkdtemp() print('Using Python tempdir: {}'.format(tempdir), flush=True) diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 16d27409ea..155592af82 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -114,7 +114,11 @@ pub async fn insert_w2l_template(pool: &MySqlPool, job: &W2lTemplateUploadJobRecord, private_bucket_hash: &str, private_bucket_object_name: &str, - private_bucket_cached_faces_object_name: &str) + private_bucket_cached_faces_object_name: &str, + frame_width: u32, + frame_height: u32, + frame_count: u64, + fps: f32) -> AnyhowResult { let model_token = random_token(32); @@ -137,7 +141,11 @@ SET private_bucket_object_name = ?, private_bucket_cached_faces_object_name = ?, public_bucket_hash = 'TODO', - public_bucket_object_name = 'TODO' + public_bucket_object_name = 'TODO', + frame_width = ?, + frame_height = ?, + frame_count = ?, + fps = ? "#, model_token, template_type, @@ -148,7 +156,11 @@ SET job.download_url.clone(), private_bucket_hash.to_string(), private_bucket_object_name.to_string(), - private_bucket_cached_faces_object_name.to_string() + private_bucket_cached_faces_object_name.to_string(), + frame_width, + frame_height, + frame_count, + fps ) .execute(pool) .await; diff --git a/src/script_execution/wav2lip_process_upload_command.rs b/src/script_execution/wav2lip_process_upload_command.rs index ae2268b5dc..5092a1d0f1 100644 --- a/src/script_execution/wav2lip_process_upload_command.rs +++ b/src/script_execution/wav2lip_process_upload_command.rs @@ -29,6 +29,7 @@ impl Wav2LipPreprocessClient { pub fn execute(&self, image_or_video_filename: &str, output_cached_faces_filename: &str, + output_metadata_filename: &str, is_image: bool, spawn_process: bool) -> AnyhowResult<()> { @@ -48,6 +49,8 @@ impl Wav2LipPreprocessClient { command.push_str(image_or_video_filename); command.push_str(" --output_cached_faces_filename "); command.push_str(output_cached_faces_filename); + command.push_str(" --output_metadata_filename "); + command.push_str(output_metadata_filename); if is_image { command.push_str(" --is_image "); diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 2b8521f6e9..04ffc45c31 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -5,6 +5,8 @@ #![warn(unused_must_use)] //#![allow(warnings)] +#[macro_use] extern crate serde_derive; + pub mod buckets; pub mod job_queries; pub mod script_execution; @@ -215,6 +217,22 @@ async fn process_jobs(downloader: &Downloader, jobs: Vec, +} + +fn read_metadata_file(filename: &str) -> AnyhowResult { + let mut file = File::open(filename)?; + let mut buffer = String::new(); + file.read_to_string(&mut buffer)?; + Ok(serde_json::from_str(&buffer)?) +} + async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. (DONE) @@ -240,25 +258,38 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) // ==================== PROCESS FACES ==================== // + // This is the Python Pickle file with all the face frames. + // We'll retain it forever since it's expensive to compute. let cached_faces_filename = format!("{}_detected_faces.pickle", &download_filename); + + // This is a file that we'll use to determine if the file is an image or video. + let output_metadata_filename = format!("{}_metadata.json", &download_filename); + let is_image = false; // TODO: Don't always treat as video. let spawn_process = false; downloader.w2l_processor.execute( &download_filename, &cached_faces_filename, + &output_metadata_filename, is_image, spawn_process)?; - // ==================== UPLOAD TO BUCKET ==================== // + // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // let video_or_image_path = PathBuf::from(&download_filename); let cached_faces_path = &PathBuf::from(&cached_faces_filename); + let output_metadata_path = &PathBuf::from(&output_metadata_filename); info!("Checking that both files exist (original source + cached faces) ..."); check_file_exists(&video_or_image_path)?; check_file_exists(&cached_faces_path)?; + check_file_exists(&output_metadata_path)?; + + let file_metadata = read_metadata_file(&output_metadata_filename)?; + + // ==================== UPLOAD TO BUCKET ==================== // let private_bucket_hash = get_file_hash(&download_filename)?; @@ -286,8 +317,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) &full_object_path_cached_faces, &cached_faces_path).await?; - // TODO: - let template_type = "image"; + let template_type = if file_metadata.is_video { "video" } else { "image" }; info!("Saving model record..."); let id = insert_w2l_template( @@ -296,7 +326,11 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) job, &private_bucket_hash, &full_object_path, - &full_object_path_cached_faces) + &full_object_path_cached_faces, + file_metadata.width, + file_metadata.height, + file_metadata.num_frames, + file_metadata.fps.unwrap_or(0.0f32)) .await?; info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}", From d45b1f6fb26b5dde48a7938f594cd658daad3c6b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 01:27:26 -0400 Subject: [PATCH 0130/1042] ffmpeg to generate video previews! --- migrations/2021-05-15-071601_assets/up.sql | 4 + src/job_queries/w2l_download_job_queries.rs | 6 + .../ffmpeg_generate_preview_video_command.rs | 107 ++++++++++++++++++ src/script_execution/mod.rs | 1 + src/w2l_download_job.rs | 51 ++++++++- 5 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 src/script_execution/ffmpeg_generate_preview_video_command.rs diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 2628966066..a96a3a2693 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -201,6 +201,10 @@ CREATE TABLE w2l_templates ( -- The "full url" of the cached faces object private_bucket_cached_faces_object_name VARCHAR(255) NOT NULL, + -- The "full url" of the preview image and/or video + maybe_private_bucket_preview_image_object_name VARCHAR(255) DEFAULT NULL, + maybe_private_bucket_preview_video_object_name VARCHAR(255) DEFAULT NULL, + -- For the thumbnail we show. public_bucket_hash CHAR(64) NOT NULL, -- The "full url" version of the path diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 155592af82..67d66a0343 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -115,6 +115,8 @@ pub async fn insert_w2l_template(pool: &MySqlPool, private_bucket_hash: &str, private_bucket_object_name: &str, private_bucket_cached_faces_object_name: &str, + maybe_image_preview_object_name: Option<&str>, + maybe_video_preview_object_name: Option<&str>, frame_width: u32, frame_height: u32, frame_count: u64, @@ -142,6 +144,8 @@ SET private_bucket_cached_faces_object_name = ?, public_bucket_hash = 'TODO', public_bucket_object_name = 'TODO', + maybe_private_bucket_preview_image_object_name = ?, + maybe_private_bucket_preview_video_object_name = ?, frame_width = ?, frame_height = ?, frame_count = ?, @@ -157,6 +161,8 @@ SET private_bucket_hash.to_string(), private_bucket_object_name.to_string(), private_bucket_cached_faces_object_name.to_string(), + maybe_image_preview_object_name, + maybe_video_preview_object_name, frame_width, frame_height, frame_count, diff --git a/src/script_execution/ffmpeg_generate_preview_video_command.rs b/src/script_execution/ffmpeg_generate_preview_video_command.rs new file mode 100644 index 0000000000..e4f2f429a4 --- /dev/null +++ b/src/script_execution/ffmpeg_generate_preview_video_command.rs @@ -0,0 +1,107 @@ +use crate::AnyhowResult; +use log::{info,warn}; +use std::process::{Command, Stdio}; +use subprocess::{Popen, PopenConfig, Redirection}; +use std::fs::OpenOptions; + +/// This command is used to generate ffmpeg previews for videos. +#[derive(Clone)] +pub struct FfmpegGeneratePreviewVideoCommand { +} + +impl FfmpegGeneratePreviewVideoCommand { + pub fn new() -> Self { + Self { + } + } + + pub fn execute(&self, + input_video_filename: &str, + output_video_filename: &str, + output_width: u32, + output_height: u32, + spawn_process: bool) -> AnyhowResult<()> + { + let mut command = String::new(); + + let dimensions = format!("{}x{}", output_width, output_height); + + command.push_str("echo 'test'"); + command.push_str(" && "); + command.push_str("ffmpeg"); + command.push_str(" -i "); + command.push_str(&input_video_filename); + command.push_str(" -lossless 0 "); + command.push_str(" -ss 00:00:00 "); + command.push_str(" -t 00:00:03 "); + command.push_str(" -s "); + command.push_str(&dimensions); + command.push_str(" "); // NB: no arg flag for output filename + command.push_str(&output_video_filename); + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + if spawn_process { + // NB: This forks and returns immediately. + //let _child_pid = command_builder.spawn()?; + + let stdout_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/ffmpeg_stdout.txt")?; + + let stderr_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/ffmpeg_stderr.txt")?; + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + stdout: Redirection::File(stdout_file), + stderr: Redirection::File(stderr_file), + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + p.detach(); + + } else { + // NB: This is a blocking call. + /*let output = command_builder.output()?; + + info!("Output status: {}", output.status); + info!("Stdout: {:?}", String::from_utf8(output.stdout)); + error!("Stderr: {:?}", String::from_utf8(output.stderr)); + + if !output.status.success() { + bail!("Bad error code: {:?}", output.status); + }*/ + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + let exit_status = p.wait()?; + + info!("Exit status: {:?}", exit_status); + } + + Ok(()) + } +} diff --git a/src/script_execution/mod.rs b/src/script_execution/mod.rs index 23c8b87876..4e01cca42a 100644 --- a/src/script_execution/mod.rs +++ b/src/script_execution/mod.rs @@ -1,3 +1,4 @@ +pub mod ffmpeg_generate_preview_video_command; pub mod google_drive_download_command; pub mod wav2lip_inference_command; pub mod wav2lip_process_upload_command; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 04ffc45c31..0e6958c9c4 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -39,6 +39,7 @@ use std::process::Command; use std::time::Duration; use tempdir::TempDir; use crate::script_execution::wav2lip_process_upload_command::Wav2LipPreprocessClient; +use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; // Buckets const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -62,6 +63,7 @@ struct Downloader { pub bucket_client: BucketClient, pub google_drive_downloader: GoogleDriveDownloadCommand, pub w2l_processor: Wav2LipPreprocessClient, + pub ffmpeg_preview_generator: FfmpegGeneratePreviewVideoCommand, // Command to run pub download_script: String, // Root to store W2L templates @@ -140,6 +142,7 @@ async fn main() -> AnyhowResult<()> { bucket_client, download_script, google_drive_downloader, + ffmpeg_preview_generator: FfmpegGeneratePreviewVideoCommand {}, w2l_processor: w2l_preprecess_command, bucket_root_w2l_template_uploads: bucket_root.to_string(), }; @@ -289,7 +292,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) let file_metadata = read_metadata_file(&output_metadata_filename)?; - // ==================== UPLOAD TO BUCKET ==================== // + // ==================== BASE OBJECT NAMES BASED ON HASH ==================== // let private_bucket_hash = get_file_hash(&download_filename)?; @@ -300,6 +303,38 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) &private_bucket_hash, Some(&downloader.bucket_root_w2l_template_uploads))?; + // ==================== GENERATE VIDEO PREVIEWS ==================== // + + let mut maybe_image_preview_filename : Option = None; + let mut maybe_image_preview_object_name : Option = None; + + let mut maybe_video_preview_filename : Option = None; + let mut maybe_video_preview_object_name : Option = None; + + if file_metadata.is_video { + let preview_filename = format!("{}_preview.webp", &download_filename); + + downloader.ffmpeg_preview_generator.execute( + &download_filename, + &preview_filename, + 500, + 500, + false + )?; + + let video_preview_path = PathBuf::from(&preview_filename); + check_file_exists(&video_preview_path)?; + + let preview_object_path = format!("{}_preview.webp", full_object_path); + maybe_video_preview_object_name = Some(preview_object_path); + maybe_video_preview_filename = Some(video_preview_path); + + } else { + // TODO + } + + // ==================== UPLOAD TO BUCKET ==================== // + info!("Image/video destination bucket path: {}", full_object_path); // Full path to cached faces @@ -317,6 +352,18 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) &full_object_path_cached_faces, &cached_faces_path).await?; + // TODO: Fix this ugh. + if let Some(video_preview_filename) = maybe_video_preview_filename.as_deref() { + if let Some(video_preview_object_name) = maybe_video_preview_object_name.as_deref() { + info!("Uploading video preview..."); + downloader.bucket_client.upload_filename( + &video_preview_object_name, + video_preview_filename).await?; + } + } + + // ==================== SAVE RECORDS ==================== // + let template_type = if file_metadata.is_video { "video" } else { "image" }; info!("Saving model record..."); @@ -327,6 +374,8 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) &private_bucket_hash, &full_object_path, &full_object_path_cached_faces, + maybe_image_preview_object_name.as_deref(), + maybe_video_preview_object_name.as_deref(), file_metadata.width, file_metadata.height, file_metadata.num_frames, From 6036934dd9b5091f6ed325548f42173e73c4e545 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 01:39:20 -0400 Subject: [PATCH 0131/1042] looping webp --- src/buckets/bucket_client.rs | 29 +++++++++++++++++++ .../ffmpeg_generate_preview_video_command.rs | 6 ++++ src/w2l_download_job.rs | 6 ++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/buckets/bucket_client.rs b/src/buckets/bucket_client.rs index f7915b7e12..3956ab8d51 100644 --- a/src/buckets/bucket_client.rs +++ b/src/buckets/bucket_client.rs @@ -78,6 +78,24 @@ impl BucketClient { Ok(()) } + pub async fn upload_file_with_content_type(&self, object_name: &str, bytes: &[u8], content_type: &str) -> anyhow::Result<()> { + info!("Filename for bucket: {}", object_name); + + let object_name = self.get_rooted_object_name(object_name); + info!("Rooted filename for bucket: {}", object_name); + + let (body_bytes, code) = self.bucket.put_object_with_content_type(&object_name, bytes, content_type).await?; + + info!("upload code: {}", code); + + if code != 200 { + let body = String::from_utf8(body_bytes)?; + warn!("upload body: {}", body); + } + + Ok(()) + } + // NB: New version has blocking client rather than blocking calls. // pub fn upload_file_blocking(&self, object_name: &str, bytes: &[u8]) -> anyhow::Result<()> { // info!("Filename for bucket: {}", object_name); @@ -112,6 +130,17 @@ impl BucketClient { self.upload_file(object_name, &buffer).await } + pub async fn upload_filename_with_content_type(&self, object_name: &str, filename: &Path, content_type: &str) -> anyhow::Result<()> { + // TODO: does a newer version of this crate handle streaming/buffering file contents? + let mut file = File::open(filename).await?; + let mut buffer : Vec = Vec::new(); + file.read_to_end(&mut buffer).await?; + + info!("Uploading with content type..."); + + self.upload_file_with_content_type(object_name, &buffer, content_type).await + } + // NB: New version has blocking client rather than blocking calls. // pub fn upload_filename_blocking(&self, object_name: &str, filename: &Path) -> anyhow::Result<()> { // // TODO: does a newer version of this crate handle streaming/buffering file contents? diff --git a/src/script_execution/ffmpeg_generate_preview_video_command.rs b/src/script_execution/ffmpeg_generate_preview_video_command.rs index e4f2f429a4..d160c6e077 100644 --- a/src/script_execution/ffmpeg_generate_preview_video_command.rs +++ b/src/script_execution/ffmpeg_generate_preview_video_command.rs @@ -20,6 +20,7 @@ impl FfmpegGeneratePreviewVideoCommand { output_video_filename: &str, output_width: u32, output_height: u32, + looping: bool, spawn_process: bool) -> AnyhowResult<()> { let mut command = String::new(); @@ -36,6 +37,11 @@ impl FfmpegGeneratePreviewVideoCommand { command.push_str(" -t 00:00:03 "); command.push_str(" -s "); command.push_str(&dimensions); + + if looping { + command.push_str(" -loop 0 "); // NB: -loop 1 disables looping + } + command.push_str(" "); // NB: no arg flag for output filename command.push_str(&output_video_filename); diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 0e6958c9c4..21c8c6bbca 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -319,6 +319,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) &preview_filename, 500, 500, + true, false )?; @@ -356,9 +357,10 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) if let Some(video_preview_filename) = maybe_video_preview_filename.as_deref() { if let Some(video_preview_object_name) = maybe_video_preview_object_name.as_deref() { info!("Uploading video preview..."); - downloader.bucket_client.upload_filename( + downloader.bucket_client.upload_filename_with_content_type( &video_preview_object_name, - video_preview_filename).await?; + video_preview_filename, + "image/webp").await?; } } From 6afe1c168be5fcf13ee0220623053606b2eaeceb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 01:50:02 -0400 Subject: [PATCH 0132/1042] save duration millis --- migrations/2021-05-15-071601_assets/up.sql | 1 + models/Wav2Lip/vocodes_process_upload.py | 10 +++++++++- src/job_queries/w2l_download_job_queries.rs | 9 ++++++--- src/w2l_download_job.rs | 4 +++- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index a96a3a2693..1c797e8d60 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -190,6 +190,7 @@ CREATE TABLE w2l_templates ( frame_height INT(10) NOT NULL DEFAULT 0, frame_count INT(10) NOT NULL DEFAULT 0, fps FLOAT NOT NULL DEFAULT 0.0, + duration_millis INT(1) NOT NULL DEFAULT 0, -- Where the file was originally downloaded (if it was downloaded) original_download_url VARCHAR(512) DEFAULT NULL, diff --git a/models/Wav2Lip/vocodes_process_upload.py b/models/Wav2Lip/vocodes_process_upload.py index 4359c1971e..d766af1ec1 100644 --- a/models/Wav2Lip/vocodes_process_upload.py +++ b/models/Wav2Lip/vocodes_process_upload.py @@ -271,7 +271,9 @@ def main(tempdir): is_video = False frame_w = 0 frame_h = 0 - fps = 0 + frame_count = 0 + fps = 0.0 + duration_millis = 0.0 if not os.path.isfile(args.image_or_video_filename): raise ValueError('--image_or_video_filename argument must be a valid path to video/image file') @@ -282,6 +284,7 @@ def main(tempdir): fps = args.fps frame_h, frame_w = full_frames[0].shape[:-1] is_video = False + frame_count = len(full_frames) else: video_stream = cv2.VideoCapture(args.image_or_video_filename) @@ -313,8 +316,12 @@ def main(tempdir): if len(full_frames) < 2: is_video = False + frame_count = len(full_frames) else: is_video = True + frame_count = len(full_frames) + duration_seconds = frame_count / fps + duration_millis = int(duration_seconds * 1000) print("Number of frames available for inference: "+str(len(full_frames)), flush=True) print("Frame dimensions: {}x{}".format(frame_w, frame_h), flush=True) @@ -339,6 +346,7 @@ def main(tempdir): if is_video: metadata['fps'] = fps + metadata['duration_millis'] = duration_millis with open(args.output_metadata_filename, 'w') as json_file: json.dump(metadata, json_file) diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 67d66a0343..01de3299de 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -120,7 +120,8 @@ pub async fn insert_w2l_template(pool: &MySqlPool, frame_width: u32, frame_height: u32, frame_count: u64, - fps: f32) + fps: f32, + duration_millis: u64) -> AnyhowResult { let model_token = random_token(32); @@ -149,7 +150,8 @@ SET frame_width = ?, frame_height = ?, frame_count = ?, - fps = ? + fps = ?, + duration_millis = ? "#, model_token, template_type, @@ -166,7 +168,8 @@ SET frame_width, frame_height, frame_count, - fps + fps, + duration_millis ) .execute(pool) .await; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 21c8c6bbca..e020182757 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -227,6 +227,7 @@ struct FileMetadata { pub height: u32, pub num_frames: u64, pub fps: Option, + pub duration_millis: Option, } fn read_metadata_file(filename: &str) -> AnyhowResult { @@ -381,7 +382,8 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) file_metadata.width, file_metadata.height, file_metadata.num_frames, - file_metadata.fps.unwrap_or(0.0f32)) + file_metadata.fps.unwrap_or(0.0f32), + file_metadata.duration_millis.unwrap_or(0)) .await?; info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}", From aa493ad1974e6fa9329abc67cfab374092eb9127 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 02:09:36 -0400 Subject: [PATCH 0133/1042] generate preview image (fails though...) --- .../ffmpeg_generate_preview_image_command.rs | 105 ++++++++++++++++++ src/script_execution/mod.rs | 1 + src/w2l_download_job.rs | 37 +++++- 3 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 src/script_execution/ffmpeg_generate_preview_image_command.rs diff --git a/src/script_execution/ffmpeg_generate_preview_image_command.rs b/src/script_execution/ffmpeg_generate_preview_image_command.rs new file mode 100644 index 0000000000..310e1b2061 --- /dev/null +++ b/src/script_execution/ffmpeg_generate_preview_image_command.rs @@ -0,0 +1,105 @@ +use crate::AnyhowResult; +use log::{info,warn}; +use std::process::{Command, Stdio}; +use subprocess::{Popen, PopenConfig, Redirection}; +use std::fs::OpenOptions; + +/// This command is used to generate ffmpeg previews for videos. +#[derive(Clone)] +pub struct FfmpegGeneratePreviewImageCommand { +} + +impl FfmpegGeneratePreviewImageCommand { + pub fn new() -> Self { + Self { + } + } + + pub fn execute(&self, + input_image_filename: &str, + output_image_filename: &str, + output_width: u32, + output_height: u32, + spawn_process: bool) -> AnyhowResult<()> + { + let mut command = String::new(); + + let scale = format!("scale={}:{}", output_width, output_height); + + command.push_str("echo 'test'"); + command.push_str(" && "); + command.push_str("ffmpeg"); + command.push_str(" -i "); + command.push_str(&input_image_filename); + command.push_str(" -vf "); + command.push_str(&scale); + + command.push_str(" "); // NB: no arg flag for output filename + command.push_str(&output_image_filename); + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + if spawn_process { + // NB: This forks and returns immediately. + //let _child_pid = command_builder.spawn()?; + + let stdout_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/ffmpeg_stdout.txt")?; + + let stderr_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/ffmpeg_stderr.txt")?; + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + stdout: Redirection::File(stdout_file), + stderr: Redirection::File(stderr_file), + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + p.detach(); + + } else { + // NB: This is a blocking call. + /*let output = command_builder.output()?; + + info!("Output status: {}", output.status); + info!("Stdout: {:?}", String::from_utf8(output.stdout)); + error!("Stderr: {:?}", String::from_utf8(output.stderr)); + + if !output.status.success() { + bail!("Bad error code: {:?}", output.status); + }*/ + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + let exit_status = p.wait()?; + + info!("Exit status: {:?}", exit_status); + } + + Ok(()) + } +} diff --git a/src/script_execution/mod.rs b/src/script_execution/mod.rs index 4e01cca42a..d433b74d78 100644 --- a/src/script_execution/mod.rs +++ b/src/script_execution/mod.rs @@ -1,3 +1,4 @@ +pub mod ffmpeg_generate_preview_image_command; pub mod ffmpeg_generate_preview_video_command; pub mod google_drive_download_command; pub mod wav2lip_inference_command; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index e020182757..7752a4de66 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -40,6 +40,7 @@ use std::time::Duration; use tempdir::TempDir; use crate::script_execution::wav2lip_process_upload_command::Wav2LipPreprocessClient; use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; +use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; // Buckets const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -63,7 +64,8 @@ struct Downloader { pub bucket_client: BucketClient, pub google_drive_downloader: GoogleDriveDownloadCommand, pub w2l_processor: Wav2LipPreprocessClient, - pub ffmpeg_preview_generator: FfmpegGeneratePreviewVideoCommand, + pub ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand, + pub ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand, // Command to run pub download_script: String, // Root to store W2L templates @@ -142,7 +144,8 @@ async fn main() -> AnyhowResult<()> { bucket_client, download_script, google_drive_downloader, - ffmpeg_preview_generator: FfmpegGeneratePreviewVideoCommand {}, + ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, + ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, w2l_processor: w2l_preprecess_command, bucket_root_w2l_template_uploads: bucket_root.to_string(), }; @@ -315,7 +318,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) if file_metadata.is_video { let preview_filename = format!("{}_preview.webp", &download_filename); - downloader.ffmpeg_preview_generator.execute( + downloader.ffmpeg_video_preview_generator.execute( &download_filename, &preview_filename, 500, @@ -332,7 +335,22 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) maybe_video_preview_filename = Some(video_preview_path); } else { - // TODO + let preview_filename = format!("{}_preview.webp", &download_filename); + + downloader.ffmpeg_image_preview_generator.execute( + &download_filename, + &preview_filename, + 500, + 500, + false + )?; + + let image_preview_path = PathBuf::from(&preview_filename); + check_file_exists(&image_preview_path)?; + + let preview_object_path = format!("{}_preview.webp", full_object_path); + maybe_image_preview_object_name = Some(preview_object_path); + maybe_image_preview_filename = Some(image_preview_path); } // ==================== UPLOAD TO BUCKET ==================== // @@ -354,6 +372,17 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) &full_object_path_cached_faces, &cached_faces_path).await?; + // TODO: Fix this ugh. + if let Some(image_preview_filename) = maybe_image_preview_filename.as_deref() { + if let Some(image_preview_object_name) = maybe_image_preview_object_name.as_deref() { + info!("Uploading image preview..."); + downloader.bucket_client.upload_filename_with_content_type( + &image_preview_object_name, + image_preview_filename, + "image/webp").await?; + } + } + // TODO: Fix this ugh. if let Some(video_preview_filename) = maybe_video_preview_filename.as_deref() { if let Some(video_preview_object_name) = maybe_video_preview_object_name.as_deref() { From 107f14104e1b680497a611b6e0102e565efc9df9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 02:17:29 -0400 Subject: [PATCH 0134/1042] imagemagick works better! --- ...gemagick_generate_preview_image_command.rs | 108 ++++++++++++++++++ src/script_execution/mod.rs | 1 + src/w2l_download_job.rs | 5 +- 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 src/script_execution/imagemagick_generate_preview_image_command.rs diff --git a/src/script_execution/imagemagick_generate_preview_image_command.rs b/src/script_execution/imagemagick_generate_preview_image_command.rs new file mode 100644 index 0000000000..878bf75d80 --- /dev/null +++ b/src/script_execution/imagemagick_generate_preview_image_command.rs @@ -0,0 +1,108 @@ +use crate::AnyhowResult; +use log::{info,warn}; +use std::process::{Command, Stdio}; +use subprocess::{Popen, PopenConfig, Redirection}; +use std::fs::OpenOptions; + +/// This command is used to generate ffmpeg previews for videos. +#[derive(Clone)] +pub struct ImagemagickGeneratePreviewImageCommand { +} + +impl ImagemagickGeneratePreviewImageCommand { + pub fn new() -> Self { + Self { + } + } + + pub fn execute(&self, + input_image_filename: &str, + output_image_filename: &str, + output_width: u32, + output_height: u32, + spawn_process: bool) -> AnyhowResult<()> + { + let mut command = String::new(); + + let dimensions = format!("{}x{}", output_width, output_height); + + // magick wizard.png -quality 50 -define webp:lossless=true wizard.webp + command.push_str("echo 'test'"); + command.push_str(" && "); + //command.push_str("magick"); + command.push_str("convert "); + command.push_str(&input_image_filename); + command.push_str(" -size "); + command.push_str(&dimensions); + command.push_str(" -quality 50 "); + command.push_str(" -define webp:loseless=false "); + + command.push_str(" "); // NB: no arg flag for output filename + command.push_str(&output_image_filename); + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + if spawn_process { + // NB: This forks and returns immediately. + //let _child_pid = command_builder.spawn()?; + + let stdout_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/ffmpeg_stdout.txt")?; + + let stderr_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/ffmpeg_stderr.txt")?; + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + stdout: Redirection::File(stdout_file), + stderr: Redirection::File(stderr_file), + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + p.detach(); + + } else { + // NB: This is a blocking call. + /*let output = command_builder.output()?; + + info!("Output status: {}", output.status); + info!("Stdout: {:?}", String::from_utf8(output.stdout)); + error!("Stderr: {:?}", String::from_utf8(output.stderr)); + + if !output.status.success() { + bail!("Bad error code: {:?}", output.status); + }*/ + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + let exit_status = p.wait()?; + + info!("Exit status: {:?}", exit_status); + } + + Ok(()) + } +} diff --git a/src/script_execution/mod.rs b/src/script_execution/mod.rs index d433b74d78..fcd02f3909 100644 --- a/src/script_execution/mod.rs +++ b/src/script_execution/mod.rs @@ -1,5 +1,6 @@ pub mod ffmpeg_generate_preview_image_command; pub mod ffmpeg_generate_preview_video_command; pub mod google_drive_download_command; +pub mod imagemagick_generate_preview_image_command; pub mod wav2lip_inference_command; pub mod wav2lip_process_upload_command; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 7752a4de66..804d714c9d 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -41,6 +41,7 @@ use tempdir::TempDir; use crate::script_execution::wav2lip_process_upload_command::Wav2LipPreprocessClient; use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; +use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; // Buckets const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -66,6 +67,7 @@ struct Downloader { pub w2l_processor: Wav2LipPreprocessClient, pub ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand, pub ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand, + pub imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand, // Command to run pub download_script: String, // Root to store W2L templates @@ -146,6 +148,7 @@ async fn main() -> AnyhowResult<()> { google_drive_downloader, ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, + imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, w2l_processor: w2l_preprecess_command, bucket_root_w2l_template_uploads: bucket_root.to_string(), }; @@ -337,7 +340,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) } else { let preview_filename = format!("{}_preview.webp", &download_filename); - downloader.ffmpeg_image_preview_generator.execute( + downloader.imagemagick_image_preview_generator.execute( &download_filename, &preview_filename, 500, From 6b73255eae1e46629686efe892a170755a48dc64 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 02:18:11 -0400 Subject: [PATCH 0135/1042] install imagemagick --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index c9b5eaaf50..c183320932 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ RUN apt-get update \ gcc-7 \ git \ htop \ + imagemagick \ libc++-7-dev \ libffi-dev \ libgcc-7-dev \ From 43736d01e284739be4cd93960482f568a9d18565 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 02:40:27 -0400 Subject: [PATCH 0136/1042] get original file mimetype --- Dockerfile | 1 + migrations/2021-05-15-071601_assets/up.sql | 1 + models/Wav2Lip/requirements.txt | 3 ++- models/Wav2Lip/vocodes_process_upload.py | 4 ++++ src/job_queries/w2l_download_job_queries.rs | 3 +++ src/w2l_download_job.rs | 14 +++++++++++--- 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c183320932..3be99a7629 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,7 @@ RUN apt-get update \ libc++-7-dev \ libffi-dev \ libgcc-7-dev \ + libmagic1 \ libsm6 \ libsndfile1 \ libssl-dev \ diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 1c797e8d60..b454b509bc 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -186,6 +186,7 @@ CREATE TABLE w2l_templates ( original_filename CHAR(255) DEFAULT NULL, -- Dimensions and other characteristics + mime_type VARCHAR(32) NOT NULL DEFAULT '', frame_width INT(10) NOT NULL DEFAULT 0, frame_height INT(10) NOT NULL DEFAULT 0, frame_count INT(10) NOT NULL DEFAULT 0, diff --git a/models/Wav2Lip/requirements.txt b/models/Wav2Lip/requirements.txt index c9cf4ce923..de89d8f04b 100644 --- a/models/Wav2Lip/requirements.txt +++ b/models/Wav2Lip/requirements.txt @@ -1,9 +1,10 @@ librosa==0.7.0 +numba==0.48 numpy==1.17.1 opencv-contrib-python==4.2.0.34 opencv-python==4.1.0.25 +python-magic==0.4.22 tensorflow==1.12.0 torch==1.1.0 torchvision==0.3.0 tqdm==4.45.0 -numba==0.48 diff --git a/models/Wav2Lip/vocodes_process_upload.py b/models/Wav2Lip/vocodes_process_upload.py index d766af1ec1..4714fa2154 100644 --- a/models/Wav2Lip/vocodes_process_upload.py +++ b/models/Wav2Lip/vocodes_process_upload.py @@ -10,6 +10,7 @@ import tempfile import shutil import datetime import pickle +import magic from PIL import Image """ @@ -337,11 +338,14 @@ def main(tempdir): print('Done detecting faces!', flush=True) + mime_type = magic.from_file(args.image_or_video_filename, mime=True) + metadata = { 'is_video': is_video, 'width': frame_w, 'height': frame_h, 'num_frames': len(full_frames), + 'mimetype': mime_type, } if is_video: diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 01de3299de..194cc73294 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -117,6 +117,7 @@ pub async fn insert_w2l_template(pool: &MySqlPool, private_bucket_cached_faces_object_name: &str, maybe_image_preview_object_name: Option<&str>, maybe_video_preview_object_name: Option<&str>, + maybe_mime_type: Option<&str>, frame_width: u32, frame_height: u32, frame_count: u64, @@ -147,6 +148,7 @@ SET public_bucket_object_name = 'TODO', maybe_private_bucket_preview_image_object_name = ?, maybe_private_bucket_preview_video_object_name = ?, + mime_type = ?, frame_width = ?, frame_height = ?, frame_count = ?, @@ -165,6 +167,7 @@ SET private_bucket_cached_faces_object_name.to_string(), maybe_image_preview_object_name, maybe_video_preview_object_name, + maybe_mime_type.unwrap_or(""), frame_width, frame_height, frame_count, diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 804d714c9d..c833d8c59c 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -32,7 +32,7 @@ use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; -use std::fs::File; +use std::fs::{File, metadata}; use std::io::{BufReader, Read}; use std::path::{PathBuf, Path}; use std::process::Command; @@ -234,6 +234,7 @@ struct FileMetadata { pub num_frames: u64, pub fps: Option, pub duration_millis: Option, + pub mimetype: Option, } fn read_metadata_file(filename: &str) -> AnyhowResult { @@ -366,9 +367,15 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) info!("Cached faces destination bucket path: {}", full_object_path_cached_faces); info!("Uploading image/video..."); - downloader.bucket_client.upload_filename( + + let original_mime_type = file_metadata.mimetype + .as_deref() + .unwrap_or("application/octet-stream"); + + downloader.bucket_client.upload_filename_with_content_type( &full_object_path, - &video_or_image_path).await?; + &video_or_image_path, + original_mime_type).await?; info!("Uploading cached faces..."); downloader.bucket_client.upload_filename( @@ -411,6 +418,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) &full_object_path_cached_faces, maybe_image_preview_object_name.as_deref(), maybe_video_preview_object_name.as_deref(), + file_metadata.mimetype.as_deref(), file_metadata.width, file_metadata.height, file_metadata.num_frames, From 1da4ab55880634190016b18a6766e459ae54d8e4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 02:46:14 -0400 Subject: [PATCH 0137/1042] fix imagemagick resize --- debug/curl/upload_w2l_image_gdrive.py | 2 ++ .../imagemagick_generate_preview_image_command.rs | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/debug/curl/upload_w2l_image_gdrive.py b/debug/curl/upload_w2l_image_gdrive.py index aadb14da89..250afcd4b1 100755 --- a/debug/curl/upload_w2l_image_gdrive.py +++ b/debug/curl/upload_w2l_image_gdrive.py @@ -37,6 +37,8 @@ cookies = { 'session': session_cookie } # elon_integration_test.jpg download_url = 'https://drive.google.com/file/d/1yoBSb6nwFUZVI4CD-nM2BSofGdBjC8Nn/view?usp=sharing' +#download_url = 'https://i.imgur.com/...' + payload = { 'idempotency_token': str(uuid.uuid4()), 'title': 'this is a new image template', diff --git a/src/script_execution/imagemagick_generate_preview_image_command.rs b/src/script_execution/imagemagick_generate_preview_image_command.rs index 878bf75d80..cb6a4b7eb8 100644 --- a/src/script_execution/imagemagick_generate_preview_image_command.rs +++ b/src/script_execution/imagemagick_generate_preview_image_command.rs @@ -24,7 +24,8 @@ impl ImagemagickGeneratePreviewImageCommand { { let mut command = String::new(); - let dimensions = format!("{}x{}", output_width, output_height); + // NB: `\!` ignores aspect ratio preservation + let dimensions = format!("{}x{}\\!", output_width, output_height); // magick wizard.png -quality 50 -define webp:lossless=true wizard.webp command.push_str("echo 'test'"); @@ -32,7 +33,7 @@ impl ImagemagickGeneratePreviewImageCommand { //command.push_str("magick"); command.push_str("convert "); command.push_str(&input_image_filename); - command.push_str(" -size "); + command.push_str(" -resize "); command.push_str(&dimensions); command.push_str(" -quality 50 "); command.push_str(" -define webp:loseless=false "); From 40f403be69a082a574d2aee286d10f07d0031dbf Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 03:03:52 -0400 Subject: [PATCH 0138/1042] messy, but upload to PUBLIC + PRIVATE buckets --- .env | 3 +- src/w2l_download_job.rs | 66 ++++++++++++++++++++++++++++++++--------- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/.env b/.env index 4f9feebeb5..91b73890b7 100644 --- a/.env +++ b/.env @@ -15,7 +15,8 @@ REGION_NAME=us-east1 TTS_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models -W2L_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads +W2L_PRIVATE_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads +W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index c833d8c59c..683a606ca9 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -43,13 +43,17 @@ use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGenera use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; -// Buckets +// Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; const ENV_REGION_NAME : &'static str = "REGION_NAME"; -const ENV_BUCKET_NAME : &'static str = "W2L_DOWNLOAD_BUCKET_NAME"; const ENV_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT"; +// Buckets (private data) +const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; +// Buckets (public data) +const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; + // Python code const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; @@ -62,15 +66,20 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Downloader { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, - pub bucket_client: BucketClient, + + pub private_bucket_client: BucketClient, + pub public_bucket_client: BucketClient, + pub google_drive_downloader: GoogleDriveDownloadCommand, pub w2l_processor: Wav2LipPreprocessClient, pub ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand, pub ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand, pub imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand, + // Command to run pub download_script: String, - // Root to store W2L templates + + // Root to store W2L templates (public and private) pub bucket_root_w2l_template_uploads: String, } @@ -89,18 +98,29 @@ async fn main() -> AnyhowResult<()> { info!("Hostname: {}", &server_hostname); - // Bucket stuff + // Bucket stuff (shared) let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; - let bucket_name = easyenv::get_env_string_required(ENV_BUCKET_NAME)?; let bucket_root = easyenv::get_env_string_required(ENV_BUCKET_ROOT)?; - let bucket_client = BucketClient::create( + // Private and Public Buckets + let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?; + let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?; + + let private_bucket_client = BucketClient::create( &access_key, &secret_key, ®ion_name, - &bucket_name, + &private_bucket_name, + None, + )?; + + let public_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &public_bucket_name, None, )?; @@ -143,7 +163,8 @@ async fn main() -> AnyhowResult<()> { let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, - bucket_client, + public_bucket_client, + private_bucket_client, download_script, google_drive_downloader, ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, @@ -357,7 +378,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) maybe_image_preview_filename = Some(image_preview_path); } - // ==================== UPLOAD TO BUCKET ==================== // + // ==================== UPLOAD TO BUCKETS ==================== // info!("Image/video destination bucket path: {}", full_object_path); @@ -372,13 +393,18 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) .as_deref() .unwrap_or("application/octet-stream"); - downloader.bucket_client.upload_filename_with_content_type( + downloader.private_bucket_client.upload_filename_with_content_type( + &full_object_path, + &video_or_image_path, + original_mime_type).await?; + + downloader.public_bucket_client.upload_filename_with_content_type( &full_object_path, &video_or_image_path, original_mime_type).await?; info!("Uploading cached faces..."); - downloader.bucket_client.upload_filename( + downloader.private_bucket_client.upload_filename( &full_object_path_cached_faces, &cached_faces_path).await?; @@ -386,7 +412,13 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) if let Some(image_preview_filename) = maybe_image_preview_filename.as_deref() { if let Some(image_preview_object_name) = maybe_image_preview_object_name.as_deref() { info!("Uploading image preview..."); - downloader.bucket_client.upload_filename_with_content_type( + downloader.private_bucket_client.upload_filename_with_content_type( + &image_preview_object_name, + image_preview_filename, + "image/webp").await?; + + info!("Uploading image preview..."); + downloader.public_bucket_client.upload_filename_with_content_type( &image_preview_object_name, image_preview_filename, "image/webp").await?; @@ -397,7 +429,13 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) if let Some(video_preview_filename) = maybe_video_preview_filename.as_deref() { if let Some(video_preview_object_name) = maybe_video_preview_object_name.as_deref() { info!("Uploading video preview..."); - downloader.bucket_client.upload_filename_with_content_type( + downloader.private_bucket_client.upload_filename_with_content_type( + &video_preview_object_name, + video_preview_filename, + "image/webp").await?; + + info!("Uploading video preview..."); + downloader.public_bucket_client.upload_filename_with_content_type( &video_preview_object_name, video_preview_filename, "image/webp").await?; From f536ce02cf3c365c67a95676e4b07a8ee4beeba0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 03:07:22 -0400 Subject: [PATCH 0139/1042] ignore sqlx logging --- src/main.rs | 4 +++- src/tts_download_job.rs | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index fca1ed55a2..6acc5b7061 100644 --- a/src/main.rs +++ b/src/main.rs @@ -42,7 +42,9 @@ use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; -const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; + +// NB: sqlx::query is spammy and logs all queries as "info"-level +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; pub type AnyhowResult = anyhow::Result; diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index d0b8b9efb3..17b62a7b3b 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -43,7 +43,8 @@ const ENV_REGION_NAME : &'static str = "REGION_NAME"; const ENV_BUCKET_NAME : &'static str = "TTS_DOWNLOAD_BUCKET_NAME"; const ENV_BUCKET_ROOT : &'static str = "TTS_DOWNLOAD_BUCKET_ROOT"; -const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; +// NB: sqlx::query is spammy and logs all queries as "info"-level +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Downloader { From 3ce65d8c25c1db562867b59ad0bc7c2d098bcf04 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 May 2021 03:19:20 -0400 Subject: [PATCH 0140/1042] improve alpha mode enrollment --- src/http_server/endpoints/misc/enable_alpha.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/http_server/endpoints/misc/enable_alpha.rs b/src/http_server/endpoints/misc/enable_alpha.rs index c062dbde56..c2bbe54e5c 100644 --- a/src/http_server/endpoints/misc/enable_alpha.rs +++ b/src/http_server/endpoints/misc/enable_alpha.rs @@ -9,6 +9,11 @@ use std::sync::Arc; const ALPHA_COOKIE_NAME : &'static str = "enable-alpha"; const CONTENT_TYPE : &'static str = "text/html; charset=utf-8"; +const ENABLE_LINK : &'static str = "enable"; +const DISABLE_LINK : &'static str = "disable"; +const STATUS_LINK : &'static str = "status"; + + #[derive(Deserialize)] struct QueryFields { enable: Option, @@ -34,20 +39,22 @@ pub async fn enable_alpha(http_request: HttpRequest, HttpResponse::build(StatusCode::OK) .content_type(CONTENT_TYPE) - .body(format!("

use ?enable={{true, false}}; current `{}` cookie state = {}

", - ALPHA_COOKIE_NAME, cookie_exists)) + .body(format!("

Alpha mode.

Current `{}` cookie state = {}

{} | {} | {}", + ALPHA_COOKIE_NAME, cookie_exists, ENABLE_LINK, DISABLE_LINK, STATUS_LINK)) } Some(true) => { HttpResponse::build(StatusCode::OK) .content_type(CONTENT_TYPE) .cookie(cookie) - .body(format!("

?enable=true; setting `{}` cookie

", ALPHA_COOKIE_NAME)) + .body(format!("

Alpha mode

setting `{}` cookie

{} | {} | {}", + ALPHA_COOKIE_NAME, ENABLE_LINK, DISABLE_LINK, STATUS_LINK)) } Some(false) => { HttpResponse::build(StatusCode::OK) .content_type(CONTENT_TYPE) .del_cookie(&cookie) - .body(format!("

?enable=false; unsetting `{}` mode cookie

", ALPHA_COOKIE_NAME)) + .body(format!("

Alpha mode

unsetting `{}` cookie

{} | {} | {}", + ALPHA_COOKIE_NAME, ENABLE_LINK, DISABLE_LINK, STATUS_LINK)) } } } From 0d634896a1058c7c7a6c60025b731b113d60fa70 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 26 May 2021 02:40:28 -0400 Subject: [PATCH 0141/1042] profile endpoint --- .../endpoints/users/get_profile.rs | 165 ++++++++++++++++++ src/http_server/endpoints/users/mod.rs | 1 + src/main.rs | 6 + 3 files changed, 172 insertions(+) create mode 100644 src/http_server/endpoints/users/get_profile.rs diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs new file mode 100644 index 0000000000..dcde459261 --- /dev/null +++ b/src/http_server/endpoints/users/get_profile.rs @@ -0,0 +1,165 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use anyhow::anyhow; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_token::random_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use chrono::{DateTime, Utc}; + +#[derive(Serialize)] +pub struct UserProfileRecord { + pub user_token: String, + pub username: String, + pub display_name: String, + pub profile_markdown: String, + pub profile_rendered_html: String, + pub user_role_slug: String, + pub banned: i8, + pub dark_mode: String, + pub avatar_public_bucket_hash: Option, + pub disable_gravatar: i8, + pub hide_results_preference: i8, + pub discord_username: Option, + pub twitch_username: Option, + pub twitter_username: Option, + pub created_at: DateTime, +} + +#[derive(Serialize)] +pub struct ProfileSuccessResponse { + pub success: bool, + pub user: Option, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum ProfileError { + ServerError, + NotFound, +} + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct PathInfo { + username: String, +} + +impl ResponseError for ProfileError { + fn status_code(&self) -> StatusCode { + match *self { + ProfileError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + ProfileError::NotFound => StatusCode::NOT_FOUND, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ProfileError::ServerError => "server error".to_string(), + ProfileError::NotFound => "not found error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn get_profile_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_profile_record = sqlx::query_as!( + UserProfileRecord, + r#" +SELECT + users.token as user_token, + username, + display_name, + profile_markdown, + profile_rendered_html, + user_role_slug, + banned, + dark_mode, + avatar_public_bucket_hash, + disable_gravatar, + hide_results_preference, + discord_username, + twitch_username, + twitter_username, + created_at +FROM users +WHERE users.username = ? +AND users.deleted_at IS NULL + "#, + &path.username, + ) + .fetch_one(&server_state.mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let profile_record = match maybe_profile_record { + Ok(profile_record) => profile_record, + Err(err) => { + match err { + RowNotFound => { + warn!("Invalid user"); + return Err(ProfileError::NotFound); + }, + _ => { + warn!("User profile query error: {:?}", err); + return Err(ProfileError::ServerError); + } + } + } + }; + + let response = ProfileSuccessResponse { + success: true, + user: Some(profile_record), + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ProfileError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/users/mod.rs b/src/http_server/endpoints/users/mod.rs index 09256cb3e6..b2811538a7 100644 --- a/src/http_server/endpoints/users/mod.rs +++ b/src/http_server/endpoints/users/mod.rs @@ -1,4 +1,5 @@ pub mod create_account; +pub mod get_profile; pub mod login; pub mod logout; pub mod session_info; diff --git a/src/main.rs b/src/main.rs index 6acc5b7061..efd07a8244 100644 --- a/src/main.rs +++ b/src/main.rs @@ -40,6 +40,7 @@ use log::{info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; +use crate::http_server::endpoints::users::get_profile::get_profile_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -203,6 +204,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service( + web::resource("/profile/{username}") + .route(web::get().to(get_profile_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(get_root_index) .service(enable_alpha) .default_service( web::route().to(default_route_404)) From 204e1ecd8805847602291c6467696e3fc5d89eae Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 26 May 2021 10:54:28 -0400 Subject: [PATCH 0142/1042] gravatars --- .../endpoints/users/get_profile.rs | 63 ++++++++++++++++++- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index dcde459261..a88a766fdf 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -26,11 +26,14 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use chrono::{DateTime, Utc}; +use md5::{Md5, Digest}; #[derive(Serialize)] pub struct UserProfileRecord { pub user_token: String, pub username: String, + /// DO NOT LEAK THE EMAIL! + pub email_address: String, pub display_name: String, pub profile_markdown: String, pub profile_rendered_html: String, @@ -46,10 +49,37 @@ pub struct UserProfileRecord { pub created_at: DateTime, } +/// This changes the record: +/// - removes email address +/// - adds gravatar_hash +/// - changes banned to bool +/// - changes hide_results_preference to bool +/// - changes disable_gravatar to bool +#[derive(Serialize)] +pub struct UserProfileRecordForResponse { + pub user_token: String, + pub username: String, + pub display_name: String, + /// Gravatar hash was added + pub gravatar_hash: String, + pub profile_markdown: String, + pub profile_rendered_html: String, + pub user_role_slug: String, + pub banned: bool, + pub dark_mode: String, + pub avatar_public_bucket_hash: Option, + pub disable_gravatar: bool, + pub hide_results_preference: bool, + pub discord_username: Option, + pub twitch_username: Option, + pub twitter_username: Option, + pub created_at: DateTime, +} + #[derive(Serialize)] pub struct ProfileSuccessResponse { pub success: bool, - pub user: Option, + pub user: Option, } #[derive(Serialize)] @@ -113,6 +143,7 @@ pub async fn get_profile_handler( SELECT users.token as user_token, username, + email_address, display_name, profile_markdown, profile_rendered_html, @@ -135,7 +166,7 @@ AND users.deleted_at IS NULL .fetch_one(&server_state.mysql_pool) .await; // TODO: This will return error if it doesn't exist - let profile_record = match maybe_profile_record { + let profile_record : UserProfileRecord = match maybe_profile_record { Ok(profile_record) => profile_record, Err(err) => { match err { @@ -151,9 +182,35 @@ AND users.deleted_at IS NULL } }; + let email = profile_record.email_address.trim().to_lowercase(); + + let mut hasher = Md5::new(); + hasher.update(email); + let hash = hasher.finalize(); + let gravatar_hash = format!("{:x}", hash); + + let profile_for_response = UserProfileRecordForResponse { + user_token: profile_record.user_token.clone(), + username: profile_record.username.clone(), + display_name: profile_record.display_name.clone(), + gravatar_hash: gravatar_hash, + profile_markdown: profile_record.profile_markdown.clone(), + profile_rendered_html: profile_record.profile_rendered_html.clone(), + user_role_slug: profile_record.user_role_slug.clone(), + banned: if profile_record.banned == 0 { false } else { true }, + dark_mode: profile_record.dark_mode.clone(), + avatar_public_bucket_hash: profile_record.avatar_public_bucket_hash.clone(), + disable_gravatar: if profile_record.disable_gravatar == 0 { false } else { true }, + hide_results_preference: if profile_record.hide_results_preference == 0 { false } else { true }, + discord_username: None, + twitch_username: None, + twitter_username: None, + created_at: profile_record.created_at.clone(), + }; + let response = ProfileSuccessResponse { success: true, - user: Some(profile_record), + user: Some(profile_for_response), }; let body = serde_json::to_string(&response) From be87c4b2812fdff89538f283fa8956239ff92998 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 26 May 2021 11:02:49 -0400 Subject: [PATCH 0143/1042] add gravatar to db --- migrations/2021-05-15-071534_users/up.sql | 3 +++ .../endpoints/users/create_account.rs | 13 +++++++++---- .../endpoints/users/get_profile.rs | 19 ++++--------------- src/util/email_to_gravatar.rs | 12 ++++++++++++ src/util/mod.rs | 3 ++- 5 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 src/util/email_to_gravatar.rs diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 4bfd3d80be..4da43640be 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -15,6 +15,9 @@ CREATE TABLE users ( email_address VARCHAR(255) NOT NULL, email_confirmed BOOLEAN NOT NULL DEFAULT false, + -- Gravatar image hashes are precomputed. + email_gravatar_hash CHAR(32) NOT NULL, + -- The profile of the user in markdown (user editable). profile_markdown TEXT NOT NULL, diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index 05eaf5a622..c9e303a132 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -19,6 +19,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::util::email_to_gravatar::email_to_gravatar; const NEW_USER_ROLE: &'static str = "new-user"; @@ -110,10 +111,12 @@ pub async fn create_account_handler( } }; - let username = request.username.to_lowercase(); - let display_name = request.username.to_string(); + let username = request.username.trim().to_lowercase(); + let display_name = request.username.trim().to_string(); - let email_address = request.email_address.to_lowercase(); + let email_address = request.email_address.trim().to_lowercase(); + + let email_gravatar_hash = email_to_gravatar(&email_address); let profile_markdown = ""; let profile_rendered_html = ""; @@ -127,6 +130,7 @@ INSERT INTO users ( username, display_name, email_address, + email_gravatar_hash, profile_markdown, profile_rendered_html, user_role_slug, @@ -135,12 +139,13 @@ INSERT INTO users ( ip_address_last_login, ip_address_last_update ) -VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) +VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) "#, user_token.to_string(), username, display_name, email_address, + email_gravatar_hash, profile_markdown, profile_rendered_html, NEW_USER_ROLE, diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index a88a766fdf..780a7d1c59 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -32,8 +32,7 @@ use md5::{Md5, Digest}; pub struct UserProfileRecord { pub user_token: String, pub username: String, - /// DO NOT LEAK THE EMAIL! - pub email_address: String, + pub email_gravatar_hash: String, pub display_name: String, pub profile_markdown: String, pub profile_rendered_html: String, @@ -50,8 +49,6 @@ pub struct UserProfileRecord { } /// This changes the record: -/// - removes email address -/// - adds gravatar_hash /// - changes banned to bool /// - changes hide_results_preference to bool /// - changes disable_gravatar to bool @@ -60,8 +57,7 @@ pub struct UserProfileRecordForResponse { pub user_token: String, pub username: String, pub display_name: String, - /// Gravatar hash was added - pub gravatar_hash: String, + pub email_gravatar_hash: String, pub profile_markdown: String, pub profile_rendered_html: String, pub user_role_slug: String, @@ -143,8 +139,8 @@ pub async fn get_profile_handler( SELECT users.token as user_token, username, - email_address, display_name, + email_gravatar_hash, profile_markdown, profile_rendered_html, user_role_slug, @@ -182,18 +178,11 @@ AND users.deleted_at IS NULL } }; - let email = profile_record.email_address.trim().to_lowercase(); - - let mut hasher = Md5::new(); - hasher.update(email); - let hash = hasher.finalize(); - let gravatar_hash = format!("{:x}", hash); - let profile_for_response = UserProfileRecordForResponse { user_token: profile_record.user_token.clone(), username: profile_record.username.clone(), display_name: profile_record.display_name.clone(), - gravatar_hash: gravatar_hash, + email_gravatar_hash: profile_record.email_gravatar_hash.clone(), profile_markdown: profile_record.profile_markdown.clone(), profile_rendered_html: profile_record.profile_rendered_html.clone(), user_role_slug: profile_record.user_role_slug.clone(), diff --git a/src/util/email_to_gravatar.rs b/src/util/email_to_gravatar.rs new file mode 100644 index 0000000000..26f4f0fea8 --- /dev/null +++ b/src/util/email_to_gravatar.rs @@ -0,0 +1,12 @@ +use md5::{Md5, Digest}; + +pub fn email_to_gravatar(email_address: &str) -> String { + let email = email_address.trim().to_lowercase(); + + let mut hasher = Md5::new(); + hasher.update(email); + let hash = hasher.finalize(); + let gravatar_hash = format!("{:x}", hash); + + gravatar_hash +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 5bdef0899a..99cdea2fa8 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,3 +1,4 @@ pub mod anyhow_result; pub mod filesystem; -pub mod random_token; \ No newline at end of file +pub mod random_token; +pub mod email_to_gravatar; \ No newline at end of file From c0f74bad3d482e4bbaea39729f2c733dc15403d9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 26 May 2021 11:11:38 -0400 Subject: [PATCH 0144/1042] return gravatar with session --- src/http_server/endpoints/users/session_info.rs | 4 +++- src/http_server/web_utils/session_checker.rs | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/http_server/endpoints/users/session_info.rs b/src/http_server/endpoints/users/session_info.rs index 22078b4b9d..fc8dce545d 100644 --- a/src/http_server/endpoints/users/session_info.rs +++ b/src/http_server/endpoints/users/session_info.rs @@ -29,6 +29,7 @@ pub struct UserInfo { pub user_token: String, pub username: String, pub display_name: String, + pub email_gravatar_hash: String, } #[derive(Serialize)] @@ -100,7 +101,8 @@ pub async fn session_info_handler( user_info = Some(UserInfo { user_token: session_data.user_token.clone(), username: session_data.username.to_string(), - display_name: session_data.display_name.to_string() + display_name: session_data.display_name.to_string(), + email_gravatar_hash: session_data.email_gravatar_hash.to_string(), }); } } diff --git a/src/http_server/web_utils/session_checker.rs b/src/http_server/web_utils/session_checker.rs index 68a491b0aa..39fbd671a1 100644 --- a/src/http_server/web_utils/session_checker.rs +++ b/src/http_server/web_utils/session_checker.rs @@ -22,6 +22,7 @@ pub struct SessionUserRecord { pub display_name: String, pub email_address: String, pub email_confirmed: i8, + pub email_gravatar_hash: String, pub profile_markdown: String, pub profile_rendered_html: String, pub user_role_slug: String, @@ -105,6 +106,7 @@ SELECT display_name, email_address, email_confirmed, + email_gravatar_hash, profile_markdown, profile_rendered_html, user_role_slug, From 594eb0f1264f1998ab3b3a1bd6606ec6e05b6bc0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 26 May 2021 22:50:47 -0400 Subject: [PATCH 0145/1042] add more fields to the database --- migrations/2021-05-15-071534_users/up.sql | 28 +- sqlx-data.json | 807 +++++++++++++----- .../endpoints/users/get_profile.rs | 30 +- src/http_server/web_utils/session_checker.rs | 18 +- 4 files changed, 626 insertions(+), 257 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 4da43640be..ecbcaa6e81 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -50,14 +50,6 @@ CREATE TABLE users ( cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0, cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0, - -- Settings - -- DO NOT REORDER. - dark_mode ENUM( - 'light-mode', - 'dark-mode', - 'use-clock' - ) NOT NULL DEFAULT 'light-mode', - -- An uploaded avatar. Public hash in our bucket. avatar_public_bucket_hash CHAR(32) DEFAULT NULL, @@ -69,11 +61,31 @@ CREATE TABLE users ( -- If the URLs are shared, they'll be visible. hide_results_preference BOOLEAN NOT NULL DEFAULT false, + -- Auto play preferences + auto_play_audio_preference BOOLEAN NOT NULL DEFAULT false, + auto_play_video_preference BOOLEAN NOT NULL DEFAULT false, + + -- Favorite TTS voice to use by default + maybe_preferred_tts_model_token VARCHAR(32) DEFAULT NULL, + + -- Favorite W2L model to use by default + maybe_preferred_w2l_template_token VARCHAR(32) DEFAULT NULL, + + -- Settings + -- DO NOT REORDER. + dark_mode_preference ENUM( + 'light-mode', + 'dark-mode', + 'use-clock' + ) NOT NULL DEFAULT 'light-mode', + -- Social media usernames -- These are not confirmed. We'll need to build an OAuth system to handle that. discord_username VARCHAR(36) DEFAULT NULL, twitter_username VARCHAR(36) DEFAULT NULL, twitch_username VARCHAR(36) DEFAULT NULL, + patreon_username VARCHAR(36) DEFAULT NULL, + github_username VARCHAR(36) DEFAULT NULL, -- Incremented with every update. version INT NOT NULL DEFAULT 0, diff --git a/sqlx-data.json b/sqlx-data.json index df9df95c4e..c7fc87babc 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -20,16 +20,6 @@ "nullable": [] } }, - "26fbb143f69b900a722ce49e14942c6eb14b53dad85aa551f13e742dd6a21573": { - "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 11 - }, - "nullable": [] - } - }, "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -40,6 +30,16 @@ "nullable": [] } }, + "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { + "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 12 + }, + "nullable": [] + } + }, "345cbabbe69652c89ccd384ed7db44f70c53791488cf63721f9563291b45db3e": { "query": "\nINSERT INTO user_sessions (\n token,\n user_token,\n ip_address_creation,\n expires_at\n)\nVALUES ( ?, ?, ?, NOW() + interval 1 year )\n ", "describe": { @@ -50,6 +50,16 @@ "nullable": [] } }, + "4db1b60fd29489e897d958db8eb5818ba80cf198277b7e8e8095f0d34b93fc76": { + "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n public_bucket_hash = 'TODO',\n public_bucket_object_name = 'TODO',\n maybe_private_bucket_preview_image_object_name = ?,\n maybe_private_bucket_preview_video_object_name = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 18 + }, + "nullable": [] + } + }, "53d77ee58e9b1f7fb900914895c25bcccb9297378ff018c1e553b187d1956e92": { "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE username = ?\nLIMIT 1\n ", "describe": { @@ -451,226 +461,6 @@ ] } }, - "95a8d3bea2906e3c3607989e329fe925dea7e8307a4924d9b3d52d24193d77c4": { - "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_address,\n email_confirmed,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n banned,\n dark_mode,\n avatar_public_bucket_hash,\n disable_gravatar,\n hide_results_preference,\n discord_username,\n twitch_username,\n twitter_username\nFROM users\nLEFT OUTER JOIN user_sessions\nON users.token = user_sessions.user_token\nWHERE user_sessions.token = ?\nAND user_sessions.deleted_at IS NULL\nAND users.deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "email_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "email_confirmed", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 5, - "name": "profile_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 6, - "name": "profile_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 7, - "name": "user_role_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 8, - "name": "banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "dark_mode", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 40 - } - }, - { - "ordinal": 10, - "name": "avatar_public_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 11, - "name": "disable_gravatar", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "hide_results_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "discord_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 14, - "name": "twitch_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 15, - "name": "twitter_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - false, - false, - true, - true, - true - ] - } - }, "b7535ecf2e63b7f02b5b96f833fd8cd28285b24473df6f89795ff360c1723215": { "query": "\nSELECT *\nFROM w2l_template_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { @@ -944,14 +734,315 @@ "nullable": [] } }, - "e228a3f6766d19b50fa0e01ef2e085bdcb57813cf30e88948f0ec2592e27dbab": { - "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?\n ", + "e568e1c13676e5cd06b2136808e71d10ddd87f6ef4a39848c03531ca65833a15": { + "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_address,\n email_confirmed,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n banned,\n dark_mode_preference,\n avatar_public_bucket_hash,\n disable_gravatar,\n hide_results_preference,\n auto_play_audio_preference,\n auto_play_video_preference,\n maybe_preferred_tts_model_token,\n maybe_preferred_w2l_template_token,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username\nFROM users\nLEFT OUTER JOIN user_sessions\nON users.token = user_sessions.user_token\nWHERE user_sessions.token = ?\nAND user_sessions.deleted_at IS NULL\nAND users.deleted_at IS NULL\n ", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "email_confirmed", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 5, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "profile_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 7, + "name": "profile_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 8, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 9, + "name": "banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "dark_mode_preference", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 40 + } + }, + { + "ordinal": 11, + "name": "avatar_public_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 12, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "hide_results_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "auto_play_audio_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "auto_play_video_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "maybe_preferred_tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 17, + "name": "maybe_preferred_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 18, + "name": "discord_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 19, + "name": "twitch_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 20, + "name": "twitter_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 21, + "name": "patreon_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 22, + "name": "github_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + } + ], "parameters": { - "Right": 9 + "Right": 1 }, - "nullable": [] + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + false, + true, + true, + true, + true, + true, + true, + true + ] } }, "e73192a3e763a287d5737821443c5a81649f17e39978497b99c52e1e896a75f6": { @@ -991,5 +1082,251 @@ false ] } + }, + "e76b9212a263b286e5a012bcf725c33580f9c4a8da800f0da124a1ceff3086b0": { + "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n banned,\n avatar_public_bucket_hash,\n disable_gravatar,\n maybe_preferred_tts_model_token,\n maybe_preferred_w2l_template_token,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n created_at\nFROM users\nWHERE users.username = ?\nAND users.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "profile_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "profile_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 6, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 7, + "name": "banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "avatar_public_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "maybe_preferred_tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "maybe_preferred_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 12, + "name": "discord_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 13, + "name": "twitch_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 14, + "name": "twitter_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 15, + "name": "patreon_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 16, + "name": "github_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 17, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + false + ] + } } } \ No newline at end of file diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index 780a7d1c59..b6bd97ece0 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -38,13 +38,15 @@ pub struct UserProfileRecord { pub profile_rendered_html: String, pub user_role_slug: String, pub banned: i8, - pub dark_mode: String, pub avatar_public_bucket_hash: Option, pub disable_gravatar: i8, - pub hide_results_preference: i8, + pub maybe_preferred_tts_model_token: Option, + pub maybe_preferred_w2l_template_token: Option, pub discord_username: Option, pub twitch_username: Option, pub twitter_username: Option, + pub patreon_username: Option, + pub github_username: Option, pub created_at: DateTime, } @@ -62,13 +64,15 @@ pub struct UserProfileRecordForResponse { pub profile_rendered_html: String, pub user_role_slug: String, pub banned: bool, - pub dark_mode: String, pub avatar_public_bucket_hash: Option, pub disable_gravatar: bool, - pub hide_results_preference: bool, + pub maybe_preferred_tts_model_token: Option, + pub maybe_preferred_w2l_template_token: Option, pub discord_username: Option, pub twitch_username: Option, pub twitter_username: Option, + pub patreon_username: Option, + pub github_username: Option, pub created_at: DateTime, } @@ -145,13 +149,15 @@ SELECT profile_rendered_html, user_role_slug, banned, - dark_mode, avatar_public_bucket_hash, disable_gravatar, - hide_results_preference, + maybe_preferred_tts_model_token, + maybe_preferred_w2l_template_token, discord_username, twitch_username, twitter_username, + patreon_username, + github_username, created_at FROM users WHERE users.username = ? @@ -187,13 +193,15 @@ AND users.deleted_at IS NULL profile_rendered_html: profile_record.profile_rendered_html.clone(), user_role_slug: profile_record.user_role_slug.clone(), banned: if profile_record.banned == 0 { false } else { true }, - dark_mode: profile_record.dark_mode.clone(), avatar_public_bucket_hash: profile_record.avatar_public_bucket_hash.clone(), disable_gravatar: if profile_record.disable_gravatar == 0 { false } else { true }, - hide_results_preference: if profile_record.hide_results_preference == 0 { false } else { true }, - discord_username: None, - twitch_username: None, - twitter_username: None, + maybe_preferred_tts_model_token: profile_record.maybe_preferred_tts_model_token.clone(), + maybe_preferred_w2l_template_token: profile_record.maybe_preferred_w2l_template_token.clone(), + discord_username: profile_record.discord_username.clone(), + twitch_username: profile_record.twitch_username.clone(), + twitter_username: profile_record.twitter_username.clone(), + patreon_username: profile_record.patreon_username.clone(), + github_username: profile_record.github_username.clone(), created_at: profile_record.created_at.clone(), }; diff --git a/src/http_server/web_utils/session_checker.rs b/src/http_server/web_utils/session_checker.rs index 39fbd671a1..21d966400f 100644 --- a/src/http_server/web_utils/session_checker.rs +++ b/src/http_server/web_utils/session_checker.rs @@ -27,13 +27,19 @@ pub struct SessionUserRecord { pub profile_rendered_html: String, pub user_role_slug: String, pub banned: i8, - pub dark_mode: String, pub avatar_public_bucket_hash: Option, pub disable_gravatar: i8, pub hide_results_preference: i8, + pub auto_play_audio_preference: i8, + pub auto_play_video_preference: i8, + pub dark_mode_preference: String, + pub maybe_preferred_tts_model_token: Option, + pub maybe_preferred_w2l_template_token: Option, pub discord_username: Option, pub twitch_username: Option, pub twitter_username: Option, + pub patreon_username: Option, + pub github_username: Option, } impl SessionChecker { @@ -111,13 +117,19 @@ SELECT profile_rendered_html, user_role_slug, banned, - dark_mode, + dark_mode_preference, avatar_public_bucket_hash, disable_gravatar, hide_results_preference, + auto_play_audio_preference, + auto_play_video_preference, + maybe_preferred_tts_model_token, + maybe_preferred_w2l_template_token, discord_username, twitch_username, - twitter_username + twitter_username, + patreon_username, + github_username FROM users LEFT OUTER JOIN user_sessions ON users.token = user_sessions.user_token From 765d4b597d43da4434d091f16b306d12ea935059 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 00:56:52 -0400 Subject: [PATCH 0146/1042] record w2l file size --- migrations/2021-05-15-071601_assets/up.sql | 18 +++++++++++------- models/Wav2Lip/vocodes_process_upload.py | 2 ++ sqlx-data.json | 20 ++++++++++---------- src/job_queries/w2l_download_job_queries.rs | 3 +++ src/w2l_download_job.rs | 2 ++ 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index b454b509bc..8bc40ae7e8 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -69,12 +69,15 @@ CREATE TABLE tts_models ( 'private' ) NOT NULL DEFAULT 'public', - -- The filename that was used at upload time. - original_filename VARCHAR(255) DEFAULT NULL, - -- Where the file was originally downloaded (if it was downloaded) original_download_url VARCHAR(512) DEFAULT NULL, + -- The filename that was used at upload time (if available) + original_filename VARCHAR(255) DEFAULT NULL, + + -- File characteristics + file_size_bytes INT(10) NOT NULL DEFAULT 0, + -- The pytorch model -- For now, this will be a hash of the file contents. -- NB: NOT UNIQUE! We can allow duplicate uploads. @@ -182,10 +185,14 @@ CREATE TABLE w2l_templates ( 'private' ) NOT NULL DEFAULT 'public', - -- The filename that was used at upload time. + -- Where the file was originally downloaded (if it was downloaded) + original_download_url VARCHAR(512) DEFAULT NULL, + + -- The filename that was used at upload time (if available) original_filename CHAR(255) DEFAULT NULL, -- Dimensions and other characteristics + file_size_bytes INT(10) NOT NULL DEFAULT 0, mime_type VARCHAR(32) NOT NULL DEFAULT '', frame_width INT(10) NOT NULL DEFAULT 0, frame_height INT(10) NOT NULL DEFAULT 0, @@ -193,9 +200,6 @@ CREATE TABLE w2l_templates ( fps FLOAT NOT NULL DEFAULT 0.0, duration_millis INT(1) NOT NULL DEFAULT 0, - -- Where the file was originally downloaded (if it was downloaded) - original_download_url VARCHAR(512) DEFAULT NULL, - -- The original source image/video and the "precomputed" faces private_bucket_hash CHAR(64) NOT NULL, -- The "full url" version of the path diff --git a/models/Wav2Lip/vocodes_process_upload.py b/models/Wav2Lip/vocodes_process_upload.py index 4714fa2154..dc4be013f0 100644 --- a/models/Wav2Lip/vocodes_process_upload.py +++ b/models/Wav2Lip/vocodes_process_upload.py @@ -339,6 +339,7 @@ def main(tempdir): print('Done detecting faces!', flush=True) mime_type = magic.from_file(args.image_or_video_filename, mime=True) + file_size_bytes = os.path.getsize(args.image_or_video_filename) metadata = { 'is_video': is_video, @@ -346,6 +347,7 @@ def main(tempdir): 'height': frame_h, 'num_frames': len(full_frames), 'mimetype': mime_type, + 'file_size_bytes': file_size_bytes, } if is_video: diff --git a/sqlx-data.json b/sqlx-data.json index c7fc87babc..78d034153b 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -50,16 +50,6 @@ "nullable": [] } }, - "4db1b60fd29489e897d958db8eb5818ba80cf198277b7e8e8095f0d34b93fc76": { - "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n public_bucket_hash = 'TODO',\n public_bucket_object_name = 'TODO',\n maybe_private_bucket_preview_image_object_name = ?,\n maybe_private_bucket_preview_video_object_name = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 18 - }, - "nullable": [] - } - }, "53d77ee58e9b1f7fb900914895c25bcccb9297378ff018c1e553b187d1956e92": { "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE username = ?\nLIMIT 1\n ", "describe": { @@ -461,6 +451,16 @@ ] } }, + "b0de8ae0637ed5c77d4c697aaf68e89e30a0542a619dd68dd43851c2f701b503": { + "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n public_bucket_hash = 'TODO',\n public_bucket_object_name = 'TODO',\n maybe_private_bucket_preview_image_object_name = ?,\n maybe_private_bucket_preview_video_object_name = ?,\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 19 + }, + "nullable": [] + } + }, "b7535ecf2e63b7f02b5b96f833fd8cd28285b24473df6f89795ff360c1723215": { "query": "\nSELECT *\nFROM w2l_template_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 194cc73294..080c151f62 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -117,6 +117,7 @@ pub async fn insert_w2l_template(pool: &MySqlPool, private_bucket_cached_faces_object_name: &str, maybe_image_preview_object_name: Option<&str>, maybe_video_preview_object_name: Option<&str>, + file_size_bytes: u64, maybe_mime_type: Option<&str>, frame_width: u32, frame_height: u32, @@ -148,6 +149,7 @@ SET public_bucket_object_name = 'TODO', maybe_private_bucket_preview_image_object_name = ?, maybe_private_bucket_preview_video_object_name = ?, + file_size_bytes = ?, mime_type = ?, frame_width = ?, frame_height = ?, @@ -167,6 +169,7 @@ SET private_bucket_cached_faces_object_name.to_string(), maybe_image_preview_object_name, maybe_video_preview_object_name, + file_size_bytes, maybe_mime_type.unwrap_or(""), frame_width, frame_height, diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 683a606ca9..c9b899e992 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -256,6 +256,7 @@ struct FileMetadata { pub fps: Option, pub duration_millis: Option, pub mimetype: Option, + pub file_size_bytes: u64, } fn read_metadata_file(filename: &str) -> AnyhowResult { @@ -456,6 +457,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) &full_object_path_cached_faces, maybe_image_preview_object_name.as_deref(), maybe_video_preview_object_name.as_deref(), + file_metadata.file_size_bytes, file_metadata.mimetype.as_deref(), file_metadata.width, file_metadata.height, From 395f3694b150ef0f65826d29f9ef2c642527144a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 01:12:17 -0400 Subject: [PATCH 0147/1042] simplify bucket hashes --- migrations/2021-05-15-071601_assets/up.sql | 31 +++++++++++---------- sqlx-data.json | 4 +-- src/job_queries/w2l_download_job_queries.rs | 6 ++-- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 8bc40ae7e8..e6a1e04d23 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -93,6 +93,8 @@ CREATE TABLE tts_models ( calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0, calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0, + -- ========== MODERATION DETAILS ========== + -- In this case, a moderator disables it. is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, @@ -101,6 +103,8 @@ CREATE TABLE tts_models ( -- The last moderator that made changes. maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + -- ========== TIMESTAMPS ========== + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -139,9 +143,6 @@ CREATE TABLE w2l_templates ( 'video' ) NOT NULL DEFAULT 'not-set', - -- Mods have to approve of w2l templates. - is_mod_approved BOOLEAN NOT NULL DEFAULT FALSE, - -- Optional Pointer to a newer version of the template -- If there's a newer version, we can disable this one. maybe_updated_template_token VARCHAR(32) DEFAULT NULL, @@ -200,21 +201,17 @@ CREATE TABLE w2l_templates ( fps FLOAT NOT NULL DEFAULT 0.0, duration_millis INT(1) NOT NULL DEFAULT 0, - -- The original source image/video and the "precomputed" faces + -- The hash of the original source image/video private_bucket_hash CHAR(64) NOT NULL, - -- The "full url" version of the path + + -- The "full path" version of the path. Contains the hash. private_bucket_object_name VARCHAR(255) NOT NULL, - -- The "full url" of the cached faces object + -- The "full path" of the cached faces object. Contains the hash. private_bucket_cached_faces_object_name VARCHAR(255) NOT NULL, - -- The "full url" of the preview image and/or video - maybe_private_bucket_preview_image_object_name VARCHAR(255) DEFAULT NULL, - maybe_private_bucket_preview_video_object_name VARCHAR(255) DEFAULT NULL, - - -- For the thumbnail we show. - public_bucket_hash CHAR(64) NOT NULL, - -- The "full url" version of the path - public_bucket_object_name VARCHAR(255) NOT NULL, + -- The "full url" of the preview image and/or video. Contains the hash. + maybe_public_bucket_preview_image_object_name VARCHAR(255) DEFAULT NULL, + maybe_public_bucket_preview_video_object_name VARCHAR(255) DEFAULT NULL, -- Calculated average, on a scale of 0-100 -- Null with zero ratings. @@ -226,6 +223,9 @@ CREATE TABLE w2l_templates ( -- ========== MODERATION DETAILS ========== + -- Mods have to approve of w2l templates. + is_mod_approved BOOLEAN NOT NULL DEFAULT FALSE, + -- In this case, a moderator disables it. -- This also disables it for the creator. is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, @@ -236,6 +236,8 @@ CREATE TABLE w2l_templates ( -- The last moderator that made changes. maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + -- ========== TIMESTAMPS ========== + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -256,7 +258,6 @@ CREATE TABLE w2l_templates ( KEY index_creator_ip_address (creator_ip_address), KEY index_creator_set_visibility (creator_set_visibility), KEY index_private_bucket_hash (private_bucket_hash), - KEY index_public_bucket_hash (public_bucket_hash), KEY index_is_mod_disabled (is_mod_disabled) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/sqlx-data.json b/sqlx-data.json index 78d034153b..634b86c163 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -451,8 +451,8 @@ ] } }, - "b0de8ae0637ed5c77d4c697aaf68e89e30a0542a619dd68dd43851c2f701b503": { - "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n public_bucket_hash = 'TODO',\n public_bucket_object_name = 'TODO',\n maybe_private_bucket_preview_image_object_name = ?,\n maybe_private_bucket_preview_video_object_name = ?,\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", + "ab78a81f85310750668e2c7de920c067290f2b004561ab706a5edfd893fd06fe": { + "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n maybe_public_bucket_preview_image_object_name = ?,\n maybe_public_bucket_preview_video_object_name = ?,\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", "describe": { "columns": [], "parameters": { diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 080c151f62..2a1b1e55e2 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -145,10 +145,8 @@ SET private_bucket_hash = ?, private_bucket_object_name = ?, private_bucket_cached_faces_object_name = ?, - public_bucket_hash = 'TODO', - public_bucket_object_name = 'TODO', - maybe_private_bucket_preview_image_object_name = ?, - maybe_private_bucket_preview_video_object_name = ?, + maybe_public_bucket_preview_image_object_name = ?, + maybe_public_bucket_preview_video_object_name = ?, file_size_bytes = ?, mime_type = ?, frame_width = ?, From 753e0edce69e943f24e10a5f29ecf8c914e9c9e9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 01:47:37 -0400 Subject: [PATCH 0148/1042] list w2l templates --- migrations/2021-05-15-071601_assets/up.sql | 2 +- .../endpoints/w2l/list_w2l_templates.rs | 189 ++++++++++++++++++ src/http_server/endpoints/w2l/mod.rs | 3 +- src/main.rs | 6 + 4 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 src/http_server/endpoints/w2l/list_w2l_templates.rs diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index e6a1e04d23..3f5a2db84d 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -199,7 +199,7 @@ CREATE TABLE w2l_templates ( frame_height INT(10) NOT NULL DEFAULT 0, frame_count INT(10) NOT NULL DEFAULT 0, fps FLOAT NOT NULL DEFAULT 0.0, - duration_millis INT(1) NOT NULL DEFAULT 0, + duration_millis INT(10) NOT NULL DEFAULT 0, -- The hash of the original source image/video private_bucket_hash CHAR(64) NOT NULL, diff --git a/src/http_server/endpoints/w2l/list_w2l_templates.rs b/src/http_server/endpoints/w2l/list_w2l_templates.rs new file mode 100644 index 0000000000..0b55d5865a --- /dev/null +++ b/src/http_server/endpoints/w2l/list_w2l_templates.rs @@ -0,0 +1,189 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_token::random_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use chrono::{DateTime, Utc}; + +#[derive(Serialize)] +pub struct W2lTemplateRecordForResponse { + pub template_token: String, + pub template_type: String, + pub creator_user_token: String, + pub username: String, + pub display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: u32, + pub frame_height: u32, + pub duration_millis: u32, + pub maybe_image_object_name: Option, + pub maybe_video_object_name: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct ListW2lTemplatesSuccessResponse { + pub success: bool, + pub templates: Vec, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum ListW2lTemplatesError { + ServerError, +} + +#[derive(Serialize)] +pub struct W2lTemplateRecord { + pub template_token: String, + pub template_type: String, + pub creator_user_token: String, + pub username: String, + pub display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + pub maybe_public_bucket_preview_image_object_name: Option, + pub maybe_public_bucket_preview_video_object_name: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl ResponseError for ListW2lTemplatesError { + fn status_code(&self) -> StatusCode { + match *self { + ListW2lTemplatesError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListW2lTemplatesError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn list_w2l_templates_handler( + http_request: HttpRequest, + server_state: web::Data>) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_templates = sqlx::query_as!( + W2lTemplateRecord, + r#" +SELECT + w2l.token as template_token, + w2l.template_type, + w2l.creator_user_token, + users.username, + users.display_name, + w2l.updatable_slug, + w2l.title, + w2l.frame_width, + w2l.frame_height, + w2l.duration_millis, + w2l.maybe_public_bucket_preview_image_object_name, + w2l.maybe_public_bucket_preview_video_object_name, + w2l.created_at, + w2l.updated_at +FROM w2l_templates as w2l +JOIN users +ON users.token = w2l.creator_user_token +WHERE w2l.deleted_at IS NULL + "#, + ) + .fetch_all(&server_state.mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let templates : Vec = match maybe_templates { + Ok(templates) => templates, + Err(err) => { + match err { + RowNotFound => { + return Err(ListW2lTemplatesError::ServerError); + }, + _ => { + warn!("w2l template list query error: {:?}", err); + return Err(ListW2lTemplatesError::ServerError); + } + } + } + }; + + let templates_for_response = templates.into_iter() + .map(|template| { + W2lTemplateRecordForResponse { + template_token: template.template_token.clone(), + template_type: template.template_type.clone(), + creator_user_token: template.creator_user_token.clone(), + username: template.username.clone(), + display_name: template.display_name.clone(), + updatable_slug: template.updatable_slug.clone(), + title: template.title.clone(), + frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, + frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, + duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, + maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), + maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), + created_at: template.created_at.clone(), + updated_at: template.updated_at.clone(), + } + }) + .collect::>(); + + let response = ListW2lTemplatesSuccessResponse { + success: true, + templates: templates_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListW2lTemplatesError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index d34936265e..82bb2a1d8f 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -1,2 +1,3 @@ pub mod enqueue_infer_w2l; -pub mod enqueue_upload_w2l_template; \ No newline at end of file +pub mod enqueue_upload_w2l_template; +pub mod list_w2l_templates; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index efd07a8244..9b9d8e5dc5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,6 +41,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use crate::http_server::endpoints::users::get_profile::get_profile_handler; +use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -203,6 +204,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::post().to(infer_w2l_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/list") + .route(web::get().to(list_w2l_templates_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service( web::resource("/profile/{username}") From 363d47bceffe7508ca54281294c5a8283aef5b82 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:05:55 -0400 Subject: [PATCH 0149/1042] GET template --- .../endpoints/users/get_profile.rs | 4 +- .../endpoints/w2l/get_w2l_template.rs | 195 ++++++++++++++++++ src/http_server/endpoints/w2l/mod.rs | 1 + src/main.rs | 6 + 4 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 src/http_server/endpoints/w2l/get_w2l_template.rs diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index b6bd97ece0..3d9831279f 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -96,7 +96,7 @@ pub enum ProfileError { /// For the URL PathInfo #[derive(Deserialize)] -pub struct PathInfo { +pub struct GetProfilePathInfo { username: String, } @@ -132,7 +132,7 @@ impl ResponseError for ProfileError { pub async fn get_profile_handler( http_request: HttpRequest, - path: Path, + path: Path, server_state: web::Data>) -> Result { // NB: Lookup failure is Err(RowNotFound). diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs new file mode 100644 index 0000000000..455c1670b6 --- /dev/null +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -0,0 +1,195 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_token::random_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetW2lTemplatePathInfo { + slug: String, +} + +#[derive(Serialize)] +pub struct W2lTemplateRecordForResponse { + pub template_token: String, + pub template_type: String, + pub creator_user_token: String, + pub username: String, + pub display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: u32, + pub frame_height: u32, + pub duration_millis: u32, + pub maybe_image_object_name: Option, + pub maybe_video_object_name: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct GetW2lTemplateSuccessResponse { + pub success: bool, + pub template: W2lTemplateRecordForResponse, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum GetW2lTemplateError { + ServerError, +} + +#[derive(Serialize)] +pub struct W2lTemplateRecord { + pub template_token: String, + pub template_type: String, + pub creator_user_token: String, + pub username: String, + pub display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + pub maybe_public_bucket_preview_image_object_name: Option, + pub maybe_public_bucket_preview_video_object_name: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl ResponseError for GetW2lTemplateError { + fn status_code(&self) -> StatusCode { + match *self { + GetW2lTemplateError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetW2lTemplateError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn get_w2l_template_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_templates = sqlx::query_as!( + W2lTemplateRecord, + r#" +SELECT + w2l.token as template_token, + w2l.template_type, + w2l.creator_user_token, + users.username, + users.display_name, + w2l.updatable_slug, + w2l.title, + w2l.frame_width, + w2l.frame_height, + w2l.duration_millis, + w2l.maybe_public_bucket_preview_image_object_name, + w2l.maybe_public_bucket_preview_video_object_name, + w2l.created_at, + w2l.updated_at +FROM w2l_templates as w2l +JOIN users +ON users.token = w2l.creator_user_token +WHERE w2l.updatable_slug = ? +AND w2l.deleted_at IS NULL + "#, + &path.slug + ) + .fetch_one(&server_state.mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let template : W2lTemplateRecord = match maybe_templates { + Ok(templates) => templates, + Err(err) => { + match err { + RowNotFound => { + return Err(GetW2lTemplateError::ServerError); + }, + _ => { + warn!("w2l template list query error: {:?}", err); + return Err(GetW2lTemplateError::ServerError); + } + } + } + }; + + let template_for_response = W2lTemplateRecordForResponse { + template_token: template.template_token.clone(), + template_type: template.template_type.clone(), + creator_user_token: template.creator_user_token.clone(), + username: template.username.clone(), + display_name: template.display_name.clone(), + updatable_slug: template.updatable_slug.clone(), + title: template.title.clone(), + frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, + frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, + duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, + maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), + maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), + created_at: template.created_at.clone(), + updated_at: template.updated_at.clone(), + }; + + let response = GetW2lTemplateSuccessResponse { + success: true, + template: template_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetW2lTemplateError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index 82bb2a1d8f..7e1ee59b46 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -1,3 +1,4 @@ pub mod enqueue_infer_w2l; pub mod enqueue_upload_w2l_template; +pub mod get_w2l_template; pub mod list_w2l_templates; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 9b9d8e5dc5..5c6bdeaa33 100644 --- a/src/main.rs +++ b/src/main.rs @@ -42,6 +42,7 @@ use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use crate::http_server::endpoints::users::get_profile::get_profile_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; +use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -209,6 +210,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(list_w2l_templates_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/template/{slug}") + .route(web::get().to(get_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service( web::resource("/profile/{username}") From 27adcb5d7fa028073331a036d0e72fedaa5a1353 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:06:07 -0400 Subject: [PATCH 0150/1042] populate with lots of templates from the web --- debug/curl/upload_w2l_lots_of_data.py | 61 +++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 debug/curl/upload_w2l_lots_of_data.py diff --git a/debug/curl/upload_w2l_lots_of_data.py b/debug/curl/upload_w2l_lots_of_data.py new file mode 100755 index 0000000000..e960113a74 --- /dev/null +++ b/debug/curl/upload_w2l_lots_of_data.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +import requests +import re +import uuid + +login_url = 'http://localhost:12345/login' +upload_url = 'http://localhost:12345/w2l/upload' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +# NB: May not be set due to cookie domain: +#r.cookies['session'] + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + + +def upload(url): + print("===== Upload W2L =====") + + cookies = { 'session': session_cookie } + + payload = { + 'idempotency_token': str(uuid.uuid4()), + 'title': 'this is a new image template', + 'download_url': url, + } + + r = requests.post(upload_url, cookies=cookies, json=payload) + + print('Status: {}'.format(r.status_code)) + print(r.content) + + +upload('https://i.imgur.com/W1EYiPU.jpg') + +upload('https://i.imgur.com/NneE7Eq.jpg') + +upload('https://i.imgur.com/v8k9yau.jpeg') + +upload('https://drive.google.com/file/d/1yoBSb6nwFUZVI4CD-nM2BSofGdBjC8Nn/view?usp=sharing') + +upload('https://drive.google.com/file/d/1ysBAdRjlSmfWQPFjGlMJpv_zcGmVw7RC/view?usp=sharing') + +upload('https://i.imgur.com/lKaQ4Er.jpg') + +upload('https://i.imgur.com/uoGuTJo.jpeg') + From 6262588697aa342be3a3be1259c00a6c39522dbf Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:21:27 -0400 Subject: [PATCH 0151/1042] reserved db (wip) --- db/reserved_usernames.txt | 215 ++++++++++++++++++++++++++++ db/reserved_usernames_including.txt | 5 + 2 files changed, 220 insertions(+) create mode 100644 db/reserved_usernames.txt create mode 100644 db/reserved_usernames_including.txt diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt new file mode 100644 index 0000000000..77d3bc856e --- /dev/null +++ b/db/reserved_usernames.txt @@ -0,0 +1,215 @@ +# This is a list of reserved usernames to allow for vanity URLs, etc. +# Taken from the following: +# https://gist.github.com/caseyohara/1453705/bd38325ae2b562572afaa119efa59529c057bc85 + +# Stuff we care about +storyteller +vocodes + +# Companies we'd love to have use our service +# so we'll reserve them to be safe +apple +epic +epicgames +facebook +github +google +microsoft +netflix +twitter +unreal + +# Generic reserved words +about +account +activate +add +admin +administrator +api +app +apps +archive +archives +auth +blog +cache +cancel +careers +cart +changelog +checkout +codereview +compare +config +configuration +connect +contact +create +delete +direct_messages +documentation +download +downloads +edit +email +employment +enterprise +faq +favorites +feed +feedback +feeds +fleet +fleets +follow +followers +following +friend +friends +gist +group +groups +help +home +hosting +hostmaster +idea +ideas +index +info +invitations +invite +is +it +job +jobs +json +language +languages +lists +login +logout +logs +mail +map +maps +mine +mis +news +oauth +oauth_clients +offers +openid +order +orders +organizations +plans +popular +post +postmaster +privacy +projects +put +recruitment +register +remove +replies +root +rss +sales +save +search +security +sessions +settings +shop +signup +sitemap +ssl +ssladmin +ssladministrator +sslwebmaster +status +stories +styleguide +subscribe +subscriptions +support +sysadmin +sysadministrator +terms +tour +translations +trends +unfollow +unsubscribe +update +url +user +weather +webmaster +widget +widgets +wiki +ww +www +wwww +xfn +xml +xmpp +yaml +yml + +# Top 50 languages by speaking population +chinese +mandarin +spanish +english +bengali +hindi +portuguese +russian +japanese +german +wu +javanese +korean +french +vietnamese +telugu +chinese +marathi +tamil +turkish +urdu +min-nan +jinyu +gujarati +polish +arabic +ukrainian +italian +xiang +malayalam +hakka +kannada +oriya +panjabi +sunda +panjabi +romanian +bhojpuri +azerbaijani +farsi +maithili +hausa +arabic +burmese +serbo-croatian +gan +awadhi +thai +dutch +yoruba +sindhi diff --git a/db/reserved_usernames_including.txt b/db/reserved_usernames_including.txt new file mode 100644 index 0000000000..6b42928461 --- /dev/null +++ b/db/reserved_usernames_including.txt @@ -0,0 +1,5 @@ +# This is a more restrictive list of reserved usernames. +# Any of the substrings may not be used. +# You should check string.remove('_') as well. +storyteller +vocodes From 571f5e8a29fe69fc93aa47c6a0fb56876cf7603d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:23:07 -0400 Subject: [PATCH 0152/1042] combine lists --- db/reserved_usernames.txt | 887 ++++++++++++++++++++++++++++++++------ 1 file changed, 745 insertions(+), 142 deletions(-) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index 77d3bc856e..97e1cdb05a 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -1,6 +1,7 @@ # This is a list of reserved usernames to allow for vanity URLs, etc. # Taken from the following: # https://gist.github.com/caseyohara/1453705/bd38325ae2b562572afaa119efa59529c057bc85 +# https://github.com/shouldbee/reserved-usernames # Stuff we care about storyteller @@ -19,148 +20,6 @@ netflix twitter unreal -# Generic reserved words -about -account -activate -add -admin -administrator -api -app -apps -archive -archives -auth -blog -cache -cancel -careers -cart -changelog -checkout -codereview -compare -config -configuration -connect -contact -create -delete -direct_messages -documentation -download -downloads -edit -email -employment -enterprise -faq -favorites -feed -feedback -feeds -fleet -fleets -follow -followers -following -friend -friends -gist -group -groups -help -home -hosting -hostmaster -idea -ideas -index -info -invitations -invite -is -it -job -jobs -json -language -languages -lists -login -logout -logs -mail -map -maps -mine -mis -news -oauth -oauth_clients -offers -openid -order -orders -organizations -plans -popular -post -postmaster -privacy -projects -put -recruitment -register -remove -replies -root -rss -sales -save -search -security -sessions -settings -shop -signup -sitemap -ssl -ssladmin -ssladministrator -sslwebmaster -status -stories -styleguide -subscribe -subscriptions -support -sysadmin -sysadministrator -terms -tour -translations -trends -unfollow -unsubscribe -update -url -user -weather -webmaster -widget -widgets -wiki -ww -www -wwww -xfn -xml -xmpp -yaml -yml - # Top 50 languages by speaking population chinese mandarin @@ -213,3 +72,747 @@ thai dutch yoruba sindhi + +# Generic reserved words +0 +about +about +access +account +account +accounts +activate +activate +activities +activity +ad +add +add +address +adm +admin +admin +administration +administrator +administrator +ads +adult +advertising +affiliate +affiliates +ajax +all +alpha +analysis +analytics +android +anon +anonymous +api +api +app +app +apps +apps +archive +archive +archives +archives +article +asct +asset +atom +auth +auth +authentication +avatar +backup +balancer-manager +banner +banners +beta +billing +bin +blog +blog +blogs +board +book +bookmark +bot +bots +bug +business +cache +cache +cadastro +calendar +call +campaign +cancel +cancel +captcha +career +careers +careers +cart +cart +categories +category +cgi +cgi-bin +changelog +changelog +chat +check +checking +checkout +checkout +client +cliente +clients +code +codereview +codereview +comercial +comment +comments +communities +community +company +compare +compare +compras +config +config +configuration +configuration +connect +connect +contact +contact +contact-us +contact_us +contactus +contest +contribute +corp +create +create +css +dashboard +data +db +default +delete +delete +demo +design +designer +destroy +dev +devel +developer +developers +diagram +diary +dict +dictionary +die +dir +direct_messages +direct_messages +directory +dist +doc +docs +documentation +documentation +domain +download +download +downloads +downloads +ecommerce +edit +edit +editor +edu +education +email +email +employment +employment +empty +end +enterprise +enterprise +entries +entry +error +errors +eval +event +exit +explore +facebook +faq +faq +favorite +favorites +favorites +feature +features +feed +feed +feedback +feedback +feeds +feeds +file +files +first +flash +fleet +fleet +fleets +fleets +flog +follow +follow +followers +followers +following +following +forgot +form +forum +forums +founder +free +friend +friend +friends +friends +ftp +gadget +gadgets +game +games +get +ghost +gift +gifts +gist +gist +github +graph +group +group +groups +groups +guest +guests +help +help +home +home +homepage +host +hosting +hosting +hostmaster +hostmaster +hostname +howto +hpg +html +http +httpd +https +i +iamges +icon +icons +id +idea +idea +ideas +ideas +image +images +imap +img +index +index +indice +info +info +information +inquiry +instagram +intranet +invitations +invitations +invite +invite +ipad +iphone +irc +is +is +issue +issues +it +it +item +items +java +javascript +job +job +jobs +jobs +join +js +json +json +jump +knowledgebase +language +language +languages +languages +last +ldap-status +legal +license +link +links +linux +list +lists +lists +log +log-in +log-out +log_in +log_out +login +login +logout +logout +logs +logs +m +mac +mail +mail +mail1 +mail2 +mail3 +mail4 +mail5 +mailer +mailing +maintenance +manager +manual +map +map +maps +maps +marketing +master +me +media +member +members +message +messages +messenger +microblog +microblogs +mine +mine +mis +mis +mob +mobile +movie +movies +mp3 +msg +msn +music +musicas +mx +my +mysql +name +named +nan +navi +navigation +net +network +new +news +news +newsletter +nick +nickname +notes +noticias +notification +notifications +notify +ns +ns1 +ns10 +ns2 +ns3 +ns4 +ns5 +ns6 +ns7 +ns8 +ns9 +null +oauth +oauth +oauth_clients +oauth_clients +offer +offers +offers +official +old +online +openid +openid +operator +order +order +orders +orders +organization +organizations +organizations +overview +owner +owners +page +pager +pages +panel +password +payment +perl +phone +photo +photoalbum +photos +php +phpmyadmin +phppgadmin +phpredisadmin +pic +pics +ping +plan +plans +plans +plugin +plugins +policy +pop +pop3 +popular +popular +portal +post +post +postfix +postmaster +postmaster +posts +pr +premium +press +price +pricing +privacy +privacy +privacy-policy +privacy_policy +privacypolicy +private +product +products +profile +project +projects +projects +promo +pub +public +purpose +put +put +python +query +random +ranking +read +readme +recent +recruit +recruitment +recruitment +register +register +registration +release +remove +remove +replies +replies +report +reports +repositories +repository +req +request +requests +reset +roc +root +root +rss +rss +ruby +rule +sag +sale +sales +sales +sample +samples +save +save +school +script +scripts +search +search +secure +security +security +self +send +server +server-info +server-status +service +services +session +sessions +sessions +setting +settings +settings +setup +share +shop +shop +show +sign-in +sign-up +sign_in +sign_up +signin +signout +signup +signup +site +sitemap +sitemap +sites +smartphone +smtp +soporte +source +spec +special +sql +src +ssh +ssl +ssl +ssladmin +ssladmin +ssladministrator +ssladministrator +sslwebmaster +sslwebmaster +staff +stage +staging +start +stat +state +static +stats +status +status +store +stores +stories +stories +style +styleguide +styleguide +stylesheet +stylesheets +subdomain +subscribe +subscribe +subscriptions +subscriptions +suporte +support +support +svn +swf +sys +sysadmin +sysadmin +sysadministrator +sysadministrator +system +tablet +tablets +tag +talk +task +tasks +team +teams +tech +telnet +term +terms +terms +terms-of-service +terms_of_service +termsofservice +test +test1 +test2 +test3 +teste +testing +tests +theme +themes +thread +threads +tmp +todo +tool +tools +top +topic +topics +tos +tour +tour +translations +translations +trends +trends +tutorial +tux +tv +twitter +undef +unfollow +unfollow +unsubscribe +unsubscribe +update +update +upload +uploads +url +url +usage +user +user +username +users +usuario +vendas +ver +version +video +videos +visitor +watch +weather +weather +web +webhook +webhooks +webmail +webmaster +webmaster +website +websites +welcome +widget +widget +widgets +widgets +wiki +wiki +win +windows +word +work +works +workshop +ww +ww +wws +www +www +www1 +www2 +www3 +www4 +www5 +www6 +www7 +wwws +wwww +wwww +xfn +xfn +xml +xml +xmpp +xmpp +xpg +xxx +yaml +yaml +year +yml +yml +you +yourdomain +yourname +yoursite +yourusername From 6182c07d6abbdb790efabc9d657d410666003384 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:24:13 -0400 Subject: [PATCH 0153/1042] remove dupes --- db/reserved_usernames.txt | 140 -------------------------------------- 1 file changed, 140 deletions(-) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index 97e1cdb05a..24d6f6eff6 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -76,25 +76,19 @@ sindhi # Generic reserved words 0 about -about access account -account accounts activate -activate activities activity ad add -add address adm admin -admin administration administrator -administrator ads adult advertising @@ -109,21 +103,15 @@ android anon anonymous api -api -app app apps -apps archive -archive -archives archives article asct asset atom auth -auth authentication avatar backup @@ -134,7 +122,6 @@ beta billing bin blog -blog blogs board book @@ -144,36 +131,29 @@ bots bug business cache -cache cadastro calendar call campaign cancel -cancel captcha career careers -careers -cart cart categories category cgi cgi-bin changelog -changelog chat check checking checkout -checkout client cliente clients code codereview -codereview comercial comment comments @@ -181,15 +161,10 @@ communities community company compare -compare compras config -config -configuration configuration connect -connect -contact contact contact-us contact_us @@ -198,14 +173,12 @@ contest contribute corp create -create css dashboard data db default delete -delete demo design designer @@ -221,32 +194,24 @@ dictionary die dir direct_messages -direct_messages directory dist doc docs documentation -documentation domain download -download -downloads downloads ecommerce edit -edit editor edu education email -email -employment employment empty end enterprise -enterprise entries entry error @@ -257,32 +222,22 @@ exit explore facebook faq -faq favorite favorites -favorites feature features feed -feed feedback -feedback -feeds feeds file files first flash fleet -fleet -fleets fleets flog follow -follow followers -followers -following following forgot form @@ -291,8 +246,6 @@ forums founder free friend -friend -friends friends ftp gadget @@ -304,24 +257,17 @@ ghost gift gifts gist -gist github graph group -group -groups groups guest guests help -help -home home homepage host hosting -hosting -hostmaster hostmaster hostname howto @@ -336,52 +282,39 @@ icon icons id idea -idea -ideas ideas image images imap img index -index indice info -info information inquiry instagram intranet invitations -invitations -invite invite ipad iphone irc is -is issue issues it -it item items java javascript job -job -jobs jobs join js json -json jump knowledgebase language -language -languages languages last ldap-status @@ -392,22 +325,17 @@ links linux list lists -lists log log-in log-out log_in log_out login -login logout -logout -logs logs m mac mail -mail mail1 mail2 mail3 @@ -419,8 +347,6 @@ maintenance manager manual map -map -maps maps marketing master @@ -434,8 +360,6 @@ messenger microblog microblogs mine -mine -mis mis mob mobile @@ -458,7 +382,6 @@ net network new news -news newsletter nick nickname @@ -480,25 +403,18 @@ ns8 ns9 null oauth -oauth -oauth_clients oauth_clients offer offers -offers official old online openid -openid operator order -order -orders orders organization organizations -organizations overview owner owners @@ -522,20 +438,16 @@ pics ping plan plans -plans plugin plugins policy pop pop3 popular -popular portal post -post postfix postmaster -postmaster posts pr premium @@ -543,7 +455,6 @@ press price pricing privacy -privacy privacy-policy privacy_policy privacypolicy @@ -553,13 +464,11 @@ products profile project projects -projects promo pub public purpose put -put python query random @@ -569,14 +478,10 @@ readme recent recruit recruitment -recruitment -register register registration release remove -remove -replies replies report reports @@ -588,27 +493,21 @@ requests reset roc root -root -rss rss ruby rule sag sale sales -sales sample samples save -save school script scripts search -search secure security -security self send server @@ -618,14 +517,11 @@ service services session sessions -sessions setting settings -settings setup share shop -shop show sign-in sign-up @@ -634,10 +530,8 @@ sign_up signin signout signup -signup site sitemap -sitemap sites smartphone smtp @@ -649,12 +543,8 @@ sql src ssh ssl -ssl -ssladmin ssladmin ssladministrator -ssladministrator -sslwebmaster sslwebmaster staff stage @@ -665,30 +555,22 @@ state static stats status -status store stores stories -stories style styleguide -styleguide stylesheet stylesheets subdomain subscribe -subscribe -subscriptions subscriptions suporte support -support svn swf sys sysadmin -sysadmin -sysadministrator sysadministrator system tablet @@ -703,7 +585,6 @@ tech telnet term terms -terms terms-of-service terms_of_service termsofservice @@ -727,10 +608,7 @@ topic topics tos tour -tour translations -translations -trends trends tutorial tux @@ -738,18 +616,13 @@ tv twitter undef unfollow -unfollow unsubscribe -unsubscribe -update update upload uploads url -url usage user -user username users usuario @@ -761,21 +634,16 @@ videos visitor watch weather -weather web webhook webhooks webmail webmaster -webmaster website websites welcome widget -widget widgets -widgets -wiki wiki win windows @@ -784,10 +652,8 @@ work works workshop ww -ww wws www -www www1 www2 www3 @@ -797,20 +663,14 @@ www6 www7 wwws wwww -wwww -xfn xfn xml -xml -xmpp xmpp xpg xxx yaml -yaml year yml -yml you yourdomain yourname From cfd1401b7f81c11a1bac45586af4ceda3247e88a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:24:58 -0400 Subject: [PATCH 0154/1042] a few custom words --- db/reserved_usernames.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index 24d6f6eff6..6f2d3ac87d 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -496,6 +496,7 @@ root rss ruby rule +rust sag sale sales @@ -614,6 +615,8 @@ tutorial tux tv twitter +type +typescript undef unfollow unsubscribe @@ -661,6 +664,8 @@ www4 www5 www6 www7 +www8 +www9 wwws wwww xfn From cdc271f70a169c92bc0be29ed6d77aadd8cbfee5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:31:15 -0400 Subject: [PATCH 0155/1042] new list + few of my own --- db/reserved_usernames.txt | 345 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index 6f2d3ac87d..25d4b975b4 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -2,6 +2,7 @@ # Taken from the following: # https://gist.github.com/caseyohara/1453705/bd38325ae2b562572afaa119efa59529c057bc85 # https://github.com/shouldbee/reserved-usernames +# https://github.com/Mottie/github-reserved-names # Stuff we care about storyteller @@ -195,6 +196,9 @@ die dir direct_messages directory +display +display_name +displayname dist doc docs @@ -534,6 +538,7 @@ signup site sitemap sites +slug smartphone smtp soporte @@ -629,6 +634,9 @@ user username users usuario +vanity +vanity_url +vanityurl vendas ver version @@ -681,3 +689,340 @@ yourdomain yourname yoursite yourusername +400 +401 +402 +403 +404 +405 +406 +407 +408 +409 +410 +411 +412 +413 +414 +415 +416 +417 +418 +419 +420 +421 +422 +423 +424 +425 +426 +427 +428 +429 +430 +431 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +510 +511 +about +access +account +admin +advisories +anonymous +any +api +apps +attributes +auth +billing +blob +blog +bounty +branches +business +businesses +c +cache +case-studies +categories +central +certification +changelog +cla +cloud +codereview +collection +collections +comments +commit +commits +community +companies +compare +contact +contributing +cookbook +coupons +customer-stories +customer +customers +dashboard +dashboards +design +develop +developer +diff +discover +discussions +docs +downloads +downtime +editor +editors +edu +enterprise +events +explore +featured +features +files +fixtures +forked +garage +ghost +gist +gists +graphs +guide +guides +help +help-wanted +home +hooks +hosting +hovercards +identity +images +inbox +individual +info +integration +interfaces +introduction +invalid-email-address +investors +issues +jobs +join +journal +journals +lab +labs +languages +launch +layouts +learn +legal +library +linux +listings +lists +login +logos +logout +mac +maintenance +malware +man +marketplace +mention +mentioned +mentioning +mentions +migrating +milestones +mine +mirrors +mobile +navigation +network +new +news +none +nonprofit +nonprofits +notices +notifications +oauth +offer +open-source +organisations +organizations +orgs +pages +partners +payments +personal +plans +plugins +popular +popularity +posts +press +pricing +professional +projects +pulls +raw +readme +recommendations +redeem +releases +render +reply +repositories +resources +restore +revert +save-net-neutrality +saved +scraping +search +security +services +sessions +settings +shareholders +shop +showcases +signin +signup +site +spam +sponsors +ssh +staff +starred +stars +static +status +statuses +storage +store +stories +styleguide +subscriptions +suggest +suggestion +suggestions +support +suspended +talks +teach +teacher +teachers +teaching +team +teams +ten +terms +timeline +topic +topics +tos +tour +train +training +translations +tree +trending +updates +username +users +visualization +w +watching +wiki +windows +works-with +www0 +www1 +www2 +www3 +www4 +www5 +www6 +www7 +www8 +www9 +avatars +assets +bookmarks +checks +conversations +labels +media +nodes +owners +page +registry +seats +shared +tasks +uploads +account +apps +blog +c +contact +contributing +customer +customers +developer +edu +guides +help +launch +mac +mirrors +pages +plans +press +readme +repositories +resources +security +settings +signup +terms +topic +tos +training +users +wiki +windows +advisories +collections +hovercards +inbox +invalid-email-address +suggestions +suspended +case-studies +watching +save-net-neutrality +ghost +status +included +true +false \ No newline at end of file From 21d3210f665a591fbbb129818749a33581b2d0de Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:32:32 -0400 Subject: [PATCH 0156/1042] more custom words --- db/reserved_usernames.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index 25d4b975b4..fd7a0f6842 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -5,7 +5,13 @@ # https://github.com/Mottie/github-reserved-names # Stuff we care about +storied +stories +story storyteller +tell +teller +vocode vocodes # Companies we'd love to have use our service From 4a8e2a00cdec4694619e6102fca812b229a653b1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:33:08 -0400 Subject: [PATCH 0157/1042] remove dupes --- db/reserved_usernames.txt | 1195 ++++++++++++++++--------------------- 1 file changed, 520 insertions(+), 675 deletions(-) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index fd7a0f6842..afc7306e1e 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -82,619 +82,6 @@ sindhi # Generic reserved words 0 -about -access -account -accounts -activate -activities -activity -ad -add -address -adm -admin -administration -administrator -ads -adult -advertising -affiliate -affiliates -ajax -all -alpha -analysis -analytics -android -anon -anonymous -api -app -apps -archive -archives -article -asct -asset -atom -auth -authentication -avatar -backup -balancer-manager -banner -banners -beta -billing -bin -blog -blogs -board -book -bookmark -bot -bots -bug -business -cache -cadastro -calendar -call -campaign -cancel -captcha -career -careers -cart -categories -category -cgi -cgi-bin -changelog -chat -check -checking -checkout -client -cliente -clients -code -codereview -comercial -comment -comments -communities -community -company -compare -compras -config -configuration -connect -contact -contact-us -contact_us -contactus -contest -contribute -corp -create -css -dashboard -data -db -default -delete -demo -design -designer -destroy -dev -devel -developer -developers -diagram -diary -dict -dictionary -die -dir -direct_messages -directory -display -display_name -displayname -dist -doc -docs -documentation -domain -download -downloads -ecommerce -edit -editor -edu -education -email -employment -empty -end -enterprise -entries -entry -error -errors -eval -event -exit -explore -facebook -faq -favorite -favorites -feature -features -feed -feedback -feeds -file -files -first -flash -fleet -fleets -flog -follow -followers -following -forgot -form -forum -forums -founder -free -friend -friends -ftp -gadget -gadgets -game -games -get -ghost -gift -gifts -gist -github -graph -group -groups -guest -guests -help -home -homepage -host -hosting -hostmaster -hostname -howto -hpg -html -http -httpd -https -i -iamges -icon -icons -id -idea -ideas -image -images -imap -img -index -indice -info -information -inquiry -instagram -intranet -invitations -invite -ipad -iphone -irc -is -issue -issues -it -item -items -java -javascript -job -jobs -join -js -json -jump -knowledgebase -language -languages -last -ldap-status -legal -license -link -links -linux -list -lists -log -log-in -log-out -log_in -log_out -login -logout -logs -m -mac -mail -mail1 -mail2 -mail3 -mail4 -mail5 -mailer -mailing -maintenance -manager -manual -map -maps -marketing -master -me -media -member -members -message -messages -messenger -microblog -microblogs -mine -mis -mob -mobile -movie -movies -mp3 -msg -msn -music -musicas -mx -my -mysql -name -named -nan -navi -navigation -net -network -new -news -newsletter -nick -nickname -notes -noticias -notification -notifications -notify -ns -ns1 -ns10 -ns2 -ns3 -ns4 -ns5 -ns6 -ns7 -ns8 -ns9 -null -oauth -oauth_clients -offer -offers -official -old -online -openid -operator -order -orders -organization -organizations -overview -owner -owners -page -pager -pages -panel -password -payment -perl -phone -photo -photoalbum -photos -php -phpmyadmin -phppgadmin -phpredisadmin -pic -pics -ping -plan -plans -plugin -plugins -policy -pop -pop3 -popular -portal -post -postfix -postmaster -posts -pr -premium -press -price -pricing -privacy -privacy-policy -privacy_policy -privacypolicy -private -product -products -profile -project -projects -promo -pub -public -purpose -put -python -query -random -ranking -read -readme -recent -recruit -recruitment -register -registration -release -remove -replies -report -reports -repositories -repository -req -request -requests -reset -roc -root -rss -ruby -rule -rust -sag -sale -sales -sample -samples -save -school -script -scripts -search -secure -security -self -send -server -server-info -server-status -service -services -session -sessions -setting -settings -setup -share -shop -show -sign-in -sign-up -sign_in -sign_up -signin -signout -signup -site -sitemap -sites -slug -smartphone -smtp -soporte -source -spec -special -sql -src -ssh -ssl -ssladmin -ssladministrator -sslwebmaster -staff -stage -staging -start -stat -state -static -stats -status -store -stores -stories -style -styleguide -stylesheet -stylesheets -subdomain -subscribe -subscriptions -suporte -support -svn -swf -sys -sysadmin -sysadministrator -system -tablet -tablets -tag -talk -task -tasks -team -teams -tech -telnet -term -terms -terms-of-service -terms_of_service -termsofservice -test -test1 -test2 -test3 -teste -testing -tests -theme -themes -thread -threads -tmp -todo -tool -tools -top -topic -topics -tos -tour -translations -trends -tutorial -tux -tv -twitter -type -typescript -undef -unfollow -unsubscribe -update -upload -uploads -url -usage -user -username -users -usuario -vanity -vanity_url -vanityurl -vendas -ver -version -video -videos -visitor -watch -weather -web -webhook -webhooks -webmail -webmaster -website -websites -welcome -widget -widgets -wiki -win -windows -word -work -works -workshop -ww -wws -www -www1 -www2 -www3 -www4 -www5 -www6 -www7 -www8 -www9 -wwws -wwww -xfn -xml -xmpp -xpg -xxx -yaml -year -yml -you -yourdomain -yourname -yoursite -yourusername 400 401 402 @@ -742,207 +129,664 @@ yourusername about access account +accounts +activate +activities +activity +ad +add +address +adm admin +administration +administrator +ads +adult +advertising advisories +affiliate +affiliates +ajax +all +alpha +analysis +analytics +android +anon anonymous any api +app apps +archive +archives +article +asct +asset +assets +atom attributes auth +authentication +avatar +avatars +backup +balancer-manager +banner +banners +beta billing +bin blob blog +blogs +board +book +bookmark +bookmarks +bot +bots bounty branches +bug business businesses c cache +cadastro +calendar +call +campaign +cancel +captcha +career +careers +cart case-studies categories +category central certification +cgi +cgi-bin changelog +chat +check +checking +checkout +checks cla +client +cliente +clients cloud +code codereview collection collections +comercial +comment comments commit commits +communities community companies +company compare +compras +config +configuration +connect contact +contact-us +contact_us +contactus +contest +contribute contributing +conversations cookbook +corp coupons -customer-stories +create +css customer +customer-stories customers dashboard dashboards +data +db +default +delete +demo design +designer +destroy +dev +devel develop developer +developers +diagram +diary +dict +dictionary +die diff +dir +direct_messages +directory discover discussions +display +display_name +displayname +dist +doc docs +documentation +domain +download downloads downtime +ecommerce +edit editor editors edu +education +email +employment +empty +end enterprise +entries +entry +error +errors +eval +event events +exit explore +facebook +false +faq +favorite +favorites +feature featured features +feed +feedback +feeds +file files +first fixtures +flash +fleet +fleets +flog +follow +followers +following +forgot forked +form +forum +forums +founder +free +friend +friends +ftp +gadget +gadgets +game +games garage +get ghost +gift +gifts gist gists +github +graph graphs +group +groups +guest +guests guide guides help help-wanted home +homepage hooks +host hosting +hostmaster +hostname hovercards +howto +hpg +html +http +httpd +https +i +iamges +icon +icons +id +idea +ideas identity +image images +imap +img inbox +included +index +indice individual info +information +inquiry +instagram integration interfaces +intranet introduction invalid-email-address investors +invitations +invite +ipad +iphone +irc +is +issue issues +it +item +items +java +javascript +job jobs join journal journals +js +json +jump +knowledgebase lab +labels labs +language languages +last launch layouts +ldap-status learn legal library +license +link +links linux +list listings lists +log +log-in +log-out +log_in +log_out login logos logout +logs +m mac +mail +mail1 +mail2 +mail3 +mail4 +mail5 +mailer +mailing maintenance malware man +manager +manual +map +maps +marketing marketplace +master +me +media +member +members mention mentioned mentioning mentions +message +messages +messenger +microblog +microblogs migrating milestones mine mirrors +mis +mob mobile +movie +movies +mp3 +msg +msn +music +musicas +mx +my +mysql +name +named +nan +navi navigation +net network new news +newsletter +nick +nickname +nodes none nonprofit nonprofits +notes notices +noticias +notification notifications +notify +ns +ns1 +ns10 +ns2 +ns3 +ns4 +ns5 +ns6 +ns7 +ns8 +ns9 +null oauth +oauth_clients offer +offers +official +old +online open-source +openid +operator +order +orders organisations +organization organizations orgs +overview +owner +owners +page +pager pages +panel partners +password +payment payments +perl personal +phone +photo +photoalbum +photos +php +phpmyadmin +phppgadmin +phpredisadmin +pic +pics +ping +plan plans +plugin plugins +policy +pop +pop3 popular popularity +portal +post +postfix +postmaster posts +pr +premium press +price pricing +privacy +privacy-policy +privacy_policy +privacypolicy +private +product +products professional +profile +project projects +promo +pub +public pulls +purpose +put +python +query +random +ranking raw +read readme +recent recommendations +recruit +recruitment redeem +register +registration +registry +release releases +remove render +replies reply +report +reports repositories +repository +req +request +requests +reset resources restore revert +roc +root +rss +ruby +rule +rust +sag +sale +sales +sample +samples +save save-net-neutrality saved +school scraping +script +scripts search +seats +secure security +self +send +server +server-info +server-status +service services +session sessions +setting settings +setup +share +shared shareholders shop +show showcases +sign-in +sign-up +sign_in +sign_up signin +signout signup site +sitemap +sites +slug +smartphone +smtp +soporte +source spam +spec +special sponsors +sql +src ssh +ssl +ssladmin +ssladministrator +sslwebmaster staff +stage +staging starred stars +start +stat +state static +stats status statuses storage store +stores stories +style styleguide +stylesheet +stylesheets +subdomain +subscribe subscriptions suggest suggestion suggestions +suporte support suspended +svn +swf +sys +sysadmin +sysadministrator +system +tablet +tablets +tag +talk talks +task +tasks teach teacher teachers teaching team teams +tech +telnet ten +term terms +terms-of-service +terms_of_service +termsofservice +test +test1 +test2 +test3 +teste +testing +tests +theme +themes +thread +threads timeline +tmp +todo +tool +tools +top topic topics tos @@ -952,15 +796,62 @@ training translations tree trending +trends +true +tutorial +tux +tv +twitter +type +typescript +undef +unfollow +unsubscribe +update updates +upload +uploads +url +usage +user username users +usuario +vanity +vanity_url +vanityurl +vendas +ver +version +video +videos +visitor visualization w +watch watching +weather +web +webhook +webhooks +webmail +webmaster +website +websites +welcome +widget +widgets wiki +win windows +word +work +works works-with +workshop +ww +wws +www www0 www1 www2 @@ -971,64 +862,18 @@ www6 www7 www8 www9 -avatars -assets -bookmarks -checks -conversations -labels -media -nodes -owners -page -registry -seats -shared -tasks -uploads -account -apps -blog -c -contact -contributing -customer -customers -developer -edu -guides -help -launch -mac -mirrors -pages -plans -press -readme -repositories -resources -security -settings -signup -terms -topic -tos -training -users -wiki -windows -advisories -collections -hovercards -inbox -invalid-email-address -suggestions -suspended -case-studies -watching -save-net-neutrality -ghost -status -included -true -false \ No newline at end of file +wwws +wwww +xfn +xml +xmpp +xpg +xxx +yaml +year +yml +you +yourdomain +yourname +yoursite +yourusername From 75735ce3c7e7da5151e1a8e851a24ec3399573f6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:44:31 -0400 Subject: [PATCH 0158/1042] more custom words --- db/reserved_usernames.txt | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index afc7306e1e..e1b0219a2f 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -5,6 +5,7 @@ # https://github.com/Mottie/github-reserved-names # Stuff we care about +monogatari storied stories story @@ -12,11 +13,26 @@ storyteller tell teller vocode +vocoder +vocoders vocodes +# Types of models +first_order_model +fom +hifigan +melgan +tacotron +tt2 +tts +w2l + # Companies we'd love to have use our service # so we'll reserve them to be safe +amazon apple +cash +cashapp epic epicgames facebook @@ -24,8 +40,21 @@ github google microsoft netflix +nintendo +paypal +reddit +snap +snapchat +softbank +sony +square +stripe +tiktok +twitch twitter unreal +venmo +youtube # Top 50 languages by speaking population chinese @@ -343,6 +372,7 @@ free friend friends ftp +ftps gadget gadgets game @@ -401,6 +431,9 @@ information inquiry instagram integration +integration-test +integration_test +integrationtest interfaces intranet introduction @@ -682,6 +715,7 @@ sessions setting settings setup +sftp share shared shareholders @@ -702,6 +736,8 @@ slug smartphone smtp soporte +sort +sorted source spam spec @@ -801,6 +837,7 @@ true tutorial tux tv +tweet twitter type typescript @@ -825,6 +862,8 @@ ver version video videos +view +viewer visitor visualization w From eb3c327ddea0d8af2355e72bb3f494c454aaa8a3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:46:23 -0400 Subject: [PATCH 0159/1042] another list --- db/reserved_usernames.txt | 618 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 618 insertions(+) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index e1b0219a2f..43a1750d8d 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -3,6 +3,7 @@ # https://gist.github.com/caseyohara/1453705/bd38325ae2b562572afaa119efa59529c057bc85 # https://github.com/shouldbee/reserved-usernames # https://github.com/Mottie/github-reserved-names +# https://github.com/mvila/reserved-usernames # Stuff we care about monogatari @@ -916,3 +917,620 @@ yourdomain yourname yoursite yourusername +0 +100 +101 +102 +1xx +200 +201 +202 +203 +204 +205 +206 +207 +226 +2xx +300 +301 +302 +303 +304 +305 +307 +308 +3xx +400 +401 +402 +403 +404 +405 +406 +407 +408 +409 +410 +411 +412 +413 +414 +415 +416 +417 +418 +422 +423 +424 +426 +428 +429 +431 +451 +4xx +500 +501 +502 +503 +504 +505 +506 +507 +511 +5xx +7xx +about +abuse +access +account +accounts +activate +activities +activity +ad +add +address +adm +admin +administration +administrator +ads +advertising +affiliate +affiliates +all +analysis +analytics +anon +anonymous +api +app +apps +archive +archives +asset +auth +authentication +autoconfig +avatar +backup +bank +beta +billing +bin +blog +blogs +board +bot +bots +broadcasthost +bug +bugs +cache +call +cancel +captcha +career +careers +cart +categories +category +cgi +cgi-bin +changelog +chat +check +checking +checkout +client +clients +code +comment +comments +communities +community +company +config +configuration +connect +contact +contact-us +contact_us +contactus +contribute +corp +create +crypt +css +dashboard +data +db +default +delete +demo +destroy +dev +devel +developer +developers +die +dir +directory +dist +dns +doc +docker +docs +documentation +download +downloads +edit +editor +email +employment +empty +end +enterprise +entries +entry +error +errors +eval +event +exit +explore +export +faq +favorite +favorites +fbi +feature +features +feed +feedback +feeds +file +files +firewall +first +follow +followers +following +forgot +forgot-password +forgot_password +forgotpassword +form +forum +forums +free +friend +friends +ftp +get +gift +gifts +gist +git +graph +group +groups +guest +guests +help +home +homepage +hooks +host +hosting +hostmaster +hostname +howto +html +http +httpd +https +i +id +image +images +imap +img +index +info +information +inquiry +intranet +invitations +invite +ip +irc +is +isatap +issue +issues +it +item +items +java +javascript +job +jobs +join +js +json +keys +keyserver +knowledgebase +language +languages +last +legal +license +link +links +list +lists +local +localdomain +localhost +log +log-in +log-out +log_in +log_out +login +logout +logs +m +mail +mail1 +mail2 +mail3 +mail4 +mail5 +mailer +mailer-daemon +mailing +maintenance +manager +manual +map +maps +marketing +master +me +media +member +members +message +messages +mis +mob +mobile +movie +movies +msg +music +mx +my +name +named +names +namespace +namespaces +nan +navi +navigation +net +network +new +news +newsletter +nick +nickname +no-reply +nobody +noc +noreply +notes +notification +notifications +notify +ns +ns1 +ns10 +ns2 +ns3 +ns4 +ns5 +ns6 +ns7 +ns8 +ns9 +null +oauth +offer +offers +official +old +online +openid +operator +ops +order +orders +organization +organizations +orgs +overview +owner +owners +package +page +pages +panel +passwd +password +patch +payment +photo +photos +pic +pics +ping +plan +plans +plugin +plugins +policy +pop +pop3 +popular +portal +post +postmaster +posts +pr +premium +press +price +pricing +privacy +privacy-policy +privacy_policy +privacypolicy +private +product +products +profile +project +projects +promo +pub +public +put +pw +query +random +ranking +read +readme +recent +recruit +recruitment +register +registration +release +releases +remote +remove +replies +reply +report +reports +repositories +repository +req +request +requests +res +reset +reset-password +reset_password +resetpassword +resource +resources +root +rss +rule +rules +sale +sales +sample +samples +save +script +scripts +search +secure +security +self +send +server +server-info +server-status +service +services +session +sessions +setting +settings +setup +share +shop +show +sign-in +sign-up +sign_in +sign_up +signin +signout +signup +site +sitemap +sites +smtp +source +spec +special +src +ssh +ssl +ssladmin +ssladministrator +sslwebmaster +staff +stage +staging +start +stat +state +static +stats +status +store +stores +stories +style +styles +stylesheet +stylesheets +subdomain +subscribe +subscriptions +support +sys +sysadmin +sysadministrator +system +tag +tags +talk +task +tasks +team +teams +tech +telnet +term +terms +terms-of-service +terms_of_service +termsofservice +test +test1 +test2 +test3 +testing +tests +theme +themes +thread +threads +tls +tmp +todo +token +tokenserver +tool +tools +top +topic +topics +tos +tour +translations +trends +tutorial +undef +unfollow +unsubscribe +update +upload +uploads +uptime +url +usage +usenet +user +username +users +usr +util +uucp +ver +version +video +videos +visitor +vpn +watch +web +webmail +webmaster +website +websites +welcome +widget +widgets +wiki +work +works +wpad +ww +wws +www +www1 +www2 +www3 +www4 +www5 +www6 +www7 +wwws +wwww +xfn +xml +xmpp +xpg +xxx +yaml +year +yml +you +yourdomain +yourname +yoursite +yourusername From d4800f8b62039f9801828b5ae2667f11b26078fe Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:47:06 -0400 Subject: [PATCH 0160/1042] merge lists --- db/reserved_usernames.txt | 698 +++++--------------------------------- 1 file changed, 81 insertions(+), 617 deletions(-) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index 43a1750d8d..24266359fb 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -112,6 +112,29 @@ sindhi # Generic reserved words 0 +100 +101 +102 +1xx +200 +201 +202 +203 +204 +205 +206 +207 +226 +2xx +300 +301 +302 +303 +304 +305 +307 +308 +3xx 400 401 402 @@ -144,6 +167,8 @@ sindhi 429 430 431 +451 +4xx 500 501 502 @@ -156,7 +181,10 @@ sindhi 509 510 511 +5xx +7xx about +abuse access account accounts @@ -198,10 +226,12 @@ atom attributes auth authentication +autoconfig avatar avatars backup balancer-manager +bank banner banners beta @@ -218,7 +248,9 @@ bot bots bounty branches +broadcasthost bug +bugs business businesses c @@ -280,6 +312,7 @@ cookbook corp coupons create +crypt css customer customer-stories @@ -314,7 +347,9 @@ display display_name displayname dist +dns doc +docker docs documentation domain @@ -341,11 +376,13 @@ event events exit explore +export facebook false faq favorite favorites +fbi feature featured features @@ -354,6 +391,7 @@ feedback feeds file files +firewall first fixtures flash @@ -364,6 +402,9 @@ follow followers following forgot +forgot-password +forgot_password +forgotpassword forked form forum @@ -385,6 +426,7 @@ gift gifts gist gists +git github graph graphs @@ -442,10 +484,12 @@ invalid-email-address investors invitations invite +ip ipad iphone irc is +isatap issue issues it @@ -461,6 +505,8 @@ journals js json jump +keys +keyserver knowledgebase lab labels @@ -481,6 +527,9 @@ linux list listings lists +local +localdomain +localhost log log-in log-out @@ -499,6 +548,7 @@ mail3 mail4 mail5 mailer +mailer-daemon mailing maintenance malware @@ -542,6 +592,9 @@ my mysql name named +names +namespace +namespaces nan navi navigation @@ -552,10 +605,14 @@ news newsletter nick nickname +no-reply +nobody +noc nodes none nonprofit nonprofits +noreply notes notices noticias @@ -584,6 +641,7 @@ online open-source openid operator +ops order orders organisations @@ -593,12 +651,15 @@ orgs overview owner owners +package page pager pages panel partners +passwd password +patch payment payments perl @@ -650,6 +711,7 @@ public pulls purpose put +pw python query random @@ -667,6 +729,7 @@ registration registry release releases +remote remove render replies @@ -678,7 +741,12 @@ repository req request requests +res reset +reset-password +reset_password +resetpassword +resource resources restore revert @@ -687,6 +755,7 @@ root rss ruby rule +rules rust sag sale @@ -769,6 +838,7 @@ stores stories style styleguide +styles stylesheet stylesheets subdomain @@ -789,6 +859,7 @@ system tablet tablets tag +tags talk talks task @@ -819,8 +890,11 @@ themes thread threads timeline +tls tmp todo +token +tokenserver tool tools top @@ -849,12 +923,17 @@ update updates upload uploads +uptime url usage +usenet user username users +usr usuario +util +uucp vanity vanity_url vanityurl @@ -867,6 +946,7 @@ view viewer visitor visualization +vpn w watch watching @@ -889,6 +969,7 @@ work works works-with workshop +wpad ww wws www @@ -917,620 +998,3 @@ yourdomain yourname yoursite yourusername -0 -100 -101 -102 -1xx -200 -201 -202 -203 -204 -205 -206 -207 -226 -2xx -300 -301 -302 -303 -304 -305 -307 -308 -3xx -400 -401 -402 -403 -404 -405 -406 -407 -408 -409 -410 -411 -412 -413 -414 -415 -416 -417 -418 -422 -423 -424 -426 -428 -429 -431 -451 -4xx -500 -501 -502 -503 -504 -505 -506 -507 -511 -5xx -7xx -about -abuse -access -account -accounts -activate -activities -activity -ad -add -address -adm -admin -administration -administrator -ads -advertising -affiliate -affiliates -all -analysis -analytics -anon -anonymous -api -app -apps -archive -archives -asset -auth -authentication -autoconfig -avatar -backup -bank -beta -billing -bin -blog -blogs -board -bot -bots -broadcasthost -bug -bugs -cache -call -cancel -captcha -career -careers -cart -categories -category -cgi -cgi-bin -changelog -chat -check -checking -checkout -client -clients -code -comment -comments -communities -community -company -config -configuration -connect -contact -contact-us -contact_us -contactus -contribute -corp -create -crypt -css -dashboard -data -db -default -delete -demo -destroy -dev -devel -developer -developers -die -dir -directory -dist -dns -doc -docker -docs -documentation -download -downloads -edit -editor -email -employment -empty -end -enterprise -entries -entry -error -errors -eval -event -exit -explore -export -faq -favorite -favorites -fbi -feature -features -feed -feedback -feeds -file -files -firewall -first -follow -followers -following -forgot -forgot-password -forgot_password -forgotpassword -form -forum -forums -free -friend -friends -ftp -get -gift -gifts -gist -git -graph -group -groups -guest -guests -help -home -homepage -hooks -host -hosting -hostmaster -hostname -howto -html -http -httpd -https -i -id -image -images -imap -img -index -info -information -inquiry -intranet -invitations -invite -ip -irc -is -isatap -issue -issues -it -item -items -java -javascript -job -jobs -join -js -json -keys -keyserver -knowledgebase -language -languages -last -legal -license -link -links -list -lists -local -localdomain -localhost -log -log-in -log-out -log_in -log_out -login -logout -logs -m -mail -mail1 -mail2 -mail3 -mail4 -mail5 -mailer -mailer-daemon -mailing -maintenance -manager -manual -map -maps -marketing -master -me -media -member -members -message -messages -mis -mob -mobile -movie -movies -msg -music -mx -my -name -named -names -namespace -namespaces -nan -navi -navigation -net -network -new -news -newsletter -nick -nickname -no-reply -nobody -noc -noreply -notes -notification -notifications -notify -ns -ns1 -ns10 -ns2 -ns3 -ns4 -ns5 -ns6 -ns7 -ns8 -ns9 -null -oauth -offer -offers -official -old -online -openid -operator -ops -order -orders -organization -organizations -orgs -overview -owner -owners -package -page -pages -panel -passwd -password -patch -payment -photo -photos -pic -pics -ping -plan -plans -plugin -plugins -policy -pop -pop3 -popular -portal -post -postmaster -posts -pr -premium -press -price -pricing -privacy -privacy-policy -privacy_policy -privacypolicy -private -product -products -profile -project -projects -promo -pub -public -put -pw -query -random -ranking -read -readme -recent -recruit -recruitment -register -registration -release -releases -remote -remove -replies -reply -report -reports -repositories -repository -req -request -requests -res -reset -reset-password -reset_password -resetpassword -resource -resources -root -rss -rule -rules -sale -sales -sample -samples -save -script -scripts -search -secure -security -self -send -server -server-info -server-status -service -services -session -sessions -setting -settings -setup -share -shop -show -sign-in -sign-up -sign_in -sign_up -signin -signout -signup -site -sitemap -sites -smtp -source -spec -special -src -ssh -ssl -ssladmin -ssladministrator -sslwebmaster -staff -stage -staging -start -stat -state -static -stats -status -store -stores -stories -style -styles -stylesheet -stylesheets -subdomain -subscribe -subscriptions -support -sys -sysadmin -sysadministrator -system -tag -tags -talk -task -tasks -team -teams -tech -telnet -term -terms -terms-of-service -terms_of_service -termsofservice -test -test1 -test2 -test3 -testing -tests -theme -themes -thread -threads -tls -tmp -todo -token -tokenserver -tool -tools -top -topic -topics -tos -tour -translations -trends -tutorial -undef -unfollow -unsubscribe -update -upload -uploads -uptime -url -usage -usenet -user -username -users -usr -util -uucp -ver -version -video -videos -visitor -vpn -watch -web -webmail -webmaster -website -websites -welcome -widget -widgets -wiki -work -works -wpad -ww -wws -www -www1 -www2 -www3 -www4 -www5 -www6 -www7 -wwws -wwww -xfn -xml -xmpp -xpg -xxx -yaml -year -yml -you -yourdomain -yourname -yoursite -yourusername From 62b85a11663f421a5213d1e3b12354d7d147fee3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 12:49:04 -0400 Subject: [PATCH 0161/1042] more --- db/reserved_usernames.txt | 6 ++++++ db/reserved_usernames_including.txt | 1 + 2 files changed, 7 insertions(+) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index 24266359fb..24860afab0 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -882,6 +882,12 @@ test test1 test2 test3 +test4 +test5 +test6 +test7 +test8 +test9 teste testing tests diff --git a/db/reserved_usernames_including.txt b/db/reserved_usernames_including.txt index 6b42928461..8961b7e60c 100644 --- a/db/reserved_usernames_including.txt +++ b/db/reserved_usernames_including.txt @@ -2,4 +2,5 @@ # Any of the substrings may not be used. # You should check string.remove('_') as well. storyteller +test vocodes From 88305f5d10f14e9de0022a94b4444a91953450a3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 13:53:01 -0400 Subject: [PATCH 0162/1042] functions to check resreved usernames --- src/main.rs | 3 ++ src/validations/mod.rs | 3 +- src/validations/username_reservations.rs | 64 ++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 src/validations/username_reservations.rs diff --git a/src/main.rs b/src/main.rs index 5c6bdeaa33..fe1326742e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,9 @@ #[macro_use] extern crate lazy_static; #[macro_use] extern crate serde_derive; +pub const RESERVED_USERNAMES : &'static str = include_str!("../db/reserved_usernames.txt"); +pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../db/reserved_usernames_including.txt"); + pub mod buckets; pub mod common_queries; pub mod database_helpers; diff --git a/src/validations/mod.rs b/src/validations/mod.rs index ef3b154a7c..fa53768548 100644 --- a/src/validations/mod.rs +++ b/src/validations/mod.rs @@ -1,3 +1,4 @@ pub mod model_uploads; pub mod passwords; -pub mod username; \ No newline at end of file +pub mod username; +pub mod username_reservations; \ No newline at end of file diff --git a/src/validations/username_reservations.rs b/src/validations/username_reservations.rs new file mode 100644 index 0000000000..dddc8fdc25 --- /dev/null +++ b/src/validations/username_reservations.rs @@ -0,0 +1,64 @@ +use std::collections::HashSet; +use crate::RESERVED_SUBSTRINGS; +use crate::RESERVED_USERNAMES; + +pub fn is_reserved_username(username: &str) -> bool { + lazy_static! { + static ref RESERVED_USERNAMES_SET : HashSet = RESERVED_USERNAMES.lines() + .map(|line| line.trim()) + .filter(|line| !(line.starts_with("#") || line.is_empty())) + .map(|line| line.to_string()) + .collect::>(); + } + + if RESERVED_USERNAMES_SET.contains(username) { + return true; + } + + is_reserved_substring(username) +} + +fn is_reserved_substring(username: &str) -> bool { + lazy_static! { + static ref RESERVED_SUBSTRINGS_LIST : Vec = RESERVED_SUBSTRINGS.lines() + .map(|line| line.trim()) + .filter(|line| !(line.starts_with("#") || line.is_empty())) + .map(|line| line.to_string()) + .collect::>(); + } + + for substr in RESERVED_SUBSTRINGS_LIST.iter() { + if username.contains(substr) { + return true; + } + } + + return false; +} + +#[cfg(test)] +mod tests { + use crate::validations::username_reservations::{is_reserved_username}; + + #[test] + fn reserved_usernames() { + assert_eq!(is_reserved_username("vocodes"), true); + assert_eq!(is_reserved_username("user"), true); + assert_eq!(is_reserved_username("username"), true); + } + + #[test] + fn unreserved_usernames() { + assert_eq!(is_reserved_username("echelon"), false); + assert_eq!(is_reserved_username("asdfasdfadsf"), false); + assert_eq!(is_reserved_username("bobdole11"), false); + } + + #[test] + fn reserved_substrings() { + assert_eq!(is_reserved_username("test112345"), true); + assert_eq!(is_reserved_username("12345test"), true); + assert_eq!(is_reserved_username("test"), true); + assert_eq!(is_reserved_username("111vocodes111"), true); + } +} \ No newline at end of file From 6a717d386e3d927438dc49491277669fb9504bee Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 13:56:12 -0400 Subject: [PATCH 0163/1042] withold reserved usernames --- debug/curl/create_account_reserved.sh | 7 +++++++ src/http_server/endpoints/users/create_account.rs | 8 ++++++++ 2 files changed, 15 insertions(+) create mode 100755 debug/curl/create_account_reserved.sh diff --git a/debug/curl/create_account_reserved.sh b/debug/curl/create_account_reserved.sh new file mode 100755 index 0000000000..020b8534a7 --- /dev/null +++ b/debug/curl/create_account_reserved.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# LOCALHOST ONLY + +curl -vvv \ + -H "content-type: application/json" \ + --data '{"username": "vocodes", "password": "testing", "password_confirmation": "testing", "email_address": "vocodes@gmail.com"}' \ + http://localhost:12345/create_account diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index c9e303a132..99ca184609 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -20,6 +20,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::util::email_to_gravatar::email_to_gravatar; +use crate::validations::username_reservations::is_reserved_username; const NEW_USER_ROLE: &'static str = "new-user"; @@ -46,6 +47,7 @@ pub struct ErrorResponse { pub enum CreateAccountError { BadInput(String), UsernameTaken, + ReservedUsername, EmailTaken, ServerError, } @@ -55,6 +57,7 @@ impl ResponseError for CreateAccountError { match *self { CreateAccountError::BadInput(_) => StatusCode::BAD_REQUEST, CreateAccountError::UsernameTaken => StatusCode::BAD_REQUEST, + CreateAccountError::ReservedUsername => StatusCode::BAD_REQUEST, CreateAccountError::EmailTaken => StatusCode::BAD_REQUEST, CreateAccountError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } @@ -64,6 +67,7 @@ impl ResponseError for CreateAccountError { let error_reason = match self { BadInput(reason) => reason.to_string(), CreateAccountError::UsernameTaken => "username already taken".to_string(), + CreateAccountError::ReservedUsername => "username is reserved".to_string(), CreateAccountError::EmailTaken => "email already taken".to_string(), CreateAccountError::ServerError => "server error".to_string(), }; @@ -97,6 +101,10 @@ pub async fn create_account_handler( return Err(CreateAccountError::BadInput(reason)); } + if is_reserved_username(&request.username) { + return Err(CreateAccountError::ReservedUsername); + } + if !request.email_address.contains("@") { return Err(CreateAccountError::BadInput("invalid email address".to_string())); } From 5eda3a9ed39c2c7ffdfe36671ca122fedf2fc2a0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 19:14:41 -0400 Subject: [PATCH 0164/1042] list models --- .../endpoints/tts/list_tts_models.rs | 169 ++++++++++++++++++ src/http_server/endpoints/tts/mod.rs | 3 +- .../endpoints/w2l/get_w2l_template.rs | 16 +- .../endpoints/w2l/list_w2l_templates.rs | 16 +- src/main.rs | 6 + 5 files changed, 193 insertions(+), 17 deletions(-) create mode 100644 src/http_server/endpoints/tts/list_tts_models.rs diff --git a/src/http_server/endpoints/tts/list_tts_models.rs b/src/http_server/endpoints/tts/list_tts_models.rs new file mode 100644 index 0000000000..338b573f29 --- /dev/null +++ b/src/http_server/endpoints/tts/list_tts_models.rs @@ -0,0 +1,169 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_token::random_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use chrono::{DateTime, Utc}; + +#[derive(Serialize)] +pub struct TtsModelRecordForResponse { + pub model_token: String, + pub tts_model_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct ListTtsModelsSuccessResponse { + pub success: bool, + pub templates: Vec, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum ListTtsModelsError { + ServerError, +} + +#[derive(Serialize)] +pub struct TtsModelRecord { + pub model_token: String, + pub tts_model_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl ResponseError for ListTtsModelsError { + fn status_code(&self) -> StatusCode { + match *self { + ListTtsModelsError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListTtsModelsError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn list_tts_models_handler( + http_request: HttpRequest, + server_state: web::Data>) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_templates = sqlx::query_as!( + TtsModelRecord, + r#" +SELECT + tts.token as model_token, + tts.tts_model_type, + tts.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + tts.updatable_slug, + tts.title, + tts.created_at, + tts.updated_at +FROM tts_models as tts +JOIN users +ON users.token = tts.creator_user_token +WHERE tts.deleted_at IS NULL + "#, + ) + .fetch_all(&server_state.mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let templates : Vec = match maybe_templates { + Ok(templates) => templates, + Err(err) => { + match err { + RowNotFound => { + return Err(ListTtsModelsError::ServerError); + }, + _ => { + warn!("w2l template list query error: {:?}", err); + return Err(ListTtsModelsError::ServerError); + } + } + } + }; + + let templates_for_response = templates.into_iter() + .map(|template| { + TtsModelRecordForResponse { + model_token: template.model_token.clone(), + tts_model_type: template.tts_model_type.clone(), + creator_user_token: template.creator_user_token.clone(), + creator_username: template.creator_username.clone(), + creator_display_name: template.creator_display_name.clone(), + updatable_slug: template.updatable_slug.clone(), + title: template.title.clone(), + created_at: template.created_at.clone(), + updated_at: template.updated_at.clone(), + } + }) + .collect::>(); + + let response = ListTtsModelsSuccessResponse { + success: true, + templates: templates_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListTtsModelsError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/tts/mod.rs b/src/http_server/endpoints/tts/mod.rs index 371b17ad4d..e0e4f48731 100644 --- a/src/http_server/endpoints/tts/mod.rs +++ b/src/http_server/endpoints/tts/mod.rs @@ -1,2 +1,3 @@ pub mod enqueue_infer_tts; -pub mod enqueue_upload_tts_model; \ No newline at end of file +pub mod enqueue_upload_tts_model; +pub mod list_tts_models; \ No newline at end of file diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index 455c1670b6..8a332cc10b 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -37,8 +37,8 @@ pub struct W2lTemplateRecordForResponse { pub template_token: String, pub template_type: String, pub creator_user_token: String, - pub username: String, - pub display_name: String, + pub creator_username: String, + pub creator_display_name: String, pub updatable_slug: String, pub title: String, pub frame_width: u32, @@ -72,8 +72,8 @@ pub struct W2lTemplateRecord { pub template_token: String, pub template_type: String, pub creator_user_token: String, - pub username: String, - pub display_name: String, + pub creator_username: String, + pub creator_display_name: String, pub updatable_slug: String, pub title: String, pub frame_width: i32, @@ -127,8 +127,8 @@ SELECT w2l.token as template_token, w2l.template_type, w2l.creator_user_token, - users.username, - users.display_name, + users.username as creator_username, + users.display_name as creator_display_name, w2l.updatable_slug, w2l.title, w2l.frame_width, @@ -168,8 +168,8 @@ AND w2l.deleted_at IS NULL template_token: template.template_token.clone(), template_type: template.template_type.clone(), creator_user_token: template.creator_user_token.clone(), - username: template.username.clone(), - display_name: template.display_name.clone(), + creator_username: template.creator_username.clone(), + creator_display_name: template.creator_display_name.clone(), updatable_slug: template.updatable_slug.clone(), title: template.title.clone(), frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, diff --git a/src/http_server/endpoints/w2l/list_w2l_templates.rs b/src/http_server/endpoints/w2l/list_w2l_templates.rs index 0b55d5865a..849f96a1fc 100644 --- a/src/http_server/endpoints/w2l/list_w2l_templates.rs +++ b/src/http_server/endpoints/w2l/list_w2l_templates.rs @@ -30,8 +30,8 @@ pub struct W2lTemplateRecordForResponse { pub template_token: String, pub template_type: String, pub creator_user_token: String, - pub username: String, - pub display_name: String, + pub creator_username: String, + pub creator_display_name: String, pub updatable_slug: String, pub title: String, pub frame_width: u32, @@ -65,8 +65,8 @@ pub struct W2lTemplateRecord { pub template_token: String, pub template_type: String, pub creator_user_token: String, - pub username: String, - pub display_name: String, + pub creator_username: String, + pub creator_display_name: String, pub updatable_slug: String, pub title: String, pub frame_width: i32, @@ -119,8 +119,8 @@ SELECT w2l.token as template_token, w2l.template_type, w2l.creator_user_token, - users.username, - users.display_name, + users.username as creator_username, + users.display_name as creator_display_name, w2l.updatable_slug, w2l.title, w2l.frame_width, @@ -160,8 +160,8 @@ WHERE w2l.deleted_at IS NULL template_token: template.template_token.clone(), template_type: template.template_type.clone(), creator_user_token: template.creator_user_token.clone(), - username: template.username.clone(), - display_name: template.display_name.clone(), + creator_username: template.creator_username.clone(), + creator_display_name: template.creator_display_name.clone(), updatable_slug: template.updatable_slug.clone(), title: template.title.clone(), frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, diff --git a/src/main.rs b/src/main.rs index fe1326742e..ca2100b514 100644 --- a/src/main.rs +++ b/src/main.rs @@ -46,6 +46,7 @@ use std::sync::Arc; use crate::http_server::endpoints::users::get_profile::get_profile_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handler; +use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -195,6 +196,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::post().to(infer_tts_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/list") + .route(web::get().to(list_tts_models_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service( web::scope("/w2l") From eecaa076381dc8089368783ddba78862b60ec6d6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 May 2021 19:23:00 -0400 Subject: [PATCH 0165/1042] fix/rename templates -> models --- .../endpoints/tts/list_tts_models.rs | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/http_server/endpoints/tts/list_tts_models.rs b/src/http_server/endpoints/tts/list_tts_models.rs index 338b573f29..65f61fd18e 100644 --- a/src/http_server/endpoints/tts/list_tts_models.rs +++ b/src/http_server/endpoints/tts/list_tts_models.rs @@ -41,7 +41,7 @@ pub struct TtsModelRecordForResponse { #[derive(Serialize)] pub struct ListTtsModelsSuccessResponse { pub success: bool, - pub templates: Vec, + pub models: Vec, } #[derive(Serialize)] @@ -102,7 +102,7 @@ pub async fn list_tts_models_handler( { // NB: Lookup failure is Err(RowNotFound). // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_templates = sqlx::query_as!( + let maybe_models = sqlx::query_as!( TtsModelRecord, r#" SELECT @@ -124,40 +124,40 @@ WHERE tts.deleted_at IS NULL .fetch_all(&server_state.mysql_pool) .await; // TODO: This will return error if it doesn't exist - let templates : Vec = match maybe_templates { - Ok(templates) => templates, + let models : Vec = match maybe_models { + Ok(models) => models, Err(err) => { match err { RowNotFound => { return Err(ListTtsModelsError::ServerError); }, _ => { - warn!("w2l template list query error: {:?}", err); + warn!("tts models list query error: {:?}", err); return Err(ListTtsModelsError::ServerError); } } } }; - let templates_for_response = templates.into_iter() - .map(|template| { + let models_for_response = models.into_iter() + .map(|model| { TtsModelRecordForResponse { - model_token: template.model_token.clone(), - tts_model_type: template.tts_model_type.clone(), - creator_user_token: template.creator_user_token.clone(), - creator_username: template.creator_username.clone(), - creator_display_name: template.creator_display_name.clone(), - updatable_slug: template.updatable_slug.clone(), - title: template.title.clone(), - created_at: template.created_at.clone(), - updated_at: template.updated_at.clone(), + model_token: model.model_token.clone(), + tts_model_type: model.tts_model_type.clone(), + creator_user_token: model.creator_user_token.clone(), + creator_username: model.creator_username.clone(), + creator_display_name: model.creator_display_name.clone(), + updatable_slug: model.updatable_slug.clone(), + title: model.title.clone(), + created_at: model.created_at.clone(), + updated_at: model.updated_at.clone(), } }) .collect::>(); let response = ListTtsModelsSuccessResponse { success: true, - templates: templates_for_response, + models: models_for_response, }; let body = serde_json::to_string(&response) From 074b985222de427bbcee57daf2a7cf9dca93cfbd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 01:00:36 -0400 Subject: [PATCH 0166/1042] wip create w2l inference job --- Cargo.lock | 39 ++ Cargo.toml | 2 + .../up.sql | 2 +- .../2021-05-15-212913_inference_jobs/up.sql | 20 + .../w2l/enqueue_infer_w2l_with_uploads.rs | 347 ++++++++++++++++++ src/http_server/endpoints/w2l/mod.rs | 1 + src/http_server/web_utils/mod.rs | 1 + .../web_utils/read_multipart_field_bytes.rs | 66 ++++ src/util/mod.rs | 4 +- src/util/random_prefix_crockford_token.rs | 77 ++++ src/util/random_uuid.rs | 9 + 11 files changed, 566 insertions(+), 2 deletions(-) create mode 100644 src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs create mode 100644 src/http_server/web_utils/read_multipart_field_bytes.rs create mode 100644 src/util/random_prefix_crockford_token.rs create mode 100644 src/util/random_uuid.rs diff --git a/Cargo.lock b/Cargo.lock index 3b69872523..32a217cb85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,6 +138,24 @@ dependencies = [ "syn", ] +[[package]] +name = "actix-multipart" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774bfeb11b54bf9c857a005b8ab893293da4eaff79261a66a9200dab7f5ab6e3" +dependencies = [ + "actix-service", + "actix-utils 2.0.0", + "actix-web", + "bytes 0.5.6", + "derive_more", + "futures-util", + "httparse", + "log", + "mime", + "twoway", +] + [[package]] name = "actix-router" version = "0.2.7" @@ -2967,6 +2985,7 @@ dependencies = [ "actix-cors", "actix-files", "actix-http", + "actix-multipart", "actix-rt 2.2.0", "actix-utils 3.0.0", "actix-web", @@ -3001,6 +3020,7 @@ dependencies = [ "time 0.2.26", "tokio 0.2.25", "toml", + "uuid", ] [[package]] @@ -3359,6 +3379,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "twoway" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57ffb460d7c24cd6eda43694110189030a3d1dfe418416d9468fd1c1d290b47" +dependencies = [ + "memchr", + "unchecked-index", +] + [[package]] name = "typenum" version = "1.13.0" @@ -3371,6 +3401,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" +[[package]] +name = "unchecked-index" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" + [[package]] name = "unicase" version = "2.6.0" @@ -3450,6 +3486,9 @@ name = "uuid" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.2", +] [[package]] name = "v_escape" diff --git a/Cargo.toml b/Cargo.toml index b303005d4f..1ff3ce6007 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ log = "0.4.14" rand = "0.8.3" regex = "1.5.4" time = "0.2.26" +uuid = { version = "0.8.2", features = ["v4"] } # Runtime # NB(1): We switched to Tokio from `async-std` for broader ecosystem support. @@ -79,6 +80,7 @@ tempdir = "0.3.7" actix-cors = "0.5.4" actix-files= "0.5" actix-http = "2.2.0" +actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade actix-rt = "2.2.0" actix-utils = "3.0.0" actix-web = "3.3.2" diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index 63da573ff0..1224bdb9ec 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -104,7 +104,7 @@ CREATE TABLE w2l_template_upload_jobs ( -- Idempotency token uuid_idempotency_token VARCHAR(36) NOT NULL, - -- Foreign key to user + -- Foreign key to user creator_user_token VARCHAR(32) NOT NULL, -- For abuse tracking. diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index 5cf19f515a..6e5abdea93 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -3,6 +3,12 @@ CREATE TABLE tts_inference_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- Idempotency token from client + uuid_idempotency_token VARCHAR(36) NOT NULL, + -- ========== INFERENCE DETAILS ========== -- The model to use. @@ -55,6 +61,8 @@ CREATE TABLE tts_inference_jobs ( -- INDICES -- PRIMARY KEY (id), + UNIQUE KEY (token), + UNIQUE KEY (uuid_idempotency_token), KEY fk_model_token (model_token), KEY fk_maybe_creator_user_token (maybe_creator_user_token), KEY index_status (status), @@ -65,6 +73,12 @@ CREATE TABLE tts_inference_jobs ( CREATE TABLE w2l_inference_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- Idempotency token from client + uuid_idempotency_token VARCHAR(36) NOT NULL, + -- ========== INFERENCE DETAILS ========== -- The W2L template to use @@ -100,6 +114,10 @@ CREATE TABLE w2l_inference_jobs ( 'private' ) NOT NULL DEFAULT 'public', + -- Flags to disable branding + disable_end_bump BOOL NOT NULL DEFAULT false, + disable_watermark BOOL NOT NULL DEFAULT false, + -- ========== JOB SYSTEM DETAILS ========== -- Jobs begin as "pending", then transition to other states. @@ -124,6 +142,8 @@ CREATE TABLE w2l_inference_jobs ( -- INDICES -- PRIMARY KEY (id), + UNIQUE KEY (token), + UNIQUE KEY (uuid_idempotency_token), KEY fk_maybe_w2l_template_token (maybe_w2l_template_token), KEY fk_maybe_tts_inference_result_token (maybe_tts_inference_result_token), KEY fk_maybe_creator_user_token (maybe_creator_user_token), diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs new file mode 100644 index 0000000000..23d44a2dfb --- /dev/null +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -0,0 +1,347 @@ +//use bytes::{Bytes, BytesMut}; +use actix_http::http::header; +use actix_multipart::{Multipart, Field}; +use actix_web::body::ResponseBody::Body; +use actix_web::dev::HttpResponseBuilder; +use actix_web::http::StatusCode; +use actix_web::http::header::ContentDisposition; +use actix_web::web::{Data, Json, BytesMut}; +use actix_web::{middleware, web, App, Error, HttpResponse, HttpServer, HttpRequest, Either, ResponseError}; +use anyhow::anyhow; +use crate::buckets::bucket_client::BucketClient; +use crate::buckets::bucket_paths::hash_to_bucket_path; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::read_multipart_field_bytes::checked_read_multipart_bytes; +use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_boolean; +use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_text; +use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_bytes; +use crate::server_state::ServerState; +use crate::util::anyhow_result::AnyhowResult; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::util::random_token::random_token; +use crate::util::random_uuid::generate_random_uuid; +use derive_more::{Display, Error}; +use futures::{StreamExt, TryStreamExt}; +use log::{warn, info}; +use sqlx::MySqlPool; +use sqlx::error::Error::Database; +use std::io::Write; +use std::sync::Arc; +use uuid::Uuid; + +const BUCKET_AUDIO_FILE_NAME : &'static str = "input_audio_file"; +const BUCKET_IMAGE_FILE_NAME: &'static str = "input_image_file"; +const BUCKET_VIDEO_FILE_NAME : &'static str = "input_video_file"; + +const MIN_BYTES : usize = 10; +const MAX_BYTES : usize = 1024 * 1024 * 20; + + + +/// Just to query for existence +#[derive(Serialize)] +pub struct W2lTemplateExistenceRecord { + pub template_token: String, +} + +#[derive(Serialize)] +pub struct InferW2lWithUploadSuccessResponse { + pub success: bool, + pub job_token: String, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum InferW2lWithUploadError { + BadInput(String), + EmptyFileUploaded, + ServerError, +} + +impl ResponseError for InferW2lWithUploadError { + fn status_code(&self) -> StatusCode { + match *self { + InferW2lWithUploadError::BadInput(_) => StatusCode::BAD_REQUEST, + InferW2lWithUploadError::EmptyFileUploaded => StatusCode::BAD_REQUEST, + InferW2lWithUploadError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + InferW2lWithUploadError::BadInput(reason) => reason.to_string(), + InferW2lWithUploadError::EmptyFileUploaded => "empty file uploaded".to_string(), + InferW2lWithUploadError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +/// This handles audio uploads w/ W2L templates. +pub async fn enqueue_infer_w2l_with_uploads( + http_request: HttpRequest, + server_state: web::Data>, + mut payload: Multipart) + //-> Result, HttpResponse> + -> Result +{ + // ==================== TODO: READ IP AND RATE LIMIT ==================== // + + let ip_address = get_request_ip(&http_request); + + // TODO: Check IP address holds + // TODO: Rate Limiting! + /*let mut hold_exists = handler_state.redis_client.get_ip_hold(&ip_address) + .map_err(|err| HttpResponse::InternalServerError() + .body(format!("Error: {:?}", err)))?; + + match ip_address.as_ref() { + "127.0.0.1" => { + warn!("Allowing IP address hold bypass for {}", &ip_address); + hold_exists = false + }, + _ => {}, + } + + if hold_exists { + return Err(HttpResponse::TooManyRequests().body("you already have pending requests").into()) + }*/ + + // ==================== READ MULTIPART REQUEST ==================== // + + let mut maybe_uuid_idempotency_token: Option = None; + let mut maybe_template_token: Option = None; + let mut maybe_audio_file_name : Option = None; + + while let Ok(Some(mut field)) = payload.try_next().await { + let content_disposition: ContentDisposition = field.content_disposition().unwrap(); + let field_name = content_disposition.get_name().unwrap_or(""); + let filename = content_disposition.get_filename().unwrap_or(""); + + match field_name { + "uuid_idempotency_token" => { + // Form text field. + maybe_uuid_idempotency_token = read_multipart_field_as_text(&mut field).await + .map_err(|_| InferW2lWithUploadError::ServerError)?; + }, + "template_token" => { + // Form text field. + maybe_template_token = read_multipart_field_as_text(&mut field).await + .map_err(|_| InferW2lWithUploadError::ServerError)?; + }, + "audio" => { + // Form binary data. + maybe_audio_file_name = Some(filename.to_string()); + + let maybe_bytes = checked_read_multipart_bytes(&mut field).await + .map_err(|_| InferW2lWithUploadError::ServerError)?; + + let bytes = match maybe_bytes { + Some(bytes) => bytes, + None => { + return Err(InferW2lWithUploadError::EmptyFileUploaded); // Nothing was uploaded! + }, + }; + }, + _ => continue, + } + + info!("Saved file: {}.", &filename); + } + + // ==================== CHECK REQUEST ==================== // + + let template_token = match &maybe_template_token { + Some(ref token) => token.to_string(), + None => { + return Err(InferW2lWithUploadError::BadInput("No template selected".to_string())); + } + }; + + let uuid_idempotency_token = match maybe_uuid_idempotency_token { + Some(token) => token, + None => { + return Err(InferW2lWithUploadError::BadInput("No uuid idempotency token".to_string())); + } + }; + + let exists = check_template_exists(&template_token, &server_state.mysql_pool).await + .map_err(|e| { + warn!("error checking tmpl existence : {:?}", e); + InferW2lWithUploadError::ServerError + })?; + + if !exists { + return Err(InferW2lWithUploadError::BadInput("Template does not exist".to_string())); + } + + // ==================== READ SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + InferW2lWithUploadError::ServerError + })?; + + let mut maybe_user_token : Option = maybe_user_session + .as_ref() + .map(|user_session| user_session.user_token.to_string()); + + + // ==================== TODO: UPLOAD AUDIO FILE ==================== // + + let upload_uuid = generate_random_uuid(); + + let audio_upload_bucket_path = hash_to_bucket_path( + &upload_uuid, + Some("/audio_uploads") // TODO: pull from configs + ).map_err(|e| { + warn!("Hash bucket path error: {:?}", e); + InferW2lWithUploadError::ServerError + })?; + + // TODO: Upload audio to bucket! + + // ==================== SAVE JOB RECORD ==================== // + + // This token is returned to the client. + let job_token = random_prefix_crockford_token("TIJ:", 32) + .map_err(|e| { + warn!("Error creating token"); + InferW2lWithUploadError::ServerError + })?; + + info!("Creating w2l inference job record..."); + + let query_result = sqlx::query!( + r#" +INSERT INTO w2l_inference_jobs +SET + token = ?, + uuid_idempotency_token = ?, + + maybe_w2l_template_token = ?, + maybe_audio_bucket_location = ?, + + maybe_creator_user_token = ?, + creator_ip_address = ?, + disable_end_bump = false, + creator_set_visibility = "public", + status = "pending" + "#, + job_token.to_string(), + uuid_idempotency_token.to_string(), + maybe_template_token.clone(), + Some(audio_upload_bucket_path.clone()), // TODO THIS ISN'T WIDE ENOUGH + maybe_user_token.clone(), + ip_address.to_string() + ) + .execute(&server_state.mysql_pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + warn!("New w2l template upload creation DB error: {:?}", err); + + // NB: SQLSTATE[23000]: Integrity constraint violation + // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access) + match err { + Database(err) => { + let maybe_code = err.code().map(|c| c.into_owned()); + /*match maybe_code.as_deref() { + Some("23000") => { + if err.message().contains("username") { + return Err(UsernameTaken); + } else if err.message().contains("email_address") { + return Err(EmailTaken); + } + } + _ => {}, + }*/ + }, + _ => {}, + } + return Err(InferW2lWithUploadError::ServerError); + } + }; + + // TODO: IP Address holds. + //handler_state.redis_client.set_ip_hold(&ip_address) + // .map_err(|err| HttpResponse::InternalServerError() + // .body(format!("Redis Err: {:?}", err)))?; + + let response = InferW2lWithUploadSuccessResponse { + success: true, + job_token: job_token.to_string(), + }; + + let body = serde_json::to_string(&response) + .map_err(|e| InferW2lWithUploadError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} + +async fn check_template_exists(template_token: &str, mysql_pool: &MySqlPool) -> AnyhowResult +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_template = sqlx::query_as!( + W2lTemplateExistenceRecord, + r#" +SELECT + token as template_token +FROM w2l_templates +WHERE token = ? +AND deleted_at IS NULL + "#, + &template_token + ) + .fetch_one(mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let record_exists = match maybe_template { + Ok(record) => { + true + }, + Err(err) => { + match err { + RowNotFound => { + false + }, + _ => { + warn!("Infer w2l query error: {:?}", err); + return Err(anyhow!("infer w2l query error: {:?}", err)); + } + } + } + }; + + Ok(record_exists) +} + diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index 7e1ee59b46..eb1514f0b4 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -1,4 +1,5 @@ pub mod enqueue_infer_w2l; +pub mod enqueue_infer_w2l_with_uploads; pub mod enqueue_upload_w2l_template; pub mod get_w2l_template; pub mod list_w2l_templates; \ No newline at end of file diff --git a/src/http_server/web_utils/mod.rs b/src/http_server/web_utils/mod.rs index 845d875119..53c5f9f362 100644 --- a/src/http_server/web_utils/mod.rs +++ b/src/http_server/web_utils/mod.rs @@ -1,3 +1,4 @@ pub mod cookie_manager; pub mod ip_address; +pub mod read_multipart_field_bytes; pub mod session_checker; \ No newline at end of file diff --git a/src/http_server/web_utils/read_multipart_field_bytes.rs b/src/http_server/web_utils/read_multipart_field_bytes.rs new file mode 100644 index 0000000000..57fa2fd81f --- /dev/null +++ b/src/http_server/web_utils/read_multipart_field_bytes.rs @@ -0,0 +1,66 @@ +//use bytes::BytesMut; +use actix_multipart::Field; +use actix_web::web::BytesMut; +use anyhow::anyhow; +use futures::{StreamExt, TryStreamExt}; +use log::warn; + +const MIN_BYTES : usize = 10; +const MAX_BYTES : usize = 1024 * 1024 * 20; + +/// Read any field from a multipart form as bytes, but do some error checking. +pub async fn checked_read_multipart_bytes(field: &mut Field) -> anyhow::Result> { + let bytes = read_multipart_field_bytes(field).await?; + + if bytes.is_empty() { + // This field is empty + return Ok(None); + } + + if bytes.len() < MIN_BYTES { + warn!("Bytes of upload too few: {}", bytes.len()); + return Err(anyhow!("Too few bytes")); + } + + if bytes.len() > MAX_BYTES { + warn!("Bytes of upload too much: {}", bytes.len()); + return Err(anyhow!("Too many bytes")); + } + + Ok(Some(bytes)) +} + +/// Read any field from a multipart form as bytes. +pub async fn read_multipart_field_bytes(field: &mut Field) -> anyhow::Result { + let mut bytes = BytesMut::new(); + while let Some(chunk) = field.next().await { + let chunk_data = chunk.map_err(|err| anyhow!("Error: {:?}", err))?; + bytes.extend_from_slice(&chunk_data); + } + Ok(bytes) +} + +/// This reads inputs, text boxes, etc. because form-multipart is hard. +pub async fn read_multipart_field_as_text(field: &mut Field) -> anyhow::Result> { + let bytes = read_multipart_field_bytes(field).await?; + let value = String::from_utf8(bytes.to_vec())?; + + if value.is_empty() || value == "none" { + return Ok(None); + } + + Ok(Some(value)) +} + +/// Read a html form boolean (eg. input type=checkbox); assumes default is false. +pub async fn read_multipart_field_as_boolean(field: &mut Field) -> anyhow::Result { + let result = read_multipart_field_as_text(field).await?; + Ok(match result.as_ref().map(String::as_str) { + Some("true") => true, + Some("TRUE") => true, + Some("false") => false, + Some("FALSE") => false, + None => false, + _ => true, + }) +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 99cdea2fa8..2104d3d1ba 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,4 +1,6 @@ pub mod anyhow_result; +pub mod email_to_gravatar; pub mod filesystem; +pub mod random_prefix_crockford_token; pub mod random_token; -pub mod email_to_gravatar; \ No newline at end of file +pub mod random_uuid; \ No newline at end of file diff --git a/src/util/random_prefix_crockford_token.rs b/src/util/random_prefix_crockford_token.rs new file mode 100644 index 0000000000..cbb134b1e5 --- /dev/null +++ b/src/util/random_prefix_crockford_token.rs @@ -0,0 +1,77 @@ +use rand::Rng; +use anyhow::anyhow; +use crate::util::anyhow_result::AnyhowResult; + +// Crockford characters +const CROCKFORD_CHARSET: &[u8] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + +// Random part can't be any less than this. +const MIN_ENTROPY_LENGTH : usize = 8; + +pub fn random_prefix_crockford_token(prefix: &str, length: usize) -> AnyhowResult { + let mut prefix = prefix.trim().to_string(); + + if prefix.is_empty() { + return Err(anyhow!("prefix not set; use other crockford functions")); + } + + if !prefix.ends_with(":") { + prefix = format!("{}:", prefix); + } + + let entropy_length : usize = length.saturating_sub(prefix.len()); + + if entropy_length < MIN_ENTROPY_LENGTH { + return Err(anyhow!("not enough entropy in token")); + } + + let mut rng = rand::thread_rng(); + + let entropy_part: String = (0..entropy_length) + .map(|_| { + let idx = rng.gen_range(0..CROCKFORD_CHARSET.len()); + CROCKFORD_CHARSET[idx] as char + }) + .collect(); + + let token = format!("{}{}", prefix, entropy_part); + Ok(token) +} + +#[cfg(test)] +mod tests { + use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; + + #[test] + fn no_prefix_error() { + assert!(random_prefix_crockford_token("", 0).is_err()); + assert!(random_prefix_crockford_token("", 10).is_err()); + assert!(random_prefix_crockford_token("", 32).is_err()); + } + + #[test] + fn not_enough_entropy_error() { + assert!(random_prefix_crockford_token("foo", 0).is_err()); + assert!(random_prefix_crockford_token("foo", 5).is_err()); + assert!(random_prefix_crockford_token("foo", 11).is_err()); + assert!(random_prefix_crockford_token("foo:", 11).is_err()); + } + + #[test] + fn starts_with() { + assert_eq!(random_prefix_crockford_token("w", 10).unwrap().starts_with("w:")); + assert_eq!(random_prefix_crockford_token("w", 10).unwrap().starts_with("w:")); + assert_eq!(random_prefix_crockford_token("prefix", 16).unwrap().starts_with("prefix:")); + assert_eq!(random_prefix_crockford_token("prefix:", 16).unwrap().starts_with("prefix:")); + } + + #[test] + fn length() { + assert_eq!(random_prefix_crockford_token("w", 10).len(), 10); + assert_eq!(random_prefix_crockford_token("w:", 10).len(), 10); + assert_eq!(random_prefix_crockford_token("prefix", 16).len(), 16); + assert_eq!(random_prefix_crockford_token("prefix:", 16).len(), 16); + assert_eq!(random_prefix_crockford_token("asdf:", 32).len(), 32); + assert_eq!(random_prefix_crockford_token("asdf", 32).len(), 32); + } +} diff --git a/src/util/random_uuid.rs b/src/util/random_uuid.rs new file mode 100644 index 0000000000..806fa27145 --- /dev/null +++ b/src/util/random_uuid.rs @@ -0,0 +1,9 @@ +use uuid::Uuid; + +pub fn generate_random_uuid() -> String { + let uuid = Uuid::new_v4(); + let mut buffer = Uuid::encode_buffer(); + let hyphenated_uuid = uuid.to_hyphenated() + .encode_lower(&mut buffer); + hyphenated_uuid.to_string() +} From 5c5039c99921208dbd54d10e371fbd98195d6f8e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 01:26:59 -0400 Subject: [PATCH 0167/1042] successful inference job creation --- .../w2l/enqueue_infer_w2l_with_uploads.rs | 36 +++++++++++++++---- src/main.rs | 3 +- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 23d44a2dfb..e32882c451 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -127,36 +127,56 @@ pub async fn enqueue_infer_w2l_with_uploads( // ==================== READ MULTIPART REQUEST ==================== // + info!("Reading multipart request..."); + let mut maybe_uuid_idempotency_token: Option = None; let mut maybe_template_token: Option = None; let mut maybe_audio_file_name : Option = None; while let Ok(Some(mut field)) = payload.try_next().await { - let content_disposition: ContentDisposition = field.content_disposition().unwrap(); - let field_name = content_disposition.get_name().unwrap_or(""); - let filename = content_disposition.get_filename().unwrap_or(""); + let mut field_name = "".to_string(); + let mut filename = "".to_string(); - match field_name { + if let Some(content_disposition) = field.content_disposition() { + field_name = content_disposition.get_name() + .map(|s| s.to_string()) + .unwrap_or("".to_string()); + filename = content_disposition.get_filename() + .map(|s| s.to_string()) + .unwrap_or("".to_string()); + } + + match field_name.as_ref() { "uuid_idempotency_token" => { // Form text field. maybe_uuid_idempotency_token = read_multipart_field_as_text(&mut field).await - .map_err(|_| InferW2lWithUploadError::ServerError)?; + .map_err(|e| { + warn!("Error reading idempotency token: {:}", e); + InferW2lWithUploadError::ServerError + })?; }, "template_token" => { // Form text field. maybe_template_token = read_multipart_field_as_text(&mut field).await - .map_err(|_| InferW2lWithUploadError::ServerError)?; + .map_err(|e| { + warn!("Error reading template token: {:}", e); + InferW2lWithUploadError::ServerError + })?; }, "audio" => { // Form binary data. maybe_audio_file_name = Some(filename.to_string()); let maybe_bytes = checked_read_multipart_bytes(&mut field).await - .map_err(|_| InferW2lWithUploadError::ServerError)?; + .map_err(|e| { + warn!("Error reading audio upload: {:}", e); + InferW2lWithUploadError::ServerError + })?; let bytes = match maybe_bytes { Some(bytes) => bytes, None => { + warn!("Empty file uploaded"); return Err(InferW2lWithUploadError::EmptyFileUploaded); // Nothing was uploaded! }, }; @@ -221,6 +241,8 @@ pub async fn enqueue_infer_w2l_with_uploads( InferW2lWithUploadError::ServerError })?; + let audio_upload_bucket_path = "todo".to_string(); // TODO TODO + // TODO: Upload audio to bucket! // ==================== SAVE JOB RECORD ==================== // diff --git a/src/main.rs b/src/main.rs index ca2100b514..628fea5b7c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,6 +47,7 @@ use crate::http_server::endpoints::users::get_profile::get_profile_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; +use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -211,7 +212,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> ) .service( web::resource("/inference") - .route(web::post().to(infer_w2l_handler)) + .route(web::post().to(enqueue_infer_w2l_with_uploads)) .route(web::head().to(|| HttpResponse::Ok())) ) .service( From 3eea12e45bfcdd999bf3ceaec9291e3b05476826 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 01:36:54 -0400 Subject: [PATCH 0168/1042] fix bucket length limitation --- migrations/2021-05-15-212913_inference_jobs/up.sql | 4 ++-- src/http_server/endpoints/w2l/enqueue_infer_w2l.rs | 4 ++-- .../endpoints/w2l/enqueue_infer_w2l_with_uploads.rs | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index 6e5abdea93..dc1afac1ad 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -90,10 +90,10 @@ CREATE TABLE w2l_inference_jobs ( maybe_tts_inference_result_token VARCHAR(32) DEFAULT NULL, -- If we're using custom uploaded audio, this will be present. - maybe_audio_bucket_location CHAR(16) DEFAULT NULL, + maybe_public_audio_bucket_location VARCHAR(255) DEFAULT NULL, -- If we're using a custom uploaded image, this will be present. - maybe_image_bucket_location CHAR(16) DEFAULT NULL, + maybe_public_image_bucket_location VARCHAR(255) DEFAULT NULL, -- ========== CREATOR DETAILS ========== diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index 58fbf961be..6d11f757dd 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -125,8 +125,8 @@ INSERT INTO w2l_inference_jobs SET maybe_w2l_template_token = ?, maybe_tts_inference_result_token = ?, - maybe_audio_bucket_location = NULL, - maybe_image_bucket_location = NULL, + maybe_public_audio_bucket_location = NULL, + maybe_public_image_bucket_location = NULL, maybe_creator_user_token = ?, creator_ip_address = ?, creator_set_visibility = ?, diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index e32882c451..2f688ac1d6 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -241,8 +241,6 @@ pub async fn enqueue_infer_w2l_with_uploads( InferW2lWithUploadError::ServerError })?; - let audio_upload_bucket_path = "todo".to_string(); // TODO TODO - // TODO: Upload audio to bucket! // ==================== SAVE JOB RECORD ==================== // @@ -264,7 +262,7 @@ SET uuid_idempotency_token = ?, maybe_w2l_template_token = ?, - maybe_audio_bucket_location = ?, + maybe_public_audio_bucket_location = ?, maybe_creator_user_token = ?, creator_ip_address = ?, @@ -275,7 +273,7 @@ SET job_token.to_string(), uuid_idempotency_token.to_string(), maybe_template_token.clone(), - Some(audio_upload_bucket_path.clone()), // TODO THIS ISN'T WIDE ENOUGH + Some(audio_upload_bucket_path.clone()), maybe_user_token.clone(), ip_address.to_string() ) From 79be37a60ce6e75d2c320b36f1fd647a0fe8ef8d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 02:10:30 -0400 Subject: [PATCH 0169/1042] upload audio to bucket --- .env | 3 ++ .../w2l/enqueue_infer_w2l_with_uploads.rs | 14 ++++-- src/main.rs | 48 +++++++++++++++++++ src/server_state.rs | 7 +++ src/w2l_download_job.rs | 2 + 5 files changed, 71 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 91b73890b7..86389b51fb 100644 --- a/.env +++ b/.env @@ -17,6 +17,9 @@ TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models W2L_PRIVATE_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public + +# Bucket roots +AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 2f688ac1d6..df185ada83 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -132,6 +132,7 @@ pub async fn enqueue_infer_w2l_with_uploads( let mut maybe_uuid_idempotency_token: Option = None; let mut maybe_template_token: Option = None; let mut maybe_audio_file_name : Option = None; + let mut audio_bytes = BytesMut::with_capacity(0); while let Ok(Some(mut field)) = payload.try_next().await { let mut field_name = "".to_string(); @@ -173,7 +174,7 @@ pub async fn enqueue_infer_w2l_with_uploads( InferW2lWithUploadError::ServerError })?; - let bytes = match maybe_bytes { + audio_bytes = match maybe_bytes { Some(bytes) => bytes, None => { warn!("Empty file uploaded"); @@ -235,13 +236,20 @@ pub async fn enqueue_infer_w2l_with_uploads( let audio_upload_bucket_path = hash_to_bucket_path( &upload_uuid, - Some("/audio_uploads") // TODO: pull from configs + Some(&server_state.audio_uploads_bucket_root) ).map_err(|e| { warn!("Hash bucket path error: {:?}", e); InferW2lWithUploadError::ServerError })?; - // TODO: Upload audio to bucket! + info!("Uploading audio to bucket..."); + server_state.private_bucket_client.upload_file( + &audio_upload_bucket_path, + audio_bytes.as_ref()).await + .map_err(|e| { + warn!("Upload audio bytes to bucket error: {:?}", e); + InferW2lWithUploadError::ServerError + })?; // ==================== SAVE JOB RECORD ==================== // diff --git a/src/main.rs b/src/main.rs index 628fea5b7c..c800d58f00 100644 --- a/src/main.rs +++ b/src/main.rs @@ -48,18 +48,36 @@ use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_h use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; +use crate::buckets::bucket_client::BucketClient; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; // NB: sqlx::query is spammy and logs all queries as "info"-level const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; +// Buckets (shared config) +const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; +const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; +const ENV_REGION_NAME : &'static str = "REGION_NAME"; + +// Buckets (private data) +const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; +// Buckets (public data) +const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; + +// Various bucket roots +const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &'static str = "AUDIO_UPLOADS_BUCKET_ROOT"; + pub type AnyhowResult = anyhow::Result; #[actix_web::main] async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + // NB: Do not check this secrets-containing dotenv file into VCS. + // This file should only contain *development* secrets, never production. + let _ = dotenv::from_filename(".env-secrets").ok(); + info!("Obtaining hostname..."); let server_hostname = hostname::get() @@ -93,6 +111,33 @@ async fn main() -> AnyhowResult<()> { let cookie_manager = CookieManager::new(&cookie_domain, &hmac_secret); let session_checker = SessionChecker::new(&cookie_manager); + let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; + let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; + let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; + + // Private and Public Buckets + let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?; + let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?; + + // Bucket roots + let audio_uploads_bucket_root= easyenv::get_env_string_required(ENV_AUDIO_UPLOADS_BUCKET_ROOT)?; + + let private_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &private_bucket_name, + None, + )?; + + let public_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &public_bucket_name, + None, + )?; + let server_state = ServerState { env_config: EnvConfig { num_workers, @@ -105,6 +150,9 @@ async fn main() -> AnyhowResult<()> { mysql_pool: pool, cookie_manager, session_checker, + private_bucket_client, + public_bucket_client, + audio_uploads_bucket_root, }; serve(server_state) diff --git a/src/server_state.rs b/src/server_state.rs index 0a8f4f01c4..01824d453e 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,6 +1,7 @@ use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; use sqlx::MySqlPool; +use crate::buckets::bucket_client::BucketClient; /// State that is injected into every endpoint. #[derive(Clone)] @@ -15,6 +16,12 @@ pub struct ServerState { pub cookie_manager: CookieManager, pub session_checker: SessionChecker, + + pub private_bucket_client: BucketClient, + pub public_bucket_client: BucketClient, + + /// Where to store audio uploads for w2l + pub audio_uploads_bucket_root: String, } #[derive(Clone)] diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index c9b899e992..680b6942a6 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -87,6 +87,8 @@ struct Downloader { async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + // NB: Do not check this secrets-containing dotenv file into VCS. + // This file should only contain *development* secrets, never production. let _ = dotenv::from_filename(".env-secrets").ok(); info!("Obtaining hostname..."); From abfd1a69e572150c4728a0bf52bc99897677594e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 02:16:50 -0400 Subject: [PATCH 0170/1042] rename crockford functions, fix tests, change prefix --- src/common_queries/sessions.rs | 4 ++-- .../endpoints/tts/enqueue_infer_tts.rs | 2 +- .../endpoints/tts/enqueue_upload_tts_model.rs | 2 +- .../endpoints/tts/list_tts_models.rs | 2 +- .../endpoints/users/create_account.rs | 4 ++-- .../endpoints/users/get_profile.rs | 2 +- src/http_server/endpoints/users/login.rs | 2 +- src/http_server/endpoints/users/logout.rs | 2 +- .../endpoints/users/session_info.rs | 2 +- .../endpoints/w2l/enqueue_infer_w2l.rs | 2 +- .../w2l/enqueue_infer_w2l_with_uploads.rs | 4 ++-- .../w2l/enqueue_upload_w2l_template.rs | 2 +- .../endpoints/w2l/get_w2l_template.rs | 2 +- .../endpoints/w2l/list_w2l_templates.rs | 2 +- src/job_queries/tts_download_job_queries.rs | 4 ++-- src/job_queries/w2l_download_job_queries.rs | 4 ++-- .../google_drive_download_command.rs | 4 ++-- src/tts_download_job.rs | 2 +- src/util/mod.rs | 2 +- ...dom_token.rs => random_crockford_token.rs} | 12 +++++------ src/util/random_prefix_crockford_token.rs | 20 +++++++++---------- src/w2l_download_job.rs | 2 +- 22 files changed, 42 insertions(+), 42 deletions(-) rename src/util/{random_token.rs => random_crockford_token.rs} (53%) diff --git a/src/common_queries/sessions.rs b/src/common_queries/sessions.rs index 1c88e372c7..42fc984a71 100644 --- a/src/common_queries/sessions.rs +++ b/src/common_queries/sessions.rs @@ -1,13 +1,13 @@ use anyhow::anyhow; use log::{info, warn}; use crate::AnyhowResult; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use sqlx::MySqlPool; pub async fn create_session_for_user(user_token: &str, ip_address: &str, mysql_pool: &MySqlPool) -> AnyhowResult { - let session_token = random_token(32); + let session_token = random_crockford_token(32); let query_result = sqlx::query!( r#" diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 45f572857a..010f1e0820 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -8,7 +8,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index 7b35d976de..d3fb2b1e0a 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -8,7 +8,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/src/http_server/endpoints/tts/list_tts_models.rs b/src/http_server/endpoints/tts/list_tts_models.rs index 65f61fd18e..4f58eb9874 100644 --- a/src/http_server/endpoints/tts/list_tts_models.rs +++ b/src/http_server/endpoints/tts/list_tts_models.rs @@ -12,7 +12,7 @@ use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index 99ca184609..3fdf226853 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -9,7 +9,7 @@ use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; @@ -109,7 +109,7 @@ pub async fn create_account_handler( return Err(CreateAccountError::BadInput("invalid email address".to_string())); } - let user_token = random_token(15); + let user_token = random_crockford_token(15); let password_hash = match bcrypt::hash(&request.password, bcrypt::DEFAULT_COST) { Ok(hash) => hash, diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index 3d9831279f..05bff72431 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -14,7 +14,7 @@ use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; diff --git a/src/http_server/endpoints/users/login.rs b/src/http_server/endpoints/users/login.rs index c46919975b..07200fa014 100644 --- a/src/http_server/endpoints/users/login.rs +++ b/src/http_server/endpoints/users/login.rs @@ -10,7 +10,7 @@ use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; diff --git a/src/http_server/endpoints/users/logout.rs b/src/http_server/endpoints/users/logout.rs index 4b1b172c88..f66d445653 100644 --- a/src/http_server/endpoints/users/logout.rs +++ b/src/http_server/endpoints/users/logout.rs @@ -11,7 +11,7 @@ use crate::http_server::endpoints::users::create_account::CreateAccountError::{B use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; diff --git a/src/http_server/endpoints/users/session_info.rs b/src/http_server/endpoints/users/session_info.rs index fc8dce545d..7dbb35bb08 100644 --- a/src/http_server/endpoints/users/session_info.rs +++ b/src/http_server/endpoints/users/session_info.rs @@ -12,7 +12,7 @@ use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index 6d11f757dd..fcf046dd2f 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -8,7 +8,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index df185ada83..43e46ecfe0 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -18,7 +18,7 @@ use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_fi use crate::server_state::ServerState; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_uuid::generate_random_uuid; use derive_more::{Display, Error}; use futures::{StreamExt, TryStreamExt}; @@ -254,7 +254,7 @@ pub async fn enqueue_infer_w2l_with_uploads( // ==================== SAVE JOB RECORD ==================== // // This token is returned to the client. - let job_token = random_prefix_crockford_token("TIJ:", 32) + let job_token = random_prefix_crockford_token("W2L_INFER:", 32) .map_err(|e| { warn!("Error creating token"); InferW2lWithUploadError::ServerError diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index b2b39198a2..d017799dd7 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -8,7 +8,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index 8a332cc10b..a80457dd86 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -14,7 +14,7 @@ use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; diff --git a/src/http_server/endpoints/w2l/list_w2l_templates.rs b/src/http_server/endpoints/w2l/list_w2l_templates.rs index 849f96a1fc..754a57e057 100644 --- a/src/http_server/endpoints/w2l/list_w2l_templates.rs +++ b/src/http_server/endpoints/w2l/list_w2l_templates.rs @@ -12,7 +12,7 @@ use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index 78e6fa2203..be2f229573 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -5,7 +5,7 @@ use anyhow::anyhow; use chrono::Utc; use crate::util::anyhow_result::AnyhowResult; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use sqlx::MySqlPool; /// table: tts_model_upload_jobs @@ -115,7 +115,7 @@ pub async fn insert_tts_model(pool: &MySqlPool, private_bucket_object_name: &str) -> AnyhowResult { - let model_token = random_token(32); + let model_token = random_crockford_token(32); let updatable_slug = model_token.clone(); let query_result = sqlx::query!( diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 2a1b1e55e2..d7120d2dc2 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -5,7 +5,7 @@ use anyhow::anyhow; use chrono::Utc; use crate::util::anyhow_result::AnyhowResult; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use sqlx::MySqlPool; /// table: w2l_template_upload_jobs @@ -126,7 +126,7 @@ pub async fn insert_w2l_template(pool: &MySqlPool, duration_millis: u64) -> AnyhowResult { - let model_token = random_token(32); + let model_token = random_crockford_token(32); let updatable_slug = model_token.clone(); let query_result = sqlx::query!( diff --git a/src/script_execution/google_drive_download_command.rs b/src/script_execution/google_drive_download_command.rs index f3ad11abc0..a3dd54f4da 100644 --- a/src/script_execution/google_drive_download_command.rs +++ b/src/script_execution/google_drive_download_command.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use crate::util::anyhow_result::AnyhowResult; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use log::{info, warn}; use std::io::{BufReader, Read}; use std::path::{PathBuf, Path}; @@ -37,7 +37,7 @@ impl GoogleDriveDownloadCommand { .unwrap_or("/tmp") .to_string(); - let temp_filename = random_token(10); + let temp_filename = random_crockford_token(10); let temp_filename = format!("{}/{}.bin", temp_dir_path, temp_filename); info!("Downloading {} to: {}", download_url, temp_filename); diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 17b62a7b3b..512d3273f2 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -23,7 +23,7 @@ use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; diff --git a/src/util/mod.rs b/src/util/mod.rs index 2104d3d1ba..badcaf6032 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -2,5 +2,5 @@ pub mod anyhow_result; pub mod email_to_gravatar; pub mod filesystem; pub mod random_prefix_crockford_token; -pub mod random_token; +pub mod random_crockford_token; pub mod random_uuid; \ No newline at end of file diff --git a/src/util/random_token.rs b/src/util/random_crockford_token.rs similarity index 53% rename from src/util/random_token.rs rename to src/util/random_crockford_token.rs index 9805463674..2b64810467 100644 --- a/src/util/random_token.rs +++ b/src/util/random_crockford_token.rs @@ -3,7 +3,7 @@ use rand::Rng; // Crockford characters const CROCKFORD_CHARSET: &[u8] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; -pub fn random_token(length: usize) -> String { +pub fn random_crockford_token(length: usize) -> String { let mut rng = rand::thread_rng(); let token: String = (0..length) @@ -18,13 +18,13 @@ pub fn random_token(length: usize) -> String { #[cfg(test)] mod tests { - use crate::util::random_token::random_token; + use crate::util::random_crockford_token::random_crockford_token; #[test] fn random_token_length() { - assert_eq!(random_token(0), "".to_string()); - assert_eq!(random_token(1).len(), 1); - assert_eq!(random_token(10).len(), 10); - assert_eq!(random_token(32).len(), 32); + assert_eq!(random_crockford_token(0), "".to_string()); + assert_eq!(random_crockford_token(1).len(), 1); + assert_eq!(random_crockford_token(10).len(), 10); + assert_eq!(random_crockford_token(32).len(), 32); } } diff --git a/src/util/random_prefix_crockford_token.rs b/src/util/random_prefix_crockford_token.rs index cbb134b1e5..9ff0588d26 100644 --- a/src/util/random_prefix_crockford_token.rs +++ b/src/util/random_prefix_crockford_token.rs @@ -59,19 +59,19 @@ mod tests { #[test] fn starts_with() { - assert_eq!(random_prefix_crockford_token("w", 10).unwrap().starts_with("w:")); - assert_eq!(random_prefix_crockford_token("w", 10).unwrap().starts_with("w:")); - assert_eq!(random_prefix_crockford_token("prefix", 16).unwrap().starts_with("prefix:")); - assert_eq!(random_prefix_crockford_token("prefix:", 16).unwrap().starts_with("prefix:")); + assert!(random_prefix_crockford_token("w", 10).unwrap().starts_with("w:")); + assert!(random_prefix_crockford_token("w", 10).unwrap().starts_with("w:")); + assert!(random_prefix_crockford_token("prefix", 16).unwrap().starts_with("prefix:")); + assert!(random_prefix_crockford_token("prefix:", 16).unwrap().starts_with("prefix:")); } #[test] fn length() { - assert_eq!(random_prefix_crockford_token("w", 10).len(), 10); - assert_eq!(random_prefix_crockford_token("w:", 10).len(), 10); - assert_eq!(random_prefix_crockford_token("prefix", 16).len(), 16); - assert_eq!(random_prefix_crockford_token("prefix:", 16).len(), 16); - assert_eq!(random_prefix_crockford_token("asdf:", 32).len(), 32); - assert_eq!(random_prefix_crockford_token("asdf", 32).len(), 32); + assert_eq!(random_prefix_crockford_token("w", 10).unwrap().len(), 10); + assert_eq!(random_prefix_crockford_token("w:", 10).unwrap().len(), 10); + assert_eq!(random_prefix_crockford_token("prefix", 16).unwrap().len(), 16); + assert_eq!(random_prefix_crockford_token("prefix:", 16).unwrap().len(), 16); + assert_eq!(random_prefix_crockford_token("asdf:", 32).unwrap().len(), 32); + assert_eq!(random_prefix_crockford_token("asdf", 32).unwrap().len(), 32); } } diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 680b6942a6..d438772d8f 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -26,7 +26,7 @@ use crate::script_execution::google_drive_download_command::GoogleDriveDownloadC use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; -use crate::util::random_token::random_token; +use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; From 95b5fa8daa8c1cafb1bb2f01638e88f3ea34eb2c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 02:26:47 -0400 Subject: [PATCH 0171/1042] switch to using prefix tokens --- src/common_queries/sessions.rs | 3 ++- src/http_server/endpoints/users/create_account.rs | 7 ++++++- src/job_queries/tts_download_job_queries.rs | 3 ++- src/job_queries/w2l_download_job_queries.rs | 3 ++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/common_queries/sessions.rs b/src/common_queries/sessions.rs index 42fc984a71..ce3acbe725 100644 --- a/src/common_queries/sessions.rs +++ b/src/common_queries/sessions.rs @@ -3,11 +3,12 @@ use log::{info, warn}; use crate::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; use sqlx::MySqlPool; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; pub async fn create_session_for_user(user_token: &str, ip_address: &str, mysql_pool: &MySqlPool) -> AnyhowResult { - let session_token = random_crockford_token(32); + let session_token = random_prefix_crockford_token("SESSION:", 32)?; let query_result = sqlx::query!( r#" diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index 3fdf226853..3209066cb4 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -21,6 +21,7 @@ use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::util::email_to_gravatar::email_to_gravatar; use crate::validations::username_reservations::is_reserved_username; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; const NEW_USER_ROLE: &'static str = "new-user"; @@ -109,7 +110,11 @@ pub async fn create_account_handler( return Err(CreateAccountError::BadInput("invalid email address".to_string())); } - let user_token = random_crockford_token(15); + let user_token = random_prefix_crockford_token("U:", 15) + .map_err(|e| { + warn!("Bad crockford token: {:?}", e); + CreateAccountError::ServerError + })?; let password_hash = match bcrypt::hash(&request.password, bcrypt::DEFAULT_COST) { Ok(hash) => hash, diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index be2f229573..20c25b5127 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -7,6 +7,7 @@ use chrono::Utc; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; use sqlx::MySqlPool; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; /// table: tts_model_upload_jobs #[derive(Debug)] @@ -115,7 +116,7 @@ pub async fn insert_tts_model(pool: &MySqlPool, private_bucket_object_name: &str) -> AnyhowResult { - let model_token = random_crockford_token(32); + let model_token = random_prefix_crockford_token("TTS_MDL:", 32)?; let updatable_slug = model_token.clone(); let query_result = sqlx::query!( diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index d7120d2dc2..033fa40fb5 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -7,6 +7,7 @@ use chrono::Utc; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; use sqlx::MySqlPool; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; /// table: w2l_template_upload_jobs #[derive(Debug)] @@ -126,7 +127,7 @@ pub async fn insert_w2l_template(pool: &MySqlPool, duration_millis: u64) -> AnyhowResult { - let model_token = random_crockford_token(32); + let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?; let updatable_slug = model_token.clone(); let query_result = sqlx::query!( From 226019b09f1e1d3898b48b9991a43d9e85f01c59 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 02:45:05 -0400 Subject: [PATCH 0172/1042] store metadata and mime types --- Cargo.lock | 20 ++++++++++++++++++ Cargo.toml | 1 + .../2021-05-15-212913_inference_jobs/up.sql | 17 ++++++++++++--- .../w2l/enqueue_infer_w2l_with_uploads.rs | 21 +++++++++++++++---- 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 32a217cb85..751c664262 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -814,6 +814,16 @@ version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +[[package]] +name = "cfb" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca453e8624711b2f0f4eb47076a318feda166252a827ee25d067b43de83dcba0" +dependencies = [ + "byteorder", + "uuid", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -1590,6 +1600,15 @@ dependencies = [ "hashbrown 0.9.1", ] +[[package]] +name = "infer" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea70330449622910e0edebab230734569516269fb32342fb0a8956340fa48c6c" +dependencies = [ + "cfb", +] + [[package]] name = "instant" version = "0.1.9" @@ -3002,6 +3021,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", + "infer", "jwt", "lazy_static", "log", diff --git a/Cargo.toml b/Cargo.toml index 1ff3ce6007..4534d19d5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ ring = "0.17.0-alpha.10" #rust-s3 = "0.27.0-rc4" #rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } rust-s3 = { version = "0.26" } +infer = "0.5.0" subprocess = "0.2.6" tempdir = "0.3.7" diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index dc1afac1ad..54dc4ed46d 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -79,21 +79,32 @@ CREATE TABLE w2l_inference_jobs ( -- Idempotency token from client uuid_idempotency_token VARCHAR(36) NOT NULL, - -- ========== INFERENCE DETAILS ========== + -- ========== INFERENCE DETAILS : FACE TEMPLATE ========== -- The W2L template to use -- Can be an image or video. -- This is null if we're using a custom uploaded image. maybe_w2l_template_token VARCHAR(32) DEFAULT NULL, + -- If we're using a custom uploaded image, this will be present. + -- NOTE: This may never be supported. + maybe_public_image_bucket_location VARCHAR(255) DEFAULT NULL, + + -- ========== INFERENCE DETAILS : AUDIO SOURCE ========== + -- If we're using TTS results, this will be present maybe_tts_inference_result_token VARCHAR(32) DEFAULT NULL, -- If we're using custom uploaded audio, this will be present. maybe_public_audio_bucket_location VARCHAR(255) DEFAULT NULL, - -- If we're using a custom uploaded image, this will be present. - maybe_public_image_bucket_location VARCHAR(255) DEFAULT NULL, + -- The filename that was used at upload time (if available) + maybe_original_audio_filename CHAR(255) DEFAULT NULL, + + -- Where the audio file was originally downloaded (if it was downloaded) + maybe_original_audio_download_url VARCHAR(512) DEFAULT NULL, + + maybe_audio_mime_type VARCHAR(32) DEFAULT NULL, -- ========== CREATOR DETAILS ========== diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 43e46ecfe0..341c253d90 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -230,7 +230,13 @@ pub async fn enqueue_infer_w2l_with_uploads( .map(|user_session| user_session.user_token.to_string()); - // ==================== TODO: UPLOAD AUDIO FILE ==================== // + // ==================== ANALYZE AND UPLOAD AUDIO FILE ==================== // + + let mut audio_type = "application/octet-stream".to_string(); + + if let Some(maybe_type) = infer::get(audio_bytes.as_ref()) { + audio_type = maybe_type.mime_type().to_string(); + } let upload_uuid = generate_random_uuid(); @@ -243,9 +249,11 @@ pub async fn enqueue_infer_w2l_with_uploads( })?; info!("Uploading audio to bucket..."); - server_state.private_bucket_client.upload_file( + server_state.private_bucket_client.upload_file_with_content_type( &audio_upload_bucket_path, - audio_bytes.as_ref()).await + audio_bytes.as_ref(), + &audio_type) + .await .map_err(|e| { warn!("Upload audio bytes to bucket error: {:?}", e); InferW2lWithUploadError::ServerError @@ -254,7 +262,7 @@ pub async fn enqueue_infer_w2l_with_uploads( // ==================== SAVE JOB RECORD ==================== // // This token is returned to the client. - let job_token = random_prefix_crockford_token("W2L_INFER:", 32) + let job_token = random_prefix_crockford_token("W2L_INF:", 32) .map_err(|e| { warn!("Error creating token"); InferW2lWithUploadError::ServerError @@ -272,6 +280,9 @@ SET maybe_w2l_template_token = ?, maybe_public_audio_bucket_location = ?, + maybe_original_audio_filename = ?, + maybe_audio_mime_type = ?, + maybe_creator_user_token = ?, creator_ip_address = ?, disable_end_bump = false, @@ -282,6 +293,8 @@ SET uuid_idempotency_token.to_string(), maybe_template_token.clone(), Some(audio_upload_bucket_path.clone()), + maybe_audio_file_name.clone(), + Some(audio_type.clone()), maybe_user_token.clone(), ip_address.to_string() ) From c11ccc36d766c3779dd0ff4ab19dfb04ff00d262 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 20:32:58 -0400 Subject: [PATCH 0173/1042] WIP on w2l inference job (doesn't compile) --- .env | 5 +- src/buckets/bucket_path_unifier.rs | 25 + src/buckets/mod.rs | 1 + src/job_queries/mod.rs | 3 +- src/job_queries/w2l_inference_job_queries.rs | 204 ++++++++ src/main.rs | 6 + src/script_execution/mod.rs | 1 + .../wav2lip_inference_command.rs | 24 +- .../wav2lip_inference_command_OLD.rs | 125 +++++ src/w2l_download_job.rs | 12 +- src/w2l_inference_job.rs | 493 +++++++++++++++++- 11 files changed, 880 insertions(+), 19 deletions(-) create mode 100644 src/buckets/bucket_path_unifier.rs create mode 100644 src/job_queries/w2l_inference_job_queries.rs create mode 100644 src/script_execution/wav2lip_inference_command_OLD.rs diff --git a/.env b/.env index 86389b51fb..78fe012901 100644 --- a/.env +++ b/.env @@ -21,8 +21,11 @@ W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public # Bucket roots AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates +W2L_INFERENCE_OUTPUT_BUCKET_ROOT=/w2l_output W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip #W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/s3fd-619a316812.pth W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/wav2lip_gan.pth -W2L_SCRIPT_NAME=vocodes_process_upload.py \ No newline at end of file + +W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py +W2L_INFERENCE_SCRIPT_NAME=vocodes_process_inference.py diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs new file mode 100644 index 0000000000..e7ce5d28cb --- /dev/null +++ b/src/buckets/bucket_path_unifier.rs @@ -0,0 +1,25 @@ + +/// This is designed to make it centrally configurable where +/// different types of objects are stored. +pub struct BucketPathUnifier { + +} + +impl BucketPathUnifier { + + pub fn user_audio_for_w2l_inference_path(audio_token: &str) -> String { + "".to_string() + } + + pub fn end_bump_video_for_w2l_path(end_bump_filename: &str) -> String { + "".to_string() + } + + pub fn precomputed_faces_for_w2l_path(token: &str) -> String { + "".to_string() + } + + pub fn pretrained_w2l_model_path(model_name: &str) -> String { + "".to_string() + } +} \ No newline at end of file diff --git a/src/buckets/mod.rs b/src/buckets/mod.rs index 477fc55a78..1860d6bdad 100644 --- a/src/buckets/mod.rs +++ b/src/buckets/mod.rs @@ -1,3 +1,4 @@ pub mod bucket_client; +pub mod bucket_path_unifier; pub mod bucket_paths; pub mod file_hashing; diff --git a/src/job_queries/mod.rs b/src/job_queries/mod.rs index de78fe1474..e2b567b9df 100644 --- a/src/job_queries/mod.rs +++ b/src/job_queries/mod.rs @@ -1,2 +1,3 @@ pub mod tts_download_job_queries; -pub mod w2l_download_job_queries; \ No newline at end of file +pub mod w2l_download_job_queries; +pub mod w2l_inference_job_queries; \ No newline at end of file diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs new file mode 100644 index 0000000000..7ee79feb68 --- /dev/null +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -0,0 +1,204 @@ +//! NB: This seems required for sqlx to generate the cached queries. +//! Sqlx's prepare needs a *single* binary to work against, so we need to +//! include these in the main binary to generate all the queries. + +use anyhow::anyhow; +use chrono::Utc; +use crate::util::anyhow_result::AnyhowResult; +use crate::util::random_crockford_token::random_crockford_token; +use sqlx::MySqlPool; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; + +/// table: w2l_template_upload_jobs +#[derive(Debug)] +pub struct W2lInferenceJobRecord { + pub id: i64, + pub inference_job_token: String, + pub uuid_idempotency_token: String, + + // ===== FACE TEMPLATE OPTIONS ===== + pub maybe_w2l_template_token: Option, + pub maybe_public_image_bucket_location: Option, + + // ===== AUDIO SOURCE OPTIONS ===== + pub maybe_tts_inference_result_token: Option, + pub maybe_public_audio_bucket_location: Option, + + pub creator_ip_address: String, + pub maybe_creator_user_token: Option, + pub creator_set_visibility: String, // TODO + + pub disable_end_bump: bool, + pub disable_watermark: bool, + //pub maybe_subject_token: Option, + //pub maybe_actor_subject_token: Option, + pub status: String, // TODO + pub attempt_count: i32, + pub failure_reason: Option, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub retry_at: Option>, +} + +pub async fn query_w2l_inference_job_records(pool: &MySqlPool, num_records: u32) + -> AnyhowResult> +{ + let job_records = sqlx::query_as!( + W2lInferenceJobRecord, + r#" +SELECT + token AS inference_job_token, + * +FROM w2l_inference_jobs +WHERE + ( + status IN ("pending", "attempt_failed") + ) + AND + ( + retry_at IS NULL + OR + retry_at < CURRENT_TIMESTAMP + ) + LIMIT ? + "#, + num_records, + ) + .fetch_all(pool) + .await?; + + Ok(job_records) +} + +pub async fn mark_w2l_inference_job_failure(pool: &MySqlPool, + job: &W2lInferenceJobRecord, + failure_reason: &str) + -> AnyhowResult<()> +{ + // statuses: "attempt_failed", "complete_failure", "dead" + let status = "attempt_failed"; + + let query_result = sqlx::query!( + r#" +UPDATE w2l_inference_jobs +SET + status = ?, + failure_reason = ?, + retry_at = NOW() + interval 2 minute +WHERE id = ? + "#, + status, + failure_reason.to_string(), + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} +pub async fn mark_w2l_inference_job_done(pool: &MySqlPool, + job: &W2lInferenceJobRecord, + success: bool) + -> AnyhowResult<()> +{ + let status = if success { "complete_success" } else { "complete_failure" }; + + let query_result = sqlx::query!( + r#" +UPDATE w2l_inference_jobs +SET + status = ?, + failure_reason = NULL, + retry_at = NULL +WHERE id = ? + "#, + status, + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} + +pub async fn insert_w2l_result(pool: &MySqlPool, + template_type: &str, // TODO: ENUM! + job: &W2lInferenceJobRecord, + private_bucket_hash: &str, + private_bucket_object_name: &str, + private_bucket_cached_faces_object_name: &str, + maybe_image_preview_object_name: Option<&str>, + maybe_video_preview_object_name: Option<&str>, + file_size_bytes: u64, + maybe_mime_type: Option<&str>, + frame_width: u32, + frame_height: u32, + frame_count: u64, + fps: f32, + duration_millis: u64) + -> AnyhowResult +{ + let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?; + let updatable_slug = model_token.clone(); + + let query_result = sqlx::query!( + r#" +INSERT INTO w2l_templates +SET + token = ?, + template_type = ?, + updatable_slug = ?, + title = ?, + description_markdown = '', + description_rendered_html = '', + creator_user_token = ?, + creator_ip_address = ?, + original_download_url = ?, + private_bucket_hash = ?, + private_bucket_object_name = ?, + private_bucket_cached_faces_object_name = ?, + maybe_public_bucket_preview_image_object_name = ?, + maybe_public_bucket_preview_video_object_name = ?, + file_size_bytes = ?, + mime_type = ?, + frame_width = ?, + frame_height = ?, + frame_count = ?, + fps = ?, + duration_millis = ? + "#, + model_token, + template_type, + updatable_slug, + job.title.to_string(), + job.creator_user_token.clone(), + job.creator_ip_address.clone(), + job.download_url.clone(), + private_bucket_hash.to_string(), + private_bucket_object_name.to_string(), + private_bucket_cached_faces_object_name.to_string(), + maybe_image_preview_object_name, + maybe_video_preview_object_name, + file_size_bytes, + maybe_mime_type.unwrap_or(""), + frame_width, + frame_height, + frame_count, + fps, + duration_millis + ) + .execute(pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + Ok(record_id) +} diff --git a/src/main.rs b/src/main.rs index c800d58f00..891991499d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,6 +50,12 @@ use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; use crate::buckets::bucket_client::BucketClient; +/// +/// TODO TODO TODO TODO +/// TODO TODO TODO TODO +/// Make a central class that configures which bucket root paths things live in. +/// + const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; // NB: sqlx::query is spammy and logs all queries as "info"-level diff --git a/src/script_execution/mod.rs b/src/script_execution/mod.rs index fcd02f3909..36879f0f3b 100644 --- a/src/script_execution/mod.rs +++ b/src/script_execution/mod.rs @@ -3,4 +3,5 @@ pub mod ffmpeg_generate_preview_video_command; pub mod google_drive_download_command; pub mod imagemagick_generate_preview_image_command; pub mod wav2lip_inference_command; +pub mod wav2lip_inference_command_OLD; pub mod wav2lip_process_upload_command; diff --git a/src/script_execution/wav2lip_inference_command.rs b/src/script_execution/wav2lip_inference_command.rs index cb26a5be61..7ed13a717e 100644 --- a/src/script_execution/wav2lip_inference_command.rs +++ b/src/script_execution/wav2lip_inference_command.rs @@ -3,9 +3,10 @@ use log::{info,warn}; use std::process::{Command, Stdio}; use subprocess::{Popen, PopenConfig, Redirection}; use std::fs::OpenOptions; +use crate::script_execution::wav2lip_inference_command_OLD::Wav2LipInferenceCommand; -/// This command is used to preprocess the face detection frames from user-submitted video. -/// This should only ever need to run once. The frames can then be uploaded to Buckets and saved. +/// This command is used to run inference. +/// It uses preprocessed face files so that it's much faster. #[derive(Clone)] pub struct Wav2LipInferenceCommand { w2l_directory: String, @@ -28,10 +29,9 @@ impl Wav2LipInferenceCommand { pub fn execute(&self, audio_filename: &str, - image_or_video_filename: &str, - output_video_filename: &str, + output_cached_faces_filename: &str, + output_metadata_filename: &str, is_image: bool, - disable_end_bump: bool, spawn_process: bool) -> AnyhowResult<()> { let mut command = String::new(); @@ -46,12 +46,12 @@ impl Wav2LipInferenceCommand { command.push_str(&self.script_name); command.push_str(" --checkpoint_path "); command.push_str(&self.checkpoint_path); - command.push_str(" --face "); - command.push_str(image_or_video_filename); - command.push_str(" --audio "); + command.push_str(" --audio_filename"); command.push_str(audio_filename); - command.push_str(" --outfile "); - command.push_str(output_video_filename); + command.push_str(" --cached_faces_filename "); + command.push_str(output_cached_faces_filename); + command.push_str(" --output_metadata_filename "); + command.push_str(output_metadata_filename); if is_image { command.push_str(" --is_image "); @@ -74,14 +74,14 @@ impl Wav2LipInferenceCommand { .write(true) .create(true) .truncate(true) - .open("/tmp/stdout.txt")?; + .open("/tmp/wav2lip_upload_stdout.txt")?; let stderr_file = OpenOptions::new() .read(true) .write(true) .create(true) .truncate(true) - .open("/tmp/stderr.txt")?; + .open("/tmp/wav2lip_upload_stderr.txt")?; let mut p = Popen::create(&command_parts, PopenConfig { //stdout: Redirection::Pipe, diff --git a/src/script_execution/wav2lip_inference_command_OLD.rs b/src/script_execution/wav2lip_inference_command_OLD.rs new file mode 100644 index 0000000000..5e91b2300b --- /dev/null +++ b/src/script_execution/wav2lip_inference_command_OLD.rs @@ -0,0 +1,125 @@ +use crate::AnyhowResult; +use log::{info,warn}; +use std::process::{Command, Stdio}; +use subprocess::{Popen, PopenConfig, Redirection}; +use std::fs::OpenOptions; + +/// This command is used to preprocess the face detection frames from user-submitted video. +/// This should only ever need to run once. The frames can then be uploaded to Buckets and saved. +#[derive(Clone)] +pub struct Wav2LipInferenceCommandOLD { + w2l_directory: String, + script_name: String, + checkpoint_path: String, +} + +impl Wav2LipInferenceCommandOLD { + pub fn new( + w2l_directory: &str, + script_name: &str, + checkpoint_path: &str, + ) -> Self { + Self { + w2l_directory: w2l_directory.to_string(), + script_name: script_name.to_string(), + checkpoint_path: checkpoint_path.to_string(), + } + } + + pub fn execute(&self, + audio_filename: &str, + image_or_video_filename: &str, + output_video_filename: &str, + is_image: bool, + disable_end_bump: bool, + spawn_process: bool) -> AnyhowResult<()> + { + let mut command = String::new(); + + command.push_str("echo 'test'"); + command.push_str(" && "); + command.push_str(&format!("cd {}", self.w2l_directory)); + command.push_str(" && "); + command.push_str("source python/bin/activate"); + command.push_str(" && "); + command.push_str("python "); + command.push_str(&self.script_name); + command.push_str(" --checkpoint_path "); + command.push_str(&self.checkpoint_path); + command.push_str(" --face "); + command.push_str(image_or_video_filename); + command.push_str(" --audio "); + command.push_str(audio_filename); + command.push_str(" --outfile "); + command.push_str(output_video_filename); + + if is_image { + command.push_str(" --is_image "); + } + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + if spawn_process { + // NB: This forks and returns immediately. + //let _child_pid = command_builder.spawn()?; + + let stdout_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/stdout.txt")?; + + let stderr_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/stderr.txt")?; + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + stdout: Redirection::File(stdout_file), + stderr: Redirection::File(stderr_file), + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + p.detach(); + + } else { + // NB: This is a blocking call. + /*let output = command_builder.output()?; + + info!("Output status: {}", output.status); + info!("Stdout: {:?}", String::from_utf8(output.stdout)); + error!("Stderr: {:?}", String::from_utf8(output.stderr)); + + if !output.status.success() { + bail!("Bad error code: {:?}", output.status); + }*/ + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + let exit_status = p.wait()?; + + info!("Exit status: {:?}", exit_status); + } + + Ok(()) + } +} diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index d438772d8f..a20c7cb5d4 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -47,17 +47,19 @@ use crate::script_execution::imagemagick_generate_preview_image_command::Imagema const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; const ENV_REGION_NAME : &'static str = "REGION_NAME"; -const ENV_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT"; // Buckets (private data) const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; // Buckets (public data) const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; +// Various bucket roots +const ENV_DOWNLOAD_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT"; + // Python code const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; -const ENV_SCRIPT_NAME : &'static str = "W2L_SCRIPT_NAME"; +const ENV_DOWNLOAD_SCRIPT_NAME : &'static str = "W2L_DOWNLOAD_SCRIPT_NAME"; // NB: sqlx::query is spammy and logs all queries as "info"-level const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; @@ -104,12 +106,14 @@ async fn main() -> AnyhowResult<()> { let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; - let bucket_root = easyenv::get_env_string_required(ENV_BUCKET_ROOT)?; // Private and Public Buckets let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?; let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?; + // Bucket roots + let bucket_root = easyenv::get_env_string_required(ENV_DOWNLOAD_BUCKET_ROOT)?; + let private_bucket_client = BucketClient::create( &access_key, &secret_key, @@ -127,7 +131,7 @@ async fn main() -> AnyhowResult<()> { )?; let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?; - let py_script_name = easyenv::get_env_string_required(ENV_SCRIPT_NAME)?; + let py_script_name = easyenv::get_env_string_required(ENV_DOWNLOAD_SCRIPT_NAME)?; let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?; let w2l_preprecess_command = Wav2LipPreprocessClient::new( diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 559e546bd4..7434302279 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -1,7 +1,498 @@ +#![deny(unused_must_use)] +#![allow(dead_code)] +#![allow(unused_mut)] +#![allow(unused_imports)] +#![warn(unused_must_use)] +//#![allow(warnings)] + +#[macro_use] extern crate serde_derive; + +pub mod buckets; +pub mod job_queries; +pub mod script_execution; pub mod util; +use anyhow::anyhow; +use chrono::Utc; +use crate::buckets::bucket_client::BucketClient; +use crate::buckets::bucket_paths::hash_to_bucket_path; +use crate::buckets::file_hashing::get_file_hash; +use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; +use crate::job_queries::w2l_inference_job_queries::insert_w2l_result; +use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_done; +use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failure; +use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records; +use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::util::anyhow_result::AnyhowResult; +use crate::util::filesystem::check_directory_exists; +use crate::util::filesystem::check_file_exists; +use crate::util::random_crockford_token::random_crockford_token; +use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; +use log::{warn, info}; +use ring::digest::{Context, Digest, SHA256}; +use sqlx::MySqlPool; +use sqlx::mysql::MySqlPoolOptions; +use std::fs::{File, metadata}; +use std::io::{BufReader, Read}; +use std::path::{PathBuf, Path}; +use std::process::Command; +use std::time::Duration; +use tempdir::TempDir; +use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand; +use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; +use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; +use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; + +// Buckets (shared config) +const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; +const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; +const ENV_REGION_NAME : &'static str = "REGION_NAME"; + +// Buckets (private data) +const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; +// Buckets (public data) +const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; + +// Various bucket roots +const ENV_DOWNLOAD_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT"; +const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &'static str = "AUDIO_UPLOADS_BUCKET_ROOT"; +const ENV_INFERENCE_OUTPUT_BUCKET_ROOT : &'static str = "W2L_INFERENCE_OUTPUT_BUCKET_ROOT"; + +// Python code +const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; +const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; +const ENV_INFERENCE_SCRIPT_NAME : &'static str = "W2L_INFERENCE_SCRIPT_NAME"; + +// NB: sqlx::query is spammy and logs all queries as "info"-level +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; +const DEFAULT_TEMP_DIR: &'static str = "/tmp"; + +struct Inferencer { + pub download_temp_directory: PathBuf, + pub mysql_pool: MySqlPool, + + pub private_bucket_client: BucketClient, + pub public_bucket_client: BucketClient, + + pub google_drive_downloader: GoogleDriveDownloadCommand, + pub w2l_inference: Wav2LipInferenceCommand, + //pub ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand, + //pub ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand, + //pub imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand, + + // Command to run + pub inference_script: String, + + // Root to retrieve W2L templates (public and private) + pub w2l_template_uploads_bucket_root: String, + // Root to retrieve uploaded audio (public and private) + pub audio_uploads_bucket_root: String, + // Root to upload inference output + pub inference_output_bucket_root: String, +} + +#[tokio::main] +async fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + // NB: Do not check this secrets-containing dotenv file into VCS. + // This file should only contain *development* secrets, never production. + let _ = dotenv::from_filename(".env-secrets").ok(); + + info!("Obtaining hostname..."); + + let server_hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or("w2l-inference-job".to_string()); + + info!("Hostname: {}", &server_hostname); + + // Bucket stuff (shared) + let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; + let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; + let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; + + // Private and Public Buckets + let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?; + let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?; + + // Bucket roots + let w2l_template_bucket_root = easyenv::get_env_string_required(ENV_DOWNLOAD_BUCKET_ROOT)?; + let audio_bucket_root = easyenv::get_env_string_required(ENV_AUDIO_UPLOADS_BUCKET_ROOT)?; + let inference_output_bucket_root = easyenv::get_env_string_required(ENV_INFERENCE_OUTPUT_BUCKET_ROOT)?; + + let private_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &private_bucket_name, + None, + )?; + + let public_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &public_bucket_name, + None, + )?; + + let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?; + let py_script_name = easyenv::get_env_string_required(ENV_INFERENCE_SCRIPT_NAME)?; + let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?; + + let w2l_inference_command = Wav2LipInferenceCommand::new( + &py_code_directory, + &py_script_name, + &py_model_checkpoint, + ); + + let temp_directory = easyenv::get_env_string_or_default( + "DOWNLOAD_TEMP_DIR", + DEFAULT_TEMP_DIR); + + // TODO: In the future, we may want to enable downloading images or audio files. + //let download_script = easyenv::get_env_string_or_default( + // "DOWNLOAD_SCRIPT", + // "./scripts/download_gdrive.py"); + // + //let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); + + let temp_directory = PathBuf::from(temp_directory); + + check_directory_exists(&temp_directory)?; + + let db_connection_string = + easyenv::get_env_string_or_default( + "MYSQL_URL", + "mysql://root:root@localhost/storyteller"); + + info!("Connecting to database..."); + + let mysql_pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(&db_connection_string) + .await?; + + let inferencer = Inferencer { + download_temp_directory: temp_directory, + mysql_pool, + public_bucket_client, + private_bucket_client, + inference_script, + google_drive_downloader, + //ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, + //ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, + //imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, + w2l_inference: w2l_inference_command, + w2l_template_uploads_bucket_root: w2l_template_bucket_root.to_string(), + audio_uploads_bucket_root: audio_bucket_root.to_string(), + inference_output_bucket_root: inference_output_bucket_root.to_string(), + }; + + main_loop(inferencer).await; + + Ok(()) +} + +const START_TIMEOUT_MILLIS : u64 = 500; +const INCREASE_TIMEOUT_MILLIS : u64 = 1000; + +async fn main_loop(inferencer: Inferencer) { + let mut timeout_millis = START_TIMEOUT_MILLIS; + + loop { + let num_records = 1; + + let query_result = query_w2l_template_upload_job_records( + &inferencer.mysql_pool, + num_records) + .await; + + let jobs = match query_result { + Ok(jobs) => jobs, + Err(e) => { + warn!("Error querying jobs: {:?}", e); + std::thread::sleep(Duration::from_millis(timeout_millis)); + timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; + } + }; + + if jobs.is_empty() { + info!("No jobs!"); + std::thread::sleep(Duration::from_millis(1500)); + continue; + } + + let result = process_jobs(&inferencer, jobs).await; + + match result { + Ok(_) => {}, + Err(e) => { + warn!("Error querying jobs: {:?}", e); + std::thread::sleep(Duration::from_millis(timeout_millis)); + timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; + } + } + + timeout_millis = START_TIMEOUT_MILLIS; // reset + + std::thread::sleep(Duration::from_millis(500)); + } +} + +async fn process_jobs(inferencer: &Inferencer, jobs: Vec) -> AnyhowResult<()> { + for job in jobs.into_iter() { + let result = process_job(inferencer, &job).await; + match result { + Ok(_) => {}, + Err(e) => { + warn!("Failure to process job: {:?}", e); + let failure_reason = ""; + let _r = mark_w2l_inference_job_failure( + &inferencer.mysql_pool, + &job, + failure_reason) + .await; + } + } + } + + Ok(()) +} + +#[derive(Deserialize)] +struct FileMetadata { + pub is_video: bool, + pub width: u32, + pub height: u32, + pub num_frames: u64, + pub fps: Option, + pub duration_millis: Option, + pub mimetype: Option, + pub file_size_bytes: u64, +} + +fn read_metadata_file(filename: &str) -> AnyhowResult { + let mut file = File::open(filename)?; + let mut buffer = String::new(); + file.read_to_string(&mut buffer)?; + Ok(serde_json::from_str(&buffer)?) +} + +async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> AnyhowResult<()> { + + // TODO: 1. Check if w2l model is downloaded and download it + // TODO: 2. Check if w2l template faces are downloaded and download it + // TODO: 3. Download audio + + + // TODO: 1. Mark processing. + // TODO: 2. Download. (DONE) + // TODO: 3. Process template with face detection (DONE) + // TODO: 4. Determine if picture or video + // TODO: 5. Take a screenshot/gif + // TODO: 6. Upload all (partially done). + // TODO: 7. Save record. (DONE) + // TODO: 8. Mark job done. (DONE) + + let temp_dir = format!("temp_{}", job.id); + let temp_dir = TempDir::new(&temp_dir)?; + + let download_url = job.download_url.as_ref() + .map(|c| c.to_string()) + .unwrap_or("".to_string()); + + // ==================== DOWNLOAD FILE ==================== // + + info!("Calling downloader..."); + let download_filename = inferencer.google_drive_downloader + .download_file(&download_url, &temp_dir).await?; + + // ==================== PROCESS FACES ==================== // + + // This is the Python Pickle file with all the face frames. + // We'll retain it forever since it's expensive to compute. + let cached_faces_filename = format!("{}_detected_faces.pickle", &download_filename); + + // This is a file that we'll use to determine if the file is an image or video. + let output_metadata_filename = format!("{}_metadata.json", &download_filename); + + let is_image = false; // TODO: Don't always treat as video. + let spawn_process = false; + + inferencer.w2l_processor.execute( + &download_filename, + &cached_faces_filename, + &output_metadata_filename, + is_image, + spawn_process)?; + + // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // + + let video_or_image_path = PathBuf::from(&download_filename); + let cached_faces_path = &PathBuf::from(&cached_faces_filename); + let output_metadata_path = &PathBuf::from(&output_metadata_filename); + + info!("Checking that both files exist (original source + cached faces) ..."); + + check_file_exists(&video_or_image_path)?; + check_file_exists(&cached_faces_path)?; + check_file_exists(&output_metadata_path)?; + + let file_metadata = read_metadata_file(&output_metadata_filename)?; + + // ==================== BASE OBJECT NAMES BASED ON HASH ==================== // + + let private_bucket_hash = get_file_hash(&download_filename)?; + + info!("File hash: {}", private_bucket_hash); + + // Full path to video/image + let full_object_path = hash_to_bucket_path( + &private_bucket_hash, + Some(&inferencer.w2l_template_uploads_bucket_root))?; + + // ==================== GENERATE VIDEO PREVIEWS ==================== // + + let mut maybe_image_preview_filename : Option = None; + let mut maybe_image_preview_object_name : Option = None; + + let mut maybe_video_preview_filename : Option = None; + let mut maybe_video_preview_object_name : Option = None; + + if file_metadata.is_video { + let preview_filename = format!("{}_preview.webp", &download_filename); + + inferencer.ffmpeg_video_preview_generator.execute( + &download_filename, + &preview_filename, + 500, + 500, + true, + false + )?; + + let video_preview_path = PathBuf::from(&preview_filename); + check_file_exists(&video_preview_path)?; + + let preview_object_path = format!("{}_preview.webp", full_object_path); + maybe_video_preview_object_name = Some(preview_object_path); + maybe_video_preview_filename = Some(video_preview_path); + + } else { + let preview_filename = format!("{}_preview.webp", &download_filename); + + inferencer.imagemagick_image_preview_generator.execute( + &download_filename, + &preview_filename, + 500, + 500, + false + )?; + + let image_preview_path = PathBuf::from(&preview_filename); + check_file_exists(&image_preview_path)?; + + let preview_object_path = format!("{}_preview.webp", full_object_path); + maybe_image_preview_object_name = Some(preview_object_path); + maybe_image_preview_filename = Some(image_preview_path); + } + + // ==================== UPLOAD TO BUCKETS ==================== // + + info!("Image/video destination bucket path: {}", full_object_path); + + // Full path to cached faces + let full_object_path_cached_faces = format!("{}_detected_faces.pickle", full_object_path); + + info!("Cached faces destination bucket path: {}", full_object_path_cached_faces); + + info!("Uploading image/video..."); + + let original_mime_type = file_metadata.mimetype + .as_deref() + .unwrap_or("application/octet-stream"); + + inferencer.private_bucket_client.upload_filename_with_content_type( + &full_object_path, + &video_or_image_path, + original_mime_type).await?; + + inferencer.public_bucket_client.upload_filename_with_content_type( + &full_object_path, + &video_or_image_path, + original_mime_type).await?; + + info!("Uploading cached faces..."); + inferencer.private_bucket_client.upload_filename( + &full_object_path_cached_faces, + &cached_faces_path).await?; + + // TODO: Fix this ugh. + if let Some(image_preview_filename) = maybe_image_preview_filename.as_deref() { + if let Some(image_preview_object_name) = maybe_image_preview_object_name.as_deref() { + info!("Uploading image preview..."); + inferencer.private_bucket_client.upload_filename_with_content_type( + &image_preview_object_name, + image_preview_filename, + "image/webp").await?; + + info!("Uploading image preview..."); + inferencer.public_bucket_client.upload_filename_with_content_type( + &image_preview_object_name, + image_preview_filename, + "image/webp").await?; + } + } + + // TODO: Fix this ugh. + if let Some(video_preview_filename) = maybe_video_preview_filename.as_deref() { + if let Some(video_preview_object_name) = maybe_video_preview_object_name.as_deref() { + info!("Uploading video preview..."); + inferencer.private_bucket_client.upload_filename_with_content_type( + &video_preview_object_name, + video_preview_filename, + "image/webp").await?; + + info!("Uploading video preview..."); + inferencer.public_bucket_client.upload_filename_with_content_type( + &video_preview_object_name, + video_preview_filename, + "image/webp").await?; + } + } + + // ==================== SAVE RECORDS ==================== // + + let template_type = if file_metadata.is_video { "video" } else { "image" }; + + info!("Saving w2l inference record..."); + let id = insert_w2l_result( + &inferencer.mysql_pool, + template_type, + job, + &private_bucket_hash, + &full_object_path, + &full_object_path_cached_faces, + maybe_image_preview_object_name.as_deref(), + maybe_video_preview_object_name.as_deref(), + file_metadata.file_size_bytes, + file_metadata.mimetype.as_deref(), + file_metadata.width, + file_metadata.height, + file_metadata.num_frames, + file_metadata.fps.unwrap_or(0.0f32), + file_metadata.duration_millis.unwrap_or(0)) + .await?; + + info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}", + job.id, id); + + mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?; -fn main() -> AnyhowResult<()> { Ok(()) } From 69fba14fa498ea726b463c0591368ce740fec9f0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 21:05:49 -0400 Subject: [PATCH 0174/1042] no longer fails to compile (but w2l inference is broken) --- src/buckets/bucket_path_unifier.rs | 64 +++++++++++++++++-- src/buckets/bucket_paths.rs | 4 +- src/job_queries/w2l_inference_job_queries.rs | 14 ++-- src/main.rs | 5 +- .../wav2lip_inference_command.rs | 1 - src/tts_download_job.rs | 3 +- src/tts_inference_job.rs | 7 ++ src/w2l_download_job.rs | 4 +- src/w2l_inference_job.rs | 36 ++++++----- 9 files changed, 99 insertions(+), 39 deletions(-) diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs index e7ce5d28cb..09d98173f3 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/buckets/bucket_path_unifier.rs @@ -1,25 +1,75 @@ +use crate::buckets::bucket_paths::hash_to_bucket_path; +use std::path::PathBuf; /// This is designed to make it centrally configurable where /// different types of objects are stored. pub struct BucketPathUnifier { - + pub user_uploaded_w2l_templates_root: PathBuf, } impl BucketPathUnifier { - pub fn user_audio_for_w2l_inference_path(audio_token: &str) -> String { + pub fn user_audio_for_w2l_inference_path(&self, audio_token: &str) -> String { "".to_string() } - pub fn end_bump_video_for_w2l_path(end_bump_filename: &str) -> String { + pub fn end_bump_video_for_w2l_path(&self, end_bump_filename: &str) -> String { "".to_string() } - pub fn precomputed_faces_for_w2l_path(token: &str) -> String { + // eg. /user_uploaded_w2l_templates/1/5/1/151a[...60]_detected_faces.pickle + pub fn precomputed_faces_for_w2l_path(&self, template_file_hash: &str) -> PathBuf { + let faces_filename = format!("{}_detected_faces.pickle", &template_file_hash); + let hashed_path = Self::hashed_directory_path(template_file_hash); + + self.user_uploaded_w2l_templates_root + .join(hashed_path) + .join(faces_filename) + } + + pub fn pretrained_w2l_model_path(&self, model_name: &str) -> String { "".to_string() } - pub fn pretrained_w2l_model_path(model_name: &str) -> String { - "".to_string() + pub fn hashed_directory_path(file_hash: &str) -> String { + match file_hash.len() { + 0 => "".to_string(), + 1 => file_hash.to_string(), + 2 => format!("{}/", &file_hash[0..1]), + 3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]), + _ => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]), + } } -} \ No newline at end of file +} + +#[cfg(test)] +mod tests { + use crate::buckets::bucket_path_unifier::BucketPathUnifier; + + #[test] + fn hashed_directory_path_length_zero() { + assert_eq!(&BucketPathUnifier::hashed_directory_path(""), ""); + } + + #[test] + fn hashed_directory_path_length_one() { + assert_eq!(&BucketPathUnifier::hashed_directory_path("a"), ""); + } + + #[test] + fn hashed_directory_path_length_two() { + assert_eq!(&BucketPathUnifier::hashed_directory_path("ab"), "a/"); + } + + #[test] + fn hashed_directory_path_length_three() { + assert_eq!(&BucketPathUnifier::hashed_directory_path("abc"), "a/b/"); + } + + #[test] + fn hashed_directory_path_length_more() { + assert_eq!(&BucketPathUnifier::hashed_directory_path("abcd"), "a/b/c/"); + assert_eq!(&BucketPathUnifier::hashed_directory_path("abcde"), "a/b/c/"); + assert_eq!(&BucketPathUnifier::hashed_directory_path("abcdef01234"), "a/b/c/"); + } +} diff --git a/src/buckets/bucket_paths.rs b/src/buckets/bucket_paths.rs index 52c1c1fad0..6848e4176d 100644 --- a/src/buckets/bucket_paths.rs +++ b/src/buckets/bucket_paths.rs @@ -11,7 +11,7 @@ pub fn hash_to_bucket_path(file_hash: &str, bail!("File length is too short"); } - let optional_directory = optional_root_directory + let root_directory = optional_root_directory .map(|root| PathBuf::from(root.to_string())) .unwrap_or(PathBuf::from("/")); @@ -23,7 +23,7 @@ pub fn hash_to_bucket_path(file_hash: &str, &file_hash, ); - let maybe_path = optional_directory + let maybe_path = root_directory .join(file_path) .to_str() .map(|s| s.to_string()); diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index 7ee79feb68..a32bb8c38e 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -43,7 +43,7 @@ pub struct W2lInferenceJobRecord { pub async fn query_w2l_inference_job_records(pool: &MySqlPool, num_records: u32) -> AnyhowResult> { - let job_records = sqlx::query_as!( + /*let job_records = sqlx::query_as!( W2lInferenceJobRecord, r#" SELECT @@ -67,7 +67,9 @@ WHERE .fetch_all(pool) .await?; - Ok(job_records) + Ok(job_records)*/ + + return Err(anyhow!("TODO")); } pub async fn mark_w2l_inference_job_failure(pool: &MySqlPool, @@ -138,7 +140,7 @@ pub async fn insert_w2l_result(pool: &MySqlPool, duration_millis: u64) -> AnyhowResult { - let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?; + /*let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?; let updatable_slug = model_token.clone(); let query_result = sqlx::query!( @@ -170,10 +172,7 @@ SET model_token, template_type, updatable_slug, - job.title.to_string(), - job.creator_user_token.clone(), job.creator_ip_address.clone(), - job.download_url.clone(), private_bucket_hash.to_string(), private_bucket_object_name.to_string(), private_bucket_cached_faces_object_name.to_string(), @@ -200,5 +199,6 @@ SET } }; - Ok(record_id) + Ok(record_id)*/ + return Err(anyhow!("TODO")); } diff --git a/src/main.rs b/src/main.rs index 891991499d..77aea47895 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,8 @@ +#![deny(unused_must_use)] #![allow(dead_code)] -#![allow(unused_mut)] #![allow(unused_imports)] -#![warn(unused_must_use)] +#![allow(unused_mut)] +#![allow(unused_variables)] //#![allow(warnings)] #[macro_use] extern crate lazy_static; diff --git a/src/script_execution/wav2lip_inference_command.rs b/src/script_execution/wav2lip_inference_command.rs index 7ed13a717e..361e6d89f1 100644 --- a/src/script_execution/wav2lip_inference_command.rs +++ b/src/script_execution/wav2lip_inference_command.rs @@ -3,7 +3,6 @@ use log::{info,warn}; use std::process::{Command, Stdio}; use subprocess::{Popen, PopenConfig, Redirection}; use std::fs::OpenOptions; -use crate::script_execution::wav2lip_inference_command_OLD::Wav2LipInferenceCommand; /// This command is used to run inference. /// It uses preprocessed face files so that it's much faster. diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 512d3273f2..4999e596c6 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -1,7 +1,8 @@ #![deny(unused_must_use)] #![allow(dead_code)] -#![allow(unused_mut)] #![allow(unused_imports)] +#![allow(unused_mut)] +#![allow(unused_variables)] #![warn(unused_must_use)] //#![allow(warnings)] diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 559e546bd4..2cec8a7b50 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -1,3 +1,10 @@ +#![deny(unused_must_use)] +#![allow(dead_code)] +#![allow(unused_imports)] +#![allow(unused_mut)] +#![allow(unused_variables)] +//#![allow(warnings)] + pub mod util; use crate::util::anyhow_result::AnyhowResult; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index a20c7cb5d4..d837779dd4 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -1,8 +1,8 @@ #![deny(unused_must_use)] #![allow(dead_code)] -#![allow(unused_mut)] #![allow(unused_imports)] -#![warn(unused_must_use)] +#![allow(unused_mut)] +#![allow(unused_variables)] //#![allow(warnings)] #[macro_use] extern crate serde_derive; diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 7434302279..373de68068 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -1,8 +1,8 @@ #![deny(unused_must_use)] #![allow(dead_code)] -#![allow(unused_mut)] #![allow(unused_imports)] -#![warn(unused_must_use)] +#![allow(unused_mut)] +#![allow(unused_variables)] //#![allow(warnings)] #[macro_use] extern crate serde_derive; @@ -153,11 +153,10 @@ async fn main() -> AnyhowResult<()> { DEFAULT_TEMP_DIR); // TODO: In the future, we may want to enable downloading images or audio files. - //let download_script = easyenv::get_env_string_or_default( - // "DOWNLOAD_SCRIPT", - // "./scripts/download_gdrive.py"); - // - //let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); + let download_script = easyenv::get_env_string_or_default( + "DOWNLOAD_SCRIPT", + "./scripts/download_gdrive.py"); + let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); let temp_directory = PathBuf::from(temp_directory); @@ -175,6 +174,8 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; + let inference_script = "TODO".to_string(); + let inferencer = Inferencer { download_temp_directory: temp_directory, mysql_pool, @@ -205,7 +206,7 @@ async fn main_loop(inferencer: Inferencer) { loop { let num_records = 1; - let query_result = query_w2l_template_upload_job_records( + let query_result = query_w2l_inference_job_records( &inferencer.mysql_pool, num_records) .await; @@ -302,15 +303,16 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; - let download_url = job.download_url.as_ref() - .map(|c| c.to_string()) - .unwrap_or("".to_string()); + //let download_url = job.download_url.as_ref() + // .map(|c| c.to_string()) + // .unwrap_or("".to_string()); // ==================== DOWNLOAD FILE ==================== // info!("Calling downloader..."); - let download_filename = inferencer.google_drive_downloader - .download_file(&download_url, &temp_dir).await?; + //let download_filename = inferencer.google_drive_downloader + // .download_file(&download_url, &temp_dir).await?; + let download_filename = "TODO"; // ==================== PROCESS FACES ==================== // @@ -324,12 +326,12 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An let is_image = false; // TODO: Don't always treat as video. let spawn_process = false; - inferencer.w2l_processor.execute( + /*inferencer.w2l_processor.execute( &download_filename, &cached_faces_filename, &output_metadata_filename, is_image, - spawn_process)?; + spawn_process)?;*/ // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // @@ -364,7 +366,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An let mut maybe_video_preview_filename : Option = None; let mut maybe_video_preview_object_name : Option = None; - if file_metadata.is_video { + /*if file_metadata.is_video { let preview_filename = format!("{}_preview.webp", &download_filename); inferencer.ffmpeg_video_preview_generator.execute( @@ -400,7 +402,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An let preview_object_path = format!("{}_preview.webp", full_object_path); maybe_image_preview_object_name = Some(preview_object_path); maybe_image_preview_filename = Some(image_preview_path); - } + }*/ // ==================== UPLOAD TO BUCKETS ==================== // From e2a9bf81e58055fba4b7ac02a48cee62cb5f8fce Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 21:06:38 -0400 Subject: [PATCH 0175/1042] fix bug --- src/buckets/bucket_path_unifier.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs index 09d98173f3..f6dbace786 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/buckets/bucket_path_unifier.rs @@ -33,8 +33,7 @@ impl BucketPathUnifier { pub fn hashed_directory_path(file_hash: &str) -> String { match file_hash.len() { - 0 => "".to_string(), - 1 => file_hash.to_string(), + 0 | 1=> "".to_string(), 2 => format!("{}/", &file_hash[0..1]), 3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]), _ => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]), From 843b9755a1068192d025d07cad22880bcca6ce5c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 21:20:48 -0400 Subject: [PATCH 0176/1042] two types of path --- src/buckets/bucket_path_unifier.rs | 37 ++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs index f6dbace786..3b3fbf2d61 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/buckets/bucket_path_unifier.rs @@ -4,19 +4,17 @@ use std::path::PathBuf; /// This is designed to make it centrally configurable where /// different types of objects are stored. pub struct BucketPathUnifier { + pub user_uploaded_audio_for_w2l_root: PathBuf, pub user_uploaded_w2l_templates_root: PathBuf, } impl BucketPathUnifier { - pub fn user_audio_for_w2l_inference_path(&self, audio_token: &str) -> String { - "".to_string() - } - pub fn end_bump_video_for_w2l_path(&self, end_bump_filename: &str) -> String { "".to_string() } + // These share the same directory as the uploaded w2l template media. // eg. /user_uploaded_w2l_templates/1/5/1/151a[...60]_detected_faces.pickle pub fn precomputed_faces_for_w2l_path(&self, template_file_hash: &str) -> PathBuf { let faces_filename = format!("{}_detected_faces.pickle", &template_file_hash); @@ -27,6 +25,15 @@ impl BucketPathUnifier { .join(faces_filename) } + // User-uploaded audio. + // eg. /user_uploaded_w2l_audio/0/0/b/00bcc7a4-bdf5-43a5-9603-a15ca780d866 + pub fn user_audio_for_w2l_inference_path(&self, audio_uuid: &str) -> PathBuf { + let hashed_path = Self::hashed_directory_path(audio_uuid); + self.user_uploaded_audio_for_w2l_root + .join(hashed_path) + .join(audio_uuid) + } + pub fn pretrained_w2l_model_path(&self, model_name: &str) -> String { "".to_string() } @@ -44,6 +51,28 @@ impl BucketPathUnifier { #[cfg(test)] mod tests { use crate::buckets::bucket_path_unifier::BucketPathUnifier; + use std::path::PathBuf; + + fn get_instance() -> BucketPathUnifier { + BucketPathUnifier { + user_uploaded_audio_for_w2l_root: PathBuf::from("/test_path_w2l_audio"), + user_uploaded_w2l_templates_root: PathBuf::from("/test_path_w2l_templates"), + } + } + + #[test] + fn test_user_audio_for_w2l_inference_path() { + let paths = get_instance(); + assert_eq!(paths.user_audio_for_w2l_inference_path("foobar").to_str().unwrap(), + "/test_path_w2l_audio/f/o/o/foobar"); + } + + #[test] + fn test_precomputed_faces_for_w2l_path() { + let paths = get_instance(); + assert_eq!(paths.precomputed_faces_for_w2l_path("foobar").to_str().unwrap(), + "/test_path_w2l_templates/f/o/o/foobar_detected_faces.pickle"); + } #[test] fn hashed_directory_path_length_zero() { From e629166da3e429d54d733698fb5c09669c409dee Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 22:11:01 -0400 Subject: [PATCH 0177/1042] filesystem path tool, tests pass again --- src/buckets/bucket_path_unifier.rs | 47 +++++++++++++++-- src/util/mod.rs | 5 +- src/util/semi_persistent_cache_dir.rs | 68 ++++++++++++++++++++++++ src/w2l_inference_job.rs | 75 ++++++++++++++------------- 4 files changed, 152 insertions(+), 43 deletions(-) create mode 100644 src/util/semi_persistent_cache_dir.rs diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs index 3b3fbf2d61..b5252ccfdc 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/buckets/bucket_path_unifier.rs @@ -1,15 +1,35 @@ use crate::buckets::bucket_paths::hash_to_bucket_path; use std::path::PathBuf; +use crate::util::anyhow_result::AnyhowResult; /// This is designed to make it centrally configurable where /// different types of objects are stored. pub struct BucketPathUnifier { - pub user_uploaded_audio_for_w2l_root: PathBuf, pub user_uploaded_w2l_templates_root: PathBuf, + pub user_uploaded_audio_for_w2l_root: PathBuf, + pub tts_inference_output_root: PathBuf, + pub w2l_inference_output_root: PathBuf, } impl BucketPathUnifier { + // TODO + //pub fn from_env_vars() -> AnyhowResult { + // Ok(Self { + // user_uploaded_audio_for_w2l_root: , + // user_uploaded_w2l_templates_root: , + // }) + //} + + pub fn default_paths() -> Self { + Self { + user_uploaded_audio_for_w2l_root: PathBuf::from("/user_uploaded_w2l_audio"), + user_uploaded_w2l_templates_root: PathBuf::from("/user_uploaded_w2l_templates"), + tts_inference_output_root: PathBuf::from("/tts_inference_output"), + w2l_inference_output_root: PathBuf::from("/w2l_inference_output"), + } + } + pub fn end_bump_video_for_w2l_path(&self, end_bump_filename: &str) -> String { "".to_string() } @@ -34,10 +54,20 @@ impl BucketPathUnifier { .join(audio_uuid) } - pub fn pretrained_w2l_model_path(&self, model_name: &str) -> String { - "".to_string() + // W2L inference output videos + pub fn w2l_inference_video_output_path(&self, w2l_inference_token: &str) -> PathBuf { + let hashed_path = Self::hashed_directory_path(w2l_inference_token); + let video_filename = format!("{}_video.mp4", &w2l_inference_token); + + self.w2l_inference_output_root + .join(hashed_path) + .join(video_filename) } + //pub fn pretrained_w2l_model_path(&self, model_name: &str) -> String { + // "".to_string() + //} + pub fn hashed_directory_path(file_hash: &str) -> String { match file_hash.len() { 0 | 1=> "".to_string(), @@ -55,8 +85,10 @@ mod tests { fn get_instance() -> BucketPathUnifier { BucketPathUnifier { - user_uploaded_audio_for_w2l_root: PathBuf::from("/test_path_w2l_audio"), user_uploaded_w2l_templates_root: PathBuf::from("/test_path_w2l_templates"), + user_uploaded_audio_for_w2l_root: PathBuf::from("/test_path_w2l_audio"), + tts_inference_output_root: PathBuf::from("/test_path_tts_output"), + w2l_inference_output_root: PathBuf::from("/test_path_w2l_output"), } } @@ -74,6 +106,13 @@ mod tests { "/test_path_w2l_templates/f/o/o/foobar_detected_faces.pickle"); } + #[test] + fn test_w2l_inference_video_output_path() { + let paths = get_instance(); + assert_eq!(paths.w2l_inference_video_output_path("foobar").to_str().unwrap(), + "/test_path_w2l_output/f/o/o/foobar_video.mp4"); + } + #[test] fn hashed_directory_path_length_zero() { assert_eq!(&BucketPathUnifier::hashed_directory_path(""), ""); diff --git a/src/util/mod.rs b/src/util/mod.rs index badcaf6032..fcbc04cc11 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,6 +1,7 @@ pub mod anyhow_result; pub mod email_to_gravatar; pub mod filesystem; -pub mod random_prefix_crockford_token; pub mod random_crockford_token; -pub mod random_uuid; \ No newline at end of file +pub mod random_prefix_crockford_token; +pub mod random_uuid; +pub mod semi_persistent_cache_dir; \ No newline at end of file diff --git a/src/util/semi_persistent_cache_dir.rs b/src/util/semi_persistent_cache_dir.rs new file mode 100644 index 0000000000..50abf148a0 --- /dev/null +++ b/src/util/semi_persistent_cache_dir.rs @@ -0,0 +1,68 @@ +use std::path::{PathBuf, Path}; +use std::fs; +use crate::util::anyhow_result::AnyhowResult; + +/// These are for files on the worker filesystems +pub struct SemiPersistentCacheDir { + cache_root: PathBuf, + tts_model_root: PathBuf, + w2l_model_root: PathBuf, + w2l_face_templates_root: PathBuf, + video_asset_root: PathBuf, // end bump, etc. +} + +impl SemiPersistentCacheDir { + + + /// Everything is rooted at `/file_cache`. + pub fn default_paths() -> Self { + // NB: This is the root of the filesystem + Self::configured_root("/file_cache/") + } + + pub fn configured_root(root_path: &str) -> Self { + let cache_root = PathBuf::from(root_path); + Self { + cache_root: cache_root.clone(), + tts_model_root: cache_root.join("tts/models/"), + w2l_model_root: cache_root.join("w2l/models/"), + w2l_face_templates_root: cache_root.join("w2l/face_templates/"), + video_asset_root: cache_root.join("static_video_assets/"), + } + } + + /// We cache TTS models here. + pub fn tts_model_path(&self) -> &Path { + &self.tts_model_root + } + + /// We cache W2L models here. + pub fn w2l_model_path(&self) -> &Path { + &self.w2l_model_root + } + + /// Face templates go here. + pub fn w2l_face_templates_path(&self) -> &Path { + &self.w2l_face_templates_root + } + + /// End bump, etc. + pub fn video_asset_path(&self) -> &Path { + &self.video_asset_root + } + + pub fn create_w2l_model_path(&self) -> AnyhowResult<()> { + let _ = fs::create_dir_all(self.w2l_model_path())?; + Ok(()) + } + + pub fn create_w2l_face_templates_path(&self) -> AnyhowResult<()> { + let _ = fs::create_dir_all(self.w2l_face_templates_path())?; + Ok(()) + } + + pub fn create_video_asset_path(&self) -> AnyhowResult<()> { + let _ = fs::create_dir_all(self.video_asset_path())?; + Ok(()) + } +} diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 373de68068..350de0c48d 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -15,6 +15,7 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; use crate::buckets::bucket_client::BucketClient; +use crate::buckets::bucket_path_unifier::BucketPathUnifier; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::buckets::file_hashing::get_file_hash; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; @@ -22,11 +23,16 @@ use crate::job_queries::w2l_inference_job_queries::insert_w2l_result; use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_done; use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failure; use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records; +use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; +use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; +use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; +use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; @@ -38,10 +44,6 @@ use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; use tempdir::TempDir; -use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand; -use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; -use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; -use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -53,10 +55,8 @@ const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME // Buckets (public data) const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; -// Various bucket roots -const ENV_DOWNLOAD_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT"; -const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &'static str = "AUDIO_UPLOADS_BUCKET_ROOT"; -const ENV_INFERENCE_OUTPUT_BUCKET_ROOT : &'static str = "W2L_INFERENCE_OUTPUT_BUCKET_ROOT"; +// Where models and other assets get downloaded to. +const ENV_SEMIPERSISTENT_CACHE_DIR : &'static str = "SEMIPERSISTENT_CACHE_DIR"; // Python code const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; @@ -74,6 +74,9 @@ struct Inferencer { pub private_bucket_client: BucketClient, pub public_bucket_client: BucketClient, + pub bucket_path_unifier: BucketPathUnifier, + pub semi_persistent_cache: SemiPersistentCacheDir, + pub google_drive_downloader: GoogleDriveDownloadCommand, pub w2l_inference: Wav2LipInferenceCommand, //pub ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand, @@ -82,13 +85,6 @@ struct Inferencer { // Command to run pub inference_script: String, - - // Root to retrieve W2L templates (public and private) - pub w2l_template_uploads_bucket_root: String, - // Root to retrieve uploaded audio (public and private) - pub audio_uploads_bucket_root: String, - // Root to upload inference output - pub inference_output_bucket_root: String, } #[tokio::main] @@ -117,11 +113,6 @@ async fn main() -> AnyhowResult<()> { let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?; let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?; - // Bucket roots - let w2l_template_bucket_root = easyenv::get_env_string_required(ENV_DOWNLOAD_BUCKET_ROOT)?; - let audio_bucket_root = easyenv::get_env_string_required(ENV_AUDIO_UPLOADS_BUCKET_ROOT)?; - let inference_output_bucket_root = easyenv::get_env_string_required(ENV_INFERENCE_OUTPUT_BUCKET_ROOT)?; - let private_bucket_client = BucketClient::create( &access_key, &secret_key, @@ -176,6 +167,17 @@ async fn main() -> AnyhowResult<()> { let inference_script = "TODO".to_string(); + let persistent_cache_path = easyenv::get_env_string_or_default( + ENV_SEMIPERSISTENT_CACHE_DIR, + "/tmp"); + + let semi_persistent_cache = SemiPersistentCacheDir::configured_root(&persistent_cache_path); + + info!("Creating pod semi-persistent cache dirs..."); + semi_persistent_cache.create_w2l_model_path()?; + semi_persistent_cache.create_w2l_face_templates_path()?; + semi_persistent_cache.create_w2l_model_path()?; + let inferencer = Inferencer { download_temp_directory: temp_directory, mysql_pool, @@ -187,9 +189,8 @@ async fn main() -> AnyhowResult<()> { //ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, //imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, w2l_inference: w2l_inference_command, - w2l_template_uploads_bucket_root: w2l_template_bucket_root.to_string(), - audio_uploads_bucket_root: audio_bucket_root.to_string(), - inference_output_bucket_root: inference_output_bucket_root.to_string(), + bucket_path_unifier: BucketPathUnifier::default_paths(), + semi_persistent_cache, }; main_loop(inferencer).await; @@ -286,19 +287,17 @@ fn read_metadata_file(filename: &str) -> AnyhowResult { async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> AnyhowResult<()> { - // TODO: 1. Check if w2l model is downloaded and download it - // TODO: 2. Check if w2l template faces are downloaded and download it - // TODO: 3. Download audio + // TODO 1. Mark Processing + // + // TODO: 2. Check if w2l model is downloaded / download it to a stable cache location + // TODO: 3. Check if w2l template faces are downloaded and download it + // TODO: 4. Download user audio + // TODO: 5. Process Inference - // TODO: 1. Mark processing. - // TODO: 2. Download. (DONE) - // TODO: 3. Process template with face detection (DONE) - // TODO: 4. Determine if picture or video - // TODO: 5. Take a screenshot/gif - // TODO: 6. Upload all (partially done). - // TODO: 7. Save record. (DONE) - // TODO: 8. Mark job done. (DONE) + // TODO 6. Upload result + // TODO 7. Save record + // TODO 8. Mark job done let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; @@ -354,9 +353,11 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An info!("File hash: {}", private_bucket_hash); // Full path to video/image - let full_object_path = hash_to_bucket_path( - &private_bucket_hash, - Some(&inferencer.w2l_template_uploads_bucket_root))?; + //let full_object_path = hash_to_bucket_path( + // &private_bucket_hash, + // Some(&inferencer.w2l_template_uploads_bucket_root))?; + + let full_object_path = "TODO"; // ==================== GENERATE VIDEO PREVIEWS ==================== // From 125b60b508539accbf3fbcf7edb709ce05918afb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 22:43:05 -0400 Subject: [PATCH 0178/1042] builds again --- .env | 2 ++ src/buckets/bucket_client.rs | 19 +++++++++++++ src/buckets/bucket_path_unifier.rs | 19 ++++++++++--- src/util/semi_persistent_cache_dir.rs | 40 ++++++++++++++++++--------- src/w2l_inference_job.rs | 26 +++++++++++++++++ 5 files changed, 89 insertions(+), 17 deletions(-) diff --git a/.env b/.env index 78fe012901..06dd83f350 100644 --- a/.env +++ b/.env @@ -26,6 +26,8 @@ W2L_INFERENCE_OUTPUT_BUCKET_ROOT=/w2l_output W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip #W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/s3fd-619a316812.pth W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/wav2lip_gan.pth +W2L_MODEL_FILENAME=wav2lip_gan.pth W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py W2L_INFERENCE_SCRIPT_NAME=vocodes_process_inference.py + diff --git a/src/buckets/bucket_client.rs b/src/buckets/bucket_client.rs index 3956ab8d51..715055c032 100644 --- a/src/buckets/bucket_client.rs +++ b/src/buckets/bucket_client.rs @@ -164,6 +164,25 @@ impl BucketClient { Ok(bytes) } + pub async fn download_file_to_disk(&self, object_path: &str, filesystem_path: &str) + -> anyhow::Result<()> + { + info!("downloading from bucket: {}", object_path); + + let mut output_file = std::fs::File::create(filesystem_path)?; + let status_code = self.bucket.get_object_stream(object_path, &mut output_file).await?; + + match status_code { + 404 => bail!("File not found in bucket: {}", object_path), + _ => {}, + } + + info!("download code: {}", status_code); + + Ok(()) + } + + // NB: New version has blocking client rather than blocking calls. // pub fn download_file_blocking(&self, path: &str) -> anyhow::Result> { // info!("downloading from bucket: {}", path); diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs index b5252ccfdc..8410901a2a 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/buckets/bucket_path_unifier.rs @@ -9,6 +9,7 @@ pub struct BucketPathUnifier { pub user_uploaded_audio_for_w2l_root: PathBuf, pub tts_inference_output_root: PathBuf, pub w2l_inference_output_root: PathBuf, + pub w2l_model_root: PathBuf, } impl BucketPathUnifier { @@ -27,6 +28,7 @@ impl BucketPathUnifier { user_uploaded_w2l_templates_root: PathBuf::from("/user_uploaded_w2l_templates"), tts_inference_output_root: PathBuf::from("/tts_inference_output"), w2l_inference_output_root: PathBuf::from("/w2l_inference_output"), + w2l_model_root: PathBuf::from("/w2l_pretrained_models"), } } @@ -34,6 +36,11 @@ impl BucketPathUnifier { "".to_string() } + // W2L pretrained models. There are only two. + pub fn w2l_pretrained_models_path(&self, w2l_model_name: &str) -> PathBuf { + self.w2l_model_root.join(w2l_model_name) + } + // These share the same directory as the uploaded w2l template media. // eg. /user_uploaded_w2l_templates/1/5/1/151a[...60]_detected_faces.pickle pub fn precomputed_faces_for_w2l_path(&self, template_file_hash: &str) -> PathBuf { @@ -64,10 +71,6 @@ impl BucketPathUnifier { .join(video_filename) } - //pub fn pretrained_w2l_model_path(&self, model_name: &str) -> String { - // "".to_string() - //} - pub fn hashed_directory_path(file_hash: &str) -> String { match file_hash.len() { 0 | 1=> "".to_string(), @@ -89,9 +92,17 @@ mod tests { user_uploaded_audio_for_w2l_root: PathBuf::from("/test_path_w2l_audio"), tts_inference_output_root: PathBuf::from("/test_path_tts_output"), w2l_inference_output_root: PathBuf::from("/test_path_w2l_output"), + w2l_model_root: PathBuf::from("/test_path_w2l_pretrained_models"), } } + #[test] + fn test_w2l_pretrained_models_path() { + let paths = get_instance(); + assert_eq!(paths.w2l_pretrained_models_path("model.pth").to_str().unwrap(), + "/test_path_w2l_pretrained_models/model.pth"); + } + #[test] fn test_user_audio_for_w2l_inference_path() { let paths = get_instance(); diff --git a/src/util/semi_persistent_cache_dir.rs b/src/util/semi_persistent_cache_dir.rs index 50abf148a0..488b03b8ff 100644 --- a/src/util/semi_persistent_cache_dir.rs +++ b/src/util/semi_persistent_cache_dir.rs @@ -31,36 +31,50 @@ impl SemiPersistentCacheDir { } } + // ==================== TTS MODELS ==================== + /// We cache TTS models here. + /// We'll likely need to LRU cache them. pub fn tts_model_path(&self) -> &Path { &self.tts_model_root } - /// We cache W2L models here. - pub fn w2l_model_path(&self) -> &Path { + // ==================== W2L MODELS (there are only two of them) ==================== + + /// There are only two pretrained W2L models, so we won't run out of space. + pub fn w2l_model_path(&self, model_filename: &str) -> PathBuf { + self.w2l_model_root.join(model_filename) + } + + pub fn w2l_model_directory(&self) -> &Path { &self.w2l_model_root } - /// Face templates go here. + pub fn create_w2l_model_path(&self) -> AnyhowResult<()> { + let _ = fs::create_dir_all(self.w2l_model_directory())?; + Ok(()) + } + + // ==================== W2L FACES ==================== + + /// We cache W2L faces here. + /// We'll likely need to LRU cache them. pub fn w2l_face_templates_path(&self) -> &Path { &self.w2l_face_templates_root } - /// End bump, etc. - pub fn video_asset_path(&self) -> &Path { - &self.video_asset_root - } - - pub fn create_w2l_model_path(&self) -> AnyhowResult<()> { - let _ = fs::create_dir_all(self.w2l_model_path())?; - Ok(()) - } - pub fn create_w2l_face_templates_path(&self) -> AnyhowResult<()> { let _ = fs::create_dir_all(self.w2l_face_templates_path())?; Ok(()) } + // ==================== VIDEO ASSETS (End bump, etc) ==================== + + /// There is only a single end bump, and we'll add a watermark file. + pub fn video_asset_path(&self) -> &Path { + &self.video_asset_root + } + pub fn create_video_asset_path(&self) -> AnyhowResult<()> { let _ = fs::create_dir_all(self.video_asset_path())?; Ok(()) diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 350de0c48d..76cc89b125 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -85,6 +85,7 @@ struct Inferencer { // Command to run pub inference_script: String, + pub w2l_model_filename: String, } #[tokio::main] @@ -178,6 +179,9 @@ async fn main() -> AnyhowResult<()> { semi_persistent_cache.create_w2l_face_templates_path()?; semi_persistent_cache.create_w2l_model_path()?; + let w2l_model_filename = easyenv::get_env_string_or_default( + "W2L_MODEL_FILENAME", "wav2lip_gan.pth"); + let inferencer = Inferencer { download_temp_directory: temp_directory, mysql_pool, @@ -191,6 +195,7 @@ async fn main() -> AnyhowResult<()> { w2l_inference: w2l_inference_command, bucket_path_unifier: BucketPathUnifier::default_paths(), semi_persistent_cache, + w2l_model_filename, }; main_loop(inferencer).await; @@ -299,6 +304,27 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // TODO 7. Save record // TODO 8. Mark job done + // ==================== CONFIRM OR DOWNLOAD W2L MODEL ==================== // + + let model_filename = inferencer.w2l_model_filename.clone(); + let model_fs_path = inferencer.semi_persistent_cache.w2l_model_path(&model_filename); + + if !model_fs_path.exists() { + info!("Model file does not exist: {:?}", &model_fs_path); + + let model_object_path = inferencer.bucket_path_unifier + .w2l_pretrained_models_path(&model_filename); + + info!("Download from bucket path: {:?}", &model_object_path); + + inferencer.private_bucket_client.download_file_to_disk( + model_object_path.to_str().ok_or(anyhow!("invalid path"))?, + &model_filename).await?; + + info!("Downloaded model from bucket!"); + } + + let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; From 0924dd7cc3ff57a26af0915643467472e9b39ada Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 23:31:47 -0400 Subject: [PATCH 0179/1042] download w2l model to persist dir --- src/job_queries/w2l_inference_job_queries.rs | 43 ++++++++++++++++---- src/w2l_inference_job.rs | 13 ++++-- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index a32bb8c38e..d34fe3c152 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -24,12 +24,17 @@ pub struct W2lInferenceJobRecord { pub maybe_tts_inference_result_token: Option, pub maybe_public_audio_bucket_location: Option, + pub maybe_original_audio_filename: Option, + pub maybe_original_audio_download_url: Option, + pub maybe_audio_mime_type: Option, + pub creator_ip_address: String, pub maybe_creator_user_token: Option, - pub creator_set_visibility: String, // TODO - pub disable_end_bump: bool, - pub disable_watermark: bool, + pub creator_set_visibility: String, // TODO + pub disable_end_bump: i8, // bool + pub disable_watermark: i8, // bool + //pub maybe_subject_token: Option, //pub maybe_actor_subject_token: Option, pub status: String, // TODO @@ -43,12 +48,36 @@ pub struct W2lInferenceJobRecord { pub async fn query_w2l_inference_job_records(pool: &MySqlPool, num_records: u32) -> AnyhowResult> { - /*let job_records = sqlx::query_as!( + let job_records = sqlx::query_as!( W2lInferenceJobRecord, r#" SELECT + id, token AS inference_job_token, - * + uuid_idempotency_token, + + maybe_w2l_template_token, + maybe_public_image_bucket_location, + maybe_tts_inference_result_token, + maybe_public_audio_bucket_location, + + maybe_original_audio_filename, + maybe_original_audio_download_url, + maybe_audio_mime_type, + + creator_ip_address, + maybe_creator_user_token, + + creator_set_visibility, + disable_end_bump, + disable_watermark, + + status, + attempt_count, + failure_reason, + created_at, + updated_at, + retry_at FROM w2l_inference_jobs WHERE ( @@ -67,9 +96,7 @@ WHERE .fetch_all(pool) .await?; - Ok(job_records)*/ - - return Err(anyhow!("TODO")); + Ok(job_records) } pub async fn mark_w2l_inference_job_failure(pool: &MySqlPool, diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 76cc89b125..f780398601 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -63,8 +63,9 @@ const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; const ENV_INFERENCE_SCRIPT_NAME : &'static str = "W2L_INFERENCE_SCRIPT_NAME"; -// NB: sqlx::query is spammy and logs all queries as "info"-level -const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; +/// NB: `sqlx::query` is spammy and logs all queries as "info"-level +/// NB: `hyper::proto::h1::io` is incredibly spammy and logs every chunk of bytes in very large files being downloaded +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn,hyper::proto::h1::io=warn"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Inferencer { @@ -319,7 +320,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An inferencer.private_bucket_client.download_file_to_disk( model_object_path.to_str().ok_or(anyhow!("invalid path"))?, - &model_filename).await?; + &model_fs_path.to_str().ok_or(anyhow!("invalid path"))? + ).await?; info!("Downloaded model from bucket!"); } @@ -328,6 +330,11 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; + if true { + info!("FAKE DONE"); + return Ok(()); + } + //let download_url = job.download_url.as_ref() // .map(|c| c.to_string()) // .unwrap_or("".to_string()); From 4a706cf5a74f66894e7b73481614bce60c139f48 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 May 2021 23:45:41 -0400 Subject: [PATCH 0180/1042] improve ergonomics --- src/buckets/bucket_client.rs | 25 +++++++++++++++++-------- src/w2l_inference_job.rs | 4 ++-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/buckets/bucket_client.rs b/src/buckets/bucket_client.rs index 715055c032..4b52677591 100644 --- a/src/buckets/bucket_client.rs +++ b/src/buckets/bucket_client.rs @@ -1,3 +1,4 @@ +use anyhow::anyhow; use anyhow::bail; use log::info; use log::warn; @@ -7,9 +8,9 @@ use s3::region::Region; use std::io::Read; use std::path::{PathBuf, Path}; use std::str::FromStr; -use tokio::io::{AsyncBufRead, BufReader, AsyncReadExt}; -use tokio::io::AsyncBufReadExt; use tokio::fs::File; +use tokio::io::AsyncBufReadExt; +use tokio::io::{AsyncBufRead, BufReader, AsyncReadExt}; #[derive(Clone)] pub struct BucketClient { @@ -164,16 +165,24 @@ impl BucketClient { Ok(bytes) } - pub async fn download_file_to_disk(&self, object_path: &str, filesystem_path: &str) - -> anyhow::Result<()> - { - info!("downloading from bucket: {}", object_path); + pub async fn download_file_to_disk>( + &self, + object_path: P, + filesystem_path: P, + ) -> anyhow::Result<()> { + let object_path_str = object_path.as_ref() + .to_str() + .map(|s| s.to_string()) + .ok_or(anyhow!("could not convert object path to string"))?; + + info!("downloading from bucket: {:?}", &object_path_str); let mut output_file = std::fs::File::create(filesystem_path)?; - let status_code = self.bucket.get_object_stream(object_path, &mut output_file).await?; + + let status_code = self.bucket.get_object_stream(&object_path_str, &mut output_file).await?; match status_code { - 404 => bail!("File not found in bucket: {}", object_path), + 404 => bail!("File not found in bucket: {}", &object_path_str), _ => {}, } diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index f780398601..ad9c91e711 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -319,8 +319,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An info!("Download from bucket path: {:?}", &model_object_path); inferencer.private_bucket_client.download_file_to_disk( - model_object_path.to_str().ok_or(anyhow!("invalid path"))?, - &model_fs_path.to_str().ok_or(anyhow!("invalid path"))? + &model_object_path, + &model_fs_path ).await?; info!("Downloaded model from bucket!"); From 9e9c4deb34cb1c9fbdd8dff4b21b85b8998a5761 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 01:11:40 -0400 Subject: [PATCH 0181/1042] download face templates --- .../endpoints/w2l/get_w2l_template.rs | 2 +- src/job_queries/w2l_inference_job_queries.rs | 76 ++++++++++++++++++- src/util/semi_persistent_cache_dir.rs | 11 ++- src/w2l_inference_job.rs | 52 ++++++++++++- 4 files changed, 133 insertions(+), 8 deletions(-) diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index a80457dd86..8a076cd62b 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -157,7 +157,7 @@ AND w2l.deleted_at IS NULL return Err(GetW2lTemplateError::ServerError); }, _ => { - warn!("w2l template list query error: {:?}", err); + warn!("w2l template query error: {:?}", err); return Err(GetW2lTemplateError::ServerError); } } diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index d34fe3c152..bdc0904a81 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -3,11 +3,12 @@ //! include these in the main binary to generate all the queries. use anyhow::anyhow; -use chrono::Utc; +use chrono::{Utc, DateTime}; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; -use sqlx::MySqlPool; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use log::{warn, info}; +use sqlx::MySqlPool; /// table: w2l_template_upload_jobs #[derive(Debug)] @@ -229,3 +230,74 @@ SET Ok(record_id)*/ return Err(anyhow!("TODO")); } + +#[derive(Serialize)] +pub struct W2lTemplateRecord2 { + pub template_token: String, + pub template_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub private_bucket_hash: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + pub maybe_public_bucket_preview_image_object_name: Option, + pub maybe_public_bucket_preview_video_object_name: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +pub async fn get_w2l_template_by_token(pool: &MySqlPool, template_token: &str) + -> AnyhowResult> +{ + + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_template = sqlx::query_as!( + W2lTemplateRecord2, + r#" +SELECT + w2l.token as template_token, + w2l.template_type, + w2l.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + w2l.updatable_slug, + w2l.title, + w2l.frame_width, + w2l.frame_height, + w2l.duration_millis, + w2l.private_bucket_hash, + w2l.maybe_public_bucket_preview_image_object_name, + w2l.maybe_public_bucket_preview_video_object_name, + w2l.created_at, + w2l.updated_at +FROM w2l_templates as w2l +JOIN users +ON users.token = w2l.creator_user_token +WHERE w2l.token = ? +AND w2l.deleted_at IS NULL + "#, + &template_token + ) + .fetch_one(pool) + .await; // TODO: This will return error if it doesn't exist + + match maybe_template { + Ok(template) => Ok(Some(template)), + Err(err) => { + match err { + RowNotFound => { + Ok(None) + }, + _ => { + warn!("w2l template query error: {:?}", err); + Err(anyhow!("Mysql error: {:?}", err)) + } + } + } + } +} diff --git a/src/util/semi_persistent_cache_dir.rs b/src/util/semi_persistent_cache_dir.rs index 488b03b8ff..6070f4a0cc 100644 --- a/src/util/semi_persistent_cache_dir.rs +++ b/src/util/semi_persistent_cache_dir.rs @@ -59,12 +59,17 @@ impl SemiPersistentCacheDir { /// We cache W2L faces here. /// We'll likely need to LRU cache them. - pub fn w2l_face_templates_path(&self) -> &Path { + pub fn w2l_face_template_path(&self, template_private_bucket_hash: &str) -> PathBuf { + self.w2l_face_templates_root.join(template_private_bucket_hash) + } + + pub fn w2l_face_template_directory(&self) -> &Path { &self.w2l_face_templates_root } - pub fn create_w2l_face_templates_path(&self) -> AnyhowResult<()> { - let _ = fs::create_dir_all(self.w2l_face_templates_path())?; + + pub fn create_w2l_face_template_path(&self) -> AnyhowResult<()> { + let _ = fs::create_dir_all(self.w2l_face_template_directory())?; Ok(()) } diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index ad9c91e711..53170a02c7 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -19,6 +19,7 @@ use crate::buckets::bucket_path_unifier::BucketPathUnifier; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::buckets::file_hashing::get_file_hash; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; +use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; use crate::job_queries::w2l_inference_job_queries::insert_w2l_result; use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_done; use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failure; @@ -177,7 +178,7 @@ async fn main() -> AnyhowResult<()> { info!("Creating pod semi-persistent cache dirs..."); semi_persistent_cache.create_w2l_model_path()?; - semi_persistent_cache.create_w2l_face_templates_path()?; + semi_persistent_cache.create_w2l_face_template_path()?; semi_persistent_cache.create_w2l_model_path()?; let w2l_model_filename = easyenv::get_env_string_or_default( @@ -295,7 +296,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // TODO 1. Mark Processing // - // TODO: 2. Check if w2l model is downloaded / download it to a stable cache location + // TODO: 2. Check if w2l model is downloaded / download it to a stable cache location (DONE) // TODO: 3. Check if w2l template faces are downloaded and download it // TODO: 4. Download user audio @@ -326,6 +327,53 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An info!("Downloaded model from bucket!"); } + // ==================== CONFIRM OR DOWNLOAD W2L TEMPLATE FACE ==================== // + + // Template is based on the `private_bucket_hash`: + // - private_bucket_hash: 1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 + // - private_bucket_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 + // - private_bucket_cached_faces_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_detected_faces.pickle + // - maybe_public_bucket_preview_image_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_preview.webp + + let template_token = match &job.maybe_w2l_template_token { + Some(token) => token.to_string(), + None => { + warn!("non-template token based inference not yet supported"); + return Err(anyhow!("non-template token based inference not yet supported")) + }, + }; + + info!("Looking up w2l template by token: {}", &template_token); + + let query_result = get_w2l_template_by_token(&inferencer.mysql_pool, &template_token).await?; + + let w2l_template = match query_result { + Some(template) => template, + None => { + warn!("W2L Template not found: {}", &template_token); + return Err(anyhow!("Template not found!")) + }, + }; + + let face_template_fs_path = inferencer.semi_persistent_cache.w2l_face_template_path( + &w2l_template.private_bucket_hash); + + if !face_template_fs_path.exists() { + info!("W2L face template file does not exist: {:?}", &face_template_fs_path); + + let template_object_path = inferencer.bucket_path_unifier + .precomputed_faces_for_w2l_path(&w2l_template.private_bucket_hash); + + info!("Download from face template path: {:?}", &template_object_path); + + inferencer.private_bucket_client.download_file_to_disk( + &template_object_path, + &face_template_fs_path + ).await?; + + info!("Downloaded face template from bucket!"); + } + let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; From e2cc57dd1e850d4081502f2fe577fd0c7e5280f8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 01:28:38 -0400 Subject: [PATCH 0182/1042] change name --- src/util/semi_persistent_cache_dir.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/semi_persistent_cache_dir.rs b/src/util/semi_persistent_cache_dir.rs index 6070f4a0cc..a3e226c863 100644 --- a/src/util/semi_persistent_cache_dir.rs +++ b/src/util/semi_persistent_cache_dir.rs @@ -60,14 +60,14 @@ impl SemiPersistentCacheDir { /// We cache W2L faces here. /// We'll likely need to LRU cache them. pub fn w2l_face_template_path(&self, template_private_bucket_hash: &str) -> PathBuf { - self.w2l_face_templates_root.join(template_private_bucket_hash) + let filename = format!("{}_detected_faces.pickle", template_private_bucket_hash); + self.w2l_face_templates_root.join(filename) } pub fn w2l_face_template_directory(&self) -> &Path { &self.w2l_face_templates_root } - pub fn create_w2l_face_template_path(&self) -> AnyhowResult<()> { let _ = fs::create_dir_all(self.w2l_face_template_directory())?; Ok(()) From b4b8222649f92b5a02c7a6ffa4df99e6f1c7bf5c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 02:27:16 -0400 Subject: [PATCH 0183/1042] download audio --- .../2021-05-15-212913_inference_jobs/up.sql | 1 + .../w2l/enqueue_infer_w2l_with_uploads.rs | 4 ++++ src/job_queries/w2l_inference_job_queries.rs | 3 ++- src/w2l_inference_job.rs | 22 ++++++++++++++++++- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index 54dc4ed46d..50f285c46b 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -96,6 +96,7 @@ CREATE TABLE w2l_inference_jobs ( maybe_tts_inference_result_token VARCHAR(32) DEFAULT NULL, -- If we're using custom uploaded audio, this will be present. + maybe_public_audio_bucket_hash VARCHAR(64) DEFAULT NULL, maybe_public_audio_bucket_location VARCHAR(255) DEFAULT NULL, -- The filename that was used at upload time (if available) diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 341c253d90..4d78fe5589 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -240,6 +240,8 @@ pub async fn enqueue_infer_w2l_with_uploads( let upload_uuid = generate_random_uuid(); + let audio_upload_bucket_hash = upload_uuid.clone(); + let audio_upload_bucket_path = hash_to_bucket_path( &upload_uuid, Some(&server_state.audio_uploads_bucket_root) @@ -278,6 +280,7 @@ SET uuid_idempotency_token = ?, maybe_w2l_template_token = ?, + maybe_public_audio_bucket_hash = ?, maybe_public_audio_bucket_location = ?, maybe_original_audio_filename = ?, @@ -292,6 +295,7 @@ SET job_token.to_string(), uuid_idempotency_token.to_string(), maybe_template_token.clone(), + Some(audio_upload_bucket_hash.clone()), Some(audio_upload_bucket_path.clone()), maybe_audio_file_name.clone(), Some(audio_type.clone()), diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index bdc0904a81..348d356078 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -23,6 +23,7 @@ pub struct W2lInferenceJobRecord { // ===== AUDIO SOURCE OPTIONS ===== pub maybe_tts_inference_result_token: Option, + pub maybe_public_audio_bucket_hash: Option, pub maybe_public_audio_bucket_location: Option, pub maybe_original_audio_filename: Option, @@ -60,6 +61,7 @@ SELECT maybe_w2l_template_token, maybe_public_image_bucket_location, maybe_tts_inference_result_token, + maybe_public_audio_bucket_hash, maybe_public_audio_bucket_location, maybe_original_audio_filename, @@ -231,7 +233,6 @@ SET return Err(anyhow!("TODO")); } -#[derive(Serialize)] pub struct W2lTemplateRecord2 { pub template_token: String, pub template_type: String, diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 53170a02c7..d8000edd4f 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -297,7 +297,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // TODO 1. Mark Processing // // TODO: 2. Check if w2l model is downloaded / download it to a stable cache location (DONE) - // TODO: 3. Check if w2l template faces are downloaded and download it + // TODO: 3. Check if w2l template faces are downloaded and download it (done) // TODO: 4. Download user audio // TODO: 5. Process Inference @@ -374,10 +374,30 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An info!("Downloaded face template from bucket!"); } + // ==================== DOWNLOAD USER AUDIO ==================== // let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; + let audio_bucket_hash = match &job.maybe_public_audio_bucket_hash { + Some(l) => l.clone(), + None => { + warn!("Only W2L jobs with user-uploaded audio are supported right now"); + return Err(anyhow!("Only W2L jobs with user-uploaded audio are supported right now")) + }, + }; + + let audio_fs_path = temp_dir.path().join(&audio_bucket_hash); + + let audio_object_path = inferencer.bucket_path_unifier + .user_audio_for_w2l_inference_path(&audio_bucket_hash); + + inferencer.private_bucket_client.download_file_to_disk( + &audio_object_path, + &audio_fs_path + ).await?; + + if true { info!("FAKE DONE"); return Ok(()); From 6d7f9ba8724f790f51a216f5a4826af628652541 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 02:41:57 -0400 Subject: [PATCH 0184/1042] inference job (doesn't work) --- models/Wav2Lip/vocodes_process_inference.py | 458 ++++++++++++++++++ .../wav2lip_inference_command.rs | 15 +- src/w2l_inference_job.rs | 14 +- 3 files changed, 478 insertions(+), 9 deletions(-) create mode 100644 models/Wav2Lip/vocodes_process_inference.py diff --git a/models/Wav2Lip/vocodes_process_inference.py b/models/Wav2Lip/vocodes_process_inference.py new file mode 100644 index 0000000000..7fd4a2032d --- /dev/null +++ b/models/Wav2Lip/vocodes_process_inference.py @@ -0,0 +1,458 @@ +from os import listdir, path +import numpy as np +import scipy, cv2, os, sys, argparse, audio +import json, subprocess, random, string +from tqdm import tqdm +from glob import glob +import torch, face_detection +from models import Wav2Lip +import tempfile +import shutil +import datetime +import pickle +from PIL import Image + +if 'PRINT_ENV' in os.environ and os.environ['PRINT_ENV']: + for k, v in os.environ.items(): + print('{}={}'.format(k, v)) + +print('========================================') +print('Python interpreter', sys.executable) +print('PyTorch version', torch.__version__) +print('CUDA Available?', torch.cuda.is_available()) +print('CUDA Device count', torch.cuda.device_count()) +print('========================================', flush=True) + +parser = argparse.ArgumentParser(description='Inference code to lip-sync videos in the wild using Wav2Lip models') + +parser.add_argument('--checkpoint_path', type=str, + help='Name of saved checkpoint to load weights from', required=True) + +parser.add_argument('--face', type=str, + help='Filepath of video/image that contains faces to use', required=True) +parser.add_argument('--audio', type=str, + help='Filepath of video/audio file to use as raw audio source', required=True) +parser.add_argument('--outfile', type=str, help='Video path to save result. See default for an e.g.', + default='results/result_voice.mp4') + +parser.add_argument('--static', type=bool, + help='If True, then use only first video frame for inference', default=False) + +# NB: I'm adjusting the FPS to match that of the end bump I made. The old default FPS was "25." (float 25) +parser.add_argument('--fps', type=float, help='Can be specified only if input is a static image (default: 29.97)', + default=29.97, required=False) + +parser.add_argument('--pads', nargs='+', type=int, default=[0, 10, 0, 0], + help='Padding (top, bottom, left, right). Please adjust to include chin at least') + +parser.add_argument('--face_det_batch_size', type=int, + help='Batch size for face detection', default=16) +parser.add_argument('--wav2lip_batch_size', type=int, help='Batch size for Wav2Lip model(s)', default=128) + +parser.add_argument('--resize_factor', default=1, type=int, + help='Reduce the resolution by this factor. Sometimes, best results are obtained at 480p or 720p') + +parser.add_argument('--crop', nargs='+', type=int, default=[0, -1, 0, -1], + help='Crop video to a smaller region (top, bottom, left, right). Applied after resize_factor and rotate arg. ' + 'Useful if multiple face present. -1 implies the value will be auto-inferred based on height, width') + +parser.add_argument('--box', nargs='+', type=int, default=[-1, -1, -1, -1], + help='Specify a constant bounding box for the face. Use only as a last resort if the face is not detected.' + 'Also, might work only if the face is not moving around much. Syntax: (top, bottom, left, right).') + +parser.add_argument('--rotate', default=False, action='store_true', + help='Sometimes videos taken from a phone can be flipped 90deg. If true, will flip video right by 90deg.' + 'Use if you get a flipped result, despite feeding a normal looking video') + +parser.add_argument('--nosmooth', default=False, action='store_true', + help='Prevent smoothing face detections over a short temporal window') + +parser.add_argument('--audio_start_pad_millis', default=0, type=int, + help='Seconds to pad the start of the audio') + +parser.add_argument('--audio_end_pad_millis', default=0, type=int, + help='Seconds to pad the end of the audio') + +parser.add_argument('--end_bump_file', type=str, + help='Video file to concatenate at the end') + +# This is a little more deliberate than "static" and causes us to pick an FPS, etc. +# This is useful when the filename doesn't have an extension. +parser.add_argument('--is_image', default=False, action='store_true', + help='Denote that the input is a single-frame image, not a video') + +# Purely for debugging on the host machine: +parser.add_argument('--preserve_tempdir', default=False, action='store_true', + help='Keep the tempdir arround for debugging') + +args = parser.parse_args() +args.img_size = 96 + +if os.path.isfile(args.face) and os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg']: + args.static = True +elif args.is_image: + args.static = True + +def get_smoothened_boxes(boxes, T): + for i in range(len(boxes)): + if i + T > len(boxes): + window = boxes[len(boxes) - T:] + else: + window = boxes[i : i + T] + boxes[i] = np.mean(window, axis=0) + return boxes + +def face_detect(images): + detector = face_detection.FaceAlignment(face_detection.LandmarksType._2D, + flip_input=False, device=device) + + batch_size = args.face_det_batch_size + + while 1: + predictions = [] + try: + for i in tqdm(range(0, len(images), batch_size)): + predictions.extend(detector.get_detections_for_batch(np.array(images[i:i + batch_size]))) + except RuntimeError: + if batch_size == 1: + raise RuntimeError('Image too big to run face detection on GPU. Please use the --resize_factor argument') + batch_size //= 2 + print('Recovering from OOM error; New batch size: {}'.format(batch_size)) + continue + break + + results = [] + pady1, pady2, padx1, padx2 = args.pads + for rect, image in zip(predictions, images): + if rect is None: + cv2.imwrite('temp/faulty_frame.jpg', image) # check this frame where the face was not detected. + raise ValueError('Face not detected! Ensure the video contains a face in all the frames.') + + y1 = max(0, rect[1] - pady1) + y2 = min(image.shape[0], rect[3] + pady2) + x1 = max(0, rect[0] - padx1) + x2 = min(image.shape[1], rect[2] + padx2) + + results.append([x1, y1, x2, y2]) + + boxes = np.array(results) + if not args.nosmooth: boxes = get_smoothened_boxes(boxes, T=5) + results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2)] for image, (x1, y1, x2, y2) in zip(images, boxes)] + + del detector + return results + +def detect_faces_in_frames(frames, video_faces_pickle_file): + t0 = datetime.datetime.now() + if args.box[0] == -1: + if not args.static: + face_det_results = face_detect(frames) # BGR2RGB for CNN face detection + else: + face_det_results = face_detect([frames[0]]) + else: + print('Using the specified bounding box instead of face detection...') + y1, y2, x1, x2 = args.box + face_det_results = [[f[y1: y2, x1:x2], (y1, y2, x1, x2)] for f in frames] + t1 = datetime.datetime.now() + + d = t1 - t0 + print('Total seconds to detect faces in {} frames: {}'.format(len(frames), d.total_seconds())) + + with open(video_faces_pickle_file, 'wb') as f: + print('Saving pickle file: {}'.format(video_faces_pickle_file)) + pickle.dump(face_det_results, f) + + return face_det_results + +#def datagen(frames, mels): +def datagen(frames, face_det_results, mels): + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + #face_det_results = detect_faces_in_frames(frames) + + for i, m in enumerate(mels): + idx = 0 if args.static else i%len(frames) + frame_to_save = frames[idx].copy() + face, coords = face_det_results[idx].copy() + + face = cv2.resize(face, (args.img_size, args.img_size)) + + img_batch.append(face) + mel_batch.append(m) + frame_batch.append(frame_to_save) + coords_batch.append(coords) + + if len(img_batch) >= args.wav2lip_batch_size: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] + + if len(img_batch) > 0: + img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch) + + img_masked = img_batch.copy() + img_masked[:, args.img_size//2:] = 0 + + img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255. + mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) + + yield img_batch, mel_batch, frame_batch, coords_batch + +mel_step_size = 16 +device = 'cuda' if torch.cuda.is_available() else 'cpu' +print('Using {} for inference.'.format(device)) + +def _load(checkpoint_path): + if device == 'cuda': + checkpoint = torch.load(checkpoint_path) + else: + checkpoint = torch.load(checkpoint_path, + map_location=lambda storage, loc: storage) + return checkpoint + +def load_model(path): + model = Wav2Lip() + print("Load checkpoint from: {}".format(path), flush=True) + checkpoint = _load(path) + s = checkpoint["state_dict"] + new_s = {} + for k, v in s.items(): + new_s[k.replace('module.', '')] = v + model.load_state_dict(new_s) + + model = model.to(device) + return model.eval() + +def maybe_pad_audio_file(tempdir, args): + if not args.audio_start_pad_millis and not args.audio_end_pad_millis: + return + + print('Padding audio with {} millis at start, {} millis at end.'.format( + args.audio_start_pad_millis, args.audio_end_pad_millis), flush=True) + + padded_wav_filename = os.path.join(tempdir, 'padded_audio.wav') + # ffmpeg padding: https://superuser.com/a/579110 + # Structure of filters: https://stackoverflow.com/a/55463101 + command = ' '.join([ + 'ffmpeg', + '-i {}'.format(args.audio), + '-filter_complex "[0] adelay={}ms:all=true [a] ; [a] apad=pad_dur={}ms"'.format(args.audio_start_pad_millis, args.audio_end_pad_millis), + '{}'.format(padded_wav_filename) + ]) + print('command:', command, flush=True) + subprocess.call(command, shell=True) + args.audio = padded_wav_filename + + +def maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h): + if not args.end_bump_file: + print('No end bump file.', flush=True) + return + + print('Adding end bump file: {}'.format(args.end_bump_file), flush=True) + + bumpfile = args.end_bump_file + + # May need to resize our bump file... + # After many attempts to get this into the same ffmpeg pipeline, + # I'm giving up and just adding another step. + # + # NOTE: The FPS of the bump needs to match the video. Especially + # important for videos generated from still photos, because that defaults + # to the odd choice of "25 fps". + if frame_w is not 1920 or frame_h is not 1080: + print('Need to resize bump.', flush=True) + bumpfile = os.path.join(tempdir, 'end_bump_resized.mp4') + + # Scale / aspect ratio / sar / resizing / padding: + # https://stackoverflow.com/a/51946719 + command = ' '.join([ + 'ffmpeg', + '-i {}'.format(args.end_bump_file), + '-vf "scale={}:{}:force_original_aspect_ratio=decrease,pad={}:{}:(ow-iw)/2:(oh-ih)/2,setsar=1"'.format(frame_w, frame_h, frame_w, frame_h), + '{}'.format(bumpfile) + ]) + print('command:', command, flush=True) + failure_code = subprocess.call(command, shell=True) + + if failure_code: + print('Failed to resize end bump') + return + + padded_wav_filename = os.path.join(tempdir, 'output_with_end_bump.mp4') + # ffmpeg video concatenation: https://stackoverflow.com/a/11175851 + # Structure of filters: https://stackoverflow.com/a/55463101 + # More info on filters: https://stackoverflow.com/a/22958746 + command = ' '.join([ + 'ffmpeg', + '-i {}'.format(args.outfile), + '-i {}'.format(bumpfile), + '-filter_complex "concat=n=2:v=1:a=1"', + '{}'.format(padded_wav_filename) + ]) + print('command:', command, flush=True) + failure_code = subprocess.call(command, shell=True) + + if failure_code: + print('Failed to concatenate end bump') + else: + print('Done concatenating end bump; renaming file...', flush=True) + os.rename(padded_wav_filename, args.outfile) + + +def main(tempdir): + video_faces_pickle_file = args.face + '.faces' + print('Video faces pickle file: {}'.format(video_faces_pickle_file), flush=True) + + frame_w = 0 + frame_h = 0 + + if not os.path.isfile(args.face): + raise ValueError('--face argument must be a valid path to video/image file') + + elif os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg'] \ + or args.is_image: + full_frames = [cv2.imread(args.face)] + fps = args.fps + frame_h, frame_w = full_frames[0].shape[:-1] + + else: + video_stream = cv2.VideoCapture(args.face) + fps = video_stream.get(cv2.CAP_PROP_FPS) + + print('Reading video frames...') + + full_frames = [] + while 1: + still_reading, frame = video_stream.read() + if not still_reading: + video_stream.release() + break + if args.resize_factor > 1: + frame = cv2.resize(frame, (frame.shape[1]//args.resize_factor, frame.shape[0]//args.resize_factor)) + + if args.rotate: + frame = cv2.rotate(frame, cv2.cv2.ROTATE_90_CLOCKWISE) + + y1, y2, x1, x2 = args.crop + if x2 == -1: x2 = frame.shape[1] + if y2 == -1: y2 = frame.shape[0] + + frame = frame[y1:y2, x1:x2] + + full_frames.append(frame) + + frame_h, frame_w = full_frames[0].shape[:-1] + + print("Number of frames available for inference: "+str(len(full_frames)), flush=True) + print("Frame dimensions: {}x{}".format(frame_w, frame_h), flush=True) + + if not args.audio.endswith('.wav'): + print('Extracting raw audio...') + temp_wav_filename = os.path.join(tempdir, 'temp.wav') # previously 'temp/temp.wav' + command = 'ffmpeg -y -i {} -strict -2 {}'.format(args.audio, temp_wav_filename) + + print('command:', command, flush=True) + subprocess.call(command, shell=True) + args.audio = temp_wav_filename + + maybe_pad_audio_file(tempdir, args) + + wav = audio.load_wav(args.audio, 16000) + mel = audio.melspectrogram(wav) + print(mel.shape) + + if np.isnan(mel.reshape(-1)).sum() > 0: + raise ValueError('Mel contains nan! Using a TTS voice? Add a small epsilon noise to the wav file and try again') + + mel_chunks = [] + mel_idx_multiplier = 80./fps + i = 0 + while 1: + start_idx = int(i * mel_idx_multiplier) + if start_idx + mel_step_size > len(mel[0]): + mel_chunks.append(mel[:, len(mel[0]) - mel_step_size:]) + break + mel_chunks.append(mel[:, start_idx : start_idx + mel_step_size]) + i += 1 + + print("Length of mel chunks: {}".format(len(mel_chunks)), flush=True) + + if len(mel_chunks) > 2000: + # TODO: Catch that the file is too long earlier on. + raise Exception("Too many mel chunks: {}".format(len(mel_chunks))) + + full_frames = full_frames[:len(mel_chunks)] + + # TODO: Do this offline. + print('Face detection time...', flush=True) + if not os.path.isfile(video_faces_pickle_file): + print('Detecting faces...', flush=True) + face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) + else: + with open(video_faces_pickle_file, 'rb') as f: + face_det_results = pickle.load(f) + if len(face_det_results) < len(full_frames): + print("Face detect results have {} frames, but we need {}".format(len(face_det_results), len(full_frames))) + print('Detecting faces again...', flush=True) + face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) + else: + print("We don't need to find faces! Woo!", flush=True) + + face_det_results = face_det_results[:len(mel_chunks)] + + batch_size = args.wav2lip_batch_size + #gen = datagen(full_frames.copy(), mel_chunks) + gen = datagen(full_frames.copy(), face_det_results.copy(), mel_chunks) + + output_video_filename = os.path.join(tempdir, 'result.avi') # previously 'temp/result.avi' + + for i, (img_batch, mel_batch, frames, coords) in enumerate(tqdm(gen, + total=int(np.ceil(float(len(mel_chunks))/batch_size)))): + if i == 0: + model = load_model(args.checkpoint_path) + print ("Model loaded") + + #frame_h, frame_w = full_frames[0].shape[:-1] + out = cv2.VideoWriter(output_video_filename, + cv2.VideoWriter_fourcc(*'DIVX'), fps, (frame_w, frame_h)) + + img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device) + mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device) + + with torch.no_grad(): + pred = model(mel_batch, img_batch) + + pred = pred.cpu().numpy().transpose(0, 2, 3, 1) * 255. + + for p, f, c in zip(pred, frames, coords): + y1, y2, x1, x2 = c + p = cv2.resize(p.astype(np.uint8), (x2 - x1, y2 - y1)) + + f[y1:y2, x1:x2] = p + out.write(f) + + out.release() + + command = 'ffmpeg -y -i {} -i {} -strict -2 -q:v 1 {}'.format(args.audio, output_video_filename, args.outfile) + print('command:', command, flush=True) + subprocess.call(command, shell=True) + + maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h) + +if __name__ == '__main__': + tempdir = tempfile.mkdtemp() + print('Using Python tempdir: {}'.format(tempdir), flush=True) + try: + main(tempdir) + finally: + if not args.preserve_tempdir: + shutil.rmtree(tempdir) + diff --git a/src/script_execution/wav2lip_inference_command.rs b/src/script_execution/wav2lip_inference_command.rs index 361e6d89f1..5c48c9ee13 100644 --- a/src/script_execution/wav2lip_inference_command.rs +++ b/src/script_execution/wav2lip_inference_command.rs @@ -3,6 +3,7 @@ use log::{info,warn}; use std::process::{Command, Stdio}; use subprocess::{Popen, PopenConfig, Redirection}; use std::fs::OpenOptions; +use std::path::Path; /// This command is used to run inference. /// It uses preprocessed face files so that it's much faster. @@ -26,10 +27,10 @@ impl Wav2LipInferenceCommand { } } - pub fn execute(&self, - audio_filename: &str, - output_cached_faces_filename: &str, - output_metadata_filename: &str, + pub fn execute>(&self, + audio_filename: P, + output_cached_faces_filename: P, + output_metadata_filename: P, is_image: bool, spawn_process: bool) -> AnyhowResult<()> { @@ -46,11 +47,11 @@ impl Wav2LipInferenceCommand { command.push_str(" --checkpoint_path "); command.push_str(&self.checkpoint_path); command.push_str(" --audio_filename"); - command.push_str(audio_filename); + command.push_str(&audio_filename.as_ref().display().to_string()); command.push_str(" --cached_faces_filename "); - command.push_str(output_cached_faces_filename); + command.push_str(&output_cached_faces_filename.as_ref().display().to_string()); command.push_str(" --output_metadata_filename "); - command.push_str(output_metadata_filename); + command.push_str(&output_metadata_filename.as_ref().display().to_string()); if is_image { command.push_str(" --is_image "); diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index d8000edd4f..eb57547605 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -298,7 +298,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // // TODO: 2. Check if w2l model is downloaded / download it to a stable cache location (DONE) // TODO: 3. Check if w2l template faces are downloaded and download it (done) - // TODO: 4. Download user audio + // TODO: 4. Download user audio (done) // TODO: 5. Process Inference @@ -377,7 +377,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // ==================== DOWNLOAD USER AUDIO ==================== // let temp_dir = format!("temp_{}", job.id); - let temp_dir = TempDir::new(&temp_dir)?; + let temp_dir = TempDir::new(&temp_dir)?; // NB: Exists until it goes out of scope. let audio_bucket_hash = match &job.maybe_public_audio_bucket_hash { Some(l) => l.clone(), @@ -398,6 +398,16 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An ).await?; + // ==================== RUN INFERENCE ==================== // + + inferencer.w2l_inference.execute( + &audio_fs_path, + &face_template_fs_path, + &PathBuf::from("todo"), + false, + false + )?; + if true { info!("FAKE DONE"); return Ok(()); From 62b6b70e361744158fed5ff2d54f454cbffbe40e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 12:07:05 -0400 Subject: [PATCH 0185/1042] download template media, start work on process script --- models/Wav2Lip/vocodes_process_inference.py | 213 +++++++++++--------- models/Wav2Lip/vocodes_process_upload.py | 38 ++-- src/buckets/bucket_path_unifier.rs | 16 ++ src/util/semi_persistent_cache_dir.rs | 21 +- src/w2l_inference_job.rs | 51 ++++- 5 files changed, 215 insertions(+), 124 deletions(-) diff --git a/models/Wav2Lip/vocodes_process_inference.py b/models/Wav2Lip/vocodes_process_inference.py index 7fd4a2032d..1e0b0f5a85 100644 --- a/models/Wav2Lip/vocodes_process_inference.py +++ b/models/Wav2Lip/vocodes_process_inference.py @@ -28,8 +28,22 @@ parser = argparse.ArgumentParser(description='Inference code to lip-sync videos parser.add_argument('--checkpoint_path', type=str, help='Name of saved checkpoint to load weights from', required=True) -parser.add_argument('--face', type=str, +parser.add_argument('--image_or_video_filename', type=str, help='Filepath of video/image that contains faces to use', required=True) + +#parser.add_argument('--image_or_video_type', type=str, +# help='either "image" or "video', required=True) + +# This is a little more deliberate than "static" and causes us to pick an FPS, etc. +# This is useful when the filename doesn't have an extension. +parser.add_argument('--is_image', default=False, action='store_true', + help='Denote that the input is a single-frame image, not a video') + +parser.add_argument('--cached_faces_filename', type=str, + help='Filename for the cached faces file to use', required=True) + +#parser.add_argument('--face', type=str, +# help='Filepath of video/image that contains faces to use', required=True) parser.add_argument('--audio', type=str, help='Filepath of video/audio file to use as raw audio source', required=True) parser.add_argument('--outfile', type=str, help='Video path to save result. See default for an e.g.', @@ -76,93 +90,92 @@ parser.add_argument('--audio_end_pad_millis', default=0, type=int, parser.add_argument('--end_bump_file', type=str, help='Video file to concatenate at the end') -# This is a little more deliberate than "static" and causes us to pick an FPS, etc. -# This is useful when the filename doesn't have an extension. -parser.add_argument('--is_image', default=False, action='store_true', - help='Denote that the input is a single-frame image, not a video') # Purely for debugging on the host machine: parser.add_argument('--preserve_tempdir', default=False, action='store_true', help='Keep the tempdir arround for debugging') +#parser.add_argument('--frame_width', type=int, help='Width of the frame') +#parser.add_argument('--frame_height', type=int, help='Height of the frame') + args = parser.parse_args() args.img_size = 96 -if os.path.isfile(args.face) and os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg']: - args.static = True -elif args.is_image: - args.static = True +#if os.path.isfile(args.face) and os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg']: +# args.static = True +#elif args.is_image: +# args.static = True -def get_smoothened_boxes(boxes, T): - for i in range(len(boxes)): - if i + T > len(boxes): - window = boxes[len(boxes) - T:] - else: - window = boxes[i : i + T] - boxes[i] = np.mean(window, axis=0) - return boxes +#def get_smoothened_boxes(boxes, T): +# for i in range(len(boxes)): +# if i + T > len(boxes): +# window = boxes[len(boxes) - T:] +# else: +# window = boxes[i : i + T] +# boxes[i] = np.mean(window, axis=0) +# return boxes -def face_detect(images): - detector = face_detection.FaceAlignment(face_detection.LandmarksType._2D, - flip_input=False, device=device) +#def face_detect(images): +# detector = face_detection.FaceAlignment(face_detection.LandmarksType._2D, +# flip_input=False, device=device) +# +# batch_size = args.face_det_batch_size +# +# while 1: +# predictions = [] +# try: +# for i in tqdm(range(0, len(images), batch_size)): +# predictions.extend(detector.get_detections_for_batch(np.array(images[i:i + batch_size]))) +# except RuntimeError: +# if batch_size == 1: +# raise RuntimeError('Image too big to run face detection on GPU. Please use the --resize_factor argument') +# batch_size //= 2 +# print('Recovering from OOM error; New batch size: {}'.format(batch_size)) +# continue +# break +# +# results = [] +# pady1, pady2, padx1, padx2 = args.pads +# for rect, image in zip(predictions, images): +# if rect is None: +# cv2.imwrite('temp/faulty_frame.jpg', image) # check this frame where the face was not detected. +# raise ValueError('Face not detected! Ensure the video contains a face in all the frames.') +# +# y1 = max(0, rect[1] - pady1) +# y2 = min(image.shape[0], rect[3] + pady2) +# x1 = max(0, rect[0] - padx1) +# x2 = min(image.shape[1], rect[2] + padx2) +# +# results.append([x1, y1, x2, y2]) +# +# boxes = np.array(results) +# if not args.nosmooth: boxes = get_smoothened_boxes(boxes, T=5) +# results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2)] for image, (x1, y1, x2, y2) in zip(images, boxes)] +# +# del detector +# return results - batch_size = args.face_det_batch_size - - while 1: - predictions = [] - try: - for i in tqdm(range(0, len(images), batch_size)): - predictions.extend(detector.get_detections_for_batch(np.array(images[i:i + batch_size]))) - except RuntimeError: - if batch_size == 1: - raise RuntimeError('Image too big to run face detection on GPU. Please use the --resize_factor argument') - batch_size //= 2 - print('Recovering from OOM error; New batch size: {}'.format(batch_size)) - continue - break - - results = [] - pady1, pady2, padx1, padx2 = args.pads - for rect, image in zip(predictions, images): - if rect is None: - cv2.imwrite('temp/faulty_frame.jpg', image) # check this frame where the face was not detected. - raise ValueError('Face not detected! Ensure the video contains a face in all the frames.') - - y1 = max(0, rect[1] - pady1) - y2 = min(image.shape[0], rect[3] + pady2) - x1 = max(0, rect[0] - padx1) - x2 = min(image.shape[1], rect[2] + padx2) - - results.append([x1, y1, x2, y2]) - - boxes = np.array(results) - if not args.nosmooth: boxes = get_smoothened_boxes(boxes, T=5) - results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2)] for image, (x1, y1, x2, y2) in zip(images, boxes)] - - del detector - return results - -def detect_faces_in_frames(frames, video_faces_pickle_file): - t0 = datetime.datetime.now() - if args.box[0] == -1: - if not args.static: - face_det_results = face_detect(frames) # BGR2RGB for CNN face detection - else: - face_det_results = face_detect([frames[0]]) - else: - print('Using the specified bounding box instead of face detection...') - y1, y2, x1, x2 = args.box - face_det_results = [[f[y1: y2, x1:x2], (y1, y2, x1, x2)] for f in frames] - t1 = datetime.datetime.now() - - d = t1 - t0 - print('Total seconds to detect faces in {} frames: {}'.format(len(frames), d.total_seconds())) - - with open(video_faces_pickle_file, 'wb') as f: - print('Saving pickle file: {}'.format(video_faces_pickle_file)) - pickle.dump(face_det_results, f) - - return face_det_results +#def detect_faces_in_frames(frames, video_faces_pickle_file): +# t0 = datetime.datetime.now() +# if args.box[0] == -1: +# if not args.static: +# face_det_results = face_detect(frames) # BGR2RGB for CNN face detection +# else: +# face_det_results = face_detect([frames[0]]) +# else: +# print('Using the specified bounding box instead of face detection...') +# y1, y2, x1, x2 = args.box +# face_det_results = [[f[y1: y2, x1:x2], (y1, y2, x1, x2)] for f in frames] +# t1 = datetime.datetime.now() +# +# d = t1 - t0 +# print('Total seconds to detect faces in {} frames: {}'.format(len(frames), d.total_seconds())) +# +# with open(video_faces_pickle_file, 'wb') as f: +# print('Saving pickle file: {}'.format(video_faces_pickle_file)) +# pickle.dump(face_det_results, f) +# +# return face_det_results #def datagen(frames, mels): def datagen(frames, face_det_results, mels): @@ -308,8 +321,9 @@ def maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h): def main(tempdir): - video_faces_pickle_file = args.face + '.faces' - print('Video faces pickle file: {}'.format(video_faces_pickle_file), flush=True) + video_faces_pickle_file = args.cached_faces_filename + #video_faces_pickle_file = args.face + '.faces' + #print('Video faces pickle file: {}'.format(video_faces_pickle_file), flush=True) frame_w = 0 frame_h = 0 @@ -351,9 +365,13 @@ def main(tempdir): frame_h, frame_w = full_frames[0].shape[:-1] + #frame_w = args.frame_width + #frame_h = args.frame_height + print("Number of frames available for inference: "+str(len(full_frames)), flush=True) print("Frame dimensions: {}x{}".format(frame_w, frame_h), flush=True) + # TODO(bt): This could be more efficient if not args.audio.endswith('.wav'): print('Extracting raw audio...') temp_wav_filename = os.path.join(tempdir, 'temp.wav') # previously 'temp/temp.wav' @@ -365,6 +383,7 @@ def main(tempdir): maybe_pad_audio_file(tempdir, args) + # TODO(bt): Save the spectrogram? wav = audio.load_wav(args.audio, 16000) mel = audio.melspectrogram(wav) print(mel.shape) @@ -386,26 +405,32 @@ def main(tempdir): print("Length of mel chunks: {}".format(len(mel_chunks)), flush=True) if len(mel_chunks) > 2000: - # TODO: Catch that the file is too long earlier on. + # TODO(bt): Catch that the file is too long earlier on. raise Exception("Too many mel chunks: {}".format(len(mel_chunks))) full_frames = full_frames[:len(mel_chunks)] - # TODO: Do this offline. - print('Face detection time...', flush=True) - if not os.path.isfile(video_faces_pickle_file): - print('Detecting faces...', flush=True) - face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) - else: - with open(video_faces_pickle_file, 'rb') as f: - face_det_results = pickle.load(f) - if len(face_det_results) < len(full_frames): - print("Face detect results have {} frames, but we need {}".format(len(face_det_results), len(full_frames))) - print('Detecting faces again...', flush=True) - face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) - else: - print("We don't need to find faces! Woo!", flush=True) + #print('Face detection time...', flush=True) + #if not os.path.isfile(video_faces_pickle_file): + # print('Detecting faces...', flush=True) + # face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) + #else: + # with open(video_faces_pickle_file, 'rb') as f: + # face_det_results = pickle.load(f) + # if len(face_det_results) < len(full_frames): + # print("Face detect results have {} frames, but we need {}".format(len(face_det_results), len(full_frames))) + # print('Detecting faces again...', flush=True) + # face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) + # else: + # print("We don't need to find faces! Woo!", flush=True) + + # Load face detection results from pickle file + video_faces_pickle_file = args.cached_faces_filename + with open(video_faces_pickle_file, 'rb') as f: + face_det_results = pickle.load(f) + + #face_det_results = pickle.load(args.cached_faces_filename) face_det_results = face_det_results[:len(mel_chunks)] batch_size = args.wav2lip_batch_size diff --git a/models/Wav2Lip/vocodes_process_upload.py b/models/Wav2Lip/vocodes_process_upload.py index dc4be013f0..5261ffd0e1 100644 --- a/models/Wav2Lip/vocodes_process_upload.py +++ b/models/Wav2Lip/vocodes_process_upload.py @@ -243,26 +243,26 @@ mel_step_size = 16 device = 'cuda' if torch.cuda.is_available() else 'cpu' print('Using {} for inference.'.format(device)) -def _load(checkpoint_path): - if device == 'cuda': - checkpoint = torch.load(checkpoint_path) - else: - checkpoint = torch.load(checkpoint_path, - map_location=lambda storage, loc: storage) - return checkpoint +#def _load(checkpoint_path): +# if device == 'cuda': +# checkpoint = torch.load(checkpoint_path) +# else: +# checkpoint = torch.load(checkpoint_path, +# map_location=lambda storage, loc: storage) +# return checkpoint -def load_model(path): - model = Wav2Lip() - print("Load checkpoint from: {}".format(path), flush=True) - checkpoint = _load(path) - s = checkpoint["state_dict"] - new_s = {} - for k, v in s.items(): - new_s[k.replace('module.', '')] = v - model.load_state_dict(new_s) - - model = model.to(device) - return model.eval() +#def load_model(path): +# model = Wav2Lip() +# print("Load checkpoint from: {}".format(path), flush=True) +# checkpoint = _load(path) +# s = checkpoint["state_dict"] +# new_s = {} +# for k, v in s.items(): +# new_s[k.replace('module.', '')] = v +# model.load_state_dict(new_s) +# +# model = model.to(device) +# return model.eval() def main(tempdir): diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs index 8410901a2a..b314715a99 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/buckets/bucket_path_unifier.rs @@ -41,6 +41,15 @@ impl BucketPathUnifier { self.w2l_model_root.join(w2l_model_name) } + // The video or images uploaded as templates + // eg. /user_uploaded_w2l_templates/1/5/1/151a[...60]... + pub fn media_templates_for_w2l_path(&self, template_file_hash: &str) -> PathBuf { + let hashed_path = Self::hashed_directory_path(template_file_hash); + self.user_uploaded_w2l_templates_root + .join(hashed_path) + .join(template_file_hash) + } + // These share the same directory as the uploaded w2l template media. // eg. /user_uploaded_w2l_templates/1/5/1/151a[...60]_detected_faces.pickle pub fn precomputed_faces_for_w2l_path(&self, template_file_hash: &str) -> PathBuf { @@ -110,6 +119,13 @@ mod tests { "/test_path_w2l_audio/f/o/o/foobar"); } + #[test] + fn test_media_templates_for_w2l_path() { + let paths = get_instance(); + assert_eq!(paths.media_templates_for_w2l_path("foobar").to_str().unwrap(), + "/test_path_w2l_templates/f/o/o/foobar"); + } + #[test] fn test_precomputed_faces_for_w2l_path() { let paths = get_instance(); diff --git a/src/util/semi_persistent_cache_dir.rs b/src/util/semi_persistent_cache_dir.rs index a3e226c863..fd03496b03 100644 --- a/src/util/semi_persistent_cache_dir.rs +++ b/src/util/semi_persistent_cache_dir.rs @@ -8,6 +8,7 @@ pub struct SemiPersistentCacheDir { tts_model_root: PathBuf, w2l_model_root: PathBuf, w2l_face_templates_root: PathBuf, + w2l_templates_media_root: PathBuf, video_asset_root: PathBuf, // end bump, etc. } @@ -27,6 +28,7 @@ impl SemiPersistentCacheDir { tts_model_root: cache_root.join("tts/models/"), w2l_model_root: cache_root.join("w2l/models/"), w2l_face_templates_root: cache_root.join("w2l/face_templates/"), + w2l_templates_media_root: cache_root.join("w2l/template_media/"), video_asset_root: cache_root.join("static_video_assets/"), } } @@ -55,7 +57,24 @@ impl SemiPersistentCacheDir { Ok(()) } - // ==================== W2L FACES ==================== + // ==================== W2L MEDIA ==================== + + /// We cache W2L media here. + /// We'll likely need to LRU cache them. + pub fn w2l_template_media_path(&self, template_private_bucket_hash: &str) -> PathBuf { + self.w2l_templates_media_root.join(template_private_bucket_hash) + } + + pub fn w2l_template_media_directory(&self) -> &Path { + &self.w2l_templates_media_root + } + + pub fn create_w2l_template_media_path(&self) -> AnyhowResult<()> { + let _ = fs::create_dir_all(self.w2l_template_media_directory())?; + Ok(()) + } + + // ==================== W2L CACHED FACES ==================== /// We cache W2L faces here. /// We'll likely need to LRU cache them. diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index eb57547605..989c9575f6 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -179,6 +179,7 @@ async fn main() -> AnyhowResult<()> { info!("Creating pod semi-persistent cache dirs..."); semi_persistent_cache.create_w2l_model_path()?; semi_persistent_cache.create_w2l_face_template_path()?; + semi_persistent_cache.create_w2l_template_media_path()?; semi_persistent_cache.create_w2l_model_path()?; let w2l_model_filename = easyenv::get_env_string_or_default( @@ -327,13 +328,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An info!("Downloaded model from bucket!"); } - // ==================== CONFIRM OR DOWNLOAD W2L TEMPLATE FACE ==================== // - - // Template is based on the `private_bucket_hash`: - // - private_bucket_hash: 1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 - // - private_bucket_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 - // - private_bucket_cached_faces_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_detected_faces.pickle - // - maybe_public_bucket_preview_image_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_preview.webp + // ==================== LOOK UP TEMPLATE RECORD ==================== // let template_token = match &job.maybe_w2l_template_token { Some(token) => token.to_string(), @@ -355,19 +350,55 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An }, }; + // ==================== CONFIRM OR DOWNLOAD W2L TEMPLATE AUDIO OR VIDEO ==================== // + + // Template is based on the `private_bucket_hash`: + // - private_bucket_hash: 1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 + // - private_bucket_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 + // - private_bucket_cached_faces_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_detected_faces.pickle + // - maybe_public_bucket_preview_image_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_preview.webp + + let template_media_fs_path = inferencer.semi_persistent_cache.w2l_template_media_path( + &w2l_template.private_bucket_hash); + + if !template_media_fs_path.exists() { + info!("W2L template media file does not exist: {:?}", &template_media_fs_path); + + let template_media_object_path = inferencer.bucket_path_unifier + .media_templates_for_w2l_path(&w2l_template.private_bucket_hash); + + info!("Download from template media path: {:?}", &template_media_object_path); + + inferencer.private_bucket_client.download_file_to_disk( + &template_media_object_path, + &template_media_fs_path + ).await?; + + info!("Downloaded template media from bucket!"); + } + + + // ==================== CONFIRM OR DOWNLOAD W2L TEMPLATE FACE ==================== // + + // Template is based on the `private_bucket_hash`: + // - private_bucket_hash: 1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 + // - private_bucket_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 + // - private_bucket_cached_faces_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_detected_faces.pickle + // - maybe_public_bucket_preview_image_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_preview.webp + let face_template_fs_path = inferencer.semi_persistent_cache.w2l_face_template_path( &w2l_template.private_bucket_hash); if !face_template_fs_path.exists() { info!("W2L face template file does not exist: {:?}", &face_template_fs_path); - let template_object_path = inferencer.bucket_path_unifier + let face_template_object_path = inferencer.bucket_path_unifier .precomputed_faces_for_w2l_path(&w2l_template.private_bucket_hash); - info!("Download from face template path: {:?}", &template_object_path); + info!("Download from face template path: {:?}", &face_template_object_path); inferencer.private_bucket_client.download_file_to_disk( - &template_object_path, + &face_template_object_path, &face_template_fs_path ).await?; From f886cecf8d9398864e6918b6354b40327391ccc9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 12:20:46 -0400 Subject: [PATCH 0186/1042] it works?? I need to save the results somewhere --- models/Wav2Lip/vocodes_process_inference.py | 40 ++++++++++++------- .../wav2lip_inference_command.rs | 10 +++-- src/w2l_inference_job.rs | 5 ++- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/models/Wav2Lip/vocodes_process_inference.py b/models/Wav2Lip/vocodes_process_inference.py index 1e0b0f5a85..40f6da7d86 100644 --- a/models/Wav2Lip/vocodes_process_inference.py +++ b/models/Wav2Lip/vocodes_process_inference.py @@ -44,8 +44,12 @@ parser.add_argument('--cached_faces_filename', type=str, #parser.add_argument('--face', type=str, # help='Filepath of video/image that contains faces to use', required=True) -parser.add_argument('--audio', type=str, +parser.add_argument('--audio_filename', type=str, help='Filepath of video/audio file to use as raw audio source', required=True) + +parser.add_argument('--output_metadata_filename', type=str, + help='Output filename for the JSON containing width, height, etc.', required=True) + parser.add_argument('--outfile', type=str, help='Video path to save result. See default for an e.g.', default='results/result_voice.mp4') @@ -101,7 +105,7 @@ parser.add_argument('--preserve_tempdir', default=False, action='store_true', args = parser.parse_args() args.img_size = 96 -#if os.path.isfile(args.face) and os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg']: +#if os.path.isfile(args.image_or_video_filename) and os.path.splitext(args.image_or_video_filename)[1] in ['.jpg', '.png', '.jpeg']: # args.static = True #elif args.is_image: # args.static = True @@ -255,13 +259,13 @@ def maybe_pad_audio_file(tempdir, args): # Structure of filters: https://stackoverflow.com/a/55463101 command = ' '.join([ 'ffmpeg', - '-i {}'.format(args.audio), + '-i {}'.format(args.audio_filename), '-filter_complex "[0] adelay={}ms:all=true [a] ; [a] apad=pad_dur={}ms"'.format(args.audio_start_pad_millis, args.audio_end_pad_millis), '{}'.format(padded_wav_filename) ]) print('command:', command, flush=True) subprocess.call(command, shell=True) - args.audio = padded_wav_filename + args.audio_filename = padded_wav_filename def maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h): @@ -322,23 +326,29 @@ def maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h): def main(tempdir): video_faces_pickle_file = args.cached_faces_filename - #video_faces_pickle_file = args.face + '.faces' + #video_faces_pickle_file = args.image_or_video_filename + '.faces' #print('Video faces pickle file: {}'.format(video_faces_pickle_file), flush=True) frame_w = 0 frame_h = 0 - if not os.path.isfile(args.face): - raise ValueError('--face argument must be a valid path to video/image file') + if not os.path.isfile(args.image_or_video_filename): + raise ValueError('image_or_video_filename is not a file') - elif os.path.splitext(args.face)[1] in ['.jpg', '.png', '.jpeg'] \ + if not os.path.isfile(args.cached_faces_filename): + raise ValueError('cached_faces_filename is not a file') + + if not os.path.isfile(args.audio_filename): + raise ValueError('audio_filename is not a file') + + if os.path.splitext(args.image_or_video_filename)[1] in ['.jpg', '.png', '.jpeg'] \ or args.is_image: - full_frames = [cv2.imread(args.face)] + full_frames = [cv2.imread(args.image_or_video_filename)] fps = args.fps frame_h, frame_w = full_frames[0].shape[:-1] else: - video_stream = cv2.VideoCapture(args.face) + video_stream = cv2.VideoCapture(args.image_or_video_filename) fps = video_stream.get(cv2.CAP_PROP_FPS) print('Reading video frames...') @@ -372,19 +382,19 @@ def main(tempdir): print("Frame dimensions: {}x{}".format(frame_w, frame_h), flush=True) # TODO(bt): This could be more efficient - if not args.audio.endswith('.wav'): + if not args.audio_filename.endswith('.wav'): print('Extracting raw audio...') temp_wav_filename = os.path.join(tempdir, 'temp.wav') # previously 'temp/temp.wav' - command = 'ffmpeg -y -i {} -strict -2 {}'.format(args.audio, temp_wav_filename) + command = 'ffmpeg -y -i {} -strict -2 {}'.format(args.audio_filename, temp_wav_filename) print('command:', command, flush=True) subprocess.call(command, shell=True) - args.audio = temp_wav_filename + args.audio_filename = temp_wav_filename maybe_pad_audio_file(tempdir, args) # TODO(bt): Save the spectrogram? - wav = audio.load_wav(args.audio, 16000) + wav = audio.load_wav(args.audio_filename, 16000) mel = audio.melspectrogram(wav) print(mel.shape) @@ -466,7 +476,7 @@ def main(tempdir): out.release() - command = 'ffmpeg -y -i {} -i {} -strict -2 -q:v 1 {}'.format(args.audio, output_video_filename, args.outfile) + command = 'ffmpeg -y -i {} -i {} -strict -2 -q:v 1 {}'.format(args.audio_filename, output_video_filename, args.outfile) print('command:', command, flush=True) subprocess.call(command, shell=True) diff --git a/src/script_execution/wav2lip_inference_command.rs b/src/script_execution/wav2lip_inference_command.rs index 5c48c9ee13..5826f2ac87 100644 --- a/src/script_execution/wav2lip_inference_command.rs +++ b/src/script_execution/wav2lip_inference_command.rs @@ -29,8 +29,10 @@ impl Wav2LipInferenceCommand { pub fn execute>(&self, audio_filename: P, - output_cached_faces_filename: P, + media_template_filename: P, + cached_faces_filename: P, output_metadata_filename: P, + output_media_filename: P, is_image: bool, spawn_process: bool) -> AnyhowResult<()> { @@ -46,10 +48,12 @@ impl Wav2LipInferenceCommand { command.push_str(&self.script_name); command.push_str(" --checkpoint_path "); command.push_str(&self.checkpoint_path); - command.push_str(" --audio_filename"); + command.push_str(" --image_or_video_filename "); + command.push_str(&media_template_filename.as_ref().display().to_string()); + command.push_str(" --audio_filename "); command.push_str(&audio_filename.as_ref().display().to_string()); command.push_str(" --cached_faces_filename "); - command.push_str(&output_cached_faces_filename.as_ref().display().to_string()); + command.push_str(&cached_faces_filename.as_ref().display().to_string()); command.push_str(" --output_metadata_filename "); command.push_str(&output_metadata_filename.as_ref().display().to_string()); diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 989c9575f6..2f4a7a8758 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -45,6 +45,7 @@ use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; use tempdir::TempDir; +use std::thread; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -377,7 +378,6 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An info!("Downloaded template media from bucket!"); } - // ==================== CONFIRM OR DOWNLOAD W2L TEMPLATE FACE ==================== // // Template is based on the `private_bucket_hash`: @@ -433,14 +433,17 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An inferencer.w2l_inference.execute( &audio_fs_path, + &template_media_fs_path, &face_template_fs_path, &PathBuf::from("todo"), + &PathBuf::from("todo"), false, false )?; if true { info!("FAKE DONE"); + thread::sleep(Duration::from_millis(5000)); return Ok(()); } From 6ceed517deff709e823197a2375080a80d9159ea Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 12:36:45 -0400 Subject: [PATCH 0187/1042] end filename --- models/Wav2Lip/vocodes_process_inference.py | 13 +++++--- .../wav2lip_inference_command.rs | 4 ++- src/util/semi_persistent_cache_dir.rs | 32 +++++++++++++++++++ src/w2l_inference_job.rs | 2 ++ 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/models/Wav2Lip/vocodes_process_inference.py b/models/Wav2Lip/vocodes_process_inference.py index 40f6da7d86..5f604c0995 100644 --- a/models/Wav2Lip/vocodes_process_inference.py +++ b/models/Wav2Lip/vocodes_process_inference.py @@ -47,11 +47,14 @@ parser.add_argument('--cached_faces_filename', type=str, parser.add_argument('--audio_filename', type=str, help='Filepath of video/audio file to use as raw audio source', required=True) +parser.add_argument('--output_video_filename', type=str, + help='Output filename for the final video', required=True) + parser.add_argument('--output_metadata_filename', type=str, help='Output filename for the JSON containing width, height, etc.', required=True) -parser.add_argument('--outfile', type=str, help='Video path to save result. See default for an e.g.', - default='results/result_voice.mp4') +#parser.add_argument('--outfile', type=str, help='Video path to save result. See default for an e.g.', +# default='results/result_voice.mp4') parser.add_argument('--static', type=bool, help='If True, then use only first video frame for inference', default=False) @@ -309,7 +312,7 @@ def maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h): # More info on filters: https://stackoverflow.com/a/22958746 command = ' '.join([ 'ffmpeg', - '-i {}'.format(args.outfile), + '-i {}'.format(args.output_video_filename), '-i {}'.format(bumpfile), '-filter_complex "concat=n=2:v=1:a=1"', '{}'.format(padded_wav_filename) @@ -321,7 +324,7 @@ def maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h): print('Failed to concatenate end bump') else: print('Done concatenating end bump; renaming file...', flush=True) - os.rename(padded_wav_filename, args.outfile) + os.rename(padded_wav_filename, args.output_video_filename) def main(tempdir): @@ -476,7 +479,7 @@ def main(tempdir): out.release() - command = 'ffmpeg -y -i {} -i {} -strict -2 -q:v 1 {}'.format(args.audio_filename, output_video_filename, args.outfile) + command = 'ffmpeg -y -i {} -i {} -strict -2 -q:v 1 {}'.format(args.audio_filename, output_video_filename, args.output_video_filename) print('command:', command, flush=True) subprocess.call(command, shell=True) diff --git a/src/script_execution/wav2lip_inference_command.rs b/src/script_execution/wav2lip_inference_command.rs index 5826f2ac87..b33f7e0497 100644 --- a/src/script_execution/wav2lip_inference_command.rs +++ b/src/script_execution/wav2lip_inference_command.rs @@ -32,7 +32,7 @@ impl Wav2LipInferenceCommand { media_template_filename: P, cached_faces_filename: P, output_metadata_filename: P, - output_media_filename: P, + output_video_filename: P, is_image: bool, spawn_process: bool) -> AnyhowResult<()> { @@ -54,6 +54,8 @@ impl Wav2LipInferenceCommand { command.push_str(&audio_filename.as_ref().display().to_string()); command.push_str(" --cached_faces_filename "); command.push_str(&cached_faces_filename.as_ref().display().to_string()); + command.push_str(" --output_video_filename "); + command.push_str(&output_video_filename.as_ref().display().to_string()); command.push_str(" --output_metadata_filename "); command.push_str(&output_metadata_filename.as_ref().display().to_string()); diff --git a/src/util/semi_persistent_cache_dir.rs b/src/util/semi_persistent_cache_dir.rs index fd03496b03..1aa0f8d51f 100644 --- a/src/util/semi_persistent_cache_dir.rs +++ b/src/util/semi_persistent_cache_dir.rs @@ -1,6 +1,7 @@ use std::path::{PathBuf, Path}; use std::fs; use crate::util::anyhow_result::AnyhowResult; +use tempdir::TempDir; /// These are for files on the worker filesystems pub struct SemiPersistentCacheDir { @@ -103,4 +104,35 @@ impl SemiPersistentCacheDir { let _ = fs::create_dir_all(self.video_asset_path())?; Ok(()) } + + // ==================== W2L OUTPUT RESULTS ==================== + + /// We cache W2L faces here. + /// We'll likely need to LRU cache them. + pub fn w2l_output_results_path(&self, temp_dir: &TempDir, inference_job_token: &str) -> PathBuf { + // NB: We don't want colons from the token in the filename. + let filename = inference_job_token.replace(":", ""); + let filename = format!("{}_result.mp4", filename); + + temp_dir.path().join(&filename) + } +} + +#[cfg(test)] +mod tests { + use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; + use tempdir::TempDir; + + #[test] + fn test_w2l_output_results_path() { + let dirs = SemiPersistentCacheDir::default_paths(); + let temp_dir = TempDir::new("test").unwrap(); + + let filename = dirs.w2l_output_results_path(&temp_dir, "TYPE:TOKEN") + .to_str() + .map(|s| s.to_string()) + .unwrap(); + + assert!(&filename.ends_with("TYPETOKEN_result.mp4")); + } } diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 2f4a7a8758..6f5bd64aa4 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -441,6 +441,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An false )?; + let output_video_fs_path = temp_dir.path().join(&audio_bucket_hash); + if true { info!("FAKE DONE"); thread::sleep(Duration::from_millis(5000)); From 9075f97d7a9673cc034e82802e644e43be5ecc59 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 13:03:19 -0400 Subject: [PATCH 0188/1042] output metadata (wrongish?) --- models/Wav2Lip/vocodes_process_inference.py | 24 ++++++++++ src/util/semi_persistent_cache_dir.rs | 53 +++++++++++---------- src/w2l_inference_job.rs | 19 ++++++-- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/models/Wav2Lip/vocodes_process_inference.py b/models/Wav2Lip/vocodes_process_inference.py index 5f604c0995..385baca10b 100644 --- a/models/Wav2Lip/vocodes_process_inference.py +++ b/models/Wav2Lip/vocodes_process_inference.py @@ -10,6 +10,7 @@ import tempfile import shutil import datetime import pickle +import magic from PIL import Image if 'PRINT_ENV' in os.environ and os.environ['PRINT_ENV']: @@ -334,6 +335,7 @@ def main(tempdir): frame_w = 0 frame_h = 0 + fps = 0 if not os.path.isfile(args.image_or_video_filename): raise ValueError('image_or_video_filename is not a file') @@ -485,6 +487,28 @@ def main(tempdir): maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h) + # ==== METADATA ==== + mime_type = magic.from_file(args.output_video_filename, mime=True) + file_size_bytes = os.path.getsize(args.output_video_filename) + + frame_count = len(full_frames) + duration_seconds = frame_count / fps + duration_millis = int(duration_seconds * 1000) + + metadata = { + 'is_video': True, + 'width': frame_w, + 'height': frame_h, + 'num_frames': frame_count, + 'fps': fps, + 'duration_millis': duration_millis, + 'mimetype': mime_type, + 'file_size_bytes': file_size_bytes, + } + + with open(args.output_metadata_filename, 'w') as json_file: + json.dump(metadata, json_file) + if __name__ == '__main__': tempdir = tempfile.mkdtemp() print('Using Python tempdir: {}'.format(tempdir), flush=True) diff --git a/src/util/semi_persistent_cache_dir.rs b/src/util/semi_persistent_cache_dir.rs index 1aa0f8d51f..6e3bd89875 100644 --- a/src/util/semi_persistent_cache_dir.rs +++ b/src/util/semi_persistent_cache_dir.rs @@ -107,32 +107,33 @@ impl SemiPersistentCacheDir { // ==================== W2L OUTPUT RESULTS ==================== - /// We cache W2L faces here. - /// We'll likely need to LRU cache them. - pub fn w2l_output_results_path(&self, temp_dir: &TempDir, inference_job_token: &str) -> PathBuf { - // NB: We don't want colons from the token in the filename. - let filename = inference_job_token.replace(":", ""); - let filename = format!("{}_result.mp4", filename); + // We cache W2L faces here. + // We'll likely need to LRU cache them. + //pub fn w2l_output_results_path(&self, temp_dir: &TempDir, inference_job_token: &str) -> PathBuf { + // // NB: We don't want colons from the token in the filename. + // let filename = inference_job_token.replace(":", ""); + // let filename = format!("{}_result.mp4", filename); - temp_dir.path().join(&filename) - } + // temp_dir.path().join(&filename) + //} } -#[cfg(test)] -mod tests { - use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; - use tempdir::TempDir; - - #[test] - fn test_w2l_output_results_path() { - let dirs = SemiPersistentCacheDir::default_paths(); - let temp_dir = TempDir::new("test").unwrap(); - - let filename = dirs.w2l_output_results_path(&temp_dir, "TYPE:TOKEN") - .to_str() - .map(|s| s.to_string()) - .unwrap(); - - assert!(&filename.ends_with("TYPETOKEN_result.mp4")); - } -} +//#[cfg(test)] +//mod tests { +// use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; +// use tempdir::TempDir; +// +// #[test] +// fn test_w2l_output_results_path() { +// let dirs = SemiPersistentCacheDir::default_paths(); +// let temp_dir = TempDir::new("test").unwrap(); +// +// let filename = dirs.w2l_output_results_path(&temp_dir, "TYPE:TOKEN") +// .to_str() +// .map(|s| s.to_string()) +// .unwrap(); +// +// assert!(&filename.ends_with("TYPETOKEN_result.mp4")); +// } +//} +// \ No newline at end of file diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 6f5bd64aa4..a7097c3940 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -431,21 +431,32 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // ==================== RUN INFERENCE ==================== // + let output_video_fs_path = temp_dir.path().join("output.mp4"); + let output_metadata_fs_path = temp_dir.path().join("metadata.json"); + + let is_image = w2l_template.template_type.contains("image"); + + // Metadata is wrong: + // {"is_video": true, "width": 1024, "height": 768, + // "num_frames": 1, -- wrong + // "fps": 25.0, -- wrong?? + // "duration_millis": 40, -- wrong + // "mimetype": "video/mp4", "file_size_bytes": 1024594} inferencer.w2l_inference.execute( &audio_fs_path, &template_media_fs_path, &face_template_fs_path, - &PathBuf::from("todo"), - &PathBuf::from("todo"), + &output_metadata_fs_path, + &output_video_fs_path, false, false )?; - let output_video_fs_path = temp_dir.path().join(&audio_bucket_hash); + info!("Output filename: {:?}", &output_video_fs_path); if true { info!("FAKE DONE"); - thread::sleep(Duration::from_millis(5000)); + thread::sleep(Duration::from_millis(50000)); return Ok(()); } From a8211a1f9bdc351bb49b083def5f6ca65398c246 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 13:20:50 -0400 Subject: [PATCH 0189/1042] this fixes duration metadata --- models/Wav2Lip/vocodes_process_inference.py | 26 +++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/models/Wav2Lip/vocodes_process_inference.py b/models/Wav2Lip/vocodes_process_inference.py index 385baca10b..3fc758a005 100644 --- a/models/Wav2Lip/vocodes_process_inference.py +++ b/models/Wav2Lip/vocodes_process_inference.py @@ -487,20 +487,36 @@ def main(tempdir): maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h) + command = [ + "ffprobe", + "-loglevel", "quiet", + "-print_format", "json", + "-show_format", + "-show_streams", + args.output_video_filename + ] + + pipe = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + out, err = pipe.communicate() + ffmpeg_metadata = json.loads(out) + # ==== METADATA ==== mime_type = magic.from_file(args.output_video_filename, mime=True) file_size_bytes = os.path.getsize(args.output_video_filename) - frame_count = len(full_frames) - duration_seconds = frame_count / fps - duration_millis = int(duration_seconds * 1000) + print(ffmpeg_metadata) + print(ffmpeg_metadata['format']) + print(ffmpeg_metadata['format']['duration']) + #frame_count = len(full_frames) + #duration_seconds = frame_count / fps + duration_millis = int(float(ffmpeg_metadata['format']['duration']) * 1000) metadata = { 'is_video': True, 'width': frame_w, 'height': frame_h, - 'num_frames': frame_count, - 'fps': fps, + #'num_frames': frame_count, + #'fps': fps, 'duration_millis': duration_millis, 'mimetype': mime_type, 'file_size_bytes': file_size_bytes, From 2d7f46cc57d081c332d93020862a6b5b83ab7323 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 13:58:06 -0400 Subject: [PATCH 0190/1042] progress, but broke one of the jobs --- .../up.sql | 10 +- models/Wav2Lip/vocodes_process_inference.py | 6 +- src/buckets/bucket_client.rs | 14 +- src/buckets/bucket_path_unifier.rs | 29 ++- src/job_queries/w2l_inference_job_queries.rs | 76 +++----- .../wav2lip_inference_command.rs | 3 +- src/w2l_inference_job.rs | 183 +++--------------- 7 files changed, 105 insertions(+), 216 deletions(-) diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index c5c0dab260..108ff3a44e 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -100,7 +100,15 @@ CREATE TABLE w2l_results ( ) NOT NULL DEFAULT 'public', -- Where the wav, spectrogram, and etc. are located. - public_bucket_hash CHAR(64) NOT NULL, + public_bucket_video_path VARCHAR(255) NOT NULL, + + -- ========== METADATA ========== + + file_size_bytes INT(10) NOT NULL DEFAULT 0, + mime_type VARCHAR(32) NOT NULL DEFAULT '', + frame_width INT(10) NOT NULL DEFAULT 0, + frame_height INT(10) NOT NULL DEFAULT 0, + duration_millis INT(10) NOT NULL DEFAULT 0, -- ========== MODERATION DETAILS ========== diff --git a/models/Wav2Lip/vocodes_process_inference.py b/models/Wav2Lip/vocodes_process_inference.py index 3fc758a005..37e7329834 100644 --- a/models/Wav2Lip/vocodes_process_inference.py +++ b/models/Wav2Lip/vocodes_process_inference.py @@ -505,10 +505,12 @@ def main(tempdir): file_size_bytes = os.path.getsize(args.output_video_filename) print(ffmpeg_metadata) - print(ffmpeg_metadata['format']) - print(ffmpeg_metadata['format']['duration']) + + # We can't really get frame count anymore. The original source is invalid. + # ffmpeg reports fps, but it's a ratio, eg. 25/1 #frame_count = len(full_frames) #duration_seconds = frame_count / fps + duration_millis = int(float(ffmpeg_metadata['format']['duration']) * 1000) metadata = { diff --git a/src/buckets/bucket_client.rs b/src/buckets/bucket_client.rs index 4b52677591..ad12e894f9 100644 --- a/src/buckets/bucket_client.rs +++ b/src/buckets/bucket_client.rs @@ -131,7 +131,17 @@ impl BucketClient { self.upload_file(object_name, &buffer).await } - pub async fn upload_filename_with_content_type(&self, object_name: &str, filename: &Path, content_type: &str) -> anyhow::Result<()> { + pub async fn upload_filename_with_content_type>( + &self, + object_path: P, + filename: P, + content_type: &str + ) -> anyhow::Result<()> { + let object_path_str = object_path.as_ref() + .to_str() + .map(|s| s.to_string()) + .ok_or(anyhow!("could not convert object path to string"))?; + // TODO: does a newer version of this crate handle streaming/buffering file contents? let mut file = File::open(filename).await?; let mut buffer : Vec = Vec::new(); @@ -139,7 +149,7 @@ impl BucketClient { info!("Uploading with content type..."); - self.upload_file_with_content_type(object_name, &buffer, content_type).await + self.upload_file_with_content_type(&object_path_str, &buffer, content_type).await } // NB: New version has blocking client rather than blocking calls. diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs index b314715a99..e47ea2e0a1 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/buckets/bucket_path_unifier.rs @@ -71,9 +71,26 @@ impl BucketPathUnifier { } // W2L inference output videos - pub fn w2l_inference_video_output_path(&self, w2l_inference_token: &str) -> PathBuf { - let hashed_path = Self::hashed_directory_path(w2l_inference_token); - let video_filename = format!("{}_video.mp4", &w2l_inference_token); + pub fn w2l_inference_video_output_path(&self, w2l_inference_job_token: &str) -> PathBuf { + // NB: We don't want colons from the token in the filename. + if w2l_inference_job_token.contains(":") { + if let Some((token_type, token)) = w2l_inference_job_token.split_once(":") { + let video_filename = w2l_inference_job_token.replace(":", ""); + let video_filename = format!("vocodes_video_{}.mp4", video_filename); + + let hashed_path = Self::hashed_directory_path(token); + + return self.w2l_inference_output_root + .join(hashed_path) + .join(video_filename); + } + } + + + let video_filename = w2l_inference_job_token.replace(":", ""); + let video_filename = format!("vocodes_video_{}.mp4", video_filename); + + let hashed_path = Self::hashed_directory_path(w2l_inference_job_token); self.w2l_inference_output_root .join(hashed_path) @@ -137,7 +154,11 @@ mod tests { fn test_w2l_inference_video_output_path() { let paths = get_instance(); assert_eq!(paths.w2l_inference_video_output_path("foobar").to_str().unwrap(), - "/test_path_w2l_output/f/o/o/foobar_video.mp4"); + "/test_path_w2l_output/f/o/o/vocodes_video_foobar.mp4"); + + // Note: it also removes the token type from dir path and handles the colon: + assert_eq!(paths.w2l_inference_video_output_path("type:abcdef").to_str().unwrap(), + "/test_path_w2l_output/a/b/c/vocodes_video_typeabcdef.mp4"); } #[test] diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index 348d356078..20760dfc7e 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -9,6 +9,7 @@ use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn, info}; use sqlx::MySqlPool; +use std::path::Path; /// table: w2l_template_upload_jobs #[derive(Debug)] @@ -153,67 +154,53 @@ WHERE id = ? Ok(()) } -pub async fn insert_w2l_result(pool: &MySqlPool, - template_type: &str, // TODO: ENUM! - job: &W2lInferenceJobRecord, - private_bucket_hash: &str, - private_bucket_object_name: &str, - private_bucket_cached_faces_object_name: &str, - maybe_image_preview_object_name: Option<&str>, - maybe_video_preview_object_name: Option<&str>, - file_size_bytes: u64, - maybe_mime_type: Option<&str>, - frame_width: u32, - frame_height: u32, - frame_count: u64, - fps: f32, - duration_millis: u64) - -> AnyhowResult +pub async fn insert_w2l_result>( + pool: &MySqlPool, + job: &W2lInferenceJobRecord, + bucket_video_results_path: P, + file_size_bytes: u64, + maybe_mime_type: Option<&str>, + frame_width: u32, + frame_height: u32, + duration_millis: u64 +) -> AnyhowResult { - /*let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?; - let updatable_slug = model_token.clone(); + let inference_result_token = random_prefix_crockford_token("W2L_RES:", 32)?; + + let bucket_video_result_path = &bucket_video_results_path + .as_ref() + .display() + .to_string(); let query_result = sqlx::query!( r#" -INSERT INTO w2l_templates +INSERT INTO w2l_results SET token = ?, - template_type = ?, - updatable_slug = ?, - title = ?, - description_markdown = '', - description_rendered_html = '', - creator_user_token = ?, + maybe_w2l_template_token = ?, + maybe_creator_user_token = ?, creator_ip_address = ?, - original_download_url = ?, - private_bucket_hash = ?, - private_bucket_object_name = ?, - private_bucket_cached_faces_object_name = ?, - maybe_public_bucket_preview_image_object_name = ?, - maybe_public_bucket_preview_video_object_name = ?, + creator_set_visibility = 'public', + + public_bucket_video_path = ?, + file_size_bytes = ?, mime_type = ?, frame_width = ?, frame_height = ?, - frame_count = ?, - fps = ?, duration_millis = ? "#, - model_token, - template_type, - updatable_slug, + inference_result_token, + job.maybe_w2l_template_token.clone(), + job.maybe_creator_user_token.clone(), job.creator_ip_address.clone(), - private_bucket_hash.to_string(), - private_bucket_object_name.to_string(), - private_bucket_cached_faces_object_name.to_string(), - maybe_image_preview_object_name, - maybe_video_preview_object_name, + + bucket_video_result_path, + file_size_bytes, maybe_mime_type.unwrap_or(""), frame_width, frame_height, - frame_count, - fps, duration_millis ) .execute(pool) @@ -229,8 +216,7 @@ SET } }; - Ok(record_id)*/ - return Err(anyhow!("TODO")); + Ok(record_id) } pub struct W2lTemplateRecord2 { diff --git a/src/script_execution/wav2lip_inference_command.rs b/src/script_execution/wav2lip_inference_command.rs index b33f7e0497..346425eae4 100644 --- a/src/script_execution/wav2lip_inference_command.rs +++ b/src/script_execution/wav2lip_inference_command.rs @@ -28,6 +28,7 @@ impl Wav2LipInferenceCommand { } pub fn execute>(&self, + checkpoint_path: P, audio_filename: P, media_template_filename: P, cached_faces_filename: P, @@ -47,7 +48,7 @@ impl Wav2LipInferenceCommand { command.push_str("python "); command.push_str(&self.script_name); command.push_str(" --checkpoint_path "); - command.push_str(&self.checkpoint_path); + command.push_str(&checkpoint_path.as_ref().display().to_string()); command.push_str(" --image_or_video_filename "); command.push_str(&media_template_filename.as_ref().display().to_string()); command.push_str(" --audio_filename "); diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index a7097c3940..bf526e20fc 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -280,14 +280,14 @@ struct FileMetadata { pub is_video: bool, pub width: u32, pub height: u32, - pub num_frames: u64, - pub fps: Option, + //pub num_frames: u64, + //pub fps: Option, pub duration_millis: Option, pub mimetype: Option, pub file_size_bytes: u64, } -fn read_metadata_file(filename: &str) -> AnyhowResult { +fn read_metadata_file(filename: &PathBuf) -> AnyhowResult { let mut file = File::open(filename)?; let mut buffer = String::new(); file.read_to_string(&mut buffer)?; @@ -436,13 +436,11 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An let is_image = w2l_template.template_type.contains("image"); - // Metadata is wrong: - // {"is_video": true, "width": 1024, "height": 768, - // "num_frames": 1, -- wrong - // "fps": 25.0, -- wrong?? - // "duration_millis": 40, -- wrong - // "mimetype": "video/mp4", "file_size_bytes": 1024594} + info!("Is image? {}", is_image); + info!("Running W2L inference..."); + inferencer.w2l_inference.execute( + &model_fs_path, &audio_fs_path, &template_media_fs_path, &face_template_fs_path, @@ -454,123 +452,29 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An info!("Output filename: {:?}", &output_video_fs_path); + // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // + + info!("Checking that output files exist..."); + + check_file_exists(&output_video_fs_path)?; + check_file_exists(&output_metadata_fs_path)?; + + let file_metadata = read_metadata_file(&output_metadata_fs_path)?; + if true { info!("FAKE DONE"); thread::sleep(Duration::from_millis(50000)); return Ok(()); } - //let download_url = job.download_url.as_ref() - // .map(|c| c.to_string()) - // .unwrap_or("".to_string()); - // ==================== DOWNLOAD FILE ==================== // - - info!("Calling downloader..."); - //let download_filename = inferencer.google_drive_downloader - // .download_file(&download_url, &temp_dir).await?; - let download_filename = "TODO"; - - // ==================== PROCESS FACES ==================== // - - // This is the Python Pickle file with all the face frames. - // We'll retain it forever since it's expensive to compute. - let cached_faces_filename = format!("{}_detected_faces.pickle", &download_filename); - - // This is a file that we'll use to determine if the file is an image or video. - let output_metadata_filename = format!("{}_metadata.json", &download_filename); - - let is_image = false; // TODO: Don't always treat as video. - let spawn_process = false; - - /*inferencer.w2l_processor.execute( - &download_filename, - &cached_faces_filename, - &output_metadata_filename, - is_image, - spawn_process)?;*/ - - // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // - - let video_or_image_path = PathBuf::from(&download_filename); - let cached_faces_path = &PathBuf::from(&cached_faces_filename); - let output_metadata_path = &PathBuf::from(&output_metadata_filename); - - info!("Checking that both files exist (original source + cached faces) ..."); - - check_file_exists(&video_or_image_path)?; - check_file_exists(&cached_faces_path)?; - check_file_exists(&output_metadata_path)?; - - let file_metadata = read_metadata_file(&output_metadata_filename)?; - - // ==================== BASE OBJECT NAMES BASED ON HASH ==================== // - - let private_bucket_hash = get_file_hash(&download_filename)?; - - info!("File hash: {}", private_bucket_hash); - - // Full path to video/image - //let full_object_path = hash_to_bucket_path( - // &private_bucket_hash, - // Some(&inferencer.w2l_template_uploads_bucket_root))?; - - let full_object_path = "TODO"; - - // ==================== GENERATE VIDEO PREVIEWS ==================== // - - let mut maybe_image_preview_filename : Option = None; - let mut maybe_image_preview_object_name : Option = None; - - let mut maybe_video_preview_filename : Option = None; - let mut maybe_video_preview_object_name : Option = None; - - /*if file_metadata.is_video { - let preview_filename = format!("{}_preview.webp", &download_filename); - - inferencer.ffmpeg_video_preview_generator.execute( - &download_filename, - &preview_filename, - 500, - 500, - true, - false - )?; - - let video_preview_path = PathBuf::from(&preview_filename); - check_file_exists(&video_preview_path)?; - - let preview_object_path = format!("{}_preview.webp", full_object_path); - maybe_video_preview_object_name = Some(preview_object_path); - maybe_video_preview_filename = Some(video_preview_path); - - } else { - let preview_filename = format!("{}_preview.webp", &download_filename); - - inferencer.imagemagick_image_preview_generator.execute( - &download_filename, - &preview_filename, - 500, - 500, - false - )?; - - let image_preview_path = PathBuf::from(&preview_filename); - check_file_exists(&image_preview_path)?; - - let preview_object_path = format!("{}_preview.webp", full_object_path); - maybe_image_preview_object_name = Some(preview_object_path); - maybe_image_preview_filename = Some(image_preview_path); - }*/ // ==================== UPLOAD TO BUCKETS ==================== // - info!("Image/video destination bucket path: {}", full_object_path); + let result_object_path = inferencer.bucket_path_unifier.w2l_inference_video_output_path( + &job.inference_job_token); - // Full path to cached faces - let full_object_path_cached_faces = format!("{}_detected_faces.pickle", full_object_path); - - info!("Cached faces destination bucket path: {}", full_object_path_cached_faces); + info!("Image/video destination bucket path: {:?}", &result_object_path); info!("Uploading image/video..."); @@ -578,54 +482,11 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An .as_deref() .unwrap_or("application/octet-stream"); - inferencer.private_bucket_client.upload_filename_with_content_type( - &full_object_path, - &video_or_image_path, - original_mime_type).await?; - inferencer.public_bucket_client.upload_filename_with_content_type( - &full_object_path, + &result_object_path, &video_or_image_path, - original_mime_type).await?; - - info!("Uploading cached faces..."); - inferencer.private_bucket_client.upload_filename( - &full_object_path_cached_faces, - &cached_faces_path).await?; - - // TODO: Fix this ugh. - if let Some(image_preview_filename) = maybe_image_preview_filename.as_deref() { - if let Some(image_preview_object_name) = maybe_image_preview_object_name.as_deref() { - info!("Uploading image preview..."); - inferencer.private_bucket_client.upload_filename_with_content_type( - &image_preview_object_name, - image_preview_filename, - "image/webp").await?; - - info!("Uploading image preview..."); - inferencer.public_bucket_client.upload_filename_with_content_type( - &image_preview_object_name, - image_preview_filename, - "image/webp").await?; - } - } - - // TODO: Fix this ugh. - if let Some(video_preview_filename) = maybe_video_preview_filename.as_deref() { - if let Some(video_preview_object_name) = maybe_video_preview_object_name.as_deref() { - info!("Uploading video preview..."); - inferencer.private_bucket_client.upload_filename_with_content_type( - &video_preview_object_name, - video_preview_filename, - "image/webp").await?; - - info!("Uploading video preview..."); - inferencer.public_bucket_client.upload_filename_with_content_type( - &video_preview_object_name, - video_preview_filename, - "image/webp").await?; - } - } + original_mime_type) + .await?; // ==================== SAVE RECORDS ==================== // From 063780db3a35f67f58b22d261dfe404796f3d62c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 14:13:36 -0400 Subject: [PATCH 0191/1042] this might be close... --- src/buckets/bucket_client.rs | 2 +- src/w2l_inference_job.rs | 23 +++-------------------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/src/buckets/bucket_client.rs b/src/buckets/bucket_client.rs index ad12e894f9..912d0fcc60 100644 --- a/src/buckets/bucket_client.rs +++ b/src/buckets/bucket_client.rs @@ -134,7 +134,7 @@ impl BucketClient { pub async fn upload_filename_with_content_type>( &self, object_path: P, - filename: P, + filename: &Path, content_type: &str ) -> anyhow::Result<()> { let object_path_str = object_path.as_ref() diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index bf526e20fc..68df41a0be 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -461,14 +461,6 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An let file_metadata = read_metadata_file(&output_metadata_fs_path)?; - if true { - info!("FAKE DONE"); - thread::sleep(Duration::from_millis(50000)); - return Ok(()); - } - - - // ==================== UPLOAD TO BUCKETS ==================== // let result_object_path = inferencer.bucket_path_unifier.w2l_inference_video_output_path( @@ -484,34 +476,25 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An inferencer.public_bucket_client.upload_filename_with_content_type( &result_object_path, - &video_or_image_path, + &output_video_fs_path, original_mime_type) .await?; // ==================== SAVE RECORDS ==================== // - let template_type = if file_metadata.is_video { "video" } else { "image" }; - info!("Saving w2l inference record..."); let id = insert_w2l_result( &inferencer.mysql_pool, - template_type, job, - &private_bucket_hash, - &full_object_path, - &full_object_path_cached_faces, - maybe_image_preview_object_name.as_deref(), - maybe_video_preview_object_name.as_deref(), + &result_object_path, file_metadata.file_size_bytes, file_metadata.mimetype.as_deref(), file_metadata.width, file_metadata.height, - file_metadata.num_frames, - file_metadata.fps.unwrap_or(0.0f32), file_metadata.duration_millis.unwrap_or(0)) .await?; - info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}", + info!("Job {} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}", job.id, id); mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?; From dbca8e7fd9b1e6946a5ea54530d876dc85f1c854 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 14:40:04 -0400 Subject: [PATCH 0192/1042] add tokens to upload jobs, reorganize schemas, codegen --- .../up.sql | 84 +- .../2021-05-15-212913_inference_jobs/up.sql | 50 +- sqlx-data.json | 1355 +++++++++++++++-- .../endpoints/tts/enqueue_upload_tts_model.rs | 14 + .../w2l/enqueue_infer_w2l_with_uploads.rs | 1 + .../w2l/enqueue_upload_w2l_template.rs | 14 + src/job_queries/tts_download_job_queries.rs | 1 + src/job_queries/w2l_download_job_queries.rs | 1 + src/w2l_inference_job.rs | 5 +- 9 files changed, 1364 insertions(+), 161 deletions(-) diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index 1224bdb9ec..c777210654 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -5,24 +5,15 @@ CREATE TABLE tts_model_upload_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, + -- Effective "primary key" (PUBLIC) + -- This is so the in-progress results can be looked up by the UI. + token VARCHAR(32) NOT NULL, + -- Idempotency token + -- This is so the frontend client doesn't submit duplicate jobs. uuid_idempotency_token VARCHAR(36) NOT NULL, - -- Foreign key to user - creator_user_token VARCHAR(32) NOT NULL, - - -- For abuse tracking. - -- Wide enough for IPv4/6 - creator_ip_address VARCHAR(40) NOT NULL, - - -- (THIS MIGHT NOT BE USED) - -- NB: DO NOT SORT! - -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. - creator_set_visibility ENUM( - 'public', - 'hidden', - 'private' - ) NOT NULL DEFAULT 'public', + -- ========== UPLOAD DETAILS ========== -- The name of the voice -- The "name" of the voice model, which might be complicated. @@ -51,6 +42,26 @@ CREATE TABLE tts_model_upload_jobs ( 'web' ) NOT NULL DEFAULT 'web', + -- ========== CREATOR DETAILS AND PREFERENCES ========== + + -- Foreign key to user + creator_user_token VARCHAR(32) NOT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- ========== JOB SYSTEM DETAILS ========== + -- Jobs begin as "pending", then transition to other states. -- -- * Pending = job is ready to go @@ -88,6 +99,7 @@ CREATE TABLE tts_model_upload_jobs ( -- INDICES -- PRIMARY KEY (id), + UNIQUE KEY (token), UNIQUE KEY (uuid_idempotency_token), KEY fk_creator_user_token (creator_user_token), KEY fk_maybe_subject_token (maybe_subject_token), @@ -101,24 +113,15 @@ CREATE TABLE tts_model_upload_jobs ( CREATE TABLE w2l_template_upload_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, + -- Effective "primary key" (PUBLIC) + -- This is so the in-progress results can be looked up by the UI. + token VARCHAR(32) NOT NULL, + -- Idempotency token + -- This is so the frontend client doesn't submit duplicate jobs. uuid_idempotency_token VARCHAR(36) NOT NULL, - -- Foreign key to user - creator_user_token VARCHAR(32) NOT NULL, - - -- For abuse tracking. - -- Wide enough for IPv4/6 - creator_ip_address VARCHAR(40) NOT NULL, - - -- (THIS MIGHT NOT BE USED) - -- NB: DO NOT SORT! - -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. - creator_set_visibility ENUM( - 'public', - 'hidden', - 'private' - ) NOT NULL DEFAULT 'public', + -- ========== UPLOAD DETAILS ========== -- The title of the template title VARCHAR(255) NOT NULL, @@ -144,6 +147,26 @@ CREATE TABLE w2l_template_upload_jobs ( 'web' ) NOT NULL DEFAULT 'web', + -- ========== CREATOR DETAILS AND PREFERENCES ========== + + -- Foreign key to user + creator_user_token VARCHAR(32) NOT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- ========== JOB SYSTEM DETAILS ========== + -- Jobs begin as "pending", then transition to other states. -- -- * Pending = job is ready to go @@ -181,6 +204,7 @@ CREATE TABLE w2l_template_upload_jobs ( -- INDICES -- PRIMARY KEY (id), + UNIQUE KEY (token), UNIQUE KEY (uuid_idempotency_token), KEY fk_creator_user_token (creator_user_token), KEY fk_maybe_subject_token (maybe_subject_token), diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index 50f285c46b..37b9759ed7 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -4,9 +4,11 @@ CREATE TABLE tts_inference_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) + -- This is so the in-progress results can be looked up by the UI. token VARCHAR(32) NOT NULL, -- Idempotency token from client + -- This is so the frontend client doesn't submit duplicate jobs. uuid_idempotency_token VARCHAR(36) NOT NULL, -- ========== INFERENCE DETAILS ========== @@ -40,11 +42,26 @@ CREATE TABLE tts_inference_jobs ( -- ========== JOB SYSTEM DETAILS ========== -- Jobs begin as "pending", then transition to other states. - -- Pending -> Started -> Complete - -- \-> Failed -> Started -> { Complete, Failed, Dead } - status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending', + -- + -- * Pending = job is ready to go + -- * Started = job is running + -- * Complete_Success = job is done (success) + -- * Complete_Failure = job is done (failure) + -- * Attempt_Failed = job failed but may retry. + -- * Dead = job failed permanently. + -- + -- Pending -> Started -> Complete_Success + -- |-> Complete_Failure + -- \-> Attempt_Failed -> Started -> { Complete, Failed, Dead } + status ENUM( + 'pending', + 'started', + 'complete_success', + 'complete_failure', + 'attempt_failed', + 'dead') NOT NULL DEFAULT 'pending', - -- We can track this against a "max_attempt_count" + -- We can track this against a "max_attempt_count" attempt_count INT(3) NOT NULL DEFAULT 0, -- If there is a failure, tell the user why. @@ -74,9 +91,11 @@ CREATE TABLE w2l_inference_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- Effective "primary key" (PUBLIC) + -- This is so the in-progress results can be looked up by the UI. token VARCHAR(32) NOT NULL, -- Idempotency token from client + -- This is so the frontend client doesn't submit duplicate jobs. uuid_idempotency_token VARCHAR(36) NOT NULL, -- ========== INFERENCE DETAILS : FACE TEMPLATE ========== @@ -107,7 +126,7 @@ CREATE TABLE w2l_inference_jobs ( maybe_audio_mime_type VARCHAR(32) DEFAULT NULL, - -- ========== CREATOR DETAILS ========== + -- ========== CREATOR DETAILS AND PREFERENCES ========== -- Foreign key to user -- If no user is logged in, this is null. @@ -133,9 +152,24 @@ CREATE TABLE w2l_inference_jobs ( -- ========== JOB SYSTEM DETAILS ========== -- Jobs begin as "pending", then transition to other states. - -- Pending -> Started -> Complete - -- \-> Failed -> Started -> { Complete, Failed, Dead } - status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending', + -- + -- * Pending = job is ready to go + -- * Started = job is running + -- * Complete_Success = job is done (success) + -- * Complete_Failure = job is done (failure) + -- * Attempt_Failed = job failed but may retry. + -- * Dead = job failed permanently. + -- + -- Pending -> Started -> Complete_Success + -- |-> Complete_Failure + -- \-> Attempt_Failed -> Started -> { Complete, Failed, Dead } + status ENUM( + 'pending', + 'started', + 'complete_success', + 'complete_failure', + 'attempt_failed', + 'dead') NOT NULL DEFAULT 'pending', -- We can track this against a "max_attempt_count" attempt_count INT(3) NOT NULL DEFAULT 0, diff --git a/sqlx-data.json b/sqlx-data.json index 634b86c163..aa3790e4be 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1,15 +1,5 @@ { "db": "MySQL", - "10c82778c4228a4f82ac310371b64171b13d4e6089232bd8c536f1df2614b4c4": { - "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 10 - }, - "nullable": [] - } - }, "1eeb7a88f5f8c2961694622b739e0b0689d89dff8a5f6b79d5a57c5b807a23f8": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { @@ -20,6 +10,210 @@ "nullable": [] } }, + "215f82f4034ccb47bfa000c3fead66c3c9f7d5caa905aa99a702d396aeeebbc6": { + "query": "\nINSERT INTO tts_model_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 11 + }, + "nullable": [] + } + }, + "293b56f5ef46fa7a860ff47b64b9784038883f742cb0c23bf9775cee78d4acef": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + } + }, "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -30,6 +224,26 @@ "nullable": [] } }, + "2b04dd7db52ef3a97a6f420a22a4dbee7416ae58697438df69e87ffd47bbf350": { + "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "2bfb24d71e1fccb98b0c27912214dca2e2c2833355fdfa9f70e4d7a4d99a038b": { + "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 11 + }, + "nullable": [] + } + }, "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", "describe": { @@ -50,6 +264,417 @@ "nullable": [] } }, + "36ff2cf2bdea42da6649923163013cdb5eaf719532d99305410284c69bedd2c0": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.private_bucket_hash,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\nAND w2l.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "private_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + } + }, + "4995b1688dab1a25fa16ea59d6282b052be5c4ed6bf740a0840e24c9ad67e55a": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.updatable_slug = ?\nAND w2l.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + } + }, + "52b8b6c64114ec754d15f8931fa88c280a2a5e614a885c6be01915d195a795e2": { + "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "53d77ee58e9b1f7fb900914895c25bcccb9297378ff018c1e553b187d1956e92": { "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE username = ?\nLIMIT 1\n ", "describe": { @@ -114,6 +739,16 @@ ] } }, + "746526e2e150150ef804d717f743de85010972d98cbb137e44d8348cfdb4dddc": { + "query": "\nINSERT INTO w2l_results\nSET\n token = ?,\n maybe_w2l_template_token = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_video_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, "7ab54466ca3021d90e0feb234acba56bf53d415dde69bc1c35120ed0c8dd1858": { "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { @@ -132,6 +767,18 @@ }, { "ordinal": 1, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, "name": "uuid_idempotency_token", "type_info": { "type": "VarString", @@ -142,44 +789,8 @@ "max_size": 144 } }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, { "ordinal": 3, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 4, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 5, "name": "title", "type_info": { "type": "VarString", @@ -191,7 +802,7 @@ } }, { - "ordinal": 6, + "ordinal": 4, "name": "tts_model_type", "type_info": { "type": "String", @@ -203,7 +814,7 @@ } }, { - "ordinal": 7, + "ordinal": 5, "name": "maybe_subject_token", "type_info": { "type": "VarString", @@ -215,7 +826,7 @@ } }, { - "ordinal": 8, + "ordinal": 6, "name": "maybe_actor_subject_token", "type_info": { "type": "VarString", @@ -227,7 +838,7 @@ } }, { - "ordinal": 9, + "ordinal": 7, "name": "download_url", "type_info": { "type": "VarString", @@ -239,7 +850,7 @@ } }, { - "ordinal": 10, + "ordinal": 8, "name": "download_url_type", "type_info": { "type": "String", @@ -250,8 +861,44 @@ "max_size": 48 } }, + { + "ordinal": 9, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, { "ordinal": 11, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 12, "name": "status", "type_info": { "type": "String", @@ -263,7 +910,7 @@ } }, { - "ordinal": 12, + "ordinal": 13, "name": "attempt_count", "type_info": { "type": "Long", @@ -275,7 +922,7 @@ } }, { - "ordinal": 13, + "ordinal": 14, "name": "failure_reason", "type_info": { "type": "VarString", @@ -287,7 +934,7 @@ } }, { - "ordinal": 14, + "ordinal": 15, "name": "created_at", "type_info": { "type": "Timestamp", @@ -299,7 +946,7 @@ } }, { - "ordinal": 15, + "ordinal": 16, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -311,7 +958,7 @@ } }, { - "ordinal": 16, + "ordinal": 17, "name": "retry_at", "type_info": { "type": "Timestamp", @@ -332,11 +979,12 @@ false, false, false, + true, + true, + true, + false, false, false, - true, - true, - true, false, false, false, @@ -347,12 +995,12 @@ ] } }, - "7e05d44d9301158232b9fa5789e22bc0b2193f7c619368feec791e710c5bc3e7": { - "query": "\nINSERT INTO w2l_inference_jobs\nSET\n maybe_w2l_template_token = ?,\n maybe_tts_inference_result_token = ?,\n maybe_audio_bucket_location = NULL,\n maybe_image_bucket_location = NULL,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "8347b64356502ae32979d48680e6b2097e7f8c4cdc492b7285044856b58ba017": { + "query": "\nINSERT INTO w2l_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n maybe_w2l_template_token = ?,\n maybe_public_audio_bucket_hash = ?,\n maybe_public_audio_bucket_location = ?,\n\n maybe_original_audio_filename = ?,\n maybe_audio_mime_type = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n disable_end_bump = false,\n creator_set_visibility = \"public\",\n status = \"pending\"\n ", "describe": { "columns": [], "parameters": { - "Right": 5 + "Right": 9 }, "nullable": [] } @@ -377,16 +1025,6 @@ "nullable": [] } }, - "8bda35672e05e17329659fcb56211978c727cb34210bf91173fbcb4b9389f615": { - "query": "\nINSERT INTO tts_model_upload_jobs\nSET\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 10 - }, - "nullable": [] - } - }, "8d61bb77449a78044135a3d6e5009a3a27b4671b7e40501d318b05ae401f02a8": { "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE email_address = ?\nLIMIT 1\n ", "describe": { @@ -451,6 +1089,16 @@ ] } }, + "9d4932baa4023accc2d28af98b58a47149d37c4937cd0171e6b955c09855c5f8": { + "query": "\nINSERT INTO w2l_inference_jobs\nSET\n maybe_w2l_template_token = ?,\n maybe_tts_inference_result_token = ?,\n maybe_public_audio_bucket_location = NULL,\n maybe_public_image_bucket_location = NULL,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + } + }, "ab78a81f85310750668e2c7de920c067290f2b004561ab706a5edfd893fd06fe": { "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n maybe_public_bucket_preview_image_object_name = ?,\n maybe_public_bucket_preview_video_object_name = ?,\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", "describe": { @@ -479,6 +1127,18 @@ }, { "ordinal": 1, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, "name": "uuid_idempotency_token", "type_info": { "type": "VarString", @@ -489,44 +1149,8 @@ "max_size": 144 } }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, { "ordinal": 3, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 4, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 5, "name": "title", "type_info": { "type": "VarString", @@ -538,7 +1162,7 @@ } }, { - "ordinal": 6, + "ordinal": 4, "name": "template_type", "type_info": { "type": "String", @@ -550,7 +1174,7 @@ } }, { - "ordinal": 7, + "ordinal": 5, "name": "maybe_subject_token", "type_info": { "type": "VarString", @@ -562,7 +1186,7 @@ } }, { - "ordinal": 8, + "ordinal": 6, "name": "maybe_actor_subject_token", "type_info": { "type": "VarString", @@ -574,7 +1198,7 @@ } }, { - "ordinal": 9, + "ordinal": 7, "name": "download_url", "type_info": { "type": "VarString", @@ -586,7 +1210,7 @@ } }, { - "ordinal": 10, + "ordinal": 8, "name": "download_url_type", "type_info": { "type": "String", @@ -597,8 +1221,44 @@ "max_size": 48 } }, + { + "ordinal": 9, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, { "ordinal": 11, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 12, "name": "status", "type_info": { "type": "String", @@ -610,7 +1270,7 @@ } }, { - "ordinal": 12, + "ordinal": 13, "name": "attempt_count", "type_info": { "type": "Long", @@ -622,7 +1282,7 @@ } }, { - "ordinal": 13, + "ordinal": 14, "name": "failure_reason", "type_info": { "type": "VarString", @@ -634,7 +1294,7 @@ } }, { - "ordinal": 14, + "ordinal": 15, "name": "created_at", "type_info": { "type": "Timestamp", @@ -646,7 +1306,7 @@ } }, { - "ordinal": 15, + "ordinal": 16, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -658,7 +1318,7 @@ } }, { - "ordinal": 16, + "ordinal": 17, "name": "retry_at", "type_info": { "type": "Timestamp", @@ -679,11 +1339,12 @@ false, false, false, + true, + true, + true, + false, false, false, - true, - true, - true, false, false, false, @@ -734,6 +1395,433 @@ "nullable": [] } }, + "d14d0e03acc71081726ef0e31bade27110f1ac8eff9ccfe155df5dbd47d06560": { + "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n maybe_w2l_template_token,\n maybe_public_image_bucket_location,\n maybe_tts_inference_result_token,\n maybe_public_audio_bucket_hash,\n maybe_public_audio_bucket_location,\n\n maybe_original_audio_filename,\n maybe_original_audio_download_url,\n maybe_audio_mime_type,\n\n creator_ip_address,\n maybe_creator_user_token,\n\n creator_set_visibility,\n disable_end_bump,\n disable_watermark,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM w2l_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "inference_job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_public_image_bucket_location", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_public_audio_bucket_hash", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 7, + "name": "maybe_public_audio_bucket_location", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "maybe_original_audio_filename", + "type_info": { + "type": "String", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 9, + "name": "maybe_original_audio_download_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 10, + "name": "maybe_audio_mime_type", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 12, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 13, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 14, + "name": "disable_end_bump", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "disable_watermark", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 17, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 18, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 19, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + false, + false, + false, + false, + false, + true, + false, + false, + true + ] + } + }, + "d15d602c1089dd5aaa75bc0f97697e9a51a058496efbcaa1a59ffef72dfcd2fa": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 8, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "e568e1c13676e5cd06b2136808e71d10ddd87f6ef4a39848c03531ca65833a15": { "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_address,\n email_confirmed,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n banned,\n dark_mode_preference,\n avatar_public_bucket_hash,\n disable_gravatar,\n hide_results_preference,\n auto_play_audio_preference,\n auto_play_video_preference,\n maybe_preferred_tts_model_token,\n maybe_preferred_w2l_template_token,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username\nFROM users\nLEFT OUTER JOIN user_sessions\nON users.token = user_sessions.user_token\nWHERE user_sessions.token = ?\nAND user_sessions.deleted_at IS NULL\nAND users.deleted_at IS NULL\n ", "describe": { @@ -1328,5 +2416,30 @@ false ] } + }, + "f6e625dde66da14091591f321eb65b0137fceb7168305bf17806f1876a644954": { + "query": "\nSELECT\n token as template_token\nFROM w2l_templates\nWHERE token = ?\nAND deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + } } } \ No newline at end of file diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index d3fb2b1e0a..eaa69e8ee0 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -19,6 +19,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; const NEW_USER_ROLE: &'static str = "new-user"; @@ -37,6 +38,8 @@ pub struct UploadTtsModelRequest { #[derive(Serialize)] pub struct UploadTtsModelSuccessResponse { pub success: bool, + /// This is how frontend clients can request the job execution status. + pub job_token: String, } #[derive(Serialize)] @@ -126,10 +129,19 @@ pub async fn upload_tts_model_handler( let maybe_subject_token : Option = None; let maybe_actor_subject_token : Option = None; + // This token is returned to the client. + let job_token = random_prefix_crockford_token("TTS_UP:", 32) + .map_err(|e| { + warn!("Error creating token"); + UploadTtsModelError::ServerError + })?; + + let query_result = sqlx::query!( r#" INSERT INTO tts_model_upload_jobs SET + token = ?, uuid_idempotency_token = ?, creator_user_token = ?, creator_ip_address = ?, @@ -142,6 +154,7 @@ SET download_url_type = ?, status = "pending" "#, + job_token.to_string(), uuid.to_string(), user_session.user_token.to_string(), ip_address.to_string(), @@ -189,6 +202,7 @@ SET let response = UploadTtsModelSuccessResponse { success: true, + job_token: job_token.to_string(), }; let body = serde_json::to_string(&response) diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 4d78fe5589..1d841f3c05 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -47,6 +47,7 @@ pub struct W2lTemplateExistenceRecord { #[derive(Serialize)] pub struct InferW2lWithUploadSuccessResponse { pub success: bool, + /// This is how frontend clients can request the job execution status. pub job_token: String, } diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index d017799dd7..8682f86673 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -19,6 +19,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; const NEW_USER_ROLE: &'static str = "new-user"; @@ -37,6 +38,8 @@ pub struct UploadW2lTemplateRequest { #[derive(Serialize)] pub struct UploadW2lTemplateSuccessResponse { pub success: bool, + /// This is how frontend clients can request the job execution status. + pub job_token: String, } #[derive(Serialize)] @@ -126,10 +129,19 @@ pub async fn upload_w2l_template_handler( let maybe_subject_token : Option = None; let maybe_actor_subject_token : Option = None; + // This token is returned to the client. + let job_token = random_prefix_crockford_token("W2L_UP:", 32) + .map_err(|e| { + warn!("Error creating token"); + UploadW2lTemplateError::ServerError + })?; + + let query_result = sqlx::query!( r#" INSERT INTO w2l_template_upload_jobs SET + token = ?, uuid_idempotency_token = ?, creator_user_token = ?, creator_ip_address = ?, @@ -142,6 +154,7 @@ SET download_url_type = ?, status = "pending" "#, + job_token.to_string(), uuid.to_string(), user_session.user_token.to_string(), ip_address.to_string(), @@ -189,6 +202,7 @@ SET let response = UploadW2lTemplateSuccessResponse { success: true, + job_token: job_token.to_string(), }; let body = serde_json::to_string(&response) diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index 20c25b5127..49b2a649f7 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -13,6 +13,7 @@ use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; #[derive(Debug)] pub struct TtsUploadJobRecord { pub id: i64, + pub token: String, pub uuid_idempotency_token: String, pub creator_user_token: String, pub creator_ip_address: String, diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 033fa40fb5..a8ea1ee508 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -13,6 +13,7 @@ use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; #[derive(Debug)] pub struct W2lTemplateUploadJobRecord { pub id: i64, + pub token: String, pub uuid_idempotency_token: String, pub creator_user_token: String, pub creator_ip_address: String, diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 68df41a0be..aa93e1874b 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -494,10 +494,11 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An file_metadata.duration_millis.unwrap_or(0)) .await?; + info!("Marking job complete..."); + mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?; + info!("Job {} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}", job.id, id); - mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?; - Ok(()) } From ece7d66e58977962ec6b8d0a649d271b107825ce Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 15:12:11 -0400 Subject: [PATCH 0193/1042] end bump --- .env | 1 + models/Wav2Lip/vocodes_process_inference.py | 7 ++--- src/buckets/bucket_path_unifier.rs | 29 ++++++++++++++--- .../wav2lip_inference_command.rs | 24 ++++++++------ src/util/semi_persistent_cache_dir.rs | 17 ++++++++++ src/w2l_inference_job.rs | 31 +++++++++++++++++-- 6 files changed, 88 insertions(+), 21 deletions(-) diff --git a/.env b/.env index 06dd83f350..e7f1c54d72 100644 --- a/.env +++ b/.env @@ -27,6 +27,7 @@ W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip #W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/s3fd-619a316812.pth W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/wav2lip_gan.pth W2L_MODEL_FILENAME=wav2lip_gan.pth +W2L_END_BUMP_FILENAME=vocodes-short-end-bump.mp4 W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py W2L_INFERENCE_SCRIPT_NAME=vocodes_process_inference.py diff --git a/models/Wav2Lip/vocodes_process_inference.py b/models/Wav2Lip/vocodes_process_inference.py index 37e7329834..2348a4b976 100644 --- a/models/Wav2Lip/vocodes_process_inference.py +++ b/models/Wav2Lip/vocodes_process_inference.py @@ -48,6 +48,9 @@ parser.add_argument('--cached_faces_filename', type=str, parser.add_argument('--audio_filename', type=str, help='Filepath of video/audio file to use as raw audio source', required=True) +parser.add_argument('--end_bump_file', type=str, + help='Video file to concatenate at the end') + parser.add_argument('--output_video_filename', type=str, help='Output filename for the final video', required=True) @@ -95,10 +98,6 @@ parser.add_argument('--audio_start_pad_millis', default=0, type=int, parser.add_argument('--audio_end_pad_millis', default=0, type=int, help='Seconds to pad the end of the audio') -parser.add_argument('--end_bump_file', type=str, - help='Video file to concatenate at the end') - - # Purely for debugging on the host machine: parser.add_argument('--preserve_tempdir', default=False, action='store_true', help='Keep the tempdir arround for debugging') diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs index e47ea2e0a1..88dec45496 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/buckets/bucket_path_unifier.rs @@ -10,6 +10,7 @@ pub struct BucketPathUnifier { pub tts_inference_output_root: PathBuf, pub w2l_inference_output_root: PathBuf, pub w2l_model_root: PathBuf, + pub w2l_end_bump_root: PathBuf, } impl BucketPathUnifier { @@ -29,18 +30,20 @@ impl BucketPathUnifier { tts_inference_output_root: PathBuf::from("/tts_inference_output"), w2l_inference_output_root: PathBuf::from("/w2l_inference_output"), w2l_model_root: PathBuf::from("/w2l_pretrained_models"), + w2l_end_bump_root: PathBuf::from("/w2l_end_bumps"), } } - pub fn end_bump_video_for_w2l_path(&self, end_bump_filename: &str) -> String { - "".to_string() - } - // W2L pretrained models. There are only two. pub fn w2l_pretrained_models_path(&self, w2l_model_name: &str) -> PathBuf { self.w2l_model_root.join(w2l_model_name) } + // W2L "end bumps" are videos added at the end. + pub fn end_bump_video_for_w2l_path(&self, end_bump_filename: &str) -> PathBuf { + self.w2l_end_bump_root.join(end_bump_filename) + } + // The video or images uploaded as templates // eg. /user_uploaded_w2l_templates/1/5/1/151a[...60]... pub fn media_templates_for_w2l_path(&self, template_file_hash: &str) -> PathBuf { @@ -79,6 +82,7 @@ impl BucketPathUnifier { let video_filename = format!("vocodes_video_{}.mp4", video_filename); let hashed_path = Self::hashed_directory_path(token); + let hashed_path = hashed_path.to_lowercase(); return self.w2l_inference_output_root .join(hashed_path) @@ -119,6 +123,7 @@ mod tests { tts_inference_output_root: PathBuf::from("/test_path_tts_output"), w2l_inference_output_root: PathBuf::from("/test_path_w2l_output"), w2l_model_root: PathBuf::from("/test_path_w2l_pretrained_models"), + w2l_end_bump_root: PathBuf::from("/test_path_w2l_end_bumps"), } } @@ -129,6 +134,13 @@ mod tests { "/test_path_w2l_pretrained_models/model.pth"); } + #[test] + fn test_end_bump_video_for_w2l_path() { + let paths = get_instance(); + assert_eq!(paths.end_bump_video_for_w2l_path("logo.mp4").to_str().unwrap(), + "/test_path_w2l_end_bumps/logo.mp4"); + } + #[test] fn test_user_audio_for_w2l_inference_path() { let paths = get_instance(); @@ -153,12 +165,19 @@ mod tests { #[test] fn test_w2l_inference_video_output_path() { let paths = get_instance(); + + // Case 1: Tokens without a "token type" assert_eq!(paths.w2l_inference_video_output_path("foobar").to_str().unwrap(), "/test_path_w2l_output/f/o/o/vocodes_video_foobar.mp4"); - // Note: it also removes the token type from dir path and handles the colon: + // Case 2: Tokens with a "token type" + // Note: that it removes the token type from dir path and handles the colon: assert_eq!(paths.w2l_inference_video_output_path("type:abcdef").to_str().unwrap(), "/test_path_w2l_output/a/b/c/vocodes_video_typeabcdef.mp4"); + + // It also handles capitalization + assert_eq!(paths.w2l_inference_video_output_path("TYPE:ABCDEF").to_str().unwrap(), + "/test_path_w2l_output/a/b/c/vocodes_video_TYPEABCDEF.mp4"); } #[test] diff --git a/src/script_execution/wav2lip_inference_command.rs b/src/script_execution/wav2lip_inference_command.rs index 346425eae4..bfe83a6be9 100644 --- a/src/script_execution/wav2lip_inference_command.rs +++ b/src/script_execution/wav2lip_inference_command.rs @@ -27,16 +27,18 @@ impl Wav2LipInferenceCommand { } } - pub fn execute>(&self, - checkpoint_path: P, - audio_filename: P, - media_template_filename: P, - cached_faces_filename: P, - output_metadata_filename: P, - output_video_filename: P, - is_image: bool, - spawn_process: bool) -> AnyhowResult<()> - { + pub fn execute>( + &self, + checkpoint_path: P, + audio_filename: P, + end_bump_video_filename: P, + media_template_filename: P, + cached_faces_filename: P, + output_metadata_filename: P, + output_video_filename: P, + is_image: bool, + spawn_process: bool + ) -> AnyhowResult<()> { let mut command = String::new(); command.push_str("echo 'test'"); @@ -53,6 +55,8 @@ impl Wav2LipInferenceCommand { command.push_str(&media_template_filename.as_ref().display().to_string()); command.push_str(" --audio_filename "); command.push_str(&audio_filename.as_ref().display().to_string()); + command.push_str(" --end_bump_file "); + command.push_str(&end_bump_video_filename.as_ref().display().to_string()); command.push_str(" --cached_faces_filename "); command.push_str(&cached_faces_filename.as_ref().display().to_string()); command.push_str(" --output_video_filename "); diff --git a/src/util/semi_persistent_cache_dir.rs b/src/util/semi_persistent_cache_dir.rs index 6e3bd89875..30f3211858 100644 --- a/src/util/semi_persistent_cache_dir.rs +++ b/src/util/semi_persistent_cache_dir.rs @@ -8,6 +8,7 @@ pub struct SemiPersistentCacheDir { cache_root: PathBuf, tts_model_root: PathBuf, w2l_model_root: PathBuf, + w2l_end_bump_root: PathBuf, w2l_face_templates_root: PathBuf, w2l_templates_media_root: PathBuf, video_asset_root: PathBuf, // end bump, etc. @@ -28,6 +29,7 @@ impl SemiPersistentCacheDir { cache_root: cache_root.clone(), tts_model_root: cache_root.join("tts/models/"), w2l_model_root: cache_root.join("w2l/models/"), + w2l_end_bump_root: cache_root.join("w2l/end_bumps/"), w2l_face_templates_root: cache_root.join("w2l/face_templates/"), w2l_templates_media_root: cache_root.join("w2l/template_media/"), video_asset_root: cache_root.join("static_video_assets/"), @@ -58,6 +60,21 @@ impl SemiPersistentCacheDir { Ok(()) } + // ==================== W2L END BUMPS (short video logos) ==================== + + pub fn w2l_end_bump_path(&self, end_bump_filename: &str) -> PathBuf { + self.w2l_end_bump_root.join(end_bump_filename) + } + + pub fn w2l_end_bump_directory(&self) -> &Path { + &self.w2l_end_bump_root + } + + pub fn create_w2l_end_bump_path(&self) -> AnyhowResult<()> { + let _ = fs::create_dir_all(self.w2l_end_bump_directory())?; + Ok(()) + } + // ==================== W2L MEDIA ==================== /// We cache W2L media here. diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index aa93e1874b..c43d2debf7 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -89,6 +89,7 @@ struct Inferencer { // Command to run pub inference_script: String, pub w2l_model_filename: String, + pub w2l_end_bump_filename: String, } #[tokio::main] @@ -179,13 +180,16 @@ async fn main() -> AnyhowResult<()> { info!("Creating pod semi-persistent cache dirs..."); semi_persistent_cache.create_w2l_model_path()?; + semi_persistent_cache.create_w2l_end_bump_path()?; semi_persistent_cache.create_w2l_face_template_path()?; semi_persistent_cache.create_w2l_template_media_path()?; - semi_persistent_cache.create_w2l_model_path()?; let w2l_model_filename = easyenv::get_env_string_or_default( "W2L_MODEL_FILENAME", "wav2lip_gan.pth"); + let w2l_end_bump_filename = easyenv::get_env_string_or_default( + "W2L_END_BUMP_FILENAME", "vocodes-short-end-bump.mp4"); + let inferencer = Inferencer { download_temp_directory: temp_directory, mysql_pool, @@ -200,6 +204,7 @@ async fn main() -> AnyhowResult<()> { bucket_path_unifier: BucketPathUnifier::default_paths(), semi_persistent_cache, w2l_model_filename, + w2l_end_bump_filename, }; main_loop(inferencer).await; @@ -314,7 +319,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An let model_fs_path = inferencer.semi_persistent_cache.w2l_model_path(&model_filename); if !model_fs_path.exists() { - info!("Model file does not exist: {:?}", &model_fs_path); + warn!("Model file does not exist: {:?}", &model_fs_path); let model_object_path = inferencer.bucket_path_unifier .w2l_pretrained_models_path(&model_filename); @@ -329,6 +334,27 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An info!("Downloaded model from bucket!"); } + // ==================== CONFIRM OR DOWNLOAD W2L END BUMP ==================== // + + let end_bump_filename = inferencer.w2l_end_bump_filename.clone(); + let end_bump_fs_path = inferencer.semi_persistent_cache.w2l_end_bump_path(&end_bump_filename); + + if !end_bump_fs_path.exists() { + warn!("End bump file does not exist: {:?}", &end_bump_fs_path); + + let end_bump_object_path = inferencer.bucket_path_unifier + .end_bump_video_for_w2l_path(&end_bump_filename); + + info!("Download from bucket path: {:?}", &end_bump_object_path); + + inferencer.private_bucket_client.download_file_to_disk( + &end_bump_object_path, + &end_bump_fs_path + ).await?; + + info!("Downloaded end bump from bucket!"); + } + // ==================== LOOK UP TEMPLATE RECORD ==================== // let template_token = match &job.maybe_w2l_template_token { @@ -442,6 +468,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An inferencer.w2l_inference.execute( &model_fs_path, &audio_fs_path, + &end_bump_fs_path, &template_media_fs_path, &face_template_fs_path, &output_metadata_fs_path, From 42d221bd79da7edc068fc85f97b44a93fb442792 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 16:25:55 -0400 Subject: [PATCH 0194/1042] endpoint to list w2l templates --- src/common_queries/list_w2l_templates.rs | 249 ++++++++++++++++++ src/common_queries/mod.rs | 5 +- .../users/list_user_w2l_templates.rs | 114 ++++++++ src/http_server/endpoints/users/mod.rs | 1 + src/main.rs | 12 +- 5 files changed, 378 insertions(+), 3 deletions(-) create mode 100644 src/common_queries/list_w2l_templates.rs create mode 100644 src/http_server/endpoints/users/list_user_w2l_templates.rs diff --git a/src/common_queries/list_w2l_templates.rs b/src/common_queries/list_w2l_templates.rs new file mode 100644 index 0000000000..d9a7994706 --- /dev/null +++ b/src/common_queries/list_w2l_templates.rs @@ -0,0 +1,249 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::util::anyhow_result::AnyhowResult; +use log::{warn, info}; +use sqlx::MySqlPool; + +#[derive(Serialize)] +pub struct W2lTemplateRecordForList { + pub template_token: String, + pub template_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: u32, + pub frame_height: u32, + pub duration_millis: u32, + pub maybe_image_object_name: Option, + pub maybe_video_object_name: Option, + pub is_mod_approved: bool, // converted + pub created_at: DateTime, + pub updated_at: DateTime, +} + +struct RawW2lTemplateRecordForList { + pub template_token: String, + pub template_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + pub maybe_public_bucket_preview_image_object_name: Option, + pub maybe_public_bucket_preview_video_object_name: Option, + pub is_mod_approved: i8, // NB: needs conversion + pub created_at: DateTime, + pub updated_at: DateTime, +} + +pub async fn list_w2l_templates( + mysql_pool: &MySqlPool, + scope_creator_username: Option<&str>, + require_mod_approved: bool +) -> AnyhowResult> { + + let maybe_templates = match scope_creator_username { + Some(username) => { + list_w2l_templates_creator_scoped(mysql_pool, username, require_mod_approved) + .await + }, + None => { + list_w2l_templates_for_all_creators(mysql_pool, require_mod_approved) + .await + }, + }; + + let templates : Vec = match maybe_templates { + Ok(templates) => { + info!("Template length: {}", templates.len()); + templates + }, + Err(err) => { + warn!("Error: {:?}", err); + + match err { + RowNotFound => { + return Ok(Vec::new()); + }, + _ => { + warn!("w2l template list query error: {:?}", err); + return Err(anyhow!("w2l template list query error")); + } + } + } + }; + + Ok(templates.into_iter() + .map(|template| { + W2lTemplateRecordForList { + template_token: template.template_token.clone(), + template_type: template.template_type.clone(), + creator_user_token: template.creator_user_token.clone(), + creator_username: template.creator_username.clone(), + creator_display_name: template.creator_display_name.clone(), + updatable_slug: template.updatable_slug.clone(), + title: template.title.clone(), + frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, + frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, + duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, + maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), + maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), + is_mod_approved: if template.is_mod_approved == 0 { false } else { true }, + created_at: template.created_at.clone(), + updated_at: template.updated_at.clone(), + } + }) + .collect::>()) +} + +async fn list_w2l_templates_for_all_creators( + mysql_pool: &MySqlPool, + require_mod_approved: bool +) -> AnyhowResult> { + // TODO: There has to be a better way. + // Sqlx doesn't like anything except string literals. + let maybe_templates = if require_mod_approved { + info!("listing w2l templates for everyone; mod-approved only"); + sqlx::query_as!( + RawW2lTemplateRecordForList, + r#" +SELECT + w2l.token as template_token, + w2l.template_type, + w2l.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + w2l.updatable_slug, + w2l.title, + w2l.frame_width, + w2l.frame_height, + w2l.duration_millis, + w2l.maybe_public_bucket_preview_image_object_name, + w2l.maybe_public_bucket_preview_video_object_name, + w2l.is_mod_approved, + w2l.created_at, + w2l.updated_at +FROM w2l_templates as w2l +JOIN users +ON users.token = w2l.creator_user_token +WHERE + w2l.deleted_at IS NULL + AND w2l.is_mod_approved IS TRUE + "#) + .fetch_all(mysql_pool) + .await? + } else { + info!("listing w2l templates for everyone; all"); + sqlx::query_as!( + RawW2lTemplateRecordForList, + r#" +SELECT + w2l.token as template_token, + w2l.template_type, + w2l.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + w2l.updatable_slug, + w2l.title, + w2l.frame_width, + w2l.frame_height, + w2l.duration_millis, + w2l.maybe_public_bucket_preview_image_object_name, + w2l.maybe_public_bucket_preview_video_object_name, + w2l.is_mod_approved, + w2l.created_at, + w2l.updated_at +FROM w2l_templates as w2l +JOIN users +ON users.token = w2l.creator_user_token +WHERE + w2l.deleted_at IS NULL + "#) + .fetch_all(mysql_pool) + .await? + }; + + Ok(maybe_templates) +} + +async fn list_w2l_templates_creator_scoped( + mysql_pool: &MySqlPool, + scope_creator_username: &str, + require_mod_approved: bool +) -> AnyhowResult> { + // TODO: There has to be a better way. + // Sqlx doesn't like anything except string literals. + let maybe_templates = if require_mod_approved { + info!("listing w2l templates for user; mod-approved only"); + sqlx::query_as!( + RawW2lTemplateRecordForList, + r#" +SELECT + w2l.token as template_token, + w2l.template_type, + w2l.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + w2l.updatable_slug, + w2l.title, + w2l.frame_width, + w2l.frame_height, + w2l.duration_millis, + w2l.maybe_public_bucket_preview_image_object_name, + w2l.maybe_public_bucket_preview_video_object_name, + w2l.is_mod_approved, + w2l.created_at, + w2l.updated_at +FROM w2l_templates as w2l +JOIN users +ON + users.token = w2l.creator_user_token +WHERE + w2l.deleted_at IS NULL + AND w2l.is_mod_approved IS TRUE + AND users.username = ? + "#, + scope_creator_username) + .fetch_all(mysql_pool) + .await? + } else { + info!("listing w2l templates for user; all"); + sqlx::query_as!( + RawW2lTemplateRecordForList, + r#" +SELECT + w2l.token as template_token, + w2l.template_type, + w2l.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + w2l.updatable_slug, + w2l.title, + w2l.frame_width, + w2l.frame_height, + w2l.duration_millis, + w2l.maybe_public_bucket_preview_image_object_name, + w2l.maybe_public_bucket_preview_video_object_name, + w2l.is_mod_approved, + w2l.created_at, + w2l.updated_at +FROM w2l_templates as w2l +JOIN users +ON + users.token = w2l.creator_user_token +WHERE + w2l.deleted_at IS NULL + AND users.username = ? + "#, + scope_creator_username) + .fetch_all(mysql_pool) + .await? + }; + + Ok(maybe_templates) +} diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index ca47e7283b..51ead07798 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -1,3 +1,4 @@ -pub mod users; pub mod badges; -pub mod sessions; \ No newline at end of file +pub mod list_w2l_templates; +pub mod sessions; +pub mod users; \ No newline at end of file diff --git a/src/http_server/endpoints/users/list_user_w2l_templates.rs b/src/http_server/endpoints/users/list_user_w2l_templates.rs new file mode 100644 index 0000000000..c774499bb1 --- /dev/null +++ b/src/http_server/endpoints/users/list_user_w2l_templates.rs @@ -0,0 +1,114 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::list_w2l_templates::{W2lTemplateRecordForList, list_w2l_templates}; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use actix_web::web::Path; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetProfilePathInfo { + username: String, +} + +#[derive(Serialize)] +pub struct ListW2lTemplatesForUserSuccessResponse { + pub success: bool, + pub templates: Vec, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum ListW2lTemplatesForUserError { + ServerError, +} + +impl ResponseError for ListW2lTemplatesForUserError { + fn status_code(&self) -> StatusCode { + match *self { + ListW2lTemplatesForUserError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListW2lTemplatesForUserError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn list_user_w2l_templates_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result +{ + info!("Fetching templates for user: {}", &path.username); + + let query_results = list_w2l_templates( + &server_state.mysql_pool, + Some(path.username.as_ref()), + false + ).await; + + let templates = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListW2lTemplatesForUserError::ServerError); + } + }; + + let response = ListW2lTemplatesForUserSuccessResponse { + success: true, + templates: templates, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListW2lTemplatesForUserError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/users/mod.rs b/src/http_server/endpoints/users/mod.rs index b2811538a7..7306f7227e 100644 --- a/src/http_server/endpoints/users/mod.rs +++ b/src/http_server/endpoints/users/mod.rs @@ -1,5 +1,6 @@ pub mod create_account; pub mod get_profile; +pub mod list_user_w2l_templates; pub mod login; pub mod logout; pub mod session_info; diff --git a/src/main.rs b/src/main.rs index 77aea47895..00581fc107 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,6 +50,7 @@ use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handl use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; use crate::buckets::bucket_client::BucketClient; +use crate::http_server::endpoints::users::list_user_w2l_templates::list_user_w2l_templates_handler; /// /// TODO TODO TODO TODO @@ -60,7 +61,8 @@ use crate::buckets::bucket_client::BucketClient; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; // NB: sqlx::query is spammy and logs all queries as "info"-level -const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; +//const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -286,6 +288,14 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_profile_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::scope("/user") + .service( + web::resource("/{username}/w2l_templates") + .route(web::get().to(list_user_w2l_templates_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) .service(get_root_index) .service(enable_alpha) .default_service( web::route().to(default_route_404)) From f90839f87f5482c2a0bc5a0ee4f4c330a9d1f75c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 16:29:27 -0400 Subject: [PATCH 0195/1042] fix query, share code --- .../users/list_user_w2l_templates.rs | 2 +- .../endpoints/w2l/list_w2l_templates.rs | 77 ++++--------------- 2 files changed, 15 insertions(+), 64 deletions(-) diff --git a/src/http_server/endpoints/users/list_user_w2l_templates.rs b/src/http_server/endpoints/users/list_user_w2l_templates.rs index c774499bb1..6bd5f5a995 100644 --- a/src/http_server/endpoints/users/list_user_w2l_templates.rs +++ b/src/http_server/endpoints/users/list_user_w2l_templates.rs @@ -102,7 +102,7 @@ pub async fn list_user_w2l_templates_handler( let response = ListW2lTemplatesForUserSuccessResponse { success: true, - templates: templates, + templates, }; let body = serde_json::to_string(&response) diff --git a/src/http_server/endpoints/w2l/list_w2l_templates.rs b/src/http_server/endpoints/w2l/list_w2l_templates.rs index 754a57e057..15e57bc313 100644 --- a/src/http_server/endpoints/w2l/list_w2l_templates.rs +++ b/src/http_server/endpoints/w2l/list_w2l_templates.rs @@ -24,6 +24,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use chrono::{DateTime, Utc}; +use crate::common_queries::list_w2l_templates::{list_w2l_templates, W2lTemplateRecordForList}; #[derive(Serialize)] pub struct W2lTemplateRecordForResponse { @@ -46,7 +47,7 @@ pub struct W2lTemplateRecordForResponse { #[derive(Serialize)] pub struct ListW2lTemplatesSuccessResponse { pub success: bool, - pub templates: Vec, + pub templates: Vec, } #[derive(Serialize)] @@ -110,74 +111,24 @@ pub async fn list_w2l_templates_handler( http_request: HttpRequest, server_state: web::Data>) -> Result { - // NB: Lookup failure is Err(RowNotFound). - // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_templates = sqlx::query_as!( - W2lTemplateRecord, - r#" -SELECT - w2l.token as template_token, - w2l.template_type, - w2l.creator_user_token, - users.username as creator_username, - users.display_name as creator_display_name, - w2l.updatable_slug, - w2l.title, - w2l.frame_width, - w2l.frame_height, - w2l.duration_millis, - w2l.maybe_public_bucket_preview_image_object_name, - w2l.maybe_public_bucket_preview_video_object_name, - w2l.created_at, - w2l.updated_at -FROM w2l_templates as w2l -JOIN users -ON users.token = w2l.creator_user_token -WHERE w2l.deleted_at IS NULL - "#, - ) - .fetch_all(&server_state.mysql_pool) - .await; // TODO: This will return error if it doesn't exist + let scope_creator = None; + let query_results = list_w2l_templates( + &server_state.mysql_pool, + scope_creator, + false, + ).await; - let templates : Vec = match maybe_templates { - Ok(templates) => templates, - Err(err) => { - match err { - RowNotFound => { - return Err(ListW2lTemplatesError::ServerError); - }, - _ => { - warn!("w2l template list query error: {:?}", err); - return Err(ListW2lTemplatesError::ServerError); - } - } + let templates = match query_results { + Ok(results) => results, + Err(e) => { + warn!("w2l template list query error: {:?}", e); + return Err(ListW2lTemplatesError::ServerError); } }; - let templates_for_response = templates.into_iter() - .map(|template| { - W2lTemplateRecordForResponse { - template_token: template.template_token.clone(), - template_type: template.template_type.clone(), - creator_user_token: template.creator_user_token.clone(), - creator_username: template.creator_username.clone(), - creator_display_name: template.creator_display_name.clone(), - updatable_slug: template.updatable_slug.clone(), - title: template.title.clone(), - frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, - frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, - duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, - maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), - maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), - created_at: template.created_at.clone(), - updated_at: template.updated_at.clone(), - } - }) - .collect::>(); - let response = ListW2lTemplatesSuccessResponse { success: true, - templates: templates_for_response, + templates, }; let body = serde_json::to_string(&response) From cf060ea1ff16c04936ec1232e1c96398f9d7e5f6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 19:15:47 -0400 Subject: [PATCH 0196/1042] user inference result endpoint (wip) --- .../up.sql | 10 +- .../list_w2l_inference_results.rs | 282 ++++++++++++++++++ src/common_queries/mod.rs | 1 + .../users/list_user_w2l_inference_results.rs | 115 +++++++ src/http_server/endpoints/users/mod.rs | 1 + src/main.rs | 10 +- 6 files changed, 415 insertions(+), 4 deletions(-) create mode 100644 src/common_queries/list_w2l_inference_results.rs create mode 100644 src/http_server/endpoints/users/list_user_w2l_inference_results.rs diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 108ff3a44e..3c40113267 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -61,7 +61,10 @@ CREATE TABLE tts_results ( KEY fk_model_token (model_token), KEY fk_maybe_creator_user_token (maybe_creator_user_token), KEY fk_maybe_mod_user_token (maybe_mod_user_token), - KEY index_creator_ip_address (creator_ip_address) + KEY index_creator_ip_address (creator_ip_address), + KEY index_is_mod_hidden_from_public(is_mod_hidden_from_public), + KEY index_deleted_at(deleted_at), + KEY index_deleted_at_and_is_mod_hidden_from_public(deleted_at, is_mod_hidden_from_public) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -134,6 +137,9 @@ CREATE TABLE w2l_results ( KEY fk_maybe_w2l_template_token (maybe_w2l_template_token), KEY fk_maybe_tts_inference_result_token (maybe_tts_inference_result_token), KEY fk_maybe_mod_user_token (maybe_mod_user_token), - KEY index_creator_ip_address (creator_ip_address) + KEY index_creator_ip_address (creator_ip_address), + KEY index_is_mod_hidden_from_public(is_mod_hidden_from_public), + KEY index_deleted_at(deleted_at), + KEY index_deleted_at_and_is_mod_hidden_from_public(deleted_at, is_mod_hidden_from_public) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/common_queries/list_w2l_inference_results.rs new file mode 100644 index 0000000000..8ad4eb5973 --- /dev/null +++ b/src/common_queries/list_w2l_inference_results.rs @@ -0,0 +1,282 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::util::anyhow_result::AnyhowResult; +use log::{warn, info}; +use sqlx::MySqlPool; + +#[derive(Serialize)] +pub struct W2lInferenceRecordForList { + pub w2l_result_token: String, + pub maybe_w2l_template_token: Option, + pub maybe_tts_inference_result_token: Option, + + pub template_type: Option, + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + + pub file_size_bytes: u32, + pub frame_width: u32, + pub frame_height: u32, + pub duration_millis: u32, + + //pub is_mod_hidden_from_public: bool, // converted + //pub template_is_mod_approved: bool, // converted + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +struct RawW2lInferenceRecordForList { + pub w2l_result_token: String, + + pub maybe_w2l_template_token: Option, + pub maybe_tts_inference_result_token: Option, + + pub template_type: Option, + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + + pub file_size_bytes: i32, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + + //pub is_mod_hidden_from_public: i8, // needs convert + //pub template_is_mod_approved: i8, // needs convert + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +pub async fn list_w2l_inference_results( + mysql_pool: &MySqlPool, + scope_creator_username: Option<&str>, + require_mod_approved_templates: bool +) -> AnyhowResult> { + + let maybe_results = match scope_creator_username { + Some(username) => { + list_w2l_inference_results_creator_scoped(mysql_pool, username, require_mod_approved_templates) + .await + }, + None => { + list_w2l_inference_results_for_all_creators(mysql_pool, require_mod_approved_templates) + .await + }, + }; + + let results : Vec = match maybe_results { + Ok(results) => { + info!("Results length: {}", results.len()); + results + }, + Err(err) => { + warn!("Error: {:?}", err); + + match err { + RowNotFound => { + return Ok(Vec::new()); + }, + _ => { + warn!("w2l inference result list query error: {:?}", err); + return Err(anyhow!("w2l inference result list query error")); + } + } + } + }; + + Ok(results.into_iter() + .map(|ir| { + W2lInferenceRecordForList { + w2l_result_token: ir.w2l_result_token.clone(), + maybe_w2l_template_token: ir.maybe_w2l_template_token.clone(), + maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), + + template_type: ir.template_type.clone(), + + maybe_creator_user_token: None, + maybe_creator_username: None, + maybe_creator_display_name: None, + //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, + //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, + + file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, + frame_width: if ir.frame_width > 0 { ir.frame_width as u32 } else { 0 }, + frame_height: if ir.frame_height > 0 { ir.frame_height as u32 } else { 0 }, + duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, + + created_at: ir.created_at.clone(), + updated_at: ir.updated_at.clone(), + } + }) + .collect::>()) +} + +async fn list_w2l_inference_results_for_all_creators( + mysql_pool: &MySqlPool, + require_mod_approved_templates: bool +) -> AnyhowResult> { + // TODO: There has to be a better way. + // Sqlx doesn't like anything except string literals. + let maybe_results = if require_mod_approved_templates { + info!("listing w2l inference results for everyone; mod-approved only"); + sqlx::query_as!( + RawW2lInferenceRecordForList, + r#" +SELECT + w2l_results.token as w2l_result_token, + w2l_results.maybe_tts_inference_result_token, + + w2l_templates.token as maybe_w2l_template_token, + w2l_templates.template_type, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + w2l_results.file_size_bytes, + w2l_results.frame_width, + w2l_results.frame_height, + w2l_results.duration_millis, + w2l_results.created_at, + w2l_results.updated_at + +FROM w2l_results +LEFT OUTER JOIN w2l_templates + ON w2l_results.maybe_w2l_template_token = w2l_templates.token +LEFT OUTER JOIN users + ON w2l_results.maybe_creator_user_token = users.token +WHERE + w2l_results.deleted_at IS NULL + AND w2l_results.is_mod_hidden_from_public IS FALSE + AND w2l_templates.is_mod_approved IS TRUE + "#) + .fetch_all(mysql_pool) + .await? + } else { + info!("listing w2l inference results for everyone; all"); + sqlx::query_as!( + RawW2lInferenceRecordForList, + r#" +SELECT + w2l_results.token as w2l_result_token, + w2l_results.maybe_tts_inference_result_token, + + w2l_templates.token as maybe_w2l_template_token, + w2l_templates.template_type, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + w2l_results.file_size_bytes, + w2l_results.frame_width, + w2l_results.frame_height, + w2l_results.duration_millis, + w2l_results.created_at, + w2l_results.updated_at + +FROM w2l_results +LEFT OUTER JOIN w2l_templates + ON w2l_results.maybe_w2l_template_token = w2l_templates.token +LEFT OUTER JOIN users + ON w2l_results.maybe_creator_user_token = users.token +WHERE + w2l_results.deleted_at IS NULL + "#) + .fetch_all(mysql_pool) + .await? + }; + + Ok(maybe_results) +} + +async fn list_w2l_inference_results_creator_scoped( + mysql_pool: &MySqlPool, + scope_creator_username: &str, + require_mod_approved_templates: bool +) -> AnyhowResult> { + // TODO: There has to be a better way. + // Sqlx doesn't like anything except string literals. + let maybe_results = if require_mod_approved_templates { + info!("listing w2l inference results for user; mod-approved only"); + sqlx::query_as!( + RawW2lInferenceRecordForList, + r#" +SELECT + w2l_results.token as w2l_result_token, + w2l_results.maybe_tts_inference_result_token, + + w2l_templates.token as maybe_w2l_template_token, + w2l_templates.template_type, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + w2l_results.file_size_bytes, + w2l_results.frame_width, + w2l_results.frame_height, + w2l_results.duration_millis, + w2l_results.created_at, + w2l_results.updated_at + +FROM w2l_results +LEFT OUTER JOIN w2l_templates + ON w2l_results.maybe_w2l_template_token = w2l_templates.token +LEFT OUTER JOIN users + ON w2l_results.maybe_creator_user_token = users.token +WHERE + w2l_results.deleted_at IS NULL + AND w2l_results.is_mod_hidden_from_public IS FALSE + AND w2l_templates.is_mod_approved IS TRUE + AND users.username = ? + "#, + scope_creator_username) + .fetch_all(mysql_pool) + .await? + } else { + info!("listing w2l inference results for user; all"); + sqlx::query_as!( + RawW2lInferenceRecordForList, + r#" +SELECT + w2l_results.token as w2l_result_token, + w2l_results.maybe_tts_inference_result_token, + + w2l_templates.token as maybe_w2l_template_token, + w2l_templates.template_type, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + w2l_results.file_size_bytes, + w2l_results.frame_width, + w2l_results.frame_height, + w2l_results.duration_millis, + w2l_results.created_at, + w2l_results.updated_at + +FROM w2l_results +LEFT OUTER JOIN w2l_templates + ON w2l_results.maybe_w2l_template_token = w2l_templates.token +LEFT OUTER JOIN users + ON w2l_results.maybe_creator_user_token = users.token +WHERE + w2l_results.deleted_at IS NULL + AND users.username = ? + "#, + scope_creator_username) + .fetch_all(mysql_pool) + .await? + }; + + Ok(maybe_results) +} diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index 51ead07798..0d32c80f74 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -1,4 +1,5 @@ pub mod badges; +pub mod list_w2l_inference_results; pub mod list_w2l_templates; pub mod sessions; pub mod users; \ No newline at end of file diff --git a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs new file mode 100644 index 0000000000..ce5a17ada7 --- /dev/null +++ b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs @@ -0,0 +1,115 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::list_w2l_inference_results::W2lInferenceRecordForList; +use crate::common_queries::list_w2l_inference_results::list_w2l_inference_results; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use actix_web::web::Path; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetProfilePathInfo { + username: String, +} + +#[derive(Serialize)] +pub struct ListW2lInferenceResultsForUserSuccessResponse { + pub success: bool, + pub results: Vec, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum ListW2lInferenceResultsForUserError { + ServerError, +} + +impl ResponseError for ListW2lInferenceResultsForUserError { + fn status_code(&self) -> StatusCode { + match *self { + ListW2lInferenceResultsForUserError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListW2lInferenceResultsForUserError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn list_user_w2l_inference_results_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result +{ + info!("Fetching inference results for user: {}", &path.username); + + let query_results = list_w2l_inference_results( + &server_state.mysql_pool, + Some(path.username.as_ref()), + false + ).await; + + let results = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListW2lInferenceResultsForUserError::ServerError); + } + }; + + let response = ListW2lInferenceResultsForUserSuccessResponse { + success: true, + results, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListW2lInferenceResultsForUserError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/users/mod.rs b/src/http_server/endpoints/users/mod.rs index 7306f7227e..f16ee1b6cb 100644 --- a/src/http_server/endpoints/users/mod.rs +++ b/src/http_server/endpoints/users/mod.rs @@ -1,5 +1,6 @@ pub mod create_account; pub mod get_profile; +pub mod list_user_w2l_inference_results; pub mod list_user_w2l_templates; pub mod login; pub mod logout; diff --git a/src/main.rs b/src/main.rs index 00581fc107..d0e5e35abd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -51,6 +51,7 @@ use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; use crate::buckets::bucket_client::BucketClient; use crate::http_server::endpoints::users::list_user_w2l_templates::list_user_w2l_templates_handler; +use crate::http_server::endpoints::users::list_user_w2l_inference_results::list_user_w2l_inference_results_handler; /// /// TODO TODO TODO TODO @@ -61,8 +62,8 @@ use crate::http_server::endpoints::users::list_user_w2l_templates::list_user_w2l const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; // NB: sqlx::query is spammy and logs all queries as "info"-level -//const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; -const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; +//const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -295,6 +296,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(list_user_w2l_templates_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/{username}/w2l_results") + .route(web::get().to(list_user_w2l_inference_results_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service(get_root_index) .service(enable_alpha) From bef1036d406d5ba73302f7f9a3d06d5ebefb47e2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 19:59:00 -0400 Subject: [PATCH 0197/1042] session debugging --- .../list_w2l_inference_results.rs | 2 +- .../endpoints/w2l/enqueue_infer_w2l.rs | 8 +++-- .../w2l/enqueue_infer_w2l_with_uploads.rs | 33 ++++++++++--------- src/main.rs | 6 ---- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/common_queries/list_w2l_inference_results.rs index 8ad4eb5973..bbcd2fc8e6 100644 --- a/src/common_queries/list_w2l_inference_results.rs +++ b/src/common_queries/list_w2l_inference_results.rs @@ -242,7 +242,7 @@ WHERE .fetch_all(mysql_pool) .await? } else { - info!("listing w2l inference results for user; all"); + info!("listing w2l inference results for user `{}`; all", scope_creator_username); sqlx::query_as!( RawW2lInferenceRecordForList, r#" diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index fcf046dd2f..98e2680b34 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -81,19 +81,21 @@ pub async fn infer_w2l_handler( request: web::Json, server_state: web::Data>) -> Result { - let maybe_user_session = server_state + let maybe_session = server_state .session_checker - .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .maybe_get_session(&http_request, &server_state.mysql_pool) .await .map_err(|e| { warn!("Session checker error: {:?}", e); InferW2lError::ServerError })?; - let mut maybe_user_token : Option = maybe_user_session + let mut maybe_user_token : Option = maybe_session .as_ref() .map(|user_session| user_session.user_token.to_string()); + info!("Enqueue infer w2l by user token: {:?}", maybe_user_token); + let w2l_template_token = match &request.w2l_template_token { None => { // TODO: Allow image uploads. diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 1d841f3c05..e86a323ed7 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -126,6 +126,23 @@ pub async fn enqueue_infer_w2l_with_uploads( return Err(HttpResponse::TooManyRequests().body("you already have pending requests").into()) }*/ + // ==================== READ SESSION ==================== // + + let maybe_session = server_state + .session_checker + .maybe_get_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + InferW2lWithUploadError::ServerError + })?; + + let mut maybe_user_token : Option = maybe_session + .as_ref() + .map(|user_session| user_session.user_token.to_string()); + + info!("Enqueue infer w2l by user token: {:?}", maybe_user_token); + // ==================== READ MULTIPART REQUEST ==================== // info!("Reading multipart request..."); @@ -215,22 +232,6 @@ pub async fn enqueue_infer_w2l_with_uploads( return Err(InferW2lWithUploadError::BadInput("Template does not exist".to_string())); } - // ==================== READ SESSION ==================== // - - let maybe_user_session = server_state - .session_checker - .maybe_get_user_session(&http_request, &server_state.mysql_pool) - .await - .map_err(|e| { - warn!("Session checker error: {:?}", e); - InferW2lWithUploadError::ServerError - })?; - - let mut maybe_user_token : Option = maybe_user_session - .as_ref() - .map(|user_session| user_session.user_token.to_string()); - - // ==================== ANALYZE AND UPLOAD AUDIO FILE ==================== // let mut audio_type = "application/octet-stream".to_string(); diff --git a/src/main.rs b/src/main.rs index d0e5e35abd..1cf04c6fe1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -53,12 +53,6 @@ use crate::buckets::bucket_client::BucketClient; use crate::http_server::endpoints::users::list_user_w2l_templates::list_user_w2l_templates_handler; use crate::http_server::endpoints::users::list_user_w2l_inference_results::list_user_w2l_inference_results_handler; -/// -/// TODO TODO TODO TODO -/// TODO TODO TODO TODO -/// Make a central class that configures which bucket root paths things live in. -/// - const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; // NB: sqlx::query is spammy and logs all queries as "info"-level From b6c54f23fa3924105cc788e397752aa748e5eccc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 20:12:56 -0400 Subject: [PATCH 0198/1042] fix username bug --- src/common_queries/list_w2l_inference_results.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/common_queries/list_w2l_inference_results.rs index bbcd2fc8e6..3961525b0f 100644 --- a/src/common_queries/list_w2l_inference_results.rs +++ b/src/common_queries/list_w2l_inference_results.rs @@ -100,9 +100,9 @@ pub async fn list_w2l_inference_results( template_type: ir.template_type.clone(), - maybe_creator_user_token: None, - maybe_creator_username: None, - maybe_creator_display_name: None, + maybe_creator_user_token: ir.maybe_creator_user_token.clone(), + maybe_creator_username: ir.maybe_creator_username.clone(), + maybe_creator_display_name: ir.maybe_creator_display_name.clone(), //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, @@ -205,7 +205,7 @@ async fn list_w2l_inference_results_creator_scoped( // TODO: There has to be a better way. // Sqlx doesn't like anything except string literals. let maybe_results = if require_mod_approved_templates { - info!("listing w2l inference results for user; mod-approved only"); + info!("listing w2l inference results for user `{}`; mod-approved only", scope_creator_username); sqlx::query_as!( RawW2lInferenceRecordForList, r#" From 510c8ed09bcbb6a7f1908b8f78ea852aeb0b7d1f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 20:29:17 -0400 Subject: [PATCH 0199/1042] add a seed script that populates a lot more data --- debug/curl/seed.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100755 debug/curl/seed.py diff --git a/debug/curl/seed.py b/debug/curl/seed.py new file mode 100755 index 0000000000..3e65c2c017 --- /dev/null +++ b/debug/curl/seed.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +import requests +import re +import uuid + +def print_response(response): + print('\tStatus: {}'.format(response.status_code)) + for k, v in response.headers.items(): + print('\t header - {}: {}'.format(k, v)) + print('\tbody: {}', response.json()) + +# ==================== CREATE ACCOUNTS ==================== + +def create_user_account(username, password, email): + print('Create account: {}'.format(username)) + signup_url = 'http://localhost:12345/create_account' + payload = { + 'username': username, + 'password': password, + 'password_confirmation': password, + 'email_address': email, + } + r = requests.post(signup_url, json=payload) + print_response(r) + +create_user_account('brandon', 'testing', 'brandon@storyteller.company') +create_user_account('echelon', 'testing', 'echelon@gmail.com') +create_user_account('nanashi', 'nanashi', 'somebody@somewhere.com') + +# ==================== LOGIN ==================== + +def login(username, password): + print('Login: {}'.format(username)) + login_url = 'http://localhost:12345/login' + payload = { + 'username_or_email': username, + 'password': password, + } + r = requests.post(login_url, json=payload) + print_response(r) + raw_set_cookie_header = r.headers['set-cookie'] + m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) + session_cookie = m.group(1) + return session_cookie + +session_echelon = login('echelon', 'testing') +session_nanashi = login('nanashi', 'nanashi') + +# ==================== UPLOAD DATA ==================== + +def upload_w2l_template(content_url, title, session_cookie): + print('Upload W2L template') + upload_url = 'http://localhost:12345/w2l/upload' + cookies = { 'session': session_cookie } + payload = { + 'idempotency_token': str(uuid.uuid4()), + 'title': title, + 'download_url': content_url, + } + r = requests.post(upload_url, cookies=cookies, json=payload) + print_response(r) + + +upload_w2l_template('https://i.imgur.com/W1EYiPU.jpg', 'foo', session_echelon) +upload_w2l_template('https://i.imgur.com/NneE7Eq.jpg', 'bar', session_echelon) + +upload_w2l_template('https://drive.google.com/file/d/1yoBSb6nwFUZVI4CD-nM2BSofGdBjC8Nn/view?usp=sharing', 'baz', session_nanashi) + + From 910b3db1eb53647b94c53ea3a701be861bfaeb95 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 20:46:08 -0400 Subject: [PATCH 0200/1042] update seed tool --- debug/curl/seed.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/debug/curl/seed.py b/debug/curl/seed.py index 3e65c2c017..7cd89469ee 100755 --- a/debug/curl/seed.py +++ b/debug/curl/seed.py @@ -27,6 +27,8 @@ def create_user_account(username, password, email): create_user_account('brandon', 'testing', 'brandon@storyteller.company') create_user_account('echelon', 'testing', 'echelon@gmail.com') create_user_account('nanashi', 'nanashi', 'somebody@somewhere.com') +create_user_account('ryu', 'testing', 'ryu@storyteller.company') +create_user_account('raptor9', 'testing', 'raptor9@storyteller.company') # ==================== LOGIN ==================== @@ -46,6 +48,7 @@ def login(username, password): session_echelon = login('echelon', 'testing') session_nanashi = login('nanashi', 'nanashi') +session_brandon = login('brandon', 'testing') # ==================== UPLOAD DATA ==================== @@ -62,9 +65,12 @@ def upload_w2l_template(content_url, title, session_cookie): print_response(r) -upload_w2l_template('https://i.imgur.com/W1EYiPU.jpg', 'foo', session_echelon) -upload_w2l_template('https://i.imgur.com/NneE7Eq.jpg', 'bar', session_echelon) - -upload_w2l_template('https://drive.google.com/file/d/1yoBSb6nwFUZVI4CD-nM2BSofGdBjC8Nn/view?usp=sharing', 'baz', session_nanashi) +upload_w2l_template('https://drive.google.com/file/d/1yoBSb6nwFUZVI4CD-nM2BSofGdBjC8Nn/view?usp=sharing', 'Mr. Musk', session_nanashi) +upload_w2l_template('https://drive.google.com/file/d/1ysBAdRjlSmfWQPFjGlMJpv_zcGmVw7RC/view?usp=sharing', 'short bill clinton clip', session_brandon) +upload_w2l_template('https://i.imgur.com/NneE7Eq.jpg', 'itsa me mario!', session_echelon) +upload_w2l_template('https://i.imgur.com/W1EYiPU.jpg', 'Creepy Sonic does a gasp', session_echelon) +upload_w2l_template('https://i.imgur.com/lKaQ4Er.jpg', 'hasta la vista', session_brandon) +upload_w2l_template('https://i.imgur.com/uoGuTJo.jpeg', 'what do you mean vern?', session_brandon) +upload_w2l_template('https://i.imgur.com/v8k9yau.jpeg', 'Miyamoto-san', session_brandon) From 4f62a865a594e66d72654820411be068cf20d19d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 21:17:42 -0400 Subject: [PATCH 0201/1042] add title --- src/common_queries/list_w2l_inference_results.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/common_queries/list_w2l_inference_results.rs index 3961525b0f..8b21e82454 100644 --- a/src/common_queries/list_w2l_inference_results.rs +++ b/src/common_queries/list_w2l_inference_results.rs @@ -11,6 +11,7 @@ pub struct W2lInferenceRecordForList { pub maybe_tts_inference_result_token: Option, pub template_type: Option, + pub template_title: Option, pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, @@ -30,12 +31,13 @@ pub struct W2lInferenceRecordForList { } struct RawW2lInferenceRecordForList { - pub w2l_result_token: String, + pub w2l_result_token: String, // from field `w2l_results.token` pub maybe_w2l_template_token: Option, pub maybe_tts_inference_result_token: Option, pub template_type: Option, + pub template_title: Option, // from field `w2l_templates.title` pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, @@ -99,6 +101,7 @@ pub async fn list_w2l_inference_results( maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), template_type: ir.template_type.clone(), + template_title: ir.template_title.clone(), maybe_creator_user_token: ir.maybe_creator_user_token.clone(), maybe_creator_username: ir.maybe_creator_username.clone(), @@ -135,6 +138,7 @@ SELECT w2l_templates.token as maybe_w2l_template_token, w2l_templates.template_type, + w2l_templates.title as template_title, users.token as maybe_creator_user_token, users.username as maybe_creator_username, @@ -170,6 +174,7 @@ SELECT w2l_templates.token as maybe_w2l_template_token, w2l_templates.template_type, + w2l_templates.title as template_title, users.token as maybe_creator_user_token, users.username as maybe_creator_username, @@ -215,6 +220,7 @@ SELECT w2l_templates.token as maybe_w2l_template_token, w2l_templates.template_type, + w2l_templates.title as template_title, users.token as maybe_creator_user_token, users.username as maybe_creator_username, @@ -252,6 +258,7 @@ SELECT w2l_templates.token as maybe_w2l_template_token, w2l_templates.template_type, + w2l_templates.title as template_title, users.token as maybe_creator_user_token, users.username as maybe_creator_username, From ea566c465e7669f38a0f2589435e1c158695826a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 May 2021 23:44:18 -0400 Subject: [PATCH 0202/1042] new endpoint to show singular inference results --- .../endpoints/w2l/get_w2l_result.rs | 224 ++++++++++++++++++ .../endpoints/w2l/list_w2l_templates.rs | 36 --- src/http_server/endpoints/w2l/mod.rs | 1 + src/main.rs | 6 + 4 files changed, 231 insertions(+), 36 deletions(-) create mode 100644 src/http_server/endpoints/w2l/get_w2l_result.rs diff --git a/src/http_server/endpoints/w2l/get_w2l_result.rs b/src/http_server/endpoints/w2l/get_w2l_result.rs new file mode 100644 index 0000000000..7577a6dcd4 --- /dev/null +++ b/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -0,0 +1,224 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetW2lResultPathInfo { + token: String, +} + +#[derive(Serialize)] +pub struct W2lResultRecordForResponse { + pub w2l_result_token: String, + pub maybe_w2l_template_token: Option, + pub maybe_tts_inference_result_token: Option, + + pub template_type: Option, + pub template_title: Option, + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + + pub file_size_bytes: u32, + pub frame_width: u32, + pub frame_height: u32, + pub duration_millis: u32, + + //pub is_mod_hidden_from_public: bool, // converted + //pub template_is_mod_approved: bool, // converted + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct GetW2lResultSuccessResponse { + pub success: bool, + pub result: W2lResultRecordForResponse, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum GetW2lResultError { + ServerError, +} + +pub struct RawW2lResultRecord { + pub w2l_result_token: String, // from field `w2l_results.token` + + pub maybe_w2l_template_token: Option, + pub maybe_tts_inference_result_token: Option, + + pub template_type: Option, + pub template_title: Option, // from field `w2l_templates.title` + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + + pub file_size_bytes: i32, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + + //pub is_mod_hidden_from_public: i8, // needs convert + //pub template_is_mod_approved: i8, // needs convert + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl ResponseError for GetW2lResultError { + fn status_code(&self) -> StatusCode { + match *self { + GetW2lResultError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetW2lResultError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn get_w2l_inference_result_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_result = sqlx::query_as!( + RawW2lResultRecord, + r#" +SELECT + w2l_results.token as w2l_result_token, + w2l_results.maybe_tts_inference_result_token, + + w2l_templates.token as maybe_w2l_template_token, + w2l_templates.template_type, + w2l_templates.title as template_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + w2l_results.file_size_bytes, + w2l_results.frame_width, + w2l_results.frame_height, + w2l_results.duration_millis, + w2l_results.created_at, + w2l_results.updated_at + +FROM w2l_results +LEFT OUTER JOIN w2l_templates + ON w2l_results.maybe_w2l_template_token = w2l_templates.token +LEFT OUTER JOIN users + ON w2l_results.maybe_creator_user_token = users.token +WHERE + w2l_results.deleted_at IS NULL + AND w2l_results.token = ? + "#, + &path.token + ) + .fetch_one(&server_state.mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let ir : RawW2lResultRecord = match maybe_result { + Ok(result) => result, + Err(err) => { + match err { + RowNotFound => { + return Err(GetW2lResultError::ServerError); + }, + _ => { + warn!("w2l inference result query error: {:?}", err); + return Err(GetW2lResultError::ServerError); + } + } + } + }; + + let result_for_response = W2lResultRecordForResponse { + w2l_result_token: ir.w2l_result_token.clone(), + maybe_w2l_template_token: ir.maybe_w2l_template_token.clone(), + maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), + + template_type: ir.template_type.clone(), + template_title: ir.template_title.clone(), + + maybe_creator_user_token: ir.maybe_creator_user_token.clone(), + maybe_creator_username: ir.maybe_creator_username.clone(), + maybe_creator_display_name: ir.maybe_creator_display_name.clone(), + //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, + //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, + + file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, + frame_width: if ir.frame_width > 0 { ir.frame_width as u32 } else { 0 }, + frame_height: if ir.frame_height > 0 { ir.frame_height as u32 } else { 0 }, + duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, + + created_at: ir.created_at.clone(), + updated_at: ir.updated_at.clone(), + }; + + let response = GetW2lResultSuccessResponse { + success: true, + result: result_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetW2lResultError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/w2l/list_w2l_templates.rs b/src/http_server/endpoints/w2l/list_w2l_templates.rs index 15e57bc313..21e9b67cbd 100644 --- a/src/http_server/endpoints/w2l/list_w2l_templates.rs +++ b/src/http_server/endpoints/w2l/list_w2l_templates.rs @@ -26,24 +26,6 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use crate::common_queries::list_w2l_templates::{list_w2l_templates, W2lTemplateRecordForList}; -#[derive(Serialize)] -pub struct W2lTemplateRecordForResponse { - pub template_token: String, - pub template_type: String, - pub creator_user_token: String, - pub creator_username: String, - pub creator_display_name: String, - pub updatable_slug: String, - pub title: String, - pub frame_width: u32, - pub frame_height: u32, - pub duration_millis: u32, - pub maybe_image_object_name: Option, - pub maybe_video_object_name: Option, - pub created_at: DateTime, - pub updated_at: DateTime, -} - #[derive(Serialize)] pub struct ListW2lTemplatesSuccessResponse { pub success: bool, @@ -61,24 +43,6 @@ pub enum ListW2lTemplatesError { ServerError, } -#[derive(Serialize)] -pub struct W2lTemplateRecord { - pub template_token: String, - pub template_type: String, - pub creator_user_token: String, - pub creator_username: String, - pub creator_display_name: String, - pub updatable_slug: String, - pub title: String, - pub frame_width: i32, - pub frame_height: i32, - pub duration_millis: i32, - pub maybe_public_bucket_preview_image_object_name: Option, - pub maybe_public_bucket_preview_video_object_name: Option, - pub created_at: DateTime, - pub updated_at: DateTime, -} - impl ResponseError for ListW2lTemplatesError { fn status_code(&self) -> StatusCode { match *self { diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index eb1514f0b4..eded8523e8 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -1,5 +1,6 @@ pub mod enqueue_infer_w2l; pub mod enqueue_infer_w2l_with_uploads; pub mod enqueue_upload_w2l_template; +pub mod get_w2l_result; pub mod get_w2l_template; pub mod list_w2l_templates; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 1cf04c6fe1..dbbc954d98 100644 --- a/src/main.rs +++ b/src/main.rs @@ -52,6 +52,7 @@ use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_ use crate::buckets::bucket_client::BucketClient; use crate::http_server::endpoints::users::list_user_w2l_templates::list_user_w2l_templates_handler; use crate::http_server::endpoints::users::list_user_w2l_inference_results::list_user_w2l_inference_results_handler; +use crate::http_server::endpoints::w2l::get_w2l_result::get_w2l_inference_result_handler; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -277,6 +278,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_w2l_template_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/result/{token}") + .route(web::get().to(get_w2l_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service( web::resource("/profile/{username}") From e84070f5bbb661e7de39ab18f4322e464109ca6d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 00:35:24 -0400 Subject: [PATCH 0203/1042] add video bucket path --- src/http_server/endpoints/w2l/get_w2l_result.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/http_server/endpoints/w2l/get_w2l_result.rs b/src/http_server/endpoints/w2l/get_w2l_result.rs index 7577a6dcd4..e58290127f 100644 --- a/src/http_server/endpoints/w2l/get_w2l_result.rs +++ b/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -38,6 +38,8 @@ pub struct W2lResultRecordForResponse { pub maybe_w2l_template_token: Option, pub maybe_tts_inference_result_token: Option, + pub public_bucket_video_path: String, + pub template_type: Option, pub template_title: Option, @@ -81,6 +83,8 @@ pub struct RawW2lResultRecord { pub maybe_w2l_template_token: Option, pub maybe_tts_inference_result_token: Option, + pub public_bucket_video_path: String, + pub template_type: Option, pub template_title: Option, // from field `w2l_templates.title` @@ -143,6 +147,8 @@ SELECT w2l_results.token as w2l_result_token, w2l_results.maybe_tts_inference_result_token, + w2l_results.public_bucket_video_path, + w2l_templates.token as maybe_w2l_template_token, w2l_templates.template_type, w2l_templates.title as template_title, @@ -192,6 +198,8 @@ WHERE maybe_w2l_template_token: ir.maybe_w2l_template_token.clone(), maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), + public_bucket_video_path: ir.public_bucket_video_path.clone(), + template_type: ir.template_type.clone(), template_title: ir.template_title.clone(), From b3a0b49bc1cef8990f65b1a4afcfe9f2a566a8ab Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 00:42:22 -0400 Subject: [PATCH 0204/1042] report gravatar and template creator for inf result --- .../endpoints/w2l/get_w2l_result.rs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/http_server/endpoints/w2l/get_w2l_result.rs b/src/http_server/endpoints/w2l/get_w2l_result.rs index e58290127f..02dc49528c 100644 --- a/src/http_server/endpoints/w2l/get_w2l_result.rs +++ b/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -46,6 +46,12 @@ pub struct W2lResultRecordForResponse { pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub maybe_template_creator_user_token: Option, + pub maybe_template_creator_username: Option, + pub maybe_template_creator_display_name: Option, + pub maybe_template_creator_gravatar_hash: Option, pub file_size_bytes: u32, pub frame_width: u32, @@ -91,6 +97,12 @@ pub struct RawW2lResultRecord { pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub maybe_template_creator_user_token: Option, + pub maybe_template_creator_username: Option, + pub maybe_template_creator_display_name: Option, + pub maybe_template_creator_gravatar_hash: Option, pub file_size_bytes: i32, pub frame_width: i32, @@ -156,6 +168,12 @@ SELECT users.token as maybe_creator_user_token, users.username as maybe_creator_username, users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_gravatar_hash, + + template_users.token as maybe_template_creator_user_token, + template_users.username as maybe_template_creator_username, + template_users.display_name as maybe_template_creator_display_name, + template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash, w2l_results.file_size_bytes, w2l_results.frame_width, @@ -169,6 +187,8 @@ LEFT OUTER JOIN w2l_templates ON w2l_results.maybe_w2l_template_token = w2l_templates.token LEFT OUTER JOIN users ON w2l_results.maybe_creator_user_token = users.token +LEFT OUTER JOIN users as template_users + ON w2l_templates.creator_user_token = template_users.token WHERE w2l_results.deleted_at IS NULL AND w2l_results.token = ? @@ -206,6 +226,13 @@ WHERE maybe_creator_user_token: ir.maybe_creator_user_token.clone(), maybe_creator_username: ir.maybe_creator_username.clone(), maybe_creator_display_name: ir.maybe_creator_display_name.clone(), + maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash.clone(), + + maybe_template_creator_user_token: ir.maybe_template_creator_user_token.clone(), + maybe_template_creator_username: ir.maybe_template_creator_username.clone(), + maybe_template_creator_display_name: ir.maybe_template_creator_display_name.clone(), + maybe_template_creator_gravatar_hash: ir.maybe_template_creator_gravatar_hash.clone(), + //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, From fca64a6cca53f28ceaaf6e45a7bc5807b94014ba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 02:12:27 -0400 Subject: [PATCH 0205/1042] firehose publishing --- debug/mysql/last_firehose_events.sh | 7 ++ debug/mysql/last_w2l_inference_results.sh | 45 +++++++++++ migrations/2021-05-15-071534_users/up.sql | 2 +- .../2021-05-30-010200_firehose/down.sql | 4 + migrations/2021-05-30-010200_firehose/up.sql | 51 ++++++++++++ src/common_queries/firehose_publisher.rs | 81 +++++++++++++++++++ src/common_queries/mod.rs | 1 + .../endpoints/users/create_account.rs | 7 ++ src/main.rs | 6 ++ src/server_state.rs | 3 + 10 files changed, 206 insertions(+), 1 deletion(-) create mode 100755 debug/mysql/last_firehose_events.sh create mode 100755 debug/mysql/last_w2l_inference_results.sh create mode 100644 migrations/2021-05-30-010200_firehose/down.sql create mode 100644 migrations/2021-05-30-010200_firehose/up.sql create mode 100644 src/common_queries/firehose_publisher.rs diff --git a/debug/mysql/last_firehose_events.sh b/debug/mysql/last_firehose_events.sh new file mode 100755 index 0000000000..3e4077969b --- /dev/null +++ b/debug/mysql/last_firehose_events.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u root \ + -proot \ + -h localhost \ + -D storyteller \ + -e "select * from firehose_entries\G" diff --git a/debug/mysql/last_w2l_inference_results.sh b/debug/mysql/last_w2l_inference_results.sh new file mode 100755 index 0000000000..ebc1e305b8 --- /dev/null +++ b/debug/mysql/last_w2l_inference_results.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV + +mysql -u root \ + -proot \ + -h localhost \ + -D storyteller \ + -e "select * from w2l_results\G" + + +echo '===============================================' + +read -r -d '' query << EOF + + SELECT + w2l_results.token as w2l_result_token, + w2l_results.maybe_tts_inference_result_token, + w2l_templates.token as maybe_w2l_template_token, + w2l_templates.template_type, + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + w2l_results.file_size_bytes, + w2l_results.frame_width, + w2l_results.frame_height, + w2l_results.duration_millis, + w2l_results.created_at, + w2l_results.updated_at + FROM + w2l_results + LEFT OUTER JOIN w2l_templates ON w2l_results.maybe_w2l_template_token = w2l_templates.token + LEFT OUTER JOIN users ON w2l_results.maybe_creator_user_token = users.token + WHERE + w2l_results.deleted_at IS NULL + AND users.username = 'echelon'\G +EOF + +echo $query + +mysql -u root \ + -proot \ + -h localhost \ + -D storyteller \ + -e "${query}" + diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index ecbcaa6e81..1eae07bce6 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -5,7 +5,7 @@ CREATE TABLE users ( -- Not used for anything except replication. id BIGINT(20) NOT NULL AUTO_INCREMENT, - -- Effective "primary key" + -- Visible "primary key" token VARCHAR(32) NOT NULL, -- Username is a lookup key; display_name allows the user to add custom case. diff --git a/migrations/2021-05-30-010200_firehose/down.sql b/migrations/2021-05-30-010200_firehose/down.sql new file mode 100644 index 0000000000..6bc740efcd --- /dev/null +++ b/migrations/2021-05-30-010200_firehose/down.sql @@ -0,0 +1,4 @@ +# noinspection SqlResolveForFile +# noinspection SqlNoDataSourceInspectionForFile + +drop table firehose_entries; diff --git a/migrations/2021-05-30-010200_firehose/up.sql b/migrations/2021-05-30-010200_firehose/up.sql new file mode 100644 index 0000000000..374c8a34ee --- /dev/null +++ b/migrations/2021-05-30-010200_firehose/up.sql @@ -0,0 +1,51 @@ +# noinspection SqlResolveForFile +# noinspection SqlNoDataSourceInspectionForFile + +CREATE TABLE firehose_entries ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Visible "primary key" + token VARCHAR(32) NOT NULL, + + -- The type of the event + event_type VARCHAR(32) NOT NULL, + + -- The target user + maybe_target_user_token VARCHAR(32) DEFAULT NULL, + + -- The target "entity", which varies by event type. + maybe_target_entity_token VARCHAR(32) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- Settings + -- DO NOT REORDER. + -- event_type ENUM( + -- 'not-set', + -- 'user_sign_up', + -- 'tts_model_upload_started', + -- 'tts_model_upload_completed', + -- 'tts_inference_started', + -- 'tts_inference_completed', + -- 'w2l_template_upload_started', + -- 'w2l_template_upload_completed', + -- 'w2l_inference_started', + -- 'w2l_inference_completed', + -- 'twitter_mention', + -- 'twitter_retweet', + -- 'discord_join', + -- 'discord_message', + -- 'twitch_subscribe', + -- 'twitch_follow' + -- ) NOT NULL DEFAULT 'not-set', + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY fk_maybe_target_user_token (maybe_target_user_token), + KEY fk_maybe_target_entity_token (maybe_target_entity_token), + KEY index_event_type (event_type) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/src/common_queries/firehose_publisher.rs b/src/common_queries/firehose_publisher.rs new file mode 100644 index 0000000000..3fecd0a7da --- /dev/null +++ b/src/common_queries/firehose_publisher.rs @@ -0,0 +1,81 @@ +use anyhow::anyhow; +use crate::util::anyhow_result::AnyhowResult; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use log::{warn,info}; +use sqlx::{MySqlPool}; +use std::sync::Arc; +use sqlx::error::Error::Database; + +enum FirehoseEvent { + UserSignUp, +} + +impl FirehoseEvent { + pub fn to_db_value(&self) -> &'static str { + match self { + FirehoseEvent::UserSignUp => "user_sign_up", + } + } +} + +#[derive(Clone)] +pub struct FirehosePublisher { + pub mysql_pool: MySqlPool, +} + +impl FirehosePublisher { + + pub async fn publish_user_sign_up(&self, user_token: &str) -> AnyhowResult<()> { + let token = random_prefix_crockford_token("EV", 32)?; + let event_type = FirehoseEvent::UserSignUp; + + let query_result = sqlx::query!( + r#" +INSERT INTO firehose_entries +SET + token = ?, + event_type = ?, + maybe_target_user_token = ?, + maybe_target_entity_token = ? + "#, + token, + event_type.to_db_value(), + user_token, + user_token, + ) + .execute(&self.mysql_pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + warn!("Insert record DB error: {:?}", err); + + // NB: SQLSTATE[23000]: Integrity constraint violation + // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access) + match err { + Database(err) => { + let maybe_code = err.code().map(|c| c.into_owned()); + /*match maybe_code.as_deref() { + Some("23000") => { + if err.message().contains("username") { + return Err(UsernameTaken); + } else if err.message().contains("email_address") { + return Err(EmailTaken); + } + } + _ => {}, + }*/ + }, + _ => {}, + } + return Err(anyhow!("Error inserting record")); + } + }; + + + Ok(()) + } +} diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index 0d32c80f74..bdefa2998f 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -1,4 +1,5 @@ pub mod badges; +pub mod firehose_publisher; pub mod list_w2l_inference_results; pub mod list_w2l_templates; pub mod sessions; diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index 3209066cb4..00d96e7a44 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -215,6 +215,13 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) info!("new user session created"); + server_state.firehose_publisher.publish_user_sign_up(&user_token) + .await + .map_err(|e| { + warn!("error publishing event: {:?}", e); + CreateAccountError::ServerError + })?; + let session_cookie = match server_state.cookie_manager.create_cookie(&session_token) { Ok(cookie) => cookie, Err(_) => return Err(ServerError), diff --git a/src/main.rs b/src/main.rs index dbbc954d98..f743dd510e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -53,6 +53,7 @@ use crate::buckets::bucket_client::BucketClient; use crate::http_server::endpoints::users::list_user_w2l_templates::list_user_w2l_templates_handler; use crate::http_server::endpoints::users::list_user_w2l_inference_results::list_user_w2l_inference_results_handler; use crate::http_server::endpoints::w2l::get_w2l_result::get_w2l_inference_result_handler; +use crate::common_queries::firehose_publisher::FirehosePublisher; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -104,6 +105,10 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; + let firehose_publisher = FirehosePublisher { + mysql_pool: pool.clone(), // NB: Pool is clone/sync/send-safe + }; + info!("Reading env vars and setting up utils..."); let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); @@ -153,6 +158,7 @@ async fn main() -> AnyhowResult<()> { }, hostname: server_hostname, mysql_pool: pool, + firehose_publisher, cookie_manager, session_checker, private_bucket_client, diff --git a/src/server_state.rs b/src/server_state.rs index 01824d453e..97d0be1178 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -2,6 +2,7 @@ use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; use sqlx::MySqlPool; use crate::buckets::bucket_client::BucketClient; +use crate::common_queries::firehose_publisher::FirehosePublisher; /// State that is injected into every endpoint. #[derive(Clone)] @@ -17,6 +18,8 @@ pub struct ServerState { pub session_checker: SessionChecker, + pub firehose_publisher: FirehosePublisher, + pub private_bucket_client: BucketClient, pub public_bucket_client: BucketClient, From f601ec9aa1944a762f3cea8f2f2a1061694c8dc0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 02:41:17 -0400 Subject: [PATCH 0206/1042] events for several things --- src/common_queries/firehose_publisher.rs | 86 ++++++++++++++++++- .../w2l/enqueue_upload_w2l_template.rs | 8 ++ src/job_queries/w2l_download_job_queries.rs | 4 +- src/w2l_download_job.rs | 27 ++++-- 4 files changed, 115 insertions(+), 10 deletions(-) diff --git a/src/common_queries/firehose_publisher.rs b/src/common_queries/firehose_publisher.rs index 3fecd0a7da..e266218275 100644 --- a/src/common_queries/firehose_publisher.rs +++ b/src/common_queries/firehose_publisher.rs @@ -5,15 +5,48 @@ use log::{warn,info}; use sqlx::{MySqlPool}; use std::sync::Arc; use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDone; +#[derive(Debug, Clone, Copy)] enum FirehoseEvent { UserSignUp, + + TtsModelUploadStarted, + TtsModelUploadCompleted, + TtsInferenceStarted, + TtsInferenceCompleted, + + W2lTemplateUploadStarted, + W2lTemplateUploadCompleted, + W2lInferenceStarted, + W2lInferenceCompleted, + + TwitterMention, + TwitterRetweet, + DiscordJoin, + DiscordMessage, + TwitchSubscribe, + TwitchFollow, } impl FirehoseEvent { pub fn to_db_value(&self) -> &'static str { match self { FirehoseEvent::UserSignUp => "user_sign_up", + FirehoseEvent::TtsModelUploadStarted => "tts_model_upload_started", + FirehoseEvent::TtsModelUploadCompleted => "tts_model_upload_completed", + FirehoseEvent::TtsInferenceStarted => "tts_inference_started", + FirehoseEvent::TtsInferenceCompleted => "tts_inference_completed", + FirehoseEvent::W2lTemplateUploadStarted => "w2l_template_upload_started", + FirehoseEvent::W2lTemplateUploadCompleted => "w2l_template_upload_completed", + FirehoseEvent::W2lInferenceStarted => "w2l_inference_started", + FirehoseEvent::W2lInferenceCompleted => "w2l_inference_completed", + FirehoseEvent::TwitterMention => "twitter_mention", + FirehoseEvent::TwitterRetweet => "twitter_retweet", + FirehoseEvent::DiscordJoin => "discord_join", + FirehoseEvent::DiscordMessage => "discord_message", + FirehoseEvent::TwitchSubscribe => "twitch_subscribe", + FirehoseEvent::TwitchFollow => "twitch_follow", } } } @@ -29,6 +62,49 @@ impl FirehosePublisher { let token = random_prefix_crockford_token("EV", 32)?; let event_type = FirehoseEvent::UserSignUp; + let _record_id = self.insert( + event_type, + Some(user_token), + Some(user_token) + ).await?; + + Ok(()) + } + + pub async fn publish_w2l_template_upload_enqueue(&self, user_token: &str, job_token: &str) -> AnyhowResult<()> { + let token = random_prefix_crockford_token("EV", 32)?; + let event_type = FirehoseEvent::W2lTemplateUploadStarted; + + let _record_id = self.insert( + event_type, + Some(user_token), + Some(job_token) + ).await?; + + Ok(()) + } + + pub async fn publish_w2l_template_upload_finished(&self, user_token: &str, template_token: &str) -> AnyhowResult<()> { + let token = random_prefix_crockford_token("EV", 32)?; + let event_type = FirehoseEvent::W2lTemplateUploadCompleted; + + let _record_id = self.insert( + event_type, + Some(user_token), + Some(template_token) + ).await?; + + Ok(()) + } + + async fn insert( + &self, + event_type: FirehoseEvent, + user_token: Option<&str>, + entity_token: Option<&str> + ) -> AnyhowResult { + let token = random_prefix_crockford_token("EV", 32)?; + let query_result = sqlx::query!( r#" INSERT INTO firehose_entries @@ -41,11 +117,16 @@ SET token, event_type.to_db_value(), user_token, - user_token, + entity_token, ) .execute(&self.mysql_pool) .await; + let record_id = Self::handle_results(query_result)?; + Ok(record_id) + } + + fn handle_results(query_result: Result) -> AnyhowResult { let record_id = match query_result { Ok(res) => { res.last_insert_id() @@ -75,7 +156,6 @@ SET } }; - - Ok(()) + Ok(record_id) } } diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index 8682f86673..e9b505d266 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -200,6 +200,14 @@ SET info!("new w2l template upload job id: {}", record_id); + server_state.firehose_publisher.publish_w2l_template_upload_enqueue(&user_session.user_token, &job_token) + .await + .map_err(|e| { + warn!("error publishing event: {:?}", e); + UploadW2lTemplateError::ServerError + })?; + + let response = UploadW2lTemplateSuccessResponse { success: true, job_token: job_token.to_string(), diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index a8ea1ee508..d3c782ea75 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -126,7 +126,7 @@ pub async fn insert_w2l_template(pool: &MySqlPool, frame_count: u64, fps: f32, duration_millis: u64) - -> AnyhowResult + -> AnyhowResult<(u64, String)> { let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?; let updatable_slug = model_token.clone(); @@ -190,5 +190,5 @@ SET } }; - Ok(record_id) + Ok((record_id, model_token.clone())) } diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index d837779dd4..b731c82efd 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -10,6 +10,7 @@ pub mod buckets; pub mod job_queries; pub mod script_execution; +pub mod common_queries; pub mod util; use anyhow::anyhow; @@ -17,12 +18,17 @@ use chrono::Utc; use crate::buckets::bucket_client::BucketClient; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::buckets::file_hashing::get_file_hash; +use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; use crate::job_queries::w2l_download_job_queries::insert_w2l_template; use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_done; use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_failure; use crate::job_queries::w2l_download_job_queries::query_w2l_template_upload_job_records; +use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; +use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; +use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; +use crate::script_execution::wav2lip_process_upload_command::Wav2LipPreprocessClient; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; @@ -38,10 +44,6 @@ use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; use tempdir::TempDir; -use crate::script_execution::wav2lip_process_upload_command::Wav2LipPreprocessClient; -use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; -use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; -use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -72,6 +74,8 @@ struct Downloader { pub private_bucket_client: BucketClient, pub public_bucket_client: BucketClient, + pub firehose_publisher: FirehosePublisher, + pub google_drive_downloader: GoogleDriveDownloadCommand, pub w2l_processor: Wav2LipPreprocessClient, pub ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand, @@ -165,6 +169,10 @@ async fn main() -> AnyhowResult<()> { .max_connections(5) .connect(&db_connection_string) .await?; + + let firehose_publisher = FirehosePublisher { + mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe + }; let downloader = Downloader { download_temp_directory: temp_directory, @@ -173,6 +181,7 @@ async fn main() -> AnyhowResult<()> { private_bucket_client, download_script, google_drive_downloader, + firehose_publisher, ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, @@ -454,7 +463,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) let template_type = if file_metadata.is_video { "video" } else { "image" }; info!("Saving model record..."); - let id = insert_w2l_template( + let (id, model_token) = insert_w2l_template( &downloader.mysql_pool, template_type, job, @@ -475,6 +484,14 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}", job.id, id); + downloader.firehose_publisher.publish_w2l_template_upload_finished(&job.creator_user_token, &model_token) + .await + .map_err(|e| { + warn!("error publishing event: {:?}", e); + anyhow!("error publishing event") + })?; + + mark_w2l_template_upload_job_done(&downloader.mysql_pool, job, true).await?; Ok(()) From 6ab5bac9dc8b53a039dd24613c2fe90319e2354b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 03:02:47 -0400 Subject: [PATCH 0207/1042] add events endpoint --- .../endpoints/events/list_events.rs | 140 ++++++++++++++++++ src/http_server/endpoints/events/mod.rs | 1 + src/http_server/endpoints/mod.rs | 1 + src/main.rs | 6 + 4 files changed, 148 insertions(+) create mode 100644 src/http_server/endpoints/events/list_events.rs create mode 100644 src/http_server/endpoints/events/mod.rs diff --git a/src/http_server/endpoints/events/list_events.rs b/src/http_server/endpoints/events/list_events.rs new file mode 100644 index 0000000000..2a7f9b9b44 --- /dev/null +++ b/src/http_server/endpoints/events/list_events.rs @@ -0,0 +1,140 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use chrono::{DateTime, Utc}; +use crate::common_queries::list_w2l_templates::{list_w2l_templates, W2lTemplateRecordForList}; + +#[derive(Serialize)] +pub struct EventRecord { + pub event_token: String, + pub event_type: String, + pub maybe_target_user_token: Option, + pub maybe_target_username: Option, + pub maybe_target_display_name: Option, + pub maybe_target_entity_token: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct ListEventsSuccessResponse { + pub success: bool, + pub events: Vec, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum ListEventsError { + ServerError, +} + +impl ResponseError for ListEventsError { + fn status_code(&self) -> StatusCode { + match *self { + ListEventsError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListEventsError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn list_events_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_events = sqlx::query_as!( + EventRecord, + r#" +SELECT + events.token as event_token, + events.event_type, + events.maybe_target_user_token, + users.username as maybe_target_username, + users.display_name as maybe_target_display_name, + events.maybe_target_entity_token, + events.created_at, + events.updated_at +FROM firehose_entries as events +LEFT OUTER JOIN users +ON events.maybe_target_user_token = users.token +ORDER BY events.id DESC +LIMIT 25 + "#, + ) + .fetch_all(&server_state.mysql_pool) + .await; + + let events : Vec = match maybe_events { + Ok(events) => events, + Err(err) => { + match err { + RowNotFound => { + return Err(ListEventsError::ServerError); + }, + _ => { + warn!("tts models list query error: {:?}", err); + return Err(ListEventsError::ServerError); + } + } + } + }; + + let response = ListEventsSuccessResponse { + success: true, + events, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListEventsError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/events/mod.rs b/src/http_server/endpoints/events/mod.rs new file mode 100644 index 0000000000..409b5152d2 --- /dev/null +++ b/src/http_server/endpoints/events/mod.rs @@ -0,0 +1 @@ +pub mod list_events; \ No newline at end of file diff --git a/src/http_server/endpoints/mod.rs b/src/http_server/endpoints/mod.rs index 60bd52fecb..e4b320f98c 100644 --- a/src/http_server/endpoints/mod.rs +++ b/src/http_server/endpoints/mod.rs @@ -1,4 +1,5 @@ pub mod default_route_404; +pub mod events; pub mod misc; pub mod root_index; pub mod tts; diff --git a/src/main.rs b/src/main.rs index f743dd510e..1e6c16ecea 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,6 +27,7 @@ use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; use crate::http_server::endpoints::default_route_404::default_route_404; +use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::misc::enable_alpha::enable_alpha; use crate::http_server::endpoints::root_index::get_root_index; use crate::http_server::endpoints::tts::enqueue_infer_tts::infer_tts_handler; @@ -308,6 +309,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service( + web::resource("/events") + .route(web::get().to(list_events_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(get_root_index) .service(enable_alpha) .default_service( web::route().to(default_route_404)) From f27908d4a200d99e8861ab86fd5b3a1ae895f9a9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 03:50:58 -0400 Subject: [PATCH 0208/1042] add gravatar hash --- src/http_server/endpoints/events/list_events.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/http_server/endpoints/events/list_events.rs b/src/http_server/endpoints/events/list_events.rs index 2a7f9b9b44..a1a48fb112 100644 --- a/src/http_server/endpoints/events/list_events.rs +++ b/src/http_server/endpoints/events/list_events.rs @@ -33,6 +33,7 @@ pub struct EventRecord { pub maybe_target_user_token: Option, pub maybe_target_username: Option, pub maybe_target_display_name: Option, + pub maybe_target_user_gravatar_hash: Option, pub maybe_target_entity_token: Option, pub created_at: DateTime, pub updated_at: DateTime, @@ -98,6 +99,7 @@ SELECT events.maybe_target_user_token, users.username as maybe_target_username, users.display_name as maybe_target_display_name, + users.email_gravatar_hash as maybe_target_user_gravatar_hash, events.maybe_target_entity_token, events.created_at, events.updated_at From fd2b01334b0003d0757025ea22ffb4503128f3be Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 04:12:32 -0400 Subject: [PATCH 0209/1042] more firehose event types --- src/common_queries/firehose_publisher.rs | 38 +++++++++++-------- .../endpoints/events/list_events.rs | 4 +- .../w2l/enqueue_infer_w2l_with_uploads.rs | 8 ++++ .../w2l/enqueue_upload_w2l_template.rs | 3 +- src/job_queries/w2l_inference_job_queries.rs | 4 +- src/w2l_inference_job.rs | 22 ++++++++++- 6 files changed, 56 insertions(+), 23 deletions(-) diff --git a/src/common_queries/firehose_publisher.rs b/src/common_queries/firehose_publisher.rs index e266218275..235f2462d6 100644 --- a/src/common_queries/firehose_publisher.rs +++ b/src/common_queries/firehose_publisher.rs @@ -59,44 +59,52 @@ pub struct FirehosePublisher { impl FirehosePublisher { pub async fn publish_user_sign_up(&self, user_token: &str) -> AnyhowResult<()> { - let token = random_prefix_crockford_token("EV", 32)?; - let event_type = FirehoseEvent::UserSignUp; - let _record_id = self.insert( - event_type, + FirehoseEvent::UserSignUp, Some(user_token), Some(user_token) ).await?; - Ok(()) } - pub async fn publish_w2l_template_upload_enqueue(&self, user_token: &str, job_token: &str) -> AnyhowResult<()> { - let token = random_prefix_crockford_token("EV", 32)?; - let event_type = FirehoseEvent::W2lTemplateUploadStarted; - + pub async fn enqueue_w2l_template_upload(&self, user_token: &str, job_token: &str) -> AnyhowResult<()> { let _record_id = self.insert( - event_type, + FirehoseEvent::W2lTemplateUploadStarted, Some(user_token), Some(job_token) ).await?; + Ok(()) + } + pub async fn enqueue_w2l_inference(&self, maybe_user_token: Option<&str>, job_token: &str) -> AnyhowResult<()> { + let _record_id = self.insert( + FirehoseEvent::W2lInferenceStarted, + maybe_user_token, + Some(job_token) + ).await?; Ok(()) } pub async fn publish_w2l_template_upload_finished(&self, user_token: &str, template_token: &str) -> AnyhowResult<()> { - let token = random_prefix_crockford_token("EV", 32)?; - let event_type = FirehoseEvent::W2lTemplateUploadCompleted; - let _record_id = self.insert( - event_type, + FirehoseEvent::W2lTemplateUploadCompleted, Some(user_token), Some(template_token) ).await?; - Ok(()) } + pub async fn w2l_inference_finished(&self, maybe_user_token: Option<&str>, job_token: &str) -> AnyhowResult<()> { + let _record_id = self.insert( + FirehoseEvent::W2lInferenceCompleted, + maybe_user_token, + Some(job_token) + ).await?; + Ok(()) + } + + // ======================================================================= + async fn insert( &self, event_type: FirehoseEvent, diff --git a/src/http_server/endpoints/events/list_events.rs b/src/http_server/endpoints/events/list_events.rs index a1a48fb112..8055feb660 100644 --- a/src/http_server/endpoints/events/list_events.rs +++ b/src/http_server/endpoints/events/list_events.rs @@ -5,7 +5,9 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::common_queries::list_w2l_templates::{list_w2l_templates, W2lTemplateRecordForList}; use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; @@ -23,8 +25,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use chrono::{DateTime, Utc}; -use crate::common_queries::list_w2l_templates::{list_w2l_templates, W2lTemplateRecordForList}; #[derive(Serialize)] pub struct EventRecord { diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index e86a323ed7..e196ae6035 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -341,6 +341,14 @@ SET // .map_err(|err| HttpResponse::InternalServerError() // .body(format!("Redis Err: {:?}", err)))?; + server_state.firehose_publisher.enqueue_w2l_inference(maybe_user_token.as_deref(), &job_token) + .await + .map_err(|e| { + warn!("error publishing event: {:?}", e); + InferW2lWithUploadError::ServerError + })?; + + let response = InferW2lWithUploadSuccessResponse { success: true, job_token: job_token.to_string(), diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index e9b505d266..609badcf96 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -200,14 +200,13 @@ SET info!("new w2l template upload job id: {}", record_id); - server_state.firehose_publisher.publish_w2l_template_upload_enqueue(&user_session.user_token, &job_token) + server_state.firehose_publisher.enqueue_w2l_template_upload(&user_session.user_token, &job_token) .await .map_err(|e| { warn!("error publishing event: {:?}", e); UploadW2lTemplateError::ServerError })?; - let response = UploadW2lTemplateSuccessResponse { success: true, job_token: job_token.to_string(), diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index 20760dfc7e..e10a573686 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -163,7 +163,7 @@ pub async fn insert_w2l_result>( frame_width: u32, frame_height: u32, duration_millis: u64 -) -> AnyhowResult +) -> AnyhowResult<(u64, String)> { let inference_result_token = random_prefix_crockford_token("W2L_RES:", 32)?; @@ -216,7 +216,7 @@ SET } }; - Ok(record_id) + Ok((record_id, inference_result_token.clone())) } pub struct W2lTemplateRecord2 { diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index c43d2debf7..2bc7e220e1 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -8,6 +8,7 @@ #[macro_use] extern crate serde_derive; pub mod buckets; +pub mod common_queries; pub mod job_queries; pub mod script_execution; pub mod util; @@ -18,6 +19,7 @@ use crate::buckets::bucket_client::BucketClient; use crate::buckets::bucket_path_unifier::BucketPathUnifier; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::buckets::file_hashing::get_file_hash; +use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; use crate::job_queries::w2l_inference_job_queries::insert_w2l_result; @@ -43,9 +45,9 @@ use std::fs::{File, metadata}; use std::io::{BufReader, Read}; use std::path::{PathBuf, Path}; use std::process::Command; +use std::thread; use std::time::Duration; use tempdir::TempDir; -use std::thread; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -77,6 +79,8 @@ struct Inferencer { pub private_bucket_client: BucketClient, pub public_bucket_client: BucketClient, + pub firehose_publisher: FirehosePublisher, + pub bucket_path_unifier: BucketPathUnifier, pub semi_persistent_cache: SemiPersistentCacheDir, @@ -190,6 +194,10 @@ async fn main() -> AnyhowResult<()> { let w2l_end_bump_filename = easyenv::get_env_string_or_default( "W2L_END_BUMP_FILENAME", "vocodes-short-end-bump.mp4"); + let firehose_publisher = FirehosePublisher { + mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe + }; + let inferencer = Inferencer { download_temp_directory: temp_directory, mysql_pool, @@ -203,6 +211,7 @@ async fn main() -> AnyhowResult<()> { w2l_inference: w2l_inference_command, bucket_path_unifier: BucketPathUnifier::default_paths(), semi_persistent_cache, + firehose_publisher, w2l_model_filename, w2l_end_bump_filename, }; @@ -510,7 +519,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // ==================== SAVE RECORDS ==================== // info!("Saving w2l inference record..."); - let id = insert_w2l_result( + let (id, inference_result_token) = insert_w2l_result( &inferencer.mysql_pool, job, &result_object_path, @@ -524,6 +533,15 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An info!("Marking job complete..."); mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?; + inferencer.firehose_publisher.w2l_inference_finished( + job.maybe_creator_user_token.as_deref(), + &inference_result_token) + .await + .map_err(|e| { + warn!("error publishing event: {:?}", e); + anyhow!("error publishing event") + })?; + info!("Job {} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}", job.id, id); From f5f5a8dd9317a75564fd4b4eab6df5cf60de3b33 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 04:27:51 -0400 Subject: [PATCH 0210/1042] add tacotron2 --- models/tacotron2/.gitmodules | 4 + models/tacotron2/Dockerfile | 10 + models/tacotron2/LICENSE | 29 + models/tacotron2/README.md | 81 + models/tacotron2/audio_processing.py | 93 + models/tacotron2/data_utils.py | 111 + models/tacotron2/demo.wav | Bin 0 -> 394298 bytes models/tacotron2/distributed.py | 173 + .../ljs_audio_text_test_filelist.txt | 500 + .../ljs_audio_text_train_filelist.txt | 12500 ++++++++++++++++ .../filelists/ljs_audio_text_val_filelist.txt | 100 + models/tacotron2/hparams.py | 95 + models/tacotron2/inference.ipynb | 266 + models/tacotron2/layers.py | 80 + models/tacotron2/logger.py | 48 + models/tacotron2/loss_function.py | 19 + models/tacotron2/loss_scaler.py | 131 + models/tacotron2/model.py | 529 + models/tacotron2/multiproc.py | 23 + models/tacotron2/plotting_utils.py | 61 + models/tacotron2/requirements.txt | 8 + models/tacotron2/stft.py | 141 + models/tacotron2/tensorboard.png | Bin 0 -> 174241 bytes models/tacotron2/text/LICENSE | 19 + models/tacotron2/text/__init__.py | 74 + models/tacotron2/text/cleaners.py | 90 + models/tacotron2/text/cmudict.py | 65 + models/tacotron2/text/numbers.py | 71 + models/tacotron2/text/symbols.py | 18 + models/tacotron2/train.py | 290 + models/tacotron2/utils.py | 29 + 31 files changed, 15658 insertions(+) create mode 100644 models/tacotron2/.gitmodules create mode 100644 models/tacotron2/Dockerfile create mode 100644 models/tacotron2/LICENSE create mode 100755 models/tacotron2/README.md create mode 100644 models/tacotron2/audio_processing.py create mode 100644 models/tacotron2/data_utils.py create mode 100755 models/tacotron2/demo.wav create mode 100644 models/tacotron2/distributed.py create mode 100755 models/tacotron2/filelists/ljs_audio_text_test_filelist.txt create mode 100755 models/tacotron2/filelists/ljs_audio_text_train_filelist.txt create mode 100755 models/tacotron2/filelists/ljs_audio_text_val_filelist.txt create mode 100644 models/tacotron2/hparams.py create mode 100644 models/tacotron2/inference.ipynb create mode 100644 models/tacotron2/layers.py create mode 100644 models/tacotron2/logger.py create mode 100644 models/tacotron2/loss_function.py create mode 100644 models/tacotron2/loss_scaler.py create mode 100644 models/tacotron2/model.py create mode 100644 models/tacotron2/multiproc.py create mode 100644 models/tacotron2/plotting_utils.py create mode 100644 models/tacotron2/requirements.txt create mode 100644 models/tacotron2/stft.py create mode 100644 models/tacotron2/tensorboard.png create mode 100644 models/tacotron2/text/LICENSE create mode 100644 models/tacotron2/text/__init__.py create mode 100644 models/tacotron2/text/cleaners.py create mode 100644 models/tacotron2/text/cmudict.py create mode 100644 models/tacotron2/text/numbers.py create mode 100644 models/tacotron2/text/symbols.py create mode 100644 models/tacotron2/train.py create mode 100644 models/tacotron2/utils.py diff --git a/models/tacotron2/.gitmodules b/models/tacotron2/.gitmodules new file mode 100644 index 0000000000..051ba22691 --- /dev/null +++ b/models/tacotron2/.gitmodules @@ -0,0 +1,4 @@ +[submodule "waveglow"] + path = waveglow + url = https://github.com/NVIDIA/waveglow + branch = master diff --git a/models/tacotron2/Dockerfile b/models/tacotron2/Dockerfile new file mode 100644 index 0000000000..44864b936c --- /dev/null +++ b/models/tacotron2/Dockerfile @@ -0,0 +1,10 @@ +FROM pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 +ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} + +RUN apt-get update -y + +RUN pip install numpy scipy matplotlib librosa==0.6.0 tensorflow tensorboardX inflect==0.2.5 Unidecode==1.0.22 pillow jupyter + +ADD apex /apex/ +WORKDIR /apex/ +RUN pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" . diff --git a/models/tacotron2/LICENSE b/models/tacotron2/LICENSE new file mode 100644 index 0000000000..2a718d63da --- /dev/null +++ b/models/tacotron2/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2018, NVIDIA Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/models/tacotron2/README.md b/models/tacotron2/README.md new file mode 100755 index 0000000000..dde0f1a542 --- /dev/null +++ b/models/tacotron2/README.md @@ -0,0 +1,81 @@ +# Tacotron 2 (without wavenet) + +PyTorch implementation of [Natural TTS Synthesis By Conditioning +Wavenet On Mel Spectrogram Predictions](https://arxiv.org/pdf/1712.05884.pdf). + +This implementation includes **distributed** and **automatic mixed precision** support +and uses the [LJSpeech dataset](https://keithito.com/LJ-Speech-Dataset/). + +Distributed and Automatic Mixed Precision support relies on NVIDIA's [Apex] and [AMP]. + +Visit our [website] for audio samples using our published [Tacotron 2] and +[WaveGlow] models. + +![Alignment, Predicted Mel Spectrogram, Target Mel Spectrogram](tensorboard.png) + + +## Pre-requisites +1. NVIDIA GPU + CUDA cuDNN + +## Setup +1. Download and extract the [LJ Speech dataset](https://keithito.com/LJ-Speech-Dataset/) +2. Clone this repo: `git clone https://github.com/NVIDIA/tacotron2.git` +3. CD into this repo: `cd tacotron2` +4. Initialize submodule: `git submodule init; git submodule update` +5. Update .wav paths: `sed -i -- 's,DUMMY,ljs_dataset_folder/wavs,g' filelists/*.txt` + - Alternatively, set `load_mel_from_disk=True` in `hparams.py` and update mel-spectrogram paths +6. Install [PyTorch 1.0] +7. Install [Apex] +8. Install python requirements or build docker image + - Install python requirements: `pip install -r requirements.txt` + +## Training +1. `python train.py --output_directory=outdir --log_directory=logdir` +2. (OPTIONAL) `tensorboard --logdir=outdir/logdir` + +## Training using a pre-trained model +Training using a pre-trained model can lead to faster convergence +By default, the dataset dependent text embedding layers are [ignored] + +1. Download our published [Tacotron 2] model +2. `python train.py --output_directory=outdir --log_directory=logdir -c tacotron2_statedict.pt --warm_start` + +## Multi-GPU (distributed) and Automatic Mixed Precision Training +1. `python -m multiproc train.py --output_directory=outdir --log_directory=logdir --hparams=distributed_run=True,fp16_run=True` + +## Inference demo +1. Download our published [Tacotron 2] model +2. Download our published [WaveGlow] model +3. `jupyter notebook --ip=127.0.0.1 --port=31337` +4. Load inference.ipynb + +N.b. When performing Mel-Spectrogram to Audio synthesis, make sure Tacotron 2 +and the Mel decoder were trained on the same mel-spectrogram representation. + + +## Related repos +[WaveGlow](https://github.com/NVIDIA/WaveGlow) Faster than real time Flow-based +Generative Network for Speech Synthesis + +[nv-wavenet](https://github.com/NVIDIA/nv-wavenet/) Faster than real time +WaveNet. + +## Acknowledgements +This implementation uses code from the following repos: [Keith +Ito](https://github.com/keithito/tacotron/), [Prem +Seetharaman](https://github.com/pseeth/pytorch-stft) as described in our code. + +We are inspired by [Ryuchi Yamamoto's](https://github.com/r9y9/tacotron_pytorch) +Tacotron PyTorch implementation. + +We are thankful to the Tacotron 2 paper authors, specially Jonathan Shen, Yuxuan +Wang and Zongheng Yang. + + +[WaveGlow]: https://drive.google.com/open?id=1rpK8CzAAirq9sWZhe9nlfvxMF1dRgFbF +[Tacotron 2]: https://drive.google.com/file/d/1c5ZTuT7J08wLUoVZ2KkUs_VdZuJ86ZqA/view?usp=sharing +[pytorch 1.0]: https://github.com/pytorch/pytorch#installation +[website]: https://nv-adlr.github.io/WaveGlow +[ignored]: https://github.com/NVIDIA/tacotron2/blob/master/hparams.py#L22 +[Apex]: https://github.com/nvidia/apex +[AMP]: https://github.com/NVIDIA/apex/tree/master/apex/amp \ No newline at end of file diff --git a/models/tacotron2/audio_processing.py b/models/tacotron2/audio_processing.py new file mode 100644 index 0000000000..b5af7f723e --- /dev/null +++ b/models/tacotron2/audio_processing.py @@ -0,0 +1,93 @@ +import torch +import numpy as np +from scipy.signal import get_window +import librosa.util as librosa_util + + +def window_sumsquare(window, n_frames, hop_length=200, win_length=800, + n_fft=800, dtype=np.float32, norm=None): + """ + # from librosa 0.6 + Compute the sum-square envelope of a window function at a given hop length. + + This is used to estimate modulation effects induced by windowing + observations in short-time fourier transforms. + + Parameters + ---------- + window : string, tuple, number, callable, or list-like + Window specification, as in `get_window` + + n_frames : int > 0 + The number of analysis frames + + hop_length : int > 0 + The number of samples to advance between frames + + win_length : [optional] + The length of the window function. By default, this matches `n_fft`. + + n_fft : int > 0 + The length of each analysis frame. + + dtype : np.dtype + The data type of the output + + Returns + ------- + wss : np.ndarray, shape=`(n_fft + hop_length * (n_frames - 1))` + The sum-squared envelope of the window function + """ + if win_length is None: + win_length = n_fft + + n = n_fft + hop_length * (n_frames - 1) + x = np.zeros(n, dtype=dtype) + + # Compute the squared window at the desired length + win_sq = get_window(window, win_length, fftbins=True) + win_sq = librosa_util.normalize(win_sq, norm=norm)**2 + win_sq = librosa_util.pad_center(win_sq, n_fft) + + # Fill the envelope + for i in range(n_frames): + sample = i * hop_length + x[sample:min(n, sample + n_fft)] += win_sq[:max(0, min(n_fft, n - sample))] + return x + + +def griffin_lim(magnitudes, stft_fn, n_iters=30): + """ + PARAMS + ------ + magnitudes: spectrogram magnitudes + stft_fn: STFT class with transform (STFT) and inverse (ISTFT) methods + """ + + angles = np.angle(np.exp(2j * np.pi * np.random.rand(*magnitudes.size()))) + angles = angles.astype(np.float32) + angles = torch.autograd.Variable(torch.from_numpy(angles)) + signal = stft_fn.inverse(magnitudes, angles).squeeze(1) + + for i in range(n_iters): + _, angles = stft_fn.transform(signal) + signal = stft_fn.inverse(magnitudes, angles).squeeze(1) + return signal + + +def dynamic_range_compression(x, C=1, clip_val=1e-5): + """ + PARAMS + ------ + C: compression factor + """ + return torch.log(torch.clamp(x, min=clip_val) * C) + + +def dynamic_range_decompression(x, C=1): + """ + PARAMS + ------ + C: compression factor used to compress + """ + return torch.exp(x) / C diff --git a/models/tacotron2/data_utils.py b/models/tacotron2/data_utils.py new file mode 100644 index 0000000000..ed8723864d --- /dev/null +++ b/models/tacotron2/data_utils.py @@ -0,0 +1,111 @@ +import random +import numpy as np +import torch +import torch.utils.data + +import layers +from utils import load_wav_to_torch, load_filepaths_and_text +from text import text_to_sequence + + +class TextMelLoader(torch.utils.data.Dataset): + """ + 1) loads audio,text pairs + 2) normalizes text and converts them to sequences of one-hot vectors + 3) computes mel-spectrograms from audio files. + """ + def __init__(self, audiopaths_and_text, hparams): + self.audiopaths_and_text = load_filepaths_and_text(audiopaths_and_text) + self.text_cleaners = hparams.text_cleaners + self.max_wav_value = hparams.max_wav_value + self.sampling_rate = hparams.sampling_rate + self.load_mel_from_disk = hparams.load_mel_from_disk + self.stft = layers.TacotronSTFT( + hparams.filter_length, hparams.hop_length, hparams.win_length, + hparams.n_mel_channels, hparams.sampling_rate, hparams.mel_fmin, + hparams.mel_fmax) + random.seed(hparams.seed) + random.shuffle(self.audiopaths_and_text) + + def get_mel_text_pair(self, audiopath_and_text): + # separate filename and text + audiopath, text = audiopath_and_text[0], audiopath_and_text[1] + text = self.get_text(text) + mel = self.get_mel(audiopath) + return (text, mel) + + def get_mel(self, filename): + if not self.load_mel_from_disk: + audio, sampling_rate = load_wav_to_torch(filename) + if sampling_rate != self.stft.sampling_rate: + raise ValueError("{} {} SR doesn't match target {} SR".format( + sampling_rate, self.stft.sampling_rate)) + audio_norm = audio / self.max_wav_value + audio_norm = audio_norm.unsqueeze(0) + audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False) + melspec = self.stft.mel_spectrogram(audio_norm) + melspec = torch.squeeze(melspec, 0) + else: + melspec = torch.from_numpy(np.load(filename)) + assert melspec.size(0) == self.stft.n_mel_channels, ( + 'Mel dimension mismatch: given {}, expected {}'.format( + melspec.size(0), self.stft.n_mel_channels)) + + return melspec + + def get_text(self, text): + text_norm = torch.IntTensor(text_to_sequence(text, self.text_cleaners)) + return text_norm + + def __getitem__(self, index): + return self.get_mel_text_pair(self.audiopaths_and_text[index]) + + def __len__(self): + return len(self.audiopaths_and_text) + + +class TextMelCollate(): + """ Zero-pads model inputs and targets based on number of frames per setep + """ + def __init__(self, n_frames_per_step): + self.n_frames_per_step = n_frames_per_step + + def __call__(self, batch): + """Collate's training batch from normalized text and mel-spectrogram + PARAMS + ------ + batch: [text_normalized, mel_normalized] + """ + # Right zero-pad all one-hot text sequences to max input length + input_lengths, ids_sorted_decreasing = torch.sort( + torch.LongTensor([len(x[0]) for x in batch]), + dim=0, descending=True) + max_input_len = input_lengths[0] + + text_padded = torch.LongTensor(len(batch), max_input_len) + text_padded.zero_() + for i in range(len(ids_sorted_decreasing)): + text = batch[ids_sorted_decreasing[i]][0] + text_padded[i, :text.size(0)] = text + + # Right zero-pad mel-spec + num_mels = batch[0][1].size(0) + max_target_len = max([x[1].size(1) for x in batch]) + if max_target_len % self.n_frames_per_step != 0: + max_target_len += self.n_frames_per_step - max_target_len % self.n_frames_per_step + assert max_target_len % self.n_frames_per_step == 0 + + # include mel padded and gate padded + mel_padded = torch.FloatTensor(len(batch), num_mels, max_target_len) + mel_padded.zero_() + gate_padded = torch.FloatTensor(len(batch), max_target_len) + gate_padded.zero_() + output_lengths = torch.LongTensor(len(batch)) + for i in range(len(ids_sorted_decreasing)): + mel = batch[ids_sorted_decreasing[i]][1] + mel_padded[i, :, :mel.size(1)] = mel + gate_padded[i, mel.size(1)-1:] = 1 + output_lengths[i] = mel.size(1) + + return text_padded, input_lengths, mel_padded, gate_padded, \ + output_lengths diff --git a/models/tacotron2/demo.wav b/models/tacotron2/demo.wav new file mode 100755 index 0000000000000000000000000000000000000000..54d1dbc4fedd7796c9b43b4118055dc6a0e94249 GIT binary patch literal 394298 zcmW(+c{Eo~AGRfHWKSx)2nkv4%-k#`Nkxk!NfK?O(q6KKBzq!6Whaz~@66ngLP;VL zDwQQltDkng&imIrXU>^(?wRx4=RTk3d1mbjJ3C_`VZM#lF3WuU0##-B`1l0*`1#ab z`1qn+`T2zSRHs&awrvj-ntJku@bh_Z3*5#xwL|>EY~(e15xVb@rIS#GZn%>LEtYB{ z$>d!;<aI6>mFJ)bO~*-6sMzZ{3Lv ztt4oJ?^Tc&xd3@(ykJBNbZIT#PNcreqEU-maI?=C@k=Ww3D^6f-1rLE6Ml`ik$IRm2Y5B>=w|p zi6r`i6}V*B8frOfLcfpVr1N4n=@FO%+vDP)@{yUu);*pSmPo_lVGX$HzjxS1PzW0s zz2@!wk!mTSk^vm+^qGZwMzCjp7|(v|>kIorz&`A|<@-`=>7M-Ae|17wB zdLl0;)rp87-^EBV5wP1b178soBb(kt11CX$=FMg;Y?-S912|*GMCKsLR@(!<%svMb zlrQ2IB_+~6X$!_)-Ug8^6~Mfihi41C!aj2U6Gh0VO6aT(}ia0Cqfl^`MMs@Nr>8mLFhGod_!=NsJsUw=k1@9HUz z7!3eP3#FKUc6YFqc$9@_cdI4)?j{I)Yyyt=nvrR%=i>xN2`KSIA1twsz+J~fc)!%1 z@eBjafS~0WFws$fbH;wxw4JR5e10o&$hI+z--|%@Wj1*9Hk8Oc@L>XPs>57KF|yP6 zx231{MSz55NX@7^-eckjJ;OxsQyXK_C>G7g9J&f5uI~nJ3fWjyWG&BBFqaoT9)$-l zu7EQZD1aTg=ZM0R<6yq~Zm7<`44*fUBR@Cuff6?ikTb-N1<5tV%GQ8Z-mHmVPkH3r zC+7H<^}Nfs1o4ak@P;j`ndy(Cft}BKCg^}UZqmJ3^WX9W9$UE=tmG=e z`;8{pb)*eDMkRv!0(q#+FGP%Lvw0aFR`9v+bDnC=CmbF!4QPH@2hGC0@LsA$>hW&o zP1|EowqX!>2MZGO?g*@84MFkv5^zu72za#Q;~1NhSaQDr@MKDWk6j{f<93XlRc(2m z4vWBKyBt{Sv5%=&^T03P?8cSL3IH$YKI8v+f_Ln*AWu5Lg5epi0H!m))kOE-;(Z|k zIQ3sSFdRF{+#CLdxrM$!S0Nt=M26!TVc9r^_Z#RwPz9>-A8`0MQzQII5IAb3g2wkJ z@YiVv@E?Z@;P#6KX6-R`oS4Ih?@yM2A4jEm44#Gg$5!LEO|3P3x>tbL%3F+~${5cu z>I)9F2n9L2LP5HP2u~)S!2v2-ylrz6z}E6L*h}CUPcLJHXZAZ9+*mpjw4Hl_&HZ|K z5p{9ko};NNf&FVX5e4+fY3`A2=cvo=0V#J2(7>NPuXlrj%kRSEKp{!HSvx@!Pm z;Vcj~r;67;pp7fu3u80wLrgBu~uy)}eCgFW7 ze%W;dJ1As=n6e|58NZg-Ea=SbF ze^XhFT=ga<+9r$29Zlfx^3$MoRzHqvmBk&klZ@g8HDG2s6Mt9|R^u4> zm+@+L!5RNm@nY=rnc&GPW}TKPSjR{MZ*UKX-gDuppKJw>l#YS>d*AV-vfkn#r$W%D zoDZZgwenWoZ|8X*-C{X%&kb1dDcAh{)XLQDYv)xc{$ezFqQD9w{7BsZWI0NMSwnWX zDKM9(E|v_6I+StYj2K3FwlFB?(lEr3C%t)Z%}{&^@E>X7jURr6e`-fFc15yG|84V{ zfPhE~A?Hk9@ZuzFrr=u-ZJ$Aj|uS>0?#eonM)V8 z<8#`N(MtrF`2J*Ys9zDYnMF)S&U0S>8Zj)LVguZqQ-FQgLwsL- z4sYDko>97doY9XFgg@^a5>NBh)!I$rVAG&2&b?*;{A*OPoWy+QaX>Y%kN2)d`S@B! zNL&{jmi)vU_)){~Jr@8gY7XQ1$E`q~^GaUz@j+niun;S|4d9KlpMwK&5EzBd$1280 zv77Z3@Ulaish<4JME73751}b`D>jE~geG{ET9J6rB8~pBTUuLeH?}``J_f(`x4F_!azntN}~t z&44R5J>$7A%;X(4DB&sBEW{Um7ee=07RiEGq80dP+$P?udnv$Cv>MDm`3S4+pHJ3HRe;gA zzdZ74eCJFBLr(4n{Zz{$?_wJE4lxC#H-1}6Ea+jXpN=y# zY$ov+PjT43a~ilMvmWdIGX9ZJX*fcW0Iu$st!@llTXe zCkXIDzjt9t_!0L=yFsUCzLp6M$#|YYF+OhS1YWLifT_NDL?G=JaN*~N>f*J$UyJy$ z@Gfy;@WTvl(-FaICtu@bQQkm$nGoFaQj8$mDlDRX2fU6K0G;c?z@=@7NO#TS%^%nT zB|K*ktFH@c;E*)9SM(O=!S(Q5a0@osbq>d-ZHNEKnvt99C5eAa7A%@K6Y7LSlRy1M zxb1~3JblRsdRIQ<-EmeXT_=mcA6F5OJpBn+q8y9MmA_-Lwj;1tQ;>{*)dA+lpMY{= zIH`1c1SZwnz{PvpL4QXFb~0N?iv0~??{Z0SU}YJ&aDNAJXy&l3x*ANN{LtHS4KMac z9+6of24AdKf)*!sf~sjFxK*qMpT77Ibmv~IAzw9N zf2CO3*c;D!jbNVsJ&=(-f`2Z3jIa6_fEi{TK+UQO1eY$Z5!{qdCcmmdA^!;2B&9$; z6$%rFKmQn6-d-^7mIG8>(=ng^B=*pfhvj|xAXWEK&BuenL?viDqq@-sT72}ttGCW1 zTVpPO>f4UMuW}XRL|)WXznV$n0R@W%;z7=ZIk@0#43FAQ}{jQbBeKy{0mkEd;$f9eLUrPn4DXA6NHZ4 zhXDu1aGBmw@?*nUu+_c?yk3itvPNfsm)KUCin2pxPJhg&#Woktd2cj zw_A$fl$poQ829R1%y zVi>UsOFQnt$~O#Q_pt@A>3Tl)Dtu4Q_N2h1SCOzK+?=SAlcdMwD==TI4F?-k$jIbCa~3@F!>|Y z4fhHb;;Yz~{P$-zbb2WOPW2PD2olDUG@33rwc#bxFZjh~N6S*QZ)az76iR`e4?!MrIb!5%N4dPjG398#TLxJ!8K)LJ&u`Nv}<;()~L1#A@i4G=3^9+g7+#X={ zkb^>wmgHXQ1)}DW0e`aNAiyMwterj^zAfm$nvW|%mwY1#I#rBsT-yfjq?Qni^Bkl- z{!mz5ibl>X0w1+fV4lM@FjjPmXnDpEnXett`Pm1+yS$I6hQFvuo5xP!H3+Uhdm5Su zdXkEGE7D}Sp~j};E_~lUliZZJP0*R!(Egt=#O)7=LUB1GIHU)!FDM0>=_aJ>_+D(E za}W60M&KQa*Ffw$4Z@ekZ|R`i4_2INoZ_hwux#Hq%)2?4aEnjD`_9U6u8cPsexFKq z47I@v`Wpd>)gg+rV}Zq@IpmLa4v_v*45x2TCefYhWFS%ssg0Kqd!3_Oc)kyO$hHWd88c#6$;wmm%_*7$@~&>=J^+B zVqFHKoemO^Ekjr33!w!Am*LkFt%UDjhq>Z~F!*`*5BzAJ6?yP;C#?GM3%uX7maMN7 zBhMu>;NLh|ykaqxXV!sHMxEz zSt84+;-e$TahoU#)Qg}xXIsft8wb>JpcXFfx28+YMVWv{7`_%RCHbR@&|WVd_Q%E% zujF>{eLHiq_KJM?b!SYrWN|unIr9 zrn(fp__dH;Ueyf28$}rOZ7U@?{zyF~461HTpm%1@Koc(?L+htAsckHPW!FwYulIH2 zMo0i}G)osOt>fzE>`X~A*{ly`F#%K08a1KksdReK2f zt5!koyWJ#$YCEQUoQI@BbOB2F_7|W0t3Y3sU58Qj;Xr9aHRua6BNM|1NtWXTKRgsx6GTMIzzbY=du(Zb97RMO5X3J@IN-hP)qo(x^yfw69O9(Qw*m#!&Y|;rOi7_Hwd$=3ao zpo?anrb~Ib>|XUBIse6SYT2w0(SAFk5>wBxW=d+m@d|cX_ zKgfsbBQNiVRG`^`^obXvjr;!LCq5}OLTedH4p|NReEW!JIxn|iFtp4f~^lkQMnjfddMg^MCEYsVRq>83ZBLy8AgtY8>Ct->= zqO>0)=+KPe~Lhvv< ziJUr?1t$;Z)AYb}WZoNr4!ADl?)cw9`Sq!213Qn?xbXs|T)ImVg0sobVu)0w*VAv^ z$z<@n1G4>XimpF8PyZU&&_gx}=-rlH7*pa(tAvLj-?#*wjOw(!Mb4qlNJO8VT1Lgl zO$5!XsNLJS#LqJcMF}_3>5Daxhnzer?p35~G*83+o2Hbt4?`p=9)^ATN0%P|jGCXc zz==2d+<*S&tjhPrwBgJPy0{Lqmz!Iiakp;Uw=jg-8aeS;UbD6@{r8WETVTgg`VgTWOey0soujjYLWH_eU_U6 znm)Xx$?{1^C^Hf*uO6fyz3;+Gejnsy+f5`o3~1Tsoyc0>1MYBqKto)5(dqh+K>2YJEol{_e_x#l1I)Z_-Lsw<(2c_auXfD>bOgq>*M;1;dz@Bj_3zMz1uAkQp5b z$Y&~T(( z^o9tGoh5x4dl0)Qhc{Hk zm*TgY*U$rjPc%?G0hO2*pzd-l?tGFY60p06EP1C$*z!^u9>9T1ZwgRp=y8x4HAt&; zqtWFRE=a7tfj&r=K#}8SXvkWYYKQ$IO78oRx7;5Re8ZgV-l2l}PSIGh3XR1{!9%&fE0GGVy+C_sNV7I?=hEIEHPqIA3DTRs3e5;PMPzSpqmJESa8|rGI%>Cx&i`ve zEUjmwJ+E0P^e2RRFDd}4mFH2t?-04KuS@@X@feHxUPD(eucZP9C&)C*GsrFB7SY@u zL5~JRpov%$v_If3{W)z265*|I={Fwi^|(mJ_ZZSkRW3+1^;}KL2_eco_lBDjb?p_|u&}+aQnnkF5l_yoeCpd{P>7 z1vP!mAir)G(;dE>(5dugX!ItarDb19Pn7GME{L2Bt$AJa`y>NW4xg))x@JKXs@*bt$bWtD%+Z?P#nq5$vv#rSoqW zldyaUTg}VSyB(v{bgmJ0xB7t&zssW=OUudP3#*{Jiwkn;O#!Xmqx4yB3kn?m2aKP( z(|jih)U>*gv_!2%&M^gaakB@^${a)nkxf*^$(!VL!fTUbZ(kV?=8qofGb##HZ674n~M{NQ#Xxln*uG#7d+I*oEUAR0!tB+<- z89NoWN_rJ}Q76i^e|U&|w{AwG+V|*;R)*e;r^7v;X~kYvvs9EA$j znw-uj0XBWg^BPcTqFMPp=!)|;+O^UhriI>w!pkqwf7T5ss$)8NaC04Xk!-~C&mBQm z&o{#V*0j+7G9Lr!kK(LddmSAw>7Z>58(3`t3+f}afHP|DK;`s1TDnx2+j)(p#=D*( z`;?S3k@_dVAXr8 zsrYs=j(#;}Eu5~PlY7>2yyCfZeQF5%YJV3!?5f1EPbApE%3|cW#h9}z_Jo(r+Yl3* z13Vo+(Dw$#W7Ca(M8qIwdL(s-N~b?obRpwMi9_g@^jmtc){J^77?L^-0a zYlvdy&(MkkzG%abOnjmD9~n8@3fKNU#&q+#nT&UK1iMv{^hX8AZ+|mYE6Ra$n+fW2 z8>U$yvr#pk1j+8TPQTNJFy4+xrXw)_$jm~XU;7lkL{wB}1ibhk_!a8cb zPJt!G9<+Z@pPo>-j#jtC(p&lmaJNM`s@4Z|b*2kyFmR{4_8+ACgCSc&M7hS8btJU4 z5J}kT(X~7Eark2aGM7z4U!p@OxxSpPQIA1s^g4oM23OA43O5c0p?o73YWi<3x_s<4 zw#r^ioAw2uT`Id_PeweQAplX7;Ws>YDnv?dIgDCXuBM)~5j0|fIh%Z0fHVDYhLYK8 zY_01vqP|a-yVX&Tf|aTf+^)s7Nv=Ta9U_q7wR&nlDT|ULgAu6eqXS>0r%Y)+*7e90 zdfNIo{rA_5-M;KOzBn?AdwV|>9lbJypo|_@@!v{hG`$t=K0SkTS5ZMiKiyEf!wo#` zP&RcF_C{xg1khM*I^F$V4P6c~Kz+tn>9=MfSZ;Y1EnWDNRNob)&%Hl^KRevOvj?N_ zgi0J4)9xWrlB2wGe|IF1SVo$Tm(ja*Gg!?bKt=rdI8_rRw!Oj!rZtYzikvdIw^Op7tr$VFn zIq>YBCvbhlO!Q~(Jg(+_G1@&L&Q{&FSg?jAH>ZGlQ>T%_9X^-0l)JNj~919{q}O%@*+AlV+vP~pNeu^pqPst! z^Ge&fRb~f~p1Kt)uLQZg3wx>g8W&c=?-s2t4yMPWo}l&p_h^Nx3sT~;(IwltT=l^# zs3)x-iT0UuxabD7L95xFA&a@!+pp1hhdu1-d!;n1@f98A8$irDWju}VBu(Kek&B%T z{czTXYJSW_!u9sFQ`sD7&M8G@)?%F3$TIZR;tCQddrP}}E0No&IX&9;WZSV!qCB^r)ciq z8LZXERkZ8V1G;zm9drq=LPnvD)MLq%cl`4-Qe6>2pXb`4;5G$DC7DNeq|ahoRwmIS z&H~&*1AkWGd^Nq>q0B89-p#hnFQcVfhp8FXV{2Z;QjxMLEfoULe}blT&zCSXaefx9 zPdP*Hj=ext{91I|$2b~f@(}5NQ=(r5F*!VZ3yz3t(_2;XXuQ=DG^@X+3m0BRySrSF z<**=^G{u(|CsonYSGTFx)^||HDi?it7))J122tPqS19#xKYs2>sLu~e zCqFDh-x`zAg9GC>fLXn7Sw`vzr6_7PiP`aRlN1$R59N81^y?st95-*J z{f|a*=k!j*sV=3TluKxcZWQvmcLRk5*3sy89csVrKAI-Fhz@&+q37MRX@kTtX^|vI z_?ABXxNj#_dTog8vX7vrePy&&PY?U$SfHfwJn%BJg_!30(C3~VYQYm*-*y(x%yaNi+T<>@Mp<0fqS@?zTBxQR-XP*f87fCNvZ)234w(An%? z@Oj}Y%87KL-9cSsLb#raIT*6MUw5eG!3VTqhc$a;Ll%A0xRiMo!uMBb6b`yo6($Pa(@5$d;k4*Rtt zie4lisc|b}%{|KKmvBS+o;5*&H^iwmXE6D3v2|ah>Hwtfh4jgk2 zaVbLaEbpx*^kG)uk>_s5Td~*DTWD=sHDO~L(5AK)sx^{*L) zY|T7$qTh{sja=AzQ-DU2yts0UTr^Z-!zK-RavKs?(80t=*3k4BUFo=zi;`Q$>YliQ z4prK7ZVO7#tSi${$4^0e+vgd%`0X6}Z4-A3qvYuR^+e zT)29DV^;H`4Vsk@&h1}(fNk>e#OBqZ+G%{q{Jl1}`$Gb&PNl*>WzBxd*LBA`{fy~x%vg_J`{nb?EqY{w-;Nm zn4eR5-#`O5pJAh#8P3frp6V_OVK2BjaGIyw(DXAQqob`t!Hq0r9)=J)|G4EEeX*zz~q}OAlqpr;ge=Fiz8&|R6e6P^Q zzIZOGSd-;_bVp^PmuO$F1?`X3NBtVkXvUdPVzcM*l$H`@_jqoky(Yy}Bi)*PIikwN z%m2pqHpZ-qk2QBd_A(01+K!g^F5zzNk7jjU)^eTGdvULo8w(5)xD5#sYy`+f!U=xd z->?{#U(u9K5w$uqioxgNd0Xa{Tl z>@Zg&>Vldt^RW)Y~o4(d5Z(*|>vwKNyx+=|BNd*5g_zfG3oMw{psA6s^^Xp$}t zZGpR0qEOXr0nY1$5bKK5sc`o^)O-IaaWP3E+itxy&$_TWT8TXcCMf_ot)E1@9=)X#`HqHt3h+dh{>!2_?%UsgmWaavL@@e zk(*Jhd-D^Tz3c+rpL8FsSbB_(pE`^TdmhmTck0l)UxTkq5B-xTO*0;AFd#+ zDoIG7G?r?+uSWAtZ-TEKyJ)@nDU`WoCoJAMrG2Fe@lJa=HbYsHQ=X$kHmvhvC;Gy; zU8}F7LvA=)T*tm+tkZ>mWJd2i?yFN9vJ*5xc{6^JhL&9FzAF)Z*Gooh z%OY-@+cb8scRLu`_l0U5(r0B1<+*L2Qc>5{hu~sb7WD=1(d5iw>N zUWl@nt6~uhlI83jMPbwE2r6B1h&mtJOdqvhMNYcM=*aI#wEoaJxHKLyab8x@GC4N&vHHSlM7 zDsp&xotF6SLeKwBN4tuRxz8R4+0ov+)FNGs(^B5ZhAg^BFUnn}8#Ya)sC^SO#N;NL z3l^Zod>)+Mo>Q#;I!~(YH=iqfE67f?WFV)~d0cSmdyDtBi`Z@H)40n|_tQg41ANtGuw$(wz<5m#J`t*rJ!rr($HdpN+H6W7qZMiZ)WB#mlHD6_?Zx2Pr;PW|ho z*r4WJbW7EAdi|_3d?s)K`S?asks1T=Ems&BJ}{=DR%WPtb|f$NX#&NzHApqr2Adob z;s{Y<&*s*VI~teh6?Z)p-)98#I%06hesTKFdOCbBaRe!P$a5VX60ClpG+MoPFUKDh z$X1BnqW&m}I{ejSZGAMk@>6MW`;Z>{+#hfOxAmbz1wXs}SQI^>mq||^Yeu~0M5KGg zk;{Xv==Stftzkm~!?@ng0o1ec2kI_4!d*MvheUd2 zuz!5_am((eAfIW9>@87S&NS>P5U0*;`0;g|{Ss4#9q?ww3`04uXS2}gdKdPHw>u~N zbpg`9EWk5-(wS(o>l|G>JAqu)YebU?lnS_&5~(zGHfx0;*Xw+nc&YATFJODFovUM> z$4auXYWiG`jSy??T!=MTe(rdMJnOn=3sWvH&CRJ)VA;G*5}=YxJD)V66AD&T@8}r( z=yHU_>?;F>6^iubx?RW#Yy|d;J!r)7Ubx+39>Qf8X+-O1*d2-|KR6CO>JXUn-k_z>T|=Rz_pGqS)&{&A5Yx31s(UW7ck71b5wi zBWt7LLzJ9nPU+7P#J5(E*&ThK4DkV~{P-~ns$NY?GP^ZZEg(+l>GU3|LOGubp^HsDO%Ae#0d2v^#Zr@AS&|8>r4P_F(FlaAgJ7idbZ$N)l>%gkdIY6&yPMW^XHc4tYZgvgri=YbS`T7 zc6Q`sCQ{sgm>a${hQuFLp{_zzPJMp|dSHH$>@HqUiP|8_>XqZBg~y?HYp$cYS7vjk z7c4?s7h9n!txytDaTN|n)zDLRvFO0@Sd`ap&6y?umZsX10B#v~@7-}W(Dw>G_@Rk% ze)6p3!Xe84N(XHp`-Qfi@#A`Reoy5;imcWqIHcfY*$?h zEj|@W1(p~>`vYDm{RcntNS5LB3oFsgfC4l#SAg5ggyFd#E0HSSP4dt;jU-;ZKtv`>oT3(v44t@u?mnr?o;f)k(t?H-Bf);GgbzSZk*+*Z6MX|TN^*B8T z4ep=|W?MYmxNpAC>HeRW*iwhXoV(Hj7?Zn^9k?9C`Ty-gJ56?=`gdKlv%eYjC`HjK zA2*QWs|fiPkHDSEZe+vk7^*H6i+YsBkniZZ;*u|Ual8hN_ctaLFB~V81 zORBM+C90f*Qwo|``U^Ft1#|5`Oj%d$VWz%JobwfzV%IN~rgz@@(UoT9A)!t=e7;zdJN<=^eeh=% zYOs~!t~7l_36Zm?3U;7swhU6S8Y99oMkpsKm;7F9OLd|X(SdR)s@MIN{9PW59QOv0 z6U`gwkC7p$r~Vx2vfDWYR|)prG<8;U`7N&N`Zm_PS%R(DQq09!tzy$`bCJC9UT*WD zjjY+r82anPe-tj?!%jOT!UY>$r+ao-u(EsQ;m>`EihcN-i6l=W2aa?VOwsO_9gNzQlGu4HJ3YT-GyAbV#uEM zpY+_q?X08Fe9m|({~fa0^8Z;twXm1@zQloj`Suj&P%?m)z3M|jtqxp197DlC37v1g zNqg9G*tbRiIf?VpSI^C1ZH+FoDxV=;JPY)oU7I*M71HUN9jN736~5G%OY>#&(WHtH zmC}`>I>!xuka{kgtno*j`xYEhy-0* zeGTmkFQIK&%?y8DIy&$pg{rN0L(xJ;u+hAgX6Z|^{%#xSiYHTUI~sdzK@ZMuf@){^$^GUhWZRw(=$DTy7npV&9Vz_*0~_vBD={@Ts49&f zR!F8|^M9icktNhm?lS6%%7CTomAG5`4A}nr-t-XyIrgX)4!1QPb?HcXB2XisMk=%TTJ^e27%rgdmB- zZ+V-}o~9r5#Gr`bk-#Xx(xsF68gpo7Xe!?~ERnk&X2%Q?vX4TE<&RpX!X0qf{L| zo?}ehhtH7%MsDB|a~C=)uLpjcG^k5;20YZ<2ZL%k(sWiIuN|!gZI_A2;E}8FYr!X25G+g2ip9Z)S^3ZP(j)TzM8PI6sO%kQkL4tA~LFv3+%k~Ys$!CdNSiZM|G^J@_ zi5h$KX8lYWVsZb@mH+;AC&8=Qspk?)E0{F&&7p(1aBG?4SVS!nG36xuyMNp|1RBS&*y z!quIFM7CQ9nr0T@XOAk#1xpS_s=dbkPfn1W+pTc+8Bd}bB1bD;K7(TZUcf8Nkd&(F zPxYNo!#Br#$OeTreEr`!=x;xOoo=0mUZuA{^bbcOvZxX^wWScvR!6cmryq8YZ)0BW zkSBe~@fZ#8;1zm~NHr>wA@hgu-jTDQZCMrhSpEziiJa=2%i7Si{sMTR{T1lH8%*@( zw8BJ16Y_mfl>YblD->efh{S7QB6|G^v{`e4XfhLwfoD39xvvk9bQBg^=tER$qXGZo zZfK|R0?&Q^9w%Rj2f3Z5BzV(FXm`2_OL%#b+GCeM(kvP_D z)Ml>N3BY@ma#QcyjK%g`gvE#A$dQf@Sl^}*-n^K?yng+MC^sg-y-!BKgDX48&s~!6 zt>!k8axx$D$*rm>pUIG&fq6VdTMDnwlO~JKFxXRIJ^ry^j8~iXpt^a#GU;;-fU3Tc zSov}hcJk^24R_t}+KV^AfyYWycP8z~iBSY+C&@tdGG(&YOqiUB-UIpe>(r=D-X{4z zm*CtPYvCsQSW;);3a#ALs&fwm5mW#1uP38^Widg1FLt zGZ{d?;HA6#WNg%fT$VM0MLjz3k1mF~O+ke!^V52@ICpVG*h@kIbC69C8+lApG7DG9dK> zW?qKmAAcCh^PC26%$3AdAzOHrYChO@?oxQas1<9?SAi`*(?C$ibNu6QE-y3R2Z|$6 zvWb5?_A!!%kC&!kd}kwXjj{~xejfrQ+~<)SOX9${79qIsrZUOuE5TOgvn`iEKRjjH z;AoXsAW-|k8~<|cfB0`njM9E_0qA~$*Y8u(|{4LAMkGFeJ06+ zT0r6ZIY^Gq2H_fW@Pn0N@QJ$xX*0>LNm{oa2ON0^jt8$L%%M40A@CxMGe1M-yf6n= zqltrwdcUwV>M7B3|3gdho$}5sc9=C(dh+LeV>+P+rak`zS+jGGYc4 zeN+t_PfR0Ue$K*wE)ysmYfZkM9tN2E0K=}JPM(9^HHg|sm zvPYY%m)`%tdzm9cN?!NXEO;&jrvGvx+4r|$*Y<;OBw-pooc{m|w+(=!3l>n1j%j3{ z$pE783|RZ#G1F?5$2!`jpdI6LyGUn zS($jU{9Ypv-lIwl`5S>_<|5eECq>@6c|oIpve16sLCkHr3YEUEg&iyFn7QW&BeJd# zUOEv3uV&a0jdyd<&X=9=@`yZ1?sx&Cui7AG|2ioBO%CZ8oJZ!B!AN6QIokK&0dmW~ zizat|Mu&fYKsHU?Q<<+J{`F zw`V3%-|}!e$s|yzt(kPs`l*iQsziF+BbM_2TSIH&7SOu^GpW-=C)pw0P28j7u=c`z z#QUo{7MlzK)JTH3W-jNAhsu%%e#c1qt|a2N@)&V^_=5aM>|>P1rO3s^qN1 z>CG0xqz)4erht4^m7w|t8>s&Xq8o&jY3@)Gl_Wt_F4uzAy|1G-v+vTK4(F)GhMm;9 z{1cQB6lJd&ENAbDhOu7uhuJ*^2UxA*^X!_)c=lIp44VoAr}EVetlCBc_J`8FGAcUAY?#+qhV3z@6R%I0Zp1F8AIS zYPqJCmRCB_1^fNUyi5&PVy%n{Y{JlzoCbK+@i=+xltJd6m`laZ8sjZ%HKFja8!$q5 z2*&*L!Vj%u!KKnNQkb%Z&hFpvKZ?#fs^{;I;|-EhL{f=Dib{hDectz~6e-FqzV@bN z6QWca8ZwKLs3=Vpsn31h29lP&Ln34(DU|hl|NiK&I_GmvpL5Uab)VyLF}BAU_+G0+ zhvOab4)+6=p1;t(?gq4?+~6`fR_rUa*jVS-O139tNGu!ANe(QS$F>{)WfQi&W4hJMeAIsa@ zj^nE6<9dXK3nO0S6Yu;?^7G{fQg$?-)Tb>a6TbfwM!NqMBG%3$zJ8P}Y;zsy7h4|cGKhOz9$-oui32QB!&nvc6a7cq3;W5k-gMps-9e)DEM@_91i> zY``w76|ncZQX3gV!KVM45LTT#U&7@?t*TESyp%D*e-lH|W)^^;s;#I~S^zDhFj&Pz zqkKXvG*T|%*0#?`D^TKDUVZtuPlLE>{YBL3O(!zBCbWBEIQ=#=iCzmjNi)uzp?nah zmz^r;Jg-8USAC44uN__YONDyZFruB`Ab1Tu%TC(;Vg1hgvgw9p>~7P3C{F1{ap4S3 zHN$xJ#sEIW+=oA5tGITJD>u=l+|#@p_VP#Ieo})u7o?EquR6&v7Yk~AX*^vSq)xM* zeIkC&JILe3`9h%ITAQ;0^}>Qx){^%9vRF}C!KN=;io~~$IJ>xp*{)K>fl@gW zOQoO&m+-Kq47&$^MmmeZ$nhi5-Wt#Jo;pigwz~^VZjW$u(GisAmGY|3?D)FoRD|)E*;1i|~<{S5! z@O|5V;eO63W;I`#en<$TL%h@Ir#C?LqYCEV)Y3JY-~CRx&D9Aw3TxCG;e-AH;|7S92Yus0@!3YPVB?r%o0Eo|_By|DH)JZVF8MSS#DwZ#o3Sqew1)0G9{f5!dwx zi&xyhl?PX$cj+)R9!0{)t{$G5P1vT{o9~{Y#(VYB;s5sb=e`T}VM1UmG5)@QGVRlJ ztVTACpL~&?si>f8TdQeG#x1%kK}z)|gwY^5E9w)GP5NEZmENvh16lEV?rnM>T1!u( z>{c$y+&Zx3OJ9C0-ibfnvVxNWd#-|UeBTWfererHtSv9af|*VT**?X_c;iY^p8bue zJK52;=J|AZ)@(XAc04`5p@Yb~9V8cW-w0oEl%+Wg#MaFg2zO40fAtysJDrA8)|I#( zu?VdlqcGE{i@j@k%lfnl@EBo*;9I_Mf43ILJxAd3w|o|BlTu=;lOUw4f0EptmRREE zXUUq>)x=RAFO9YwjkBj`;>p6(nCVf4!~a^aYi}>U)$}ta4tflS)3s2Tb_o`=9CZU8 zLPO^n^z0S5_Ba_nE=G^*){NwD7DhAA1wAP<*+a`tT%wMTuT#h6#Wdh%9er7Im*&)h zmi*137DnDQv`&EzZE+)u!mJQv-iXL2z4)A8viybJ8$6qvjRVHd&>FA6$8;F-x0u3v zp0VYo4@UCAqZ?4;aSfR}L*XHFh}ox{l_V6cB5Sl2sb=H^8s|BKeptPPem-D9-Il*6 zJ-^N+%PbUy?y@{)VY(AuW;3B#y&iV%$I!U?82oRpz?pD)FwJ^)u78;H$}2&#BzFV5 zHK!joKhwv$-OJ%qHXVJ}s$pu%BSsS^v7E5kY~J_NtoW4~YP=S~>aP!UEhBJXT!rYx zd_<~lJNn1>Kw;{At& zqT2a9X*}jl3!X~p@6tQ8D7&8CK3zwv_dlXv>#Jy`@kM&YCYIVn>(IQcjiN7X1C`Wf z9MIL|A)}1A!G8mI`q*~FYj+{^l{TN~Hk{wQ>c)rHIrF>a=3I#m;+t(NVR0$|yL;5L z75&ahKQ9n>#bu@B?u+5{am+N@wC_Kvv2G5Xw`d?u(JmwE`cA}K@wsGo?jvUBm&)3X z$zjubXB-St!t}8dVKQF_uVyb}%e=~^wYFmgy;-*=dZfH%brHj`;mjK3ema4sb?5P3 zKN_FbZ-vq)KiDrjj0g7781*F!+K(DgfBQYU4!ncz%Vw<9@5QHneu4ax-|?pQ8>+3( zin*?{7+sWz30gte8vY;7EZmQZ$99PJIfOkI%Mm&|8?zOku`TO+(g>4d^vvC7G(G-3 zjhNFwZ`nPhd&+Oow^rr!dP6+5ZkUjE)nt6equzkf|$rl9}ZS%x<|XHflX$6GGeAet89m)gGq&=@;8n z8qdt-o!HxicQ*aUamkG%dCWFR1wS;mVR}t1w8p$b#Gado8uAd%>xAHZ#EDb!<5AU8A^bDw+R`E&!6 zpLNCTRbhz8SO*8yOUT{!4ae1OV0G&a$&ZepRtEQ}e||G94SGOxT^i`M(KqOd)Li;| z-XR)nKbl5AIY9dK=?&kWYJ8!x4ga*Bh?#L4KCQSHm-fC4Ve=jwa!SY8!|!qVhaw;B zWWiUz9?Tu1-eBk80-THvJ|QI zyZt1jMoIE6b`gSQ1bptLieBSxGW{Kkm{G+_>CjJ<>FmxgNlLO6thAUUVnKo+`}iOW z_|FzR<2crZeZc$Ezo4Aog0=v$k8$!eGT!d>opOJqedFwe19ARKw9*eIF7+5tVL;GYoJbha5I`so)-IC|c`pTSbQxeZnJ7kh>LQka%^K|yZuxUQzM~}w6 zhUd&m?J|36V#-YWJ!kXXD_KaJBa*)yK*xmhXg`nz{egD`)leH6$IsKwy?3aFSv}p= ztAG~Y%ciD3PEh03E9kLGD?0SmEYhN-AkOeAe0HD6&D?Z({XThqla%6J!89!BUCqin zgIUhKpS+a80yyY=A)c#V}Hm`p9Fj5HAn^EZ$aL&g5uwI1a1>u-Y0KZy|i zWUcUa#U^3v+$2V?n;{`*KjO2xpt-RXSNFG|XUR(#=3l^uL1FLp-CTK^z}b}g=F|3c9oS?23jLIiVUNENd!tgzx_2L!gq*p?F1>n*yJDU>BtxCg z*6PWBN1Q~-0tHxRjgzQ(?`7QZyHlWvOQ5$M!~c z+AJ2aJvPE!$p(k-j1g>>?;`OhG|0l|JBf$mUJ~J$DhxPVER0M@U?ER!VZ1yX89TZ` z+%uii?_mN?M~qc}<}+3rACH(-J-D-rEoJ(bf@LhRrXkJkV z?`N;^bk;|7x{ctyvc_`n{|xxkyOw;fv@ehABgc)SzoF?XgH_aN^!+m#?q;i5qrw=$ zzhRcZZ$wL0KDfxf9r(Zuhs=P{f3fIwLez5}rXg2$6j7Tvh58nrp_lvA(Pdsz+I{%~ zZT3G;7y6u}Y4-zYNumWEBHc&ccGWXc(uYd70ngM>=X$IEpk-+(1{{<@N`&a=Xe5_t zd>xIs+wbB3In-j=VSL$OMebu=f|kIYNViO5H@v?J(?Z4W-ub`eM-K-|6c*6!UslkR z0&$Lv`$Wu?y~x~7aaXLZ! zuTGmJY;cnfUOf*-7XC#28ZADpd?+8EsmHgse8q+__i%alJtze~gL(D<{?2|FUwUa2 z-_X;JD_k+<0e?pDkH?4e?$LiSqP_tZKO^ArcP8ry3YVmkPlCgs7Qs2}pN+h1Ad9S2 zgGIJ4;$8*feg7QT-Z}*1$zf#Bt7&xcj8uB`P7QTAmPL1sIYS59$I;x2zVt|#2mR&s zmG~lFxc;;+vZlQf?=OA%*%!b8-xS>3;{YA~eZq6E4C%~hE_I`tFkLFgCw?EyZ}u9) zAH37$iN3YCm%kXFq%l(e9#hDQqt}VjYHix8F`eFOm`$IH`NX3)m8lmCB_HKug`5*w zShn38nqRC@==GRcEh-Rl#_c15PceDZ-9)B-t|o8Xi^wID{lswGEFrY%3sc!%fjLdu z9O?!<*wlot=+&D)b*w~9$z^QVdKP|%p5btMA3lWX^4RY-yfDd{o2?wpxBb%NzoTXO zor5(vT6i4I*JUw^#FxBT6e9$Rn$D+Bm4Zy<<%w)@D>HoJi2Lh<5O!lT4A!4QOUgbr zgAb)mUXj%1_Dy<5?E&5TJB?1oN&0+!5LN7)N%;?R`lDeVv3zFEh877CgI)AnURr+d0U4s!7z(+<-;px_s`3P@oZt`RZ_tj9~{etg6F6Y| zQ;{??c>^u!97?z9>r+$J)5KuPDzjG`J8y&3#I0}GwFMs`_y+=F)eEKq6%U2sI*v#rno#I&1UW7(VX+b zrl)r#*LzHo9v{6NiqAHcbXM;dhSVE~IhoTE`w0tixaA$b7c248S>D{Q-)6pLl^JhS z{fQ}i2e8_@`DEL_V+{!JMG`8@7jq~{Yu<_h{ng%~A?ZR9KdH!wo zC%D(t<3(H{HlB*afgVwK`6~!>3bx{L*>0S=y#ZG{-4XrwBYRt;E4lBdN=^rjAkR#D z(VzbvpaZ{V(wu!)Y392mdcVk#cHgz4Bd_$M20s<)_22#J-#dY%`0o#4SD>dbH|7Z& z?_?!h8dxvHeechLr-iWiwp7U34&-B+C-T?&{yfuT3vWF#lIQ#1K$K3E^z~0yQYbIZ z%%dr!n7EVDJ~* zi+YZS6LZk}_zt|(vVc{lJRWP^W%J|Jm_dApWbf!>LhXrMNre3lvVPP`N_aX&RwaGr ze3>>$*U@Xo1sXKNh_;?mqHBY6so#-1q*OtdtdmO0-Bfa{-IY{+qC@_U3Hx$5Xm-Zrl%{$^Xq(X2F* z+i;LvedTuib|Wx+>h^zAQJI*#KvkljvMD1&(U3 z*wBAfHs?025t6@J3O_Se3aVaJ>_J}I zeiJtR(ibMA_^}$}FKo(oZ9w-h_}wSax;2z5+fL!{I!$@;_uf3j^B$J9#NmTi1@cG8 z@F!Q)xw@SZKQqgK%LHrluPb`J>g5T;3>%JKfdgjlVBo?`C7B zJ!G12JTj{!T;D**D?P>fY-(VeYW51-R|E^D_iKcIf33*E{-&hR`x5D~>Nl||?j%k& zva~Jp4mnkIkfc8{B3GME2xBUY$y>EJazBxg@i}?KQgs^<9(4&p8|~QW#o8EG-VaVo ztT0{dIJ}8yM6-q-Ke>4Xzi+3?+dqlgT~j&E72HPAQ#rnKi6-Cmdjd~0a^TiJD~ zW4`L6GJi6$9D6QBpmNV%m}dE+&uvd+Y8%04mogMeN8x(rG(_!9Wk)hpsKL1;x3b|PKu3Ju=y3qiJx z=v;Xl_9a!=neq?w9uDF6qRse}9+P>Zs=%Y;t@u|3LwJ66^tpkVxF@Dy95on4Ql zu`Zy8hrs5~9#r}I!f$d2rZ)E_-^(}AU5B!$V#*Er#65@JJvW`+NRx=%o-y>hnkw~w zZAv#fno);_p|onZA(ff?j+`HQitPD!lK7vC6&5{OuE?!PDgDnD0@FkXhyEd+9NxqvZLNU!s4UXu|WaP2)d< zr|`R{$ML?`jd=%E;+HF~!}9leRE>#3hs9x(?u-lD(g5rB)^Ci21Y-)1Iuw2D{C8k&+Gnu&0=D`6+h4eh9nW zPY|sB8688LVO^lW6Ag!RyYm*@S7{2LH^W)n^Gx{hog?^P>t5VzPd%>2-otX)Yj`&0 z3N-4oVD)nkHmF44?b=*uhg}uIm%GwYQR(#EixT?r-$~lu&yiLg97`{(pFqpUjHRas zET9=$fpnJaF1mNwe>8fjHr@20jD$VTAYU#x5kHSfSI*wiLtJ(OW^Y&KY3Hr@)e;+S zK17wT9ex2%5BcEt?Q<}1tAege=Q_fP}GxUxt$a2x+%H(QtWDp zJd$-6OGt8Q1nFI~h2-4WNUl|RlC@V)lfZySBw>Lv{VA_XKbqLCV(i9U}Q2I^;^TT+2hHE+eVPc!%*A zKD6IpKAbA^>vD3ue9%9v5IdFa-bHv@k%gkk;fU2bjD3~`Q0-lTaC1o4J~F3syV7WO z(G|M6DUL3 z1@#TWrSYch*{Q)|{-Xrmcg4Jf=qn8hQ09BGZX&?;6cSP%BPeJ9uLvK?<-UyOl{y++ z^|RRZTpo-!Z*MU3rzc7hZw?a#V|}t@>0FW?I-PV^NXUp^_GH+s-9(}75PAJX)K%uc zCCjfU(dL0lbfxi2;&jK4440`Es7Q;>Yh= zME)p3JUfaVlT#7wTno$ZfviQMMO_+?QvVYL)TJew1~v?!?dN}z5|3eYhUQed$t{HT ziaJVF{Sw6A9igRJZZy`ti|qaQ+4^#lHW@>5va3Y+o|zuCgWlE4zg4ukHy6 zh5LlXQ>IGY-=AfVS~Xy;F%K?2iRk=Qiav>T*d5h~?XOy)e!UAZ6NmD)siXK`jlul< z4jumcf+}A+{{v()KA^Gl56lO5V&m?1lt1r4y7fDhUX|g;2K+){fQw+((~2(3J50U* zCeZzE<7tcDaq_LZk*u~EKx39oq2K$2QkCHc>45A+>UQ)fb$jSUi@H^4);AgY)TM>U z?|MK|9z>E``}&gBm>M=L`xJUSEP(pw)9C!Z6n7p5z~EU2?v5ME>zzk&#E<49cWLo= zQS#jJWHy|wJ@7Zk6hqDpM&OcfY>n?i7TX*q*=&*|8PH3KoQVt~V-~F^A#Gk{%FDIn z)*T;GD2*Z$2M3T{nr0+mz9sqE&z79=HY4=|N`%1S_iQdYN3aJ=r!iss$B;G=UZC0SH>&F7VR*fZ+)3C>MB$-`*Z(lvCI zeh{6z={TLvj?))4YiZsrQ|cL_B<>*J$qp({-->%xcW5Yins`7M9zG1ed+bNX?}Mln zHetrnDEQ^RhxY?je(}{Hp889Nhem7h6A}$>aN#Q^L}sCT%`vPj41(p2C8$-_LPO~t z=_0vJf!=tjh&PB|7Mo~1#@wFpNS$Z(4;9j>`ao9Ac_ z;Vb$K;#Um25c~Qoj!by~iP2TK^GEQiD8(y-VmvxsikXX^VYo>vk^`jd;E^AsX3Hwt z+dhn*`=Cz~v5gFSx0__8T_%N*`jp47r=}lw(WQeA(fmH*J@&pc-SS+Q`XBg4-tBB6 z*%}IT-=yCp+NnakZyFMt@FLdV+YC#USHrS-IkbOzLFIWlzHa{nzb&%-N_#(U=cmLM zn<;az`(JRRBpExVrN9;m*!Ff2vQFvY(^grwhuT)m?k?x}slrS^6D+=sk=aT8WdZbFtueI)+`ng6AEV zz`kF^)QWt}JXL|%qt|e7l!x?*#b2^z??O7$--lLj?Itole}wx=Q^=g$6mn*h7G2V@ zjH(oS(dE?<^mordda#Gsi_GmL-(QN{EuY)u!4M7Vn%9@^yzrbDzn@D^|JlJN-5rfA z#n})_ozbt+1qlNZFsCjLR%fr{>e$~n{z8uX`787Jr@z3V-)VSxBx1pr7{pC+f{UdN zrj9pZ8ME@F<7L#@g6j2xwMvLkS)C?ipHn1)$!o!Bh8oe9Dv;LD9fBba2s-4OEPerPQ7baIFA)?_1j-34gxizf_ z-17)l%9Wtmxd^XH#nqH!&{~;*=hM#PXKOvXd9{>EQESHVIO+;g&KKk5tjjp+8HcYw;;?*oH0pXU zLfWK2Y&v=fC!FrVVM;9aI&UL~j_c4216O)ocL*JHHAoncd|deYIG!|_zb0KD7SN>6 ztEu$|Z#qvcnl9}WDA#HyRh}Qow~`DJyj+jgjJ2Y-yWW!NHyEi^KQEbfViex&In0Jj zbumGIHe;{y#12X}j2CBOqf8Gz+Hf%U@7IH?A5!7nQ!+6{b_z}`i^Tp!H~2mY!}P{| z@Rw_6^>1xZ+L*~~TDZh~gN@CEk)=XL=`^8M+)(OS zxKLB)?+?F{)XZL*cb6f_QgNblCk%rIG zLLAYmLS{oT%w}GOv@H!jD~{pY`zXZE4o6e(1WcTmhpf%w-Yez_f*+W()u~rV{A+dE zbDcJIKB`Wd^s57f`niF7$?80981*hQ{YAP}{F9B(e8HvaO#S z^&LNvhMnw7uZz0kl0H7f+Z|aNc(28O=KDMDs-a}Rm13f9~rv$?FRB)(LSa#c9*bn!#g3t+l*DOlE;fB@sMBMiJ_IEA7!k{|MPhQ+lxUMdFd?lGZJ91 zB@3!^YeYujG1#BjkL8t>Ibf>hsy-1Sh>RD3NGJs9B;mja<3|5XFhCYg> z7;n7*Mbj5y*pDMv_aG5B>n@@_^#W?euH4rDh&e?o^Wzyw-DR7R2sPLcJU#_e!|DIt&+AO#{~Tz z?j_k1Zb_Z443hL_5v=X^cXsrjqh72pU}dv}aAjDTNxJWk9^Mrr*) zxXSy1Z(9w?*uB`jQ{)fMufdZ1W*nc>pRdrqgOjywg1gmSQukVsj`WyF#fsug{bg?rI+eFW~O=!)m$2pcgrp1rHkkq$o_#x9laV69rgLfVHb z!u}Dx$(m~yh4?Nt_F3Iga>M_CMEUq=$=bqE0($(DE*hU%GUq~q)KdO0OaJ(n*;56W ze*D0cAFW~^ZN4+3C^Za|)e-fP`C?8a5ckhSq5MD~GXAwbWJVRuqK1SH` zS9tQiKUcng7)_ysEE8EYez6VY#p8EUqZ3cDR-+%j*jN$rvW1kZOr#m34Cq2n2Wr~K zjehL)hphR0gH+$SMdWLG(o=TMw9k$)bp3oex?R*T%j^0O$17vWGBbTr@KuupOd#aS z$oYcO%brXjMh&H?!mU|#@Y`IDV6$@w88!*|;UY67^&0w*XE2!a1cw)&f@Z*4M2=X4 zjPM(5H;%9~jsDDBS(f$KEQcK$m)XZTS0#x7Hp0uxw}ch3`Zo2e&#<@iZN!x455k4nk@(Z&99CYf#@U^{xa&|kp1q&5+dUJ>wyDjeVDnU> z;Cuuh9$&(y$C|9jLHvhyE7KWQ^r**FeHxWV>7(VEwEXvFVzB=#3G7=*T2l3?g0Cap zvs9PHZ)qao34!G4=gDNpqNN1I3&@31AL3UukL*z@6}-#zrQ2**LVPv{tKBo9d+#Ln zCQQZIZ4R)jx`<5&Z@}&TT})NEiu?V|RZd6g!ipK2NVms=@;#IZfLn3y)0{*00fJMH9cX z^=cjLMtmD%tD+fO7|xX6-eGwG?X3Fc7&t3LVdLjGnE$s6YVTdJOq{<9zgj>D+l+I1 zCvfe^Q_R{nh$|JH6wibKS-0f{G1PfZV)p%&etyVt*}6aWuB;FST9uR97XzuSMt^$! zj~R{rK8E(qxlV4Ug%Ph`XUTq58QM}hl{#;=rYHX?&|@9BIo>@<%xOKWK^6A!?q!M25j)oV*$#CSNQGK=fphIgg;>8X!_T3j! zyRVM;`3%IT!6k?|I*0wt9Y(Hx{6t3XRimyad(+?BtZ3Dn!8FzC6j3Y*Ady9>WOQp^ zI@)Ie4aza5!@m6?>LF*zSSvje`fm?uR^3WW-UN|Cvovz`gca$LlP^dca@m&hqiDH# z2q8W}c=dTYt`^JSNKG()NQ;p9_cGpx-9PYO? zhj8EkJM>wH1s`gaE}j!5mEJqR)URG*lMfGJu4y78==g9Z_tTJNh#8zhxUoq0Aa-}| z4d(r;iEaGT&XVd<*h`ONEO^%=*4w9@WiOfx-SgXF8FCy+qmLoEc@|oK>0@%ic(}M3 z;RLqAZ%jJwe0hiPMSc0LLUl1`et=v%(n^$6%82EKerW#?i!tl8rG^uY$-EuE$w_q` z>J_L%2kn>85|h64^zJCK?qv)yI$1(me-Eb{bXJMp!#Jv>-kZK`I88=u8ASr(0tpWL zktYrjWXte)(z$RXneTl_@S3EC#H6bZ(w^(RB)xLQH!mOF zVG@(|tpBicOsAhUi&}Y3`qwR)sYph#L2s|I*+crHS-&6Z2ixF;R}1t1X~)i6m$H9- zbg)WfIyP6QAwYTrrN8#Wv2g`1y_to;{hRP*$ZBNShho~IZ1LW68|Oxfxj*sUfk*SS zN#Xl)a@pq=`PD}%JyDko-xIZLv%vsT?emnZoimu;h|{3+caEa({B`LK&ph&FYXb2f z`I_uE(5L$jZJ-aHPp9jg2U6ivE&0(rlVqp_5tXu2#CS#$3AvI@#sy6!HEx>1wf?`^ ztDq#r85H2nlyF=xnSnIJ_w1Xw$Yn_`$Dh!8_&n~!j}B2M7eY|ge-X+&ud~n|6|7Rt z1hQiG>0zrY8jN})bLI?bP+F}->8rj_a&Cg;lR`Yp81BVdjHj`kos(E@{6CwPj$SOX zs!6)1s~>x{SqT@!&f(*J&d~Jm5OP{8(6q#l+H@pH3UE*>8auSa9@^u<_}M!E ziRt&m9=n)BHTorTE^9C>H5zSIGY}kB%#;s=N(KTcw39Ve7G zW(YBdZG@GF<(T&81#HX$XXewjo$0wYPY8_ikxnU|Bz5VXD^)c8$HE6u@PMTl|7kvI zisZ3&N9p_Mi%ARUf#tsmn|Fp}$Xk&<-eJV#dlk{mc|bz#ipVMx zGjbr|t5kYxEvkJ}(H?dJb(igMv*8I-{G^Dm*dV;V`5bF6e}{YQFN`BMapLiQs2CYR zzV|t{dgE=Tc49D!$J!&>$qp-nZnMB`3FEKjMhc=2C%n87DRCXSo~<@_U`PH~OF!$* zlfKB(6wC+RDG815A&vbLDV;p>1N%elFf`N`<9ly{+I|gWetN?`AF;rTxjS*SB}L2# z7NN)ZbLido6jtp{L?5esEN!X6*VUECbFab#_v_ey;Uk*9{KJ5g|6#MI1!=sWMn1Hz zCBr5!!QS4Y{v>-}I@snMQCXow&z0HHynDtpd$Gs%^ODa5wCg={ozBWvv|$gAcFWQFVxX)0ZZkM-HusuqT< zSS`#r?#m{3|6+R#6OfVi6^)C3pzg#QM4vB#qy2uU$}UFIfI+B@>|hTKhvJ~C4qm@h zL9+A#3sQeCoxb$S#MYIAY?g(um(C4vkiN}2E=f1j5z5do@tsnRG$$^CMXpSg#+nf6 ze}=W}rkpy4yq*afI}YE){V@Ay4=lPo8?WVq#q9nmSd=E>-O*Sy4NbxPLFE`^Sc$!( z9^i}eRgBwu4`I7IFy@yszjpj0PX2HaKDsR+UO!C8`OVf?`rsAb2zprVW=pDfwvznp z(R6yyaN4%UfVw#kr8e&G$eHRgBIi1g-t0M(u9)Ocn}4sSjraT0K$(1U+w3@Tye}nf z7rMwg`=8`ZlgRY+2_Z?JYlX|xS2Hd0AFAVRV07mbvkeMiwmX7Y&EZA3wxk>x-yY&y z(@U6_T}L0g6S#HN2Z1Jfa6DGW(u0P<;g%ih`!9fCD*EM<vL%^^7faG=TS~fD ze3M>VdPAy|+8|xj(Vq#I1omxtB&#aj$0Q$9S@6(N2<))I&QlKP96ko2{hQe&pRq96 zw;#?=(~(%iu&*)`gZ4#Z!QF5S9uX%pS@N+kunG;1<%n2#3stlQM_>O%;FdR7);Eg< z%^x7L{Rb2M(ADhWj7LbbNJrM6KEkcq^TgbJ2>o(z7&YFaK})yvqgxW6lCr_+Byf2P z>0UgBR;^x5?bmFf+hslB71-Hu5;xPf;%s0zQhM*j!s@B`C2GjG zI^+=6R}r&!|6)CU1+#$lv)Cc`;nG`KLv3Vq4_;okW0DZQa=-97?~`OUGqagG>WuJ;~8yhIMuHgSJ`;oF3xquZFe zoHMC={+DFw=+oHKeW_)%8eQ+$i@tquhg62VCJrsS^kJ79y;Z)AZa?8pe`jmatcWK> zrM`yftF@DNhP|k9LkHQj{T8t`3??6v>`7u?mhkV60~^$TEsJ|u!g|d4%tlt|p)eyA zQI~H*wQm8AHlIgJVgOe53`3@<_tZN@W8=ypdu{q_*GcS^)|WgyH(I(x?Sb@L&2%=~_EkwzXqi;=b^(h_Q^P5|@cT$b7xmq@byl z?3`3ju18!V&LQu~xSzvl?4jAzeBomHYtTgcJV060mL3wB!FNQ)Tq{{}uZu)2{!EIq z3dmw5Px9!VCAt4;xA0l>qjW%R61%qlBU8I4gST4*Y$%CG@{TyHF58PGjv||A%O33h z9tW4PsZhDJ11fYpa&&&NLld%CtKW0B^?Mtu{?@`QCZA$+duEp8h0YOPo~af5=Hv>K zHY!LOcSW<0&jz3~r5|3@oM5zQDl{Wh!#KZ?Z@ zGjLYaMy?wi!2LR3^!^xunX}Wy9LZ%^_*J57Uoj5nXF_$V6jw)DLgRv~P`0F9@Hu26 zZ1#^rRFv39Sw9^MnGuoC_(EzdhtPFqiuA{v7E<%Ag6O~7N#2RsX2ZOHr1hBiC-^7;xne5j3tR?1PU#s>1yG@3L;7?6e&NkZ`%JE`dgPqu!v zlwIp6W(!tSvaRZq(6lE63pAHu{?&0PE1!rNE;C_h?S~HM|F9?PKKpRQU(6`xNLy`+ zOD43 zKEK2I|1`mGr|s~pI)NRp4m)A3(v^8W6?x?ZiC=|ZIcjrq(CBxG-VE5h0HWi0q&;CxcIL(&Moyt=b11>9iD<6 z%I5g8O9$io+Ct~xVqAFC$%5RgrFlaNq93p<ABCdWFbw7jr;#u)4cd!O!nGs~_r-44uf=B}?+m>4GscfN zM}elF5h~4fh3#MVBStqHqg+P`lO`6EG7(zaxpxq?(RoP%E1Ss`>n}v|jFTQmnn-=! zD4KF-Ce0{XOYPo{r46~_+Z~!mi1w>9r2i)p>2TVBs( z>y$sR(h1`G2>bgWDq07#>TRK7nu0r$$GFPVaC@E?g8G=@tcC>MzF}~g>VqVuQWjel z&Hn8EDE<5Dl5}8MFShQ~Kw+i3ico11C3$ZndR*t135#4@1n)_1l0|F$S>wWB=Aw3( z>AN0ep6((?O#GgNiSxvMRuG1~J&gfL`>{yV6Yn#8vC=#UpZ+F6HvK#-+>_DRnu{Gv za$vIJ5_aw6m|IORkN3P~D-kc23t$V!y7OB?NymLgdVqwe$ku9OHNJ0xYVQbxo2|IfSo;=Vff zoO{ms{l4GN2TyHw!p~e+fS%FMGIwL02w8te*s|KS`$4)RV5*-{i?SIc_fPBYWQFlcecNd^Q1$Hrhh_r*-gRbQV~14g-tlRwP9Dc?aWH9T-x{u7kUOmO#$D`?z1F>GLOg#Mk3VV3&> z*pE+3+1W#}%*LdXC9IeaHKDt~ecork*mx3lZZ^$f|I+et#lfyGzJHJq6M)tfHN-J(bOPFToQ8;Zx8Q`wL%8|2 z3wka|icGHd!k_Cs;8|4yNBDD6t*)EgK0ZXsqEtAxTaA0V z`4vgsl1X%{N0PM8lk`z|3Oiq|3~_U|z@_R{V77Z7B=0>1mKK*_h0-gqICdE(&x?e6 z%M!r7BpoW-4g;3W1&bzAsQiA4g^044Sg*)hY}`3?XtX3YeO!mLKV8I8AMfChFhw%m z{~Jz_I)-2D&O%9?3cQ z8E}$8p4VKvpY?JFSXX)+YyY_ug=WPdN4;3#iSgO!z@NKF(Jcw%HMlf}r z<5d~cNdJVFvU90kU_BTJ!2#y*q;?WS$+%`E`B|b`Q9j&%nJ9V1E34n4wY)4exHSgIFZ2=(EJqjlY>sDerZ- zmnaOUXGxQcEH`!tC-4%rTknp1&YMZk=3sG#?XW!0JiCsv z0UF9C&$-D~@%#iVKZRY+_)7iHBvCv4C^miCQ?`$1QtGW8X7BEGuvI}rtl{xw*cCht zqP=wBdh;ZxJYo*vZhoK}#LrrFyy56Ocld(WfN|k25bEc`H?KpmM=}N`j3@x#*@bXm zO9ec;UI(%2bIsG>LdM8*nvts9V81q(G_1~@ z^SMVp>c*35k6_ZL^dAlz)q~8kGniw*EGQo*4rzZzLg$GPaH&h>&$UXp6kiO{@76$j z?ICzkcLplB(@_1s9Hh6of}gcGG_2Qy2p?(qq&OZdu9`r1ye7ma*uk*{+Ay%eiD}n% zP@jaKG+3*I6}4Hx`E9d6!_E*ED$j(QO=BS5K?Vd;@}NF$ke!=n27xJSV6J&6ge$~D zXMiuLH7|ypdOP7gjsxkhmEcB%utfPJ>`*@kyK~CH@L&n7TyqjCdkW#rhSSjPd>xFR zlt8mU7Uo=DPh*95km{a5CVsn-{aN-FOWrt1em8cK35&jwyj?#?;VA`9=g}0-Lr#M; z(V4)dyIFAumn`AN2-a|44S9co!UXPnk^;visdE1gPUId*OLM`iZ;_`z3rXO{eWWr= zos20@!ABsErVqNXm!^A}%~%;Is}F!2#YA`l2Y3hHN>~#z3pyunhA-)1@N{l49L!%0 znTKY>_Qe71%E|5z5X9{q|OGrrp1t5F&`}a zXTwRENzk=l8+;`NfQsjU)tp5jVY&l`Hy?zXpUOcp`Y1#@hrpeQ%VCFE0JzOK3~WIG z%o1jR<-!B7{c;3c!ZDz!z7O=`*1;cnZy0xf2Sl3ljxYfiG5A`eIz5y-4 znH+P(GdE~{jF7ci-)0Jnbx#2mYxp#@2!E? zsT;sx?H=&sXA8g5_CWWhC@4R?8?-M60t?&;Y5&6ENf)0Zd5{5i*AIc?q+}3gM?>|6 zNZ9l(8e-q@$5JW*HUuO<*uN;)TNwqH`}V@rh!Cim5(SEpiO^Y@20BlZAp2Jo*em&f zP2*zFO@~5x)c}jMvm(@kmhW+jpDLR$8i32irlix{Ig864A;E+ z7a2MAGnrb{&bvr2ksF6H$?5QIj*bc9#{tgbsuQ)+X1t^H^Y$oLb$u0 zgKzH_!Li0rI98Mlr#>fvj7%hqjSJ$LWBZ`(N)$*0MuUc59Efd8 zddd;f3&(>o{=g1RtYqi6-KE~eZs>844^EwO20OGp!==GmF}Z1l_uRy#I;b(&9LdJR^z%1Z|8HlMz#G<5< z?6f;ef{$Dv*E)a{$lV|xemo#98IMWZm&YU^_c?K{?IQm@c|oYwZK6E0o|LXVLHx?{ z$o+FkWKWeJN$YVUHY;pN`5sGhD_n*6`n|>vbdKYDuBkZO!UKyfjYS{sIMdtV?ey#Y zB>F>eS?GK{l8*J0WHOUIS?uBz7Gzz;mP{C6&ofma)nh8$^wx*WC;Bk4WCobTEP_GD z^>9XN7wF5!fltgqSpPO1o=;DNCfmJGGd&QFF5Uw-0l?*XWqoCsC<$)GBh0E55w z0EZUB%KGWxuzNa$7TJPClmH~OC&8BJQJ}BU!B+OwvDK)Mt)Fw8wR>M<5~+{aAN?k_ zqdbnCTO`fo=dGrm8A@}>nsXdyukgn0Q6sQTeFRFq@BgZ-uQps?{L^K@@!t{YNW!hU@wxiOwD zdzVySv-$!;^B3da7ZdRlr9!MaoPlqM1>o3~?)dq`ZTPoE3f_3E3};QM#e1vyZ1q0C zWuL3Ces~TZwvWNzUPj=*68zl0mBxgIqr6 zNMxGVl1#mwkd>X%#P#bL+>|qy=l+0OefB}PSqKc>+XLYz!a#L(1f<-H1jEcY zF!1Nk)4q5ZKES`ftL}i&=*XcG^h>@1!R{{z?U;=HtpaggMma9r$*}R0n|RWj z7kJ94?^q>3j95(=P2xu?k$+xBq~B-(=}%lu3WnAarxPp5UB_iaoUS4Nt@bB!W7m)= z+c?susYW6f5976mUgN!9ckzY=`FQP52OLv126s*vgIlkS!CThGqLs5F>0cv#)>yoV zt>Bp_hMtM6BDjj(n(&DIT=bHuXt%Mysn^)D-gXux_m6E|sRBp#=)jZ?O_&@zf#>Zh z!qdiaa4CfMzu2ilzl0n_M!jWkRv%>^$8Fh(`LC(6!~?2rI+h)*S&&^_^A*=t9s4lsr+Jeu$7|?{}?Zcb4*u@p>O-8vx8*wgfx%)Z=_{cK*utlgYhiM$z;7^%lD znp?BRtqv?L%Y+%IU!@Z_1PeEgVK$!xwzTQrcd9F4#i~zY_F$Dd`}6P!z0)Nncy%Wg ziGMqTPG1-hTrf$b722}wxr!c>_$kLO*J012S$Njm1)wvDua}cw`Bm53Nl_kNzDL*or0zdiK?z7PA$2 z>fbE9y)_57FOS71^rLb9^aHs6OA0=^Dh{{ZiNq$`qj5p=K72mFANS03#!7nTIN_@n z_Ozdbk3X4)&)w0&-g`&jwe96dY>ynenqY_)jx9i^>W-p@_ipHif4sm>Tk=v|_$NVW za|R0B*M&S+Nn`sxDp*ab7n$xri1;W_zUx89z89x%a*l}P^r85R!)R$#Bg$@@g3?be zr_Mx?9UQh|d1VSrEF+HQ?en0E#>%j_<)N(SOc6Wyu8f5xWbsMoWOiC1hOy6{?ABP0 z<&HuuE!LlXlS*R8Rnys#JF)C2f1VCM7|*t@2YU9zcG_OV(QSJE^rre6n*U9g-kz;1 z960qwa5y&+8He6PzojZrZSEyO{^nBQ)&D@~c;=#@Qtt%%{_Q&&J8d*R`c4A>`1KH- z&5S`@!FZ(S(kYnmSC@aITZx>zqmY>O0rcL%1$8^?pg-3qptLqu6gF3Y%4Spu#5QMI z?^mm+cPDEF4zu-9*s}y=P+Ec(4CkTg&CV!A(*z~*#bS#aW3Xzi8y<9z##%HSXTJ}? z&9=+&qzK-D(`AaIJp-}E;2B&x{|#QwvnfmmAK~cz`Pk>88-71|6keEn4c(g2fFhJ1 zAmQRVz6qs)T8=KIx`8?@Z{8+`r2W{}Z}zP3vpMV2)Mou9)@)z&cD5!tm)(6;!=e@e zJG%NJ8@Z~S1=y#sxhq$(rr9S4VRszGRtlVaqCq<_a`&7CR!6YZg^i;xNI@4H%g_CPHv>P zQ{D(0>MVsvill^9cBksJtgZ=;=D!luUpgRoaeRcp>|d%N<6#i8uP;U#V>YASCHHKa zMrhNXycjyvwVnokN)Se6cL@AzQc?PgE|ifm5{t=-W801q*s}FIGTQMIrHg$)pW}Pc znhOfpIo1dp)vv*wS_kmcghD*7tsI;2p!=+KN%-nHKkTBj2JfD;3Y(Ae#t&Q^aN8DL zeC*L56e4yOow~meW%;xT=H^`yn5VuFoN_7>c>dC%rmfQK*iT>PH9LbfPDo)xfs2_N zr^qJWRAIj^Su=SF2R82}}S0ggyyhpi9M%QT+Kj6s(+qqT=+BmrtocHRoxa_7gwh)mM>% zb91i>Tx*@tLX9kRvwS}K)?$s$UpGY(H{S`guC5Z?HK-Pro%N-q2mEN+yiL^2cr|qj zjG*{#It`agrGBe?X;GXJZIqi#9e-aH+I63^ek1o+5UWcClGAjAp6cdwaI7mG9Ni~u z89$z0Ugkj?`pc+vbrtQli=fJu#`JlrBt7NeNt*;`=!{cMw02@Al@Iwv?eJ?_#Hwi4 zOrE2*#J%1j{hYw}agxB$`M6C)Q=stfV(Wrh$01bMVT?8Ab2wtEHl8II zfm`Q&Mf6xNy5cB_m31Vs#TEs8J>C@Gskg;l>%6hl)&)5AxDkH6Whyr8)5hb2)p6vG zUuf;jn@DzE7c!3ifpW+8qD1jFWNpb5-p^$o z>eg)BK|>ZdU7dY6s?TQ1+OTWsj?ALTnLRsf%bLranc3xqtf`4(pX~Km-fbn8;j7Ae zvj5P_b1&1wd7;!i+Lo@Ix1Y+XgwukX2dUn@O;q)9zHstZZ|g#jmx7afwUGFdUO}$p zT|s*73_*#S3Oe^T7nxXfA=M~ZtXM3K$HypRMLSd6wP6l^y}}+hKQYFS_zrbK`V`*b zqJ}NoCu6A@5;&~UzI{atB|p*%w@&(4qLIq{D4}9Q`>BD8CY64^Uij#SvS9MTXM#zjU!Z+Y0#&xF zphLGjQNyyO=+zS?WcgDXxwEOWLD)hRS2Aw=oi~9Wy5&nywQ`dFDcBaWGap8Y!qJ>%c6NR@=CJT$7`qm{K(6iZ+ z=u_Y5^-);z@t?5cztMEd-aMhxCspqwpbyCj8B#4XMpk?UD?8JY@1S zyN#Gny&g-c(PZB(M=-^OCur-z{Zva@i7NluFTA||oKWj*gV15Ow($9vM>fCW%>@hM z9tj>^8IK%~J+*0_E=~>fJZRqK<@C?lAYortzrf^+CGt3{j{=uJ6Zq?G5`^383jHps zQ-xWVNcL^oT+KWo=DWU#Z(&(^>16rV-f=nY%q4mM{QQEfAyhlL=Pvm_r z-8a4=ztHcR}^ei8!hYfewrk-H|Q)Ik-w_ z;W&w^bb3;mwrqOKsg0_Qdri%DjAC+8axCte3NwvXX0Li=nblc6Cb~M66p9z zQ?%nC&vzf4MgK&F)A83V=n^w4di`sq&@MjA=JBgyLGWuY!K;eWdOPu}!kNA1^nCsz z8sI;LPT5o|d}VAU=;)9_`!m*}^yGL{P#1}m&fBA#3VQ?vH(prlIV207D^EggXB|-d zFDI0)a~SEk9YT+bBT++Q7SjJ5gYG_vL~|D%Koy?^NThOJ5IjFkkdS=crt`e9zI@gMjn1=}YPNnEr%vZF0c^~>7r-a95nqdELqwxDWIXr*%Z?yPe zJGxsx0l(4F!cP~B#@j~RLEforkl2+U7gsN}q3wr`QupE*y8qsCDj%Ikr%%bCETxi? zA49a|l^XloV#c;!G%p!gxTfF)k>$ug($}FC+ z;B7)yR}{^5FZ#>S2FF;hgM`RwoD|1BmEz2MJeX9#0(yGq5{O!BMq;*gk{laF_~)ze}7(?ujTB$;{Y`-1Vq19;azN!F=T$NbhN3FoMkVD;-+ z@Y7owjGVN{@U~Ve8D9W8WxdqIL=#V6z|Sp)*W=K_8;@9>UL&W#ow515*3A z9W+!w;+^0BBW@w_^!I0ZfKXi`wMh?&j#U#)cMY=Y6NSl{uH=nl6`nrM2}EL(NY(rR z(D*hJx67sy-|XY?^}{f=oaRE7_$M)G-zZ4?F3-M=J4`N?cd@8D1yFEt5LZ@=CY5Wo ziQfAKu)$FteAm?wi@Fi);gozVB|DN_=*eX6^DT(v)q6OifU> z{lK-0!^tZ{9gvqXhS-Ud$(XB_I3eeX;8QAMcH3&0C}siCw-|x(>Xo2s){nntZN?RD zieP7zhTHpA;Qxl~;r>HIQXOA}M0=J%;^Uckev&mgJO3o>l=_c-K5_$B6gS~%fyb%$ z!h3XD6Q5VNoIo3OUa)`(QskFGGXCIMg-kLE8G77J%_>4kQ1nR_=oNcHPeWd%md+jMj#0(s>H3+jzPC z8Q|T$L)Ya<)w&+MdjA6`|CviF=4q2gpGJ5xG>wST){~^=J7L`B`FQxi0wTQW3?`R~ zkxcqLveD}rOOIL#a?*a}-#%MXxGxB*Uaw>ZzGKJ*@pYu&h$Mt-ABDx;Yl*kaA`<>p z4kC;ufC6nqZtu@xF;ka%pJ^3r*`Loe!Q}?_6}_9whmCknfnNdH zIP6W1e&BoUn#W0Vpd;~&E`sXrGNc`wi1)@l!RwFEnl znEQ!tp3VE~om9XkS{%mx;oX4k$H?q+Bbg{A6JiqFNQFT)Gw>|LhwJCUz`SMn3pFQF z6&>u_X=OOMHH6g7QiS!JPqH^~nFMXV0H3C2vZH@D6L0q-b~WZ0i{^7e^R5^243PuG zdpH5>B?g0AW)||jh2YoUN-_%7QlE*D@Ve_Y_Sv39EnQ^*b=s4*$&oDM+7q^Ah74Bz zSOia9(#e@q%gL2h{P8>fmCYZTNp^2*z^!$C0{0u1aQa;j8?n)mT*f+1e-I?ybxF)VE3#=3zkmOik9~LP5z`Yg{B>dt*Ca>didE~0 z;?DDQ-^iKpE-;0OZeX&|w;Q*=_Xmwzmyp6|o^fLSk16O!Azj5L$ONyOO^b#!iZoM0P ziOnX-u4qUnms;ftrU^$4aco7BAB>yG`slkJf8VWk6hPvW*;l2vk%R(#PP-|Torym*f}sB zT$auyPp5>E*Bg>S6sLkd?9LqX&l^Y{Y4gH`2@?0`Xr> zvFdCa(w-5;Qg%IMi6O?gBfXHR&uGMvcGG8FSKxRi{Uj{=+y)u#n~IYQF5Y{>J!wDCh|K}6d5?#kmywBBzzfNS8lgY-cZDfVl zRah_~87@sRC$6g;iQ;b#zj}O%jTyfS5<P<@ zjO<0bQ55>~>Ob^1GnMrf{U&o>#DY*Y4x$a}{X+wGQUJL4*1ASK@^W4^z+Xcs3_s6kK+h zN!+ESNaC4Nh*gzEmB}9DzSK`v@Z~Uv~r)swPc>~`6Mg>1k z)r6%d-0>QoHT>8*f$sWwjIGO6$6pK|zmIJ%nokOYShQ^vg=vwxXPcZa#|}`@676ejd-A55{D3AQ9}~`*?EZP{E(mUPfbZpp7{` zIc^#IX4=m?K(z7aQAY&dh7GXY(n2h5EMiJ?udxG93h|b@Zfv>KnSFOnW3}3<}h>J&Pym?Pa8U4|B%O zctuX9(Ek2ncF6ex3#$DlSpPa4R~@>8Cx;o+cL&Vi-cT*wtA7}$J)O#O)%5VL`R+JG z;}MgYIfKSF4C49YGMQM{95y*NjQ(xO7pfhP!CFf-SfF+*wfMY~KAV?F+uk@}tzD^D z!62Qr$6scw)Cb$-Sq?nESptW z!_-6gx%&v5#dm2Isie^bJU8eu*bA~B|6}fh)3HzTGqhS$1L=$~L`@&x+r;0pMM~8H z_++OxzBf0Pp5N1m+aDcfkNNwf%!45;X7^HfwXubjSpLFAn^Vy1$u&$Tp@80WSuZr$ z#0ggH_NT!*h3L-;j%qGki!3dw=odXxR+*xL-||csW4v5Y`E(Nvm0i#N-i=_#fAJn# z{x=Xf2-v7u7DCrW3-QHiBeAXhMK+Xsm`bxpi4v${zfIE+evm3eT zY`p9fq1X~voN{v#?tDC#wpIqy0`m#@*}tiXlTxPhp9~7h+Vq5jGHcncS^l^To?vu8 zkIDI5W?FkP(LS|S{Jms3{lm0WE-(5K|N3~QMeeVwTeiMn6!`xYGh$8)7JQ7cb zBmucQOxyRy3J)@d{@TVOmkl2TWocuv=P4t$d)@_x&qdQojvm-8ry2PwUSrF}ide4L zBb#E+bbRH88vZ1RW(JA-nTL)tm0q8N^g>kWiga^UequfK?^%zQWk=&>O8fCpw-yt% zA7?JtjF|FVaTLNIQ@X?j9k1TN_KIionvli#J2k~wA-3%1uT~Z@R7wY=w&I>|1E|cp zPq=Ny2pIm$j$O0!!=(|5i1(bd)ov`^QP zas#vICI20Gw?qsMeESKVTvODA3&>=SJvv*B82k2XE( zy7G6ZCG4!Ad^5j?XDg`5;w=KHQ$s>0-j&Sh$+G?`hJLt+1Xf}9=-vIt17KCIGDt9SY8KVqp+LumEg z*YxjQEjAe!qEQj+(W{cPR4XToJ};k#ZKNe|#V;9lYe0*gT+mIg9==#F6Lm=VK->U_ zD5&GIKOd<-*t4KN{nW8*M}5!e8?@Uqi@KK;p`fSJg>!#QU?21u9o(~!dE5s&+1vo{ zovMKLwBDzQbIsVFWWsLD?H4}K-iZzcR@XJU3y}Qju_#htZ{twYPqSA3q6zWM^_jC3 z=%Pu6SmO6syxlQa7_DrMs#L}Z2M^VtA)jJ&`cx(D{(2PUZXb=c+&9rfL0^T&>qg)W z!?V%xhB%b16oH4LU!ZqC|I!YT3Og6CgTh)?@@@_}yu)x9`MD3F@m7avSd^*Ix$P*r z=2%WURV2_-J#XBas*dORq@i59DjJ%+k;S*GB40N%^jElus`;d%g(JV!PvK`+Hrb}E z?XWeP`r!yQ*S$}x?0(b2-A4tVv_>P>Cmbsr5yAx0d?4oMEL3sZOwiOhhVG45Wb5bN z6rQupKnZiZ(3hAJq5F!xv^LzBrNz|HGh#ci^EoH%P_%{4jv2w?*>U6;oGv(je*)7g ztESgE{vLo*(Cz*cNOPo);Lya`!btr`*411$owK=(N~pA;OFy5Yb&p@s_1E}W+rK>e zNM<^IQznV`h5FHZT05z<{565pt_#8y1-Gbc!Yt~VdkUE-kHaVb6{1UHD+Eg|cA~l! zUP7_I8P*TKwWH;mW(v00S|K9|d35>WF;txDWV3AE47!9)Mk?Z_$ir@%Ao56`FgRcn znrC<&mFgGM&KtGVapQ9I@Nu4Ha;BmiZPX(`hMhm( zB`hvaKp{((u$gZJtXf=3xJKn7`lntZXi(i#?>9-GZpk@E*W`|120j*S=$1Kcoc|Hs z8AwF0)iRJo$nuK;{nbrpFn&^?e5Bl1ZiB1YWA}ObNNPK22 z6)RdybFrs&9S0Lp0G&`c`q4iWAMT7ddCI-$;W@6gp*si)#dl2=g1f9JuP4CZXL)Pm!d^6fb!Yg{^=2V)*Hf zGC$;@&EHM%inv0=#p@#dCFjs`jbP-Qy%?7?TVe;j0D;#4DYX0fa~Q_;BC>*{O;dp=64^m=Rc$%Zyul_ z(QRAFOpj#Zee4jKzWgkSWDkhrfo2kV1<1>R8)WsH+r+H*Hi72z1V5-DI-f5SpRdPA z$B}d*zRHU@7$}hA&p+b(cH41y{;<&FPYHWp^PVj*0C87K|3ekVUS-msr0 zMeJmUCmv@CO3&E`(1-KNKd4t~C$_rcN|qXy5&fjoWMB7HqVo1Bsi?>xYtNh{C6~u= z?OUgF*NxY4l_F0LMcQ-DFo)A$XV2BWnaxEcnsVlr3f!R;W4LWEMsgQtJSTzjhsfXN z$>j93G345b0zAfClG*9=u>K<73m^Re#=O4+=cnEQ`uQg)-V_&k_YQ)G+aK`z zC@Gq8d9)~R)_eH%=_IthDuVj9Gw`G@4cae;gLm3m_@to=F~RC^=k{b+qpS+wFUdg$ z-zzVwf5;qFidYaNvE&fOB4qxu_{-KX-o_aAtTlkY7pFpD@CkOhKZAX76^CpcEBLEq z4nv));F$58Ewg{apkbKp)g54dqeejd)(P0pv+2(w?$g!+DCNcrvz zJI1Yrg6)f;$z=}ASTYBqpM^q$LmqsVR{}{LYqHg@f~;%2PO`%riMVVRY3-Nh+_cAY zTTLW6gCH#~qj^0Szv%#X@JAXq;q5+7W~nzf!Xc2mzHJYe^K~)zw$X%p6(h-&iQbdD z!fT{+V=~#lc?uc-<|r;aD$jQJC_wOVDLAc@5v6@J5Y?5KhztT0MQ>E(L~atAqBZA@ zM6#;-q74%Iq8TG~MQMI2q8(EIfs5cA#4bpH68TL~tuGC4J}_3Y%7@t~t-(=~p5PtA zd>nafFB%^ailB*W)@zR->vCm2u+<0IdWiR5`HTiayqAm*&w_Dk_ zmgg+?8iq?355Oj|WKdZh3SRPC!1rn}_{e6!)}Q$>usjp`S_>fH%vq3YF9VM?2Vr`| zAy_{noo7<4^ZVQ+$8D_^%Ys|aTWc}Fcy8tk{8WY>xY5OLKyL>6#n&vKv-=d3yeOB zs_7u^*qB8=*3^)K+|#7@LIJUzpH3c|Rg(Ftx5l8vSt9;9mDs(QOH=|4h=y_z z?i39p=Q+m0-TzKAEh9ykbZjRy4xfZE9+x4{;ST7bx3E6;I~XP?h>lV%5sT0lUCz@M z`EQsgaz7z0l3OGux^?Csc-;98r%p+T2LDKjW|T;YN`}6}qp=sjtRsWxS|)?_3cjlu zZ3O3kT_PXiO}U+$mUBy85I1~RjGOjSk24Ni%^5mHa`$9nxj!m}+_zxh&K6wfCM-D1 zO}C8WhNISTViOZNpGPsAW~LA46r#gLt!^NFI#J~2YK|zj)Z;hfRpF8EVKC=DL%IKW zQP(m~WUaSJBzJ1BC`4?nNL4sjq;b+#RK0SeNNQleNW4BwRCs5dXuGYs=;t|QQG@)mFZ?`J{9{?qWZzZ>NAB}GT> zN$~53v7-1UT~Su`B#~Q}k?7}^X(BoGsUl}p6VXrpT6;Was>me9P_$;Vj!5-@gy_VQ zuOQ>t4gUBxi2ptd@9fhcUuqvbdg=;)`W;#MsJEotVgaZ1B#_%2u!P%qLW{F_#c}_A zjONl=78kiHhdZfU$elk>!&O`ra=*8iaNbX%xywEN+{9@IxP9_boS&;B7a=i7x-JEh zTZ?;eIGu$3N{eaJpa$%gse_dx6-7p)1){>`>qLkD?h@@d9V$wT-XN;kHD5Gi{8G^( z*exoIh!kmaJ4AyT&iuM&k|-C(i@w+XhT@Qu;A_1TRxP~3?5n@vT^nPGhE@Zq$mu5G zzW0f1;%)L{+)pAkZ477UqsPr?B%I+{-g!{3%7q5@k*xoUNbv<6Q+-%vJ1N;J|}MO0^EEYh+!5slkzB08{LUv%%{6w$@b zX`)$!(?m*>r-@qRjYIzAbRh&XgEtghb zz-27l%ORdo@G{Du%jNmzpQLTL29p%l<;^@V zdh;Ksv%C(!cy`8noDB63qo8NeE;w}B171DRgH5~R+0I3LKdOHkXLWZCH;d=yPxLq7 z#G=%>r6tR_%@vW{53x8dyg#1PYd*|5TOH>V&y;ctHb-%b!d$q$o-W)@IZsY0kZ@D8 zx=H_7S<;j|N+_I?i{gvbY4k}k$R5gu%X@i_{|7zMlJWCJ3#-?O{9Sj58gF=u3|lx+ z=+|kYv)D@XqiTid%6)&)``Y;;Vm(1Ll=T}j*S`mc86voj*FnN_ZCZV7G|8SmpCpSH zlD&62NS5JG@;^ss8VKe0zH!QyC`6W2NF{5F3iI4&B9xL6NeU%tS4q2&>=H?KQVB^! zNaoyUD4|WhCGAT4PLhP`_x%6w%?mSU&U5D6*Y){0zK|gG1_LloPKwO6ok;BDInq2} zLpn_b#57Bf)IE|R7xW*(kS*t6ZQ(%({52MwE-mDOp57Hr4t_!H`F` z{f_sJj%Fn;#;nYOV`EOwVOul>>~zC)Ccn{?nQbs*vyTZ_M4JuUJ#;1;zEq#tc8p>( zWc!h(Ji?;oh~H;qVdAiZc)Y_E2S!Qp9hq`^Gwvvzxp_JyKOITlYc3`_{jKmCs(p#y^_Q z{9W~#M#XHl%i4`KX*n^u0VDR}q#R4#^AayBU&Yxv38*S{o{nsaH7r&k5a z=bys>bb`)`FTkbz1l>nUWXCQ;A|FY}xd9ZZaOEvGPNNE(#~+#t|f>>I9iN zwTR3*pGwv(^(M!JGsuT?R^$VnO9K5zlc~Z;Skcob2C|?b_Rbx{#$Uv3 zW`pQaZpiLOEn+9VHZk*WJJ=1Sm2B{@C3|&Lk7*w=VR{P|uw}m-SYntl%UC{)83n&U zg|Zer`;qUsy^%#Fvp~VrbKRVG&3@nyg)l6+7y5?{CE4e`LEPu>Ft|j4l&&%$kDaW@ zLO&rnA25|{o}@#^3R+W;$;OT+iwY250LQ=%awX5f@r$=K{!f{nwg z@XV|Gc&l2Do#6KigGmqnHka|0i}Dn?t)SgbiY7t>nX>F*KIR5ACssKNGe%{wCnXl?6<_?X4y<>&*% z#FJ+^A228CzwF5CLqWv8`w-EbvX2yJ1{2NjW8`sqF6rERl*~swN`vvp%YOn*&)=aBsHOpS)!yaZi zFdu5h=5I7)_Znxjmoo`7j4@$yhm_gWHGfgfzY|#i?>}pcL*FfL>73chmOdvK=kO^I z!dEoGRSllUo%sn){e1$BtzRIY|9w=bHI_^?*C!g!#}Rp+Szt3ojxfhA2-ydV^%(fSY0> zX6zwS8E}l)KT0RTb0dgy%3|VG24vyXg=E4TLacp;l5l;0c&CxUna|$OqqrKDEk43YXU)V&}Z#+s|GK``Dw6mFHca+k}diI%zK(!ZbQp!jSSs*f$j zwVB2E`617s8P$kc>Yveisw}I%qQtC{HQDONN%8qljrAM6YO zrvOzFXt{(OII@C7T`?kQ)3nLUbO*xLuOZ7=zNG%gZW8$U2-$BDM+dygJWHd$>kqV4rCBJ`D5!Oc-@{YPacQLVr%Gi6yzfi#P4lw(d$WtmR%PyAEx2!HF=qm;A=O%gBS ztJPW9oD+!FGbiB5ke_s~u`Ko|M$tbr7sExKGa&dZB-<*4#KBaXEO;}4Y`0!a)-PR0 zR;K!rIdQ8=wcCF3#5{?(_{EcqziWuMtTlPHejdp&Tg;!YW{`laXJCHJ4%8Rir6P;osm!FFj$t>a zsjxeBU(qQ>jL|-Y_^vGtcj898YyFtA=(nOx`Ex~q!y-ACsujRpx&&jsJc2nV8^OMy z6z)GLgN%)r!Q;S1i20NQdTWy)VrBw_*T;bE(N%D|?GM+OJ(sIkx3VUo$(5RaljSo3 z2k>QPK9)I`;_U}#`88gG_7|J5D!v`nKYqlCZSS!>=m)Af$gsb%3e4X}fvu8OV4C9; z*fodtxtM^-Ntpe?N&ARc8C!~4Oq75Dmp}+#e-gZ(0Z~1-X1mw$JETGHzox^ z+)4?Ow%C9y_17ZmK@#Lyn+B0`wIq8A?a3RLMI>e6agBHWm~eyHMWR{vis;$jv+zB{pm}~X=1o;( z)kDaJmJ5@=T$&lYh|E>jS3Ah%m_e6mA-wjw@C~W1y`*_HOGJb=_#O zGEDOn6^iC^4ePC-^X_r@JuL_FEP`O;MK`#2D-^z@B!KJDP<~qG4KM1KK+A-AAiqN& zO7|3VVcu7*cI>vKfBJH$(7T>m#cN@=n?KJ0OGU4h`M50ZBu=o;M>}&dChX+dkoiyX z%$s&h@_K~_U-sZar9mv7@)d6;x8tdWkI`Pb4OJdBBe$`hzbm|it_M%!jt9w@(s~^C z1n?ZfaYyjo4o{S}5uo-pX}o!VJGcDtDKPexA(O9On~&eMF1sy)`BQ z=LF=+ITzAEc9U)K`-psoD{-9to&8Hx{4sxmXLY$Mj^qFExL2n10$nkzSVz<@{r3zyftI=rA&bb8CNd$MZ(R zp)CaN7}&x_ubFUg=yVt_w1bXG7I3Qi7PshCsNnXo3!;`)_eDPAC(@H&9?*mVJ2Ytw z!MY7G=&&Uo&m1_89&gWJ$mVifwo-(%G;g4r*HgSy@DZ23dyh7mPkE=`O+2M`9YH1V$oO={Y+F~&5=U!ZO!Urb|+l40;`e;c(ALlAK z2eae9L6F~lux`wP-iZk3VtV-wzXEyYt4#zy%*fiYE~KY$8>uT?O?H@Ak+ofh#Cokc zi5PB9ju>l@p~G*175IYlfjOXj!3he^>w>Mr8*a>*LtIJS&nuF5Q)!pp8Z7p$!{2xB z;V~78$ua5J|2`U5#pR&e^$1C4 za+VTpxZ%$|nRA2F|M8UjItVYV(-%1y)mFZ1cXs}sSOzmqpU_W_D7)WDg!X>g;j1)hKE<8!8bq0&H& zZ0=MiRZE2=BziHKy;MMkJe@=w12oC7oXMm{W(u*17zF$8nGi8$HdH8aAbnsfWPGv# zsjtI9EhmrjRJRmZ)UBu2Z+hTa-r16R>k{r7c>%AiPQkHnMi5|bgaO=a( zJPUC#`mJAyYw{Q3F`*_VeUZf8JuNgvSVZT)EEYxYisu@d>bMi6iSs*jmfM%?&V6_q z$$3p#%vIM<;N*9o;5xFixp{}PxJS>oadVpXFpL75Ta1qH$5! zvMvx^bjaetgf(OAxkk(CXqEb%90B z6p@hH(a)V+4KA3_uI_ArUwC+dAmom33^>$3LR0Y<1D{m^>bGlf>d*LvU{*KbPC9hf~Yd za7L3f*7l6S)oeUg-O|CE`r0_tMHxQ?X<gtMn~g40;o>ry zQ+12JPp+c2zv8J>Rso%9lR?osiS}w0(eRs;+HLwlt2g&k>0Phry5*l}m#Pd-&l!*M z9h30u^(m-%b0$u|WR1SM^U+FrF1B8?!2eFpK)()C9N9Sz`y4bd%~ch5d5y#+Gvx4t z%zfJWsDy?uJ4I8TB+wByd?xJhT3UH(6g7W6igpKn7dd9`68+xqE|Sn6WwE;CvdHDf z0;=~`ifgx&g>1dGV7v7Q6dP>@5A9eeqeXD{RvE;YHh{VZ!oTC!pySwkxH9D#u$Tuh zW_cUvoM-@@&8={0>NV(}lma`d4uRAW9oT;9J~wjk9ygdUmviv8<@}^H1XXgOR3qs) z{WSJBW&Zkjq+SJkp*sZHpSOw);yPt;H!DY_)lsIUhI{}S;~?)J5mk(hmFSc zucL8ktt9T#xk>%cHc%(+92&mZk!t*ZpLrSIeJP|?lk@5HFKg&DQ+Mh+KZf3#lt)#v zPSRwhAUY>MoyJ|eL=A@K)A_$rsE{~hX}d6~XU&7k)u9-}j6 z`qP~}=Wf#OWI9j~K-F!H=_ornT4FzwO6(m)KO~Og%Hsz)lkan3;ag{@_|FbjK3okm zH^x9k`Uxm$KM4&FazM7Y0HPJeFztH-7+tsqQ~uPzdiRU)w&o(NpPvU>(J^pxg9l{D zPJy?Dqu`dyXYTflV6NuFFwXpMwP4}T@S4~$BGEI+%XD8S&xKc0KpFWdC~snjO41YY zyvsDa954yfUYp{_E3bOlv&8qJ)Iv$B%yT+*G7-TzD$)aOpc*mRmW*-Y5<*OxR^%xThMOHNV+d7moA7&r9Iw9 zXrp}=tvg#r6X!S6s?)b=-hL64I8#X{FS$Vb&X>?Aw>+BF9YeQNAHl+3lW5K_C;CXWTXdu|g<3k@qtENR>6xhK)VuLA)!Fur-g={n2Fv>Cz2DdP z9=H=#I@}|wd=pIXCmf-dQsQVYJW3EJa+T~ro0DEgqGPB%P>qa|m; zX#{snG;YhSny=4(iE5cWjoBJWWn?~xE_NIey;-71#cvepkj^O4hw}}B*ViSu1LtMv z%H(DA^($T4Fiel8toc+`>B%!8uN4oDmhm~ z)iaZ?6iSv@X67ji@=o}3uX~HRiu)Hhon|`@eur^0cMStO{xj$fQG}DP)xl&?3lh3b zVNAkw(BOGYD{mRXzty&&D#5{HFEvmQH*mW3t=y(tLqPsB<(e+|b4}Yua3#M5T%Sg- z;N65|QNvg{YW7;1_WB#p?aQXoty0_Q%XPxoZJ#!ccG-JSqh=BH>sdwh zU8?B3$IWz``wLnt&Y@E!1vO+|qQEM)M4(@-M-!vdMdyQ+IGw--E2mj)BCpbNQUAi< z0u2*gF1){nTdI6g)V!9^j^S6i#>b7E(@`Bx&2kZ~oOV{U_5E9sg!%)ENiU~x?aO0m z(Tv4(iVyJ5lVzNld$1sX*m!)lU<|eT>n6%Gj^p$r-iX{6sEL;FIs4VUOX!QNet}%} zLuxVM4)>_-EVuslKhcB|FOC_!vMSoB0xi4zx&AX@0-KSg0&|ysLEpY9U=cP;koeT4 zCU|wW$X8JUI)})ElI4mk;koYwVb7$f-bZE7E3l;gITr-gFII9(olQmG4y6mO^kz_> zC$?N<FEr477dYV8ZK8p6R5H9r6UCUw5 zYy|Vv_E5RzUMpi?k-%bWnaHiBgnQRhAu3K!61Z5Xa#yv@xT3ZQPUXU4s{JmJ(|(k} z^(PsK5@n*eiH}!twbM(u32i?GGMBOiblw1GKiY(vYCRPROj0=I%>pho$l9vqjUf#w zp#l}pMO>KHA3A=?dqKOSSP(;XDGhg^PxmWQUB5fr^ypvo$ecJDxky4Vcnqzo<5W=Y zwluDGPUn1#mkEr!zR*7FPgYAWR|~W)XV4OUU*6)RNIf2^P-U~jw9lYYklT@A_1Hp@ zGmkT*`cm0~H&UCpeCvZehl2>>+uqTu#>Yk1D|$IPdJV17oWn_uSRiogsSyMol&AHo z+o^YUB-f=p1t;N-6_?Z8xxcv`d=Akb-DttM9`16HFEm}qr^)Zv z;fh8vY`4%C^*b)4^AGc!z)}L=uJ&-cTFp4A^8~o*JQuYuKaNsUd$|o>#e(v*ap<#M zf%lDk;fl%^;>`^cQF*%ybdF3%+f#;utePp@U#-13i=V~j`i{IZbi^F|u*VpzawVYo zzk}#xP%oIcW(HiS7K%#W%s~?eV@Qu8&^E^vk9)5I+gBg0zPfwSw3+T4H-5c9S;38K z`k^9PkfS}uq6xuIO5{0D(XXP4D0v)`g;Zf&&Z^$@{|IYWUd9z;Jz=;Mq+PE?tJ z^IlB@S8pMPKC!;CV!jk@>Dh~;*DZ(sh3hdxqf792wH8N~N8#uN#?Y)E&Ud}m(|8jC zkLS!pZApK=({dRkcX`l=X_ImOzXk|RT8LkcyyK>Z76>XURluvp6CNrxB=!M)$W+c#H1S6&lE%RZ$0i}>veQAIRYc!9pn5! z0Uhd}(r+7napQ0icXUxO+Lb1XrcSM*$Cf_i6fYjbh)dw_QeG4QJ3o%A(^kNu@w2(rO}$nX_)HYsDMwn4>oEPE5T2zk zLxS}-GmVKHnJ1@@0S9xjw(mMbz0b#Qv1yRAQV+8WB5Cp01kNMH5H~I?fWG7wbas-Y zg9AS}{~Hk^DX%jaj8@#EjX5AJlw{h*Z@7CedpP3YgGXNRJtM<=xIZEpS`Iw3@(*5v zU&nm~t$A|n&)y;AW@kB-F!;y!<0@g)x!JUwgkoB`9Eq$Qi(v`Q*y$lnN_-{R0S$ff z)$BUYw>~Sfnt2{HCWhlFTVs%16@;=C`JfXKhg}Ev3)-VNkSq7%dwzvf5c`{(mV6dKs~>x zhDeDxpP*sP{P_#et~y+!Gxry6f7A*O_GV#qMX{j&{ROJ( z{`ku2S~-j>N`%(_7x<~w4Z2?@3r=V8>up^v7}g!9-%EJj{J1d4kxk^f7X?wR!FJBr z^{r@<$~5quHv^OYa|C0xSAzG>p-?Vw!g6*^COyZ(I0cIkd{`kvzuHD~OGpqvEug z%v2>3EN_cyMMv zttn_P7>N>RYN5fZgI<5lac)!ppv}@KvbxQLC4&hz`A#6@V>o&~QvpkH7&TiXrbF}U zVNyjWSO07!yVf2>l4h>QKWZ8*tX@bG`MbKk+hz)q6}55FUPo}~9uJ>W6j@eJJ8ZgD zNX;GPnM?CR()!hojp4t7K6TdADC8l!Ch8EYg__Ji@iDbo{}D3B9Km5`tvJ_yGqG~o z&4y-u1B0!H;ORySX6E2dCLNYyKkgso>`r|KfBtpP*GZH8CQ+P{t0Ws-p-Zk=JK)j@ zfI(v(!>yiE*gvI`@5_i`)eTivj#{shYND-oUpcS5H`Rew&&$W;;wQG ztN(~GKfWA(+%6WF>>j|QuWo~?gB9mE@++?0p-#l%vzSG#Dd`dU;O2u5(5FL#$QZTb zL9=U+uU?0}4^kKjR7{qzvICdjCqT}ds~Eyg?Z!+As?@++I_&VO=>? z`VOGW1`*u!_(c5;uG3A~KR5$dPUM<1leX;E6W`lU)P)k&RP$Ns&_bj?*c*G<6-D^L>l*;a0czr zt>Db(N@n02kZNdz@oTMF`tI*AOSuXo4P^+opdK@DuS#AI5mti$9 zws0ux+WQ(}|GveAYkqQ8 zp%VvJCmzF;^lY%4u7vk@U#3#m)8NoIRYu0GBL`Bvn9&7Q;?JJq#G3;0PJIUpSn(b1 z+_(+ulMb;g_hj-bF@}BCpFr|leW=aB^ElSS1HvA(LhYd=EHfvBd~&{q^E?=?>NX|H zIm_9{0p6!ls)Slg?O|i5G20>OMkGwFSl9RmpnbOqqjwI6FWMHcD_xKIhB=XYv9np^ z`lqma@+Iz%Sd2Cg(r8~y7@4yBI@`1I1W~EiK>0NW>|jtV3H>sQoj7(JYG#{&+{rnp z!CXNqU=m#4w15S^C?|DYZp?E2aF)T{A+M%~v6Kt7xLC7-RNg+t7F85M(Up;8-qzzR zVOAJ9xWx!9qz^NjFEvCfX&2ixu7&EcMI@{ti+Oisk=y*=&5jBnyZkiS0nr1I{Kjrb zYJG_}9jmxtBY-CvvY0kzEE>w~B0qh4nS+9waQ2)UHY!*Z0z7sT|NPCo^eB{W>Ik5(Met2?r36oabN!I^2kA+Rr;r^-f*`%acrg6lRoc*lH z8Z+8pgv~!ZZ#skcyq(Wne=3m0sipY7?=W;io9mbN!67C3Xf@%dTEc81{9dmzzXw)>OP`HvMy=`HAbHZ0b5UQN9d=dY@t3yYE88PjTW@F`T$kCisYVLCu4^O=T ze`l;m`CLnUXRk^W9~{774Q)1RO&vKs`6DArB_y;yvnKNEVYaitk@y4_!Gas3)uwbd8L^=9C?~?z;m`oPXz6Ab;w3IK-a{unxcjplrvBvE|(?PpJQ9`m1rla z{TRjU!~CGkdnqaVR>H!&ge1u72TVz-WPiCN;wXMghwIE@c49LU+2tUZ;5wRx{^cDy zd*kSu^EJ3V_7*%FkjHEs4Fg{5(6Py#4&HMhEq50%dEIm_ewiQ1G~3CxB!_{^5-)P) z%mKFZs6535=Sa*z5qqfc6BL%4k_Q{I*mIXPq&U$Ya%S&ipZ_i-(Yv}tO3}+$(8nmE z`e`^T8}}1!A`TL>uVdJXSS9vkX%?}~ie)1|c*5O)`S9}U4$^-?z@q*z zrVx6TjCqj5f}M7dOJ|xeLp>I?{;CjfsZ?C9k&4q2)X1;yvA8;V5K|W?l9^SxtT1~f zndKvfe})n4=6nI!yOTn7bT+eEfkga$1-rQSG;V3lBWRMsmiVsXvff9Kw_7tY~ z)mu-rkJm7tK3C#AMIXuy)L36)5l9a4z`MR?+~O)vo{2J?MGcW8qUvyTzgbyxRUid& zt2bfDJzv-wVU4n-$EYJKfknlQ7}uB#7CZ-`e9=hAtVx5F`=&Cl?0Nh?GMu$)mO(@6 zH|j-WsD%6VD{IGff>bQW>{l6+_IdiS#3hs6>v~B%UYuYp{Z#@sVg_-Fc#1P0-ovcQ zc(T^pk)EWX0D)w!d5xhI^kP$1M9;b)-L06Go8DS@PoK=nKRxVJDm^KFv1% zK2NgVC^1j9WHz>OlrXUG6Z1Yl63b`5BGb%nu`80cIOg_ivi936cH<3&AWS9u?0z%H zuXSW?OdT4Wn87qIog#Tph3wynY_#fQncn8gy_e! zMF#%7ygZxwM`^KwXXd0~&lPGHzn%?B-6cMC>CCM12)du7#MAl`d%Nrx9A8lbjd2rL z^6`3s_xU5l=hinS_u&e8RW=HzDWFXteALgO=tVpsSpR}qwpb} zV{0Y!$(IGkI2Ex4TQ<~@G-T+~=^tEK!n=5aWwNYEVI6wcjUzX*V)4()bky}yCV7*e zVTeL7cc`uvibink{FffMXn&tG-am{TJvfmx73AX(yH0$0;|Oui-_6b(IL~c)pi9o# ztzy@dl*#Y)(|K>qG0sM13yyuf91>T3=FBXl*%)U7!kIqAj~a(?$Y~w&@y<^Sm~jxE z9UDrsr$?ho^%v;fcbm@Je+~CO>Vc#C&tu|pp#6&WmyzJO95`CXW@~sryw3W9G|g0C~Sy@OTWL+?8*q- z>?a`C%2u=8lj9)%&RsAYGnUOvJON#t5qLc0yDvkfG5ok58R}%iCaJ2C_Qi`)Y9G%H z3hV{F<+D&l>oxasL@a)~5H6sJf8j^BDjSL)KxvNvm)(?Sjb|^A!P|S-^Vw1?uICz& zzZ=FfFHB@Y{6^XWpD}sf8`LmINTTwJ*}7x##O*uJQvYVm0`BROEzZ`oaJoF(_;49n zX}F4cUW|aX-cH25F^~O=m`Jn*i7;Y(EIJft{*F5UXVB%AJ^Tzh1cOsd*@7XJzzttOzS-Yqqo;Zi^~3%|THy=JD?Lmy zrDl?my`8LcNef9%e}_45joIY+=g3UIVrHk!uY1=FvTTF2_`|ps5^N+xy@UU<)>M1p z8DS?Y{T|Fr>p>WMSXVr&b|h(h|Ca2ub{5y&Fc7|*cZQCPR1#}7nF@{d%h>#3MfgIP zP4u@MW+gV~_R!iIYXQ}e5CM5lMDB>TS_edNm1CF;>X6d*y9O4 z!*R~%IY`>y#}7X(F}v1=qTZ zdke`162f;ooW)UOicqKjGVk+~5xe-$7nUrR6sucUu=FW>2z2ufan7CrvcT{WWJbw| zjYiH8c1p{O)rJotO;56jc&~yuZ}$jcreZQfFB5j5X}r+()Dm&Fnx4?#bPn6Pd@CFJ z`T?;$cZY>qFC(|o9>Z22F>@(xA~H#uENMj-UzI*2o>o|B>PUw~VNq`1Y(Sg2MlCw?~aC(M<; zKuEZXSZUQT;oE7+Y|B6&x*zT*yB_~zS{C`RvZ9SNnvD~0*;Gp|u9PN!^5n!FS5$=G zxy!6H;4^smtAZSkWmgoUi1PLWtbVyH^z8{HN2Q9H?j9f~C3z;C+h%5!_LekGQV~0y zcuq9!KA?e>92=(OOWbMI*>I6* zTB-?^E=>@d$*Bq-+;3(t)+>^%MHV1^Z#X+SdkINi8o?HHIg{j0mh-*+=!`#P-#D9C^nOwrS;Vq*Jm3Kw$r zB|8y4O?dh7Wbv1YZ%Fv9C`L17$?~TWAbklh%$9XPShNfxoI`^ragtCGaNMn`X+=bh9z| zfjV*ao&oA@2~67h6d9=`$Nc}5(dFWQuudY3rS&(EMBO@ea^6_-WxgEw6cxrQU;$at zr_KHa&mnuRu3@@GZp3ko88aJ{fJb`sh)3{s)=+nkO!7I$Mt+`3wkz2RrbiCqro}R3 z>YaO-Iky3lZaA@xr^V#7)>}6H+g@U;m7nb%#Ep6EbY%xIyIjmvTTasZb*%(_wZuJrO2X9%x0y=%TC(Rq z1sv=}oGh(PT*~>(ecd9Wnq0_yj&=|_rJg;^{sLyA0J3*ZA3MIZgBayaVu4b8N1!8> zBrjBAJGvII^b6NW)6?TjFMSoh_HQHiBn-rJ?@9`bUp0tIAIpk27?=s0-acpc5B6Zy z10F;>wU-H%Ucm6sM1C-Go=tI3f&rGwCo-y8V4pdu^8E_&a>#cJKM^<2ST;)M4Bneo zOpeY;VqRf$VD9WuWZyz%CcJ43j_GSR>Ve z@0+HRC?j2VQ~o$-zetN5?ucNU#)puRE!yldHGy2aZc+H#Qf>;{OI0^IppCLBxpdc` z1uu#wPw(keSb7aBdj6eMOpq00eIa@Ico*H-Mp*ERWhAKFf|a|;lkaD{akwAf-AiA> zY~#dm==67RmOjfG`a6jJ{r^}|lR0TS`w+}W9%iP}31qE#EE7lUC*{*tF|D-mqx9>Q4B+2{q`VX=o(;wvQpO0*X-&B(C z4y1MWDYk}x-n};lBAFY`(&qSbpXSw&v**;r!Mop)4>F!KY4AQ9^>>Ky<&{V_>((cc z&G?yMsymc~+zVnCI(0~el_r@Y+sl-1j~9NCyT?WwJKzMz=OpvYE2i8so=9k`5`%*_ zY|j@{ZhwR$xw>aBOOQVgOTsNkc|!$z_&JoUxh@S)0zz4P=@ydPcoIF;GcmKC!V-3e zQ@FN_Df=8Iw*IfsX2oKbVf2yg@&3(btmJ$7A6Aka6A5wOe~-u+xtn-stB?hpvnLl9 zrHN|6h$$!;k-j&}1z&vp*`Z8I&M!K}V&&dJ%kV+4Y7J(Fg-%3icqz3R{SHN*2(ju* zU~|Y=7F;_B){6JJ@{3~_fD*|b@{?88>H#*sotZ8q+9KehIoPAclQvXB&8q89W3 zjbyaIyI?48&^gTQjI;z;H-q``=Rw7QlkAfGJmPcxo+$RC1#|nEKdb-s^g@ z&>l@u=EXVWz@B(k*_%aFbW>Pb>2C5qU4<2j?s1ZL-@$f|Smw99nvCu`&DMNSC+AX> zz%p8q?}$l}drL*wtXu_GuB)+?Qgg@=TMefF_a*nI;4b8SnaxZ_tRk=cH?mPuD~atM zC1xzt0B6t3ux5ihi>}Neo`;IqDVJDsQ~Nhgdp;Vqb#zIBzc0J-G=v0Q-OMx(5pr-F z-^G~r9Ih?7O1US$vC1!j46UtXcl=KimvwWQde3i8QT`U3y<*M;rGfw3e3AWkY5YjnSE`=|dHn z`@4uOYuQJ}J#Is5Pc8f}X)-Z5vyZ(=h$c4rYgniCB=SY!5qAG{hJU4+p!59?mRRs` zxcXzvDLS4!`(?zGHMfDqxi)UdzaX3(@&ODM&t`YS!pZlQ2@FoHB@!~nai!x6tbeeK z97duIeC?`+UM?L6mTej4ocoY<0IJBi53oy9hcART_1 zc+=x0K9vt31;?`3Cyf*2+t)Sh$=rH))xpo&msqmy1EJ)uX$*^AxRtnn*JqEFyg{x{&%-HGoFPbs2+<_P{0D0w_EOwC$$^3F00|PG$TKxZVV;k0E^`=TVI=K_4>?;T1 zx%qTe`XQ_j%z@;Ha2ylm14sG%v$6atkev2|rnz3>Gs3f=)A$BHbLxT-T^CVj{cG;( z)Q#NmccysHDGhc`zlWVRH^J|+1)h032Hi5B!@OAo=vdYW9^nfy?Hc21H`EFIjGCyj zc_gg)#9tBae+?U-q+-H!#wo}Tg9CbMsO6Z>IGq8g3$oe~Z#VZg?wx+dK%`*%EBu z?IC1*?jVlHsfLj(Ly!^p6Q?b+BMoEcG7~#B@--?CA84hCoEATUem_U{aK>gLh_Yaz z+m1r?u?=DURI>pMTb57eih^= z+i;4#eK_mj3bJr;1M4p~BsLFT()cr9FjT>nw0&5{w$9QglV2qvc%G$!RTSD*_TZCM z?_vD@GW>Pg1zgXsrAGhSsMNMYFs`l!kG?nqCAt0dzNIB9Dz<`-xg^Weu7|(zQ!&JG zKP~#U36|Pk!=)+xu=DS5)>ZWuQd&$gC&&fQCG|kFb06v)J`5&$L7Z296i(Qk0NYth9RMn`nyZhzIlko-kEI_LJ zGmZaFMb)xKF!4b+mMpXe$Ir*<^#%1*+0GB@K8SI<;T`aqpN1|EmvIhvN5E}o-i0l< z6{OSc&}wZ8XYW@f*nYc{J~C8+$GhDzw=xvQoEnZXwsKg#@ft*)`Gl(@YvHBBax|DA z0o~3YsPljSakX<&xSeKUxNP7KkCc`zwujugdDJ+-}jth!kaEUIj1^e@6V9=CQm=sfiJGo4_8fJpeb~#hsgZ^M@ zQi>-g#KZF=uW7w^8-3~(0PZUru-xx2+)5tGDpzMiSCSDOoxBh|1G2&RaVgqG=@>JdkWRfU(wCw9#mLu zz|5UfInz5moW1c1l(Q*-QT$9zyYD|p;2Ao0vV5P*U>ud`ETNAboneM$7CIGH!Fc&p z44ZX?Gf0hx(c!~cHQ&?QYTAcSLSn&5yM{s<&R# z=gsMY9~t@F%FbRYmGFbp?ntM`3zl-rMP;Id+heG_bP{Ja?IiUcJA~^x{y&P&G@z!g z4a1e@In7FwB7{uU+0WXQA}Mb&(@X;y5|VjJkyI)gC83FuB-Po^+NDWUWJn~E3>iWc zGUYqp&-3@3eV(<~TK9cj;zH%2H?+mcot?MqI6u#CAzP65jy}7kMq9EX$qFxD`f!ph zom3S}E_~cfOP^i`pOlaIy;L3bY(0tIo7Z&2EhSdVa1wvPVhKCF=Rdy6Nb)fT|++H+<<-P97XrMk)n6cOSA8e zYw_im>dC*C3AkWXAY~$}Nlx)Q`a&{~1f^wAE^rh1Q(Z-`3a*iyBd=-7?E=ykeu3t1 zy$oqn-f%6?RjEtnBV3t0maM&{Nb6r+pmIyJS+DD}_!FVp?7UDJp6_WVvzOkX6@G_E zj=BJYqz$RW!BfPpB%3lj#*n#zYspYg6Fs|Tn0U$!QD*}q(To_POZG>T&@VBhw{Hxe zE%LN>iu~G+TI*RoUc~15)=I9XjwcoKt7v%0EAr%41O4#%5eXYrOAkizWWucoy0hXv ztc|Gz$5=J$e7F+~f4$-?G#BB>)-hnu`~p&$O1HO%lSOm3a&~1M*lbvX-R?KZ)r4-k zD9?ce5jir-I+G4RD1vt*qBSyE6(0$s+$edy$s(r|pOj&S-s+#{KOz0~b zRuWC(-7bUil5_O6^)2G}el4y2;(~5Tu4JK;0@co)O5b(ABieQb{2#|&;xua?ogg0$ zOQLnCKy3rj(l|oDlm$b0SrS}V-6zOxKS!O?`bm*{13hD$3!|6n5>L}8T6u0GX*zj; z?sK0-YFl*azZXI{F}n++6Q#+*God1v(Qy*Z?W4*olkZmcuafaYz?3pE?r*#mBUxRf2tE;7#kCM7iW!r^&13N>oQxi@sd3 zohTM1QMa9E$o*$RS~IkmSmr&!35z#Ek#sYz`w@>3hkQhVkPCElZvcszuo}z%bECh* z&XGHAakOvR54gCMa&2SGx$JSv;ZWE@v?$AgaUnX?OzJktStP}W480_8*LrZTwiwg= zpt}&J7C{butr4+FACl}~OX}z~j{Bg!foQtzquWI55+mwpY-&>?x10o2p)inKa{7ST z)n9Om&pJ}aM$j3NNsv<=1fQo~pqJeW$d@-&H1E4688U0Y*#q6o6u;^8h4Cz~*&Gi2 z1ESx!Y8N^4^A!D}5l9@b&ZHGm=Y^5G9|1eTmr8U7k&hNzXinfFvck0!|1&)SQag8` z^sFaLRMcqlb7MG7Sa^)gpD9CkwMD|xT8 z^IHbQ;A7r74R;)yU=xnUc-Rh~SG>bkNd=M>;zw)c z-hg7pBJf!nOp{fE$=ZAknpRT`eRkil&PJT9vw4iQO?j~6`wUnu6!`>0(#ia67fLx3 z@}xEl>&>T&=0Y#d`Jzo~-Ex_G#r>R)&N~ona%SobqnMmS&zUF3CV~29S4>zvnfPZ+ zqcfM@VRnKR31|qUBWq+xT6qC?ZqZ}ZkG2+ZG+k+gt>`%%`%O51TN;K*^f5_~{c&ZY zD>Q{R;!LeWP~DV(Jr2E<^V*L5t7DNW>C-dSm}8&UTVI`ToI5VxT$V#+ zrVW;*JOZZ^(sarPJE&EufoRcOg3iOBzu%FGCfm8`)oSp^Tn2rNRzbq755CHfF;Pe{a|!b`Jg4 zr-RAWHIjxJs;AWnU5$pdlu^)A~DU!{wd*3+v2PcwI zLk!AFcvD6)oh%zzNT)uv!-)6FMCQXS{MVJh?XFG)<+e}U{w>onH1QK02p>!P3JEzE zs7jBFDF-{ri_FOUcC7LUBAZ2Bj?%mo$m>gi#S*1>U}>DS>dRBGr0b4w`-m$zxz3%a z$Z1d?k>_gaBo^F!c%0BEAam6x)5v28%UiE9?q^Ey*u;lWr?&%Lm!@$gTT2CtqP*aF zg*cTr`vh7^UG9qs%AW)qZ+kLlWzWDUheWgrzQ`!7 ze+_T0y})wKJczQD0jEW$vE$rGaxvsG?%NS0=$Cy0(=tV@&-Z&F)cH84c`KS5v2`_5 z5-5x5TqWo5&mBBMU!&gnESNa%1XnrbCFf-{PY@%wN-(p|5>t*pgdcZIs9lQ`IZ&St zC;wYTk8eIH;?}ct^w$DdF?|u7|8fDxMMuIqVFFyrHln3clgO!K&p5%vllW%6IT`CT zjGliE!=!5&jM?9DU?Tnyom5y-eQF5h(GWC;9>VWM-%wlgGmKJ~#P{mIg)g!az$>{C zC(eEcD||Pi{}MfP8~h9JaWt)#tB1;!Wz1^lSJ-1Ma#anENV(?Co7hvuqzi zz&~%;)*Fo)ZbzZS+ZRTLdgC{H9uzfng|ewlf~6DkVY77xSJlxYte4Rt`sa^|e85w= z!p*aYWSD?%d9{#4+!JxWJ1k)AI3?Jk^nwv3@NqMSz%``I?S-$=Qf(4hCM`q9lJV;B}hI>9fH- zF~y8h?sN1Clpw$LmeDg~rje(93ZgZqm1|w9MW8j1iW^Fj;%VzK^5iP~quI~6i~3P# za>rs?aZKfySe`SqjK`ps`!FaJr;p~o;&NRV632)Fy3=qsdGoUslP>T$+@(xrpSPe7 z0<+=D!2}FA77u?+rc*L*5qWA%XnThf7q)sjaTj$8;%2%L{rr!(LT576i{+v}dk+d8 zq@mTSR_=X|9x*GKO=;{`c+>KdIsILXmV9Z05>bfG>X8!XK$5}eawkr#ksy*=w_t|3 z5i>BnmoXid1{w2{cy|9sc)8aRYro54&r%@P`)5+y?ulfP`Eij;PDk)NYds`|uff`% zDkNOtH~Kw104{S1ph4mq`W){>|TN@zyINlO#<@vFGbN2BO20)YR zAoZ*4Y4@|~u;yAV?Aqx;d!_Zsw4X|N=j}+;mwU?9ULHxt%=M&hKC{Wp9}@I>QZxK; zFJqQE$6?_lGa`cgVOcj}BvmV*V(Btmb32SVn*Icw`}OGa#Vg5S#ot)et|;)}JHP~j z@j;*!+PL3=qAN*Q_t_GyWgEa*Plle^Z3Tg&`?(jJjj?!QBraOf0PLU%z1I=Oy}f=C z?j09JqGkp|-*$O?uXh0l9V#oUyhp-=CVe{OSOQ1i^$Urj9G#fc0-70#oLJvUtj`LC zMsGEk7FUixOxz(vB?f{f$xudIkyzU%qN0fd_8S?J~Rz4oH}&ZiL%`_1(4 z;2iMoY6AEDr?9NIh&yoN8Ms{8kMY<2m@upLqCL|}&in0hFx61yB)jjx#*j^Dy-$`b zv^tIbJCzCPIDsWKX2gvjMZ@o`BeHfk@WK@VE*W)-Lz^?<8NI1wbji1=mE~gI}GgL6mGOT3L9tDFiHLzRHHwVEKZD|PX5khYzzVU`gkhM z+ue&lrJacL>t)n@2$_Q1(SrY8jl$OKW7ujq8sx*bqOZg|?rq#9E?)W{*U@$m|8+Mr znlGDBCd?ATU2fyA(-utkr!LNZ+e37z)g$79}70*eP`a_kXi@{Mx#STg3ab9Kwj+?P8hrexw%b((VQK1c^ySYhdI%+eHXc;3YL(A zqHfEty(EF{#NnK=xaY5c=oov@(oZU6-bNYDQYRX{zHf&wCl=sAu{@Y$Do5XhiJSqA zjSQ>x4+{oOa2-l)MVi{gs*^lzKqGLp|F0VhYI=)1|X355hFPxoE062dQKp z1obFVx$6;-_AyX+|LjdJJN_59`(+}ijv6B7QZn5XY%jblR*6=>&=N4R~bH zL&!586mC(_z&7u%;PmhZs`iQ#!;j_Yp<)VN*R)}@bqkIvlp>m)kMPMIX;|BLUXaq| z$JDt-pmXakSfSP?yhe9$srMfXb`@P_CPd~lqePtl7VC5v?w-!I`Oe`AL!!W03mDm! zk*Lw9O{@$x=+G1~GPd(RcYWzp!Rx#JuuNknhW6iv<6k7H;VxV9bm=EtyCDFMI~hXk zm&2GU@+4f=f6D1MdoyjzlNcYhOWcZ4VPLp&J~j{LGO~-Lz#^g!`LafcoUA})b&i4a z>NqrLej+HDAB?SMjnGHefC=El=(L;^C|omzURf=2T}8^G@hfrqbi)xQyQfO9;JuQF zn+}uIE6u7Z=moqqEzU7G}O5PeQ&4WlD0^^V^1N>MGAu) z8ifBw&VfH|Vwme9fV7bo%+u|~OiTYl;q|>u%+%B~a5ASJr<^W^yhpRa0Hx@pQ-9#* zhoSA-BmfgS*nQ94`E)AnNCzgl?Ok!iaQHXL|Yt#xKVg#LFvC*3AjjPQPVb z7M$W@?M?|+lMHxfC0i7%ne@AAxr{ix53-O1>m152_XVZ1qLm z4r@}p!4Xf|%AoEmDPnM`i0KXz;;4yl;K{WPMr-|T%xxG!&PhjbGu8=24vMKT=jU2* z*1m$7UAe;V-u-a;i#}BkyTT+dy9iz?^7Jle#?=%RLzJwE$nW(|U^>PO1Rfq5@Ks+ED8VraBb5@-L|4wH5VAZ=PNjA|GSvmPXK<{`<9#U2IhH(vl| z=Ojcu`Zb*8g=aQe`VW~U5jwcj9Eo<8V zkqZ@VC=iI8 zN9Fy;LE~u=#sq7Tv(=$^IKq{C@l1uNr2W9hjsIZn;?0;)drGjE`oXb-SGi@*Yq+k5 z=b*5<5|0#$!!)m4c(0*Hlj0u21a1#Z^B7B?MAX91dtvbAp%={<3?}18NK%KvMO=tf zE%>Ppiu&>=;DydbY`T973_MNfvepeyqV!TA{jv}H{r)k*F;blMik}$lQv+GXws?Ja zH=c}{NBk7M=&-*H30XFPY?}g8+brUuXy{VKD?RXKlTc_{SHu`8OveqKjZigSS2U}3 z;`+a$ce*wOI?rS!8t5{_^F}|qg{;IWCqBUjD`$L^br8oMivzOk7M`oFV5qYalO0`! zL27#-PPQ0h`TJn^;sRzLP^167nM?k&`Hnd{_b~j+Z1`Gnh0zoQU|!-A)HtX_##vqPT<;tOC=&nsN8ruIHFc) zS{OjeA5Eset=pkXKwyKEAD)^th73kY(EcYTv}SU_DbM?umOMe=i&Bc0%(OI+%XgVS1by0dW*GEca}?z=8{ z|LP&AtS)6PKPd;-M>e!Lo06@M{b;liqD1<0Vw#mhuP16jcCrPTDRBkm-p1l9ms>DY z@k{V9lVIM-a_-fS47fH&fkxij!KsV9gU7Zd(`n7}L~z{(ytp{3n>ULj>dE1g8*bE9 zq6tdOCt7Ry$kUA0Hi(e1#saO|c-Z$3bSgxnn1T>L&N&JFj@z&;LX26~9fx}7BDnj$ zyI`WP61CR44YoPu_{Cd~1V>fjp1Z5SYhxLAAhHOwzRt&j7IW@RZxZZqU(3|&+Xl9Q zKxY;!vIRG_`M;-4*?rfaQC263{PAuBb4G>Z>!7S@K$GcEWSq9ZW1HTBP(&_ zf@4Ief1B<-7e_)zzGe2b5pAmz?8Eut} z_Ky)ii9B%+zMsVY5RvIXuab;NQln{l#(YCY3ft~r#b3CWNLPPk*mDm>{qgz;dd)F` zY?Bwk5$A-{tUWu(?*}=w{oWCBTFjaT&pAXgFKwq;&yDfvizS54NTfp0oU>F}%=+25A$i^U4S!75D*Xm`YQB33uY9D-c9@CSXdt0-3C?DB^x;kqsg? zkfZq<#!4!kOgr<9zOz3e;_^wd+e*gsyN`Y*YqNgC(L04AS7JFawEYZXRrxe7eLwl| zPlv=GX``9>eI#1y3~KHUrR8^6(wuh|+7@Qh8^`7ltv{k3%m_`YF0BJoT9l}1+-KZ! zK!ZGfwV8946uCWr8M3RM4O5-Jxxx#CBNsNP^Ho}TWJb0bwO0$o@6+o@MD}ayZn{N~ zY5tw0k8h;L=k&3}JeN5Byn#%V3SDzTlav&Np`YgnQaLt}S&_7y&ap@(o?WM*XNxzD zKYfJkeCJJ{^^7D}&;J2?qg1+Q-bLc6l|X;3+(8<*#L)iVhsn0hUvX$m6b%aOAOgd7 z`bc!A8teOsp1W;N&JbOqXn%?xlTjz}&;3AAuNKc0YtR#k1;qaRJ$hsILE^AN4n5va z!;hI0K(+M(!+;HDrs&Y!<)hgdzUKVt9uM-PqL9p2)fKT3W!ad_qgd2Q?)Ktk_vchlW;isv0*)xV01?ns5)C$^)o$AIxS>r^7ifAn8Cedo@p(O{e1+wmEwuG>mypw`~VO5&SAweHuFP1k4fi! zcT%**kq>hSWFLC2J9gcek^HIFGvum-KV6e5ActKx&~HBy$Zfr9 zI<@{Ec^~?P)|!aXr$Y-^|BhtdYx)ru$9nVqM{kgV!v~3$)=K`c<^fjON|8@H4BS2` zI})*1Nr0SHi8Ko&`f@lwA>v94zh)8xzNMSIMX#J69W_MMbqo$bB(yc?%T=W<=zQZznTX`_NMAbnirHpL%wY!yXgrsX$n#o=#m_51(LK{nm1c6%XWs1C!A9Zb$Q%G^~6Qa1CftlzIX(AXOz#ztx~R1`h1y{ z3btV*XUSDrtA1e%vID5it4hA+S{GXYGx;rXBIeiV8Ej$EN~*~P)5q8KiK49|(T@I2 zbB$}rvKD39^;<>+Gxw%%B_nC$Sc|+S`V#HrII>>W&GEFEZbFY2DL&3erI<% z`4;jWGk?2*?m{hkluTmP{X+PSbL)xE>jn~0AtCzCmlHZt3pPCZKyvP<@~)yju!_%b z5KBtq8;wiY)Q5BU`a=`Z+Rlv~w~^uLMFLJUY>3I%PNFR-&9^*4YT|f@^*8F`ZSU5R zTWy!v&lf-O5i{4ZUy5qUi^PNcC(+ID9eTWeW-xplSWPr%q>!v34?cOPE<0}CD)M#f zF&?68*p23@{NrJsU!ghP_SWvt{I|0UX~Wz>c5JysRfW+28Q31rimr5cef24( zDxyWz_HU^_?|aLczx-3gsyF_|a}kfp;z!5X(ium2m93)geB&)zS~!pWn{%38SNls% z_4l%-7BRfPDA=0gxSz;>_v6PX8?tSF^_XmMg6_^vCtr2*N!`f^e!aLn+rM=sTiTAi z#3KQ_dE8x6^7#P2=}-_W{j{0BQkH-bvw1@A%85FJJBZ(mqcA>HlGWK$!(aceh?QBc z!1|h+RsCogYddY!G~Otyg14KaW;;sr8eectk1qYVi1m30d{6#!BAup9Z@L{thgk}A z{k`$<>cM#u{YHmZ@SVoy)qJONt*U(cx(+sQUppVOY%vL5bB2{){eoAw&18oHThMe< z5YHWtVW(!M@>2Tw?B1kRypxD!XqoVtbrRTDH4r;nlTVlUxt#?fXVWsnxmRsYD(b{T1_?xs)`4am+H9;1KjviSHnMz-5FjH$BSSi*LNdC^y=|B3cU&)Jn7 zldJaI%h~!HJ%FB4xvD}vd)tH3GF86<<=LLg)*_Cn6>s=&fM^_+=2QP1VK+;B=3}De zZEx9i)7E6ks!vB(+Oqtls=OQH*)Pm}c2)VPs{40dv+n~=^WTn7WPhqXAS*v#=jDIA zWalyQe5vXLUd&R(cAl0*m4lZYv432~mVa*Mw?5Ql7x*i&tz~gMyk5daNEDIgakV_z zKgeF}5_wVnX3*lHP&QQm3w`TbK};GJ@{zKsY^`GizjmA?`(aNYD~y+~`g^IH)yR1Q zWw-9}#?Bwv$=QPx6VLHO*M;obq5H~h`BQ_{`(=4)H{ zL)-w{mlVWXt&Zi>F5YL~OGxvM?^5Zm&tFjEYZMz&$MXwT5cax>h?{#wu4?xBpKR8c z6dE5?&!?r!*w*iF;!S_eC9+13S)F;Ks%i{f*oSEW>`3u#{JBZZL{O>5C;ywku9d7N z??m5cOqQj4 zf!d7EqoW>QCaZk6kxdC>c+R5`zcrm@{lZT0cPfQU%-;xhSLz|&$8a2b^@am4xKhRb zZI0s0^i}!FLM_{>b&geIJ{#LUy{b>nB#svaA&1yv-IHu_t5lWwPfIZTC~y1U;44v% zo$y^vqHbt&E`KxY9`8QG*;ch`i2pU;kzeO5nsrZ2s%*cWX8j`_*?-cORa4IB*cRCS zWLTkE)$J!kfq0rPD@KX*wQfw;zaMAM0CI-H9@?E%@fo*WErWnlHE5 zAhqrM@7Sl%60?{X$x6|0pV#uMj+L=FSu1Eqtz?y^gr)6%<21gdS*q%^&t6-P>DpD> z49@e~rPFPfpKIg~aBlp8m``l&cdZw@MLQ&=aWg_3;9(_ zJ*F?^%XC%#@y%eGjq7sb~+;+N*%V&^U7iP_IiKJLv~_TOYv zws^3aS5k>!`PpKu#P_p&>jPk?jI?6s_*~>w^hG>}b-rw(jdaymna8Z{@OTm>70WyQ zlV%kyMLuh{<-EIL02^<#jgFkI&8MXVvGGN@eAu!NtYL9CpRmE5>Dnk}J1bu@- zvNAlMHT`^sf4WkZXv?^=3E_umbJrohep?RvN_iD}_(bq~ldrLQy=Qor8U@zK^D=q! zxrP=r=<|B|O)T7>%Ll%R;=f+F%r+0j=_Gqebg9qm&Ib7Km1EOC4+uY=So7M|%#?6$c)&?<^wunELVj`3KEx%n1!XIKoZvs* z0@{=`K(5~`=l&Kc^VeSfA_x7R68nmS`~w?ZHrhv#BuUNF}$_e-WF2cs}xZ9D8|fFmDkO3-{CJu}T~4`3e30v|yhv zYiT`&A8u6R6JpEQpR-o*a=q2`cf>6AOxqpY)l)`~j0+}PbhSuTs6JIpSS}dWizK^d z_R+?eE~3+UhWQX`%FoqW!D_18(np@A{Ki#oY}}+`IJIIkZ?ChEZNH*T#lo^}VmB3H z(Cltp7amEPr6TCc{YOY+uQH8O&LLOa`l#X;3p#b7FPo_*%_ruk(N)SVq}%5XJ*G03 z{cuo?Us9ik|E;uRzpGmCBhO@jMATe%si`b~>bZz7ZhL@rsGGn)b{|7lNB)M-Sc3hj z^GJYd3O!bCM`oTZhj!M1in)g~$8KDKwGr7U{c9z0TKI?_%DYa??OzD)pDLymg<9+@ ziAK7+%8>S-)nS{K%hT4P8am==2T`4FOIMG}nH5D*qiWVRWLGAhUE zF8h(}F9U|=?60Kp+uhi`q0{-gqtnTA)z^fhn*3_J1JKbpmTlSgjUInlLI1t>VedLL z(FnIT`mNcNP0Qoqqo)<$KDCd0(^^Yro>u11{p87~i^W`2)i2sKNuJ$8L^Ep8o7cW% z#>#%5Mb0|-@D_2B?3-O`M1Hj_@6rwA$RcUdS1eCUrfs2@TK15bzBC;1a})LJ84aWM zcyW_w4PcdqHior!GR>+{^vjx`kYK)wijBJi#~Re=JKmMl7bT;A|9QIb>^5R9>xBaj zR&alm481t`9?IK8F=f?pP6V`K=J^vi-FO@G{{DgYW}|5N?I@B_bU@Hrc%FXS#1N~y zv&evZJ)Cz#dT@CUxgs4!eSMC?gl#uT$fXz5{f{2=bw@4<8VsaXiQZ5*tWBoLET9+H z861F4|M-j5sW)(p0BA z4h{=lCisxo$9$>9Y8i6#t_tVmHUUc0qd=$gI2S#?6q=3SGCie5IP-D~6kpsx4tpM; zWrJ(TsEs=yt3`zlTAGqu*K(lcjvNi2pFrl%YJ*JKd^+@zBlqTY;inR3>M~<9QFW~p zb~$dKOkyNq&&_>fL_H!Ai_ktzD^NUSd&$ELrDsFd6Zj*Z_L9n(~J7iC73ZqH-h zM)tupp%r!59F2)mRUk8CH2oS8jVBdWWAh(P`c#tx*8Uw%E7(hCxcq|J5qi`$eje3l z#ksy`vvJ&i#&m*N0s1eKCugrDQ2nhj%%bwET)R^P_Ft~!PCKo`mel#ecb>?^S}n#J zdpD}>aE>S*Jco0dCevQcspPDEAM^9b7CKH+m6#2tb15gr(;E_TBsfixiY3)qOX(*= z?C}h6%`~A#w`M}Sk10tjy~52p(FFflT!7OdZs2DtMW#HMLQ3fi$Q&=q>UOG?X?`g2dU}n-juQC-&P5ZkU8hLhyPuF1Y)tNS#1s3chXol|G)VEfRpgYvIQsr< zWcKQ=AC4fwe9O)j+hO#wwP_bW$xaLuCza&Q< z%4ER2U0QIwYX#x{ieb?+NAzs;B1fwKA^m{Ny0691w8ffUuXtNo*?a*yJ?!cC@eeR4 zD;^w=i+WG_krYpUhubSnsOQ*YbcgIBGDds@RUUZ{$z}^;vTZ!Q>vbFh213dD8V%~A z8qNH%3?g2WuHv$=42ZkFfqZfK%G~H!4^3{NBx|xNQJvSu$OTJ~RL+ao2Jgq-dLW^{ zF2Tnb8Jsc{P4wqYL6f_s++7I=a`DdgMG%a^icD2NJ;gl$@!A>*o0i1f7gXmZEqFY^-ZFNhQVa^ z;*%INxpT>B5M*1$fi3>NzJ%ya{h-9n&!Kb<5}m3%Jewc zxpg!-y>u$EnHmMJo=J0VZbGozzXktE=+Mk#5?qO|0ljLYNhe8&Q%B8cx^Bl5Iws=? zo$G&umZj~X(#s}O+7&=GZx_&^fkUXebToBqvZiOu6{%_DRwi{;J*T@{f^yTpg4p@J z%s5wZc$X$i?(WP6`6~>$=OQ4n%oZ5=ay_|QGqLPj!3>NrBDoo_ zpwxFNGxm8lGxKmX%xt1e(7_Qje2W=VyL&8pDcI6~;S~Q<6y>EF+-T6~atvS7ggRm} zv}3j*%5NA&Q>Dx?DpC=*%o&4^T#FId@v!H5v+(K1QnWZSnP@x{0rWh0ZjWCY0?$`$|p8~Mb2_Z!70_<|lz|Wi4b8|Q5G3M??=)R>8C$w+I-`o!@d-4>Q zu2rWJ33@c+?s{A-=1O~Q%xK}&19?vhLmX7(O|lx3e$ZSWfQq-uHhU?-1H!2BR*rP?S97Nwj)W8 zFvo>4u~469M3z+PLRS3*XgEGbl&cG2x|N^7hVB30kJUFuZr(5~^8XL!rlm2He`JEC zk~Fh+WI?6#^&(I=ipJvKs_?rrQLv&o1=W(Q;Kl7-7(usV<~Mn+@?#9{WjIXpZr~)v zE?}0^Ai5=tVRk+L%h4}+*r%{XaHZh{SFV+buOG`GF|>mH%Pyhpjdo0N@g$;KSghd}0p!}Nske8YQzT!VP^*{!4+z-HeS$ocF z^D4&k%1gNRpDv0m_q85tmjy{TZLuWF1-`zY$D~F|;i?QJkhc`zwxw^l>_4XjB8ns$ zg*yplcUlP_EUmzktCBfykhIB@K7l4;F?jvJbLNx4oO65@htG~GV*K5W5OvuZ|5?vq zs^ufVO(m5p@4gQMc4}}-!xof|3L)s|HD*^X}=obz;H4oybs2qJRxwM2N;tH2hs1+duG+W_ri{9 zFV}r|NX3W;nqR;MlN~@N+WN(7!EnB#@m0rvSF-07^ z!;ss#*aG(0=%U+PMJU+~5Z9*9{IMCujhJo?A796V_RA|cE+r5){z!l)qc3A{ls32* zK7dJw`>@9B8BXR7Exz* zRX5{4kdLJk9&q1>L}ET+4tHSNN1F?Ho2&h_n3-R<2c+&*3G0#unZ<6hL}vL_ra3JK zF1i)MN6rJ@Zj~n48qu)pRu=5~)Cxwet&CgL4S15B$oPe<6DVus!n7@!Xwh+1C@834 ze${s2ZKEq_rFRtLYU;3ejW;gZXh@kU^Kn2a6TNnr(s&U(Aa<}47r#A;6?JpbjGuz4 zxpg);c?Gv2-xL?TECP$Z5awiq6ek`eI^)wD1@?K{gw^v@2wQ#x{MOxPB0i`R#lObj z61@bBn>iT0?LH&DtBzTHUWVv+J`@_AoyAp`Hi3$+3I?S~;;J(#a9{NM)Y6q`o6!Kn z>g_mpU>weE%Z7j`Bf3Rq8s>KvLVuj-gzl!jFy@mnM0gt0PaPwHan2N~?-yla?uwZB zdU1lVlj-=MTsU;Eds$%IfW;8eyR zJU(wGTpD)~IKK=m^fG4Z=A}S@^*C(b?aaw$o?{*_&_J=s#}!RSw{YnmUEKN;<=jb8 zI_=8T9?q<=mh(090IBq!vKU)Pi1bVn=VlvF1eSq06v5aXP-3(FXVVDtL2QqdC zVKJ#j4bd7{)zHGg>@xKJqCsi|2e<`Aw%Fd63j>#nuz%=n<+Y5X+^4g3qO8MAF4FQO z>UhfI(U34SQqM*&v2>e&vrXtEu8g5<8?!*_Fy2Y(V@Bo6FfrT53ue8)0rDE0K$xBb zKbMLT^*{RHxA-D-e7OTc(K@?db1}5$9E35W9s=`U1bFPJXNJxffYfg}PIhh_bNP4$ zBXEwzFS^H>mz#3&RPPxK>{@|?VGSsKu@I$SoI@LN(O&BR=cYH#W3z=9-r5+18yC(- ziThDpo9!B$b~%eNQ~V%+Pp7!OHtA5ZIS2|qv@yHiH^Zq5QSdO!1fuRf1a9Gea4zVw zId=3WoLxT??sXdA7yBHrwW#O9_Q_(t)iLgmnl4J8mqR7z8vHur9w)8%0IR!1b5nmL zhF^-uyX%^`dnrfIVM(OWQmTowsTmYX&T;|$`%z4$Ssv3~Qwo)Te=*y(FNPJPnqc$- zFX&&B3zPHHVSP>%ME=_WBmC`QPXB7=?vhl{m^MyW;(3j0^Vtg?e-iN27Zzv!ea7s` z;!w`t5^wDo5Z>J(#3k2MusVGm9u&snNizX9%$CFC3(IhI^(6d})h0~M?&0noDCH~- z?o>Rri{+GKjtN$l3K?b7Pu%Jo^-TXCKf!sAi$bkq*-UA79C!0by+GRWrm!p01VgJ2 z2;-OD;-;%PW4v*sFsJJscX^Ew1{`f=CT*8Poe}@I2eA&|9UFme3)Nx7nil4b!CbVQ zbP7nvOhN4J5zGz;eP}$2j7fYObKhe5bQ!nE@z}DXOk1n^(JWJJ#8b%mXu|d#i=nn48lMl2f|u<U?U|!{XhwIFR_ZIj#B3Ss?&J)V57>rN2 z$3)Jw0gnVNOgW|w58?x0o6};hIlu-;_f`mXxWG;17Xk0H81nKuxW$)`!n|}DQ2Q*6 zX&dt4!Vf#4opd7?IcXR3raqDTuDzaHpJsyBpI8g$o8@ylQJ(nZR*TSf|52g!(fOF` z^MPq!Tr8BXUBIcf)Rr=p~cN>vE+N?h0D2M?#6z1ep3<9OT@q;G26c zvs80EM0v);sQhqd=lNCOGVBWv^3_2>?>Q6PvIB1F*9wYczX~!~Ehse`!EG^q!pRwX zGrX|_{#lfdp%T(u#e%DtZ!Vg(XKr#5v;eiYHQ<50&RpB~)7Yh6jDZo`m}mJLaeLc% zoR<8UseQSQlXuo-jtmkwCtJ*vh-SjA)RQp5Q4*4t5tyAA4_^bdn5**J;pzunIIti= zu;>nfL%+T;EuX&#`RVe|yt=>?kyyM+bv8-!MWBA~5l z6z6I<9*%F5fca0Qxh#ieuvRukC|4)V9Njg{%s3TiBROd^clbzyz-Nsjo;Z7%%U;sU zWp>7jIwhL~PiAFeV`U;{{w){yHP&Le*D1W|)>@g>pNS`CKH_F+EN8w<7>jpeCo%4w zVa%IHOSqwBMsO)(4CpRNWzJ7r0wYR{;r7rUhV41bi2HtKWN+%g_O%Ag_+b;!SDOL3 zix)FiSAQ|V+a5B+#!dJkIg=R^^iGg4&5C;`m&JV9a7r+}MTT2F5XXsWJ#e0kmo=oA{)Jn({ZCC9x8Oo% zSl6DbuXKg(f5&W=j5^6YnXwP-UN7K2zLbTo6kA9DM?v=1Y^K3N0q(iq6O_yrFw?{z zGv~T@aiRsExqV8UxoV)o{q$;QR!JNcX6Jf{p65D7-$a`8U2Fm~ia#)Sw%-*7eAEZ_ z?0Y8gZ<^3AS0Azz9x)Pk1cIr%bwRteiJ3i6!bDn+gY2R~X29Hz@oXFg{WJen7EI0I zX1_}jDC;8E;yH=)Tl`$;eEAs{k@QZOa#jo1ey~9Oed72cISxHV|KmHoYj7VEjms2t zF~M*mzST3vqR=;7$-awRfJqlOapDSYw9+%qXG_2E$ZjFCYtC<>UFim9()BW?vA0n0 zt)!kAv@{TUFVYn*Y#CynGY5rdOrJ0o7xkf8s+OydH2`M|ai+^ZpL^4203k((xVXAA z!u!#m7@eu%T*|_H&S*GY;BKBR%6Y!$q}vWLfy<+~8-CBYsjuD(?yW4~HcSZNRBfhL z9Q(dnsd^p`xu%(S~*C!@dXxoitE`^-q z6#k?OI`&jEF6WWkxOoAyr9_D_+BC){yJ@kFeSk0b<@_D( zd^eKo-J#B`Zk{V>AESWwKmJy1JSE9FXkFvvj^qoQ>)pB8FD!+3|JxxDtK2vObF>op~VXHuTVy6v@R?de0*SuGt_f5fcCF@6`d_`M3%aeW@;^Tw2a%ljHN$E%il z7of=|-<%3M(t0UR(bcT4^%StcwwQV#&u8}-rBm-7=Tl#U4zTkNtMJ1gTT}MCE7*Vj z{j9;?scca554M#NL%1M;)zNNY4aQcofdeO7Vk<8-op`;87hyG<@;tMgI@W8$-)bC9 zZ5)oKrWC%mJzKGq(%Yv>l?1)zzg8*Z`Mg*`DR-o>%gzb-^Nu9&1uZqKnoJ_=u&Rtr zpDK-}iKnyd%n3I7)D&b#m#{Nirl3o@Jl5Rg9Jl}c#tN-`fH$xwD4zn)jM_A9@MXc-=KC3%NogJE)+U!`-!Dfs8;>VrcMj4nYP?-}8sD{P= zsBN!AfiKtpD!w~Gy>S?#WM%-+Nv46T@r6`*(M{^`CmA4U&!1%i zM(RjghHdtNV#;;XK`MM(5qpC>FMoS-pI6o(iaNaIDbtrTC?%@}9GJY0a=wv3sm~Or zn%^k_#iJRN++KO0T&NAUsBs;Fd;@Uy&Ma`s_B=Jvt^m%D$Z)gu9LmLEI#3(FK$V>B z;>E73rvC0Qqb?kbWGm1xzuq^N^<658Ht&DR{xH)+`!3#K-3{)sUA=B7%dm#6(K*Cw zf7M21*sc$1G}GR z!ShAyfZ4lJ@ST&-SU!sYZ#X|l%Evs=vn~!S2tNzrsUi>-wh!d0CV*Ye`#@T|IykvQ z3)q}e0P~mJrsfJi<_VwMNcD*(QAXdb*}l>=eriw%UvXkR0?}e#TsG+79AtcdJ3CE_;koQ{`ccsSSTb+0ZGr z;8)dMs{cO+-mkQE;184qK0-xQIa9F!M& z6LnjiM(ZMrki)rNG=(ij2M(S>eWf`_yLTaSoF0b)Vhzyx1N+zsp>?SBLki10N~MTO z4f}BmHe3i^tI^`&+o<*X*Cl-~@}GnNCX4Qqi?VFH-5M->#hZUjkEI{~8` zKz$lA1A|6#;J8gL-)XTJh+8a9t!&d`@0FxbThFdy6}M)vC7&Pg61SKjeWM!o_8lI} zpS1%$v`l1I7+0{eO9Rl;U7z@FuX9-4)mCWg<@MATRbv!f`H5YsnL;u3ZfH@0GTWr; zN9DhgMb10tQFH3)owuHPnX#C)m57g`HEALa8MUu)-nb?8DhBTIOFCMn3PocwVgzm+H z+{8E#JLQ9|3vUT@D8Eyr48ViaToRX=?vh+>;!!~l&Aq}Hn?7E3_5#r*gnr* zyKT1IdoTVn>xb8{0r|Zx2Hf1C!C1EC$DeQnk0bk@O)(N$)dx{Re?`I0iq7L zqE5aE&=~7NK6xua^Oq%DpSBuB^Aupy?-<@dObpWfr3V8dR-&zfH0sun7JTZq5eeme z03zWMkUA8InpfLELFFT$amN@P3Q>n4pX8B+>`k=3JO(_UK8fbd3`TKPtH8>?@_4qq z4YmJJD45!)itij-M1@Y@jM4+$u!q=Q{?jpG>|3XV1HZqfCid9iy^rJ3ZF0r+-)Eden4H^&l@;o44Y164QVN^A`PlIgI0v z%>$aDQ}DNMX4tGH1Ww-#uxIH6S|U(^z1sWn!KdG;hHgiAxkekG%XbCkM&Wqd-TB00 zi2=5=^@gD$9&`h5oIcI*_oJnE(bJ3u;OAXNB+$W$I9N)P5A1Sc7m|ZbM1Qd3%S~{w zMkDN(SqCGO`N;p5I&@gH1OJ`HWscL&;G1@PNftkahkCUi+6CgyaWC-?E@~#btmI@J;>cKQBqa84I3h5cpaRB!IkMy@kJ~QGWCS`z8v_N zz7IQP8j%^Ev1GT^9>Q4 zH$;h1buo@?UxSl#WU=6AH;UBzjU2B4{=mpLFwWhd&y}47hLR=-Ue3YCJ`n6NsYvG9 zs+0EXm+_do3{Lo73oXCP(w$Lfp}~#Q@Y>@iVBBpwHuVt2w=NyWXLEIN@|_+^cGGJV zpQ8eNcBxadf3Ct;OzY7^*A%3>Vk`L6@f2JXX-3gejVQ~j9xaV9!*<`cqF-X#@M}Up zl!}+6tHS=k^v_qIX#EP9ep?i3_0M7>BP?-ne>qM(H;0rddXrya+X?kOip12@WOSxF zsVI1cr)TU2(@iC4-NHP&Lavuq3ap`RD>l(dl4(%WJ{WuY=U~ZLm@JtWkft$9EtPGG-9pOOr-vMLB3dxk%2RaIGgSwpLOH0 zquwT1{&N&su2JRKYN~YpmlgEQluA0|@(nujzhU}{dl#)Mag=Uy4X4$97}2GxhhSO9 zXXw&C4CCtl!5wDbp=w$Ku<6RhXQa8zPwEkJj%1Veg-6Iett^rx9YL&Ef0AQjOsu62 zm9U9Q$_W z;gW=X`1P%HJP_P~fprc($3KCsnIznCYZ)%M`UKtDqXGX&5@@}y4#vnF1}gE{IDWPq z@!+y0GfpodVvAi#a{`xT&DSGX&y4WR=9A=dM-rO5hA90oBX2gT5!G{|s93TEHCO6G?;VOB0KW@2jB zF#cGHd7{3H7Bch4iGfb!OIkY-Q|={#qnukwpdirSCM3|S8zxObCrSP6I=rG&h2%7r zk!}kSfse#00p+n#kW*_Tu>IIi5)*{z^?&~F7PMzwEd(lcTVX~pqTxcrDJp4MhUmfqe+=C;+4xBMn@#G;NkUCbb>r8UU* z<+)gG>nwcqKswUgY=e6x_H(&0j6)`u62nVJiDw;4`2242%&(T{JV_xHp-af;laKJB zt$CP8mf-YPqGSYm5GTBo`1|^ippm6yNJEc!hs%?J`S-D0P&J+e*HO!>*YKWM4tI{Csh-dtgguGmxDVj{EoL;WfIqQJYQ3}@I|KimVUL1N{{%lZHp1=!0JYWsQ17Z2Vx{5d*mBu?vcq8y5y8b|cX1n8^6vq0 zJTXGB`wi0nsFdUmMUuObYsmTE%gK{a53;y_75Q>+IWbdrB2IzIWX|k$ILF5h3}olP z#>Hy%u%!oGQkz8AZA_z|d^|$qo$^ecH;-GRxpI37OVMelMR#{?yJVRA&V7dXh(&H5260 zKTNjl)FQXT8Te7BlRniU&0P7Y%4{39VAeiYVrozQrPtWpr-kOm(Ar<-(T0``90=7U zM@Cc0;$0mCG~6QFt`(A*j^U(ZU9y$D+F>OatfEBQc1&li8xAnyl@-jL zqZb(4b*T)i;l@0Esmj#tJ3?Rcv8H#`+($JTX(Vt-3W=H=ByJCXkcRK~$g^ugg3e19 zh!Juj2_%(hStk&;ch89QPDO$L>rrxGsE+LAe?$v*pQ3#gzSA>5+b}sX%b6Anz#J^K zWomZHF;Mk2JwvpI&fo1$Q}2$z)X`Dwc07liI-W>2n)r}!GRma1v<~NHYm&A{&g8Xg zJUPAU9FbNi<-Chg1lEWUDLd|7exW-3&M<;*ym*opURFWVkpg%zQc9vB5>N0 zWAIa(AN`=?7ah;}uEs|5nG@UVnP%Tg=9SR_M(yZwX4B?b%(3x$F1NdwZoZU^OI`Pp zna}nUo%e0zD%DF4dtD>_F<(i?t4>mdx`^h)Zz9|+A$YDhSMb5eRIt2*;~?KTLJpUG z0g1Pd)6yc(=l~^MX7?ErreT>LvsXcrxjFof7Cm~E_E>$AHhQT{XYuzz39I86JvAXZ zWq)y7g*dq*I)OpfT^z-mO9sl#38l4?;G`77vkxFudjOet^D>tA0#IJ39zL!Sp+D`J zORv?V>Hp#m(|yk6^eNE{+U$uZ9W%cII@QWx?WU7h^_@8BuNNoRq}uV9`#t#E@(bAa z)II!uOqsMjoJwT+pK=_}LL75)7;SrJjB^L_a8Sz?Y~*qq2fxn5$ugnXVRr}izo$+f zX73?diKXOkbPH)rzeZTiZnATzi@aIiP7dfal1GV^WYztn zN5-aDmoe*Yr{CRMNxN^=fQuH{6MxG%5*o^q)~x@?{ZsGA)3Sf$N8AhY?|B2M&mAHO z2Y-{_Z3=?tS9Jv1BQMGI;QeINILBp>y4UhcvfOlp5@%Q`@ zyzS-^ystbSr&_MXj(#)n*+)WTy_z(+b$KpXw0j?Uy4jR`7!fBIBAf7dl|SYAYZRSy zdyoFJ{e+}u4BO{jTrHtIRQyXZnRye!N9o^-hsRJF>{t zm@VY{*nTpYl0lw-TSdB!my-33Tqo5l3|s2H#LGK9@w=bu@Z25?7{1k(*3)*QBLd6m z%JnYHM~O^k)7whMm8xPwCw4Ouw>_Duhpd^i6@zqRXFaW;Z3Y`3&nA~uOh{LCFS&j0 zC5fH&ANj4NBM9FAf>Y<+A|+Da$p`$E?CdrbXb&3+(psO8rA~$9^9C7kF(QT5IMhLF z$Eq_W%k-K1=?cs$QDw&Z^C$Z4o#%AgwNZLi<{nxnsRz<-{#Yht2lsqAkbS+L#N&D_ z=@i>SY!ptDp0zb3`d${d4wVr>TnJg&B1a~ZU!Vb}xpY|TcDnkO2mQuzIgM-$XuXSm z^ue=7=oOy(=><{dbY0j|XsIpB^?XIJ-vJHW^=2lnSi!@w2Q#ok>=Aq_Wek_RnMal- zxRRI&d-6V*k2}O&Kv$>()br^BzT52a#ENpP>v$S#jl9IMs!HTlohy-g7)w5-XOpm> z)ufekh<&|VOCke`Nd3Vm;%Kmn$p2hRO22s%E%&XYv1SE%XDCHB+K%BXS0{1lLjkr~ z{|pn`Jgn~ek5U!A0R>~@^m4F`IsLPMndNqoiHj{}dR`r1q6U4K5pxY@r|?_)-$*gt zm}CyV9y^FZJxiOzm5y&zyv+oV|oJY zx+X%#CRY;6Y!x!#6hMM2SCe6(&183LF_|@1%zZeZF)5tO*h5^F^G;`M_U zaO{B|T#$7QyBq0{{AsJm*3KYq4cJL81}Kt!-Wc0dvABifPb=2ZWaqC{L^Iui97|kJ zUg)nOs}^n`(v^G3I`df4cm5a|^{gT>wWVay!aNdlAeqQWdXWm7r6iwcN#0yAA`#KH zjEfgyo|1_P-4d3M>9`UE16?H7n#%KIP+}L z0cP{^WlaCfDUAK$9{Qw{9{p;e8BRFfi)UBXkmitAWWZWX(AsM#5NL`Cj#j-QAD(i1 zfGw|x>_tt%$-i?2r+$eG)O+uf6p6psqF9r@xG;yd%9dmltu+{G(=_JqCUxewoCq_g zSD3LWmthtjYo-s5Y~dWZkHOgR5)%K(k~l3|O}<>+K;|&or25_gl3^WBM#|#IA-ailZljmHJ4+9`wbKg%Sz5_)KfNzn zo-Pe8g29&E?C*Vp2r5ySJ(GsLvXk(dDhcv9Z~@tUF@TH@WD<)zyNROIa`NMlIT?Je zM4UrblSM1Hk+czt9!E6k87bq>J@s95XC%_ z%V3i4oMUtXsu}ff$C(n_C`N3)IU{K=#T=irVVn!yJoha;swc;G|MSHQrJmwJ4PA29Qx$n99&L%PWXWU{}E^;v}|{O((3#`{*h>`DQ7;)u0H4L#NTw zX6bZP(+tM#ju&%me+p9~p2t+KNMrsU+RMa$Sjjw`Ih|R3VTgVi5kwE$z6aCY=3~bj z{$!r;8Pa{PlL(~$619wbM+LqiW3p#qqhe%PupuW67F#3jM>R%@lBgQ0jb2=QI zvyuiPkLmM1B21W&G!wi-lWB65WB%03F=_LpnGvN*WpZ}MIcP|{J$_~wewnL#% zSX;iQBc0xcZ_2ey+ylo6Z>GCJ}Cc{ei!tT*4}c0zIY&nK6h{?yupZ( z*C9G&FSn;E;?}Cp+?V*muW4j%{3+}^`W-iK(jo1fr!%O1Jt+%VLS`&7AfeT|q`TA~ z%f50!R-QXxS{+5#RYcJLPS(;tb@=r3v10n+;dokXzbYN#5)J*}M=JP8IIi+;#F?4* z@ob-Gc%IfZytHlzhqnB~MUNCnl7%tZzi=@rjy*^!^2*4;y8_Z$)JM)Y^pe!f4l*9u zM3yhDCnZ-J$ir9FMB#Wb@fOP>@5{H7K}Rog{Gk&GKExxcC3Ymp`7?f{wg)r5r;y!( zFI@iX1m)s4#Mbm}1Lu5xf_YJnw6gdudZCIcv#(|iqwL|!Xzkp<%&2i?gcG%y@B8JM zqdM>C+h)heIv6oKQxlAA5QASfiRkX0~Wx9J~676+uFRk=d zneO(=rA}>`f_vo8U}C->&ovi?MWS=AEw-49QTgN--#WGd&P-qB!oca$cJ=5U0iX!-`=3yL< zOK^g4HcpT1!#DR2;$m(@$sz~zB!$fLb1lgk#O|}K@CY6EyWcJSWB;9feDb{r0X66+6zD0+4nyHZgrb&wM%{@y(!`*y)HnVd3a?ZWBmg%5)T$L zHIHqWhmo3$yOIX;XyIG>hRtRA!}Q}c%6kc8VhUiLS}yx}dpBNwL=<7Yx%h2JJns5_ z9Mdid3Le;1!K>9H;663TX?(;i_r)yvQ3oXIz9-%FB>x zE~aEt`&^4yCT;JS>4pp$Hi!K?_*QTh@bsar$ zV)Ze&!Mp(6J>!Ne+Zu7MvOdw@=1js~c#`GZ9%14Q=Q{e7NhZ1~NIkA45AGro`@We- zU#lWtDoRP>SSC3Tm_(lX*hY#_GPn?@n@+(%2tiY?or{b`O4&e0-6DU<30Xq-o!QR3uxOJKUjwIfLuRq>^ zj|wZG&w?3{=K3F7!;5ez5hlWOO~`^L4&?3D)ntw62C^k}Be9g&L}uROy4UMnh@fsU z8Rh1jve_$$^uIksPA850o_>P3Ddv#ZVh4zu-g2@gLY}-?)`Oqy&BjG$mRQPM7;l3D zR4985sZ_l|jjpQr*Np`@QDz%%g}J!*cOE`-o0~OO=drTC>f!Yl^J&l$OHcoEl78x3 zNk>`L(B2y7>F=`}=rPHQbWKJLy=ZneU3D{-P81EGrw@A5zsWAzJ~^0Hdb5}A+8aqr zA6ZF%yQ)O9e*z)Hb%mtO#_+#6`lRlG9XXI}OJ%+j*j8?v_aRsv0qOhFT5nPyu@DK49_{`C#_~OG_oEfkX+kZB+n;xSMpZymP zo4q(zQe`W=x3CEc&#i)P?*ibAC5q5u>@8pt?t}c3v!T}IM5trc2xtC&0UJ(zgWGq# zghID_U@EMFkyHX4lwJ;1t>1vpT%Y9vPafn)I)G%O_aI5c7mC#Lp}D;nees_z?Neh* zi(HsTCj`u*-JVTAYmZY75!`%gyr~&Tt$XwddH7W8pf=-C}E2&4SPUQ{avZtuXoS z1Gu;8J8ZlrO^-Or(%H^3v|0B*D7*ANJaX#-{G5>nJs-NkZZ!kw|6B;p`F;=B-FZ)) zU5HRy&tlvhk%%un%)%RVlkoBOc)Zaj11I0A#0y{bV(H8wEG=;tC)u3G|8aadr={0X z>u0^u zO3x%<;+z6Mv1+Z794fu|QQ-VQN)N`|hJ^0xM^_^n?9wiw93U3=%_uY6BbS8*9^m{5h< z*C8BF3V?%U2cbxN2|QSE4l0~2gag0w;Kp->(EMZ-R9oK-WiMQTbuLx#&C3F~L2VuE z+GPi4-;se)M>WCluNiFHg)1n!VkVBUrttgh2prd+f_JMY7E1^uh!=NnVq&a77IWO3xS&_qE$b{+ z_Be!d@6uT0xe2azaKc`);n<}v2mk6njg$OqaM0RQ+&xzc4%c3SMGxLW>P-r~Pfb-I zbAvkUy91zcpfy}0Wy|H!1rzukgPVz6N$-Y)rQt1 z<0B8yryN`C5qk(P`c#G!cVERpPaj}V{uckW{ely$?_eYC8k{5c_j0%`_c+U}`89HcrBt-&WvN#|?0_Ni_2NdltN{Ukj7p zoq@Zy-GUcy-i22?$D#7de{c(yqjk>c(gMbaK0wc*8-wT3=}YI*cOd7HYtg1BO~mMG z)yGi7AQ%2xXb3m1Rsjn?%3{m6Y1nkK6(6m5i4)$u!GS+UajeJ(Bw znO>3jaA*n+7RttNI44JeZWs;;+lQr$67jN*d>m>}gX8Vb;DQfj*yvv-mhkq)enp13 zef4uRd!-jL{(hWal<|X-C|wKwij;y|9FO(<=VI{hZ5Ft9xde1qv;yCSVo)naA67T1 z!NGx#;I5}A7?R3IQD8nEA5OyJN#*!TLKD8Ab_4fZx{pQt?qi30*SIiw6>d>Y#3vG* zFpQJJ!umJR9>YBL+x9!4irED>-z2ai@GIOBDoejBR-wy=RcMhbigay-EPZyl5MBTL zE!?y8Hf-Ko4wnRMglEKCfXJ65HhkTElz4OnR-Meo-*g)A%jWC2|JV&&aOElIURn?I;|f8}!D8^KwH2gn zodlDdf9qMT6%1YN2fKRXq1NI8nB0~JmD^(B)OrfGcU%Q}2d+~4S|!l4br+FLa6jta z-HI-|6r+LE>8Rz$4&Km9L$Km(Ja}_n94LKFV=q)xAfsA6{AbJu2OdbqT_=uWtygh4 z(`*_38PbMCTB|5G~bv4g4`+?HUe$bf1R86V=eZqzDFX+zwH~ba?+>8PHWK zr-VMWQJn_IL6es-TnWse$|eOUt9cN_Hc6r5!9sYN`*K`!CmPcWj$&cia(sPWIj$8d z#2TafuvwBSc8l79%D>Z`LqrV9>e<1EsrGQ!e>0%;_eQY8#|?aK4WKTn-LQSU`~+(^ zO#^A-Tde0%Gu|6fTQJ-CCUEmrhWS$#LsA?DBP0^wp}X-gOMeGEuz`WEm6hRw_yI7> zCkcG=o1`R__Odw_5>VCxt^>aJGx~Jp9=frp7BvSQMB#!kR9y8K{hpqGaqjln1a2M3z1*a1GZ2gpp;7D0V!j_lgSQ{jxK|1 zqW8hdw0FSahA@1+WGdX8_61~XaNLTJT=4aq9=J1lp68^w9!b9Vje>XE;V*Nx;OWQW z@P)EeoDh?O-Im4T&$esvqLn7N-t8UQAL)S7<4UO~pPqr{{?+gg%z(~IieQB2F}RF- z4r3C1;0HGcD8)S^78mD3%~>?O^V=1!jav=FZ5?3gR9P6mMFGs)n}!;far}I>B>eYp zG2T^m4mZBJi1U=Iv8{9(Uj4=fU)gp7S^gNHQba3(XzY7%L+=B4=uro3d)I@vuGc6( zzj~fU<4$Vik35iz^#pIJW}rO(2e{s*2MtD-!3^t-u)cXG)VdS^6&-!yzdar>SJ4W7 zT>l@C;kvOaT%J(w5>pTxeF-@Q$a3#tQyea4g8M(KO2$n zioqLJD5nrL_z$C7@!!$Tm|Li|8l!T5j0RN*T9Vj-o-~voF2jK~eo^7QT6; zNn5zyaT6S0zYX$j7vPI_y>(D0&Ll^ z1M8_J;!_T3`2Cba_{tI=9Q6?5tvfVu!kriB6yF2Qo%xj#ANm0-ek_AUzvG~}-EkQ3 zz7z(Ro`z2OrO^0V4m>|E4w9!E;Pu$K@Z+Kvps!XBEEEW#BE@!eY^^9>Jfe#gMjUXD z#x|@am5KjZa2`9IDm-g67t37SjLW?KpjET0?I8|2{n zDHuFspHefwD5Hqvi)itZr^qSoGK$=3fl^K8Q(wKxfSmSIAW{7p%#<7f2lukz!N3_H z(Rc)SY2*MHau}$Zdw^eVpDEo-_S6`(XLVKD*+*j`>_xHT{7}a=l-&BORJ5xkC>z%R zc0D81#-Q)Ku!$Y)s#Y1aeJBnM4_-sf)O0+$ZW*>dxCOuO_Qg)swph%05Y4JEV8u9= zz0Xl0n6$zSE=#b4lSbz7#akmd@J$Ogtx|zPXCz_sf}7ylXe_ugbv9MEK?;rb#3K6V zCG???%UaewK)3X{Sw^8Lp8974RyrGp|N1B5!QNo(^v@hqY&kl&^D@t3Gzs8SUx4Ln z4d~&@gRe%HLnpo?{7G5C&6zUrgMI}_o7YLrnK_r8<86vuw{iX~kcxf<)gcF+adhmC zEUwzAh5O$a;9ZVtIPQido?7@4oklrmoAW(Z!^f0r+Or%C->U(01N(rD@C3Lg_7!aP zcm*6U^?<@1)xgR(1!QyGp)Yfu@OP77)G_T2Qu+EGIS-AY&{v&EHzysv#B}#~j>CCecnr)Ec!}Ai=W+M1GnH)stvA ze#Qa1hpWR22l-&ig(ynAFb3`1BZT)yo8w-v2%oQY$GZYOvC9*09IdhjUpsG#*J<{l z8vP0ODIBE+Ta!Rf&v~#a=P5AdOT!ad`tZS?2L7(1VA(}GnDb*6G}ImfTYM71 zwbwVO{77@&4CSvZO8Tz95hVoA&?%j!^YEC2Bu^cpd zu^RmhYU4a@i1Rx&qWl&C+WYVt+Fd$`wk2LcdTBe+qcJD8u}B=~l{SKXQ>McG)oRf8 z6z3%|5``{7l>tu-gnZRHHq z2+feux+wN*VIOa$!%ym=g&WWf&jc5yRfG274$ynyF_`$wf{`h4pen>2WSMFJ%gGr) zQo$G8v`GNVt{wr#Gb2Ea-dAdTeHLqJ8izLKkD)u(TKH>}8CE@PfGbuhVq4GO$UeCZ zMe<4zn7s~VTFIi_C-<_u(>7C&qQn62x)<2;HWnQIx)f-N^-v$L4f0hwhFJ;iGi-~V zI`6N(Fi1SH8C<-&6U-mVLd`W(V5Qq<@O#Y=aGq=htJYS6{P6;iRFn_IxIJU}3NsMRit!i8YoYl3 z2Gr0sfsQ?!K>BN+p{?x==w81)Ds{NX{y3V#U;J%1m17x0eU#%-fo@_{?0=#*yGHu> zQ9Hi!KXpxEr-GlSaw?1Np<3_CWp<0tD;V~F&<)}jr2*CNViA^MPF zfNE|@qU2o)sOy~uqIamHmxt@wz=U%=V^I@utv3zyO{)Nz!hBE_-OP1}uYwt-Pr$q7 z55TqEgFx_!4|tC5pkRC{l{R^a-QDki);fEmSq)Z5rn#9tqHW0zir2BuSOtNhJxKgx z5)!@_ghmUr(eIz_e5pxaDkJDW>dK2HVE?;JQ1`kX=!JFxEASHVc1?huvS%QfW5oo! z%?7)KvUsAaeOV~($xf9Xw)^YjK;63{Llu8>=aocb7FKH^&3AsN<8(Z7_{sI!AD5t% zgnG2c@jAM$@De31{fKfd-$(H$3s9o7CJNR$!?${wPbsE0$?OH%_)DtLQa0G&Y zI?6yIi@)di8`g8JD$3N?Kti`g(U@x#d%W}+|Gd;3Uh&=e)Cne*YQlG^zSlFrt$SMF z-DeqazCZ&!e4q)!{O5uycTZrS8Usr8Q@}YzH!yF`D{8_+p6$A+fjo;tQBOlYdc3C= zIiOQWEFu-HQqD$LQFQ086??W?l={h+1}Wv?V9%w~VAdc8Pa^w)obOv; zRUiyk*o(rfSO0*hS{s<$6ALzNSp}XfGX|}ZLSTA!Beio*Ar;zmg4Y$6$-bNNi0#}m z4b>cFP?@n6N(wbb;d{-I_(3BS((i|St5Xr-*dBFN$Ix*_HKeb0hi@Zyol;B=0NI6s zKu%Txd|PTvrF}ZarW^Cn=lpC`mwE)LEA8Rd044P0b2VRDrIcEqG7GFy%m?(16F}Xq8GP7x1~e9zfE#Pmz>D>qM@caNC^RQiG5Z7A z%Y1uOYng&JK0d|uerl0|PBohOI~SGME<`7$G_yi)-|+gi3_--SMPM@m;Kl=*W0kz1 z{y5uEa}+Z9?J_>BXm1yP^ZcV!4bKzIUv(O&Iah-D?I*$Qg=qk5xd4Y2S>WdRl=AZ) zrl^b&YLWglP^fJPc52Fl-8&O0@6X=sTPUdCiMEv4)iv=2=R}eMwG+}#Mv~Fd9^XxIHQ0UrPP9sJgA0>`2^f#<3}dCj~3u#T!pXvp|3Dsuje;>VsM6S?aM zY2=~KlPi(;i&;pm)fSz!+liK#SD>$7o}s&sf1=w#Q?R2W$3JO4kE}MhqkE0>S+!fb zV3ebW!|~7HYw!%X*i;3cQJMyy1bqd%mX|@_t8kzv`e)%2Or0pl3&{Y= z8(YAE+npdxs|#qQ7X$6M?O-HI4(LBHrY^UZ+FC>$=kIbOcAe}N%J6;{C1foJ?&L6_ z@#roP#ESy6F4=>nD_F|nY#Z;Zi#xk%ogThazflW>$ zP`S>6%>4-D(g-}w$pza6{6W(VV~~Kk|EXW{;NiN9T!+?#LdP+`=ZXQE?|K5+nGuv- z)sFt`YDF@8k0Q79aW*ds@|#vDQAQeZRAx{mZ<>xByJyBTb|}dOE%+LX?g&CqMzIPi zJ`&CDHOlARx0;|5=6M5=b!ou)K@Rx*-U}=!9;a$6vw5QzcC!||3A)O9@P^sXITzXf&;Y)? zz9dLiaRh$S*1)%8gvxmOn3rPvicQ~|jO<>WLREej(dg}Fbdf4R7gm@d`}+A`|0(+&Qt`5eJ*i1YTXu|dy&tV3!(3dI!Oo!Ha7uUn_{6{7Ol`DT;say>s( zY*&Enae}q!8doo zNv9K_bG;F0vE4)cvutK_mCcdP_Lb<@VpnwfkvR%bazpwXSE7XO0F=8W3*p|Qh_~(- z646aTPgmKXy7mS(Y!ma?<6XT#w`8_C@b{q5;a;zS02+p;~QD2Xk zu!*xf*ot0Jlz&wg$;|C$7ifs0@Vk@jmhM(oHcJ*+lnt z&0s41-~=WAXen@;xgWe26#@@B6{r)!J^Ym^)vUdI^SaAHRx&QVA(w zGyg92ZBUhJv9YFFtWxa)L$X5&s~e2j-s<;!?Y|#*2FueZ>vkSh8Bu8W zL&l09S6=}YD*C4!rFCl+-(r!HUDLl<%JcM7j!B>dG%QK&vbmAHIMtHfR|~125i{za=1uD3x(RCXdpEPukuha=;Q#xq&X$D#Bddq5-#XzxBA$14KgDXZJ zz;b39a6|-bw(A1wFUx@WPwu=i?>YEoei=N7F9hxHj{)6vjo@6@Jz!aL7Wj^z01M*_ zz)!;l@ag6`;1XH}Z0wW3-KwoXCR_+y(LGD$KhUIlYdQYUe-yg?Dib|ySdGGrHX`?i z3KaRG0D-YwTc(?>aT055$oqaU*CxOs{+tPb!#l0^b?hfRp5=` z-29-$f#$itgg>}@_51PJFyD12lyA3(eI<*LyPgLAk!yz^f8B{i6i?%o87g?T=>~L1 zHyw{U=Ar8v_u1()R8e@16{^oI!zw=+yie2(FUrcmdvAr{CwXP4-~Bbn*nAeWiU+dY z9|;&(x)AhfbfY)BPa)U+wUm%aX>+U9V`n|uuf8qUp-1mL1>%3mi=Tu4Yk*mUHJ5^Gq z6e3KNS0WEbL`(L(ke9l&Rtuq$6e$PLR5JNafneDBs>GupNF4T`m z6s9P&NqW)>C7mI0;=AoeEMMt{xb@Y3QRmMZ_NDfmc=~at_~HEP(rGJSNemp#Sn9?a zVW4`dXzFOfmgLE>tL`(|x2=lICGR{dPoK}iUnQ~Nqtg33JsCE$$A{HNjSwO>eG}6b z^3ueRC~-;52eISW8L`Y&iM7oADmp5vvZe$L_Ot1Q*ok)ppX0lQotneRZLbr=f9(PC zeAE@PFy}P+bup2w%T6a{508?j)V?IZ(1FaWO(4z^MG{*ZOCGxwk?6Y{$y)j2#5gdJ zxQufohaU7I=fYPDvr-a8c#|w{Pd`^0xvo_FHOonKtf(k?pSMwb@v1`fDTxs2`TOF# zyg%X-R||G{i5WZMWy)TC{UgrYI!qLc)5MP3>ca9N*95sC9|WZxT|&;}fkex?QAj;m zCa8W_CZlUl34t+3h36)h1y`m*VnX#vLT`QYW2baBF_b4<{*}<4yIA_a%Ls*rmmF4= zY!DT%7K<;(o)L#m$P(NCOArfhsIY&t%f<48PsE4;^`iXT=h8h=w)4`$%3o{+4XM7U2;7^oiukZ?h6PDZ_2fMA5#xHcPbe zUoDPN(-IUZmF!EoD_9jyAk&;EnXu7?Y;iUs?LF58Rjo?li_J<&{>ROw-PMs|&?hBv z#h6IZzfY#97^)={Zq^rPK0YWhE!P)c+a-xHVc8`bE%Blmm%6m~YqJ^sTf|g)OdO%# zQ`%LcQ2K81J%^W64Pe3gOcvMG~{@v~YFzZK3h8E>T=|L0IkbMwq79i>TZ; zBV~X72>mw%#r}TV zL|5}c;-CZH1-(<*F~CR zeiOHt=fu>aon%!F{Qq3Nm}LWo&eEQ9u>vIIw~+bcu9IYs6U0#3?B6n9hp0UZ7KS^Y z7Gl0;*n3`J4$aLoM41oGA|0j5PFb#Gj~ydexKP7vcD6Cq1FfvPw42Rd(a2y=4P$RO zlg-FxwsvW(eqIc-EIq*PO*p}phhJtP($1Y*UunO{Dup!~SF#=ZD_EpSCR1O}*??)e ztUTxm3;)u>9Cyg0qMHHEyD%IzOh@3b9m6o8a~Q5#s)G+ryVwVx1{PiPiUps4#FG6h z*mJ2rKq>eXi`!Dj=59@5AK#x~55+82ZWqbE6i2gx#&ekBu-?pOS1(3`eMMW74$;P7 zAiEj$S5#5|D4LkZi)TrwcrAaAn18WAT<5<)%>F8f`H!Cn0YRlC=Hqm@o{$2czFdXI zBju2nRt|eY?t%QN|6t;(SMWSn+J$IugaHx%!6DU3Sd>r=8OtGfHaB{(8QzQ8n62~J(ZIn>tz?xbEU^+cB^#1!2iB4Cvm;63`ABl@ff?Dn zU#YK8>cCX|Q6(N5nIbNq;=+0?T$y%m6#IPa6obseEdB3A_IYa+Q+y$VdD;E(W}+?b zig!fYjgA;(Ct$3)0shufz|UR1Fxs`1EkE81v)`y;`}5wI`>{7By!SjESzbt_hG;Ln6jFjD)I30VsU(dHp@z%%`_yNS%%g`_NBcq>l*V+EW2dn@Hj0_ z>XmYbdA-lUm5QfeIrtgeIq(!5E#JZf`*)y`{0=UtHNp+YMws}w5r}ysINy8$E7&vG zX7dyddRN1sS_a{EnGpST6+Ac0BQm>(u&&#`SQYd^PdOv37(51d+e=VZMZmLT?D4Me zY;>sHhR9&l=Sez_)OOSm#i2w_q-6wVukJp zeT1URMuJhjqp+;NOQ^H+7O3k_p<}Lu)GKTt{a&snCVGUN%aJ9@Ve5n;%iAQ5cG;zl z7dyp;iNWln>H~IX?n|~(_dYx4Q^$;gezLQ3wD7O#K+IJdiKhdmq49{d_*i})o+#ae zi{1RtZJ{rwYplnbW2^9A=OSEZHv#F^k(jeL5ur}6CSPTy0q0q1 z*)67gLk6>43{l6+3SaJ(cCGY0an}(~Jh*EGMyjpCL63Li!WrQ>L?;w8ckRG9X?EYH zwS}EYbYbU?d>1!$IIzZq>CC5YKC7)ZXM=VQX7AMOS?IHyqQhk~(ZBv?Y3f7KA$Y+z zNkn{!(75oCuxDYb(3R0FVRK)Jx&uR)efJ~wwBs(Dzc`N_EGS{^?RVIQ;a`~6*lyNq z%|G_UULO~vjlw}Yrs7O9H`HzP#62@sqxnEjJfyk=O&grhbD=r@Ijx1AX1#F8PIcD{} zOOm4LCSJ5QU~f&G*|+6I?11I~{B_kF-HuK~HSKx0+t3}yEpo^9q`BC*YAtH@1Y?qK zJcgwm!@DU_crJS%&McaOm-ZT=MNdD}=v2TBmGbEDrJCs#$FRL`wzC$EK*lq+v9)=l zq;v-x);+E-qwR;qnX8uxS=05&r%8RutA=EuPlUCQxpkD7kbg$}ZJs8+4XYB3)PIZj zhfS7d*G{v3Qg8g8OK;gsKQ(-J*$AI(GQkBKhhXqY;gI^2lA>Qv9DBrGvgsT)kExEK_@F5KL|5l^~HJHm$>#kH|i|$GpKuf-hsV~lonzIxHrE!4*vGEWNsKrQ9&(0TnesHn) zwhDXIZpV&Fv*bQw(j+PK<$AU0=cBTtdVq!$W!Wk)@0+Hr%04!FytsamFDwTJE7e}JiFMKE|3%&LP_ z*&XGs635l9rJh2T#QaBsP?H!hywY$M7Wk=?hjWd|O?5j`cEN)rOzWS$_a*HXV_=9gl*RtJEb{oC?*w zlR@@CA~<&?f(lK9qi2so*%fI=c*%OWm*D`*W@tijel6K@E`i)jT}BFgj7f;=Bw>!n z28Syzw~Ld%Zei{7OW4a{zgT>fHV%KIgM$L~ux->3td<{)l4Ju^9oipP?UKV&vpZPv zrY5#d;W^vKn^<$hTUK!5D_f}8%sTJXvmP_X6bFQ|fovR$RZ?Xx;fl;#*Mtp?c4m3+ zrJNzXFcvibG<#x`$M*lrVpd}=vlSB_vUyjZveN_`-seC=S^m^|0WA{yu?~JA7^(|LfP5TAi~|Qye>}6~dgxgfsi4zAWqa zVm5E@diG|CAJg|vV)rlSGYyYqcD+kv$%apvle5&*B+a5{WZq_vAD?DQ2c?<&k)`aT z$vw9JNClgG<2pOGFPFttoMSP!%Gp@=0(Rx=OSbgeQ)YiOn9WyBU=z|0w;kE^`lD>~{%5RF<}TCpTq*TO*)z+5d)e-qEH?D*Vb(BjHak~7o0Ype zGjHz9Vz%kAoZnZ)zkfAZ>OpVOEx1FlT)0!PQrRJ{378`6nGB@%+;d@74=rst?JQg= zH6YtYbPC7vY{~VaD6)2`Ke-V$kIa^KrjN&NCCze?Bx=k8a;QR`EOfOXzWK{Z{^GGD zO?$QtT>6t(uXs;9K0GBpKV)Hwxdyx#ssLu1>d@!=Uvl?LHPQTV zi#$L2f=mmzOjfMSA%+%@NM>*a(f)N?YJ|-vtziK~CMt$JTy&fS9S&yxT3z^X=Z|P6_2I;aonSg$Q`wk9aV)X^8jJam$dtax;Q6CJ*gl_MOiudDmbEM4 zq&z+BS3Vr0Pj@j5$9HUUzz%wR@dGJ*xDLKfxWY03INr6oHr4QS7Bb2$U(PQuZ_b|CB zxlC`KjOgT`AlPWw2`k(QTkf63wmOuOg;MtQ2K!rV=b!_E!HThD)kQUyU!ludP7NX| z@4pLM$9|UB7p|8yl>8LdEXXABzl=yq^>@k1X(BhXFMv3V)!F;Ib*moAY7 zOXraTo4bV=`C}yhj}rXyyGNEe4;3e*zrZiXpoT39GVE^uMNVINEXmW(EPOGgvs?RnxM@xG89 zA|ZN{wg^f#TC8t3uoCCeYcOGX%o3SfckZiQXr*xP2vfo=4SCT6ZUXvy! z$2+iW^C!&I;1atiULpRQ!dX}CO%^=$xM+NH6`OvsO>pxt!tOQ5Y%8CVcxiW|%1aL;FB$TRICotm_cn zGL=jltszzoiXmH9n6i*Z@hnsNoP0Glf<;CR%=peEmgUt4&fP6$adxLk^1}%vvsUUS zO8Q2=#?EB(Cq5OjGIgbn=O@DMpl;En?>J%W)@9^`q!;u{mm~K^pA~hKoyGhZ2EwFv9c(anJ7*{%u%=Gbf8g*?%8fF9)Jh@*J9BR9!7?xkgO4Fa7Cyk6qc?O4q5Br zStl>}_VXya**qTK|F;L69>frfxxuJ;dL($BS;y+gUe=To1bQTh9h&ZnZJ|l<)$W2& zl@*0M!wWNCah7vTk0~(5{A}p!uG>=Nl=3_43E6P#x}ZOhPN@; zT(lOxL`)T|JifD;SA)pCLsQ}9lL4qU7bUBX3sCju6R8#~@wBWAG={ZH-vjEbHuWJ1 zQx9XEW(DM~l_^OT4hf-0j@ieTQ@=sFT{J6C2&3K77 zo)UJ;@4%6h3J7yC5OrJn;j}sHS@G8!5?zmUAuG9o6{Oa%Zr6ikvq2;4m3G=bZKAcf z!?>Q^|JIM(x^_@(F}g3VxEsX!JevZ#*PGbZpqFg^4FAY=RQAep}urF$V)VpA6&PH;ZUz6_cU#Vcl@%xEBn zZzwT`YKMIrZjymlB{=O|9yvX)kjTv{VFS;7W_wmHgVe@$wprs7^DI31lKN}CQE|mq3)sxh%*_+Ivf(n@s@aSHgLc>#D`># zh=UPRw3)89C%Glq3lR}IP^)`_%(Y8luJ&Weo8S--n+#diUu$grbpmF&7O}v}-{e+# zD4X+R8}rokf|b%P=;<$&!9r*DN~iQYK~##fFlqxPaZub z{@BYi8Y-^#W+#8VMN{ZbV?;$uwa;Zd^$Y)EdkpdC0G-2hQusNhgpjs z;qXT3S>gBqsN3`%I~sb?h)p@@q%~A@NlODKvwYOQy^TF|qM$d6v6art@cVTuST$u3 zhAer-)S`C5_U;H&9-_na%=SXG=Ubd^JQhxpKzOpa9#`sF;i`tcplE!El^f|0eJo-U zrpuG!w^0{2S6uXUJ~zLv#K1=U6L`ZYn`-1v$ZzD}wU!Mlq zyC(5|3Fjbw#vO=vZNt$~!|5$EIi9M<+2*!Qbjg?9Jm*{;c>fndaNQ8LJ8B4iy4*$H z^&SSAQit_j^)^^oX3V!P*P*uhXEF5B0N~Nrh>3zS_w%u$+FSl%^QTFmxFZCXNc+_j zG*nPGP?L5qbwd5KFVW_wIyL!_f=V)DVepJN;uo&K^9{A==!scmT5t`Py*mox4>kN1 zFpTtB`xaK@81Y<9RY*|(5Bki>z}g-o_?FO{1%7x_8tGt1Z0gQnSboDqLg*H zrjyBG2V1#19YSy3XQxk;V)JSlDwkO#<*GLTFqfV0@M zd>Ck-OebB1qshUji&#GCEZEf4iNhP-W5PaHYO;C+A2uV8Y~B+FN!n!?V(<{$7i8n} z_qODw$s*7$x{J3J0wCgH09=J&A>PvZ1(pI>Jed2sDMk zSiipy-NAq2#(Om+$tVsKWPalMR0W!EDB_lr3QQ&a6gY?JV)dIfOxyPw%gj}vQMAdzP zZ1+9M+Mj)rcIGBP?V4ZYbki_y)8s^T6G!quQHJVo1bjUF1kR2QhR_8vOsPweY&_V6 z-L6CE{eQ}Q+m!1tELRq*oQI-$*%f$v=MS#({Q+*)ld*NxalDY)1sXoVY^tgp-<_#S zB^oQ?b-6PSRg$M8wwxzcsxmn5a1bipT?z{0q#oO~1Nle4G4!614j-oR9j5QSf)+0f z$e3G|5Lx;UKaTnWp7ORl$7UIwQK8PCE8G$i8wZi%-J|fFFpjp!26K(5I(U7qFW5X> zg-aJ)%yl?I8y?b9?nKb{aMlse|YJO&*3;%AE?pI34Eii zCNY}-8;&-=#kfOi(5rHLvGKU?k~?1xku6@uEGn~%eOY%KLZTPqW5dNl%-#W_Q~DcX z$R0vn_i(*u4+Ed{b(SOlZ9ol%c4< zzFH2$Lhq8PL6umQt4ynZ%5xnTHJEGs9Au7J^6kY55cx8c*gsf~840&x!jUE%J^49X zb3>p@zfR{lms23}T?@dn1AIo>GHUeIm9=({;${2RP`#=kdob1~yeV{cNuge!R3PH`DeG8(ug);AnirlPjE8X(uARoSj(o5STP;ZGQ z9G`U_|F~C@XQB0QAdB+oFXpsivh;UM2%;m6j`5q<)Tl>y0gRb3fqzlerJKh}nRi|x zBx9E@xjFJE%&F?b8$R`+F@eAF%-rWt6#I+KpYs6?rhC#=`V)B^Qur}429|uc$B{K2 zEHBg>yqxc2!7h2)8l23={%6Ej?N~quO7Wa!S324FgiKJAaxeGIoyF2+Q^07zQJlK( zH*vq`3jQ%U_-<4!S@3lS@t2-syX6(Rmj6YlSx-T^K$C}A-iM$a(zEx>blmo{nf262 zSxg>Id`g}iy=>Bp{I+r7pBAj8_WP6YvUM3wnLGt92Wil~Et_~y^*=~090`ZZmH2^d zWh(is%G;D@(8}pl$`6!!5<1q?ldbMt;_gh`GSsN0)pk64@*+yqC3LKT4gZ|C6BcX8 z&?Re}d76J5XzCvI@dJSqf4{V_NqGo_%hr%6lczXu_)uEd@fQ2`e$GtRe1fx)!RWhhCjN7I z0?%HZB+=!T-0qti{h2)Phxmp%;$mQbf%k zv*5Xk9gNHj#H+Uwv08r&-P4tbUm_plp(A6U51&D2ypH0d&5psOk)vs+e;~JozBK3g zCg{K3nCFzILfzoIkl_{1r{=}chj}gNX6}Qj)(@bkx&>zzt5DgUr`eQa>YNQ!rA<2)R>k(UnYZ8fJp%EvLrvlELl7gKttKR1oH zBQD`bU`kQ~P8%n~Cd$--!q&c=c-;cGsFmUfr5YTvcQyT%?af`{r7ZQ)S~OwV2tLsI zJel&^o}LTyq~3+%Ks#^zx);9Z{aLnG+{oix;213 z)}6qMC5AXsG7k_zHj7ajJ%ma z!E7rxxTweF5^8n&w7(!*eg1NbLhAkdHn|hZY<6$?H=$Y?2^qW;AA2mRm zR-fpES29cZuj_Z1_tI?IJt?1GesKhfx}E8-oN!*%5eC1ehttQi?(lo5W;EDvH8p(n zAJ6EuhU!`mr=OEAb7K=t8ghFzb^2V!@8Cju=w%4>>Kw~wZ(`sxs~)zL9OP47eW=&= zc!;ej)&twg9QTm>#&e=@+b=Bad9p^CM>ImxfpE@5l>m*5?=?9<8 zipbE9D%{v=6QpJ7&{XNS*nWqqIIEq~cT2Uo4wJ#)+wb7zm?>BmaFz`$i4pB59s{{3 z2dTrk4O{JPY0Td(Xx5_2UEYMzlQO+|dYlYL$xwPHDh2vwI`hQdisHMMXQ`KM2_Lec z9~|0QL`&?dc)#HX;E81f-I^cC4@vh4O-BkS3lHHk+17k}TLUfFx0RQ7s_-v0Vbss_ zJ~qDY#e>! zI*R`^4&yiK+G*M7{d{M}0p3|5KVEmzPTpM^#$zWnP_IfoURf8+JJmnYsnbt#?GKw! zVQ?a)Dk1#RlEY-m-O==ihavCKyns(%h0x@dp?uTg44mJpPDkEtBibGdcwMbmyyocu)Qq>&)#l0P1hSF4o^h%j2;yhXpJxvJXCFqr$gwao zZV`L>Sr!Xt#X(?aKAyFG0GXFx;Lav1a#hKKuFP1#8<#1<<;lya^ULF0!{R#V-kMIk zJOg>vJVq3EZKNye_w)Q8!LUM>Qa?!MkI!zRTfEZ9!lnCpu1qjpb!iiON>_5Xs`>QG zl1x~-J(Jg%$I{GNPr`&_Jis%ZD(}9JzKgZ^Oj#$YoOuhC*6&2WDizA4yu2}5`Cy*6 z7j<1{!mQhW+0!Fh$X&fa;lGb8aG z9Hg&m&#;ndZv2hva5_AE6EwZ)L1URSqSb3P`ra=GU0RRfz?8w%E@ToWYrMjXw;X7| z?u(c>{~>lOZKK~CWO=dPGgKKnfsXSXgb|(pq1*KHpd#H#yJ>zDjaAZ7;adX?)n3Zh z9+B`lZ-eQ^;ca-SO_sY1UQc_r_2w6jG+>U(box(wISzby1E0h<6L0GiP_V`x->;p7 z?Qa*shb|vHQ|Jrsrl&~Y!+Fe_BKq0intv`e=I#vz^nLbPotAv7~zMG(5-W_Ld`%(fE2KWGHJyQsnMDV~Ny1rqYi!W|Q} zPk_@QMSeBzEzGcdOlqdz!%K4hxc`O|H3elUPjW3E|LhR;`7@OJy-o$Mg{Mf`reVDN z-C`=2YQPUjpMTXc12A-@A^vLbh8STK|8}4c)oAM^ebAV{K6?vd^aoR;A4&XHsD#Q5 zYk~R$QT*4qN%Z9TBI2-6pI5%Q0BYW4aB+MS<|w|xZ?SQ7K!_PX@Z%q@O%A0w9h>>7 z`I>b8qw(l?q_1>dm=20_L)Zu3@uHKIOM0ho8_ZK~!?hF6lRMf=q1O41FjA!-_t-m- zE}M3ju@~7mtzj9!$~UgB$NNE5B~& z)8{f;Uw97loCouMSB`;&jVx`v;lq6sHRzPu?Lfn&40Hc~;Aqqz8|9u$<`#y)_iQWf z^N7-LjUZ%+4d@t1>Bgyz=vi%o9d+*D_tys6M=P+t{l@e8ribXC752PV;V;|1G?c#c zXyPOwj|P?;7os|1_`66~8hLyw)ace?%92L>cySxudh86296O$`8xTaN4NAunPg9= z(=lG>MCtpVo+PQT8NH8i@ai(hF$rDd)ERZud+iK~UQ49y1dL()0?b;~gNJ9Q(kFGz#M zO0VGaXL&wT>o`=J%}0``%e>8hLeo7hKJAw>Xk30R^)L;kKEgJ>Pkkpe_VJmTF}?4e~ItMN%)~ZvuUbuLlQb*HqZC7q5E@B zkf?pie8q-#7@)iu^#;4KW=S4cUb>ByBV|bBi9hgf%wIficar>=`J9B`{Q$M+mvWET z6nHarH2pTznLk=@ifOw;Xn2|hf7RZDA5JIIF_Q=Jo~vfO&G9g;_NrxXb{^z1=Ko=|we1vdk#cJ6BhJz>Ro8jwQEPf)ln4onBKU`m$Eo?v$2fmG@cVO-YG=gq z=fl&9@7R5G=e0vTV#Xk17#K|VnY;0+a)WUB!TFSov*b#)`fz{2L0GL2fEphT!_94X z(R5uH8Rj*SZhacbpWDU4_;jGE$@ZKrkHxuWVRX0C0N(D>kFPs@l&(&>hP(fp&x1b& z(3gfYAYDo1Rza!M_KXKCy;H%Z5hFTm`FdQsFPdk_WYa9YyLiOpE7p&Vf_YN{p}IMZ zi8Ei|_%XARCYpY}+=E&E4t$w2r@66-yr+5&XsoG`x=bFUhg&6# z$ZwW%91}r(SwH-JaS0S(?~O;w!m(eH9Q|}D0Po+9$9{_qXiRl6HojPkTNmrm9~<)V zua_;V$6bI)t8GZw=u=oQq>4DYl$G?D9f8H_XVHgFXBL&-;Cy2TJEwesZArRF`cKXl zt2~>qvacWQx@f_F$4BEj*B-c%d=j$LEx6^D!?;r}nSQO@#4Gmb;qp%_sN?ZHd{ML! z4U2GuF+;=oB&!p2z@990`#gyAJPB>g+s=rY9XD%TK~LCRWv4bS;S-L#Qz(XDvn z?S~QS=C6Z{(~q&NDh^`LDlomE3YK*KHtFaUf}RUx=;CZmtb8$xI~~iC`Zucamr@*e zsCq_QmMAhP`(`5t*Mu_=|VSbZGNJ z>C7>QZyfAPb-V0%6!U|^3nyTAR1uCi`BYFU7y+$0?}>GTJdYhsD5=%pJ6C)sP3Ie- zui_aNqvehJ;97l7;|Q?8SC0m?15lBd7lu@fW9$elssae-Y3 zEI!hS;p&U%tQkgJ=3fcwW^ASFrF`3CuFvpj*BW5U8XrxtashB4TQPj;ONTR+j zgx@m>qMzbCP+yF|GVc>)MZqI-!u*b~*j)z~&p68-_dX4!zENz+7iFH-y_9ZQE!}U+ z9wh%Pov7dPOkVtRBCS671CGU?;#ZxdJ8P3b!DIJYp644))va~;(s_y~yY3GNrdP1h za|YAxH55EN(}+7t?{pVxg5Ad}Xjw3v@&hecG1CHvw)LXPQl9)??E#p1R}shCOrqS+ zmtRz^hu|G;;9PN%Z<7zBrn?WYN^cWBb(0Q_AM_IM^-`zXFbl_BiNlNKx^&WtVLbHO zIQn!^f8NoxNyw3nf{j~W5O$~z6&)92ut65x8To*Z{A5ih&ymnKQ}6O~B!!m#*2h7) z4*bLKAli5RD*pBHOW5@~9b&G@@w|#PZ1rPVx>EKXDR&6tUCU3=Q}+gN(^6@ctEUV6 zR(?jme~EDZXgDqi8c5YN*6`PreJS))q64Ohyf|N^u|tOOd1r&MWx6xnpfjJV^tYlr zi#}r31y0WPZiX-SO!z>-271EF;B>zn^q)#mHOYkLpON8`=~=jMWH4=e=)hl;ox-nd zE1kDPkq7%9#!YT_p>BpJJa$&)RmmpcnA8M!U^uV)b_e$5I*|+4x>!S*CARJUBzW14 zr2F5jBCnreAZmPeb)_`el8P~jEC~4OLsttpAH=sw3TP5 z-U6RyZR%&>&-d-OrY?rj!cRiDQg44MqoKxgG9I#jsk(H3*Jz%idI65q7QlUG$hXDy zXP$;Dsn7MH{J*(%XmQt#CZ!p3Mf0m*d5ubOG{$_)mDO~kl@6bqL&@z^b6{JzJJ$0* zByWy8ZhV{$i_FS#;)_$zujdU(3vl307iZJQyS4G&!ah7i)TI;G&BwrP*9pd6Kt19G zkE8B@qJ=46DE;p=?93cmpZ0+t?NTLFN)Tv4JfyFg%%_yA&<`2OLh3tfu~f?59eqBL zJ@Fhz4O3?F%HJw*+O`3{*Br!u-FZZ@?i#9ny8ygE3&u4L!Oxo$uuDE0LI-%UK6&OW z^~+H5dT1IR)V~J<4~GlWE!Ftywsg3+uop#*x%|-Jq2MpHgNtb+${CCh7X~~5`%+zg zu1_wE*tQfV?CH&`TM}XQd>a^&`G78nQkeqVm=d~>8-0=la zB72lawXde3+f!kkS0=ZP+CleUjE5_0C-THR>HVh?lI>sB;a#091|K#-%gv{u%w?!F z&XULGuCId?7y9t+i;^HAx(Ga9Oyvie7PaUc1+ofue8%!I^vAan+%NTBwYV;!Rja3Q zXI~>$waJ9aync;$H@e`4L@RnIaR9$ciP#I5@jsU9AWI>Xc9m2ix> z9yaDW!_BGhds)HZqAPDoSWLG?+vC`uC8(470OH?YKzaQKKe;T<6Wh z7i!arO+$$Jv=B`8$&o{e~>%FI+moYWz8<)Y!uh2P~i+ z%F6g9b{Lm&kkD%}UQF?$F2B3I4KiALNdNQ!Jm$(6n$z0G4%plA*1oIgZNoTxvtN#X zj#@`QZL;BQ6Aj6^l6w#^TY5g-HJ#pmr_5)#dg7_q<7urfUqHIyDi!PwdBw6Dy!YZ6f}QA1`5xs}_iP$6ZWwt!>5r0xt3jDs&Y&Q z_p)5H)s&;HfyZ!@nH7%PtVZS9%CW-z0OmX$Lf;OxMh%ru=;81UnuC{OuN8%4MC($r zOgRNJ&ZdE51Hpx_l-SWLli~h)Cw6_p9V|=w3ZDW7Gv6-(Xm0k0*yjeaCciOYzGE)B zt#4-&cr~>9n&Hvf%P1`U4E`0`#A#y;I#1qhH{|Uz$I!nM z&#-&_`|%eIQ)qIN3V)CpK}xS{QD@34R^&d)1E-^DpSU`{5Q-?whuT~ zM39Koo%rmCHu>oA2ySPXa8Z4#G@t(!WR3drY>xpzPR;?hOHp`Knk}9%Jp>AV<)MFu z3yT$_p}y@tD;d=XLw(Xo)9Z1tVDli1>d?WL@%N#Ba4<&ZsIdR-qIB0torJy0;N<#6 z*!IgF?QXO{V7UgGYL%hcvo5eb_Eqfc{T{Oyyo1*7{=_XznOAk@!o;LN7(ON)r{*3a z1IEaRO^)p%Qq{ry7@C%RIDva{~Ak@PI?RP3yZ}`Z#I2bBbm>CGo|_yWVUw@=}NUhk12UVdA1wjvo*;{&}S>B&tkH_8`$yfHDvHR z8+Zx*a8+*|^obb@wiRA%;wovb&0E^*X^j+oyPgW`&IdEgtZQJDB%*wt9?Sk60?jLG z*p=(S!o3~miJxySE1fcd^;)L~CngxO?&5wJo~;kkgbF!^6)fy@INndQhsduCY_`YX z6Q?n_YrPY}9AyalJ{u}s(y{scdJJf1Fn7*m!6|M6%0(p!?a%7jkGI|=f5im&mKlYc zrFrR1|JK3i#`o+~%rzETk|{iY{7e`WDogAW;@G{+A!<|J}4n`?@0OfZ;RNdv`Y5-STULTOdEzAQ^qR!PBH)53?M62Fj0^Zm8ERw z`G+joyXYHipaUiHHcC)({}uBvt}9L7_kaXOI11}pH1WQ@vOzxlq(;B4$v!~A@ZPGkw`0<~Tsq0nYjPrN#Mdg0xy*P$U z+<%Gf6jfQ-f#ongX*u&4ua7NJ0i|1g*9m=EB#=MNl6|&Wg^L#jl6ls5NbHINmT~2l zm^1&JFjmo8)N&ptahCtfKJ8jVR?b;P%3aopid(zHS^5A)h3%bOF$Yaecq3>zc1 z{+JQV?SE59{6B`yG_IyLisH>tDjH;nXci4JG@NH2p%hXfWhz6V6iH?oN;FC&Qc5a> zCZ(ve_m!zadJRQFrjR5mW5|0x^!wcpw=dn>dG@o{TL0CcD4aQ|&c@!4W(DzEh28BJ z@qWfLHm!3dd&XvAO`d_>%cAju(|RZ7)9TEU2EIhsNH=up9S>K$6u4P?!Z69?7|JjG zEOfk71FN&DY|M@PeKktlIFZM<7eT_m^8~nuMz!>WO}DU(7gv0S}q_u+jJZS#+}4>hy9eN}Wz* z;d7Q@%!f=NZ^jDtE_Z=oDeuC~Z##;?`hGaYL6&V@uLNDoM{xh@f8*J-5lrd47MEcy z$9cV4!FEl1jTIh0xICp8X5;k{-~HS#Xv`GZy{~5Dl8O|jbh?{W*O=4JEo!*$*$?)< zI|F;d?a=to8ioTe2&dYcxRW>KDmq4v!do)B%)O5hSC*`VlH2dNEr0qmDVrFZFymE@w6)h1?4=fNm_e(*?x#I|Lm1%yc3hbEK!li2KV`0oW zwk|Kq)>Gl0;Ad-w!*IZ)wqrC{WY!?m)KwRp#t*@$$(AfyA)0*+T8I0ThT+7V zb#{lIuVPX?+RU-ONZiGH;-rK3@%_lBc8OQbn8)n+im#=%f~HPE#nOIW@48aJ;H6T8lJ&ok+E#& zDej6-X=Aa1JTo0MOt^GA9-lR~RRpAavGIGFgsHm*vCGol%xq2|Q-~Z?caXrReW~$0Q0-ImXlVB7Mju) zFw4xt!l&U{xI1LJ&G{4N?5pE&cIfCAPSc=SaM<-(u$8~cEmgQ?>mHI)zM(jjyLfI0 zm${7uodKd?y*3vT)MxrUiRcq>E(XLlvD#j6i+0WsnD$KZlc z`82Nla`yu6!Q+k0qD2u4?1$mHR)4!T%gc7|9=XC(*2`Txeag-zcQ@zneV+Taeiqv4 zbyt`zdc^IFJ|v_@?BzE59cG&vUD0f#1)5%0Vr#xiGyCa%u{QW^h0j*T?4;rasmBH7 z;~ph)o2Oq9+%uQhsT-Mbo8M)LwIQeN(*FzRQb#`)7T-_jb`3IQ?z*+ z$pzfu0y_uUHD(XR{w+I%%dOwJgeC({*87Ig7&hH*J~xSLleulDJi<=o0NL3|4cuI@ z_{~6+_d754?!+mQcFCW!*Jysi|POPv*i5^k4&sr|sISxMuik$Z)~dUv>w zohO7g->bqD^(WlmP1>B(#37iY-CxYYyyfaY^~1Zf?%6JmPUk$0orM01KW*4E6>jR{ z1iPN?w=0e>zAI?|%N2%&)^U@%71(OGH$wK(4}$6)SGyz0Dcrkb7Iv0Ph6xj$&Q-h! zbm3~#uT-qP@3m#Y3}xR zc|l^gT!=Gr=gf!Q;Y^omA+NSf5cLIvn(DJ zZ*D}_(9d>-0fuPP{#y9*sKeI3qDfFNo{K%*%6Rzn6e01J3aTCyxu~;t3ph5PVh;BJSQo{BlKWQJnwbNWtTJItzBgRGI3JuhPX><)19;~ucE721 zFnaIGTFjz2J&`>*B5Oa!Pc6dJw$IUbr7;;a8B(8X6RBq40xB-^AgLYZ)OYk)nsa16 z&6%J;ui~y@(e9skukb#uQBT5d{c!x-;faeMW#V|7OmyF5JK8Af+_+>{JrcD})X6~1SnPbjy zhb?^s&0EWbwNsX{2d{vgZ*36gsT9FzPN*=!=ptBiTm`Lev*ms9X!?E+8|y>Q-RykNn=F~IA-U`tIs?BqH+ zu+P}(G+6ZIPjM)rypLDO#pWu>WR%iiqd!!!_AA{!bb}rQUMJyp7WMhzMfmWvWT&!}Qcmdk=%UK1NPbr{&3*8%qjLm&*QSj-0r`@8G|3w5qzo8+_E z#l^Zp%a461*&Kk1`W+ZN*qHqP45WpQDiqSGPJPB2QjxqqeJazYrrmvMo)1TcosP8U zwDNyB(gNrkh?#G+8@{X0Dh^)B3s4vi?J;-_LH`RMw5BH|#;VFHU$N zXCJDj2VzD0Y4m%QhCXF~G31ph#f~|IkNV__`DhgyR8x;noEPJS6D8;yjp$zEfKO&V zLzh9LDd6e}({)g=95Td7Z7F|Bt$BKl-3siJ}*yMNqv^t6%8zW6MY z9ViE9OGIDHl5ea+N*2~Pe_%g#Z?NkL|JdgMOE5kw3*obsLG_s&lv|Xuj8jLLyAo8m zONQX7dltCwZYOSul%eP|vh>Mo4*hcVp~)rlX_VPwns4bqzjQRnVV4!1U++a_i4*Ck z+YZ`!EQ3~cMw87`A6jPSO=haAY2zGcLRo7HQPQVpCEeIIPQZVw_TWoZK?o7Q`~G{% zP^Y2{7f(HjMyJ2vn?f-QeZv^5&RoKgA1>q1NeQUz(Th*rRVj67J+4(&qSS3Rq`SC3 z#k^9ck;WWW$c%sl?Xz&yQ-(Kl7qwH~jdu%Lz~>e%;?4CI@^ZgidH?M;{M##|dEuG{ ze=YqJ?4HyJ3kFHx`oWX%;m0wUDh3`TgXN$kaR z$-^yX5;NCPlH#rrk{wI>OX6eNDQ4Y$l1@pbnd;+7+EirY#Fd~`&nRKoNM-gT{yjG< zeIJ{yqz<_b8eq7rgZ-Rj0g}j>uxi;*Sp0iA+}@fDBfh%8<}YJGr^FQar4ejPf_P49 zwg@(IiTE|?KeT>SjNx|eSmo4#3bIxBdE-koy7mie%|GF+^1FEUn<8b-o=bZ_`O}mU zJE^NrGDS^0Ngf-IQD58rl$5za>~FWGBPz~fZ?Y5lexF1q`cI+CF43DI{}b28j3g=j z;S?%;3!7s5(imSOs@>Fx=O*t#-_|JHbZ;M)+}ni-v(95m>0^9dnTT#za&V2F9Np4= zj`eeA;g``E2EHC5MRq(?xtw+!TJpB{TlXJ5_!{ z>0_|;y9m#;GQn)revr~X4Bab_L-yX~F!6^eOd9o=S&tL@1C8D3?m?t2VV$&Sw2DM^ ztAXT7g^?t|e~@HJKV^yZ-yV8XSS$LF4^w#CQc>fUrnIR6xO?s*OxP_50@`wF1sPn^ zdtcTwAcOte|A9^V3ZVGg8ouc-hvtKEFl$Q;Y_Ki_HOdD?+09_yumsGHGElSL&x}^z z&G@g<{J7ut1Wme{LkL&sM2C$HbbsAR+>(svn8 z5#=gWl=cAw8^o?=cV7$<^fBR26bAcxVeYS6XfjWoj&)o{xr3K*S>qTo++<0glKas{ z+iMtXUBuOmoCMh$FTgKt1-^Ta11}6%#mgwH@k zBOeK)f35|Y%*9Z#W*W?uY=9qS`{4ZU%@C(F08Hds1-JJ$^xxA{1a{v@r(&=q;*x=6 zeTcq9+hwTasfmi@j^Arq*-}8(X-U)?>Of0kKI4wzXV7y%k1)J{bH%99^4#(GYg~3{ zCQCSE0*PZqR)dWt7`Sf%jhHoHQ|Z6?zn}!;$kgUy`wzPKr|cB)Kw)JUhMVT}lL{@6MtO z?F%%uT%bX_uh8O=7ifuLF3mGLPQJ(XQH0&<6=`$2 zDmChNqtdcdIKg-;o^JLKJbXrRa@qYcaN1A7=%OL2O#Ojv*+4gbF*5TXO+8Z_Nmpek z-8Q-_+?gr`8&wj(-b!SN84cq-mEHL@KZAJd+adhkeyjL-3*Gs$CsushSap7KT@&oj zj)$?US3pkHY-mp6pxa>)7+l!^35zyBPt;1VDN=!rcDjQ7C$YctG{|e~Dl053+fGokDgdQe^u?^14uu{|wfm@91{H@9R|Y4yDg&6t&wqen?~a zkqW@9+SpN@VX(;F6F&Eh2US;FNHg9DQyju!>Wy%iMaN;MOD;t9iGjHn*1^4P%OIJJ z0gc!@?9cc*?ufQEe(9?~x8E}I54NO-Xh+TZUNr03csen`isVXHk&R?GX$B^dTi*hz zG`mK{D*sXAj0VbStf!cbwN(A%CT-WcO!-4I=e*W!C|7ot324~kkVj+hpNJA3kRxNaL7wfCo85698H(lHdK??6FG z91Y*U7O!-svFO`wF#BR9crZnN@;Vc~ZNW@F`s8vR&-n6-vz+prkwkp( zONT*2lr1!CTxQxOvpI*#V&qQ@BP%5l{;lswo}tckG})SNKLr}x*NOgG%_c=*8>QqO zAgj^Yl&gD*?$li&Zqrp-x4w$rN>x$fSP8W)DIfqW-|t>rQwL=6L`b>4f5;y()9Gl_~$EPlv^*>v`SN_mN{YZ zENW7JimvlLg?X(7ta99b=vdqe^FlTGR}pr+%119g-F+eduE~QpzvINamJjFGHNSx@ znNyG-V-LfEU$bJdGct960d)F#KyqC&sK>>_;?UJFSY`_BvI$^8CFjw9$s8JEaG8Fs z|4IUrmjwTjmpq&Cpa|#51t1=ma!d|nO!Gl@Vt9!!J@o&&TXClpav7q3O<7m??XFAhqORt|W8r-#vt{jS{ zB-6uGr*x7o9VjB_);u!QKS#z3FNvO)OEf{hkf2`?1u2QSShu4TKYu+JJ$DG8CmhPeuKx6?kNa56&n%j3=%>z?efXu~K^|Y5F+R9(70R+vG+{3Crlk zK@al&y?_*M%%jpRjVPDZ%#95n4D!0!ka6ZGIEM}8t81@unj|UUAFbSXmljuM)9S7uI+^?|FzF#ef!0+ zbszSzwAf-+o14Y<*KJ@&6Hc;7k>BvF&p0+=^Im3~e4qV{QUJr1=Fs942>!1U;k|SQ ze7|r8+`ebP`}jRzcXI(e{h|cNb+2&Ce%s*uzQx$VfnCwm-qfd6}G@)NCd8VwUnSW=Ki^pg> zd29qVZP%l}?Xpz1zZM&3p2x%`Q8;>vA%;wFMU`2%FeCgm?sAo)RKu~fwAzt$1}`Gb z#Bj<#9!`E5YpADaEzNxGPHXN-aQ*N=_My`gs>~|k-%K$B>}14uC{E)i$;{(5zR%~! ztn}c6ZLIk|{u+E<-}ewzcp4&ImcqjMmSFzR7=j%qLaAaPv{c4GzwUUDxfce1Z`i{V zmd=*XzKCELPWOd6D*R7DvP5mLkqUGynbjIr~>(IOdof&F*7Lqdq&4mF7ay9_mGV z8kf?!_MOBp7QIZG;S>`SLw;>ZRP!dDqSE)%4clXs6nT)A=0*`89!WoU_>%QUApcW5 z)z7q~)85K7;-4ygZqcBv7n|^sY$-N9KZ=J2n&a6Y7FalsM_m~|T)Ocb{%3O+tNtp{ z1O4fg|9UCS)!k0s?#XntHHPlIo<&Y8Utw6Q3(J+-3a4dn!}Xv(y#7loK3RJXFR@z8 z|Fl@h%j!?(iwuo<#mT?n>h`lx<>Uxg9bdBe=?~a>P6p}*8iA7_X2<>}!Oq7=K=)59 zv_y{wk5{pr?FtQg*PTI8!JRbm^FWDulcD6r4t+_>U@ghzQNL;FJ~3bZcRjuLYCyf> zpJIPv8aKW*kj>p_&$RZP;mTjX=A2v`xfa(jPVdofwsJ@di#7ekqR;9<*n*YN9k>x< zUWLQsff*p>QvemCbHV%VX$Zula3^UQ93DRw2Fz|@4#ieN){>hDGSVay;zZx(#!`A~ zES zmnh#ylk6{v44emEWGwnn6~kg^aLsO-Cg)5SC(l6hjT#UZdkP{~i8;)>X8ii%S-f=b zN`7Z&2w#18CI5Y)8~@10g8y$}FMLZ%gC~j;;d{^r)^MCOqCkNNe0zr^x$J z2JP3&B=@aZWZiX~Vpk;5wMDzBd6@^*7~9Z*0z(?|uOB(yeu@2*%J6$n3R>5%#diUV z@wU$q-0O4}2OloS8FDK0bDkAFxoAf-&DK!ur~~w_IEXys6)3jnux;G=d642!4`w^H z_)sl-e#TTE-eQ(7f5XzB-(R?re>Y<)Kk0`yzoDTKE}RMko2q8EO*W1x8iX+0W1@HR zy)Hbe@r6N+VZh&509(y;!TMV~D?1p2H%gb#hk51n*>|jwlUa^}yrC>|xBbGaS6C1ta9;Y(7FMFatk!=|=h1Cb# zVi9X}L83k!Rxh@LdyNrLqY?$Hd_*mK?=h(RoemC1qd;NXE+{^?8s?1~3h!RVuol&? z6}|ztG56U78k!hP9%nNs_g5)#PWcqko=s*$(`fm6@ws}>hq{J%Q$y8kT5)AMecI_y z7O~NE;?hC-ZhD-e!%oxRy;-CXltIzokI`@SJrr%`L8h4#DRQtZy^6>|?t{q167?Zb z@De-}05^0_z?F+k@#fDmRJhlLZ;q+btX(`&*L+&zy@AyJ?IZ2r`LrtR96AZ-nbxFe zNZ;HF0|x2y1C*TjjyzBPrTJ3+>arDl)+l#=tGx-oKkE-TXC%Vd_#3SLuNIS;%yZV| zC%7x~i`lESN-)FE0xleIfb*Gp@Oon>yKg<8neYoP?Zr1bcU?*Hk@`up zJQO6Ix0=YIB9&?`yV8N7SMXloVY@pEcd!hbW+pdV0hXyZFsI%a<{c2h_8Y`AkK!1X zJ+6o;x+=hA9aY$2@srISZw!gsX2PJEZXlx;33G3!!?xH&SfLUF+vO8snZi1F88s5l zIp1O4)jW4Hq6RNd!dKN+ZW#q&r3DX#Up2J#l_@e>zP3 z&KuG2ZZFz2BbeMDh7xX!rO0E+bT2u9CjU*Ket!$#nH z!hSjP`wadAR zWLfJNYf&3cVy9-*GI?<~SfFVE*{@u{*U%f*&fW{WN(u~XPKMR%4nh06{qQX`5KJN` zz*!Yp_-Z$aUHm-|GqoOJ$0~cOGKeJKie%d6bc()p7LeY&YiBFy z$KDuHPo*`De#?=vmJ|Jv3840z&BRY#NoI&>(H*0CX@e=ers$oVG+`0%SuleS-7=b==+uXQ^Sl)7nmA}L zy22LySU1 zC&^0e>!c-#Vcm50@pCbgd67;o+(Hw-^(QsCg(w)RGu@gR_95&63!8kL^$y(4b_`g? z%y#x;$^-hbIrDb2Z;j{Kna?+w@4Y`PE8iF{{h9_+#>?Q+s7OfkJP5yz#6e+o1UQ`8 z0BKdT!0(|ZNQa+c**- z8$oetB9m9{By!!Atjj+h)DFIYzfyzvh6Hoo_Jb{dSS(4Xm}bu#x0vuzma2TF)nk~@ z8wGO*Xv2qfh3xFER2CXi$(C63fjx2)pwBu7I5EuuO3fz%mn992=XKazt$Z|@y^3DM z7tzzSCX!p#MV7ld$ZWtp^4W4uJV#?lexp4durI^jBW}V`WovfKL}c`!C@@;RJeiF5q~&u76d=cLSW_c4KU)E z9|ZrI2JI*G#Jp+?+iKv+UI@1Z+w8m8o~Tbg*VfR9e@E!f^lW-F>MZ@3nNNc>PSJp` zNi;;9b!$h?r=c}AG)!VjI%7;J>WU>D3Us96vA%SA_b!^jM^oOW7)DoN`iv^X*`sgDQhgX|kW7`N_x|ZQc_xk$MbKdo43KxsmhSJzmhE+(#XzwNf+3`Nel3~hhfqn~YLYp!n0}YL z(0yY|%0913tzSRj)|C=`l9`4TlaAvf#dPdWJ&YO&XYh7sGq&`Xrg6?1wEoF-%52<1 zi&OW~gXsx`TsRFgS0b0HB+hB+I5?Vj85-vG;g`9I{|oo0@EJj~`6eFi%ethOA!1^iLI_ z%^V-nc_i)>!k>`q#W&=B^AXu5-lZVEjPi10$>XFg)rObj)am_%rDuk-%L_xqzK>9r zQ3x#kpDZ))xWPR*RnPtEugA7ntzbX1Q<==o1{T1p0razm@D<*mdv7a!C?|ww+a9E?7D;qD><}ec9HvFR z(WKQFOiMHt(%3(92whxB!*B{^WinbK>QMDJrqOXL2a;=>MeECF(h)^#>b0Fn7VoU+ zcIZ(0oUTDZbJZyKoD{W-y%uYyNz!S|qT06>`=} zKbr8|nRV1xy^z*&w`lmMT59P}l>0G}i(UTlQ?wg?z@bvLYD z9RiNd%i*_;3!I8H0^@!1kZD59%2JiB-(D}QJ^3H1WsIlD&H!3u5>K+rGe}jOcXz8F zqvMT7DfN96eVO7-LrbU7IUNJqxArS0{Q7`ze#y|~ls;seIfC5APN6Svohh~3fgCiP z>C!SrBicrg$|MyU>)3-Od?kL8q+zvx5f-c8#=L^(cvQU~_3_jf*;RV zI;K&H%XD(`@}lf13ut48H5KPS!}4h*c5mNlK);0t;7ENPSdCKP-&*SO%WFsTn|4|9 z--1l}@*WM|F{K@Z?M1MsYbU(fun^`(2g4tkNEmo>uedV}hc9}Y;F#7bXgM|=4z{U+ zZ?p^3bBx5aN0Vs!oWnF?KneM7yhI%}czGE`ooN(bu= zXtK=oHaL;D3p5V}KtQJxjD27$vR%f3^P+)J)_tEDeqF$xJ+T(98=c47mlSEi z0YegkXOPEIan=|cOnFDv(P?czy3jnCz8i|&BW`N6%0Z4AMejzLgC0F~nMmQkXOfrF z3bHTXMCRwh$tpO4E{dP$slQIN{Jt4+T7#);qC8o@m8RS&Ql$N`3r`Ia{UyJ&sHRAV z_LPmFqOYcu-)uk~<2C3}h6=rr(WH|P&8XX$qo2ux$o%m>v@l!2EaRqtBqa;d{xpM3 zUO)c0$Y6+D!}0g>C-cA6?f8Uidc4G6o-Yo&4@#ntl}?FpRXPd0r=>ugY9h>&4~OVQ z0nl;Q6-3P{7?Z)|!VqFHQ%vA`x#!+RwH8tLwLQ~evr^12& z+Ep4#K8M0d^Jo@GJNJ;x=nb@U`D_~4$Wgt)c)EGsfI`j>Bquc$+T0*RCzeQ& zRka*Fl#!uNhrZ&?8{g3Ns{$z=7)t6Tb`-S5g&qx=KrO}BF?h@{mR14qu2tN>HGc#r zPMu$*Zpt6aW4uDj6n;XXEuWyP$LnsA=H>V+u>Ex+D3(S*hHMP@Z%%~%&hcRRd^7xv zcZ0-4a~Nc+2>U%Zvy2`id^X0A@bUrDJ%$vbR!`Sl8^}2L4%s~x$md}mMa~zW;TPg5 zwP_ozk6lKJAv}fl4x`Uu{fV>fMbFV?xH7muK3Q~_n_IGjHSUwJBWqikS%(^AJKDkO zC|_9ic{dE0p8)>BY2Yt83!j6EV0U#sIBw5^Ns94s&Uh|7^Xgz@pIqhYy%uA%+gl7v zA53qfCeV%?cN&`JM?K;$t9(ZQEgdL6!@hV@d(i?qWE()grMA=8XM5@D!Tt17HlDtj z#8Hoa9F-dHp*2~XL?&whUAyZ^-X=Em=$r|qh&^5(M~$G<%M57YRbv{iszp5sQesx{ zH#!z|;N`|j>=S$zv$Qku^vcutRPr9TtTvz^vHwR}eh}$vo@=tj~kNH92to{RuczbpX;HghQZu zIBeG03MtVZ5GpeisxR(khDny_xo!kqmfuG=j$ELasA@`I_#X`mxKHxCYN)*IGHE=? zp}{H#sNXzM&-pZ$7U%Jl+AxB0R`jC!%z7*g&BlRWwDHe57rPRT{%rNfG^S)G3scxc z$lmS`qYPu=Ph$$mdgMUy>q4m5R|X4@6oO$!Ivltg4fRbPkUmopWNO^lJnL4WCF2rK z3zHJ>VOr#(VomQpdDDOup;WkcD}@gZrfU|PNN~P( z1*$50K`vJGC|FJ4^DON7Q;xtljWg!Qdn@rBBiq2X@*WIPz6M4c${}7HN+;-^gq*N6 z7^Ra2mDO>OaMleb6}B_?=?ggPca8Yu_9CioPowX1E68DM4ehv5ORcl+(9>r(#X0;k z-8`B_m&YF<{Qrw=PS1lA2Z|x#MiJc2ItR_>X`rFC8`_>uhB9kuc&zWlmR4U7 zDyGDsWA9x|Iw?b4nmQsU*oM4By=Zry$yM&~xZ^DB z`q=^j1#0{pvr+uNMU3Aq?i3HqpUlS|H06tQ`tz5>4k-NaSe##OLQB^rSe|$qz6GX0 zVYj%WxDW@BxfLp>G1$`o9_!ILX_qqLF@8?)rm(DZl6@vnnsptS{CzSsj6gceV?94-ZxhJ~xJQiq~L0 zW6IfwykQXFu@LS@Mnj#=Nidyo3BGQrf`$F>z^{&LkltGWaU+vp=!K;adrTGbs%NlB zP7YUERp2E<1u~sxL^(sI((3y86luPi^7n2eo!!B-tk9QChOMTSu{&t%+-NEqe~4ml zr%}S{9HLccY1Q65>S{VqBMnZ|-971K8F!e@H^+%ytX=eJ{4$bSFpcKcThMSn9eVRk zi4sjdihWb1X#OA;hZIDJ8Er9Z|1%8-m0iN3?XR))ML)_4)19qa0{Ffx0h#yb zz@sz^VppcXr5Q0W?o=qenQH?Yz9nprbD}W*iUNh+3Z`>^GHJs?q6p=BY8&)|UPeC` z88G#9Y)Umf_qaqaN1dSlvQgA5zly4IrjoRq5p@nzqpwNN(7t^?nip80T<8y>WV4M> zYV@5`Hz{K`?+k(DEel}s?r4zRe-etkE<#m@0GnFNp|RvVbg)A(MaLV|9`=J(3fow( z+s}$IRr7IYd@jx!*o-Dy`_hjxeNy^3iT0oKq-q3J&8RyJTz{REeP3h>x4j;u5V*!r#eN ztTVzFX5Y+)d6u=XrM3rdk@|GEl8v*yVX2B@e0Qfj31bW}Df*E!*;C-zgn3PL_#Eh8P z0w;!J&s3{M+$CsqXLooFkNK%yZbI6$My5*%qH*uD z@%q`7oaI3&So&iLlt&$bjb|>w?=j7AV7m)?KQ zIlQPm4Ry6AVf4VWu*#|kqJQUs_Qw?X{U!v)m0833dIi`xFqxIx?+`{Mb)j&`gWheA zppPw4G}R=Us%OVgQc^746g8OS73*nk!%VvFIGPUZ7)I0I4H4%^O^PoUJK@sb;GIu3 zczZz^I$TM`_8K0ec6bO|@>99utoiJn>?O9eSq45_8xDg%>ch6>nox7(AFG+&#s+Xt z*xj7dY=MO=d-6P8hxCMl>;NzZW62Vy|e5(bj0Tt9l-K9EW$jHMw3t`zjkhxRU>PbceUkhFn4 z)eL5onmCT?m=XO)0|;Ne5&5nqSlf9PQ{BqZ_WfN{|9l@K7a?xGnTsY5&mmiS8O{1u z;`kq8H`tmq!FT5jHu;w{%oFwS524|pJ~SKr>u!SgBGJ>+)CK21d=m}uU2r}7Bisyl z4YnfdBdhH;BqvmYze~Bugf4?=EyYlkaSp~`P6f@+d!Srl5m;{;1Mk&;vmvU7*dwnf zuGT>fV}4~~`t&Ebd(Llc-`JmYPL3qmz2?N<7)`Q$hf>+&{^WW=jUJ|`)7m2gsI^*w z_P=Pu!2vI^V7I6pjQx$XuFH^-g4q36QH<=^D(v0)QfQh#zQTQK0=MqB0_&e)$6Us( zW!o+M*v4_j?Cif}?g8IbQIqjrST|`p4$VJ;z9j+8bt(D0T#oxLGLK@Qz`TxaSAxz4!$C7r%tdBkN)Cohs1me;G!Xod@mrS@1gZ2&AX&gG;t+VDD=;81Tpn zh7TMBH+FTf2NBWi`&A9jF4hPysGY;Ctt~ins2nX`r9mgs4M^VAg6tm|QA@oBosxXP z4enLQqDnFBb0Kan&&7o<85lJCIKFep#W|PD@niW_+-z2g5o43^*e7Qk_3ybb*ZFyc zbi`CH>T4}`YLgaQTxG>RH&0|+mnyO&KfJj|aR-Dh!?~F9?J$~LIE&*-FJi6eJGk{< z3uffXQ{5L%fzG{|1 zfcAapOnLrO|h`;j?=8md^Gr_lD`qgrH$I{`t z$2#cSV+z(LpBSnZGWj+BY}nsGuAxc|bNZe{*`9jz@$JNPKSlEEKa|$qGpDw@b~JSW zkY~CLS?G(t;{C&D-&AcftWu%q=RJ5#;R8O9ZbqfN`*>aV31;;^$Afv|-)=xY_W2ki z7S_$c3oX{zJ$wu*Y*WBj4kUP34;AJ-OTAp9v#tEI)Dodzt{i&EGn{U)64meQ!wsJk zaohZCOfsv%+Qx2N)}=s)e#p}N1s_qOScB$9m$2FT2D)cI!G$lL;MAW_(BwxmI=_>l zwvHj>wMO*&jx(lGe^b$uF`hiD?I8l1H- zAD^=>w9`-%xk#F%K2TGfJ;k%PP?`K>W$1y+d(4=66(_C8$L`83R2`a$H*$0E?Yfg# z^f(ihGEd^?t7mbSR6b6VKZWux*%-0*C_Zo6i5JaXaqos<7|@`K+zBl_{bdp+eBO*v zbO615a&c93IZDM`#nz+uvF_Y!+#UNCKkRvr!vmk<4!N5+YR)Y@Abuu#rY~{DxgQvj ztV#=bEsEGah+g*7qG`7k>4AG6YX3BVj8=@K1LE#*-aJh@o&ObUCbXi0ssb%qX-J+6 znCR2CCd&c3vju#Ry2th`fr655Bw zM?^3?qheNZyp#RnO`u=f0+8Lh6V@zh*)@>z8-RK@o@5>3cQrOWE(5XnM7oP zhIILGUvx&JirWR0QF)B=yWXO={C8|Wqd?I;;+fhgPvM4NaDQe4ZXZyMTU4*(^XBX5 z6(PZ+r;D+};}W)gyn_CvwPN4fbL?LD5f#H;Vx;&!jpL`$zcCJT&mBf1pObhf;R4!J z<>8P?*(jq~fR3uSur)yJE6aF<*Va_w-1;(1NzTPsOvctVap>icjQW4eP*3$4{(JTm z1B{#SjnY&6{_YjV{_DnLi#u`0&1yW1TpAN2LwA$Kp2`o}bad4aI^BN^ zH9urxmd%9>D_khVz>XR=8q=>tQI9+}ff`<$(?LZ;`utO$tl!zv$X7np*D#o5zAhxG z>Iqb^LW^dJnK~TTiXX2>VNc&{T)zfsm{scnZ|Xu|n_U=~?23b^f-~SKTLqrmo_xUW#JC;fFU48yRpPRqH$fpe!7}kM(dOoN)9)#La zA+SewGQ_9;Wo`|N*p3MPz0Ml1cWYtt3HTHA@WbziaNv@AO3@fdF;|9Btl22?YBQnpmvkt4lQN~aDpKig1)87Si_-7} z*NUz3^)GJVO^-*o?ENE5_o>Eh+9en`FCQc1@1ysIe^{gN4=)rn;MXB%G4IR)F|QGe zOG~4%cHv@FUH?b$UO7%!Q{!9VmS-jStvWA6l}UsdCVqnKs61g(&|D0Q%f~G(PjQNR z6SfyOqLkRXF{0Xnz9@Uph>m&mcey=<92qNeN2k$cUw?XhX*0<$4W>vJ@t$cUcApe( zr^#xfU+h&Rc~x$psH1DC^jH8@FPTl}5 zr4PSY>nF^d{uch8Yk=>eHz8zc5fq$13hsM%!BtNu2%Rr-2UO3p1TSg!KSgIAPu15( zaWa)CB&En0LJFn0XYVqE1|cOWB}s)+DN6J+hp0%IOH`7mD8=3DG%1wQOo}E=G-+1S zd)|NVKlgslx%=6BKhIj<|s;C2-zvX;ciO61$kv`xtgT%7<;^t=L!dFXB;048*GE)`&6>58!?) zDRNRZT_VFJFGa)8eG-*E%M^XOy0`ZFacP#Ix`|nePqXcA&)F7jWk@cbf|pxdP!{Eh z>hTNk=+II;m0O2nPa=@hyA`)D`eUim5@ZGk0|D_!%TEI*=;lmEq~h+h9hi4B0sTva zJT{vk#G1@OI-P(K_lIMT#c!4zC)9JXJnLDSZ!`AeMbX&G30z)*n5*wLB`4J)$c**5 zBqq0vc+B_;R}7?ggxmqW=>zHYQG@93L-I6u-XJ>dycErt^o>})5pui_ zJtfaRvM2x0y`a; zSx6;Q`uBnLo_NDjcZu16JEd%(=VP|>yE+c|SU@6v3c@~G!DEkbc3rp%mn}kZYKA`s zYS|(4qaJF{jDX5Y6^xrQ816a}*mnF28@Bxyds_dKE!h2oRsZ_HiZq_HHBGmfvU53` zTCjv2{~axs8j>tJklZUWTKi43bMjl!Eb|YdfZwWI_;(2|JHuAAILl8w$yvn4S?px` z9}ch%q2K#O|1$gZbPiWox2A=qv@su#_L^*~Qt?^*)exq;~Kh=!@%F6hV{zjE28{vAyAnL6`2l3VM0;ipYy@O1I0f9C92+XnV$-aNL( zTbh|Kcr8Ak?#+~*RI^B>w`^+0Czjy#fIT~Njy+pi%|-}m&j|{uXjL1JLO(OyKc<7V z`_yrzM;F(P2o!5Q5h1Y@$71GUjCeYB?bge%bCQ?@0Bhq@Q!})AG#<>ggSu5vlaNNjevSEcPoPt*-zj7q@1bg7c4T1ICw;k^;xPrSg z7RfUV5jW5V?TY5eEtkN-mVHd>pF7hKJn1_ImavH$*{rF-lQnHoWa)*YnLJy|s^0El z4bL_)x5@gg&vsyDJ=kNa#?wiF~Sw2|a$3fcR@{xMn!#`ePd5_-$?ZZC#O>uQ*)`=fla zE8NyNAgk09bomAp{ffpu-AD{tu?Bksh5P`aR!Y4LfOLYu*5*9W(5wZgZI9Ts(A})i z%!TdFc_n@j8!no1K9?(LlOZEo41}z$spO8fJ5l%-L0S%GlD)@{k^`&GkSyzqWPnXG z=|0s#K5h~6Fm}Bos*k#b-cJu%qw|Frt?wds@f}2R?H%&c;iixSc%59e72Z?sQL=h? zIx*CZC%Wz%Nl(#Svc_M5WQ9g@slQL#Y+l90hVv}hUA02CDW{iB@YRN-sUBADR>V%# zF1EeAgX!A*WF=;DxMDRBM}r4ouX-OFoFb1%Ic-d<)fak@S~%vXg7R0=uoqxfzuvU7 zn56TpeBD7-iUJmyyqBGRpUkS(MzP{&-fXU+15@OjS;9Fh_RRXb`0v~R@f)KjwN_W> ziX&sH#Er`zi(hGe6nDA}WMg;DX5TFe*~;eoZ0El(tkV7$TW;`=9iFX-jFOSCX&R0J zno5vwR>JBobp*dOL7ny#Xxy2K4S6ComCu4(+6r8}9S&*t?J#TDDCjf;AoFYmc3%j@ zysPW+ZDA00xcIE)5`xG&=IZvKt)sl2AMx48^5b0O9h?nhM@<#R!S^BP- z>^XmrEX{dBdS|th;_bJ{gg57i#*Y%B?{$!r8)TEDs$??7b_FRaHz(h3DUs8&q{$)g zpPZ=jKKI=(k=s;sP_)9iTKq_TAzKx`pXHd`WvL?uqWX{lCU{umgs^UmwI`v&#T4f1 zdKhA+frxn{5T`u?oBT#2X!D`D zxdPAfow#pfl(@3{R!vvmGSR7fk3|nH26Kw!gQ#vykSJ=DuFb{)dNzSy-r2Yv_7}+x zN)(wTezM8TcNSM9d=mezG-P(1D;v3O2TR@Ez@~qY$Ng9n3~v_UtKv-fw#`S7&|5DM z=AFe=8?fTF52n_*!jR3wOv7c^^K>5=bjT;k5}Z)tXCx1>IL!deL^0uzek$HjigarN5C$Tr16)@ zCf&=#^V>zDc=7^q)DRQrg44t;U4U}W&LB2#qKMV81;l!Q3DL(lZtR~_ZrH14QH5$- z?S!-6#COKeW~RbEdP|{&rI$)V%Rw2_W3^Gb(*h@l*}_KB1rj0i0fEzT;HMYv&2>js zxHD|`Q7oT79`S2OLPzkcEa5xZ@~My5G^Hy{?wQc*nUKmFCi*ejrOyo8ABz)5jumgv z7%X}wFrPcejpx`4FYfc@D6Y6Io_qOhBj+VEfg9P8Dk|7NO`Lf0mw4Hu~ZS?bVtBp$Y>0y7=_*4x{%7ZLcl~PEX<#WsgbVO zc4IbTja=X`NN~{pn}rZ#dyLhYjwQ(spcfZnFjDEe@WgR?jk>?ca#1t@5$7*k0jmvgWw&0P3i+5kPCB!{HTK$h-K_Wva|Oh zSr~nmJehNZ6w2k2HwBsG$%|B?Yq^O$$Pp3y1ybbX$Rk{?ggy6l(N59eBTZsm$1HYr zbT|9-RT^bwl1LEV!I6(be|fPs7V;)I@qH}Lej1His>0c0pC#(vJL9M0e0=-kf^!7~ z-(@C2;+Z)vyfB4!s-S_K*~YF;-^W(Zp24gF--ss#DT>!lNDvtx9K)HOT_R++#&J7b zvp6>)(>Ho^758t;S#I0RaxVUT0jJdx%UzIkqf<=PH_&w(|K##=icPE}Kj!dZj|LoyBS%0IB z%wF0^?5{SHS(T3kZ)+zRwecm{Ht#W+d-gUtG5H3$KkgvnUmgKfDb zXBUWyUhWl3oSVRAL`AcjEfs9;PhwXQ~qwd;xnj+NdP8^_-x#j=xb3ydgrw(Y7h zJNR`1ld@RG=8Y?49mV&Ui`ge;viB{^DZ0xPL$0w!wk>RH`)f9?=rzj~a#NzEU$Pl1 zB;mDl6b=TP!>W2Z0)+kUT>k<=(-VMuLxfz8iR{usZ=EGhhi{NvvnOP3Vkc2K{)X^_ddU3xUu2_xFS)b)Gm%n$LB3sXA&tZC zk!ySt*{{Zvr`wK^nR;2|V@euX>zz#G3}VQ{MJvd+jke^n`&g1Ir$WNk{^Z^*IL)m* zWya}#Z?%5<;Jw)M_+pmx>@d@id&eTS4acvg#<;S33dDxCxbk^68tfKAQnUmYnuOZ9 z+7mqj|6G2MH>mJmpU`dxopfOhz7pnmPNU$^EC>5xEi6oGH~Tcyh(*@5i5G0%D?UH@ zQ0@M?3L=@tP|>BE>qWnR8;klqbVbf7OGV3l(na+?B_fC9Mv<|WBxicYn5#H5OVG$U zaajf9xzK|$TwPtE$Z5wpn`~7raY%HxIKjxCjrktJ61)@H!-WS~$Otj}y6P6IANP>G zYJJFPNhgczl0##p7Rl_glc8B5nE7<8KqzSlxp^PiAihVRJ8;lD}rp#ijNk0hNO z{fEdp^^p1>@5%h)cVz6oc5-U$4I$&IlBk6pAch~)$e0P+i6Q4jgv32#vFgh z5>B_X?5%&;Pcv0K67H3Ioj6!-orBQsC5U>z6_W)m@vupmn4Ob}9K8%=-$_S;!(O!8 z>;+BO4OyiWB!(s6^|b`t+LsLZ3u(w7zZ20n<56*DGb{z4Z&R2f3OqHDto?-j_4Z_b z3UkHZpGtC@!i%|g_vDDw92+ujq8Hij5J6^!rjfH3O9(N&MpTNQ5akIUNXo-6WbK2$ zr0uR8Rr451tBaIq#~*pxsvt#W_5P6jeP2m@`CFp5>jBwXUqimrgCt`~5@}e!mJGLc zB#$qvkSvsOZfV-wvgt84MPV{*>WN%tKdFxm-=>K%Bh8T@Hy!uFmf(f?R?HXtv#BTJ z@X~$@UiAh;@oW%WHC94j$^%hz%%P$<1V8(PzHHEK)*F7BeHHF>)1n76{Y~m(kFF;* zEbWJN&cQh%+oHkT+fpyigN1Rr-~QvAdbV;wC--n~9-ZLy#$DuGHecgTh^x4iO-0=0 zoZVbqV-m;i#&N&SEaH|k1+K4khUls3rCKgyo!I1Xf!N;cx!CHWDf^xn&fKN;v7^WH z*l)L^OnJ~Hwj<;fI~p#J{rzL`%fkZM>Q1PFACzy!!2H-AxZlo&Wy>Lyd?>-d^phYi z=kPtYM(8_MgVV0U=W~_Fmpp@6XDd;>QQ)c99YeWwA*v4)z;@MM_`5_y;hzhx33J#R zb8fK>+h?;&>7_QRSMs<8(?^iowoc?8zlBU%nL>hp7Lhui3uNTL2PFLc8}cLbD^ZS+ zq>|PORPV-cx}$UyHQJ?4>+)3TUkhcb^FomtcnE#3%YRAt!w#bKf+y;8kCDEf{p6B; zH2Lz>kvQI#Cfe}_In8u+ZtBY)HjAZYn18aM=lwJYugfiPUeKPncj z*8q(Mh61a19P$O8lVrLct{<>PqJ=jU#3As{-wGO*h?Ld2IJKb+<=jQgvc3wtMRfx4 zu>p@7>T#FcN6FEvZ`{c|1%bI)SVuVXlR?f_zsBw>7W9B#~Cicc@) zFv4XY+Y(YCGTpj{JLd6^GyW1p)_EQzXCB@oVx2Z}@y0t+I{OcC&6cGBpGQ!8Yh~J( zIh1btPo8>b45F5n!)fnmWtu)pS?~=@)A&{2$x-%|T>12jXkBa})2pk9Zh1QS*0!3s zzcwWw)(j?tbB}VBnY^g(zaQck3$vN@vnIB*Rsv%MZq=&B*+^&!M48nVG;iGwM{zvX z3p!HU&^<61ng&<@IApBfh_QmtWAaZ=%#WCc@3xw_p!t=xSXQvKk@4)Gy8-*XgNo0T z-4u1y?dCcrJ>tG}N|PqH(d2QHIpL;EC$n5@~N2Q62ZaxSE@_Fo84Q=Ezw{N^_rc+iUiJ)M6LtA`DQAf&Phi#EyTHL#hCxM0y|V0j%r*& z!iENX^129>w_^CeI*HmhC2(F>gtoLy6wl1Smxl=mP1=k>*bLoQ%aQND8V<*G@NRq= zJ0hDPZhd9U`A_uX+~t&+jxSk@F|QM` zd`%2WT314S&rBRvSb`0LM%DLB2*wHb=Fai~=nZXTni4lyq~0y|$RdWB%f#6PXGC+2 z7H7C%tHYeeEiu<|O@+wkyOQ-eVdTc)P_o1zjL2svkQ*m5NZq-;MDyKwXp0+Zp@=yrurLaIP(L_< z&0C(uWv|sHE4B#iFY`hY*D#RoHJw4%&00l=%dDc47Wz`_dSBXJx1O45h0$1}Wi)!* zB3Um?19P@nacl{5YJ3Z^1zU=KC#v3&|^_^oe* zuI`cem)gt52c2T3iWk`NqUUVh&`wsOf17!K+RUnA3s}&er%cL26S0<-us7Djvd1WR1buZfgwwXvZfbN8y`&c`c{%nbB%~mz-w;JwNOrSax#}?A^3Dxp6Axc zG;>y#dEBj)&0IGBhGUhz+}c_R@+;>w7cEi1Er|Up`VsZ8HtR?d%N_WXsp}4gladZj zD`{cjo{0$bn}EU71P}k2KkTDV7Q62xVudwM?AXaT7T0iumGl>~y!*%5>V@x^!gzfQ z+UgDIo)CmR*o*_4!}0xK7@UVDpfxlB+XcVPj7<@cjoAR=w+SM_cdkE8c=lftQE_1h z(mYdezB3LhD-!V3APSDJHls~-4Pp&vqi>TX4hUShSRu!yf12X7oFg*#&c}h! zC>RQU^Uei1SgCsgH`a^6&WbTFHUJGXU)M?=4J4=UF%rMzBhl;Cr;-i|af{Cb`pTlT4_6NgL65N66^!W4J-PAeOX^5zh+R$fO+wO^D43 zj1;_R7Kb*WS~#bCaN3U-epSfquf?ThXV84}BnE|*;+Nw-OxQOaL*3W2fC^VGZe0(T zt!O}gef1=vIw?fD>>%;pQA}QhrjQL8Q%U9MHf{_b$<5dp!4;pj;v#b$Z5FgBGiL#n z_R>OG{Q8vwcQIrSH%avrx3sg6TN_`*87LQVf3;3?TWWK;e&WOZyr;nZNL(kXeVJiH zg=~Z+y`|#97E|$sc@so0vi^#!s_R8wMn|oGWoOo8cYd}$HDjCTkylntZ@MwNq*%`+ z->IWR%?OXKo1*o*;Mp1N4mo96beztz|gdoCkA{9Y`Iu23{v-AV+5$9L^2LEhPn<`mKafiSkHk z8HSEclOR{V95d2(V)T zsAY0^)Tb!knKz8|KT9Gz_!9Et%W-nDE}J~vn??>9N0K0J0kJh6N*pa7h-{XgBf1fQI0?jJeXi4%{C5KO%jv0Gs~d|7tsY zW{G}g>2azO+1$xfm$~5uFF4nXpWK3Q@?sMlCWN|M(X^3Kfg|QDO z+nL4oKKA9S1lFZW!LnBYSE9A>FJ>&Z3#`VE~h)w_S5n2m(#7U zjOo~}VRW979CiJ1m(-;>kv$eO#NN(hpj;|w(CU9Mw+Vy6clg7o^)yz6-$%^%c5GVl z8XD(bfd6<0&*vY3+FNI6{XW6GZr&5+G>;;CO7cnB$LFN+(g1ohqmNuV*g%xnPGb1q zSTe-^GN+O7Tom+tKg(*NsF2SX!1mPJ~7l7L5{2( zNc_(HfwRnZrMb z!z2=kq_#Y$l%7XnWDS;x%F$<+2fbw(xVm*0T#Xa(_G=iPO%H~TO9Wcp2y+kn<(Tg@ zRmhe%!MB;#7(K)pzc)_B@@7S(&G^igPVZ)R+M^JWV1vAQ)1mQB4-pH7V0hqA{K3m=Y^`^_Uf%^39T74D+Mi zYI(Fx{xsbqb%-kLTt}Pj45;LY!BqdxV0vwY4DB(mBdo=mY|_*eizN=Tun!?jdc6m$ zd@vd-4pl&*@(-MLDe|`2%Dkr32tGbTg}?ddGkT8i#-iIN+0ovGT>58Aa^PYS+2Qt) zv_z@WZuwDkv8FW5+I^CoR}=~Sr_)@-uux7a$6eH1CuZ*_uYivCUWENlz(}PzSTRKl z*Sf}Hp^gJoGFKq&+8P{Nz7l#X-SI4V8U_j)!8vnsSnJhP@jBTPqA?+1+`Mvm;v$YA z6<5oN&V&v_zU^2+cTUCyRIlob1X zs~64Rzf-(j`YdyJJs77iOu{{XSK)a_VBB1RiTCOVE(M%~p^i`&HA*0@orUUE+fmp& z4MA!ecvEEne*X$oUQ5EIL9ux9*A{hWh6?-y9h}cxjL&zX@J-ntR7D3Z#oa7&?{`7# zr-%FjPJ-^#1&eKkeES_bFmqDFdjpI>ZCZFCj8e$a)x%0 z<%t)F+a1B%c=|YZyB_MYgLG|rwQp+o=H54BFRvn}C`%?9tk1Z0Maw?9P(SEK0IKtgjIz4t|s%@V^FPuh%SWn7RquT6g09 zhCD>36(S)e8*2lT1;0rgek(^}%K1R#j-l{8rGy2SG!T}$0JF4H(EW24-YN_C7?X+c z{BI1_pB)e7AV+Mx>Wp$nK|45E8@mUK5d2~#WPhe$_Npw%Kg+_Jrd%}Dq(f42EA$4f zf^Ot0I2%VJ)Fc^4trDFXpq+29cE zc*WCPiw4@%d7S>6v57KGV>Gss9p@7{yPr)jy6Q< zD)ZZg%v_CcR=mMd8$RjX6h8IwR6f^VpLdmgi0_ZunbTbha!SV;b4xp9^sby;rmX?FCcDqcJeM1evcuD>nErmuwloW|(C9Vv&XcqED@58}*_&^$%moz#(CT-pwE{qmGj8<1dnQ zc8{E#{eWCGVC3hH2%_^r%w5#_A)XY?Sl74ftb(;MuWn_0ygLIHQw045n}(xQ38U?w zGh-d${2wm|+CLEo>lS0=h6or132So9E}V@?!RL*!xDylt{dzC>{F{y91`er#Q!uJ* zCKS^*9r@ij=N1lwIrBlRXJE|>J8%^O8`fB0?QT}Z&`X*q^LBvO;V`Im zB_U8Y1wk$`NHdAYg|G;uj|f3qZh&yN+6=>W5g_klk$Yay(NC+u;)+JthdV;U-Or##oG@5sSW((DfI|t5i?I zwK7pfx-k}>JB<0|kMSmDApgp70&i#I!&~}n{anF zY&+|K+nqmM_YE|u%EGFg?>)IjArI(CH`HwlU>D>UA)Od8X$K$^3 z8gxWDln;LirRdk_QvU*Op(O8G`vc2vJ-~E} zv)FH&1;_pIf=5C)J1Y0G1;WfE!#tFPIJ_mv>xR+kKh)?y^?}rUMmPDVD7ajp~lq{SM+!7Lg8`;MNA6ehxF_4*O4u^6*9Jo3f z_3@+OpgbFiBNAa+k&lfsxd=X&ik9!&;PH4ZUf*Ahs+s;cSQ&xhqx-RKNHyG2YGBuY z2)_kw&xM#tC>Qo$omJ|v+aQ7ZutxTLNC{iIu7hRtjfb&+Al`4@hpj3nF|MlyqCaOa zd-5@44M;-u`bdhy=Bwcur(-TA$i{(RlUApXy!0RFP?9R5p`4gXQdD30hCLn?c?AOO8fK6Nw?=ixvs;kUxz!8y|yB9e>^fh@W z(xRdb!u=pXjoM}ZCR<8xl3se0d@S%HZ5vl{CTayN+`|?dTVr5&WjjQIrfP!6M3@@& zGQ$^7S@h{%W_>~)H{GYe+(pnfuU>)Fo(L?xvKvNG&2(2Bj}hT=F&;sA=K~Adb;JiADx;$gO)Efq4y+I=-VA) z5*1#@wK-2g;=vmTy(z)ZxhlgO_w`|4>unT=mO|Y$7w#SfxcK-Ou8lc}o%grmg_R+G z?K#QDWJxez^9|>Xp89gvObm#J%uyn4?I$9EOE+0rg?jbBBkIZrh}#PrvP-Itd#z$4 zYIt*oef~QEi>?Z}tnn)`e(@v>DOSM0VON<(9%Tk5<>C_-n`>oXUKh7=mzjl$kQ?)2 zHD(Aih)vS<YCyU$+4`%X&>|49>83khOnD~3Ct^05=y;VNZL0E zIq|wMoAa4DYCK_5XPTMhLun*$^}^;U`!QL&9*;|Vg)_hd!LM7)`n-CH;*$hnEghwu zEk#s|6jQb4TDs#+3;lQF5xv)WhW?V;LgNQ1(9=rRUmvulm}N&8g| z+NI?|57h+GEBAcp?K}}xjy9xi5o)yj`%7}BZYHU3dCoqJ5bhaaZ_vMR5I=T~0-v?? z2duVUf~vwU%xR5B)S)EA>`p?{yF}V7c=o{MRfKf7>>YMvO?I3YSXh^g|`|c>Q5IGwm!LIJc3`S=A)u;~b~2 z@`C8g<-Z6fdT_^te(3X=SD?K2GJdT-k3El?vEjNjKT&%qzw_a6e(`cs{#_d3*&r+a z`bkZGPT6k+8+Tz)Z#zb>NQZd)Vpi$9g#@%I(*@RZ=nG4K>i^q=KJGQ3FV_yD^FO~N zr?2cIwAk^A*g-io2ENU7B^VNFL zN;E=$$XyoUm@7{2AIhzHy^AxbxWTDaiOB1`gCwVe9n*LF4x|XdJr|XC?L?|JORn@{%aZIF^&Qe!y9tE#U0;?-olf zABN{vX}EaoJp2?Jgc@9fTfa*Y6nhrts;$^2{~crczr$zmV{Gj?gFTC~kleZ%vnQ2HVp`{(vgU*)b}XokZ8D#V6PB5X`uzi%$zS1CAc3aR&!jmjnQ9F> zMR)p?Q<=sS^hsz9m3viB#SJ&8On521Le^8?E=j6eSSWIpFNRmgOE?&I=I{n8U%%sTn%4&ogHUO}}o{eXV15LQ|pPRUZ$@QF7saSMZ zGLr?#Dxg+iu6#>UM_Gd;44u2#c%>W6S-O$s%S&SZa!tIqQA5ja2_SM8`@TJmIW#E2 zFF6+88=hdd-A7EQmZrR+!!T~R^d*dKpPHr%tbxn~^x}?QdE}z6J6xs00uaDwq z*mc3pv1Ur1HXthIYn|WEF@kA zU7|1dCPQs`32cTv$C39keD$Y+{MDu|C>a;wme9L*aY$f==PRc!`nt)ct9~g0f6O{}r>X zqvKfcwsFkKtlCDY!;)F|d@Upv9!?}g}x75YjUKJ_P?`eZi>@3g@wL6UcyEX%8Te!wdA zQfv)#L(+)*?5wMb*nRj%k$h{ujlHS~n|SU&_VVpe(Nc1oqvsV#aKk{7c;_egrDP~s z-(^b_#Vbk0`!FIKyn@Ik%M<68`19+ZDZ(UX9i%6P;;x;L`5pL&8I;7a>i!`t(QCHY zEBLsLXXSIz71@PctkZn%)b&d1dYzjrENT^YwHBk)<~-i-zXm7m`!GAu4Q<^Cj>}iG%Zk zBZ3F`6T1I?_*(KN%b7bK4{Ig6;`=3@53~Ri0iG)@*YBed<_nnZ6j3LnB|$ zpg%TjArIbK!R=oe;^OO(tKEcoPG3;SpM(zGu20@+&~PE^az za(s0evr3zd$-;R!{7@?bZgs(dv`Zfa^L=RW)kH_A|!c)Q6EiDhHR%F#2)(METoRNzX!X{!-?T{AA>xf{#f ztS021%z*1aGvsE5dUjqkos&*-&$+tg?kKH-K1`C?=Z1)>1_gpHAZq%^0d7aLU zJ6giYoFB-U2-3MBy9H+B1WUNy^uRQOIMjYCfOpz6NIMMXXAENKXLVffoT;?fJBNmS zE~N)_PEy4t(8J;;>c6^`&NjbAYqHAetvfOFU&&t*A}SIaZQ6w|h0Q3n@4{wjdA`Vh z0zYcPc>a^{ZZE9l_>@>De$ISn{v$KvwVY*n@5i4|u=ygooL3<0#d{{_eNXf~)rZVm z{FcNxTho=>=F)4wMHEg(^ygJMI!adP9oe`LpTT^j zvv3AYzlq`Mc~HGL4(p%(5dWEECQ^DJkbgAO#Cso`iZ{%i$(e>6*qI zd?33$K-ng6@N|*o_i>`0{eGg`O^d`mm%P~n$1kkGcLCOVr647{0JqN_h5E8%7@SxI zGmi%JWIu(q%Wr&j{)YpiH+cGl$Ak?DxX`18+(||3_>D^8YNjSG`?kr(FZ+SW+Wds* z*{NUW{#)qCmQTCIB&YSW`GbdJ>gX|0JpF?8B~M}P^92p1t|oUb^OWcaKS})i*#_oY z^P8PeoQ~e1i3kt420OFCJZ%2p--nOv$Hw<0Z(B4C%r2nD^2PM%>QnT34ruN3oAko^ z2h=ybj_!V3OqYburuq4Hr1-5VPI{h%`lBb<`%r?nN;2lZSX=O1lRkgM*M^sZGcS2* zHs4($;xncV<>fs4VEynWKJ>4_R(*BkM<%oNk7BuOyENi+QiCS=%%p>U+R>qlr%;`| z(e#<-S7Mt{M24%~<=&1%`OTlXeJ8ZZNf(YB2(l)(z8R4L=e)_eMe(Fo zVLOp`noSf149*a@Y2wb`)vVG^5%CLq7=K^L?vxiX*VMJ_!l)Yd`Ah>F(XoQfFGv-a zSH0Z6#!=J89Z2uL9=6iw2yc7iAik+zbR1S%I4T!M6i)p_u zqN!cTRhqU%m>~$Wqpo4N8`Z)-3<_fp?@wWtug0^VFMJpoxPk2{ie?Kw?P6EVcCy7a z8EnmtSZ06Rg~?x9!rr(lveu9K?DgvcrfJStbEPEq3HAA4<4$lJ?;tqrj=-Onh3(B< zB)oMRja`&P^YZr62bKrvwTcotRa8OUH-KtBsHJ^}OK5>pEN%R65Pg1BnaB@1&Z=E9 zkh$Up3T&kL=Vx^IwXWLyqsdnMB83I~eP=J;Ey|036D#81c&hU`euMak$ZOa;I0ZZF zX2EZ*H@k7Joog;VLZp8Wp^~*mbXk)%eb`25mytg8tCFQNlrNIuR-4F%W11wp*kP*v z_8~|xUWehUN+5f+7M(9{BDViNBxBE@Y;b`vKNgsO>-8{q@?AE-Ukrl`-|l*_al!aZr2FIuIo z%vuGnYL2-N1e+cj!x*|Nwcsh`OmroO(NTr7sh(jVUEY{Li-zo>VW0A8XG0nNGx;RV zKq(zLErt48nNUsHlf)t7EEkg6#4bm!z=NlCSQ_>YfmuIcB0HKdtgz%OvK{%I`Y!z1 zZPvWT1Pgv=ur9BjIR!-UjIz#rVAES z_ofydr2m-=9Db2Z`H)Y}S*;;cgjxIcGya@&e5d%t&8O^1b}(GYevB*1g}QwP9)4ek zj+rZwuOj61#2Z5QxW3TmR0aL`jKxk%U>}zWH7qfot5#~`3LhGf>6LRx+?f?5Vdo-} zHhd`YmCod@8Hq(x=r-}Dk1pbKF}K97R$JI-??(3Wp*&tW3`So0W7ZRYlpW3y)^o>E zW+(9ITc26u-qYn!nX&{owfq`TzeRHuLChCT$S54<`zjb zGpF`}OR3(s82Zj2g$@Z%qf3kT(gvSwnl!h7RxRE`g`8sA_eP!y`HEx)naat=4Z>x| zd`vvZV~dcDx&Gm6^let;x9b@3H{MwCm$ppgXD_nk)hvzqfP$g?tv6l5oJ0(bc_(nB zau3YywUOuYKZ?#Y9;>el!y#jmOl4|7(m)c$*?T|rFO{ScO^Q-!B+-OQAxWkXGG$I7 zQmHt5Z7C`t8Z;`TqD++vsd&%(!O!={vyXk&y6@{!1l`(39Gx3Uu0(z!Lx)sF(=Vus zbkhcjxBP2TwycoU`h=2Hb>Z&QCPrf4+`;dLUZaVxnnCZ32QxFtW-d}!*uwr4W||bm zn(RH;SvOPmwpEECG5||UYQQbe6b6+3p~k*weA)4qkn?ZF`3b6|b3jDgt>=;g%L(MO zfh2kESdDXAqp`P|4xZZnjNdtYH8|b<3?r*7*vjGi**WJBre1u8^{t6y?g3FOYfKdT z@j09w$U4cYKe)5b=B3PV*95lj^H(TVx((aDP-u?%2~TsTGW8WgPw&eCChqLb{(acS zuD!Hj#o>Cw&bJZld``l%U}Nz5=nf0QD`3H7F&6PmiB+=k%ii71CLD9#3on#MSPBw>?YQ@yV$IWAZ$kq#)0%)@zb}+t=h} z`y|mx*EOPjFT6!&B_X1~YjL8XoOIFlo{J)W$$8OQ6@O7HMWT^g0*H8}aY!{JxDtF1NxRIjD6FQ>&Z4*UHC#r~c#tsvWpH)xx6(o}j zS+j^j;#u6hHI;7geGW;2NB&aPA@=ir4D;!|#O%zA*_p}$HdZ2%^^XyF$AaJH?;i`s zR7_dyWDQ2Abbv#02*f`z;4l7;#_jzY#ALHAk(s}nBpUB09k!c^y~9G{o}@*ppMAzI zG3oe3a0gltHWehMe+1vd9CMI3$OgVgvsA4Np`(3)1$8E~D9dp6uEdu~zHn!`t~;6e znXT;n&h@Oi7YnYusSLMFU@gnE*r1T1+;>xxtt0Bp>#i2tE%?2iT?-(weg+&8`O>5d zy>zP83HWVY2s)jm5VI~HmX0|MpUoYC{hdY63%rX-tO)NN`3=v>c#RK6gyP6A=E$=8 zESweC-kH}iE4sUr8I(mb*|rjPG6vYA27e~!evGba&mt3R^+lbTZld_aD3Qy=6w&&Q zEYaF|mqhCWvqUDt6GSrPxae$%xhO>92Z`(qB?h8DSZ3xg^K&T@?1=GRwx~Iu>9l=e zm-I)Lo~zL<%_^Hy>fE@r^!bSGrE=#yOI7>oAsUTMSCy zW#N1AO#E!t5N=RhO{6}bARhfO#H%ZlSR|*B;9pT>+6PZkxyFFpXv@V@&RwQrCuNw} zBNvwYBZfUcn#~%{7O+N%LgBqVkBK?vvhIj1))jV^ksIEO4$Nmi0|dsTz5+|WHH>+h z{Dg^SjS!$-1QDwPA=PLKb-Lgiyg1Vb?rBBal%2ob7CXXH+7=1oxbo*q64CAWLUYL80+>Kg4-2B zM>K9ebJlTZQXAda_YG56oO=j${O3$Y81|DBW)>pt+MS}W3xh;(C`M$}dtM~goFWTt0o!_lyrtzlV^2-_~Jk+7zw^$(?^W?CjDj+E~85KzSSx%sMRmM zBX3-4Q8KqQC)T#qP_(9W^W!C@J5r~V{?QOCRd|@sTINn=r{5}~)eB~kMw~^yZF^5( zlB~%7yqakIIyKRzW;u~{L=WkW6L?_VvE~P~>(Cn;&9nS2Ki5c^9Q(~X3f57_wHVDod1n#gBdt9K&{;X1AS%t&d0h3?Q zd~pz}g&9-tVl&oJvWgj;c3}tm*R%0za~Y?q&AhJnz%a#3Xq%(}^Dy_Qm5Dtc+vE02ZYX!x8Cteq=ztz|frRW+@Px|++ub#= z_``jeeDM^l(2e1us()j3gAl^$G?F7$nxbkUFQI9_Pt>~ev}jvGl&JCU8PS~H69OM< zqiFK1X`=Ap9oihZox?@>|^u4tYntOi&@|O z|JY!v1#>(f=PX}z6xRWf z9l%3~!#Y+7A?p zvcXm5Hdr>7LRUvR`1kq2NdKKMTr~tMbvj!&9La1227am3j&?wCb_}A|? zY?HK#xT#oN;tDWcTW^-f0P2+#Tl@nKMwBgjs(3E5fE)E z{N9)V-25xBZBiwahc|)8m#^?U?=v({Z3L6842~{JfVkH_Fn_)!tT?C(&92S#biEi= z^ie@KXIG+T>EW1n7sKO+WU%6b5%}+ZBm9=%hef_;@wCzD_|4uJ9Ps`G{&dU-56n1- z9}Avd=?OMCwQwYk$y$LjMy#Pjp9~;W$Z=t@({Su;1QgXr34yI>NQpcRg{Mz|_pea6 zwLb=?3qLulqrf2d3_MDSg_A1+As?*ZkEIg)-P1{@YtaCpvA%YnA2j^1RDj^t=}oXRHF<%K~@rO&XLpPzXBn2(%iW zfyt)|SlwF)JJU13%O?;NmraFQcW;_zvJ}mKIvjr(T!rsV3c{E4Be9Wo99B1s!AAt= zz)(&y*87-%mpn+tG4G4;=#qLYme`3ufB1*zybw6DV>c?$63-9g^r95}0FgYLovD3P~^-Tf|9EwB;grJul&+E=inERQ$e&d2T#lCXkK z7{0i{56fiu;J|@Hc-t{wJTv4Jew`PJe=a?Z|I_uv3%*Xk3lesqFzGE+HOvsIW(Prv zZ7$3$x&x|JWuUPm53cbFS@Sk10B3lPeq9`aL;ls*nZpxS_$VsvOOPOBrrI0x*TSR zT?c)!csL}v8`@h&f7&Qti)&8c@9Jgv<(GCWsoRCO|9*lG1fRvjM(E?q zVrJ;p&LSGUa5bz^z5<2rKY%o=uqhAqSWCx*C{_Xo@VZX|TZ3 zj@z+i>Is}<7=f#Y2VmWI?pPic<2e>O_=V>WgndKM%ehOCd_oN8q?X4i?fuGap5c#z zGpdo!FFCAGtdF;kn}VAMU!nc|$GNVXF?8L@dTQ{*l`j99#8u50gO9)3iQBX<FOG7s)-iYRO%E=0plUEgp)GP4L2nCH{Eq7=J9C5RUiGPr<#vZ{WG6 zRd`PSb9~;X4qv=igdOyfv4`t^te0ttt5inf`LA8kpH;*81=9=Z;VV)QcWDeTi&2mt z+f1`vA5!<8C-ic*G`!L?1dI935HJu3rTfo8q-PrZyq66Aqm#h#Z5VvZ-wr0->QEoE znWq1IXz}8N6dD+H-@+o`3|+iZ6|xT6LesXbFkrR;lr2S2)~^VgyR+$*a4|k*f;Y0r z5W@??MA+o^UR)d=g#S~D!?K5yv3d7doZ#k&^X%WFXTNoM`SmJr+Ajo#f@&i6rIKv#NW=}_fPJw8yc0RhN@e0kZZ9;l43y_LT75DkV82UxEfmSrl zgkZxh0zYIg$Sy(fvOI+*-`RpjKGnsS7Wv@KgQu}!;TNI*7T^7KN~NRVzt z@Gs5~W?#Qdk38Fs_C7Gdmpy{^08Zd3Eq?O2#aldffFUF zaMRgRY@eEsqxW9GldG>`uC@|)$24MO+=QdY-ouekBJrLvQ!pnMf`-L7ng?#3Nw>m$H0Z;<9)mW0`p6=3$%*-)@}FNk?Z zz+Cf87=Qm7IM|j$_K7F($KwMecJ%?c4`%dl@)nypJ8g7TOy_?g`X;X2K*`|L~j zcCi`cKD0splNLDX^b+`qC2&ad0{B%$gWZk*(26()JvRd&Y#LJ-WqVf63h3AY@$h9DUp(j_2!YV#ih^Tr6dY^B1^b%@gtX>C-Z7Y1@r^nJh7~ zA4^sZ8%tWM#EI0ATNob9!BUdpT+{qEsuQ^Z?o5vYt(+K$eB%XPJ4S;2w~Uf1xiNT= zTMVugoTeUrpKyS_7&*LAo_sY@CqeNO$wSAKt6WVi7imkgw0Gwiu-2pZp~!M3LuylyN2e}SuAA0-EIy)E>HtCYZzw*=+f zF!)(^4KyTb;HG9B)Hg7==2!qr1i#UhTd|<`Jqiv7g$T29UkD5JgxGC+q2S7X_+)no z65O}J^1mE>jO?USvcK?(o8yto)oEDE#UD$?M+jc2eYkd)CLV6rV`08(Gz7dj0|#6l z0AJ7udF`*@*ZdoBqhKHG4B)BDOkV`=Z16gP$;7NH@#zI^cwg^uqB~DnVD@T|kM$Es z>}y?eV1ofM&znK=#?L0}3`In_a2}bgiHYIO>BQ%}68S0Kg_nx1;yc%l;c1%)evl!H zU#QiiUBk~K^V@R~5m!ZJ2IEnA)>(ABt_^iQn1+LW-0K?m%%}eIQ9EZef|(L#s>V{<$&}i)6$Z|7HI?T(49SN z@urtC*zs5%c9gq-#fwj3XMqRvX7!eaL7FnIHK#mUn#YYb};#Dsy@rmyx`2REtkyAF#9CZPojJkx=&`ta~ z?-o`(N%8G>JU%~_$3|0c;h|E7hbcY8lA|7DrQ8PGapo1iWAhTXmDXadUWUz1CF7wv z9(ex7@pxcaD4I4umGgVASBw_lpLN;7wMGo2 z^w~nU(X1=92Q=(!VU~3=sG>z+cl{z=rT2z2w)lW#%8c;Py*W5#zY3mr;4Zo&qkwLy zNYiPRjGpqD0LFA17<$G){jVH27E=O29e0HuGlS=F3w*xkKw9~E_&zHFX3GVDYn(r9 z7v`^j9X1JWS#u~DdL?in!%K`F)}!+~S7Mpuv-skti};V?4g5m+3ZA$>6yLfz1=m;4 zMUfWwY13|RkP9t_ePcer_IOz)_dr@ zB<+RdkQ*#E1@BY!kjBP z@byCq1QlO{0;T&vGFrf_Z5T^9CdWQ2Dzd^U3QTILI9p=d28X_s!K+GtI5@qVju<}* znLV9{$A7wjm&(1wUD@BTLwzS+^{Ec8{E~^YYBuAfC;)4>(}b*BaswV~SC6Ks1j z;F5PWtX%j7a=eAzTC^m)6eh*;LL?bd7iW!^2H=Ch-w6({1ijoGs4x`xP^%Cuxt~j0 zlAduZw~xVb`tDde?hFpR9ffZzAIAg9>#$$_6x>#jjmj1&Qj;t4P_^{{JP5o7t)HL5 zi{@X@-Yd!O=}R*y1sRqiF`Ts=XoHy8C*VdJ-UxSI%90nBS{zCum&~7Y?vBX2{ZpZgFox5fz)MzWA{Ng-Z>e%zm(FS z@@Z7jET1;$j04$G0YZ8<9YzJ*fuqkJLdWihFf8E(xUYH!!;(5dFGT2|bW5jg z9nMZolVJ-#D=?i#aVF#Q7QCd=AZ4!>taBZ0q55(rR@$A8vHw-PE4UB`@yXa|`VMUR zWg?EBV1~*~oaq8A1=o69z^O3^CKqIal5?5xK5-wcGwWc^(x;Gku^J*4Jb{b1>mmQ# zV`wm{go9Suu&X)%UWQsi>A_#LPGT3meseBYS<;On%NFC4n?rGMcLttrn}9uxqp;tn zy?B?hDc17;g&Gn@qin53)OMN@gl*dc8(18agcZTN_l+>MVE}%P9m&*VwOP=*$?Vfu zbC%mQpB1dOVnz3?S@f^jY-aKp7G&B0N`q@4Lf(xYc~XJCjopN8)@9)tanJDd*+1~2 z*5PFMBPpU>EJpGteZ>nLTJhWXcKoPk5Ss+ckTO3dlIo^MZko;{^(jPPWmysKusxaH z>Q2fE1Bi^u8Dcy>g{+ahKpahS$x)L$;@h1`)TSkqeA5WBH*+Vs{mPhZ?R|&CH;=@= zgOzk=K^nLn>H)7^BiZ=i@$A6L2`nv7gViXgFmp>8*0l2v{I}x=qzkOi$(}8s`S2~M zIn{x;N(D3;WPsZDFj!-`6Bfpa;I;B{a0=K23x|$ChDRE>%qxP8Gs|Fb&TSBlNCwX! zH#m~gPeq+PS5w)FPPMGYuN5P(k7PVp-gmbF zlcC3PF=*=^fhU(!!1_utY)g9rYxlH4ZQu{EER|-zS{2#mrJ5|`y&jvHHD36qG?r}? zI#eU{o-$Aq)K0Ky zr99dp?1-1$@&|`cw_wHVpD?;yiG6=RiET-r!(={KFp1TKeG=yI;!^}BOwI!47(b8s zZ!u@T98FnTqArsZ+(%k%s;uda1k>|>3vXv9!_{bQ*fcer-{~rjEyX=?gw%Q5v%MH^ z)GEZSIZ=4YyW*jjE%Vseyp61rxUyaGo@{ZM4;v;rzz)7Wz$P~DV-7a1%rMZE z9 zSlw;oHlT@Ey)h7by}5yHKRv>{kU!nKWdK*?|G-Z(e&OHS+iaqa5f4^a;{qM2b@@l-h38n!;bLmK*doieU?UxU0%k-6AP+)w$cVOZvS^Ph z`L~KAW=^BYP)0ot)epqhAtNyVG0fcUT9qB&Lt_nNtyiLbgyX$)97O{ZMpErlSR-4Tpo?pQ37_VUoi+8cBU%iE$_aXMm zc?Vmz)`~rOB;3Q@gsjqrKXgxj1G>_27W;TL<3Nxib$wFAdFU5b&b*DSR(Rtd@0*aK zYd7yVN*-?B=b+c%KR7W*L)epFpm8&QSnO?YL~mtgs z@1$A4{*+`0IPerKV!y(syWe5pVI?R#`M_m9f?CJlMeUbuv2k%ER-Kn9%-W9Oi#dAu z{5>~rW4ba7pPmLSj(wo6CG2^ejF{o(nQZ<$1Ey}G$_)Ga!R`(ZAC$a7Cv6mrmI|hV z2Z(;#uS z(;O&HH-+t`A~+Jf6>{89!Im|tFnWBEFh4AUsEj0V30eTbqD-zWCK6Zd(iVg z*`!4C8EH@LB>|VDMUt~ei?*Fm5*;s65Z&;W7PTb(A_vws5wVym;@5SZNR?h74h@Oq z^{8mFqdAKBd4-TVr{lzI@pjU3(U=_C+KtyGuE08@pU`sQ9oAxx2Aia{iWLgy-DXn= zJ0;{l_u(j}_U|0)P)cOJ#p!Hya1kp`zr%*zc*;`Fy=R9y1%`%u3)As@$?WM}He0-q z9h)7++#YRX_3!5heJ^p=U0(`#TNs>wqYbZ1|I*zXkI|mPR;WpR2)#KGf^E(`!|&W= z$dSrf(Ox`NB^>o6Z{~Ntj;6*x zfu2cGqx6m*uk=P8C5(853QAw0D22PoYPT`swIk``i5xr^&jXA0VXVbxJo6J!SGRRn zFxM%z%(C2y{ctj2H^fFW&GPr~cvlSgUmOP;?*&sg#k(bGzDE2gY)iLY|4By=w}Dez zg2B!%5nA6R!v_Coh~)#o|Iro*e2?Mf!O74SGYO{WabRn*4d%2)!GFJQ0iN0fQ)2(Z zOig+AEP4zZmN=em-Kxuc7Yn@llcU&$`d3iDeJQw4X+!Iy+i`{Q2BNVxofLIFCAPZ- zj^fY=(Qe`I{L)`Z)VM)Tv@3Cl9DUSI*79%2%Ofv{RQn4ua>-kAKC_K9|L!F1@7u`Z z*q7wmwd>?fbubC^H6&4k9{72-I2iwK1{+-iHut}|Y}Y><7ErmA%{j7&H5sj9&kyfr zQFf=nf(Gyw*IRaJ1GAZ`fIA->(n4P+Mog}N2no}(H{8FvBy|)*LY$g@Fllx*+RB> z`I3a*01^@(Kmu%hiT4v{GI8Bva?sR>IGD+kX8B&+Z1fWUH*^>KkEzC!Qg~czbs0b2 zc@clV8iqHlScBJWd5KOu^0%v^Ie3q4G z%vK6J>&<(VnTz0fQk^8h?1k)OfU4)tqL}KR*M2?$uMEHlQC~KRzXyf88GT}i3nG*k)%#&yk=5rs3ZrKlF z_f=d}sy9fAj5~ zLoQC6PuAu;l7zW_MCR!!qURez&V2|c%69{aRN(;4|~Dz{CehjQCdvVjBk z$T*1`tbK6Jab28r8llRfN3`?iKA62E4Z zpW#G}5qs9}$++d`SXcNZ*5f3wX{ys%rr06o+B1UL#TRf@_7ln8Rj0`pN1mMIUXo<@ z_hk5z4)V!SaMt~)BU853l4)KK$e#VrNs7`>@&ZeXUU2H7`ZPV!Z-Hr|X+BY8Nt8wF zihq-)8P()~VhBm78AsN|p2G^)f#0IG6oxiO!i}&xu=&#j4os4LEttw2Qs*=Eb>3{| z#UwWQ;tl4m|C$}N_|Cos{$ox|s&v6wsZzHd@ltc=zs%&rC-!9jb9VRKeWs1Cv7cKm zu+;doY{}|TwD9pix^VE-|T!8|}PZ#8|xKW@iP zzp-OY9jn=H4@YJpzLI5~UdOJ;tzk=kFJbDID_EHEBsSrXi;e&UziA|d; zsdmaBW9n~_Z?ZMy?DrQW$gh?dK6yrr-`115@h^$K?N1V$Dk=IwM~iORju&k*n;@Fv zI$p#(Ylt*UB}FR-8p-zMdF1*IN7CRnnk*<6dUYvZ(4(^%$nsK`h3VEP8mQMrALpgQ z&9lSVuRHUZwvaJDZOFRLH$fUO04vXr)23pKk3q&Jz}MH{sZiz;a3(M*T~`% zUa+`v)htq%F%#J$*6p3nPW=sMS{8?y*U?2RqE3a~kuHU}X{Ge8x++e!e~6pgS@NRZ7oRa$sYUj0B%H_EXrtD-D*8ADMWv*;zD96fVi@=S_ zqtp$9Sj^p%#8sUqyB-12YP(M?#Y@Q_y%LfZdWm>lOD1QkF9;dLYh+T+W8xFqOKR52 zh^V!SXx1rpQC0hBQBkj?X#Dji;<6}*G&=YYH&YF=!mAJuTknb|RsTZXmDi9*s3)3k z6HOybte{7}0=(x>Wq|?DO*1W8BP+ ztG;3&bo9pW7c$-Th0H7BJX7)yWsdibG3WdC%vbQ(4=IdfB%}y3#&9sDSBF!deE_>8 zHwcbqebQLyOq}F`$fJrlaXDm8z{f1mg`J09iQ*Z^9(m>{HS#y;zp*+$qb7 z{wXpGO<@iXQ`piI^VoH<73@LZYWBBoIs4Xt*^XP2SwyBRGrn5~8!Mxr=Fl3jT&My& zc739gKfI#HOp57$qp#4U2p<|mo|ZJ;8-vaTWuZ;#gQ)hPDmLraz}edkvB%Nr_{gx~ zIMI@#55Ya$7F|O={N+iizv>@dk;=g)gTpX%^gI~6ya$tlAAu;d5oFIafl28nz$JZP z*WLz(g5TCVAR1Oh7SdhkB=D8ybS$+?f>hc~B5I~f$jHduWa}AEVw&VZOi#LzO|!O> z4qabje?Oe;nv_b~?J2qTxt4rMe?y9=d?1qBUy(!yM zWPaD)vkHaRY}kT&W;^XJdwB3V^Ucj;avKuagvv-}A|1l|gxsUK-5!=(vzqA?8?g&- zRoEja1HC`$puXaqMM~N(fm{0o&+wKYU$;ypt^Fe6VY`I<3UwmM0(M;xg^}*b zk%VN$kZ*G5NTPcA z(k$Z30%qCc$Fkg#Sc%ak_U&69^N!D9509jAwkq10Cb}LmwC6WHN~%sGZoa{DgnU6#%XQT4 zH0wFN zY-~VpvqQ1X^NYA*-g&IoG70bfnnKaoTQKjj9{c-i8(Y^Jz?!y%Fe#@%CZ4mO&3m?$ zS>>)|^@Gb;$C-_6uHimb{5gQ>ZVY9+E(Nj$p&pD{uVQwIGntvIEL-gL1aAIFg6`WP zFm6jIWb$qhFi+UME=s2}+SO4-<5KJx$Z!`(5)TVqQgX$VxSW|uvi%H+#D5b=WZ`IX z^NT!LK%~ik3i2ch#*j@HG|31D74p9O2kzT0WDOd};-xdf_(^CBj7q)_nc@GSWwR1{ zrL4%F+9@;3Yvb6n#p9Xb<*96jtU1fJwPgNLYna!1d$u4&By^L7{QLrC=HA`|uZ65( z{o|eRrhhWr)vco${u$J0Oc~v&yo~zwwwTM=o?;SV&+9|Zwrv*)~ zRl|3W%)+((uJ}VoAYQ9*0(%5*#czb!u3wc8Zn_eSA2?8Kyzd?U@>Pn2y&Ok+XBv}W zPeY+|B-{!9iILl=3}ZNj^+KlNxa|ArY-uHOQyoIq+h^l7?%_D`K^{&uxP>SBUBH!_ zo$-Ogw~*ZMF;t{J7P6$b!QfY+cNy;ok4iU#wBa$F`y!MY>2xwv^Ov-9%yZReEPb4Bb?^jw;#C;G^j*uD&nWVy$ix&+Z?g zyA$LgJHQHFJoW^w(Vh^q-xJad)&N{rh8LMPsc`+FA1ZN)#_{9&!Fp^RAn(Y zF5(^c>K|{B@aqLzSo{wIM`hZf@b@d5Z|Sr{G~OvLr}X?S^85Z)|x0H1bSf)7so zi>z7`(dUzy-1H0fw7z$;FcYcc)f6uC$6~s;jY3XO>w^b!`u7C+u93oi_P>yknFL2%m=O6B-d^tR$4 z?>%NFSL=5awJX&k+ZTOk_mpN-XIy}yyR1-K!hW86e1MKy8IfAa3YvcPn@0M;O0yeK?go+(CC7nFuuv z9`PA z6duJ_#lx}BtwYqoRD!*Trh+MB6g%NcSDy?dn6mRhVsA$dBbGH#u+~ zpPf*kQX0}6Rfy!ZBar_U6Z9gOan@%-`8AD`=!?$(Xyoq&bo!O?G=AB-bV3-(s=AXWo#6$j+MkEam}&&=%_>( zvI|y0%Ko>wD5GAB(&2_&)Y%8z%FV{;x^*gb#17 z*ut9~5YdI!%jw2;JF2jCF1@u`fmS5l;fHhaylGzyzcwkAA5iP!#Z70@$fw7sm~jTR zC(SfbSsa`dN5TPfZTJPcH4WwXdE=C^yjPj!rsX?g#z!Py)83 z%E41xd3bCj3E~^xQ9t21I3QzAvz(8XtgLC|rfyC_W|oi9o-eP_V*mGOgJL;yvCDGB`Y6FStTdk3w(d5~&ahf;~gBpfzbX(ba%5q@$3Hih_3`b!5d=r&jTX{lWBDY!xMaKdIrx_jEAk z0{z-yLC+;?bAE+O(3mGTki~}gX!g|?sG;vXGFqU7ESH#L@yM{}TRnvL)X>Y8M}C_>Pas)uwr>ljziUOX*yuxm%V<#lCk_#7 z|4@a(I=bv!5iP0Aq@Q!bY3~?6YFT%h_V%Bpb!(%kn`$JLxEw`&UA^dMQxUaVs7mku zI?sDIZ#DOtf>|whv9`sbI_51nP^>$bIG?=X0%|J8*P0UO@o#nriI!I>2?WMy6TXyznGXs z?+f4WxcJ|6(peeEUZnzAor>@*;yazUr<^w0M9`JeN;K(iDeq?2TJm*!qeaMr9PU@j zXp~jA6?ulHqv@Iq<^6nzy1%_c-wm43e}^BU;Yo$aG${{F-c*3vzt*9m>~BaiTLuri zI~F@BYvMhJ74Y(%eW+ILIhtpZiD=moizD4c3dbxZ$1>G^6%y7ao|-H_#y}~ z#R;ffWhu{_dC+vjDq6ox3VNQ80dY+&*jBFvpi) zk-mHLimuNV3S#5JXra(ObggsZ-Otal;5rs0vLH6s&)Rvl|KmQj7yWvWfVDHyI@YqQ)gyT7*)+x&HK(h z`|lTbQ|c=>;ra`UExVISq}L4bmYj$#G7hGXgYsz1syKQ)JBDVDai=tVGp!fyxVf?G zsO91?npPe`4Mv@z21kAAQqNN~M(8?(?AT2=_SjKZ|LL?PP=&7VddY7&J%ityu-f8k zTp>rS=b>9h-iW+uLhrw-VFM{5WcA(g6pa9!P#%H{7lmVYhhRKK&l9gYN-*q{!nmmz zUH&^4WgidZu7#vnT)nDc(I9=8d(&{6tJR%?ZW@e2k5ziOqh^X+$np35i1M{m#r8VQ zI@?TR{(Yy`>=)Ia+f6rz-lezRr_!7Du2fLAQTvSsG%`__o?Nhro|6uw`m3|));HJa zy*qhykM>zwe?EpjYdJM#X{1zf zgX8B0@gPxUB%ilaLbj+mr)hFnhBRgr$kO|ElHJe^AG^T37`t*>I z2{n0WOxqTYqrXqz;=T86EOGhMV9`sTa4qk3P<)>fia#Tbin=|7c&id`@2NmbUrnZ8 zryJ3a>38@yVmC^*^xxn%Ow&NiKdeP}J9ePrkA+;&mIQvswLt3FUqs`Se^P1lX5qV^ zM~zZi`Q&8FMVOOI?%VJ56o8OoBPpTV_M+WcJacS?g)k zjQKR_#a~{sqP?Ulb^|BR>N&sa=bWhG6=xE1fO`Ul{JKSFc&~x4e63#~HIIp-d(uwP z12w_);vQdGHHhfJz6-p_WIxYRX7G=vrv#)N%{XQD|qKx zd%k7gMxOsUtYrVOV2jn=THM+nx4A20TRHEFktlHSE3V^EEH?v%SOnPQ@IUTo(}a`m z)NUYx{@xu-h+NZrL5X?pEP*PQ30 z@h_Jl`Q2-gTzv-@een%<;cF)M=aWB|+%=NJYh}5ScEhBm4)1Kec}H6qL1r6 zIu6BK%tRBP_j7+VrlP4GLtM`v9qw?WB$qYM$s+6Kb-s2;g36_5@OLly@bcle_=uiv z{?xft{J4`--NsrBD5QX$$LwLnQt5`MtN-zUR5j? zEV|^lOUff9d+m!W{vAGOue5EBuuZL1xPP}-_@b{#{Xv!P{h}w--?XZzZT(mNOxVXA zHhIih{Zc>*6Gx(*55^(`FAb!=X$Z=UxWg^UJH$@w3>)Fe3OHO{{|xOI8So z7XL1X`xS<@B4cq5kwh%F;$}U4$Nk*a%GqtSLEnPHP;tjp)S(}PrgUYakmY-k!VrIC zTO@-PWxTGqw|6X^qCJZ$X8BWRlrEmXKN1b| zN`h8@jztA@f>nQRg=4-Nj<9P`v#f8D$MNhcU zqf7B#l_U5>RsjAp`XF83Q4Wzhc@QcmjoXc-iJIgaHe02Si#TUc`?(Q(65Qa)hv8yx zb_DL58jB}x9fsIXtuW=(?kAvusoBWn zW+K(KSp_;Jtu*`8B;n57;jm=ISvpGhfPLqVBV6q_11@B}JWQ91rT^J}MR#n5;Chfn zYwB~Uu6rt{K60br+}K8&d|q>JJbP)U-haa2Q|GZlWeaK<=!$+SGW1<(H2BXvEmSuh z7w`;Yn(`$Ahik4w4sZhXTb=ZLc@ceRQcGK!=Lvr|Wm3^4kM#*D^&#%lg9`b#^1G1-|kmOI^ z4(&m~z$Lk(mv(LFQrjNnWjY9NTGxuS{S~0PNE#Ndn}L$`%JJ?dHL+*72$o0sVxJyA z)Mp(6C(p=XsRzIDQ87=u=xrY4DSW{u6Nb~ zCHE`9C#T|?5l66`^Cg)3ROHDm+>Q_45}k&AXMo>6k{aKBhlk7Ug0NqDY}1<>tlfA6 zf8p$y)BxW}!8Y4*juwR-{lA6PBm%jHR9F03qD8XL%h}hjI1A49sdPhZG(OSr2Au8M zq4_~9aWgv!J#izL%wKOZ^tJ+v4=DnrK5>Wjd_RQOhjM1BV&7tmCgz$}!g8`7KJ2K5 zC((hZWSkrsVz?cTZ&qQ~<|jd~v?eZ9O#_PtS-h=hC~@rQgpj*oEO@sXDN1!CpB9+1 zV>*Mdsii9n^_t5bRIMd*r5kW^{VP~=vl=!fsS?9-N95)=i)D`v$M(d zLpILK;jz|EoEK7sRg9;ybDcj>$ojFk;%*WBIRcTPygMoSD{@w+?P8PftAYoA2>X<; z1XV!^!?P%IVc=9qm>vV;v$EmDnyJijYyveeYQ&Gu4JFxq1rU)whPk%o;jz60e+|9| zQ7(Dl;vIvRf6BlmGXv?8ndLZaWe5)1@(`YZ4{KDMgwKrV6up)g1^L?+WZPVAX6mH_ zk0LnYdqjp!&Po8e6$N;Id>}kfPDhp}n{i@)1bymy0>3E`Aa?aqrY>hi2AL?5vD@63 z%BJsl{UKBGyKn?sFgg<23;glOTW`SFte2XUenZ_AA`fVG0RF7$#%e72E3s#N9$Izf z0ZLdM3UhrdV4JEfTGcojyH1tDEn{*~&qg&gZ%r{=$c)4`RuTBwhImNWjUZ@cBfPkC z8GniUgc}BmnU*`U)a$Vq-50wbTGwP?mD^_^ZO3JJR67Fe4LpJtza0mqQ#RqqrF!_( z9vNnPzZwOvoW*sH8N`imeS}`wyhj`Bv+&AQpJ3F*2XM}G9DaR99n8Jd$ntCjlHHpN z<+r^>$MQ>bGv6GDQ4SsTSQdv(X$BL~cir>d2d7-of`Ixeylr(a3}3hudQ`UJx}VwN zENLIMl8D`Dbwh07Hv>m4T?jITGr?d3VNok?3R(vik?vEKNcw{tj6eAmNuHg=!_Fzd zf!Bw`ck?jhDrPuMIXj%XKNt+w-$h@%_hOCc8hE5M2V}k^;Tppf)HgP(QZ&fzZ(`J~NGUOPt%Vnu$^kv^zF`N&E65w>~J3g?OqVP^VBQj@31 zxYWlmF>VbB?rEkkW!6fpnkJB8??$mpm5MCq*F>E5X$=f?|AY^#2I6@G<=6$;8~FI~ z%ed;{EiTb;BLs{0hZ86o{Q=`=L@q z>i%kc#v=z#yT7DWLpo@MsTZ7>UWQ&zOTeovmB9X1Hdi{J6g^kJg*+DcqD?!JVCzU( zoIc45t^ctDtb{_m%V;d_YA%Gnxqp#+lQf>XUkbEmM{ucI*5bn&Ib82|d65s0LO<(; zK)CX9+~U0fjEC)m$=z>ort1b+Fy$w@P-{dM{`v&BZ*N92>SefMy|}v@^Pc{6&c{6k z`zp#V%79jJBI;hF1yZA*T(+(Mh>dktz?~IND9?NgZg^D-2i;F||E+kAqO@FDTG4*k zce)sNFS5qBuCBtZT1T)|-W<+LPyGBF74hl-D=5)w6?rN;xL$oUZG85b)|+3zKP!vr zE$`{je{TkU)8~R0T^)lpy;e|z$nD(zKTUL~n8#jd>;(-mr(o9k71+D~6s~VfLIWpy zVPoA)=*oUC4EnhiU+iB8L;mfjEe)-xuR4pW{NicbjXP9IuEBo(nQ?fMehPg%ZU7!N z)gM37ZKPhVgHT+-Cp7$|8)$6UhccVyVVmI(=%lvGT%W^UTAU0o;8wDYoL`K|)c zUQ&fhlEvR&`7tP+dmf#9ua0MwI>OTXZ&3IsA6(Mc0WlYSAbf#4zCI$H@-=q&Wk>Q0&(o3CX9Hg8B_lJk)a!{#3e-7I&V8=P7bAjP27#7VWO%WBo2E=z56}D zXH_+_eli7XikPCHxlhp3h&}JQcvFvz^BJ9HdpxC*r!j zcKGWm4|KV6D(X8GjS3t)#NV45zWr|_ZQr>Ysz?5$dqa+M$BrMNxzY2fp{yNlHVzil zYm33M-({X2zVN5-eh@WSF zOC+vd@g0S3PNCKN*PtG?p}1>8g77QZ3Rue|+ByKcWmELN)rx8_y@(3sE}*oVGX?dTY6xO&;a>y?Nk2=mMr=Hup>TrU z^M8*5?6+e3-cpJlDnQ1m_w>{(S9rg9GNjP+=yAkGJWkO{Fq6}V(LZbGj&MQJ{?igB z?jHkkHw6?YBko%ZH_|!D8@YnIQ#5t13v5c*&dCfmK(8lSQ_WZ-topMV^*EGEA_^X$ zKL`Ek8uM^icjXL49XHU0p&7{G=X-Q;-V@XiECpR7x6n|Xc7a=MO4n$-MdtcrkyK0` zZ8f-B z^>UjPobL8eC55Fu2av^!zH2b*A; z;l)i9a7Pxl8YFYR33=#3?`%5gwm&-LC@Ve-JZRTwN1;RG8rpI0gy5*;V}E0H2HL(g z7rp)wj)tt5h0@ZVNY1-PaA>PEs%mJO`OnO-#d$iby-!YUZ(+l9Cw)BQ=g2c zuCc%!+4fZZpb@mmFXNn|_tN?Ebm^yCafn1UbGkd%(#f*hXd2ZLW{#2J4mG?*@B5?C zy&tl)hgX&a7=_bcQQDCCzJgApVbpQxVzepe9g2_7M_xD)T}-q?eJ^D09 z9`4jWGewKs;~_~~-_a-B0M>D;;sfH)M8z-M17P_R)hHl<%(C@=_KTmRNH24KpeZlxHAgV^{hH?G;oym1#1X z3EQmC(qRF4oMd?%^*0QrX6Tp@b9V-pSgwa_4;kXj##Zk3##pp)=>XCBnI(8k&yq~m zh(pOIw$V*bLg>iJ4k+#YTy#do3FW&sqKx`K$o54TGG8Q~OMiINmtVSQ$kcmuo8wdA zd|#pDhj@+WRSc(>%noyMR@1rULl3xYeu-qfUljWH`#bvYcnZ)}fjQD!6(?52{o>gmR4X(7X7HXwrfQC?s_`;&0Y*=cHu0s12KRHU^25X?h$4*PEAL$7#N0higt0%}RC=Z3kWpdIEwOsrkAL??nl}-wl zq2~@gpd0>iFm&B9+8SIhnEN==x5qq?oJf_n{c%xJe)w|5OcFrrhgwkbEuNcIHkv-a zUP%A8CJXIp((v}O3q0AAThV)QEIN{XpX*b}M^>4)IM-S^!8}AC1#6zJP|5vI_`@xd z+}%1v@Ccw4L%*z}8>24Js^U44zM#L7;4!o5kV7tLrHKp|;v%M&)}KcMCKz#Y(Zw`H zSAk~L9HQnk!)T$DAG$qHjWe&$tEe4lhT6qFYwf}&6uWOTT4M5~oSXfGYYFY;ZWcWj zI#N1?fC<@xspkb@wPcat^xBqMJrAHGGWH9%v>Gdv@+YA6E#@flzZ7P3=lASE!`zG{-qumVQNIFI zId%b3>q+LC{)Hj~%~#xIg~gK11`ZQA< zI$oSWb3)d=V3AiD8QB7HGFk=9L>r5$JANH(X9wr{$zk(5NjM!A!5LYc=bjvWAWTefqb~L(LVEcr$sYT7;pJZkZm!-&ba(Dh z)EcLPmKS;>SG#bIzhFiOU$>$?I(Bqc=M~!c@*(|txk5;Z3=%GGk)rPh>7hdbfykgS z6uB)6MIPs7quY`0C{DeIt8N}GY9cN!q)bd&woElqBL_9dbx8t zYLWLt+Tuu!CPbmS7ymdZDu@2;8BS}D%EFXMDzN_7I$FNZg!X$n(Mu7pTx|wPSSncpECEo-Q#hk)v+g6zO+QW%Ocl zFbbWT%bl@_;EX+rxtP>sVaBjSg8GIUuKwp^`@&o6B${6SNCizn7nH0Jz2}IkhmA#@ zlQfX&S2Z-JzLeW_WVG=7W}9%LHj>Wh2&Qqvo(p}UZ-uxK?UGSmW=Pt8HuvE}8YiQ9 zjnn$nC0O$j)XD2My*^(PwtN@}wKi4s-*j;nx_BF{e*edw*BZe+G>k{OV$LHeN)fBR z`HGfi>_QV2^=MPeV%m||D~vo&%a%#*H0lyS|vO zA9Mr#VY+DR=QG^lh8Z;O>qV+?Y9QUTco$7r*h?FoYw7BR?sTzbju6^rkD6CpMHZon zX!B7`lyL7n*E8yh5H&S}=6=hiUM~*P6OHHS`=~PN6%s|2@4M3Sv<7au>jJc@N=?kH zq#(a;Kyjb;p`~HIoZ+6|lGUFn*Ju5hGb38ucJEHX%wPevSmjLpBFE5!b4SpT$u=aXChpn0R#oh7`pg-QRONieWe9lRR<7f>gD_&eKUJJ;MD1Mv3hnc! zQOjjrLiCvvg74bFG%KKvn|J*PO6nbqTJG4O#lE}IZ_0BaH@0&jYmSMSkcG&tJ&@aO z>xKlMRh*R5p-GXFx#-bxHKdgElbf>SE9ZW8w&44IG`A?-8u^{S$Av#UimHvukfE9q zQh6AJrfi>r6q8dbw(fR8YnQJ@dVaZrZ17aNYw9cd@PHIJ-;P16XCj=M`kbrsOyOYD1|d|{3(YSIK;`%Ig~iG*1ntrZbo|I}YLfekejgP@CHjx) z}Q6Cw_2i=qtByTemSU&$0)l! z3sr@@LaXnTBK1~7G=7;aTD@>H>T|k|9(rrwO06My_&Ifac!?Iyn5U0dZAJL?>XEqn z#CG)hZW@wpe}g8w$DpCPfhg?ee)Q2ryjGq`=t|NEe9_Snj}G_2ml}d_O2HodFYyqz zTH%GyrM^O4PtKuh4=cH@S>+OkuQF&`UA~YV+AG+Nk3`o#okm_Z`_L`ZX0+t@W3)B9 z61isIM^9E+qt7G03eLt}bW(Z|4e5SJOPo3>d94GT`j(*LKLlo2>B5Ew4&e6036w%j zVB&0f7?SaurcF_RFeVoDY3Q?W(^pODt-%GjEgE4EdR&;bn-~rJTHbGYfYw_ za!=^H&p&AQq&OPsI)$z>A3&vQ+b%U8T*g_38lo!+D^ZEcAhfXfC)fJtD3>(wnJ~nq zQ&^eiOaJ>=O>y^2`X%ul{nuSe(*~&0$zL+L&$BJj0NDa$_MbIgC~}5IEjo@Z-__!n zl3)1P3k~vn0!J#+*AtcIL&QJ+9Le~3mu%2zCr?v*$+d-Y{M>U2{Oj%k{Mjqgy#J+c zqPy`iadEjsO8b&UpWAApcUp_w+LVZQ{vL*Ugh<%`P?eqC!?CeHmNBb6kt~wMu?NkE zSbEuE_UmmTt5|!0J>9#F!Q!=ScKl+NV>p?Wf45;s-Hdg;(_*=2)mi(!0qjTfN9efO z3A*z?z{jdak=ZFgUQaPN^dEtqFX8az$bVpe)fn>AuTzWu^VFyGEKN8WLyf)DsNJI+ zYHW~7_x-*?xxtUAd`cZX(cMiY^2*R(t_jm0Tfqp=1<<)D2KJPc!sbaW5YQ;ghT3Vf zmD9}GpEP?mqt8)vB|ETlZW6{e*s|vb%*8pL0n^>D%q$mvhRH9U!p5gH@a_|Z%?C=L z?(Jc49325G`WJxbb``Lmo=1B(KCD<@nSuPy_8}u#1-$8pJhnOd3mLyFM|TG=M$1b# zqp6$sBm40lNcYhyg!QA4b6P%%cm9e(>*VnO>A|=*Nf+OqDud@FcA&|jH_!{32 z9;+Kn#fPg>u>HH2_*RNGxq5dtc@P>$PJJyU=SMe@v(x&>vq8#yYL5m#uyh2kbyAlv zG1uZN(>3_P35xuz@@{fJx}Jn57n3z1d&%`2J2FP@5{}6=lx$WkfT)V`Y^!%T+j93f z`?3L;>!62hbJGjvyXXy@UE9X4B|l}p)z?{IRWXZSkj5-O?__iHXR)%X(X0u5gR2iq zV9}7hpsQ;S@kOtwQ9ug4bT^v%U38*Gjxv-+5U+v(m#c*XiQ-x_mf$+T?||BXFEH>`LULuY3$nt3)a`H zE&6>FS&D<`x~qKzmM+zxBr?|SMl66s4W?klHPQvu`>94}01dC6T&}1;0r_kSM>9St z;Y&m3;K<%(xZr6dHr^41Pa1k*3$Vl=mMG!%3lI`=?g@RH=Fn5sSLv{a{q$~MGwr>e zN-bZ{q%y7Elz&l9AB8m0gc-fNx*ztDa@FWWMmuir`dl1pa%qB3=U?dgx?+>xsU7ABHqUOps6cL)i%wGsz7 zHqpY_ci_FVC7Zk4i;eb-V7Yrz*xsldwoG&eAJMwVHuqCz{wV^0xsh`PWTTU}5yU6rR%h`zX zGS=r&z)TmWv0BS0rh7h+1r>>|$4UAu>SZrHcdUZcJ#p}3wL1j2^I&IW4Hgq@z`tW5 z$eA60*O5t(HYEdYIOT|Jhgb+S7w2K-ir~3mJ#{`{i85Ohu>6Ik`01krygNS^*BRzw z_kp>1gph0SO)aoqR7E%%`}H;Ukl6_^&4julR@O)ng?5 zf$J80w2D5zXSE`)9MeXgy(uHAmf_@IlL?6*lZKmr?x7C9I$-|F8BFta9Gk~PSIre* zB@=FluK9axX?Hz)mDI?bhBdJQoo4nr?+JTr(7=Y7+-4hgTxWAWma~kjXIYm`E*o|y zfi<~?GS8sttaIiF_TG;Yl`K7{CibzKT#OIek?=LV7ch6*IfK#C+dbYT6$GuU@hIDXpD zi-Ju@(pA_A-mJYS_NNE3yK&>0y1O^a-L;3cg~l>7RTckX)X6g`z|YZ!<8es4wRyK~URB4@N~Lps-)a<;;` z&Yr$XPNh1(t7x3s5NI{AhKs|rVO6~*eEUxUCNzGePu|_3EAQseh3-A#d1b+*-Zx#lb|j;Qg>k* zagtd;bg%i6dzDMbzW^Wde9k;_t(qt7w;56D5jk{apRwAiYj~wj3ch4H9p9Ash{l<` z;!LOSr#9=xz@5M-IOcs7dRxDO?WDo%$81BU|9w0gV2hdR=P9h{feRa2?95h8KHt;1v z0`=1@p^ceAllNFCTB{Dx1+BDrhCu61l+d3yk5Ka?#dPlLQu^R)6Wtf_m8zbWhb|o% z_@*rdD?Rckm6Ap)wok|VWm>SN#DOeRi6M{XULxney&{E1GW@%IbzYn3@K5Cp`BiQP z{EcC{e74nKUMfe1U)I}3)G{l{&&z4#^@cUX?UoKv-JXvN4y1A0zU4r?-bl7GX(=n3 z6vuk~j`s)xh6k6k=AY-;f`VeUcwYgtP|0P-Iy2ah?iALO z8qYGXhBNtHi`jY22~2shGHdu&2MZU*z`ImE2-~!fZf)6)QZI}bIgv>?|N2e5`Snly zYl|9bd|*IK@dUE;k0UYu5VBKq5_v<$km*H3iAsPx8C2YfM_FCPzgA}B0+H43 zcV`q1%V|bu^fB_@6d<&!AEM{wwczg5AjmU24%U0_!>33YR`*_u{WxyUdVV=DXZ_i1 zhQ(sm^=$=P-oK2+H2AWXhIuT%eG2<<--_*jqRWnrm0|NvUK4jD5wN{m3O*K;3RA6B zQC4>>O7z=8yl}rmNakOns-?+a;!OCs zxmNs@T3i0&rwM%7SyNsyPMc4!Rph7N?Ic^uE6MZM2TAwpY2;*n1D>#R6bkuX06nk9 zGF>^*Kckq=j(ePA^Xn_wE03G3`O*!x)wGIv4=HDddd{%zVfk#)jAQIf-w~!)bdW8O zi)A{(9%e4Pft7F`%+uR~U9%j-UT=8{JKYQ6`s7{Ub6^6zeEgU`U#U%}H_b;DWg0jv zI27NLsl<1`_2T|kHPRpt8ISj)vF4gVi0Nnj+0U|I}d5 z3XNFSIa}7GKaI^?Kc5}%U%=jbd$1b?PHf$3GnN&j!7iNYg?Y&jq4wY@D7TM;yRPeC z(5aaaA8Zd7HjIHmw?~3P7Aqla|b*uc|-M zZcjI0LGdub^fKHW{2WIA_Z^yM{svW%ud~G}69O9QsqH`yY_wR7tg75Xj*Y)SVBdR^ z<)O+i3^3$nzF6?(jS^lcApDPX#2>7*;p?{>^J}jR=67{}Cyv}TvVVCD=`%4Rf8I~W z%bHezTcZwp`(XpyfwNef+GUnG;SN*#{Dhs4X=Sy+Z&<|qw`{mm8(SLxf(=W2#I&E* zGNmoRDyz;gt*Ke8-#wX4bU47i|J%V>XCRw@)Qi2dab|&wMzdRq;`?)aAzU^7O4Cli zL>tOW@UOIiq?ofN%DHpNkkS8<<(?sA(5)?mPYojzn!`x+kx+8HW;3bF3MQ}10*G_c zGBT#koechLPqqxuCH4`$SW}AO1f_61S62gXR8>PR1JmeB2P<$&PlU?5b>MMXo-Hjl zVb8ZtX4ca@+2bgG1_dE3=E^1(V-UGQekfC;u*}U5q2qD zfJ=}HXD3EN(Ss#Wo=G4_#|l)2kAZJ5#(?Jtb6D&v=G{uifp>sD_)OOTsWts{!@y^B z=9CgTq`{fy%?=VwjxFP&LWiQ$G855@J}b1%e*$`|KM;-Ho6FT$?~zC)qzG5GdeGCR zK>a=sfVV#!AUP!zG)ASt>QfitpJ6Suv^;~sd^_B{QxCfaoP_y06QFtDL~*Zl99O|8 zqENq$L_a!BTzj4p>n>@&c914NWQ`HO>8LqBK;4?(@y3#0J=>HYGFq1>j>>%He?7$L z+;y_-@gY)neky64R*5gkRY?pg&ce_*6SgxafXy{J%*MVr%@*cYu}h8*SjnYk_H4;3 zHeByD^Xz)Y#>F?XZMow6fN>>z%9XK3)dJ@7Go95PPGFH^Bbh>12wT_g%k(p6Fw3Ti ztX5l#X{_&n&W}+rFMc0ArKE(zlg{9!7Bb|&)nkd3x*JL7SCAEUVdScMI2m^-g6Q@~ zkmsX!k^Ls2Z=`duHaep0ntPG0I^qLcC28wDf2baVLF#p*)=51 zwoHEmT^7}_J^C~hZ;S(j`5Qpl)dMWPPk`DnW{^F}44UsrV9W36uwBa=s%4jfdyF6G z$SefgdPi|)GYX!Jc|kev2-@UdBY5zw_R31DxoiGf62=y}28U%;)YxPiOk0 zPbZbpgT>z^tJ+(I_JbufX66WR9qt9ebqB$uuoNseKLm%cPq1>zKcKy`?B>osxGZUb z_hK$5>V*dEdT}3B**4%w+b0wL??Gbm?>brW=POxZq0V@o}VC}v}9Pp}--3>JR* z5ZiDvmg!#E!)A{QW*YB3*iR>sk2L-VG(b2MseP#ES`mgPk{>v=&60S|6WL^yJBi86 zSmHPCFv;4OPP{WS$*)lvWS7zrlJG5-e4M|V+>G%fhiZ7zJofdPFAGuf zW7Rj8umsMV&DWX5lCv=L^EYFyaawG9ygX|={tnXL)UYO;Hr+t`$N6PBq2~(blTp2BkRbX!VzR@T{8BG zzhfVJD+aziQ)Nxj9!zG#0rvKGKI`37#hRravCG%mnc4P_Y-Ha@w&u%Qw)0~PD^R@8 zS~9P)M<>czP2qV~o?pxk^%k+Jg#~QzgJaBm%|Uj@H-ZJR#jND78Cy~J8t&N6gbSfA zs6+8EE_^CQ$^)&*!ESG&pR$Wsua77G{i(!r({b`MCWolZ%O;E0q?02(N#yXPy+rT# z8say6IvH}+i0ts_6Zw57aE!MB&UyY@A|0pu<$<)9RU^0Hk7DMdAMKlW0VN&AXzbDHoNHoB#o>{z73tlX z+=cJ})T_{np2X;5?Iq55anmxK)fRxi2Cl`I-!8*D;wIwVKFavVk5csY<1MZv<)5H0 zc8Q+dGKUdgL*Z(DHq`9C1;-9|L)R_^rhi|Zg_w!HuyKRgyf^a9b$%Cg)?X3dJ$A#m zp%-Xv=NDu)=O*43JCOujj3w=>uaVRtzsTB-GUKJfl`O-*Z0gQ;tVZNGx{t18>RqL*`r#>-aWbFfs^zhrOOLY` zzf;(s@+hXhV>JtWY{gc8XoqS02`oy>L?>4r$336q$&2x}M5k^ENfDjJ^&>_1)bAt2 z|H&~DH!Y7$|B_Fv!V1XgXE~&!Erk@TMUm~J{7K0Zj$n}iB9jf+fuD=bj%-Jj!a*AQ z*#%^eWC4HR2`D8hGq2onY^}o##ywxdE-6GX#m%wo>Xsy?)RM@=JPF%gzngtdTE)g& z%wjvD?U>Ch1E#-i5VLdt1&3}mKtx_49MoM8E0UF9_>OVZZrUz0-wgbEM+|nc-H3H9m*E48|HC#_L3qI6t+-yyi-!emz2co$yD?cX)@|6!Jwii3DA~O1Qvpq*gDSoe zkWv=4HkT!?Ol3$dkqu9dVU}74n8U~2Y+UFXHpR=0eVSq@GXFZ^_wjx3Gwmo1;_sqM zEE@aVsK;$-isa^RV=^?CkY&kpNW-_~#5{2`8TT%d6#B*xGyiyEAu|6u0>X*f*`?$^ zUu*Iu{15j0mmoHR_M_V4(tt)}z@Y7)VKCNVm(NSsm~u~cyC#IqG1|+jCdaef&cn>- zU>ZB*md2j%JLwd?*(7cgOL4r z{DB61AyYV%S1%=xPg624vZXou(!{`~F<{Af8n zzS3Tf@3puib{S&GO0N+_`MwoS+Bh9Fum6N=Ov3)n3uKepquGzM$C;+td1eq?&G`IB ztk}4Xp@Cg&z=Th1LRkmO~v zj9A7_7xv3{-W((M;K1o#|Zt0J_I0G@aiMmE!KeH*Y7MdDR@f%GSV7OIP8t#w>jI?KR9vJ;lB4 z@34#GCv4&H0!tg*#7;L(;F2F(v1gzme(fEEr2WnaGIEtvugwTJF-z$>J`A3nkA~x- zb8Z()1E=h4C^XK8%`#`8T~Y>8otMG5R{$feN|>it3(0AB;E?bbevNto+TqV((Y02v zQEdV_t!lWmuL!;_iiD~@O{l+9B5eO9g$L;uW2Jx*M0!#PnKt7LIX|S0a3cG+qQjI| zb#~zE|GM*ba`XAV@3VR7$4KJ|)Cr6@o z*^=?AJjpNP)uinBCUSCp2yvOdiu@VqM;dG9k&L6$NW>Qh;3lY}`mU7CU4SQ}0>AwudfbS3fRg zjr$j|;F<2sdktpaw~b-nRfn?4ufM>M%{34selICiyP)Lp96;->VAm#jSn*7t7iNs7 zky7Q{-lStFrcnjAyqk+PFGu2By$pPBz)5`N=?UC#l!s}+Aw2NJZoG-k#)Gt!@xSpo zsNS7%y_SabP*5!mkJN$v-cHbF>rufX>8GdR4b338%p;Nc<$AMX@_*Sf>dX|NVL7v86R zvggGPG{riToJfw#QS##KQ&Q=r%1_B3%S-1u@fTmZ^EcvV^D-XO_&ckt`J=|e`0_tp z9l@~pN_kY_4p!Gf-GWRcSM@BI8{6x@u=MFyJE5W;T z157d718WvWz%--%5LtW}l9G>smQV!d!z&^9W-TPNJOW{0GpMWGf$UjTa3`byvR1`` zjZ!d78G#|9!2ne927sO2W?GtIi#FJV;l;9p$b#SN$k+E5NMP4@GA3S&FSuyUyLmYC z13tU*Pp(ek9fEB5vZ>lUocu}7jVU9EFN4XVOUmS?Rwz~+7mcizhR~h+)nKRLEa(?b zgY_1%^Rt>~wRiWilw*Z#^P^iV#iyOgseNZ=?S0I)=__NuU$F+WI(FRQ5=&Z~#oTwr zvgz&XS&Z2nmZvw7jXN`(wH^HmUWba|<}4?uwpdH${4&r^^;tOlR|=MScNK4Z@fII^ zBu`pmN07-U%*nV8p3G03OI(I6C$AmW5>1m3vZx}EY~8+?)J~pCRz;YSmz@KNp5h~1 zHY)`y{FPw&&GG2(s$q1PwkqU32!r;p5{O;%1pNK~!VU3^bV77BW}nh!2b6T#k5@V@ zeWeb&dPH<_{}{>6ej3I89UsefmD{nNowjUWqy;N+Gh*sW3T%|z6Y$ebfVFxu@TJH= z@b_p(Ni!U=qhTOEZoUue>Tbp-Z_UTkURdD(c!pBX#G)sEq>+Gb}oAk{1I9bAXB~BIn+b%qwG?9P%SDF3sCnQj+4jjhHM~ zB#PIj;i)nM(WWmMv|vIVUGnk*ZBQ}>M~6h%`Q{J2zdwta=q9q$U6gGb(#B*;f3wwR z23&bFcfggyPx=}C`i?0Z++`0hl(GcB3^sV^J{Huyo~gX`WIo3^miEY)jeIJ{e)#}g zejNz1?T6_s`$V+JkYGETEbNqaA3vWaO|okANMosl%z5iW^0ju8?NJFN`*bGh-I+@Q zcjpjTo=%3&jwVK00i<#=NB(Y5B)O~-cdhov0Rx_+9{VTU#5>FB)3$-I+{_nt2V}vl z^$oCbgaXsHGGbTmU^Z#Ud^WCiIU5|imYsODnceyt%Cx_SveySg*e({p3i_6@74fr~ zrRX|XI${JnJfj^PMjr!(xyF#_8z(gSh}kKBTO3vDhnHSokJp{wjJ>M8uxZL_x{wUK=0}c34J82^n{iI>6?|sHW8D6}6MxKX!3}rwv5`g^ zN?Dc+cU8sj(>{aYiUyXsLG(uJs$S7qIrvKPUX?44d!?@oknLdjzZy0-@eJFNn#v3# z;+eU7G#ePakNw; z&gG#}@!j=b?r=PHnh|z=ITSCHQNyMm#Y|89P<-s!L~LQ(O@a4U;a7m0b z=A6E8Zik!cDzj~HDC-iK+kA$f#RJ**IU3B*ZYUd2robNE{{usUUcj=c;=H8$B#amv z4Sj!>f@-ZNRMl9*dOb6^-#;3Dv<(6ec|(|1r3khwGwIJ`NgS)+h7JE`=se@Gdc!zw zOJ$`YWE3T-Bn{8Gt|y^V5lN+z($J8S_Cm;rY!Z=(B+-!Zocnr+DE+0Of%Y&fE$#I` zFTC*9@j2(duj}{wejEFtR^4t)KT?h9xeDT`vqp=D|ClMhHQinOW|^n>4=xaoJPG1i z_WEM`?2nkFTa4RvtWe$f>dP@<|AJ z%li|zX)P&l_aJ7rn7BqKa=R<0{Kk=lD-~#+dKvpsKY>-5jGwGpJru4FjskPN)6gnY z1`kV1;gMW6WRE)nugrWQ_xn`%JwX$6^3HPDwq4{q!CJ^`YO=F7-b~tXmRU`D$@YgU zl3&Ikx*lmrhiiw?%WgfA_G;0vg?;Eoq%vtd_{e5zwKKU(57}sQo;{1qXS+)iS*N2f zyD77dJ)3CB-d@vW(=~eebD0hjHun=Jd%hBCYwa;}Mh5z+wBcaIA>xhZR$^a%t~k-r zRlM%r9C6>jQ^j3vI^v!c-_dbJC3dck#hll>aAU|$jIfEq!>jVKdS(SqUB_eDoNWAT z=!AVrhk*S~LsI;HnNl?rr3!jR(#sFWNXJJSOQrSdQsHYu{}KzSaNq&j{Mv;=^V})m z(qVE7DW^YA+v&^zWvNcCnslspAF0dGH#AW99Q_(NmkMi^vNx*7q2cUUw0Pu)BQ(R& zqdOc$ZHG|vOaLxy-;4S_AvkK?5q!}%1J8UZ#9FN~w3r~pbukra+f;~&2hygCI^-#yP0dzNdfZDy6qT{p8nP1sCwoF@zR(Xx3yIFI|(o*2&e)pq!p*!jHq`f4Q zv5U43*iWv)?xbsa1f3oiPwP@s1Qvb@&F+^#N-m)^+rg7u9-2}2+Q%#+){Auu+#M^0 zgWPJPOI)XSI(It2QRMUf0RJD{+zJN03 z@Jqt_WL#%*1=rZDF_&4(%WPI%xtW=bG-bwj-}6V>M(}=nearIb&x|S zrmU&QyblWEK^>#T(?(4f&kvj@UMA}*Zq}V9b~t4!UMcJ&GA}fv(TQ@LVt*3jPaVe( z(|G)FxB-iFUSVM0PJ9_i_;7zHhMms<7oUN&?R`E?*(WD`l&UAKd2b+9^3s;xUiyO^ zONexIW9eY&BD$Hof?B*1NbzD2Q zGaSi?ji-j|Luk~?@9eI5H4AskV3p-wtZkbK^OXMOJMAm@X@$r63m@n5>XUCsxnb=#An7(VAHW*1bQ^NG#?Z;a992z#77LG0oi<+2)j2Y-WH0 z+1*y7anCdbcDoAgdhm-)-G84Aes+Ov>wAn@9oWYLt(UU8qc(!C&4{^YDzPzzjr`M= z0N&l2BqDDEE?#2#ZTAYt#|{*SR$GcqE$zfF{>>C;8=-jWR}=B? zn114`pI)PUIgb&OF5!s)3A#5wz=wZi#JkV+7c2cy7i-$eh&`8G!jY5pu<2$6JM$`+ z?$S?k`Z7rRJxxnmru~b8B}8}peJR&c@T2tYL+;Fmm?W3986Ck>RQ91yaJyqG3hUFM$`m4H)E8`N8*f^YA1QM+C`Qxd;tbH_YjYjd>Ovfi_x zdt*KBUL(Q06`!$TxU#tKcqQ@P($APM{5oQBCJr>(gS!lzvASk9UYI=}lOL|YNu8_l zGtI)F8*1nl832=cRd(&N45jsRqHv{ylyM<~DvQGC+sM7-KXDDExO$P=CtuoV6GpY7 zbPDb{OG@W1(_pL1B(IxG&CW+C@%RclD>s~E_H{9D9G~zW%Pm>nk)MmboTfPYI5BF{CyqhK4 zSa+RY1iHM%xki!8;1urE=aXE)!XMn|<4*9Ut`?*Z#MtbghMe$Cc&4f)etFqUyq}va z)>O9;FIhBP?6bJPxaaa4>|S>hhon~E^p6bX3hrVV-+@gk}ll9No{_aN|V+oNaG(~B>OG4lsO}vMV*jgrN#@{hn#j+sclPh{rsr0!khTl z!|Bz^mwcS8EOy$);Dr7S=g z583|YQMi%zD&^8I_8*OT@PyWB-6whf3K}b%B@gS*pl#3oFw3-j7VtBR%{kD-RGigl ztHvZcTQQ5;--@V1JcNEFmop9X2Ykc!+g$#bUEsec5LT@p0huZ1L?0)pvUj2TnBMvX z_U~mV>+@zT)AfxJc$D9*zrEMz@>}giuPk%;;&U#n%-}3Lf4i0KU;LEy9wlbAFp$NX zpW}x_8E|>W9&=A!`$MO395-+1K+*8iTY2SaGVFxO9o~GfSH(vMM~G-Q#r<9xxU;Jl zb3#Xm)pk!357eG2zO6Z2eDIKrxMb>0d~q)uP1Uo|-R}y@UhBr>BU<8sfu`cqi_OFb zcN&UA{lDRm-|=WAn+$&!noz*y6BInHhSDBYlHSt@x?^ojb$^bs>c`rw!oP(%|K3O* z((Cl)`T%LF!g%TFmmvL8JW0C2a){LbYA2QYr;^hG3+g-Pli;}TV4SWnPq-FI8+8jw z`1hf-^SQLzcQJMB+RRjKU%`-;19(Fr4>XmbQ>*WsWRPemrwHoXET6q;jc-Bf;p> z1Fl_l{I$D~^yt;UW6?ZVSoA%%ZKrv$wWuXA3)u+4g0}SjxkE*0Q;bEvgV)5_fJhlb0oI zph70wx^)p#YW*&Bt`qpkWs-`j5BgJI$3#_>rKL--JA-#VP{xOQOZenFp}gmUI_r5s zdpY&7V_?^-9k8nPI8EFQZ9xMWlMQh=~lV^}` zwhYyG72t_nS#g7-j(A_avDo(eFmc_RA2>PXIvQDL;aZDWw6bStV)O#-+xm&O`;QPm zTRu$OvuKbw`(+n`>tS4b`73z7j%S(YIQnkoLXq){DNDPFy*;nU#+4@XUgK`_M$tMH z;(nA46t&X*85+`>BSWRXbO%a%IYsFv$qm}Zr_uQ4t+ckuma@KhQ-nh_jgLD`3zDvp ztlJgJ*?XFN`})wlj#4(j^%m#u&<)CkqcG>_Ae_d4PI>k&^l{r1Y8DTs@Yf&M z)$U5>G42%GyC93Hk4s~7G9%cv)%(~!GGqtU<(TELFy7i^DEDmNH7>evG6b1v{yg7^o$K`O7`3FhD&}NpYSI3fSq->tO4ATqh1Jzpu9>N|k z{@5%f`eSyOw*RT5@%_G$(J}?8>AxRzb8aV17+yuNK9`F2r_m_W1k%o9 z7_NA!;*8#XsCe%rMlCKw0d{MMA9B!9CtuqEblF;qOx9Y`k|~aKD$Ab^G)0qM zewl_0goJbfiLGOi*1h@it~%q#8>w}z|(#O=${gTANg!N zzP<~G%+(O@7EKXP@SZH5zIdc~xu>Fd!rnYQdCCL#Yu*N>-!WiWIvz#^MMI`^2@G&9 zg0eIJxX>x@`P!puR3sBhO4dcR#pVX7YBdq(e~)%`-lLC>_h~{!1D(siO8w3VnWdfO zbhnXc9cv^0-~}~pzfSpQi%8Gl5Z$^rnXY`4qgTJn*{O+_S?!fGtWKQCM&y{V zokPb~K%YQxyeW^NQvg@{Y{K(T_u=zdJ25id7xf$0VzjV>jQk#kGWMyc+>nW7htlzW zM-&=qt;X}O#^BYqM))2R1no|VoY+vEpHcIapM85K%lqld;?5*7`gfk0ta;6@3OTle zs#9rS!Wt@645rhU<4FH*5{+ASl%`Bipx6gT$lfTL&fX8DRL?{7J#HU8ez}DPTwFuy z{g;xpXfZkMm`!^^?Wtt!IEtNQK$Ff3nWFSp>{!(`c5UuXA%o$@0#Z)%0X+`g9Lj^k z{9DlZ_a1CD?S(0UZD2m59k~AaAa6GqUl|6WZXXHGd8#57kLtn5-5+tzjvPecZj&}s@<3Eo+kooEHfYnL zfrHof2lc*fBKdJbKBizYh3EUwvI!AnleClc_odKXxf~jlUqk->P4xKVLyEdmPlZ2f z=yWMhBc5HSD#g1L^Xw_@eB45Ht+#2y-%FG$car+w2&0mLTPW3bG1*v~(}iFY`t?*B8Bd6fkSQEb4AFL!+g$F-bNE*9XU7?$mh9 zJ97j_{|UkEUfXb2&N8fdKMNyiAP#I$#3M$>VcLH&Fz@(qxI3s)w0V^sziwd)TTxld zECg4Lc*i)hjG0bD7qD>vZqGVaFxSkCH%f=Xb8QF z4yH?~K9uk3NlVwQro0pLgt_^A+VaJmW|XVblJI)LU!BQXK5MeC8WP^Ji{osSUULCj zySec;{ouolE7pEZ_qpJ-XWX+OJFw_`40N)D=WZ&2w4640M5SHcbu^#z z8Ft10fxR7rP#k59cySVbyEz=Y+SE~V(-&CTeFVjF*{FR%OZ;N$7nJY%fIBMkuyy?u zJZPMRqGhot)_sAqk1C26%KpaC#Qx&7pPTVQULk((n}o`$rZ~>_Aoeua;&$4Afq|yj zqo9F?KC6MARdSK9+u6^7CS>r;nX+A1QDM3{IVJ?tmYO)K53Z&%i#w#Cb%zf2xla$5 z+@l*YPiWwgHu~D#LTYo{DgSvB-S2rxh713rp*EF-JM-vNayV6Q+C~=2VjBKsGO3&& zN>f&<(}#EG*vIc$Ea$x@oc+k4QQ)AZ{2q_T1s7$+yd^k6X+6$rUx>cHx1j6sop?Yv zzeFxi#=lFVa9Tzf8hd%;}UKyQ!=R-!+H-mN;HuzVB+7ROczH{%;(WO?Rdq%`kdonk4ial1Mf#h4gNPQs5fF z4S9VzeX*KH*-~e^JZJ$08ZV+L+HN$Ud^Umm1PTZj*pqiVn8%20Hgcvf>)N2lYzOPG zr?x6=c~CeX?RtivKL4M!R4q~B6^~$EvgC zxH<0(PH6-@pASV~ohf;=OoJ`e5$-RQ-qf)+jk-ry(2q-J zX!(d38vizjPAV~)6Ly=n+g~Hq;XE~Ouc7^CnrTaI7sbB4MS=o~)SFYOQ8S5}`i0Zb zg{#Ose?HAzVM$gAljy{pzO-g`rqCCf!tENo5qh$&L&xhL(05V9C58I9$GZ>Kr)%Mo zMh(H4YL6DSoA8Y=>l|sa1Ly0{$K}`Ekz!|IzninM-M>z%QuEn-KS}E8|Txg%}Z$Gg2iNh zZ#gBqd6RpPFQsJ)dF+*Ysi85Dwu^)5lgbXNNn1e=7mLW?o-ysaF@VO@DASqpzNA0> z4f9L5&-lHe%xXa$A2D!ng}3u1?#Gx5oP*q1PW97Z!0lR)*DwdxpNfFJ0cRmxQ5OH+ zRKtDxf1rEfV^9^kwVfJLc+~zGq{aVW{zD~9l~=$uO79_WrUH%+`U+h&eepxpUUb@Z z618tV!DZ|M?wGP0eVz=(ztsj9-RCssf4q)`*Lv_-V?AE%SAoCn%8|*P#COWaQ2NOo z&6VfjV7s;05aEo=co8PA-wXY6pK_jKqOJG%PUT$7l=;NWI2QNtIh(IyOzoQXl7_f)ao>~wkJ{5;W#?fkw;EnuhRAl59kG}qiZ2W^uj-rLPSUDfo>{E3?gZ~T?Bpe z45u=?NLv12H+^xNKqnS-!CIJyq~> zyf(VpY2cpbp=kVb94<5xb`52l@bwH|w9#FLp)Y6Q$RsPov~l?G>0n&fsfbUPsx%}lrXQIHIClr0e#)$NkO`s z$y0SB{i|I@bC%De1EKCz9pFXDM?5KK?R+x2JDp@oCQ@k4XnLEkMt*;L*&&lF?C8{N zb~h}9Mazw0=Hrg@Pk(Dj=4jN2T&gBqLJ?BxMwr6~|vSO#~yn_$Pn=dj>& z4|v7gg)FaoaPxgVI2tyBZQ2KjP;$DY&>S5T#zJm=G(>4I5&y@qHA!Ej^41Vf#`4!8-i2 z&K3`>6XQ{D8}vUj7WcIXodEk<*!3SlO<*2G9?XY=N^|gh`CGE+(t5Tz{VKb&^%pa9 z8c7cEGsz)(g^*QSOKk8~+E@}y&24dXp*Wqmd&ekInm~U3iKMb1g$xo;(pl?F@@Yw? z=!6JLpSX+q`>Y|(#gRlNA}V}oLfx-*NzeW-tABNYZDP7?;z48HXIHl9W%^34WxpoG zZ;gQP&Gq1Y`8~|I)&r$)zCpppZ{Sy{fXlN!!iMw?fpz&5xXgdxRNfD#nf`^!geusb zJq4mB2XSe6MfzpncR<&F!QtK>Xy5%B4oQ{Jr_lr>zT4ne>!oR8Blx#4K*`B=Pe8D4t6AJ=R;ghdY{uqVtHH!N|(7mwV~@7!9Pe0LKrPISS) zZ^okj1|1COGQj<(4KQ6Iu$=GifM9KZZgANg(bF!_SwI6Y>VhVb2a$HCXf9i ze9!s_IYk3%jGjzWQfE+ngfq1a@}$0P{xpXVAWiEan!9HojW0bwLsPa;*ZLKdaNU7Q zVyviIV+D#<6z)qH?W+!0M2AC0BH9Ey}F%Hx_S|mxaz>T zgD<&9(bb$z_IS>`T9;STY~x=qQ)b_ez2h_e<=L0*Hf&dXFoP}0?9lNjHg0b$J6W5; z>|dW?IbjLR(k+O^wI5`G^~r3i_eItrImS$va%}IPEZ%M5LS9`xiqBefoL^__%&%UT zA!)m6D$;dL<|>;e!J2ciFzRm(Fu4H8{HX-PE?nj!S`1)`)d?86|0SG0_!G7^$zb^5 z_s|r1N5~FdfP>OBxbh(pzMo5js8?AqS}qk_E+xS8`Y2fSAp}^=5omp$15*p7pmL`M z&IG5x-K>T1XZ3ivyJ8c>^*Ig;L-OFmwS4&7aRdI-RK$pFLve(!7TQ1i0ekiCfSvwr z@U?1&b35-qo8C?EnD!D* zg3W2|mk|{3Q-=%&8d2%E(X=Vvm>&M8MgGdaS#H^TcB;37DXhE0Ud}CKImZsMP8AR|N3E08N$1N;5@!s+i#+!w`Z@J8Mb=4N<6 z`XD`6$vK0MXAJn-l);y%G|)}n1z*}-;EZH81Wyiy?)ptoHEAu>%h-c+^IRwxrUy=A z`$5}K_d4`B}GG^Nv56Nz?xYF zGq!SNy;?tc{V^(R<34-7-DiOitube-B*rY}m(U3e&XwHp^p*^o5F+vJDdZQnA;0L6 z9QUB=6?bw=4kv+QytY^*T4*v7=G*IoCp*Ix-l`F`osQ;hq@BFIVWmVyy>trpvw+1$ z4j@qjKH`BE%YPlmD@gUA^yf3^mD<6eC|CBvAVBaI9ASSQ@;RN4$uL}R3!J{P1X}30 zB*;dNfA9H$vwDOe7Z(S&b;7wj1(AIJ00&69Zw|>uHC*4#&hXeGjqfwEMAVBl1lfL{LdyUaH|jEZfZ1gncw>IG9!Xn;@Zdj#laEW zbw^9sE!)AlhuW~-j%&PvA;1zv8#ox$$<>Q*a2Ef`!?mzTPA2$7d4TvLXC-ig4e#BC zfLk>%a!7{6YdYhrtCvB?=Rs(E}cZ{b~psvO&VtjY{YBZN> z`S~4Vzm_Yq%`+`vaB2p8dQ-vu(mTd%E4bq$WCq=VA?XYBkZTN)|pWdTpl zf@1DrRBMWcU75iYwkni1Zra4l2=~mSxIU=kaf-ziPoP0-G%4+X8)(cJgopN@fVwj~ z`Lx7I6!_;3gSKdB6F&pfuu2&DV>K%+e8=W!Zso-{Ho%X=BQe0BKlUodvYpAnR5x&p z5F}oMcDo93YWF^FalrwSo(iIwxhKIsq8d+qlgDEJJ`~8r^i|+%Bred$+E=^rLy&^V zs>P7WAYJXV_W zls$fS4dx9r#RB;^F#Fee?(;TdA!n1rE%x7nm;amqixCec>n>&UyRWQZhRf=qacB(; zey7Q4?2F(oD+`%e<@wzC3j;7{OEgq@N0;qv)1cPiKWy)-aj13;@T2Wfc42fqtJA&4 z8g5=;3&IppY<&gF?QTmV*(-i>wmMz!cbe5rn~g@9ikRMz!VJe2un8Kw$>P%%&f~ck zZuzE%p)8Q!;2ueyPCl&qNDSOMs)l<;_rqI;zSQv2j_O7!;pR!EIL}E1Z>>-u`7Jz6 zQkux?U0{;h*jBHNY1OzUf5v_;)fp|Gk-CGSvY) z9Ph9lK8|$fhXE@}H$uzs637eJqV+djDWdQybY6AFE6*3OY9Vv{^{bR!x9w$9-94bt zeIs{1ULDn5KjP{&fMjI+%IE(`K(C?O;K{-#EbD>LeJHc!=4*Q5=$UO`ed4Te-j8L! zw+@4YfwHKb^%LZTTweHvK9raj3Mxvo(5CiN#Ymf6R`_}nB>4xhX_;rC=kh_AcVQqa zwF+W#YsD zbZuWIy?wKaIq9B;(CcY<@%KHLKP`#|n(0u-<}^OOZ#1s8zXQ3q=F<(mer$zmDs266 zlsG z1OIjDVwmGPfQ5}+&VtrvSLDxq#;Rt#WaiPXSbHZAPU>3Ic8e42cybg!<<=Q^89o|E zPZ`ESrKd=4uQ7e7S%=g0gK=|22K%rvl$>||Wvch~!^+SvFm1tau+mjw-Knu;IV^?u z7@>n2n=Em3m^Bm`In(#=FZmy3HDK%i12*jg2$BS_v)q1GtoW8UyB`ORe&1l_;V=B| zAOE<+8-}9)u0hOFJdR!L(q$z+nk>$Bkm&92A7Hb1BitEtiy4hu&F8Jlg>mzf!87qF z*J#JHrBW%kXpsee7Q7puoT`Ob(>UB^oCu;=9>x_6rr~WjS=z=5wA@?Djk=&sE{a>2 zpWxP+v11}0yDrOp%vB@xH(#M%qaTiZuSSuQFsAXhoAYL>u=&_1EXhjX-Z`a{On$s1 zENUzop0z+4+sx(`M3C7+duBgLD9mnt019f(tlVc3`7YI?hcpTA#fovFqOjdj3M5Mv zXVKQWskmt9RX+Y&3smrF}()KjAYvn$=?em^R9G;CSrejefIgwe`3io@@dy?z3 z>iPL+51SnTY}AM%VAp>@@Q-vDKTV#6Z8xBI%aPcq2*3``wI_ z)O4L;a??cg&v80x-w~Ylc5%$&M-D4-n@6Fq?67F*GTiOU@bqa@ae<8?D@tj!Rs3gr^db&&O}pT zl!87M)u@v8z+SHU-F^6(J(~u4jR3d#7a-xegCy1NkZ67AGTu{3fx2crWO_LZaNeBP zFep^$zlp5qZNv~}+EIos|4o7O7f#ZPlcPx5=O1@z>?QPcKf#at8%T01J!r;?c`!qA z2n+PWMA?B-R{wS{osW3R$$meAE2E1mg8iD=97jcp5M5z6q^(e5coAp3G^aY%%T(vq z4$(HV;DNp{N4)bDHi|D0bk?!^ou8m6BaB-wuZ>+12iUly-eeG?f}Qfkut8n8^A9Z) zdXNuTgz)^LHtj;n@?+^I50S~9^RRT*9u&C`qG9r98D7s|3$K{7YSjvSyjqs#it9-H zWe8qZy8|Vc53w6j9QuA7P0ME$(vB;qFjUqO|Mm|jL&IZ|i%ayOwy&NjA~Fm%pHt>L zS9SB<3mV|6xsWAxo4`Np(8R}XK1}a}!oe`x>lH>H)rGW9&eiLZ%0>t_h;kc?J(eIFwTo#M)qU&(wC@QXz@RXEg$=_LScUU z{kbX2N>btX3MJ%STV?3m;4xI784SKoGWb=;1s4}a(c_GR^gA>gYat9bg<4Qz|HTv- zI9TA+J%TC$t)WsK!J3bmk%El{zw|}`-1d~mlBQJF&+!KfGx^C~Sk(?iZ!7r|L%%ZZ zwHH9!q&%sF)3 zW)QnI^eJmxRtwGnn{h|~-Q?YBNC!AJYew+ABj>(ib1RWzb_8*3 zO+OjypFSmcGUEtyyLpkm-|f%&f3kqPRvR(yye7K8eoCpQ1QzD%CLAN=8x+!`_2$t`5H4EzckBlSs@w1NxSHN54)J z>b2;gel^_LG$`Fepp4%#9~H&&*tnNBPx*ay9er@}b5NOGHSRkB|#0bgB=U}2GF zB(pIBCOuooI|y9YQRyquq=E^#>Lej=HU-lTsKOB8eSEHOF)XgV1O@r>^y+FldmQM( zrtA5j;glixbx%3v7TsWT;ytl$%VGF0JOo}!l-XU|A_^9A$L`tDSQr#e#~j}C*>ALY zvqD#>s%?WS@8rn7)&utbB`D1}Oa_zU;kEUCJm0v6N%nnbwWq!@({Gi~J#r$(me#QH zwBa;;P&52H9R!NenM{C_@@3}&LFds^W^wVUXzSrvDA$p!N^ zcdF3a7yv$Fl4#Dwsrbb#1Z}xUI({&PWhA!2p6l&=R>E_T{&@~J1qPR&>v_qvBfB8( zF|x|mY^+`~pDJ#qQTCm?s8?Jgy4>}L$+e7vdow*?@Y5Pr_-Hpb^VMcN?{SQO`9@47 zN1yYH!dAhkCwp<5!Y^*n^dr<-c8tAIkHjFW{EE@T+u0JA{^*!?fbMPz#jZas(D!;a z8Pv8w=b&`{;IbS@sQL#-s~@npBg3exJOpKvr=$7%=QR148t!WU0zv0;E7Z*l>2B+H zF6_MmCJULmOh+O6pfQTfBaPsL$_)G(6V6Lj?e8r=-GhF`jPY@??8KM^@ z&`s-HJk%KpVJoc2>i7qC>-~Q8Yq`uBzb&G;C#mSu1)T3lO+J2WA?xlF#vG^j!-I*D zAeS4@EU$2&G~bkNi0?s>jXhezc^a(K3;UI)Ub7*XtCtwroJ%3yc$NdY-AxrBWY@8HF(bT!>%PQ%<<7l=I55f zHFo@B7LV8Ab*m;W_`Vh?k1S`uAADw(ik%n}Gm(xPpQ1rAi?Fg$9-pr+t5~zej||tI zJEpW(=6RjJ&bzx>~c zO5%>m)2Mc?9VvXBi4Kx!tkW3yylDe)rs@ic-ZO%;Z;iv>DfignA46%$xD9Z%?-MnrbU7>=?_gz9v#TAnXpxkD-nfHWfzR^FL-`pg$&MY5&uuV z9Y%x&aWSj<)A-$yqN%%kM26o3VAh`9U^VO}%U7{rrE!l~r zuh~%DWPukSwz0p(I@IuaA=;f;$G#4K%Vct%!uq;HT;MwgIuc-t^ji&&+}KKah8f&? z8R0w>D`7jmb!fx@!~lm{uHtf-V7CzVW1srsxDqqUzwwQ=W_^IjG3ziZY7D))eqZQv z8luiV3vQZbe>%~whspj?Y>1E(RNWAUft@qpKg$C8UR?$C#wRdu^aA>}D4w3bxeLet z^%r)mqgbfZO>W`hQ{)~S1&!m{FgW)&jaJIyjtcYn{{Hq4uc7MP|w;57Y9*+4reyt&|9LM0TwOz8*Rmm$oI4&nevStF`HKnVW$DrD4T#~TRP#*u54D?(j#&8&){ub z68R0UOJLfYb?j}tG9;eXC-22GxuX03;iu06D)Z(jdelD`H@_Xl`|UKBoPM;68cL!# z6Vobmeiuo49~QGIUL`PVlq&O(e*y)=&v2#d8rY~|v)M}nH?*ml#I71np?c8_Tq+s| zRlDQq{*v=p@Ms(>=odg$R;O|DEp>99+|E*-EW<4qgno6kAKPNt0cLL=@;-g!$=s|I zjFuh-!xKfKJCD_ASF;A$iX_-4Dw6zGE?|awAvo546FFC2Wclwj@rPzCOofAN>B@Z8 zZ+8wGD)g6~m0MuxlR0EmcaS|@nv8FnSCESL6=pmp6qSCSWW85AxT+@xl2RetQ?O_$ zf{-l>d3u`aRg;*gPX?wHM6w2-5!7lr0$*!+^3D-6$!xz8hBX+VR=^?3@R!A^1%pNF zmV}eHz>aL}$lwoqSJH=Z!fN-1z?Cjo&GZ`ofnVtmDo_(sRMJ{>AJG7RZ#vVc*IT)J zcAc!uW*&|h`-LlY+9L3*OMwOD2|P{@)>I$Gj_th(>4ICR^rZz0J0Uo?VqS2Mr< z>>BtUc!sPBYT?x84D2)cF}vf+vvJOC>7)0yo`R{4qC<=xrao1SygKTcM zpA{C=is)Y8UP9Z!aiiXvkOg9E_~lJF5q4eM%>P$hI4|3Buc&vRsj3I$Lv31)nYprNNu0 zp`=>iNGFC+CPia@;0wVy6+_W6AFz9VG^MG>vn`(2Ftz9>WpnF**i{Y9)0$ zltaoLBVp#YiO#=UD)P7+g8e`7^dw{ih6xzTo%gPg_9a4hW=G$*nKC`)ZKx-Agyw1p z=k^RG{MnmLOJ;-!%!5n#?Q=Z&dh<+nQ$C)2JCB~mw6aZ;k}>rdM^%02gM0oqcFJiG zYY=8W%UX^yn~C2geZ~)Eo}*`Q*EK7c?ifQ%64fyuu94SF`p#<~6!J?QpMlr)VK?kw zv&lzVL0*_Ubw5^Q6*+)j4uX60ZvbD?+z2*LPqIg|ZnORKZ{RMsK(c#lNHp;e$Ps(7d&qI&yPdkjGb(f>uZp{@`l5eJ*=XQk@x^abP>B1` z{A>$a+A|w}+iZjDYbG-Jeg^c>cQKB8vrWhqjlvI~-@%6ojdbsb&=u%i1rLhER3r2S z8n?@lpF=t8wD=CKRx3fTS%p%g?((5xYy3B90?jv4h3yTCU}oe^D2eUEN{VKaVQmdO zuZY9c{C1W+Aeb)7MPowC4$f-zB&N) z{%K%0OpFWNMzJXw)huYe5xYH8nJ;O7&V>#(1FN3>EMUnJ)-&iTr>0%aK7JaCW%Aif zttO3I_pX&`sn)Wixt55m%^h9A>xLFood;I!jFELS%qzxXX+F>E=kYD{CP zKjXmW=sS_u+A{*rVgdIqa}rj(RH4byD)jEmA@sQ#%0@RVpx0eKn7u4p=t725KZkt~ zo8yF|ga(Wx{tt`kT8ZE0*qSOWR?o9JRst# zWR%EwmnNoX9%nV%!`Xp-Jcu=B(v4^nSkhL77h=0e5|t#(+?#R1^Nln(UyscH#o=hV zC+zElFglQ%i#JvnW3;NQv}nye5LtF&)8c%(e=|vPJir3FPC2uJ)FoJX-;Gx99l-A1 znt@YNK7UEQC1_#&ZRi7AQ*HO z%h3d7OFAUvdgYcUu|r*#*||_Pyk6Ij*>v4y3YQEyl{bgb_}>p!yD*k~6~gh#gc&F( zuW0hCBk)P$h1!o4NMWim?YWHi^c!-vemH{s~3L+JfnMP|BZ zi7*?u%hpCMr`6eeuyC~noOo+Zr+h_Ft*pi1wKDF#*q4PTEnx3P<}meZXQBUFZ>;!` zLY7-M@u{yeaje-%dYrfoWCMb*Q7MSDN40PRN_iOj^$mY$j5cGL2~gL_a~-)_WMLHu z!;2c=q~K_?qx3RmGft2NkEo=3T+Ct*dV z0dBOAp>#`qJde}ZhVBmLIczINDjZ{XW=|8kD624FhBaiguqQ*XwtEO&RJh8t(v(pl@)mz4#fF8b`0zWVg6ICnNbt;cVLI8j zxRNQB5URn^GlMF~_^yJ@p9DU0TR(QAyaz_8xUl^5UfjEw3V8c$CIxG(hJUIc3EZ=lG9z*$ZSx`im^X~u^Mcvf$BEMNV(?k`a%!k7V6uCbS(e zt`3=t6P8KH_~At`e_?`td;H1&tMGk0q66xg)olLNUe@=;3F!AR5Q7Bw&DsG9)W6UZ z7gc1llC{g2jWXb~44`7YN$k4nK-}Q0v+5AH?l*n$XE*gx|>ID9~k7LWU%qBC*C z>S@EUwJ1qM$WF+XB+GkdhLjQ#C5e)sB&9{PN7)J4D@!7UR!J!DndelLP*S3Wl0u70 zD($83eE)&h*=C-(@9UaH{Y$%0u;yQZ5%-V@w&C**rW(}oyETqI-o$q$&eN}t^zjh> zN!7vcfrv+?zpk@3Vpax9K2iOacjwJ zy16nLBAWK&G51E$RMT3pf0K#(ue+jaBBh{lZWF#cLecuyb6_9$4OyPvz;tFKsQ=}S z!#Y2p!`Zd4U_}Y~ukR3Esk93g&JLhwR3vbRk1-W%l|zk+vww!zl_(KYI;F!1@f4=3f2D1pdISsR9R92hkZ_Gzg1S#Wc5_^&>)bx zI7tb8SYb!?))rBrp$)zw7fcn8MpKEjQe>?l3daVg;vMg!nH@W3!W?52{2^GJ3dgC_ zUS&_gixdasQ+poSkQHEUosMeHTtfQ_%Bl6J2ySTB0-GBt^zA?(E}zs$qn32mwMs>x zyHavcK}Q(xs5Z2qvw=CGI)jd%jp5zJNx1r4AG%(_cZO3>pe0L77&}2JRWCTgp8j?R z-J13oDW?R$7EYc$l;w`yUL$y0AmG=Ep@QO`LeQMJk)P8}#Lgf5V0fwu4cB+V&NEx+ z^pxe$00nq(+HWSZxCusm!?8~9Fgk5_03`NHV!!3vkhbDV7@lz#-KJ-diNPd@QOFhy z6&Ip`pV_p3`XT0At|*%L>ZI32tJ&e=dFVxMD02U! zD0mj62T$hnzICazE1qAKki|eDQk+r(*;C`N)KXm}))Ei9WQ*{~NEhqKI~pYB9>mFN z-;mPs>(Ji&7~kl)O=YE?K~Cf!yip;L{j#?PS|gX?C5LkGogJ#&z1hi3W1kE0JgLjI zw?;wa-6MeSfv0a_j{P+wf$(Y zvWnU0n?&E1Z=+%+OY!t{QK(sTp6=Dz%KvR*jKjmj(0AcCnk-DmpD$)Yh=Lw^*~Fn; z{2aaExf;6mjlcpqQY)`6T{r%G}&|ZH_?c+GjN}> zS<6|T_}ObM$P_fuUz<&EcE1F@8ZnRWx1}TRUj&EfGhnJ$!E>soV8hDsZ1kncaQF5g z`f%wB`k3zyKXP>Gsn>RRmvtffdCU!#7*w!T!Qpsc);B8i>>&-ik%Vv93+bgJ82m$h zv3;I3+GZ~YW?np_HuNX+DQJj2ADTvFC*DC9`4mcwMaOkWH6PaY9C44loXP zPw^eXIVdW0zwjT~gWg0J)9UR()K6vz8E%}zY#!E!F1<-u`LZ9gqhkUnseDHr`<$5H z2TEDxt^9r|Jdmoso{l~y6{Floddw$DKKp>kh=d-+-C^)kGx zDV;8>vIc|yq;VwQ9oZJU2$GCVu=ThK)DyKG{6b!%!o$B&NCSpy$rr52_gHKf76bNH zKiIhK$v9#-3hr3V5mtCxW2q4fsL%kmQ(_zbcq1N$bT7~kMveH++4FE_G?8{6y^GzB zl|y@HH~lGf3%g8O0jEk`;EMbW?CGQom2Mc$C>+9b{zcIkw?(jg(lUHn;~^98AqmSK zjG*Kv2WY_C4%RSO0T(QepnVt1=(+9rNaoZFVQ|Fr7)ijvTS0q z-+(~R=7&6seW(WEi60!`(E5ox<_Q8pk7Ss5j;$~>!Qc%327 z%WkB1E{a2=gASId@u&LtpV05YlGuF01^QTO0?gD+!sO;tsuVa6*zH?z)#{5(UeqxV znH7t(U^UkFcn4y=cTmwS4sUQOhu)EcbhzdYt}#r3ohH)2bzQ;hKk0#TnmoKxJC9ek zu7}OP^dT{~A8+<|fwp=606dp9phJOec|4ZSN!&ptqtn?yqYx?|n1affBzjeG8NbI6 z3T!X5(DByM=y~rX)OqRxr5C%I6;E=JTC+GDOix3>AM=pexG`Yi-+|8Dn1<$QOF}|# z6^h*7j21gtz2 zVhe3^S0bxPzfeIS1B0>2jPskh_*tAH>>VCkdm(4%b}>IE*Tb8fkywLH5YN2egkO#jC^j~^P z%WOU|*-Rap;eDQFeP7QoBD2x04?pN9O?7NCJ&OHsMH;GCMWQR+W6=ot4qs4UkKEtEC|)6149pgI!29v2P;^8~O)a2;DtA46+5 z7lLrvA{@m$M%=b3(0cnclNzAPuMH5Sw=e=`8x)0^=_mN+zftC z>4Nmc(&z`DO6K6qM)WpT2R08*MPfWN_Hly({LWXUlJ794soiwLm=hFLZNiJa=7H?H zUe@QLHTIg!=Qrdeu%1K=vtf$^IQZFPo9TYc`B*#1-n0sTYt3ih;y|$RQpas;hnP~| z8hY}{017TkrVFtQL}-q|$L&IpVf{&=OEjN(ytbC&6H560ndgj6b1z+f_!)D%!=9Pd z-$XO`Mctf;CrE8$9(|nqlI|+1Wh!k-P>(o6KRoAIu=2C$ty6EP2j8L48k9p0yD;7M zbBO+NeuAD9#GtqphLHMbF=KC|jmNC`M@`@M(Wbw?cvbdJ`Xg%+tg;zKQS(+&lsw90 z_FY9)>6;*+sRhlfGQubP4I$~)4ECW0zdzU~2FD9B1Y4%4V*_tlxIN&>EI3k)ey&TT zJ4QT_ONSj}u)=~i-f~4z)-KF7uVd6lmbmLBsoBvekQ7p`Yd@ zrX*xB6+LnU_4fZ}7E1A)%%vGfXLC0?wfHNwP}{@Q-sd|e)(2_)$`N$nv;md6uL#j= z4e?jm1A@KQw(#?%Ilk2OA9L`YF{J6dLC=aq8LPFwS%V4N7?Xk+HqhffE2mR{wwC)a zHc_H563p{+Hx;47rE_VkMm$xk--0xc9zqw@$AE!Dh49Ef6+F4-1^sh4hz4i3qF#rr zy0EZqbc{_3n^Hd&@IkwTbC}8DkJvlk0+@kfmWdGOJKW2Q(X{_|Go89()Xm!r ztvaxbzSsFJNG%>kTCWSK;U+U$5+ILf3-7S_+Xrg4O%*q8jYcc`KU1>hI2t?aF~i-- zpcWdwXqI^-irQgO=hXR37^l0BX89+f(%16H{oJt7H1RsqI&&F4r7KArlz9Hs!W8D| zj7!vh=yBb+06gjIRdLP_O@NS6MBmDh1pV_ugmiN?HvG3(y zGsA=D*y@f*HZ)$Xt_P;l(AIHive|uhSTvdHPCmdy_zW?=^?CGHs~U|n(L~(LNpwhl zoPax#%-)=FfO?l*6FTjaN6gO|f{BSvH2s>8jrA3$D=(a62Ko17BmWE;@$-w0_ZdvE z@krh7&RLB2Kne;I*-b+Wuha1@KT$*;|GslMN$;-HV9t7KqjNdpG-v!;+8?XQb~lJpm03ne@1qtY_U4Cx zaT;I*UP<)YfEgP;C`3~4D{Wm{C(^HP$I&~HmxXGoDQIg*0MFeyMpb_H2j-}TP=249n1;*Ju1?g{m&TLQ2q~|PDS&OHGj9c{>G!UwY zHe7zro{&98BTp(IyXrLbKzSKEIJ=(x+fLbE#DRUe+={7fb4Jx0n}pcRlNOiV7rZR{ z$_(DM7o5;W>~Pm%VM*X0cK0)$^FJtsoGiuY)CHdf%F~w!2XrUX*#~TqWl1dBYD4Jz z7YX#UTM{E1|B4a2|A9GoP>YQpt4%L_(_+P}otdthndr&;bk^fe9sBy&6;>=Dd#jNdNi$V6<`?fk8 ze}-+b%M`BZc*g9v7iA;6w5cAQDeUn+!el?Y$(#(4VJDs7v)F0r%(@$e!p3$TW|Er) zGB=NBZ?1BnC&MfRViL(nq(0FWhWD`5VoKEi#*HiARvI7~E`+(_J&u)iXIZ~(Z-l4! z*s*^`oREx_me6AEIx6J)gi&*+Aq5A0MsyK>E@QD!P*#;72z+GA>isTas^SWnG~X9& z;=mQQriWwhx!W-M@AQP6VTLg1X)}AvszH$36O1O*ZeX0|Q4&B|n{ zuoK!`P?UZqtKD;!bu_!qc+XfPoP5BAky$Cj91e6~=GnbxWxC>-+|p*oOXfNI)WDm` z35{hWG@O}AyLy|~Ys&=kM%K&^*Cn=%c^id)jw-Qr-HREMx{r*v_+xfrm^Hihkv>|i zn#anX2w|h0@))tSJV9u&Qr%|Q&B|`~6e#|;n)#8{B-}PDrEcxmP}XEdGt*@>Dr_+< zVm-?8m?mgEdp)TEE?St%y0e&15}Y}zr_%OgcNm6c;k zQ{&hYv87txL#T=M@pQ+FM zB^3M2XP@@~6P$8W(CBlM&Cyh64o$km>=YET)m8Cz?~ar*(VwH(@E#||_)IP%tQ4dD z@=?r)ZoVMlR5Pp6m>|@X$Yi2NMd?kqYph#kJ2N6ai3+oAh57zZuUoy4;>jg!{%1qG!>O+3XBH_f&|B-q+yW@pV{XeKmfa z(1Lx+9^qDr`{~zUh3HCL&m70i8oRN>3>7Ta`UyFBzh+z>O{1==Rn%ww zcvzpl60Su=z}TXl5a)9mdcy%kPt?Fzg^QqIN8!k{Yw&XI8NQ!=5Q<}S!J~Tv+?3M; zFGU?V>e)}{9~Psxe=lb@&)!7M7wxufelft9h$-5zcswe)6T_%3*GCyg zo(W&+>9Q58zp?X53+U4K=c%#Cb{bGnMfdz3q{EVYPj3mIfg5|1+TWmb_*OT)Jy8n$ z94EjXKYb{8rU^|=Iv}gUXPw@Hh%Uz$h?;+2W9>>##WVupiPcTz18-m9(i3M^x~;S_rr zYIaqF+R{2$c)k`MqgHS;6yf?sj5$r`SzO@-ea?KT9QRRe1YAW#xGd{VXjhMh1GU}s zyrwt$7&ZaVn<9os%EsfE#LGw`Zz_Ip65*xc^RRVC9gg2AO2#B8l3ag9GR;krB(>Gy zjo-?!xl;z;@Ak(od~c&~=q@_+^*qwccSW0(7$nhWfcBj!V;!<(p}2AeIJWKri%)0Z zZ)yij59k5k$5+9Kdd z16n(!;Og5Muz!UQEV5e#as5G1WMvNpr&hrF{tVcAF&#vBuT9CHE6gjG+xXT?J#zIG zB2ibAiFK_w$@3aR1c#JJ`$I)CyGMo`H`XUt#=4N6dCN#epbHr%X-c$Brx5eUQe?A% zD0whMalpR_K11UU>yjS8y@Us_H|IJG{XGFAt>vKRb{5Wc9fg)l1K@vCjm!Coxx^QX zIO_p3PR`^D4BzO4|CS5k=gxTOdH0Y0(cdi0+0}p&Z^+?gGdT3m{1s|{JRYBOR>h(X z1i#>!NxcRyv1On#d6{HMGRCQrebOKBip_C2+SmeftES=p?(0bDRt(bE`Gm3g@xXRe z`3Y+kBg;sIbkM(c>%h<;6{h*u!im5~FqrWJRxXy|#s-b!=$~G=V4n&Le=EZh6=%@7 zU;s`{?e!(Mf)(pGt7f3}2CcrwbKI$g(xSf6EQ4=2;R z?}q6+MNbfOE`a&n`H(&~3I=V$L2-OMD2qnIJCX#~$ECobb7^4nu@ELnp9Yvv3BP`1 z(x|3foUql36o2`T3~uKA1A5cRm=OaK-%N<~hWX^+PHS@gku`a8eKi?)v5{;w3?fol z?j)3V*mO*qO!i7j5hQg5k4amP{$AjFjQhp7oFrN9*VG>%^m+h-pf2e6)d8g+dLh(H znw!?C#|4Bg;lg(XaP8fQTd{OJ7x=IZ=DmuBn=b`0v$38^kNJg+Hf_Pj0y6NS@q6)& z=WbYUS_uBCmVrs;S*)aAjYV_5VrH2#*>Y|cscg|Be+K`bG&zRr`PoSP+@E|_vIad< zs9*-}9HifFU84~^i!h>&rH>SUQ04v=5E)wz3p0iAV8dnDJ+>dtZvO=m-hcRe=NVW% zOa@$}0fq<0K$@KhNU!du(-pgD8f~YujIYvVyr*XW*5mZCvN~0qV#pj(OJbC3GKC6e zezaoyJvtB}2Mebx1RJdwco=>Rip>fkJ-7f|ThoB!du&UtpN5Ui7kGxsSqR#G0v7)~ z4*PY_Lhr&V(2uze+3N=A@QzAsDsN9bd&9`b#8u>;o+;V;Mw6ucMda60LMBDpk+Hmc ztVu4A-1!wp62tb8Ya^@3%_9b6UY|G_a(s{XTra>I-o~@V-;3en6b+7Xu;AWrROK4> z{{?x+w{Y{o7dWx^58RwQm2=KP+{U)8+-$9F+}$*1u4uj@7qjysw2yUzwX!4B@8mL? zq9lQ*ya>lN%VV+o%?w;HDGSeK@^O&oWxQVP1~v@r!DwV6@l?_#|EerW?>c=l{o??3 z9e)6SZR|#}Cu-2=H%C$BFIyy|o5%j1Swt^-aCCUNIP9o10qMkeSSeNw)kBToT7C-d zN;bjR%MYRd%YBepQweL|OaaIDPw3`5(bTD}kb25Wz(YJ1d_;K<)lfV=?S7r!=RM)= z|M8AtxeO+%?1ZhO{2o3RHbO_YJHs`N04O^i0U25)FnNH5h{S5RcqkR}2GgK|%K@BP z2VW&$f!Eyz5U;rlGV0}^Kf44Y37Ode8_U`0n7y5BG4`^~} zCFfy8(-J(Cqi9*{3P$f@C2~+$i~Y`&;C6E%mVMZdZ+&}@vpd8|l#4w1cXbj8S-`VG zU+^qAD=RWB*MhVpPbLrJujA)Sys_Wo3y3T_%GlozL@vo1Ozq2Z8l<2PPg>T)N%K?C zC)Ehf`fou=b}Z-a{uQdYr;r?b2W;OoLMnf4{=GR4Mqlh9!c7ZG%%$L-lngwWIY`9| zPSK{8XzCD9#F}>;W}eLTWomACGPS*??1@>$bX7n-W(tsx#xHpD@WyEAm0;5l51 zgwT6_=yS!LirgCyd2Vf#0ry$XoD1IK%!R(& z%%v5?aJ3>EI73fEuJrdQSol*6{vH&i=0T&3%*keCnYkKYT53hs-y<%k&hF!N&j{uqW8jr{9dX?3~q^#nI(L;>g7b-FR`1ss&B#j(Z@5AIW6pW z>vFo!)DM=dKME400eWrwV9LZFu&Y{{6So_J8P;v^1XsiA-R0nAR}2}y4#C@-7O*tG zh6bN3qyd%x(Su`_(YNDdsLq|2f{J~K=&U2}@bD``v-O9WYZp(lj=D#wyMq!$ciO_s z@*rsM+X;tSw}OM~Zm>x^2>9EsOnX}w5pEGE5XSHG_x9Oz}r;XNuR$ZvT$Ko2_qx^t6wOnxN zi)c)-|(sS3FJ?nBDp?YousN8k?O~K#NfUGS;y&-nOo89DL}z2nAt>;OKutjE1T-3G~@d?tITC#f`hj zV3Rp%JE>1T-gY970@jl+aZzNkXFMVEGReQ+ndD>qHqy7wl4SMu<1s_&_(4f1wqhjj*390l(MzhH3w7ubLJDHtxSfoO$;aOUhr7&j{%l!iPYuFDGYWGZQ@`Zd8nA6ur( zI*rl07s+TE2Oxn__sq z;}V$6u7c3PTX1^PTQDpC3j^`rfl(0Q5|h3|hS~ruPI(SPX#ic#Wk3|tK|d!0o+=#W zS-a)5*&ITStjr_VpQVzjmGemD1yf>m&Xrsq-9qN?-$%ZrL=e53$t3;2K@y!2 zMec-K6V30<_)SY3?_;pRvrd$wi>I>aA2$wMv^2O%hPAzL(gH*%>V8E~0G2F8n$<`4JrmBBn*%d{Xm&5ydpkAs8` zTc|htL?3jF0n-3KAeVB%{ayz+`2U0L%_5x3kSM3`I}Ce&--RICLP(wE4h22-VEoh) zww@dhY5!8`#qe=#gOxs7wk!%6&%c2ld%Qq?BVt(G(j4EDP{K>MJwq06A`x1i$8N3~ zqRG24cp2>l{hAaA$;yGHOR}N)Lm9kus)NHD+F?@+e_sTMaH||dI6Y4Z&PeP%nAF~c zk4kr8rpPsz!}rF#ADxHi!@2O#-jhDv-HD$q2_T7;StJDFiAJ|Iu^Uk!s_Itcpx=5@ z?H^C#o*W={&8cKkOgcGNy`4N$oleHp-okye?D095ad@81VRSP6F=hIXf@}UH&f&lL zTxq-qcTkMs#=n}ztv@}3vmCJI)ZfhKLZ|q0uOBYwlJ>iB=SQb;TI(udUBDtLb!{Bp z-nao0-p1yf7_rsSBpXhclKG(~M8DgFbgnQWcZmkMxm1cg%^JW9 z9fdft)(4B6r>ITIfUaMn2=_f@;g#+%t?>Cx=ileqPJi>E^;HK*8w|msB_F{n{3DFh zYljo3PQc5&-Jp5M2y}GBLAPWa%y`pJBcHm{^P>Wu2(}X)|5A&3?AD@pi6_wC&X*|i z`e$_a(Ni=_+7q2f%3|;9l~AXba-bn}0ST2jnCzYllIwC|%Cvkio5M53%Wr{c;Ct{) z_zj1ReuLMXH=y~t3*3_L!}70}VbJmv(EJhzNGSoi?o42>$HJO)Sx{Ao#NRsY$m^<1 zB76G?vA?*Ah)h=^>qgaxYt15Je>99dR^3bX{D>s}X|bf>br?x*w&PD?; z^2*@PRa@5z}kJ6#Vd|DC2> z;a8#ZHwWRUxFq^=z8-yT`HOTG%Hc<+f1(pRWJ@2$9iuVSev*ST~z@NA;bAj%%e* zrdP(ZqmfdPf+l{5-hOWii34l;zFry z7@sY}J$A2fgrPiHQ>H;yt(Z#o7-*2hCUx>-EAMpm6es z8Izus^opb}ojN~?o;MT)Gd{DnxZna5hdqHI$1a%p`T;!p(FEx2CCDw!fiTZ4@coq? zkl48(r?>_#E8D<8?;By# zi+Y%|={)%Ko`KxR3Sj67(C^CUGfpW`{(J{$DVl)h+wJ(|s|94@_;hk_G@INR@g#W~ zvgB}*5)m~pC6gz+5@**nByRq8!sR3onWhMGGuMtZx4yzl!acCXl5fn@=X2y@frGb-A$J*4($L)3_$5iJaU+B`(9#i2G+|#Z8T!$5}3%&qcXfasM5k$VJW0 zgE!w4Xji*9&WiBB&l1+-;idj~Dc>!YEU3iaGhShv3OVAhqee96>yl3|^~r}?{<-F_ z4?FKG_D}1?W-8aPiC#W7I!v&`t`IbG)0H-Lw$r#Wo=;?H1KqRZz-s;(_%r_=1O&Z+ zIjX&I>(*oV;MD+EqYv}fEChl&{b2WKFk}Sng4TcAVRw@$Bs*nN;QZJ!dlB?v`AKx; zFNc0SKZ;xem!Zdp%j&jSEuyP-rcsH1HMDf13`};I&gWSWjJt(lk^zRNi(GkUSOT~` zI0a|Mv_Q<2ZcvzX4^HiDgC#!KVB3yT{(G1Kw|NI26PL~VzVjeBcsHEoomXf7t_K<2 z31D_M4c|N9N=&}wkop7Ji=1iIv%9h@8?f;-wG!`EMiI9+pT2& zdqh5(e8gA1H{yidNk}u`IjbDb^AZv=U`y;6E^@$;L#nE+cJ?}Ct#7RndxZ5uYZ@zpGYp$rnUl#nt zHbGNJsGkm59AQXwJ*JUW%chWZ24cix;X}OA^%Bm(dHC>l4}8Ee6BQ4yqxZ7)!0PQX zSRq&h{dP8Bd@~5PY%PJG4KE=xQ--^6PMZ5%@Et_=J_L{3N5EBdJ?zPw2~q!9gM?l* z0RLW+|FRUM$pikqD8;&c;(H#+ELuKO9;2UH`15fEEOq50inz2I1&C+cbt6&@rAEIG_D;)M{a?4g%CdJUVxa% z$06-VHoSYZ8@43OfCY!fz&d*|-c4x;S(ARF`S0~f$HN2UYG5gOt`bY;_!tpcS#>fw zS&u0D5OOzW8Cm^$Es6XWOZJ4vlGhQ7h{v^mIBA|Y4pJRYLr(sp&3^MBZHIT}BHc!u}G4C8&bHOQWuMkHXW1~J+`i5zYFgYWxw;N)q5Ehpz; z<(tvCH*7I}xwjlmHeX45ED)^RkOZ+m6F|->01Wr6heHKN;P3c*@LO&av?Iki-^btK z^R*86HKQ1U{_}?of5ZU{8)#+YFPgMT1^$+bfZRzR+I;dLbF%OPdTKY8pWEB;YxWTQ zMI;>m%Js!Xj5>bf7luv_=~4eLziHCS8PN519yHDlfN{^lA)+t_2LH>43&l6#$M9dU zbCu(|`=vM&#Xn#m+6zktn?U7S9SljFfX4X4@MwA%tTa=B&sWlE>RwTFtZN+;f2t5A zCcVOnB0i+#=@F9gset@0^Cu5;jEP3qbVB@RkS*AWl&tU|hi3SYt)d&qpWgW-S?W9f z^2iUHn>$ih)&&l);rZ~#g zhZgPVna*gw$Uvpb_)J^1E?(|sfI~Jq;G<&t7>%hyNr$Gh6V?sV8J8U(eEM# z;-KWQiAr>DU`D6Kp!l^@k=o2%IMY<0C{Kwa5+BOR=79`S5-lKyT2#nlFIjRSQ-j18 z%_0$vrsU+txkNDCfhb&4As7B##oc>iQI!992=3VnQzq1c6@PZ)Sk(u9yzlUfwhcF- z#+H+=(BN|68~mOt!d1*4%dOCoB(Furkj0*1HFsziuKA^O6MN( z+9`(?iCa@))f@WVUJb7H8Nr+^!p}3k;Lo%Dpgq3`+TYbe$)Y=Ouk|tHr96fPohR@} z`93%p+<O9~RquBDsZk%2dh`}raOV{1RNgZ$S;eTK7%Jco$ zE~^c%pWKKi7oWgEXZK(qUo+hB`XD$o=&a9z8B z=RzNZs@Bsm=U_9OJ=X!+yRSlIVIfGTY=szkQ`nv-4T*UpRQu&$YX5p1sBC&q$1?8J ztEqtr%H}zYV{ahZdKnqbjz&J6I_O0jD-5dJNBiFXp#^F)pmdmkqmu_bRoM#6%tR1j zav+FRg1}r?4DS*A_w2XeOxXtc0eXX&}fJgN4eQ>5tOsZ13M$ zNUZH8^7Yz;jyaFVPo9t9j}KN5x3AgcrCln?jJ6>wzkJ3uCJlI&{Tn>RZz2(?pF;9; z%*kA@8RX3ZdE&4NaHi^6v}xi4DpN981JlB|Nd+)HrpQaL!g9)=W;WRNi03|}0V z!A22RxLoM~56c`uoaeeJetJQz%&e%p*ldB`(0xJ4*0q9SpSN`jBhCrmoy-^3XI^5f z!$Ro(#t$^dkY{!9Js;`O|DfGF3RDlI@xG%1(3dWN?YnZ|!0#;B&2v;X>Fj~iSz+Kr zLt)i)AGl<)0B-G5gQP=GXh&@r^)!!UO`beqjJ17GbAmEDwq-Y7@J5gDR9+(GkxsT} zEGHJCpYSK$V(g`U6yI%siB~ww5V1qkh_#-8DE!bO4I+cMPs1IX%&%g*#wf!%wSB<- z{twh^jNrALA6$HU2AUVVhY;>3*t<7Fz3(}waJ&Kotykf9VLg-vU4g+(c@UVe8xF?a zq;9V-AiteDHV|#yOxHaq zpw9D+=(4RJ*kLY#MtrWJHvD>G-Rn z8{!unfho1w5GbAktxNZU{iRTNpz8(wg6*>)xY z8&x9qwhISjt;1*hjtim#MZj#oKkp^=hQeP4@H=ZKw2xAdy!oByto?(Lvu$wMkAqs@ zTX5`58}vy(1`&SW=ID7A>Q_YZ?vz1lRvyQ+$G$+`zH_kp;9Pt~PZ=XCaV)=81$XXF z#CHzf#%?yR@%^Mv*meG6Jl?1pYq_Q2*9%SYy_Rbz!uT8te%OO1O#g-gFO{NaT?WkP zcX2u{JDAp$)X*qpNpRX_2z%2pn7>&L7BNwTIa&V7ZjJ@_Cr@>!1|1rM-pCeVF& z7Tl}Q0bk>nbkz3%tv+nS78tKYwx3%OH>VHvCI#Rbuf&N&wKtjA7)xfQ`;&r?6Udv? ztN7K=D>!h?TP$TQL+%Bt5N}mulHg=SW?3kaziur!f?q@3OrFo?UseG5J^JuaUlGP8 zIKkK6JP5!26yo#0LGzwon7{5Wm}%dJigS-3)~yG`|MkM{Ry~jd1u%MGE`Psn zq!3__PB?d?aT=0%SIk#*eBuN=HarlQB$s2aH=S7G#V_n>DMpq#{lg#B2JlDyi}+es zI&OXtjT85V7MY&-acr9TJPFGxchiVw?|CI_; z+={`t>=Z07zXZZtwUBIA1NZoI>x!$TAj~d>Vf%deekK{5Ed1fj_$go#7flV^-ZEq7 zcB2cdG2gw#cwX%V{DVv<$-DhYqJ9*yU*Sp0={O=#TIihB!PdrPN$zY^1 z+1WUe#3lEGbA<+ckmHp;$Z{1FqqPGi-t>Pf-a~aGUd~+I;z0sz=J##wG%D;z5qu;7+$s}3V%M7h#Ao$ ze9yNLKW^#6san7Bq|Kt_#i0?bZSoX<`F0j>o4EyJe}r#4is3yn9Vmk5Hnh4L(zkza z(CC_JkW<4mhYmSFj5NO%U$_cVgc<0x$|vS_k_IbTCPCvwovCAjH?6Zsqmh$~DO*`c z`xNidO-LQOd^f-}w=A&VUI@CnMZBYpKdY`j4yVT)hv=zA;PB@ZytOEYio|kAe|sES zJ5!;}d^bokzF?oA2W8@KXjcCknp*ZiuoyznjyDsr-8)Cz^Ck#C7x>~M?<;Y?zcKM@ z3L!-av83$!a`Kh;HY*%>fSs1K;GPvv@v2B^a=&XVzmJt6@4t%@@6z*FEJzt|nzWLJ zWbJ|q&vQ^-a1n;m%b|@R>Y-xmJyk=zIe<&IO(wcoBlx z92i+QA8vK-r>CEtMxtB)!{T|xct%PcHZMGlAGT*q{|g7y;j;*lg3oNN+gtkejt1C(J(wMv3%V|5a7=;$)inNn zqr3-djS^s8dMK#0&4qGo2!`)eKyr>86iQsBeIItxjULKWbbkzM7NE#J-qR%b{QUrP zaK#=bDt4pwuAoHQeF<`Gb;DrYK5IpW8GS6Ayv~}w4?ai7HBN*n?iSEp;sMJu*Mq9l z7RZ^q6~<234BNIw!{tRe{C~d+o)2D!j?6~BcU=qaey8D|<{?-k$*((xmOvY;0~^yu z=nl1`w45TkYjG&+8exvccL~v`ix%z-n}e72yWziozTjDdbBW!~ZA3>toX9=YBF}P~ z{?GWr^j$4p68av0O_L@G7L&-mRT{)|mL$35aS?AFr-<(rnbEtt4$wDwAH=_pFGY9Algh-A^eZb=Z~=E3ZI7vAA$4^{8oLCib~#)TAuK)D=} zz8ryz$Kqkf<`6iz)CNMgPk%06nqXnFEn?T}YBfLG$&!9OF6oZFqTnAD;H(32xQu#2s;$uy0r@t}0uBW!*IK^`bxMNAq3exgrAjz5Bp; zm-MlAr_R$UG2)=eex*`Fzv%cR6`+sj!=?4nkQ*KggRL=8EE@>hO7&s=sakrg(3=Ky zg;K{O7irdq%Ty^NpIW7ErteSd(Npe+>-LyWNAG(UphxOUkzm;{Q{?@MIr;q(bHnVZ zu$b=y5`T^zN;RTWD$D7KRig00N(-7=wPAI}Y?y7b7>*y<4?Q9|u;LrCB1u^d6-{Yt>v#SFua|qzJKZa@ ztQ>@D&VWfjtzqmaU3lILlR|2mE`jOxV%RM?2B!>I;-a_9@Z52Kyc->WR`d2^ z%a&zW;{TB5O@2m`=|^g`SqHaW;h#NS zH$47;-xG$8W3nm#d6H>hmICmKEW19i0W^t7|ZgoZE- zSL?&kg;JtI9eGg?^9y%>e}&cWKcTqWAZEyQ^WBnnaH>iJ&f&}PYJ$TtseU3#m8_wv zTV|8Xg7ZXa&Lgt=70-9ERRJH>V3@u4Ed2X%8Kf56fOElhFvp`D*3HX>#>g)a$#K+Et)1#r zYvHSKM_j144aaMR;!?3FjCG1f{q4!PZgU0(C}iQrRk>)Yl8dwLPGi}$DC`&8j^X8A zDCgye)h%mrWQQwG{Ar1qG54s{_m5n2Y%-ad!!zNukAuX(O>n9h%dT{q#Ts9;WAz#* zuu5MfSp)tKeynN?`z+0zopDjXcJ8ucQOTSg4pnD+Wcl~dK7W`Hd4g!ROVAXzUS4+= z$6ZO&F!|RB)RKFH?>;Dq)($Rd6}4r)I-cf>qCr1hZT8WWsQO;>%jop zCSJu=J$(MX+XPD%M$pc!$%3rH*(53EGP#oE41(e^7?J)K^2E_H$MJ81@ZRrKjM-Fx^LU-%!<`0Rdw77)^g7Wu zzZpA3SI|un(Tkt&{xiLbvMYE`VaWw7T9kz8vsh$yzN24s*U`)E?cCU#g93loZDg_K zXL3((JVX|3fo)&&!5;s?;TTu8C-(rmF7XK4uyHv%Vzv>xIA#p%>~GELZDZLBbC~b~U)3?}!{pD)P9-C*vnx~9m=l5u^|9CFhH%lzmmHH#r=NSBQSNvihK}jM(Vt~RRiG^zex)fA-y$zs{zqD* zmn$#YWF;rM**`**ulO0iCUj%j5=0+oo|*7y5?)GQP5%@G5%X)7FmwA+IAxm(F2$$8 z`0_cpj~w(!+=5%3x8eJLjSw^U7UU>b@iSu@*xla@ZmK+AqB?|p)s^BzUGdb>?g^c{ zk-@f%<7jrU3^(0tKndzl1f4$)i_e|_t;B2)8&M2F zUKNlVJp|`wEMtog#Imy#Q`v?&!K`5OG&cN zdnch4zS~%U;E*mIIch(?np=f>hfDBEaUo{c^rGS;BhgCVB_bQS4WiqVJVi(C7K+xU zED(*(oFx*g;{6--R-()+yyyA-OVm7Hj9zheSgIXLeGHC~-?`Jl@T>>CeRCWPtob_(mi+ku@s!jUz&YR#?_PGWz1>ahJQ6j^ANX3d*F!D^pYFn2i%>soo1?NdtR zM*rYsxf!T$m59|>FQLw;`>1{V1m}wQOY_k(clbak*T?)NT*}~ z!J!!=>hn?d+%Dc%K8p7!g;3GN5@!1HPb4_o73{u-z(7VI|x3v-eTjV7`_m7_;|@f3qRUNf}8aSGLg#MSV=nUW-2;rl4L{DT;W8 z>s$Bxs1VVC8iF3&S=Pn#p4)I&!)=WFa2fses&V*T3BIsT$MT0^sP)kkSC;DI(Tz7K zCaTfK9XgD^mklXvP9PGhC1la|>!k2wADMMb2Gk}i!*&h6OVd*V!t12KQK5^BcRoPA zA0169I=UFipfk+$8BxT4^Ea}?<}CayHe?6?3fP4!CbQb7dZDW&5I&tR0gqeXAbjT- zHigV#%@)pLvr?w9{gZirkMKUs`IQJ2O}bz^LYMfgx98dxyK$=3`IKE2jdnG^uy>1| zNGxoe2)i^y!4;!L-#Q&)W{??hPiBa42qnTj{}M(~}qm8f{|30~higkPe6q59{)xa;vS zR_cxry?P)i%1|C53i1DjJNCEW$D*@1d)F@9esL83Te^mhu}&cg*1Aw8&-^+qqiQo9?f6E5>y(34OzYZEATsza~MX=3cB!W_Dj zNs|;O;QrtPn0zA>Q+jIfs8|!`+B`;?z(MqaANY3K53GLm9k(tT#C@`_(Cy9>oT_&R za}+s@dR&Z-fr)r+#cquCv%x&kW7=Z5oa#L^;!?;R#&Y&lQXP~_+@rgQTD2JL_{w|q z*6P5r*CxEC%D_@R8_-J|4G(^Hl7Nkogl%^tU+vbAQ`6f>@MA|v(0B)$B{uBU=mo6F zzDcZFf(S}(@frmb!S|J7to}O-wyE2dJ^gJids}@fYq3*?t?Q_Qkd0fx#qS3R2yJ7G zUY0VkcFByeeJ?!`6^s(1@3{A!u1G#dS7e#2B6^~uBihkoCOVQM5S_j-SEPKKh}z#6 zi@eg5MP8L}QA6@JK6ft1Qr!?#|6q;d=RT(!&x8tR^IYbcjpZP9{x_`jl4oazj${{@ z@VCJ^GHmu@WwxEawX}BYvW2^}ST$Q&HeqQ$RPPXit5GccR2mP%?LK5!)tio+rHK+I zdoXZ!4bIU2jIBpUiuyLGh)yn46Ajp_iWbQ0h=v`;iH2Y6i-JaLiX7)jh~j?Tz!^)E z@Z%uQY&m_He)@Eo`J~=SLjGBTgZnY)OTG@p`6Jm~Y7SlbBKFMZj*v^Z>O+lXI%~^4{J=+?#ZxKM*8*3=~X$dQM-puJVzLRsl8jK$Q zlr&B~M|Py0BByU0Ba79R5T`#CwyS&j{6Lo!R;7HQ?`GCftE3rp;dTLKHYZS}=?+_VW55`8=fDr} zajyi!CvKoVT>`i|p3(W=J+hu|C&lkx)2dzkJeBwl4SS?TYjkBrMeCGB{x9@I`zn;j7>SlnDIKwwwZXzw}UQH*QLi^=aT={ghEb0 zGc-hf1n&J1oZ9~uzMLBXMI|xTaiT2y+*y%r<8_)JUf;p6wGBoToCB4CQ22Jv8thvi zlVKm0*e$K)-kp6-*Q9Sk!@OeLfA=OPUT#OT7oC{){5gL8`it-3mJuB^;XM*ElAhWpZ){AMGa7{@Er6U-|>9e z0hp{a2;XZ4U~b@hkO+DWFR%p`msdc{wm4W><^eadv>|DA40%vLkFy&oq%TBMaMHjL ze4cy;qyJpNj3>=FwzvyT6T2`>trIKFd+;*Wi9hrB41wVdJUQ|#t{+T7%_cr? zu-Oao<5aAx&_pYf4^-`35zmjBO>58W6k5nnAyx5}WRc2WGEzz$oHHH}sS;ioqJJwGuAy+I7kGTsROZ%Zgu|#}2D|vZLBpvi~m4 zWj~7)*v;OJ;OY<#hU$vI+);#@%&qXXMC8)1zl-Tj^!K7Fusq~C#vy{WGbFz!?9A#0|yEJIgM5% zrZFyX_(&eOIh=<*4QD~wybM0ph#>LCD{$fGOBZV?Rvdpp*RNi9A5afpY71abOfWng zI~RQ3$bs~iGejwX6?p%-N*8o3!8>~C$hus@%`Z5#5MIS=^da6Z9KwB43L@)7WzojF z5~9PiUtr$@#J@~A7S6K}|zdg>gQ%uPuFAI3!kqid(GB`TC1UHvA z5NP}m+!o%11Aez)E|oK+u$UYYf$VF-NwbAZsNq$%h7<9^cyE&ydb^om!5VGcT6c$*D*3_Bkz#Cs^jNk(^#@4&*bJ@)BSE4h z9cHZ-XA@UgvxX~HvunlO*nPaOKL8Ba_y2xCljSYA>`?)um!AS@3l5Ggz5rpoNA=+H zx!nBid(mxCEnfOlg*z>C@f1IoZwtPS;wn6AB3oTFrBh$@AX86t{*SgOCrVb7T=Esa zJ?%lSsZD59e1pG5h%n1B5jTG@!45BZ8hk8*Y|UsSGDDP1x&DgW^0NYqzDS7Gy#(DO z8sVnkKKOp`gd#q>yZK}-tXoqAZ?~qxR`F!W%8Q0n9WSt-paYr~E<~d45UqH4lcp>k zqU?g#H0PcYzl(aKrphT4uepeRYa3C9w4q5%8-L@yiSd^&V75aM&TG!bu)vdeZAJvX z9J35xu2DrtlNkD!_nJfj|2)gwCal#k8896OV~?zXb6#QK*%k+P9-e|DQW+riB^6w! zo`f>FI7m5i9IR}%LDiKh@F!x3D9eVBvV;&u>B1u}eISCKTXBWPX^G)y#u&v$@Oz;B z7Hqs2juTg>qFG!4>e5`i-I9uHm&D^$`7rEG4!~;u-Sr3O;rEdYDtwuWd%tMm%eRlI zkBu`u<5(@cd|isn&<`cJwT$pKGcsXaF|jbtCOja4jN$VrOHU1vazCEawOs-lk2DjV zczY7i70ZadTQ2l?WXX*?y_bvim8OCx?li*s1Qk!Lq|rBi(rNkcDVHXJb9$cAId?`P zv7Ul^EjHkS@+@xd3rwil3iGB3Zm06(eG> zIFt8oH$BFP4-%r@#R{UlHbs%{rIDgl`5&>^?gcK=zKa6>`y`ol7vDK_qUR8YA-zZN zTgG9U=bK1Qq_mK=&dua-T^9MhvxcPKGlU&q_5#;m43oWY!t-OTFm-NFom+=@rt=cUMG zU&hM!4Olg;9?iaA$E4iLIGJ)dPm{wgxe~l2lZ=@q2eG<(8LB>T#*C7wXmh)jE^q7S z?5y}+Ej3-x^PUU)*ZV=h@|~bGEfB)ZqG03mcu1Ic3dDvBU}8c6i23t7mSY4&{__Dj zepYE$83Uhod?#+vm84c$jd+`MacOErbgTUcR5jAU&dZi~D8mtRpZcNVs9;=b6pKSS zDcBpFi3LxyQP%Smj;;>F73Wst6x~_K$QYx-d^Ic%kU$&gpcyO6>4}IXboqn7!t`Bd znfAkbh{#w-!nZvj3FjM$mleM!?OZ@wo%a)y+;Xzu%|}wE`Gx$oyG5GDIFdc*h6KKQ zlDP8ZVD9y(9o&&Ihq!f!YhH5z~(4Fi`8YmV+O9JDl_NW*d^XWWYseP9w+DafJ z%3-1v_kh&ahp_Q(FL+G}g@3t4gz;%2S0lCqt$hOXzi6-;3v^hS3~e^@x&#}ytOMSx zuYhqkGhomAcnFOyhT4l2aO!3Xa0mVpkwPskUE+hAd42tDsTryroru0Sd~nvTbJ$bd zi!1#kMD|TGB7v>6s4w$7dRDx_x+CqFy5cI%{f}aw>LpCPcNQhPBQg4y30CJv(iOi# zIc3NFjL$(wqGa-hjMrHT{*w|RWNI-S8e0MTPMilB@eA{7dANS43?iqN z!Jo7oIJGhge$~%{D5)F7_vBb|VlYqedgdg~Lw_QTT=#??Nai_{T7mdkKM~JKr=pbG zX*9|`gA*2&qw4E&e14=9M}+30r7#1Z0pD$W+!Hk-jIq?BliCOGrBM@YxNRj0WYmv! zMCx`s&-7M;t)r$v?|=vI_uLF4EW#niAr4-rMMB`NeX!-oD#)~Dz+uBoXytdKrRpDf z4@W7P_tcs^-)qIJ%gy4Zd4$r=s$S~cposzRY;nmjgVx4N(201X%*vy9Wp)t8*9G99 zlpjv6T8}Pg{IF1K2Xec%;D+6+@r2ZDjBZxN${i1>`uy$m#!nS$IQ5%QcBe8^CSK3T zXT4$UizgAU_j|~}kVX<3BMC|JGH~Uz7`PZ*BQXINiT|5O;wis@+*KY;oa;xDx}-l$ z>{UlWx5gwY9($Emw>+a_EAG;LvC(wkoD=Q4?MWY*w9tL)l~DUhA6+;kB$2C1pnJ`8 zXnj-h6%GcL zgR1&{sIcgPD7QS=6aJ29-~36XO4r~bgQZy7ua9l5D(EiZibcOpqDx65F24E&tJ_D4 zB45jjH1fnm%JTiFy5~OHICY?-{411QGk|*Tov4^mge{d`SaC!fJvR?h*WEH$J?0ZF zc@j$_Z){`g?+%hjYVKe>B@(VKO@TKZ@zCyh5Y|Z^0-iMv!vQCNC>O!`)de7MNQG6O zcfnH`YZ$Xt6&_ycCC9sRh|7;*W@tknw{Y+*eeJG}ruy>`{Wqat{$U(rvlS1l@Inal z#N-iyXvp)hW{xSu2^xG)Kw1=jl3Ru2mZ{>CggiQ=dr9c~{S?!>zLU8-&73)Z)R<{L zx0O@{{~(e@^WgI7K!|jR1Qopycx|@}_7^OLEFUKb`Dg-PWi(-y`7r6Ic}L!yc}cbh zRg(HNd%{X6ay8k9Y5OYP54rL$9VexOif2Y6t2_}qPMM;fk{Q}NGC1<7BYO2MM~hEu zQTosdoaj3fxhM@ZzyFFB#5||#^PW?Uac}6hXZNVxTAp7#(~_zvh6uas9hl+f>rBhe zN+!ZjP5^11LG!MjZTb3Hq$&c;}UzMI_COvc@9}}EO z;oB2T^vdl5rIBCim?cv=gNCHql~Q+wK_7H$rHZ6rp?NqAsMbQus7pL6VLiOrWC(cJ z6XsXuL6TZ87#;Wvg_-XltojxB-@6C%0_xyy>^bl&ONZ@lq2RPN6dD=>;MORBp&cT! zI%^RPaZ|=shxO5YKpg|#n&J989;kaf4f*jB?WeS1Y0nd!GyV}87qw%3P9wfNcnx3L zUBpPKvl!S_h%QR$sP!foJ^Gw+jYtm9)*`)iFNDSm<+#LCnq+@DCI3t{AjriIrZ{_p zxJDrOE)Ir@S%;w1GYDQghJvqT5~Mn3@EUCzklF~?pS=mx;wC}!jc)SoTp}5_lqFqt z-OTTz7@?a0sO|O~8hn?37BBy%OR6TIYvpnr)v+7*2trW>k6>k}FZ!Qe%I~tXvH0}@ z44AzH8=kr#4$MNm4?JVHYXpC@N~U=e=5w0ikC|9qWwIvs8}ohvUy84=o21UVOW=qK z-2FNYXoww{nOno7LPLm;Rs>M|NFK#MB;5z@kWuR^3G_x1`#*N%`S_Cpt+zt1CUzXX z!uJ?Hs4b<_hFhqU$6tEm+DMF^!|P!dj;OlX8IR0#LWv^`p8q`!?IJDE>efWQC)NfV z{#m2a93u?dt&DRnKH~Ypl=sC{Q;7i=I#=gCUmEE`BXZOD{_e}%-GW?3_X3c8HzkS7 z5)otbcOO%{DzPqt?dQ_7CsLn2BTj70Ea8@Z9rBb*C;5I;$?P3Xl<0rXC|P= z&Z)dsumMf{PU7E3h3M;2g?stgrqcc{?pg8-^-lKVvLz32)X)t)VRac_m0iNqRp)WO zSSGf(uf;}}F_;wHL67H!(mk_g^0!$B;kQxoWMGmw+?oO)*0me@4n#nOMhYZu%Yn{) zIj~HX*Wcsvq1Yu4O55_mJBs&086?44vIiC&TLu}|%%J( z%9dnVU@DRF?jjHMUXx9-cgc){CB$?vg6t6AL>lc5lZ{q;NNdO(628ou>^q;#Z2v08 zrCxZ;sVw%Pn_9L|wcB3Q#MOf?H;Ja6A8*lxS7or9OvGQ;Cm=~O##igdqrN8Z`;|23 z_ihUm-@|)^`m8aw)e=u1o{kf%`5YKC2A#f1;8V%B^!lex`et+$b*@XN$(A3u2R($_ zRC`T$@|F@a>?cJoIR=sgqr}1Np*+k98x3(!40-;y7DTKc58KWW7^~|B$F3DZd{ZAd zJAQ^YDUZQqd^8m8c7nm^RCvL=q_ZUtVh-iB?C8~K};Gdkx^#+0ZDsG4YnpB?An2&=hhGI<)F`J#qh=B-q7 z_&i0`6gueWOY3}=(_jA5)Nw=ur#K;$i&6XHV-$i~`Y$GaYsA>?F@10$B@?`1C0y0c1c&|~Q1U~IykNcQD~9H4L!T@+2}>c zbt&NA+dN}h${ja0<>R6D$C$Zr2sPLK#&bu9cwh4;^zMF&)-SuTprr%NRNL`UYbWmy zy@`u&<)h=YAigim33Hs)P_}7^9-mx7rwh}BOQX%n9M7l3=ZYT88qe#%#a>XkGz`jL z^L#e9(_sHT8_3K5px|dQ%-Z}P*!?;QgEJyw{rp4lU3xoA+T;hLzq`WBiN-Lr;vPA4 zb_;2CzrswEP!m2%y1@k=^Pp|bLK;-@gqF=w#qD1wVW!Uc zNk~~JQAw~MUf+H&vF|PjG)9f!7B4x$8M`Xe=Vo)L`P^jM5+tJTqn_}+Q@`jWfhw}* zR`@M;GVeFA!o=6(aG$Xr?o^wCyIgc}gWFg>7p{uyh06RKEXHT7-qXW_DRfVlKMij& zq{2}-+{zg*xE`XzO|tkWX!Dpxp3W{L`JVc4rb`PR{L*(2G8B`+apVg~pQAA+67 zmV(*NeIVMA2O*zJA?K(KSStlFC%!r0%kC`nxtNSGqDY)>8j0R&8F;z*Dhj%Opxuy^ zNXJcGm4<{4Nv`VuhyS3pg>9!A?;hBcdRfu<~l zhWJW&5mf}A_N2pPn|Mf<3<8U?Rj@op6V&$)5cBGA@;!8b$@jk?e5VvZYaYC%KU%b~ z;TZ3|j-8K3Y`szaZXh1~9*fs}6ZyS08QW*3@w#?8{_IJ`+GEjJH7}57%C1AEU?rZM zG!ySHm&0E3=hVRK3=LkINM}R@(L9>ZF+B=Izo?W%_WvbsH%r0EJws%)q8Kojq+!Qx z1+Z-!4dD*sz}?9dP8jjG?>D*-zW6QK5D`yeTNz?~(DF|$k`g+~QAcx3@zN}7*n883`eUW21WyHQyo7z-EpV|UypOz>EZ^&36$ zdVnjIU3bBIyuYQ;TnmRYBvJMIExK5#h;A>RN>!wf2v>C~ldU`a$&X8|gxZY&vkA7a z&20fFsd&L-wd1gA=}k~LAkW@59>cEXJ&i|#JE6F<0w$lm2^WI-@687@_NzF<>I6?= zt+uGK6CHoUY^7RYa$?~$_<`o;jj-}aGW@(;412T>fT6`k5-B$tw|Qsby+g&gY-T+A zX!E}Cv=nqKK9Aehy+lS_LS)mSBzm_`Pt@A0E3$D?616=2jusQU&`@|EzZwjp^w>{m zxW5~Xw_d_@gHYU*XNP3EBBuV1qlH&k&Y}Q$Ho6B<=*lC9TZaiYdq5vegTGnlVAH4? z*lb%4K`JGn^}7m=Y`g>`<*H$maRq$5Uj}zoDqv&yd06y014>5uLblEr$ggZ7Fu9og zwu&VM(c{QqS4Caj=jqhJuz>0;lEJsjt#QZoJ=hX_obT>Df-7eG;gK^-QQw_ssg94w z`W1QjkAGGSWg<{oV>z~On}qYGj=}jY;@Ds_Ocz~xOe`e_r=(ou23zGw`rJ|? z3cpIuGD5;1OvrZr*_b!t9hv@K9t`GAgx7x!KzfrroLt#XhSS4H*aDuzp6dw+U(NBJUO0o&?lsEdM@yH ztcY69FtJqkfz4{$AUkX;=#0tWb(`BTc>FIEm5gM!9UO%9a`$0FVJ}=iEX~^OHDa$` zpUL`c?1q0aJrJZ|#@1+M>ad7`4oGhbKaeNaoZSW{87@tU|ucW*x~*LsfS z=f7aB+efU}(ubc?ZXmQLp={PlT%Tl)|L9x#esVHRE0Cn3Y4T)!U?|ZsXd?G3#zV@Y zRZt)m4pPT6pq6BVAR`xMWR=1Du(L2K`wVos{|AlTr{UXR4iwEV0`azNm;=#p`mQH< zJk*7a4|>V$^vgUqIEu6`oJXeWePtB9S?)%i9X*pLq@4nNymxUvb~G)+PqW;xX=obG zI>+ma`V2l>setakGjV~={;F&f}zOS&KzNjy!7qe3+x7v|Dw46=-M_AIU zqy4#JjrYv##ixjZ-7B&;M*?d9JtyBrbP}(|dUB|U=Ns(cJsxqF$mdIGWa|4^GI4kT zInx)$94pD^dap*&;vel)d)_zN?cE~y*KrY$#Nmw4;2dT0K?eUmP*@)W^#UOTKVs)Bdl3c0{(ci5Nn*1(6;hA=7sg6e#95_miUEoVc${n;sDAXc#f;R z+tK}f8_Enl#Z=8+9DnH!{`NV8`c;SUpXn?-$eQD$Et=Rr_5ppfVKyBv^(d`(jYxubPtM9F?~%+-X?$f(&`h=bep1TWzpKj_O*_h{g}|EOE+e0s#VoLlqd zx$v;yxZq-8J|m|kLF8_1B@Pj%$jY7q(%e-<_Jvmxo5btny>Sas>JpJtzf;NR!z;-1 zCGzBqcMRk7>xu0>ah7}8*1<);CUlugE*<^kG4*?(f$!EjXm;i#w?7$?hMXP_fq zG55h41zSdU4ygOV7H7Lm#s-l#nt#&34##_xu}!A3vd_2%!+W-b z*DWKOjfsM67n$g80NHulp*`&=EDMN)_I<}-!^P9!<<Ssk^2;aGqU7v`vSDi-H@Kn4Qd=9<7RpaUJH!#4G?~A+j6cr}DLeGzl=&rys z>K?@NjK)xO=!n8r{ZQ0P-iBWiCS$oz1ATqMf@Z3%<;bF5X7Q^?GPU-m@U*ZX?&6dK` zxH+)PeiCeoeL!@Koym-EiOh0W1MbEBB+k%Poi5rDNt?_&=)q7qjQVSiOsNAVHh5rv z@CKYd$rqKImZF852YT82;Unq8m~br=|LY0B=OQmW_h>F^l=1AT(CH{Yo!81sM`6Z3 zDSTb@m>Ms+Nw=*^p>LPXquBVHbN7zrbbX|_>cStwetNSuBWD(b1gQcjbYR1o=R^zd2b4}|8S1> zG*r@7y?lD+Vm6H`KTe-LEu@MEW9gT(p|m~1gZkuH(2K4NU2AcZyQVNwuz;U4Y?Wfk z;TL@Hrt>!WWc!bZ?oS7SgaCf8_X4LI`@z+-2qO0JndSps@VdDP#`b0M{TX?Xvikyr zNIir2&VOL`jPHHA&<#b?t6*AeIv87rz{(#Rz|3eN*ho9TOK^lmF1pY?X)W=d5<{oG zF+gqg`3p$`n-tNjGiSrwXs-*<) zmob4U>+IoNj5l;VS_5gi8^ParCp@%20umhGi4b}KG~;%`3(qxh)yN&{mFGgnX?@7c zRe|k~dP%c>9r^qIC>g$&#?V1$ZcXcSTCNpL&u+R(ZAM7ot-EGe-m(Dau3d(^t=FN| z)lK-XU=Pmcg7Mt1FgzU}hJHIju(~b~r!Lus|8d?JJ;dVdDHAbCRvY6^4pZOmUi#LG z=e{QxP`fpYxbs0D1#O3`n0u?t$#B~a;=a3-h~Ga?q^oPlMT2tER(FQHJXT12O|wbB z*jQqv9!OX_SE6Oy!5CdOW9Zpb&TjB4H|n$^-Sfzqitanp1KR)56)M-M&%Q3|d-@N} zEqY4RV?NRiVb`dA&ONGKbcF`Wwo&oP&2(Hu8a*4fgo=#T33sO6WDYNJBV=GD;RDWO z&Al38yi68GrrN@|FJ5pmF&aX(PQXId8u(j%3BFISfp4+rLG#W@*nIdZyxhwC zahEt7GQStjxL*ZDhkUsBD+g-7oPrZBr{Pdo3i!?sf-)H|IH1D&P3CBllYCD4uk$Tx zyY&%O*l>;FU1{9C&jnqS`8(;Ii|FXjcRxurpyJt^2ywUYaL+BYHn@e{&n8qp(u57a zn{g$dOZdL09MgC3Ihlk#Xen^UMm=B9EP=ZJRn^|o9(@BRzAWRfCoNi;yu*lD=UYBs8_ zUd?v`9m0&7P<*#C7>hpb#$WyGG5LWHW)*M2h8G*rGG#I1zUh3w6VDK1UeXupeD_mv z0Zr&jqzD44H~Sp&x6+%3xFNF4xFvl!nGS~K=g4TT;T5-X2-`t{3%12_d^$A3V)FQ z3d@MFOP>hqW-<21Dj6{yUqQvL4sOQW^>m8l1De(LjTU_UL(xPV7uXr0^cEA;>Y9S@ z*YR`jw5ho4?|gJ>_QZMm%g}SD6;AL{!}Pd*x?S@-o%4+65cAqfe$6G?zQUc#rgw7O zt+QOVj89$O)qEzvVk}v3VF3x)>PVjNm`}=jJjwOt8N|^%lYBUMocvODByTETGA4!1 z%z=Cba$BQp<`=Lss~;}i>xAmID^MR> z0;?6X`0vyy=oxt$E`2J3YwmfV;*kXZ3i*A#`Y!L+kS8@c8Maq*Hwe-O1U5MbXHczE zopj|!0ftZ8#q-ajaq^3k=*xGT?_Q9P)>8{nOsN#5-c{jxSs{wAp-7%p`P6|tY0 zP86(fl83efBx#BmT=4iw)Xl|Vcc>&trz^p6X$4qV`j*^rx=wB`DQf>8+aBL(=MR3<%j7s8J^$S^ogD> z`$T)MJ)_2}o2Y|W9Ub|-p57x>G@QS4OxDk$h0hA;D~C(;{jv(mp508#B%X1CjO|?H z>vZl@&v&lSOr2W!m2uOA_QIKe>jkbII|Z*Q&Iu;ZGGGq-u42~TTgTk$m*o0QPI439 zrf?tC?6^6-ZMN6`WeeLBV3eEu9iAQXFWVmbFS3U`m#FO zJ?#pu;om)D{mbYbg(Nz4Sf6T%M$(5u6FN9xLc4vG=;G;?jPP45dG6gw#Jl;t+f^3i zQ#Qc{(@=QhejFbE*aCAJ4}#V59Ee@f01hsL5GeTx!cTR;*zpL`<0_!}^Eq()$>%&B zI7kerhQ~qYVe`FAFl9D_vG*t#kd7e9Qm=U}T$XH)9mD%Ep9t2+h0)?odN?lB2ld4d z;y5-8EB41?&?r8muA70UcV=PwlRPv|E5@nI%K6`xQhd#O%>%whqjkehjIY>;!qq-_ z5YF(V!l5jV@^CKXEJV+d%pRk{p&&UdB~kiow1Q9Jn*G z9qPjRS8-gh#zW5G^HXj>QjGo@X+kwW+tEV_9u#Uf($}^j)Koi@7T27kYr+a>@QM?3 z^@VV{JopfGvW=nc5i!*0P$E4Q7)`Cz4^p{LJ~Vv(EGlVgK(C9sIWPCyoN4QGZqxL? zoJqAh6*H5g-3mS2fxBhgBD;GWb5)#!4LXO&Znmw8lub1;=r(xbxJah%PD z<-#D^ZF^o@ml^pXl3AWFL%ui#kqwd$$X3O2CXrv(5ieIe)F3AFWopezI;85-98?EhJ7F; z6N#!;A=%G_6M=^q8F*pA?Ks(@tDAekYFb-iIf}g3;k~ z6c*;5L>W;gp52{}bN8m=x1w0gun5Ik%|keL{W@G(>55+@`QM*0hB(0UQCISejZW)R zw6%IV6-v}_jv_0e&A$Jbe`l@8htHA3@9qV%*5VGYy?2u2hrE7!<0tWQ6^AEjvan%+ zILw{-hxBHBBx{cJkR4JFh@%Z9eN!^Xl;r*7!(B(>o?=8MYX4>=d;^)HlAJoSD47e% zm789MLDK3d)xOMgV3pm}^R!!-2*I_CZbS`u=bp4-NAWPE$5{%|v$|DLZW4KAgx zzUEU`<1@4@AdNm2oT7%Qp;XJ?hnnawr_#0tRO9|Pu6w~{ZgXTTx7fj+Tj0Eg+pII5 ztDUdR{hcs|YhV4>_VN2s%$Bv`Or2OdBOQB^(MuY|%z2(*+dRoaU|0XS=8~oz&q;1# zY(I@462mjeiK)}b<~hpb?(HGQZQL-E=Br2itXC6@sR_h%QUM8lcY{p-)k79#yd*VZ z?}(n<3lg{WC0XnBi@Zow0c~YHI1oP(<}H~FxA#wkwXWk}+q_Yby+aO69VG#JB*BR9 z4DyKmN8sse@_6tfsoj=L5<-@du}*5l;P5@>+J8%#*K7aS5LYu!Wu}nJ*lbE0y7$r; zw+uR5R7DHN)XhmxSx!I$a_eOA`n8(7SFc=;)&9)Y)2|!f7#@67+^EIr*C_ z@spvwRXyCk!4mF;_i9d4rb`$#;#OV4&0T`s-#P?N&glYy-VNd7m~GsgzlS-UbUCj2 zkcXh$W-7D$k`@z3|JZ7pe-;Y1UF4=ee#pgMdB}Y@l+Fb#dL}&J`L#|#=7li2v4L~k zZ%+3*yVBg2>2&$C5!50^$PGX>x0p0@A2dF3j#;|&iq8}(Un-y}4io5?lDC}X;6+Y# zPAZobd7l5wuW)yM7IHo<@tmQ=IPTCFaW1b$g_C$=!ada*$w}S&RHwCcgFyS&O~K+r zNrLmc^#tTspN(-)c%8z-)>dS90eO}_2H4TKFHi01yvVE zf$Nkp@Hf&7T=J{|&a&Wf+Xk+9n!=8)qd^Pg;E~=ikqi4uTxWbBk1_{{6nR7xDyqm4 z*X^Y1=pQCS!iW*0UDdbrgM{TrCUeODAIk@==uXvRluDM;s~I<``NZdRufSBLp0LOljbDaQD66oG~v`Z8YFH^{d|pS!^$z#V2T)BVttxh z_~4Up=eEnWcD~zd=C3>_)Jt{XiXW$N%}2AiQpad6@tZ#9m%3T_;r6{cL$~L3R*q&| zy;e0h(o&j6ywsv%xf-;!Ux7~JJC^J$6zP#3JsSFVCY?U$K>PF-(o|=6YFECBw#NHW z#o7LJ)_8wppce8H{RIl$>gf8e5Hd%1x5@myqxVx4vc z??pLJz>R3<~a9AN1Ge z5{;|5yQR9+ahwO$G>E1D4rI}h183;H$TF&2aGp*X&rx~%oAhk>eR|~MQ)<=qnEq>T zrL94isNeNMS|H4(CqJK}A6K5DOLU^?frJC}fQkn_8PC%38zxZy_u9N>tVQ>?>(Fjh z6MFZ~I4TU#qQ*S)Y<0CH)vs5feyNu9(b^TXB5@s^ab*k5w_Qi42dt*PZ;_t z>Mr^|FpBP6l1d-PpP=p6qG3jD&<@(s<T&g!?rG4Gi+C12(|tCD6@7^mx(Y_ zD{YvEg3U}tRyy-%cMO6B~Nj3A@?+TObSjQyv-ef9n zJZIeQOOn=>3FOM~bh3KEH1foAJke2CA&qPQ{Lj#Nhg1EAVcaZRMr0(}BO@c6_jyl4 zsi>rZhIT5ImWE2%BOx=pL6Qh1ao*>p;n$!=dn##9(q6yszvsGK*KwWic%S>Z@6Syx zCMZD2w1Mzt=@7`R8zJ;0xeK$Q2Xr2D0RuN1@D8yB>rh*WbP&O-8xx@aSw9H;FdKHN z`N52*esH2^P0TUe@(s>-XU+!(wsfl>?Upo{r%0{_4q_ur{h5tnG;8TQz=ke5&$<-fv51!bn3o}k_borL z@f$v~9!Gg}QyYd3k_otLR}iic&%)e2z5+96DmL3rL%Yx^=pOBhUdt9>cilRKh(yfT zk%mTQDVXmQg$?so;27t{`0d3Ktjt`F-M9>$WtZa|?M?XaLW!>Rla$=Uz|2EbrOq>M{+hdjY3(o8q(X{%A3=i?yV5vyv147>rcM@E$EZ z_2fGX_*#!7z8G7>*iQDs10O0hIDa)lG?u`cgdu9dRF(crMp%{8*xq_9h87#l526eks z!8Jo22H((z&`LY-ygm+`xi1V7O#tiJ!(qj%L2#f&2Zj$D3SqaLV21Pr*mBtib{dQV zmtqH4=Vk&KjRW9sv;us{QHG6*njo&xfj@IKp=I=662GdML?z^s*s+OZ@YGTwOhlw4 z_$b+yzLFgJa!mBZ=wV}6SD!@w^Dy@Hksj;O`6Wrc{8lnd$(}WjPhgujx3e+s%6O_u zUFay5!{u&|n56s+^S*w8nJCKOP7fp8oHi2mL^DuFa{>n0I3ot@qrxS1tgAJ~>!uTM zP54SYIX@2fn@3~gk`NrKG6`>#xuD(=XH@nZgG?qE*WBBQaZ=Iv=*LFP$eDt+Q*3ce zi7s~S*TWHZ6dzBXiE%A+@a!yiv=R$>oStSDX%WWio&~b+TgS5H0ZZA0i%x9zyKu?- z*|$ZubJNMA(t9N2#ye8y{E$Q*yF!#E+#)|+6yR`=A&iN&gkxo9@b~Wkh#Mz(HJ{%g zGUMvV*eeGK{g*-PtD{KHC|fcpsM%v^Yo3Jsp3keI~?Yu~~kV@iRn@$#6Cy~F&No3^4 z4MZcdKS}*LNA&4Vd84mGr6j=cv?O$xC0lmLgw1JoX4CpDVPP*9u#Phstb2SjOG$5M z&Ox0_vg-xApMRIVw|&5ty542F4)@uqh{tS)(MR^>w=DWE66U*XJ@j)`$BG?)*!4SC zn7Fxx`NBme3^`1*p8>8qVu7^|L-E>cZH!p>k;P9Iux_fQY%e*$VxHb)--2Y(Q$`b& z8{!jdhD~KFJFVG+)!!v|EnZ3-S-V7qKajM-Wyx=OU&-}z z``qy57SXi^dEzQ`%-5tQ6H~Qn^6*9nkvsg4n7$YQV_S`(z1RgVoE{Hi>JK+nm%x*# zFqnFFC*)T|!>jc>U`72#n3A~(sI)tGCFh;s242^h@N&+Zr-GB9{DhHz4)z zpNaH`OcAAO#dxHiKkoshMv@PYpGl%mFJx|kCG5!C3(Wh{SvKoP2OF@UmmOT8h%0Ux zBR%1OMJEWBEuVzbPR+!J_CB~sR*YWM1#4yv!>hu-uWvIJ8=w2(j7hVw|5R^u3?eu$ z-wMAT8jQQXD`EBZ-)!G9DJ=9<#z0GB96!PwmrSw5zG+&x*IEW&o&3w5kCeh7P{Ubs zba7{$&|x^i5{Hkq#pS~J$W`bB-&di7U2@tubF4Z(cq@+?_0qV|LLN=0$e~u}8@6l5 zJGSTQ1E#dCmQ6QGVDUqiu+fEKxUM0A_mtM5%a^dNW{`hH5~e zhAj9lkp<;<0|nZv9r)^wgl|Q@Fn+WjJW(1Co}URcs|<(hIZn_%3gFCmUsxLx0HZ_; zq2zd=kPn>%bKEAuu6QrFc4z{$4jT`}sbX-S=O&bw+QPNAL16tr3GVNEPm-=)Czqm+ zkoKZP^35%f4EQTUnCdQ3hPzCo=A}Bxyx{)qo4+lSbK1$ytu13VXY1J5eI3mHOE0^T z-X9~f2VzXX5R`P9V!W0)-Ym1mUMFWPE%w0X+a4GjBFu?dB4h<4v3VFOJh41QglIJk7dM*X2|-J^vS=uVCYhq5oe>U-*F;}6X*{p;lARuXg^gHt zkX2U2GbNq*EThblZ5nIGE@cj2-WT3VCj6TyDIetOK2~F<=;x(JBB^g&ByIFc)ZLpa zYA{(L3f33=vLnw)p6uJz&q!waw-2*?>ys?8=>|Jo@Qi&< z>tYK-x>@qmXY6a`V?nFG%Cv`^W_kxD>|A{n)5$x)AZ05X5#qrJGhi--=OtHu&6mt* zBaLO_YD7PM+=+B|92xFdN5uUulcfuvkofN$dJEq{DJAx!W(3G`MXcdxz~LgPw$vM+;@hqO2kh{Znrwk?%({ zMfWv~UX5n132|)F;uO}mBAZ>yEnsSP2biaPIa91HW3SfKu%&&KY@u%r+lFQA4M}73 zXJj%dnLKu_u9RI4$Y*<>?qYqL_pm!zne4`ez3h%+4SU;D%^s{MW5*62Vq2bNvlrWT zvqdUX*`5wx=Ag5d9S%!lCMrkRnc{s+_GcD*=^w-1CCy|fE(WnVv!Yq;?QE9qQNtWh z2rR=*hgjE=MQpLnSXLbF%0_!EVR8=TY^hrzGQFvUNOVDKbV!h45mq|;=m3qBqud-e1jH_E7Qi1 z3QhERWPxc%2jh{mnmBpVTh{jXBs18)k6pgIhfNBvV85_~wS-?`mo}VZyIU&RmwlE zUJ2L#9EFVu6>#p>F$m!mFyO>77-3KWW*Q|h(76ca6(qofyp^E3ZYmVa8V+^896?jc zRA6SSg4a}OXntHqtS-zT)5cW^o#o|{)SM2<{;&u(G(4KMKibSP4qRmYE;aAl1>>IM(SzpX}ivrrXxNo%8rhbET%{>7dzQoya7pD~SX zSJ;meXW7iIwM_Z%I97J7Px4}WwdA6sDpM$^mxvoqOJ4LTOHS*CigM~LL??W*M1ie( z4NG~IWUBmjN&GrFcDiwo3ZH;dvh=d=$c1gYRm#!+I^-_)LoZUNnRa2s|lS zvT>y3`QD($xnDJipTH(I=*=Q?YfqB(^GZpSawm~CeM$muy&@kMYeU4mesJcDCRC*k z0E2n*Aa}MS_D@=T7lPsp^!A&9oFX#hg!q&uz1A+@VvDg zzTQuU)HjLHZn_KF({sUmSq{j@l|${M6JXI&1E0!I!|Z7YGOA}`pY0{s5#9{Su`Q69 zc@pfjYrw6b0=mr$pt(ZGTjfN8R%Hl`z3&Hi&<>)`=)%g?&q$VQ3`xBGLbP{lro>?B z0w$YN$x2$^uzSq|F#XS9)cE3zN47^k%M`SsWIQ(` z2^F+s@$m4SSoC!p>UjjCxy2l;it@yr+U{s^&JNqhXkl}b6uzB*ndxR7WY1q_u(5Yn zGoRQXmeIGGC2ewGIvD{>ZqF>XD0Ct7e7K&ScbUk{Bum)mGjkYeNMl~-4l(7>3+$P= zgS|+)&dl8(FoWYCSlCKgG*;+h%ltmDcXcmW&$n*2$5ss!#%f}!pDC7#tnkKID|AE= zPT%8%y;EKA#us(GI{z<|3|7RFx4rD?O?h1YP7aOde`nIKp0ej3lt7*%zGl-Wx#BF|vU zkq6M@ehVH9X#tZpCqYS6CTQsyaIGX(I4{hBn{Hwdfi`f3n`Gds9I`1ZMP$G1mE@8| z89SQ#nf-aDft_lj@WZxg_-EV-{8toq221^}O!A2HsV#8dG*+5sqPK@)C zgejLuzWY3rD3?S@te-VVB)|KMF5XZVDQ-*mspGy4Go8kZvJbA1v_;5D3LM0u z<%^#+Slr1I{Yr|IjDKOrdbM{;tQI=47fl_KBd6y|9xrK>+}(ukW3%rx ztk9Uy7+I1xV#d^sjXBqJB^_>iC09Srl{jr!E7_qO;&J$qf#|T3qG&*jvI{aKOW>f%zlJgN_(>aot@e6qMnT z4K}#&sXs;yU59G=vFMqehQ~58QTbB=E?jv4Wy;Dh^;0!^WY%NJz%xj-FJS7uYuK>y z3a;955kEUN<9hd6^j1BETi(>8@|e^3JDOwR<2sC9RfD(cN^pPZ0ZgzsfbqgLhTn2< z%*5ST%p$RER2b^$F2LEM*{BpX2KU_ZK$~1UJn&T$4R$JFqxCD6F{Og_9!g+6eU2>h z>p#h_DcO>{2}32jPY!FW+os}iELcNyUa3UXGOSM2sgI(YD;|nchI|oqcXx~Q3oS{F zx*?gdmXK8LLxwOn^0dg4{H)R?bNgoz+k|Rz{pxA5!l{>3-~C9sr>lVHRXt%(YX&|A zj-W71c;9X#aL#r#RJ4u<-WCK&;r_58b36p8iXb@J2++?Et}j)9=zu=5?A%9^aI1;r zTQrkP;W=dd&)sCq&O~x~k_lN?K8kd77?OkTS48jEbcsxS6Gc{e^`gt4pNiHkGa_>i z?;>Z8SCgFVvt;nOc4A+4i#&bMO*CB<;J`m!nC9dJwii4>+QAp5PMQZviVGn4=X#h) zW1uG_8J3OM3*+>2z|4Fv1jX%v4cB+Vz6r5#MSX+7OkD$!YnMS+kT z&AA$%A+Zl#$gJ{Qk?f84l9wjY%qg*(^?%`xks5yJdv`N-*T-OwR0c*(EJI7dmm+)T zE@o>zLVtrd=z8xNZk>G}*Q!6qt8Z>&$fU;@A9x4vO}mQF)`n3p>u~6yN>r%LL9gxv zTrxWnUDCKH7Gpi+K{myuZo@j_JPWdic`F0JNV3STN zHdF}+^Bgj0LOdC>ERxKO9YuB;4JA=N-J)#gfudmrzdR%v2O6h2rby)fQ)8c&Td<&V zOJ?EZ#QHk{dl93{VwFcQpCsT$B+ck-FL@DqPAYq{frt9kp*u&-OMF!-rpp_srJlkZY-OO zCG7C>MmEdlIMWz+ilu!kXH#~UFe|g&?8Lf2HsiY<%j>F_v@NxfgtbIUen)3W=H|Tj zh*iHPS`-;dG{>}%f2KmmiJ~r46}Uo@zAuchjRLENBrqJB0}5t^aH{kGoJ}nU=iz1G zIk6VbJgtB(^=i0lbquOf_Q5Fo9QYcR2tVF$18u`OFy_1~v^FY2%7aSMe$JJ=aC8;r zKl&}H^bKJ5K0RblPUxZ6J&HA{YjBJ7HXNv$hy@ktLWg=5o|=_~q77O2_eMG%8k377 z_GjYLfNUJwm5iH|hZL)Kv2THYUJ|Gs2j@?W!+A0IJxs+|4% zkO!9LSD_rg`nSc97<3hRn zlI-}Qk|js3OJ*GPZhUU6L@NA#i;hf7AaTm!B*bSK`RW%*JdEQ=@w+badA$_$s4Ia_ z%>Wpur35vZL*davZMdbT267LTz^>vYao+ToROeO`?}LlTk~L#UeVVk${7kMW`P>kR zwa*dB#-Udv)TfLsNv~uz`R5t6eZdZDbTZBFRyg)6#qBadI6pNG?PNA$r*sxZZ%oCa zDT(-c&K^{`IS(DbuE+81%Wz53Oia8HfjynRXfk^}){fhX(dvL_d`$3TgAOTrX9H5z zu`pupDOmJ10Zbnk!&>)af^O6dL)Iu#+eHItPPa7;@f$_0zl@_x{{_-Vvi|gSjxV+H z8cRK94yQYe%xLi3{a4s#?v8*7`e&}SDP!a@CTmc z1Jo1ezlX_3by+gE*Im?mzuTkMNRO!AI!j*qyeFH!4uouL9WWka0{v{o@K0n5TX$JN z^a6QsS#yDm-Mg36?lUK2CS4b8Sa3@eueqJD<46|0|3_9A%7DCTF&S>%A{w;wqU8Rk zv24ADE$gTjW_^)9yYNnr&1|)2)hAup(y;}?%=?NhtRISM#sOILeF;|YU4w7^Lh$O; z6?iB$63eEC;gj`S@ZGz4I6(OP)WuyWUyzKx8Ha?uUkW~$y#tr$MWXer(b$tY9}n7& z#s`n=@cg=EIMj+_+B|=hwy?+g<5Em2I>0@Eb`fl7CRnXT9GdHiu(=_Nn-9U-lZvze z22-6uV!EZlg#Nf^MEx&1(RW*{>A$?m^nOAB{dYZxras<5e|oH>b(Vh z>1aZSjaQ-S9g5U4>@Czx4uk!fQjpd9Qskf3&*QqzKoW3UnAHd9W7_guoUQW-2Q~NO z258QUW$gK?D;C^;qX~CXQRPu~;8hsuI~?~x9jn#Jl+^XRD9ZeKg!H@iinwi)fz1W7aC_Hda{ie#oDSE6=~pbl z`icn@CNvS$rdXnTj1%@ z>eyE(=>M(hET%?==`?ppJYOnH5>JIoO26LpXx&~!ZpOSOk<-*+#t}u(SgHZnXIevl zqlvx%+dee-_Q>oX+sq~QL z9}qvzfRPg{=!M{Mw02A^HR{NxS)Oa@{qsrm*rr&j>#>;HT5q8f-$g2WJa+!oKc=h-n-AJjexg$0|glfF}} zl7=Ox8|!*4L`_~1BIA@cw=J3~SRUnt^E;*^b(xKV2?2dl#Ivcy8W={%L6u`Cza(Mo6L^itznX1=Oo{U3?&{<-w}IzFEH6*4l@#a3A-Q# zO52dw*Ga>y4|32xWj7f|HO@%>U{DGeLnA#2{&=k=9Wy24-Ef~JsDr{ ztN0oAKk^vUr5|J8jD0vHF$HxxO)%M65sPbou~8L+CCAPVhg3~@TJX=8-nkV|uMEtl z&t}I_xvGt{r*$@Mw%I_{HUEeTpeB?7|ca`6NQYI(9x-w z&Q-LdxvFXePkRx?hi*^d3X&r3b*q>ge%;IMKF9N`r)P0B<>5jHqAbsAzJ{4ACHTHJ z1CO2$!bJhq?C+2LBv^YYcpNN(8~P>ib$TU?=|XUL`U*aN)ui4+KG;x7oof5_fP?Qz zsEtd8MyF`FwJ`uHGB-lNdx5PRtOE}>8<7Qz->|N|q4;6NL3~lui2K5hU~0cQ>^{_l zm%jk zno*#=PAsq?Y#}bi0~AeG!NJlPSl5vaWXLY4H3@(}3*6v-?sL-DahQB*R3S_=mrb@B zjaSY0;wp9xC)<9++tZY}ZQek>@}x8GeLjj`^Iyz8#)a{jjcfUy+;#l;(qP`%G>@OG zSIM%x|ArTa_wdXg zKe#6PLc&eM7LMLplp!GWSbXH(JwM?$1yKffJi&i`7_rj^va`g~e8`>l6Qv1;-n;t{V zfMW2!f1P+gOUIbS>U^okcz(UgpAWk-gX_OsDR^+B`9kXqzN$Wthm~aWV4H0IWLg?; zIlO~AnTPVS5TO%!p&6HDhPLSQ=@Mj~>1wc3BG`weK2`5CrP?4&EL!rYEmB z(H9OgX|VPTS~Y4a-I^0f7oD9-xBE?`u0D2D<+(OhY`X=;m(yWU!ZMg^J496TO$DVI zbFs=vhBsc+;K@DKynWzuUTYG?*R9Xu;z#*BH=&qo@88SwXXo&Gi#$HXXfrQzUd=P> zoj5m=<78zO`WMt=asDW3?3_a9ebj04x~krEs7Iyg>-v68|FwPPhb+Hv*E1h^VDTNUzT-Zh z^QVb>^cC^L-t)P8bPAfb8L$y^6~S3y5DX>4eTjB-r$sV7*?gF$=GW2Bn@>>GuArs| zYN*W3Dk?YTBweX>oKDOvqZi)f(-nU4)U?`67zS=Zef>i+DA__}CZ12EzJ6t1nT42E zsK$Tm&*UKkc5=H5G5l~~Hvi_jhp(KG%N0Cy@xmO2_LueLl5`xJV_c4IGxH(J%pi@{R-LDsaW`m)nu%Ea+Ed*A#94g4lZYP&jSzde zDvJlY>579MzNSYsPSRB)uh7feB58U41{$8HM2~JiE6m+W-0R#texq(4zcOPx{}6GN zKMj^^${J?al&fRfG_z|^la~3wCN~xJCPfR?CO>EOCY5~IrkI)@zAveh%M>u4zPE%w z4)Ejt#&@wr&JTaDza-i#8qVrq4@kS_eMl*Zkxr6Z5m z(;2lVXz_n#^s#v^-5Qoa$Gn+AUw3NLPm9vPUaScrS<F_VIcALg7@ND6ovYQJ(LYw4&?g z4Y4x2cf4cSV zD^R(g1e=ty3qC(&%9Jb+?4mw2#-h|%b)(U=ROX49>y|nQ6MS0ljrv`Uu zGwpeDj~XkQiH(Hca6OBl?R7$j#fmfZ^zwn?zP;nbzr#f0U%!WoM?{Pgd)!bI54$TR zE|UI6wVuRLv#NUfCU_6^w)CcE^ON8=#}kdwAcSQbs14XyKGA6Syh2W zr;!M-O2lOBYAyMEU*6)%^UBo5YCa~c*_zgZdLLWyCunZOe+v83tuz2 zJD)^dD>NW*LL#J%JPct|-obvMgTX_hN8hv$rfO7&#$HjS?7B2ft^N#y1|NV8x3b}@ zfd{x2rjx>Z|B0>!D&dt`2k_y&S`1w*!*}%Q@x-5+JR!|P=sfb|%dd{-*#a9HUajE4 zQO^APrdd2wd9aXW7x7DGgSod1+W9EhNXk?&+njNdzV5* zFP*I5D8BRKCw(KZF{|}DXyuJoS~OEn9RGHdc+3uCan^oo@v8kc;(0o+=)_r{=)2y7 z)ZT3j-9I9oMlhjAbL~`W``-tM)sp8a59jmeQ)2nvh0)wYE0Tvdp5fzlrJ4fX$T!8f z3~V|$yI<42BhpQeJr$asgni-aHs83kPY<6Pe~q`^xyoN&ZQy(UM)SI%4qR$WI%+&u z#S`InXx8@Jqp*K5JpO7$UmuU8h5oTr)jFO!L?%;LDxdO{C zj*eIwN3W>Oq-BnZ)XJw2?mqN{O08s~7okBW-y4Vbw>?6aTPA#U&c?O8tl^V= zqIvr8WPv+Y#C=**`Rez%+&epmFR4r9r^iKeQ!<6;)Z6eM@9p_UArsN|?h0zWIfyFH zu8CY^OUc{mE1-4Cf8e`5A6_iD4V{+-Hp`_>s9PdM118B)p9U5BT}z4nDeHv|i7kRQ zF&mD?UnI-Vyb&d|g|ghQf0^NT8N5S|;IgXYIQ)?ekJ7i|{HqS1;p5JIdW`ts0i(Iy z^)-YtH~7qW+3D2m~R$1AyKT^_GDn83X{Rk=~)7~DKIUeF}& zNwgoh3BL9xp!aG#?YD9#<*I2k_HG27Dc(fazS=@@PBQJhExe!G576JjS;E~mnTCXK zrD+D{)LwXgMr_*&2ebRg;?-)fLno1JI4gr|8**@}aORoR>%+Hx6ga@yTlkkN8@cj^ zXf8D=mVXq^K=*PZxO?$x?mlECKeA{Me--b|-=1^gxoyiy~eg zoyNlA%E*oUJ>*+J3Q=pff@%2jX)pdLol6 zlf|#+)$saZz(u7dC^AgK0@F)qH1`J{KV-;%)4^PO#ZxrxH-v9T13sa5D6f`t;xy(r z-oK&4y{soN!* z@oyR}$g86x44P=nP%E+hyXoSA**4;zbKS+82RMp%?tV^P*GP#YW>wMN(+RXJlTiKch(*E+ZwOUP+biVL2uRifb zW2Bp`qd)Ox$#>qr_%mO*=dV&8QSkEnDg!j=E5uaCEjk?xbaieY)zWHL081tPp zMD@W{LvK1IVF^8PVhVLD_o07m1L!W3ZB(Z)nf^#hr@OZ9pcN)zbl(FXT5{HnChF_Z zFE1Gc)`h{mO%4znRYoF5&v$eFwghi}RpP&OhjF8Gv-pgq{(Nt_AHOfPh)>Ph%+EOP z;CcoLT(4p)_nfm%;M#5F$q$$C?IF|opb~-A6x)jti}#_CeF~0Tmn$(Y}Fy==w)eSm;3sZ< z?mZv1>5ZVFJmCQ!g}%q?Dqh-E%HQ6JF&Hea!WCZ>@&0%=HHwOU$pv2P5Hk@&91lXe0Cob6a-1$woxQl>_m*G43nr#ead2^ zlrhD4GJdp}jb6baSTt?}YSzWzpj$=QH0CINnp}_R&u8K7$W17(u7JNIhhRhS7M!Ja z2=ARu#cO*OV$BU+FQ8U=T`o!ww2@a zF1~lv3!!`AHFu7CEb#CcPi{NOH$*k^es(3onIx5GrO)Hi(l$I?;NzL>>czpWQAjSQ zu%euPF!gx?qKQuGn(?LKi&C) z2gCT=uNDF)#hUv))Z>NkW%=D_Pw-D&8+!h|iMm6&@cZi)tiBP2D&N(y$pu+M)kh}f zs)}a=kFxUj>SRe+GkJBfjKqZ$lPB)RBrvGMUB^*_6r{_L$vYO2(j7~PcIpz*z9}No zdpKP(MrR%yGhr|$uChXzLqnKE$(>ECTgz5vxuCH@JXXHR$AZiN^nNxM^G5z*>+S!t zrF#aV@vK7TUa^{G?EfaYJ57rGTKFGn5O^@PEn6UAUL-_(+(jKWU!(4iKGOH^@6p_i zKJ-7cRrK=NV^n4SM|v*ZLVRU{p?FN8ide-@O)M+@fGWnlq|UDGbjsQzbaLA<+Npnp z4v#OPITyld;pe?jcs>_vzh1$nm#0x{Qz>rlrzB`7k-Sv5lCLp5$E!WB3GDM0F1!9Z zuNun{ ziBj6y_^&xhRAhca zbV7v8I%t0~0|v+j!XO)0IA?T6MW%Vu`5q=&g`Ks}g6D;`3aU$*( z;lPL76rTH6U3u4uQJg&=$GZ;?!?@I*oYjGu(|* zlftnso#2*#2Uy9fR8jv7LlSx;jRbxzCE7PWlh>Ww;oH77uyk-bSjTUNTUIGxuvN%J z%jd&k;aOHUHyCpLBH*V;3kDU7V8`hfzcGO$d!*t;U<#talwu!*(H) zmG{Eh0psv;`$uN=ClD`sykp~T%i)fLS43yc--*_G8o*RD)kbOUVu9ThMaAWDw0qwP zs;22DupB z6>?)0R64MjMogPe!z-7;4mU3xd`s{ve+|dyfqO8wT$9IIZ{lNbMf22~ncVWf5`IVa z2#=~c&ovI7;|rG7@SfHhuF!Lk8|<#*cJ~_jnd620Mr1UfE&ceF ztF>q}EFQO*{9z|ucC&8}e8})GDd4Mj3AzW~hmP=1pg89qtZeOr&J{W|a-lj^{U$}X zxwb+2up9_HQVo}_t3iE-FPu?ICgS-0aSc5CPt7Ti*V*CKlH*iJbB0PY|!TOta`2?bKDazlCro?)|KkP>XK1VS~Ln)=@f(a&@i~1oeWQ6 zbHV4{7#LuX1(olnf!1ML=uY;6`n}POZ`Rj~tP^60^N)vQ=M;IEwaryB^-Ug|ZfJ^; ze$A(zGz}DCgAYL1A1bEz^eV3sA^3SOg|Awy_92V+s{fm z`PXQg-u@3xg^i>~1Rl*q>n?h7K@a`3{1){Ox69imlf#g%(&PgZoO$Y(AfCA@gjcu4 zbESj=p4gbh{pKg}J@;ez*6+#ueu$vA1eI|&rA*#ov7GnDc=H1pl;b2XzNOQV53l)% zp1;rI<;}4ezH~c!pB;_*2aZT0e<+fr!w-^+FON)}wj)PBFjb16;kAa*g8QJZkqa9yWale`7s>XJ3}%+fH?2qL6Jg zA2<)Ss%5Zn@F&UGz8|7VhS{Qrz6B)tO+INY7!EZyM4qD>br?|VS5AM7TIp8rH2 zFps1>I7Z%oSwwc%Hj;;-TZxpsIhc44go{h^pe}t2%xQcAN`L3j%+Lk&>#$5gb2Fx= zg`OZUx`4+2x1T=#_kv0lbkm*b4RpI&5#6f0kLtG-(B`wH^xwJ~I?lC_eh$l{%gfWK z@kl})uB%dSA$weS^oqz+@bp^`Y$1;hUYDF_9PPFW;EAFM{8^$O&wj9!Ptb|v{u*)I z#5$3$+8xOsXeM*Np&5K%b_W0XG@XBV9L1gAFBf?6%ebWmB6-TnLKeggET<}i_O_v56>AN!V}YAk>D z+l@~N6j+LKCfvNrn75vD;G2B~j=qW^-)yYHvz-+91J~bZKIaKa=B48%6@nf%!|{;6 zCf?M3$h6OWmrU0xCLJ<8B;Fig!Q;6w9wQ)elF)}xkOn`D8o(v`5G>e!7;;$-XgrO9 z>#K@jrf~uMt1JekXXZk_MIRo=T_dx%$iViseZ*15)uVXDVTp0TY>%7Qq)B9FIdRpk zBtFHP$lC43@MpQd;8_uNpmAwX`?wxtXI=qa*SYkoXFB!Xvybk}Or=@xt?8+v88owh zHdV=Lr+&Ae)376#XvB(A>aaGQ4pzygedqVk`2jifL_-pFShb$oWiO)j!REAj`3qRE z&Hwxi``W=#p0>W!lFu-`>WdXWAQFLD3_;b_8P zuuqQ!IX_$2uBHR~oTfpt;%3NyFblM<1GL$`B~QJdkbO;#u-k7h`MdZ835+~U0*;x& z@|FV-THFfTexHIF3a#MUc@EObZo+Y873$t%K-X7~rXzG#QUBr5bglJH`ph_zE_4W> zIW}wPyLr3mdVz`P5Osj=IG;jC&t6P_X!uem&#Bbp<1`vX=F$0cN6;JT@-*GP0ESz7 zfr@26_;p7G3^TGMslTPMD^>WN{@leVD=D6BY{b2O8FJl)CVWGV4Sysj=El!Q^U;$g z^Z&N_@c+^$^O?<4_>DvlftP5(qsoo=dV4MYZ<7{}OBZym)c2^})QZ|N2k~sd4xDA_ zgS*d-#v>vxTqZvr2j0ES@@j8O{wOMuoi(wd_n#72_mA^zT6-nSn%l>s`st(D`xSU6 zeJ|!kM`1>2B;IyN$Ir7khFHmQDdBt3nZo(4VKCR-r_E0dQW84Dbb0s>Ev~XngV*F} z^AWFfdHZHlzOh4-pH_H;ub#$X)HqwrE7!qFx(0uEhht*jWZc-p+3xQ}XT6^0UjJ|A-n`#SaE%hO-trl)PAq`#m||uaFcJ2g8QjrBL783F{x-1^?m(I8Gv=<9QgUe~*Pd z(MdoHtDx;-1bS{)aM)56}GC+lHY__G!zb)yIomN!+s~I(`bff!%#8ltgg+h-79n-B# z{n3qviAK>CPsdQzivwv{=S6_4iID6laMd-kVX1Z$gtXYe_pz!Xl||CzX82`E{WcXG zkm-vm@&))d_9Ql3D8{O#$MNLyE9kfX9?FG&!}ZnA@%WMhc;~-jEH|phmhjj3szcbn zExL&gY?;7tkw+V9ifQ@rs2q`mZSy>E_uD!aa8m|votlqFlosF+39^yldSuR@@nnH& z2g7#y_BhmwMLkQ)#W8wW*0X1@aX>D~nWZrz8r{Bg{VkH>i_&-%)5CtpP4hy`F`*Fx}1?;#{!hOUlNVHRz-&j0@3jH zQ)qj2G`h2WEsC4`6pzSuvNGz=#J*ijZ{LSUL4Sb&M>EI+TB3PEI6UuY~x-Thqd;cc8z?iy~+6*F$hbY0G?#GF&9v*EriGD(h%xL68RlfB(=FKq*aEEbc)vtdN;X>>i4A5X0^q{qqPXm zvDSunYJQUEFAL~YofInn)Qp}GF~e#tR&dGFiBNuXDc*f~Jl*^!k*?0;vybA04lX%I zkI3wz8L3h9bnAT@tUZS9nly=(gex+i=`w8cVNI41Y{BeKFJYopwhUX(VG|u@vFDsQ zyZ?m$A9_58*=uXCjoZ|i{Xb2%!c(86_e^9>$*<|W{&Z?t974Oeoz%^N&rK0k>KXDM zk?S-gi)l2t*K!k{HMd1oXO1Fw9e+et>_P$IKFDBB6j~u#f=b8LqtxF`Xxi~xXtGWv zda>#%^14}xdX%ptgA+H_6N70z4(FoW2 zqbKvvphfTYqk9*Qq1A8wQO04OZKm$UKT8y&oUYgCMLEyLQ&Q%#-+V;v``Xdzi52LU z*#}fuH;eN!U&5shxpLgw1>BbF3fy!n5zcp02MTv?M#~nfaMjaHxiV8@?xCd&XQ8(h z-5Qt)Pj3l=W$yDJFKkD@wy#7}&J{tc?dDKE?KfGTyNcFT9im6e&(V>|JL%=40pz2? z5wiVWG-&=UXDq3z(<^8b%iweN04(q9i%O! zjpRinKvnk%NYVX1>{}%iuH8448n)}uML$Y$%ehH{%5Z7eaGK}m1#Y7=U+DS8C6T}qUZmKvIDQQS==NW76>eu9cvI`s{?zg5C97!luN`X9qNAG3&V#S?=Lcy7uoksyD5dHr;EX5jz+yPA#D~ z$EVWi8iCaEpg6r`J)YE=M#DX#B501N6q^4*43#-aBGn6)NV6^!t&ht=cR@CqTVI59 zXQrYzXHKJ|LBZ(l&L!wujsqHgx)_}u)4YU;&g6fA4VgDapG$utG z4URnu-8&V@%z9UFz^@5PS1R!xSWWbbt>CfbxoFekHss@U7xlsbWH4bRqI>hvtz0S2 z*Uy9temRZf!R+XfW-k)6lIM08F68gt7-CF`r2wF>$44Mz1fHE1Vp0GdrFaY^A&Xbw-+j3A%V<2KEo(U z1GLY2KFSgAhMSVg!CsHO@b5xJ1kCtcJf*2z40)6t~PS~XN*KN z`{9BRH#j_`3#Y`$!kf}b0{w(AqHsfuDyhiRJC$N|VzGdFJ15Y6>^}Wb@P~%={Goq7 zeB;mDVOmrq&5TNPm|e+yW-sBu60Mi9(c3H7e%?o5knYaDJGrtp%T-Li(VZQhzn01U zx0*f5U%`wcSF`&sH#3L9O)OG#G3yA>V15Vw(7uWv^qGY;vkp{aXO%RVi=Q&PmGX;z zTvEt0y-a9c+jnwzw4m+&&dwWFP7z%XQ6*^9&pd=!WW)~lX7EQ zGV)sks%kYsfpk9vnd-=*_8VL$84FfORFk#&Hq?GqDY<^H4pt}aLsL)hLmmsZL6?GA zRC>=UT9VmIy5cHevx_e(=-r8Y`F>34pO>(u5cY)h7H_*tmDdt48A*a^KUQYLQkyZx?Og2_Z|juulr-T ztC!AkhD+l)?P)=rx-RdznYe^=`LvFUX*$4d`n`ucwcm+5-#?wRzA4Skc{Gg9dUd0D z<_}Q-$mDnJC(-NzA0+wG0im!ND9SJpOdXb@?~xL1*n5;rJTeJNiE@_BJNV^AMJ`pm zbdg?p>Q3!VC(x3&OQ@Pj88u$eOI>*W?y8BeXwt7I)Y-L5F8Nvt+Um3@07!TROD(Z8FX(M8uj(>0qXu%R$5W^r^T z>-;*4*_t-p zFlu|9frbo=Q3vZpO4q-ln;i{kpGrL6kC}(oz0g5|>#O*@d?uRM+kn>e_Mnlm{BL{Q zFEr-GYxFtsHuBZ4LAp93oJf})cVuiBcdGt8_f#&K3p=u#;}f3T>uLk;?mSO!iFXpW z7hdMB1m$pRKAz@cGZt}!^CxqPRE5hmFyPc$%(#>HR5tLn7&4D{#w>wn!{0VE zVR_4DvXq0W%63vtFA0Qu|FCXO+>RVIR8Upg&#Kewn7ectxN8 z8N+tzOk!gF@(fQI%X)I|(EPGfwCvG8GA!Xl3KmrhllI2}P5qVvnqGQJ%qal7iYku-Ma?-6v3fhrq@U}a$uyjPeul$hu%@hRw zC8N*mG;&LrfxIWp1v4gR61N-YNWGj4njznTjK9iq-6K-mS-Wyn`^6cxUk^im556Gl zRg1XPNryR~+ymU-yrUex7Qtm+O6M-;^8VYAD9$kK6xXDlz{MAoaE4xW+~VPKF5D`b z+jVd~w|`i`al;NATd|Qd^t9v(ss_l}(+0q|zQuIrjB~6w|a5Vec;V(f7ER zmZUh+XX~3uYg|oMF*h-^fb**0J%1KCTu071Zs(F+-0FwsoT}(F zj+-ROX$NCmTf<#x~Lo1IjFRkt;j@c^^xQ^JT#n6CLcwo=rI^(fh}`3_d>`T|SVWIl_XeWT2>-j88PntUdA@-XG% zf6*A3r&MfDA&q&MOlul0(#-8u^iyaxJvK9sdZa{CE2AKK{b?js3CN^PXA0<@m6xb{ zdl{W{;SPn4ZPb2aCq4hJjRt>gr>RjNsLKU0rtwyh{oJU+)I(I+3~4nMDyGRM;Tg=P z+lbv9%kT9DwOE|iWHv8SnziYTWmzg8sLr~F)Worl;-n%foElEEw#!jlMOUmd*#&k^ zyAM4^XQ4&>-ul19A|%p7QFKEIatW+N^$ph$^I*u^;T7V&n8-~qh>X9Fpd_)M=-%8x zv}{~HvOH>p(*9;Z@9h)eU{fy)_LzW-9+kkx_RDbgq{Hz3t4cUgLk6jyeF;Yi^eW}nX|!^`eztI(PB%Gpy_9=O zC};hynX}96;ue2?%0=$4=Hhy<;oX;IvTqLRl1*%iW{v)dY>ILWJ5nFW-kjw-6do?@Z3Se* zw{+Rb6H=^a-&;EGz&(1or-@DuYNCl1lz!N7h6avVMOP*RdOdLibrTDx3AxAUzkDYe zwRbAzH0RQ}wr6O;+H%^{TS0r)Wz&<*`{{#`J@nzabF{dT&yuHA(;IQO=m*VPRIa&= zPG0wiu0nF`W~ma>-m1#RT-IS951KN_dl+38Eo7$0t(Z}(IUC?+vjt64n1RB0w%+O) zZQPejGnIGK4b?O0hLsZZ+skS)YyEBfam5)p>$(-vyB3R99mq#8yc`u|<{}-5NHnE1 z0$uM4L9K5Sk>~t$-n+OPDJ!o)X>ZifhS71T?v4Wz*}DPVjqpUzP4v-cGc)ifaX)#H z_mI?Vkfc){=8$ch>PTLa54q}Thv}6<(%dX0)k~fbm#yVw-f9cDZ(Kgy87+bGg`Vi^ zkwoMg4>*m=JZ@TkH`kLTVI`tG)=EylpHu(#nM-{7k+c0Z!~w}MR!3g^<&NGGwR*RA zl>27a$Gbcpb1uB2Y+>7c&gXG0+Kb1d+7D7>U_GPltCzB7ofswv&SvU-4{aw7)=*l`eu6Ug`}I|3G`5h%Wu~#Tyb$(ut{sDTluA1CyD;5+8f%_Q^#jh) z0-M>?t@Z{f^-d)tC;ua&7cueN`d%1w=N}YbHVch@utaZ)O5hgRF(m110*Q;MAzzIi zll<#)^wYYX^j-29DslJ}owz5E4hG?z#{wKxm=%};5s})(qcR6O+q{XyO z&Su#fCagSlHZwY)#h$5avG6)g_I$lEGn*p89%{azk*(M0TDcfH?#X)ket{|tlsiM@ zlurvv!=2!I*$~+3Q3j(6cwbYRHCpk&4xPNLiz55J!slN*;kJhoXp!nv^vq5a&C-a0 zk;!V{st^O15pT0$h*QU{-iSB&1Wjb}fp-lrCO=;&mjwW84O#66FePe9DFz=5N zEH9FQN;eCHv3vjGdHdo?3B637?N1{+A{vP0#OGwpiDq)Rw3V!SeU+GPt0mp^2eJA3 zh#XL=BBXXd8SWcPB)neX3uB+)56|Vua&8Ry5jP@meXRu6&+IL7uF>Jf*(Y*qq+WBy4H8zX+*Pd9H%zd~ zIwoRu>VTM)OaCZ$UQOJ}v_!(n`_B)q(`|rTeD@_cudIp_6{+BGNHC}1ZpI~T>_fx< zje!3OHqp3N4VGnkj%8jgW5th}*-ptemKk5eEcetfpR0G6g<(D0H>HhD^lD)xZ||@e zOTvsFl_&4?B%TQpv;rvYb`1PICd*2p%t70nM zmrzAsyw@Yz(ktOnpE{V3rihOHRpog-5-3Az94h}RgBC2cMAO3dp&PpK=(Z|GY7;w< zP5M}FP=5j!@ZcX({oR9J{(Oiss`$D1*a&XD@_o+yxR{lLsG?Q#F*&P`XJoBro*HN6 z{da_0Kl+24uK$wT(Rhz*jl|rMm-$@9#e8nl^Av8lX&?9R#%%7)%nT%_*DH+Yzw6%o z40*ILiKTYcvKHs}ELG6Wmbt%V9itCfk$E*6?7zjfs#de~_$u~a2w{CMud!7{naux6 zEXz_q$i{UG*bSEv8geayz6m=;s#e~GAIgHzkXkFh#}VgT_f6x>KI?L>U22^Eusmni zBgIu5m*726vYaW;Zf`u_i@pT&+=3JV8vLyVbS9|LQAxf>%<5?Q!XH$ezwZVcPhrPI)c(b#nwv=naH&jp8us{4c?y z^k;Zm!&b7gJBw_1+eVzLevl?KO6I@s<9W{g#PDQ0sjrD6?NQp~anKAhX?6->DZQk{ zNsRJ26QUQNL_$>`<4BVT5Eq>W`%EO!gQxS++}xdL%ll1e-D_7A^ldY8I~s*XUs1F^ z>=hEN??3^O_mFn_BlPi@4)?Y@io5TApBumEFW0e9#!6XM)=ErT(&}5}SgV|YK5k^{ zYwqW!N8Gv3b=>LvYOee^<3c1bCwiio?;Rv@74!YL_hXH@N`ATLe?lH{(>GEhOMUkB zZ!ptU%4ac=C2aG*i%fA(3hUVv!-nhQnAgz^rlC{FUWOO5N3!K?^l2FjdVGbIZ9LCb z#D=odDW1$pAFzFXs%(eDPx@;s&wpr@r(J?X!OdW4baK5XvhdlCwmd$JaC8L!Jeh

uxdGiNa~A>SJ_WyufqS(URE^A}fRZ`R4N<}-ik zd)uc}O1F~t=bfiYR*}@i-iIFXn@gLse-q)k91^s10ZDev$5q2t!2OUeOe#4GUJYfr#;s3yxc?b#}9ATt&CwO;TK`^_+S@?fG$i^m7lvp+30?xZMzGZGC#B@{us}W3q}r^{N3B>Dk^;5j#T)4*ulN_+@Qf>uK8yq z=gi-+`qqSV$mBE^bTEv&TXu$f=MuvSLSs0(H=3)cJH>@>*v(aTZQvG*IdCiO=Wrcj z@?4DEU9{bJFZ%E`2R7sj$(tvKXqDDoS}y&UR$rH9cNQu$kBrG|kWOd&jX7qv)tw2} z`LcG% z6O*V1WcOBW-g972Q^_j&c%u_7R9sEz)dN(T&#gW@6iWY|2&9|;+eT&W*U_I_oM>^f z5uKVNORb^{h^3Y`vHI1HBfd@KnKTY0g6t&Xk*kQ&mc?YQ!YMK&oP(yX5=% zt3;%ufD~*hC;q>`64lbFRHx3IZdG!m8}nUglWJgR=vl5SF7Kr1?}saw4o{W4LMrXK4iFE5-UpOYoX;p}&UYg{XWvj7KT~S!m{>QnYzFL;igCspU&ODv54DX{pU9y1x}AYB!*THwha1 zIS2h^QRtw1FxqP4iykcFp98AwP*B5EH1nny8Y3Z!b`(xVe{8JLo-{|a&cqe1FW-VR ztpZW%=vg#s6@$V*hNHNq6KH$q4y2Oghj#Os^_Z5;C~V3Gw0QqUWcv`7K-)H+MD)XhcK5eBB`#kCL)9$naucgMM^(HBEtvx5!bPIZWIVy2;}=H3SyMkr@i>i1@8> zB-A(>%WeFC$9QRzc`^scpWD&I$vB9puGmi!et44C+cuH8!>;6lsU4Agw1ZT8pCvC- zlgTW_Y%=}%RkCa5Wnw>CN=p0hl0|>+k=4tJ$t2-ca>B3)`|aC;cRl+ooPPg5L6Y_@ zV6U(dE^|B$jV`9bf88~3G=B^lYd;MgikyY2PV1tfG0N!EOiA?Roj6+ZWg5D>&ItKQ z&O;9l%t1o}eH3kOiV|%t(BNik6q)IWM8`X$nkGk-aKa58_T=w~pH`vdRv(n?x(gNm zJBm!zLQuiC)5ymv1)V>Tflj%cL6t!v=+P72gP=BKw)r2A$e&2QY_cnCN zW-}^#>5J4;JkV34g(y%$3!M%802TJNL&f6^em!{{mgNrs&y&|GMnog9s`@Rw#aDu8 z{4ydcPT|C%vYS-fOVD1dLR$u=Q2m`sbabaa{j=AE=L$kv#b*y%R~yhj4NA12PJ&+C z+C%K!nn}*dMiQk_K<;OS5_#rLZuiY4@_HIXeYGA@`K?04yHApX!D*x=j*;{6pGaMB zFEQEjp4^gnP4q8)AmQ7G$kVU#H26OQ3gwp28~-*?o7&aXXPyI9akitMURlzBJVYmI z&!gRrMl`v{kS-{iP4$A!XmznZ^-dPln;~!IKdX5(TSj6{w_|IsFcT4I>L8fm#0pk?n?&AO39Dp%bOSE=eS1l zTH^}YQ{zYGmfMiLUAAPu?h&H=GJ?2;h7i;1Gl^UAD{Q2A99v#<12;1@V18N{oT%9Z zcX$sdLiD<`0L8B@?o4-@ovwmo|3x&l2M-i5^7gV2anByu&5M0dkO zk*?TrBtX7s+8R$}`g1j!HgyfMXm>@{GuEK1)jQG1)ME%Lg`vGqqS4d~5hyD;gg-w5 zkw4tU&lWbIxj~L->BMtK>!l|)v~zeS|YUy-}o1LSGHq(P)RQbXocy8fbBdzG>b@ z_a9qDyLUL!hF*S8x@IBWeAAu^_c+pX{P+LI5=%O@*Mth@YSR;&p5ddvRxM5VquC6l#d9U$7KN7#7Qv2IRdV?t>t;beNbwNDB90W zKvq8T=-pL)v``A6;XjL!jp!z{reY_WX1)crmn}zsQWv4MO#qc>%t4RX40P?sB;>(+ zRaT`vflpem!h!ri=-s6R+2at<02T|)6IbC^j}mb~>t`JCLX4c{djqd~WQojfT!FklLctPvTXYHdq(7@B9u}#j#A6@vlwU(U92b&089%b^ zeKeV?o=i-hoFS8qLW$A)jie#OmDomZCO$uQkT{X!<#zNJq;xr&cVc4`B3>~39NR^ zgU~%2UMmcO{ofs+L5m?QetZRdeBuX+pUnfc>8F6^=6>+Ot{3cR{{)uKXacG3-9Rb5 z2Yitj0`CJc7*37_Yj;DSv2HeS3OEa{kYC_;>3rxZ?gcAuoPk^8FT(8B1gQNk9ol3S zLqSg_v}qASu}$~kE}dq0)UX3;W+`vR zqpyF+(GvrcD5;xH{mrJ)mMIfxv5Yuf9{imQt{f!Oe~*&Cy(2{S{u?5Ho{-`5*T}hl zspQ?yNRs>`gqZ!ZBU3adkhVuJFx_wri#GdVc5E{~pXiDk`*+|=Maj5`-{)&TdoL^~ zvlB!GEEhBs8-Z_qjsnpRX)H1;Q&`4x5NFTdjP-u5!>3f*u!FP=3E$I>k7mBdeN$!0 zw6V%WQ%8Zc?fr#Ybx&Z=uC4f;&Pg1owYbu2=CL0rVW!&Ms6HjRz!rc{OgiV}Bc6Y0h3g1$Eyif)g>`DR(c~jvr zvnBj_eE<%uiGZ?6Q7~ccWmsrm3T+GWV25-zG+%Zeis3?dC7=v`n*9JCVt3*D%qMWg z^G(F5qq`vMQ0FNKFYb0N5s2|e0!;bEB+sQD!tR$YjPb|z^sZC4Hq8#)hv-8&0C zKJvaL6I0mDyQUV+nE{u-*MaH#)Zn8_P2kA3Vz7GYG*EE!lwju$RiT4Rr_krSCe|+> zk7Z)V;W17+<=zjy!1w14K<~9Tz=~%_ZtU+x#eF_eE2yC?|KHZ)^vfQ z{ck}|`Xiu!D;=zNdnG8I=*?sH&k3%M?X!&kvP-C*a#!f_-zn^Z=Hl#OeVlYB2(O>E z8E5V;!ArO{Z2HcK$X#_JYczI}4=1C@Wh1_a{a+|~e%6Qd6!W&XHeIq)Ly9P$>%zk7PTB$C|y6GZN*Y$nSJ1Bls=V4~?2Ld-{Zk^H~_QWJlc9DZU+W~(~z ze!$s8Vof_NZ&Ji9Vb%7?iHNZZ04XFL` z4$O98;C{_%@N&l&Apbf7yjd^-1k+|vg(5(be}=#`X+K!E+ZN23zfJhAxla&#&;eu) zr~~iPA>grhIxKEghWc}+Lpd`Jn(LcGr!{6!X_Y<{x7C1-+Z5sI?4Q6OOap#Wm<@TZ zH#~G`I{dVa|4-E@2e&n&!0?{Cg07ZLf^F(sg(Z5+@tDCyxMq2}@Q=pj${KC+%5Ce% zfxz`bL8o4tz|%+{WDtMQ_ADNlPM3lkKK}%&xji89?KGH!C&MITGr0Y=Ep+;^9(uSz z=ol>yOQN2E;(rSTzT?ycjHPnv7aIkzZAh zvByjUTy|b1yJzL(#Ukzl0JHX$O!(jWvGa&4e6WG{DfE<*BdROK^a@zqO+!q98@|MCcGZw;p z8#kUWuL+k8uY&QNJD_Fk3aCA6F%&6MgIa2m@ZgKdu*FIgeuH^najy#4C>baCGUABW ziV88Um`X;%)kwLF8bJZ-$ z9MRPRDG^%WgN!W5eB%#nd{TkurXa8~u>$mn)PwQm6`*V7DR3#XMbKK4S-EfLd!by{ zN_;ib1|L;2$5J!waMy}PVdl82!YHpuVdN~naXMQWhgx42jxBv!Iiq@$z{F__SUx@y z?36nOmO1k|tcj5zRIDBh4m}6<+i!ux?X5sOKMQ1=^neAAdI6g75v;MF2-j_G0piMc zz=oH_AX!TQ7T^{?&l9%Z zcgACF&SH~H6CAX63N926!ys!bu9Ci8d0^RkiwK!jl_H8A7Wsus@Qi?Y!Xtf+!Y!d) zLis9DtlemgA1n^Ue_EH}-6i*he^PD;n^*r5dKEk4{?H}(isKc$L;eZARQwYsi0P7# zK##0Ud5V`CjwJ;>uW|c*lU8tCk>R|JCEYYu@4Y37_$EO&Wa+ z^?lYPtdC!Ezg8!^bspm-85{Ayh3CRN-4lYq6gAM9eGA+TEd%o|#(`yfLcrM>^)lDP%lkhdQj`QH$gKjg`fI@1N&rHJJiscC zM9>h!@2{sX0ZVE@uzSrxVLVKH(3WJ``&Y$CaHrjdu0GGuPXW!&?*2Jf;-#_Mi)VmIAi!q4g>Lhq1F zVQ*2BVAm~ad?*#+5U*dBPFboJp31ufv46uXuG+W?hh$nRKYAnxD&1U#BFj%<-xXKz zo4->@$+kh9b73k`x?xM4?mLr(cf85!Lt9Aavj8$L{UDh-`4};6h$LfNwiDw+Qsje) zGd= z@{-AKON~8qKuPUR;21p_MB1zr^zC>e&^sUna!ywZ1{Tc+dk@8cr)A9`Z{c{@XsrO9 z4)lV8fCr$F6N3M?UIKmFZiDSzcR|^!dN6;{8}Lu&2bfVI0v*AiFOK2c#yz z)wkY*IY~7@V>AS88I}R*O>YEop{l^RAP%IM@y|19b)X<91!#=*0COTN!N>8REY1&` zVxu{GaOLxKYfMl<8Kr`?&aLN;c%@?1659fb?n$@asZLk!);`az>TXukey;)$+lT| zj^2(BPX7-lObf!*XRl*=VGutwpF~`&A=$kVl21Elkqbp*$c5S`*l35~+Fuuwk&59#LM zNw^qWOa8$R94Y?hd=0PImw+>M=i|v1UN|gxCjN4KCbr~1=afBvaov@l_|w1Sm1FC0=rL=p!xQQK&(RsL?j3WWp>0;^QuJU+zM5pufPWE-gp)?iN=C2 zm(szI{wIM!?lr;pwatRZtu?}#cMjlHeg0VDA%_nyID;pZ`{SDqSA^|VN3iAI0X%JE zE8e*A8qU!V#W~IvSmA{sKD+Le&|f87aHY{5WJN9m1@5Q7yX^D8!#^C9sr!I}Y2d#QDL5!I0wO2=1;TC&^iI715tiA&CM*@0PHF{?jTgXDyAsfS z^Czg7Bm-?e>B7)^mQYul@8e$5hV!2ef&$b4+_cKT#@G2Ee>(tcg?|LAz6XMvjRoNT z(*mH=wGEtpa0EF2OB7TU&K6W$BSLSjD|pQ-99GAs&>^v6T*FLE&PM;EBv zkqg>qWrM`VIxsOd0r=F6f<3+8K&?j)aNIutY=eD4b@MsF4CrI|Id!q1{Mrg(*;I3( zeYTr$L!q+pkC&CuS4kD8&lJUfFG%8R#^Z3SYl?7)>k%rnvI0btzavmnVW0sLM&DCkvD6si_(#+P>S?%tLT?EdsE*3P<%zkZCuGf#Ho zqGc1w^vn8WpdXUUZ8OQT{!K)=+>I#vO(%Wswxl9wBk96hNu28e!UaZ=t^S!rHa&`% zl?M>}f6ioFf)3eP^8p`!`4)HTzQSEg!f}R4yzrEz9Ijj(jTFQdyjV zUqy8YH7}J~j@RlJv~K)p`Nlp|7#&)HGdN>1g?If-J-Uf(8I31R&Hs_ff^%e5K_EX1 zb0nAFX_5MlT|`zUg!c~FlW&LS6NhQqVrN*<4eQ@bJj6B-I@ z+ykKBI1H>)dW3CZwZcm_7lmU!Q*%wdI~G>R2t0T)02Za}hvLfj;8VR9ublG5x0^L;%|1gfEFGcGjic!@?lipgEgwE>*-ZULU(zz=J@l(WADr`38a=R^ z2IA)rk=fr)!Y-{KWHj*;OsQ9(ZT!BqO56g?)#Y=?cK4CdxfWC-83}`nqv-3)m*L*0 zuTgEoEN*pq3;d<_kAjWU$w$6-zgz7IYVppYM~{TjiE2sc+$j~VS9LsD0D14zaw#g+ zQH2)G8ivi1x!^*qA+3J=jhwk~9#Jh?wX$)Nx;o%RIi2t*rB8BOkOlnW9>> z;l2)in)VzyERAC|$_{Mxzw2DnaXqf1Aesg|Imw=>MX;%b>$#SFVO;8QX;eY^T+j56 zY^3rSC6XVwwI}^i{rsJv;-dn~K2yp%oIW8H2@lS7)dcRKL@O#(%V8Qj`J9vYe6DUv z8n;PHit9^OW2fsbFl(z9)VeX4!=e$KvfE1fs4tK;Ex%3k90B^+F3QclBEda7C(bin za;aC(HzG6TARSq!#c7>RLzA9Jv2Ply;W4p#c=N6py&jZ@R7;BB`;==maBU^2aBV>6 zf0(0!#cFinqPG<8m`cB&HsNM;Z9|UJfqG>+q3wsSq^)=lEgmd}F6H9frhUfr{8Taa*&zt+p2f3s zd?ukqt_8%~U0X9GqjezF)S9I z{po}1K4|henpA4@pqXT8AHma{sHOa9F%`axVBz_3+0(`W%}`Sot-4h);mVmiC#r+N21Wa!sDdP zF_E4;`L%3>6gO_LLCucM>3V zdzavn0Yg|*!ob5_+lldQ5l#i53SHQfKvpOgu=`puW2Qm!Jw&1AMp3xyj5Mx}l))lR zU$FJC4j%UjhKGNPLBk(Cc){>1HhM=OzH;>-m{pw)WdxNNGppNJ!22H9O^d*Z7ehdi zq=M`F42f@F7*4cJhKo4PkjU(=qugZLFiShP9=+q_5#1 z6DgC0yzbR$HWD|Kc`3R=NS`I`Um{lV*tPY zTa1SuCS&&?Iq>sp5;#AnS@GtS;>Yqh+_)%G?xY zyPq@YA2ord{VcS~xDAR@V{ym}N4VMWFg&ZQ2lr&{gT?A<@bTCOa7E!53>z4jU?q~Ox{xd@{EL0-hjH8)5fWc%jD^=(V=p@$ za?Y<9uea%E^`wnR{8?wR?wuK4^IIR3c}HU)r$@5ve_`?4vf#Sd4-oX*mQ}FN!g8nT zE9ae$F^~El!Puo%fB=~vK%wzAnCZ=(*(n0B)g%yhM|8n`u5X}J&LOx}M+>@6Uj>(~ ze-D*D8lr`os%R%Jfvu`npjWQ~nk#IN&XyRUsWoq*(}QAowYdwL|5HalPwJxe=l(*G zflMeROky_>R`WWW)<7}&WE~8^<%Ge3lF}ws5aA@;(W@N5Cvt4;PXtE4uEV%xl{j0<5 zznSq^b3+?uQjg&Gn+I6w*?IW#x)H?txZ{%eVCg$lV~cL}ezRxx)4@2cRp0yZ!D6X+jvho9<> z!hm1rptINon7*zNrk$FC9t=%K_qNPLZMRmV)0Up7YhpJ#YrYxv#W|pgkEh`qFL?gwENsFnjt)!8qb_XHD?;ou^IEJ zsfa4wp|40MZD&xWNi_<8Nm7+FJ>=7pHo^&cp1T(eTjXAV_lq;4fn?D?4vDESRz!9;o$% z1MjThGPxRNp8N&&$Cd`5UMvRfwuLf3!zb{?qZ;G~8;4b1g<|X9w#2n4pBOy2Pb@sj z$l+^=#J{7Ma}WF@vDc^4!Ipo7UK%8w*Zz=|2mg@^*Q995yyrSRMtjE*81_GPUw?V0E9#C909Zru)fe+lC!6h<-&}f$u+Gjo&^*&vM2KU+^p>l5& z))Ro7+=7t&8(-x1FA(iay^iknVzgSW6lEPuMWtV&kdIsp@`${QoCe~N@daNLHnaha z^({q1JdU|Es*Bd!C9GkpL^d9NhA$ew!a_3lN%E)FbiUqk${5?w3-#0K zxuMzAb;N@{`VvFQ-t*M%Cg&G=x0a3+ZKsWWhiNc9Ko6EM^vBbw)Oh|Nv3S`^M5n(X z`+^F{f80)S;KK}Nm#Q4xE;t*87=8g}U9(_-O931n8-*TLB52=4!FmtT-OkHgq_oNoP)4YiD8hSy3ksU=s;8<+HO`vT{{ zrq~<6_StlpYkUCa-}i=S8H7o7yI|mjchKFLLGD+ak=!4Mgnw!vZG&YeEmJonur(2?3YJD(ZM>L%F zUk=>EYk;|-sZhe$8uYxnNL)F_kj-r($lmtX&bdH-c6ST zN6<5xiS*v~C>nM86rG*mP7P)sp)XVdsr7Szn(pCDcTbs1?~I$tA9KZ1!js*##JIj^ONY{K+5^sopuG2<* zo+y&gcm`+wxdX+s-@@FH5xB7SBfOKu!lF0faMAh=@a=p(SiCL@%-fKHcU2xBUks9n z;*Cght0{*_qt|3h=qTB5K0*Xnc9N*#QF60oDm8TGxR3g?=~J+nX8SIo7DqV7$K2`k znymyC^B1J2K{r{}Rzw^>_>nJ00b~(ZCnU;H;@lQOmVW<;b33wd;(}3Je6ZJdW*YpA1LqCHUQ z;XBy9Ney|@|Ip6M|DpS`=7<{$AvLMpNN0K|+O{Vc?a^^Zwa-@}C%y^#_HhYvu>weF z-8?k8a00%{On_^G-+8%*g3FDg*=>Et^S{Za*nqEkbS1$kQ9cg0y&A z3i139K~7BGBTs9t5o0qM(tBYcSvo(02=3TLCfXk0Bi*}z%0V44iN4`k#&UQqZ$7l_ zm;=AhdJncN2?1`_{@{kfIcWDg99|xn0)Bp82HID7frVM;z^i|&VeqLNU~4`%dk&}u zjo+Sur(68-_6Mo>iTwlS*yLU&b@Bqxbvgm^4okCAzD3|y{9ADMz6f;BPk{0Nu0ZAL zJSe@i7k)S{f+nY`BPX4CD85<+6}M_3?^qk;2|Q3xyf2E`vjZJboQ-6kE=Iv2ThXkx zo#=_#awJ))j%x0TBP-1bDChPL-gsC7yJFnnM8RvYZOtmsvTh@*_I8xZS+63a95b)$ z;7<}4Hl5!7sYnm(1d|5~YR-zMoB||6W4^EQ&Bw;Y-dO z8YNYWDY3k7N#wT35j!_?(mQdT)SfRT6_2e+#-}9HAM$rWgkTF+u((1T71YS?76=0!E}2yjC*6z30{v8G~-zwQ7J7 zS~0++TyP}YZWWNdZi`5`h9JADS(tQ$r<0iG2-5IIg?tj%C-%RdVkwE=*fT4fh8|mD(-+vnR|^gj zJI}A=D)lF}tS5ZryB01u6i96MmXgNH5!gr}7o7PkO=1@AB0kfjfLQq!VE7}8jBIEm zKOVGzjk4ls;aC^0?p{LWcdsHkJuJ-0(15ScT_9r?Iow@aj||+XW2NrQfbPM+z`l-D z(sjR^*o{cAtN&etvS9yM&-TDV~%kP5$Exq{phzijhIRk{>T!hS{B;2+$ zi41k-!LM?gU{`E8aahm{EO)$vD;)O2f!-H*=lgQ<_t9baV7&<%tk@5Q<&KgS?|SiG zt5@K_{Wv%}?=76u9RQ>fmF~_bmCNI+k{@4zxij9wvN=BR_H&dr$!qNY0;f6*WtEaVPrHGhhrjUbF3m8IxzJY*nLX_$uBA)yN@E;6~C5< z9S$RdJPze?cpkL7KS1W@#gbr^b}YlbAf+3+u@c@1o0GXZ`SwpzdeMPE|4hd3Z7FPO z-;6`=%%DumYg`l#q2dBTxT3BO)Ysj>X~Rimy7hD-1^2?ML4oj@peFqDyprt7)gY<8 z?eNXX_qe#q1Q#fpLDd_V$X%}Y?!Lwq#MECUB2~i3vg89;9I}(fRTme^EO4gM?9XYX;PN=vJla=T6F6uM=@^N2sf^13m(>t9E`P-72LR*QA#(7w-7!Eh(*Ma8c!g$HM z5E7c3PpmsUVe@~cNbigmBOPT%opRiWPhtlA_}?(JzxWw%w^5*;331Hsf*C+FRu2e> zEr3&>?$fB)stEWQQ}O??E9`pa@TS2yyx z;|^iohme|A<*;p33Ym!t(!4qjYa-hL-Mwp}%JWIQJFkqmKT2iRY#M|vYwkm#^d6!m z*+)7~tU$i%A7S!wj=N{Sr^;%@MsQ+s7PNY;j$%{xP>;J0$XXnY?7t-u@iH;$t9<~j zu{IzpQ{>^}pmuiPsxu!GeTFpC?WpE0BZT zGGwwikf#1iA=dmpbj1Odyn(diT^5`Qd@I@AnmX32-5EZ=ph5xxpqsiBGSoB2-jXk^u*4_(3s~>2<NFOC>UXS$tFL*`fOuKTDo#WFm08Hn z{RFL@8$>NK1(ETn2(o%x0p}%rCGGQ0(M+zlKB^jzb}l|kpZd+AO5I*)SB5*;-SnK? z%M620p3ETS9~MGjzMt4`oZqRXW&!!7d{AOUB4ntwW<9JKXD zn!bCW^et<8I`t8mx`c-kUFzVkwsz8VVLD^BvXbOFj>FcZS!hk45FO}MC+#n<*XmGFF)?bo=rK8CpoJO*)*u7F8dApbPjx=X3(kwA3JJDon>EJceaw0WR2i}D`w>D;sgNnfZtWt9|3p!{JE9rwQRuMhW&B`c8CBO|;3=gpwCr>eU9&lq8U?&S zK3#+a6d$b$F0`Oml1AX|lq95I5J*2?DWzjxQ+dUw520}jE55v8B--T^hNK?EfUl=J z;ohH8e9It+WJ0F$?mnudN-=N9#dwIkcgoSz0xMxXNkb{GF41q4d)_o;c$rBtH2HTi zozo%8tLJ6m{Gn8Ow+BNnr{yr@wm8&yU5*Rp`jEs}N#xR{!`0xf#M(v)&8fJ_sK8zl zCu&V%?}($ZpNr|7Lme<}A0H+c$kEbxP4f5WcQ}0N40(0?2B?v9MkoF6(P{I`$r~3f z-r-F#wD^1+J=lH+EwKE^IY(^B-Mi+fb)!1AE)1Y4Ykbkmb>Fd@%MY?5RT{~; z6C5iq3BU55gM$AAQIGJesM%^Z)czsL7nOd5+;>e!!vP;Cs;ovCFGul~s7ZR+ZyK+< zcMLApD!2TDzxv&vV?_o;_x><)I3Z6J z2ES0~BFifjcA=pM3Tg58r97d4Wcuc?KL44bJul#p8!?g8;9q&yhpsLVLRC8N>94sA0hAe$r_TOry!lEOsj?c!T+tNc@$X(E z`p*Suc$PX+inXMIY#@45Qh{#8tH7M`9rVP3 zRVd?v7V~QJ73$HViV`{>pdRyk^odj>Y99mj(><%M6lv*6|H~Iv8LwSqrUIHM#(sK=I(o9*D z7v)98efCf##UV6)UJPC@xkVmqPM|-B`|--h0=!MA0zZ1@L+$;OXpLzbvULbSSMnNY z70M!wJszmdUlJaWPKNh&ROq1?cY4_9F&eAMfW9pUXrqxMRk?W)wdNf|9yj}`*e5Gg zGI|$2?XIFf21n2(og3u$wlDNo+a$6Pn@Y9AchU2`BiuJ&j7s!RM^jz$(3GCvH05s? z+O%g1Hc=P{%3A|S|3L>5IuMMc_iUpc$@0`K|2gO4)}ZhHYS8qm^Qe_=p=qU!XiQ=! zN)MRE?+7qwQujf;tH(IDIwH^fTS0KNVL{Z(4ISrNQCrL}M zA&2oC5+Ni`!)3>SyL1cq@k{|}9nz=wOAw8G--R|^mZ7|l`PAb?0+QI74#yw4P(@is zB;>M-UhvI^*L?!f&{b)E>WOb?AjuXbwwv|Tzkk@7mvKmtDjUV~6$AJ1#Pq+SHMjQgw;Vc) zGJkDGM%@KmKOq$5P4}Z~b_O6}=@J6i96IH1A$r~z3in%e(FSFVq;|~3&Q{g5Jx+i( z8QDU;|Fpn)>LKW`TQ$9uxEgJWc?Djir_%T%g1mT>i*(b8V0uzRm?ys}gGv@#!@E7( zQOB3BwBO(kB5A3pNW`A6tvHR>W!V7dMh(%%%*W`?{b~aDNYb<7rbzssF#V{M2EW(L zMJgw|sh;f(B-bX4b~jhiO@->{ht3=nduumcB$fw#HoC!c2Ih40yjy7h?Ib$+b1}TW zMhs<|Cezh3x)Hf~5uKY+Goa*q3c3^oe&Sa z*b;_D-Zhehy|S)lWR79Q_J zzb~qh%SEBo;B7J5@I;NeoO7jLX8k~iYny5N3}ICJsT>LY`%O3fx&kksJ%ZGK7va87 z5%8);0bTV(0tK9HLlIwP`M2WB(2}!D>72R*d^WHRIkhS9FKgX|8n;DxrULJ10dglh z5B>-TUZ`{|SyWmo79iXlFlGYt4N2#N+=(OlmzUtrS z;J(!W^3Ue{thK^meU2@fAb)8DS;$j(w2B7SDf5pTYVgMOq8YOv<+L22b1M-WwSek;x1|#YE+FiEgJ^H;g!hWm zsb^g;Q`n`>3%@SI?;3hZ2DUy$rHKn@^->S=yK^(*XNOQG`vJ_Dw;2s|Z=_@P*NLI! z4rGyl(6g)Ws1GiKfvXD8TMN!JkdjE}Hn8Z&=wC8&I)nV~aYL7u`P28>mC#Sh1YKKe zPwj;hl6*@% z10CL3fKJa$r{>1RWX{QpXx7Z7^t*Hqd1jc6;&xFw%AbNHUI(M-ZS(n4PrpIg_bbq7 zs|7z|*-F&(wjJ5X_0tafA^75;GU8KPDibgc3;ZSN+_^!pOSKBcz8a^EtF}`6%Zj|T zqzW)k?KvF;K~QZypJQFtf!Fg7ppT{>C^KpRg)a*5RDTuHLhT+Rx|VZUtlf=Dd$K@^ zYZ~pimjrYp+E6(t!a+Ob`OjR^kX^$h8r$}o)`#Aq`I*wZ@HZF8WO@ksaQZSf|FD>T z%t%G6_Q#VmJDsV?SQu)I>?Mj@#HddLfrn)6*#Z9)sP=art=L}%+xm~ut$c1y>r#yB z?~c)kUqjg4f}4woF@VL|s5|Nk?NKcPQ*0g48?G-S;J&l+zPt%?@pfj`fs_7Q9(W*s%|1zft;`Fg=*dHx0wVloxkU6@PYW46t)c@xE6}-sKonuC#~J( z!`WJ|=^vS|Fz8b~49tVbYGf-Nb3RA2yo=F`rdcR>;xFx3d<_Ym%Yaw^rqhrMH{jmN zyXcWmI5j)Jm$uHm1%rR*A(53?WTQeQeeAZ3n~TdJ)_X4J!z-qe8B)A=)^q4uuj5p< zfJbsVqR|a6M>H8L!j~xw0{Urp(F2JXYPwPf^~v4GPdA3r%0F*Vop}O1=6w^kuc-xt zi{>#6pLT)1vUj9HyBbTZdx9DoI%uQB3$W~CDVmsPP6wuoQ$bQp6uhN)+g)R*!m0wQ zb0`h1wVQzboF_VbsUT{7uSC!mV-nqx1g|f<1o#V_DZMxg>0I4{ByI=N?DwAZq`@3? z;f5@4;U8iCttbh;rlBuSjH@U5+LrMz8d~zA#+0bRy>zzPI9_48!va%zY9d5=5DXHhWKd>=%&?$_n@ zmYCCNSH$^iBWLj3#8*PszF+hR3`XaUe1yNV`J~9&2DYp*p&cD^_(0c1BxCf6Jv99f zmG*jwD7OoeqjZTLNhNTvz6<>FB^+xXT!qDy57BtH5@cZ;O4R>-qHWsBynAuc& zRGe3XhBtqsH>U7VoQ)c9QJFViK{5#zZ#m984%hL|eM+QfKA+{~hzatWmle{T8Q+j^ zsxDnnp^c0bb`hI<_vp2hTGYSa7_HxQmu7Gr<=bb^pknC&9N1ey>peK1Xn+C=dy_{c z>L0+ZvM-_DBQsiH^#D2B`O=KRbb5zT;Z?6%OH&N%Nlt|m@o`c^BiX_zc4|D0Q@;Xj zo?k?-?~C(SoOq8WIIe@?fj(N+GmCfSmNfq`&lO!6%z*!zy6Ni(E#6WKb*g<*ho9>| zmA7q=16nq3I=|KP5Nf&Qh0+fGq&IrrA~%Oja&h`=dbnl=&slvwHTrg!cK2OCn;u_A z1)V~C>l#P2a0iCB^X;i!w;dXk(4^xA-{I!<9ADY83(3rOryVil)VjincWShpDiw6m zNz=K!6Gq|GPbY~MNb-?UFo1dyg#MZL1+K2HM2+9OXs5waGGHLavp$nequ%vXqmdoF zMAd5gQ1dT6a%VFyWCi!#u$V<Ibuf(y358nfrXd%ZZ3eMO|tbuH{ ztb!fR6`X7IE4sRdQbCS=uT*Ht<3KZfA~_E!J+()&*FtH5VJ0fx-b#Nx+KXHis^P&a zJZk&F2qktVa-4D}wEES0WVj`UF8OLko<>i=OQ z-S5NbGFkNN#wJ?*(*rK|kOzM!q*0!6A{|imL@-(%J=>K-FIiN>#L!r{rosVbs<+S^ zUI{2;WG%jA@{OqUI&ydA-E>?r3JvFOqh8!Qr})JT&iDHd8r_j3lQk(cvP+J)e&|;s)u6-;d~gojJNeVkK139MF>D{0fWV0@iep=0=t+7|p9AmC_C|*lC%r@dAAxSvnlsL+e%caMTa=X&&-lO%m z@6zaS7dpD77QIhC3b#m0@$Y{9jSAu|AV*cUDSky=LwjhYwGmXkVEV^9znC z;@PG0JjK_$sNn1Z>Tp4VSM#WWE?*vm_V3U~V&TU8d!=tsrx$4~m1a8rdnU@C z$VTrbO!>Jn$~=Q3KFDp%n;&Aaig#paf^+f7@c(q1@r*==>D5hDR1QvKu$A#V?K6)zpt9c)uU86>~ zhAyM6QFGC5hd}gr!8y+La-KGien-~tdP(w&r6V(bv;Jl7c-%(c}!-w(Kst`t=_5f2&M9qnr?FNe4^IyD&N0 zO}h1NvEOr}U__M-y7V)KimXt^!*|NzrQ>!;Q`w(hKfZv3d$^;*=TY?fsRDFZGmuU? zHgT?%$0$bV6Kn0-N%f{%mnqs6@ZH-oo3-XJ# zk~jyH6k2F>j#{K-z%`Fo0a)||rdEhxn?v`hGuwtPs9ZwR+&A*qCEP)dK{wEeDSv5a z(=*iDqDBwi38VA4K8b6@RoJzEKe>4(lLfp-Zjql0kHTTa2=JqqWyk!RnKa`_tnE-} z-q@#1TEA}xM|P;f4=v}&<-_VO|F*kt5?%;h*d<@HAjMIUJk{+fN-_NB`-e%(`6kkt z&T(0(mX{^;8G-xV%Bl)*!-y^AlASgKW5|frp9(}BZeR^{2M2-~Z*|m^T>|{lti`_6eaZrP*-7rGm|0&{t{(5$ATqQ}hVu{?T`RJ03 zEsh&}L)2zp$DWh*@Zx6${QKw=^21gUswODIWer_y^7k;jJW!W(q$V?)Lh|65r3O_Z zN-^Z7fGIfH7zjOm{sX&Kg^)}55tDSk9r}%YAiD3Tll1~Ea86$qR$CDbBi*uy%CQn` z)G`&xL}y};m#c7G$5I@lh0S9vf}s#Q4$5?AQe~@W(E2ZiQO^~rl1dERTfK`sUVIPA z7M;V@`Q1c5ydN5`79+Fg+7iv?vz)_u3lN>CAlrw^q5b`7B=NHlzOW(@9@U0NEJoJ@t~|Cy5Z>z6@y^eV{H>L<$XrEq_} zGYo!-iQC2;@On`atd?^ng?qcf4*NXNWH^l+($QrOy)Xe6EE6DeIGT(_Z6wp@PC?C? zpNZSE1cqdOgMU{m(w#$lpkq=$$ZzQ(UKje{n8_>VQ;R%3wIdrwi|z%&D?EsG%O?0E zKatFTQVD{uBPfxQK>VL?fX7-p*&C0(;O$dg;VdpoW#94@ANpAWmssp2(WZvvrE4@? zzRwL0y3Hb^f}Q}`9{>Y=iA?ghBULq*qsZ=onQ(c40aQ?tC6ARnNKecsR;!`~-t({} zm+#8ZeT6@vOwSCM+4qiQZW#p@>bh9ZJOhXbwFBYSE2Nhn1~NY3cc^FCE<>JxFbm)#@m>I?e87fh_FcfbW1MNb+?DP_3<}wY8D|?9hb1tL>08= z+#ydo&XK{VJ#dptIpz%1#LPGs*f&3B9g?Kr;BPk^o;JZ=aVx{6Wns|d<^%lsnK(Jo zvN697-2)-ho@N$a^Cx z*>ZZk8G0+*S6STj=>jv_gd zs~U~(3BjDM2>elu2d@-)#Jq7nbeqx* zKAF04xr2p}Y3&4+6ON?rhzy+QS_Z}wN?3=@+^qiY65`_VxJvhi6^wJch7Iqjk+b@o zGvoV5u)fWu>dB87lIte}6%LPZ(}+4uJ(6lTLqJ91ed;V*Qkc@CSiV@GFr85A4qa8~0%Hq4hABH^&zi#4*I$;RxR0 z_z65cV~PXUwlbFVex;3xE?IUu2;Ow-VC76s5X}o*z1pb(roFt)_E+m;4RRdEIeJ6$ z_En%PI*Qzkm4_$O*T8EPN@OtoBWV65iEWoH0?VhHLeK4UNig$~8UGdx-==3V0=;MO z>E2Ph=Y3;qzi8uyB6}*;+h{9P3Yq_R6P7!1@9#sHFlUQ!qik=j^PumNqc&%e% z{%$4k&UTpU{TZ~^ZXoL|W7rjifw1ymAC~L?i?eRV!GXd%c)WT&8J#!;*S?iyEWMNP zzRTM{rAQP$^IaA`6|BYX87sk_+3NwixP%yYZ-eD$RdIUe5N^IU4&IzhU?M-tVR_%A zs^r26u-f&zc|lP-n_)Lm)%g1oC>JaO{hsnzVX}seNmnsH%z0!Ko;Tr`!ko%DE`wIQ z!-8WEECvUkcd)zn*)iK?BEY5B^1zx%lXJUrL4cw?*z4JdzgMQ?@7CF%WcO0g{LGYG zp12OsR~1a}Hel%=LQppRHm=Pm!cxcXFw3ibaAR8%n9PvE{`FnhO4b$^X9~i_!B3gk zE55`B8NuR#9}Is$oFpu51+eKivqbYgTYm=O-%qR|&q5>+tO9`@wF#4$uHHu{V9nzBH-<%2S+xVd)n9Q}G8AY8DGd6~eJ% z^-YjAZ$I;W+izT{AqHd4KB!dxbPGFnwlbN=G=bXL1?&aCv)Fj2Fc9d>VdA`n$s3he zpaph={XmVJ0^Lk~=Q}2FSQ87x%wB4l?SdbFc*y1)`3|;+R^nHdi8#5=f=RBeXVzNQ z0LMKW$*e7}%%!vB;A!r6tAl^oxQplU7lBtmcF2_JZQ&7v)t|v8uWjZ~=8pLik2max zBgMEuOBN-=hwz}X1YUD13EUMK01Y~u$UlA$b7f2wp7=P5kInYLrvAZzm&-xc zas)`RZ!uW1ND~K!e84A<`h)8yw_{xoeVm+WX)YuARU8{rD}mh%GEDm0FHypoH;{rOrId}I>G$*qF(t}JJU^44I(oX23Wu)|F7>@u*e zxse(8{hpn4vVa9iZ*il!1D@7)2)b}*?_ReMc5=iWoSYrZEMJ%oj-Ox2TE)C)9rJB~ zz?z*^Egw{%bN(D6kJRyv!F^D3)(95pU5Q`XcmS=mVWw160@&aB!%n?CjOYJZ30H(Q zF)nm5DO4B)KD7m4#g`0RI6s8#5NKl-&WZrbcZaa@=M!<`vdI-S< z!qbS-uo5)SJ;fFXsF0Azr$9#L79-NQ1FTAThZ~(YGK-pGfaR-~c%{)nF#N{|2Q(L$ zo%Bn=Wj^chk?cCK&Atc+pW2H*N0s2((~rTEj38|OM3$(`SPD}|9Z6u>C8n)`1&5lA zN!OGB&{UO-_nMRg2`=;Y*4CZ5|71EmTz`aYxGo7F?rUc&uRp^7W_<N z+7R2ZCK&t}{tw6I@JLE8gf%xg2e;839HkZvs`g#O_OS&(ASjTXkP;&&g3o~^B^_Y! zyae9v`v%*WJ^;cYjo`P@6fEC+h*^+x0=yW#2X>v^j&sF}K((YQlVQiQvf5jq$F+Z0 zj30#8oH_vo433(m)!oLkxG=WVsJR`4xW~n`?SVO$;f5+8x@ zZc1R|W=mkH?Q5Aj%XBQMVS)Fr8wKNPQsiLeb^KcFDTv$rmAyTzf+NkInG0x!GJf0T z@LmyZro^KR_;{7#XtCw2URox2+Wvx7?NVi(IUZTcB^kV3i_dJ#wgN5VlEhqb8EhV_ zW8Ht5l7_5hkSxl>r>5-1OVw(@1%*5K@kVFdKEknC{hFD$>$6Cl#vQ|eZ+2~*Gl(MEQx-aGDWQ?xh7)#f1jl`=Ld39Wj(pt0&} z-e%l*^dgXXUB^y7^uiD4ak-l4BUqJt-xfTa1+GZPvxdnZD)$;Y;h1-~D?9uz0%ONi z>?wI3ytfs`CR@)ldSd5+@!>&y+h;nQlWtWNvbqOv{e*?#QS;%D{E^$TJ z3E&v81-~(_1|8YYn6pY8%l)Dl!pj1_w+4~9Mi3%@JJ^L{8_VEpl zn;y=bSe*xw9NyrUBLU!giyOZyfUN7K?45T;jBQ|3mE1wOichUM?E7JN#;R2l_{b+?XO5Hh z;`?n@%=$Z<6ch-;j})`XPmkc`5klbR`3z>YPZj$hX9oLnTn|^TyKL&t@i<$T&BS?| zkKmR5dCb%GoN{w(KI=KGg+245s~Wr;|zZV7n8aHC+?Ko)%vW zc#j&YDt2bEvn;K^+k3*G`&$PS@ZUdHASMSmb9_eI^`&frRU@d2zmCT}Jiz4m9gxwT zhTq-20Mfc+f#s=ZRbm^2nH)S7(4%+f#5@-aqr@in?I;;A?EYF${E=?Kij3OJnPX=HW*7jH>bLp+Ka( z5YNe6#gcRXqv%Y-sruSDERit{h@uFUBxxY_T5Crs6_rvEDpLI$G*22-X30D@5T(*Y zp$vPiXQv4X4J1jZRFX7_2I<}J2iJ8z_;SwK`>eH}_50oTEaX!yPu~x25(3vmqchER zwDhF|t-PXxe=9DbZ{sh~L2I6&#w2w?O=GTbEYlB#6=eyYZ`3KDQ-U;G82EwaU^{CngH>2AHEBwa?an&dsK3?T-aPlX0pVS=^lyz_o|A z)5J|{QH8&N{L7VCd5SUGtcnq@oF;tf8%Q;C6woIhAGH0uHT!6>2OT<@jXob~qcJ*G zsCh7mFU?+l)NSchRuFlKv}n8S1o> z_72s1gngrGP)$rS4GQ~8dtZowSy5? zEsN)$e@0zwFL2woUq(*58icCiubj(!Lwe%=G4yJq7Pm`Zi>7QJgdPOm7u3!MAoITZ zg@?xvr3b&wK$YA8p^ES15>7X9XX;XEo5m+jV@Va-bTW(L+fxO{C%$O8p)n0#d>Zx5 zN)ck)Cevws)6k%sy4=}s+XdyR5!@Y+*jOthx1m8cw_0yDs=DM(pM@oJ7eW)Pcjn!< zE_*&2eao_>=eta3`}jAU2U&><&3P(>Wzq9XA97~)`_XFY9?akPgm(3&6j}aDMG2<& zig=qW;k4~gx>e%QxeuR&3O^jTu2{K-i`jF&P{Y;=_4wSkZb+xd$5RXKTla`oRyCs9 z+%3q%aH@1JeAoI+`EFYGJrl_kc5y?dSyHEeUyFQ=W$1_Z0mw3I8)ui`PusUN3SzDj zO4^*vb@@IP-n`l@y+O@%1!qKe+`Rv{_EOX6w_Zy4}cZ@+xW_wVZlJ zPqG?B!@16=W1QuSWSSV6Nzazg;LZ(-NA3<4)Lr6!1fFq3mv({B8T49!uVbiP#RJ-` z6<>JJtq(V4i8nLwMN#AzGm*_=hiOoOTrIch-heT~sQ}{;G>!Df$*Q)SGg4n?#g7AkzB3 zwex9>&3RPkd2#4hPDm4h?eR^(v3F~=_84q&*ucw7I#BbuI){~y}L$BW~QN)?Wwfj zgDdqgv*3n}d_i|cmI#+`>7edS%53MeEA(A^4_drFjLs_W$4;;sq!zV@?r}U%i}eQM zIyFW|6&uk&D%HJsY_;5=7$2>^D&+->$Va0 zJ-vraRd!KL`{T4-ZU}P8mqYtg@`T+26^eZnt^LoN2%X|vZdQX8D!#Xj8k{s2Dvn%4 zuHP4ME9NRv(}xRC#l+8oY@cSX|DyzSJl>Dit;pn7&sRnVKOC?wHp{Sz6jab;?JTZW z;_RPWZie<%z7$B=ZJ|7ZqAAyO>FaO@WS!lYwm%#rxV4SI3&wn>2QK6Y2UcxGtzWGL zNZEk6%oaLh(`D|uUkRG^Hh`|v`N-WIRmI70a|(_3cXG+H6SxEQKEfH9zVxK@n-EJ% z1f!XAX~m(f+=Ak40q4tEH-yUwKfSlmyYD{;lMJ$io}{PN?>DDXxvBotY3)6vwe}&M zv3w*A_-_re3+|yQw)@bv)AzWAr+a9G>IPx_2z4&Su8s@azm&=ytS;&?*-Yg;UkDGY z^{|rlQz}#YRqzaYfjq?iw9zyNE!6FYZeP4A$o*M|TBNza;g4?$6pA?d>*X@^=~^>q z=>Df@{vrQ z5B8(ATgRcl4=>U7?}JfN-Um)iHDAy+H9~5d4phHdiCcMA4TYdFbVuKGp*62W*fQ*w z&~lSji)Np-jI6>yY^QlUg=TMo`qo~`zyJ+Xn)pXsfnRNcdd0dFzSSs$)Mph1!Xz{F< zLUuvA@a>2d{TCmAt~foUVe7V1Zh125OfRDT3n$Yd>)#3Aywj-h^S{>b%CzWgljqca z!9*c!)p_fC<5K9>ig#AL$~|ttml$f^UkA;WAB0Rj)##nancTsF?~4i&k8{mJGxt++ z0UP<6a;>#aMfb@YM300EVMpxHmZksD3JZVw`t@1Vqi_@DKg$+ktW(gi#CvG&snKZT zs~ll-of9$%^Fx#5&j>by7jpsbgo+`?=wM1I*S@}*j;lF|;;VWJOLAi9YjhR)7&Qu| zdJ)1SWh0ckZzz2!vy*GJKg$it+Eo-cF;`gBBST$!dbu$TmYj-QsN}tiDZ1dwiaL}E zg^r+}LQU5s`d^=0T<5U?-2AZXG`MsSa!b|Xd@B98K)C|Jv8Ghe{CN|Zj2b|v4ZFgf zDksRq`%mFf=Y#0Leh^|j($Gh1gmx@dqmvikMh_PCMF$F8kV@)PPVVu2t~dD#D*H7> zcvW}_HQsv3-KlL8Rvwcy-S1Sb-=4|f)_<75#nv3)s(NdGc5x7TWEM$xY#u@% z&$@-uSH#jko<@{yJH%lt8DWrOw6J%w5p|mWxv;l(JXH!AUsN&TlMq~Shk6_gr#76b)$*xhDN2>Q-b*-*FCATv~XK2Lbz>PhFt*c54hoJ)=18j6uOD)Mc4Oqo?a7ZV{zuXF~YveVbC zUp4=wTf^mwc6ew}pBEEQVe@sN`MVs8wADo?&F={L!!+rt5p$@O{YiR3b*pv$w^Ca7 z^1PsWabHnhbXJrLW=BEmMS5y|>?t2b}q^z{YXWfL?w zl6_NLbX4g8O47&_misLfwm2!GVKsAwkPWAh?~p?@=GQVh-A-CFEtKedGgZ{SV!j}} zpDs*{yePytKNMWApFq2{5`|o=J3>HYudubTA6-@b5b<&@f}+&7GWLos%6`_(wS_06 zIp5+0-Nrub@J>?n`1u7i-u^OOoH3S~u4q7I64zGW@_wQ1xPHi3_BQ=JLXloPVu5Op zw$bUf&uQv7JG@Cp7e(g{VY+iAXT~~z`ls(KQqi)Sn2vU)#ar?O&v{Kn8w!%}SqDWj z(Aa>b^wzSyzw&88?J}(MXEqM_t;ix)KBEVH?xGWy`{3?b{^*?$$WjKcVXw!XV}n2b zWC5cqn0s=U#H#4#oPy=h;oKy)d{aE_tjVM=dkyi;)lc#64I}WQRvY%%){{NIpU)!R zd}NNLub5`=0&wRMbls_8sZxG6rGzoNJ-|FPhXVPb3Jc~vVj5BoWX9b^a3bVC=vp5Q zy-)T)d>?ZuALa<7%(~f>CH5>SF_M{WG(|-#2H|hrp~$0oFixBB2Nm}wS~a}PMB@#9 zqnoc5)3Et>h4a8+44~6^IcZ)jWYWdv48FA2jAYkN4*r z|F#jY{iV{GST-3hbC|Sl+D1$k>?eKX50Ijv_GDDB3|XrjkGy-D*sG!dXw~cn?b*Y{ zf2qcz?)%jKG z8CZm0!r*P~F#hKQs9bXaEC#2-bC)>qOb&0%2##2?!4fga_`c!S#z1 zgxntsIqUnuTKAc(*{KiCPE*F^muKKUH!X0b`f1$de*#ZP0Xi*MVE-AMVR1E5KG9?e zTP>X{9@U--2aj{`ZT1GZF=0IPj+j9PT$3_nGCO&pY!6@BG?)K*ZZ_YZe1JC|8pf+` z-O6tYoy8lhoydnLPv!UhGv?Phz9)7|E65e~-{e-sD>7Fzo^;I(!k4e*u}x8F-jPEZU4)_RK`v8y7OgLqp(cQW(5hodxR#WJ80L*v>Wh1Lt!_h%Y9Ziq3B) ziMMu46&05gv1!ayapOWwadAZ@yfO@cgj8)<5g*KyuN=T3W=6!<&We0a#YEF}GUo_+}m`^%t5iNd$~JUI7pC&-$nV!is$ghbo(?X@%byLFTK12WTj z`&b8l%Zrt~m)|@dcOpJ4dphrHZOr%mqs(W?{~+y8I>@p`vb>wsYjSEUCIL4wo;Avi zE#ksq{Iu^d=!=7R>d}8<=WSOJ5}n1HTkS=?W;=1nFg?+H|961mjM@ouxUq=LT>Y3RFHq)##~bo`?c@2Mp2K;acsbs; z_ZP|C^_Cb(z0!>JlSrJ@w;z|w*v}|E&|k6~ns%Lq&?lE+S^r8f@#=&NPnE^SlLN%d z-(^J`rB6_k{|5|?D~tU#d*QWd3zWXdg4&)8xYczC+<*LJ8J~@q&2|TT$o(#^>gY$- zy6Ta|PKpGQp5fQtT4bu978(Ak2hWZNZ2s4QJ2J_d$(0Xdy|zCowwg!h-tuCOiMh<{ z##5%VQXaP490^+u*F)`#KsaO+52s8*0GWn?m*GiZwO63yr!Ab9y7p}PINqB@^6Snd z^E+Jwd428ee5TJHzIB;DuV1*4CtlO|;|C4-D`Cof@(W6eOLq{z5w@hEXdxMP-ipM( z4`w~-M`7<35x!jSg0B^0#p9=g#j?6nVs1v9*k|B=sY~~8YyWWAU8n#B z5y^~qiNu~=p(Ibal-QihCTDj0k@Rbk#Cp+f;_UU6w48oQto_SL+<7_v?t!7au-b^9 zZ#|6HF>ELOXI~~?>sFHcS5D)egRi)1zew7&F`PNf83nVP)1Z0(69_YGhBb?x!k5q@ zFgyMPUS>ByrzL=b+jW>2FTlOVN~t6I45D2NVCUONNdJ`px$VKQIL`=V&jz#gdkyG1 zl{Ls#cO`b>zv0KkmJEwABxlBz;*W|e@m+X@9tH}+;7dzcp6WLCHKdB2a!~=BR5LIt zaDeu2^C0;9Y^bX|0JQEpWLrOjwV&ERvEw6z)qa2+smt-_RXt39dkLllI>0!k{^Z!) zQT*DoNBOM1=Xi2HiLcrn!k2CggDe8uU91D zi+%aYz1n=Hh5;|@Je+@LuFU85)RUnpDP+3BK*CSZ!~OK1Qe<|LIZQAF{SP@Hd*M4w z`cFkX-l8Uc668dud>L`3)J0x7w*k)2tpMXZ0ahCo0r&3$3?CH-e}@Obs^!z*rp{+( zXko#$caOwtw2JWXk6&@&+z})xd_F-QyGZ2lt)yy)3F$Jug%8WJqUlN3*?`qu?0du{ z2v3*_cl;0xxwaAXhXjDd{#e*^^C$?70U-N15w_c8gSM1io6>p<()>yxGp_<{x0FKc zk1H_8^)v{D@vzCs6k5Gs;a6j|`BySQJfD4z|1DnR%WY2bwsTT>nxD$ot0eFdr}pz! z?Hn%|Nca%AM=Ba7kk>xjkbH*`Gcl=WeY;#?*@@e*B;+@|kCGQlyhn+jCa)BogCfO@ zck$xIs&J8w+9{eXTPQxgVIq*r?1nJXHT8nP;rc&(X5vf0$0@!}gi- z`vNEN1LP<3vT-B%pFp!b+0s%J^wdw%%rL*vmpXlhju<>S=EqT6!f4dY+XI<^tA+X}$1Uld%N>I`GQTf)bJudJ=k zlg)k`Y3<{59_638DX26m<8kAk;H(L1gf01iYdeGS)OZ(s-gyyTp`L(0+`WifCe6h; zABs_f@)Q=e(*QhoxPsQ)b#S+4BUr@l1XuACsJtoy^@0M}7nu*Ondjlm_w!(Twg8T* z6hOcf23~J2z+tW9ko7MaPCNU;vJvZ`ahMx?OI3u?oII-y^C8wvL-{o(KD?7{GCyEp zD*xW?1h2Lxh3`0;z%RUdgul?_!`EM!#J^i2#}A0QM(`VT@~m2k2^mI!OZLDG%PWv) z@e&NPyTD+kswlo0DSFMBDW1HwTilQnAbPyoB}OAR(LI+Fn=fdK?%9vQ$zd~m*%`&o zerd#c-UXz1&~ScL^$fnBf)n3#b`kHsekp(1Za#0XV96t?bg}1GCn+@eMAmM)Np=m7 zAYV(&$lrMnu-k-dD8f`xo{Heh01R zFW`InRj6FD71RzG!@h^z>?4a{n^umZznZ+TNvOmKjSj+Rop0km=?WyENtyh%eTMIk zD8>nI((v8G*KnG1GFIJRj9&k_i6GipxO;pL%lPq*efw_`{F^@$-qqN{EBS4}Z%Tm7 z2N&VBdJa5oF8~F-N^rRL42%~&hKq_1VZQb)a9D8_1cx-xTzCjZ8Lb7A^?^$y|RSZ?O8TJwBV zqcZ=cx|E#i<%no8m>oMX2wVh-&}M!S-uu>r(kun>*(nwAnb%-(yOM=?19^#Y+x*0P zL$`@tlbuBNV6y1!tS)xWE`gRxNAQT9&vK+^h`*z9iMrn~-cp#$|KJz$I#Pc6#vVI9 zrrw74Q#awGfA-}M{3<05zps(zV<(BhEl1*3Qijjw&EXzfCNp1~F|gx#0xXot0l#6F zr9IYl7<#n{2BpZ0K3VeOey^`^BI7Ac=zkl!Z7+ju&~7M!A<$cNn_1egWH~BJIp4%^ z{MfY+?;d>|8-10CS5xr0Q(cnBQIqa> z+{dnWceAyxC%`9;gHImjFe!H>{ObsXif{3d_3RYXs$GHHLpP!1`~x`DzY4aDC;{W9 zBKT;35$qG=VSCVK_`PBQpqbVXy7?aq>ORBnO3!bLKT8}Qi61%4U5~te{gDhPo5K&N z3*&e1OyYaQX#V#17(S#shQCm9gukY9m_N908Lt&Si9hgEo{#PqN{m(+;1vd6*c#O% zP;XxjF%SBS1r}q(4kJA=Bg9DbisD4QON+!szgLJy-#Cf$Yp|F#$ViO3@e7>eQz3n# zCX8R^EcN1(@z*`aiQRi8{(^@&e@%t(*UhCpp#3a9?(THncF+X={o=v=vY<}Vc(RnV zLn!Iq^$mwNRw5z#B()6aX1SvSKziJUrH>wigXV46@Sy_EX0}6z(QlY;(GB@e+hN&% ztr8L9DO^d<0mbYXNbr(+J_q-}wrOJ_c(y;=^Km;e4vN9|$Mqq(*XKyjSOMgZdjfej zIga$@j*t~MHY9le zjl9TDUm)gZIfw_}PZP6en2M*?^b=*i7sB5g?l61IKW1*Rk>PE5cL&D2N`=fzQXgma7C5@u z4$z%uR$^Kp&5KO2%9bDac%U8maCbjBA?4V;lO@~T$m3+RV>HPgzKLw#Z$hT`sm7k0 z*5I;jhTNlV|FKzb9R%Tb zJ8mvn__~gM3RnS2rIp}3Mnk-M)k^fxo+-Zcw-sN@P8XLu%@(uoIf^5H+li4srie%S zj}{a9D~PVwbK!v0d-$~a8I$MRso|=8?C~jxoUd#miEI$R(#eERb;tbU06V^)v_@s` zvEb+KAHyHk*5If9XdwAnM~S_RGMRlsg1*eDXA`83Ys7Z}CKi4Heu%ti=%FH-=qihd z_JT*@SEzgW6}+YOsJQes$i&`;ku7Nu9kmxeg zBkRwG5u5MHBp~V>IdCVJJf5CS60XLPeZ`(6W{^5rXm|woC_4!GMse)kjUVh-yD@yI z^Ms_IAuv2J4h|V6z^kkApl5mlnvydhAU_wlm$%`B?gRK9`VflT%V1Sw5j4EL2rXH0 zFzELt7_!O=h;2VeH_BuOU)nI;iG%*niBS0(6UkX6SF%IuK4{to zlaG(~6LsU|oX$jK>Vakz9%4f=ish+QA?S@Cdj)9zuS zo{5@hovJLJ8>uE{XKIN(({#m-9Ye)xQA14e>jka*Qg8lIBxn{dg1QI8VO#%WZ05a8 zd~U!F@<-~#w|?#>y__aLrhPQ8*~gTZeP+Z@Ow;8L#;ftGv_6voV+%;8`)2ZflqxB_ zISt!q)w4*gF!(w17X1780hUU$InVlEU|{wamPkyE?j!G@DY_Y|x~jn?<|d3VJ`1Bl zBH`XHe}G-9V9I+_c(c8k@njA=vf7M$P~wGmx3u9{FB2k{F7?@M_LAin50e@H4iJ6O zl_VW9AfQj5aDwJ zhV{J%F~*PKYHKYN_dbCSSMI{P(^+8gI|lk{Y=^YCZ4>~#a-?56S$w{789pAY8Wi${5Dg9v`0eJKC*!2v#| zV@9m?y>dqCXcFzGWg0BySbkzH2W09#HK0qgh#7e@SmCnX;tX!J)QHh(~K ztCBeFjhZ-a$UyOrqPqBRw6e&~G{ZuPOH{ve3p^M;5vE!cvBi(BB2(>wWS5aYX^6f| zB9_*W0eU^;c%&Tvf8eC#^Be3A{5CuhObhHFsKcnz97Ps3!lL|C3139t42;KUR! zh)Ep-w&steKFMi`k+^`F9gn3wG8ed<(wTVEjyxQ>{0&~SRhfA1R3Mq}-(hj&104GG zEcQ?F#?$0vvEss0MT)Jp)J}4y#N`LFsy1L||JaL8pjqF9wmG)$HF&*_Mt^=uhGY1TSEe$N)3t>3|)(e&X{=ehDW&u#fW52aZJ z>C2PZRN4zJC!;?-!iow`2;aEDu35~0_nbdCm4?D!%LuqUE)5n=E`s#LN}&B~p=v@M z*gzfR%`1VeRhK|(aS|-C4S~sc3t0D`3Y)bvnODbov|&pLZdqtXlq@%p$8I5H+KqVf z`CAfM@%Wh3dptpMZyhBIdnN9~02`v}GK2(d_=Y)`7tCD+1kjF;5^p` z##Ai>ZG|mxVxccs`T9dJI|2)Kgu@uAhw69F55_h7LYUSra9pqxUg)g`@1ZtOqNNQx z%A44P?ntK7Y|M5$4W>7qR-;BsGyHq34{qKQgViR5<0WMQc=`t)$%``uD@Fc7bKRZM zuk|`qkuWM$y``_luVws;3btp30`#jK3-kV)4v`7-A>xQ9I7qyY>ezL#cJfNt<-80Q z?3oV>cFzDGW&`2!7BKab0eFoV3~@&rSVmeLn>f&y&2O`0lMjz&E@l&$q30+r?`Z~p zxMMb{oSIGUZfGSNauoTSAw&7p6azladNl9)a10;*&5+*~HJo?4tj=p0^x^Bm>PeJp z5pi}rP99ZmBQt}ClfAK-SbrLa%eTHkv%BJu!+JluarGHCDo6v`Z@R#|)KGZo5D$y{ z$HCgPqhNXW2#ner4R&v1;hKIp#00Gc&8$%{p{0fO9zMj1s}zLP!BVz<&LynXrwX^H zmf=1l?_lN30(?I$8MpZd;~7>V_`UyLJVWxKtag3CZLK>><>kAmV#aT}^qdU4TCT~Y zKrQ>A6UbU{8XKqakc}$nW)<>7VD6RiV7A;EQax*mlsa+|JDiRs-A)Y3one=wbeRywk(}q8%ce*$Fx$y4M&u1 zWwH*@%zfA;HqtSb6$wX}=bZU$hU+`ZkMpMfT4RM8)%Aj9^i1l;ZKCSF0{#Dv@uuy^PoR_ss2hZII|1|{YgYu&YeT%p;<_AekHmS zrh~)x9>wk_U*Q0?5rhWKCDn^J5WIXZS&$V(R_}`<5tm{}M|~WbdN-Q9e0-P$p4>!y z)_IZZYut#%MlUilYb`N-=|x-zx|8~kOC=wLGdZU!aRfZalN6tRKi(h!`ChdN?U_y z`WspH-f#@tXKTg!J@R0@Q5cJ2SJ<;-z@C*9F>}LW*2^e+aq2QlYENK+`+V88Pm9<) z*}2UAkR9V@yD=5D1I+er5W7`=l%1^D%wiOL*e%y27U5gK*3YS9%E_Nu$edp+^sYSo zH$xp(2Wi2y(7{kypb7&m2f=g0L6B*s0#6cEVZ0~@s&79q8U_rfANzcCBEFB#3>Lt^@B88`&Bab0!;DPQ$G2>Au%o z(H^Bhv~WT-TDJcgdZ%;~WgN;vgD1-14lNEFOHB2#Qg>T@O&C2f0Bw|gG>_kJ!Zel>?E+S`$*yQh*F=K3ULr3~43 z?+*U4Djom*FB!kHOT(59SMZ|8rP#ztV&p34;OBn(@j7YDg>RVpj8xvQ2BYv7qsLSZDS& zX1&ydoqS@$$h$FYWUeuL6*!35@9Cw1x(aN?A2sGWR+CL}(_-o#Q`pt)eQd$cH0E1N z+3mb~CSTUcw!8L)Stk_1v|b);G-Y7Xm^aM%$s;y=*B#b#{Ty3q7s)mRIwP!6^~fA9Nb(}+rd&rMuh*hRTXoKQo)SGoSJ8K6y7cD8Yg8RHtbU^O8+qGlu1{fI~M~MqVzH*G*%!`%+mle}QTIxy#xt zo7i)iPUg`5lZ~wV%A9%{*hBSF_IT+%)_n0A%Z?6WM@I6@@_v6dJK`BV(UVDMxp-5P zez`(S5-n(`?H0`Z%jjRF(M)J|VinWZv+y4Q%s(cAjei%-Mx{w!iA{%DqA-f}-U_9@ zX#yAS{25Km<8l1D!#HDZ0_Jud!$S`p$BE4;c;(X|{AqwKMwdF#?@x=6xz2DdS^cH3 z^3pkKKc<70hkc=oNByC*7Avu1ef3yeswI0lZ5kWaXvMY$jbooqYOulnicDce4^?^g zm%jTa%jS9{QlBan6J9d1 z0cWmyhR@u2hAYoK#IpI<@NSC;Y;U{=ud&#K(b>azN8}EC$=Y!h=})h* zy@vN$s@*I0z4a?oAJ@YUt^CU-mbS6fht+Ir-?PlD)|+WBH)32v6Rn8JqC>nl((*{2 zUVpWps`u}vP5bTH;3YvU&Lxe#nk}8J*R-*x>!s^pS_@Na&0=J^J+qhJOM|C`q4P&3 z;@+qbocuBcFIB#Xi)Q8H6O#+EY|eQ+cgkiwOzj`4zjz7h&e@F$m*o}h2tG-l$f&To zA7hxD&LEbj`IKU>74y)&lrR8^9~+QUHuJRQ-6S(e5CdG zUpDusIgM*wu3hwc>RIbhS;~#NH)aCfaZ^M=n|EPO1?W ztR2WghD>1|9t+uZsdK0rzlz0g4`)4OKEucU(Pyqn)aU$URC?V9g(SM83;Q#BlFl5k9s!U`8u;GOJW^kCb282pQ!St2h{wRDRtZ-2oWCMXoF28 z>YjcH@y?Ib0?AXaTY|wWDGGrHIyt((r+_q{T2LqG0r$L{@RCUG1-g9UuA7VsVwwdUX%?`o<@kA03Ux z-Plf}HjJffQ$vuQ?G2&i_Dgg;)(aKKTX3bZX6St2Bkt^QdHmgG44!j(JAQgS7i;uV z?0>8kJ7m>k6PH26UT-Xkb(~CUSMp?Fh&HMGHh~!C=n$W*eniLjJw83|941l5_*H`e z_qrE#1s#K7xf%~04esEsvJniI?|^jI^{{1uCv0z;58_E17<_CzSREP! zXC-&dg@u(&XKxfcb?q2azp|TEADYbq&W>kp(Q5RCN&;7V^B)TOF$#O>r(kj{8OLd) z<11~&c=W{c_+i#{yrez|Z{D{Q>l<-ccYrL`2oFU;$3}4j7mL(wa(`BQOP8HEwwhJ# zb7j+1k1+X+b8Ov{CjOhoYv1xI>Y|W(AEM#~Rd*)~%IgYe1r3D_cYGVbu4h`*?*;_EVo zI3xT%s`yU>C!AKpm#S?sf4~d->e*ul8jm|=ui(bd&+w+DazrI!ASsoc7z*pONY&2) zr2m)xq<+>w@@UInydznTT>kZ2@*}*!2L0+VdUF*o@jH&=!j@qL14S%9@+;zWhoR3Z zD^NfE!^pc4p(~!QD1G34>tAV!w6@P|YH?yFT{JLRXz9=u+&)~Px^7?TuM@s(r^FuU z+f*oh&Qub02GjC_cT`aIW9buO z*!t_Ym|6{ExeCwOfL*2R(0j&w?_Fh%HmBH5ziaH>KgpqMmcc@fhq069!R(5rD;qpw z3%hIU!3x@@OX2%ftfrE%hVEsIZymrMXXjC+YBzeIeW$Qy%~fu?l?*<=^cT81%o)#` z>5se2gR#Q5Sgcgzho_tm#oqM+xU4A^-`c(%ulu_cA0Iv$`;Q%r&0J)0Mp_|ie)J7h zU93l=L$q+1*oic9exRMx=i?n89dM1f8vppb1lP%_;;rT7sLP5+PT6W)P~iTeq5fUm zxB4K#t1A#KdhUnr$9zEBqTKMi(oNVgvqUsRt77W1$XU9k#;4nPt_cE+^ zY9E%L(1d;--Gp|Z$`Inmy``fw`msj~=CZR}b}(hfwXFWSD|1nq&Fm&}th(2W#cydEEuxF1}9AH^X;#izPHnXkHWrvqPkaz(( zOgFrY?KMee1Mc2pJ62UN^=N^$^si+P_T;hnvy7DwjAP|3v8>}jDr?s}CGj`Hn4*>s zd(dB<>1&*#3HwIT$sR`Nn(BS7@ToHX-l>PRKXbSue=%;B*Tv=+C*lil%c5 zu{GVsj2w?(?Wxz;D6W{9yLYlz!+$fcDX*BN)qVE$i@@~ctXbwP8TMu8eH4Bt3j5jD zV_GqS9JiiC>Qfhxy*kS!CZ!J{lQ$9h!K=y4`K!p$!OrAPu?zVdy@VWHyoCG_9LdhT zvq-ng1hUfAh?pT!0XVX{k^n#^g&KWk%M1C?RugmLgQ zeF{JW4_y8XSk9)xBa3N}i6+A0WBRa0K^qR7R0YR~U+lDBf6zFq0%PTsAmV``bZ7L3 znv26>Mv^pdS*ZlF&I93K-(Hqn{G46)&SXMr3|pHvhdo*|fmxM>(FKv6!l2Est=rF~ zqJ7l?T#!GN?o(UtrK8j_DxQb_d%c^}U-J!ZAFy8Psy;W+g)1tV-AZ*9?UdkkFtLwt=XHA z0c>)QJX6{Fo_2gwrNqPx-ya==R~agi&Xg&{ZO007GXD_yd@P2vt%@R_&c~61_ruA) zRG1@v8#4V@PpS zl?L{0|0GPi5zBJte`Wo?PJsyR%v-}@UrZRJtv(F(Iy+(F z(B<&Fbv-<>bcT60OQ5HB9+<3~3BTz~a0o&$!NUlY`l`dfS1)eVLWdrdGBxr|WTSi)sK%y`I3zRTr>rwKrM5cQUij zy2I`yr%G_%6l2MAzg8xzSojnl0}T1#?PD?Q_VI4LM}L(RGQ# zc8wgLMajLjB1y<9B-JyolBV4W#OdlHav)xjl!x}gr6oRWkmo><0%7pv-YNLzDL~(` zwXpnZBdEB%gT-y{z%%g;G~-&x53GR7bqw&AJn&e11?HJvfF`Z8aAfcq5ZY6~VCqr$ zov;g<%<5^{u6CI5uCXvGpd_PK&>}b{`zWO7G&M!@38rh%Z>;J{+3SV&R z6N+^nUBU*xf^paD4OmRrhTrD7;e3Tq>@a#2zLg~Lc?uU|?aIwKzgYuMa@~V`9$u%b z4lHL2va*=}z$&Kw;t6BZdsvkBPqxZf5eB*pfOhHi;w5eDnZ&tUly;xJ9x1uukNsrL zk2~0n=r=5IbUm|Be91P>>H`}ZWufYW8U(IVfYD`d+0t!I>`&88ws1!di`rhmyljKm z!^UK`ZJP8U*ws<;DHwg}Zooa`W|0K>lf)(G37P*#j-M4eg#Yi73ICyCEZ^40j2{)G z$LB8{%wN9QmoI+$j*Q(^OKj#;kiR{*$j#DQ#5=K+#7w$H?td;O6YEpRkIH@I^{=6% zX!jUgb$0`^le{C(Nf_KW&4T1C={#XgD?DBH8#Lh`gtY#HKl}fJ#^NrCQ}_n_l3v1S zS_yYe+>md(+wrfn6|aEJ0c+_ln- zxT(w|xmTpQYV3M4=CvJ}Coz>Lx(z3)yZe(xo4(<{Rn-#zzXq>$DaXEU&v4r8TKwLx z6W>p+!ORVrGf#nFnR%*W%ck_m?H+)Oq&Mwv)YmlEaLbo@McW#-l~&&SR^%>EvBN z3h7$iK!%?h$nRfe%o9>e^oczSG}4$P(|_T zBsFn!gtDk|NKxE(w~u%xqYJ*be*oj)R?ta(DRGlOLZbUySoWqJ62jiY_M=Vkb$$gb zRL+InGb6yo!4BBVJ4}%2DY~&f7%vz$gbX#7GTDwxNTcr(;y&J)6dO5`jL$Pj>!8VG zkjxaaq}QCNv~Z-LXbwpl=tAb)lHANAHxmof)ns|N2l1D)A=5_ak)NA7aDd_>eE8E7 z?wM1_^>t%x+fih&kKX$ z>y99JHDn9?xx5(`pIZcZtEPjfHyRp)20_swMJWsWh0T;70NLAKu)nk9z-7e+c1H6B zYjZ4MPG36NY|9?@y;ypGn&B){Yc5s|c!K}^4J4Kt7bM1|EZ=^A2+svs@t35W)bd!u zXD**4ouTURNx|y8tIBWbZ1W#6+SWoY+W#eL?SCX5>_Gm2r4~QDei(mj@j$*MwuLB0 zWRe7LRq{zTP{?(e1wzev=)5952&O8Eg=5sksTmsL+hSF*%urFh6YvYhp+De-+FW6K8 zG>EanLQ-epLxiB6q=D`swWrsT8i}cIIeZ>D^wLi9$xJ8ncPt_j5t|%V^dqZ6qsaEM z<0LWj1i43|NYGq=GUmcUvb|J~oM^j=cfPrV@b7VK(9nE_|MjxmWPMOf9}ml4PY1U* z^C9kpbnR*Q!hy&zShzkOoK~cP;p1!Yh0lkNe+pqq)>SB-c@BqGy7BcjcFS9g-P!X!FHAm^z3}f_I&t& z{vGeeW9KB25hgVx;=&+a>$4fZ)X|QAEaSv`o}R^b#SwnQO9Q^Fzc$}(*NgJG#JS*-z;B(hKN7UvZwEqL?vIRot*!U95H17RPAl zh>CXF;`Dv}MaL~aA!K_gOj#KZa>W+V$NvZmEUiGXhg^ zmb7G!B5ivONP2}OF&gbc-q#->v873p!{rJYX>^~``Awzip(a#keTjq+9nM8xBri# zGYzNe?cy*ZGa-bKRLWR}wRLE2*$&^qOB1D9<*E$)Zl;%m% zJg79%JihzAeD%fUoV}lCt>5py1$RYYImKm5X-#D-ISD-J_SxsjLIug@Vm|d%KS+Vm zJLqxXJo>KgN&lV?rIt_=YFVpBvMYy^p}q+%kL75Kj|H7s%V@FD5XzW3lKM;ZD17x1 zJhIro&TX^<=pIP{|NdWKGHP+b`!#L*Nk88=$;FL2| zxWg7^+_YuZT!x1ycXF{Ww|`XtS9;BhD}O$ct2?E{S=Z%*T-`-xJW7UaxdT*1S1DKf z8;u$yC*89{L3$xtM(Wh|g-)uzro)C^RBGHqKOQPbJsS0;cMlAa4oMg()$w(d&N6Y9 z292|qs#gq^3XU4-1ph8t+FD9d_389i{0JwHpUGbd^JK*vud#?$MIi^T4PQO`!F~3L z#isPK-d8`FYq|w!&hvvuS9c1}y+iPKWEGT;ZxviEcc9eg0Zgj92Fgdzfa8k&5PE$o zOq1s*Bea)pW-=DeArn2-ue{YZ;Zetx$}#c$WF)IZDnY(NQ+k4PAA22duigr z6I2j%jx_e(r1Bw;30t1g%%AP_@@F%tPbsE5XVPhw(GL1KY#F&%`O(iU{?zlwgFXv8 z_=VQ>%BdS z`|^cxDrS>7_37SRNCM-2J+|ZwO?0?fv07YNnFd!GH=L9CCX}fe7Nk3X553(SD{7l0=<=wMgOGrblIDyf!6J$Pmk!g)r13F>rOafs^zTm(sl~|g(lcMiNrT-%x?sJX^vEY8Y0ocNX<1G)1$6A9 zV1F|T`I3sZo>G2A{%-NngJ!JVauQ3M5z4L$zD-xX8O$v$jmb{<#q4}sAwwq-PAsW_ zmNPd&^~x()dg&bueD)SLwRFI0>jo%#mkAPwP&lMA1QzIgV+Z`sv5d-6*5g&e-l`pC zvGO_W`H%T5erd1x!{|eN@b42C%h`}+&`#<+R!Hw>oTD>s_i5PUm!zcBO{+&dq1Ln4 zsH&-!M5pt~>EeFs9T892lcLGRAcpF`B~akac)Ha;jeG{BQ?+^?dAsZ*ulRjrGHnSJ z4-KP7Qw*qP?@64T?#qVv^M@&^r$8^|4_tTD<6cj2;6%1#xQV`A+`gpI-1!%yIK{t) z+y~16T(W2&r+(a;^UE8@o$v|bPE8KyZsaZFc1&By<*WE|Be`LmROK1yFYpKZUO)c& zym53p^&m~tDW~cur)gDTF?|azpcN|F)H|S%d@IT+H}EcfEtHW?nK4K@@3gaYQP6m4 zx7$Q%oRXKc-#n21`8YzlaipH~UtBL;InzK3<@Zv7yO6Dr(Iocy7G^eH#oUdhc<{|} ztkd3xUE3UR--67`=U~3$D%^~%f&H2p zf=7NG6s;10(k4@Qv(N|}r&z#f7bn>A-3g37ScB;i1t3TTjV5e;NLekE#*A{Og#(r7!qGUaUGkcJ$eIJkjN2jO z@NalB&76CA(2Cps*@fGH4&3X#7ToXUCftNu>fAYNBkpMCKyG8MD|ep)xa=n@xO2Lj zxkYxXxhv!6aLdPx=4Opo<-!cIVDIQAX1et?KHL>dy@D^{!h;>uJ9Y}avhbo2b!PPW zy&K)~-$=O$71X`{J#Cq0B(1g(Nw0BJq=gG+N#Bi}B=yr|($-~zrSBS*r3-gGrB%+A z)U-X0ltzV6aQrkfjt!(ImzU9p!lm?n#6r4p%Z>C!illI2D~@gdCw}q!4?AA99L6j< z3Z}sg;B0>h)?d2-ic3yItVsqK;8u|Hbcap1$H0jv_5jiTP_45WROcUrwJQqX->?&q z>5vH)BbPv1urdsJo+mJkO0nyn2aT8$L+3wdQgQhadOkdp-Y4v(rQ^1d#j~ZYd8I7SZIOQNS)K+U%zNT2l&idN`zIalquR2vs=*{HEx{W2?V zi*UYs*!U9+N?${zUk~g!uEwqHW6zC9@#Qu}E#R&t%;)xa%-~c;G4AXLeXeI}1JopH z!SL}y7Wd>L+NqF0*}Ha9%0FKctuvt?TB@|JP@CW^&@dst-zoS#+Gi?Aw|I|`$}4+F zap^?qh$eSwc(0XIrAt%ll=PO)&XrQW;4_){a2cJR?L+Lx1j@g&nDTBWQqHJU3f!}w z&U{%*a*>u~H7pPReKHnHdv3BT!a2fx(;k?`oq$cvrEpI52%PJ;6^2%M!SF;QD48S+ z+&vjc>+BCjo#Wxp(;eV5JR4fR2!5H#1#s_J0_5)<1$t+q*=z4WtP(5J!=HZicSHI)nZE&K5#$(lm09c$=@%>qiH{{VEPHxc~F#eMaPwPK0ee-mFRrL{^C7X_KPR$dkIVH2Dm;SK7 zPB)p-Bpvo?a}G8p&!d<9BqUQQCvEI7k;=&1N|nA_N|myWq$Z}a(%aeBXj@Glm2@Z2 z`jIizEuTQuhcf8Jl44p@MC6@ui7N6h&@06%YMHx_E(_hbm34xPTW^HqZ({=U9{-q~ zwUUQ5S&C3u-^u<&oMf*8{8&rjQ}L-^nk+FSi|HTMfbt8|p)+e2G=!zW@2(se;*tt% z(pK1`ZVPu(YMFyb=qOj};fbGd_(g9k-qJTgwf7=E^y*S^`!yx;n12)b`6@fHB}|2C zN<(Pj+6>xQbebLr&$cHAODU^2mt-l1X88ot@n^%y1C4RU`&?0oJ zj>XW&a%<|tOVDSf3;r=w!pj#O@Q`IH&KM-YBY_?GxV{S?_LX3pUpUHYHA(uV*|TlB z6)a!;jc^we;AWkw3*zVGZKDVma1lrxkNNJdgdW zoXH%wIx>wZx5Ov^S-J3`ZTyBs5$JbHif?9r!|C-(Lbphb+`D^FGUqy$3fYm1q8a!> z)d8P4jX-BVJO@z}5W?wo z*J)nEiu5no6+Mtrh4aHwza`XiXcLA0*h2NkSJU|9x#Sx$fkq}fkl#F08g^NW_N!`8 z;s{OpKE{BO>P+Z~_dweASey3U`h|A4&tuK8eHdTkk0V0<@;U<9e$m|({1Wkae%#e2 z$Lud_dM-?EI?U)hJW zZuTvqgB`NC#%47(GqvI-wp{xjbI<(Ck`HP@LxL%c`e_07^_DQ32z63!E{q_&l><=Z>hsf={M$UV5nbEs? z79+Ts4(sPK>CA0R@3)8*tdwVScJ2}P|9DqqQWq}ydCHJ~Fz~I!_vdYiVnwlJ+4%vy z`mSxfOH>iB`A6WRY`w$hzJJEMwEW>k!CDyX?uf&F&BbP!C^VJsLCamKcr`Z-fB)Ns z*fkBO=X@6Y3%evWeFZPyaTAueZz9vnU&{&)#Q;d{=PMw z9%Kza4mg5~4?xzu@lfsS1$Dzkka&489Dk|+4Yh5|^in)ioUG4`JQc;u8)W#wVbMYk zcn!bps3YIzXvc>t@H`vmhCjLEXqo;KjgOeoi_;uwf1XSs!9f%@UDf%OstKhfMj>=MVj4|c36v;&Cr9baQO)XhJmp!BqW5R8<5?ZX z^)#Wb%@r(o*og0sAH|rn)3EwdE$@6fQ)GEuDrUhlY#IMVY-nmKK42Wrzg%sLA=j6q zQdcC#OZ+iV=RH5q|3_V8e1Z6Rlq##6q0Fp8PKbZC9U5sm9uSS!v+mB@=+ z#4`fg#P6D(iEEbDh`;tX5l7ycDsj1L$q%ZS#QR;=ll;kDAl5&i&Mw1hW;mmOjWWE! zy6=2pMqkvx*yWhFii4VRpK7oCxF`#=@bup777g8$x^h;H}`p8u4)!JbW=1 z9NGgxG;IPT?zD$<-UFcRkSy2-b+G^bAyfNNz`Q$>+4}TNELL|atBV-IZt1)evxLFo z8Q(Jbhqou;n(Azf*&^(D9B<;ezV~q8y1UqF)`73zKEp8@-!R!wiS9hrrIJhw8n5d_ z8)U|k;HJ`iVo6o6 z#HHo#iMqg#p?qkJ0`?3JM5hUfDEr_rhKx9j@qOZOys9UDmZ<;{8V_0D|IV%;Ieb`kOV|L!@p4dvczqtHAD?Y6G zA1`k-4KJC-quc&`ygsfDHJV#7YvD7LpDjZ!Z~Ku_*I;3%Vo!5M0NpI%Xr-qMJwZEx zCt*$*#`@I%hdS-Ou0)R~%aQfrUw8;Up=$PXEIHDFew|k_Qd*C*f0p3vo>a8Vj=^@T zMYzycjMhme_|8feJw{aW^V`<(Zs!N_0fyRqn2;Hm$z*W0GQ+V(Yw>n?JcbD#)tS3@ zqR2KBU2cp-x2pGijZQ58yTOj%9UIM;7JcD^oyF+Bbr*hGpO15N;kNYO6$={B*PK-U3f?3(h|WeB z(~fZi$aB92IqvL389zRtg343$8FLTw^4f5|Y(0K!KZZ4PcVR@d1G?@+K1KhiM01mg zc>Ug8;_Mac#q}3fi~g-1%X30T{OVdS47(~gc~{TJH`O+{Z%P@zIJv^b^7mG8%Y{7g z2G;=b_Q_Kvs<}1%>34v=fpG{u*;wy<6c4Z6ft%cou%CfCpEKpW*!Hw5ODqdvRmH*V zt-z3fzdTfY!`zBDun;os%O>FTuEl6-I}NAiSfFv`CEnd^ljPLv`QoaHcf>I(x&@ER zL$T_OO0lkdgt%VLTWtEzT&$rpQ#@|aA@Qb=7h+p~duD62gPpT3WCt{_vNwvKnV-Bo zJkV5v5o&5MCtCxy2lWS&35Kw3qaoBzF@kHR1K?G>78n<(L2jY~h@F42+4zd>F1^8? zJ#1jHha~JNNLcQTGYn);G2fm{mY%YTvB@UPcjj`jf1CzSzh3j99z${26<-{9csk~V z%)|jR0#P}@8=ZZH-o&1%c>2Q%+|VF=*2B}WOPr1EmU;N!lf$^R?`|}my#xEYZo<>= z;xPVwDh`Rv#&P41;xU6P-0r;_`Q{)De5#2*SMT9!bW3MIiG~u0q60~ z_t*HX>KUF+sKvDSWhgIjPVv`Naj`_5ZFn+>9nh6yN!OIc6j#IVov<2@xtu|>>g#x+ zy#d#sO~i=sUg7=zP84uOhYeHrV}8p8hsqulU%j0xnR8VWqrG;b*@csM&a(mYCRgMA z+BDRkHc@yMU*_w)m+^@zllk@$ReYI`EOuSi#hy1R=-QgW$JtC5mGtz8v-d7#Go}=> zbC++k7xQERUiO2I(MI6^ZZNzm7zX+K?7-zEgDochu%>z;6s!FQzcqJ&hI2I7IPHY_ z*%8odzXtp_hl2kifAA}G5xiw)kYT0)!_;M=D&Re9d-{NF6Zjg%ZmI099b=y+q>5Y2 zZ28U_6YRfaC!Y5xMJ;{d?$G`WWmH~cg~2_Xc(55cn|eI{?i@K7B5&d+e zgg!!->4FpecWMN&CEDb0sTcJPn=x=?DweKXjp1wNVab6F_(%0H&L2~agKjjS|Kl1o zn3937RxU>6i$+)`)4|sm5I=nQOCG!jV)Umu_~3CeUJE{j@;UWraI*qeE=olILnc`I z&XHd;_P!{`UsEi1d_Z0BbqOyw*bR>x#$rj)QM^`t7#}Ogpl5(1>Y8rmvr9LM#Ayhd?4^N!lbu%(ENJW|aTtViax zU+4iAw6m6uU(Eiu5zNYUgPeU!!T-iCIN2{5M0RNqaxE1ez26TV;@v`@J^>E*O@OVI ziBOTd2d;?rK&;~)Snw$hTq8C@-OeCTbaa5>Kb1kYTJVGi?__=rI!s3IxcK@jBk_c( z#iGT&`(4hx59gIb^>FjWXq>w1JXR?v(6g^Y$ok`0l6p?2oC8zIHOiMBKX;+i&&_G5 zpEf=EB1fh=AMmpC3yibw!mTpzaJBhI-1733&~v$qp9CGK^lUQrwsGjJFX68YPLT|F zVNjn+0@Z!85@U}KIh?_7NN^!k|lHlreTQ{dR!()!=Jlm*WH2lsOYwZd?kR|5wN! zM(Dv`Q#-Jm!k|K!2fEw(fkMt4xbL_O#NiQe_;muvv}Sdn!DSeFy9Sf*|3;VWe!}cYhdcsxXzh#vWGi&M z|4!GZy&B5YPwzWktAB(iUbmy^u1<_Q`v%_>e!^w353!Ovi7)l%;Ex}wm{roomk7D@ z?`}@GxL+Vnbz3I5N><_FQR8sgwp`vKGFp6hodG*m)1P^(8jFv_E#ohQm+=mN&huAp z*75s`tZ_acjxTm^#CJcYV9ur&eCd%1{Lu0|Ub}h;4&L02vVxOw;u{~*DRZMyPt7T9 zjxa}F_6Pe7mL*3AS!#JFOLHqVi2XAs{WuYYxw=u-H(Lt(-j6m&gj3m_!xKKH@VZ;?bm|;Y2@PZkF~t)xJb63P6O7i^n!j3L9n@bI(&aS84mbNg81+t zC~gdg=@mO*xZ!>ny)PGj=bVD#Z7txG(+wVBcB>u-hAt@izrtn8MlbXmZeyMrWCm z2M(gLvwCE;ryq?e(IQ*p{`A+%oW2Ti%!$Wl)B2RP^tA9lTGYCbT2-c#e86OqJP#n0 z4Wunq{&Ye-j@}yycfjH1^x?ZKE!bLwMhf;wYg71d&lmBffoJ&OcR%>5*I)S9uv-3z zc9`VO*T-W0iY;vH{U&zE=RNyjsQ~Uj`@-a5pO}rmgmwG}ENz{NNY+;s?aS>^M^ypm z_ZiCv&rA|czAIC=TqcjdH#!78xmsKqEO;k(IMbt*^XTuk_0%|Q6>XMVL^aXVDRe;q z$E=}gbJkEz&1?#d@}lUqPV_;TX*XGP)s8 z@1q;we^HKWD%R&#e=*^@o(|xKAMeXWmcN79|J)7Rt?-clI@wD|tEct3~p~|!CsqgeC zTJ&c>`7X_;#Vbz|%{)mF4f#~@A%pHMK0xki33U5uB;_=Ol2;GV=KwP*c%ev*t6rl~ zssxh;hvDkgo&0j2WKm>StJrzDKARg7#`2sqSx)3B_H0%I)6YE5W~X(r%m6jWEF1{G z{+YmhKYgGJeW3HmJ2pSNoprA|%c4w$-wlan_XlQ+MYbI7@3@L$){KRXhjFQUE{@FU zOP=Qz(4Z;tWLdF`o_w57Ga6ke__9Cs^W029VF&41K?Q}(;wi5Y$z9-W{Mwm7r*%VV zw}K-b4l$y{f6A2pQl3P${)Iicb%#SF}Ay4cUviR z+^dBt9+mLr!YNSED})`^#qe%qB@7vQ7F;tP3ID%7+(SJBZsr#oPT`I-7v$u``Sc&c zg_$UG61S(|GuaL z@e|X+b=%3JG?P^K9ivOfj*-W_TspWekLH~^LHp01qNseHetv7A3wGCuT(41hhm@@L z6%!O>kioXS6y3aowp2`|-7g1G`kW8AJL)W+Y|F-E#hKV$(Zm0}mnB)5OJcK!OIh=* z3ruT;0=#Y=2&Z=2f{WAzd<6ZUi=J1EyH znOcqrerGe2_}Lp1a73W>Se0BankT9xU>+?Z7%_t6IJkM z@p*6>^B(+SG`af%FZ7I=1E=lj%3Yp4j?0;S5D@J4+km8Ys<2 zLZXEw?2jucV4NGp-ZrC=zx7DQY!EG6V?{pbLQ@*2(GP=l)O$#9HqRB_ ziJABL3sNVzpj!ZsG%G;eG7s*3KLqLP%3AO3|uhMTi`z@(`!_t2m}m*!>0 zt%TM53}W(|LAi=UiRT!(_g}rhditwy$^JvHp2UT5g;=n8C*Zd z!)%qY(B+fLf~(CCFa5(DJ3o>f-A%18N`?PR18It`(x=gP=+&f7(%E^Rx=h+g@i$Mg zyQ*o~g+@~8f1BFX9@Bzb?PRm$9L?@uO0f#bWVu=3ZLL(K*7Y;7_I|Wv!ri0d;AeeU z?Y8Z#!T152l&%WFlZQaJn+=3-8wXNuGgOaAgWT>CI3ITkW|kDf#yjbddN>Ber6J(6 z7(hkC0Me9%9DvqoRuOW5%`RWSM%Bu&jQ#Tb1>xS1^}>Muo?lH8mPsS9oIW(y)2DN- zbkX1*J+XTzuvYGo!sj-Ug$AmzZKjQ8H>s=eF)2KGMn6{FBD055(tUi4?tG3V$H75l zdeoSF-n_%QjypJh$uE?s52LlGylGS4wPbuaiW;}?BKLbhXLa}TyBghKl8A@egd6Z- z3c>cH8DQVD2d*n;LG_sm7(M(P>~neyt=96K?o%VqJj#L7UF^pBIr?#~CcfO0XpZw$ z8OFsFsc?G=p23m|9*n;p0+U(WA-m5uI2^GLf>sqnT;g$KqX zl=^1hqF1X=({&ettGj9>X&K+Zap`M?UQjjvl`=&~(}uFmRlC@f-^836df87uBZ!+i z76vCSfWc#SL1RKDm|K;@d~qEt46Xv5&Lhxxe;0UI2Em|Y8z{XZ13Htc*_`b40=sb- zyJ*oYE*~|D4^rBRN&P-xZj%+IO$ef|+ha)9JcsgcR#W@Wi3GD!MJ zD{`L;JB_>K8`efwzMm(icaDB8BGNrvK~L`=p%Y&F=}^&5l8jnTddZW>(#nR$s;JS< zXYbG@wi9K`ZljX>PdunKlfzJut;zmctTlZn)A2ZmRAWZb>?DyNrR8H4<_6X4rF=oy<6^d1~CRmtSD~ z!@FR@YN1Xq4Kx#CL8m?mKCj4ynFG_nYwuXNchi75jJt$OkBaE)%f&(tbr=0vpG7Mc zSI`y>Buh`8_BU41$e=1RR;eQczKIqDT%-9yuB+q7YpTh3LsqvR(!m4GG`irZkVD%^ z+N-^&aiIo{PC?Yt3qw2YAvmJ2onNP1R#!f7BCEY|j!kouhff0rLSsKCnD%Wc)cYNP z=eG->&y5mTGCCicN|NDx(R$c-LIj7SWudw44Et_yfUOBnVC~!Yu?s@M`-xuj+{xw?vqFKm6mXX|wG%C#9Oq1okXxkfY`ae(Mm3k?TdR8N3iUo#S@n4}+ zp+}o?9caKIFG~NTOG7@#N|L;TV61-=lux<`-xr<$6{mSn7vKVBXCmO!?Fv}4`W|eT z{RKadsdEa$%(&IIBRQ?}&YU-jxOIlk+*qk4_iL>dm$IQ7+6o$AVb&?Q=3fk=t~_X3 zo(-Cg$AR^&glT2P@VWLs@OhEXgoqiY=Pw|UauQWE2%MO-MDnbNr{7ce(iizmvQIfe zqe2TQG`*Cr3tZ+ML)&Qd*9R1D{DNvto>H{oRa%!>LEPp&v|>U4NoE++_o5$o;M-Na zlv#m%>0un1y#>{>hM-q?b={Ctu54{sGkYLWhL! z!NZQkVXWdk#BZy@VX^$Kc_33npG}hObYhaHXph zL_0RX#mUE6|EK~i&GMqBvqI^vaHs0=@+H&M=_D&~trq>tq}5*4wC3t%+T!?#zNfyX zak*beC;bO~*!YPmG#^nv>vPm5uywoc943_!dufIK7K-i@L|b?fIa>~;50(As_v72> zFn_7Q7H$%anvu;?E&IUx+eQ#{Qxk^Ae`etWzYBg_V^Ccuf}6o(;9%`Qh~NC1-8l7= zO&HM^nlgT|4MR^eL)*cu@9q5}ljm9dpW2~#{Lvl^_tVD>$P2X{-9$qS zl4xRV8WkMKq^P^uWKnpG25KG?zCT&C@JTYIUf)BL3X^F?NIE4^GKqw2|Eia>=%1c9 zNj`c|`Zgx;kIYH))d!SwI)&Rij|j7bb2y=-1=S8#;P&!V9I){Su2ZYRx=&~LO@;kn zMtCKd+kb*j7doJGbRv9NsSP(r^#@Kkujd?X1-rdp;XgA)Zr!DRT;wJ_4zdPt`78T# z7b`WmdlFTyRY{&RRP2VVri*ZMZ7m#pUk4g%nqb(|Mo4~A2kPIeVcy*`2#897J;pN4psa1V6KTBy}h`U%10*A=hFogu2VeXvt*27mr0L|MI$G#C$Wzzj64W}o5a)My@G`nF z;vk!N_VVu~wkY)uOR1}6A=l#B;z4tn)oM3(d)j)o^+!GHUnd9aLo~s1*9#{3zJ%@H zJWiY_QkVR`87?{S&w@YUsEVWGci{7zId~(z5?@DM$I3UdRPH-O*z0;ya_&5GS6xS@ zQzPiz$PF|=eFFt$ZY2}JaWVb+PI@&Uf=Yiz(lQ~>`p$np<@ec54&BR0``Q@VYi}yt z8I0(<(5Wf2wIv1XA>^yppC%d0k-PDI+}hrRkM7i>>~q57DHrk4fk=F-53Kr+KiEc` zfd9UffyI_3@ZR?xi}UB%%O5i#Z_8P*i2MLb`Mt1M^9@9D-7xp-XIS{~H`sgqfEJB! zu)ymZaGviW|7{1{cD?{*Drcbw&Vlo`1_*4d0M(qMpgAxLMz*Jb*5<(J$&0!gZs@5=-wnx@;SSJR_5;@&nKzW^h@ZOHiH)1BaPXcx5|4F-;tl6EinnMU;Kg8r z8!TOLX**y?!(Lo+`8Lj9E>EADmFSb@cWhYv5&stb!B>{5l+Fx={gVx;>Q5%uMXM=( z_-1Ol8$x+oMdbR;fL6@uONWmBMsc6-C^r9rD>nSVX|v?$+wQ(JMy3z#dGHMzr1!A5 z@i>Nd&cI8LqWGFId&MmtS}gB{K2y~x6Cb@S&(B+|he9z9<%%4{7n9^*MBf;QI(ZCk zIw!#j-U)mL%R=THSJ)bv4G+IvhPsE>;DUAyBx>cs=d*>dMyUag)pfw3uiX$({Q+JF zzJSL3HaM?x3U;()3Ny=OP_0OX(mjVjIx!t;rXB>jioG!Z?{-*l%Nw@teZmsYcZw4| zcJhlJmGLbRw)!z>U`xX!-&b+Hg;e)+d_L*(si6uxJ6441j5<8VsGs;g=Eg0gTJ9>5Vo9>sCeR9yUd47!YUL){nS@s8RgOt19D95ZiW zU$!0nJ8SU6{V%xjsV23=YE!V58fjU`(5i>MxX)6S#=KM^KL;J!l5apwv1T;E)t0jU z6Vt3}FrlQ_+ zf7~SFgNdt*vD5uLUs-*KAK%f-%a#vCU2PlmeW!-~atip9Y2o5w2D{k&aUYrcA59Qf zzh~O*C)tU9<;<{E3q~%Q2O+kxFeES*Ebc_YVv2$PIx--9Wf_=EMws-RK*m{6F!`T> z4c4WwZ*MLbPzt;}m;&9yvcTA@5X5dLAv5rpkS$7sj}QS`M#Aj#oEGRNwXrvn5|)3y zkYzUHu_Hq_vh;K3#mleHPN2EWl7)tA*xT5Vu83E*OZ;Y zk2C6V!}M#|Rd@}ZJeqJuZ85q;W#d4jO!U~8gu#C|Bi|H`Uppd^w>gBS1$Z|m1FvMp;j^Uz&w`D|lShYPX`j2i zA&cN!{;2W~2f6c62g>+KI(@P7{&3v?*8{uHPsj5Q#$eqPJAOjvzq zv!@C*%uVGM3+kgTyaz2nS7$V6#&Fg(KQo@cC&TWIic_);Ge; z<-loh`L7ffI~IeJ`w4KLa~vQ!8#X>V0OntJgM)Sg{Bw^1^(X5gxN-`Fq}#yzw_1?q z{*BGLah+|DNnxAxY}wYFVzJ#AFL6?)ta$C0rxHEw>->OYo;a~`JC+X-cmq{6_-1lD zn%((@Uq32PfQu%D*AJw3Cbl%%#+ABuc~ae&vE=yGg{obgs7UbpTynIhuTSjAW|$3G zds~v`3k$ORHi!n^HK6Jx>eMYOM?=Tr)Vs;>8j$OGvD~rXZi@=Q3agDZt>J|l7DKM%)5{NA(?)!K|J;JLDpyA zH`W=W1B+K_!NeXF82PCmuzj}B+%O5YC#{9^bK>BEQ7UX#I0_}33ShBY5j^NDfuLu# z@a$4E2mmpt>uG?p8&$A#NI6_kssfE`HSqadCFCU@g`%FlklhgplQqKOvqd1Ro!}0- zQ!K%GxIX-QEDs6XZDv{&C+yarh{wivN$h9#!CrMQoLI08)zvf6@zfdoIra`-<9^{< z&weyXYD)jUSkqW}7g~CPqru1A=;9rB`mfAg;O@FpNTVAyx4Kbdk0-6S8!ybR#*wBO zBaK&!jNKf%VG_{E-U z*}&k-J}`Iv1Q^%P6IA>9z<*_75V$o4#$DM5Z03G2+_(>(RHs8sN-oSDUIY<*39QUJ z2_FI~VP$?D^xsnpN1`i1`ndp(dS!#!)@(Q#o(-E755e4~{cz`$z^{~F1%sB)g#B;E zg6)69;nxHWaA|tYR$M#I94~c=$JGlN)UpG3|7AVudo`oxAA#p{{{w1-8So6N1{>WDonh30@;rHI3V%|8Vh^g{}g1Y+wc#*5y?`{Nd+3p6zOJz z8eJ$IKv}vYD0GxCuTXKNDz!1RlG)I`sKJy_GJs^SYSWo2H98ZnNGBG_l7oE@t~+rH z&0|jBrbYi@sGAA?2z$V5y^G|-k`yI=JJyJc2i_5PYRR*0%8KlgjUt;Y=f>`=NoS7Y zF7_$T6!O-Mg&~5MDQ@)~;LXC}_`a<`uJN#RUlJ%ROos3w>5%*}4~B{=;k#J_?D=pJ zmj1pDV#U}e9aql+B zPnZc#^)?VQtq(k)N_OSWJZ5#KU3_Wbj=D|Oiui5zW?Z6Kf{pg~aiUpY5|x-!uOCMr zbtY3Qn@&Mnr%_|KKOH(eft);iXoj|sQL>vw{{=55{jhK{ie5lvHd9DHQ0Pu4x=@GU zC#hdxNz($&=&y$%&Ck{+%Yk~7q^Ls++LdYN%v0bea zrDiqgJh}Ra?t_=zhf$dh75Kk~R_KnFP!8nMlgD&_lAQD=X0N*zf1 z-&<11HAixO;Y=DrCegff6wNzgM`<@l(&Tp|DWTY!ri~v?(?^@pLOVT5$?iiNcXpx( zYtg>!7rak@|Gp~NqtOHteqVvJ zCvU>1U$>y%?Kap-Zoo3HtB{x81_vW9z~rEFP!z_)fE^`Zdh-x;ZP*P@qPK&Z{4$uk z%Lk5+GKPKgpR!q%>FnqCts+Sdby5vA#D`COFI!tnI#Q|l{@mKSAe9-X^y~h8>>RMT{Q&FYY zX$I6e(}MmCvZW^_HgxRz2vj(Rc$|eqWmu->T7p zjk2^jvjYc)6=BDZP}I@*$7da!$UDTQi#|TRFTQee24nR(Y_v=>+bZ{zjT!ZW^-0VQ?H%436@`%rv(a#l&t*2~cA+y*NB+hC$WE3Ej^0x|0vVdg7@u$mfBt}6qnZ!WY6nU>a*ONET57ifux zLBD1NsK9ep$)v{iRz-`sO%Lj_LT>Qp14Hqneldo?Wo&)-6c2a*#+vgQ^nA04 zzzH8g=BsRJg>a_Xoo`8QKg?-GfC*_|H5F#6BLpww7;-mrr-TLWbp6>_Iz8QuhG|Wp z-4mu$%9lXmzE33o!)~N~SxnKNI6Ao7jfw}l(iS(7kPmSpW1;s}R^v>s^zCU$gkW+_ z?N6y^)o9>p73#ZLi3X&|(lCc!%!v4cnSKJt_qgzx(AK9L7P|EJnkFfpRHes4rU5Va zr`5Vf^k%6!CH%D{zbpq@^H5C3-f)y~OGF(PMp6ZDMlO?xav!n1 zZx+t)2*x>kcCW!Af=WU`tmjKP4Kp;`W3?53AqsIn+{&@lb}{F9z5E%!Gi4zp=z`T+~^tv z*KGShzp8c?u>TAjZjr)%z7{xL5A($*U)u2*Qw{Nj{bD>Bx(~OR=HiZXg$QaDxca{) zJbL0DPTKPsi!J48pt?GVHtAB;8zWk_eJDNJYDp_L3!D_PrF$86wDY4qjr=;A)UtrU zdm=TegwpT8EreHM$Y4ygz{T50Mc%pM# zV-$ErdP05ARA|xaTRc+wHnH1)WJu634Btk z1>>{T5IDRF%(kBfC+#X>?|T|bCY*wXl43}hc?7q1k7yUHAL_L|C|}o; z2JIb7;Xg-HsF58__qL|qIaXBbEj%C3n$VkO9r`^~g@*5tr|U(sBoY0^*xFtk?EDdJ z`t{(=@J~2*lsa@s6_|$&RXOa=syige}w~lU{W*&gFK@?deZTGs4Gt{ziK# zziABJqFh2J7k1GT^*`uaJwf#O!;i+BKhq;&!?bzBC|%t3haTX1R(e4KPbEuW#$gF$ z(#6r^rx?mbisJX5f;fJP4=-9xP~A@)f64R<{dtV@3Yxs9gS@XaI#38R=S)S1$?{k? zF&z~y&ADc_Zt_ShH;?i-_q##+o-V~Y6)x8j8~ zOOAhHiBfJ`aTk}_jo|i5PoDv9xu}nw&KyI?P6fY*NuzD;WPHKRqn;8%=(P0@-6T9n zx38_Dh39i=LwF=zr{qnSZQD;>b}XSM69nk)hXuTWGoidH<6}I&apB@u>O5xQx>Y3Q zvl|gk3nSmc(#h_3JaR3S(^jm%5CL0ZxN=Y$I;>}d#VbSjc3~5YDq4d;@-f(d{32u( z`$G4pAn3OVfoSgC5&b+Ip0>om!huAn???rw=ybU8G9B74Wx!6^40t{_8D7nchQLz+ zu-(BOivK!6&jouhTC^QfRM&&Mks*A%JP&d#WL?TYjCoyqCWX`$@ zChO&Fro+Cz)Mb4$?{caRjeEO`K0D<>J)>i3zNqr6)N*V9u|lIjGu^8`h9{dxsbMVL>9n4?71bwC~=X#QtV5y!C{uP;na~x-J zcbhtX4&-_ud^FHFSqp!z#S&L5}2Wn$?VXGx zUngTvr6jIOnuhOlq|jDY4Bz+(U~j-zIzDHB?hJWJPyK75$88?bmCiZzzY!lAyfqMNYnI% zZ7agz$V4p6tWAdfLFuqhD+kVexdGYS%=4=F4xqsua8$Vk*^W11{Bb@UI+YJE^mCyn zJPmZ(ufc7Lt01uIEab)5KzH>fFrHWm`_C)_1!gA5rU=2dj%QrPHiMijJVu<;RmemA zCrqQ%2_~d4(XeaVXlbl8k5`=_P17@1&_9KCbcdD)wci~@Q+ha9q*6Yu&A(318s4Bb z_4zbOAd6}qjHjvl!zpjUfXa`RgByJv~F z*4Uw?%VE^qZi9|v=6FfU6o*DPV%p5DsC#cc4z#SqPhG3W^9)*F0^ zLqIJ(8ixNRz+bBjFz(3(lgI*aJ#!nfn(u-3#$xUcECyNLa+f zxdYMgGa(dG>-@R+vODCT`42v8?gp-)__ z|Jqz4oA-m+Q4zyTON%#LJe^k>7iq#vkM`wV7|Z6FyS4L%ZN_-cgQB$f)FdipB+Plq z#p%)+B9y;ckR~Jx(uHdBwBeo(?Yg&?3c4Pm**5M}Z%r^Q=5i!mFK^KUWu>1~}5nmYNt{ASjw{|;Tl(xaaQft%=w8G^a_v_t> zJt+3&2!8W8jY)SeVE=#{&U|_a>l81ehWG{S5pczTX(#c+_y4f&o)hMsb4K+qXSlog z6ngG*K%e)AId-N!Zb;mNnsuD+`yH@u#yqs>Rm5)}CGpW=K{WdLktV+{qo2&8Y0e{a zsuuE>7kyTjH~53cJo{xy)~0azk#BFvmYpK-aa;)|+7>|Txi#GTeH;7-hv1v~dEi^? z3t6+nA$`R)koAp+%>9YnJ0uZ4b|k{zg^95EY8;&S5($6H!@%D*6mCfd!+?T6^mn_1 zX^{(5)Exz@S{ulDYz}`~^r3Q*20Ry)1`Bg>u(>S&(bl8nW=jjPXL+P;sEB;{kVG6} zy-3~lwM0ZohSc1cU|i1rWHkB3$l1P0txG&*4 zp66xId0~}QFy#aNpfU+hZ=A;Ivx+EnT?0Gs&&NNuE3nUF6^?IMhZ|H(P*}tq$I`Z= z8TT93oVLNx`+MIyqt+R(0KhHzrUmsZ2 z6aeErp%CC41-t*m!NB_@5U)yx5wm2F=|}?Ekz}xtO#$KY1gLx;4ZMA!@L*aX_S zi55?&7(D|EuN$s+P#lN@oIHOMrbjA5RxHD(kxi84y7)-ag|V;Lj=GA6|IB_n_S1M^CRpYZr6 zlWCJ=$y%yI#O|ykqJtb~TI3k1TDG4EPu)rSWlYG0U>&l+UVwn#b*5=HwHDDFS9szXmh5CBl15fzr@)(7{aD9G?wGC*^{{{%m;mBon&NrNC1f4-pA5 zkor6fgq#9EV6_iCR&(XCef0Z5)sVkM(Fw3xD&wA$VjVi`1 zdn(yluSM)XZXuTW7fE+u3fU5gM1Qo282@@sc&ppU-Tqc0XID?8gDc4!i~Gd4JB8Hk zxI`Y!)giumH<>d_lS-Y;`DwqF6CK)ii!L7-q`8jL_-trCt}5S%wg2pK6>}EvW%{D* z?r@v~2^d|T!DW1J;KKJsTsFIydy689IzPm%UggN|SdRMz%JIeL2Pi$K49kv}U}xlQ z6jZ;76HQr|Bb9 zves07a0*>`U?0y^SAg*qe#HpCX2@dotK?raCGSstCFAzfU|WDDoGDlW%&;;1PPTx9 z1NIPB=?ou;2lW5&1Euybc*++CjAI%kDdqy7bO9XMUj*w*?!araJJ8u%1T*&)!nLh= zQ2QhsRIaB(g9gXLoN*0?%((t0As>zt%ypKZKLAZbn?XQu8Rx%J1{nh(sNVjHM91DH zX@&vhxAbbFIYoe^UI=7fy?$qKzMBia>*w=im;dDTSxuv_)m5mH=uDasuR(uqoJ(aB z)#%F*e)@3l4W84owLHyN(xvS2^@bbFHJJ&eAm-BAJmzy1kNM+~$K+m&X8vtGz*Oy5 zV%n{e4VT5wE}gKI%;|PejF2h0zjfF{PJ2`6P&y z7v%AipcX0~Uyhd-7~#9%O(@3g+W9gMBO~LAN!Fey_tg*Yh6bbC*$A{UjX|S(@u+H+ zjE9$|q6S|o-rtdk54OePzl;cMy2^RNoc-}=ffqIzy5ZI#Cw#ku^BZm3f_ZmY&YP!& zMt>wZuN9|FE^4Fa@j9J-=Mq)fIh&?U{*ULiLyb{cFop1M@FdR!Yl*@MQMgg1$rT8* z@MFU^0M%o#>aiD8jfKFZlbj!TX)2UX&VkU7Lf9I74^B045xKfDDDZy(V{Q*2`9T?6 z+{gn3yE~v8cO6c1?+v*H32=KwB-qq*-kr$Hu$p`C8+_dW_up_`z;cREnIH(35pAR{ zIfeM_I7zmL$&o+qZcLD5Qt9RT$GrDT8OD0DKOXqSb$v7^ZfKD|r_&Oj0=b8kg{o^ZGXMYj5$R9`L%{$Q6%?Oz} z+GzVr8ap|z_(E>~{!!=#4Z7+_JIl=JifS=>XUB2g{r6dhV@*Gq`DTYm;Qc}}L*px1 zr#KbL8?~USdIh|9vw{l^F0kZ{ABZ|da&?1oFg%bBx8B}>iHLh}A|Bzw>_@Q0v=Xcn z>NxgfJt)7bhg7L5;M05n>(YzConwM+-g*Nb4Wz*-SMJW-?+@Q~F2JzHA=ofu6STL@ zgY%MmooccU|OrQDi=)OI<)G{xY_W1`=6ULo>n6sZI%wXwa9T_ThwVD@d6~H?x zxtX`b{ab0{p%0wOLq!cT^a4s1DyQ=5ZkzD7ObzBuo?p)M+9*Z+R~k`Iuj5o> zzCRT_o#w=fgeS!7{>n0-IzVT zd}xlh`WY^drHSEPQ}8Ia`+0Y%f{qI%(0~s%bp0A3E-z@uYk#9+=%^dQoXe6Y*@+Hh zyda0DYP}&d&4oeuk2+Y+UIIPWH^PYaLCCN41m;ILBs9lDr)w(M^<=@$gN2|aR|4i9 zWpHKLBM>=W1HKMTFifApuBEM@r(O@?A1a`!>;YIbQ25kz2ac!af`(5#v>xTO;+#9eYb@G;qf z`i2LPXK@Wx&U5_3oLKZb8jS0#&SG1|CH#5j3hF}uc2@Xc_nh-Mc4!v|~TB0tx^?vyl4$Gjrxah@kVA&^LEcRda15y$&_nbcX` zf`~h$d`Azw%_F7$HVdqm+r~>tr&~`GJHV)B;}v1DHKe37W6)!}qOE ziI;3U*{=11+}zke@|tdwl;^=jd8aA)`~DIm5VM%Z$n&Vufgd!oT#{q0E2G<%mAF=F zKbHAi!DZ<&Si39(o22g`ImyFm<_|IRSQ#FfU4a5$s?a#79z&BGvF~#|J{qjV6I36xLha##c@NfHG)bP~D!`<>2Rog@-+~X-GpP@nt9dyNl zdvrpokG?t3PIAnyzzqFLh>@)UJ-=RiHR&<>{+jGE({oS^W{d96AJ2`p|Ti^5-l$jThU(p7|Pd-7D1lKt_ zA;2E5L=cZRh7)3kh|Z&S-tLFb7{Q%unW+8CY4Gj=%1G_PCZ#L5$t?Ed&b%z<=JL`^9wmSblI28J8h!u^h(QB_mkB=r>yL`GZ~# zvPS7k*la!> z_V4N$?A-ISS;lZKyC7%H{oT1GsTL)co}T$Dugr zT9pM0{<{G;^HRV;HycF8G9WuH5>k(OfMW7Gu(Q{LM+!@!DM=AlK^K{E$A@`v-iIFR z<-@W)bI~?x5hi{x!&S4~(6uxPE%P4WcEwiwE$|xe#EfD0!LQih$5&=j`U}?|`HS5~ z0%h7?1j~+CPAV%g{f6}xKah_1ppZ{5p7CtKaE&IsW?g~>y;*2b7J%7u`|x!32Hc+@ zgZ3A{(OIiY=x_f*T9@~bnrKQH) zPaq64=9Zuk`GVd=pvZh&fVO5Gj1<_c2+#OME3-12T=}zCp@oPvP70%tq=yIN;_y79H1{01$lqLg- zHrjA_>uFG#@PP>BAUN|e9x7)?L6K@2&=a8$Dt-~nUhjfBpJlNA;W8MMQioA%KG^2w zN3^XM@LuoApu;W#xbBK7r%x`!-mgb-#h5RCzn{T*LhfTyQ5{|o>&MhvpV6di0$c3I z(bn@PRyU5Io!LJO72_-OJ1$VRnGr0zk=l>t)_r*BYB~3tJ;w9CTplszI#P9RzJ2V8 znbxK#H(3`Y{Ku)_s&YEAGM6e|%BDjFSLx(Mr(pSuSa{v@3GU{#gUOk9gg;;#C~Jm8 zc8?(Inzxo6xw?g&bZ<4A@@Xae@T(5HSyh|W713a4EzxIxzumw#SL|S=m5;NR+IF$k zzYJMxlwcPZy?`@U%V6bRKDJ9|8hiVO80$A$3fJFmgwpI7(viXPo@AdejJ5Eh(?vhf z)ZYSlmtBvl7d-K+brME2w4t2J2UO-4E(?189s8LvE~ott3-`UpmUywUh&|$El6jNM z_?myC>xTDu?eJ^VzgCX{(@U|VFBZ?Ip2GAQ$I+i%jR)Li@p8-qS}B*pTu!`8Lw>s282a+Od~BMej3DFnz2P z$Fx$oGarh3Opjsa$5r_8{Y>2VxQyO;QAE9V-=d#;i)e}33XnDMfUsZPkWkP7yKczA zr`-+Y;7knl4+uT8ABasl`4uV%cpEjo35W&DrLQ zwrrHL6`NRT$o4Ot!dlh617FKpm~Sq`mh6>cml_DLhNi`^marMfs6T9J>*PJ{vu;bHy>V*`kM!>)z03U6R!Mh9FTl zOd#tzbBWHGY7!780B~0i?iIU2va=T?{)>WV_XELrN+jI1N`QIW0$}f-0O&A33wP6P zAs~MbSTO+Z2W4S_OBQK65W%eY{FCSR&yW6{$f47^$LWQ=OK`TD9g1F##MMzZFz+nI zJA2!4m&h|z*z*zt-*j^f>~5TE`3XBOf5q)0LS=jU13KodUTB9x(i-2d-UEV7(f&SedZJto_hDHht?t_K4jq*0Vv0{VO-0-JiXH z9p-uf>MOUigVVRNc1QJDUr$MP>C6#`k8g)d9egau8fKl3PQa+w9q8S48q}gBVMpf@ zGG|E_vv`UM-Dz}#9++BAJ>zC#_k;dN?eD7(ZFxqBH4?N*e7^`p;-A}7H+ZQ-e}JQvBK~o=C3)7_KW9W;_uINQ@1{S zC$^Mm|4b+IVtYum=NPH4ctiI3&4+0rHejIQ1G2ZHL3~RbRBecd5RQ*?N+c9|HN7Ep z#c_DwxfAryn?uvt#jqt=4SYY8k!|dCX1q;|PEPZp$8M+6c()Ldi`31AEewv$0M=0FW(;n8~0;U+yeYiJ{1R7)=-7$cKSLmnRX9Ip=0n=viJLT zcyQhe4$*GdW_T1T7AeERGGCau8wodl{e-T#nQVE08XNRphi!7#VShAgvk_0_Sn4&C z)$CSbW3)9{vmPV1)^8PCII@a$x}(V26F&C9`9?6-tOx#pZ!qfm7QU~144y6=AL#aK z7=F%mX*WM)eBHZwez)7`>V?9%O+^U%hfEOcEwFlfEH|H5;K8^zxPAOHm*H;2!@bRT zVqXpRrFY{*=Xcz>as-!r>__vG7pUr1j&}2l@x;yy489+TC(YcsUG-kvB(N3>r)Z+U z=`m_lI>3|F&PK}-KmcxmJW3(XF~-=wpkD0v3L zPdh-T(jEw2x*Qhu>4MlBWoRKX5czI^lr|h7PAgSQ^NkJYr_**+DAS)lfANOi4VFg_ ze{&S6aX}Ju3CpeGk$)r)JHtw`|JGxiG#9x$j)w`E#rV6T0be$?an*Nkk+<><%A9V( zAdPCw++Tt|W5u}fTn?AnNyX>!VR-AcJN}+_3a3nE&{<|2 zZVMKa*Wlu+59sPVg4g^;Fq!SaX9*p+DX0S578jz;-3;trz0yH+?#%*O{%vqj?-*G9a|gTd%iuif4SS?d zLD>yk=sK{5+chqO$|`*r*vVy9-KRruz%wFL3*>yZ2CpW6J5|oULgQuA=;7#2n*LM{ zb;iulRqYVw8TetuizLimcpH;GJw$uQNBFK2aYjQ4TFY?Xm!2AIz2AYE0dG)iO&>0e zeunBtYw*t!#16|lcw}86G9%nh-6atF^L_A@{!XsFWFGp83S)6bJ5@hDPPt2i3K+cL zI9PI=Hem^3b&2qyyAl=ztHN1rFR+$~f)COepqR_YCTC7#PyJPA1GVR~V=I@j|LvO1 zPDz(#UASE8xS=TfFMkeexXY0JcF>sZnNL`YU@2BeS(tqt(+I=Oy)e#u1@75AFkf5% zQ|5ZZmc1K6$gqM~@Me+j^XgPCSRB7es9;X~SM*`1u9rbHCS^ z+gsF~GKy|X1~K-NU|H7ruV^;@7e8F+=gw&-PEmS{OSCf4OD6$G+s>ipmIKH)vJ)2u z&Bu;UAL+Z$zq}5?|499YUedT+8k(f|KrdAVHXEzM{sUGJc<3Tr`Rfn&B$MFK-za!` zFBaDHd%#YaeW3Mw9qi>i3kQnk!h%?7kbU-!v{eU_eAnBCDH#e>yznx8`8SCk33@^c zRHg8z@LII&b3)rE-bnJUVRU~I#wlIrI!TJ~^!_4z{iqCEKT+)9bP*=K9#6SH!4dHW zG}dT9%UiX$_0c1g-&le9lMrRBaweI=i#t%7G4bHOIm6|@>R!BA2RL=!)F!tL1nRtT}DlEm4@7qi$_mnH1m z*~?hpZcWzbN$3nDH&BBjx}jOOz!y85R8TFQ!G%zta}+H@{YZJv&P6`dG8{2uky zxjwg=7pUdcfg2oK(c-``UX=NR^*8@ueB5t5VfG$7^*S+4z6|{(ldxmmC7fbrgl{6( z;rm7tEZC`u6UnW#AP@98mnF2lg#?uz#5=?2>VZHd8Mc zcJT)H<$e$?e+u^3Z-t{T7Q#a{3Fwp*hBd+;$t1r7(pTHXh)E9d)OIIQh4any{p5GF znCo#EOIV0@V1ZdG7x2HUp?F9z4|kFKxas6WyuO!X+jDton~g=ds;dx#9~R@=BRup| zdV-Iq)uVW8EnZ*t08OH~z7NwB+<7|~cXIdrHSsejo_7>wJxq~}=X_dIgwRv`DUAwl zrE^n6uzThYW~1>-Zojz#rcY0R8-~uX^RFj3IwZrF@JO(azXuKFop4&^CnOO`R$g0$ zU0c79?Pb;2++j6#cB>pat}MmMuUBE6@2auyzALi}uTNti4vDdEjsJnH!v`?@I|zou zH6W0h3Po4lAtQbP1T3rLbjCN#y)WB&7iJYvfwdL1Q{f@4Xw|_OPfIR8n2N(snow9` z7};%~@y(B3Olxn$C-19qb!-DJ6z}BNP+b@~@P^YDda?P}b2Q}c$>q)onCg2D6&D%f zZwXx#d#8W`$cLR<(rInV{nBq&Y{}e{jpTOGJ91f7m}ACi!?dRxp}yun`0u+rtda79 zle7I`Ypg#MDEokmi8BO$u>}oD7K9IRtmF3_V}7?ROkX!f^s0S`%ciUql{PGm+ml5% zJ$goGjd3j=6S}Auxdrd$UPP@RPG6iGiM8hvaTQ+tXK?Jn8dQ8f6Ptay>1V|k z)W9j6F5gr{o2>WIGE4<%nM_Ca(sO=`TkM@m}lzn@z4gibJ!jZe)NVbveBS_ zC<@+*Cc?b)iE!~sG}P)xgRnK{4}9tbGhbVQKmkDdT209I6NRIVJR+eni`3lQ$h-gd z60KX%Ma5Or@luc>_J|+E{6T+wmmPzb_FqRKAs+5I`w%BdHshSZ7F_+T6>Ug6USwKP z?%PYO-ufIb*S2BM%X(B%uE1q~i%>fz73tJyT+RJ2`gOff*uotpJk4>UauNP25yQoL zjkM)cBn{9Eq3U-!d4>iy~2MmOfFvq!#Zu#=W5vW+sD?Dc9n)~a5CU9oZ&8%~wjsan(6V18kCquD3O+1ms) zl9kY{k`G?Ze&BG=8P?`30@bT;NcEIS#Ek2pCl|)K?(AKd=(-d8*Ph3wA6HS%DG+tv z6ydDU3hdzC8}}{_V(`KdEF1fTX6O460^Xt*w4um-#8eF)uJJ3LYFH7C>BzUTj4oc zhOO71VJ+0)PQ7xR`n?R7FE7Hd&o^+vf2rv6DiP-?2VrKNBhG$eh2BBxxc$2*&T{Ue zT&$MrehZ@Ae!dj%dGnsG5rAaQ53D-t3iwDQa(iMQc)v0aJZId7AAOI&vGpz3c7F#a z#VPDkW*S?td=@J{Pm2}br_Q#x&SAg0%wU7A$+PT!Mb=DzDy#6HAUixV3J*vpWV=>_ zsA(zayv>9q49A2!y$bAt8c586dM3EujlPT?rXxG%q2TB-%zti&GKL=L+!%s6A2N_F zeS!<6dho)6&p4~+BPJgk!dH9WV3hZB+^GKo6YX1(e@g`(nOuxbOdd8*MB;dtCq{-H zLd!SHaO)H`6p9nTupOmT+GG=TPtamEE_WvxQ@)X({j#vxb|$nPXCZKS4eWcj3rbI( zfv%?)U|*OIXr1+iKK?6^k?R7^0mqTp%`e2j z9k8oEZWaH;XqGdB5;uE~UZ0UNCE%JjY>to?tVJ0X^RKn-+YA9ts zfl$#en64_t{^Ynf;@Jx9q1)Q5wtxovt51;~;l8i2H|5zE7ev|XZ@$Ad?Otd**AA)X z&mqI77G7Py4hDMRU^=iJwwB97Xp$!}36bH2a@=@T3$9NkJ?{{m&QZ)9;TTQWf%XP{cus5tzdn71Qjzs&zL$r2|6)C3h>@HR_#XHD5#Q8G&yTI8ziyS4=v}`^HkeXEHoYSW&a=Txk?YNH+YGt;9H6W0 zFx2p!;}{&SkPzbtVWY<&W}Ou@8Lo!7bvh8fb1v+5(}ekAli{mwHwihnnUtPQFJ0%l zj%r3U&<JX^VMw})VljnzYc#%4k+G)C22f|>Y-`)#J3 zx{ouK6yy5Rn`rtW2Y34x;hr@mSaPQt1+O%s&xIzmQsLNKQ6=0=myatOBQV6_0A{N# z!RqqgG~4zz{ktrUmiw%xS2}0%bTgZoJH{U5ZqIe{XX#^-$BYs@r~u>hS%>v z`_A-?V;XOANhNdl)N^JLszOM{QZ5s93g)!O!^c%_;O&tOjdGcwI$8pbdat3OsU3(3 zKi9+g14eH0v8$}5*%jO#y(34Cjcb)*FV%>%(`4mYk29j|r={Ou(VUm?Yt`5moRKi#a2jS%CI`l_pnAR^TM+MzIbSQ z9@;tHz(c(IIJV>=e&u+4i*A;nm1qH$8|NWU_C7W~PDTBIbi7*-i?jX<#CzHt)9CpT zj)h}^PVOrBqq>gnY>lEFtFQ3nk0EnZeH*bm(LhdY7$No{!z3_C2u9t+VA~00PzWGU z?_~l%6s+J1wE*MBozOeB1LB4_-fq-Zs7jv?zbmDoCcBH|d%Ba^YipQWl23WV%RK1Y zqB{EL!VfyMkK;tWn}c`rmtnv8b~Jf?5k*_Q(CJ_p&T@#ut|`grH>VIcKE8|h!zjnT z;o*44UA+0CkmI&yVYh1v&WjDlpQqfBK0S)pUv9wh155CTpe~xnNMK;*A9`(1J578R zLIn|F)e++se~;twC|YA?w;Y^s}*ZxZSW3DoKUDm~cZMG;FcL@(^ z$6?hP%Jr{5#1kKz(dqINl&mgC*_D-e^};RmxLk<661ix(DiRkSOc<2ngq{xpZfi={voTbID+<8fUg5Ew~g0o96?6st#;@sSMt3lAzVe51sek5|`>4a!Wjh{9Y$bWW`?)w}C4FNpMUCEMmmbj*U|b}+$;@x>$l#Tg;P>(*e0sGD)a&!$PgWL8^*}gm*$de> z8Xz=q6sFbn!9L%2V7}`o%r+l_^|8Wie+eIZGEkKLdRB&&Y!zhPXA840?~el1HiNNJ z4IFuy3rc^ZK_&1kutJ+bXh<5`AKWIJQg;!Tm1(8&MRHVw4Wz5yx6nOTL{L{_F@D#y zM90EF{2LjEC#7%TTFT9$Pi~-h(mhyP4BF&{LY?mTZt)c?vI)fU zXQ9|58-i9}f^gb%f1Hx%juQiy@$(Bi{28(aix$kocmoNJ1;~#fwOzDIpqM7Dyh7bl z=TToDU25|)iw863^Ah&lV01<+$fgyqh~-fOFg36OKfO~>w=NTc7heY$tB0GJE$~{h z8Ej+TfP&O>=&2usg+_0|VnaK(SAPRNLeF8uXb7%0kHO4`gHSUx2+vQqf|nu3wmote zK3`1$?}a|_HP{3ePMr?lRjSBDlQoHzv0+{d3R7NGG}jj~N?ZG-@WE>>6p~ww2?sg; ztExLHyZYcV{cwEtB!Sa?67b6Yd<<&K!PDH^P_Hi+k5yj7YLO_6aPUU<%xUDGcNm>q z4e_h_0t|mEfy@34&~xXXQo`l2?gcVbDr70oyD*zkZILH4r-YI@^KKKqW5Z<3O$fFQ z34yk&I81*b2ujPupxb5&{3sBEGdZG=vU-GE@u(wnZDL5yKRYtvB1i;}tvB5HUktCz z*_8IJ38ngu<+OH@5N={LvHcrhUzr&ewd}^5-)!)(>=8`lxMS_y-oDA&7v;YA|HIuK)c6 z%eeP2x=n!far+Lcr^li8+b>xCsSiwFd;^QigHXG>2dbaFfYj3aQ1G<~vi2rG{zxdi zcJv1$jwfJ~w;Cowev=EiF{E8il6<}+L)D-8(ht(_xVMG`F0N3+{4_34?Xn3YQqQ1w zw-@F;3dMgOu^eCd8V)rjV_ZNY-VeyYTiguiBACi4b!pgN6^75KN?xghqKgiadG6Q|2&(ZerxEnjA5^bApgRjzmKJ3UNsa zAbe`BLO=TJJRZ%2`Q~H?BDJ6__z6*2ne44@ihGV=vN zd@p)0_!c!x=%pj6f9Z~XY209-f$zfRWBfq_JpFJbw+~!}&tI=Wi76{_i8YIVnwFv6 z(WUsCEXC+EdRTv3A4jBgJ&SP-&;}|{f|C9#ZOrYPY zUFbsh=R6HpW!|BO{>^jz|Z`gl>nW*Q>zA?E##cR183KVRl$H%v%uy*_W@v z!|f-b*_j2yZA##N>m`9RNu)_kinMC^8$MYzm(JbufG*hak18!Q#8sCq(MI7kp0x7A zS8x1K_Ei+hu8F}bjw$GHHU|rwGq7$dx0`H>|9^&@P*E_>DL9XxvrgfF{8l`&eFZl4 zso}DrF?uiPF|}CZKo3TVQD2<_&w3zI;TSNjU(#fn10c4B1J;(HxB-id7V#N9dc;{^esKI<=I{jE6^|@3) z>7GV<)%Q33?I3|+vdYLl;Aq_&^>M*0V`MDWV58SsRGPdN`4_I_dZ3MP;3|vFwg#BF zO&@FbFGBMfT4)fVj%8}o@oSbe{>>1_+DHNXbnz=a>)T0-T8pXS(`dSLgAx6u^qHr0 zTeMVp_iM&z@+MMqypp)ayeIR$)In8v6=d4&hjQ(!@UAif=G@MK!oRnmljHQ-U8sW3 zO{Fj^vK;=rD}n0GcR=SAh10GDaBk6U2rN#6ccu}L_Aw9!ww#1Kdnd3fG6zfE5}1;$ z4Jp3kpHwrbf)+)m zP-3>5&ilBKwx~Ve-G4iu_p3Ukbg)Q`8I`}n@Yg?PBF?CjIu}9+-A+d2?MSKHIx@|| zm|SbrBl<;)$SUPI0bWI!|{cWZ0d;6*V@o)6}^sh95eMcY79iSC6o2l^CMw(_>MNPAwP`mzCPOxdB zFV8)tLZ|Dfabpc79#xcYFIr<@*^z}X@gGXNz@31MbCrlo`i!TKM{u~y-J2%xG zgETU95_-8yVsYa%4A+yx@-5Q%Z>16*YE;E5;~KcqR12%Gt8?!vReZEp21hT7;Gz41 z$lvgRW=4IWe5VI!lt43`m|IU9HgNqt&!gzf8IE+`8(m6PNYM0oO}q`3iM)k-9C;mE zu9oiWT3XC!^TnX0x7Dyecb{S5(Y)f;)bP@72Q}WMr|!JD7o&N8`DMJtx4U>@L0@?- zZSr*9!$tJJqTMvX&Y51y^Pw@>A++8tj0XLVruK2y=#^Qq^xfq+8Ub9N&!1HK(j}MX zZ{_?lEhTh?Y9-CkX`p2XUs0CxP&=IcLFa|=;a2jGzPSE{&Kn+~cBwC^i%v5g&3{ZE z3Fgozb@B90^cniYXB%x7)24Gqa(HTTdOVv!FRpj?592UqM9i1Q5>>$xqL(WKN7R)- zV&4jI)U|@FpSIxAa1nm}_JFOMu7c}CH2k%R1x6+tXipwYOur63z4@SXI~8`Fjt3DM z3NGXRAokG%c8xm$Z?g^PlQnQnPzU(7O2YQAVdAy%4tc46mB@5il6~@m__KKERVD?^uJi2Xub z{Ye*lZ)qXpt${uKYS?mC31=Ojj=QC%;4dSN=R55s-LS5hCN#!Vsi~gy;!#U#FEyW* zANj}AH7n)avvuO>w04xnX`L&bquXXc|I{01P7E75PMgY{KAvf4WirXo!I#g#X7kNb z2NeUJPQZ2Ej;>MO>n&2WZit~SFRW;=)=9eGHjK`nlR^9D@u;rHQ~F7(l{%m6q&>VI z8hgBt+Gvf?g?oO`s_H*9_lOVn|JOByV=fJd&sCg3dPw+Vg4y=&NI0gKKL6!>E@-7RXH1WuAB~8^F-m1 z-4`O^Q$s}EQ;DaK1KDaLO~(4t7}4{&#oPG&dDpdt=?@TYO!vc1CO5T~ znS%=$mm`On7pE68p(jQ$o|QLo&MZ@09Jhls9E^sS6+`e}x(&^hT&tG8q;q0*k>0yg zM`hC=Q)R_o`u^V#^{|lUKdn^YhuKT+SMdm5+^)u}Pf_Ei?N#S9WhD#@!ARaoUz-O5 zZGMG|E?}BlQ zDEc&60spMofOCT5aHx7AK4Z`0pX00X289o}Wbg|%$W&y$Gs;ZxZaGE;t25#s`po?( zGv@Fw0rU4BX0~MdF_o&TnL{3tjI?Y#!vrNVSG`gg(fw>DR`mjN#lD#NNz0golnO>c zL(I(2FJa0nOPFh_Wz2A|GDbmgnUR&dz+kB}%(?tyjLE@trVJ-Ai5GV>IgT5c0i7Tw z?Ry}D17|VnIUbDs7H8&1jR|vni86Dgp$%`dFT&>2WAW554P2!djcx?T36hivX&=yr zMQ!1*Xgq=2#m^xH52xZee~0ewuOzMJ3g3Gqvf_VC&Y$c zdT&RaEhIS%FMApnXHUb{Inc|lPIP9q3zgeFi`K=?qwTRv=uWRNI^|0+J+H8yf_en) zZ{I}SjkeO@iZE)G8%nKn=288}lWD_jb9yX9mtNMDqmK={BqEPG`0jolOdK=da%3pT z3dTWpz-N*)`4Ul>-b@4w&yjr38>A@d0g2z$LXzSX!FFW}DbQAjOHFn#-*o{L9Z!MS z@-)Z{$bpL%slYB>31`k4z>6vsc&;o5Ge18ipS+4mQ;QC{w7Z<^u-t)8EK$Hx9xHJ5 z*mGEQLOouU{0+ygkYft3k7agUbYW7m{F$tu;f&+?ZOpcqSf;l>mf3MNmI*!<%ba)J z&m5m|kkPmp&!jwzXBu4(Ftft;GLHXtF_%V0Fv$y6FgLhB=9k(GCTD>wW3$SJ`D|&x zWZzL@tn!}XI;#R)w>lN)PV~jE!xZsG?PI9F#Z1sqR!e^Eo&mbeu~3?F5^m2s4@`3v z{MmgUJ{)-k>TDgX-FXkBVm`sN`Y&)tQJ$I^DpBPbzo8)EGi2KL!jrld*naaN>?&x5 zA@UH+#gE{N!gEmi+6)%w#E`r5BB*8;!;BdfASdaue*7y@37_EaomLpse*}#sx4_EqHkdqX1i4w2P?}p0W|9)Hms2w|T3kuyD>rP#hZyZ>4{b;yObT)6SsbA04n<#$lZ2*?<*lWSEJI$1tYr zT^MogOh)PVT85g&FhuzfBYWW}vut`QBTkgCRZ~-#bt}`D>N`gnUxmYrblL%C`o`Ui zVZlabLh({2o%Lg0CwVY6rY_8T-kFhWabjvMte6!C#xRQRa!goRHx@NCVA)-l@yEy{ zoc%l!M;0u{f<|X-CN#uVi7$~&-5k`lx>l$&{SPN6P9$@@T8Y|iSumZh2+f6Zuyyt@ zNT3?vaq%NLa;AY?@i|4_c9@Z?pmuopDTTN zc{1P9gZg=o~!3jKdG7x8Ye{Z}5oF zUTk#qE53Q?HO_iHfc*}A!_IGqaJ17OTs7(^K5*;@{?qXdryF$P`6KS(nTCLsiVE-y zwg9*NK8E?Lhw-A0C_LVNA9g5;#a}bFVWu@0-*ooF3C7EC3^x@Eb5qF09m*GGCXYdvM$JYFr;ee&X)UP4_dVJWS&Ym!??i7mjTT;(ZRGydgcH5} z9NEqekY!`kVbIJRJT)A_{j)D@;3Q$#f#Yzi`vN>JD}|G6HaM1^0bTiWc)$22s2_U- z=-D%{S=$30>tDg%zh5C>XCDZnCA@{tL-6E%KN!b+1*M7IU^lZH%td$M;f-QY2{;aE zXJX;0@jTEtZVe$WvS8JnLkf017c4J`K^p%6wVbfW%cVoGyf72*F};p;?C;_~dR@5U zaX%jQOo`bPK7y(JHJn-FE5~SWmt*{`6qzeq)tDXg#xm0uSup0*h8bCD#Y7j5W714@ zn9jAbjKj%~Sa_!fM_jpycWk(f2NI59ozVAq5)d*B&O7KsZo@=S-|h|C=Ke5N zYCE(^GQ#9nGI(##gIWLbVC#ZnIB=DKnhAv;tDE8I$u4MEIRt&VQuNv(DLOwvkrsSW zq|T?6sGpxQ?Vl@8tBn-tbkQ){si8v0OPF~_KS=5xqOUMs{R7DNzl2j2?O>eT08%Ti zLiG52NVYD8cT4lZOe+_3=0SZ;7%+}e@ZNJ5{Kz;8Hik(MSegnwPh;WN;xOo$9}X{r*TC2- zzTor60iL<3g46qVEq6{)VVXuK6xW2!@wa+Ci{yS6R0+Z(Qg~g9M4u^6x(E&rh$HJ7w`hJzpL;M#X>ym@<|-O zVheupXD;4&aT0!px>3ro4XD+HMRI~<+L;t?`%;*)<0kkO0PG6o;lz*IkT~KNoXW3(hs&Gb`++J*_Baju zf|B8mUkcQZ&VY3`84x=5BHW8jhm`HHU~X~{Djnv->8Fce+~k>%>*NjV+GoSDe-W_k zR}jcgbAxBI#)4GebHd_VBt5U5yc5)tj#rdS^5`X2@n=ZGBp_0*(d3X;8dnqWSFGOn zM;O1v5@kIYjq)a+Li-=*p;YN3XsS^y;!mDM_uE>LIQ}JC{zMV)%+$pi4!_Zcc`wkX z3xAN*tuN?1Qo#0I`Zz(y3^$DO!V%#XxTn(&ADlND8}66E2X;3hC8HFsTPmQE7(wsndXM&96-sdjTm>KZvgKsh)2`hBkE z^F6Lg>kXIoY!|6XN+k6b7s(itljQegOuTevaMN~4AyMgAlqtg^_a*~mbpfM=A8rU& zB=1Gd1*zz(%y%@Z(*&oOOW_T(jqn47QP}W<0X~(1aBzbaHhe6D7fjN@t?VFbx_Apk z$elz+c1uuffCZ8Yu&Ma)^cYv*t4!{EOC?b=3(3WsC8W9bDQTWKNUlFsf@PbZkT&ms zBq%`_vh#g`XnFwaG!p{m`GA7PM96RQ0=Z~CFqam>jYvb#dhnfuX!MY_>?HF2MG<){ zTS7ea4iG74S)x;~h?c+JgFc=vMB*qDEK@!iXIJ{*_v^;uyNUuV&Qr%B5t`UY!wowW zjKDg6!?4vFEo?tr4PUP7MTg5fk^hwIsB_dKbhh~`@?H24;h0XO=-Q9=26ZB8`U=fg z{*FQ?$>L`#<*@9hTZkz#MtZ}Ygfdy@#05rrq$4euq&N_AY>G5Yp8K0LJk^3V`~DK` zshwoSr5d6*{xs3?t|c4%E)$c`95Qb5HFBy#8cId-Al`hPB>JQg8|SqoddQU|cb+7A zvFW6E%s%qQGnhE_o+3Z?G>|=&6=dE$HP|)n9r=l#lAx0>$=!ld$uoM5Ouc!5u=OX1 z=y4LcpMI7gy%MrvQzv=4yP1sAIZrIV)RFJ6Q^}+o40&(uL8j? zoMF)&?)R;9E~V3$`*+2gGfvx75x>$)9NIma+wL$;*mkLm%UsjQnGf?IM#`(ml7~My z%OzeUL;*;smo$X^rw%j{!~rFI&iuA3idlaX)k)Sz*&TCnTiZq)R+@;%XyxEf@})Rr zIFH+6>#>v6eSG!NOB|JT6Nf`1?p)k~9sDRRbiazH*k<8<^HcHdsnPf`@x~X-E%CCG zBk|loS?u^q1}j%bpgx~0VQ|2Z5Vt9z$5sK# zw8{XKw%#R0&ss>K^evKWbeUYW>f(YXBeD9o4??+_N=RMmj7$30r^3M%jL>nuCR$@Q z9{J~N74B3UK{m3!Bx2i0qM7PObW10a2X`uno}w8TpRt6v2|`$s?FHU%OI z+?w_i?Q!cuv(Cw4>w0-SA;lcGWayz)G73biE*t%h_CPz&*r80pYSg@UJ@-8Ci?DUs zE)JbbA?0h-NJm})G0)sh>~yY@9!q(UQcfX%+fI|D2mgq#^iOg+^&tshuaH-fM~OJl zn|#oSBh+Cr37ID#Lw%WKo#_r@R9VW!mE~~H)E00hUWH=aswDzae?xdK#T!+t$s!;1 zbHaq65on8vE=s-cgA7J};Z#p+lTBT!q}yZ#sl+M2p(r; z;oQSX@NRT4ls`*^tBM<;OWqCw!$mO3-WamOJmB5!C14jd2BJR=gVBkjK;hs|(wopl z%3d}Tce7Ju*`X|=(|CZ~klI5AR&M4d2IL4f4ExAg#*QUB$Lt^lGdO|wB#czv$O!x9 zJQmn(tr7-4^+uI4)~J_Ca}Ugi3soM@Le{of!fAnC!l8{T#TMih*Nc2OvU?7CzXHb3Qtyxqk+5{*cwaTRyeaTV9z9naN}HjWL?=T=MhyNh*mh`Tx>dxXP? z$;vq-@9uPR^13sb_gPF9uj(aA{@=*_s8ZrEjS&Ac_2e{DN46e+LOg$bAjg6m$?{p{ z+S*Krr@tv?wzOmW8X?)rF| zV=3Bkog%-{GB~m29NH>egpO41K;@U?kTB;GdiQiQTBsp~D(tTaLoa0t`(l@&LC8P` zGa}K@k6Lb^(4Vx}OcEOO@Zyq_$y~$ztz4d3AhF6_Lw0C;kQ=8ya>g1x-1bi<6o z(l9}R?Azqag;nhmXa5%?Uhw*XU`Q{SYo1M9@>BDLhwok_OLrBJnAVTw^DiCYcEuKv zLo2@$$8Yu^-m4Emq>@}`Z;^ma&q-mI6!dk~lkFy_$-4FzWciH`B=vj?*UE^Ny$^lpRK~Y>cU)qi zjcq>YV1<>N@yXG#_`i?NSYyFReB#nroM-BWJKp%=GEZgva_|_6KirO*%c_xPJB!NF zLJ(D#BgL1m5bw58GQ85if`_c?+uG2H8jb7?@~(o+Xl&Z_Z}j!+e4I&6_6Q+ z8cD$8HRN_x6E{ay%>5I!b7t1F$u6Wt3i|`lHr*;DzhWgSy?at9QkjQj&&uEoGc)X^ zHHh59pP&r6Re@-3W&M(j#Qnkl)+ar0+k37?KW zeGEg7ksNWYkR!6ojW{!0CN67R$axtik(0_f!DZCYUK&shX4i(1fUZ_jF|L-Z{E$L=qm0Rfm~!Hu zD1@vCe=tAq4{1MbK;?rbc=XA`t9{z=wnQIF3N2wbnN6AvP-K6b!5jkLS zn*=_ShrA~Yc~|$T!tduCWbDBpFQsxp|K}j|b;&I>Y3xHJa?V9Dnah!b&3SYy?;8?d zKxpPqW1;Jw8excVJ(|3Opyj4}(O^+9dVeuSI95g*X-x}7jU}gryUX7r&e}%8+S!Gw zbypLEb$3ay*FN%XXr0hQX%nI|&ZB95?T9%PCiHKwC%f;+lWvVa;@=w{kv#@!M77+W zYzaS!23j@A*bVxyqgo2Q4NPIbPX`fHubM+|n|o6K>9gn_hZ*$nhd7$?Bc7givZe8-y9qC^OW*cP zqL1@;Ldw$*B;w2lkm^^bJDhI7!bJ#d?_S2cmgVB_TgNb28-1D3&mXYY*_C*Hr3ahx zbqXs#>LF8p(VLwUqRj64Ce6B~nz7~Qs+e$%2Y9dEV!ZN6B(0SAkx*|r^_(Ni`$o#~ zf8WXQW53Jt)4EE!NiDGF?*C77;T3q%$DOV3^)2Yb6LKg*|C>9bOx%6~cjg9S z>n}1$=9@9ek9a{Q_@0Mf`kP>xB>YCOFEYX>(>yEuh8r^$k=4iY5j1TyoHWG~pBTG4H! zBQRP&fmpmhBz6|$BB@ixf~W@yqNu}IcgG2AzyAgv8+8xc zj=hU#MiyYwRf>J}PU41$hq!-N3oaf?!vTwr<9u^1Jk}-%3qpeM{2ymf#v~Wieef~r zAGnTUSGptgP7cT19)bTXb;swf{6=;2&!Oq(Gf}hMaB|~~F7%r!gReN3^ve|qd^R-+ zwzM~JBxtTb0RHC}Tc3cAzdwim=*ZwE$+>ym_*>}6lh5c{+)p%c zWFt;omw?BA-HBuFr(!+*BwQPH20#6tfwyev#!%ml#hVLpUe^$QIDRtFK6$jpcAGsR^DZJpw)>7uHD~fNwXpf@#}+m>aPZobxd(%e4oC*XziPsn>~C z_$4AYP)9E5J|}N?al#iR)P>Rd;|^TBq~k@T3nD^q}1-}L|=b%vpH$KcYT6;Lv664XAD)K&8zk>AeK!8U}2 z9nwk=+c6EQrnDS^Su;qdQ^pGU6sa9<%i+*dzA1fQyaXYb{NhSC8ORGbwWWVBW!su zjqW>$(9)zYD8{lMxy-+SbeVdjS*wH@8r^@2}E3yN^Xsif{>Dfq)YUT^zUCpLKaz(VNYI= zO+EL>{*2p1KXDTgSDSI?R#y_&UdbRd8$jRND3dcx5kfFj>Jy(@_1)TA-Xy- zLD((5Lo9V>3}9pe z(h+V%ECTv)1XPnLup<8;2?0Hnbl@zS`R_2g`d*Px!y^K-OG9XTs5{P4jKOUWLvY0J zC#bOM9qMcn;fz_)cxt!+C%=4zMmBq*hRzbsJ;92*Wq1zpFT2sPmQN`DXBwI?hN7Rz z*HNOt0_}YMS@`(iLc;g0fUbYB5VSA?5|71z6gLu-Qg3n{KPZ}f#uTp#2*%gd^|75J zpkI`ARvHd(0KLQK@f#LW>4 z5}OQg@*Ye4?zaav#;s!E_Yjhwy#mDy1rx1DP6FviBS_!(2juoodG7F+9307xLQR_S8nfL+`Kab-^hR48}@Kq4^?GmUcMS{P(4^nY2K#jPE zs0ueqdR@B!o!AcfHhUpH_&mwW^~Y11wvwTPi{Qi^PRz#Fk|hgHL;Av#!htJVP;3=O zoh~U;VlM?+-%ddG^Alk7?Fa0h+YSTAPcgU77Bh!dzQ@kPa+zxj#xtzcN31=pfLR|j znz_|}2KgUu!Giy8L*B^85cD*cvv8KC`^h87EvSYEmJ{itQ5sZcf&om!GE`>eez>&x z1{t{GM%y+>WcTYWVaB|F5U^SR-S?f5$;apL$9gWUPec|yD8nd*iz(+Dmmc1N~s;NTb z7IWlv?;?@eJqs02o`^@r9YkY~8{u%ZvAB823pGpX1e?OL@r%Zz`09W8cy+TRn-u?1 z_}pF>OPvTS4>V!OUd1Pf`HNBbSvhpOdJKN&8p>UFR3nXfcaZ#hA98r7LKIiafL+!$4WQcviUiq3_El^rA#Uw$NwO2+u#sx)jGn}BP(vatD&y;y4gJ$!S) zC%o0P6w5!mhD(ZLuwM{C6T|w%@VSO?hnK+8tlO}3!aJDq1z=x95zO{2fqdmEIP#|o zuGeHpp7oV*vU?mfpA98Hj+&$G9~^~$hN~m{@4wN3flB0fS%C7!{6RZyOz`Z(s@Pk7 zGM?5Ph~<<0@#z;=aE)#iK9mrL)xDkY8|`6OMW+-shaMGsWn3h_D`rBHLNa7*=HbEE zYcN>7A3{{KA;)?@ycuB%mFvfn=aI6)fIB1cWyQ64@t1Abc`=V0v(i!Q8Y7Y>Z-{pn zsiW`>StR2{0J$@`7PiPcz`*&p6|Hmqko@#89MrM_Zr zmn7j?W@m*9goY%4&13Rsu?OfTZ{!3Eroprk*C9x=uzuuC{hf>ma1uP6e7~21g@1#X9mU@l*F&9H!iaZ;gF{ub~Hcjp!4;`E{7Y zV>N^`wG(l(i!Ao@Qp9YRtt98+h+_*TV3B7esXKlV3RWM5C+v`94_c1AWM>E)H%NMP zXJ5gS0~8j1%i#=BJVsad;!n9*XvSqB+)eTUmo?jnS&s_}ie6vwKJWpFnmPx9x}#zI zm%n7Eq!?uRX)RVee;Vg#?!wJ8({cP+DQ5qn@l15TKC}JJZ~XmyB4(E6p}&#Z_)+a1 zT-!Ys8|%l3k3LK#WpgIM>-H63VXFsjQQ=^_ehQe(ze$Wr18>BaStDok>yEOVb} z$HR25;MBr&?7i$19-gret893IwmV7T2bE*-uMi(R;d?IjeOHUmnPuY9##6BWF-<%r zk_femvWV|d1<=!|mh{H1lk5wWFcuexugwFZu*#4)JNF2RRyCqxKNmb!r3nSN93*u$ zG4R&x80>ks1{RE(0G}pYB0E}S$edZ45c(zz;y%@qcd7B>$X+>YJ#`iy<@6pI=ZxTv zeySEuzo?AuE(GC2q#j=>?!?_$V(jDFg&XU}Gct;pDKVeO#7&fD7T&MJ(VGfzZ$K{A zk<1zNMk`|7y>c*m)I0EAt4X&s8qsmq|G|U!YFNbIf#ppq^oV3aJlD~c?!Ws7__JYf zO<{lx4YPtBGaO+ayd{tBVzPVTc%+;25JhWfV%n~UQ|3r!Rsa8Oc_ald`Z*m(+%d+{ zs>5;BSOL1h=%SywJJ7T?bs>|g4JV3Yz@X$f$j(lL=L%tv6Jihl`mI6YxFK}MT;aQt zU)<~ZN5l<uf`Sv z9#_6ShX3vh!Ev2dl5fz>c*BE9_`rU9?tX9?@mx3@9AB70(N!Ht^}J2)4-SwOWy9dA z=4dF3_)5z3^tr+I<4D)`AL{vQiq&sBi*wuXyCC?*x%sk1BzwiLNR9(WlM*>N1vKzR~Hb-aky|7B_ChXN{%`q7- z;k=I;W%X{sTth9Wk*$)@TsgAREETM;RKiHM zhss22l@6kSan79_d{p5$K6QE|zT|2~#0ob^*~dtNE?-5})>ClydsVz!D;LcVHAK%R z7og?o({RS^LpV9v3t#R_KwURYL4QjgsM=J3UiwQ|sV7a1dvxf%*ACR~*eDvKy%mOn z@1dJpE@6_u;~qVVKm2@&!vz`)6JgGjZ5+UF;#{#`_ha#e)5UOA^bc%Syn*2@+ElSZ zhprrwrJBp-sjXZ-l$1Y2!SOvfv2`Nz_JA`pTWrBh?ssA?t(LsFS(BNw>&%(kl377r zVkR`Y5(~@!OM+3e4?|mU3|M*}fLRw5Xr{ddHEnMN`L9Os$KMxiPac65N;^V7-VD_( zF7VdsJy&*g5+2cLgxl1u_UupSb=c9YxRrVyg^l=LtoNQK5Zr1hf+dH(7`OHLSI-`y^FqWdK5{$&h)lz$#= z9lXqqJ1s}T{lAN+mhMA&ofnXL;3DCw6*I`}uxJvHxsER!fR+N_sw$ zI^mG@R_e&itx#>Nhwbs_WyXU4r2RxYjF3P`S`&J5$2yP z#LirCU2LNljOfyM7I46nWxGCcQXil%vXaxT>`ed3}9Se z3aQ(EfQ;E&Oh%99$fE56(0-^~qa-B>2vnc6CD^-K^}ewOr3-;^Ms z@o21DHUclc{s(=^X+nJt&k>yziTXXAkZDS%FyiN2Vdc$}TyKYfJRdwtO1C#j=0^rd zu4g0Z+nqxeKdd7)rI(1;^$t$TXSifZ zHE7!iWms_c3z6h&$;$__NF})#HTWGs{&@+4kvS)bUAYuwukIsDpGZNy@kltdZ5pI# z?tr)3*TKHsqd_OXnTvdQ5Dh*w#&vHZu=BPIJU1i@%Sj!?o1(n%;Fnj(+b$DT$J|6X zK?9$RpMi^?D&u=|)VcO`Z^;9ncJkUso^uT@LoPo$QTN6rDD&Sg67Q=5U5^c*a-pOS z_H!7#H~B=GqI7_M7eMkNbI4zIjp)V}ijT-AB7>d_=*)v{=--lVWIE0s`$q_|?spSB z>V7V|tsl=VRaGTBLf3K)nw>(lGgU1TB;}0|zCJ38s@9D|OXNa?Gi@D(TE`!Wc^`Am|MoXw zL_{)D2qp0Mjhd|k+Y$W&}yw5ELac)_HoJ3IY(0e{XGlRr53@B8;4E8`0>hF< zP}z75A|~oXYE>n2&)tfD?+V3Hnw99KBN6+}RTI8Um&O)NX}I^^Mf`EEJML|c=cYWI z0^>qgfq$$FoO!yKoF9D1eK<3nJUsQ6q`K=?y)h)G^=$-FwD&4k__!7N?-1v|RrVof(g+7>D8h`tB zJm0u3gdbsK#vkY_p(bZs;m!JKjLEfn=KNx1Hf5m&E1cxU>b3i^H`V8}_wM?z6HZBT zT$eT3hvz$)k(!xI#0NDdEHnU4N5#&_P@NqCR- z3_qrD9KTS)L`({9q{H9E(bw}jV0&!|n)G%!W6p18Y$sl3Ha>a8yij|~Y#jNBK`lj0 zQU5k(ySx&!eD5UTa7SDEg#~&>cLZNA@ui%s_vG1TKVH>r7GF2lli%yd@Kzz#e4r%H z8M&#BzS=m4ZdtGdI=(C6-^b5k^B1qM&+q^6*1M8ANy}yUQyohWy=ta?ixv4xJ5BiS zGd%dc=_~odmLMnnZk3UB^;X?NZDFK=Sni#mirv}8m+l}P3IQ= z+WW)2jA1H&uEXbgYDOPi0Gt;UZ?Rp7Pcf6=jfK2qiJujrZCFX^TK+G*+Z7Mi@Io!)0Z z)0mh4_u*Rb+Zh+$;>TqEoXa%c)@ceqRxIS*JR}U}wLj>8pHI;eDoj5=S?2}AT)~~ zwRkCya!jcG!f9;pqNi-YNJCK*<0`tAuux>XIb5_wCQ5RDw~C@2f<$S(SX7m!Cfa%L z2D{x&B+1UT2>W{NC9Il4{>}Zbe9Zlkm3r~Ul{J41D$}z^RQB)u#$S|f;$JQ=;-44B z^ZF}o_yO}+dU5wFyy?|tX7h+|%~k|3;2msLiq-hD1OV)gZ$afqkL`naf##j1P>jl{AtM=Q}c8i-~S|# zS9B2Zhty2?>=!cppy4gL@^CaI?pk!i(HIz!(}@Nv<(O$FwllWLh0NpJ8;rW)HKyTf zGD9;aGY#^!c*=(`Me|UI$+*Ygp-P=}|dM zI;e8Z;;_nX@=Gh%KM+-pveK@!TlbXTA>sVI%9dmhwbtU}5)av_xD#yc#ySZbu!Fs` zqlaB|_buxb{hY14c$;m{C}5|~+QU|Po3nkkY0R}RDiA$%m^PpIPGxN^`JIoJ^0y$7 zzbib$8-!fu3y)po`A>O#cIgTJe)IvpIddU@`LY7v6FP#Zt(KfH|p~`kYBqggIC*F%xldT z^P{Jg@tI4{@+a0D;R}Cndhk>S%_@394gP+i+4tr7+j;8z zjc7yuow_YQXCLBsee&Qt|C`2}M7Z*NtP`(VJAt3rrO#JO^3vjMf9ci2mo(_;UD_X1 zM#rV3(Ve+#Xz+-MRJE}cS{k&#I7tdyZn%jJ&dM?tL*3Y0I~6@YE=#q(6w*@(YP^1$ z1K;(|jkoNv(Vv8qqNtfi zL#+ zjLLiYx|LUNHtJWeK%yfR$@=m&K>qhca_>3{%Fo)e@eT0?$cbX09zQBI$JuBh$X0T%= z%<=RdPc}P8ntkEn$|PC;gqWcwTKC6@KfB$HKilumuUHbye{5ULKh9sp|9m~4CzDvd zd(edcutI~kS~-$$9BIg}Z8n$8X4vpoeXV&-6*Jy5Lx;CkRN}f?KpANdNh{P`h4|2Q}2BL&KNRT44f3rS^2; z-?#YLbyGI^1(j^sV?=iF|QDzo`3$MtV={997%7j@|(~s&iM5cFvhhgU6ku7YEFS~HBu;sc^N+weeugkyrU*}|)?^qB zd#3J!F%#jGjjtZs0cX`i>E-xhdU4(ZI%3@y+PFrJf4W_fulZX~o5x+{+U#vu!@O4Z z#0fhQ*|bSCtSDX7mU>=vIj~SPu&6+!y8F1O4Q&?{o}VZhHj`(?Ci9rOtWA91x-a~v zY}3knN|P&>FZ8c`JZX7lozt4i_*21^E*?IWn*yvVOG;!bwjvySSqtWT3Y z+iG`_DIa?Q?s@)44?8U3Z(caT%M_RKL9O-tMYk3{Qt<)5wxo`){8_}mdKb?}jg@%l z4jrJ~H$0f?x-o26?Jo9*eIC0(zKm@bma#YA=d)Ws9A=#!d$Tr?@0qL|MaIuwpVn-> zMhovs@jCOy^RnSyyjOMz|849R{=?fX{J4AT`P8%({Fy1Ucx#ahKhoTQSBO*O*M9p- z?^-;gPuJb2&n3K|YmOE4pxrfUr+S87K9E9{KOCpOb&BceCC{l#r83{enDQC3TzKhR ziT|g5I$yrhlmBet!sBY)|50?NfmD5695$0Vgb+fJCP{J6-sfsSGb$ntRHT8Vxqp<9 zWX=?lBo(1#$~}ASn?xZ>g$6^?q)D^VT-3vgjr?bdD_zEml~$;T*VuK_q*;0N zFDb55!O`kVjL_r(Zw~{a%`WRCo2^n&AQcR^5-?1)!mcACcKJd8!maVhpP-( z#|(nmu1ut`V?G#Oe+X}#jA>1oBb7Jvp)-H&pw6NDXj=VF`e4^a%I;Z8fBzEFuEOb5 z;rnD7yK4e)Z`SLxLyf2+`q&rW=w|n_9sE^(?d8cCrj5)9z*AKYg1-tCcOC;fg{%^ zvLEFXg>G*1g+F$M2?v*?31_~~5q>Mq5nfuFEPOF_r_ip#O1P=2hQIbVV>ulUy4s|T z((|ffh#n{Y>}(@Gk;90iYy{$~VOHWqDI@XG=kj919i8;`nF88ty^VH9$W!UJ`fxzs z4F9=Vi=DduVCB7P?8~dhY}=(7>{6dqtc6YtTcT9N9-Pnf?X2&!B7-J2)uNPDz7olz z-TEx++K#L`yFkKbE8TFoh@J|4Kn0QgbV2o7s=l&|x-Gat?bGvVz3w3ztv8N(*aX7} zWQPrEc~(WQBU?{4u&T?qvHRH_teaK<>uSG_4ScbfofXTo=k;Y+E$wUAxGDgfWZpwt zoNseahv)PA^Cqa;t4p`uUOto5=C?BKtVY55Wa9H!S2{(*a)64WmF3q&9M41R~(V1f1($S5xYn~w$Lo77Kix%DLLaaLXE)Vx$^ zgbxWb8&ZYp>Dj`XLnnlxl8M5AbwR=(2d4^cN4B%I(JlCT!&qw7b(!A&I7I7StBQl( z=!;MLn~EKL#)_ZMF%}PB)e>Lml@V_&e@CCMs-bUdPE+0J{q%6o7MVcS5j!>Ts`a6f6;1$Vo6VI|aCStbl++DW&O(WaYf0h0GFO~hZ+n$|Yl!a%0 zvEj>$6KMR~XnKW|(dX4ov_JkX-BHy*vo`a**YPK5PwQ6d<~fRPy0sBPjim5}rdK$~ zond#m1+nj=BiR{wJR8yY0QH)A<@2G*i$FiVN3*s<|s^l9$HK z*6G;&RV%j4|BK@cMzFCi671ZAH}DF-QP^nZbU1(O7fd}cpGJotq2BjS()fyV^n2iG zTF*1zcP#Xyg}Rfd)S@?_1Boy=WDPP-e@TgwB%pZ{!Q{Rh>_3zSW<2lo(8%Xd7a>ap zU5Ye>k)vIeDzx8Bf|@Xoz<6Fdv^v(4vK0s^FZRLC^HQ;C>Lt8gnqg0SWU!0Yer8*m zrVER+RtY6aBZLz(Q-#sH(uI8m8N$y`zL=o z7A%!vHy1r*lHWZ8)!FV;^j|z3aUz-i`4vq)RJYQ&83MX0Vif)2-3}7bv2be#hNI7~ zl3t$zqV;VhIR~|jakw7(hqF+AS2D8ynZ~Vq)Bl6a3>{Q5SbeQi1g| zAi_G3d3Flg3If2Qel?V82Ed8(W3c9L8O(tuxc{*gR`WiOdeZ{DzdkuWj z_dIq>e2;@SDY4+8z_xaE;&&-W@U4}NOyllA7~~xX-ER!&os+ZZzzr9A{OVe&dv^^j z6)&TcU0kTb(OERsY6?{|GNCgxM$wyDBj^zsX}aq8Pk3?aDXi=00G};l7&kW@)(2~X z{r7!*-;o-A-LwWzsm;Si4{~s`W(AI_x{H%c3$S4=haV}W;Pt&HutDlPHgH@nn|WJ8 z7@m!U0rOT1ul$G*z6(keX3NG14{VDPzU){h{IPYs@NRe~J9woBJ9W#^&yMN5Gqs-< zE&WT+l*o%0HmZn)-z3C6(}($W+D95_{gASxjlSG{gL)klQT?t8TE07l=DhWyGoz%b zr{f*cpXH3tSPf!b)`?Av3}rvh$YyuiTxFN)USn_P0sGCoh`rzx%l0L$WqsX8u$~34 z(7MUVpnGaA?^#Qx$7Qb23%0ju)`}*oIEV1pb}_AYil;v7)>4zH6Rvt;dUKZDu)jT3kX^+Ow%feFPoG z^QkNP13K^QL4kXU)c)1r_;givZ^Z=G*l{Ym*+rXO()tw3l<+*Bd8ug3(_W&#cO6u6 zC!s;+3|!f|2h8$T!6D7Du=dS$k{0uVC$keI)7V>c6WC8< zkF!U&#;_IoJJ_Vu1w1EAk`*+1V)n>Rvgdm>?0F|opU9chhq4Z|{emm+M%qes{SH#& z)nRmR{ZZ;LA)XGtNTidGrP2Lw;%TYzE}G@Y(%JftAiVz~xp906ZruG6pKBk(?)A52 zQ!HHA_G9ksdD|sym!6QlKiY^@ekZ|R{wv1sI(=~a)?DOXS4OsGMZ-STPFP~ALBCJ6 zq($a4spGdfbT&7i+S<9$p($%=X8IPIp0S@Mv*FaWf%gE1?5A`0`ca>!LYlc%g&NDI zfS%heZq4>7*uSJ4?|$$W+u!CVmc6(PT}mg_9wnA{7kxJcJX?Qq*|69HomS z>H4n?fJ!34fptP7WQ^I9ra7#4w6t)~CtKm$;g!OIq7dP*Z=~?rk`Uqk``d))z)|RZ zccjq#)-Cq_{&8$mog$j)8w0%sinMV23z#fl28-=)!$|Att-w1pKY88Q+YT zU}e{uux6dsY>^zx`gwS;Pr;KtIq1cv-u7d)(|lN0~AWUs`hUJbfD0K>rMP(XfJ7^i|p;YP3yE-FK%`1u~0g8{dXe zM&HQt*O$>a&DprqcON!h5RKgxBk)4G|8Qc=c3dC08FMNJ@Z0HW*k`*K?@4%!b8kPy z=6rr;D0L2>aoCAVGbdq{!yahNmWyPc^E#*x>xDdXo*OvDmrj(5p>~_o==6?s>hM05 zIvt3n1-tpZ-5U#PZh8y8aO2^h?`y_VI37Fha>Gw-J@F;2nfM?#8ON`*$Cq8r@y3C1 zxMI^Zy!6C$td+A5Z_h}@pNunb^~wS)=bVQF^5XI4#{KxtlyNwC(lHdVU<;Wrbq~}< zKZGVvU7D(CMZ+%H(LO0V%Hh#8_1Q1j@M#l-m7c-d8hv;dRx2yBW3+JIyBWe?1xtln zu$M3)VZQK_DkC&A(H2Jkxx>nM#%VI?>#4Yw z4MRNF@#<8(pgjo&?gh}5_rz2s@E0|Y)fM+1G!}pD86%#bp&^d{^OdekXr=P`pnYVjoK5j>YJ8f~Zh)PZ0{!GT ziTZ7^r#}}jqJ}3I(|-$CYQ($vYJ7RG0_p;nCzs%Bc_hpjzaF-I_W`93Zcw#-4IDo1 z1-e@W5aKivw67-<2f4F$;o--TypC)|my!LudECQ8 zsxUnMD8vkuL4~LddRO-Vz5Ed7oUVq#<%Mu_WG2kQULbOx2{Phn6xTa|ZAY$Tm2QdH zB!@wE`vVnW(-AY_u{cxVwj2XtRtN7BTlb0G;#bSAs6D|N`z~dJCah$oyn@*u7746h zX$8B~SHvzqR>s~_O=pWYSg-~jYIyG*N&0z1COz=0o1T59B9=KnMy!>hC!XP`CZ755 zGoMe^Qti;Q)b?E@?b)=R{&V-B@=A_W^#DsRG*73|Q#Gk7c?tV-cEhSE3CJ=j|{y;{^I7 zb_}(_l2lc-00xb8f$QjIj0;V%WYA_D>^%>M7N0^1)=|V32Y|%FYtZ`W4@~sap&4Hd zsC$+Mt!w)P`l~L0()rnNt#=&>_-={ngDR1NZZ@)+jB^sI3CG9u*bF%zS!r-20ZA9v9Lyjq#LIZqP_{XMKnRz zE_o{RVGLaWmb7Z^WcqTX9=&k+BP4j7gi>pHnEFQsIT|6{J~|S=mdn8-W}d^FRVr{@ zi6%SPv4XukBaM|_)yg|7zp_*QQxv|^7$v+kpe%e)JVNMl^fxP3gv({LAcF`P;L!gQ&;#}L?5-he$xg)p#Y6MR)@Aa{$_ zFye3<+-zBj-`ah{qgLv&J-(*wDyQ-6Ie&9DYNHXGHdT{F@4jNcsipYJ(IwbnC>k{k zf8ykMMqRM%5NWU~As?oVCL3qTKRStkT+!HFNPtT#4OZ$_&i!3eg_FGzoH4(Bycm5#SflJ;x%8sBc0@T=-0MuNG;hJU0V1{ zq`f+eWF=34kmSR>tKlS^5ub)EYzOtwjZ8jwV9CY>?27(Gc1g%Z)-|P`t>Yb@t~+k9 z6HIH^+Qodc>8@gS%Cd4cf4G#r{e`e&WNxu>5!cwQa;2<2ZydwZmDpeP?&!iJNm_a# znBE?Bme$Mx`eHAk|5;w4Cwxv)6c$0J`1sOb>4ki5?LsxoHqp=j!s+W5DO4gOgSI|M zp;?#1Y2o+z^slr$4dgk-ehUoIn60rmeAH{a%u0nVzNo{FRT#xyjT*sLFaCu+{krjr zm<0hOgU~CbK)2=c*;}>>ooqUi9{8zEYiH=wt;-FlkmX$*x|(#d z)(D!_{u!nnX@sW9$Kf?+3}PQKx6$AW${Fj0og1UE@8vZ7H!c>N1&3hALr&O9>m9P~ zdB()P%^)ienn7H%H_V9G3k}!Sz=1%*Gj;rb8EvNPI z(M%;QjIBh=$R$SNLOR)I!%>>lwZeop3?TZh#GM+Q z38X)T`)J#;?G&x?pwFgT(U!Hc^bTLamm577lv|dOX3cCQ<8Ok0D7#>5+ig7S?GQdw zwG)@!_rPi1M);s_49dJFMLO2=PKIMKKniN0wYd+@Ec^n8@BpOj?}SAjcY!gWkmP$6 z=Jwx$`TV{AX}moBxn7sfy*-A$-=agWbdI2tnqR{z|8fYF35Jy(av;l(rK%RS2*y;l zF&q9xGmRlfxDhM#h?2xq(j1S-?y%XUddgOEb5SAbD{3d*RlVeJ>J#F$?=3NYH%Kaw z8i?*#L(qj~Abhn6SZ*2QZkY&PCzN5$W=a&cuOk))BSan7Ly?oi9pu;1k3_qM(B8TI z$oao}$kcuq`8b>5Gg;m^MlTx2>c``?N{4a!-=+A*DMPGK`35ZxIE6;Hm?F!WQ<#GX zS_FHpv+0~;s@Y@v} z=5ZhEZg_=%%fH2;J>58@<~sftcLBeZO2@;pXYu5NxACc0!?-|aEc>`tz+U?5$i}KH zWWSZ!vkC7kSoiTNtXyIjK5;)CkC-ULrHS1rneSEk`??VMGifMf{8nUF|CWiKkj4F3 zwTOIOnLuI-LkVcz;9uSj;2SjV}40BK~D}pgYOgv+Z!UF z`{*`Uh;3m@krY_{&Lz$FXAu+Q&E);U{bbiFKN6pzOPn;7m_M8cVtn5t+c%E*PJSd- z4$sEp9-PEA4G~yRZVE2-&qoU{t>7jHWRV?SK>Y3=Ctph?xHpZ5(MAJhZ1K?%Prtnq zudQ*$;eQPAuPP2zn_ghXrm7LW2`QxWWFg6#l}e%u6UpI<17t;y7daOjKpLAIiKVg( zS*ugd_f{|CY(^G}p1vAnvhw)e)&6nVZ$~)xcvFha3DAGXhTwJIg+G4UcjtMEJbf(7~H5BjqmB6$5RZhV(YC}@!R17+`Bggmt5G5 zPifoZj28-6nA3sGPSv3T??%)b^&Q<-9z=Jt`VePbgqF+dqO$tQBx$=etXQ!IF1e<{ zCXWknWc~%nQ$G%=b%F4+U;_Bu8XywtP2|tqX9VrifnyyDpfNZSOt+i?&zl8Mza^J% zBRL8SlHB1sOohO_G4QI>82;?Fg}=&cVa=#$XwOcE*KbZkfZPQrtS^AkiWk99F&|!R zOofgS!7%mbBv|$DK9R7oCVxJvqJQS^k=DmKxIrQmZ}LpUyR);f!M!9rPJ1u@Uhax_ zlxX3{Hnm9S?i|$6o@i%1nsU-{-?=ZBUAaJE0W-}w3>|b9qw&Yv(e;u>^wPHgZNb{; z+2LI7#ffty`HcpAUT+82HqC=qljcAVwuTdmMqvJWn8aD|9NgXtVxn3|>V9XF6ITzC zCxNEKqR@(Ic%F+Es#xLETT*bt?3;LE+;c2y{=`EQyD)xNjNdl9U{-!5a+f+z#z;%&ToD2L(#d z!JIRSL}z>Nqr>Y*;6)1NIAZD~JpI;KeBprwuK#0-i;{G3eJ1}L|7n8ncTdDe6KwIL z+e5%pNt1f=A?Y3o>ia>m zz4c&CH-@wdABYb<3~O#Cf>XtDDBc?b-XVvf&v-Ey>uG~Z>N_%3vx;mRx=4Kct`YOj zHgYq*ll0AKBf;5c$<1gNazChp%N*NSZ8`EDqw>a+Z;xJvBsNzt#;QvvSuG zb-J6hl};rm!Yf7jV@IPI{%29)>c2>1v>~3OF$w>Y8;6^XzoFT2lhFvDxx{&_1jHI3 zP|0U{ z392|W)5Cu5xq39?wZ{S(?~FwzBa@L#QaDmFR6}kxlgSDrRml0g2rTqHq57je zEHisX9u+R)hCM2f{DEoM`H3gK7(XAo+-9)ln~6C2=rpWZH3vuPuEI~Z9KgAeM{usp zaoqGX3$Il;jpyww!p7blR+xVcCvL963u>z{vdG77GBMaob{S5aCWQ~qk3x=9{t6;& z){v*Wo{-pyTG01#W93T*EKhNH**1)f z{&VI|rw!T`XOC8Y=x6diriuJM4s%n7Cy^CJhNN|i6e%vM;=cVZ<&MZ`kXg#UWUWR6 znemW!m&nb4Ym?{0YuANv|DGj8f9BcK=rCC^*?_ZL^oVi$WP}vopJjaBNs+TdmE=gE z4D`Jo1GB3wz%$MatYS6bU;iNSu#F(EzJ3)*&WS)=^EJeoMxcWBrvm5OX{5=*0!KdLUeWNO>Hv3Wi^5Y>QBc}x4yeC0>um*gp z>?8k5Fj3g3!+iVlgQ@))iOvtCA*tb=Xzblm#%#F`v6Idx^ExFUr$QSHdPak^>j;oK zaE{28k0eE{)lA2cXk@y%4cYDFIUM_VuI{oQsO#`Y^m6VW0n+7ZZmTybomlW}A?+v2`SqpKX-* zDFn+}2Y9;C0yZ=ckeKBcNcibBWO1c3vGKGchAvs8F}Rr+D|C~Zo~Qh0K;8vjCdc*f zk}W?Uk!82r$&MpDBgD0b)Fw*9%>%MvD0)crv}cmleR0gI6TNtnk;5le9mSQtt;ptZAjxX6hxytuptUg#woge0=_|?5b|eAXC8OaJ z?;P-6xExFZZQ<(#Ef_VUf!tOOA)W#i@^XJYw<*X)Fwp*&S>wTTA>b5>51xgJ%gk*% zbs|Zh`UIerQBWFO0>aK_2srB;c<3CG=CYp(Hr^EUh>}?IQ{f2{CPJIZeR5$S}_(#TQVL=Ys^MdJkB$gvwjOK9uVe7 z>T0y4<{Yx(`q5oO8Jx_%K<)E-(Y9~($i}P;b!lHl;#0Gc=_z+^(=7+`Dk6wj?wLul zAD`s1tVeSpOZIY#vH|3x_EUcL20@rw1Vn7lfOo3tU>3Ongs*Rsj7}-!>irY77tg}p zhRg6~A2;l<#|3|4FdiBd;BrSh9tdxV|2>w%7mu7k9tDYvV(M8@Md&r=sY(hO<{FT3 zCP5$FW}x{U4Wj@2w8?)jM-kJ!F46epDM-8X0qT5hjAfJ;;8>-V_|RM*T*-LiUluGb zeX4}*7F+OB_?>X_eG9P~GY`%8;om1V-q`$c5azB0oEC~hm4x`xu1AH=P z6fR>4GCrTiy!7E`;KqI>S2t-uqn!?XOg4brJF*aWu!D4##*i1ITt%@wTm9@oS-gDv zWV~%afVWIC!#k3j&@91bCV$2iGP~CV*7(kWB&{_NShx+o<;21vnSA(fS_Q~p3Zm|E z7*~)E!l|C{Zj&6ueOODj1e@7StW-h!Wirv@9e>e|ZPpk{{qTvhNc{Hm5qxFIX6&Rh z5&x=rj2r?Jk-Gm3H219`ieI!CElOO6PHsGc=H*8sF!MmJ4wDdbqKt`pCe1`yY-8e{ z*E5?p+90O!7>d@tjmp;<hi3h=oyM|>sE89&iof+ZZ?@XMVpcuSKLcA%p%v?L%LRLEJ4>?GSZ zm_dNmG`N~J8lo0G=6S2z$gErPL~p(&adM9!&5hk8v1uGEXj=r2)ZHOqoeBuM*KksC zkC0F%2%zf7Zt(kmNHO0UI^n;GvRFeHt4jN2(7>WkZ(sml7D(? zAl_#UF6UN&!DByYy0RVwkq*!mpaT1pqlmMN4DIxC!1~ECNrD&0rJoh!i}0DR?_rgZ8Z$iBdsAQWBdTpootR@Bn9KOlh5H7Zntrd_6MwbU!LuorpIplHkBne7P0eWwy}pF zMYDa6)7iWS*=&7fCL2B`mEHa_mR&YJoSkaBiSB0U3s z&fW==7N$W~Zxy^%dU|J6McXq37!9p9)f%ZNuhWd`FD3M$}ez@+^r1f{+Jxj)q~KW`tLy33#8%|PN9~;u`=0^q zaLpqQ))^f0K912ynuW-&3N*TXB(B-K1eP}6Ru=5#H zCx~Q@RsZEuKECFAY#rh6+`|wcp9p8i#zTE}B+t$XhiB!{z&0d;rbjGjUR({vHoe3> z{|uA4eFz0gEWirKcj7wf;pNIHcvMjw&Q0{g0XDaUnNigA1HAuO%_ z52+7Z2!2T{AUE?_;&=8ZH>{Y-nX#SRP!J|*eu?B-Od-MljuRa(GqTgYfO$JnjBaS# z;(agTvH8lYc%$tj9F*0E``mBgUFGR`{D$dx`q(2V#%nZ@7(WU|E#3iUu{p5TEe}lM z65!@uNBD9{MEcyH32u3JBe$=!u)kR<4$^GJb7!ltr<5kK;}tQxGRukWo$ksiH2Sk4 z3ZZP>!f>|Q<^W%NT*q!|nafHInzAKJ6OyRwId2<`# zk=-*e+x!-em=8lrxelGr-`lP>n9l2C=*K&@bYJOAYVBc52VYv!(Cun;O=mA$7<(3E zmE6JG(DP$ zp=6J9)K*~0G-o_ne*(7uDTTKbG@z)lkYs({Vwcf@&TF$vULMe-hNlON{5S!7S+wptG|y=@QC{Q?gx zd9)BGRKLSxXX~;4JDk{l>HpZY(q#5e=xKJ(+_UVBpj`HM%xShv^AwwK@dR7UyCkPh z+sRrrE@SoA%w#RM8nK*?G@B#Yg0s`I@s#~X@U=ZYyq9<~UN%&W3=Woa6GIwFj)D`o zO*{pE_VhqNy&6B;WlrPPOye1uHdH#xhOWz)LZ8Zvr}wb|Jz=d*CoNT=`>*|gT(1}K zLa`m9v#!I3OB7DOBklf5xrYP7D{nwX{8uNdmp23;e=>Bf(apE{G3eClucM5S!-35H? zeGU%eox-MyN3m;(Ki>1(49h0xqnwum0)snaNiqCK(mo`R0;{tm!|M`JuqY;udy>dR z3nA$mAxZYA^4!066XIKIO`6XFr{VXN(Ob}jTqVu1@7c|GuTeBEZ%f9rBu?U0_fF#X z33+%^_*uLrDig=}$Kel6A$ZH!09;+R0YBQh312$kk2gB3!uiDuvFQs_{M1mMcR)Wu z&d-X_r*^()?Bjlaqbt{rqd5IK;cal3*lHl@88)m(; zgRS=4!1VfY2wR*7idAP|>ai5a^f?H6%htnOllidOmgQ@at(i+=W*HNYh0g_dM}?z{cfTU_*ZlL*F$8-a&%wV7%W$iy2wPQhc!7E` zZiq?2tEmqz8?TPf)TW?`gKo^J*P+~(y;nG2lSIz-qAQpGwp~;)<*De-y4l<`=1NnSpb|#r=-b=PH zQqU784=c()lEH1y$?h!~#5X|1g}M(iVP!eU-@g=v*d`;JCRv1Q)42CB@ucnIU6MWc zgdDwohwQdGN*0dq;Cjtm?6hyFAfEsSsiL= ztU)f9n6!@sS!R=MUoVl__SNJEADP{`#E~;L6=eIE%l!MKoH$$DBu^H-CrNp_uHp>r)NhYpy~vSsn3HD{{m?2D1h?fd>HpH8`P#H!pln`U{bjn6yIUEn)k=^_}cEtO&vWMi3-32Mk)gK{+u5E*v=mv6I5#<;+k>z83(aJXXOMJ4e`b zYz*i-ydiT}o+fLx)JSdEBT=#0Pdn?eSt3Q=OOse;O9cC~h^E93o;x!Ng4`#7e7QAf z-y8*JzI-GvZ4${c)eGEv&vQ)nEMGLA@A2JHbPnx#coMB!=Yi~tWRQR8S?1qIN#@cz zkpS4cqJ{uN&g7>Z7kyP*aC^)kqqr^`&6z!bBJ|2ogVj$oGj$|R?KZ;^Lo2ae(hBU} z?Sb8d!FT`#;Kx`sO5LqE?fe!@Mh4;TZyWKR z_baiNssj$@nQV!fBXFqN2Yx?Vjq<$G(D?p?=zQfyr02Z@HP22(rFvJ<*!wj|F1QGF z|JsL^=#NBt8;1n96tV232U5_ z!C#-fMvt}2(f9%vl(6nOb3aeOJUQPckW*J@-k-b7sQy(z3x67*^r?$VFw~82T-%n~}RuVl^M3g2^CCM#PWZPi{GC0wiXo`bK z>x&XHd-w|}=j-gFv+bbwhzsDQK9E?n1ClQv1dkPmU_;w>m|VC9f@564)5jjpo}39m z16GhyGYLR|?1|O^^=^IW`=JYmXQ+c|i!3-O4H3<)Pf4rxb>0_NM^dgg zk!sg>`xV3F+r4+>wZ<*7wDu(NUa*cF{-{Fam=)_z)dP1Oi+(r;qr_We@d&6w(T8kz~o5hobd7Bkhg2iJ9|sSnw`~0ZEwdon&oDU_U`O0j=8$`R4#Y=+ciy~biS|S-qLVzGo19ZDFi;gUvO(j~QyX&> zGh!6_&>@R~7BmVbZQmvGo#e_b%#Y-3x)M1(R}C(uUX6R-w1n%+T*)Q5RC3vA*bv!NDxgY&&;z)c6m4n$hRSGQ(L*TFCRo2X2%4c+s&CdAN`oHsRx;M zX)`9dMpLlQ*%7mdxMyh* zle@>b&mRO_XUSF3`M{xS^|?-0h4TzV-vZqQ9^UTElnWWmg#4$BgyMVVaM(ZQ$*y5$ zaD@``wH}4;4gO`Ot!iM}FFG@=qSpf1(b|G5kqV+t(}~>azG`meTHqEvuHtNE5U0?W zDcZ6?TJ-G87lB&)B4+HY*V*<-s= z?%nrDGy4lV=O&FIfX`2N)S+R&)2Q5gK3XJt#Jrp-FKCQA%H4l;n6o*c%9(zOvvaup zmT8e(fi$0-Mrmtrpmo#kp}sG-(B(^KP)hGkRIjXpl&3lhW@h_xxh-*AS!WEl!9%~5$_1wS;<5T8<|;EG0Yy{OM>;~Jln?l12>^`5qbS6mbjcK;@68ndKB*vFN;S+ zkGv*r-v`KmR5w{+`hdutY9$HFZWEi9tHf%4J~=KpLTX!ViN%IqPQq}Zpw-40Nwpn8 zdxV!!dwv}XExv=+$lXLUkDo)2{_Q|l-s_>B897XG*i(Tg-6!~H*eQ^0w_w;qKFrkJ zlv%)hVjMJYF{ggYGZ&{E;wnCjC1*mSNkDD|IcD~VtW1AJ{BFJ_DHZMH?ZqlGd2=|q z`Ou6U&AZMy5KXSbw%@MFX%W+IzLL2VeUb6|Rlu0MX<)Jyo-TL2riwTDmwd6z}XHQ;G8leIb-!R+~DUOT(h<`mrxlgxHmqM z(TV=Tn2R(}btOirp>`;x@*k6!UC#V8`5>?x;cpiwb3v3lEl6OU5yq6w{lYl4c%i|^ z3CKyW2yJ^)iAI$dqq_kKsP&s0vN>*n^v?2L3^fg=z|CKDzg?Nzb!9pCGbDjqwD%Yn zu`rHn(o5w$R~B))>%+N8tMa%@dlR`&^3I$elH`&k)?RfzmM6IL(27~H>n?M8zcG4f z`(6^=I&`m)j6B4qWiCMW)uu?*q+a9jVS$y8ZZQwbY2R{vR zNhjq=`Cb#!@^(D=SZ+>?8_kH?#4+UTBn7hQr6dU{m`NfEc#qb(y`*Al2$_5(oJ2%K zkzH~}_%+)uQqwYnL`gj1PVF=0lIFJx${I1F{b&!<>$;W6{-w+GS}ha3OD*SK#C_yW zFVG@S*2|Nl<=41nj~qEFH&bNrpTA(s)uV!;7)MbpFSAndTEeZbF6PuU(zs_sS48Sx zp9=PTKf-u-)-W2$kC|on8yTaL3dX8nCL?35E2vFpIK7N*T#e>dZsam6PV4z`yJ!FY z?>|;SD>g4dVbg-q<`N%tVa^V8`q&24Y-oWZl7BKY&ebxmW)~R!8?sF7gJS}TX^$(W z9HLy@Z>Tq{vkq|6!-DTPJWEzm<_%$h-HNe>2TR8YnA63C$k;fC(Kw z&8+YCVeV{y%-r#=V}hMD(clPYghTcqx!qBy@l!Aw_~nQO=8i&6=c<{>MvlDpcu-)BaXi2Kt>=xlrPNX41=B(5aqVLCB>C`pp6>bXC`6K-`ALH|mb_p`%|B8F4cA0ArNakKG z+|Er{r^#s<9pgqVb>wO+LpY6ko{OoH$e9{cb6Y1AaqSKX{C}as9sWo}+p`93BPCUt zLk|}+sZT1HE$!7z%ZbNK-?w+nT_Yv5p3h0$N>`wMuR|!XEe@$_#Gt3cnFu_SP{NYk z=%|xB%Cwt~j_iERY&w03fo^lAH7i`Su=AxT^|mC}AvcASx~Ig=8QmaSzxpuO7@Ww3 zg~oBpAMbL4r`_C%DlvB>{5*G|ZUI;Oxl*)r*L%USM;n-aj|iswb_t`<7Qq;rtYG|K z1~3tNl>#qSf59KOV}dz7mjpZd+XStHpX}sx z0>)`}iC}Zk_v)qnvE1LqvLy2301JLk5A&?E_sm`xV#E%=nZU`;?B4;y!Ub2kUL*MF3hbdh7@@P)~Y%~}8H;udUHkr$#%ekH33Pd&C zR|VnfiHt|UcVPc%pm*XG&91>I}nB)n#F({}X;VB2{{L3te7USfx= z;vG>znG@PJ*AS(5ykI@j z_jfH}Mn}D2x(v;cb2;zTNS}bxhTbu@;*-quz*0evzBji?>K^BoX+Z2^Oi6FQ3fYsz z|ArqjT(`?dI~0r=YrR~?`qmH=eCZ+c_Vz<2rtmr=K6HwS_ef%VMu#(W_esWeEMk^* zw+lWMe5$gkdRVW6T;+hI@+9kEBiHBO%pLOH z$wm0x5}g(1iFEaq?Mjci2^Mt8F?h*R!Ju@vVE2@Aft=HL0Ui+BZH%UN0k!P{b>mRM zPUC3NKW-nlM>Cz1{!q>>AFSZIOZIY>gGOA&wE|l=hbe-B@>zDnUl+n zuPf*N+x~#_^B>?$=Vo!!Yt6WPLl4pF^@r^4YL8@cQy(y$cO=o34+`knB(s06UA^%Vw@#G!V-tL#QkZU zuap)ywKQB5^goKuJD#iWkKoO4r&qS6w|*Q$tyrqWXO-m4IbP)Qob z=bUpT(bAAKj3g~o646w@`}^y|AD;&gpX1zn-{tJHNyI!abp5-Tg>nC zB?bWAuVd*xl#seOO2kh2O;;!s`F_<84iA@VKKDuHI85a83Inm~>wa zx4h55>jk2u=9x5EbA1MB>9-(L=4>SQH2uhIcWaWc#(;>uy^GCb*5g%&Osp?lNe6}V z6rrfREu2~52+_6g|!!3&I-z>ofAkTtib0y&*P1qs$};wBQo>5IXPfD zi-gn}ki%}m#MI$BUJ?<8wfy7-PJ!OcW1E>E<8L}>-Cqf^gfD}_Z|A@#_2b}L+Hp`> zkOTgt#Dl22PC!p;18`Q;0#R+hn5}Z(nX3io7_)zA%qgyb*(;<7eAQTBAFT-@Kki|c zpR8kM9Cl`c-!xlm?cE`WDo7HDk(bt~eWrr<(N_dtXIbFwhc;t>n+UApq=WaFB?&4; zRq(`j%DDSSGQPlj4A0y;p*S-Od1_GQ{o&(&1b-edFi%D{LH?MV%Du2I!LEx@_oUvHq!F0 zF+#yltCm*9*Y=3?3$kuQTZ&~s11YR6v@rm}d<+IN8o*tT+n*}g;`m>wKrtj?A) zZh8l;lh!H=3L6&AGx8f1sC8`-G(M2673vuiEbflPAqCYqZ`(^OSp6F->M4*bv((71 z9onSPVI~p(s6pPfe8Ce+dvTmi2JZiMQE*&L!fNirXN+=17?|St0_>H22qdk}gYIDr z7KoPt`BQN~59fmAmU+NNDHEu)HUL%i8t~@88E}~?1pCA9fTgS2fQQvPm++QE2uEMXMBB-NVcS_*VknrZ_@ zjID>zvw6GT8h}=z7JzTO1^&J|Fm=BT?C0!ZI&$F8ebz#mOLkDf_ZCa>nj=Q)yi>$iBQ;URvqJsi)zktQ%4t*o^g zZ)Lij+L;i4Z4kT)f|RB?K#YrF49@t~Zt%3^@6l}WBI7vx=tdmQbaux_l7<8>$1-bk zrin9O$F)IT;09pv@jf`reg*Tr%7M1;MX);17tFOi%PjtoR#UuE6zerH_>I(ktS>o> zRCFyOUmE@+LV@liCeM!K9`PmnzXg+b`}unk*KOqEIzMvB0uuCi4iVxVxNIfCQ<;-k zsL`TkaqZx|eX>G;=aU1G>#o4C+dW zYWg1Fu*#pYes0M#>~G^8Wq0t=&YO6Vk2q1DpiJc7zQp!!*|_S`O1ypHWx=ikT8v%Q zJP>Mo3%rm}hkp)O*Ph<~s)dj>Q4t3lo6B zcX1GrCN1#hJ)fH=l;i)@kK^;_cHy@8W`VeGoB-Ujsr{}g4X)*xfqlCtgU3V4V61R8 zSTNBL=wEuztbg|1y5PGVShF?_EWg+ZT9?Q{2m7heuH6ALpPb>ES9_pvcmRC$+75cHaqvST8{9Qk1HC^yt+nb$1lO2i!KNFMxIE`$ZS1>mwVI1fv6S*6tZM0wyXCU* zj` zCS~KN2U@Uau?mSdQsw8DQwi6znBiMl ztpoSicGk!Z$%5_|?m+$JL-0|wA1oia1;(CV0D4oh!O9owf%oO}jP}F=X82QvASPfY zKDT91kQ1nbjmH?lu8*%7lx+e`#$Pe#! zwez($k~;-qleKVlQ?}qvT~+PQ!q3c1vqB)L+Y25)F^4N$SHq&lAo%$g-$xcMgp)VK z!Y^Z6VYj+DOf|m@_Sa4X3pUBko8mhSi#xaAgVTTEjQAm3FYyw~=RLs__D#ij`m@^G zPa~Lh*9*WN{25%ktp;ZXO@U7}b)bET5`6MO8=m&pgv8|+m@|+ER&^FLCv;V6<*u0t z7L^n-k}~;>-|eN$>pm|IEZ_c~@=W)H4F7ul#fC=`NxjrAgOJWXWvZ z1n?+-2Ht$*D-LvuBhG2YaMh}XC}yH2I$B``OY|lS)`{!|mk;E_4$%&<^YsXsdc&Je zI%rKhr{5rAMUtP7p+Ns&6ZnUHY}XRv0bA@&g~zdk9_ z5lPul#r6)I|4s>wtdv5vC+p#nc6!2a)6W{&b%e$}!v7z-p ze0ZM^Y3tlV1`8#~nT>aut6yv(JXryUZJxvS3#~9?ZwvH&Es9PaRz=p)BT%cB_s~5W z#ZGSuN$S=IWZ;zsy}Bug8Z;Ert1HgXNqzBDH{YM`bYD*|@fnI#;fZvovKjH}*Mhc} zD6AeDfdR=f2)~|*R!uQS8bLg(<@!_Dm7WEUu5*Gnbn8Kfh$2%zT8x*QRIGlfUlq~LBMmjEnx?agbRQ2xzCI?sH@cmKdih3BOGd= z>6;=bB(W8)pVkM2tmLg%>@C4j_Nt^y9}u@cR^(1KKl7fhPNo`t!7{uf*G>IDCN2Fe z7;?D)R((ta4$l%n;gZ9^GR|4>DN~Vz%?c*bi}J}D=W}F1`x8DB@R7)lju5Z-C#1mQ z0@hWUGyougpLqZ>FRE5EW$E@f}+0wZpKW zO6W8Wpl9)P&^R>%pX{APymvT|Y32LK@Zr^Dun3a4E7$NfmrU%~882u!FdbC;H-q{4 z%CO$C9az474BUV7^Y6xQ;OO-AAg5jwztc4&?H{$sjM9Z<(GzQel*I_J{eowT4++W| z>uR^hyJMl)qxfnS#tzxji1RZ?5`Hv=MA$VG8@m^z0sbYvM&h*V!64Cz?iM3!pTjrpvDA>de?$Kx2A%up9 z`GY%Te=tzS6% zKMn4W>4L=fo=i$*71JRr49teb;fq28cw%NX*s$sg6Cv>*Bds?JOMA(XAb}ydu`-JU zZoEZKC4D9z8`bH+25mm~sYnwaPUN|5zX&_If{YwINRIm0;Q9UK;A7lec;HeQ{JQHi zEcqyd%6K05XyXLb5k3TsD>`8B#|rp*M=UgJ7yyI2E(3Eb2`K3P2NF)2z}Y{np_4d- zw=NXIgNvW?Eb3mU@Z`9~8- z7^xutS_a52?_MG-|Au%>ZY0~dDzfZH6e;Q0M$WX}#H!x1f}lhWyl~Ej&yL-JyYKvk zv*jewigZ6kyL{GdQ`<3}!ia!pIvAaMm(Ecv*HWoUP;l zC3Y=?_lp<6)jb=bp3olH#00@sV-NURco}^EbQUyT0pXl!rf|zY7Ouz{0;^`)f&3^2 ztefTzevO>~evO7;PE{Q#;Jl&^dUixae_0lPS)<3TPNcfU%-FC!0Nd4)tgelHW> z_bb7PtFB|G_EEe)*orKCvXy8a*h);t-T4`=Hwkv$LvpySMBw5{icZInW9N%VSSBXz z&8=kN=iB7ro>o%wxtpY{Zy_VbgotXDlC|mw$aDMw@7<9K?H!l%pH)68CJ)fO-~H&Y zv?LpGSC_SEHen4vn6PvF^w{+-lI*VO9mrNL8G-d*U~%6vvc%twK0kYvuD$=AhRAAg zm99oyU*0Tky|)pkR5gh+e?O7yDgHtoR#x$6*g2I^|kUqRxuoe`!br4Pxs^*Q8Q_-ew;NW{lQT|w=MxzNw(6#Q0u z9oFA}0wt3^K&R_(p>6Y9xIgGFtoV5nt}qLS_x9Pq`v>~K-0h3O_QqK2KiwjDasD1` zu-6#pOrOTQ3pE5QKPiFJ(bGYx>O>%Zd$pM~Gf)5@*Jr^s>Ot`8;@R-Nz8pNU_XYUj(E!dZ>H@b~i^1r!M6fgN0n;gJ2Kwh_ zfU=7hK$n#aoLV>mW-L>J?X`=c?>Y~d`tuO{Z}SdV!v7reJ#lcLFd2TWOoSmqzHrX? zY8X{)1N&N+z|BkiA=}^!)h8T=%LC$JwZ*)W@crK1=DDP5@DjPPEQO372p~uLr;>e^qc}&s z5tpvW5WMXZ7Zl}nGx=d@OmFZx+;ng)d0LjnpP|)|X$pO$qfC}Q_Sd6M7w6E<2!?(! zT|j3)pG{v+m7+@XKajqESIDmVWYQnGndIq6kO}%d9)5s&)H1c&8o%kn_F7}J11-@al z?R6maUjHB6+Ps*~nK_MCn~v}4n2Vs8#k zWfRiv*=5{j_V3*z>~fyV8pJaKlYS<$@zzo7MzbC4f5W!ybAJ`q=};r`iPAzQ->OOR zSP{*Z731=Lm~n?rxpH5Y?&Z{9hjV+y!a3ZxpSu(4&c!)G?yQ#0t%Jvu#$YHEy; z(S3H199e}-J*s$)-zQZ4Qk?OAm&Hl&>2fU1E(fJ?kQ@~n3du5 z$({4G?0pj*4tz>4AL^yky53U(dQX3hbe)Jj3e6rNg+e7nPHq-iD5Ho~3iV*HXCv0sx#4a?U(SfzC8RoqCl3-_&`jo0UcPkpZ>m; zN!JJ}=-+*H^n~Y4n%aDej(@&MO+Qll#_Bw^E;~UR`opN?KQ|ikdKS&S{ev_eNF#&Y z53%kcCt!>=LTzOVv)U!m_w_T89p5vIf8~svoc++n--nQiYa~k6jYiWt!%_V75Y%RL z5Pcokf^O$JA)m2XNa3_FYWKVZxAU2vWqwWgg!Tmzy;+vpe_B8%JlaV=DkM@*r98^G zo}LSz2jNNl-30WNjWK)-_gPz6pweM~9Z?_PtZ?x{v5 zL6t~sP7WG33_?45jgZE#61c!`GxPLj9(nJoNn2lTrN{a*s0&v@LwhgMQ`<}F^3ELk zTt19OOE0DqZ~h?ab(6{bI}z~G0c#Y#I|Kc-zl&bD4kJA|S@zlsUH0+*S**__OV-AF z4x4h&lzpT-gAHFjnKiXmV$U5FVc%}!*Ro49>hUi@Pp2M5ho)MhB|ksG8FxM5syrJ( z$@K{G@~0^6EU~A<9wGGefjs)0 zoP636u59*l?#;wy+*00WK)1~2jy<;E7Bo-g+&0N^87`mb){Ywb$n6+iYbHj`%q442 zZW2Q#%Oa7zTN8S1`WNK}@{F2xLsrPxiv4)chSjGw?0^%)+CP}VKB}0=h9>kPi9;ul zk)J9`a2Ul}{cGrH3rbIA3v)f1lez9PEAEch67H4qGM-gz$2kcsIfsNvoQUfLuA!lY z?(xo~D|x?kgOLK+K5Gp2Wk;ZtUz`E{o zVEyhaXVoNEvgXy!Y{w68w!mvQtLYrZhDyY<#}=iqt_xCG?SCn(++ZS`DjdV}ID^=i zn;qFatiUoyQqb_eKI|WUi;6F`<;3r$aOavS_eAiPdvRN|uD4RL?ys(Pos*ni-TDQ3 zbqlMs>jLGJ>b@)%tvfLJHP>8K!-dtxbJEixC#`XfcKsay#v08i&6s6_*l_mGm(%Qo zJp%UDvu5__wstn7y@&Nb`G#%bdf4#77wnJO_t^syb?kp87x~{Mhkbo5k}Vgwv#U=U zv8T89q3Q!$P|A=TW4*{bV2z8aLpnc<;hQEGZ{w~C>>$d-lwvctWLA% z)}3b`1zcjAFPF2vht9Ci^Rifn@kloFg)h6v3bKD)WZC=KRme|&D#AYou-=lDlpC+4 zJ`P{#OgUxl%2s2}d+S23;?_zoIM9vjT;k0g{Lh=iMjo8tsS7uBXeD>F8gVlVr*VZ_ zWVkE5e<9zxh_3y%fG&9#My$M&;e0s<^w>2AEy!;`BSSCIy}FOceC9h8rT!e9_)~*g z9z~(!n#$<#(r1iVlN5bE7(?sJZqs8yKj^5bJQv)c%^5tN$~`O9=j@)VaSJzya0&|_ z(P8CGDqU$nhpJV{4}~hYUvdqK(>Z~D)YqZ+kK57O9dA(Kt#?S&wi8)I-9dw2FQD}b zu_#Y`G5YB88`eCFhiZ~BKqM{~TfB85j$1>C$h3SS$`ujuOZ&;0n?tzeZKt)bT@Uy= zV=6T9XW$0MO;FY@5Qf}vfvqkq%=xSc#l)h3(bsmt`a697#|#kVQ5SMgCWLgK-%bvM zEGJe0fAQ>zwY3Tc3cMpB4Qi|Q!a*?|1m@018{)PgJ;fB{&hz2hJ~yK+f)7X`@;B0v zm16ZDsj{&z^;z_D7W?g-6&vz*9@~{+&K{~aWCK^Iv+rK~L;cdXQP!mpG<)Yw*eM=C z6vTsR@T;eMS5cLl!*l#p<-NIQ&LP}U-#BjGcry1eF@+OEBydLBkzCWiz1&WRb=*%%8Iiz+p5uGdLdp5Jhk?YU~c;xUPo`U;` zc;@b`-5Wy$^MCxuPg`4x0OEBI@JaET1*}p{`sR z*g9`5PO_CI!CUr_t!HzIeohhD>RwE|r)QG;v7zK~;aalb%M{|Wa5~X6u_JnhQ3S8N zN>=y`kZ=thI;q={?vUO`lSkv}rP~EmX~HG?XIc&QjUZI>d^O);zCdg1bLpjw2x>mZ zjcR?FM%(5-AYlqqiD`ZsSlyHhL&m;C`6hmT_uU;ycO6F7&%*isOBCWv640HZ40Py4 zF*@_D4mG^#L`x(8p`&-y*~LP0SQ4;`U3?~xz26$e#=B**I_(AQFV|9buh4n+F)3kF zTTZdP4-(lNw{2`o*|$mOYZadj#U+|tf7+^F_(&i%t&o-y^9ChNZ;jjhJ0UA_x-R?K8m zwLI8G=R(=frpc_UOE#-fn#1;>lk64QbXIeI9J^)iVYaF6KQ{LiVC$4sS+PT%NK`cz zxk&Q4&bJOD{j3Vxji8EKcitg{)-IA(;V zhCP6CU%r6LhjoZkaX(48Z9{F3MbI&(lNN(8|?HsN>9Jq*$ec-2cg;*fM#v{;w!X zG#Y>)>3(rBG${YB=l?3`-1-L9L8-H_T>?xbeFQmCTYd7AvNo?gDvO3(c5q|K49X)o=er?cB> zi}O9&wD1bOOtR_8J$q?$`3#!Au7aG3-+_Aqw?TeSMd`m6p}e{yJVz-LZFDa|aL+|l zQd5p<5=)SaO#xawo{K61GSMFCM6~iq1kw)Pi4)c@r}A?E>2SB#tigxI}OBjswZDPP%xwo0<=`(dCxcX_HYQot<@zPM*O~!!@@E z$9opLyOLnGr4|ZG^G2x|vB>>H-inMvmgqpM=8a2U@dMtOMHFy~fm|{mG z70jqIqes)XD$(Ja!qoWf7jltjfDGEV6M=aj(Ki*M!vD$B_C6IV`pJM^?=qn`@64w1 zMW%Gt^l9{Dy(T?r@tfF6oF`&lF68@$3Y@C91Z2uhhF|#nq;X~t6xy>A+V|X{0h9M-T|16pEvz+{vC?q5ynjB_%59{{`F#C2P{CPzeMVbVnSzn6LrM`9) zlp@A1(9vR7516n)z6=|rzlhy-*q+^KyqNv`1hSKF&StgDbXmP)6WDLsPm#}14!X1s zp#-uY+Mf0!!{)OoJ&-_o{VH9u{xy9y{+A|q$a2QE>fFSC+FbpB4kxRu&4s*G=Kf2O z;ARI6Ql_|xiVHGn**1In-lvR&6e~k7X)UA{AB7sXT}FQ&JVdzzZ_&e#gXqkkhj5gX@`hlibf23(`{q%HF zH$5o*fc9Lhq=x5`sP(Q*bX0N@O_Zr33rnSV{_gd8t@qx6w&eg`s9OX3&~_N!=LNTS z89~|lQy`4Vw$@KOjnyKjkrQkfVS5PC2zX2O^#3KsQ)Q@)kutrLr%&GpPNM_Ajp=Oz zNT;REr%aPQy_xGolZ;nV-HGm0DqsuE74xOHhWu%Tuoum;SWCC=TtU+$ZE5*(Q~Ewv zjxIOl_ikE3TX~5vwden{|*kV?%$>W^YfN#pcd1VlTbs_vA5McJd@;HekCrJG{Fe z9esTRNlB+8*NqHn)C_~04EU^MrX}xPJ3;e#&xdia80WJ{n>$o)!o7TM#cg}Ta^M@w zEmE=KqDzc9ArWoPaNh(@=1&Ke=siP^vTNxtxl2UL>N$vR|IX*ey^wBODr&Q@Ko|8I z(B-z9sN+=w5>Bf@T8YJ|g~XwH{`0rmD}_?64?_{T6}W}x4E?}~&D#MD!V?bqpZp6Ra9Sx#pKo}vNKu~hZ0KTY3c%lFEasolDpWd8J}#NR5S z#!!7VoHTGBruFl4bum+P`+*Ib(P)Pb&9_BIlgv^3@KnUa%A(S=7qF^58JeG`p#4oh z)_9*xauhm=*j`QQ_-Zcg6;~ z@^L2q5)~sb>30>l-kpX~(hqzH`I7jrRixZZh)TE2q*>YSRQOmJT?lfhipqJq;J+&R zM2pb9KA47WucD-_n2y*a(eZ8%I^RZ_`jrNfh}n{G>%DR48M_fZipWJiN9uXrcQ;DM zpHb1TG35Du49z(39t{mVKuhPBp?5yP$ojVm8m(UpE1MS*%|1yw^3a}|x9p`auLjdu z-`%OU(PZi^2q2y>;^38~p6FIa6WX#{o=w_-*v^yw?5*cf?7-PfmKx@>dWnUsZbu&L zBc8^7S`o_b__~hO+M&;O?7oerog0DgHYw9BscqC<)0mS{;T=sYDjW#x8fv9J@bYy z@hh+s=gnXhmq6C*rXBlH%7InEOW6p%Lm$90?EDMHY>TZft0E%Db~uhAGt(AyeBD_j z{c#t{`lN;e!FrhZUYvms1Uz%C&V* z<+fsd?jFx%Mqd><-p$PIa2urs)-UKL2~4wIpP&_WfwU`T20bUzLbkvCir-IEgk7%f z@YE+0^f_`1(uzq(S>BasvO+U*5${1e|BRsS%R=mz8zQWpmKeJ`NrWww{DD@Ky+syb z_fU20dGvB(47%sK6p8!yL&sD;v!%h1xt-#4+m^NTFYl0ACvuT$Cg0-snNE6icRwxd z{zm6${h)VVe4*r2Kh+L>LqpTsdG1p)B_V{avM8Z1+>)uQTp%@FG?z9wj*|4RhsoXe zd}euG1_Z05P>9TOWS10<6x2^4g*R7F)&864@vTSbx#|;SZ{CIu>+oO8-axm^tI@Q= zBD7y432nc%50yx*KodDbRGlk{%6fZX|KAF@p)V7Pe%lW_otH!P!}FnVv#OC|v~UBR_05kq&)h~g^E;Yy-a^{* zU7y~0J4Pb*#}jmbhgTn80sl#M!^B=gls4ZLrR9a95zkbl?spR1f0>JhbkfiX5RPWs zyP>ArO6aZsb{Kh5ni$v%(^W&8=yO2^iC;H?%HYp=%T z^viHn`oHPhKdtogmlCS}DS#r+A4KGu6872G470Z{LsJsdQ4DNChkkuRiLnz{r-v%6 z!fS1|Y|kWCPks`+;-UsCI3dd}0OP1Kpc4&~Ye=pt4&5BJL}xyi!_zcf@Z|s|Rc&*q z<+}sa=yn!$Z7HXbPipBF4UTrT5c-LqlQ!y}qTktY`s(fxwx-;lP={ZzXRD>>=pFmgCLs5N>EsA*Y4{k7=4%Hq{A%Ysin zt?5gor_Pnr%aM(A;psN|-{vk_47%v?P_wVv)?dx6?nrBh{YD{ZuxMGsg!AWI)u z65Sn#z?Ey6P<&hn{XS@l-Z9Ro83&=`CNan{HWh7sl7${QoJ3iDCy-QbCfav1p6619 zp^YVeNDyU@68*GMQS2i)Z}AFP@!_=~*k}tGnE#4s$FUa8s(TzbFw0-<%HGreyI%IB)5l52|NR4DBp*xvgRYRKXItx!9_IR z=stS8_7yUB>qi-DJ|Q8|cj#BeV}!h`(8|6HR5IX(o~n#P*?%vAvv(HJl{TVL_D86u zfOpaHbL*=8UG&wt4^&RPpT=!}L&@4U8XC*d@b(hw-xo{oJXlTNcB|4k$>)izK$EOk zY{-PO3NY@&9=OK!EF8NbfM>^Q;9R~JIj&LzBc>NYCC3~XAxMP9J$bNyO(`^RErD?* z<*;423QA?xLkX`&D3n$Qm3HO9DYgfo_AXWUOFt-as>VdD5H( z&a{4wJ$;jAPA}w5qA$mUY4Lb7IUik0e!fc}_rAE2&3ZG*z6aAtMT{KTb48ns*&^b9 z+MV35j3rNXc#nn%B{P#=5mg5<8vH|%K6f^tdLEWk^)^fUjcj@LgaeIGUO{!IIneRS zMRdhKYnodyg)SeJpn-;uNzH{M;&ob_FrqntDUFAdmOg@`$5fD4nl;+>Z6#_-+J!C_ z97S4O7?ODyfo5I{Mlnw|qxgk($fjTtsyo*OKfofqued2~XN zLC0m?&@{bWXqR~qx(fE82WnfAefy&j7e$*Lop6`e3wc^`C3j{~K5Z{mNRvE*~` z9kO=+2(fLEqFb*j)0%1p{{D5GaCh2>ifI8s%~qtPz#RKHi9+!!$xywv4OZ+`L|Kt$ z$UEH;ZMO7Au}XeOMkN3l=4?Z2LN}t%u`AK$*Oo~7gED%3?=Ad2dJ=}H=)()w5}C_Y z$|U4PE}8SWgSZcgQ~e-KTDovLO_4I9<~yd-n{V{!6s$yJ&i*5g&P^n-GMl_rvmnBU zqj6J%H@G*&60WXHg{pG(aL4)<7`t%<+KY+v`-u$l5m!X!50#KkhzyFIB#O)eI-ym2 z1XnN=*y=ukBe#6 zqZ0bgzli?TIZYpmWYX5z(KHb6rnh;nyNjm*4ITPQY8y_G%*kTRJA1#K5tS>@) z{VrI;a|ffhDWF7qEtJxugSMQ~L)nw{kdcB0%4w5B%{wRXxwYSL#rGlDxS$_4c635{ zk!BbiTm!TC=S)NWOt|}aGR#^W!Dn29Ve6tG*wGmP|F-(U!Mxq@$U0x>6TKQbEwP5% zp)QQKREB!*B;ddzaaed!0!|J51XkUC2b9=vpjaRbcaBVlLlu+Z!{@W1*kKk9+%@6z zXOb|)_a4v8g5b|gYrKn9AP-VQiF!;0L9aVW=B9CCDk)FTy^yD09x79b#gf!8ttEWTpY5nAAA^1fZ^(&;qYo5^xf4C9kB2~b~_KE$boQ_QW=4EX@(-7WxLRD zj2rrg%+bAaNp$Ym894Xk0GK`4g5U1RA)iDg=!dOy=*To@y20)MZ7GSO2bz=U(a>}{ zRGvyZKE%=qRmbSPg`4S`GzSVZr%>&Xk3>X2hlG#llj08%IMDAH126Fz^_VuW%A*NH zRfPic$BBa0vX^+%uFYh${46rqMA11HHPpCE8$EBGf;Nrm zAb)8URBS4X#3ufNY398!_)ayP<-v1~_zJ`)wLf6;DQRHzgcTgMPQ|So?_rA7gt^PDO&?(-Hm&W1R$V zK{6pa=?BQ*u2UpWy^d78|3Vz1rK!ylU0UI0N-ya_dLVca^=Y=D4!O3}&y1nT2@Li4 zVo0xbtI>_~)G6)}q53C=NC|pFd}lo(LYhrvK%;@^9lc6IwpNfy=L$%UAe|&^NF=8$ z_K+}3zQda`kql*Vcx)LISlHeLk}-?nXm2U34}1$RS*akoao)Xl#1j?2KZyFdaG6-rEqgz~(}t@;i1jrkndbqs8m4`36~G9s%a zOMly0(@VLV=?U2|dMYrH9zKvopZ%9Zeg9?CtiOr$g7+~x>xVN{ygP-)oATEjLn{(6 zs0J*SN5NfuHZA3k70NQ)j65EPBTar?=~!eS**jTi-;PvdTpfe{i5x}C%Y4v4>H_|3 zSQ-8N+YUc(i-Tu6q~Y%cO8A+U897!KPqrFdC)JC3NpPVU4VtS&bq?y&rT|0U#b`#K zr2x9iWFD;xx1KM2y_HbqP791VGy>IQgi!8W3I4o89__d;j*LP?(ZH3ju+pX#ZZf$G zzv*0n#_x{9Kvmu&vi%RwUF3aYmkb2+&cxwYDZ*s=MHO;ymIX0a_8|-AZz7xEZW7QJ zO>V4?B%1Rxh@;F&vd--+*|evUbRRAw^hgm=tW71}L+K>@=RWdL(U^RFq(t^g7h|9G zvf$(6Zm|2UBWzq01J7PBfi;_M!K&D&kP#h%Ykqx(i9NqzIO&7+6WX9i>=hWEavUDH zqzK#JWiT!ip5raq@#OT&)mkzm z_bu_2`c6W^g{V%6B!#*XR3(AW`4~!2?_U#mkJ~s|llF5Q)SqlVG4E2m%_CVeV`o)fCK5iP=WD* z4I4b6@OpcA5SqbgB}G{8)B|>I`T+KBcmWbhib2!!IIzU36iEN1;6hv>pvyjhCQb%E z($s{@&GljEZF?As{9uh;IDBrB3h!#3fHHGSVfw-xsJgEhPAoeOrN>iX?X^_+!#@V* z{0WBEPjr&RZ{4uNc`q05Y?+P z@#Nb`5IKG}jLhfz=3nF;iQh{F z^1AC1p7byT;}feGw@JxB=gMyo(F)=Bg#EDfdOCDkb&=-w}9aF-3A!yZ}>qt@y{E17Dr`c)Jox9lvo4`Ikb=VQdGTg#DG6Z}Zg z`xp{7QcP|vyg@Xr?vSWeog`B93$gz2o(x)iBJNXPk?q+nWa;lZl0*tg&R`gEJ+y?h zzEC1JXaJtSE4kLdG!QWOEtu+#VAzj?{Q8fCFLY92oJtyOh$(>g=M=#GOd>oQ9RY*O z_Crv-95zj!1G^S{0#yllK#!RKHXm;nlzu42J*&kCU2jZg`fedB_U$E|B5`E4-f^;I zD1=Os4JHp~EhNdy0g?M7LpIf3#(y~n!C#;Z*5W&0_k|g7TKfi=9+d>IN1fyMsq1jY zotyAp<_qZfxEC6(Xn|KwaqyHz0eo+p0P7WO;NwZJL6)Tx5NkHUe|~r50&QGU{=uf&>Ftu?a8n2&G5yo%~12A7rd0d4i;U}hJg{H z5Z^ul4(H!sK!zG#>eqlHlXb})^@XJT;dYYU6-)5!Bc%9kBAGfCMM6dW$>y>p#C%wl z^p9rpkbXJfA29@41DxTJhbQ6tuzL8zeheZw@Ln$o)HFAP;jJ79@OcbO zdPLx0;#A1bc;UT>`S5C>8%&f6f;Q_y;T^tHVr!EEhYz2C;obRg`+-yNa&IA=v?3YC zu8oHKu0_IqL)+mAZC7aRfMD<*05{i4!6~;#Kvi}<$gmFsk=5(KJojfza`a7R+ls5q z_OL6=B*9DT!KQSElX${x)SU=IC4GVX4to%-5e`D%7Jx5bi$V3VbKuvzOW>OAS7606 zIXNa>ftF+c_<%x7u%Kw-dUY?k zJ>P*SYs@FlXUdc0#rJXPzZN8w4UC1+`N*P(VVMac_iwU^z!tA!4XRYXVLJ(n7 zAUGLfh_4?`!p;*ZUgDrcZG9xyr87weki;7IE)kL+2ThI!lj=#!^->}(6Q}5csbM_{w_6#S4Rxs7;mT_ z!8JfXT@DOS_NfhhriDATA7bX;AG{`6i@1$Dk)zvv$jZ&0#H-$g6!5-^-n>QR_cT@F zHy}j3BE0aSmk*fcDg!0F`Zi+Id6aD1 z5J3*O#gT#=aYQ0x54mTwo@_~XBHL6alkI;dk^_~)_+8NxEJkkNIg^vIe2*bcn-?e0 zQkqsHJ$RmZ!rx=8dM5{310k^PlmPp@q=Dq~Xkc9)48|AcfMw4j!EzfD;CAsnW0>)T z@!RRa4EZL_+x_;SV7ihNevi#@^3%n5Nwg2Xwa)~ndmq7dVi`DB-yJ9BMe|*=g}CjF zJKowEhUxb_yysgwp1Sca{;Sc0>-Sv2;`_$2e@!R;5s3LAZv{Two`ws5&B5ni&lhx* z7g}FWxW$OK&IDTve88rsx?o_=Y~c96hR(zvtFMdV5{e{+C_>0oDKgx%A0bJokhwId zB=u^bNhO(*S%yrhOd%9S+_N>vTq%)ChLog~`k6`8`}_;{p3gpeuYJ~csjEOyRSB_v zyM)o`xk#6{ugA6@SMj=20@v5xgVzIQm1jDA#vg7A%CmhH%j3rQ$_uthls{NIfoAOw zv9^`#9d9(mJx9gx?8+u)_@ONPVjUoF&t)hw$^mV~Dp;HG3|fL(U^L`6ylSn7oUSUE z=hgrVq$?q)A_-P^o`>%ASHy56f$_OMn@M$Pqt`e8qim#YSDx)?_ru}zC^UJTfSwzJ zu<(H&_BLO_gqbNAv_1q)CQLClq=~xP8nXq(>c7gcQqWYSqujbOp~IJnM7#55q;8jgT8;9irp!Jn39-_5AHp{gk4F9 z_D?b3Y!0UF3c>HWM^QEUEGkwyHFs%3)kLOPx!y*+9Ue>x!uP0Bi8x7)!V(W3zswc%nIo&5HDGipP`Q)>C1Nj^` zLHgSo7&GtG;;|{ac|(_Z<32d!%i#@N_ipEqaKb-*TOTvI7{s_zfDY zdyj{QYjNUA9F}Y?z#(2kEJ#qo?vUp+x6_n*=v|;8U2hqZWJNUo%7YP6gG;}s$gJuq zGB8p_^6I%CGiV(^KaxvvJRJ^|cT5rRrpNw|7^2AuTd2S=Ahuujbj=Kd;xfWIHW zrVr&^?A0U!?GEu`0D{RUlP$n~8tpDyX<91w+1Ku-0)Cyp-g6^>(O0|0@ruc3B3= zIx?VMBm$rEw1Dz)K)-YeWbYDyBBxD6!8xAH&~Rm1ymgq;TNkOJf(q(o-^SVHZ5ZzH z9Ji?sqNnQ!&MX%!-)B96clHlr(LfD8n@B>7q%c&qwn3dBVSJ`KNjVvmWQi1c%i#i1RRcPLZA9IGUHtzQL2@L zYOP8lc4iy#I`oXaJ+PMwU7AfTEJaaCbR8~MvO)FCm1tF{id!C?K~aO%*fVN~m30o7 z{!AX_$oLb!uRrlzm*^Hq~65j#Ys}DH(mN^mO1r_+C?+oNuu#~VT}D2h|M)o zDC8Q6twGuN=0*XwUw?|Vr@Bxm}MR1~524HfSSo7Q?A0j$Qg4uIo`s_RDYT)MfwZA2+Maqb2&jm7B zC`H@)^6Bm$9n{y>0;?YDpnQxY7X03iHmf91nMWV1o&V7~$!faEcNT_RJx?tb%*4Aj z8MMtvnGQ{kv6d^pGxI`)NydMrOwmOiy*p>;v)Owx7$d7@=DWEg`-R7ud9!RYxx9?( znP~rDx_;Ub)9f3>utNmQ8f8KKydmtd7KZ&3!T|IB5z(vD%)f>NX1a1dRThe*P_q;Z z*I1&YR5-4bDa2at?EAqbANd0PG57FZTr7MUTTh%tpE>(+&IYbOZ8#jclo9M!-;HxT z#L&R>1=V<7Pq~qrwBI6xshICW8rSl|<8}w|>WG7EwN`N2#AQhB_ys8n?NA&$3HH(o z9P#hMoLu{F@OQ%;&g^CHp{Fbpb{zyz$lwEkPB$P3d6VxqLUP?+I zv=AG!TjYI(2pHefhJ@`AFuOPyCb!#gwbDSC7Ks6Gff#5KHUzJ$|B(utt7Jr+A=-}? zGDX6BseEcI-EP)RTXH3_<=;=*bxD%TTC%~nZoCNZK2i_mR;soxmAWOh(VTxFG?+D_ zb`oB6YBZO|rAp$yKXXuMr8eD{*Ul|FOQ`Io=Can4x0u~_O~m}~0%%&j9u!SnLB=u> zh91PgsCYRX+*l6PIdS0gI2+PqZ^E9D8?bUs4kYvhz}u=ING$hR6>iO9ES^+&)o1(pd4!)W;LmBHqnz3SnF8G*12hw<`?X(o#*quZlXV_A|-NLN& zW)oJ*zK1dDx=iw0M8G0?H8gLM0e5l^A`>pc(85v*T{Z!NQuSqKOyK?kbtu| zQOKs;z|Z3s@!mOkbkOI=j)_k+cov-CQqt}4ZPomvbGWijYV7j7PYgstDQ zuwUyIhFG7#Mga#*oMD2^Z|uTe26#sTy!(BNU*f28@qpua9ubekpYOX|gxzlL<#~90_q|h2F zY1ZHlc7}{J9sZF*nx*Q=zw966wennG`ILd(d<6V5FF~pW3qzIVVE*p|>@;`{CvHB0 z8i;{{3&vo2XcK(j&Jv45NvutN2W9g#@t<@6u1d^A9qmfIJp2{?#(&^rnZKxdlefIh zt_y=jGx1bpBF@UQLznH3=$qtSbVx9pU2#zq^xye_eO4jl`TYdLb8|Q!9!^1Y$1ohS z7U0qvYdOL9LuMXHHNf7{XKoRmxIMKxr$bJB4#g2z`E!Z^foKR zF5YI`^RO0&vbf)%zZB2d)?xcy7MGJa+*9F$be{=+cC4k|(UDYYO$%c%a}AvC^@GSa ziE#LDC|pTPg4B@|7-_l+bG~1NXS0%EVND$Du@40IIwk13@P}+&c#|}9T_Y7^V~lOp z1DX?OfYN3M@MpI@s)PmObH|%_O5{E^v^>Sp2Z+Af6X?`AvwX3`H1aEU;k>Wh{IBW; z?Dnd~+bvBvaPSE#S=FH2b{0+e6R`5J6_yuBA_uW zh8YjFcxFQ7Zpu?L8eUfB;zn*?nn(QH-VqaxOn~ASpvq~F@qPx?{tZ98f;bm509Ex6~@LyQb*#0!hUaYn~+JYuDRlkwu%pmKw*a@Jr>IfS@; zxJHCOe` zFfWApC_18}FvFh?J6F+TTEn#Zqb91YUyG^ld2nCdKRU=U#&-f@IMo-x_oSXzYf23eWPYPt5ET|98TWmr;bwzq~svC zw>TybC)fNBqOwoJocK`Kx8oFOnwo;bY(+R5KMQy*#o_FkY;r@&kMNx+B-@4V5cR)q zWd06yD(^q0_fg(U_tlM+WtUW+(4Bz;)aShvHgRjrLYI6x6(fqf!Y1hYq5;Y;#L{oS zlIhc8Ir_u>Gp!EtrxnL^bS;h6v%5lrNaFcKV!K8NEdQ+`Vfz)p0lBlOs5?C2-sfi1 z=OE(QaW3n~1dhFl0UqVUkdSW+hf=sRL3%CuTFMXJ8}*4_NNTBZ>~Z?4OPjv>Duyy$ z_PAi73vL*^j8+@MaIn%1eaEcPIw}Iwbu4l0)*vmPmPhpbLS1jp!k@xRaTz0qbLNX+ z3C9M#pBtffnFjqMTSNz@I;a`>LKmM>pve^vsdxJ_=4i=JGQTsI-1#jAT02xhTki<` zYYm0*yOFTL$Pl6fD~NJXHqkiJMtFh*_0A8P(bY*jv}R=*jh>#QL6aS{zB;1JA#4k+ zm3&Qq!w%|U(aI`knNg#Kcl9hgx%%7gBJx!F7Xi*V(avZfUsD!>bxa|t4>u*XR|5$n zFG_huuF+gsQS{hmg=yY)sPs1uwbz`(dO;~{v#q0n_C7T1c^o~avYh@|yopB2B0H6L zfT@o0U~5xj*@fStX_DOtRd*9azI)0ztH zW#2EqM&6nW!mtVtR21jJ^1dF3(is3BeF;u&+d|G+J~ht&wk+rPaoaS+6RTzeesG!CidOW!Xl$^932nApi9S4v3nZ^u3UroP99xrG_X{Eg349) z(NBR@l+o{@E*b@8pHH13|63$Z*r}KU3{@knt*sXM+xnG0UZ| zkb+14Wbb}Y^6+~(Sxw%OTz6qsAVrRB8yBWU;|yKdp3FVRRrH85cUFGniho{*Vl{C^ zdFBi{WJ{rb!C#tQv;!x^SD}7>9sT5{Nz*mL*{69+NQ(bm#%ywgoNkbXc117n`rr@K z#c{CcLO$qfpMoV(*--x?9q#@vfJfsM5Y_Vxf$N2GBDvXse27h>0(;k^qT_iS<2i^L`_AE4#~d7M$VXm= z;<3JBeD{7AcIuepRUJjV;~7SCUjC0Ba_eQJtM`(O*Db`7$^gHS0x0a<3X^i&vr`@j zQd)5^L+~JY83%&VbvGEiq7OFNwlM0v1q2S8f#7;0DC8R@fkBH&sk|sL`Jzm&yc8fJ z-WJSXn#~Rds588gg@i}@bg5URD67y`Owt_kiNs1pBKh?9oicT#Hc2ULZTe+8 zZ77LilS^@N;TDu+XJGsu2KTiGp=ZEO+|lfY&%!AkjmxJzlMGI$ufzE}XrBUT(3&#FLk}J>XGv zv)O}(q-}76tOg$0d>U)EEkKw1Kd4Ht8_thDgKtixix6FP+5 zBMyy$>!CL>5XNdlVc~EfXbRnefwBU~7QYKJe;z=E+>B+EWv48GLN%OT!m8; zuEgHEaJSFp2Z2(`UD?8Hv;dC`=G8I2W4vhP_V=WayLlBxKsmavUed0Z#NUu zK4W@iZv*WO?VtyohN$UHD{T5^fj3rikPRB7VN5o?>Bxsfbs_fmETSRn&oGgC@{H1G zC>gyUMe^SZ!l1Q2_|Cfoo<0d6QI-fL?&WYqsu3iQ_Cj(=J*eHUhSNrToCU+5VeS1N zu+FLva(F+&t=vKA*!K&z7dAnJPZ6y0xC2jhGC@<|Hf%l~z|DJI2c@HHVEZ07Vs@dO zDdZ0)7QFk}HM~~zRU=aE$tKG8#TtFSg<}4!D0H{Fi~j|sWB2Au7(5Nwn_-K+`VyF$ zU`pREl3*(ye`TD{`w)fm0+97?iVR;^2fJ2y!@|c~;lBD7SS+y|?ulu_2^}E_aXm|t zzw|MgCvG!6=9Y9=T7vS_pP{B2+i2?Z6V$zPFW&H!Mfs`C+&$)5oGdoPUtXth;iEiLCu6kz-Wav`HBKdOtD>(M2kXmpv8;SQo;rIH6GtrY#CdI0J-rOI z^mMWQ$2?r{Q~=xDO)!3Z6~=Ykq&CSf*)8#RX`bwBmX-X+RHy}(y$DYx^0LEZO*bXA zJRisj$5>MFE{{|g^^@ycsd5XbK|aizz6Let z(?H1@;Mq#V4d<>tty~S{FNT@Lg%;?m(9Fcq2_?O4(kgXxdi$ zl&)wL!yVq&QD(s{)DMZpqlbf0L*W({ExV704>qHZ*jKEu;XbjPU4=`Jm2%li&oP0^ z6}RRqM&(tRxJ9c3c{Z-Yu;-2R)creDHEcO?H!@=U=6Vy+M+V^Z+Y`jnj{~IH!|=%i z@cD&5oLRaL)E*?j;L2dQw^$vT6!an3P#C`5wc-sFu@xYOcxB%jvV(#;+GB-;PnLf5(pvPZ4MRUi9lJuDwiq>ze58Bxy{Gb;3Jm zDj{$H0|hT z>Yz*Lcdnj#vH3Q|#s<2@a4mY(Dq{C3JzP=yoE}mcp#fjRX@Gt!ZHl-?jjBd<@w@U+Wj59RfH3t2|+aM?37+Pv$VB^nakZEv>9MP^MQVUaiUCKwte&?`! zHVv$EUmhLxZKn#0Lg;tl?{xS6Svcnmr5ZI&^iNJPjW3YJ1KrE9n$tjC<@4#6i|gpG z;tbYBUWOQWnUL1NuS`N%40&3Aob;PLB%(qyVa3{Oq&YE~^zpWk*?wb8=H+qbTXtI6 zOwS~`Y-)lU?dD+jWovvSbOD!H-9kbAJmg(`1r>{KVb_jqbkj}1isjq!p??Hk)HTHm zOU9|4cRCI8(xcv`{?v%;$sY?G`YSq>{m$)Pavfj5JsWgye_mXQ9s GD)N5>!^z12 literal 0 HcmV?d00001 diff --git a/models/tacotron2/distributed.py b/models/tacotron2/distributed.py new file mode 100644 index 0000000000..cce74944bf --- /dev/null +++ b/models/tacotron2/distributed.py @@ -0,0 +1,173 @@ +import torch +import torch.distributed as dist +from torch.nn.modules import Module +from torch.autograd import Variable + +def _flatten_dense_tensors(tensors): + """Flatten dense tensors into a contiguous 1D buffer. Assume tensors are of + same dense type. + Since inputs are dense, the resulting tensor will be a concatenated 1D + buffer. Element-wise operation on this buffer will be equivalent to + operating individually. + Arguments: + tensors (Iterable[Tensor]): dense tensors to flatten. + Returns: + A contiguous 1D buffer containing input tensors. + """ + if len(tensors) == 1: + return tensors[0].contiguous().view(-1) + flat = torch.cat([t.contiguous().view(-1) for t in tensors], dim=0) + return flat + +def _unflatten_dense_tensors(flat, tensors): + """View a flat buffer using the sizes of tensors. Assume that tensors are of + same dense type, and that flat is given by _flatten_dense_tensors. + Arguments: + flat (Tensor): flattened dense tensors to unflatten. + tensors (Iterable[Tensor]): dense tensors whose sizes will be used to + unflatten flat. + Returns: + Unflattened dense tensors with sizes same as tensors and values from + flat. + """ + outputs = [] + offset = 0 + for tensor in tensors: + numel = tensor.numel() + outputs.append(flat.narrow(0, offset, numel).view_as(tensor)) + offset += numel + return tuple(outputs) + + +''' +This version of DistributedDataParallel is designed to be used in conjunction with the multiproc.py +launcher included with this example. It assumes that your run is using multiprocess with 1 +GPU/process, that the model is on the correct device, and that torch.set_device has been +used to set the device. + +Parameters are broadcasted to the other processes on initialization of DistributedDataParallel, +and will be allreduced at the finish of the backward pass. +''' +class DistributedDataParallel(Module): + + def __init__(self, module): + super(DistributedDataParallel, self).__init__() + #fallback for PyTorch 0.3 + if not hasattr(dist, '_backend'): + self.warn_on_half = True + else: + self.warn_on_half = True if dist._backend == dist.dist_backend.GLOO else False + + self.module = module + + for p in self.module.state_dict().values(): + if not torch.is_tensor(p): + continue + dist.broadcast(p, 0) + + def allreduce_params(): + if(self.needs_reduction): + self.needs_reduction = False + buckets = {} + for param in self.module.parameters(): + if param.requires_grad and param.grad is not None: + tp = type(param.data) + if tp not in buckets: + buckets[tp] = [] + buckets[tp].append(param) + if self.warn_on_half: + if torch.cuda.HalfTensor in buckets: + print("WARNING: gloo dist backend for half parameters may be extremely slow." + + " It is recommended to use the NCCL backend in this case. This currently requires" + + "PyTorch built from top of tree master.") + self.warn_on_half = False + + for tp in buckets: + bucket = buckets[tp] + grads = [param.grad.data for param in bucket] + coalesced = _flatten_dense_tensors(grads) + dist.all_reduce(coalesced) + coalesced /= dist.get_world_size() + for buf, synced in zip(grads, _unflatten_dense_tensors(coalesced, grads)): + buf.copy_(synced) + + for param in list(self.module.parameters()): + def allreduce_hook(*unused): + param._execution_engine.queue_callback(allreduce_params) + if param.requires_grad: + param.register_hook(allreduce_hook) + + def forward(self, *inputs, **kwargs): + self.needs_reduction = True + return self.module(*inputs, **kwargs) + + ''' + def _sync_buffers(self): + buffers = list(self.module._all_buffers()) + if len(buffers) > 0: + # cross-node buffer sync + flat_buffers = _flatten_dense_tensors(buffers) + dist.broadcast(flat_buffers, 0) + for buf, synced in zip(buffers, _unflatten_dense_tensors(flat_buffers, buffers)): + buf.copy_(synced) + def train(self, mode=True): + # Clear NCCL communicator and CUDA event cache of the default group ID, + # These cache will be recreated at the later call. This is currently a + # work-around for a potential NCCL deadlock. + if dist._backend == dist.dist_backend.NCCL: + dist._clear_group_cache() + super(DistributedDataParallel, self).train(mode) + self.module.train(mode) + ''' +''' +Modifies existing model to do gradient allreduce, but doesn't change class +so you don't need "module" +''' +def apply_gradient_allreduce(module): + if not hasattr(dist, '_backend'): + module.warn_on_half = True + else: + module.warn_on_half = True if dist._backend == dist.dist_backend.GLOO else False + + for p in module.state_dict().values(): + if not torch.is_tensor(p): + continue + dist.broadcast(p, 0) + + def allreduce_params(): + if(module.needs_reduction): + module.needs_reduction = False + buckets = {} + for param in module.parameters(): + if param.requires_grad and param.grad is not None: + tp = param.data.dtype + if tp not in buckets: + buckets[tp] = [] + buckets[tp].append(param) + if module.warn_on_half: + if torch.cuda.HalfTensor in buckets: + print("WARNING: gloo dist backend for half parameters may be extremely slow." + + " It is recommended to use the NCCL backend in this case. This currently requires" + + "PyTorch built from top of tree master.") + module.warn_on_half = False + + for tp in buckets: + bucket = buckets[tp] + grads = [param.grad.data for param in bucket] + coalesced = _flatten_dense_tensors(grads) + dist.all_reduce(coalesced) + coalesced /= dist.get_world_size() + for buf, synced in zip(grads, _unflatten_dense_tensors(coalesced, grads)): + buf.copy_(synced) + + for param in list(module.parameters()): + def allreduce_hook(*unused): + Variable._execution_engine.queue_callback(allreduce_params) + if param.requires_grad: + param.register_hook(allreduce_hook) + + def set_needs_reduction(self, input, output): + self.needs_reduction = True + + module.register_forward_hook(set_needs_reduction) + return module diff --git a/models/tacotron2/filelists/ljs_audio_text_test_filelist.txt b/models/tacotron2/filelists/ljs_audio_text_test_filelist.txt new file mode 100755 index 0000000000..4c51f4c51c --- /dev/null +++ b/models/tacotron2/filelists/ljs_audio_text_test_filelist.txt @@ -0,0 +1,500 @@ +DUMMY/LJ045-0096.wav|Mrs. De Mohrenschildt thought that Oswald, +DUMMY/LJ049-0022.wav|The Secret Service believed that it was very doubtful that any President would ride regularly in a vehicle with a fixed top, even though transparent. +DUMMY/LJ033-0042.wav|Between the hours of eight and nine p.m. they were occupied with the children in the bedrooms located at the extreme east end of the house. +DUMMY/LJ016-0117.wav|The prisoner had nothing to deal with but wooden panels, and by dint of cutting and chopping he got both the lower panels out. +DUMMY/LJ025-0157.wav|Under these circumstances, unnatural as they are, with proper management, the bean will thrust forth its radicle and its plumule; +DUMMY/LJ042-0219.wav|Oswald demonstrated his thinking in connection with his return to the United States by preparing two sets of identical questions of the type which he might have thought +DUMMY/LJ032-0164.wav|it is not possible to state with scientific certainty that a particular small group of fibers come from a certain piece of clothing +DUMMY/LJ046-0092.wav|has confidence in the dedicated Secret Service men who are ready to lay down their lives for him +DUMMY/LJ050-0118.wav|Since these agencies are already obliged constantly to evaluate the activities of such groups, +DUMMY/LJ043-0016.wav|Jeanne De Mohrenschildt said, quote, +DUMMY/LJ021-0078.wav|no economic panacea, which could simply revive over-night the heavy industries and the trades dependent upon them. +DUMMY/LJ039-0148.wav|Examination of the cartridge cases found on the sixth floor of the Depository Building +DUMMY/LJ047-0202.wav|testified that the information available to the Federal Government about Oswald before the assassination would, if known to PRS, +DUMMY/LJ023-0056.wav|It is an easy document to understand when you remember that it was called into being +DUMMY/LJ021-0025.wav|And in many directions, the intervention of that organized control which we call government +DUMMY/LJ030-0105.wav|Communications in the motorcade. +DUMMY/LJ021-0012.wav|with respect to industry and business, but nearly all are agreed that private enterprise in times such as these +DUMMY/LJ019-0169.wav|and one or two men were allowed to mend clothes and make shoes. The rules made by the Secretary of State were hung up in conspicuous parts of the prison; +DUMMY/LJ039-0088.wav|It just is an aid in seeing in the fact that you only have the one element, the crosshair, +DUMMY/LJ016-0192.wav|"I think I could do that sort of job," said Calcraft, on the spur of the moment. +DUMMY/LJ014-0142.wav|was strewn in front of the dock, and sprinkled it towards the bench with a contemptuous gesture. +DUMMY/LJ012-0015.wav|Weedon and Lecasser to twelve and six months respectively in Coldbath Fields. +DUMMY/LJ048-0033.wav|Prior to November twenty-two, nineteen sixty-three +DUMMY/LJ028-0349.wav|who were each required to send so large a number to Babylon, that in all there were collected no fewer than fifty thousand. +DUMMY/LJ030-0197.wav|At first Mrs. Connally thought that her husband had been killed, +DUMMY/LJ017-0133.wav|Palmer speedily found imitators. +DUMMY/LJ034-0123.wav|Although Brennan testified that the man in the window was standing when he fired the shots, most probably he was either sitting or kneeling. +DUMMY/LJ003-0282.wav|Many years were to elapse before these objections should be fairly met and universally overcome. +DUMMY/LJ032-0204.wav|Special Agent Lyndal L. Shaneyfelt, a photography expert with the FBI, +DUMMY/LJ016-0241.wav|Calcraft served the city of London till eighteen seventy-four, when he was pensioned at the rate of twenty-five shillings per week. +DUMMY/LJ023-0033.wav|we will not allow ourselves to run around in new circles of futile discussion and debate, always postponing the day of decision. +DUMMY/LJ009-0286.wav|There has never been much science in the system of carrying out the extreme penalty in this country; the "finisher of the law" +DUMMY/LJ008-0181.wav|he had his pockets filled with bread and cheese, and it was generally supposed that he had come a long distance to see the fatal show. +DUMMY/LJ015-0052.wav|to the value of twenty thousand pounds. +DUMMY/LJ016-0314.wav|Sir George Grey thought there was a growing feeling in favor of executions within the prison precincts. +DUMMY/LJ047-0056.wav|From August nineteen sixty-two +DUMMY/LJ010-0027.wav|Nor did the methods by which they were perpetrated greatly vary from those in times past. +DUMMY/LJ010-0065.wav|At the former the "Provisional Government" was to be established, +DUMMY/LJ046-0113.wav|The Commission has concluded that at the time of the assassination +DUMMY/LJ028-0410.wav|There among the ruins they still live in the same kind of houses, +DUMMY/LJ044-0137.wav|More seriously, the facts of his defection had become known, leaving him open to almost unanswerable attack by those who opposed his views. +DUMMY/LJ008-0215.wav|One by one the huge uprights of black timber were fitted together, +DUMMY/LJ030-0084.wav|or when the press of the crowd made it impossible for the escort motorcycles to stay in position on the car's rear flanks. +DUMMY/LJ020-0092.wav|Have yourself called on biscuit mornings an hour earlier than usual. +DUMMY/LJ029-0096.wav|On November fourteen, Lawson and Sorrels attended a meeting at Love Field +DUMMY/LJ015-0308.wav|and others who swore to the meetings of the conspirators and their movements. Saward was found guilty, +DUMMY/LJ012-0067.wav|But Mrs. Solomons could not resist the temptation to dabble in stolen goods, and she was found shipping watches of the wrong category to New York. +DUMMY/LJ018-0231.wav|namely, to suppress it and substitute another. +DUMMY/LJ014-0265.wav|and later he became manager of the newly rebuilt Olympic at Wych Street. +DUMMY/LJ024-0102.wav|would be the first to exclaim as soon as an amendment was proposed +DUMMY/LJ007-0233.wav|it consists of several circular perforations, about two inches in diameter, +DUMMY/LJ013-0213.wav|This seems to have decided Courvoisier, +DUMMY/LJ032-0045.wav|This price included nineteen dollars, ninety-five cents for the rifle and the scope, and one dollar, fifty cents for postage and handling. +DUMMY/LJ011-0048.wav|Wherefore let him that thinketh he standeth take heed lest he fall," and was full of the most pointed allusions to the culprit. +DUMMY/LJ005-0294.wav|It was frequently stated in evidence that the jail of the borough was in so unfit a state for the reception of prisoners, +DUMMY/LJ016-0007.wav|There were others less successful. +DUMMY/LJ028-0138.wav|perhaps the tales that travelers told him were exaggerated as travelers' tales are likely to be, +DUMMY/LJ050-0029.wav|that is reflected in definite and comprehensive operating procedures. +DUMMY/LJ014-0121.wav|The prisoners were in due course transferred to Newgate, to be put upon their trial at the Central Criminal Court. +DUMMY/LJ014-0146.wav|They had to handcuff her by force against the most violent resistance, and still she raged and stormed, +DUMMY/LJ046-0111.wav|The Secret Service has attempted to perform this function through the activities of its Protective Research Section +DUMMY/LJ012-0257.wav|But the affair still remained a profound mystery. No light was thrown upon it till, towards the end of March, +DUMMY/LJ002-0260.wav|Yet the public opinion of the whole body seems to have checked dissipation. +DUMMY/LJ031-0014.wav|the Presidential limousine arrived at the emergency entrance of the Parkland Hospital at about twelve:thirty-five p.m. +DUMMY/LJ047-0093.wav|Oswald was arrested and jailed by the New Orleans Police Department for disturbing the peace, in connection with a street fight which broke out when he was accosted +DUMMY/LJ003-0324.wav|gaming of all sorts should be peremptorily forbidden under heavy pains and penalties. +DUMMY/LJ021-0115.wav|we have reached into the heart of the problem which is to provide such annual earnings for the lowest paid worker as will meet his minimum needs. +DUMMY/LJ046-0191.wav|it had established periodic regular review of the status of four hundred individuals; +DUMMY/LJ034-0197.wav|who was one of the first witnesses to alert the police to the Depository as the source of the shots, as has been discussed in chapter three. +DUMMY/LJ002-0253.wav|were governed by rules which they themselves had framed, and under which subscriptions were levied +DUMMY/LJ048-0288.wav|might have been more alert in the Dallas motorcade if they had retired promptly in Fort Worth. +DUMMY/LJ007-0112.wav|Many of the old customs once prevalent in the State Side, so properly condemned and abolished, +DUMMY/LJ017-0189.wav|who was presently attacked in the same way as the others, but, but, thanks to the prompt administration of remedies, he recovered. +DUMMY/LJ042-0230.wav|basically, although I hate the USSR and socialist system I still think marxism can work under different circumstances, end quote. +DUMMY/LJ050-0161.wav|The Secret Service should not and does not plan to develop its own intelligence gathering facilities to duplicate the existing facilities of other Federal agencies. +DUMMY/LJ003-0011.wav|that not more than one bottle of wine or one quart of beer could be issued at one time. No account was taken of the amount of liquors admitted in one day, +DUMMY/LJ008-0206.wav|and caused a number of stout additional barriers to be erected in front of the scaffold, +DUMMY/LJ002-0261.wav|The poorer prisoners were not in abject want, as in other prisons, +DUMMY/LJ012-0189.wav|Hunt, in consideration of the information he had given, escaped death, and was sentenced to transportation for life. +DUMMY/LJ019-0317.wav|The former, which consisted principally of the tread-wheel, cranks, capstans, shot-drill, +DUMMY/LJ011-0041.wav|Visited Mr. Fauntleroy. My application for books for him not having been attended, I had no prayer-book to give him. +DUMMY/LJ023-0089.wav|That is not only my accusation. +DUMMY/LJ044-0224.wav|would not agree with that particular wording, end quote. +DUMMY/LJ013-0104.wav|He found them at length residing at the latter place, one as a landed proprietor, the other as a publican. +DUMMY/LJ013-0055.wav|The jury did not believe him, and the verdict was for the defendants. +DUMMY/LJ014-0306.wav|These had been attributed to political action; some thought that the large purchases in foreign grains, effected at losing prices, +DUMMY/LJ029-0052.wav|To supplement the PRS files, the Secret Service depends largely on local police departments and local offices of other Federal agencies +DUMMY/LJ028-0459.wav|Its bricks, measuring about thirteen inches square and three inches in thickness, were burned and stamped with the usual short inscription: +DUMMY/LJ017-0183.wav|Soon afterwards Dixon died, showing all the symptoms already described. +DUMMY/LJ009-0084.wav|At length the ordinary pauses, and then, in a deep tone, which, though hardly above a whisper, is audible to all, says, +DUMMY/LJ007-0170.wav|That in this vast metropolis, the center of wealth, civilization, and information; +DUMMY/LJ016-0277.wav|This is proved by contemporary accounts, especially one graphic and realistic article which appeared in the 'Times,' +DUMMY/LJ009-0061.wav|He staggers towards the pew, reels into it, stumbles forward, flings himself on the ground, and, by a curious twist of the spine, +DUMMY/LJ019-0201.wav|to select a sufficiently spacious piece of ground, and erect a prison which from foundations to roofs should be in conformity with the newest ideas. +DUMMY/LJ030-0063.wav|He had repeated this wish only a few days before, during his visit to Tampa, Florida. +DUMMY/LJ010-0257.wav|a third miscreant made a similar but far less serious attempt in the month of July following. +DUMMY/LJ009-0106.wav|The keeper tries to appear unmoved, but his eye wanders anxiously over the combustible assembly. +DUMMY/LJ008-0121.wav|After the construction and action of the machine had been explained, the doctor asked the governor what kind of men he had commanded at Goree, +DUMMY/LJ050-0069.wav|the Secret Service had received from the FBI some nine thousand reports on members of the Communist Party. +DUMMY/LJ006-0202.wav|The news-vendor was also a tobacconist, +DUMMY/LJ012-0230.wav|Shortly before the day fixed for execution, Bishop made a full confession, the bulk of which bore the impress of truth, +DUMMY/LJ005-0248.wav|and stated that in his opinion Newgate, as the common jail of Middlesex, was wholly inadequate to the proper confinement of its prisoners. +DUMMY/LJ037-0053.wav|who had been greatly upset by her experience, was able to view a lineup of four men handcuffed together at the police station. +DUMMY/LJ045-0177.wav|For the first time +DUMMY/LJ004-0036.wav|it was hoped that their rulers would hire accommodation in the county prisons, and that the inferior establishments would in course of time disappear. +DUMMY/LJ026-0054.wav|carbohydrates (starch, cellulose) and fats. +DUMMY/LJ020-0085.wav|Break apart from one another and pile on a plate, throwing a clean doily or a small napkin over them. Break open at table. +DUMMY/LJ046-0226.wav|The several military intelligence agencies reported crank mail and similar threats involving the President. +DUMMY/LJ014-0233.wav|he shot an old soldier who had attempted to detain him. He was convicted and executed. +DUMMY/LJ033-0152.wav|The portion of the palm which was identified was the heel of the right palm, i.e., the area near the wrist, on the little finger side. +DUMMY/LJ004-0009.wav|as indefatigable and self-sacrificing, found by personal visitation that the condition of jails throughout the kingdom was, +DUMMY/LJ017-0134.wav|Within a few weeks occurred the Leeds poisoning case, in which the murderer undoubtedly was inspired by the facts made public at Palmer's trial. +DUMMY/LJ019-0318.wav|was to be the rule for all convicted prisoners throughout the early stages of their detention; +DUMMY/LJ020-0093.wav|Rise, wash face and hands, rinse the mouth out and brush back the hair. +DUMMY/LJ012-0188.wav|Probert was then admitted as a witness, and the case was fully proved against Thurtell, who was hanged in front of Hertford Jail. +DUMMY/LJ019-0202.wav|The preference given to the Pentonville system destroyed all hopes of a complete reformation of Newgate. +DUMMY/LJ039-0027.wav|Oswald's revolver +DUMMY/LJ040-0176.wav|He admitted to fantasies about being powerful and sometimes hurting and killing people, but refused to elaborate on them. +DUMMY/LJ018-0354.wav|Doubts were long entertained whether Thomas Wainwright, +DUMMY/LJ031-0185.wav|From the Presidential airplane, the Vice President telephoned Attorney General Robert F. Kennedy, +DUMMY/LJ006-0137.wav|They were not obliged to attend chapel, and seldom if ever went; "prisoners," said one of them under examination, "did not like the trouble of going to chapel." +DUMMY/LJ032-0085.wav|The Hidell signature on the notice of classification was in the handwriting of Oswald. +DUMMY/LJ009-0037.wav|the schoolmaster and the juvenile prisoners being seated round the communion-table, opposite the pulpit. +DUMMY/LJ006-0021.wav|Later on he had devoted himself to the personal investigation of the prisons of the United States. +DUMMY/LJ006-0082.wav|and this particular official took excellent care to select as residents for his own ward those most suitable from his own point of view. +DUMMY/LJ016-0380.wav|with hope to the last. There is always the chance of a flaw in the indictment, of a missing witness, or extenuating circumstances. +DUMMY/LJ019-0344.wav|monitor, or schoolmaster, nor to be engaged in the service of any officer of the prison. +DUMMY/LJ019-0161.wav|These disciplinary improvements were, however, only slowly and gradually introduced. +DUMMY/LJ028-0145.wav|And here I may not omit to tell the use to which the mould dug out of the great moat was turned, nor the manner wherein the wall was wrought. +DUMMY/LJ018-0349.wav|His disclaimer, distinct and detailed on every point, was intended simply for effect. +DUMMY/LJ043-0010.wav|Some of the members of that group saw a good deal of the Oswalds through the fall of nineteen sixty-three, +DUMMY/LJ027-0178.wav|These were undoubtedly perennibranchs. In the Permian and Triassic higher forms appeared, which were certainly caducibranch. +DUMMY/LJ041-0070.wav|He did not rise above the rank of private first class, even though he had passed a qualifying examination for the rank of corporal. +DUMMY/LJ008-0266.wav|Thus in the years between May first, eighteen twenty-seven, and thirtieth April, eighteen thirty-one, +DUMMY/LJ021-0091.wav|In this recent reorganization we have recognized three distinct functions: +DUMMY/LJ019-0129.wav|which marked the growth of public interest in prison affairs, and which was the germ of the new system +DUMMY/LJ018-0215.wav|William Roupell was the eldest but illegitimate son of a wealthy man who subsequently married Roupell's mother, and had further legitimate issue. +DUMMY/LJ015-0194.wav|and behaved so as to justify a belief that he had been a jail-bird all his life. +DUMMY/LJ016-0137.wav|that numbers of men, "lifers," and others with ten, fourteen, or twenty years to do, can be trusted to work out of doors without bolts and bars +DUMMY/LJ002-0289.wav|the latter raised eighteen pence among them to pay for a truss of straw for the poor woman to lie on. +DUMMY/LJ023-0016.wav|In nineteen thirty-three you and I knew that we must never let our economic system get completely out of joint again +DUMMY/LJ011-0141.wav|There were at the moment in Newgate six convicts sentenced to death for forging wills. +DUMMY/LJ016-0283.wav|to do them mere justice, there was at least till then a half-drunken ribald gaiety among the crowd that made them all akin." +DUMMY/LJ035-0082.wav|The only interval was the time necessary to ride in the elevator from the second to the sixth floor and walk back to the southeast corner. +DUMMY/LJ045-0194.wav|Anyone who was familiar with that area of Dallas would have known that the motorcade would probably pass the Texas School Book Depository to get from Main Street +DUMMY/LJ009-0124.wav|occupied when they saw it last, but a few hours ago, by their comrades who are now dead; +DUMMY/LJ030-0162.wav|In the Presidential Limousine +DUMMY/LJ050-0223.wav|The plan provides for an additional two hundred five agents for the Secret Service. Seventeen of this number are proposed for the Protective Research Section; +DUMMY/LJ008-0228.wav|their harsh and half-cracked voices full of maudlin, besotted sympathy for those about to die. +DUMMY/LJ002-0096.wav|The eight courts above enumerated were well supplied with water; +DUMMY/LJ018-0288.wav|After this the other conspirators traveled to obtain genuine bills and master the system of the leading houses at home and abroad. +DUMMY/LJ002-0106.wav|in which latterly a copper had been fixed for the cooking of provisions sent in by charitable persons. +DUMMY/LJ025-0129.wav|On each lobe of the bi-lobed leaf of Venus flytrap are three delicate filaments which stand out at right angles from the surface of the leaf. +DUMMY/LJ044-0013.wav|Hands Off Cuba, end quote, an application form for, and a membership card in, +DUMMY/LJ049-0115.wav|of the person who is actually in the exercise of the executive power, or +DUMMY/LJ019-0145.wav|But reformation was only skin deep. Below the surface many of the old evils still rankled. +DUMMY/LJ019-0355.wav|came up in all respects to modern requirements. +DUMMY/LJ019-0289.wav|There was unrestrained association of untried and convicted, juvenile with adult prisoners, vagrants, misdemeanants, felons. +DUMMY/LJ048-0222.wav|in Fort Worth, there occurred a breach of discipline by some members of the Secret Service who were officially traveling with the President. +DUMMY/LJ016-0367.wav|Under the new system the whole of the arrangements from first to last fell upon the officers. +DUMMY/LJ047-0097.wav|Agent Quigley did not know of Oswald's prior FBI record when he interviewed him, +DUMMY/LJ007-0075.wav|as effectually to rebuke and abash the profane spirit of the more insolent and daring of the criminals. +DUMMY/LJ047-0022.wav|provided by other agencies. +DUMMY/LJ007-0085.wav|at Newgate and York Castle as long as five years; "at Ilchester and Morpeth for seven years; at Warwick for eight years, +DUMMY/LJ047-0075.wav|Hosty had inquired earlier and found no evidence that it was functioning in the Dallas area. +DUMMY/LJ008-0098.wav|One was the "yeoman of the halter," a Newgate official, the executioner's assistant, whom Mr. J. T. Smith, who was present at the execution, +DUMMY/LJ017-0102.wav|The second attack was fatal, and ended in Cook's death from tetanus. +DUMMY/LJ046-0105.wav|Second, the adequacy of other advance preparations for the security of the President, during his visit to Dallas, +DUMMY/LJ018-0206.wav|He was a tall, slender man, with a long face and iron-gray hair. +DUMMY/LJ012-0271.wav|Whether it was greed or a quarrel that drove Greenacre to the desperate deed remains obscure. +DUMMY/LJ005-0086.wav|with such further separation as the justices should deem conducive to good order and discipline. +DUMMY/LJ042-0097.wav|and considerably better living quarters than those accorded to Soviet citizens of equal age and station. +DUMMY/LJ047-0126.wav|we would handle it in due course, in accord with the whole context of the investigation. End quote. +DUMMY/LJ041-0022.wav|Oswald first wrote, quote, Edward Vogel, end quote, an obvious misspelling of Voebel's name, +DUMMY/LJ015-0025.wav|The bank enjoyed an excellent reputation, it had a good connection, and was supposed to be perfectly sound. +DUMMY/LJ012-0194.wav|But Burke and Hare had their imitators further south, +DUMMY/LJ028-0416.wav|(if man may speak so confidently of His great impenetrable counsels), for an eternal Testimony of His great work in the confusion of Man's pride, +DUMMY/LJ007-0130.wav|are all huddled together without discrimination, oversight, or control." +DUMMY/LJ015-0005.wav|About this time Davidson and Gordon, the people above-mentioned, +DUMMY/LJ016-0125.wav|with this, placed against the wall near the chevaux-de-frise, he made an escalade. +DUMMY/LJ014-0224.wav|As Dwyer survived, Cannon escaped the death sentence, which was commuted to penal servitude for life. +DUMMY/LJ005-0019.wav|refuted by abundant evidence, and having no foundation whatever in truth. +DUMMY/LJ042-0221.wav|With either great ambivalence, or cold calculation he prepared completely different answers to the same questions. +DUMMY/LJ001-0063.wav|which was generally more formally Gothic than the printing of the German workmen, +DUMMY/LJ030-0006.wav|They took off in the Presidential plane, Air Force One, at eleven a.m., arriving at San Antonio at one:thirty p.m., Eastern Standard Time. +DUMMY/LJ024-0054.wav|democracy will have failed far beyond the importance to it of any king of precedent concerning the judiciary. +DUMMY/LJ006-0044.wav|the same callous indifference to the moral well-being of the prisoners, the same want of employment and of all disciplinary control. +DUMMY/LJ039-0154.wav|four point eight to five point six seconds if the second shot missed, +DUMMY/LJ050-0090.wav|they seem unduly restrictive in continuing to require some manifestation of animus against a Government official. +DUMMY/LJ028-0421.wav|it was the beginning of the great collections of Babylonian antiquities in the museums of the Western world. +DUMMY/LJ033-0205.wav|then I would say the possibility exists, these fibers could have come from this blanket, end quote. +DUMMY/LJ019-0335.wav|The books and journals he was to keep were minutely specified, and his constant presence in or near the jail was insisted upon. +DUMMY/LJ013-0045.wav|Wallace's relations warned him against his Liverpool friend, +DUMMY/LJ037-0002.wav|Chapter four. The Assassin: Part six. +DUMMY/LJ018-0159.wav|This was all the police wanted to know. +DUMMY/LJ026-0140.wav|In the plant as in the animal metabolism must consist of anabolic and catabolic processes. +DUMMY/LJ014-0171.wav|I will briefly describe one or two of the more remarkable murders in the years immediately following, then pass on to another branch of crime. +DUMMY/LJ037-0007.wav|Three others subsequently identified Oswald from a photograph. +DUMMY/LJ033-0174.wav|microscopic and UV (ultra violet) characteristics, end quote. +DUMMY/LJ040-0110.wav|he apparently adjusted well enough there to have had an average, although gradually deteriorating, school record +DUMMY/LJ039-0192.wav|he had a total of between four point eight and five point six seconds between the two shots which hit +DUMMY/LJ032-0261.wav|When he appeared before the Commission, Michael Paine lifted the blanket +DUMMY/LJ040-0097.wav|Lee was brought up in this atmosphere of constant money problems, and I am sure it had quite an effect on him, and also Robert, end quote. +DUMMY/LJ037-0249.wav|Mrs. Earlene Roberts, the housekeeper at Oswald's roominghouse and the last person known to have seen him before he reached tenth Street and Patton Avenue, +DUMMY/LJ016-0248.wav|Marwood was proud of his calling, and when questioned as to whether his process was satisfactory, replied that he heard "no complaints." +DUMMY/LJ004-0083.wav|As Mr. Buxton pointed out, many old acts of parliament designed to protect the prisoner were still in full force. +DUMMY/LJ014-0029.wav|This was Delarue's watch, fully identified as such, which Hocker told his brother Delarue had given him the morning of the murder. +DUMMY/LJ021-0110.wav|have been best calculated to promote industrial recovery and a permanent improvement of business and labor conditions. +DUMMY/LJ003-0107.wav|he slept in the same bed with a highwayman on one side, and a man charged with murder on the other. +DUMMY/LJ039-0076.wav|Ronald Simmons, chief of the U.S. Army Infantry Weapons Evaluation Branch of the Ballistics Research Laboratory, said, quote, +DUMMY/LJ016-0347.wav|had undoubtedly a solemn, impressive effect upon those outside. +DUMMY/LJ001-0072.wav|After the end of the fifteenth century the degradation of printing, especially in Germany and Italy, +DUMMY/LJ024-0018.wav|Consequently, although there never can be more than fifteen, there may be only fourteen, or thirteen, or twelve. +DUMMY/LJ032-0180.wav|that the fibers were caught in the crevice of the rifle's butt plate, quote, in the recent past, end quote, +DUMMY/LJ010-0083.wav|and measures taken to arrest them when their plans were so far developed that no doubt could remain as to their guilt. +DUMMY/LJ002-0299.wav|and gave the garnish for the common side at that sum, which is five shillings more than Mr. Neild says was extorted on the common side. +DUMMY/LJ048-0143.wav|the Secret Service did not at the time of the assassination have any established procedure governing its relationships with them. +DUMMY/LJ012-0054.wav|Solomons, while waiting to appear in court, persuaded the turnkeys to take him to a public-house, where all might "refresh." +DUMMY/LJ019-0270.wav|Vegetables, especially the potato, that most valuable anti-scorbutic, was too often omitted. +DUMMY/LJ035-0164.wav|three minutes after the shooting. +DUMMY/LJ014-0326.wav|Maltby and Co. would issue warrants on them deliverable to the importer, and the goods were then passed to be stored in neighboring warehouses. +DUMMY/LJ001-0173.wav|The essential point to be remembered is that the ornament, whatever it is, whether picture or pattern-work, should form part of the page, +DUMMY/LJ050-0056.wav|On December twenty-six, nineteen sixty-three, the FBI circulated additional instructions to all its agents, +DUMMY/LJ003-0319.wav|provided only that their security was not jeopardized, and dependent upon the enforcement of another new rule, +DUMMY/LJ006-0040.wav|The fact was that the years as they passed, nearly twenty in all, had worked but little permanent improvement in this detestable prison. +DUMMY/LJ017-0231.wav|His body was found lying in a pool of blood in a night-dress, stabbed over and over again in the left side. +DUMMY/LJ017-0226.wav|One half of the mutineers fell upon him unawares with handspikes and capstan-bars. +DUMMY/LJ004-0239.wav|He had been committed for an offense for which he was acquitted. +DUMMY/LJ048-0112.wav|The Commission also regards the security arrangements worked out by Lawson and Sorrels at Love Field as entirely adequate. +DUMMY/LJ039-0125.wav|that Oswald was a good shot, somewhat better than or equal to -- better than the average let us say. +DUMMY/LJ030-0196.wav|He cried out, quote, Oh, no, no, no. My God, they are going to kill us all, end quote, +DUMMY/LJ010-0228.wav|He was released from Broadmoor in eighteen seventy-eight, and went abroad. +DUMMY/LJ045-0228.wav|On the other hand, he could have traveled some distance with the money he did have and he did return to his room where he obtained his revolver. +DUMMY/LJ028-0168.wav|in the other was the sacred precinct of Jupiter Belus, +DUMMY/LJ021-0140.wav|and in such an effort we should be able to secure for employers and employees and consumers +DUMMY/LJ009-0280.wav|Again the wretched creature succeeded in obtaining foothold, but this time on the left side of the drop. +DUMMY/LJ003-0159.wav|To constitute this the aristocratic quarter, unwarrantable demands were made upon the space properly allotted to the female felons, +DUMMY/LJ016-0274.wav|and the windows of the opposite houses, which commanded a good view, as usual fetched high prices. +DUMMY/LJ035-0014.wav|it sounded high and I immediately kind of looked up, +DUMMY/LJ033-0120.wav|which he believed was where the bag reached when it was laid on the seat with one edge against the door. +DUMMY/LJ045-0015.wav|which Johnson said he did not receive until after the assassination. The letter said in part, quote, +DUMMY/LJ003-0299.wav|the latter end of the nineteenth century, several of which still fall far short of our English ideal, +DUMMY/LJ032-0206.wav|After comparing the rifle in the simulated photograph with the rifle in Exhibit Number one thirty-three A, Shaneyfelt testified, quote, +DUMMY/LJ028-0494.wav|Between the several sections were wide spaces where foot soldiers and charioteers might fight. +DUMMY/LJ005-0099.wav|and report at length upon the condition of the prisons of the country. +DUMMY/LJ015-0144.wav|developed to a colossal extent the frauds he had already practiced as a subordinate. +DUMMY/LJ019-0221.wav|It was intended as far as possible that, except awaiting trial, no prisoner should find himself relegated to Newgate. +DUMMY/LJ003-0088.wav|in one, for seven years -- that of a man sentenced to death, for whom great interest had been made, but whom it was not thought right to pardon. +DUMMY/LJ045-0216.wav|nineteen sixty-three, merely to disarm her and to provide a justification of sorts, +DUMMY/LJ042-0135.wav|that he was not yet twenty years old when he went to the Soviet Union with such high hopes and not quite twenty-three when he returned bitterly disappointed. +DUMMY/LJ049-0196.wav|On the other hand, it is urged that all features of the protection of the President and his family should be committed to an elite and independent corps. +DUMMY/LJ018-0278.wav|This was the well and astutely devised plot of the brothers Bidwell, +DUMMY/LJ030-0238.wav|and then looked around again and saw more of this movement, and so I proceeded to go to the back seat and get on top of him. +DUMMY/LJ018-0309.wav|where probably the money still remains. +DUMMY/LJ041-0199.wav|is shown most clearly by his employment relations after his return from the Soviet Union. Of course, he made his real problems worse to the extent +DUMMY/LJ007-0076.wav|The lax discipline maintained in Newgate was still further deteriorated by the presence of two other classes of prisoners who ought never to have been inmates of such a jail. +DUMMY/LJ039-0118.wav|He had high motivation. He had presumably a good to excellent rifle and good ammunition. +DUMMY/LJ024-0019.wav|And there may be only nine. +DUMMY/LJ008-0085.wav|The fire had not quite burnt out at twelve, in nearly four hours, that is to say. +DUMMY/LJ018-0031.wav|This fixed the crime pretty certainly upon Müller, who had already left the country, thus increasing suspicion under which he lay. +DUMMY/LJ030-0032.wav|Dallas police stood at intervals along the fence and Dallas plain clothes men mixed in the crowd. +DUMMY/LJ050-0004.wav|General Supervision of the Secret Service +DUMMY/LJ039-0096.wav|This is a definite advantage to the shooter, the vehicle moving directly away from him and the downgrade of the street, and he being in an elevated position +DUMMY/LJ041-0195.wav|Oswald's interest in Marxism led some people to avoid him, +DUMMY/LJ047-0158.wav|After a moment's hesitation, she told me that he worked at the Texas School Book Depository near the downtown area of Dallas. +DUMMY/LJ050-0162.wav|In planning its data processing techniques, +DUMMY/LJ001-0051.wav|and paying great attention to the "press work" or actual process of printing, +DUMMY/LJ028-0136.wav|Of all the ancient descriptions of the famous walls and the city they protected, that of Herodotus is the fullest. +DUMMY/LJ034-0134.wav|Shortly after the assassination Brennan noticed +DUMMY/LJ019-0348.wav|Every facility was promised. The sanction of the Secretary of State would not be withheld if plans and estimates were duly submitted, +DUMMY/LJ010-0219.wav|While one stood over the fire with the papers, another stood with lighted torch to fire the house. +DUMMY/LJ011-0245.wav|Mr. Mullay called again, taking with him five hundred pounds in cash. Howard discovered this, and his manner was very suspicious; +DUMMY/LJ030-0035.wav|Organization of the Motorcade +DUMMY/LJ044-0135.wav|While he had drawn some attention to himself and had actually appeared on two radio programs, he had been attacked by Cuban exiles and arrested, +DUMMY/LJ045-0090.wav|He was very much interested in autobiographical works of outstanding statesmen of the United States, to whom his wife thought he compared himself. +DUMMY/LJ026-0034.wav|When any given "protist" has to be classified the case must be decided on its individual merits; +DUMMY/LJ045-0092.wav|as to the fact that he was an outstanding man, end quote. +DUMMY/LJ017-0050.wav|Palmer, who was only thirty-one at the time of his trial, was in appearance short and stout, with a round head +DUMMY/LJ036-0104.wav|Whaley picked Oswald. +DUMMY/LJ019-0055.wav|High authorities were in favor of continuous separation. +DUMMY/LJ010-0030.wav|The brutal ferocity of the wild beast once aroused, the same means, the same weapons were employed to do the dreadful deed, +DUMMY/LJ038-0047.wav|Some of the officers saw Oswald strike McDonald with his fist. Most of them heard a click which they assumed to be a click of the hammer of the revolver. +DUMMY/LJ009-0074.wav|Let us pass on. +DUMMY/LJ048-0069.wav|Efforts made by the Bureau since the assassination, on the other hand, +DUMMY/LJ003-0211.wav|They were never left quite alone for fear of suicide, and for the same reason they were searched for weapons or poisons. +DUMMY/LJ048-0053.wav|It is the conclusion of the Commission that, even in the absence of Secret Service criteria +DUMMY/LJ033-0093.wav|Frazier estimated that the bag was two feet long, quote, give and take a few inches, end quote, and about five or six inches wide. +DUMMY/LJ006-0149.wav|The turnkeys left the prisoners very much to themselves, never entering the wards after locking-up time, at dusk, till unlocking next morning, +DUMMY/LJ018-0211.wav|The false coin was bought by an agent from an agent, and dealings were carried on secretly at the "Clock House" in Seven Dials. +DUMMY/LJ008-0054.wav|This contrivance appears to have been copied with improvements from that which had been used in Dublin at a still earlier date, +DUMMY/LJ040-0052.wav|that his commitment to Marxism was an important factor influencing his conduct during his adult years. +DUMMY/LJ028-0023.wav|Two weeks pass, and at last you stand on the eastern edge of the plateau +DUMMY/LJ009-0184.wav|Lord Ferrers' body was brought to Surgeons' Hall after execution in his own carriage and six; +DUMMY/LJ005-0252.wav|A committee was appointed, under the presidency of the Duke of Richmond +DUMMY/LJ015-0266.wav|has probably no parallel in the annals of crime. Saward himself is a striking and in some respects an unique figure in criminal history. +DUMMY/LJ017-0059.wav|even after sentence, and until within a few hours of execution, he was buoyed up with the hope of reprieve. +DUMMY/LJ024-0034.wav|What do they mean by the words "packing the Court"? +DUMMY/LJ016-0089.wav|He was engaged in whitewashing and cleaning; the officer who had him in charge left him on the stairs leading to the gallery. +DUMMY/LJ039-0227.wav|with two hits, within four point eight and five point six seconds. +DUMMY/LJ001-0096.wav|have now come into general use and are obviously a great improvement on the ordinary "modern style" in use in England, which is in fact the Bodoni type +DUMMY/LJ018-0129.wav|who threatened to betray the theft. But Brewer, either before or after this, succumbed to temptation, +DUMMY/LJ010-0157.wav|and that, as he was starving, he had resolved on this desperate deed, +DUMMY/LJ038-0264.wav|He concluded that, quote, the general rifling characteristics of the rifle are of the same type as those found on the bullet +DUMMY/LJ031-0165.wav|When security arrangements at the airport were complete, the Secret Service made the necessary arrangements for the Vice President to leave the hospital. +DUMMY/LJ018-0244.wav|The effect of establishing the forgeries would be to restore to the Roupell family lands for which a price had already been paid +DUMMY/LJ007-0071.wav|in the face of impediments confessedly discouraging +DUMMY/LJ028-0340.wav|Such of the Babylonians as witnessed the treachery took refuge in the temple of Jupiter Belus; +DUMMY/LJ017-0164.wav|with the idea of subjecting her to the irritant poison slowly but surely until the desired effect, death, was achieved. +DUMMY/LJ048-0197.wav|I then told the officers that their primary duty was traffic and crowd control and that they should be alert for any persons who might attempt to throw anything +DUMMY/LJ013-0098.wav|Mr. Oxenford having denied that he had made any transfer of stock, the matter was at once put into the hands of the police. +DUMMY/LJ012-0049.wav|led him to think seriously of trying his fortunes in another land. +DUMMY/LJ030-0014.wav|quote, that the crowd was about the same as the one which came to see him before but there were one hundred thousand extra people on hand who came to see Mrs. Kennedy. +DUMMY/LJ014-0186.wav|A milliner's porter, +DUMMY/LJ015-0027.wav|Yet even so early as the death of the first Sir John Paul, +DUMMY/LJ047-0049.wav|Marina Oswald, however, recalled that her husband was upset by this interview. +DUMMY/LJ012-0021.wav|at fourteen he was a pickpocket and a "duffer," or a seller of sham goods. +DUMMY/LJ003-0140.wav|otherwise he would have been stripped of his clothes. End quote. +DUMMY/LJ042-0130.wav|Shortly thereafter, less than eighteen months after his defection, about six weeks before he met Marina Prusakova, +DUMMY/LJ019-0180.wav|His letter to the Corporation, under date fourth June, +DUMMY/LJ017-0108.wav|He was struck with the appearance of the corpse, which was not emaciated, as after a long disease ending in death; +DUMMY/LJ006-0268.wav|Women saw men if they merely pretended to be wives; even boys were visited by their sweethearts. +DUMMY/LJ044-0125.wav|of residence in the U.S.S.R. against any cause which I join, by association, +DUMMY/LJ015-0231.wav|It was Tester's business, who had access to the railway company's books, to watch for this. +DUMMY/LJ002-0225.wav|The rentals of rooms and fees went to the warden, whose income was two thousand three hundred seventy-two pounds. +DUMMY/LJ034-0072.wav|The employees raced the elevators to the first floor. Givens saw Oswald standing at the gate on the fifth floor as the elevator went by. +DUMMY/LJ045-0033.wav|He began to treat me better. He helped me more -- although he always did help. But he was more attentive, end quote. +DUMMY/LJ031-0058.wav|to infuse blood and fluids into the circulatory system. +DUMMY/LJ029-0197.wav|During November the Dallas papers reported frequently on the plans for protecting the President, stressing the thoroughness of the preparations. +DUMMY/LJ043-0047.wav|Oswald and his family lived for a brief period with his mother at her urging, but Oswald soon decided to move out. +DUMMY/LJ021-0026.wav|seems necessary to produce the same result of justice and right conduct +DUMMY/LJ003-0230.wav|The prison allowances were eked out by the broken victuals generously given by several eating-house keepers in the city, +DUMMY/LJ037-0252.wav|Ted Callaway, who saw the gunman moments after the shooting, testified that Commission Exhibit Number one sixty-two +DUMMY/LJ031-0008.wav|Meanwhile, Chief Curry ordered the police base station to notify Parkland Hospital that the wounded President was en route. +DUMMY/LJ030-0021.wav|all one had to do was get a high building someday with a telescopic rifle, and there was nothing anybody could do to defend against such an attempt. +DUMMY/LJ046-0179.wav|being reviewed regularly. +DUMMY/LJ025-0118.wav|and that, however diverse may be the fabrics or tissues of which their bodies are composed, all these varied structures result +DUMMY/LJ028-0278.wav|Zopyrus, when they told him, not thinking that it could be true, went and saw the colt with his own eyes; +DUMMY/LJ007-0090.wav|Not only did their presence tend greatly to interfere with the discipline of the prison, but their condition was deplorable in the extreme. +DUMMY/LJ045-0045.wav|that she would be able to leave the Soviet Union. Marina Oswald has denied this. +DUMMY/LJ028-0289.wav|For he cut off his own nose and ears, and then, clipping his hair close and flogging himself with a scourge, +DUMMY/LJ009-0276.wav|Calcraft, the moment he had adjusted the cap and rope, ran down the steps, drew the bolt, and disappeared. +DUMMY/LJ031-0122.wav|treated the gunshot wound in the left thigh. +DUMMY/LJ016-0205.wav|he received a retaining fee of five pounds, five shillings, with the usual guinea for each job; +DUMMY/LJ019-0248.wav|leading to an inequality, uncertainty, and inefficiency of punishment productive of the most prejudicial results. +DUMMY/LJ033-0183.wav|it was not surprising that the replica sack made on December one, nineteen sixty-three, +DUMMY/LJ037-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ018-0218.wav|In eighteen fifty-five +DUMMY/LJ001-0102.wav|Here and there a book is printed in France or Germany with some pretension to good taste, +DUMMY/LJ007-0125.wav|It was diverted from its proper uses, and, as the "place of the greatest comfort," was allotted to persons who should not have been sent to Newgate at all. +DUMMY/LJ050-0022.wav|A formal and thorough description of the responsibilities of the advance agent is now in preparation by the Service. +DUMMY/LJ028-0212.wav|On the night of the eleventh day Gobrias killed the son of the King. +DUMMY/LJ028-0357.wav|yet we may be sure that Babylon was taken by Darius only by use of stratagem. Its walls were impregnable. +DUMMY/LJ014-0199.wav|there was no case to make out; why waste money on lawyers for the defense? His demeanor was cool and collected throughout; +DUMMY/LJ016-0077.wav|A man named Lears, under sentence of transportation for an attempt at murder on board ship, got up part of the way, +DUMMY/LJ009-0194.wav|and that executors or persons having lawful possession of the bodies +DUMMY/LJ014-0094.wav|Discovery of the murder came in this wise. O'Connor, a punctual and well-conducted official, was at once missed at the London Docks. +DUMMY/LJ001-0079.wav|Caslon's type is clear and neat, and fairly well designed; +DUMMY/LJ026-0052.wav|In the nutrition of the animal the most essential and characteristic part of the food supply is derived from vegetable +DUMMY/LJ013-0005.wav|One of the earliest of the big operators in fraudulent finance was Edward Beaumont Smith, +DUMMY/LJ033-0072.wav|I then stepped off of it and the officer picked it up in the middle and it bent so. +DUMMY/LJ036-0067.wav|According to McWatters, the Beckley bus was behind the Marsalis bus, but he did not actually see it. +DUMMY/LJ025-0098.wav|and it is probable that amyloid substances are universally present in the animal organism, though not in the precise form of starch. +DUMMY/LJ005-0257.wav|during which time a host of witnesses were examined, and the committee presented three separate reports, +DUMMY/LJ004-0024.wav|Thus in eighteen thirteen the exaction of jail fees had been forbidden by law, +DUMMY/LJ049-0154.wav|In eighteen ninety-four, +DUMMY/LJ039-0059.wav|(three) his experience and practice after leaving the Marine Corps, and (four) the accuracy of the weapon and the quality of the ammunition. +DUMMY/LJ007-0150.wav|He is allowed intercourse with prostitutes who, in nine cases out of ten, have originally conduced to his ruin; +DUMMY/LJ015-0001.wav|Chronicles of Newgate, Volume two. By Arthur Griffiths. Section eighteen: Newgate notorieties continued, part three. +DUMMY/LJ010-0158.wav|feeling, as he said, that he might as well be shot or hanged as remain in such a state. +DUMMY/LJ010-0281.wav|who had borne the Queen's commission, first as cornet, and then lieutenant, in the tenth Hussars. +DUMMY/LJ033-0055.wav|and he could disassemble it more rapidly. +DUMMY/LJ015-0218.wav|A new accomplice was now needed within the company's establishment, and Pierce looked about long before he found the right person. +DUMMY/LJ027-0006.wav|In all these lines the facts are drawn together by a strong thread of unity. +DUMMY/LJ016-0049.wav|He had here completed his ascent. +DUMMY/LJ006-0088.wav|It was not likely that a system which left innocent men -- for the great bulk of new arrivals were still untried +DUMMY/LJ042-0133.wav|a great change must have occurred in Oswald's thinking to induce him to return to the United States. +DUMMY/LJ045-0234.wav|While he did become enraged at at least one point in his interrogation, +DUMMY/LJ046-0033.wav|The adequacy of existing procedures can fairly be assessed only after full consideration of the difficulty of the protective assignment, +DUMMY/LJ037-0061.wav|and having, quote, somewhat bushy, end quote, hair. +DUMMY/LJ032-0025.wav|the officers of Klein's discovered that a rifle bearing serial number C two seven six six had been shipped to one A. Hidell, +DUMMY/LJ047-0197.wav|in view of all the information concerning Oswald in its files, should have alerted the Secret Service to Oswald's presence in Dallas +DUMMY/LJ018-0130.wav|and stole paper on a much larger scale than Brown. +DUMMY/LJ005-0265.wav|It was recommended that the dietaries should be submitted and approved like the rules; that convicted prisoners should not receive any food but the jail allowance; +DUMMY/LJ044-0105.wav|He presented Arnold Johnson, Gus Hall, +DUMMY/LJ015-0043.wav|This went on for some time, and might never have been discovered had some good stroke of luck provided any of the partners +DUMMY/LJ030-0125.wav|On several occasions when the Vice President's car was slowed down by the throng, Special Agent Youngblood stepped out to hold the crowd back. +DUMMY/LJ043-0140.wav|He also studied Dallas bus schedules to prepare for his later use of buses to travel to and from General Walker's house. +DUMMY/LJ002-0220.wav|In consequence of these disclosures, both Bambridge and Huggin, his predecessor in the office, were committed to Newgate, +DUMMY/LJ034-0117.wav|At one:twenty-nine p.m. the police radio reported +DUMMY/LJ018-0276.wav|The first plot was against Mr. Harry Emmanuel, but he escaped, and the attempt was made upon Loudon and Ryder. +DUMMY/LJ004-0077.wav|nor has he a right to poison or starve his fellow-creatures." +DUMMY/LJ042-0194.wav|they should not be confused with slowness, indecision or fear. Only the intellectually fearless could even be remotely attracted to our doctrine, +DUMMY/LJ029-0114.wav|The route chosen from the airport to Main Street was the normal one, except where Harwood Street was selected as the means of access to Main Street +DUMMY/LJ014-0194.wav|The policemen were now in possession; +DUMMY/LJ032-0027.wav|According to its microfilm records, Klein's received an order for a rifle on March thirteen, nineteen sixty-three, +DUMMY/LJ048-0289.wav|However, there is no evidence that these men failed to take any action in Dallas within their power that would have averted the tragedy. +DUMMY/LJ043-0188.wav|that he was the leader of a fascist organization, and when I said that even though all of that might be true, just the same he had no right to take his life, +DUMMY/LJ011-0118.wav|In eighteen twenty-nine the gallows claimed two more victims for this offense. +DUMMY/LJ040-0201.wav|After her interview with Mrs. Oswald, +DUMMY/LJ033-0056.wav|While the rifle may have already been disassembled when Oswald arrived home on Thursday, he had ample time that evening to disassemble the rifle +DUMMY/LJ047-0073.wav|Hosty considered the information to be, quote, stale, unquote, by that time, and did not attempt to verify Oswald's reported statement. +DUMMY/LJ001-0153.wav|only nominally so, however, in many cases, since when he uses a headline he counts that in, +DUMMY/LJ007-0158.wav|or any kind of moral improvement was impossible; the prisoner's career was inevitably downward, till he struck the lowest depths. +DUMMY/LJ028-0502.wav|The Ishtar gateway leading to the palace was encased with beautiful blue glazed bricks, +DUMMY/LJ028-0226.wav|Though Herodotus wrote nearly a hundred years after Babylon fell, his story seems to bear the stamp of truth. +DUMMY/LJ010-0038.wav|as there had been before; as in the year eighteen forty-nine, a year memorable for the Rush murders at Norwich, +DUMMY/LJ019-0241.wav|But in the interval very comprehensive and, I think it must be admitted, salutary changes were successively introduced into the management of prisons. +DUMMY/LJ001-0094.wav|were induced to cut punches for a series of "old style" letters. +DUMMY/LJ001-0015.wav|the forms of printed letters should be beautiful, and that their arrangement on the page should be reasonable and a help to the shapeliness of the letters themselves. +DUMMY/LJ047-0015.wav|From defection to return to Fort Worth. +DUMMY/LJ044-0139.wav|since there was no background to the New Orleans FPCC, quote, organization, end quote, which consisted solely of Oswald. +DUMMY/LJ050-0031.wav|that the Secret Service consciously set about the task of inculcating and maintaining the highest standard of excellence and esprit, for all of its personnel. +DUMMY/LJ050-0235.wav|It has also used other Federal law enforcement agents during Presidential visits to cities in which such agents are stationed. +DUMMY/LJ050-0137.wav|FBI, and the Secret Service. +DUMMY/LJ031-0109.wav|At one:thirty-five p.m., after Governor Connally had been moved to the operating room, Dr. Shaw started the first operation +DUMMY/LJ031-0041.wav|He noted that the President was blue-white or ashen in color; had slow, spasmodic, agonal respiration without any coordination; +DUMMY/LJ021-0139.wav|There should be at least a full and fair trial given to these means of ending industrial warfare; +DUMMY/LJ029-0004.wav|The narrative of these events is based largely on the recollections of the participants, +DUMMY/LJ023-0122.wav|It was said in last year's Democratic platform, +DUMMY/LJ005-0264.wav|inspectors of prisons should be appointed, who should visit all the prisons from time to time and report to the Secretary of State. +DUMMY/LJ002-0105.wav|and beyond it was a room called the "wine room," because formerly used for the sale of wine, but +DUMMY/LJ017-0035.wav|in the interests and for the due protection of the public, that the fullest and fairest inquiry should be made, +DUMMY/LJ048-0252.wav|Three of these agents occupied positions on the running boards of the car, and the fourth was seated in the car. +DUMMY/LJ013-0109.wav|The proceeds of the robbery were lodged in a Boston bank, +DUMMY/LJ039-0139.wav|Oswald obtained a hunting license, joined a hunting club and went hunting about six times, as discussed more fully in chapter six. +DUMMY/LJ044-0047.wav|that anyone ever attacked any street demonstration in which Oswald was involved, except for the Bringuier incident mentioned above, +DUMMY/LJ016-0417.wav|Catherine Wilson, the poisoner, was reserved and reticent to the last, expressing no contrition, but also no fear -- +DUMMY/LJ045-0178.wav|he left his wedding ring in a cup on the dresser in his room. He also left one hundred seventy dollars in a wallet in one of the dresser drawers. +DUMMY/LJ009-0172.wav|While in London, for instance, in eighteen twenty-nine, twenty-four persons had been executed for crimes other than murder, +DUMMY/LJ049-0202.wav|incident to its responsibilities. +DUMMY/LJ032-0103.wav|The name "Hidell" was stamped on some of the "Chapter's" printed literature and on the membership application blanks. +DUMMY/LJ013-0091.wav|and Elder had to be assisted by two bank porters, who carried it for him to a carriage waiting near the Mansion House. +DUMMY/LJ037-0208.wav|nineteen dollars, ninety-five cents, plus one dollar, twenty-seven cents shipping charge, had been collected from the consignee, Hidell. +DUMMY/LJ014-0128.wav|her hair was dressed in long crepe bands. She had lace ruffles at her wrist, and wore primrose-colored kid gloves. +DUMMY/LJ015-0007.wav|This affected Cole's credit, and ugly reports were in circulation charging him with the issue of simulated warrants. +DUMMY/LJ036-0169.wav|he would have reached his destination at approximately twelve:fifty-four p.m. +DUMMY/LJ021-0040.wav|The second step we have taken in the restoration of normal business enterprise +DUMMY/LJ015-0036.wav|The bank was already insolvent, +DUMMY/LJ034-0041.wav|Although Bureau experiments had shown that twenty-four hours was a likely maximum time, Latona stated +DUMMY/LJ009-0192.wav|The dissection of executed criminals was abolished soon after the discovery of the crime of burking, +DUMMY/LJ037-0248.wav|The eyewitnesses vary in their identification of the jacket. +DUMMY/LJ015-0289.wav|As each transaction was carried out from a different address, and a different messenger always employed, +DUMMY/LJ005-0072.wav|After a few years of active exertion the Society was rewarded by fresh legislation. +DUMMY/LJ023-0047.wav|The three horses are, of course, the three branches of government -- the Congress, the Executive and the courts. +DUMMY/LJ009-0126.wav|Hardly any one. +DUMMY/LJ034-0097.wav|The window was approximately one hundred twenty feet away. +DUMMY/LJ028-0462.wav|They were laid in bitumen. +DUMMY/LJ046-0055.wav|It is now possible for Presidents to travel the length and breadth of a land far larger than the United States +DUMMY/LJ019-0371.wav|Yet the law was seldom if ever enforced. +DUMMY/LJ039-0207.wav|Although all of the shots were a few inches high and to the right of the target, +DUMMY/LJ002-0174.wav|Mr. Buxton's friends at once paid the forty shillings, and the boy was released. +DUMMY/LJ016-0233.wav|In his own profession +DUMMY/LJ026-0108.wav|It is clear that there are upward and downward currents of water containing food (comparable to blood of an animal), +DUMMY/LJ038-0035.wav|Oswald rose from his seat, bringing up both hands. +DUMMY/LJ026-0148.wav|water which is lost by evaporation, especially from the leaf surface through the stomata; +DUMMY/LJ001-0186.wav|the position of our Society that a work of utility might be also a work of art, if we cared to make it so. +DUMMY/LJ016-0264.wav|The upturned faces of the eager spectators resembled those of the 'gods' at Drury Lane on Boxing Night; +DUMMY/LJ009-0041.wav|The occupants of this terrible black pew were the last always to enter the chapel. +DUMMY/LJ010-0297.wav|But there were other notorious cases of forgery. +DUMMY/LJ040-0018.wav|the Commission is not able to reach any definite conclusions as to whether or not he was, quote, sane, unquote, under prevailing legal standards. +DUMMY/LJ005-0253.wav|"to inquire into and report upon the several jails and houses of correction in the counties, cities, and corporate towns within England and Wales +DUMMY/LJ027-0176.wav|Fishes first appeared in the Devonian and Upper Silurian in very reptilian or rather amphibian forms. +DUMMY/LJ034-0035.wav|The position of this palmprint on the carton was parallel with the long axis of the box, and at right angles with the short axis; +DUMMY/LJ016-0054.wav|But he did not like the risk of entering a room by the fireplace, and the chances of detection it offered. +DUMMY/LJ018-0262.wav|Roupell received the announcement with a cheerful countenance, +DUMMY/LJ044-0237.wav|with thirteen dollars, eighty-seven cents when considerably greater resources were available to him. +DUMMY/LJ034-0166.wav|Two other witnesses were able to offer partial descriptions of a man they saw in the southeast corner window +DUMMY/LJ016-0238.wav|"just to steady their legs a little;" in other words, to add his weight to that of the hanging bodies. +DUMMY/LJ042-0198.wav|The discussion above has already set forth examples of his expression of hatred for the United States. +DUMMY/LJ031-0189.wav|At two:thirty-eight p.m., Eastern Standard Time, Lyndon Baines Johnson took the oath of office as the thirty-sixth President of the United States. +DUMMY/LJ050-0084.wav|or, quote, other high government officials in the nature of a complaint coupled with an expressed or implied determination to use a means, +DUMMY/LJ044-0158.wav|As for my return entrance visa please consider it separately. End quote. +DUMMY/LJ045-0082.wav|it appears that Marina Oswald also complained that her husband was not able to provide more material things for her. +DUMMY/LJ045-0190.wav|appeared in The Dallas Times Herald on November fifteen, nineteen sixty-three. +DUMMY/LJ035-0155.wav|The only exit from the office in the direction Oswald was moving was through the door to the front stairway. +DUMMY/LJ044-0004.wav|Political Activities +DUMMY/LJ046-0016.wav|The Commission has not undertaken a comprehensive examination of all facets of this subject; +DUMMY/LJ019-0368.wav|The latter too was to be laid before the House of Commons. +DUMMY/LJ010-0062.wav|But they proceeded in all seriousness, and would have shrunk from no outrage or atrocity in furtherance of their foolhardy enterprise. +DUMMY/LJ033-0159.wav|It was from Oswald's right hand, in which he carried the long package as he walked from Frazier's car to the building. +DUMMY/LJ002-0171.wav|The boy declared he saw no one, and accordingly passed through without paying the toll of a penny. +DUMMY/LJ002-0298.wav|in his evidence in eighteen fourteen, said it was more, +DUMMY/LJ012-0219.wav|and in one corner, at some depth, a bundle of clothes were unearthed, which, with a hairy cap, +DUMMY/LJ017-0190.wav|After this came the charge of administering oil of vitriol, which failed, as has been described. +DUMMY/LJ019-0179.wav|This, with a scheme for limiting the jail to untried prisoners, had been urgently recommended by Lord John Russell in eighteen thirty. +DUMMY/LJ050-0188.wav|each patrolman might be given a prepared booklet of instructions explaining what is expected of him. The Secret Service has expressed concern +DUMMY/LJ006-0043.wav|The disgraceful overcrowding had been partially ended, but the same evils of indiscriminate association were still present; there was the old neglect of decency, +DUMMY/LJ029-0060.wav|A number of people who resembled some of those in the photographs were placed under surveillance at the Trade Mart. +DUMMY/LJ019-0052.wav|Both systems came to us from the United States. The difference was really more in degree than in principle, +DUMMY/LJ037-0081.wav|Later in the day each woman found an empty shell on the ground near the house. These two shells were delivered to the police. +DUMMY/LJ048-0200.wav|paying particular attention to the crowd for any unusual activity. +DUMMY/LJ016-0426.wav|come along, gallows. +DUMMY/LJ008-0182.wav|A tremendous crowd assembled when Bellingham was executed in eighteen twelve for the murder of Spencer Percival, at that time prime minister; +DUMMY/LJ043-0107.wav|Upon moving to New Orleans on April twenty-four, nineteen sixty-three, +DUMMY/LJ006-0084.wav|and so numerous were his opportunities of showing favoritism, that all the prisoners may be said to be in his power. +DUMMY/LJ025-0081.wav|has no permanent digestive cavity or mouth, but takes in its food anywhere and digests, so to speak, all over its body. +DUMMY/LJ019-0042.wav|These were either satisfied with a makeshift, and modified existing buildings, without close regard to their suitability, or for a long time did nothing at all. +DUMMY/LJ047-0240.wav|They agree that Hosty told Revill +DUMMY/LJ032-0012.wav|the resistance to arrest and the attempted shooting of another police officer by the man (Lee Harvey Oswald) subsequently accused of assassinating President Kennedy +DUMMY/LJ050-0209.wav|The assistant to the Director of the FBI testified that diff --git a/models/tacotron2/filelists/ljs_audio_text_train_filelist.txt b/models/tacotron2/filelists/ljs_audio_text_train_filelist.txt new file mode 100755 index 0000000000..237c55bf62 --- /dev/null +++ b/models/tacotron2/filelists/ljs_audio_text_train_filelist.txt @@ -0,0 +1,12500 @@ +DUMMY/LJ050-0234.wav|It has used other Treasury law enforcement agents on special experiments in building and route surveys in places to which the President frequently travels. +DUMMY/LJ019-0373.wav|to avail himself of his powers, as it was difficult to bring home the derelictions of duties and evasion of the acts. Too much was left to the inspectors. +DUMMY/LJ050-0207.wav|Although Chief Rowley does not complain about the pay scale for Secret Service agents, +DUMMY/LJ048-0203.wav|The three officers confirm that their primary concern was crowd and traffic control, +DUMMY/LJ003-0182.wav|The tried and the untried, young and old, were herded together +DUMMY/LJ044-0166.wav|According to Marina Oswald, he thought that would help him when he got to Cuba. +DUMMY/LJ019-0208.wav|The proposal made was to purchase some fifty thousand square feet between Newgate, Warwick Lane, and the Sessions House, +DUMMY/LJ021-0146.wav|I shall seek assurances of the making and maintenance of agreements, which can be mutually relied upon, +DUMMY/LJ013-0214.wav|who took a carving-knife from the sideboard in the dining-room, went upstairs to Lord William's bedroom, and drew the knife across his throat. +DUMMY/LJ011-0256.wav|By this time the neighbors were aroused, and several people came to the scene of the affray. +DUMMY/LJ014-0083.wav|which, having possessed herself of the murdered man's keys, she rifled from end to end. +DUMMY/LJ035-0121.wav|This is the period during which Oswald would have descended the stairs. In all likelihood +DUMMY/LJ049-0118.wav|Enactment of this statute would mean that the investigation of any of the acts covered and of the possibility of a further attempt +DUMMY/LJ006-0132.wav|All the wardsmen alike were more or less irresponsible. +DUMMY/LJ049-0084.wav|Murder of the President has never been covered by Federal law, however, so that once it became reasonably clear that the killing was the act of a single person, +DUMMY/LJ012-0052.wav|He claimed to be admitted to bail, and was taken from Newgate on a writ of habeas before one of the judges sitting at Westminster. +DUMMY/LJ011-0203.wav|Monsieur le Maire was appealed to, and decided to leave it to the young lady, who at once abandoned Wakefield. +DUMMY/LJ019-0141.wav|The old wards, day rooms and sleeping rooms combined, of which the reader has already heard so much, +DUMMY/LJ003-0322.wav|except for the use of the debtors, or as medical comforts for the infirmary. +DUMMY/LJ027-0028.wav|Such structures or organs are most often found internally. +DUMMY/LJ047-0068.wav|and his recommendation was accepted. He decided, however, not to interview Marina Oswald at that time, and merely determined +DUMMY/LJ036-0053.wav|End quote. As Mrs. Bledsoe said these words, she pointed to her right elbow. +DUMMY/LJ007-0046.wav|laughing and uproarious, utterly unmindful of the companionship of men upon whom lay the shadow of an impending shameful death. +DUMMY/LJ033-0149.wav|the FBI Laboratory developed a latent palmprint and latent fingerprint on the bag. +DUMMY/LJ018-0334.wav|Webster, it may be mentioned here, was one of the worst prisoners ever remembered in Newgate +DUMMY/LJ043-0073.wav|On October nine, nineteen sixty-two he went to the Dallas office of the Texas Employment Commission +DUMMY/LJ028-0169.wav|a square enclosure two furlongs each way, with gates of solid brass; which was also remaining in my time. +DUMMY/LJ032-0052.wav|According to Klein's shipping order form, one Italian carbine six point five x four x scope, +DUMMY/LJ015-0030.wav|The bank had been conducted on false principles; +DUMMY/LJ006-0115.wav|Their original capital had been a few shillings, and for this they purchased the right to tax their fellows to the extent of pounds per week. +DUMMY/LJ039-0237.wav|The Commission agrees with the testimony of Marine marksmanship expert Zahm +DUMMY/LJ027-0081.wav|exhibit a remarkable series of homologies pointing to a five-toed ancestor, +DUMMY/LJ031-0045.wav|Dr. Carrico noted two wounds: a small bullet wound in the front lower neck, +DUMMY/LJ019-0393.wav|Three years after the advent of the prison commissioners, it was decided that Newgate was an excessively costly and redundant establishment. +DUMMY/LJ040-0209.wav|well I've got to live with her. I guess I love her, end quote. +DUMMY/LJ004-0222.wav|Meat was no longer issued raw, to be imperfectly cooked before a ward fire and bolted gluttonously, the whole two pounds at one sitting. +DUMMY/LJ009-0139.wav|In spite of the throng, owing to the excellent arrangements made by the sheriffs, +DUMMY/LJ020-0087.wav|One word with regard to getting up early in order to give dough a chance for the second rising. +DUMMY/LJ008-0175.wav|This went on for more than an hour, and until the malefactors were cut down and the gallows removed; +DUMMY/LJ025-0058.wav|but the fact, important as it was, fell into oblivion and had to be rediscovered by Treviranus in eighteen oh seven. +DUMMY/LJ022-0011.wav|When one of these ships is under construction and the steel frames have been set in the keel, +DUMMY/LJ037-0219.wav|Oswald's Jacket +DUMMY/LJ008-0190.wav|were blocked by persons who had overflowed from the area in front of the jail. +DUMMY/LJ019-0357.wav|The Secretary of State was empowered to deal rather summarily with "inadequate" prisons, in other words, +DUMMY/LJ043-0122.wav|who had previously interviewed Oswald, saw him on television and heard a radio debate in which he engaged on August twenty-one, nineteen sixty-three. +DUMMY/LJ013-0161.wav|the furniture turned upside down, the drawers of the escritoire open and rifled, +DUMMY/LJ002-0319.wav|issued once a week, and divided as far as it would go -- a very precarious and uncertain ration. +DUMMY/LJ001-0178.wav|The due relation of letter to pictures and other ornament was thoroughly understood by the old printers; so that +DUMMY/LJ015-0170.wav|Redpath was called in and informed of the intended investigation. He tried to stave off the evil hour by declaring that everything was perfectly right; +DUMMY/LJ012-0224.wav|who examined Bishop and his fellows, and further incriminating evidence adduced, to the effect that the prisoners had bartered for a coach to carry "a stiff 'un"; +DUMMY/LJ047-0047.wav|According to Fain's contemporaneous memorandum and his present recollection, +DUMMY/LJ020-0052.wav|Cut this into round cakes with a biscuit-cutter or a sharp-edged tumbler and fold, not quite in the middle, +DUMMY/LJ013-0077.wav|Barber and Fletcher were both transported for life, although Fletcher declared that Barber was innocent, and had no guilty knowledge of what was being done. +DUMMY/LJ048-0153.wav|while on the overpass overlooking the assassination scene railroad and yard terminal workmen were permitted to remain under police supervision, +DUMMY/LJ003-0028.wav|and no doubt made the meat also pay toll. +DUMMY/LJ002-0277.wav|There were fifteen rooms of various sizes, and as the numbers imprisoned rarely exceeded five-and-twenty, the place was never overcrowded, +DUMMY/LJ009-0050.wav|He has, however, doubly earned his sentence, and is actually condemned for burglary committed since his arrival in England. +DUMMY/LJ003-0084.wav|Quote, a common-sized man, says the keeper, Mr. Newman, can turn in nineteen inches, end quote. +DUMMY/LJ018-0305.wav|The bills were sent as a matter of form to the drawer to have the date added, and the forgery was at once detected. +DUMMY/LJ015-0118.wav|paid him the compliment of describing him as "a young man of great intelligence, considerable powers of mind, +DUMMY/LJ034-0128.wav|But the testimony of these employees, together with photographs subsequently taken of them at the scene of the assassination, +DUMMY/LJ035-0074.wav|The second test run required one minute and fifteen seconds. +DUMMY/LJ028-0466.wav|This inner part was about twenty-four feet wide, and at intervals of about one hundred and forty feet it was surmounted with towers. +DUMMY/LJ048-0241.wav|These agents were each relieved for a half hour break during the night. +DUMMY/LJ028-0184.wav|he extended his empire to the Mediterranean and even to Egypt. +DUMMY/LJ030-0034.wav|Approximately ten minutes after the arrival at Love Field, the President and Mrs. Kennedy went to the Presidential automobile to begin the motorcade. +DUMMY/LJ005-0222.wav|and as the window-frames would not shut tight, the prisoners complained much of the cold, especially at night. +DUMMY/LJ004-0067.wav|or to make him sleep in close contact with the victims of contagious and loathsome disease, +DUMMY/LJ010-0215.wav|This letter went on to say that a new member had been introduced by Lt. Mars, "a fine, tall, gentlemanly young man, +DUMMY/LJ013-0096.wav|Suspicions were aroused when it was found that he had been employed in selling stock for Mr. Oxenford, which developed into certainty +DUMMY/LJ028-0397.wav|When Julian saw this, he caused a large part of the wall to be overthrown, and as the beasts escaped they were shot down by his soldiers. +DUMMY/LJ046-0069.wav|I think he felt very strongly that the President ought to get out of Washington, and go meet the people on a regular basis. End quote. +DUMMY/LJ022-0096.wav|and of making certain that people who are employed will be taken from the relief rolls. +DUMMY/LJ041-0151.wav|An old lady handed me a pamphlet about saving the Rosenbergs. I looked at that paper and I still remember it for some reason, I don't know why. End quote. +DUMMY/LJ030-0109.wav|The Vice-Presidential car +DUMMY/LJ014-0064.wav|He lived at Mile End, whence he walked often to call at three, Minver Place, Bermondsey, the residence of his old love. +DUMMY/LJ015-0273.wav|But Saward's operations were mainly directed to the fabrication and uttering of forged cheques. +DUMMY/LJ046-0012.wav|there have been attempts on the lives of one out of every three. +DUMMY/LJ009-0100.wav|struggles violently; his feet, legs, hands, and arms, even the muscles of his back, +DUMMY/LJ050-0181.wav|The Commission regards this as a most useful innovation and urges that the practice be continued. +DUMMY/LJ002-0264.wav|Legacies had also been left to free a certain number of debtors, notably that of one hundred pounds per annum +DUMMY/LJ009-0118.wav|could hardly witness this ceremony without being affected. The men, who were so lately snatched from the jaws of death, +DUMMY/LJ025-0154.wav|and which certainly hold good for the vast majority of animals and plants, are of universal application. +DUMMY/LJ006-0077.wav|The fact was, he did not keep the classification of prisoners on first arrival in his own hands, nor even in that of his officers. +DUMMY/LJ049-0167.wav|in the field of preventive investigation in regard to the President's security. +DUMMY/LJ009-0012.wav|This was noticeable with Fauntleroy, who, on account of his birth and antecedents, was allowed to occupy a turnkey's room, +DUMMY/LJ035-0200.wav|Fritz believed that he learned of Oswald's absence after the rifle was found. +DUMMY/LJ027-0059.wav|These, therefore, may well be called backboned animals. +DUMMY/LJ016-0022.wav|A harrowing scene followed; +DUMMY/LJ040-0055.wav|The discussion below will describe the events known to the Commission which most clearly reveals the formation and nature of Oswald's character. +DUMMY/LJ019-0178.wav|and abandoned because of the expense. As to the entire reconstruction of Newgate, nothing had been done as yet. +DUMMY/LJ018-0286.wav|and Austin Bidwell opened a bona fide credit in the Burlington or West End branch of the Bank of England, +DUMMY/LJ044-0145.wav|By August of nineteen sixty-three, Oswald had for some time been considering the possibility of leaving the United States again. +DUMMY/LJ003-0222.wav|The jail was neither suitable nor sufficiently large. It was not even kept weather-tight. +DUMMY/LJ034-0086.wav|This clipboard had been made by Kaiser and had his name on it. +DUMMY/LJ044-0050.wav|Police reports also fail to reflect any activity on Oswald's part prior to August nine, nineteen sixty-three, +DUMMY/LJ003-0198.wav|Few realizing that the dreadful fate would overtake them, dismissed the prospect of death, +DUMMY/LJ028-0352.wav|whether of former or of later times, except only Cyrus with whom no person ever yet thought himself worthy to compare. +DUMMY/LJ038-0022.wav|I don't know if this is the man they want. But he is running from them for some reason, end quote, She then called the police. +DUMMY/LJ017-0278.wav|No soldiers either. +DUMMY/LJ014-0096.wav|and at their request two police officers were sent to Bermondsey to inquire for him at the Mannings, with whom it was well known that he was very intimate. +DUMMY/LJ011-0105.wav|His defense was that the forged acceptances would have been met on coming to maturity, and that he had no real desire to defraud. +DUMMY/LJ002-0038.wav|Of these wards, three were appropriated to the "cabin side," so called because +DUMMY/LJ028-0123.wav|I dug its moat to the water level. +DUMMY/LJ003-0097.wav|with convicts of all ages and characters, to render it next to impossible but that, with the obliteration of all sense of self-respect, +DUMMY/LJ019-0107.wav|District prisons were to be established for this purpose, each of which would be in the heart of a farm of a thousand acres. +DUMMY/LJ039-0187.wav|In fact, one of the firers in the rapid fire test in firing his two series of three shots, +DUMMY/LJ011-0227.wav|After their release from Lancaster and Newgate respectively, both Wakefields went abroad. +DUMMY/LJ032-0270.wav|Having reviewed the evidence that (one) Lee Harvey Oswald purchased the rifle used in the assassination, +DUMMY/LJ015-0019.wav|The Strahan of the firm which came to grief was a Snow who changed his name for a fortune of two hundred thousand pounds; +DUMMY/LJ015-0073.wav|Within a year he was advanced to the post of chief clerk in the transfer department, at a salary of one hundred fifty pounds a year. +DUMMY/LJ043-0165.wav|Leaving the note and picture as he did would seem to indicate that he had considered the possibility of capture. +DUMMY/LJ023-0064.wav|Having in mind that in succeeding generations many other problems then undreamed of would become national problems +DUMMY/LJ008-0223.wav|The approaching ceremony had evidently no terrors for these "idolaters of the gallows." +DUMMY/LJ034-0114.wav|Brennan's description should also be compared with the eyewitness description broadcast over the Dallas police radio at one:twenty-two p.m. +DUMMY/LJ045-0105.wav|She testified that she told him, quote, +DUMMY/LJ012-0281.wav|The most ghastly part of the story is that which deals with his getting rid of the head. +DUMMY/LJ019-0090.wav|The alternative between labor or idleness, or the selection of the form of labor, +DUMMY/LJ045-0020.wav|In any event, the Commission has been unable to find any credible evidence that Oswald had direct contact or association with +DUMMY/LJ025-0096.wav|Amylaceous and saccharine substances are largely manufactured, even by the highest animals. +DUMMY/LJ029-0212.wav|"Welcome Mr. Kennedy to Dallas," sponsored by the American Fact-finding Committee, which the sponsor later testified was an ad hoc committee +DUMMY/LJ047-0192.wav|On November eighteen the FBI learned that Oswald recently had been in communication with the Soviet Embassy in Washington +DUMMY/LJ021-0195.wav|the great Chief Justice White said: +DUMMY/LJ004-0125.wav|in the balance some work was done, but with the most meager results. +DUMMY/LJ003-0201.wav|unlimited drink was not denied them provided it was obtained in regulated quantities at one time. +DUMMY/LJ023-0078.wav|to presume in favor of its validity until its violation of the Constitution is proved beyond all reasonable doubt. +DUMMY/LJ035-0040.wav|As Baker reached the second floor, he was about twenty feet from the vestibule door. He intended to continue around to his left toward the stairway going up +DUMMY/LJ010-0188.wav|Oxford expressed little anxiety or concern. +DUMMY/LJ045-0051.wav|On the other hand, Oswald objected to the invitation that his wife had received to live with Mrs. Ruth Paine, +DUMMY/LJ019-0174.wav|It was less prompt apparently in dealing with structural alterations and improvements. +DUMMY/LJ042-0004.wav|Defection to the Soviet Union +DUMMY/LJ010-0002.wav|In chapter two of the present volume I brought down the record of crime to the second decade of the present century. +DUMMY/LJ017-0177.wav|Wilson was acquitted on this charge, but other suspicious facts cropped up while she was in Newgate. +DUMMY/LJ037-0139.wav|he subsequently testified before a Commission staff member and, when shown two photographs of Oswald, stated that they were photographs of the man he saw. +DUMMY/LJ015-0119.wav|and possessed of an education very much beyond the rank of life to which he originally belonged." +DUMMY/LJ004-0032.wav|The onus of improvement lay upon the magistracy, the local authorities administering local funds, +DUMMY/LJ028-0413.wav|In his quaint way he says: +DUMMY/LJ012-0203.wav|Eventually the law overtook them, but almost by accident. +DUMMY/LJ034-0155.wav|When specifically asked before the Commission +DUMMY/LJ021-0118.wav|During the last twelve months our industrial recovery has been to some extent retarded by strikes, +DUMMY/LJ013-0052.wav|The executors to the estate disputed the validity of this document. +DUMMY/LJ012-0120.wav|one of them of considerable size. +DUMMY/LJ034-0213.wav|Just before snapping the picture Altgens heard a noise which sounded like the popping of a firecracker. +DUMMY/LJ005-0284.wav|of more depraved and systematic criminals. +DUMMY/LJ004-0246.wav|They used their own tools, and this without any dangerous consequences as regards facilitating the escape of others, +DUMMY/LJ049-0040.wav|The assassination suggests that it would have been of prime importance +DUMMY/LJ020-0056.wav|Arrange the rolls in regular rows in the pan about a quarter of an inch apart. +DUMMY/LJ015-0281.wav|Saward's method of negotiating the cheques was equally well planned. +DUMMY/LJ032-0071.wav|Oswald's use of the name "Hidell" to purchase the assassination weapon was one of several instances in which he used this name as an alias. +DUMMY/LJ022-0089.wav|charged with carrying on work relief projects. +DUMMY/LJ035-0139.wav|They reentered the building by the rear door several minutes after Baker and Truly rushed through the front entrance. +DUMMY/LJ016-0439.wav|Often the convicts give way to despair. They are too closely watched to be allowed to do themselves much mischief, or suicides would probably be more frequent. +DUMMY/LJ046-0011.wav|One out of every five Presidents since eighteen sixty-five has been assassinated; +DUMMY/LJ022-0099.wav|to move as rapidly as possible back into private employment when such employment is available. +DUMMY/LJ005-0170.wav|that many of them will build new jails, and that when in a year or two we come to examine their situation, +DUMMY/LJ029-0021.wav|The three agreed that the President would come to Texas in late November nineteen sixty-three. +DUMMY/LJ012-0036.wav|On one occasion the whole of the proceeds of a robbery from a boot shop was traced to Solomons'; +DUMMY/LJ027-0082.wav|and any other series of organs of vertebrates would give the same evidence of fundamental resemblances (homologies). +DUMMY/LJ046-0129.wav|The total volume of information received by PRS has risen steadily. +DUMMY/LJ001-0143.wav|For where these are boldly and carefully designed, and each letter is thoroughly individual in form, +DUMMY/LJ018-0293.wav|The ground thus cleared, the forgers poured in from Birmingham numbers of forged acceptances, +DUMMY/LJ018-0017.wav|The friends of Mr. Briggs were communicated with, and it was ascertained that when he left home the morning of the murderous attack, +DUMMY/LJ024-0125.wav|This proposal of mine will not infringe in the slightest upon the civil or religious liberties so dear to every American. +DUMMY/LJ030-0127.wav|to the intersection of Houston Street, which marks the beginning of Dealey Plaza. +DUMMY/LJ008-0211.wav|Gin-shops and coffee-houses were the first to open doors, and touts began to bid for tenants for the various rooms upstairs. +DUMMY/LJ036-0215.wav|He approached the car and apparently exchanged words with Tippit through the right front or vent window. +DUMMY/LJ030-0250.wav|Other Secret Service agents assigned to the motorcade remained at their posts during the race to the hospital. +DUMMY/LJ020-0072.wav|Yolk of one egg beaten light. +DUMMY/LJ029-0198.wav|They conveyed the pleas of Dallas leaders that citizens not demonstrate or create disturbances during the President's visit. +DUMMY/LJ003-0206.wav|and their gibes and jollity counteracted the ordinary's counsels or the independent preacher's earnest prayers. +DUMMY/LJ022-0083.wav|six. Projects will be allocated to localities or relief areas in relation to the number of workers on relief rolls in those areas. +DUMMY/LJ008-0125.wav|The sheriff arrived, attended by his officers, to receive the prisoner from the keeper. +DUMMY/LJ048-0137.wav|there have been references to the numerous discussions between Secret Service representatives and the Dallas Police Department. +DUMMY/LJ016-0061.wav|and then following her down a step ladder into the house, told his story, appealed to and won her compassion. +DUMMY/LJ036-0212.wav|About one hundred feet past the intersection Tippit stopped a man walking east along the south side of Patton. +DUMMY/LJ038-0268.wav|the FBI refuses to draw any conclusion as to probability. +DUMMY/LJ005-0202.wav|An examination of this report shows how even the most insignificant township had its jail. +DUMMY/LJ025-0087.wav|On the other hand, the definition thus amended will exclude all ordinary vegetable organisms. +DUMMY/LJ018-0311.wav|had undoubtedly the command of large funds while there, and would have readily disbursed it to effect their enlargement. +DUMMY/LJ002-0185.wav|The King's Bench was a national prison, in which were confined all debtors arrested for debt or for contempt of the court of the King's Bench. +DUMMY/LJ050-0257.wav|Recognition that the responsibility must be shared increases the likelihood that it will be met. +DUMMY/LJ004-0132.wav|the old evils of indiscriminate association still continued unchecked. +DUMMY/LJ019-0382.wav|As the years passed, great want of uniformity continued to prevail throughout the prisons of the United Kingdom. +DUMMY/LJ028-0428.wav|has been removed, and the surrounding city walls have been traced. +DUMMY/LJ017-0244.wav|The murders were perpetrated on the tenth September, and the ship continued her voyage for nearly three weeks, meeting and speaking one ship only. +DUMMY/LJ009-0047.wav|The next convict is clearly and unmistakably a villain. +DUMMY/LJ010-0225.wav|The whole of the evidence pointed so strongly towards insanity, that the jury brought in a verdict of acquittal on that ground, +DUMMY/LJ011-0164.wav|Miss Turner, greatly agitated, accompanied the messenger who had brought this news, a disguised servant of Wakefield's, +DUMMY/LJ006-0069.wav|consign many of the more petty and unpracticed offenders to this place, +DUMMY/LJ047-0131.wav|In early September nineteen sixty-three +DUMMY/LJ044-0055.wav|While the legend, quote, FPCC, five four four Camp Street New Orleans, Louisiana, end quote, +DUMMY/LJ016-0214.wav|a silver coin bearing the same relation to the Scottish pound that an English shilling does to an English pound, was made to pass current in England. +DUMMY/LJ038-0192.wav|six. You can either throw out or give my clothing, etc. away. Do not keep these. +DUMMY/LJ011-0046.wav|As I have already said, they took the chaplain seriously to task for the bad taste shown in the condemned sermon preached before Fauntleroy. +DUMMY/LJ020-0067.wav|Five tablespoonfuls of yeast. +DUMMY/LJ027-0017.wav|All organisms have the same need of reproduction. +DUMMY/LJ028-0181.wav|He was followed by three kings whose reigns were short, +DUMMY/LJ015-0100.wav|They drove there, and Robson on arrival did the honors of his house, rang for lunch to gain time, +DUMMY/LJ024-0005.wav|has reached the age of seventy and does not avail himself of the opportunity to retire on a pension, +DUMMY/LJ006-0133.wav|The turnkeys complained bitterly that these old prisoners had more power than they themselves. +DUMMY/LJ003-0302.wav|the work being constant and suitable, with certain hours of relaxation and for food and exercise. +DUMMY/LJ017-0280.wav|The convicts were pinioned one by one and sent singly out to the gallows. +DUMMY/LJ035-0149.wav|She ran inside and up the front stairs into the large open office reserved for clerical employees. +DUMMY/LJ030-0208.wav|Hill heard a noise, which seemed to be a firecracker, coming from his right rear. +DUMMY/LJ001-0099.wav|It must be remembered, however, that most modern printing is done by machinery on soft paper, and not by the hand press, +DUMMY/LJ004-0013.wav|some prisons that had been ameliorated under the persuasive influence of his kind advice were relapsing into their former horrid state of privation, +DUMMY/LJ041-0109.wav|The hearing officer agreed with the latter claim but found Oswald guilty of wrongfully using provoking words and sentenced him to twenty-eight days, +DUMMY/LJ014-0045.wav|that Delarue had suffered by the hands of imaginary outraged brothers acting as the avengers of females deeply injured by him. +DUMMY/LJ006-0123.wav|The man intrusted with this privilege was often the inner gatesman, +DUMMY/LJ016-0250.wav|It may, however, be remarked that the wisdom of appointing any regular hangman is very open to question, +DUMMY/LJ038-0128.wav|He also denied that he had received the rifle through this box. +DUMMY/LJ039-0174.wav|The marksmen took as much time as they wanted for the first target and all hit the target. +DUMMY/LJ023-0141.wav|can, if they choose, hold office for life, no matter how old they may get to be. +DUMMY/LJ032-0049.wav|The specific rifle shipped against the order had been received by Klein's from Crescent on February twenty-one, nineteen sixty-three. +DUMMY/LJ042-0103.wav|Oswald apparently resented the exercise of authority over him and the better treatment afforded to Communist Party officials. +DUMMY/LJ029-0115.wav|in preference to a short stretch of the Central Expressway, which presented a minor safety hazard +DUMMY/LJ008-0064.wav|When the criminals are tied up and prepared for their fate, this floor suddenly falls down, upon withdrawing the supporters inwards. +DUMMY/LJ019-0238.wav|The first "glass house," or room in which prisoners could talk in private with their attorneys, but yet be seen by the warder on the watch, had been constructed +DUMMY/LJ037-0185.wav|Records of Seaport Traders, Incorporated, a mail-order division of George Rose and Co., +DUMMY/LJ044-0036.wav|then national director of FPCC. In one of those letters, dated August one, nineteen sixty-three, +DUMMY/LJ045-0131.wav|Neither Hosty nor any other agent of the FBI spoke to Oswald on any subject from August ten, nineteen sixty-three, +DUMMY/LJ039-0228.wav|Concerning the shots which struck the President in the back of the neck, +DUMMY/LJ041-0058.wav|which might appear to be inconsistent with his desire to join the Marines, could have been another manifestation of Oswald's rejection of his environment. +DUMMY/LJ007-0060.wav|In his evidence before the inspectors he declared that "for years he gave his whole time to his duties, from an early hour in the morning till late in the afternoon. +DUMMY/LJ001-0076.wav|and things got worse and worse through the whole of the seventeenth century, so that in the eighteenth printing was very miserably performed. +DUMMY/LJ045-0091.wav|She said he was different from other people in, quote, At, least his imagination, his fantasy, which was quite unfounded, +DUMMY/LJ010-0308.wav|by the death of one of the partners, a large sum in cash had to be withdrawn from the bank to pay his heirs. +DUMMY/LJ008-0222.wav|of those who had shown the white, and those who had died game. +DUMMY/LJ047-0019.wav|Oswald's defection was also the occasion for the opening of files by the Department of State, CIA, and the Office of Naval Intelligence. +DUMMY/LJ008-0059.wav|Under this bar at a proper distance is a piece of flooring or platform, +DUMMY/LJ035-0029.wav|Neither elevator was there. +DUMMY/LJ004-0076.wav|"Disease, cold, famine, nakedness, and contagious and polluted air are not lawful punishments in the hands of the civil magistrates; +DUMMY/LJ003-0119.wav|This was carried out by putting the criminal's head through the legs of a chair, and stretching out his arms and tying them to the legs. +DUMMY/LJ003-0228.wav|and the general insufficiency was such +DUMMY/LJ003-0326.wav|so that the various classes might be seated separately, and so as not to see each other. +DUMMY/LJ031-0019.wav|These rooms were for the emergency treatment of acutely ill or injured patients. Although the first message mentioned an injury only to President Kennedy, +DUMMY/LJ010-0275.wav|but it was proved that the pistol was not loaded with ball, +DUMMY/LJ040-0127.wav|or for detention pending court appearance or commitment to a child-caring or custodial institution such as a training school. +DUMMY/LJ048-0164.wav|For the inauguration and certain other parades in Washington where the traditional route is known to the public long in advance of the event, +DUMMY/LJ032-0082.wav|and the photograph of Lee Harvey Oswald. +DUMMY/LJ033-0110.wav|Frazier and Mrs. Randle testified +DUMMY/LJ004-0207.wav|and all the material for prisoners' apparel was manufactured in the jail. +DUMMY/LJ041-0189.wav|He was concerned with his image in history and I do think that is why he chose the particular method of defecting he chose and did it in the way he did. +DUMMY/LJ048-0264.wav|the special agent is officially employed and should not use liquor, until the completion of all of his official duties for the day, +DUMMY/LJ010-0171.wav|On that day Oxford was on the watch at Buckingham Palace. +DUMMY/LJ021-0077.wav|There is no magic formula, +DUMMY/LJ021-0015.wav|The underlying necessity for such activity +DUMMY/LJ021-0177.wav|Did England hold to the gold standard when her reserves were threatened? +DUMMY/LJ019-0303.wav|of these, sixty-three gave admittance during the entire year to less than twenty-five prisoners; +DUMMY/LJ016-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section nineteen: Later Records +DUMMY/LJ049-0009.wav|When President and Mrs. Kennedy shook hands with members of the public along the fences surrounding the reception area, they were closely guarded by Secret Service agents +DUMMY/LJ037-0216.wav|in the picture she took in late March or early April nineteen sixty-three when the family was living on Neely Street in Dallas. +DUMMY/LJ008-0292.wav|but it was seldom less than six weeks. It all depended upon the sovereign's disposition to do business. +DUMMY/LJ002-0066.wav|The master felons' side consisted of a yard the same size as the preceding, appropriated nominally to the most decent and better-behaved prisoners, +DUMMY/LJ039-0044.wav|Mr. Johnson was the then Vice President and his visit took place on April twenty-third. +DUMMY/LJ039-0220.wav|and that one would not have to be an expert marksman to have accomplished the assassination with the weapon which was used. +DUMMY/LJ006-0274.wav|Perhaps the worst feature of the visiting system was the permission accorded to male prisoners "under the name of husbands, brothers, and sons" +DUMMY/LJ005-0250.wav|The prisoners were crowded together in the jail, contrary to the requirements of the four George the fourth +DUMMY/LJ038-0050.wav|they might have heard a snapping noise resulting from the police officer grabbing the cylinder of the revolver and pulling it away from Oswald while he was attempting to pull the trigger. +DUMMY/LJ012-0144.wav|and passed it on to Solomons by his daughter, a widow named Abrahams. +DUMMY/LJ004-0139.wav|"In the morning the stench and heat were so oppressive that he and every one else on waking rushed unclothed into the yard;" +DUMMY/LJ020-0100.wav|Unless it is too hot, they will get no harm while you finish dressing in ten minutes, just in season to turn the pan. +DUMMY/LJ017-0247.wav|The rest of the crew were also permitted to embark, except the Chinamen, one of whom was thrown into the water and drowned, +DUMMY/LJ009-0108.wav|The two masses of prisoners for trial undulate and slightly murmur, +DUMMY/LJ044-0066.wav|Oswald wrote that he had appeared on Mr. William Stuckey's fifteen-minute television program over WDSU-TV called, quote, +DUMMY/LJ032-0105.wav|quote, I knew there was no such organization. And I know Hidell is merely an altered Fidel, and I laughed at such foolishness. +DUMMY/LJ035-0186.wav|After he returned to the street he directed Sergeant Harkness to station two patrolmen at the front door and not let anyone in or out; +DUMMY/LJ036-0125.wav|and that sometimes he made his entry right after a trip while at other times he waited to record three or four trips. +DUMMY/LJ045-0241.wav|that it can ascribe to him any one motive or group of motives. +DUMMY/LJ006-0153.wav|It was not strange that subordinates should neglect their duty when superiors set the example. +DUMMY/LJ024-0010.wav|to make the administration of all federal justice speedier and, therefore, less costly; +DUMMY/LJ033-0139.wav|Oswald's Mannlicher-Carcano rifle, serial Number C two seven six six, which was also found on the sixth floor. +DUMMY/LJ032-0265.wav|Each man testified that he thought he could detect the outline of a rifle in the blanket, even though the blanket was empty. +DUMMY/LJ022-0155.wav|Power production in this country is virtually back to the nineteen twenty-nine peak. +DUMMY/LJ049-0183.wav|regarding such threats and that its Protective Research Section is not adequately staffed or equipped +DUMMY/LJ035-0085.wav|from the southeast corner of the sixth floor down the stairway to the lunchroom. +DUMMY/LJ005-0178.wav|Many of them consisted of one or two rooms at most. +DUMMY/LJ011-0130.wav|This was on the last day of eighteen twenty-nine. In the following session Sir Robert Peel brought in a bill to consolidate the acts relating to forgery. +DUMMY/LJ045-0005.wav|especially an atmosphere of extreme opposition to President Kennedy that was present in some parts of the Dallas community +DUMMY/LJ028-0333.wav|he put them also to the sword. +DUMMY/LJ024-0105.wav|to block the amendment, although I would be awfully glad to help get some other kind of amendment ratified. +DUMMY/LJ018-0330.wav|Violent passions too easily aroused prompted the Frenchwoman Marguerite Dixblanc +DUMMY/LJ042-0062.wav|It shows how willing he was to act dramatically and decisively when he faced an emotional crisis +DUMMY/LJ043-0164.wav|and his role would have been made clear by the evidence which he had left behind. +DUMMY/LJ012-0277.wav|It is scarcely probable that he would have gone to this extremity if he had had no previous evil intention, +DUMMY/LJ045-0128.wav|that the FBI was, quote, not now, end quote, interested in his Fair Play for Cuba Committee activities, but noted that the FBI, quote, +DUMMY/LJ042-0170.wav|There are two great representatives of power in the world, +DUMMY/LJ023-0139.wav|In the case of Supreme Court justices, that pension is twenty thousand dollars a year. +DUMMY/LJ015-0179.wav|The Central Criminal Court, when the trial came on, +DUMMY/LJ031-0063.wav|When Dr. Perry noted free air and blood in the President's chest cavity, he asked that chest tubes be inserted to allow for drainage of blood and air. +DUMMY/LJ034-0208.wav|In evaluating the evidence that Oswald was at the southeast corner window of the sixth floor at the time of the shooting, +DUMMY/LJ003-0292.wav|This committee was to compare allowances, examine rules, and certify as to the condition of prisoners; +DUMMY/LJ005-0035.wav|the Society pursued its laudable undertaking with remarkable energy and great singleness of purpose. +DUMMY/LJ034-0074.wav|I discovered I left my cigarettes in my jacket pocket upstairs, and I took the elevator back upstairs to get my jacket with my cigarettes in it, end quote. +DUMMY/LJ006-0143.wav|There was no penalty attached to drunkenness. +DUMMY/LJ016-0337.wav|In the houses opposite the prison numbers of detectives mixed with the spectators; +DUMMY/LJ029-0145.wav|in order to reach the Dallas-Fort Worth Turnpike, which has the same access road from Elm Street as does the Stemmons Freeway. +DUMMY/LJ025-0104.wav|even in the case of the green plant, which in the dark absorbs oxygen and gives out carbonic acid like any animal. +DUMMY/LJ005-0063.wav|It was thought that this would guarantee constant supervision and inspection, but it did nothing of the kind, and only the presence of warders on duty +DUMMY/LJ019-0216.wav|Eventually Holloway was decided upon as a site for the new city prison. +DUMMY/LJ042-0142.wav|He was very irritable, sometimes for a trifle, end quote, and that, quote, Lee was very unrestrained and very explosive, end quote. +DUMMY/LJ018-0078.wav|The next time Thain entered his cabin he fired at him point-blank, and lodged three bullets in his breast. +DUMMY/LJ001-0169.wav|The paper used for printing the small highly ornamented French service-books about the beginning of the sixteenth century is a model in this respect, +DUMMY/LJ011-0021.wav|Further investigations brought other similar frauds to light, +DUMMY/LJ049-0145.wav|As improvements are recommended for the advance detection of potential threats to the President, it could act as a final review board. +DUMMY/LJ021-0056.wav|Under its guidance, trades and industries covering over ninety percent of all industrial employees +DUMMY/LJ039-0163.wav|the Infantry Weapons Evaluation Branch of the Ballistics Research Laboratory had expert riflemen fire the assassination weapon +DUMMY/LJ003-0290.wav|the court of aldermen appointed a committee of its own body, assisted by the town clerk, Mr. Dance, city surveyor, son to the architect of Newgate, +DUMMY/LJ016-0122.wav|He made a strong rope with several of the sheets; then, returning to the male yard, +DUMMY/LJ022-0179.wav|speaking through their government. +DUMMY/LJ022-0101.wav|I have felt it to be essentially wise and prudent to avoid, so far as possible, +DUMMY/LJ002-0147.wav|that thirteen hundred and twelve debtors were committed by them to Newgate between seventeen ninety-seven and eighteen oh eight, +DUMMY/LJ045-0010.wav|The Commission has found no evidence that the extreme views expressed toward President Kennedy +DUMMY/LJ015-0076.wav|Some time elapsed before Robson succumbed to temptation. +DUMMY/LJ014-0226.wav|Only a few have vied with Cannon in fiendish cruelty and brutality. +DUMMY/LJ006-0031.wav|that punishment must be reformatory as well as deterrent. +DUMMY/LJ047-0210.wav|I think all of those, if we had them all together, +DUMMY/LJ022-0072.wav|through the most comprehensive work plan in the history of the nation. +DUMMY/LJ038-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. +DUMMY/LJ008-0281.wav|They found at Newgate, under disgraceful conditions as already described, +DUMMY/LJ042-0064.wav|The entry in his self-styled, quote, Historic Diary, end quote, for October twenty-one, nineteen fifty-nine, reports, quote, +DUMMY/LJ002-0307.wav|only out of a satisfaction of all his former debts, end quote. The fees were not always extorted, it is true; +DUMMY/LJ021-0100.wav|Let me call your attention to the fact that the national Industrial Recovery Act +DUMMY/LJ030-0241.wav|President Johnson emphasized Youngblood's instantaneous reaction after the first shot: +DUMMY/LJ005-0135.wav|and the average daily number committed that year amounted to three thousand, nine hundred eighty-five. +DUMMY/LJ022-0144.wav|As we have proceeded with the administration of this Act, +DUMMY/LJ012-0081.wav|Being inexperienced, they decided to call in the services of a couple of professional housebreakers, +DUMMY/LJ048-0204.wav|and that they had no opportunity to scan the windows of the Depository or any other building in the vicinity of Elm and Houston when the motorcade was passing. +DUMMY/LJ009-0141.wav|Lord Paget, Lord Bruce, several members of the House of Commons, and a few ladies. +DUMMY/LJ012-0065.wav|There he devoted himself to the circulation of forged notes. He was also anxious to do business in watches, +DUMMY/LJ015-0020.wav|he was a man esteemed and respected in society and the world of finance, incapable as it was thought of a dishonest deed. +DUMMY/LJ017-0209.wav|more particularly as regards locomotion. +DUMMY/LJ043-0112.wav|Mrs. Paine brought Oswald's family to New Orleans. +DUMMY/LJ003-0193.wav|Once, during the long illness of George the third, as many as one hundred were there waiting the "Report," as it was called. +DUMMY/LJ049-0158.wav|These informal and part-time arrangements led to more systematic protection in nineteen oh two, after the assassination of President McKinley; +DUMMY/LJ039-0055.wav|In deciding whether Lee Harvey Oswald fired the shots which killed President Kennedy and wounded Governor Connally, +DUMMY/LJ017-0233.wav|The second mate, who had heard the hammering of the capstan-bars and the handspikes, with the first mate's and captain's agonized cries, had come out, +DUMMY/LJ044-0032.wav|in which one man single handedly created publicity for his cause or for himself. +DUMMY/LJ002-0310.wav|that debtors who could afford the cabin and master's side were not permitted to share in the prison charities. +DUMMY/LJ004-0134.wav|The debtors were crowded almost inconceivably. In a space twenty feet long by six wide, +DUMMY/LJ006-0130.wav|had a key of both the master's side and middle side yards, was the only person present at the distribution of beer, and was trusted to examine, +DUMMY/LJ037-0201.wav|Heinz W. Michaelis, office manager of both George Rose and Co., Incorporated and Seaport Traders, Incorporated. +DUMMY/LJ050-0222.wav|would take approximately twenty months to implement and require expenditures of approximately three million dollars during that period. +DUMMY/LJ023-0084.wav|When the Congress has sought to stabilize national agriculture, to improve the conditions of labor, +DUMMY/LJ036-0210.wav|At approximately one:fifteen p.m., Tippit, who was cruising east on tenth Street, +DUMMY/LJ009-0206.wav|parish officers, peace officers, the high constable of the county of Middlesex on horseback, and then the body of Williams, +DUMMY/LJ005-0128.wav|The interior of one prison was illuminated throughout with gas, -- still a novelty, which had been generally adopted in London only four years previously, +DUMMY/LJ001-0062.wav|Even in Italy most of the theological and law books were printed in Gothic letter, +DUMMY/LJ037-0005.wav|By the evening of November twenty-two, five of them had identified Lee Harvey Oswald in police lineups as the man they saw. +DUMMY/LJ016-0052.wav|he passed to the top of the ordinary's residence, hoping to find an open sky-light by which he might enter and so work downstairs. +DUMMY/LJ022-0157.wav|under holding company domination the utility industry has long been hopelessly at war within itself and with public sentiment. +DUMMY/LJ009-0133.wav|the outside public continued to be excluded from the Newgate chapel on the day the condemned sermon was preached. +DUMMY/LJ029-0173.wav|beginning on September thirteen, when the Times-Herald announced in a front page article that President Kennedy was planning a brief one-day tour of four Texas cities +DUMMY/LJ026-0091.wav|This must, in the first instance, be provided for them by green plants, and hence without chlorophyll-bearing plants, +DUMMY/LJ018-0153.wav|the officers ensconced themselves in the latter, and waited for Buncher's expected visit. +DUMMY/LJ049-0114.wav|what this bill means to punish is the crime of interruption of the Government of the United States and the destruction of its security by striking down the life +DUMMY/LJ005-0182.wav|that for long no idea of interfering with them was entertained. +DUMMY/LJ028-0399.wav|The spread of Mohammedanism caused new cities to be built, and Babylon was the quarry for their building material. +DUMMY/LJ032-0243.wav|Rifle Among Oswald's Possessions +DUMMY/LJ001-0008.wav|has never been surpassed. +DUMMY/LJ010-0173.wav|where he waited till four p.m., the time at which the Queen and Prince Consort usually took an afternoon drive. +DUMMY/LJ028-0193.wav|and to them came hope of returning to Jerusalem. +DUMMY/LJ033-0059.wav|A short time later Mrs. Paine told her that someone had shot the President, quote, from the building in which Lee is working, end quote. +DUMMY/LJ012-0222.wav|plainly pointing to the perpetration of other crimes. +DUMMY/LJ014-0157.wav|Nevertheless, she attempted to commit suicide by driving her nails, purposely left long, into her throat. +DUMMY/LJ035-0034.wav|The stairs from one floor to the next are "L-shaped," with both legs of the "L" approximately the same length. +DUMMY/LJ029-0051.wav|in the territory of the Secret Service regional office which includes Dallas and Fort Worth. +DUMMY/LJ042-0104.wav|After he returned to the United States +DUMMY/LJ048-0244.wav|Chief Rowley testified that agents on duty in such a situation usually stay within the building during their relief, +DUMMY/LJ032-0260.wav|He thought it contained tent poles, or possibly other camping equipment such as a folding shovel. +DUMMY/LJ016-0441.wav|The man, Bousfield, however, whose execution was so sadly bungled, +DUMMY/LJ006-0155.wav|He may have erred in some points through ignorance, but in others he was clearly guilty of culpable neglect. +DUMMY/LJ048-0223.wav|After the President had retired at his hotel, +DUMMY/LJ047-0027.wav|In this way, it learned that when Oswald had arrived in the Soviet Union +DUMMY/LJ040-0140.wav|Mrs. Evelyn D Siegel, a social worker who interviewed both Lee and his mother while Lee was confined in Youth House, +DUMMY/LJ011-0189.wav|the other brother was accordingly sent on a pretended mission to Shrigley to bring Mr. Turner on to London, whither Wakefield and Miss Turner also proceeded. +DUMMY/LJ033-0019.wav|According to the testimony of Frazier, Marina Oswald, and Ruth Paine, it appears that Oswald never returned to Irving in midweek +DUMMY/LJ008-0255.wav|the iron knell that rang upon the beating heart of the living man who was about to die; +DUMMY/LJ036-0189.wav|at the southeast corner of tenth Street and Patton Avenue, moments before the Tippit shooting. +DUMMY/LJ047-0122.wav|Our interest in this man at this point was to determine whether his activities constituted a threat to the internal security of the country. +DUMMY/LJ048-0263.wav|During entire periods of travel status, +DUMMY/LJ007-0141.wav|and "to dispose him, by meditation and seclusion, to return to an honest life." +DUMMY/LJ019-0327.wav|No less close was the care as regards preservation of health. +DUMMY/LJ026-0117.wav|are directly oxidized to excretions and, lacking nitrogen, cannot serve for making new animal protoplasm. +DUMMY/LJ046-0205.wav|with the headquarters of the Federal intelligence agencies, particularly the FBI, +DUMMY/LJ028-0173.wav|When one is about halfway up, one finds a resting place and seats, where persons are wont to sit sometimes on their way to the summit. +DUMMY/LJ018-0122.wav|but not before the forged paper had been put in circulation for more than a couple of years. In eighteen sixty-one, +DUMMY/LJ028-0492.wav|and surmounted with towers, and then finally a sixth wall +DUMMY/LJ043-0125.wav|because the president of the photographic firm for which he had previously worked +DUMMY/LJ025-0004.wav|are often designated Organic Functions. +DUMMY/LJ026-0090.wav|This power is possessed by green plants alone; all other organisms being dependent for energy upon the potential energy of ready-made food. +DUMMY/LJ019-0018.wav|moreover, the gross expenditure was increased "partly from its being considered necessary, as it was a national prison, +DUMMY/LJ019-0148.wav|The prisoners were still left to themselves till next morning's unlocking, +DUMMY/LJ002-0287.wav|There were no beds or bedding, no straw even. +DUMMY/LJ046-0004.wav|four Presidents of the United States have been assassinated +DUMMY/LJ044-0182.wav|When faced with that situation Oswald became greatly agitated, and although he later unsuccessfully attempted +DUMMY/LJ044-0091.wav|with a fellow who had lived in Russia for three years and who was an admitted Marxist. +DUMMY/LJ019-0389.wav|passed under the more direct control of the State. +DUMMY/LJ011-0069.wav|and that after hanging for the regulated time he was taken down and easily restored to consciousness. +DUMMY/LJ046-0003.wav|In the one hundred years since eighteen sixty-five +DUMMY/LJ007-0096.wav|It was high time that the unsatisfactory state of the law as regards the treatment of criminal lunatics should be remedied +DUMMY/LJ044-0113.wav|After he had begun his Cuban activity in New Orleans, quote, +DUMMY/LJ034-0081.wav|Givens thought this was about eleven:fifty-five a.m. +DUMMY/LJ017-0103.wav|This tetanus, according to the prosecution, was produced by strychnia, and followed the administration of pills by Palmer +DUMMY/LJ040-0185.wav|He exhibits some difficulty in relationship to the maternal figure suggesting more anxiety in this area than in any other. +DUMMY/LJ012-0157.wav|It may be added that prison life did not agree with "Money Moses"; a striking change came over his appearance while in Newgate. +DUMMY/LJ050-0152.wav|The Secret Service and the Department of the Treasury now recognize this critical need. +DUMMY/LJ021-0190.wav|We do not get greatly excited nor is our peace of mind disturbed, whether we be businessmen or workers or farmers, +DUMMY/LJ003-0151.wav|All who could scrape together the cash seem to have gladly availed themselves of the privilege of entering the master's side. +DUMMY/LJ014-0330.wav|He had several narrow escapes. +DUMMY/LJ018-0367.wav|but she eventually passed into Broadmoor Lunatic Asylum, where she busies herself with watercolor drawing. +DUMMY/LJ001-0028.wav|but by printers in Strasburg, Basle, Paris, Lubeck, and other cities. +DUMMY/LJ003-0012.wav|and debtors might practically have as much as they liked, if they could only pay for it. +DUMMY/LJ003-0314.wav|the surgeon should see all prisoners, whether ill or well, once a week, and take general charge of the infirmaries. +DUMMY/LJ022-0169.wav|It will not destroy values. +DUMMY/LJ048-0240.wav|by members of the midnight to eight a.m. shift of the White House detail. +DUMMY/LJ031-0195.wav|Air Force One landed at Andrews Air Force Base, where President Kennedy had begun his last trip only thirty-one hours before. +DUMMY/LJ046-0048.wav|Desired by both the President and the public, it is an indispensable means of communication between the two. +DUMMY/LJ025-0050.wav|Thus wrote Cuvier in eighteen twenty-eight. +DUMMY/LJ032-0096.wav|listing the names of those, other than the owner of the box, entitled to receive mail through the box. +DUMMY/LJ003-0174.wav|Mr. Davison, sent to Newgate for embezzlement, and whose case is given in the preceding chapter, +DUMMY/LJ012-0201.wav|Bishop confessed that he was moved to this by the example of Burke and Hare. +DUMMY/LJ032-0160.wav|He compared them with the fibers found in the shirt which Oswald was wearing when arrested in the Texas Theatre. +DUMMY/LJ003-0291.wav|and Mr. Addison, keeper of Newgate, to make a visitation of the jails supposed to be the best managed, including those of Petworth and Gloucester. +DUMMY/LJ048-0088.wav|Examination of these procedures shows that in most respects they were well conceived and ably executed by the personnel of the Service. +DUMMY/LJ033-0079.wav|sister of the man with whom Oswald drove to work -- Buell Wesley Frazier. +DUMMY/LJ042-0028.wav|Oswald had managed to save enough money to cover the expenses of his forthcoming trip. +DUMMY/LJ031-0085.wav|Since the Dallas doctors directed all their efforts to controlling the massive bleeding caused by the head wound, and to reconstructing an airway to his lungs, +DUMMY/LJ024-0024.wav|Why was the age fixed at seventy? +DUMMY/LJ009-0196.wav|Another brutal practice had nearly disappeared about the time of the abolition of dissection. This was the public exhibition of the body, +DUMMY/LJ010-0025.wav|those against the person became more and more limited to the most heinous, or those which menaced or destroyed life. +DUMMY/LJ028-0486.wav|yet the palace of Nebuchadnezzar was protected by a third defense far stronger. +DUMMY/LJ010-0194.wav|but gazed with complacency around the crowded court, and eagerly inquired what persons of distinction were present. +DUMMY/LJ012-0246.wav|The murder in both cases was long a profound mystery. In this which I am now describing, +DUMMY/LJ038-0265.wav|and, further, on this basis the bullet could have been fired from the rifle on the basis of its land and groove impressions, end quote. +DUMMY/LJ019-0361.wav|who was bound to report any deficiencies and abuses he might find at his periodical visits. The Secretary of State might go further. +DUMMY/LJ044-0024.wav|While Oswald publicly engaged in the activities described above, his, quote, organization, end quote, was a product of his imagination. +DUMMY/LJ050-0010.wav|Daily supervision of the operations of the Secret Service within the Department of the Treasury should be improved. +DUMMY/LJ017-0266.wav|They none of them spoke much English except Leon, commonly called Lyons. +DUMMY/LJ043-0104.wav|His supervisor admitted, however, +DUMMY/LJ002-0262.wav|owing to many charitable gifts and bequests, which included annual donations from the Archbishop of Canterbury, +DUMMY/LJ012-0128.wav|A young man named Caspar, clerk to a steam-ship company, +DUMMY/LJ022-0135.wav|but I am jealous of the right of every citizen to call to the attention of his or her government +DUMMY/LJ001-0123.wav|this is awkward if you have to read Bradshaw's Guide in a hurry. +DUMMY/LJ015-0035.wav|produced immediate embarrassment and financial distress. +DUMMY/LJ049-0188.wav|in one or another Government agency, to clarify the division of authority between the agencies involved, and to retain the existing system +DUMMY/LJ050-0191.wav|More importantly, the lack of carefully prepared and carefully transmitted instructions for typical visits to cities +DUMMY/LJ021-0188.wav|that much of our New Deal program is only an attempt to catch up with English reforms that go back ten years or more. +DUMMY/LJ006-0085.wav|If a man is poor and ragged, however inexperienced in crime, or however trifling may be the offense for which he has been committed +DUMMY/LJ024-0026.wav|and the rules of many of our universities and of almost every great private business enterprise, +DUMMY/LJ024-0067.wav|have now given us a Court in which five justices will be over seventy-five years of age before next June +DUMMY/LJ050-0179.wav|Based on its experience during this period, the Secret Service now recommends that additional personnel be made available to PRS +DUMMY/LJ016-0098.wav|or any of the new establishments at home, at Portland, Dartmoor, or elsewhere. +DUMMY/LJ011-0218.wav|spoke severely of the gross deception practiced upon an innocent girl, and sentenced the brothers each to three years' imprisonment, +DUMMY/LJ042-0037.wav|and under which, quote, art, culture and the sprit of man are subjected to commercial enterprising, +DUMMY/LJ032-0197.wav|Photograph of Oswald With Rifle +DUMMY/LJ037-0220.wav|Approximately fifteen minutes before the shooting of Tippit, Oswald was seen leaving his roominghouse. +DUMMY/LJ006-0055.wav|and when the population fell they shut up one half the jail and crowded up the other. +DUMMY/LJ031-0121.wav|Dr. George T. Shires, assisted by Drs. Robert McClelland, Charles Baxter, and Ralph Don Patman, +DUMMY/LJ019-0144.wav|the indiscriminate visitation of friends, and the almost unlimited admission of extra food, these more glaring defects had disappeared. +DUMMY/LJ003-0118.wav|Various punishments were inflicted, the heaviest of which was standing in the pillory. +DUMMY/LJ028-0013.wav|The Damascus mosques and their minarets sink beneath the western sky. +DUMMY/LJ038-0289.wav|As shown above, the note and the photographs of Walker's house and of the nearby railroad tracks +DUMMY/LJ015-0190.wav|that of dishonest rogues who assume piety and philanthropy as a cloak for their misdeeds. +DUMMY/LJ028-0504.wav|designed in colors of white and blue and yellow and black. +DUMMY/LJ035-0007.wav|The encounter in the lunchroom. +DUMMY/LJ010-0092.wav|Lord Harrowby's dinner-party was postponed, but the conspirators knew nothing of it, +DUMMY/LJ002-0250.wav|Their conduct too was orderly on the whole. +DUMMY/LJ006-0033.wav|The ink was barely dry upon their letters of appointment before they appeared at Newgate, and commenced a searching investigation. +DUMMY/LJ009-0262.wav|The crowd vociferously yelled their disapproval, and at length +DUMMY/LJ032-0029.wav|The order coupon was signed, in handprinting, "A. Hidell, P.O. Box two nine one five, Dallas, Texas." +DUMMY/LJ038-0106.wav|These photographs were shown to Oswald on the evening of November twenty-three and again on the morning of the twenty-fourth. +DUMMY/LJ047-0193.wav|and so advised the Dallas office in the ordinary course of business. +DUMMY/LJ014-0237.wav|I will now pass to grave but less atrocious crimes. +DUMMY/LJ011-0191.wav|The strangely-married couple thereupon pressed on to Dover, and crossed over to Calais. +DUMMY/LJ039-0208.wav|this was because of a defect in the scope which was recognized by the FBI agents and which they could have compensated for if they were aiming to hit a bull's-eye. +DUMMY/LJ021-0183.wav|Is it not a fact that ever since the year nineteen oh nine, +DUMMY/LJ002-0057.wav|These wards were all fitted with barrack-beds, but no bedding was supplied. +DUMMY/LJ009-0044.wav|His features have no felonious cast; +DUMMY/LJ050-0244.wav|the assistance of trained Federal law enforcement officers. +DUMMY/LJ007-0128.wav|"what must be the state of discipline in a place filled with characters so various as were assembled there, where the tried and the untried, the sick and the healthy, +DUMMY/LJ037-0213.wav|Marina Oswald gave as her opinion that the mail-order coupon was in Oswald's handwriting. +DUMMY/LJ002-0232.wav|Disturbances were frequent, owing to the riotous conduct of intoxicated women. +DUMMY/LJ023-0126.wav|In other words, we said we would seek an amendment only if every other possible means by legislation were to fail. +DUMMY/LJ042-0178.wav|He thought the new alternative would have its best chance to be accepted after, quote, conflict between the two world systems leaves the world country +DUMMY/LJ017-0185.wav|She was in good health on leaving home, and had with her a large sum of money. +DUMMY/LJ045-0078.wav|Oswald read the letter, but refused to believe that it was sincere, even though his wife insisted to him that it was. +DUMMY/LJ002-0335.wav|so were candles, salt, pepper, mops and brooms. But the latter could have been of little service. Dirt prevailed everywhere; +DUMMY/LJ022-0132.wav|I, therefore, hope you will watch the work in every corner of this Nation. +DUMMY/LJ050-0183.wav|Advice by the Secret Service to local police in metropolitan areas relating to the assistance expected in connection with a Presidential visit +DUMMY/LJ017-0274.wav|When the warder went in to call him he asked for his clothes. He was told he would have to wear his own. +DUMMY/LJ018-0351.wav|and that, in spite of the verdict of the jury, "he left the dock with a calm and quiet conscience;" +DUMMY/LJ042-0175.wav|could possibly make a choice between them, there is no choice, +DUMMY/LJ002-0273.wav|Ludgate to the last was the debtors' prison for freemen of the city of London, +DUMMY/LJ030-0131.wav|which curves in a southwesterly direction as it proceeds downgrade toward the Triple Underpass and the Stemmons Freeway. +DUMMY/LJ017-0255.wav|repaired there secretly, and so gave information to the Brazilian authorities. +DUMMY/LJ009-0129.wav|that many of the kneeling men or boys laughed while they knelt, pinched each other, and, when they could do so without fear of being seen by any officer of the prison, +DUMMY/LJ007-0213.wav|Again the following year the inspectors repeat their charge. +DUMMY/LJ027-0063.wav|enlarged and consolidated to form the pelvis, to serve as a firm support to the hind-limbs. +DUMMY/LJ007-0240.wav|that a strict compliance with their duties gave them no choice "but to report matters as we found them, +DUMMY/LJ011-0144.wav|but Mr. Joseph Hume pressed the Government hard, and obtained an assurance that the men should not be executed. +DUMMY/LJ017-0204.wav|who said that he feared it was only too true that secret poisoning was at that time very rife in the metropolis. +DUMMY/LJ028-0071.wav|Late in his life, in five sixty-seven, he invaded Egypt. +DUMMY/LJ013-0071.wav|Here again the money, with ten years' interest, was handed over to Barber and another calling himself Thomas Hunt, an executor of the will. +DUMMY/LJ019-0072.wav|It is scarcely necessary to follow the controversy further. I have only introduced the subject as showing how little as yet the State +DUMMY/LJ043-0055.wav|He never saw or communicated with her in any way again until she came to see him after the assassination. +DUMMY/LJ050-0166.wav|The Commission was struck by the apparent lack of effort, on an interagency basis, +DUMMY/LJ017-0138.wav|Three years later came the case of Dr. Smethurst, +DUMMY/LJ002-0129.wav|would have amounted to sixty-eight thousand, seven hundred twenty-eight pounds, and if defended, +DUMMY/LJ007-0019.wav|Even in the daytime, when supervision was withdrawn, "the language used to be dreadful," says one of the women when under examination; +DUMMY/LJ015-0172.wav|The inquiry soon revealed the colossal character of the frauds. +DUMMY/LJ009-0256.wav|Still he resisted. +DUMMY/LJ019-0341.wav|The old custom, so fruitful of the worst evils, of keeping a tap inside the prison was made illegal. +DUMMY/LJ018-0386.wav|the whole manufactured while in the condemned cell of the crumb of bread made into paste. +DUMMY/LJ006-0275.wav|to have access to the female side on Sundays and Wednesdays, in order to visit their supposed relations there. +DUMMY/LJ050-0267.wav|between Secretary Dillon and Donald F. Hornig, Special Assistant to the President for Science and Technology, is a useful effort in the right direction. +DUMMY/LJ032-0189.wav|since there was limited, if any, opportunity for Oswald to handle the weapon during the two months prior to November twenty-two. +DUMMY/LJ041-0162.wav|Thornley also testified about an incident which grew +DUMMY/LJ017-0008.wav|who had poisoned his master and many of his dependents. +DUMMY/LJ013-0138.wav|Ashley was the son of an army agent and banker, +DUMMY/LJ048-0020.wav|The interview with him in jail is not significant from the standpoint of whether he had a propensity for violence. +DUMMY/LJ010-0087.wav|and it was arranged that one of their number should knock at Lord Harrowby's door on the pretense of leaving a parcel, +DUMMY/LJ019-0182.wav|Lord John Russell, commenting upon the offer of the Corporation to improve Newgate, +DUMMY/LJ020-0053.wav|in the form of turnovers, pinching the corners of the fold pretty hard to hinder the flap of dough from flying up as the rising proceeds. +DUMMY/LJ050-0036.wav|is not consistent with the standards which the responsibilities of the Secret Service require it to meet. +DUMMY/LJ042-0212.wav|I have often wondered why it is that the communist, anarchist capitalist, and even the fascist and anarchist elements in American, +DUMMY/LJ003-0021.wav|The head of the debtors' prison was a prisoner called the steward, who was chosen by the whole body from six whom the keeper nominated. +DUMMY/LJ011-0157.wav|He thereupon conceived an idea of carrying her off and marrying her willy nilly at Gretna Green. +DUMMY/LJ046-0161.wav|Many of these cases were not investigated by PRS. +DUMMY/LJ025-0075.wav|with the advance of microscopic anatomy, the universality of the fact itself among animals has ceased to be predicable. +DUMMY/LJ002-0314.wav|was after eighteen oh seven, through the exertions of the keeper of the jail, spent in the purchase of necessaries. +DUMMY/LJ044-0230.wav|His unhappy experience with the Cuban consul seems thus to have reduced his enthusiasm for the Castro regime and his desire to go to Cuba. +DUMMY/LJ038-0250.wav|Another photograph of railroad tracks found among Oswald's possessions was not identified by his wife, +DUMMY/LJ013-0227.wav|The order was, however, at length obeyed, and the whole of the prisoner's clothes were minutely searched. +DUMMY/LJ038-0117.wav|He falsely alleged that he bought the revolver in Fort Worth, when in fact he purchased it from a mail-order house in Los Angeles. +DUMMY/LJ031-0131.wav|An agent was stationed at the entrance to stop anyone who was not a member of the Presidential party. +DUMMY/LJ026-0011.wav|Because of these uncertain forms of life, +DUMMY/LJ015-0307.wav|The evidence was corroborated by that of many of the victims who had acted as messengers, +DUMMY/LJ009-0080.wav|The text, without another word, is enough to raise the wildest passions of the audience +DUMMY/LJ049-0146.wav|The expert assistance and resources which it could draw upon would be particularly desirable in this complex and sensitive area. +DUMMY/LJ040-0083.wav|After the divorce Mrs. Oswald complained considerably about how unfairly she was treated, +DUMMY/LJ017-0097.wav|He had swallowed his pills as usual, at least Palmer had administered them +DUMMY/LJ028-0165.wav|There is, however, a second inner wall, of less thickness than the first, but very little inferior to it in strength. +DUMMY/LJ047-0004.wav|No information concerning Lee Harvey Oswald appeared in PRS files before the President's trip to Dallas. +DUMMY/LJ046-0218.wav|Advise the Bureau at the same time by teletype of the information so furnished to the Secret Service and the fact that it has been so disseminated. +DUMMY/LJ048-0058.wav|Agent Hosty testified +DUMMY/LJ002-0009.wav|On the twenty-seventh April, in the following year, +DUMMY/LJ010-0078.wav|On his second release, goaded by his fancied wrongs, he began to plot a dark and dreadful revenge, +DUMMY/LJ011-0192.wav|The fact of the abduction did not transpire for some days. +DUMMY/LJ029-0181.wav|called on the people of Dallas to be "congenial hosts" even though "Dallas didn't vote for Mr. Kennedy in nineteen sixty, +DUMMY/LJ025-0031.wav|there were no means by which the motion of its fluids could be produced by internal causes. +DUMMY/LJ005-0043.wav|and obtained full details, from places where they had been adopted, of the nature of these new machines +DUMMY/LJ003-0197.wav|This procrastination bred certain callousness. +DUMMY/LJ011-0139.wav|This time they sent back the bill, re-enacting only the two penalties for will forging and the forging of powers of attorney; +DUMMY/LJ028-0122.wav|To the west of Babylon, at a greater distance from the outer wall, I constructed an enclosing wall four thousand cubits in length about the city. +DUMMY/LJ011-0052.wav|The sermon ended with an appeal to the dying man, exhorting him to penitence. +DUMMY/LJ029-0011.wav|President Kennedy's visit to Texas in November nineteen sixty-three had been under consideration for almost a year before it occurred. +DUMMY/LJ016-0202.wav|Calcraft's emoluments were a guinea per week, and an extra guinea for every execution. +DUMMY/LJ024-0039.wav|and no Senate of honorable men fit for their office would confirm, that kind of appointees to the Supreme Court. +DUMMY/LJ030-0122.wav|to the right side of the President's car. +DUMMY/LJ028-0098.wav|the wise, the pious, the maintainer of Esagil and Ezida, +DUMMY/LJ007-0102.wav|Pilgrim, Bond, and Clamp, had refused to give evidence before the House of Commons' Committee; a Speaker's warrant was issued for their arrest when they absconded. +DUMMY/LJ008-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section eleven: Executions, part one. +DUMMY/LJ026-0042.wav|The lower plants and animals referred to above are so far from everyday observation +DUMMY/LJ047-0074.wav|Under a general Bureau request to be on the alert for activities of the Fair Play for Cuba Committee +DUMMY/LJ034-0064.wav|This finding, in addition to the freshness of one of the prints and the presence of Oswald's prints on two of the four cartons and the paper bag +DUMMY/LJ008-0114.wav|His hands were clasped, and he was truly penitent. +DUMMY/LJ003-0298.wav|for giving every prisoner a sleeping cell to himself, an amount of enlightenment which is hardly general among European nations at this +DUMMY/LJ011-0195.wav|The terror and dismay of her parents were soon intensified by the receipt of a letter from Wakefield, at Carlisle, announcing the marriage. +DUMMY/LJ045-0129.wav|has visited us here in Dallas, Texas, on November one. Agent James P. Hasty +DUMMY/LJ014-0040.wav|that he rushed to a slaughterhouse in Hampstead and purposely stained his clothes with blood. +DUMMY/LJ014-0209.wav|was perpetrated in eighteen fifty-two. The case was accompanied with the most shocking brutality. +DUMMY/LJ042-0007.wav|At the age of nineteen, Oswald thus committed an act which was the most striking indication he had yet given +DUMMY/LJ021-0080.wav|In these gains and in the policies of the administration there are assurances that hearten all forward- looking men and women +DUMMY/LJ039-0221.wav|The various tests showed that the Mannlicher-Carcano was an accurate rifle and that the use of a four-power scope +DUMMY/LJ010-0029.wav|the cold-blooded, calculating atrocity born of self-interest, were still the irresistible incentives to kill. +DUMMY/LJ025-0028.wav|before it can be converted into substances fitted for absorption, +DUMMY/LJ016-0148.wav|This was not strong enough to carry him, and he fell through. +DUMMY/LJ019-0041.wav|Other jurisdictions were less prompt to recognize their responsibilities, the city of London among the number, as I shall presently show at length. +DUMMY/LJ004-0140.wav|and the turnkey told Mr. Buxton that the "smell on first opening the door was enough to knock down a horse. +DUMMY/LJ009-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section twelve: Executions, part two. +DUMMY/LJ039-0069.wav|Maj. Eugene D. Anderson, assistant head of the Marksmanship Branch of U.S. Marine Corps +DUMMY/LJ007-0034.wav|At times the numbers congregated together were very great; as many as fifty and sixty, even more, were crowded indiscriminately into the press-yard. +DUMMY/LJ046-0131.wav|in nineteen fifty-three this had increased to more than seventeen thousand items; +DUMMY/LJ009-0002.wav|It is satisfactory to be able to record that some consideration was shown the capital convict actually awaiting execution. +DUMMY/LJ027-0139.wav|and developmental changes which may be observed during the life history of now existing individuals belonging to the same group of animals. +DUMMY/LJ028-0366.wav|Therefore, to destroy the power of the old capital, he planned to build Seleucia on the Tigris about fifty miles to the east. +DUMMY/LJ019-0386.wav|that no complete reform could be accomplished so long as the prisons were left under the jurisdiction of the local authorities. +DUMMY/LJ039-0232.wav|the assassin was then required to hit the target one more time within a space of from four point eight to five point six seconds. +DUMMY/LJ047-0175.wav|or telephone number in Dallas. Mrs. Paine testified that she learned Oswald's telephone number at the Beckley Street roominghouse in the middle of October +DUMMY/LJ005-0031.wav|no visiting of friends, no education but religious education, no freedom of diet, +DUMMY/LJ047-0233.wav|his lies to Agent Quigley, his recent visit to Mexico City -- indicated that Oswald was capable of violence. +DUMMY/LJ011-0005.wav|A lady in the country, who had thirteen thousand pounds in the stocks, desired her London agent to sell them out. +DUMMY/LJ048-0231.wav|According to their statements, the agents remained at the Press Club for periods varying from thirty minutes to an hour and a half, +DUMMY/LJ020-0010.wav|One heaping cup of sifted flour. +DUMMY/LJ012-0207.wav|The porter asked what he had got, and the answer was, a male subject. +DUMMY/LJ009-0070.wav|The youth's hands tremble as they hold the book upside down. +DUMMY/LJ008-0006.wav|as the exposure was brutal and cruel towards the principal actors. +DUMMY/LJ046-0121.wav|At the time of the assassination PRS was a very small group, comprised of twelve specialists and three clerks. +DUMMY/LJ019-0138.wav|at the most crowded periods, or just before the sessional jail delivery; and at its lowest it fell sometimes to fifty or sixty. +DUMMY/LJ041-0037.wav|and subsequently, after this conversation, my father came in and we were kind of arguing back and forth about the situation, and my father came in the room, +DUMMY/LJ019-0105.wav|and that a new system of prison discipline should be introduced. +DUMMY/LJ036-0056.wav|Mrs. Bledsoe recalled that Oswald sat halfway to the rear of the bus which moved slowly and intermittently as traffic became heavy. +DUMMY/LJ032-0111.wav|Letters received by Marina Oswald from her husband signed "Alek" were given to the Commission. +DUMMY/LJ017-0045.wav|The Central Criminal Court was crowded to suffocation. +DUMMY/LJ005-0092.wav|and every prisoner was to be provided with a hammock or cot to himself, suitable bedding, and, if possible, a separate cell. +DUMMY/LJ003-0165.wav|The better class of inmate complained bitterly of this enforced companionship with the vile, +DUMMY/LJ040-0016.wav|and in the history of his life which does give some insight into his character and, possibly, into the motives for his act. +DUMMY/LJ003-0237.wav|The fees on reception and discharge must be deemed exorbitant, when it is remembered the impoverished class who usually crowded the jail; +DUMMY/LJ026-0037.wav|It will no doubt occur to the reader that, on the theory of evolution, +DUMMY/LJ026-0007.wav|Many of the lowest forms of life have long been puzzles +DUMMY/LJ004-0173.wav|and sometimes slept in "the pit," loaded with heavy irons for a whole year, waiting the jail delivery. +DUMMY/LJ003-0069.wav|In this heterogeneous society were also thrown the unfortunate journalists to whom I have already referred, and on whom imprisonment in Newgate +DUMMY/LJ032-0255.wav|About one week after the return from New Orleans, +DUMMY/LJ015-0160.wav|he patronized the best tailors, and had a fashionable coiffeur from Hanover Square daily to curl his hair. +DUMMY/LJ032-0002.wav|Chapter four. The Assassin: Part one. +DUMMY/LJ016-0290.wav|There were but few women, except of the most degraded sort; the men were mostly young men -- +DUMMY/LJ044-0018.wav|and was arrested because of a dispute with three anti-Castro Cuban exiles, and again on August sixteen, nineteen sixty-three. +DUMMY/LJ016-0271.wav|The reply evinced equal satisfaction, and the speaker, with a profane oath, declared that he would like to act as Jack Ketch to the whole lot. +DUMMY/LJ001-0016.wav|The Middle Ages brought calligraphy to perfection, and it was natural therefore +DUMMY/LJ006-0151.wav|Many of them were otherwise and improperly occupied for hours every day in menial services for the governor, cleaning his windows or grooming his horse. +DUMMY/LJ020-0002.wav|Bread raised with what is known to bakers as a “sponge,” requires more time and a trifle more work than the simpler form for which I have just already given directions. +DUMMY/LJ001-0070.wav|and was used there with very little variation all through the sixteenth and seventeenth centuries, and indeed into the eighteenth. +DUMMY/LJ007-0097.wav|and not the least of the good services rendered by the new inspectors was their inquiry into the status of these unfortunate people, and their recommendation to improve it. +DUMMY/LJ022-0024.wav|They know that the process of the constructive rebuilding of America cannot be done in a day or a year, +DUMMY/LJ022-0139.wav|with which we have been concerned for two years. +DUMMY/LJ021-0050.wav|I sincerely hope that as a result +DUMMY/LJ028-0392.wav|The statement of St. Jerome is confirmed by the following passage from Zosimus, a Greek writer of the fifth century A.D.: +DUMMY/LJ049-0215.wav|the protection of the President is in a real sense a Government-wide responsibility which must necessarily assumed by the Department of State, +DUMMY/LJ029-0130.wav|the President's motorcade would pass the Texas School Book Depository Building on the northwest corner of Houston and Elm Streets. +DUMMY/LJ028-0194.wav|But how did the "mighty city" fall? How could Cyrus take Babylon whose walls were strong enough to resist any army? +DUMMY/LJ028-0452.wav|The outer wall bore the name of Nimitti-Bel. Its direction was northeast and southwest, forming a triangle with the river. +DUMMY/LJ028-0192.wav|When the news came to the Hebrews, who were held there in exile, they excitedly rushed about the streets, crying: "Babylon is fallen," +DUMMY/LJ048-0259.wav|However, Chief Rowley did not condone the action of the off-duty agents, particularly since it violated a regulation of the Secret Service, +DUMMY/LJ006-0144.wav|A wardsman did not necessarily lose his situation for it. Nor was drink the only creature comfort he might enjoy. +DUMMY/LJ005-0157.wav|Here and there food was not issued in kind, but a money allowance which the prisoner might expend himself. +DUMMY/LJ016-0133.wav|There is, however, no explanation of the motives which prompt prisoners to attempt escapes. +DUMMY/LJ018-0226.wav|leaving everything to the widow, and constituting William sole executor. +DUMMY/LJ045-0184.wav|Then on Thursday morning, November twenty-one, +DUMMY/LJ010-0072.wav|Returning to England, +DUMMY/LJ002-0118.wav|a table showing the figures for the year ending Michaelmas eighteen oh two. +DUMMY/LJ011-0291.wav|Two Bow Street runners were dispatched to the house in York Street, which had evidently been taken on purpose for the outrage. +DUMMY/LJ036-0023.wav|On the basis of the date and time on the transfer, McWatters was able to testify that the transfer had been issued by him +DUMMY/LJ016-0276.wav|Never, indeed, had an execution been more generally patronized. +DUMMY/LJ008-0073.wav|One case is preserved by Catnach, +DUMMY/LJ010-0066.wav|which under Thistlewood as dictator was to rule the nation, by first handing over its capital to fire and pillage. +DUMMY/LJ003-0226.wav|The rations of food were notoriously inadequate, and so carelessly distributed, that many were left to starve. +DUMMY/LJ046-0240.wav|Bouck explained the failure to try to identify the individuals involved in the Stevenson incident after it occurred on the ground that +DUMMY/LJ002-0251.wav|Drunkenness was not common, chiefly because liquor was not to be had freely, although the tapster paid a rent of two guineas a week for permission to sell it. +DUMMY/LJ013-0028.wav|and at last, when within fifteen miles of port, with wind and weather perfectly fair, he ran her on to the rocks. +DUMMY/LJ017-0092.wav|He decided to use strychnia, or the vegetable poison otherwise known as nux vomica; +DUMMY/LJ031-0112.wav|in its greatest diameter, was treated by cutting away the damaged skin and suturing the back muscle and skin. +DUMMY/LJ038-0144.wav|that must have been some other time he picked me up, end quote, +DUMMY/LJ034-0184.wav|Approximately one week after the assassination, according to Fisher, policemen showed him a picture of Oswald. +DUMMY/LJ042-0067.wav|Soak fist in cold water to numb the pain. Then slash my left wrist. Than plunge wrist into bathtub of hot water. +DUMMY/LJ015-0271.wav|He dealt too in the precious metals, when they had been improperly acquired, +DUMMY/LJ049-0222.wav|Pending any such determination, however, this Commission is convinced of the necessity of better coordination +DUMMY/LJ042-0233.wav|His first set of questions and answers indicated his belief that there were no outstanding differences between the Soviet Union and the United States, +DUMMY/LJ017-0213.wav|the first and second mates, Karswell and Taffir; there were two other Englishmen on board, and the rest of the crew were a polyglot lot, +DUMMY/LJ008-0117.wav|"You have tied me very tight," upon which Dr. Forde ordered him to slacken the cord, which he did, but not without muttering. +DUMMY/LJ008-0074.wav|that of Phoebe Harris, who in seventeen eighty-eight was "barbariously" executed and burnt before Newgate for coining. +DUMMY/LJ021-0108.wav|from the standpoint of the good of the industries themselves, as well as the general public interest, +DUMMY/LJ029-0045.wav|to select the luncheon site and motorcade route, and to plan security measures for the luncheon and the motorcade. +DUMMY/LJ013-0179.wav|The latter was so placed that it could throw no light on the former, which was a 'Life of Sir Samuel Romilly.' +DUMMY/LJ028-0336.wav|Darius now, still keeping to the plan agreed upon, +DUMMY/LJ028-0514.wav|The great outer wall seems to have disappeared completely beneath the desert surface. +DUMMY/LJ015-0023.wav|Mr. Bates had been confidential managing clerk, and was taken into the firm not alone as a reward for long and faithful service +DUMMY/LJ040-0062.wav|The Early Years +DUMMY/LJ038-0225.wav|which would indicate that it was probably written before the birth of Oswald's second child on October twenty, nineteen sixty-three. +DUMMY/LJ027-0161.wav|It is essentially a fish, and would be so classed if it remained in this condition. +DUMMY/LJ006-0023.wav|had made such prisons as Auburn a model for imitation. +DUMMY/LJ046-0044.wav|in accordance with George Washington's maxim that Americans have a government, quote, of accommodation as well as a government of laws, end quote, +DUMMY/LJ008-0020.wav|Here John Sutherland, commander of the British armed transport 'The Friends,' suffered on the twenty-ninth June, eighteen oh nine, +DUMMY/LJ017-0056.wav|Although the strain lasted fourteen days, he showed no signs of exhaustion, either physical or mental. +DUMMY/LJ038-0172.wav|Although the bullet was recovered from Walker's house, in the absence of a weapon it was of little investigatory value. +DUMMY/LJ030-0110.wav|and Vice-Presidential follow-up car used portable sets with a separate frequency for their own car-to-car communication. +DUMMY/LJ025-0122.wav|The contractility which is the fundamental condition of locomotion," continues Huxley, +DUMMY/LJ039-0178.wav|This angle was used in the test because the majority of the eyewitnesses to the assassination +DUMMY/LJ031-0169.wav|Motorcade policemen who escorted the automobiles were requested by the Vice President and Agent Youngblood not to use sirens. +DUMMY/LJ018-0144.wav|The amount of Brewer's abstractions (who was eventually acquitted) was never exactly estimated. +DUMMY/LJ012-0251.wav|the lockman of "Ben Jonson lock," in Stepney Fields, found a human head jammed into the lock gates. +DUMMY/LJ044-0101.wav|He had previously sent, apparently unsolicited, to the Party newspaper, the Worker, +DUMMY/LJ040-0205.wav|and of the, quote, protective shell he has drawn around himself, end quote. Dr. Hartogs reported that Mrs. Oswald did not understand +DUMMY/LJ027-0073.wav|may be shown to be modifications one of another. +DUMMY/LJ020-0031.wav|Into the “crater” dug out in the middle, pour the sponge, warm water, the molasses, and soda dissolved in hot water. +DUMMY/LJ004-0130.wav|incommodious, as has been stated, insecure, unhealthy, and unprovided with the printed or written regulations required by law. +DUMMY/LJ029-0062.wav|It also advised the Secret Service of the circulation on November twenty-one of a handbill sharply critical of President Kennedy, +DUMMY/LJ015-0217.wav|The next step was to get impressions and fabricate false keys. +DUMMY/LJ006-0292.wav|male prisoners carried coal into the female prison, when they saw and could speak or pass letters to the female prisoners; +DUMMY/LJ037-0040.wav|She saw the man come to the right window of the police car. As he talked, he leaned on the ledge of the right window with his arms. +DUMMY/LJ043-0183.wav|Oswald did not lack the determination and other traits required +DUMMY/LJ050-0009.wav|disclosed by the Commission's investigation. Other measures should be taken as well to improve the overall operation of the Secret Service. +DUMMY/LJ005-0246.wav|where "prisoners before and after trial are under no efficient superintendence," and where "there was no restraint, or attempt at restraint." +DUMMY/LJ011-0073.wav|several persons were sentenced to or suffered death for this crime. +DUMMY/LJ025-0025.wav|he naturally sees the primary and the most important distinction between animals and plants, +DUMMY/LJ025-0017.wav|He affirms that living beings have been subdivided from the earliest times into animated beings, which possess sense and motion, +DUMMY/LJ004-0050.wav|Mr. Buxton had already been associated with Mrs. Fry in the Newgate visitation, and his attention had thus been drawn to the neglected state of English prisons. +DUMMY/LJ050-0058.wav|which was the possibility of an attempt against the person or safety of the President. +DUMMY/LJ026-0146.wav|They are absorbed by the circulating liquids and carried to the eliminating organs, lungs and kidneys chiefly, for elimination. +DUMMY/LJ031-0150.wav|Other Secret Service agents posted themselves in the corridors and other areas near the emergency room. +DUMMY/LJ017-0217.wav|Navigation and discipline could not be easy with such a nondescript crew. +DUMMY/LJ017-0175.wav|The circumstances were strange. Mrs. Wilson had gone to the chemist's for medicine, +DUMMY/LJ036-0045.wav|Mrs. Bledsoe told him, quote, I am not going to rent to you any more, end quote, +DUMMY/LJ002-0284.wav|The Borough Compter was in a disgraceful state to the last. The men's ward had an earth, or rather a mud, floor, +DUMMY/LJ018-0212.wav|The annals of fraudulent crime probably contain nothing +DUMMY/LJ014-0253.wav|there was no independent check upon him if he chose to tamper with it. +DUMMY/LJ017-0222.wav|hatched amongst them a plot to murder their officers and seize the ship. +DUMMY/LJ040-0051.wav|While there is doubt about how fully Oswald understood the doctrine which he so often espoused, it seems clear +DUMMY/LJ036-0008.wav|(one) positive identification of the killer by two eyewitnesses who saw the shooting +DUMMY/LJ008-0166.wav|The pieman stooped down to pick up his scattered stock, and some of the mob, not seeing what had happened, stumbled over him. +DUMMY/LJ019-0360.wav|This he might do on the representation of the inspector of prisons, +DUMMY/LJ048-0102.wav|In particular, the Commission believes that the motorcade route selected by Agent Lawson, upon the advice of Agent in Charge Sorrels +DUMMY/LJ047-0219.wav|Quote, Mr. Shanklin advised us, among other things, +DUMMY/LJ033-0135.wav|Location of Bag +DUMMY/LJ006-0262.wav|all of them instruments calculated to facilitate attempts at breaking out of prison, +DUMMY/LJ007-0149.wav|the jail, and the scaffold. +DUMMY/LJ044-0118.wav|He anticipated that the full disclosure of his defection would hinder him in, quote, the struggle for progress and freedom in the United States, end quote. +DUMMY/LJ014-0072.wav|and that during the whole time he was never in his right senses. +DUMMY/LJ036-0073.wav|A bus moving through heavy traffic on Elm from Murphy to Lamar was timed at four minutes. +DUMMY/LJ046-0149.wav|There are many actions, situations, and incidents that may indicate such potential danger. Some are specific, such as threats; +DUMMY/LJ011-0061.wav|Let this monster give his name; I am ready to fight him. I am still determined to put myself in the place of Mr. Fauntleroy. +DUMMY/LJ040-0003.wav|The evidence reviewed above identifies Lee Harvey Oswald as the assassin of President Kennedy and indicates that he acted alone in that event. +DUMMY/LJ047-0063.wav|Agent Hosty was told by Mrs. M. F. Tobias, a former landlady of the Oswalds at six oh two Elsbeth Street in Dallas, +DUMMY/LJ015-0268.wav|Instead of following an honorable profession, he preferred to turn his great natural talents and ready wits to the most nefarious practices. +DUMMY/LJ039-0016.wav|he finished reading a morning newspaper, quote, and put on a good suit. I saw that he took a pistol. +DUMMY/LJ037-0195.wav|The order was signed in ink by, quote, +DUMMY/LJ018-0338.wav|Greed in another form led the Stauntons to make away with Mrs. Patrick Staunton, murdering her with devilish cruelty by slow degrees. +DUMMY/LJ015-0184.wav|and he looked about him with wayward, furtive glances. +DUMMY/LJ040-0132.wav|with Puerto Ricans and Negroes and everything, end quote. +DUMMY/LJ002-0242.wav|to be held before the steward of the royal household, the knight marshal, and the steward of the court, +DUMMY/LJ042-0150.wav|Marina Oswald's judgment of her husband's state of mind may be substantiated by comparing material which he wrote in the Soviet Union +DUMMY/LJ044-0056.wav|was stamped on some literature that Oswald had in his possession at the time of his arrest in New Orleans, +DUMMY/LJ019-0014.wav|had embarked on a considerable expenditure in order to provide a model prison for general imitation. +DUMMY/LJ018-0059.wav|He saw Mr. Briggs' watch-chain, and followed him instantly into the carriage, determined to have it at all costs. +DUMMY/LJ036-0200.wav|the radio dispatcher on channel one ordered all downtown patrol squads to report to Elm and Houston, code three (emergency). +DUMMY/LJ007-0027.wav|Bad as were the various parts of the jail already dealt with, +DUMMY/LJ003-0030.wav|These subordinate chiefs were also rewarded out of the scanty prison rations. +DUMMY/LJ014-0268.wav|whither he was taken every morning in a smart brougham, they would have seen him alight from it in Cornhill, +DUMMY/LJ006-0119.wav|When the wardsman was a man of some education, with some knowledge of legal chicanery gained by personal experience, he might add considerably to his emoluments +DUMMY/LJ022-0075.wav|We are losing no time getting the government's vast work relief program underway, +DUMMY/LJ025-0066.wav|and his skepticism was the more justified since Ehrenberg in his elaborate and comprehensive work on the infusoria, +DUMMY/LJ004-0137.wav|and that it was accomplished by "sleeping edgewise." +DUMMY/LJ023-0079.wav|But since the rise of the modern movement for social and economic progress through legislation, +DUMMY/LJ007-0100.wav|These were the gentlemen concerned in the bribery case at Ipswich in eighteen thirty-five, +DUMMY/LJ048-0032.wav|so far as the FBI knew he had not shown any potential for violence. +DUMMY/LJ045-0049.wav|Lee would say that it would be better for me if I went to Russia. I did not know why. I did not know what he had in mind. +DUMMY/LJ035-0018.wav|end quote, After the third shot, he, quote, revved that motorcycle up, end quote, +DUMMY/LJ041-0174.wav|and had not intended any criticism of Oswald's political views which is the way in which, Thornley thought, Oswald took his remarks. +DUMMY/LJ016-0087.wav|Among the escapes still remembered was one in eighteen forty-nine, accomplished by a man who had been employed +DUMMY/LJ018-0322.wav|Nor were these two abortive efforts all that were planned. +DUMMY/LJ012-0186.wav|The crime was brought home to Thurtell by the confession of Hunt, +DUMMY/LJ004-0162.wav|It was "a scene of infernal passions and distresses," says Buxton, "which few have imagination sufficient to picture, +DUMMY/LJ036-0016.wav|Lee Harvey Oswald left the building approximately three minutes after the assassination. +DUMMY/LJ038-0227.wav|From Neely Street the Oswalds would have traveled downtown on the Beckley bus, across the Commerce Street viaduct +DUMMY/LJ006-0263.wav|and capable of becoming most dangerous weapons in the hands of desperate and determined men. +DUMMY/LJ030-0054.wav|Because the skies had cleared in Dallas, Lawson directed that the top not be used for the day's activities. +DUMMY/LJ018-0074.wav|Sattler probably misunderstood, and he declared that the police officer had broken faith with him, having, moreover, stated that +DUMMY/LJ048-0221.wav|In the early morning hours on November twenty-two, nineteen sixty-three, +DUMMY/LJ011-0211.wav|when indictments were preferred against both brothers "for having carried away Ellen Turner, spinster, +DUMMY/LJ044-0153.wav|While that first request, made according to Marina Oswald at her husband's insistence, specifically stated that Oswald was to remain in the United States, +DUMMY/LJ030-0015.wav|Late in the evening, the Presidential party flew to Fort Worth where they spent the night at the Texas Hotel. +DUMMY/LJ007-0165.wav|any awakening truth, salutary exhortation, or imperfect resolutions of amendment can take root or grow. +DUMMY/LJ043-0184.wav|to carry out a carefully planned killing of another human being and was willing to consummate such a purpose if he thought there was sufficient reason to do so. +DUMMY/LJ035-0159.wav|Mrs. Reid believes that she returned to her desk from the street about two minutes after the shooting. +DUMMY/LJ014-0215.wav|threw the constable on his back, and violently assaulted him by jumping on his chest and stomach, +DUMMY/LJ018-0220.wav|The old gentleman's signature to this deed of gift was a forgery, +DUMMY/LJ017-0054.wav|His features were not careworn, but rather set, and he looked older than his age. +DUMMY/LJ012-0030.wav|Thus, a watch was paid for as a watch, whether it was of gold or silver; a piece of linen as such, whether the stuff was coarse or fine. +DUMMY/LJ010-0286.wav|nine shillings, all in shillings, and no other coin. +DUMMY/LJ022-0137.wav|I now come, my friends, to a part of the remaining business before the Congress. +DUMMY/LJ007-0009.wav|They had tried, moreover, +DUMMY/LJ014-0197.wav|the police magistrate, that Marley was the man who had beaten him to death. +DUMMY/LJ039-0113.wav|as a, quote, rather poor shot, end quote. +DUMMY/LJ003-0082.wav|The middle yard, as far as its limits would permit, was appropriated to felons and transports. The wards here were generally very crowded. +DUMMY/LJ011-0038.wav|Fauntleroy meanwhile lay in Newgate, not herded with other condemned prisoners, as the custom was, +DUMMY/LJ002-0258.wav|Bad language, quarreling, throwing water over one another was forbidden on pain of fine and being sent to Coventry; +DUMMY/LJ036-0033.wav|asked for a transfer and got off at the same place in the middle of the block where the lady did. +DUMMY/LJ011-0116.wav|A society which had already been started against capital punishment +DUMMY/LJ044-0087.wav|Oswald was confronted with those facts at the beginning of the debate and was so thrown on the defensive by this that he was forced to state that +DUMMY/LJ009-0246.wav|Good people, do not be hurried; I am not, I can wait. +DUMMY/LJ021-0099.wav|in order that the functions of N.R.A. which have proved their worth may be made a part of the permanent machinery of government. +DUMMY/LJ035-0114.wav|Neither elevator could have been used by Oswald as a means of descent. +DUMMY/LJ030-0246.wav|Almost in the same moment in which he hit or pushed me, he vaulted over the back seat and sat on me. +DUMMY/LJ046-0173.wav|to advise the field office if the subject displays signs of increased danger or plans to leave his home area. +DUMMY/LJ022-0154.wav|I consider this legislation a positive recovery measure. +DUMMY/LJ045-0053.wav|Many times Oswald has recalled this matter to me and said that I am just waiting for an opportunity to hurt him. +DUMMY/LJ022-0082.wav|In all cases projects must be of a character to give employment to those on the relief rolls. +DUMMY/LJ028-0099.wav|the first-born son of Nabopolassar, King of Babylon, am I. +DUMMY/LJ045-0116.wav|While the facts of his defection had become known in New Orleans as a result of his radio debate with Bringuier, +DUMMY/LJ048-0071.wav|Most important, notwithstanding that both agencies have professed to the Commission that the liaison between them was close and fully sufficient, +DUMMY/LJ028-0033.wav|There is a Hebrew tradition that it was the oldest of all cities, but now we know that great empires flourished and passed away before Babylon was built. +DUMMY/LJ025-0016.wav|and justify biologists in regarding them as representative expressions of the most extensive, if not the profoundest, knowledge of his time. +DUMMY/LJ011-0123.wav|He forged the names of people who held stock on the Bank books, and got the value of the stock; +DUMMY/LJ019-0255.wav|Great numbers were not lighted at night, and were without means by which their inmates could communicate, in case of urgent necessity, with their keepers. +DUMMY/LJ042-0241.wav|Soon after his arrival he wrote to the Soviet Embassy in Washington +DUMMY/LJ041-0013.wav|Another time, he fought with two brothers who claimed that he had picked on the younger of them, three years Oswald's junior. +DUMMY/LJ040-0169.wav|She observed that since Lee's mother worked all day, he made his own meals and spent all his time alone +DUMMY/LJ047-0007.wav|It had interviewed him twice shortly after his return to the United States, again a year later at his request +DUMMY/LJ003-0059.wav|The sleeping accommodation in the chapel-yard wards, and indeed throughout the prison, consisted of a barrack bed, +DUMMY/LJ017-0167.wav|Smethurst was found guilty by the jury, and sentenced to death. +DUMMY/LJ040-0211.wav|This would make him approximately ten, well, almost eleven years old. End quote. +DUMMY/LJ011-0009.wav|Nothing more was heard of the affair, although the lady declared that she had never instructed Fauntleroy to sell. +DUMMY/LJ006-0064.wav|In the master's side yard, which had only one washing place, as many as seventy-eight prisoners, frequently more, +DUMMY/LJ030-0142.wav|Governor Connally sustained bullet wounds in his back, the right side of his chest, right wrist, and left thigh. +DUMMY/LJ049-0102.wav|the President, Vice President, or other officer next in the order of succession to the Office of President, the President-elect and the Vice-President-elect, +DUMMY/LJ028-0224.wav|I quieted their sighings and soothed their sorrows. +DUMMY/LJ005-0088.wav|Every prison containing female prisoners was to have a matron who was to reside constantly in the prison. +DUMMY/LJ048-0002.wav|Chapter eight. The Protection of the President. Part three. +DUMMY/LJ036-0130.wav|Whaley described the ensuing events as follows, quote, +DUMMY/LJ036-0038.wav|and who, during the ride south on Marsalis, had an argument with a woman passenger. +DUMMY/LJ012-0034.wav|diamonds refaced; all marks by which other articles might be identified, the selvages of linen, the stamps on shoes, +DUMMY/LJ004-0158.wav|All were in ill health; almost all were in rags; almost all were filthy in the extreme. +DUMMY/LJ017-0194.wav|that immediately afterwards Mrs. Soames was taken ill with vomiting and purging, +DUMMY/LJ018-0302.wav|For these cash was obtained in notes; the notes were exchanged by one of the conspirators for gold at the Bank of England, and again the same day +DUMMY/LJ042-0201.wav|He wrote, quote, The Communist Party of the United States has betrayed itself! +DUMMY/LJ003-0089.wav|Occasionally the transports made themselves so useful in the jail that they were passed over. +DUMMY/LJ015-0114.wav|Thence he was transferred to Copenhagen and surrendered in due course to a London police officer. +DUMMY/LJ035-0056.wav|Meanwhile, Truly had run up several steps toward the third floor. +DUMMY/LJ026-0076.wav|The green plant owes its power of absorbing the energy of sunlight +DUMMY/LJ037-0241.wav|The suspect last seen running west on Jefferson from four hundred East Jefferson. +DUMMY/LJ002-0241.wav|The court of the Marshalsea was instituted by Charles the first in the sixth year of his reign, +DUMMY/LJ021-0076.wav|could not be brought about in one year. +DUMMY/LJ014-0281.wav|which he had paid into his own bankers, and on this he was committed to Newgate for trial. +DUMMY/LJ033-0010.wav|the presence of a long handmade brown paper bag near the point from which the shots were fired, and (five) the palmprint, +DUMMY/LJ050-0208.wav|salaries are below those of the FBI and leading municipal police forces. +DUMMY/LJ022-0055.wav|the second is to establish the practical means to help those who are unemployed in this present emergency. +DUMMY/LJ004-0047.wav|One of the moving spirits was the Honorable H. G. Bennet, M.P., whose vigorous protests against the lamentable condition of Newgate have already been recorded. +DUMMY/LJ018-0393.wav|Messrs. Swindlehurst, Saffery, and Langley cleared a large profit +DUMMY/LJ049-0218.wav|between the Secret Service and the President and his family is contemplated. +DUMMY/LJ050-0218.wav|Chief Rowley explained that this would not provide enough additional manpower to take all the measures which he considers required. +DUMMY/LJ030-0023.wav|Arrival at Love Field +DUMMY/LJ003-0276.wav|Although attempted partially at Bridewell, and more systematically at the new Millbank penitentiary, +DUMMY/LJ007-0039.wav|and when the inspectors visited Newgate they found the three certain to die in a day-room by themselves; +DUMMY/LJ016-0190.wav|"I have no idea who the sheriffs will get to do the work after me," said Foxen, adding that his assistant, Tom Cheshire, +DUMMY/LJ033-0089.wav|After entering the car, Frazier glanced over his shoulder and noticed a brown paper package on the back seat. +DUMMY/LJ037-0217.wav|Police found an empty revolver holster when they searched Oswald's room on Beckley Avenue after his arrest. +DUMMY/LJ018-0372.wav|mutinied, murdered the captain and mates, sparing the steward only on condition that he would navigate the ship to the Mediterranean. +DUMMY/LJ002-0132.wav|An elaborate machinery planned for the protection of the trader, and altogether on his side, had long existed for the recovery of debts. +DUMMY/LJ033-0050.wav|Marina Oswald testified that it was between nine and ten p.m. +DUMMY/LJ016-0355.wav|Mackay's fortitude, which had been great, +DUMMY/LJ007-0022.wav|After locking-up time, which varied, as on the male side, according to the daylight, the scenes were often riotous and disgraceful. +DUMMY/LJ014-0144.wav|She uttered loud imprecations, cursing judge, jury, barristers, witnesses, and all who stood around. +DUMMY/LJ044-0039.wav|they said something about remodeling, etc. I'm sure you understand, end quote. +DUMMY/LJ038-0259.wav|The oral report was negative because of the battered condition of the bullet. +DUMMY/LJ032-0202.wav|Two pictures were taken. The Commission has concluded that the rifle shown in these pictures is the same rifle which was found on the sixth floor of the Depository Building +DUMMY/LJ028-0246.wav|But, as it was, the Persians came upon them by surprise and so took the city. +DUMMY/LJ041-0106.wav|At the court-martial hearing which followed, Oswald admitted that he had been rather drunk when the incident occurred. +DUMMY/LJ044-0008.wav|and into his possible motivation for the assassination. +DUMMY/LJ018-0073.wav|This reply of his contained no promise of immediate release. +DUMMY/LJ045-0050.wav|He said he loved me but that it would be better for me if I went to Russia, and what he had in mind I don't know. End quote. +DUMMY/LJ027-0024.wav|is the theory of descent -- i.e., that similarities are due to blood relationship and that differences come from adaptive modifications. +DUMMY/LJ024-0074.wav|instead of leaving the composition of our federal courts, including the highest, +DUMMY/LJ018-0125.wav|Burnett had only just come out of jail after completing a sentence of penal servitude. +DUMMY/LJ007-0091.wav|The lunatic became the sport of the idle and the depraved. His cure was out of the question; +DUMMY/LJ026-0116.wav|The foods containing only the elements carbon, hydrogen and oxygen (fats and carbohydrates) +DUMMY/LJ014-0325.wav|Goods would be landed at this wharf; +DUMMY/LJ013-0025.wav|But another ship, concluding that he was acting in ignorance, rendered him assistance. +DUMMY/LJ020-0099.wav|When habited for the day in all except the outer gown, collar, etc., slip on the wrapper again and run down to put the biscuits in the oven. +DUMMY/LJ005-0216.wav|But it was pointed out in eighteen twenty-three that defective construction must always bar the way to any radical improvement in Newgate. +DUMMY/LJ022-0134.wav|Neither you nor I want criticism conceived in a purely fault-finding or partisan spirit, +DUMMY/LJ038-0141.wav|therefore, he made his weekly visit on Thursday night. +DUMMY/LJ009-0188.wav|One was attached to Hicks' Hall, the Clerkenwell Sessions House, built out of monies provided by Sir Baptist Hicks, a wealthy alderman of the reign of James the first +DUMMY/LJ013-0187.wav|near it was his Waterloo medal, and the above-mentioned ten-pound note. +DUMMY/LJ036-0217.wav|As Tippit reached the left front wheel the man pulled out a revolver and fired several shots. +DUMMY/LJ009-0288.wav|Calcraft, of whom mention has just been made, was by trade a lady's shoemaker, +DUMMY/LJ013-0254.wav|and evidence was given of close relation between Good and a third woman, a girl named Butcher, residing at Woolwich. +DUMMY/LJ008-0170.wav|the man, needing all his care for his own life, threw the child from him, +DUMMY/LJ009-0132.wav|Although this misapplication of religious services still went on, +DUMMY/LJ016-0123.wav|got hold of the step ladder used in lighting the gas, and which under our more careful supervision would have been, as now-a-days, chained up. +DUMMY/LJ019-0211.wav|It was urged that the area selected for purchase must be excessively costly to acquire, and still quite inadequate for the city needs. +DUMMY/LJ003-0092.wav|There were frequently in the middle yard seven or eight children, the youngest barely nine, +DUMMY/LJ011-0057.wav|urging that Fauntleroy was a father, a citizen: "his life is useful, mine a burthen, to the State." +DUMMY/LJ019-0158.wav|Later on a more efficacious but still imperfect method of supervision was introduced. Iron cages, which are still to be seen in Newgate, +DUMMY/LJ017-0004.wav|I have attempted to draw of crime in connection with Newgate, from first to last, I must make some mention, in this my penultimate chapter, +DUMMY/LJ044-0034.wav|and of his need to present himself to others as well as to himself in a light more favorable than was justified by reality. +DUMMY/LJ034-0073.wav|Givens testified that after reaching the first floor, quote, +DUMMY/LJ050-0241.wav|The FBI has indicated that it is willing to continue to make such assistance available, +DUMMY/LJ002-0144.wav|and when once appointed continued to serve sine die; they were generally near in rank to the parties whose causes they decided. +DUMMY/LJ018-0375.wav|The 'Lennie's' men were all Greeks, except one known as French Peter, +DUMMY/LJ011-0263.wav|It was perpetrated upon a respectable country solicitor, +DUMMY/LJ014-0190.wav|Marley, disturbed, picked up a cigar and parcel from the counter, then ran out, pursued by Lerigo only. +DUMMY/LJ014-0133.wav|She had not been treated like a Christian, but like a wild beast of the forest. She declared that the money found in her possession had been sent her from abroad; +DUMMY/LJ018-0247.wav|William Roupell himself was brought as a principal witness to clench the case by a confession altogether against himself. +DUMMY/LJ039-0216.wav|It should be noted, however, that the President's car was curving slightly to the right when the third shot was fired. +DUMMY/LJ005-0156.wav|in others meat, soup, gruel, beer were given. +DUMMY/LJ048-0026.wav|He gave evidence of settling down. +DUMMY/LJ048-0189.wav|the Dallas police to scan windows of buildings lining the motorcade route, although it was his usual practice to do so. +DUMMY/LJ018-0100.wav|took to appropriating the bills intrusted to him, and so lost his business, after which he became a clerk to Messrs. Wagner and Bateman. +DUMMY/LJ030-0137.wav|The Assassination +DUMMY/LJ019-0164.wav|the personal superintendence of night officers, as already described, became possible. +DUMMY/LJ046-0160.wav|A case file was established if the information available suggested that the subject might be a danger to the President. +DUMMY/LJ009-0186.wav|Pennant speaks of Surgeons' Hall as a handsome building, ornamented with Ionic pilasters, and with a double flight of steps to the first floor. +DUMMY/LJ038-0119.wav|The arresting officers found a forged selective service card with a picture of Oswald and the name "Alek J. Hidell" +DUMMY/LJ012-0112.wav|notwithstanding a minute search through the room they had occupied. +DUMMY/LJ047-0054.wav|This is an administrative classification indicating that no further work has been scheduled. +DUMMY/LJ002-0010.wav|these numbers had risen to two hundred seventy-five and three hundred seventy-five respectively, or six hundred fifty in all. +DUMMY/LJ027-0032.wav|Internal organs may persist unchanged and hence they offer good guides to classification. +DUMMY/LJ003-0190.wav|Except in murder cases, where the execution was generally very promptly performed, +DUMMY/LJ032-0148.wav|fingerprint expert with the New York City Police Department, conducted an independent examination and also determined that this was the right palmprint of Oswald. +DUMMY/LJ043-0024.wav|Even though it appears that they may have left Oswald a few days before, it seems that he resisted the move as best he could. +DUMMY/LJ031-0091.wav|You have to determine which things, which are immediately life threatening and cope with them, before attempting to evaluate the full extent of the injuries. +DUMMY/LJ044-0074.wav|an agent of Bringuier's attempting to learn more about the true nature +DUMMY/LJ020-0008.wav|One tablespoonful of butter. +DUMMY/LJ028-0201.wav|"Thou art weighed in the balances and art found wanting." +DUMMY/LJ016-0140.wav|No report of it was made public, as the man was almost immediately recaptured. +DUMMY/LJ030-0016.wav|On the morning of November twenty-two, President Kennedy attended a breakfast at the hotel and afterward addressed a crowd at an open parking lot. +DUMMY/LJ014-0118.wav|Mrs. Manning was transferred to London and lodged in the Horsemonger Lane Jail, where her husband soon afterwards joined her. +DUMMY/LJ006-0098.wav|There was some provision of clothing, but it was quite insufficient, and nothing at all was given if prisoners had enough of their own to cover their nakedness. +DUMMY/LJ007-0195.wav|to give better light and ventilation; the construction of bed-places, three tiers high alongside the walls for males, two tiers for females; +DUMMY/LJ046-0112.wav|and requests to other agencies, Federal and local, for useful information. +DUMMY/LJ018-0021.wav|The facts of the murder and its object, robbery, were thus conclusively proved. +DUMMY/LJ023-0004.wav|For the many messages which have come to me after that speech, and which it is physically impossible to answer individually, +DUMMY/LJ016-0073.wav|Mr. Cope, the governor of Newgate, having been communicated with, proceeded to Winchester, where he at once identified Williams. +DUMMY/LJ028-0388.wav|The old walls were restored to a height sufficient to prevent the escape of the animals, and among the ruins the kings enjoyed their favorite sport. +DUMMY/LJ041-0184.wav|who might, quote, lead an expedition to some of these other islands and free them too, end quote. +DUMMY/LJ029-0093.wav|he was told that forty-five minutes had been allotted for a motorcade procession from Love Field to the luncheon site. +DUMMY/LJ014-0088.wav|This was on a Thursday evening. +DUMMY/LJ049-0082.wav|Had there been reason to believe that the assassination was the result of a conspiracy, Federal jurisdiction could have been asserted; +DUMMY/LJ023-0113.wav|reached the point as a nation where we must take action to save the Constitution from the Court and the Court from itself. +DUMMY/LJ021-0202.wav|the equally untenable theory that it is an interference with liberty to offer reasonable help when private enterprise is in need of help. +DUMMY/LJ019-0108.wav|The prisoners were to cultivate the land and raise sufficient produce for their own support. +DUMMY/LJ003-0014.wav|Quarreling among the debtors was not unfrequent. Blows were struck, and fights often ensued. +DUMMY/LJ012-0040.wav|at the price Solomons had paid for it, and it cost him about a hundred pounds to re-stock his shop with his own goods. +DUMMY/LJ040-0045.wav|In the Marines, he evidenced a strong conviction as to the correctness of Marxist doctrine, +DUMMY/LJ002-0235.wav|were places set apart for skittles, fives, and tennis, which strangers frequented as any other place of public amusement. +DUMMY/LJ012-0135.wav|The police were at once employed, and after infinite pains they discovered the person, one Moss, who had acted as the messenger. +DUMMY/LJ010-0209.wav|Louis and Amadeus among the captains; and Hercules, Neptune, and Mars among the lieutenants of the association. +DUMMY/LJ008-0176.wav|then the mob began to thin, and the streets were cleared by the city marshals and a number of constables. +DUMMY/LJ017-0032.wav|The first intimation conveyed to society of the new terror which threatened it was in the arrest and arraignment of William Palmer, +DUMMY/LJ044-0085.wav|The basic facts of the event were uncovered independently by William Stuckey, who arranged the debate, and Edward Butler, +DUMMY/LJ037-0059.wav|as the man she saw kill Officer Tippit. In evaluating Mrs. Markham's identification of Oswald, the Commission considered certain allegations +DUMMY/LJ041-0153.wav|at least in the unit to which he was assigned after his return from the Far East. +DUMMY/LJ039-0205.wav|five point six, and six point five seconds. +DUMMY/LJ023-0051.wav|It is the American people themselves who are in the driver's seat. +DUMMY/LJ008-0256.wav|and with blanched cheek, and sinking, we turned away from the scene. +DUMMY/LJ014-0005.wav|but too late to give substantial aid. +DUMMY/LJ001-0182.wav|the books so ornamented are amongst the most delightful works of art that have ever been produced. +DUMMY/LJ041-0032.wav|Oswald also became interested in the New Orleans Amateur Astronomy Association, an organization of high school students. +DUMMY/LJ022-0085.wav|I have set up a Division of Applications and Information +DUMMY/LJ044-0212.wav|It also dealt with the fear of the Castro regime that such a policy might result in its abandonment by the Soviet Union. +DUMMY/LJ014-0245.wav|It was the custom in this office to make the banker's passbook the basis of the entries in the company's ledgers. +DUMMY/LJ027-0007.wav|There are numberless similarities and correlations and surprising uniformities. +DUMMY/LJ009-0152.wav|The crowd outside Newgate on the day of execution has already been described; but there was also a select gathering of distinguished visitors within the jail. +DUMMY/LJ014-0037.wav|Soon afterwards Hocker heard cries of "murder," +DUMMY/LJ035-0204.wav|Oswald was seen in the vicinity of the southeast corner of the sixth floor approximately thirty-five minutes before the assassination +DUMMY/LJ015-0302.wav|who was at length taken in a coffee-shop near Oxford Street, under the name of Hopkins. He resisted at first, and denied his identity, +DUMMY/LJ035-0028.wav|to the northwest corner of the floor where Truly hoped to find one of the two freight elevators. +DUMMY/LJ003-0133.wav|prevented me from going inside but seldom, and I used to communicate with him through the bars from the passage. End quote. +DUMMY/LJ025-0080.wav|Finally amid the lowest forms of animal life the speck of gelatinous protoplasm, which constitutes the whole body, +DUMMY/LJ050-0109.wav|Since the assassination, the Service has recognized that these relationships must be far more formal +DUMMY/LJ033-0177.wav|On December one, nineteen sixty-three, a replica bag was made from materials found on that date in the shipping room. +DUMMY/LJ036-0022.wav|the transfer was conclusively identified as having been issued by Cecil J. McWatters, a busdriver for the Dallas Transit Co. +DUMMY/LJ006-0159.wav|The prisoners declared that they did not see him oftener than twice a week; +DUMMY/LJ030-0037.wav|are designed to provide protection while permitting large numbers of people to see the President. +DUMMY/LJ050-0275.wav|materially improve upon the procedures in effect at the time of President Kennedy's assassination and result in a substantial lessening of the danger. +DUMMY/LJ035-0158.wav|A blue jacket, later identified by Marina Oswald as her husband's, was subsequently found in the building, apparently left behind by Oswald. +DUMMY/LJ046-0068.wav|It helped him in his job here, he was able to come back here with a fresh view of many things. +DUMMY/LJ048-0208.wav|Agent Sorrels, riding in the lead car, did observe the Texas School Book Depository Building as he passed by, +DUMMY/LJ038-0016.wav|heard police sirens and then saw a man as he, quote, ducked into, end quote, the outer lobby space of the theatre near the ticket office. +DUMMY/LJ015-0026.wav|Moreover, the partners were sober, steady men, who paid unremitting attention to business. +DUMMY/LJ013-0170.wav|the fresh marks of violence upon the door had evidently been made inside, and not from outside; +DUMMY/LJ047-0104.wav|reluctant and actually as far as I was concerned, was completely evasive on them. End quote. +DUMMY/LJ021-0034.wav|Closely following this great constructive effort we have, through various federal agencies, +DUMMY/LJ015-0109.wav|and he discussed the latter with appetite, but begged that they would never give him curry again, as he did not like it. +DUMMY/LJ014-0337.wav|wishing to verify the value of warrants they held, and sending to Maltby and Co.'s wharf, found out half the truth. +DUMMY/LJ021-0122.wav|has been far less than in any previous, comparable period. +DUMMY/LJ036-0196.wav|Tippit patroled district Number seventy-eight in the Oak Cliff area of Dallas during daylight hours. +DUMMY/LJ042-0036.wav|that that government supported an economic system, quote, which exploits all its workers, end quote, +DUMMY/LJ050-0239.wav|From February eleven through June thirty, nineteen sixty-four, +DUMMY/LJ005-0100.wav|For this purpose it kept up an extensive correspondence with all parts of the kingdom, and circulated queries to be answered in detail, +DUMMY/LJ002-0332.wav|Sometimes rugs were urgently required and not forthcoming; +DUMMY/LJ016-0002.wav|The old notion always prevailed that Newgate was impregnable, so to speak, from within, +DUMMY/LJ021-0136.wav|We have passed through more than a year of education. +DUMMY/LJ035-0090.wav|possible delayed reaction to the shot, jostling with the crowd of people on the steps and scanning the area along Elm Street and the parkway. +DUMMY/LJ030-0065.wav|Governor Connally occupied the right jump seat, Mrs. Connally the left. +DUMMY/LJ032-0173.wav|The testimony of those who saw him after the assassination was inconclusive about the color of Oswald's shirt, +DUMMY/LJ028-0196.wav|Every child knows the story of "the writing of the hand on the wall." It was the night that Babylon fell. +DUMMY/LJ048-0183.wav|Admittedly, protective measures cannot ordinarily be taken with regard to all buildings along a motorcade route. +DUMMY/LJ032-0171.wav|and that this was the same shirt which Oswald wore on the morning of the assassination. +DUMMY/LJ015-0078.wav|who preferred to risk his future reputation and liberty to the present discomfort of living upon narrow means. +DUMMY/LJ015-0138.wav|His fault was generosity, +DUMMY/LJ037-0204.wav|was shipped on March twenty, nineteen sixty-three, to A. J. Hidell, Post Office Box two nine one five, Dallas, Texas. +DUMMY/LJ044-0093.wav|and was extremely believable before this. We thought the fellow could probably get quite a few members if he was really indeed serious about getting members. +DUMMY/LJ002-0257.wav|and a master of the ale-room, who kept this the scene of their revels clean, and saw that boiling water was provided for grog. +DUMMY/LJ046-0221.wav|where it will be made available to Secret Service headquarters in Washington. +DUMMY/LJ045-0069.wav|Katherine Ford, with whom Marina Oswald stayed during her separation from her husband in November of nineteen sixty-two, +DUMMY/LJ003-0070.wav|was frequently adjudged for so-called libels, or too out-spoken comments in print. +DUMMY/LJ045-0074.wav|In the letter Marina Oswald stated that her husband had changed a great deal and that she was very lonely in the United States. +DUMMY/LJ045-0149.wav|She said, quote, +DUMMY/LJ012-0009.wav|It turned out that he was really married to Mrs. Canning, +DUMMY/LJ001-0024.wav|But the first Bible actually dated (which also was printed at Maintz by Peter Schoeffer in the year fourteen sixty-two) +DUMMY/LJ008-0068.wav|But the fall apart and inwards of two leaves is considered superior. +DUMMY/LJ002-0143.wav|The commissioners who presided were, quote, little otherwise than self-elected +DUMMY/LJ028-0489.wav|Had the enemy of Babylon succeeded in breaking through the outer and inner defenses of the city the royal palace would have still been far from his reach. +DUMMY/LJ018-0102.wav|Besides his employers, a jeweler named Humphreys was in the "swim," at whose shop in Red Lion Square was discovered a quantity of base gold +DUMMY/LJ018-0038.wav|and Müller at the time of his capture was actually wearing Mr. Briggs' hat, cut down and somewhat altered. +DUMMY/LJ002-0064.wav|directly under the chapel, in which there were three cells, used either for the confinement of disorderly and refractory prisoners, +DUMMY/LJ040-0037.wav|His wife testified that he compared himself with great readers of history. +DUMMY/LJ032-0139.wav|Day, on the other hand, believed that sufficient traces of the print had been left on the rifle barrel, +DUMMY/LJ012-0176.wav|To obtain possession of this, Thurtell with his two associates resolved to kill him. +DUMMY/LJ016-0300.wav|Such too the scene remained with little change or respite till the old hangman (Calcraft) slunk again along the drop, +DUMMY/LJ014-0227.wav|One of these was Mobbs, who lived in the Minories, +DUMMY/LJ018-0270.wav|The assistant called with the jewels on approbation at a house specially hired for the purpose in the West End, +DUMMY/LJ025-0151.wav|And it must be allowed to be possible that further research may reveal the existence of something comparable to a nervous system in plants. +DUMMY/LJ015-0013.wav|He was found guilty, and sentenced to the seemingly inadequate punishment of four years' transportation. +DUMMY/LJ035-0171.wav|Barnett ran to the building, noted its name, and then returned to the corner. +DUMMY/LJ008-0178.wav|the bodies for identification, the wounded to hospitals, a cart-load of shoes, hats, petticoats, and fragments of wearing apparel were picked up. +DUMMY/LJ006-0234.wav|More cruel injuries were common enough, which did not result from honest hand-to-hand fights. +DUMMY/LJ023-0137.wav|Congress has provided financial security +DUMMY/LJ003-0018.wav|and discipline maintained by a system open to grave abuses, and which had the prescription of long usage, +DUMMY/LJ002-0102.wav|near it a grating through which the debtors receive their beer from the neighboring public-houses. +DUMMY/LJ043-0022.wav|it appears that De Mohrenschildt and his wife actually went to Oswald's apartment early in November of nineteen sixty-two +DUMMY/LJ008-0269.wav|Already the severity of our criminal code, and the number of capital felonies upon the statute book, had brought a reaction; +DUMMY/LJ019-0258.wav|Both, however, varied greatly in mechanism and in the amount of energy they called forth, +DUMMY/LJ048-0261.wav|a. Employees are strictly enjoined to refrain from the use of intoxicating liquor +DUMMY/LJ007-0029.wav|The condition of the capitally-convicted prisoners after sentence was still very disgraceful. The side they occupied, still known as the press-yard, +DUMMY/LJ028-0211.wav|He appointed Gobrias governor of Babylon. +DUMMY/LJ045-0041.wav|Oswald's diary reports that he married his wife shortly after his proposal of marriage to another girl had been rejected. +DUMMY/LJ006-0171.wav|He was satisfied to let matters run on as in the old times, he said in his own justification; with him what was, was right, +DUMMY/LJ008-0230.wav|fell with a strange unnatural sound upon the ear +DUMMY/LJ016-0194.wav|Calcraft thought no more of what had occurred till the next sessions at the Old Bailey, when the sheriffs sent for him, +DUMMY/LJ045-0019.wav|Could you advise me as to the general view we have on the American Civil Liberties Union? End quote. +DUMMY/LJ007-0030.wav|consisted of two dozen rooms and fifteen cells. In these various chambers, until just before the inspectors made their report, +DUMMY/LJ034-0095.wav|When the shots were fired, Brennan was in an excellent position to observe anyone in the window. He was sitting on a concrete wall +DUMMY/LJ027-0074.wav|Sometimes a piece is enlarged, sometimes diminished, or even becomes obsolete; sometimes several pieces are consolidated into one; +DUMMY/LJ020-0107.wav|The grand secret of keeping house well and without worry, lies in the art of packing and fitting different kinds of work and in picking up the minutes. +DUMMY/LJ049-0004.wav|The motorcade in Dallas. +DUMMY/LJ007-0164.wav|forbid the faintest shadow of a hope that in a soil so unfavorable for moral culture +DUMMY/LJ033-0111.wav|that the bag which Oswald was carrying was approximately twenty-seven or twenty-eight inches long, whereas the wooden stock of the rifle, which is its largest component, +DUMMY/LJ044-0213.wav|The October seven, nineteen sixty-three, issue of the Militant +DUMMY/LJ037-0074.wav|and her positive identification of Oswald at a police lineup, the Commission considers her testimony reliable. +DUMMY/LJ028-0316.wav|Then the doorkeepers, when they heard this, carried him at once before the Magistrates. +DUMMY/LJ021-0192.wav|We are not frightened by reactionary lawyers or political editors. +DUMMY/LJ016-0132.wav|he was still untried, but would almost certainly have escaped with a minor penalty, say of three or four months' imprisonment. +DUMMY/LJ016-0328.wav|Nevertheless, in the very next session +DUMMY/LJ032-0271.wav|(two) Oswald's palmprint was on the rifle in a position which shows that he had handled it while it was disassembled, +DUMMY/LJ009-0226.wav|gaming-tables were set up, cards were played under the gibbet, to the disturbance of the public peace and the annoyance of all decent people. +DUMMY/LJ003-0238.wav|and they were exacted to relieve a rich corporation from paying for the maintenance of their own prison. +DUMMY/LJ044-0176.wav|He went almost directly to the Cuban Embassy and applied for a visa to Cuba in transit to Russia. +DUMMY/LJ012-0141.wav|Moss presently turned approver, +DUMMY/LJ030-0130.wav|The spectators were still thickly congregated in front of the buildings which lined the east side of Houston Street, but the crowd thinned abruptly along Elm Street, +DUMMY/LJ015-0162.wav|Just after Robson's frauds had agitated the minds of all directors of companies, the chairman of the Great Northern (Mr. Denison) +DUMMY/LJ030-0088.wav|proceeded approximately two to three car lengths behind the President's follow-up car. +DUMMY/LJ013-0139.wav|and many witnesses were brought to attest to his previous good character, but he was found guilty and sentenced to seven years' transportation. +DUMMY/LJ047-0146.wav|that he had sought and obtained a passport on June twenty-five, nineteen sixty-three. +DUMMY/LJ011-0060.wav|He wrote afterwards to the effect that the moment he had offered himself, an unknown assassin came to aim a blow at him. +DUMMY/LJ047-0177.wav|As discussed in chapter six, she failed to report this to Agent Hosty +DUMMY/LJ028-0476.wav|The entire width of this inner defense was about fifty-five feet; its height is uncertain. +DUMMY/LJ002-0315.wav|But this weekly pittance did not go far when the debtors' side was crowded, as it often was; +DUMMY/LJ021-0204.wav|of regulating only to meet concrete needs -- a practice of courageous recognition of change. +DUMMY/LJ004-0154.wav|One hundred and fifty were lodged in a prison just capable of holding fifty-two. +DUMMY/LJ014-0097.wav|The Mannings had seen or heard nothing of him, of course. As O'Connor still did not turn up, the police after a couple of days returned to Minver Place. +DUMMY/LJ050-0141.wav|As a beginning step to improve liaison with local law enforcement officials, the Secret Service on August twenty-six, nineteen sixty-four, +DUMMY/LJ050-0240.wav|the Service had the advantage of nine thousand, five hundred hours of work by other enforcement agencies. +DUMMY/LJ036-0214.wav|Tippit stopped the man and called him to his car. +DUMMY/LJ013-0105.wav|Elder, the former, was soon apprehended at his house, but he evaded the law by hanging himself with his pocket-handkerchief. +DUMMY/LJ027-0013.wav|the mysterious semi-fluid network of protoplasm, which is, so far as is known, the physical basis of all life; +DUMMY/LJ026-0129.wav|Sooner or later the starch grains are changed into a kind of sugar (glucose), which, unlike starch, dissolves in the sap +DUMMY/LJ050-0200.wav|The Secret Service should utilize the personnel of other Federal law enforcement offices +DUMMY/LJ040-0117.wav|The short-lived stay with the Pics was terminated after an incident in which Lee allegedly pulled out a pocket knife during an argument +DUMMY/LJ036-0165.wav|The Greyhound Bus Station at Lamar and Jackson Streets, where Oswald entered Whaley's cab, +DUMMY/LJ016-0281.wav|Rain fell heavily at intervals, but did not thin the concourse. +DUMMY/LJ007-0206.wav|In endeavoring to arrest contamination, prisoners were more closely confined, and associated in smaller numbers; +DUMMY/LJ004-0150.wav|Matters were on much the same footing at St. Albans. +DUMMY/LJ047-0089.wav|inasmuch as the facts relating to Oswald's activities at that time did not warrant such action. +DUMMY/LJ014-0184.wav|The noise of the struggle was heard outside in the street, +DUMMY/LJ007-0223.wav|The prison officials appear to be on the side of the inspectors, to the great dissatisfaction of the corporation, who claimed the full allegiance and support of its servants. +DUMMY/LJ019-0127.wav|or the still more costly process of walling in the whole farm, would have greatly added to the charges of these establishments. +DUMMY/LJ010-0245.wav|one of the equerries. The Queen was untouched, and at first, it is said, hardly realized the danger she had escaped. +DUMMY/LJ049-0051.wav|Planning for motorcade contingencies. +DUMMY/LJ027-0004.wav|are grouped by Romanes and other writers on organic evolution under the heads of morphology, embryology, classification, +DUMMY/LJ028-0472.wav|Its outer part, about twelve feet in width, was protected with towers at intervals of sixty-five feet. +DUMMY/LJ020-0009.wav|Three cups of lukewarm water in which the potatoes were boiled, strained through a coarse cloth. +DUMMY/LJ031-0034.wav|The Governor was lifted onto a stretcher and taken into trauma room two. +DUMMY/LJ011-0117.wav|devoted its efforts first to a mitigation of the forgery statute, but could not immediately accomplish much. +DUMMY/LJ024-0119.wav|When before have you found them really at your side in your fights for progress? +DUMMY/LJ037-0157.wav|taken from Oswald. +DUMMY/LJ046-0159.wav|accumulated over a twenty-year period, some of which included more than one individual. +DUMMY/LJ042-0040.wav|He complained in his letter about segregation, +DUMMY/LJ012-0195.wav|and of these Bishop and Williams, who were guilty of many peculiar atrocities, ended their murderous careers in front of the debtors' door at Newgate. +DUMMY/LJ044-0003.wav|Lee Harvey Oswald: Background and Possible Motives, Part five. +DUMMY/LJ028-0408.wav|Some are Arabs of the same tribes which used to roam the desert in Nebuchadnezzar's days. Some are the children of the Hebrew exiles of old. +DUMMY/LJ016-0261.wav|simultaneously, of the five pirates, part of the mutinous crew of the 'Flowery Land,' was greater than on any previous occasion. +DUMMY/LJ029-0081.wav|The third possibility, the Trade Mart, a handsome new building with all the necessary facilities, presented security problems. It had numerous entrances, +DUMMY/LJ015-0304.wav|He then confessed that he was the redoubtable Jem Saward, or Jem the Penman, and was conveyed to a police-court, and thence to Newgate. +DUMMY/LJ013-0095.wav|Burgess' continued absence was soon noticed at the bank. +DUMMY/LJ001-0164.wav|which should be indications of a delicacy of material and manufacture which would of necessity increase its cost. +DUMMY/LJ045-0171.wav|What did he say to that? +DUMMY/LJ021-0088.wav|We have passed through the formative period of code making in the National Recovery Administration +DUMMY/LJ006-0092.wav|and under the scanty covering of a couple of dirty stable-rugs apiece. +DUMMY/LJ030-0072.wav|On previous occasions, the President had requested that, to the extent possible, these flanking motorcycles keep back from the sides of his car. +DUMMY/LJ018-0181.wav|and was perpetually in trouble. Commencing life as a resurrection man, when that trade failed through the change in the law, +DUMMY/LJ028-0024.wav|gazing down upon the great Euphrates winding along the valley beneath. +DUMMY/LJ025-0021.wav|yet none of these movements justify the ascription to plants of perception of will. +DUMMY/LJ014-0143.wav|On being removed to Newgate from the court Mrs. Manning became perfectly furious. +DUMMY/LJ012-0212.wav|The snatchers were detained, the police sent for, and arrest followed as a matter of course. +DUMMY/LJ014-0124.wav|Manning, who was dressed in black, appeared to be a heavy, bull-necked, repulsive-looking man, with a very fair complexion and light hair. +DUMMY/LJ027-0088.wav|Extensive comparison, on the contrary, shows them to be the same, although the essential identity is obscured by adaptive modifications. +DUMMY/LJ032-0185.wav|Moreover, the Commission found no reliable evidence that Oswald used the rifle at any time between September twenty-three, +DUMMY/LJ031-0006.wav|On receipt of the radio message from Kellerman to the lead car that the President had been hit, +DUMMY/LJ044-0048.wav|which occurred eight days after Oswald wrote the above letter to V. T. Lee. +DUMMY/LJ010-0317.wav|The bank began first to refuse our acceptances, and to destroy the credit of our house; the bank shall smart for it. +DUMMY/LJ015-0166.wav|"What do you know of our clerk?" asked Mr. Denison of his lordship. "Only that he is a capital fellow, who gives the best dinners and balls in town." +DUMMY/LJ007-0181.wav|A committee of aldermen was forthwith appointed to report upon the inspectors' report, +DUMMY/LJ031-0219.wav|but it was difficult to measure accurately because multiple crisscross fractures radiated from the large defect. +DUMMY/LJ045-0246.wav|He sought for himself a place in history -- a role as the "great man" who would be recognized as having been in advance of his times. +DUMMY/LJ017-0232.wav|The captain's brother, a passenger on board the 'Flowery Land,' was also stabbed to death and his body thrown overboard. +DUMMY/LJ003-0144.wav|the wards being furnished with barrack bedsteads, upon which each prisoner had the regulation allowance of sleeping room +DUMMY/LJ002-0252.wav|The inmates, who euphemistically styled themselves "collegians," +DUMMY/LJ030-0245.wav|I was pushed down by Agent Youngblood. +DUMMY/LJ010-0316.wav|I kept up the payments of the dividends, but made no entries of such payments in my books. +DUMMY/LJ028-0106.wav|His palace in Babylon was one of the world's largest buildings, but the walls with which he protected his palace and city were the wonder of the whole world. +DUMMY/LJ046-0020.wav|prepared by the Secret Service for the Secretary of the Treasury following the assassination. As a result of this study, +DUMMY/LJ008-0067.wav|The machinery in use in Dublin is much the same as that employed at many jails now-a-days. +DUMMY/LJ024-0122.wav|its meaning would depend upon the kind of justices who would be sitting on the Supreme Court bench. +DUMMY/LJ038-0160.wav|he asked me what were the people gathering around on the corner for and I told him that the President was supposed to pass that morning, +DUMMY/LJ035-0039.wav|This vestibule door is solid except for a small glass window in the upper part of the door. +DUMMY/LJ040-0156.wav|No finding of neurological impairment or psychotic mental changes could be made. +DUMMY/LJ038-0088.wav|The results were positive for the hands and negative for the right cheek. +DUMMY/LJ014-0183.wav|meaning to kill him and rifle the shop. +DUMMY/LJ009-0302.wav|Another man was hired, himself a convict, whose fees for self and wife were twelve guineas. +DUMMY/LJ018-0357.wav|Poisoning has still its victims. +DUMMY/LJ002-0109.wav|called the clerk of the papers, attends daily. End quote. +DUMMY/LJ040-0175.wav|through which they could not reach him, but that he preferred the veil to remain intact. +DUMMY/LJ049-0002.wav|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ031-0089.wav|considerable time which at this juncture was not available. +DUMMY/LJ033-0032.wav|In the Paines' garage, along with many other objects of a household character, +DUMMY/LJ016-0208.wav|The onus and responsibility of carrying out the sentence is personal to the sheriff. A good story is told illustrating this. +DUMMY/LJ046-0047.wav|Exposure of the President to public view through travel among the people of this country is a great and historic tradition of American life. +DUMMY/LJ033-0083.wav|Oswald gripped the bag in his right hand near the top, quote, It tapered like this as he hugged it in his hand. +DUMMY/LJ005-0146.wav|and was indeed destined to continue for another half century, in other words, until the introduction of the Prison Act of eighteen seventy-seven. +DUMMY/LJ046-0249.wav|about persons other than those who were obvious threats to the President. +DUMMY/LJ036-0205.wav|Tippit must have heard the description of the suspect wanted for the President's shooting; it was broadcast over channel one at twelve:forty-five p.m., +DUMMY/LJ019-0124.wav|thinking perhaps that prisoners so well disposed would cheerfully remain in jail of their own accord. +DUMMY/LJ049-0037.wav|In contrast, the Vice Presidential vehicle, although not specially designed for that purpose, +DUMMY/LJ039-0224.wav|show that he possessed ample capability to commit the assassination. +DUMMY/LJ048-0104.wav|There were far safer routes via freeways directly to the Trade Mart, +DUMMY/LJ028-0239.wav|where he did exactly what she had done formerly: he turned the Euphrates by a canal into the basin, which was then a marsh, +DUMMY/LJ036-0192.wav|Tippit's murder was recorded on the police radio tape at about one:sixteen p.m. +DUMMY/LJ047-0187.wav|It was then my plan to interview Marina Oswald in detail concerning both herself and her husband's background. Question: +DUMMY/LJ041-0026.wav|from which Oswald appeared to suffer. +DUMMY/LJ045-0021.wav|any of the personalities or groups epitomizing or representing the so-called right wing, +DUMMY/LJ047-0008.wav|and was investigating him at the time of the assassination. The Commission has taken the testimony of Bureau agents +DUMMY/LJ014-0056.wav|while in Ireland a wife dashed out her husband's brains with a hammer. +DUMMY/LJ033-0094.wav|As they sat in the car, Frazier asked Oswald where his lunch was, and Oswald replied that he was going to buy his lunch that day. +DUMMY/LJ045-0119.wav|which had led to the disclosure of his defection in New Orleans. +DUMMY/LJ050-0175.wav|Since the assassination, Secret Service procedures have been changed to require that a member of PRS accompany each advance survey team +DUMMY/LJ004-0061.wav|by want of bed-clothing by night or firing by day +DUMMY/LJ032-0227.wav|Shaneyfelt testified that the published photographs appeared to be based on a copy of the original which the publications had each retouched differently. +DUMMY/LJ047-0087.wav|According to the Bureau, quote, +DUMMY/LJ028-0225.wav|A much longer account of the capture of Babylon by Cyrus appears in the writings of Herodotus. +DUMMY/LJ028-0457.wav|The moat, ten feet deep, and of a width no longer known, ran close to its base. The wall was double. +DUMMY/LJ028-0140.wav|The city stands on a broad plain, and is an exact square, a hundred and twenty furlongs in length each way, +DUMMY/LJ024-0127.wav|You who know me can have no fear that I would tolerate the destruction by any branch of government of any part of our heritage of freedom. +DUMMY/LJ014-0078.wav|that she stood behind him as he stood near the open grave she herself had dug for him, +DUMMY/LJ007-0065.wav|on the one hand, the Ladies' Association, supported and encouraged by public approval, trenched upon his peculiar province; +DUMMY/LJ005-0233.wav|There was no instruction of adult prisoners, in accordance with the law. The sleeping accommodation was still altogether contrary to the latest ideas. +DUMMY/LJ010-0053.wav|and when the people were beginning to agitate for a larger share of political power, was among the earliest, and to some extent the most desperate, of these. +DUMMY/LJ022-0100.wav|Moreover, this Division will be charged with keeping projects moving on schedule. +DUMMY/LJ037-0143.wav|Two of the arresting officers placed their initials on the weapon and a third inscribed his name. +DUMMY/LJ039-0105.wav|Following that training, Oswald was tested in December of nineteen fifty-six, and obtained a score of two hundred twelve, +DUMMY/LJ028-0009.wav|At first, paying little heed to you, she hesitates and glances anxiously about the desert as if in search of an enemy. +DUMMY/LJ009-0295.wav|who had been a convicted prisoner at York, but who consented to act as hangman when Calcraft was engaged, and no other functionary could be obtained. +DUMMY/LJ028-0048.wav|In six twenty-six Assurbanipal died, +DUMMY/LJ006-0293.wav|and the men could also at any time go for tea, coffee, and sugar to Mrs. Brown's shop, which was inside the female gate. +DUMMY/LJ012-0174.wav|Thurtell was a gambler, and Weare had won a good deal of money from him. +DUMMY/LJ016-0365.wav|Moreover, after the execution, under the old system, the latter had only to receive the body for burial after it had been cut down by the hangman, +DUMMY/LJ008-0282.wav|seventeen capital convicts, upon all of whom the sentence of death had been passed. Eventually two only of the whole number suffered; +DUMMY/LJ028-0484.wav|Grant me as a gift a life of distant days. +DUMMY/LJ014-0280.wav|But it was proved that Watts had appropriated one cheque for fourteen hundred pounds, +DUMMY/LJ045-0130.wav|warned me that if I engaged in F.P.C.C. activities in Texas the F.B.I. will again take an interest in me, end quote. +DUMMY/LJ036-0157.wav|Oswald could not possibly have been wearing the blue jacket during the trip with Whaley, since it was found in the "domino" room of the Depository late in November. +DUMMY/LJ009-0038.wav|In the center of the chapel was the condemned pew, a large dock-like erection painted black. +DUMMY/LJ049-0013.wav|and on one occasion Agent Kellerman left the front seat of the President's car to take a similar position. +DUMMY/LJ028-0301.wav|Now, therefore, if there be no failure on thy part, we shall take Babylon. +DUMMY/LJ029-0117.wav|According to Lawson, the chosen route seemed to be the best. +DUMMY/LJ030-0027.wav|Vice President Johnson's airplane, Air Force Two, had arrived at Love Field at approximately eleven:thirty-five a.m., +DUMMY/LJ028-0325.wav|They were ready, therefore, to grant him anything he asked; +DUMMY/LJ028-0303.wav|I think they will believe my words and entrust me with a command of troops. Thou, on thy part, must wait +DUMMY/LJ011-0243.wav|Mr. Mullay went, and a second interview was agreed upon, when a third person, Mr. Owen, +DUMMY/LJ035-0112.wav|The west elevator was not on the fifth floor when Baker and Truly reached that floor, +DUMMY/LJ016-0121.wav|Krapps walked at once into the yard and across to the female side, where he found some of the washing still hanging out to dry. +DUMMY/LJ039-0141.wav|he again went hunting with his brother, Robert, and used a borrowed twenty-two caliber bolt-action rifle. +DUMMY/LJ021-0105.wav|and the vital necessity of improving labor conditions, +DUMMY/LJ020-0083.wav|prick each with a fork and let them rise near the fire for half an hour, covered with a light cloth. +DUMMY/LJ036-0020.wav|The transfer was dated "Friday November twenty-two, 'sixty-three" and was punched in two places by the bus driver. +DUMMY/LJ022-0124.wav|There are chiselers in every walk of life; there are those in every industry who are guilty of unfair practices; +DUMMY/LJ015-0202.wav|The precious metal was sent from place to place carefully locked up and guarded, no doubt; +DUMMY/LJ003-0106.wav|The first night in Newgate, and for the subsequent fortnight, +DUMMY/LJ011-0033.wav|deeming him the last person capable of a dishonorable action. +DUMMY/LJ007-0115.wav|slept there, and had their meals provided for them from the Sessions House or London Coffee-House. +DUMMY/LJ018-0285.wav|With this as capital they came back to England via Buenos Aires, +DUMMY/LJ037-0188.wav|Among these guns was a thirty-eight Special caliber Smith and Wesson revolver, serial Number V five one zero two one zero, +DUMMY/LJ009-0169.wav|Within a couple of years the number of persons sentenced to death in England had fallen from four hundred and thirty-eight in eighteen thirty-seven +DUMMY/LJ013-0097.wav|as soon as that gentleman was referred to. +DUMMY/LJ007-0089.wav|It was manifestly wrong that such persons, "visited by the most awful of calamities," should be detained in a common prison. +DUMMY/LJ018-0160.wav|They next watched Buncher, and found that he paid frequent visits to Birmingham. +DUMMY/LJ049-0023.wav|Since the assassination, the Secret Service, with the assistance of other Federal agencies and of private industry, +DUMMY/LJ016-0099.wav|These men managed to cut a hole in the ceiling of the ward near the iron cage on the landing, and so got access to the roof. +DUMMY/LJ008-0250.wav|and lay disregarded beside the body of the poor dog which, while searching for its master in the crowd, +DUMMY/LJ035-0169.wav|testified that immediately after the shots he went to the rear of the building to check the fire escape. +DUMMY/LJ012-0122.wav|The condign punishment meted out to these Custom House robbers had no deterrent effect seemingly. +DUMMY/LJ025-0010.wav|One of the most important summaries of this similarity +DUMMY/LJ013-0167.wav|The discovery of the murdered man immediately followed. The neighborhood was alarmed, the police sent for, and a close inquiry forthwith commenced. +DUMMY/LJ049-0129.wav|while the CIA has the primary responsibility for collecting intelligence overseas to supplement information acquired by the Department of State. +DUMMY/LJ013-0250.wav|In the same room a large axe and saw were found covered with blood. +DUMMY/LJ010-0069.wav|She died early, and Thistlewood, left to his own resources, +DUMMY/LJ008-0313.wav|who in solemn tones communicated to each in turn the fate in store for him. +DUMMY/LJ013-0134.wav|and it was contended that these with the files were used to alter the marks on the plate. +DUMMY/LJ038-0105.wav|They discovered two photographs, each showing Oswald with a rifle and a pistol. +DUMMY/LJ013-0169.wav|It was also clearly proved that no forcible entry had been made into the house; +DUMMY/LJ018-0397.wav|out of large sums in sham sporting speculations. The conviction of the principals in this nefarious transaction, +DUMMY/LJ030-0067.wav|on his right sat Kellerman. +DUMMY/LJ042-0144.wav|After the assassination she wrote that, quote, +DUMMY/LJ019-0195.wav|A proviso was also made that the magistrates should continue to exercise full control over the new jail, +DUMMY/LJ044-0027.wav|to her husband's membership card in the New Orleans chapter. +DUMMY/LJ003-0268.wav|This delectable society enticed into their set a clerk who had been imprisoned for fraud, +DUMMY/LJ031-0096.wav|Question: Was any effort made to inspect the President's back after he had expired? Answer: No, sir. +DUMMY/LJ023-0048.wav|Two of the horses are pulling in unison today; the third is not. +DUMMY/LJ002-0077.wav|from twenty-one by eighteen feet to fifteen feet square, which were furnished with bedsteads and bedding. +DUMMY/LJ038-0077.wav|Statements of Oswald during Detention. +DUMMY/LJ009-0165.wav|House-breaking, as distinguished from burglary, was similarly exempted in the following year; +DUMMY/LJ026-0043.wav|and hence so unfamiliar that to most people the comparison made will mean little in terms of ordinary green flowering plants and common vertebrate animals. +DUMMY/LJ035-0044.wav|If the man had passed from the vestibule into the lunchroom, Baker could not have seen him. +DUMMY/LJ048-0226.wav|All of the agents stayed for a drink of beer, or in several cases, a mixed drink. +DUMMY/LJ008-0316.wav|"A. B., your case has been taken into consideration by the king in council, and His Majesty has been mercifully pleased to spare your life." +DUMMY/LJ009-0198.wav|and her body publicly exhibited in a place built for the purpose in the Old Bailey. +DUMMY/LJ033-0078.wav|Neither she nor Mrs. Paine saw him leave the house. About half-a-block away from the Paine house was the residence of Mrs. Linnie Mae Randle, +DUMMY/LJ035-0087.wav|Baker required fifteen seconds after the simulated shot to ride his motorcycle one hundred eighty to two hundred feet, +DUMMY/LJ029-0068.wav|The adequacy of the intelligence system maintained by the Secret Service at the time of the assassination, +DUMMY/LJ022-0153.wav|legislation to provide for the elimination of unnecessary holding companies in the public utility field. +DUMMY/LJ017-0058.wav|He was confident to the very last that it would be impossible to find him guilty; +DUMMY/LJ021-0054.wav|that the way to wealth is through work. +DUMMY/LJ032-0123.wav|the rifle was released to the FBI and forwarded to Washington where it was examined on the morning of November twenty-three +DUMMY/LJ028-0391.wav|that there is a royal hunting ground at Babylon, and that wild game of every kind is contained within the circuit of its walls. +DUMMY/LJ028-0482.wav|but I, the devout petitioner, the worshipper of the gods, built the moat, and made its wall of burned brick and bitumen mountain high. +DUMMY/LJ011-0208.wav|On this Wakefield gave in. +DUMMY/LJ032-0192.wav|The rifle used in the assassination probably had been wrapped in a blanket for about eight weeks prior to November twenty-two. +DUMMY/LJ009-0292.wav|He was well known in the neighborhood where he resided, and the street gamins cried "Jack Ketch" as he went along the street. +DUMMY/LJ027-0057.wav|In all vertebrates, and in none other, the axis of this skeleton is a jointed backbone (vertebral column) +DUMMY/LJ011-0254.wav|A life and death struggle ensued. Mullay cried "Murder!" +DUMMY/LJ018-0245.wav|in all good faith to another, but a criminal member of the family. +DUMMY/LJ044-0234.wav|one hundred seventy dollars in a wallet in his wife's room in Irving. +DUMMY/LJ032-0258.wav|Ruth and Michael Paine both noticed the rolled-up blanket in the garage during the time that Marina Oswald was living in their home. +DUMMY/LJ010-0136.wav|He told the executioner to "do it tidy," to pull it tight, and was in a state of hysterical exaltation up to the very last. +DUMMY/LJ017-0029.wav|The crime of poisoning is essentially one which will be most prevalent in a high state of civilization, +DUMMY/LJ046-0139.wav|to refer all communications on identified existing cases and, in addition, +DUMMY/LJ048-0079.wav|mutual understanding of FBI and agency jurisdictions, and an indicated willingness by the agency representative +DUMMY/LJ027-0124.wav|For instance, although in nearly all the numerous species of snakes there are no vestiges of limbs, +DUMMY/LJ011-0133.wav|This amendment was strongly supported outside the House, and a petition in favor of its passing was presented, +DUMMY/LJ050-0060.wav|Subversives, ultrarightists, racists and fascists (a) possessing emotional instability or irrational behavior, +DUMMY/LJ009-0243.wav|Other cases might be quoted, +DUMMY/LJ049-0128.wav|The FBI is the major domestic investigating agency of the United States, +DUMMY/LJ011-0173.wav|Miss Turner consented to go on, and they traveled night and day towards the north. +DUMMY/LJ034-0211.wav|who was standing on the south side of Elm Street between the Triple Underpass and the Depository Building. +DUMMY/LJ041-0035.wav|and then came out with a statement that he was looking for a Communist cell in town to join but he couldn't find any. +DUMMY/LJ033-0071.wav|And I then translated this to the officers that she knew that her husband had a gun that he had stored in here. +DUMMY/LJ010-0161.wav|two years after she ascended the throne. +DUMMY/LJ008-0246.wav|Never once from that pale face passed away its benign and patient expression. +DUMMY/LJ015-0141.wav|from which he rose to be assistant registrar, with the special duties of transferring shares. +DUMMY/LJ032-0127.wav|the formations, the ridge formations and characteristics, were insufficient for purposes of either effecting identification +DUMMY/LJ013-0123.wav|The club spoons and other articles of plate were long a source of profitable income to a gentleman named Ashley, +DUMMY/LJ018-0304.wav|They had omitted to insert the dates in certain bills. +DUMMY/LJ030-0057.wav|If the weather is clear and it is not raining, have that bubbletop off, end quote, +DUMMY/LJ018-0180.wav|But he was not long at large; he was too active an evil-doer +DUMMY/LJ035-0099.wav|Baker, not realizing that there were two elevators, thought that only one elevator was in the shaft and that it was two or three floors above the second floor. +DUMMY/LJ021-0082.wav|lines which as I have so often made clear, are in complete accord with the underlying principles of orderly popular government +DUMMY/LJ031-0156.wav|The Secret Service group awaiting the President in Austin were instructed to return to Washington. +DUMMY/LJ032-0014.wav|(six) the lies told to the police by Oswald, (seven) +DUMMY/LJ006-0254.wav|Other regulations laid down by the Jail Acts were still defied. One of these was that prisoners should be restricted to the jail allowance of food; +DUMMY/LJ035-0184.wav|He then entered the building and took the front passenger elevator as far as it would go -- the fourth floor. +DUMMY/LJ048-0170.wav|While certain streets thought to be too narrow could be avoided and other choices made, +DUMMY/LJ028-0204.wav|It refers to the death of the King's son, possibly to Belshazzar of the Bible story. +DUMMY/LJ026-0022.wav|if chlorophyll is absent, carbon is obtained from sugar or some similar compound, +DUMMY/LJ049-0074.wav|Their actions demonstrate that the President and the Nation can expect courage and devotion to duty from the agents of the Secret Service. +DUMMY/LJ009-0054.wav|He is quite content to die; +DUMMY/LJ049-0096.wav|There have been a number of efforts to make assassination a Federal crime, particularly after the assassination of President McKinley +DUMMY/LJ012-0265.wav|A woman named Gale, who lived with him, was arrested at the same time. The prisoners were examined at the Marylebone police court. +DUMMY/LJ048-0081.wav|Each field office should determine those Federal agencies which are represented locally and with which liaison should be conducted. End quote. +DUMMY/LJ009-0163.wav|In the same year capital punishment was further restricted, and ceased to be the legal sentence for coining, +DUMMY/LJ034-0077.wav|Oswald said, quote, No, sir. When you get downstairs, close the gate to the elevator, end quote. +DUMMY/LJ003-0332.wav|A certain number of bedsteads were provided, and there was a slight increase in the ration of bread. +DUMMY/LJ045-0231.wav|two officers have testified that at the time of his arrest Oswald said something to the effect that, quote, it's all over now, end quote. +DUMMY/LJ040-0080.wav|That situation, however, was short-lived, +DUMMY/LJ049-0067.wav|the radio net in use in motorcades is elaborate and permits a number of different means of communication with various local points. +DUMMY/LJ027-0016.wav|The same laws of heredity, variability and of response to outside stimulus hold in all parts of the organic world. +DUMMY/LJ049-0113.wav|This point was ably made in the nineteen oh two debate by Senator George F. Hoar, the sponsor of the Senate bill, quote, +DUMMY/LJ041-0196.wav|even though as his wife suggested, that interest may have been motivated by a desire to gain attention. He used his Marxist and associated activities +DUMMY/LJ019-0239.wav|and others were subsequently added. +DUMMY/LJ016-0256.wav|It was thought that capital punishment would lose its deterrent effect if it ceased to be public, +DUMMY/LJ018-0147.wav|The next step was to take the principals, and under such circumstances as would insure their conviction. +DUMMY/LJ014-0003.wav|The cries of his victim, a Mr. Delarue, +DUMMY/LJ004-0141.wav|The hospital was filled with infectious cases, and in one room, seven feet by nine, with closed windows, +DUMMY/LJ005-0136.wav|The want of sleeping cells long continued a crying need. +DUMMY/LJ002-0280.wav|It was generally crowded, as debtors who would have gone to the Poultry Compter were sent to Giltspur Street when the former was condemned as unfit to receive prisoners. +DUMMY/LJ030-0176.wav|In his testimony before the Commission, Governor Connally was certain that he was hit by the second shot, which he stated he did not hear. +DUMMY/LJ042-0002.wav|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter seven. Lee Harvey Oswald: +DUMMY/LJ047-0072.wav|stating that he had distributed its pamphlets on the streets of Dallas. This information did not reach Agent Hosty in Dallas until June. +DUMMY/LJ004-0092.wav|A much later act, the fourteen George the third c. fifty-nine (seventeen seventy-four), +DUMMY/LJ032-0149.wav|Latona's findings were also confirmed by Ronald G. Wittmus, another FBI fingerprint expert. +DUMMY/LJ041-0011.wav|Several witnesses testified that Lee Oswald was not aggressive. He was, however, involved in some fights. +DUMMY/LJ007-0148.wav|prefers that description which are expressly prepared for his own class, and which abound in vulgar adventure in criminal enterprise, and in the histories of the police, +DUMMY/LJ005-0016.wav|The Society for the Improvement of Prison Discipline was taxed with a desire to introduce a system +DUMMY/LJ001-0056.wav|It was these great Venetian printers, together with their brethren of Rome, Milan, +DUMMY/LJ014-0187.wav|Lerigo, was also attracted by the noise of the row, but after walking a few paces he felt dissatisfied, and returned to the spot. +DUMMY/LJ027-0092.wav|A man having made a steam-engine, and desiring to use it for a different purpose from that for which it was first designed and used, +DUMMY/LJ004-0108.wav|which was the first legislative attempt to compel the classification of prisoners, or their separation into classes +DUMMY/LJ024-0046.wav|then I say that I and with me the vast majority of the American people favor doing just that thing -- now. +DUMMY/LJ038-0276.wav|In explaining the difference between his policy and that of the FBI on the matter of probable identification, Nicol said, quote, +DUMMY/LJ044-0222.wav|Oswald was asked during the New Orleans radio debate in which he engaged on August twenty-one, nineteen sixty-three, +DUMMY/LJ028-0177.wav|However, they teach us that in those early days when most cities were surrounded by enormous walls, +DUMMY/LJ011-0178.wav|But another bank had since failed, and nothing could save Mr. Turner but the transfer of some property to Miss Turner, and its settlement on her, +DUMMY/LJ002-0036.wav|one. The male debtors' side consisted of a yard forty-nine feet by thirty-one, +DUMMY/LJ017-0256.wav|The mutineers were arrested, the case inquired into by a naval court-martial, +DUMMY/LJ017-0263.wav|the remaining five were executed in one batch. +DUMMY/LJ030-0248.wav|Clifton C. Carter, riding in the Vice President's follow-up car a short distance behind, +DUMMY/LJ002-0224.wav|but a number of rooms, fifteen more or less, were reserved for poor debtors under the name of Bartholomew Fair. +DUMMY/LJ042-0182.wav|and they would never accept a new order complete beyond their understanding, end quote. +DUMMY/LJ043-0139.wav|He outlined his plans in a notebook and studied them at considerable length before his attack. +DUMMY/LJ039-0004.wav|Chapter four. The Assassin: Part eight. +DUMMY/LJ038-0212.wav|which appeared to be the work of a man expecting to be killed, or imprisoned, or to disappear. +DUMMY/LJ008-0273.wav|At the Old Bailey almost every one capitally convicted by a jury was sentenced to be hanged. +DUMMY/LJ037-0054.wav|She identified Lee Harvey Oswald as the man who shot the policeman. +DUMMY/LJ002-0108.wav|watch all night. Adjoining the felons' side lodge is the keeper's office, where the prison books are kept, and his clerk, +DUMMY/LJ037-0221.wav|He was wearing a zipper jacket which he had not been wearing moments before when he had arrived home. +DUMMY/LJ041-0168.wav|even though Oswald seemed to be lost in his own thoughts. After a brief period of silence Oswald remarked on the stupidity of the parade +DUMMY/LJ009-0242.wav|and the sentence was properly and completely carried out. +DUMMY/LJ019-0250.wav|In many prisons the prisoners were still associated, and, from the want of a sufficient number of cells, the principle of separation was still greatly neglected. +DUMMY/LJ022-0129.wav|in making this the most efficient and the cleanest example of public enterprise the world has ever seen. +DUMMY/LJ019-0308.wav|the penultimate of such enactments, many of the provisions of which still remain in force. +DUMMY/LJ038-0166.wav|an active and controversial figure on the American political scene since his resignation from the U.S. Army in nineteen sixty-one +DUMMY/LJ043-0130.wav|and had been seen reading a foreign newspaper, end quote. +DUMMY/LJ031-0103.wav|While Dr. Carrico went on to attend the President, Dr. Dulany stayed with the Governor and was soon joined by several other doctors. +DUMMY/LJ015-0227.wav|Still watching and waiting for the first chance, they seized it when the clerks left the office empty for a moment. +DUMMY/LJ009-0278.wav|Bousfield slowly drew himself up, and rested with his feet on the right side of the drop. +DUMMY/LJ038-0080.wav|Throughout this interrogation he denied that he had anything to do either with the assassination of President Kennedy or the murder of Patrolman Tippit. +DUMMY/LJ024-0080.wav|Our difficulty with the Court today rises not from the Court as an institution but from human beings within it. +DUMMY/LJ050-0018.wav|and to keep the Secretary fully informed regarding all significant developments relating to Presidential protection. +DUMMY/LJ019-0081.wav|The use of gas or some other means of lighting might be adopted, but more often was dispensed with. +DUMMY/LJ038-0174.wav|Their results were negative. +DUMMY/LJ040-0220.wav|could have led anyone to predict the outburst of violence which finally occurred. +DUMMY/LJ025-0061.wav|Agardh and other of the botanists of Cuvier's generation who occupied themselves with the lower plants had observed that, +DUMMY/LJ041-0113.wav|Oswald was transferred from active duty to the Marine Corps Reserve under honorable conditions in September of nineteen fifty-nine, +DUMMY/LJ004-0022.wav|The executive, although harassed by internal commotion and foreign war, was not entirely callous to the crying need for amelioration in jails. +DUMMY/LJ017-0260.wav|Vartos the Turk, and Carlos the Greek. +DUMMY/LJ008-0087.wav|The change from Tyburn to the Old Bailey had worked no improvement as regards the gathering together of the crowd or its demeanor. +DUMMY/LJ015-0300.wav|Saward's letter to Hardwicke fell into the hands of the police and compromised him. +DUMMY/LJ029-0067.wav|nor did PRS develop any additional information between November twelve, when Lawson left Washington, and November twenty-two. +DUMMY/LJ045-0174.wav|Thank you. That it would be better if he bought something for himself -- that I would manage. End quote. That night Oswald went to bed before his wife retired. +DUMMY/LJ047-0011.wav|and the Assistant to the Director in charge of all investigative activities under the Director and Associate Director. In addition, +DUMMY/LJ041-0131.wav|and also because he had not received any notice of the original discharge proceedings, since his whereabouts were not known. +DUMMY/LJ016-0096.wav|They were penal servitude men, their names Bell, Brown, and Barry, and they were awaiting transfer to Leicester, +DUMMY/LJ007-0184.wav|The inspectors retorted without loss of time, reiterating their charges, and pointing out that the committee of aldermen by its own admission +DUMMY/LJ030-0120.wav|had to leave the left front running board of the President's follow-up car four times to ride on the rear of the President's limousine. +DUMMY/LJ044-0123.wav|Stating that he had used his "position" with what he claimed to be the local branch of the Fair Play for Cuba Committee to, quote, foster communist ideals, end quote, +DUMMY/LJ031-0228.wav|concluded that the bullet had emerged from the front portion of the President's neck that had been cut away by the tracheotomy at Parkland. +DUMMY/LJ031-0010.wav|Chief Curry radioed, quote, Go to the hospital -- Parkland Hospital. Have them stand by, end quote, +DUMMY/LJ004-0161.wav|"the indescribable stench," presented together a concentration of the utmost misery and the utmost guilt. +DUMMY/LJ028-0479.wav|Nebuchadnezzar, King of Babylon, +DUMMY/LJ001-0136.wav|modern printers understand this, but it is only practiced in the very best establishments. +DUMMY/LJ042-0147.wav|Lee became very irritable, and sometimes some completely trivial thing would drive him into a rage. +DUMMY/LJ034-0187.wav|Fischer described the man's hair as some shade of brown, quote, it wasn't dark, and it wasn't light, end quote. +DUMMY/LJ037-0064.wav|that Mrs. Markham strongly reaffirmed her positive identification of Oswald and denied having described the killer +DUMMY/LJ008-0304.wav|to utter imprecations against the Recorder, the Secretary of State, the Council, and the King; +DUMMY/LJ035-0083.wav|Howlett was not short winded at the end of either test run. +DUMMY/LJ043-0007.wav|Somewhat of a nonconformist, +DUMMY/LJ039-0101.wav|He went through a series of exercises called dry firing where he assumed all positions which would later be used in the qualification course. +DUMMY/LJ045-0097.wav|whom she compared to, quote, a puppy dog that everybody kicked, end quote, had a lot of good qualities, in spite of the fact that, quote, +DUMMY/LJ038-0217.wav|indicated that the note was written when they were living in a rented apartment; therefore it could not have been written while Marina Oswald was living with the Paines. +DUMMY/LJ044-0102.wav|samples of his photographic work, offering to contribute that sort of service without charge. +DUMMY/LJ004-0114.wav|King's evidences were also to be lodged apart. +DUMMY/LJ016-0257.wav|and the raison d'être of the penalty, which in principle so many opposed, would be gone. +DUMMY/LJ028-0130.wav|And after he had walled the city, and adorned its gates, he built another palace before his father's palace; but so that they joined to it: +DUMMY/LJ019-0017.wav|This building was a costly affair. The site was uneven, and had to be leveled; +DUMMY/LJ043-0021.wav|While the exact sequence of events is not clear because of conflicting testimony, +DUMMY/LJ028-0188.wav|In five thirty-eight the city fell, and for a time it became the home of the Persian King. +DUMMY/LJ005-0004.wav|unless they could afford to pay for a vehicle out of their own funds. +DUMMY/LJ019-0172.wav|prisoners who had been acquitted were not permitted to return to the prison to show their joy and receive the congratulations of their unfortunate fellows. +DUMMY/LJ004-0204.wav|Masons, bricklayers, carpenters, painters had been employed upon the buildings, and the work was pronounced excellent by competent judges. +DUMMY/LJ023-0052.wav|It is the American people themselves who want the furrow plowed. +DUMMY/LJ003-0019.wav|and which was never wholly rooted out for many years to come. +DUMMY/LJ015-0206.wav|He laid the suggestion before Agar, a professional thief, who was of opinion it contained elements of success. +DUMMY/LJ014-0011.wav|Even then he followed it as far as Belsize Lane. +DUMMY/LJ039-0239.wav|would include the President's head. +DUMMY/LJ045-0147.wav|She asked Oswald, quote, +DUMMY/LJ036-0177.wav|knew Lee Harvey Oswald under the alias of O. H. Lee. +DUMMY/LJ018-0087.wav|Several cases of gigantic fraud, rivaling any already recorded, were brought to light between eighteen fifty-six and eighteen seventy-three. +DUMMY/LJ045-0179.wav|He took with him thirteen dollars, eighty-seven cents and the long brown package that Frazier and Mrs. Randle saw him carry +DUMMY/LJ001-0074.wav|the best, mostly French or Low-Country, was neat and clear, but without any distinction; +DUMMY/LJ022-0043.wav|I shall not confuse my discussion by a multitude of figures. So many figures are quoted to prove so many things. +DUMMY/LJ038-0043.wav|The other officers who helped subdue Oswald corroborated McDonald in his testimony +DUMMY/LJ034-0037.wav|Someone sitting on the box facing the window would have his palm in this position if he placed his hand alongside his right hip. +DUMMY/LJ001-0111.wav|the ends of many of the letters such as the t and e are hooked up in a vulgar and meaningless way, +DUMMY/LJ017-0022.wav|was that of Eliza Fenning, who was convicted of an attempt to poison a whole family +DUMMY/LJ026-0094.wav|This, however, is probably not a source of vital energy, but only contributes to the maintenance of the body temperature. +DUMMY/LJ045-0095.wav|She said that if she ever spoke to her husband the way Marina Oswald spoke to her husband, quote, we would not last long, end quote. +DUMMY/LJ012-0293.wav|But the medical evidence was clear that the decapitation had been effected during life, and the jury, after a short deliberation, +DUMMY/LJ050-0272.wav|and the traditions of the office in a democracy such as ours are so deep-seated as to preclude absolute security. +DUMMY/LJ042-0222.wav|Judged by his other statements and writings, however, he appears to have indicated his true feelings in the set of answers first presented +DUMMY/LJ016-0078.wav|but had to come down again covered with soot and filth just as the officers entered the ward. +DUMMY/LJ045-0094.wav|Jeanne De Mohrenschildt, however, thought that Marina Oswald, quote, said things that will hurt men's pride, end quote. +DUMMY/LJ007-0168.wav|"We cannot close these remarks," say the inspectors, "without an expression of the painful feelings with which we submit to your Lordship +DUMMY/LJ005-0212.wav|and the prison allowance was still limited to bread and water. +DUMMY/LJ012-0116.wav|A fire was lit in it, and the maid in doing so threw a lot of rubbish, as it seemed, which had accumulated under the grate, on top of the burning coals. +DUMMY/LJ049-0151.wav|Responsibilities for Presidential Protection +DUMMY/LJ043-0123.wav|He consulted with his supervisor and, quote, it was determined that we should not undertake to furnish employment references for him, end quote. +DUMMY/LJ005-0289.wav|In these the commissioners found "additional proof of the evils of continuing the present constitution of the local tribunals. +DUMMY/LJ038-0008.wav|Brewer knew from radio broadcasts that the President had been shot and that a patrolman had also been shot in Oak Cliff. +DUMMY/LJ046-0177.wav|which is maintained on a geographical field office basis. +DUMMY/LJ042-0132.wav|Return to the United States. In view of the intensity of his earlier commitment to the Soviet Union, +DUMMY/LJ006-0148.wav|Probably authority would not have been so recklessly usurped by the wardsmen had not the proper officials too readily surrendered it. +DUMMY/LJ041-0088.wav|which he strove to maintain and, quote, felt the Marine Corps kept a pretty close watch on him because of his subversive activities, end quote. +DUMMY/LJ045-0024.wav|to shoot General Walker. +DUMMY/LJ033-0150.wav|Sebastian F. Latona, supervisor of the FBI's Latent Fingerprint Section, +DUMMY/LJ050-0070.wav|The FBI now transmits information on all defectors, a category which would, of course, have included Oswald. +DUMMY/LJ006-0170.wav|and was surprised when the inspectors proved to him that practices of which he was ignorant, and which he admitted that he reprehended, went on without hindrance. +DUMMY/LJ030-0045.wav|The lead car. -- Described as a "rolling command car," this was an unmarked Dallas police car, driven by Chief of Police Curry +DUMMY/LJ011-0205.wav|he carried me away by fraud and stratagem, and forced me to accompany him to Gretna Green +DUMMY/LJ001-0167.wav|a device which deceives nobody, and makes a book very unpleasant to read. +DUMMY/LJ013-0259.wav|The actual method of the murder was never exactly ascertained. +DUMMY/LJ048-0283.wav|so that nothing can interfere with their bringing to their task the finest qualities and maximum resources of mind and body. +DUMMY/LJ011-0272.wav|The coach immediately drove off; Mr. Gee entered the house, asked for Mr. Heath, was told he would find him in the back kitchen at breakfast. +DUMMY/LJ034-0168.wav|Ronald Fischer and Robert Edwards were standing on the curb at the southwest corner of Elm and Houston Streets, +DUMMY/LJ039-0209.wav|They were instead firing to determine how rapidly the weapon could be fired and the area within which three shots could be placed. +DUMMY/LJ025-0077.wav|Their food is provided for them, +DUMMY/LJ033-0122.wav|Mrs. Randle said, when shown the paper bag, +DUMMY/LJ016-0109.wav|but not till they had had an exciting chase down the street. +DUMMY/LJ048-0138.wav|The wholehearted support of these local authorities was indispensable to the Service in carrying out its duties. +DUMMY/LJ004-0040.wav|They felt that private enterprise might +DUMMY/LJ011-0027.wav|The veil was lifted from his private life, and he was accused of persistent immorality. +DUMMY/LJ028-0446.wav|There lie the ruins of the famous temple of Esagil, sacred to Marduk. +DUMMY/LJ041-0173.wav|Thornley said that he had made his remark only in the context of "nineteen eighty-four" +DUMMY/LJ028-0282.wav|for it seemed to him that there was a divine providence in the man having used the phrase, and then his mule having foaled. +DUMMY/LJ027-0091.wav|See, then, the difference between man's mode of working and Nature's. +DUMMY/LJ018-0043.wav|It was really evidence of facts which could not be controverted or explained away. +DUMMY/LJ016-0129.wav|Down below were market-carts waiting for daylight, and among them Krapps found a refuge and friends. +DUMMY/LJ041-0138.wav|and that she could not think of any reason why Oswald would want to kill President Kennedy. +DUMMY/LJ025-0074.wav|has broken down, so far, at least, as his mode of stating it goes. And +DUMMY/LJ018-0224.wav|It had been supposed up to this date that he had willed his property, amounting in all to upwards of two hundred thousand pounds, +DUMMY/LJ017-0166.wav|but was careful to prime them with his facts and lead them if possible to accept his diagnosis of the case. +DUMMY/LJ003-0331.wav|The privileges of the master's side also disappeared; fees were nominally abolished, and garnish was scotched, although not yet killed outright. +DUMMY/LJ003-0056.wav|This was the chapel yard, with its five wards, which were calculated to hold seventy prisoners, but often held many more. +DUMMY/LJ046-0076.wav|He cannot and will not take the precautions of a dictator or a sovereign. +DUMMY/LJ039-0085.wav|it just takes a minor move in aiming to bring the crosshairs to bear, and then it is a quick squeeze. +DUMMY/LJ040-0223.wav|his lack of friends, the apparent unavailability of any agency assistance and the ineffectualness of his mother +DUMMY/LJ008-0075.wav|She is described as a well-made little woman, something more than thirty years of age, of a pale complexion and not disagreeable features. +DUMMY/LJ010-0207.wav|Thus, among the presidents were those of Gowrie, Justinian, Aloman, Colsman, Kenneth, and Godfrey; +DUMMY/LJ023-0039.wav|by meeting squarely our modern social and economic conditions. +DUMMY/LJ029-0105.wav|The route was further reviewed by Lawson and Sorrels with Assistant Chief Batchelor and members of the local host committee on November fifteen. +DUMMY/LJ034-0120.wav|the police radio reported that, quote, an eyeball witness, end quote, described the suspect in the Tippit shooting as, quote, a white male, +DUMMY/LJ032-0145.wav|also bore impressions of the same irregularities that appeared on the barrel of the rifle. +DUMMY/LJ002-0240.wav|At one time the Marshalsea was the receptacle of pirates, but none were committed to it after seventeen eighty-nine. +DUMMY/LJ046-0189.wav|PRS had received, over a twenty-year period, basic information on some fifty thousand cases; +DUMMY/LJ035-0162.wav|approximately thirty to forty-five seconds after Oswald's lunchroom encounter with Baker and Truly. +DUMMY/LJ040-0063.wav|Significant in shaping the character of Lee Harvey Oswald was the death of his father, a collector of insurance premiums. +DUMMY/LJ023-0093.wav|Chief Justice Hughes said in a dissenting opinion that the majority opinion was "a departure from sound principles," +DUMMY/LJ024-0098.wav|And thirteen states which contain only five percent of the voting population can block ratification +DUMMY/LJ028-0036.wav|But in those very early days Babylon was little more than a shrine, surrounded with mud huts and date palms. +DUMMY/LJ002-0155.wav|for which the costs were six shillings ten pence. +DUMMY/LJ017-0034.wav|The case contained elements of much uncertainty, and yet it was so essential, +DUMMY/LJ048-0129.wav|In view of the constant change in the nature of threats to the President and the diversity of the dangers which may arise in the various cities within the United States, +DUMMY/LJ006-0176.wav|the punishments he inflicted were unequal, +DUMMY/LJ028-0082.wav|it was recognized that the cameo was genuine, and that it bore the likeness of the great King. +DUMMY/LJ009-0166.wav|next, the offenses of returning from transportation, stealing post-office letters, and sacrilege were no longer punishable with death. +DUMMY/LJ019-0337.wav|both in controlling his subordinates and in protecting the prisoners from petty tyranny and oppression, every one of whom he was to see once every twenty-four hours. +DUMMY/LJ050-0231.wav|Before the assassination the Secret Service infrequently requested other Federal law enforcement agencies to provide personnel +DUMMY/LJ024-0112.wav|To them I say: I do not think you will be able long to fool the American people as to your purposes. +DUMMY/LJ031-0075.wav|In the absence of any neurological, muscular, or heart response, the doctors concluded that efforts to revive the President were hopeless. +DUMMY/LJ033-0021.wav|after the birth of their second child. +DUMMY/LJ015-0101.wav|but at Mr. Fasson's pressing demands went upstairs to fetch the certificates. +DUMMY/LJ038-0228.wav|and into downtown Dallas through the Triple Underpass. +DUMMY/LJ041-0006.wav|His mother exercised little control over him and thought he could decide for himself whether to go on in school. +DUMMY/LJ003-0177.wav|if they happened to be in funds -- among whom was the Marquis of Sligo in eighteen eleven. +DUMMY/LJ048-0173.wav|arrangements were made for building and roof security by posting police officers where appropriate. +DUMMY/LJ031-0230.wav|After the autopsy was concluded at approximately eleven p.m., the President's body was prepared for burial. +DUMMY/LJ026-0063.wav|The food supply of the higher plant, like that of the animal, is the source of the required matter and energy, +DUMMY/LJ030-0112.wav|The motorcade left Love Field shortly after eleven:fifty a.m. and drove at speeds up to twenty-five to thirty miles an hour +DUMMY/LJ005-0013.wav|One smug alderman, a member of the House of Commons, sneered at the ultra philanthropy of the champions of prison improvement. +DUMMY/LJ003-0307.wav|but as yet no suggestion was made to provide prison uniform. +DUMMY/LJ005-0174.wav|in the deplorable situation in which many of them now are." +DUMMY/LJ042-0215.wav|but our, the people, culture, traditions, heritage, and very people itself, and yet +DUMMY/LJ005-0204.wav|Clun, in Shropshire, had a lock-up under the town hall. +DUMMY/LJ006-0086.wav|his place is assigned among the most depraved, the most experienced, and the most incorrigible offenders in the middle yard. +DUMMY/LJ037-0111.wav|Guinyard and Callaway ran to tenth and Patton and found Tippit lying in the street beside his car. +DUMMY/LJ039-0177.wav|was greater than from the second to the third shot and required a movement in the basic firing position of the marksmen. +DUMMY/LJ006-0038.wav|"a subject of magnitude and importance sufficient to exclude other jails," they soon narrowed their inquiry still further, +DUMMY/LJ010-0070.wav|followed the profession of arms, first in the British service, and then in that of the French revolutionary Government. +DUMMY/LJ027-0022.wav|An examination of the facts in each of the lines of evidence makes it clear +DUMMY/LJ018-0053.wav|he whispered in the ear of the German pastor who attended him on the scaffold, +DUMMY/LJ018-0027.wav|In little more than a week a cabman came forward and voluntarily made a statement which at once drew suspicion to a German, Franz Müller, +DUMMY/LJ023-0067.wav|That, my friends, is what I honestly believe +DUMMY/LJ010-0011.wav|This led to a rapid and marked increase in all kinds of fraud; +DUMMY/LJ023-0031.wav|It was almost too late. +DUMMY/LJ022-0022.wav|That is a tremendous gain for the principles of democracy. +DUMMY/LJ016-0220.wav|On another occasion an importunate person begged Calcraft eagerly to claim his right to the clothes, and give them to him. +DUMMY/LJ013-0166.wav|While Courvoisier opened the shutters, the housemaid, approaching the bed, saw that the pillow was saturated with blood. +DUMMY/LJ050-0165.wav|the Commission had occasion to become informed, to a limited extent, about the data processing techniques of other Federal intelligence and law enforcement agencies. +DUMMY/LJ024-0055.wav|We think it so much in the public interest to maintain a vigorous judiciary +DUMMY/LJ014-0214.wav|They walked together quietly for some little distance, then Cannon, without the slightest warning, +DUMMY/LJ036-0028.wav|McWatters' recollection is that he issued this transfer to a man who entered his bus just beyond Field Street, +DUMMY/LJ029-0135.wav|Elm Street, the northernmost of the three, after intersecting Houston curves in a southwesterly arc +DUMMY/LJ012-0027.wav|He had such great aptitude for business, and such a thorough knowledge of the real value of goods, +DUMMY/LJ032-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ046-0148.wav|that the safety of the President is or might be in danger, either at the present or in the future. +DUMMY/LJ030-0134.wav|because it convinced him that the average Dallas resident was like other American citizens in respecting and admiring the President. +DUMMY/LJ006-0042.wav|Relapse was rapid and inevitable, so that the latter state of the prison was worse than the first. +DUMMY/LJ017-0252.wav|The place at which they landed was not far from the entrance to the River Plate. +DUMMY/LJ047-0194.wav|Hosty received this information on the afternoon of November twenty-two, nineteen sixty-three. +DUMMY/LJ002-0190.wav|The best, or at least the most influential prisoners, got lodging in the State House, which contained "eight large handsome rooms." +DUMMY/LJ010-0032.wav|Pegsworth, and Greenacre, and Daniel Good merely reproduced types that had gone before, and that have since reappeared. +DUMMY/LJ045-0148.wav|After all, when will all your foolishness come to an end? All of these comedies. First one thing and then another. And now this fictitious name, end quote. +DUMMY/LJ018-0279.wav|assisted by Macdonell and Noyes, all of them citizens of the United States, by which the bank lost upwards of one hundred thousand pounds. +DUMMY/LJ032-0102.wav|According to her testimony, he compelled her to write the name "Hidell" on membership cards in the space designated for the signature of the "Chapter President." +DUMMY/LJ031-0191.wav|Nine minutes later, the Presidential airplane departed for Washington, D.C. +DUMMY/LJ028-0491.wav|then a second wall still higher, a third and fourth and a fifth, each stronger and higher than the others, +DUMMY/LJ029-0169.wav|so that crowds can get a good view of President Kennedy and his wife. +DUMMY/LJ008-0032.wav|a scaffold was erected in front of that prison for the execution of several convicts named by the Recorder. +DUMMY/LJ016-0391.wav|He ate constantly and voraciously after sentence, as though eager to cram as many meals as possible into the few hours still left him to live. +DUMMY/LJ006-0007.wav|as the inspectors of prisons found them in eighteen thirty-five to six. +DUMMY/LJ005-0082.wav|unless such ability (to work) should cease by reason of sickness, infirmity, the want of sufficient work, or from any other cause. +DUMMY/LJ044-0231.wav|While some of Castro's more severe criticisms of President Kennedy might have led Oswald to believe that he would be well received in Cuba +DUMMY/LJ011-0159.wav|At Manchester, en route, a traveling carriage was purchased, which was driven up to Mrs. Daulby's door at eight in the morning, +DUMMY/LJ030-0004.wav|The trip to Texas began with the departure of President and Mrs. Kennedy from the White House +DUMMY/LJ035-0023.wav|Baker testified that he entered the lobby of the building and, quote, spoke out and asked where the stairs or elevator was +DUMMY/LJ006-0012.wav|but he would not arm them with any authority lest their cooperation might be offensive. +DUMMY/LJ028-0216.wav|commanded him (Cyrus) to go forth to his city Babylon, and he went by his side as a friend and companion. +DUMMY/LJ042-0049.wav|I never believed I would find more material advantages at this stage of development in the Soviet Union than I might of had in the U.S. +DUMMY/LJ046-0053.wav|In recent years, Presidential journeys have been frequent and extensive, +DUMMY/LJ005-0153.wav|Sometimes this money might be expended in the purchase of extra articles of food. +DUMMY/LJ005-0152.wav|Earnings were very differently appropriated. Here the prisoners were given the whole amount, there a half or a third. +DUMMY/LJ034-0085.wav|found a clipboard hidden by book cartons in the northwest corner of the sixth floor at the west wall a few feet from where the rifle had been found. +DUMMY/LJ048-0154.wav|as discussed in chapter three. +DUMMY/LJ030-0107.wav|Presidential car, Presidential follow-up car, +DUMMY/LJ034-0177.wav|Fischer placed the man in the easternmost window on the south side of the Depository Building on either the fifth or the sixth floor. +DUMMY/LJ025-0140.wav|are not such, but are simply the visible aggregations of vastly more attenuated filaments, +DUMMY/LJ035-0024.wav|and this man, Mr. Truly, spoke up and says, it seems to me like he says, +DUMMY/LJ018-0050.wav|His arguments were specious and evasive when pressed to confess. "Why should man confess to man?" he replied; +DUMMY/LJ007-0179.wav|defeats the ends of justice, and disgraces the profession of a Christian country. +DUMMY/LJ044-0104.wav|He later wrote to another official of the Worker, seeking employment, and mentioning the praise he had received for submitting his photographic work. +DUMMY/LJ050-0178.wav|to make certain that all protective intelligence activities are coordinated. +DUMMY/LJ034-0162.wav|Brennan described to the police the man he saw in the window and then identified Oswald as the person who most nearly resembled the man he saw. +DUMMY/LJ016-0306.wav|and the question, as part of the whole subject of capital punishment, had been referred to a royal commission in January eighteen sixty-four. +DUMMY/LJ030-0128.wav|From Main Street the motorcade turned right and went north on Houston Street, passing tall buildings on the right, +DUMMY/LJ024-0101.wav|Bar Association, Manufacturers' Associations, who are trying to give the impression that they really do want a constitutional amendment +DUMMY/LJ006-0036.wav|They visited the wards after locking-up time, and saw with their own eyes what went on. +DUMMY/LJ003-0131.wav|He was an inmate of the same ward with others of the most dreadful sort, quote, +DUMMY/LJ048-0075.wav|each SAC should specifically designate an Agent (or Agents) to be responsible for developing +DUMMY/LJ011-0190.wav|On arrival, Wakefield pretended that they had missed Mr. Turner, and must follow him over to France. +DUMMY/LJ006-0117.wav|that none of its inmates were suffered to make tea or coffee for themselves lest it should interfere with his sales. +DUMMY/LJ010-0048.wav|These crimes had their origin too often in the disordered brains of lunatics at large, like Captain Goode. +DUMMY/LJ024-0096.wav|No amendment which any powerful economic interests or the leaders of any powerful political party have had reason to oppose +DUMMY/LJ027-0076.wav|These remarkable similarities in the common general plan alone are convincing evidences of descent, +DUMMY/LJ009-0167.wav|In eighteen thirty-seven Lord John Russell's acts swept away a number of capital offenses, including cutting and maiming, rick-burning, +DUMMY/LJ016-0030.wav|they preferred to credit it to carelessness or collusion from officers of the jail. +DUMMY/LJ039-0142.wav|After Oswald purchased the Mannlicher-Carcano rifle, he told his wife that he practiced with it. +DUMMY/LJ010-0190.wav|A craze for notoriety, to be achieved at any cost, was the one absorbing idea in young Oxford's disordered brain. +DUMMY/LJ035-0195.wav|Satisfying himself that Oswald was missing, Truly obtained Oswald's address, phone number, and description from his employment application card. +DUMMY/LJ034-0121.wav|twenty-seven, five foot eleven, one hundred sixty-five pounds, black wavy hair, end quote, +DUMMY/LJ003-0153.wav|Idleness was not so universally the rule in this part of the jail. +DUMMY/LJ015-0061.wav|They both reproduced many of the features of the case of Watts, already described, +DUMMY/LJ012-0139.wav|Solomons, had sold bar gold to the value of one thousand two hundred pounds to certain bullion dealers. +DUMMY/LJ022-0118.wav|No sectional, no political distinctions can be permitted. +DUMMY/LJ004-0006.wav|Howard's energy and devotion were rewarded by lively sympathy, but the desire to improve which followed his exposures was but short-lived. +DUMMY/LJ029-0049.wav|after undertaking the responsibility for advance preparations for the visit to Dallas, Agent Lawson went to the PRS offices in Washington. +DUMMY/LJ003-0127.wav|Quote, by insensible degrees he began to lose his repugnance to their society, +DUMMY/LJ015-0034.wav|The engagements of the bank on these two heads amounting to nearly half a million of money, +DUMMY/LJ044-0038.wav|promptly closed three days later for some obscure reasons by the renters, +DUMMY/LJ033-0053.wav|Only if disassembled could the rifle fit into the paper bag found near the window from which the shots were fired. +DUMMY/LJ019-0224.wav|With the reduction of numbers to be accommodated, there was ample space in Newgate for its reconstruction on the most approved modern lines. +DUMMY/LJ005-0266.wav|that food and fuel should be issued in kind, and never provided by the prisoners themselves out of monies granted them. +DUMMY/LJ009-0059.wav|Friends, even utter strangers, have interceded for him, and to the last he has buoyed himself up by hope of reprieve. +DUMMY/LJ035-0124.wav|Neither Jarman, Norman, Williams, or Dougherty saw Oswald. +DUMMY/LJ042-0238.wav|freedom of speech, travel outspoken opposition to unpopular policies, freedom to believe in god, end quote, while the Soviet Union did not. +DUMMY/LJ048-0229.wav|The statements of the agents involved are supported by statements of members of the Fort Worth press who accompanied or observed them +DUMMY/LJ016-0379.wav|All as a rule, unless poignant remorse has brought a desire to court their richly-merited retribution, are buoyed up +DUMMY/LJ004-0010.wav|with a few bright exceptions, still deplorable and disgraceful. +DUMMY/LJ049-0063.wav|Beyond these two principles the Secret Service believes a detailed contingency or emergency plan is not feasible +DUMMY/LJ042-0013.wav|that while he had begun to read Marx and Engels at the age of fifteen, the conclusive thing that made him decide that Marxism was the answer +DUMMY/LJ043-0173.wav|and the trail of evidence he left behind him on that occasion. +DUMMY/LJ005-0094.wav|to call for information as to the observance of its provisions. +DUMMY/LJ030-0166.wav|On turning she saw a quizzical look on her husband's face as he raised his left hand to his throat. +DUMMY/LJ016-0343.wav|The sufferer was a porter on the London, Chatham, and Dover railway, sentenced to death for shooting the station-master at Dover. +DUMMY/LJ030-0136.wav|end quote, the President replied, "That is very obvious." +DUMMY/LJ002-0133.wav|Alfred the Great established the Court Baron, the Hundred Court, and the County Court, which among other matters entertained pleas for debt. +DUMMY/LJ014-0339.wav|asked Davidson and Gordon, a firm with which Cole was closely allied, whether the warrants meant goods or nothing. +DUMMY/LJ003-0248.wav|The irons weighed from three to four pounds, but heavier irons, seven or eight pounds' weight, +DUMMY/LJ045-0219.wav|that he wanted to make peace with her and bring his family to Dallas where they could live with him again. +DUMMY/LJ049-0192.wav|It was also suggested that it would take a substantial period of time for the Secret Service to build up the experience and skills necessary to meet the problem. +DUMMY/LJ049-0006.wav|These precautions included reserving a ceremonial area for the Presidential party, +DUMMY/LJ043-0177.wav|and his Communist and Socialist Worker's Party newspapers would probably have appeared on the front pages of newspapers or magazines all over the country, +DUMMY/LJ003-0041.wav|There was no lack of air and light for the new jail, and several exercising yards. +DUMMY/LJ006-0026.wav|England was represented by Mr. Crawford, and the result of his inquiry was given to the public as an appendix to the House of Commons' Report on Secondary Punishments. +DUMMY/LJ047-0046.wav|Agent Fain was not satisfied by this interview and arranged to see Oswald again on August sixteen, nineteen sixty-two. +DUMMY/LJ044-0192.wav|and as far as the performance of the Cubans themselves was concerned, he was, quote, disappointed at not being able to get to Cuba +DUMMY/LJ006-0157.wav|This was only one of many grave omissions on his part. He did not feel it incumbent on himself to visit his prison often or see his prisoners. +DUMMY/LJ039-0233.wav|On the basis of Oswald's training and the accuracy of the weapon as established by the tests, +DUMMY/LJ018-0119.wav|His offer was not, however, accepted. +DUMMY/LJ044-0220.wav|The general conflict of views between the United States and Cuba was, of course, reflected in other media to such an extent +DUMMY/LJ038-0237.wav|The Commission confirmed, by comparison with other photographs, that these were, indeed, photographs of the rear of Walker's house. +DUMMY/LJ043-0065.wav|Oswald experienced some difficulty finding employment. Perhaps this was partially because of his lack of any specific skill or training. +DUMMY/LJ019-0333.wav|were elected to inspect the prisons frequently, to examine the prisoners, hear complaints, and check abuses. +DUMMY/LJ014-0251.wav|In this way Watts had complete control over the whole of the monetary transactions of the company. +DUMMY/LJ007-0067.wav|stigmatized his often most just strictures on abuses as "a bundle of nonsense," and the aldermen, when he appealed to them for protection and countenance, +DUMMY/LJ008-0171.wav|and it passed along the heads of the crowd, to be caught at last by a person who struggled with it to a cart and deposited it there in safety. +DUMMY/LJ014-0275.wav|but it was long before the public realized that the fraudulent clerk and the great theatrical manager were one and the same person. +DUMMY/LJ040-0238.wav|Such a placement was postponed, however, perhaps in part at least because Lee's behavior suddenly improved. +DUMMY/LJ038-0194.wav|seven. Certain of my documents are in the small blue valise. +DUMMY/LJ003-0218.wav|The chapel was filled with a curious but callous congregation, who came to stare at the miserable people thus publicly exposed. +DUMMY/LJ002-0039.wav|they each contained four small rooms or "cabins" seven feet square, +DUMMY/LJ041-0063.wav|He said that Oswald, quote, seemed to guard against developing real close friendships, end quote, +DUMMY/LJ002-0219.wav|and to have been guilty of extortion to others. One Sir William Rich, Bart., he had loaded with heavy irons +DUMMY/LJ038-0030.wav|The man was Oswald. He was sitting alone in the rear of the main floor of the theatre near the right center aisle. +DUMMY/LJ028-0363.wav|and there on June thirteen, three twenty-three B.C., he met his death. +DUMMY/LJ024-0066.wav|But chance and the disinclination of individuals to leave the Supreme bench +DUMMY/LJ026-0121.wav|The destruction of protoplasm, excess proteids or the fat and carbohydrate foods is catabolism, destructive metabolism. +DUMMY/LJ027-0062.wav|viz., the brain; and also usually, but not always, a number of posterior joints, +DUMMY/LJ048-0022.wav|Answer: Yes. It was a self-serving interview. The visits with the Soviet Embassy were evidently for the purpose of securing a visa, +DUMMY/LJ032-0144.wav|was confirmed by FBI Laboratory tests which established that the adhesive material bearing the print +DUMMY/LJ013-0041.wav|He was an honest sea-captain, he said, trading from Liverpool, where once he had the misfortune to be introduced to a man of wealth, +DUMMY/LJ023-0029.wav|we cannot delay one moment in making certain that our national government has power to carry through. +DUMMY/LJ012-0019.wav|The name of Ikey Solomons was long remembered by thief and thief-taker. +DUMMY/LJ022-0044.wav|Sometimes it depends upon what paper you read and what broadcast you hear. +DUMMY/LJ017-0085.wav|Palmer's plan was to administer poison in quantities insufficient to cause death, but enough to produce illness which would account for death. +DUMMY/LJ050-0121.wav|Detailed formal agreements embodying these arrangements should be worked out between the Secret Service and both of these agencies. +DUMMY/LJ027-0102.wav|which in other and allied kinds of animals and plants are of large size and functional utility. +DUMMY/LJ046-0032.wav|or their desire to have frequent and easy access to the people. +DUMMY/LJ016-0134.wav|Cases well authenticated have been known of men who had all but completed their sentences, and for whom the prison gates would open within a few days, +DUMMY/LJ002-0170.wav|There was in the City Road a temporary bar, with a collector of tolls who was sometimes on the spot and sometimes not. +DUMMY/LJ024-0058.wav|or make independent on upon the desire or prejudice of any individual justice? +DUMMY/LJ004-0028.wav|Various acts were also passed to consolidate and amend previous jail acts. +DUMMY/LJ005-0190.wav|the knowledge and practice of every species of criminality. +DUMMY/LJ048-0160.wav|either by police or by custodians of the buildings, since it was not the usual practice of the Secret Service to do so. +DUMMY/LJ004-0104.wav|By the thirty-one George the third c. forty-six, s. five, +DUMMY/LJ039-0007.wav|In January nineteen sixty-four, Marina Oswald and her business manager, James Martin, +DUMMY/LJ029-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ046-0074.wav|If the sole goal were to protect the life of the President, it could be accomplished with reasonable assurance despite the multiple roles he must play. +DUMMY/LJ027-0180.wav|The general similarity of the three series is complete. +DUMMY/LJ028-0051.wav|He enlarged the old city, erected temples, and began the construction of its walls. +DUMMY/LJ005-0067.wav|The Prison Society reproves the misdirected efforts of ambitious architects, who by a lavish and improvident expenditure of public money +DUMMY/LJ028-0237.wav|with orders to march into the town by the bed of the stream, as soon as the water became shallow enough: +DUMMY/LJ027-0083.wav|For such a series of facts the reader must be referred to special books like Wiedersheim's "Comparative Anatomy of the Vertebrates," +DUMMY/LJ029-0029.wav|When the trip was planned for only one day, Governor Connally had opposed the motorcade because there was not enough time. The Governor stated, however, that, quote, +DUMMY/LJ037-0028.wav|He saw him empty the gun and throw the shells into some bushes on the southeast corner lot. +DUMMY/LJ013-0141.wav|whose ship brought home a mixed cargo from Bahia and other ports. +DUMMY/LJ050-0230.wav|Manpower and Technical Assistance From Other Agencies +DUMMY/LJ032-0065.wav|In accordance with postal regulations, the portion of the application which lists names of persons, other than the applicant, entitled to receive mail +DUMMY/LJ033-0073.wav|Mrs. Paine had the actual blanket before her as she testified and she indicated that the blanket hung limp in the officer's hand. +DUMMY/LJ013-0013.wav|with other underwriters for a second sum of two thousand pounds. +DUMMY/LJ013-0062.wav|to draw the dividends on more than one sum. +DUMMY/LJ035-0116.wav|Three employees -- James Jarman, Jr., +DUMMY/LJ026-0153.wav|the other part of respiration, elimination of carbon dioxide, has been treated under excretions. +DUMMY/LJ039-0116.wav|he had just completed a very intensive preliminary training period. +DUMMY/LJ045-0229.wav|He then killed Patrolman Tippit when that police officer apparently tried to question him after he had left his roominghouse and he vigorously resisted arrest +DUMMY/LJ028-0157.wav|This river is the Euphrates, a broad, deep, swift stream, which rises in Armenia, and empties itself into the Erythraean Sea. +DUMMY/LJ031-0181.wav|onto the rear of the Presidential plane where seats had been removed to make room. +DUMMY/LJ016-0072.wav|where he presently committed a burglary at Lymington, was caught, and lodged in Winchester Jail. +DUMMY/LJ043-0154.wav|after he thought that what he had written in his book might be proof against him, and he destroyed it (the book), end quote. +DUMMY/LJ036-0112.wav|Whaley said that Oswald was the man under Number two. +DUMMY/LJ006-0161.wav|Again, a turnkey deposed that his chief did not enter the wards more than once a fortnight. +DUMMY/LJ013-0038.wav|The merchant Wallace said he had been led into the crime by the advice and example of a city friend who had gone largely into this nefarious business; +DUMMY/LJ009-0033.wav|while behind their pew stood a couple of tall footmen in state liveries. +DUMMY/LJ044-0070.wav|WDSU has no program of any kind called, quote, Latin American Focus, end quote. +DUMMY/LJ039-0218.wav|Simmons described it as, quote, quite accurate, end quote, in fact, as accurate as current military rifles. +DUMMY/LJ022-0068.wav|Another helpful feature of unemployment insurance is the incentive it will give to employers to plan more carefully +DUMMY/LJ042-0106.wav|He is reported to have expressed the conclusion that they had, quote, fat stinking politicians over there just like we have over here, end quote. +DUMMY/LJ002-0228.wav|The Committee on Jails reported that, quote, although the house of the warden looked into the court, +DUMMY/LJ007-0215.wav|are the association of prisoners, and the unusual contamination to which such association gives rise. +DUMMY/LJ014-0048.wav|But at the fatal hour his hardihood forsook him, and he was almost insensible when taken out of his cell for execution. +DUMMY/LJ025-0053.wav|to the investigation of organic structure, +DUMMY/LJ034-0090.wav|Eyewitness Identification of Assassin +DUMMY/LJ035-0183.wav|At twelve:thirty-four p.m. Sawyer heard a call over the police radio that the shots had come from the Depository Building. +DUMMY/LJ014-0060.wav|Marie de Roux up to the time of her marriage had been in service as lady's maid to Lady Blantyre, daughter of the Duchess of Sutherland, +DUMMY/LJ011-0265.wav|This Mr. Canning had left his widow a life interest in two thousand pounds so long as she remained unmarried. +DUMMY/LJ022-0070.wav|Provisions for social security, however, are protections for the future. +DUMMY/LJ032-0175.wav|and identified the shirt as being the one worn by Oswald primarily because of a distinctive hole in the shirt's right elbow. +DUMMY/LJ003-0336.wav|but it at once deprecated the idea that the city could follow the laudable example thus set in the provinces. Quote, +DUMMY/LJ033-0155.wav|Both concluded that the prints were the right palm and left index finger of Lee Oswald. No other identifiable prints were found on the bag. +DUMMY/LJ026-0045.wav|e.g., a bean plant with a higher animal, e.g., frog or even man. +DUMMY/LJ011-0136.wav|Still no sentence of death was carried out for the offense, and in eighteen thirty-two +DUMMY/LJ034-0061.wav|The results of this investigation are fully discussed in chapter six, page two forty-nine. +DUMMY/LJ016-0339.wav|troops were held in readiness to act if required. Everything passed off quite quietly, however, +DUMMY/LJ023-0135.wav|In forty-five out of the forty-eight states of the Union, judges are chosen not for life but for a period of years. +DUMMY/LJ042-0032.wav|and of his initial commitment to that country can best be understood, however, in the context +DUMMY/LJ049-0186.wav|the Commission received a number of proposals designed to improve current arrangements for protecting the President. +DUMMY/LJ041-0030.wav|One of his fellow employees, Palmer McBride, stated that Oswald said he would like to kill President Eisenhower because he was exploiting the working class. +DUMMY/LJ019-0093.wav|and because the machine was often so imperfect that the amount of effort could not be exactly regulated. +DUMMY/LJ018-0107.wav|eighty years of age, who had spent his whole life in imitating other people's signatures, and had acquired the most consummate skill in the practice. +DUMMY/LJ010-0142.wav|Be this as it may, the weapon used was only an ordinary axe, which rather indicates that force, not skill, was employed. +DUMMY/LJ027-0033.wav|On the other hand, external structures are likely to undergo adaptation when habits or conditions of life change. +DUMMY/LJ025-0134.wav|A molecular change takes place in the nerve of the tentacle, +DUMMY/LJ022-0125.wav|every profession has its black sheep, but long experience in government has taught me +DUMMY/LJ041-0041.wav|This was one year before his actual enlistment and just a little over two point five years after he left New York. +DUMMY/LJ049-0156.wav|Secret Service men accompanied the President and his family to their vacation home in Massachusetts +DUMMY/LJ039-0042.wav|I was absolutely convinced it was Nixon and now after all these questions I wonder if I am right in my mind? End quote. +DUMMY/LJ029-0092.wav|On November eight, when Lawson was briefed on the itinerary for the trip to Dallas, +DUMMY/LJ026-0030.wav|The distinction between the higher plants and animals is perfectly sharp and obvious, +DUMMY/LJ048-0017.wav|We satisfied ourselves that we had met our requirement, namely to find out whether he had been recruited by Soviet intelligence. The case was closed. +DUMMY/LJ014-0070.wav|a heavy brutish fellow, was yet aghast at his wife's resolve, and tried hard to dissuade her from bad purpose. +DUMMY/LJ013-0119.wav|Howse, the steward, accused the other servants, but they retorted, declaring that he had been visited by the thief the day previous, +DUMMY/LJ036-0092.wav|and said, quote, Everybody will know who I am now, end quote. +DUMMY/LJ029-0025.wav|When Governor Connally called at the White House on October four to discuss the details of the visit, +DUMMY/LJ042-0112.wav|said that Oswald told her that he had returned because, quote, I didn't find what I was looking for, end quote. +DUMMY/LJ045-0156.wav|Answer: He tried to talk to me but I would not answer him, and he was very upset. Question: Were you upset with him? +DUMMY/LJ005-0044.wav|the method by which they were worked, and the dietaries of the prisoners employed upon them. +DUMMY/LJ037-0125.wav|As Oswald ran south on Patton Avenue toward Jefferson Boulevard he was moving in the direction of a used-car lot +DUMMY/LJ024-0129.wav|brings again to mind that crude and cruel strategy tried by the same opposition +DUMMY/LJ006-0017.wav|with those who made the selection of the first inspectors, and the two gentlemen appointed were probably the most fitted in England to be so employed. +DUMMY/LJ025-0047.wav|The plant withdraws water and carbonic acid from the atmosphere, the animal contributes both to it. +DUMMY/LJ035-0057.wav|Missing Baker, he came back to find the officer in the doorway to the lunchroom facing Lee Harvey Oswald. +DUMMY/LJ005-0046.wav|The good it tried to do took active shape in the establishment of temporary refuges -- at Hoxton for males, and in the Hackney Road for females +DUMMY/LJ002-0211.wav|The sale of spirits was forbidden, but gin could always be had at the whistling shops, where it was known as Moonshine, Sky Blue, +DUMMY/LJ013-0189.wav|next, and in the same place, a chased gold key; +DUMMY/LJ008-0238.wav|as it lay, white and ghastly, upon the black boards, and then again was as suddenly withdrawn, as if ashamed of the deed it had done. +DUMMY/LJ010-0292.wav|which was authorized by a recent act, on account of Mr. Pate's family and position in life. +DUMMY/LJ006-0270.wav|But no search was made to intercept prohibited articles at the gate, and there was no permanent gate-keeper, +DUMMY/LJ048-0242.wav|Three members of this shift separately took this opportunity to visit the Cellar Coffee House. +DUMMY/LJ040-0134.wav|that anybody entering this home had to be searched in case the parents were bringing cigarettes or narcotics or anything, end quote. +DUMMY/LJ001-0057.wav|Parma, and one or two other cities, who produced the splendid editions of the Classics, which are one of the great glories of the printer's art, +DUMMY/LJ027-0050.wav|Homology thus means identity of structure which is the result of identity of parentage. It is the stamp of heredity. +DUMMY/LJ014-0300.wav|Sentence of death had been recorded against Donovan, who, like Watts, had seemingly been overcome with sudden despair. +DUMMY/LJ018-0163.wav|Griffiths was an unusually clever and skilful workman, +DUMMY/LJ025-0107.wav|They absorb oxygen and give out carbonic acid. +DUMMY/LJ003-0208.wav|a number of amateurs were ever ready to give their gratuitous ministrations to the condemned. +DUMMY/LJ037-0048.wav|Helen Markham was screaming as she leaned over the body. +DUMMY/LJ003-0167.wav|The fee for admission to the state side, as fixed by the table of fees, was three guineas, but Mr. Newman declared that he never took more than two. +DUMMY/LJ032-0066.wav|was thrown away after the box was closed in May nineteen sixty-three. +DUMMY/LJ007-0174.wav|should be for a number of years in undisturbed operation, not only in contempt of religion and humanity, +DUMMY/LJ039-0147.wav|sighting with the telescopic lens and operating the bolt. +DUMMY/LJ049-0205.wav|this Commission has necessarily examined into the functioning of the various Federal agencies concerned with the tragic trip of President Kennedy to Dallas +DUMMY/LJ036-0129.wav|The man asked, quote, May I have the cab?, end quote, and got into the front seat. +DUMMY/LJ005-0048.wav|which he could bestow on prisoners on release, and so save the better-disposed or the completely destitute from lapsing at once into crime. +DUMMY/LJ022-0170.wav|Not only business recovery, but the general economic recovery of the nation will be greatly stimulated by the enactment of legislation +DUMMY/LJ007-0053.wav|"Joseph Coleman put in irons for three days for striking one of the prisoners," in the same place. +DUMMY/LJ036-0168.wav|If the cab ride was approximately six minutes, as was the reconstructed ride, +DUMMY/LJ028-0351.wav|As for Zopyrus he was considered by Darius to have surpassed, in the greatness of his achievements, all other Persians, +DUMMY/LJ037-0227.wav|Mrs. Markham told Poe +DUMMY/LJ011-0220.wav|The marriage was annulled by an Act of Parliament, although Wakefield petitioned against it, +DUMMY/LJ044-0112.wav|Marina Oswald said that such correspondence from people he considered important meant much to Oswald. +DUMMY/LJ037-0038.wav|at the southeast corner of the intersection, approximately fifty feet away. +DUMMY/LJ017-0205.wav|Wilson was duly sentenced to death, and suffered impenitent, hardened, and without any confession of her guilt. +DUMMY/LJ025-0152.wav|So that I know not where we can hope to find any absolute distinction between animals and plants, unless we return to their mode of nutrition +DUMMY/LJ030-0227.wav|Special Agent George W. Hickey, Jr., in the rear seat of the Presidential follow-up car, +DUMMY/LJ025-0123.wav|has not only been discovered to exist far more widely among plants than was formerly imagined, +DUMMY/LJ006-0193.wav|Idle hands and unoccupied brains found in mischief the only means of whiling away the long hours of incarceration. +DUMMY/LJ021-0065.wav|but also to the owners and managers of industry because, +DUMMY/LJ015-0290.wav|the forgers always escaped detection. But fate overtook two of the gang, +DUMMY/LJ035-0042.wav|Since the vestibule door is only a few feet from the lunchroom door, the man must have entered the vestibule only a second or two before Baker arrived at the top of the stairwell. +DUMMY/LJ044-0221.wav|that there can be no doubt that Oswald was aware generally of the critical attitude that Castro expressed about President Kennedy. +DUMMY/LJ018-0103.wav|and silver coins, with all the latest appliances for coining, including those of electroplating; +DUMMY/LJ014-0002.wav|Hocker's murder is in its way interesting, as affording another proof of the extraordinary way in which the culprit returned to the scene of his guilt. +DUMMY/LJ027-0043.wav|and mean nothing more than similarity of environment. +DUMMY/LJ006-0294.wav|In the bail-dock, where most improper general association was permitted, the female prisoners were often altogether in the charge of male turnkeys. +DUMMY/LJ002-0032.wav|These were: one. The male debtors' side. +DUMMY/LJ043-0082.wav|and noted that Oswald's verbal and clerical potential was, quote, outstanding, end quote. +DUMMY/LJ006-0296.wav|and was in the habit of drawing frequently upon the female prison for prisoners to act as domestic servants in his own private dwelling. +DUMMY/LJ004-0138.wav|One poor wretch, who had slept next the wall, said he had been literally unable to move for the pressure. +DUMMY/LJ028-0283.wav|As soon therefore as he felt within himself that Babylon was fated to be taken, he went to Darius and asked him if he set a very high value on its conquest. +DUMMY/LJ028-0012.wav|she realizes that she is bound on the long journey across the desert. Hour after hour she bears you over the hard monotonous plain. +DUMMY/LJ001-0075.wav|the worst, which perhaps was the English, was a terrible falling-off from the work of the earlier presses; +DUMMY/LJ007-0159.wav|Forced and constant intercourse with the most depraved individuals of his own class; +DUMMY/LJ021-0174.wav|England has her peculiarities and we have ours +DUMMY/LJ029-0200.wav|Two days before the President's arrival Chief Curry warned that the Dallas police would not permit improper conduct during the President's visit. +DUMMY/LJ018-0094.wav|Forged cheques and bills were soon uttered in great numbers, as well as base coin. +DUMMY/LJ014-0023.wav|and many of the same sort were found in the possession of the accused. This was enough to obtain a committal, +DUMMY/LJ005-0243.wav|Nevertheless a committee of the House of Commons was appointed in eighteen thirty-one to report upon the whole system of secondary punishments, +DUMMY/LJ018-0248.wav|He told his story with perfect coolness and self-possession, but in a grave and serious tone. +DUMMY/LJ033-0209.wav|The preponderance of the evidence supports the conclusion that Lee Harvey Oswald (one) +DUMMY/LJ049-0117.wav|that the interruption shall not take place for an hour, end quote. +DUMMY/LJ007-0140.wav|The primary object of committing a prisoner to jail, as the inspectors pointed out, was to deter not only the criminal himself, but others from crime, +DUMMY/LJ030-0119.wav|The crowds were so dense that Special Agent Clinton J. Hill +DUMMY/LJ019-0187.wav|and he was therefore prepared to submit to Parliament a proposal that half the cost of reconstruction should be borne by public funds. +DUMMY/LJ007-0114.wav|A certain number, who could afford the high rate of a guinea per diem, fixed by the under sheriff, were lodged in the governor's house, +DUMMY/LJ014-0316.wav|Inquiries were instituted when the cheque was protested, which led to the discovery of the forgeries. +DUMMY/LJ032-0081.wav|On the Hidell Selective Service card there appeared a signature, "Alek J. Hidell," +DUMMY/LJ044-0090.wav|I think that we finished him on that program because we had publicly linked the Fair Play for Cuba Committee +DUMMY/LJ022-0166.wav|This legislation will not only in the long run result in providing lower electric and gas rates to the consumer, +DUMMY/LJ028-0270.wav|This was said by a Babylonian who thought that a mule would never foal. +DUMMY/LJ009-0182.wav|This dissection was carried out for Newgate prisoners in Surgeons' Hall, adjoining Newgate, +DUMMY/LJ003-0168.wav|Ten and sixpence a week more was charged as rent for a single bed; where two or more slept in a bed the rent was seven shillings a week each. +DUMMY/LJ035-0105.wav|They took the east elevator to the seventh floor and ran up a stairway to the roof where they searched for several minutes. +DUMMY/LJ028-0394.wav|he passed other unimportant fortresses, and came at last to a walled enclosure, which the natives pointed out as a royal hunting ground. +DUMMY/LJ016-0182.wav|he was a shoemaker by trade, and settled in London after his marriage in eighteen twenty-five. +DUMMY/LJ018-0281.wav|Their operations were no less worldwide. +DUMMY/LJ022-0060.wav|We must begin now to make provision for the future. +DUMMY/LJ042-0073.wav|Oswald went on October thirty-one, to the American Embassy to renounce his U.S. citizenship. +DUMMY/LJ022-0071.wav|Our responsibility for the immediate necessities of the unemployed has been met by the Congress +DUMMY/LJ003-0036.wav|Authority was given to raise money on the Orphans' Fund to the extent of ninety thousand pounds. +DUMMY/LJ004-0119.wav|which had continued through that long period, are forcibly pointed out. +DUMMY/LJ031-0149.wav|A nurse was asked to identify hospital personnel and to tell everyone, except necessary medical staff members, to leave the emergency room. +DUMMY/LJ004-0131.wav|To specify more particularly one or two of the worst, it may be mentioned that in the Borough Compter +DUMMY/LJ016-0265.wav|the crowd had come to witness a popular and gratuitous public performance -- better than a prize-fight or a play. +DUMMY/LJ010-0090.wav|Hand-grenades were to be thrown into the dining-room, and during the noise and confusion the assassination of the ministers was to be completed, +DUMMY/LJ026-0141.wav|The construction in the cells of new proteid from the absorbed carbohydrate and the materials from the soil is true anabolism. +DUMMY/LJ034-0194.wav|Their testimony is of probative value, however, because their limited description is consistent with that of the man who has been found by the Commission, +DUMMY/LJ039-0015.wav|and testified that a few days before her husband's departure from Dallas to New Orleans on April twenty-four, nineteen sixty-three, +DUMMY/LJ007-0037.wav|In the midst of the noisy and blasphemous talk no one could pursue his meditations; any who tried to pray became the sport and ridicule of his brutal fellows. +DUMMY/LJ033-0091.wav|Frazier told the Commission, quote, +DUMMY/LJ029-0056.wav|Although there was no mention in PRS files of the demonstration in Dallas against Ambassador Adlai Stevenson on October twenty-fourth, +DUMMY/LJ026-0009.wav|The fact is that they are on the border line, are neither plants nor animals but simply organisms. +DUMMY/LJ030-0090.wav|Vice President Johnson sat on the right-hand side of the rear seat, Mrs. Johnson in the center, and Senator Yarborough on the left. +DUMMY/LJ035-0143.wav|and she actually came down the stairs several minutes after Oswald and after Truly and Baker as well. +DUMMY/LJ021-0071.wav|although millions of hitherto underprivileged workers are today far better paid than ever before. +DUMMY/LJ001-0122.wav|that he has a five, an eight, or a three before him, unless the press work is of the best: +DUMMY/LJ034-0108.wav|man in his early thirties, fair complexion, slender, but neat, neat slender, possible five foot ten +DUMMY/LJ011-0278.wav|addressed him, insisting that he should forthwith sign a cheque for the eight hundred pounds of the Canning inheritance still uninvested, +DUMMY/LJ015-0280.wav|was the return from transportation of an old "pal" and confederate, who brought with him some bills of exchange. +DUMMY/LJ030-0254.wav|Forrest V. Sorrels, special agent in charge of the Dallas office, was the first Secret Service agent to return to the scene of the assassination, +DUMMY/LJ048-0218.wav|However, the Commission does not believe that agents stationed in a car behind the Presidential car, +DUMMY/LJ016-0062.wav|She suffered him to pass downstairs. +DUMMY/LJ001-0086.wav|are dazzling and unpleasant to the eye owing to the clumsy thickening and vulgar thinning of the lines: +DUMMY/LJ030-0100.wav|three cars for press photographers, an official party bus for White House staff members and others, and two press buses. +DUMMY/LJ002-0027.wav|and this number did not include the debtors, a numerous class, who were still committed to Newgate pending the completion of the White Cross Street prison. +DUMMY/LJ002-0034.wav|five. The master felons' side. six. The female felons' side. seven. The state side. +DUMMY/LJ005-0008.wav|they were followed by a crowd of reckless boys, who jeered at and insulted them. +DUMMY/LJ032-0063.wav|Since the rifle was shipped from Chicago on March twenty, nineteen sixty-three, it was received in Dallas during the period when Oswald rented and used the box. +DUMMY/LJ001-0176.wav|because the modern practice is to disregard the relation between the printing and the ornament altogether, +DUMMY/LJ040-0172.wav|Mrs. Siegel concluded that Lee, quote, just felt that his mother never gave a damn for him. +DUMMY/LJ018-0271.wav|and was rendered insensible by chloroform, after which he was bound and the precious stones stolen. +DUMMY/LJ010-0204.wav|to wear a black crape mask, to obey punctually the orders of their commander-in-chief, +DUMMY/LJ022-0032.wav|to go fishing or back home to Hyde Park, so that I can have a chance to think quietly about the country as a whole. +DUMMY/LJ015-0225.wav|This was at length effected by Agar and Pierce. +DUMMY/LJ004-0002.wav|While Mrs. Fry was diligently engaged upon her self-imposed task in Newgate, +DUMMY/LJ015-0062.wav|but in neither did the sums misappropriated reach quite the same high figure. +DUMMY/LJ041-0203.wav|to engage in activities on behalf of the Fair Play for Cuba Committee in the summer of nineteen sixty-three, +DUMMY/LJ011-0156.wav|While on a visit to Macclesfield he heard by chance of Miss Turner, and that she would inherit all her father's possessions. +DUMMY/LJ022-0108.wav|For many months preparations have been under way. +DUMMY/LJ007-0007.wav|had provided work for those who had hitherto passed their time in total idleness, and had made the treatment of female transports on the way to New South Wales +DUMMY/LJ048-0107.wav|Love Field and the Trade Mart. +DUMMY/LJ024-0103.wav|Oh! I was for an amendment all right, but this amendment you proposed is not the kind of amendment that I was thinking about. +DUMMY/LJ017-0077.wav|The brother too died conveniently, but the life office took some exception to the manner of the death, and hesitated to disburse the funds claimed by Palmer. +DUMMY/LJ044-0136.wav|an event which his wife thought upset him and as a result of which, quote, he became less active, he cooled off a little, end quote. +DUMMY/LJ017-0200.wav|but a more careful investigation attributed it beyond doubt to over-doses of colchicum. +DUMMY/LJ004-0071.wav|Nor even when found guilty and his liberty forfeited did his privileges cease. The law appointed a suitable punishment for the offense; +DUMMY/LJ011-0080.wav|His first forgery was the marvelous imitation of the signature of the Hon. Mr. Neville, M.P., who wrote an extremely cramped and curious hand. +DUMMY/LJ002-0168.wav|The lad in question was found in Coldbath Fields prison, to which he had been sent for a month in default of paying a fine of forty shillings. +DUMMY/LJ010-0270.wav|exactly tallied with that of the deformed person "wanted" for the assault on the Queen. +DUMMY/LJ025-0043.wav|The essential character of vegetable life is the exhalation of oxygen, which is effected through the agency of light. +DUMMY/LJ016-0315.wav|Colonel (now Sir Edmund) Henderson was strongly in favor of them, +DUMMY/LJ019-0044.wav|The south and west of England were also very laggard, and many years were still to elapse before the prisons in these parts were properly reconstituted. +DUMMY/LJ026-0118.wav|Fat and carbohydrate foods, then, never become living matter. +DUMMY/LJ015-0081.wav|He conceived the idea of transferring shares, bogus shares from a person who held none, to any one who would buy them in the open market. +DUMMY/LJ035-0199.wav|but it was probably no earlier than one:twenty-two p.m., the time when the rifle was found. +DUMMY/LJ023-0043.wav|I want to talk with you very simply about the need for present action in this crisis +DUMMY/LJ023-0026.wav|Individual or local or state effort alone cannot protect us in nineteen thirty-seven any better than ten years ago. +DUMMY/LJ019-0220.wav|the uses of Newgate were narrowed almost entirely to those of a prison of detention. +DUMMY/LJ048-0167.wav|buildings are not checked either by Secret Service agents or by any other law enforcement officers at the request of the Secret Service. +DUMMY/LJ012-0126.wav|The next year twelve thousand sovereigns were cleverly stolen in the Mile End Road. +DUMMY/LJ025-0111.wav|But has the advance of biology simply tended to break down old distinctions without establishing new ones? +DUMMY/LJ035-0202.wav|Fingerprint and palmprint evidence establishes that Oswald handled two of the four cartons next to the window +DUMMY/LJ033-0068.wav|Mrs. Paine testified, quote, +DUMMY/LJ009-0071.wav|The burglar is heard to mutter an angry oath. +DUMMY/LJ013-0102.wav|and thence to Halifax, whither he followed them, accompanied by a confidential clerk from the bank. +DUMMY/LJ004-0034.wav|Moreover, the laws applied more particularly to county jurisdictions. +DUMMY/LJ013-0206.wav|Next morning he made a full confession in presence of his attorney, and the governor, Mr. Cope. +DUMMY/LJ037-0008.wav|Two witnesses testified that Oswald resembled the man they had seen. +DUMMY/LJ031-0007.wav|Chief of Police Curry and police motorcyclists at the head of the motorcade led the way to the hospital. +DUMMY/LJ029-0063.wav|discussed in chapter six of this report. +DUMMY/LJ004-0153.wav|In eighteen eighteen the old jail was so densely packed that it was nearly impossible to pass through the yards for the throng. +DUMMY/LJ013-0027.wav|He crept along the coast close in shore, looking for a quiet spot to cast away the ship, +DUMMY/LJ008-0169.wav|She was killed, but in the act of falling she forced her child into the arms of a man near her, and implored him in God's name to save it; +DUMMY/LJ006-0238.wav|"There was a disturbance in the transport yard yesterday evening, and the police were called in. +DUMMY/LJ028-0259.wav|That same year, when Darius Hystaspes came to the Persian throne, the Babylonians openly rebelled. +DUMMY/LJ014-0155.wav|Now she abused the jury, now called Manning a vagabond, +DUMMY/LJ017-0161.wav|But probably the lump sum was the bait, or perhaps Smethurst wished to return to his temporarily deserted first wife. +DUMMY/LJ012-0296.wav|The execution was, as usual, attended by an immense concourse, and Greenacre died amidst the loudest execrations. +DUMMY/LJ020-0064.wav|Breakfast Biscuit. +DUMMY/LJ050-0035.wav|Nevertheless, such a breach, in which so many agents participated, +DUMMY/LJ002-0182.wav|the Fleet, and the Marshalsea prisons especially devoted to them, +DUMMY/LJ030-0224.wav|heard noises that sounded like firecrackers and ran toward the President's limousine. +DUMMY/LJ036-0093.wav|If Oswald had made such a statement, Captain Fritz and others present would probably have remembered it. +DUMMY/LJ029-0020.wav|and had stopped in El Paso to discuss the proposed visit and other matters with the Vice President and the Governor. +DUMMY/LJ007-0078.wav|As the law stood since the passing of the ninth George the fourth c. forty, any two justices might remove a prisoner found to be insane, either on commitment +DUMMY/LJ010-0279.wav|I shall mention briefly one more case, in which, however, there was no murderous intent, before I pass on to other crimes. +DUMMY/LJ003-0215.wav|from Friday to Monday, so as to include one Sunday, on which day there was a special service for the condemned in the prison chapel. +DUMMY/LJ007-0183.wav|traversing where it was possible the statements of the inspectors, and offering explanation and palliation of such evils as could not be denied. +DUMMY/LJ014-0274.wav|Rumors were abroad that serious defalcations had been discovered in one of the insurance offices, +DUMMY/LJ042-0181.wav|Oswald realized that, quote, their thinking and education will be steeped in the traditions of those systems +DUMMY/LJ010-0073.wav|he found himself rich in a small landed property, which he presently sold to a man who became bankrupt before he had paid over the purchase money. +DUMMY/LJ028-0354.wav|And he honored Zopyrus greatly; year by year he presented him with all the gifts which are held in most esteem among the Persians; +DUMMY/LJ040-0239.wav|Before the court took any action, the Oswalds left New York in January of nineteen fifty-four, +DUMMY/LJ030-0046.wav|and occupied by Secret Service Agents Sorrels and Lawson and by Dallas County Sheriff J. E. Decker. +DUMMY/LJ018-0365.wav|Kate Webster followed the same course; but these pleas of pregnancy are not common now-a-days. +DUMMY/LJ021-0011.wav|Men may differ as to the particular form of governmental activity +DUMMY/LJ023-0095.wav|And three other justices agreed with him. +DUMMY/LJ001-0036.wav|But about the same year Mentelin at Strasburg began to print in a type which is distinctly Roman; +DUMMY/LJ022-0045.wav|Therefore, let us keep our minds on two or three simple, essential facts in connection with this problem of unemployment. +DUMMY/LJ043-0043.wav|When they arrived from the Soviet Union, Oswald and his family lived at first with his brother Robert. +DUMMY/LJ010-0238.wav|but draw it back again, exclaiming presently, "I wish I had done it." +DUMMY/LJ015-0156.wav|Down at Weybridge, where he had a country place, his name was long remembered with gratitude by the poor. +DUMMY/LJ032-0067.wav|Postal Inspector Harry D. Holmes of the Dallas Post Office testified, however, that when a package is received for a certain box, +DUMMY/LJ012-0175.wav|Weare was supposed to carry a "private bank" about with him in a pocket in his under waistcoat. +DUMMY/LJ029-0208.wav|It was fashioned after the "wanted" circulars issued by law enforcement agencies. +DUMMY/LJ015-0094.wav|All this time his position was one of extreme insecurity. He was standing over a mine which at any moment might explode. +DUMMY/LJ015-0178.wav|The Stock Market was greatly affected, and society, more especially that which frequents Exeter Hall, was convulsed. +DUMMY/LJ013-0084.wav|Burgess fraudulently transferred consols to the above amount, standing in the name of Mr. Oxenford, to another party. +DUMMY/LJ034-0171.wav|Edwards said, quote, Look at that guy there in that window, end quote, +DUMMY/LJ040-0198.wav|On the other hand he also told her that he wished his mother had been more firm with him in her attempts to get him to return to school. +DUMMY/LJ025-0088.wav|Cuvier himself practically gives up his second distinctive mark when he admits that it is wanting in the simpler animals. +DUMMY/LJ005-0215.wav|The regular daily visitation of the chaplain was also insisted upon. +DUMMY/LJ024-0009.wav|By bringing into the judicial system a steady and continuing stream of new and younger blood, I hope, first, +DUMMY/LJ035-0077.wav|He placed the rifle on the floor near the site where Oswald's rifle was actually found after the shooting. +DUMMY/LJ001-0046.wav|their type is on the lines of the German and French rather than of the Roman printers. +DUMMY/LJ002-0266.wav|His bequest, which was charged upon his manor at Goring, Oxon, and hence called the Oxford Charity, +DUMMY/LJ050-0108.wav|The Secret Service's liaison with the agencies that supply information to it has been too casual. +DUMMY/LJ015-0108.wav|where a fish curry and a brace of partridges were set before him, +DUMMY/LJ006-0230.wav|Often the victim, goaded to madness, attacked his tormentors; a fight was then certain to follow. +DUMMY/LJ024-0073.wav|In this way I propose to enforce a sound public policy by law +DUMMY/LJ019-0038.wav|The cost in each varied considerably, the general average being from one hundred twenty pounds to one hundred thirty pounds per cell. +DUMMY/LJ025-0106.wav|which contain no chlorophyll and are not green, are always, so far as respiration is concerned, in the exact position of animals. +DUMMY/LJ040-0114.wav|Relations soon became strained, however, so in late September Lee and his mother moved to their own apartment in the Bronx. +DUMMY/LJ028-0262.wav|having first set apart their mothers, each man chose besides out of his whole household one woman whomsoever he pleased; +DUMMY/LJ010-0040.wav|Men like Mobbs, the miscreant known as "General Haynau" on account of his blood-thirstiness, still murdered their wives; +DUMMY/LJ019-0133.wav|The condition of that prison in eighteen fifty may be gathered from the pages of the report. Not much had been done to remedy the old defects; +DUMMY/LJ028-0405.wav|Many of the ten thousand people living in Hillah still gain their livelihood by digging the bricks from the ruins to sell to the modern builders. +DUMMY/LJ003-0342.wav|and checked in its efforts towards reform by the prohibitory costliness of the land about Newgate. +DUMMY/LJ042-0080.wav|My request for the revoking of my American citizenship +DUMMY/LJ028-0339.wav|he threw open the Cissian and Belian gates, and admitted the enemy. +DUMMY/LJ047-0009.wav|who interviewed Oswald after his return from the Soviet Union and prior to November twenty-two, nineteen sixty-three, +DUMMY/LJ017-0240.wav|Immediately after the murder cases of champagne, which formed part of the cargo, were brought on deck and broached; +DUMMY/LJ045-0073.wav|Oswald once struck his wife because of a letter which she wrote to a former boyfriend in Russia. +DUMMY/LJ041-0144.wav|Oswald would have had other and more favorable opportunities to strike at the Governor than on this occasion when, as a member of the President's party, +DUMMY/LJ017-0141.wav|could not divest his mind of serious doubt, and of which the murderer got the benefit. +DUMMY/LJ013-0185.wav|Three days later a close search of the butler's pantry produced fresh circumstantial evidence. +DUMMY/LJ018-0023.wav|Marylebone; while within a few days Mr. Briggs' gold chain was traced to a jeweler's in Cheapside, +DUMMY/LJ041-0054.wav|It probably contributed greatly to the general dissatisfaction which he exhibited with his environment, +DUMMY/LJ033-0211.wav|and the obvious bulk of the package which he intended to bring to work the next day; +DUMMY/LJ016-0413.wav|No less than sixty-seven persons were present, admitted by special permission of the sheriff. +DUMMY/LJ002-0099.wav|to which access was granted rarely and as a great favor. But they were allowed the luxury of drink -- if they could pay for it. +DUMMY/LJ035-0016.wav|He saw pigeons flutter upward. He was not certain, quote, but I am pretty sure they came from the building right on the northwest corner, end quote. +DUMMY/LJ019-0111.wav|healthful, easily learnt, and well adapted to the circumstances of unskilled laborers. +DUMMY/LJ044-0081.wav|to make himself and his activities appear far more important than they really were. +DUMMY/LJ002-0005.wav|The prison population fluctuated a great deal, +DUMMY/LJ013-0253.wav|with a son of Good's by a former wife. Another wife, real or fictitious, existed in Spitalfields, +DUMMY/LJ042-0038.wav|and religion and education are used as a tool to suppress what would otherwise be a population questioning their government's unfair +DUMMY/LJ041-0119.wav|stating that he would, quote, employ all means to right this gross mistake or injustice, end quote. +DUMMY/LJ043-0072.wav|Even though he told his wife that he had been fired, he voluntarily left on October eight, nineteen sixty-two, and moved to Dallas. +DUMMY/LJ037-0242.wav|Police Capt. W. R. Westbrook and several other officers concentrated their search along Jefferson Boulevard. +DUMMY/LJ002-0180.wav|Before dealing with the debtors in Newgate, I may refer incidentally +DUMMY/LJ037-0224.wav|At one:twenty-two p.m. the Dallas police radio described the man wanted for the murder of Tippit as, quote, a white male about thirty, +DUMMY/LJ027-0044.wav|Turning to more fundamental resemblances, such as the wing of a bat and the wing of a bird, +DUMMY/LJ012-0022.wav|He early saw the profits to be made out of purchasing stolen goods, but could not embark in it at first for want of capital. +DUMMY/LJ047-0227.wav|He testified that he did not read the newspaper story describing the motorcade route in detail, since he was interested only in the fact +DUMMY/LJ050-0077.wav|and in taking preventive steps. +DUMMY/LJ015-0104.wav|Robson again left him, but only to gather together hastily all the money and valuables on which he could lay his hands, with which he left the house. +DUMMY/LJ028-0373.wav|The Hebrew exiles, whose ancestors Nebuchadnezzar had brought from Jerusalem, +DUMMY/LJ047-0106.wav|Oswald, quote, was probably making a self-serving statement in attempting to explain to me why he was distributing this literature, and for no other reason, +DUMMY/LJ009-0075.wav|All have sung 'the Lamentation of a Sinner,' and have seemed to pray 'especially for those now awaiting the awful execution of the law.' +DUMMY/LJ028-0102.wav|There is scarcely a royal record from his reign which is not chiefly occupied with descriptions of his building operations. +DUMMY/LJ005-0224.wav|but nothing had been done to improve the prison buildings. +DUMMY/LJ013-0215.wav|"He appeared to die instantly," said the murderer, in conclusion. +DUMMY/LJ042-0180.wav|after which the survivors would, quote, seek an alternative opposed to those systems which have brought them misery, end quote. +DUMMY/LJ034-0016.wav|Behind these boxes was another carton placed on the floor on which a man sitting +DUMMY/LJ003-0271.wav|Some of the worst and most extensive burglaries were planned there. +DUMMY/LJ010-0240.wav|The Queen was apprised of the danger, and begged not to go abroad; +DUMMY/LJ015-0286.wav|A confederate followed the emissary closely, +DUMMY/LJ002-0117.wav|Neild gives, on the authority of Mr. Burchell, the under sheriff of Middlesex, +DUMMY/LJ042-0119.wav|Under the entry for August to September of that year he wrote, quote, +DUMMY/LJ006-0096.wav|The only convenience for personal ablutions were the pumps in the yards, and the far-off baths in the condemned or press-yard. +DUMMY/LJ019-0234.wav|The old buildings were entirely disused, and the whole of the inmates of Newgate were kept constantly in separate confinement. +DUMMY/LJ030-0135.wav|Mrs. Connally, elated by the reception, turned to President Kennedy and said, quote, Mr. President, you can't say Dallas doesn't love you. +DUMMY/LJ009-0204.wav|A long procession was formed, headed by constables, who cleared the way with their staves. +DUMMY/LJ046-0132.wav|in nineteen sixty-three the total exceeded thirty-two thousand items. +DUMMY/LJ050-0274.wav|made certain recommendations which it believes would, if adopted, +DUMMY/LJ023-0116.wav|In our courts we want a government of laws and not of men. +DUMMY/LJ029-0037.wav|Advance preparations for President Kennedy's visit to Dallas were primarily the responsibility of two Secret Service agents: +DUMMY/LJ026-0079.wav|After passing through manifold but imperfectly known processes, the elements of these substances finally reappear as starch, +DUMMY/LJ043-0089.wav|to a commercial advertising photography firm in Dallas, where he was employed as a trainee starting October twelve, nineteen sixty-two. +DUMMY/LJ014-0266.wav|No one cared too closely to inquire into the sources of wealth. Some said he was a fortunate speculator in stocks, +DUMMY/LJ037-0189.wav|the only revolver made by Smith and Wesson with this serial number. When first manufactured, it had a five-inch barrel. +DUMMY/LJ002-0100.wav|A recent reform had closed the tap kept by the jailer within the precincts, but +DUMMY/LJ036-0036.wav|At about six:thirty p.m. on the day of the assassination, McWatters viewed four men in a police lineup. +DUMMY/LJ016-0325.wav|The judge of the Admiralty Court, the Right Hon. Stephen Lushington, the Right Hon. James Moncrieff, +DUMMY/LJ012-0051.wav|While thus incarcerated he managed to escape from custody, but not actually from jail, by an ingenious contrivance which is worth mentioning. +DUMMY/LJ028-0503.wav|and decorated here and there with large reliefs representing bulls and lions and dragons, +DUMMY/LJ029-0091.wav|The Motorcade Route. +DUMMY/LJ010-0162.wav|The cowardly crime was probably encouraged by the fearless and confiding manner in which the Queen, +DUMMY/LJ044-0144.wav|Interest in Cuba +DUMMY/LJ021-0207.wav|I am not for a return to that definition of liberty under which for many years a free people were being gradually regimented +DUMMY/LJ034-0062.wav|Although a person could handle a carton and not leave identifiable prints, +DUMMY/LJ044-0058.wav|that an anti-Castro organization had maintained offices there for a period ending early in nineteen sixty-two. +DUMMY/LJ047-0167.wav|On November four, Hosty telephoned the Texas School Book Depository and learned that Oswald was working there +DUMMY/LJ046-0082.wav|must depend upon the utmost cooperation and understanding from the public and the President. +DUMMY/LJ002-0270.wav|The latter's salary, with fees, the rent of the tap and of the chandler's shop, amounted to about six hundred pounds a year. +DUMMY/LJ046-0099.wav|thoroughly professional personnel, using the best technical equipment that can be devised. +DUMMY/LJ030-0223.wav|Special Agent Ready, on the right front running board of the Presidential follow-up car, +DUMMY/LJ009-0270.wav|A worse case still was that of William Bousfield, who, when awaiting execution for murder, about the same date, +DUMMY/LJ032-0056.wav|bearing serial number C two seven six six. +DUMMY/LJ006-0256.wav|Visitors were still permitted to come with supplies on given days of the week, about the only limitation being that the food should be cooked, and cold; +DUMMY/LJ006-0104.wav|but its issue was precarious, and dependent on the good will of the wardsmen, who measured out the portions to each according to his eye, +DUMMY/LJ005-0138.wav|that in four prisons, which at one time of the year contained one thousand three hundred eight prisoners, there were only sixty-eight sleeping rooms or cells, +DUMMY/LJ010-0291.wav|Pate was found guilty, and sentenced to seven years' transportation, the judge, Baron Alderson, abstaining from inflicting the penalty of whipping, +DUMMY/LJ032-0028.wav|on a coupon clipped from the February nineteen sixty-three issue of the American Rifleman magazine. +DUMMY/LJ002-0164.wav|for the paltry sum of four pence, with costs of seven shillings, six pence. +DUMMY/LJ010-0202.wav|Its sworn members, known only to Oxford, and all of them mere shadows, +DUMMY/LJ025-0072.wav|Hence the teleological argument for Cuvier's first diagnostic character +DUMMY/LJ047-0012.wav|the Director and Deputy Director for Plans of the CIA testified concerning that Agency's limited knowledge of Oswald before the assassination. +DUMMY/LJ019-0095.wav|which was yet under full control, and might be made to work corn-mills or prove otherwise productive; +DUMMY/LJ015-0054.wav|Police officers went down at night to Nutfield, near Reigate, and arrested Sir John Paul, but allowed the prisoner to sleep there. +DUMMY/LJ003-0008.wav|Friends arrived about nine a.m., and went out at nine p.m., when as many as two hundred visitors have been observed leaving the debtors' yards at one time. +DUMMY/LJ043-0057.wav|he attempted for the first time to learn something about his family background when he went to New Orleans in April of nineteen sixty-three. +DUMMY/LJ042-0246.wav|In July of nineteen sixty-three, Oswald also requested the Soviet Union to provide a visa for his return to that country. +DUMMY/LJ008-0203.wav|Numbers of ladies were present, although the public feeling was much against their attendance. +DUMMY/LJ008-0231.wav|Hush! the unceasing murmur of the mob now breaks into a loud deep roar, +DUMMY/LJ010-0071.wav|It was during this period that he was said to have imbibed his revolutionary ideas. +DUMMY/LJ035-0203.wav|and also handled a paper bag which was found near the cartons. +DUMMY/LJ025-0137.wav|but it suggests a suspicion of their identity which needs careful testing. +DUMMY/LJ036-0147.wav|The five hundred block of North Beckley is five blocks south of the roominghouse. +DUMMY/LJ018-0041.wav|The case was one of circumstantial evidence, but, as Sir Robert Collyer the Solicitor-General pointed out, +DUMMY/LJ026-0021.wav|nitrogen is obtained from simple salts and the nutritive processes result in deoxidation; +DUMMY/LJ034-0054.wav|In addition, Mandella was of the opinion that the print taken from the carton on the floor +DUMMY/LJ050-0263.wav|it will continue to rely in many respects upon the greater resources of the Office of Science and Technology and other agencies. +DUMMY/LJ013-0125.wav|the Junior United Service, the Union, Reform, Colonial, and Erechtheum clubs. +DUMMY/LJ001-0023.wav|and was in fact the kind of letter used in the many splendid missals, psalters, etc., produced by printing in the fifteenth century. +DUMMY/LJ009-0216.wav|Hanging in chains upon the gibbet which had served for the execution, +DUMMY/LJ013-0068.wav|and a person calling herself Miss Slack duly attended at the bank, where the money was handed over to her in proper form. +DUMMY/LJ002-0200.wav|The marshal was supposed to be resident either within the prison or the rules. +DUMMY/LJ047-0035.wav|Oswald's file at the Department of State Passport Office was reviewed in June nineteen sixty-two. +DUMMY/LJ025-0045.wav|They get rid of the superfluous hydrogen and carbon and accumulate nitrogen. +DUMMY/LJ033-0132.wav|I didn't pay too much attention the way he was walking because I was walking along there looking at the railroad cars and watching the men on the diesel switch them cars +DUMMY/LJ032-0172.wav|Marina Oswald testified that she thought her husband wore this shirt to work on that day. +DUMMY/LJ025-0011.wav|is Huxley's famous essay, "The Border Territory Between the Animal and Vegetable Kingdoms," written in eighteen seventy-six, +DUMMY/LJ007-0038.wav|Owing to the repeated entreaties of the criminals who could hardly hope to escape the gallows, some show of classification was carried out, +DUMMY/LJ006-0188.wav|The days were passed in idleness, debauchery, riotous quarreling, immoral conversation, +DUMMY/LJ030-0073.wav|Presidential follow-up car. +DUMMY/LJ041-0139.wav|It should be noted, however, that at the time Oswald fired the shots at the Presidential limousine the Governor occupied the seat in front of the President, +DUMMY/LJ039-0199.wav|The three FBI experts each fired three shots from the weapon at fifteen yards in six, seven, and nine seconds, +DUMMY/LJ013-0049.wav|A notorious miser, Robert Smith, had recently died in Seven Dials, where he had amassed a considerable fortune. +DUMMY/LJ048-0250.wav|In Dallas, one of the nine agents was assigned to assist in security measures at Love Field, and four had protective assignments at the Trade Mart. +DUMMY/LJ017-0068.wav|and public opinion at the termination of the trial coincided with the verdict of the jury. +DUMMY/LJ014-0150.wav|and expressed the utmost contempt for her husband, whom she never intended to acknowledge or speak to again. +DUMMY/LJ043-0167.wav|Even after his wife told him to destroy the notebook +DUMMY/LJ006-0142.wav|The matron deposed to having seen the gates-woman "exceedingly drunk," and having been insulted by her. +DUMMY/LJ021-0047.wav|The country now enjoys the safety of bank savings under the new banking laws, +DUMMY/LJ019-0040.wav|the difficulty of access, and the admitted necessity of giving architectural importance to this the national model prison. +DUMMY/LJ015-0314.wav|He was in person a short, square-built man of gentlemanly address, sharp and shrewd in conversation and manner. +DUMMY/LJ037-0230.wav|Mrs. Davis gave Poe the same general description: a, quote, white male in his early twenties, around five foot seven inches +DUMMY/LJ042-0029.wav|While there is no proof that he saved fifteen hundred dollars, as he claimed, +DUMMY/LJ013-0180.wav|The intention of the real murderer to shift the crime to burglars was evident although futile, +DUMMY/LJ013-0228.wav|In the pocket of the coat Mr. Cope, the governor, found a neatly-folded cloth, and asked what it was for. +DUMMY/LJ039-0248.wav|possessed the capability with a rifle which would have enabled him to commit the assassination. +DUMMY/LJ046-0209.wav|In the absence of more specific instructions, other Federal agencies interpreted the Secret Service's informal requests +DUMMY/LJ016-0124.wav|Cutting the cord which fastened the two legs of the step ladder, he opened them out and made one long length; +DUMMY/LJ030-0203.wav|From the left front running board of the President's follow-up car, +DUMMY/LJ031-0033.wav|Then he experienced his first sensation of pain, which became excruciating. +DUMMY/LJ013-0067.wav|a note purporting to be from Miss Slack was addressed to the governor of the Bank of England, begging that the said stock might be handed over to her, +DUMMY/LJ005-0221.wav|and the panes were filled in with oiled paper, an insufficient protection against the weather; +DUMMY/LJ016-0142.wav|who permitted him to go up on to the roof of the old wards, in order to throw water for flushing purposes down a shoot. +DUMMY/LJ029-0057.wav|nineteen sixty-three, Lawson inquired about the incident and obtained through the local police photographs of some of the persons involved. +DUMMY/LJ015-0187.wav|Redpath passed away into the outer darkness of a penal colony, where he was still living a year or two back +DUMMY/LJ007-0191.wav|has been productive of at least some advantage, +DUMMY/LJ023-0080.wav|the Court has more and more often and more and more boldly +DUMMY/LJ015-0134.wav|had assets in the shape of land, house, furniture, pictures, and objets d'art to the value of fifty thousand pounds. +DUMMY/LJ012-0105.wav|The police were of opinion that these robberies were both the work of the same hand. +DUMMY/LJ016-0366.wav|and placed decently in a shell by the workmen who removed the gallows. +DUMMY/LJ026-0024.wav|There is no special excretory organ, and, except in the case of certain reproductive bodies, there is usually no locomotion. +DUMMY/LJ036-0111.wav|Whaley's memory of the lineup is inaccurate. There were four men altogether, not six men, in the lineup with Oswald. +DUMMY/LJ030-0050.wav|The Presidential limousine. +DUMMY/LJ049-0089.wav|The murder of Federal judges, U.S. attorneys and marshals, and a number of other specifically designated +DUMMY/LJ018-0299.wav|Long before they came to maturity the forgers hoped to be well beyond arrest. They had, moreover, sought to destroy all clue. +DUMMY/LJ024-0143.wav|You and I will do our part. +DUMMY/LJ028-0515.wav|Such were the walls of Babylon, +DUMMY/LJ037-0009.wav|One witness felt he was too distant from the gunman to make a positive identification. +DUMMY/LJ029-0048.wav|On November eight, nineteen sixty-three, +DUMMY/LJ034-0045.wav|Since cartons absorb perspiration, powder can successfully develop a print on such material only within a limited time. +DUMMY/LJ040-0006.wav|The Commission has considered many possible motives for the assassination, including those which might flow from Oswald's commitment to Marxism or communism, +DUMMY/LJ029-0213.wav|quote, formed strictly for the purpose of having a name to put in the paper, end quote, +DUMMY/LJ047-0053.wav|Fain determined that nothing further remained to be done at that time and recommended that the case be placed in a closed status. +DUMMY/LJ037-0238.wav|she identified a picture of Oswald as being the same person she saw on November twenty-two. +DUMMY/LJ001-0014.wav|And it was a matter of course that in the Middle Ages, when the craftsmen took care that beautiful form should always be a part of their productions whatever they were, +DUMMY/LJ008-0253.wav|Then came the cry of 'Hats off!' and 'Down in front!' as at a theatre. +DUMMY/LJ014-0035.wav|in a lonely part of Hampstead. +DUMMY/LJ011-0049.wav|Fauntleroy constantly groaned aloud while the sermon proceeded, and contemporary reports declared +DUMMY/LJ001-0183.wav|Therefore, granted well-designed type, due spacing of the lines and words, and proper position of the page on the paper, +DUMMY/LJ013-0149.wav|The circumstances of this purchase of brilliants from a stranger at such an inadequate price was strongly commented upon at Ker's trial. +DUMMY/LJ041-0025.wav|It has been suggested that this misspelling of names, apparently on a phonetic basis, was caused by a reading-spelling disability +DUMMY/LJ030-0132.wav|As the motorcade approached the intersection of Houston and Elm Streets, there was general gratification in the Presidential party about the enthusiastic reception. +DUMMY/LJ035-0208.wav|Oswald's known actions in the building immediately after the assassination are consistent with his having been at the southeast corner window of the sixth floor +DUMMY/LJ022-0168.wav|who have little protection under the old laws against what used to be called frenzied finance. +DUMMY/LJ009-0173.wav|from eighteen thirty-two to eighteen forty-four not a single person had been executed in the metropolis except for this the gravest crime. +DUMMY/LJ027-0164.wav|At this stage it breathes equally by lungs and by gills -- i.e., both air and water. +DUMMY/LJ042-0138.wav|His return to the United States publicly testified to the utter failure of what had been the most important act of his life. +DUMMY/LJ020-0098.wav|Having made out your rolls and tucked them up snugly for the final rise, return to your chamber for a comfortable bath and toilet. +DUMMY/LJ032-0088.wav|Post Office Box three zero zero one six, New Orleans +DUMMY/LJ043-0096.wav|became involved in conflicts, some of which were fairly heated, with his fellow employees. +DUMMY/LJ003-0003.wav|Prisoners were committed to it quite without reference to its capacity. +DUMMY/LJ050-0277.wav|with the active cooperation of the responsible agencies and with the understanding of the people of the United States in their demands upon their President, +DUMMY/LJ003-0348.wav|Again, there was the new Millbank penitentiary now ready for occupation. +DUMMY/LJ043-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. +DUMMY/LJ016-0254.wav|Executions long continued to be in public, in spite of remonstrance and reprobation. +DUMMY/LJ011-0162.wav|and it stated that Mrs. Turner had been stricken with paralysis. +DUMMY/LJ012-0028.wav|that he was soon admitted to be one of the best judges known of all kinds of property, from a glass bottle to a five hundred guinea chronometer. +DUMMY/LJ008-0233.wav|the wide dark sea of heads is all at once in motion; +DUMMY/LJ018-0016.wav|T. Briggs, Esq., Robarts and Co., Lombard Street. +DUMMY/LJ029-0047.wav|maintains records of people who have threatened the President or so conducted themselves as to be deemed a potential danger to him. +DUMMY/LJ015-0057.wav|All three were tried at the Central Criminal Court, and sentenced to fourteen years' transportation, passing some time in Newgate en route. +DUMMY/LJ034-0181.wav|and he had brown hair, a slender face and neck with light complexion, and looked to be twenty-two or twenty-four years old. +DUMMY/LJ027-0071.wav|But much more remains which is not only suggestive, but demonstrative of such origin. +DUMMY/LJ034-0023.wav|The carton on the windowsill and the large carton below the window contained no prints which could be identified as being those of Lee Harvey Oswald. +DUMMY/LJ048-0130.wav|the Commission believes that standard procedures in use for many years and applied in all parts of the country may not be sufficient. +DUMMY/LJ010-0107.wav|and a military sash. +DUMMY/LJ036-0095.wav|but the Commission concluded that this man was not Lee Harvey Oswald, +DUMMY/LJ015-0248.wav|with Ostend tickets previously procured, returned to London without mishap, and by degrees disposed of much of the stolen gold. +DUMMY/LJ018-0284.wav|Dresden, Bordeaux, Marseilles, and Lyons. +DUMMY/LJ018-0128.wav|Brown took several sheets, and then was detected by Brewer, a fellow-workman of superior grade, +DUMMY/LJ038-0221.wav|and the second month's rent was paid on either April two or April three. +DUMMY/LJ017-0084.wav|It was there that Cook was first taken ill with violent retchings and vomitings, all dating from visits of Palmer, who brought him medicines and food. +DUMMY/LJ014-0015.wav|picked up a coat-button, which afterwards played an important part in the identification of the murderer. +DUMMY/LJ043-0031.wav|I am surprised that he didn't do something worse, end quote. +DUMMY/LJ011-0262.wav|A more complicated and altogether most extraordinary case of assault, with intent to extort money, occurred a few years later. +DUMMY/LJ005-0115.wav|or an imaginary boundary line, and nothing prevented parties from passing to either side +DUMMY/LJ026-0013.wav|Parker's definition of animals and plants, +DUMMY/LJ004-0160.wav|the uproar of oaths, complaints, and obscenity, +DUMMY/LJ002-0187.wav|Every new-comer was entitled to a "chummage" ticket, but did not always get it, +DUMMY/LJ047-0168.wav|and that he had given as his address Mrs. Paine's residence in Irving. +DUMMY/LJ017-0110.wav|He said nothing, but began to feel uneasy when he found that Cook's betting-book was missing, and that Palmer put it forward +DUMMY/LJ013-0070.wav|Mrs. Hunt had left money in the funds which remained unclaimed, and had been transferred, as in Miss Slack's case. +DUMMY/LJ017-0157.wav|For some months Smethurst and Miss Bankes lived together as man and wife, first in London, and then at Richmond. +DUMMY/LJ025-0112.wav|With a qualification, to be considered presently, the answer to this question is undoubtedly in the affirmative. +DUMMY/LJ013-0230.wav|The next inquiry was how he hoped to open a vein. "With a bit of sharpened stick picked out of the ordinary firewood." +DUMMY/LJ046-0107.wav|Finally, the performance of those charged with the immediate responsibility of protecting the President on November twenty-two is reviewed. +DUMMY/LJ042-0009.wav|While his defection resulted in part from Oswald's commitment to Marxism, +DUMMY/LJ026-0061.wav|These substances, which are dissolved in a watery liquid to form the digestive fluid, are secreted by the walls of the alimentary tube. +DUMMY/LJ043-0150.wav|She indicated that she had no advance knowledge of Oswald's plans, +DUMMY/LJ004-0019.wav|were as yet but imperfectly understood, and such portions of the "improved" jails of that period as were still extant a few years back, +DUMMY/LJ035-0097.wav|When Truly, accompanied by Baker, ran to the rear of the first floor, +DUMMY/LJ025-0167.wav|and in the seeds which it produces are exactly equivalent to the weights of the same elements which have disappeared from the materials supplied to the bean during its growth. +DUMMY/LJ032-0126.wav|He examined these prints, as well as photographs of them which the Dallas police had made, and concluded that: +DUMMY/LJ027-0054.wav|Since backboned animals are best known to most readers, they may be taken as an illustration. +DUMMY/LJ012-0274.wav|and that in a moment of anger he knocked her down. +DUMMY/LJ041-0066.wav|his general personality would alienate the group against him, end quote. +DUMMY/LJ027-0045.wav|careful study shows detailed internal as well as external similarities of structure. Such cases are "homologies". +DUMMY/LJ007-0084.wav|The inspectors in the following year, on examining the facts, found that some of these poor creatures had been in confinement for long periods: +DUMMY/LJ043-0006.wav|and he did not establish any close friendships or associations, although it appears that he came to respect George De Mohrenschildt. +DUMMY/LJ008-0241.wav|were vended under the names of notorious murderers, highwaymen, and criminals, +DUMMY/LJ018-0272.wav|Mrs. Tarpey was almost immediately captured and put on her trial, but she was acquitted on the plea that she had acted under the coercion of her husband. +DUMMY/LJ049-0068.wav|A doctor is in the motorcade. +DUMMY/LJ026-0138.wav|Apparently when this reserve supply is finally needed at any point in the plant, it is again changed to glucose and transported thither. +DUMMY/LJ026-0002.wav|Life Processes. Part One. +DUMMY/LJ016-0321.wav|Duly impressed with the weight of evidence in favor of abolition, +DUMMY/LJ009-0205.wav|Then came the newly-formed horse patrol, with drawn cutlasses, +DUMMY/LJ041-0132.wav|He continued his efforts to reverse the discharge by petitioning the Navy Discharge Review Board, +DUMMY/LJ033-0166.wav|Among other tests, the paper and tape were submitted to fiber analysis and spectrographic examination. +DUMMY/LJ027-0120.wav|This reasonable expectation, however, is far from being realized. +DUMMY/LJ019-0369.wav|In the same way, six months' notice was required in cases where the closing of a prison was contemplated; +DUMMY/LJ032-0068.wav|a notice is placed in that box regardless of whether the name on the package is listed on the application as a person entitled to receive mail through that box. +DUMMY/LJ038-0208.wav|I don't remember the exact time, but it was very late. And he told me not to ask him any questions. He only told me he had shot at General Walker. +DUMMY/LJ050-0080.wav|The suggestions of Federal agencies for revision of these guidelines were solicited. +DUMMY/LJ004-0218.wav|a week's allowance at a time, was abolished, and there was a regular scale of daily rations adopted. +DUMMY/LJ009-0191.wav|as we have heard some relate, and with pale and terrified features, to get a breath of air. +DUMMY/LJ035-0189.wav|Special Agent Forrest V. Sorrels of the Secret Service, who had been in the motorcade, +DUMMY/LJ031-0167.wav|Chief Curry drove one automobile occupied by Vice President Johnson, U.S. Representatives Thomas and Thornberry, and Special Agent Youngblood. +DUMMY/LJ013-0159.wav|The fact of the murder was first discovered by the housemaid, +DUMMY/LJ023-0070.wav|for ourselves and our posterity. +DUMMY/LJ009-0015.wav|I will quote an extract from the reverend gentleman's own journal. +DUMMY/LJ011-0179.wav|so that it might become the exclusive property of her husband, "whoever he might be." +DUMMY/LJ007-0178.wav|and when the first municipal authority of our land will be no longer subjected to the reproach of fostering an institution which outrages the rights and feelings of humanity, +DUMMY/LJ014-0180.wav|secure the stock of watches and jewelry, then lock up the place and take on the keys to Mr. Berry's private house in Pimlico. +DUMMY/LJ017-0083.wav|whom he persuaded to go and stay at Rugeley in an hotel just opposite his own house. +DUMMY/LJ001-0019.wav|and which developed more completely and satisfactorily on the side of the "lower-case" than the capital letters; +DUMMY/LJ009-0117.wav|The firmest disbeliever in religion, if he had not lately been irritated by taking part in such a scene as the condemned service in Newgate, +DUMMY/LJ016-0212.wav|Calcraft's salary was more than the proverbial "thirteenpence halfpenny -- hangman's wages." +DUMMY/LJ045-0191.wav|The next day that paper announced the final approval of the Trade Mart as the luncheon site and stated that the motorcade, quote, +DUMMY/LJ026-0078.wav|Entering the chlorophyll bodies, the kinetic energy of sunlight is applied to the decomposition of carbon dioxide and water. +DUMMY/LJ012-0150.wav|Mrs. Abrahams imposed upon her father by abstracting a portion of the dust and selling it on her own account; +DUMMY/LJ016-0249.wav|The strange competition amongst hundreds to succeed Marwood is a strange fact too recently before the public to need mention here. +DUMMY/LJ006-0304.wav|committed by the House of Commons, who had been lodged in the governor's own house. +DUMMY/LJ014-0286.wav|The sentence of the court, one of ten years' transportation, struck the prisoner with dismay. +DUMMY/LJ035-0160.wav|Reconstructing her movements, Mrs. Reid ran the distance three times and was timed in two minutes by stopwatch. +DUMMY/LJ023-0104.wav|And two other justices agreed with him. +DUMMY/LJ031-0126.wav|As President Kennedy and Governor Connally were being removed from the limousine onto stretchers, +DUMMY/LJ006-0030.wav|the great architectural experiment which grew out of the strong representations of Jeremy Bentham and others, and was the first national recognition of the principle +DUMMY/LJ029-0126.wav|to reach the northbound lanes of the Stemmons Freeway is via Elm Street, which Route Number seventy-seven traffic is instructed to follow in this part of the city. +DUMMY/LJ003-0234.wav|The bedding was scanty; fuel and light had to be purchased out of prisoners' private means; clothing was issued but rarely, +DUMMY/LJ016-0153.wav|would make death a certainty, so limited and imperfect are the means generally available. +DUMMY/LJ022-0092.wav|and such projects as they approve will be next submitted to the President who under the Act is required to make final allocations. +DUMMY/LJ048-0005.wav|There was nothing up to the time of the assassination that gave any indication that this man was a dangerous character who might do harm to the President +DUMMY/LJ019-0245.wav|The inquiry was most searching and complete, and the committee spoke plainly in its report. +DUMMY/LJ021-0006.wav|have had to do with industry and labor and with respect to these, certain developments have taken place which I consider of importance. +DUMMY/LJ033-0189.wav|enabled the Commission to conclude that the bag was made from these materials. +DUMMY/LJ036-0043.wav|Riding on the bus was an elderly woman, Mary Bledsoe, who confirmed the mute evidence of the transfer. +DUMMY/LJ030-0189.wav|he realized that something was wrong, and he pressed down on the accelerator as Kellerman said, quote, Get out of here fast, end quote. +DUMMY/LJ016-0297.wav|The crowd was for a moment awed and stilled by the quiet rapid passage from life to death! +DUMMY/LJ046-0187.wav|In June nineteen sixty-four PRS had arrangements to be notified about the release or escape of approximately one thousand persons. +DUMMY/LJ028-0263.wav|these alone were allowed to live, while all the rest were brought to one place and strangled. +DUMMY/LJ048-0042.wav|There were no Secret Service criteria which specifically required the referral of Oswald's case to the Secret Service; +DUMMY/LJ031-0092.wav|Did you ever have occasion to look at the President's back? +DUMMY/LJ008-0141.wav|the doubtfulness of the issue, or the superior position of the perpetrator, +DUMMY/LJ046-0225.wav|or crank visitors and furnished reports concerning any assassination or attempted assassination of a ruler or other major official anywhere in the world. +DUMMY/LJ016-0136.wav|On the other hand, at the great convict establishments, such is the moral restraint of a systematic discipline, +DUMMY/LJ027-0053.wav|The most striking fact of similar structure among plants and among animals is the existence of a common general plan in any group. +DUMMY/LJ013-0145.wav|Ker came at once to London, and, by the help of the landlord of a public-house in Smithfield and others, disposed of the whole of the diamonds. +DUMMY/LJ019-0002.wav|The time at length approached when a radical and complete change was to come over the old city jail. +DUMMY/LJ018-0227.wav|This will was a deliberate forgery. +DUMMY/LJ043-0135.wav|Oswald shot at Maj. Gen. Edwin A. Walker (Resigned, U.S. Army), +DUMMY/LJ028-0203.wav|Less picturesque than this Hebrew legend is the royal record of Babylon, which fortunately was inscribed upon a clay cylinder from the ruins of the city. +DUMMY/LJ027-0079.wav|a series of either fore or hind limbs of a mammal with one toe (horse), +DUMMY/LJ024-0069.wav|Thus a sound public policy has been defeated. +DUMMY/LJ012-0047.wav|But it was eventually discovered by Mrs. Solomons, a very jealous wife, +DUMMY/LJ030-0218.wav|the right rear tail, when she noticed that I was trying to climb on the car. +DUMMY/LJ013-0260.wav|Good himself remained at large for some weeks. He had tramped as far as Tunbridge, where he obtained work as a bricklayer's laborer; +DUMMY/LJ017-0043.wav|The direful suspicions which surrounded the case filled the whole country with uneasiness and misgiving, +DUMMY/LJ018-0117.wav|Wagner, after conviction, offered to reveal, for a reward of three thousand pounds +DUMMY/LJ011-0270.wav|but on arrival he was met by a young sailor with a letter which begged Mr. Gee to go to Heath's house, as the latter was not well. +DUMMY/LJ010-0024.wav|While the varying conditions of social life thus brought about many changes in the character of offenses against property, +DUMMY/LJ037-0159.wav|Cunningham compared four lead bullets recovered from the body of Patrolman Tippit with test bullets fired from Oswald's revolver. +DUMMY/LJ028-0155.wav|Lumps of bitumen are found in great abundance in this river. +DUMMY/LJ028-0133.wav|He also erected elevated places for walking, of stone; and made it resemble mountains: and built it so that it might be planted with all sorts of trees. +DUMMY/LJ018-0176.wav|Cummings, who had introduced Buncher to Griffiths, was also tried for being in possession of stolen bank paper for improper purposes. +DUMMY/LJ041-0198.wav|His use of those excuses to present himself to the world as a person who was being unfairly treated +DUMMY/LJ016-0159.wav|who contrived to hang himself from a hammock hook only eighteen inches from the ground. +DUMMY/LJ030-0187.wav|When he heard the same noise again, Greer glanced over his shoulder and saw Governor Connally fall. +DUMMY/LJ049-0120.wav|At present, Federal agencies participate only upon the sufferance of the local authorities. +DUMMY/LJ010-0003.wav|I propose now to continue the subject, and to devote a couple of chapters to criminal occurrences of a more recent date, +DUMMY/LJ011-0215.wav|and found guilty, remaining in Lancaster Jail for a couple of months, when they were brought up to the court of King's Bench for judgment. +DUMMY/LJ021-0061.wav|The emergency purpose of the N.R.A. was to put men to work and since its creation more than four million persons have been reemployed, +DUMMY/LJ014-0305.wav|So enormous were his transactions, that they often affected the markets, and caused great fluctuations in prices. +DUMMY/LJ046-0096.wav|Nevertheless the procedures and advice should be accepted if the President wishes to have any security. +DUMMY/LJ015-0068.wav|He began life as a law-writer, earning thereby some fifteen or eighteen shillings a week; +DUMMY/LJ028-0027.wav|For ten days you follow down the river, through little villages and black tent encampments, among scenes of strange Arab life which never lose their charm. +DUMMY/LJ047-0002.wav|Chapter eight. The Protection of the President. Part two. +DUMMY/LJ034-0080.wav|When he reached the first floor, the west elevator -- the one with the gate was not there. +DUMMY/LJ047-0206.wav|I would think his continued association with the Russian Embassy after his return, +DUMMY/LJ009-0154.wav|Mr. Charles Kean the tragedian was also present, +DUMMY/LJ001-0034.wav|They printed very few books in this type, three only; but in their very first books in Rome, beginning with the year fourteen sixty-eight, +DUMMY/LJ009-0284.wav|It was stated in evidence before the Commission on Capital Punishment in eighteen sixty-four, +DUMMY/LJ006-0124.wav|the prisoner official already mentioned, who held the fate of new arrivals as regards location in his hands. +DUMMY/LJ014-0036.wav|Hocker and the brother went to the spot, where the latter left him to meet his sister's seducer alone. +DUMMY/LJ018-0131.wav|All that was taken was handed over to Burnett, or a "woman in black" whom Brown met by appointment at Waterloo station. +DUMMY/LJ005-0161.wav|too much discretion was still left to the magistracy to fill in the details. The legislature only recommended, +DUMMY/LJ018-0060.wav|His crime under this aspect of it was less premeditated, and less atrocious therefore, than that of Lefroy. +DUMMY/LJ025-0128.wav|can be distinguished from those acts of contraction following upon stimuli, which are called "reflex" in animals. +DUMMY/LJ011-0146.wav|and no person ever after suffered death for any variety of this crime. +DUMMY/LJ026-0025.wav|The important point to recognize is that these boundaries are artificial and that there are no scientific frontiers in Nature. +DUMMY/LJ045-0138.wav|In fact, his wife testified that she only said that she would prefer not to receive any more visits from the Bureau +DUMMY/LJ015-0235.wav|and last, but not least, Agar frequently traveled up and down the line to test the false keys he had manufactured with Pierce's assistance. +DUMMY/LJ030-0249.wav|reported that Youngblood was in the rear seat using his body to shield the Vice President before the second and third shots were fired. +DUMMY/LJ048-0061.wav|Under proper procedures knowledge of the pending Presidential visit might have prompted Hosty +DUMMY/LJ003-0249.wav|were imposed in case of misconduct; and when there had been an attempt at escape, +DUMMY/LJ014-0299.wav|that of a prize-fighter named Donovan, tried the same day, and convicted of manslaughter. +DUMMY/LJ045-0220.wav|The Commission does not believe that the relations between Oswald and his wife caused him to assassinate the President. +DUMMY/LJ033-0148.wav|Using a standard chemical method involving silver nitrates +DUMMY/LJ040-0191.wav|Lee told Carro, his probation officer, that he liked to be by himself because he had too much difficulty in making friends. +DUMMY/LJ008-0047.wav|This movable platform is raised six inches higher than the rest of the scaffold, and on it the convicts stand; +DUMMY/LJ028-0276.wav|a marvelous thing happened to Zopyrus, son of the Megabyzus who was among the seven men that overthrew the Magus. +DUMMY/LJ016-0371.wav|The interment in a shell filled with quicklime in the passage-way leading to the Old Bailey is also a part of the duty of the prison officials. +DUMMY/LJ050-0259.wav|some of which must be specially designed to fit unique requirements. Even before the assassination, and to a far greater extent thereafter, +DUMMY/LJ006-0201.wav|went at once to his customers, who bought their paper and paid for it themselves. +DUMMY/LJ016-0184.wav|his attention was drawn early one morning to a man who leant against a lamp-post in Finsbury Square, coughing violently. +DUMMY/LJ040-0098.wav|Marguerite Oswald worked in miscellaneous jobs after her divorce from Ekdahl. +DUMMY/LJ040-0212.wav|The factors in Lee Oswald's personality which were noted by those who had contact with him in New York indicate +DUMMY/LJ039-0087.wav|It allows you to see your target clearly, and it is still of a minimum amount of power that it doesn't exaggerate your own body movements. +DUMMY/LJ030-0157.wav|The car maintained this average speed over a distance of approximately one hundred eighty-six feet immediately preceding the shot which struck the President in the head. +DUMMY/LJ049-0080.wav|Assassination a Federal Crime +DUMMY/LJ011-0124.wav|he also forged dividend receipts and got the dividends. He was only six-and-twenty when he was hanged. +DUMMY/LJ006-0056.wav|Some rooms remained quite empty and unoccupied, while others were full to overflowing. +DUMMY/LJ039-0190.wav|Simmons testified that familiarity with the bolt could be achieved in dry practice and, as has been indicated above, +DUMMY/LJ010-0100.wav|The first constable who entered Thistlewood ran through the body with a sword, but others quickly followed, +DUMMY/LJ042-0185.wav|economic, political, or military crisis, internal or external, +DUMMY/LJ007-0201.wav|each was provided with an iron bedstead, and a small desk at which the condemned man might read or write. +DUMMY/LJ041-0052.wav|Oswald's inability or lack of desire to enter into meaningful relationships with other people +DUMMY/LJ032-0009.wav|(two) the means by which the weapon was brought into the Depository Building, +DUMMY/LJ028-0249.wav|continued dancing and reveling until they learned the capture but too certainly. +DUMMY/LJ015-0047.wav|Its debts amounted to three-quarters of a million, and the dividend it eventually paid was three and twopence in the pound. +DUMMY/LJ050-0158.wav|the Department hopes to design a practical system which will fully meet the needs of the Protective Research Section of the Secret Service. +DUMMY/LJ040-0149.wav|intense anxiety, shyness, feelings of awkwardness and insecurity, end quote. +DUMMY/LJ048-0168.wav|With the number of men available to the Secret Service and the time available, surveys of hundreds of buildings and thousands of windows is not practical. +DUMMY/LJ005-0006.wav|who handed a shilling to the escort warder to provide her with a hackney coach; but this functionary pocketed the cash, and obliged the woman to walk +DUMMY/LJ021-0073.wav|This is because of the establishment of fair, competitive standards and because of relief from unfair competition +DUMMY/LJ019-0242.wav|Newgate naturally shared in any advantages due to these reforms. I propose, therefore, to refer to them in the concluding pages of this work, +DUMMY/LJ047-0045.wav|and promised to advise the FBI if he heard from them. +DUMMY/LJ038-0239.wav|indicated that the bullet was fired from a position near the point where one of the photographs was taken. +DUMMY/LJ038-0046.wav|until Oswald had been knocked to the floor by McDonald and the others. +DUMMY/LJ046-0170.wav|which requires a checkup at least, every six months. +DUMMY/LJ005-0281.wav|that the greatest mischief followed from the intercourse which was still permitted in so many prisons; to use its words, +DUMMY/LJ049-0172.wav|questions inevitably arise as to the scope of each agency's authority and responsibility. +DUMMY/LJ010-0153.wav|William the fourth was also the victim of a murderous outrage on Ascot race-course in eighteen thirty-two, +DUMMY/LJ050-0064.wav|whose prior acts or statements depict propensity for violence and hatred against organized government, end quote. +DUMMY/LJ007-0054.wav|There were disputes and quarrels constantly among these doomed men; it was a word and blow, an argument clenched always with a fight. +DUMMY/LJ027-0109.wav|Wiedersheim has recorded more than one hundred and eighty such structural reminiscences in man. +DUMMY/LJ003-0147.wav|Besides the jail fees, there was garnish of half-a-guinea, collected by the steward, +DUMMY/LJ044-0045.wav|events which, quote, robbed me of what support I had leaving me alone, end quote. +DUMMY/LJ037-0049.wav|A few minutes later she described the gunman to a policeman. +DUMMY/LJ010-0105.wav|Thistlewood was discovered next morning in a mean house in White Street, Moorfields. +DUMMY/LJ006-0222.wav|and enlivened by flash songs and thrilling long-winded descriptions of robberies and other "plants." +DUMMY/LJ042-0084.wav|referring to them as experiences that he did not intend to have himself, even though he stated that he had never held a civilian job. +DUMMY/LJ031-0224.wav|with a sizable metal fragment lying just above the right eye. +DUMMY/LJ022-0152.wav|There is likewise pending before the Congress +DUMMY/LJ003-0317.wav|No prisoner should be allowed to obtain superior accommodation on the payment of any fees. Fees indeed should be generally abolished, garnish also. +DUMMY/LJ002-0186.wav|The population generally amounted to from five hundred to seven hundred, the accommodation being calculated for two hundred. +DUMMY/LJ028-0042.wav|The place where the old city had stood for three thousand years again became a desert. +DUMMY/LJ020-0034.wav|When light, knead well and long; +DUMMY/LJ050-0043.wav|The Secret Service has embarked upon a complete overhaul of its research activities. The staff of the Protective Research Section (PRS) +DUMMY/LJ038-0086.wav|During the evening of November twenty-two, the Dallas Police Department performed paraffin tests on Oswald's hands and right cheek +DUMMY/LJ030-0019.wav|According to O'Donnell, the President commented that, quote, +DUMMY/LJ028-0055.wav|and surrounded it with walls the like of which no other city has ever seen. +DUMMY/LJ009-0236.wav|To the great surprise of all who witnessed this dreadful scene, +DUMMY/LJ004-0120.wav|In eighteen eighteen, out of five hundred and eighteen prisons in the United Kingdom, +DUMMY/LJ011-0209.wav|He surrendered the bride who had never been a wife, and she returned to England with her friends, while Wakefield went on alone to Paris. +DUMMY/LJ002-0175.wav|The costs in heavier debts always doubled the sum; if the arrest was made in the country it trebled it. +DUMMY/LJ021-0038.wav|and with the full expectation that the money used to assist these enterprises will eventually be repaid. +DUMMY/LJ015-0037.wav|and the partners had to decide between suspending payment or continuing to hold its head above water by flagitious processes. +DUMMY/LJ028-0020.wav|Late at night the weary dromedary kneels, and on the ground, close beside her, you lie down to sleep. +DUMMY/LJ023-0065.wav|they gave to the Congress the ample broad powers "to levy taxes +DUMMY/LJ025-0103.wav|of attempting to draw any general distinction between animals and vegetables on this ground. In fact, the difference vanishes with the sunshine, +DUMMY/LJ028-0451.wav|In places even the bases have disappeared, and their moats have long been filled with the drifting sand. +DUMMY/LJ002-0033.wav|two. The female debtors' side. three. The chapel yard. four. The middle yard. +DUMMY/LJ005-0062.wav|Experience has since shown that in some respects these plans are defective, especially in the placing of the governor's residence in the center of the prison. +DUMMY/LJ035-0063.wav|But I cannot recall any change in expression of any kind on his face, end quote. +DUMMY/LJ029-0074.wav|and that following the luncheon the President would return to the airport by the most direct route. +DUMMY/LJ046-0063.wav|stated the President's views of his responsibilities with simplicity and clarity, quote, +DUMMY/LJ013-0060.wav|In the first a maiden lady, Miss Slack, +DUMMY/LJ047-0118.wav|that Oswald had told him that he had worked and been married in the Soviet Union. +DUMMY/LJ018-0308.wav|Large sums, as they were realized, were transmitted to the United States, and invested in various American securities, +DUMMY/LJ018-0179.wav|On his return to Newgate to be finally discharged, Cummings jumped up the stairs and fairly danced for joy. +DUMMY/LJ011-0106.wav|Hunton accepted his sentence with great resignation, although he protested against the inhumanity of the laws which condemned him to death. +DUMMY/LJ006-0250.wav|The officers were obliged to go in among the prisoners to restore order with drawn cutlasses, +DUMMY/LJ018-0183.wav|he devoted his energies to coining and forgery, and in the latter line was a friend and associate of Saward's. +DUMMY/LJ043-0077.wav|While that counselor found that he was qualified to handle many different types of jobs, +DUMMY/LJ024-0051.wav|I suggest only the addition of justices to the bench +DUMMY/LJ050-0174.wav|Protective Research participation in advance arrangements. +DUMMY/LJ023-0124.wav|we shall seek such clarifying amendment as will assure the power to enact those laws, +DUMMY/LJ002-0236.wav|Matters were rather better at the Marshalsea. +DUMMY/LJ028-0343.wav|Darius having become master of the place, destroyed the wall, and tore down all the gates; +DUMMY/LJ014-0311.wav|His extensive business had been carried on by fraud. +DUMMY/LJ028-0298.wav|Surely thou hadst gone out of thy mind when thou didst so misuse thyself. +DUMMY/LJ005-0041.wav|a much-needed and, according to our ideas, indispensable reform, already initiated by the Ladies' Committee at Newgate. +DUMMY/LJ010-0021.wav|The credulity of investors, the unscrupulous dishonesty of bankers, +DUMMY/LJ045-0204.wav|he had never found anything to which he felt he could really belong. +DUMMY/LJ012-0233.wav|According to the confession, death had been inflicted by drowning in a well, whereas the medical evidence all pointed to violence. +DUMMY/LJ016-0440.wav|But it is neither easy to obtain the instruments of self-destruction nor to elude the vigilance of their guard. +DUMMY/LJ037-0131.wav|When he reached a gasoline service station one block away he turned north and walked toward a parking area in the rear of the station. +DUMMY/LJ022-0138.wav|It has under consideration many measures which provide for the rounding out of the program of economic and social reconstruction +DUMMY/LJ002-0160.wav|and John Lancaster for one shilling, eight pence, with costs of seven shillings, six pence. Quote. +DUMMY/LJ003-0315.wav|All three, governor, chaplain, and surgeon, should keep journals, which should be inspected periodically by the visiting magistrates. +DUMMY/LJ012-0215.wav|and the jury returned a verdict of willful murder against persons unknown, expressing a strong opinion that Bishop, +DUMMY/LJ039-0099.wav|In accordance with standard Marine procedures, Oswald received extensive training in marksmanship. +DUMMY/LJ003-0284.wav|that when they were not carousing, plotting, or scheming, +DUMMY/LJ019-0340.wav|The latter was rendered nearly impossible by the penalties imposed on persons bringing spirituous liquors into the jail. +DUMMY/LJ031-0211.wav|weighed one hundred seventy pounds, had blue eyes and reddish-brown hair. +DUMMY/LJ012-0044.wav|When his trade was busiest he set up a second establishment, at the head of which, although he was married, +DUMMY/LJ026-0005.wav|that is, the forms may possess a mouth for the ingestion of solid food and green coloring matter, chlorophyll, +DUMMY/LJ026-0056.wav|It is, however, impossible for the animal to build these materials directly into the substance of its own body. +DUMMY/LJ011-0242.wav|Howard replied, desiring Mr. Mullay to call upon him in a house in Red Lion Square. +DUMMY/LJ011-0125.wav|The other and the last criminal executed for forgery in this country was one Maynard, who was convicted of a fraud upon the Custom House. +DUMMY/LJ048-0094.wav|Lawson was responsible for working out a great many arrangements for the President's trip. +DUMMY/LJ035-0140.wav|On entering, Lovelady saw a girl on the first floor who he believes was Victoria Adams. +DUMMY/LJ028-0300.wav|as it is, I kept my own counsel, and so accomplished my plans. +DUMMY/LJ003-0293.wav|also to make such proposals as might appear salutary, and calculated to improve Newgate and the rest of the city jails. +DUMMY/LJ045-0061.wav|While Marina Oswald said that her husband wanted her to learn English, +DUMMY/LJ040-0214.wav|His usual reaction to the problems which he encountered there was simply withdrawal. +DUMMY/LJ048-0028.wav|Consequently, there was no basis for Hosty to go to Secret Service and advise them of Oswald's presence. End quote +DUMMY/LJ030-0059.wav|was a metallic frame with four handholds that riders in the car could grip while standing in the rear seat during parades. +DUMMY/LJ008-0289.wav|Capitally convicted prisoners did, as a matter of fact, "treat with habitual and inexpressible levity the sentence of death." +DUMMY/LJ035-0079.wav|The first test, run at normal walking pace, required one minute, eighteen seconds; +DUMMY/LJ025-0054.wav|by the introduction of exact and easily manageable methods of conducting the chemical analysis of organic compounds and finally +DUMMY/LJ006-0210.wav|Drink, in more or less unlimited quantities, was still to be had. +DUMMY/LJ031-0203.wav|Mrs. Kennedy and the Attorney General, +DUMMY/LJ013-0234.wav|The cloth above referred to belonged to the inner seam of his trousers, which he had managed to tear out. +DUMMY/LJ048-0015.wav|We talked to him twice. +DUMMY/LJ045-0087.wav|Mrs. Oswald told another of her friends that Oswald was very cold to her, that they very seldom had sexual relations +DUMMY/LJ014-0041.wav|Such an extravagant defense did not weigh with judge or jury; +DUMMY/LJ028-0159.wav|thence from the corners of the wall there is carried along each bank of the river a fence of burned bricks. +DUMMY/LJ016-0282.wav|"Till three o'clock it was one long revelry of songs and laughter, shouting, and often quarreling, though, +DUMMY/LJ002-0070.wav|The state side was the part stolen from the female felons' side. +DUMMY/LJ006-0182.wav|Under the reckless contempt for regulations, +DUMMY/LJ001-0029.wav|But though on the whole, except in Italy, Gothic letter was most often used +DUMMY/LJ049-0079.wav|Many changes have already been made and others are contemplated, some of them in response to the Commission's questions and informal suggestions. +DUMMY/LJ016-0047.wav|But the worst part of the business was to jump from this irksome foothold of the iron grating on to the top of the building just mentioned, +DUMMY/LJ048-0127.wav|as those followed in nineteen thirty-six during a trip to Dallas by President Roosevelt, +DUMMY/LJ014-0173.wav|It was understood also that he had served in the army as a private, and had, moreover, undergone a sentence of transportation. +DUMMY/LJ030-0118.wav|In the downtown area, large crowds of spectators gave the President a tremendous reception. +DUMMY/LJ017-0090.wav|and so got the money for which he was prepared to barter his soul. +DUMMY/LJ010-0086.wav|The occasion was considered excellent by the conspirators for disposal of the whole Cabinet at one blow, +DUMMY/LJ045-0112.wav|When Oswald called the next day his wife became very angry about his use of the alias. He said that he used it because, quote, +DUMMY/LJ002-0021.wav|the grand jury made a special presentment to the Court of Common Council, pointing out that on the debtors' side, which was intended for only one hundred, +DUMMY/LJ006-0245.wav|After severe suffering prolonged for many months, the inflammation was subdued, but the cartilage of the knee-joint was destroyed, and he was crippled for life. +DUMMY/LJ015-0133.wav|Even when the bubble burst Redpath, who had lived at the rate of twenty thousand a year, +DUMMY/LJ032-0135.wav|"Lifting" a print involves the use of adhesive material to remove the fingerprint powder which adheres to the original print. +DUMMY/LJ016-0336.wav|But Newgate and its neighborhood was carefully held by the police, both city and metropolitan. +DUMMY/LJ018-0134.wav|In this part of the mills a large quantity of bank-note paper was kept at the period of the robbery, +DUMMY/LJ045-0137.wav|I and my wife strongly protested these tactics by the notorious F.B.I., end quote. +DUMMY/LJ011-0022.wav|and fixed the whole sum misappropriated at one hundred seventy thousand pounds, the first forgery dating back to eighteen fourteen. +DUMMY/LJ007-0079.wav|or arraignment, to an asylum, and the Secretary of State had the same power as regards any who became insane while undergoing sentence. +DUMMY/LJ041-0014.wav|Two days later, quote, some big guy, probably from a high school -- he looked like a tremendous football player, end quote, +DUMMY/LJ006-0217.wav|There were more varied, and at times, especially when beer had circulated freely, more uproarious diversions. +DUMMY/LJ014-0174.wav|But it was supposed that he had been once in a good position, well born, and well educated. +DUMMY/LJ028-0210.wav|Nabonidus was taken prisoner in Babylon. On the third of Marchesvan Cyrus entered Babylon and proclaimed peace to all the city. +DUMMY/LJ047-0242.wav|Revill testified that Hosty said also that the FBI had information that Oswald was, quote, capable of committing this assassination, end quote. +DUMMY/LJ034-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. +DUMMY/LJ044-0044.wav|to an attack by Cuban exiles in a street demonstration and being, quote, officialy cautioned, end quote, by the police. +DUMMY/LJ038-0157.wav|The only employee at the Depository Building named "Junior" was James Jarman, Jr. +DUMMY/LJ041-0081.wav|Oswald probably engaged his superiors in arguments on a subject that he had studied +DUMMY/LJ013-0120.wav|whom he had shown over the plate closet. +DUMMY/LJ040-0174.wav|Lee confirmed some of those observations by saying that he felt almost as if there were a veil between him and other people +DUMMY/LJ035-0098.wav|he was certain that both elevators, which occupy the same shaft, were on the fifth floor. +DUMMY/LJ023-0062.wav|and which could not be met by merely local action. +DUMMY/LJ048-0284.wav|This is the salutary goal to which the Secret Service regulation is directed, +DUMMY/LJ050-0206.wav|suggest that the Secret Service is trying to accomplish its job with too few people and without adequate modern equipment. +DUMMY/LJ012-0080.wav|This was known to some clerks in the office, who began to consider how they might lay hands on a lot of cash. +DUMMY/LJ041-0009.wav|Mrs. Murret believes that he talked at length with a girl on the telephone, but no one remembers that he had any dates. +DUMMY/LJ036-0137.wav|And he never said anything. So I figured he was one of these people that don't like to talk so I never said any more to him. +DUMMY/LJ021-0064.wav|not only to labor in the form of new jobs, in relief from overwork and in relief from underpay, +DUMMY/LJ018-0190.wav|"By all means," replies Bob, and a close investigation follows, without any detection of the corpse concealed. +DUMMY/LJ009-0138.wav|all the avenues to the prison gates were blocked by ticket-holders. +DUMMY/LJ015-0124.wav|but it was based upon more extended and audacious forgeries. +DUMMY/LJ020-0059.wav|When the time is up and the rolls are puffy and promising, set them in a pretty quick oven and bake half an hour, +DUMMY/LJ016-0284.wav|There were preachers among the crowd, but they could not get a patient hearing. +DUMMY/LJ020-0036.wav|Take care that it does not burn in baking. The molasses renders it liable to scorching. +DUMMY/LJ040-0113.wav|They moved shortly after Robert joined the Marines; they lived for a time with John Pic who was stationed there with the Coast Guard. +DUMMY/LJ028-0146.wav|As fast as they dug the moat, the soil which they got from the cutting was made into bricks, +DUMMY/LJ036-0211.wav|passed the intersection of tenth and Patton, about eight blocks from where he had reported at twelve:fifty-four p.m. +DUMMY/LJ045-0071.wav|Mrs. Ford said that Marina Oswald admitted that she provoked Oswald on occasion. +DUMMY/LJ003-0308.wav|A laundry should be established, and a matron appointed on the female side, where all the prisoners' washing could be performed. +DUMMY/LJ032-0129.wav|Accordingly, my opinion simply was that the latent prints which were there were of no value, end quote. +DUMMY/LJ050-0246.wav|which would permit it to provide adequate protective manpower for all situations. +DUMMY/LJ008-0213.wav|By this time the workmen might be heard busily erecting the gallows; +DUMMY/LJ041-0065.wav|testified that Oswald seemed, quote, always to be striving for a relationship, but whenever he did +DUMMY/LJ015-0090.wav|To account for his revenues he pretended to have been very lucky on the Stock Exchange, which was at one time true to a limited extent, +DUMMY/LJ012-0152.wav|which he refused to redeem on account of the row about the robbery. +DUMMY/LJ041-0182.wav|and believed that our Government did not have, quote, too much to offer, end quote, but was not in favor of, quote, the Communist way of life, end quote. +DUMMY/LJ038-0179.wav|(three) firearm identification of the bullet found in Walker's home, and (four) +DUMMY/LJ043-0020.wav|The relations between Oswald and his wife became such that Bouhe wanted to "liberate" her from Oswald. +DUMMY/LJ002-0058.wav|The chapel yard led to the chapel, and on the staircase were two rooms frequently set apart for the king's witnesses, +DUMMY/LJ008-0221.wav|They talked of the days when the convicts were hung up a dozen or more in a row; +DUMMY/LJ017-0196.wav|The last time Mrs. Soames showed great reluctance to take it, but Wilson said it would certainly do her good. +DUMMY/LJ033-0210.wav|told the curtain rod story to Frazier to explain both the return to Irving on a Thursday +DUMMY/LJ032-0055.wav|Information received from the Italian Armed Forces Intelligence Service has established that this particular rifle was the only rifle of its type +DUMMY/LJ042-0227.wav|His second answer was that he, quote, went as a citizen of the U.S. (as a tourist) residing in a foreign country which I have a perfect right to do. +DUMMY/LJ015-0083.wav|This ingenious method of raising funds he adopted and carried on without detection, +DUMMY/LJ010-0223.wav|You must attend; and if your master will not give you leave, you must come in defiance of him. +DUMMY/LJ049-0069.wav|This basic approach to the problem of planning for emergencies is sound. +DUMMY/LJ050-0027.wav|While manuals and memoranda are no guarantee of effective operations, +DUMMY/LJ009-0036.wav|Down below between the galleries was the mass of the prison population; +DUMMY/LJ041-0089.wav|Thornley added, quote, I think it was kind of necessary to him to believe that he was being picked on. +DUMMY/LJ017-0036.wav|that the trial was transferred to the Central Criminal Court, +DUMMY/LJ040-0029.wav|He seemed to prefer the Soviet Union and he spoke highly of Cuba. +DUMMY/LJ043-0084.wav|business suit, alert replies -- Expresses self extremely well, end quote. +DUMMY/LJ050-0245.wav|In view of the ever-increasing mobility of American Presidents, it seems unlikely that the Service could or should increase its own staff to a size +DUMMY/LJ033-0144.wav|At the time the bag was found, Lieutenant Day of the Dallas police wrote on it, quote, Found next to the sixth floor window gun fired from. +DUMMY/LJ050-0015.wav|The Commission recommends that the Secretary of the Treasury appoint a special assistant with the responsibility of supervising the Service. +DUMMY/LJ011-0006.wav|He went to the bank, and found that no stocks stood in her name. He called at once upon Fauntleroy, his client's bankers, for an explanation, +DUMMY/LJ009-0009.wav|This rule has some, but very few, exceptions; such as where a hardened offender behaves with great levity and brutality, as if he cared nought for his life, +DUMMY/LJ006-0308.wav|The governor sent down wine on festive occasions, of which no doubt the prisoner housemaid had her share. +DUMMY/LJ050-0021.wav|Actions by the Service since the assassination indicate its awareness of the necessity for substantial improvement in its administration. +DUMMY/LJ002-0282.wav|Those who could not pay were thrown into the wards with the night charges, +DUMMY/LJ036-0048.wav|She boarded the Marsalis bus at St. Paul and Elm Streets to return home. She testified further, quote, +DUMMY/LJ024-0023.wav|ever since a similar proposal passed the House of Representatives in eighteen sixty-nine. +DUMMY/LJ038-0234.wav|Although Oswald destroyed the notebook, three photographs found among Oswald's possessions after the assassination +DUMMY/LJ038-0089.wav|Expert testimony before the Commission +DUMMY/LJ025-0105.wav|On the other hand, those plants, such as the fungi, +DUMMY/LJ019-0347.wav|chairman of quarter sessions, or even by a couple of justices. +DUMMY/LJ029-0065.wav|Neither the Dallas police nor the FBI had yet learned the source of the handbill. +DUMMY/LJ041-0142.wav|Once it had started down Elm Street toward the Triple Underpass, however, +DUMMY/LJ031-0143.wav|and decided that there would be no release of the news of the President's death until the Vice President had left the hospital. +DUMMY/LJ021-0096.wav|under the able and energetic leadership of General Johnson. +DUMMY/LJ020-0084.wav|Bake from twenty to twenty-five minutes in a quick oven, turning the pan around once, quickly and lightly. +DUMMY/LJ028-0470.wav|Time has dealt even less kindly with it, for it may be traced only for the distance of about a mile along its eastern side. +DUMMY/LJ015-0288.wav|so that Saward and the rest might make themselves scarce. +DUMMY/LJ019-0045.wav|Not less remarkable than this diverse interpretation of a manifest duty +DUMMY/LJ027-0005.wav|paleontology, distribution and domestication. +DUMMY/LJ044-0098.wav|He did not think Oswald looked like the, quote, type, end quote, that he would have expected to find associating with a group such as the Fair Play for Cuba Committee. +DUMMY/LJ003-0105.wav|he was sent on to Newgate in a coach, handcuffed to a noted house-breaker, who was afterwards cast for death. +DUMMY/LJ028-0252.wav|Only the King's son, Belshazzar, was killed. +DUMMY/LJ030-0113.wav|through thinly populated areas on the outskirts of Dallas. +DUMMY/LJ018-0289.wav|When all was ready, Bidwell first "refreshed his credit" at the Bank of England, as well as disarmed suspicion, +DUMMY/LJ016-0008.wav|Charles Thomas White, awaiting execution for arson, made a desperate effort to escape from Newgate in eighteen twenty-seven. +DUMMY/LJ029-0119.wav|It afforded us a chance to have alternative routes if something happened on the motorcade route. It was the type of suburban area a good part of the way +DUMMY/LJ041-0094.wav|He had the nickname, quote, Ozzie Rabbit, end quote. +DUMMY/LJ044-0089.wav|Stuckey testified that uncovering Oswald's defection was very important, quote, +DUMMY/LJ049-0049.wav|About three point seven seconds after the President received this wound, +DUMMY/LJ028-0513.wav|Should you cross the river to search for the western inner wall, you would find but a small fragment of it. +DUMMY/LJ047-0092.wav|On August nine, nineteen sixty-three, +DUMMY/LJ038-0019.wav|She said, quote, No; by golly, he didn't, end quote, and turned around, but the man was nowhere in sight. +DUMMY/LJ011-0237.wav|The jewelers were always a favorite prey of the London thieves. +DUMMY/LJ020-0003.wav|But it keeps fresh longer, is softer and more nutritious, and a second-rate brand of flour thus treated produces a better loaf +DUMMY/LJ036-0187.wav|Mrs. Roberts saw him a few seconds later standing near the bus stop in front of the house on the east side of Beckley. +DUMMY/LJ047-0107.wav|and when I got to questioning him further then he felt that his purpose had been served and he wouldn't say anything further, end quote. +DUMMY/LJ041-0146.wav|it was more in terms of a general hostility against the government and its representatives rather than a grudge against any particular person. +DUMMY/LJ003-0162.wav|the convicted felon, the transport awaiting removal, the lunatic whose case was still undecided, +DUMMY/LJ047-0117.wav|William Stuckey, who had appeared on the radio program with Oswald, told the Bureau on August thirty +DUMMY/LJ047-0101.wav|Quigley testified, quote, +DUMMY/LJ046-0116.wav|The main job of the Protective Research Section (PRS) +DUMMY/LJ032-0229.wav|This information enabled the Commission to conclude +DUMMY/LJ044-0211.wav|The Militant also reflected a critical attitude toward President Kennedy's attempts to reduce tensions between the United States and the Soviet Union. +DUMMY/LJ033-0013.wav|During October and November of nineteen sixty-three, +DUMMY/LJ002-0191.wav|Besides those actually resident within the walls, +DUMMY/LJ039-0247.wav|(seven) attempted, in April nineteen sixty-three, to kill Maj. Gen. Edwin A. Walker, and (eight) +DUMMY/LJ024-0114.wav|and who would be willing to support a reasonable amendment if they could agree on one. +DUMMY/LJ031-0183.wav|He was informed that takeoff would be delayed until Vice President Johnson was sworn in. +DUMMY/LJ033-0049.wav|According to Mrs. Paine, Oswald had gone to bed by nine p.m.; +DUMMY/LJ027-0106.wav|Other well-known examples among vertebrates are: Vestiges of hind limbs in certain snakes, reduced wings in the Apteryx and ostriches, +DUMMY/LJ038-0293.wav|until that Wednesday because he had heard that there was to be a gathering at the church next door to Walker's house on that evening. +DUMMY/LJ012-0268.wav|Suspicion grew almost to certainty as the evidence was unfolded. +DUMMY/LJ049-0058.wav|Secondly, agents are instructed to remove the President as quickly as possible from known or impending danger. +DUMMY/LJ010-0313.wav|the sum total amounting to some one hundred seventy thousand pounds, with a declaration in his own handwriting to the following effect. +DUMMY/LJ007-0202.wav|But the one great and most crying evil remained unremedied. +DUMMY/LJ011-0058.wav|He was summoned to the Mansion House, where he repeated his request, crying, "Accordez moi cette grâce," with much urgency. +DUMMY/LJ010-0250.wav|of the criminal intent to kill. +DUMMY/LJ047-0051.wav|particularly since his employment did not involve any sensitive information. +DUMMY/LJ021-0106.wav|the representatives of trade and industry were permitted to write their ideas into the codes. +DUMMY/LJ048-0159.wav|Agent Lawson did not arrange for a prior inspection of buildings along the motorcade route, +DUMMY/LJ032-0191.wav|quote, put aside, end quote, after catching the fibers. +DUMMY/LJ028-0186.wav|but when Nabonidus joined with the King of Egypt and with the wealthy Croesus of Lydia in an alliance against him, +DUMMY/LJ013-0018.wav|beef, pork, butter, and earthenwares, on all of which they effected insurances. +DUMMY/LJ003-0337.wav|Were a metropolitan prison erected on the same lines, with all the space not only for air and exercise, but for day rooms and sleeping cells +DUMMY/LJ016-0065.wav|To walk out into the street was an easy affair, and he was now free, with one and fourpence in his pocket and a shirt and trousers for all his clothing. +DUMMY/LJ007-0243.wav|It was not till the erection of the new prison at Holloway in eighteen fifty, and the entire internal reconstruction of Newgate according to new ideas, +DUMMY/LJ037-0056.wav|testified that she was, quote, quite hysterical, end quote, and was, quote, crying and upset, end quote. +DUMMY/LJ006-0229.wav|which they dropped smoldering into their fellow-prisoners' pockets. +DUMMY/LJ022-0051.wav|In spite of the fact that unemployment remains a serious problem +DUMMY/LJ011-0034.wav|These arguments availed little with the jury, who after a short deliberation found Fauntleroy guilty, and he was sentenced to death. +DUMMY/LJ040-0072.wav|he was cared for principally by his mother's sister, by babysitters and by his mother, when she had time for him. +DUMMY/LJ026-0162.wav|In non-green plants like the Indian pipe and mushrooms the breathing of oxygen and the excretion of carbon dioxide are as in the animal. +DUMMY/LJ022-0017.wav|The general good was at a discount. +DUMMY/LJ037-0190.wav|George Rose and Co. had the barrel shortened by a gunsmith to two and one quarter inches. +DUMMY/LJ048-0272.wav|Chief Rowley testified +DUMMY/LJ003-0025.wav|In the latter duties he was, however, supervised by three auditors, freely chosen by the prisoners among themselves. +DUMMY/LJ013-0240.wav|Good was followed to his stables by a policeman, but obstinately denied the theft. +DUMMY/LJ018-0256.wav|Roupell made a long statement more in exculpation than in his defense. +DUMMY/LJ011-0128.wav|Maynard was convicted of uttering the forged document, Jones of being an accessory; the third prisoner was acquitted. +DUMMY/LJ008-0122.wav|where the murder for which he was condemned had been committed. +DUMMY/LJ033-0081.wav|she looked out the breakfast-room window and saw Oswald cross the street and walk toward the driveway where her brother parked his car near the carport. +DUMMY/LJ028-0014.wav|The desert about you shows no signs of life; +DUMMY/LJ017-0171.wav|Smethurst was therefore given a free pardon for the offense of murder, +DUMMY/LJ038-0038.wav|McDonald struck back with his right hand and grabbed the gun with his left hand. They both fell into the seats. +DUMMY/LJ037-0228.wav|that the man was a, quote, white male, about twenty-five, about five feet eight, brown hair, medium, end quote, and wearing a, quote, +DUMMY/LJ032-0184.wav|For ten days prior to the eve of the assassination Oswald had not been present at Ruth Paine's house in Irving, Texas, where the rifle was kept. +DUMMY/LJ047-0160.wav|found it to be four one one Elm Street. End quote. +DUMMY/LJ032-0153.wav|A palmprint could not be placed on this portion of the rifle, when assembled, because the wooden foregrip covers the barrel at this point. +DUMMY/LJ045-0143.wav|and use it against him as had been done in New Orleans. +DUMMY/LJ049-0064.wav|because the variations possible preclude effective planning. +DUMMY/LJ029-0127.wav|Elm Street was to be reached from Main by turning right at Houston, +DUMMY/LJ038-0176.wav|The Commission evaluated the following evidence in considering whether Lee Harvey Oswald fired the shot which almost killed General Walker: +DUMMY/LJ013-0235.wav|There is nothing to show that Courvoisier really contemplated self-destruction. +DUMMY/LJ016-0048.wav|a distance of eight or nine feet. +DUMMY/LJ001-0064.wav|many of whose types, indeed, like that of the Subiaco works, are of a transitional character. +DUMMY/LJ003-0020.wav|This was the pernicious plan of governing by prisoners, or of setting a favored few in authority over the many. +DUMMY/LJ002-0259.wav|but the prevailing moral tone may be guessed from the penalty inflicted upon persons singing obscene songs before nine p.m. +DUMMY/LJ045-0009.wav|and in the extreme anti-Kennedy newspaper advertisement and handbills that appeared in Dallas at the time of the President's visit there. +DUMMY/LJ029-0177.wav|Articles, editorials, and letters to the editor in the Dallas Morning News and the Dallas Times-Herald after September thirteen +DUMMY/LJ004-0029.wav|The erection of new prison buildings was made imperative under certain conditions and following certain rules +DUMMY/LJ042-0091.wav|The Soviet authorities finally permitted Oswald to remain in their country. +DUMMY/LJ042-0008.wav|of his willingness to act on his beliefs in quite extraordinary ways. +DUMMY/LJ046-0220.wav|When the threat is in the form of a written communication, give a copy to local Secret Service and forward the original to the Bureau +DUMMY/LJ017-0073.wav|In eighteen fifty-four he owed a very large sum of money, but he was temporarily relieved by the death of his wife, +DUMMY/LJ039-0244.wav|killed Dallas Police Officer J. D. Tippit in an apparent attempt to escape, +DUMMY/LJ016-0352.wav|the officials spoke in whispers; there was but little moving to and fro. +DUMMY/LJ027-0160.wav|that the embryo or larva of a frog or toad, when first hatched, is a legless, tail-swimming, water-breathing, gill-breathing animal. +DUMMY/LJ016-0188.wav|He told Calcraft that he was Foxen the executioner, and that he was that moment on his way to Newgate to hang a man, +DUMMY/LJ022-0069.wav|in order that unemployment may be prevented by the stabilizing of employment itself. +DUMMY/LJ013-0221.wav|that he could not resist the opportunity for robbery offered him by his situation at Lord William Russell's; that when vexed with his master +DUMMY/LJ019-0188.wav|He forwarded plans prepared by the inspectors of prisons, not for blind adoption, but as a guide. +DUMMY/LJ027-0067.wav|The exceptions are of two kinds, viz.: (a) some lowest fishes, amphioxus and lampreys, +DUMMY/LJ047-0140.wav|advising that an individual tentatively identified as Oswald had been in touch with the Soviet Embassy in Mexico City +DUMMY/LJ045-0122.wav|While Oswald's concern about the FBI had some basis in fact, in that FBI agents had interviewed him in the past and had renewed their interest +DUMMY/LJ038-0098.wav|Denial of Rifle Ownership +DUMMY/LJ030-0097.wav|Remainder of motorcade. +DUMMY/LJ038-0110.wav|Fritz told him that the two small photographs were found in the Paine garage. +DUMMY/LJ018-0370.wav|unhappily show that society is more than ever at the mercy of the insidious and unscrupulous administration of poisonous drugs. +DUMMY/LJ045-0100.wav|She thought that he might not have become involved in the assassination if people had been kinder to him. +DUMMY/LJ013-0196.wav|This placed them in a position of much embarrassment. To have thrown up their brief would have been to have secured Courvoisier's conviction. +DUMMY/LJ006-0136.wav|These wardsmen, besides thus ruling the roast, had numerous special privileges, if such they can be called. +DUMMY/LJ042-0044.wav|I will not say your grandchildren will live under communism, look for yourself at history, look at a world map! America is a dicing country, +DUMMY/LJ044-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. +DUMMY/LJ006-0099.wav|The inspectors paraded the prisoners, and found them generally ragged and ill-clad, squalid and filthy in the extreme; +DUMMY/LJ037-0266.wav|(two) the cartridge cases found near the scene of the shooting were fired from the revolver in the possession of Oswald at the time of his arrest, +DUMMY/LJ022-0053.wav|These measures are of two kinds. +DUMMY/LJ011-0011.wav|and took the instrument out to a clerk with the ink not dry. +DUMMY/LJ013-0029.wav|Even then she might have been saved, but the captain would not suffer the crew to act. Nearly the whole of the cargo was lost as well as the ship. +DUMMY/LJ008-0299.wav|On one occasion, Mr. Wakefield tells us, the Recorder, who had attended the council at Windsor, did not deliver the report till the following day. +DUMMY/LJ019-0117.wav|That industry might thus be stimulated and encouraged was probable enough, +DUMMY/LJ036-0094.wav|Craig may have seen a person enter a white Rambler station wagon fifteen or twenty minutes after the shooting and travel west on Elm Street +DUMMY/LJ014-0117.wav|An examination of her boxes disclosed a quantity of O'Connor's property. +DUMMY/LJ049-0039.wav|to take a protective position in the passenger compartment before the third shot was fired. +DUMMY/LJ043-0158.wav|Question: And then he burned the book? +DUMMY/LJ002-0134.wav|The County Court was the sheriff's, who sat there surrounded by the bishop and the magnates of the county; +DUMMY/LJ048-0237.wav|Most of the agents were there from about one:thirty or one:forty-five a.m. to about two:forty-five or three a.m.; +DUMMY/LJ040-0154.wav|Quote, this thirteen year old well built boy has superior mental resources and functions only slightly below his capacity level +DUMMY/LJ004-0220.wav|for breakfast a pint of gruel; for dinner half a pound of boiled meat, or a quart of soup with vegetables, on alternate days. +DUMMY/LJ028-0207.wav|On the fourteenth day Sippar was taken without a battle. +DUMMY/LJ021-0007.wav|I am happy to report that after years of uncertainty, culminating in the collapse of the spring of nineteen thirty-three, +DUMMY/LJ034-0051.wav|and a Dallas police officer after the cartons had been processed with powder by the Dallas Police. +DUMMY/LJ003-0135.wav|She went up to the ward and found him lying down, quote, +DUMMY/LJ016-0171.wav|were as far as possible mitigated and toned down. But there was little improvement in the ceremony itself. +DUMMY/LJ035-0017.wav|He heard two more shots spaced, quote, pretty well even to me. +DUMMY/LJ021-0069.wav|Now it should not be expected that even employed labor and capital would be completely satisfied with present conditions. +DUMMY/LJ049-0130.wav|The Secret Service must rely in large part +DUMMY/LJ013-0197.wav|Mr. Phillips, who led in the case, went to the other extreme, +DUMMY/LJ019-0232.wav|During this reconstruction the female prisoners were lodged in Holloway, +DUMMY/LJ003-0100.wav|yet the society they were driven to live in, the language they daily heard, and the lessons they were taught in this academy, +DUMMY/LJ004-0087.wav|and the poor there living idle and unemployed become debauched, and come forth instructed in the practice of thievery and lewdness." +DUMMY/LJ040-0115.wav|Pic and his wife would have been happy to have kept Lee, however, +DUMMY/LJ004-0186.wav|It is consoling to know that there were a few brilliant exceptions to this cruel, callous neglect. +DUMMY/LJ046-0056.wav|in seventeen eighty-nine in less time than it took George Washington to travel from New York to Mount Vernon +DUMMY/LJ003-0138.wav|Though they could not force him to gamble, he was compelled to drink, +DUMMY/LJ050-0133.wav|In his testimony Secretary of the Treasury C. Douglas Dillon informed the Commission +DUMMY/LJ049-0194.wav|should be transferred to the FBI, +DUMMY/LJ016-0368.wav|It was they who formed the chief part of the small select group of spectators; +DUMMY/LJ005-0198.wav|In another borough, with a population of ten thousand, the prison was of the same dimensions. +DUMMY/LJ031-0141.wav|It was decided that the Vice President should return on the Presidential plane rather than on the Vice-Presidential plane because it had better communication equipment. +DUMMY/LJ044-0073.wav|It appears that Oswald had only one caller in response to all of his FPCC activities, +DUMMY/LJ028-0171.wav|upon which was raised a second tower, and on that a third, and so on up to eight. +DUMMY/LJ031-0161.wav|Secret Service agents working with police cleared the areas adjacent to the aircraft, including warehouses, +DUMMY/LJ038-0202.wav|Prior to the Walker shooting on April ten, Oswald had been attending typing classes on Monday, Tuesday, and Thursday evenings. +DUMMY/LJ037-0176.wav|And though only one bullet of Remington-Peters manufacture was recovered, two empty cartridge cases of that make were retrieved. +DUMMY/LJ011-0287.wav|but at length managed to wriggle out of the chain which confined his body, and soon loosened the ropes round his feet. +DUMMY/LJ041-0147.wav|Interest in Marxism +DUMMY/LJ031-0077.wav|my direct services to him at that moment would have interfered with the action of the team which was in progress, end quote, +DUMMY/LJ038-0300.wav|Based on (one) the contents of the note which Oswald left for his wife on April ten, nineteen sixty-three, +DUMMY/LJ020-0016.wav|it is well to stir into the yeast a bit of soda no larger than a grain of corn already wet up in a teaspoonful of boiling water. +DUMMY/LJ016-0039.wav|Williams surveyed these formidable obstacles to evasion, and calmly proceeded to surmount them. +DUMMY/LJ014-0304.wav|He was a corn merchant who operated largely in grain. +DUMMY/LJ002-0095.wav|and the rest being provided with barrack beds, and in dimensions varying from thirty feet by fifteen to fifteen feet by ten. +DUMMY/LJ020-0023.wav|If too stiff, warm water, a spoonful at a time until you can handle the paste easily. The danger is in getting it too stiff. Now. +DUMMY/LJ047-0171.wav|Mrs. Paine had nothing to add to what she had already told him, except that during a visit that past weekend, +DUMMY/LJ012-0143.wav|"Money Moses" had received the stolen gold-dust from Moss' father-in-law, Davis, or Isaacs, who was never arrested, +DUMMY/LJ021-0041.wav|has been to clean up thoroughly unwholesome conditions in the field of investment. +DUMMY/LJ049-0065.wav|A number of steps are taken, however, to permit appropriate steps to be taken in an emergency. +DUMMY/LJ005-0005.wav|Even then they were not certain of the favor, for I find a reference to a decent and respectable woman sent to Newgate +DUMMY/LJ009-0055.wav|indeed the chaplain and others have brought him firmly to believe that his situation is enviable, and that the gates of heaven are open to receive him. +DUMMY/LJ013-0186.wav|Behind the skirting board several of his lordship's rings were discovered; +DUMMY/LJ047-0064.wav|that other tenants had complained because Oswald was drinking to excess and beating his wife. +DUMMY/LJ015-0305.wav|At his trial Atwell and Hardwicke, two of his chief allies and accomplices, turned approvers, +DUMMY/LJ002-0274.wav|clergymen, proctors, attorneys, and persons specially selected by the Corporation. +DUMMY/LJ035-0181.wav|At that time, Inspector Herbert V. Sawyer's car was parked in front of the building. +DUMMY/LJ020-0096.wav|Before beginning operations downstairs eat a half-slice of dry bread or a biscuit. +DUMMY/LJ046-0134.wav|In the period from November nineteen sixty-one to November nineteen sixty-three, +DUMMY/LJ050-0074.wav|They emphasized the necessity that the information now being furnished be handled with judgment and care. +DUMMY/LJ016-0287.wav|This was presently superseded by a fresh catch -- +DUMMY/LJ046-0104.wav|is set out and the reasons why this information was not furnished to the Secret Service appraised. +DUMMY/LJ019-0125.wav|But an open farm of a thousand acres would have offered abundant chances of escape, which some at least would have attempted, probably with success. +DUMMY/LJ022-0090.wav|The group will also include representatives of cities, and of labor, farming, banking and industry. +DUMMY/LJ046-0125.wav|Robert I. Bouck, special agent in charge of PRS, +DUMMY/LJ040-0057.wav|It will examine his defection to the Soviet Union in nineteen fifty-nine, his subsequent return to the United States and his life here +DUMMY/LJ027-0171.wav|the highest order (anoura) of this class. Thus, then, in ontogeny the fish goes no further than the fish stages. +DUMMY/LJ011-0194.wav|Friends went in pursuit and traced her to Huddersfield and further north. +DUMMY/LJ002-0165.wav|Bird was, however, discharged within three days by a subscription raised among his fellow-prisoners. +DUMMY/LJ004-0184.wav|There reigned through the whole edifice a chilly, damp, unwholesome atmosphere, and the effluvia from the prisoners was so nauseous +DUMMY/LJ013-0252.wav|One called his sister, but supposed to be his wife, had occupied a room in South Street, Manchester Square, +DUMMY/LJ043-0013.wav|Part of the problem resulted from the fact that, as Jeanne De Mohrenschildt testified, +DUMMY/LJ037-0210.wav|testified before the Commission that the writing on the coupon was Oswald's. +DUMMY/LJ027-0038.wav|between the heart of an insect and the heart of a vertebrate animal +DUMMY/LJ015-0226.wav|After hanging about the Folkestone office for some time, they saw at last that the key was kept in a certain cupboard. +DUMMY/LJ019-0010.wav|As regards the first, however, Sir James Graham, when Home Secretary in eighteen forty-three, +DUMMY/LJ046-0202.wav|The head of PRS testified that the Secret Service requested other agencies to provide, quote, +DUMMY/LJ039-0030.wav|mentioned any proposed visit by Mr. Nixon to Dallas. +DUMMY/LJ022-0098.wav|of making full use of existing employment services and to assist people engaged in relief work +DUMMY/LJ002-0104.wav|can be permitted to enter the interior of this prison. End quote. The tap-room and bar were just behind the felons' entrance lodge, +DUMMY/LJ049-0127.wav|agencies other than the Secret Service have become involved in phases of the overall problem of protecting our national leaders. +DUMMY/LJ047-0144.wav|The FBI representative in Mexico City arranged to follow up this information with the CIA and to verify Oswald's entry into Mexico. +DUMMY/LJ039-0119.wav|We have nothing here to show under what conditions the B course was fired. It might well have been a bad day for firing the rifle +DUMMY/LJ042-0188.wav|which would achieve the goal, which was, quote, the emplacement of a separate, democratic, pure communist society +DUMMY/LJ018-0209.wav|It was not easy to detect coiners, or bring home their guilt to them. +DUMMY/LJ045-0227.wav|and suggests the possibility, as did his note to his wife just prior to the attempt on General Walker, that he did not expect to escape at all. +DUMMY/LJ045-0022.wav|even though he did, as he told Johnson, attend a meeting at which General Walker spoke to approximately thirteen hundred persons. +DUMMY/LJ004-0237.wav|The first still produced deplorable results -- results to be observable for many years to come. +DUMMY/LJ044-0052.wav|Furthermore, the general tenor of Oswald's next letter to V. T. Lee, +DUMMY/LJ028-0061.wav|at the head of the Babylonian army, he defeated the Egyptians in the famous battle of Carchemish, the old Hittite capital, +DUMMY/LJ003-0325.wav|The feelings of the condemned prisoners should no longer be outraged by their exposure in the chapel, and the chapel should be rearranged, +DUMMY/LJ014-0049.wav|Restoratives were applied, but he was in a fainting condition when tied, and had to be supported by the assistant executioner +DUMMY/LJ048-0132.wav|which would bring to bear the judgment and experience of members of the White House detail other than the advance agent. +DUMMY/LJ005-0133.wav|The provision of separate sleeping cells was still quite inadequate. For instance, +DUMMY/LJ020-0022.wav|Wash out the bowl with a little lukewarm water and add this to the dough. If it should prove too soft, work in, cautiously, a little more flour. +DUMMY/LJ022-0202.wav|That faith is receiving its just reward. +DUMMY/LJ012-0003.wav|The two men on guard had gone off immediately after Mr. Gee had escaped, but they returned later in the day, and were apprehended. +DUMMY/LJ015-0247.wav|At that station the safes were given out, heavy with shot, not gold; the thieves went on to Dover, and by-and-by, +DUMMY/LJ018-0169.wav|In one corner was a printing-press actually in use, and on it were twenty-one forged Bank of England notes, without date or signature. +DUMMY/LJ038-0003.wav|Oswald's Arrest +DUMMY/LJ031-0015.wav|Arriving almost simultaneously were the President's follow-up car, the Vice President's automobile, and the Vice President's follow-up car. +DUMMY/LJ025-0008.wav|This is true only as a matter of detail, +DUMMY/LJ024-0016.wav|If, for instance, any one of the six justices of the Supreme Court now over the age of seventy should retire as provided under the plan, +DUMMY/LJ027-0034.wav|Hence, as Jordan has said, "the inside of an animal tells the real history of its ancestry; the outside tells us only where its ancestors have been." +DUMMY/LJ039-0134.wav|During one of his leaves from the Marines, Oswald hunted with his brother Robert, +DUMMY/LJ031-0214.wav|The autopsy examination revealed two wounds in the President's head. +DUMMY/LJ018-0331.wav|to murder her mistress, Madame Riel, in Park Lane, as Courvoisier, the Swiss, had been tempted to murder Lord William Russell. +DUMMY/LJ037-0077.wav|Barbara Jeanette Davis and Virginia Davis, were in an apartment of a multiple-unit house on the southeast corner of tenth and Patton +DUMMY/LJ018-0012.wav|It was that of an aged man, whose head had been battered in by a life-preserver. +DUMMY/LJ003-0285.wav|they should gamble with dice or cards, and play at bumble puppy or some other disreputable game of chance. +DUMMY/LJ010-0018.wav|jewel-boxes cleverly stolen under the very noses of owners or care-takers. +DUMMY/LJ028-0430.wav|The Euphrates flowed through it, but the greater part of the city was on the eastern shore. +DUMMY/LJ035-0190.wav|testified that after driving to Parkland Hospital, he returned to the Depository Building about twenty minutes after the shooting, +DUMMY/LJ018-0126.wav|His object in visiting Whitchurch was to undermine the honesty of some workman in the mills; +DUMMY/LJ012-0227.wav|one that of the murder of the Italian boy, the other that of a boy unknown. +DUMMY/LJ043-0128.wav|The plant superintendent of the new firm testified that, one of the employees of the old firm, quote, +DUMMY/LJ019-0395.wav|at others an expensive staff was maintained with little or nothing to do. +DUMMY/LJ021-0126.wav|to organize themselves for collective bargaining as embodied in Section seven (a) of the national Industrial Recovery Act. +DUMMY/LJ014-0091.wav|This cold-blooded indifference after the event was only outdone by the premeditation of this horrible murder. +DUMMY/LJ028-0286.wav|He therefore reviewed all ways of bringing the city under, +DUMMY/LJ036-0002.wav|Chapter four. The Assassin: Part five. +DUMMY/LJ014-0244.wav|which offered peculiar chances of profit to an ingenious and unscrupulous man. +DUMMY/LJ037-0214.wav|When shown the revolver, she stated that she recognized it as the one owned by her husband. +DUMMY/LJ041-0023.wav|and, quote, Arthor Abear, end quote, most likely Arthur Hebert, a classmate who has said that he did not know Oswald well. +DUMMY/LJ017-0030.wav|when the spread of scientific knowledge places nefarious means at the disposal of many, +DUMMY/LJ023-0092.wav|But in the case holding the Railroad Retirement Act unconstitutional, for instance, +DUMMY/LJ006-0046.wav|There was no longer the faintest possible excuse for overcrowding. The numbers now committed to Newgate had sensibly diminished. +DUMMY/LJ008-0174.wav|One cart-load of spectators having broken down, some of its occupants fell off the vehicle, and were instantly trampled to death. +DUMMY/LJ044-0121.wav|handicapped as it were, by my past record and compete with anti-progressive forces, above-ground or whether in your opinion +DUMMY/LJ049-0033.wav|to move into the passenger section without hindrance or delay. Had the vehicle been so designed it is possible that an agent riding in the front seat +DUMMY/LJ005-0089.wav|The religious and moral welfare of the prisoners were to be attended to, +DUMMY/LJ045-0060.wav|He apparently attempted to be "the Commander" by dictating many of the details of their married life. +DUMMY/LJ043-0120.wav|His Fair Play for Cuba Committee activities, however, made it more difficult for him to obtain other employment. +DUMMY/LJ037-0095.wav|Smith failed to make himself known to the police on November twenty-two. +DUMMY/LJ016-0436.wav|pointing to an aperture for ventilating the cell. On the morning of execution he asked how far it was to the gallows, and was told it was quite close. +DUMMY/LJ041-0167.wav|While waiting for the parade to start they talked briefly about "nineteen eighty-four" +DUMMY/LJ046-0027.wav|However, all information considered by the Commission which pertains to the protective function as it was carried out in Dallas has been published as part of this report. +DUMMY/LJ011-0247.wav|Mr. Mullay became alarmed, and as Mr. Owen did not appear, withdrew; +DUMMY/LJ027-0131.wav|of descent discovered in the development of plant and animal embryos. +DUMMY/LJ027-0173.wav|The caducibranch takes first the fish form, then the perennibranch form, and finally the caducibranch form, but goes no further. +DUMMY/LJ032-0094.wav|listing Marina Oswald and A. J. Hidell +DUMMY/LJ038-0131.wav|Oswald replied, quote, I don't know anything about that, end quote. +DUMMY/LJ011-0119.wav|One was Richard Gifford, a well-educated youth who had been at Christ's Hospital, and afterwards in the National Debt Office. +DUMMY/LJ024-0142.wav|I seek to make American democracy succeed. +DUMMY/LJ010-0255.wav|This was in May eighteen forty-two. +DUMMY/LJ010-0221.wav|Another letter directed Oxford to attend an extraordinary meeting of "Young England" +DUMMY/LJ050-0192.wav|can lead to lapses in protection, such as the confusion in Dallas about whether members of the public were permitted on overpasses. +DUMMY/LJ016-0292.wav|bricklayers' laborers, dock workmen, German artisans and sugar-bakers +DUMMY/LJ037-0211.wav|The signature of the witness, D. F. Drittal, +DUMMY/LJ041-0016.wav|Voebel took Oswald back to the school to attend to his wounds, and their, quote, mild friendship, end quote, stemmed from that incident. +DUMMY/LJ008-0026.wav|Palm, as soon as he was seated in the cart, +DUMMY/LJ013-0176.wav|also the jewel-box and note-case, from the latter of which was abstracted a ten-pound note known to have been in the possession of the deceased. +DUMMY/LJ013-0238.wav|He was coachman to a gentleman at Roehampton. One day he went into a pawnbroker's at Wandsworth, and bought a pair of breeches on credit. +DUMMY/LJ005-0060.wav|The Society did not limit its remarks to the description of what had already been done +DUMMY/LJ009-0237.wav|the poor criminal rose straight upon his feet, with the hangman on his shoulders, and immediately loosened the rope with his fingers. +DUMMY/LJ001-0059.wav|the greater part of these Italian printers, it should be mentioned, were Germans or Frenchmen, working under the influence of Italian opinion and aims. +DUMMY/LJ042-0204.wav|but in servile conformity to the wishes of the Soviet Union and in anticipation of Soviet Russia's complete domination of the American continent. +DUMMY/LJ042-0184.wav|Expanding on his ideas on how his alternative to communism and capitalism might be introduced, he wrote of a, quote, readily foreseeable +DUMMY/LJ028-0066.wav|transported the Jews to Babylon, and brought the Hebrew kingdom to an end. +DUMMY/LJ012-0221.wav|In another portion of the garden more clothing, partly male and partly female, was discovered, +DUMMY/LJ040-0058.wav|after June of nineteen sixty-two. +DUMMY/LJ033-0188.wav|and the paper and tape found in the shipping room of the Depository on November twenty-two +DUMMY/LJ028-0053.wav|The next year, in six oh five, Nabopolassar died, and Nebuchadnezzar succeeded him to the throne. +DUMMY/LJ018-0229.wav|The day his father died he got the keys of his private bureau, opened it, and took out the authentic will. +DUMMY/LJ030-0048.wav|Their main function was to spot trouble in advance and to direct any necessary steps to meet the trouble. +DUMMY/LJ038-0136.wav|In concluding that Oswald was carrying a rifle in the paper bag on the morning of November twenty-two, nineteen sixty-three, +DUMMY/LJ028-0320.wav|will prove the greatest gain that you could possibly receive, while to Darius and the Persians it will be the severest loss. +DUMMY/LJ039-0056.wav|the Commission considered whether Oswald, using his own rifle, +DUMMY/LJ012-0162.wav|Having brought down the records of great frauds, forgeries, and thefts from about eighteen twenty-five to eighteen forty, +DUMMY/LJ040-0203.wav|but essentially a, quote, defensive, rigid, self-involved person +DUMMY/LJ046-0216.wav|members of the immediate family of the President, the President-Elect or the Vice-President +DUMMY/LJ003-0309.wav|Proper hours for locking and unlocking prisoners should be insisted upon; +DUMMY/LJ022-0127.wav|The most effective means of preventing such evils in this Works Relief program will be the eternal vigilance of the American people themselves. +DUMMY/LJ028-0271.wav|Now when a year and seven months had passed, Darius and his army were quite wearied out, finding that they could not anyhow take the city. +DUMMY/LJ047-0178.wav|because she thought the FBI was in possession of a great deal of information and certainly would find it very easy to learn where Oswald was living. +DUMMY/LJ048-0141.wav|Fire Department, County Sheriff's Department, and the Texas Department of Public Safety. +DUMMY/LJ003-0245.wav|preparatory to their appearance in the Old Bailey. Irons were seldom removed from the convicted until discharge; +DUMMY/LJ013-0218.wav|His last statement contains the words, "The public now think I am a liar, and they will not believe me when I say the truth." +DUMMY/LJ044-0029.wav|It appears to have been a solitary operation on Oswald's part in spite of his misstatements to the New Orleans police that it had thirty-five members, +DUMMY/LJ035-0165.wav|At that time the building had not yet been sealed off by the police. +DUMMY/LJ009-0179.wav|the most recent enactment in force was the ninth George the fourth cap. thirty-one, which directed the dissection of all bodies of executed murderers, +DUMMY/LJ006-0127.wav|in view of speaking to it at the trial, and he could easily do them a good turn -- or a very bad one. +DUMMY/LJ034-0144.wav|Prior to the lineup, +DUMMY/LJ044-0015.wav|He first distributed his handbills and other material uneventfully in the vicinity of the U.S.S. Wasp, +DUMMY/LJ050-0046.wav|it has obtained the services of outside consultants, such as the Rand Corporation, +DUMMY/LJ042-0115.wav|The best description of Oswald's state of mind, however, is set forth in his own "Historic Diary." +DUMMY/LJ013-0009.wav|This was the willful shipwreck and casting away of a vessel which, with her supposed cargo, had been heavily insured. +DUMMY/LJ007-0238.wav|At the same time the inspectors animadvert strongly upon the misconduct of prisoners and the frequency of prison punishments, +DUMMY/LJ035-0088.wav|park it, and run forty-five feet to the building. +DUMMY/LJ050-0232.wav|to assist in its protection functions. +DUMMY/LJ027-0042.wav|All such cases of resemblance in function but not in detailed plan of structure are called "analogies," +DUMMY/LJ002-0248.wav|The population, as reported by the committee in eighteen fourteen, averaged about sixty, +DUMMY/LJ029-0034.wav|In his experience, quote, it would be automatic, end quote, for the Secret Service to arrange a route which would, within the time allotted, +DUMMY/LJ022-0109.wav|The allotment of funds for desirable projects has already begun. +DUMMY/LJ011-0100.wav|to which the signature of one Wilkins of Abingdon was found to be forged. +DUMMY/LJ037-0236.wav|She last saw him in the parking lot directly behind the service station. +DUMMY/LJ047-0179.wav|Hosty did nothing further in connection with the Oswald case until after the assassination. On November one, nineteen sixty-three, +DUMMY/LJ033-0043.wav|On the west end of the house is the attached garage, which can be reached from the kitchen or from the outside. +DUMMY/LJ010-0218.wav|in an instant our faces were covered, we cocked our pistols, and with drawn swords stood waiting to receive the enemy. +DUMMY/LJ021-0124.wav|when the farmers were demanding legislation which would give them opportunities and incentives to organize themselves for a common advance, +DUMMY/LJ010-0168.wav|after which he devoted himself to pistol practice in shooting-galleries, sometimes in Leicester Square, +DUMMY/LJ042-0250.wav|He sought an immediate grant of visa on his trip to Mexico City in late September of nineteen sixty-three. +DUMMY/LJ008-0131.wav|the other he kept between his hands. +DUMMY/LJ003-0339.wav|could possibly afford. +DUMMY/LJ050-0176.wav|to establish liaison with local intelligence gathering agencies and to provide for the immediate evaluation of information received from them. +DUMMY/LJ047-0069.wav|that the Oswalds were living at two one four Neely Street in Dallas. +DUMMY/LJ001-0125.wav|this is the narrowing of the modern letters. +DUMMY/LJ021-0032.wav|have been restored to complete public confidence. +DUMMY/LJ019-0168.wav|Renewed recommendations to provide employment resulted in the provision of a certain amount of oakum for picking, +DUMMY/LJ028-0116.wav|I raised its summit mountain high. I constructed a moat-wall of burned bricks about the west wall of Babylon. +DUMMY/LJ042-0055.wav|In the event of war I would kill any American who put a uniform on in defense of the American government -- any American. +DUMMY/LJ034-0201.wav|but a radio reporter testified that Euins described the man to him as, quote, colored, end quote. +DUMMY/LJ019-0367.wav|and with the warning a copy of the particular defects and allegations was to be sent to the local authority. +DUMMY/LJ039-0189.wav|The others would have been able to reduce their times if they had been given the opportunity to become familiar with the movement of the bolt and the trigger pull. +DUMMY/LJ025-0090.wav|and resemblances between the constituents of animal and vegetable organisms, for which Cuvier is not responsible, +DUMMY/LJ002-0044.wav|the whole about fifteen feet wide. +DUMMY/LJ031-0035.wav|For a moment, Mrs. Kennedy refused to release the President, whom she held in her lap, +DUMMY/LJ015-0294.wav|He forgot to add that it was to be placed to Ralph's credit, and when he called as Ralph, +DUMMY/LJ016-0308.wav|yet the witnesses were not unanimous. +DUMMY/LJ003-0085.wav|These twenty-four could just sleep on the barrack bed; when the number was higher, and it often rose to forty, the surplus had to sleep on the floor. +DUMMY/LJ039-0012.wav|The Commission first learned of this incident when Robert Oswald related it to FBI agents on February nineteen, nineteen sixty-four, +DUMMY/LJ028-0235.wav|In this distress either someone made this suggestion to him, or he bethought himself of a plan which he proceeded to put in execution. +DUMMY/LJ043-0074.wav|where he expressed a reluctance to work in the industrial field. +DUMMY/LJ015-0284.wav|In order to obtain messengers of this sort, Saward answered advertisements of persons seeking employment, +DUMMY/LJ011-0202.wav|The uncle claimed her. The husband resisted. +DUMMY/LJ027-0036.wav|For example, there is a superficial resemblance between the wing of an insect and the wing of a bird +DUMMY/LJ027-0030.wav|is associated with an adaptive modification of locomotor structures, legs and wings, +DUMMY/LJ019-0228.wav|it also contained eleven reception cells, six punishment cells, and a couple of cells for condemned criminals. +DUMMY/LJ036-0040.wav|In a later interview, Jones confirmed that he had exchanged words with a woman passenger on the bus during the ride south on Marsalis. +DUMMY/LJ010-0263.wav|The policemen treated the matter as a hoax, and allowed the culprit to make off. +DUMMY/LJ034-0142.wav|During the evening of November twenty-two, Brennan identified Oswald as the person in the lineup who bore the closest resemblance to the man in the window +DUMMY/LJ027-0014.wav|and the equally mysterious nuclear substance or chromatin +DUMMY/LJ019-0275.wav|that the committee of eighteen sixty-three fully endorsed Sir Joshua's recommendations as regards a "hard bed," and recommended that +DUMMY/LJ029-0009.wav|An evaluation of the procedures employed to safeguard the President, with recommendations for improving these procedures, appears in Chapter eight of the report. +DUMMY/LJ030-0153.wav|William Greer, operator of the Presidential limousine, estimated the car's speed at the time of the first shot as twelve to fifteen miles per hour. +DUMMY/LJ021-0022.wav|between the small producer, the small trader, the consumer, +DUMMY/LJ047-0095.wav|On the next day, he asked the New Orleans police to arrange for him to be interviewed by the FBI. +DUMMY/LJ003-0204.wav|I saw Cashman a few hours before his execution, smoking and drinking with the utmost unconcern and indifference, end quote. +DUMMY/LJ045-0081.wav|Although she denied it in some of her testimony before the Commission, +DUMMY/LJ018-0175.wav|This was penal servitude for life, Buncher's sentence being twenty-five, and Burnett's twenty years. +DUMMY/LJ002-0169.wav|He had been in the employ of a corn-chandler at Islington, and went into London with his master's cart and horse. +DUMMY/LJ035-0104.wav|The east elevator was on the fifth floor when they arrived; the west elevator was not. +DUMMY/LJ037-0037.wav|As she waited on the northwest corner of the intersection for traffic to pass, she noticed a young man as he was, quote, almost ready to get up on the curb, end quote. +DUMMY/LJ013-0110.wav|but four hundred sovereigns were found on Elder, while two hundred more were found in Burgess' effects. +DUMMY/LJ011-0276.wav|"A chain fixed to staples at his back passed round his chest under his arms, and was padlocked on the left side;" +DUMMY/LJ048-0038.wav|The Commission believes, however, +DUMMY/LJ049-0119.wav|would be conducted by Federal law enforcement officials, in particular, the FBI with the assistance of the Secret Service. +DUMMY/LJ048-0063.wav|The formal FBI instructions to its agents outlining the information to be referred to the Secret Service were too narrow at the time of the assassination. +DUMMY/LJ036-0158.wav|Moreover, Mrs. Bledsoe saw Oswald in the bus without a jacket and wearing a shirt with a hole at the elbow. +DUMMY/LJ049-0189.wav|but expand both the scope and the operations of the existing agencies, particularly those of the Secret Service and the FBI. +DUMMY/LJ049-0201.wav|and becoming merely the recipient of information gathered by others would become limited solely to acts of physical alertness and personal courage +DUMMY/LJ038-0279.wav|I think is going overboard in the other direction. +DUMMY/LJ008-0024.wav|of the trading vessel 'Adventure,' upon the high seas. +DUMMY/LJ003-0120.wav|The culprit was then compelled to carry the chair about with him. +DUMMY/LJ011-0277.wav|his feet were bound with cords and made fast to rings in the floor. Thus manacled, one of the party, who pretended to be Mrs. Canning's brother, +DUMMY/LJ025-0165.wav|Neither protein, nor fat, nor starch, nor sugar, nor any substance in the slightest degree resembling them has formed part of the food of the bean. +DUMMY/LJ021-0176.wav|Did England let nature take her course? No. +DUMMY/LJ037-0094.wav|He looked west on tenth and saw a man running to the west and a policeman falling to the ground. +DUMMY/LJ018-0314.wav|Certain friends of the prisoners were watched, and found to be in communication with these warders, +DUMMY/LJ007-0036.wav|one man declared that the language of the condemned rooms was disgusting, that he was dying a death every day in being compelled to associate with such characters. +DUMMY/LJ019-0390.wav|Since then a strong central authority has labored steadfastly to compass concentration, +DUMMY/LJ034-0002.wav|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter four. The Assassin: Part three. +DUMMY/LJ010-0315.wav|and sold out to the amount here stated, and without the knowledge of my partners. +DUMMY/LJ038-0219.wav|would be consistent with the period when the Oswalds were living on Neely Street since the apartment was rented on March three, nineteen sixty-three. +DUMMY/LJ013-0236.wav|A murder which reproduced many of the features of that committed by Greenacre soon followed, and excited the public mind even more than that of Courvoisier's. +DUMMY/LJ008-0088.wav|As many spectators as ever thronged to see the dreadful show, +DUMMY/LJ049-0059.wav|Agents are instructed that it is not their responsibility to investigate or evaluate a present danger, +DUMMY/LJ025-0166.wav|But the weights of the carbon, hydrogen, oxygen, nitrogen, phosphorus, sulfur and other elementary bodies contained in the bean-plant +DUMMY/LJ021-0180.wav|to issue new bonds therefore bearing only three and one half percent interest, +DUMMY/LJ007-0203.wav|"The mischief of jail associations," say the inspectors, +DUMMY/LJ014-0205.wav|On the dread morning he came out to execution quite gaily, and tripped up the stairs to the scaffold. +DUMMY/LJ006-0307.wav|There was reveling and roistering, as usual, with "high life below-stairs. +DUMMY/LJ005-0123.wav|with a few of the more important city or borough prisons, these acts more especially applied. +DUMMY/LJ008-0025.wav|They were taken in a cart to the place of execution, amidst a vast concourse of people. +DUMMY/LJ014-0219.wav|Once more Dwyer got to his feet, and managed, by drawing his staff, to keep Cannon at bay until a second constable came to his aid. +DUMMY/LJ004-0241.wav|Other cases of moral deterioration have already been recorded. +DUMMY/LJ032-0233.wav|reinforce the belief that the rifle in the photograph is the rifle which Oswald bought from Klein's. +DUMMY/LJ009-0257.wav|and having got from the falling drop to the firm part of the platform, he nearly succeeded in tearing the handkerchief from his eyes. +DUMMY/LJ005-0126.wav|Religious worship became more generally the rule; chaplains were appointed, and chapels provided for them; surgeons and hospitals also. +DUMMY/LJ040-0034.wav|Marina Oswald thought that he would not be happy anywhere, quote, Only on the moon, perhaps, end quote. +DUMMY/LJ009-0028.wav|Probably the reader will side with the aldermen against the ordinary. +DUMMY/LJ015-0095.wav|The blow fell suddenly, and when least expected. One morning Mr. Fasson asked casually for certain certificates, +DUMMY/LJ006-0282.wav|the pump was the only provision, and this in a place within sight of visitors, of the windows of the male turnkeys, and unprotected from the weather. +DUMMY/LJ017-0024.wav|and as Fenning, although convicted and executed, protested her innocence in the most solemn manner to the last, +DUMMY/LJ044-0031.wav|Oswald's Fair Play for Cuba activities may be viewed as a very shrewd political operation +DUMMY/LJ039-0064.wav|An aerial photograph of Dealey Plaza shows that Elm Street runs at an angle +DUMMY/LJ030-0005.wav|by helicopter at ten:forty-five A.M., Eastern Standard Time, on November twenty-one, nineteen sixty-three, for Andrews Air Force Base. +DUMMY/LJ002-0124.wav|Quite half of the foregoing writs and arrests applied to sums under thirty pounds. +DUMMY/LJ046-0228.wav|the Secret Service had no standard procedure for the systematic review of its requests for and receipt of information from other Federal agencies. +DUMMY/LJ028-0415.wav|but only part, and that a small part, of the great tower, which God hath suffered to stand +DUMMY/LJ029-0152.wav|to inspect buildings along the motorcade route since the Secret Service did not normally request or make such a check. +DUMMY/LJ012-0123.wav|Within three months, three new and most mysterious burglaries were committed at the West End, all in houses adjoining each other. +DUMMY/LJ008-0052.wav|it is supported by strong posts fixed into grooves made in the street; +DUMMY/LJ002-0210.wav|the keeper of the large tap-room called the Brace, because it was once kept by two brothers named Partridge, also paid him toll. +DUMMY/LJ045-0209.wav|but it was also, at least in part, because his wife did not want to live there with him. +DUMMY/LJ038-0013.wav|The man wore a T-shirt beneath his outer shirt and he had no jacket. +DUMMY/LJ049-0027.wav|Presidents have made it clear, however, that they did not favor this or any other arrangement which interferes with the privacy of the President and his guests. +DUMMY/LJ020-0051.wav|Flour a rolling-pin and roll the dough into a sheet not more than half an inch thick. +DUMMY/LJ013-0092.wav|The thieves, for Elder was soon joined by Burgess, drove together to Ben Caunt's, the pugilist's, public-house in St. Martin's Lane, +DUMMY/LJ005-0175.wav|At this time there were in England one hundred and seventy boroughs, cities, towns, and liberties +DUMMY/LJ030-0068.wav|Kellerman's responsibilities included maintaining radio communications with the lead and follow-up cars, +DUMMY/LJ039-0112.wav|evaluated the sharpshooter qualification as a, quote, fairly good shot, end quote, and a low marksman rating +DUMMY/LJ039-0013.wav|and to the Commission on February twenty-one. +DUMMY/LJ033-0107.wav|No other employee has been found who saw Oswald enter that morning. +DUMMY/LJ014-0230.wav|For this he was executed in front of Newgate in eighteen thirty-three. +DUMMY/LJ038-0059.wav|that there was no one near Oswald who had a shotgun and he saw no one strike Oswald in the back with a rifle butt or the butt of a gun. +DUMMY/LJ044-0099.wav|Stuckey thought that Oswald acted very much as would a young attorney. +DUMMY/LJ009-0215.wav|All the shops in the neighborhood were shut, and the windows and tops of the houses were crowded with spectators. +DUMMY/LJ050-0260.wav|the Secret Service has been receiving full cooperation in scientific research and technological development +DUMMY/LJ014-0058.wav|These great criminals suffered at Horsemonger Lane Jail, but they were tried at the Central Criminal Court, and were for some time inmates of Newgate. +DUMMY/LJ005-0007.wav|chained to the rest. As the miserable crew filed through the public streets, exposed to the scornful gaze of every passenger, +DUMMY/LJ029-0066.wav|No one else was identified to the Secret Service through local inquiry as potentially dangerous, +DUMMY/LJ016-0204.wav|For acting as executioner of Horsemonger Lane Jail +DUMMY/LJ019-0295.wav|A single officer was the only custodian and disciplinary authority in the jail. +DUMMY/LJ025-0145.wav|Hence it is conceivable that even the simplest living being may possess a nervous system. +DUMMY/LJ033-0098.wav|Oswald left the car first, picked up the brown paper bag, and proceeded toward the building ahead of Frazier. +DUMMY/LJ006-0060.wav|The classification prescribed by the Jail Act, which laid down that certain prisoners should not intermix, was openly neglected, +DUMMY/LJ006-0298.wav|while older women in infirm health were sent across the seas. +DUMMY/LJ030-0155.wav|A more precise determination has been made from motion pictures taken on the scene by an amateur photographer, Abraham Zapruder. +DUMMY/LJ016-0245.wav|Until the time of his death he kept a small shop close to the church in Horncastle. +DUMMY/LJ038-0242.wav|An examination of certain construction work appearing in the background of this photograph revealed that the picture was taken between March eight +DUMMY/LJ021-0172.wav|They point to England. +DUMMY/LJ038-0042.wav|Detective Bob K. Carroll, who was standing beside McDonald, seized the gun from him. +DUMMY/LJ010-0079.wav|and thus the conspiracy in which he was the prime mover took shape, and came to a head. +DUMMY/LJ018-0236.wav|One large item of his expenditure was a contested election at Lambeth, which he gained at a cost of ten thousand pounds. +DUMMY/LJ013-0177.wav|His lordship's watch was gone. +DUMMY/LJ045-0235.wav|the testimony of the officers present indicates that he handled himself with considerable composure during his questioning. +DUMMY/LJ011-0037.wav|Appeals were made to the Home Secretary, and all possible political interest brought to bear, but without success. +DUMMY/LJ005-0229.wav|By another clause of the Jail Act, two justices were to be appointed to visit the prison at least thrice in every quarter, and "oftener if occasion required." +DUMMY/LJ047-0021.wav|information regarding his relations with the U.S. Embassy in Moscow and background data relating largely to his prior military service, +DUMMY/LJ019-0204.wav|a serious effort was made to reconstruct Newgate. +DUMMY/LJ005-0242.wav|In the year immediately preceding this, Parliament was too busy with the great question of its own reform to spare much time for domestic legislation. +DUMMY/LJ018-0112.wav|It was one of these, Glendinning, who had allowed himself to be utilized for some time in this way, whose capture led to the breaking up of the gang. +DUMMY/LJ049-0123.wav|over the assassination of President Kennedy led to embarrassment and confusion in the subsequent investigation by Federal and local authorities. +DUMMY/LJ022-0031.wav|That is why I occasionally leave this scene of action for a few days +DUMMY/LJ005-0054.wav|traced the progress of prison architecture from the days when the jail was the mere annexe of the baronial or episcopal castle +DUMMY/LJ014-0038.wav|and proceeding to where they came from, found Delarue dead, slain by the furious brother. +DUMMY/LJ032-0093.wav|but Oswald had rented post office box three zero zero six one in New Orleans on June three, nineteen sixty-three, +DUMMY/LJ002-0333.wav|a severe winter set in, the new stock had not been supplied by the contractors, and the poor debtors perished of cold. +DUMMY/LJ048-0279.wav|They work long, hard hours, under very great strain, and must travel frequently. +DUMMY/LJ048-0012.wav|According to Belmont, when Oswald returned from the Soviet Union, quote, +DUMMY/LJ017-0262.wav|Two of the seven, Santos and Marsolino, were reprieved, and their sentences commuted to penal servitude for life; +DUMMY/LJ050-0172.wav|The Commission, therefore, recommends that the President consider ordering an inquiry into the possibility +DUMMY/LJ019-0134.wav|radical improvement was generally considered impossible. The great evil, however, had been sensibly diminished. +DUMMY/LJ006-0185.wav|Edward Gibbon Wakefield, who spent three years in Newgate a little before the time of the inspectors' first report, +DUMMY/LJ011-0083.wav|For a long time justice did not overtake him for any criminal offense, but he was frequently in Newgate and in the King's Bench for debt. +DUMMY/LJ006-0169.wav|He really did not know what passed in his jail +DUMMY/LJ044-0202.wav|by a desire to aid the Castro regime, which President Kennedy so outspokenly criticized. +DUMMY/LJ050-0083.wav|Information is requested also concerning individuals or groups who have demonstrated an interest in the President +DUMMY/LJ021-0137.wav|Step by step we have created all the government agencies necessary to insure, as a general rule, industrial peace, +DUMMY/LJ024-0087.wav|I have thus explained to you the reasons that lie behind our efforts to secure results by legislation within the Constitution. +DUMMY/LJ046-0230.wav|prior to November twenty-two, nineteen sixty-three, were inadequate. +DUMMY/LJ006-0156.wav|We have seen that he took no pains to classify and separate prisoners on reception. +DUMMY/LJ046-0062.wav|His friend and Special Assistant Kenneth O'Donnell, who accompanied him on his last visit to Dallas, +DUMMY/LJ032-0165.wav|to the exclusion of all others because there are not enough microscopic characteristics present in fibers. +DUMMY/LJ018-0194.wav|Cummings was repeatedly "run in" for the offense of coining and uttering bad money, whether coin or notes. +DUMMY/LJ003-0311.wav|The committee took upon itself to lay down stringent rules for the discipline of the prison. +DUMMY/LJ041-0092.wav|an, quote, individual that you would brainwash, and quite easy, but I think once he believed in something, he stood in his beliefs, end quote. +DUMMY/LJ020-0043.wav|After mixing your bread in the morning either with sponge or with yeast, divide the kneaded dough into two portions. +DUMMY/LJ017-0104.wav|prescribed nominally by the fresh doctor, for which Palmer had substituted his own. +DUMMY/LJ011-0028.wav|In his defense +DUMMY/LJ035-0080.wav|the second test, at a "fast walk" took one minute, fourteen seconds. +DUMMY/LJ030-0244.wav|hitting me on the shoulder, and shouted to all of us in the back seat to get down. +DUMMY/LJ041-0158.wav|Thornley, who thought Oswald had an "irrevocable conviction" that his Marxist beliefs were correct, testified, quote, +DUMMY/LJ003-0010.wav|wine and beer might be had in any quantity, the only limitation being +DUMMY/LJ034-0170.wav|Fischer testified that about ten or fifteen seconds before the motorcade turned onto Houston Street from Main Street, +DUMMY/LJ021-0121.wav|But I would point out that the extent and severity of labor disputes during this period +DUMMY/LJ006-0269.wav|Decency was, however, insured by a line of demarcation, and visitors were kept upon each side of a separated double iron railing. +DUMMY/LJ029-0187.wav|On the evening of October twenty-four, nineteen sixty-three, after addressing a meeting in Dallas, +DUMMY/LJ004-0030.wav|the principle of classification was freshly enunciated; prison regulations were framed for general observance. +DUMMY/LJ014-0340.wav|They could not deny that the latter was the truth, and were forthwith stigmatized by Mr. Chapman, Overend and Gurney's representative, as rogues. +DUMMY/LJ048-0136.wav|In the description of the important aspects of the advance preparations, +DUMMY/LJ006-0139.wav|Nobody interfered with them or regulated their conduct. They might get drunk when so disposed, and did so frequently, alone or in company. +DUMMY/LJ018-0290.wav|by paying in a genuine bill of Messrs. Rothschilds' for forty-five hundred pounds, which was duly discounted. +DUMMY/LJ041-0201.wav|Of greater importance, +DUMMY/LJ043-0144.wav|and the March eleven, nineteen sixty-three, issue of the Militant. +DUMMY/LJ043-0088.wav|On October eleven, nineteen sixty-two, the Employment Commission referred Oswald +DUMMY/LJ036-0039.wav|In his Commission testimony, McWatters said he had been in error and that a teenager named Milton Jones was the passenger he had in mind. +DUMMY/LJ030-0017.wav|The President liked outdoor appearances because more people could see and hear him. +DUMMY/LJ017-0144.wav|There was no convincing proof that the accused had administered the poison, although beyond question that poison had occasioned the death. +DUMMY/LJ040-0158.wav|Lee has to be seen as an emotionally, quite disturbed youngster +DUMMY/LJ016-0086.wav|were fixed over the fireplaces, and the prisoners had no longer access to the chimneys. +DUMMY/LJ039-0008.wav|told Robert Oswald, Lee Harvey Oswald's brother, that Oswald had once threatened to shoot former Vice President Richard M. Nixon. +DUMMY/LJ008-0265.wav|although in the long run mercy generally prevailed, and very few capitally convicted for crimes less than murder actually suffered. +DUMMY/LJ047-0236.wav|was capable or potentially an assassin of the President of the United States, end quote. +DUMMY/LJ003-0275.wav|These malpractices were fostered by the absence of all supervision and the generally unbroken idleness. +DUMMY/LJ033-0116.wav|Frazier placed the upper part of the package under his armpit and attempted to cup his right hand beneath the bottom of the bag. +DUMMY/LJ010-0274.wav|Much the same motives of seeking notoriety seem to have impelled Bean, who was perfectly sane, to his rash act; +DUMMY/LJ009-0225.wav|but not before the greatest scandal had been caused by the unseemly proceedings of the crowd that flocked to see the sight. A sort of fair was held, +DUMMY/LJ003-0172.wav|The keeper went still further in his efforts to make money. +DUMMY/LJ028-0077.wav|offering for sale a small cameo engraved with a helmeted head of a Greek type. +DUMMY/LJ021-0109.wav|whether the methods and policies adopted in the emergency +DUMMY/LJ034-0099.wav|After hearing the first shot, which he thought was a motorcycle backfire, Brennan glanced up at the window. He testified that, quote, +DUMMY/LJ022-0150.wav|to eliminate so far as humanly possible, the kinds of unfair practices by selfish minorities which unfortunately +DUMMY/LJ021-0107.wav|It is now time to review these actions as a whole to determine through deliberative means in the light of experience, +DUMMY/LJ016-0278.wav|and which contributed in no small degree to the introduction of private executions. A great crowd was expected, and a great crowd came. +DUMMY/LJ043-0168.wav|he removed at least some of the pictures which had been pasted in it and saved them among his effects, where they were found after the assassination. +DUMMY/LJ013-0024.wav|At a place called the Silver Keys he ran the ship on the reef. +DUMMY/LJ037-0187.wav|the company received from Empire Wholesale Sporting Goods, Ltd., Montreal, a shipment of ninety-nine guns in one case. +DUMMY/LJ010-0063.wav|The massacre of the whole of the Cabinet Ministers at one stroke was to be followed by an attack +DUMMY/LJ009-0252.wav|On the scaffold he made a violent attempt to loosen his bonds, and succeeded in getting his hands free. +DUMMY/LJ024-0002.wav|Section ten. March nine, nineteen thirty-seven. Part two. +DUMMY/LJ002-0295.wav|This was the reprehensible claim for "garnish," which had already been abolished in all well-conducted prisons, but which still was demanded in Newgate. +DUMMY/LJ024-0091.wav|Each one is radically different from the other. +DUMMY/LJ037-0152.wav|Identification was effected through breech face marks and firing pin marks. +DUMMY/LJ028-0208.wav|Nabonidus fled. +DUMMY/LJ016-0309.wav|Some of the judges would have retained the public spectacle; the ordinary of Newgate was not certain that public executions were not the best. +DUMMY/LJ003-0158.wav|from whom still higher fees were exacted, with the same discreditable idea of swelling the revenues of the prison. +DUMMY/LJ021-0170.wav|complain that all we have done is unnecessary and subject to great risks. +DUMMY/LJ045-0063.wav|Oswald apparently wished to continue practicing his own Russian with her. Lieutenant Martello of the New Orleans police testified +DUMMY/LJ005-0087.wav|Female prisoners were in all cases to be under the charge of female officers. +DUMMY/LJ045-0200.wav|that over the weekend he did think about his wife's request that he not come to Irving, which was prompted by the birthday party being held at the Paine home. +DUMMY/LJ037-0089.wav|She identified Oswald, who was the Number two man in the lineup, as the man she saw running with the gun: +DUMMY/LJ011-0091.wav|In one corner after much search a phial was found labeled "Prussic acid," +DUMMY/LJ050-0131.wav|unless a system is established for the frequent formal review of activities thereunder. In this regard +DUMMY/LJ050-0204.wav|Secret Service Personnel and Facilities +DUMMY/LJ038-0159.wav|Jarman did talk to Oswald that morning: Quote, +DUMMY/LJ015-0016.wav|the failure and subsequent sentence of the bankers Messrs. Strahan, Paul, and Bates, +DUMMY/LJ006-0266.wav|On these occasions precautions were supposed to be taken to exclude bad characters, +DUMMY/LJ002-0138.wav|and four common councilmen to hold Courts of Requests, or Courts of Conscience, to hear and determine all causes of debt +DUMMY/LJ035-0064.wav|Truly thought that the officer's gun at that time appeared to be almost touching the middle portion of Oswald's body. +DUMMY/LJ009-0040.wav|whose desk and pulpit were just in front of the condemned pew, and within a couple of yards of it. +DUMMY/LJ041-0122.wav|and he must have known that President Kennedy had had nothing to do with it. +DUMMY/LJ003-0220.wav|quote, so discreditable to the metropolis, end quote. +DUMMY/LJ038-0147.wav|Frazier testified before the Commission that Oswald carried no lunch sack that day. +DUMMY/LJ004-0027.wav|it was subsequently amplified, and the rates of salaries fixed. +DUMMY/LJ028-0172.wav|The ascent to the top is on the outside, by a path which winds round all the towers. +DUMMY/LJ050-0104.wav|Each of them was a failure in his work and in his relations with others, a victim of delusions and fancies which led to the conviction +DUMMY/LJ050-0102.wav|John Schrank, and Guiseppe Zangara -- four assassins or would-be assassins +DUMMY/LJ046-0024.wav|to the Bureau of the Budget for review and approval. +DUMMY/LJ028-0269.wav|One even shouted to them and said, "Why sit ye there, Persians? Why do ye not go back to your homes? Till mules foal ye will not take our city!" +DUMMY/LJ045-0233.wav|He consistently refused to admit involvement in the assassination or in the killing of Patrolman Tippit. +DUMMY/LJ011-0036.wav|but the result in both cases was unfavorable. +DUMMY/LJ008-0104.wav|Here's the identical rope at sixpence an inch. +DUMMY/LJ033-0076.wav|On the morning of November twenty-two, nineteen sixty-three, +DUMMY/LJ041-0155.wav|He always wanted to play the red pieces in chess because, as he said in an apparently humorous context, +DUMMY/LJ017-0269.wav|let down. A full view of them was thus at all times obtainable by the officers who, without intermission, day and night patrolled the ward. +DUMMY/LJ025-0037.wav|while in plants nitrogen is only accidentally joined with the three other fundamental constituents of organic beings +DUMMY/LJ024-0120.wav|And remember one thing more. +DUMMY/LJ005-0261.wav|As a means of securing this uniformity, +DUMMY/LJ038-0251.wav|but investigation revealed that it was taken from a point slightly less than half a mile from General Walker's house. +DUMMY/LJ002-0217.wav|the warden of the Fleet at the commencement of the eighteenth century, are too well known to need more than a passing reference. +DUMMY/LJ044-0235.wav|If there was no conspiracy which would help him escape, the possibility of which has been considered in chapter six, +DUMMY/LJ002-0061.wav|or they lodged with the gatesmen, the prisoner officers who had charge of the inner gates. +DUMMY/LJ035-0075.wav|A test was also conducted to determine the time required to walk from the southeast corner of the sixth floor to the second-floor lunchroom by stairway. +DUMMY/LJ050-0199.wav|The Commission strongly encourages these efforts to improve protection along a motorcade route. +DUMMY/LJ036-0173.wav|but in either event he would have been in the roominghouse at about one p.m. +DUMMY/LJ012-0234.wav|It was, however, pretty clear that this victim, like preceding ones, had been lured to Nova Scotia Gardens, and there drugged with a large dose of laudanum. +DUMMY/LJ013-0156.wav|a scion of the ducal house of Bedford, by his confidential valet and personal attendant. +DUMMY/LJ022-0091.wav|This Allotment Division will consider all of the recommendations submitted to it +DUMMY/LJ009-0218.wav|But there was an attempt to revive it at that date, when the act for dispensing with the dissection of criminals was passed. +DUMMY/LJ041-0017.wav|Voebel also recalled that Oswald once outlined a plan to cut the glass in the window of a store on Rampart Street and steal a pistol, +DUMMY/LJ046-0023.wav|The Secretary of the Treasury submitted this planning document on August thirty-one, nineteen sixty-four, +DUMMY/LJ040-0189.wav|On the Monroe Silent Reading Test, Lee's score indicated no retardation in reading speed and comprehension; +DUMMY/LJ036-0121.wav|a twelve:fifteen p.m. pickup at Continental to Greyhound, unloaded at twelve:thirty p.m., +DUMMY/LJ046-0049.wav|More often than not, Presidential journeys have served more than one purpose at the same time: ceremonial, +DUMMY/LJ003-0192.wav|Hence there was a terrible accumulation of prisoners in the condemned cells. +DUMMY/LJ035-0069.wav|Baker placed himself on a motorcycle about two hundred feet from the corner of Elm and Houston Streets where he said he heard the shots. +DUMMY/LJ017-0005.wav|of some of the most heinous offenses of modern times. +DUMMY/LJ038-0103.wav|On the afternoon of November twenty-three, Officers H. M. Moore, +DUMMY/LJ006-0129.wav|He was on the most intimate and improperly familiar terms with the turnkeys, +DUMMY/LJ009-0140.wav|no inconvenience was suffered by the congregation, among whom were Lord Adolphus Fitz Clarence, Lord Coventry, +DUMMY/LJ023-0072.wav|Congress passed a statute which, in eighteen oh three, the Court said violated an express provision of the Constitution. +DUMMY/LJ021-0173.wav|They would have you believe that England has made progress out of her depression by a do-nothing policy, by letting nature take her course. +DUMMY/LJ039-0021.wav|I called him into the bathroom and I closed the door and I wanted to prevent him and then I started to cry. +DUMMY/LJ016-0444.wav|They were still much swollen when he was led out to execution. +DUMMY/LJ045-0120.wav|Furthermore, even though it appears that at times Oswald was really upset by visits of the FBI, +DUMMY/LJ033-0198.wav|Despite these matches, however, Stombaugh was unable to render on opinion that the fibers which he found in the bag had probably come from the blanket, +DUMMY/LJ035-0066.wav|In an effort to determine whether Oswald could have descended to the lunchroom +DUMMY/LJ048-0181.wav|The danger from a concealed sniper on the Dallas trip was of concern to those who had considered the problem. +DUMMY/LJ041-0079.wav|Nelson Delgado, one of Oswald's fellow Marines, testified that Oswald tried to, quote, cut up anybody that was high ranking, end quote, +DUMMY/LJ019-0287.wav|They were very numerous, very imperfect in construction and management, and they were very little required. +DUMMY/LJ038-0107.wav|According to Fritz, Oswald sneered, saying that they were fake photographs, that he had been photographed a number of times the day before by the police, +DUMMY/LJ017-0270.wav|On the morning of execution the noise of fixing the gallows in the street outside awoke one or two of them. +DUMMY/LJ012-0007.wav|Edwards, whom Mr. Gee easily identified with the others, at once admitted that he was the prime mover of the conspiracy. +DUMMY/LJ024-0004.wav|It is simply this: whenever a judge or justice of any federal court +DUMMY/LJ019-0054.wav|But about eighteen fifty the two sides were distinctly hostile, and the controversy ran high. +DUMMY/LJ049-0230.wav|can and should be recommended by this Commission in the interest of the more efficient protection of the President. +DUMMY/LJ047-0229.wav|Even if he had recalled that Oswald's place of employment was on the President's route, +DUMMY/LJ008-0291.wav|The time thus spent varied considerably, +DUMMY/LJ029-0026.wav|it was agreed that the planning of events in Texas would be left largely to the Governor. +DUMMY/LJ008-0157.wav|By eight o'clock not an inch of ground in front of the platform was unoccupied. +DUMMY/LJ030-0228.wav|picked up and cocked an automatic rifle as he heard the last shot. +DUMMY/LJ040-0138.wav|On the other hand, Lee told his probation officer, John Carro, that, quote, +DUMMY/LJ029-0164.wav|Also on November nineteen, the Morning News reported that the President's motorcade would travel from Love Field along specified streets, then +DUMMY/LJ015-0219.wav|At last he decided to enlist one Tester, a clerk in the traffic department, whom he thought would prove a likely tool. +DUMMY/LJ044-0150.wav|and that it would be better to have less and not have to be concerned about tomorrow. +DUMMY/LJ001-0045.wav|fourteen sixty-nine, fourteen seventy; +DUMMY/LJ019-0033.wav|Some jurisdictions, greatly to their credit, strove at once to follow the lead of the central authority. +DUMMY/LJ016-0041.wav|and working with his hands behind him, while he used his bare feet like claws upon the other side of the wall angle. +DUMMY/LJ012-0082.wav|Jordan and Sullivan, who at once set to work in a business-like way to obtain impressions of the keys of the strong room and chest. +DUMMY/LJ045-0164.wav|He said that he was tired of living alone and perhaps the reason for my being so angry was the fact that we were not living together. +DUMMY/LJ017-0079.wav|Meanwhile the bill discounters who held the forged acceptances, with other promissory notes, began to clamor for payment, and talk of issuing writs. +DUMMY/LJ028-0321.wav|Verily he by whom I have been so mutilated shall not escape unpunished. And truly all the paths of his counsels are known to me. +DUMMY/LJ045-0008.wav|in the treatment of Ambassador Adlai Stevenson late in October of nineteen sixty-three +DUMMY/LJ034-0046.wav|When the FBI in Washington received the cartons, the remaining prints, including Oswald's on the Rolling Readers carton, +DUMMY/LJ002-0037.wav|leading to thirteen wards on various floors, and a day room. +DUMMY/LJ005-0139.wav|making an average of nineteen persons occupying each room. +DUMMY/LJ036-0026.wav|and he estimated that it took him three to four minutes to drive three blocks west from the checkpoint to Field Street, +DUMMY/LJ010-0116.wav|on the one inveigling silly enthusiasts to join in the plot, and denouncing them on the other. +DUMMY/LJ020-0086.wav|Hot rolls and muffins should never be cut. +DUMMY/LJ028-0078.wav|About the head was an inscription in Greek characters saying that the face was that of Nebuchadnezzar. +DUMMY/LJ013-0074.wav|A third similar fraud to the amount of two thousand pounds was also brought to light. +DUMMY/LJ050-0040.wav|and reports from other agencies which independently evaluate their information for potential sources of danger. +DUMMY/LJ048-0059.wav|that several matters brought to his attention in late October and early November, including the visit to the Soviet Embassy in Mexico City, +DUMMY/LJ038-0167.wav|narrowly escaped death when a rifle bullet fired from outside his home passed near his head as he was seated at his desk. +DUMMY/LJ018-0202.wav|But the police nearly paid the penalty of capture with their lives. +DUMMY/LJ016-0213.wav|The origin of this expression dates, it is said, from the time when the Scottish mark, +DUMMY/LJ016-0201.wav|When he had taken the oath he was dismissed with the words, "Get thee hence, wretch!" +DUMMY/LJ001-0117.wav|which title can only be claimed by artistic practice, whether the art in it be conscious or unconscious. +DUMMY/LJ046-0190.wav|it had arrangements to be notified about release from confinement in roughly one thousand cases; +DUMMY/LJ022-0199.wav|Fear is vanishing and confidence is growing on every side, +DUMMY/LJ013-0203.wav|On his removal to Newgate after sentence, +DUMMY/LJ009-0083.wav|while the eyes of the latter are wandering round the chapel, and one of them is occasionally winked impudently at some acquaintance amongst the prisoners for trial. +DUMMY/LJ028-0404.wav|The courtyards and streets are paved with them, and as you walk about the city the name of Nebuchadnezzar everywhere meets your eye. +DUMMY/LJ030-0181.wav|Turning to his right in the direction of the noise, Kellerman heard the President say +DUMMY/LJ032-0072.wav|When arrested on the day of the assassination, he had in his possession a Smith and Wesson thirty-eight caliber revolver +DUMMY/LJ017-0120.wav|He had already been arrested on a writ issued at the instance of the money-lenders, and an action had been commenced against Mrs. Palmer on her acceptances. +DUMMY/LJ022-0046.wav|It is true that while business and industry are definitely better our relief rolls are still too large. +DUMMY/LJ042-0070.wav|Oswald was discovered in time to thwart his attempt at suicide. +DUMMY/LJ006-0204.wav|In the same way the wardsman laid in his stock to be retailed. Other light literature besides the daily journals were in circulation: +DUMMY/LJ042-0039.wav|economic system and plans for war, end quote. +DUMMY/LJ028-0152.wav|In the circuit of the wall are a hundred gates, all of brass, with brazen lintels and sideposts. +DUMMY/LJ021-0092.wav|first, the legislative or policy making function; +DUMMY/LJ042-0108.wav|Although Marina Oswald told the Commission that her husband had good personal relationships in the Soviet Union, Katherine Ford, +DUMMY/LJ038-0203.wav|He had quit these classes at least a week before the shooting, which occurred on a Wednesday night. +DUMMY/LJ008-0015.wav|was executed on a gibbet in front of his victim's house. +DUMMY/LJ018-0362.wav|The trial of Miss Edmunds was transferred to the Central Criminal Court under Lord Campbell's Act, already referred to. +DUMMY/LJ024-0110.wav|And the strategy of that last stand is to suggest the time-consuming process of amendment in order to kill off by delay +DUMMY/LJ044-0042.wav|In his letter to V. T. Lee, +DUMMY/LJ018-0195.wav|His regular trade, followed before he took to the life of resurrectionist, was that of an engraver. +DUMMY/LJ040-0111.wav|with no behavior or truancy problems. +DUMMY/LJ042-0207.wav|with the mass extermination so typical of Stalin, and the individual suppression and regimentation under Khrushchev. +DUMMY/LJ036-0047.wav|On November twenty-two, Mrs. Bledsoe came downtown to watch the Presidential motorcade. +DUMMY/LJ008-0065.wav|They are both drawn at once by a windlass, and the unhappy culprits remain suspended. +DUMMY/LJ030-0007.wav|They were greeted by Vice President Johnson and Governor Connally, who joined the Presidential party in a motorcade through San Antonio. +DUMMY/LJ033-0213.wav|(three) removed the rifle from the blanket in the Paines' garage on Thursday evening; +DUMMY/LJ019-0091.wav|were mere matters of chance, and decided according to the views of the local magistracy. +DUMMY/LJ022-0079.wav|Projects shall be of a nature that a considerable proportion of the money spent will go into wages for labor. +DUMMY/LJ049-0170.wav|indicating the possibility of an attempt against the person or safety of the President, end quote, or other protected persons. +DUMMY/LJ027-0029.wav|For illustration: a change in the locomotive habit of a bird from that of flying to that of an ostrich +DUMMY/LJ043-0156.wav|After he brought the rifle home, then, he showed you the book? +DUMMY/LJ014-0250.wav|after they had been cashed by the bank. +DUMMY/LJ043-0099.wav|It appears that his discharge was hastened by the fact that he brought a Russian language newspaper to work. +DUMMY/LJ040-0023.wav|may be obtained from a study of the events, relationships and influences which appear to have been significant in shaping his character and in guiding him. +DUMMY/LJ010-0111.wav|and there seems to have been good reason for supposing that he was a greater villain than any of those arraigned. +DUMMY/LJ025-0070.wav|in nowise distinguishable from that of one of the simpler animals, and +DUMMY/LJ033-0109.wav|the Commission has carefully considered the testimony of these two witnesses with regard to the length of the bag. +DUMMY/LJ042-0149.wav|but I had to change my character a great deal in order to maintain a more or less peaceful family life. End quote. +DUMMY/LJ016-0172.wav|There were still untoward accidents occasionally at executions, and even the chief practitioner of recent times, Calcraft, +DUMMY/LJ049-0139.wav|and the Attorney General at any meetings which are concerned with Presidential protection. +DUMMY/LJ042-0200.wav|even though he later referred to the latter as, quote, trusted long time fighters for progress, end quote. +DUMMY/LJ037-0093.wav|William Arthur Smith was about a block east of tenth and Patton when he heard shots. +DUMMY/LJ029-0050.wav|A check of the geographic indexes there revealed no listing for any individual deemed to be a potential danger to the President +DUMMY/LJ023-0027.wav|It will take time -- and plenty of time -- to work out our remedies administratively even after legislation is passed. +DUMMY/LJ006-0068.wav|We have reason to fear that poverty, ragged clothes, and an inability to pay the ward dues, elsewhere exacted for better accommodation, +DUMMY/LJ029-0144.wav|In conformity with these arrangements, traffic proceeding west on Main is directed to turn right at Houston +DUMMY/LJ035-0011.wav|a strong wind blowing from the north almost unseated him. +DUMMY/LJ007-0122.wav|Even Mr. Cope admitted that the committal of this class of prisoners to Newgate was most inconvenient, +DUMMY/LJ019-0334.wav|Under them the governor or jailer was held strictly responsible. +DUMMY/LJ019-0312.wav|with the local jurisdictions, although still very leniently disposed. +DUMMY/LJ028-0465.wav|served as an elevated road where several chariots might have been driven abreast. +DUMMY/LJ027-0002.wav|Biology. Chapter ten. Morphology and Embryology, Part One. +DUMMY/LJ019-0163.wav|Thus the wards, which, as I have said, were left in complete darkness, were now to be lighted with gas; and after this most salutary addition, +DUMMY/LJ027-0168.wav|Now this is the permanent, mature condition of many amphibians, +DUMMY/LJ012-0089.wav|he broke open the chest and stole four thousand, seven hundred pounds in notes, with a quantity of gold and some silver. +DUMMY/LJ045-0163.wav|On that day, he suggested that we rent an apartment in Dallas. +DUMMY/LJ020-0015.wav|Beat hard one minute before pouring in the yeast. +DUMMY/LJ031-0204.wav|with three Secret Service agents, accompanied President Kennedy's body on the forty-five-minute automobile trip from Andrews Air Force Base to the hospital. +DUMMY/LJ037-0240.wav|At one:twenty-four p.m., the police radio reported, quote, +DUMMY/LJ050-0003.wav|By The President's Commission on the Assassination of President Kennedy. Chapter eight. The Protection of the President. Part five. +DUMMY/LJ003-0343.wav|With the seeming impossibility of extending the limits of the prison as it then stood, +DUMMY/LJ042-0202.wav|It has turned itself into the traditional lever of a foreign power to overthrow the government of the United States; +DUMMY/LJ001-0040.wav|and is very simple and legible, and unaffectedly designed for use; but it is by no means without beauty. +DUMMY/LJ004-0094.wav|This act set forth that "whereas the malignant fever commonly called the jail distemper +DUMMY/LJ018-0057.wav|He was short in stature, his legs were light for the upper part of his body, which was powerful, almost herculean. +DUMMY/LJ018-0049.wav|Müller knew that any confession would ruin his chances of escape. +DUMMY/LJ006-0224.wav|New arrivals, especially the innocent and still guileless debutant, were tormented with rude horse-play, and assailed by the most insulting "chaff." +DUMMY/LJ041-0141.wav|Oswald could have shot the Governor as the car approached the Depository or as it was making the turn onto Elm Street. +DUMMY/LJ040-0152.wav|turning around the topics of omnipotence and power, through which he tries to compensate for his present shortcomings and frustrations, end quote. +DUMMY/LJ044-0080.wav|was apparently not enough to satisfy him. He exaggerated in his letters to V. T. Lee in an apparent attempt +DUMMY/LJ048-0184.wav|Levels of risk can be determined, however, as has been confirmed by building surveys made since the assassination for the Department of the Treasury. +DUMMY/LJ008-0056.wav|The Dublin "engine of death," as the gallows are styled in the account from which the following description is taken, consisted of an iron bar +DUMMY/LJ017-0063.wav|He made no distinct admissions even on the scaffold; but when the chaplain at the last moment exhorted him to confess, +DUMMY/LJ041-0049.wav|According to Marguerite Oswald, quote, Lee lived for the time that he would become seventeen years old to join the Marines -- that whole year, end quote. +DUMMY/LJ043-0081.wav|The counselor testified that there was some indication that Oswald was capable of doing college work +DUMMY/LJ015-0188.wav|But his name lingers still in this country as that of the first swindler of his time, +DUMMY/LJ007-0123.wav|and the inspectors expressed themselves still more strongly in reprehension of the practice. +DUMMY/LJ010-0133.wav|and catching sight of the coffins ranged around the gallows, he smiled at the show with contemptuous indifference. +DUMMY/LJ035-0103.wav|There was no elevator on the third or fourth floor. +DUMMY/LJ018-0241.wav|forthwith brought an ejectment on the possessor of lands purchased on the authority of the forged conveyance and will. +DUMMY/LJ033-0096.wav|quote, when he rode with me, I say he always brought lunch except that one day on November twenty-two he didn't bring his lunch that day, end quote, +DUMMY/LJ045-0027.wav|During the period from Oswald's return from Mexico to the assassination, +DUMMY/LJ040-0216.wav|In view of his experiences when he visited his relatives in New Orleans in the spring of nineteen fifty, and his other solitary habits, +DUMMY/LJ040-0125.wav|Truancy charges were brought against him alleging that he was, quote, beyond the control of his mother insofar as school attendance is concerned, end quote. +DUMMY/LJ014-0324.wav|Cole's plan was to have a wharf of his own, nominally occupied by a creature trading as Maltby and Co. +DUMMY/LJ049-0185.wav|During the period the Commission was giving thought to this situation, +DUMMY/LJ003-0121.wav|But all punishments might readily be commuted into a fine to be spent in gin for judge and jury. +DUMMY/LJ037-0043.wav|She raised her hands to her eyes as the man started to walk back toward Patton. +DUMMY/LJ008-0186.wav|The concourse was very great, notwithstanding these warnings. +DUMMY/LJ043-0078.wav|because of his need for immediate employment she attempted to obtain for him any job that was available at the time. Oswald made qualifying marks +DUMMY/LJ037-0019.wav|The next day Scoggins viewed a lineup of four persons and identified Oswald as the man whom he had seen the day before at tenth and Patton. +DUMMY/LJ013-0229.wav|Courvoisier admitted that he had intended to bind it tightly round his arm and bleed himself to death in the night. +DUMMY/LJ018-0341.wav|But it was scarcely worse than that of Mrs. Brownrigg, or that of the Meteyards, both of whom did their helpless apprentices to death. +DUMMY/LJ004-0171.wav|was by a kind of chimney, which the prisoners kept hermetically sealed, and which had never been opened in the memory of the turnkey. +DUMMY/LJ009-0282.wav|who for the fourth time regained his foothold. Again he was repelled, +DUMMY/LJ039-0203.wav|The shots fired by Frazier at the range of twenty-five yards landed within an area of two inches and five inches respectively. +DUMMY/LJ008-0199.wav|At the George public-house to the south of the drop, Sir W. Watkin Wynn, Baronet, +DUMMY/LJ020-0088.wav|It is not a wholesome practice for any woman -- least of all a young girl to be out of bed two hours before she eats her breakfast. +DUMMY/LJ035-0176.wav|Barnett estimated that approximately three minutes elapsed between the time he heard the last of the shots and the time he started guarding the front door. +DUMMY/LJ006-0020.wav|William Crawford had been one of the promoters and managers of the Philanthropic Society's farm school. +DUMMY/LJ016-0390.wav|and was arrested on board ship just as he was about to leave the country, had a little spare cash, which he devoted entirely to the purchase of extra food. +DUMMY/LJ037-0135.wav|Russell confirmed this statement in a sworn affidavit for the Commission. +DUMMY/LJ017-0132.wav|He suffered at Stafford in front of the jail. +DUMMY/LJ019-0167.wav|wore no distinguishing costume, although its use was admitted, not only for cleanliness, but as a badge of condition, and a security against escape. +DUMMY/LJ032-0007.wav|In this chapter the Commission evaluates the evidence upon which it has based its conclusion concerning the identity of the assassin. +DUMMY/LJ025-0046.wav|The relations of plants and animals to the atmosphere are therefore inverse. +DUMMY/LJ044-0075.wav|of the alleged FPCC, quote, organization, end quote, in New Orleans. +DUMMY/LJ028-0132.wav|Yet as large and lofty as they were, they were completed in fifteen days. +DUMMY/LJ034-0137.wav|The two men, Harold Norman and James Jarman, Jr., each confirmed that when they came out of the building, +DUMMY/LJ048-0227.wav|According to their affidavits, the drinking in no case amounted to more than three glasses of beer or one and a half mixed drinks, +DUMMY/LJ028-0062.wav|and drove them from Asia. +DUMMY/LJ008-0275.wav|but there was a further appeal then, as now, to the king himself, or practically to the Home Secretary. +DUMMY/LJ050-0129.wav|and this source should not be neglected by undue concentration on relationships with other Federal agencies. +DUMMY/LJ012-0055.wav|While there he was joined by his wife and other friends. +DUMMY/LJ045-0054.wav|It has been the cause of many of our arguments. End quote. Oswald claimed that his wife preferred others to him. +DUMMY/LJ045-0083.wav|On that issue George De Mohrenschildt, who was probably as close to the Oswalds as anyone else during their first stay in Dallas, +DUMMY/LJ033-0031.wav|had curtains and curtain rods, and that Oswald had never discussed the subject with her. +DUMMY/LJ033-0136.wav|A handmade bag of wrapping paper and tape was found in the southeast corner of the sixth floor alongside the window from which the shots were fired. +DUMMY/LJ022-0067.wav|but it will, by sustaining purchasing power, cushion the shock of economic distress. +DUMMY/LJ042-0093.wav|They sent him to Minsk to work in a radio and television factory as a metal worker. +DUMMY/LJ001-0060.wav|It must be understood that through the whole of the fifteenth and the first quarter of the sixteenth centuries +DUMMY/LJ023-0086.wav|to protect our national resources, and in many other ways, to serve our clearly national needs, +DUMMY/LJ017-0091.wav|The last act now approached, and in order to avoid the detection of this last fraud, Palmer laid his plans for disposing of Cook. +DUMMY/LJ019-0350.wav|The funds necessary would be advanced by the Public Works Loan Commissioners, and the interest might be charged against the county or borough rates. +DUMMY/LJ006-0118.wav|He made collections when it suited him for ward purposes, to be spent as he chose, in candles and so forth. +DUMMY/LJ001-0128.wav|it is not a real gain, for the modern printer throws the gain away by putting inordinately wide spaces between his lines, which, probably, +DUMMY/LJ028-0139.wav|yet he at least tried to be accurate. He says: +DUMMY/LJ046-0219.wav|The above action should be taken without delay in order to attempt to verify the information and no evaluation of the information should be attempted. +DUMMY/LJ006-0053.wav|and the whole by proper management might have been so accommodated as to prevent overcrowding. +DUMMY/LJ013-0184.wav|He had hung over the body in a state of dreadful agitation, answering no questions, and taking no part in the proceedings. +DUMMY/LJ018-0033.wav|Directly the foregoing facts were established, a couple of detective officers, armed with a warrant to arrest Müller, +DUMMY/LJ037-0078.wav|when they heard the sound of gunfire and the screams of Helen Markham. +DUMMY/LJ032-0211.wav|The authenticity of these pictures has been established by expert testimony which links the second picture, +DUMMY/LJ029-0058.wav|On November twenty-two a Secret Service agent stood at the entrance to the Trade Mart, where the President was scheduled to speak, with copies of these photographs. +DUMMY/LJ026-0028.wav|it is futile to argue about the exact period when, for instance, the egg becomes a tadpole or the tadpole a frog, +DUMMY/LJ007-0242.wav|No complete and permanent improvement was indeed possible while Newgate remained unchanged. +DUMMY/LJ018-0394.wav|by swindling the Artisans' Dwellings Company; and Madame Rachel passed through Newgate on her way to Millbank +DUMMY/LJ013-0129.wav|and one day a police constable tracked him to a silversmith's in Holborn Hill, +DUMMY/LJ037-0091.wav|Barbara Jeanette Davis and Virginia Davis were sitting alongside each other when they made their positive identifications of Oswald. +DUMMY/LJ005-0154.wav|The rations varied considerably everywhere. +DUMMY/LJ028-0072.wav|During all his reign there was little peace in his great mixed turbulent empire. +DUMMY/LJ030-0154.wav|Other witnesses in the motorcade estimated the speed of the President's limousine from seven to twenty-two miles per hour. +DUMMY/LJ030-0031.wav|Secret Service agents formed a cordon to keep the press and photographers from impeding their passage and scanned the crowd for threatening movements. +DUMMY/LJ028-0223.wav|I gave heed to the needs of Babylon and its cities, and the servitude of the Babylonians, whatever was oppressive, I removed from them. +DUMMY/LJ013-0246.wav|Some time elapsed before the imprisoned party could force open the doors, and by then the fugitive had escaped. +DUMMY/LJ044-0053.wav|in which he supported his report on the Bringuier incident with a copy of the charges made against him +DUMMY/LJ006-0072.wav|I shall have occasion to speak again, and more at length, of Mr. Cope's careless and perfunctory discharge +DUMMY/LJ028-0358.wav|Cyrus had permitted them to stand, and as long as he made Babylon his home, the city was as strongly protected as ever. +DUMMY/LJ050-0136.wav|the Committee will include representatives of the President's Office of Science and Technology, Department of Defense, CIA, +DUMMY/LJ014-0331.wav|Once a warrant-holder sent down a clerk to view certain goods, and the clerk found that these goods had already a "stop" upon them, or were pledged. +DUMMY/LJ011-0019.wav|when it was found that a sum of ten thousand pounds, standing in the name of three trustees, of whom Fauntleroy was one, +DUMMY/LJ045-0188.wav|While there is no way to tell when he first began to think specifically of assassinating the President +DUMMY/LJ019-0274.wav|It is interesting to note here +DUMMY/LJ031-0043.wav|evidenced no palpable pulse; and had a few chest sounds which were thought to be heartbeats. +DUMMY/LJ042-0021.wav|shortly after he arrived in the Soviet Union, and probably before Soviet authorities had given him permission to stay indefinitely, +DUMMY/LJ006-0259.wav|Many other articles were introduced by visitors, including money, tobacco, pipes, and snuff. +DUMMY/LJ049-0216.wav|the FBI, the CIA, and the military intelligence agencies as well as the Secret Service. +DUMMY/LJ027-0023.wav|that the only reasonable explanation for the existence of a fundamental unity in organic life +DUMMY/LJ029-0027.wav|At the White House, Kenneth O'Donnell, special assistant to the President, acted as coordinator for the trip. +DUMMY/LJ028-0442.wav|At a distance of about two miles to the south of Babel is the larger and lower mound called the Kasr, or the Fortress, +DUMMY/LJ022-0181.wav|but that its resources be most fully utilized in the economic life of the country +DUMMY/LJ035-0060.wav|Baker stated later that the man did not seem to be out of breath; he seemed calm. Quote, He never did say a word or nothing. +DUMMY/LJ037-0186.wav|disclosed that on January three, nineteen sixty-three, +DUMMY/LJ039-0195.wav|were able to fire the rounds within the time period which would have been available to the assassin under those conditions. +DUMMY/LJ034-0031.wav|The box on the floor, behind the three near the window, +DUMMY/LJ017-0273.wav|Lopez was more talkative. +DUMMY/LJ039-0222.wav|was a substantial aid to rapid, accurate firing. +DUMMY/LJ001-0091.wav|so far as fine printing is concerned, though paper did not get to its worst till about eighteen forty. +DUMMY/LJ014-0210.wav|Cannon, by trade a chimney-sweep, had long been characterized by the bitterest hatred of the police force, +DUMMY/LJ015-0038.wav|They chose, unhappily for themselves, the latter alternative. +DUMMY/LJ011-0204.wav|As he still urged his rights over his wife, Miss Turner cried out in protest, "No, no, I am not his wife; +DUMMY/LJ032-0039.wav|and opposite the printed word "From" +DUMMY/LJ012-0092.wav|This occurred in November eighteen thirty-four. The Custom House officials were in a state of consternation, +DUMMY/LJ019-0285.wav|In one jail fifteen hours were spent in bed out of the twenty-four. This was in keeping with other grave defects and omissions. +DUMMY/LJ005-0147.wav|At the time of which I am writing there was great diversity of practice as regards the hours of labor. +DUMMY/LJ030-0094.wav|Driven by an officer of the Dallas Police Department, +DUMMY/LJ017-0080.wav|Palmer, alive to the danger he ran of a prosecution for forgery, should the fraud he had committed be brought to light, +DUMMY/LJ020-0024.wav|knead and set for risings first and second, as you have already been instructed. This sponge will be found especially useful in making Graham Bread. +DUMMY/LJ043-0179.wav|The circumstances of the attack on Walker coupled with other indications that Oswald was concerned about his place in history +DUMMY/LJ038-0065.wav|It is unlikely that any of the police officers referred to Oswald as a suspect in the assassination. +DUMMY/LJ022-0025.wav|but that it is being done in spite of the few who seek to confuse them and to profit by their confusion. +DUMMY/LJ021-0027.wav|which obtained through the attrition of individuals before the new conditions arose. +DUMMY/LJ046-0243.wav|if the presence in Dallas of the Stevenson pickets might have created a danger for the President on a visit to that city, +DUMMY/LJ007-0005.wav|The inspectors paid tribute to the excellence of the motives of these philanthropic ladies, and recognized the good they did. +DUMMY/LJ017-0148.wav|Smethurst's victim was a Miss Bankes, with whom he had contracted a bigamous marriage. +DUMMY/LJ002-0093.wav|There were nine wards in all on the female side, one of them in the attic, +DUMMY/LJ019-0183.wav|provided it was henceforth used only for untried prisoners, suggested that Newgate should be entirely reconstructed, and the new building adopted as a model. +DUMMY/LJ018-0164.wav|who had devoted all his talent and all his energies for some seventeen years to the fabrication of false bank-notes. +DUMMY/LJ049-0126.wav|Committee of Cabinet Officers. As our Government has become more complex, +DUMMY/LJ029-0162.wav|The motorcade will then pass through downtown on Harwood +DUMMY/LJ016-0266.wav|No notion that they were assisting at a vindication of the law filled the minds of those present with dread. +DUMMY/LJ045-0150.wav|On Monday, November eighteen, nineteen sixty-three, he called several times, but after I hung up on him and didn't want to talk to him he did not call again. +DUMMY/LJ002-0052.wav|three. The chapel yard was about forty-three feet by twenty-five. +DUMMY/LJ010-0224.wav|No serious importance could be attached to these, the manifest inventions of a disordered intellect. +DUMMY/LJ025-0135.wav|is propagated to the muscles by which the body is retracted, and causing them to contract, the act of retraction is brought about. +DUMMY/LJ012-0011.wav|which Mrs. Canning had lost by remarriage. +DUMMY/LJ024-0042.wav|who understand those modern conditions, +DUMMY/LJ021-0196.wav|There is great danger it seems to me to arise from the constant habit which prevails where anything is opposed or objected to, +DUMMY/LJ014-0093.wav|and during that time he must frequently have stood or sat over his own grave. +DUMMY/LJ007-0018.wav|and that when the restraining influences of the ladies were absent, the female prisoners relapsed into immoral and uncleanly discourse. +DUMMY/LJ019-0007.wav|By other acts local authorities were empowered to construct new jails or hire accommodation in the district; +DUMMY/LJ034-0087.wav|Kaiser identified it as the clipboard which Oswald had appropriated from him when Oswald came to work at the Depository. +DUMMY/LJ009-0063.wav|The sheriffs shudder, their inquisitive friends crane forward, the keeper frowns on the excited congregation, +DUMMY/LJ040-0182.wav|He appears to be a somewhat insecure youngster exhibiting much inclination for warm and satisfying relationships to others. +DUMMY/LJ029-0210.wav|end quote, followed by a scurrilous bill of particulars that constituted a vilification of the President. +DUMMY/LJ047-0191.wav|The official Bureau files confirm Hosty's statement that from November five until the assassination, no active investigation was conducted. +DUMMY/LJ020-0035.wav|make into two loaves, then put into well-greased pans and leave for an hour, or until it becomes more than twice the original size of the dough. +DUMMY/LJ039-0213.wav|So that if you aimed with this weapon as it actually was received at the laboratory, it would not be necessary to take any lead whatsoever +DUMMY/LJ003-0171.wav|But the keeper protested that no single prisoner could thus monopolize space if the state side was crowded. +DUMMY/LJ008-0268.wav|but of these three hundred and ninety-six were reversed by the king in council, and only fifty-two were really executed. +DUMMY/LJ034-0165.wav|Lee Harvey Oswald. +DUMMY/LJ003-0186.wav|Only when the order of execution came down were those about to suffer placed apart in one of the rooms in the arcade of the middle ward. +DUMMY/LJ028-0242.wav|entered the stream, which had now sunk so as to reach about midway up a man's thigh, and thus got into the town. +DUMMY/LJ012-0006.wav|A watch was set on him at her house, where he was soon afterwards arrested. +DUMMY/LJ028-0329.wav|Then the Babylonians, seeing that his deeds were as brave as his words, were beyond measure pleased, and set no bounds to their trust. +DUMMY/LJ002-0223.wav|The principle of "chummage" prevailed as in the King's Bench, +DUMMY/LJ028-0160.wav|The houses are mostly three and four stories high; +DUMMY/LJ038-0169.wav|he saw two men, in separate cars, drive out of a church parking lot adjacent to Walker's home. A friend of Walker's testified that +DUMMY/LJ002-0050.wav|and the other eighteen by fifteen; and they nominally held twenty-two persons. +DUMMY/LJ028-0083.wav|Unfortunately, the little stone seal, perhaps the only one to preserve for us his features, appears to have been lost for ever. +DUMMY/LJ037-0003.wav|Eyewitnesses +DUMMY/LJ037-0261.wav|that the jacket belonged to Lee Harvey Oswald, and that when he was arrested at approximately one:fifty p.m., he was in shirt sleeves. +DUMMY/LJ033-0214.wav|(four) carried the rifle into the Depository Building, concealed in the bag; +DUMMY/LJ011-0145.wav|The new Forgery Act with the Lords' amendment passed into law, but the latter proved perfectly harmless, +DUMMY/LJ028-0044.wav|He rebuilt Babylon that in accordance with the ancient custom he might be crowned in the sacred city. +DUMMY/LJ031-0023.wav|four anesthesiologists, Drs. Marion T. Jenkins, Adolph H. Giesecke, Jr., Jackie H. Hunt, +DUMMY/LJ018-0238.wav|and in eighteen sixty-two he was obliged to fly the country, hopelessly and irretrievably ruined. +DUMMY/LJ016-0410.wav|nodded pleasantly to the governor, who stood just opposite, and then walked briskly towards the execution shed, smiling as he went along. +DUMMY/LJ033-0158.wav|The palmprint was found on the closed end of the bag. +DUMMY/LJ018-0315.wav|to whom it was said one hundred pounds apiece had been given down as the price of their infidelity. +DUMMY/LJ041-0053.wav|continued during this period in New Orleans (nineteen fifty-four to nineteen fifty-six). +DUMMY/LJ014-0087.wav|A quantity of quicklime was thrown in with the body to destroy all identification. +DUMMY/LJ011-0095.wav|He had prospered in early life, was a slop-seller on a large scale at Bury St. Edmunds, and a sugar-baker in the metropolis. +DUMMY/LJ050-0057.wav|specifying criteria for information to be furnished to the Secret Service in addition to that covered by the former standard, +DUMMY/LJ029-0036.wav|Advance Preparations for the Dallas Trip +DUMMY/LJ037-0226.wav|According to Patrolman Poe this description came from Mrs. Markham and Mrs. Barbara Jeanette Davis. +DUMMY/LJ045-0042.wav|He stated that the other girl rejected him partly because he was an American, a fact that he said she had exploited. He stated that, quote, +DUMMY/LJ034-0065.wav|led the Commission to attach some probative value to the fingerprint and palmprint identifications in reaching the conclusion +DUMMY/LJ039-0025.wav|After further questioning +DUMMY/LJ039-0246.wav|(six) lied to the police after his arrest concerning important substantive matters, +DUMMY/LJ025-0109.wav|has been as completely invalidated as the third and second, and even the first can be retained only in a modified form +DUMMY/LJ028-0434.wav|the large high mound, which resembles a mountain from a distance, still bears the ancient name Babel. +DUMMY/LJ014-0105.wav|He was lying on his face, his legs tied up to his hips so as to allow of the body fitting into the hole. +DUMMY/LJ028-0073.wav|The walls of the palaces of many of the Assyrian kings were lined with great stone slabs engraved with reliefs and sometimes with the portrait of a king. +DUMMY/LJ036-0071.wav|In a reconstruction of this bus trip, agents of the Secret Service and the FBI walked the seven blocks from the front entrance of the Depository Building +DUMMY/LJ009-0056.wav|The last of the four is said to have been a clergyman of the Church of England, condemned for forgery, "a miserable old man in a tattered suit of black. +DUMMY/LJ001-0112.wav|instead of ending in the sharp and clear stroke of Jenson's letters; +DUMMY/LJ022-0165.wav|both in its public relations and in its internal relations. +DUMMY/LJ008-0272.wav|Judges on assize were satisfied with simply recording a sentence of death against offenders whom they did not think deserved the extreme penalty. +DUMMY/LJ013-0048.wav|The Rev. W. Bailey, LL.D., was convicted at the Central Criminal Court, in February eighteen forty-three, of forgery. +DUMMY/LJ008-0303.wav|The crowd of men and women who passed the night in front of Newgate, began, as soon as the hour was passed when they had expected the report, +DUMMY/LJ049-0066.wav|For instance, the lead car always is manned by Secret Service agents familiar with the area and with local law enforcement officials; +DUMMY/LJ004-0062.wav|and the reason is plain: you have taken him from his home, and have deprived him of the means of providing himself with the necessaries or comforts of life, +DUMMY/LJ036-0180.wav|Mrs. Roberts testified that on Thursday, November twenty-one, Oswald did not come home. +DUMMY/LJ046-0124.wav|or with undue persistence. +DUMMY/LJ016-0361.wav|the moment too that the condemned man had passed through the debtors' door on to the scaffold the prison had done with him, +DUMMY/LJ016-0348.wav|The same was realized in the first private execution within Newgate, +DUMMY/LJ031-0003.wav|Parkland Memorial Hospital. The Race to the Hospital +DUMMY/LJ002-0113.wav|in an age when insolvent acts and bankruptcy courts do so much to relieve the impecunious, +DUMMY/LJ048-0256.wav|as a result of the investigation he ordered, he was satisfied that each of the agents performed his duties in an entirely satisfactory manner +DUMMY/LJ010-0259.wav|a deformed lad among the crowd was seen to present a pistol at Her Majesty's carriage, +DUMMY/LJ005-0134.wav|in twenty-two county jails there were one thousand sixty-three sleeping cells in all (in eighteen twenty-three) +DUMMY/LJ012-0279.wav|His measures for the disposal of the corpus delicti remind us of those taken by Mrs. Hayes and her associates, +DUMMY/LJ005-0002.wav|Newgate prisoners were the victims to another most objectionable practice which obtained all over London. +DUMMY/LJ034-0017.wav|could look southwesterly down Elm Street over the top of the "Rolling Readers" cartons. +DUMMY/LJ049-0045.wav|It is clear that at the time of the shots in Dallas, Agent Clinton J. Hill leaped to the President's rescue as quickly as humanly possible. +DUMMY/LJ036-0166.wav|is three to four short blocks south of Lamar and Elm. If Oswald left the bus at twelve:forty-four p.m. +DUMMY/LJ031-0054.wav|At that point, direction of the President's treatment was undertaken by Dr. Malcolm O. Perry, who arrived at trauma room one a few moments after the President. +DUMMY/LJ022-0173.wav|for the regulation of transportation by water, for the strengthening of our Merchant Marine and Air Transport, +DUMMY/LJ030-0195.wav|Observing his blood-covered chest as he was pulled into his wife's lap, Governor Connally believed himself mortally wounded. +DUMMY/LJ021-0003.wav|Three months have passed since I talked with you shortly after the adjournment of the Congress. +DUMMY/LJ013-0194.wav|but on the second day the discovery of fresh evidence, more particularly the recovery of some of Lord William's stolen plate, +DUMMY/LJ044-0187.wav|a person like him (Oswald) in place of aiding the Cuban Revolution, was doing it harm, end quote. +DUMMY/LJ002-0159.wav|detained for one shilling nine pence, with costs of five shillings. +DUMMY/LJ043-0069.wav|the evidence indicates that Oswald usually told his prospective employers and employment counselors that he had recently been discharged from the Marine Corps. +DUMMY/LJ004-0178.wav|The bedclothes consisted only of a single "very slight" rug. +DUMMY/LJ007-0138.wav|who constantly breathed an air heavy, and, when the wards were first opened in the morning, particularly offensive. +DUMMY/LJ047-0023.wav|In April nineteen sixty, Mrs. Marguerite Oswald and Robert Oswald were interviewed +DUMMY/LJ016-0400.wav|Wainwright's demeanor was one of reckless effrontery steadily maintained to the last. +DUMMY/LJ040-0008.wav|a desire to effect changes in the structure of society or simply to go down in history as a well publicized assassin. +DUMMY/LJ007-0105.wav|counsel for the two M.P.'s, was also concerned. +DUMMY/LJ047-0220.wav|that in view of the President's visit to Dallas, that if anyone had any indication of any possibility of any acts of violence or any demonstrations against the President, +DUMMY/LJ042-0247.wav|In August of nineteen sixty-three, he gave the New Orleans police as a reason for refusing to permit his family to learn English, +DUMMY/LJ029-0132.wav|From Houston Street, which forms the base of the triangle, three streets -- Commerce, Main, and Elm -- +DUMMY/LJ002-0137.wav|So much inconvenience ensued, that in fifteen eighteen the Corporation obtained from Parliament an act empowering two aldermen +DUMMY/LJ046-0207.wav|in the training programs of other law enforcement agencies, and agents from other agencies attended the regular Secret Service training schools. +DUMMY/LJ031-0206.wav|Mrs. Kennedy was guarded by Secret Service agents in quarters assigned to her in the naval hospital. +DUMMY/LJ047-0151.wav|On the same day Hosty interviewed neighbors on Fifth Street and learned that the address was that of Mrs. Ruth Paine. +DUMMY/LJ011-0030.wav|was substantially true +DUMMY/LJ003-0134.wav|One day he was too ill to come down and meet her. +DUMMY/LJ016-0032.wav|Williams as a capital convict was lodged in the press-yard or condemned ward. +DUMMY/LJ015-0237.wav|One night Tester whispered to Agar and Pierce, "All right," as they cautiously lounged about London Bridge. +DUMMY/LJ034-0136.wav|When the three employees appeared before the Commission, Brennan identified the two whom he saw leave the building. +DUMMY/LJ011-0153.wav|Edward Gibbon Wakefield, the prime mover in the abduction, was a barrister, not exactly briefless, but without a large practice. +DUMMY/LJ023-0007.wav|I am reminded of that evening in March, four years ago, when I made my first radio report to you. +DUMMY/LJ014-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section seventeen: Newgate notorieties continued, part two. +DUMMY/LJ027-0137.wav|"It is an observable fact," says Romanes, +DUMMY/LJ027-0167.wav|Then the gills gradually dry up, as the lungs develop, and they now breathe wholly by lungs, but still retain the tail. +DUMMY/LJ009-0190.wav|whom the horrid scene, with the additional effect of some noted criminals hanging on the walls, drove out again sick and faint, +DUMMY/LJ005-0183.wav|All that was urged was that the borough magistracy had no right to govern their jails +DUMMY/LJ028-0368.wav|learned of his purpose, and when they were consulted as to the most favorable time for beginning the work upon the new city, +DUMMY/LJ028-0043.wav|Esarhaddon, the son of Sinacherib, was the next King of Nineveh. +DUMMY/LJ028-0150.wav|On the top, along the edges of the wall, they constructed buildings of a single chamber facing one another, +DUMMY/LJ002-0234.wav|In the yard behind the prison +DUMMY/LJ041-0099.wav|Powers believed that when Oswald arrived in Japan he acquired a girlfriend, quote, +DUMMY/LJ016-0015.wav|White, when taxed with his attempt, admitted that the accusation was true, and spoke "with pride and satisfaction of the practicability of his scheme." +DUMMY/LJ045-0085.wav|Why don't you make some money? Poor guy was going out of his mind. We told her she should not annoy him -- poor guy, he is doing his best, "Don't annoy him so much." +DUMMY/LJ019-0346.wav|They might raise funds for this purpose, provided a certificate for the necessity for the new works was given, either by the recorder +DUMMY/LJ037-0181.wav|but not with the other eyewitnesses who claim to have heard from two to four shots. +DUMMY/LJ016-0079.wav|Lears was rewarded by being obliged to wear cross irons on his legs, a punishment rarely inflicted in Newgate, +DUMMY/LJ049-0053.wav|of emergency procedures to be taken in a contingency such as that which actually occurred, the Secret Service responded, quote, +DUMMY/LJ020-0030.wav|Sift into it white flour, meal and salt, and stir up well while dry. +DUMMY/LJ019-0063.wav|who maintained that under this system prisoners were more industrious and more healthy +DUMMY/LJ014-0115.wav|and they had given information to the police. +DUMMY/LJ009-0279.wav|One of the turnkeys rushed forward and pushed him off. +DUMMY/LJ038-0036.wav|As McDonald started to search Oswald's waist for a gun, he heard him say, quote, Well, it's all over now, end quote. +DUMMY/LJ044-0209.wav|for the three-month period prior to the assassination reflects an extremely critical attitude toward President Kennedy and his administration +DUMMY/LJ028-0464.wav|The intervening space, which was filled with dirt probably to the upper inner edge of the outer part, +DUMMY/LJ013-0057.wav|A gigantic conspiracy to defraud was discovered in the following year, when a solicitor named William Henry Barber, +DUMMY/LJ028-0076.wav|However, several decades ago, an Oriental appeared at the Berlin Museum, +DUMMY/LJ034-0012.wav|Atop this carton was a small carton marked "Rolling Readers," measuring approximately thirteen by nine by eight inches. +DUMMY/LJ033-0142.wav|because the cartons stacked around the southeast corner would shield him. +DUMMY/LJ016-0095.wav|In eighteen fifty-three three men escaped in company from one of the wards in the middle yard. +DUMMY/LJ013-0162.wav|a bundle lying on the floor, as though thieves had been interrupted in the act. +DUMMY/LJ017-0264.wav|They were an abject, miserable crew, cowards at heart; but some, especially Lopez, continued bloodthirsty to the last. +DUMMY/LJ003-0164.wav|or the daring reporter of parliamentary debates. +DUMMY/LJ033-0088.wav|Frazier met Oswald at the kitchen door and together they walked to the car. +DUMMY/LJ030-0013.wav|David F. Powers of the President's staff later stated that when the President asked for his assessment of the day's activities, Powers replied +DUMMY/LJ036-0072.wav|to Murphy and Elm three times, averaging six point five minutes for the three trips. +DUMMY/LJ039-0074.wav|you should not have any difficulty in hitting your target. I mean it requires no training at all to shoot a weapon with a telescopic sight +DUMMY/LJ047-0174.wav|During neither interview did Hosty learn Oswald's address +DUMMY/LJ018-0061.wav|One other curious murder may be added to the two foregoing. +DUMMY/LJ050-0203.wav|in this crucial area of Presidential protection. +DUMMY/LJ014-0169.wav|It will be in the memory of many that Mrs. Manning appeared on the scaffold in a black satin dress, which was bound tightly round her waist. +DUMMY/LJ028-0468.wav|its height was probably more than double its width, but that may never be determined. +DUMMY/LJ036-0160.wav|He also stated he saw a silver identification bracelet on his passenger's left wrist. +DUMMY/LJ048-0162.wav|Except for inauguration or other parades involving foreign dignitaries accompanied by the President in Washington, +DUMMY/LJ028-0323.wav|The Babylonians, seeing a Persian of such exalted rank in so grievous a plight, his nose and ears cut off, +DUMMY/LJ015-0008.wav|These indeed were out to the value of three hundred sixty-seven thousand, eight hundred pounds. +DUMMY/LJ029-0194.wav|Any thought that the President might cancel his visit to Dallas was ended +DUMMY/LJ028-0170.wav|In the middle of the precinct there was a tower of solid masonry, a furlong in length and breadth, +DUMMY/LJ028-0478.wav|there were drains of large burned bricks, some of which bore the following long inscription: +DUMMY/LJ013-0003.wav|They not only became more numerous, but they were on a wider scale. +DUMMY/LJ012-0266.wav|Greenacre, a stout, middle-aged man, wrapped in a brown greatcoat, assumed an air of insolent bravado; +DUMMY/LJ017-0109.wav|while the muscles of the fingers were tightly clenched, not open, as usual in a corpse. +DUMMY/LJ012-0249.wav|returned a verdict of willful murder against some person unknown. +DUMMY/LJ007-0236.wav|Sometimes the stench in hot weather is said to be very bad. +DUMMY/LJ034-0056.wav|Another expert with the FBI, Ronald G. Wittmus, conducted a separate examination and also agreed with Latona that the prints were Oswald's. +DUMMY/LJ030-0121.wav|Several times Special Agent John D. Ready came forward from the right front running board of the Presidential follow-up car +DUMMY/LJ050-0128.wav|Much useful information will come to the attention of local law enforcement agencies in the regular course of their activities, +DUMMY/LJ048-0249.wav|and then at a breakfast, after which the entourage would proceed to Dallas. +DUMMY/LJ043-0004.wav|Personal Relations +DUMMY/LJ001-0103.wav|but the general revival of the old forms has made no way in those countries. +DUMMY/LJ024-0076.wav|If such a law as I propose is regarded as establishing a new precedent, +DUMMY/LJ021-0057.wav|have adopted codes of fair competition, which have been approved by the President. +DUMMY/LJ009-0301.wav|who deserted, and was pursued, but without success. +DUMMY/LJ019-0071.wav|moreover, it was nearly impossible to prevent communication and mutual contamination. +DUMMY/LJ020-0027.wav|Half the quantity of sponge given in preceding receipt. +DUMMY/LJ034-0186.wav|That that could have been the man that I saw in the window in the School Book Depository Building, but that I was not sure, end quote. +DUMMY/LJ028-0317.wav|Introduced into their assembly, he began to bewail his misfortunes, telling them that +DUMMY/LJ022-0077.wav|In directing it, I shall recognize six fundamental principles: +DUMMY/LJ009-0195.wav|of people who had died friendless, would voluntarily surrender them for the advancement of medical science. +DUMMY/LJ015-0055.wav|Next morning they only just saved the train to town, and left Sir John behind on the platform, but he subsequently surrendered himself. +DUMMY/LJ028-0444.wav|Deep down in the mound the Germans discovered the palace of Nebuchadnezzar with its hundreds of small chambers and its huge surrounding walls. +DUMMY/LJ031-0093.wav|Answer: No, sir. Before -- well, in trying to treat an acutely injured patient, you have to establish an airway, adequate ventilation +DUMMY/LJ006-0277.wav|But there were evils akin to those on the male side, prominent amongst which was the undue influence accorded to prisoners. +DUMMY/LJ008-0315.wav|"So-and-so, I am sorry to tell you that it is all against you;" or, +DUMMY/LJ033-0005.wav|was brought into the Depository Building, where it was found on the sixth floor shortly after the assassination. +DUMMY/LJ041-0202.wav|Oswald's commitment to Marxism contributed to the decisions which led him to defect to the Soviet Union in nineteen fifty-nine, and later +DUMMY/LJ046-0186.wav|the prosecution of persons who have committed an offense such as threatening the President. +DUMMY/LJ045-0173.wav|Question: What did you say to that? Answer: +DUMMY/LJ002-0305.wav|and thus when the debtor's debt had been actually paid, or when he had abandoned his property to the creditors, and, almost destitute, +DUMMY/LJ042-0063.wav|with few readily available alternatives at hand. He was shocked to find that the Soviet Union did not accept him with open arms. +DUMMY/LJ031-0129.wav|The agents moved a nurse and patient out of a nearby room, lowered the shades, and took emergency security measures to protect the Vice President. +DUMMY/LJ042-0016.wav|He said it was in Japan that he made up his mind to go to Russia +DUMMY/LJ048-0172.wav|At the two places in Dallas where the President would remain for a period of time, Love Field and the Trade Mart, +DUMMY/LJ010-0056.wav|Later on, the Chartists agitated persistently for the concession embraced in the so-called People's Charter, many of which +DUMMY/LJ033-0173.wav|The papers I also found were similar in fiber composition, therefore, in addition to the visual characteristics, +DUMMY/LJ030-0232.wav|Roberts noticed that the Vice President's car was approximately one-half block +DUMMY/LJ009-0069.wav|The ordinary observes him, looks to the sheriffs, and says aloud, 'The service for the dead!' +DUMMY/LJ025-0121.wav|but present a close resemblance when those of animals and those of plants are compared together. +DUMMY/LJ016-0215.wav|The mark was valued at thirteenpence halfpenny, or rather more than the shilling, which from time immemorial had been the hangman's wages. +DUMMY/LJ015-0292.wav|One named Hardwicke assumed the name of Ralph, and, to obtain commercial credit in Yarmouth, paid in two hundred fifty pounds to a Yarmouth bank +DUMMY/LJ026-0145.wav|In the animal carbon dioxide, water and nitrogen compounds are the chief excretions. +DUMMY/LJ022-0015.wav|It is that way with the making of a national policy. The objective of the nation has greatly changed in three years +DUMMY/LJ016-0059.wav|Here he came upon a woman on the leads hanging out clothes to dry. +DUMMY/LJ014-0044.wav|He drew up several long statements, containing narratives purely fictitious, imputing crimes to his victim, and repeating his line of defense +DUMMY/LJ016-0312.wav|Foreign witnesses, too, were in favor of publicity. +DUMMY/LJ037-0179.wav|was discarded along with the others as Oswald left the scene. +DUMMY/LJ044-0210.wav|concerning Cuban policy in general as well as on the issues of automation and civil rights, issues which appeared to concern Oswald a great deal. +DUMMY/LJ014-0231.wav|Emmanuel Barthelemy again, +DUMMY/LJ012-0155.wav|and kept the Black Lion in Vinegar Yard, Drury Lane, where secretly he did business as one of the most daring and successful fencers ever known in the metropolis. +DUMMY/LJ040-0196.wav|into which he appeared to be retreating. +DUMMY/LJ032-0106.wav|end quote. Hidell was a fictitious president of an organization of which Oswald was the only member. +DUMMY/LJ022-0001.wav|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt, Section seven. +DUMMY/LJ011-0002.wav|Many stories were in circulation at the time of Fauntleroy's trial with regard to his forgeries. It was said that he had by means of them +DUMMY/LJ021-0102.wav|If the codes which have been written have been too complicated, +DUMMY/LJ036-0087.wav|Captain Fritz stated that a deputy sheriff whom he could not identify did ask to see him that afternoon +DUMMY/LJ047-0048.wav|while Oswald remained somewhat evasive at this interview, he was not antagonistic and seemed generally to be settling down. +DUMMY/LJ004-0216.wav|The most noticeable of the improvements introduced was a better regulation of dietaries within the prison. +DUMMY/LJ035-0047.wav|and as I got to this door here I caught a glimpse of this man, just, you know, a sudden glimpse. And it looked to me like he was going away from me. +DUMMY/LJ014-0127.wav|She was smartly dressed in a plaid shawl, a white lace cap; +DUMMY/LJ034-0156.wav|whether or not he could positively identify the man he saw in the sixth-floor window as the same man he saw in the police station, +DUMMY/LJ020-0049.wav|When it is light for the second time flour your board, rubbing in the flour and blowing lightly away what does not adhere to the surface. +DUMMY/LJ008-0254.wav|It was followed by the deep and solemn booming of the death-bell from the church of St. Sepulchre +DUMMY/LJ041-0140.wav|and it would have been almost impossible for Oswald to have hit the Governor without hitting the President first. +DUMMY/LJ012-0062.wav|He passed through and out at the back, and was soon beyond pursuit. +DUMMY/LJ038-0122.wav|On Sunday morning, November twenty-four, Oswald denied that he knew A. J. Hidell. +DUMMY/LJ048-0029.wav|As reflected in this testimony, +DUMMY/LJ030-0033.wav|Vice President and Mrs. Johnson followed along the fence, guarded by four members of the Vice-Presidential detail. +DUMMY/LJ011-0184.wav|Nothing could release him from his trouble but the arrival of the marriage certificate from Gretna Green. +DUMMY/LJ046-0178.wav|At the time of the assassination the names of about one hundred persons were in this index, all of whom were included in the group of four hundred +DUMMY/LJ038-0082.wav|Representatives of other law enforcement agencies were also present, including the FBI and the U.S. Secret Service. +DUMMY/LJ010-0156.wav|Collins, when charged, pleaded that he had lost his leg in action, that he had petitioned without success for a pension, +DUMMY/LJ010-0140.wav|Contemporary reports state that from the skillful manner in which he performed the decapitation, +DUMMY/LJ029-0015.wav|The party itself saw an opportunity to raise funds by having the President speak at a political dinner eventually planned for Austin. +DUMMY/LJ021-0208.wav|into the service of the privileged few. +DUMMY/LJ024-0088.wav|I hope that thereby the difficult process of constitutional amendment may be rendered unnecessary. But, +DUMMY/LJ004-0118.wav|published some six-and-thirty years after the promulgation of this act, the flagrant and persistent violations of it and others +DUMMY/LJ007-0052.wav|Benjamin Vines and Daniel Ward put in irons for two days for breaking the windows of the day room in the condemned cells. +DUMMY/LJ006-0281.wav|in some yards +DUMMY/LJ037-0062.wav|The Commission reviewed the transcript of a phone conversation in which Mrs. Markham is alleged to have provided such a description. +DUMMY/LJ026-0031.wav|but when the two groups are traced downward they are found gradually to merge, +DUMMY/LJ039-0063.wav|with the alinement of the assassin's rifle, at a range of one hundred seventy-seven to two hundred sixty-six feet. +DUMMY/LJ002-0042.wav|and supposed to accommodate twenty persons. The eight remaining wards were for the common side debtors, +DUMMY/LJ048-0023.wav|and he had told us during one of the interviews that he would probably take his wife back to Soviet Russia some time in the future. +DUMMY/LJ014-0071.wav|In his confession after sentence he declared that she plied him well with brandy at this period, +DUMMY/LJ016-0017.wav|John Williams, a young fellow only twenty-three years of age, awaited execution in eighteen twenty-seven for stealing in a dwelling-house. +DUMMY/LJ006-0113.wav|The authority of these wardsmen so improperly exalted, and so entirely unchecked, degenerated into a baneful despotism. +DUMMY/LJ021-0149.wav|through the mediation or arbitration of state or federal agencies. +DUMMY/LJ029-0202.wav|the president of the Dallas Chamber of Commerce referred to the city's reputation for being the friendliest town in America and asserted that citizens would, quote, +DUMMY/LJ050-0019.wav|This report has already pointed out several respects +DUMMY/LJ047-0248.wav|that he ever said that Oswald was capable of violence, or that he had any information suggesting this. +DUMMY/LJ028-0166.wav|The center of each division of the town is occupied by a fortress. +DUMMY/LJ035-0145.wav|Within a minute after Baker and Truly left Oswald in the lunchroom, Mrs. R. A. Reid, clerical supervisor for the Texas School Book Depository, +DUMMY/LJ047-0232.wav|In his opinion, none of the information in the FBI files -- Oswald's defection, his Fair Play for Cuba activities in New Orleans, +DUMMY/LJ031-0147.wav|Secret Service Emergency Security Arrangements +DUMMY/LJ009-0023.wav|and that from the time of his conviction nothing but what is consolatory should be addressed to a criminal. +DUMMY/LJ044-0190.wav|by the officials of both Cuba and the Soviet Union in Mexico City. +DUMMY/LJ007-0048.wav|so subversive of meditation, so disturbing to the thoughts; +DUMMY/LJ023-0130.wav|was to infuse new blood into all our courts. +DUMMY/LJ028-0137.wav|Perhaps Herodotus had never been in Babylon; +DUMMY/LJ021-0184.wav|Great Britain in many ways has advanced further along lines of social security than the United States? +DUMMY/LJ001-0025.wav|imitates a much freer hand, simpler, rounder, and less spiky, and therefore far pleasanter and easier to read. +DUMMY/LJ005-0226.wav|and that consequently the observance of their most important provisions was habitually neglected. +DUMMY/LJ028-0109.wav|In one of them he says: +DUMMY/LJ040-0130.wav|and interviewed and observed by other members of the Youth House staff. +DUMMY/LJ035-0131.wav|she would have seen Baker or Truly on the first floor or on the stairs, unless they were already in the second-floor lunchroom talking to Oswald. +DUMMY/LJ046-0009.wav|when visiting Miami on February fifteen, nineteen thirty-three; and on President Harry S. Truman on November one, nineteen fifty, +DUMMY/LJ011-0032.wav|declared that they had hitherto formed a high opinion of his honor, integrity, and goodness of disposition, +DUMMY/LJ016-0045.wav|and at length reached a point opposite the top of the building containing the condemned wards. This had been a perilous and painful task; +DUMMY/LJ003-0094.wav|Mr. Bennet mentions also the case of young men of better stamp, clerks in city offices, and youths of good parentage, +DUMMY/LJ039-0083.wav|Using the scope, rapidly working a bolt and using the scope to relocate your target quickly and at the same time when you locate that target +DUMMY/LJ016-0141.wav|He was at work under the supervision of the artisan warder of the prison, +DUMMY/LJ041-0145.wav|he had more protection than usual. It would appear, therefore, that to the extent Oswald's undesirable discharge affected his motivation, +DUMMY/LJ018-0085.wav|He would shoot any man or any policeman like a dog, or any number of them, who had treated him in that way. +DUMMY/LJ042-0143.wav|during the period from November nineteen, nineteen sixty-two to March of nineteen sixty-three. +DUMMY/LJ039-0065.wav|so that the President would have been moving in an almost straight line away from the assassin's rifle. +DUMMY/LJ001-0147.wav|The position of the page on the paper should be considered if the book is to have a satisfactory look. +DUMMY/LJ024-0095.wav|Then would come the long course of ratification by three-fourths of all the states. +DUMMY/LJ017-0218.wav|The captain was kindly but somewhat intemperate, the first mate a man of some determination, +DUMMY/LJ043-0114.wav|Oswald told his wife and Mrs. Paine that he was working as a commercial photographer. +DUMMY/LJ037-0117.wav|Both men picked Oswald as the man who had run south on Patton with a gun in his hand. +DUMMY/LJ036-0161.wav|Oswald was wearing such a bracelet when he was arrested. +DUMMY/LJ034-0075.wav|He saw Oswald, a clipboard in hand, walking from the southeast corner of the sixth floor toward the elevator. +DUMMY/LJ007-0026.wav|and with all this "the most dreadful oaths, the worst language, too bad to be repeated," were made use of every night. +DUMMY/LJ017-0192.wav|Here, however, the evidence was strong and sufficient. +DUMMY/LJ003-0007.wav|hundreds of women and children came in every morning to spend the day there, and there was no limitation whatever to the numbers of visitors admitted to the debtors' side. +DUMMY/LJ033-0201.wav|All I would say here is that it is possible that these fibers could have come from this blanket. +DUMMY/LJ013-0233.wav|The bed was searched, but no piece of sharpened wood was found. It was thought that it might have been lost in changing the mattresses. +DUMMY/LJ042-0031.wav|The extent of Oswald's desire to go to the Soviet Union +DUMMY/LJ018-0110.wav|Much the same plan was adopted by these forgers as by Saward to get their cheques cashed. +DUMMY/LJ042-0078.wav|I have entered the Soviet Union for the express purpose of applying for citizenship in the Soviet Union, through the means of naturalization. +DUMMY/LJ003-0068.wav|the man guilty of a common assault found himself side by side with the fraudulent, or others who had attempted abominable crimes. +DUMMY/LJ037-0088.wav|Similarly, Virginia Davis had not been shown pictures of anyone prior to the lineup and had not seen either television or the newspapers during the afternoon. +DUMMY/LJ020-0004.wav|than when mixed up with yeast and water only. +DUMMY/LJ030-0204.wav|Special Agent Hill was scanning the few people standing on the south side of Elm Street after the motorcade had turned off Houston Street. +DUMMY/LJ004-0145.wav|At Guildford prison, which Mr. Buxton also visited in eighteen eighteen, +DUMMY/LJ020-0058.wav|Peep under the cloth two or three times to see whether they rise evenly, and turn the pan around once that all may be equally exposed to the heat. +DUMMY/LJ020-0007.wav|Five tablespoonfuls of yeast. One tablespoonful of white sugar. +DUMMY/LJ024-0022.wav|It has been discussed and approved by many persons of high authority +DUMMY/LJ008-0035.wav|were told that the plan had been well considered, and would be persevered in. +DUMMY/LJ046-0079.wav|The rights of private individuals must not be infringed. +DUMMY/LJ005-0051.wav|At the time of its greatest prosperity, its annual income from donations and subscriptions was about one thousand six hundred pounds. +DUMMY/LJ015-0181.wav|He is described by a contemporary account as a fresh-looking man of forty years of age, slightly bald, inclined to embonpoint, +DUMMY/LJ009-0213.wav|The procession traversed Ratcliffe twice, halting for a quarter of an hour in front of the victims' dwelling, +DUMMY/LJ032-0158.wav|On November twenty-three, nineteen sixty-three, +DUMMY/LJ012-0002.wav|On the arrival of the police the house was empty. +DUMMY/LJ002-0322.wav|Share in the weekly allowance of meat might also be denied to debtors who had not paid "garnish," as well as in the weekly grant from the charitable fund. +DUMMY/LJ015-0147.wav|Leading social and artistic personages were to be met with at his house, and his hospitality was far famed. +DUMMY/LJ036-0149.wav|The route of the taxicab was retraced under the direction of Whaley. +DUMMY/LJ008-0288.wav|The consequences were plainly proved to the inspectors. +DUMMY/LJ002-0007.wav|Neild gives some figures which well illustrate this. +DUMMY/LJ038-0111.wav|At that point, Oswald refused to answer any further questions. +DUMMY/LJ015-0046.wav|then came a run on the bank, and it was compelled to stop payment. +DUMMY/LJ044-0189.wav|In spite of his former residence in the Soviet Union and his Fair Play for Cuba Committee activities he had been rebuffed +DUMMY/LJ039-0184.wav|Considering the various probabilities which may have prevailed during the actual assassination, +DUMMY/LJ038-0002.wav|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter four. The Assassin: Part seven. +DUMMY/LJ042-0083.wav|Oswald stated, quote, I am a Marxist, end quote. He also alluded to hardships endured by his mother as a worker, +DUMMY/LJ033-0207.wav|In light of the other evidence linking Lee Harvey Oswald, the blanket, and the rifle to the paper bag found on the sixth floor, +DUMMY/LJ032-0131.wav|He stated that the poor quality of the wood and the metal would cause the rifle to absorb moisture from the skin, thereby making a clear print unlikely. +DUMMY/LJ036-0027.wav|which he reached at about twelve:forty p.m. +DUMMY/LJ034-0175.wav|He looked like he was looking down toward the Trinity River and the Triple Underpass down at the end -- toward the end of Elm Street. And +DUMMY/LJ028-0086.wav|the lips are thin, the chin prominent; the neck is that of a strong vigorous man. +DUMMY/LJ048-0151.wav|asked the police to station guards at each overpass to keep, quote, unauthorized personnel, end quote, off, this term was not defined. +DUMMY/LJ042-0127.wav|Finally, the entry of January four to thirty-one of nineteen sixty-one, quote, +DUMMY/LJ009-0134.wav|This very proper rule was, however, set aside on the Sunday preceding Courvoisier's execution. +DUMMY/LJ010-0132.wav|As the executioner fastened the noose, he nodded to a friend he saw in the crowd; +DUMMY/LJ040-0005.wav|There remains the question of what impelled Oswald to conceive and to carry out the assassination of the President of the United States. +DUMMY/LJ017-0259.wav|Eight were arraigned at the same time: six Spaniards, Leon, Blanco, Duranno, Santos, and Marsolino; +DUMMY/LJ035-0154.wav|Oswald mumbled something and walked by. She paid no more attention to him. +DUMMY/LJ014-0042.wav|the first summed up dead against the prisoner, and the latter, after retiring for ten minutes, found him guilty. +DUMMY/LJ005-0287.wav|The same language was used by the commissioners appointed to inquire into the municipal corporations in eighteen thirty-five, +DUMMY/LJ041-0120.wav|Governor Connally had just resigned to run for Governor of Texas, so he advised Oswald that he had forwarded the letter to his successor. +DUMMY/LJ028-0299.wav|"Had I told thee," rejoined the other, "what I was bent on doing, thou wouldst not have suffered it; +DUMMY/LJ037-0018.wav|Scoggins saw him and heard him mutter either "Poor damn cop" or "Poor dumb cop." +DUMMY/LJ030-0129.wav|and headed toward the Texas School Book Depository Building. +DUMMY/LJ026-0039.wav|the earliest organisms were protists and that from them animals and plants were evolved along divergent lines of descent. +DUMMY/LJ033-0154.wav|and by Arthur Mandella, a fingerprint expert with the New York City Police Department. +DUMMY/LJ032-0100.wav|Marina Oswald +DUMMY/LJ016-0189.wav|but that his cough was getting so much the master of him that he feared he would not be able to carry on his duties much longer. +DUMMY/LJ043-0136.wav|demonstrating once again his propensity to act dramatically and, in this instance violently, in furtherance of his beliefs. +DUMMY/LJ044-0206.wav|The Castro regime severely attacked President Kennedy in connection with the Bay of Pigs affair, the Cuban missile crisis, the ban on travel to Cuba, +DUMMY/LJ008-0040.wav|A full description of this new gallows, which was erected in front of the debtors' door, is to be found in contemporary records. +DUMMY/LJ019-0338.wav|But discipline was to be maintained if necessary by punishment, +DUMMY/LJ022-0143.wav|After careful consideration, I have asked the Congress to extend the life of this useful agency of government. +DUMMY/LJ001-0152.wav|The modern printer, in the teeth of the evidence given by his own eyes, considers the single page as the unit, and prints the page in the middle of his paper +DUMMY/LJ014-0212.wav|a fine young officer who had been summoned to take Cannon into custody when the latter was drunk and riotous in front of a public-house. +DUMMY/LJ015-0014.wav|Davidson and Gordon were also sentenced to imprisonment. +DUMMY/LJ044-0122.wav|I should always remain in the background, i.e. underground, end quote. +DUMMY/LJ009-0300.wav|Traveling expenses of these agents cost fifteen pounds, and another ten pounds were spent in the hire of a Shropshire man, +DUMMY/LJ037-0070.wav|within minutes of the shooting and before the lineup, Mrs. Markham described the man to the police as five foot eight inches tall. +DUMMY/LJ008-0149.wav|and after a long interval his dead body was discovered, shockingly disfigured, in a ditch. This was in eighteen oh two. +DUMMY/LJ046-0181.wav|and who do not have a fixed place of residence. Members of the White House detail of the Secret Service have copies of this album. +DUMMY/LJ017-0188.wav|Mrs. Atkinson's symptoms had been the same as Dixon's. Then Mrs. Wilson went to live with a man named Taylor, +DUMMY/LJ026-0109.wav|but no system of complete circulation as in the blood vessels of a higher animal. +DUMMY/LJ018-0048.wav|and powerful influence was exerted both here and abroad to obtain a reprieve. +DUMMY/LJ018-0165.wav|On a certain day, the twenty-seventh October, eighteen sixty-two, the two were arrested simultaneously; +DUMMY/LJ018-0257.wav|He complained that he had at first been the dupe of others, and admitted that he had too readily fallen astray. +DUMMY/LJ013-0058.wav|Joshua Fletcher a surgeon, and three others were charged with forging wills for the purpose of obtaining unclaimed stock in the funds. +DUMMY/LJ028-0107.wav|The ancients never tired of describing them. +DUMMY/LJ037-0047.wav|Mrs. Markham then ran to Officer Tippit's side and saw him lying in a pool of blood. +DUMMY/LJ030-0117.wav|On the other occasion, the President halted the motorcade to speak to a Catholic nun and a group of small children. +DUMMY/LJ030-0074.wav|This vehicle, a nineteen fifty-five Cadillac eight-passenger convertible especially outfitted for the Secret Service, +DUMMY/LJ013-0190.wav|and at last his lordship's watch was found secreted under the leads of the sink. +DUMMY/LJ034-0131.wav|a person squatting or kneeling exposes more of his body than would normally be the case. +DUMMY/LJ034-0167.wav|of the sixth floor approximately one minute before the assassination, although neither witness saw the shots being fired. +DUMMY/LJ002-0152.wav|fourteen men and women who had lain there ten, eleven, and thirteen years for debts of a few shillings, +DUMMY/LJ025-0092.wav|It is now established that nitrogen is as essential a constituent of vegetable as of animal living matter +DUMMY/LJ045-0197.wav|On November fifteen, nineteen sixty-three, the same day that his wife told him not to come to Irving, Oswald could have assumed +DUMMY/LJ005-0247.wav|Mr. Samuel Hoare was examined by this committee +DUMMY/LJ038-0091.wav|The Commission has, therefore, placed no reliance on the paraffin tests administered by the Dallas police. +DUMMY/LJ016-0362.wav|and the great outside world shared in the completion of the sacrifice. +DUMMY/LJ048-0266.wav|and special agents cooperating with them on Presidential and similar protective assignments are considered to be subject to call for official duty +DUMMY/LJ011-0251.wav|Mullay protested, and then Howard, under the influence of ungovernable rage, as it seemed, jumped up, +DUMMY/LJ005-0148.wav|In some prisons the prisoners worked seven hours a day, in others ten and ten and a half. +DUMMY/LJ040-0009.wav|None of these possibilities satisfactorily explains Oswald's act if it is judged by the standards of reasonable men. +DUMMY/LJ009-0223.wav|but it was actually tried in two provincial towns, Leicester and Durham. +DUMMY/LJ032-0203.wav|on November twenty-two, nineteen sixty-three. One of these pictures, Exhibit Number one thirty-three A, shows most of the rifle's configuration. +DUMMY/LJ047-0037.wav|in which he protested his discharge and declared that he would use, quote, all means, end quote, to correct it. +DUMMY/LJ039-0206.wav|Each series of three shots landed within areas ranging in diameter from three to five inches. +DUMMY/LJ018-0298.wav|Time was clearly an important factor in the fraud, hence the bills were sent forward in quick succession. +DUMMY/LJ008-0011.wav|This was partly due to the survival of the old notion that the scene of the crime ought also to witness the retribution; +DUMMY/LJ039-0201.wav|fired two series of three shots at twenty-five yards in four point six and four point eight seconds. +DUMMY/LJ018-0250.wav|but at the same time with an air of the most entire truthfulness and candor. +DUMMY/LJ044-0035.wav|This is suggested by his misleading and sometime untruthful statements in his letters to Mr. V. T. Lee, +DUMMY/LJ047-0013.wav|Finally, the Commission has reviewed the complete files on Oswald, as they existed at the time of the assassination, of the Department of State, +DUMMY/LJ048-0254.wav|advised, in the course of the Secret Service investigation of these events, that each agent reported for duty on time, +DUMMY/LJ017-0221.wav|eight truculent and reckless scoundrels, who, brooding over their fancied wrongs, and burning for revenge, +DUMMY/LJ046-0025.wav|This planning document has been made a part of the Commission's published record; the underlying staff and consultants' reports reviewed by the Commission have not, +DUMMY/LJ031-0221.wav|When put together, these fragments accounted for approximately three-quarters of the missing portion of the skull. +DUMMY/LJ019-0342.wav|So was the employment of prisoners in any position of trust or authority; +DUMMY/LJ024-0081.wav|But we cannot yield our constitutional destiny to the personal judgment of a few men who, being fearful of the future, +DUMMY/LJ012-0284.wav|It was not until he left the bus, and walked up by the Regent's Canal, that he conceived the idea of throwing the head into the water. +DUMMY/LJ008-0191.wav|At Courvoisier's execution in eighteen forty it was the same, or worse. +DUMMY/LJ031-0174.wav|After the President was pronounced dead, +DUMMY/LJ048-0110.wav|Since the President was to be at the Trade Mart longer than at any other location in Dallas and in view of the security hazards presented by the building, +DUMMY/LJ034-0020.wav|The cartons were forwarded to the FBI in Washington. +DUMMY/LJ001-0157.wav|of this it may be said that though there is some good paper made now, +DUMMY/LJ025-0041.wav|and carbonic acid containing carbon and oxygen. +DUMMY/LJ032-0010.wav|(three) the identity of the person present at the window from which the shots were fired, +DUMMY/LJ011-0175.wav|Wakefield found it necessary to become more explicit regarding her father's affairs. +DUMMY/LJ003-0013.wav|No attempt was made to check drunkenness, beyond the penalty of shutting out friends from any ward in which a prisoner exceeded. +DUMMY/LJ050-0202.wav|Lack of adequate resources is an unacceptable excuse for failing to improve advance precautions +DUMMY/LJ032-0030.wav|It was sent in an envelope bearing the same name and return address in handwriting. +DUMMY/LJ019-0379.wav|matters went on after the eighteen sixty-five Act much the same as they had done before. Districts differed greatly in the attention they paid to prison affairs. +DUMMY/LJ049-0048.wav|and therefore approximately one point six seconds after the President was shot in the head. +DUMMY/LJ008-0158.wav|The pressure soon became so frightful that many would have willingly escaped from the crowd; but their attempts only increased the general confusion. +DUMMY/LJ002-0074.wav|and tells us that in the state side, quote, such prisoners were safely associated whose manners and conduct evince a more liberal style of education, +DUMMY/LJ041-0137.wav|In support of her conclusion Mrs. Oswald noted her husband's undesirable discharge +DUMMY/LJ048-0234.wav|described by some as a beatnik place and by its manager as, quote, a unique showplace with continuous light entertainment all night +DUMMY/LJ007-0099.wav|were the ten persons committed to Newgate by the House of Commons in eighteen thirty-five. +DUMMY/LJ018-0174.wav|and the prominent part he played secured for him on conviction the heaviest sentence of the law. +DUMMY/LJ010-0091.wav|the heads of Lord Castlereagh and Lord Sidmouth being carried away in a bag. +DUMMY/LJ034-0018.wav|Next to these cartons was the handmade paper bag, previously discussed, +DUMMY/LJ002-0123.wav|Barely half of these gave bail bonds on arrests, and the remainder went to prison. +DUMMY/LJ002-0316.wav|notably as when numbers filled Newgate in anticipation of Lord Redesdale's bill for insolvent debtors, +DUMMY/LJ022-0073.wav|Our problem is to put to work three and one-half million employable persons now on the relief rolls. +DUMMY/LJ049-0180.wav|The Commission has the impression +DUMMY/LJ008-0031.wav|when, in pursuance of an order issued by the Recorder to the sheriffs of Middlesex and the keeper of His Majesty's jail, Newgate, +DUMMY/LJ008-0159.wav|Very soon women began to scream with terror; +DUMMY/LJ038-0090.wav|was to the effect that the paraffin test was unreliable in determining whether or not a person has fired a rifle or revolver. +DUMMY/LJ014-0130.wav|Manning, when sentence of death was passed on him, said nothing; +DUMMY/LJ050-0054.wav|Since the assassination, both the Secret Service and the FBI have recognized +DUMMY/LJ016-0443.wav|He was promptly rescued from his perilous condition, but not before his face and hands were badly scorched. +DUMMY/LJ011-0107.wav|On entering Newgate he said, +DUMMY/LJ011-0126.wav|In conjunction with two others, one of whom was a clerk in the Custom House, +DUMMY/LJ002-0313.wav|It generally ran to about six pounds per week. The money, which at one time had been distributed quarterly, and all went in drink, +DUMMY/LJ028-0015.wav|only a tall column of whirling sand, rearing its head until it is lost in the blue above, moves majestically along. +DUMMY/LJ034-0190.wav|Robert Edwards said that, while looking at the south side of the Depository Building shortly before the motorcade, +DUMMY/LJ015-0022.wav|he was himself a prominent member of the Low Church, of austere piety, active in all good works. +DUMMY/LJ012-0260.wav|The murdered individual was thus identified. The next step was to ascertain where and with whom she had last been seen. +DUMMY/LJ008-0055.wav|for that city claims the priority in establishing the custom of hanging criminals at the jail itself. +DUMMY/LJ016-0260.wav|has already been quoted. Again, the concourse of people collected in front of Newgate to witness the execution, +DUMMY/LJ037-0174.wav|but only two of the four discarded cartridge cases found on the lawn at tenth Street and Patton Avenue were of Winchester-Western manufacture. +DUMMY/LJ018-0097.wav|At length a man was caught in the act of passing a forged cheque at the Union Bank, +DUMMY/LJ025-0044.wav|Animals, on the contrary, derive their nourishment either directly or indirectly from plants. +DUMMY/LJ039-0197.wav|The purpose of this experiment was not to test the rifle under conditions which prevailed at the time of the assassination +DUMMY/LJ025-0139.wav|converge toward the conclusion that the nerve fibers, which have been regarded as ultimate elements of nervous tissue, +DUMMY/LJ008-0083.wav|Two cart-loads of faggots were piled about her, and after she had hung for half-an-hour the fire was kindled. +DUMMY/LJ046-0114.wav|the arrangements relied upon by the Secret Service to perform this function were seriously deficient. +DUMMY/LJ046-0194.wav|Members of the White House detail were expected to familiarize themselves with the descriptions and photographs of the highest risk cases. +DUMMY/LJ043-0080.wav|with fifty percent of the people taking it who score less than one hundred. +DUMMY/LJ036-0025.wav|McWatters was sure that he left the checkpoint on time +DUMMY/LJ028-0174.wav|Other ancient descriptions of the walls have been left us by Ctesias of the fifth century B.C., and by Strabo of the beginning of the Christian era, +DUMMY/LJ029-0134.wav|almost five hundred feet from Houston Street. +DUMMY/LJ043-0067.wav|that his employment difficulties were caused by his telling prospective employers that he had last been employed in Minsk. +DUMMY/LJ007-0127.wav|"It may easily be imagined," say the inspectors, in speaking of the prison generally, +DUMMY/LJ028-0032.wav|Babylon, even in the days of Nebuchadnezzar, was an old, old city. +DUMMY/LJ018-0384.wav|and to show his gratitude to the warder who procured this privilege for him, he made him a present. +DUMMY/LJ016-0404.wav|The man who had made a pious life a cloak for his misdeeds, the once exemplary young man and indefatigable Sunday School teacher, +DUMMY/LJ004-0042.wav|The association was organized under the most promising auspices. +DUMMY/LJ010-0166.wav|From this he was promoted to barman, and as such had charge of the business in various public-houses. +DUMMY/LJ017-0137.wav|In the latter hope he was happily disappointed. But as this case is beyond my subject, I merely mention it as one of the group already referred to. +DUMMY/LJ017-0150.wav|and whom he left without scruple to join Miss Bankes. +DUMMY/LJ009-0145.wav|and he seemed to suffer great inward agitation when the ordinary particularly alluded to the crime for the perpetration of which he stood condemned. +DUMMY/LJ031-0110.wav|by cutting away the edges of the wound on the front of the Governor's chest and suturing the damaged lung and lacerated muscles. +DUMMY/LJ028-0420.wav|That was the first object taken from Babylon to Europe; +DUMMY/LJ021-0023.wav|and the great transporting and manufacturing and distributing agencies, +DUMMY/LJ003-0304.wav|with the penalty of forfeiting the day's allowance of food, an increase of which the committee had recommended. +DUMMY/LJ017-0121.wav|It came out at once that these had been forged, and the whole affair at once took the ugliest complexion. +DUMMY/LJ005-0254.wav|upon the rules and discipline therein established with regard to the treatment of unconvicted as well as convicted persons." +DUMMY/LJ025-0057.wav|such as the Charae, are in constant and regular motion was made out by Bonaventura Corti a century ago; +DUMMY/LJ050-0115.wav|This is especially necessary with regard to the FBI and CIA, +DUMMY/LJ007-0126.wav|All the evils of indiscriminate association were strongly accentuated by the crowd collected within its narrow limits. +DUMMY/LJ003-0258.wav|Hence the frequent cases of drunkenness, of which no notice was taken, unless people grew riotous in their cups +DUMMY/LJ019-0022.wav|On the other hand, it must be admitted +DUMMY/LJ028-0396.wav|they were supplied with food by keepers, and gave the king the opportunity of hunting whenever he felt inclined. +DUMMY/LJ032-0221.wav|it is reasonably certain that Exhibit Number one thirty-three A was taken by the same camera at the same time, as Marina Oswald testified. +DUMMY/LJ032-0210.wav|to the exclusion of all other rifles of the same general configuration, end quote. +DUMMY/LJ004-0159.wav|The state of the prison, the desperation of the prisoners, broadly hinted in their conversation and plainly expressed in their conduct, +DUMMY/LJ028-0091.wav|and most of the records which have come from his time speak chiefly of his deeds of piety. +DUMMY/LJ002-0173.wav|The lad was proved to be of good character and the son of respectable parents. +DUMMY/LJ035-0167.wav|the earliest estimates would still have permitted Oswald to leave the building by twelve:thirty-three. +DUMMY/LJ013-0209.wav|he was annoyed with his man for various small omissions and acts of forgetfulness, and on the night of the murder had taken Courvoisier to task rather sharply. +DUMMY/LJ012-0090.wav|He went out next morning with the booty when the doors were re-opened, and attracted no attention. +DUMMY/LJ004-0068.wav|or amidst the noxious effluvia of dirt and corruption. +DUMMY/LJ010-0276.wav|and he was only convicted of an attempt "to harass, vex, and grieve the sovereign. +DUMMY/LJ042-0172.wav|must have as its nucleus the traditional ideological best of both systems, and yet be utterly opposed to both systems. +DUMMY/LJ033-0167.wav|In addition the tape was compared to determine whether the sample tape and the tape on the bag had been taken from the tape dispensing machine at the Depository. +DUMMY/LJ027-0156.wav|According to the evolutionary interpretation of such facts, we can see a very good reason why the life-history of the individual +DUMMY/LJ038-0294.wav|He indicated that he wanted more people in the vicinity at the time of the attempt so that his arrival and departure would not attract great attention. +DUMMY/LJ017-0096.wav|The night he bought it, Cook, who had been taking certain pills under medical advice, not Palmer's, was seized with violent convulsions. +DUMMY/LJ031-0047.wav|He observed shredded brain tissue and, quote, considerable slow oozing, end quote, from the latter wound, +DUMMY/LJ044-0154.wav|she wrote in her letter of July nineteen sixty-three, +DUMMY/LJ038-0170.wav|two nights before the shooting he saw, quote, two men around the house peeking in windows, end quote. +DUMMY/LJ013-0047.wav|A clergyman nearly a century later followed in the steps of Dr. Dodd, but did not under more humane laws lose his life. +DUMMY/LJ036-0218.wav|Four bullets hit Tippit and killed him instantly. +DUMMY/LJ005-0273.wav|or prisoners employed in positions of trust, who should not be permitted to traffic with their fellow-prisoners in any way. +DUMMY/LJ016-0046.wav|the spikes of the railing penetrated his flesh and made progression slow and difficult. +DUMMY/LJ027-0108.wav|as well as useless muscles to move the ears and the skin, and also a very much reduced hairy covering over the surface of the body. +DUMMY/LJ015-0116.wav|When brought to trial he took matters very coolly, and at first pleaded "Not Guilty," but subsequently withdrew the plea. +DUMMY/LJ045-0109.wav|On Sunday, November seventeen, nineteen sixty-three, +DUMMY/LJ002-0336.wav|indeed the place, with its oak floors caulked with pitch, and smoked ceilings, +DUMMY/LJ008-0108.wav|The ordinary "gravely uttered, 'Come this way, Mr. Smith.' +DUMMY/LJ024-0107.wav|The first includes those who fundamentally object to social and economic legislation along modern lines. +DUMMY/LJ037-0132.wav|Neither Reynolds nor Patterson saw the man after he turned off Jefferson at the service station. +DUMMY/LJ044-0022.wav|one of the Cuban exiles who had been arrested with him on August nine. Bringuier claimed that on August five, nineteen sixty-three, +DUMMY/LJ017-0211.wav|which left London for Singapore on the twenty-eighth July, eighteen sixty-three, with a cargo of wine and other goods. +DUMMY/LJ042-0118.wav|I begin to feel uneasy inside, its true! End quote. +DUMMY/LJ001-0119.wav|and the elegance and legibility of the ancient more striking than in the Arabic numerals. +DUMMY/LJ044-0002.wav|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter seven. +DUMMY/LJ047-0198.wav|prior to President Kennedy's visit. +DUMMY/LJ029-0106.wav|The police officials agreed that the route recommended by Sorrels was the proper one and did not express a belief that any other route might be better. +DUMMY/LJ005-0171.wav|we shall find but few which have not in one or other of these ways removed the grievance of which such just complaint is made. +DUMMY/LJ046-0029.wav|It is unlikely that measures can be devised to eliminate entirely the multitude of diverse dangers that may arise, +DUMMY/LJ027-0011.wav|It is, moreover, true that all living forms are but series of modifications and extensions of one single plan of structure. +DUMMY/LJ022-0066.wav|will not only help to guard the individual in future periods of lay-off against dependence upon relief, +DUMMY/LJ001-0092.wav|The Chiswick press in eighteen forty-four revived Caslon's founts, printing for Messrs. Longman the Diary of Lady Willoughby. +DUMMY/LJ011-0068.wav|It was said that strangulation had been prevented by the insertion of a silver tube in his wind-pipe, +DUMMY/LJ038-0287.wav|Additional corroborative evidence. +DUMMY/LJ038-0127.wav|but denied that he had received a package in this box addressed to Hidell. +DUMMY/LJ038-0057.wav|Deputy Sheriff Walthers brought a shotgun into the theatre but laid it on some seats before helping subdue Oswald. +DUMMY/LJ003-0027.wav|He had a double allowance of bread, deducted, of course, from the already too limited portion of the rest, +DUMMY/LJ049-0073.wav|who were immediately responsible for the President's safety reacted promptly at the time the shots were fired. +DUMMY/LJ021-0116.wav|We also question the wisdom of extending code requirements suited to the great industrial centers and to large employers, +DUMMY/LJ046-0059.wav|Since nineteen forty-five, Roosevelt's successors have ranged the world, +DUMMY/LJ036-0110.wav|Whaley believes that Oswald's conduct did not aid him in his identification, quote, because I knew he was the right one as soon as I saw him. +DUMMY/LJ035-0126.wav|claimed that within about one minute following the shots she ran from a window on the south side of the fourth floor, +DUMMY/LJ024-0021.wav|It seeks to maintain the federal bench in full vigor. +DUMMY/LJ015-0232.wav|Meanwhile the others completed their preparations with the utmost care. +DUMMY/LJ001-0080.wav|he seems to have taken the letter of the Elzevirs of the seventeenth century for his model: +DUMMY/LJ010-0081.wav|One of the conspirators, by name Edwards, +DUMMY/LJ015-0048.wav|But worse than the bankruptcy was the confession made by the partners in the court. +DUMMY/LJ005-0176.wav|which possessed the right of trying criminals for various offenses. +DUMMY/LJ003-0250.wav|the culprit was chained down to the floor by running a chain through his irons which prevented him from climbing to the window of his cell. +DUMMY/LJ046-0045.wav|it is the President's right and duty to be the active leader of his party, as when he seeks to be reelected or to maintain his party in power. +DUMMY/LJ032-0044.wav|shows an imprint made by the cash register which recorded the receipt of twenty-one dollars, forty-five cents on March thirteen, nineteen sixty-three. +DUMMY/LJ038-0205.wav|She thought he was attending a class or was on his own business. +DUMMY/LJ034-0161.wav|Immediately after the assassination, however, +DUMMY/LJ038-0177.wav|A note which Oswald left for his wife on the evening of the shooting, +DUMMY/LJ018-0010.wav|Inside the carriage was a hat, a walking-stick, and a small black leather bag. +DUMMY/LJ002-0244.wav|but its business was much reduced by the extension of the Courts of Conscience. +DUMMY/LJ034-0047.wav|were developed by chemical processes. +DUMMY/LJ022-0149.wav|to safeguard, define and enforce collective bargaining, and, while retaining fair competition, +DUMMY/LJ016-0158.wav|It was that of a "Long Firm" swindler, by name Johnson, +DUMMY/LJ042-0237.wav|He apparently concluded that the United States offered, quote, +DUMMY/LJ015-0129.wav|by inserting say one before five hundred, and thus making it +DUMMY/LJ045-0199.wav|Whether he thought about assassinating the President over the weekend can never be known, but it is reasonably certain +DUMMY/LJ001-0156.wav|The paper on which the printing is to be done is a necessary part of our subject: +DUMMY/LJ049-0207.wav|that it was not within the Commission's responsibility to make specific recommendations as to the long-range organization of the President's protection, +DUMMY/LJ018-0047.wav|He adhered to this almost to the very last. His case had been warmly espoused by the Society for the Protection of Germans in this country, +DUMMY/LJ045-0175.wav|She did not speak to him when she joined him there, although she thought that he was still awake. +DUMMY/LJ048-0207.wav|there were several Secret Service agents in it who shared the responsibility of scanning the windows of nearby buildings. +DUMMY/LJ004-0112.wav|three. Prisoners guilty of misdemeanors. four. Prisoners charged with misdemeanors. five. Debtors. +DUMMY/LJ047-0081.wav|On June twenty-four, Oswald applied in New Orleans for a passport, stating that he planned to depart by ship +DUMMY/LJ010-0010.wav|Street robberies offered fewer inducements to depredators, and evil-doers were compelled to adopt other methods of preying upon their fellows. +DUMMY/LJ019-0399.wav|and it is now, except during sessions or when the gallows is in requisition, practically and for ever closed. +DUMMY/LJ009-0265.wav|Even now his sufferings were not at an end, +DUMMY/LJ011-0224.wav|That Mr. Wakefield was a shrewd critic and close observer of all that went on in the Newgate of those days, +DUMMY/LJ042-0240.wav|he continued to be interested in that country after he returned to the United States. +DUMMY/LJ006-0051.wav|and for offenses of the most varying description, by the Central Criminal Court. +DUMMY/LJ033-0103.wav|It was the first time that Oswald had not walked with Frazier from the parking lot to the building entrance. +DUMMY/LJ040-0119.wav|When Pic returned home, Mrs. Oswald tried to play down the event but Mrs. Pic took a different view and asked the Oswalds to leave. +DUMMY/LJ039-0045.wav|This was one day before Oswald left for New Orleans and Marina appeared certain that the Nixon incident, quote, +DUMMY/LJ002-0013.wav|till, eighteen oh eight, the lowest point was touched of one hundred ninety-seven debtors and one hundred eighty-two felons, or three hundred seventy-nine in all. +DUMMY/LJ008-0113.wav|and his soul shot out so piercingly through the port-holes of his head, that the first glance of him nearly petrified me +DUMMY/LJ026-0047.wav|circulation, metabolism, excretion, oxygenation (part of respiration), +DUMMY/LJ037-0130.wav|Reynolds and Patterson decided to follow him. +DUMMY/LJ041-0135.wav|In that connection, it should be noted that Marina Oswald testified on September six, nineteen sixty-four, that she thought her husband, quote, +DUMMY/LJ038-0164.wav|The Attempt on the Life of Maj. Gen. Edwin A. Walker +DUMMY/LJ027-0121.wav|In numberless cases, such as that of the fore-limbs of serpents, no vestige of a rudiment is present. +DUMMY/LJ013-0249.wav|half consumed in the ashes of a wood fire. +DUMMY/LJ045-0195.wav|onto the Stemmons Freeway. +DUMMY/LJ019-0329.wav|every prison was ordered to keep up an infirmary, and the medical supervision was to be strict and continuous. +DUMMY/LJ026-0081.wav|Thus the leaf of a green plant in the light is continually absorbing carbon dioxide and giving forth free oxygen. +DUMMY/LJ005-0249.wav|From the moment of a person's committal he was certain to be plunged deeper and deeper in guilt. +DUMMY/LJ004-0176.wav|Sick women and their children remained in the ordinary wards, and propagated disease. +DUMMY/LJ049-0124.wav|In addition, the proposed legislation will insure +DUMMY/LJ046-0093.wav|and accepts the necessary security precautions which they recommend. +DUMMY/LJ019-0149.wav|and they spent some fourteen or fifteen hours in total darkness, and almost without check or control. +DUMMY/LJ046-0140.wav|any communication, quote, that in any way indicates anyone may have possible intention of harming the President, end quote. +DUMMY/LJ035-0138.wav|Shelley and Lovelady, at a fast walk or trot, turned west into the railroad yards and then to the west side of the Depository Building. +DUMMY/LJ004-0097.wav|to provide such accommodations in jails as may be necessary to answer this salutary purpose, +DUMMY/LJ047-0165.wav|At the end of the interview, Marina Oswald came into the room. When he observed that she seemed, quote, quite alarmed, end quote, about the visit, +DUMMY/LJ050-0071.wav|Both Director Hoover and Belmont expressed to the Commission the great concern of the FBI, which is shared by the Secret Service, +DUMMY/LJ040-0026.wav|He had very few, if any, close relationships with other people and he appeared to have great difficulty in finding a meaningful place in the world. +DUMMY/LJ028-0406.wav|The great irrigating dams across the Euphrates are constructed entirely of them. +DUMMY/LJ002-0081.wav|and beyond it was now a day room for the capital convicts or those awaiting execution. +DUMMY/LJ019-0304.wav|twenty-two others received between eleven and twenty-five; +DUMMY/LJ010-0290.wav|The blow crushed the bonnet and bruised the forehead of the Queen, who was happily not otherwise injured. +DUMMY/LJ009-0024.wav|One of the aldermen, moreover, informed me that the whole court of aldermen were unanimous in their opinion on this subject. +DUMMY/LJ016-0393.wav|called for a roast duck directly he entered the condemned cell. +DUMMY/LJ026-0072.wav|The sap travels throughout the whole plant, the main though not the only cause of movement +DUMMY/LJ045-0023.wav|Oswald's writings and his reading habits indicate that he had an extreme dislike of the right wing, an attitude most clearly reflected by his attempt +DUMMY/LJ028-0148.wav|Then they set to building, and began by bricking the borders of the moat, after which they proceeded to construct the wall itself, +DUMMY/LJ005-0164.wav|When the new acts were introduced, numerous prisons under local jurisdiction were exempted from the operation of the law. +DUMMY/LJ030-0190.wav|As he issued his instructions to Greer and to the lead car, Kellerman heard a flurry of shots within five seconds of the first noise. +DUMMY/LJ045-0244.wav|Long before the assassination he expressed his hatred for American society and acted in protest against it. +DUMMY/LJ005-0027.wav|He was altogether against too liberal a diet; he disapproved of industrial occupations in jails, as not calculated to render prisons terrible. +DUMMY/LJ018-0022.wav|It was also easily established that the hat found in the carriage had been bought at Walker's, a hatter's in Crawford Street, +DUMMY/LJ003-0253.wav|Clothes or prison uniform would not have served the purpose, for a disguise can be rapidly and secretly put on, +DUMMY/LJ004-0058.wav|The untried, and in the eyes of the law still innocent, could claim pure air, wholesome and sufficient food, and opportunities for exercise. +DUMMY/LJ044-0111.wav|for anybody who is concerned about developments in Cuba, end quote, +DUMMY/LJ015-0205.wav|who had been concerned in various "jobs" of a dishonest character, and who for the moment was a clerk in a betting office. +DUMMY/LJ030-0095.wav|this vehicle was occupied by three Secret Service agents and Clifton C. Garter, assistant to the Vice President. +DUMMY/LJ016-0157.wav|One curious instance of a suicide carried out under the most adverse and extraordinary circumstances may be quoted. +DUMMY/LJ042-0192.wav|are preferred rather than loud and useless manifestations of protest, end quote, Oswald went on to note, quote, +DUMMY/LJ031-0148.wav|Immediately after President Kennedy's stretcher was wheeled into trauma room one, Secret Service agents took positions at the door of the small emergency room. +DUMMY/LJ007-0232.wav|The plan adopted for ventilating the dining-room on the 'master's side' and that of the middle yard is very inefficient; +DUMMY/LJ026-0020.wav|If chlorophyll is present, the carbon dioxide of the air serves as a source of carbon, +DUMMY/LJ043-0040.wav|by so-called exploitation of his mother by the capitalist system. +DUMMY/LJ042-0126.wav|He is a no-nonsense party regular. +DUMMY/LJ045-0196.wav|That fact was made precisely clear in subsequent news stories on November nineteen, twenty, and twenty-two. +DUMMY/LJ049-0017.wav|The limousine used by President Kennedy in Dallas was a convertible with a detachable, rigid plastic "bubble" top +DUMMY/LJ043-0033.wav|Bouhe thoroughly disapproved of this and as a result almost all communication between the Oswalds and members of the Russian community ceased. +DUMMY/LJ009-0220.wav|the bodies of all prisoners convicted of murder should either be hung in chains, or buried under the gallows on which they had been executed, +DUMMY/LJ016-0397.wav|The craving for tobacco which so dominates the habitual smoker often leads the convicted to plead hard for a last smoke. +DUMMY/LJ046-0090.wav|Any travel, any contact with the general public, involves a calculated risk on the part of the President and the men responsible for his protection. +DUMMY/LJ002-0111.wav|Of these the debtors, male and female, formed a large proportion. +DUMMY/LJ033-0168.wav|When asked to explain the similarity of characteristics, Cadigan stated, quote, well briefly +DUMMY/LJ005-0034.wav|Undeterred by these sarcasms and misrepresentations, +DUMMY/LJ012-0078.wav|One of the largest robberies of its class was that effected upon the Custom House in the winter of eighteen thirty-four. +DUMMY/LJ048-0018.wav|We again exhibited interest on the basis of these contacts with The Worker, Fair Play for Cuba Committee, which are relatively inconsequential. +DUMMY/LJ010-0183.wav|Meanwhile the pistol-shots had attracted the attention of the bystanders, of whom there was a fair collection, as usual, waiting to see the Queen pass. +DUMMY/LJ016-0305.wav|Already the urgent necessity for abolishing public executions had been brought before the House of Commons by Mr. Hibbert, +DUMMY/LJ013-0237.wav|Daniel Good's crime might have remained long undiscovered but for his own careless stupidity. +DUMMY/LJ020-0075.wav|Into a second hollow pour the yeast and knead thoroughly for fifteen minutes. +DUMMY/LJ007-0218.wav|Newgate is only less extensively injurious than formerly because it is less crowded. +DUMMY/LJ034-0096.wav|on the southwest corner of Elm and Houston Streets, looking north at the Depository Building which was directly in front of him. +DUMMY/LJ009-0229.wav|Later on, after dark, some friends of the deceased stole the body and buried it in the sand, and this was the end of hanging in chains. +DUMMY/LJ015-0262.wav|came out at the Old Bailey, and was acknowledged to be one of the most extraordinary on record. +DUMMY/LJ041-0101.wav|aggressive and even somewhat pugnacious, although Powers, quote, wouldn't say that this guy is a troublemaker, end quote. +DUMMY/LJ030-0114.wav|At the President's direction, his automobile stopped twice, the first time to permit him to respond to a sign asking him to shake hands. +DUMMY/LJ037-0133.wav|These four witnesses were interviewed by FBI agents two months after the shooting. +DUMMY/LJ020-0108.wav|Other things besides rising dough get on quite as well without your standing by to watch them. +DUMMY/LJ043-0097.wav|In February or March of nineteen sixty-three, +DUMMY/LJ029-0073.wav|that the President would arrive and depart from Dallas' Love Field; that a motorcade through the downtown area of Dallas to the luncheon site should be arranged; +DUMMY/LJ049-0204.wav|While in accordance with its mandate +DUMMY/LJ039-0171.wav|On the second series they required five point one five, six point four five, and seven seconds. +DUMMY/LJ045-0214.wav|ended when Marina Oswald hung up and refused to talk to him. Although he may long before have decided on the course he was to follow +DUMMY/LJ043-0042.wav|He first told his wife that he did not have a mother, but later admitted that he did but that, quote, he didn't love her very much, end quote. +DUMMY/LJ001-0144.wav|the words may be set much closer together, without loss of clearness. +DUMMY/LJ007-0221.wav|The reports as the years flow on reiterate the same complaints. +DUMMY/LJ049-0010.wav|who responded to the unplanned event with dispatch. +DUMMY/LJ040-0194.wav|He had refused to salute the flag and was doing very little, if any, work. +DUMMY/LJ004-0012.wav|was in several places merely temporary: +DUMMY/LJ016-0021.wav|Although his execution was imminent, a surgeon attended to his wounds, and he was carried more dead than alive to the scaffold. +DUMMY/LJ036-0116.wav|Lee Oswald was Number three; +DUMMY/LJ005-0056.wav|It is interesting to observe that the plan of "radiation," by which the prison blocks radiated from a central hall, like spokes in a wheel +DUMMY/LJ018-0277.wav|The last great case of fraud upon the Bank of England will fitly close this branch of the criminal records of Newgate. +DUMMY/LJ043-0127.wav|and that he, quote, may be a damn Communist. I can't tell you. If I was you, I wouldn't hire him, end quote. +DUMMY/LJ008-0103.wav|who had just arrived from the purlieus of Black Boy Alley, woebegone as Romeo's apothecary, exclaiming, +DUMMY/LJ011-0271.wav|Mr. Gee went in the coach sent for him, and alighted at twenty-seven, York Street, West, Commercial Road. +DUMMY/LJ035-0055.wav|Along a side wall of the lunchroom was a soft drink rending machine, but at that time the man had nothing in his hands. +DUMMY/LJ023-0009.wav|Soon after, with the authority of the Congress, +DUMMY/LJ009-0230.wav|After this a law was passed which prescribed that the bodies of all executed murderers should be buried within the walls of the jail. +DUMMY/LJ028-0030.wav|Your journey is at an end. Before you is Babylon, the "Gate of God," as the old name means. +DUMMY/LJ050-0072.wav|that referrals to the Secret Service under the new criteria might, if not properly handled, +DUMMY/LJ017-0272.wav|"Ah!" he remarked, "they will have to wait for us then till eight." +DUMMY/LJ005-0025.wav|"A poor man who is lucky enough," he said, "to have his son committed for a felony +DUMMY/LJ032-0259.wav|On several occasions, Michael Paine moved the blanket in the garage. +DUMMY/LJ011-0025.wav|and it was openly stated in the press and in conversation that the proceeds of these frauds had been squandered in chambering, gambling, and debauchery. +DUMMY/LJ031-0217.wav|(external occipital protuberance) which juts out at the center of the lower part of the back of the skull. +DUMMY/LJ037-0031.wav|He found two empty shells in the bushes and gave them to Patrolman J. M. Poe who arrived on the scene shortly after the shooting. +DUMMY/LJ022-0059.wav|it is obvious that we cannot continue to create governmental deficits for that purpose year after year. +DUMMY/LJ042-0035.wav|and communist's would like to see the present capitalist government of the U.S. overthrown, end quote, Oswald stated +DUMMY/LJ023-0066.wav|and provide for the common defense and general welfare of the United States. +DUMMY/LJ044-0064.wav|In addition, there is no evidence that he received any substantial amount of materials from the national headquarters. +DUMMY/LJ029-0150.wav|Police were assigned to each overpass on the route and instructed to keep them clear of unauthorized persons. +DUMMY/LJ015-0169.wav|It was at once decided at the board to make a thorough examination of all his books. +DUMMY/LJ013-0032.wav|Mate, carpenter, and crew were eager to disavow complicity, and voluntarily gave information. +DUMMY/LJ019-0223.wav|misdemeanants have found their way into Newgate, and within the last few years one offender against the privileges of the House of Commons. +DUMMY/LJ006-0194.wav|Gaming of all kinds, although forbidden by the Jail Acts, was habitually practiced. +DUMMY/LJ003-0338.wav|End quote. it would cover some thirty acres, and cost a great deal more than the city, with the example of Whitecross Street prison before it, +DUMMY/LJ032-0107.wav|When seeking employment in New Orleans, +DUMMY/LJ028-0022.wav|Day after day you travel on, scorched by the heat of noon-day, shivering in the chill winds of the night. +DUMMY/LJ031-0042.wav|made no voluntary movements; had his eyes open with the pupils dilated without any reaction to light, +DUMMY/LJ006-0059.wav|the felon and the misdemeanant, the sane and the insane, the old and young offender. +DUMMY/LJ037-0191.wav|Sometime after January twenty-seven, nineteen sixty-three, +DUMMY/LJ012-0237.wav|A very painful scene occurred in Newgate when the news of his escape from death was imparted to May. +DUMMY/LJ002-0161.wav|These surely, I thought," says Mr. Neild, "were bad enough! But it was not so. End quote. +DUMMY/LJ002-0030.wav|Full details of the arrangements are to be found in Mr. Neild's "State of Prisons in England, Scotland, and Wales," published in eighteen twelve. +DUMMY/LJ023-0100.wav|that the majority were actually reading into the Constitution their own "personal economic predilections," and that +DUMMY/LJ047-0127.wav|On August twenty-one, nineteen sixty-three, Bureau headquarters instructed the New Orleans and Dallas field offices +DUMMY/LJ008-0219.wav|their conversation was of companions and associates of former years, long ago imprisoned, transported, hanged, while they, +DUMMY/LJ014-0336.wav|Last of all, the well-known bankers Overend and Gurney, whose own affairs created much excitement some years later, +DUMMY/LJ028-0025.wav|You have crossed the Arabian Desert, the first stage of the long journey to the walls of Babylon. +DUMMY/LJ009-0189.wav|Persons were still living in eighteen fifty-five who had witnessed dissections at Hicks' Hall, and +DUMMY/LJ005-0091.wav|Last, but not least, the use of irons was strictly forbidden, "except in cases of urgent and absolute necessity," +DUMMY/LJ001-0058.wav|and are worthy representatives of the eager enthusiasm for the revived learning of that epoch. By far, +DUMMY/LJ001-0172.wav|The ornamentation of printed books is too wide a subject to be dealt with fully here; but one thing must be said on it. +DUMMY/LJ043-0093.wav|He also had difficulty in working with the other employees. +DUMMY/LJ014-0135.wav|It was against common sense to charge her with murdering the only friend she had in the world; +DUMMY/LJ044-0108.wav|Arnold Johnson, director of the information and lecture bureau of the Communist Party, U.S.A., replied stating, quote, +DUMMY/LJ030-0214.wav|At the instant that Hill stepped onto the left rear step of the President's automobile and grasped the handhold, +DUMMY/LJ039-0024.wav|She stated that it was not physical force which kept him from leaving the house. Quote, I couldn't keep him from going out if he really wanted to, end quote. +DUMMY/LJ007-0199.wav|baths, fumigating places for clothing, wash-house, and the removal of dust-bins, completed the new arrangements in the main prison. +DUMMY/LJ017-0282.wav|the seemingly most courageous was selected to lead the way. +DUMMY/LJ042-0140.wav|She said that, quote, immediately after coming to the United States, Lee changed. I did not know him as such a man in Russia, end quote. +DUMMY/LJ008-0070.wav|The sentences inflicted in front of Newgate were not limited to hanging. +DUMMY/LJ040-0116.wav|who was becoming quite a disciplinary problem for his mother, having struck her on at least one occasion. +DUMMY/LJ026-0026.wav|As in the liquefaction of gases, there is a "critical point" at which the substance under experiment is neither gaseous nor liquid. +DUMMY/LJ036-0052.wav|he looked so bad in his face, and his face was so distorted. Hole in his sleeve right here. +DUMMY/LJ016-0227.wav|whose business it was to provide the rope and do the pinioning, and who was paid a fee of five shillings. +DUMMY/LJ047-0082.wav|for an extended tour of Western European countries, the Soviet Union, Finland, and Poland. +DUMMY/LJ015-0067.wav|one at least of which, 'Love and Loyalty,' with Wallack in a leading part, achieved a certain success. +DUMMY/LJ010-0023.wav|and lodged men like Cole, Robson, and Redpath in Newgate. +DUMMY/LJ015-0015.wav|A more distressing case stands next on the criminal records -- +DUMMY/LJ007-0228.wav|In their tenth report +DUMMY/LJ016-0275.wav|As much as twenty-five pounds was paid for a first-floor front on this occasion. +DUMMY/LJ004-0127.wav|In one hundred jails, +DUMMY/LJ003-0032.wav|So valuable were they deemed, that as much as fifty guineas was offered to the keeper for the post. +DUMMY/LJ013-0007.wav|A not entirely novel kind of fraud, but one carried out on a larger scale than heretofore, +DUMMY/LJ046-0017.wav|rather, it has devoted its time and resources to those broader aspects of Presidential protection +DUMMY/LJ009-0082.wav|except the clergyman and the burglar, the former of whom is still rolled up at the bottom of the condemned pew, +DUMMY/LJ047-0088.wav|We did not request the State Department to include Oswald on a list which would have resulted in advising us of any application for a passport +DUMMY/LJ018-0225.wav|but after the funeral William Roupell produced another and a later will, +DUMMY/LJ027-0100.wav|are the thousands of cases of rudimentary or vestigial structures to be found. +DUMMY/LJ012-0223.wav|These facts were represented before the police magistrate +DUMMY/LJ010-0283.wav|and walking for choice through prickly gorse bushes. +DUMMY/LJ035-0051.wav|He saw a man walking away from him in the lunchroom. +DUMMY/LJ046-0212.wav|Threats against the President of the U.S., +DUMMY/LJ017-0020.wav|would probably have had the benefit in these days of the doubts raised at his trial. +DUMMY/LJ019-0305.wav|fourteen received less than eleven and more than six; +DUMMY/LJ006-0178.wav|and for years, felons who should have been sent beyond the seas. +DUMMY/LJ023-0123.wav|If these problems cannot be effectively solved within the Constitution, +DUMMY/LJ028-0219.wav|All the people of Babylon prostrated themselves before him, and, kissing his feet, rejoiced in his sovereignty, while happiness shone on their faces. +DUMMY/LJ017-0093.wav|and one of the many links in the long chain of evidence was an entry in a book of Palmer's to the effect that Quote. +DUMMY/LJ043-0157.wav|Answer: Yes. Question: And you said it was not a good idea to keep this book? Answer: Yes. +DUMMY/LJ016-0075.wav|It was for some time after this a constant practice to go up the chimneys in the hopes of escaping by the flue. +DUMMY/LJ042-0122.wav|Compulsory attendance at lectures and the sending of the entire shop collective (except me) to pick potatoes on a Sunday, at a state collective farm +DUMMY/LJ032-0155.wav|Fibers on Rifle +DUMMY/LJ004-0149.wav|The diet was limited to dry bread, which was of the best certainly, and a pound and a half in weight. +DUMMY/LJ028-0400.wav|The walls of Babylon were transformed into the sacred cities of Kerbela and Nejef. +DUMMY/LJ010-0034.wav|Thurtell and Hunt followed in the footsteps of Billings, Wood, and Catherine Hayes. +DUMMY/LJ037-0013.wav|About one hundred feet from the corner the police car pulled up alongside a man on the sidewalk. This man, dressed in a light-colored jacket, approached the car. +DUMMY/LJ049-0190.wav|It has been pointed out that the FBI, as our chief investigative agency, +DUMMY/LJ016-0064.wav|when he explained that he was running away from the gallows they left him the road clear. +DUMMY/LJ047-0199.wav|The Secret Service and the FBI differ +DUMMY/LJ021-0200.wav|we have avoided on the one hand the theory that business should and must be taken over into an all-embracing government. +DUMMY/LJ025-0023.wav|reduces the necessity of the existence in them of an alimentary cavity, or reservoir of food, +DUMMY/LJ017-0127.wav|But the chain of circumstantial evidence was strong enough to satisfy the jury, who agreed to their verdict in an hour. +DUMMY/LJ006-0265.wav|The untried might see their friends three times a week, the convicted only once. +DUMMY/LJ013-0063.wav|The other, remaining unclaimed for ten years, was transferred at the end of that time to the commissioners for the reduction of the National Debt. +DUMMY/LJ032-0236.wav|By checking the actual mailing dates of these issues and the time it usually takes to effect delivery to Dallas, +DUMMY/LJ014-0100.wav|In the back kitchen one of the detectives remarked that the cement between certain stones looked lighter than the rest, and on trying it with a knife, +DUMMY/LJ001-0161.wav|There seems to be no reason why ordinary paper should not be better made, +DUMMY/LJ006-0103.wav|The allowance of food was not illiberal, +DUMMY/LJ012-0023.wav|He was taken up when still in his teens for stealing a pocketbook, and was sentenced to transportation, but did not get beyond the hulks at Chatham. +DUMMY/LJ044-0030.wav|five of which were usually present at meetings which were held once a month. +DUMMY/LJ008-0038.wav|After hanging the usual time they were taken down, and the machine cleared away in half-an-hour. +DUMMY/LJ021-0043.wav|most of whom recognize the past evils in the banking system, in the sale of securities, in the deliberate encouragement of stock gambling, +DUMMY/LJ035-0037.wav|This door leads into a small vestibule, and another door leads from the vestibule into the second-floor lunchroom. +DUMMY/LJ009-0123.wav|Could any one, knowing the late situation of the kneeling men, looking as they do at the empty pew, +DUMMY/LJ019-0019.wav|to make a great archway, and to make the character of it more imposing than if it had been situated in the country, and had been an ordinary prison. +DUMMY/LJ049-0203.wav|So circumscribed, it could not maintain the esprit de corps or the necessary alertness for this unique and challenging responsibility. +DUMMY/LJ032-0092.wav|There is no post office box three zero zero one six in the New Orleans Post Office +DUMMY/LJ038-0063.wav|but that once he was subdued, no officer struck him. +DUMMY/LJ023-0129.wav|the only method which was clearly constitutional, and would at the same time carry out other much needed reforms, +DUMMY/LJ033-0011.wav|fiber, and paper analyses linking Oswald and the assassination weapon to this bag. +DUMMY/LJ038-0188.wav|two. Send the information as to what has happened to me to the Embassy. +DUMMY/LJ027-0128.wav|and even in that case should have maintained his ideal very inefficiently, inasmuch as only two limbs, instead of four, are represented? +DUMMY/LJ010-0244.wav|leveled a pistol at the Queen, and fired. He was so close to the carriage that the smoke of his pistol enveloped the face of Colonel Wylde, +DUMMY/LJ029-0172.wav|The two Dallas newspapers provided their readers with a steady stream of information and speculation about the trip, +DUMMY/LJ017-0088.wav|Quantities of antimony were found in the body after death. +DUMMY/LJ037-0172.wav|even though only four bullets were recovered. +DUMMY/LJ047-0196.wav|The Commission has considered carefully the question whether the FBI, +DUMMY/LJ047-0195.wav|Nonreferral of Oswald to the Secret Service. +DUMMY/LJ009-0151.wav|The atrocity of the murder no doubt attracted extraordinary attention to it. +DUMMY/LJ018-0168.wav|but proofs of Griffiths' guilt were at once apparent on entering his work-room. +DUMMY/LJ043-0015.wav|He also expressed considerable resentment at the help given to his wife by her Russian-American friends. +DUMMY/LJ049-0104.wav|Such a statute would cover the President and Vice President or, in the absence of a Vice President, the person next in order of succession. +DUMMY/LJ017-0147.wav|the protracted dissimulation, as with unshrinking, relentless wickedness the deadly work is carried on to the end. +DUMMY/LJ019-0024.wav|Nor must it be overlooked that this, the first model prison, although obtained at a considerable cost, became actually what its name implied. +DUMMY/LJ007-0093.wav|The matter was still further complicated at Newgate by the presence within the walls of sham lunatics. Some of those included in the category +DUMMY/LJ010-0080.wav|The Government obtained early and full information of the nefarious scheme. +DUMMY/LJ026-0119.wav|They may be stored, especially as fat, until needed for oxidation to supply energy. +DUMMY/LJ035-0188.wav|This was no earlier than twelve:thirty-seven p.m. and may have been later. +DUMMY/LJ037-0129.wav|When the man reached Jefferson, he turned right and headed west. +DUMMY/LJ006-0141.wav|The female wards-women were also given to intemperance. +DUMMY/LJ027-0012.wav|All have the same ultimate substance +DUMMY/LJ009-0264.wav|His face was visible to the whole crowd, and was fearful to behold. +DUMMY/LJ018-0091.wav|Wagner and Bateman, who had already been convicted of systematic forgery, and sentenced to transportation, but they had been released on ticket-of-leave +DUMMY/LJ016-0071.wav|Dissatisfied with this remuneration, he again took to the road, and tramped into Hampshire, +DUMMY/LJ012-0258.wav|a Mr. Gay of Goodge Street came to view the head, and immediately recognized it as that of a widowed sister, Hannah Brown, +DUMMY/LJ011-0016.wav|His countenance had an expression of most subdued resignation. +DUMMY/LJ001-0118.wav|In no characters is the contrast between the ugly and vulgar illegibility of the modern type +DUMMY/LJ046-0188.wav|In summary, at the time of the assassination +DUMMY/LJ017-0258.wav|Their trial followed at the Central Criminal Court. +DUMMY/LJ002-0263.wav|the Lord Steward of the Household, the steward and officers of the Marshalsea Court, and others. +DUMMY/LJ048-0120.wav|has been that provided by a requirement that he file interim and final reports on each advance assignment. +DUMMY/LJ048-0080.wav|to coordinate activities and to discuss problems of mutual interest. +DUMMY/LJ045-0067.wav|did not want her to drink, smoke or wear cosmetics and generally treated her with lack of respect in the presence of others. +DUMMY/LJ024-0121.wav|Even if an amendment were passed, and even if in the years to come it were to be ratified, +DUMMY/LJ035-0166.wav|While it was difficult to determine exactly when the police sealed off the building, +DUMMY/LJ007-0234.wav|slanting downwards from the top of the walls to the outside adjoining the slaughterhouses of Newgate market; and occasionally, in hot weather, +DUMMY/LJ016-0376.wav|literally walked over what, in case of conviction, would be their own graves. +DUMMY/LJ036-0069.wav|Instead of waiting there, Oswald apparently went as far away as he could and boarded the first Oak Cliff bus which came along +DUMMY/LJ033-0125.wav|Frazier doubted whether the bag that Oswald carried was as wide as the bag found on the sixth floor, although Mrs. Randle testified +DUMMY/LJ005-0244.wav|which dealt with jails of all classes, as well as transportation. +DUMMY/LJ041-0110.wav|canceling the suspension of a twenty-day sentence that Oswald had received in an earlier court-martial +DUMMY/LJ037-0055.wav|Detective L. C. Graves, who had been with Mrs. Markham before the lineup +DUMMY/LJ035-0209.wav|at twelve:thirty p.m +DUMMY/LJ016-0060.wav|Williams concealed himself behind a chimney till she had re-entered her garret, +DUMMY/LJ045-0075.wav|She was, quote, sorry that I had not married him (the Russian boyfriend) instead, that it would have been much easier for me, end quote. +DUMMY/LJ044-0151.wav|As a result of that conversation, Marina Oswald wrote the Soviet Embassy in Washington concerning a request she had first made +DUMMY/LJ001-0089.wav|but his letters, though uninteresting and poor, are not nearly so gross and vulgar as those of either the Italian or the Frenchman. +DUMMY/LJ012-0045.wav|he put another lady, with whom he was on intimate terms. +DUMMY/LJ036-0078.wav|claimed that about fifteen minutes after the assassination he saw a man, whom he later identified as Oswald, +DUMMY/LJ008-0280.wav|The absurdity of the system, its irregularity and cruelty, were fully touched upon by the inspectors of prisons in their first report. +DUMMY/LJ016-0392.wav|Jeffrey, who murdered his own child, an infant of six, by hanging him in a cellar in Seven Dials, +DUMMY/LJ033-0187.wav|However, the complete identity of characteristics between the paper and tape in the bag found on the sixth floor +DUMMY/LJ009-0241.wav|They were taken back to jail, and were again brought out in the afternoon, by which time fresh and stronger ropes had been procured, +DUMMY/LJ013-0153.wav|The gravest crimes continued at intervals to inspire the town with horror, and concentrate public attention upon the jail of Newgate, +DUMMY/LJ016-0116.wav|The doors, through incomplete knowledge of prison needs, were not, as now, sheeted with iron. +DUMMY/LJ050-0145.wav|and the Commission recommends that these and the other measures suggested by the Commission be pursued vigorously by Secret Service. +DUMMY/LJ006-0286.wav|Thus, when the inspectors visited there were twenty-three strangers, and only twenty-eight prisoners. +DUMMY/LJ046-0100.wav|The assassination of President Kennedy demands an examination of the protective measures employed to safe guard him +DUMMY/LJ049-0220.wav|that the determination of whether or not there should be a relocation of responsibilities and functions should be left to the Executive and the Congress, +DUMMY/LJ032-0134.wav|quote, near the firing end of the barrel about three inches under the woodstock when I took the woodstock loose, end quote, +DUMMY/LJ047-0038.wav|The file reflected the Department's determination that Oswald had not expatriated himself. +DUMMY/LJ032-0143.wav|The print's positive identity as having been lifted from the rifle +DUMMY/LJ039-0200.wav|and one of these agents, Robert A. Frazier, +DUMMY/LJ046-0086.wav|is dependent to a considerable extent upon the degree of contact with the general public desired by the President. +DUMMY/LJ015-0072.wav|He had a pleasant address, showed good business aptitudes, and quickly acquired the approval of his superiors. +DUMMY/LJ029-0120.wav|where the crowds would be able to be controlled for a great distance, and we figured that the largest crowds would be downtown, which they were, +DUMMY/LJ043-0161.wav|Answer: It never came to me, myself, to ask him that question. +DUMMY/LJ008-0160.wav|some, especially of low stature, found it difficult to remain standing, and several, although held up for some time by the men nearest them, +DUMMY/LJ037-0229.wav|white jacket, end quote, +DUMMY/LJ001-0148.wav|Here once more the almost invariable modern practice is in opposition to a natural sense of proportion. +DUMMY/LJ014-0338.wav|These bankers, wishing for more specific information, +DUMMY/LJ041-0159.wav|I think you could sit down and argue with him for a number of years, and I don't think you could have changed his mind on that unless you knew why he believed it in the first place. +DUMMY/LJ034-0084.wav|became apparent on December two, nineteen sixty-three, when an employee, Frankie Kaiser, +DUMMY/LJ016-0403.wav|His effrontery was only outdone by his cool contempt for the consolations of religion. +DUMMY/LJ030-0051.wav|The President's automobile was specially designed nineteen sixty-one Lincoln convertible +DUMMY/LJ017-0173.wav|Catherine Wilson was a female poisoner who did business wholesale. +DUMMY/LJ018-0377.wav|Conviction was obtained through the evidence of the steward and two of the least culpable of the crew. +DUMMY/LJ034-0109.wav|one-sixty to one-seventy pounds. +DUMMY/LJ044-0049.wav|Bringuier, who seemed to be familiar with many anti-Castro activities in New Orleans, was not aware of any such incident. +DUMMY/LJ032-0216.wav|He concluded that the negative of Exhibit Number one thirty-three B was exposed in Oswald's Imperial Reflex camera to the exclusion of all other cameras. +DUMMY/LJ040-0231.wav|Lee Oswald never received that help. +DUMMY/LJ002-0131.wav|three times the amount would be expended. +DUMMY/LJ009-0177.wav|With the amelioration of the criminal code, other cruel concomitants of execution also disappeared. +DUMMY/LJ047-0129.wav|FBI informants in the New Orleans area, familiar with pro-Castro or Communist Party activity there, +DUMMY/LJ014-0236.wav|Barthelemy was generally supposed to have been a secret agent of the French police. +DUMMY/LJ019-0030.wav|Major, afterwards Sir Joshua Jebb, +DUMMY/LJ046-0095.wav|because these precautions reduce the President's privacy and the access to him of the people of the country. +DUMMY/LJ028-0255.wav|In five twenty-nine Cyrus died. +DUMMY/LJ008-0154.wav|Conviction followed mainly on his testimony; but the two men, especially Holloway, stoutly maintained their innocence to the last. +DUMMY/LJ016-0120.wav|This was considered safer than intrusting him with keys. +DUMMY/LJ009-0303.wav|Then came the cost of the gallows, +DUMMY/LJ009-0014.wav|It cannot be denied, however, that the ordinary's treatment was somewhat unfeeling, and in proof thereof +DUMMY/LJ002-0306.wav|looked forward to his liberty, he was still delayed until he had paid a new debt arising, quote, +DUMMY/LJ009-0296.wav|It was not always easy to hire a hangman. +DUMMY/LJ043-0003.wav|Background and Possible Motives, Part four. +DUMMY/LJ029-0097.wav|and on their return to Dallas drove over the route which Sorrels believed best suited for the proposed motorcade. +DUMMY/LJ010-0103.wav|Nine of the latter were captured, with all the war material, cutlasses, pistols, hand-grenades, and ammunition. +DUMMY/LJ019-0349.wav|and they met with the approval of his professional adviser, the surveyor-general of prisons. +DUMMY/LJ032-0174.wav|but Mary Bledsoe, a former landlady of Oswald, saw him on a bus approximately ten minutes after the assassination +DUMMY/LJ029-0141.wav|from Main across Elm to the access road to Stemmons Freeway and the Dallas-Fort Worth Turnpike. +DUMMY/LJ046-0145.wav|The Secret Service responded, quote, +DUMMY/LJ015-0228.wav|Pierce boldly stepped in, found the cupboard unlocked; he removed the key, handed it to Agar outside, +DUMMY/LJ045-0107.wav|She testified that he responded, quote, As you wish. If you don't want me to come, I won't, end quote. +DUMMY/LJ012-0241.wav|the memory of which is still fresh in the minds of Londoners, although half a century has passed since it was committed. +DUMMY/LJ021-0052.wav|The average person almost always loses. +DUMMY/LJ023-0106.wav|that something in the Constitution has compelled them regretfully to thwart the will of the people. +DUMMY/LJ002-0055.wav|This arrangement was, however, modified after eighteen eleven, and the chapel yard was allotted to misdemeanants and prisoners awaiting trial. +DUMMY/LJ016-0031.wav|Yet from the circumstantial account given by Williams after recapture, there can be little doubt that he got away as will be described. +DUMMY/LJ016-0084.wav|the fugitive, uncomfortably ensconced in the flue, came down of his own accord, like Colonel Colt's raccoon. +DUMMY/LJ050-0068.wav|According to Chief Rowley, by mid-June nineteen sixty-four, +DUMMY/LJ006-0225.wav|If any man presumed to turn in too early +DUMMY/LJ006-0301.wav|This woman was really his servant, employed to help in cleaning, and taken on whenever there was extra work to be done. +DUMMY/LJ006-0154.wav|Nothing was more prominently brought out by the inspectors than the inefficiency of the governor at that time, Mr. Cope. +DUMMY/LJ010-0049.wav|Their perpetrators were charged with high treason, but met with merciful clemency as irresponsible beings. +DUMMY/LJ045-0202.wav|He had great difficulty in establishing meaningful relations with other people. Except for his family he was completely alone. +DUMMY/LJ049-0142.wav|by defining responsibilities clearly and overseeing their execution. +DUMMY/LJ029-0028.wav|Everyone agreed that, if there was sufficient time, a motorcade through downtown Dallas would be the best way for the people to see their President. +DUMMY/LJ016-0118.wav|Through the aperture he crept out on to the landing at the dead of night, and so down into the central space of the building. +DUMMY/LJ038-0094.wav|While Oswald's untrue statements during interrogation were not considered items of positive proof +DUMMY/LJ036-0120.wav|unloaded at twelve:fifteen p.m., +DUMMY/LJ013-0193.wav|Courvoisier, when put on his trial, pleaded not guilty; +DUMMY/LJ031-0160.wav|The police were cautioned to prevent picture taking. +DUMMY/LJ028-0234.wav|Cyrus was now reduced to great perplexity, as time went on and he made no progress against the place. +DUMMY/LJ023-0117.wav|I want -- as all Americans want -- an independent judiciary as proposed by the framers of the Constitution. +DUMMY/LJ042-0074.wav|Mr. Richard E. Snyder, then Second Secretary and senior consular official at the Embassy, +DUMMY/LJ012-0063.wav|By-and-by the turnkeys, sobered by their loss, returned to Newgate alone, and pleaded in excuse that they had been drugged. +DUMMY/LJ028-0256.wav|During the reigns of the two following Persian kings Babylon was slowly regaining its independence, +DUMMY/LJ019-0306.wav|while twenty-seven received less than six prisoners, and were in some instances absolutely tenantless. +DUMMY/LJ009-0089.wav|childless parents, widows and helpless orphans, broken and contrite hearts, and death tomorrow morning for the benefit of society. +DUMMY/LJ035-0170.wav|He then returned to the corner of Elm and Houston where he met a sergeant who instructed him to find out the name of the building. +DUMMY/LJ008-0276.wav|Neither in town or country were cases entirely taken on their own merits. +DUMMY/LJ016-0051.wav|Clambering along the roof, +DUMMY/LJ037-0162.wav|Consecutive bullets fired from the revolver by the FBI experts could not be identified as having been fired from that revolver. +DUMMY/LJ010-0179.wav|The Queen saw this second movement, and stooped to avoid the shot; +DUMMY/LJ001-0129.wav|the lateral compression of his letters renders necessary. +DUMMY/LJ012-0013.wav|At the trial the indictment charging them with felony could not be sustained, but they were found guilty of conspiracy and assault. +DUMMY/LJ028-0233.wav|for when they saw Cyrus conquering nation after nation, they were convinced that he would never stop, and their turn would come at last. +DUMMY/LJ031-0229.wav|The nature and characteristics of this neck wound are discussed fully in the next chapter. +DUMMY/LJ030-0165.wav|which caused her to look to her right. +DUMMY/LJ010-0230.wav|who was arrested in the very act, just as he had fired one shot. +DUMMY/LJ027-0064.wav|All vertebrates, and none other, have two cavities, +DUMMY/LJ001-0039.wav|The Roman type of all these printers is similar in character, +DUMMY/LJ026-0166.wav|back to starch usable as food and the comparison of the green plant and the animal would be complete. +DUMMY/LJ050-0213.wav|field agents supplement those on the detail, particularly when the President is traveling. +DUMMY/LJ005-0290.wav|Instances rarely occur in which the borough jails admit of any proper classification of the prisoners. +DUMMY/LJ032-0200.wav|One Sunday, while his wife was hanging diapers, Oswald asked her to take a picture of him holding a rifle, a pistol +DUMMY/LJ004-0014.wav|filthiness, severity, or neglect; many new dungeons had aggravated the evils against which his sagacity could not but remonstrate; +DUMMY/LJ045-0176.wav|The next morning he left for work before anyone else arose. +DUMMY/LJ045-0134.wav|Hosty had come to the Paine residence on November one and five, nineteen sixty-three, +DUMMY/LJ006-0081.wav|The wardsman still exacted dues, of which more directly, +DUMMY/LJ026-0055.wav|These materials are used by the animal in the manufacture of new protoplasm to take the place of that which has been used up. +DUMMY/LJ019-0177.wav|A scheme for heating the whole prison with hot-water pipes, after the system now generally adopted elsewhere, was considered, +DUMMY/LJ038-0180.wav|admissions and other statements made to Marina Oswald by Oswald concerning the shooting. Note left by Oswald. +DUMMY/LJ008-0060.wav|projecting somewhat beyond the range of the iron bar, and swinging upon hinges affixed to the wall. +DUMMY/LJ027-0152.wav|for instance, were separately created, additional prongs were successively added to their antlers; and yet that, +DUMMY/LJ001-0150.wav|the page so lay on the paper that there was more space allowed to the bottom and fore margin than to the top and back of the paper, +DUMMY/LJ044-0128.wav|there are a number of organizations, including possibly Fair Play, which are of a very broad character, +DUMMY/LJ043-0038.wav|nor his mother were objects of his affection, quote, but only examples of workers in the U.S., end quote. +DUMMY/LJ024-0029.wav|There is general approval so far as the lower federal courts are concerned. +DUMMY/LJ017-0040.wav|The trial of Catherine Winsor, the baby farmer, +DUMMY/LJ017-0146.wav|He had all the characteristics of the poisoner -- the calm deliberation, +DUMMY/LJ028-0369.wav|they intentionally mentioned a most unfavorable hour. +DUMMY/LJ003-0243.wav|Ironing was still the rule, not only for the convicted, but for those charged with felonies; only the misdemeanants escaped. +DUMMY/LJ022-0050.wav|and every day that passes offers more chances to work for those who want to work. +DUMMY/LJ048-0085.wav|the Commission believes that the liaison between all Federal agencies responsible for Presidential protection should be improved. +DUMMY/LJ046-0171.wav|This might involve a personal interview or interviews with members of the person's household. Wherever possible, +DUMMY/LJ008-0243.wav|Threading his way among these itinerant vendors was seen the meek-faced deliverer of tracts, the man of good intentions, now bonneted, +DUMMY/LJ019-0213.wav|Now for the first time the Tuffnell estate in Holloway was mentioned. +DUMMY/LJ029-0030.wav|once we got San Antonio moved from Friday to Thursday afternoon, where that was his initial stop in Texas, +DUMMY/LJ003-0117.wav|A bribe to the judge was certain to secure acquittal, and the neglect of the formality was as certainly followed by condemnation. +DUMMY/LJ002-0127.wav|and the aggregate amount of debts sued for was eighty-one thousand, seven hundred ninety-one pounds. +DUMMY/LJ029-0054.wav|Upon his arrival in Dallas on November twelve +DUMMY/LJ014-0178.wav|His offense was the murder of Richard Cope, +DUMMY/LJ021-0005.wav|Recently the most notable public questions that have concerned us all +DUMMY/LJ021-0004.wav|Tonight I continue that report, though, because of the shortness of time, I must defer a number of subjects to a later date. +DUMMY/LJ003-0187.wav|I have kept till the last that part of the prison which was usually the last resting-place of so many. +DUMMY/LJ009-0045.wav|he steps boldly with head upright, looks to the women's gallery, and smiles. His intention is to pass for a brave fellow, +DUMMY/LJ024-0061.wav|Until my first term practically every President of the United States has appointed at least one member of the Supreme Court. +DUMMY/LJ046-0251.wav|to the other agencies. No specific guidance was provided. +DUMMY/LJ027-0162.wav|The fish retains permanently this form, but the frog passes on. +DUMMY/LJ016-0226.wav|To this curious etiquette was due the appointment of an official whose office has long since disappeared, "the yeoman of the halter," +DUMMY/LJ036-0090.wav|If Craig saw Oswald that afternoon, he saw him through the glass windows of the office. +DUMMY/LJ047-0211.wav|would have added up to pointing out a pretty bad individual, and I think that, together, had we known that he had a vantage point +DUMMY/LJ019-0217.wav|The necessary preliminaries took some time, but the contracts for the new building were completed in eighteen forty-nine, when the works were commenced. +DUMMY/LJ017-0107.wav|After Cook's death his stepfather, who was much attached to him, came to Rugeley. +DUMMY/LJ036-0185.wav|He hurried to his room and stayed no longer than three or four minutes. +DUMMY/LJ045-0115.wav|and he thought that he loses jobs because the FBI visits the place of his employment, end quote. +DUMMY/LJ009-0150.wav|So far as I can find, Courvoisier was the last condemned criminal who was thus exhibited to a crowd of morbidly curious spectators. +DUMMY/LJ042-0134.wav|The psychological effects of that change must have been highly unsettling. It should be remembered +DUMMY/LJ005-0111.wav|In a county prison adjoining the metropolis, the separation between the male and female quarters was supposed to be accomplished by the erection of an iron railing; +DUMMY/LJ045-0032.wav|She testified that after his return from Mexico Oswald, quote, changed for the better. +DUMMY/LJ001-0163.wav|e.g. the cheap paper should not sacrifice toughness and durability to a smooth and white surface, +DUMMY/LJ007-0080.wav|These powers were not invariably put in force, and there were in consequence many unhappy lunatics in Newgate and other jails, +DUMMY/LJ030-0215.wav|the car lurched forward, causing him to lose his footing. He ran three or four steps, regained his position and mounted the car. +DUMMY/LJ014-0161.wav|The actual execution, which took place at another prison than Newgate, is rather beyond the scope of this work. +DUMMY/LJ012-0039.wav|In the end the injured bootmaker agreed to buy back his stolen stock +DUMMY/LJ025-0078.wav|not only ready cooked but ready digested, and the alimentary canal, become superfluous, has disappeared, and, again, +DUMMY/LJ048-0260.wav|which provides, quote, Liquor, use of +DUMMY/LJ025-0076.wav|Many animals of even complex structure which live parasitically within others are wholly devoid of an alimentary cavity. +DUMMY/LJ009-0136.wav|that they reluctantly agreed to open the gallery which had formerly been occupied by strangers on these occasions. +DUMMY/LJ037-0257.wav|Scoggins thought it was lighter. +DUMMY/LJ047-0055.wav|It does not preclude the agent in charge of the case from reopening it if he feels that further work should be done. +DUMMY/LJ050-0229.wav|as soon as it can be justified. The Congress has often stressed that it will support any reasonable request for funds for the protection of the President. +DUMMY/LJ016-0156.wav|at no elevation affords the only drop, strangulation would seldom supervene but for the resolution of the miserable felo de se. +DUMMY/LJ030-0230.wav|but Hickey kept the automatic weapon ready as the car raced to the hospital. +DUMMY/LJ044-0204.wav|prominently reported President Kennedy as having, quote, all but invited the Cuban people today to overthrow Fidel Castro's Communist regime +DUMMY/LJ032-0213.wav|The negative of that picture, Commission Exhibit Number one thirty-three B, was found among Oswald's possessions. +DUMMY/LJ018-0137.wav|None of the stolen paper was glazed, and this was an important clue to the subsequent discovery of the crime. +DUMMY/LJ045-0151.wav|He then arrived on Thursday, November twenty-one, nineteen sixty-three, end quote. +DUMMY/LJ006-0302.wav|The governor had a great dislike, he said, to seeing strangers in his house. +DUMMY/LJ005-0213.wav|Newgate through all these years continued a bye-word with the Society. +DUMMY/LJ005-0144.wav|Yet to Clerkenwell were now committed the juveniles, and all who were inexperienced in crime. +DUMMY/LJ026-0036.wav|and then a balance must be struck and the doubtful form placed in the kingdom with which it has, on the whole, most points in common. +DUMMY/LJ048-0180.wav|This justification of the Secret Service's standing policy is not persuasive. +DUMMY/LJ008-0196.wav|High prices were asked and paid for front seats or good standing room. As much as five pounds was given for the attic story +DUMMY/LJ045-0240.wav|Many factors were undoubtedly involved in Oswald's motivation for the assassination, and the Commission does not believe +DUMMY/LJ018-0347.wav|and the circumstances which led to detection were scarcely novel proofs of the old adage that murder will out. +DUMMY/LJ028-0355.wav|he gave him likewise the government of Babylon for his life, free from tribute, and he also granted him many other favors. +DUMMY/LJ025-0120.wav|which are not only similar in animals and in plants respectively, +DUMMY/LJ017-0206.wav|Although murder by insidious methods had become more common, cases where violence of the most deadly and determined kind was offered +DUMMY/LJ016-0181.wav|Calcraft was born at Baddow, in Essex, in eighteen hundred; +DUMMY/LJ036-0156.wav|He was in error, however. +DUMMY/LJ014-0138.wav|Mrs. Manning became still more violent, shouting, "No, no, I will not stand it! You ought to be ashamed of yourselves!" +DUMMY/LJ045-0002.wav|Chapter seven. Lee Harvey Oswald: Background and Possible Motives, Part six. +DUMMY/LJ037-0250.wav|said that she may have seen the gray zipper jacket but she was not certain. +DUMMY/LJ030-0243.wav|but I had no time to speculate as to its origin because Agent Youngblood turned in a flash, immediately after the first explosion, +DUMMY/LJ001-0137.wav|But another point which they should attend to they almost always disregard; +DUMMY/LJ001-0005.wav|the invention of movable metal letters in the middle of the fifteenth century may justly be considered as the invention of the art of printing. +DUMMY/LJ032-0231.wav|and other details in the picture. +DUMMY/LJ010-0296.wav|At one session of the Old Bailey, in eighteen twenty-one, no less than thirty-five true bills were found for passing forged notes. +DUMMY/LJ016-0092.wav|He stepped in at a garret window, coolly walked downstairs, and entered the bar. +DUMMY/LJ036-0011.wav|(three) evidence establishing the ownership of the murder weapon, +DUMMY/LJ006-0251.wav|but the presence and authority of the governor himself became indispensable. +DUMMY/LJ005-0239.wav|The Prison Society did not relax its efforts as time passed, but its leading members had other and more pressing claims upon their energies. +DUMMY/LJ031-0115.wav|From approximately four p.m. to four:fifty p.m. on November twenty-two, +DUMMY/LJ032-0013.wav|and killing Patrolman Tippit, +DUMMY/LJ001-0018.wav|The first books were printed in black letter, i.e. the letter which was a Gothic development of the ancient Roman character, +DUMMY/LJ050-0258.wav|Much of the Secret Service work requires the development and use of highly sophisticated equipment, +DUMMY/LJ044-0207.wav|the economic embargo against that country, and the general policy of the United States with regard to Cuba. +DUMMY/LJ035-0107.wav|testified that he took the west elevator to the first floor after hearing a noise which sounded like a backfire. +DUMMY/LJ038-0068.wav|cursing a little bit and hollering police brutality, end quote. +DUMMY/LJ022-0102.wav|the creation of new governmental machinery for supervising this work. +DUMMY/LJ021-0081.wav|with the confidence that we are definitely rebuilding our political and economic system on the lines laid down by the New Deal +DUMMY/LJ047-0060.wav|Fain had been assigned to see Marina Oswald at an appropriate time. +DUMMY/LJ011-0087.wav|Montgomery was duly sentenced to death, but he preferred suicide to the gallows. After sentence his demeanor was serious yet firm. +DUMMY/LJ011-0212.wav|then a maid and heir apparent unto her father, for the sake of the lucre of her substance; and for having afterwards unlawfully and against her will +DUMMY/LJ017-0038.wav|That the administration of justice should never be interfered with by local prejudice or local feeling +DUMMY/LJ039-0092.wav|to one hundred ninety point eight feet was, quote, very easy, end quote, and the shot which struck the President in the head +DUMMY/LJ005-0142.wav|occupying barely three hundred ninety feet lineal +DUMMY/LJ015-0096.wav|whether representing real or fictitious shares does not appear; but they were certificates connected in some way with Robson's long practiced frauds +DUMMY/LJ023-0050.wav|overlook the simple fact that the President, as Chief Executive, is himself one of the three horses. +DUMMY/LJ007-0024.wav|who supped royally on the supplies provided from outside, and kept it up till ten or eleven o'clock. +DUMMY/LJ050-0255.wav|This no longer appears to be the case. +DUMMY/LJ022-0049.wav|The simple fact is that many million more people have private work today than two years ago today or one year ago today, +DUMMY/LJ007-0227.wav|The inspectors were not to be deterred, however, by any opposition from the earnest discharge of their functions, and continued to report against Newgate. +DUMMY/LJ007-0095.wav|Prisoners indeed were known to boast that they had saved their necks by feigning insanity. +DUMMY/LJ042-0217.wav|I wonder what would happen it somebody was to stand up and say he was utterly opposed not only to the governments, but to the people, +DUMMY/LJ010-0064.wav|upon "the old man and the old woman," as they styled the Mansion House and the Bank of England. +DUMMY/LJ013-0043.wav|This person made much of Wallace, encouraged his attentions to his daughter, +DUMMY/LJ046-0193.wav|and twelve to fifteen of these cases as highly dangerous risks. +DUMMY/LJ031-0220.wav|During the autopsy examination, Federal agents brought the surgeons three pieces of bone recovered from Elm Street and the Presidential automobile. +DUMMY/LJ018-0162.wav|he had been introduced to a man named Griffiths, an engraver and copper-plate printer. +DUMMY/LJ040-0011.wav|For a motive that appears incomprehensible to other men may be the moving force of a man whose view of the world has been twisted, +DUMMY/LJ046-0061.wav|John F. Kennedy's journey to Texas in November nineteen sixty-three was in this tradition. +DUMMY/LJ005-0026.wav|educates him under such a system for nothing, while the virtuous simpleton who is on the other side of the wall is paying by the quarter for these attainments." +DUMMY/LJ008-0229.wav|Above the murmur and tumult of that noisy assembly, the lowing and bleating of cattle as they were driven into the stalls and pens of Smithfield +DUMMY/LJ009-0293.wav|While Calcraft was in office other aspirants to fame appeared in the field. +DUMMY/LJ042-0026.wav|Whatever Oswald's reasons for going to the Soviet Union might have been, however, there can be little doubt that his desire to go was quite strong. +DUMMY/LJ044-0007.wav|that he was involved in any conspiracy, his political activities do provide insight into certain aspects of Oswald's character +DUMMY/LJ023-0082.wav|In the last four years the sound rule of giving statutes the benefit of all reasonable doubt has been cast aside. +DUMMY/LJ046-0097.wav|Evaluation Of Presidential Protection At The Time Of The Assassination Of President Kennedy +DUMMY/LJ029-0207.wav|On November twenty-one there appeared on the streets of Dallas the anonymous handbill mentioned above. +DUMMY/LJ014-0298.wav|Strange to say, a second suicide occurred in Newgate the same night, +DUMMY/LJ046-0040.wav|In this role he appears to the American people, in the words of William Howard Taft, as, quote, +DUMMY/LJ040-0163.wav|He also recommended that Mrs. Oswald seek, quote, psychotherapeutic guidance through contact with a family agency, end quote. +DUMMY/LJ050-0171.wav|but it seems to warrant further study before each agency becomes irrevocably committed to separate action. +DUMMY/LJ003-0283.wav|It was not strange, therefore, that the inmates of Newgate should turn their unoccupied brains and idle hands to all manner of mischief; +DUMMY/LJ004-0227.wav|Mr. Brown also, much to his own credit, brought about the abandonment of the practice of ironing all prisoners as a matter of course. +DUMMY/LJ008-0096.wav|Pieman and ballad-monger did their usual roaring trade amidst the dense throng. +DUMMY/LJ028-0488.wav|They may best be described by means of the accompanying diagram representing a cross section. +DUMMY/LJ003-0270.wav|Owing to the facility of intercourse between inside and outside, many crimes were doubtless hatched in Newgate. +DUMMY/LJ022-0146.wav|No reasonable person wants to abandon our present gains +DUMMY/LJ025-0170.wav|for it is only the green parts of the plant which, under the influence of sunlight, have the marvelous power of decomposing carbonic acid, +DUMMY/LJ027-0061.wav|in order to enclose and protect a similar enlargement of the nervous center, +DUMMY/LJ030-0206.wav|on the turn at the intersection of Houston and Elm Streets and then proceeded at a rate of twelve to fifteen miles per hour +DUMMY/LJ032-0128.wav|or a determination that the print was not identical with the prints of people. +DUMMY/LJ005-0103.wav|often highly damnatory, which were embodied in its annual reports. +DUMMY/LJ016-0303.wav|It was preposterous to claim for such a scene as this that it conveyed any great moral lesson, or had any deterring influence. +DUMMY/LJ014-0290.wav|In Giltspur Street Compter, where he was first lodged +DUMMY/LJ029-0109.wav|Representatives of the local host committee and the White House staff were advised by the Secret Service of the actual route on the afternoon of November eighteen. +DUMMY/LJ014-0327.wav|The owners of the latter would then issue a second set of warrants on these goods, in total ignorance of the fact that they were already pledged. +DUMMY/LJ028-0037.wav|It was about twenty-two fifty B.C., when the great Hammurabi made it his capital, that it became the chief city of Babylonia. +DUMMY/LJ012-0071.wav|which presently followed, and again found himself an inmate of Newgate, waiting trial as a receiver and a prison-breaker. +DUMMY/LJ008-0240.wav|Then followed sounds more distinct and audible, in which ginger-beer, pies, fried fish, sandwiches, and fruit +DUMMY/LJ033-0034.wav|Oswald never asked Mrs. Paine about the use of curtain rods, +DUMMY/LJ011-0255.wav|Howard swore he would finish him, but proved the weaker of the two, and Mullay got him down on the floor. +DUMMY/LJ002-0087.wav|a circular aperture had been let in to give ventilation and secure a free current of air. +DUMMY/LJ004-0018.wav|generally through the personal activity of influential and benevolent local magnates, but the true principles of prison construction +DUMMY/LJ028-0310.wav|one at the Belian, the other at the Cissian gates; for I expect that, on account of my successes, +DUMMY/LJ031-0152.wav|Agents Kellerman and Hill telephoned the head of the White House detail, Gerald A. Behn, to advise him of the assassination. +DUMMY/LJ040-0107.wav|but apparently was not able to spend as much time with them as he would have liked, because of the age gaps of five and seven years, +DUMMY/LJ007-0187.wav|the neglect of the condemned convicts, the filthy condition of the wards, the insufficiency of bedding and clothing, +DUMMY/LJ015-0120.wav|Robson was found guilty, and sentenced to two terms of transportation, one for twenty and one for fourteen years. +DUMMY/LJ028-0490.wav|He would have had to cross a deep moat, to scale a wall of burned bricks about twenty feet in thickness and perhaps three times as high, +DUMMY/LJ016-0302.wav|He, after failing once to cut the rope, made a second attempt more successfully, and the body of Müller disappeared from view. +DUMMY/LJ047-0190.wav|Question: Did you take any action on this case between November five and November twenty-two? Answer: No, sir. +DUMMY/LJ018-0026.wav|showed how greatly the publicity and wide dissemination of the news regarding murder facilitate the detection of crime. +DUMMY/LJ016-0151.wav|prison suicides seldom do, except as regards ingenuity and determination. +DUMMY/LJ008-0132.wav|Over the handkerchief around his brows he placed a white cap, the new hat being on top of all. +DUMMY/LJ019-0162.wav|Other changes affecting the condition and proper treatment of prisoners were not made until the inspector had urged and recommended them. +DUMMY/LJ012-0012.wav|All three men were committed for trial, although Edwards wished to exculpate the others as having only acted under his order. +DUMMY/LJ045-0144.wav|On the other hand, the concern he expressed about the FBI may have been just another story to support the objective he sought in his letter. +DUMMY/LJ003-0260.wav|Another frightful consequence of this indiscriminate admission was the influx of numbers of abandoned women, +DUMMY/LJ038-0178.wav|(two) photographs found among Oswald's possessions after the assassination of President Kennedy, +DUMMY/LJ006-0120.wav|by drawing briefs and petitions for his fellows. There was a recognized charge of five shillings per brief, +DUMMY/LJ023-0011.wav|Today's recovery proves how right that policy was. +DUMMY/LJ035-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ019-0121.wav|The hope of reward might do much, but no system of penal discipline is complete unless it can also count upon the fear of punishment. +DUMMY/LJ015-0060.wav|will long be remembered both within and without the commercial world. +DUMMY/LJ006-0054.wav|But incredible as it may appear, the authorities of Newgate declined to avail themselves of the advantages offered them, +DUMMY/LJ036-0148.wav|After a review of these inconsistencies in his testimony before the Commission, Whaley was interviewed again in Dallas. +DUMMY/LJ005-0291.wav|In some large towns, as at Berwick on Tweed, Southampton, and Southwark, they (the prisons) are in a very discreditable condition. +DUMMY/LJ023-0017.wav|that we could not afford to take the risk of another great depression. +DUMMY/LJ043-0151.wav|that she became quite angry when Oswald told her what he had done, and that she made him promise never to repeat such a performance. +DUMMY/LJ039-0133.wav|Oswald's Rifle Practice Outside the Marines +DUMMY/LJ025-0034.wav|Animals further needed muscles for locomotion and nerves for sensibility. +DUMMY/LJ003-0054.wav|An imperfect attempt at classification was, however, made in eighteen twelve, and a yard was as far as possible set apart for the untried, +DUMMY/LJ002-0337.wav|could not be made even to look clean while there was no obligation of personal cleanliness on individuals, who often came into the prison in filthy rags. +DUMMY/LJ016-0091.wav|This was a public-house. +DUMMY/LJ028-0423.wav|In eighteen twelve, James Claudius Rich, the British Resident at Baghdad, made the first complete examination of the ruins. +DUMMY/LJ030-0070.wav|Motorcycles. -- Four motorcycles, two on each side, flanked the rear of the Presidential car. +DUMMY/LJ050-0150.wav|Its present manual filing system is obsolete; +DUMMY/LJ030-0024.wav|In Dallas the rain had stopped, and by midmorning a gloomy overcast sky had given way to the bright sunshine that greeted the Presidential party +DUMMY/LJ009-0185.wav|after the post mortem had been carried out, the corpse was exposed to view in a first-floor room. +DUMMY/LJ019-0113.wav|The scheme was no doubt fascinating, and in many respects feasible; +DUMMY/LJ042-0210.wav|A suggestion that Oswald hated more than just capitalism and communism +DUMMY/LJ016-0359.wav|Private executions at their first introduction were not popular with the Newgate officials, and for intelligible reasons. +DUMMY/LJ049-0032.wav|had no special design or equipment which would have permitted the Secret Service agent riding in the driver's compartment +DUMMY/LJ018-0321.wav|An increase of policemen on duty sufficed to prevent any attempt of this kind. +DUMMY/LJ038-0298.wav|Oswald could have taken any one of several different buses to Walker's house or to a point near the railroad tracks where he may have concealed the rifle. +DUMMY/LJ010-0137.wav|Davidson, who was the only one who seemed to realize his awful situation, listened patiently and with thankfulness to the chaplain, +DUMMY/LJ010-0131.wav|Come, old cock-of-wax, it will soon be over. +DUMMY/LJ037-0170.wav|But he agreed that because the other three bullets were mutilated, he could not determine if they had been fired from the same weapon as the test bullets. +DUMMY/LJ014-0313.wav|In this way he obtained vast sums from several firms, and one to which he was indebted upwards of fifty thousand pounds subsequently stopped payment. +DUMMY/LJ004-0052.wav|which Howard had eulogized some forty years before. +DUMMY/LJ045-0215.wav|and may have told his wife the things he did on the evening of November twenty-one, +DUMMY/LJ028-0450.wav|but the walls of Babylon have so long been used to supply bricks to the builders of the neighboring cities that only their bases remain. +DUMMY/LJ029-0077.wav|One building, Market Hall, was unavailable for November twenty-two. +DUMMY/LJ033-0178.wav|This was done as an investigatory aid +DUMMY/LJ013-0244.wav|Just at this moment, however, the searchers found concealed under two trusses of hay a woman's headless and dismembered trunk. +DUMMY/LJ031-0196.wav|Detailed security arrangements had been made by radio from the President's plane on the return flight. +DUMMY/LJ024-0041.wav|that I would appoint and the Senate would confirm justices worthy to sit beside present members of the Court +DUMMY/LJ001-0030.wav|a very few years saw the birth of Roman character not only in Italy, but in Germany and France. +DUMMY/LJ034-0145.wav|Brennan had seen Oswald's picture on television, and he told the Commission that whether this affected his identification, quote, is something I do not know. +DUMMY/LJ019-0009.wav|or to insist upon the construction of prisons on the most approved plan. +DUMMY/LJ015-0085.wav|With the proceeds of these flagitious frauds Robson feasted and made merry. +DUMMY/LJ019-0119.wav|but Mr. Pearson hardly considered the converse sufficiently, and +DUMMY/LJ047-0128.wav|to conduct an additional investigation of Oswald in view of the activities which had led to his arrest. +DUMMY/LJ040-0187.wav|According to Sokolow, this indicated a, quote, present intellectual functioning in the upper range of bright normal intelligence, end quote. +DUMMY/LJ046-0150.wav|danger may be implied from others, such as membership or activity in an organization which believes in assassination as a political weapon. +DUMMY/LJ044-0140.wav|Furthermore, he had apparently not received any letters from the national headquarters of FPCC since May twenty-nine, nineteen sixty-three, +DUMMY/LJ012-0248.wav|The inquest on these remains, which medical examination showed to be those of a female, +DUMMY/LJ028-0441.wav|trees and overhanging vines were planted, and thus the travelers' tales arose. +DUMMY/LJ013-0154.wav|and the murderers immured within its walls. +DUMMY/LJ005-0268.wav|should be strictly prohibited, "as a stimulating luxury inconsistent with any notion of strict discipline and the due pressure of just punishment." +DUMMY/LJ035-0157.wav|When he left home that morning, Marina Oswald, who was still in bed, suggested that he wear a jacket. +DUMMY/LJ050-0170.wav|This matter is obviously beyond the jurisdiction of the Commission, +DUMMY/LJ027-0165.wav|Now, the lower forms of amphibians, such as siredon, menobranchus, siren, etc., +DUMMY/LJ015-0234.wav|Courier bags were bought to carry the "stuff" slung over the shoulders; +DUMMY/LJ029-0156.wav|the two Dallas newspapers carried several reports of his motorcade route. +DUMMY/LJ018-0356.wav|But a conversation overheard between the two brothers in Newgate satisfactorily exonerated Thomas Wainwright. +DUMMY/LJ011-0282.wav|he gave in, and signed the documents thus illegally extorted. +DUMMY/LJ030-0104.wav|A Dallas police car and several motorcycles at the rear kept the motorcade together and prevented unauthorized vehicles from joining the motorcade. +DUMMY/LJ044-0068.wav|as a result of which he was, quote, flooded with callers and invitations to debates, etc. as well as people interested in joining the F.P.C.C. +DUMMY/LJ023-0035.wav|For in the last three national elections +DUMMY/LJ045-0146.wav|to whom, quote, it was nothing terrible if people were to find out that he had been in Russia, end quote. +DUMMY/LJ036-0206.wav|again at twelve:forty-eight p.m., and again at twelve:fifty-five p.m. +DUMMY/LJ014-0204.wav|with his head resting on his hands, looked up and observed calmly, "Go along, clock; come along, gallows." +DUMMY/LJ017-0072.wav|whose signature he counterfeited. +DUMMY/LJ035-0135.wav|ran up and said that the President had been shot. +DUMMY/LJ041-0044.wav|While he apparently was able to induce his mother to make a false statement about his age +DUMMY/LJ013-0126.wav|When one of these clubs was taken in at the Army and Navy, that establishment also suffered. +DUMMY/LJ004-0074.wav|No judge ever condemned a man to be half-perished with cold by day, or half-suffocated with heat by night. +DUMMY/LJ002-0231.wav|There was no distinct place for the female debtors, who lived in the same galleries as the men. +DUMMY/LJ028-0518.wav|It is not strange, then, that they were included among the Seven Wonders of the World, +DUMMY/LJ007-0182.wav|and the result was another lengthy blue book, printed in the parliamentary papers, eighteen thirty-six, +DUMMY/LJ019-0013.wav|As regards the second, the Government had set a good example, and in deciding upon the erection of Pentonville prison +DUMMY/LJ044-0172.wav|and left for Irving with Marina Oswald and June and most of the Oswalds' effects three days later. +DUMMY/LJ017-0193.wav|It was proved that Mrs. Wilson had given Mrs. Soames something peculiar to drink, +DUMMY/LJ011-0249.wav|On this renewed visit Mr. Owen was still absent, and Mr. Mullay agreed to write him a note from a copy Howard gave him. +DUMMY/LJ027-0095.wav|It is as if Nature were not free to use any and every device to accomplish her end, but were conditioned by her own plans of structure; +DUMMY/LJ002-0218.wav|A committee of the House of Commons investigated the charges against Bambridge, who was proved to have connived at the escape of some debtors, +DUMMY/LJ018-0186.wav|It was well known that they were resurrectionists, and a strict watch was kept at all the turnpike gates on the southern roads leading into London. +DUMMY/LJ046-0242.wav|Such an approach seriously undermines the precautionary nature of PRS work; +DUMMY/LJ018-0187.wav|An officer was placed for this purpose at New Cross, Camberwell, and Kennington gates. +DUMMY/LJ040-0235.wav|When Lee became a disciplinary problem upon his return to school in the fall of nineteen fifty-three, +DUMMY/LJ018-0177.wav|But as there was no independent corroboration of the informer's evidence, according to the custom of the British law, +DUMMY/LJ016-0221.wav|Calcraft consented, got and bestowed the clothes, only to find that the person he had obliged exhibited them publicly. +DUMMY/LJ009-0077.wav|The ordinary of Newgate is an orthodox, unaffected, Church of England divine, +DUMMY/LJ036-0098.wav|William Whaley, a taxicab driver, told his employer on Saturday morning, November twenty-three +DUMMY/LJ014-0260.wav|In the West End he was a man of fashion, with a town house, a house at Brighton, and a cellar full of good wine at both. +DUMMY/LJ028-0156.wav|The city is divided into two portions by the river which runs through the midst of it. +DUMMY/LJ025-0029.wav|while the atmosphere and the earth supply plants with juices ready prepared and which can be absorbed immediately. +DUMMY/LJ005-0047.wav|for the reception of deserving cases discharged from prison. The governor of Newgate and other metropolitan prisons had orders of admission to this refuge +DUMMY/LJ007-0083.wav|gives a total of thirty-nine lunatics confined in various jails, many of them guilty of murder and other serious crimes. +DUMMY/LJ010-0008.wav|the streets of London were well lighted, the suburbs were more populous and regularly patrolled. +DUMMY/LJ005-0282.wav|"the comparatively innocent are seduced, the unwary are entrapped, +DUMMY/LJ018-0161.wav|They also discovered that through the intermediacy of one Robert Cummings, well known as a reputed coiner, +DUMMY/LJ006-0280.wav|The clothing was still meager and ragged: the washing places insufficient, and wanting in decency; +DUMMY/LJ016-0013.wav|It was noticed that he was most anxious to receive a pair of shoes for which he had asked, and when they arrived they were closely examined. +DUMMY/LJ050-0082.wav|the interest of the Secret Service goes beyond information on individuals or groups threatening to cause harm or embarrassment to the President. +DUMMY/LJ003-0039.wav|Giltspur Street, and the Poultry, or about four hundred and seventy-six in all. +DUMMY/LJ024-0136.wav|Third, because it will provide a series of federal courts willing to enforce the Constitution as written, +DUMMY/LJ035-0106.wav|Jack Dougherty, an employee working on the fifth floor, +DUMMY/LJ019-0077.wav|There was an absolute want of uniformity in dietaries; in some prisons it was too liberal, in others too low. +DUMMY/LJ017-0210.wav|The first case was that of the 'Flowery Land,' +DUMMY/LJ037-0137.wav|He was then shown two photographs of Oswald and he advised that Oswald was, quote, unquestionably, end quote, the man he saw. +DUMMY/LJ004-0221.wav|The food was properly prepared in the prison kitchen. +DUMMY/LJ019-0110.wav|Field labor, he urged, and with reason, was a very suitable employment; +DUMMY/LJ006-0164.wav|But when he did visit, his inspections were of the most superficial character +DUMMY/LJ017-0229.wav|Meanwhile the captain, roused from his berth, +DUMMY/LJ022-0172.wav|There is need for legislation providing for the regulation of interstate transportation by buses and trucks, +DUMMY/LJ022-0147.wav|we must continue to protect children, +DUMMY/LJ028-0501.wav|Nor were the walls about the palace a great mass of dull brick masonry. +DUMMY/LJ014-0256.wav|It seems incredible that this passbook, which when produced in court +DUMMY/LJ016-0298.wav|But before "the slight slow vibrations of the body had well ended, +DUMMY/LJ014-0247.wav|and without reference to or comparison with the documents in which the payment was claimed. +DUMMY/LJ004-0129.wav|Many of the jails were in the most deplorable condition: +DUMMY/LJ040-0136.wav|There are children in here who have killed people, and smoke. I want to get out, end quote, +DUMMY/LJ016-0424.wav|During the trial and after sentence he remained perfectly cool and collected. +DUMMY/LJ023-0134.wav|and reject the legislative powers which the courts have today assumed. +DUMMY/LJ016-0196.wav|He accepted, having at first Tom Cheshire as his assistant, then for a time, when Cheshire was dismissed for drunkenness, a man named Osborne. +DUMMY/LJ046-0204.wav|These requests were communicated in writing by the Secret Service; rather, the Service depended on the personal liaison maintained by PRS +DUMMY/LJ025-0172.wav|In fact, the bean obtains two of the absolutely indispensable elements of its substance from two distinct sources. +DUMMY/LJ038-0010.wav|a recessed area extending about fifteen feet between the sidewalk and the front door of his store. +DUMMY/LJ030-0052.wav|with two collapsible jump seats between the front and rear seats. +DUMMY/LJ019-0151.wav|stated in evidence that he was visiting Newgate one night, when he heard a great disturbance in one of the day and sleeping rooms, +DUMMY/LJ036-0024.wav|on a trip which passed a check point at St. Paul and Elm Streets at twelve:thirty-six p.m., November twenty-two, nineteen sixty-three. +DUMMY/LJ006-0221.wav|The weekly introduction of food, to which I shall presently refer, formed the basis of luxurious banquets, washed down by liquor +DUMMY/LJ016-0230.wav|His nature must have been kindly. +DUMMY/LJ005-0097.wav|and could expose all the local authorities that still lagged behind, or neglected to comply with the provisions of the new laws. +DUMMY/LJ035-0173.wav|This worker told Barnett that the shots had been fired from a window in the Depository Building, +DUMMY/LJ009-0200.wav|were publicly exposed in a stable in Little Bridge Street, near Apothecaries' Hall, +DUMMY/LJ049-0176.wav|On the other hand, the Secret Service had no knowledge whatever of Oswald, his background, or his employment at the Book Depository, +DUMMY/LJ011-0206.wav|By the same forcible means I was compelled to quit England, and to trust myself to the protection of this person, +DUMMY/LJ037-0225.wav|five foot eight inches, black hair, slender, wearing a white jacket, white shirt and dark slacks, end quote, +DUMMY/LJ013-0034.wav|The case was clearly proved against them, and both were sentenced to transportation for life. +DUMMY/LJ011-0182.wav|Miss Turner, thus pressed, consented to go on to Gretna Green. +DUMMY/LJ048-0097.wav|and had also scheduled visits to other parts of the country immediately before the Texas trip, +DUMMY/LJ004-0023.wav|Measures remedial, although at best partial and incomplete, were introduced from time to time. +DUMMY/LJ048-0014.wav|was disenchanted with Russia, and had a renewed concept -- I am paraphrasing, a renewed concept -- of the American free society. +DUMMY/LJ049-0132.wav|The Commission believes that it is necessary to improve the cooperation among these agencies +DUMMY/LJ010-0051.wav|The well-known Cato Street conspiracy, +DUMMY/LJ011-0279.wav|and write an order sufficient to secure the surrender of the other one thousand two hundred pounds. +DUMMY/LJ040-0053.wav|It was an obvious element in his decision to go to Russia and later to Cuba and it probably influenced his decision to shoot at General Walker. +DUMMY/LJ050-0094.wav|I shall employ all means to right this gross mistake or injustice to a bona fide U.S. citizen and ex-service man. +DUMMY/LJ040-0021.wav|ascertained the facts surrounding the assassination but did not draw conclusions concerning Oswald's legal guilt. +DUMMY/LJ038-0006.wav|police sirens sounded along Jefferson Boulevard. +DUMMY/LJ005-0132.wav|Only a few glaring evils still demanded a remedy. +DUMMY/LJ044-0077.wav|This would have made him a real center of attention as he must have been when he first arrived in the Soviet Union +DUMMY/LJ012-0066.wav|and begged his wife to send him over a consignment of cheap "righteous" watches, or such as had been honestly obtained, and not "on the cross." +DUMMY/LJ010-0009.wav|People, too, were getting into the habit of carrying but little cash about them, and no valuables but their watches or personal jewelery. +DUMMY/LJ034-0101.wav|Brennan saw the man fire the last shot and disappear from the window. +DUMMY/LJ028-0253.wav|The city was spared; the great walls were left standing; the daily sacrifices were continued in the temples, and Cyrus made his home in the royal palace. +DUMMY/LJ017-0130.wav|To the last, therefore, he played the part of a man wrongfully convicted, and did not abandon hope +DUMMY/LJ003-0136.wav|pale as death, very ill, and in a dreadfully dirty state, the wretches making game of him, and enjoying my distress; +DUMMY/LJ009-0095.wav|which he has bitten unconsciously, or from rage, or to rouse his fainting courage. +DUMMY/LJ043-0061.wav|at the time of his defection, when he evidenced no interest in his father and hardly mentioned him, even when questioned. +DUMMY/LJ048-0161.wav|The Chief of the Service has provided the Commission a detailed explanation of this policy, quote, +DUMMY/LJ010-0165.wav|but he came as a lad to London, and took service as a pot-boy to a publican. +DUMMY/LJ038-0060.wav|John Gibson, another patron in the theatre, saw an officer grab Oswald, and he claims that he heard the click of a gun misfiring. +DUMMY/LJ018-0326.wav|I propose to end at this point the detailed account of the more prominent criminal cases which lodged their perpetrators in Newgate. +DUMMY/LJ049-0099.wav|A number of bills were introduced immediately following the assassination of President Kennedy. +DUMMY/LJ035-0032.wav|When the elevator failed to come, Baker said, quote, let's take the stairs, end quote, and he followed Truly up the stairway, which is to the west of the elevator. +DUMMY/LJ011-0079.wav|he soon left the service and betook himself to dishonest ways. +DUMMY/LJ039-0070.wav|testified that the shots which struck the President in the neck and in the head were, quote, not particularly difficult, end quote. +DUMMY/LJ016-0420.wav|is remembered at the former prison as a defiant, +DUMMY/LJ032-0016.wav|and (eight) Oswald's capability with a rifle. +DUMMY/LJ013-0168.wav|That Lord William Russell had committed suicide was at once declared impossible. +DUMMY/LJ008-0279.wav|In this latter it might fairly be said that a man was put to death less for his own sins than for the crimes of others. +DUMMY/LJ022-0029.wav|So many people come to Washington who know things that are not so, +DUMMY/LJ013-0094.wav|The same evening both started for Liverpool, and embarking on board the mail steamer 'Britannia,' escaped to the United States. +DUMMY/LJ049-0021.wav|to permit its removal on those occasions when the President wished to ride in an open car. +DUMMY/LJ016-0068.wav|and continued his march to Kingston, where he slept soundly under a hedge till next morning. +DUMMY/LJ045-0039.wav|His past relationships with his wife had been stormy, however, and it did not seem that she respected him very much. +DUMMY/LJ008-0202.wav|Those who had hired apartments spent the night in them, keeping up their courage with liquids and cigars. +DUMMY/LJ021-0158.wav|Demoralization caused by vast unemployment is our greatest extravagance. +DUMMY/LJ030-0231.wav|Most of the other Secret Service agents in the motorcade had drawn their sidearms. +DUMMY/LJ025-0117.wav|that a fundamental unity of structure obtains in animals and plants, +DUMMY/LJ037-0136.wav|Patterson, when asked later to confirm his identification by affidavit said he did not recall having been shown the photograph. +DUMMY/LJ011-0110.wav|He was, however, visited by and received his wife, and several members of the Society of Friends. +DUMMY/LJ028-0215.wav|Marduk, the great lord, looking with joy on his pious works and upright heart, +DUMMY/LJ037-0039.wav|The man continued along tenth Street. Mrs. Markham saw a police car slowly approach the man from the rear and stop alongside of him. +DUMMY/LJ039-0078.wav|The effect of a four-power telescopic sight on the difficulty of these shots was considered in detail +DUMMY/LJ013-0258.wav|The poor creature was never seen again alive. +DUMMY/LJ042-0058.wav|That my mother and you are (in spite of what the newspaper said) not objects of affection, but only examples of workers in the U.S. +DUMMY/LJ011-0176.wav|He now pretended that Mr. Turner was also on his way to the border, pursued by sheriffs' officers. +DUMMY/LJ032-0006.wav|six point five-millimeter Italian rifle bearing the serial number C two seven six six. +DUMMY/LJ030-0111.wav|The Drive through Dallas +DUMMY/LJ038-0116.wav|When Captain Fritz asked him why he carried the revolver, he answered, quote, Well, you know about a pistol. I just carried it, end quote. +DUMMY/LJ019-0120.wav|omitted the fact that he might have to deal with that persistent idleness which is not an unknown characteristic of the criminal class. +DUMMY/LJ002-0206.wav|Dissolute habits prevailed on all sides; drunkenness was universal, gambling perpetual. +DUMMY/LJ035-0177.wav|According to Barnett, quote, there were people going in and out, end quote, during this period. +DUMMY/LJ032-0133.wav|Lieutenant Day of the Dallas Police Department had "lifted" a palmprint from the underside of the gun barrel +DUMMY/LJ017-0113.wav|After this the step-father insisted upon a post-mortem, which was conducted somewhat carelessly. +DUMMY/LJ027-0025.wav|The facts adduced from morphology, being the result of researches into the structure of adult animals and plants, +DUMMY/LJ016-0295.wav|curse, or shout, as in this heaving and struggling forward they gained or lost in their strong efforts to get nearer where Müller was to die. +DUMMY/LJ022-0167.wav|but it will protect the actual value and earning power of properties now owned by thousands of investors +DUMMY/LJ019-0376.wav|The task imposed upon them, tending as it did to the imposition of a fine upon the local authorities, was not a pleasant one, +DUMMY/LJ002-0148.wav|and that no more than one hundred ninety-seven creditors recovered debts and costs. +DUMMY/LJ034-0044.wav|The freshness of this print could be estimated only because the Dallas police developed it through the use of powder. +DUMMY/LJ010-0212.wav|With Oxford's other papers were found letters from the secretary, written as it seemed by Oxford to himself, after the manner of Mr. Toots, +DUMMY/LJ017-0149.wav|He had met her at a boarding-house, where he lived with his own wife, a person of "shady" antecedents, +DUMMY/LJ028-0505.wav|It seems that the bricks of the reliefs were molded and glazed separately and so accurately that when built into the wall they fitted perfectly. +DUMMY/LJ012-0038.wav|but could not positively identify it, and Ikey defied them to remove a single shoe. +DUMMY/LJ046-0206.wav|and at the working level with personnel of the field offices of the various agencies. The Service frequently participated +DUMMY/LJ014-0259.wav|Watts led two lives. +DUMMY/LJ018-0358.wav|Christina Edmunds had resort to strychnia, the same lethal drug that Palmer used; +DUMMY/LJ038-0049.wav|Although the witnesses did not hear the sound of a misfire, +DUMMY/LJ008-0245.wav|though pulled right and left, or sent headlong into the crowd by the swing of some brutal and muscular arm, +DUMMY/LJ011-0234.wav|Bigger "jobs" than ever were planned and attempted, +DUMMY/LJ050-0261.wav|from many Government agencies including the Department of Defense and the President's Office of Science and Technology. +DUMMY/LJ026-0101.wav|The causes of this ascent are not clear, +DUMMY/LJ024-0040.wav|But if by that phrase the charge is made +DUMMY/LJ028-0291.wav|Wrath stirred within the King at the sight of a man of his lofty rank in such a condition; +DUMMY/LJ026-0019.wav|they cannot ingest solid food, but are nourished by a watery solution of nutrient materials. +DUMMY/LJ004-0168.wav|The smell at one o'clock of the day "was something more than can be expressed by the term disgusting." +DUMMY/LJ004-0069.wav|In short, attention to his feelings, mental and bodily, a supply of every necessary, abstraction from evil society, +DUMMY/LJ038-0245.wav|A photography expert with the FBI +DUMMY/LJ050-0205.wav|Testimony and other evidence before the Commission +DUMMY/LJ042-0235.wav|Freedoms are about the same. Medical aid and the educational system in the USSR is better than in the USA, end quote. +DUMMY/LJ047-0040.wav|Oswald was first interviewed by FBI Agents John W. Fain and B. Tom Carter on June twenty-six, nineteen sixty-two, +DUMMY/LJ011-0174.wav|But at Kendal there was no Mr. Turner, and, to allay Miss Turner's growing anxiety, +DUMMY/LJ019-0276.wav|during short sentences, or the earlier stages of a long confinement, +DUMMY/LJ003-0078.wav|Nor were the misdemeanants and bail prisoners any longer separated from those whose crimes were of a more serious character. +DUMMY/LJ027-0133.wav|to present the mass of embryological evidence, but a few salient facts will illustrate the kind of evidence to be deduced from embryology. +DUMMY/LJ028-0474.wav|It too was surmounted with towers. +DUMMY/LJ030-0171.wav|and the thought immediately crossed his mind that it was an assassination attempt. +DUMMY/LJ014-0074.wav|One fine afternoon in August, O'Connor was met walking in the direction of Bermondsey. +DUMMY/LJ013-0182.wav|took Courvoisier into custody, and placed the two female servants under surveillance. +DUMMY/LJ031-0106.wav|whose major wound fell within Dr. Shaw's area of specialization. +DUMMY/LJ022-0161.wav|Even more significantly it has given the country as a whole an uneasy apprehension of overconcentrated economic power. +DUMMY/LJ028-0308.wav|Let neither these nor the former troops be armed with any weapons but their swords those thou mayest leave them. +DUMMY/LJ028-0311.wav|the Babylonians will entrust everything, even the keys of their gates, to me. Then it will be for me and my Persians to do the rest. +DUMMY/LJ005-0235.wav|Last, and worst of all, the arrangements for keeping the condemned prisoners between sentence and execution were more than unsatisfactory. +DUMMY/LJ039-0120.wav|windy, rainy, dark. There is little probability that he had good, expert coach, +DUMMY/LJ040-0200.wav|Furthermore she did not appear to understand her own relationship to Lee's psychological problems. +DUMMY/LJ016-0389.wav|Giovanni Lanni, the Italian boy who murdered a Frenchwoman in the Haymarket, +DUMMY/LJ006-0199.wav|the most popular being the "Times," "Morning Herald," and "Morning Chronicle"; on Sunday the "Weekly Dispatch," "Bell's Life," and the "Weekly Messenger." +DUMMY/LJ014-0262.wav|He played high, and spent his nights at the club, or in joyous and dissolute company. +DUMMY/LJ018-0273.wav|Tarpey was caught through his wife, +DUMMY/LJ028-0007.wav|With mingling fear and wonder at the mystery always lying beyond the desert horizon, +DUMMY/LJ037-0155.wav|At the request of the Commission, Joseph D. Nicol, superintendent of the Illinois Bureau of Criminal Identification Investigation, +DUMMY/LJ026-0085.wav|And this is due to the fact that in the manufacture of starch in the chlorophyll bodies +DUMMY/LJ027-0019.wav|And at last each organism or each alliance of organisms must come to the greatest concession of all, which is called death. +DUMMY/LJ005-0269.wav|Prison officers should not have any share in prisoners' earnings, +DUMMY/LJ007-0017.wav|It is to be feared too that although the surface was thus whitewashed and decorous, much that was vicious still festered and rankled beneath, +DUMMY/LJ003-0123.wav|The most trifling acts were magnified into offenses. +DUMMY/LJ028-0390.wav|I was informed by a certain Elamite brother, who came from those regions, and now leads the life of a monk at Jerusalem, +DUMMY/LJ003-0286.wav|The report of the Committee of the House of Commons painted so black a picture of Newgate as then conducted, that the Corporation were roused in very shame +DUMMY/LJ042-0199.wav|He also expressed hatred of the Soviet Union and of the Communist Party, U.S.A., +DUMMY/LJ014-0318.wav|Another set of frauds, which resembled those of Pries in principle, although not in practice, were soon afterwards discovered. +DUMMY/LJ038-0296.wav|Marina Oswald also testified that her husband had used a bus to return home. +DUMMY/LJ039-0068.wav|Four marksmanship experts testified before the Commission. +DUMMY/LJ011-0168.wav|Miss Turner was shown into a private room, where Mr. Wakefield soon presented himself. +DUMMY/LJ014-0257.wav|was a mass of blots and erasures, should not have created suspicion of foul play either at the bank or at the company's board. +DUMMY/LJ041-0027.wav|Other evidence of the existence of such a disability is provided by the many other misspellings that appear in Oswald's writings, portions of which are quoted below. +DUMMY/LJ008-0192.wav|As early as six a.m. the number assembled already exceeded that seen on ordinary occasions; +DUMMY/LJ002-0073.wav|Neild takes it for granted that the former rather than the latter prevailed in the selection, +DUMMY/LJ024-0075.wav|to be determined by chance or the personal indecision of individuals. +DUMMY/LJ036-0064.wav|Marsalis Street is seven blocks from Beckley. +DUMMY/LJ038-0132.wav|When asked why he lived at his roominghouse under the name O. H. Lee, +DUMMY/LJ050-0044.wav|has been augmented, and a Secret Service inspector has been put in charge of this operation. With the assistance of the President's Office of Science and Technology, +DUMMY/LJ050-0001.wav|For more information, or to volunteer, please visit librivox dot org. Report of the President's Commission on the Assassination of President Kennedy. +DUMMY/LJ039-0037.wav|Vice President Lyndon B. Johnson was in Dallas for a visit which had been publicized in the Dallas newspapers +DUMMY/LJ028-0345.wav|He then chose out near three thousand of the leading citizens and caused them to be crucified, while he allowed the remainder still to inhabit the city. +DUMMY/LJ003-0154.wav|Artisans and others were at liberty to work at their trades, provided they were not dangerous. +DUMMY/LJ041-0067.wav|Other marines also testified that Oswald had few friends and kept very much to himself. +DUMMY/LJ032-0109.wav|Both names were found to be fictitious. +DUMMY/LJ035-0070.wav|Truly stood in front of the building. +DUMMY/LJ023-0111.wav|reading into the Constitution words and implications which are not there, and which were never intended to be there. +DUMMY/LJ033-0023.wav|Frazier, surprised, asked him why he was going to Irving on Thursday night rather than Friday. +DUMMY/LJ050-0140.wav|Secretary Dillon testified that the use of such liaison officers is the only effective way to insure that adequate liaison is maintained. +DUMMY/LJ029-0071.wav|An important purpose of the President's visit to Dallas was to speak at a luncheon given by business and civic leaders. +DUMMY/LJ048-0095.wav|The Service prefers to have two agents perform advance preparations. +DUMMY/LJ019-0006.wav|laid it down that individuals might be confined separately and apart in single cells. +DUMMY/LJ041-0095.wav|Oswald read a good deal, said Powers, but, quote, he would never be reading any of the shoot-em-up westerns or anything like that. +DUMMY/LJ034-0130.wav|Since the window ledges in the Depository Building are lower than in most buildings, +DUMMY/LJ014-0163.wav|who wrote to the 'Times,' saying that he believed "a sight so inconceivably awful +DUMMY/LJ048-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ017-0139.wav|presenting still greater features of resemblance with Palmer's, for both were medical men, and both raised difficult questions of medical jurisprudence. +DUMMY/LJ040-0126.wav|Oswald was remanded for psychiatric observation to Youth House, an institution in which children are kept for psychiatric observation +DUMMY/LJ032-0246.wav|It had been purchased in March nineteen sixty-three, and taken to New Orleans where Marina Oswald saw it in their rented apartment +DUMMY/LJ010-0208.wav|Hannibal and Ethelred were on the council; Anthony, Augustus, and Frederic were among the generals; +DUMMY/LJ013-0243.wav|Good now offered to go to Wandsworth and satisfy the pawnbroker. +DUMMY/LJ016-0408.wav|During the singing of these hymns Wainwright fainted, but whether from real emotion or the desire to make a sensation was never exactly known. +DUMMY/LJ026-0016.wav|They ingest solid proteinaceous food, +DUMMY/LJ016-0320.wav|But these, it was thought, might be provided by the admission of the press and the holding of a coroner's inquest. +DUMMY/LJ039-0034.wav|The Commission has concluded, therefore, that regardless of what Oswald may have said to his wife +DUMMY/LJ045-0018.wav|As you can see, political friction between "left" and "right" is very great here. +DUMMY/LJ008-0081.wav|and after praying fervently for a few minutes, the steps on which she stood were drawn away, and she was left suspended. +DUMMY/LJ002-0157.wav|and Susannah Evans, in October the same year, for two shillings, with costs of six shillings, eight pence. +DUMMY/LJ048-0245.wav|but that their visits to the Cellar were, quote, neither consistent nor inconsistent, end quote, with their duty. +DUMMY/LJ043-0068.wav|While he might have expected difficulty from such an approach, in fact +DUMMY/LJ018-0232.wav|He himself prepared it on a blank form which he had brought with him on purpose. +DUMMY/LJ006-0189.wav|gambling, indirect contravention of parliamentary rules, instruction in all nefarious processes, +DUMMY/LJ019-0191.wav|feeling that it approached too nearly to solitary confinement. +DUMMY/LJ035-0093.wav|On the basis of this time test, therefore, +DUMMY/LJ045-0208.wav|His family lived with Mrs. Paine, ostensibly because Oswald could not afford to keep an apartment in Dallas, +DUMMY/LJ031-0116.wav|Dr. Charles F. Gregory, chief of orthopedic surgery, operated on the wounds of Governor Connally's right wrist, +DUMMY/LJ049-0153.wav|The Secret Service was organized as a division of the Department of the Treasury in eighteen sixty-five, to deal with counterfeiting. +DUMMY/LJ047-0170.wav|On November five, Hosty was traveling near Mrs. Paine's home and took the occasion to stop by to ask whether she had any further information. +DUMMY/LJ032-0032.wav|that the bold printing on the face of the mail-order coupon was in the handprinting of Lee Harvey Oswald and that the writing on the envelope was also his. +DUMMY/LJ031-0094.wav|and you have to establish adequate circulation. +DUMMY/LJ016-0415.wav|but the story was never substantiated, and we may hope that it rested only on the idle gossip of the day. +DUMMY/LJ018-0328.wav|Few of the Newgate notorieties of late years show any marked peculiarities; +DUMMY/LJ037-0207.wav|Michaelis furnished the shipping copy of the invoice, and the Railway Express Agency shipping documents, showing that +DUMMY/LJ040-0121.wav|On September thirty, nineteen fifty-two, Lee enrolled in P.S. one seventeen +DUMMY/LJ022-0009.wav|The job of creating a program for the nation's welfare is, in some respects, like the building of a ship. +DUMMY/LJ046-0154.wav|as indicating some potential danger to the President -- no matter how small -- it was indexed in the general PRS files +DUMMY/LJ032-0070.wav|Ordinarily, Inspector Holmes testified, identification is not requested because it is assumed that the person with the notice is entitled to the package. +DUMMY/LJ014-0099.wav|The circumstance was suspicious, and a search was at once made of the whole premises. +DUMMY/LJ030-0008.wav|During the afternoon, President Kennedy dedicated the U.S. Air Force School of Aerospace Medicine at Brooks Air Force Base. +DUMMY/LJ030-0040.wav|The order of vehicles in the Dallas motorcade was as follows: +DUMMY/LJ028-0510.wav|Probably their metal was far too valuable for the enemy to leave behind. +DUMMY/LJ001-0120.wav|In the old print each figure has its definite individuality, and one cannot be mistaken for the other; +DUMMY/LJ013-0075.wav|Fletcher was the moving spirit of the whole business. It was he who had introduced Barber to Miss Slack, +DUMMY/LJ032-0234.wav|The rifle was shipped from Klein's in Chicago on March twenty, nineteen sixty-three, at a time when the Oswalds were living on Neely Street. +DUMMY/LJ026-0077.wav|to the chlorophyll bodies or chromatophores, for plants which, like fungi, etc., are devoid of chlorophyll, are unable thus to acquire energy. +DUMMY/LJ022-0159.wav|The absentee management of unnecessary holding company control +DUMMY/LJ037-0180.wav|If a bullet is missing, five were fired. This corresponds with the observation and memory of Ted Callaway, and possibly Warren Reynolds, +DUMMY/LJ007-0157.wav|The evils of such association were aggravated by the unbroken idleness; one "evil inflamed the other;" reformation +DUMMY/LJ030-0163.wav|Mrs. John F. Kennedy, on the left of the rear seat of the limousine, looked toward her left and waved to the crowds along the route. +DUMMY/LJ035-0100.wav|In the few seconds which elapsed while Baker and Truly ran from the first to the second floor, +DUMMY/LJ050-0038.wav|In attempting to identify those individuals who might prove a danger to the President, +DUMMY/LJ027-0077.wav|but attention may be called to a like similarity extending to the details of structure. +DUMMY/LJ011-0113.wav|He met his death with unshaken firmness, only entreating that a certain blue handkerchief, +DUMMY/LJ003-0006.wav|Besides this, although the families of debtors were no longer permitted to live with them inside the jail, +DUMMY/LJ028-0143.wav|It is surrounded, in the first place, by a broad and deep moat, full of water, behind which rises a wall +DUMMY/LJ002-0110.wav|Having thus briefly described the plan and appropriation of the prison, I propose to deal now with the general condition of the inmates, and the manner of their life. +DUMMY/LJ038-0280.wav|And for purposes of probative value, for whatever it might be worth, +DUMMY/LJ013-0072.wav|It was shown that the will must be a forgery, +DUMMY/LJ018-0332.wav|Greed in the latter case was a secondary motive; +DUMMY/LJ047-0138.wav|He checked in Oswald's old neighborhood and throughout the Dallas-Fort Worth area but was unable to locate Oswald. +DUMMY/LJ043-0048.wav|Marguerite Oswald visited her son and his family at the first apartment which he rented after his return, and tried to help them get settled there. +DUMMY/LJ049-0155.wav|while investigating a plot to assassinate President Cleveland, the Service assigned a small protective detail of agents to the White House. +DUMMY/LJ018-0251.wav|He confessed himself a perjurer in having sworn to the false will, and a wholesale forger, having manufactured no less than ten false signatures +DUMMY/LJ028-0248.wav|long after the outer portions of the town were taken, knew nothing of what had chanced, but as they were engaged in a festival, +DUMMY/LJ016-0058.wav|he worked along to the roofs in Warwick Lane. +DUMMY/LJ016-0396.wav|but to which additions are sometimes made, chiefly of stimulants, if deemed necessary, by the medical officer of the jail. +DUMMY/LJ010-0280.wav|On June eighteen fifty the Queen was once more subjected to cowardly outrage, the offender being a Mr. Pate, a gentleman by birth, +DUMMY/LJ014-0017.wav|In the mean time the police were not idle. +DUMMY/LJ044-0163.wav|Marina Oswald testified that her husband engaged in Fair Play for Cuba Committee activities, quote, +DUMMY/LJ024-0090.wav|There are many types of amendment proposed. +DUMMY/LJ007-0225.wav|I find it ordered "that the ordinary of Newgate be restricted from making any communications to the Home Office +DUMMY/LJ023-0076.wav|It is but a decent respect due to the wisdom, the integrity and the patriotism of the legislative body, +DUMMY/LJ019-0069.wav|and it was unsafe to fix this limit above twelve months, although some rash advocates were in favor of eighteen months, some indeed of two years. +DUMMY/LJ006-0276.wav|On this female side, where the Ladies' Association still reigned supreme, more system and a greater semblance of decorum was maintained. +DUMMY/LJ010-0298.wav|That of Fauntleroy the banker, in eighteen twenty-four, +DUMMY/LJ037-0168.wav|Nicol differed with the FBI experts on one bullet taken from Tippit's body. +DUMMY/LJ018-0345.wav|As in that case, the grave had been dug long in anticipation, and the chloride of lime purchased to destroy the corpse. +DUMMY/LJ028-0435.wav|Arabs, searching for bricks, have burrowed their way down deep into it, revealing massive walls and arches. +DUMMY/LJ048-0210.wav|He was handicapped, however, by the fact that he was riding in a closed car whose roof at times obscured his view. +DUMMY/LJ010-0047.wav|while in eighteen fifty Her Majesty was the victim of another outrage at the hands of one Pate. +DUMMY/LJ024-0134.wav|Second, because it will provide a reinvigorated +DUMMY/LJ027-0080.wav|two toes (sheep), four toes (hog) and five toes (dog) +DUMMY/LJ010-0206.wav|The officers of the society were to be known only by "factitious names." +DUMMY/LJ014-0329.wav|and so developed his business that in one year his transactions amounted to a couple of millions of pounds. +DUMMY/LJ039-0067.wav|which is ordinarily required when a marksman must raise his rifle as a target moves farther away. +DUMMY/LJ013-0127.wav|Suspicion fell at length upon Ashley, who was seen to handle the forks and spoons at table in a strange manner. +DUMMY/LJ021-0160.wav|Some people try to tell me that we must make up our minds that for the future we shall permanently have millions of unemployed +DUMMY/LJ046-0037.wav|The nature of the protective assignment. +DUMMY/LJ028-0485.wav|The outer and inner defenses of Babylon were so strong and so high that no enemy could hope to take them, +DUMMY/LJ045-0249.wav|Out of these and the many other factors which may have molded the character of Lee Harvey Oswald +DUMMY/LJ028-0218.wav|Without a skirmish or a battle, he permitted them to enter Babylon, and, sparing the city, he delivered the King Nabonidus to him. +DUMMY/LJ010-0128.wav|they with Brunt and Tidd scorned the ordinary's ministrations, +DUMMY/LJ023-0090.wav|It is the accusation of most distinguished justices of the present Supreme Court. +DUMMY/LJ018-0066.wav|He at length stole a carpet-bag containing valuables, and fled to Hamburgh. +DUMMY/LJ047-0029.wav|had described himself as a Marxist, had said he would give the Soviet Union any useful information he had acquired +DUMMY/LJ035-0109.wav|but in his testimony Piper did not mention either seeing or talking with Dougherty during these moments of excitement. +DUMMY/LJ046-0019.wav|In this part of its inquiry the Commission has had full access to a major study of all phases of protective activities +DUMMY/LJ012-0168.wav|was tried at the Old Bailey the following year for horse-stealing, and hanged in front of Newgate. +DUMMY/LJ039-0003.wav|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ011-0111.wav|Two elders of the meeting sat up with him in the press yard the whole of the night previous to execution, and a third, +DUMMY/LJ036-0017.wav|He probably walked east on Elm Street for seven blocks to the corner of Elm and Murphy +DUMMY/LJ021-0150.wav|I shall not ask either employers or employees permanently to lay aside the weapons common to industrial war. +DUMMY/LJ048-0054.wav|which specifically required the referral of such a case as Oswald's to the Secret Service, +DUMMY/LJ030-0103.wav|Police car and motorcycles. +DUMMY/LJ012-0196.wav|Bishop, whose real name was Head, married a half-sister of Williams'. Williams was a professional resurrectionist, or body-snatcher, +DUMMY/LJ032-0062.wav|by which Oswald requested that mail addressed to that box be forwarded to him in New Orleans, where he had moved on April twenty-four. +DUMMY/LJ030-0061.wav|with a metallic handle for the rider to grasp. +DUMMY/LJ040-0181.wav|They reflect a considerable amount of impoverishment in the social and emotional areas. +DUMMY/LJ004-0167.wav|The floor was on the level of the river, and very damp. +DUMMY/LJ005-0073.wav|To its efforts, and their effect upon Parliament and the public mind, we must attribute the new Jail Acts of four George the fourth +DUMMY/LJ022-0203.wav|For that we can be thankful to the God who watches over America. +DUMMY/LJ028-0315.wav|He replied that he was Zopyrus, and deserted to them from the Persians. +DUMMY/LJ008-0302.wav|I have heard the protracted agony of both classes described by those who witnessed it in terms so strong, that I am unwilling to repeat them. +DUMMY/LJ048-0193.wav|Three officers from the Dallas Police Department were assigned to the intersection of Elm and Houston +DUMMY/LJ030-0200.wav|At that point, both Governor and Mrs. Connally observed brain tissue splattered over the interior of the car. +DUMMY/LJ012-0070.wav|He was, however, recognized, and ere long an order came out from home for his arrest and transfer to England, +DUMMY/LJ006-0101.wav|some, who had the semblance of covering on the upper part of their feet, had no soles to the shoes, and their bare feet were on the ground. +DUMMY/LJ044-0110.wav|We do not have any organizational ties with the Committee, and yet there is much material that we issue from time to time that is important +DUMMY/LJ012-0270.wav|She had realized all her effects, and brought them with her furniture to Greenacre's lodgings. The two when married were to emigrate to Hudson's Bay. +DUMMY/LJ047-0115.wav|Several days later, the Bureau received additional evidence that Oswald had lied to Agent Quigley. +DUMMY/LJ022-0004.wav|In the many weeks since that time the Congress has devoted itself to the arduous task of formulating legislation necessary to the country's welfare. +DUMMY/LJ016-0239.wav|Marwood till latterly seemed to have done his work more effectually, and has been known to give as much as six feet fall. +DUMMY/LJ010-0226.wav|and Oxford was ordered to be detained during Her Majesty's pleasure. +DUMMY/LJ016-0259.wav|The graphic and terrible account given by Charles Dickens of the awful scene before Horsemonger Lane Jail, at the execution of the Mannings, +DUMMY/LJ008-0136.wav|These evidently deprived him of the small portion of fortitude he had summoned up. +DUMMY/LJ030-0141.wav|a subsequent bullet, which was lethal, shattered the right side of his skull. +DUMMY/LJ026-0014.wav|based on the foregoing considerations, is convenient for distinguishing between animals and plants in all cases except the doubtful unicellar forms. +DUMMY/LJ035-0002.wav|Chapter four. The Assassin: Part four. Oswald's Actions in Building After Assassination. +DUMMY/LJ045-0113.wav|he did not want his landlady to know his real name because she might read in the paper of the fact that he had been in Russia and that he had been questioned, end quote. +DUMMY/LJ008-0146.wav|who kept a lavender warehouse in the city, and who had gardens at Feltham, +DUMMY/LJ004-0039.wav|They were hopeless of any general reform by the action of the executive alone. +DUMMY/LJ046-0108.wav|Intelligence Functions Relating to Presidential Protection at the Time of the Dallas Trip +DUMMY/LJ027-0001.wav|The Science: History of the Universe, Volume five. Edited by Francis Rolt-Wheeler. +DUMMY/LJ019-0088.wav|There was no general rule of employment. Hard labor was often not insisted upon in separate confinement; +DUMMY/LJ041-0018.wav|but he was not sure then that Oswald meant to carry out the plan, and in fact they never did. +DUMMY/LJ047-0176.wav|shortly after Oswald rented the room on October fourteen. +DUMMY/LJ002-0212.wav|Mexico, and was consumed at the rate of a hogshead per week. +DUMMY/LJ041-0048.wav|who studied it during the year following his unsuccessful attempt to enlist until, quote, He knew it by heart, end quote. +DUMMY/LJ028-0384.wav|So Babylon was buried and forgotten. +DUMMY/LJ048-0232.wav|and the last agent left the Press Club by two a.m. +DUMMY/LJ004-0080.wav|because they are not specified in his sentence; he is therefore entitled to a wholesome atmosphere, +DUMMY/LJ032-0091.wav|There is no "Dr. Hideel" licensed to practice medicine in Louisiana. +DUMMY/LJ043-0039.wav|He also indicated to officials at the American Embassy in Moscow that his defection was motivated at least in part +DUMMY/LJ006-0287.wav|The presence of so many strangers, many of them gentlemen, distracted the prisoners' attention, and could not be productive of much good. +DUMMY/LJ031-0176.wav|A casket was obtained and the President's body was prepared for removal. +DUMMY/LJ012-0147.wav|Moss peached because he declared he had been done out of the proper price of the gold-dust; but it was clear that he had tried to appropriate the whole of the stuff, +DUMMY/LJ004-0217.wav|The old haphazard system, by which meat was issued in bulk, +DUMMY/LJ034-0043.wav|The print, therefore, could have been placed on the carton at any time within this period. +DUMMY/LJ027-0122.wav|But the vacillating policy in the matter of rudiments does not end here; for it is shown in a still more aggravated form +DUMMY/LJ022-0058.wav|While our present and projected expenditures for work relief are wholly within the reasonable limits of our national credit resources, +DUMMY/LJ019-0244.wav|The last inquiry into the condition and management of our jails and houses of correction was that made by the Lords' Committee in eighteen sixty-three. +DUMMY/LJ021-0165.wav|On the contrary, +DUMMY/LJ004-0065.wav|his morals by compelling him to mix with a promiscuous assemblage of hardened and convicted criminals, +DUMMY/LJ014-0235.wav|I want forgiveness of man; I want those doors (of the prison) opened. +DUMMY/LJ048-0117.wav|all instructions from higher authority were communicated to him orally. +DUMMY/LJ045-0155.wav|Question: Did you say anything to him then? +DUMMY/LJ033-0151.wav|identified these prints as the left index fingerprint and right palmprint of Lee Harvey Oswald +DUMMY/LJ044-0021.wav|he debated over radio station WDSU, New Orleans, with Carlos Bringuier, +DUMMY/LJ040-0155.wav|in spite of chronic truancy from school which brought him into Youth House. +DUMMY/LJ049-0168.wav|As discussed above, the Bureau has attempted to meet its responsibilities in this field by spelling out in its Handbook +DUMMY/LJ038-0189.wav|and include newspaper clippings (should there be anything about me in the newspapers). I believe that the Embassy will come quickly to your assistance on learning everything. +DUMMY/LJ047-0186.wav|I can now afford to wait until New Orleans forwarded the necessary papers to me to show me I now had all the information. +DUMMY/LJ034-0027.wav|Depository employees were laying a new floor at the west end and transferring books from the west to the east end of the building. +DUMMY/LJ040-0086.wav|In the fall of nineteen forty-eight she told John Pic and Robert Oswald +DUMMY/LJ047-0213.wav|would meet our specific criteria, none of them alone. But, it is when you begin adding them up to some degree that you begin to get criteria that, are meaningful. +DUMMY/LJ012-0164.wav|No murder has created greater sensation and horror throughout England than that of Mr. Weare by Thurtell, Hunt, and Probert. +DUMMY/LJ010-0258.wav|As the Queen was driving from Buckingham Palace to the Chapel Royal, +DUMMY/LJ005-0275.wav|religious worship, and instruction, as "absolutely necessary for preventing contamination, +DUMMY/LJ029-0102.wav|After the selection of the Trade Mart as the luncheon site, +DUMMY/LJ031-0154.wav|Secret Service agents stationed at later stops on the President's itinerary of November twenty-two were redeployed. +DUMMY/LJ033-0186.wav|one cannot estimate when, prior to November twenty-two, Oswald made the paper bag. +DUMMY/LJ003-0048.wav|I pass now to the criminal side of Newgate, which consisted of the six quarters or yards already enumerated and described. +DUMMY/LJ007-0020.wav|"swearing and talking of what crimes they had committed, and how they had done it." Another witness declared she had heard the most shocking language in the yard; she said +DUMMY/LJ042-0005.wav|After Oswald left the Marine Corps in September of nineteen fifty-nine, ostensibly to care for his mother, +DUMMY/LJ037-0154.wav|independently examined the four cartridge cases and arrived at the same conclusion as Cunningham. +DUMMY/LJ013-0152.wav|at Montreuil. He was arraigned at the Old Bailey, and the case fully proved. His sentence was seven years' transportation. +DUMMY/LJ012-0069.wav|Ikey joined her at Hobart Town, where they set up a general shop, and soon began to prosper. +DUMMY/LJ038-0229.wav|Either the viaduct or the underpass might have been the "bridge" mentioned in the last paragraph of the note. +DUMMY/LJ007-0012.wav|who not strangely resented the orders issued by the aldermen +DUMMY/LJ011-0240.wav|No more curious instance of this ever occurred than the assault made by one Howard upon a Mr. Mullay, with intent to rob him. +DUMMY/LJ040-0089.wav|In order to supplement their income further she falsely swore that Pic was seventeen years old so that he could join the Marine Corps Reserves. +DUMMY/LJ018-0166.wav|Buncher in London, and Griffiths in Birmingham. +DUMMY/LJ018-0300.wav|The sums obtained by Bidwell in the name of "Warren" at the Bank of England +DUMMY/LJ011-0261.wav|He was found guilty of an assault with intent, and sentenced to transportation for fourteen years. +DUMMY/LJ042-0046.wav|This should answer your question, and also give you a glimpse of my way of thinking. So you speak of advantages. Do you think that is why I am here? +DUMMY/LJ004-0057.wav|For the first time the doctrine was enunciated that prisoners had rights of their own. +DUMMY/LJ031-0119.wav|The fracture was set, and a cast was applied with some traction utilized. +DUMMY/LJ003-0277.wav|but just open (eighteen sixteen), the regular employment of prisoners had never yet been accepted as a principle in the metropolitan prisons. +DUMMY/LJ010-0141.wav|he was generally supposed to be a surgeon. +DUMMY/LJ008-0112.wav|He was death's counterfeit, tall, shriveled, and pale; +DUMMY/LJ001-0133.wav|One very important matter in "setting up" for fine printing is the "spacing," that is, the lateral distance of words from one another. +DUMMY/LJ004-0192.wav|The prison stood on a dry, airy situation outside the town. +DUMMY/LJ016-0349.wav|that of Alexander Mackay, who murdered his mistress at Norton Folgate by beating her with a rolling-pin and furnace-rake, +DUMMY/LJ038-0222.wav|The main post office, quote, on Ervay Street, end quote, refers to the post office where Oswald rented box two nine one five +DUMMY/LJ008-0217.wav|a throng of people whom neither rain, snow, storm, nor darkness ever hindered from attending the show. +DUMMY/LJ009-0153.wav|First came the sheriffs, the under-sheriffs, and several aldermen and city officials, then Lord Powerscourt and several other peers of the realm. +DUMMY/LJ007-0041.wav|The whole of these seventeen had, however, a common airing-yard, and took their exercise there at the same time, +DUMMY/LJ006-0219.wav|leap-frog, puss in the corner, and "fly the garter," for which purpose the rugs were spread out to prevent feet slipping on the floor. +DUMMY/LJ026-0015.wav|He says: Animals are organisms of fixed and definite form, in which the cell-body is not covered with a cellulose wall. +DUMMY/LJ047-0249.wav|The only witness to the conversation was Dallas Police Detective V. J. Brian, who was accompanying Revill. +DUMMY/LJ031-0192.wav|Return to Washington, D.C. +DUMMY/LJ049-0014.wav|The Commission regards such impromptu stops as presenting an unnecessary danger, +DUMMY/LJ042-0087.wav|He stated that he had volunteered to give Soviet officials any information that he had concerning Marine Corps operations, and intimated +DUMMY/LJ019-0132.wav|in the inquiry which followed, attention was again attracted to Newgate. +DUMMY/LJ015-0252.wav|When he knew that he could not escape his fate, +DUMMY/LJ028-0460.wav|Nebuchadnezzar, King of Babylon, the restorer of the temples Esagil and Ezida, +DUMMY/LJ014-0086.wav|This job was not completed till the following day, as the hole had to be enlarged, and the only tool they had was a dust-shovel. +DUMMY/LJ003-0224.wav|Supplies of common necessaries, such as have now been part of the furniture of every British jail for many years, +DUMMY/LJ046-0119.wav|as obtaining clearance of some categories of White House employees and all tradesmen who service the White House, +DUMMY/LJ050-0005.wav|The intimacy of the Secret Service's relationship to the White House +DUMMY/LJ039-0161.wav|and the Infantry Weapons Evaluation Branch of the U.S. Army. There were no misfires. +DUMMY/LJ031-0087.wav|When asked why he did not turn the President over, Dr. Carrico testified as follows: +DUMMY/LJ046-0065.wav|and allow them to see him, and discuss, if possible, the views of the world as he sees it, the problems of the country as he sees them. +DUMMY/LJ033-0137.wav|It was not a standard type bag which could be obtained in a store and it was presumably made for a particular purpose. +DUMMY/LJ016-0425.wav|When visited one day in the condemned cell, just as St. Sepulchre's clock was striking, he looked up and said laughingly, "Go along, clock; +DUMMY/LJ042-0156.wav|Oswald prefaced his manuscript with a short autobiographical sketch which reads in part as follows, quote, +DUMMY/LJ008-0216.wav|until presently the huge stage loomed dark above the crowd which was now ranged round the barriers; +DUMMY/LJ034-0118.wav|that the description of the suspect in the Tippit shooting was similar to the description which had been given by Brennan in connection with the assassination. +DUMMY/LJ003-0300.wav|that all prisoners should always be in separate cells by night, and those of short sentences by day. +DUMMY/LJ033-0009.wav|Oswald's arrival at the Depository Building on November twenty-two, carrying a long and bulky brown paper package, +DUMMY/LJ021-0189.wav|Nearly all Americans are sensible and calm people. +DUMMY/LJ045-0013.wav|There is, of course, no way to judge what the effect of the general political ferment present in that city might have been, even though Oswald was aware of it. +DUMMY/LJ004-0200.wav|Another jail, that of Ilchester, was also worthy of all commendation. It exhibited all the good points of that at Bury. +DUMMY/LJ012-0132.wav|and he with his father arranged that a messenger should call for the stuff with forged credentials, and anticipating the rightful owner. +DUMMY/LJ029-0161.wav|From the airport, the President's party will proceed to Mockingbird Lane to Lemmon and then to Turtle Creek, turning south to Cedar Springs. +DUMMY/LJ041-0188.wav|The eyes of the future became the eyes of God. +DUMMY/LJ030-0144.wav|Special Agent Rufus W. Youngblood observed that the large electric sign clock atop the Texas School Book Depository Building +DUMMY/LJ004-0107.wav|The most important jail act of that early period, however, was the twenty-four George the third c. fifty-four, s. four (seventeen eighty-four) +DUMMY/LJ014-0067.wav|His wealth was well known to "Maria," as he called Mrs. Manning, who made several ineffectual attempts to get money out of him. +DUMMY/LJ049-0111.wav|The activities of the victim at the time an assassination occurs and the motive for the assassination +DUMMY/LJ047-0085.wav|The FBI had not asked to be informed of any effort by Oswald to obtain a passport, +DUMMY/LJ015-0197.wav|which at one time spread terror throughout London. Thieves preferred now to use ingenuity rather than brute force. +DUMMY/LJ032-0034.wav|with that appearing on documents known to have been written by Oswald, such as his letters, passport application, and endorsements of checks. +DUMMY/LJ024-0065.wav|Such a succession of appointments should have provided a Court well-balanced as to age. +DUMMY/LJ002-0080.wav|The long narrow yard still remained as we saw it in Jacobite times, +DUMMY/LJ004-0151.wav|They were far worse at Bristol, +DUMMY/LJ017-0198.wav|but its nature was betrayed when this last victim also died. +DUMMY/LJ047-0034.wav|and instructed the Dallas office to interview him when he got back to determine whether he had been recruited by a Soviet intelligence service. +DUMMY/LJ021-0024.wav|all present new questions for the solution of which the old reliance upon the free action of individual wills appears quite inadequate. +DUMMY/LJ025-0164.wav|iron and the like in combination with phosphoric, sulphuric and other acids. +DUMMY/LJ004-0188.wav|and which even fulfilled many of the exacting requirements of modern days. +DUMMY/LJ028-0295.wav|I maimed myself because I could not endure that the Assyrians should laugh at the Persians. "Wretched man," said Darius, +DUMMY/LJ044-0181.wav|The Cubans would not, however, give him a visa until he had received one from the Soviets, which involved a delay of several months. +DUMMY/LJ003-0227.wav|So unjust and unequal was the system, that the allowance to convicted criminals was better than that of the innocent debtor, +DUMMY/LJ044-0143.wav|Those events no doubt had their effects on Oswald. +DUMMY/LJ042-0076.wav|He took charge, in a sense, of the conversation right from the beginning, end quote. He presented the following signed note: +DUMMY/LJ039-0188.wav|hit the target twice within a span of four point six and five point one five seconds. +DUMMY/LJ021-0169.wav|Those, fortunately few in number, who are frightened by boldness and cowed by the necessity for making decisions, +DUMMY/LJ010-0198.wav|His father, it was proved in evidence, had been at times quite mad; and Oxford's mental state might be inferred from his own proceedings. +DUMMY/LJ015-0167.wav|Redpath had industriously circulated reports that he had prospered greatly in speculation; +DUMMY/LJ010-0124.wav|Five of the conspirators, Thistlewood, Ings, Brunt, Davidson, and Tidd, were sentenced to death, +DUMMY/LJ035-0132.wav|When she reached the first floor, she actually saw Shelley and Lovelady slightly east of the east elevator. +DUMMY/LJ028-0326.wav|and on his suing for a command, they entrusted to him a body of troops with the help of which he proceeded to do as he had arranged with Darius. +DUMMY/LJ025-0052.wav|the greatest and most rapid revolution which biological science has ever undergone was effected by the application of the modern microscope +DUMMY/LJ006-0102.wav|This, too, was in the depth of the winter, and during a most inclement season. +DUMMY/LJ039-0072.wav|From my own experience in shooting over the years, +DUMMY/LJ042-0095.wav|to attend a university in Moscow, but they gave him a monthly allowance of seven hundred rubles a month +DUMMY/LJ046-0039.wav|As the ceremonial head of the Government the President must discharge a wide range of public duties, not only in Washington but throughout the land. +DUMMY/LJ019-0157.wav|sharp tests which generally broke down the fidelity of the inmates of the ward to one another. +DUMMY/LJ002-0201.wav|He seems to have felt no responsibility as to the welfare or comfort of those in charge, and out of whom he made all his money. +DUMMY/LJ019-0152.wav|and on entering it found the prisoners engaged in kicking bundles of wood from one end of the ward to the other. +DUMMY/LJ039-0210.wav|Frazier testified that while he could not tell when the defect occurred, but that a person familiar with the weapon could compensate for it. +DUMMY/LJ020-0080.wav|Pinch the dough hard to stop the mouths of these cavities, +DUMMY/LJ003-0185.wav|There was no separation even for the women under sentence of death, who lived in a common and perpetually crowded ward. +DUMMY/LJ003-0026.wav|The auditors were paid a shilling each for their services each time the poor-box was opened. The steward was also remunerated for his trouble. +DUMMY/LJ046-0038.wav|The President is Head of State, Chief Executive, Commander in Chief, and leader of a political party. +DUMMY/LJ039-0149.wav|established that they had been previously loaded and ejected from the assassination rifle, +DUMMY/LJ007-0023.wav|The poor, who could afford no luxuries, went to bed early, but were kept awake by the revelries of the rich, +DUMMY/LJ042-0161.wav|Immediately after serving out his three years in the U.S. Marine Corps, he abandoned his American life to seek a new life in the USSR. +DUMMY/LJ028-0424.wav|Porter, Layard, and Rawlinson followed him, but the real scientific exploration of Babylon and its walls +DUMMY/LJ028-0367.wav|The priests of the temple of Bel, so a story tells us, +DUMMY/LJ002-0075.wav|and who are therefore lodged apart from all other districts of the jail. End quote. +DUMMY/LJ048-0220.wav|Conduct of Secret Service agents in Fort Worth on November twenty-two. +DUMMY/LJ033-0196.wav|The green cotton fibers found in the paper bag matched some of the green cotton fibers in the blanket +DUMMY/LJ008-0251.wav|was an hour before kicked to death by the savage and brutal mob. +DUMMY/LJ039-0054.wav|Oswald's Rifle Capability +DUMMY/LJ037-0166.wav|He concluded, however, that he could not say whether the four bullets were fired from the revolver in Oswald's possession. +DUMMY/LJ029-0053.wav|which advise it of potential threats immediately before the visit of the President to their community. +DUMMY/LJ002-0089.wav|eight. The female felons were deprived of part of the space which the architect had intended for them. +DUMMY/LJ017-0181.wav|who had been in the habit of taking colchicum for rheumatism. Mrs. Wilson heard then casually from a medical man +DUMMY/LJ048-0101.wav|The Commission concludes that the most significant advance arrangements for the President's trip were soundly planned. +DUMMY/LJ027-0070.wav|So much concerns the general plan of skeletal structures and is strongly suggestive of -- in fact it is inexplicable without -- common origin. +DUMMY/LJ007-0143.wav|"is permitted to purchase whatever his own means or the means of his friends in or out of prison can afford, +DUMMY/LJ040-0061.wav|Various possible motives will be treated in the appropriate context of the discussion outlined above. +DUMMY/LJ042-0249.wav|Even though his primary purpose probably was to get to Cuba, +DUMMY/LJ010-0143.wav|This axe is still in existence, and is preserved at Newgate with various other unpleasant curiosities, +DUMMY/LJ013-0099.wav|A smart detective, Forrester, after a little inquiry, +DUMMY/LJ032-0219.wav|They are photographs of the same scene. +DUMMY/LJ029-0008.wav|the activities at Parkland Memorial Hospital, and the return of the Presidential party to Washington. +DUMMY/LJ037-0060.wav|That Mrs. Markham described the man who killed Patrolman Tippit as, quote, short, a little on the heavy side, end quote. +DUMMY/LJ016-0363.wav|This feeling was the stronger because +DUMMY/LJ008-0116.wav|and tied the cord with so little feeling that the governor (Wall), who had not given the wretch his accustomed fee, observed, +DUMMY/LJ040-0227.wav|Mrs. Siegel concluded her report with the statement that, quote, +DUMMY/LJ015-0032.wav|and on the other by backing up +DUMMY/LJ018-0018.wav|he wore gold-rimmed eye-glasses and a gold watch and chain. +DUMMY/LJ035-0076.wav|Special Agent John Howlett of the Secret Service carried a rifle from the southeast corner of the sixth floor along the east aisle to the northeast corner. +DUMMY/LJ017-0165.wav|As she became worse and worse, Smethurst called in the best medical advice in Richmond, +DUMMY/LJ022-0084.wav|I next want to make it clear exactly how we shall direct the work. +DUMMY/LJ035-0141.wav|If Miss Adams accurately recalled meeting Shelley and Lovelady when she reached the bottom of the stairs, +DUMMY/LJ010-0236.wav|The deed was long premeditated, and would have been executed a day earlier had not his courage failed him at the last moment. +DUMMY/LJ031-0066.wav|the doctors were able to maintain peripheral circulation as monitored at the neck (carotid) artery and at the wrist (radial) pulse. +DUMMY/LJ014-0012.wav|It was here that the others engaged in their dismal office in removing the dead first got a good look at the stranger's face. +DUMMY/LJ009-0098.wav|There! there! I see the Lamb of God! Oh! how happy! Oh! this is happy! +DUMMY/LJ018-0196.wav|He was a notorious criminal, +DUMMY/LJ038-0161.wav|and he asked me did I know which way he was coming, and I told him, yes, he probably come down Main and turn on Houston and then back again on Elm. +DUMMY/LJ039-0046.wav|wasn't the day before. Perhaps three days before, end quote. +DUMMY/LJ033-0108.wav|In deciding whether Oswald carried the assassination weapon in the bag which Frazier and Mrs. Randle saw, +DUMMY/LJ018-0219.wav|he instructed certain solicitors to prepare a deed of gift as from his father, conveying to him estates near Kingston. +DUMMY/LJ040-0096.wav|Pic said, quote, +DUMMY/LJ034-0032.wav|had been one of these moved by the floor layers from the west wall to near the east side of the building in preparation for the laying of the floor. +DUMMY/LJ030-0085.wav|The two agents on the rear of the running boards were to advance toward the front of the President's car whenever it stopped or slowed down sufficiently for them to do so. +DUMMY/LJ023-0030.wav|Four years ago action did not come until the eleventh hour. +DUMMY/LJ044-0103.wav|The Worker replied, quote, Your kind offer is most welcomed and from time to time we shall call on you, end quote. +DUMMY/LJ050-0264.wav|The Commission recommends that the present arrangements +DUMMY/LJ011-0071.wav|but there is good evidence to show that the body after execution was handed over to his friends and interred privately. +DUMMY/LJ044-0195.wav|was another act which expressed his hostility toward the United States and its institutions as well as a concomitant attachment +DUMMY/LJ050-0088.wav|to further the intention to satisfy a grievance by unlawful means. +DUMMY/LJ042-0190.wav|democratic socializing of production, and without regard to the twisting apart of Marxism Marxist Communism by other powers. +DUMMY/LJ007-0049.wav|they suffered sickening anxiety, and wished to be locked up alone. This indiscriminate association lasted for months, +DUMMY/LJ044-0152.wav|on February seventeen, nineteen sixty-three, for permission for herself and June to return to the Soviet Union. +DUMMY/LJ044-0227.wav|Although Oswald could possibly have been motivated in part by his sympathy for the Castro government, +DUMMY/LJ038-0005.wav|from where several witnesses last saw Oswald running west on Jefferson Boulevard. Shortly after the Tippit murder, +DUMMY/LJ034-0176.wav|all the time I watched him, he never moved his head, he never -- he never moved anything. Just was there transfixed. +DUMMY/LJ016-0405.wav|went impenitent to the gallows. The only sign of feeling he showed was in asking to be allowed to choose the hymns on the Sunday +DUMMY/LJ023-0119.wav|that will refuse to amend the Constitution by the arbitrary exercise of judicial power -- amended by judicial say-so. +DUMMY/LJ045-0121.wav|it does not appear that he ever lost his job because of its activities, although he may well not have been aware of that fact. +DUMMY/LJ031-0020.wav|two rooms were prepared. +DUMMY/LJ035-0201.wav|The fact that Truly found Fritz in the northwest corner of the floor, near the point where the rifle was found, supports Fritz' recollection. +DUMMY/LJ003-0015.wav|For this and other acts of misconduct there was the discipline of the refractory ward, or "strong room" on the debtors' side. +DUMMY/LJ044-0043.wav|he stated that he was then alone in his efforts on behalf of FPCC, but he attributed his lack of support +DUMMY/LJ003-0210.wav|at night they were placed in the fifteen cells, two, three, or more together, according to the total number to be accommodated. +DUMMY/LJ046-0235.wav|Except for its special "trip index" file of four hundred names, +DUMMY/LJ022-0028.wav|I am reminded sometimes of what President Wilson once said: +DUMMY/LJ019-0251.wav|Yet this principle, as the committee pointed out, "must now be accepted as the foundation of prison discipline," +DUMMY/LJ044-0114.wav|he received a letter from somebody in New York, some Communist -- probably from New York -- I am not sure from where -- from some Communist leader and he was very happy, +DUMMY/LJ014-0084.wav|Returning to her own home, where Manning meantime had been calmly smoking and talking to the neighbors over the basement wall, +DUMMY/LJ043-0155.wav|She later gave the following testimony. Question: +DUMMY/LJ023-0010.wav|we asked the nation to turn over all of its privately held gold, dollar for dollar, to the government of the United States. +DUMMY/LJ046-0180.wav|PRS also maintains an album of photographs and descriptions of about twelve to fifteen individuals who are regarded as clear risks to the President +DUMMY/LJ029-0205.wav|for President Kennedy, stating that "in many respects Dallas County has isolated itself from the main stream of life in the world in this decade. +DUMMY/LJ018-0025.wav|More precise clues to the murderer were not long wanting; indeed the readiness with which they were produced and followed up +DUMMY/LJ020-0054.wav|Rub the bottom and sides of a baking-pan with sweet lard or butter. Do this with a bit of clean soft rag or tissue-paper, +DUMMY/LJ044-0167.wav|He asked his wife to help him to hijack an airplane to get there, but gave up that scheme when she refused. +DUMMY/LJ016-0395.wav|The diet of the condemned is the ordinary diet of the prison, +DUMMY/LJ041-0100.wav|finally attaining a male status or image in his own eyes, end quote. That apparently caused Oswald to become more self-confident, +DUMMY/LJ032-0257.wav|When she started to open the blanket, she saw the stock of the rifle. +DUMMY/LJ040-0118.wav|and threatened to use it on Mrs. Pic. +DUMMY/LJ028-0431.wav|The city walls, of which the ancients were so proud, appear here and there like low ridges far out on the plain; +DUMMY/LJ004-0003.wav|other earnest people, inspired doubtless by her noble example, were stirred up to activity in the same great work. +DUMMY/LJ013-0065.wav|and that he gained access to Miss Slack on pretense of conveying to her some funded property left her by an aunt. +DUMMY/LJ043-0108.wav|Oswald's employment problems became more difficult. He left his wife and child at the home of a friend, Mrs. Ruth Paine, of Irving, Texas. +DUMMY/LJ003-0023.wav|All the allowances of food passed through his hands; he had the control of the poor-box for chance charities, +DUMMY/LJ050-0273.wav|The Commission has, however, from its examination of the facts of President Kennedy's assassination +DUMMY/LJ014-0283.wav|The jury found him guilty of the latter only, with a point of law reserved. This was fully argued before three judges, +DUMMY/LJ030-0193.wav|Kellerman saw Governor Connally in his wife's lap and Special Agent Clinton J. Hill lying across the trunk of the car. +DUMMY/LJ002-0068.wav|Here were also lodged the gatesmen, the prisoners who had charge of the inner gates, and who were entrusted with the duty of escorting visitors from the gates +DUMMY/LJ023-0069.wav|with national power, intended as they said, "to form a more perfect union +DUMMY/LJ037-0067.wav|the transcript establishes that she was referring to the uncombed state of his hair, a description fully supported by a photograph of Oswald +DUMMY/LJ018-0359.wav|her object being first to dispose of the wife of a man for whom she had conceived a guilty passion, +DUMMY/LJ013-0202.wav|Coolness amounting almost to effrontery gave way to hopeless dejection. +DUMMY/LJ022-0171.wav|designed to improve the status of our transportation agencies. +DUMMY/LJ019-0126.wav|The creation of an expensive staff for supervision, +DUMMY/LJ034-0014.wav|These two small cartons had been moved from a stack about three aisles away. +DUMMY/LJ022-0188.wav|Certain proposals made to amend the Federal Reserve Act deserve prompt and favorable action by the Congress. +DUMMY/LJ014-0052.wav|In November of the first-named year occurred the wholesale slaughter of the Jermys in their house, Stanfield Hall, by the miscreant Rush. +DUMMY/LJ009-0135.wav|So many applications for admission were made to the sheriffs, +DUMMY/LJ036-0055.wav|Mrs. Bledsoe identified the shirt as the one Oswald was wearing and she stated she was certain that it was Oswald who boarded the bus. +DUMMY/LJ018-0115.wav|It was stated in evidence that the monies obtained by these forgeries amounted to eight thousand pounds or ten thousand pounds, +DUMMY/LJ011-0140.wav|in other words, they had advanced in eighteen thirty-two to the point at which the Lower House had arrived in eighteen thirty. +DUMMY/LJ046-0155.wav|under the name of the individual or group of individuals to whom that material related. End quote. +DUMMY/LJ045-0180.wav|and which he was to take to the School Book Depository. The Unanswered Questions. +DUMMY/LJ018-0139.wav|Soon afterwards the inspectors of bank-notes at the Bank of England detected the presentation at the bank of spurious notes on genuine paper. +DUMMY/LJ020-0014.wav|beating the batter smooth as you go on until all of the liquid and flour has gone in. +DUMMY/LJ018-0142.wav|By this time Brown alone had stolen three or four hundred sheets, +DUMMY/LJ042-0069.wav|I think to myself "How easy to Die" and "A Sweet Death, (to violins), end quote. +DUMMY/LJ005-0039.wav|constant inspection, regular employment, and humane treatment generally, with religious and moral instruction. +DUMMY/LJ014-0062.wav|He had failed in this as well as in the business of a publican, which he had at one time adopted. +DUMMY/LJ001-0179.wav|even when the woodcuts are very rude indeed, +DUMMY/LJ003-0163.wav|the misdemeanant tried or untried, the debtor who wished to avoid the discomfort of the crowded debtors' side, the outspoken newspaper editor, +DUMMY/LJ034-0028.wav|The "Rolling Readers" cartons, however, had not been moved by the floor layers and had apparently been taken to the window from their regular position for some particular purpose. +DUMMY/LJ039-0098.wav|Oswald's Marine Training +DUMMY/LJ009-0031.wav|The extract is from Mr. E. Gibbon Wakefield's brochure, the date eighteen twenty-eight, just three years after Fauntleroy's death. +DUMMY/LJ034-0022.wav|testified that twenty identifiable fingerprints and eight palmprints were developed on these cartons. +DUMMY/LJ027-0169.wav|such as the triton, the salamander, etc., which are therefore called "caducibranchs," but the frog still passes on. +DUMMY/LJ016-0326.wav|Lord Advocate, Mr. Charles Neate, Mr. William Ewart, and last, but not least, Mr. John Bright +DUMMY/LJ019-0294.wav|The prison buildings were in many places out of repair; other houses often overlooked them. +DUMMY/LJ046-0031.wav|The protective task is further complicated by the reluctance of Presidents to take security precautions which might interfere with the performance of their duties, +DUMMY/LJ018-0171.wav|"Mother plates" for engraving the body of the notes lay about, and other plates for various processes. +DUMMY/LJ049-0071.wav|If the advance preparation is thorough, and the protective devices and techniques employed are sound, +DUMMY/LJ019-0343.wav|they were not to be turnkeys or assistant turnkeys, neither wardsman nor yardsman, overseer, +DUMMY/LJ045-0182.wav|Instead of returning to Irving on November fifteen for his customary weekend visit, he remained in Dallas at his wife's suggestion because of the birthday party. +DUMMY/LJ017-0195.wav|and that Mrs. Wilson administered the same medicine again and again. +DUMMY/LJ047-0123.wav|It was apparent that he had made a self-serving statement to Agent Quigley. It became a matter of record in our files as a part of the case, +DUMMY/LJ049-0020.wav|It was not then replaced because the state of the art did not permit the development of a bulletproof top of sufficiently light weight +DUMMY/LJ048-0187.wav|In addition, Secret Service agents riding in the motorcade were trained to scan buildings as part of their general observation of the crowd of spectators. +DUMMY/LJ033-0044.wav|In the garage were the personal belongings of the Oswald family including, as the evidence has shown, the rifle wrapped in the old brown and green blanket. +DUMMY/LJ050-0130.wav|Finally, these agreements with Federal and local authorities will be of little value +DUMMY/LJ024-0126.wav|My record as Governor and President proves my devotion to those liberties. +DUMMY/LJ008-0016.wav|Lawrence Jones, a burglar, was in seventeen ninety-three ordered for execution in Hatton Garden, near the house he had robbed; +DUMMY/LJ043-0101.wav|or whether he brought the Russian language newspaper with him one day after his other difficulties became clear. +DUMMY/LJ013-0044.wav|and tempted him to take to fraudulent insurance as a certain method of achieving fortune. +DUMMY/LJ044-0229.wav|and had lost his desire to do so because of the bureaucracy and red tape which he had encountered. +DUMMY/LJ022-0197.wav|We have survived all of the arduous burdens and the threatening dangers of a great economic calamity. +DUMMY/LJ010-0120.wav|to commit outrage, to set fire to houses, throw hand-grenades into the carriages of ministers; +DUMMY/LJ048-0034.wav|no law enforcement agency had any information to connect Oswald with the attempted shooting of General Walker. +DUMMY/LJ020-0050.wav|Toss the lump dough upon it and knead thoroughly for five minutes. +DUMMY/LJ049-0200.wav|It is suggested that an organization shorn of its power to investigate all the possibilities of danger to the President +DUMMY/LJ024-0135.wav|liberal-minded judiciary necessary to furnish quicker and cheaper justice from bottom to top. +DUMMY/LJ005-0071.wav|differed little from that kept in view in the construction of the latest and best modern jails. +DUMMY/LJ048-0083.wav|were discussed twice officially by the special agent in charge of the FBI office in Dallas. As discussed in chapter two, +DUMMY/LJ004-0232.wav|Ironing it will be remembered, was a distinguishing badge, so that when the jail was cleared the free might be readily known from the captive, and escapes prevented. +DUMMY/LJ004-0008.wav|Mr. Neild, a second Howard, +DUMMY/LJ028-0213.wav|Nor does the royal record of Babylon contain the only contemporary account of the fall of the city, +DUMMY/LJ008-0184.wav|Precautions had been taken by the erection of barriers, and the posting of placards at all the avenues to the Old Bailey, on which was printed, +DUMMY/LJ005-0085.wav|Classification was insisted upon, in the manner laid down by the twenty-four George the third cap. fifty-four, +DUMMY/LJ012-0182.wav|After the murder the villains divided the spoil, and went on to Probert's cottage, and supped off pork-chops brought down on purpose. +DUMMY/LJ042-0099.wav|gave Oswald an income which he said approximated that of the director of the factory in which he worked. +DUMMY/LJ042-0214.wav|to the bitter destruction of all and everything. I am quite sure these people must hate not only the government +DUMMY/LJ016-0026.wav|managed to get away in the very same spot where his namesake had nine years before so miserably failed. +DUMMY/LJ044-0146.wav|On June twenty-four, nineteen sixty-three, he applied for a new passport +DUMMY/LJ045-0207.wav|He could not keep them with him in Dallas, where at least he could see his children whom, several witnesses testified, he seemed to love. +DUMMY/LJ018-0149.wav|Buncher was then tracked to North Kent Terrace, New Cross, where a Mr. and Mrs. Campbell resided, +DUMMY/LJ018-0082.wav|he had yielded to an irresistible impulse to kill him. +DUMMY/LJ016-0253.wav|that the ceremony may be made more mechanical, thus rendering the personal intervention of a skilled functionary unnecessary. +DUMMY/LJ025-0130.wav|Touch one of them with the end of a fine human hair +DUMMY/LJ003-0004.wav|No remonstrance was attended to, +DUMMY/LJ003-0232.wav|These were fetched away in a large tub on a truck by a turnkey. +DUMMY/LJ023-0041.wav|It is a quiet crisis. There are no lines of depositors outside closed banks. +DUMMY/LJ032-0132.wav|On November twenty-two, however, before surrendering possession of the rifle to the FBI Laboratory, +DUMMY/LJ003-0075.wav|thirty-five tried and thirty-seven untried. Of the former, three were transports for life, +DUMMY/LJ008-0226.wav|the latter unsexed and brutalized by dissipation, +DUMMY/LJ017-0261.wav|Seven were found guilty of murder on the high seas, and one, Carlos, acquitted. +DUMMY/LJ005-0234.wav|The visits of friends was once more unreservedly allowed, and these incomers freely brought in extra provisions and beer. +DUMMY/LJ038-0028.wav|Brewer met McDonald and the other policemen at the alley exit door, +DUMMY/LJ014-0026.wav|running at a dog-trot into London, and others swore that they plainly recognized him as the man seen soon afterwards in the lane. +DUMMY/LJ024-0036.wav|If by that phrase "packing the Court" it is charged that I wish to place on the bench spineless puppets +DUMMY/LJ014-0234.wav|He died impenitent, declaring that he had no belief, and that it was idle to ask forgiveness of God. +DUMMY/LJ009-0035.wav|in the other opposite were the convicts capitally convicted who had been respited. +DUMMY/LJ012-0178.wav|Thurtell drove him down in a gig, "to be killed as he traveled," in Thurtell's own words. +DUMMY/LJ009-0157.wav|But there is little doubt that as executions became more rare they made more impression on the public mind. +DUMMY/LJ016-0446.wav|but ineffectively, by running his head against his cell wall. +DUMMY/LJ018-0324.wav|much uneasiness was caused at one of the convict prisons by information that bribery on a large scale was again at work amongst the officials. +DUMMY/LJ032-0209.wav|He stated, however, that while he, quote, found no differences, end quote, between the rifles in the two photographs, he could not make a, quote, positive identification +DUMMY/LJ030-0003.wav|Visits to Other Texas Cities +DUMMY/LJ002-0091.wav|and what remained was divided into a master's and a common side for female felons. +DUMMY/LJ006-0048.wav|To these were still added an average of about fifty expecting the last penalty of the law; a certain number of transports awaiting removal to the colonies; +DUMMY/LJ005-0300.wav|and the establishment of municipal institutions upon a proper footing, +DUMMY/LJ009-0104.wav|the women set up a yell, which is mixed with a rustling noise, occasioned by the removal of those whose hysterics have ended in fainting. +DUMMY/LJ016-0042.wav|The condition of the stone surface just mentioned assisted him in this, and he managed to get beyond the cistern to the railing below the chevaux-de-frise. +DUMMY/LJ028-0021.wav|Again, long before the stars have been scattered by the morning sun, you are on your way. +DUMMY/LJ016-0375.wav|As this burial-ground leads to the adjacent Central Criminal Court, accused murderers, on going to and returning from trial, +DUMMY/LJ019-0363.wav|he could close the "inadequate" prison, by declaring it unfit for the reception of prisoners. +DUMMY/LJ013-0111.wav|Burgess was eventually brought back to England, tried at the Central Criminal Court, and sentenced to transportation for life. +DUMMY/LJ044-0010.wav|April six to twenty-four, nineteen sixty-three Oswald's first public identification with that cause was in New Orleans. +DUMMY/LJ028-0154.wav|at the point where the city of the same name stands, eight days' journey from Babylon. +DUMMY/LJ029-0179.wav|Although there were critical editorials and letters to the editors, the news stories reflected the desire of Dallas officials to welcome the President with dignity and courtesy. +DUMMY/LJ024-0113.wav|The other groups is composed of those who honestly believe the amendment process is the best +DUMMY/LJ005-0158.wav|Bedding and clothing was still denied, but only in a few jails; +DUMMY/LJ043-0181.wav|that such concern is an important factor to consider in assessing possible motivation for the assassination. +DUMMY/LJ028-0322.wav|Thus did Zopyrus speak. +DUMMY/LJ024-0137.wav|and unwilling to assert legislative powers by writing into it their own political and economic policies. +DUMMY/LJ016-0191.wav|was given to drink, and not to be trusted. +DUMMY/LJ004-0020.wav|contrast ludicrously with the prison architecture based upon a century's experience of our own age. +DUMMY/LJ034-0188.wav|On November twenty-two, Fischer had apparently described the man as, quote, light-headed, end quote. +DUMMY/LJ046-0147.wav|were broad and flexible. All material is and was desired, accepted, and filed if it indicated or tended to indicate +DUMMY/LJ006-0215.wav|The beer-man brought in as much as he pleased; he sold it without the controlling presence of an officer. +DUMMY/LJ050-0014.wav|The incumbent has no technical qualifications in the area of Presidential protection. +DUMMY/LJ035-0013.wav|Having recently heard the sounds of rifles while on a hunting trip, Baker recognized the shots as that of a high-powered rifle; Quote, +DUMMY/LJ034-0143.wav|but he said he was unable to make a positive identification. +DUMMY/LJ047-0065.wav|This information led Hosty to review Oswald's file, from which he learned that Oswald had become a subscriber to the Worker, +DUMMY/LJ041-0084.wav|fitted into a general personality pattern of his: to do whatever was not wanted of him, a recalcitrant trend in his personality, end quote, +DUMMY/LJ025-0097.wav|Cellulose is widespread as a constituent of the skeletons of the lower animals +DUMMY/LJ030-0255.wav|approximately twenty or twenty-five minutes after the shots were fired. +DUMMY/LJ036-0063.wav|across Houston in front of the Depository Building, past the Triple Underpass into west Dallas, and south on Beckley. +DUMMY/LJ028-0375.wav|Slowly the few remaining walls fell, and were buried in their own ruins. +DUMMY/LJ040-0081.wav|for the relations between Marguerite Oswald and Ekdahl were stormy and they were finally divorced, after several separations and reunions, +DUMMY/LJ047-0052.wav|Having concluded that Oswald was not a security risk or potentially dangerous or violent, +DUMMY/LJ045-0098.wav|Nobody said anything good about him. End quote. +DUMMY/LJ046-0022.wav|which recommends additional personnel and facilities to enable the Secret Service to expand its protection capabilities. +DUMMY/LJ025-0115.wav|and from that day to this the rapid improvement of methods of investigation and the energy of a host of accurate observers +DUMMY/LJ048-0064.wav|While the Secret Service bears the principal responsibility for this failure, +DUMMY/LJ015-0222.wav|in such a way that Tester had possession of this key for a time. +DUMMY/LJ037-0203.wav|S and W Special two-inch Commando, serial number V five one zero two one zero, end quote, +DUMMY/LJ047-0031.wav|it learned also that Oswald had been discharged from the Marine Corps Reserve as undesirable in August nineteen sixty. +DUMMY/LJ006-0131.wav|and, if he chose, pass in, all provisions, money, clothes, and letters brought for prisoners by their friends. +DUMMY/LJ004-0177.wav|No prison dress was allowed; no reception-room was provided, no soap, towels, or baths. +DUMMY/LJ028-0179.wav|It is only from their ruins that we may hope to obtain accurate information of the strongest fortifications in the ancient world. +DUMMY/LJ014-0312.wav|His method was to obtain advances twice over on the same bills of lading or corn warrants. The duplicates were forged. +DUMMY/LJ006-0039.wav|and limited it to Newgate alone. Newgate indeed became the sole theme of their first report. +DUMMY/LJ045-0101.wav|In spite of these difficulties, however, and in the face of the economic problems that were always with them, +DUMMY/LJ044-0083.wav|was greatly impeded by the fact that the radio debate over WDSU on August twenty-one, nineteen sixty-three, +DUMMY/LJ033-0052.wav|The period between eight and nine p.m., however, provided ample opportunity for Oswald to prepare the rifle for his departure the next morning. +DUMMY/LJ031-0188.wav|Members of the Presidential and Vice-Presidential parties filled the central compartment of the plane to witness the swearing in. +DUMMY/LJ047-0014.wav|the Office of Naval Intelligence, the FBI and the CIA. The information known to the FBI is summarized below. +DUMMY/LJ015-0087.wav|entertained literary, artistic, and dramatic celebrities; had a smart "turn out," attended all the race-meetings, and dressed in the latest fashion. +DUMMY/LJ038-0058.wav|Officer Ray Hawkins said +DUMMY/LJ008-0019.wav|Again, as late as eighteen oh nine and eighteen twelve, Execution Dock, on the banks of the Thames, was still retained. +DUMMY/LJ007-0068.wav|generally sided with his opponents. Nevertheless the inspectors summed up against him. +DUMMY/LJ032-0054.wav|"A. Hidell, P.O. Box two nine one five, Dallas, Texas," on March twenty, nineteen sixty-three. +DUMMY/LJ028-0126.wav|Berossus, a priest of the temple of Bel at Babylon, writing about two fifty B.C., +DUMMY/LJ004-0223.wav|Mr. Brown confidently asserted that no jail in England now fed its inmates so well as did Newgate. +DUMMY/LJ049-0225.wav|The Commission feels the Secret Service and the FBI, as well as the State Department and the CIA when the President travels abroad, +DUMMY/LJ019-0301.wav|Some idea of the comparative uselessness of these small borough prisons was conveyed by some figures quoted by the committee. +DUMMY/LJ032-0256.wav|Marina Oswald was looking in the garage for parts to the baby's crib and thought that the parts might be in the blanket. +DUMMY/LJ026-0032.wav|as it were, into an assemblage of organisms which partake of the characters of both kingdoms and cannot without a certain violence +DUMMY/LJ046-0014.wav|at the time of President Kennedy's assassination. +DUMMY/LJ037-0156.wav|also examined the four cartridge cases found near the site of the homicide and compared them with the test cartridge cases fired from the Smith and Wesson revolver +DUMMY/LJ040-0144.wav|potentially dangerous, that, quote, his outlook on life had strongly paranoid overtones, end quote, or that he should be institutionalized. +DUMMY/LJ039-0057.wav|possessed the capability to hit his target with two out of three shots under the conditions described in chapter three. +DUMMY/LJ046-0115.wav|Adequacy of preventive intelligence operations of the Secret Service. +DUMMY/LJ012-0134.wav|Presently the proper person arrived from the consignees, but found the gold-dust gone. +DUMMY/LJ016-0027.wav|Escape seemed absolutely hopeless, +DUMMY/LJ029-0033.wav|particularly in large cities where the purpose was to let the President be seen by as many people as possible. +DUMMY/LJ007-0073.wav|to that line of conduct which his duty imposed on him +DUMMY/LJ030-0219.wav|She turned toward me and I grabbed her and put her back in the back seat, crawled up on top of the back seat and lay there. +DUMMY/LJ019-0288.wav|In them, according to the committee, the old objectionable practices were still in full force. +DUMMY/LJ042-0168.wav|He must be opposed to their basic foundations and representatives, end quote. +DUMMY/LJ025-0158.wav|the former will grow down into roots, the latter grow up into the stem and leaves of a vigorous bean-plant, +DUMMY/LJ005-0283.wav|and the tendency to crime in offenders not entirely hardened is confirmed by the language, the suggestions, and the example +DUMMY/LJ048-0213.wav|watching his car, watching the sides, watching the crowds, giving advice or asking advice from the Chief +DUMMY/LJ002-0291.wav|I come now to the debtors in Newgate. The quarters they occupied were divided, as I have said, into three principal divisions +DUMMY/LJ047-0043.wav|Oswald, quote, denied that he had ever denounced his U.S. citizenship, and that he had ever applied for Soviet citizenship specifically, end quote. +DUMMY/LJ023-0120.wav|It does not mean a judiciary so independent that it can deny the existence of facts which are universally recognized. +DUMMY/LJ023-0083.wav|The Court has been acting not as a judicial body, but as a policy-making body. +DUMMY/LJ021-0035.wav|saved debtors and creditors alike in many other fields of enterprise, such as loans on farm mortgages and home mortgages; +DUMMY/LJ008-0110.wav|The prisoners had not risen. +DUMMY/LJ034-0094.wav|whom he viewed in a police lineup on the night of the assassination, was the man he saw fire the shots from the sixth-floor window of the Depository Building. +DUMMY/LJ034-0058.wav|the Commission considered the possibility that Oswald handled these cartons as part of his normal duties. +DUMMY/LJ014-0335.wav|which were soon developed, much to that firm's subsequent anger and regret. +DUMMY/LJ013-0030.wav|The captain and crew, however, got safely to Jamaica, and so to England; the captain dying on the voyage home. +DUMMY/LJ020-0061.wav|Break the rolls apart from one another and eat warm. They are also good cold, and if the directions be followed implicitly, very good always. +DUMMY/LJ012-0117.wav|By-and-by the occupant of the room noticed something glittering in the center of the fire, which, to inspect more closely, he took out with the tongs. +DUMMY/LJ045-0058.wav|nineteen sixty-three. +DUMMY/LJ018-0352.wav|but there was no doubt of his guilt, as the Lord Chief Justice told him, while expressing great regret at his rash assertion. +DUMMY/LJ005-0295.wav|that plaintiffs were unwilling to consign the defendants against whom they had obtained execution to confinement within its walls." +DUMMY/LJ046-0141.wav|Slightly more specific criteria were established for PRS personnel processing White House mail referred by the White House mailroom, +DUMMY/LJ030-0229.wav|At this point the cars were speeding through the underpass and had left the scene of the shooting, +DUMMY/LJ017-0143.wav|which followed close on its heels, although in that the verdict of "Not Guilty" was excusable, as the evidence was entirely circumstantial. +DUMMY/LJ045-0217.wav|both she and Mrs. Paine thought he had come home to make up after the fight on Monday. +DUMMY/LJ050-0195.wav|Inspection of Buildings +DUMMY/LJ046-0084.wav|for the President soon after the assassination, quote, +DUMMY/LJ014-0080.wav|She ran upstairs, told her husband, made him go down to look at her handiwork, and as O'Connor was not quite dead, +DUMMY/LJ044-0061.wav|there also seems to be no basis for his claim that he had distributed, quote, +DUMMY/LJ015-0009.wav|Cole's difficulties increased more and more; warrant-holders came down upon him demanding to realize their goods. +DUMMY/LJ006-0080.wav|by the inner gatesman, himself a convicted prisoner, and a "wardsman" or responsible head of a room. +DUMMY/LJ015-0287.wav|not only to ensure fair play and the surrender of the proceeds if the cheque was cashed, but to give timely notice if it was not, +DUMMY/LJ018-0004.wav|and as some time elapsed before the discovery and apprehension of the supposed murderer, +DUMMY/LJ046-0042.wav|As Chief Executive, the President controls the exercise of the vast, almost incalculable powers of the executive branch of the Federal Government. +DUMMY/LJ006-0177.wav|and it was not the least part of the blame imputed to him that he made special favorites of particular prisoners, retaining of his own accord in Newgate, +DUMMY/LJ046-0174.wav|At the time of the assassination there were approximately four hundred persons throughout the country who were subject to periodic review. +DUMMY/LJ037-0255.wav|testified that Commission Exhibit Number one sixty-two was the jacket worn by the man they saw on November twenty-two. +DUMMY/LJ029-0083.wav|On November four, Sorrels told Behn he believed security difficulties at the Trade Mart could be overcome by special precautions. +DUMMY/LJ021-0074.wav|in wage cutting which depresses markets and destroys purchasing power. +DUMMY/LJ015-0106.wav|A reward was forthwith offered for Robson's apprehension. +DUMMY/LJ014-0200.wav|he seemed while in Newgate to realize thoroughly that there was no hope for him, and was determined to face his fate bravely. +DUMMY/LJ022-0062.wav|It proposes, by means of old age pensions, +DUMMY/LJ033-0069.wav|As she [Marina] told me about it I stepped onto the blanket roll +DUMMY/LJ050-0215.wav|these figures suggest that the agents of the Secret Service are substantially overworked. +DUMMY/LJ041-0005.wav|He concluded that school had nothing to offer him. +DUMMY/LJ018-0141.wav|led to the employment of the police, and the offer of a reward of fifteen hundred pounds for the detection of the offenders. +DUMMY/LJ048-0070.wav|reflect keen awareness of the necessity of communicating a much wider range of intelligence information to the Service. +DUMMY/LJ006-0090.wav|within this the greatest nursery of crime in London, should exercise even the commonest care for the personal decency or comfort of the prisoners. +DUMMY/LJ003-0287.wav|to undertake some kind of reform. +DUMMY/LJ025-0161.wav|of the oily, starchy, saccharine and woody substances contained in the full-grown plant and its seeds +DUMMY/LJ010-0075.wav|He married again and came to London, where he soon became notorious as a reckless gambler and a politician holding the most extreme views. +DUMMY/LJ017-0064.wav|he made use of the remarkable words, quote, If it is necessary for my soul's sake to confess this murder, +DUMMY/LJ034-0126.wav|shows three employees looking out of the fifth-floor window directly below the window from which the shots were fired. +DUMMY/LJ015-0151.wav|The philanthropy he had displayed in a small way when less prosperous became now a passion. +DUMMY/LJ018-0088.wav|I propose next to describe the leading features of the most important of these. +DUMMY/LJ015-0224.wav|the difficulty was to get a copy of the second key. +DUMMY/LJ023-0044.wav|the need to meet the unanswered challenge of one-third of a Nation ill-nourished, ill-clad, ill-housed. +DUMMY/LJ041-0012.wav|Once a group of white boys beat him up for sitting in the Negro section of a bus, which he apparently did simply out of ignorance. +DUMMY/LJ040-0019.wav|Under our system of justice no forum could properly make that determination unless Oswald were before it. +DUMMY/LJ006-0145.wav|He could indulge in snuff if a snuff-taker, +DUMMY/LJ040-0075.wav|Apparently that action was taken in anticipation of her marriage to Edwin A. Ekdahl, which took place in May of nineteen forty-five. +DUMMY/LJ044-0026.wav|Marina Oswald said she signed that name, apparently chosen because it rhymed with "Fidel," +DUMMY/LJ050-0106.wav|to devise a practical system which has any reasonable possibility of revealing such malcontents. +DUMMY/LJ029-0112.wav|in nineteen thirty-six, as testified that the traditional parade route in Dallas was along Main Street, since the tall buildings along the street +DUMMY/LJ035-0073.wav|was one minute and thirty seconds. +DUMMY/LJ010-0175.wav|Oxford, who had been pacing backwards and forwards with his hands under the lapels of his coat, saw the carriage approach. +DUMMY/LJ025-0082.wav|But although Cuvier's leading diagnosis of the animal from the plant will not stand a strict test, +DUMMY/LJ048-0268.wav|Therefore, the use of intoxicating liquor of any kind, including beer and wine, by members of the White House Detail +DUMMY/LJ026-0139.wav|It is probable that new leaves and new tissues generally are always formed in part from this reserve starch. +DUMMY/LJ038-0112.wav|As previously indicated, Marina Oswald testified that she took the two pictures with her husband's Imperial Reflex camera +DUMMY/LJ007-0186.wav|the undue authority given to prisoners, the levying of garnish under another name +DUMMY/LJ012-0173.wav|The event which he styles calamitous we may well characterize as one of the most deliberately atrocious murders on record. +DUMMY/LJ048-0035.wav|It was against this background and consistent with the criteria followed by the FBI prior to November twenty-two +DUMMY/LJ046-0073.wav|and the greater diversity of the audiences he must face in a world torn by conflicting ideologies. +DUMMY/LJ005-0012.wav|yet there were those who, wedded to ancient ideas, were intolerant of change; they would not admit the existence of any evils. +DUMMY/LJ026-0103.wav|Just as the solid food of animals must be digested in preparation for absorption, +DUMMY/LJ002-0122.wav|The number of arrests actually made was one hundred fourteen thousand, three hundred for the kingdom, and seven thousand twenty for Middlesex. +DUMMY/LJ021-0062.wav|in great part through the cooperation of American business brought about under the codes. +DUMMY/LJ018-0297.wav|It was the discovery of this flaw in the banking system which had encouraged the Americans to attempt this crime. +DUMMY/LJ010-0262.wav|charged him with the offense. +DUMMY/LJ003-0130.wav|His wife visited him in Newgate, and wrote a pitiable account of the state in which she found her husband. +DUMMY/LJ040-0139.wav|while he liked Youth House, he missed the freedom of doing what he wanted. He indicated that he did not miss his mother, end quote. +DUMMY/LJ040-0229.wav|in a sustained relationship with one therapist might bring results. +DUMMY/LJ006-0111.wav|He could trump up a false complaint against his fellow-prisoner, and so get him punished; +DUMMY/LJ028-0500.wav|The ruins reach the height of about forty feet. +DUMMY/LJ040-0090.wav|Pic did turn over part of his income to his mother, +DUMMY/LJ015-0201.wav|There were still big prizes, however, to tempt the daring, and none appealed more to the thievish instinct than the custom of transmitting gold by rail. +DUMMY/LJ008-0283.wav|two others were sentenced to three months' imprisonment, and the balance to varying terms. +DUMMY/LJ007-0224.wav|In a resolution passed by the Court of Aldermen on eighteenth March, eighteen forty-two, +DUMMY/LJ045-0035.wav|She testified that Oswald, quote, was very happy, end quote, about the birth of the child. +DUMMY/LJ016-0025.wav|who, while detained in the press-yard as a capital convict, under sentence of death for burglary, +DUMMY/LJ012-0295.wav|The woman Gale was also found guilty, but sentence of death was only passed on Greenacre. +DUMMY/LJ007-0082.wav|At the time the Lords' Committee sat there were eight thus retained in Newgate, and a return in the appendix of the Lords' report +DUMMY/LJ048-0239.wav|The lobby of the hotel and the areas adjacent to the quarters of the President were guarded during the night +DUMMY/LJ018-0213.wav|which in dramatic interest can compare with the conviction of William Roupell for forgery. +DUMMY/LJ011-0055.wav|Edmund Angelini, to take Fauntleroy's place. +DUMMY/LJ021-0129.wav|The employer who turns away from impartial agencies of peace, +DUMMY/LJ034-0157.wav|Brennan stated, quote, I could at that time -- I could, with all sincerity, identify him as being the same man, end quote. +DUMMY/LJ046-0122.wav|Many persons call themselves to the attention of PRS by attempting to visit, the President +DUMMY/LJ048-0041.wav|six hundred thirty-six thousand, three hundred seventy-one investigative matters during fiscal year nineteen sixty-three +DUMMY/LJ039-0202.wav|At fifteen yards each man's shots landed within the size of a dime. +DUMMY/LJ046-0030.wav|particularly when the President is traveling in this country or abroad. +DUMMY/LJ039-0097.wav|made an almost stationary target while he was aiming in, very little movement if any, end quote. +DUMMY/LJ031-0016.wav|Admiral Burkley, the President's physician, arrived at the hospital, quote, between three and five minutes following the arrival of the President, end quote. +DUMMY/LJ030-0075.wav|followed closely behind the President's automobile. +DUMMY/LJ013-0226.wav|The governor insisted, but Courvoisier showed great reluctance to strip. +DUMMY/LJ050-0146.wav|Automatic data processing +DUMMY/LJ046-0043.wav|As Commander in Chief of the Armed Forces, he must maintain ultimate authority over the development and disposition of our military power. +DUMMY/LJ039-0111.wav|head, Records Branch, Personnel Department, Headquarters U.S. Marine Corps, +DUMMY/LJ039-0036.wav|On April twenty-three, nineteen sixty-three, +DUMMY/LJ021-0148.wav|may be determined and any later adjustments shall be made either by agreement or, in case of disagreement, +DUMMY/LJ038-0246.wav|was able to determine that, this picture was taken with the Imperial Reflex camera owned by Lee Harvey Oswald. +DUMMY/LJ047-0018.wav|for the purpose of correlating information inasmuch as he was considered a possible security risk in the event he returned to this country, end quote. +DUMMY/LJ003-0080.wav|two committed by the Bankruptcy Commissioner, one for perjury, and two transports. +DUMMY/LJ026-0051.wav|These comparisons will, however, be translated into terms applicable to any species of higher plants or animals. +DUMMY/LJ045-0158.wav|He spent quite a bit of time putting away diapers and played with the children on the street. +DUMMY/LJ018-0034.wav|and accompanied by Mr. Death the jeweler and the cabman, went down to Liverpool and took the first steamer across the Atlantic. +DUMMY/LJ019-0067.wav|The opposing champions were not slow to find faults and flaws in the system they condemned. +DUMMY/LJ028-0332.wav|Once more, however, he waited till the interval appointed had gone by, and then leading the troops to the place where the four thousand were, +DUMMY/LJ013-0037.wav|The crime of fraudulent insurance they declared was very common, and the underwriters must have lost great sums in this way. +DUMMY/LJ048-0192.wav|show the foot patrolmen facing the passing motorcade, and not the adjacent crowds and buildings, as the procession passed. +DUMMY/LJ037-0202.wav|identified records of Seaport Traders, Incorporated, which showed that a, quote, point three eight +DUMMY/LJ047-0124.wav|and if we determined that the course of the investigation required us to clarify or face him down with this information, we would do it at the appropriate time. +DUMMY/LJ016-0034.wav|In one corner of the airing yard stood a cistern at some height from the ground; +DUMMY/LJ011-0151.wav|only child of a gentleman of large property in Cheshire, of which county he was actually high sheriff at the time of his daughter's abduction. +DUMMY/LJ012-0088.wav|May was smuggled in through a window on the esplanade behind an opened umbrella. When the place was quite deserted +DUMMY/LJ030-0167.wav|Mrs. Kennedy then heard a second shot and saw the President's skull torn open under the impact of the bullet. +DUMMY/LJ019-0383.wav|The whole question assumed sufficient importance to become a part of the Government program when Lord Beaconsfield took office in eighteen seventy-four. +DUMMY/LJ013-0256.wav|Good had told her, much to her perturbation, that she was to move from South Street to Roehampton, and one day he fetched her. +DUMMY/LJ050-0041.wav|This was the consequence of the Service's lack of an adequate investigative staff, +DUMMY/LJ002-0196.wav|The foregoing rentals and payments for privileges, together with fees exacted on commitment and discharge, went to the marshal or keeper of the prison, +DUMMY/LJ028-0427.wav|The enormous amount of debris which buried the palaces and temples and walls of Nebuchadnezzar's city, in places to the depth of a hundred feet, +DUMMY/LJ035-0178.wav|Sgt. D. V. Harkness of the Dallas police +DUMMY/LJ034-0038.wav|This print which had been cut out of the box was also forwarded to the FBI and Latona identified it as Oswald's right palmprint. +DUMMY/LJ019-0049.wav|the principles of which were debated by disputants of widely opposite opinions with an earnestness that sometimes bordered upon acrimony. +DUMMY/LJ029-0072.wav|The White House staff informed the Secret Service +DUMMY/LJ049-0165.wav|which first appeared in the appropriation of the Department of Justice in nineteen ten under the heading, quote, Miscellaneous Objects, end quote. +DUMMY/LJ009-0116.wav|On the following day the capital convicts, whose companions have been hanged, are required to return thanks for their narrow escape. +DUMMY/LJ034-0148.wav|In another interview with FBI agents on January seven, nineteen sixty-four, +DUMMY/LJ007-0153.wav|is generally known to all the worst men in the country; not only without the inclination, but almost without the ability of returning to an honest life." +DUMMY/LJ006-0289.wav|We have seen that male prisoners visited their female relations and friends on the female side. Besides this, +DUMMY/LJ005-0206.wav|At Godmanchester there was no jail, but a cage to secure prisoners till they could be taken before a magistrate. +DUMMY/LJ004-0225.wav|the practice was falling into abeyance, and the prisoners seldom required private assistance to eke out their meals. +DUMMY/LJ006-0013.wav|Sir Frederick Roe was of the same opinion as regards the appointment, but he would give the inspectors the power of acting as well as reporting. +DUMMY/LJ045-0141.wav|in an increasingly complex conspiracy against him. +DUMMY/LJ030-0251.wav|None stayed at the scene of the shooting, and none entered the Texas School Book Depository Building at or immediately after the shooting. +DUMMY/LJ049-0036.wav|and by the passengers in the jump seats. +DUMMY/LJ021-0030.wav|Our first problem was, of course, the banking situation because, as you know, the banks had collapsed. +DUMMY/LJ047-0244.wav|Revill promptly made a memorandum of this conversation in which the quoted statement appears. +DUMMY/LJ017-0115.wav|but it was proved that Palmer tried hard to get possession of the jar containing them, +DUMMY/LJ048-0066.wav|The handbook referred thus to, quote, the possibility of an attempt against the person or safety of the President, end quote. +DUMMY/LJ034-0029.wav|The "Rolling Readers" boxes contained, instead of books, light blocks used as reading aids. +DUMMY/LJ032-0275.wav|the Commission concluded that the rifle used to assassinate President Kennedy and wound Governor Connally was owned and possessed by Lee Harvey Oswald. +DUMMY/LJ030-0002.wav|Chapter two. The Assassination: Part two. +DUMMY/LJ007-0086.wav|at Buckingham and Hereford for eleven years +DUMMY/LJ016-0427.wav|He tripped up the chapel-stairs to hear the condemned sermon, and came out with cheerful alacrity on the morning he was to die. +DUMMY/LJ021-0019.wav|working through vast agencies of commerce and employing great masses of men in movements of production +DUMMY/LJ031-0123.wav|This punctuate missile wound, about two-fifths inch in diameter (one centimeter) and located approximately five inches above the left knee, +DUMMY/LJ007-0015.wav|it had the bad effect of distracting attention, +DUMMY/LJ046-0144.wav|When requested to provide a specific statement of the standards employed by PRS in deciding what information to seek and retain, +DUMMY/LJ039-0091.wav|Sergeant Zahm expressed the opinion that the shot which struck President Kennedy in the neck at one hundred seventy-six point nine +DUMMY/LJ007-0219.wav|The effects of the imprisonment are to vitiate its inmates, to extend their acquaintanceship with each other, +DUMMY/LJ013-0183.wav|The valet's strange demeanor had attracted attention from the first. +DUMMY/LJ028-0407.wav|The people of Hillah, too, are a survival of Babylonian times. +DUMMY/LJ018-0275.wav|On Tarpey's defense it was stated that the idea of the theft had been suggested to him by a novel, at a time he had lost largely on the turf. +DUMMY/LJ050-0073.wav|result in some degree of interference with the personal liberty of those involved. +DUMMY/LJ024-0133.wav|First, because I believe that it can be passed at this session of the Congress. +DUMMY/LJ011-0044.wav|will no longer be herded indiscriminately together. +DUMMY/LJ044-0084.wav|brought out the history of his defection to the Soviet Union. +DUMMY/LJ029-0163.wav|and then west on Main, turning back to Elm at Houston and then out Stemmons Freeway to the Trade Mart. +DUMMY/LJ028-0017.wav|but soon the phantom lake vanishes, while others, still farther beyond, appear and vanish in rapid succession. +DUMMY/LJ028-0327.wav|On the tenth day after his flight he led out his detachment, and surrounding the thousand men, +DUMMY/LJ021-0119.wav|including a few of major importance. +DUMMY/LJ028-0445.wav|The mound still farther south is called Amran, because upon its summit stands the tomb of a Mohammedan saint of that name. +DUMMY/LJ017-0060.wav|The conviction that he would escape had taken so firm a hold of him, +DUMMY/LJ037-0235.wav|five feet, ten inches, wearing light clothing, a light-colored jacket" walk past her at a fast pace with his hands in his pocket. +DUMMY/LJ028-0499.wav|Ctesias mentions three hundred feet; probably they were not far from the truth. +DUMMY/LJ048-0016.wav|He likewise indicated he was disenchanted with Russia. +DUMMY/LJ010-0036.wav|Hocker was found upon the scene of his crime, irresistibly attracted thither, as was Theodore Gardelle. +DUMMY/LJ032-0015.wav|the evidence linking Oswald to the attempted killing of Maj. Gen. Edwin A. Walker (Resigned, U.S. Army) on April ten, nineteen sixty-three, +DUMMY/LJ023-0127.wav|When I commenced to review the situation with the problem squarely before me, +DUMMY/LJ019-0028.wav|Mr. Shaw-Lefevre, the Speaker of the House of Commons, Sir Benjamin Brodie, +DUMMY/LJ020-0073.wav|Sift the flour, salt and sugar into a bowl, +DUMMY/LJ016-0387.wav|All too have a fairly good appetite, and eat with relish, up to the last moment. +DUMMY/LJ048-0273.wav|that under ordinary circumstances he would have taken disciplinary action against those agents who had been drinking in clear violation of the regulation. +DUMMY/LJ003-0330.wav|The state side ceased to exist, and the female prisoners thus regained the space of which their quadrangle had been robbed. +DUMMY/LJ046-0165.wav|Most of these cases involved persons who used threatening language in communications to or about the President. +DUMMY/LJ007-0121.wav|There were no restraints, cards and backgammon were played, and the time passed in feasting and revelry. +DUMMY/LJ016-0418.wav|a tall, gaunt, repulsive-looking woman, who no more shrank from cowardly, secret crimes than from the penalty they entailed. +DUMMY/LJ015-0165.wav|But the peer rushed forward and shook Redpath warmly by the hand. +DUMMY/LJ040-0168.wav|She thought that he had detached himself from the world around him because, quote, no one in it ever met any of his needs for love, end quote. +DUMMY/LJ047-0134.wav|The FBI was advised by the rental agent for the Oswalds' apartment in New Orleans that they had moved again. +DUMMY/LJ047-0159.wav|She didn't have the exact address, and it is my recollection that we went to the phone book and looked it up, +DUMMY/LJ031-0099.wav|Moreover, the Parkland doctors took no further action after the President had expired because they concluded that it was beyond the scope of their permissible duties. +DUMMY/LJ009-0178.wav|In eighteen thirty-two the dissection of bodies cut down from the gallows, which had been decreed centuries previously, was abolished; +DUMMY/LJ016-0170.wav|greater attention was paid to the capital convicts, and the horrors of their situation while awaiting sentence +DUMMY/LJ045-0079.wav|As a result Oswald struck her, as to which she testified, quote, +DUMMY/LJ005-0276.wav|and for securing a proper system of prison discipline. +DUMMY/LJ042-0152.wav|While in the Soviet Union he wrote his longest and clearest piece of work, "The Collective." +DUMMY/LJ046-0006.wav|During this same period there were three other attacks on the life of a President, +DUMMY/LJ044-0218.wav|In this connection it should be noted that in speaking of the Worker, Oswald told Michael Paine, apparently in all seriousness, that, quote, +DUMMY/LJ008-0306.wav|I myself heard more than one of those whose lives were spared by that decision of the council, +DUMMY/LJ004-0179.wav|The allowance of food daily to felons was a fourpenny loaf, +DUMMY/LJ026-0112.wav|In an animal the foods in the circulating liquids, blood and lymph, are selected and absorbed by the cells. +DUMMY/LJ030-0082.wav|The agents on the front of the running boards had directions to move immediately to positions just to the rear of the President and Mrs. Kennedy when the President's car slowed +DUMMY/LJ007-0163.wav|all the tumultuous and diversified passions and emotions which circumstances like these must necessarily generate +DUMMY/LJ045-0159.wav|Question: How did you indicate to him that you were angry with him? Answer: By not talking to him. +DUMMY/LJ034-0196.wav|Another person who saw the assassin as the shots were fired was Amos L. Euins, age fifteen, +DUMMY/LJ019-0020.wav|Up to the twenty-first December, eighteen forty-two, +DUMMY/LJ004-0164.wav|There was still worse to come. Having explored the yards and adjacent day rooms, and sleeping cells, a door was unlocked, +DUMMY/LJ039-0165.wav|The target at two hundred sixty-five feet was placed to the right of the two hundred forty-foot target +DUMMY/LJ001-0131.wav|the size known as "Long primer" ought to be the smallest size used in a book meant to be read. +DUMMY/LJ003-0252.wav|of distinguishing a prisoner from a stranger and temporary visitor. +DUMMY/LJ045-0014.wav|His awareness is shown by a letter that he wrote to Arnold Johnson of the Communist Party U.S.A., +DUMMY/LJ003-0241.wav|A further and a more iniquitous method of extorting money +DUMMY/LJ006-0095.wav|The rugs used were never washed; an order existed that the bedding should be taken into the yards to be aired, but it was not very punctually obeyed. +DUMMY/LJ028-0158.wav|The city wall is brought down on both sides to the edge of the stream, +DUMMY/LJ028-0040.wav|In the year six eighty-nine B.C., Sinacherib, King of Nineveh, captured Babylon, +DUMMY/LJ031-0132.wav|U.S. Representatives Henry B. Gonzalez, +DUMMY/LJ007-0241.wav|and again and again to protest against Newgate as it at present exists. +DUMMY/LJ048-0165.wav|buildings along the route can be checked by teams of law enforcement officers, and armed guards are posted along the route as appropriate. +DUMMY/LJ040-0230.wav|There are indications that he has suffered serious personality damage but if he can receive help quickly this might be repaired to some extent, end quote. +DUMMY/LJ031-0114.wav|Two additional operations were performed on Governor Connally for wounds which he had not realized he had sustained until he regained consciousness the following day. +DUMMY/LJ047-0061.wav|This assignment was given to Agent James P. Hosty, Jr. of the Dallas office upon Fain's retirement. +DUMMY/LJ002-0209.wav|How to make most profit out of the wretched denizens of the jail was the marshal's only care. He got a rent for the coffee-house and the bake-house; +DUMMY/LJ044-0060.wav|In view of the limited amount of public activity on Oswald's part before August nine, nineteen sixty-three, +DUMMY/LJ050-0100.wav|The Commission recognizes that no set of meaningful criteria will yield the names of all potential assassins. Charles J. Guiteau, +DUMMY/LJ018-0127.wav|and he eventually succeeded, his wife making the first overtures, in persuading a lad named Brown to steal some of the bank paper. +DUMMY/LJ035-0033.wav|The stairway is located in the northwest corner of the Depository Building. +DUMMY/LJ022-0136.wav|examples of how the public money can be more effectively spent for the benefit of the American people. +DUMMY/LJ010-0242.wav|Nothing happened till Her Majesty returned to Buckingham Palace about six p.m., when, on descending Constitution Hill, +DUMMY/LJ034-0172.wav|Fischer looked up and watched the man in the window for ten or fifteen seconds and then started watching the motorcade, which came into view on Houston Street. +DUMMY/LJ042-0211.wav|is provided by the following, which was apparently written either on the ship coming back, or after his return from the Soviet Union, quote, +DUMMY/LJ039-0128.wav|Based on that training, his basic knowledge in sight manipulation and trigger squeeze and what not, I would say that he would be capable of sighting that rifle in well, +DUMMY/LJ032-0115.wav|A few minutes after the rifle was discovered on the sixth floor of the Depository Building +DUMMY/LJ018-0366.wav|Although sentence of death was passed on Edmunds, it was commuted to penal servitude for life; +DUMMY/LJ049-0223.wav|and direction of the activities of all existing agencies of Government which are in a position to and do, furnish information +DUMMY/LJ037-0212.wav|who attested that the fictitious Hidell was an American citizen and had not been convicted of a felony, was also in Oswald's handwriting. +DUMMY/LJ038-0270.wav|which would indicate that the bullet was not fired from the Mannlicher-Carcano rifle owned by Lee Harvey Oswald. +DUMMY/LJ022-0112.wav|and I assure my fellow citizens that no energy will be spared in using these funds effectively +DUMMY/LJ031-0170.wav|During the drive Vice President Johnson, at Youngblood's instruction, kept below window level. +DUMMY/LJ019-0215.wav|Why not move the city prison bodily into this more rural spot, with its purer air and greater breathing space? +DUMMY/LJ022-0182.wav|To this end it was decided more than twenty years ago +DUMMY/LJ011-0289.wav|and creeping out into the garden at the back, climbed the wall, and got into the street. +DUMMY/LJ050-0047.wav|International Business Machines Corporation, and a panel of psychiatric and psychological experts. +DUMMY/LJ015-0261.wav|The whole strange story, the long incubation and the elaborate accomplishment of the plot, +DUMMY/LJ016-0162.wav|he fastened one end of the strap above mentioned to the hook, and then fell down. +DUMMY/LJ002-0090.wav|More than half their quadrangle had been partitioned off for another purpose, +DUMMY/LJ012-0024.wav|On his release an uncle, a slop-seller in Chatham, gave him a situation as "barker," or salesman, +DUMMY/LJ042-0061.wav|It seems that Oswald immediately attempted suicide -- a striking indication of how much he desired to remain in the Soviet Union. +DUMMY/LJ036-0100.wav|Notified of Whaley's statement, the police brought him to the police station that afternoon. +DUMMY/LJ016-0294.wav|But all, whether young or old, men or women, seemed to know nothing, feel nothing, to have no object but the gallows, and to laugh, +DUMMY/LJ031-0017.wav|since the riders in his car, quote, were not exactly aware what had happened, end quote, and the car went on to the Trade Mart first. +DUMMY/LJ040-0133.wav|She said that her pocketbook was searched, quote, because the children in this home were such criminals, dope fiends, and had been in criminal offenses, +DUMMY/LJ025-0071.wav|while in this condition their movements are, to all appearances, as spontaneous -- as much the product of volition -- as those of such animals. +DUMMY/LJ046-0015.wav|This study has led the Commission to conclude that the public interest might be served by any contribution it can make to the improvement of protective arrangements. +DUMMY/LJ018-0123.wav|a man named Burnett came with his wife and took up his residence at Whitchurch, Hampshire, at no great distance from Laverstock, +DUMMY/LJ016-0004.wav|Yet the jail, in spite of its fortress-like aspect, was by no means really safe. +DUMMY/LJ036-0155.wav|Whaley testified that Oswald was wearing either the gray zippered jacket or the heavy blue jacket. +DUMMY/LJ038-0070.wav|Captain Fritz returned to police headquarters from the Texas School Book Depository at two:fifteen after a brief stop at the sheriff's office. +DUMMY/LJ038-0056.wav|Nor did Oswald ever complain that he was hit with a gun, or injured in the back. +DUMMY/LJ036-0182.wav|She recalled that it was subsequent to the time the President had been shot. +DUMMY/LJ048-0282.wav|that it is reasonable to expect them to meet very high standards of personal conduct, +DUMMY/LJ044-0096.wav|Mr. Oswald handled himself very well, as usual, end quote. +DUMMY/LJ047-0112.wav|Knack was aware of the facts known to the FBI and recognized Oswald's false statements. +DUMMY/LJ020-0074.wav|hollow the heap in the center and pour in the milk, working down the flour into the liquid with a spoon or your hands until it is thoroughly melted. +DUMMY/LJ041-0165.wav|Shortly after Thornley finished reading that book the Marine unit to which both men were assigned was required to take part in a Saturday morning parade +DUMMY/LJ009-0109.wav|while the capital convicts who were lately in that black pew appear faint with emotion. +DUMMY/LJ050-0169.wav|and that many strongly held views are advanced in resistance to any suggestion that an effort be made to impose any degree of coordination. +DUMMY/LJ023-0008.wav|We were then in the midst of the great banking crisis. +DUMMY/LJ014-0208.wav|A murderous assault on a police constable, which so nearly ended fatally that the culprit was sentenced to death, although not executed, +DUMMY/LJ011-0177.wav|The fact was, Wakefield went on to say, an uncle of his had advanced Mr. Turner sixty thousand pounds, which had temporarily staved off ruin. +DUMMY/LJ006-0087.wav|It must be admitted that so far but little effort had been made to counteract the evils of indiscriminate association. +DUMMY/LJ044-0160.wav|he may not have intended to go to the Soviet Union directly, if at all. It appears that he really wanted to go to Cuba. +DUMMY/LJ012-0225.wav|they had also been seen to leave their cottage, carrying out a sack with something heavy inside. On this they were fully committed to Newgate for trial. +DUMMY/LJ048-0092.wav|The advance preparations in Dallas by Agent Winston G. Lawson of the White House detail have been described in chapter two. +DUMMY/LJ006-0255.wav|but all could still obtain as much extra, and of a luxurious kind, as their friends chose to bring them in. +DUMMY/LJ003-0170.wav|A Mr. Lundy, charged with forgery, was thus accommodated on the state side for upwards of five years. +DUMMY/LJ017-0220.wav|The six Spaniards, the Greek, and the Turk were in the same watch, +DUMMY/LJ009-0021.wav|in which it was said I had enlarged upon the heinous nature of his crime, and warned the public to avoid such conduct. +DUMMY/LJ016-0093.wav|They asked him how he had cut his hand, which was bleeding, and he said he had done it while working up on the roof. +DUMMY/LJ038-0156.wav|He stated that at the time the President was shot he was having lunch with "Junior" but he did not give Junior's last name. +DUMMY/LJ018-0182.wav|and no more bodies were to be bought +DUMMY/LJ008-0089.wav|and they were packed into a more limited space, disporting themselves as heretofore by brutal horse-play, coarse jests, and frantic yells. +DUMMY/LJ026-0040.wav|And in this connection the fact that some bacteria -- the simplest organisms known and devoid of chlorophyll +DUMMY/LJ041-0050.wav|In John Pic's view, +DUMMY/LJ032-0124.wav|by Sebastian F. Latona, supervisor of the Latent Fingerprint Section of the FBI's Identification Division. +DUMMY/LJ002-0112.wav|The frequency and extent of processes against debtors seventy or eighty years ago will appear almost incredible +DUMMY/LJ050-0251.wav|the amount of assistance that it can expect from other agencies. +DUMMY/LJ032-0021.wav|was a distributor of surplus Italian six point five-millimeter military rifles. During the evening of November twenty-two, nineteen sixty-three, +DUMMY/LJ023-0099.wav|In the case holding the New York Minimum Wage Law unconstitutional, Justice Stone said +DUMMY/LJ003-0295.wav|The committee seems to have fully realized, even at this early date (eighteen fifteen), +DUMMY/LJ018-0065.wav|At the disbandment of the force, as he was without resources, he turned his attention to hotel robberies, by which he lived for some years. +DUMMY/LJ013-0232.wav|The prisoner replied that he had left it in the mattress of which he had just been deprived. +DUMMY/LJ010-0146.wav|Attacks upon the sovereign, as I have said, became more common after the accession of the young Queen Victoria in eighteen thirty-eight. +DUMMY/LJ007-0188.wav|the misemployment of officers and prisoners by the governor. +DUMMY/LJ035-0005.wav|The Commission has found that Oswald's movements, as described by these witnesses, +DUMMY/LJ018-0009.wav|When the train arrived at Hackney station, a passenger who was about to enter one of the carriages found the cushions soaked with blood. +DUMMY/LJ048-0093.wav|With the assistance of Agent in Charge Sorrels of the Dallas field office of the Secret Service, +DUMMY/LJ021-0125.wav|it was natural that the workers should seek and obtain a statutory declaration of their constitutional right +DUMMY/LJ001-0159.wav|The paper that is used for ordinary books is exceedingly bad even in this country, but is beaten in the race for vileness +DUMMY/LJ045-0203.wav|Even though he had searched -- in the Marine Corps, in his ideal of communism, in the Soviet Union and in his attempt to get to Cuba +DUMMY/LJ008-0277.wav|Convicted offenders might have good or bad luck; they might be arraigned when their particular crime was uncommon, and were then nearly certain to escape; +DUMMY/LJ031-0073.wav|According to Dr. Clark, the small bullet hole on the right rear of the President's head discovered during the subsequent autopsy +DUMMY/LJ011-0235.wav|as in the burglary at Lambeth Palace, when the thieves were fortunately disappointed, the archbishop having, before he left town, +DUMMY/LJ016-0211.wav|But, fortunately for him, just at the last moment Calcraft was set free. +DUMMY/LJ025-0116.wav|have given greater and greater breadth and firmness to Schwann's great generalization +DUMMY/LJ010-0289.wav|That offense was a brutal assault upon Her Majesty, whom he struck in the face with a small stick just as she was leaving Cambridge House. +DUMMY/LJ031-0146.wav|and wait for Mrs. Kennedy and the President's body. +DUMMY/LJ026-0135.wav|If a larger quantity of starch is formed in the chlorophyll bodies than is immediately needed by the protoplasm for purposes of repair or growth, +DUMMY/LJ035-0059.wav|Truly replied, "Yes." +DUMMY/LJ048-0019.wav|His activities for the Fair Play for Cuba Committee in New Orleans, we knew, were not of real consequence as he was not connected with any organized activity there. +DUMMY/LJ025-0014.wav|Cuvier devotes a special section to the Division of Organized Beings into Animals and Vegetables, +DUMMY/LJ014-0272.wav|and eighteen fifty to embezzle and apply to his own purposes some seventy-one thousand pounds. +DUMMY/LJ017-0082.wav|He fixed upon a sporting friend, Mr. John Parsons Cook, who had been in luck at Shrewsbury races, both as a winner and a backer, +DUMMY/LJ005-0159.wav|in others both were supplied in ample quantities, the cost varying per prisoner from twenty shillings to five pounds. +DUMMY/LJ001-0052.wav|yet their type is artistically on a much lower level than Jenson's, and in fact +DUMMY/LJ037-0084.wav|Barbara Jeanette Davis testified that no one had shown her a picture of Oswald before the identification and that she had not seen him on television. +DUMMY/LJ024-0138.wav|During the past half century the balance of power between the three great branches of the federal government, +DUMMY/LJ011-0210.wav|Mr. William Wakefield was arrested at Dover, conveyed to Chester, and committed to Lancaster Jail for trial at the next assizes, +DUMMY/LJ018-0028.wav|who had been a lodger of his. Müller had given the cabman's little daughter a jeweler's cardboard box bearing the name of Mr. Death. +DUMMY/LJ034-0093.wav|Brennan also testified that Lee Harvey Oswald, +DUMMY/LJ039-0131.wav|Sergeant Zahm concluded, quote, I would say in the Marine Corps he is a good shot, slightly above average, +DUMMY/LJ022-0189.wav|They are a minimum of wise readjustments of our Federal Reserve System in the light of past experience and present needs. +DUMMY/LJ012-0136.wav|Moss was known to be intimate with the elder Caspar, +DUMMY/LJ004-0116.wav|"Care also was to be taken that the prisoners shall not be kept in any apartment underground." +DUMMY/LJ008-0094.wav|His appearance on the scaffold was the signal for three prolonged shouts from an innumerable populace, "the brutal effusion of one common sentiment." +DUMMY/LJ016-0358.wav|all who could were glad to escape from the last act of the ghastly drama at which they had assisted. +DUMMY/LJ009-0162.wav|Nevertheless, after that date no person whatever was executed for this offense. +DUMMY/LJ050-0067.wav|more than five thousand names were referred to the Secret Service in the first four months of nineteen sixty-four. +DUMMY/LJ029-0176.wav|with Dallas scheduled as one of the stops. +DUMMY/LJ044-0208.wav|An examination of the Militant, to which Oswald subscribed, +DUMMY/LJ017-0186.wav|While with Mrs. Wilson she became suddenly and alarmingly ill, and died in great agony. +DUMMY/LJ040-0147.wav|so that he was not bothered and did not have to make the effort of communicating. +DUMMY/LJ032-0253.wav|From September twenty-four, nineteen sixty-three, when Marina Oswald arrived in Irving from New Orleans, until the morning of the assassination, +DUMMY/LJ040-0025.wav|His life was characterized by isolation, frustration, and failure. +DUMMY/LJ018-0201.wav|where the coiners, unexpectedly disturbed, fell an easy prey. +DUMMY/LJ014-0069.wav|Her husband, to whom she coolly confided her intention, +DUMMY/LJ041-0071.wav|His Marine career was not helped by his attitude +DUMMY/LJ043-0052.wav|Oswald objected to his mother visiting the apartment +DUMMY/LJ026-0165.wav|Imagine a higher animal with an attachment for turning the carbon dioxide and water excreted +DUMMY/LJ025-0013.wav|In the second edition of the "Regne Animal," published in eighteen twenty-eight, +DUMMY/LJ012-0106.wav|But it was not until the autumn that they traced some of the notes stolen from the Custom House to Jordan and Sullivan. +DUMMY/LJ005-0045.wav|Nor did it confine itself to mere verbal recommendations. +DUMMY/LJ014-0288.wav|That same evening he committed suicide in Newgate. +DUMMY/LJ006-0027.wav|It is an able and exhaustive state paper, testifying to the keenness of the writer's perception, and his unremitting labor in pursuing his researches. +DUMMY/LJ045-0242.wav|It is apparent, however, that Oswald was moved by an overriding hostility to his environment. +DUMMY/LJ003-0063.wav|but when the ward was obliged to accommodate double the ordinary number, as was frequently the case, +DUMMY/LJ011-0127.wav|and had access to the official records, he forged a warrant for one thousand nine-hundred seventy-three pounds and was paid the money by the comptroller general. +DUMMY/LJ012-0079.wav|A large amount of specie was nearly always retained here in the department of the Receiver of Fines. +DUMMY/LJ006-0271.wav|which would have greatly helped to keep out bad characters. Some idea of the difficulty and inconvenience of these lax regulations as regards visiting, +DUMMY/LJ014-0240.wav|without capital, and at railroad speed. +DUMMY/LJ033-0129.wav|Mrs. Randle saw the bag fleetingly +DUMMY/LJ022-0064.wav|and to give to all a feeling of security as they look toward old age. +DUMMY/LJ021-0037.wav|In all of these efforts the government has come to the assistance of business +DUMMY/LJ012-0255.wav|These were the missing members of the same mutilated trunk, +DUMMY/LJ016-0198.wav|I cannot find that Calcraft was sworn in when appointed, or any exact information when the old forbidding ceremony ceased to be practiced. +DUMMY/LJ018-0086.wav|His demeanor immediately preceding his execution I have referred to in the last chapter. +DUMMY/LJ034-0067.wav|Oswald's Presence on Sixth Floor Approximately thirty-five Minutes Before the Assassination +DUMMY/LJ037-0260.wav|that he was seen running along Jefferson Boulevard, that a jacket was found under a car in a lot adjoining Jefferson Boulevard +DUMMY/LJ040-0112.wav|That was not the case, however, after he and his mother moved to New York in August of nineteen fifty-two, shortly before Lee's thirteenth birthday. +DUMMY/LJ038-0121.wav|or to answer any questions concerning the card. +DUMMY/LJ016-0113.wav|but only to be taken soon afterwards for a fresh offense, and again pass through Newgate with sentences of penal servitude. +DUMMY/LJ038-0297.wav|A study of the bus routes indicates that +DUMMY/LJ031-0086.wav|the President remained on his back throughout his medical treatment at Parkland. +DUMMY/LJ014-0068.wav|At last this fiendish woman made up her mind to murder O'Connor and appropriate all his possessions. +DUMMY/LJ009-0248.wav|Sometimes the condemned man made a hard fight for life. +DUMMY/LJ047-0150.wav|West Fifth Street, Irving, Texas. After receiving this information on October twenty-nine, Agent Hosty attempted to locate Oswald. +DUMMY/LJ048-0251.wav|The remaining four had key responsibilities as members of the complement of the follow-up car in the motorcade. +DUMMY/LJ042-0206.wav|The Soviets have committed crimes unsurpassed even by their early day capitalist counterparts, the imprisonment of their own peoples, +DUMMY/LJ006-0063.wav|minor offenders charged with small thefts or non-payment of small sums were cheek by jowl with convicts sentenced to long terms of transportation. +DUMMY/LJ028-0198.wav|In the great banquet hall of the palace, when the guests were drinking from the golden cups, and the revelry was at its highest, +DUMMY/LJ006-0191.wav|No provision whatever was made for the employment of prisoners, no materials were purchased, no trade instructors appointed. +DUMMY/LJ032-0074.wav|The mail-order coupon listed the purchaser as "A. J. Hidell Age twenty-eight" +DUMMY/LJ039-0170.wav|and eight point two five seconds respectively. +DUMMY/LJ029-0016.wav|As Chief of State, the President always welcomed the opportunity to learn, firsthand, about the problems which concerned the American people. +DUMMY/LJ018-0233.wav|To this fraudulent instrument he appended forged signatures, and in due course obtained probate. +DUMMY/LJ007-0173.wav|that in the metropolis of this highly-favored country, to which the eyes of other lands turn for example, a system of prison discipline such as that enforced in Newgate +DUMMY/LJ046-0152.wav|and a determination made as to whether the information might indicate possible harm to the President. +DUMMY/LJ022-0115.wav|This is a great national crusade to destroy enforced idleness which is an enemy of the human spirit +DUMMY/LJ003-0246.wav|sometimes the wearer was declared medically unfit, or he obtained release by long good conduct, +DUMMY/LJ028-0350.wav|It is from these women that the Babylonians of our times are sprung. +DUMMY/LJ035-0038.wav|The lunchroom door is usually open, but the first door is kept shut by a closing mechanism on the door. +DUMMY/LJ011-0239.wav|Where robbery with violence was intended, the perpetrators had now to adopt various shifts and contrivances to secure their victim. +DUMMY/LJ009-0099.wav|Meanwhile the clergyman, still bent into the form of a sleeping dog, +DUMMY/LJ021-0103.wav|if they have gone too far in such matters as price fixing and limitation of production, +DUMMY/LJ046-0234.wav|or an organized group, other than specific threats, which suggested a source of danger against which timely precautions could be taken. +DUMMY/LJ046-0203.wav|any and all information that they may come in contact with that would indicate danger to the President, end quote. +DUMMY/LJ038-0214.wav|and the other paragraphs instructed her on the disposal of Oswald's personal effects and the management of her affairs if he should not return. +DUMMY/LJ045-0007.wav|Some of that feeling was expressed in the incident involving then vice-presidential candidate Johnson during the nineteen sixty campaign, +DUMMY/LJ010-0114.wav|His sudden access to means unlimited was no doubt due to the profitable role he soon adopted of Government informer and spy, +DUMMY/LJ008-0027.wav|put a quid of tobacco into his mouth, and offered another to his companion, who refused it with indignation +DUMMY/LJ016-0442.wav|made a determined effort to burn himself to death by throwing himself bodily on to the fire in the condemned ward. +DUMMY/LJ016-0304.wav|Numbers of humane and thoughtful persons had long been convinced of this. +DUMMY/LJ050-0006.wav|and the dissimilarity of its protective functions to most activities of the Department of the Treasury +DUMMY/LJ037-0042.wav|The man pulled a gun. Mrs. Markham heard three shots and saw the policeman fall to the ground near the left front wheel. +DUMMY/LJ023-0022.wav|Today we are only part-way through that program +DUMMY/LJ001-0050.wav|and though the famous family of Aldus restored its technical excellence, rejecting battered letters, +DUMMY/LJ018-0216.wav|William was brought up as an attorney, and became in due course his father's man of business. +DUMMY/LJ008-0138.wav|and at his request the ordinary drew the cap further down over his face, when in an instant, +DUMMY/LJ017-0202.wav|fairly electrified the court by declaring it his opinion that many deaths, supposed to be from cholera, were really due to poison. +DUMMY/LJ025-0091.wav|as it was current among contemporary chemists. +DUMMY/LJ013-0265.wav|was a piece of a woman's calico apron stained with blood, which he had used to save the pressure on his shoulder by the hod. +DUMMY/LJ016-0165.wav|three fingers might have been inserted between it and the flesh; the pressure was all on the arteries behind the ears, +DUMMY/LJ008-0195.wav|Every window had its party of occupants; the adjoining roofs were equally crowded. +DUMMY/LJ016-0236.wav|Marwood being, on the other hand, in favor of giving a man as much rope as possible. +DUMMY/LJ043-0116.wav|and because he spent too much time loitering in the garage next door, where he read rifle and hunting magazines. +DUMMY/LJ030-0147.wav|that it was twelve:thirty p.m., the time they were due at the Trade Mart. +DUMMY/LJ007-0108.wav|Dasent and Pilgrim were released in ten days, on making due submission. +DUMMY/LJ021-0033.wav|This has given safety to millions of depositors in these banks. +DUMMY/LJ042-0094.wav|The Soviet authorities denied Oswald permission +DUMMY/LJ003-0157.wav|The best accommodation the jail could offer was reserved for the prisoners on the state side, +DUMMY/LJ004-0182.wav|As for the debtors, they had no allowance whatever, and were often in imminent danger of starvation. +DUMMY/LJ002-0012.wav|After that they fell as steadily, +DUMMY/LJ025-0036.wav|and it is so, inasmuch as an additional substance -- nitrogen -- enters into it as an essential element; +DUMMY/LJ007-0214.wav|"The prominent evils of this prison (Newgate) -- evils which the alterations made within the last four years have failed to remove +DUMMY/LJ050-0030.wav|The Commission also recommends +DUMMY/LJ020-0005.wav|Sponge-making is, therefore, an important if not an essential accomplishment in a cook, be she novice or veteran. +DUMMY/LJ019-0062.wav|The system of associated labor in silence had also its warm supporters, +DUMMY/LJ014-0065.wav|O'Connor was a man of substance. He had long followed the profitable trade of a money-lender, +DUMMY/LJ015-0269.wav|He was known to the whole criminal fraternity as a high-class receiver of stolen goods, a negotiator more especially of stolen paper, +DUMMY/LJ029-0087.wav|In addition to the preventive measures already mentioned, they provided for controlling access to the building, closing off and policing areas around it, +DUMMY/LJ016-0088.wav|working at the roof of the chapel on the female side. +DUMMY/LJ018-0037.wav|In searching the prisoner's box, Mr. Briggs' watch was found wrapped up in a piece of leather, +DUMMY/LJ032-0004.wav|were fired from the southeast corner window of the sixth floor of the Texas School Book Depository Building +DUMMY/LJ017-0094.wav|strychnia kills by causing tetanic fixing of the respiratory muscles. End quote. +DUMMY/LJ014-0320.wav|proposed to gain the capital he needed for business purposes by raising money on dock warrants for imported goods which had no real existence. +DUMMY/LJ013-0086.wav|Burgess, who was purposely on leave from the bank, effected the sale, which was paid for with a cheque for nearly the whole amount on Lubbock's Bank. +DUMMY/LJ016-0195.wav|and offered him the post of executioner for the city of London and Middlesex. +DUMMY/LJ046-0233.wav|it made little effort to identify factors in the activities of an individual +DUMMY/LJ008-0072.wav|and the abolition of the practice of burning females for petty treason, more than one woman suffered this penalty at the Old Bailey. +DUMMY/LJ013-0131.wav|Ashley was arrested as he left the shop; the spoons were impounded, and it was found that the club monogram had been erased from them. +DUMMY/LJ030-0056.wav|Kellerman had discussed the matter with O'Donnell, whose instructions were, quote, +DUMMY/LJ005-0018.wav|an imputation which the Society indignantly and very justly repudiated, the statement being, as they said, +DUMMY/LJ006-0006.wav|I shall now return to the great jail of the city of London, and give a more detailed account of its condition and inner life +DUMMY/LJ045-0026.wav|The relations between Lee and Marina Oswald are of great importance in any attempt to understand Oswald's possible motivation. +DUMMY/LJ044-0201.wav|Oswald's activities with regard to Cuba raise serious questions as to how much he might have been motivated in the assassination +DUMMY/LJ006-0146.wav|and might always smoke his pipe undisturbed; for although the use of tobacco had been prohibited since the report of the Lords Committee, +DUMMY/LJ011-0167.wav|At Manchester, however, the carriage stopped at the Albion Hotel. +DUMMY/LJ002-0140.wav|These courts were extended two centuries later to several large provincial towns, and all were in full activity when Neild wrote, +DUMMY/LJ032-0084.wav|made by photographing the Oswald cards, retouching the resulting negatives, and producing prints from the retouched negatives. +DUMMY/LJ039-0061.wav|For a rifleman situated on the sixth floor of the Texas School Book Depository Building +DUMMY/LJ025-0049.wav|is the specially animal function of animals and constitutes their fourth distinctive character. +DUMMY/LJ012-0291.wav|as she leant back he put his foot against the chair, and so tilted it over. +DUMMY/LJ047-0067.wav|Hosty decided that the Lee Harvey Oswald case should be reopened because of the alleged personal difficulties and the contact with the Worker, +DUMMY/LJ049-0055.wav|All agents are so instructed. +DUMMY/LJ028-0361.wav|Yet Babylon continued to live, +DUMMY/LJ017-0014.wav|was never introduced into this country. Its admission was probably checked by the increased vigilance at the custom houses, +DUMMY/LJ029-0042.wav|As advance agent working closely with Sorrels, Lawson had responsibility for arranging the timetable for the President's visit to Dallas +DUMMY/LJ018-0173.wav|Griffiths afterwards admitted that he had been employed in defrauding the bank since eighteen forty-six, +DUMMY/LJ041-0080.wav|in those arguments, quote, and make himself come out top dog, end quote. +DUMMY/LJ007-0208.wav|In the inspectors' fourth report, dated eighteen thirty-nine, +DUMMY/LJ032-0140.wav|because he did not release the lifted print until November twenty-six, when he received instructions to send "everything that we had" to the FBI. +DUMMY/LJ003-0329.wav|In common justice to them, it must be admitted that immediate effect was given to all that could be easily carried out. +DUMMY/LJ037-0233.wav|two witnesses, Warren Reynolds and B. M. Patterson, saw the gunman run toward the rear of a gasoline service station on Jefferson Boulevard. +DUMMY/LJ036-0141.wav|[east side of the street]. Of course, the traffic was moving through there and I put it in gear and moved on, that is the last I saw of him. +DUMMY/LJ027-0147.wav|For as it is of the essence of this theory that new forms arise from older forms by way of hereditary descent, +DUMMY/LJ018-0136.wav|One more remains, that of "glazing," without which no paper is issued for engraving. +DUMMY/LJ030-0025.wav|when Air Force One touched down at Love Field at eleven:forty a.m., Eastern Standard Time. +DUMMY/LJ001-0184.wav|all books might be at least comely and well-looking: and if to these good qualities were added really beautiful ornament and pictures, +DUMMY/LJ014-0176.wav|His determined addiction to evil courses had led to his being cast off by his family, +DUMMY/LJ019-0214.wav|The Corporation owned lands there covering from nineteen to twenty acres. +DUMMY/LJ027-0072.wav|By extensive comparison in the taxonomic and ontogenic series, the whole vertebrate structure in all its details in different animals +DUMMY/LJ009-0103.wav|The silence is short. As the ordinary proceeds 'to conclude,' +DUMMY/LJ015-0045.wav|the firm's paper went down further and further in value; an application to the Committee of Bankers for assistance was peremptorily refused, +DUMMY/LJ020-0038.wav|Turn the pans once while baking, moving them as gently as possible. +DUMMY/LJ024-0063.wav|President Wilson, three; President Harding, four, including a Chief Justice; President Coolidge, one; +DUMMY/LJ036-0198.wav|Tippit rode alone, as only one man was normally assigned to a patrol car in residential areas during daylight shifts. +DUMMY/LJ028-0412.wav|Among the first of the modern travelers to describe the ruins of Babylon was Anthony Shirley, an Englishman who visited Mesopotamia in fifteen ninety-nine. +DUMMY/LJ007-0172.wav|where is to be found in operation every expedient by which Ignorance may be superseded by Knowledge, Idleness by Industry, and Suffering by Benevolence; +DUMMY/LJ018-0099.wav|Among them was a man named Chandler, formerly a bill discounter by profession, who by degrees, to meet his extravagant expenditure, +DUMMY/LJ013-0066.wav|By this means her signature was obtained; a forged will was prepared bequeathing the unclaimed stock to Miss Slack; +DUMMY/LJ025-0127.wav|I know of no test by which the reaction of the leaves of the Sundew and of other plants to stimuli, so fully and carefully studied by Mr. Darwin, +DUMMY/LJ037-0118.wav|Callaway told the Commission, quote, So they brought four men in. +DUMMY/LJ010-0129.wav|but Ings said he hoped God would be more merciful to him than men had been. +DUMMY/LJ012-0181.wav|Then he attacked the other's throat with a penknife, and last of all drove the pistol barrel into his forehead. +DUMMY/LJ003-0002.wav|The squalor and uncleanness of the debtors' side was intensified by constant overcrowding. +DUMMY/LJ027-0138.wav|that there is often a close correspondence between developmental changes as revealed by any chronological series of fossils which may happen to have been preserved, +DUMMY/LJ037-0015.wav|but he saw the policeman leave the car, heard three or four shots, and then saw the policeman fall. +DUMMY/LJ038-0029.wav|stepped out onto the stage with them and pointed out the man who had come into the theatre without paying. +DUMMY/LJ004-0126.wav|The want of room was still a crying evil. +DUMMY/LJ046-0013.wav|Prompted by these dismaying statistics, the Commission has inquired into the problems and methods of Presidential protection in effect +DUMMY/LJ014-0276.wav|Watts's crime was discovered by the secretary of the Globe Company, who came suddenly upon the extensive falsification of the passbook. +DUMMY/LJ038-0118.wav|The Aliases "Hidell" and "O. H. Lee" +DUMMY/LJ016-0411.wav|There was a smile on his face when it was last seen, and just as the terrible white cap was drawn over it. +DUMMY/LJ018-0150.wav|with whom he did business in exchanging the false notes. +DUMMY/LJ005-0200.wav|Most of these small jails were still in existence and in much the same state eight years later, +DUMMY/LJ026-0082.wav|Carbon dioxide and water contain no potential energy, since the affinities of their constituent elements are completely satisfied. +DUMMY/LJ040-0124.wav|This continued despite the efforts of the school authorities and, to a lesser extent, of his mother to have him return to school. +DUMMY/LJ015-0258.wav|went to the police in great fury and distress, and disclosed all she knew of the affair. +DUMMY/LJ007-0070.wav|and that he had again and again protested against the obstacles thrown in his way, the inspectors "cannot forbear expressing their opinion that he might have shown greater perseverance, +DUMMY/LJ009-0250.wav|in such a way that the executioner found a difficulty in pinioning his hands. +DUMMY/LJ048-0224.wav|nine agents who were off duty went to the nearby Fort Worth Press Club at midnight or slightly thereafter, expecting to obtain food; +DUMMY/LJ023-0096.wav|In the case of holding the A.A.A. unconstitutional, +DUMMY/LJ011-0097.wav|that of Dickson and Co. +DUMMY/LJ038-0104.wav|R. S. Stovall, and G. F. Rose obtained a search warrant and examined Oswald's effects in the Paine garage. +DUMMY/LJ028-0209.wav|On the sixteenth day the troops of Cyrus entered Babylon without a battle. +DUMMY/LJ027-0010.wav|and all the rest are modifications from these few types. +DUMMY/LJ046-0237.wav|As reported in chapter two, when the special file was reviewed on November eight, +DUMMY/LJ015-0249.wav|The theft was discovered at Boulogne, when the boxes were found not to weigh exactly what they ought. But no clue was obtained to the thieves, +DUMMY/LJ044-0133.wav|He had not found another job. His wife was expecting their second child in October and there was concern about the cost which would be involved. +DUMMY/LJ039-0109.wav|forty-five caliber pistol, and twelve-gauge riot gun. +DUMMY/LJ029-0101.wav|For the President's return to Love Field following the luncheon, the agents selected the most direct route, which was approximately four miles. +DUMMY/LJ014-0297.wav|The prison officers were called, but Watts was quite cold and stiff when he was cut down. +DUMMY/LJ047-0228.wav|that the motorcade was coming up Main Street, quote, where maybe I could watch it if I had a chance, end quote. +DUMMY/LJ003-0191.wav|strange and inconceivable delay occurred in carrying out the extreme sentence. +DUMMY/LJ049-0147.wav|This arrangement would provide a continuing high-level contact for agencies that may wish to consult respecting particular protective measures. +DUMMY/LJ002-0024.wav|Quote. the apartments set apart for them, being built to accommodate sixty persons, now contain about one hundred twenty. End quote. +DUMMY/LJ041-0157.wav|He studied the Russian language, read a Russian language newspaper and seemed interested in what was going on in the Soviet Union. +DUMMY/LJ043-0100.wav|It is not possible to tell whether Oswald did this to provide an excuse for his eventual discharge, +DUMMY/LJ016-0143.wav|He was out of sight while so employed, and remained so long absent that the warder, becoming uneasy, went in search of him. +DUMMY/LJ010-0303.wav|According to Fauntleroy's own case, he found at once that the firm was heavily involved, +DUMMY/LJ049-0198.wav|in all its ramifications and alert to every danger that might befall it, +DUMMY/LJ001-0090.wav|With this change the art of printing touched bottom, +DUMMY/LJ038-0249.wav|Investigation determined that this photograph was taken approximately seven-tenths of a mile from Walker's house. +DUMMY/LJ025-0009.wav|for in the past fifty years it has been made evident that in general principles all living things are fundamentally similar. +DUMMY/LJ038-0137.wav|the Commission found that Oswald lied when he told Frazier that he was returning to Irving to obtain curtain rods. +DUMMY/LJ050-0159.wav|The Commission recommends that prompt and favorable consideration be given to this request. +DUMMY/LJ009-0214.wav|and was accompanied throughout by "an immense concourse of persons, eager to get a sight of the murderer's remains. +DUMMY/LJ008-0039.wav|By practice the art is much improved, and there is no part of the world in which villains are hanged in so neat a manner, and with so little ceremony. +DUMMY/LJ032-0217.wav|He could not test Exhibit Number one thirty-three A in the same way because the negative was never recovered. +DUMMY/LJ014-0223.wav|The judge, in passing sentence of death, told him he richly deserved the punishment. +DUMMY/LJ009-0086.wav|But why should I repeat the whole? +DUMMY/LJ045-0225.wav|Oswald's behavior after the assassination throws little light on his motives. +DUMMY/LJ011-0180.wav|Wakefield added that it had been suggested he should marry Miss Turner, but that he had laughed at the idea. +DUMMY/LJ008-0042.wav|is enclosed by a temporary roof, under which are placed two seats for the reception of the sheriffs, one on each side of the stairs leading to the scaffold. +DUMMY/LJ011-0246.wav|there were weapons in the room -- a long knife, a heavy trap-ball bat, and a poker. +DUMMY/LJ020-0045.wav|Make a hole in the middle of the other batch and pour into it a tablespoonful of butter, just melted, but not hot. +DUMMY/LJ013-0023.wav|that he got a chance of accomplishing his crime. +DUMMY/LJ009-0010.wav|and thought every one anxious to promote his death. +DUMMY/LJ009-0011.wav|Mr. Wakefield goes on to remark that persons convicted of forgery "excited an extraordinary degree of interest in all who approached them." +DUMMY/LJ003-0334.wav|It made this too the excuse for begging the most important issue of the whole question. +DUMMY/LJ016-0317.wav|but believed that a public ceremony destroyed the whole value of an execution. +DUMMY/LJ003-0046.wav|could not be carried out till then. It is to be feared that long after the opening of White Cross Street prison, +DUMMY/LJ047-0181.wav|and realized immediately that Oswald had given false biographic information. Hosty knew that he would eventually have to investigate this, and, quote, +DUMMY/LJ030-0242.wav|I was startled by the sharp report or explosion, +DUMMY/LJ012-0145.wav|Solomons was now also admitted as a witness, and his evidence, with that of Moss, secured the transportation of the principal actors in the theft. +DUMMY/LJ020-0044.wav|Mould one into a round ball, and set aside for a loaf as already directed. +DUMMY/LJ044-0100.wav|Following the disclosure of his defection, Oswald sought advice from the Communist Party, U.S.A., concerning his Fair Play for Cuba activity. +DUMMY/LJ028-0247.wav|Owing to the vast size of the place, the inhabitants of the central parts (as the residents of Babylon declare), +DUMMY/LJ023-0012.wav|But when, almost two years later, it came before the Supreme Court its constitutionality was upheld only by a five-to-four vote. +DUMMY/LJ004-0041.wav|with advantage step in, and by the collection and diffusion of information, and the reiteration of sound advice, greatly assist the good work. +DUMMY/LJ035-0205.wav|and no one could be found who saw Oswald anywhere else in the building until after the shooting. +DUMMY/LJ029-0069.wav|including a detailed description of the available data on Lee Harvey Oswald and the reasons why his name had not been furnished to the Secret Service, +DUMMY/LJ049-0054.wav|The Secret Service has consistently followed two general principles in emergencies involving the President. +DUMMY/LJ005-0011.wav|were first made use of about eighteen twenty-seven. That the need for prison reform was imperative may be gathered from the few out of many instances I have adduced, +DUMMY/LJ025-0042.wav|They retain the hydrogen and the carbon, exhale the superfluous oxygen and absorb little or no nitrogen. +DUMMY/LJ005-0059.wav|With some important modifications this principle of radiation is still the rule. +DUMMY/LJ050-0186.wav|in formats that can be communicated to each level of local authorities. +DUMMY/LJ006-0035.wav|took voluminous evidence from all classes of individuals, from the governor down to the convict in the condemned cells. +DUMMY/LJ009-0120.wav|Almighty God, Father of all mercies, we thine unworthy servants do give thee most humble and hearty thanks +DUMMY/LJ010-0176.wav|He was on the right or north side of the road. Prince Albert occupied the same side of the carriage, the Queen the left. +DUMMY/LJ009-0143.wav|He sat on a bench before the pulpit, -- the hideous condemned pew had been swept away, -- and never once raised his eyes during the service. +DUMMY/LJ033-0082.wav|He carried a, quote, heavy brown bag, end quote. +DUMMY/LJ044-0129.wav|and often it is advisable for some people to remain in the background, not underground, end quote. +DUMMY/LJ008-0028.wav|Some indications of pity were offered for the fate of Tilling; Palm, execration alone. +DUMMY/LJ007-0025.wav|There were frequent quarrels and fights; shoes and other missiles were freely bandied about; +DUMMY/LJ006-0216.wav|Not only did prisoners come again and again for a "pint," but large quantities were carried off to the wards to be drunk later in the day. +DUMMY/LJ030-0133.wav|Evaluating the political overtones, Kenneth O'Donnell was especially pleased +DUMMY/LJ015-0098.wav|Robson said they were at Kilburn Priory. +DUMMY/LJ049-0171.wav|With two Federal agencies operating in the same general field of preventive investigation, +DUMMY/LJ028-0057.wav|but about his name so many legends have grown that it is sometimes difficult to learn the facts of his life. +DUMMY/LJ010-0220.wav|We then sent the old woman to open the door, and it proved to be some little boys who had knocked and ran away. +DUMMY/LJ019-0256.wav|Still greater were the differences with regard to employment. The various authorities held widely different opinions as to what constituted hard labor. +DUMMY/LJ015-0250.wav|and the theft might have remained a mystery but for the subsequent bad faith of Pierce to his accomplice Agar. +DUMMY/LJ050-0025.wav|or from which it derives assistance and support. +DUMMY/LJ031-0136.wav|The Vice President decided to wait until he received definitive word of the President's condition. +DUMMY/LJ024-0048.wav|The number of justices has been changed several times before, +DUMMY/LJ036-0108.wav|and they asked me which one and I told them. It was him all right, the same man. He showed no respect for the policemen, +DUMMY/LJ032-0050.wav|It bore the manufacturer's serial number C two seven six six. +DUMMY/LJ008-0236.wav|coiled up on the floor of the scaffold like a serpent, the hangman's rope! +DUMMY/LJ009-0235.wav|and having first pulled him sideways, then got upon his shoulders, so that the rope broke. +DUMMY/LJ028-0214.wav|for upon a barrel-shaped cylinder of clay bearing a long inscription we have Cyrus's account of his capture of Babylon. Extracts from it are as follows: +DUMMY/LJ043-0056.wav|Even though Oswald cut off relations with his mother, +DUMMY/LJ034-0055.wav|was probably made within a day or a day and a half of the examination on November twenty-two. +DUMMY/LJ018-0252.wav|to deeds involving on the whole some three hundred fifty thousand pounds. +DUMMY/LJ018-0380.wav|They paid close attention to the counsels of the archimandrite, and died quite penitent. A story is told of one of them, "Big Harry," +DUMMY/LJ031-0175.wav|O'Donnell tried to persuade Mrs. Kennedy to leave the area, but she refused. +DUMMY/LJ031-0056.wav|Observing that an effective airway had to be established if treatment was to be effective, Dr. Perry performed a tracheotomy, which required three to five minutes. +DUMMY/LJ045-0224.wav|and such a favorable opportunity to strike at a figure as great as the President would probably never have come to him again. +DUMMY/LJ016-0160.wav|The noose was one of his hammock straps, which he buckled round his throat. +DUMMY/LJ029-0012.wav|He had made only a few brief visits to the State since the nineteen sixty Presidential campaign and in nineteen sixty-two he began to consider a formal visit. +DUMMY/LJ049-0169.wav|the procedures which its agents are to follow in connection with information received, quote, +DUMMY/LJ011-0094.wav|The second conviction for forgery in eighteen twenty-eight was that of the Quaker Joseph Hunton, a man of previously the highest repute in the city of London. +DUMMY/LJ031-0098.wav|I suppose nobody really had the heart to do it. +DUMMY/LJ015-0123.wav|In many respects the embezzlement of which Leopold Redpath was guilty closely resembled that of Robson, +DUMMY/LJ001-0035.wav|they discarded this for a more completely Roman and far less beautiful letter. +DUMMY/LJ012-0172.wav|Since the calamitous event," he went on, "that happened at Hertford, I have been a lost man." +DUMMY/LJ007-0002.wav|Eighteen years had elapsed since the formation of the "Ladies' Association," +DUMMY/LJ035-0134.wav|that they were watching the parade from the top step of the building entrance when Gloria Calverly, who works in the Depository Building, +DUMMY/LJ026-0083.wav|Starch, however, contains potential energy, since the molecule is relatively unstable +DUMMY/LJ009-0025.wav|As to the exclusion of strangers on these occasions, +DUMMY/LJ039-0014.wav|Marina Oswald appeared before the Commission again on June eleven, nineteen sixty-four, +DUMMY/LJ046-0215.wav|Any information indicating the possibility of an attempt against the person or safety of the President, +DUMMY/LJ016-0401.wav|His conversation turned always upon his influence over the weaker sex, and the extraordinary success he had achieved. +DUMMY/LJ019-0345.wav|The Act of eighteen sixty-five also encouraged and empowered the local authorities to "alter, enlarge, or rebuild" their prisons. +DUMMY/LJ004-0157.wav|All charged with felony were in heavy irons, without distinction of age. +DUMMY/LJ037-0022.wav|He had not seen Oswald on television and had not been shown any photographs of Oswald by the police. +DUMMY/LJ027-0086.wav|To the superficial observer the bodies of animals of different classes seem to differ fundamentally in plan, +DUMMY/LJ037-0138.wav|Reynolds did not make a positive identification when interviewed by the FBI, but +DUMMY/LJ019-0226.wav|the erection of a wing or large block of cells was commenced within the original walls of the prison, and upon the north or male side. +DUMMY/LJ030-0164.wav|Soon after the motorcade turned onto Elm Street, she heard a sound similar to a motorcycle noise and a cry from Governor Connally, +DUMMY/LJ018-0344.wav|In cold-blooded premeditation it rivaled that of the Mannings. +DUMMY/LJ043-0175.wav|The items which Oswald left at home when he made his attack on Walker suggest a strong concern for his place in history. +DUMMY/LJ001-0033.wav|but which must certainly have come from the study of the twelfth or even the eleventh century MSS. +DUMMY/LJ015-0021.wav|Sir John Dean Paul had inherited a baronetcy from his father, together with an honored name; +DUMMY/LJ030-0138.wav|At twelve:thirty p.m., Eastern Standard Time, as the President's open limousine proceeded at approximately eleven miles per hour along Elm Street +DUMMY/LJ014-0307.wav|were intended by the protectionists to depress the wheat market, and secure the support of the farmers at the forthcoming election; +DUMMY/LJ010-0104.wav|Thistlewood and fourteen more succeeded for the moment in making their escape, but most of them were subsequently taken. +DUMMY/LJ045-0012.wav|or rightwing extremism which may have existed in the city of Dallas had any connection with Oswald's actions on November twenty-two, nineteen sixty-three. +DUMMY/LJ006-0032.wav|Messrs. Crawford and Russell proceeded to carry out their new functions with commendable energy, and without a moment's loss of time. +DUMMY/LJ015-0306.wav|and the whole scheme of systematic forgery was laid bare. +DUMMY/LJ032-0186.wav|when it was transported from New Orleans, and November twenty-two, the day of the assassination. +DUMMY/LJ049-0019.wav|The last Presidential vehicle with any protection against small-arms fire left the White House in nineteen fifty-three. +DUMMY/LJ042-0197.wav|seemed to place him in a situation in which he could not live with satisfaction either in the United States or in the Soviet Union. +DUMMY/LJ049-0150.wav|A Cabinet-level committee which is actively concerned with these problems would be able to discuss these matters more effectively with the President. +DUMMY/LJ028-0016.wav|In the distance your eyes detect a beautiful lake with shores fringed with trees, +DUMMY/LJ019-0237.wav|The Act for private executions led to the erection of the gallows shed in the exercising yard, and at the flank of the passage from the condemned cells. +DUMMY/LJ037-0134.wav|Russell and Patterson were shown a picture of Oswald and they stated that Oswald was the man they saw on November twenty-two, nineteen sixty-three. +DUMMY/LJ032-0193.wav|Because the relative freshness of these fibers might be explained by the continuous storage of the rifle in the blanket, +DUMMY/LJ047-0098.wav|inasmuch as the police had not given Oswald's name to the Bureau when they called the office. +DUMMY/LJ048-0271.wav|will be cause for removal from the Service, end quote. +DUMMY/LJ019-0061.wav|Another argument was, that it afforded more hope of the reformation of criminals. +DUMMY/LJ038-0288.wav|The admissions made to Marina Oswald by her husband are an important element in the evidence that Lee Harvey Oswald fired the shot at General Walker. +DUMMY/LJ042-0208.wav|The deportations, the purposeful curtailment of diet in the consumer slighted population of Russia, +DUMMY/LJ005-0230.wav|These justices were to inspect every part of the prison, and examine into the state and condition of prisoners. +DUMMY/LJ009-0224.wav|At the first-named the exhibition nearly created a tumult, and the body was taken down and buried, +DUMMY/LJ005-0076.wav|By the preamble of the first-named act it was declared +DUMMY/LJ030-0213.wav|Hill heard a second shot, proximately five seconds after the first, which removed a portion of the President's head. +DUMMY/LJ011-0085.wav|a distinguished officer, and would have married an heiress had not the imposture been discovered in time. +DUMMY/LJ027-0097.wav|Thus, in the fin of a fish, the fore-paw of a reptile or a mammal, the wing of a bird, and the arm and hand of a man +DUMMY/LJ049-0100.wav|The Commission recommends to the Congress that it adopt legislation which would: +DUMMY/LJ043-0018.wav|He was offensive with the people. And I can understand why, because that hurt him. +DUMMY/LJ019-0359.wav|He could in the first place withhold the government grant in aid of prison funds by refusing the certificate to the Treasury upon which the allowance was paid. +DUMMY/LJ003-0240.wav|with ruined character, bad habits, and often bad health contracted in the jail. +DUMMY/LJ012-0077.wav|and the thieves seem to have had no difficulty in disposing of their spoil. +DUMMY/LJ028-0240.wav|on which the river sank to such an extent that the natural bed of the stream became fordable. +DUMMY/LJ048-0082.wav|The testimony reveals that liaison responsibilities in connection with the President's visit +DUMMY/LJ017-0178.wav|It appeared that several persons with whom she was intimate had succumbed suddenly. +DUMMY/LJ014-0018.wav|It came out by degrees that the person who had been seen in Belsize Lane on the night the body was found was a friend of the deceased. +DUMMY/LJ042-0158.wav|left a far mean streak of independence brought on by neglect. +DUMMY/LJ004-0106.wav|They were to report in writing to quarter sessions as to the state of the jail, and as to all abuses which they might observe therein. +DUMMY/LJ031-0013.wav|Traveling at speeds estimated at times to be up to seventy or eighty miles per hour down the Stemmons Freeway and Harry Hines Boulevard +DUMMY/LJ011-0023.wav|A run upon the bank immediately followed, which was only met by a suspension of payment and the closing of its doors. +DUMMY/LJ018-0348.wav|Henry Wainwright's impassioned denial of his crime, even after it had been brought fully home to him, has many parallels in the criminal records. +DUMMY/LJ028-0493.wav|whose summit reached into the sky as far, perhaps, as the tallest of the modern buildings. +DUMMY/LJ007-0136.wav|But there were others remediable under a better system of management. +DUMMY/LJ004-0037.wav|Yet the borough jails were destined to survive many years, and to exhibit for a long time to come all the worst features of jail mismanagement. +DUMMY/LJ045-0038.wav|might also have been important to him. It would not have been the first time that Oswald sought closer ties with his family in time of adversity. +DUMMY/LJ023-0105.wav|In the face of these dissenting opinions, there is no basis for the claim made by some members of the Court +DUMMY/LJ028-0281.wav|Till mules foal ye shall not take our city, he thought, as he reflected on this speech, that Babylon might now be taken, +DUMMY/LJ047-0003.wav|Information known about Lee Harvey Oswald prior to the assassination. +DUMMY/LJ037-0263.wav|The foregoing evidence establishes that (one) two eyewitnesses who heard the shots and saw the shooting of Dallas Police Patrolman J. D. Tippit +DUMMY/LJ014-0195.wav|two of them supported Cope, who was still alive, although insensible, and Marley was apprehended. The evidence against him was completed +DUMMY/LJ021-0029.wav|that we approached our task of reviving private enterprise in March, nineteen thirty-three. +DUMMY/LJ046-0035.wav|After reviewing this aspect of the matter this chapter will set forth the Commission's conclusions +DUMMY/LJ037-0254.wav|Two other witnesses, Sam Guinyard and William Arthur Smith, +DUMMY/LJ009-0209.wav|which was dressed in blue trousers and a blue-and-white striped waistcoat, but without a coat, as when found in the cell. +DUMMY/LJ009-0020.wav|previous to the execution of Henry Fauntleroy for uttering a forged security +DUMMY/LJ001-0174.wav|should be a part of the whole scheme of the book. +DUMMY/LJ035-0006.wav|are consistent with his having been at the window at twelve:thirty p.m. +DUMMY/LJ020-0082.wav|Roll out into a sheet half an inch thick with a floured rolling-pin; cut into round cakes, set these closely together in a well-greased pan; +DUMMY/LJ014-0175.wav|When lying under sentence of death in Newgate, he was visited by a lady, a gentlewoman in every sense of the word, who was said to be his sister. +DUMMY/LJ019-0325.wav|the ministrations of ministers of their own form of belief. +DUMMY/LJ032-0008.wav|This evidence includes (one) the ownership and possession of the weapon used to commit the assassination, +DUMMY/LJ050-0225.wav|and be available to protect the President or Vice President when they travel; +DUMMY/LJ012-0193.wav|The crimes of these inhuman purveyors to medical science do not fall within the limits of this work. +DUMMY/LJ047-0163.wav|According to Hosty, Mrs. Paine indicated that she thought she could find out where Oswald was living and would let him know. +DUMMY/LJ012-0148.wav|instead of handing it or the price of it back to the Caspars. +DUMMY/LJ008-0100.wav|The yeoman was, however, under-sold by his wife, "Rosy Emma," +DUMMY/LJ048-0217.wav|The agents in the follow-up car also were expected to scan adjacent buildings. +DUMMY/LJ036-0097.wav|The taxicab ride. +DUMMY/LJ021-0059.wav|The work day and the work week have been shortened. +DUMMY/LJ043-0090.wav|Even though Oswald indicated that he liked photographic work, his employer found that he was not an efficient worker. +DUMMY/LJ037-0160.wav|He explained that the bullets were slightly smaller than the barrel of the pistol which had fired them. This caused the bullets to have an erratic passage through the barrel +DUMMY/LJ016-0203.wav|He got besides half-a-crown for every man he flogged, and an allowance to provide cats or birch rods. +DUMMY/LJ038-0041.wav|McDonald felt the pistol scratch his cheek as he wrenched it away from Oswald. +DUMMY/LJ008-0018.wav|extended upon a plank on the top of an open cart, in his clothes, and fettered. +DUMMY/LJ006-0008.wav|These gentlemen were appointed in October eighteen thirty-five, owing to the strong representations of the Lords' Committee +DUMMY/LJ015-0303.wav|but on being searched, two blank cheques of the London and Westminster Bank were found in his pocket. +DUMMY/LJ006-0297.wav|Some member of the Ladies' Association observed and commented upon the fact that a "young rosy-cheeked girl" had been kept by the governor from transportation, +DUMMY/LJ040-0095.wav|Referring to the period after the divorce from Ekdahl, which was apparently caused in part by Marguerite's desire to get more money from him, +DUMMY/LJ001-0065.wav|This was notably the case with the early works printed at Ulm, and in a somewhat lesser degree at Augsburg. +DUMMY/LJ011-0187.wav|Returning to Carlisle, she now heard that her father had been set free, and had gone home to Shrigley, whither they were to follow him. +DUMMY/LJ002-0062.wav|The middle yard was at first given up to the least heinous offenders. After eighteen twelve it changed functions with the chapel yard. +DUMMY/LJ040-0065.wav|That death strained the financial fortunes of the remainder of the Oswald family. +DUMMY/LJ017-0053.wav|there was certainly nothing in it which indicated cruel cunning or deliberate truculence. +DUMMY/LJ020-0047.wav|Put it into a floured bowl, cover with a cloth and set away out of draught and undue heat, for three hours. +DUMMY/LJ014-0061.wav|and Manning hoped to get some small Government appointment through his wife's interest. +DUMMY/LJ002-0076.wav|The state side contained twelve good-sized rooms, +DUMMY/LJ008-0076.wav|When she came out of prison she appeared languid and terrified, and trembled greatly as she advanced to the stake, +DUMMY/LJ046-0167.wav|When PRS learns of an individual whose conduct warrants scrutiny, it requests an investigation by the closest Secret Service field office, +DUMMY/LJ005-0028.wav|"There should be no tea and sugar, no assemblage of female felons around the washing-tub, +DUMMY/LJ045-0065.wav|Marina Oswald's inability to speak English also made it more difficult for her to have an independent existence in this country. +DUMMY/LJ036-0140.wav|He gave me a dollar bill and didn't say anything, just got out and closed the door and walked around the front of the cab over to the other side of the street +DUMMY/LJ015-0259.wav|Agar too, in Newgate, heard how Pierce had treated him, and at once readily turned approver. +DUMMY/LJ039-0234.wav|the Commission concluded that Oswald was capable of accomplishing this second hit even if there was an intervening shot which missed. +DUMMY/LJ042-0220.wav|he would be asked at a press conference when he returned. +DUMMY/LJ008-0152.wav|made a full confession, and implicated Holloway and Haggerty. +DUMMY/LJ025-0089.wav|The third distinction is based on a completely erroneous conception of the chemical differences +DUMMY/LJ048-0009.wav|Both Director Hoover and his assistant, Alan H. Belmont, +DUMMY/LJ003-0188.wav|The old press yard has been fully described in a previous chapter. +DUMMY/LJ023-0054.wav|I hope that you have re-read the Constitution of the United States in these past few weeks. +DUMMY/LJ004-0105.wav|two or more justices were appointed visitors of prisons, and directed to visit and inspect three times every quarter. +DUMMY/LJ044-0078.wav|and as he was to some extent when he returned to the United States. +DUMMY/LJ028-0454.wav|but both sections originally reached the river. +DUMMY/LJ021-0128.wav|Both employers and employees must share the blame of not using them as fully as they should. +DUMMY/LJ012-0158.wav|Before his confinement he had been a sleek round person, addicted obviously to the pleasures of the table. +DUMMY/LJ008-0017.wav|and when he evaded the sentence by suicide, his body was exhibited in the same neighborhood, +DUMMY/LJ026-0069.wav|as the potential energy of foodstuffs, but comes in principally as the kinetic energy of sunlight absorbed in the leaves. +DUMMY/LJ029-0139.wav|in order to avoid the traffic hazards which would otherwise exist if right turns were permitted from both Main and Elm into the freeway. +DUMMY/LJ030-0030.wav|President and Mrs. Kennedy walked along a chain-link fence at the reception area greeting a large crowd of spectators that had gathered behind it. +DUMMY/LJ010-0231.wav|His motives for thus imitating the dastardly crime of Oxford are shrouded in obscurity. +DUMMY/LJ006-0105.wav|and not with weights and measures, no turnkey being present. +DUMMY/LJ049-0081.wav|There was no Federal criminal jurisdiction over the assassination of President Kennedy. +DUMMY/LJ033-0146.wav|Scientific Evidence Linking Rifle and Oswald to Paper Bag +DUMMY/LJ026-0041.wav|may flourish in solutions wholly devoid of organic matter is very significant. +DUMMY/LJ012-0170.wav|According to his statement, when sentenced to death, he had been driven to horse-stealing by the execration which had pursued him after the murder. +DUMMY/LJ029-0160.wav|on its way to the Trade Mart. On November nineteen, the Times-Herald afternoon paper detailed the precise route: +DUMMY/LJ026-0136.wav|it may be reconverted into starch after journeying as glucose through the plant +DUMMY/LJ047-0111.wav|Quigley advised Knack of his interview and gave him a detailed memorandum. +DUMMY/LJ008-0147.wav|whither he often went to distill the lavender, returning to London the same evening. +DUMMY/LJ017-0245.wav|On the second October they sighted land, ten miles distant; +DUMMY/LJ040-0049.wav|He stated several times that he was a Communist but apparently never joined any Communist Party. +DUMMY/LJ044-0016.wav|which was berthed at the Dumaine Street wharf in New Orleans, on June sixteen, nineteen sixty-three. +DUMMY/LJ040-0078.wav|Lee Oswald remained with his mother and Ekdahl, to whom he became quite attached. +DUMMY/LJ034-0217.wav|One of these employees was alleged to resemble Lee Harvey Oswald. +DUMMY/LJ029-0142.wav|This concrete barrier extends far enough beyond the access road to make it impracticable for vehicles to turn right from Main directly to the access road. +DUMMY/LJ022-0119.wav|It must, however, be recognized that when an enterprise of this character is extended over more than three thousand counties throughout the nation, +DUMMY/LJ036-0201.wav|At twelve:forty-five p.m. the dispatcher ordered Number seventy-eight (Tippit) to, quote, move into central Oak Cliff area, end quote. +DUMMY/LJ029-0121.wav|and that the wide streets that we would use downtown would be of sufficient width to keep the public out of our way. +DUMMY/LJ037-0069.wav|Although in the phone conversation she described the man as, quote, short, end quote, on November twenty-second, +DUMMY/LJ019-0252.wav|while its rigid maintenance was in its opinion vital to the efficiency of the jails. +DUMMY/LJ041-0074.wav|John E. Donovan, one of his former officers, testified that Oswald thought, quote, +DUMMY/LJ048-0247.wav|had duty assignments beginning no later than eight a.m. that morning. +DUMMY/LJ016-0100.wav|At that time rope mats were still used as beds. +DUMMY/LJ045-0006.wav|and which received publicity there prior to the assassination. +DUMMY/LJ012-0154.wav|He was ostensibly a publican, +DUMMY/LJ048-0109.wav|by these Secret Service agents with the cooperation of the Dallas police and other local law enforcement agents, were carefully executed. +DUMMY/LJ011-0092.wav|which it was asserted he had been in the habit of carrying about his person ever since he had taken to passing forged notes, as an antidote against disgrace. +DUMMY/LJ013-0026.wav|The 'Dryad' was got off, repaired, and her voyage renewed to Santa Cruz. +DUMMY/LJ047-0094.wav|by anti-Castro Cubans while distributing leaflets on behalf of the Fair Play for Cuba Committee. +DUMMY/LJ009-0258.wav|However, the ceremony went forward, and when the signal was given the drop sank. +DUMMY/LJ024-0038.wav|that no President fit for his office would appoint, +DUMMY/LJ039-0146.wav|Marina Oswald testified that in New Orleans in May of nineteen sixty-three, she observed Oswald sitting with the rifle on their screened porch at night, +DUMMY/LJ005-0209.wav|through the bars of which quills or reeds were inserted, and drink conveyed to the prisoners. +DUMMY/LJ041-0108.wav|He said that he had simply wanted to discuss the question with the sergeant and the drink had been spilled accidentally. +DUMMY/LJ017-0154.wav|that Mrs. Smethurst had no right to the name, but had a husband of her own, one Johnson, alive -- a story subsequently disproved. +DUMMY/LJ024-0123.wav|An amendment, like the rest of the Constitution, +DUMMY/LJ019-0207.wav|The site of which was to be sold to Christ's Hospital, but the intention was of course to embody and absorb old Newgate in the new construction. +DUMMY/LJ011-0018.wav|The crime, long carried on without detection, was first discovered in eighteen twenty, +DUMMY/LJ002-0151.wav|Neild found at his visit to Newgate in eighteen ten, +DUMMY/LJ023-0098.wav|And two other justices agreed with him. +DUMMY/LJ018-0267.wav|His uniform good conduct gained him release from Portland on ticket-of-leave in eighteen eighty-two, just twenty years after his conviction. +DUMMY/LJ033-0006.wav|In this connection the Commission considered (one) +DUMMY/LJ015-0240.wav|Agar at once got to work on the first safe. +DUMMY/LJ010-0243.wav|with an equerry riding close on each side of her carriage, a man who had been leaning against the palace garden wall suddenly advanced, +DUMMY/LJ012-0253.wav|A further discovery was made in an osier bed near Cold Harbor Lane, Camberwell, +DUMMY/LJ037-0076.wav|Two young women, +DUMMY/LJ018-0184.wav|One narrow escape he had, however, before he abandoned his old business. +DUMMY/LJ041-0062.wav|testified that, quote, definitely the Marine Corps was not what he had expected it to be when he joined, end quote, +DUMMY/LJ038-0286.wav|when combined with the other testimony linking Oswald to the shooting. +DUMMY/LJ002-0135.wav|but as time passed, difficulties and delays in obtaining judgment led to the removal of causes to the great Court of King's Bench, +DUMMY/LJ020-0066.wav|One quart and a cup of flour sifted. +DUMMY/LJ032-0097.wav|Expert testimony confirmed that the writing on this application was that of Lee Harvey Oswald. +DUMMY/LJ039-0212.wav|Frazier said, quote, The fact that the crosshairs are set high would actually compensate for any lead which had to be taken +DUMMY/LJ006-0128.wav|The brief-drawing gatesman and wardsman at the time of the inspectors' first visit must have been a particularly powerful personage. +DUMMY/LJ034-0173.wav|He said that the man held his attention until the motorcade came because the man, quote, +DUMMY/LJ030-0234.wav|Directing the security detail for the Vice President from the right front seat of the Vice-Presidential car, Special Agent Youngblood recalled, quote, +DUMMY/LJ016-0126.wav|The top of the wall was gained without difficulty. +DUMMY/LJ033-0101.wav|so that it was carried straight and parallel to his body. +DUMMY/LJ049-0108.wav|In addition, the restriction would probably eliminate a need for the requirement which has been urged as necessary for the exercise of Federal power, +DUMMY/LJ042-0173.wav|Such an alternative was to be opposed both to capitalism and communism because, quote, +DUMMY/LJ017-0028.wav|that she had had a quarrel with her mistress, and that the latter with all others who tasted the dumplings were similarly attacked, although no one died. +DUMMY/LJ025-0101.wav|decomposes carbonic acid and exhales oxygen while the animal absorbs oxygen and exhales carbonic acid, +DUMMY/LJ015-0031.wav|it had assumed enormous responsibilities -- on one side by the ownership of the Mostyn collieries, a valueless property, +DUMMY/LJ038-0155.wav|The next day, Oswald added to his story. +DUMMY/LJ005-0069.wav|Absence of embellishment is in perfect unison with the character of the establishment. +DUMMY/LJ035-0091.wav|Baker said, quote, We simulated the shots and by the time we got there, we did everything that I did that day, +DUMMY/LJ031-0044.wav|On the basis of these findings, Dr. Carrico concluded that President Kennedy was still alive. +DUMMY/LJ038-0143.wav|When told that Frazier and Mrs. Randle had seen him carrying a long heavy package, Oswald replied, quote, Well, they was mistaken. +DUMMY/LJ004-0095.wav|is found to be owing to want of cleanliness and fresh air in the several jails, +DUMMY/LJ038-0064.wav|He further stated that while fists were flying he heard one of the officers say, quote, Kill the President, will you, end quote. +DUMMY/LJ048-0243.wav|Only one stayed as long as a half hour, and none had any beverage there. +DUMMY/LJ041-0010.wav|A friend, Edward Voebel, testified that, quote, he was more bashful about girls than anything else, end quote. +DUMMY/LJ007-0207.wav|but this had the effect of throwing them into closer contact, and of making them more intimately acquainted with, more directly influential upon, one another. +DUMMY/LJ039-0140.wav|Soon after Oswald returned from the Soviet Union +DUMMY/LJ042-0059.wav|Despite this commitment to the Soviet Union +DUMMY/LJ012-0111.wav|Both the thieves were now apprehended, but only a small portion of the lost property was recovered, +DUMMY/LJ021-0021.wav|The relations between the employer and the employed, between the owners of aggregated capital and the units of organized labor, +DUMMY/LJ043-0070.wav|Oswald obtained a job in July of nineteen sixty-two as a sheet metal worker with a company in Fort Worth. +DUMMY/LJ012-0094.wav|While the excitement was still fresh, a new robbery of diamonds was committed at a bonded warehouse in the immediate neighborhood, on Custom House Quay. +DUMMY/LJ003-0166.wav|association at one time forbidden by custom, but which greed and rapacity long made the rule. +DUMMY/LJ005-0113.wav|In another jail not far off male and female felons still occupied the same room -- underground, and reached by a ladder of ten steps. +DUMMY/LJ029-0035.wav|bring the President, quote, through an area which exposes him to the greatest number of people, end quote. +DUMMY/LJ007-0111.wav|or witnesses who did not voluntarily come forward when the chance was given them. +DUMMY/LJ037-0153.wav|Robert A. Frazier and Charles Killion, other FBI firearms experts, +DUMMY/LJ002-0327.wav|or from persons who made a trade of it, or they might bring their beds with them into the prison. +DUMMY/LJ008-0198.wav|two pounds was a common price for a window. +DUMMY/LJ034-0010.wav|Below the southeast corner window on the sixth floor was a large carton of books +DUMMY/LJ028-0111.wav|At the entrance of the great gates I erected strong bulls of bronze, and terrible serpents standing upright. +DUMMY/LJ037-0231.wav|or eight inches, about one hundred forty-five pounds, end quote, and wearing a white jacket. +DUMMY/LJ015-0128.wav|but he bought bona fide amounts, and increased their value by altering the figures, +DUMMY/LJ015-0002.wav|The course of the swindlers was by no means smooth, but it was not till eighteen fifty-four that suspicion arose that anything was wrong. +DUMMY/LJ002-0245.wav|The prison was a nest of abuses, like its neighbor the King's Bench +DUMMY/LJ028-0475.wav|No traces of its moat have appeared. +DUMMY/LJ010-0135.wav|yelled out three cheers to the populace whom he faced. +DUMMY/LJ049-0016.wav|The Presidential limousine. +DUMMY/LJ046-0106.wav|largely measures taken by the Secret Service, is considered. +DUMMY/LJ032-0042.wav|From Klein's records it was possible to trace the processing of the order after its receipt. +DUMMY/LJ014-0239.wav|which followed each other at no long intervals, which had a strong family likeness, and originated all of them to make money easily, +DUMMY/LJ022-0121.wav|When cases of this kind occur, +DUMMY/LJ013-0115.wav|Of this class was the ingenious yet peculiarly simple robbery effected at the house of Lord Fitzgerald in Belgrave Square. +DUMMY/LJ030-0066.wav|Driving the Presidential limousine was Special Agent William R. Greer of the Secret Service; +DUMMY/LJ004-0088.wav|As a remedy, justices of the peace were empowered to provide materials for the setting of poor prisoners to work, +DUMMY/LJ034-0102.wav|Within minutes of the assassination, Brennan described the man to the police. +DUMMY/LJ028-0302.wav|I will desert to the enemy as I am, and when I get into their city I will tell them that it is by thee that I have been thus treated. +DUMMY/LJ036-0150.wav|He directed the driver of the car to a point twenty feet north of the northwest corner of the intersection of Beckley and Neely +DUMMY/LJ017-0197.wav|This mysterious medicine Wilson kept carefully locked up, and allowed no one to see it, +DUMMY/LJ038-0223.wav|from October nine, nineteen sixty-two, to May fourteen, nineteen sixty-three. +DUMMY/LJ001-0077.wav|In England about this time, an attempt was made (notably by Caslon, who started business in London as a type-founder in seventeen twenty) +DUMMY/LJ018-0015.wav|His identity was established by a bundle of letters in his pocket, which bore his full address: +DUMMY/LJ018-0320.wav|in which the plan of action was changed, and the freedom of the prisoners was to be obtained by means of a rescue from the dock during the trial. +DUMMY/LJ019-0314.wav|The separation of prisoners in cells duly certified by the inspectors was insisted upon, +DUMMY/LJ028-0438.wav|However, it is possible that the hanging gardens existed only in the imagination of the Greek writers, +DUMMY/LJ013-0255.wav|The victim was the first of these three. +DUMMY/LJ020-0055.wav|visiting every corner of the pan, but not leaving thick layers and streaks of grease after it. +DUMMY/LJ019-0046.wav|was the variety of views as regards the discipline to be introduced in these new prisons. The time was one +DUMMY/LJ017-0019.wav|Captain Donellan, who was convicted of poisoning his brother-in-law, Sir Theodosius Broughton, and executed for it, +DUMMY/LJ015-0012.wav|Cole too was apprehended, and in due course tried at the Central Criminal Court. +DUMMY/LJ007-0040.wav|in a second room were fourteen more who had every hope of a reprieve. +DUMMY/LJ019-0099.wav|moved for a committee to report upon the best means of securing some uniform system which should be "punitive, reformatory, and self-supporting;" +DUMMY/LJ013-0059.wav|There were two separate affairs. +DUMMY/LJ016-0197.wav|After that he worked alone. +DUMMY/LJ033-0065.wav|With Ruth Paine acting as an interpreter, Detective Rose asked Marina whether her husband had a rifle. +DUMMY/LJ041-0085.wav|Oswald, quote, seemed to be a person who would go out of his way to get into trouble, end quote, +DUMMY/LJ044-0223.wav|whether or not he agreed with Castro that President Kennedy was a, quote, ruffian and a thief, end quote. He replied that he, quote, +DUMMY/LJ032-0031.wav|Document examiners for the Treasury Department and the FBI testified unequivocally +DUMMY/LJ031-0155.wav|Men at the Trade Mart were driven to Parkland Hospital in Dallas police cars. +DUMMY/LJ013-0208.wav|Lord William, according to the valet, was of a peevish, difficult temper; +DUMMY/LJ005-0033.wav|hard, incessant, irksome, eternal labor, a planned and regulated and unrelenting exclusion of happiness and comfort. +DUMMY/LJ005-0245.wav|This committee animadverted strongly upon the system in force at the metropolitan jails, and more especially upon the condition of Newgate +DUMMY/LJ047-0090.wav|Our investigation of Oswald had disclosed no evidence that Oswald was acting under the instructions or on behalf of +DUMMY/LJ015-0238.wav|The thieves took first-class tickets, handed their bags full of shot to the porters, who placed them in the guard's van. +DUMMY/LJ018-0398.wav|Benson, the two Kurrs, Bale, and Murray, led to strange revelations of dishonest practices amongst the detective police, +DUMMY/LJ022-0094.wav|in whose field the project falls, and also to notify another agency which I am creating -- a Progress Division. +DUMMY/LJ003-0259.wav|and attracted attention by their disorderly behavior. +DUMMY/LJ039-0071.wav|Robert A. Frazier, FBI expert in firearms identification and training, said, quote, +DUMMY/LJ004-0185.wav|that the chaplain found it necessary to take his place before they entered chapel, as he could not otherwise have faced the smell. +DUMMY/LJ024-0071.wav|I propose that hereafter, when a judge reaches the age of seventy, +DUMMY/LJ047-0130.wav|advised the Bureau that Oswald was unknown in such circles. +DUMMY/LJ014-0034.wav|Hocker, whose skill in counterfeiting handwriting was known, was asked to fabricate a letter making an assignation with Delarue +DUMMY/LJ024-0108.wav|This is the same group who during the campaign last Fall tried to block the mandate of the people. +DUMMY/LJ031-0021.wav|As the President's limousine sped toward the hospital, twelve doctors to the emergency area: surgeons, Drs. Malcolm O. Perry, +DUMMY/LJ049-0152.wav|The assignment of the responsibility of protecting the President to an agency of the Department of the Treasury was largely an historical accident. +DUMMY/LJ010-0234.wav|backed possibly with the hope that, as in a much more recent case, +DUMMY/LJ011-0171.wav|was the state of her father's affairs. +DUMMY/LJ013-0016.wav|It was fully proved in evidence that when the Dryad sailed she carried nothing but the cargo belonging to Zulueta and Co. +DUMMY/LJ049-0087.wav|without making an attack on the President a crime. Threatening harm to the President is a Federal offense, +DUMMY/LJ018-0045.wav|An alibi was set up for the defense, but not well substantiated, and the jury without hesitation returned a verdict of guilty. +DUMMY/LJ021-0166.wav|we must make it a national principle that we will not tolerate a large army of unemployed and that we will arrange our national economy +DUMMY/LJ043-0180.wav|and with the circumstances surrounding the assassination, have led the Commission to believe +DUMMY/LJ049-0094.wav|The Commission agrees with the Secret Service that it should be authorized to make arrests without warrant +DUMMY/LJ034-0030.wav|They could be easily adjusted and were still solid enough to serve as a gun rest. +DUMMY/LJ030-0089.wav|This distance was maintained so that spectators would normally turn their gaze from the President's automobile by the time the Vice President came into view. +DUMMY/LJ019-0074.wav|The legislature was content to let local jurisdictions experimentalize for themselves; with the strange, anomalous result, +DUMMY/LJ045-0125.wav|For example, in his letter of November nine, nineteen sixty-three, +DUMMY/LJ011-0101.wav|Hunton tried to fly the country on the detection of the fraud, but was arrested at Plymouth just as he was on the point of leaving England in the New York packet. +DUMMY/LJ033-0097.wav|Frazier parked the car in the company parking lot about two blocks north of the Depository Building. +DUMMY/LJ019-0311.wav|The legislature was beginning to overcome its disinclination to interfere actively or authoritatively +DUMMY/LJ007-0117.wav|where the upper ward was exclusively appropriated to their use. They also had their meals sent in, and, with the food, wine almost ad libitum. +DUMMY/LJ029-0099.wav|From Love Field the route passed through a portion of suburban Dallas, +DUMMY/LJ016-0006.wav|The inspectors' reports mention many cases of evasion accomplished. +DUMMY/LJ026-0006.wav|for the manufacture of starchy food from gaseous matter. +DUMMY/LJ037-0023.wav|Another witness, Domingo Benavides, was driving a pickup truck west on tenth Street. +DUMMY/LJ012-0119.wav|The fire was raked out, and in the ashes were found seven large and four dozen small brilliants, also seven emeralds, +DUMMY/LJ043-0036.wav|he did not see his brother Robert from Thanksgiving of nineteen sixty-two until November twenty-three, nineteen sixty-three. +DUMMY/LJ050-0002.wav|The Warren Commission Report. +DUMMY/LJ044-0006.wav|Although, as indicated above, the Commission has been unable to find any credible evidence +DUMMY/LJ012-0100.wav|Some ham sandwiches, several cigar ends, and two empty champagne bottles were found on the premises next day, showing how they had passed their time. +DUMMY/LJ012-0177.wav|The victim was invited to visit Probert's cottage in the country near Elstree. +DUMMY/LJ019-0353.wav|they were at liberty to hire suitable cell accommodation from any neighbors who might have it to spare; the only proviso, +DUMMY/LJ016-0333.wav|from Clerkenwell prison, by which many persons lost their lives. +DUMMY/LJ046-0072.wav|have been confronted in recent years with increasingly difficult problems, created by the greater exposure of the President during his travels +DUMMY/LJ046-0143.wav|These instructions to PRS personnel appear to be the only instance where an effort was made to reduce the criteria to writing. +DUMMY/LJ009-0121.wav|for all thy goodness and loving-kindness to us, and to all men; +DUMMY/LJ002-0254.wav|and fines imposed for conduct disapproved of by the "college." +DUMMY/LJ046-0080.wav|If the protective job is well done, its performance will be evident only in the unexceptional fact of its success. +DUMMY/LJ034-0140.wav|Jarman heard Brennan, quote, talking to this officer about that he had heard these shots and he had seen the barrel of the gun sticking out the window, +DUMMY/LJ001-0127.wav|the modern letters are narrowed by a third or thereabout; but while this gain of space very much hampers the possibility of beauty of design, +DUMMY/LJ045-0118.wav|would see anything in the newspaper about his defection, unless he engaged in activities similar to those +DUMMY/LJ043-0086.wav|through a work-study program at a local college. +DUMMY/LJ013-0090.wav|But when the latter was filled with gold it was too heavy to lift, +DUMMY/LJ001-0149.wav|From the time when books first took their present shape till the end of the sixteenth century, or indeed later, +DUMMY/LJ016-0232.wav|and he owned a pet pony which would follow him about like a dog. +DUMMY/LJ019-0372.wav|It was practically inoperative as regards the penalties for neglect. It was no doubt as irksome and inconvenient to the Secretary of State +DUMMY/LJ026-0049.wav|In turn these will be compared for the animal and the plant, +DUMMY/LJ032-0269.wav|It could have been caused by the telescopic sight of the rifle which was approximately eleven inches long. +DUMMY/LJ018-0325.wav|But extra precautions and close supervision have so far proved effectual, and the prisoners are still in custody after a lapse of ten years. +DUMMY/LJ006-0267.wav|yet many persons of notoriously loose life continually obtained egress. +DUMMY/LJ039-0084.wav|you identify it and the crosshairs are in close relationship to the point you want to shoot at, +DUMMY/LJ027-0047.wav|totally different structures are modified to perform the same functions. +DUMMY/LJ033-0160.wav|Materials used to make bag. +DUMMY/LJ032-0273.wav|(four) a photograph taken in the yard of Oswald's apartment showed him holding this rifle, and (five) +DUMMY/LJ006-0005.wav|just before the public mind was first awakened to the need for thorough reform. +DUMMY/LJ016-0398.wav|As a special favor +DUMMY/LJ035-0043.wav|Yet he must have entered the vestibule door before Truly reached the top of the stairwell, since Truly did not see him. +DUMMY/LJ018-0221.wav|but upon this forged and false conveyance William Roupell, who had already embarked upon a career of wild extravagance, +DUMMY/LJ001-0061.wav|the Roman letter was used side by side with the Gothic. +DUMMY/LJ013-0128.wav|A watch was set on his house, in Allington Street, Pimlico, +DUMMY/LJ037-0182.wav|Ownership of Revolver +DUMMY/LJ028-0309.wav|After the twenty days are over, bid thy whole army attack the city on every side, and put me two bodies of Persians, +DUMMY/LJ007-0056.wav|leaden hearts, and "grinding the impressions off penny-pieces, then pricking figures or words on them to give to their friends as memorials. +DUMMY/LJ042-0209.wav|the murder of history, the prostitution of art and culture, end quote. +DUMMY/LJ037-0262.wav|These facts warrant the finding that Lee Harvey Oswald disposed of his jacket as he fled from the scene of the Tippit killing. +DUMMY/LJ006-0071.wav|No doubt the governor for the time being, Mr. Cope, was in a great measure to blame for all this, and for the want of proper classification. +DUMMY/LJ048-0114.wav|that the Secret Service has inadequately defined the responsibilities of its advance agents, who have been given broad discretion +DUMMY/LJ044-0178.wav|he stated his desire that he should be accepted as a friend of the Cuban Revolution. +DUMMY/LJ030-0236.wav|I quickly observed unnatural movement of crowds, like ducking or scattering, and quick movements in the Presidential follow-up car. +DUMMY/LJ035-0136.wav|Lovelady and Shelley moved out into the street. +DUMMY/LJ012-0202.wav|They pursued their terrible trade for five years without scruple and without detection. +DUMMY/LJ011-0280.wav|Mr. Gee at first stoutly refused. +DUMMY/LJ012-0239.wav|The execution took place a week or two later, in the presence of such a crowd as had not been seen near Newgate for years. +DUMMY/LJ006-0070.wav|where they inevitably meet with further contamination from the society of the most abandoned and incorrigible inmates of the jail." +DUMMY/LJ039-0156.wav|to seven point nine seconds. +DUMMY/LJ005-0140.wav|At the New Prison, Clerkenwell, which had become the principal reception jail of Middlesex, and so took all the untried, +DUMMY/LJ001-0031.wav|In fourteen sixty-five Sweynheim and Pannartz began printing in the monastery of Subiaco near Rome, +DUMMY/LJ026-0062.wav|Through their action the solid portions are liquefied and the food is rendered capable of absorption into the body proper. +DUMMY/LJ037-0107.wav|They saw a man coming south on Patton with a revolver held high in his right hand. According to Callaway, the man crossed to the west side of Patton. +DUMMY/LJ044-0168.wav|During this period Oswald may have practiced opening and closing the bolt on his rifle in a screened porch in his apartment. +DUMMY/LJ022-0198.wav|We have in the darkest moments of our national trials retained our faith in our own ability to master our destiny. +DUMMY/LJ003-0113.wav|A prisoner, generally the oldest and most dexterous thief, +DUMMY/LJ035-0004.wav|the Commission has reviewed the testimony of witnesses who saw Oswald in the building within minutes after the assassination. +DUMMY/LJ047-0235.wav|complete surprise, end quote, because he had no reason to believe that Oswald, quote, +DUMMY/LJ006-0198.wav|But this was not the only amusement. Most of the wards took in the daily papers, +DUMMY/LJ047-0121.wav|stated the Bureau's reasoning in this way, quote, +DUMMY/LJ038-0230.wav|The county jail is at the corner of Houston and Main Streets, quote, right in the beginning of the city, end quote, after one travels through the underpass. +DUMMY/LJ015-0070.wav|whence he passed to a better position under the Crystal Palace Company. +DUMMY/LJ023-0112.wav|We have, therefore, +DUMMY/LJ013-0019.wav|Loose had his instructions to cast away the ship on the first possible opportunity, and from the time of his leaving Liverpool +DUMMY/LJ039-0117.wav|He had the services of an experienced highly trained coach. +DUMMY/LJ022-0178.wav|We all know that private banking actually exists by virtue of the permission of and regulation by the people as a whole, +DUMMY/LJ038-0024.wav|Patrol cars bearing at least fifteen officers converged on the Texas Theatre. +DUMMY/LJ020-0006.wav|Three potatoes of fair size, peeled and boiled mealy. +DUMMY/LJ017-0277.wav|The soldiers then? +DUMMY/LJ047-0109.wav|When Quigley returned to his office, he learned +DUMMY/LJ019-0264.wav|and it was decidedly of opinion that in all short sentences the hard labor of the tread-wheel, crank, and so forth should be the invariable rule. +DUMMY/LJ003-0247.wav|or the faithful discharge of some petty office, such as gatesman or captain of a ward. +DUMMY/LJ010-0265.wav|and on his information handbills were circulated, giving the exact description of the deformed youth, who had a hump-back, +DUMMY/LJ031-0078.wav|At approximately one p.m., after last rites were administered to the President by Father Oscar L. Huber, Dr. Clark pronounced the President dead. +DUMMY/LJ004-0078.wav|"The convicted delinquent has his rights," said Mr. Buxton authoritatively. +DUMMY/LJ003-0202.wav|These capital convicts, says Mr. Bennet, quote, lessened the ennui and despair of their situation by unbecoming merriment +DUMMY/LJ026-0075.wav|are dissolved by the sap in the leaves and elsewhere and thus may pass to every portion of the plant. +DUMMY/LJ033-0192.wav|When Paul M. Stombaugh of the FBI Laboratory examined the paper bag, +DUMMY/LJ045-0072.wav|There can be little doubt that some provocation existed. +DUMMY/LJ016-0167.wav|Probably dissolution came as easily and almost without pain. +DUMMY/LJ003-0179.wav|which will deal with Mrs. Fry's philanthropic exertions at this period in this particular part of the prison. +DUMMY/LJ022-0056.wav|Our social security legislation is an attempt to answer the first of these questions; our Works Relief program, the second. +DUMMY/LJ038-0018.wav|Shortly thereafter, Johnny Brewer, who had come from the nearby shoestore, asked Mrs. Postal whether the fellow that had ducked in had bought a ticket. +DUMMY/LJ022-0097.wav|It will also have the responsibility of determining work payments in various localities, +DUMMY/LJ047-0050.wav|Oswald again agreed to advise the FBI if he were approached under suspicious circumstances; however, he deprecated the possibility of this happening, +DUMMY/LJ047-0156.wav|She said that Lee Oswald was living somewhere in Dallas. She didn't know where. She said it was in the Oak Cliff area but she didn't have his address. +DUMMY/LJ040-0219.wav|It would be incorrect, however, to believe that those aspects of Lee's personality which were observed in New York +DUMMY/LJ028-0280.wav|Calling to mind then the words of the Babylonian at the beginning of the siege: +DUMMY/LJ043-0160.wav|Question: Did you ask him why he had not destroyed the book before he actually went to shoot General Walker? +DUMMY/LJ008-0274.wav|The result in the latter case was left in the first place to the king in council, +DUMMY/LJ038-0027.wav|Detective Paul L. Bentley rushed to the balcony and told the projectionist to turn up the house lights. +DUMMY/LJ048-0007.wav|Director Hoover emphasized that the first indication of Oswald's capacity for violence +DUMMY/LJ014-0164.wav|as the wickedness and levity of the immense crowd collected at the execution this morning could be imagined by no man, +DUMMY/LJ028-0429.wav|The excavations have shown that Babylon, as the ancients told us, was nearly square. +DUMMY/LJ044-0127.wav|In reply Arnold Johnson advised Oswald that, while as an American citizen he had a right to participate in such organizations as he wished, quote, +DUMMY/LJ050-0196.wav|Since the assassination of President Kennedy, the Secret Service has been experimenting with new techniques in the inspection of buildings along a motorcade route. +DUMMY/LJ012-0101.wav|They had had serious work to get at the diamonds. It was necessary to force one heavy door from its hinges, and to cut through the thick panels of another. +DUMMY/LJ012-0073.wav|At his own request he was reconveyed to Hobart Town, where his son had been carrying on the business. +DUMMY/LJ043-0071.wav|His performance for that company was satisfactory. +DUMMY/LJ050-0182.wav|Liaison With Local Law Enforcement Agencies +DUMMY/LJ006-0024.wav|Several European states had dispatched emissaries to examine and report upon them. +DUMMY/LJ010-0016.wav|thefts of bullion, bank-notes, specie, and gold-dust, planned with consummate ingenuity, +DUMMY/LJ045-0161.wav|He was upset over the fact that I would not answer him. +DUMMY/LJ014-0027.wav|A woman whom he called on the same evening declared he had worn a mackintosh, his coat was much torn, there was a stain of blood on his shirt-cuff, +DUMMY/LJ040-0160.wav|and deprivation, lack of affection, absence of family life and rejection by a self involved and conflicted mother. +DUMMY/LJ035-0123.wav|None of these three men saw Dougherty, probably because of the anxiety of the moment and because of the books which may have blocked the view. +DUMMY/LJ009-0164.wav|sheep or horse stealing, and stealing in a dwelling-house. +DUMMY/LJ023-0101.wav|if the legislative power is not left free to choose the methods of solving the problems of poverty, +DUMMY/LJ018-0339.wav|The judge, Sir Henry Hawkins, in passing sentence +DUMMY/LJ018-0230.wav|After reading it, and finding this unfavorable to himself, he resolved to carry out his deliberate plan, +DUMMY/LJ008-0022.wav|On the eighteenth December, eighteen twelve, +DUMMY/LJ032-0198.wav|During the period from March two, nineteen sixty-three, to April twenty-four, nineteen sixty-three, +DUMMY/LJ029-0006.wav|Beginning with the advance plans and Secret Service preparations for the trip, +DUMMY/LJ036-0213.wav|The man's general description was similar to the one broadcast over the police radio. +DUMMY/LJ027-0144.wav|Now, it must be obvious +DUMMY/LJ002-0047.wav|The day room was fitted with benches and settles after the manner of the tap in a public-house. +DUMMY/LJ008-0084.wav|The flames presently burned the halter, the body fell a few inches, and hung then by the iron chain. +DUMMY/LJ048-0126.wav|Special Agent in Charge Sorrels testified that the advance preparations followed on this occasion were, quote, pretty much the same, end quote, +DUMMY/LJ046-0060.wav|and their foreign journeys have come to be accepted as normal rather than extraordinary. +DUMMY/LJ018-0058.wav|It is generally supposed that he committed the murder under a sudden access of covetousness and greed. +DUMMY/LJ008-0034.wav|the scaffold hung with black; and the inhabitants of the neighborhood, having petitioned the sheriffs to remove the scene of execution to the old place, +DUMMY/LJ019-0277.wav|the prisoners should be made to dispense with the use of a mattress, and should sleep on planks. +DUMMY/LJ047-0086.wav|as it might have under existing procedures, and did not know of his application. +DUMMY/LJ018-0114.wav|the others to twenty and ten years. +DUMMY/LJ006-0248.wav|Matters were at times still worse, and the rioting went on to such dangerous lengths as to endanger the safety of the building. +DUMMY/LJ010-0151.wav|Again, a soldier, by name Hatfield, who had been wounded in the head, and discharged from the army for unsoundness of mind, +DUMMY/LJ031-0081.wav|President Kennedy could have survived the neck injury, but the head wound was fatal. +DUMMY/LJ042-0186.wav|which will bring about the final destruction of the capitalist system, and indicated that +DUMMY/LJ021-0045.wav|They saw that without changes in the policies and methods of investment +DUMMY/LJ008-0086.wav|A great concourse of people attended on this melancholy occasion. +DUMMY/LJ022-0176.wav|Finally, the reestablishment of public confidence in the banks of the nation +DUMMY/LJ049-0015.wav|but finds that the Secret Service agents did all that could have been done to take protective measures. +DUMMY/LJ045-0135.wav|but did not issue any such warning or suggest that Marina Oswald defect from the Soviet Union and remain in the United States under FBI protection, +DUMMY/LJ002-0213.wav|The Fleet, which stood in Farringdon Street, +DUMMY/LJ015-0066.wav|His proclivities were theatrical, and he was the author of several plays, +DUMMY/LJ017-0142.wav|Smethurst's escape may have influenced the jury in the Poplar poisoning case, +DUMMY/LJ009-0219.wav|A clause was inserted to the effect that +DUMMY/LJ009-0210.wav|On the left side of the head the fatal mall, +DUMMY/LJ012-0098.wav|These were deposited in an iron chest of great strength on the second floor. +DUMMY/LJ024-0131.wav|The workers were not fooled by that propaganda then. The people of America will not be fooled by such propaganda now. +DUMMY/LJ010-0196.wav|These antics may have been assumed to bear out the plea of insanity set up in his defense, +DUMMY/LJ039-0145.wav|According to George De Mohrenschildt, Oswald said that he went target shooting with that rifle. +DUMMY/LJ009-0290.wav|He was at first engaged as assistant to the executioner Tom Cheshire, but in due course rose to be chief. +DUMMY/LJ016-0081.wav|On another occasion Mr. Cope the governor came in and missed a man. +DUMMY/LJ028-0260.wav|The following story from Herodotus tells the results: +DUMMY/LJ026-0008.wav|and the beginner in biological study is surprised to find them described in textbooks of both botany and zoology. +DUMMY/LJ002-0184.wav|the latter two being also a prison for felons and vagrants arrested within certain limits. +DUMMY/LJ019-0284.wav|According to this committee of eighteen sixty-three, beds in the smaller and most carelessly conducted prisons formed a large element in the life of a prisoner. +DUMMY/LJ046-0239.wav|notwithstanding the fact that Ambassador Stevenson had been abused by pickets in Dallas less than a month before. +DUMMY/LJ044-0086.wav|executive director of the Information Council of the Americas, who also appeared on the program. +DUMMY/LJ007-0152.wav|Having thus passed his time, he returns a greater adept in crime, with a wider acquaintance among criminals, and, what perhaps is even more injurious to him, +DUMMY/LJ027-0078.wav|For example: the wings of a bat (a mammal), a bird and a fossil flying reptile all show the same bones adaptively modified; +DUMMY/LJ014-0232.wav|the French refugee, was a murderer of the same description, who dispatched his victim with a loaded cane, after which, to secure his escape, +DUMMY/LJ019-0101.wav|In eighteen fifty Sir George Grey brought forward a new motion to the same effect, +DUMMY/LJ014-0136.wav|the culprit was really her husband, who killed O'Connor out of jealousy and revengeful feelings. +DUMMY/LJ044-0194.wav|Oswald's attempt to go to Cuba +DUMMY/LJ033-0002.wav|Chapter four. The Assassin: Part two. +DUMMY/LJ032-0079.wav|Also in his wallet at that time +DUMMY/LJ044-0023.wav|Oswald had attempted to infiltrate an anti-Castro organization with which he was associated. +DUMMY/LJ021-0199.wav|In our efforts for recovery +DUMMY/LJ006-0232.wav|More often they occurred at night, and were continued to the bitter end. +DUMMY/LJ035-0050.wav|With his revolver drawn, Baker opened the vestibule door and ran into the vestibule. +DUMMY/LJ016-0234.wav|he was not unskilful, but he proceeded entirely by rule of thumb, leaving the result very much to chance and the strength of the rope. +DUMMY/LJ042-0100.wav|Even though he received more money and better living quarters than other Russians doing similar work, +DUMMY/LJ019-0200.wav|It would be far better to begin at the beginning, +DUMMY/LJ010-0294.wav|frauds indicating great boldness, extensive design, and ingenuity became more prevalent. +DUMMY/LJ016-0242.wav|The last execution at which he acted was that of Godwin, on the twenty-fifth May, eighteen seventy-four. +DUMMY/LJ027-0105.wav|a number of organs which never could have been of use to any kind of creature save a terrestrial quadruped. +DUMMY/LJ047-0141.wav|in early October of nineteen sixty-three. +DUMMY/LJ050-0103.wav|were all men who acted alone in their criminal acts against our leaders. None had a serious record of prior violence. +DUMMY/LJ002-0189.wav|No fixed rates or rules governed the hiring out of rooms or parts of a room, and all sorts of imposition was practiced. +DUMMY/LJ024-0117.wav|you will find that many of those who pretend to support you will sabotage any constructive amendment which is proposed. +DUMMY/LJ016-0070.wav|from a widow woman, who gave him "bub and grub," or food and one-and-sixpence, for every nine days' work. +DUMMY/LJ005-0231.wav|The city justices had not fulfilled this obligation. +DUMMY/LJ035-0072.wav|On the first test, the elapsed time between the simulated first shot and Baker's arrival on the second-floor stair landing +DUMMY/LJ012-0091.wav|The spoil was fairly divided; part of the notes were disposed of to a traveling "receiver," who passed over to the Continent and there cashed them easily. +DUMMY/LJ028-0257.wav|and in five twenty-one Nebuchadnezzar the third, a native Babylonian, was placed on the throne. +DUMMY/LJ044-0012.wav|he had printed a handbill headed in large letters, quote, +DUMMY/LJ043-0142.wav|Oswald posed for two pictures +DUMMY/LJ028-0038.wav|Its history for the next fifteen hundred years or more is obscure. +DUMMY/LJ018-0145.wav|Suspicion appears to have rested on Brown, who had left Laverstock, +DUMMY/LJ018-0254.wav|He declined to plead, but a plea of "Not Guilty" was recorded. +DUMMY/LJ034-0013.wav|In front of this small carton and resting partially on the windowsill was another small "Rolling Readers" carton. +DUMMY/LJ034-0193.wav|Fischer and Edwards did not see the man clearly enough or long enough to identify him. +DUMMY/LJ019-0115.wav|In his proposal he dwelt much upon the humanizing effects of healthful open-air toil, +DUMMY/LJ048-0008.wav|was his attempt on General Walker's life, which did not become known to the FBI until after the assassination. +DUMMY/LJ031-0218.wav|The second head wound measured approximately five inches (thirteen centimeters) in its greatest diameter, +DUMMY/LJ015-0298.wav|But while Hardwicke was in communication with Saward, the bank was in communication with London +DUMMY/LJ005-0299.wav|The whole question was again dealt with in Lord John Russell's bill for the reform of the municipal corporations, and with a more liberal election of town councilors, +DUMMY/LJ003-0281.wav|and the danger of losing material, raw or manufactured. +DUMMY/LJ002-0272.wav|on the opening of Whitecross prison for debtors in eighteen fifteen. +DUMMY/LJ033-0102.wav|When Oswald entered the rear door of the Depository Building, he was about fifty feet ahead of Frazier. +DUMMY/LJ031-0125.wav|Vice President Johnson at Parkland +DUMMY/LJ045-0084.wav|said that, quote, She was annoying him all the time +DUMMY/LJ019-0263.wav|and sought to make profitable returns from prisoners' labor the test of prison efficiency. In this view the committee could not coincide, +DUMMY/LJ028-0380.wav|The prophecy of the Hebrew Isaiah was fulfilled: +DUMMY/LJ031-0018.wav|When Parkland Hospital received the notification, the staff in the emergency area was alerted and trauma rooms one and two were prepared. +DUMMY/LJ035-0015.wav|and I had a feeling that it came from the building, either right in front of me [the Depository Building] or of the one across to the right of it, end quote, +DUMMY/LJ001-0105.wav|America has produced a good many showy books, the typography, paper, and illustrations of which are, however, all wrong, +DUMMY/LJ014-0261.wav|He rode a priceless hack in Rotten Row, or drove down to Richmond in a mail phaeton and pair. +DUMMY/LJ008-0051.wav|says, "The scaffold on which these miserable people suffered is a temporary machine which was drawn out of the yard of the sessions house by horses; +DUMMY/LJ005-0149.wav|The nature of the employment varied greatly in severity, especially the tread-wheel labor. +DUMMY/LJ011-0196.wav|Mr. Turner at once set off for London, where he sought the assistance of the police, +DUMMY/LJ003-0037.wav|A site was purchased between Red Lion and White Cross streets, and a new prison planned, +DUMMY/LJ002-0172.wav|For this he was summoned before a magistrate, and sentenced as already stated. +DUMMY/LJ032-0040.wav|were written the words "A. Hidell, P.O. Box two nine one five Dallas, Texas." +DUMMY/LJ002-0203.wav|The half or wholly starved prisoners fished for alms or food at the gratings. +DUMMY/LJ016-0313.wav|On the other hand, Lords Cranworth and Wensleydale recommended private executions; so did Mr. Spencer Walpole, M.P. +DUMMY/LJ033-0062.wav|She did not unroll the blanket. She saw that it was in its usual position and it appeared to her to have something inside. +DUMMY/LJ043-0032.wav|After about a two-week separation, Marina Oswald returned to her husband. +DUMMY/LJ016-0106.wav|with a civil note to the governor, saying he had no further use for them. All three fugitives were recaptured, +DUMMY/LJ013-0051.wav|was a promissory note for two thousand eight hundred seventy-five pounds, produced by Dr. Bailey, and purporting to be signed by Smith. +DUMMY/LJ010-0160.wav|None of the foregoing attempts were, however, so dastardly or determined as that made by Oxford upon our present gracious Queen +DUMMY/LJ002-0294.wav|a further fee was, however, demanded from the new-comer before he was made free of either the master's or the cabin side. +DUMMY/LJ028-0389.wav|St. Jerome said: +DUMMY/LJ010-0201.wav|"Young England" was a secret society, with no aim or object. +DUMMY/LJ004-0051.wav|When in Belgium he had examined with great satisfaction the admirable management of the great "Maison de Force" at Ghent, +DUMMY/LJ014-0007.wav|He entered into conversation with the policemen, and learnt, as it seemed for the first time, what had happened. +DUMMY/LJ008-0107.wav|in his canonicals, and with his head as stiffly erect as a sheriff's coachman. +DUMMY/LJ038-0201.wav|James C. Cadigan, FBI handwriting expert, testified that this note was written by Lee Harvey Oswald. +DUMMY/LJ012-0137.wav|father of the clerk to the steam-ship company, and these facts were deemed sufficient to justify the arrest of all three. +DUMMY/LJ030-0081.wav|They were instructed to watch particularly for thrown objects, sudden actions in the crowd, and any movements toward the Presidential car. +DUMMY/LJ019-0231.wav|and by the following year forty-seven new cells had been built on the most approved plan. +DUMMY/LJ005-0208.wav|the only aperture through which air could be admitted being an iron grating level with the street, +DUMMY/LJ037-0124.wav|The Commission is satisfied that the lineups were conducted fairly. +DUMMY/LJ002-0290.wav|Neild found the prisoners in the Borough Compter ragged, starving, and dirty. +DUMMY/LJ021-0127.wav|Machinery set up by the federal government has provided some new methods of adjustment. +DUMMY/LJ019-0323.wav|The law made it imperative that every prison should have a prison chapel, and that daily and Sunday services should be held. +DUMMY/LJ007-0142.wav|But at Newgate the convicted prisoner, instead of privation and hard fare, +DUMMY/LJ047-0238.wav|Hosty parked his car in the basement of police headquarters and there met an acquaintance, Lt. Jack Revill of the Dallas police force. +DUMMY/LJ007-0055.wav|The more peaceably disposed found some occupation in making Newgate tokens, +DUMMY/LJ015-0168.wav|but the chairman of the Great Northern could not realize that a clerk of the company could honestly be in the possession of unlimited wealth. +DUMMY/LJ003-0318.wav|No prisoners should in future be ironed, except in cases of misconduct, +DUMMY/LJ036-0037.wav|He picked Oswald from the lineup as the man who had boarded the bus at the, quote, lower end of town on Elm around Houston, end quote, +DUMMY/LJ019-0249.wav|The varieties in construction were still very marked. +DUMMY/LJ019-0114.wav|but Mr. Pearson overlooked some points in which a more practical mind would have foreseen difficulty, and perhaps forecasted failure. +DUMMY/LJ043-0045.wav|He also said that he and his family got along well with Marina Oswald and enjoyed showing her American things. +DUMMY/LJ012-0263.wav|The police wished to refer to Greenacre, but as he was not forthcoming, +DUMMY/LJ034-0104.wav|slender, weighing about one sixty-five pounds, about five foot ten inches tall, and in his early thirties. +DUMMY/LJ033-0114.wav|When Frazier appeared before the Commission and was asked to demonstrate how Oswald carried the package, he said, quote, Like I said, +DUMMY/LJ002-0178.wav|affidavit of service, and many more, amounting in all to twenty-seven, +DUMMY/LJ039-0018.wav|He also said that he would use the pistol if the opportunity arose. +DUMMY/LJ029-0196.wav|and that he would visit San Antonio, Houston, Fort Worth, Dallas, and Austin. +DUMMY/LJ031-0069.wav|Dr. Clark noted some electrical activity on the cardiotachyscope attached to monitor the President's heart responses. +DUMMY/LJ040-0197.wav|He told Mrs. Siegel that he would run away if sent to a boarding school. +DUMMY/LJ035-0137.wav|About this time Shelley saw Truly and Patrolman Baker go into the building +DUMMY/LJ005-0023.wav|He took exception to various of the proposals of the Society. He thought they leant too much to a system of indulgence and education in jails. +DUMMY/LJ001-0116.wav|still clings to a foolish, because misunderstood conventionality, deduced from what was once ornament, and is by no means useful; +DUMMY/LJ006-0295.wav|The governor was also personally responsible for gross contravention of this rule of separation, +DUMMY/LJ043-0029.wav|De Mohrenschildt admitted that, quote, +DUMMY/LJ043-0147.wav|He had left a note for his wife telling her what to do in case he were apprehended, as well as his notebook and the pictures of himself holding the rifle. +DUMMY/LJ048-0134.wav|would be a desirable innovation. +DUMMY/LJ038-0283.wav|without going so far as to say to the exclusion of all other guns. This I could not do, end quote. +DUMMY/LJ044-0191.wav|Now there appeared to be no chance to get to Cuba, where he had thought he might find his communist ideal. The U.S. Government would not permit travel there +DUMMY/LJ004-0203.wav|The new jail had been in a great measure constructed by the prisoners themselves. +DUMMY/LJ031-0157.wav|Meanwhile, the Secret Service agents in charge of security at Love Field started to make arrangements for departure. +DUMMY/LJ009-0147.wav|and to have dwelt with more emphasis than good taste upon the nature of the crime, and the necessity for repentance. +DUMMY/LJ028-0149.wav|using throughout for their cement hot bitumen, and interposing a layer of wattled reeds at every thirtieth course of the bricks. +DUMMY/LJ032-0245.wav|Moreover, it was the only rifle owned by her husband following his return from the Soviet Union in June nineteen sixty-two. +DUMMY/LJ003-0254.wav|whereas irons cannot well be exchanged without loss of time and attracting much attention. +DUMMY/LJ016-0338.wav|inside the jail was Colonel Frazer, the chief commissioner of the city police, and at no great distance, although in the background, +DUMMY/LJ006-0212.wav|brought in beer for sale from a neighboring public-house, and visited all the wards with no other escort than the prisoner gatesman. +DUMMY/LJ038-0146.wav|was a lunch sack which he kept on his lap during the ride from Irving to Dallas. +DUMMY/LJ048-0176.wav|Even if it were possible with a vastly larger force of security officers to do so, many observers have felt +DUMMY/LJ019-0159.wav|were constructed on the landings, ensconced in which warders spent the night, on duty, and alert to watch the sleepers below, +DUMMY/LJ031-0012.wav|The base station replied, quote, They have been notified, end quote, +DUMMY/LJ034-0091.wav|Howard L. Brennan was an eyewitness to the shooting. As indicated previously the Commission considered his testimony as probative +DUMMY/LJ010-0123.wav|an ample pension, so long as he did not return to England. +DUMMY/LJ004-0211.wav|The moral welfare of the inmates was as closely looked after as the physical. +DUMMY/LJ040-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ004-0101.wav|Warm and cold baths, or "commodious bathing tubs," +DUMMY/LJ008-0189.wav|All the avenues and approaches, places even whence nothing whatever could be seen of the scaffold, +DUMMY/LJ019-0192.wav|The court was, however, prepared to consider Lord John Russell's proposal with regard to the cost of rebuilding; +DUMMY/LJ012-0209.wav|and after some haggling they agreed on a price, and in the afternoon the snatchers brought a hamper which contained a body in a sack. +DUMMY/LJ034-0071.wav|At about eleven:forty-five a.m. the floor-laying crew used both elevators to come down from the sixth floor. +DUMMY/LJ002-0107.wav|Quote, On the top of the jail, continues Neild, are a watch-house and a sentry-box, where two or more guards, with dogs and firearms, +DUMMY/LJ018-0140.wav|The two facts taken in conjunction +DUMMY/LJ016-0111.wav|The officers dropped on to him while he was still in bed, but as they came upstairs he jumped up and hid in a cupboard. +DUMMY/LJ050-0256.wav|Protecting the President is a difficult and complex task which requires full use of the best resources of many parts of our Government. +DUMMY/LJ031-0144.wav|When told that Mrs. Kennedy refused to leave without the President's body, the Vice President said that he would not leave Dallas without her. +DUMMY/LJ048-0128.wav|which was Sorrels' first important assignment in connection with Presidential work. +DUMMY/LJ027-0039.wav|both adaptations for pumping blood; +DUMMY/LJ029-0154.wav|and Secret Service agents riding in the motorcade. +DUMMY/LJ002-0275.wav|At one time the Ludgate debtors, accompanied by the keeper, +DUMMY/LJ025-0175.wav|but its nitrogen is in the state of a free gas, in which condition the bean can make no use of it, +DUMMY/LJ007-0074.wav|would, it is probable, have eventually overcome the reluctance of some of the prisoners at least, and would have possessed so much moral dignity +DUMMY/LJ004-0166.wav|At the bottom was a circular space, through which ran a narrow passage, and the sides of which were fitted with barrack bedsteads. +DUMMY/LJ007-0131.wav|Enough has probably been extracted from this most damnatory report to give a complete picture of the disgraceful state in which Newgate still remained in eighteen thirty-five. +DUMMY/LJ025-0163.wav|But nothing has been supplied to the bean save water, carbonic acid, ammonia, potash, lime, +DUMMY/LJ045-0142.wav|He may have felt he could never tell when the FBI was going to appear on the scene or who else was going to find out about his defection +DUMMY/LJ028-0477.wav|To protect the sun-dried bricks of the inner wall from the winter rains +DUMMY/LJ042-0169.wav|And yet, it is immature to take the sort of attitude which says, quote, a curse on both your houses! End quote. +DUMMY/LJ014-0102.wav|The stones were at once taken up; +DUMMY/LJ043-0132.wav|He subsequently found a job with the Texas School Book Depository for which he performed his duties satisfactorily. +DUMMY/LJ032-0073.wav|purchased by mail-order coupon from Seaport-Traders, Inc., a mail-order division of George Rose and Co., Los Angeles. +DUMMY/LJ038-0183.wav|In translation, the note read as follows: one. +DUMMY/LJ012-0208.wav|Reference was made to Mr. Partridge, the demonstrator in anatomy, +DUMMY/LJ022-0162.wav|A business that loses the confidence of its customers and the goodwill of the public cannot long continue to be a good risk for the investor. +DUMMY/LJ041-0076.wav|and put them in positions of prominence, end quote. Oswald manifested this feeling about authority by baiting his officers. +DUMMY/LJ015-0211.wav|Each safe had three sets of double keys, all held by confidential servants of the company. +DUMMY/LJ032-0179.wav|In light of these findings the Commission evaluated the additional testimony of Stombaugh +DUMMY/LJ045-0003.wav|Possible Influence of Anti-Kennedy Sentiment in Dallas +DUMMY/LJ045-0086.wav|The De Mohrenschildts also testified that, quote, right in front, end quote, of Oswald Marina Oswald complained about Oswald's inadequacy as a husband. +DUMMY/LJ018-0258.wav|But while repudiating the charges made against him of systematic extravagance and immorality, +DUMMY/LJ039-0093.wav|at a distance of two hundred sixty-five point three feet was, quote, an easy shot, end quote. +DUMMY/LJ049-0214.wav|duplication, and confusion of authority where more than one agency is operating in the same field; but on the other hand +DUMMY/LJ033-0202.wav|because this blanket is composed of brown and green woolen fibers, brown and green delustered viscose fibers, and brown and green cotton fibers. +DUMMY/LJ018-0079.wav|The unfortunate man survived till he landed, but died in Guy's Hospital. +DUMMY/LJ010-0278.wav|but the place of durance was changed, to meet the existing law, to Millbank Penitentiary. +DUMMY/LJ005-0030.wav|"In prisons, which are really meant to keep the multitude in order, and to be a terror to evil-doers, there must be no sharings of profits, +DUMMY/LJ042-0051.wav|and what I observed plus my Marxist learning brought me here to the Soviet Union. +DUMMY/LJ019-0212.wav|The Home Secretary had laid it down that at least five acres would be indispensable, and such an area it was impossible to obtain within the limits of the city. +DUMMY/LJ002-0054.wav|those who were the oldest offenders, sentenced to transportation, and who had narrowly escaped the penalty of death. +DUMMY/LJ018-0185.wav|A Bow Street officer saw him leaving London in the evening by Camberwell Green, accompanied by two other men. +DUMMY/LJ018-0101.wav|Chandler while in Newgate turned informer, and betrayed the whole conspiracy. +DUMMY/LJ019-0254.wav|they were frequently below the standard size, and were therefore not certified for occupation as was required by law. +DUMMY/LJ002-0092.wav|The two yards were adjoining, that for the common side much the largest. +DUMMY/LJ010-0252.wav|the old traitor's doom, but was spared, and subsequently transported for life. +DUMMY/LJ048-0025.wav|and had told Mrs. Paine that when he got the money he was going to take an apartment, when the baby was old enough, he was going to take an apartment, and the family would live together. +DUMMY/LJ021-0068.wav|to a level of sustained profits within one year from the inauguration of N.R.A. +DUMMY/LJ048-0111.wav|the Secret Service correctly gave particular attention in the advance preparations to those arrangements. +DUMMY/LJ043-0115.wav|He lost his job on July nineteen, nineteen sixty-three, because his work was not satisfactory +DUMMY/LJ015-0033.wav|an impecunious and rotten firm of contractors with vast liabilities and pledged to impossible works abroad. +DUMMY/LJ043-0149.wav|where she had gone, contrary to his instructions, after she became, worried about his absence. +DUMMY/LJ021-0017.wav|Instead of the give and take of free individual contract, +DUMMY/LJ034-0179.wav|and, quote, seemed to be sitting a little forward, end quote, +DUMMY/LJ041-0143.wav|the President almost completely blocked Oswald's view of the Governor prior to the time the first shot struck the President. +DUMMY/LJ015-0089.wav|in receiving guests at home, or could be spared from two rival establishments in other parts of the town. +DUMMY/LJ038-0257.wav|which had come to rest on a stack of paper. +DUMMY/LJ031-0009.wav|The radio log of the Dallas Police Department shows that at twelve:thirty p.m. on November twenty-two +DUMMY/LJ008-0003.wav|which we left at the time of the discontinuance of the long-practiced procession to Tyburn. +DUMMY/LJ035-0110.wav|Both Dougherty and Piper were confused witnesses. They had no exact memory of the events of that afternoon. +DUMMY/LJ004-0195.wav|Personal cleanliness was insisted upon, and all parts of the prison were kept scrupulously clean. +DUMMY/LJ014-0116.wav|Her arrest was planned, and, when the telegram arrived from London, completed. +DUMMY/LJ003-0024.wav|he collected the garnish money, and distributed the weekly grant from the prison charitable fund. +DUMMY/LJ004-0165.wav|the visitors were furnished with candles, and they descended eighteen long steps into a vault. +DUMMY/LJ031-0101.wav|While one medical team tried to revive President Kennedy, a second performed a series of operations on the bullet wounds sustained by Governor Connally. +DUMMY/LJ002-0325.wav|or the sixpenny allowance was claimed for the creditors, which seldom happened, owing to the expense the process entailed. +DUMMY/LJ028-0092.wav|Read the introduction to any of his inscriptions, of which the following is one, and you will call him vain and proud, +DUMMY/LJ003-0109.wav|he found he must adopt the manners of his companions, or that his life would be in danger. +DUMMY/LJ045-0133.wav|The claimed warning was one more of Oswald's fabrications. +DUMMY/LJ038-0148.wav|Actions During and After Shooting +DUMMY/LJ017-0216.wav|there were also a Frenchman, a Norwegian (the carpenter), three Chinamen, a "Sclavonian," and a black on board. +DUMMY/LJ040-0042.wav|He defected to the Soviet Union, shot at General Walker, tried to go to Cuba and even contemplated hijacking an airplane to get there. +DUMMY/LJ009-0091.wav|The young stealer in a dwelling-house no longer has the least pretense to bravery. He grasps the back of the pew, +DUMMY/LJ024-0037.wav|who would disregard the law and would decide specific cases as I wished them to be decided, I make this answer: +DUMMY/LJ033-0143.wav|The presence of the bag in this corner is cogent evidence that it was used as the container for the rifle. +DUMMY/LJ036-0057.wav|She heard a passing motorist tell the driver that the President had been shot. +DUMMY/LJ034-0158.wav|Although the record indicates that Brennan was an accurate observer, he declined to make a positive identification of Oswald when he first saw him in the police lineup. +DUMMY/LJ002-0072.wav|The inmates were privileged, either by antecedents or the fortunate possession of sufficient funds to pay the charges of the place. +DUMMY/LJ037-0106.wav|Both ran to the sidewalk on the east side of Patton at a point about a half a block south of tenth. +DUMMY/LJ003-0061.wav|No beds were issued, only two rugs per prisoner. +DUMMY/LJ047-0200.wav|as to whether Oswald fell within the category of, quote, threats against the President, end quote, which should be referred to the Service. +DUMMY/LJ005-0260.wav|a theory which did not become a practical fact for forty more years. +DUMMY/LJ007-0058.wav|The chaplain, who might have been expected to make these men his peculiar care, and who at one time had visited them frequently, often several times a week, +DUMMY/LJ014-0282.wav|There were two counts in the indictment: one for stealing a cheque value fourteen hundred pounds, the second for stealing a bit of paper value one penny. +DUMMY/LJ046-0018.wav|to which the events of last November called attention. +DUMMY/LJ009-0051.wav|His look at the sheriffs and the ordinary is full of scorn and defiance. +DUMMY/LJ046-0211.wav|For example, at the time of the assassination, the FBI Handbook, which is in the possession of every Bureau special agent, provided, quote, +DUMMY/LJ048-0280.wav|It might seem harsh to circumscribe their opportunities for relaxation. +DUMMY/LJ022-0183.wav|that the government should assume the responsibility of providing a means by which the credit of the nation might be controlled, +DUMMY/LJ010-0213.wav|all of which declared their approval of the commander-in-chief. +DUMMY/LJ015-0283.wav|The bearer of the cheque was always innocent and ignorant of the fraudulent nature of the document he presented. +DUMMY/LJ002-0011.wav|For two more years these high figures were steadily maintained, and in eighteen oh three the total rose to seven hundred ten. +DUMMY/LJ006-0014.wav|They should be persons, he thought, selected from the highest class; the duty was most important, +DUMMY/LJ040-0073.wav|Marguerite Oswald withdrew Lee from the orphans' home and took him with her to Dallas when he was a little over four years old. +DUMMY/LJ046-0089.wav|His travel would be in secret; his public appearances would be behind bulletproof glass. A more practical approach necessitates compromise. +DUMMY/LJ045-0213.wav|The conversation on Monday, November eighteen, nineteen sixty-three, +DUMMY/LJ015-0296.wav|Hardwicke, or "Ralph," appealed to Saward in his difficulty +DUMMY/LJ050-0262.wav|Even if the manpower and technological resources of the Secret Service are adequately augmented, +DUMMY/LJ023-0061.wav|that they were all the powers needed to meet each and every problem which then had a national character +DUMMY/LJ012-0149.wav|"Money Moses" and Mrs. Abrahams imposed upon Moss as to the price paid by Solomons; +DUMMY/LJ027-0115.wav|it will be suffered to dwindle away in successive generations, under the influence of certain natural causes. +DUMMY/LJ038-0269.wav|Frazier testified, however, that he found no microscopic characteristics or other evidence +DUMMY/LJ019-0100.wav|but the session was far advanced, and the matter was relegated to the following year. +DUMMY/LJ045-0238.wav|and in the face of the overwhelming evidence against him +DUMMY/LJ034-0133.wav|Brennan could have seen enough of the body of a kneeling or squatting person to estimate his height. +DUMMY/LJ047-0241.wav|that the FBI had known about Oswald and, in particular, of his presence in Dallas and his employment at the Texas School Book Depository Building. +DUMMY/LJ022-0190.wav|These measures I have mentioned are, in large part, the program which under my constitutional duty I have recommended to the Congress. +DUMMY/LJ028-0141.wav|so that the entire circuit is four hundred and eighty furlongs. +DUMMY/LJ047-0246.wav|and District Attorney Henry M. Wade both testified that they saw it later that day. +DUMMY/LJ039-0135.wav|using a twenty-two caliber bolt-action rifle belonging either to Robert or Robert's in-laws. +DUMMY/LJ040-0020.wav|It certainly could not be made by this Commission which, as has been pointed out above, +DUMMY/LJ012-0032.wav|Solomons also established a system of provincial agency, by which stolen goods were passed on from London to the seaports, and so abroad. +DUMMY/LJ036-0034.wav|It was the intersection near Lamar Street, it was near Poydras and Lamar Street. +DUMMY/LJ008-0214.wav|the sounds of hammer and saw intermingled with the broad jeers and coarse jests of the rapidly increasing mob. +DUMMY/LJ015-0246.wav|The train was now approaching Folkestone, and Agar and Pierce hid themselves in a dark part of the van. +DUMMY/LJ011-0199.wav|Meanwhile, a second letter turned up from Wakefield at Calais, in which he assured Mrs. Turner that Miss Turner was fondly attached to him, +DUMMY/LJ009-0110.wav|This exhibition lasts for some minutes, and then the congregation disperses, +DUMMY/LJ020-0097.wav|You will not relish it, but take it all the same to appease the empty, discontented stomach. +DUMMY/LJ048-0286.wav|Nor is this goal served when agents remain out until early morning hours, and lose the opportunity to get a reasonable amount of sleep. +DUMMY/LJ044-0126.wav|they could say the organization of which I am a member, is Russian controlled, etc, end quote. +DUMMY/LJ032-0114.wav|Additional evidence of ownership was provided in the form of palmprint identification which indicated that Oswald had possession of the rifle he had purchased. +DUMMY/LJ013-0080.wav|the evidence to establish his (Barber's) connivance in the frauds was too doubtful for us to continue his exclusion any longer. +DUMMY/LJ013-0093.wav|where the cash was transferred from the carpet-bag to a portmanteau. +DUMMY/LJ012-0287.wav|On the other hand, it was adduced in evidence that Mrs. Gale had been at his lodgings the very day after the murder, +DUMMY/LJ033-0124.wav|And she folded the bag to length of about twenty-eight and a half inches. +DUMMY/LJ012-0254.wav|where a workman found a bundle containing two human legs, in a drain. +DUMMY/LJ041-0105.wav|led to an incident in which he spilled a drink on one of his sergeants and abusively challenged him to fight. +DUMMY/LJ008-0185.wav|Beware of entering the crowd! Remember thirty poor persons were pressed to death by the crowd when Haggerty and Holloway were executed! +DUMMY/LJ027-0145.wav|that such a recapitulation in the life history of an existing animal of developmental changes successively distinctive of sundry allied, +DUMMY/LJ040-0236.wav|and when his mother failed to cooperate in any way with school authorities, +DUMMY/LJ049-0193.wav|Consequently the suggestion has been made, on the one hand, that all preventive investigative functions relating to the security of the President +DUMMY/LJ037-0268.wav|(three) the revolver in Oswald's possession at the time of his arrest was purchased by and belonged to Oswald, and (four) +DUMMY/LJ047-0212.wav|would have seemed somewhat serious to us, even though I must admit, that none of these in themselves would be +DUMMY/LJ035-0025.wav|I am a building manager. Follow me, officer, and I will show you, end quote. +DUMMY/LJ003-0176.wav|Mr. Cobbett was also a lodger of Mr. Newman's; and so were any members of the aristocracy, +DUMMY/LJ046-0127.wav|or from the occasional investigations initiated by the Secret Service, +DUMMY/LJ012-0151.wav|Solomons cheated the whole lot by retaining half the gold in his possession, and only giving an I. O. U. for it, +DUMMY/LJ035-0147.wav|Mrs. Reid had watched the parade from the sidewalk in front of the building with Truly and Mr. O. V. Campbell, vice president of the Depository. +DUMMY/LJ017-0248.wav|while the other two were left to go down in the ship, and were seen clinging to the tops until the waters closed over them. +DUMMY/LJ036-0061.wav|A Beckley bus which also served the Oak Cliff area, +DUMMY/LJ048-0004.wav|After summarizing the Bureau's investigative interest in Oswald prior to the assassination, J. Edgar Hoover concluded that, quote, +DUMMY/LJ010-0170.wav|His acquaintances often asked his object in this, but he kept his own counsel till the tenth June. +DUMMY/LJ010-0019.wav|As time passed, +DUMMY/LJ023-0006.wav|Tonight, sitting at my desk in the White House, I make my first radio report to the people in my second term of office. +DUMMY/LJ004-0123.wav|sixty-eight had only two divisions, and so on. +DUMMY/LJ025-0073.wav|the presence in animals of an alimentary cavity, or internal pocket, in which they can carry about their nutriment +DUMMY/LJ040-0225.wav|Carro testified that, quote, There was nothing that would lead me to believe when I saw him at the age of twelve that them would be seeds of destruction for somebody. +DUMMY/LJ003-0267.wav|kept open house, so to speak, and entertained daily within the walls a select party of the most noted thieves in London. +DUMMY/LJ019-0378.wav|As the almost inevitable consequence, while the more glaring defects in prison management disappeared, +DUMMY/LJ038-0078.wav|Oswald was questioned intermittently for approximately twelve hours between two:thirty p.m., on November twenty-two, and eleven a.m., +DUMMY/LJ047-0203.wav|have made Oswald a subject of concern to the Secret Service. +DUMMY/LJ005-0120.wav|Half the jails had no code of rules properly prepared and sanctioned by the judges, according to law. +DUMMY/LJ050-0268.wav|The Service should negotiate a memorandum of understanding with each agency that has been assisting it and from which it can expect to need help in the future. +DUMMY/LJ014-0082.wav|After this Mrs. Manning changed her dress and went off in a cab to O'Connor's lodgings, +DUMMY/LJ048-0158.wav|Check of buildings along route of motorcade. +DUMMY/LJ040-0046.wav|which one associate described as, quote, irrevocable, end quote, but also as, quote, theoretical, end quote, +DUMMY/LJ034-0202.wav|In his Commission testimony +DUMMY/LJ008-0237.wav|The human hand that placed it there was only seen for a moment, +DUMMY/LJ005-0172.wav|When that time arrives +DUMMY/LJ034-0066.wav|that Oswald was at the window from which the shots were fired, although the prints do not establish the exact time he was there. +DUMMY/LJ022-0114.wav|Our responsibility is to all of the people in this country. +DUMMY/LJ023-0094.wav|and placed "an unwarranted limitation upon the commerce clause." +DUMMY/LJ025-0093.wav|and that the latter is, chemically speaking, just as complicated as the former. +DUMMY/LJ003-0173.wav|He continued the ancient practice of letting out a portion of his own house, and by a poetical fiction treated it as an annex of the state side. +DUMMY/LJ012-0060.wav|to let the coach change and pass Petticoat Lane en route to the jail, where the suffering woman might be handed over to her friends. +DUMMY/LJ011-0014.wav|A newspaper report of the time describes Fauntleroy as a well-made man of middle stature. +DUMMY/LJ021-0014.wav|but also our processes of civilization. +DUMMY/LJ028-0274.wav|The Babylonians were ever upon the watch, and he found no way of conquering them. +DUMMY/LJ038-0181.wav|On December two, nineteen sixty-three, Mrs. Ruth Paine turned over to the police some of the Oswalds' belongings, +DUMMY/LJ016-0291.wav|sharpers, thieves, gamblers, betting men, the outsiders of the boxing ring, +DUMMY/LJ031-0084.wav|But his condition was hopeless, and the extraordinary efforts of the doctors to save him could not help but to have been unavailing. +DUMMY/LJ019-0230.wav|In eighteen sixty-one a similar work was undertaken to provide separate cellular accommodation for the female inmates of Newgate, +DUMMY/LJ041-0038.wav|heard what we were arguing on communism, and that this boy was loud-mouthed, boisterous, and my father asked him to leave the house and politely put him out of the house, +DUMMY/LJ046-0217.wav|must be referred immediately by the most expeditious means of communication to the nearest office of the U.S. Secret Service. +DUMMY/LJ031-0173.wav|Mrs. Kennedy alternated between watching them and waiting outside. +DUMMY/LJ016-0406.wav|the condemned sermon was preached in the prison chapel, and this was probably only that he might hear the singing of a lady with a magnificent voice +DUMMY/LJ019-0365.wav|and with the concurrence of the other authority, and on payment. A few provisos governed these rather extensive powers. +DUMMY/LJ021-0185.wav|Is it not a fact that relations between capital and labor on the basis of collective bargaining are much further advanced in Great Britain +DUMMY/LJ021-0113.wav|or whether their effect may have been to prevent that volume of production which would make possible lower prices and increased employment. +DUMMY/LJ003-0044.wav|The reforms which were to be attempted in that prison +DUMMY/LJ019-0015.wav|The first stone of Pentonville prison was laid on the tenth April, eighteen forty, by the Marquis of Normanby, +DUMMY/LJ048-0198.wav|and although it was not a violation of the law to carry a placard, that they were not to tolerate any actions such as the Stevenson incident +DUMMY/LJ028-0279.wav|after which he commanded his servants to tell no one what had come to pass, while he himself pondered the matter. +DUMMY/LJ008-0144.wav|in eighteen oh seven, an event long remembered from the fatal and disastrous consequences which followed it. +DUMMY/LJ014-0165.wav|and presented by no heathen land under the sun. +DUMMY/LJ008-0142.wav|the attendance at the execution was certain to be tumultuous, and the conduct of the mob disorderly. +DUMMY/LJ048-0265.wav|after which time a very moderate use of liquor will not be considered a violation. However, all members of the White House Detail +DUMMY/LJ003-0065.wav|All the misdemeanants, whatever their offense, were lodged in this chapel ward. +DUMMY/LJ047-0102.wav|When I began asking him specific details with respect to his activities in the Fair Play for Cuba Committee in New Orleans as to where meetings were held, +DUMMY/LJ043-0023.wav|and helped to move the personal effects of Marina Oswald and the baby. +DUMMY/LJ032-0138.wav|Nor was there any indication that the lift had been performed. +DUMMY/LJ016-0255.wav|The old prejudices, such as that which enlisted Dr. Johnson on the side of the Tyburn procession, still lingered and prevented any change. +DUMMY/LJ047-0036.wav|It revealed his letter of January thirty, nineteen sixty-two, to Secretary of the Navy Connally, +DUMMY/LJ031-0055.wav|Dr. Perry noted the President's back brace as he felt for a femoral pulse, which he did not find. +DUMMY/LJ049-0199.wav|and ready at any instant to hazard great danger to themselves in the performance of their tremendous responsibility. +DUMMY/LJ025-0102.wav|yet the exact researches of the modern chemical investigators of the physiological processes of plants have clearly demonstrated the fallacy +DUMMY/LJ029-0188.wav|Stevenson was jeered, jostled, and spat upon by hostile demonstrators outside the Dallas Memorial Auditorium Theater. +DUMMY/LJ029-0131.wav|The building overlooks Dealey Plaza, an attractively landscaped triangle of three acres. +DUMMY/LJ036-0050.wav|Oswald got on. He looks like a maniac. His sleeve was out here. His shirt was undone. +DUMMY/LJ005-0256.wav|in order to insure uniformity of discipline. It met on the thirty-first March, eighteen thirty-five, and continued its sittings well into July +DUMMY/LJ035-0210.wav|On the basis of these findings the Commission has concluded that Oswald, at the time of the assassination, +DUMMY/LJ016-0206.wav|he was also at liberty to engage himself in the country, where he demanded and was paid ten pounds on each occasion. +DUMMY/LJ012-0056.wav|After a short carouse the prisoner went into Westminster, his case was heard, bail refused, and he was ordered back to Newgate. +DUMMY/LJ042-0213.wav|always profess patriotism toward the land and the people, if not the government; although their ideals movements must surely lead +DUMMY/LJ038-0095.wav|by the Commission, they had probative value in deciding the weight to be given to his denials that he assassinated President Kennedy +DUMMY/LJ045-0250.wav|there emerged a man capable of assassinating President Kennedy. +DUMMY/LJ036-0204.wav|According to Chief Curry, Tippit was free to patrol the central Oak Cliff area. +DUMMY/LJ029-0108.wav|Sorrels and Lawson drove over the selected route with Batchelor and other police officers, verifying that it could be traversed within forty-five minutes. +DUMMY/LJ003-0239.wav|This imposition of fees left prisoners destitute on their discharge, without funds to support them in their first struggle to recommence life, +DUMMY/LJ045-0066.wav|Oswald struck his wife on occasion, +DUMMY/LJ014-0014.wav|It was noticed that he wore a mackintosh. Next day the police, in making a careful search of the scene of the murder, +DUMMY/LJ005-0114.wav|In others the separation between the sexes consisted in a hanging curtain +DUMMY/LJ043-0110.wav|beginning May ten, nineteen sixty-three. +DUMMY/LJ031-0137.wav|At approximately one:twenty p.m., Vice President Johnson was notified by O'Donnell that President Kennedy was dead. +DUMMY/LJ005-0258.wav|embodying recommendations which may be said to have formed the basis of modern prison management. +DUMMY/LJ004-0235.wav|Indeed the best consequences followed from the removal of irons. The prisoners were much better disposed; there were no riots, and fewer disturbances. +DUMMY/LJ018-0381.wav|the wildest and most cut-throat looking of the lot, which proves that he could be grateful for kindness, and was not all bad. +DUMMY/LJ019-0165.wav|The rule became general as regards the prison dress; hitherto clothing had been issued only to such as were destitute or in rags, and all classes of prisoners, +DUMMY/LJ007-0116.wav|A few others, who could not afford a payment of more than half a guinea, were permitted to monopolize a part of the prison infirmary, +DUMMY/LJ050-0253.wav|that the job of protecting the President has not been and cannot be exclusively the responsibility of the Secret Service. +DUMMY/LJ009-0105.wav|The sheriffs cover their faces, and one of their inquisitive friends blows his nose with his glove. +DUMMY/LJ016-0139.wav|The last escape from Newgate was only three years ago, and occurred just before the final closing of the prison. +DUMMY/LJ003-0095.wav|Quote, in this dreadful situation, end quote. who had been rescued from the hulks through the kindness and attention of the Secretary of State. +DUMMY/LJ041-0093.wav|Powers also testified that Oswald was reserved and seemed to be, quote, somewhat the frail, little puppy in the litter, end quote. +DUMMY/LJ005-0125.wav|or to reappropriate, reconstruct, and patch up the existing prisons till they were more in accordance with the growing requirements of the times. +DUMMY/LJ028-0029.wav|At last you see before you a mound rising like a mountain from the level plain. +DUMMY/LJ035-0021.wav|A woman screamed, quote, Oh, they have shot that man, they have shot that man, end quote. +DUMMY/LJ039-0211.wav|Moreover, the defect was one which would have assisted the assassin aiming at a target which was moving away. +DUMMY/LJ005-0137.wav|Four years later the Prison Society reported +DUMMY/LJ020-0065.wav|Two cups of fresh milk slightly warmed. +DUMMY/LJ021-0123.wav|When the businessmen of the country were demanding the right to organize themselves adequately to promote their legitimate interests; +DUMMY/LJ006-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section nine: The first report of the inspector of prisons. +DUMMY/LJ003-0341.wav|by a new jail built on the outskirts, as Holloway eventually was, and permitted itself to be altogether countered +DUMMY/LJ037-0075.wav|However, even in the absence of Mrs. Markham's testimony, there is ample evidence to identify Oswald as the killer of Tippit. +DUMMY/LJ025-0169.wav|The bean has been able to perform this great chemical feat by the help of its green coloring matter, or chlorophyll, +DUMMY/LJ032-0263.wav|About three hours after the assassination, a detective and deputy sheriff saw the blanket-roll, +DUMMY/LJ013-0116.wav|The butler, on the occasion of a death in the family, when the house was in some confusion, arranged with a burglar to come in, +DUMMY/LJ039-0124.wav|End quote. Major Anderson concluded, quote, I would say that as compared to other Marines receiving the same type of training, +DUMMY/LJ043-0118.wav|The correctness of that conclusion is supported by the fact that he does not seem to have been publicly identified with that organization until August nine, +DUMMY/LJ048-0068.wav|as requiring evidence of a plan or conspiracy to injure the President. +DUMMY/LJ032-0087.wav|was a purported international certificate of vaccination signed by "Dr. A. J. Hideel, +DUMMY/LJ016-0344.wav|The ceremony, which was witnessed by only a few officials and representatives of the press, was performed with the utmost decency and decorum. +DUMMY/LJ037-0071.wav|During her testimony Mrs. Markham initially denied that she ever had the above phone conversation. +DUMMY/LJ003-0081.wav|Persons convicted of publishing libels were still immured in the same rooms with transports and felons. +DUMMY/LJ028-0119.wav|but I, his first-born, the beloved of his heart, +DUMMY/LJ012-0259.wav|who had been missing since the previous Christmas Day. +DUMMY/LJ025-0149.wav|that, while locomotive, their movements may have as much appearance of spontaneity as those of the lowest animals, +DUMMY/LJ027-0118.wav|"for, without waiting to dispute the wisdom of making dwarfed and useless structures merely for the whimsical motive assigned, +DUMMY/LJ022-0180.wav|Wise public policy, however, requires not only that banking be safe +DUMMY/LJ005-0015.wav|our prisoners have all that prisoners ought to have, without gentlemen think they ought to be indulged with Turkey carpets. +DUMMY/LJ011-0047.wav|This was on the text, +DUMMY/LJ002-0020.wav|In eighteen thirteen +DUMMY/LJ023-0045.wav|Last Thursday I described the American form of government as a three horse team provided by the Constitution to the American people +DUMMY/LJ006-0209.wav|the donation of a philanthropic gentleman, Captain Brown, but these, particularly the Bibles, bore little appearance of having been used. +DUMMY/LJ030-0247.wav|I was bent over under the weight of Agent Youngblood's body, toward Mrs. Johnson and Senator Yarborough, end quote, +DUMMY/LJ046-0223.wav|to Secret Service locally, end quote. +DUMMY/LJ034-0021.wav|Sebastian F. Latona, supervisor of the Latent Fingerprint Section, +DUMMY/LJ042-0020.wav|indicating that his motivation was at least in part a personal one. On November twenty-six, nineteen fifty-nine, +DUMMY/LJ006-0285.wav|which numbers of well-meaning but inquisitive people were anxious to witness. +DUMMY/LJ001-0134.wav|In good printing the spaces between the words should be as near as possible equal +DUMMY/LJ028-0113.wav|With mortar and bricks he built two moat-walls about the city, +DUMMY/LJ022-0140.wav|I can mention only a few of them tonight, but I do not want my mention of specific measures +DUMMY/LJ030-0077.wav|Each agent carried a thirty-eight-caliber pistol, and a shotgun and automatic rifle were also available. +DUMMY/LJ031-0053.wav|Dr. Carrico inserted a cuffed endotracheal tube past the injury, inflated the cuff, and connected it to a Bennett machine to assist in respiration. +DUMMY/LJ033-0130.wav|and her first remembrance is that it was held in Oswald's right hand, quote, and it almost touched the ground as he carried it, end quote. +DUMMY/LJ018-0388.wav|and the bird's legs were a couple of teeth broken off the prisoner's comb. +DUMMY/LJ038-0054.wav|He added that one officer grabbed the muzzle of a shotgun, drew back, and hit Oswald with the butt end of the gun in the back. +DUMMY/LJ042-0216.wav|they stand up and piously pronounce themselves patriots, displaying their war medals that they gained in conflicts long past between themselves. +DUMMY/LJ038-0175.wav|Until December three, nineteen sixty-three, the Walker shooting remained unsolved. +DUMMY/LJ038-0012.wav|the man in the lobby, quote, looked over his shoulder and turned around and walked up West Jefferson towards the theatre, end quote. +DUMMY/LJ006-0122.wav|and by these payments a wardsman had been known to amass as much as forty pounds. +DUMMY/LJ015-0213.wav|a third with the captain of the Folkestone and Boulogne boat. +DUMMY/LJ031-0138.wav|Special Agent Youngblood learned from Mrs. Johnson the location of her two daughters +DUMMY/LJ022-0201.wav|through the instrumentality of the democratic form of government. +DUMMY/LJ015-0111.wav|then, accompanied by a lady, not Mrs. Robson, he took steamer and started for Copenhagen. +DUMMY/LJ042-0060.wav|Oswald met disappointments there just as he had in the past. At the outset the Soviets told him that he could not remain. +DUMMY/LJ001-0108.wav|it is obvious that legibility is the first thing to be aimed at in the forms of the letters; +DUMMY/LJ019-0016.wav|then Home Secretary, and the prison, which contained five hundred and twenty cells, was occupied on the twenty-first December, eighteen forty-two. +DUMMY/LJ007-0147.wav|communicating his own skill and aptitude in crime, or acquiring the lessons of greater adepts. He has access to newspapers, and of course +DUMMY/LJ002-0324.wav|or the keeper made them a special grant of six pence per diem out of the common stock; +DUMMY/LJ008-0091.wav|The highwayman, whose exploits many in the crowd admired or emulated, was cheered and bidden to die game; +DUMMY/LJ014-0159.wav|Manning's demeanor was more in harmony with his situation, and the full confession he made +DUMMY/LJ037-0100.wav|Later, the FBI showed Smith a picture of Oswald. In the picture the hair was brown. +DUMMY/LJ011-0142.wav|The question was whether the Government would dare to take their lives at the bidding of the House of Lords, +DUMMY/LJ006-0041.wav|Changes introduced under pressure had been only skin deep. +DUMMY/LJ009-0005.wav|which persons under sentence of death obtain from all the officers of the prison. +DUMMY/LJ037-0223.wav|Shortly after Tippit was slain, policemen found a light-colored zipper jacket along the route taken by the killer as he attempted to escape. +DUMMY/LJ006-0239.wav|During the tumult a prisoner, who was one of the worst of the rioters, was bruised about the head and body. +DUMMY/LJ010-0205.wav|and to assume any disguise, if required to go into the country on the business of the association. +DUMMY/LJ037-0167.wav|Quote: The only thing I can testify is they could have on the basis of the rifling characteristics -- they could have been, end quote, +DUMMY/LJ011-0269.wav|An appointment was made and kept by Mr. Gee, +DUMMY/LJ019-0057.wav|was the true basis of a sound system of prison discipline. +DUMMY/LJ011-0267.wav|Mr. Gee had invested one thousand two hundred pounds of this, and was seeking how best to place the remaining eight hundred pounds, +DUMMY/LJ050-0212.wav|While these statistics relate to the activities of Secret Service agents stationed in field offices and not the White House detail, +DUMMY/LJ010-0147.wav|It was a form of high treason not unknown in earlier reigns. In seventeen eighty-six a mad woman, Margaret Nicholson, +DUMMY/LJ028-0356.wav|How much truth there may be in this interesting tale of Herodotus, we may never know, +DUMMY/LJ014-0264.wav|Under his auspices several "stars" appeared on the boards of the Marylebone theatre, +DUMMY/LJ008-0188.wav|Every window and roof which could command a view of the horrible performance was occupied. +DUMMY/LJ003-0005.wav|no steps taken to reduce the number of committals, and the governor was obliged to utilize the chapel as a day and night room. +DUMMY/LJ011-0216.wav|The prosecution pressed for a severe penalty. Edward Wakefield pleaded that his trial had already cost him three thousand pounds. +DUMMY/LJ047-0161.wav|Mrs. Paine told Hosty also +DUMMY/LJ014-0294.wav|and in the middle of the night one of the fellow-prisoners who slept in the same ward noticed that he was not in his bed. +DUMMY/LJ002-0256.wav|A committee of collegians was elected to act as the executive, also a secretary or accountant to receive monies and keep books, +DUMMY/LJ031-0104.wav|At approximately twelve:forty-five p.m., Dr. Robert Shaw, +DUMMY/LJ022-0175.wav|in which the benefits of private ownership are retained while the public stake in these important services is protected by the public's government. +DUMMY/LJ048-0147.wav|The Commission believes +DUMMY/LJ048-0275.wav|to an inference that the violation of the regulation had contributed to the tragic events of November twenty-two. +DUMMY/LJ013-0158.wav|One morning in May his lordship was found dead in his bed with his throat cut. +DUMMY/LJ031-0199.wav|President and Mrs. Johnson were flown by helicopter to the White House, from where Mrs. Johnson was driven to her residence under Secret Service escort. +DUMMY/LJ017-0099.wav|Cook recovered this time; it was probably Palmer's intention that he should recover, wishing to encourage the supposition that Cook was in a bad way. +DUMMY/LJ040-0054.wav|It was a factor which contributed to his character and thereby might have influenced his decision to assassinate President Kennedy. +DUMMY/LJ042-0248.wav|that, quote, he hated America and he did not want them to become Americanized, and that his plans were to go back to Russia, end quote. +DUMMY/LJ047-0080.wav|A confidential informant advised the FBI that Oswald was not known to be engaged in Communist Party activities in New Orleans. +DUMMY/LJ021-0153.wav|Closely allied to the N.R.A. +DUMMY/LJ013-0035.wav|While lying in Newgate, awaiting removal to the convict ship, both prisoners made full confessions. +DUMMY/LJ036-0178.wav|She first saw him the day he rented a room at that address on October fourteen, nineteen sixty-three. +DUMMY/LJ014-0243.wav|he with unusual quickness of apprehension discovered and promptly turned to account an inexcusably lax system of management, +DUMMY/LJ043-0062.wav|Oswald's defection, his interest in the Soviet Union, and his activities on behalf of the Fair Play for Cuba Committee +DUMMY/LJ021-0157.wav|I answer that no country, however rich, can afford the waste of its human resources. +DUMMY/LJ018-0346.wav|Henry Wainwright's attempt to get rid of the body was ingenious, but not original, +DUMMY/LJ019-0060.wav|that in districts where it was the rule, evil-doers especially dreaded coming under its irksome conditions. +DUMMY/LJ033-0033.wav|there were two flat lightweight curtain rods belonging to Ruth Paine but they were still there on Friday afternoon after Oswald's arrest. +DUMMY/LJ032-0254.wav|the rifle was, according to the evidence, stored in a green and brown blanket in the Paines' garage among the Oswalds' other possessions. +DUMMY/LJ028-0011.wav|As the taps upon her shoulder are repeated, she stretches out her long neck, and with long strides makes for the eastern horizon; +DUMMY/LJ010-0197.wav|but that there was madness in his family, and that he himself was of unsound mind, could not be well denied. +DUMMY/LJ014-0220.wav|All this time not one of a numerous body of bystanders offered to assist the policeman in his extremity. +DUMMY/LJ020-0105.wav|Weave the little duties in and under and among what seem to be the greater. +DUMMY/LJ049-0137.wav|and the other Federal agencies that assist in safeguarding the President. The Committee should include the Secretary of the Treasury and the Attorney General, +DUMMY/LJ001-0098.wav|and the whole effect is a little too gray, owing to the thinness of the letters. +DUMMY/LJ018-0152.wav|With her assistance on a certain day a couple of bricks were taken out of the wall dividing her front and back parlors; +DUMMY/LJ047-0033.wav|The Bureau made arrangements to be advised by immigration authorities of his return, +DUMMY/LJ028-0455.wav|It seems that the circuit of the outer wall was about eleven miles. +DUMMY/LJ008-0005.wav|The terrible spectacle was as demoralizing to the public, for whose admonition it was intended, +DUMMY/LJ004-0148.wav|No prison dress was allowed, and half the inmates were without shirts or shoes or stockings. +DUMMY/LJ017-0098.wav|whether the ordinary or his own pills will never be known, except as may be inferred from the results, which indicate that he had taken the latter. +DUMMY/LJ050-0016.wav|This special assistant should be required to have sufficient stature and experience in law enforcement, intelligence, or allied fields +DUMMY/LJ048-0090.wav|certain shortcomings and lapses from the high standards which the Commission believes should prevail in the field of Presidential protection are evident. +DUMMY/LJ018-0146.wav|and he was soon approached by the police. Almost directly he was questioned he made a clean breast of the whole affair. +DUMMY/LJ014-0202.wav|One of them went into his cell just at dinner-time; +DUMMY/LJ032-0108.wav|Oswald listed a "Sgt. Robert Hidell" as a reference on one job application and "George Hidell" as a reference on another. +DUMMY/LJ026-0060.wav|through which it slowly passes, being subjected meanwhile to the action of certain chemical substances or reagents, known as digestive ferments. +DUMMY/LJ002-0279.wav|The Giltspur Street Compter received sheriffs' debtors, also felons, vagrants, and night charges. +DUMMY/LJ003-0067.wav|in the then existing state of the law, +DUMMY/LJ005-0083.wav|It was distinctly laid down that male and female prisoners should be confined in separate buildings or parts of the prison, +DUMMY/LJ047-0166.wav|Hosty assured her, through Mrs. Paine as interpreter, that the FBI would not harm or harass her. +DUMMY/LJ038-0023.wav|At one:forty-five p.m., the police radio stated, quote, Have information a suspect just went in the Texas Theatre on West Jefferson, end quote. +DUMMY/LJ042-0155.wav|it appears to be the work of a fairly well organized person. +DUMMY/LJ017-0284.wav|was too much for his nerves, and he so nearly fainted that he had to be seated in a chair. +DUMMY/LJ030-0140.wav|One bullet passed through the President's neck; +DUMMY/LJ015-0041.wav|and the various discount houses would not advance sufficient sums to relieve the necessities of the bank. +DUMMY/LJ010-0253.wav|The enthusiasm of the people at the Queen's escape was uproarious, and her drive next day was one long triumphal progress. +DUMMY/LJ015-0272.wav|and it was to him that Agar, Pierce, and the rest applied when seeking to dispose of their stolen bullion. +DUMMY/LJ038-0292.wav|She testified that her husband had postponed his attempt to kill Walker +DUMMY/LJ010-0122.wav|The Government were probably not proud of their agent, for Edwards, after the conviction had been assured, went abroad to enjoy, it was said, +DUMMY/LJ028-0419.wav|and digging from the wall an inscribed square brick bearing the name of Nebuchadnezzar, he took it to Rome where it may still be seen. +DUMMY/LJ025-0126.wav|comparable to that which was found by Du Bois Reymond to be a concomitant of the activity of ordinary muscle in animals. Again, +DUMMY/LJ010-0182.wav|the Duchess of Kent. +DUMMY/LJ032-0101.wav|testified that she first learned of Oswald's use of the fictitious name "Hidell" in connection with his pro-Castro activities in New Orleans. +DUMMY/LJ046-0067.wav|that he expose himself to the actual basic problems that were disturbing the American people. +DUMMY/LJ009-0263.wav|the executioner forced the struggling criminal from the platform, so that the rope sustained his whole weight. +DUMMY/LJ008-0210.wav|the crowd began to congregate in and about the Old Bailey. +DUMMY/LJ038-0093.wav|Frequently, however, he was confronted with evidence which he could not explain, and he resorted to statements which are known to be lies. +DUMMY/LJ010-0200.wav|"Young England," which Oxford had evolved out of his own inflated self-conceit, and which had never any real corporeal existence. +DUMMY/LJ013-0124.wav|who belonged to five good London clubs -- +DUMMY/LJ048-0216.wav|stated that he scanned the Depository Building, but not sufficiently to be alerted by anything in the windows or on the roof. +DUMMY/LJ032-0076.wav|Handwriting experts from the FBI and the Treasury Department testified that the writing on the mail-order form was that of Lee Harvey Oswald. +DUMMY/LJ013-0056.wav|Subsequently Bailey was arrested on a charge of forgery, and after a long trial found guilty. His sentence was transportation for life. +DUMMY/LJ024-0032.wav|Those opposing this plan have sought to arouse prejudice and fear by crying that I am seeking to "pack" the Supreme Court +DUMMY/LJ042-0077.wav|Quote, I Lee Harvey Oswald do hereby request that my present citizenship in the United States of America, be revoked. +DUMMY/LJ006-0140.wav|Evidence was given before the inspectors of eight or ten prisoners seen "giddy drunk, not able to sit upon forms." +DUMMY/LJ016-0107.wav|Brown almost at once; then Barry, who was taken at the East End in a public-house where he had arranged to meet a pal. +DUMMY/LJ043-0079.wav|in nineteen of twenty-three categories included on the general aptitude examination and scored one hundred twenty-seven on the verbal test, as compared +DUMMY/LJ026-0133.wav|combined with nitrogen and sulphur (probably also with salts, water, etc.) to form proteid matter. +DUMMY/LJ019-0271.wav|In a word, the value of diet as a part of penal discipline was still insufficiently recognized. +DUMMY/LJ035-0127.wav|down the rear stairs to the first floor, where she encountered two Depository employees -- William Shelley and Billy Lovelady. +DUMMY/LJ034-0105.wav|In his sworn statement to the police later that day, +DUMMY/LJ003-0306.wav|The clothes of prisoners arriving dirty, or in rags, should be fumigated before worn in the jail, +DUMMY/LJ015-0112.wav|But the continental police had been warned to look out for him, and two Danish inspectors got upon his track, +DUMMY/LJ008-0044.wav|and at the distance of five feet from the same is fixed a strong railing all round the scaffold to enclose a place for the constables. +DUMMY/LJ048-0105.wav|but these routes would not have been in accordance with the White House staff instructions given the Secret Service for a desirable motorcade route. +DUMMY/LJ048-0278.wav|The Commission recognizes that the responsibilities of members of the White House detail of the Secret Service are arduous. +DUMMY/LJ044-0198.wav|as he experienced successive failures in his jobs, in his political activity, and in his personal relationships. +DUMMY/LJ002-0249.wav|and the prison, although wives and children resided within the walls, was not overcrowded. +DUMMY/LJ001-0168.wav|On the whole, a small book should be printed on paper which is as thin as may be without being transparent. +DUMMY/LJ009-0029.wav|This episode throws some doubt upon the tenderness and proper feeling exhibited by the chaplain towards the most deserving members of his criminal flock; +DUMMY/LJ019-0131.wav|Mr. Pearson and the committee of eighteen fifty have the more claim on our consideration, because, +DUMMY/LJ027-0127.wav|while neglecting to maintain his unity of ideal in the case of nearly all the numerous species of snakes, he should have added a tiny rudiment in the case of the Python +DUMMY/LJ005-0057.wav|was introduced as early as seventeen ninety by Mr. Blackburn +DUMMY/LJ032-0142.wav|quote, off underside gun barrel near end of grip C two seven six six, end quote, +DUMMY/LJ001-0048.wav|his letter is admirably clear and regular, but at least as beautiful as any other Roman type. +DUMMY/LJ027-0123.wav|where within the limits of the same natural group of organisms a rudiment is sometimes present and sometimes absent. +DUMMY/LJ029-0175.wav|Both Dallas papers cited White House sources on September twenty-six as confirming the President's intention to visit Texas on November twenty-one and twenty-two, +DUMMY/LJ044-0174.wav|and possibly to Philadelphia to look for work. +DUMMY/LJ039-0023.wav|I remember that I held him. We actually struggled for several minutes and then he quieted down. End quote. +DUMMY/LJ050-0048.wav|It has received assistance also from data processing experts at the CIA +DUMMY/LJ010-0149.wav|She was seized before she could do any mischief, +DUMMY/LJ032-0187.wav|The fact that on the morning of the assassination Oswald was wearing the shirt from which these relatively fresh fibers most probably originated, +DUMMY/LJ047-0226.wav|and never realized that the motorcade would pass the Texas School Book Depository Building. +DUMMY/LJ047-0189.wav|Answer: No. I would have to wait until I had talked to Marina to see what I could determine, and from there I could make my plans. +DUMMY/LJ049-0028.wav|The Secret Service has therefore suggested this practice only on extraordinary occasions. +DUMMY/LJ012-0187.wav|one of his accomplices, who took the police to the pond, where the remains of the unfortunate Mr. Weare were discovered, sunk in a sack weighted by stones. +DUMMY/LJ021-0104.wav|let it be remembered that so far as possible, consistent with the immediate public interest of this past year +DUMMY/LJ003-0115.wav|The judge sat in proper form; he was punctiliously styled "my lord." +DUMMY/LJ039-0127.wav|When Sergeant Zahm was asked whether Oswald's Marine Corps training would have made it easier to operate a rifle with a four-power scope, he replied, quote, +DUMMY/LJ001-0101.wav|It is discouraging to note that the improvement of the last fifty years is almost wholly confined to Great Britain. +DUMMY/LJ029-0209.wav|Beneath two photographs of President Kennedy, one full- face and one profile, appeared the caption, quote, Wanted for Treason, +DUMMY/LJ050-0065.wav|Alan H. Belmont, Assistant to the Director of the FBI, testified that this revision was initiated by the FBI itself. +DUMMY/LJ046-0213.wav|members of his immediate family, the President-elect, and the Vice-President. Investigation of threats against the President of the United States, +DUMMY/LJ038-0253.wav|he replied that he had buried it in the ground or hidden it in some bushes and that he also mentioned a railroad track in this connection. +DUMMY/LJ029-0075.wav|Accordingly, it was important to determine the luncheon site as quickly as possible, so that security could be established at the site and the motorcade route selected. +DUMMY/LJ020-0063.wav|They are even better when cold than hot. +DUMMY/LJ014-0021.wav|a coat among them much torn and stained, with three buttons missing, one of which corresponded with that picked up at Hampstead. +DUMMY/LJ003-0076.wav|four for fourteen years, and three of them persons sentenced to fines or short imprisonment -- one for little more than a month. +DUMMY/LJ045-0232.wav|Oswald was overbearing and arrogant throughout much of the time between his arrest and his own death. +DUMMY/LJ016-0388.wav|A few go further, and are almost gluttonous. +DUMMY/LJ026-0128.wav|for it is here that oxidation occurs and the need for a renewal of matter and energy consequently arises. +DUMMY/LJ010-0082.wav|made a voluntary confession to Sir Herbert Taylor one morning at Windsor; after which Thistlewood and his accomplices were closely watched, +DUMMY/LJ008-0049.wav|and this, being much more sudden and regular than that of a cart being drawn away, has the effect of immediate death. +DUMMY/LJ007-0151.wav|and his connection with them is confirmed by that devotion and generosity towards their paramours in adversity for which these otherwise degraded women are remarkable. +DUMMY/LJ038-0207.wav|When he came back I asked him what had happened. He was very pale. +DUMMY/LJ012-0057.wav|But he once more persuaded the turnkeys to pause at the public, where more liquor was consumed. +DUMMY/LJ003-0183.wav|sometimes girls of thirteen, twelve, even ten or nine years of age, were exposed to, quote, +DUMMY/LJ005-0179.wav|he total number of prisoners they received during the year varied from two persons to many hundreds. +DUMMY/LJ041-0024.wav|Oswald erased those names, however, and indicated that he had no close friends. +DUMMY/LJ029-0090.wav|were deployed in and around the Trade Mart. +DUMMY/LJ019-0356.wav|But the act was not limited to permissive legislation. Its provisions and enactments were backed up by certain penalties. +DUMMY/LJ002-0023.wav|On the female side matters were much worse; +DUMMY/LJ010-0233.wav|Here again probably it was partly the love of notoriety which was the incentive, +DUMMY/LJ022-0061.wav|That is why our social security program is an important part of the complete picture. +DUMMY/LJ003-0043.wav|and it was not ready to relieve Newgate till late in eighteen fifteen. +DUMMY/LJ033-0170.wav|the width of the tape, the knurled markings on the surface of the fiber, the texture of the fiber, the letting pattern +DUMMY/LJ037-0025.wav|Benavides saw a man standing at the right side of the parked police car. He then heard three shots and saw the policeman fall to the ground. +DUMMY/LJ005-0236.wav|They were not confined apart from each other, but were crowded thirty or forty together in the press yard, +DUMMY/LJ043-0111.wav|After securing this job and an apartment, Oswald asked his wife to join him. +DUMMY/LJ033-0185.wav|On the other hand, since two rolls could be made from the same batch of paper, +DUMMY/LJ039-0229.wav|Sergeant Zahm testified, quote, +DUMMY/LJ028-0304.wav|till the tenth day after I am entered within the town, and then place near to the gates of Semiramis a detachment of thy army, +DUMMY/LJ040-0082.wav|in the summer of nineteen forty-eight. +DUMMY/LJ013-0117.wav|and with another carry off the plate-chest in broad daylight, and as a matter of business. No one interfered or asked any questions. +DUMMY/LJ018-0032.wav|There was no mystery about his departure; he had gone to Canada, by the 'Victoria' sailing ship, starting from the London docks, and bound to New York. +DUMMY/LJ030-0058.wav|Elevated approximately fifteen inches above the back of the front seat +DUMMY/LJ038-0244.wav|Oswald purchased the money order for the rifle on March twelve, the rifle was shipped on March twenty, and the shooting occurred on April ten. +DUMMY/LJ003-0194.wav|At another time there were fifty, one of whom had been under sentence a couple of years. +DUMMY/LJ022-0186.wav|Twenty years of experience with this system have justified the efforts made to create it, +DUMMY/LJ017-0016.wav|The cases of poisoning in the British calendars are rare, nor indeed was the guilt of the accused always clearly established. +DUMMY/LJ025-0100.wav|in so far as under these circumstances the green plant +DUMMY/LJ049-0107.wav|or who hold important governmental posts. +DUMMY/LJ037-0021.wav|Scoggins stated that he thought he had seen a picture of Oswald in the newspapers prior to the lineup identification on Saturday. +DUMMY/LJ018-0378.wav|In Newgate the 'Lennie' mutineers were extremely well behaved. +DUMMY/LJ041-0051.wav|Oswald was motivated to join the Marines in large part by a desire, quote, to get from out and under the yoke of oppression from my mother, end quote. +DUMMY/LJ013-0103.wav|At Halifax Forrester learnt that the men he wanted had gone on to Boston, thence to Buffalo and Canada, and back to Boston. +DUMMY/LJ019-0073.wav|was impressed with the necessity for authoritative interference. +DUMMY/LJ004-0174.wav|Confinement for twelve months in the Bristol jail was counted a punishment equivalent to seven years' transportation. +DUMMY/LJ032-0020.wav|learned from retail outlets in Dallas that Crescent Firearms, Inc., of New York City, +DUMMY/LJ031-0039.wav|Dr. Carrico was in the emergency area, examining another patient, when he was notified that President Kennedy was en route to the hospital. +DUMMY/LJ022-0184.wav|not by a few private banking institutions, but by a body with public prestige and authority. +DUMMY/LJ032-0235.wav|From an examination of one of the photographs, the Commission determined the dates of the issues of the Militant and the Worker which Oswald was holding in his hand. +DUMMY/LJ042-0228.wav|I went there to see the land, the people and how their system works, end quote. +DUMMY/LJ005-0207.wav|Kidderminster had a prison, one damp chill room, +DUMMY/LJ036-0190.wav|If Oswald left his roominghouse shortly after one p.m. and walked at a brisk pace, +DUMMY/LJ030-0028.wav|and the Vice President and Mrs. Johnson were in the receiving line to greet President and Mrs. Kennedy. +DUMMY/LJ015-0301.wav|While Hardwicke and Atwell were in Newgate awaiting trial, active search was made for Saward, +DUMMY/LJ035-0133.wav|Shelley and Lovelady, however, have testified +DUMMY/LJ011-0285.wav|The door of his place of durance stood open, and Mr. Gee began to consider whether he might not escape. +DUMMY/LJ022-0122.wav|there will be those, of course, who will try to tell you that the exceptional failure is characteristic of the entire endeavor. +DUMMY/LJ028-0314.wav|observing him, hastened down, and setting one of the gates slightly ajar, questioned him who he was, and on what errand he had come. +DUMMY/LJ007-0035.wav|The better-disposed complained bitterly of what they had to endure; +DUMMY/LJ011-0053.wav|This "personality," and it can be called by no other name, is carefully excluded from prison pulpit utterances on the eve of an execution. +DUMMY/LJ034-0152.wav|Brennan told the Commission that he could have made a positive identification in the lineup on November twenty-two +DUMMY/LJ046-0077.wav|Under our system, measures must be sought to afford security without impeding the President's performance of his many functions. +DUMMY/LJ025-0143.wav|nothing but a linear tract of specially modified protoplasm between two points of an organism +DUMMY/LJ028-0046.wav|Another son, Assurbanipal, or the great Sardanapalus of the Greeks, became the King of Nineveh. +DUMMY/LJ040-0012.wav|possibly by factors of which those around him were only dimly aware. +DUMMY/LJ033-0015.wav|approximately fifteen miles from Oswald's place of work at the Texas School Book Depository. +DUMMY/LJ033-0197.wav|quote, in all observable microscopic characteristics, end quote. +DUMMY/LJ048-0163.wav|it has not been the practice of the Secret Service to make surveys or checks of buildings along the route of a Presidential motorcade. +DUMMY/LJ007-0211.wav|"In our late visits," they say, "we have seen manifest indications of a retrograde movement in this respect, +DUMMY/LJ036-0197.wav|He drove a police car painted distinctive colors with Number ten prominently displayed on each side. +DUMMY/LJ032-0220.wav|Since Exhibit Number one thirty-three B was taken with Oswald's camera, +DUMMY/LJ006-0162.wav|But it is only fair to Mr. Cope to state that he himself said he went whenever he could find time +DUMMY/LJ003-0149.wav|supported themselves; they had the ration of prison bread only, +DUMMY/LJ014-0046.wav|Hocker made several pretended confessions and revelations, all of which were proved to be absolutely false by the police on inquiry. +DUMMY/LJ010-0115.wav|and it is pretty certain that for some time he served both sides; +DUMMY/LJ012-0278.wav|and the most probable inference is that he inveigled Mrs. Brown to his lodgings with the set purpose of taking her life. +DUMMY/LJ006-0093.wav|So closely did they lie together, that the inspectors at their night visits found it difficult in stepping across the room to avoid treading on them. +DUMMY/LJ038-0040.wav|As McDonald fell into the seat with his left hand on the gun, he felt something graze across his hand and heard what sounded like the snap of the hammer. +DUMMY/LJ045-0064.wav|that Oswald stated that he did not speak English in his family because he did not want them to become Americanized. +DUMMY/LJ023-0021.wav|to make it bomb-proof against the causes of nineteen twenty-nine. +DUMMY/LJ008-0029.wav|But the Old Bailey gradually, and in spite of all objections urged, monopolized the dread business of execution. +DUMMY/LJ037-0103.wav|Two other important eyewitnesses to Oswald's flight were Ted Callaway, +DUMMY/LJ003-0053.wav|At one time the whole of these different categories were thrown together pell-mell, young and old, the untried with the convicted. +DUMMY/LJ023-0138.wav|by offering life pensions at full pay for federal judges on all courts who are willing to retire at seventy. +DUMMY/LJ042-0162.wav|Full of optimism and hope, he stood in Red Square in the Fall of nineteen fifty-nine, vowing to see his chosen course through, +DUMMY/LJ032-0038.wav|Opposite the printed words "Pay To" were written the words "Kleins Sporting Goods," +DUMMY/LJ016-0033.wav|He had access to the airing yard, and there was for hours no kind of supervision. +DUMMY/LJ041-0200.wav|that his use of those excuses prevented him from discovering the real reasons for and attempting to overcome his difficulties. +DUMMY/LJ033-0007.wav|the circumstances surrounding Oswald's return to Irving, Texas, on Thursday, November twenty-one, nineteen sixty-three, +DUMMY/LJ044-0147.wav|and in late June or early July he told his wife that he wanted to return to the Soviet Union with her. She said that he was extremely upset, +DUMMY/LJ033-0162.wav|from the shipping room of the Depository and forwarded it to the FBI Laboratory in Washington. +DUMMY/LJ047-0020.wav|Until April nineteen sixty, FBI activity consisted of placing in Oswald's file +DUMMY/LJ004-0100.wav|An infirmary, consisting of two distinct rooms, one for males and one for females, should be provided for the separate accommodation of the sick. +DUMMY/LJ015-0088.wav|To his wife, poor soul, he made no pretense of fidelity, and she enjoyed only so much of his company as was necessarily spent +DUMMY/LJ013-0198.wav|and in an impassioned address implored the jury not to send an innocent man to the gallows. +DUMMY/LJ004-0102.wav|were to be kept in every jail, and the prisoners directed to wash in them before release. These provisions were almost a dead letter. +DUMMY/LJ049-0031.wav|The Presidential vehicle in use in Dallas, described in chapter two, +DUMMY/LJ016-0080.wav|and probably one of the few cases of a recurrence, but under proper safeguards and limitations, to the old system of chains. +DUMMY/LJ036-0010.wav|(two) testimony of firearms identification experts establishing the identity of the murder weapon, +DUMMY/LJ050-0026.wav|The Commission urges that the Service continue this effort to overhaul and define its procedures. +DUMMY/LJ017-0122.wav|A government prosecution was instituted, and Palmer was brought to Newgate for trial at the Central Criminal Court. +DUMMY/LJ041-0083.wav|Thornley also testified that he thought that Oswald's extreme personal sloppiness in the Marine Corps, quote, +DUMMY/LJ014-0109.wav|Part of this she had deposited to be left till called for at one station, while she had gone herself to another, that at Euston Square. +DUMMY/LJ022-0048.wav|They are still declining. +DUMMY/LJ048-0048.wav|All this does seem to amount to enough to have induced an alert agency, such as the FBI, +DUMMY/LJ036-0035.wav|The man was on the bus approximately four minutes. +DUMMY/LJ028-0382.wav|and their houses shall be full of doleful creatures, and owls shall dwell there, and satyrs shall dance there, +DUMMY/LJ004-0070.wav|the conservation of his health and industrious habits, are the clear, evident, undeniable rights of an unconvicted prisoner." +DUMMY/LJ024-0003.wav|What is my proposal? +DUMMY/LJ050-0153.wav|In the planning document currently under review by the Bureau of the Budget, the Department recommends that it be permitted to hire five qualified persons, quote, +DUMMY/LJ002-0098.wav|The prisoners had few comforts, beyond the occasional use of a bath at some distance, situated in the press yard, +DUMMY/LJ042-0111.wav|Jeanne De Mohrenschildt, another member of that group, +DUMMY/LJ020-0077.wav|Study the temperature in different parts of the kitchen and kitchen closets to the end of finding the best places for raising dough and sponge. +DUMMY/LJ017-0101.wav|That day Palmer had bought more strychnia, and had called in a fresh doctor. +DUMMY/LJ042-0019.wav|The validity of the latter observation is borne out by some of the things Oswald wrote in connection with his defection +DUMMY/LJ004-0016.wav|I have shown in a previous chapter what Newgate was at this period, despite a vast expenditure and boasted efforts to introduce reforms. +DUMMY/LJ010-0109.wav|Thistlewood made a long and rambling defense, the chief features of which were abuse of Lord Sidmouth, and the vilification of the informer Edwards. +DUMMY/LJ017-0168.wav|But a long public discussion followed, and in consequence he was reprieved. +DUMMY/LJ038-0236.wav|Two of these photographs were taken from the rear of Walker's house. +DUMMY/LJ013-0108.wav|Next day a person betrayed him for the reward, and he was soon captured. +DUMMY/LJ006-0168.wav|to taking the descriptions of newly-arrived prisoners. +DUMMY/LJ033-0030.wav|Mrs. A. C. Johnson, his landlady, testified that Oswald's room at ten twenty-six North Beckley Avenue +DUMMY/LJ045-0117.wav|it would appear to be unlikely that his landlady in Dallas +DUMMY/LJ047-0096.wav|The police called the local FBI office and an agent, John L. Quigley, was sent to the police station. +DUMMY/LJ033-0212.wav|(two) took paper and tape from the wrapping bench of the Depository and fashioned a bag large enough to carry the disassembled rifle; +DUMMY/LJ034-0138.wav|they saw and heard Brennan describing what he had seen. +DUMMY/LJ017-0250.wav|from Peru bound to Bordeaux, which had foundered at sea; +DUMMY/LJ027-0049.wav|the structural elements remain, but are profoundly modified to perform totally different functions. +DUMMY/LJ014-0185.wav|and bystanders peeped in through the shutters, but no one entered or sought to interfere in what seemed only a domestic quarrel. +DUMMY/LJ009-0053.wav|but who is left to die on the supposition that this is not his first conviction, and still more because a good many sheep have of late been stolen by other people. +DUMMY/LJ003-0051.wav|(two) persons under sentence of imprisonment for a fixed period, or until they shall have paid certain fines; +DUMMY/LJ050-0085.wav|other than legal or peaceful, to satisfy any grievance, real or imagined. +DUMMY/LJ017-0276.wav|Then he wanted to know when the policemen would arrive, and was told none would come. +DUMMY/LJ002-0006.wav|but it was almost always in excess of the accommodation available, and there were times when the place was full to overflowing. +DUMMY/LJ028-0305.wav|troops for whose loss thou wilt care little, a thousand men. +DUMMY/LJ009-0294.wav|One was Askern, +DUMMY/LJ049-0159.wav|the Secret Service, then the only Federal investigative agency, assumed full-time responsibility for the safety of the President. +DUMMY/LJ002-0330.wav|female visitors carried them out of the prisons, or the debtors destroyed them when the weather was warm, +DUMMY/LJ013-0064.wav|Barber, it was said, became aware of this, +DUMMY/LJ017-0055.wav|Throughout his trial he preserved an impassive countenance, but he clearly took a deep interest in all that passed. +DUMMY/LJ037-0171.wav|The examination and testimony of the experts enabled the Commission to conclude that five shots may have been fired, +DUMMY/LJ009-0239.wav|The whole case was referred to the king, and the poor wretch, whose crime had been a military one, was eventually pardoned. +DUMMY/LJ007-0129.wav|the sane and the insane, the young and the old, the trivial offender and the man about to suffer the extreme penalty of the law, +DUMMY/LJ002-0317.wav|and there were as many as three hundred and fifty prisoners in at one time. +DUMMY/LJ018-0054.wav|While in the condemned cell he conversed freely with the warders in broken English or through an interpreter. +DUMMY/LJ036-0126.wav|As he unloaded his Continental bus station passenger in front of Greyhound, he started to get out to buy a package of cigarettes. +DUMMY/LJ014-0152.wav|On reaching the condemned cell she threw herself upon the floor and shrieked in an hysterical agony of tears. +DUMMY/LJ003-0349.wav|Why not relieve Newgate by drawing more largely upon the superior accommodation which Millbank offered? +DUMMY/LJ032-0147.wav|At the request of the Commission, Arthur Mandella, +DUMMY/LJ025-0138.wav|The results of inquiries into the structure of the nervous system of animals +DUMMY/LJ032-0207.wav|I found it to be the same general configuration. All appearances were the same, end quote, +DUMMY/LJ037-0264.wav|and seven eyewitnesses who saw the flight of the gunman with revolver in hand +DUMMY/LJ046-0120.wav|the security processing of gifts sent to the President, and technical inspections against covert listening devices. +DUMMY/LJ034-0204.wav|and not to his race. +DUMMY/LJ033-0048.wav|Mrs. Paine was certain that she had not left the light on in the garage after dinner. +DUMMY/LJ041-0178.wav|not unpopular, end quote, at that time. Donovan testified +DUMMY/LJ015-0214.wav|At the other side of the Channel the French railway authorities took charge. +DUMMY/LJ019-0056.wav|Colonel Jebb preferred it; Messrs. Crawford and Whitworth Russell were convinced that the complete isolation of criminals from one another +DUMMY/LJ021-0144.wav|in order to seek their cooperation in establishing what I may describe as a specific trial period of industrial peace. +DUMMY/LJ012-0025.wav|at which he realized one hundred fifty pounds within a couple of years. +DUMMY/LJ031-0080.wav|The time was fixed at one p.m., as an approximation, since it was impossible to determine the precise moment when life left the President. +DUMMY/LJ048-0078.wav|In each instance, liaison contacts should be developed to include a close friendly relationship, +DUMMY/LJ032-0249.wav|that Oswald may have sat on the screened-in porch at night practicing with the rifle by looking through the telescopic sight and operating the bolt. +DUMMY/LJ049-0144.wav|The Committee would be able to provide guidance in defining the general nature of domestic and foreign dangers to Presidential security. +DUMMY/LJ018-0135.wav|and in the states known as "water-leaf" and "sized," which are the penultimate processes of manufacture. +DUMMY/LJ012-0110.wav|The police lost all trace of them for some days, but at length Sullivan's brother was followed from the house in Kennington to the above-mentioned tavern. +DUMMY/LJ014-0166.wav|faded in my mind before the atrocious bearing, looks, and language of the assembled spectators. +DUMMY/LJ028-0101.wav|There is scarcely one of the thousands of ruin mounds in Babylonia which does not contain bricks bearing his name. +DUMMY/LJ012-0231.wav|although it included statements that were improbable and unsubstantiated. +DUMMY/LJ013-0046.wav|but he would not take their advice, and developing his transactions, ended as we have seen. +DUMMY/LJ013-0267.wav|He made a rambling defense, ending by saying, +DUMMY/LJ033-0012.wav|The Curtain Rod Story +DUMMY/LJ041-0156.wav|he preferred the, quote, Red Army, end quote. +DUMMY/LJ002-0276.wav|went outside and beyond the prison to call on their creditors, and try to arrange their debts, but this practice was discontinued. +DUMMY/LJ019-0397.wav|with spare accommodation sufficient to receive all prisoners who were then committed to Newgate. These arguments were unanswerable. +DUMMY/LJ028-0268.wav|Mounting upon the battlements that crowned their walls, they insulted and jeered at Darius and his mighty host. +DUMMY/LJ039-0182.wav|Five of the six shots hit the third target where the angle of movement of the weapon was small. +DUMMY/LJ049-0135.wav|(which is responsible for advising the President respecting the coordination +DUMMY/LJ005-0195.wav|There was no decency whatever in the internal arrangements; +DUMMY/LJ004-0209.wav|The looms were constantly busy. Tailors were always at work, and every article of clothing and bedding was made up within the walls. +DUMMY/LJ015-0220.wav|The four waited patiently for their opportunity, +DUMMY/LJ010-0222.wav|in consequence of having received some information of an important nature from Hanover. +DUMMY/LJ028-0330.wav|and when the next period agreed on had elapsed, again with a band of picked men he sallied forth, and slaughtered the two thousand. +DUMMY/LJ026-0120.wav|The building up of the protoplasm from proteids is anabolism, constructive metabolism. +DUMMY/LJ038-0193.wav|However, I prefer that you hold on to my personal papers (military, civil, etc.). +DUMMY/LJ027-0058.wav|enclosing and protecting the nervous centers (cerebrospinal axis). +DUMMY/LJ046-0126.wav|estimated that most of the material received by his office originated in this fashion +DUMMY/LJ027-0003.wav|The facts of biology which admit of adequate explanation only in connection with the theory of descent +DUMMY/LJ008-0267.wav|no less than four hundred and fifty-one sentences of death for capital crimes were passed at the Old Bailey; +DUMMY/LJ040-0048.wav|Oswald did not always distinguish between Marxism and communism. +DUMMY/LJ003-0175.wav|was accommodated with a room in Mr. Newman's house at the extravagant rental of thirty guineas per week; +DUMMY/LJ025-0032.wav|Hence arose the second great distinctive character of animals, or the circulatory system, which is less important than the digestive, +DUMMY/LJ013-0004.wav|The most extensive and systematic robberies were planned and carried out so as long to escape detection. +DUMMY/LJ011-0219.wav|William Wakefield in Lancaster Jail, and Edward Gibbon Wakefield in Newgate, which sentences were duly enforced. +DUMMY/LJ044-0185.wav|He engaged in an angry argument with the consul who finally told him that, quote, as far as he was concerned +DUMMY/LJ048-0106.wav|Much of Lawson's time was taken with establishing adequate security over the motorcade route and at the two places where the President would stop, +DUMMY/LJ030-0087.wav|The Vice-Presidential automobile, a four-door Lincoln convertible obtained locally for use in the motorcade, +DUMMY/LJ049-0149.wav|that it has been unable, as a practical matter, to exercise sufficient influence over the security precautions which surround Presidential activities. +DUMMY/LJ016-0016.wav|There is an attempt at escape mentioned in Mr. Wakefield's book, which might have been an intended suicide. +DUMMY/LJ041-0103.wav|Oswald once told Powers that he didn't care if he returned to the United States at all. +DUMMY/LJ022-0106.wav|This will make certain that the largest possible portion of the funds allotted +DUMMY/LJ007-0135.wav|The yards were narrow and confined, mainly because the ground plan was radically vicious. These were evils inseparable from the place. +DUMMY/LJ021-0036.wav|loans to the railroads and insurance companies and, finally, help for home owners and industry itself. +DUMMY/LJ008-0061.wav|The entrance upon this floor or leaf is from the middle window over the gate of the prison; +DUMMY/LJ037-0029.wav|It was Benavides, using Tippit's car radio, who first reported the killing of Patrolman Tippit at about one:sixteen p.m.: +DUMMY/LJ034-0203.wav|Euins stated that he could not ascertain the man's race and that the statement in the affidavit was intended to refer only to the white spot on the man's head +DUMMY/LJ040-0135.wav|She recalled that Lee cried and said, quote, Mother, I want to get out of here. +DUMMY/LJ012-0087.wav|should be introduced into the building, and secreted there during the night to accomplish the robbery. +DUMMY/LJ013-0079.wav|when Lord Campbell delivered judgment on Barber's petition, to the effect that +DUMMY/LJ040-0064.wav|This occurred two months before Lee was born in New Orleans on October eighteen, nineteen thirty-nine. +DUMMY/LJ036-0188.wav|Oswald was next seen about nine-tenths of a mile away +DUMMY/LJ029-0082.wav|several tiers of balconies surrounding the central court where the luncheon would be held, and several catwalks crossing the court at each level. +DUMMY/LJ029-0024.wav|through the evening of Friday, November twenty-two. +DUMMY/LJ017-0031.wav|instead of limiting them, as in the days of the Borgias and Brinvilliers, to the specially informed and unscrupulously powerful few. +DUMMY/LJ010-0174.wav|About six p.m. the royal carriage, a low open vehicle drawn by four horses, ridden by postilions, left the palace. +DUMMY/LJ020-0081.wav|and knead for ten minutes, carefully at first, lest the liquids should be wasted, and more boldly when they are absorbed by the paste. +DUMMY/LJ016-0421.wav|brutal creature who showed no remorse, but was subject to fits of ungovernable passion, when she broke out into language the most appalling. +DUMMY/LJ002-0286.wav|The rooms had no fireplaces, so it mattered little that no coals were allowed. +DUMMY/LJ016-0166.wav|and surgical opinion stated that the stoppage of circulation was the cause of death. +DUMMY/LJ032-0212.wav|Commission Exhibit Number one thirty-three B, to Oswald's Imperial Reflex camera, with which Marina Oswald testified she took the pictures. +DUMMY/LJ046-0169.wav|If the field office determines that the case should be subject to continuing review, PRS establishes a file +DUMMY/LJ040-0106.wav|It also appears that Lee tried to tag along with his older brothers +DUMMY/LJ036-0058.wav|People on the bus began talking about it. As the bus neared Lamar Street, Oswald left the bus and disappeared into the crowd. +DUMMY/LJ033-0039.wav|and one which provided an excuse for the carrying of a bulky package the following morning. +DUMMY/LJ031-0040.wav|Approximately two minutes later, Dr. Carrico saw the President on his back, being wheeled into the emergency area. +DUMMY/LJ020-0071.wav|One tablespoonful of butter, just melted, not hot. +DUMMY/LJ050-0185.wav|The Service should consider preparing formal explanations of the cooperation anticipated during a Presidential visit to a city, +DUMMY/LJ049-0056.wav|The first duty of the agents in the motorcade is to attempt to cover the President as closely as possible and practicable +DUMMY/LJ007-0118.wav|A prisoner, one of the wardsmen, waited on those in the infirmary; the occupants of the governor's house had their own servants, or the governor's. +DUMMY/LJ017-0021.wav|A third case, more especially interesting to us as having passed through Newgate, +DUMMY/LJ005-0194.wav|The keeper and his officers resided at a distance from the jail, and left its inmates to their own devices. +DUMMY/LJ047-0024.wav|in the course of a routine FBI investigation of transfers of small sums of money from Mrs. Oswald to her son in Russia. +DUMMY/LJ010-0187.wav|He was taken into custody, and removed first to a police cell, thence committed to Newgate, after he had been examined before the Privy Council. +DUMMY/LJ013-0113.wav|Robberies as daring in conception as they were boldly executed were common enough. +DUMMY/LJ048-0040.wav|The Commission appreciates the large volume of cases handled by the FBI +DUMMY/LJ032-0113.wav|Based on the above evidence, the Commission concluded that Oswald purchased the rifle found on the sixth floor of the Depository Building. +DUMMY/LJ022-0005.wav|It has made and is making distinct progress. +DUMMY/LJ033-0199.wav|because other types of fibers present in the blanket were not found in the bag. +DUMMY/LJ020-0029.wav|Put the brown or Graham flour unsifted into the bread-bowl. +DUMMY/LJ045-0206.wav|it must have appeared to him that he was unable to command even the attention of his family. +DUMMY/LJ006-0061.wav|and "the greatest contempt shown for the law." +DUMMY/LJ014-0207.wav|The former was given a reward of twenty and the latter of ten pounds. +DUMMY/LJ029-0122.wav|Elm Street, parallel to Main Street and one block north, +DUMMY/LJ022-0120.wav|there may be occasional instances of inefficiency, bad management, or misuse of funds. +DUMMY/LJ034-0207.wav|Accordingly, Euins' testimony is considered probative as to the source of the shots but is inconclusive as to the identity of the man in the window. +DUMMY/LJ002-0003.wav|with criminals and misdemeanants of all shades crowding perpetually into its narrow limits, the latter state of Newgate was worse than the first. +DUMMY/LJ003-0035.wav|and about eighteen twelve application was made to Parliament for funds to build a new debtors' prison. +DUMMY/LJ011-0292.wav|There was no furniture in the place, and the den in the kitchen had been recently and specially constructed of boards of immense strength and thickness. +DUMMY/LJ036-0081.wav|The station wagon stopped to pick up the man and then drove off. +DUMMY/LJ008-0135.wav|and stepped out on to the platform, to be received, as has been said, by prolonged yells. +DUMMY/LJ002-0281.wav|The demands for fees were excessive in Giltspur Street. +DUMMY/LJ028-0039.wav|We know the names of its kings, and the records speak of long wars with the Assyrians. +DUMMY/LJ028-0312.wav|Having left these instructions, Zopyrus fled towards the gates of the town, often looking back, to give himself the air of a deserter. +DUMMY/LJ011-0115.wav|Hunton's execution no doubt aroused public attention to the cruelty and futility of the capital law against forgery. +DUMMY/LJ026-0097.wav|furnish the materials and energy required for the life and growth of the plant. +DUMMY/LJ008-0205.wav|The city authorities had endeavored to take all precautions against panic and excitement among the crowd, +DUMMY/LJ031-0193.wav|On the return flight, Mrs. Kennedy sat with David Powers, Kenneth O'Donnell, and Lawrence O'Brien. +DUMMY/LJ028-0182.wav|and in five fifty-five Nabonidus, the father of the Biblical Belshazzar, came to the throne. +DUMMY/LJ004-0226.wav|It was also claimed for the more ample and more orderly distribution of victuals, that the general health of the prisoners had greatly improved. +DUMMY/LJ050-0132.wav|the Commission notes with approval several recent measures taken and proposed by the Secret Service to improve its liaison arrangements. +DUMMY/LJ028-0254.wav|The people, enjoying the greater freedom which Cyrus permitted them, were contented, and life in Babylon went on about as before. +DUMMY/LJ004-0142.wav|where a lad lay ill with fever, three other prisoners, at first perfectly healthy, were lodged. Of course they were seized with the fever; +DUMMY/LJ010-0126.wav|A crowd as great as any known collected in the Old Bailey to see the ceremony, about which there were some peculiar features worth recording. +DUMMY/LJ005-0186.wav|They neither built new jails nor contracted with the counties, as had been expected, for the transfer of their prisoners. +DUMMY/LJ041-0002.wav|Chapter seven. Lee Harvey Oswald: Background and Possible Motives, Part two. +DUMMY/LJ032-0118.wav|Capt. J. W. Fritz then ejected a cartridge by operating the bolt, +DUMMY/LJ002-0125.wav|Neild also says that in seventeen ninety-three, +DUMMY/LJ037-0092.wav|Each woman whispered Oswald's number to the detective. Each testified that she was the first to make the identification. +DUMMY/LJ004-0015.wav|the motives for a transient amendment were becoming paralyzed, and the effect had ceased with the cause." +DUMMY/LJ048-0118.wav|He did not have a checklist of the tasks he was expected to accomplish, either by his own efforts or with the cooperation of local authorities. +DUMMY/LJ038-0034.wav|When he reached the row where the suspect was sitting, McDonald stopped abruptly and told the man to get on his feet. +DUMMY/LJ010-0067.wav|This Thistlewood had seen many vicissitudes throughout his strange, adventurous career. The son of a respectable Lincolnshire farmer, +DUMMY/LJ040-0186.wav|Lee scored an I.Q. of one eighteen on the Wechsler Intelligence Scale for Children. +DUMMY/LJ040-0146.wav|He noted that Lee liked to give the impression that he did not care for other people but preferred to keep to himself, +DUMMY/LJ008-0009.wav|But the Old Bailey was not exclusively used; +DUMMY/LJ016-0105.wav|These men were all in prison dress at the time of their escape, but one of their number, Bell, sent back his clothes a few days later by parcel's delivery, +DUMMY/LJ006-0206.wav|and publications which in these days would have been made the subject of a criminal prosecution. +DUMMY/LJ017-0123.wav|There was not much reserve about him when there. +DUMMY/LJ007-0045.wav|They amused themselves after their own fashion; played all day long at blind-man's-buff and leap-frog, or beat each other with a knotted handkerchief, +DUMMY/LJ007-0033.wav|All privacy was impossible under the circumstances. +DUMMY/LJ018-0395.wav|convicted of obtaining jewelery under the false pretense of making silly women "beautiful for ever." +DUMMY/LJ013-0012.wav|Her owners insured her for a full sum of two thousand pounds, after which the Wallaces insured her privily +DUMMY/LJ008-0311.wav|The whole of the convicts were assembled together in one ward, and made to kneel down. +DUMMY/LJ024-0047.wav|Is it a dangerous precedent for the Congress to change the number of the justices? The Congress has always had, and will have, that power. +DUMMY/LJ041-0124.wav|Marina Oswald testified that she, quote, had never heard anything bad about Kennedy from Lee. And he never had anything against him, end quote. +DUMMY/LJ016-0243.wav|Marwood, who succeeded him, and who died while these sheets were in the press, was a Lincolnshire man, a native of Horncastle, +DUMMY/LJ047-0153.wav|Having determined that Mrs. Paine was a responsible and reliable citizen, Hosty interviewed her on November one. +DUMMY/LJ016-0101.wav|One of the three, shamming ill, remained all day in his ward, where he employed himself unraveling the rope from the sleeping-mats. +DUMMY/LJ010-0098.wav|The police were the first to arrive on the spot, the Guards having entered the street at the wrong end. +DUMMY/LJ023-0102.wav|subsistence, and health of large numbers in the community, then +DUMMY/LJ023-0110.wav|a super-legislature, as one of the justices has called it +DUMMY/LJ016-0432.wav|The only subject another showed any interest in was the theatres and the new pieces that were being produced. A third, Christian Satler, +DUMMY/LJ010-0057.wav|are by this time actually, and by more legitimate efforts, engrafted upon our Constitution. +DUMMY/LJ029-0078.wav|The second, the Women's Building at the State Fair Grounds, +DUMMY/LJ033-0070.wav|And she indicated to me that she had peered into this roll and saw a portion of what she took to be a gun she knew her husband to have, a rifle. +DUMMY/LJ012-0014.wav|Edwards was sentenced to two years' imprisonment in Newgate, +DUMMY/LJ001-0114.wav|an ugly pear-shaped swelling defacing the form of the letter: +DUMMY/LJ038-0048.wav|Testimony of a firearms expert before the Commission established that the hammer of the revolver never touched the shell in the chamber. +DUMMY/LJ028-0519.wav|or that the Babylonian soldier stood confidently upon their summit, and jeering at the Persian army encamped below, shouted: +DUMMY/LJ028-0487.wav|Fortunately its walls have suffered less from the hands of the brick hunters, and the German excavators have been able to reconstruct their plan. +DUMMY/LJ050-0079.wav|guidelines for an experimental program to develop more detailed criteria. +DUMMY/LJ050-0116.wav|which carry the major responsibility for supplying information about potential threats, +DUMMY/LJ050-0147.wav|Unless the Secret Service is able to deal rapidly and accurately with a growing body of data, +DUMMY/LJ048-0125.wav|the precautions taken for the President's trip were the usual safeguards employed on trips of this kind in the United States during the previous year, end quote. +DUMMY/LJ031-0005.wav|approximately four miles from the Texas School Book Depository Building. +DUMMY/LJ009-0269.wav|his chest heaved, and it was evident that his struggle was a fearful one. +DUMMY/LJ004-0043.wav|A king's son, the Duke of Gloucester, was the patron; among the vice-presidents were many great peers of the realm +DUMMY/LJ017-0006.wav|The crime of poisoning has always been viewed with peculiar loathing and terror in this country. +DUMMY/LJ024-0086.wav|The Court itself can best undo what the Court has done. +DUMMY/LJ024-0056.wav|that we encourage the retirement of elderly judges by offering them a life pension at full salary. +DUMMY/LJ018-0371.wav|A case reproducing many of the features of the 'Flowery Land' occurred twelve years later, when the crew of the 'Lennie' +DUMMY/LJ049-0034.wav|could have reached the President in time to protect him from the second and fatal shot to hit the President. +DUMMY/LJ004-0075.wav|"Who ever heard of a criminal being sentenced to catch the rheumatism or the typhus fever?" +DUMMY/LJ032-0130.wav|Latona then processed the complete weapon but developed no identifiable prints. +DUMMY/LJ035-0113.wav|probably because Jack Dougherty took it to the first floor while Baker and Truly were running up the stairs or in the lunchroom with Oswald. +DUMMY/LJ030-0116.wav|looking out toward the crowd, and Special Agent Kellerman assumed his position next to the car. +DUMMY/LJ027-0015.wav|which in some fashion presides over all the movements of the protoplasm and is the physical basis of the phenomena of heredity. +DUMMY/LJ018-0090.wav|This conspiracy was cleverly planned, but had scarcely so many ramifications as that of Saward. Its originators were a couple of men, +DUMMY/LJ043-0064.wav|which were largely of his own making. +DUMMY/LJ029-0129.wav|On this last portion of the journey, only five minutes from the Trade Mart, +DUMMY/LJ045-0029.wav|The sole exception was the weekend of November sixteen to seventeen, nineteen sixty-three, the weekend before the assassination, +DUMMY/LJ042-0042.wav|Asking his brother why he supported the American Government and what ideals he put forward, Oswald wrote, quote, +DUMMY/LJ019-0070.wav|The other side retorted that the system of associated labor was most costly, so many officers being required to maintain the discipline of silence; +DUMMY/LJ024-0115.wav|To them I say: we cannot rely on an amendment as the immediate or only answer to our present difficulties. +DUMMY/LJ042-0225.wav|as a mark of discuss and protest against American political policies in foreign countries, my personal sign of discontent +DUMMY/LJ012-0289.wav|Greenacre, when tried at the Old Bailey, admitted that he had been guilty of manslaughter. +DUMMY/LJ021-0049.wav|and the curtailment of rank stock speculation through the Securities Exchange Act. +DUMMY/LJ046-0236.wav|none of the cases in the PRS general files was available for systematic review on a geographic basis when the President planned a particular trip. +DUMMY/LJ036-0059.wav|The Marsalis bus which Oswald boarded traveled a route west on Elm, +DUMMY/LJ039-0241.wav|owned and possessed the rifle used to kill President Kennedy and wound Governor Connally, +DUMMY/LJ002-0053.wav|It had been for some time devoted principally to felons of the worst types, +DUMMY/LJ039-0102.wav|After familiarization with live ammunition in the twenty-two rifle and the twenty-two pistol, +DUMMY/LJ019-0227.wav|This block contained one hundred and thirty cells, embracing every modern improvement; +DUMMY/LJ003-0102.wav|Mr. Buxton, in the work already quoted, instances another grievous case of the horrors of indiscriminate association in Newgate. +DUMMY/LJ046-0005.wav|Abraham Lincoln, James A. Garfield, William McKinley, and John F. Kennedy. +DUMMY/LJ029-0017.wav|Moreover, he looked forward to the public appearances which he personally enjoyed. The basic decision on the November trip to Texas was made at a meeting of President Kennedy, +DUMMY/LJ039-0009.wav|When Marina Oswald testified before the Commission on February three to six, nineteen sixty-four, +DUMMY/LJ032-0077.wav|Among other identification cards in Oswald's wallet at the time of his arrest were a Selective Service notice of classification, +DUMMY/LJ013-0200.wav|The jury without hesitation found Courvoisier guilty, and he was sentenced to death. +DUMMY/LJ028-0264.wav|The women chosen were kept to make bread for the men; while the others were strangled that they might not consume the stores. +DUMMY/LJ027-0175.wav|Now, this is undoubtedly the order of succession of forms in geological times -- i.e., in the phylogenic series. +DUMMY/LJ037-0098.wav|that he saw Oswald on television the night of the murder and thought that Oswald was the man he had seen running away from the shooting. +DUMMY/LJ049-0098.wav|In nineteen oh two bills passed both Houses of Congress but failed of enactment when the Senate refused to accept the conference report. +DUMMY/LJ019-0358.wav|with those in which there was no separation, no proper enforcement of hard labor, no chapel, infirmary, and so forth. +DUMMY/LJ010-0154.wav|when John Collins, "a person in the garb of a sailor, of wretched appearance, and having a wooden leg," +DUMMY/LJ010-0014.wav|to cover enormous defalcations; the fabrication of deeds, wills, and false securities +DUMMY/LJ004-0055.wav|he personally visited several English jails, and pointed his observations by drawing forcible contrasts between the good and bad. +DUMMY/LJ045-0077.wav|because of insufficient postage, which apparently resulted from an increase in postal rates of which his wife had been unaware. +DUMMY/LJ001-0027.wav|especially as regards the lower-case letters; and type very similar was used during the next fifteen or twenty years not only by Schoeffer, +DUMMY/LJ039-0060.wav|The Nature of the Shots +DUMMY/LJ007-0226.wav|or the Inspectors of Prisons, and that he be required wholly to confine himself to the performance of his duty as prescribed by Act of Parliament. +DUMMY/LJ033-0099.wav|Frazier walked behind and as they crossed the railroad tracks he watched the switching of the cars. +DUMMY/LJ012-0124.wav|One was occupied by the Portuguese ambassador, who lost a quantity of jewelery from an escritoire, and his neighbors lost plate and cash. +DUMMY/LJ019-0286.wav|The minor borough prisons were the worst blot on the still dark and imperfect system. +DUMMY/LJ015-0099.wav|"Let us go to Kilburn for them together," said Mr. Fasson, growing suspicious. +DUMMY/LJ030-0020.wav|if anybody really wanted to shoot the President of the United States, it was not a very difficult job +DUMMY/LJ044-0200.wav|his last gambit to extricate himself from the mediocrity and defeat which plagued him throughout most of his life. +DUMMY/LJ004-0004.wav|It began to be understood that prison reform could only be compassed by continuous and combined effort. +DUMMY/LJ047-0066.wav|a Communist Party publication. +DUMMY/LJ003-0049.wav|The inmates of this part, as distinguished from the debtors, were comprised in four classes: +DUMMY/LJ040-0173.wav|He always felt like a burden that she simply just had to tolerate, end quote. +DUMMY/LJ006-0067.wav|"Here," say the inspectors, "are herded together the very worst class of prisoners; certainly a more wretched combination of human beings can hardly be imagined. +DUMMY/LJ048-0195.wav|All received their instructions early in the morning from Capt. P. W. Lawrence of the traffic division. +DUMMY/LJ028-0068.wav|They tell how he forced the exiles to carry heavy bags of sand across the desert to increase their burdens; +DUMMY/LJ020-0037.wav|The oven must be steady, but not so hot as for white bread, nor will the Graham bread be done quite so soon as that made of bolted flour. +DUMMY/LJ014-0188.wav|Pushing the shop-door open, he saw Marley finishing his murderous assault. +DUMMY/LJ017-0174.wav|She was tried in April eighteen sixty-two on suspicion of having attempted to poison a neighbor with oil of vitriol. +DUMMY/LJ042-0145.wav|In general, our family life began to deteriorate after we arrived in America. +DUMMY/LJ035-0067.wav|from the sixth floor by the time Baker and Truly arrived, Commission counsel asked Baker and Truly to repeat their movements from the time of the shot +DUMMY/LJ036-0179.wav|He signed his name as O. H. Lee on the roominghouse register. +DUMMY/LJ017-0187.wav|Her husband, who came up to town, would not allow a post-mortem, and again Mrs. Wilson escaped. +DUMMY/LJ041-0171.wav|And I remember his voice cracked as he said this. He was definitely disturbed at what I had said and I didn't really think I had said that much. +DUMMY/LJ019-0392.wav|in the administration of prisons. +DUMMY/LJ033-0200.wav|He concluded, quote, +DUMMY/LJ014-0172.wav|Robert Marley at the time of his arrest called himself a surgical instrument maker. +DUMMY/LJ006-0016.wav|These considerations no doubt had weight +DUMMY/LJ020-0091.wav|Yet our bread and rolls must be looked after at the proper time. +DUMMY/LJ015-0130.wav|fifteen hundred pounds, which larger amount was duly carried to his credit on the register, and entered upon the certificates of transfer. +DUMMY/LJ019-0058.wav|Prison chaplains of experience and high repute, such as Messrs. Field, Clay, Kingsmill, Burt, and Osborne, also advocated it. +DUMMY/LJ008-0257.wav|In thus describing the saturnalia before the gallows I have been drawn on somewhat beyond the period with which I am at present dealing. +DUMMY/LJ035-0026.wav|Baker and building superintendent Roy Truly went through a second set of doors and stopped at a swinging door where Baker bumped into Truly's back. +DUMMY/LJ005-0109.wav|or contriving some means of escape by climbing the fluted columns which supported the Gothic arches of the aisles, +DUMMY/LJ021-0039.wav|I believe it will be. +DUMMY/LJ001-0160.wav|by that made in America, which is the worst conceivable. +DUMMY/LJ019-0380.wav|In one part the most praiseworthy activity prevailed, +DUMMY/LJ031-0037.wav|Treatment of President Kennedy +DUMMY/LJ008-0105.wav|Mr. Smith's account of the condemned convict, whose cell he was permitted to enter, may be inserted here. +DUMMY/LJ037-0096.wav|Several days later he reported what he had seen and was questioned by FBI agents. +DUMMY/LJ008-0014.wav|John Hogan, the murderer of a Mr. Odell, an attorney who resided in Charlotte Street, Rathbone Place, +DUMMY/LJ041-0133.wav|which finally declined to modify the discharge and so advised him in a letter dated July nineteen sixty-three. +DUMMY/LJ049-0184.wav|to conduct the wider investigative work that is required today for the security of the President. +DUMMY/LJ003-0321.wav|No wine or beer should be in future admitted into or sold in the jail, +DUMMY/LJ047-0120.wav|was considered sufficiently unusual to necessitate another interview. Alan H. Belmont, Assistant to the Director of the FBI, +DUMMY/LJ017-0025.wav|the justice of the sentence was doubted at the time. +DUMMY/LJ012-0096.wav|At her death the diamonds were divided between her four daughters, but only half had been claimed, +DUMMY/LJ033-0134.wav|Frazier could easily have been mistaken when he stated that Oswald held the bottom of the bag cupped in his hand with the upper end tucked into his armpit. +DUMMY/LJ018-0193.wav|With the police officers it had passed muster as a living member of the party. +DUMMY/LJ016-0176.wav|After Calcraft's resignation no successor was really appointed. +DUMMY/LJ017-0172.wav|but he was subsequently again tried for bigamy, and sentenced to twelve months' imprisonment. +DUMMY/LJ017-0051.wav|covered rather scantily with light sandy hair. +DUMMY/LJ022-0030.wav|and so few people who know anything about what the people of the United States are thinking about. +DUMMY/LJ001-0022.wav|of the more formal ecclesiastical writing which obtained at that time; this has since been called "missal type," +DUMMY/LJ045-0102.wav|things apparently went quite smoothly from the time Oswald returned from Mexico until the weekend of November sixteen to seventeen, nineteen sixty-three. +DUMMY/LJ008-0137.wav|He bowed his head under extreme pressure of ignominy, +DUMMY/LJ027-0155.wav|Now I do not deny that this view is a possible view; but I do deny that it is a probable one. +DUMMY/LJ044-0025.wav|The imaginary president of the nonexistent chapter was named A. J. Hidell, the name that Oswald used when he purchased the assassination weapon. +DUMMY/LJ041-0191.wav|Thornley thought that Oswald not only wanted a place in history but also wanted to live comfortably in the present. +DUMMY/LJ046-0091.wav|Such risks can be lessened when the President recognizes the security problem, +DUMMY/LJ003-0323.wav|Drunkenness, if it ever occurred, should be visited with severe punishment; +DUMMY/LJ004-0072.wav|it was for those charged with the administration of the law to guard carefully against any aggravation of that punishment, +DUMMY/LJ043-0178.wav|as, in fact, one of them did appear after the assassination. +DUMMY/LJ043-0092.wav|which adhered with sufficient precision to the job specifications and as a result too much of his work had to be redone. +DUMMY/LJ014-0229.wav|Mobbs systematically ill-used his wife for a long space of time, and at last cut her throat. +DUMMY/LJ038-0168.wav|There were no eyewitnesses, although a fourteen-year-old boy in a neighboring house claimed that immediately after the shooting +DUMMY/LJ043-0041.wav|Consistent with this attitude +DUMMY/LJ010-0093.wav|and those who watched his house were further encouraged in their mistake by the arrival of many carriages, +DUMMY/LJ043-0083.wav|Employment Commission records concerning Oswald stated, quote, Well-groomed and spoken, +DUMMY/LJ015-0086.wav|He kept open house at Kilburn Priory; +DUMMY/LJ016-0428.wav|Some condemned convicts converse but little with the warders who have them unceasingly in charge. +DUMMY/LJ028-0088.wav|Religion and cruelty frequently go hand in hand, and Nebuchadnezzar was exceedingly religious. +DUMMY/LJ037-0151.wav|Cunningham declared that this weapon fired the four cartridges to the exclusion of all other weapons. +DUMMY/LJ023-0023.wav|and recovery is speeding up to a point where the dangers of nineteen twenty-nine are again becoming possible, +DUMMY/LJ043-0105.wav|that while he did not fire Oswald because of the newspaper incident or even weigh it heavily in his decision, quote, +DUMMY/LJ026-0057.wav|They must first undergo certain preparatory chemical changes known collectively as digestion +DUMMY/LJ030-0161.wav|This represents a speed of eleven point two miles per hour. +DUMMY/LJ035-0206.wav|An eyewitness to the shooting immediately provided a description of the man in the window which was similar to Oswald's actual appearance. +DUMMY/LJ013-0223.wav|Courvoisier wished to commit suicide in Newgate, but was prevented by the vigilant supervision to which he was subjected while in jail. +DUMMY/LJ034-0009.wav|Palmprints and Fingerprints on Cartons and Paper Bag +DUMMY/LJ040-0237.wav|authorities were finally forced to consider placement in a home for boys. +DUMMY/LJ031-0118.wav|The wound on the back of the wrist was left partially open for draining, and the wound on the palm side was enlarged, cleansed, and closed. +DUMMY/LJ025-0063.wav|and with all the appearances of spontaneity as locomotive bodies, +DUMMY/LJ005-0181.wav|The right and privilege of the borough to maintain its own place of confinement was so "ancient and indisputable," +DUMMY/LJ003-0344.wav|all chances of classification and separation vanished, and the greatest evils remained untouched. +DUMMY/LJ009-0275.wav|who bad been further upset by a letter threatening to shoot him when he appeared to perform his task. +DUMMY/LJ033-0025.wav|The two men left work at four:forty p.m. and drove to Irving. +DUMMY/LJ008-0235.wav|Every link in that great human chain is shaken, along the whole lengthened line has the motion jarred, and each in turn sees, +DUMMY/LJ003-0057.wav|A further sub-classification was attempted by separating at night those charged with misdemeanors from those charged with felony, +DUMMY/LJ014-0168.wav|denoting that they came from a concourse of boys and girls already assembled in the best places, made my blood run cold. +DUMMY/LJ040-0102.wav|return to it for lunch and then again at night, his mother having trained him to do that rather than to play with other children. +DUMMY/LJ029-0089.wav|Ultimately more than two hundred law enforcement officers, mainly Dallas police but including eight Secret Service agents, +DUMMY/LJ038-0206.wav|When he failed to return by ten or ten:thirty p.m., Marina Oswald went to his room and discovered the note. She testified: quote, +DUMMY/LJ031-0082.wav|From a medical viewpoint, President Kennedy was alive when he arrived at Parkland Hospital; +DUMMY/LJ033-0026.wav|There was little conversation between them on the way home. +DUMMY/LJ032-0059.wav|Experts on handwriting identification from the Treasury Department and the FBI +DUMMY/LJ018-0373.wav|The mutineers were of the same stamp as the crew of the 'Flowery Land' +DUMMY/LJ044-0115.wav|he felt that this was a great man that he had received the letter from, end quote. +DUMMY/LJ028-0265.wav|When tidings reached Darius of what had happened, +DUMMY/LJ006-0174.wav|The aldermen never called upon him to report, and left him nearly unsupervised and uncontrolled. +DUMMY/LJ029-0185.wav|by Vice President Johnson during the nineteen sixty campaign. +DUMMY/LJ028-0272.wav|All stratagems and all arts had been used, and yet the King could not prevail +DUMMY/LJ028-0096.wav|who reverences their lordship; the untiring governor, who ponders daily concerning the maintenance of Esagil and Ezida, +DUMMY/LJ033-0003.wav|The rifle in the building. +DUMMY/LJ014-0193.wav|Escorted by his two captors, Marley was taken back into Parliament Street to the jeweler's shop. +DUMMY/LJ016-0019.wav|and climbed up the pipe of a cistern in the corner of the press yard; some thought with the idea of drowning himself. +DUMMY/LJ044-0170.wav|He approved arrangements for his family to return to Irving, Texas, to live with Mrs. Ruth Paine. +DUMMY/LJ017-0048.wav|People came to stare at the supposed cold-blooded prisoner; +DUMMY/LJ008-0102.wav|A little further off, says Mr. Smith, was "a lath of a fellow past three-score years and ten, +DUMMY/LJ030-0159.wav|Since the camera operates at a speed of eighteen point three frames per second, +DUMMY/LJ019-0075.wav|that a thief or other criminal might be quite differently treated according as he was incarcerated on one side or another of a border line. +DUMMY/LJ001-0067.wav|In the Low Countries and Cologne, which were very fertile of printed books, Gothic was the favorite. +DUMMY/LJ014-0292.wav|His health improved, but was still indifferent when he was brought up for sentence, and he was an occupant of the Newgate infirmary. +DUMMY/LJ032-0157.wav|was a tuft of several cotton fibers of dark blue, gray-black, and orange-yellow shades. +DUMMY/LJ024-0006.wav|a new member shall be appointed by the President then in office, +DUMMY/LJ040-0162.wav|There, he suggested, Lee should be treated by a male psychiatrist who could substitute for the lack of a father figure. +DUMMY/LJ026-0149.wav|excretions which are lost by osmosis through the roots and the accumulated but useless mineral substances which are eliminated by leaf fall. +DUMMY/LJ048-0030.wav|the officials of the FBI believed that there was no data in its files which gave warning that Oswald was a source of danger to President Kennedy. +DUMMY/LJ034-0132.wav|From the street, this creates the impression that the person is standing. +DUMMY/LJ041-0177.wav|and expressed opposition to the Batista regime and sympathy for Castro, an attitude which, Donovan said, was, quote, +DUMMY/LJ003-0146.wav|without them the keeper declared that he could not pay the salaries of turnkeys and servants, nor keep the prison going at all. +DUMMY/LJ015-0210.wav|The gold, packed in an iron-bound box, was securely lodged in safes locked with patent Chubbs. +DUMMY/LJ018-0042.wav|it was the strongest circumstantial evidence which had ever been brought forward in a murder case. +DUMMY/LJ003-0203.wav|or sought relief in the constant application of intoxicating stimulants. +DUMMY/LJ036-0089.wav|Fritz did not bring him into his office to identify Oswald but turned him over to Lieutenant Baker for questioning. +DUMMY/LJ003-0091.wav|Constantly associated with these convicted felons were numbers of juveniles, infants of tender years. +DUMMY/LJ019-0222.wav|This principle became more and more generally the rule, although it has never been punctiliously observed. Now and again +DUMMY/LJ034-0206.wav|confirmed that Euins could neither describe the man in the window nor indicate his race. +DUMMY/LJ015-0142.wav|He soon proved his ability, and by unremitting attention mastered the whole work of the office. +DUMMY/LJ032-0116.wav|it was examined by Lt. J. C. Day of the identification bureau of the Dallas police. +DUMMY/LJ040-0157.wav|Lee has to be diagnosed as, quote, personality pattern disturbance with schizoid features and passive-aggressive tendencies, end quote. +DUMMY/LJ016-0130.wav|The first intimation of his escape was afforded by the police, who informed the prison authorities next day that a rope was hanging down from the cook-house roof. +DUMMY/LJ011-0109.wav|my heart has undergone that horrible penalty. +DUMMY/LJ036-0175.wav|Arrival and departure from roominghouse. +DUMMY/LJ028-0277.wav|One of his sumpter-mules gave birth to a foal. +DUMMY/LJ041-0192.wav|He testified that if Oswald could not have that, quote, degree of physical comfort that he expected or sought, +DUMMY/LJ047-0119.wav|Neither these discrepancies nor the fact that Oswald had initiated the FBI interview +DUMMY/LJ050-0164.wav|In evaluating data processing techniques of the Secret Service, +DUMMY/LJ026-0156.wav|In the plant also oxygen is absorbed through the epidermis and stomata from the air. +DUMMY/LJ027-0153.wav|in order to be so added to successive species, every individual deer belonging to later species was required to repeat in his own lifetime +DUMMY/LJ028-0498.wav|Herodotus says that it was three hundred and thirty-five feet; +DUMMY/LJ043-0113.wav|Refusing to admit that he could only get work as a greaser, +DUMMY/LJ035-0081.wav|The second test followed immediately after the first. +DUMMY/LJ002-0177.wav|affidavit of debt, drawing praecipe (one pound, five shillings), capias, fee to officer on arrest, +DUMMY/LJ044-0062.wav|thousands, end quote, of circulars, especially since he had claimed to have printed only two thousand +DUMMY/LJ023-0073.wav|The Court claimed the power to declare it unconstitutional and did so declare it. +DUMMY/LJ039-0179.wav|stated that there was a shorter interval between shots two and three than between shots one and two. +DUMMY/LJ022-0014.wav|the creation of a useful instrument for man ultimately comes. +DUMMY/LJ017-0126.wav|He relied on the absence of the strychnia. +DUMMY/LJ001-0084.wav|But for the beauty of the earlier work they might have seemed tolerable. +DUMMY/LJ029-0080.wav|and had certain unattractive features, including a low ceiling with exposed conduits and beams. +DUMMY/LJ004-0113.wav|It was further ordered that male prisoners should be kept perfectly distinct from the females. +DUMMY/LJ016-0299.wav|robbery and violence, loud laughing, oaths, fighting, obscene conduct, and still more filthy language reigned round the gallows far and near. +DUMMY/LJ039-0160.wav|In tests with the Mannlicher-Carano C twenty-seven sixty-six rifle, over one hundred rounds of this ammunition were fired by the FBI +DUMMY/LJ025-0173.wav|The watery solution, in which its roots are plunged, contains nitrogen but no carbon; +DUMMY/LJ004-0201.wav|At Ilchester the rule of employment had been carried further. +DUMMY/LJ010-0287.wav|But this was not sufficient to constitute lunacy, nor was his plea of "momentary uncontrollable impulse" +DUMMY/LJ003-0274.wav|"that there is no place in the metropolis where more crimes are projected or where stolen property is more secreted than in Newgate." +DUMMY/LJ048-0236.wav|There is no indication that any of the agents who visited the Cellar Coffee House had any intoxicating drink at that establishment. +DUMMY/LJ009-0065.wav|and the old clerk lifts up his cracked voice, saying, 'Let us sing to the praise and glory of God.' +DUMMY/LJ023-0077.wav|by which any law is passed, +DUMMY/LJ016-0296.wav|The actual execution made some impression. +DUMMY/LJ049-0206.wav|and while it has arrived at certain conclusions in respect thereto, it seems clear +DUMMY/LJ023-0074.wav|But a little later the Court itself admitted +DUMMY/LJ030-0177.wav|Mrs. Connally, too, heard a frightening noise from her right. +DUMMY/LJ017-0111.wav|that his friend had died greatly embarrassed, with bills to the amount of four thousand pounds out in his name. +DUMMY/LJ016-0356.wav|broke down at the supreme moment before the horror of the stillness, the awful impressiveness of the scene in which he was the principal actor. +DUMMY/LJ006-0233.wav|The prisoners in this way administered serious punishment on one another. Black eyes and broken noses were always to be seen. +DUMMY/LJ047-0071.wav|was advised that Oswald was in contact with the Fair Play for Cuba Committee in New York, and that he had written to the committee +DUMMY/LJ017-0236.wav|Leon, or Lyons, who spoke English, +DUMMY/LJ012-0191.wav|of atrocious murders perpetrated in Edinburgh by the miscreants Burke and Hare, +DUMMY/LJ018-0208.wav|were a low lot, the lowest among criminals except, perhaps, the 'smashers,' or those who passed the counterfeit money. +DUMMY/LJ044-0037.wav|Oswald wrote that an office which he had previously claimed to have rented for FPCC activities had been, quote, +DUMMY/LJ001-0037.wav|and the next year Gunther Zeiner at Augsburg followed suit; +DUMMY/LJ010-0239.wav|Pearson weakly allowed Francis to go off without securing his apprehension, but later he gave full information. +DUMMY/LJ010-0260.wav|in the Mall, about half-way between Buckingham and St. James's Palaces. +DUMMY/LJ032-0137.wav|The lifting had been so complete in this case that there was no trace of the print on the rifle itself when it was examined by Latona. +DUMMY/LJ022-0033.wav|"To get away from the trees", as they say, "and to look at the whole forest." +DUMMY/LJ035-0180.wav|that a witness (Amos Euins) had seen shots fired from a window of the building. +DUMMY/LJ043-0060.wav|from one of the elderly relatives with whom he spoke. Oswald's interest in such things presents a sharp contrast with his attitude +DUMMY/LJ023-0063.wav|But the framers went further. +DUMMY/LJ003-0231.wav|such as Messrs. Birch of Cornhill and Messrs. Leach and Dollimore of Ludgate Hill. +DUMMY/LJ023-0085.wav|to safeguard business against unfair competition, +DUMMY/LJ014-0198.wav|Marley at his trial was undefended, and the sheriffs offered him counsel; but he declined. The witnesses against him all spoke the truth, he said; +DUMMY/LJ035-0192.wav|Although Oswald probably left the building at about twelve:thirty-three p.m., his absence was not noticed until at least one-half hour later. +DUMMY/LJ031-0231.wav|This was finished at approximately four a.m. +DUMMY/LJ041-0186.wav|as some kind of tribunal, +DUMMY/LJ013-0087.wav|Burgess and Elder proceeded in company to cash this, but as they wanted all gold, +DUMMY/LJ021-0044.wav|in the sale of unsound mortgages and in many other ways in which the public lost billions of dollars. +DUMMY/LJ032-0272.wav|(three) fibers found on the rifle most probably came from the shirt Oswald was wearing on the day of the assassination, +DUMMY/LJ039-0121.wav|and he probably didn't have as high a motivation because he was no longer in recruit training and under the care of the drill instructor. +DUMMY/LJ023-0133.wav|judges who will retain in the courts the judicial functions of a court, +DUMMY/LJ010-0189.wav|He asked more than once whether the Queen was hurt, and acknowledged that the pistols were loaded with ball. +DUMMY/LJ016-0316.wav|based on his experience of them in Western Australia. He not only thought them likely to be more deterrent, +DUMMY/LJ043-0027.wav|De Mohrenschildt said that the whole affair made him nervous since he was, quote, interfering in other people's affairs, after all, end quote. +DUMMY/LJ002-0215.wav|It was so used for the date of the abolition of the Star Chamber in the sixteenth Charles the first +DUMMY/LJ042-0195.wav|and yet this doctrine requires the utmost, utmost restraint, a state of being in itself majestic in power, end quote. +DUMMY/LJ016-0414.wav|Rumour even went so far as to assert that among the spectators were several women, disguised in male habiliments; +DUMMY/LJ050-0037.wav|Preventive Intelligence +DUMMY/LJ040-0056.wav|It will attempt to summarize the events of his early life, his experience in New York City and in the Marine Corps, and his interest in Marxism. +DUMMY/LJ031-0076.wav|This was verified by Admiral Burkley, the President's physician, who arrived at the hospital after emergency treatment was underway and concluded that, quote, +DUMMY/LJ004-0240.wav|He left Newgate utterly corrupted, and after lapsing into crime, soon returned with a very different character. +DUMMY/LJ046-0197.wav|to a particular part of the country. These were the files reviewed by PRS on November eight, nineteen sixty-three, +DUMMY/LJ004-0190.wav|There were eighty-four separate sleeping-cells, and unless the jail was overcrowded, every inmate passed the night alone, +DUMMY/LJ019-0155.wav|The enforcement of discipline depended upon the want of honor among thieves. +DUMMY/LJ008-0218.wav|They were mainly members of the criminal classes. +DUMMY/LJ008-0079.wav|She walked from the debtors' door to a stake fixed in the ground about half-way between the scaffold and Newgate Street. +DUMMY/LJ047-0230.wav|Hosty testified that he would not have cited him to the Secret Service as a potential threat to the President. Hosty interpreted his instructions as requiring, quote, +DUMMY/LJ038-0055.wav|No other theatre patron or officer has testified that Oswald was hit by a gun. +DUMMY/LJ001-0044.wav|John of Spires and his brother Vindelin, followed by Nicholas Jenson, began to print in that city, +DUMMY/LJ005-0061.wav|but it offered suggestions for future buildings, with numerous carefully-executed drawings and designs of the model it recommended for imitation. +DUMMY/LJ031-0059.wav|Dr. Carrico treated the President's known ad-renal insufficiency by administering hydrocortisone. +DUMMY/LJ032-0266.wav|Paul M. Stombaugh, of the FBI Laboratory, +DUMMY/LJ019-0225.wav|In eighteen fifty-seven +DUMMY/LJ031-0180.wav|At approximately two:fifteen p.m. the casket was loaded, with some difficulty because of the narrow airplane door, +DUMMY/LJ026-0143.wav|Probably foods containing carbon, hydrogen and oxygen are the sources of energy in the higher plants as in animals. +DUMMY/LJ027-0084.wav|Romanes's "Darwin and After Darwin", and Le Conte's "Evolution." +DUMMY/LJ014-0333.wav|Again, some of the duplicate and fictitious warrants were held by a firm which suspended payment, and there was no knowing into whose hands they might fall. +DUMMY/LJ045-0068.wav|The difficulties which Oswald's problems would have caused him in any relationship were probably not reduced by his wife's conduct. +DUMMY/LJ016-0350.wav|and who expiated his crime on the eighth September, eighteen sixty-eight. +DUMMY/LJ014-0156.wav|and through all ate heartily at every meal, slept soundly at nights, and talked with cheerfulness on almost any subject. +DUMMY/LJ018-0269.wav|from an assistant of Loudon and Ryder's, the jewelers in Bond Street. The trick was an old one. +DUMMY/LJ009-0058.wav|Great efforts have been made to save his life. +DUMMY/LJ008-0012.wav|partly perhaps because residents in and about the Old Bailey raised a loud protest against the constant erection of the scaffold in their neighborhood. +DUMMY/LJ013-0112.wav|Within a month or two the bank of Messrs. Rogers and Co., Clement's Lane, was broken into. +DUMMY/LJ008-0004.wav|The reasons for this change were fully set forth in a previous chapter. +DUMMY/LJ021-0210.wav|to greater security for the average man than he has ever known before in the history of America. +DUMMY/LJ009-0171.wav|Gradually the application of capital punishment became more and more restricted, and was soon the penalty for murder alone. +DUMMY/LJ017-0037.wav|under the authority of an Act passed on purpose, known as the Trial of offenses Act, and sometimes as Lord Campbell's Act. +DUMMY/LJ040-0105.wav|Despite their urgings, he refused to play with the other children his own age. +DUMMY/LJ019-0260.wav|At other prisons "shot-drill," the lifting and carrying of heavy round shot, was the favorite method of inflicting penal labor. +DUMMY/LJ011-0078.wav|Montgomery was an adept at forgery. He had gone wrong early. Although born of respectable parents, and gazetted to a commission in the army, +DUMMY/LJ009-0115.wav|He says, "On the day of execution there is no service in the chapel of Newgate. +DUMMY/LJ034-0216.wav|In the background of this picture were several employees watching the parade from the steps of the Depository Building. +DUMMY/LJ038-0275.wav|and concluded, quote, that there is a fair probability, end quote, that the bullet was fired from the rifle used in the assassination of President Kennedy. +DUMMY/LJ019-0106.wav|His plan was to devote the whole labor of prisoners sentenced to any term between three months and four years to agriculture. +DUMMY/LJ036-0079.wav|coming from the direction of the Depository Building and running down the hill north of Elm Street +DUMMY/LJ040-0206.wav|that Lee's withdrawal was a form of, quote, violent but silent protest against his neglect by her +DUMMY/LJ030-0183.wav|As he told the driver, quote, Let's get out of here; we are hit, end quote, +DUMMY/LJ006-0208.wav|There was also a good supply of Bibles and prayers, +DUMMY/LJ028-0187.wav|Cyrus decided that Babylon must be taken. +DUMMY/LJ002-0128.wav|He also makes the curious calculation that the costs of these actions if undefended +DUMMY/LJ016-0244.wav|who first took to the work from predilection, and the idea of being useful in his generation, as he himself assured the writer of these pages. +DUMMY/LJ006-0028.wav|Mr. Crawford was thoroughly versed in the still imperfectly understood science of prison management, and fully qualified for his new duties. +DUMMY/LJ020-0042.wav|and these also tend to nourish and strengthen the brain. +DUMMY/LJ007-0066.wav|on the other, the governor of the jail sneered at his zeal +DUMMY/LJ040-0228.wav|Despite his withdrawal, he gives the impression that he is not so difficult to reach as he appears and patient, prolonged effort +DUMMY/LJ016-0040.wav|His first task was to gain the top of the cistern; this he effected by keeping his back to one side of the angle, +DUMMY/LJ047-0137.wav|On October three, Hosty reopened the case in Dallas to assist the New Orleans office. +DUMMY/LJ004-0146.wav|there was no infirmary, no chapel, no work, no classification. +DUMMY/LJ039-0041.wav|I am getting a little confused with so many questions. +DUMMY/LJ044-0107.wav|and advised them of some of his activities on behalf of the organization. +DUMMY/LJ019-0011.wav|had appointed a committee of prison inspectors, presided over by the Under Secretary of State, to draw up rules and dietaries, +DUMMY/LJ027-0111.wav|usually, indeed, a considerable number. +DUMMY/LJ023-0132.wav|But, at the same time, we must have judges who will bring to the courts a present-day sense of the Constitution +DUMMY/LJ016-0324.wav|But it is curious to note that there were several dissentients among the commissioners to this paragraph of the report. +DUMMY/LJ031-0145.wav|On the recommendation of the Secret Service agents, Vice President Johnson decided to board the Presidential airplane, Air Force One, +DUMMY/LJ014-0075.wav|He was dressed with particular care, as he was to dine at the Mannings and meet friends, one a young lady. +DUMMY/LJ004-0086.wav|of "poor and needy prisoners committed to the common jail for felony and other misdemeanors, who many times perish before their trial; +DUMMY/LJ029-0190.wav|Mayor Earle Cabell called on the city to redeem itself during President Kennedy's visit. +DUMMY/LJ044-0161.wav|In his wife's words, quote, +DUMMY/LJ003-0181.wav|There was a master's side for females who could pay the usual fees, but they associated with the rest in the one narrow yard common to all. +DUMMY/LJ009-0039.wav|Those who sat in it were visible to the whole congregation, and still more to the ordinary, +DUMMY/LJ013-0245.wav|At the constable's cry of alarm Good rushed from the stable and locked the door behind him. +DUMMY/LJ038-0099.wav|From the outset, Oswald denied owning a rifle. +DUMMY/LJ036-0143.wav|He marked what, he thought was the intersection of Neches and Beckley on a map of Dallas with a large "X." +DUMMY/LJ031-0071.wav|described a large, gaping wound in the right rear part of the head, with substantial damage and exposure of brain tissue, and a considerable loss of blood. +DUMMY/LJ013-0061.wav|who was the possessor of two separate sums in consols, neglected through strange carelessness on her own part and that of her friends +DUMMY/LJ046-0192.wav|it regarded approximately one hundred of these four hundred cases as serious risks +DUMMY/LJ021-0090.wav|suited to the needs of the next phase, which is, in turn, a period of preparation for legislation which will determine its permanent form. +DUMMY/LJ014-0113.wav|advised the Edinburgh police of the whole affair, and within an hour an answer was telegraphed, stating that Mrs. Manning was in custody. +DUMMY/LJ038-0135.wav|The Curtain Rod Story +DUMMY/LJ038-0231.wav|In her testimony before the Commission in February nineteen sixty-four, +DUMMY/LJ042-0089.wav|I leave Embassy, elated at this showdown, returning to my hotel I feel now my energies are not spent in vain. +DUMMY/LJ007-0180.wav|The publication of this report raised a storm in the city, and the corporation was roused to make an immediate protest. +DUMMY/LJ030-0205.wav|He estimated that the motorcade had slowed down to approximately nine or ten miles per hour +DUMMY/LJ018-0266.wav|was at length advanced to the comparative ease of a post much coveted by convicts, that of hospital nurse. +DUMMY/LJ018-0280.wav|The commercial experience of these clever rogues was cosmopolitan. +DUMMY/LJ006-0173.wav|He kept no daily journal of occurrences, and nothing, however important, was recorded at the time. +DUMMY/LJ040-0184.wav|He appears slightly withdrawn and in view of the lack of detail within the drawings this may assume a more significant characteristic. +DUMMY/LJ019-0173.wav|The Corporation seems to have introduced these salutary changes without hesitation. +DUMMY/LJ032-0048.wav|the scope was mounted on the rifle by a gunsmith employed by Klein's, and the rifle was shipped fully assembled in accordance with customary company procedures. +DUMMY/LJ019-0043.wav|Among the latter were notably the counties of Cheshire, Lincolnshire, Norfolk, Suffolk, Nottinghamshire, the East and North Ridings of Yorkshire. +DUMMY/LJ015-0279.wav|One stroke of luck which he turned to great account +DUMMY/LJ001-0124.wav|One of the differences between the fine type and the utilitarian must probably be put down to a misapprehension of a commercial necessity: +DUMMY/LJ028-0153.wav|The bitumen used in the work was brought to Babylon from Is, a small stream which flows into the Euphrates +DUMMY/LJ024-0062.wav|President Taft appointed five members and named a Chief Justice; +DUMMY/LJ034-0113.wav|The autopsy report indicated an estimated weight of one hundred fifty pounds. +DUMMY/LJ050-0154.wav|to plan and develop a workable and efficient automated file and retrieval system, end quote. +DUMMY/LJ045-0212.wav|this time concerning his use of an alias. +DUMMY/LJ038-0185.wav|This is the same street where the drugstore, in which you always waited is located. +DUMMY/LJ015-0207.wav|But the collusion and active assistance of employees of the railway carriers were indispensable, and together +DUMMY/LJ048-0060.wav|required further attention. +DUMMY/LJ040-0010.wav|The motives of any man, however, must be analyzed in terms of the character and state of mind of the particular individual involved. +DUMMY/LJ046-0128.wav|while the balance was furnished to PRS by other Federal agencies, with primary source being the FBI. +DUMMY/LJ037-0122.wav|The Dallas Police Department furnished the Commission with pictures of the men who appeared in the lineups with Oswald, +DUMMY/LJ009-0013.wav|and kept altogether separate from the other prisoners until the day of his death. +DUMMY/LJ019-0362.wav|Where the local authority had neglected to comply with the provisions of the eighteen sixty-five Act for four consecutive years, +DUMMY/LJ033-0171.wav|I found that the paper sack found on the sixth floor and the sample +DUMMY/LJ003-0200.wav|Visitors were permitted access to them without stint; +DUMMY/LJ005-0188.wav|that the jails attached to corporate jurisdictions continue to be the fruitful sources +DUMMY/LJ040-0151.wav|He was also described as having a, quote, Vivid fantasy life, +DUMMY/LJ005-0151.wav|in others women were very properly exempted from it, and also from all severe labor. +DUMMY/LJ042-0023.wav|and that he went there, quote, only to find freedom. I could never have been personally happy in the U.S., end quote. +DUMMY/LJ010-0068.wav|he became a militia officer, and married a woman with ten thousand pounds, in which, however, she had only a life interest. +DUMMY/LJ032-0208.wav|He found, quote, one notch in the stock at this point that appears very faintly in the photograph, end quote. +DUMMY/LJ011-0207.wav|whom I never saw until I was taken from Liverpool, and never want to see again. +DUMMY/LJ006-0045.wav|All these evils were set forth at length in the inspectors' first report. +DUMMY/LJ039-0126.wav|As compared to a civilian who had not received this intensive training, he would be considered as a good to excellent shot. End quote. +DUMMY/LJ041-0129.wav|he continued to be concerned about his undesirable discharge. It is clear that he thought he had been unjustly treated. +DUMMY/LJ043-0050.wav|As Marguerite Oswald testified, quote, +DUMMY/LJ039-0175.wav|For the first four attempts, the firers missed the second shot by several inches. +DUMMY/LJ032-0183.wav|The relative freshness of the fibers is strong evidence that they were caught on the rifle on the morning of the assassination or during the preceding evening. +DUMMY/LJ029-0040.wav|Lawson received a tentative schedule of the Texas trip on November eight from Roy H. Kellerman, assistant special agent in charge of the White House detail, +DUMMY/LJ001-0010.wav|Now, as all books not primarily intended as picture-books consist principally of types composed to form letterpress, +DUMMY/LJ012-0256.wav|and there was now evidence sufficient to establish conclusively that the woman thus collected piecemeal had been barbarously done to death. +DUMMY/LJ035-0049.wav|All I did was catch a glance at him, and evidently he was -- this door might have been, you know, closing and almost shut at that time, end quote. +DUMMY/LJ038-0220.wav|Oswald had paid the first month's rent in advance on March two, nineteen sixty-three, +DUMMY/LJ001-0041.wav|It must be said that it is in no way like the transition type of Subiaco, +DUMMY/LJ050-0228.wav|The Commission urges that the Bureau of the Budget review these recommendations with the Secret Service and authorize a request for the necessary supplemental appropriation, +DUMMY/LJ020-0101.wav|From the beginning of your apprenticeship in housewifery, learn how to "dovetail" your duties neatly into one another. +DUMMY/LJ003-0207.wav|For while Roman Catholics and Dissenters were encouraged to see ministers of their own persuasion, +DUMMY/LJ010-0096.wav|Edgeware Road, completing their dispositions for assuming supreme power after the blow had been struck. +DUMMY/LJ050-0193.wav|Such instructions will not fit all circumstances, of course, +DUMMY/LJ014-0222.wav|To Cannon's infinite surprise, he was indicted for attempt to murder, and not for a simple assault, and found guilty. +DUMMY/LJ016-0146.wav|from the building he was on to a distant wall, with a drop of sixty feet between. +DUMMY/LJ017-0087.wav|antimony, which caused Cook's constant sickness. +DUMMY/LJ030-0049.wav|Following normal practice, the lead automobile stayed approximately four to five car lengths ahead of the President's limousine. +DUMMY/LJ019-0066.wav|with the means of earning an honest livelihood if so disposed. +DUMMY/LJ029-0007.wav|this chapter reviews the motorcade through Dallas, the fleeting moments of the assassination, +DUMMY/LJ001-0162.wav|even allowing the necessity for a very low price; but any improvement must be based on showing openly that the cheap article is cheap, +DUMMY/LJ012-0099.wav|The thieves it was supposed had secreted themselves in the warehouse during business hours, and waited till night to carry out their plans. +DUMMY/LJ011-0066.wav|The concourse in front of Newgate was enormous, but much sympathy was evinced for this unfortunate victim to human weakness and ruthless laws. +DUMMY/LJ026-0053.wav|or animal matter in the form of various organic compounds, of which the most important are proteids (protoplasm, albumen, etc.), +DUMMY/LJ039-0238.wav|that it was an easy shot to hit some part of the President's body, and that the range where the rifleman would be expected to hit +DUMMY/LJ026-0087.wav|In this way some of the radiant and kinetic energy of the sun comes to be stored up as potential energy in the starch. +DUMMY/LJ009-0066.wav|The morning hymn is sung first, as if to remind the condemned that next morning at eight a.m. they are to die. +DUMMY/LJ017-0046.wav|Great personages occupied seats upon the bench; +DUMMY/LJ026-0093.wav|The plant absorbs also a small amount of kinetic energy, independently of the sunlight, in the form of heat. +DUMMY/LJ002-0019.wav|who seldom let a session go by without visiting Newgate. +DUMMY/LJ031-0090.wav|A thorough inspection would have involved washing and cleansing the back, and this is not practical in treating an acutely injured patient. +DUMMY/LJ030-0216.wav|Between the time he originally seized the handhold and the time he mounted the car, Hill recalled: +DUMMY/LJ036-0207.wav|The suspect was described as a, quote, +DUMMY/LJ003-0346.wav|It pointed out that the Government was to blame for the overcrowding, and might diminish it if it chose. +DUMMY/LJ039-0053.wav|was of no probative value in the Commission's decision concerning the identity of the assassin of President Kennedy. +DUMMY/LJ018-0312.wav|A plot was soon discovered, +DUMMY/LJ022-0035.wav|is one which, in a very special manner, attaches to this office to which you have chosen me. +DUMMY/LJ048-0175.wav|Nor is it practical to prevent people from entering such buildings, or to limit access in every building to those employed or having business there. +DUMMY/LJ035-0036.wav|On the second-floor landing there is a small open area with a door at the east end. +DUMMY/LJ002-0304.wav|The sheriff demanded four shillings, six pence for his liberate, the jailer six shillings, ten pence more, and the turnkey two shillings; +DUMMY/LJ034-0083.wav|The significance of Givens' observation that Oswald was carrying his clipboard +DUMMY/LJ039-0005.wav|Richard M. Nixon Incident +DUMMY/LJ004-0233.wav|Under the new rule visitors were not allowed to pass into the interior of the prison, but were detained between the grating. +DUMMY/LJ022-0104.wav|and the experience and the competence necessary to carry on the two hundred and fifty or three hundred kinds of work that will be undertaken. +DUMMY/LJ028-0074.wav|But in Babylonia stone was difficult to obtain, and sculptures were very rare. +DUMMY/LJ035-0094.wav|the Commission concluded that Oswald could have fired the shots and still have been present in the second-floor lunchroom when seen by Baker and Truly. +DUMMY/LJ008-0177.wav|The catastrophe exceeded the worst anticipations. Nearly one hundred dead and dying lay about; and after all had been removed, +DUMMY/LJ033-0172.wav|had the same observable characteristics both under the microscope and all the visual tests that I could conduct. +DUMMY/LJ019-0352.wav|or hesitated to embark upon any considerable expenditure to alter or rebuild, +DUMMY/LJ048-0087.wav|The President's trip to Dallas called into play many standard operating procedures of the Secret Service in addition to its preventive intelligence operations. +DUMMY/LJ043-0143.wav|with his recently acquired rifle and pistol, a copy of the March twenty-four, nineteen sixty-three, issue of the Worker, +DUMMY/LJ026-0132.wav|after absorption into the cells the elements of the starch (or glucose) are, by the living protoplasm, in some unknown way +DUMMY/LJ047-0247.wav|Hosty has unequivocally denied, first by affidavit and then in his testimony before the Commission, +DUMMY/LJ011-0186.wav|by the blacksmith in the usual way. +DUMMY/LJ048-0027.wav|Nowhere during the course of this investigation or the information that came to us from other agencies was there any indication of a potential for violence on his part. +DUMMY/LJ012-0058.wav|When the journey was resumed, Mrs. Solomons accompanied her husband in the coach. Half-way to Newgate she was taken with a fit. +DUMMY/LJ001-0071.wav|Most of Caxton's own types are of an earlier character, though they also much resemble Flemish or Cologne letter. +DUMMY/LJ048-0142.wav|Despite this dependence on local authorities, which would be substantially the same on a visit by the President to any large city, +DUMMY/LJ014-0151.wav|Later her mood changed to abject despair. +DUMMY/LJ050-0107.wav|Liaison with other agencies regarding intelligence. +DUMMY/LJ043-0017.wav|Marina had a hundred dresses given to her, and he objected to that lavish help, because Marina was throwing it into his face. +DUMMY/LJ005-0166.wav|Mr. Peel, who as Home Secretary had charge of the bill, +DUMMY/LJ030-0212.wav|At about the time he reached the President's automobile, +DUMMY/LJ005-0297.wav|They therefore recommended that the prisoners should be removed +DUMMY/LJ005-0210.wav|At Walsall, in Staffordshire, +DUMMY/LJ050-0173.wav|that coordination might be achieved to a greater extent than seems now to be contemplated, without interference with the primary mission of each agency involved. +DUMMY/LJ018-0084.wav|While in Newgate awaiting trial he frequently tried to justify his murder by declaring that the police officer had broken faith with him. +DUMMY/LJ012-0245.wav|in the hope that the features might some day be recognized, were alike in both. +DUMMY/LJ028-0045.wav|When Esarhaddon died, one of his sons, Samas-sum-yukin, was made King of Babylon. +DUMMY/LJ038-0097.wav|the Commission gave little weight to his denials of guilt. +DUMMY/LJ042-0176.wav|One offers oppression, the other poverty. Both offer imperialistic injustice, tinted with two brands of slavery, end quote. +DUMMY/LJ042-0041.wav|unemployment, automation, and the use of military forces to suppress other populations. +DUMMY/LJ041-0187.wav|and he wanted to be on the winning side so that ten thousand years from-now people would look in the history books and say, "Well, this man was ahead of his time." +DUMMY/LJ030-0139.wav|toward the Triple Underpass, shots fired from a rifle mortally wounded President Kennedy and seriously injured Governor Connally. +DUMMY/LJ033-0175.wav|Mr. Cadigan concluded that the paper and tape from the bag were identical in all respects to the sample paper and tape +DUMMY/LJ002-0094.wav|with four casements and two fireplaces, being allotted for a female infirmary +DUMMY/LJ018-0189.wav|The officers rushed out to detain him. "What have you got here? We must search the cart," they cry. +DUMMY/LJ031-0050.wav|Observing the serious problems presented by the head wound and inadequate respiration, Dr. Carrico directed his attention to improving the President's breathing. +DUMMY/LJ042-0166.wav|The latter material expresses great hostility to both communism and capitalism. He wrote, that to a person knowing both of those systems, quote, +DUMMY/LJ044-0131.wav|Oswald had fallen on difficult times. +DUMMY/LJ002-0227.wav|absence or neglect of divine service, were present as in the King's Bench, but in an exaggerated form. +DUMMY/LJ032-0237.wav|it was established that the photographs must have been taken sometime after March twenty-seven. +DUMMY/LJ039-0082.wav|this is the ideal type of weapon for moving targets +DUMMY/LJ023-0003.wav|Last Thursday I described in detail certain economic problems which everyone admits now face the nation. +DUMMY/LJ004-0244.wav|A few fortunate criminals, many of them kept back from transportation on purpose, who were skilled in trades, were employed at them. +DUMMY/LJ010-0314.wav|In order to keep up the credit of our house, I have forged powers of attorney for the above sums and parties, +DUMMY/LJ048-0086.wav|Other Protective Measures and Aspects of Secret Service Performance +DUMMY/LJ038-0072.wav|who had driven from the theatre with Oswald. +DUMMY/LJ028-0371.wav|Then the world metropolis, stripped of most of its population, became a mere village. +DUMMY/LJ046-0224.wav|The State Department advised the Secret Service of all crank and threat letter mail +DUMMY/LJ014-0024.wav|after several remands; but the case contained elements of doubt, and the evidence at the trial was entirely circumstantial. +DUMMY/LJ022-0076.wav|and we have every reason to believe that it should be in full swing by autumn. +DUMMY/LJ017-0078.wav|Palmer tried to get a new insurance on the life of a hanger-on, one Bates, but no office would accept it, no doubt greatly to Bates's longevity. +DUMMY/LJ049-0025.wav|Access to passenger compartment of Presidential car. +DUMMY/LJ008-0298.wav|Sometimes the report was delayed. +DUMMY/LJ002-0208.wav|into holes marked with numbers. +DUMMY/LJ037-0175.wav|Therefore, one cartridge case of this type was not recovered. +DUMMY/LJ015-0243.wav|At Redhill Tester met the train and relieved the thieves of a portion of the stolen gold. +DUMMY/LJ010-0272.wav|This son, John William Bean, was fully identified by Dasset, and presently examined by the Privy Council. +DUMMY/LJ004-0111.wav|one. Prisoners convicted of felony. two. Prisoners committed on a charge or suspicion of felony. +DUMMY/LJ028-0110.wav|I completed Imgur-Bel and Nimitti-Bel, the great walls of Babylon, the mighty city, the city of his exalted power. +DUMMY/LJ017-0169.wav|The Home Secretary, in a letter to the Lord Chief Baron, stated that, quote, although the facts are full of suspicion against Smethurst, +DUMMY/LJ005-0102.wav|Upon these and the private visitations made by various members the Society obtained the facts, +DUMMY/LJ015-0186.wav|and received his sentence of transportation for life without much surprise. +DUMMY/LJ026-0058.wav|and only after the completion of this process can all the food be absorbed into the circulation. +DUMMY/LJ038-0051.wav|Two patrons of the theatre and John Brewer +DUMMY/LJ019-0189.wav|This plan was on the principle of cellular separation, a system, according to Lord John Russell, desirable in all prisons, +DUMMY/LJ030-0102.wav|was in a car following those, quote, containing the local and national representatives, end quote, +DUMMY/LJ049-0091.wav|Equally anomalous are statutory provisions which specifically authorize the Secret Service to protect the President, +DUMMY/LJ029-0055.wav|Lawson conferred with the local police and the local office of the Federal Bureau of Investigation about potential dangers to the President. +DUMMY/LJ019-0039.wav|At Pentonville the rate was higher, but there the expense had been increased by the site, +DUMMY/LJ038-0145.wav|In one interview, he told Fritz that the only sack he carried to work that day +DUMMY/LJ024-0104.wav|I am therefore, going to spend my time, my efforts and my money +DUMMY/LJ013-0076.wav|and held all the threads of these intricate and nefarious transactions. +DUMMY/LJ048-0178.wav|In accordance with its regular procedures, no survey or other check was made by the Secret Service, or by any other law enforcement agency at its request, +DUMMY/LJ014-0051.wav|There was an epidemic of murder in the United Kingdom about eighteen forty-eight to nine. +DUMMY/LJ046-0078.wav|The protection of the President must be thorough but inconspicuous to avoid even the suggestion of a garrison state. +DUMMY/LJ026-0110.wav|However, the result in distributed food is the same in the plant and in the animal. +DUMMY/LJ001-0083.wav|The seventeenth century founts were bad rather negatively than positively. +DUMMY/LJ008-0118.wav|"Thank you, sir," said the governor to the doctor, "it is of little moment." +DUMMY/LJ030-0071.wav|They provided some cover for the President, but their main purpose was to keep back the crowd. +DUMMY/LJ005-0010.wav|the germs, more or less developed, of contagious disease. "Caravans," the forerunners of the prison vans, +DUMMY/LJ031-0024.wav|Gene C. Akin; +DUMMY/LJ022-0078.wav|one. The projects should be useful. +DUMMY/LJ042-0165.wav|which appears to be more an expression of his own psychological condition than of a reasoned analysis. +DUMMY/LJ042-0048.wav|Happiness is taking part in the struggle, where there is no borderline between one's own personal world, and the world in general. +DUMMY/LJ042-0006.wav|he almost immediately left for the Soviet Union where he attempted to renounce his citizenship. +DUMMY/LJ005-0124.wav|Most of the local authorities embarked into considerable expenditure, determined to rebuild their jails de novo on the most approved pattern, +DUMMY/LJ030-0064.wav|President Kennedy rode on the right-hand side of the rear seat with Mrs. Kennedy on his left. +DUMMY/LJ033-0058.wav|On the day of the assassination, Marina Oswald was watching television when she learned of the shooting. +DUMMY/LJ031-0117.wav|assisted by Drs. William Osborne and John Parker. +DUMMY/LJ022-0192.wav|They contemplate the enrichment of our national life +DUMMY/LJ031-0210.wav|The autopsy report noted that President Kennedy was forty-six years of age, seventy-two and one half inches tall, +DUMMY/LJ013-0054.wav|then brought an action against the administrators, and at the trial Dr. Bailey swore that the note had been given him by Smith. +DUMMY/LJ003-0160.wav|and no lodger was rejected, whatever his status, who offered himself and could bring grist to the mill. +DUMMY/LJ008-0194.wav|Some persons were kept for more than five hours standing against the barriers, and many nearly fainted from exhaustion. +DUMMY/LJ034-0218.wav|The Commission has determined that the employee was in fact Billy Lovelady, who identified himself in the picture. +DUMMY/LJ016-0409.wav|On the fatal morning he came gaily out of his cell, +DUMMY/LJ019-0035.wav|and the total number of separate cells provided amounted to eleven thousand odd. +DUMMY/LJ012-0053.wav|He was conveyed in a coach driven by a confederate, and under the escort of a couple of turnkeys. +DUMMY/LJ016-0053.wav|If the worst came to the worst, he intended to have gone down some chimney, as he had often done before in the way of business. +DUMMY/LJ028-0414.wav|All the ground on which Babylon was spread is left now desolate; nothing standing in that Peninsula between the Euphrates and the Tigris, +DUMMY/LJ008-0082.wav|A chain fastened by nails to the stake was then put round her body by the executioner with his assistants. +DUMMY/LJ046-0245.wav|Other agencies occasionally provided information to the Secret Service concerning potentially dangerous political groups. +DUMMY/LJ006-0220.wav|Feasting alternated with fighting. +DUMMY/LJ006-0252.wav|The worst fights occurred on Sunday afternoons; but nearly every night the act of locking up became, from the consequent removal of all supervision, +DUMMY/LJ028-0056.wav|Nebuchadnezzar, or Nebuchadrezzar, as his name should be spelled, was the greatest character in Babylonian history, +DUMMY/LJ016-0216.wav|That very ancient perquisite the convict's clothes was never claimed by Calcraft, and it may be doubted whether he was entitled to it. +DUMMY/LJ030-0198.wav|but then she noticed an almost imperceptible movement and knew that he was still alive. She said, quote, It's all right. Be still, end quote, +DUMMY/LJ034-0100.wav|this man I saw previously was aiming for his last shot. As it appeared to me he was standing up and resting against the left window sill, end quote. +DUMMY/LJ049-0227.wav|Without, therefore, coming to final conclusions respecting the long-range organization of the President's security, +DUMMY/LJ050-0270.wav|This Commission can recommend no procedures for the future protection of our Presidents which will guarantee security. +DUMMY/LJ011-0160.wav|and a servant hurriedly alighted from it, bearing a letter for Miss Turner. +DUMMY/LJ019-0243.wav|and thus bring the history of prison discipline down to our own times. +DUMMY/LJ004-0238.wav|Mr. Buxton mentions the case of a boy whose apparent innocence and artlessness had attracted his attention. +DUMMY/LJ049-0191.wav|is properly manned and equipped to carry on extensive information gathering functions within the United States. +DUMMY/LJ048-0205.wav|They had, however, occasionally observed the windows of buildings in the area before the motorcade arrived, in accordance with their own understanding of their function. +DUMMY/LJ031-0068.wav|While these medical efforts were in progress, +DUMMY/LJ005-0116.wav|but an empty regulation which all so disposed could defy. +DUMMY/LJ019-0298.wav|The total want of administration was very marked, +DUMMY/LJ040-0190.wav|he had better than average ability in arithmetical reasoning for his age group. +DUMMY/LJ005-0189.wav|of vice and misery, debasing all who are confined within their walls, and disseminating through their respective communities +DUMMY/LJ011-0222.wav|He also wrote and published a pamphlet from the jail to show that Miss Turner had been a consenting party to the marriage, and was really his wife. +DUMMY/LJ032-0151.wav|Experts testifying before the Commission agreed that palmprints are as unique as fingerprints for purposes of establishing identification. +DUMMY/LJ019-0085.wav|the prisoners might be supervised and watched at every step, and made liable to punishment for a trifling breach of an irksome code of regulations, +DUMMY/LJ047-0234.wav|Hosty's initial reaction on hearing that Oswald was a suspect in the assassination, was, quote, shock +DUMMY/LJ028-0293.wav|Zopyrus answered, "There is not a man in the world, but thou, O King, that could reduce me to such a plight +DUMMY/LJ010-0266.wav|and a long, sickly, pale face, with light hair; +DUMMY/LJ036-0134.wav|I asked him where he wanted to go. And he said, "five hundred North Beckley. Well, I started up, +DUMMY/LJ009-0273.wav|two supporting his body and two his legs. +DUMMY/LJ011-0266.wav|The money went after her to her children. +DUMMY/LJ014-0158.wav|She was discovered just as she was getting black in the face. +DUMMY/LJ004-0170.wav|The only ventilation of this pit, this "dark, cheerless, damp, unwholesome cavern -- a dungeon in its worst sense" +DUMMY/LJ031-0233.wav|The President's body was taken to the East Room of the White House where it was placed under ceremonial military guard. +DUMMY/LJ029-0039.wav|special agent in charge of the Dallas office. Both agents were advised of the trip on November four. +DUMMY/LJ017-0235.wav|He was soon summoned on deck, but as he would not move, the mutineers came down and stood in a circle round his berth. +DUMMY/LJ002-0130.wav|two hundred eighty-five thousand, nine hundred fifty pounds; in other words, that to recover eighty odd thousand pounds, +DUMMY/LJ028-0463.wav|The inner part of the wall was constructed of unburned bricks, and at a distance of about thirty-six feet from the outer part. +DUMMY/LJ019-0236.wav|the architectural records of the prison end. Nothing much was done at Newgate in the way of building, outside or in, after eighteen sixty-two. +DUMMY/LJ043-0106.wav|it didn't do his case any good, end quote. +DUMMY/LJ041-0136.wav|was shooting at Connally rather than President Kennedy, end quote. +DUMMY/LJ028-0001.wav|The Seven Wonders of the Ancient World. By Edgar J. Banks. Chapter two. The Walls of Babylon. +DUMMY/LJ015-0241.wav|opened it, took out and broke into the bullion box, removed the gold, substituted the shot from a carpet bag, +DUMMY/LJ043-0037.wav|At the time of his defection, Oswald had said that neither his brother, Robert, +DUMMY/LJ032-0205.wav|photographed the rifle used in the assassination, attempting to duplicate the position of the rifle and the lighting in Exhibit Number one thirty-three A. +DUMMY/LJ039-0129.wav|firing it, with ten rounds. +DUMMY/LJ013-0101.wav|Forrester next traced the fugitives to Liverpool, +DUMMY/LJ030-0222.wav|Mrs. Kennedy had no recollection of climbing onto the back of the car. +DUMMY/LJ021-0075.wav|But it is an undeniable fact that the restoration of other billions of sound investments to a reasonable earning power +DUMMY/LJ005-0096.wav|It had now a legal and authoritative standard of efficiency to apply, +DUMMY/LJ036-0151.wav|the point at which he said his passenger alighted. +DUMMY/LJ035-0191.wav|found no police officers at the rear door and was able to enter through this door without identifying himself. +DUMMY/LJ018-0243.wav|the hardship to the holders of these lands being plain, should the allegations of invalidity be made good. +DUMMY/LJ039-0153.wav|that the assassin in all probability hit two out of the three shots during the maximum time span of +DUMMY/LJ036-0068.wav|Both buses stopped within one block of the Depository Building. +DUMMY/LJ033-0077.wav|Lee Harvey Oswald left the Paine house in Irving at approximately seven:fifteen a.m., while Marina Oswald was still in bed. +DUMMY/LJ018-0323.wav|A year or two after, when the prisoners were undergoing their life sentences of penal servitude, +DUMMY/LJ011-0075.wav|One was that of Captain Montgomery, who assumed the aliases of Colonel Wallace and Colonel Morgan. +DUMMY/LJ012-0118.wav|It was a large gold brooch set in pearls, but a portion of the mounting had melted with the heat. +DUMMY/LJ046-0198.wav|at the request of Special Agent Lawson, advance agent for President Kennedy's trip to Dallas. +DUMMY/LJ033-0147.wav|Oswald's fingerprint and palmprint found on bag. +DUMMY/LJ027-0094.wav|Nature rarely makes new parts -- never, if she can avoid it -- but, on the contrary, adapts an old part to the new function. +DUMMY/LJ032-0121.wav|He applied fingerprint powder to the side of the metal housing near the trigger, and noticed traces of two prints. +DUMMY/LJ014-0271.wav|Nevertheless, in this position, through the culpable carelessness which left him unfettered, he managed between eighteen forty-four +DUMMY/LJ046-0168.wav|of which there are sixty-five throughout the country. +DUMMY/LJ028-0403.wav|The walls of the houses are built of them. +DUMMY/LJ039-0006.wav|Another alleged threat by Oswald against a public figure involved former Vice President Richard M. Nixon. +DUMMY/LJ045-0192.wav|apparently will loop through the downtown area, probably on Main Street, +DUMMY/LJ037-0035.wav|He testified that the picture of Oswald which he saw later on television bore a resemblance to the man who shot Officer Tippit. +DUMMY/LJ008-0261.wav|In the case of convicted murderers only was prompt punishment inflicted, +DUMMY/LJ036-0144.wav|He said, quote, Yes, sir; that is right, because that is the five hundred block of North Beckley, end quote, +DUMMY/LJ024-0020.wav|There is nothing novel or radical about this idea. +DUMMY/LJ040-0215.wav|Those factors indicated a severe inability to enter into relationships with other people. +DUMMY/LJ028-0002.wav|In the old city of Damascus you climb to the hump of a tall fleet dromedary. +DUMMY/LJ050-0238.wav|which represents a departure from its prior practice. +DUMMY/LJ012-0199.wav|After a little +DUMMY/LJ018-0056.wav|blue eyes which were generally half closed, and very fair hair. +DUMMY/LJ017-0158.wav|She had a little fortune of her own, some one thousand seven hundred pounds or one thousand eight hundred pounds, +DUMMY/LJ011-0233.wav|that thieves or depredators were idle or entirely unsuccessful. +DUMMY/LJ011-0062.wav|If the law of this country can receive such a sacrifice, my death will render to heaven an innocent man, and to earth a repentant sinner. +DUMMY/LJ012-0217.wav|Meanwhile, a search had been made at Nova Scotia Gardens, Bethnal Green, where Bishop and Williams lived. +DUMMY/LJ015-0299.wav|and the circumstances were deemed sufficiently suspicious to warrant the arrest of the gentlemen at Yarmouth on a charge of forgery and conspiracy. +DUMMY/LJ048-0003.wav|Hosty's interpretation of the prevailing FBI instructions on referrals to the Secret Service was defended before the Commission by his superiors. +DUMMY/LJ045-0165.wav|That if I want to he would rent an apartment in Dallas tomorrow -- that he didn't want me to remain with Ruth any longer, but wanted me to live with him in Dallas. +DUMMY/LJ034-0024.wav|The other "Rolling Readers" carton, however, contained a palmprint and a fingerprint which were identified by Latona +DUMMY/LJ044-0079.wav|The limited notoriety that Oswald received as a result of the street fracas and in the subsequent radio debate +DUMMY/LJ002-0022.wav|no less than three hundred forty were crowded, to the great inconvenience and danger of the inmates. +DUMMY/LJ009-0211.wav|and on the right the ripping chisel, with which the murders had been committed, were exposed to view. +DUMMY/LJ012-0229.wav|but the jury, after a short deliberation, did not hesitate to bring in a verdict of guilty, and all three were condemned to death. +DUMMY/LJ038-0021.wav|She sent Brewer into the theatre to find the man and check the exits, told him about the assassination, and said, quote, +DUMMY/LJ016-0193.wav|Foxen asked him his name and address, and went away. +DUMMY/LJ041-0116.wav|after it was learned that he had defected to the Soviet Union. +DUMMY/LJ047-0076.wav|In New Orleans. In the middle of May of nineteen sixty-three, Agent Hosty checked Oswald's last known residence and found that he had moved. +DUMMY/LJ048-0177.wav|that such a procedure would not be consistent with the nature and purpose of the motorcade to let the people see their President and to welcome him to their city. +DUMMY/LJ025-0150.wav|and that many exhibit actions comparable to those which are brought about by the agency of a nervous system in animals. +DUMMY/LJ035-0068.wav|until Baker came upon Oswald in the lunchroom. +DUMMY/LJ013-0212.wav|"You can have no good intentions; you must quit my service tomorrow morning." +DUMMY/LJ024-0025.wav|Because the laws of many states, the practice of the Civil Service, the regulations of the Army and Navy, +DUMMY/LJ040-0165.wav|Lee's withdrawal was also noted by Mrs. Siegel, who described him as a, quote, seriously detached, withdrawn youngster, end quote. +DUMMY/LJ037-0169.wav|He declared that this bullet was fired from the same weapon that fired the test bullets to the exclusion of all other weapons. +DUMMY/LJ001-0155.wav|and that laterally the page looks as if it were being driven off the paper. +DUMMY/LJ042-0125.wav|I am increasingly aware of the presence, in all thing, of Lebizen, shop party secretary, fat, fortyish, and jovial on the outside. +DUMMY/LJ041-0029.wav|Oswald started to read Communist literature, which he obtained from the public library. +DUMMY/LJ015-0282.wav|Like his great predecessor Old Patch, he never went to a bank himself, nor did any of his accomplices. +DUMMY/LJ010-0043.wav|although happily never passing beyond dastardly attempts, cropped up in these times, and was often frequently repeated within a short interval. +DUMMY/LJ041-0057.wav|His study of Communist literature, +DUMMY/LJ047-0237.wav|Shortly after Oswald was apprehended and identified, Hosty's superior sent him to observe the interrogation of Oswald. +DUMMY/LJ009-0030.wav|and the idea will be strengthened by the following account of the Sunday service in the prison chapel on the occasion when the condemned sermon was preached. +DUMMY/LJ038-0204.wav|According to Marina Oswald's testimony, on the night of the Walker shooting, her husband left their apartment on Neely Street shortly after dinner. +DUMMY/LJ040-0234.wav|for some reason the record does not show, advantage was never taken of the chance afforded to Oswald. +DUMMY/LJ039-0020.wav|Marina Oswald related the events which followed, quote, +DUMMY/LJ047-0221.wav|or Vice President, to immediately notify the Secret Service and confirm it in writing. +DUMMY/LJ045-0124.wav|Marina Oswald thought he did so in order to emphasize his importance. +DUMMY/LJ014-0107.wav|The corpse settled all doubts, and the next point was to lay hands upon the Mannings. +DUMMY/LJ046-0046.wav|In all of these roles the President must go to the people. +DUMMY/LJ039-0169.wav|In the first series the firers required time spans of four point six, six point seven five, +DUMMY/LJ042-0065.wav|I am shocked! My dreams! I have waited for two years to be accepted. +DUMMY/LJ042-0052.wav|I have always considered this country to be my own. End quote. +DUMMY/LJ048-0285.wav|when it absolutely forbids drinking by any agent accompanying the President on a trip. +DUMMY/LJ033-0085.wav|She then opened the kitchen door and saw Oswald open the right rear door of her brother's car and place the package in the back of the car. +DUMMY/LJ014-0225.wav|A handsome sum was subscribed for the injured constable, who was disabled for life. +DUMMY/LJ040-0109.wav|Whatever problems may have been created by Lee's home life in Louisiana and Texas, +DUMMY/LJ042-0124.wav|they don't seem to be especially enthusiastic about any of the "collective" duties, a natural feeling. +DUMMY/LJ015-0053.wav|Bates was at once captured in Norfolk Street, Strand. +DUMMY/LJ011-0026.wav|Fauntleroy was scouted as a licentious libertine, a deep and determined gamester, a spendthrift whose extravagance knew no bounds. +DUMMY/LJ020-0089.wav|Studying upon an empty stomach provokes dyspepsia and injures the eyes. +DUMMY/LJ033-0208.wav|the Commission considered Stombaugh's testimony of probative value in deciding whether Oswald carried the rifle into the building in the paper bag. +DUMMY/LJ025-0039.wav|he afterward affirms that nitrogen is peculiar to animals, and herein he places the third distinction between the animal and the plant. +DUMMY/LJ037-0051.wav|describing the slayer as, quote, about thirty, five foot eight inches, black hair, slender, end quote. +DUMMY/LJ013-0268.wav|Good ladies and gentlemen all, I have a great deal more to say, but I am so bad I cannot say it. +DUMMY/LJ016-0384.wav|when the first shock of the verdict and the solemn notification of the impending blow keeps nearly all awake, or at least disturbs their night's rest. +DUMMY/LJ016-0422.wav|The man Marley displayed fortitude of a less repulsive kind. He acknowledged his guilt from the first. +DUMMY/LJ050-0114.wav|and the respective responsibilities for any further investigation that may be required. +DUMMY/LJ006-0083.wav|"So great is the authority exercised by him, +DUMMY/LJ048-0084.wav|some limited information was made available to the Secret Service. But there was no fully adequate liaison between the two agencies. Indeed, +DUMMY/LJ016-0090.wav|Taking advantage of being unobserved, he got out through the roof on to the leads, and traveled along them towards Number one, Newgate Street. +DUMMY/LJ010-0097.wav|Here they were surprised by the police, headed by a magistrate, and supported by a strong detachment of Her Majesty's Guards. +DUMMY/LJ043-0008.wav|De Mohrenschildt was a peripheral member of the so-called Russian community, with which Oswald made contact through Mr. Peter Gregory, +DUMMY/LJ018-0063.wav|He had led a wild life; had left his native land and enlisted first in the French army in Algeria, +DUMMY/LJ008-0162.wav|Cries of Murder! murder! were now raised, and added greatly to the horrors of the scene. +DUMMY/LJ001-0181.wav|When, as is most often the case, there is actual beauty in the cuts, +DUMMY/LJ019-0064.wav|that their condition was more natural, and approximated more nearly to that of daily life. +DUMMY/LJ006-0211.wav|Spirits certainly were now excluded; but a potman, with full permission of the sheriffs, +DUMMY/LJ040-0035.wav|While Oswald appeared to most of those who knew him as a meek and harmless person, he sometimes imagined himself as, quote, the Commander, end quote, +DUMMY/LJ034-0034.wav|Lieutenant Day of the Dallas police dusted this carton with powder and developed a palmprint on the top edge of the carton on the side nearest the window. +DUMMY/LJ038-0210.wav|according to Marina Oswald when he learned on the radio and in the newspapers the next day that he had missed, +DUMMY/LJ040-0128.wav|He was in Youth House from April sixteen to May seven, nineteen fifty-three, +DUMMY/LJ024-0017.wav|no additional place would be created. +DUMMY/LJ018-0093.wav|As a blind for their new frauds, they set up as law-stationers in York Buildings, Adelphi, and at once commenced their nefarious traffic. +DUMMY/LJ009-0032.wav|Strangers were now excluded, but the sheriffs attended in state, wearing their gold chains, +DUMMY/LJ018-0108.wav|His copies were generally pronounced indistinguishable from and as good as the originals. +DUMMY/LJ005-0177.wav|Nearly every one of these jurisdictions had its own prison, and there were one hundred and sixty such jails in all. +DUMMY/LJ007-0200.wav|In the press-yard, the press-room and ward above it were parceled out into nine separate sleeping cells; +DUMMY/LJ006-0184.wav|the most hardened and the oldest in vice had the best of it, while the inexperienced beginner went to the wall. +DUMMY/LJ018-0295.wav|The fraud was rendered possible by the absence of a check usual in the United States. +DUMMY/LJ018-0342.wav|It was to effect the rupture of an irksome tie that led Henry Wainwright to murder Harriet Lane deliberately and in cold blood. +DUMMY/LJ010-0309.wav|"During these numerous and trying difficulties" -- it is Mr. Fauntleroy who speaks -- +DUMMY/LJ048-0131.wav|There is, for example, no Secret Service arrangement for evaluating before a trip particular difficulties that might be anticipated, +DUMMY/LJ003-0148.wav|and spent in providing coals, candles, plates, knives, and forks; while all the occupants of this part of the prison +DUMMY/LJ020-0090.wav|Active exercise in like circumstances tempts debility and disease. +DUMMY/LJ041-0169.wav|and on how angry it made him, to which Thornley replied, quote, Well, comes the revolution you will change all that, end quote. +DUMMY/LJ002-0029.wav|as it was occupied and appropriated in eighteen ten. +DUMMY/LJ017-0033.wav|a medical practitioner, charged with doing to death persons who relied upon his professional skill. +DUMMY/LJ016-0335.wav|There was no interference with the crowd, which collected as usual, although not to the customary extent. +DUMMY/LJ007-0204.wav|"which has been demonstrably proved to be the fruitful source of all the abuses and irregularities which have so long disgraced Newgate, +DUMMY/LJ006-0244.wav|Violent inflammation and extensive suppuration ensued, and for a considerable time amputation seemed inevitable. +DUMMY/LJ013-0266.wav|Good was committed to Newgate, and tried at the Central Criminal Court before a crowded court. +DUMMY/LJ015-0039.wav|Money they must have, and money they raised to meet their urgent necessities upon the balances and securities deposited with them by their customers. +DUMMY/LJ030-0038.wav|Every effort is made to prevent unscheduled stops, although the President may, and in Dallas did, order stops in order to greet the public. +DUMMY/LJ014-0032.wav|as the seducer of an innocent girl to whom he (Hocker) had been fondly attached. +DUMMY/LJ009-0217.wav|or on another specially erected on some commanding spot, had fallen into disuse by eighteen thirty-two. +DUMMY/LJ045-0037.wav|after his recent rebuffs in Mexico City +DUMMY/LJ024-0044.wav|that I will appoint justices who will act as justices and not as legislators +DUMMY/LJ009-0222.wav|The revival of this barbarous practice caused much indignation in certain quarters, +DUMMY/LJ020-0001.wav|Marion Harland's Cookery for Beginners. Bread Sponge and Breakfast Breads. +DUMMY/LJ028-0236.wav|He placed a portion of his army at the point where the river enters the city, and another body at the back of the place where it issues forth, +DUMMY/LJ009-0094.wav|but his face is of an ashy paleness; and if you look carefully you may see the blood trickling from his lip, +DUMMY/LJ015-0223.wav|He lent it to Agar for a brief space, who promptly took an impression on wax. But the safes had a double lock; +DUMMY/LJ013-0263.wav|He was recognized by an ex-policeman who had known him at Roehampton, and immediately arrested. +DUMMY/LJ043-0063.wav|not only caused him difficulties in his employment relations, but they also provided him with excuses for employment failures +DUMMY/LJ006-0100.wav|many without stockings, and with hardly shoes to their feet; +DUMMY/LJ019-0310.wav|which had long been admitted as indispensable, and had never as yet been properly obtained. +DUMMY/LJ046-0117.wav|is to collect, process, and evaluate information about persons or groups who may be a danger to the President. +DUMMY/LJ010-0148.wav|tried to stab George the third as he was alighting from his carriage at the gate of St. James's Palace. +DUMMY/LJ019-0375.wav|There were only two of them, and they could not visit each prison more than once in each year, sometimes not oftener than once in eighteen months. +DUMMY/LJ011-0257.wav|Howard was secured, given into custody, and committed to Newgate. +DUMMY/LJ047-0078.wav|and Hosty asked the New Orleans FBI office to determine Oswald's address and what he was doing. +DUMMY/LJ028-0005.wav|Leaving the city by the eastern gate, and passing a small village or two, +DUMMY/LJ004-0124.wav|In four hundred and forty-five prisons no work of any description had been introduced for the employment of prisoners; +DUMMY/LJ041-0179.wav|that he never heard Oswald express a desire personally to take part in the elimination of injustices anywhere in the world and that he, quote, +DUMMY/LJ009-0233.wav|The rope still broke sometimes, although it was not often that the horrid scene seen at Jersey at the beginning of the century was repeated. +DUMMY/LJ004-0093.wav|which was contemporaneous with Howard's first journeys, laid down precise rules as regards cleanliness, and the proper supply of space and air. +DUMMY/LJ014-0218.wav|after which he kicked his prostrate foe in the most brutal and cowardly manner, and until he was almost senseless, and bruised from head to foot. +DUMMY/LJ021-0060.wav|Minimum wages have been established and other wages adjusted toward a rising standard of living. +DUMMY/LJ015-0265.wav|was the prime mover, +DUMMY/LJ049-0161.wav|and also for the preventive investigation of potential threats against the President. +DUMMY/LJ014-0008.wav|His remark was, "It is a nasty job;" he took hold of the dead hand, and confessed that he felt "queer" at the shocking sight. +DUMMY/LJ049-0003.wav|Chapter eight. The Protection of the President. Part four. +DUMMY/LJ034-0069.wav|was provided by the testimony of Charles Givens, who was the last known employee to see Oswald inside the building prior to the assassination. +DUMMY/LJ003-0128.wav|caught their flash terms and sung their songs, was admitted to their revels, and acquired, in place of habits of perfect sobriety, +DUMMY/LJ047-0149.wav|the New Orleans office of the FBI learned that in September Oswald had given a forwarding address of two five one five +DUMMY/LJ028-0047.wav|War broke out between the two brothers, and again Babylon was captured. +DUMMY/LJ050-0028.wav|no sizable organization can achieve efficiency without the careful analysis and demarcation of responsibility +DUMMY/LJ017-0116.wav|and even sought to upset the vehicle by which they were being conveyed a part of the way to London. +DUMMY/LJ038-0026.wav|Other policemen entered the front door and searched the balcony. +DUMMY/LJ017-0125.wav|He never actually said that he was not guilty, but he was confident he would not be convicted. +DUMMY/LJ028-0026.wav|Here in the valley the water is sweet and the food abundant. +DUMMY/LJ034-0026.wav|The Commission has considered the possibility that the cartons might have been moved in connection with the work that was being performed on the sixth floor on November twenty-two. +DUMMY/LJ009-0285.wav|that Calcraft's method of hanging was very rough, much the same as if he had been hanging a dog. +DUMMY/LJ018-0030.wav|Last of all, the cabman swore that he had bought the very hat found in the carriage for Müller at the hatter's, Walker's of Crawford Street. +DUMMY/LJ028-0258.wav|Then the Babylonians secretly plotted to throw off the Persian yoke. +DUMMY/LJ037-0121.wav|Both Callaway and Guinyard testified that they had not been shown any pictures by the police before the lineup. +DUMMY/LJ022-0195.wav|But it is more than the recovery of the material basis of our individual lives. +DUMMY/LJ014-0308.wav|others, that Napoleon the third, but recently proclaimed Emperor of the French, wished to gain the popularity necessary to secure the people. +DUMMY/LJ042-0109.wav|one of the members of the Russian community in Dallas with which the Oswalds became acquainted upon their arrival in the United States, +DUMMY/LJ011-0169.wav|Miss Turner, not knowing him, would have left the room, but he said he came from her father, and she remained. +DUMMY/LJ003-0310.wav|a bell should give notice thereof, and of meal-hours, working-hours, or of escapes. +DUMMY/LJ004-0109.wav|according to their categories or crimes. +DUMMY/LJ017-0124.wav|He frequently declared before and during the trial that it would be impossible to find him guilty. +DUMMY/LJ039-0049.wav|He was the kind of person who could try and wound somebody in that way. Possibly he didn't want to go out at all but was just doing this all as a sort of joke, +DUMMY/LJ050-0034.wav|and the breach of regulations which occurred on the night of November twenty-one at Fort Worth. +DUMMY/LJ025-0067.wav|had declared the greater number of what are now recognized as locomotive plants to be animals. +DUMMY/LJ008-0053.wav|the whole is temporary, being all calculated to take to pieces, which are preserved within the prison." +DUMMY/LJ037-0104.wav|manager of a used-car lot on the northeast corner of Patton Avenue and Jefferson Boulevard, and Sam Guinyard, a porter at the lot. +DUMMY/LJ002-0071.wav|It was large and comparatively commodious, being maintained on a better footing than any other part of the prison. +DUMMY/LJ010-0232.wav|He could not plead insanity like his predecessor, and no attempt was made at his trial to prove him of unsound mind. +DUMMY/LJ016-0185.wav|Calcraft, who, in spite of the dreadful calling he subsequently followed, was always reputed a kindly man, +DUMMY/LJ032-0110.wav|Moreover, the use of "Alek" as a first name for Hidell is a further link to Oswald because "Alek" was Oswald's nickname in Russia. +DUMMY/LJ045-0248.wav|He also had demonstrated a capacity to act decisively and without regard to the consequences when such action would further his aims of the moment. +DUMMY/LJ015-0103.wav|Mr. Fasson, more and more ill at ease, would not accept this subterfuge, and declared they must be found. +DUMMY/LJ028-0288.wav|To do this seeming to him a light matter, he mutilated himself in a way that was utterly without remedy. +DUMMY/LJ028-0142.wav|While such is its size, in magnificence there is no other city that approaches to it. +DUMMY/LJ029-0199.wav|On November eighteen the Dallas City Council adopted a new city ordinance prohibiting interference with attendance at lawful assemblies. +DUMMY/LJ044-0109.wav|It is good to know that movements in support of fair play for Cuba has developed in New Orleans as well as in other cities. +DUMMY/LJ038-0125.wav|Now, I've told you all I'm going to tell you about that card in my billfolds -- you have the card yourself and you know as much about it as I do, end quote. +DUMMY/LJ014-0322.wav|The dock warrant was issued by the wharfinger as certificate that he held the goods. +DUMMY/LJ029-0211.wav|And on the morning of the President's arrival, there appeared in the Morning News a full, black-bordered advertisement headed: +DUMMY/LJ047-0105.wav|In Quigley's judgment, +DUMMY/LJ009-0149.wav|was more appropriate for serious reflection and profitable ministration than this exciting occasion before a large and public assembly. +DUMMY/LJ041-0043.wav|In fact, he had quit school in an attempt to obtain his mother's assistance to join the Marines. +DUMMY/LJ026-0038.wav|the fact of the animal and vegetable kingdoms being related to one another like two trees united at the roots may be accounted for by the hypothesis that +DUMMY/LJ017-0074.wav|whose life he had insured for thirteen thousand pounds. +DUMMY/LJ037-0193.wav|S T. W. two inch BBL, unquote, cost twenty-nine dollars, ninety-five cents. +DUMMY/LJ044-0155.wav|that things are improving due to the fact that my husband expresses a sincere wish to return together with me to the USSR. Unknown to his wife, however, +DUMMY/LJ009-0283.wav|this time Calcraft adding his weight to the body, and the strangulation was completed. +DUMMY/LJ009-0161.wav|capital punishment was abolished for forgery, except in cases of forging or altering wills or powers of attorney to transfer stock. +DUMMY/LJ010-0177.wav|As the carriage came up to him Oxford turned, put his hand into his breast, drew a pistol, and fired at the Queen. +DUMMY/LJ007-0171.wav|distinguished as the seat of religion, worth, and philanthropy, +DUMMY/LJ047-0016.wav|The FBI opened a file on Oswald in October nineteen fifty-nine, when news reports appeared of his defection to the Soviet Union. +DUMMY/LJ003-0145.wav|or about a foot and a half laterally. These fees were in reality a substantial contribution towards the expenses of the jail; +DUMMY/LJ036-0003.wav|The Killing of Patrolman J. D. Tippit +DUMMY/LJ047-0017.wav|The file was opened, quote, +DUMMY/LJ014-0079.wav|and which he mistook for a drain, and that while he was speaking to her she put the muzzle of a pistol close to the back of his head and shot him down. +DUMMY/LJ005-0032.wav|no weavers' looms or carpenters' benches. There must be a great deal of solitude, coarse food, a dress of shame, +DUMMY/LJ048-0052.wav|and less narrow interpretation of their responsibilities. +DUMMY/LJ007-0061.wav|He left off because he was so much interfered with and laughed at, and from seeing that no success attended his efforts, owing to the evils arising from association. +DUMMY/LJ006-0299.wav|His excuse was that he had given the girl his promise that she should not go, an assumption of prerogative which by no means rested with him; +DUMMY/LJ036-0006.wav|At about one:sixteen p.m., a Dallas police officer, J. D. Tippit, was shot less than one mile from Oswald's roominghouse. +DUMMY/LJ036-0164.wav|And when interrogated about a cab ride, Oswald also admitted that he left the slow-moving bus and took a cab to his roominghouse. +DUMMY/LJ015-0257.wav|Kay, who had been living with Agar at the time of the bullion robbery, +DUMMY/LJ003-0142.wav|to purchase the greater ease and comfort of the master's side. +DUMMY/LJ030-0080.wav|scanning not only the crowds but the windows and roofs of buildings, overpasses, and crossings. +DUMMY/LJ003-0242.wav|was still practiced, that of loading newly-arrived prisoners until they paid certain fees. +DUMMY/LJ034-0182.wav|The person in the window was a white man and, quote, looked to me like he was looking straight at the Triple Underpass, end quote, down Elm Street. +DUMMY/LJ045-0025.wav|Relationship With Wife +DUMMY/LJ006-0261.wav|together with four bradawls, several large iron spikes, screws, nails, and knives; +DUMMY/LJ046-0050.wav|administrative, political. +DUMMY/LJ003-0038.wav|which would accommodate the inmates of Newgate and of the three compters, Ludgate, +DUMMY/LJ009-0102.wav|Suddenly he utters a short sharp scream, and all is still. +DUMMY/LJ003-0347.wav|It was very desirable that there should be a more speedy removal of transports from Newgate to the ships. +DUMMY/LJ041-0148.wav|As indicated above, Oswald started to read Communist literature after he and his mother left New York and moved to New Orleans. +DUMMY/LJ016-0280.wav|"There were well-dressed and ill-dressed, old men and lads, women and girls." +DUMMY/LJ004-0082.wav|These somewhat novel but undoubtedly indisputable propositions were backed up, not by sound arguments only, but by the letter of the law. +DUMMY/LJ046-0157.wav|The files are manually maintained, without use of any automatic data-processing techniques. +DUMMY/LJ050-0123.wav|and investigative capabilities of the agencies now operating in this field but will continue +DUMMY/LJ038-0247.wav|A fourth photograph, showing a stretch of railroad tracks, +DUMMY/LJ032-0178.wav|the evidence indicates that he continued wearing the same shirt which he was wearing all morning and which he was still wearing when arrested. +DUMMY/LJ042-0072.wav|Still intent, however, on staying in the Soviet Union, +DUMMY/LJ040-0094.wav|Pic thought that his mother overstated her financial problems and was unduly concerned about money. +DUMMY/LJ006-0172.wav|and evils that should have been speedily rooted out remained because they had the prescription of long usage. +DUMMY/LJ029-0128.wav|going one block north and then turning left onto Elm. +DUMMY/LJ012-0232.wav|He asserted that the victim was a Lincolnshire lad, and not an Italian boy, although the latter was fully proved. +DUMMY/LJ048-0037.wav|overlooking the motorcade route as a source of danger to the President and did not inform the Secret Service of his employment in the Depository Building. +DUMMY/LJ040-0137.wav|Marguerite Oswald said that she had not realized until then in what kind of place her son had been confined. +DUMMY/LJ041-0059.wav|His difficulty in relating to other people and his general dissatisfaction with the world around him continued while he was in the Marine Corps. +DUMMY/LJ030-0174.wav|Unable to see the President as he turned to the right, +DUMMY/LJ012-0280.wav|or of Gardelle's frantic efforts to conceal his crime. +DUMMY/LJ005-0169.wav|but I indulge a hope that many of them will contract with the counties, +DUMMY/LJ006-0196.wav|Prisoners confessed that it was a favorite occupation, the chief games being "shoving halfpence" on the table, +DUMMY/LJ015-0229.wav|who quickly took the wax impression, handed it back to Pierce; Pierce replaced it, left the office, and the thing was done. +DUMMY/LJ007-0064.wav|and the day of whose execution was fixed. He had no doubt a difficult mission to discharge; +DUMMY/LJ018-0092.wav|in eighteen fifty-six. +DUMMY/LJ022-0086.wav|to which all proposals for the expenditure of money must go for preliminary study and consideration. +DUMMY/LJ042-0121.wav|Mass gymnastics, compulsory afterwork meeting, usually political information meeting. +DUMMY/LJ013-0136.wav|Several pawnbrokers were subpoenaed and obliged to surrender plate, to the extent in some cases of a couple of dozen of spoons or forks, +DUMMY/LJ021-0070.wav|Employed workers have not by any means all enjoyed a return to the earnings of prosperous times, +DUMMY/LJ018-0067.wav|Thither, he was pursued by a detective officer, +DUMMY/LJ040-0221.wav|Carro was the only one of Oswald's three principal observers who recommended that he be placed in a boy's home or similar institution. +DUMMY/LJ023-0018.wav|We also became convinced that the only way to avoid a repetition of those dark days was to have a government with power to prevent +DUMMY/LJ044-0149.wav|He said that nothing kept him in the United States, that he would not lose anything if he returned to the Soviet Union, that he wanted to be with her +DUMMY/LJ003-0090.wav|Mr. Newman admitted that he had petitioned that certain "trusty men" might be left in the jail. +DUMMY/LJ004-0090.wav|Again, the twenty-two Charles the second c twenty ordered the jailer to keep felons and debtors "separate and apart from one another, +DUMMY/LJ050-0276.wav|As has been pointed out, the Commission has not resolved all the proposals which could be made. The Commission nevertheless is confident that, +DUMMY/LJ031-0194.wav|At five:fifty-eight p.m. Eastern Standard Time, +DUMMY/LJ042-0075.wav|testified that Oswald was extremely sure of himself and seemed, quote, to know what his mission was. +DUMMY/LJ021-0101.wav|gave businessmen the opportunity they had sought for years to improve business conditions through what has been called self-government in industry. +DUMMY/LJ022-0151.wav|did more than anything else to bring about the recent collapse of industries. +DUMMY/LJ009-0168.wav|robbery, burglary, and arson. +DUMMY/LJ011-0274.wav|fell upon him, and in spite of his resistance carried him into a sort of den partitioned off at the end of the back kitchen. +DUMMY/LJ014-0073.wav|Meanwhile this woman, unflinching in her cold, bloody determination, carefully laid all her plans for the consummation of the deed. +DUMMY/LJ013-0078.wav|Barber was subsequently pardoned, but was not replaced on the rolls as an attorney till eighteen fifty-five, +DUMMY/LJ027-0166.wav|retain permanently this form, and are therefore called "perennibranchs," but the frog still passes on. +DUMMY/LJ009-0254.wav|The hangman summoned assistance, and with help tied the cap over White's face with a handkerchief. +DUMMY/LJ012-0109.wav|But they at once made tracks, and took up their residence under assumed names in a tavern in Bloomsbury. +DUMMY/LJ045-0226.wav|The fact that he took so little money with him when he left Irving in the morning indicates that he did not expect to get very far from Dallas on his own +DUMMY/LJ015-0251.wav|The latter was ere long arrested on a charge of uttering forged cheques, convicted, and sentenced to transportation for life. +DUMMY/LJ025-0133.wav|The reflex action of the snail is the result of the presence of a nervous system in the animal. +DUMMY/LJ034-0129.wav|establishes that they were either squatting or kneeling. +DUMMY/LJ008-0030.wav|The first affair of the kind on this spot was on the third December, seventeen eighty-three, +DUMMY/LJ049-0211.wav|It would have been necessary for the Commission to take considerable testimony, much of it extraneous to the facts of the assassination of President Kennedy, +DUMMY/LJ012-0285.wav|Another day elapsed before he got rid of the rest of the body, +DUMMY/LJ003-0180.wav|These wards were always full to overflowing; sometimes double the number the rooms could accommodate were crowded into them. +DUMMY/LJ037-0044.wav|She peered through her fingers, lowered her hands, and saw the man doing something with his gun. Quote, +DUMMY/LJ033-0105.wav|One employee, Jack Dougherty, +DUMMY/LJ007-0124.wav|The infirmary at this particular period epitomized the condition of the jail at large. +DUMMY/LJ037-0184.wav|agents of the FBI determined that George Rose and Co. of Los Angeles was a major distributor of this type of revolver. +DUMMY/LJ044-0071.wav|Stuckey had a radio program called, quote, Latin Listening Post, end quote, +DUMMY/LJ029-0124.wav|To reach the Trade Mart from Main Street the agents decided to use the Stemmons Freeway (Route Number seventy-seven), the most direct route. +DUMMY/LJ003-0212.wav|But they nevertheless frequently managed to secrete the means of making away with themselves, and accomplished their purpose. +DUMMY/LJ017-0153.wav|Probably the doctor had told her the story he brought forward when tried for bigamy, namely, +DUMMY/LJ009-0085.wav|'Now for you, my poor fellow mortals, who are about to suffer the last penalty of the law.' +DUMMY/LJ036-0154.wav|The walk from Beckley and Neely to ten twenty-six North Beckley was timed by Commission counsel at five minutes and forty-five seconds. +DUMMY/LJ002-0193.wav|In these cases security was given for the amount of the debt, +DUMMY/LJ039-0050.wav|not really as a joke but rather to simply wound me, to make me feel bad, end quote. +DUMMY/LJ006-0243.wav|He was in the transport side, when one of his fellows, in endeavoring to strike another prisoner with a large poker, missed his aim, and struck Watkins' knee +DUMMY/LJ047-0110.wav|that another Bureau agent, Milton R. Knack, had been conducting a background investigation of Oswald at the request of Agent Hosty in Dallas. +DUMMY/LJ010-0178.wav|The shot missed, and as the carriage passed on, Oxford drew a second pistol and fired again. +DUMMY/LJ006-0002.wav|In the preceding chapter I have been tempted by the importance of the general question to give it prominence and precedence over the particular branch of which I am treating. +DUMMY/LJ022-0177.wav|is one of the most hopeful results of our efforts as a Nation to reestablish public confidence in private banking. +DUMMY/LJ015-0006.wav|who had fraudulent warrants out of their own to the extent of one hundred fifty thousand pounds, suspended payment and absconded. +DUMMY/LJ026-0124.wav|In the plant, starch, as has been seen, is first formed in the chlorophyll-bodies. +DUMMY/LJ024-0013.wav|This plan will save our national Constitution from hardening of the judicial arteries. +DUMMY/LJ002-0246.wav|and came under the strong animadversion of the Jail Committee of seventeen twenty-nine. +DUMMY/LJ004-0091.wav|in distinct rooms, on pain of forfeiting his office and treble damages to the party aggrieved." +DUMMY/LJ013-0081.wav|Banks and bankers continued to be victimized. +DUMMY/LJ024-0111.wav|the legislation demanded by the mandate. +DUMMY/LJ006-0240.wav|Watkins' knee-joint is very severely injured. +DUMMY/LJ015-0135.wav|He began in a very small way. +DUMMY/LJ017-0066.wav|Yet he was silent when specifically pressed to confess that he had killed his wife and his brother. +DUMMY/LJ050-0148.wav|the increased information supplied by other agencies will be wasted. +DUMMY/LJ029-0123.wav|was not used for the main portion of the downtown part of the motorcade because Main Street offered better vantage points for spectators. +DUMMY/LJ041-0008.wav|He took walks and visited museums, and sometimes rode a rented bicycle in the park on Saturday mornings. +DUMMY/LJ028-0483.wav|O Marduk, great god, look joyfully upon the precious work of my hands. Be thou my protector. +DUMMY/LJ008-0126.wav|A new hat was partly flattened on his head, for, owing to its being too small in the crown, it stood many inches too high behind. +DUMMY/LJ034-0178.wav|He said that he could see the man from the middle of his chest to the top of his head, and that as he was facing the window the man was in the lower right-hand portion of the window +DUMMY/LJ015-0155.wav|that so good a man had really been for years a swindler and a rogue. +DUMMY/LJ019-0008.wav|but no steps had been taken in Parliament to enforce a better system of discipline, +DUMMY/LJ028-0251.wav|When Cyrus took Babylon, little or no force was employed. +DUMMY/LJ047-0215.wav|including the significant fact that Oswald was employed in a building which overlooked the motorcade route. +DUMMY/LJ011-0045.wav|The kindliness of the city authorities to Fauntleroy was not limited to the assignment of a separate place of durance. +DUMMY/LJ041-0161.wav|It was just -- never getting back to looking at things from any other way once he had become a Marxist, whenever that was, end quote. +DUMMY/LJ001-0100.wav|and these somewhat wiry letters are suitable for the machine process, which would not do justice to letters of more generous design. +DUMMY/LJ034-0005.wav|He worked principally on the first and sixth floors of the building, gathering books listed on orders and delivering them to the shipping room on the first floor. +DUMMY/LJ017-0199.wav|The first post-mortem indicated death from natural causes, +DUMMY/LJ004-0085.wav|As far back as the reign of Charles the second, a law was passed declaring that sufficient provision should be made for the relief and setting on work +DUMMY/LJ034-0153.wav|but did not do so because he felt that the assassination was, quote, a Communist activity, +DUMMY/LJ010-0277.wav|Lord Abinger sentenced him to eighteen months' imprisonment in Newgate, +DUMMY/LJ012-0061.wav|On stopping at a door in this low street, Ikey jumped out, ran into the house, slamming the door behind him. +DUMMY/LJ028-0331.wav|After this second exploit, his praise was in all mouths. +DUMMY/LJ002-0060.wav|But these king's witnesses were also put at times into the press yard among the capital convicts, seemingly a very dangerous proceeding, +DUMMY/LJ038-0302.wav|(three) the testimony of firearms identification experts, and (four) the testimony of Marina Oswald, +DUMMY/LJ039-0103.wav|Oswald, like all Marine recruits, received training on the rifle range at distances up to five hundred yards, +DUMMY/LJ028-0346.wav|Further, wishing to prevent the race of the Babylonians from becoming extinct, +DUMMY/LJ003-0265.wav|the thieves and burglars who carried on the active business of their profession, from which their confederates were temporarily debarred. +DUMMY/LJ001-0132.wav|Here, again, if the practice of "leading" were retrenched larger type could be used without enhancing the price of a book. +DUMMY/LJ014-0131.wav|but Mrs. Manning, speaking in a foreign accent, addressed the court with great fluency and vehemence. +DUMMY/LJ038-0079.wav|on November twenty-four. +DUMMY/LJ032-0047.wav|According to the vice president of Klein's, William Waldman, +DUMMY/LJ018-0192.wav|But they had the body, all the same; it had been dressed up in decent clothes and made to stand upright in the cart. +DUMMY/LJ014-0120.wav|Each tried to throw the blame on the other; Manning declared his wife had committed the murder, Mrs. Manning indignantly denied the charge. +DUMMY/LJ003-0125.wav|The evidence was invariably sufficient to convict, and the judge never hesitated to inflict the heaviest penalties. +DUMMY/LJ006-0186.wav|said with justice that "incredible scenes of horror occur in Newgate." +DUMMY/LJ028-0147.wav|and when a sufficient number were completed they baked the bricks in kilns. +DUMMY/LJ038-0304.wav|on April ten, nineteen sixty-three. +DUMMY/LJ008-0036.wav|The following twenty-third April, it is stated that the malefactors ordered for execution on the eighteenth instead +DUMMY/LJ018-0296.wav|There such bills would be sent to the drawer to be initialed, and the forgery would have been at once detected. +DUMMY/LJ044-0069.wav|New Orleans branch, end quote. +DUMMY/LJ002-0166.wav|Mr. Buxton, in his "Inquiry into the System of Prison Discipline," +DUMMY/LJ010-0299.wav|caused much excitement at the time on account of the magnitude of the fraud, and the seeming probity of the culprit. +DUMMY/LJ034-0007.wav|The Commission evaluated the physical evidence found near the window after the assassination and the testimony of eyewitnesses +DUMMY/LJ041-0033.wav|The association's then president, William E. Wulf, testified that he remembered an occasion when Oswald, quote, +DUMMY/LJ012-0085.wav|The chest was opened to give change, and a heavy tray lifted out which plainly held some four thousand pounds in cash. +DUMMY/LJ016-0385.wav|But the uneasiness soon wears off. The second night sleep comes readily, and is sound; +DUMMY/LJ018-0313.wav|deep laid, and with many ramifications, by which some of the Newgate warders were to be bribed to allow the prisoners to escape from their cells at night. +DUMMY/LJ030-0210.wav|and I saw President Kennedy grab at himself and lurch forward and to the left, end quote. +DUMMY/LJ006-0197.wav|pitch in the hole, cribbage, dominoes, and common tossing, at which as much as four or five shillings would change hands in an hour. +DUMMY/LJ048-0156.wav|and suggested the desirability of such a statement. +DUMMY/LJ003-0199.wav|and until the day was actually fixed, spent the time in roistering, swearing, gambling, or playing at ball. +DUMMY/LJ005-0049.wav|The refuge, which had for its object the training of its inmates in habits of industry, +DUMMY/LJ001-0066.wav|In fact Gunther Zeiner's first type (afterwards used by Schussler) is remarkably like the type of the before-mentioned Subiaco books. +DUMMY/LJ031-0216.wav|was located about an inch (two point five centimeters) to the right and slightly above the large bony protrusion +DUMMY/LJ019-0118.wav|and later experience has fully proved the advantage of a judicious system of gratuities for labor; +DUMMY/LJ029-0013.wav|During nineteen sixty-three, the reasons for making the trip became more persuasive. +DUMMY/LJ044-0226.wav|Oswald had expressed approval of President Kennedy's active role in the area of civil rights. +DUMMY/LJ046-0162.wav|The case file served merely as a repository for information until enough had accumulated to warrant an investigation. +DUMMY/LJ036-0152.wav|This was the seven hundred block of North Beckley +DUMMY/LJ021-0094.wav|consumer complaints and the settlement of disputes between employers and employees and between one employer and another. +DUMMY/LJ028-0221.wav|When I made my gracious entry into Babylon, with exceeding joy I took up my abode in the royal palace. +DUMMY/LJ022-0093.wav|The next step will be to notify the proper government agency +DUMMY/LJ049-0085.wav|the State of Texas had exclusive jurisdiction. +DUMMY/LJ048-0179.wav|of the Texas School Book Depository Building or those employed there prior to the time the President was shot. End quote. +DUMMY/LJ011-0166.wav|The road taken was via Manchester, where the servant said a Dr. Hull was to be picked up to go on with them to Shrigley. +DUMMY/LJ024-0014.wav|The number of judges to be appointed would depend wholly on the decision of present judges now over seventy, +DUMMY/LJ033-0027.wav|Mrs. Linnie Mae Randle, Frazier's sister, commented to her brother about Oswald's unusual midweek return to Irving. +DUMMY/LJ021-0008.wav|we are bringing order out of the old chaos +DUMMY/LJ018-0191.wav|Bob was therefore allowed to pass on. +DUMMY/LJ022-0160.wav|has lost touch with, and has lost the sympathy of, the communities it pretends to serve. +DUMMY/LJ031-0184.wav|Swearing in of the New President +DUMMY/LJ045-0028.wav|he and his wife spent every weekend but one together at the Irving, Texas home of Mrs. Ruth Paine, who was then separated from her husband. +DUMMY/LJ030-0185.wav|quote, we are hit. Get us to the hospital immediately, end quote. +DUMMY/LJ041-0175.wav|Lieutenant Donovan testified that Oswald thought that, quote, there were many grave injustices concerning the affairs in the international situation, end quote. +DUMMY/LJ011-0031.wav|He called many witnesses, seventeen in all, to speak of him as they had found him; and these, all respectable city merchants and business men, +DUMMY/LJ023-0118.wav|That means a Supreme Court that will enforce the Constitution as written +DUMMY/LJ047-0172.wav|Oswald had said that he was a, quote, Trotskyite Communist, end quote. +DUMMY/LJ010-0192.wav|or of the fatal consequences which might have ensued. +DUMMY/LJ019-0068.wav|Separation was injurious to health, mental or physical, said one side; men broke down when subjected to it for more than a certain period, +DUMMY/LJ037-0119.wav|I stepped to the back of the room, so I could kind of see him from the same distance which I had seen him before. And when he came out I knew him, end quote. +DUMMY/LJ006-0247.wav|The prisoner who used the knife was secured, but it was the wardsman, and not the officers, to whom the report was made, and no official inquiry or punishment followed. +DUMMY/LJ040-0076.wav|In the fall of that year John Pic and Robert Oswald went to a military academy +DUMMY/LJ035-0071.wav|At a given signal, they reenacted the event. Baker's movements were timed with a stopwatch. +DUMMY/LJ010-0285.wav|invariably went out in a cab, for which he always paid the same fare, +DUMMY/LJ008-0007.wav|The decision to remove the scene of action to the immediate front of the jail itself +DUMMY/LJ034-0116.wav|The suspect was described as, quote, a white male about thirty, five foot eight, black hair, slender, end quote. +DUMMY/LJ014-0053.wav|Soon afterwards, in Gloucestershire, +DUMMY/LJ015-0152.wav|His name headed every subscription list; his purse was always open. +DUMMY/LJ040-0066.wav|It had its effect on Lee's mother, Marguerite, his brother Robert, who had been born in nineteen thirty-four, +DUMMY/LJ044-0165.wav|He wanted to be arrested. I think he wanted to get into the newspapers, so that he would be known. End quote. +DUMMY/LJ005-0084.wav|"so as to prevent them from seeing, conversing, or holding any intercourse with each other." +DUMMY/LJ010-0295.wav|The increase of bank forgeries, and its cause, I referred to in a previous chapter. +DUMMY/LJ003-0266.wav|One notorious character, while a prisoner awaiting transfer to the hulks, +DUMMY/LJ032-0244.wav|Marina Oswald testified that the rifle found on the sixth floor of the Depository Building was the "fateful rifle of Lee Oswald." +DUMMY/LJ025-0160.wav|The weight of the nitrogenous protein compounds, +DUMMY/LJ015-0077.wav|He was not the first man of loose morality and expensive tastes +DUMMY/LJ004-0084.wav|Some might be in abeyance, but they had never been repealed, and some were quite freshly imported upon the Statute Book. +DUMMY/LJ030-0042.wav|Manned by officers of the Dallas Police Department, this automobile preceded the main party by approximately quarter of a mile. +DUMMY/LJ038-0014.wav|Brewer said, quote, He just looked funny to me. His hair was sort of messed up and looked like he had been running, and he looked scared, and he looked funny, end quote. +DUMMY/LJ037-0259.wav|that the man who killed Tippit was wearing a light-colored jacket, +DUMMY/LJ041-0075.wav|that authority, particularly the Marine Corps, ought to be able to recognize talent such as his own, without a given magic college degree, +DUMMY/LJ028-0244.wav|but would have destroyed them utterly; for they would have made fast all the street gates which gave upon the river, +DUMMY/LJ009-0158.wav|Already a strong dislike to the reckless and almost indiscriminate application of the extreme penalty was apparent in all classes, +DUMMY/LJ007-0098.wav|The other inmates of the prison of an exceptional character, and exempted from the regular discipline, such as it was, +DUMMY/LJ042-0085.wav|He said that his Marine service in Okinawa and elsewhere had given him, quote, a chance to observe American imperialism, end quote. +DUMMY/LJ018-0132.wav|To facilitate his operations, Brewer obtained a false master key from Burnett, +DUMMY/LJ009-0087.wav|It is enough to say that in the same solemn tone he talks about the minutest of crimes, punishments, bonds, shame, +DUMMY/LJ023-0114.wav|We must find a way to take an appeal from the Supreme Court to the Constitution itself. +DUMMY/LJ028-0064.wav|In five ninety-seven, when he sent his army to Jerusalem, he won the hatred of the Jews by taking Jehoiakin, the King, captive. +DUMMY/LJ027-0134.wav|Most remarkable of all the principles which have been discovered by embryologists is the "Recapitulation Doctrine" +DUMMY/LJ038-0115.wav|At the first interrogation, Oswald claimed that his only crime was carrying a gun and resisting arrest. +DUMMY/LJ011-0258.wav|The defense he set up was, that Mullay had used epithets towards him while they were negotiating a business matter, +DUMMY/LJ006-0283.wav|There was the same crowding in the sleeping arrangements as on the male side; the same scarcity of bedding. +DUMMY/LJ045-0172.wav|Answer: He said he would buy me a washing machine. +DUMMY/LJ005-0274.wav|The committee most of all insisted upon the entire individual separation of prisoners, except during the hours of labor, +DUMMY/LJ010-0044.wav|The present Queen very soon after her accession +DUMMY/LJ015-0236.wav|Burgess admitted him into the guard's van, where he fitted and filed the keys till they worked easily and satisfactorily in the locks of the safe. +DUMMY/LJ040-0038.wav|Such ideas of grandeur were apparently accompanied by notions of oppression. +DUMMY/LJ019-0004.wav|Elsewhere the spirit had been more or less active, although not uniformly or always to the same extent. +DUMMY/LJ036-0163.wav|When queried the following morning concerning a bus transfer found in his possession at the time of his arrest, he admitted receiving it. +DUMMY/LJ031-0163.wav|The agents decided not to shift the Presidential aircraft to the far side of the airport because the original landing area was secure +DUMMY/LJ024-0049.wav|in the administration of John Adams and Thomas Jefferson -- both signers of the Declaration of Independence +DUMMY/LJ032-0232.wav|The dates surrounding the taking of this picture and the purchase of the rifle +DUMMY/LJ040-0103.wav|An indication of the nature of Lee's character at this time was provided in the spring of nineteen fifty, +DUMMY/LJ008-0062.wav|and this floor is supported below, +DUMMY/LJ049-0070.wav|Any effort to prepare detailed contingency plans might well have the undesirable effect of inhibiting quick and imaginative responses. +DUMMY/LJ011-0081.wav|He was not prosecuted for this fraud on account of the respectability of his family, and soon after this escape +DUMMY/LJ003-0235.wav|even to prisoners almost in nakedness, and as a special charitable gift. Extortion was practiced right and left. +DUMMY/LJ003-0029.wav|Under the steward there were captains of wards, chosen in the same way, and performing analogous duties. +DUMMY/LJ019-0229.wav|This block was completed in eighteen fifty-nine, after which the hitherto unavoidable and long-continued promiscuous association of prisoners +DUMMY/LJ033-0195.wav|The single brown viscose fiber found in the bag matched some of the brown viscose fibers from the blanket in all observable characteristics. +DUMMY/LJ025-0094.wav|Starchy substances, cellulose and sugar, +DUMMY/LJ005-0050.wav|and in moral and religious duty, and which after a time sought to provide them with suitable situations, was supported entirely out of the funds of the Society. +DUMMY/LJ008-0262.wav|and with them indeed this dispatch amounted to undue precipitancy. +DUMMY/LJ045-0157.wav|Answer: I was angry, of course. He was not angry -- he was upset. I was angry. He tried very hard to please me. +DUMMY/LJ024-0141.wav|You who know me will accept my solemn assurance that in a world in which democracy is under attack, +DUMMY/LJ032-0274.wav|the rifle was kept among Oswald's possessions from the time of its purchase until the day of the assassination, +DUMMY/LJ029-0104.wav|Deputy Chief N. T. Fisher, and several other command officers to discuss details of the motorcade and possible routes. +DUMMY/LJ036-0076.wav|he would have boarded the bus at approximately twelve:forty p.m. and left it at approximately twelve:forty-four p.m. +DUMMY/LJ039-0095.wav|Zahm stated further, quote, +DUMMY/LJ021-0093.wav|second, the administrative function of code making and revision; and, third, the judicial function, which includes enforcement, +DUMMY/LJ018-0319.wav|A second was nevertheless set on foot, +DUMMY/LJ019-0171.wav|Another custom which had led to disorder was abolished; +DUMMY/LJ028-0085.wav|The eyes are suggestive of the Semitic; the nose is of the Greek type; +DUMMY/LJ045-0052.wav|which Mrs. Paine had made in part to give her an alternative to returning to the Soviet Union. Marina Oswald wrote to Mrs. Paine that, quote, +DUMMY/LJ035-0010.wav|As he turned the corner from Main onto Houston at a speed of about five to ten miles per hour, +DUMMY/LJ015-0216.wav|This was an important step, and they might easily be robbed some day when Burgess was the guard, provided only that they could be opened. +DUMMY/LJ044-0156.wav|Oswald apparently enclosed a note with her letter of July in which +DUMMY/LJ014-0047.wav|His demeanor was a strange compound of wickedness, falsehood, and deceit. +DUMMY/LJ002-0329.wav|the supply of which was, however, limited, and there were not always enough to give bedding to all. The stock was diminished by theft; +DUMMY/LJ025-0162.wav|will be vastly greater than the weight of the same substances contained in the bean from which it sprang. +DUMMY/LJ036-0075.wav|walked seven blocks directly to Murphy and Elm, and boarded a bus almost immediately, +DUMMY/LJ005-0197.wav|One borough prison consisted of nothing more than a couple of cells, about ten yards square, and absolutely nothing more. +DUMMY/LJ008-0187.wav|It was still greater at Fauntleroy's execution in eighteen twenty-four, when no less than one hundred thousand persons assembled, it was said. +DUMMY/LJ014-0066.wav|and by dint of usurious interest on small sums advanced to needy neighbors, had amassed as much as eight thousand pounds or ten thousand pounds. +DUMMY/LJ032-0215.wav|Shaneyfelt compared this negative with a negative which he made by taking a new picture with Oswald's camera. +DUMMY/LJ005-0241.wav|till the whole body of the slaves were manumitted in eighteen thirty-three. +DUMMY/LJ050-0214.wav|Although the Commission does not know whether the cases involved are entirely comparable, +DUMMY/LJ013-0140.wav|A robbery of a somewhat novel kind was executed in rather a bungling fashion by Ker, a sea-captain, +DUMMY/LJ013-0033.wav|The Wallaces were arrested, committed to Newgate, and tried at the Old Bailey. +DUMMY/LJ040-0145.wav|Dr. Hartogs did find Oswald to be a tense, withdrawn, and evasive boy who intensely disliked talking about himself and his feelings. +DUMMY/LJ025-0159.wav|and this plant will, in due time, flower and produce its crop of beans just as if it were grown in the garden or in the field. +DUMMY/LJ028-0191.wav|The old enemies of Babylon rejoiced. +DUMMY/LJ004-0143.wav|so that the culprit, in addition to his sentence, +DUMMY/LJ034-0122.wav|As will be discussed fully below, the Commission has concluded that this suspect was Lee Harvey Oswald. +DUMMY/LJ042-0056.wav|two. That in my own mind I have no attachments of any kind in the U.S. +DUMMY/LJ028-0052.wav|In six oh six, Nineveh, the old enemy of Babylon, fell, never to rise again. +DUMMY/LJ041-0115.wav|He was undesirably discharged from the Marine Corps Reserve, to which he had been assigned on inactive status following his transfer from active duty, +DUMMY/LJ032-0086.wav|In Oswald's personal effects found in his room at ten twenty-six North Beckley Avenue in Dallas +DUMMY/LJ016-0240.wav|This generally produces instantaneous death, although cases where complete fracture of the spinal cord occurred are said to be rare. +DUMMY/LJ028-0284.wav|When he found that Darius did indeed value it highly, he considered further with himself how he might make the deed his own, and be the man to take Babylon. +DUMMY/LJ042-0120.wav|As my Russian improves I become increasingly conscious of just what sort of a society I live in. +DUMMY/LJ037-0068.wav|taken at the time of his arrest. +DUMMY/LJ003-0263.wav|which might have done something towards increasing the prison receipts, had it not been appropriated by the turnkey who winked at this evasion of the rules. +DUMMY/LJ008-0101.wav|exuberant in talk and hissing hot from Pie Corner, where she had taken her morning dose of gin-and-bitters. +DUMMY/LJ030-0202.wav|Reaction by Secret Service Agents +DUMMY/LJ021-0161.wav|just as other countries have had them for over a decade. +DUMMY/LJ023-0115.wav|We want a Supreme Court which will do justice under the Constitution -- not over it. +DUMMY/LJ014-0081.wav|Manning gave the coup de grace with a crowbar. +DUMMY/LJ008-0248.wav|Another fight followed the score which had already taken place; this time two women were the combatants. +DUMMY/LJ028-0261.wav|At last when the time came for rebelling openly they did as follows: +DUMMY/LJ037-0065.wav|as short, stocky and having bushy hair. She stated that the man weighed about one hundred fifty pounds. +DUMMY/LJ037-0164.wav|All four bullets were fired from a weapon +DUMMY/LJ044-0020.wav|On August seventeen, nineteen sixty-three, he appeared briefly on a radio program and on August twenty-one, nineteen sixty-three, +DUMMY/LJ010-0015.wav|for the purpose of misappropriating funds or feloniously obtaining cash; +DUMMY/LJ033-0095.wav|Frazier testified that Oswald carried no lunch bag that day. +DUMMY/LJ008-0092.wav|the man of better birth could hope for no sympathy, whatever his crime. +DUMMY/LJ032-0043.wav|A bank deposit made on March thirteen, nineteen sixty-three, included an item of twenty-one dollars, forty-five cents. Klein's shipping order form +DUMMY/LJ011-0200.wav|and went on to say, "I do assure you, madam, that it shall be the anxious endeavor of my life to promote her happiness by every means in my power." +DUMMY/LJ029-0159.wav|quote, apparently will loop through the downtown area, probably on Main Street, en route from Dallas Love Field, end quote. +DUMMY/LJ042-0146.wav|Lee was always hot-tempered, and now this trait of character more and more prevented us from living together in harmony. +DUMMY/LJ015-0028.wav|the bank was insolvent, and instead of starting on a fresh life with a new name, it should then and there have closed its doors. +DUMMY/LJ049-0103.wav|whether or not the act is committed while the victim is in the performance of his official duties or on account of such performance. +DUMMY/LJ013-0006.wav|who was convicted in eighteen forty-one of uttering false exchequer bills to an almost fabulous amount. +DUMMY/LJ031-0186.wav|who advised that Mr. Johnson take the Presidential oath of office before the plane left Dallas. +DUMMY/LJ049-0043.wav|However, these agents could not reach the President's car when it was traveling at an appreciable rate of speed. +DUMMY/LJ007-0192.wav|inasmuch as it has aroused the attention of those upon whom parliamentary reports and grand jury presentments had hitherto failed to make the slightest impression. +DUMMY/LJ004-0175.wav|In this prison there was no female infirmary. +DUMMY/LJ034-0070.wav|During the morning of November twenty-two, Givens was working with the floor-laying crew in the southwest section of the sixth floor. +DUMMY/LJ037-0052.wav|At about four:thirty p.m., Mrs. Markham, +DUMMY/LJ019-0299.wav|but in one prison it was such that the prisoners' food was supplied daily from the neighboring inn, and the innkeeper's bill constituted the only accounts kept. +DUMMY/LJ028-0511.wav|Should you walk along the shore of the Euphrates at Babylon, you would still see the embankments which Nebuchadnezzar constructed of bricks bearing his name, +DUMMY/LJ002-0292.wav|the master's side, the cabin side, and the common side. Payment of a fee of three shillings gained the debtor admission to the two first named; +DUMMY/LJ016-0175.wav|I may be permitted to digress for a moment to give a few particulars concerning the last officially appointed hangman of the city of London. +DUMMY/LJ018-0157.wav|In the course of his remarks, however, he said, +DUMMY/LJ005-0036.wav|The objects it had in view were set forth in one of its earliest meetings. +DUMMY/LJ027-0021.wav|Whatever emphasis is laid upon the diversity of life, the essential unity of all organisms must not be forgotten. +DUMMY/LJ027-0117.wav|"Now, here again the former theory appears to be triumphant over the latter," says Romanes, +DUMMY/LJ049-0005.wav|Rigorous security precautions had been arranged at Love Field with the local law enforcement authorities by Agents Sorrels and Lawson. +DUMMY/LJ011-0121.wav|Suddenly, after being at the lowest depths, he emerged, and was found by his friends living in comfort in the Waterloo Road. +DUMMY/LJ009-0006.wav|Before sentence a prisoner has only to observe the regulations of the jail in order to remain neglected and unnoticed. +DUMMY/LJ017-0039.wav|is obviously of paramount importance, and the powers granted by this Act have been frequently put in practice since. +DUMMY/LJ018-0020.wav|A desperate and deadly struggle must have taken place in the carriage, and the stain of a bloody hand marked the door. +DUMMY/LJ033-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ031-0127.wav|a protective circle of Secret Service agents surrounded Vice President and Mrs. Johnson +DUMMY/LJ013-0148.wav|four thousand pounds +DUMMY/LJ006-0195.wav|This was admitted in evidence by the turnkeys, and was proved by the appearance of the prison tables, which bore the marks of gaming-boards deeply cut into them. +DUMMY/LJ045-0036.wav|While those considerations no doubt had an effect on Oswald's attitude toward his family it would seem that the need for support and sympathy +DUMMY/LJ001-0003.wav|For although the Chinese took impressions from wood blocks engraved in relief for centuries before the woodcutters of the Netherlands, by a similar process +DUMMY/LJ001-0043.wav|A further development of the Roman letter took place at Venice. +DUMMY/LJ029-0189.wav|The local, national, and international reaction to this incident evoked from Dallas officials and newspapers strong condemnations of the demonstrators. +DUMMY/LJ031-0213.wav|Under "Pathological Diagnosis" the cause of death was set forth as "Gunshot wound, head." +DUMMY/LJ017-0114.wav|The intestines were, however, preserved and sent for analysis, +DUMMY/LJ028-0379.wav|fearing the wild beasts and evil spirits which lurk among all old ruins, refused to pitch their tents there. +DUMMY/LJ018-0234.wav|As he possessed nearly unbounded influence over his mother, her accession to the property meant that William could dispose of it as he pleased. +DUMMY/LJ003-0340.wav|The committee does not seem to have yet understood that Newgate could be only and properly replaced +DUMMY/LJ037-0165.wav|with five lands and grooves and a right twist which were the rifling characteristics of the revolver taken from Oswald. +DUMMY/LJ046-0101.wav|and an inquiry whether improvements can be made which will reduce the risk of another such tragedy. +DUMMY/LJ029-0203.wav|greet the President of the United States with the warmth and pride that keep the Dallas spirit famous the world over, end quote, +DUMMY/LJ034-0082.wav|None of the Depository employees is known to have seen Oswald again until after the shooting. +DUMMY/LJ031-0036.wav|but then Kellerman, Greer, and Lawson lifted the President onto a stretcher and pushed it into trauma room one. +DUMMY/LJ005-0288.wav|when speaking more particularly of the borough jails. +DUMMY/LJ004-0208.wav|There were work-rooms for wool-washing, dyeing, carding, and spinning. +DUMMY/LJ002-0181.wav|to those in other London prisons, for Newgate was not the only place of durance for these unfortunate people. There were also the King's Bench, +DUMMY/LJ033-0017.wav|who also worked at the Depository. +DUMMY/LJ021-0020.wav|and transportation and trade, so great in the mass that each individual concerned in them is quite helpless by himself. +DUMMY/LJ008-0127.wav|As we were crossing the press yard, +DUMMY/LJ016-0223.wav|Capital convicts go to the gallows in their own clothing, and not in prison dress, unless the former is quite unfit to be worn. +DUMMY/LJ015-0075.wav|that ere long the whole management of the transfer department was entrusted to him. +DUMMY/LJ032-0112.wav|Oswald's Palmprint on Rifle Barrel +DUMMY/LJ028-0197.wav|Belshazzar, the King, he was really the King's son, gave a feast to a thousand of his nobles. +DUMMY/LJ003-0261.wav|only a few of whom had the commendable prudery to pass themselves off as the wives of prisoners. +DUMMY/LJ015-0150.wav|But his chief extravagance, his favorite folly, was the exercise of an ostentatious benevolence. +DUMMY/LJ016-0082.wav|The ward was one short of its number. What had become of the fellow? +DUMMY/LJ002-0016.wav|Previous to that date there had been seven hundred or eight hundred frequently, and once, in Mr. Akerman's time, one thousand. +DUMMY/LJ012-0286.wav|all of which, according to his own confession, made no doubt with the idea of exonerating Mrs. Gale, he accomplished without her assistance. +DUMMY/LJ027-0027.wav|First, it must be noted that some structures are not non-adaptive, that is, do not change to fit changed habits or conditions of life. +DUMMY/LJ006-0203.wav|and he had thus ample means of introducing to the prisoners the prohibited but always much-coveted and generally procurable weed. +DUMMY/LJ045-0185.wav|he asked Frazier for a ride to Irving that night, stating falsely that he wanted to pick up some curtain rods to put in an apartment. +DUMMY/LJ004-0194.wav|No irons were worn except as a punishment. +DUMMY/LJ037-0006.wav|A sixth did so the next day. +DUMMY/LJ007-0104.wav|the parliamentary agents of the members whose election was impugned, were implicated in aiding and abetting the others to abscond, and a Mr. O'Mally, +DUMMY/LJ006-0107.wav|he sold tea, coffee, sugar, tobacco, although prohibited, and extra beer. +DUMMY/LJ016-0383.wav|All almost without exception sleep soundly at night, except the first after sentence, +DUMMY/LJ047-0250.wav|Brian did not hear Hosty make any statement concerning Oswald's capacity to be an assassin +DUMMY/LJ003-0126.wav|The unfortunate man was compelled at length to adopt the habits of his associates; +DUMMY/LJ002-0085.wav|Security was provided for in these condemned cells by lining the substantial stone walls with planks studded with broad-headed nails; +DUMMY/LJ002-0026.wav|in the three years between eighteen thirteen and eighteen sixteen, +DUMMY/LJ027-0158.wav|But according to the opposite view no reason can be assigned why such should be the case. +DUMMY/LJ008-0247.wav|but ever the same form moved along in the fulfillment of his mission, in spite of all persecution. +DUMMY/LJ036-0044.wav|Oswald had rented a room from Mrs. Bledsoe about six weeks before, on October seven, but she had asked him to leave at the end of a week. +DUMMY/LJ014-0278.wav|The latter, when first taxed with his offense, protested his innocence boldly, and positively denied all knowledge of the affair; +DUMMY/LJ037-0199.wav|The mail-order form had a line for the name of a witness to attest that the person ordering the gun was a U.S. citizen and had not been convicted of a felony. +DUMMY/LJ015-0183.wav|His manner was generally self-possessed, but his face was marked with "uneasy earnestness," +DUMMY/LJ030-0151.wav|and issued his initial orders at twelve:thirty p.m. +DUMMY/LJ019-0092.wav|They were approved of and employed at some prisons, at others objected to because they were unproductive, +DUMMY/LJ006-0025.wav|France had sent Misseurs Beaumont and De Tocqueville, who subsequently published several interesting works on the subject. +DUMMY/LJ007-0057.wav|Turnkeys occasionally visited the press-yard, but its occupants were under little or no control. +DUMMY/LJ025-0006.wav|that the functions or workings of the organs of plants, animals or man are quite distinct, +DUMMY/LJ041-0176.wav|He recalled that Oswald had a specific interest in Latin America, particularly Cuba, +DUMMY/LJ010-0138.wav|and died in a manner strongly contrasting with that of his fellows. +DUMMY/LJ021-0198.wav|that the Constitution is but a barrier to progress instead of being the broad highway through which alone true progress may be enjoyed. +DUMMY/LJ009-0187.wav|Beneath is a door for the admission of the bodies of murderers and other felons. There were other public dissecting rooms for criminals. +DUMMY/LJ024-0116.wav|When the time comes for action, +DUMMY/LJ042-0102.wav|Reminiscent of his attitude toward his superiors in the Marine Corps, +DUMMY/LJ028-0006.wav|you ascend the hill to the plateau, and before you, as far as the eye can reach, stretches the great Arabian Desert. +DUMMY/LJ040-0161.wav|Dr. Hartogs recommended that Oswald be placed on probation on condition that he seek help and guidance through a child guidance clinic. +DUMMY/LJ026-0001.wav|The Science: History of the Universe, Volume five. Edited by Francis Rolt-Wheeler. Biology. Chapter eight. +DUMMY/LJ031-0205.wav|On the seventeenth floor of the Hospital, Mrs. Kennedy and the Attorney General joined other members of the Kennedy family to await the conclusion of the autopsy. +DUMMY/LJ002-0283.wav|and denied admission to the "charity wards," which partook of all the benefits of bequests and donations to poor debtors. +DUMMY/LJ004-0214.wav|Its evils were inherent and irremediable, but some ameliorating measures had been introduced, +DUMMY/LJ050-0075.wav|The Commission shares this concern. +DUMMY/LJ017-0145.wav|Dr. Smethurst was long an inmate of Newgate, and was tried at the Central Criminal Court. +DUMMY/LJ029-0168.wav|the Morning News noted that the motorcade would travel through downtown Dallas onto the Stemmons Freeway, and reported that, quote, the motorcade will move slowly +DUMMY/LJ007-0155.wav|to be subjected to the same baneful influences, and to suffer the same moral deterioration, whether ultimately convicted or set free. +DUMMY/LJ011-0284.wav|His captors having thus succeeded in their designs, left him, no doubt to realize the money. +DUMMY/LJ047-0136.wav|they had vacated their apartment, and Marina Oswald had departed with their child in a station wagon with Texas registration. +DUMMY/LJ018-0029.wav|A photograph of Müller shown the jeweler was identified as the likeness of the man who had exchanged Mr. Briggs' chain. +DUMMY/LJ014-0315.wav|through a dishonored cheque for three thousand pounds, paid over as an installment of eighteen thousand pounds owing for an advance on warrants. +DUMMY/LJ038-0158.wav|Jarman testified that he ate his lunch on the first floor around five minutes to twelve, and that he neither ate lunch with nor saw Oswald. +DUMMY/LJ050-0089.wav|While these tentative criteria are a step in the right direction, +DUMMY/LJ028-0516.wav|the strongest, the thickest, the loftiest, the most intricate, perhaps the most beautiful that ever protected a city, +DUMMY/LJ037-0215.wav|She also testified that this appeared to be the revolver seen in Oswald's belt +DUMMY/LJ006-0052.wav|The sum total thus produced was inconsiderable compared with the hundreds that had formerly filled the jail, +DUMMY/LJ030-0239.wav|Youngblood was not positive that he was in the rear seat before the second shot, but thought it probable because of President Johnson's statement +DUMMY/LJ011-0143.wav|and in defiance of the vote of the assembly which more accurately represented public opinion. It was indeed announced that their fate was sealed; +DUMMY/LJ007-0081.wav|whose proper place was the asylum. +DUMMY/LJ018-0172.wav|More than this, Griffiths took the police to a field where, in a bank, a number of other plates were secreted. +DUMMY/LJ016-0114.wav|A later case was still more remarkable, as it was effected after the alteration of the prison and its reconstruction on the newest lines. +DUMMY/LJ050-0210.wav|the caseload of each FBI agent averaged twenty to twenty-five, and he felt that this was high. +DUMMY/LJ008-0300.wav|The prisoners and their friends, therefore, were kept in a state of the most violent suspense for many hours, +DUMMY/LJ047-0116.wav|On August twenty-two, it learned that Oswald had appeared on a radio discussion program on August twenty-one. +DUMMY/LJ003-0209.wav|The prisoners in the press yard had free access during the day to the yard and large day room; +DUMMY/LJ005-0214.wav|Some reforms had certainly been introduced, such as the abolition of irons, already referred to, and the establishment of male and female infirmaries. +DUMMY/LJ033-0206.wav|Stombaugh confirmed that the rifle could have picked up fibers from the blanket and transferred them to the paper bag. +DUMMY/LJ042-0203.wav|not in the name of freedom or high ideals, +DUMMY/LJ009-0176.wav|Seven other crimes, however, were still capital by law, and so continued till the passing of the Criminal Consolidation Acts of eighteen sixty-one. +DUMMY/LJ037-0232.wav|As has been discussed previously, +DUMMY/LJ039-0038.wav|throughout April. The Commission asked Marina Oswald whether she might have misunderstood the object of her husband's threat. She stated, quote, +DUMMY/LJ015-0105.wav|Mr. Fasson waited and waited for his subordinate to re-appear, and at last discovered his flight. +DUMMY/LJ021-0143.wav|with small groups of those truly representative of large employers of labor and of large groups of organized labor, +DUMMY/LJ039-0029.wav|No edition of either Dallas newspaper during the period January one, nineteen sixty-three, to May fifteen, nineteen sixty-three, +DUMMY/LJ040-0041.wav|There was some quality about him that led him to act with an apparent disregard for possible consequences. +DUMMY/LJ009-0081.wav|For a while the preacher addresses himself to the congregation at large, who listen attentively +DUMMY/LJ039-0231.wav|Having fired this slot +DUMMY/LJ015-0082.wav|He took it for granted that the certificates representing these bogus shares, and which practically did not exist, would never be called for. +DUMMY/LJ039-0017.wav|I asked him where he was going, and why he was getting dressed. He answered Nixon is coming. I want to go and have a look, end quote. +DUMMY/LJ002-0048.wav|two. The female debtors' side consisted of a court-yard forty-nine by sixteen feet, +DUMMY/LJ017-0212.wav|Her captain was John Smith; +DUMMY/LJ028-0376.wav|As the centuries passed the mounds into which the city had turned grew higher and higher with the ruins of the huts later built upon them, +DUMMY/LJ029-0155.wav|As the date for the President's visit approached, +DUMMY/LJ025-0113.wav|The famous researches of Schwann and Schleiden in eighteen thirty-seven and the following years +DUMMY/LJ041-0190.wav|It got him in the newspapers. It did broadcast his name out, end quote. +DUMMY/LJ031-0168.wav|In another car Mrs. Johnson was driven to the airport accompanied by Secret Service agents and Representative Brooks. +DUMMY/LJ050-0233.wav|Since the assassination, the Service has experimented with the use of agents borrowed for short periods from such agencies. +DUMMY/LJ009-0155.wav|drawn to this terrible exhibition by the example of his father, the more celebrated Edmund Kean, who had witnessed the execution of Thistlewood +DUMMY/LJ027-0163.wav|Next, it forms first one pair and then another pair of legs; and meanwhile it begins to breathe also by lungs. +DUMMY/LJ007-0239.wav|both offenses and punishments affording a sufficient index to the practices going forward; and they wind up by declaring +DUMMY/LJ003-0016.wav|Bad cases were removed to a cell on the felons' side, and here they were locked in solitary confinement for three days at a time. +DUMMY/LJ027-0125.wav|in the Python we find very tiny rudiments of the hindlimbs. Now, +DUMMY/LJ009-0004.wav|would be astonished to observe the peculiar tenderness, I was going to add respect, +DUMMY/LJ018-0329.wav|their crimes follow in the lines of others already found, and often more than once, in the calendars. +DUMMY/LJ008-0290.wav|Of this I have treated at length in the last chapter. +DUMMY/LJ039-0144.wav|Oswald told her he was going to practice with it. +DUMMY/LJ026-0071.wav|Water and dissolved salts continually pass by diffusion from the soil into the roots, where together they constitute the sap. +DUMMY/LJ048-0021.wav|Question: This is the Quigley interview you are talking about? +DUMMY/LJ011-0064.wav|He was also attended by the Rev. Mr. Springett and the indefatigable Mr. Baker, whose name has already been mentioned. +DUMMY/LJ017-0227.wav|He was struck down, imploring mercy, but they beat him about the head and face +DUMMY/LJ008-0209.wav|Soon after midnight on the Sunday night, for by this time the present practice of executing on Monday morning had been pretty generally introduced, +DUMMY/LJ045-0057.wav|He specifically made that claim when his wife refused to come to live with him in Dallas as he asked her to do on the evening of November twenty-one, +DUMMY/LJ050-0098.wav|determination to use a means, other than legal or peaceful, to satisfy his grievance, end quote, within the meaning of the new criteria. +DUMMY/LJ050-0226.wav|eighteen agents are proposed for a rotating pool which will go through an intensive training cycle and also be available to supplement the White House detail +DUMMY/LJ002-0116.wav|The number of processes against debtors annually was extraordinary. +DUMMY/LJ050-0151.wav|it makes no use of the recent developments in automatic data processing which are widely used in the business world and in other Government offices. +DUMMY/LJ009-0016.wav|He seems to have improved the occasion when preaching the condemned sermon before Fauntleroy, by pointing a moral from that unhappy man's own case. +DUMMY/LJ027-0151.wav|The only alternative view is that as species of deer, +DUMMY/LJ019-0324.wav|The chaplain's duties were enlarged, and the principle of toleration accepted to the extent of securing to all prisoners +DUMMY/LJ019-0109.wav|Mr. Pearson backed up his recommendations by many sound arguments. +DUMMY/LJ033-0127.wav|The Commission has weighed the visual recollection of Frazier and Mrs. Randle against the evidence here presented +DUMMY/LJ023-0032.wav|If we learned anything from the depression +DUMMY/LJ028-0387.wav|The Sassanian kings of Persia were fond of hunting, and Babylon, then overgrown with trees, was their game preserve. +DUMMY/LJ026-0161.wav|with the part played by the same substances in starch formation (photosynthesis). +DUMMY/LJ007-0094.wav|had actually been returned as sane from the asylum to which they had been sent, and there was always some uncertainty as to who was mad and who not. +DUMMY/LJ010-0172.wav|He saw Prince Albert return there from a visit to Woolwich, and then passed on to Constitution Hill, +DUMMY/LJ026-0106.wav|The change is from starch to a sugar capable of diffusion. +DUMMY/LJ028-0497.wav|We do not know their height, for the statements of the ancient writers disagree. +DUMMY/LJ038-0092.wav|Oswald provided little information during his questioning. +DUMMY/LJ047-0006.wav|The FBI had been interested in him, to some degree at least, since the time of his defection in October nineteen fifty-nine. +DUMMY/LJ008-0293.wav|Sometimes the Privy Council did not meet for months, and during all that time the convicts languished with hope +DUMMY/LJ036-0191.wav|he would have reached tenth and Patton shortly after one:fifteen p.m. +DUMMY/LJ019-0206.wav|This jail was nominally to replace the Giltspur Street Compter, +DUMMY/LJ022-0145.wav|we have found from time to time more and more useful ways of promoting its purposes. +DUMMY/LJ017-0140.wav|In both the jury had no doubt as to the guilt of the accused, only in Smethurst's case the then Home Secretary, Sir George Cornewall Lewis, +DUMMY/LJ048-0258.wav|or in the slightest way prevent them from taking any action that might have averted the tragedy. +DUMMY/LJ047-0039.wav|From return to Fort Worth to move to New Orleans. +DUMMY/LJ014-0112.wav|A telegraphic message, then newly adapted to the purposes of criminal detection, +DUMMY/LJ022-0187.wav|but these twenty years have shown by experience definite possibilities for improvement. +DUMMY/LJ015-0311.wav|had not been guided by a sense of virtue, and directed to more honorable and useful pursuits. +DUMMY/LJ040-0153.wav|Dr. Hartogs summarized his report by stating: +DUMMY/LJ010-0246.wav|Francis had already been seized by a policeman named Trounce, who saw his movement with the pistol, but too late to prevent its discharge. +DUMMY/LJ003-0294.wav|This committee made its report in September the following year, and an excellent report it is, so far as its recommendations are concerned. +DUMMY/LJ031-0151.wav|Special Agent Lawson made certain that the Dallas police kept the public and press away from the immediate area of the hospital. +DUMMY/LJ028-0115.wav|I laid its foundation deep to the water level; +DUMMY/LJ009-0203.wav|it was determined that a public exhibition should be made of the body through the neighborhood which had been the scene of the monster's crimes. +DUMMY/LJ032-0242.wav|the revolver shipped from Los Angeles on the same date, and the two newspapers which he was holding in the picture. +DUMMY/LJ002-0199.wav|but in the twenty-seventh George the second the right of presentation was bought by the Crown for ten thousand, five hundred pounds. +DUMMY/LJ050-0201.wav|in the locality to assure adequate manpower for this task, as it is now doing. +DUMMY/LJ045-0222.wav|The feelings of hostility and aggression which seem to have played such an important, part in Oswald's life +DUMMY/LJ032-0156.wav|In a crevice between the butt plate of the rifle and the wooden stock +DUMMY/LJ028-0334.wav|This last victory gave him the finishing stroke to his power and made him all in all with the Babylonians: +DUMMY/LJ032-0023.wav|to Klein's Sporting Goods Co., of Chicago, Illinois. +DUMMY/LJ018-0207.wav|The community of coiners of which he was so notorious a member +DUMMY/LJ012-0236.wav|Bishop's confession was endorsed by Williams, and the immediate result was the respite of May. +DUMMY/LJ009-0026.wav|the experience I have had convinces me that one, and perhaps the only, good of an execution, i. e. the solemn admonition to the public, +DUMMY/LJ008-0259.wav|and of their demeanor after sentence until the last penalty was paid. +DUMMY/LJ028-0359.wav|Darius, who besieged the rebellious city twice, weakened it by destroying some of its walls. +DUMMY/LJ026-0114.wav|and even of proteids only a limited amount, seventy-five to one hundred grams a day for a man, is built into new protoplasm. +DUMMY/LJ020-0033.wav|It will not swell so fast as the white, so give yourself more time for making it. +DUMMY/LJ020-0106.wav|While your bread is taking a three hours’ rise, you are free in body and mind for other things. +DUMMY/LJ048-0157.wav|Agent Lawson agreed that such a procedure would assist him and other agents in fulfilling their responsibilities as advance agents. +DUMMY/LJ006-0272.wav|may be gathered from the statement that as many as three hundred were often admitted on the same day +DUMMY/LJ014-0206.wav|His captors, it may be added (Lerigo and Allen), were warmly commended by the judge for their courage and activity. +DUMMY/LJ002-0297.wav|on the master's side it was thirteen and fourpence, and a gallon of beer on entrance, although Mr. Newman, +DUMMY/LJ023-0059.wav|In its Preamble, the Constitution states that it was intended to form a more perfect Union and promote the general welfare; +DUMMY/LJ011-0065.wav|When led out on the morning of his execution, these two last-named gentlemen each took hold of one of his arms, and so accompanied him to the scaffold. +DUMMY/LJ008-0220.wav|hoary-headed and hardened in guilt, were still at large. +DUMMY/LJ012-0179.wav|The others followed, and on overtaking Thurtell, found he had done the job alone in a retired part of the road known as Gill's Hill Lane. +DUMMY/LJ006-0112.wav|he might keep him from the fire, or give him his soup or gruel in a pail instead of a basin. +DUMMY/LJ037-0149.wav|Cortlandt Cunningham, of the Firearms Identification Unit of the FBI Laboratory, testified +DUMMY/LJ012-0159.wav|He did not thrive on prison fare, +DUMMY/LJ046-0102.wav|This section considers first the means used to locate potential sources of danger to the President in time to take appropriate precautions. +DUMMY/LJ029-0184.wav|that President Kennedy would receive a "good welcome" and would not face demonstrations like those encountered +DUMMY/LJ031-0088.wav|This man was in obvious extreme distress and any more thorough inspection would have involved several minutes -- well, several +DUMMY/LJ036-0136.wav|What the hell. I wonder what the hell is the uproar? +DUMMY/LJ018-0376.wav|who was the ringleader, and who had long been an habitual criminal, a reputed murderer, and certainly an inmate more than once of a French bagne. +DUMMY/LJ001-0109.wav|this is best furthered by the avoidance of irrational swellings and spiky projections, and by the using of careful purity of line. +DUMMY/LJ030-0235.wav|As we were beginning to go down this incline, all of a sudden there was an explosive noise. +DUMMY/LJ050-0138.wav|In addition, the Department of the Treasury has requested five additional agents for its Protective Research Section +DUMMY/LJ022-0027.wav|The most difficult place in the world to get a clear open perspective of the country as a whole is Washington. +DUMMY/LJ038-0108.wav|that they had superimposed upon the photographs a rifle and a revolver. He told Fritz a number of times that the smaller photograph +DUMMY/LJ016-0186.wav|invited the man with the cough to enter a neighboring house and try a little peppermint for it. +DUMMY/LJ036-0086.wav|The Commission could not accept important elements of Craig's testimony. +DUMMY/LJ013-0122.wav|Stealing plate was about this period the crime of a more aristocratic thief. +DUMMY/LJ017-0214.wav|most of them, as was proved by their subsequent acts, blackguards of the deepest dye. +DUMMY/LJ018-0283.wav|and by means of forged letters of credit and introduction from London, obtained large sums from continental banks, in Berlin, +DUMMY/LJ044-0232.wav|after he had assassinated the American President, it does not appear that he had any plans to go there. +DUMMY/LJ017-0047.wav|the rest of the available space was allotted by ticket, to secure which the greatest influence was necessary. +DUMMY/LJ035-0020.wav|As he was parking he noted that people were, quote, falling, and they were rolling around down there grabbing their children, end quote, and rushing about. +DUMMY/LJ013-0050.wav|But among the charges on the estate he left +DUMMY/LJ002-0221.wav|and many reforms instituted. But the condition of the prison and its inmates remained unsatisfactory to the last. +DUMMY/LJ046-0246.wav|This was done in the case of the Nationalist Party of Puerto Rico, for example, but only after members of the group had resorted to political violence. +DUMMY/LJ043-0051.wav|he strongly put me in my place about buying things for his wife that he himself could not buy, end quote. +DUMMY/LJ032-0181.wav|Although Stombaugh was unable to estimate the period of time the fibers were on the rifle he said that the fibers, quote, +DUMMY/LJ038-0152.wav|Oswald told Fritz that after lunch he went outside, talked with Foreman Bill Shelley for five or ten minutes and then left for home. +DUMMY/LJ019-0247.wav|which existed in the various prisons, +DUMMY/LJ016-0225.wav|which his predecessors were not, and who were paid their wages over the gate to obviate the necessity for letting them enter. +DUMMY/LJ010-0005.wav|taking in preference those which are typical and invested with peculiar interest. It is somewhat remarkable that a marked change soon comes over the Calendar. +DUMMY/LJ031-0079.wav|He made the official determination because the ultimate cause of death, the severe head injury, was within his sphere of specialization. +DUMMY/LJ009-0127.wav|But what are the feelings of those who take part in it? +DUMMY/LJ003-0217.wav|The condemned occupied an open pew in the center of the chapel, hung with black; in front of them, upon a table, was a black coffin in full view. +DUMMY/LJ042-0187.wav|preparation in a special party could safeguard an independent course of action after the debacle, end quote. +DUMMY/LJ047-0042.wav|and unwilling to answer questions regarding his motive for going to the Soviet Union. +DUMMY/LJ018-0242.wav|The case was tried at Guildford Assizes, and caused intense excitement, +DUMMY/LJ014-0216.wav|and by getting his hand inside Dwyer's stock, with the idea of strangling him. +DUMMY/LJ009-0090.wav|The dying men are dreadfully agitated. +DUMMY/LJ003-0034.wav|This gradually was forced upon the consciousness of the Corporation, +DUMMY/LJ042-0116.wav|Under the entry for May one, nineteen sixty, +DUMMY/LJ045-0153.wav|Question: Did your husband give any reason for coming home on Thursday? +DUMMY/LJ040-0024.wav|Perhaps the most outstanding conclusion of such a study is that Oswald was profoundly alienated from the world in which he lived. +DUMMY/LJ008-0002.wav|I propose to return now to the subject of Newgate executions, +DUMMY/LJ027-0119.wav|surely if such a method were adopted in so many cases, we should expect that in consistency it would be adopted in all cases. +DUMMY/LJ027-0143.wav|whereby the primitive horn was gradually superseded by horns presenting a greater and greater number of prongs in successive species of extinct deer. +DUMMY/LJ048-0155.wav|Assistant Chief Batchelor of the Dallas police noted the absence of any formal statement by the Secret Service of specific work assigned to the police +DUMMY/LJ039-0166.wav|which was in turn placed to the right of the closest silhouette. +DUMMY/LJ009-0199.wav|About this time I find that the bodies of two murderers, Clench and Mackay, +DUMMY/LJ023-0057.wav|because the Articles of Confederation under which the original thirteen States tried to operate after the Revolution +DUMMY/LJ021-0053.wav|Only a very small minority of the people of this country believe in gambling as a substitute for the old philosophy of Benjamin Franklin +DUMMY/LJ040-0195.wav|It appears that he did not want to do any of the things which the authorities suggested in their efforts to bring him out of the shell +DUMMY/LJ032-0251.wav|Oswald loaded their possessions into a station wagon owned by Ruth Paine, who had invited Marina Oswald and the baby to live at her home in Irving, Texas. +DUMMY/LJ005-0278.wav|which was eventually to replace the attempted arrangement of prisoners by classes according to antecedents and crimes, +DUMMY/LJ004-0066.wav|or his health by forcing him at night into a damp, unventilated cell, with such crowds of companions as very speedily render the air foul and putrid; +DUMMY/LJ040-0104.wav|when he was sent to New Orleans to visit the family of his mother's sister, Mrs. Lillian Murret, for two or three weeks. +DUMMY/LJ014-0196.wav|by his identification by Cope in Westminster Hospital, who survived long enough to make a formal deposition before Mr. Jardine, +DUMMY/LJ037-0148.wav|in Oswald's possession to the exclusion of all other weapons. +DUMMY/LJ006-0264.wav|The nearly indiscriminate admission of visitors, although restricted to certain days, continued to be an unmixed evil. +DUMMY/LJ043-0053.wav|and became quite incensed with his wife when she would open the door for her in spite of his instructions to the contrary. +DUMMY/LJ012-0113.wav|After their arrest, Jordan's wife and Sullivan's brother came to the inn, and begged to be allowed to visit this room; +DUMMY/LJ038-0154.wav|Shelley denied seeing Oswald after twelve noon or at any time after the shooting. +DUMMY/LJ040-0233.wav|and when one of the city's clinics did find room to handle him, +DUMMY/LJ046-0185.wav|PRS attempts to eliminate serious risks by hospitalization or, where necessary, +DUMMY/LJ031-0232.wav|Shortly thereafter, the President's wife, family and aides left Bethesda Naval Hospital. +DUMMY/LJ028-0124.wav|I walled up its side with mortar and burned bricks, and I united it securely with the moat-walls of my father. +DUMMY/LJ030-0011.wav|At Rice Stadium a very large, enthusiastic crowd greeted the President. +DUMMY/LJ023-0125.wav|adequately to regulate commerce, protect public health and safety, and safeguard economic security. +DUMMY/LJ028-0418.wav|About that same time Pietro della Valle, an Italian, visited Babylon, +DUMMY/LJ008-0010.wav|at first, and for some few years after seventeen eighty-four, executions took place occasionally at a distance from Newgate. +DUMMY/LJ029-0151.wav|No arrangements were made for police or building custodians +DUMMY/LJ028-0075.wav|Therefore it was useless to hope that Nebuchadnezzar's portrait would be found on his palace walls. +DUMMY/LJ034-0191.wav|he saw nothing of importance, quote, except maybe one individual who was up there in the corner room of the sixth floor which was crowded in among boxes, end quote. +DUMMY/LJ047-0010.wav|the agent who was assigned his case at the time of the assassination, the Director of the FBI, +DUMMY/LJ015-0264.wav|The series of boldly-conceived and cleverly-executed forgeries in which James Townshend Saward, commonly called "Jem the Penman," +DUMMY/LJ011-0223.wav|Neither his address nor his pamphlet availed much, for the bill for the divorce passed both Houses. +DUMMY/LJ047-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ005-0003.wav|Persons committed to a metropolitan jail at that time were taken in gangs, men and women handcuffed together, or linked on to a long chain, +DUMMY/LJ016-0374.wav|Underfoot and upon the walls roughly cut into the stones, are single initial letters, the brief epitaphs of those who lie below. +DUMMY/LJ018-0327.wav|The most recent affairs are still too fresh in the public mind to need more than a passing reference. +DUMMY/LJ038-0015.wav|Mrs. Julia Postal, selling tickets at the box office of the Texas Theatre, +DUMMY/LJ004-0169.wav|On the dirty bedstead lay a wretched being in the throes of severe illness. +DUMMY/LJ017-0242.wav|Leon wished to make every one on board share and share alike, so as to implicate the innocent with the guilty; +DUMMY/LJ001-0171.wav|However, the fact must not be blinked that machine-made paper cannot in the nature of things be made of so good a texture as that made by hand. +DUMMY/LJ045-0048.wav|if he wanted me to go then that meant that he didn't love me, and that in that case what was the idea of coming to the United States in the first place. +DUMMY/LJ015-0267.wav|A man of birth and education, a member of the bar, and of acknowledged legal attainments, his proclivities were all downward. +DUMMY/LJ026-0017.wav|their nutritive processes result in oxidation, they have a definite organ of excretion and are capable of automatic movement +DUMMY/LJ038-0140.wav|He explained that a party for the Paine children had been planned for the weekend and he preferred not to be in the Paine house at that time; +DUMMY/LJ015-0174.wav|Thither police officers followed, only to find that he had returned to London. +DUMMY/LJ048-0196.wav|According to Captain Lawrence, quote, +DUMMY/LJ046-0164.wav|PRS investigated thirty-four newly established or reactivated cases concerning residents of Texas. +DUMMY/LJ012-0156.wav|His arrest and conviction cast dismay over the whole gang of receivers, and for a time seriously checked the nefarious traffic. +DUMMY/LJ026-0150.wav|In both animals and plants oxygen is essential to the catabolic part of metabolism. +DUMMY/LJ023-0140.wav|But all federal judges, once appointed, +DUMMY/LJ020-0104.wav|Master these, and do not let them master you. +DUMMY/LJ009-0101.wav|move with a quick, jerking motion, not naturally, but, as it were, like the affected parts of a galvanized corpse. +DUMMY/LJ039-0136.wav|After he left the Marines and before departing for Russia, Oswald, his brother, and a third companion went hunting for squirrels and rabbits. +DUMMY/LJ040-0093.wav|he accepted the responsibility for that decision himself and signed his mother's name to all his own excuses and report cards. +DUMMY/LJ024-0140.wav|It is my purpose to restore that balance. +DUMMY/LJ016-0108.wav|The Newgate officers obtained information of this, and went to the spot, where they effected the capture, +DUMMY/LJ033-0060.wav|Marina Oswald testified that at that time, quote, My heart dropped. +DUMMY/LJ005-0205.wav|At Eye, in Suffolk, the jail was part of the poor-house; so it was at Richmond, in Yorkshire, where the master of the workhouse was also keeper of the jail. +DUMMY/LJ014-0221.wav|On the contrary, many of them encouraged the brutal assailant in his savage attack. +DUMMY/LJ018-0071.wav|a proceeding which he vehemently resented, and begged that they might be removed, as the handcuffs hurt his wrists. +DUMMY/LJ025-0027.wav|and its appurtenances must needs vary according to the nature of the aliment and the operations which it has to undergo +DUMMY/LJ044-0196.wav|to a country in which he must have thought were embodied the political principles to which he had been committed for so long. +DUMMY/LJ019-0293.wav|In one prison the bedsteads had been removed lest the prisoners should break them up and convert them into weapons of offense. +DUMMY/LJ039-0130.wav|After reviewing Oswald's marksmanship scores, +DUMMY/LJ050-0032.wav|This involves tight and unswerving discipline as well as the promotion of an outstanding degree of dedication and loyalty to duty. +DUMMY/LJ017-0207.wav|had not quite disappeared. I will mention two cases of this class, one accompanied with piracy on the high seas, +DUMMY/LJ044-0011.wav|There, in late May and early June of nineteen sixty-three, under the name Lee Osborne, +DUMMY/LJ029-0183.wav|On October three the Dallas Morning News quoted U.S. Representative Joe Pool's hope +DUMMY/LJ008-0244.wav|now laughed at, the skirt of his seedy black coat torn across; yet, +DUMMY/LJ049-0187.wav|These proposals included suggestions to locate exclusive responsibility for all phases of the work +DUMMY/LJ039-0031.wav|Mr. Nixon advised the Commission that the only time he was in Dallas in nineteen sixty-three +DUMMY/LJ002-0008.wav|On the fourteenth June, eighteen hundred, there were one hundred ninety-nine debtors and two hundred eighty-nine felons in the prison. +DUMMY/LJ030-0126.wav|According to plan, the President's motorcade proceeded west through downtown Dallas on Main Street +DUMMY/LJ043-0186.wav|Marina Oswald indicated that her husband had compared General Walker to Adolph Hitler. +DUMMY/LJ044-0175.wav|Oswald left for Mexico City on September twenty-five, nineteen sixty-three, and arrived on September twenty-seven, nineteen sixty-three. +DUMMY/LJ034-0025.wav|as being the left palmprint and right index fingerprint of Lee Harvey Oswald. +DUMMY/LJ031-0162.wav|other terminal buildings and the neighboring parking lots, of all people. +DUMMY/LJ003-0216.wav|This latter was an ordeal which all dreaded, and many avoided by denying their faith. +DUMMY/LJ006-0237.wav|"A lad named Matthew White has had a wound in his eye by a bone thrown at him, which very nearly destroyed vision." +DUMMY/LJ047-0113.wav|For example, Oswald claimed that his wife's maiden name was Prossa +DUMMY/LJ016-0252.wav|Possibly such changes may be introduced into the method of execution +DUMMY/LJ017-0176.wav|and on her return had administered a dose of something which burnt the mouth badly, but did not prove fatal. +DUMMY/LJ019-0278.wav|This suggestion was adopted in the Act of eighteen sixty-five, which followed the committee's report, and of which more directly. +DUMMY/LJ042-0043.wav|Ask me and I will tell you I fight for communism. +DUMMY/LJ026-0157.wav|This process is, however, obscured during the day because of the oxygen freed in the manufacture of starch which goes on at that time. +DUMMY/LJ038-0266.wav|Frazier testified further that the FBI avoids the category of "probable" identification. +DUMMY/LJ016-0029.wav|Even after Williams had got out, persons were disposed to disbelieve that the escape had been accomplished in the manner indicated; +DUMMY/LJ036-0128.wav|The man was dressed in faded blue color khaki work clothes, a brown shirt, and some kind of work jacket that almost matched his pants. +DUMMY/LJ042-0105.wav|he took the position that the Communist Party officials in the Soviet Union were opportunists who were betraying their positions for personal gain. +DUMMY/LJ034-0150.wav|but, in his testimony before the Commission, Brennan stated that his remarks of January seven were intended by him merely as an accurate report +DUMMY/LJ008-0204.wav|One well-dressed woman fell out of a first-floor window on to the shoulders of the crowd below, but neither she nor any one else was greatly hurt. +DUMMY/LJ048-0185.wav|An attempt to cover only the most obvious points of possible ambush along the route in Dallas might well have included the Texas School Book Depository Building. +DUMMY/LJ037-0158.wav|He concluded that all of these cartridges were fired from the same weapon. +DUMMY/LJ042-0114.wav|of the presence of an American wore off and he began to be less the center of attention. +DUMMY/LJ005-0185.wav|As time passed, however, these magistrates made no effort at reform. +DUMMY/LJ050-0242.wav|even though it agrees with the Secret Service that it is preferable for the Service to have enough agents to handle all protective demands. +DUMMY/LJ045-0031.wav|During the week, Oswald lived in a roominghouse in Dallas, but he usually called his wife on the telephone twice a day. +DUMMY/LJ034-0063.wav|none of these employees except Oswald left identifiable prints on the cartons. +DUMMY/LJ043-0148.wav|She testified that she was agitated because she had found the note in Oswald's room, +DUMMY/LJ028-0069.wav|how he cast Hebrew lads into a fiery furnace and into the lions' den, +DUMMY/LJ010-0125.wav|and suffered in the usual way in front of Newgate, with the additional penalty of decapitation, as traitors, after they had been hanged. +DUMMY/LJ015-0242.wav|re-fastened and re-sealed the bullion box, and replaced it in the safe. +DUMMY/LJ001-0032.wav|and used an exceedingly beautiful type, which is indeed to look at a transition between Gothic and Roman, +DUMMY/LJ001-0177.wav|so that if the two are helpful to one another it is a mere matter of accident. +DUMMY/LJ023-0046.wav|so that their field might be plowed. +DUMMY/LJ028-0206.wav|he subdued the people, and wherever they collected, he slew them. +DUMMY/LJ017-0075.wav|There is every reason to suppose that he poisoned his wife to obtain possession of this sum upon her death. +DUMMY/LJ015-0040.wav|This borrowing continued, and on such a scale that their paper was soon at a discount, +DUMMY/LJ010-0155.wav|threw a stone at the king, which hit him on the forehead, but did no serious injury. +DUMMY/LJ002-0230.wav|The state of morals was disgraceful. Any woman obtained admission if sober, and if she got drunk she was not turned out. +DUMMY/LJ032-0262.wav|with the rifle wrapped inside and testified that it appeared to be the same approximate weight and shape as the package in his garage. +DUMMY/LJ013-0251.wav|Inquiry into the character of Good exposed him as a loose liver, who "kept company" with several women. +DUMMY/LJ049-0221.wav|perhaps upon recommendations based on further studies by the Cabinet-level committee recommended above or the National Security Council. +DUMMY/LJ003-0066.wav|As many various and, according to our ideas, heinous crimes came under this head, +DUMMY/LJ044-0225.wav|It should also be noted, however, that one witness testified that shortly before the assassination +DUMMY/LJ035-0095.wav|That Oswald descended by stairway from the sixth floor to the second-floor lunchroom +DUMMY/LJ034-0147.wav|Brennan stated that he was sure that the person firing the rifle was Oswald. +DUMMY/LJ003-0244.wav|At the commencement of every sessions, such of the untried as had purchased "easement" of irons were called up and re-fettered, +DUMMY/LJ018-0040.wav|His trial followed at the next sessions of the Central Criminal Court, and ended in his conviction. +DUMMY/LJ001-0115.wav|in short, it happens to this craft, as to others, that the utilitarian practice, though it professes to avoid ornament, +DUMMY/LJ046-0166.wav|An additional one hundred fifteen cases concerning Texas residents were established but not investigated. +DUMMY/LJ028-0338.wav|While the Babylonians, crowding to the walls, did their best to resist the Persian assault, +DUMMY/LJ049-0136.wav|of departmental policies relating to the national security) the responsibility to review and oversee the protective activities of the Secret Service +DUMMY/LJ009-0174.wav|In eighteen thirty-seven the death penalty was practically limited to murder or attempts to murder, +DUMMY/LJ029-0043.wav|and coordinating local activities with the White House staff, the organizations directly concerned with the visit, and local law enforcement officials. +DUMMY/LJ011-0072.wav|Some years were still to elapse before capital punishment ceased to be the penalty for forgery, and in the interval +DUMMY/LJ049-0181.wav|that too much emphasis is placed by both on the investigation of specific threats by individuals and not enough on dangers from other sources. +DUMMY/LJ045-0243.wav|He does not appear to have been able to establish meaningful relationships with other people. He was perpetually discontented with the world around him. +DUMMY/LJ041-0003.wav|Return to New Orleans and Joining the Marine Corps +DUMMY/LJ018-0374.wav|foreigners, vindictive, reckless, and truculent ruffians, easily moved to murderous rage. +DUMMY/LJ014-0189.wav|Lerigo turned for assistance to take the man into custody. +DUMMY/LJ029-0041.wav|who was the Secret Service official responsible for the entire Texas journey. +DUMMY/LJ022-0013.wav|It may seem confused to some, but out of the multitude of detailed parts that go into the making of the structure +DUMMY/LJ014-0050.wav|while Calcraft adjusted the noose. +DUMMY/LJ010-0288.wav|deemed valid as any palliation of his offense. +DUMMY/LJ047-0079.wav|The New Orleans office investigated and located Oswald, learning his address and former place of employment on August five, nineteen sixty-three. +DUMMY/LJ019-0047.wav|when thoughtful people who concerned themselves closely with social questions were greatly exercised as to the best system of treating the inmates of a jail. +DUMMY/LJ017-0018.wav|was ignorant of the destructive character of the powders, probably arsenic, which she administered. +DUMMY/LJ003-0009.wav|The day passed in revelry and drunkenness. Although spirituous liquors were forbidden, +DUMMY/LJ014-0085.wav|the corpse lying just inside the kitchen all the while, the two set to work to strip the body and hide it under the stones of the floor. +DUMMY/LJ038-0233.wav|He showed her a notebook three days later containing photographs of General Walker's home and a map of the area where the house was located. +DUMMY/LJ037-0113.wav|Callaway picked up the gun. +DUMMY/LJ029-0084.wav|Lawson also evaluated the security hazards at the Trade Mart on November thirteen. +DUMMY/LJ039-0158.wav|were capable of firing the shots which were fired by the assassin on November twenty-two, nineteen sixty-three. +DUMMY/LJ024-0052.wav|in accordance with a clearly defined principle relating to a clearly defined age limit. +DUMMY/LJ037-0265.wav|positively identified Lee Harvey Oswald as the man they saw fire the shots or flee from the scene, +DUMMY/LJ008-0164.wav|The most affecting and distressing scene was at Green Arbor Lane, just opposite the debtors' door of the prison. +DUMMY/LJ005-0285.wav|This committee, as well as the one preceding it, also reported in terms of strong reprobation on the small prisons and jails +DUMMY/LJ050-0101.wav|Leon F. Czolgosz, +DUMMY/LJ038-0067.wav|As Oswald, handcuffed, was led from the theatre, he was, according to McDonald, quote, +DUMMY/LJ009-0277.wav|For a second or two the body hung motionless, then, with a strength that astonished the attendant officials, +DUMMY/LJ030-0180.wav|Roy Kellerman, in the right front seat of the limousine, heard a report like a firecracker pop. +DUMMY/LJ018-0154.wav|He came to complete a sale of forged notes, and he wanted a couple of hundred pounds for what he had. +DUMMY/LJ046-0123.wav|for bizarre reasons or by writing or in some other way attempting to communicate with him in a threatening or abusive manner +DUMMY/LJ033-0104.wav|When Frazier entered the building, he did not see Oswald. +DUMMY/LJ027-0110.wav|Now, rudimentary organs of this kind are of such frequent occurrence, that almost every species of organism presents one or more of them +DUMMY/LJ040-0028.wav|When he was in the United States he resented the capitalist system which he thought was exploiting him and others like him. +DUMMY/LJ031-0095.wav|Before this was accomplished the President's cardiac activity had ceased and closed cardiac massage was instituted, which made it impossible to inspect his back. +DUMMY/LJ017-0257.wav|and the prisoners eventually surrendered to the British authorities, brought to England, and lodged in Newgate. +DUMMY/LJ048-0119.wav|The only systematic supervision of the activities of the advance agent +DUMMY/LJ011-0149.wav|The abduction of Miss Turner by the brothers Wakefield bore a strong resemblance to the carrying off and forcible marrying of heiresses as already described. +DUMMY/LJ018-0246.wav|At first the case was contested hotly, but, to the profound astonishment of every one inside and outside the court, +DUMMY/LJ039-0040.wav|When asked later whether it might have been Mr. Johnson, she said, quote, Yes, no. +DUMMY/LJ040-0022.wav|Indications of Oswald's motivation +DUMMY/LJ028-0411.wav|dressing the same, eating the same food as did their ancestors when Nebuchadnezzar built the walls of Babylon. +DUMMY/LJ008-0145.wav|They were accused by a confederate, who, goaded by conscience, had turned approver, of the murder of a Mr. Steele, +DUMMY/LJ015-0198.wav|It was no longer possible to stop a coach or carriage, or rob the postman who carried the mail. +DUMMY/LJ046-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By +DUMMY/LJ028-0117.wav|My father built the moat-wall of the Arachtu canal securely with mortar and bricks. +DUMMY/LJ002-0179.wav|and costing eleven pounds, fifteen shillings, eight pence, within ten days. +DUMMY/LJ010-0284.wav|He always kept the blinds down at his chambers in Jermyn Street; and as the St. James's clock chimed quarter-past three, +DUMMY/LJ035-0053.wav|The man turned and walked back toward Baker. +DUMMY/LJ028-0447.wav|Upon the little mound Jumjuma farther on, an Arab village has long stood. +DUMMY/LJ003-0236.wav|Garnish continued to be demanded long after it had disappeared in other and better-regulated prisons. +DUMMY/LJ028-0134.wav|He also erected what is called a pensile paradise: +DUMMY/LJ025-0125.wav|as Dr. Burdon Sanderson's interesting investigations have shown, by a disturbance of the electrical state of the contractile substance +DUMMY/LJ006-0062.wav|In another part there were men charged with and convicted of unnatural offenses shut up with lads of tender years; +DUMMY/LJ017-0203.wav|This fact was referred to by the judge in his summing up, +DUMMY/LJ011-0132.wav|except the forgery of wills and powers of attorney. +DUMMY/LJ041-0082.wav|in an attempt to attract attention to himself and to support his exaggerated idea of his own abilities. +DUMMY/LJ015-0159.wav|In his person he was neat and fastidious; +DUMMY/LJ034-0192.wav|He said that this was a white man about average in size, quote, possibly thin, end quote, and that he thought the man had light-brown hair. +DUMMY/LJ035-0117.wav|Harold Norman, and Bonnie Ray Williams -- were watching the parade from the fifth floor, directly below the window from which the shots were fired. +DUMMY/LJ013-0042.wav|the foundations of which had been laid by buying old ships on purpose to cast them away. +DUMMY/LJ028-0028.wav|Everywhere the valley is dotted with the mounds of buried cities carefully guarding the secrets of the centuries of long ago. +DUMMY/LJ012-0244.wav|The decapitation and dismemberment, the bestowal of the remains in various parts of the town, the preservation of the head in spirits of wine, +DUMMY/LJ038-0052.wav|testified regarding the arrest of Oswald, as did the various police officers who participated in the fight. +DUMMY/LJ013-0031.wav|The crime soon became public. +DUMMY/LJ021-0182.wav|And let it be recorded that the British bankers helped. +DUMMY/LJ028-0370.wav|The priests' deception was unavailing, and in two seventy-five B.C., the inhabitants of Babylon were transported to Seleucia. +DUMMY/LJ037-0032.wav|Benavides never saw Oswald after the arrest. +DUMMY/LJ031-0029.wav|Special Agent Hill removed his suit jacket and covered the President's head and upper chest to prevent the taking of photographs. +DUMMY/LJ028-0112.wav|My father did that which no previous king had done. +DUMMY/LJ003-0279.wav|There was fear as to the unrestricted use of tools, +DUMMY/LJ010-0130.wav|Ings was especially defiant. He sought to cheer Davidson, who seemed affected, crying out, +DUMMY/LJ007-0047.wav|"Men whose cases were dangerous, and those most seriously inclined, complained of these annoyances," +DUMMY/LJ005-0255.wav|The committee was also to report upon the manner in which sentences were carried out, and to recommend any alterations necessary in the rules +DUMMY/LJ038-0062.wav|Johnny Brewer testified he saw Oswald pull the revolver and the officers struggle with him to take it away +DUMMY/LJ011-0250.wav|While thus engaged, Howard thrust the poker into the fire. +DUMMY/LJ012-0016.wav|At no period could thieves in London or elsewhere have prospered had they been unable to dispose of their ill-gotten goods. +DUMMY/LJ048-0013.wav|he indicated that he had learned his lesson, +DUMMY/LJ005-0101.wav|whence it deduced the practice and condition of every prison that replied. +DUMMY/LJ008-0078.wav|seemed to strike her mind with horror and consternation, to the exclusion of all power of recollectedness in preparation for the approaching awful moment. +DUMMY/LJ026-0131.wav|Wherever there is need for new protoplasm, whether to repair previous waste or to supply materials for growth, +DUMMY/LJ022-0111.wav|I well realize that the country is expecting before this year is out to see the "dirt fly", as they say, in carrying on the work, +DUMMY/LJ029-0103.wav|Lawson and Sorrels met with Dallas Chief of Police Jesse E. Curry, Assistant Chief Charles Batchelor, +DUMMY/LJ010-0256.wav|Undeterred by the well-merited punishment which had overtaken Francis, +DUMMY/LJ045-0186.wav|He must have planned his attack at the very latest prior to Thursday morning when he spoke to Frazier. +DUMMY/LJ018-0121.wav|was the case of the forgeries upon the Bank of England discovered in eighteen sixty-three, +DUMMY/LJ028-0285.wav|Noble exploits in Persia are ever highly honored and bring their authors to greatness. +DUMMY/LJ037-0150.wav|that he compared the four empty cartridge cases found near the scene of the shooting with a test cartridge fired from the weapon in Oswald's possession when he was arrested. +DUMMY/LJ018-0070.wav|Sattler was ironed for safe custody, +DUMMY/LJ032-0046.wav|The rifle without the scope cost only twelve dollars, seventy-eight cents. +DUMMY/LJ003-0225.wav|were meager or altogether absent. +DUMMY/LJ027-0066.wav|All vertebrates, with few exceptions, and no other animals, have two and only two pair of limbs. +DUMMY/LJ045-0160.wav|Question: And how did he show that he was upset? +DUMMY/LJ014-0267.wav|others that he had had extraordinary luck as a gold-digger. Had his West End and little-informed associates followed him into the city, +DUMMY/LJ028-0041.wav|tore down its palaces and temples and walls, and scraped even the foundations of the city into the river. +DUMMY/LJ045-0189.wav|it should be noted that mention of the Trade Mart as the expected site of the Presidential luncheon +DUMMY/LJ017-0027.wav|that arsenic was within her reach in the house, +DUMMY/LJ012-0166.wav|where the principal actor suffered death, the case hardly comes within the limits of my subject. +DUMMY/LJ018-0310.wav|The prisoners, who were committed to Newgate for trial, +DUMMY/LJ036-0083.wav|and told Captain Fritz that Oswald was the man he saw. +DUMMY/LJ024-0079.wav|But the welfare of the United States, and indeed of the Constitution itself, is what we all must think about first. +DUMMY/LJ019-0316.wav|Hard labor of the first and second class was carefully defined. +DUMMY/LJ041-0055.wav|a dissatisfaction which seemed to find expression at this particular point +DUMMY/LJ007-0209.wav|they return to the charge, and again call the corporation to task for their mismanagement of Newgate. +DUMMY/LJ017-0117.wav|The examination of the stomach betrayed the presence of antimony in large quantities, but no strychnia, +DUMMY/LJ039-0191.wav|Oswald engaged in such practice. If the assassin missed either the first or third shot, +DUMMY/LJ036-0070.wav|rather than wait for one which stopped across the street from his roominghouse. +DUMMY/LJ005-0263.wav|and not, as heretofore, to the judges of assize; that, both to check abuses and watch the progress of improvement, +DUMMY/LJ029-0031.wav|then we had the time, and I withdrew my objections to a motorcade. End quote. +DUMMY/LJ003-0313.wav|instruct the prisoners, quote, give spiritual advice and administer religious consolation, end quote. to all who might need them; +DUMMY/LJ010-0061.wav|That some thirty or more needy men should hope to revolutionize England is a sufficient proof of the absurdity of their attempt. +DUMMY/LJ003-0083.wav|Each ward was calculated to hold twenty-four, allowing each individual one foot and a half; +DUMMY/LJ018-0217.wav|As such he had pretty general control over his father's estates and affairs. +DUMMY/LJ045-0059.wav|The instability of their relations was probably a function of the personalities of both people. Oswald was overbearing in relations with his wife. +DUMMY/LJ050-0126.wav|the Secret Service, through its field offices, +DUMMY/LJ049-0229.wav|while assuming no radical relocation of responsibilities, +DUMMY/LJ019-0209.wav|the situation having been proved by long experience to be salubrious. +DUMMY/LJ030-0098.wav|The remainder of the motorcade consisted of five cars for other dignitaries, including the mayor of Dallas and Texas Congressmen, +DUMMY/LJ028-0059.wav|His military career began while he was still the crown prince, and his father was on the throne. +DUMMY/LJ025-0146.wav|And the question whether plants are provided with a nervous system or not thus acquires a new aspect and presents the histologist and physiologist +DUMMY/LJ046-0137.wav|PRS expressed its interest in receiving information on suspects in very general terms. For example, +DUMMY/LJ027-0170.wav|Finally, it loses the tail, or rather its tail is absorbed and its material used in further development, and it becomes a perfect frog, +DUMMY/LJ018-0255.wav|The case was easily and rapidly disposed of. +DUMMY/LJ042-0071.wav|He was taken to a hospital in Moscow where he was kept until October twenty-eight, nineteen fifty-nine. +DUMMY/LJ033-0040.wav|Before dinner on November twenty-one, Oswald played on the lawn of the Paines' home with his daughter June. +DUMMY/LJ034-0151.wav|of what he said on November twenty-two. +DUMMY/LJ008-0043.wav|Round the north, west, and south sides are erected galleries for the reception of officers, attendants, etc. +DUMMY/LJ050-0177.wav|This PRS agent will also be responsible for establishing an informal local liaison committee +DUMMY/LJ048-0096.wav|In the case of Dallas, because President Kennedy had scheduled visits to five Texas cities +DUMMY/LJ026-0134.wav|The particles of this newly formed compound are incorporated into the protoplasm. +DUMMY/LJ047-0091.wav|any foreign government or instrumentality thereof. End quote. +DUMMY/LJ015-0313.wav|Saward spent all his share at low gaming houses, and in all manner of debaucheries. +DUMMY/LJ039-0033.wav|An investigation failed to reveal any invitation extended to Mr. Nixon during the period when Oswald's threat reportedly occurred. +DUMMY/LJ016-0419.wav|Kate Webster, who was tried at the Central Criminal Court, and passed through Newgate, although she suffered at Wandsworth, +DUMMY/LJ003-0060.wav|which was a wooden flooring on a slightly inclined plane, with a beam running across the top to serve as a pillow. +DUMMY/LJ020-0076.wav|Wrap bowl and biscuit in a thick cloth and set to rise where it will neither become chilled nor sour over night. +DUMMY/LJ050-0122.wav|It should be made clear that the Secret Service will in no way seek to duplicate the intelligence +DUMMY/LJ034-0141.wav|and he said that the shots came from inside the building, end quote. +DUMMY/LJ016-0399.wav|Wainwright was allowed a cigar the night before execution, which he smoked in the prison yard, walking up and down with the governor, Mr. Sydney Smith. +DUMMY/LJ021-0051.wav|people will be discouraged in unhappy efforts to get rich quick by speculating in securities. +DUMMY/LJ037-0251.wav|It seemed to her that the jacket Oswald wore was darker than Commission Exhibit Number one sixty-two. +DUMMY/LJ009-0008.wav|the keepers, and the sheriffs all seem interested in his fate, and he can make no reasonable request that is not at once granted by whomsoever he may address. +DUMMY/LJ040-0047.wav|That associate did not think that Oswald was a Communist. +DUMMY/LJ043-0133.wav|Attack on General Walker +DUMMY/LJ016-0036.wav|About fifty feet from the ground level, and above the cistern, a revolving chevaux-de-frise of iron was fixed, +DUMMY/LJ042-0242.wav|requesting information on how to subscribe to Russian newspapers and magazines and asked for, quote, +DUMMY/LJ039-0172.wav|None of the marksmen had any practice with the assassination weapon except for exercising the bolt for two or three minutes on a dry run. +DUMMY/LJ005-0098.wav|The Society did not shrink from its self-imposed duty, but continued year after year, with unflagging energy and unflinching spirit, to watch closely +DUMMY/LJ042-0033.wav|of his concomitant hatred of the United States, which was most clearly expressed in his November twenty-six, +DUMMY/LJ048-0230.wav|and by a Secret Service investigation. +DUMMY/LJ013-0002.wav|As the century advanced crimes of fraud increased. +DUMMY/LJ011-0122.wav|His funds, which he pretended came to him with a rich wife, were really the proceeds of frauds upon the Bank of England. +DUMMY/LJ046-0156.wav|The general files of PRS consist of folders on individuals, card indexed by name. +DUMMY/LJ019-0330.wav|Dietaries were drawn up for adoption on the recommendation of a committee of experts. +DUMMY/LJ008-0314.wav|The form of imparting the intelligence was generally the same. +DUMMY/LJ048-0146.wav|Discussions with the Dallas authorities and requests made of them were entirely informal. +DUMMY/LJ007-0103.wav|Mr. J. E. Sparrow and Mr. Clipperton +DUMMY/LJ014-0104.wav|Six inches lower the body of O'Connor was uncovered. +DUMMY/LJ005-0075.wav|which having gone through several sessions, at last became law in eighteen twenty-three to four +DUMMY/LJ006-0187.wav|It was, moreover, in his opinion undoubtedly the greatest nursery of crime in London. +DUMMY/LJ038-0007.wav|One of the persons who heard the sirens was Johnny Calvin Brewer, manager of Hardy's Shoestore, a few doors east of the Texas Theatre. +DUMMY/LJ009-0231.wav|Although these objectionable practices had disappeared, +DUMMY/LJ016-0351.wav|A more marked change from the old scene can hardly be conceived. Instead of the roar of the brutalized crowd, +DUMMY/LJ014-0246.wav|Thus, when a payment was made by the company, the amount disbursed was carried to account in the general books from its entry in the passbook, +DUMMY/LJ047-0041.wav|Agent Fain reported to headquarters that Oswald was impatient and arrogant, +DUMMY/LJ019-0283.wav|Beds might well be made hard and their use strictly limited. +DUMMY/LJ030-0182.wav|"My God, I am hit," and saw both of the President's hands move up toward his neck. +DUMMY/LJ026-0067.wav|We have here the direct absorption into the body proper of food-stuffs precisely as the animal takes in water and oxygen. +DUMMY/LJ031-0002.wav|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter two. The Assassination: Part three. +DUMMY/LJ019-0370.wav|but if these conditions were observed, the Secretary of State could deal sharply enough with the defaulting jurisdictions. +DUMMY/LJ005-0219.wav|female prisoners were still exposed to the full view of the males, the netting in front of the gallery being perfectly useless as a screen. +DUMMY/LJ050-0156.wav|this money would be used to compensate consultants, to lease standard equipment or to purchase specially designed pilot equipment. +DUMMY/LJ011-0024.wav|Meanwhile public gossip was busy with Fauntleroy's name, +DUMMY/LJ025-0084.wav|And if we substitute for the possession of an alimentary cavity the power of taking solid nutriment into the body and there digesting it, +DUMMY/LJ045-0154.wav|Answer: He said that he was lonely because he hadn't come the preceding weekend, and he wanted to make his peace with me. +DUMMY/LJ045-0167.wav|Question: What did you say to that? +DUMMY/LJ016-0038.wav|in the wall above the chevaux-de-frise projected a series of iron spikes sharp enough to forbid further ascent. +DUMMY/LJ048-0150.wav|while the Secret Service representatives in Dallas +DUMMY/LJ008-0080.wav|She was immediately tied by the neck to an iron bolt fixed near the top of the stake, +DUMMY/LJ004-0038.wav|It was in eighteen seventeen that a small band of philanthropists resolved to form themselves into an association for the improvement of prison discipline. +DUMMY/LJ010-0282.wav|Pate was said to be an eccentric person, given to strange acts and antics, such as mixing whiskey and camphor with his morning bath-water, +DUMMY/LJ005-0105.wav|It was long before the many jurisdictions imitated the few. +DUMMY/LJ002-0025.wav|Returns laid before the House of Commons showed that six thousand, four hundred thirty-nine persons had been committed to Newgate +DUMMY/LJ033-0117.wav|The disassembled rifle was too long to be carried in this manner. +DUMMY/LJ039-0081.wav|Referring to a rifle with a four-power telescope, Sergeant Zahm said, quote, +DUMMY/LJ014-0028.wav|and he was in possession, the first time to her knowledge, of a watch. +DUMMY/LJ019-0296.wav|Complete idleness was tolerated; there was neither penal labor nor light employment. +DUMMY/LJ038-0271.wav|It was a six point five-millimeter bullet +DUMMY/LJ023-0121.wav|How then could we proceed to perform the mandate given us? +DUMMY/LJ024-0007.wav|with the approval, as required by the Constitution, of the Senate of the United States. +DUMMY/LJ001-0106.wav|oddity rather than rational beauty and meaning being apparently the thing sought for both in the letters and the illustrations. +DUMMY/LJ015-0115.wav|Little more remains to be said about Robson. He appears to have accepted his position, and to have at once resigned himself to his fate. +DUMMY/LJ019-0332.wav|The administration of good government was to be watched over by the local magistracy, certain of whom, styled visiting justices, +DUMMY/LJ016-0066.wav|Denied admission everywhere as a ragged, half-naked beggar, +DUMMY/LJ013-0191.wav|All this was evidence sufficient to warrant Courvoisier's committal for trial; +DUMMY/LJ022-0113.wav|to make a major attack upon the problem of unemployment. +DUMMY/LJ027-0085.wav|The existence of great similarities in vertebrate structure is not always fully recognized. +DUMMY/LJ030-0220.wav|David Powers, who witnessed the scene from the President's follow-up car, +DUMMY/LJ011-0165.wav|who had plausibly explained that he had only recently been engaged at Shrigley. +DUMMY/LJ027-0087.wav|to be entirely different machines, made each for its own purposes, at once, out of hand. +DUMMY/LJ009-0274.wav|His wretched, abject condition, seated in a chair under the drop, was such as almost to unnerve the executioner Calcraft, +DUMMY/LJ027-0126.wav|is it a worthy conception of Deity that, +DUMMY/LJ015-0110.wav|After dinner he went into hiding for a day or two, +DUMMY/LJ047-0223.wav|I don't recall the exact date. It was about a week prior. End quote. +DUMMY/LJ005-0251.wav|Again in eighteen thirty-five prisons and their inmates became once more the care of the senate, and the subject was taken up this time by the House of Lords. +DUMMY/LJ002-0046.wav|and were occupied as a rule by ten to fifteen people when the prison was not crowded, but double the number was occasionally placed in them. +DUMMY/LJ023-0103.wav|government is to be rendered impotent. +DUMMY/LJ038-0130.wav|as one entitled to receive mail. +DUMMY/LJ007-0010.wav|by their presence and their pious, disinterested efforts, to restrain the dissolute manners and vicious language of the unhappy and depraved inmates. +DUMMY/LJ036-0080.wav|toward a light-colored Rambler station wagon, which was moving slowly along Elm toward the underpass: +DUMMY/LJ011-0003.wav|sold out so large an amount of stock, that he paid sixteen thousand pounds a year in dividends to escape detection. +DUMMY/LJ031-0178.wav|two Dallas officials informed members of the President's start that the body could not be removed from the city until an autopsy was performed. +DUMMY/LJ047-0224.wav|In fact, Hosty participated in transmitting to the Secret Service two pieces of information pertaining to the visit. +DUMMY/LJ007-0087.wav|at Appleby for thirteen years, at Anglesea for fifteen years, at Exeter for sixteen years, and at Pembroke +DUMMY/LJ018-0249.wav|Every word he uttered was said with consideration, and sometimes with a long pause, +DUMMY/LJ037-0144.wav|All three identified Exhibit Number one forty-three as the revolver taken from Oswald when he was arrested. +DUMMY/LJ005-0117.wav|Numbers of the jails were still unprovided with chaplains, and the prisoners never heard Divine service. +DUMMY/LJ028-0449.wav|Parts of the walls of Nineveh are still standing to the height of one hundred and twenty-five feet, +DUMMY/LJ017-0234.wav|verified the murderers, and then shut himself up in his cabin. +DUMMY/LJ015-0312.wav|The proceeds of these forgeries amounted, it was said, to some thousands per annum. +DUMMY/LJ028-0344.wav|for Cyrus had done neither the one nor the other when he took Babylon. +DUMMY/LJ018-0052.wav|But on the gallows, when the cap was over his eyes and the rope had been adjusted round his neck, and within a second of the moment when he would be launched into eternity, +DUMMY/LJ033-0022.wav|During the morning of November twenty-one, Oswald asked Frazier whether he could ride home with him that afternoon. +DUMMY/LJ018-0109.wav|The aged but wary Kerp, the moment the plot was discovered, vanished, and was never more heard of. +DUMMY/LJ021-0147.wav|under which wages, hours and working conditions +DUMMY/LJ033-0131.wav|Frazier's view of the bag was from the rear. He continually advised that he was not paying close attention. For example, he said, quote, +DUMMY/LJ002-0041.wav|Two other wards were appropriated to the master's side debtors; they were each twenty-three feet by fourteen and a half, +DUMMY/LJ042-0066.wav|My fondest dreams are shattered because of a petty official. I decide to end it. +DUMMY/LJ018-0240.wav|were fictitious documents. His next brother, who should have inherited under the authentic will, +DUMMY/LJ039-0173.wav|They had not even pulled the trigger because of concern about breaking the firing pin. +DUMMY/LJ038-0305.wav|The finding that Lee Harvey Oswald attempted to murder a public figure in April nineteen sixty-three +DUMMY/LJ029-0136.wav|through the underpass and leads into an access road, +DUMMY/LJ030-0093.wav|Vice-Presidential follow-up car. +DUMMY/LJ038-0037.wav|Oswald then struck McDonald between the eyes with his left fist; with his right hand he drew a gun from his waist. +DUMMY/LJ048-0139.wav|The Service had twenty-eight agents participating in the Dallas visit. +DUMMY/LJ007-0145.wav|Instead of seclusion and meditation, his time is passed in the midst of a body of criminals of every class and degree, in riot, debauchery, and gaming, +DUMMY/LJ012-0292.wav|Mrs. Brown fell with it, and Greenacre, to his horror, found that she was dead. +DUMMY/LJ008-0123.wav|"Sir," he answered, "they sent me the very riff-raff." +DUMMY/LJ046-0010.wav|when his temporary residence, Blair House, was attacked by Puerto Rican Nationalists. +DUMMY/LJ002-0121.wav|Fifteen thousand of these were issued in Middlesex alone, which at that time was reckoned as only a fifteenth of Great Britain. +DUMMY/LJ046-0070.wav|Whatever their purposes Presidential journeys have greatly enlarged and complicated the task of protecting the President. +DUMMY/LJ046-0182.wav|Individuals who are regarded as dangerous to the President +DUMMY/LJ038-0165.wav|At approximately nine p.m., on April ten, nineteen sixty-three, in Dallas, Texas, Maj. Gen. Edwin A. Walker, +DUMMY/LJ018-0014.wav|Strange to say, the unfortunate man was not yet dead, and he actually survived more than four-and-twenty hours. +DUMMY/LJ027-0089.wav|The simplest, in fact the only scientific, explanation of the phenomena of vertebrate structure is the idea of a primal vertebrate, +DUMMY/LJ021-0155.wav|both directly on the public works themselves, and indirectly in the industries supplying the materials for these public works. +DUMMY/LJ017-0119.wav|All the circumstances were so suspicious that he could not escape the criminal charge. +DUMMY/LJ003-0110.wav|They viewed him with some suspicion, as one of whom they knew nothing. +DUMMY/LJ011-0161.wav|This purported to be from the medical attendant of Mr. Turner, written at Shrigley, Mr. Turner's place of residence; +DUMMY/LJ028-0035.wav|seems to have been the first to mention Babylon, and one of his inscriptions seems to say that he built the city and gave it its name. +DUMMY/LJ037-0140.wav|L.J. Lewis said in an interview that because of the distance from which he observed the gunman he would hesitate to state whether the man was identical with Oswald. +DUMMY/LJ026-0100.wav|In the plant the inorganic matter in water from the soil are absorbed by the roots and carried up definite tubes in the woody part of the stem. +DUMMY/LJ042-0117.wav|he noted that one of his acquaintances, quote, relates many things I do not know about the U.S.S.R. +DUMMY/LJ034-0036.wav|the bottom of the palm rested on the box. +DUMMY/LJ008-0143.wav|This was notably the case at the execution of Holloway and Haggerty +DUMMY/LJ007-0230.wav|Newgate, Giltspur St. Compter, and the City Bridewell, +DUMMY/LJ032-0119.wav|but only after Day viewed the knob on the bolt through a magnifying glass and found no prints. +DUMMY/LJ016-0416.wav|Many, like Wainwright, were calm and imperturbable throughout their trying ordeal. +DUMMY/LJ049-0160.wav|Since that time, the Secret Service has had and exercised responsibility for the physical protection of the President +DUMMY/LJ049-0143.wav|Major needs of personnel or other resources might be met more easily on its recommendation than they have been in the past. +DUMMY/LJ018-0104.wav|also a furniture dealer and one or two more commonplace rogues. The arch villain was never taken into custody. +DUMMY/LJ030-0173.wav|he instinctively turned to his right because the shot appeared to come from over his right shoulder. +DUMMY/LJ015-0277.wav|When he could get nothing but the blank cheque, he set in motion all sorts of schemes for obtaining signatures, such as +DUMMY/LJ009-0212.wav|The countenance of Williams was ghastly in the extreme, and the whole had an appearance too horrible for description. +DUMMY/LJ025-0148.wav|"Thus it must be admitted," he says again, "that plants may be contractile and locomotive; +DUMMY/LJ008-0033.wav|Ten were executed +DUMMY/LJ038-0020.wav|Brewer told Mrs. Postal that he had seen the man ducking into his place of business and that he had followed him to the theatre. +DUMMY/LJ001-0165.wav|One fruitful source of badness in paper +DUMMY/LJ020-0103.wav|are better than mere personal strength in the accomplishment of such tasks as fall to women to perform. +DUMMY/LJ046-0176.wav|that an individual presents a significant danger to the life of the President, his name is placed in a "trip index file" +DUMMY/LJ004-0046.wav|An active committee was appointed, comprising many names already well known, some of them destined to become famous in the annals of philanthropy. +DUMMY/LJ035-0144.wav|Oswald's departure from building. +DUMMY/LJ019-0381.wav|in another there was half-heartedness, even apathy and an almost complete contempt for the provisions of the act. +DUMMY/LJ045-0011.wav|by some rightwing groups centered in Dallas or any other general atmosphere of hate +DUMMY/LJ008-0008.wav|was in the right direction, as making the performance shorter and diminishing the area of display. +DUMMY/LJ002-0014.wav|The numbers soon increased, however, and by eighteen eleven had again risen to six hundred twenty-nine; and Mr. Neild was told that there had been at one time +DUMMY/LJ037-0237.wav|When interviewed by FBI agents on January twenty-one, nineteen sixty-four, +DUMMY/LJ016-0251.wav|and must be strongly deprecated on moral grounds, as tending to the utter degradation of one individual. +DUMMY/LJ031-0207.wav|The Secret Service established a communication system with the White House and screened all telephone calls and visitors. +DUMMY/LJ026-0023.wav|nitrogen either from simple salts or from proteids, and the process of nutrition is one of oxidation. +DUMMY/LJ013-0217.wav|varied so considerably in the several documents he left behind, that too much reliance cannot be placed upon his confession. +DUMMY/LJ031-0032.wav|Although he was reclining in his wife's arms, he lurched forward in an effort to stand upright and get out of the car, but he collapsed again. +DUMMY/LJ025-0020.wav|although the parts of some plants exhibit oscillating movements without any perceptible cause and the leaves of others retract when touched, +DUMMY/LJ038-0262.wav|Robert A. Frazier, an FBI ballistics identification expert, testified that he was, quote, unable to reach a conclusion, end quote, +DUMMY/LJ050-0033.wav|The Commission emphasizes that it finds no causal connection between the assassination +DUMMY/LJ014-0111.wav|At the second exact information was obtained of Mrs. Manning's movements. She had gone to Edinburgh. +DUMMY/LJ041-0072.wav|that he was a man of great ability and intelligence and that many of his superiors in the Marine Corps were not sufficiently competent to give him orders. +DUMMY/LJ050-0093.wav|because of Oswald's demonstrated hostility toward the Secretary of the Navy in his letter of January thirty, nineteen sixty-two. Quote, +DUMMY/LJ010-0184.wav|Oxford was seized by a person named Lowe, who was at first mistaken for the assailant. +DUMMY/LJ010-0235.wav|he would be in some way provided for, he having been for some time previously in abject circumstances. +DUMMY/LJ032-0053.wav|control number V C eight three six, serial number C two seven six six, was shipped parcel post to +DUMMY/LJ034-0057.wav|In evaluating the significance of these fingerprint and palmprint identifications, +DUMMY/LJ049-0109.wav|that the hostile act occur while the victim is engaged in or because of the performance of official duties. +DUMMY/LJ026-0115.wav|The excess undergoes oxidation and forms nitrogen excretions. +DUMMY/LJ011-0093.wav|This phial he had managed to retain in his possession in spite of the frequent searches to which he was subjected in Newgate. +DUMMY/LJ027-0040.wav|between the fin of a fish and the paddle of a whale +DUMMY/LJ020-0025.wav|One quart of Graham flour, one cup of white flour. +DUMMY/LJ003-0296.wav|many of the indispensable conditions of a model prison according to modern ideas. +DUMMY/LJ039-0215.wav|Frazier added that the scope would cause a slight miss to the right. +DUMMY/LJ032-0214.wav|Using a recognized technique of determining whether a picture was taken with a particular camera, +DUMMY/LJ016-0035.wav|the wall beneath and above it was "rusticated," in other words, the granite surface had become roughened, and offered a sort of foothold. +DUMMY/LJ041-0036.wav|He was a little dismayed at this, and he said that he couldn't find any that would show any interest in him as a Communist, +DUMMY/LJ033-0123.wav|that the bag she saw Oswald carrying, quote, wasn't that long, I mean it was folded down at the top as I told you. It definitely wasn't that long, end quote, +DUMMY/LJ007-0235.wav|instead of ventilating the apartments, they only serve to convey the offensive effluvia arising from the decaying animal matter into the dining-rooms. +DUMMY/LJ017-0095.wav|The purchase by Palmer of strychnia was proved. +DUMMY/LJ041-0160.wav|I certainly don't. I don't think with any kind of formal argument you could have shaken that conviction. And that is why I say irrevocable. +DUMMY/LJ050-0254.wav|The Secret Service in the past has sometimes guarded its right to be acknowledged as the sole protector of the Chief Executive. +DUMMY/LJ007-0175.wav|but in opposition to the recorded denunciations of authority, and in defiance of the express enactments of the law, +DUMMY/LJ003-0137.wav|and I learned he had been up with the others the whole night. +DUMMY/LJ008-0271.wav|This was more particularly the practice in London. +DUMMY/LJ014-0177.wav|and he must have been at the end of his resources when he committed the crime for which he suffered. +DUMMY/LJ008-0090.wav|It was still the custom to offer warm encouragement or bitter disapproval, according to the character and antecedents of the sufferer. +DUMMY/LJ038-0186.wav|You will find the mailbox in the post office which is located four blocks from the drugstore on that street. +DUMMY/LJ050-0099.wav|It is apparent that a good deal of further consideration and experimentation will be required before adequate criteria can be framed. +DUMMY/LJ029-0192.wav|On October twenty-six the press reported Chief of Police Curry's plans +DUMMY/LJ033-0067.wav|She pointed to the blanket which was on the floor very close to where Ruth Paine was standing. +DUMMY/LJ030-0221.wav|stated that Mrs. Kennedy would probably have fallen off the rear end of the car and been killed if Hill had not pushed her back into the Presidential automobile. +DUMMY/LJ031-0108.wav|Rubber tubes were inserted between the second and third ribs to reexpand the right lung, which had collapsed because of the opening in the chest wall. +DUMMY/LJ012-0114.wav|but their request, in spite of their earnest entreaties, +DUMMY/LJ019-0076.wav|This variety was often extended to all branches of prison economy. +DUMMY/LJ034-0060.wav|who, like Oswald, might have handled the cartons. They were also compared with the prints of those law enforcement officials who might have handled the cartons. +DUMMY/LJ017-0281.wav|As the first to appear would have some time to wait for his fellows, a difficult and painful ordeal, +DUMMY/LJ037-0036.wav|Just prior to the shooting, Mrs. Helen Markham, a waitress in downtown Dallas, was about to cross tenth Street at Patton. +DUMMY/LJ044-0082.wav|His attempt to express himself through his Fair Play for Cuba activities, however, +DUMMY/LJ050-0059.wav|he new instructions require FBI agents to report immediately information concerning, quote, +DUMMY/LJ041-0004.wav|After his return to New Orleans Oswald was teased at school because of the northern accent which he had acquired. +DUMMY/LJ026-0084.wav|i.e., capable of decomposition into simpler, stabler molecules in which stronger affinities are satisfied. +DUMMY/LJ037-0011.wav|was eating lunch in his cab which was parked on Patton facing the southeast corner of tenth Street and Patton Avenue a few feet to the north. +DUMMY/LJ048-0212.wav|Lawson stated that he, quote, was looking back a good deal of the time, +DUMMY/LJ010-0013.wav|forgeries committed systematically, and for long periods, as in the case of Fauntleroy, +DUMMY/LJ027-0130.wav|these evidences have been in the past thirty years somewhat overshadowed by the far more surprising evidences +DUMMY/LJ016-0112.wav|All three after recapture passed on, as originally intended, to Leicester, where they did their "bit" and were released; +DUMMY/LJ016-0115.wav|A sailor, Krapps by name, occupied one of the upper cells in the new block. +DUMMY/LJ027-0068.wav|which probably represent the vertebrate condition before limbs were acquired; +DUMMY/LJ028-0163.wav|which are, like the great gates in the outer wall, of brass, and open on the water. +DUMMY/LJ010-0269.wav|whose description, as given by his father to the police, +DUMMY/LJ035-0054.wav|He had been proceeding toward the rear of the lunchroom. +DUMMY/LJ028-0313.wav|The men upon the towers, whose business it was to keep a lookout, +DUMMY/LJ049-0101.wav|Punish the murder or manslaughter of, attempt or conspiracy to murder, kidnaping of and assault upon +DUMMY/LJ032-0117.wav|He lifted the rifle by the wooden stock after his examination convinced him that the wood was too rough to take fingerprints. +DUMMY/LJ037-0141.wav|When Oswald was arrested, he had in his possession a Smith and Wesson thirty-eight Special caliber revolver, +DUMMY/LJ036-0170.wav|If he was discharged at Neely and Beckley and walked directly to his roominghouse, +DUMMY/LJ003-0184.wav|all the contagion and profligacy which prevailed in this part of the prison, end quote. +DUMMY/LJ010-0191.wav|After his arrest he thought only of the excitement his attempt had raised, nothing of its atrocity, +DUMMY/LJ004-0135.wav|twenty men slept on eight straw beds, with sixteen rugs amongst them, and a piece of timber for a bolster. +DUMMY/LJ030-0022.wav|Upon concluding the conversation, the President prepared to depart for Dallas. +DUMMY/LJ015-0107.wav|Meanwhile the absconding clerk had coolly driven to a favorite dining-place in the West End, +DUMMY/LJ039-0026.wav|she stated that she might have been confused about shutting him in the bathroom, but that, quote, there is no doubt that he got dressed and got a gun, end quote, +DUMMY/LJ016-0386.wav|many of the most abandoned murderers snore peacefully their eight hours, even on the night immediately preceding execution. +DUMMY/LJ014-0129.wav|The case rested upon the facts which have been already set forth, and was proved to the satisfaction of the jury, who brought in a verdict of guilty. +DUMMY/LJ012-0130.wav|brought in a man-of-war from Brazil had been transhipped at Falmouth for conveyance to London. +DUMMY/LJ012-0282.wav|This, wrapped up in a silk handkerchief, +DUMMY/LJ039-0062.wav|the shots were at a slow-moving target proceeding on a downgrade in virtually a straight line +DUMMY/LJ014-0077.wav|It came out in the husband's confession that Mrs. Manning induced O'Connor to go down to the kitchen to wash his hands, that she followed him to the basement, +DUMMY/LJ011-0201.wav|The game, however, was nearly up. Miss Turner was met by her uncle on Calais pier as she was walking with Wakefield. +DUMMY/LJ005-0150.wav|In some county jails, as I have already said, female prisoners were placed upon the tread-wheel; +DUMMY/LJ016-0402.wav|No woman could resist him, he calmly assured Mr. Smith that night as they walked together, and he recounted his villanies one by one. +DUMMY/LJ042-0183.wav|As a result he thought it would be, quote, necessary to oppose the old systems but at the same time support their cherished traditions, end quote. +DUMMY/LJ028-0178.wav|the walls of Babylon were so long and wide and high that all who saw them were amazed. +DUMMY/LJ032-0089.wav|It certified that Lee Harvey Oswald had been vaccinated for smallpox on June eight, nineteen sixty-three. +DUMMY/LJ049-0209.wav|The Commission was not asked to apply itself as did the Hoover Commission in nineteen forty-nine, +DUMMY/LJ002-0069.wav|to the various wards their friends occupied. +DUMMY/LJ018-0396.wav|The greatest causes célèbre, however, of recent times were the turf frauds by which the Comtesse de Goncourt was swindled +DUMMY/LJ022-0158.wav|By far the greater part of the general decline in utility securities had occurred before I was inaugurated. +DUMMY/LJ027-0065.wav|enclosed and protected by the skeleton, viz., the neural cavity above, and the visceral or body cavity below, the vertebral column. +DUMMY/LJ032-0238.wav|Marina Oswald testified that the photographs were taken on a Sunday about two weeks before the attempted shooting of Maj. Gen. Edwin A. Walker +DUMMY/LJ013-0132.wav|On a search of the prisoner's lodgings in Allington Street, a silver fork was found, +DUMMY/LJ031-0060.wav|Dr. Robert N. McClelland entered at that point and assisted Dr. Perry with the tracheotomy. +DUMMY/LJ025-0022.wav|From the mobility of animals Cuvier, with his characteristic partiality for teleological reasoning, +DUMMY/LJ018-0111.wav|They advertised for clerks, and employed the most likely of the applicants by sending them to the bank. +DUMMY/LJ039-0086.wav|I consider it a real advantage, particularly at the range of one hundred yards, in identifying your target. +DUMMY/LJ002-0265.wav|left by a Mr. Henry Allnutt, who was long a prisoner in the Marshalsea, and came into a fortune while there. +DUMMY/LJ018-0044.wav|There was the prisoner's poverty, his inability to account for himself on the night of the murder, and his possession of the property of the murdered man. +DUMMY/LJ038-0069.wav|At one:fifty-one p.m., police car two reported by radio that it was on the way to headquarters with the suspect. +DUMMY/LJ033-0084.wav|It was more bulky toward the bottom, end quote, than toward the top. +DUMMY/LJ019-0354.wav|That no such contract was valid between one jurisdiction and another unless the Secretary of State was satisfied that the prison it was intended to use +DUMMY/LJ045-0047.wav|and I told him that +DUMMY/LJ003-0178.wav|The female felons' wards I shall describe at length in the next chapter, +DUMMY/LJ002-0285.wav|and was so unfit to sleep on that it had not been used for many years, so that the men and women associated together indiscriminately. +DUMMY/LJ011-0076.wav|His offense was uttering forged notes, and there was strong suspicion that he had long subsisted entirely by this fraud. +DUMMY/LJ033-0133.wav|and I didn't pay too much attention on how he carried the package at all, end quote. +DUMMY/LJ048-0098.wav|there were not enough men available to permit two agents to be assigned to all the advance work. +DUMMY/LJ011-0129.wav|Maynard was the only one who suffered death. +DUMMY/LJ013-0022.wav|The ship, however, met with exasperatingly fine weather, and it was not until the captain reached the West India Islands +DUMMY/LJ039-0194.wav|All three of the firers in these tests +DUMMY/LJ043-0094.wav|This was at least in part because of the close physical confines in which some of the work had to be done. +DUMMY/LJ049-0228.wav|the Commission believes that the facts of the assassination of President Kennedy point to certain measures which, +DUMMY/LJ038-0073.wav|Hill testified that Fritz told the detective to get a search warrant, go to an address on Fifth Street in Irving, +DUMMY/LJ018-0383.wav|He was glad, therefore, to get an extra allowance of bread, +DUMMY/LJ002-0214.wav|was a prison for debtors and persons committed for contempt by the courts of Chancery, Exchequer, and Common Pleas. +DUMMY/LJ007-0110.wav|The rest were detained for more than a month, it being considered that they were the most guilty, as being either professional agents, who advised the others to abscond, +DUMMY/LJ011-0244.wav|through whose interest an appointment under Government was to be obtained for Mullay, would be present. +DUMMY/LJ050-0096.wav|I ask you to look into this case and take the necessary steps to repair the damage done to me and my family. End quote. +DUMMY/LJ008-0021.wav|for the murder of his cabin-boy, whom he stabbed after much ill-usage on board the ship as it lay in the Tagus. +DUMMY/LJ043-0028.wav|Oswald attempted to get his wife to come back and, over Bouhe's protest, De Mohrenschildt finally told him where she was. +DUMMY/LJ033-0106.wav|believed that he saw Oswald coming to work, but he does not remember that Oswald had anything in his hands as he entered the door. +DUMMY/LJ048-0269.wav|and special agents cooperating with them, or by special agents on similar assignments, while they are in a travel status, is prohibited. End quote. +DUMMY/LJ013-0107.wav|but he escaped through a back door on to the river, and rowed off in a boat to a hiding-place in the woods. +DUMMY/LJ011-0102.wav|He had gone on board in his Quaker dress, but when captured was found in a light-green frock, +DUMMY/LJ021-0111.wav|There may be a serious question as to the wisdom of many of those devices to control production, +DUMMY/LJ018-0069.wav|where the arrest was made. +DUMMY/LJ050-0143.wav|county, and State law enforcement agencies in their districts. +DUMMY/LJ015-0127.wav|All the signatures in the transfer were forged. Not only did he thus transfer and realize "bogus" stock +DUMMY/LJ006-0138.wav|They had a standing bedstead to sleep on, and a good flock mattress; double allowance of provisions, filched from the common stock. +DUMMY/LJ046-0051.wav|From George Washington to John F. Kennedy, such journeys have been a normal part of the President's activities. +DUMMY/LJ014-0019.wav|His name was Hocker; he was by trade a ladies' shoemaker; and it was also ascertained that after the day of the murder he was flush of money. +DUMMY/LJ023-0042.wav|But to the far-sighted it is far-reaching in its possibilities of injury to America. +DUMMY/LJ045-0062.wav|he made no attempt to help her and there are other indications that he did not want her to learn that language. +DUMMY/LJ011-0017.wav|The impression which his appearance altogether was calculated to make was that of the profoundest commiseration. +DUMMY/LJ048-0006.wav|or to the Vice President, end quote. +DUMMY/LJ046-0142.wav|but again the standards were very general. +DUMMY/LJ035-0012.wav|At about this time he heard the first shot. +DUMMY/LJ005-0020.wav|Among those whom the Society found arrayed against it was Sydney Smith, +DUMMY/LJ033-0193.wav|he found, on the inside, a single brown delustered viscose fiber and several light green cotton fibers. +DUMMY/LJ034-0115.wav|of the man who shot Patrolman J. D. Tippit. +DUMMY/LJ007-0120.wav|some to the infirmary, many more to the governor's house. +DUMMY/LJ048-0091.wav|Advance preparations. +DUMMY/LJ013-0178.wav|Further suspicion was caused by the position of a book and a wax candle by the bedside. +DUMMY/LJ028-0105.wav|Tradition says that to please his foreign wife from the mountainous country he built the famous hanging gardens, but that may be only a tradition. +DUMMY/LJ011-0231.wav|Miss Turner subsequently married Mr. Legh of Lym Hall, Cheshire. +DUMMY/LJ048-0246.wav|Each of the agents who visited the Press Club or the Cellar Coffee House (apart from the three members of the midnight shift) +DUMMY/LJ015-0185.wav|When the jury found a verdict of guilty he remained unmoved. He listened without emotion to the judge's well-merited censures, +DUMMY/LJ025-0124.wav|but in the plants the act of contraction has been found to be accompanied, +DUMMY/LJ049-0092.wav|without authorizing it to arrest anyone who harms him. The same provisions authorize the Service to arrest without warrant +DUMMY/LJ043-0187.wav|She testified that Oswald said that General Walker, quote, was a very bad man, that he was a fascist, +DUMMY/LJ010-0094.wav|bound, as it happened, to the Archbishop of York's. +DUMMY/LJ004-0243.wav|The bulk of the prisoners were still left in idleness. +DUMMY/LJ047-0157.wav|I asked her if she knew where he worked. +DUMMY/LJ002-0176.wav|Neild gives a list of the various items charged upon a debt of ten pounds, which included instructions to sue, +DUMMY/LJ049-0086.wav|It is anomalous that Congress has legislated in other ways touching upon the safety of the Chief Executive or other Federal officers, +DUMMY/LJ005-0141.wav|the sleeping space per head was only sixteen inches, and often as many as two hundred ninety-three men had to be accommodated on barrack beds +DUMMY/LJ032-0159.wav|these fibers were examined by Paul M. Stombaugh, a special agent assigned to the Hair and Fiber Unit of the FBI Laboratory. +DUMMY/LJ028-0512.wav|but the river walls have disappeared, and the buttresses of the bridges have been torn or washed away. +DUMMY/LJ008-0095.wav|It was said that so large a crowd had never collected since the execution of Mrs. Brownrigg, nor had the public indignation risen so high. +DUMMY/LJ016-0341.wav|The sufferer was stolid and reticent to the last. +DUMMY/LJ028-0294.wav|no stranger's hands have wrought this work on me, but my own only. +DUMMY/LJ047-0152.wav|He conducted a limited background investigation of the Paines, intending to interview Mrs. Paine and ask her particularly about Oswald's whereabouts. +DUMMY/LJ020-0013.wav|Add a few spoonfuls at a time, the potato-water alternately with the flour by the handful, +DUMMY/LJ005-0093.wav|The second act, passed in the following year, enlarged and amended the first, and at the same time gave powers to the House +DUMMY/LJ006-0257.wav|hot meat, poultry, and fish were also forbidden. +DUMMY/LJ003-0223.wav|The roof of the female prison, says the grand jury in their presentment in eighteen thirteen, let in the rain. +DUMMY/LJ048-0281.wav|Yet their role of protecting the President is so important to the well-being of the country +DUMMY/LJ009-0049.wav|For this offense the punishment is death. +DUMMY/LJ032-0060.wav|testified that the signature and other writing on the application for that box were in the handwriting of Lee Harvey Oswald +DUMMY/LJ050-0112.wav|and the leading State and local agencies that might be a source of such information. +DUMMY/LJ036-0049.wav|And, after we got past Akard, at Murphy -- I figured it out. Let's see. I don't know for sure. +DUMMY/LJ010-0305.wav|Its embarrassments were greatly increased by the bankruptcy of two of its clients in the building trade, +DUMMY/LJ040-0084.wav|dwelling on the fact that she was a widow with three children. +DUMMY/LJ018-0158.wav|I am the man that has got all the bank paper; I have thirty thousand pounds now, and the Bank of England cannot stop it. +DUMMY/LJ043-0085.wav|Oswald said that he hoped eventually to develop qualifications for employment as a junior executive +DUMMY/LJ032-0005.wav|and that the weapon which fired these bullets was a Mannlicher-Carcano +DUMMY/LJ030-0252.wav|Secret Service procedure requires that each agent stay with the person being protected +DUMMY/LJ045-0099.wav|She had, quote, the impression that he was just pushed, pushed, pushed, and she Marina Oswald was probably nagging, nagging, nagging, end quote. +DUMMY/LJ005-0270.wav|which should be paid into general prison funds, and no part of them handed over to the prisoners themselves. +DUMMY/LJ045-0127.wav|He absolved the Soviet Embassy in Mexico City of any blame for his difficulties there. He advised the Washington Embassy +DUMMY/LJ026-0130.wav|and may thus be easily transported to all parts of the plant. +DUMMY/LJ019-0240.wav|But no structural alterations were made from the date first quoted until the time of closing the prison in eighteen eighty-one. +DUMMY/LJ028-0517.wav|walls which no ancient army was ever able to take by storm. +DUMMY/LJ004-0231.wav|This was effected by stopping the nearly indiscriminate admission of visitors, which had hitherto prevailed all over the jail. +DUMMY/LJ023-0014.wav|In effect, four Justices ruled that the right under a private contract +DUMMY/LJ029-0182.wav|may not endorse him in 'sixty-four. +DUMMY/LJ044-0205.wav|and promised prompt U.S. aid if they do, end quote. +DUMMY/LJ042-0226.wav|and horror at the misguided line of reasoning of the United States Government, end quote. +DUMMY/LJ038-0100.wav|On November twenty-three, Fritz confronted Oswald with the evidence that he had purchased a rifle under the fictitious name of "Hidell." +DUMMY/LJ026-0160.wav|In comparing a higher animal and a green plant confusion must be avoided regarding the part played by oxygen and carbon dioxide in true respiration +DUMMY/LJ028-0195.wav|It is a long story. Poets have sung it. Historians have written it. Prophets have preached it. Legends have gathered about it. +DUMMY/LJ043-0153.wav|When asked if Oswald requested the note back she testified that, quote, He forgot about it. But apparently +DUMMY/LJ006-0179.wav|But, indeed, his whole rule was far too mild, and under this mistaken leniency +DUMMY/LJ043-0025.wav|He even threatened to tear up his wife's dresses and break all the baby things. According to De Mohrenschildt, +DUMMY/LJ029-0079.wav|was a one-story building with few entrances and easy to make secure, but it lacked necessary food-handling facilities +DUMMY/LJ050-0049.wav|and from a specialist in psychiatric prognostication at Walter Reed Hospital. +DUMMY/LJ005-0193.wav|All descriptions of offenders congregated together in the felons' wards. +DUMMY/LJ034-0212.wav|As the motorcade started its descent down Elm Street., Altgens snapped a picture of the Presidential limousine with the entrance to the Depository Building in the background. +DUMMY/LJ043-0124.wav|Ironically, he failed to get a job in another photographic firm after his return to Dallas in October of nineteen sixty-three, +DUMMY/LJ049-0030.wav|the Commission does believe that there are aspects of the protective measures employed in the motorcade at Dallas which deserve special comment. +DUMMY/LJ049-0097.wav|and the attempt on the life of President-elect Franklin D. Roosevelt. +DUMMY/LJ014-0291.wav|he showed symptoms of delirium tremens, and admitted that he had been addicted to the excessive use of stimulants. +DUMMY/LJ008-0120.wav|"Do tell me, sir; I am informed I shall go down with great force; is it so?" +DUMMY/LJ014-0310.wav|Pries, although enjoying a high reputation in the city, had long been in a bad way. +DUMMY/LJ050-0157.wav|On the basis of such a feasibility study, +DUMMY/LJ018-0198.wav|It is told of "Bob" Brennan, an official specially employed for years by the Mint +DUMMY/LJ026-0155.wav|and later is absorbed from the blood and lymph by all the living cells. +DUMMY/LJ043-0141.wav|Sometime after March twenty-seven, but according to Marina Oswald, prior to April ten, nineteen sixty-three, +DUMMY/LJ006-0279.wav|Wards-women were allowed much the same authority, with the same temptations to excess, and intoxication was not unknown among them and others. +DUMMY/LJ047-0025.wav|During the next two years the FBI continued to accumulate information, +DUMMY/LJ046-0232.wav|Although the Service recognized that its advance preventive measures must encompass more than these most obvious dangers, +DUMMY/LJ039-0168.wav|each fired two series of three shots. +DUMMY/LJ029-0166.wav|On November twenty a front page story reported that the streets on which the Presidential motorcade would travel included "Main and Stemmons Freeway." +DUMMY/LJ029-0204.wav|Two days later, a local Republican leader called for a "civilized nonpartisan" welcome +DUMMY/LJ050-0020.wav|in which the Commission believes that the Secret Service has operated with insufficient planning or control. +DUMMY/LJ007-0006.wav|They had introduced "much order and cleanliness," +DUMMY/LJ031-0100.wav|Treatment of Governor Connally +DUMMY/LJ038-0306.wav|was considered of probative value in this investigation, although the Commission's conclusion concerning the identity of the assassin +DUMMY/LJ016-0152.wav|Only great resolution indeed, persisted in to the bitter end, +DUMMY/LJ021-0079.wav|Nevertheless the gains of trade and industry, as a whole, have been substantial. +DUMMY/LJ017-0041.wav|was thus brought to the Central Criminal Court from Exeter assizes, and that of the Stauntons from Maidstone. +DUMMY/LJ011-0077.wav|The act for which he was taken into custody was the payment of a forged ten-pound note for half-a-dozen silver spoons. +DUMMY/LJ049-0061.wav|No responsibility rests upon those agents near the President for the identification or arrest of any assassin or an attacker. +DUMMY/LJ039-0240.wav|On the basis of the evidence reviewed in this chapter, the Commission has found that Lee Harvey Oswald (one) +DUMMY/LJ012-0184.wav|Meanwhile the discovery of pistol and knife spattered with human blood and brains +DUMMY/LJ032-0026.wav|Post Office Box two nine one five, Dallas, Texas, on March twenty, nineteen sixty-three. +DUMMY/LJ050-0155.wav|Also the Department requests the sum of one hundred thousand dollars to conduct a detailed feasibility study; +DUMMY/LJ041-0091.wav|Powers considered Oswald to be meek and easily led +DUMMY/LJ025-0171.wav|setting free the oxygen and laying hold of the carbon which it contains. +DUMMY/LJ012-0018.wav|One of the most notorious, and for a time most successful practitioners in this illicit trade, passed through Newgate in eighteen thirty-one. +DUMMY/LJ019-0026.wav|of course with the added improvements following longer experience, of the pattern originated by the architectural genius of Sir Joshua Jebb. +DUMMY/LJ039-0138.wav|Lee Oswald exhibited an average amount of proficiency with that weapon. +DUMMY/LJ028-0127.wav|was living in the city while the walls were still standing, though in a ruinous condition. +DUMMY/LJ002-0031.wav|The jail at that date was divided into eight separate and more or less distinct departments, each of which had its own wards and yard. +DUMMY/LJ009-0119.wav|kneel, whilst the rest of the congregation sit, and the ordinary, in a tone of peculiar solemnity, says, +DUMMY/LJ005-0090.wav|the first by daily services, the latter by the appointment of schoolmasters and instruction in reading and writing. +DUMMY/LJ042-0068.wav|Somewhere, a violin plays, as I watch my life whirl away. +DUMMY/LJ039-0052.wav|the Commission concluded that the incident, as described by Marina Oswald, +DUMMY/LJ037-0173.wav|Three of the bullets recovered from Tippit's body were manufactured by Winchester-Western, and the fourth bullet by Remington-Peters, +DUMMY/LJ044-0076.wav|Oswald's statements suggest that he hoped to be flooded with callers and invitations to debate. +DUMMY/LJ011-0283.wav|One was a cheque for eight hundred pounds on his bankers, the other an order to Mr. Bell of Newport, Essex, requesting the surrender of a deed. +DUMMY/LJ046-0248.wav|to Federal intelligence and law enforcement agencies were not well designed to elicit information from them +DUMMY/LJ031-0197.wav|The public had been excluded from the base, and only Government officials and the press were permitted near the landing area. +DUMMY/LJ049-0138.wav|and, if the Council is used, arrangements should be made for the attendance of the Secretary of the Treasury +DUMMY/LJ025-0001.wav|The Science: History of the Universe, Volume five. +DUMMY/LJ014-0103.wav|beneath them was a layer of fresh mortar, beneath that a lot of loose earth, amongst which a stocking was turned up, and presently a human toe. +DUMMY/LJ015-0121.wav|Newgate officers who remember Robson still describe him as a fine young man, who behaved well as a prisoner, +DUMMY/LJ011-0226.wav|which was based on his jail experiences, and of which I have availed myself in the last chapter. +DUMMY/LJ021-0085.wav|and the incentive of fair private profit, +DUMMY/LJ006-0079.wav|The inspectors found that prisoners had their places assigned to them +DUMMY/LJ004-0236.wav|But nothing short of radical reform and complete reconstruction could touch the deep-seated evils of association, overcrowding, and idleness. +DUMMY/LJ047-0204.wav|Bouck pointed to a number of characteristics besides Oswald's defection the cumulative effect of which would have been to alert the Secret Service +DUMMY/LJ009-0027.wav|will thereby be lost. +DUMMY/LJ041-0118.wav|Oswald wrote to then Secretary of the Navy Connally on January thirty, nineteen sixty-two, +DUMMY/LJ018-0350.wav|He might swear he was not the murderer, that he never fired a pistol in his life, +DUMMY/LJ018-0013.wav|There was a deep wound just over the ear, the skull was fractured, and there were several other blows and wounds on the head. +DUMMY/LJ042-0223.wav|and to have stated in the second what he thought would be least harmful to him as he resumed life in the United States. +DUMMY/LJ010-0169.wav|sometimes in the Strand, or the West End. +DUMMY/LJ015-0024.wav|but that he might strengthen it by his long experience and known business capacity. +DUMMY/LJ030-0018.wav|Before leaving the hotel, the President, Mrs. Kennedy, and Kenneth O'Donnell talked about the risks inherent in Presidential public appearances. +DUMMY/LJ019-0315.wav|also their constant employment in labor appropriate to their condition. +DUMMY/LJ035-0027.wav|They went through the swinging door and continued at, quote, a good trot, end quote, +DUMMY/LJ038-0278.wav|However, to say that because one does not find sufficient marks for identification that it is a negative, +DUMMY/LJ022-0080.wav|Projects will be sought which promise ultimate return to the federal treasury of a considerable proportion of the costs. +DUMMY/LJ022-0088.wav|they will be sent to an Allotment Division composed of representatives of the more important governmental agencies +DUMMY/LJ024-0139.wav|has been tipped out of balance by the courts in direct contradiction of the high purposes of the framers of the Constitution. +DUMMY/LJ017-0184.wav|Soon afterwards a friend, Mrs. Atkinson, came to London from Westmoreland, and stayed in Mrs. Wilson's house. +DUMMY/LJ010-0301.wav|He had entered the house as clerk in eighteen hundred; +DUMMY/LJ035-0196.wav|The address listed was for the Paine home in Irving. +DUMMY/LJ037-0085.wav|She was not sure whether she had seen his picture in a newspaper on the afternoon or evening of November twenty-two prior to the lineup. +DUMMY/LJ036-0176.wav|Earlene Roberts, housekeeper for Mrs. A. C. Johnson at ten twenty-six North Beckley +DUMMY/LJ011-0004.wav|Once he ran a narrow risk of being found out. +DUMMY/LJ019-0012.wav|which were then recommended to and generally adopted by the visiting justices all over the kingdom. +DUMMY/LJ010-0004.wav|only premising that as accounts become more voluminous I shall be compelled to deal with fewer cases, +DUMMY/LJ038-0282.wav|there is enough on it to say that it could have come, and even perhaps a little stronger, to say that it probably came from this, +DUMMY/LJ037-0014.wav|Scoggins lost sight of him behind some shrubbery on the southeast corner lot, +DUMMY/LJ002-0338.wav|Only now and again, in extreme cases, an unusually nasty companion was stripped, haled to the pump, +DUMMY/LJ048-0201.wav|I stressed the fact that this was our President and he should be shown every respect due his position and that it was our duty to see that this was done. +DUMMY/LJ018-0089.wav|Another case of long-continued successful forgery was brought to light two years after the convictions of Saward and his accomplices. +DUMMY/LJ002-0097.wav|they had dust-bins, sewers, and so forth, "properly disposed," and the city scavenger paid periodical visits to the prison. +DUMMY/LJ013-0015.wav|being a party to the intended fraud, they obtained further insurances on goods never shipped. +DUMMY/LJ021-0164.wav|a permanent army of unemployed. +DUMMY/LJ002-0119.wav|In that period upwards of two hundred thousand writs +DUMMY/LJ006-0109.wav|a perpetuation under another form of the old detestable custom of garnish. +DUMMY/LJ003-0017.wav|Order throughout the debtors' side was preserved +DUMMY/LJ013-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section sixteen: Newgate notorieties continued, part one. +DUMMY/LJ037-0073.wav|Addressing itself solely to the probative value of Mrs. Markham's contemporaneous description of the gunman +DUMMY/LJ037-0087.wav|I was pretty sure it was the same man I saw. When they made him turn sideways, I was positive that was the one I seen, end quote. +DUMMY/LJ011-0225.wav|will be admitted by those who have read his book on "the punishment of death, +DUMMY/LJ006-0181.wav|It was the same old story -- evil constantly in the ascendant, the least criminal at the mercy of the most depraved. +DUMMY/LJ021-0018.wav|the tremendous power of organization has combined great aggregations of capital in enormous industrial establishments +DUMMY/LJ025-0038.wav|carbon, hydrogen and oxygen. +DUMMY/LJ050-0224.wav|one hundred forty-five are proposed for the field offices to handle the increased volume of security investigations +DUMMY/LJ009-0122.wav|particularly to those who desire now to offer up their praises and thanksgivings for thy late mercies vouchsafed unto them. +DUMMY/LJ012-0210.wav|The porter received it, but from its freshness became suspicious of foul play. +DUMMY/LJ037-0066.wav|Although she used the words, quote, a little bit bushy, end quote, to describe the gunman's hair, +DUMMY/LJ010-0119.wav|It was asserted, not without foundation, at these trials, that Edwards repeatedly incited the associates he was betraying +DUMMY/LJ009-0092.wav|his legs give way, he utters a faint groan, and sinks on the floor. +DUMMY/LJ048-0045.wav|his pro-Castro tendencies, his lies when interrogated by the FBI, +DUMMY/LJ005-0068.wav|sought to "rank the prisons they built among the most splendid buildings of the city or town." +DUMMY/LJ029-0059.wav|Dallas detectives in the lobby of the Trade Mart and in the luncheon area also had copies of these photographs. +DUMMY/LJ006-0049.wav|an occasional prisoner or two committed by the Houses of Parliament, the Courts of King's Bench, Common Pleas, +DUMMY/LJ039-0137.wav|On that occasion Oswald again used a bolt-action twenty-two caliber rifle; and according to Robert, +DUMMY/LJ044-0014.wav|the New Orleans branch of the Fair Play for Cuba Committee. +DUMMY/LJ038-0102.wav|Oswald also denied owning a rifle and said that since leaving the Marine Corps he had fired only a small bore twenty-two rifle. +DUMMY/LJ021-0133.wav|are likewise not fully cooperating with their government. +DUMMY/LJ023-0025.wav|National laws are needed to complete that program. +DUMMY/LJ002-0035.wav|eight. The press yard. +DUMMY/LJ019-0051.wav|the other supported the theory of labor in association, but under a stringent rule of silence, with isolation only at night. +DUMMY/LJ035-0125.wav|Victoria Adams, who worked on the fourth floor of the Depository Building, +DUMMY/LJ036-0085.wav|Oswald rose from his chair, looked directly at Fritz, and said, quote, Everybody will know who I am now, end quote, +DUMMY/LJ039-0214.wav|in order to hit the intended object. The scope would accomplish the lead for you, end quote, +DUMMY/LJ039-0243.wav|(three) was present, at the time of the assassination, at the window from which the shots were fired +DUMMY/LJ019-0176.wav|The wards had open fires, but the separate cells were not warmed at all. +DUMMY/LJ019-0193.wav|but as the plan was "confessedly experimental, for the benefit of the country generally, the amount for which the city should be responsible should be distinctly limited +DUMMY/LJ019-0036.wav|This list included Wakefield, Leeds, Kirkdale, Manchester, Birmingham, and Dublin. +DUMMY/LJ050-0236.wav|Thus, in the four months following the assassination, +DUMMY/LJ028-0267.wav|The Babylonians, however, cared not a whit for his siege. +DUMMY/LJ048-0055.wav|a more alert and carefully considered treatment of the Oswald case by the Bureau might have brought about such a referral. +DUMMY/LJ037-0115.wav|Early in the evening of November twenty-two, +DUMMY/LJ050-0051.wav|nineteen sixty-four, makes several significant recommendations in this field. +DUMMY/LJ011-0063.wav|Fauntleroy was not entirely dependent upon the ordinary for ghostly counsel in his extremity. +DUMMY/LJ005-0184.wav|so as to corrupt those committed, "to the injury of the peace and morals of the public." +DUMMY/LJ009-0107.wav|The children round the communion-table stare and gape with childish wonder. +DUMMY/LJ017-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section twenty: Newgate Notorieties, part one. +DUMMY/LJ001-0001.wav|Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition +DUMMY/LJ040-0177.wav|He took the position that such matters were his own business. +DUMMY/LJ037-0045.wav|He was just fooling with it. I didn't know what he was doing. I was afraid he was fixing to kill me. End quote. +DUMMY/LJ041-0117.wav|In an attempt to have this discharge reversed, +DUMMY/LJ027-0018.wav|All are forced to make concession after concession to their surroundings, and in these concessions all progress in life consists. +DUMMY/LJ034-0185.wav|In his testimony he said, quote, I told them that that could have been the man. +DUMMY/LJ025-0069.wav|innumerable plants and free plant cells are known to pass the whole or part of their lives in an actively locomotive condition, +DUMMY/LJ016-0224.wav|Calcraft shared the odium which his office, not strangely, has always inspired. But he was admitted into the jail, +DUMMY/LJ036-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ009-0048.wav|He is a hardened offender, previously cast for life, reprieved, transported to Australia, and since returned without pardon. +DUMMY/LJ041-0125.wav|Mrs. Oswald said that her husband did not say anything about Governor Connally after his return to the United States. She testified, quote, +DUMMY/LJ005-0279.wav|an incomplete and fallacious method of preventing contamination. +DUMMY/LJ006-0163.wav|and that he was constantly engaged attending sessions and going with drafts to the hulks. +DUMMY/LJ031-0130.wav|Two men from the President's follow-up car were detailed to help protect the Vice President. +DUMMY/LJ011-0236.wav|sent his plate-chests, eight in number, to the silversmith's for greater security. +DUMMY/LJ045-0126.wav|to the Soviet Embassy in Washington, he asked about the entrance visas for which he and his wife had previously applied. +DUMMY/LJ032-0146.wav|Latona testified that this palmprint was the right palmprint of Lee Harvey Oswald. +DUMMY/LJ030-0211.wav|Hill jumped from the follow-up car and ran to the President's automobile. +DUMMY/LJ012-0037.wav|the owner came with the police, and was morally convinced that it was his property, +DUMMY/LJ027-0114.wav|Namely that when from changed conditions of life, an organ which was previously useful becomes useless, +DUMMY/LJ046-0021.wav|the Secretary of the Treasury has prepared a planning document dated August twenty-seven, nineteen sixty-four, +DUMMY/LJ009-0097.wav|He throws his hands far from him, and shouts aloud, 'Mercy, good Lord! mercy is all I ask. The Lord in His mercy come! +DUMMY/LJ028-0243.wav|Had the Babylonians been apprised of what Cyrus was about, or had they noticed their danger, they would never have allowed the Persians to enter the city, +DUMMY/LJ018-0155.wav|Mrs. Campbell offered him less, and there was an altercation, in the course of which Buncher became very violent, and at length, +DUMMY/LJ035-0182.wav|Harkness did not know whether or not two officers with Sawyer were guarding the doors. +DUMMY/LJ010-0037.wav|Now and again there seemed to be a recurrence of a murder epidemic, +DUMMY/LJ030-0172.wav|From his position in the right jump seat immediately in front of the President, +DUMMY/LJ037-0079.wav|They ran to the door in time to see a man with a revolver cut across their lawn and disappear around a corner of the house onto Patton. +DUMMY/LJ008-0156.wav|An enormous crowd assembled to witness the execution, amounting, it was said, to the hitherto unparalleled number of forty thousand. +DUMMY/LJ021-0209.wav|I prefer and I am sure you prefer that broader definition of liberty under which we are moving forward to greater freedom, +DUMMY/LJ007-0156.wav|The whole system, or more correctly the want of system, was baneful and pernicious to the last degree. +DUMMY/LJ032-0099.wav|to serve as president of the so-called New Orleans Chapter of the Fair Play for Cuba Committee. +DUMMY/LJ047-0059.wav|However, pursuant to a regular Bureau practice of interviewing certain immigrants from Iron Curtain countries, +DUMMY/LJ005-0119.wav|No attempt was made to maintain discipline. +DUMMY/LJ030-0201.wav|According to Governor and Mrs. Connally, it was after this shot that Kellerman issued his emergency instructions and the car accelerated. +DUMMY/LJ025-0141.wav|the diameter of which dwindles down to the limits of our present microscopic vision, +DUMMY/LJ040-0192.wav|The reports of Carro and Mrs. Siegel also indicate an ambivalent attitude toward authority on Oswald's part. +DUMMY/LJ007-0016.wav|it tended to "dissipate reflection, diminish the gloom of the prison, and mitigate the punishment which the law has sentenced the prisoner to undergo." +DUMMY/LJ012-0131.wav|The letter informed him of the marks and sizes of the cases containing the precious metal, +DUMMY/LJ009-0287.wav|has come more by chance than fitness or special education to exercise his loathsome office. +DUMMY/LJ048-0144.wav|It had no prepared checklist of matters to be covered with local police on such visits to metropolitan areas +DUMMY/LJ028-0448.wav|All of the ancient writers agree in saying that Babylon was surrounded with both inner and outer walls, and the ruins confirm their statements. +DUMMY/LJ045-0193.wav|en route from Dallas Love Field, end quote, on its way to the Trade Mart on Stemmons Freeway. +DUMMY/LJ037-0058.wav|In testimony before the Commission, Mrs. Markham confirmed her positive identification of Lee Harvey Oswald +DUMMY/LJ004-0197.wav|No idleness was permitted among the inmates. Trades were taught, or prisoners were allowed to follow their own if suitable. +DUMMY/LJ017-0224.wav|A simultaneous attack was made upon the captain and the first mate. +DUMMY/LJ041-0061.wav|who, shortly after Oswald's defection, wrote an as yet unpublished novel based in considerable part on Oswald's life, +DUMMY/LJ035-0119.wav|until after they saw Patrolman Baker's white helmet on the fifth floor moving toward the elevator. +DUMMY/LJ041-0069.wav|he did not adjust well to conditions which he found in that service. +DUMMY/LJ035-0030.wav|Truly pushed the button for the west elevator which operates automatically if the gate is closed. +DUMMY/LJ032-0239.wav|on April ten, nineteen sixty-three. +DUMMY/LJ019-0391.wav|to close useless prisons, and to insure that uniformity of system which all thoughtful persons had long admitted to be of paramount importance +DUMMY/LJ040-0070.wav|she placed John Pic and Robert Oswald in an orphans' home. +DUMMY/LJ036-0117.wav|and Daniel Lujan, aged twenty-six, was Number four. +DUMMY/LJ024-0043.wav|that I will appoint justices who will not undertake to override the judgment of the Congress on legislative policy, +DUMMY/LJ031-0030.wav|Governor Connally, who had lost consciousness on the ride to the hospital, regained consciousness when the limousine stopped abruptly at the emergency entrance. +DUMMY/LJ028-0348.wav|These he levied from the nations bordering on Babylonia, +DUMMY/LJ050-0247.wav|The Commission recommends that the agencies involved determine how much periodic assistance they can provide, and that each such agency +DUMMY/LJ048-0215.wav|Agent Roy H. Kellerman, riding in the front seat of the Presidential car, +DUMMY/LJ049-0208.wav|except as conclusions flowing directly from its examination of the President's assassination can be drawn. +DUMMY/LJ021-0178.wav|Has England gone back to the gold standard today? +DUMMY/LJ008-0099.wav|describes as "a most diabolical-looking little wretch -- Jack Ketch's head man. +DUMMY/LJ026-0144.wav|In both plants and animals simple waste substances result from the catabolic processes in the cells. +DUMMY/LJ016-0237.wav|With Calcraft's method there were undoubtedly many failures, and it was a common custom for him to go below the gallows +DUMMY/LJ018-0039.wav|The prisoner was forthwith extradited and sent back to England, which he reached with his escort on the seventeenth September the same year. +DUMMY/LJ043-0117.wav|Oswald apparently concluded that his Fair Play for Cuba Committee activities were not related to his discharge. +DUMMY/LJ018-0019.wav|The stick and bag were his, but not the hat. +DUMMY/LJ026-0010.wav|Since they cannot be classified, it is necessary that they be listed both under botany and zoology, in order to make sure that they will not be omitted entirely. +DUMMY/LJ019-0300.wav|The committee might well suggest the abolition of these jails, or their amalgamation with the larger county establishments in their immediate neighborhood. +DUMMY/LJ019-0089.wav|sometimes it embraced the tread-wheel or the newly-invented instruments known as cranks, which ground air. +DUMMY/LJ029-0088.wav|securing the roof and insuring the presence of numerous police officers inside and around the building. +DUMMY/LJ001-0158.wav|it is never used except for very expensive books, although it would not materially increase the cost in all but the very cheapest. +DUMMY/LJ002-0114.wav|and imprisonment for debt has almost entirely disappeared. +DUMMY/LJ011-0054.wav|A very curious and, in its way, amusing circumstance in connection with this case was the offer of a certain Italian, +DUMMY/LJ039-0110.wav|Based on the general Marine Corps ratings, Lt. Col. A. G. Folsom, Jr. +DUMMY/LJ012-0129.wav|learnt through the firm's correspondence that a quantity of gold-dust +DUMMY/LJ008-0148.wav|One night he was missing +DUMMY/LJ032-0122.wav|At eleven:forty-five p.m. on November twenty-two, +DUMMY/LJ041-0068.wav|While there is nothing in Oswald's military records to indicate that he was mentally unstable or otherwise psychologically unfit for duty in the Marine Corps, +DUMMY/LJ041-0152.wav|Oswald studied Marxism after he joined the Marines and his sympathies in that direction and for the Soviet Union appear to have been widely known, +DUMMY/LJ030-0186.wav|The driver, William Greer, heard a noise which he took to be a backfire from one of the motorcycles flanking the Presidential car. +DUMMY/LJ008-0071.wav|In the few years which elapsed between the establishment of the gallows at Newgate +DUMMY/LJ011-0260.wav|It was, however, proved that Howard was in needy circumstances, and that his proposals to Mr. Mullay could only have originated in a desire to rob him. +DUMMY/LJ044-0179.wav|He apparently based his claim for a visa in transit to Russia +DUMMY/LJ003-0062.wav|When each sleeper had the full lateral space allotted to him, it amounted to one foot and a half on the barrack bed; +DUMMY/LJ033-0194.wav|The blanket in which the rifle was stored was composed of brown and green cotton, viscose and woolen fibers. +DUMMY/LJ031-0113.wav|This operation was concluded at three:twenty p.m. +DUMMY/LJ020-0011.wav|Put the potatoes into a large bowl or tray and mash them to powder with a potato beetle, or a wooden spoon. +DUMMY/LJ031-0027.wav|Upon arriving at Parkland Hospital, +DUMMY/LJ013-0188.wav|Further investigation was rewarded by the discovery in the pantry of a split gold ring, used by Lord William to carry his keys on; +DUMMY/LJ006-0011.wav|that inspectors should watch over the observance of the law. He saw no objection on the score of their probable interference with the local jurisdiction, +DUMMY/LJ003-0229.wav|that it multiplied beyond all reason the number of visitors, many of whom came merely as the purveyors of food to their friends. +DUMMY/LJ030-0168.wav|As she cradled her mortally wounded husband, Mrs. Kennedy cried, quote, +DUMMY/LJ005-0106.wav|Jails, of which the old prison at Reading was a specimen, were still left intact. +DUMMY/LJ010-0074.wav|After this he tried farming, but failed. +DUMMY/LJ041-0019.wav|Voebel said that Oswald, quote, wouldn't start any fights, but if you wanted to start one with him, he was going to make sure that he ended it, +DUMMY/LJ009-0181.wav|that authentic cases were known previous to the first cited act of criminals selling their own bodies to surgeons for dissection. +DUMMY/LJ006-0242.wav|Watkins' case, referred to above, is made the subject of another and a special report from the surgeon. +DUMMY/LJ016-0272.wav|To the disgrace of the better-educated and better-bred public, executions could still command the attendance of curious aristocrats from the West End. +DUMMY/LJ036-0172.wav|From the five hundred block of North Beckley, the walk would be a few minutes longer, +DUMMY/LJ011-0213.wav|married the said Ellen Turner. +DUMMY/LJ012-0247.wav|a bricklayer found a human trunk near some new buildings in the Edgeware Road, one morning in the last week of eighteen thirty-six. +DUMMY/LJ046-0007.wav|a President-elect, and a candidate for the Presidency, which narrowly failed: +DUMMY/LJ038-0232.wav|Marina Oswald stated that when Oswald returned home on the night of the Walker shooting, he told her that he had been planning the attempt for two months. +DUMMY/LJ004-0245.wav|Painters, plasterers, and carpenters were allowed to follow their handicrafts, with the reward of sixpence per diem and a double allowance of food. +DUMMY/LJ017-0003.wav|Nevertheless, in order to give completeness to the picture +DUMMY/LJ049-0093.wav|persons committing certain offenses, including counterfeiting and certain frauds involving Federal checks or securities. +DUMMY/LJ037-0046.wav|The man, quote, in kind of a little trot, end quote, headed down Patton toward Jefferson Boulevard, a block away. +DUMMY/LJ012-0033.wav|Jewels were re-set, +DUMMY/LJ032-0224.wav|One of the photographs taken by Marina Oswald was widely published in newspapers and magazines, +DUMMY/LJ041-0015.wav|accosted Oswald on the way home from school and punched him in the mouth, making his lip bleed and loosening a tooth. +DUMMY/LJ028-0473.wav|A space of about twenty-three feet separated it from its inner part, which was about twenty feet in width. +DUMMY/LJ016-0076.wav|Even then, however, irons across barred the ascent after a certain distance, and in no one case did a fugitive get clear away. +DUMMY/LJ013-0114.wav|One night a quantity of plate was stolen from Windsor Castle; another time Buckingham Palace was robbed. +DUMMY/LJ050-0113.wav|Such agreements should describe in detail the information which is sought, the manner in which it will be provided to the Secret Service, +DUMMY/LJ006-0015.wav|one which required discretion, judgment, and knowledge of law, with sufficient insight and experience to discover defects in prison discipline. +DUMMY/LJ049-0174.wav|to notify the Secret Service of the substantial information about Lee Harvey Oswald which the FBI had accumulated +DUMMY/LJ018-0068.wav|Inspector Thain, who, being unable to obtain his extradition legally, had him inveigled on board an English steamer, +DUMMY/LJ050-0243.wav|The Commission endorses these efforts to supplement the Service's own personnel by obtaining, for short periods of time, +DUMMY/LJ034-0015.wav|The boxes in the window appeared to have been arranged as a convenient gun rest. +DUMMY/LJ030-0079.wav|The agents in this car, under established procedure, had instructions to watch the route for signs of trouble, +DUMMY/LJ031-0052.wav|and also ragged tissue which indicated a tracheal injury. +DUMMY/LJ040-0170.wav|because he didn't make friends with the boys in the neighborhood. She thought that he, quote, withdrew into a completely solitary and detached existence +DUMMY/LJ050-0086.wav|Under these criteria, whether the case should be referred to the Secret Service depends on the existence of a previous history of mental instability, +DUMMY/LJ048-0171.wav|it was not practical to select a route where the President could not be seen from roofs or windows of buildings. +DUMMY/LJ048-0067.wav|It is clear from Hosty's testimony that this was construed, at least by him, +DUMMY/LJ004-0249.wav|As the Prison Discipline Society pertinently observed in a report dated eighteen twenty, +DUMMY/LJ019-0003.wav|It was impossible for Newgate to escape for ever the influences pressing so strongly towards prison reform. +DUMMY/LJ009-0221.wav|according to the discretion of the court before whom the prisoners might be tried. +DUMMY/LJ015-0193.wav|By-and-by, when escape seemed hopeless, and after sentence, he suddenly degenerated into the lowest stamp of criminal, +DUMMY/LJ050-0062.wav|(c) who express or have expressed strong or violent anti-U.S. sentiments +DUMMY/LJ013-0135.wav|At most of the clubs the servants had been mulcted to make good lost plate, which had no doubt been stolen by the prisoner. +DUMMY/LJ029-0010.wav|Planning the Texas Trip +DUMMY/LJ047-0026.wav|and kept itself informed on Oswald's status by periodic reviews of State Department and Office of Naval Intelligence files. +DUMMY/LJ026-0099.wav|are absorbed by the circulating liquids (blood and lymph) and transported to all cells of the animal body. +DUMMY/LJ004-0044.wav|several bishops, and a number of members of the House of Commons, including Mr. Manners Sutton, +DUMMY/LJ012-0276.wav|While she was senseless, but really still alive, he cut off her head, and dismembered the body in the manner already described. +DUMMY/LJ023-0001.wav|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt, Section nine. +DUMMY/LJ040-0199.wav|The reports of the New York authorities indicate that Lee's mother gave him very little affection and did not serve as any sort of substitute for a father. +DUMMY/LJ011-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section fourteen: Newgate notorieties, part one +DUMMY/LJ049-0116.wav|of such persons as have been constitutionally and lawfully provided to succeed thereto in case of a vacancy. It is important to this country +DUMMY/LJ001-0130.wav|Commercialism again compels the use of type too small in size to be comfortable reading: +DUMMY/LJ016-0056.wav|running a great risk of discovery as he passed by a lot of workmen at Tyler's manufactory in Warwick Square, which had formerly been the College of Physicians. +DUMMY/LJ010-0249.wav|There was some doubt as to whether the pistol when fired was actually loaded with ball, but the jury brought in a verdict of guilty +DUMMY/LJ038-0235.wav|were identified by Marina Oswald as photographs of General Walker's house. +DUMMY/LJ019-0023.wav|that this was an experimental construction, and that too strict a limitation of outlay would have militated seriously against the usefulness of the building. +DUMMY/LJ016-0311.wav|that any secrecy in the treatment of the condemned would invest them with a new and greater interest, which was much to be deprecated. +DUMMY/LJ048-0219.wav|who must concentrate primarily on the possibility of threats from crowds along the route, provide a significant safeguard against dangers in nearby buildings. +DUMMY/LJ002-0015.wav|three hundred debtors and nine hundred criminals in Newgate, or twelve hundred prisoners in all. +DUMMY/LJ014-0279.wav|and he had so cleverly destroyed all traces that it was not easy to bring home the charge. +DUMMY/LJ050-0227.wav|in case of unexpected need; and twenty-five additional agents are recommended to provide the Vice President full protection. +DUMMY/LJ042-0012.wav|who had arranged a radio debate on Oswald's activities on behalf of the Fair Play for Cuba Committee, +DUMMY/LJ014-0228.wav|generally known by the soubriquet of "General Haynau," a name execrated in England about this time. +DUMMY/LJ033-0118.wav|Similarly, when the butt of the rifle was placed in Frazier's hand, it extended above his shoulder to ear level. +DUMMY/LJ022-0131.wav|If you will help, this can be done. +DUMMY/LJ041-0130.wav|Probably his complaint was due to the fact that his discharge was not related to anything he had done while on active duty +DUMMY/LJ026-0164.wav|These constitute a sort of extra mechanism, enabling green plants to make their own carbohydrate food. +DUMMY/LJ040-0014.wav|He cannot, of course, be questioned or observed by those charged with the responsibility for this report or by experts on their behalf. +DUMMY/LJ016-0319.wav|The only doubts expressed were as to the sufficiency of the safeguards, as to the certainty of death and its subsequent publication. +DUMMY/LJ038-0011.wav|A police car made a U-turn, and as the sirens grew fainter, +DUMMY/LJ012-0197.wav|a trade almost openly countenanced when "subjects" for the anatomy schools were only to be got by rifling graves, or worse. +DUMMY/LJ014-0332.wav|Cole escaped by throwing the blame on a careless partner, and at once removed the "stop." +DUMMY/LJ015-0161.wav|There was something dramatic in Redpath's detection +DUMMY/LJ038-0258.wav|The Dallas City-County Investigation Laboratory tried to determine the type of weapon which fired the bullet. +DUMMY/LJ016-0147.wav|Then he ran along the coping of the wall towards its angle with Tyler's manufactory, and dropped down on to the gridiron below. +DUMMY/LJ010-0134.wav|He roared out snatches of a song about Death or Liberty, and just before he was turned off, +DUMMY/LJ032-0182.wav|were clean, they had good color to them, there was no grease on them and they were not fragmented. They looked as if they had just been picked up, end quote. +DUMMY/LJ044-0017.wav|He distributed literature in downtown New Orleans on August nine, nineteen sixty-three, +DUMMY/LJ040-0004.wav|There is no evidence that he had accomplices or that he was involved in any conspiracy directed to the assassination of the President. +DUMMY/LJ028-0131.wav|to describe whose vast height and immense riches it would perhaps be too much for me to attempt. +DUMMY/LJ030-0191.wav|According to Kellerman, Mrs. Kennedy then cried out, quote, +DUMMY/LJ011-0008.wav|whereupon he produced exchequer bills to the amount. +DUMMY/LJ023-0107.wav|In the face of such dissenting opinions, it is perfectly clear that, as Chief Justice Hughes has said, +DUMMY/LJ015-0149.wav|peas at ten shillings a quart, five-guinea pines, and early asparagus were to be found on his table. +DUMMY/LJ006-0165.wav|sometimes he looked at his bolts and bars, but he never examined the cupboards, coal-boxes, or other possible hiding-places for cards +DUMMY/LJ012-0163.wav|I will now retrace my steps and give some account of the more remarkable murders during that period. +DUMMY/LJ030-0226.wav|who did not believe that he could reach, the President's car at the speed it was then traveling. +DUMMY/LJ016-0037.wav|with only a short interval between it and the wall, supported by a horizontal iron railing with upright points; +DUMMY/LJ031-0166.wav|Unmarked police cars took the Vice President and Mrs. Johnson from Parkland Hospital to Love Field. +DUMMY/LJ028-0440.wav|Possibly along the terraces of the walls, or upon the stages of some lofty temple tower, +DUMMY/LJ014-0269.wav|and walk forward on foot to enter as a humble and unpretending employee the doors of the Globe Assurance office. +DUMMY/LJ003-0273.wav|"I believe," says Mr. Bennet in the letter already largely quoted, +DUMMY/LJ028-0347.wav|he provided wives for them in the room of those whom (as I explained before) they strangled to save their stores. +DUMMY/LJ008-0297.wav|and its contents were anxiously awaited by both convicts in the press-yard and their friends collected in a crowd outside the gates. +DUMMY/LJ028-0467.wav|The entire width of the outer defense, not including the moat, was therefore about eighty-two feet; +DUMMY/LJ026-0066.wav|both from the soil through the roots (liquids) and from the atmosphere through the leaves (gases). +DUMMY/LJ017-0015.wav|the necessity for which was urged by Mr. Addison, when Secretary of State, in seventeen seventeen. +DUMMY/LJ032-0035.wav|In addition to the order coupon the envelope contained a. U.S. postal money order for twenty-one dollars, forty-five cents, +DUMMY/LJ010-0304.wav|through advances made to various builders, and that it could only maintain its credit by wholesale discounting. +DUMMY/LJ035-0008.wav|The first person to see Oswald after the assassination was Patrolman M. L. Baker of the Dallas Police Department. +DUMMY/LJ030-0053.wav|It was outfitted with a clear plastic bubbletop which was neither bulletproof nor bullet resistant. +DUMMY/LJ035-0086.wav|The time actually required for Baker and Truly to reach the second floor on November twenty-two was probably longer than in the test runs. For example, +DUMMY/LJ040-0040.wav|which he expressed in striking and sometimes violent acts long before the assassination. +DUMMY/LJ038-0171.wav|General Walker gave this information to the police before the shooting, but it did not help solve the crime. +DUMMY/LJ041-0104.wav|While in Japan, Oswald's new found apparent self confidence and pugnaciousness +DUMMY/LJ027-0098.wav|is found the same part, variously modified for many purposes. +DUMMY/LJ017-0152.wav|in spite of the existence of the other Mrs. Smethurst. +DUMMY/LJ021-0117.wav|to the great number of small employers in the smaller communities. +DUMMY/LJ050-0187.wav|Thus, the local chief of police could be given a master plan, prepared for the occasion, of all protective measures to be taken during the visit; +DUMMY/LJ047-0239.wav|The two men disagree about the conversation which took place between them. +DUMMY/LJ021-0067.wav|a rise from a deficit figure in the first quarter of nineteen thirty-three +DUMMY/LJ019-0059.wav|It was claimed for it that it was more deterrent; +DUMMY/LJ014-0054.wav|a maidservant, Sarah Thomas, murdered her mistress, an aged woman, by beating out her brains with a stone. +DUMMY/LJ016-0412.wav|Wainwright's execution was within the jail, but only nominally private. +DUMMY/LJ010-0033.wav|Esther Hibner was as inhuman in her ill-usage of the parish apprentice she killed as Martha Brownrigg had been. +DUMMY/LJ003-0047.wav|Newgate continued to be a reproach to those responsible for its management. +DUMMY/LJ026-0086.wav|the kinetic energy of sunlight was expended in lifting the atoms into position of vantage, thus endowing them with energy of position. +DUMMY/LJ004-0210.wav|There was a prison laundry too, where all the prisoners' linen was regularly washed. +DUMMY/LJ043-0172.wav|The attempt on General Walker's life deserves close attention in any consideration of Oswald's possible motive for the assassination +DUMMY/LJ029-0180.wav|An editorial in the Times-Herald of September seventeen +DUMMY/LJ016-0104.wav|whence they let themselves down into the street by the rope. +DUMMY/LJ026-0098.wav|The circulatory system distributes these foods. In animals foods prepared for absorption in the stomach and intestine (by digestion) +DUMMY/LJ028-0120.wav|built the moat-walls of Arachtu with mortar and bricks, and, joining them together with those of my father, made them very solid. +DUMMY/LJ022-0012.wav|it is difficult for a person who does not know ships to tell how it will finally look when it is sailing the high seas. +DUMMY/LJ039-0090.wav|Characterizing the four-power scope as, quote, a real aid, an extreme aid, end quote, in rapid fire shooting, +DUMMY/LJ042-0174.wav|No man, having known, having lived, under the Russian Communist and American capitalist system, +DUMMY/LJ037-0127.wav|Four men -- Warren Reynolds, Harold Russell, Pat Patterson, and L. J. Lewis +DUMMY/LJ050-0163.wav|the Secret Service should attempt to develop a system compatible with those of the agencies from which most of its data will come. Note: +DUMMY/LJ012-0243.wav|There were many features of resemblance in these crimes. +DUMMY/LJ030-0047.wav|The occupants scanned the crowd and the buildings along the route. +DUMMY/LJ023-0068.wav|to have been the clear and underlying purpose of the patriots who wrote a federal constitution to create a national government +DUMMY/LJ011-0050.wav|that he appeared to feel deeply the force of the reverend gentleman's observations, especially when the chaplain spoke of +DUMMY/LJ019-0307.wav|The result of the recommendation of the committee of eighteen sixty-two was the Prison Act of eighteen sixty-five, +DUMMY/LJ019-0094.wav|Opinions differed greatly with regard to the tread-wheel; some authorities advocated it as a very severe and irksome punishment, +DUMMY/LJ012-0206.wav|This man, May, asked the porter at King's College if "he wanted anything?" the euphemism for offering a body. +DUMMY/LJ002-0183.wav|whilst Ludgate, the Giltspur Street, and Borough Compters also received them +DUMMY/LJ007-0107.wav|A resolution at once passed the House without division to commit the whole to Newgate, where they remained for various terms. +DUMMY/LJ040-0188.wav|Sokolow said that although Lee was, quote, presumably disinterested in school subjects he operates on a much higher than average level, end quote. +DUMMY/LJ019-0210.wav|But when this suggestion was brought before the court of aldermen, various amendments were proposed. +DUMMY/LJ025-0024.wav|whence their nutrition may be drawn by vessels, which are a sort of internal roots; and, in the presence of this alimentary cavity +DUMMY/LJ006-0180.wav|the interior of the jail was more like a bear-garden or the noisy purlieus of a public-house than a prison. +DUMMY/LJ019-0246.wav|It animadverted strongly on "the many and wide differences as regards construction, labor, diet, and general discipline" +DUMMY/LJ048-0253.wav|The supervisor of each of the off-duty agents who visited the Press Club or the Cellar Coffee House +DUMMY/LJ020-0028.wav|Warm water for rinsing bowl -- about half a cup. +DUMMY/LJ035-0172.wav|There he was met by a construction worker -- in all likelihood Howard Brennan, who was wearing his work helmet. +DUMMY/LJ042-0245.wav|While Marina Oswald tried to obtain permission to return to the Soviet Union, she testified that she did so at her husband's insistence. +DUMMY/LJ050-0248.wav|and the Secret Service enter into a formal agreement defining such arrangements. +DUMMY/LJ047-0207.wav|his association with the Castro groups would have been of concern to us, a knowledge that he had, I believe, +DUMMY/LJ011-0154.wav|He had, it was said, a good private income, and was already a widower with two children at the time of his committing the offense for which he was subsequently tried. +DUMMY/LJ041-0107.wav|He testified that he had felt the sergeant had a grudge against him and that he had unsuccessfully sought a transfer from the sergeant's unit. +DUMMY/LJ023-0108.wav|We are under a Constitution, but the Constitution is what the judges say it is. +DUMMY/LJ046-0071.wav|The Secret Service and the Federal, State, and local law enforcement agencies which cooperate with it, +DUMMY/LJ026-0035.wav|the organism must be compared in detail with all those which resemble it closely in structure, physiology and life history, +DUMMY/LJ014-0090.wav|On the Sunday Mrs. Manning roasted a goose at this same kitchen fire, and ate it with relish in the afternoon. +DUMMY/LJ003-0335.wav|The committee did not deny the superior advantages offered by such prisons as Gloucester and Petworth, +DUMMY/LJ010-0028.wav|The causes also continued much the same. Passion, revenge, cupidity, sudden ebullitions of homicidal rage, +DUMMY/LJ024-0077.wav|is it not a most desirable precedent? +DUMMY/LJ010-0020.wav|the extraordinary extension of all commercial operations led to many entirely novel and often gigantic financial frauds. +DUMMY/LJ028-0067.wav|Centuries afterward, even to this day, Jewish mothers teach their children to hate his name. +DUMMY/LJ015-0092.wav|When his friends asked why he, a wealthy man of independent means, continued to slave on as a clerk on a pittance, +DUMMY/LJ042-0011.wav|On August seventeen, nineteen sixty-three, Oswald told Mr. William Stuckey, +DUMMY/LJ012-0198.wav|Bishop was a carpenter, but having been suddenly thrown out of work, he joined his brother-in-law in his line of business. +DUMMY/LJ046-0229.wav|The Commission believes that the facilities and procedures of the Protective Research Section of the Secret Service +DUMMY/LJ032-0075.wav|with the address of post office box two nine one five in Dallas. +DUMMY/LJ031-0164.wav|and a move would require new measures. +DUMMY/LJ039-0047.wav|Marina Oswald speculated that the incident may have been unrelated to an actual threat. +DUMMY/LJ023-0024.wav|not this week or month perhaps, but within a year or two. +DUMMY/LJ039-0226.wav|the Marine marksmanship experts, Major Anderson and Sergeant Zahm, concurred in the opinion that Oswald had the capability to fire three shots, +DUMMY/LJ047-0057.wav|until March nineteen sixty-three, the FBI continued to accumulate information regarding Oswald but engaged in no active investigation. +DUMMY/LJ030-0209.wav|He immediately looked to his right, quote, and, in so doing, my eyes had to cross the Presidential limousine +DUMMY/LJ049-0213.wav|There are always dangers of divided responsibility, +DUMMY/LJ010-0117.wav|The employment of Edwards, and the manner in which the conspirators were allowed to commit themselves further and further before the law was set in motion against them, +DUMMY/LJ047-0147.wav|The Department of State did not advise either the CIA or the FBI of these facts. +DUMMY/LJ027-0101.wav|Throughout both the animal and vegetable kingdoms dwarfed and useless representatives of organs are constantly met with, +DUMMY/LJ008-0161.wav|presently fell, and were at once trampled to death. +DUMMY/LJ049-0112.wav|bear no relationship to the injury to the United States which follows from the act. +DUMMY/LJ019-0319.wav|while the latter, which included various forms of industrial employment, +DUMMY/LJ016-0097.wav|which with Wakefield was utilized as a receptacle for convicts not going to Western Australia, +DUMMY/LJ014-0039.wav|Hocker was so overcome, feeling himself the principal cause of the tragedy, +DUMMY/LJ011-0163.wav|She was not in immediate danger, but she wished to see her daughter, "as it was possible she might soon become incapable of recognizing any one." +DUMMY/LJ046-0252.wav|Although the CIA had on file requests from the Treasury Department for information on the counterfeiting of U.S. currency and certain smuggling matters, +DUMMY/LJ016-0085.wav|After this great iron guards, just as are to be seen in lunatic asylums, +DUMMY/LJ010-0050.wav|But at various dates treason more distinct and tangible came to the front: attempts to levy war against the State. +DUMMY/LJ025-0086.wav|and the few and exceptional cases of non-parasitic animals which do not feed at all. +DUMMY/LJ047-0155.wav|readily admitted that Mrs. Marina Oswald and Lee Oswald's two children were staying with her. +DUMMY/LJ030-0178.wav|Looking over her right shoulder, she saw that the President had both hands at his neck but she observed no blood and heard nothing. +DUMMY/LJ034-0098.wav|In the six to eight minute period before the motorcade arrived, Brennan saw a man leave and return to the window, quote, a couple of times, end quote, +DUMMY/LJ037-0197.wav|and the return address was Post Office Box two nine one five, Dallas, Texas. +DUMMY/LJ031-0140.wav|When consulted by the Vice President, O'Donnell advised him to go to the airfield immediately and return to Washington. +DUMMY/LJ042-0027.wav|In addition to studying the Russian language while he was in the Marines, +DUMMY/LJ048-0057.wav|there might conceivably have been additional investigation of the Oswald case between November five and November twenty-two. +DUMMY/LJ038-0290.wav|provide important corroboration for her account of the incident. +DUMMY/LJ038-0273.wav|Joseph D. Nicol, +DUMMY/LJ036-0162.wav|On November twenty-two, Oswald told Captain Fritz that he rode a bus to a stop near his home and then walked to his roominghouse. +DUMMY/LJ047-0132.wav|the FBI transferred the principal responsibility for the Oswald case from the Dallas office to the New Orleans office. +DUMMY/LJ001-0007.wav|the earliest book printed with movable types, the Gutenberg, or "forty-two line Bible" of about fourteen fifty-five, +DUMMY/LJ044-0130.wav|By August of nineteen sixty-three, after a short three months in New Orleans, the city in which he had been born and had lived most of his early life, +DUMMY/LJ032-0041.wav|These words were also in the handwriting of Lee Harvey Oswald. +DUMMY/LJ010-0312.wav|Among the prisoner's private papers, one was found giving full details of the stock he had feloniously sold out, +DUMMY/LJ018-0138.wav|Some time in eighteen sixty-two, a large deficiency in stock of bank paper unglazed was discovered at the mills. +DUMMY/LJ015-0208.wav|they sounded one Burgess, a guard on the South-Eastern Railway, a line by which large quantities of bullion were sent to the Continent. +DUMMY/LJ041-0031.wav|Oswald praised Khrushchev and suggested that he and McBride join the Communist Party, quote, to take advantage of their social functions, end quote. +DUMMY/LJ008-0093.wav|At the execution of Governor Wall, in eighteen oh two, the furious hatred of the mob was plainly apparent in their appalling cries. +DUMMY/LJ032-0136.wav|In this way the powdered impression is actually removed from the object. +DUMMY/LJ038-0009.wav|When he heard police sirens, he, quote, looked up and saw the man enter the lobby, end quote. +DUMMY/LJ012-0272.wav|They were apparently good friends when last seen together at a neighbor's, where they seemed "perfectly happy and sociable, and eager for the wedding day." +DUMMY/LJ050-0078.wav|In June nineteen sixty-four, the Secret Service sent to a number of Federal law enforcement and intelligence agencies +DUMMY/LJ012-0004.wav|Inquiries set on foot also elicited the suspicion that the person who had represented Mrs. Canning's brother +DUMMY/LJ005-0196.wav|still no separation of the sexes, no means of ablution or other necessary services. +DUMMY/LJ016-0023.wav|the wounds broke open and bled profusely while the last dread penalty was being performed, to the manifest excitement and indignation of the crowd. +DUMMY/LJ004-0136.wav|Mr. Buxton, who found this, declared that it seemed physically impossible, but he was assured that it was true, +DUMMY/LJ018-0072.wav|The inspector said that they could not be removed till he reached England. +DUMMY/LJ050-0066.wav|The volume of references to the Secret Service has increased substantially since the new instructions went into effect; +DUMMY/LJ038-0211.wav|he said that he, quote, was very sorry that he had not hit him, end quote. Marina Oswald's testimony was fully supported by the note itself +DUMMY/LJ035-0193.wav|Truly, who had returned with Patrolman Baker from the roof, saw the police questioning the warehouse employees. +DUMMY/LJ039-0010.wav|she had failed to mention the incident when she was asked whether Oswald had ever expressed any hostility +DUMMY/LJ019-0266.wav|The efforts made by Sir James Graham years before to introduce uniformity in this particular had failed of effect. +DUMMY/LJ044-0188.wav|Oswald must have been thoroughly disillusioned when he left Mexico City on October two, nineteen sixty-three. +DUMMY/LJ028-0180.wav|In the year five sixty-two, after a long reign of forty-three years, Nebuchadnezzar died. +DUMMY/LJ012-0238.wav|He fainted, and the warrant of mercy nearly proved his death-blow. The other two looked on at his agitation with an indifference amounting to apathy. +DUMMY/LJ042-0024.wav|He wrote in another letter that he would, quote, never return to the United States which is a country I hate, end quote. +DUMMY/LJ001-0093.wav|This experiment was so far successful that about eighteen fifty Messrs. Miller and Richard of Edinburgh +DUMMY/LJ024-0078.wav|Like all lawyers, like all Americans, I regret the necessity of this controversy. +DUMMY/LJ041-0078.wav|When the officers were unable to discuss foreign affairs satisfactorily with him, Oswald regarded them as unfit to exercise command over him. +DUMMY/LJ046-0133.wav|Since many items may pertain to a single case, these figures do not show the caseload. +DUMMY/LJ006-0166.wav|dice, dangerous implements, or other prohibited articles. +DUMMY/LJ019-0190.wav|"but in a metropolitan prison absolutely essential." The Corporation in reply demurred rather to accepting strict separation as a rule, +DUMMY/LJ002-0002.wav|Under the conditions referred to in the previous chapter, +DUMMY/LJ003-0132.wav|whose language and manners, whose female associates of the most abandoned description, and the scenes consequent with such lost wretches +DUMMY/LJ044-0054.wav|and a newspaper clipping reporting the event, suggests that his previous story of an attack by Cuban exiles was at least greatly exaggerated. +DUMMY/LJ045-0093.wav|She said that she, quote, always tried to point out to him that he was a man like any others who were around us. But he simply could not understand that? End quote. +DUMMY/LJ019-0374.wav|It was nearly impossible for them to exercise a very close supervision over the whole of the prisons of the country. +DUMMY/LJ036-0133.wav|and he opened the door a little bit like he was going to get out and he said, "I will let you have this one," and she says, "No, the driver can call me one." +DUMMY/LJ001-0006.wav|And it is worth mention in passing that, as an example of fine typography, +DUMMY/LJ010-0307.wav|New liabilities were incurred to the extent of one hundred thousand pounds by more failures, and in eighteen nineteen, +DUMMY/LJ005-0065.wav|and none are more commendable than that which deprecates the excessive ornamentation of the external parts of the edifice. +DUMMY/LJ045-0043.wav|In spite of fact I married Marina to hurt Ella (the girl that had rejected him) I found myself in love with Marina, end quote. +DUMMY/LJ029-0061.wav|The FBI office in Dallas gave the local Secret Service representatives the name of a possibly dangerous individual in the Dallas area who was investigated. +DUMMY/LJ007-0177.wav|We trust, however, that the day is at hand when this stain will be removed from the character of the city of London, +DUMMY/LJ009-0112.wav|the burglar muttering curses and savage expressions of defiance; whilst the poor sheep-stealer shakes hands with the turnkeys, +DUMMY/LJ019-0203.wav|But the condition of the great city jail was evidently considered a reproach by the city authorities, and a year after the opening of the new "model" at Pentonville, +DUMMY/LJ032-0057.wav|The post office box to which the rifle was shipped was rented to "Lee H. Oswald" from October nine, nineteen sixty-two, +DUMMY/LJ012-0167.wav|But Probert, who turned king's evidence, and materially assisted conviction, +DUMMY/LJ045-0080.wav|Generally, I think that was right, for such things that is the right thing to do. There was some grounds for it. End quote. +DUMMY/LJ010-0088.wav|and that when it was opened the whole band should rush in. +DUMMY/LJ038-0153.wav|He said that he left work because Bill Shelley said that there would be no more work done that day in the building. +DUMMY/LJ030-0152.wav|Speed of the Limousine +DUMMY/LJ004-0026.wav|In eighteen fourteen a bill was brought in to insist upon the appointment of chaplains in jails, and when this had passed into law, +DUMMY/LJ027-0129.wav|Convincing as are the evidences of descent recorded in the structure of plants and animals, +DUMMY/LJ031-0139.wav|and made arrangements through Secret Service headquarters in Washington to provide them with protection immediately. +DUMMY/LJ014-0296.wav|He had made use of a piece of rope cut out from the sacking of his bedstead, and had tied his feet together with a silk pocket-handkerchief. +DUMMY/LJ014-0254.wav|This he did to an enormous extent, +DUMMY/LJ017-0268.wav|and they were each kept in an ordinary separate cell of the newly-constructed block, with the "traps," or square openings in the cell door, +DUMMY/LJ029-0113.wav|gave more people an opportunity to participate. +DUMMY/LJ004-0060.wav|"You have no right," he says, addressing the authorities, "to subject a prisoner to suffering from cold, +DUMMY/LJ038-0045.wav|Deputy Sheriff Eddy R. Walthers recalled such a remark but he did not reach the scene of the struggle +DUMMY/LJ018-0264.wav|Roupell was quiet and submissive while in Newgate, unassuming in manner, and ready to make the best of his position. +DUMMY/LJ010-0145.wav|These were the last executions for high treason, but not the last prisoners by many who passed through Newgate charged with sedition. +DUMMY/LJ034-0019.wav|on which appeared the print of the left index finger and right palm of Lee Harvey Oswald. +DUMMY/LJ046-0110.wav|is the identification and elimination of possible sources of danger to the President before the danger becomes actual. +DUMMY/LJ001-0068.wav|The characteristic Dutch type, as represented by the excellent printer Gerard Leew, is very pronounced and uncompromising Gothic. +DUMMY/LJ033-0038.wav|the Commission gave weight to the fact that Oswald gave a false reason for returning home on November twenty-one, +DUMMY/LJ008-0295.wav|When the council had decided, the news was conveyed to Newgate by the Recorder, who made his "report," as it was called. +DUMMY/LJ015-0176.wav|He was arrested, examined before a police magistrate, and committed to Newgate. +DUMMY/LJ008-0163.wav|Panic became general. More women, children, and many men were borne down, to perish beneath the feet of the rest. +DUMMY/LJ010-0059.wav|Some five hundred in all were arrested, but as only three of these were lodged in Newgate, I shall not recur to them in my narrative. +DUMMY/LJ003-0262.wav|Any reputed, and indeed any real, wife might spend the night in Newgate if she would pay the shilling fee, commonly known as the "bad money," +DUMMY/LJ016-0301.wav|amid hisses and sneering inquiries of what he had had to drink that morning. +DUMMY/LJ008-0193.wav|by seven a.m. the whole space was so thronged that it was impossible to move one way or the other. +DUMMY/LJ038-0134.wav|An examination of the roominghouse register revealed that Oswald actually signed the name O. H. Lee. +DUMMY/LJ016-0209.wav|Some wags in Scotland seized Calcraft and kept him in durance the night before the execution. +DUMMY/LJ008-0270.wav|and while the courts adhered to the letter of the law, appeals were constantly made to the royal prerogative of mercy. +DUMMY/LJ036-0066.wav|He could not reach his roominghouse on the Marsalis bus, but the Beckley bus stopped across the street. +DUMMY/LJ007-0161.wav|drink, gaming, obscene and blasphemous language; utter idleness, the almost unrestricted admission of money and luxuries; +DUMMY/LJ050-0217.wav|the Secret Service sought funds for twenty-five new positions, primarily in field offices. This increase has been approved by the Congress. +DUMMY/LJ028-0383.wav|and the wild beasts of the islands shall cry in their desolate houses, and dragons in their pleasant palaces. +DUMMY/LJ014-0182.wav|the shutters were up, and he was preparing to close, when Marley entered and fell upon him with a life-preserver, +DUMMY/LJ033-0204.wav|So if I found all of these then I would have been able to say these fibers probably had come from this blanket. But since I found so few, +DUMMY/LJ048-0099.wav|Agent Lawson did the advance work alone from November thirteen to November eighteen, when he was joined by Agent David B. Grant, +DUMMY/LJ047-0133.wav|Soon after, on October one, nineteen sixty-three, +DUMMY/LJ032-0036.wav|purchased as Number two two zero two one three zero four six two in Dallas, Texas, on March twelve, nineteen sixty-three. +DUMMY/LJ001-0012.wav|especially as no more time is occupied, or cost incurred, in casting, setting, or printing beautiful letters +DUMMY/LJ038-0241.wav|Also seen in the picture is the fence on which Walker's assailant apparently rested the rifle. +DUMMY/LJ019-0027.wav|The internal arrangements of the new model were carefully supervised by a body of distinguished men, among which were many peers, Lord John Russell, +DUMMY/LJ040-0202.wav|Mrs. Siegel described her as a smartly dressed, gray haired woman, very self-possessed and alert and superficially affable, +DUMMY/LJ018-0368.wav|The still more recent cases of poisoning which have occurred were not connected with Newgate. +DUMMY/LJ013-0036.wav|According to their own statements the loss of the 'Dryad' was only one of six intentional shipwrecks with which they had been concerned. +DUMMY/LJ035-0045.wav|Baker said, quote, He [Truly] had already started around the bend to come to the next elevator going up, +DUMMY/LJ043-0163.wav|If he had been successful and had been apprehended even for routine questioning, his apartment would undoubtedly have been searched, +DUMMY/LJ045-0111.wav|They asked for Lee Oswald who was not called to the telephone because he was known by the other name. +DUMMY/LJ016-0331.wav|The last public execution in front of Newgate was that of the Fenian Michael Barrett, +DUMMY/LJ002-0293.wav|those who could pay nothing went, as a matter of course, to the common side; +DUMMY/LJ008-0048.wav|it is supported by two beams, which are held in their place by bolts. The movement of the lever withdraws the bolts, the platform falls in; +DUMMY/LJ033-0128.wav|that the bag Oswald carried contained the assassination weapon and has concluded that Frazier and Randle are mistaken as to the length of the bag. +DUMMY/LJ011-0040.wav|I find in the chaplain's journal, under date eighteen twenty-four, various entries relative to this prisoner. +DUMMY/LJ016-0323.wav|to prevent abuse, and satisfy the public that the law had been complied with. +DUMMY/LJ031-0142.wav|The Vice President conferred with White House Assistant Press Secretary Malcolm Kilduff +DUMMY/LJ026-0074.wav|The gaseous matters (carbon dioxide, oxygen, nitrogen) enter the plant mainly by diffusion from the atmosphere, +DUMMY/LJ024-0068.wav|and one over seventy. +DUMMY/LJ050-0081.wav|The new tentative criteria are useful in making clear that +DUMMY/LJ007-0014.wav|The admission of a crowd of visitors to assist in these lay services has already been remarked upon; as the inspectors pointed out, +DUMMY/LJ015-0010.wav|Cole now suspended payment. +DUMMY/LJ046-0241.wav|PRS required a more direct indication of a threat to the President, and that there was no such indication until the President's scheduled visit to that area became known. +DUMMY/LJ008-0173.wav|In this convulsive struggle for bare existence people fought fiercely with one another, and the weakest, of course the women, went under. +DUMMY/LJ018-0260.wav|Mr. Justice Byles, in passing sentence, commented severely upon the commission of such crimes by a man in Roupell's position in life, +DUMMY/LJ003-0156.wav|All the money earned by prisoners was at their own disposal, and was spent almost habitually in drink, chambering, and wantonness. +DUMMY/LJ022-0081.wav|Funds allotted for each project should be actually and promptly spent and not held over until later years. +DUMMY/LJ043-0103.wav|In any event, Oswald was discharged on April six, nineteen sixty-three, ostensibly because of his inefficiency and difficult personality. +DUMMY/LJ026-0137.wav|and be laid down as "reserve starch" in the cells of root or stem or elsewhere. +DUMMY/LJ012-0064.wav|Ikey left no traces, and the police could hear nothing of him. He had in fact gone out of the country, to Copenhagen, whence he passed on to New York. +DUMMY/LJ018-0391.wav|Mr. Bamell Oakley made a rich harvest for a time, and was said at the time of his trial +DUMMY/LJ043-0134.wav|The Commission has concluded that on April ten, nineteen sixty-three, +DUMMY/LJ036-0202.wav|At twelve:fifty-four p.m., Tippit reported that he was in the central Oak Cliff area at Lancaster and Eighth. +DUMMY/LJ038-0162.wav|Then he said, "Oh, I see," and that was all. +DUMMY/LJ037-0177.wav|Therefore, either one bullet of Remington-Peters manufacture is missing +DUMMY/LJ037-0041.wav|The man appeared to step back as the policeman, quote, calmly opened the car door, end quote, and very slowly got out and walked toward the front of the car. +DUMMY/LJ045-0046.wav|Marina Oswald expressed one aspect of her husband's attitude toward her when she testified that, quote, Lee wanted me to go to Russia. +DUMMY/LJ047-0154.wav|The interview lasted about twenty to twenty-five minutes. In response to Hosty's inquiries, Mrs. Paine, quote, +DUMMY/LJ026-0096.wav|water, nitrites or nitrates and various mineral substances in solution in water +DUMMY/LJ003-0116.wav|A jury having been selected and duly sworn, the culprit was then arraigned. Justice, however, was not administered with absolute integrity. +DUMMY/LJ010-0180.wav|the Prince too rose to shield her with his person. Again, providentially, the bullet went wide of the mark, +DUMMY/LJ041-0056.wav|in his intense desire to join the Marines and get away from his surroundings and his mother. +DUMMY/LJ019-0309.wav|The main object of this act was to compass that uniformity in discipline and treatment generally +DUMMY/LJ004-0199.wav|This made exertion compulsory, and imposed hard labor as a proper punishment. +DUMMY/LJ024-0118.wav|Look at these strange bed-fellows of yours. +DUMMY/LJ029-0023.wav|In September, the White House decided to permit further visits by the President and extended the trip to run from the afternoon of November twenty-one +DUMMY/LJ047-0142.wav|The Bureau had no earlier information suggesting that Oswald had left the United States. +DUMMY/LJ010-0007.wav|Police protection was better and more effective; +DUMMY/LJ037-0192.wav|Seaport Traders, Incorporated, received through the mail a mail-order coupon for one, quote, point three-eight +DUMMY/LJ012-0183.wav|During the night they sought to dispose of the body by throwing it into a pond, but two days later had to throw it into another pond. +DUMMY/LJ048-0276.wav|Since he was convinced that this was not the case, he believed that it would be unfair to the agents and their families to take explicit disciplinary measures. +DUMMY/LJ003-0278.wav|Insuperable difficulties were still supposed to stand in the way of any general employment of prisoners at their trades. +DUMMY/LJ025-0040.wav|The soil and the atmosphere supply plants with water composed of hydrogen and oxygen +DUMMY/LJ011-0020.wav|had been sold out under a forged power of attorney. +DUMMY/LJ047-0143.wav|The possible contact with the Soviet Embassy in Mexico intensified the FBI's interest in learning Oswald's whereabouts. +DUMMY/LJ010-0041.wav|or like Cannon the chimney-sweeper, who savagely killed the policeman. +DUMMY/LJ030-0055.wav|He acted on instructions he had received earlier from Assistant Special Agent in Charge Roy H. Kellerman, who was in Fort Worth with the President. +DUMMY/LJ048-0124.wav|The Secret Service has advised the Commission that no unusual precautions were taken for the Dallas trip, and that, quote, +DUMMY/LJ021-0114.wav|Another question arises as to whether in fixing minimum wages on the basis of an hourly or weekly wage +DUMMY/LJ029-0005.wav|although in many instances documentary or other evidence has also been used by the Commission. +DUMMY/LJ020-0041.wav|The phosphates which the process of “bolting” removes to a large extent from white flour, go directly to the manufacture of bone, +DUMMY/LJ021-0072.wav|Also, billions of dollars of invested capital have today a greater security of present and future earning power than before. +DUMMY/LJ015-0204.wav|This was the question which presented itself to the fertile brain of one Pierce, +DUMMY/LJ038-0215.wav|It is clear that the note was written while the Oswalds were living in Dallas before they moved to New Orleans in the spring of nineteen sixty-three. +DUMMY/LJ009-0207.wav|extended at full length on an inclined platform +DUMMY/LJ010-0229.wav|Within a couple of years a second attempt to assassinate the Queen was perpetrated in nearly the same spot, by a man named John Francis, +DUMMY/LJ016-0258.wav|This line of argument prevailed over the manifest horrors of the spectacle. These increased as time passed. +DUMMY/LJ033-0051.wav|Neither Marina Oswald nor Ruth Paine saw Oswald in the garage. +DUMMY/LJ014-0141.wav|and stretching out her hand, she gathered up a quantity of the rue which, following ancient custom dating from the days of the jail fever, +DUMMY/LJ048-0145.wav|and no written description of the role the local police were expected to perform. +DUMMY/LJ006-0288.wav|The separation of the sexes was not indeed rigidly carried out in Newgate as yet. +DUMMY/LJ048-0248.wav|President Kennedy was scheduled to speak across the street from his hotel in Fort Worth at eight:thirty a.m. +DUMMY/LJ025-0136.wav|Of course the similarity of the acts does not necessarily involve the conclusion that the mechanism by which they are effected is the same +DUMMY/LJ035-0108.wav|Eddie Piper, the janitor, told Dougherty that the President had been shot, +DUMMY/LJ025-0147.wav|with a problem of extreme difficulty, which must be attacked from a new point of view and by the aid of methods which have yet to be invented. +DUMMY/LJ008-0155.wav|Very great excitement prevailed in the town throughout the trial, and this greatly increased when the verdict was known. +DUMMY/LJ036-0184.wav|When Oswald came in she said, quote, Oh, you are in a hurry, end quote, but Oswald did not respond. +DUMMY/LJ049-0134.wav|The Commission suggests that consideration might be given to assigning to a Cabinet-level committee or the National Security Council +DUMMY/LJ006-0192.wav|There was no school for adults; only the boys were taught anything, and their instructor, with his assistant, were convicted prisoners. +DUMMY/LJ015-0125.wav|Redpath's crime arose from his peculiar and independent position as registrar of stock of the Great Northern Railway Company. +DUMMY/LJ002-0197.wav|whose net annual income thus entirely derived from the impecunious amounted to between three and four thousand pounds. +DUMMY/LJ018-0228.wav|Five or six years later, William Roupell minutely described how he had effected the fraud. +DUMMY/LJ022-0065.wav|The unemployment insurance part of the legislation +DUMMY/LJ046-0036.wav|regarding certain protective measures in force at the time of the Dallas trip and propose recommendations for improvements. +DUMMY/LJ013-0106.wav|The inn belonging to Burgess was surrounded +DUMMY/LJ011-0043.wav|that this may prove the beginning of a better system of confinement, and that every description of persons who may be unfortunately under sentence of death +DUMMY/LJ048-0077.wav|FBI-agency community of interests, location of agency head quarters, and the responsiveness of agency representatives. +DUMMY/LJ004-0031.wav|But the effect of this legislation was rather weakened by the remoteness of the pressure exercised. +DUMMY/LJ006-0213.wav|The quantity to be issued per head was limited by the prison regulations to one pint +DUMMY/LJ015-0260.wav|As the evidence he gave incriminated Pierce, Burgess, and Tester, all three were arrested and committed to Newgate for trial. +DUMMY/LJ016-0177.wav|Marwood, whose name is so familiar with the present generation, +DUMMY/LJ030-0188.wav|At the sound of the second shot +DUMMY/LJ016-0310.wav|Another distinguished witness feared +DUMMY/LJ014-0119.wav|He had fled to Jersey, where he was recognized and arrested. +DUMMY/LJ028-0003.wav|With guides and guards about you, you ride through the covered bazaars crowded with dark-faced Arabs in strange costumes, +DUMMY/LJ036-0042.wav|However, McWatters' recollection alone was too vague to be a basis for placing Oswald on the bus. +DUMMY/LJ033-0046.wav|went out to the garage to paint some children's blocks, and worked in the garage for half an hour or so. +DUMMY/LJ028-0287.wav|but found none by which he could hope to prevail, unless he maimed himself and then went over to the enemy. +DUMMY/LJ038-0291.wav|Other details described by Marina Oswald coincide with facts developed independently of her statements. +DUMMY/LJ007-0185.wav|justified the original allegations. It was impossible to deny the indiscriminate association; the gambling, drinking, smoking, quarreling in the jail; +DUMMY/LJ009-0160.wav|In eighteen thirty-two +DUMMY/LJ010-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section thirteen: Newgate notorieties, part one. +DUMMY/LJ016-0069.wav|Entering a town, he obtained employment at once as a chimney-sweep +DUMMY/LJ013-0121.wav|Howse and his accomplice were arrested; the former was found guilty and sentenced to fifteen years, but the latter was acquitted. +DUMMY/LJ015-0189.wav|and the prototype of a class not uncommon in our later days +DUMMY/LJ017-0017.wav|It is quite possible that Catherine Blandy, who poisoned her father at the instigation of her lover, +DUMMY/LJ020-0017.wav|Now whip up the batter with a wooden spoon for another minute, and the sponge is made. +DUMMY/LJ011-0108.wav|I wish after this day to have communication with nobody; let me take leave of my wife, and family, and friends. I have already suffered an execution; +DUMMY/LJ039-0183.wav|On the basis of these results, Simmons testified that in his opinion the probability of hitting the targets at the relatively short range at which they were hit +DUMMY/LJ029-0107.wav|On November eighteen, +DUMMY/LJ019-0321.wav|The infliction of punishment more or less uniform was thus aimed at. +DUMMY/LJ027-0051.wav|It means blood relationship. +DUMMY/LJ003-0195.wav|Mr. Bennet speaks of thirty-eight capital convicts he found in the press yard in February eighteen seventeen, +DUMMY/LJ015-0215.wav|The safes while on the line en route between London and Folkestone were in the guard's van. +DUMMY/LJ022-0164.wav|It will put the public utility operating industry on a sound basis for the future, +DUMMY/LJ050-0095.wav|The U.S. government has no charges or complaints against me. +DUMMY/LJ038-0066.wav|While the police radio had noted the similarity in description of the two suspects, the arresting officers were pursuing Oswald for the murder of Tippit. +DUMMY/LJ001-0139.wav|a blemish which can be nearly, though not wholly, avoided by care and forethought +DUMMY/LJ016-0307.wav|Full evidence was taken on all points, and on that regarding public executions there was a great preponderance of opinion towards their abolition, +DUMMY/LJ040-0213.wav|that he had great difficulty in adapting himself to conditions in that city. +DUMMY/LJ034-0200.wav|Shortly after the assassination, Euins signed an affidavit describing the man as "white," +DUMMY/LJ015-0293.wav|as coming from a Mr. Whitney. +DUMMY/LJ034-0076.wav|Givens said to Oswald, quote, Boy are you going downstairs? It's near lunch time, end quote. +DUMMY/LJ012-0041.wav|As a general rule Ikey Solomons confined his purchases to small articles, mostly of jewelery and plate, +DUMMY/LJ049-0075.wav|Recommendations. +DUMMY/LJ010-0035.wav|Courvoisier might have lived a century earlier. +DUMMY/LJ049-0212.wav|to put it in a position to reach final conclusions in this respect. +DUMMY/LJ018-0036.wav|The officers went on board the 'Victoria' at once, Müller was identified by Mr. Death, and the arrest was made. +DUMMY/LJ035-0046.wav|I was coming out this one on the second floor, and I don't know, I was kind of sweeping this area as I come up, I was looking from right to left +DUMMY/LJ002-0101.wav|there was still a "convenient room" which served, and, quote, +DUMMY/LJ043-0145.wav|He told his wife that he wanted to send the pictures to the Militant and he also asked her to keep one of the pictures for his daughter, June. +DUMMY/LJ011-0221.wav|and was brought from Newgate, at his own request, to oppose the second reading of the bill. +DUMMY/LJ003-0303.wav|The personal cleanliness of all prisoners was to be insisted upon; they should be made to wash at least once a day, +DUMMY/LJ022-0141.wav|to be interpreted as lack of interest in or disapproval of many other important proposals that are pending. +DUMMY/LJ040-0179.wav|Irving Sokolow, a Youth House psychologist reported that, quote, +DUMMY/LJ002-0079.wav|Its name and its situation were the same as those of the old place of carrying out the terrible sentence inflicted on accused persons who stood mute. +DUMMY/LJ002-0247.wav|As the business of the Marshalsea Court declined, the numbers in its prison diminished. +DUMMY/LJ033-0161.wav|On the day of the assassination, the Dallas police obtained a sample of wrapping paper and tape +DUMMY/LJ004-0144.wav|had to endure by "the regulations of the city a disease very dangerous in its nature," and ran the risk of a lingering and painful death. +DUMMY/LJ004-0017.wav|Some of the county jails, and one or two borough jails, had been rebuilt, +DUMMY/LJ019-0005.wav|There had been a pause in legislation, except of a permissive kind. The second and third Victoria, cap. fifty-six +DUMMY/LJ016-0050.wav|His next job was to descend outside Newgate. +DUMMY/LJ004-0115.wav|Infirmaries separating the sexes were also to be provided, a chapel too, and warm and cold baths. +DUMMY/LJ015-0196.wav|Highway robberies, for instance, had disappeared, if we except the spasmodic and severely repressed outbreak of "garotting," +DUMMY/LJ028-0480.wav|the exalted prince, the protector of Esagil and Ezida, son of Nabopolassar, King of Babylon, am I. +DUMMY/LJ044-0236.wav|it is unlikely that a reasoning person would plan to attempt to travel from Dallas, Texas to Cuba +DUMMY/LJ025-0060.wav|and now such movements of the living substance of plants are well known to be some of the most widely prevalent phenomena of vegetable life. +DUMMY/LJ026-0158.wav|Probably this freed oxygen is used for the purpose of oxygenation, but more is freed in the photosynthetic process than is needed for oxygenation +DUMMY/LJ013-0231.wav|"Where is it?" asked the governor. +DUMMY/LJ004-0228.wav|In eighteen eighteen prisoners awaiting trial in Newgate, were at length relieved from this illegal infliction. +DUMMY/LJ016-0231.wav|When he came to the prison for his wages his grandchildren often accompanied him, affectionately clinging to his hands; +DUMMY/LJ031-0200.wav|The President then walked to the Executive Office Building, where he worked until nine p.m. +DUMMY/LJ016-0340.wav|and Calcraft, who had been threatened with summary retribution if he executed Barrett, carried out the sentence without mishap. +DUMMY/LJ002-0222.wav|It contained generally from six to seven hundred inmates, while another hundred more or less resided in the rules outside. +DUMMY/LJ016-0329.wav|a bill was introduced by Mr. Hibbert, M.P., and accepted by the Government, providing for the future carrying out of executions within prisons. +DUMMY/LJ025-0099.wav|Moreover, although it remains true that there is an inverse relation between the green plant in sunshine and the animal, +DUMMY/LJ004-0064.wav|"You have for the same reason," he goes on, "no right to ruin his habits by compelling him to be idle, +DUMMY/LJ012-0192.wav|the first of whom has added to the British language a synonym for illegal suppression. +DUMMY/LJ040-0183.wav|There is some indication that he may relate to men more easily than to women in view of the more mature conceptualisation. +DUMMY/LJ026-0070.wav|Of the substances the solids (salts, etc.) must be dissolved in water before they can be taken in. +DUMMY/LJ015-0146.wav|He had a nice taste in bric-à-brac, and was considered a good judge of pictures. +DUMMY/LJ032-0170.wav|the Commission has concluded that the fibers in the tuft on the rifle most probably came from the shirt worn by Oswald when he was arrested, +DUMMY/LJ030-0148.wav|Seconds after the shooting, Roy Kellerman, riding in the front seat of the Presidential limousine, +DUMMY/LJ031-0133.wav|Jack Brooks, Homer Thornberry, and Albert Thomas joined Clifton C. Carter and the group of special agents protecting the Vice President. +DUMMY/LJ018-0363.wav|She was found guilty. +DUMMY/LJ042-0128.wav|I am stating to reconsider my desire about staying. The work is drab. The money I get has nowhere to be spent. +DUMMY/LJ010-0195.wav|He smiled continually, and when the indictment was read, burst into loud and discordant fits of laughter. +DUMMY/LJ019-0137.wav|The prison population of Newgate was still subject to great fluctuations, but it seldom rose above two hundred and fifty or three hundred +DUMMY/LJ031-0025.wav|urological surgeon, Dr Paul C. Peters; an oral surgeon, Dr. Don T. Curtis; and a heart specialist, +DUMMY/LJ040-0226.wav|I couldn't in all honesty sincerely say such a thing. +DUMMY/LJ027-0055.wav|"All vertebrate animals, and none other," says Le Conte, "have an internal jointed skeleton worked by muscles on the outside. +DUMMY/LJ005-0228.wav|As yet no rules or regulations had been printed or prepared. +DUMMY/LJ033-0100.wav|Frazier recalled that one end of the package was under Oswald's armpit and the lower part was held with his right hand +DUMMY/LJ016-0011.wav|The ladder was actually made, of black sewing-thread firmly and closely interwoven. But White could not remove the bars; +DUMMY/LJ050-0127.wav|should negotiate similar arrangements with such other State and local law enforcement agencies as may provide meaningful assistance. +DUMMY/LJ049-0012.wav|At these stops, agents from the Presidential follow-up car stood between the President and the public, +DUMMY/LJ026-0142.wav|It is also clear that catabolism or oxidation for the liberation of energy occurs as in animals, but this process is slower. +DUMMY/LJ014-0059.wav|Their victim was a man named Patrick O'Connor, a Custom-House gauger, who had been a suitor of Marie de Roux before she became Mrs. Manning. +DUMMY/LJ037-0267.wav|to the exclusion of all other weapons, +DUMMY/LJ018-0379.wav|Resolute, determined-looking men, their courage broke down in confinement. +DUMMY/LJ046-0066.wav|And he felt that leaving Washington for the President of the United States was a most necessary -- not only for the people, but for the President himself, +DUMMY/LJ014-0321.wav|When such goods arrived they were frequently left at a wharf, paying rent until it suited the importer to remove them. +DUMMY/LJ021-0179.wav|Did England hesitate to call in ten billion dollars of her war bonds bearing five percent interest, +DUMMY/LJ002-0142.wav|These courts were open to many and grave objections. +DUMMY/LJ005-0121.wav|By degrees, however, +DUMMY/LJ007-0059.wav|had relaxed his efforts, because, according to his own account, he was so frequently stopped in the performance of his duties. +DUMMY/LJ030-0225.wav|But he was immediately called back by Special Agent Emory P. Roberts, in charge of the follow-up car, +DUMMY/LJ044-0217.wav|was also reported in the October one, nineteen sixty-three, issue of the Worker, to which Oswald also subscribed. +DUMMY/LJ043-0054.wav|Oswald moved to Dallas on about October eight, nineteen sixty-two, without telling his mother where he was going. +DUMMY/LJ044-0138.wav|It would not have been possible to have followed Arnold Johnson's advice to remain in the background, +DUMMY/LJ018-0167.wav|Nothing was found in Buncher's premises in Strutton Ground, which were thoroughly searched, +DUMMY/LJ014-0031.wav|The prisoner made an elaborate defense, in which he sought to vilify the character of deceased +DUMMY/LJ008-0301.wav|during which they counted the moments -- the prisoners in their cells as usual, and their friends in the street in front of Newgate, where they passed the night. +DUMMY/LJ011-0217.wav|Mr. Justice Bayley, in summing up, +DUMMY/LJ047-0183.wav|When asked what his next step would have been, Hosty replied, quote, +DUMMY/LJ011-0193.wav|Then Mrs. Daulby learnt that Miss Turner had not arrived at Shrigley, but that she had gone to Manchester. +DUMMY/LJ019-0184.wav|The Corporation had agreed to spend twenty thousand pounds on alterations, but sixty thousand pounds would suffice to reconstruct. +DUMMY/LJ017-0062.wav|In the condemned cell he frequently repeated, quote, I go to my death a murdered man, end quote. +DUMMY/LJ002-0145.wav|Often a commissioner had to leave the bench because he was himself a party to the suit that was sub judice. +DUMMY/LJ019-0112.wav|Such excellent returns might be counted upon, that a margin of profit would be left after the cost of the prisons had been defrayed. +DUMMY/LJ009-0113.wav|whistles merrily, and points upwards with madness in his look. +DUMMY/LJ015-0276.wav|Cheques blank and canceled were his stock-in-trade. The former he filled up by exact imitation of the latter, signature and all. +DUMMY/LJ011-0275.wav|There he was seated on some sort of wooden bench and securely fastened. +DUMMY/LJ044-0199.wav|In retrospect his attempt to go to Cuba or return to the Soviet Union may well have been Oswald's last escape hatch, +DUMMY/LJ015-0153.wav|Not content with giving where assistance was solicited, he himself sought out deserving cases and personally afforded relief. +DUMMY/LJ012-0228.wav|The evidence from first to last was circumstantial, +DUMMY/LJ040-0171.wav|where he did as he wanted and he didn't have to live by any rules or come into contact with people, end quote. +DUMMY/LJ015-0245.wav|The two remaining safes were attacked and nearly entirely despoiled in the same way as the first, and the contents transferred to the courier bags. +DUMMY/LJ021-0171.wav|Now that these people are coming out of their storm cellars, they forget that there ever was a storm. +DUMMY/LJ024-0128.wav|The present attempt by those opposed to progress to play upon the fears of danger to personal liberty +DUMMY/LJ045-0076.wav|The letter fell into Oswald's hands when it was returned to his post office box +DUMMY/LJ041-0028.wav|Sometime during this period, and under circumstances to be discussed more fully below, +DUMMY/LJ018-0055.wav|He is described as not a bad-looking man, with a square German type of face, +DUMMY/LJ028-0065.wav|Eleven years later, in five eighty-six, he destroyed the sacred Hebrew city, +DUMMY/LJ015-0049.wav|They admitted that they had made away with many of the securities entrusted to their keeping. +DUMMY/LJ002-0141.wav|and indeed supplied the bulk of the poor debtors committed to prison. +DUMMY/LJ002-0302.wav|and so were unable either to warm themselves or to cook their food. +DUMMY/LJ033-0075.wav|The Long and Bulky Package +DUMMY/LJ036-0012.wav|(four) evidence establishing the ownership of a zipper jacket +DUMMY/LJ019-0032.wav|The movement thus laudably initiated by the Government soon spread to the provinces. +DUMMY/LJ035-0058.wav|Baker turned to Truly and said, quote, Do you know this man, does he work here? end quote. +DUMMY/LJ028-0200.wav|High up, where all might see it, the armless hand wrote the King's fate. +DUMMY/LJ041-0150.wav|I'm a Marxist. I became interested about the age of fifteen. From an ideological viewpoint. +DUMMY/LJ008-0050.wav|A broadsheet dated April twenty-fourth, seventeen eighty-seven, describing an execution on the newly-invented scaffold before the debtors' door, +DUMMY/LJ045-0136.wav|as Oswald went on to say. In Oswald's imagination, quote, +DUMMY/LJ030-0060.wav|At the rear on each side of the automobile were small running boards, each designed to hold a Secret Service agent, +DUMMY/LJ042-0054.wav|Oswald told his brother, quote, on what terms I want this arrangement, end quote. He advised Robert that: one. +DUMMY/LJ046-0196.wav|were filed on a geographic basis, and could conveniently be reviewed by a Secret Service agent preparing for a Presidential trip +DUMMY/LJ021-0168.wav|I do not want to think that it is the destiny of any American to remain permanently on relief rolls. +DUMMY/LJ015-0148.wav|The choicest wines, the finest fruits, +DUMMY/LJ010-0095.wav|Meanwhile the main body remained at their headquarters, a ruined stable in Cato Street, +DUMMY/LJ040-0178.wav|A psychological human figure-drawing test corroborated the interviewer's findings that Lee was insecure and had limited social contacts. +DUMMY/LJ019-0235.wav|With the last re-edification of Newgate, a work executed some seven centuries after the first stone of the old jail was laid, +DUMMY/LJ047-0245.wav|His secretary testified that she prepared such a report for him that afternoon and Chief of Police Jesse E. Curry +DUMMY/LJ010-0006.wav|Certain crimes, those against the person especially, diminished gradually. They became less easy or remunerative. +DUMMY/LJ012-0125.wav|Not the slightest clue to these large affairs was ever obtained, but it is probable that they were "put up" jobs, or managed with the complicity of servants. +DUMMY/LJ028-0353.wav|Darius, as the story goes, would often say that "he had rather Zopyrus were unmaimed, than be master of twenty more Babylons." +DUMMY/LJ004-0247.wav|thus disposing of the objection so long raised against the industrial employment of prisoners in Newgate. +DUMMY/LJ021-0206.wav|whatever they need to have done but cannot do at all or cannot do so well for themselves in their separate and individual capacities." +DUMMY/LJ012-0107.wav|About this time also suspicion fell upon Huey, one of the clerks, who was arrested soon afterwards, and made a clean breast of the whole affair. +DUMMY/LJ050-0266.wav|The exchange of letters dated August thirty-one, nineteen sixty-four, +DUMMY/LJ005-0038.wav|to suggest beneficial regulations, and circulate tracts demonstrating the advantages of classification, +DUMMY/LJ031-0172.wav|While the team of doctors at Parkland Hospital tried desperately to save the life of President Kennedy, +DUMMY/LJ037-0205.wav|The invoice was prepared on March thirteen, nineteen sixty-three; the revolver was actually shipped on March twenty by Railway Express. +DUMMY/LJ002-0088.wav|In each cell there was a barrack bedstead on the floor without bedding. +DUMMY/LJ047-0114.wav|and that they had been married in Fort Worth and lived there until coming to New Orleans. He had told the New Orleans arresting officers that he had been born in Cuba. +DUMMY/LJ035-0142.wav|then her estimate of the time when she descended from the fourth floor is incorrect, +DUMMY/LJ013-0192.wav|but still he found friends, and a liberal subscription was raised among the foreign servants in London to provide funds for his defense. +DUMMY/LJ017-0182.wav|that it was a very dangerous medicine, and she profited by what she had heard. +DUMMY/LJ025-0048.wav|Respiration -- that is, the absorption of oxygen and the exhalation of carbonic acid +DUMMY/LJ015-0285.wav|and when these presented themselves, entrusted them as a beginning with the duty of cashing cheques. +DUMMY/LJ016-0364.wav|all the ghastly paraphernalia, the gallows itself and the process of erecting and removing it, rested with the city architect, and not with the prison officials. +DUMMY/LJ013-0157.wav|Lord William Russell lived alone in Norfolk Street, Park Lane. He was a widower, and seventy-three years of age. +DUMMY/LJ039-0001.wav|For more information, or to volunteer, please visit librivox dot org. +DUMMY/LJ015-0017.wav|for the fraudulent disposal of securities lodged in their hands. This firm was one of the oldest banking establishments in the kingdom, +DUMMY/LJ031-0225.wav|From this head wound two small irregularly shaped fragments of metal were recovered and turned over to the FBI. +DUMMY/LJ021-0186.wav|than in the United States? +DUMMY/LJ016-0074.wav|The success, although very short-lived, which attended him, no doubt inspired other inmates of Newgate to follow his example. +DUMMY/LJ025-0064.wav|which, from their similarity to animals of simple organization, were called "zoospores." +DUMMY/LJ049-0047.wav|reveals that Hill first placed his hand on the Presidential car at frame three forty-three, thirty frames +DUMMY/LJ023-0109.wav|The Court in addition to the proper use of its judicial functions has improperly set itself up as a third house of the Congress +DUMMY/LJ035-0078.wav|Then Howlett walked down the stairway to the second-floor landing and entered the lunchroom. +DUMMY/LJ046-0210.wav|to relate principally to overt threats to harm the President or other specific manifestations of hostility. +DUMMY/LJ032-0230.wav|that the published pictures were the same as the original except for retouching done by these publications, apparently for the purpose of clarifying the lines of the rifle +DUMMY/LJ026-0122.wav|Catabolism is probably due to enzyme action, but the final result is chiefly carbon dioxide and water, +DUMMY/LJ003-0205.wav|Those who were thus reckless reacted upon the penitent who knew their days were numbered, +DUMMY/LJ035-0035.wav|Because the stairway itself is enclosed, neither Baker nor Truly could see anything on the second-floor hallway until they reached the landing at the top of the stairs. +DUMMY/LJ025-0003.wav|The facts dealing with the physiology of organisms, the activities associated with that which we call life +DUMMY/LJ006-0273.wav|enough to altogether upset what small show of decorum and discipline was still preserved in the prison. +DUMMY/LJ037-0120.wav|Guinyard said, quote, I told them that was him right there. I pointed him out right there, end quote. +DUMMY/LJ019-0136.wav|This was obviated by the frequent sessions of the Central Criminal Court, and the utilization of the two subsidiary prisons in Giltspur Street and Southwark. +DUMMY/LJ003-0161.wav|The luxury of the state side was for a long time open to all who could pay +DUMMY/LJ020-0057.wav|Cover with a cloth and set nearer the fire than you dared trust the dough, and let them rise for an hour. +DUMMY/LJ038-0256.wav|In the room beyond the one in which General Walker was sitting on the night of the shooting the Dallas police recovered a badly mutilated bullet +DUMMY/LJ049-0179.wav|The Commission believes that both the FBI and the Secret Service have too narrowly construed their respective responsibilities. +DUMMY/LJ018-0116.wav|and that the forged cheques which had been presented, but refused, amounted to double the sum. +DUMMY/LJ011-0131.wav|Upon the third reading of this bill Sir James Macintosh moved as an amendment that capital punishment should be abolished for all crimes of forgery, +DUMMY/LJ045-0205.wav|After he returned from his trip to Mexico where his application to go to Cuba had been sharply rejected, +DUMMY/LJ035-0185.wav|After inspecting this floor, Sawyer returned to the street about three minutes after he entered the building. +DUMMY/LJ037-0097.wav|Smith subsequently told a Commission staff member +DUMMY/LJ050-0265.wav|with the Office of Science and Technology and the other Federal agencies that have been so helpful to the Secret Service be placed on a permanent and formal basis. +DUMMY/LJ008-0317.wav|The fatal news was not always received in the same way. +DUMMY/LJ005-0267.wav|The use of tobacco, hitherto pretty generally indulged in both by men and women, +DUMMY/LJ016-0327.wav|declared that they were not prepared to agree to the resolution respecting private executions. +DUMMY/LJ028-0381.wav|Wild beasts of the desert shall lie there; +DUMMY/LJ036-0171.wav|he would have arrived there about twelve:fifty-nine to one p.m. +DUMMY/LJ004-0213.wav|Compared with those highly meritorious institutions Newgate still showed but badly. +DUMMY/LJ005-0129.wav|"a measure which must greatly tend to discourage attempts to escape." +DUMMY/LJ003-0055.wav|or class (one), with whom, under the imperious demand for accommodation, were also associated the misdemeanants, or class (two). +DUMMY/LJ018-0148.wav|A watch was set on Burnett, who was followed to the shop of one Buncher, a butcher in Strutton Ground. +DUMMY/LJ037-0161.wav|and impressed upon the lead of the bullets inconsistent individual characteristics which made identification impossible. +DUMMY/LJ028-0034.wav|Old King Sargon I., who may have lived as early as three thousand eight hundred B.C., +DUMMY/LJ038-0150.wav|Oswald told him that he ate lunch in the first-floor lunchroom and then went to the second floor for a Coke which he brought downstairs. +DUMMY/LJ050-0142.wav|directed its field representatives to send a form request for intelligence information to all local, +DUMMY/LJ031-0065.wav|As a result of the infusion of liquids through the cutdowns, the cardiac massage, and the airway, +DUMMY/LJ027-0041.wav|both adaptive swimming organs, yet the resemblance in these cases does not go deeper than the surface -- it is one of function only. +DUMMY/LJ011-0084.wav|After three years' confinement in the latter prison he passed himself off as his brother, Colonel Montgomery, +DUMMY/LJ035-0065.wav|Truly also noted at this time that Oswald's hands were empty. +DUMMY/LJ042-0160.wav|the Philippines, and the scores of odd islands in the Pacific. +DUMMY/LJ022-0142.wav|The National Industrial Recovery Act expires on the sixteenth of June. +DUMMY/LJ041-0123.wav|In that connection, it does not appear that Oswald ever expressed any dissatisfaction of any kind with either the President or Governor Connally. +DUMMY/LJ042-0030.wav|it would have taken considerable discipline to save whatever amount was required to finance his defection out of the salary of a low ranking enlisted man. +DUMMY/LJ032-0037.wav|The canceled money order was obtained from the Post Office Department. +DUMMY/LJ030-0156.wav|Based on these films, the speed of the President's automobile is computed at an average speed of eleven point two miles per hour. +DUMMY/LJ012-0074.wav|Whether Ikey was "assigned" to his own family is not recorded, but no doubt he succeeded to his own property when the term of servitude had expired. +DUMMY/LJ008-0023.wav|two sailors, Charles Palm and Sam Tilling, were hanged at the same place for the murder of their captain, James Keith +DUMMY/LJ003-0280.wav|limits of space, the interference of the ill-disposed, who would neither work nor let others do so, +DUMMY/LJ033-0035.wav|and Marina Oswald testified that Oswald did not say anything about curtain rods on the day before the assassination. +DUMMY/LJ025-0155.wav|A bean may be supplied with water in which salts of ammonia and certain other mineral salts are dissolved in due proportion, +DUMMY/LJ015-0274.wav|His method was comprehensive and deeply laid. +DUMMY/LJ011-0090.wav|But next morning he was found dead in his cell. +DUMMY/LJ031-0226.wav|The autopsy also disclosed a wound near the base of the back of President Kennedy's neck slightly to the right of his spine. +DUMMY/LJ032-0161.wav|This shirt was also composed of dark blue, gray- black and orange-yellow cotton fibers. +DUMMY/LJ023-0075.wav|that it was an extraordinary power to exercise and through Mr. Justice Washington laid down this limitation upon it: +DUMMY/LJ038-0263.wav|as to whether or not the bullet recovered from Walker's house had been fired from the rifle found on the sixth floor of the Texas School Book Depository Building. +DUMMY/LJ049-0076.wav|The Commission's review of the provisions for Presidential protection at the time of President Kennedy's trip to Dallas demonstrates the need for substantial improvements. +DUMMY/LJ005-0296.wav|The Lords' Committee on Jails were of the same opinion, and considered the prisons under corporate or peculiar jurisdiction in a very unsatisfactory condition. +DUMMY/LJ015-0117.wav|Sergeant Ballantine, who prosecuted, +DUMMY/LJ047-0201.wav|Robert I Bouck, special agent in charge of the Protective Research Section, +DUMMY/LJ046-0183.wav|and who are in penal or hospital custody are listed only in the general files of PRS, +DUMMY/LJ038-0240.wav|The third photograph identified by Marina Oswald depicts the entrance to General Walker's driveway from a back alley. +DUMMY/LJ008-0133.wav|He was dressed in a mixed-colored loose coat with a black collar, swandown waistcoat, blue pantaloons, and white silk stockings. +DUMMY/LJ008-0057.wav|parallel to the prison wall, and about four feet from it, but strongly affixed thereto with iron scroll clamps. +DUMMY/LJ019-0366.wav|It was necessary, for instance, to give due notice when the government grant was to be withdrawn, +DUMMY/LJ005-0223.wav|There was a diminution in the numbers in custody, due to the adoption of the practice of not committing at once to Newgate every offender for trial at the Old Bailey +DUMMY/LJ006-0246.wav|On another occasion a young man, who was being violently teased, seized a knife and stabbed his tormentor in the back. +DUMMY/LJ048-0024.wav|He had come back to Dallas. Hosty had established that he had a job, he was working, +DUMMY/LJ008-0139.wav|without waiting for any signal, the platform dropped, and he was launched into eternity. +DUMMY/LJ011-0013.wav|replaced the stock in the names of the original holders, who might otherwise have been completely ruined. +DUMMY/LJ011-0155.wav|He had eloped with his first wife from school. +DUMMY/LJ001-0142.wav|modern printers generally overdo the "whites" in the spacing, a defect probably forced on them by the characterless quality of the letters. +DUMMY/LJ026-0102.wav|but root pressure due to osmosis, capillary action and evaporation from the leaves are factors. +DUMMY/LJ008-0212.wav|Cries of "Comfortable room!" "Excellent situation!" "Beautiful prospect!" "Splendid view!" resounded on every side. +DUMMY/LJ033-0036.wav|No curtain rods were known to have been discovered in the Depository Building after the assassination. +DUMMY/LJ045-0166.wav|He repeated this not once but several times, but I refused. And he said that once again I was preferring my friends to him, and that I didn't need him. +DUMMY/LJ026-0111.wav|In the cells the foods undergo metabolic changes. +DUMMY/LJ001-0140.wav|the desirable thing being "the breaking of the line" as in bonding masonry or brickwork +DUMMY/LJ019-0290.wav|There were dormitories without light, control, or regulation at night, +DUMMY/LJ007-0162.wav|uncontrolled conversation with visitors of the very worst description -- prostitutes, thieves, receivers of stolen goods +DUMMY/LJ040-0085.wav|John Pic, however, did not think her position was worse than that of many other people. +DUMMY/LJ029-0195.wav|when Governor Connally confirmed on November eight that the President would come to Texas on November twenty-one and twenty-two, +DUMMY/LJ006-0226.wav|he was "toed," that is to say, a string was fastened to his big toe while he was asleep, and he was dragged from off his mat, +DUMMY/LJ035-0118.wav|They rushed to the west windows after the shots were fired and remained there +DUMMY/LJ029-0118.wav|It afforded us wide streets most of the way, because of the buses that were in the motorcade. +DUMMY/LJ021-0120.wav|I would not minimize the inevitable losses to employers and employees and to the general public through such conflicts. +DUMMY/LJ019-0384.wav|The Home Secretary in that administration, +DUMMY/LJ014-0154.wav|mocking at the chaplain, and turning a deaf ear to the counsels of a benevolent lady who came to visit. +DUMMY/LJ036-0118.wav|When he first testified before the Commission, +DUMMY/LJ010-0150.wav|and eventually lodged in Bethlehem Hospital, where she died after forty years' detention, at the advanced age of one hundred. +DUMMY/LJ036-0018.wav|where he boarded a bus which was heading back in the direction of the Depository Building, on its way to the Oak Cliff section of Dallas. +DUMMY/LJ037-0027.wav|Benavides stopped and waited in the truck until the gunman ran to the corner. +DUMMY/LJ045-0187.wav|There is, of course, no way to determine the degree to which he was committed to his plan at that time. +DUMMY/LJ040-0088.wav|which he did for four months in the fall of nineteen forty-eight. +DUMMY/LJ030-0039.wav|Men the motorcade slows or stops, agents take positions between the President and the crowd. +DUMMY/LJ012-0214.wav|Carlo Ferrari, who made a living by exhibiting white mice about the streets, +DUMMY/LJ022-0148.wav|to enforce minimum wages, to prevent excessive hours, +DUMMY/LJ010-0203.wav|were bound to provide themselves with sword, rifle, dagger, and a pair of pistols; +DUMMY/LJ032-0152.wav|Oswald's palmprint on the underside of the barrel demonstrates that he handled the rifle when it was disassembled. +DUMMY/LJ039-0080.wav|of the Marine Corps School at Quantico, Virginia. +DUMMY/LJ033-0176.wav|taken from the Texas School Book Depository shipping room on November twenty-two, nineteen sixty-three. +DUMMY/LJ038-0124.wav|Oswald became angry and said, quote, +DUMMY/LJ008-0318.wav|The men who were doomed often fell down in convulsions upon the floor. +DUMMY/LJ036-0004.wav|After leaving the Depository Building at approximately twelve:thirty-three p.m., Lee Harvey Oswald proceeded to his roominghouse by bus and taxi. +DUMMY/LJ042-0244.wav|Oswald subsequently did subscribe to several Soviet journals. +DUMMY/LJ044-0106.wav|and Benjamin J. Davis honorary membership cards in his nonexistent New Orleans chapter of the Fair Play for Cuba Committee, +DUMMY/LJ010-0163.wav|secure as it seemed in the affections of her loyal people, freely appeared in public. +DUMMY/LJ013-0264.wav|In his effects were found the clothes he had on at the time of his escape from the stables, and under the jacket he was wearing +DUMMY/LJ010-0045.wav|became the victim of the most cowardly and unmanly outrages, and the attempted murder of the sovereign by Oxford in eighteen forty +DUMMY/LJ040-0036.wav|and, apparently seriously, as a political prophet -- a man who said that after twenty years he would be prime minister. +DUMMY/LJ048-0214.wav|and also looking ahead to the known hazards like overpasses, under-passes, railroads, et cetera, end quote. +DUMMY/LJ041-0045.wav|he was nevertheless unable to convince the proper authorities that he was really seventeen years old. +DUMMY/LJ025-0085.wav|the definition so changed will cover all animals, except certain parasites, +DUMMY/LJ028-0508.wav|others have been taken to the Berlin Museum. +DUMMY/LJ047-0135.wav|According to the information received by the Bureau +DUMMY/LJ042-0189.wav|but one with union communes, +DUMMY/LJ039-0204.wav|Frazier later fired four groups of three shots at a distance of one hundred yards in five point nine, six point two, +DUMMY/LJ042-0113.wav|George De Mohrenschildt thought that Oswald must have become disgusted with life in the Soviet Union as the novelty +DUMMY/LJ040-0167.wav|which grows as one speaks to him, end quote. +DUMMY/LJ003-0108.wav|Spirits were freely introduced, and although he at first abstained, +DUMMY/LJ036-0062.wav|followed the same route as the Marsalis bus through downtown Dallas, except that it continued west on Elm, +DUMMY/LJ024-0094.wav|It would take months and years thereafter to get a two-thirds majority in favor of that amendment in both Houses of the Congress. +DUMMY/LJ008-0097.wav|No sooner was the "job" finished than half-a-dozen competitors appeared, each offering the identical rope for sale at a shilling an inch. +DUMMY/LJ045-0221.wav|It is unlikely that the motivation was that simple. +DUMMY/LJ013-0151.wav|Forrester, the detective who had pursued and captured Burgess at Boston, tracked Ker to France, and following him there, eventually captured him +DUMMY/LJ035-0022.wav|Baker, quote, had it in mind that the shots came from the top of this building here, end quote, so he ran straight to the entrance of the Depository Building. +DUMMY/LJ007-0167.wav|framing a terrible indictment against those responsible for the condition of Newgate. Their words deserve to be quoted in full. +DUMMY/LJ029-0201.wav|Meanwhile, on November seventeen +DUMMY/LJ034-0068.wav|Additional testimony linking Oswald with the point from which the shots were fired +DUMMY/LJ015-0126.wav|This offered him great facilities for the creation of artificial stock, its sale from a fictitious holder, and transfer to himself. +DUMMY/LJ012-0103.wav|which if introduced into a keyhole, and worked like a bit and brace, will soon destroy the strongest lock. +DUMMY/LJ001-0078.wav|to improve the letter in form. +DUMMY/LJ037-0178.wav|or one used Remington-Peters cartridge case, which may have been in the revolver before the shooting, +DUMMY/LJ028-0385.wav|It had become, as Dio Cassius said, "Mounds and legends and ruins." +DUMMY/LJ010-0251.wav|Francis was sentenced to be hanged, decapitated, and quartered, +DUMMY/LJ049-0164.wav|In the appropriations of the FBI there has recurred annually an item for the, quote, protection of the person of the President of the United States, end quote, +DUMMY/LJ038-0299.wav|It would have been possible for him to take different routes in approaching and leaving the scene of the shooting. +DUMMY/LJ013-0014.wav|After this, on the faith of forced bills of lading, the captain, Loose by name, +DUMMY/LJ020-0020.wav|When the sponge is light sift a quart and a cup of flour into a bowl or tray with two teaspoonfuls of salt. +DUMMY/LJ046-0026.wav|since a disclosure of such detailed information relating to protective measures might undermine present methods of protecting the President. +DUMMY/LJ006-0058.wav|but the occupants of each were huddled together indiscriminately. The inspectors found in the same wards in the chapel yard the convicted and the untried, +DUMMY/LJ006-0135.wav|the victim of this cruel ill-usage having "more fear of the power of the wardsman to injure him, than confidence in the governor's power to protect him." +DUMMY/LJ046-0163.wav|During the period November nineteen sixty-one to November nineteen sixty-three, +DUMMY/LJ044-0005.wav|Oswald's political activities after his return to the United States center around his interest in Cuba and in the Fair Play for Cuba Committee. +DUMMY/LJ017-0061.wav|that he steadily refused to confess his guilt, lest it should militate against his chances. +DUMMY/LJ048-0039.wav|that the FBI took an unduly restrictive view of its responsibilities in preventive intelligence work, prior to the assassination. +DUMMY/LJ043-0162.wav|Marina Oswald's testimony indicates that her husband was not particularly concerned about his continued possession of the most incriminating sort of evidence. +DUMMY/LJ050-0237.wav|the FBI, on sixteen separate occasions, supplied a total of one hundred thirty-nine agents to assist in protection work during a Presidential visit, +DUMMY/LJ040-0217.wav|Lee had apparently been experiencing similar problems before going to New York, +DUMMY/LJ025-0156.wav|with atmospheric air containing its ordinary minute dose of carbonic acid and with nothing else but sunlight and heat. +DUMMY/LJ026-0050.wav|following in part the comparisons of certain animals and plants by Sedgwick and Wilson and others. +DUMMY/LJ041-0154.wav|His interest in Russia led some of his associates to call him "comrade" or "Oswaldskovitch." +DUMMY/LJ046-0085.wav|The degree of security that can be afforded the President of the United States +DUMMY/LJ001-0049.wav|After his death in the "fourteen eighties," or at least by fourteen ninety, printing in Venice had declined very much; +DUMMY/LJ032-0083.wav|Experts on questioned documents from the Treasury Department and the FBI testified that the Hidell cards were counterfeit photographic reproductions +DUMMY/LJ036-0105.wav|He said, quote, +DUMMY/LJ013-0053.wav|Miss Bailey, the doctor's sister, in whose favor the note was said to have been given, +DUMMY/LJ034-0219.wav|Standing alongside him were Buell Wesley Frazier and William Shelley, who also identified Lovelady. +DUMMY/LJ045-0004.wav|It has been suggested that one of the motivating influences operating on Lee Oswald was the atmosphere in the city of Dallas, +DUMMY/LJ015-0192.wav|From the moment of his reception he gave himself great airs, as a martyr and a man heavily wronged. +DUMMY/LJ037-0090.wav|she testified, quote, I would say that was him for sure, end quote. +DUMMY/LJ047-0222.wav|He had made the same statement about a week prior at another special conference which we had held. +DUMMY/LJ039-0150.wav|which would indicate that Oswald practiced operating the bolt. +DUMMY/LJ036-0013.wav|found along the path of flight taken by the gunman from the scene of the shooting to the place of arrest. +DUMMY/LJ045-0239.wav|which has been set forth above, only served to prolong the period during which he was the center of the attention of the entire world. +DUMMY/LJ014-0241.wav|Walter Watts was an inventor, a creator, who struck an entirely new and original line of crime. +DUMMY/LJ045-0108.wav|Ruth Paine testified that she heard Marina Oswald tell Oswald about the birthday party. +DUMMY/LJ004-0215.wav|mainly through the exertions of a new governor, Mr. Brown, who succeeded Mr. Newman at Newgate in eighteen seventeen. +DUMMY/LJ012-0146.wav|In the course of the trial it came out that almost every one concerned except the Caspars had endeavored to defraud his accomplices. +DUMMY/LJ016-0262.wav|It was a callous, careless crowd of coarse-minded, semi-brutalized folk, who came to enjoy themselves. +DUMMY/LJ015-0004.wav|The goods having no existence, Cole of course could not deliver them. +DUMMY/LJ049-0125.wav|that any suspects who are arrested will be Federal prisoners, subject to Federal protection from vigilante justice and other threats. +DUMMY/LJ004-0196.wav|There was an infirmary, properly found and duly looked after. +DUMMY/LJ008-0151.wav|but in the beginning of eighteen oh seven one of them, at that time just sentenced to transportation, +DUMMY/LJ003-0093.wav|the oldest only twelve or thirteen, exposed to all the contaminating influences of the place. +DUMMY/LJ012-0153.wav|Moses, it may be added, was a direct descendant of Ikey Solomons. +DUMMY/LJ046-0222.wav|The referral of the copy to local Secret Service should not delay the immediate referral of the information by the fastest available means of communication +DUMMY/LJ034-0174.wav|appeared uncomfortable for one, and secondly, he wasn't watching. he didn't look like he was watching for the parade. +DUMMY/LJ018-0294.wav|all of which were discounted to the value of one hundred two thousand, two hundred seventeen pounds. +DUMMY/LJ011-0281.wav|Then, as they warned him that he would be kept a prisoner in total darkness in this horrible den until he agreed to their demands, +DUMMY/LJ031-0171.wav|Removal of the President's Body +DUMMY/LJ015-0139.wav|an open-handed, unthinking charity which gave freely to the poor and needy the money which belonged to his creditors. +DUMMY/LJ008-0077.wav|where the apparatus for the punishment she was about to experience +DUMMY/LJ011-0229.wav|after which he went to New Zealand, and held an important post in that colony. +DUMMY/LJ038-0260.wav|On November thirty, nineteen sixty-three, the FBI requested the bullet for ballistics examination; +DUMMY/LJ015-0164.wav|Redpath passed and lifted his hat to his chairman; the latter acknowledged the salute. +DUMMY/LJ016-0018.wav|On the very morning on which he was to suffer he eluded the vigilance, such as it was, of his officers +DUMMY/LJ044-0095.wav|In spite of the fact that Oswald had been surprised and was on the defensive throughout the debate, according to Stuckey, quote, +DUMMY/LJ017-0049.wav|with morbid curiosity to scan his features and watch his demeanor through the shifting, nicely-balanced phases of his protracted trial. +DUMMY/LJ002-0139.wav|under forty shillings arising within the city. +DUMMY/LJ005-0173.wav|I shall not hesitate to ask Parliament for powers to compel them to make the necessary alterations, for it is not to be endured that these local jurisdictions should remain +DUMMY/LJ014-0022.wav|The letter found in the pocket of the deceased was sealed with a wafer marked F, +DUMMY/LJ021-0197.wav|of referring without rhyme or reason to the Constitution as a means of preventing its accomplishment, thus creating the general impression +DUMMY/LJ024-0064.wav|President Hoover, three, including a Chief Justice. +DUMMY/LJ008-0285.wav|on the one hand the gallows, on the other a short imprisonment. +DUMMY/LJ003-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section five: Newgate down to eighteen eighteen, part two. +DUMMY/LJ049-0157.wav|and special details protected him in Washington, on trips, and at special functions. +DUMMY/LJ013-0143.wav|These packages were consigned to Messrs. Shroeder of London; and as it was known that they were to arrive in Ker's ship, +DUMMY/LJ048-0238.wav|one agent was there from two until five a.m. +DUMMY/LJ018-0046.wav|Müller protested after sentence of death had been passed upon him that he had been convicted on a false statement of facts. +DUMMY/LJ017-0136.wav|with exquisite torture, and with a poison that would leave, as he thought, no trace. +DUMMY/LJ019-0034.wav|Within half-a-dozen years no less than fifty-four new prisons were built on the Pentonville plan, others were in progress, +DUMMY/LJ017-0160.wav|as indicating a motive for keeping her alive rather than killing her. +DUMMY/LJ039-0185.wav|the highest level of firing performance which would have been required of the assassin and the C two seven six six rifle +DUMMY/LJ004-0230.wav|It was found that shackles might be safely dispensed with, even in the case of the most desperate characters. +DUMMY/LJ035-0156.wav|Mrs. Reid testified that when she saw Oswald, he was wearing a T-shirt and no jacket. +DUMMY/LJ027-0093.wav|will nearly always be compelled to add new parts not contemplated in the original machine. +DUMMY/LJ013-0069.wav|A second will, also forged, was propounded at Doctors Commons as that of a Mrs. Hunt of Bristol. +DUMMY/LJ003-0072.wav|that some other and less mixed prison should be used for the confinement of persons convicted of libels. But this suggestion was ignored. +DUMMY/LJ030-0158.wav|While the car traveled this distance, the Zapruder camera ran one hundred fifty-two frames. +DUMMY/LJ019-0037.wav|Liverpool was building a new prison with a thousand cells, the county of Surrey one with seven hundred. +DUMMY/LJ031-0198.wav|Upon arrival, President Johnson made a brief statement over television and radio. +DUMMY/LJ004-0098.wav|it is enacted that the justices shall order the walls of every room to be scraped and white-washed once every year." +DUMMY/LJ040-0050.wav|His attachment to Marxist and Communist doctrine was probably, in some measure, an expression of his hostility to his environment. +DUMMY/LJ002-0188.wav|being often obliged to pay a high rent for a bed at the coffee-house or in some room which was vacated by its regular occupant. +DUMMY/LJ008-0263.wav|Forty-eight hours was the limit of time allowed to the unhappy man to make his peace, and during that time he was still kept on a bare allowance of bread and water. +DUMMY/LJ034-0164.wav|Brennan saw a man in the window who closely resembled Lee Harvey Oswald, and that Brennan believes the man he saw was in fact +DUMMY/LJ001-0020.wav|the "lower-case" being in fact invented in the early Middle Ages. +DUMMY/LJ006-0106.wav|Too much was left to the wardsman. It was he who could issue small luxuries; +DUMMY/LJ016-0445.wav|Miller, the Chelsea murderer, who packed his victim's body in a box, and tried to send it by parcels delivery, tried to kill himself, +DUMMY/LJ031-0107.wav|Governor Connally had a large sucking wound in the front of the right chest which caused extreme pain and difficulty in breathing. +DUMMY/LJ023-0020.wav|We then began a program of remedying those abuses and inequalities -- to give balance and stability to our economic system +DUMMY/LJ049-0217.wav|Moreover, a number of imponderable questions have to be weighed if any change in the intimate association now established +DUMMY/LJ050-0050.wav|As a result of these studies, the planning document submitted by the Secretary of the Treasury to the Bureau of the Budget on August thirty-one, +DUMMY/LJ044-0186.wav|he would not give him a visa, end quote, and that, quote, +DUMMY/LJ032-0226.wav|The Commission sought to determine whether these photographs were touched prior to publication. +DUMMY/LJ011-0248.wav|Howard, strange to say, making no attempt to detain him; probably because Mullay promised to return a few days later, and to bring more money. +DUMMY/LJ006-0305.wav|The house at this time was full of men and visitors; waiters came in from the taverns with meals. +DUMMY/LJ028-0019.wav|The stars come out one by one and shine brighter than elsewhere as if to light you on your way. +DUMMY/LJ003-0269.wav|and offered him half the booty if he would give full information as to the transactions and correspondence of his late employers. +DUMMY/LJ006-0110.wav|He had power where his exactions were resisted of making the ward most uncomfortable for the defaulter. +DUMMY/LJ016-0199.wav|It was customary to make the executioner take the Bible in his hand, and swear solemnly that he would dispatch every criminal condemned to die, +DUMMY/LJ030-0240.wav|to that effect immediately after the assassination. +DUMMY/LJ013-0118.wav|The thief walked into the house in Belgrave Square, and openly carried off the plate-chest, deposited it in a light cart at the door, and drove away. +DUMMY/LJ013-0173.wav|The researches of the police soon laid bare other suspicious facts. +DUMMY/LJ038-0113.wav|when they lived on Neely Street. +DUMMY/LJ002-0269.wav|he did nothing beyond visiting the prison occasionally, and left the administration to the deputy marshal. +DUMMY/LJ006-0278.wav|A female prisoner kept the registers. +DUMMY/LJ026-0064.wav|but unlike that of the animal, it is not chiefly an income of foods, but only of the raw materials of food. +DUMMY/LJ008-0066.wav|This mode of execution, it is alleged, gave rise to the old vulgar "chaff," "Take care, or you'll die at the fall of the leaf." +DUMMY/LJ003-0058.wav|but all mingled freely during the day in the yard. +DUMMY/LJ042-0047.wav|For personal, material advantages? Happiness is not based on oneself, it does not consist of a small home, of taking and getting, +DUMMY/LJ039-0236.wav|thereby leaving a time span of four point eight to five point six seconds between the two shots which struck their mark. +DUMMY/LJ022-0126.wav|that the exceptional instances of wrong-doing in government are probably less numerous than in almost every other line of endeavor. +DUMMY/LJ024-0027.wav|commonly fix the retirement age at seventy years or less. +DUMMY/LJ003-0073.wav|Indeed the partial classification attempted seems to have been abandoned within a year or two. +DUMMY/LJ032-0019.wav|Shortly after the Mannlicher-Carcano rifle was found on the sixth floor of the Texas School Book Depository Building, agents of the FBI +DUMMY/LJ011-0183.wav|Passing through Carlisle, she was told that Mr. Turner was in the town, but could not show himself. +DUMMY/LJ032-0078.wav|a Selective Service registration certificate, and a certificate of service in the U.S. Marine Corps, all three cards being in his own name. +DUMMY/LJ022-0196.wav|It is the recovery of confidence in our democratic processes and institutions. +DUMMY/LJ031-0011.wav|A moment later Curry added, quote, Looks like the President has been hit. Have Parkland stand by, end quote, +DUMMY/LJ007-0217.wav|They go on to say +DUMMY/LJ030-0184.wav|Kellerman grabbed his microphone and radioed ahead to the lead car, +DUMMY/LJ016-0187.wav|The other accepted, and they got into conversation. +DUMMY/LJ028-0318.wav|Darius had maltreated him in the way they could see, only because he had given advice that the siege should be raised, since there seemed no hope of taking the city. +DUMMY/LJ014-0249.wav|The cheques drawn by the directors also passed through his hands; to him too they came back to be verified and put by, +DUMMY/LJ006-0284.wav|It was a special evil of this part of the prison, that the devotional exercises, originally so profitable, had grown into a kind of edifying spectacle, +DUMMY/LJ007-0077.wav|One of these were the criminal lunatics, who were at this time and for long previous continuously imprisoned there. +DUMMY/LJ008-0296.wav|The time of the arrival of this report was generally known at Newgate, +DUMMY/LJ014-0255.wav|continually altering, erasing, and adding figures to correspond with and cover the abstractions he made of various cheques as they were drawn. +DUMMY/LJ026-0029.wav|so in the case under discussion. +DUMMY/LJ009-0128.wav|I have been present at the scene not less than twenty times, and have invariably observed +DUMMY/LJ017-0100.wav|Next night Cook had a second and a more violent attack. +DUMMY/LJ038-0303.wav|the Commission has concluded that Lee Harvey Oswald attempted to take the life of Maj. Gen. Edwin A. Walker (Resigned, U.S. Army) +DUMMY/LJ026-0012.wav|Haeckel proposed once to include all one-celled animals and plants in a third kingdom to be called Protista (meaning the first of all life). +DUMMY/LJ042-0050.wav|I have been a pro-communist for years and yet I have never met a communist, instead I kept silent and observed, +DUMMY/LJ009-0072.wav|The sheep-stealer smiles, and, extending his arms upwards, looks with a glad expression to the roof of the chapel. +DUMMY/LJ049-0195.wav|leaving with the Secret Service only the responsibility for the physical protection of the President, that is, the guarding function alone. +DUMMY/LJ019-0116.wav|anticipating the best results from a system which made earnings, and indeed release, dependent upon the amount of work done. +DUMMY/LJ048-0050.wav|This conclusion may be tinged with hindsight, but +DUMMY/LJ027-0008.wav|The great variety of life as exhibited in the countless species of plants and animals has been referred to, and yet, great as this variety is, +DUMMY/LJ046-0195.wav|The cases subject to periodic review and the one hundred or so cases in the higher risk category +DUMMY/LJ011-0112.wav|Mr. Sparks Moline, came to attend him to the scaffold. +DUMMY/LJ032-0176.wav|Moreover, the bus transfer which he obtained as he left the bus was still in the pocket when he was arrested. +DUMMY/LJ004-0035.wav|The borough jails, those in fact under corporate management, were not included in the new measures; +DUMMY/LJ010-0113.wav|he had neither a bed to lie upon nor a coat to his back. +DUMMY/LJ048-0206.wav|As the motorcade approached Elm Street +DUMMY/LJ025-0030.wav|As the animal body required to be independent of heat and of the atmosphere, +DUMMY/LJ044-0193.wav|and he didn't have any great desire to do so any more because he had run into, as he himself said -- into bureaucracy and red tape, end quote. +DUMMY/LJ050-0092.wav|Chief Rowley believed that they would, +DUMMY/LJ011-0197.wav|and presently ascertained that Wakefield had gone to the Continent with his involuntary bride. +DUMMY/LJ002-0028.wav|In order to realize the evils entailed by incarceration in Newgate in these days, it is necessary to give some account of its interior +DUMMY/LJ021-0130.wav|who denies freedom of organization to his employees, or fails to make every reasonable effort at a peaceful solution of their differences, +DUMMY/LJ043-0005.wav|Apart from his relatives, Oswald had no friends or close associates in Texas when he returned there in June of nineteen sixty-two, +DUMMY/LJ009-0297.wav|There is still extant a curious petition presented to the Treasury by Ralph Griffith, Esq., high sheriff of Flintshire, +DUMMY/LJ046-0088.wav|to operate in a sort of vacuum, isolated from the general public and behind impregnable barriers. +DUMMY/LJ013-0073.wav|as its signature was dated eighteen twenty-nine, whereas Mrs. Hunt actually died in eighteen oh six. +DUMMY/LJ026-0059.wav|For this purpose the food is taken not into the body proper, but into a kind of tubular chemical laboratory; called the alimentary canal, +DUMMY/LJ036-0084.wav|Craig also claimed that when Fritz pointed out to Oswald that Craig had identified him, +DUMMY/LJ008-0308.wav|The Recorder's report generally reached Newgate late at night. +DUMMY/LJ036-0082.wav|Craig testified that later in the afternoon he saw Oswald in the police interrogation room +DUMMY/LJ031-0190.wav|Mrs. Kennedy and Mrs. Johnson stood at the side of the new President as he took the oath of office. +DUMMY/LJ041-0046.wav|There is evidence that Oswald was greatly influenced in his decision to join the Marines by the fact that his brother Robert had done so +DUMMY/LJ036-0019.wav|When Oswald was apprehended, a bus transfer marked for the Lakewood-Marsalis route was found in his shirt pocket. +DUMMY/LJ039-0032.wav|was on November twenty to twenty-one, nineteen sixty-three. +DUMMY/LJ029-0070.wav|is discussed in chapter eight. +DUMMY/LJ045-0103.wav|Mrs. Paine was planning a birthday party for one of her children on that weekend and her husband, Michael, was to be at the house. +DUMMY/LJ027-0104.wav|which are never destined to cut the gums; and throughout its life this animal retains, in a similarly rudimentary condition, +DUMMY/LJ038-0114.wav|Her testimony was fully supported by a photography expert who testified that in his opinion the pictures were not composites. +DUMMY/LJ036-0015.wav|According to the reconstruction of time and events which the Commission found most credible, +DUMMY/LJ014-0160.wav|elucidated all dark and uncertain points in connection with the crime. +DUMMY/LJ002-0067.wav|but really kept for the few who had funds sufficient to gain them admission to these more comfortable quarters. +DUMMY/LJ048-0149.wav|is essential. This would help to eliminate varying interpretations of Secret Service instructions by different local law enforcement representatives. For example, +DUMMY/LJ024-0093.wav|It would take months or years to get substantial agreement upon the type and language of the amendment. +DUMMY/LJ011-0138.wav|It passed the Commons, but opposition was again encountered in the Lords. +DUMMY/LJ013-0210.wav|Finally, on coming downstairs after bed-time, Lord William had found Courvoisier in the dining-room. +DUMMY/LJ009-0007.wav|Once ordered to the cells, friends of all classes suddenly rise up; his fellow-prisoners, the turnkeys, the chaplain, +DUMMY/LJ016-0434.wav|When they brought in the two watchers to relieve guard one night, Sattler said, +DUMMY/LJ001-0017.wav|that the forms of printed letters should follow more or less closely those of the written character, and they followed them very closely. +DUMMY/LJ037-0114.wav|He and Scoggins attempted to chase down the gunman in Scoggin's taxicab, but he had disappeared. +DUMMY/LJ045-0034.wav|Marina Oswald attributed that to their living apart and to the imminent birth of their second child. +DUMMY/LJ015-0122.wav|but who had all the appearance of a careless, thoughtless, happy-go-lucky fellow. +DUMMY/LJ032-0228.wav|Several of the publications furnished the Commission with the prints they had used, or described by correspondence the retouching they had done. +DUMMY/LJ050-0135.wav|According to Secretary Dillon, +DUMMY/LJ041-0134.wav|Governor Connally's connection with the discharge, although indirect, caused the Commission to consider whether he might have been Oswald's real target. +DUMMY/LJ027-0056.wav|The relation of skeleton and muscle in arthropods is exactly the reverse. +DUMMY/LJ023-0088.wav|and to approve or disapprove the public policy written into these laws. +DUMMY/LJ035-0048.wav|I can't say whether he had gone on through that door [the lunchroom door] or not. +DUMMY/LJ015-0079.wav|The temptation was all the greater because the chances of successful fraud lay ready to hand. +DUMMY/LJ005-0227.wav|It was enacted that the court of aldermen should make rules for the government of the prison, and that these should be posted publicly within the walls. +DUMMY/LJ048-0277.wav|He felt that each agent recognized the seriousness of the infraction and that there was no danger of a repetition. +DUMMY/LJ025-0051.wav|But in the fourth and fifth decades of this century +DUMMY/LJ027-0179.wav|Finally, only in the Tertiary, so far as we yet know, do the highest form (anoura) appear. +DUMMY/LJ034-0006.wav|He had ready access to the sixth floor, from the southeast corner window of which the shots were fired. +DUMMY/LJ039-0039.wav|there is no question that in this incident it was a question of Mr. Nixon, end quote. +DUMMY/LJ014-0334.wav|Cole found out where they were, and redeemed them at a heavy outlay, thus obtaining business relations with the firm that held them, +DUMMY/LJ025-0005.wav|The terms animal physiology, plant physiology and human physiology are in common use and often suggest to the lay reader +DUMMY/LJ013-0165.wav|The housemaid suggested that they should see if his lordship was all right, and the three went to his bedroom. +DUMMY/LJ015-0102.wav|He came back to explain that he had mislaid them. +DUMMY/LJ042-0101.wav|he envied his wife's uncle, a colonel in the MVD, because of the larger apartment in which he lived. +DUMMY/LJ012-0068.wav|For this she received a sentence of fourteen years' transportation, and was sent to Van Diemen's Land. +DUMMY/LJ042-0218.wav|to the entire land and complete foundations of his society, end quote. +DUMMY/LJ021-0141.wav|the benefits that all derive from the continuous, peaceful operation of our essential enterprises. +DUMMY/LJ015-0051.wav|the specific charge being the unlawful negotiation of Danish bonds and other shares belonging to the Rev. Dr. Griffiths of Rochester +DUMMY/LJ024-0100.wav|A very large percentage of newspaper publishers, Chambers of Commerce, +DUMMY/LJ001-0088.wav|were the leaders in this luckless change, though our own Baskerville, who was at work some years before them, went much on the same lines; +DUMMY/LJ048-0043.wav|nor was there any requirement to report the names of defectors. However, there was much material in the hands of the FBI about Oswald: +DUMMY/LJ013-0181.wav|and the police, feeling convinced that the crime had been committed by some inmate of the house, +DUMMY/LJ024-0082.wav|would deny us the necessary means of dealing with the present. +DUMMY/LJ002-0083.wav|The ground floor cells were nine feet by six; +DUMMY/LJ047-0216.wav|Agent Hosty testified that he was fully aware of the pending Presidential visit to Dallas. +DUMMY/LJ032-0168.wav|There is no way, however, to eliminate the possibility of the fibers having come from another identical shirt, end quote, +DUMMY/LJ039-0104.wav|firing fifty rounds each day for five days. +DUMMY/LJ038-0274.wav|superintendent of the Illinois Bureau of Criminal Identification and Investigation, conducted an independent examination of this bullet +DUMMY/LJ011-0181.wav|Wakefield's uncle took the matter more seriously, and declared that unless the marriage came off Mr. Turner must be sold up. +DUMMY/LJ031-0158.wav|As soon as one of the agents learned of the shooting, he asked the officer in charge of the police detail at the airport +DUMMY/LJ038-0218.wav|Moreover, the reference in paragraph three to paying, quote, the house rent on the second, end quote, +DUMMY/LJ014-0277.wav|An inquiry was at once set on foot, and the frauds were traced to Watts. +DUMMY/LJ046-0008.wav|on Theodore Roosevelt while campaigning in October of nineteen twelve; on President-elect Franklin Delano Roosevelt, +DUMMY/LJ044-0051.wav|except for the uneventful distribution of literature at the Dumaine Street wharf in June. +DUMMY/LJ017-0162.wav|Whatever the exact cause which impelled him to crime, it seems certain that he began to give her some poison, +DUMMY/LJ049-0095.wav|for all offenses within its jurisdiction, as are FBI agents and Federal marshals. +DUMMY/LJ022-0018.wav|Three years of hard thinking have changed the picture. +DUMMY/LJ041-0098.wav|All the Marine Corps did was to teach you to kill and after you got out of the Marines you might be good gangsters, end quote. +DUMMY/LJ015-0309.wav|and the judge, in passing sentence on him of transportation for life, expressed deep regret that "the ingenuity, skill, and talent, +DUMMY/LJ029-0167.wav|On the morning of the President's arrival, +DUMMY/LJ017-0191.wav|Last of all Mrs. Wilson poisoned her landlady, Mrs. Soames, under precisely the same conditions as the foregoing. +DUMMY/LJ006-0094.wav|Sometimes two mats were allotted to three sleepers. Sometimes four slept under the same bedding, and left their mats unoccupied. +DUMMY/LJ017-0241.wav|the captain's cabin ransacked, his money and clothes divided amongst the mutineers, as well as much of the merchandise on board. +DUMMY/LJ017-0010.wav|and it is hinted that James himself nearly fell a victim to a nefarious attempt of the Duke of Buckingham. +DUMMY/LJ045-0123.wav|to some extent after his Fair Play for Cuba Committee activities had become known, he exaggerated their concern for him. +DUMMY/LJ048-0211.wav|Lawson, also in the lead car, did not scan any buildings since an important part of his job was to look backward at the President's car. +DUMMY/LJ022-0107.wav|will be spent for actually creating new work and not for building up expensive overhead organizations here in Washington. +DUMMY/LJ050-0271.wav|The demands on the President in the execution of His responsibilities in today's world are so varied and complex +DUMMY/LJ024-0045.wav|if the appointment of such justices can be called "packing the Courts," +DUMMY/LJ044-0157.wav|he requested the Embassy to rush his wife's entrance visa because of the impending birth of the second child but stated that, quote, +DUMMY/LJ028-0100.wav|However cruel and religiously intolerant Nebuchadnezzar may have been, he was undoubtedly the greatest builder the world has ever seen. +DUMMY/LJ010-0127.wav|The reckless demeanor of all the convicts except Davidson was most marked. Thistlewood and Ings sucked oranges on the scaffold; +DUMMY/LJ049-0105.wav|During the period between election and inauguration, the President-elect and Vice-President-elect would also be covered. +DUMMY/LJ046-0254.wav|in advance of Presidential trips outside the United States. +DUMMY/LJ036-0199.wav|At about twelve:forty-four p.m. on November twenty-two, +DUMMY/LJ018-0336.wav|Webster's devices for disposing of the body of her victim will call to mind those of Theodore Gardelle, +DUMMY/LJ006-0076.wav|He blamed the construction of Newgate for the neglect of classification, and was yet compelled to confess that he had made no attempt whatever to carry it out. +DUMMY/LJ049-0106.wav|Restricting the coverage in this way would avoid unnecessary controversy over the inclusion or exclusion of other officials who are in the order of succession +DUMMY/LJ015-0263.wav|Scarcely had the conviction of these daring and astute thieves been assured, than another gigantic fraud was brought to light. +DUMMY/LJ044-0180.wav|on his previous residence, his work permit for that country, and several unidentified letters in the Russian language. +DUMMY/LJ016-0322.wav|the commission recommended that death sentences should be carried out within the jail, under such regulations as might be considered necessary +DUMMY/LJ035-0150.wav|As she approached her desk, she saw Oswald. +DUMMY/LJ028-0125.wav|Along its edge I built a great wall of mortar and burned bricks mountain high. +DUMMY/LJ003-0233.wav|Amongst the heap was often the meat that had made turtle soup, which, when heated and stirred together in a saucepan, was said to be very good eating. +DUMMY/LJ022-0185.wav|The answer to this demand was the Federal Reserve System. +DUMMY/LJ038-0213.wav|The last paragraph directed her to the jail +DUMMY/LJ015-0059.wav|Two cases of extensive embezzlement which were discovered almost simultaneously, those of Robson and Redpath, +DUMMY/LJ030-0149.wav|looked at his watch and said "twelve:thirty" to the driver, Special Agent Greer. +DUMMY/LJ015-0137.wav|afterwards he set up as an insurance broker on his own account, but presently failed. +DUMMY/LJ018-0214.wav|As the case must still be well remembered by the present generation, it will be necessary to give here only the briefest summary. +DUMMY/LJ029-0165.wav|Harwood to Main, Main to Houston, Houston to Elm, Elm under the Triple Underpass to Stemmons Freeway, and on to the Trade Mart. +DUMMY/LJ014-0192.wav|then into Palace Yard, where the waterman of the cab-tank, in obedience to Lerigo's shouts, collared the fugitive. +DUMMY/LJ021-0135.wav|which is one of the high purposes of the Recovery Act. +DUMMY/LJ042-0236.wav|In the second simulated transcript which ended with the statement, quote, Newspapers thank you, sir. You are a real patriot! End quote. +DUMMY/LJ015-0297.wav|and that clever schemer sent an elaborate letter of instructions how to ask for the money. +DUMMY/LJ036-0208.wav|white male, approximately thirty, slender build, height five foot ten inches, weight one hundred sixty-five pounds, end quote. +DUMMY/LJ048-0174.wav|Similar arrangements for a motorcade of ten miles, including many blocks of tall commercial buildings is not practical. +DUMMY/LJ007-0216.wav|For nearly twenty-two hours out of the twenty-four the prisoners are locked up, during which time no officer is stationed in the ward with them. +DUMMY/LJ046-0227.wav|According to Special Agent in Charge Bouck, +DUMMY/LJ028-0469.wav|The inner wall of Babylon was called Imgur-Bel, and like the outer wall, it was double. +DUMMY/LJ038-0031.wav|About six or seven people were seated on the theatre's main floor and an equal number in the balcony. +DUMMY/LJ004-0103.wav|Yet another act passed in seventeen ninety-one, if properly observed, should have insured proper attention to them. +DUMMY/LJ035-0031.wav|He shouted twice, quote, Turn loose the elevator, end quote, +DUMMY/LJ005-0203.wav|Thus Dinas Mwddy, in Merionethshire, had, "besides the pinfold and the stocks or crib, a little prison." +DUMMY/LJ030-0115.wav|During this brief stop, agents in the front positions on the running boards of the Presidential follow-up car came forward and stood beside the President's car, +DUMMY/LJ028-0205.wav|In the month Tammuz, when Cyrus fought the troops of Akkad (Babylonia) at Opis on the river Salsallat, +DUMMY/LJ011-0088.wav|The night previous to that fixed for his execution he wrote several letters, one of them being to Edward Gibbon Wakefield, a fellow-prisoner, +DUMMY/LJ036-0107.wav|because he was bawling out the policeman, telling them it wasn't right to put him in line with these teenagers and all of that +DUMMY/LJ039-0022.wav|And I told him that he shouldn't do this, and that he had promised me. +DUMMY/LJ049-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. +DUMMY/LJ043-0009.wav|a Russian-speaking petroleum engineer whom Oswald met as a result of his contact with the Texas Employment Commission office in Fort Worth. +DUMMY/LJ038-0267.wav|Unless the missile or cartridge case can be identified as coming from a particular weapon to the exclusion of all others, +DUMMY/LJ015-0175.wav|A further search discovered him at breakfast at a small house in the New Road. +DUMMY/LJ036-0051.wav|Was a hole in it, hole, and he was dirty, and I didn't look at him. I didn't want to know I even seen him. +DUMMY/LJ011-0228.wav|Mr. W. Wakefield served in a continental army, and rose to the rank of colonel, +DUMMY/LJ019-0104.wav|That gentleman had come to the conclusion that the ordinary and hackneyed methods of treatment were practically inefficacious, +DUMMY/LJ035-0052.wav|Baker stopped at the door of the lunchroom and commanded, quote, Come here, end quote. +DUMMY/LJ049-0044.wav|Even if the car is traveling more slowly, the delay involved in reaching the President may be crucial. +DUMMY/LJ038-0053.wav|George Jefferson Applin, Jr., confirmed that Oswald fought with four or five officers before he was handcuffed. +DUMMY/LJ006-0009.wav|backed up by the evidence of several influential witnesses. +DUMMY/LJ044-0162.wav|I only know that his basic desire was to get to Cuba by any means, and that all the rest of it was window dressing for that purpose. End quote. +DUMMY/LJ049-0088.wav|as is advocacy of the overthrow of the Government by the assassination of any of its officers. +DUMMY/LJ048-0056.wav|Had such a review been undertaken by the FBI, +DUMMY/LJ044-0132.wav|He had not liked his job as a greaser of coffee processing machinery and he held it for only a little over two months. +DUMMY/LJ019-0197.wav|No doubt wiser counsels prevailed with Lord John Russell, +DUMMY/LJ010-0210.wav|The various grades were distinguished by cockades and bows of different colors. +DUMMY/LJ006-0029.wav|The second inspector, the Rev. Whitworth Russell, was the chaplain of Millbank penitentiary, +DUMMY/LJ002-0049.wav|leading to two wards, one of which was thirty-six feet by fifteen, +DUMMY/LJ003-0098.wav|the inevitable consequence of such a situation, their morals must have been destroyed; +DUMMY/LJ023-0028.wav|To complete our program of protection in time, therefore, +DUMMY/LJ012-0097.wav|and at the time of the robbery there were still six thousand pounds worth in the warehouse. +DUMMY/LJ050-0269.wav|The essential terms of such memoranda might well be embodied in an Executive order. +DUMMY/LJ017-0265.wav|Lopez took a violent dislike to the officer of the ward in charge of them, and often expressed a keen desire to do for him. +DUMMY/LJ050-0012.wav|through an Assistant Secretary whose duties also include the direct supervision of the Bureau of the Mint +DUMMY/LJ032-0177.wav|Although Oswald returned to his roominghouse after the assassination and when questioned by the police, claimed to have changed his shirt, +DUMMY/LJ022-0039.wav|I speak, therefore, tonight, to and of the American people as a whole. +DUMMY/LJ004-0081.wav|decent clothing and bedding, and a diet sufficient to support him." +DUMMY/LJ014-0025.wav|A witness deposed to meeting Hocker, soon after the cries of murder were heard, +DUMMY/LJ017-0223.wav|The mutiny was organized with great secrecy, and broke out most unexpectedly in the middle of the night. +DUMMY/LJ005-0112.wav|in this same prison capital convicts were chained to the floor until execution. +DUMMY/LJ011-0170.wav|Wakefield, in reply to her inquiries, satisfied her that her mother was well, and that the real reason for summoning her from school +DUMMY/LJ022-0008.wav|Each of our steps has a definite relationship to every other step. +DUMMY/LJ033-0014.wav|Lee Harvey Oswald lived in a roominghouse in Dallas while his wife and children lived in Irving, at the home of Ruth Paine, +DUMMY/LJ017-0105.wav|Cook's death was horrible +DUMMY/LJ050-0211.wav|Chief Rowley testified that the present workload of each Secret Service agent averages one hundred ten point one cases. +DUMMY/LJ010-0060.wav|The Cato Street conspiracy would have been simply ridiculous but for the recklessness of the desperadoes who planned it. +DUMMY/LJ009-0247.wav|Similar cases were not wanting as regards the executions before Newgate. Others were not less horrible, although there was no failure of apparatus. +DUMMY/LJ050-0097.wav|Even with the advantage of hindsight, this letter does not appear to express or imply Oswald's, quote, +DUMMY/LJ017-0225.wav|The latter had the watch on deck. +DUMMY/LJ011-0029.wav|he sought to rebut these charges, which indeed were never clearly made out, and it is pretty certain that his own account of the causes which led him into dishonesty +DUMMY/LJ013-0216.wav|His account of his acts and movements after the deed +DUMMY/LJ028-0401.wav|In the eleventh century, on the site of the southern part of Babylon, the city of Hillah was built. +DUMMY/LJ015-0154.wav|When the crash came there were pensioners and other recipients of his bounty who could not believe +DUMMY/LJ012-0242.wav|The horror with which Greenacre's crime struck the town was unparalleled since the time when Catherine Hayes slew her husband. +DUMMY/LJ006-0253.wav|the signal for the commencement of obscene talk, revelry, and violence. +DUMMY/LJ009-0093.wav|Why does no one stir to help him? Where would be the use? The hardened burglar moves not, nor does he speak; +DUMMY/LJ033-0004.wav|The Commission has evaluated the evidence tending to show how Lee Harvey Oswald's Mannlicher-Carcano rifle, serial number C two seven six six, +DUMMY/LJ014-0147.wav|shaking her clenched and manacled hands in the officers' faces. +DUMMY/LJ042-0022.wav|he wrote to his brother Robert that the Soviet Union was a country which, quote, I have always considered to be my own, end quote, +DUMMY/LJ043-0035.wav|Shortly after his return from the Soviet Union, Oswald severed all relations with his mother; +DUMMY/LJ031-0072.wav|Dr. Clark did not see any other hole or wound on the President's head. +DUMMY/LJ034-0125.wav|It is understandable, however, for Brennan to have believed that the man with the rifle was standing. A photograph of the building taken seconds after the assassination +DUMMY/LJ018-0287.wav|to which he was introduced by a well-known tailor in Saville Row. +DUMMY/LJ036-0142.wav|Whaley was somewhat imprecise as to where he unloaded his passenger. +DUMMY/LJ036-0153.wav|The elapsed time of the reconstructed run from the Greyhound Bus Station to Neely and Beckley was five minutes and thirty seconds by stopwatch. +DUMMY/LJ005-0055.wav|or a dungeon above or below the gate of a town, to the first attempts at systematic reconstruction carried out under the advice and supervision of Howard. +DUMMY/LJ022-0026.wav|Americans as a whole are feeling a lot better -- a lot more cheerful than for many, many years. +DUMMY/LJ036-0145.wav|However, Neches and Beckley do not intersect. +DUMMY/LJ005-0095.wav|The promulgation of these two Jail Acts strengthened the hands of the Prison Discipline Society enormously. +DUMMY/LJ039-0028.wav|was shipped from Los Angeles on March twenty, nineteen sixty-three, and he left for New Orleans on April twenty-four, nineteen sixty-three. +DUMMY/LJ023-0019.wav|and to cure the abuses and the inequalities which had thrown that system out of joint. +DUMMY/LJ040-0224.wav|and not on any particular mental disturbance, in the boy himself. +DUMMY/LJ023-0040.wav|We are at a crisis in our ability to proceed with that protection. +DUMMY/LJ032-0169.wav|Having considered the probabilities as explained in Stombaugh's testimony, +DUMMY/LJ015-0244.wav|At the same station Pierce joined Agar in the guard's van, and there were now three to carry on the robbery. +DUMMY/LJ004-0198.wav|There was, besides, a mill for grinding corn, somewhat similar to a turn-spit, which prisoners turned by walking in rows. +DUMMY/LJ038-0123.wav|Captain Fritz produced the selective service card bearing the name "Alek J. Hidell." +DUMMY/LJ018-0301.wav|were lodged at once by drafts to "Horton," another alias, in the Continental Bank. +DUMMY/LJ034-0112.wav|The New Orleans police records of his arrest in August of nineteen sixty-three show a weight of one hundred thirty-six pounds. +DUMMY/LJ006-0019.wav|The first named had long been an active philanthropist, devoting himself more particularly to the reformation of juvenile criminals. +DUMMY/LJ046-0098.wav|The history of Presidential protection shows growing recognition over the years that the job must be done by able, dedicated, +DUMMY/LJ017-0163.wav|either arsenic or antimony, or both, in small quantities, +DUMMY/LJ009-0183.wav|the site of the present Sessions House of the Old Bailey, and the operation was witnessed by students and a number of curious spectators. +DUMMY/LJ034-0004.wav|Lee Harvey Oswald was hired on October fifteen, nineteen sixty-three, by the Texas School Book Depository as an "order filler." +DUMMY/LJ005-0199.wav|One cell was a dungeon, and the other an "improper and unhealthy abode for any human being," with a watercourse running through it. +DUMMY/LJ045-0211.wav|where he had spent every previous weekend since his return from Mexico and his wife was once again calling into question his judgment, +DUMMY/LJ023-0013.wav|The change of one vote would have thrown all the affairs of this great Nation back into hopeless chaos. +DUMMY/LJ032-0022.wav|a review of the records of Crescent Firearms revealed that the firm had shipped an Italian carbine, serial number C two seven six six, +DUMMY/LJ048-0152.wav|At some overpasses all persons were excluded +DUMMY/LJ046-0200.wav|Secret Service requests to other agencies for intelligence information +DUMMY/LJ021-0016.wav|is indeed as strong now as it was years ago when Elihu Root said the following very significant words: +DUMMY/LJ028-0507.wav|Some of these decorations, the most valuable objects found in the ruins of the great city, still remain in their places on the walls; +DUMMY/LJ008-0183.wav|but there were no serious accidents, beyond those caused by the goring of a maddened, over-driven ox which forced its way through the crowd. +DUMMY/LJ013-0082.wav|In eighteen forty-four +DUMMY/LJ049-0197.wav|It is also contended that the agents should be intimately associated with the life of the Presidential family +DUMMY/LJ019-0170.wav|more officers were appointed, as the time of so many of those already on the staff was monopolized by attendance at the Central Criminal Court. +DUMMY/LJ038-0044.wav|except that they did not hear Oswald say, quote, It's all over now, end quote. +DUMMY/LJ009-0272.wav|He was in consequence so weak when brought out for execution, that he had to be carried by four men, +DUMMY/LJ019-0098.wav|and the question was once more taken up in the House of Commons. In eighteen forty-nine Mr. Charles Pearson, M.P., +DUMMY/LJ024-0085.wav|and to have it resume its high task of building anew on the Constitution "a system of living law." +DUMMY/LJ016-0289.wav|As day broke the character of the crowd was betrayed. +DUMMY/LJ018-0204.wav|Where it had lain was a yawning gulf or trap sufficient to do for the whole body of police engaged in the capture. +DUMMY/LJ016-0334.wav|Unusual precautions were taken upon this occasion, as some fresh outrage was apprehended. +DUMMY/LJ003-0256.wav|By this means spirits, otherwise unattainable and strictly prohibited, were smuggled into the jail. +DUMMY/LJ006-0227.wav|or his bedclothes were drawn away across the room. +DUMMY/LJ008-0119.wav|He then made some observations to the attendant about the fire, and turning to the doctor, questioned him. +DUMMY/LJ005-0080.wav|Accordingly due provision was made for the enforcement of hard labor on all prisoners sentenced to it, and for the employment of all others. +DUMMY/LJ030-0091.wav|Rufus W. Youngblood, special agent in charge of the Vice President's detail, +DUMMY/LJ044-0171.wav|On September twenty, nineteen sixty-three, Mrs. Paine and her two children arrived in New Orleans from a trip to the East Coast +DUMMY/LJ050-0111.wav|Once the Secret Service has formulated its new standards for collection of information, it should enter into written agreements with each Federal agency +DUMMY/LJ033-0164.wav|He testified, quote, +DUMMY/LJ021-0201.wav|We have avoided on the other hand +DUMMY/LJ029-0038.wav|Special Agent Winston G. Lawson, a member of the White House detail who acted as the advance agent, and Forrest V. Sorrels, +DUMMY/LJ013-0163.wav|The housemaid summoned the cook, and both went to call the valet, Courvoisier, +DUMMY/LJ013-0219.wav|This was no doubt the case, but this much truth his confession may be taken to contain: +DUMMY/LJ009-0131.wav|boast to their companions after the service that they had wiped their eyes during the thanksgiving, to make the ordinary believe they had been crying. +DUMMY/LJ022-0074.wav|It is a problem quite as much for private industry as for the government. +DUMMY/LJ008-0124.wav|The poor soul then joined the doctor in prayer, and never did I witness more contrition at any condemned sermon than he then evinced. +DUMMY/LJ037-0146.wav|Barbara Jeanette Davis, and Virginia Davis. +DUMMY/LJ043-0095.wav|He did not seem to be able to make the accommodations necessary when people work under such conditions and as a result +DUMMY/LJ006-0073.wav|of his many manifest duties, but I shall here confine myself to animadverting on his neglect as regards the appropriation of his prison. +DUMMY/LJ012-0043.wav|He lived in Rosemary Lane, and sometimes he had as much as twenty thousand pounds worth of goods secreted on the premises. +DUMMY/LJ021-0152.wav|and to experiment for a reasonable time with measures suitable to civilize our industrial civilization. +DUMMY/LJ003-0214.wav|in couples, from the time of sentence to that of execution, which was about three or four days generally, +DUMMY/LJ006-0003.wav|Newgate has remained rather in the background while the whole of the jails as a body were under discussion. +DUMMY/LJ015-0093.wav|he replied gaily that his regular work at the Crystal Palace office was useful as a sort of discipline, and kept him steady. +DUMMY/LJ015-0256.wav|Pierce made Kay only a few small payments, then appropriated the rest of the money. +DUMMY/LJ018-0237.wav|No fortune could stand the inroads he made into his mother's money, +DUMMY/LJ037-0200.wav|The name written in this space was D. F. Drittal. +DUMMY/LJ033-0156.wav|Oswald's palmprint on the bottom of the paper bag indicated, of course, that he had handled the bag. +DUMMY/LJ020-0062.wav|Graham Rolls are made by treating the dough mixed for Graham bread as above and following the foregoing receipt in every section, but allowing more time for rising and baking. +DUMMY/LJ038-0243.wav|and twelve, nineteen sixty-three, and most probably on either March nine or March ten. +DUMMY/LJ042-0179.wav|without defense or foundation of government, end quote, +DUMMY/LJ012-0252.wav|Closer investigation proved that it belonged to the trunk already discovered on the second February. +DUMMY/LJ016-0433.wav|laughed and jested with the officers about "Jack Ketch," who, through the postponement of the execution, would lose his Christmas dinner. +DUMMY/LJ038-0295.wav|An official of this church told FBI agents that services are held every Wednesday at the church except during the month of August. +DUMMY/LJ033-0153.wav|These prints were examined independently by Ronald G. Wittmus of the FBI, +DUMMY/LJ010-0293.wav|I have already remarked that as violence was more and more eliminated from crimes against the person, +DUMMY/LJ008-0129.wav|he soon arrived at the room where the sheriff was to give a receipt for his body, according to the usual custom. +DUMMY/LJ025-0083.wav|it remains one of the most constant of the distinctive characters of animals. +DUMMY/LJ047-0225.wav|Hosty testified that he did not know until the evening of Thursday, November twenty-one, that there was to be a motorcade, however, +DUMMY/LJ018-0006.wav|Mr. Briggs left the bank one afternoon as usual, dined with his daughter at Peckham, +DUMMY/LJ028-0437.wav|Here, it has been suggested, were the famous hanging gardens which some ancient authors included among the Seven Wonders of the World. +DUMMY/LJ030-0076.wav|It carried eight Secret Service agents -- two in the front seat, two in the rear, and two on each of the right and left running boards. +DUMMY/LJ015-0069.wav|but the firm he served got him a situation as clerk in the office of the Great Northern Railway, +DUMMY/LJ016-0217.wav|On one particular occasion, however, he got them. A gentleman whose sins brought him to the gallows at Maidstone +DUMMY/LJ018-0178.wav|the case was considered not proved, and he was acquitted. +DUMMY/LJ004-0133.wav|All prisoners passed their time in absolute idleness, or killed it by gambling and loose conversation. +DUMMY/LJ040-0039.wav|He had a great hostility toward his environment, whatever it happened to be, +DUMMY/LJ042-0224.wav|For example, in response to his questions about his decision to go to the Soviet Union, his first draft answered, quote, +DUMMY/LJ028-0432.wav|other parts of them have disappeared entirely. +DUMMY/LJ042-0015.wav|He said living conditions over there convinced him something was wrong with the system, and that possibly Marxism was the answer. +DUMMY/LJ049-0072.wav|those in command should be able to direct the response appropriate to the emergency. The Commission finds that the Secret Service agents in the motorcade +DUMMY/LJ036-0132.wav|Driver, will you call me a cab down here? She had seen him get this cab and she wanted one, too, +DUMMY/LJ028-0324.wav|his body red with marks of scourging and with blood, had no suspicion but that he spoke the truth, and was really come to be their friend and helper. +DUMMY/LJ006-0114.wav|They bought their offices from one another, and were thus considered to have a vested interest in them. +DUMMY/LJ011-0148.wav|had invariably been visited with the death penalty, and which was of a distinctly fraudulent nature. +DUMMY/LJ042-0159.wav|entering the US Marine corp at seventeen, this streak of independence was strengthened by exotic journeys to Japan, +DUMMY/LJ007-0229.wav|they state that they are compelled by an imperative sense of duty to advert in terms of decided condemnation to the lamentable condition of the prisons of the city of London, +DUMMY/LJ029-0095.wav|Even before the Trade Mart had been definitely selected, Lawson and Sorrels began to consider the best motorcade route from Love Field to the Trade Mart. +DUMMY/LJ007-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section ten: The first report of the inspector of prisons. +DUMMY/LJ011-0241.wav|The latter had advertised, offering a sum of one thousand pounds to anyone who would introduce him to some mercantile equipment. +DUMMY/LJ037-0194.wav|Ten dollars in cash was enclosed. +DUMMY/LJ009-0245.wav|He soon rose to his feet, and hearing the sorrowful exclamations of the populace, coolly said, +DUMMY/LJ035-0197.wav|Truly gave this information to Captain Fritz who was on the sixth floor at the time. +DUMMY/LJ014-0242.wav|Employed as a clerk in the Globe Assurance, +DUMMY/LJ002-0051.wav|A high wall fifteen feet in height divided the females' court-yard from the men's. +DUMMY/LJ004-0122.wav|fifty-nine had no division whatever to separate males and females; one hundred and thirty-six had only one division for the purpose; +DUMMY/LJ028-0190.wav|A great empire which had existed for more than three thousand years was brought to an end. +DUMMY/LJ037-0206.wav|The balance due on the purchase was nineteen dollars, ninety-five cents. +DUMMY/LJ029-0147.wav|Sorrels and Lawson reviewed the route in cooperation with Assistant Chief Bachelor and other Dallas police officials who took notes on the requirements +DUMMY/LJ037-0004.wav|At least twelve persons saw the man with the revolver in the vicinity of the Tippit crime scene at or immediately after the shooting. +DUMMY/LJ018-0005.wav|the public mind was greatly agitated by the affair for several months. The story of the murder must be pretty familiar to most of my readers. +DUMMY/LJ039-0225.wav|Based on the known facts of the assassination, +DUMMY/LJ043-0137.wav|The shooting occurred two weeks before Oswald moved to New Orleans and a few days after he had been discharged by the photographic firm. +DUMMY/LJ018-0096.wav|and thus obtained personal knowledge of all who passed in and out, but without obtaining any direct evidence. +DUMMY/LJ050-0134.wav|that an interagency committee has been established to develop more effective criteria. +DUMMY/LJ019-0084.wav|The greatest pains might be taken to secure isolation, +DUMMY/LJ028-0220.wav|The inscription continues: I am Cyrus, king of the world. +DUMMY/LJ016-0228.wav|They did not dislike Calcraft, however, at Newgate. He was an illiterate, simple-minded man, who scarcely remembered what executions he had performed. +DUMMY/LJ003-0150.wav|but they had no share in the prison meat or other charities, and they or their friends found them in food. +DUMMY/LJ016-0381.wav|Even when in the condemned cell, with a shameful death within measurable distance, +DUMMY/LJ010-0310.wav|the house was nearly without resources, and the whole burthen of management falling on me, I sought resources where I could; +DUMMY/LJ022-0133.wav|Feel free to criticize. Tell me of instances where work can be done better, or where improper practices prevail. +DUMMY/LJ035-0146.wav|saw him walk through the clerical office on the second floor toward the door leading to the front stairway. +DUMMY/LJ048-0036.wav|that agents of the FBI in Dallas did not consider Oswald's presence in the Texas School Book Depository Building +DUMMY/LJ015-0270.wav|cheques and bills, of which he made a particular use. +DUMMY/LJ013-0261.wav|he there gave satisfaction for industry, but he was taciturn, and would hold no converse with his fellows. +DUMMY/LJ028-0084.wav|Its impression shows the face of a beardless young man, intelligent and refined. +DUMMY/LJ014-0132.wav|She complained that she had no justice; there was no law for her, she had found no protection either from judges, the prosecutor, or her husband. +DUMMY/LJ037-0116.wav|Guinyard and Callaway viewed the same lineup of four men from which Mrs. Markham had earlier made her identification of Lee Harvey Oswald. +DUMMY/LJ004-0202.wav|A system not adopted generally till nearly half a century later had already prevailed at Ilchester. +DUMMY/LJ010-0185.wav|But Oxford at once assumed the responsibility for his crime, saying, "It was I. I did it. I'll give myself up. +DUMMY/LJ040-0077.wav|where they stayed, except for vacations, until the spring of nineteen forty-eight. +DUMMY/LJ031-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. +DUMMY/LJ016-0430.wav|When vanity is strongly developed there is the keen anxiety to hear what is being said about them outside. +DUMMY/LJ048-0089.wav|Against the background of the critical events of November twenty-two, however, +DUMMY/LJ028-0097.wav|and is continually anxious for the shrines of Babylon and Borsippa; +DUMMY/LJ004-0155.wav|In the crowd, all of them persons who had "no other avocation or mode of livelihood but thieving," Mr. Buxton counted eleven children +DUMMY/LJ030-0237.wav|So I turned around and hit the Vice President on the shoulder and hollered, get down, +DUMMY/LJ011-0099.wav|and losing heavily, to meet the claims upon him he put out a number of forged bills of exchange or acceptances, +DUMMY/LJ010-0254.wav|At the Italian Opera in the evening the audience, on the Queen's appearance, greeted her with loud cheers, and called for the national anthem. +DUMMY/LJ048-0148.wav|that a more formal statement of assigned responsibilities, supplemented in each case to reflect the peculiar conditions of each Presidential trip +DUMMY/LJ039-0196.wav|Three FBI firearms experts tested the rifle in order to determine the speed with which it could be fired. +DUMMY/LJ032-0250.wav|In September nineteen sixty-three, +DUMMY/LJ036-0009.wav|and seven eyewitnesses who heard the shots and saw the gunman flee the scene with the revolver in his hand, +DUMMY/LJ011-0059.wav|There were doubts of his sanity. +DUMMY/LJ025-0019.wav|Although the roots of plants direct themselves toward moisture and their leaves toward air and light, +DUMMY/LJ029-0137.wav|which branches off to the right and is used by traffic going to the Stemmons Freeway and the Dallas-Fort Worth Turnpike. +DUMMY/LJ042-0193.wav|But these preferred tactics now may prove to be too limited in the near future, +DUMMY/LJ019-0218.wav|The prison was to contain four hundred and four prisoners, and the estimated expenditure was seventy-nine thousand pounds. +DUMMY/LJ033-0169.wav|it would be the thickness of both the paper and the tape, the color under various lighting conditions of both the paper and the tape, +DUMMY/LJ002-0065.wav|or female convicts ordered for execution. +DUMMY/LJ049-0166.wav|Although the FBI is not charged with the physical protection of the President, it does have an assignment, as do other Government agencies, +DUMMY/LJ041-0194.wav|I think he wanted both if he could have them. If he didn't, he wanted to die with the knowledge that, or with the idea that he was somebody, end quote. +DUMMY/LJ033-0074.wav|Marina Oswald testified that this was her first knowledge that the rifle was not in its accustomed place. +DUMMY/LJ016-0246.wav|Over the door, in gilt letters, were the words "Crown Office"; in the window was a pile of official envelopes, ostentatiously displayed, +DUMMY/LJ041-0126.wav|But while we were in Russia he spoke well of him. Lee said that when he would return to the United States he would vote for him for Governor, end quote. +DUMMY/LJ029-0085.wav|Kenneth O'Donnell made the final decision to hold the luncheon at the Trade Mart; Behn so notified Lawson on November fourteen. +DUMMY/LJ003-0316.wav|It should be peremptorily forbidden to the keeper or any officer to make a pecuniary profit out of the supplies of food, fuel, or other necessaries. +DUMMY/LJ005-0009.wav|Many thus led in procession were in a shocking condition of dirt and misery, frequently nearly naked, and often bearing upon them +DUMMY/LJ028-0443.wav|because great masses of masonry used to project from its surface. +DUMMY/LJ024-0124.wav|is what the justices say it is rather than what its framers or you might hope it is. +DUMMY/LJ019-0291.wav|which warders, dreading assault, were afraid to enter after dark, even to check rioting and disturbance. +DUMMY/LJ016-0423.wav|When the sheriff offered him counsel for his defense, he declined, saying he wished to make none -- "the witnesses for the prosecution spoke the truth." +DUMMY/LJ041-0042.wav|He wrote a note in his mother's name to school authorities in New Orleans saying that he was leaving school because he and his mother were moving to San Diego. +DUMMY/LJ010-0054.wav|Its ringleaders, Thistlewood and the rest, were after capture honored by committal as State prisoners to the Tower, +DUMMY/LJ021-0203.wav|The course we have followed fits the American practice of government -- a practice of taking action step by step, +DUMMY/LJ035-0198.wav|Truly estimated that he gave this information to Fritz about fifteen or twenty minutes after the shots, +DUMMY/LJ026-0152.wav|Oxygenation is the term used to denote the oxygen-supplying part of respiration; +DUMMY/LJ011-0259.wav|and that, being an irritable temper, he had struck Mullay, after which a violent scuffle took place. +DUMMY/LJ025-0131.wav|and the lobes of the leaf instantly close together in virtue of an act of contraction of part of their substance, +DUMMY/LJ011-0134.wav|signed by more than a thousand members of banking firms. +DUMMY/LJ022-0041.wav|Its first objective is to put men and women now on the relief rolls to work and, incidentally, +DUMMY/LJ004-0156.wav|children hardly old enough to be released from the nursery. +DUMMY/LJ047-0162.wav|that Oswald was living alone in Dallas because she did not want him staying at her house, although she was willing to let Oswald visit his wife and children. +DUMMY/LJ005-0078.wav|but shall also tend more effectually to preserve the health +DUMMY/LJ005-0262.wav|it was suggested that the rules framed for prison government should be subjected to the Secretary of State for approval, +DUMMY/LJ007-0205.wav|is not only permitted still to exist in the prison, but is rendered more powerful than before. +DUMMY/LJ039-0186.wav|would have been to fire three times and hit the target twice within a span of four point eight to five point six seconds. +DUMMY/LJ028-0425.wav|was begun by the Deutsche Orientgesellschaft, in eighteen eighty-nine, and continued till the summer of nineteen fifteen. +DUMMY/LJ043-0174.wav|While there are differences between the two events as far as Oswald's actions and planning are concerned, there are also similarities that should be considered. +DUMMY/LJ050-0008.wav|The Commission believes that the recommended Cabinet-level committee will help to correct many of the major deficiencies of supervision +DUMMY/LJ018-0389.wav|Of the lesser criminals, forgers, thieves, swindlers, Newgate continued to receive its full share up to the last. +DUMMY/LJ002-0154.wav|Thus, amongst others, Thomas Blackburn had been committed on October fifteenth for a debt of one shilling five pence. +DUMMY/LJ003-0052.wav|(three) transports awaiting removal to the colonies, and (four) capital convicts, condemned to death and awaiting execution. +DUMMY/LJ018-0303.wav|a second conspirator exchanged the gold for notes. But just as all promised well, the frauds were detected through the carelessness of the forgers. +DUMMY/LJ036-0096.wav|because of the overwhelming evidence that Oswald was far away from the building by that time. +DUMMY/LJ039-0048.wav|She said, quote, It might have been that he was just trying to test me. +DUMMY/LJ050-0013.wav|and the Department's Employment Policy Program, and who also represents the Secretary of the Treasury on various committees and groups. +DUMMY/LJ016-0164.wav|When discovered next morning, quite dead, it was found that the strap actually did not touch his throat; +DUMMY/LJ050-0011.wav|The Chief of the Service now reports to the Secretary of the Treasury +DUMMY/LJ006-0121.wav|for a petition of from one shilling, half pence to eight shillings, according to its length, +DUMMY/LJ032-0017.wav|Ownership And Possession Of Assassination Weapon +DUMMY/LJ040-0180.wav|The Human Figure Drawings are empty, poor characterizations of persons approximately the same age as the subject. +DUMMY/LJ050-0042.wav|its inability to process large amounts of data, and its failure to provide specific descriptions of the kind of information it sought. +DUMMY/LJ016-0373.wav|and for the greater security of prisoners it is roofed in with iron bars which gives it, at least overhead, the aspect of a huge cage. +DUMMY/LJ003-0086.wav|The crowding was in consequence of the delay in removing transports. +DUMMY/LJ046-0231.wav|Its efforts appear to have been too largely directed at the "crank" threat. +DUMMY/LJ001-0185.wav|printed books might once again illustrate to the full +DUMMY/LJ019-0320.wav|was the boon to which willing industry extending over a long period established a certain claim. +DUMMY/LJ001-0135.wav|it is impossible that they should be quite equal except in lines of poetry +DUMMY/LJ045-0139.wav|because of the, quote, very exciting and disturbing effect, end quote, they had upon her husband, who was not even present at that time. +DUMMY/LJ033-0041.wav|After dinner Ruth Paine and Marina Oswald were busy cleaning house and preparing their children for bed. +DUMMY/LJ004-0172.wav|Untried persons were often lodged in this nauseous underground den, +DUMMY/LJ018-0317.wav|and one warder admitted that he was to have one thousand pounds more paid to him, and to be provided with a passage to Australia. +DUMMY/LJ005-0052.wav|Another point to which the Society devoted infinite pains was the preparation of plans for the guidance of architects in the construction of prisons. +DUMMY/LJ050-0091.wav|It is questionable whether such criteria would have resulted in the referral of Oswald to the Secret Service. +DUMMY/LJ050-0144.wav|Each of these efforts appears sound, +DUMMY/LJ037-0198.wav|Also on the order form was an order, written in ink, for one box of ammunition and one holster, but a line was drawn through these items. +DUMMY/LJ021-0156.wav|To those who say that our expenditures for public works and other means for recovery are a waste that we cannot afford, +DUMMY/LJ035-0102.wav|Truly and Baker continued up the stairs after the encounter with Oswald in the lunchroom. +DUMMY/LJ001-0180.wav|the proportions of the page still give pleasure by the sense of richness that the cuts and letter together convey. +DUMMY/LJ015-0136.wav|First a lawyer's clerk, he then got an appointment in the Peninsular and Oriental Company's office; +DUMMY/LJ041-0087.wav|as a means of getting or attempting to get sympathy, end quote. In Thornley's view, Oswald labored under a persecution complex +DUMMY/LJ013-0171.wav|moreover, the instruments, poker and chisel, by which they had no doubt been effected, +DUMMY/LJ029-0110.wav|The route impressed the agents as a natural and desirable one. +DUMMY/LJ015-0163.wav|was standing at a railway station talking to a certain well-known peer of the realm. +DUMMY/LJ015-0158.wav|of the St. Ann's Society, and one of the supporters and managers of the Patriotic Fund. +DUMMY/LJ017-0052.wav|His skin was extraordinarily fair, his cheeks fresh and ruddy; altogether his face, though commonplace, was not exactly ugly; +DUMMY/LJ002-0040.wav|intended to accommodate a couple of prisoners apiece, but often much more crowded. +DUMMY/LJ008-0310.wav|which was done with a sort of ceremony intended to be impressive. +DUMMY/LJ044-0033.wav|It is also evidence of Oswald's reluctance to describe events accurately +DUMMY/LJ027-0113.wav|Of course the theory of descent with adaptive modification has a simple answer to supply +DUMMY/LJ046-0094.wav|Many Presidents have been understandably impatient with the security precautions which many years of experience dictate +DUMMY/LJ030-0029.wav|After a welcome from the Dallas reception committee, +DUMMY/LJ037-0016.wav|Scoggins hurriedly left his seat and hid behind the cab as the man came back toward the corner with gun in hand. +DUMMY/LJ030-0096.wav|These agents performed for the Vice President the same functions that the agents in the Presidential follow-up car performed for the President. +DUMMY/LJ008-0140.wav|Whenever the public attention had been specially called to a particular crime, either on account of its atrocity, +DUMMY/LJ001-0082.wav|In spite, however, of his praiseworthy efforts, printing had still one last degradation to undergo. +DUMMY/LJ004-0025.wav|and two other acts more peremptory and precise followed on the same subject in succeeding years. +DUMMY/LJ005-0131.wav|In many places too where the prisoners earned money by their work, they were granted a portion of it for their own use after proper deduction for maintenance. +DUMMY/LJ022-0036.wav|Did you ever stop to think that there are, after all, only two positions in the nation +DUMMY/LJ009-0003.wav|Even so severe a critic as Mr. Wakefield states that "a stranger to the scene +DUMMY/LJ049-0121.wav|While the police work of the Dallas authorities in the early identification and apprehension of Oswald was both efficient and prompt, +DUMMY/LJ025-0035.wav|Hence, says Cuvier, it was necessary that the chemical composition of the animal body should be more complicated than that of the plant; +DUMMY/LJ033-0057.wav|and insert it into the paper bag. +DUMMY/LJ046-0002.wav|The President's Commission on the Assassination of President Kennedy. Chapter eight. The Protection of the President. Part one. +DUMMY/LJ036-0091.wav|And neither Captain Fritz nor any other officer can remember that Oswald dramatically arose from his chair +DUMMY/LJ003-0289.wav|Upon the twenty-ninth July the same year, +DUMMY/LJ049-0224.wav|and services related to the security of the President. +DUMMY/LJ020-0069.wav|One even teaspoonful of salt. +DUMMY/LJ039-0073.wav|when you shoot at one hundred seventy-five feet or two hundred sixty feet, which is less than a hundred yards, with a telescopic sight, +DUMMY/LJ010-0181.wav|and the royal party drove back to Clarence House, the Queen being anxious to give the first news of the outrage and of her safety to her mother, +DUMMY/LJ041-0102.wav|Powers said, quote, now he was Oswald the man rather than Oswald the rabbit, end quote. +DUMMY/LJ033-0020.wav|prior to November twenty-one, nineteen sixty-three, except on Monday, October twenty-one, when he visited his wife in the hospital +DUMMY/LJ034-0146.wav|In an interview with FBI agents on December seventeen, nineteen sixty-three, +DUMMY/LJ003-0320.wav|which recommended restrictions upon the number of visitors admitted. +DUMMY/LJ007-0032.wav|were jumbled up together, higgledy-piggledy, the old and the young, the murderer and the child who had broken into a dwelling. +DUMMY/LJ006-0205.wav|novels, flash songs, play-books, such as "Jane Shore," "Grimm's German Tales," with Cruikshank's illustrations, +DUMMY/LJ025-0095.wav|once supposed to be exclusively confined to plants, are now known to be regular and normal products of animals. +DUMMY/LJ018-0261.wav|and passed the heaviest sentence of the law, transportation for life. +DUMMY/LJ002-0017.wav|Trustworthy evidence is forthcoming to the effect that these high figures were constantly maintained for many months at a time. +DUMMY/LJ033-0115.wav|I remember that I didn't look at the package very much but when I did look at it he did have his hands on the package like that, end quote, and at this point +DUMMY/LJ009-0304.wav|four pounds, twelve shillings; and finally the funeral, cart, coffin, and other petty expenses, amounting to seven pounds ten +DUMMY/LJ021-0205.wav|I believe with Abraham Lincoln, that "The legitimate object of government is to do for a community of people +DUMMY/LJ001-0069.wav|This type was introduced into England by Wynkyn de Worde, Caxton's successor, +DUMMY/LJ033-0037.wav|In deciding whether Oswald carried a rifle to work in a long paper bag on November twenty-two, +DUMMY/LJ014-0089.wav|For the remainder of that week and part of the next the murderers stayed in the house, and occupied the kitchen, close to the remains of their victim. +DUMMY/LJ036-0183.wav|After a friend had called and told her, "President Kennedy has been shot," she turned on the television. +DUMMY/LJ019-0385.wav|Mr. (now Sir Richard) Cross, having applied himself vigorously to the task of reorganizing the whole system, became convinced +DUMMY/LJ048-0166.wav|But on out-of-town trips where the route is decided on and made public only a few days in advance, +DUMMY/LJ006-0228.wav|The ragged part of the prisoners were very anxious to destroy the clothes of the better dressed, and often lighted small pieces of cloth, +DUMMY/LJ035-0122.wav|Dougherty took the elevator down from the fifth floor after Jarman, Norman, and Williams ran to the west windows and were deciding what to do. +DUMMY/LJ009-0073.wav|The forger has never moved. +DUMMY/LJ033-0182.wav|Since the Depository normally used approximately one roll of paper every three working days, +DUMMY/LJ022-0040.wav|My most immediate concern is in carrying out the purposes of the great work program just enacted by the Congress. +DUMMY/LJ009-0022.wav|I was informed that this unnecessarily harassed his feelings, and that the object of such sermons was solely to console the prisoner, +DUMMY/LJ009-0289.wav|and before he took to hanging he was employed as a watchman at Reid's brewery in Liquorpond Street. +DUMMY/LJ019-0267.wav|The Secretary of State's suggested scale of diet had seldom been closely followed. +DUMMY/LJ033-0163.wav|James C. Cadigan, a questioned-documents expert with the Bureau, compared the samples with the paper and tape in the actual bag. +DUMMY/LJ011-0158.wav|The two brothers started at once for Liverpool, where Miss Turner was at school with a Mrs. Daulby. +DUMMY/LJ015-0080.wav|Shares in the company were represented by certificates, which often enough never left the company's, or more exactly Robson's, hands. +DUMMY/LJ005-0077.wav|"expedient to introduce such measures and arrangements as shall not only provide for the safe custody, +DUMMY/LJ008-0252.wav|Another deep roar, louder than any which had preceded it, broke from the multitude. +DUMMY/LJ006-0074.wav|He was unable to give any reason whatever for not utilizing the whole of the wards. +DUMMY/LJ048-0100.wav|who had just completed advance work on the President's trip to Tampa. +DUMMY/LJ016-0346.wav|a fact duly advertised as completed by the hoisting of the black flag over the jail, +DUMMY/LJ014-0314.wav|Pries at length was discovered +DUMMY/LJ026-0154.wav|In the animal oxygen is absorbed by the blood, in excess by the hemoglobin of the red cells of the blood +DUMMY/LJ050-0197.wav|According to Secretary Dillon, +DUMMY/LJ013-0205.wav|Later in the day he tried to commit suicide by cramming a towel down his throat, but was prevented. +DUMMY/LJ005-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section eight: The beginnings of prison reform. +DUMMY/LJ022-0087.wav|After the Division of Applications and Information has sifted those projects, +DUMMY/LJ042-0205.wav|There can be no sympathy for those who have turned the idea of communism into a vile curse to Western man. +DUMMY/LJ044-0197.wav|It should be noted that his interest in Cuba seems to have increased along with the sense of frustration which must have developed +DUMMY/LJ030-0043.wav|Its function was to alert police along the route that the motorcade was approaching and to check for signs of trouble. +DUMMY/LJ016-0145.wav|Encouraged by the shouts and signals of some workmen employed on a building outside, the prisoner made one of the most marvellous jumps on record, +DUMMY/LJ030-0175.wav|the Governor started to look back over his left shoulder, but he never completed the turn because he felt something strike him in the back. +DUMMY/LJ036-0194.wav|Patrolman J. D. Tippit joined the Dallas Police Department in July nineteen fifty-two. +DUMMY/LJ022-0020.wav|cause of clearer thinking and a better understanding, are considering the whole rather than a mere part relating to one section or to one crop, +DUMMY/LJ001-0166.wav|is the habit that publishers have of eking out a thin volume by printing it on thick paper almost of the substance of cardboard, +DUMMY/LJ010-0300.wav|Mr. Fauntleroy was a member of a banking firm, which his father had established in conjunction with a gentleman of the name of Marsh, and others. +DUMMY/LJ013-0011.wav|She was freighted by the firm of Zulueta and Co. for a voyage to Santa Cruz. +DUMMY/LJ028-0341.wav|the rest who did not see it kept at their posts, till at last they too learned that they were betrayed. +DUMMY/LJ012-0275.wav|He thought he had killed her, and in his terror began at once to consider how he might dispose of the body and escape arrest. +DUMMY/LJ047-0139.wav|The next word about Oswald's location was a communication from the CIA to the FBI on October ten, +DUMMY/LJ013-0242.wav|This increased when the officer, accompanied by two others, a neighbor and a bailiff, entered one of the stables. +DUMMY/LJ013-0262.wav|The woman where he lodged noticed that he was very restless at night, moaning and sighing much. Detection came unexpectedly. +DUMMY/LJ031-0048.wav|followed by, quote, more profuse bleeding, end quote, after some circulation was established. +DUMMY/LJ041-0185.wav|Thornley also believed that Oswald's Marxist beliefs led to an extraordinary view of history under which, quote, He looked upon the eyes of future people +DUMMY/LJ037-0142.wav|serial number V five one zero two one zero. +DUMMY/LJ007-0028.wav|there still remained one where the general callous indifference and mismanagement culminated in cruel culpable neglect. +DUMMY/LJ010-0055.wav|but they came one and all to Newgate for trial at the Old Bailey, and remained there after conviction till they were hanged. +DUMMY/LJ019-0160.wav|and check by remonstrance or threat of punishment all who broke the peace of the prison. +DUMMY/LJ007-0196.wav|the provision of dining-rooms and dining-tables. +DUMMY/LJ032-0194.wav|the Commission was unable to reach any firm conclusion as to when the fibers were caught in the rifle. +DUMMY/LJ002-0158.wav|Other cases are recorded elsewhere, as at the Giltspur Street Compter, where in eighteen oh five Mr. Neild found a man named William Grant +DUMMY/LJ043-0126.wav|told the prospective employer that Oswald was, quote, kinda peculiar sometimes and that he had some knowledge of the Russian language, end quote. +DUMMY/LJ001-0055.wav|some of which -- as, e.g., that of Jacobus Rubeus or Jacques le Rouge -- is scarcely distinguishable from his. +DUMMY/LJ028-0185.wav|Perhaps Babylon was so strongly fortified that at first he made no attempt to add it to his empire, +DUMMY/LJ031-0067.wav|A femoral pulse was also detected in the President's leg. +DUMMY/LJ005-0298.wav|to the county jails from such prisons as were past improvement, and that the borough funds should be charged for the accommodation. +DUMMY/LJ020-0012.wav|While still hot, mix in the sugar and butter, beating all to a lumpless cream. +DUMMY/LJ029-0138.wav|The Elm Street approach to the Stemmons Freeway is necessary +DUMMY/LJ006-0167.wav|He only attended chapel once on Sunday, never on the week-day, and generally devoted the time service was in progress +DUMMY/LJ038-0061.wav|He saw no shotgun in the possession of any policeman near Oswald. +DUMMY/LJ018-0392.wav|to have obtained as much as forty thousand pounds by false and fraudulent pretenses. +DUMMY/LJ037-0269.wav|Oswald's jacket was found along the path of flight taken by the gunman as he fled from the scene of the killing. +DUMMY/LJ009-0062.wav|buries his head under his body. +DUMMY/LJ002-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section four: Newgate down to eighteen eighteen. +DUMMY/LJ022-0130.wav|It is time to provide a smashing answer for those cynical men who say that a democracy cannot be honest and efficient. +DUMMY/LJ029-0003.wav|This chapter describes President Kennedy's trip to Dallas, from its origin through its tragic conclusion. +DUMMY/LJ002-0303.wav|Besides these fees, legitimate and illegitimate, there were others which must be paid before release. +DUMMY/LJ049-0029.wav|Without attempting to prescribe or recommend specific measures which should be employed for the future protection of Presidents, +DUMMY/LJ034-0110.wav|Oswald was five foot nine inches, slender and twenty-four years old. +DUMMY/LJ033-0029.wav|It would appear, however, that obtaining curtain rods was not the purpose of Oswald's trip to Irving on November twenty-one. +DUMMY/LJ050-0119.wav|they should be responsible for advising the Secret Service if information develops indicating the existence of an assassination plot +DUMMY/LJ034-0169.wav|the same corner where Brennan was sitting on a concrete wall. +DUMMY/LJ042-0045.wav|I do not wish to be a part of it, nor do I ever again wish to be used as a tool in its military aggressions. +DUMMY/LJ022-0194.wav|Never since my inauguration in March, nineteen thirty-three, have I felt so unmistakably the atmosphere of recovery. +DUMMY/LJ039-0162.wav|In an effort to test the rifle under conditions which simulated those which prevailed during the assassination, +DUMMY/LJ043-0146.wav|Following his unsuccessful attack on Walker, Oswald returned home. +DUMMY/LJ050-0167.wav|to develop coordinated and mutually compatible systems, even where such coordination would not seem inconsistent +DUMMY/LJ017-0011.wav|But secret poisoning on a wholesale scale such as was practiced in Italy and France was happily never popularized in England. +DUMMY/LJ016-0382.wav|many cling still to life, expecting much from the intercession of friends or the humanitarianism of the age. +DUMMY/LJ006-0214.wav|but no steps were taken to prevent any prisoner from obtaining more if he could pay for it. +DUMMY/LJ041-0112.wav|At his own request, +DUMMY/LJ008-0319.wav|Sometimes any who had had a narrow escape fainted, but the bulk of those respited looked on with unfeeling indifference. +DUMMY/LJ028-0386.wav|But the walls of the old city had not yet served their full purpose. +DUMMY/LJ031-0159.wav|to institute strict security measures for the Presidential aircraft, the airport terminal, and the surrounding area. +DUMMY/LJ017-0230.wav|came out of the cabin, was caught near the 'companion' by the rest of the mutineers, and promptly dispatched with daggers. +DUMMY/LJ015-0132.wav|The total amount was never exactly made out, but the false stock created and issued by him was estimated at two hundred twenty thousand pounds. +DUMMY/LJ001-0009.wav|Printing, then, for our purpose, may be considered as the art of making books by means of movable types. +DUMMY/LJ018-0035.wav|This was the 'City of Manchester,' which was expected to arrive some days before the 'Victoria,' and did so. +DUMMY/LJ028-0393.wav|As the Emperor Julian was marching forward through Babylonia, +DUMMY/LJ007-0008.wav|their especial care. +DUMMY/LJ034-0003.wav|Oswald at Window +DUMMY/LJ029-0064.wav|Shortly before, the Dallas police had reported to the Secret Service that the handbill had appeared on the streets of Dallas. +DUMMY/LJ032-0247.wav|during the summer of nineteen sixty-three. +DUMMY/LJ011-0214.wav|They were tried in March of the following year, Edward Wakefield having apparently given himself up, +DUMMY/LJ049-0226.wav|could improve their existing capacities and procedures so as to lessen the chances of assassination. +DUMMY/LJ042-0057.wav|three. That I want to, and I shall, live a normal happy and peaceful life here in the Soviet Union for the rest of my life. +DUMMY/LJ032-0051.wav|On that date, Klein's placed an internal control number V C eight three six on this rifle. +DUMMY/LJ033-0112.wav|measured thirty-four point eight inches. +DUMMY/LJ026-0027.wav|as in a mountainous country, it is impossible to say where mountain ends and valley begins; as in the development of an animal, +DUMMY/LJ005-0292.wav|In many of the smaller boroughs they are totally unfit for the confinement of human beings. +DUMMY/LJ048-0287.wav|It is conceivable that those men who had little sleep, and who had consumed alcoholic beverages, even in limited quantities, +DUMMY/LJ028-0031.wav|About you is all that remains of the second of the Seven Wonders of the World. +DUMMY/LJ028-0372.wav|The poor of the surrounding country occupied its dismantled palaces. +DUMMY/LJ035-0179.wav|said that to his knowledge the building was not sealed off at twelve:thirty-six p.m. when he called in on police radio +DUMMY/LJ026-0080.wav|often in the form of granules embedded in the chlorophyll bodies and free oxygen, most of which is returned to the atmosphere. +DUMMY/LJ010-0077.wav|On his release he sent a challenge to Lord Sidmouth, the Home Secretary, and was again arrested and imprisoned. +DUMMY/LJ024-0008.wav|That plan has two chief purposes. +DUMMY/LJ023-0087.wav|the majority of the Court has been assuming the power to pass on the wisdom of these acts of the Congress +DUMMY/LJ012-0261.wav|This brought suspicion on to a certain James Greenacre, +DUMMY/LJ016-0180.wav|But Calcraft regularly succeeded Foxen, who followed Botting, and Dennis, the actor in the seventeen eighty riots. +DUMMY/LJ004-0183.wav|With all this, the inmates were crowded together at night to such a degree as to excite surprise that they should escape suffocation. +DUMMY/LJ041-0039.wav|and that is the last I have seen or spoken with Oswald. End quote. +DUMMY/LJ017-0007.wav|It will be remembered that as far back as the reign of Henry the eighth a new and most cruel penalty was devised for the punishment of the Bishop of Rochester's cook, +DUMMY/LJ019-0396.wav|At a short distance stood another prison of detention, that of Clerkenwell, +DUMMY/LJ031-0064.wav|Drs. Paul C. Peters and Charles R. Baxter initiated these procedures. +DUMMY/LJ013-0039.wav|this Wallace added that his friend had made several voyages with the distinct intention of superintending the predetermined shipwrecks. +DUMMY/LJ035-0152.wav|carrying a full bottle of Coca-Cola in his hand, presumably purchased after the encounter with Baker and Truly. +DUMMY/LJ042-0053.wav|Responding to Robert's statement that he had not "renounced" him, +DUMMY/LJ005-0191.wav|The Society proceeded to support this indictment by facts. It is much the old story. +DUMMY/LJ049-0018.wav|which was neither bulletproof nor bullet resistant. +DUMMY/LJ030-0099.wav|telephone and Western Union vehicles, a White House communications car, +DUMMY/LJ033-0008.wav|(two) the disappearance of the rifle from its normal place of storage, (three) +DUMMY/LJ002-0278.wav|while the funds of several bequests and charities were applied in adding to the material comfort of the prisoners. +DUMMY/LJ005-0259.wav|It was laid down as a first and indispensable principle that uniformity of discipline should prevail everywhere, +DUMMY/LJ007-0063.wav|It is only fair to Mr. Cotton to add that, according to his own journal, he was unremitting in his attentions to convicts who were actually cast for death, +DUMMY/LJ040-0143.wav|Contrary to reports that appeared after the assassination, the psychiatric examination did not indicate that Lee Oswald was a potential assassin, +DUMMY/LJ042-0003.wav|Background and Possible Motives, Part three. +DUMMY/LJ028-0395.wav|It was a low rampart, enclosing a wide space planted with trees of every sort, in which all kinds of beasts were shut up; +DUMMY/LJ022-0002.wav|April twenty-eight, nineteen thirty-five. +DUMMY/LJ041-0181.wav|Delgado testified that Oswald was, quote, a complete believer that our way of government was not quite right, end quote. +DUMMY/LJ019-0292.wav|Prisoners still slept two in a bed. +DUMMY/LJ004-0147.wav|The irons, which nearly every one wore, were remarkably heavy; those double ironed could not take off their small clothes. +DUMMY/LJ009-0259.wav|The wretched man did not fall with it, but jumped on to the platform, and seizing the rope with his hands, tried to avoid strangulation. +DUMMY/LJ016-0268.wav|The remarks heard amongst the crowd were of coarse approval. +DUMMY/LJ015-0018.wav|and dated back to the Commonwealth, when, under the title of Snow and Walton, it carried on business as pawnbrokers. +DUMMY/LJ039-0035.wav|he was not actually planning to shoot Mr. Nixon at that time in Dallas. +DUMMY/LJ029-0046.wav|Preventive Intelligence Activities. The Protective Research Section (PRS) of the Secret Service +DUMMY/LJ007-0088.wav|for not less a period than twenty-four years. +DUMMY/LJ043-0098.wav|it began to appear that Oswald was having considerable difficulty doing accurate work and in getting along with the other employees. +DUMMY/LJ024-0050.wav|Andrew Jackson, Abraham Lincoln and Ulysses S. Grant. +DUMMY/LJ009-0240.wav|A somewhat similar event happened at Chester not long afterwards; the ropes by which two offenders were turned off broke a few inches from their necks. +DUMMY/LJ040-0141.wav|reported that Lee, quote, confided that the worse thing about Youth House was the fact that he had to be with other boys all the time, +DUMMY/LJ009-0156.wav|"with a view," as he himself said, "to his professional studies." +DUMMY/LJ044-0094.wav|We figured after this broadcast of August twenty-one, why, that was no longer possible, end quote. +DUMMY/LJ019-0087.wav|or make signs; or the chances of recognizing or being recognized were reduced to a minimum by the use of a mask. +DUMMY/LJ050-0061.wav|(b) who have made threats of bodily harm against officials or employees of Federal, state or local government or officials of a foreign government, +DUMMY/LJ009-0299.wav|The man to be hanged belonged to Wales, and no Welshman would do the job. +DUMMY/LJ038-0032.wav|McDonald first searched two men in the center of the main floor, about ten rows from the front. +DUMMY/LJ004-0079.wav|"All measures and practices in prison which may injure him in any way are illegal, +DUMMY/LJ039-0157.wav|A series of tests were performed to determine whether the weapon and ammunition used in the assassination +DUMMY/LJ008-0260.wav|One of the worst evils was the terrible and long-protracted uncertainty as to the result. +DUMMY/LJ029-0170.wav|Dallas Before the Visit +DUMMY/LJ028-0398.wav|The walls of Babylon were destined to serve still another purpose. +DUMMY/LJ010-0267.wav|his nose was marked with a scar or black patch, and he was altogether of a dirty appearance. +DUMMY/LJ050-0007.wav|have made it difficult for the Treasury to maintain close and continuing supervision. +DUMMY/LJ013-0220.wav|that Courvoisier was idle, discontented, ready to take offense, greedy of gain; +DUMMY/LJ030-0233.wav|behind the Presidential follow-up car at the time of the shooting and signaled for it to move in closer. +DUMMY/LJ005-0280.wav|The Lords' Committee fully recognized the painful fact +DUMMY/LJ007-0106.wav|Pilgrim and Dasent were caught and given into the custody of the sergeant-at-arms, and the rest were either arrested or they surrendered. +DUMMY/LJ038-0133.wav|Oswald responded that the landlady simply made a mistake, because he told her that his name was Lee, meaning his first name. +DUMMY/LJ045-0201.wav|Oswald had a highly exaggerated sense of his own importance, but he had failed at almost everything he had ever tried to do. +DUMMY/LJ028-0199.wav|there suddenly appeared upon the wall an armless hand. +DUMMY/LJ045-0110.wav|Ruth Paine and Marina Oswald decided to call Oswald at the place where he was living, unbeknownst to them, under the name of O. H. Lee. +DUMMY/LJ036-0186.wav|Oswald had entered the house in his shirt sleeves, but when he left, he was zipping up a jacket. +DUMMY/LJ031-0120.wav|While the second operation was in progress, +DUMMY/LJ009-0234.wav|There the hangman added his weight to that of the suspended culprit, +DUMMY/LJ046-0153.wav|If the material was evaluated +DUMMY/LJ028-0495.wav|It must have been an imposing sight to one standing without to have seen the walls, one after another, +DUMMY/LJ036-0138.wav|But when I got pretty close to five hundred block at Neches and North Beckley which is the five hundred block, he said, "This will do fine," and I pulled over to the curb right there. +DUMMY/LJ048-0011.wav|Neither believed that the Bureau investigation of him up to November twenty-two revealed any information which would have justified referral to the Secret Service. +DUMMY/LJ011-0172.wav|Mr. Turner was on the verge of bankruptcy. He was at that moment at Kendal, and wished her to join him there at once. +DUMMY/LJ028-0063.wav|Then Syria and Palestine were added to his future empire. +DUMMY/LJ019-0078.wav|The amount of exercise varied from one or two hours daily to half the working day. +DUMMY/LJ019-0123.wav|On that account he made no provision to insure safe custody, +DUMMY/LJ008-0264.wav|But the murderers formed only a small proportion of the total number sentenced to death, and for the rest there was a long period of anxious suspense, +DUMMY/LJ050-0249.wav|It may eventually be desirable to codify the practice in an Executive order. +DUMMY/LJ035-0161.wav|The reconstruction was the minimum time. Accordingly, she probably met Oswald at about twelve:thirty-two, +DUMMY/LJ014-0303.wav|Nothing to equal the excitement caused by the forgeries of Robert Ferdinand Pries had been known before in the city of London. +DUMMY/LJ038-0301.wav|(two) the photographs found among Oswald's possessions, +DUMMY/LJ019-0377.wav|and it is not strange if they did not very frequently hand up the offenders to the reproof and correction of the Secretary of State. +DUMMY/LJ009-0291.wav|He was always known as a mild-mannered man of simple tastes, much given to angling in the New River, and a devoted rabbit fancier. +DUMMY/LJ011-0042.wav|Visited Mr. Fauntleroy. The sheriffs have very kindly permitted him to remain in the turnkey's room where he was originally placed; nor can I omit expressing a hope +DUMMY/LJ015-0145.wav|Now he launched out into great expenditure, took a house in Chester Terrace, and became known as a Maecenas and patron of the arts. +DUMMY/LJ049-0175.wav|before the President reached Dallas. +DUMMY/LJ028-0342.wav|Thus was Babylon taken for the second time. +DUMMY/LJ007-0004.wav|although their ministrations were occasionally extended to the male side. +DUMMY/LJ034-0214.wav|Investigation has established that Altgens' picture was taken approximately two seconds after the firing of the shot +DUMMY/LJ044-0040.wav|He wrote that, quote, thousands of circulars were distributed, end quote. +DUMMY/LJ017-0201.wav|Dr. Alfred Taylor, the great authority and writer on medical jurisprudence, corroborated this, and in his evidence on the trial +DUMMY/LJ043-0131.wav|As a result Oswald was not hired. +DUMMY/LJ001-0042.wav|and though more Roman than that, yet scarcely more like the complete Roman type of the earliest printers of Rome. +DUMMY/LJ017-0271.wav|Lyons asked the time, and was told it was only five. +DUMMY/LJ049-0148.wav|For various reasons the Secret Service has functioned largely as an informal part of the White House staff, with the result +DUMMY/LJ002-0239.wav|also for prisoners committed by the Admiralty Court. +DUMMY/LJ014-0238.wav|In eighteen fifty occurred the first of a series of gigantic frauds, +DUMMY/LJ034-0215.wav|which entered the back of the President's neck. +DUMMY/LJ016-0014.wav|Sewn in between the upper and lower leathers several spring saws were found, which would have easily cut through any bars. +DUMMY/LJ032-0267.wav|examined the blanket and discovered a bulge approximately ten inches long midway in the blanket. +DUMMY/LJ034-0159.wav|The Commission, therefore, does not base its conclusion concerning the identity of the assassin +DUMMY/LJ001-0053.wav|they must be considered to have ended the age of fine printing in Italy. +DUMMY/LJ027-0031.wav|but not in any striking way is there change in the internal organs. +DUMMY/LJ013-0008.wav|came to light in this same year, eighteen forty-one. +DUMMY/LJ036-0041.wav|McWatters also remembered that a man received a transfer at Lamar and Elm Streets and that a man in the lineup was about the size of this man. +DUMMY/LJ031-0124.wav|was cleansed and closed with sutures; but a small metallic fragment remained in the Governor's leg. +DUMMY/LJ006-0241.wav|A prisoner Baxter is in the infirmary in consequence of a severe injury to his wrist-joint. +DUMMY/LJ016-0377.wav|The older officers, with several of whom I have conversed, have thus had unusual opportunities of watching the demeanor of murderers both before trial +DUMMY/LJ009-0144.wav|In fact his looks denoted extreme sorrow and contrition, +DUMMY/LJ004-0234.wav|This change led to some discontent, until it was found that the much greater boon of relief from irons accompanied it, and the reform was quietly accepted. +DUMMY/LJ021-0163.wav|But as for this country, I stand or fall by my refusal to accept as a necessary condition of our future +DUMMY/LJ045-0089.wav|Marina Oswald also ridiculed her husband's political views, thereby tearing down his view of his own importance. +DUMMY/LJ001-0138.wav|this is the tendency to the formation of ugly meandering white lines or "rivers" in the page +DUMMY/LJ032-0003.wav|The preceding chapter has established that the bullets which killed President Kennedy and wounded Governor Connally +DUMMY/LJ041-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ040-0067.wav|and his half-brother John Pic, who had been born in nineteen thirty-two during Marguerite's previous marriage. +DUMMY/LJ046-0041.wav|the personal embodiment and representative of their dignity and majesty, end quote. +DUMMY/LJ037-0012.wav|A police car moving east on tenth at about ten or twelve miles an hour passed in front of his cab. +DUMMY/LJ038-0101.wav|Oswald said that this was not true. Oswald denied that he had a rifle wrapped up in a blanket in the Paine garage. +DUMMY/LJ036-0101.wav|He was taken to the lineup room where, according to Whaley, five young teenagers, all handcuffed together, were displayed with Oswald. +DUMMY/LJ034-0008.wav|in deciding whether Lee Harvey Oswald was present at this window at the time of the assassination. +DUMMY/LJ004-0089.wav|and to pay overseers or instructors out of the county rates. +DUMMY/LJ016-0370.wav|All that the hangman, whoever he may be, does under the new regime is to unhook the halter and remove the pinioning straps. +DUMMY/LJ040-0074.wav|About six months later she also withdrew John Pic and Robert Oswald. +DUMMY/LJ005-0237.wav|so that "corrupt conversation obliterated from the mind of him who is doomed to suffer every serious feeling and valuable impression." +DUMMY/LJ030-0170.wav|Governor Connally testified that he recognized the first noise as a rifle shot +DUMMY/LJ027-0020.wav|The unity in life, then, is not less a fact than is life's great diversity. +DUMMY/LJ050-0023.wav|Work is going forward +DUMMY/LJ033-0018.wav|Oswald generally would go to Irving on Friday afternoon and return to Dallas Monday morning. +DUMMY/LJ038-0033.wav|He walked out of the row up the right center aisle. +DUMMY/LJ038-0191.wav|five. The money from work will possibly be coming. The money will be sent to our post office box. Go to the bank and cash the check. +DUMMY/LJ047-0058.wav|Agent Fain retired from the FBI in October nineteen sixty-two, and the closed Oswald case was not reassigned. +DUMMY/LJ010-0106.wav|He was in bed with his breeches on (in the pockets of which were found a number of cartridges), the black belt he had worn at Cato Street, +DUMMY/LJ008-0284.wav|Nothing could be more strongly marked than the contrast between the ultimate destiny of different individuals all abiding the same awful doom: +DUMMY/LJ027-0009.wav|there are, after all, only a few types of structure among all animals and plants, some three or four or eight or ten general modes of development, +DUMMY/LJ019-0096.wav|other authorities as strongly condemned it as brutalizing, unequal in its operation, and altogether a "deplorable invention." +DUMMY/LJ040-0033.wav|At the same time he professed his love for her and said that he could not get along without her. +DUMMY/LJ049-0178.wav|should have constituted a sufficient basis to warn the Secret Service of the Oswald risk. +DUMMY/LJ018-0353.wav|Wainwright's demeanor after sentence has been described in the last chapter. +DUMMY/LJ010-0159.wav|He was eventually sentenced to death, but the plea of lunacy was allowed, and he was confined for life. +DUMMY/LJ020-0048.wav|Knead it again, then, and wait upon its rising for another three hours. The dough should be as soft as can be handled. +DUMMY/LJ026-0107.wav|Dissolved in water, the sugar is transported down delicate tubes, chiefly in the growing bark region of the stem. +DUMMY/LJ046-0138.wav|PRS instructed the White House mailroom, a source of much PRS data, +DUMMY/LJ002-0063.wav|It was fifty feet by twenty-five, and had five wards each thirty-eight by fifteen. At one end of the yard was an arcade, +DUMMY/LJ042-0154.wav|While it was apparently intended for publication in the United States, and is in many respects critical of certain aspects of life in the Soviet Union, +DUMMY/LJ015-0295.wav|he was told it was only at Mr. Whitney's disposal, and that it could be paid to no one else. +DUMMY/LJ033-0138.wav|It was the appropriate size to contain, in disassembled form, +DUMMY/LJ044-0041.wav|and that he continued to receive inquiries through his post office box which he endeavored, quote, to keep answering to the best of my ability, end quote. +DUMMY/LJ010-0261.wav|Only one person saw the movement, a lad named Dasset, who at once collared the cripple, and taking him up to two policemen, +DUMMY/LJ016-0144.wav|He had disappeared. +DUMMY/LJ009-0159.wav|and the mitigation of the criminal code, for which Romilly had so strenuously labored, was daily more and more of an accomplished fact. +DUMMY/LJ019-0339.wav|while decency and good order were to be insured by the strict prohibition of gambling and drunkenness. +DUMMY/LJ001-0141.wav|The general solidity of a page is much to be sought for +DUMMY/LJ022-0057.wav|The program for social security now pending before the Congress is a necessary part of the future unemployment policy of the government. +DUMMY/LJ003-0031.wav|The same system was extended to the criminal side, and cases were on record of the place of wardsman being sold for considerable sums. +DUMMY/LJ036-0114.wav|Only two of the men in the lineup with Oswald were teenagers: John T. Horn, aged eighteen, was Number one; +DUMMY/LJ042-0229.wav|To the question of, quote, Are you a Communist? End quote, he first answered "Yes," +DUMMY/LJ039-0164.wav|from a tower at three silhouette targets at distances of one hundred seventy-five, two hundred forty, and two hundred sixty-five feet. +DUMMY/LJ040-0013.wav|Oswald's complete state of mind and character are now outside of the power of man to know. +DUMMY/LJ032-0201.wav|and issues of two newspapers later identified as the Worker and the Militant. +DUMMY/LJ007-0231.wav|in which the master evil of jail association and consequent contamination still continues to operate directly to the encouragement of crime. +DUMMY/LJ031-0135.wav|Concern that the Vice President might also be a target for assassination prompted the Secret Service agents to urge him to leave the hospital and return to Washington immediately. +DUMMY/LJ030-0217.wav|quote, Mrs. Kennedy had jumped up from the seat and was, it appeared to me, reaching for something coming off the fight rear bumper of the car, +DUMMY/LJ050-0278.wav|the recommendations we have here suggested would greatly advance the security of the office without any impairment of our fundamental liberties. +DUMMY/LJ028-0232.wav|Here they shut themselves up and made light of his siege, having laid in a store of provision for many years in preparation against this attack; +DUMMY/LJ039-0132.wav|and as compared to the average male of his age throughout the civilian, throughout the United States, that he is an excellent shot, end quote. +DUMMY/LJ013-0137.wav|which the various club secretaries identified as the property of their respective clubs. +DUMMY/LJ018-0008.wav|He lived at Hackney, but he never reached it alive. +DUMMY/LJ042-0171.wav|simply expressed, the left and right, and their offspring factions and concerns. Any practical attempt at one alternative +DUMMY/LJ012-0010.wav|both having been recognized by the clergyman who had performed the ceremony, and the assault had been committed to secure the money +DUMMY/LJ002-0195.wav|Besides these, a number had the privilege of a "run on the key," which allowed a prisoner to go into the rules for the day. +DUMMY/LJ002-0086.wav|they were lighted by a double-grated window two feet nine inches by fourteen inches; and in the doors, which were four inches thick, +DUMMY/LJ044-0215.wav|it also was increasing its efforts to tighten the noose around Cuba. +DUMMY/LJ028-0365.wav|one of Alexander's generals, who for a time made it his home, but he was a Greek and cared little for things Babylonian. +DUMMY/LJ048-0140.wav|Agent Lawson's advance planning called for the deployment of almost six hundred members of the Dallas Police Department, +DUMMY/LJ011-0007.wav|and was told by Mr. Fauntleroy that the lady had desired him to sell out, "which I have done," added the fraudulent banker, "and here are the proceeds," +DUMMY/LJ029-0191.wav|He asserted that Dallas had shed its reputation of the twenties as the, quote, Southwest hate capital of Dixie, end quote +DUMMY/LJ004-0187.wav|Already, as early as eighteen eighteen, a prison existed at Bury St. Edmunds which was a model for imitation to others at that time, +DUMMY/LJ046-0054.wav|partly because of the greater speed and comfort of travel and partly because of the greater demands made on the President. +DUMMY/LJ050-0045.wav|and of the Advanced Research Projects Agency of the Department of Defense, +DUMMY/LJ038-0074.wav|and pick up a man named Lee Oswald. When Hill asked why Oswald was wanted, +DUMMY/LJ015-0199.wav|The improved methods of locomotion had put a stop to these depredations. People traveled in company, as a rule; +DUMMY/LJ021-0083.wav|which Americans have demanded since the white man first came to these shores. +DUMMY/LJ031-0227.wav|The doctors traced the course of the bullet through the body and, as information was received from Parkland Hospital, +DUMMY/LJ039-0235.wav|The probability of hitting the President a second time would have been markedly increased if, in fact, he had missed either the first or third shots +DUMMY/LJ010-0214.wav|One expressed pleasure that Oxford improved so much in speaking, and declared that his (Oxford's) speech the last time "was beautiful." +DUMMY/LJ041-0021.wav|In a space for the names of, quote, close friends, end quote, on the ninth grade personal history record, +DUMMY/LJ002-0146.wav|The activity as well as the futility of these courts may be estimated from the statement given by Neild +DUMMY/LJ017-0246.wav|the mutineers took command of the ship, put her about till night-fall, by which time they had scuttled her, got out the boats, and all left the ship. +DUMMY/LJ041-0097.wav|According to Powers, Oswald said, quote, +DUMMY/LJ027-0149.wav|that the phases of development presented by the individual organism would follow, in their main outlines, those phases of development +DUMMY/LJ018-0259.wav|he confessed that his whole life had been a gigantic mistake, and he was ready to make what atonement he could. +DUMMY/LJ002-0078.wav|seven. The press yard was that part set aside for the condemned. +DUMMY/LJ016-0119.wav|Under superior orders all the doors and gates of this block were left open at night, to allow the night watchman to pass freely to all parts. +DUMMY/LJ010-0248.wav|On searching him the pistol was found in his pocket, the barrel still warm; also some loose powder and a bullet. +DUMMY/LJ010-0302.wav|in eighteen oh seven, and when only twenty-two, he succeeded to his father's share in the business. +DUMMY/LJ019-0079.wav|The cells inhabited by prisoners were of very varying dimensions; +DUMMY/LJ016-0131.wav|Nothing more was heard of Krapps. The curious thing in his case was that his offense was a trifling one; +DUMMY/LJ001-0026.wav|On the whole the type of this book may be considered the ne-plus-ultra of Gothic type, +DUMMY/LJ034-0033.wav|During the afternoon of November twenty-two, +DUMMY/LJ033-0157.wav|Furthermore, it was consistent with the bag having contained a heavy or bulky object when he handled it since a light object is usually held by the fingers. +DUMMY/LJ006-0152.wav|One turnkey had been so employed several hours daily for nearly eleven years. +DUMMY/LJ033-0061.wav|I then went to the garage to see whether the rifle was there and I saw that the blanket was still there and I said Thank God, end quote. +DUMMY/LJ028-0202.wav|"In that night," so the story ends, "Belshazzar, the Chaldean King, was slain." +DUMMY/LJ042-0110.wav|stated that Mrs. Oswald told her everybody in Russia, quote, hated him, end quote. +DUMMY/LJ017-0228.wav|till every feature was obliterated, and then, still living, flung him into the sea. +DUMMY/LJ043-0166.wav|Possibly he might have wanted to be caught, and wanted his involvement made clear if he was in fact apprehended. +DUMMY/LJ008-0232.wav|a sound as if the ocean had suddenly broken through some ancient boundary, against which its ever restless billows had for ages battered; +DUMMY/LJ028-0439.wav|for none of the many building inscriptions from Nebuchadnezzar mentions them. +DUMMY/LJ011-0070.wav|Afterwards, according to the common rumor, he went abroad and lived there for many years; but the story is not only wholly unsubstantiated, +DUMMY/LJ022-0010.wav|At different points on the coast where I often visit they build great seagoing ships. +DUMMY/LJ008-0224.wav|With them were younger men and women: +DUMMY/LJ042-0153.wav|This was a fairly coherent description of life in that country, basically centered around the radio and television factory in which he worked. +DUMMY/LJ019-0265.wav|In dietaries, again, the same wide diversity of practice obtained. +DUMMY/LJ006-0066.wav|In the middle yard it was still worse. +DUMMY/LJ026-0126.wav|as a preliminary to the real processes of nutrition. +DUMMY/LJ007-0176.wav|is indeed a subject which cannot but impress every considerate mind with humiliation and sorrow. +DUMMY/LJ049-0024.wav|has developed a vehicle for the better protection of the President. +DUMMY/LJ011-0293.wav|It was a cell five feet by three, within another, the intervening being filled with rammed earth to deaden the sound. +DUMMY/LJ036-0014.wav|Oswald's Movements After Leaving Depository Building +DUMMY/LJ019-0253.wav|Even where cells had been built +DUMMY/LJ019-0364.wav|His order would at the same time specify some neighboring and more satisfactory prison which the local authority would be compelled to utilize instead, +DUMMY/LJ003-0022.wav|This steward was practically supreme. +DUMMY/LJ050-0063.wav|and who have been involved in bombing or bomb-making or whose past conduct indicates tendencies toward violence, and (d) +DUMMY/LJ016-0435.wav|"Two fresh men! May I speak to them? Yes! I must caution you," he went on to the warders, "not to go to sleep, or I shall be off through that little hole," +DUMMY/LJ004-0206.wav|Blanket weaving and cloth spinning was carried on prosperously, +DUMMY/LJ014-0122.wav|A great number of distinguished people assembled as usual at the Old Bailey on the day of trial. +DUMMY/LJ017-0180.wav|vomiting, violent retching, purging, such as are visible in cholera, and all dated from the time when she knew a young man named Dixon, +DUMMY/LJ046-0136.wav|Before the assassination of President Kennedy, +DUMMY/LJ028-0453.wav|The northeastern section may now be traced for a distance of less than three miles, and the southwestern "for more than a mile, +DUMMY/LJ026-0163.wav|This is true also of green plants in darkness and even in the light of all parts of green plants except the chlorophyll-bodies. +DUMMY/LJ019-0196.wav|free from any other interference than that of the inspectors on the part of Government. +DUMMY/LJ019-0128.wav|I have lingered too long perhaps over Mr. Pearson's proposal, but some reference was indispensable to a scheme +DUMMY/LJ047-0099.wav|Quigley recalled that Oswald was receptive when questioned about his general background +DUMMY/LJ004-0181.wav|but as no similar variation occurred in the prisoner's appetite, his ration was somewhat precarious. +DUMMY/LJ037-0256.wav|Mrs. Markham and Barbara Davis thought that the jacket worn by the slayer of Tippit was darker than the jacket found by Westbrook. +DUMMY/LJ027-0035.wav|In the second place, it must be noted that adaptations to similar conditions may result in superficial resemblances. +DUMMY/LJ018-0360.wav|then to divert suspicion from herself by throwing it on a confectioner, whose sweetmeats she bought, +DUMMY/LJ022-0128.wav|I call upon my fellow citizens everywhere to cooperate with me +DUMMY/LJ008-0168.wav|Among the rest was a woman with an infant at the breast. +DUMMY/LJ028-0222.wav|My many troops marched peacefully into Babylon. +DUMMY/LJ022-0156.wav|The operating companies in the gas and electric utility field are by and large in good condition, but +DUMMY/LJ015-0209.wav|Burgess detailed the whole system of transmission. +DUMMY/LJ016-0178.wav|had no official status, and was merely an operator selected by the Corporation, and who, on the strength of it, +DUMMY/LJ050-0184.wav|has hitherto been handled on an informal basis. +DUMMY/LJ004-0049.wav|Mr. Gurney, Mr. Fry, Messrs. Forster, and Mr. T. F. Buxton, the coadjutor of Wilberforce in the great anti-slavery struggle. +DUMMY/LJ018-0222.wav|obtained a mortgage of seven thousand pounds. +DUMMY/LJ013-0224.wav|The attempt was to have been made by opening a vein and allowing himself to bleed to death. +DUMMY/LJ033-0016.wav|Oswald traveled between Dallas and Irving on weekends in a car driven by a neighbor of the Paines, Buell Wesley Frazier, +DUMMY/LJ043-0159.wav|Answer: Yes. +DUMMY/LJ036-0119.wav|Whaley displayed a trip manifest which showed a twelve o'clock trip from Travis Hotel to the Continental bus station, +DUMMY/LJ032-0162.wav|Stombaugh testified that the colors, shades, and twist of the fibers found in the tuft on the rifle matched those in Oswald's shirt. +DUMMY/LJ011-0098.wav|He soon, however, became deeply involved in Stock Exchange speculations, +DUMMY/LJ041-0111.wav|for possessing an unauthorized pistol with which he had accidentally shot himself. +DUMMY/LJ050-0139.wav|to serve as liaison officers with law enforcement and intelligence agencies. On the basis of the Department's review during the past several months, +DUMMY/LJ032-0033.wav|Oswald's writing on these and other documents was identified by comparing the writing and printing on the documents in question +DUMMY/LJ012-0072.wav|He was indicted on eight charges, two only of which were substantiated, but on each of them he received a sentence of seven years' transportation. +DUMMY/LJ013-0160.wav|who, on going down early, was surprised to find the dining-room in a state of utter confusion; +DUMMY/LJ035-0163.wav|After leaving Mrs. Reid in the front office, Oswald could have gone down the stairs and out the front door by twelve:thirty-three p.m. +DUMMY/LJ044-0169.wav|In September he began to review Spanish. +DUMMY/LJ032-0120.wav|Day continued to examine the rifle with the magnifying glass, looking for possible fingerprints. +DUMMY/LJ041-0047.wav|approximately three years before. Robert Oswald had given his Marine Corps manual to his brother Lee, +DUMMY/LJ019-0048.wav|A new and still imperfectly understood science had arisen, +DUMMY/LJ001-0146.wav|which requires the constant exercise of judgment and taste on the part of the printer. +DUMMY/LJ028-0481.wav|Nabopolassar, the father, my begetter, built Imgur-Bel, the great wall of Babylon, +DUMMY/LJ004-0219.wav|The diet was now ample. It consisted of a pound and a half of bread per diem; +DUMMY/LJ005-0163.wav|One great impediment to wide amelioration was that a vast number of small jails lay out of reach of the law. +DUMMY/LJ006-0200.wav|The newsman had free access to the prison; he passed in unsearched and unexamined, and, unaccompanied by an officer, +DUMMY/LJ027-0107.wav|rudiments of eyes in cave fishes, hind limbs beneath the skin of whales, the vermiform appendix in man, +DUMMY/LJ017-0238.wav|Taffir, the second mate, agreed, but constantly went in fear of his life for the remainder of the voyage; +DUMMY/LJ023-0136.wav|In many states judges must retire at the age of seventy. +DUMMY/LJ039-0094.wav|After viewing photographs depicting the alignment of Elm Street in relation to the Texas School Book Depository Building, +DUMMY/LJ048-0257.wav|and that their conduct the night before did not impede their actions on duty +DUMMY/LJ034-0059.wav|Since other identifiable prints were developed on the cartons, the Commission requested that they be compared with the prints of the twelve warehouse employees +DUMMY/LJ044-0173.wav|While Marina Oswald knew of her husband's plan to go to Mexico and thence to Cuba if possible, Mrs. Paine was told that Oswald was going to Houston +DUMMY/LJ018-0118.wav|a system which had long been in practice of defrauding the Exchequer of vast sums by means of forged stamps. +DUMMY/LJ002-0082.wav|Beyond the press yard were three stories, condemned cells, fifteen in all, with vaulted ceilings nine feet high to the crown of the arch. +DUMMY/LJ025-0114.wav|founded the modern science of histology or that branch of anatomy which deals with the ultimate visible structure of organisms as revealed by the microscope, +DUMMY/LJ037-0239.wav|She confirmed this interview by a sworn affidavit. +DUMMY/LJ021-0138.wav|with justice for all those willing to use these agencies whenever their voluntary bargaining fails to produce a necessary agreement. +DUMMY/LJ048-0113.wav|The Commission believes, however, +DUMMY/LJ014-0153.wav|After this, until the day of execution, she recovered her spirits, and displayed reckless effrontery, +DUMMY/LJ027-0172.wav|The perennibranch passes through the fish stage to the perennibranch amphibian. +DUMMY/LJ021-0131.wav|is not fully supporting the recovery effort of his government. +DUMMY/LJ037-0243.wav|Westbrook walked through the parking lot behind the service station and found a light-colored jacket lying under the rear of one of the cars. +DUMMY/LJ028-0250.wav|Such, then, were the circumstances of the first taking of Babylon. +DUMMY/LJ036-0203.wav|The dispatcher ordered Tippit to be, quote, at large for any emergency that comes in, end quote. +DUMMY/LJ022-0110.wav|The key men for the major responsibilities of this great task already have been selected. +DUMMY/LJ038-0039.wav|Three other officers, moving toward the scuffle, grabbed Oswald from the front, rear and side. +DUMMY/LJ013-0144.wav|one of the owners had met her at Deal, but the captain had already absconded with the packages of precious stones in his pocket. +DUMMY/LJ035-0151.wav|He was walking into the office from the back hallway, +DUMMY/LJ042-0196.wav|Oswald's decided rejection of both capitalism and communism +DUMMY/LJ050-0250.wav|The Secret Service will be better able to plan its own long-range personnel requirements if it knows with reasonable certainty +DUMMY/LJ012-0048.wav|and this, with the danger arising from an extensive robbery of watches in Cheapside, in which Ikey was implicated as a receiver, +DUMMY/LJ014-0114.wav|She had been to brokers to negotiate the sale of certain foreign railway stock, with which they had been warned from London not to deal, +DUMMY/LJ018-0223.wav|In eighteen fifty-six the father died. +DUMMY/LJ048-0062.wav|to have made more vigorous efforts to locate Oswald's roominghouse address in Dallas and to interview him regarding these unresolved matters. +DUMMY/LJ010-0102.wav|The Guards, headed by Lord Adolphus Fitz Clarence, now reinforced the police, and the conspirators gave way. +DUMMY/LJ016-0353.wav|Almost absolute silence prevailed until the great bell began to toll its deep note, and broke the stillness with its regular and monotonous clangour, +DUMMY/LJ015-0143.wav|Later on he became registrar, and in this more independent position +DUMMY/LJ002-0163.wav|a market porter, was arrested and committed at the suit of a publican +DUMMY/LJ044-0057.wav|extensive investigation was not able to connect Oswald with that address, although it did develop the fact +DUMMY/LJ040-0101.wav|When she worked during the school year, Lee had to leave an empty house in the morning, +DUMMY/LJ050-0189.wav|that written instructions might come into the hands of local newspapers, to the prejudice of the precautions described. +DUMMY/LJ003-0328.wav|Had they been accepted in their entirety, little fault could in future have been found with the managers of Newgate. +DUMMY/LJ030-0143.wav|The exact time of the assassination was fixed by the testimony of four witnesses. +DUMMY/LJ043-0102.wav|It is possible that his immediate supervisor noticed the newspaper at that time because his attention had otherwise been drawn more directly to Oswald. +DUMMY/LJ035-0061.wav|In fact, he didn't change his expression one bit, end quote. +DUMMY/LJ042-0025.wav|His idea that he was to find, quote, freedom, end quote, in the Soviet Union was to be rudely shattered. +DUMMY/LJ036-0124.wav|Whaley testified that he did not keep an accurate time record of his trips but recorded them by the quarter hour, +DUMMY/LJ038-0252.wav|Marina Oswald stated that- when she asked her husband what be had done with the rifle, +DUMMY/LJ031-0074.wav|quote, could have easily been hidden in the blood and hair, end quote, +DUMMY/LJ030-0062.wav|The President had frequently stated that he did not want agents to ride on these steps during a motorcade except when necessary. +DUMMY/LJ012-0127.wav|The gold-dust robbery of eighteen thirty-nine, the first of its kind, was cleverly and carefully planned with the assistance of a dishonest employee. +DUMMY/LJ036-0113.wav|Actually Oswald was under Number three. +DUMMY/LJ029-0158.wav|The following day, the newspaper reported that the Presidential party +DUMMY/LJ036-0216.wav|Tippit got out and started to walk around the front of the car +DUMMY/LJ049-0052.wav|In response to inquiry by the Commission regarding the instructions to agents in a motorcade +DUMMY/LJ028-0360.wav|During the reign of Xerxes again the city rebelled, and in four eighty-four B.C. he captured it, and completely demolished its defenses. +DUMMY/LJ005-0145.wav|Great want of uniformity in treatment in the various prisons was still noticeable +DUMMY/LJ014-0106.wav|The lime had done its work so rapidly that the features would have been indistinguishable but for the prominent chin and a set of false teeth. +DUMMY/LJ013-0175.wav|Upstairs in the bedroom a rouleaux box for sovereigns had been broken open, +DUMMY/LJ040-0142.wav|was disturbed about disrobing in front of them, taking showers with them etc., end quote. +DUMMY/LJ027-0135.wav|which, briefly stated, is that individual development (ontogeny) recapitulates ancestral history (phylogeny). +DUMMY/LJ016-0354.wav|and the ordinary, in a voice trembling with emotion, read the burial service aloud. +DUMMY/LJ018-0291.wav|Then he explained to the bank manager +DUMMY/LJ033-0140.wav|Three cartons had been placed at the window apparently to act as a gun rest and a fourth carton was placed behind those at the window. +DUMMY/LJ015-0171.wav|but finding he could not escape, he said he would resign his post, and leaving the boardroom, disappeared. +DUMMY/LJ019-0142.wav|now seldom contained more than ten or a dozen each. Some sort of decorum was maintained among the occupants in the day-time. +DUMMY/LJ012-0104.wav|The thieves were satisfied with the diamonds; they broke open other cases containing gold watches and plate, but abstracted nothing. +DUMMY/LJ009-0111.wav|the condemned returning to the cells: the forger carried by turnkeys; the youth sobbing aloud convulsively, as a passionate child; +DUMMY/LJ045-0030.wav|when his wife asked Oswald not to come to Irving. +DUMMY/LJ001-0011.wav|it is of the first importance that the letter used should be fine in form; +DUMMY/LJ034-0195.wav|based on other evidence, to have fired the shots from the window. +DUMMY/LJ034-0183.wav|Boxes and cases were stacked behind him. +DUMMY/LJ021-0087.wav|We have the right to expect that this driving power will be given patriotically and whole-heartedly to our nation. +DUMMY/LJ040-0091.wav|but he returned to high school in January of nineteen forty-nine, where he stayed until three days before he was scheduled to graduate, +DUMMY/LJ047-0005.wav|Oswald was known to other Federal agencies with which the Secret Service maintained intelligence liaison. +DUMMY/LJ046-0109.wav|A basic element of Presidential protection +DUMMY/LJ048-0072.wav|the Commission does not believe that the liaison between the FBI and the Secret Service prior to the assassination was as effective as it should have been. +DUMMY/LJ049-0035.wav|However, such access to the President was interfered with both by the metal bar some fifteen inches above the back of the front seat +DUMMY/LJ037-0050.wav|Her description and that of other eyewitnesses led to the police broadcast at one:twenty-two p.m. +DUMMY/LJ030-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ014-0191.wav|Marley ran along the street, down into Cannon Row +DUMMY/LJ037-0017.wav|The man cut across the yard through some bushes, passed within twelve feet of Scoggins, and ran south on Patton. +DUMMY/LJ028-0229.wav|The Babylonians, encamped without their walls, awaited his coming. +DUMMY/LJ018-0307.wav|Through Noyes the rest of the conspirators were eventually apprehended. Very little if any of the ill-gotten proceeds, however, was ever recovered. +DUMMY/LJ021-0167.wav|to end our present unemployment as soon as we can and then to take wise measures against its return. +DUMMY/LJ003-0139.wav|and I was obliged afterwards to let him have five shillings to pay his share, +DUMMY/LJ015-0074.wav|His immediate chief was a Mr. Fasson, upon whose confidence he gained so rapidly, through his activity, industry, and engaging manners, +DUMMY/LJ028-0436.wav|The Germans maintain that it is the ruin of the Tower of Babel. +DUMMY/LJ046-0135.wav|PRS received items in eight thousand, seven hundred nine cases. +DUMMY/LJ033-0184.wav|had different characteristics from both the actual bag and the sample taken on November twenty-two. +DUMMY/LJ015-0071.wav|He now married, although his salary was only a pound a week; but he soon got on. +DUMMY/LJ025-0056.wav|That the semi-fluid contents (which we now term protoplasm) of the cells of certain plants, +DUMMY/LJ019-0103.wav|The latter had reference more especially to a proposal emanating from Mr. Charles Pearson himself. +DUMMY/LJ013-0130.wav|where Ashley produced four silver spoons, and begged that his initials might be engraved upon them. +DUMMY/LJ035-0115.wav|Oswald's use of the stairway is consistent with the testimony of other employees in the building. +DUMMY/LJ036-0099.wav|that he recognized Oswald from a newspaper photograph as a man whom he had driven to the Oak Cliff area the day before. +DUMMY/LJ043-0182.wav|In any event, the Walker incident indicates that in spite of the belief among those who knew him that he was apparently not dangerous, +DUMMY/LJ038-0255.wav|Firearms identification. +DUMMY/LJ007-0062.wav|Latterly his ministrations to the condemned had been restricted to a visit on Sunday afternoons, and occasionally about once a fortnight on a week-day. +DUMMY/LJ031-0153.wav|The telephone line to Washington was kept open throughout the remainder of the stay at the hospital. +DUMMY/LJ045-0152.wav|The events of that evening can best be appreciated through Marina Oswald's testimony +DUMMY/LJ040-0159.wav|who suffers under the impact of really existing emotional isolation +DUMMY/LJ033-0086.wav|Mrs. Randle estimated that the package was approximately twenty-eight inches long and about eight inches wide. +DUMMY/LJ011-0230.wav|Mr. E. G. Wakefield took part in the scheme for the colonization of North Australia, and for some years resided in that colony. +DUMMY/LJ028-0509.wav|Nebuchadnezzar speaks of great bronze gates and of images of bronze, but none have been discovered. +DUMMY/LJ001-0145.wav|No definite rules, however, except the avoidance of "rivers" and excess of white, can be given for the spacing, +DUMMY/LJ037-0244.wav|Westbrook identified Commission Exhibit Number one sixty-two as the light-colored jacket which he discovered underneath the automobile. +DUMMY/LJ022-0037.wav|that are filled by the vote of all of the voters -- the President and the Vice-President? +DUMMY/LJ006-0126.wav|had been known to employ threats, declaring that he was often consulted by the governor as to a prisoner's character, +DUMMY/LJ028-0121.wav|A thing which no king before had ever done: +DUMMY/LJ002-0288.wav|In one room Mr. Neild found a woman ill of a flux shut up with three men; +DUMMY/LJ034-0189.wav|Fischer explained that he did not mean by the earlier statement that the man was blond, but rather that his hair was not black. +DUMMY/LJ019-0143.wav|Drinking and gaming, +DUMMY/LJ006-0089.wav|to be pitchforked by chance anywhere, into any sort of company, +DUMMY/LJ028-0010.wav|Now and then she reaches down to graze the thorny argool along the way. +DUMMY/LJ041-0073.wav|While Oswald did not seem to object to authority in the abstract, he did think that he should be the one to exercise it. +DUMMY/LJ031-0187.wav|Federal Judge Sarah T. Hughes hastened to the plane to administer the oath. +DUMMY/LJ028-0456.wav|The small portions of it which have been excavated suffice to show its construction. +DUMMY/LJ042-0098.wav|The subsidy, apparently similar to those sometimes given to foreigners allowed to remain in the Soviet Union, together with his salary, +DUMMY/LJ016-0102.wav|By evening he manufactured a good long length, +DUMMY/LJ048-0074.wav|To insure adequate and effective liaison arrangements, +DUMMY/LJ016-0229.wav|He kept no record of them, and when asked questions, referred to the officers of the jail. +DUMMY/LJ021-0028.wav|It was in this spirit thus described by Secretary Root +DUMMY/LJ016-0168.wav|A laudable desire to invest executions with more and more solemnity and decorum gained ground as they became more rare. +DUMMY/LJ009-0125.wav|could any one, not disgusted with the religious ceremonials of Newgate, witness this scene without emotion? +DUMMY/LJ030-0124.wav|When a teenage boy ran toward the rear of the President's car, Ready left the running board to chase the boy back into the crowd. +DUMMY/LJ039-0011.wav|toward any official of the United States. +DUMMY/LJ032-0150.wav|In the opinion of these experts, it was not possible to estimate the time which elapsed between the placing of the print on the rifle and the date of the lift. +DUMMY/LJ033-0064.wav|Mrs. Paine pointed out that most of the Oswalds' possessions were in the garage. +DUMMY/LJ002-0301.wav|They were made to wash and swab the ward, or they were shut out from the ward fireplace, and forbidden to pass a chalked line drawn on the floor, +DUMMY/LJ030-0012.wav|In Houston, as elsewhere during the trip, the crowds showed much interest in Mrs. Kennedy. +DUMMY/LJ038-0120.wav|in Oswald's billfold. On November twenty-two and twenty-three, Oswald refused to tell Fritz why this card was in his possession, +DUMMY/LJ031-0026.wav|Dr. Fouad A. Bashour. +DUMMY/LJ019-0102.wav|which was promptly carried, with the additional instruction to the committee to suggest any improvements. +DUMMY/LJ014-0162.wav|But it may be mentioned that the concourse was so enormous that it drew down the well-merited and trenchant disapproval of Charles Dickens, +DUMMY/LJ048-0270.wav|The regulations provide further that, quote, violation or slight disregard, end quote, of these provisions, quote, +DUMMY/LJ016-0210.wav|Meanwhile the convener or sheriff was in despair, expecting that, failing the executioner, he would have to do the job himself. +DUMMY/LJ042-0141.wav|She added that while he helped her as he had done before, he became more of a recluse, that, quote, +DUMMY/LJ040-0032.wav|He accused his wife of preferring others to himself and told her to return to the Soviet Union without him but without a divorce. +DUMMY/LJ028-0426.wav|For fifteen years Dr. Koldewey and his assistants, with a force of two hundred native workmen, have labored there winter and summer. +DUMMY/LJ036-0159.wav|On the other hand, Whaley identified Commission Exhibit Number one fifty (the shirt taken from Oswald upon arrest) as the shirt his passenger was wearing. +DUMMY/LJ032-0080.wav|were a Selective Service notice of classification and a Marine certificate of service in the name of Alek James Hidell. +DUMMY/LJ029-0157.wav|The selection of the Trade Mart as the possible site for the luncheon first appeared in the Dallas Times-Herald on November fifteen, nineteen sixty-three. +DUMMY/LJ014-0137.wav|When the judge assumed the black cap +DUMMY/LJ046-0199.wav|The general files of PRS were not indexed by geographic location and were of little use in preparing for a Presidential visit to a specific locality. +DUMMY/LJ004-0021.wav|The neglect of prison reform in those days was not to be visited upon the legislature. +DUMMY/LJ024-0030.wav|The plan has met opposition only so far as the Supreme Court of the United States itself is concerned. +DUMMY/LJ033-0113.wav|The bag found on the sixth floor was eighty-eight inches long. +DUMMY/LJ022-0095.wav|This Division will have the duty of coordinating the purchases of materials and supplies +DUMMY/LJ001-0126.wav|Most of Jenson's letters are designed within a square, +DUMMY/LJ030-0083.wav|to a walking pace or stopped, +DUMMY/LJ006-0207.wav|One of these, published by Stockdale, the inspectors styled "a book of the most disgusting nature." +DUMMY/LJ045-0145.wav|Those arguments, however, were not persuasive to Marina Oswald, +DUMMY/LJ031-0208.wav|The hospital received the President's body for autopsy at approximately seven:thirty-five p.m. +DUMMY/LJ027-0090.wav|modified more and more through successive generations by the necessities of different modes of life. +DUMMY/LJ012-0200.wav|Bishop got weary of the dangers and fatigues of exhumation, and proposed to Williams that instead of disinterring they should murder their subjects. +DUMMY/LJ017-0012.wav|The well-known and lethal aqua Toffania, +DUMMY/LJ040-0232.wav|Few social agencies even in New York were equipped to provide the kind of intensive treatment that he needed, +DUMMY/LJ018-0203.wav|Proceeding cautiously down the stairs, they found that the flooring at the bottom had been taken up. +DUMMY/LJ016-0263.wav|Few, if any, showed any feeling of terror, none were impressed with the solemnity, or realized the warning which the sight conveyed. +DUMMY/LJ001-0104.wav|Italy is contentedly stagnant. +DUMMY/LJ027-0048.wav|On the other hand are found structures which are perfectly homologous and yet in no way analogous: +DUMMY/LJ011-0074.wav|There were two notable capital convictions for forgery in eighteen twenty-eight. +DUMMY/LJ035-0187.wav|he also directed that the back door be sealed off. +DUMMY/LJ009-0180.wav|the idea being to intensify the dread of capital punishment. That such dread was not universal or deep-seated may be gathered from the fact +DUMMY/LJ036-0074.wav|If Oswald left the Depository Building at twelve:thirty-three p.m., +DUMMY/LJ038-0087.wav|in an apparent effort to determine, by means of a scientific test, whether Oswald had recently fired a weapon. +DUMMY/LJ014-0108.wav|It was soon ascertained that the wife had gone off in a cab with a quantity of luggage. +DUMMY/LJ028-0090.wav|He was fond of restoring the ruined temples of the old Babylonian cities, +DUMMY/LJ017-0179.wav|In all cases the symptoms were much the same, +DUMMY/LJ027-0154.wav|the process of successive additions which had previously taken place in a remote series of extinct species. +DUMMY/LJ024-0109.wav|Now they are making a last stand. +DUMMY/LJ028-0402.wav|Hillah might be called a child of Babylon, for it is almost entirely constructed with Nebuchadnezzar's bricks. +DUMMY/LJ036-0193.wav|Description of Shooting +DUMMY/LJ015-0044.wav|with money enough to retrieve the position of the bank. But that passed from bad to worse; +DUMMY/LJ048-0133.wav|Constant reevaluation of procedures, with attention to special problems and the development of instructions specific to particular trips +DUMMY/LJ011-0067.wav|A report was, moreover, widely circulated, and the impression long prevailed, that he actually escaped death. +DUMMY/LJ031-0028.wav|Lawson jumped from the lead car and rushed into the emergency entrance, where he was met by hospital staff members wheeling stretchers out to the automobile. +DUMMY/LJ050-0198.wav|the studies indicate that there is some utility in attempting to designate certain buildings as involving a higher risk than others. +DUMMY/LJ021-0193.wav|All of these cries have been heard before. +DUMMY/LJ029-0148.wav|for controlling the crowds and traffic, watching the overpasses, and providing motorcycle escort. +DUMMY/LJ021-0031.wav|Some banks could not be saved but the great majority of them, either through their own resources or with government aid, +DUMMY/LJ039-0180.wav|As has been shown in chapter three, if the three shots were fired within a period of from four point eight to five point six seconds, +DUMMY/LJ025-0153.wav|and inquire whether certain differences of a more occult character than those imagined to exist by Cuvier, +DUMMY/LJ009-0067.wav|The service proceeds. At last the burial service is reached. +DUMMY/LJ038-0126.wav|At the last interrogation in November Oswald admitted to Postal Inspector Holmes that he had rented post office box two nine one five, Dallas, +DUMMY/LJ012-0115.wav|was refused, at the instigation of the police. A few days later a frequent guest at the tavern arrived, and had this same room allotted to him. +DUMMY/LJ012-0171.wav|Every door had been closed against him, every hope of future support blasted. +DUMMY/LJ004-0248.wav|But this boon of toil was denied to all but a very limited number. +DUMMY/LJ033-0045.wav|At approximately nine p.m., after the children had been put to bed, Mrs. Paine, according to her testimony before the Commission, quote, +DUMMY/LJ042-0232.wav|I have never even know a communist, outside of the ones in the USSR but you can't help that, end quote. +DUMMY/LJ013-0085.wav|A person, Elder of course, who personated Oxenford, attended at the bank to complete the transfer and sell the stock. +DUMMY/LJ024-0097.wav|has ever been ratified within anything like a reasonable time. +DUMMY/LJ014-0057.wav|London did not escape the contagion, and prominent among the detestable crimes of the period stands that of the Mannings at Bermondsey. +DUMMY/LJ031-0057.wav|While Dr. Perry was performing the tracheotomy, Drs. Carrico and Ronald Jones made cutdowns on the President's right leg and left arm, respectively, +DUMMY/LJ006-0134.wav|The governor himself admitted that a prisoner of weak intellect who had been severely beaten and much injured by a wardsman did not dare complain +DUMMY/LJ019-0351.wav|Nor were these the only inducements offered. Where local authorities were indisposed to set their prisons in order, +DUMMY/LJ038-0184.wav|This is the key to the mailbox which is located in the main post office in the city on Ervay Street. +DUMMY/LJ049-0078.wav|have properly taken the initiative in reexamining major aspects of Presidential protection. +DUMMY/LJ025-0002.wav|Edited by Francis Rolt-Wheeler. Biology. Chapter seven. Organic Functions. Part One. +DUMMY/LJ018-0156.wav|after using much intemperate language, he left the place in a huff. +DUMMY/LJ005-0107.wav|In that prison, with its cells and yards arranged within the shell of an ancient abbey chapel, +DUMMY/LJ019-0050.wav|One school were strongly in favor of the continuous separation of prisoners, +DUMMY/LJ002-0328.wav|Failing any of these methods, seeing that straw was forbidden for fear of fire, they had to be satisfied with a couple of the rugs provided by the city +DUMMY/LJ010-0216.wav|and it is said that he is a military officer, but his name has not yet transpired. +DUMMY/LJ043-0026.wav|Oswald submitted to the inevitable, presumably because he was, quote, small, you know, and he was rather a puny individual, end quote, +DUMMY/LJ016-0067.wav|he tramped across London Bridge to Wandsworth, where he refreshed himself with a pint of strong ale, the first sustenance he had taken since his escape, +DUMMY/LJ033-0024.wav|Oswald replied, quote, I'm going home to get some curtain rods to put in an apartment, end quote. +DUMMY/LJ007-0011.wav|But it was already plain that they constituted an independent authority within the jails; they were frequently in conflict with the chaplain, +DUMMY/LJ007-0003.wav|and Mrs. Fry with her colleagues still labored assiduously in Newgate, devoting themselves mainly to the female prison, +DUMMY/LJ022-0034.wav|This duty of seeing the country in a long-range perspective +DUMMY/LJ037-0063.wav|A review of the complete transcript has satisfied the Commission +DUMMY/LJ028-0114.wav|and I, with mortar and bricks, built a third great moat-wall, and joined it and united it closely with the moat-walls of my father. +DUMMY/LJ018-0106.wav|He was a German named Kerp, +DUMMY/LJ050-0220.wav|and requests for additional personnel were not made because of the studies then being conducted. +DUMMY/LJ042-0107.wav|Oswald apparently continued to have personal difficulties while he was in Minsk. +DUMMY/LJ045-0169.wav|That this was better because while he was living alone and I stayed with Ruth, we were spending less money. And I told him to buy me a washing machine, because two children +DUMMY/LJ038-0187.wav|I paid for the box last month so don't worry about it. +DUMMY/LJ012-0190.wav|Widespread horror and indignation was evoked throughout the kingdom by the discovery of the series +DUMMY/LJ035-0194.wav|Approximately fifteen men worked in the warehouse and Truly noticed that Oswald was not among those being questioned. +DUMMY/LJ002-0334.wav|Again, there was no regular allowance of fuel. Coals were purchased out of the garnish money and the charitable fund; +DUMMY/LJ037-0145.wav|Four cartridge cases were found in the shrubbery on the corner of tenth and Patton by three of the eyewitnesses -- Domingo Benavides, +DUMMY/LJ014-0258.wav|Implicit confidence appears to have been placed in Watts, who was the son of an old and trusted employee, and, moreover, a young man of plausible address. +DUMMY/LJ016-0342.wav|The first private execution under the new law took place within the precincts of Maidstone Jail. +DUMMY/LJ024-0060.wav|Normally every President appoints a large number of district and circuit court judges and a few members of the Supreme Court. +DUMMY/LJ028-0422.wav|Among the later visitors to Babylon was the great Niebuhr. +DUMMY/LJ031-0102.wav|Governor Connally was originally seen by Dr. Carrico and Dr. Richard Dulany. +DUMMY/LJ026-0048.wav|movement, irritability (nervous activity) and reproduction. +DUMMY/LJ011-0188.wav|They set out, but at Leeds Wakefield found himself called suddenly to Paris; +DUMMY/LJ021-0010.wav|These governmental and industrial developments hold promise of new achievements for the nation. +DUMMY/LJ007-0133.wav|Valuable space was cumbered with many long and winding passages, numerous staircases, and unnecessarily thick and cumbrous inner walls. +DUMMY/LJ014-0287.wav|He had been led to suppose that twelve months' imprisonment was the utmost the law could inflict, and he broke down utterly under the unexpected blow. +DUMMY/LJ009-0068.wav|The youth alone is able to read, but from long want of practice he is at a loss to find the place in his prayer-book. +DUMMY/LJ005-0218.wav|The chapel still continued incommodious and insufficient +DUMMY/LJ047-0125.wav|In other words, he committed no violation of the law by telling us something that wasn't true, and unless this required further investigation at that time, +DUMMY/LJ043-0152.wav|She said that she kept the note to use against him, quote, if something like that should be repeated again, end quote. +DUMMY/LJ021-0048.wav|the careful checking of new securities under the Securities Act +DUMMY/LJ014-0125.wav|Mrs. Manning was not without personal charms; +DUMMY/LJ018-0343.wav|In this case the tie was unsanctified, but it was not more inconvenient than that which urged Greenacre to a similar crime. +DUMMY/LJ044-0233.wav|Oswald was carrying only thirteen dollars, eighty-seven cents at the time of his arrest, although he had left, apparently by design, +DUMMY/LJ028-0060.wav|In six oh five, +DUMMY/LJ016-0024.wav|A more daring and skilful escape was effected in eighteen thirty-six by the chimney-sweep Henry Williams, +DUMMY/LJ030-0086.wav|Vice-Presidential car. +DUMMY/LJ016-0173.wav|was not always to be trusted to do his fell work efficiently. +DUMMY/LJ014-0179.wav|a working jeweler, shopman to a Mr. Berry of Parliament Street. It was Cope's duty to stay in the shop till the last, close the shutters, +DUMMY/LJ005-0211.wav|the jail consisted of six cells, frequently so damp that the moisture trickled down the walls; there was not space for air or exercise, +DUMMY/LJ014-0123.wav|The Mannings were arraigned together; the husband standing at one of the front corners of the dock, his wife at the other end. +DUMMY/LJ009-0281.wav|Calcraft was forced to return, and he once more pushed Bousfield off, +DUMMY/LJ026-0088.wav|In short, the green plant is able by cooperation with sunlight to use simple raw materials (carbon dioxide, water, oxygen, etc.) +DUMMY/LJ021-0187.wav|It is perhaps not strange that the conservative British press has told us with pardonable irony +DUMMY/LJ021-0002.wav|September thirty, nineteen thirty-four. +DUMMY/LJ041-0114.wav|three months prior to his regularly scheduled separation date, ostensibly to care for his mother who had been injured in an accident at her work. +DUMMY/LJ036-0109.wav|he told them what he thought about them. They knew what they were doing and they were trying to railroad him and he wanted his lawyer. +DUMMY/LJ047-0214.wav|End quote. Mr. Bouck pointed out, however, that he had no reason to believe that any one Federal agency had access to all this information, +DUMMY/LJ016-0222.wav|It may be added that of late years the clothes in which a convict has suffered are invariably burnt. +DUMMY/LJ013-0257.wav|They were seen together on Barnes Common, and again in Putney Park Lane, where they were talking loud and angrily. +DUMMY/LJ008-0115.wav|After the yeoman had requested him to stand up, 'he pinioned him,' as the Newgate phrase is +DUMMY/LJ037-0128.wav|were on the lot at the time, and they saw a white male with a revolver in his hands running south on Patton. +DUMMY/LJ002-0153.wav|weighted by treble the amount of costs. +DUMMY/LJ005-0232.wav|Idleness was still the general rule for all prisoners in Newgate, in defiance of the law. +DUMMY/LJ016-0163.wav|He might have saved himself at any moment by merely extending an arm; but he lay there patiently till death supervened. +DUMMY/LJ006-0075.wav|He saw certain rooms fill up, and yet took no steps to open others that were locked up and empty. +DUMMY/LJ029-0178.wav|reflected the feeling in the community toward the forthcoming Presidential visit. +DUMMY/LJ007-0144.wav|and he can almost invariably procure the luxuries of his class of life, beer and tobacco, in abundance. +DUMMY/LJ030-0010.wav|spoke at the Rice University Stadium, and attended a dinner in honor of U.S. Representative Albert Thomas. +DUMMY/LJ029-0146.wav|The planning for the motorcade also included advance preparations for security arrangements along the route. +DUMMY/LJ044-0088.wav|Fair Play for Cuba was, quote, not at all Communist controlled regardless of the fact that I had the experience of living in Russia, end quote. +DUMMY/LJ003-0040.wav|The evils of association for these debtors were perpetuated, although the plan provided for the separation of the various contingents committed to it. +DUMMY/LJ048-0209.wav|at least for a sufficient number of seconds to gain a, quote, general impression, end quote, of the lack of any unusual activity. +DUMMY/LJ016-0009.wav|He had friends and auxiliaries inside the jail and out. The cell he occupied was near the outer wall, +DUMMY/LJ008-0037.wav|were brought out of Newgate about eight in the morning, and suspended on a gallows of a new construction. +DUMMY/LJ035-0062.wav|Truly said of Oswald: quote, He didn't seem to be excited or overly afraid or anything. He might have been a bit startled, like I might have been if somebody confronted me. +DUMMY/LJ019-0282.wav|Their origin it will be seen dates back much further than that. +DUMMY/LJ019-0268.wav|In some places the dietary was too full, in others too meager. Its constituents were not of the most suitable character. +DUMMY/LJ006-0303.wav|This girl had been first engaged on account of the extra work entailed by certain prisoners +DUMMY/LJ041-0086.wav|and then used the, quote, special treatment, end quote, he received as an example of the way in which he was being picked on and, quote, +DUMMY/LJ017-0151.wav|The latter seems to have succumbed only too willingly to his fascinations, and to have as readily agreed to marry him, +DUMMY/LJ018-0340.wav|characterized this as a crime more black and hideous than any in the criminal annals of the country. +DUMMY/LJ019-0194.wav|not to exceed a certain sum to be agreed upon. +DUMMY/LJ018-0170.wav|On the bed were twenty forged ten-pound notes complete and ready for use, and twenty-five five-pound notes. +DUMMY/LJ039-0043.wav|She stated further that Oswald had only mentioned Nixon's name once during the incident. Marina Oswald might have misunderstood her husband. +DUMMY/LJ001-0081.wav|type cast from his matrices is still in everyday use. +DUMMY/LJ016-0043.wav|The least slip now would have been fatal to him. But he could not thrust his body in through the narrow space left by the chevaux-de-frise, +DUMMY/LJ005-0168.wav|"It is not," he said, "that I am insensible of the lamentable and disgraceful situation in which many of them are, +DUMMY/LJ040-0060.wav|and his unsuccessful attempt to go to Cuba in late September of nineteen sixty-three. +DUMMY/LJ044-0141.wav|even though he had written four detailed letters since that time to Mr. V. T. Lee and had also kept the national headquarters informed +DUMMY/LJ038-0238.wav|An examination of the window at the rear of the house, the wall through which the bullet passed, and the fence behind the house +DUMMY/LJ032-0104.wav|Marina Oswald testified, +DUMMY/LJ015-0291.wav|partly through their own carelessness, when transferring their operations to Yarmouth. +DUMMY/LJ002-0300.wav|Numerous tyrannies were practiced on all who would not and could not pay the garnish. +DUMMY/LJ048-0169.wav|In Dallas the route selected necessarily involved passing through the principal downtown section between tall buildings. +DUMMY/LJ047-0103.wav|who was involved, what occurred, he was reticent to furnish information, +DUMMY/LJ028-0049.wav|and in that same year Nabopolassar, the father of the great Nebuchadnezzar, became the King of Babylon. +DUMMY/LJ018-0205.wav|Cummings was caught shortly afterwards. +DUMMY/LJ028-0471.wav|Nebuchadnezzar says that he built it of burned bricks, but only sun-dried bricks laid in mud now appear. +DUMMY/LJ013-0010.wav|The 'Dryad' was a brig owned principally by two persons named Wallace, one a seaman, the other merchant. +DUMMY/LJ049-0046.wav|Even so, analysis of the motion picture films taken by amateur photographer Zapruder +DUMMY/LJ006-0010.wav|Mr. Samuel Hoare, when examined, considered it indispensably necessary, to carry out whatever system might be established, +DUMMY/LJ014-0201.wav|After sentence, the Newgate officers who had special charge of him noticed that he slept well and ate well, enjoying all his meals. +DUMMY/LJ028-0296.wav|thou coverest the foulest deeds with the fairest possible name, when thou sayest thy maiming is to help our siege forward. +DUMMY/LJ003-0033.wav|Enough has been said, probably, to prove that there was room for improvement in the condition and treatment of debtors in the prisons of the city of London. +DUMMY/LJ049-0060.wav|but to consider any untoward circumstances as serious and to afford the President maximum protection at all times. +DUMMY/LJ018-0076.wav|As Sattler brooded over his wrongs, his rage got the upper hand, and he resolved to wreak it upon Thain. +DUMMY/LJ002-0167.wav|quotes a case which came within his own knowledge of a boy sent to prison for non-payment of one penny. +DUMMY/LJ017-0215.wav|Six were Spaniards, or rather natives of Manilla, and men of color; one was a Greek, another a Turk; +DUMMY/LJ007-0109.wav|O'Mally sent in a medical certificate, declaring that the imprisonment was endangering his life, and after some question he was also released. +DUMMY/LJ006-0235.wav|The surgeon's journal produced to the inspectors contained numerous entries of terrible wounds inflicted in a cowardly way. +DUMMY/LJ036-0122.wav|and a pickup from Greyhound (bus station) at twelve:thirty p.m., +DUMMY/LJ018-0382.wav|He had steadfastly refused to eat meat on some religions scruples, and for the same reason would not touch soup. +DUMMY/LJ048-0188.wav|These substitute measures were of limited value. Agent Lawson was unable to state whether he had actually instructed +DUMMY/LJ040-0108.wav|which became more significant as the children grew older. +DUMMY/LJ011-0056.wav|Angelini wrote to the Lord Mayor to this effect, +DUMMY/LJ005-0238.wav|I shall have more to say on this subject, and upon the state of Newgate generally, in the following chapter. +DUMMY/LJ010-0152.wav|fired a pistol at George the third from the pit of Drury Lane theatre in eighteen hundred. +DUMMY/LJ030-0160.wav|it was calculated that the car required eight point three seconds to cover the one hundred thirty-six feet. +DUMMY/LJ030-0150.wav|The Dallas police radio log reflects that Chief of Police Curry reported the shooting of the President +DUMMY/LJ028-0337.wav|attacked the walls on every side, whereupon Zopyrus played out the remainder of his stratagem. +DUMMY/LJ016-0028.wav|and would certainly have been impossible to any one less nimble than a chimney-sweep, trained under the old system to ascend the most intricate flues. +DUMMY/LJ018-0333.wav|it was the principal incentive with Kate Webster, that fierce and brutal female savage who took the life of her mistress at Richmond. +DUMMY/LJ008-0234.wav|each wave seems trying to overleap the other as they are drawn onwards towards this outlet. +DUMMY/LJ027-0146.wav|though now extinct species, speaks strongly in favor of evolution. +DUMMY/LJ050-0221.wav|The Secret Service has now presented its recommendations to the Bureau of the Budget. The plan proposed by the Service +DUMMY/LJ045-0162.wav|He tried to start a conversation with me several times, but I would not answer. And he said that he didn't want me to be angry at him because this upsets him. +DUMMY/LJ017-0071.wav|To meet his liabilities, he raised large sums on forged bills of acceptance drawn upon his mother, a woman of some means, +DUMMY/LJ004-0250.wav|"It is obvious that reformation must be materially impeded, and in some cases utterly defeated, when the prisoners are defectively classed, +DUMMY/LJ017-0249.wav|The boats reached the shore on the fourth October. Leon had prepared a plausible tale to the effect that they belonged to an American ship +DUMMY/LJ010-0099.wav|The conspirators were in a loft, approached by a ladder and a trap-door, access through which could only be obtained one by one. +DUMMY/LJ037-0163.wav|Cunningham testified that all of the bullets were mutilated, one being useless for comparison purposes. +DUMMY/LJ028-0175.wav|but they add little to our knowledge. +DUMMY/LJ029-0206.wav|Another reaction to the impending visit -- hostile to the President -- came to a head shortly before his arrival. +DUMMY/LJ033-0066.wav|Mrs. Paine, who had no knowledge of the rifle, first said "No," but when the question was translated, Marina Oswald replied "Yes." +DUMMY/LJ013-0147.wav|who declared that the rough diamond market was in such a depressed condition that they could only afford to give one thousand seven fifty pounds for stones worth +DUMMY/LJ029-0100.wav|through the downtown area along Main Street and then to the Trade Mart via Stemmons Freeway. +DUMMY/LJ006-0004.wav|But this digression was necessary in order to present a more complete picture of the state of jails in the early part of the present century, +DUMMY/LJ006-0218.wav|Wrestling, in which legs were occasionally broken, was freely indulged in; also such low games as "cobham," +DUMMY/LJ013-0017.wav|Yet the Wallaces pretended to have put on board quantities of flannels, cloths, cotton prints, +DUMMY/LJ012-0160.wav|now more strictly meager, thanks to the inspectors and the more stringent discipline, and before he embarked for Australia to undergo his fourteen years, +DUMMY/LJ038-0198.wav|sixty dollars on the second of the month. You and the baby can live for another two months using ten dollars per week. +DUMMY/LJ046-0253.wav|it had no written specification of intelligence information collected by CIA abroad which was desired by the Secret Service +DUMMY/LJ050-0180.wav|so that these arrangements can be made permanent without adversely affecting the operations of the Service's field offices. +DUMMY/LJ004-0180.wav|a price which in those days fluctuated enormously -- as much as a hundred percent in a couple of years; +DUMMY/LJ039-0176.wav|The angle from the first to the second shot +DUMMY/LJ013-0100.wav|established the fact that the man who had personated Mr. Oxenford was a horse-dealer named Joseph Elder, an intimate acquaintance of Burgess'. +DUMMY/LJ002-0216.wav|The shameful malpractices of Bambridge, +DUMMY/LJ030-0101.wav|Admiral George G. Burkley, physician to the President, +DUMMY/LJ003-0096.wav|Quote, yet they had been long enough, he goes on to say, in the prison associated with the lowest and vilest criminals, +DUMMY/LJ009-0202.wav|In eighteen eleven Williams, who murdered the Marrs in Ratcliffe Highway, having committed suicide in jail to escape hanging, +DUMMY/LJ007-0119.wav|As a rule, visitors, many of them persons of good position, came and went all day long, and as late as nine at night; +DUMMY/LJ017-0219.wav|and punishment such as rope's-ending and tying to the bulwarks had to be applied to get the work properly done. +DUMMY/LJ031-0031.wav|Despite his serious wounds, Governor Connally tried to get out of the way so that medical help could reach the President. +DUMMY/LJ047-0028.wav|he had attempted to renounce his U.S. citizenship and applied for Soviet citizenship, +DUMMY/LJ016-0218.wav|wished to do Calcraft a good turn, and sent to his London tailor for a complete new suit, in which he appeared at his execution. +DUMMY/LJ031-0209.wav|X-rays and photographs were taken preliminarily and the pathological examination began at about eight p.m. +DUMMY/LJ001-0047.wav|Of Jenson it must be said that he carried the development of Roman type as far as it can go: +DUMMY/LJ042-0131.wav|Oswald opened negotiations with the U.S. Embassy in Moscow looking toward his return to the United States. +DUMMY/LJ018-0210.wav|Those who manufactured and those who passed had no direct dealings with each other. +DUMMY/LJ018-0318.wav|The vigilance of the Newgate officials, assisted by the city police, completely frustrated this plot. +DUMMY/LJ046-0247.wav|However, the vague requests for information which the Secret Service made +DUMMY/LJ014-0302.wav|It outshone the defalcations of Watts. +DUMMY/LJ012-0059.wav|One turnkey was stupidly drunk, and Ikey persuaded the other, who was not much better, +DUMMY/LJ037-0030.wav|quote, We've had a shooting out here, end quote, +DUMMY/LJ033-0090.wav|He asked, quote, What's the package, Lee? End quote. Oswald replied, quote, curtain rods, end quote. +DUMMY/LJ016-0174.wav|Having mentioned Calcraft's name, +DUMMY/LJ019-0065.wav|Better industrial results were obtained from it, and instruction in trades was easier, and prisoners were more likely to leave jail +DUMMY/LJ028-0128.wav|His brief description of them should not be omitted. He says that Nebuchadnezzar +DUMMY/LJ013-0222.wav|he did not shrink from murder, both for revenge and to conceal his other crimes. +DUMMY/LJ006-0300.wav|but he afterwards admitted that the girl had been recommended to him by the principal turnkey, who knew something of her friends. +DUMMY/LJ009-0052.wav|The third convict is a sheep-stealer, a poor ignorant fellow in whose crime are mitigating circumstances, +DUMMY/LJ029-0193.wav|to call in one hundred extra off-duty officers to help protect President Kennedy. +DUMMY/LJ001-0151.wav|the unit of the book being looked on as the two pages forming an opening. +DUMMY/LJ019-0181.wav|is an interesting document, and shows that even at that date the Government contemplated the erection of a model prison. +DUMMY/LJ002-0323.wav|Hence starvation stared many in the face, unless friends from outside came to their assistance, +DUMMY/LJ048-0121.wav|The interim report must be in the hands of the agent supervising the protective group traveling with the President +DUMMY/LJ024-0011.wav|secondly, to bring to the decision of social and economic problems younger men who have had personal experience and contact +DUMMY/LJ025-0119.wav|from the metamorphosis of morphological units (termed cells in a more general sense than that in which the word "cells" was at first employed), +DUMMY/LJ038-0163.wav|Prior attempt to kill. +DUMMY/LJ012-0083.wav|But before committing themselves to an attempt on the latter, it was of importance to ascertain how much it usually contained. +DUMMY/LJ010-0237.wav|A youth named Pearson had seen him present a pistol at the Queen's carriage, +DUMMY/LJ021-0055.wav|In meeting the problems of industrial recovery the chief agency of the government has been the National Recovery Administration. +DUMMY/LJ015-0254.wav|together with the unrealized part of the bullion, amounting in all to some fifteen thousand pounds, +DUMMY/LJ009-0244.wav|especially that of William Snow, alias Sketch, who slipped from the gallows at Exeter and fell to the ground. +DUMMY/LJ013-0083.wav|the Bank of England was defrauded of a sum of eight thousand pounds by one of its clerks, Burgess, in conjunction with an accomplice named Elder. +DUMMY/LJ047-0169.wav|Hosty took the necessary steps to have the Dallas office of the FBI, rather than the New Orleans office, reestablished as the office with principal responsibility. +DUMMY/LJ033-0080.wav|Mrs. Randle stated that on the morning of November twenty-two, while her brother was eating breakfast, +DUMMY/LJ035-0041.wav|but through the window in the door he caught a fleeting glimpse of a man walking in the vestibule toward the lunchroom. +DUMMY/LJ023-0128.wav|I came by a process of elimination to the conclusion that, short of amendments, +DUMMY/LJ023-0037.wav|not after long years of debate, but now. +DUMMY/LJ018-0083.wav|Sattler was a very excitable although not an ill-tempered man. +DUMMY/LJ049-0007.wav|stationing police on the rooftops of all buildings overlooking the reception area, +DUMMY/LJ005-0040.wav|It earnestly advocated the appointment of female officers to take exclusive charge of female prisoners, +DUMMY/LJ022-0007.wav|The administration and the Congress are not proceeding in any haphazard fashion in this task of government. +DUMMY/LJ018-0024.wav|Mr. Death, who had given another in exchange for it to a man supposed to be a foreigner. +DUMMY/LJ044-0063.wav|and actually had only one thousand printed. +DUMMY/LJ007-0069.wav|While admitting that he had had many difficulties to contend with, +DUMMY/LJ019-0328.wav|Stringent rules were prescribed for the prison surgeons; +DUMMY/LJ015-0221.wav|which came when the safes were sent to Chubbs' to be repaired; and Chubbs sent them back, but only with one key, +DUMMY/LJ017-0135.wav|Dove, a fiendish brute, found from the evidence in that case that he could kill his wife, whom he hated, +DUMMY/LJ011-0152.wav|The elder brother, +DUMMY/LJ031-0004.wav|In the final instant of the assassination, the Presidential motorcade began a race to Parkland Memorial Hospital, +DUMMY/LJ049-0057.wav|and to shield him by attempting to place themselves between the President and any source of danger. +DUMMY/LJ040-0059.wav|The review of the latter period will evaluate his personal and employment relations, his attempt to kill General Walker, his political activities, +DUMMY/LJ029-0171.wav|The President's intention to pay a visit to Texas in the fall of nineteen sixty-three aroused interest throughout the State. +DUMMY/LJ001-0107.wav|To say a few words on the principles of design in typography: +DUMMY/LJ019-0272.wav|The prisons were still far from inflicting the three punishments, hard labor, hard fare, and a hard bed +DUMMY/LJ034-0079.wav|and rode down in the east elevator. +DUMMY/LJ009-0137.wav|Cards were issued, and to such an extent, that although the service was not to commence till half-past ten, by nine a.m. +DUMMY/LJ050-0124.wav|to use the data developed by these agencies to carry out its special duties. +DUMMY/LJ044-0059.wav|The Commission has not been able to find any other indication that Oswald had rented an office in New Orleans. +DUMMY/LJ005-0220.wav|In eighteen twenty-four Newgate had no glass in its windows, except in the infirmary and one ward of the chapel yard; +DUMMY/LJ044-0203.wav|For example, the Dallas Times Herald of November nineteen, nineteen sixty-three, +DUMMY/LJ038-0182.wav|including a Russian volume entitled, quote, Book of Useful Advice, end quote. In this book was an undated note written in Russian. +DUMMY/LJ045-0114.wav|Oswald also said that he did not want the FBI to know where he lived, quote, Because their visits were not very pleasant for him +DUMMY/LJ014-0181.wav|Cope, a small man, crippled, and of weakly constitution, was alone in the shop about nine:thirty. +DUMMY/LJ014-0285.wav|and not for the slight offense as it appeared on the record. +DUMMY/LJ023-0036.wav|an overwhelming majority of them voted a mandate that the Congress and the President begin the task of providing that protection +DUMMY/LJ003-0189.wav|The name still survived in the new press yard, which was the receptacle of the male condemned prisoners. It was generally crowded, like the rest of the prison. +DUMMY/LJ022-0042.wav|to assist materially in our already unmistakable march toward recovery. +DUMMY/LJ016-0005.wav|Year after year prisoners determined to get free, and occasionally succeeded in their efforts. +DUMMY/LJ036-0088.wav|and told him a similar story to Craig's. +DUMMY/LJ043-0049.wav|After she had bought some clothes for Marina Oswald and a highchair for the baby, Oswald emphatically told her to stop. +DUMMY/LJ009-0088.wav|ignominy, sorrow, sufferings, wretchedness, pangs, +DUMMY/LJ045-0055.wav|He said this about members of the Russian-speaking group in the Dallas-Ft. Worth area, whom she said he tried to forbid her from seeing, +DUMMY/LJ042-0014.wav|was his service in Japan. +DUMMY/LJ030-0036.wav|Secret Service arrangements for Presidential trips, which were followed in the Dallas motorcade, +DUMMY/LJ039-0193.wav|and a total minimum time period of from seven point one to seven point nine seconds for all three shots. +DUMMY/LJ027-0096.wav|as, indeed, she must be according to the derivation theory. +DUMMY/LJ028-0496.wav|rising higher and higher, like a great terraced, turreted mountain. +DUMMY/LJ032-0018.wav|Purchase of Rifle by Oswald +DUMMY/LJ039-0114.wav|When asked to explain the different scores achieved by Oswald on the two occasions when he fired for record, +DUMMY/LJ003-0042.wav|The completion of this very necessary building was, however, much delayed for want of funds, +DUMMY/LJ014-0203.wav|the great clock of St. Sepulchre's close by was striking the hour, and Marley, who had his elbows on the table, +DUMMY/LJ036-0115.wav|David Knapp, aged eighteen, was Number two; +DUMMY/LJ025-0026.wav|Following out his teleological argument, Cuvier remarks that the organization of this cavity +DUMMY/LJ019-0326.wav|Steps were taken to provide the illiterate with secular instruction. +DUMMY/LJ038-0272.wav|and, according to Frazier, "relatively few" types of rifles could produce the characteristics found on the bullet. +DUMMY/LJ034-0119.wav|Approximately seven or eight minutes later +DUMMY/LJ008-0208.wav|Some years later an eye-witness published a graphic account of one of these scenes. +DUMMY/LJ010-0089.wav|While a few secured the servants, the rest were to fall upon Lord Harrowby and his guests. +DUMMY/LJ010-0193.wav|When brought to trial he hardly realized his position, +DUMMY/LJ042-0163.wav|After, however, two years and a lot of growing up, I decided to return to the USA. +DUMMY/LJ008-0128.wav|the dreadful execrations of some of the felons so shook his frame that he observed "the clock had struck;" and quickening his pace, +DUMMY/LJ005-0160.wav|It was plain that although the law had defined general principles of prison government, +DUMMY/LJ028-0374.wav|settled there, and finally the place was abandoned to the Arabs of the desert. +DUMMY/LJ048-0233.wav|Two of the nine agents returned to their rooms. The seven others proceeded to an establishment called the Cellar Coffee House, +DUMMY/LJ038-0109.wav|was either made from the larger, or the larger photograph was made from the smaller and that at the proper time he would show that the pictures were fakes. +DUMMY/LJ038-0004.wav|The Texas Theatre is on the north side of Jefferson Boulevard, approximately eight blocks from the scene of the Tippit shooting and six blocks +DUMMY/LJ043-0138.wav|As indicated in chapter four, Oswald had been planning his attack on General Walker for at least one and perhaps as much as two months. +DUMMY/LJ005-0021.wav|who, in a caustic article contributed to the 'Edinburgh Review,' protested against the pampering of criminals +DUMMY/LJ009-0078.wav|who preaches plain, homely discourses, as fit as any religious discourse can be fit for the irritated audience. +DUMMY/LJ020-0026.wav|One half cup of Indian meal. One half cup of molasses. Two teaspoonfuls of salt. Soda, the size of a pea. +DUMMY/LJ039-0245.wav|(five) resisted arrest by drawing a fully loaded pistol and attempting to shoot another police officer, +DUMMY/LJ048-0065.wav|the FBI instructions did not reflect fully the Secret Service's need for information regarding potential threats. +DUMMY/LJ049-0140.wav|The Council already includes, in addition to the President and Vice President, the Secretaries of State and Defense and has a competent staff. +DUMMY/LJ018-0077.wav|Although manacled, he managed to get a pistol from his chest and load it. +DUMMY/LJ013-0199.wav|It will be remembered that the question whether Mr. Phillips had not exceeded the limits usually allowed to counsel was much debated at the time. +DUMMY/LJ004-0056.wav|Mr. Buxton's small work on prison discipline gave a new aspect to the question he had so much at heart. +DUMMY/LJ001-0054.wav|Jenson, however, had many contemporaries who used beautiful type, +DUMMY/LJ016-0369.wav|upon them devolved the painful duty of cutting down the body and preparing for the inquest. +DUMMY/LJ019-0080.wav|some were not sufficiently ventilated, others were warmed artificially, and were unwholesomely close. +DUMMY/LJ024-0057.wav|Why then should we leave the fulfillment of this public policy to chance +DUMMY/LJ050-0120.wav|and for reporting such events as a change in leadership or dogma which indicate that the group may present a danger to the President. +DUMMY/LJ009-0079.wav|The sermon of this day, whether eloquent or plain, useful or useless, must produce a striking effect at the moment of its delivery. +DUMMY/LJ027-0046.wav|On the one hand, then, are found structures which are perfectly analogous and yet in no way homologous: +DUMMY/LJ013-0172.wav|were found in the butler's pantry, used by Courvoisier. +DUMMY/LJ014-0140.wav|After judgment was passed she repeatedly cried out Shame! +DUMMY/LJ045-0070.wav|thought that Marina Oswald was immature in her thinking and partly responsible for the difficulties that the Oswalds were having at that time. +DUMMY/LJ014-0289.wav|The details of the suicide were given at the inquest. Watts had been in ill-health from the time of his first arrest. +DUMMY/LJ026-0089.wav|poor in energy or devoid of it, and out of them to manufacture food -- i.e. complex compounds rich in available potential energy. +DUMMY/LJ021-0112.wav|or to prevent destructive price cutting which many business organizations have insisted were necessary, +DUMMY/LJ023-0060.wav|and the powers given to the Congress to carry out those purposes can be best described by saying +DUMMY/LJ030-0069.wav|scanning the route, and getting out and standing near the President when the cars stopped. +DUMMY/LJ039-0077.wav|Well, in order to achieve three hits, it would not be required that a man be an exceptional shot. A proficient man with this weapon, yes, end quote. +DUMMY/LJ002-0296.wav|Garnish on the cabin side was a guinea at entrance for coals, candles, brooms, etc., and a gallon of beer on discharge; +DUMMY/LJ036-0030.wav|About two blocks later, a woman asked to get off to make a one o'clock train at Union Station +DUMMY/LJ033-0028.wav|Frazier told her that Oswald had come home to get curtain rods. +DUMMY/LJ024-0015.wav|or those who would subsequently reach the age of seventy. +DUMMY/LJ036-0135.wav|I started to that address, and the police cars, the sirens was going, running crisscrossing everywhere, just a big uproar in that end of town and I said, +DUMMY/LJ045-0237.wav|His denials under questioning, which have no probative value in view of the many readily demonstrable lies he told at that time +DUMMY/LJ037-0222.wav|When Oswald was arrested, he did not have a jacket. +DUMMY/LJ002-0267.wav|was applied only to the release of poor debtors whom four pounds each could free. +DUMMY/LJ016-0010.wav|and had he but been able to remove its iron bars, he might have descended into Newgate Street by means of a rope ladder. +DUMMY/LJ031-0128.wav|and escorted them into Parkland Hospital through the emergency entrance. +DUMMY/LJ007-0134.wav|The wards were in some cases spacious, but they were entirely unsuited for separation or the inspection of prisoners. +DUMMY/LJ012-0084.wav|For this purpose Jordan waited on the receiver to make a small payment, for which he tendered a fifty-pound note. +DUMMY/LJ045-0181.wav|No one will ever know what passed through Oswald's mind during the week before November twenty-two, nineteen sixty-three. +DUMMY/LJ036-0123.wav|unloaded at five hundred North Beckley at twelve:forty-five p.m. +DUMMY/LJ003-0050.wav|(one) those awaiting trial; +DUMMY/LJ019-0139.wav|These numbers would have still further decreased, and the jail would have been almost empty, but for the misdemeanants who were still sent to Newgate +DUMMY/LJ011-0015.wav|His hair, though gray, was thick, and lay smooth over his forehead. +DUMMY/LJ009-0232.wav|there were still many shocking incidents at executions, owing to the bungling and unskilful way in which the operation was performed. +DUMMY/LJ018-0292.wav|that his transactions at Birmingham would shortly be very large, owing to the development of his business there in the alleged manufacture of Pullman cars. +DUMMY/LJ005-0108.wav|the prisoners, without firing, bedding, or sufficient food, spent their days "in surveying their grotesque prison, +DUMMY/LJ043-0091.wav|He was not able to produce photographic work +DUMMY/LJ005-0066.wav|"The new jails," as Howard says, "having pompous fronts, appear like palaces to the lower class of people, and many persons are against them on this account." +DUMMY/LJ024-0001.wav|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt +DUMMY/LJ043-0075.wav|He indicated an interest in writing. An employment counselor testified, on the basis of a general aptitude test Oswald had taken, +DUMMY/LJ018-0080.wav|Sattler was tried for murder and convicted; +DUMMY/LJ032-0196.wav|This adds to the conviction of the Commission that Oswald owned and handled the weapon used in the assassination. +DUMMY/LJ046-0057.wav|or Thomas Jefferson from Washington to Monticello. +DUMMY/LJ012-0269.wav|Mrs. Brown was a washer-woman, supposed to be worth some money; hence Greenacre's offer of marriage. +DUMMY/LJ014-0009.wav|This sight was his own handiwork, +DUMMY/LJ022-0019.wav|More and more people, +DUMMY/LJ045-0170.wav|it became too difficult to wash by hand. +DUMMY/LJ032-0188.wav|provides some evidence that they were placed on the rifle that day +DUMMY/LJ014-0317.wav|Pries was lodged in Newgate, tried at the Old Bailey, and transported for life. +DUMMY/LJ050-0087.wav|propensity toward violent action, or some similar characteristic, coupled with some evaluation of the capability of the individual or group +DUMMY/LJ041-0163.wav|out of a combination of Oswald's known Marxist sympathies and George Orwell's book "nineteen eighty-four," one of Oswald's favorite books +DUMMY/LJ022-0006.wav|Before I come to any of the specific measures, however, I want to leave in your minds one clear fact. +DUMMY/LJ012-0290.wav|While conversing with Mrs. Brown, he declared the unfortunate woman was rocking herself to and fro in a chair; +DUMMY/LJ043-0119.wav|nineteen sixty-three, almost a month after he lost his job. +DUMMY/LJ018-0200.wav|He went there with a posse of officers and forced his way upstairs to the first floor, +DUMMY/LJ016-0438.wav|that the convict's clothes were still the executioner's perquisite. +DUMMY/LJ015-0212.wav|One pair was with the traffic superintendent in London, another with an official in Folkestone, +DUMMY/LJ019-0280.wav|They are now the universal rule, +DUMMY/LJ018-0235.wav|He embarked forthwith in a career of the wildest extravagance, and ere long he had parted in his mother's name with most of the landed estates. +DUMMY/LJ003-0264.wav|Among the daily visitors were members of the criminal classes still at large, +DUMMY/LJ034-0078.wav|Oswald was referring to the west elevator which operates by pushbutton and only with the gate closed. Givens said, "Okay," +DUMMY/LJ005-0240.wav|Mr. Buxton had succeeded to the great work which William Wilberforce had commenced, and led the repeated attacks upon slavery in British colonies +DUMMY/LJ021-0084.wav|We count, in the future as in the past, on the driving power of individual initiative +DUMMY/LJ026-0105.wav|This is done by diastase, an enzyme of plant cells. +DUMMY/LJ044-0214.wav|reported Castro as saying Cuba could not accept a situation where at the same time the United States was trying to ease world tensions +DUMMY/LJ039-0219.wav|Frazier testified that the rifle was accurate, that it had less recoil than the average military rifle +DUMMY/LJ040-0030.wav|When he was in the Soviet Union, he apparently resented the Communist Party members, +DUMMY/LJ006-0236.wav|"A serious accident: one of the prisoners had a hot poker run into his eye." +DUMMY/LJ034-0205.wav|A Secret Service agent who spoke to Euins approximately twenty to thirty minutes after the assassination +DUMMY/LJ036-0139.wav|He gave me a dollar bill, the trip was ninety-five cents. +DUMMY/LJ002-0156.wav|Thomas Dobson, on twenty-second August, seventeen ninety-nine, for one shilling, with costs of eight shillings, ten pence. +DUMMY/LJ047-0062.wav|In March nineteen sixty-three, while attempting to locate Marina Oswald, +DUMMY/LJ015-0140.wav|After his bankruptcy he obtained a place as clerk in the Great Northern Railway office, +DUMMY/LJ030-0194.wav|Mrs. Connally heard a second shot fired and pulled her husband down into her lap. +DUMMY/LJ038-0138.wav|When asked about the curtain rod story, Oswald lied again. +DUMMY/LJ003-0169.wav|Prisoners who could afford it sometimes paid for four beds, at the rate of twenty-eight shillings, and so secured the luxury of a private room. +DUMMY/LJ002-0318.wav|The city also allowed the poor debtors fourteen ounces of bread daily, and their share of eight stone of meat, an allowance which never varied, +DUMMY/LJ043-0046.wav|After about a month with his brother, +DUMMY/LJ002-0207.wav|The yards were taken up with rackets and five courts, and here and there were "bumble puppy grounds," a game in which the players rolled iron balls +DUMMY/LJ019-0219.wav|It was to accommodate only the convicted prisoners sentenced to terms short of penal servitude, and after its completion +DUMMY/LJ021-0162.wav|What may be necessary for those countries is not my responsibility to determine. +DUMMY/LJ034-0050.wav|Most of the prints were found to have been placed on the cartons by an FBI clerk +DUMMY/LJ005-0122.wav|the changes necessary to bring the prisons into conformity with the recent acts were attempted, if not actually introduced into the county prisons, to which, +DUMMY/LJ034-0209.wav|the Commission has considered the allegation that Oswald was photographed standing in front of the building when the shots were fired. +DUMMY/LJ017-0267.wav|After condemnation, as the rules now kept capital convicts strictly apart, they could not be lodged in the two condemned cells, +DUMMY/LJ028-0433.wav|In the northern part of the enclosure to the east of the river, +DUMMY/LJ003-0122.wav|The prisoner mentioned above was continually persecuted by trials of this kind. +DUMMY/LJ046-0201.wav|were no more specific than the broad and general instructions its own agents and the White House mailroom. +DUMMY/LJ048-0182.wav|President Kennedy himself had mentioned it that morning, as had Agent Sorrels when he and Agent Lawson were fixing the motorcade route. +DUMMY/LJ002-0150.wav|Costs were the gallons of sack to the pennyworth of debt. +DUMMY/LJ021-0098.wav|modifying it where it needs modification and finally making recommendations to the Congress, +DUMMY/LJ002-0311.wav|These were lumped together into a general fund, +DUMMY/LJ021-0159.wav|Morally, it is the greatest menace to our social order. +DUMMY/LJ003-0099.wav|and though distress or the seduction of others might have led to the commission of this their first offense, +DUMMY/LJ043-0185.wav|Some idea of what he thought was sufficient reason for such an act may be found in the nature of the motive that he stated for his attack on General Walker. +DUMMY/LJ005-0187.wav|As the Society put it in eighteen twenty-seven, "the friends to the improvement of prison discipline will regret to learn +DUMMY/LJ043-0019.wav|He could never give her what the people were showering on her no matter how hard he worked -- and he worked very hard, end quote. +DUMMY/LJ042-0191.wav|While, quote, resourcefulness and patient working towards the aforesaid goals +DUMMY/LJ006-0047.wav|The prison had become more or less a place of detention only, harboring mainly those awaiting trial. +DUMMY/LJ031-0097.wav|Question: And why was no effort made at that time to inspect his back? Answer: +DUMMY/LJ014-0006.wav|While the body lay there still warm, battered and bleeding from the cruel blows inflicted upon him by his cowardly assailant, a man came by singing. +DUMMY/LJ012-0283.wav|he carried under his coat-flaps through the streets, and afterwards on his cap in a crowded city omnibus. +DUMMY/LJ018-0105.wav|He, like Saward, was an artist in penmanship. +DUMMY/LJ050-0105.wav|that society and its leaders had combined to thwart him. It will require every available resource of our Government +DUMMY/LJ032-0252.wav|Marina Oswald has stated that the rifle was among these possessions, although Ruth Paine testified that she was not aware of it. +DUMMY/LJ033-0190.wav|The Depository shipping department was on the first floor to which Oswald had access in the normal performance of his duties filling orders. +DUMMY/LJ017-0159.wav|and a life-interest in five thousand pounds, a fact on which Smethurst's counsel dwelt with much weight, +DUMMY/LJ041-0170.wav|Thornley testified, quote, At which time he looked at me like a betrayed Caesar and screamed, screamed definitely, "Not you, too, Thornley!" +DUMMY/LJ021-0191.wav|by awesome pronouncements concerning the unconstitutionality of some of our measures of recovery and relief and reform. +DUMMY/LJ007-0237.wav|Many rats also come through these so-called ventilators, as they open close to the ground at the back of the prison. +DUMMY/LJ031-0046.wav|and an extensive wound in the President's head where a sizable portion of the skull was missing. +DUMMY/LJ044-0148.wav|very unhappy, and that he actually wept when he told her that. +DUMMY/LJ026-0046.wav|In each the life is the sum total of a series of definite processes -- nutrition or food supply, +DUMMY/LJ033-0121.wav|The distance between the point on the seat and the door was twenty-seven inches. +DUMMY/LJ026-0033.wav|be either included in or excluded from either. +DUMMY/LJ037-0099.wav|On television Oswald's hair looked blond, whereas Smith remembered that the man who ran away had hair that was brown or brownish black. +DUMMY/LJ050-0017.wav|to be able to provide effective continuing supervision +DUMMY/LJ011-0273.wav|He was about to descend the stairs when three persons, one of them the young sailor, +DUMMY/LJ017-0118.wav|and it was on the entire absence of the latter that the defense was principally based when Palmer was brought to trial. +DUMMY/LJ039-0115.wav|Major Anderson said, quote, when he fired that two twelve +DUMMY/LJ050-0110.wav|and each agency given clear understanding of the assistance which the Secret Service expects. +DUMMY/LJ016-0044.wav|and was compelled to work along the railing round three-quarters of the square of the yard, +DUMMY/LJ014-0134.wav|that O'Connor had been more to her than her husband, that she ought to have married him. +DUMMY/LJ011-0135.wav|Macintosh's amendment was carried in the Commons, but the new law did not pass the Lords, who re-enacted the capital penalty. +DUMMY/LJ010-0012.wav|and prominent in the criminal annals of Newgate in these later years will be found numerous remarkable instances of this class of offense -- +DUMMY/LJ006-0175.wav|In his administration of discipline he was quite uncertain; +DUMMY/LJ034-0039.wav|In Latona's opinion, quote, +DUMMY/LJ029-0019.wav|The President had spoken earlier that day at the Air Force Academy in Colorado Springs, Colorado. +DUMMY/LJ002-0268.wav|The supreme control of the Marshalsea was vested in the marshal of the royal household; but although he drew a salary of five hundred pounds a year, +DUMMY/LJ015-0058.wav|Bates, the least guilty, was pardoned in eighteen fifty-eight. +DUMMY/LJ013-0239.wav|At the same time he was seen to steal and secrete a pair of trousers. The shop-boy gave information. +DUMMY/LJ024-0035.wav|Let me answer this question with a bluntness that will end all honest misunderstanding of my purposes. +DUMMY/LJ028-0183.wav|Cyrus, the King of Persia, was rising to power, and after he had defeated the Medes +DUMMY/LJ028-0095.wav|the beloved of Nabu, the arbiter, the possessor of wisdom, who seeks out the path of their divinity, +DUMMY/LJ027-0136.wav|Illustrations quoted from the works of Romanes and Le Conte will make this principle clear. +DUMMY/LJ049-0038.wav|had no passenger in a jump seat between Agent Youngblood and Vice President Johnson to interfere with Agent Youngblood's ability +DUMMY/LJ048-0010.wav|stressed also the decision by the Department of State that Oswald should be permitted to return to the United States. +DUMMY/LJ019-0205.wav|In eighteen forty-five the Jail Committee brought forward a definite proposal to purchase ground in the immediate vicinity for the erection of a new jail. +DUMMY/LJ042-0251.wav|He also inquired about visas for himself and his wife +DUMMY/LJ043-0012.wav|In fact, his relations with some of them, particularly George Bouhe, became quite hostile. +DUMMY/LJ026-0095.wav|Food (starch) thus produced in the green leaves of higher plants and the inorganic foods +DUMMY/LJ040-0166.wav|She also noted that there was, quote, a rather pleasant, appealing quality about this emotionally starved, affectionless youngster +DUMMY/LJ047-0032.wav|In June nineteen sixty-two, the Bureau was advised by the Department of State of Oswald's plan to return to the United States. +DUMMY/LJ049-0162.wav|Although the Secret Service has had the primary responsibility for the protection of the President, +DUMMY/LJ050-0125.wav|Once experience has been gained in implementing such agreements with the Federal and leading State and local agencies, +DUMMY/LJ033-0165.wav|In all of the observations and physical tests that I made I found the bag and the paper sample were the same, end quote. +DUMMY/LJ004-0163.wav|and of which fewer still would believe that the original is to be found in this enlightened and happy country." +DUMMY/LJ014-0328.wav|Cole quickly raised money on both sets of warrants. He carried on this game for some time with great success, +DUMMY/LJ032-0090.wav|This, too, was a forgery. The signature of "A. J. Hideel" was in the handwriting of Lee Harvey Oswald. +DUMMY/LJ010-0121.wav|that he was, to use Thistlewood's words, "a contriver, instigator, and entrapper." +DUMMY/LJ038-0149.wav|During the first interrogation on November twenty-two, Fritz asked Oswald to account for himself at the time the President was shot. +DUMMY/LJ004-0189.wav|The great principles of classification, cleanliness, and employment were closely observed. +DUMMY/LJ039-0107.wav|In May of nineteen fifty-nine, on another range, Oswald scored one hundred ninety-one, which was one point over the minimum for ranking as a "marksman." +DUMMY/LJ018-0274.wav|who was followed, disguised, and with her hair dyed black, to a house in the Marylebone Road, where she met her husband. +DUMMY/LJ010-0101.wav|the lights were extinguished, and a desperate conflict ensued. +DUMMY/LJ039-0155.wav|or, if either the first or third shots missed, the assassin fired the three shots during a minimum time span of seven point one +DUMMY/LJ001-0121.wav|in reading the modern figures the eyes must be strained before the reader can have any reasonable assurance +DUMMY/LJ014-0149.wav|On this journey her manner changed completely. She became flippant, joked with the officers, asked how they liked her "resolution" in the dock, +DUMMY/LJ008-0180.wav|Among the dead was a sailor lad whom no one knew; +DUMMY/LJ025-0108.wav|Thus, by the progress of knowledge, Cuvier's fourth distinction between the animal and the plant +DUMMY/LJ026-0065.wav|Matter enters the plant in the liquid or gaseous form by diffusion, +DUMMY/LJ014-0252.wav|He could do what he liked with the passbook, and by its adoption, as described as the basis of all entries, +DUMMY/LJ014-0126.wav|her face was comely, she had dark hair and good eyes, and was above the middle height, yet inclined to be stout. +DUMMY/LJ011-0120.wav|Unfortunately he took to drink, lost his appointment, and fell from bad to worse. +DUMMY/LJ018-0007.wav|then returned to the city to take the train from Fenchurch Street home, traveling by the North London Railway. +DUMMY/LJ019-0082.wav|In a great number of prisons no provision was made for the education of prisoners, in some others there was a sufficient staff of schoolmasters and instructors. +DUMMY/LJ008-0013.wav|As regards the first, I find that in seventeen eighty-six +DUMMY/LJ012-0211.wav|Mr. Partridge was sent for, and he with some of the students soon decided that the corpse had not died a natural death. +DUMMY/LJ012-0142.wav|and implicated "Money Moses," another Jew, for the whole affair had been planned and executed by members of the Hebrew persuasion. +DUMMY/LJ006-0183.wav|the apathy of the authorities, and the undue ascendancy of those who, as convicted felons, should have been most sternly repressed, +DUMMY/LJ034-0092.wav|in reaching the conclusion that the shots came from the sixth floor, southeast corner window of the Depository Building. +DUMMY/LJ016-0332.wav|who was convicted of complicity in the Clerkenwell explosion, intended to effect the release of Burke and Casey +DUMMY/LJ015-0131.wav|By these means Redpath misappropriated vast sums during a period extending over ten years. +DUMMY/LJ019-0021.wav|with the additions made to that date, the total expenditure amounted to nearly ninety thousand pounds, or about one hundred eighty pounds per cell. +DUMMY/LJ036-0031.wav|and requested a transfer which she might use if she got through the traffic. +DUMMY/LJ012-0220.wav|were known to be what Ferrari had worn when last seen. +DUMMY/LJ021-0132.wav|The workers who turn away from these same impartial agencies and decline to use their good offices to gain their ends +DUMMY/LJ042-0231.wav|His second answer to this question was, quote, No of course not, +DUMMY/LJ031-0177.wav|Before the body could be taken from the hospital, +DUMMY/LJ005-0074.wav|cap. sixty-four, and five George the fourth cap. eighty-five +DUMMY/LJ028-0070.wav|and how, in punishment for all his wickedness, he became a calf, and for seven years grazed the grass in the fields about the city. +DUMMY/LJ003-0077.wav|Two of the untried were for murder, and several for house-breaking and highway robbery. +DUMMY/LJ003-0213.wav|Convicted murderers were kept continuously in the cells on bread and water, +DUMMY/LJ019-0140.wav|at times on long terms of imprisonment, and for the transports, whom the Home Office were often, as of old, slow to remove. +DUMMY/LJ004-0033.wav|and they were not threatened with any particular penalties if they evaded or ignored the new acts. +DUMMY/LJ032-0125.wav|In his testimony before the Commission, Latona stated that when he received the rifle, the area where prints were visible was protected by cellophane. +DUMMY/LJ018-0335.wav|most violent in temper, and addicted to the most frightful language. +DUMMY/LJ005-0272.wav|Various other recommendations were made as regards the appointment of chaplain and schoolmasters; the limitation of the powers of wardsmen, +DUMMY/LJ015-0050.wav|Following this, warrants were issued for their arrest, +DUMMY/LJ040-0017.wav|Since Oswald is dead, +DUMMY/LJ016-0286.wav|Oh, my! Think I've got to die. +DUMMY/LJ013-0195.wav|induced the prisoner to make a full confession of his crime to the lawyers who defended him. +DUMMY/LJ009-0208.wav|erected on the cart, about four feet high at the head, and gradually sloping towards the horse, giving a full view of the body, +DUMMY/LJ043-0059.wav|While it does not appear that he established any new relationships as a result of his investigation, he did obtain a large picture of his father +DUMMY/LJ046-0087.wav|Absolute security is neither practical nor possible. An approach to complete security would require the President +DUMMY/LJ016-0149.wav|Suicides and executions were, however, always the most effectual methods of making exit from durance. +DUMMY/LJ029-0174.wav|Dallas, Fort Worth, San Antonio, and Houston +DUMMY/LJ015-0191.wav|In Newgate Redpath is remembered by the prison officer as a difficult man to deal with. +DUMMY/LJ018-0385.wav|It was his own handiwork -- a bird pecking at a flower; +DUMMY/LJ028-0364.wav|The city then fell to Seleucus, +DUMMY/LJ036-0167.wav|and walked directly to the terminal, he would have entered the cab at twelve:forty-seven or twelve:forty-eight p.m. +DUMMY/LJ009-0249.wav|When Charles White was executed in eighteen twenty-three for arson, he arranged a handkerchief +DUMMY/LJ019-0185.wav|Lord John, with great fairness, admitted that the whole of this burthen could not be imposed upon the city +DUMMY/LJ029-0143.wav|A sign located on this barrier instructs Main Street traffic not to make any turns. +DUMMY/LJ010-0046.wav|was followed in the very next year by those of Francis and of Bean in two consecutive months, +DUMMY/LJ007-0210.wav|Abuses and irregularities, which had been partially remedied by the reform introduced in eighteen thirty-seven, were once more in the ascendant. +DUMMY/LJ017-0106.wav|fearful paroxysms and cramps, ending in suffocation by the tetanic rigor which caught the muscles of the chest. +DUMMY/LJ011-0104.wav|Hunton was put upon his trial at the Old Bailey, and in due course sentenced to death. +DUMMY/LJ020-0019.wav|If you intend to bake in the forenoon, make the sponge at bedtime. If in the afternoon, early in the morning. +DUMMY/LJ014-0055.wav|Next year John Gleeson Wilson, at Liverpool, murdered a woman, Ann Henrichson, also a maidservant and two children; +DUMMY/LJ022-0116.wav|generated by this depression +DUMMY/LJ038-0139.wav|He denied that he had ever told Frazier that he wanted a ride to Irving to get curtain rods for an apartment. +DUMMY/LJ004-0212.wav|There was an attentive chaplain, a schoolmaster, and regular religious and other instruction. +DUMMY/LJ007-0013.wav|that women should be frequently kept from chapel in order that they might attend the ladies' lectures and exhortations. +DUMMY/LJ025-0068.wav|"At the present day," writes Huxley, +DUMMY/LJ045-0132.wav|to the time of the assassination. +DUMMY/LJ012-0026.wav|With this capital he returned to London and set up as a fence. +DUMMY/LJ004-0099.wav|Ventilators, hand and others, were to be supplied. +DUMMY/LJ016-0135.wav|who yet faced extraordinary risks to advance their enlargement by only a few hours. +DUMMY/LJ002-0320.wav|The bread was issued every alternate day; and while some prisoners often ate their whole allowance at once, +DUMMY/LJ028-0417.wav|and that Arke of Nebuchadnezzar for as perpetual a memory of his great idolatry and condigne punishment. +DUMMY/LJ021-0175.wav|but I do not believe any intelligent observer can accuse England of undue orthodoxy in the present emergency. +DUMMY/LJ039-0198.wav|but to determine the maximum speed at which it could be fired. +DUMMY/LJ008-0200.wav|hired a room for the night and morning, which he and a large party of friends occupied before and during the execution; +DUMMY/LJ019-0154.wav|If any disturbance reached his ears, he reported the case to the governor, who next morning visited the ward in fault, and asked for the culprit. +DUMMY/LJ009-0261.wav|the convict was half on the platform, half hanging, and the convulsions of his body were appalling. +DUMMY/LJ042-0018.wav|On the other hand, at least one person who knew Oswald after his return thought that his defection had a more personal and psychological basis. +DUMMY/LJ014-0248.wav|This passbook, when not at the bank, was in the exclusive custody of Watts. +DUMMY/LJ022-0016.wav|Before that time individual self- interest and group selfishness were paramount in public thinking. +DUMMY/LJ046-0028.wav|The protection of the President of the United States is an immensely difficult and complex task. +DUMMY/LJ006-0022.wav|At that time the mild and intelligent prison discipline in force in Pennsylvania, the legacy of the old Quaker immigrants, +DUMMY/LJ043-0058.wav|He visited some of his father's elderly relatives and the cemetery where his father was buried in an effort to develop the facts of his genealogy. +DUMMY/LJ019-0025.wav|Pentonville has really been the model on which all subsequent prison construction has been based. All prisons at home and abroad are but variations, +DUMMY/LJ018-0113.wav|The principals in this conspiracy, Wagner and Bateman, were sentenced to penal servitude for life, +DUMMY/LJ012-0095.wav|The jewels had belonged to a Spanish countess recently deceased, who had sent them to England for greater security on the outbreak of the first Carlist war. +DUMMY/LJ009-0175.wav|and in eighteen forty-one this was accepted as the almost universally established rule. +DUMMY/LJ038-0248.wav|was also identified by Marina Oswald as having been taken by her husband, presumably in connection with the Walker shooting. +DUMMY/LJ035-0207.wav|This witness identified Oswald in a lineup as the man most nearly resembling the man he saw and later identified Oswald as the man he observed. +DUMMY/LJ019-0261.wav|With these differences were others as opposed concerning industrial occupation. +DUMMY/LJ021-0142.wav|Accordingly, I propose to confer within the coming month +DUMMY/LJ048-0051.wav|it stated primarily to direct the thought of those responsible for the future safety of our Presidents to the need for a more imaginative +DUMMY/LJ045-0198.wav|that the Presidential motorcade would pass in front of his place of work. +DUMMY/LJ044-0028.wav|She testified that he threatened to beat her if she did not do so. The chapter had never been chartered by the national FPCC organization. +DUMMY/LJ015-0063.wav|But neither Robson nor Redpath would have been able to pursue their fraudulent designs with success had they not, like Watts, +DUMMY/LJ046-0158.wav|At the time of the assassination, the active PRS general files contained approximately fifty thousand cases +DUMMY/LJ027-0037.wav|both adaptations to an aerial environment; +DUMMY/LJ049-0110.wav|The governmental consequences of assassination of one of the specified officials give the United States ample power to act for its own protection. +DUMMY/LJ012-0180.wav|The murderer explained that he had first fired a pistol at Weare's head, but the shot glanced off his cheek. +DUMMY/LJ042-0234.wav|quote, except in the US, the living standard is a little higher. +DUMMY/LJ008-0134.wav|Thus appareled he ascended the stairs at the debtors' door, +DUMMY/LJ047-0070.wav|On April twenty-one, nineteen sixty-three, the FBI field office in New York +DUMMY/LJ001-0004.wav|produced the block books, which were the immediate predecessors of the true printed book, +DUMMY/LJ019-0053.wav|and our modern practice has prudently tried to steer between the two extremes, accepting as the best system a judicious combination of both. +DUMMY/LJ003-0071.wav|It was particularly recommended by the Committee on Jails in eighteen fourteen +DUMMY/LJ047-0188.wav|Had you planned any steps beyond that point? +DUMMY/LJ023-0131.wav|We must have men worthy and equipped to carry out impartial justice. +DUMMY/LJ034-0149.wav|Brennan appeared to revert to his earlier inability to make a positive identification, +DUMMY/LJ040-0015.wav|There is, however, a large amount of material available in his writings +DUMMY/LJ029-0014.wav|As a political leader, the President wished to resolve the factional controversy within the Democratic Party in Texas before the election of nineteen sixty-four. +DUMMY/LJ021-0063.wav|Benefits of the Industrial Recovery Program have come, +DUMMY/LJ005-0192.wav|The prisoners were lodged in rooms whence they could converse with passengers in the streets, and freely obtain spirits and other prohibited articles. +DUMMY/LJ046-0052.wav|To promote nationwide acceptance of his administration Washington made grand tours that served also to excite interest in the Presidency. +DUMMY/LJ025-0168.wav|Whence it follows that the bean has taken in only the raw materials of its fabric and has manufactured them into bean-stuffs. +DUMMY/LJ022-0191.wav|They are essential factors in a rounded program for national recovery. +DUMMY/LJ041-0007.wav|Neighbors and others who knew him at that time recall an introverted boy who read a great deal. +DUMMY/LJ045-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +DUMMY/LJ028-0241.wav|Hereupon the Persians who had been left for the purpose at Babylon by the river side +DUMMY/LJ024-0099.wav|even though the thirty- five states with ninety-five percent of the population are in favor of it. +DUMMY/LJ017-0009.wav|Sir Thomas Overbury was undoubtedly poisoned by Lord Rochester in the reign of James the first, +DUMMY/LJ049-0008.wav|and detailing police in civilian clothes to be scattered throughout the sizable crowd. +DUMMY/LJ006-0223.wav|There was much swearing and bad language, the very worst that could be used, from the first thing in the morning to the last thing at night. +DUMMY/LJ012-0008.wav|He had sought by all legal means to obtain possession of the two thousand pounds, but had failed, and had had recourse to more violent means. +DUMMY/LJ042-0239.wav|Despite the hatred that Oswald expressed toward the Soviet Union after his residence there, +DUMMY/LJ023-0071.wav|For nearly twenty years there was no conflict between the Congress and the Court. +DUMMY/LJ027-0148.wav|we should antecedently expect, if the theory is true, +DUMMY/LJ007-0044.wav|The latter, light-hearted and reckless, conducted themselves in the most unseemly fashion, and "with as much indifference as the inmates of the other parts of the prison." +DUMMY/LJ013-0150.wav|The moment it was discovered that the diamonds had disappeared, the affair was taken up by the police. +DUMMY/LJ008-0106.wav|He was introduced by the ordinary, Dr. Forde, a name familiar to the reader, who met him at the felons' door +DUMMY/LJ005-0217.wav|Without enlargement no material change in discipline or interior economy could possibly be introduced. +DUMMY/LJ039-0143.wav|Marina Oswald testified that on one occasion she saw him take the rifle, concealed in a raincoat, from the house on Neely Street. +DUMMY/LJ009-0260.wav|The spectacle was horrible; +DUMMY/LJ014-0301.wav|In eighteen fifty-three a second case of gigantic fraud alarmed and scandalized the financial world. +DUMMY/LJ037-0105.wav|They heard the sound of shots to the north of their lot. Callaway heard five shots, and Guinyard three. +DUMMY/LJ024-0084.wav|it seeks to restore the Court to its rightful and historic place in our constitutional government +DUMMY/LJ012-0031.wav|This rule in dealing with stolen goods continues to this day, and has made the fortune of many since Ikey. +DUMMY/LJ050-0117.wav|particularly those arising from organized groups, within their special jurisdiction. +DUMMY/LJ042-0148.wav|I myself do not have a particularly quiet disposition, +DUMMY/LJ008-0130.wav|Before the colonel had been pinioned he had pulled out two white handkerchiefs, one of which he bound over his temples so as nearly to conceal his eyes, +DUMMY/LJ003-0152.wav|It was the only way to escape the horrors, the distress, penury, and rags of the common yards. +DUMMY/LJ034-0127.wav|Brennan testified that they were standing, which is their apparent position in the photograph. +DUMMY/LJ023-0049.wav|Those who have intimated that the President of the United States is trying to drive that team +DUMMY/LJ029-0111.wav|Sorrels, who had participated in Presidential protection assignments in Dallas since a visit by President Franklin D. Roosevelt +DUMMY/LJ049-0163.wav|the FBI, which was established within the Department of Justice in nineteen oh eight, has had in recent years an increasingly important role to play. +DUMMY/LJ017-0089.wav|While Cook lay ill, Palmer in his name pocketed the proceeds of the Shrewsbury settling, +DUMMY/LJ028-0144.wav|fifty royal cubits in width, and two hundred in height. +DUMMY/LJ007-0043.wav|were associated continually with a number of those who could look with certainty on a mitigation of punishment. +DUMMY/LJ005-0225.wav|In eighteen twenty-seven the Society was compelled to report that "no material change had taken place in Newgate since the passing of the prison laws, +DUMMY/LJ040-0204.wav|who had real difficulty in accepting and relating to people, end quote, and who had, quote, little understanding, end quote, of Lee's behavior +DUMMY/LJ011-0089.wav|and listened attentively to the ordinary, who read him the well-known address written and delivered by Dr. Dodd previous to his own execution for forgery. +DUMMY/LJ031-0222.wav|The surgeons observed, through X-ray analysis, thirty or forty tiny dustlike fragments of metal +DUMMY/LJ026-0125.wav|But the formation of starch, all important as it is, is after all only the manufacture of food +DUMMY/LJ011-0288.wav|Thus free, he eluded the vigilance of two of the party, who were at dinner in the front kitchen, +DUMMY/LJ008-0058.wav|From this bar hang several iron loops, in which the halters are tied. +DUMMY/LJ014-0033.wav|When her ruin was discovered her brother panted for revenge. +DUMMY/LJ030-0169.wav|Oh, my God, they have shot my husband. I love you, Jack. +DUMMY/LJ047-0164.wav|At this point in the interview, Hosty gave Mrs. Paine his name and office telephone number on a piece of paper. +DUMMY/LJ026-0147.wav|In the higher plants the excretions are carbon dioxide, which escapes through the epidermis of root, stem and leaf and through the stomata; +DUMMY/LJ048-0135.wav|Liaison with local law enforcement authorities. +DUMMY/LJ012-0093.wav|and the police were unable at first to get on the track of the thieves. +DUMMY/LJ046-0081.wav|The men in charge of protecting the President, confronted by complex problems and limited as they are in the measures they may employ, +DUMMY/LJ016-0094.wav|No further notice was taken of him; no one seemingly suspected that he was a prisoner, and he was suffered to walk off without let or hindrance. +DUMMY/LJ005-0155.wav|It was still limited to bread in some places, the allowance of which varied from one to three pounds; +DUMMY/LJ040-0210.wav|It may also be significant that, as reported by John Pic, quote, Lee slept with my mother until I joined the service in nineteen fifty. +DUMMY/LJ028-0161.wav|the streets all run in straight lines, not only those parallel to the river, but also the cross streets which lead down to the waterside. +DUMMY/LJ032-0095.wav|as additional persons entitled to receive mail in the box. The New Orleans postal authorities had not discarded the portion of the application +DUMMY/LJ016-0360.wav|The change added greatly to the responsibilities of the governor and his subordinates. Hitherto the public had seemed to assist at the ceremony; +DUMMY/LJ046-0083.wav|The problem and the reasonable approach to its solution were ably stated in a memorandum prepared by FBI Director J. Edgar Hoover +DUMMY/LJ034-0103.wav|This description most probably led to the radio alert sent to police cars at approximately twelve:forty-five p.m., which described the suspect as white, +DUMMY/LJ012-0267.wav|but his despair must have been great, as was evident from his attempt to strangle himself in the station-house. +DUMMY/LJ014-0309.wav|Few realized that these mysterious operations were the "convulsive attempt" of a ruined and dishonest speculator to sustain his credit. +DUMMY/LJ002-0226.wav|The same evils of overcrowding, uncleanliness, want of medical attendance, +DUMMY/LJ007-0072.wav|as regards the private teaching of prisoners; and they went on to say that "a resolved adherence, in spite of discouragements the most disheartening, +DUMMY/LJ034-0011.wav|measuring approximately eighteen by twelve by fourteen inches which had been moved from a stack along the south wall. +DUMMY/LJ019-0031.wav|was the moving spirit among these commissioners, and he is now generally recognized as the originator of modern prison architecture. +DUMMY/LJ016-0357.wav|No time was lost in carrying out the dread ceremony; but it was not completed without some of the officials turning sick, and the moment it was over, +DUMMY/LJ038-0151.wav|He acknowledged the encounter with the police officer on the second floor. +DUMMY/LJ028-0378.wav|Even the shepherds ceased to graze their sheep there, and the wandering Arabs, +DUMMY/LJ016-0055.wav|He traversed vainly all the roofs in Newgate Street, +DUMMY/LJ019-0198.wav|and on a more mature consideration he realized that the limited area of the existing Newgate site, +DUMMY/LJ017-0023.wav|by putting arsenic in the dumplings she had prepared for them. The charge rested entirely on circumstantial evidence, +DUMMY/LJ048-0235.wav|serving only coffee, fruit juices and no hard liquors or beer, end quote. +DUMMY/LJ050-0160.wav|The Commission further recommends that the Secret Service coordinate its planning as closely as possible with all of the Federal agencies from which it receives information. +DUMMY/LJ004-0191.wav|and in comparative comfort, with a bed and proper bedding. +DUMMY/LJ044-0184.wav|partisanship, and personal activities on behalf of the Cuban government. +DUMMY/LJ034-0088.wav|Three invoices on this clipboard, each dated November twenty-two, were for Scott-Foresman books, located on the first and sixth floors. +DUMMY/LJ008-0225.wav|the former already vowed to the same criminal career, and looking up to their elders with the respect due to successful practitioners; +DUMMY/LJ014-0295.wav|This man got up to look for him, and found him hanging from the bars of a neighboring room. +DUMMY/LJ036-0029.wav|where a man beat on the front door of the bus, boarded it and paid his fare. +DUMMY/LJ006-0258.wav|But the inspectors found in the ward cupboards mince-pies and other pasties, cold joints, hams, and so forth. +DUMMY/LJ013-0020.wav|he acted in a manner which excited the suspicions of the crew. +DUMMY/LJ029-0133.wav|trisect the plaza, converging at the apex of the triangle to form a triple underpass beneath a multiple railroad bridge +DUMMY/LJ039-0108.wav|The Marine Corps records maintained on Oswald further show that he had fired and was familiar with the Browning Automatic rifle, +DUMMY/LJ023-0002.wav|March nine, nineteen thirty-seven. Part one. +DUMMY/LJ040-0193.wav|Carro reported that Lee was disruptive in class after he returned to school on a regular basis in the fall of nineteen fifty-three. +DUMMY/LJ003-0074.wav|The Hon. H. G. Bennet, who visited Newgate in eighteen seventeen, saw in one yard, in a total of seventy-two prisoners, +DUMMY/LJ038-0284.wav|Although the Commission recognizes that neither expert was able to state +DUMMY/LJ010-0039.wav|the Gleeson Wilson murder at Liverpool, that of the Mannings in London, and of many more. +DUMMY/LJ038-0085.wav|A full discussion of Oswald's detention and interrogation is presented in chapter five of this report. +DUMMY/LJ028-0227.wav|He certainly mentions details which neither Nabonidus nor Cyrus would care to have appear in their royal records. His story is as follows: +DUMMY/LJ028-0362.wav|for history mentions the names of two of its later rulers. The palace of Nebuchadnezzar was occupied by Alexander the Great, +DUMMY/LJ015-0230.wav|After this nothing remained but to wait for some occasion when the amount transmitted would be sufficient to justify the risks of robbery. +DUMMY/LJ003-0112.wav|When any prisoner committed an offense against the community or against an individual, he was tried by a court in the jail. +DUMMY/LJ004-0121.wav|to which a total of upwards of one hundred thousand prisoners had been committed in the year, only twenty-three prisons were divided according to law; +DUMMY/LJ004-0011.wav|Mr. Neild was compelled to admit in eighteen twelve that "the great reformation produced by Howard +DUMMY/LJ033-0092.wav|the main reason he was going over there that Thursday afternoon when he was to bring back some curtain rods, so I didn't think any more about it when he told me that, end quote, +DUMMY/LJ028-0228.wav|Cyrus, with the first approach of the ensuing spring, marched forward against Babylon. +DUMMY/LJ028-0306.wav|Wait, after that, seven days, and post me another detachment, two thousand strong, at the Nineveh gates; +DUMMY/LJ027-0060.wav|All vertebrates, and none other, have a number of their anterior vertebral joints enlarged and consolidated into a box to form the skull, +DUMMY/LJ045-0106.wav|that he shouldn't come every week, that perhaps it is not convenient for Ruth that the whole family be there, live there, end quote. +DUMMY/LJ021-0042.wav|In this we have had assistance from many bankers and businessmen, +DUMMY/LJ032-0225.wav|and in many instances the details of these pictures differed from the original, and even from each other, particularly as to the configuration of the rifle. +DUMMY/LJ003-0255.wav|The unchecked admission of crowds of visitors to the felons' as well as the debtors' side was another unmixed evil. +DUMMY/LJ007-0137.wav|More attention to ventilation, which was altogether neglected and inadequate, would have secured a better atmosphere for the unhappy inmates +DUMMY/LJ040-0240.wav|and returned to New Orleans where Lee finished the ninth grade before he left school to work for a year. +DUMMY/LJ017-0069.wav|Originally a doctor in practice at Rugeley, in Staffordshire, he had gradually withdrawn from medicine, and devoted himself to the turf; +DUMMY/LJ002-0312.wav|and a calculation made as to the amount that might be expended per week from the whole sum, so that the latter might last out the year. +DUMMY/LJ034-0111.wav|When arrested, he gave his weight as one hundred forty pounds. On other occasions he gave weights of both one hundred forty and one hundred fifty pounds. +DUMMY/LJ037-0183.wav|By checking certain importers and dealers after the assassination of President Kennedy and slaying of Officer Tippit, +DUMMY/LJ044-0159.wav|Thus while Oswald's real intentions, assuming that they were known to himself, are not clear, +DUMMY/LJ028-0189.wav|The fall of Babylon with its lofty walls was a most important event in the history of the ancient world. +DUMMY/LJ024-0028.wav|The statute would apply to all the courts in the federal system. +DUMMY/LJ045-0088.wav|and that Oswald, quote, was not a man, end quote. She also told Mrs. Paine that she was not satisfied with her sexual relations with Oswald. +DUMMY/LJ030-0207.wav|with the follow-up car trailing the President's automobile by approximately five feet. +DUMMY/LJ032-0098.wav|Hidell's name on the post office box application was part of Oswald's use of a nonexistent Hidell +DUMMY/LJ038-0076.wav|Hill said, quote, Captain, we will save you a trip. There he sits, end quote. +DUMMY/LJ028-0328.wav|whom Darius according to agreement had sent first, he fell upon them and slew them all. +DUMMY/LJ019-0387.wav|The Prisons Bill of eighteen seventy-six contemplated the transfer of the prisons to Government. +DUMMY/LJ026-0104.wav|so starch manufactured in the leaves must be digested (dissolved) before it can be transported. +DUMMY/LJ037-0010.wav|A taxi driver, William Scoggins, +DUMMY/LJ038-0096.wav|and killed Patrolman Tippit. Since independent evidence revealed that Oswald repeatedly and blatantly lied to the police, +DUMMY/LJ016-0110.wav|The third, Bell, remained longest at large. He too was run into at a lodging in the Kingsland Road. +DUMMY/LJ009-0096.wav|The poor sheep-stealer is in a frenzy. +DUMMY/LJ013-0204.wav|he admitted that he had been justly convicted, and expressed great anxiety that his fellow-servants should be relieved from all suspicion. +DUMMY/LJ038-0081.wav|Captain Fritz of the homicide and robbery bureau did most of the questioning, but he kept no notes and there were no stenographic or tape recordings. +DUMMY/LJ049-0133.wav|and to emphasize that the task of Presidential protection is one of broad national concern. +DUMMY/LJ044-0072.wav|on which Oswald was heard for less than five minutes on August seventeen, nineteen sixty-three. +DUMMY/LJ014-0043.wav|Hocker's conduct in Newgate while under sentence of death was most extraordinary. +DUMMY/LJ005-0180.wav|It was in these jails, withdrawn from the pressure of authority, that the new rules were invariably ignored. +DUMMY/LJ008-0312.wav|To them entered the chaplain or ordinary of Newgate in full canonicals, +DUMMY/LJ048-0267.wav|at any time while in travel status. +DUMMY/LJ037-0246.wav|The blue jacket was found in the Texas School Book Depository and was identified by Marina Oswald as her husband's. +DUMMY/LJ048-0047.wav|his presence in the School Book Depository job and its location along the route of the motorcade. +DUMMY/LJ006-0231.wav|These fights sometimes took place in the daytime, when a ring was regularly formed, and two or three stood by the door to watch for the officer's approach. +DUMMY/LJ034-0180.wav|The man was dressed in a light-colored, open-neck shirt which could have been either a sports shirt or a T-shirt, +DUMMY/LJ038-0190.wav|three. I paid the house rent on the second so don't worry about it. four. Recently I also paid for water and gas. +DUMMY/LJ048-0123.wav|Agent Lawson's interim report was received by Agent Kellerman on November twenty, the day before departure on the Texas trip. +DUMMY/LJ008-0153.wav|They were accordingly apprehended and brought to trial, the informer, Hanfield by name, being accepted as king's evidence. +DUMMY/LJ006-0018.wav|One was Mr. William Crawford, the other the Rev. Whitworth Russell. +DUMMY/LJ047-0217.wav|He recalled that the special agent in charge of the Dallas office of the FBI, J. Gordon Shanklin, +DUMMY/LJ035-0148.wav|She testified that she heard three shots which she thought came from the building. +DUMMY/LJ008-0309.wav|Its receipt was immediately followed by the promulgation of its contents to the persons most closely concerned, +DUMMY/LJ017-0208.wav|the other perpetrated in a railway-carriage, and showing the promptitude with which criminals accept and utilize altered conditions of life, +DUMMY/LJ003-0327.wav|It will hardly be denied that these proposals went to the root of the matter. +DUMMY/LJ018-0051.wav|man cannot forgive man, only God can do so. Man is therefore only accountable to God. +DUMMY/LJ018-0197.wav|an habitual offender in his own particular line, one who would stick at no trifles to evade detection or escape capture. +DUMMY/LJ016-0273.wav|At Müller's execution there was great competition for front seats, +DUMMY/LJ028-0103.wav|He rebuilt scores of the ancient temples, surrounded many cities with walls, +DUMMY/LJ002-0103.wav|The felons' side has a similar accommodation, and this mode of introducing the beverage is adopted because no publican as such +DUMMY/LJ009-0064.wav|the lately smirking footmen close their eyes and forget their liveries, the ordinary clasps his hands, the turnkeys cry 'Hush!' +DUMMY/LJ010-0264.wav|Later on, however, Dasset was himself seized and interrogated, +DUMMY/LJ025-0142.wav|greatly as these have been extended by modern improvements of the microscope, and that a nerve is, in its essence, +DUMMY/LJ048-0274.wav|However, he felt that any disciplinary action might have given rise +DUMMY/LJ010-0042.wav|A not altogether new crime, however, akin to murder, +DUMMY/LJ042-0167.wav|There can be no mediation between those systems as they exist today, and that person +DUMMY/LJ028-0094.wav|Nebuchadnezzar, King of Babylon, the exalted prince, the favorite of Marduk, the lofty patesi, +DUMMY/LJ028-0058.wav|Early he married Amuhia, a daughter of the Medean king. +DUMMY/LJ013-0021.wav|The larboard pump was suffered to remain choked up, and the long-boat was fitted with tackles and held ready for use at a moment's notice. +DUMMY/LJ039-0079.wav|by Master Sgt. James A. Zahm, noncommissioned officer in charge of the Marksmanship Training Unit in the Weapons Training Battalion +DUMMY/LJ003-0196.wav|five of whom had been condemned the previous July, four in September, and twenty-nine in October. +DUMMY/LJ038-0025.wav|Patrolman M. N. McDonald, with Patrolmen R. Hawkins, T. A. Hutson, and C. T. Walker, entered the theatre from the rear. +DUMMY/LJ015-0275.wav|Burglars brought him the cheques they stole from houses, thieves what they got in pocketbooks. +DUMMY/LJ040-0150.wav|He was reported to have said, quote, I don't want a friend and I don't like to talk to people, end quote, and, quote, I dislike everybody, end quote. +DUMMY/LJ033-0145.wav|May have been used to carry gun. Lt. J. C. Day, end quote. +DUMMY/LJ048-0255.wav|with full possession of his mental and physical capabilities and entirely ready for the performance of his assigned duties. Chief Rowley testified that, +DUMMY/LJ006-0260.wav|From the same source came the two or three strong files which the inspectors found in one ward, +DUMMY/LJ045-0040.wav|They had been married after a courtship of only about six weeks, a part of which Oswald spent in the hospital. +DUMMY/LJ012-0205.wav|an avowed "snatcher" and habitué of the Fortune of War, a public-house in Smithfield frequented openly by men of this awful profession. +DUMMY/LJ018-0337.wav|of Good, and Greenacre, and Catherine Hayes. +DUMMY/LJ037-0033.wav|When questioned by police officers on the evening of November twenty-two, Benavides told them that he did not think that he could identify the man who fired the shots. +DUMMY/LJ029-0153.wav|Under standard procedures, the responsibility for watching the windows of buildings was shared by local police stationed along the route +DUMMY/LJ029-0116.wav|and could not accommodate spectators as conveniently as Harwood Street. +DUMMY/LJ033-0179.wav|since the original bag had been discolored during various laboratory examinations and could not be used for valid identification by witnesses. +DUMMY/LJ028-0238.wav|he then himself drew off with the unwarlike portion of his host, and made for the place where Nitocris dug the basin for the river, +DUMMY/LJ002-0326.wav|The poor debtors were not supplied with beds. Those who could pay the price might hire them from each other, +DUMMY/LJ003-0064.wav|the sleepers covered the entire floor, with the exception of a passage in the middle. +DUMMY/LJ019-0175.wav|Well-founded complaints had been made of the want of heating appliances in the jail. +DUMMY/LJ017-0129.wav|Even after the death sentence had been passed upon him he clung to the hope that the Government would grant him a reprieve. +DUMMY/LJ024-0031.wav|If such a plan is good for the lower courts it certainly ought to be equally good for the highest court from which there is no appeal. +DUMMY/LJ035-0089.wav|No allowance was made for the special conditions which existed on the day of the assassination +DUMMY/LJ028-0104.wav|lined the shores of the rivers with embankments, and spanned the rivers with bridges. +DUMMY/LJ022-0174.wav|for the strengthening of the Interstate Commerce Commission to enable it to carry out a rounded conception of the national transportation system +DUMMY/LJ034-0124.wav|The half-open window, the arrangement of the boxes, and the angle of the shots virtually preclude a standing position. +DUMMY/LJ006-0290.wav|the gatesman who prepared the briefs had interviews with female prisoners alone while taking their instructions; a female came alone and unaccompanied by a matron +DUMMY/LJ014-0063.wav|After the marriage a close intimacy was still maintained between O'Connor and the Mannings. +DUMMY/LJ048-0049.wav|possessed of this information to list Oswald as a potential threat to the safety of the President. +DUMMY/LJ011-0264.wav|Mr. Gee, of Bishop Stortford, who administered the estate of a certain Mr. Canning, deceased. +DUMMY/LJ043-0169.wav|His behavior was entirely consistent with his wife's testimony that, quote, +DUMMY/LJ030-0108.wav|White House communications car, Trade Mart, Love Field, and the Presidential and Vice-Presidential airplanes. +DUMMY/LJ028-0319.wav|"And now," he went on to say, "my coming to you, Babylonians, +DUMMY/LJ002-0192.wav|another two hundred more or less took advantage of "the rules," and lived outside within a circumference of two miles and a half. +DUMMY/LJ011-0086.wav|He then took to forging bank-notes, and was arrested as I have described above. +DUMMY/LJ011-0114.wav|to which he seemed fondly attached, should be used to bandage his eyes, which request was readily granted. +DUMMY/LJ004-0059.wav|They had a right, Mr. Buxton affirmed, to be employed in their own crafts, provided it could be safely followed in prison. +DUMMY/LJ043-0170.wav|I asked him what for he was making all these entries in the book and he answered that he wanted to leave a complete record so that all the details would be in it. +DUMMY/LJ036-0046.wav|She testified, quote, I didn't like his attitude. There was just something about him I didn't like or want him. Just didn't want him around me, end quote, +DUMMY/LJ021-0086.wav|strengthened with the acceptance of those obligations to the public interest which rest upon us all. +DUMMY/LJ009-0197.wav|as was done in the case of Mrs. Phipoe, the murderess, who was executed in front of Newgate in seventeen ninety-eight, +DUMMY/LJ002-0233.wav|Twice a week there was a wine and beer club held at night, which lasted till two or three in the morning. +DUMMY/LJ001-0073.wav|went on apace; and by the end of the sixteenth century there was no really beautiful printing done: +DUMMY/LJ011-0103.wav|a pair of light gray pantaloons, a black stock and a foraging cap. +DUMMY/LJ027-0116.wav|On the other hand, the theory of special creation can only maintain that these rudiments are formed for the sake of adhering to an ideal type. +DUMMY/LJ024-0072.wav|a new and younger judge shall be added to the court automatically. +DUMMY/LJ028-0292.wav|leaping down from his throne he exclaimed aloud and asked Zopyrus who it was that had disfigured him, and what he had done to be so treated. +DUMMY/LJ028-0164.wav|The outer wall is the main defense of the city. +DUMMY/LJ017-0283.wav|This was Duranno; but the sight of the heaving mass of uplifted, impassioned faces +DUMMY/LJ025-0012.wav|extracts from which follow. +DUMMY/LJ008-0242.wav|famous in the annals of Newgate for the hardihood they had displayed in the hour of execution, when they terminated their career of crime at the gallows. +DUMMY/LJ016-0394.wav|The request was not granted, as the old custom of allowing capital convicts whatever they asked for in the way of food has not been the rule in Newgate. +DUMMY/LJ038-0281.wav|in the absence of very definite negative evidence, I think it is permissible to say that in an exhibit such as five seven three +DUMMY/LJ033-0126.wav|that the width was approximately the same. +DUMMY/LJ016-0293.wav|with the rakings of cheap singing-halls and billiard-rooms, the fast young men of London. +DUMMY/LJ001-0113.wav|there is a grossness in the upper finishings of letters like the c, the a, and so on, +DUMMY/LJ010-0085.wav|It was the day on which Lord Harrowby was to entertain his colleagues at dinner in Grosvenor Square. +DUMMY/LJ008-0305.wav|they never ceased cursing until the passion of anger so excited was exchanged for joy in some and grief in others. +DUMMY/LJ034-0210.wav|The picture which gave rise to these allegations was taken by Associated Press Photographer James W. Altgens, +DUMMY/LJ031-0182.wav|Concerned that the local officials might try to prevent the plane's departure, O'Donnell asked that the pilot take off immediately. +DUMMY/LJ047-0218.wav|had discussed the President's visit on several occasions, including the regular biweekly conference on the morning of November twenty-two +DUMMY/LJ009-0042.wav|Upon the occasion which I am describing they were four in number; and here I will continue the narrative in Mr. Wakefield's own words: +DUMMY/LJ031-0179.wav|Despite the protests of these officials, the casket was wheeled out of the hospital, placed in an ambulance, and transported to the airport shortly after two p.m. +DUMMY/LJ044-0065.wav|In another letter to V. T. Lee, dated August seventeen, nineteen sixty-three, +DUMMY/LJ012-0216.wav|Williams, and May had been concerned in the transaction. +DUMMY/LJ038-0200.wav|the city jail is located at the end of the bridge through which we always passed on going to the city (right in the beginning of the city after crossing the bridge). +DUMMY/LJ005-0127.wav|Workshops were built at many prisons, various kinds of manufactures and trades were set on foot, including weaving, matting, shoe-making, and tailoring. +DUMMY/LJ002-0120.wav|had been issued for the arrests of debtors in the kingdom, for sums varying from fourpence to five hundred pounds and upwards. +DUMMY/LJ046-0208.wav|Presidential protection was an important topic in these training programs. +DUMMY/LJ046-0250.wav|The requests shifted the responsibility for evaluating difficult cases from the Service, the agency most responsible for performing that task, +DUMMY/LJ014-0148.wav|From Newgate the Mannings were taken in separate cabs to Horsemonger Lane Jail. +DUMMY/LJ002-0237.wav|This very ancient prison, which stood in the High Street, Southwark, +DUMMY/LJ025-0174.wav|the air, to which the leaves are exposed, contains carbon, +DUMMY/LJ006-0078.wav|A new prisoner's fate, as to location, rested really with a powerful fellow-prisoner. +DUMMY/LJ010-0144.wav|but is only an ordinary commonplace tool. +DUMMY/LJ037-0057.wav|He said that Mrs. Markham started crying when Oswald walked into the lineup room. +DUMMY/LJ006-0190.wav|lively discourse upon past criminal exploits, elaborate discussion of others to be perpetrated after release. +DUMMY/LJ005-0162.wav|it did not peremptorily insist. Too often the letter of the law was observed, but not its spirit. +DUMMY/LJ012-0086.wav|Some difficulty then arose as to gaining admission to the strong room, and it was arranged that a man, May, another Custom House clerk, +DUMMY/LJ010-0306.wav|and the bank became liable for a sum of one hundred seventy thousand pounds. +DUMMY/LJ012-0005.wav|was a blind man named Edwards, who had taken this house in York Street, and who was known to be a frequent visitor at Mrs. Canning's. +DUMMY/LJ042-0137.wav|The dramatic break with society in America now had to be undone. +DUMMY/LJ011-0147.wav|I will include in this part of the present chapter almost one of the last instances of a crime which in time past +DUMMY/LJ001-0002.wav|in being comparatively modern. +DUMMY/LJ018-0265.wav|He carried this character with him into penal servitude, and after enduring the full severity of his punishment for several years, +DUMMY/LJ002-0136.wav|and the disuse of the inferior courts. +DUMMY/LJ004-0054.wav|In order to give greater value to the pamphlet, +DUMMY/LJ050-0076.wav|The problem is aggravated by the necessity that the Service obtain the assistance of local law enforcement officials in evaluating the information which it receives +DUMMY/LJ033-0063.wav|Soon afterward, at about three p.m., police officers arrived and searched the house. +DUMMY/LJ027-0174.wav|Last, the anoura takes first the fish-form, then that of the perennibranch, then that of the caducibranch, and finally becomes anoura. +DUMMY/LJ016-0267.wav|On the contrary, the prevailing sentiment was one of satisfaction at the success of the spectacle. +DUMMY/LJ013-0142.wav|Part of the freight were four hundred rough diamonds valued at four thousand pounds. +DUMMY/LJ002-0321.wav|others who arrived just after the time of distribution were often forty-eight hours without food. The latter might also be six days without meat. +DUMMY/LJ037-0109.wav|He slowed down, halted, said something, and then kept on going to the corner, turned right, and continued west on Jefferson. +DUMMY/LJ007-0198.wav|The windows were to be glazed and painted to prevent prisoners from looking out; +DUMMY/LJ045-0218.wav|Thoughts of his personal difficulties must have been at least partly on his mind when he went to Irving on Thursday night and told his wife that he was lonely, +DUMMY/LJ038-0224.wav|Another statement which limits the time when it could have been written is the reference, quote, you and the baby, end quote, +DUMMY/LJ005-0110.wav|and so passing by the roof down into the garden and on to freedom. +DUMMY/LJ008-0201.wav|in an adjoining house, that of an undertaker, was Lord Alfred Paget, also with several friends. +DUMMY/LJ020-0070.wav|Bit of soda as large as a pea, dissolved in hot water. +DUMMY/LJ029-0098.wav|This route, eventually selected for the motorcade from the airport to the Trade Mart, measured ten miles and could be driven easily within the allotted forty-five minutes. +DUMMY/LJ029-0018.wav|Vice President Johnson, and Governor Connally on June fifth, nineteen sixty-three, at the Cortez Hotel in El Paso, Texas. +DUMMY/LJ004-0242.wav|Some attempt was made to reduce the overcrowding, on the recommendation of the House of Commons Committee of eighteen eighteen, but this applied only a partial remedy. +DUMMY/LJ033-0203.wav|We found no brown cotton fibers, no green viscose fibers, and no woolen fibers. +DUMMY/LJ008-0286.wav|The inspectors very properly desired to call attention to the inevitable tendency in this mode of dealing with "the most awful sanctions of the law," +DUMMY/LJ012-0294.wav|without hesitation brought in a verdict of willful murder. +DUMMY/LJ016-0161.wav|Having carefully spread out a blanket on the floor just below the hammock as it lay suspended, +DUMMY/LJ009-0298.wav|which sets forth that the petitioner had been at great expense by sending clerks and agents to Liverpool and Shrewsbury to hire an executioner. +DUMMY/LJ035-0153.wav|As Oswald passed Mrs. Reid she said, quote, Oh, the President has been shot, but maybe they didn't hit him, end quote. +DUMMY/LJ007-0160.wav|the employment of those means and agents by which the lowest passions and the most vulgar propensities of man are perpetually kept in the highest state of excitement +DUMMY/LJ002-0271.wav|The compters of Ludgate, Giltspur Street, and the Borough were discontinued as debtors' prisons (as was Newgate also) +DUMMY/LJ031-0022.wav|Charles R. Baxter, Robert N. McClelland, Ronald C. Jones; the chief neurologist, Dr. William Kemp Clark; +DUMMY/LJ018-0199.wav|to watch and prosecute coiners, that he received information that coining was carried on by Cummings and others at a place in Westminster. +DUMMY/LJ008-0165.wav|Here a couple of piemen had been selling their wares; the basket of one of them, which was raised upon a four-legged stool, was upset. +DUMMY/LJ044-0019.wav|Following his arrest, he was interviewed by the police, and at his own request, by an agent of the FBI. +DUMMY/LJ042-0136.wav|His attempt to renounce his citizenship had been an open expression of hostility against the United States and a profound rejection of his early life. +DUMMY/LJ028-0266.wav|he drew together all his power and began the war by marching straight upon Babylon and laying siege to the place. +DUMMY/LJ018-0361.wav|tampered with, and returned to the shop. +DUMMY/LJ032-0222.wav|Moreover, Shaneyfelt testified that in his opinion the photographs were not composites of two different photographs +DUMMY/LJ037-0126.wav|located on the southeast corner of this intersection. +DUMMY/LJ042-0139.wav|Marina Oswald confirmed the fact that her husband was experiencing psychological difficulties at the time of his return. +DUMMY/LJ017-0155.wav|Miss Bankes seems to have counted upon some species of whitewashing, no less than the repudiation of the other marriage, +DUMMY/LJ002-0238.wav|was used for debtors arrested for the lowest sums within twelve miles of the palace of Whitehall; +DUMMY/LJ001-0095.wav|These and similar founts, cast by the above firm and others, +DUMMY/LJ004-0005.wav|The pleadings, however eloquent, of a single individual were unable to more than partially remedy the widespread and colossal evils of British prisons. +DUMMY/LJ047-0145.wav|The CIA message was sent also to the Department of State where it was reviewed by personnel of the Passport Office, who knew from Oswald's file +DUMMY/LJ004-0117.wav|In an early report of the Prison Discipline Improvement Society, +DUMMY/LJ019-0269.wav|More animal food was given than was necessary. +DUMMY/LJ016-0219.wav|He expressly bequeathed them to Calcraft, who was graciously pleased to accept them. +DUMMY/LJ047-0030.wav|as a marine radar technician and had displayed an arrogant and aggressive attitude at the U.S. Embassy; +DUMMY/LJ003-0301.wav|It recommended day cells or rooms for regular labor, which should be compulsory upon all transports and prisoners sentenced to hard labor, +DUMMY/LJ033-0181.wav|The science of paper analysis enabled him to distinguish between different rolls of paper even though they were produced by the same manufacturer. +DUMMY/LJ009-0018.wav|and informed that the public would not in future be admitted to hear the condemned sermon. "I was also informed," writes Mr. Cotton, +DUMMY/LJ023-0015.wav|to exact a pound of flesh was more sacred than the main objectives of the Constitution to establish an enduring Nation. +DUMMY/LJ032-0064.wav|It is not known whether the application for post office box two nine one five listed "A. Hidell" as a person entitled to receive mail at this box. +DUMMY/LJ009-0148.wav|But the chaplain admitted that the solitude of the convict's cell +DUMMY/LJ009-0271.wav|had attempted to throw himself upon the fire in his condemned cell. +DUMMY/LJ012-0050.wav|He was about to emigrate to New South Wales, when he was arrested at Islington and committed to Newgate on a charge of receiving stolen goods. +DUMMY/LJ041-0020.wav|or you were going to really have one, because he wasn't going to take anything from anybody, end quote. +DUMMY/LJ049-0141.wav|The foremost assignment of the Committee would be to insure that the maximum resources of the Federal Government are fully engaged in the job of protecting the President, +DUMMY/LJ003-0143.wav|The entrance fee was at least thirteen shillings, six pence a head, with half-a-crown a week more for bed and bedding, +DUMMY/LJ021-0151.wav|But I shall ask both groups to give a fair trial to peaceful methods of adjusting their conflicts of opinion and interest, +DUMMY/LJ012-0240.wav|I will close this chapter with a brief account of another murder, +DUMMY/LJ003-0297.wav|It admitted the paramount necessity +DUMMY/LJ032-0240.wav|By Sunday, March thirty-one, nineteen sixty-three, +DUMMY/LJ019-0336.wav|His disciplinary powers were defined by the act, and his duties, +DUMMY/LJ002-0084.wav|those on the first floor were rather larger on account of a set-off in the wall; and the uppermost were the largest, for the same reason. +DUMMY/LJ008-0249.wav|Blinded with their long hair, they tore at each other like two furies; their bonnets and caps were trodden underfoot in the kennel, +DUMMY/LJ026-0151.wav|Hence oxygen must be supplied to the cells. +DUMMY/LJ042-0017.wav|and see for himself how a revolutionary society operates, a Marxist society. +DUMMY/LJ035-0168.wav|One of the police officers assigned to the corner of Elm and Houston Streets for the Presidential motorcade, W. E. Barnett, +DUMMY/LJ005-0042.wav|It made the subject of the newly-invented tread-wheels, or stepping-wheels, as they were at first called, its peculiar affair, +DUMMY/LJ009-0017.wav|For this the chaplain was a few days later summoned before the jail committee of aldermen, +DUMMY/LJ047-0205.wav|to potential danger, quote, +DUMMY/LJ009-0019.wav|that this resolution was in consequence of their (the aldermen's) disapproving of the last discourse delivered by me, +DUMMY/LJ005-0165.wav|They were so radically bad that reform seemed hopeless, and it was thought wiser not to bring them under provisions which clearly could not be enforced. +DUMMY/LJ007-0146.wav|vaunting his own adventures, or listening to those of others; +DUMMY/LJ002-0162.wav|He recites another most outrageous and extraordinary case, in which one John Bird, +DUMMY/LJ018-0064.wav|afterwards in the British German Legion raised for the Crimean War. +DUMMY/LJ017-0013.wav|so called after its inventress, a Roman woman named Toffana, and which was so widely adopted by ladies anxious to get rid of their husbands, +DUMMY/LJ032-0241.wav|ten days prior to the Walker attempt, Oswald had undoubtedly received the rifle shipped from Chicago on March twenty, +DUMMY/LJ012-0075.wav|No doubt, on the removal of Ikey Solomons from the scene, his mantle fell upon worthy successors. +DUMMY/LJ003-0219.wav|Well might Mr. Bennet write that the condition of the condemned side was the most prominent of the manifold evils in the present system of Newgate, +DUMMY/LJ001-0097.wav|a little reduced in ugliness. The design of the letters of this modern "old style" leaves a good deal to be desired, +DUMMY/LJ018-0133.wav|which gave him access to all parts of the mills, the packing-room included. +DUMMY/LJ015-0233.wav|A weight of shot was bought and stowed in carpet bags ready to replace exactly the abstracted gold. +DUMMY/LJ040-0208.wav|Carro reported that when questioned about his mother Lee said, quote, +DUMMY/LJ021-0134.wav|It is time that we made a clean-cut effort to bring about that united action of management and labor, +DUMMY/LJ002-0255.wav|A court of the collegians was held every Monday to manage its affairs, at which all prisoners were required to attend. +DUMMY/LJ040-0122.wav|a junior high school in the Bronx, where the other children apparently teased him because of his, quote, western clothes and Texas accent. +DUMMY/LJ014-0217.wav|Dwyer managed to overpower his assailant, and got to his feet; but Cannon butted at him with his head, and again threw him to the ground, +DUMMY/LJ008-0167.wav|No one who fell ever rose again. +DUMMY/LJ009-0060.wav|Now his doom is sealed irrevocably, and he has given himself up to despair. +DUMMY/LJ028-0018.wav|Like a great ball of fire the sun sinks in the west. +DUMMY/LJ035-0174.wav|where upon Barnett posted himself at the front door to make certain that no one left the building. +DUMMY/LJ007-0139.wav|Again, the discipline commonly deemed inseparable from every place of durance was entirely wanting. +DUMMY/LJ018-0253.wav|For these crimes William Roupell was tried at the Central Criminal Court on the twenty-fourth September, eighteen sixty-two. +DUMMY/LJ007-0021.wav|"she had never witnessed such scenes before, and hopes she never shall again -- it was dreadful!" +DUMMY/LJ004-0048.wav|Mrs. Fry's brother, Mr. Samuel Hoare, Junior, was chairman of the committee, on which also served many noted members of the Society of Friends +DUMMY/LJ039-0159.wav|The ammunition used by the assassin was manufactured by Western Cartridge Co. of East Alton, Illinois. +DUMMY/LJ021-0001.wav|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt, Section six. +DUMMY/LJ030-0044.wav|Motorcycles. -- Next came four to six motorcycle policemen whose main purpose was to keep the crowd back. +DUMMY/LJ017-0026.wav|Yet it was clearly proved that the dumplings contained arsenic, that she, and she alone, had made the dough, +DUMMY/LJ021-0095.wav|We are now prepared to move into this second phase, on the basis of our experience in the first phase +DUMMY/LJ007-0031.wav|all classes of the condemned, those certain to suffer, and the larger number who were nearly certain of a reprieve, +DUMMY/LJ020-0102.wav|A wise accommodation of parts and angles, and compactness in the adjustment of "must-be-dones" +DUMMY/LJ032-0061.wav|as was a change-of-address card dated May twelve, nineteen sixty-three, +DUMMY/LJ047-0173.wav|and that she found this and similar statements illogical and somewhat amusing. On this occasion Hosty was at the Paine residence for only a few minutes. +DUMMY/LJ042-0092.wav|No evidence has been found that they used him for any particular propaganda or other political or informational purposes. +DUMMY/LJ009-0142.wav|Contemporary accounts give a minute description of the demeanor of the convict upon this solemn occasion. +DUMMY/LJ009-0193.wav|with the idea that ignominy would no longer attach to an operation which ceased to be compulsory for the most degraded beings; +DUMMY/LJ029-0002.wav|Chapter two. The Assassination: Part one. +DUMMY/LJ015-0173.wav|Warrants were issued for Redpath's arrest, but he had flown to Paris. +DUMMY/LJ007-0092.wav|he was placed in a situation "beyond all others calculated to confirm his malady and prolong his sufferings." +DUMMY/LJ036-0032.wav|So I gave her a transfer and opened the door and she was going out the gentleman I had picked up about two blocks [back] +DUMMY/LJ030-0009.wav|Late in the afternoon he flew to Houston where he rode through the city in a motorcade, +DUMMY/LJ035-0084.wav|The minimum time required by Baker to park his motorcycle and reach the second-floor lunchroom was within three seconds of the time needed to walk +DUMMY/LJ020-0078.wav|Do all this at bedtime. Early in the morning turn out the dough upon a floured board, work it for a minute into manageable shape; +DUMMY/LJ013-0211.wav|"What are you doing here?" asked his lordship. +DUMMY/LJ031-0049.wav|Dr. Carrico felt the President's back and determined that there was no large wound there which would be an immediate threat to life. +DUMMY/LJ010-0084.wav|The day appointed for the murder and rising actually arrived before the authorities interfered. +DUMMY/LJ037-0083.wav|Barbara Jeanette and Virginia Davis viewed a group of four men in a lineup and each one picked Oswald as the man who crossed their lawn while emptying his pistol. +DUMMY/LJ037-0196.wav|A. J. Hidell, aged twenty-eight, end quote. The date of the order was January twenty-seven. No year shown. +DUMMY/LJ011-0286.wav|For three hours he struggled without success with his bonds, +DUMMY/LJ039-0122.wav|There is some possibility that the rifle he was firing might not have been as good a rifle as the rifle that he was firing in his A course firing, +DUMMY/LJ038-0083.wav|They occasionally participated in the questioning. +DUMMY/LJ024-0106.wav|Two groups oppose my plan on the ground that they favor a constitutional amendment. +DUMMY/LJ019-0199.wav|and the costliness of enlarging it, forbade all idea of entirely reconstructing the jail so as to constitute it a model prison. +DUMMY/LJ027-0132.wav|A dozen volumes would be necessary +DUMMY/LJ018-0124.wav|where are Messrs. Portal's mills for the manufacture of bank-note paper. +DUMMY/LJ011-0253.wav|Mullay defended himself, and managed to break the knife, but not before he had cut himself severely. +DUMMY/LJ036-0181.wav|On Friday, November twenty-two, about one p.m., he entered the house in unusual haste. +DUMMY/LJ018-0003.wav|As the circumstances under which it was perpetrated were somewhat novel, +DUMMY/LJ022-0038.wav|That makes it particularly necessary for the Vice- President and for me to conceive of our duty toward the entire country. +DUMMY/LJ019-0262.wav|The jail authorities often gave the highest, possibly undue, importance to the value of remunerative employment, +DUMMY/LJ037-0253.wav|looked like the jacket he was wearing but, quote, I thought it had a little more tan to it, end quote. +DUMMY/LJ003-0288.wav|The above-mentioned report was ordered to be printed upon the ninth May. +DUMMY/LJ025-0110.wav|and subject to exceptions. +DUMMY/LJ022-0047.wav|However, for the first time in five years the relief rolls have declined instead of increased during the winter months. +DUMMY/LJ048-0190.wav|If such instructions were in fact given, they were not effectively carried out. +DUMMY/LJ046-0034.wav|with particular attention to the diverse roles which the President is expected to fill. +DUMMY/LJ008-0041.wav|The criminals are not exposed to view till they mount the fatal stage. The last part of the stage, or that next to the jail, +DUMMY/LJ034-0135.wav|two of these employees leaving the building and immediately identified them as having been in the fifth-floor windows. +DUMMY/LJ024-0132.wav|I am in favor of action through legislation: +DUMMY/LJ050-0039.wav|the Secret Service has largely been the passive recipient of threatening communications to the President +DUMMY/LJ042-0082.wav|End quote. As his, quote, principal reason, end quote, for renouncing his citizenship +DUMMY/LJ019-0083.wav|The discipline also varied greatly, from the severely penal to the culpably lax. +DUMMY/LJ046-0214.wav|members of his immediate family, the President-Elect, and the Vice-President is within the exclusive jurisdiction of the U.S. Secret Service. +DUMMY/LJ046-0238.wav|it contained the names of no persons from the entire Dallas-Fort Worth area, +DUMMY/LJ044-0124.wav|Oswald wrote that he felt that he might have compromised the FPCC and expressed concern lest, quote, Our opponents could use my background +DUMMY/LJ044-0097.wav|Stuckey thought Oswald, quote, appeared to be a very logical, intelligent fellow, end quote, and, quote, was arrested by his cleancutness, end quote. +DUMMY/LJ005-0293.wav|In these places the prisoners are often without a proper supply of air and light; frequently the jails are mere dungeons under the town hall. +DUMMY/LJ020-0039.wav|If rudely shaken or jarred, there will be heavy streaks in loaves. +DUMMY/LJ014-0098.wav|The house was empty, bare and stripped of all its furniture, and its former occupants had decamped. +DUMMY/LJ021-0089.wav|and have effected a reorganization of the N.R.A. +DUMMY/LJ044-0067.wav|Latin American Focus, end quote, +DUMMY/LJ002-0198.wav|The office of marshal had been hereditary, +DUMMY/LJ007-0132.wav|The inspectors, however, honestly admitted that although the site of the prison was convenient, its construction was as bad as bad could be. +DUMMY/LJ019-0122.wav|Mr. Pearson seems to have taken for granted that all prisoners would behave well in his district prisons. +DUMMY/LJ030-0026.wav|Governor and Mrs. Connally and Senator Ralph W. Yarborough had come with the President from Fort Worth. +DUMMY/LJ011-0252.wav|locked the door, and attacked Mullay violently with the trap-ball bat and knife. +DUMMY/LJ047-0243.wav|According to Revill, Hosty indicated that he was going to tell this to Lieutenant Wells of the homicide and robbery bureau. +DUMMY/LJ024-0053.wav|Fundamentally, if in the future, America cannot trust the Congress it elects to refrain from abuse of our Constitutional usages +DUMMY/LJ034-0040.wav|not too long, end quote, a time had elapsed between the time that the print was placed on the carton and the time that it had been developed by the Dallas police. +DUMMY/LJ030-0078.wav|Presidential Assistants David F. Powers and Kenneth O'Donnell sat in the right and left jump seats, respectively. +DUMMY/LJ024-0092.wav|There is no substantial groups within the Congress or outside it who are agreed on any single amendment. +DUMMY/LJ010-0110.wav|Several of the other prisoners took the same line as regards Edwards, +DUMMY/LJ009-0170.wav|to fifty-six in eighteen thirty-nine. +DUMMY/LJ037-0108.wav|From across the street Callaway yelled, quote, Hey, man, what the hell is going on? End quote. +DUMMY/LJ016-0200.wav|without favoring father or mother or any other relation or friend. +DUMMY/LJ040-0100.wav|she would sometimes take Lee with her, apparently leaving him alone in the car while she transacted her business. +DUMMY/LJ015-0065.wav|Robson was of humble origin, but he was well educated, and he had some literary abilities. +DUMMY/LJ019-0394.wav|It was only filled at the periods when the sessions of the Central Criminal Court were in progress; +DUMMY/LJ014-0095.wav|On the third day his friends began to inquire for him, +DUMMY/LJ016-0431.wav|One was vexed to think that his victims had a finer funeral than he would have. +DUMMY/LJ038-0071.wav|When he entered the homicide and robbery bureau office, he saw two detectives standing there with Sgt. Gerald L. Hill, +DUMMY/LJ010-0052.wav|which grew out of disturbed social conditions after the last French war, amidst general distress, +DUMMY/LJ040-0164.wav|The possibility of commitment was to be considered only if the probation plan was not successful. +DUMMY/LJ018-0151.wav|The police officers now taxed Mrs. Campbell with complicity, and frightened her into collusion. +DUMMY/LJ048-0076.wav|and maintaining liaison with other Federal Agencies. This liaison should take into consideration +DUMMY/LJ036-0007.wav|In deciding whether Oswald killed Patrolman Tippit the Commission considered the following: +DUMMY/LJ013-0174.wav|The bundle found in the dining-room contained, with clothes, various small articles of plate and jewelery which a thief would probably have put into his pocket. +DUMMY/LJ019-0322.wav|On the other hand, new and careful regulations were framed to secure the moral and material well-being of the inmates of the jails. +DUMMY/LJ010-0026.wav|There was no increase in murder or manslaughter; the number of such crimes remained pretty constant proportionately to population. +DUMMY/LJ008-0239.wav|The loud shout of the multitude once more subsided, or only fell upon the abstracted ear like the dreamy murmur of an ocean shell. +DUMMY/LJ039-0100.wav|During the first week of an intensive eight-week training period he received instruction in sighting, aiming, and manipulation of the trigger. +DUMMY/LJ029-0125.wav|The only practical way for westbound traffic on Main Street +DUMMY/LJ010-0022.wav|the slackness of supervision over wholly irresponsible agents, produced many terrible monetary catastrophes, +DUMMY/LJ006-0158.wav|The act prescribed that he should do both every twenty-four hours, but days passed without his entering the wards. +DUMMY/LJ038-0261.wav|the Dallas Police Department forwarded it on December two, nineteen sixty-three. +DUMMY/LJ013-0155.wav|Courvoisier's case made a great stir. There was unusual atrocity in this murder of an aged, infirm gentleman, +DUMMY/LJ014-0273.wav|The detection of these frauds came while he was still prominently before the world as the lessee of the Olympic. +DUMMY/LJ043-0014.wav|Oswald was, quote, very, very disagreeable and disappointed, end quote, +DUMMY/LJ022-0105.wav|These agencies, therefore, will simply be doing on a somewhat enlarged scale the same sort of things that they have been doing. +DUMMY/LJ015-0064.wav|been afforded peculiar facilities by the slackness of system and the want of methodical administration in the concerns by which they were employed. +DUMMY/LJ037-0110.wav|Guinyard claimed that the man ran down the east side of Patton and passed within ten feet of him before crossing to the other side. +DUMMY/LJ041-0077.wav|He led them into discussions of foreign affairs about which they often knew less than he did, since he had apparently devoted considerable time to a study of such matters. +DUMMY/LJ006-0306.wav|Some of the prisoners had their valets, and all these were constantly in and out of the kitchen where this female prisoner was employed. +DUMMY/LJ027-0157.wav|is thus a condensed resume of the life history of its ancestral species. +DUMMY/LJ010-0186.wav|There is no occasion to use violence. I will go with you. +DUMMY/LJ002-0331.wav|and they were not in great demand, in order to convert them into mop-heads or cleaning-rags. +DUMMY/LJ016-0063.wav|Below he met another woman and a girl, both of whom were terrified at his appearance, but +DUMMY/LJ028-0290.wav|he came in this plight before Darius. +DUMMY/LJ016-0183.wav|The story goes, that about eighteen twenty-eight +DUMMY/LJ043-0171.wav|I am guessing that perhaps he did it to appear to be a brave man in case he were arrested, but that is my supposition, end quote. +DUMMY/LJ010-0017.wav|eluding the keenest vigilance, and carried out with reckless daring; +DUMMY/LJ010-0273.wav|He was eventually charged with a misdemeanor, the capital charge having been abandoned, and committed for trial. +DUMMY/LJ007-0190.wav|Next year we find the inspectors expressing their satisfaction that "the full and faithful exposure which we felt it our duty to make of Newgate +DUMMY/LJ014-0101.wav|he found that it was soft and new, while elsewhere it was set and hard. +DUMMY/LJ005-0022.wav|While fully admitting the good intentions of the Society, he condemned their ultra humanitarianism as misplaced. +DUMMY/LJ050-0216.wav|In its budget request for the fiscal year beginning July one, nineteen sixty-four, +DUMMY/LJ044-0119.wav|into which Oswald, in his own words, had, quote, thrown himself. He sought advice from the central committee of the Communist Party, U.S.A., +DUMMY/LJ028-0118.wav|He built well the quays along the opposite shore of the Euphrates, but he did not finish all his work, +DUMMY/LJ045-0016.wav|On October twenty-third, I had attended a ultra-right meeting headed by General Edwin A. Walker, who lives in Dallas. +DUMMY/LJ009-0043.wav|First is a youth of eighteen, condemned for stealing in a dwelling-house goods valued above five pounds. +DUMMY/LJ019-0302.wav|In eighteen sixty-two there were in all one hundred and ninety-three jails in England and Wales; +DUMMY/LJ016-0235.wav|He was so much in favor of short drops that his immediate successor, Marwood, stigmatized him as "short-drop" man. +DUMMY/LJ013-0089.wav|Thither Elder went alone, provided with a number of canvas and one large carpet-bag. +DUMMY/LJ039-0242.wav|(two) brought this rifle into the Depository Building on the morning of the assassination, +DUMMY/LJ048-0031.wav|While he had expressed hostility at times toward the State Department, the Marine Corps, and the FBI as agents of the Government, +DUMMY/LJ046-0244.wav|PRS should have investigated and been prepared to guard against it. +DUMMY/LJ026-0018.wav|Plants are organisms of constantly varying form in which the cell body is surrounded by a cellulose wall; +DUMMY/LJ018-0263.wav|and left the dock with evident satisfaction and relief at the termination of a most painful ordeal. +DUMMY/LJ017-0243.wav|but Vartos, or Watto, the Turk, would not allow any but the eight mutineers to have anything. +DUMMY/LJ032-0024.wav|After searching their records from ten p.m. to four a.m. +DUMMY/LJ012-0218.wav|At first nothing peculiar was found; but at a second search the back-garden ground was dug up, +DUMMY/LJ012-0046.wav|The second house was in Lower Queen Street, Islington, and he used it for some time as a depot for valuables. +DUMMY/LJ018-0120.wav|A more elaborate plot in many ways, more secretly, more patiently prepared than the preceding, or indeed than any in the calendar, +DUMMY/LJ001-0038.wav|while in fourteen seventy at Paris Udalric Gering and his associates turned out the first books printed in France, also in Roman character. +DUMMY/LJ044-0117.wav|that there were people who understood his activity, end quote. +DUMMY/LJ010-0217.wav|Soon after he was introduced we were alarmed by a violent knocking at the door; +DUMMY/LJ048-0046.wav|his trip to Mexico where he was in contact with Soviet authorities, +DUMMY/LJ007-0222.wav|Much bitterness of feeling is evidently engendered, and the corporation grows more and more angry with the inspectors. +DUMMY/LJ012-0121.wav|A part of the "swag" stolen from the bonded warehouse was thus recovered, but it was supposed that a number of the stolen notes had perished in the fire. +DUMMY/LJ011-0150.wav|Miss Turner was a school-girl of barely fifteen, +DUMMY/LJ043-0087.wav|He indicated, however, that he would have to delay that program because of his immediate financial needs and responsibilities. +DUMMY/LJ019-0147.wav|this occurred in summer at eight, but in the winter months it took place at dusk, and was often as early as four or five. +DUMMY/LJ047-0180.wav|he had received a copy of the report of the New Orleans office which contained Agent Quigley's memorandum of the interview in the New Orleans jail on August ten, +DUMMY/LJ027-0099.wav|Another striking class of the facts of morphology which admit of scientific explanation only along the line of homology +DUMMY/LJ005-0286.wav|still under the borough corporations. The Commons' Committee gave it as their opinion that they were in a deplorable state. +DUMMY/LJ037-0247.wav|Marina Oswald also identified Commission Exhibit Number one sixty-two, the jacket found by Captain Westbrook, as her husband's second jacket. +DUMMY/LJ038-0017.wav|Attracted by the sound of the sirens, Mrs. Postal stepped out of the box office and walked to the curb. +DUMMY/LJ046-0130.wav|In nineteen forty-three PRS received approximately nine thousand items of information; +DUMMY/LJ002-0149.wav|The latter indeed hung like millstones round the neck of the unhappy insolvent wretches who found themselves in limbo. +DUMMY/LJ025-0018.wav|and inanimated beings, which are devoid of these functions and simply vegetable. +DUMMY/LJ037-0080.wav|Barbara Jeanette Davis assumed that he was emptying his gun as, quote, he had it open and was shaking it, end quote. She immediately called the police. +DUMMY/LJ042-0086.wav|but he also displayed some sensitivity at not having reached a higher rank in the Marine Corps. +DUMMY/LJ016-0279.wav|They collected over night in the bright light of a November moon. +DUMMY/LJ015-0177.wav|Great excitement prevailed in the city and the West End when Redpath's defalcations were made public. +DUMMY/LJ041-0166.wav|in honor of some retiring noncommissioned officers, an event which they both approached with little enthusiasm. +DUMMY/LJ028-0080.wav|yet they took an impression of it, and returned it to its owner. +DUMMY/LJ030-0192.wav|What are they doing to you! end quote. Looking back from the front seat, +DUMMY/LJ040-0120.wav|Lee refused to discuss the matter with Pic, whom he had previously idolized, and their relations were strained thereafter. +DUMMY/LJ019-0388.wav|This bill, reintroduced in eighteen seventy-seven, became law that year, after which the whole of the prisons, including Newgate, +DUMMY/LJ038-0197.wav|ten. I left you as much money as I could, +DUMMY/LJ035-0092.wav|and this would be the minimum, because I am sure that I, you know, it took me a little longer, end quote, +DUMMY/LJ006-0125.wav|It was not strange that he should sometimes misuse his power, and when prisoners were not to be cajoled into securing his legal services, +DUMMY/LJ048-0044.wav|the knowledge of his defection, his arrogance and hostility to the United States, +DUMMY/LJ007-0212.wav|and a tendency to return to much of that laxity and remissness which formerly marked the management of this prison." +DUMMY/LJ014-0013.wav|He wanted a light for a cigar, and got it from a lantern which was lifted up and fully betrayed his features. +DUMMY/LJ023-0038.wav|The courts, however, have cast doubts on the ability of the elected Congress to protect us against catastrophe +DUMMY/LJ041-0197.wav|as excuses for his difficulties in getting along in the world, which were usually caused by entirely different factors. +DUMMY/LJ014-0213.wav|Dwyer found Cannon bleeding profusely from a wound in the head, and persuaded him to go to a doctor's. +DUMMY/LJ049-0062.wav|Their primary responsibility is to stay with and protect the President. +DUMMY/LJ034-0049.wav|so no conclusions can be drawn as to whether these remaining prints preceded or followed the print developed in Dallas by powder. +DUMMY/LJ040-0099.wav|When she worked for a time as an insurance saleslady, +DUMMY/LJ036-0131.wav|And about that time an old lady, I think she was an old lady, I don't remember nothing but her sticking her head down past him in the door and said, +DUMMY/LJ012-0133.wav|The fraudulent messenger, by the help of young Caspar, established his claim to the boxes, paid the wharfage dues, and carried off the gold-dust. +DUMMY/LJ039-0058.wav|The Commission evaluated (one) the nature of the shots, (two) Oswald's Marine training in marksmanship, +DUMMY/LJ023-0091.wav|I have not the time to quote to you all the language used by dissenting justices in many of these cases. +DUMMY/LJ029-0094.wav|Lawson was not specifically instructed to select the parade route, but he understood that this was one of his functions. +DUMMY/LJ020-0040.wav|Graham bread is wholesome and sweet, and ought to be eaten frequently in every family, particularly by young people whose bones and teeth are in forming. +DUMMY/LJ009-0268.wav|through the improper adjustment of the rope his sufferings were prolonged for five minutes; +DUMMY/LJ018-0369.wav|The mysterious Bravo case, that of Dr. Lamson, and that of Kate Dover +DUMMY/LJ037-0034.wav|As a result, they did not take him to the police station. +DUMMY/LJ040-0079.wav|John Pic testified that he thought Lee found in Ekdahl the father that he never had. +DUMMY/LJ004-0128.wav|capable of accommodating only eight thousand five hundred and forty-five persons, as many as thirteen thousand and fifty-seven were crowded. +DUMMY/LJ044-0009.wav|While it appears that he may have distributed Fair Play for Cuba Committee materials on one uneventful occasion in Dallas sometime during the period +DUMMY/LJ047-0084.wav|and his application was approved on the following day. +DUMMY/LJ040-0123.wav|He began to stay away from school, preferring to read magazines and watch television at home by himself. +DUMMY/LJ016-0127.wav|Along this Krapps crawled, and then dropped down on to the cook-house. +DUMMY/LJ032-0199.wav|the Oswalds lived on Neely Street in Dallas in a rented house which had a small back yard. +DUMMY/LJ050-0024.wav|toward the preparation of formal understandings of the respective roles of the Secret Service and other agencies with which it collaborates +DUMMY/LJ044-0177.wav|Representing himself as the head of the New Orleans branch of the, quote, organization called Fair Play for Cuba, end quote, +DUMMY/LJ037-0072.wav|She has subsequently admitted the existence of the conversation and offered an explanation for her denial. +DUMMY/LJ038-0084.wav|The reports prepared by those present at these interviews are set forth in appendix eleven. +DUMMY/LJ007-0220.wav|to corrupt the prisoner charged with an offense of which he may be innocent, and to confirm in guilt the young and inexperienced offender. +DUMMY/LJ008-0197.wav|of the Lamb's Coffee House; +DUMMY/LJ044-0120.wav|in a letter dated August twenty-eight, nineteen sixty-three, about whether he could, quote, continue to fight, +DUMMY/LJ026-0003.wav|Nutrition thus, as has been pointed out, makes it possible to classify most organisms as animals or plants. +DUMMY/LJ039-0151.wav|Accuracy of Weapon +DUMMY/LJ033-0180.wav|Cadigan found that the paper used to make this replica sack had different characteristics from the paper in the original bag. +DUMMY/LJ039-0123.wav|because he may well have carried this rifle for quite some time, and it got banged around in normal usage. +DUMMY/LJ044-0142.wav|of each of his changes of mailing address. +DUMMY/LJ028-0245.wav|and mounting upon the walls along both sides of the stream, would so have caught the enemy as it were in a trap. +DUMMY/LJ049-0083.wav|it has long been a Federal crime to conspire to injure any Federal officer, on account of, or while he is engaged in, the lawful discharge of the duties of his office. +DUMMY/LJ010-0271.wav|A visit to the father's residence was followed by the arrest of the son, who had by this time returned. +DUMMY/LJ016-0247.wav|while round about were shoe-strings, boot-laces, and lasts. Marwood, strange to say, followed the same trade as Calcraft. +DUMMY/LJ022-0163.wav|This legislation will serve the investor by ending the conditions which have caused that lack of confidence and goodwill. +DUMMY/LJ034-0048.wav|The freshness of prints developed in this manner cannot be estimated, +DUMMY/LJ019-0097.wav|This want of uniformity in prison discipline became ere long an acknowledged evil pressing for some remedy, +DUMMY/LJ040-0068.wav|It forced Marguerite Oswald to go to work to provide for her family. +DUMMY/LJ011-0238.wav|Shops were broken into, as when that of Grimaldi and Johnson, in the Strand, was robbed of watches to the value of six thousand pounds. +DUMMY/LJ011-0082.wav|he came to London, where he practiced as a professional swindler and cheat. +DUMMY/LJ016-0150.wav|Suicides at Newgate were numerous enough, but they seldom possessed any novel or unusual features; +DUMMY/LJ005-0167.wav|which became the four George the fourth cap. sixty-four, said that he had abstained from legislating for these small jurisdictions "on mature deliberation." +DUMMY/LJ008-0069.wav|The latter is the method still followed at Newgate. +DUMMY/LJ040-0129.wav|during which time he was examined by its Chief Psychiatrist, Dr. Renatus Hartogs, +DUMMY/LJ025-0132.wav|just as the body of a snail contracts into its shell when one of its "horns" is irritated. +DUMMY/LJ032-0141.wav|The print arrived in the FBI Laboratory in Washington on November twenty-nine, mounted on a card on which Lieutenant Day had written the words +DUMMY/LJ016-0207.wav|It was not always easy to get a hangman so cheap, as I have already indicated on a previous page. +DUMMY/LJ044-0183.wav|to obtain a Soviet visa at the Soviet Embassy in Mexico City, he insisted that he was entitled to the Cuban visa because of his background, +DUMMY/LJ037-0218.wav|Marina Oswald testified that this was the holster which contained the revolver in the photographs taken on Neely Street. +DUMMY/LJ018-0387.wav|The flower had berries also of bread fixed on stems made from the fiber drawn from the stuffing of his mattress, +DUMMY/LJ016-0154.wav|When a bit of rope carefully secreted, +DUMMY/LJ007-0194.wav|the fixing of "inspection holes" in the doors and walls, so as to insure more supervision; of windows opening into the well-holes, +DUMMY/LJ050-0055.wav|that the PRS files can no longer be limited largely to persons communicating actual threats to the President. +DUMMY/LJ009-0267.wav|When Luigi Buranelli was executed in eighteen fifty-five, +DUMMY/LJ036-0021.wav|On the basis of this punchmark, which was distinctive to each Dallas driver, +DUMMY/LJ048-0073.wav|The FBI Manual of Instructions provided, quote, Liaison With Other Government Agencies +DUMMY/LJ049-0177.wav|and Robert I. Bouck, who was in charge of the Protective Research Section of the Secret Service, believed that the accumulation of the facts known to the FBI +DUMMY/LJ004-0205.wav|Industrial labor had also been introduced with satisfactory results. +DUMMY/LJ041-0096.wav|Normally, it would be a good type of literature; and the one that I recall was "Leaves of Grass," by Walt Whitman, end quote. +DUMMY/LJ039-0167.wav|Using the assassination rifle mounted with the telescopic sight, three marksmen, rated as master by the National Rifle Association, +DUMMY/LJ045-0056.wav|and also about Mrs. Paine. +DUMMY/LJ048-0202.wav|End quote. Captain Lawrence was not instructed to have his men watch buildings along the motorcade route and did not mention the observation of buildings to them. +DUMMY/LJ045-0247.wav|His commitment to Marxism and communism appears to have been another important factor in his motivation. +DUMMY/LJ008-0063.wav|while the criminals stand upon it, by two pieces of timber, which are made to slide in and out of the prison wall through apertures made for that purpose. +DUMMY/LJ025-0033.wav|since it was unnecessary, and therefore is absent, in the more simple animals. +DUMMY/LJ015-0310.wav|which had received so perverted and mistaken direction, +DUMMY/LJ016-0429.wav|Others talk freely enough on various topics, but principally upon their own cases. +DUMMY/LJ002-0309.wav|There was this much honest forbearance in Newgate in these days, +DUMMY/LJ015-0003.wav|A firm which held a lot of warrants suddenly demanded the delivery of the goods they covered. +DUMMY/LJ042-0090.wav|I'm sure Russians will accept me after this sign of my faith in them, end quote. +DUMMY/LJ012-0204.wav|They presented themselves about noon one day at the dissecting room of King's College Hospital, accompanied by a third man, +DUMMY/LJ017-0254.wav|Taffir, the mate, finding there was a man who could speak English at another place twenty miles off, +DUMMY/LJ046-0118.wav|In addition to this function, PRS is responsible for such tasks +DUMMY/LJ028-0230.wav|A battle was fought at a short distance from the city, in which the Babylonians were defeated by the Persian King, +DUMMY/LJ028-0079.wav|The museum authorities believed that the cameo was one of the many spurious objects which the Eastern forgers were constantly sending to Europe, +DUMMY/LJ004-0110.wav|It was made incumbent upon the justices to provide distinct places of confinement for five classes of prisoners, viz. +DUMMY/LJ002-0004.wav|The new jail fell as far short of the demands made on it as did the old. +DUMMY/LJ018-0268.wav|A daring and cleverly-planned robbery of diamonds was that of the Tarpeys, man and wife, +DUMMY/LJ020-0060.wav|turning the pan once in this time, and covering with clean -- never printed -- paper, should they brown too fast. +DUMMY/LJ047-0209.wav|that he had owned a weapon and did a good deal of hunting or use of it, perhaps in Russia, plus a number of items about his disposition and unreliability of character, +DUMMY/LJ041-0128.wav|Even though Oswald apparently did not express any hostility against the President or Governor Connally, +DUMMY/LJ003-0312.wav|The jailer should be required to visit every part and see every prisoner daily; the chaplain should perform service, visit the sick, +DUMMY/LJ028-0129.wav|built three walls round about the inner city, and three others about that which was the outer; and this he did with burnt brick. +DUMMY/LJ043-0129.wav|implied that Oswald's fellow employees did not like him because he was propagandizing +DUMMY/LJ012-0108.wav|There was a hunt for the two well-known house-breakers, who were eventually heard of at a lodging in Kennington. +DUMMY/LJ012-0169.wav|The murder was still fresh in the memory of the populace, and Probert was all but lynched on his way to jail. +DUMMY/LJ018-0364.wav|It will be remembered that she made a statement which led to the empaneling of a jury of matrons, who decided that there was no cause for an arrest of judgment. +DUMMY/LJ039-0066.wav|In addition, the three degree downward slope of Elm Street was of assistance in eliminating at least some of the adjustment +DUMMY/LJ001-0013.wav|than in the same operations with ugly ones. +DUMMY/LJ022-0054.wav|The first is to make provisions intended to relieve, to minimize, and to prevent future unemployment; +DUMMY/LJ037-0147.wav|It was the unanimous and unequivocal testimony of expert witnesses before the Commission that these used cartridge cases were fired from the revolver +DUMMY/LJ032-0167.wav|He concluded, quote, There is no doubt in my mind that these fibers could have come from this shirt. +DUMMY/LJ005-0017.wav|tending to divest punishment of its just and salutary terrors; +DUMMY/LJ036-0127.wav|He saw a man walking south on Lamar from Commerce. +DUMMY/LJ003-0087.wav|These often remained in Newgate for six months, sometimes a year, in some cases longer; +DUMMY/LJ018-0282.wav|In eighteen seventy-one they crossed the Atlantic, +DUMMY/LJ030-0146.wav|Just prior to the shooting, David F. Powers, riding in the Secret Service follow-up car, remarked to Kenneth O'Donnell +DUMMY/LJ035-0111.wav|Truly was probably correct in stating that the west elevator was on the fifth floor when he looked up the elevator shaft from the first floor. +DUMMY/LJ023-0005.wav|I take this means of saying "thank you." +DUMMY/LJ004-0063.wav|and therefore you are bound to furnish him with moderate indeed but suitable accommodation." +DUMMY/LJ030-0123.wav|Special Agent Glen A. Bennett once left his place inside the follow-up car to help keep the crowd away from the President's car. +DUMMY/LJ006-0097.wav|Water might not be taken into the ward for washing purposes. +DUMMY/LJ045-0104.wav|Marina Oswald said that she knew her husband did not like Michael Paine and so she asked him not to come out that weekend, even though he wanted to do so. +DUMMY/LJ033-0119.wav|Moreover, in an interview on December one, nineteen sixty-three, with agents of the FBI, Frazier had marked the point on the back seat of his car +DUMMY/LJ041-0090.wav|It wasn't anything extreme. I wouldn't go as far as to call it, call him a paranoid, but a definite tendency there was in that direction, I think, end quote. +DUMMY/LJ002-0243.wav|with jurisdiction to hold pleas in all actions within the prescribed limits. The court was chiefly used for the recovery of small debts under ten pounds +DUMMY/LJ010-0164.wav|Oxford, who was only nineteen at the time his offense was committed, had been born at Birmingham, +DUMMY/LJ042-0088.wav|that he might know something of special interest. Oswald's "Historic Diary" describes the event in part as follows, quote, +DUMMY/LJ043-0109.wav|In New Orleans he obtained work as a greaser and oiler of coffee processing machines for the William B. Reily Co., +DUMMY/LJ030-0253.wav|and not be diverted unless it is necessary to accomplish the protective assignment. +DUMMY/LJ040-0031.wav|who were accorded special privileges and who he thought were betraying communism, and he spoke well of the United States. +DUMMY/LJ020-0018.wav|Throw a cloth over the bowl and set by for five or six hours to rise. +DUMMY/LJ012-0020.wav|He began as an itinerant street vendor at eight, at ten he passed bad money, +DUMMY/LJ004-0224.wav|So plentiful was this dietary, that although the old permission remained in force of allowing the friends of prisoners to bring them supplies from outside, +DUMMY/LJ027-0142.wav|the antlers of an existing deer furnish in their development a kind of "resume," or recapitulation, of the successive phases +DUMMY/LJ037-0245.wav|This jacket belonged to Lee Harvey Oswald. Marina Oswald stated that her husband owned only two jackets, one blue and the other gray. +DUMMY/LJ035-0003.wav|In considering whether Oswald was at the southeast corner window at the time the shots were fired, +DUMMY/LJ031-0201.wav|Given a choice between the National Naval Medical Center at Bethesda, Maryland, and the Army's Walter Reed Hospital, +DUMMY/LJ035-0175.wav|The sergeant did the same thing at the rear of the building. +DUMMY/LJ004-0193.wav|Prisoners on reception were treated as they are now-a-days -- bathed, dressed in prison clothes, and inspected by the surgeon. +DUMMY/LJ006-0116.wav|The wardsman had a monopoly in supplying provisions, gave dinner and breakfast at his own price, and was such complete master of the ward +DUMMY/LJ015-0180.wav|was densely crowded, and many curious eyes were turned upon the somewhat remarkable man who occupied the dock. +DUMMY/LJ015-0084.wav|till the defalcations from fraudulent transfers and fraudulent issues combined amounted to twenty-seven thousand pounds. +DUMMY/LJ009-0130.wav|winked at other prisoners in derision of what was taking place; and I have frequently heard men and lads who had been of the kneeling party +DUMMY/LJ043-0044.wav|The latter testified that they, quote, were just together again, end quote, as if his brother, quote, had not been to Russia, end quote. +DUMMY/LJ043-0011.wav|and attempted to help Mrs. Oswald particularly, in various ways. In general, Oswald did not like the members of the Russian community. +DUMMY/LJ012-0262.wav|whom she was to have married, and in whose company she had left her own lodgings to visit his in Camberwell. +DUMMY/LJ039-0152.wav|It will be recalled from the discussion in chapter three +DUMMY/LJ030-0145.wav|showed the numerals twelve:thirty as the Vice-Presidential automobile proceeded north on Houston Street, a few seconds before the shots were fired. +DUMMY/LJ018-0062.wav|Christian Sattler was by birth a German. +DUMMY/LJ010-0268.wav|It happened that a lad named Bean had absconded from his father's home some weeks before, +DUMMY/LJ025-0065.wav|Even as late as eighteen forty-five, however, a botanist of Schleiden's eminence dealt very skeptically with these statements, +DUMMY/LJ025-0007.wav|so much so as to require discussion in different treatises. +DUMMY/LJ044-0134.wav|His brief foray on behalf of the Fair Play for Cuba Committee had failed to win any support. +DUMMY/LJ006-0034.wav|They attended early and late; they mustered the prisoners, examined into their condition, +DUMMY/LJ040-0007.wav|the existence of some personal grievance, +DUMMY/LJ048-0115.wav|to determine what matters require attention in making advance preparations and to decide what action to take. +DUMMY/LJ030-0199.wav|The Governor was lying with his head on his wife's lap when he heard a shot hit the President. +DUMMY/LJ012-0165.wav|As this was accomplished beyond the limits of the metropolis, and its perpetrators arraigned at Hertford, +DUMMY/LJ038-0226.wav|Oswald had apparently mistaken the county jail for the city jail. +DUMMY/LJ008-0227.wav|slipshod and slovenly, in crushed bonnet and dirty shawl, the gown fastened by a single hook, +DUMMY/LJ045-0183.wav|He had argued with her over the use of an alias and had not called her after that argument, although he usually telephoned once or twice a day. +DUMMY/LJ002-0059.wav|those who had turned king's evidence, whose safety might have been imperiled had they been lodged with the men against whom they had informed. +DUMMY/LJ028-0167.wav|In the one stood the palace of the kings, surrounded by a wall of great strength and size; +DUMMY/LJ012-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section fifteen: Newgate notorieties, part three +DUMMY/LJ019-0233.wav|and when it was completed, both sides of the prison were brought into harmony with modern ideas. +DUMMY/LJ003-0333.wav|But here the recommendations touched at once upon the delicate subject of expense, and it is clear that the committee hesitated on this score. +DUMMY/LJ037-0082.wav|On the evening of November twenty-two, +DUMMY/LJ050-0149.wav|PRS must develop the capacity to classify its subjects on a more sophisticated basis than the present geographic breakdown. +DUMMY/LJ042-0079.wav|My request for citizenship is now pending before the Supreme Soviet of the U.S.S.R. I take these steps for political reasons. +DUMMY/LJ002-0194.wav|and a heavy fee at the rate of eight pounds per one hundred pounds, with four pounds for every additional hundred. +DUMMY/LJ005-0064.wav|is found now-a-days to be really efficacious. The main recommendations, however, are based upon common sense +DUMMY/LJ005-0118.wav|In many others there were no infirmaries, no places set apart for the confinement of prisoners afflicted with dangerous and infectious disorders. +DUMMY/LJ024-0089.wav|let us examine the process. +DUMMY/LJ002-0126.wav|five thousand, seven hundred nineteen writs and executions for debts between ten pounds and twenty pounds were issued in Middlesex, +DUMMY/LJ014-0170.wav|This preference brought the costly stuff into disrepute, and its unpopularity lasted for nearly thirty years. +DUMMY/LJ028-0087.wav|Such was the appearance of the builder of the walls of Babylon. +DUMMY/LJ028-0273.wav|not even when he tried the means by which Cyrus had made himself master of the place. +DUMMY/LJ028-0176.wav|Should we compare these ancient descriptions of the walls, we should find them hopelessly conflicting. +DUMMY/LJ038-0254.wav|She testified that several days later Oswald recovered his rifle and brought it back to their apartment. +DUMMY/LJ006-0147.wav|it was still freely introduced into the prison. +DUMMY/LJ015-0200.wav|only when single and unprotected were they in any danger of attack, and that but rarely. +DUMMY/LJ026-0004.wav|Yet there are many unicellular forms in which both kinds of nutrition go on at the same time; +DUMMY/LJ031-0051.wav|He noted contusions, hematoma to the right of the larynx, which was deviated slightly to the left, +DUMMY/LJ010-0167.wav|He left his last situation in April eighteen forty, and established himself in lodgings in Lambeth, +DUMMY/LJ017-0002.wav|As these records draw to a close, the crimes I chronicle become so much more recent in date that they will be fresh in the memory of most of my readers. +DUMMY/LJ025-0079.wav|the males of most Rotifers have no digestive apparatus. +DUMMY/LJ010-0139.wav|After the five bodies had hung for half-an-hour, a man in a mask came forward to complete the sentence. +DUMMY/LJ046-0175.wav|If PRS concludes after investigation +DUMMY/LJ034-0052.wav|In his independent investigation, Arthur Mandella of the New York City Police Department +DUMMY/LJ017-0065.wav|I ought also to confess the others: I mean my wife and my brother's. End quote. +DUMMY/LJ027-0159.wav|"It is well known," likewise comments Le Conte, +DUMMY/LJ006-0091.wav|Their treatment was also a matter of chance. They still slept on rope mats on the floor, herded together in companies of four or more to keep one another warm +DUMMY/LJ005-0143.wav|The "scenes of tumult and obscenity" in these night rooms are said to have been beyond description; a prisoner in one nocturnal riot lost an eye. +DUMMY/LJ017-0081.wav|sought about for a fresh victim to supply him with funds. +DUMMY/LJ012-0102.wav|The lock and fastenings of the chest were forced by means of a "jack," an instrument known to housebreakers, +DUMMY/LJ017-0112.wav|Palmer too showed an indecent haste in preparing the body for interment, and in obtaining the usual certificate. +DUMMY/LJ040-0148.wav|Oswald's withdrawn tendencies and solitary habits were thought to be the result of, quote, +DUMMY/LJ015-0056.wav|Mr. Strahan was arrested at a friend's house in Bryanston Square. +DUMMY/LJ035-0009.wav|Baker was riding a two-wheeled motorcycle behind the last press car of the motorcade. +DUMMY/LJ038-0173.wav|General Walker hired two investigators to determine whether a former employee might have been involved in the shooting. +DUMMY/LJ050-0190.wav|However, the instructions must be communicated to the local police in any event and can be leaked to the press whether or not they are in writing. +DUMMY/LJ037-0112.wav|Apparently he had reached for his gun; it lay beneath him outside of the holster. +DUMMY/LJ037-0086.wav|Her reaction when she saw Oswald in the lineup was that, quote, +DUMMY/LJ002-0229.wav|and the turnkeys slept in the prison, yet scenes of riot, drunkenness, and disorder were most prevalent, end quote. +DUMMY/LJ045-0245.wav|Oswald's search for what he conceived to be the perfect society was doomed from the start. +DUMMY/LJ002-0205.wav|A man dying of liver complaint lay on the cold stones without a bed or food to eat. +DUMMY/LJ009-0255.wav|The miserable wretch during the whole of this time was struggling with the most determined violence, to the great horror of the spectators. +DUMMY/LJ016-0378.wav|and after sentence. +DUMMY/LJ012-0273.wav|But Greenacre in his confession pretended that he and his intended had quarreled over her property or the want of it, +DUMMY/LJ028-0377.wav|until at last the foundations of the temples and palaces were buried fully a hundred feet beneath the surface. +DUMMY/LJ004-0229.wav|Convicts were not even compelled to wear irons, providing they behaved well. +DUMMY/LJ032-0069.wav|The person having access to the box then takes the notice to the window and is given the package. +DUMMY/LJ040-0071.wav|From the time Marguerite Oswald returned to work until December twenty-six, nineteen forty-two, when Lee too was sent to the orphans' home, +DUMMY/LJ005-0053.wav|A very valuable volume published by the Society +DUMMY/LJ007-0050.wav|during the whole of which time the unhappy convicts who had but little hope of commutation were exposed to the mockery of their reckless associates. +DUMMY/LJ029-0076.wav|On November four, Gerald A. Behn, agent in charge of the White House detail, asked Sorrels to examine three potential sites for the luncheon. +DUMMY/LJ041-0040.wav|Despite this apparent interest in communism, Oswald tried to join the Marines when he was sixteen years old. +DUMMY/LJ014-0139.wav|and would have left the dock had not Mr. Cope, the governor of Newgate, restrained her. +DUMMY/LJ046-0075.wav|But his very position as representative of the people prevents him from effectively shielding himself from the people. +DUMMY/LJ032-0011.wav|(four) the killing of Dallas Patrolman J. D. Tippit within forty-five minutes after the assassination, (five) +DUMMY/LJ026-0123.wav|which could be derived by the ordinary chemical evolution of protoplasm, proteid, sugar, starch or fats. +DUMMY/LJ024-0033.wav|and that a baneful precedent will be established. +DUMMY/LJ021-0097.wav|we shall watch carefully the working of this new machinery for the second phase of N.R.A., +DUMMY/LJ035-0130.wav|Actually she noticed no one on the back stairs. If she descended from the fourth to the first floor as fast as she claimed in her testimony, +DUMMY/LJ039-0089.wav|in relation to the target as opposed to iron sights with aligning the sights and then aligning them on the target, end quote. +DUMMY/LJ039-0217.wav|Based on these tests the experts agreed that the assassination rifle was an accurate weapon. +DUMMY/LJ007-0169.wav|this picture of the existing state of Newgate +DUMMY/LJ015-0255.wav|and begged his accomplice to invest it as a settlement on a woman named Kay, by whom he had had a child. +DUMMY/LJ019-0086.wav|or they might herd together or communicate freely as in the old worst days. They might see each other when they liked, and converse sotto voce, +DUMMY/LJ004-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section seven: The beginnings of prison reform. +DUMMY/LJ041-0060.wav|Kerry Thornley, a marine associate, +DUMMY/LJ005-0130.wav|There were tread-wheels at most of the prisons, and regular employment thereon or at some other kind of hard labor. +DUMMY/LJ020-0095.wav|Cover your hair entirely with a handkerchief or sweeping cap. +DUMMY/LJ047-0184.wav|Well, as I had previously stated, I have between twenty-five and forty cases assigned to me at any one time. I had other matters to take care of. +DUMMY/LJ013-0241.wav|The policeman insisted on searching the premises, at which Good displayed some uneasiness. +DUMMY/LJ047-0208.wav|been courtmartialed for illegal possession of a gun, of a handgun in the Marines, +DUMMY/LJ050-0252.wav|The occasional use of personnel from other Federal agencies to assist in protecting the President has a further advantage. It symbolizes the reality +DUMMY/LJ034-0163.wav|The Commission is satisfied that, at the least, +DUMMY/LJ002-0308.wav|nor was non-payment made a pretext for further imprisonment, thanks to the humanity of the jailer, or the funds provided by various charities. +DUMMY/LJ013-0088.wav|the cashier gave them eight Bank of England notes for one thousand pounds each, saying that they could get so much specie nowhere else. +DUMMY/LJ027-0177.wav|Then in the Carboniferous, fishes still continuing, there appeared the lowest -- i.e., most fish-like forms of amphibians. +DUMMY/LJ019-0130.wav|since admirably developed in the convict prisons of this country. +DUMMY/LJ017-0251.wav|that they had been in the boats five days and nights, but that the captain and others had been lost. +DUMMY/LJ049-0042.wav|At that time the agents on the framing boards of the follow-up car were expected to perform such a function. +DUMMY/LJ017-0076.wav|His brother was supposed to have been his next victim, upon whose life he had also effected an insurance for another thirteen thousand pounds. +DUMMY/LJ019-0135.wav|There was no longer, or at worst but rarely, and for short periods, the same overcrowding. +DUMMY/LJ047-0108.wav|During the interview Quigley obtained background information from Oswald which was inconsistent with information already in the Bureau's possession. +DUMMY/LJ006-0108.wav|He charged a weekly sum as ward dues for the use of knives, forks, and plates +DUMMY/LJ045-0044.wav|Many of the people with whom the Oswalds became acquainted after their arrival in the United States thought that Marina Oswald had married her husband primarily in the hope +DUMMY/LJ041-0127.wav|Oswald must have already learned that the Governor could not help him with his discharge because he was no longer Secretary of the Navy, at the time he made that remark. +DUMMY/LJ011-0268.wav|when he was asked to meet a Mr. Heath in London with regard to the sale of certain lands at Bishop Stortford. +DUMMY/LJ042-0123.wav|A "patriotic duty" to bring in the harvest. The opinions of the workers (unvoiced) are that it's a great pain in the neck: +DUMMY/LJ027-0103.wav|Thus, for instance, the unborn whale has rudimentary teeth, +DUMMY/LJ012-0029.wav|But he never paid more than a fixed price for all articles of the same class, whatever their intrinsic value. +DUMMY/LJ019-0331.wav|Baths were provided, ablutions ordered, and all appliances to insure personal cleanliness. +DUMMY/LJ010-0112.wav|He had been in a state of abject misery, and when he first joined "the reformers," as the Cato Street conspirators called themselves, +DUMMY/LJ003-0221.wav|Yet it must have been abundantly plain to the reader that the other evils existing were great and glaring. A brief summary of them will best prove this. +DUMMY/LJ006-0065.wav|were associated together, "of every variety of age, habit, and delinquency, without employment, oversight, or control." +DUMMY/LJ003-0079.wav|Mr. Bennet refers to a gentleman confined for want of bail, who occupied a room with five others +DUMMY/LJ038-0216.wav|The references to house rent and payments for water and gas +DUMMY/LJ018-0075.wav|while at sea the captain of the ship was responsible for the security of the prisoner. +DUMMY/LJ022-0117.wav|Our attack upon these enemies must be without stint and without discrimination. +DUMMY/LJ012-0185.wav|raised the alarm, and suspicion fell upon the three murderers, who were arrested. +DUMMY/LJ016-0155.wav|braces, shoe-strings, shirt torn into strips are the only instruments, and a bar or small hook +DUMMY/LJ016-0407.wav|who generally attended the prison services. +DUMMY/LJ039-0106.wav|which was two points above the minimum for qualifications as a "sharpshooter" in a scale of marksman, sharpshooter, expert. +DUMMY/LJ037-0101.wav|According to his testimony, Smith told the FBI, quote, It looked more like him than it did on television, end quote, +DUMMY/LJ013-0040.wav|The other Wallace, the sailor, also traced his lapse into crime to evil counsel. +DUMMY/LJ024-0059.wav|It is the clear intention of our public policy to provide for a constant flow of new and younger blood into the judiciary. +DUMMY/LJ046-0172.wav|the Secret Service arranges for the family and friends of the individual, and local law enforcement officials, +DUMMY/LJ009-0227.wav|At Jarrow Stake, where the Durham murderer's body was exposed, there were similar scenes, mingled with compassion for the culprit's family, +DUMMY/LJ020-0046.wav|Close the dough over it, dust your hands and kneading-board with flour and work in the shortening until the dough is elastic and ceases to be sticky. +DUMMY/LJ025-0055.wav|by the employment of instruments of precision for the measurement of the physical forces which are at work in the living economy. +DUMMY/LJ041-0121.wav|It is thus clear that Oswald knew that Governor Connally was never directly concerned with his discharge +DUMMY/LJ048-0122.wav|long enough before his departure to apprise him of any particular problems encountered and the responsive action taken. +DUMMY/LJ037-0258.wav|There is no doubt, however, that Oswald was seen leaving his roominghouse at about one p.m. wearing a zipper jacket, +DUMMY/LJ014-0004.wav|brought passers-by and policemen to the spot, a lonely place near a dead wall beyond Belsize Hall, Hampstead, +DUMMY/LJ050-0052.wav|Based on the Commission's investigation, the following minimum goals for improvements are indicated: +DUMMY/LJ005-0037.wav|It sought to obtain and diffuse useful information +DUMMY/LJ022-0103.wav|The national government now has at least sixty different agencies with the staff +DUMMY/LJ003-0251.wav|Among other excuses offered for thus manacling all almost without exception, was that it was the best and safest method +DUMMY/LJ032-0223.wav|and that Oswald's face had not been superimposed on another body. +DUMMY/LJ043-0176.wav|If the attack had succeeded and Oswald had been caught, the pictures showing him with his rifle +DUMMY/LJ002-0202.wav|The prison was always in "the most filthy state imaginable." +DUMMY/LJ005-0058.wav|an architect of eminence who was very largely employed in the erection of prison buildings at the close of the last century. +DUMMY/LJ040-0087.wav|that she could not afford to send them back to the military school and she asked Pic to quit school entirely to help support the family. +DUMMY/LJ041-0193.wav|I think he would then throw himself entirely on the other thing he also wanted, which was the image in history. +DUMMY/LJ035-0128.wav|If her estimate of time is correct, she reached the bottom of the stairs before Truly and Baker started up, +DUMMY/LJ007-0101.wav|when a petition was presented against the return of Messrs. Adam Dundas and Fitzroy Kelly. Various witnesses, including Messrs. J. B. Dasent, +DUMMY/LJ042-0001.wav|Report of the President's Commission on the Assassination of President Kennedy. +DUMMY/LJ003-0104.wav|Having been committed to Clerkenwell, +DUMMY/LJ041-0183.wav|Delgado and Oswald talked more about Cuba than Russia, and sometimes imagined themselves as leaders in the Cuban Army or Government, +DUMMY/LJ001-0175.wav|Simple as this proposition is, it is necessary to be stated, +DUMMY/LJ008-0287.wav|to make those sanctions an object of contemptuous mockery. +DUMMY/LJ029-0044.wav|Lawson's most important responsibilities were to take preventive action against anyone in Dallas considered a threat to the President, +DUMMY/LJ045-0168.wav|Answer: I said it would be better if I remained with Ruth until the holidays, he would come, and we would all meet together. +DUMMY/LJ019-0398.wav|Accordingly, it was ordered by Sir William Harcourt, the present Secretary of State, that Newgate should cease to be used as a regular prison, +DUMMY/LJ011-0039.wav|but in a separate chamber, that belonging to one of the warders of the jail. +DUMMY/LJ007-0051.wav|The brutal callousness of the bulk of the inmates of the press-yard may be gathered from the prison punishment-book, which frequently recorded such entries as the following: +DUMMY/LJ010-0108.wav|The trial of the conspirators came on some six weeks later, at the Old Bailey. +DUMMY/LJ016-0128.wav|He now put in requisition the rope made of the sheets, and with its help lowered himself into the street. +DUMMY/LJ005-0070.wav|These are principles fully recognized now-a-days, and it may fairly be conceded that the Prison Discipline Society's ideal +DUMMY/LJ027-0112.wav|How, then, are they to be accounted for? +DUMMY/LJ024-0070.wav|I now propose that we establish by law an assurance against any such ill-balanced court in the future. +DUMMY/LJ003-0305.wav|The provision of more baths was also suggested, and the daily sweeping out of the prison. +DUMMY/LJ021-0046.wav|there could be no recovery of public confidence in the security of savings. +DUMMY/LJ003-0101.wav|must have had a tendency to turn them into the world hardened and accomplished in the ways of vice and crime. End quote. +DUMMY/LJ007-0042.wav|so that men in the most awful situation, daily expecting to be hanged, +DUMMY/LJ023-0081.wav|asserted a power to veto laws passed by the Congress and state legislatures in complete disregard of this original limitation. +DUMMY/LJ036-0195.wav|He was described by Chief Curry as having the reputation of being "a very fine, dedicated officer." +DUMMY/LJ047-0231.wav|some indication that the person planned to take some action against the safety of the President of the United States or the Vice President. End quote. +DUMMY/LJ006-0150.wav|and then only went round to count the number. +DUMMY/LJ018-0390.wav|But there were few cases so remarkable as the great ones already recorded. +DUMMY/LJ019-0153.wav|Some attempt at supervision was exercised by the night watchman stationed on the leads, who might hear what went on inside. +DUMMY/LJ037-0026.wav|By this time the pickup truck was across the street and about twenty-five feet from the police car. +DUMMY/LJ016-0330.wav|It was read for the first time in March eighteen sixty-six, but did not become law till eighteen sixty-eight. +DUMMY/LJ015-0029.wav|In December eighteen fifty-one the balance sheet showed a deficiency of upwards of seventy thousand pounds. +DUMMY/LJ034-0199.wav|According to Euins, however, as the man lowered his head in order to aim the rifle down Elm Street, he appeared to have a white bald spot, on his head. +DUMMY/LJ016-0012.wav|the instruments needed for the purpose never reached him. +DUMMY/LJ036-0065.wav|Oswald lived at ten twenty-six North Beckley. +DUMMY/LJ018-0002.wav|In July eighteen sixty-four occurred the murder of Mr. Briggs, a gentleman advanced in years and chief clerk in Robarts' bank. +DUMMY/LJ010-0076.wav|In this way he formed the acquaintance of Watson and others, with whom he was arraigned for treasonable practices, and imprisoned. +DUMMY/LJ005-0104.wav|The progress of improvement was certainly extremely slow. +DUMMY/LJ020-0068.wav|One even tablespoonful of white sugar. +DUMMY/LJ001-0154.wav|the result as measured by the eye being that the lower margin is less than the top one, and that the whole opening has an upside-down look vertically +DUMMY/LJ025-0015.wav|in which the question is treated with that comprehensiveness of knowledge and clear critical judgment which characterize his writings +DUMMY/LJ010-0211.wav|The society was supposed to meet regularly, and its proceedings, together with the speeches made, were duly recorded. +DUMMY/LJ022-0123.wav|It should be remembered that in every big job there are some imperfections. +DUMMY/LJ007-0166.wav|Strong as were the foregoing remarks, the inspectors wound up their report in still more trenchant language +DUMMY/LJ013-0201.wav|The prisoner's demeanor had greatly changed during the trial. +DUMMY/LJ009-0251.wav|White managed to keep his wrists asunder, and continued to struggle with the officials for some time. Eventually he was pinioned with a cord in the usual manner. +DUMMY/LJ013-0133.wav|a number of pawnbrokers' duplicates, and three small files. It was proved at the trial that Ashley had asked his landlady for brick-dust and leather, +DUMMY/LJ037-0123.wav|and the Commission has inquired into general lineup procedures used by the Dallas police as well as the specific procedures in the lineups involving Oswald. +DUMMY/LJ039-0002.wav|Report of the President's Commission on the Assassination of President Kennedy. +DUMMY/LJ009-0034.wav|The sheriffs were in one gallery; +DUMMY/LJ048-0116.wav|Agent Lawson was not given written instructions concerning the Dallas trip or advice about any peculiar problems which it might involve; +DUMMY/LJ011-0137.wav|the Attorney-General introduced a bill to abolish capital punishment entirely for forgery. +DUMMY/LJ012-0288.wav|and was seen to be busily engaged in washing down the house with bucket and mop. +DUMMY/LJ017-0042.wav|Palmer's trial caused the most intense excitement. +DUMMY/LJ008-0150.wav|Four years passed without the detection of the murderers, +DUMMY/LJ026-0073.wav|being the constant transpiration (evaporation) of watery vapor from the leaves, especially through the stomata. +DUMMY/LJ040-0044.wav|Oswald apparently started reading about communism when he was about fifteen. +DUMMY/LJ009-0253.wav|Then with a strong effort he pushed off the white cap, and tried to liberate his neck from the halter, which by this time had been adjusted. +DUMMY/LJ044-0164.wav|primarily for purposes of self-advertising. +DUMMY/LJ022-0193.wav|by a sound and rational ordering of its various elements and wise provisions for the protection of the weak against the strong. +DUMMY/LJ044-0219.wav|you could tell what they wanted you to do by reading between the lines, reading the thing and doing a little reading between the lines, end quote. +DUMMY/LJ036-0005.wav|He arrived at approximately one p.m. and left a few minutes later. +DUMMY/LJ003-0045.wav|more particularly as regarded the classification of prisoners, and which were dependent on the space to be gained by the removal of the debtors, +DUMMY/LJ027-0150.wav|through which their long line of ancestors had passed. +DUMMY/LJ042-0081.wav|is made only after the longest and most serious considerations. I affirm that my allegiance is to the Union of Soviet Socialist Republics. +DUMMY/LJ043-0066.wav|Some of his acquaintances, feeling that Oswald tried to impress people with the fact that he had lived and worked in Russia, were led to the belief +DUMMY/LJ013-0146.wav|A Jew named Benjamin effected the sale to certain merchants named Blogg and Martin, +DUMMY/LJ028-0054.wav|He continued the building operations of his father, until Babylon became the greatest city of its age, +DUMMY/LJ001-0087.wav|for the seventeenth-century letters are at least pure and simple in line. The Italian, Bodoni, and the Frenchman, Didot, +DUMMY/LJ026-0127.wav|These processes must take place everywhere in ordinary protoplasm, +DUMMY/LJ049-0182.wav|In addition, the Commission has concluded that the Secret Service particularly tends to be the passive recipient of information +DUMMY/LJ032-0248.wav|It appears from his wife's testimony +DUMMY/LJ042-0010.wav|it appears that personal and psychological factors were also involved. +DUMMY/LJ012-0264.wav|a warrant was issued for his apprehension, which was effected at Kennington on the twenty-fourth March. +DUMMY/LJ026-0092.wav|animals (and colorless plants as well) apparently could not long exist. +DUMMY/LJ047-0182.wav|was quite interested in determining the nature of his contact with the Soviet Embassy in Mexico City, end quote. +DUMMY/LJ009-0201.wav|Surgeons' Hall being let to the lieutenancy of the county for the accommodation of the militia. +DUMMY/LJ007-0113.wav|were revived for the convenience of these gentlemen, whose incarceration was thus rendered as little like imprisonment as possible. +DUMMY/LJ008-0109.wav|As we crossed the press yard a cock crew, and the solitary clanking of a restless chain was dreadfully horrible. +DUMMY/LJ003-0114.wav|was appointed judge, and a towel tied in knots was hung on each side in imitation of a wig. +DUMMY/LJ033-0141.wav|A person seated on the fourth carton could assemble the rifle without being seen from the rest of the sixth floor +DUMMY/LJ006-0249.wav|On one occasion a disturbance was raised which was not quelled until windows had been broken and forms and tables burnt. +DUMMY/LJ014-0323.wav|The warrant thus represented money, and was often used as such, being endorsed and passed from hand to hand as other negotiable bills. +DUMMY/LJ018-0095.wav|The police suspecting the house in York Buildings, put a watch on the premises, which they kept up for more than a year, +DUMMY/LJ030-0179.wav|She watched as he slumped down with an empty expression on his face. +DUMMY/LJ010-0311.wav|in other words, he forged powers of attorney, and proceeded to realize securities lodged in his bank under various names. +DUMMY/LJ049-0210.wav|for examples to a determination of the optimum organization of the President's protection. +DUMMY/LJ003-0257.wav|Searches were made certainly, but they were too often superficial, or they might be evaded by a trifling bribe. +DUMMY/LJ002-0204.wav|When they were sick no more notice was taken of them than of a dog. +DUMMY/LJ028-0151.wav|leaving between them room for a four-horse chariot to turn. +DUMMY/LJ036-0146.wav|Neches is within one-half block of the roominghouse at ten twenty-six North Beckley where Oswald was living. +DUMMY/LJ030-0106.wav|A base station at a fixed location in Dallas operated a radio network which linked together the lead car, +DUMMY/LJ030-0092.wav|occupied the right-hand side of the front seat, and Hurchel Jacks of the Texas State Highway patrol was the driver. +DUMMY/LJ019-0259.wav|while the former was intended for the congregate labor of a number, and the latter, as its name implies, imposed continuous solitary toil. +DUMMY/LJ038-0075.wav|Fritz replied, quote, Well, he was employed down at the Book Depository and he had not been present for a roll call of the employees, end quote. +DUMMY/LJ014-0167.wav|When I came upon the scene at midnight, the shrillness of the cries and howls that were raised from time to time, +DUMMY/LJ040-0043.wav|He assassinated the President, shot Officer Tippit, resisted arrest and tried to kill another policeman in the process. +DUMMY/LJ003-0103.wav|It was that of a person, quote, who practiced in the law, and who was connected by marriage with some very respectable families. +DUMMY/LJ023-0058.wav|showed the need of a national government with power enough to handle national problems. +DUMMY/LJ046-0064.wav|The President's views of his responsibilities as President of the United States were that he meet the people, that he go out to their homes and see them, +DUMMY/LJ013-0247.wav|Medical assistance having been summoned, it was ascertained how the dismemberment had been effected. +DUMMY/LJ034-0089.wav|Oswald had not filled any of the three orders. +DUMMY/LJ017-0239.wav|and although the mutineers spared him, they ill-treated the Chinamen, and cut one badly with knives. +DUMMY/LJ014-0016.wav|A letter, which afforded an additional clue, was also found in the pocket of the deceased. Still it was many weeks before any arrest was made. +DUMMY/LJ008-0046.wav|on the middle of which is placed the gibbet, extending from the jail across the Old Bailey. +DUMMY/LJ003-0124.wav|He was charged with moving something which should not be touched, with leaving a door open, or coughing maliciously to the disturbance of his companions. +DUMMY/LJ028-0458.wav|Its outer part was about twenty-four feet in thickness, and its foundations, as Nebuchadnezzar said, were carried down to the water level. +DUMMY/LJ019-0279.wav|Clause ninety-two, Schedule one of that act authorized the use of plank beds, which were adopted in many prisons. +DUMMY/LJ035-0120.wav|While they were at the west windows their view of the stairwell was completely blocked by shelves and boxes. +DUMMY/LJ036-0102.wav|He testified that Oswald looked older than the other boys. +DUMMY/LJ025-0062.wav|under particular circumstances, the contents of the cells of certain water-weeds were set free and moved about with considerable velocity +DUMMY/LJ019-0156.wav|Unless the guilty prisoner was given up, the whole ward was punished, either by the exclusion of visitors or the deprivation of fire, +DUMMY/LJ015-0091.wav|and to have succeeded in other speculations. +DUMMY/LJ026-0159.wav|and hence the excess oxygen is eliminated while starch manufacture is in process. +DUMMY/LJ041-0172.wav|I never said anything to him again and he never said anything to me again, end quote. +DUMMY/LJ018-0306.wav|Noyes was seized without difficulty, as it was a part of the scheme that he should act as the dupe, and remain on the spot in London till all the money was obtained. +DUMMY/LJ015-0097.wav|and he could not produce them. His chief asked sternly where they were. +DUMMY/LJ015-0011.wav|Maltby, who had bolted, was pursued and arrested, to end his life miserably by committing suicide in a Newgate cell. +DUMMY/LJ040-0069.wav|Reminding her sons that they were orphans and that the family's financial condition was poor, +DUMMY/LJ028-0231.wav|whereupon they withdrew within their defenses. +DUMMY/LJ047-0100.wav|but less than completely truthful or cooperative when interrogated about the Fair Play for Cuba Committee. +DUMMY/LJ032-0264.wav|tied with a string, lying on the floor of the Paines' garage. +DUMMY/LJ028-0461.wav|the first-born son of Nabopolassar, King of Babylon. +DUMMY/LJ018-0011.wav|About the same time a body was discovered on the line near the railway-bridge by Victoria Park. +DUMMY/LJ011-0290.wav|His first act was to send a messenger to stop the cheque and the order to Mr. Bell, his next to seek the help of the police. +DUMMY/LJ017-0067.wav|Palmer was ably defended, but the weight of evidence was clearly with the prosecution, led by Sir Alexander Cockburn, +DUMMY/LJ013-0225.wav|The Sunday night before his execution he would not go to bed when ordered. +DUMMY/LJ049-0219.wav|These considerations have induced the Commission to believe +DUMMY/LJ023-0055.wav|Like the Bible, it ought to be read again and again. +DUMMY/LJ015-0182.wav|and thoroughly embodying the idea of English respectability. +DUMMY/LJ020-0079.wav|drill several finger-holes in it and fill them with the melted butter, the dissolved soda and the beaten yolk of egg. +DUMMY/LJ018-0143.wav|each containing two notes, many of the sheets suitable for engraving any kind of note from one thousand pounds downwards. +DUMMY/LJ047-0083.wav|The Passport Office of the Department of State in Washington had no listing for Oswald requiring special treatment, +DUMMY/LJ008-0207.wav|and although one of these gave way owing to the extraordinary pressure, no serious accident occurred. +DUMMY/LJ009-0046.wav|but the attempt fails; he trembles, his knees knock together, and his head droops as he enters the condemned pew. +DUMMY/LJ031-0212.wav|The body was muscular and well developed with no gross skeletal abnormalities except for those caused by the gunshot wounds. +DUMMY/LJ021-0154.wav|is the program of Public Works provided for in the same Act and designed to put more men back to work, +DUMMY/LJ038-0129.wav|Holmes reminded Oswald that A. J. Hidell was listed on post office box three zero zero six one, New Orleans, +DUMMY/LJ020-0094.wav|Put on stockings and slippers, such underclothing as may be needed to prevent cold, a wrapper and the kitchen apron. +DUMMY/LJ050-0194.wav|and should not be relied upon to the detriment of the imaginative application of judgment in special cases. +DUMMY/LJ044-0238.wav|The fact that Oswald left behind the funds which might have enabled him to reach Cuba suggests the absence of any plan to try to flee there +DUMMY/LJ032-0154.wav|The print is additional proof that the rifle was in Oswald's possession. +DUMMY/LJ025-0144.wav|one of which is able to affect the other by means of the communication so established. +DUMMY/LJ020-0021.wav|Into a hollow, like a crater in the middle of the flour, empty your sponge-bowl, and work the flour down into it. +DUMMY/LJ015-0195.wav|It has been already remarked in these pages that with changed social conditions came a great change in the character of crimes. +DUMMY/LJ042-0151.wav|with what he wrote while on the way back to the United States and after his return. +DUMMY/LJ004-0007.wav|It was so powerless against the persistent neglect of those intrusted with prison management, that, five-and-twenty years later, +DUMMY/LJ006-0050.wav|the Exchequer, the Commissioners of bankruptcy and of taxes; smugglers, and a larger number sentenced for very short terms, +DUMMY/LJ048-0199.wav|and arrest any person who might attempt to throw anything or try to get at the President and his party; +DUMMY/LJ021-0009.wav|with a greater certainty of the employment of labor at a reasonable wage and of more business at a fair profit. +DUMMY/LJ031-0061.wav|Dr. Fouad Bashour, chief of cardiology, Dr. M. T. Jenkins, chief of anesthesiology, and Dr. A. H. Giesecke, Jr., +DUMMY/LJ047-0185.wav|I had now established that Lee Oswald was not employed in a sensitive industry. +DUMMY/LJ038-0277.wav|I am aware of their position. This is not, I am sure, arrived at without careful consideration. +DUMMY/LJ015-0278.wav|commencing sham actions, and addressing formal applications, merely for the reply. +DUMMY/LJ048-0262.wav|during the hours they are officially employed at their post of duty, or when they may reasonably expect that they may be called upon to perform an official duty. +DUMMY/LJ017-0156.wav|and told her sister as much when they last met. +DUMMY/LJ028-0089.wav|Though a great warrior, it was not for his military deeds that he was best known. +DUMMY/LJ007-0193.wav|The measures of improvement introduced were mainly as follows: +DUMMY/LJ015-0113.wav|followed him over to Sweden, and arrested him at Helsingfors. +DUMMY/LJ034-0106.wav|Brennan described the man in similar terms, except that he gave the weight as between one hundred sixty-five and one hundred seventy-five pounds and the height was omitted. +DUMMY/LJ041-0064.wav|Daniel Powers, another marine who was stationed with Oswald for part of his marine career, +DUMMY/LJ018-0188.wav|Presently "Old Bob" drove up to Camberwell Gate in the same cart in which he had been seen to start. +DUMMY/LJ041-0034.wav|started expounding the Communist doctrine and saying that he was highly interested in communism, that communism was the only way of life for the worker, et cetera, +DUMMY/LJ009-0266.wav|and his death was not compassed until the executioner terminated his sufferings by hanging on to his legs. +DUMMY/LJ014-0211.wav|and had been repeatedly sentenced to imprisonment for most desperate and ferocious attacks upon various constables. His last victim was Dwyer, +DUMMY/LJ041-0164.wav|which Thornley read at Oswald's suggestion. +DUMMY/LJ016-0103.wav|and after all was quiet the three got on to the roof through the hole, and so on to Tyler's manufactory close by, +DUMMY/LJ022-0003.wav|Since my annual message to the Congress on January fourth, last, I have not addressed the general public over the air. +DUMMY/LJ005-0271.wav|As a means of increasing the severity of imprisonment, letters and visits from outside should not be permitted during the first six months of an imprisonment. +DUMMY/LJ019-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section twenty-two. Newgate Reformed. +DUMMY/LJ028-0162.wav|At the river end of these cross streets are low gates in the fence that skirts the stream, +DUMMY/LJ007-0189.wav|The corporation evidently had the worst of it, and began to feel the necessity for undertaking the great work of reform. +DUMMY/LJ012-0017.wav|The trade of fence, or receiver, therefore, is very nearly as old as the crimes which it so obviously fostered. +DUMMY/LJ009-0146.wav|Mr. Carver, the ordinary, appears to have addressed himself directly to Courvoisier, +DUMMY/LJ040-0207.wav|and represents his reaction to a complete absence of any real family life, end quote. +DUMMY/LJ028-0297.wav|How will thy disfigurement, thou simpleton, induce the enemy to yield one day sooner? +DUMMY/LJ022-0063.wav|to help those who have reached the age of retirement to give up their jobs and thus give to the younger generation greater opportunities for work +DUMMY/LJ027-0026.wav|lead to a preview of certain principles of adaptation, necessary for their interpretation. +DUMMY/LJ029-0140.wav|To create this traffic pattern, a concrete barrier between Main and Elm Streets presents an obstacle to a right turn +DUMMY/LJ045-0223.wav|were part of his character long before he met his wife +DUMMY/LJ048-0225.wav|they had little opportunity to eat during the day. No food was available at the Press Club. +DUMMY/LJ005-0024.wav|He objected to the instruction of prisoners in reading and writing. +DUMMY/LJ037-0024.wav|As he crossed the intersection a block east of tenth and Patton, he saw a policeman standing by the left door of the police car parked along the south side of tenth. +DUMMY/LJ039-0230.wav|With the equipment he [Oswald] had and with his ability I consider it a very easy shot, end quote. +DUMMY/LJ011-0010.wav|On another occasion the banker forged a gentleman's name while the latter was sitting with him in his private room, +DUMMY/LJ033-0054.wav|A firearms expert with the FBI assembled the rifle in six minutes using a ten-cent coin as a tool, +DUMMY/LJ032-0218.wav|Both pictures, however, have identical backgrounds and lighting and, judging from the shadows, were taken at the same angle. +DUMMY/LJ022-0021.wav|or to one industry, or to an individual private occupation. +DUMMY/LJ017-0128.wav|At the last moment Palmer tossed a bit of paper over to his counsel, on which he had written, quote, I think there will be a verdict of Not Guilty, end quote. +DUMMY/LJ012-0076.wav|There was an increase rather than an abatement in jewel and bullion robberies in the years immediately following, +DUMMY/LJ016-0057.wav|As his coat was an encumbrance, he left it on the top of the third house in Newgate Street, and thus in shirt-sleeves, barefoot and bareheaded, +DUMMY/LJ045-0236.wav|He admitted nothing that would damage him but discussed other matters quite freely. +DUMMY/LJ015-0157.wav|During the days of his prosperity he was a governor of Christ's Hospital, +DUMMY/LJ044-0228.wav|it should be remembered that his wife testified that he was disappointed with his failure to get to Cuba +DUMMY/LJ011-0012.wav|It must be added that the Bank of England, on discovering the forgeries, +DUMMY/LJ033-0087.wav|She thought that its color was similar to that of the bag found on the sixth floor of the School Book Depository after the assassination. +DUMMY/LJ040-0092.wav|when he left school in order to get into the Coast Guard. Since his mother did not approve of his decision to continue school +DUMMY/LJ036-0054.wav|When Oswald was arrested in the Texas Theatre, he was wearing a brown sport shirt with a hole in the right sleeve at the elbow. +DUMMY/LJ019-0166.wav|those for trial, and those sentenced for short terms or long +DUMMY/LJ003-0141.wav|Felons who could pay the price were permitted, irrespective of their character or offenses, +DUMMY/LJ014-0092.wav|The hole must have been excavated and the quicklime purchased quite three weeks before O'Connor met his death, +DUMMY/LJ028-0409.wav|Some, calling themselves Christians, are the descendants of Babylonians, perhaps of Nebuchadnezzar himself. +DUMMY/LJ037-0102.wav|He stated further that from, quote, What I saw of him, end quote, the man looked like the man in the picture. +DUMMY/LJ014-0293.wav|He returned from court in a state of gloomy dejection, +DUMMY/LJ043-0076.wav|that he had some aptitude in that area, quote, because the verbal score is high and the clerical score is high, end quote, +DUMMY/LJ031-0083.wav|the doctors observed that he had a heartbeat and was making some respiratory efforts. +DUMMY/LJ010-0031.wav|the same and happily often futile precautions taken to conceal the crime. +DUMMY/LJ049-0122.wav|FBI Director J. Edgar Hoover, who strongly supports such legislation, testified that the absence of clear Federal jurisdiction +DUMMY/LJ017-0057.wav|On returning to jail each day he talked freely and without reserve to the warders in charge of him, chiefly on incidents in the day's proceedings. +DUMMY/LJ031-0105.wav|chief of thoracic surgery, arrived at trauma room two, to take charge of the care of Governor Connally, +DUMMY/LJ041-0180.wav|never heard him in any way, shape or form confess that he was a Communist, or that he ever thought about being a Communist, end quote. +DUMMY/LJ049-0011.wav|As described in chapter two, the President directed that his car stop on two occasions during the motorcade so that he could greet members of the public. +DUMMY/LJ038-0142.wav|Actually, the party for one of the Paine's children was the preceding weekend, when Marina Oswald suggested that Oswald remain in Dallas. +DUMMY/LJ016-0169.wav|As more humane principles were introduced into prison management, +DUMMY/LJ029-0186.wav|Increased concern about the President's visit was aroused by the incident involving the U.S. Ambassador to the United Nations, Adlai E. Stevenson. +DUMMY/LJ032-0268.wav|This bulge was apparently caused by a hard protruding object which had stretched the blanket's fibers. +DUMMY/LJ042-0177.wav|Oswald actually did attempt to formulate such an alternative which he planned to, quote, put forward, end quote, himself. +DUMMY/LJ047-0077.wav|Oswald was tentatively located in New Orleans in June, +DUMMY/LJ022-0052.wav|here as in every other nation, we have come to recognize the possibility and the necessity of certain helpful remedial measures. +DUMMY/LJ048-0103.wav|and with the concurrence of the Dallas police, was entirely appropriate, in view of the known desires of the President. +DUMMY/LJ011-0185.wav|Filial affection rose superior to all scruples, and Miss Turner, having crossed the border, was married to Wakefield +DUMMY/LJ028-0004.wav|and along the narrow winding lane which was once called the "Street Called Straight." +DUMMY/LJ038-0209.wav|Oswald told his wife that he did not know whether he had hit Walker; +DUMMY/LJ017-0237.wav|when asked said they would spare his life if he would navigate the ship for them to the River Plate or Buenos Aires. +DUMMY/LJ016-0003.wav|and that none of its inmates could hope to escape from its secure precincts. +DUMMY/LJ049-0090.wav|Federal law enforcement officers is a Federal crime. +DUMMY/LJ028-0050.wav|The building of the Babylon so famous in history began with Nabopolassar. +DUMMY/LJ026-0044.wav|In order to emphasize the fundamental similarity of organic function in higher and lower animals and plants, let us compare any higher plant +DUMMY/LJ023-0034.wav|The American people have learned from the depression. +DUMMY/LJ015-0042.wav|Then it was that instead of merely pledging securities, the bank sold them outright, and thus passed the Rubicon of fraud. +DUMMY/LJ049-0077.wav|Since the assassination, the Secret Service and the Department of the Treasury +DUMMY/LJ042-0157.wav|Lee Harvey Oswald was born in October nineteen thirty-nine in New Orleans, Louisiana, the son of a Insurance Salesmen whose early death +DUMMY/LJ021-0181.wav|thereby saving the British treasury one hundred and fifty million dollars a year in interest alone? +DUMMY/LJ003-0155.wav|Tailoring and shoemaking was permitted, but it was deemed unsafe to allow a carpenter or blacksmith to have his tools. +DUMMY/LJ036-0060.wav|south on Houston, and southwest across the Houston viaduct to service the Oak Cliff area along Marsalis. +DUMMY/LJ032-0190.wav|On the other hand Stombaugh pointed out that fibers might retain their freshness if the rifle had been +DUMMY/LJ035-0096.wav|is consistent with the movements of the two elevators, which would have provided the other possible means of descent. +DUMMY/LJ038-0285.wav|that the bullet which missed General Walker was fired from Oswald's rifle to the exclusion of all others, this testimony was considered probative +DUMMY/LJ039-0019.wav|She reminded him that after the Walker shooting he had promised never to repeat such an act. +DUMMY/LJ024-0012.wav|with modern facts and circumstances under which average men have to live and work. +DUMMY/LJ046-0103.wav|In this connection the information available to Federal agencies about Lee Harvey Oswald +DUMMY/LJ007-0197.wav|The infirmary was enlarged, the admission of visitors limited, and the passing of articles prevented by a wire screen. +DUMMY/LJ027-0075.wav|but, in spite of all these obscurations, corresponding parts usually may be made out. +DUMMY/LJ029-0149.wav|To control traffic, arrangements were made for the deployment of foot patrolmen and motorcycle police at various positions along the route. +DUMMY/LJ011-0232.wav|It must not be imagined that although highway robbery was now nearly extinct, and felonious outrages in the streets were rare, +DUMMY/LJ046-0151.wav|All material received by PRS was separately screened +DUMMY/LJ018-0355.wav|who was convicted as an accessory after the fact, had not really taken an active part in the murder. +DUMMY/LJ010-0199.wav|Among his papers was found a curious document, purporting to be the rules of an association called +DUMMY/LJ048-0191.wav|Television films taken of parts of the motorcade by a Dallas television station +DUMMY/LJ016-0285.wav|Then one struck up the hymn of the Promised Land, and the refrain was at once taken up with a mighty chorus -- +DUMMY/LJ048-0186.wav|Instead of such advance precautions, the Secret Service depended in part on the efforts of local law enforcement personnel stationed along the route. +DUMMY/LJ031-0062.wav|then joined in the effort to revive the President. +DUMMY/LJ015-0239.wav|Just as the train was starting Agar slipped into the van with Burgess, and Pierce got into a first-class carriage. +DUMMY/LJ042-0164.wav|"The Collective" contrasts sharply with material which Oswald seems to have written after he left the Soviet Union. +DUMMY/LJ040-0222.wav|But Carro was quite specific that his recommendation was based primarily on the adverse factors in Lee's environment +DUMMY/LJ005-0029.wav|nothing but beating hemp and pulling oakum and pounding bricks -- no work but what was tedious, unusual." +DUMMY/LJ021-0194.wav|More than twenty years ago, when Theodore Roosevelt and Woodrow Wilson were attempting to correct abuses in our national life, +DUMMY/LJ020-0032.wav|Knead as you would white bread, and set aside for the rising. +DUMMY/LJ031-0111.wav|The elliptical wound in the Governor's back, located slightly to the left of the Governor's right armpit approximately five-eighths inch (a centimeter and a half) +DUMMY/LJ022-0200.wav|renewed faith in the vast possibilities of human beings to improve their material and spiritual status +DUMMY/LJ043-0034.wav|Contacts with De Mohrenschildt and his wife did continue and they saw the Oswalds occasionally until the spring of nineteen sixty-three. +DUMMY/LJ011-0198.wav|An uncle of Miss Wakefield's, accompanied by his solicitor and a Bow Street runner, at once went in pursuit. +DUMMY/LJ032-0195.wav|The Commission was able to conclude, however, that the fibers most probably came from Oswald's shirt. +DUMMY/LJ019-0297.wav|The prisoners inter-communicated freely, and exercised the most injurious, corrupting influences upon one another. +DUMMY/LJ015-0253.wav|he handed over to Pierce a sum of three thousand pounds, his own, whether rightly or wrongly acquired never came out, +DUMMY/LJ011-0035.wav|Every endeavor was used, however, to obtain a commutation of sentence. His case was twice argued before the judges on points of law, +DUMMY/LJ045-0017.wav|This meeting preceded by one day the attack on A. E. Stevenson at the United Nations Day meeting at which he spoke +DUMMY/LJ044-0239.wav|and raises serious questions as to whether or not he ever expected to escape. +DUMMY/LJ013-0207.wav|In this he gave as the motives of his crime a quarrel he had with his master, who threatened to discharge him without a character. +DUMMY/LJ009-0057.wav|Already he is half dead. +DUMMY/LJ003-0129.wav|a taste for spirits. End quote. +DUMMY/LJ019-0146.wav|There was as yet no control over the prisoners after locking-up time; +DUMMY/LJ025-0059.wav|Robert Brown noted the more complex motions of the protoplasm in the cells of Tradescantia in eighteen thirty-one, +DUMMY/LJ012-0213.wav|An inquest was held on the body, which was identified as that of an Italian boy, +DUMMY/LJ024-0130.wav|to frighten the workers of America in a pay-envelope propaganda against the Social Security Law. +DUMMY/LJ018-0316.wav|Further sums were to have been paid after the escape; +DUMMY/LJ012-0138.wav|They also ascertained that a gold-refiner, +DUMMY/LJ006-0057.wav|Not only were the wards thus needlessly crammed, and for no reason but the niggardliness of the corporation which refused a proper supply of bedding +DUMMY/LJ032-0166.wav|Judgments as to probability will depend on the number and types of matches. +DUMMY/LJ034-0107.wav|In his testimony before the Commission, Brennan described the person he saw as, quote, +DUMMY/LJ039-0051.wav|In the absence of other evidence that Oswald actually intended to shoot someone at this time, +DUMMY/LJ011-0051.wav|the great magnitude of our erring brother's offense, one of the most dangerous description in a trading community. +DUMMY/LJ045-0210.wav|Now it appeared that he was not welcome at the Paine home, +DUMMY/LJ032-0058.wav|to May fourteen, nineteen sixty-three. +DUMMY/LJ018-0001.wav|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section twenty-one: Newgate Notorieties, part two. +DUMMY/LJ036-0209.wav|A similar description was given on channel two at twelve:forty-five p.m. +DUMMY/LJ010-0118.wav|were not altogether creditable to the Government. +DUMMY/LJ009-0228.wav|and a subscription was set on foot for them then and there at the foot of the gibbet. +DUMMY/LJ002-0045.wav|The various wards were all about eleven feet in height, +DUMMY/LJ029-0086.wav|Once the Trade Mart had been selected, Sorrels and Lawson worked out detailed arrangements for security at the building. +DUMMY/LJ023-0097.wav|Justice Stone said of the majority opinion that it was a "tortured construction of the Constitution." +DUMMY/LJ028-0108.wav|Fortunately in several of his long inscriptions, recently discovered in the Babylonian mounds, Nebuchadnezzar speaks of the building of the walls. +DUMMY/LJ041-0149.wav|He told Aline Mosby, a reporter who interviewed him after he arrived in Moscow, quote, +DUMMY/LJ028-0217.wav|His many troops, whose number, like the waters of the river, could not be counted, marched in full armor at his side, +DUMMY/LJ008-0045.wav|In the middle of this machinery is placed a movable platform, in form of a trap-door, ten feet long by eight wide, +DUMMY/LJ010-0241.wav|but she declared she would not remain a prisoner in her own palace, and next day drove out as usual in an open barouche. +DUMMY/LJ010-0058.wav|But the Chartists sought their ends by riot and rebellion, and gained only imprisonment for their pains. +DUMMY/LJ017-0253.wav|A farmer took them in for the night, and drove them next day to Rocha, a place north of Maldonado. +DUMMY/LJ043-0121.wav|A placement interviewer of the Louisiana Department of Labor +DUMMY/LJ001-0021.wav|The earliest book printed with movable type, the aforesaid Gutenberg Bible, is printed in letters which are an exact imitation +DUMMY/LJ012-0226.wav|This trial came off in due course at the Central Criminal Court, where the prisoners were charged on two counts, +DUMMY/LJ039-0181.wav|the shots would have been evenly spaced and the assassin would not have incurred so sharp an angular movement. +DUMMY/LJ049-0173.wav|As the testimony of J. Edgar Hoover and other Bureau officials revealed, the FBI did not believe that its directive required the Bureau +DUMMY/LJ035-0101.wav|neither of these slow elevators could have descended from the fifth to the second floor. Furthermore, no elevator was at the second floor when they arrived there. +DUMMY/LJ019-0313.wav|However, it now laid down in plain language and with precise details the requirements of a good jail system. +DUMMY/LJ004-0073.wav|to see that "no circumstances of severity are found in his treatment which are not found in his sentence." +DUMMY/LJ019-0029.wav|Major Jebb, R.E., and the two prison inspectors, Messrs. Crawford and Russell, with whose names the reader is already familiar. +DUMMY/LJ030-0041.wav|Motorcycles. -- Dallas police motorcycles preceded the pilot car. +DUMMY/LJ044-0092.wav|The interesting thing, or rather the danger involved, was the fact that Oswald seemed like such a nice, bright boy +DUMMY/LJ031-0223.wav|running in a line from the wound in the rear of the President's head toward the front part of the skull, +DUMMY/LJ050-0053.wav|Broader and more selective criteria +DUMMY/LJ002-0056.wav|The wards in this part were five in number, all in dimensions twenty feet by fifteen, with a sixth ward fifteen feet square. +DUMMY/LJ034-0154.wav|and I felt like there hadn't been more than one eyewitness, and if it got to be a known fact that I was an eyewitness, my family or I, either one, might not be safe. +DUMMY/LJ033-0191.wav|Fibers in paper bag matched fibers in blanket +DUMMY/LJ027-0069.wav|and (b) degenerate forms like snakes and some lizards, which have lost their limbs by disuse. +DUMMY/LJ017-0170.wav|there is not absolute and complete evidence of his guilt. End quote. +DUMMY/LJ006-0160.wav|one man who had been in the condemned ward for two months, said the governor only came there four times. +DUMMY/LJ031-0215.wav|One wound, approximately one-fourth of an inch by five-eighths of an inch (six by fifteen millimeters), +DUMMY/LJ036-0106.wav|You could have picked him out without identifying him by just listening to him. +DUMMY/LJ040-0131.wav|Marguerite Oswald visited her son at Youth House, where she recalled that she waited in line, quote, +DUMMY/LJ044-0116.wav|Since he seemed to feel that no one else understood his political views, the letter was of great value to him for it, quote, was proof +DUMMY/LJ021-0058.wav|Under these codes, in the industries covered, child labor has been eliminated. +DUMMY/LJ001-0170.wav|being thin, tough, and opaque. +DUMMY/LJ006-0291.wav|to clean the governor's office in the male prison; +DUMMY/LJ032-0163.wav|Stombaugh explained in his testimony that in fiber analysis, as distinct from fingerprint or firearms identification, +DUMMY/LJ010-0227.wav|He went from Newgate first to Bethlehem, from which he was removed to Broadmoor on the opening of the great criminal lunatic asylum at that place. +DUMMY/LJ013-0248.wav|At the same time an overpowering odor attracted them to the adjoining harness-room, where the missing remains were raked out +DUMMY/LJ049-0041.wav|in the protection of the President if the Presidential car permitted immediate access to the President by a Secret Service agent at the first sign of danger. +DUMMY/LJ040-0218.wav|and as will be shown below, this failure to adapt to his environment was a dominant trait in his later life. +DUMMY/LJ019-0281.wav|introduced, as was erroneously supposed, by the prison commissioners appointed under the Prison Act of eighteen seventy-seven. +DUMMY/LJ012-0140.wav|Solomons was not straightforward in his replies as to where he got the gold, and he was soon placed in the dock with the Caspars and Moss. +DUMMY/LJ001-0085.wav|It was reserved for the founders of the later eighteenth century to produce letters which are positively ugly, and which, it may be added, +DUMMY/LJ017-0086.wav|For this purpose he gave, or there was the strongest presumption that he gave, +DUMMY/LJ026-0113.wav|Only proteid foods form new protoplasm +DUMMY/LJ025-0176.wav|and the chlorophyll is the apparatus by which the carbon is extracted from the atmospheric carbonic acid, the leaves being the chief laboratories +DUMMY/LJ037-0020.wav|In his testimony before the Commission, +DUMMY/LJ005-0277.wav|This was the first enunciation of the system of separate confinement, +DUMMY/LJ019-0150.wav|Captain Williams, who was the inspector of prisons for the home district in succession to Messrs. Crawford and Russell, +DUMMY/LJ008-0172.wav|In another part seven persons met their death by suffocation. +DUMMY/LJ050-0219.wav|However, the nineteen sixty-four to sixty-five budget request was submitted in November nineteen sixty-three +DUMMY/LJ023-0053.wav|It is the American people themselves who expect the third horse to pull in unison with the other two. +DUMMY/LJ037-0209.wav|Handwriting experts, Alwyn Cole of the Treasury Department and James C. Cadigan of the FBI, +DUMMY/LJ010-0247.wav|The prisoner was conveyed without delay to the Home Office, and there examined by the Privy Council, which had been hastily summoned for the purpose. diff --git a/models/tacotron2/filelists/ljs_audio_text_val_filelist.txt b/models/tacotron2/filelists/ljs_audio_text_val_filelist.txt new file mode 100755 index 0000000000..4f605f6564 --- /dev/null +++ b/models/tacotron2/filelists/ljs_audio_text_val_filelist.txt @@ -0,0 +1,100 @@ +DUMMY/LJ022-0023.wav|The overwhelming majority of people in this country know how to sift the wheat from the chaff in what they hear and what they read. +DUMMY/LJ043-0030.wav|If somebody did that to me, a lousy trick like that, to take my wife away, and all the furniture, I would be mad as hell, too. +DUMMY/LJ005-0201.wav|as is shown by the report of the Commissioners to inquire into the state of the municipal corporations in eighteen thirty-five. +DUMMY/LJ001-0110.wav|Even the Caslon type when enlarged shows great shortcomings in this respect: +DUMMY/LJ003-0345.wav|All the committee could do in this respect was to throw the responsibility on others. +DUMMY/LJ007-0154.wav|These pungent and well-grounded strictures applied with still greater force to the unconvicted prisoner, the man who came to the prison innocent, and still uncontaminated, +DUMMY/LJ018-0098.wav|and recognized as one of the frequenters of the bogus law-stationers. His arrest led to that of others. +DUMMY/LJ047-0044.wav|Oswald was, however, willing to discuss his contacts with Soviet authorities. He denied having any involvement with Soviet intelligence agencies +DUMMY/LJ031-0038.wav|The first physician to see the President at Parkland Hospital was Dr. Charles J. Carrico, a resident in general surgery. +DUMMY/LJ048-0194.wav|during the morning of November twenty-two prior to the motorcade. +DUMMY/LJ049-0026.wav|On occasion the Secret Service has been permitted to have an agent riding in the passenger compartment with the President. +DUMMY/LJ004-0152.wav|although at Mr. Buxton's visit a new jail was in process of erection, the first step towards reform since Howard's visitation in seventeen seventy-four. +DUMMY/LJ008-0278.wav|or theirs might be one of many, and it might be considered necessary to "make an example." +DUMMY/LJ043-0002.wav|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter seven. Lee Harvey Oswald: +DUMMY/LJ009-0114.wav|Mr. Wakefield winds up his graphic but somewhat sensational account by describing another religious service, which may appropriately be inserted here. +DUMMY/LJ028-0506.wav|A modern artist would have difficulty in doing such accurate work. +DUMMY/LJ050-0168.wav|with the particular purposes of the agency involved. The Commission recognizes that this is a controversial area +DUMMY/LJ039-0223.wav|Oswald's Marine training in marksmanship, his other rifle experience and his established familiarity with this particular weapon +DUMMY/LJ029-0032.wav|According to O'Donnell, quote, we had a motorcade wherever we went, end quote. +DUMMY/LJ031-0070.wav|Dr. Clark, who most closely observed the head wound, +DUMMY/LJ034-0198.wav|Euins, who was on the southwest corner of Elm and Houston Streets testified that he could not describe the man he saw in the window. +DUMMY/LJ026-0068.wav|Energy enters the plant, to a small extent, +DUMMY/LJ039-0075.wav|once you know that you must put the crosshairs on the target and that is all that is necessary. +DUMMY/LJ004-0096.wav|the fatal consequences whereof might be prevented if the justices of the peace were duly authorized +DUMMY/LJ005-0014.wav|Speaking on a debate on prison matters, he declared that +DUMMY/LJ012-0161.wav|he was reported to have fallen away to a shadow. +DUMMY/LJ018-0239.wav|His disappearance gave color and substance to evil reports already in circulation that the will and conveyance above referred to +DUMMY/LJ019-0257.wav|Here the tread-wheel was in use, there cellular cranks, or hard-labor machines. +DUMMY/LJ028-0008.wav|you tap gently with your heel upon the shoulder of the dromedary to urge her on. +DUMMY/LJ024-0083.wav|This plan of mine is no attack on the Court; +DUMMY/LJ042-0129.wav|No night clubs or bowling alleys, no places of recreation except the trade union dances. I have had enough. +DUMMY/LJ036-0103.wav|The police asked him whether he could pick out his passenger from the lineup. +DUMMY/LJ046-0058.wav|During his Presidency, Franklin D. Roosevelt made almost four hundred journeys and traveled more than three hundred fifty thousand miles. +DUMMY/LJ014-0076.wav|He was seen afterwards smoking and talking with his hosts in their back parlor, and never seen again alive. +DUMMY/LJ002-0043.wav|long narrow rooms -- one thirty-six feet, six twenty-three feet, and the eighth eighteen, +DUMMY/LJ009-0076.wav|We come to the sermon. +DUMMY/LJ017-0131.wav|even when the high sheriff had told him there was no possibility of a reprieve, and within a few hours of execution. +DUMMY/LJ046-0184.wav|but there is a system for the immediate notification of the Secret Service by the confining institution when a subject is released or escapes. +DUMMY/LJ014-0263.wav|When other pleasures palled he took a theatre, and posed as a munificent patron of the dramatic art. +DUMMY/LJ042-0096.wav|(old exchange rate) in addition to his factory salary of approximately equal amount +DUMMY/LJ049-0050.wav|Hill had both feet on the car and was climbing aboard to assist President and Mrs. Kennedy. +DUMMY/LJ019-0186.wav|seeing that since the establishment of the Central Criminal Court, Newgate received prisoners for trial from several counties, +DUMMY/LJ028-0307.wav|then let twenty days pass, and at the end of that time station near the Chaldasan gates a body of four thousand. +DUMMY/LJ012-0235.wav|While they were in a state of insensibility the murder was committed. +DUMMY/LJ034-0053.wav|reached the same conclusion as Latona that the prints found on the cartons were those of Lee Harvey Oswald. +DUMMY/LJ014-0030.wav|These were damnatory facts which well supported the prosecution. +DUMMY/LJ015-0203.wav|but were the precautions too minute, the vigilance too close to be eluded or overcome? +DUMMY/LJ028-0093.wav|but his scribe wrote it in the manner customary for the scribes of those days to write of their royal masters. +DUMMY/LJ002-0018.wav|The inadequacy of the jail was noticed and reported upon again and again by the grand juries of the city of London, +DUMMY/LJ028-0275.wav|At last, in the twentieth month, +DUMMY/LJ012-0042.wav|which he kept concealed in a hiding-place with a trap-door just under his bed. +DUMMY/LJ011-0096.wav|He married a lady also belonging to the Society of Friends, who brought him a large fortune, which, and his own money, he put into a city firm, +DUMMY/LJ036-0077.wav|Roger D. Craig, a deputy sheriff of Dallas County, +DUMMY/LJ016-0318.wav|Other officials, great lawyers, governors of prisons, and chaplains supported this view. +DUMMY/LJ013-0164.wav|who came from his room ready dressed, a suspicious circumstance, as he was always late in the morning. +DUMMY/LJ027-0141.wav|is closely reproduced in the life-history of existing deer. Or, in other words, +DUMMY/LJ028-0335.wav|accordingly they committed to him the command of their whole army, and put the keys of their city into his hands. +DUMMY/LJ031-0202.wav|Mrs. Kennedy chose the hospital in Bethesda for the autopsy because the President had served in the Navy. +DUMMY/LJ021-0145.wav|From those willing to join in establishing this hoped-for period of peace, +DUMMY/LJ016-0288.wav|"Müller, Müller, He's the man," till a diversion was created by the appearance of the gallows, which was received with continuous yells. +DUMMY/LJ028-0081.wav|Years later, when the archaeologists could readily distinguish the false from the true, +DUMMY/LJ018-0081.wav|his defense being that he had intended to commit suicide, but that, on the appearance of this officer who had wronged him, +DUMMY/LJ021-0066.wav|together with a great increase in the payrolls, there has come a substantial rise in the total of industrial profits +DUMMY/LJ009-0238.wav|After this the sheriffs sent for another rope, but the spectators interfered, and the man was carried back to jail. +DUMMY/LJ005-0079.wav|and improve the morals of the prisoners, and shall insure the proper measure of punishment to convicted offenders. +DUMMY/LJ035-0019.wav|drove to the northwest corner of Elm and Houston, and parked approximately ten feet from the traffic signal. +DUMMY/LJ036-0174.wav|This is the approximate time he entered the roominghouse, according to Earlene Roberts, the housekeeper there. +DUMMY/LJ046-0146.wav|The criteria in effect prior to November twenty-two, nineteen sixty-three, for determining whether to accept material for the PRS general files +DUMMY/LJ017-0044.wav|and the deepest anxiety was felt that the crime, if crime there had been, should be brought home to its perpetrator. +DUMMY/LJ017-0070.wav|but his sporting operations did not prosper, and he became a needy man, always driven to desperate straits for cash. +DUMMY/LJ014-0020.wav|He was soon afterwards arrested on suspicion, and a search of his lodgings brought to light several garments saturated with blood; +DUMMY/LJ016-0020.wav|He never reached the cistern, but fell back into the yard, injuring his legs severely. +DUMMY/LJ045-0230.wav|when he was finally apprehended in the Texas Theatre. Although it is not fully corroborated by others who were present, +DUMMY/LJ035-0129.wav|and she must have run down the stairs ahead of Oswald and would probably have seen or heard him. +DUMMY/LJ008-0307.wav|afterwards express a wish to murder the Recorder for having kept them so long in suspense. +DUMMY/LJ008-0294.wav|nearly indefinitely deferred. +DUMMY/LJ047-0148.wav|On October twenty-five, +DUMMY/LJ008-0111.wav|They entered a "stone cold room," and were presently joined by the prisoner. +DUMMY/LJ034-0042.wav|that he could only testify with certainty that the print was less than three days old. +DUMMY/LJ037-0234.wav|Mrs. Mary Brock, the wife of a mechanic who worked at the station, was there at the time and she saw a white male, +DUMMY/LJ040-0002.wav|Chapter seven. Lee Harvey Oswald: Background and Possible Motives, Part one. +DUMMY/LJ045-0140.wav|The arguments he used to justify his use of the alias suggest that Oswald may have come to think that the whole world was becoming involved +DUMMY/LJ012-0035.wav|the number and names on watches, were carefully removed or obliterated after the goods passed out of his hands. +DUMMY/LJ012-0250.wav|On the seventh July, eighteen thirty-seven, +DUMMY/LJ016-0179.wav|contracted with sheriffs and conveners to work by the job. +DUMMY/LJ016-0138.wav|at a distance from the prison. +DUMMY/LJ027-0052.wav|These principles of homology are essential to a correct interpretation of the facts of morphology. +DUMMY/LJ031-0134.wav|On one occasion Mrs. Johnson, accompanied by two Secret Service agents, left the room to see Mrs. Kennedy and Mrs. Connally. +DUMMY/LJ019-0273.wav|which Sir Joshua Jebb told the committee he considered the proper elements of penal discipline. +DUMMY/LJ014-0110.wav|At the first the boxes were impounded, opened, and found to contain many of O'Connor's effects. +DUMMY/LJ034-0160.wav|on Brennan's subsequent certain identification of Lee Harvey Oswald as the man he saw fire the rifle. +DUMMY/LJ038-0199.wav|eleven. If I am alive and taken prisoner, +DUMMY/LJ014-0010.wav|yet he could not overcome the strange fascination it had for him, and remained by the side of the corpse till the stretcher came. +DUMMY/LJ033-0047.wav|I noticed when I went out that the light was on, end quote, +DUMMY/LJ040-0027.wav|He was never satisfied with anything. +DUMMY/LJ048-0228.wav|and others who were present say that no agent was inebriated or acted improperly. +DUMMY/LJ003-0111.wav|He was in consequence put out of the protection of their internal law, end quote. Their code was a subject of some curiosity. +DUMMY/LJ008-0258.wav|Let me retrace my steps, and speak more in detail of the treatment of the condemned in those bloodthirsty and brutally indifferent days, +DUMMY/LJ029-0022.wav|The original plan called for the President to spend only one day in the State, making whirlwind visits to Dallas, Fort Worth, San Antonio, and Houston. +DUMMY/LJ004-0045.wav|Mr. Sturges Bourne, Sir James Mackintosh, Sir James Scarlett, and William Wilberforce. diff --git a/models/tacotron2/hparams.py b/models/tacotron2/hparams.py new file mode 100644 index 0000000000..8886f18fec --- /dev/null +++ b/models/tacotron2/hparams.py @@ -0,0 +1,95 @@ +import tensorflow as tf +from text import symbols + + +def create_hparams(hparams_string=None, verbose=False): + """Create model hyperparameters. Parse nondefault from given string.""" + + hparams = tf.contrib.training.HParams( + ################################ + # Experiment Parameters # + ################################ + epochs=500, + iters_per_checkpoint=1000, + seed=1234, + dynamic_loss_scaling=True, + fp16_run=False, + distributed_run=False, + dist_backend="nccl", + dist_url="tcp://localhost:54321", + cudnn_enabled=True, + cudnn_benchmark=False, + ignore_layers=['embedding.weight'], + + ################################ + # Data Parameters # + ################################ + load_mel_from_disk=False, + training_files='filelists/ljs_audio_text_train_filelist.txt', + validation_files='filelists/ljs_audio_text_val_filelist.txt', + text_cleaners=['english_cleaners'], + + ################################ + # Audio Parameters # + ################################ + max_wav_value=32768.0, + sampling_rate=22050, + filter_length=1024, + hop_length=256, + win_length=1024, + n_mel_channels=80, + mel_fmin=0.0, + mel_fmax=8000.0, + + ################################ + # Model Parameters # + ################################ + n_symbols=len(symbols), + symbols_embedding_dim=512, + + # Encoder parameters + encoder_kernel_size=5, + encoder_n_convolutions=3, + encoder_embedding_dim=512, + + # Decoder parameters + n_frames_per_step=1, # currently only 1 is supported + decoder_rnn_dim=1024, + prenet_dim=256, + max_decoder_steps=1000, + gate_threshold=0.5, + p_attention_dropout=0.1, + p_decoder_dropout=0.1, + + # Attention parameters + attention_rnn_dim=1024, + attention_dim=128, + + # Location Layer parameters + attention_location_n_filters=32, + attention_location_kernel_size=31, + + # Mel-post processing network parameters + postnet_embedding_dim=512, + postnet_kernel_size=5, + postnet_n_convolutions=5, + + ################################ + # Optimization Hyperparameters # + ################################ + use_saved_learning_rate=False, + learning_rate=1e-3, + weight_decay=1e-6, + grad_clip_thresh=1.0, + batch_size=64, + mask_padding=True # set model's padded outputs to padded values + ) + + if hparams_string: + tf.logging.info('Parsing command line hparams: %s', hparams_string) + hparams.parse(hparams_string) + + if verbose: + tf.logging.info('Final parsed hparams: %s', hparams.values()) + + return hparams diff --git a/models/tacotron2/inference.ipynb b/models/tacotron2/inference.ipynb new file mode 100644 index 0000000000..83a23f7e23 --- /dev/null +++ b/models/tacotron2/inference.ipynb @@ -0,0 +1,266 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tacotron 2 inference code \n", + "Edit the variables **checkpoint_path** and **text** to match yours and run the entire code to generate plots of mel outputs, alignments and audio synthesis from the generated mel-spectrogram using Griffin-Lim." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Import libraries and setup matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib\n", + "%matplotlib inline\n", + "import matplotlib.pylab as plt\n", + "\n", + "import IPython.display as ipd\n", + "\n", + "import sys\n", + "sys.path.append('waveglow/')\n", + "import numpy as np\n", + "import torch\n", + "\n", + "from hparams import create_hparams\n", + "from model import Tacotron2\n", + "from layers import TacotronSTFT, STFT\n", + "from audio_processing import griffin_lim\n", + "from train import load_model\n", + "from text import text_to_sequence\n", + "from denoiser import Denoiser" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_data(data, figsize=(16, 4)):\n", + " fig, axes = plt.subplots(1, len(data), figsize=figsize)\n", + " for i in range(len(data)):\n", + " axes[i].imshow(data[i], aspect='auto', origin='bottom', \n", + " interpolation='none')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Setup hparams" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "hparams = create_hparams()\n", + "hparams.sampling_rate = 22050" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Load model from checkpoint" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "checkpoint_path = \"tacotron2_statedict.pt\"\n", + "model = load_model(hparams)\n", + "model.load_state_dict(torch.load(checkpoint_path)['state_dict'])\n", + "_ = model.cuda().eval().half()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Load WaveGlow for mel2audio synthesis and denoiser" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ ] + } + ], + "source": [ + "waveglow_path = 'waveglow_256channels.pt'\n", + "waveglow = torch.load(waveglow_path)['model']\n", + "waveglow.cuda().eval().half()\n", + "for k in waveglow.convinv:\n", + " k.float()\n", + "denoiser = Denoiser(waveglow)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Prepare text input" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "text = \"Waveglow is really awesome!\"\n", + "sequence = np.array(text_to_sequence(text, ['english_cleaners']))[None, :]\n", + "sequence = torch.autograd.Variable(\n", + " torch.from_numpy(sequence)).cuda().long()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Decode text input and plot results" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6YAAAD8CAYAAABkf7nnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsvV2sbVl2HvSNOedae+9z7rm37q2ualfaxp1fk+AIm7QEJhIKmCCRhzS8WOQhMiiS8wAIizzE8JRHP0CiPEVqMKiRTESUBCX8JJJlkYfIkoUxrQTSRFaS7rjd5a6udtW952fvtdacc/Awf8ZY++xTdau66p6quuOTqu46a6+f+be+tfcc3/wGMTMMBoPBYDAYDAaDwWC4L7j7LoDBYDAYDAaDwWAwGF5u2A9Tg8FgMBgMBoPBYDDcK+yHqcFgMBgMBoPBYDAY7hX2w9RgMBgMBoPBYDAYDPcK+2FqMBgMBoPBYDAYDIZ7hf0wNRgMBoPBYDAYDAbDvcJ+mBoMBoPBYDAYDAaD4V5hP0wNBoPBYDAYDAaDwXCvsB+mBoPBYDAYDAaDwWC4V4QXebORNrzzD8sf3oHHcvu4c8hj2c0OANUTiNcXuGv/qYMygERyOKvrq8tQqtsRcLEc5BKALPfIYzmJSc51ics1uV2LwZ7qcQRS5/djAIBlPxP1c+VmvK5e+4xI3Z/W57RtvQ8EuHJC2nlkXw/xUg9iaQ8eGC5kOMf1bAbXtsyZkHM9KTpps1z+a3Vs22U/9+1WLibqXZg9wLV8HFQ5CNLPGqpqlGQI6DIwlfoB5Xq9r4+nX7I6P6lyJ4BSKzcf9aHqN1/Hg6NVWaX/pQ5MdwxXXR9eXz/u6nU2DOdL4ZxjUD3Jq23UW7W/Ert+qWUJQLz9DICPyqTLsurDup3UmMy8KuvJsXqrnnftL9jHp5jT/lSvf2pRuO6i/OE9eDMAAOKWTnPdqhPU/vfiOr6D69rHd3CdW4TrKPGK63hQY7udm7k+M+q49jnV+/f73FFeamVVY+eO5+vW/Vkd07lOX1uun7YBeaiH+HURNNdRyPD1wkRHXJfqgZHWXKefH/2cKM7o5dK84Agc6vW96hdS7xQ9+vXzqbkqSx/CQTj9Q3Cdi9zLfec7B3dwHSnuVvd7Xq7T/d64Lm/XXOfqhTypwYXbXJdrA8bFf39c1/owqTGsePlOnnsOfjv+/Nn81tvM/Nrtkz6dGGnDW5zfdzE+2aC7/uDbXPaeF7j9vfJjx4lbf2pB/X/QW6z+/5HVU92LiMAnv6d/TND3BvpvAZADKqfDudV7vG3nQKt3KIfKgYHhXCFNT4zgystocBlDfTFt3IJQvxAESvj6P1w+FNe90B+mW5zjJx5+GQBAjx5i/uLnAABv/+gO118olY8XWb7Ub9Pq1yCNpfLkuf+AOu7iHOuL9ODhn5U3t5upv4TSVs5wC2F4Vq6/+y7j7Lvl+uOzCH+99OOufvisXHugfsPxMsMtub/Q/CFheTjUeziE69TPd0t7uTGofiGEA3JwtXy5v6zdYQHFVlgGpXId9h68rd1FBJrjrWMQ5Z4gAo+lPE9/9BUcXin1nF+RQUcRiGflvvMbC86f7HGxO5RyEPeX/s004upyW056e4PxaSl3uAH8Qa4V9uVaww1jvEr1mAQ3l/rEne9fiOdHAfNFuc7hsUMsTYw8lB+qAJCDkLafpe2HS8DV6ocbxuayXD8NhMNjV/sA/Zppx6svqX5PvdzDFTBelguPlxnj01ivu8AdojRna28ipItNqc/Z0CctAGB65Hsdsvoi2srKTr7g+Jn70HYRCJN8AXv7XyoNsP+9Mx4+uQYAnG1mDJUUHm4OGF1Ert8Kg0uI9Vvq5bLBIZbzv/3mY/i3y68glwCqQ9otBCfDu2+7BLhZteuzXNtF+tBNCW6p4ywxKOcT2+rL5OrHhPqxn+WYX/mtX8RnDVuc4yceVK57/AjT7ync/L0/uMVN5brlAQO+8lhQTOYZNJT2IZ9BvrWf+ubPhFy/iPM+ILxb+t/P8mMhj0KfbgHGxnXfUVx3uQjXJcb+B8sXzLiTcT1cJ/hDhlO8FM/LGMuDQ7hRXKf6vnEdO+qTNm5K/Yu/O0SgXpNylnHiHHKdtIQDqI43WhKwxHrt1H+MsnfApozzyx95Bdevl7LPj6i3hUvAUr87T28s2D3Z48FuKvXzqXPd9TTi6t1CHO57A4Zn9Rm7BvxUu2dmhH1rm4xBcV3jbh5dr//8cMBcueHwmLCcU++fXKvJXv2AXGQ7XJf7AZWrrivXjdQ5PZ4R4q52zRkXnmk/QCfq5R4uheu27yaM75Z+9zczaJI+7O8T505yHXtgfuB7ORq3Zl8ndY/QOL+0HRBu2i9C4O0/XBrg5vfNuKhcd76ZsQ2lny/GCYHU+CLGXBvtehlxPZd+f+utR3DfrVwXhdMoUudfsHCxi4A/1Ha9BjbPah9eJfhD2XZzAtXxTDEDfQIiC8fdxW96v+ZDR/i73/hL37zdSp9ebHGOf5V+8r6L8ckA0dHfrv5D633t75TAerLOHZ1/dB2wjCVO6eP7kUPyQ6aViTPL/Y/vS0f164U8+i7wQe7/UdatBYG8l/I5ArV3CDPQvmunEyR2DObbfX20n7wHvO/b7bo8TXLu91vH9xpvbds70Fhnwzcb4NGDcuvtBvms/BjIm4C0K2W9eS3g5vPl3OkJY35Svyu8esB5fWc+3E54/ewSAPAD22d4fSzbv2/zHTzxVwCA1/0V/sgXf/NDcZ1JeQ0Gg8FgMBgMBoPBcK94oRFTAH0GAZlBS43CRJZZzUxAix44iPSLAGoyUwKoRo+I1zMG3CKpjteyCC2ZOiG3Ov77LklSn9VmhlsYbq6z+Yn7zLB/GjE8q6EnFRkCRBoFR73+lFmipBoOgJ68adIryGzsLemcnkFxcngeJIKXJRghkUTPGHzCxssNWxQh+NSjNtlJFEZLyMipa62kXoQ8SJ1FakvITfqqpbzHUyVaEtuq42WiKQ8lUgpUmeRG9nc5Qitzu5YX2XUegTS0c6i3EzvqUmsAfWYNQO8rPyWRLMcsfajqpmWZaXTwNfJIaS01bFH1POiKAr6O88Fl7EIJAzwYJlyECVE11lIjphkkMrhN6pE4ZpJJ18BgPiE7zqr9meCq+sBFRmgR/YV631K/AACvt52aCaRVdKGfe/TcfibRZr811yUGdckhI7sWQleSLsflP7TJ9U46/dIMNYnuuD/rullXTfweXNcjQazKwLySuLs5w02iIPBDU00k+KvCdZSzRDGJgHB7CQRlVjymy7OeIe+8djx53WfyacV1+lntXDccSU3beyUwhpAwVq4LLgvXuQyqktJV+ymu4+PtzmMEOJG+8ij7Uy8TrZYbaMl2v5+nLo/mgL6MIo2MWMdO2hDSRtWzcR0pvkORX/EixzVOTCP1CKibXK8zmMs7uBVLcV2PhmfAH2obeVrLetuQDySRUpb3AWXu56bNmuybtHrjE86HMqYeDgfs/NL7J4Ow5KpqUZFUPybkoXJdIsWtjKzHSes3r8YMM6jKtyni5BIGdgDVTmE4GZ8Zq+gHo73TVcVW0bKXgPcMBoPh+4BFTA0Gg8FgMBgMBoPBcK948RHTFgGNEf6q6JXHyx3my/IbOQcg1bWBPFCPHADoUR5kRqpTopwJRBIV4hrlweJAdabYRTWDnNRssDbRSVDRAkik02kHEXUMALdkeL02p84QD88muGd1EVLOcg0i8K45n6gLEfW1Vjx4nNKdEzPQ1lodr185ZYREMusepiwzwtpAg9FnuN2QMYSEoUYRMlMJj7a61shd9gxu6+J0BDRjtdaor7EMBBWE67P0aZS1VsuZrP1dGZaslipwN7fIg8xIsy/XAoC0wcrkqc+gt6igbta2nfW6I17N8vf7Z5ldJ+a+5s0xoy2IdVOE3y/9eH1urmuD89bD1fFCmVfGIm0t03IxrMqZah94l7EJErFylDHWRmhrrgBgdBEz1bXVLiO2iGmSqAWghAhZPRtYQ6JCskaQnUTCELNsJ1kviUyy3hRYR7n0+HwRJgD3idbIy4Jw2bhui/lSImmxDpTssnAdo0etMoCcjlx86jGN62h2oD6GqT+fKqBUjb3qMQmyFi7x3YZF7dyY4ZbUI6YcXF9z7K8m0PWhHpfAQa2p2ZRxSUBfi8+eZDvoB5yVMVLu66HLn2rt8h1lbePNzetI78orpHNdwmaInesccec6IhZ1SBCuy546jbOnsv69bnf1hSdZj+VFNRF3DktZ1oPlHKLqCKLqWRlVEcO1yOgo/cZhzXWs1qdmrTI65rreRmo9uTI/IsZ6zZOOJCqua3xFS0ZQ6ypZvSPzLtQyub4WnUnaCFBjIQxSOFVe7zLGSsqOMgaXkGrnzTnA1Q4OLiPUbeczUuMfR7LskwCHtjaUbpMcSnvxKfWOJ+T6fDoA3GwGoKQlpN7FOnq6cll6CbjOYDAYPiJYxNRgMBgMBoPBYDAYDPcK+2FqMBgMBoPBYDAYDIZ7xYuX8lZwzqCbIm/bfi8iboushzJhrhLcmIC8UVLepUof+/8AiqQMW0SqG/aEcF3ll9pohkQ+5Rax0g8Tw83N4EXSwDDyKk+m/ilPWaROWpZGcwQtLUeIGJmwU2lejlK7dDOjfCRXSyJj0wY8Ih/KOGk9zZJGZniWsDmXRHepZn7hgL4NLpK2ljfOgbppBDPJLbySnxFWRkhNupaDSMu03LdeuGOVb/BEHr7jvHhdosdKfjvLfpdUahZfU8xAytLkcX5fUi8AwHDN2D6t7XSZEGrqDFJpLRCzpFDIIn2kJYkcbo7SV47W+RqVURV1KW8WCZwXIxdKIkUEKXU5UzcAuVo2tanKp4cUEFUjT6lp/LSRkqRMWuU2jNJmfgZcSy1xzQg1nYKf5XkAlCTYOzS9eDFCEnkbq44kravr1zgyCPkMoucti7HLXXdvnyPWnKbEBFe5bjkn5E0bP+gS0mK41pYwYGWM1cfzQbjOJZHRs+a6mXoqID9L2ioXc19GACW3LNJGNYYT4+QSg5hL6pZS4S5J1/luKSYxv9EmYtnLc8KsJLs4Kf+mdHrAUMrgyrfj5YLNu23ZAnXpK3sgqlSLjhjByfWyWg7SzKZu5RxdGR7Vog4EX42N3OJ6fVjJQ7MXQyb2ZTlE2+5ciju4Ti29WBvzoMu3KaCPo8wMIvk77IFQuW68ZGzfuZ0miObYl32sUqHkLOlSvOt8RUtcmfr1HnViCsSe1oZ+2hSr0UQMMsQIXa6bsuvLE96d/Wqpwpx8T5OVQZ3rWBk26TyzyPKdgCLgm5R5VulibtC5TkvBdeXYU38OGU6+OWkpvJL1sqPT/GbmR5893JGyo2y2d6XmutxN3Y5TvvAn5Z3Y39np+cq0+u75HOlWPsj1Pgq05zIl0eTj5IqHglNLGwDpW73Le5DKDUqbulbDO1Co/BQjsC/fAXiavr/6Ofmup+9N4wg6L+sg+XwHflByiC2Pt9i/VpYQ7l8lLA9kmQh3MzxW3wklneD4LuH8W+V+23fPEPblB4O7Tnh69RAAcHX5OfyTm1K3X7n5F4CpfIks7+SvfrgqfqizDAaDwWAwGAyfOhDRDxHR/0FE/4iI/l8i+s/q/r9ARL9FRF+r//2J+y6rwWB4uXBvEVODwWAwGAwGwwtHBPDnmPnXiegCwP9FRL9UP/tLzPxf3WPZDAbDS4wX+8OUaJXTiw5FW7Z5e99zXVLyygHVIS1NUgvwoFwr+YTjLiBOtDcEX6VrUIaPTrsDRiWH0k6VR2Vukqy4lXP97OAPvsfC2RHiWQl5+web03lJlaskxSQ5TbWDacQ6zK8lBfn2NSnltdRAy0KoSVRn7KobbpgY00XZXi4IVN0iOTmk7JBuJRIt8jadlpKV7La78opBbckN2mSGI8Htm2yQwVUSOjjCeNn6mZBq26ZBrrmStkX0/vSHIjUFgOE6K9mblnVTl6imsciqfFEbYLxkjJc1P+hVxnBZdAt+SiK1TiySM93uzEpenUTRsYg8BM6vXZ3rWGiy73adLntLKA7UvbL1H+VITcRdyruPA/Zx6I6Ukd0qD+N+GWrxnJLoQeTvjNX+3n4R8PV58wvDNfluPuoL/Zi055lFuohMPedf+Uzrt5WbZ9eO4LMHUrJVZtC+yFs2b90gD0VTShzQKk+JkLZ1qYIXd2ktB6SEVW7iJvH0B1mSoPuGvUi5KRWZb7umlmb3HKjB9dyYyxkVF1MAfvFwcYBreUmDQzyvjrspwyuH1t6/OStpburXKvdTSzJ6YdVYz3xa03Ysf1L3bTzun804+25pvHDwWM4q1z0A5rYkIzksSZ4ZQNyvC9eJFPqu3KXddXzgLhd2M/UcnS5DllJ4ID2r/ZwJcSdLHnTu5/58JsV1e2C8Kh+Efe59GFWuTlJcl4fKddUUfrhibBrXXSYMzwrXuUOEm9VyE2mI3m+UsvRRzqC29CApHnNOcR3JuVqMlWQ5CwHgJkXTzz3x6s8m0Y3Z4XoZRV6tljQ4YtxoruvrHlRbZlot9ejO1FE5FC/ixk7v5VLdnmePvlSBkKX5MnVHZxCB+xogPv3d4h7BzG8CeLNuXxLR1wF84X5LZTAYDCblNRgMBoPBYHgpQURfBPDjAH617vpPiOgfENF/R0SP7zjnZ4jo14jo1xZML6ikBoPhZYD9MDUYDAaDwWB4yUBEDwD8DQA/y8zPAPwVAL8XwI+hRFT/61PnMfNXmPlLzPylAZsXVl6DwfDZxwteY3rkcFU1MO4QsXmnaJfYj1guSrHyqORThO4SCzGbLDKd/odI2egOJWH5ux7kRQ6VB3Q5cdp4UJJE4UuVXs2PtNjIIfsBYWpaVmB65Ou1tjjLJ5xbl7SW497p/HXCZfc5wE6S1xdtrTjGhqsmV3VwsdYteFCV3HIkzNFjSeL4tVR525I8Uiz7KYk0iu9yqhypS6ZidEpymOCXKvXKwK6eu3lGXXIXd+JgSZFFHkrorsnDNWO4Kdvjs9hldS455MHX8jDaRG4r83BZztm+m7F5p7bHzSLOkzEX2RmKfJdXFsyqf5pcLSYxlVyiOHJuBnEqVVI34mHd/05kX2iuzo7QTCgp5O4cOjqRzzETpuSx0O3xkbLDtNT+TSLHvGU/px/DnlAeXZZISSTybpHrUFIuqWvj1i7RhHIcJpY+JDh5QONH4Nz3SYdvkkXq8kfazxjfqfLDjcN8UZ+rUTmrZjXyFL8p08+VXLFYutZNkn4rsnvhutydZIG8qc6miwelUp48eizVvXt6RQZI9h5pdPBTGYvsCctZld5vtthWObebYper05Kkj527m9NOupTeLdm9a9lCe67ctGB4VpdU7D1CfZewC719ORKW5DFVHnTEmCvvzdEjxbashEQaTWrZQhalcRrRXV9dJFCqSyYOCW6q77ckUtF46TrXpa2SbLO4mhNzl5kO14zxqtwsXCcxvs4eaaxcR3J82hRHxbFx3TsJY5Xv+utF+ifmu9tVS9CV43LfrznMy7IFBsQ52Svu1DJW70TKTbL8ww0Zwad6iFy/vY/cCa5bmFZcp12/pS7clwax6sMcAGyavFoOd7Pr71CfqQ/FYnB+W/5ePmj3zerLRpaDclb99smR9BLRgPKj9BeZ+W8CADN/R33+3wD4X++peAaD4SWFRUwNBoPBYDAYXhJQWQD/CwC+zsx/Ue1/Qx327wP4f1502QwGw8uNFxsxZRwZLXQnB7i5zpaqfKJMhDzWUx2Qtip3WpZZUNnJffYaAPxeRYva5GVgMY1wjDy2nD7UjT/cQsgtguWp59tMGwn45LGYIbXIkIvcI105iLmIu5okp2nO60iRPxGRuysioHE0u91zBGa+IwKB3r40SU5Bf+FXZim5RuLKrQlLnf2P0SHXHLIUSYxz7gp6EaQtBvR2BflulkOJ4arRjltY5UN1fVbZzZKDFCSGIOHAPXrj5tz7nL1Eaimpvspr8xc/Z2mPJcv0jA4BPk8/pNz7lm/2Mp7HoeStAkpEoZp90GZcHdMjat5JxDRLPikw4JUBUosanA0zsjJpIdWJUwpwNeLgAiNXwzCOQLcXiej15EArnxletUWt5uJ6xIej5PMDQxmPqQj9cXu166so/kuB3rBOcpomzXW55L5EGbtpWw1eAnfOASSiww4yqJXhTRm/Ev2RqNAR120011XeWnzPaZoHh1RVeWmL/iikDWGJQFZ8lYZm4ENwVQXhLg+guYbuosrPR1TGe91md4KjjnIwS+VVROrY/O0U16UMdygN4w6xP9v+kUoaCiBn16OkALBURUiMvufLdsuaT3TATPiKOkelgUBbyXPXjJBKBLQaGOWsiuF6SjrKkt8UEBMyzXVecXce5ZmkvDa5osyi0okMmpUi5C7o6Ompvlhi5y6eZuGxIci290DN20dB7Q9B+ir4vn0cBG1cFyj3qKkfb5e58d6UQo+yuiFL7uxwxHXKoC8pBZbmulaWtEjUmzLgsvRh57rMa05ryOpvlcf3k2h+BOCPAvjTAP4hEX2t7vsvAfwpIvoxlNH+DQB/9n6K9ynCMV+pFyrn9l6PWL9oP3Hj4eXA87b7KQ486ludr1b2kTKEvOP7+PN8tzx1Tr2f2256rlTaboBt2V7eeAWH18v2/rHHzQ9UnnzCSOeVJ2eH8Z2yP1wD43dFhdhyOW/ejRjfvinHP7sBX5ZE2Jr3OaWeOzxx/sjHs6WLMRgMBoPBYHhJwMx/Hzjph/6/v+iyGAwGg4ZJeQ0Gg8FgMBgMBoPBcK948RHTrKQuzexhmuHeLSHy7bQAXJJr+mnA/vPlmHgGoMoi0nlGbhKflZQXQFUPJZWzjKJInXQ+OrCYT6zz1FEvGzuR+LKX6wNF6uOVHLWFwoebLDkrUwKavA01HA6AghejCGZRiRGJiUz9bPVv275DsiuFI5EPxQzHS22LjKYSdK8PXTJIQ4YjPikP5eI8pa4NaQ+1r0mdsgdQDYmilgSOhFAlgG7mbnKUA7rpynJG3RCDlJzRJe59lQcCT7VPwrpcvT+imChlz3BadnysOtBqsdb0q5yRqj9U3zCRSBfHQfTLnLukreR2PKF5PpY0dhms2pUc5iYxZAdfZSTB5VU+v7YPKPK2wVdppU9IoRqTBN8fPQK6DJSVH1GRit4uKjuIgYgn5NoeLmcZY8zdOGolucyQPLA6z+VnXdZ7a4zV/t1P8FXOv5sjKF0AAPwcsH+tPjPnVUoLIO1yfx5YybpB3M1ykKmb3yBqieL7cx2gZKmeuvkVe0iOZy45bf0sY7SNmXCTe+5fWiJQ+Q3M4GYqFlRiVqDnuCzjTS1DOIVjrlMGWyclUTmD9rUMKcHXY9znN13WTkORip4y1GFleLPitCBeZeywkod2mmV5z6TRw3e5NEuO50CYz8XorRuDJRY5bkaXy+dB50x1K5l8W9pAieUdRWuuW/EJkeROVsseVvBOTNxGGSTknLzHYhS5WmaRtKUkfeLumPNe5dlWYy8TUn3HZBCGWvDg1lJezXuUuXOd91m4zvuuuOMAuLZsgWnNdbpYSprd8/o6eb+s8vBCcZrKB0z5yFCqcbqW9b7fe9tgMBhecljE1GAwGAwGg8FgMBgM9wr7YWowGAwGg8FgMBgMhnvFi5XyEkQC5JTbInN3N6XLa5y9U1ygdt/YYvrCIwDA/vUB80U5/vDEY37cnHsBDlqvVC8/U89bB+VUuSqOVsZlnJTZMKlcpyPDTeJIqvPTucirfGjzq7ty3Ve23YXTLRk0Lb2YPedoTOJg+Lxo0kolB2VHJyVxHFx3D2bvu+Mwe3TX4zAmbIaIbSj9sA0LpipHXZJHnGuO1o1D3ouOzVX9mVbWkYdqM8nL6aLI0vwgLsZxS0g7kWwr1V9vU38ghNq5CwguNomV7+2YRtclWeHASL0/CG4Rt1vKqt2C6zIrJgfENj6V5ExJWfMuIG1bnl0HNzd3YJGxuUXcM92cuhtmPBvW/dZlwyJrjzvf20WXIWWHqT6uS2bE7LrsGgCG6k65JI+Ub+s0KUEGvBozlEXW6xbAT629lXQzKVdJPb6O5OVdZqiN67RkV5/7WZe0ESnXZTWgU+qOzfT0EmfffRcAsPvmBQ5feAgA2L+muO5Vj+VhHZ8OYK+lg228AbS0fVjLsdWYb88SZdmvcZzn0TdKYlRX6zYe5DkGgOVJ5bqLDVwd67Qk0JR6Eajx2xJlCQPen7tWruN35RXWGAK4O1wHcMtNPZb/AMAPecV1Gx8718XsOtfxKMsNABZ5LQrHAUUq2o7R93AL4OfaP1HckeOWECvXpZ2WlFKXThdZtuiJXWzaUukDKRcQJs1tJZe1b+MhKhdgLy7y9eB6XSWXZu4y3LQbkDeV9wN1d2Dt7uu006/azttwclkAZQb7lsfVi2u0ehnH7KC/msS8nj9vEuw7uS6LNHdltJ7QJe9+BlzjumnNdd11/mg49r+V+y4dL7FJGccoY1u2DQaDwXA33jdiSkQ/QkRfU/89I6KfJaInRPRLRPQb9d/HL6LABoPB8HHAuM5gMBgMBoPh/vC+EVNm/scAfgwAiMgD+C0A/zOAnwPwy8z880T0c/XvP/8xltVgMBg+NhjXGQwGw6cQKs8jPoa8is+N4/vyB1TCfVZxrHDROUA/Lfld7+hbzgnczAeJgJsbOf5kHlMndT7RLjTUXNDeg3bVBfEHXsPV7y/q0Td/asZf+CP/CwDg28sr+O2p7P/aO4w333oCAEhvb3D+z4sc5Y1fybj4x0WZhTe/i9zKlxJ4pYATBU3rkffIfP2x4oNKeX8SwD9h5m8S0ZcB/LG6/6sA/h4+wJc1ck6Srju37qCl9vL39thU9zt/uMByUY7fPA24OpSBnTfojoSr60eIOyErw9QgBrPHroRdGamkm+18oMqulHNv3EDtQJffLWehS5TYibulX9Cde93CCDelAOGQ4K+KxaLbL8rZ73jQnhjkzuGUEyB7aVMePNKutN38aOjtNV9Ql5J5X5wqm6Rq6yPGquXbLwMOoWjUsmNwdbdkEDKapInkWWORurW2KnVGd8bME3VH4LhVcmklA2YnarO1yy7gm0ybJdF8DnJu2tBKEqyRg8jImVS7askV1DHBdfnD7mWQAAAgAElEQVTZ/MqAm9fKYJoe0WqM9bacRLrnFhlneUR3Ui0J26useebusJkDkLbVkXKMPdH8kh2W5kqdHZYkUl4ixnYog2w/D5hjKV9aPCjVY6KMHcroMneKa/luqJw17Bm+yZQnRjikeryMMRdzl1FTUi6Uz/NyWY3lT6S87fvnuubES7SqbxeHpgTe78sfy4JtdesNNxdYHpbnbfPU4+b12u+a67REMYtEkbIYh1Ok7khKSRxJT5jRCrqClPtzW5zJASg33SZHnR8EsJPXSBvrfmaESbhuuCof+H2Ev67O5AdxKweRcqjNa8ljl7/jTrlvOyZvBqQHpe2WBwFpV5/bC+EY74sjb3u2zoepS+EPMeAwlPOj575MhJmQ27iHcB28cB0N4rLrlrL0o2wL16VRtnNQFK+GCGWIOzKxLElhB47iqttdlkmWSyhj4PJ30NLd9ThcOdO2beeQq/x5fjzi5nN1+cAFKYmr9IGLSjabZGzlQV2fFY8v4pyeRkLalW0/pL7CYE4ecyqVK3JdKbMj9H6bloClHhdn3zmOkpTVJSWpngF/KPvDnjHcVJ49MHwbq3PuvEeLuI5TzF2qTCnJMpz34rpTbf1ZX8JgMBgM3yc+6A/T/wDAX63bn2fmN+v2bwP4/KkTiOhnAPwMAGxx9mHKaDAYDC8axnUGg8FgMBgMLxDP/cOUiEYAfxLAf3H8GTMz0el5eGb+CoCvAMBDesKrPIbtGO/AZ5uy/eoFlosanRtdjxCkUeXYcxKpygys8s61mUkvYWiXcNIQBFD+DyTH5IGQag46UrO9lCT3G4cyi7w8kNn85UGrj4r6ee5RKxeLKRNQZm6Hq3KPYR8wPqvR4HdmuJsSSXCTiijomdm7DGiO/66zs2kbsH+9tOnNa06MhnZlRh4AYvQ4zJI49GIkjNUVZzcsCEM11wkSDeRFeWYQem4/KPMJAKA2m58kepqUsUhW+UpXIPksZelbF0u+03Jf6jPwJR+qjJfVpZjgD81ExoFSKdTg0HPRUeIS3aptJ/kdHdK2VG5+4Hu+ycNrvMotKeWjvp9UXkQO3Ntbm9G4Bd1Uyy1AOi9tvQ0ipJhj6DlNU3ZIya1S4/XIQfRY5mrOdAjwB2V+VMtHsUQPACDsS/QAAIZrxnhVc6XeZLilGTvlblhDSY/DLA+Z3v4ky3GeAx8Z19WwGqcMqhF3hAA+K/Icfv0x4sOync5C55y0IaSe45dWpmpSyHXE6y5zN61i6ONQnZsDgcYW3idlkCRclwdgfkj9oeYALPV3Nwd0QyYmwLUI/UJdBRD2QLgu9xhuBozPajT4nQnupkZPF5Gv8HtGlU5wnzJIyorr9q+6HtmNZ+gPXIoe+3noOTLz4LCtbkG7YcEwFt6LQRkpqcggK9Vg+7sVp3FdySfb+E24jgNUyBwSqaQ1D4pqhFC9sgCIUiQHYK7vnjzSOvKaCeGmtZMH1Qv7m7iKAJKO4rWIc3Dd3G2+8D2P+OFVPoqwtuschWh7+XU/YfUO9Pv6bk1AOiv9vvHcI6MHFQlNqXCd0DLD1Xun5LAs1ZzpEOAnMT/qaoJIa66r7TJeMcbLFsXPcI3fNNep9yzF3PMza3XIc7+XDQaDwfBc+CDpYv5dAL/OzN+pf3+HiN4AgPrvWx914QwGg+EeYFxnMBgMBoPB8ILxQX6Y/imItA0A/jaAn67bPw3gb31UhTIYDIZ7hHGdwWAwGAwGwwvGc0l5iegcwB8H8GfV7p8H8NeI6M8A+CaAn/ogN2ZWuRG9x/KkaMOuf9fYzT6WByJv0nI2t4h8Ko9KNkQigaIkhi88o8uN+FiC1eWakj8zBwINTW4kuUopiSwzD8WMRBv1NCOHHLjnGMwDK/kU9ZyReSRwaDIzQty04zfYvFPuHZ6i5/9byYccJF8a0Un5LoiAoRlXDDg8KdecHhNSNfriIPnv4uRxGERLNo0BZ9uigdqFBUOo5iCO1yZRzfvJ46TkkDJWeeG6FJTEKMMluQ6papYEiFVqvANoIyYWIpOTPJxFTk21ztwHSbtPuFYy3y6vpZ6LNNyklQuZNvnoBh9auTUy8qZVDl3elrOqnNY0OxYlYiZQleWxF2MWHNQYy9Rz+C2JEJuUNzrk7FbdnqpULmeHdKiubgcHv28SOkKuz4mfSUyOrrlLecfrjKHJ2w6pS9NczGvpmjZBUblL39f06E7Z+SdH9vaRcl3jtxjBrj5b2xHL50u+0qsf3GD/WjPnAbhxXQJ07kUormt9qJ89ygArKXh/Jh0r3lNmRkHykOaRQCoXpJhZybjNGyAN6NOYOQBpU+W7g4xp9ur8SEj1udTLMOKZWpLhCeO7lesup1V+zFNLPm5B5wSuuWLjxYD9q+Wah1ep52lmL3yQJo9pCAjN8GgMeCVUKa/iuv3Ru4I0753ICUsZ3eiu5MxsSwfQZdGkljyQw+r9043bAhcOAeACuqFSGo+WLVSui+eKZqgsHRn6+wTgqi8eB4I/VDPBvdIHvwf6e3bDyFv9Em7PPa1J8dSSmUz9XZwXkR37I65r8l1m6hLdlBxYmR8V1bHwbK45ZzXX6Tymbi4SXgAI14zxunLdZUK4rvnFpyT5WNUYXPHZ8VKF55HqqmPadwD+IKEAg8FgeAnxXD9MmfkawKtH+76H4lxpMBgMnwkY1xkMBoPBYDDcDz6oK6/BYDAYDAaDwfD9QecfBUCrNFG3c11yVo5uAMiL/ICTyv/XXRk/OWqcFwa63YardtWfqbbs7fdxtNlHmd+1qwLX5X/ucq9SQd5xzvuldSLXxx551xU7NAQgVMXabge+KErQ+GiHb3y5bP87P/nr+IPn3wYA/OHtb+I1V+RrFy5jJEKqZUoAfqfmAfs/91/E33n7RwEAl3/1D+B//M+/VK771tu9SCH/c/we+s3bZWXGh27tY0Xm++3/iPDif5jWgUREQKjyq1fPcf27iubq6e92mB9X98QHGWjStSTSRzdTdzEFlJKIoeQ2WOXt61In3ZaElZthl4d6wHnlnlpVT06Z5OZQpXVBLtjzyilZq5Z6QZUjB5Vb1aFLfyd2oPqBWzJ8lR4xcskhCZTcpf2mvCbhEw9U2rnuVhvPWe7rIY20OCxz6DKp62XE+VASXAaXe+44YN3eXaKWpM5OqcTcInkyoZxom5SwYSVxWvVJrcNWbEXdQICS8na3UFfqBwDxAatcc1WaqCRhc5O+Bo/xujnXMnzPxak4jxl+Ebfa8Vk5fn5EiLWAHI7bnuSf1l5EIoNdqOfXo0XyirpFXCvjIo9nzoTcpG4ZyNGtZMJ5aTpLAuay7fdaygv4ljd1D2yelkINN4xwU+u2T0XCC4CWJFL7xOLoe4cr70rWe0xY6Y6Xx2f8S0N70ZP3oCqTj5+7wOUPFwfyZz/sMD+pXHeeRR4ZCa72oXa3LRdV/z6PElo5yerlDKyWLTSuQ5Znl9QznH1dMlHLxx6S31PJVG+Vo91vAHL7zqNyExO77hjrYhaHXiblVutW46dxNKWj5626HsedQ2xcd8ZdHl3q37iOEJeAgy/XvRlHnIWaR5oYTktT1TtEcgIf8XuF5jq3sLTL8Tv8lIstoS9JSRuRkWopbx6B2JaI+MJxpc684mQXlRSblVw2eGyeNbkri8EEy/2IGW4uBR+ufOe65SEhQnHdit9OcB0guUQX9e6O6O65blZcN4utNGdCSvVeicDpDv1rBlB5z98orlPfwsINMD6VpQrhWnNdGeQUszizH8t3K57biffEMZRl/0mHbYPBYDB02IoHg8FgMBgMBoPBYDDcK15sxJQADNWNYjMiPzoHAFz/4A5XP1gNK17P4F2NNIx5ZbLAdVY8eQI3iUKSnKYUZfbeJQLVCKeepaRM6xxrDXfNaGuTngUrsyOwXFvPyK8Nf2TG2ilDJmJtUoJu0hETsJxVQ5DzIDnVYpZyO4Cm2kaUV0ZIbXYdzvXt+dwh7mozDhLtyEFFPzMhLw5LNcq4mQdcDyWyM/iElGUG26koqYsS6Wv7dWTZTYCfWySAel7BvFlHmPPYZpqlf1aRna0yclERaXbUo9bsW2S11LEbCnE1TAqtDahHkNmJURUlL309Z4kiZAYWmWnfPG3mXK5HOtOW12NBTfn03Ld6LE2EcGj3UpGqxMjVHOYw+ZU/UI4qKqqMuMC0Gm9UIxLhRu7hJ/SI+3jF2DytJihThr8pN3cxiwFNYpCKGHQlQtKRgKNIaD7xXLXPjkCrKOrp0z7VoCrrAUCbEfmVCwDA1Q8prvt8Au/q8z3kPjg4ODQ/IgqiDtB5lI+5rj1zLklXHXOd9uHibhykopAsY9gt8ry0KCfVC2TVYZRUJDEplUoUbtBKEQTJMRx3wPKgPKR+GjAkRZQqR2nPcRqTqEWymDmVPKb1mTxziNXcLQ8o5ky1DpI/lJBmh5lKOPVqGLELheu8y8itsElMeygKr7moVDRaHTIL17FHzxedNmLilz16FLeUS7i489Mu92eClQSPVX7kYr5X7zWw9EjNP9uulTaQihPgavSRsu8vfzeriGHM8HV7uI7YVHOqeKa4bpQI7bGZz+pd28bnLFF/N0sOZRe5t9Fh8mhNyUzgznWoXCdjqfE1suQFD9eE4UbdYxGuG5+pKGk1fXJL7kZHlJW52+q7QpaofM7vb8ilIqY6ByqOedJgMBgMd8IipgaDwWAwGAwGg8FguFfYD1ODwWAwGAwGg8FgMNwrXrz5UculNwTEiyKf2j8hzI+qLGmTRbIp6k1QEFkve4c8iFFIy/lGymSBI8PX/azymoHEyKUYTtRtLRtlkZtpQwc/NWlUrcpS5FXtuCbXYq/UwFlJmiJ1GRNwZITAsq8b/mwc3K7KASN3+SM7aqnwQAfuJlLa3KdIeX0vj8hglcQ18EoqypGQq/x3WgKeTkUTdz7OXcqrc9L5WerjZyVvS+iSKL+wMt8AlqJoRHzAvQ/ZATxKf65ygLY+H1QnekDL06Ak0d2kR8neUGV8q5y43dSEELsM0vWctQBUX4m0jhIj7EvlxqfUZYxpol5/DmuJm86zqsfSUI2t/IR+XyZ0mTEmt8qrijbWcpXxdtkknZS3uUnq4CdGuCknbJ5mDFdV0janbnYC1oZRLLJdLUu+y7woc3+2kZQEDsBL65DYXCW3G6RH5Vk6PHGYH9a+Hllp6dGHtBuSGA0lyfuJJOONFnkOeWHZVr40JXepXFsvHeg5MweRRrpFOMnPRUFZTpAVFQBAnsRUyCn5boKSGou8WC+HoIyVUU+T86etA6VqhDS7lby7c52SSq5GEhEQ6lKDFdeJwRz08g0GEB1S3XeYBzwNpX+2IQrX8VqO2uqjuY6yyDZdlIKlkTrXLQ+4L1VgL/JiZKz11W1JQsidyzkRtIFVf18p3qOoJNtUJM6p8SnkfGKpAyXXOTpAcV1WebeXjOFGcV3LRbqRpQMc1lXQxkP9+gfA11zJfkbPxcoEzM14bvKr11cffJlqjuT6d6SV8VSTF/u58F25H2OoXDc+Sxgum3w3dVk4JRaJrcrBzESnOU5Lc5/DkZIdreW8+lyDwWAw3AmLmBoMBoPBYDAYDAaD4V5heUwNBoPBYDAYDC8eWcLsnN/juBPgGN//oI8Ld0W/T+RfvZVzs3/+MaiIVhH/tCrGJxLvpyJ4rzbSbU2Qin6InKZuU+WQ3oO2ZZu2Wzl2COBNcSnl7dAVn/PjEfODahK3JTG4jJIyzC+MH/47xYnyn/7CF/DP4g8AAP63yz8AnopchecZSOk98smWnKWv422sRv3z5GX9sHgvo7ePES/4hymBqnsiEyEPTX5FXSpDkytyXgA0MsJQOikMCa45Embq2ym53kYx+p7PMe0DcpW4+oPIx/Io8jFi5ZY4EFJu0lIS10ovcjhSyjtGdaVV7oTicKuqzEeulToHtM65eiIfaA6EtCmFdZRX6UvRy0civ3ROpG5e5G05rGXG3fXWi8y25ICl7vwao8ccy72DCz2v3HHd7gKdGLjsxUkynmXwVl1sUJa27dR4B2HpY6DaTkllQegaQPZcFGEqf6vrjqPK8ZEAplrn0cEfxLnWNRm1lzL5GeCbJuOTfmBPqzHWHCLLOK/nLozhquwPE6/ObQ6WtNDqfq09KNOqHyjSSh6pW03nmPRNsrzk4kpZt0+SDLPI2/KHkJ/dJfl9aeS81BNtc/DIY30W1fNKkwO3jhvyiutaPmHm5+O6Jtv3B9HNrrguQ1xfg0g92VF3fg2OV/meOz9RHUf6na8+69Bcp3IWk5adpyOua7w8ENKmES26gzRl7lwM58QJ+iiHKXeuI8V13CW8t7kO3fk1Rte5zrusuI7WKUeVpJaO6r36F1g5hMdzBm+bJhhAUFzX3a5FcnsrF+gH4TpiZJe7s3ceADe2z0gkwiSu7XkkxXWucwMP8sLxE3pnpUXkruxJ6JMhPEYEl2SpQpPW+jl3GXQelNw7ktKO4zbX1ZtoufTxe6Ddj7IsjXCRu9M4Rcm7zHTUt3fhefjqlOu4yl1qMBgMhueHSXkNBoPBYDAYDAaDwXCvsB+mBoPBYDAYDAaDwWC4V7z4NaZa3kLyr+sSMC4OrADCJuJ8V3SN23GBq1IiArAJRQ80xYBUZT4xeUxVknUzbJBQNUzwXVbGQTlVKgdc9oQ8VJmTdrPUroaRgbEd0+pTLzUoB0jHXf5IQJdAMZHIgrOq/zFUu3QXxkDIUFK3oUp8k9bcCTi4LpWOZ4S0q2UYWNxqAdBcE64zwCyJ2uPiMQ+lQmfjAufqPXSieu306fslkR1ATWpMIqsq8j6RaHXZYMhwg6pDa1PyInVjApos1TG49lUm7vJBEHfHSx5zlwdTlfG1PomTB01VRh5EnpsHcUANN4xhX7f3VCSSAEBreW1rerfIegLOrNwvxZU3B5bjI+BnVudWmZyjtVStyWgZoEX6CloqeZfUlkTCDSj5ZiBwaI6u0leA9CPBgbPq8/a5upc+79YU1/s58b4MMjcW2WmTywKK61jG67CNON8VW9HNEOErZwSXMfgysDTXzTHgsJTncx82iF1aK89Medbr/kirZ/UU12mDYBdZHdOuXf5JoxpXiusc+pKmlbv4sSy4Q/OHkzGZE/WxRZlBQxv3rJpUllvAE3Llw7gTCe2K646eHx4YXBsqLR5LKufvoLlOSZuV4p+DPJbk0d1qAV4vz8iqn1sbD7k4zEOOA4C8eHnus3AAe5Fgg4RjQOVdBqAsfWlcd/TYpcUhz23JjJM2HghpU7aHG0LYu7qd4fdqzZ9eSnDCaR3KxVwvQ2G3Xl/VljO4mdW7WOpPip/B0qaUqgtvf29gjXaKQ68PE8NPtZ6BuswbUZz9y218bbO8Xm2g390Nmt8yrxv61PbxOpeXge8MBoPhI4BFTA0Gg8FgMBgMBoPBcK94oRFTArohCIaAtK3mHRsgnlXDjouIzUWJHDw8P2BbI6ODT8h1ypqZ+nZwGU7t99WlyxH3AFtxZZAI3inwrYiP/NtnflMxFAGAtK0R0jYZu2WkixYu4H6jYmBTyxS5z35TotVsseQxVdErAlKNBLToWkOoM99hdKClTUEDroZO8uCQq3FS2kBFy9QMdyT4vcwO5wjkavyRncdczaPSjuB9S0qXu3lS9oCv0Z/sZZaDSc14kAocqZyuYPTZazfkbvBCKmzDTODltvMUs+v1cY7EXMihl83tInyNTPiQkTN1U5PsGexqhAUq1EskuWwHMVFJg0eoxiyUuUdV06ii5EA3IMlezEEonY4qpwFwG4letKhyGkly5arci5SpR16RqURilZHWKh9vO2xcBTTQ3LN0Htchcc9jWsahRPe74Z123XLqGDgxuVIRLCJaRxGO853W6/f9z+VC8ukCAaChDCAeh851WXPdg4ht5bpH5/vOdd7lrg7Jiuu84rqkjlk9M4SVUeFJg7KjqKU23eljbJKoaNrUyF0dY/GckR5GOb9yVp4JrnMdVF5jMfnR5m9gUU2wU1w3utXzkyvX+UPq/KbzbbIj5G3jOkIemlEZi9plOeK6hcQgyAHzWAqWt5rrWIzinOIGKK5zygAvq+c+Q6k90LmLwprruL+7snCdThobGKm+c9xCK7O9xuWa65zP5f3YuC5k5Mp1ibEKf3czLM11o8ewadFKIHaOWvOXzgW+ioaf4jqS3KN58KIO2RDyph2vI6+SM9XVvKVtLK1URCqKnTaQd+iZqFpIResps+RrZf3SVQ8EkXrPQCK1mseCesDuMHbT/EZJGcxZHlODwWB4T1jE1GAwGAwGg8FgMBgM9wrLY2owGAwGg8FgeLH4NK+9vbPsp/KV3pGWzfDB26Ud3w0NPoAK4fjYKi+iIYAenJft8zPkR2U7XmwRd0WiMT8KmC/K8cuZyALZiXLDLaJuzEFS/7kE+OtiPMLf/g7yVJRSH0ke3s/guHqxP0yJgCoPzZsB84OyvZwD8WEZZMP5gouz0mmDT/DViCJlh0MM9TLSEdfTiKWagDCX/H4AkG4C3KHJykQ263htQOS6ZEjyTZbtckyYuOcCZAfEZnRRlaXZKYnsWA0oQhbziuCQquQqR1I5TVnJhFQ+1ZHgzqpJz0Fkqm6hLt1c5UJlwHfJmEjDVoYrSm5Fi2qLhbpJBDFAg0iJk0fPoZqy62YspORTJaGxlOP9wATkbk7EcFU+N4zycKbopQ4MkeL5DG75BZXxRwbE/KidU9EkkJSpKEqbTDqLqZCLAPX8niIZoyx1YqckbYGQas7ltJV8pat6emV8ogxB0mZt3rHMyhCnloEDsFy0+7JIIBcqfYdCchRVDkCGkuXxug3G9i934xOwU/I2IFzVMsQMamyrJbvhSFjR8gVCmYZoAyW4tZa757Pk9UCRk/GZA1FftpDH0Lluvlhz3YNqeBRc7ly3JI8llwb0xN3w6GYaEeszkJLDMtfr7zXXQfqfqXdL4bq6rYy3KCmuO4hZUGJCTMINWgWaFde5kPtzxUEk9lnlpaQkhjeURELrN4WDACBOBFfL5GeRfrqFMepn7NDqw2iOT+ydGCGRevZUHmQ3OYSD/E2BheuCQ1xOcJ0TKfBqecYx19HRv3W7mxMNDLeRHLUNOTnk1nYqjyk5MaEruVjrMZnLcSfQuK7Lqtt1s+qHhVZjQHNdA/uSJB4o8uWouU5/W2hUot8tytAvHxkbtX7WObs5APMFSz0V17lZuM4t1Dl69a7Uy1J4zXXNuEuvbaDM/QsPLUkU7LpMXt4NAEC1clrWuzZCUutWMlZLG9ByX/cL4BMj5SWiHwLwPwD4PEoRv8LMf5mIngD4nwB8EcA3APwUM79zX+U0GAwvH55LyktErxDRXyei/4+Ivk5EP0FET4jol4joN+q/jz/uwhoMBsPHCeM6g8HwEiAC+HPM/IcA/GsA/mMi+kMAfg7ALzPz7wfwy/Vvg8FgeGF43jWmfxnA32XmfxHAvwzg6zACMxgMnz0Y1xkMhs80mPlNZv71un2JwnNfAPBlAF+th30VwL93PyU0GAwvK95XyktEjwD8GwD+QwBg5hnATERfBvDH6mFfBfD3APz5971jk7Q45a7n0fVROTtcH4omJ42Eq2rb513uEs+YXd8m4u6iGKMSe2eCqzLV9m+5F6+lvE0DvjD8VLcjw1fpkJ8yaBBpk1dys6zkPd4DqcmVnDj7UWBwkwMNazlpl6BmSC7NUcqURoKv1/R7wE/lmmGP7khJkYvrH8o+avI2on6Mm9Elu+XA+s9CSmJXZM7NNTUHQp6bvI0QvJIpq7yGKxmXlvV2LZrcNp4B8UHd8WCBr7I255Q7JY4k881wWNvNAlIGnVdVgZnAUWRyzASu+fyQCDTL2AjVrdNPJX8pUPOMTq1txIl3ekyIu1q0Ucn11NBjL3I/StIGklNUfVaL38YopSJFK9tKdj2r9QpzkeTJ2JVnKW3XjdHkd9kD+bwVRI099r3Bh6uI1fqY+qDoHJyAyNW0Ky8riRohg5sDsJLvrlwugS51+6QoeT82rsMR11Ucc93loXCdU1pRzXXOiaNrVMsTkNGlj/4gktDsuT8nWrLr5zXXteUCYZ+7zJa9cF2a673q/fxEkhtT951nwAnXQclU+3hLIinPo8jRNdfxjXCdi/KcuCT5fl3MQJaxqrnO1boxuT72tMSZEuAy9f7JEyFXR9yUCUOoLtU+i6O2UPqqLTTv6Ucn7op7MQDggbjmrpx4od4HLJLbW3LdZsVLhGO5MLDmugwHZPkbiXrOZj+JM3HhunKIHg/EjFTfd8sFIZ7Vy2yUdFa/u2nNdTrXq4bO79rfRVk4kZJaXnHMdYvmYrl2Gtft0SW+iusokizXSeII7JnFzZ6o57QFUXdXL+s/1A3a/pWLL8vfxKLYTSjfA1qD9ZzQnzy/SSL6IoAfB/CrAD7PzG/Wj34bReprMBgMLwzPw5K/G8B3Afz3RPR/E9F/S0TneE4CI6KfIaJfI6Jfm/nw0ZTaYDAYPnoY1xkMhpcGRPQAwN8A8LPM/Ex/xqxz6tw6r3PdgukFlNRgMLwseJ4fpgHAvwLgrzDzjwO4xpGU7b0IjJm/wsxfYuYvjbT9fstrMBgMHxeM6wwGw0sBIhpQfpT+IjP/zbr7O0T0Rv38DQBvnTpXc92AzalDDAaD4UPheVx5vwXgW8z8q/Xvv47yZe07RPQGM7/5XgR2C7HKNw8Rw02RtwzXDstNlY0SMFXVy7QfMG5r0nmfMVdp6TgmuCoZO+xHxEmq0eSa7saLVFK52LJby09PbbuELvmhhN5KWtVDqSh4eqL1WSRTvNJ9Ye3aqF0eu+ZunWi+y/0IXR7mFsZQZabDNSNcl0r5KcHNtYKJQVUy5CDyYDeLK2IejuR33ZqwutI2+d2ALm+blgEPtmVW1IeE2JLO01pOqGXBvS0jdyloPGfkXfkgDLkroFJykhB+9iJl05K2sHbIlDzxqu08FzkhAB+y/HpgAkconRn19u4e5VEAACAASURBVPDz2qnSKYfe7lya0ZPcpw2wVCfJtGPwqOSoLUm8U32fVRsNRzbyrX7RiavqQbs80moM96TzC+BnkfKGvdwvqclrdoR4Xm+RlQRuy1geiOtlqPf2s1u5X+rr9DoQukSNonLlda5LI7UrL2cSp8pjHEmEPwH46LiOGbyUxnTTguFac12VjVLAoY4BzXVEXJYloHBdcyHf32yQKgcyA5jKtrtxMobVECMiNH9XyrTmOsVvze1b/9w+5jrdVX4iRM11Gl12ur6edgq+i+u6ZHfmLqkfr3PnOndIcIfaRlmlX8iAq7JMt7Bw3ahklgql/orrApBrW07LgLNNebCcZyQ9datkmqslEEpqnL1wXTovB/kxyVKV5JCr43JeHHiR5QUdQXcieuOzen8cc11DzoSsZLHIiusmkiUqM9SyFNkGgFQlsWmruG7LyFvloKv7WdNaq0bgNYkorkuK63r/H3Gd8HORZjdpt1f8lkesHOIbp6WtLGFIO0Y8L/vn2cEf6jMTc1u1AmLubuTr4aze40nVxUmdn5vB2jj8hHAeERGAXwDwdWb+i+qjvw3gpwH8fP33b91D8QwGw0uM942YMvNvA/hNIvqRuusnAfwjCIEBRmAGg+FTDuM6g8HwkuCPAvjTAP4tIvpa/e9PoPwg/eNE9BsA/u36t8FgMLwwPG8e0/8UwC8S0QjgnwL4j1B+1P41IvozAL4J4Kfe7yLMDJ7rlO00Y/NO2d78jsf8sM40k+852OgsYtqX6dswRvg6QzxPoc8S73YzXM17GrPDdCjHLzTCHWp+U2VGA4JE21TQUs/eZ9az6SzGRIPMxJaZbKxm/6WiEJOXeGRY0U0zjvJSttniSH1W2B+A4aqcsHnGGJ/VXHj7BHeoUYQ5dsMjAJJjknKvczENEcOJPsmsDYvyeoLbzegR4OkwYPA67Czn9GhBVHkSE69mv1NV+uSAPsvPTEg156xzGc430xRCbrPKnsRMQkUhmZ3KkYh1ZKaZvShTLACICEizOu4uI5MGkr4lHEUQm+HRmIFt6uXTZSQ1HnIrq+NVAIfr/TIpw5NmNIN1FP+WeFSN3ZZTF5CIb7kAI1ejGX+gnkPWJYnK5oB+TB6cBHtjlln+U5GvYzD3CBYxA+n2/hJVz7I/33Wxe8VHw3UAULmO9hM2vyNc1yI7C4TrcBYx3RTu8kOGrwY88xT6s7E7m0DVjGaJHvNYjk8YwDXit+IhEpWBjgQVrqv8lkV9wIHUWFA5eitn9uOcmMJQgkTMsnbCgXBgQjegWakpFupGRWG/5rrhsuZ6vYlwU33GpgRa6rYeV0An5lU0U+XVXJkU8fpZ9xP1KN7hiOtERUMrNYXmOtkGcjNGG9DbgjMh5cZ13NU+GKTptMEYeakXE8DN8EmrLyDXJ+I+RhwTlkxgleNYoqeQSN+ReKO3U5L8rkzoYyBvGVxzsVJQEdN6/15efg6ua8ZMdBzFJ12tdTVJbZ/iOpb3d9oQqBnIRVL5d0U1koODa1HQBFBTFtyhGiifSX+ueKxB5zHNWfhtxYGfDM5j5r+PuwO+P/kiy/Kpx/Pkpr1LNfQyYWUadjsmRo7AKcnnLCRF/kTC+HocDeWhdk8egy/qC/K7vyNt7jzoQdnPQ0C+KMts4sMNvv2vl+03/s1v4ccefwsA8Hj4Lfyzm88BAL51/Qou9+Xcd56dgd8qx4dLYPNuqc/5mxm7twoZheuI8Kz6SywRdHld7rss4KuynWdNXN8n3mvsfYrH3HP9MGXmrwH40omPjMAMBsNnBsZ1BoPBYDAYDPeDT553ucFgMBgMBoPBYDAYXio8r5T3owEzEKt5xRLhb8r2cMMIzRBkq2RBk8fwoOgviYBUDUF2uxljlbpdH0bM01CPYcSp6XlIJJdRqbj8WqLTpUoDIbUcnp6UbM31vG5pLMcBVQLpVe60gcFVPoQhi9lDJpGdZnQDG8raaEeMH3T+zHDDGKu8bbxMGK5r280Zrkna0lG4vkn0PHV5WPbUJcg6jyL1/6GXaZU3syoS4k3AYSgVzcmLCU9U8jZloOGiSMWYgLRT927qqdnBV+OgnCUfJojhWr8lEtmxmkJJB4JrufkOYl6UIG2dmTBWubdzxWipmTkxEThUA5Igclen+lM3Di+8kjWuzKxUY/YyKhkxESOSSvpXwQyVwxBYyQxXUl4ZL/1WfHSOkh2vJMj6WgnwSn5JSn7Yr63y4FLmkitSF7hUSCS7+nMoudrRmOx9yyy5S7W87dOrOLkbzOC5ctcS4fflQRluNvD7xnVK7XfwCBdV+utyl7nvzqYuLb05bDAro7dm2EOJVJ5UacwcoMaoGPPkQEiVq9ipcx2J8c1Gcd1wJIMcAN7UfvQs/LZgtWRgJd+t481FUtvoHBP2jPGyct2zhHBVjaPmpPI0H41HLWNqUlidY/M4x3GT+Vcpb5dtzjX/K4DlesC+vltyUoY8yhjNzyw5hCO6JJSVGx479Acxzx5urNfMtOIJ6sZFknOVPPdngqcgXDdR516o92FmwmYQ+TEzsHQzJSfvwSAS7uwBWr39G6cftV/P48qrtmzLFogg0mSgGztplEf99LIAvaxEeEgdd8x1UtRaLrVLcV3PC63fsyonLuU7cuIe+U717ZREBu3Ue1fL5fTyhBW/HR1jMBgMhjthEVODwWAwGAwGg8FgMNwr7IepwWAwGAwGg8FgMBjuFS9Wygsoez4+nc8O6I5//iwi1URy23HG2aZocuboEev+R+d7nL3yDAAQKONqKQkn373e4YbP2wVXToOSm4zATYoblGzSsZIwiQw2DyJna1JfkQbJdSlkkdQSVA435dCbRN6UmeFYHa+b65TplhOZLohOW+s5hzwoyWCT7g2q3bMopAIT6HAk/ayS4rT3mMbWUISm3KJFydsWhu/yNu6yrKTcPYuDp9iENjfHzXbBZigXikkGQsquy8RScthfFXtfig5urm6zM/Xcm2AgV3fNNAQsVU57vosYtlNXUS1z6LLefPDILUdpRpcH5oQuKXYDdTk3e+XM7MWJlxyLSitTkfUBq6kf57JI2ggi1yORYOaBe910bj8XaeX+6WLJ2QiUsZdVrt0uQyclNR/VWGXZLg6jch3KSqar5G06F2mv2y0ZW72+llzq4zLLNVeSts+ovE1L+WqbaXdbAH18uLPY3Zs3Y8b5tmhcC9eVZ+LR+R67V8TR7xBLp79zeYYZZ3JBpQ5dFUfxWCMB9iLxXY0XJd0Vx9b6r+5ezXWOurs0RerFIKJu4srMQmrK6PuWG2prL+/6mCy6UXlo5BkTrotbQtrIs3Ry6jUTwgGSIzqKlDfuXXc75uQQlPu3XrYQal7Nkvu5bKcRvS1JOeCCixsvcJvrcnsHZOrHpOSwv6xctwjXuYW6hJgyIW2E6+bKddvNgvPdjJv2ypk9cpX/5skj6zzFLT/qIvxDiZBGxXXVyRuBxS3YKakxA7n2w0o1reS9zKTch+VePDCwtPZdc90qp2nk3vaUZflN4c3exH3scoDIerPmOpHvFilv47csyxKOpbatbnp5QrzDZfdIvkunZL3u6PoGg8FgWMEipgaDwWAwGAwGg8FguFe8+IhpqLcMHums5hw9EzOOvGPQKLOtT14puX/GEHFYyrmPdgc82d6U40F4NpXcQlMOuJnGevkM2lXDiUUMNwCsDUF6REmimZQJuR7vSfL85YHEbKHNxHajGcJSTSrYc8/XSUMG2kxuJHA1rNCGJeRJTEsciXeMylvnF6/MHRihTok7l9ZmMy3XmqduXpK2QNrV9j1P0Pn18rbNjvsSfdUz6qyiH5Oamm77tSlFlshbMT9qhiBHBj4tqhYyQjXsWBbJ57gdlx5JTdnhsC/9maID1yinv3EI+1r/SXKpEgP50KI/AXObsM6E87OpGxLxkLCc1bbJBLjab06i4y5BTJ4WmY1PI3fjF7eNPYrgtNMQU69bSq6bHCWmVY7CU2jGLK293KLq2aLTE+APKkKdeJUHsefjDYRYjafSjsXwKMuxxCrC6tCvQ44kouUgRk0O0l4qwkxR3GQYbm14dKqeq0jDezbJpxa04rqyvZwRco3opV0uRmkAQMJ1wSfM1fxIc11kh6u5RNKm5DvXDUPCdIrrVlIKyWOruS5n6s8P3GmuQ0ZRSdS//Z5Ac4uSOqAqCzBkoBqJcSBwkGhYz5saqUf6i5FW3WSSCGZy3ZnHzbk/6+SSmCs5XnPdWCOmZ4rrzlLnYTAhb1qZC9e5E3mN3UJIh9tmZeXxrs96+v/Ze7dYW7b0vuv/jVFVc85125dzc/fpNraxm5CAYymWiQQPxiBEiEW4WBEXRZawaF6QIvGADS8JipCIBDJIQYGOYsV5AIMsIkdKhIgMlpFAiDaB2Inv7bbTp899X9aaa85ZVWOMj4cxxvd9Nffa59b77L17n/GXjs5cc9Vl1KhRX+1V41f/v+aYOpPZzM5cu7EcU+mXWg9CcOCC6az6gMqHjKmTDO4we8DWulrrJ90vGPJ9oA5TJYLOCJvNhK7T6ej5pPRNIjCq6RstDZxk5lJnyuOKkTal1m3CwrTJCVrDki8eg5NZ/5gcXJ0ZNf27kK11Qc2yspFe/t4fyn+TUh0yU056imytCycmRzsRjk3qgFLnas1MJHXMGmpZM6uF0VZIhiAx+MDCC+7I/Mjmmza9WPogQ6sbxtWjy9QbcMJN+Z7yu4+z3+dBH3TMKT7yFTMtj998lnxTAG6V74Hu9i3wrXOMr98CANz7Qys8+P5cIP/EH/uH+NLJWwAAD4YrhWZmj/fmcwDA165fxtf+ny8BAOJPv4bf+PXy75r37iHtM7FE/Bbuln+r3F009jHXMTOinHOX/x0FAN4D9RhuOPZPrE8yBm46L48bdx9Xx/3yCYdomzFtampqampqampqampqeqZqf5g2NTU1NTU1NTU1NTU1PVM9XZSXCFTyMHnoMZ/n3U+3CPNFwSzXEZuz7Lrz8vk1pmL8MfiI106u8vKpw9u7MwAZtV11efp+5QO6YrpweViBCrqZOmOCsZhpJoM9YonpVsyWFWlbZIDa/Meyjq9ZrEyCt3GfFHVzyGYhyJjQAps0IXsVVbImIFMAkveyr7pudyBQ0PlyQWg7QlwXhOuUkdZ5v93pDDLGFKHgY5EBFz18ZZ0sbbBAdo8wADr6P5CNH6rhhDMmPRGC1rkuiUHG+ckkWY2JSQyCQkwYVvncHrgHFcTORSzOiT2lkpE4qTlIKkZL55s8rmJyuPZD2S4hGkcrOccjKfFwbFhT++KIEpPcQqaM4wEZdatYHZPgril6pKmcz70XNNJNJOgikmLKfoSgd92e0e0ZviKRbPtDG8WkplfxJGmGLmfTKAAIayBsCl5usx2RTWtuVMVTTKc4syzNUQkOi72ZjS8Ijw8gfr5tRQQU5IhXA6aLfJ0tat0mYnVeat3FNULJgFx3Aa+fPczLpw7v7rOJGwFS6858xKpcM/d3G6l13CuOCybT5ZrXG3s9dZQAjro8zKklQyuSrQcEzZ1OJi96SIrOOgYKOszGrI1nZ+qbIui5PfnzmDQ72s9OMGA/Os3ZjWpew51TI6ANFD89DXLtEQHzvqCyDFDUQGv7Sob0Q+6yxVeLdtflWWuui0fIaekL3yVBa09Xk9yjIpMYW3U+gUqtS8khjfWVD23bo7VOMf+KKScmOGJcnGQULUSPbTHcOTC01pHTcTJqbreL5rI+QmDJXr+sRcbGGfuay2qXNbUOo4MrBnVuIsGRbY6pH/MxASXj3NY6s+l8z66oXH5lBViivGCoUd6GMM/2lZRyzFNU5M7KjFtKLEZH5EmItUW+KaUFWsdinLT8vqmpqanp8Wozpk1NTU1NTU1NTU1NTU3PVO0P06ampqampqampqampqZnqqeK8hKRuPJy36lb4CYjvACwuXXAnbPsQjlFj1dPtwCAwQVs54zGzcnji+cPAACvrray/XvTCe6NOc8vDA4PK53jFb0i4wTJUJQs9WpMRVE/pwhBdSzOSbFglpXQCVCn2Kh5dmmgnNeGgnYZpLi6NtIxHmtU9xlXgDJdgAu1seqGi6T5oSAIAhcHCEJMxOCCDDIgCCBCl4/TukpWA8xASvBa99kjBlPdXRVZpch6fA6CNa/X6r5b3UWB7L4cS/t2uxXCrgzR2emxJTyS6Vn3X/MIM2qdsa3AA64Accbsuij7dn1C7PT81BxGNkaNdXtAPsdxm7cbPWd8EcW1dlaXx3TTOWUI9ggGaCrOm3uCHwuavQO6vR5PPWY3s6C8fuRFViwA+OLYm5gNQmb6noA0aCZsMG0VV1bznMqTccxc5JVicdorxpiIBFezT7sIyNmVQEa8JQPVGbztxWN5iQjU5XHCfYdQ3K/DBoLVr2+NuG1q3Xec5VcVbK0bYye17uXhWrb/3nSKB2O2IT1bO2yvMseYPEOSjdlk60JxTe6BZOqQs5nM9pRU1DEe/Q6ArwjmDHHfjaujWrfAXQ2y+ZjTvqh1NU+410xLEBn3WBaUNaO8tQ2stc5prUs4qnXGEdZFfUOBImnWZKIlwl9F2vbjewLZe0Vph61110e1bipZtPv9gHidUWOanLgPUzKvJwSDVBOAg2lzeQchINe6w5S31XVR3Mhdn/JrLaj3IsWoLfpa9+f3BL8tiLTrkKqDdKKcyQ1kXLw6JR/RqlLrEkDVZfhAUqP7a8A/ptbVDG0/MlzkxbYli5RI3HsBU+s8q7NwgtQ6Mi73gL6X4x0tXMJrvWP7egKzuN9TNK7jgdQV3/xbgUw95A9yJ21qampqWqjNmDY1NTU1NTU1NTU1NTU9Uz39HNOmpqampqampqbPtj7ubPLjTKQWeJPTPMVnbTr1UfbP8ebPn1Q39emT7ge7jyeUpUne57xPAP7ObcTPvQwA+IM/eQv/2o/97wCAH7v1VTxImRT6++Pr+O39a/ity1cBAA9/7Yt49f/I63/tr/yj+P33XskbHiekq0xW8jhqJirfx/fh/5L9BzwhCSEWNdbT5LDC+SeXZer8xx/rNy33JMbdE9TT/8O0ut+ZuVr2AApm6H2CL9jTH37pTexjRpJ2ocdZn/mez28e4t6UnSp/8/JV3NvlgRqixzjlQwrBI82VDaMlLlvPiwl1X7TJkEPJm2vI4o0Fe3VRT3J1/6NIcKVnk0Hd2APsqmujwYSCRVPJIJuAm+v3pp0G02XrEIiMJ+fGGHTPYHUp20vKptK+K/tycDNpO45wWcHbyCC75nPqIK6flBhc1zXXQOoAX5Bt55K4U97aHLDp8oGOscP1lHG3EJ2ch7jr4CrWS0DqFeeSbrHuwfW4AcAzyCWsh1n2PUdvVsJyG8fbSnoeXFA0a+ZOXG8XeKtdl7QdNBNcQXbdpEhbDZEHCrpW9sXm6qSguK6gywsnUR1XgnZHyP5oJGBVDqcDYmm3GwhhU9o0A5SqQ68NqTeYWzjqrwVjp2OyOljCL1FMrh0c2bj14sVUYWQXKJ8DsModclzrxpRPuK1133P+Pi7njOn+5uWreP+6vKqQHMaxOGoHj1TcRike1bSb7kEENQGHuY+6o8uhnp5UxoMxXK2uzi4QUr0sg61LWDj/LrDg6g4d9bryo45rZ/+dZuuNL8h8bVtX+tcb118PxWwTLcZWKvXDTxkJlvFtXlughKWzsASPLx3ctdZBX1tgfWUkdYAzta66jt8+qnVXY74oY3SYysqRO7i9l+NPfTkeZ+rQI/eD8rljOFPrOp8whjI2jq8zc35lU0e1rh7/HElqLpzWnkdqXb2/BYI7aK2rr7lYx93uoLUumVrnwtFYOH59wLRb7r+T3jdpIvCqYLQdkMrnOBNiqXVzyEhu7Yda4yiyecXGOOyCtNZR/hko9x9nLqBy8TEM7muw3qampqamD1ZDeZuampqampqampqampqeqZ7+jGkouXIhoTuUWdKDA8byhJhJsvp+5/JlWe2l9TXOyyzCb12+ivd3ecZ01QW8fJINRBJIZsLefnguZhJ5h6YN6dGvj2fbHitjDuRnFqMIkD5p5gThAhzR8om/rzMNuj8XjMFFNOYTE8tMmgssT8mZAD8VQ4uZ4eZi9pFMLqBts50sTqSZcgni+uHmZZso6BN2fyDM1eCiY306byZf2ZMYa1AkODEQyQYmABDXCUOfD27dKzhxeVjhXswzQTE5hGLsNG5XwORkm/Wpupt01iXP8plDNVl40hedx9z32JbZwGqCBAAcCVTNMYKZ3TMmLS4s8wnF7IOomFKVfU6k65o22fxb+cxmtmjKswdANj6qua9x0BmfbPJUZjmrCYc9ycZ0xc6mVCMlfyDUZsP6Vz1uzC+2bWZME4v5SG5XkuXrDIH9/XL2jj87MwfM4JDHOMUo59cfILUO0FzS3718WQxyXlpf46TL0z+/ffmKECHrPuCl01LrmDBv8nbeeXiGeG2n1007zAzjh05MG28YMuZAFPLMVh1XqdNMXfZmBn86qnXWTM6aqtWMYzszNjE6U+vsbL0fy7ifGW7SHNP6WJUNOsDHtS4oNSO1bqLF8big7UsHwlyNffplrbOGR1KLgtYZQGtdWif0g9Y6V87tg/0a78V870pMmArhM22HRa0TWmIy5Eow1zbrZ7K1rveYu5trXQrHtc4YLCXdR+1va1oFJqRBj1NIHjMbLv2E5ay8NatzttbtIPmk0WybIhtahx+9N1uqpZ5DZp2VPdAihvs4j1bWtc22dECtY0HrFbGpdYl1fZvTnJL+++Bxta4ZITU1NTV9oD7SH6ZE9HUAV8i3gcDMP0hEdwH8DwC+C8DXAfxpZr7/6TSzqamp6dNXq3VNTU1NTU1NTc9GHwfl/WeZ+QeY+QfLzz8F4BeZ+fsA/GL5uampqenbXa3WNTU1NTU1NTU9ZX0rKO+fAvDD5fPPAvglAD/5oWulwrrEhP4qAzfDZYex5KXtzwZ8ky8AAF+4/RCvnVwCALbzCr/9ILtsnfYTXiuZfyE5jDEfxvU0YJwt0qbmA9UUiPhmlIYNqmVXBQyWacyPXOBiSKPYU0WJLMaWOsU3OQEw5hWC7xoc1c2s+OVsMLZgEEoC3Jge/Z5ZDEEW7cbRsRXMlkGgathUcKmFuVORmwAai9GIi6Zfl9tnZz8rN5pKdB9vIjaruTRVV970AZuTfNCJSXJM7w8B223JZ9z2Yi6VBjUZcREm2xCCUqUBiCe5neksYHU6yb4jEw6HjHmn4DRjERDW69gExuKHtb+Z1MAILhtl5T4yq5qx4GZIPp9F2vyon13gBapmj1MNOrDse4PmWuyNEsOX7fZb0nPKmvfaHYBuX9C6AysiPiWDMutgoDktzZAstmvySgVpS2mJtB2jvd8e+oS1rhzfHDBc5rE3PPQ4XOWBsj8d8GbKte71Ow/x2ibXtO28wu8+zK8xDD7ildOcX8pMy1oXDCNer2myOaY3N4ud4XqZdCiZ6/94zDtT69xsDXkU3029GnaxOc/W3M0FyBjwM8vYczPQlZqWl9exVMfkYuyxvhZBjs2YXNZ3qXUMuIJQyzVicHY51tHUOh+XaKr0n5p4HXnJCdqfTpe1LpaFNn3AapOZ5Y6SnM/3hxPstnnltO2lLqeBIZHVwdS6oNd86oFgat1wMuNknVlrW+s4OLik9e0YZ5XvF4hz7W/Fi7P5U13B9ot+dhFwYugGqUN+UlTYz7w4B7quMdpKOZd5kaVcjoGdIrUuah07rnX1vtztgG5nal1tx6TjimatexSTXMPEAGodVI+jXN/qqxXJFjdofTP1Er6hvE1NTU0fpI86Y8oA/hci+hUi+nL57jVmfrN8fgvAa0+8dU1NTU1PV63WNTU1NTU1NTU9A33UGdN/hpnfIKJXAfwdIvoN+0tmZqKbnr8C5R93XwaANZ1+S41tampq+pTVal1TU1PT09CTombsdp5lJuPj8lSfZrbq0yCRPuo+StwUOQJtsokfrdfg114CABxeP8P285nouPweAF/KdNCtsz1urTMtOf7qOb76E98PAPiV37gAjxk748T5XPM3AQDfh29q8/AEc0mfhD6t8fmk8lCfM32kP0yZ+Y3y/3eI6G8A+CEAbxPR55j5TSL6HIB3HrPuVwB8BQBu+Ze5ZvuBGX7MndrtWdDCad9hdZGdJ4kYX7/MA3jVBVysMhvkiPF7798FAOyv1uCDyaSsc8BRMVU3OsFz2CkfSVGRxpqtBixdCrsDFvhhxdn8ISNH4mBIQCjOrTbnzlt3VmeQ3Ulddt1knFajOsD6keEPOvAEbyNIniQl4/7HkAw+Nhab1cE296mhphLAne6XjBOtRULdDPjr3LHBG77U4mBHn2UXvXGu7ZMgbR0AX3JMH+w2eC/kf8hPhx6uHEM8eKBk0VIgOQ4/amasxaCJYRArRf1S12HqGNO+IG2JUPk4Gp26Oe5NhuykSFu3Z/S7itou3UYrpgxWXNF+ZneMOOtnQbkDy7n1B833jMkpBhx4gadZp186HmOCojH6fCkhDVAXzpiPDwD6rR5bf53E9dQFbQfNugM3250xbrxBpcf8fMxnPIcOlU+21mkeotvnk93tNLM217q9rP97l7mmDT7irM8nKIHw+6XWHa5WwGgCSKvztb02Dk6uv8VYSI+vdXWs+hELTLti537khTM1eyCO9dpicaL1E8Bz3fByfNdryc+Kf7tFrUvwB3V4lnHG+qoCxbTAT5OrTu7QWhXFqBXkWHNPI2mecKSlm6ypXW4G/K7Uus4g6RaRZ17WyaI4aK2jPiHUTGBiDF1u1YPdBnPJnA2TFxQ27TtA8l3VMdcfAD/qdSuuvEdO5JKxOnSYPGM+5Fs7R6fbPTj4Xam/ptaJIzsy6tpf11rHUkP9BMRqck9aG+298hjrXaC98toKm3toMq8daBatm3mB7rLD4p6i2cmK+FKEtDsOBGeycus47q4Zw3WtddG8DpP0fmrx3Wg62bj1Lix/r4Ik/AAAIABJREFUednWG+VoifM2NTU1NT1WH4ryEtEpEZ3XzwD+BQC/BuBvAvjxstiPA/iFT6uRTU1NTZ+2Wq1rampqampqanp2+igzpq8B+BuUZzc6AP8dM//PRPR/A/gfiegnAPw+gD/9oVsiArryaNc7uDE/Ru13DL/Xv5F3hzwN9c10gc+dZ0OQBMK9fc66fO/BmWSUUp/Q39IpgJodt9+uQPtBvq9Py/MPpTnWAMM2k80sXOTF02GZXZjzk3z7tNg+ga4bTr0+OV48QZ511qpuC8imDE4MlVhmUmtb8spYzFTVJ83ZLMdmzZVt7gjxLO/cetUgEZwYfeT/7INdOW5jIAHndXbC5K/mfMvyfdQZheTUKIUcw5dfJAbu3TvPnycvUw+u1/w/5xLC+9n8yG+18+Ka1WRl0tlTioAz56MadPAqZROUOnsyesnzcyM9OsOHPCm8oHLMeLBGLumg+4u9ml/Jvq35UbAmR1iab5in8HVC2k9JZqOsKZKY1Nwwa5Nn0CHrSL4hOTHVcrOOjX7P8PuSJzwaE5BgPifWgbFAUvjm2VBjDgIiwNvf2Skms8zzoSda66grs1aO4KZ8sP0uodvlDpmgte7N6PHK+VZWr7Xu/Xtn4F0ZTEOCv5jL5lmuk8N2BbpfaAA7kwromE+4cZzbWmdn5SlqVmU1PrIGQ3Wm0wUCDkqKsDEAU0M3HW/OGBt1h6S1LtmZUV7MVsmMlNO5MiZamA7JzOuOEApBs5ikYq0TtdZVsi7BZIKmXC/z/rzpG233IvfTeMGxMwQFsWSXMhPefb/UulFnSf06yjmcHSO9l6dbu51eTGml9y43E7jeY8z5SWTMqEqtk5niyQk55A/uZkO8YyM1c8yo2ckzo/PVdOgxte7I/EiMrQJL3aNg7mmJUakCzFjQIVYUlcywhnCUyIyxpPdZIsRCivhFrUvLWhc0/9vWPTUtMm143KyoGYcfWMWElHlual1TU1PTc6kP/cOUmb8G4I/e8P37AP65T6NRTU1NTU9brdY1NTU1NTU1NT07fZwc06ampqampqampqampqamJ65vJcf044ugKItFWiw+dO2RzvPvvufOPVnka/fv4voqY53OM07uZlaHmZAKtpSiQyjZfv06YFrlw3OGMSJWNIojLTBbQbJIkdCcW1k+z2oOQgVXtWYhFUFiT4iD2a5FggwyJofvFMtiD8k6tRmRTGRMHyB5aGxMfo6NaOr6/RYIBeVNSU1A4BhpXRqy8xlftRswqGg1kLC4XsbYDN62MMSoeJ9D6uuGgMOUmbMYjQuTYVHTrsNYzqfvEvg0o26ROsF53Wxz6paNXuQw1t1OBEanSNvolojeqAYf1ZjGTQY53BssbWbZHzuGk74HhiuDhlVMlVnGOiVGqssbQw9KEKzMRRYcO/WkBJjNqz0+1tK3AMAm39CFJPvrOyDN9fhZ0cd9lOM8NjaSsb3gu49QNP+Y7wVjhOKkWCJxTMvjeeFka525TsT0bOfBpdZ91917SIUJ/IP7d7C7ylin6xj9S5nBZobUOk4OoRh4desZ8yqfCJ+cooWm1lEieQxpax0ANfgJpm1hiebX72T9WM8pIw6KdVqzoJtqHciir5oH6tJybNjxZBHRJV9fX2HQ1x+GK2A+r7UOYu6Wa53ivrUtx7JmOanXZShCr9doj03R+dST1DpyjHHK958QvKK1pNuJ1x1S6UfnGXyWNxo8TK0z+zK4/wIntvfPg0PiTl7RcJPBXc1rD35SEyw/sbyG0u8Y/a7mydpaZ8yjEjBIdvaj98T6PXck68r4N6+/uDnpo/Hk5D62eG0BZVywfq71kZ2aH7kpqSefJ6RRjZRqDq7fJ0Hqc1sNRn5TKbJ51KaGHb+qQOUHPh5QBgmu++KWY9rU1NT0gWozpk1NTU1NTU1NTU1NTU3PVE93xrSpqampqampqemZiYh+BsCPAniHmf+J8t2fB/DvAni3LPYfM/PffjYt/DaVIaTEXe3484ss5+GG4sTW92IASOsV+DxHAvLJCofXsrlf3Dj0hUZ7+VeB/v/MpNBwn9C9m2m5L73995GuCyFpczsNifTM9XFMzZ6XNj/Hevp/mFa3RWaxiPWzOp2mVcLt08xTvnV9jnuXeQBzcuhX1a2VsXtYwnr3HjQa7GsoyMyQJNuv35rfE8DlunEjiVNpd62omgswaFPSfLUFhsQLjDf1msOWf1++BwyyqlgaGbzHYlluNNt1JH1k80qZnHGq1K5lUrSJmWQ73W7pSCkajftjn9Euyck7crTUTEJCrCi0UxdgZ/qCSXNc4wqI63LMjtGVPD/nCHPBFf2DTvoLDojr3MjYM7qrgqQFLPDditz6Sfva4rtp0L7mPqNgTlBWPT43k7ojj8Ypd9Tt9nvNWCTWY7PnkwmIq5JbmGCwMnWwpKTuoc64PftDEjQ7dkuIQVyWj9gGiorZ1f3LZ8G8ST8fIWSC3/ERQpke83nRJsi6Nxbkx+Sbfmje34skgvaB6Q8/6nlPq4Rbpda9eXWOy6tS65gwbGbZ1OEyXye084Kdg4C4Ktdhn0Al77e7poztIqPm9Xp3k14z3c645Bp38FzryvJRx4ibc5tlTFv3XSKt3ebwmcwyBuu16Keb0tKBVa5vU+scgcxAtzXWrldR+G7HcjzsDeY/uUVt8KNBZBmL1xacqXUVzWUH3ZYdxkRIBReOAzSzmYCu0x4Zr/JNpzO1jr3WutQx+uua2Www3UAmT1bbtsyK1r5OQ84Gra7j/gAZD2TOtZuWta4rn7sDw4/VrpiRes3NFYdvD6R1xaixRGhtpHh1bI/66oCbFN+NTosae1psf1mH6Ajz1o8W6+VSOy36TcY5fJk/y8t9PAYjX3z/OL5M/hB5zPasnp+/T/4agL8E4K8fff/TzPyfP/3mNDU1NWU1lLepqampqamp6TMiZv5lAPc+dMGmpqamp6z2h2lTU1NTU1NTU9O/T0R/j4h+hojuPOvGNDU1ffb09FHeimIRCb6YOpKQbvQJu0PmoVIiDENxKgyMubgcpsse/royaoqrccfgk8JnBVrwjfLRQXEjrw5/CxfJEigP5P/Hvm4ExpmRwY40dN3QP7EHwkk5tuHIlbKun4AQFBH2h4JerQl9CQF3I6M+O8gugorxpYJBWZdC615Y1wGK62JFizsCu4KnraNiVfAZIYuKndZTtYCSDL6auoz1AchOwuLgqP0d1oS4yR17cjJh6DLfdXl1AioocTyLEJPPIcGv6vIjDvtsbzzfX4lTJTuWIPs4K6ZsEdr5FiO8lA96OJswjx0i5cYyObigPKr0q3GQte67yZM6TDIQVzpuqyNpPW5giem6WcPl/V4xNibFDNmTImaJBRVOvaK4bs6IZ10mO1LWbWGJdJcTFweHqTiUzqck10nXWTdqAhV3Vz8luFAxPigyZ97lWIyFx6C+GVfT62rp0KuLvfD+lBVVJJLPaTC1rmNc73Ot40Toh+JAHR3msYzVhwO6XRn3BHGZPa51FnO3GLmtdeoObXFVU+smRurNtVBPb7muqjsvRz1zcdBaxx2MW6nBUY1jtwtAPBQH4ZUTB1g3JdRB7GLScUa0eP1gUeuqMyyROgcb51nngVjvDeuo9wD2iBuDuCaAzCsW1qm91rfUK8LvZtYa6MvBAggbQiq1brOZ4Iu1+34aQFOtdQmhXrirBFdq3enpiH2pdcnUOgym1q1J6ritddMtRngpn9D+bEKYPeRNLHJ6ro+we0qPHg9786oCSF5PSD3pfdBuw4wTP7G4l/tD0pppXzHpCKivzCTIeEu90/McIHWYGGA2NRoGHXak96+Vw3xWa52T4/GjQycu9IDrq1txEvybGPKKEZs+svjwsSO6tfEVN94jbFiV1NX3+Z4K+MsA/gLyGf0LAP4LAP/OTQsS0ZcBfBkA1jh5Wu1ramr6DOj5LpNNTU1NTU1NTU2fqpj5bWaOzJwA/BUAP/QBy36FmX+QmX+wx+pxizU1NTV9bD39GVOT7WfNClKtbZFwss5uD69fXOLNqwsAwHsPLkBllpQAxDv50bFbxTwbAICY0PX58TAzYeIy87rv9Am8A9Kgy6diBpE6Y9ARHmMm49QgKc8IsDH5ITG+SL2ai+TZJsj6kouXdAYi5wfqDK3kWA4AFeMLMvmWi9k2+xQ8IWfPAXBIMsO2mEXoGJRKv0+EeJEPmlcJfPA6A5p0tqG2FwCStzPUJesPQJrUIMmaYKQVwENeOUaHEXmFFEm+RyIx63DbDmnIK+8SsDnNJ2i8A8RyA/RXTo/H5PyVAy+NAFAMYeZ9D1x1GB7W2WedNXZRZyHcrIZH3piDdIck55M9ZAzHgTCf1g4yM+MjicEWdyTmR9xZ5yld3h/Y7JfFgIqdzt5Qr7NClGiRn2iNSQA1WwKA+Sx/ni5oMdOdfJkJ6VhmzNgTfJ19jmpIZmc8FrLGNXZ2wSzCgM4iJGjuqRnPL6zqOXGa0ZhNssrvGTg7yY5Enzu/wjcvc627/+651DoQZObfDVFn9gCpdQAwpmwGlw7LGfxa6wASQzf2WMxO6QyUzrACUKrAZQLA1rqwrjNdauiVeLlvK0uj2Fqns3MOVJwrOZESKMZgbTGwjDkToA48fmatDT1A+9LO2SOe50akTQKPXmauiSHjcpkvzctrpl7TU+43YDnDGNda61JyEJ8iW+siwZXZU7f1Yn6045trXXd5VOssfVPHAiMTQgDmQwfadhgeaOazLBcAX2td0HuUn40R0oHL7HWZxSzrhhUhbMp+ycyAG1LGUhmp75YmRGL0xtIGP7LcT5OhOFzQGpgzofUeWvdf10nmXzCVDpnO9Zj9wc7KkslvXtY6GW82PxeQukZmhnRpqKTXDzHdSIcQEeCef+M3IvocM79ZfvxXAfzas2xPU1PTZ1MtLqapqampqamp6TMiIvrvAfwwgJeJ6BsA/hyAHyaiH0D+m/zrAP69Z9bApqamz6zaH6ZNTU1NTU1NTZ8RMfO/ecPXf/WpN+RF1Isciyb0yhKJcesSaXZ6Ctw6AwCkO2cYb+UX5Pev9DjcyevsPs+Im0KjbR1O38if7/76AcPvvQMA4O0WaXudP8d4c5/WzNhn1d+W+DL9QTX2KjEkv5YcPjNZtk9AzwzlZSKkgvLGXrPgNi/t8cWLhwByjul7b2W8DcGBTzPf060DYjFsSftOTHQAYOwzb+UvJtC6YKroFhF0jzONEbOOeIR0WaysxrASgaAoEneKr8W1Yk9L4w7jmZCM+UQE5lFxrX6XFxq2SRAoJqf5cmTwO2ucc2zQUD0fIqupkUWPIsE/LIZSJwlpYCSbk2dQ0Yr1LY7HoKb2dwmKvcUeQDm362FG5/Mx7LYruG1eob9yas7UA6n0aaAB19e9bNTv9Rg0I0+z+VIHKXjhVsTJK7mwna4nvMsXmAun7EY1BGG2OCEEEwtBfyB2oFTwNoPmxnU+17Lvgmgu+tiMo26nyJxTCjNnSQr6nQ1lgGK6ZExZKgJXx6cdxxYVregaO0V55zMssETJDOyAJDnASUxGyKDBedxatr202zoZpcfcINjUbPtrg0TbTb84Ir1xeQfuSzbvoNmYJy/tpNZ9c3uB+2+XWhcJfHZDrdt1YqIDBsZBax0qKkr+KFC0/N9eq7ysdYpNGiScyo21ruK0vliDoLDRVxgy5n5DV9haFwAvtY7R5dz0XOtq/upxrZN8S9I24IaxhqPabWn3QPCXpdZtEuLA8uqBzT8GG1Mxc3dkf1TrxPyI9HWGDota56r5EVbwtdZdmlrXARTzhoLTWkeR4A7GmKlisxGLdWO5x8y3Izav5I48XU94n84xl0Z1e33FwrGir9a0iRIvjaCSGh4lU+vCRu8B0b5WaG6udSz5PeAmrXWS0x30dRh2EPO4nANej02RW0p5SFupUZPWYlBBeJFrnZgiDQYv7oDuULeiJ9NFVqO/Y4OoWtOyC1PZEGCzS9l8XrxWUl95gdPdtX+bNjU1NX2gmvlRU1NTU1NTU1NTU1NT0zNV+8O0qampqampqampqamp6ZnqKaO8BC5uoOgc0iqzLvOpIpinw4y3d5lRf/vtW5oreTbDFWe7sO/g7xWMbSrOg8iOsdXdN2AA+pJTdow9CiZv8tvczc5/7BRnSp12WIpmIWRkqKJOyWc3QKAgT+L4p58pKZblDH7mR5bPYUWCsXW7JJlylNg4BdPCKVhxykfRy/q5Hn/aJLCv+X8OlKBOlUcoc0Wt08CSpRijU0Q4khyzi2wQMAaV83C6mvBgl/kz3nUopBumO1H7e5WA0qbNrQM2q8zpPrw6AY/rsoJpp8kaZAeEk3I8ZzPON9nSOCYTrArkDMiKDT4GR17g0k6dN9krKpvPuS7ji4NydzDYY8TiVNR1pxMsHJC7mmN7UP4w9TpuyaDFnnOOqd2uOP967ZvUqWtwOGXNr+10IHKnGD2Tk/Hmx6TXKi/Hg4wxcjq2vHFJjbxA17g6W3qz7uJaeyFZXsku5d4jbvJJmc4JsYzRky7im9uM7777zoUi0mczqNa66x7dvbyuHyHXmx3rEYMMFIoQXNC6gLPjhbu47X+LUCqWSlLrYlo+v8wO5IpQ1nGfeoNZHjuQGzdYdWhVh964InDpr36XEE1+c+qrXasOFRcACetcZE9i+bkinbbWjTfUujosHWt9XOn9AUnvMyFSvk5LOyqOGjcM15cM5tWEB9da66rGl0yt61kQ7NPbe6z63EkPHp6CR2Vlq8N3dNAsUQfMp1rrzqTWUX71qja7Y7hyQBz1/FA0DswWl7bjxKCycSAEE1dZkVg3aa1zBmPN7tOlv060rlKCZnYf1H23vr4AFPfhei2EjOJK+aaMT+djM07JHgil1s3n2gFugowP6+LLRKgUsB9ZtplzWa0Db/meWOubceVlZslj58iKHRsMnjrWn/sXtNY1NTU1PSG1GdOmpqampqampqampqamZ6r2h2lTU1NTU1NTU1NTU1PTM9VHRnmJyAP4KoA3mPlHiei7AfwcgJcA/AqAP8PM0wdvBECfd8m9V0SLgHiS2Zhx7iTM+u7LV7LqdrfG9DDjTW7rBQFKPRA2JZjdA3xSbWwhoeMgqBseQTghNq6ngHGqNLhrxjjz5ziou2R2g1VcJ3VAKjjSfM6IhTp189G+KwWZSLBjioq3zWck2NvqIUDXXL53EojuwhGWV1A6CgQq+CWTYsCUls6bFb3zkxMckB3gZwIJXqxocxoYcV2WWxlWzqKCXrEqkCKBAOC73AHX44Ddtlp46jmvbQcAf+kF+xpXPb545wEA4HPnV/gt9woAILx9Ar+v7dQ+tW0gBq72eV/7qzX8+z36rXEcrR+N0+XCUXHhXGrOsyfB6cJGEfTUAXMxVXVBxwx3LC6scHqcYHPMByAWDNxNtHDsFUfOALA49xLIYHmAHnfqrdNlRniBZV+DFEvMh/aoOyeYZIxRNG6TpP2SiARvY9K2WkdfeONsmVjwS3uN3ejk+gz1pGodd3kgc+8FFQQBqVxLU/DiUv3Kq5eIpe+vrteYH+QC4q+8YI1xBaTe1LpNlG1ivqETaTlGlAunJZptlq/Ielxh4dQqLrkoSKTUOiBs6usA5pUJp9gxmdpoa910QTIOh0tGf13X9VJbXTCu2QT4UkzZEai6OjtauqvXscoEV12wZ611dfzLdR+071LPSOV4eDCuxOZeYdF+gE3dA1zBhbeHFfbXK/lerj/Or00AgN85pPJ6xmHd4wu3c6177WyL3/G51s3vbNDtzXGZe0l9pYIT4WqXx8thO8C/N2itYyhSbfreBYM9s2L4dlwkr4ht2EDuadwx5lt1IcV0uWP97NUJ3mLQfgTcida66qhu64frAS73KDeXNmlZ0lcVBvtaBTCflVp3mgya7Rb1tG7IBc73uPKVuLzbfgHEhd+Zz8R6byTjyktOkV0KR9uQMfycFbumpqam50wf5x3TPwvg1wGUf37jLwL4aWb+OSL6bwD8BIC//ITb19TU1PS01WpdU1NT06ehj/LHuY0e+7hZlc8y2/LbTTdkcZL3oBJF5i7OgfJ5+s6Xcf8P5adT8Ufv4z/5w38TAPBHhnfwfplleTee4x8cXgcA/OrV6/iDq7sAgLfeeAmrf5ifct39B4w7X30XAMBvvgOeytOpGBGiPN3GIvfzpnY6emQ5iTjj9OmMARMNtZBtw2KCoy5n2vMExyd13ad/zM9AH+kPUyL6AoA/CeA/BfAfUH5U/SMA/q2yyM8C+PP40H+sEWx24cJ4qJzvV86v8fJmCwDYzit8/b08sKerATSXfLWThHS7GBt1SZ5SIwFdmZ0jx5i2Q9k+FiYgdaqGyXzfLc0wXPlFsmYyAyEmNb0Ia7eYNa1yoz6Nn0/1CTyMB48LLE+RaYY8ve12kGy/vH59ugx488SejQGEZMTNDhzrDKjOeOXflWOLmqNIIefq5WPLuX5iTDFBZ0xnEqMMPtGn0Xkj5f/HBjl1gsDMnAKAK+cnEjTb74oW5ii1v6dpjd9OrwIA7tzZak1YJXAZC8ykT6cp/w4Abl3scfc0d+S9LuLB5JAGL/twE0m/JJlSMGMgkpoFOScGU6lT45fUm9kCl/sQANLKnudstgIA/ZayGQfKTK/pLzEOYZ1dqj8DeRzWdXPu4lFmnswe6VepB+Kqzv6YGa/kEOtsUNIxQsHkRCaSAutmwEHHlYw3mAw/Mm1YmIeYmQYiyT5l4JHMwOdBT7TW6Ub1evUkJjyvXmzx2kmmQh6MG6l189VKZvHieURc5RPhuqSZsMe1rtAkyZt7pzOzP7beWrqBzURqUtIhDrSYRQtrncVk0hkjNwJ9NQU6YxlXeV9qyCS1Lmgmcre/udZVoqNuR4zHnM7iZlM6J59td8vsVzIERTS1ruecp7p/dH9xpX3A7iPUOvNZzIQAOGIhRQKA7srUuloDjHHSNG2k1t2+dS3mV7xKSKXWOSYlhXw2rwOA27d3WuuGEzwIWuv8aPZHtBgDclhJ+49J7w+x15nxXOtY1q3LLLZj7h/dtTMz0st7g3ympQFT1eLfo6XWyQwloHUmAqjnqstmVQCyqVSlopIeAyWtXXFFi/pZ6SJHrBnXXmc+E5QEYegMM5uGc4LM4jukiqNkc6TnsNY1NTU1PY/6qO+Y/pcA/kPoP6VfAvCAmeufBN8A8PoTbltTU1PT01ardU1NTU1NTU1Nz0Af+ocpEf0ogHeY+Vc+yQ6I6MtE9FUi+uqU9p9kE01NTU2fulqta2pqampqamp6dvooKO8/DeBfJqJ/CcAa+b2r/wrAbSLqykzCFwC8cdPKzPwVAF8BgFvDaywYjneI6/x38XwG0CZPSHzh7AHe3p8DAH73G68ADwurMzD4JC/TrSJSQbrSvkPaFR7IMeq0hrs7gTo1nIBB2kTEgvAkb/FYzbKjpHlscQ1hNF3IiGNYF4xtQzk/DQXFlFxSQrd7FONxweCeUZEhYs1jS4HQ7ZRhqply4GWupiCnpEhSNuswmGZF94P2RTxhYdL7rYObFPn1o8GYQsaYASBcQHJG2Zsc2I7A1vDBIK6uGLw4l5Cm/Ivu0gtKB6fGGtOGBRmLFxG+fH7w4BRpW7Jrt14NQQyKmhwD5Zyvh1m+n4IHEi0zDiteGmiRJ2vPuy5s+1VRW4t/A8Dqnnuk7ygucbdY/FDmUzUvST6PE0Dzb6tq1uvCEKnmLlpTE/O5Hg9IcT93EjS7FUAqKDNFSFZfGiAGPBSBVPbdAcvrpmLuPRYY9eIiq0M1KQKHxEiVRV0gks8N5vYp1TpCOCmZzeeAO80n6LWTK7y9y7Xu9994Gbgsg2lg8GnuWD8kpIL5p+se7lDNzYBQcznvTED3we+WsOOSX5vPm+Q2dowkTm8s9Syu9dqqY6r+bj4lMfpKvWKW/qC1jskgwsagjJKpRQmSjekiwe+11lkjNal7BPBYazTJU1X2pEZafFTryvUWT1hqckXqq/FOzoet2DEh1prOkH7Nta4cWwdwvcaYFuY/XRdLsxnR1DrpF28MyQaD/59FOZ7798/A2zwWuq2Hq0ZAxpCOHEtO98lKfbjGuQPikblVXacaCQFL87EjXPkmkyc2maHEwOr9ghfPBp029RMOCKWmxzNoVrKpdc68LgIATgyFjtp+VLdtrav1h13OqgXy9cW1pjEhzuUEJT3oJV5McGWRDsvXIZx0Hi2ySRevIcg9Q5dJ5ICa3Wrf+3oxXgFrampq+tT0oTOmzPwfMfMXmPm7APwbAP5XZv63AfxvAH6sLPbjAH7hU2tlU1NT06esVuuampqampqamp6dvpUc059ENgf5HeT3sP7qk2lSU1NT03OlVuuampqampqamj5lfZy4GDDzLwH4pfL5awB+6GPtjQjcl2y/zi1w1M1Z5pXe3p/j995+KS//sBfckdYRrqBL4bpHd6+gThPgJ8XHqlvgjEEcWgGDHPZQVsgZbIdwhCuWYyZFtVIHkLhW5v15wUA1ty31SbAiOAj6usB4Ei1y3miuiDChu86f+63BiAfNTU09LfHQSdeFRSUrYsQGLU1AVxBankjySVPH6HYkKGl34AXKW/E2NzqkkhXLBEWjouk/05cZvcr7eHh1Arrfy/I1VzCt0o19709nfP6lhwCAVRfw9TourjaK8KYlElv18HqDd6eMSeLNNdZXTpC4Bapl0Fc3aT/5kRVrngyy7NXlMa5ZnSA7RriobsfqvIqOQcURl3wSnDZjibnBNLkbc0z9qPgxR4tv8xJ3s/3NxjHSEVCugWEVBH+fEyGV6zCfN8UYRUziRAwQjMEzfHXoZVLUTRpyJILg4kyKXyLpMTyPdNu3XusALpnNqXNaQ3pgfZLRy7euL/CNd+/kX1x1i1pHBX+PVz36++qwSgbTro7LMwZBTsmgrBhYMH9b65hw8yNJwqLWSTZuGY/qnG1q3YlmdLJnw2NCHZ/ZIPcR4jhsa123A2CC6F4LAAAgAElEQVTcrqtTLsWMkQIF/a11cnZ6bMeHUd2ko9a6NJNcq1LryusA3YHNOizX4jw6JK9ZsXLNJe1LSgb/N9fPw0utdUjAfJ77KJlsVCSS89afT1LrvEv4fal13mQLY+mqW/Z37+oU81Q66c1VrnU3pOzaWmdzkG2tq1nZdR/1FYuw0SxreMZ8oTXD1rp6PK6P4KiDjEt2K00OoeaYzibfdK+1LnUAdZBl6PhVnA+rdWutdVNwSEPZdyLBamMkeYUBYEWzQXJPXyDEZDBok6sqbSoNkkxTgtRP69z7vGU2NzU1NT1v+lh/mDY1NTU1NTU1NTV9In3crMWPury8fOz0AUCKj138hdZR7qfb5CdM7vYtzN/5CgDg7T9+ijt/4psAgH/99b+Lf2TI2aIDRfzq4QsAgN/YEvYxP2D7zXsb3L+ft/vy/3QH/+2f/ecBAOnd9/Xh8zSBaxYprrBCjkP7En990bwPPSv8AUuU39lYqaemxfviH2Mcf9L1PmyzIXz4Qt+Gerp/mFKeKQWA1DuZ4Uod0JWn39tpEAOJ+PKIrs+fp7FDvCrmN1d+8UTY5o/WmQC/cxIjBmhOaFw8fWU1aSFj3GAnjpwx8vHWGIIQTjSqLHWQJ8puIvTFyCSueTErW5/yU9CZMRd0JsSPWOSoVbMcP6phiQOreUk6GuQ3GFq4wDLb6ieqD9rhR4gpR+oAJH3K66Y8e5C7iSRv0O8JXAJVKdLCyERMd5KZuHXAdMjnjScnEyppUNOU/tLJk/Nur+dhvN7grfIY/fu/8Ab+sc+/DQD4DbyGw4M8belGpzmCK4bf5AathxmxPLGfe0ZYM1wd7axZiglLE5A6O5PzBfO+Q9IZ6vmEEDb5czjlPNtb+73kTcIzXGl3Ck4yV/Gw1/zUoCYg1iiGHS8MldiMT83izbOPcu7NmKZkTD0YYgDWdVENQaJDWBVyITj93mTiUgBQzF7yLAM9sn1SP5ly7emgow+5aSyMqD540W9TEeo0M3dOrlf20q04hA7DKl804eUkJmHz2IEv8/jurjQPkh0Dnc6eVflrJ7OBi9lJNgua+maJAXb2e7q51hHyLFe9zgZT62agfzuvFNes6xsTHRch9c0FzR3uDjq7mbqc/wsAfr8kWaTuHd/QzXVbx5OftdbFWU2R+hHA9XKc11rnJzZjWmtdtyPMUusgx29rHSXWbFXPWutGL+c5Dbr9/qFbGJ3V4xyvT/BmuX7+yc9/E/3ncsf8Dl7FWAwA3eSUYukBVwwDhz4glqIeBkbcsLknao1mM2vuJu2/1JP8Y4mSZp3OpyTmVPGENQv5qNbJaQgOmGqt64RkcrPmRtsZz+RZZ+6PZyDtWD3+N6qpdTb/mcrMra11cR0QS440BydGYs6aOUVrnmVrHeusKhkECYybzKWsaFGT+UUtck1NTU1PXN/KO6ZNTU1NTU1NTU1NTU1NTd+y2h+mTU1NTU1NTU1NTU1NTc9Uz+wdU+5IM0BPGH1hsW6tNMjxar/G/jojbWnbw+317+i4qaYzAHvNKxUDkUhisgGoKZIYNUhDyv8dFviQoKjemIB0kLwz7rJpROorMqR4W9okzGU/tHeCMZExPKKk22XHcOWdgHiiGardXg06mAAuqBsHUjQokuDBsSc1X7CIL6uZSLcz2a2dolD9VcbVKiLtAms+XWLJDe32JJmmFI+QTcHhWN5xYCqIV+3jIn8gdFvFd8V8JLDkybIjjG9kbvbv0efxHXfy+wqbzYTtPg9dZpJ3DbhjrDf5AG5tDoJzPbzoEX2n2Z2BgEPB+BKZdutwyOdd+7IaD6XBZPIp7QsKBF/x4qCosBsVM3STOedeEcq0YTG/okDwBo8WxDBCsxZTbpPF2HwxLYkggxlCDGiIWEyonE/SKKbHmA85fRXCYsTHInPOFwhxXVl3lTdoxuUjGPqLJptjamrdUMJpb6/36MrnB7sNdttygV/1cIfS4U5NwtibWld+ByAj+LOikrXW4THmQAsjJPOqQiJ9LSLvq34m+JEL3r6sdXGT1PBm53Ucm7xSmFoHU+vGjSKObiSpMezU9M0azbAx/4kDLTFO0y1dubZTj2WtK/Wsv6qvN5R9B8V6szlc/t4fSNtxlA1ax/eC8ATA8p6EFpNuR+ivHq11FLXWgQgHn7nZX6PP4dVbWwDAejPhel+we9b6zkPCyWk+gLunOzx0+R2y++c9guvgCi5Ms+a12sxnwNSfTn+IrPefx9a6yaG756Xv6n3WG/M4F5b3zXre0kqNs2TMota38tlkjNYaKL8zhkc5llTHSS05i1rnEqLJ+b7R4NDc749NEJnM12lZf3UhPPI9xeW198LXuqampqYnpDZj2tTU1NTU1NTU1NTU1PRM1f4wbWpqampqampqampqanqmeuoor7jydpoHmdaM01XmjW6v9njj4S0AwPU7p3C7vLyDYrppkzQb1LHkuVkukU8iuLjxuckv3CYresaeZZvsSXEzVsdTppwhCmQMKRX8KKaMiVX01Y8QfIp7zXajuxOookRMghiF4BRxDQQqzq00kmTvAZojF1cWidT+9CMQVrp8PU5nXKSZFCVyE2G4zJ/nMz1mYqDbG/feWXEkTuqq6CYs8y6NqqMtIonjss3JxMFjuMzHWdFhYIkTzmck25nPWXL/4oM13gh5obTr0JccW5oVGYsbxjznZd7bnmJ7P6Nx/n4HvydFqg2C7CfFnP2obpbW9ZYiwHUMGPdQ7hNQEfGO1Txy7yRjki8YXD8PCVSyeF2fkMo550TiZkkzwR3KuJ2Bzrrhiqtu5gcFJzSulccOuvI9gKG4XafkMK+L5XpwBoWG5P8t8UjFxfO4KmMhKq7oAgk6SgmS3UdgdfyMvBzDFvF90UTLWifX8UnCrU1+XeGsH/GNB7cB5Frnr/U5obx6cKKoLOjmWod1QqqZuMHUQA+g1j3HUpNsraNE4HqdO3UBj2tevgrRkcm61HqQTK3ju5M4o9ozGiOpM7WtdRPBH0ytq/sejl4RKOr2JMuAFJenuHRJlWt71JbMp+o4XHHdivzmXNZybGBBTN18hL+S/r+i/RQ1P3mx7MFheKi1TpyVzashcTBu32dae8b7a7wxlgzca82xpaA1NhA9vtbtSJx/j2tdzXJ2YekWb3HU+qpG6iH3aB4SUGoXd4xQBpY/EFCw7qlncB23K6111CW513EkoJ7/QPJ6jp8g6LutdblmkDoyH7nhkml3lSdGV3OAe4dgal115U2duf+QwdbJ3LsDCQbNAabWGX7bvrZgX1Uw3xPzst1NTU1NTY9VyzFtampqampqamr69pU8IPgI2aU25/NpPRi1Oav26/IePCcG+fqE5MOfZGheKPIx3LT9sh0e8xOp+M57cO+8BwD4/N91wFfy/v62+07Q6nvzOt4D56d5vZOVbOrVBHzHe28AANKDhwiHgx6X7UPbt1XOPx+ZstUkBqXfa38n1r467tcnoeM++jTkvIylvE8zDsz3MsZS0uNnBnXVt8X4gBAt1pVtOtK+dKQP5OZJPnMIwPUnPJRPtlpTU1NTU1NTU1NTU1NT05PR050xNS56qScNne8TzobMzHzj6ja2b54BAPy1FzwwbhSbBDGoOBW6keAN+lpxqulOFKyI2LjN9uo8CWAZCF8fOCV1F4TBfFLH4IJw8QzMpwBtyvodEC70SYu/LNjp3i3wyCQB5+oq6SZ1KPSjomFpUKTPHwyiZ8/aBHm8kLwJCi/7q8dA2nWKc80saBwTgRiC8loUOP+s7ai/Y6/7sH2WHWNLP3Us6FZ/36O7rvsD4olux/Z3VeoYdCefuPVmRox5O/M4iKMvRXMOO8Z8yJ0zXQ0Y3szMXH+dMcSK04GXyLMvSF93UMdidoq6UbKukhBMFxZXc4zV3bwDi2w7xwbf9ogFaUvJgQXfdXquHJBWZZvRLRB0HYcF6yNdB3bsGiff6hLKgLRj6AKmVd7YPDukWMbqvLTotYiw9MtsHCaJzPlXTHcxLlixXgBA1GvyhQ6dzx2eP3Yk6CcPCSd9HtNvbG9hV2pdd+0EIXyk1h1Krds7+IN8LZpukyC71t2U+/xf/oW5ThxLHUbC4nqW1yU6uzwwn+u4Sj0QLvSJvr8sT1r3Xh/0k14zbtKasfx8dF3ZWmccY2VcWQfhTvlTIuvQa572GoddN7E47ObltKa5wGbck77OsCepS2we4VJcuvhW7JQ9AwWpHu57dVT3wFRQbrjj66oeP4Pu5sK8WgeEguni4LTWJXv8DtMun9zp4Qqrt/I56K5p4Y5r9+GC4sv1GIFy37CTLOWeZt2XsUpwQ264c4z+7l77w9S6KlvrYnDyWg0FB66O4I6l1rnotC86bb/z5bzf8AjduvW6AMHcIxNWxe161QfMQ+6beeWQCv6dDKZNj7hU62sr9iaqDuQwgzL/7lHx4jzLFMCRW29TU1NT01JtxrSpqampqampqampqanpmerpzpgSIZWZmrB2CBljB5/o492H+7V8jqcRqLNHnmX2h649hgflCWwkkwunT5S7nUM4zesmDziZXrJTQiymLuwNo28WyWYVZRZhYMRixkMzwc9ALDMS8xnD38ozZhfnO1ys89Pvy8MKhykvlBIhRZ0xQ3mqOweHdDCnojyNpYOHL+ZPdgbPmlWwJ3nCa40uKOgTXhfUHCQOOhtBxryGOJtv1FkFdsuZsfqA2E8sxidxzTJ7GwedlWanJiXxNEmWJhwwn9VlWJ5Y+wPQ7XT7kk0XHPar7L6xutjjtbs5x/Te2QnePbvIy2w7yU5MtwJOznK/h+AxXxcDkWGZt+dm7T8/Q2fu7awI68w1CAhldne6lcDnebye3tZZg3n2mKcycwQgTSV7cHJi9uK3ugPHkLzSnCNY+sVDz78xCKKE5bh0Oou5mD5bDm/5OSXdt3eMvs87j+uohiBmxpSdmsssZ6ogU7V51sUYLdlXEersGpkZoqRtdpEhblE3vZPy7S4ixFrrNlrraBMRy0C72q+l/oSzo1pXjX2uO/SXddzqbCNFNabxeyczW+yPTF3s2JBaZ2qAmWFkZ3J2e5bPbs6zcNUEbj5juDv5OrtzscPZSmvdfiy504kkRzhFU+tmf3OtG72aPzljQuZ0LLEjzRaOOuPlQqY/alvrscXeHGfiRZZk6tU0zjvCTbmUfmJ4U+tqm9KghIs1x4snSUxu2Od+yo0yRMze5KSOLLXHBYfdOnf4ye1rvPJSRkvevzjFexel1l1rTmy6CNicl1o3e0yFIIqrYk5Vx4/JBHWzzgamg35vZ7dTR1Kjp9sJKPe0s1vLWjdN4hKHWMgPjB5U+qvbab3pzMxh8pBzwp4V+zjObLang8w6zvzOTj4y5D7DTEi1RjlGP+SLJkaHVImV2dTiADH0s7Uu94n+u8EaLVG9R7H2HSV9HYsdyfXmZps9/ALWuqampqYnqDZj2tTU1NTU1NTU1NTU1PRM1f4wbWpqampqampqampqanqmesoor+a/pQ4IxWSh38wYQzFvcAn+dmZCuy4KDjbterirvIzfkzErYFBBb/xBv6cAyYNcKBGoIqdEglhRNOhrZDHTiJ0xG3KQP+XjmuFnQlfMSLgnHIpJSTh16Aun+aWX3sXGZxxqFjY0K5TGHkKP9/aZ9Xuw22B3lTlYnhziuobQ2WNQ9ChYYwhW5NYiXP5Ayxy9iqT1ahrSXQMIipbB5PmVH3MXzAbZNSY37BUtRFIkkGYCF/w5bFgGnD8Q+kzmot/ywtQl1dxGR+Bv5gN/iFtYf08+iO+5/T7WXcaz3n5wLoZHm9MJt08zcjZHj3uv5I4J2x40OkV5R+0PYkXBU6fZoAwYoy7Nk02nEauCC9862QsytqUVtg/ygKZdJ0ibzfdMPUtf8JAU9VpHyaFEJDH2AgNUzWg8ASZnF1A8mxJlNBYG760HUQ1BosNccmCJWIxKnGOkmj1pxjcDYmRDznjKeL0eOCqJ9zgjIybKWaZYLsfGYvyFtAMhgPti/tKT1rpVwFzMpoYuYLqdx5LzSUzS5utBa92B1NxsYEEx/QGLa/ImHJoiwRXTGe4VwaewfP1BUFmDdObc04qxAu5acfvUE8Z9bl84dVj5fC1+390rDMeuaUVTKTqP1LptqXWz1rpqiiYyucnWuM6ivPXa7vZq5pRMTUqdYtDdDrnWdabWVfTVmHi5oEZxyStCmh5X6wKJ4V44SfAlo7PbEYaHeZl+y5KbXF+fKD+BXf7hHt3G5rtyY7/79vsYSq179/45QnlFYHM64e5ZPiFj6HC/oPrhugMdnMli1axtmFpHncGXSetGrnWl5pxGrE/zyhebg9S6y7TGeJnPG+284M4UjWlVrxnUPGgWrxsiUjV2mkmMvShpfjV3mj4gta7mmiaogZB9dcCgvCF4jNWQydQ6ciz552wM4+x9DF5rHZt7Pwe9H4Agg8H6IB1LjfToBS1yTU1NTU9eLce0qampqampqanp6YoINm+RHvcOrsmb5DDDrJD/5/XJPceoT1Uflx35tLJLb9rnUc6qjSzlj5P1SSTH7V66i/GPfBEAcPXFAe/+U3k7P/nDfws/sP4DWeX9dIpfuvzHAQA////+MXzxF3L/nf9/byG99Q4AIE0z6P17panL9qSb+u34uxuXOTou8XPh5eePo0+ynulfTgDCzQ9Sn7iYJfeT+g485/2SI3B63BhNem0sBsljlk/xo8Tffls8I2sob1NTU1NTU1PTZ0hE9DNE9A4R/Zr57i4R/R0i+u3y/zvPso1NTU2fPX3ojCkRrQH8MoBVWf7nmfnPEdF3A/g5AC8B+BUAf4aZp8dvCdmVdyh424oEGRqGCF9cG18+u8Z6yE/ErscB24eFgTt4cTxM3bLlkv+3hrhcWlwto0omd26qT+WcuBxmZz7IMoKomnWtMy73wOElFkwq9YBf5w0c9gN+98GrAIDfGyKcCRHtuuK8SSy5nDF4hIKjugedZoD2mvOWj0efJlYsz40ZTa3HKV1tnA1dwCJ2TVC3Xh0l3QxBzPJxk/aZcR4EA16OmRYOiZVUdgw5V/7gEIoDLCVI5mx3rUhw6ggo2aBxpY6+qYc8Ohned3hruAsAuLy7xjgWp+P7K3F33QeHULPzZo+0zct0Vx5uNq68k+KL3QHw+7zvbm+wRl97CwhrxQbhGZtVHp/rLuC0ZFLeWe+xPckY8dVhJdjb0EWcloze26s91l1ed+NnjLHgkOxwf8z84tW4wnv3z/MxbDukdXGR7N0iC9JPBF/PXdBzsngcRhDWNiVCrNwtq2OqLIfiSFm/cnbcGxSNl66Vi/w/Z5+CmiZY1BwGpeucLvQc6EnXOnHlXRPiJh/3ZjXLGFidBmxKrbvcrxUFP3jp7+oIXlXPSdzoObHZtfmL8r9o3EYTPrTW5azTakmr5zk6zjhvOeK4Btwqb+B6t8Jv3c9oruuTZFqSS4taFwpGHoNHrBnUV528SpB6CO4J47gLaLv9uHTDld9HaCZ0gNQeMpmhPJhaZ6+XetxOf5BtJ80+rqhvlXVilVdAJkL06g7c7fI6/RaLVynqawFx5aRN3Ol2hnc93vAvAQDevTjDPJab3cNecNV9cHi7OOPGyQPFgbzbupyLbRzI5ZWOPdCVWufHZa2r7s1hQ9r3XZJ78Uk/47TTWvfwVGtd7ZlVH6TW3VntsC6I98oHeY0lsMP9Q651D8e11Lpw1SOtinv5gUClv/xE8CPJqx5uNq+YHLmUy2fWX0VT6wgwN0WzDkHYYWJSB3IGqHS9O6p1WjS1Efk+WffF8qvUu6Xr8/OjvwbgLwH46+a7nwLwi8z8nxHRT5Wff/IZtK2pqekzqo8yYzoC+BFm/qMAfgDAv0hEfxzAXwTw08z8vQDuA/iJT6+ZTU1NTZ+6Wq1ramr6TIiZfxnAvaOv/xSAny2ffxbAv/JUG9XU1PSZ14f+YcpZ2/JjX/5jAD8C4OfL962ANTU1fVur1bqmpqbPuF5j5jfL57cAvHbTQkT0ZSL6KhF9dcZ40yJNTU1Nn0gfyfyIiDwywva9AP5rAL8L4AEz1zeHvwHg9Q/bDhMhnOS/hedTQhoySzR0AedDLm770GNXQtq3D07Au/LCcFTHQx4Y3BdMbB3hy2dmEpQsjB60LXjswYmL4iLU22CJeWP5f8lrz7Anxek6dRqksMSy+DTgiy8/AABsuhnXc97hOw/OECbt5liC4MkzvFf+jsrndDvIZzABxVWTgjMugiyNJf24OAZgiffJITvFyro9FkH2cSBBk/xkkE2zD2LF4+x2CYoIW7yNPYsz6PDQSbg8CBLknlaK68UTFpfLNDDSJsrxUwly3719Cn+dG7e+VPRuHh1COefdjrDakhwL2LQvqttiDqMvnyNL31j3T9cZx+Yh4mSVcbXTfsJFnxmzjZ/xyu0rXecGq8Z97LEvA3EfexwKynt/PMFUnFpjcvAlkT56L2M+rRlcMUaXDQPqcbvZIGQWQzdjmxMJOg5gifJKo1kC4DkZbwGn2wRr3zhPgn6mTscOoG7PzKxh9E5JOoqmfc8Jygt8OrUunBBiqXWbYcbtVb4ItvMKu22tdRvQrnRshNY6h+xqCsCtA3zBY1PSzgz7DlSuB39w4HL9sOlv4KjWOf0uyXghcMFpuU9Ig7r4cseIJ2XB84AvvKK1bltq3bsPzhAqdkoOMajzs/OK9VJfa90MquhwInBxaHWTU8zSs9SGekz5AxavEUhNYnNsZGod9JpPvtS6im8mErdWK0paG3BUZ6nWEl7WWb/LO+8vtdaxB6aL8rkjQY3jmuW+FNcs55kCgYqj/HzYwF+XbV4pZhoOPVKfD6i/Vmy4orvWHLmu44J+72deuBcLptpBDtQPCael1p33B9wZ8gFt/Izbd3Z6zKYDYtnQ/8/eu8Xqlp3ZQeObc93+y/735dxd5XLZbrvLbhq7idVqk0aEhJCAIiVIqCUEKFIi9QsSQgJExEteeAgXgXhCGDVKgzqQliCkxQOh1UoUhRDa7m5jx/d2uVxVp859X/7bus05Px7mde06p+uU23VOXeZ4Oeus/f9rzTXXXGPtPb8xx+hNgZ1bk7FVFbaj3d6pKjjwGyYI516vCxmdjllAUPLCQXTbT12GraN0sjTADyUtJlxnTDL/HjiN49KDxGPEgCEQ94d34ITrEik8J07EFDmTOLYnXVbzQQIzM9HjW87MXwHwFQBY0ckH8OoyMjLer3gq8yNm1sz8RQAvAvhFAK887QkmM2tq9xM2MyMjI+O9R+a6jIyMjzDuEdEtAHD/3n/O7cnIyPiI4V3FxTDzORH9fQBfBnBERIWrJLwI4PYTvhNn1pYvsKpd1tjcVoEABAMQALhzscLu1JmAtDJMUrPkmIU2U6hqO90rBGN0FUk2cXZU1jqavXQizljqOPMZjgvEjDJgWsFJg8oKhnFZezQSik2c2Ve1QOdmgq/PN/j06iEA4OFyAeUz5pKSxbLoUYhYelTOHGKrapx39vrvnh1gHOxMs5kZoPZOIeT9fmw+X2Lk4ZtOKhokpVVSXcXcPaGiaYjdgUlGZcgPJIR8T3BqCJJUXNMsuMQsiWU0PBJ9NEjSDYIhDBgoEKszAnFmWi7tfa7qMeQ8Dm0JPyE+CBEMQcw8ztyzpFA50gJ29tsXc1QyE04I37cT/4nhz2NAklE6o655MUC4asFG1dipKnxucBd6McywH21DTjcLaGfOpEYJHv00/aWcO5/P2CbZqyNiXqCrfPj7LpQ1BQnX4G9bYphljAhVUq1FUBYYQ+F8iJdvo/qeZD3+ODf49BBp7imSbFgRrw2SJ5X39xt++lxnO7MpVOCDN86PsH5kjYNCdi1gjYdcVVHMFZrGPnBEjMGZfrEB2D1kcqZCtCM68VgjJHuwx3AdppXHcB8TrhOdQLEjmNJdTy1Chf9jywt8ZvUAAHB6MA/j3jAF1cC8GFAmjWq1vYatqnHaWiOc+2cHGJ0iwiw0yFUP2VAYMmNBsVKZKAOEQshcTStnqaEQ6VjltCZPCQ+mXCdijimZ6TMWVBQU+4xpWokOXDfG46tZVIEQu2xaOIMi/xzWAB3Y+1zVKmbadgWUq4z6XFz7+ejwY0pKFEG23SZU9BK1DCX9YSgYHoHePiYAQAgT8rgbqWDcYDod5jgf7TvKMAXlx8Uww7qzEqLNvg6GV2ZIuM43CrBc4PmtE9HQb0SSwzqt9NqfJVzHcb/nSpMYHhkz3U65Lh33gfeS7fAz929UHcUMWBez6hqR7E94L8Xj+vl9ht8C8JcB/A337999vs3JyMj4qOFpXHmvARjdL2ozAH8W1gzk7wP4N2DdKjOBZWRkfKCRuS4jI+OjAiL6nwH8KQBXiehNAH8d9g/S3ySivwrgxwB+5T1tBDPSjMtJDmP6R/yT8ibdd9no6eefdU7pU0w4pFmrVLhlZjevQ908AgCoRYn6DbtEgvYduE8qDr1d6sbMoI/ZZb+n/zXwa5//nwAAj8wMv/HwywCA//vNT2H+1UMAwG/9pV/Cb939NMzeSu9txqvtm8/ia+Hwl3v3afIw/1hI789Peq+eRxbt43D53j8me5SkDHmlPAyh7e/Yz5fzXx937neTdfoBwdNUTG8B+HW39koA+E1m/j+I6NsA/hci+k8B/AGAX3sP25mRkZHxXiNzXUZGxkcCzPxvPuFHf+aZNiQjIyMjwTv+YcrM3wDwC4/Z/yrsGqynB9mcSgAwFcBzOxswL0fc31knnN2jOahNTFq82dBMoajc7BgT+o2VuFInIbpoBOTz2/SBAZxRiNBRYsWSrVELpvJTSiWqY5S4GuawEpcEg90xuQC0oijL2gucXlhZ3nrXBGOjg1mPysmhBHHIazVMGHWcQfP7H+4XOF9bedvYlkF+J+YK5DMM90WUDxEmK4XDRIlENDJR0TSDC0A5CTUl/VJ0VpZknCSOSvantnIwpH3s2jTSxDRDzd2pewQZreCmUmcAACAASURBVOwoSLT0jENuHygaSBRbQum8UK1ky0kGZ4TW2Ps83AKunaztMVccZNraiGCWJYWBdjLJcZToXTYsKwEM0TxJ9lESWLQ0MbjwGapITE1MSRMzJJ9RqoyAchq6h+0Sd89tJt84FDBeutbLIKcV3fQ4vn+54Ji9mMhvrUQXoa99m4VyBlNerpd8Dom0jDTCca3M3T17WoDceGNNQcZIOsmlNQjaNZuH6Y5pMPlMMLkSSHTkl6W9XgOXzpLGhj7Oh+l54L3iOjUDaGFvXik1bm/tbPb60SIYtIF4wnVlwnXt2sojqZUQrX8OCe4WQh3oyFE6SkVZJEsVikTCqBOJamL+RYbDoCRpwnfMXEOxhOw918mQP3m+nQWuW8071I7riBiFa2CnyyBfTvFgt8B6Y0lDdUVYqiEbFQaFTjJdU7DkkLfJMsr0TWI0o2tAu3xkoWK/yN5LcN31FZHrgMQUKeU6RZPlCd70LjXIkR2FDGo1j8ZGSE2YUq4bOTwz40KgdeQz3mLccFxXHpu4FMQItIMlVkEc9g9DEbluFMAYuU4MlJjdUXLNiDyjE1lwRTB+zDAFPh2MxN4tVbi3X+KRe9epvogy3UGA3LZIlksQx6ICS45LJ3Qi2U2WnqTZq+S5Lkh5o0HdRJqsEY7LmiZc56W8rETkOhUN7VKuJ0PJdvw9gJin8u2E65DwYWrONXFC0vGZzMjIyMh4MjJNZmRkZGRkZGRkZGRkZDxX5D9MMzIyMjIyMjIyMjIyMp4r3pUr7x8bRDGjsmQr2QJQCo1d720Fp/Ld+dIv+qbgSKk3JYq1z7xL5TyUuFCKkMOHxL2PTJQ2GfA0/84380kLkgUDXp4kGfLWPub2AVgtbKbldtegf2RdC1uzDD9nyVFndikb0svYMMbwQZrpkFvIsO6GAAAlIDufbxrlUKkF40T+pKbX5yVWuuYgLSUmSBOla0yXZEkOJnGgTCWe1sY16SqXuS0GgnL5h7qJByr2hNLljBZ7QHbRoTcc3wC473LxVIMHbv/V4w2OGpupt6o6POqsrCzNDjVM2A0uI7Kr0bUVtHM+VbWAcHJxK4mN0jIvdUuvzRQI94QNYXTytr2qwjlTJ0gAEO6+oTAhq1EQo3Fu0qnsmIgxjrZtY19Aty7nb4jSM6GiBC5IeVWUJhZeqp7I23SNKPFMnKiFNODUidfjsiOlw+Wsxsfl4xKiDJxFqmJjryYGMU2kbMER+v3vVPnuQRRk66ZmFN5FHIxN63SgimJ25UxjlnKdk2aa7ZTrvFQU6f1gGbOWJ7JEJ9WGXZLgpa+XMxYn99zft2TZgigM5GE/yWOez21b97saw7nL5sUyfN9m7T6hb7xu1lBcqpAs1TBaQPWOaHoRskHFGJ9Jpig7J0ORb0ZMXHKDa/qc434mFEkGJlKuoyg9T+XppABKcnf9uJcakJ3bPRLUwrvsRnlssSeUG/uFcsuQvW93bKdti+O6cYZ77gTXTtY4dlw3LwZshiZ+x3XAaGRww911FfquhHGZsLoS4DbKaz1nUJVwt6IwnlJ+N5qC0/xmaML5mAmcSLN9Li0qE5abCGlQVtE5P+XGlOuU42QaxISHp9vxHSf7qQt9cJtvEN6hKdeR4PAM8GX3XY902yAuq+C4ffn9GeTBhOAib8dLuobBHcdMx05GRkZGxpORK6YZGRkZGRkZGRkZGRkZzxXPtmIKROOh6DaBdd+EWVRqNAo303rtaBuqSo/OlzBndma+2IlLs83RNCEYUfSJuQEhVBRs9Yfetj3J8AOS7DcKJkJSclqoQFlqHC7tdHlVKNyYbwAAu2WNB0tbxTs/XUZzCMHT2VlfJWUCuepEddBjMbNT6k2psHbVlf22DjO/zo3Ibupk5p2nxg1+RlwMsSJACpB7d/113G8KQMjkOyqZVeZpRSFFqHqlWXN93DYloBb+3ITCnbtaxyqpuOxV7k1GBIX2VOeEAbYv7m1KPFjYqfJmPoRKuix0MGJhJgyuwqO3JahPqsyc9k2cgZd9rKIIhTBbLjQwjtFYY3D5fKORmBW2HSfNHsvr9r4JYjTS7j8oexwUdozUly50rez1POiW6Fy247pvcLaz1fZuH3NRjSGwC2/VioDEJEm2IhjtpJVyLhHyMIukbwCEjEFb6X5MtTqtvCVjbDKuND9RXTDJ8PNjhjEZP77C+mEE4xLXOUJZDzWUy7KlWeS6k9Ueo7u/5xcLsKtCljuRZHfG54EUYl/uCeRMu1jEvE1KjIAmVe/HVEjtdyPXCcmxogigrhWODmzlrpQaV2Y2fmB3UOH+wqpCtuczQKVObO5fQ4CvYgkOFbZmPmDpuK6UOnDdbtPEStUlc5lQSUNyPSZWzuQQ80N5AAqfYZlwHReuqp/05ROzX5OqVzAi08n2wMG0Ry1CLCdoJJQ7VyVdA0XrDeOm1TpfoUyzputHhMG5K929KHHPZTmXtYJxYyTlOq0Fhs7l2+4LUC9Q9LGy7jla9BSqtbJzpkeuTcH8KKlOGiXQj3Yw9WWBZWW/fHOxwYm7/4Ct5AKW6xbSbtfBjQ3QENiM9nrOhlkwUdoMNc4917WR65gR8p4xWtUIjZHrQsVZR+421ZTrhDfxIoPR30MhpoKQ9NlIDOdS46XQd3r6eQ8rUvK/T3BUkCRcx5KQK6UZGRkZT4dn/odpRkZGRkZGRkZGxhPxbrMYn0V24xOWnaQZpSFXEgBJt12WoMbJ8A+X6D9xAgDY3Sqxecl+prtm8IU/Yd24P7+6ixerUwDAgexws7D5pkeixde7lwAA/91/9q/jP/rf/zwAwOxagO1k0Iv4ns0rBaA/4HmWHwhc7uPHZI+m+bE/9XM/TdbpE3/29KJZEnR5R9z2P9NpJjED79C0JyFLeTMyMjIyMjIyMjIyMjKeK55pxZQpyqxMhSBv61WBwklZ50c7zCs786OMwKNzKxNTFxWkN6zRmMgD6TF/lZMBpM/PrDic10oR48yFl65RH2cDmBK5TmpSRBwmCcwgsX84R3/gJJuLDvvKypGuzbZ4aXkGAHhwuIRw2iBBPDHLUU7HZUBBDnVYtmH/67tjnG4WoV1ypsK5o9lJlN9Z6ZG7jkSSZM17/DUksrc+ydUU9p54+aFQj5clTTIqU7OjEZC9a4cGjJtBUQ2CLK/cE4rWfX6IJiOmYJjSbVfJGCmjLFGMQOEMRIpdAVPYHyjZQHqDl5qhvGRbE5yaFmVPk0xQFlHWJ0aENsmOw3fIRLm0KWOGKJtociTAKNzgO6w6XK1sQOHVchukbPPEqePheIC3+iMAwFrVQdJ2Z7cKx9SJqYhJZZGGgvkRDIFGMZHXeqTPgikY1NgdZalRFfYilJZhEk1rAe1mdVliKrtNpNxBhXxJEjrJMfVtSI1CiEBJXmmaBYjHjKMPDQghx9IUCA/NoCRqJ989mPeo3T3pVYFzlw3J5xXkznFdIv1MZdqJmt/lkrrxUzNMFSWawaRFxuUTPs8Tvlnp9ORjJK2ml9i2C7RLO6YX8x6HtZWnv7A4D1x3djJHIWJms8egCxh3sII0lqXVYi6LAa2TsL+2OQnSezAgZ27GXwCm8hrchOs0QmY1JW211xkvIUhXE65j+Xaum4z7xyExFXsb17nj6jLh1pZQ7P250/cNoJwpnynjGNHVlJO8WZLcF2DHdSzi6g9dMbSXbBug8rm0wztw3d4eoGgB6TSuKX9oHc0BYSh0hxQGlSOBK/UeJ9UOAHBS7ALHlaTQuQDrh+MBHgz23b0ZG5wPVrJ72s7juYyAcXzHSsSCAlPURDsZbzBtSvOVk3abgkG1HTNFYUIueAqjJbTPaE1zl5PfIVLJNiVGSGlm80RennLXE3jsbc9YRkZGRsYTkekyIyMjIyMjIyMjIyMj47ki/2GakZGRkZGRkZGRkZGR8Vzx7F15vXSp5iBR61VcOK6NwCMnX+0ezoKkrVSU5HJSkKUxAWoW5TlRxkZBfkgGQWZDjImsl30P8FS+G1x5E8kP6+iMSoJB+wJ0bg9wMa+xObFypdfrMTgmLpsejZPrFcKgTKRuuzE6EXo86hZ4uLXXv9k10C7PT5QmyOP0IIJMWSR5m2KIrp1iTCRJHOVm6R1PHQi9PMn3B4so3ySOEkLDgEgkbUHpVPiD2PZo5xJqKgR3StlFWZmeJe6Zl/Jt/baeG/DSXZAmiK3tC9lSzLXrKMiqZE9BEpw6KvosvDRzNUiEE6Nc684Ysw0nUq9EZpbmj3o33f2+wps7K9OdFSOU+0yrSmx72xnn6zm0c2j24wgAMFzSwbrBV6xllOuOiO7TTqoXpIVDMgZGDhLpHhQcKY0hSPfMKA0IEeXlXv2dnluMiYPnEMeSP0doh47ZhmE7kT0S8dR12en1LruqfhgRpLwVB9fPQRXhOVZaYL2zcsfhrIHc2PFd9NEZNV22wAIhE5iL6EBOOpH4aiSu1on7eSrBTzCRbxNA3sVWiegCDkCuJcSpy7Sc1/jeiTXy+FF9BUVhB1/KdZXQKKXdr4xA73SnldAY3MP3xvY4cN1u04CddF2UOmRmYhAQfeKY6t1Tk0xTOUTJLibyzrhtv+v/4z4XFcKB01IJ/6Q/Lsluo87Zcpn/vM9m9tmmAKCbhN+KhPcKDsdUcwOe+QBignDZrTa38+3XLPp4zPTarMQ5kdhT5HQyl2Th6XMX9se8W6TLFoixGy2P7cYar+MYAFAXCoO2A3E/VrhwzsrbbQPj3egNhTxmTLLGEe5XsZWT3O3gvqym1ydTrkuWmwCEIeHUwo09Y0Ti0MsxQ5eSYyZuxWJM3gkGkIOTv4/Td8Nj+y55f1zOSU0d7jMyMjIynoxcMc3IyMjIyMjIyMjIyMh4rsh/mGZkZGRkZGRkZGRkZGQ8VzxbKW8ioeTaBAlYIQx6J8NZn81BW/uhYiemEhhvODtnqJX9Ls0VqsbqDJkJs9pu7/c1hrXVScmNjDIuad1bAYAUgZ2Lq6kJxsuvDIGc5JCTeCrW0TlQ1hrmGNAXXpcFsJP3tBcNqLVf3MpllO8IRCkREOVNAFC5L4/JXEGtUTinwaoeMfT2XNSLKFNVMWRcJvJdMQKk/DVQvNMEqIXbv6coB9VTWSvLJB4pdSZGKiFle2xYmZszmYVIQu5lH2WGagao5du1TF6qBlgn5SBDKwnG9Vd90IMP3XHGAsr1HXcS5PtM0URD5Y9Lhlx/xPOkMlgRngIKUmNz6ckI8m9pUJf2ghqp0DmJ4nk3w/nWavqMFiEg3owScLJMsY9uuCAEi00WU/mcb2faZjEk8liOErdwDSq6hLL0crIomytk1J4VUkNpGQ+WyNtSJ96J1PaSNG3yL5xkML22d3DapfT4H0Z9G1mXVsBxXen4ihijW7rQbmrQxnHdPnmmTZS8T7huEbmOgOBk3rUV9NqerNjIt7mVpm0CAFNFGSgljqRGIt4KE2+gaDT0McBrT94UZLfdvgF1dnsnDqLsWF66p75NAvEZTWWdtUE5s9dWVgp9Z8mEepFwWnSMTeXlpKLDLAsEB2CWgHbLPESfcN045bp06QYSF2kgjlGZcl0NqHk8d+oc7vtyXMRzpxJqkXBQuvTCFAR9YE9WHw6gK66LlMDouaQrQEPCdb6POeE656obXWYp9L0cgHR9gvZt8dJmtx24T3Jw8i6FDnLs87bBdm8luynX8SDCsgTRiRBtx+l7L+UYE5fbiBETybL/jNBTrpM9R9l68pwAdnkNYB2EZXjPmiDHJmEm8u3YjoRbdeyLiaw76Tqm6Xsm/R3lsoTXXgM/Njkg430KIUFl8guAce/TqgQ1Vs5Osxn40C7D0IsKemY/P6xKqJkdZP0hQc3soBkXwHDolmLNOLz7izXh4Ed2++o3Dbr/dgUA+IO1wO+P1wAA3PVgZQc9D0NYDnOM/+cnjYnMeNZ4nnmyf9S53ykDNf3oM1xylSumGRkZGRkZGRkZGRkZGc8Vz9z8KFQgJWPmZv/rUmHjqk3ivAw5e5xUWM3MgF1VsT7qcOvQZkbWhcLoqj+7oQqmB4t6wKm0xhqjmkH47D4TZ+l9tRSYZmmSwqRiEJCYw4AJy6M9+lkZfnyystlu3VBi19oZ/3FTh1luZo7VPY7GJMQE9vtLg/rAOjFcWe0g3RTsaATuu2Nywc6Aw5pS+Blh1pPmhUlqI6IBhqmA0c3M6yZW54odoTCA6z5rnOLvVTJTMsmf9H0FQFDsMpFMwghtZwsBYDg20At7MLkV4T4XLeLsskpz9wRGV4btrhBmR7akffV4g1lpx442IhhuDGMczrN6CCZFw1ig31XgTobrkc5cpNjFuRltYrbhpOpnrFkXAJDkYOpSyVh2KYQJRiGGCZyOH1+VP1AgN4aFZEinGKgqFTIc1Shh9m6bZDTT0LHyQcAku1L21sADwGRm3pSxilAVOpjuyMKE/ECi+JlwcMBWkUSyLxlXHiyS/gKC6ZL9fqzmGF+cNe6ESIxEPsQIz09pAtdJYbDdWa6Tp2Uw9knVJHrJYJdZPL+yx4sry3VVUrXqVBEMgpZNj7PClvBGPYPwec+Mx+bLpgY8QpGtDLk2cGoUEypNjOXJHv2icD8DDpc2/LcbC7R7W0VQmzLwWzpewYhToGmVqzKoV5brjg/2qN1ztR9LdHtHcJJhqljl88qCNJ+TkoqnkbEarGsOCg1acOC6ckOgfeyXCdcRwnNs+yN+zpt+MVE0FNKxcAkGxgO72V8xMLXnOoliH7luYlTlOXMQGBzX9dcI8yPbv1eOd5Os201n+3pUMhjsNaXC4Krw/VBg2Ffg3t0HJYKBILaxaqyIplW85HH0hkxUGJTuHJWI/LERdeA3oymaZBGC8seUDKqc0V/BEDLmKY+jbaseJbTjOlPIoPqwiiXfGPsO8e8K2UWFkNCx3VxYNQsw5Toh+YlcF+5tMpZYxOopi1gZNRIgSvjNG/0xT8aICfsRnytc4tKMjIyMjCciV0wzMjIyMjIyMjIyMjIynivyH6YZGRkZGRkZGRkZGRkZzxXvKOUloo8D+B8B3IAVznyFmf8bIjoB8LcBvAzgNQC/wsxnf9SxWFCURJbRRAZAMFBAyVBO+sgLhcWhlW9eO9hiUVoNT0EGOyd7uugabJwRgxoljJMYLZYdqsoef6gM2Gel6iT7THCQQ7KM2ZuU5KaZxPxIFAzjjGx0K7EzDWpn2NFUI44bK7+6cXwfh6Xd3oxNlBWRgXIHNCCMbluxQOXcHWqhsSisvK3VJW7vbTbmve1BaEd51GMs7PWbVgQTi6KkibyTncxSV4C2XQQ1Z7BbeK+r6NagGwlTUszhSyRxdMn8JuTwiSgPlAMmubG+/9QM6K86eddcB72v7Amy9frQRLJdTM0kyo0zxFAlOifFHY5LNDM7FmbVCMO+zSbIbJf1gNFJuLQREJUOGaKgeA2m4mBIYhIjGNKYZNz6bVlozJ2M+KDocTC7AAC8tDhFe/z2XNpF0aN2HXOQhBuKpEN7LvBosEYK5+MMP7ywzicPz+I9N4k0mA1ZoxF3PaKNubayi3m/as6hn5Z1j8KFNbZjibKI0jrtjMe05MT8iSG82UuBiQTNt4UETzISp2zydqOp1JQnzUWdSICfI37aXOczlak0qFx/Ky2CbJ8FoJysnpcK85UdH9dXWxxUlgMEGBeDfXgf7BbYOtmsTrKf54sepTNXGmoD9qY2CtOpx9FzXZTHGoUgr04zTUlwyNrVuxJ7Q6hqO7Bm9YDrCysvvtFsMJP2ZvamgHA32bAIz+XIYsJ7KdfNrCMPdqrGndYaf1y0Dcg/qscDlDMiEa0MPCb7RMo+RKmnKQHlPmOquFzDFNHkSzcS1blAaS8BQhKEX5FhogSTRZTsApiYJyFwXTRFGpdWwgsAZqnC8znJWU24xMRVIAASrhtLtDvbX91xFd4xZaGDcRYRh+fZGxQBVuJLhYljTHIwwDIlYFTkXK9MnZgIkZXhAkBRaiwqe39WVYuTyuqfP7V8iPaq5TrFAoXrmJkcAtfN5RA4riQdJK4jS5yNVnb+sF/iVcd1pxeLcA2crBcwhqB6CThpstyLeN+7aIY1Ljj006IawnugHUuYhOuUkxGzjHnPpgCEe5xYxiUpZILvDSgxBptkmKbEyFOui7nTH/5lCxkZGRk/LTxNxVQB+A+Y+fMAfgnAv0tEnwfw1wD8DjN/BsDvuP9nZGRkfFCRuS4jIyMjIyMj4znhHf8wZeY7zPz7bnsD4DsAXgDwFwH8uvvYrwP4S+9VIzMyMjLea2Suy8jIyMjIyMh4fnhXrrxE9DKAXwDw/wK4wcx33I/uwsrf3uEA0f2QGUGCS8SQLpNPXGtx1TnuvnJ0H1dru62MwO3Oylrf3Bzh7qmVfemuSNwfKWSDbjVBlu4HRZSvAkkGWZpfh6RtxaV93qUvVeQoATor0RdWWtfONXrnCquPBNrG6rSuVrsgzZWJva0gxtppznpTBAlUqyu86eS7t7eHuHAOnlqL4OJaFAZjyF2jx2anTTImRZRnsWRIJxODYJjG7a8M9FxAJNmqIesvkbhCIDgCmzLulxRlTKaMn+mvGhjvRLuXwR1ZDBT6W9cIskffXmB6HwCgWDt52rZBJ23ftSWHzEQuGHD3/BxLQMVMPdJA2UdHz3RKxsuIKZU+po6myXZVaswKJxkrehwXVt52UuxwKK0r8yqR7DY0Qidyr9vjCQAraRtdB2yHOkggj8o2SMIvqln4nlKxwcYIK0tOHHSD861kCO/SXHDILxXEod29KiCdDFAKE6TmWiCNOZwgfU6CQ2+RJD4m8jZKHhSmmAlMSV4gpw6X7w8l7wQ/Fa7z41gT1js7XpkB4bKJ5WLE9eMNAOBzx/dwUtnx05sicMCbmyM8OLWSbtNOuY7cs7pTAsJznYjPQ+oszoJBSUf7Zy+Vk7JEOGYqp8RIEGcNRnfcbjHDoOxDMx5KXGlsu0+qfeC6MrF8NUzYunUSiiUK97OdqvHazko572xWoY+MFsHFFQA0e9394zMj03GVSlEBoNi6DNCCYXyuaGmgFgA5qX9BAHt38cSRnUVcAmFkdBtPM4VNSVDuMe2vmCAdFtsiLFUQfcxH1lXiACwA47krlVFroLyIXDcK5zpeROk8F4y9e6edI943GghiJBTOuXbCdcn7FwqPzQjnZNwWhQnLZ47KFtdLO1avlhushOWoJtHkNzSic/dqYIlTZZcnjFygcw3fj1UYG0dVi4PajpdNVUdXc0Nhm32Dfftk8lxJhGw9Lji4FBMxahmdjFOu8668JnHnJcbb+C72RfLO8OMt/UUg6TsA0zxU7xj8Plmq8IEFEd4xB5Iu9fHT5EY6/bZoavAY5fBi1gDXLS/d+9M38W/9e38PAPAvL7+N0Q2+B/oA3+xeBADc7o9xt7O/j76xOcK5+52t3TSAy6muTiUWb9k2NqeM1av2d4TyrTPw6TkAwPQ9tG8Hm6e7hoz3By6PP+DZ3L+fZNx/APDU5kdEtATwvwL495l5nf6MmZ9A7QAR/SoRfY2IvjZ2uz9WYzMyMjLea2Suy8jIyMjIyMh49niqP0yJqIT9Re03mPl/c7vvEdEt9/NbAO4/7rvM/BVm/hIzf6lsFo/7SEZGRsb7ApnrMjIyMjIyMjKeD57GlZcA/BqA7zDzf5X86LcA/GUAf8P9+3ff6ViMKMMhAsbBnr6sFI4PbYXhs8cP8MryLgDguNjhQlkHvx8NV/HahZVBnm3mMINzJxQMeJmuIbD7W5u1gBeDkWBw7TU2ApxIeeGNVHsKdRAbpu22C0zknd7ZUa4GaK5CeDl1Eso5Jt7ZHOC1h7atVaUSV16GEFHz6OVwWgsbaA/bJ945EACEkzhXlULhpLxdW4UgezFEKa8Yo0uhlaTFvvdyVRbWJdJ/3jiHQ10zwM6pF1EaBlySYCZyT5aJu20Z95uCYZzDsT5UEE7OIrvYx2qWyD2LqaSYGhfMLuNnjCLwzh2nFRBjlK55V0wjATN3+0cKsmEytm1BZsVR5ZjKnImT0PlEvksmfqYqNOZOJ7cqomT3VC1woa2EpySNvUupH40MMsaH/RKPOvsHy2BkGBf7MeopJTF2vf1ut4suvzyKIE0mTc6R0suiEwfIHvE7AsH5elEOaJx76qombAcnrSw0BjfGRslTya7HJRlv6lQddqdTXMnYASeyXeJ4rMcc43njp851yTV6F92qHnHFcd0rx/fxysJy3aHcB+njD/rreH19DAA4X89h3LNOZXygWUeugyYYvy0YXEWuC58XDOMIQQ6JLDx5AxhpPwe4W+XGBa00NFPkuoEwOP69fXEYuK4sdeC3QpjAacCU68K+oQg8DsTzVXXkuv2uBg1R/h/ktCnXjclSAyRLMkR8HopewAxOxlkxwBSeaa0oDEeD6BjNYnpM5V3K6yhtZ2ndrwFAL03oIzFELtENTzhTN86JuTYg917y1w4AZhSgvXciJki3BAEjJctNKBxHDJELvMQ5SJt52lbvig+mKP9PPgMgyJHLUk24zjvr3h9XOKM48eK5rjMlTge7/2yY4WKwfKhM7MhOFYH3CMC6szzU7Suwl1APMnAIjYSiFcFp/PJ9D6DoTrwoh9BuABOu65zcV8vknhAiL13aDs/wZLlMfI+l+4lTx//knflER9+MjIyMjMt4mjWmfxLAvwPgm0T0dbfvP4H9Je03ieivAvgxgF95b5qYkZGR8UyQuS4jIyMjIyMj4znhHf8wZeZ/hCfbk/yZd3W2xJiCpAnrdOdNj5+/Yr1FvnjwBg6kNVb4fnsT37h4AQDw1nqF9cZWT80opmt8e+/kgVgK0xQrNTJm2LFMXTMAdrPOpqNJpmlqsOA/TwIh2Gwx71Gt9ti4XEEi4NPXHtpjMeHe1lY/tvsGY+/LlUmbOZowsZoqqr05ynLZ4XBmq3KCGBtXSdtpEbNIZwak4u2J1amYzabmDD13U7UyLWjF74VZ+ZBjiTBLDST9QDE21gAAIABJREFUQZdm46WvQiSmI0wwvkI9iEtZse6aZzpURmeLHvPaTn/PyhFHzvxnXgwhDxYAXttYQ4I3zw/R7my/cyuTkgBAc9toMwpwEccFaZr0U8iY5HidaS5rampBGuF6ykJjIZPZeFcNvdMd4vbuEIDNzvNVT6UktOvwsU2dZggkHzN9TggZhLQtkpxYelvVQIT/J19XmOSv+urEsujDdlPvQhVDGYG+tP3UVzr0GctYnTGcaP7p0qx/vOUTA5rLPw/7k0p1WuV6P+CnzXX+GSXJoXp4uGjxC1dvAwB+bnE7GMH8sLuOb13cAgC8cX6E7dYZAQ0y8gQjZGNCU6iSMwTYZyXKeLOi4RtsJdXnWXYCwg9hkagpEnUICQa5Z3u1bCGPdsGcqCo0PnFiY1wNE+57rts1Ue3BwOPK76wpcrTgwHWLZYdV47JbibF1zw8bilmkDQeTIlvlorAtXbvVDNDB5ChmjMoeYSzKniYmP1wAxntHXcr0TCtjwayt5FhJo1hhpD4qOVjGdpjGBF6aLXosGtv583LEqrb8nnKdYYHXt7ZifvviEPut47ouIW/BoJntO91LmN4T95TrKOFoIDG3khyf14lxGWAcR5dSh8xZALhwLk9vtMe4u7NmL/uxRDtYXhtHCeUNAIepsyAlZkOB1Jgi13UitE32aVWUIJKM7FQJRAYTgzzpzrEs+5CtWlXthOtCtnlZheeBJWBk8g5J3m/iMZzGwCSvdPIDJJ9PKrLZ/igjIyPj6fDU5kcZGRkZGRkZGRkZGRkZGe8F8h+mGRkZGRkZGRkZGRkZGc8V7yrH9KcBL58pSo2msXqdF1ZrHJRW0vSN7Yu401qZ0I8enaDvrKRLtzKavwwECgYXDOFkTJzKhSQQBDQdWYmbb4OXnAqGcHJSvYiSU6ES7c3bYoLsjq4vcTxvcTSz8qvDqsXPr94CALxYnQY58qla4qGyOYQjS+xdwKdhCtmlvSnQOxchQTFXEgAuRiufetAtcbqfhb7jQydBNgLGybbGXoKcsYnoCeRkZXpuwEtnKFTqkBk77kqQk0GLzn4+qKwkgwsn+zKYyJI8iBFNiHqbRwo4iZVrBxOgnTmIORxRze09ryoVzCpkct+2fQXtTjJUEtdrm533hcUb+PLqhwCAt64d463e5Tzuj3DaWYl3pwqczGyuaCMVWudW0usCm77C3hltaBVlZrotAC/xNcl9T/PpEolwU6iQkSeTe7VXJba9Pf6+L4OxF3PM+kzlkf7/ACCEATu5mTFk5Y4e/uOXVb/8hO0UhJDhVwiN2jnHtDpKigVxHOKXjT+iKnxqBOP3m6doR5L/x4jyYKH5Q69v8xLZolJYzq1M9dZiHeSRX9+8hDd2dhy/fnqMvrP3xbQF4J+rUUTuYoRnOs1JJSDelC4uEYBI5LySg3mSngsI93xKE8dneo9BgHF82w0lrq+2E6774uGbAIBP1g/QkH2mH6iDYODUc4HOcZphCpLllOsKMdVye667s1/hvLWy4bJW4NJ+zhgRjKB0J6GC+RdBuKUIesYwK9u/otbhGRv3BaiLXCdGSsxpouTXhA7F24y+RGoa542QGDB9/Lw3QjJHCtXCfqGuR9TOzImIwztk01cYvcy0FrjZ2GSiz8/fwp88/AEA4N71Q9xOuO5ha/tXM+HazJpoCTLYK2dApEps+gqtX0owSmgnr+ZOJq5veOzzR5rAXnZcKsycYZokA+PavR3rYFq076pg7MVMYYmN5bpIWv4+TNTRmsBegsyIHJe27Sk5ghOuq4SCdM9Ar+OvOIKipJ6kiVLu1NAveQYoee9NuC55xKZtSN8fCc9z/C7nUkBGRkbGH4ln/odpRkZGRkZGRkbG+w9E9BqADQANQDHzl55rg/hJM69TUGUnY0jG2STT9W5mHaCihLx+FQDQfu4W3vgrdrLoN7783+Pj0s4uzYXEmwr4ve7jAID//Nt/Dr/9b/8SAOC3v3sdrOykDRsOx7XdZNfcL3GG5VO2FwDUO38k44OAd3HPPxTnfY+R/zDNyMjIyMjIyMjw+JeY+eHzbkRGRsZHD8/0D1MmhPzCqtI4co6zg5b45tnHAABvnR2id/mN3CfOfgbRbbJiiGWUhEonGSpklFa2fYnBu6CeVxM3w6hH5SibrAz0zOdEAsK7F6bSVWIIJ40bthVeb6+gWdqZtnZZ4mOzCwBWynuzsNufLe9DO33PjiuMTsuoWUA6LdnIBdbGStc2ehakv9/d3cQPzq8BsHlvo3M8rOsRg5OKmjSE7bIToN8uGbJR4bs+2xKLLjgW7vc11LaE2Dm52xD1SqQR+4kAI6OboXdWLfaEQiWfcSNrXBnIG1YCuJrHkM1uKIN0ceyK4KxMI+HcHeat2uDNK1bGdvf6Cp9d3gcA3CjX+NTsgT1m0eJebaXfG1XjE/NTAMBL9Wno63vjCne6Fe7srWturwq0Ljv0QjRBigZGkNqyppirZyhKApNMPs0CJ4WV0316SWicxHer6tCvldCo3P5GjkFOWws1kTKeDlaOvB1r/OjMug+fny6CY7OpowsrDdZB2jtVyo4mTpIeLKNMVxkZZHnbsY7ZfkZAG5/9+/jjXEaQkCYSNaGT71ySQXvYnL/EQTpxev6wwXKdvfiq0phXtu83Y4Pff2hn4+9fLNE7x1Uk2aKkKSxL4EZDOGmlLEyQ4ReFDrLEvi+j4/O6SNx6OcojTXTD5cpAz6KkUyTPbQp//Pa8wY83NZqllaa2ByXO5na8/kxt8EJhqwWfLh9gcCTdcQnjbrBODnyZ6+6M9vn+9uYWfrS2eaibtgl5p2WpofwzYBI3drqkRZ1wnXvGmiFy3TLmabb7GuOmhNw6rhspZFsLRVFumWQcs0BY6iETl1gmgNySh+HIgG7ad9pq3gfp6zAUaJ17u+oKoPdLUqIN+Fu1wY+vWifet24c4nMHMcv7hfrcXo9QWJaWQwdT4FNz+3fDreoicN2d4RB3ukM8cJLf7Vhh793cy9ouY4CV3QanXEbkOiUiB8behWaBK6Xlup85eBAykTdDEz5TSh32z4sh8M1MjpDw7zqJjbLfedQv8NqZvebN6QLspNlmlnBdT5CJVNtmNtvzTSSyEkG+q4xE6c69HhtsHNdpI8K4YiWC427qrHtZohvGAsefkQbIPIEgn7T7XUqTMzIyMj6qyCseMjIyMjIyMjIyAPvn9f9FRL9HRL/6vBuTkZHx0cKzlfJSzFQrZawWPdovcLa2M/B6X8TqZskQbsa7ahQOF7bydnW+w88d2txTX7Hy8IYbD4YDfG99HQDwQ7o2MZQJRgyJ6Y5odDjtWEiIzpkIaQCuSiqkCTloPTHGixrtYA077g4FTlcLAMB3xS383vZlAMBB0aFxoWyjkShdlUyzCLPcygi0xs5qv7U/xIPWHudsOw8mOiQMjg7a0HcP2ir2lzeF6kXIIxVDNCZSB7bCAtiq8qDsMatChezAg6bHZl5je2GvR29KcBfNoMTgTD2qmEVqGhOy50wpYhWhAHTj7vPHdriysvdo3TZofbsvqmBaVQwUKq9p5hvvBfadnfn/2vYT+NbiJgDg5uEmGBBpI4LBBRGHGXtBHCok9/oVHnYLXHSN+w5BuJPMmhGjslPmnJTutKBYhdIE41xQ9n0VzINaUYbx93LzEC/Vp+7cJlSLajGicgGkJelg7NSIEZW76LVucNrY67xQ89CGbycmSqYQIfOPBQFCAIMvE3Go8CMx7ICMz5kgxuDK2JVUoR06qULRZfMjX0hOjwlMMn7TLNK0khEMktIKbtK/aaH/Q4lLXOf7++76IORSmi7lOhMyPetmDFx3a7HGKwf3AACHRQvhOtmwwN5xxt1+he+d3wAAvFEcB4Og1MyLpAnbYqagQ/6whGwdfxiEvGcpDarajttOVNCbEp2rjt3XhHsrq1L4PXoZ/0h/FgAwk0MwOQLwRK5buzzMe90B7m6tOuRiOwvKBSE1DpddOM5Zl7wbvOFRJyC9WV0f1QNqab8P2PHcO5VJU41BobNqeqznNba1y8VeF5BdlAEId69MzVA+E7XmYDwl24SvCkC5jOjyRovDpb1v+75C65Q/ZlNGrutpkvcbqts7gW5v++Kr6wbfWlmuu3awC6Y+2oig2KiSjNHeFKF/H/ZLPOiWOG9tHystQuW7mQ1BdWNMNCqyplI+9JrAj+G6kSXmbh3eK8UWn26sYiWthjc0hvvfJOHKC9FDuIrpuZ5jY2zb7o+r8JnvD0XI+zYqch0gMLXIin2fVkwhOKilBDGMa1dBJihCRi1gvOKFplXMCdclRkXeZJF0rJLbhYGPMZFigKLTW8DEUOmDsSTsl5n5NhFdB/DbRPRdZv6H6QfcH6y/CgAN5o87RkZGRsZPhFwxzcjIyMjIyMjIADPfdv/eB/B3APziYz7zFWb+EjN/qUT9rJuYkZHxIUb+wzQjIyMjIyMj4yMOIloQ0YHfBvCvAPinz7dVGRkZHyU8e/Ojykp6hDDBgKYdyiA1bU5GzGtrsvGx5RovLaw88guLN/ByZeVDB6ILUskdV3igrCRoZ2pcaCsrOZzvsSicMdFYonfyVaVFMKUopEY3xExHrp1ZRV1Cb52sqJNRYlePOJk7OfHVLR5eWeLC5e3N6wE/f3AbgJU9fe3RSwCAB5tlMO+w1+2kYUxBYmVlVdFox+t9Zs2IG4c2x/Og6rF013OvPYgyZBOlVMSJnFJHyRMKxryxfSoF4/TMtnmvG2xdluzBosO8GsErl1WnBYzXNxGC7skUDNPYk8jVCOnumzkRUE6WSJKxcEZHLx2d4621vT/bR/Ng8iIGEYxZTMlA5e5BEfMtWTLgshehBPYPrMT51YeXpEOue+ViDPl638JN7Fq7PewrsKIgeUZhUK+chHneB6mcFAzt+rPtK4xB2y2CNKvrSzxw8uIiyTGdJzLGuRiCpHxvqrC9Vk3IcAQQZJkb1UC4E4wsoDjmApohSuzIyQFFJ0AG8Go5oZJ7TYnUVjAqJ6c8qXZ42Nt2HxQ9VpW9/t1YQTlDkL40MJUbkxqTbD8vOQRx3DaJcYhAuJ9kEoMkk8jXkvGJxATpwyjpTbmOiIOkVCmJonKy+vmAeW1v4q2DNT7jzL2+sHgdL1fW2GZF0TDs3MxwX1u556laonYD4LjYBaMZwOZjAoDSMo1exKhijAI7M7SxKaA2dkxSLyBmziStGXF1aWXqx1f3OO0WgeuO5m3gur2p8NWHnwAAPNgsoJJzRK5L+oUTU7GEA5vZgJNjm+N5VLeYF5av3twe4Tx8GSDHQ8RRRk4aYexxyZg1Uc5/cWG5ot002CRct6gHmJV7h+hZkFrbG+eegQIwM/e+OhiDyZ5JrkFKg0PHdVeXO9zb2GdsdzqLGdEjRXloyfBq0gnXFRzzapXA/r7luh8/WEzyM4PUejmG+6GNQOuM5FRXWhlsYvZXeq5bdEHaz0xhqUPXVtDOAIvG+K5q+xKnvW3HshgAWKOquRiCVLckHWTEW24C1211DZWEwHqu26k6cKZigUEn0R5eTjyIuJTGLfMIGeNj5BAmxKl1AdQuF/uobHHuMnFXVYudy3jdDhVGtzRoKAtw5YwIdTS/YgmkbmxhjJlo6CcUJnLisGxBx/E54boPhnzX4waAv0OWnwsAf4uZ/8/n26SMjIyPEnJcTEZGRkZGRkbGRxzM/CqALzzvdkxABJBbB18WgHdEFgRR28lnWh3AXLWu++0LC9z/op0g+dSf/RH+w4//PQDAJ4o1XlX2M3/rwZdx/us/DwD463/lX4AZ3MSema5ofgHfCsbmGRkZzwZZypuRkZGRkZGRkZGRkZHxXPHMXXmlk8WOiYTnZLHHzx5ZGdvnFnfwz81eAwB8qlxj7iR/h6LBQ21ltG/pCv+4/TQA4I3uBA8GK5/aqwp7J9u5NVsH59ZlFeVwhosgt6kKHVz69l0F5aREZkzyU0XMOiUgyD5fnJ/j51Z3oJ1msREjfsG1uyKN8qa9zj+YfxzbMZoDmEQmVLhjFYmT5bVmi5u1lbS9UJ2hIStpu9ALfGP7IgAr3VrM7DUxI1yDriVM5XNIRZA/FcsRVxZ7ADbD00sJh53EuLZtO91VELUO8jtoCo67E3dPQ8FhUu8LeEO+qh7Dd42hoM68fXGI9XoWj1M7B9ylihmYBaNwEqt5M2DppNzzcgj9ctE3ONvYkw1tCe5kPKZXNQ8SvZOHD4PEuHX64FGAFMUcXIiYXQo7Dvz9CFmHPYJMmjTZ7wNQg8Sj1rl5MuF1Y3P4zroZdp1z4TQijiVDYfxwL6M0GTEzFQCKUodjeqnbuK0gkqxF4Yax9G7JiczMg0WcUAYQ3ClTnA2zIKEUqU0kcZC0Jf6SLnvYW/fGbUrGhZFxlivNJWURv5rK2z6M2aUTECDcc6aMCBmLN4/W+NyRddn9mdn9wHWfKNY4ErYHl6IOXPdjNcM/cVz3Wnc1yLHPh1kYqy/ML0Im7rwc0Gs/9uK8Y1nocBu6NnJd6lYO4iBRJYoy8E8uHuELh7fROy3jUvaB6yQZ1M51+uvzFydc52GYAtcJ4sB3l7lu4Qb4A3WAb25eDN+fO6lsKwyMuzbdSZjWZR8PFBzI5WoIEuROFdiVtj3jvsR4MeW6oCZX9LZnyHVHkJQaWQBO5lzWKvIkEGTAdy5W2G/c9TOBZ+4BXZo44KUJ/LuY9ROu88s5zroZzjfOVbcrbZ43MMmiNaMI8vBxKBKuS3gb1jXY9xkzYebydCVxWErTEcdnVEeH46Et8XC/CMfy7+zTdo5976TDSoZsUKNEyF2GoihNJucK7bZF8fYalN4lGdodofAcN8Bl7bqvp/dJWt7xSN+tyt3E/RiXSUy5DokDOVunc3fMlEBTaW7qRh5POm1P6Md0CQPhgybnzcjIyHhuyBXTjIyMjIyMjIyMjIyMjOeK/IdpRkZGRkZGRkZGRkZGxnPFM5fyVs6FclEPqJ1D4C9f/yH+wurrAIBPlR2uSysfGnmGM2ND0b8zjvhq+zMAgO+2t/CNsxcAWFdRLxPrtQxyqP1YBQdCwzRR0ngZoyQODsC7fQ2zt91BY3SMhYkOjECUA93tVmjLEqsiBsG/pays80Ru8Qnnqrk8jj83PJ0H8E6FDY2QbvtAdDgQVsZnIHDXLda/N65wr7OOnMoIrJooTw6SLuLgJGsEgjxp3oy4MduEz580Vtb7oF3gfGclY92+gmmLqExSFOWWMnGSTJV/vYBWzllZFlG6ZQht4R1JAemka6sbmyCze3FxHgLib9ZrHBe7cP0fLx8BAI7kHt5X9K5e4Q/2LwMAvr25hXut7Yv9WIYA9VJqHNad21/htLHXxkwwhoKrJmDHHwA0hQr7CmGwd/I2IUyQ3bKhyXhQTtLWqjLI4c7Xc4z7MvTdxC3ZS8E5cSi9NCbHIsr1tFfAdRKij7JdL1d8m9tjGvKeItl3Ps6CtP1eexBkeZqnY1u7e8hFcoJEsouonLYSOe+wmkrjUsVcElgP8wQZ3IdR1ktW8gnYseZdkP/Faz/Av3bwDQDAJwqNY2ll4SPPgnz31UHjq+3PAgD+6e5FfOfiBgA71j3/DKoIDtLbocastLxqLmmkhePGplBhGcJ+X4Md18Ek49MQjH77XOUb7TGOq/3E+ddz3bVijU/UlusOEq7TT7ipczGE5Qkr2WHuXIdTrrszHAWuA4DDmT2uFCZwXU9luH4WBJa23fNmxK35Onz3ysxx3X6Bi5Tr9kVUpysxdcelZEw7UC/Ao5XL9rKMXMeYSFb9UpXF4RY3Dyzn3pivQ99dq7a4Wtr9J3KLm8WFbafchXO9oU7w/+2tq/t3tjfxqLPvw5TrKqmxcly3HWqcziLXpW7HRIzGLZNoChWcawFgdMcqCgNVR65jjmQ/uvHQqhK7wV7/+XoO5cePiu9K4kvu4GUkgnBMAvRjHOVFJyC8fHcEKJXuMiacMjFQDs7h8QNrVYd3y91+FSTvhuMSExIME7gucWIGgVJ5sDuuQLJsQURTcUraRgyw7wuB8M5M3ctTNXFGRkZGxtuRK6YZGRkZGRkZGRkZGRkZzxXPPC6mceYLLywvcLOxM8dfWvwIh8744q6WeHW0067f6l/A97ubAIA73Qr3XZVsM9TYuLzK1OBjHGUwpVgTB4MPWZhwXiJG7WaQF+WAWWH3t0OJ8yEaggTTg2SKUxCHz6+HBt95cCNM5C7qAX+4ugYAOCzbUB2thcKqsJWQ3hQTAwZf3ZjJMWRg9qaAdvMF9/uDUBlsVayWnMz2YUYYAM5cHyjBiQNN7POqULjV2Jn5W9UFTootAJv7+lp3FQDwjbMX8ONHxxhcHh4LAfYVw4lBCkKWHoCQl0edCPmWLBhcuf3LET/7MWv28s+fvBqqK1fkFsJNtS9EH655YxqshK0EVDA4dHl51+RDfMZVUh8sZzg3ttJ0rhfYaJfnB4GGYlWnY3stEiZs2z4uQ/bew3GJzdiEvr/DNnO1LhX0wlZ2VFVYoycAsjSoXNVhVoxhPIxaYONzabWcZDf62XVZ6GBaJQWnrvcYnBlT25UYW9fWPk7Ns3B5rwBY0nSmPjXgIFf1AQDBwfhjryo0Mo77R8r2nzYiGiFJA3KVEwNXSQIA5mD+xCIG9wlJIO2NkABOKrq+cmIzUGM/TKoIH8ZKqQMTUFe2v19YXuDFuU3j/PzsNkrXIXc18L2E677b3gJgK5QPWmtytOlrtC5rWWsRFCE6eSb3XRUqnUWpA78RcTD2WlZ9UJZ0Q4kLl1fJbREzQJP7IYmD4uRRt7Bc5z6wbHq8emh54zLXzaR9ZkaWkIkzjH++52JA6UyVRiODodK9foXT3o7J/ViFMXnUtIHrBC1x6gZQOnQoqbxVhcILM9vXL9WPgvpkY2aB675+9iLeeHSEYW8rgCy8QsB1QlrJ988SI3Cd6GKVkCXA7pmRByM+67nuyqv4ZG1zt6/JWMGtSIdq8shFMHyS4MB1J9Vd/GxpzQBPD5rAdRs9w7mOGc5zEVUzexNNpzREuCcdF9hr+7MzNcdO2e2dqmDYvluGOpYIe1nCCMd1lQ7jp5YKZeNM85YSW3Kcq+KYBBDytYtSo3HqqELqyWe8WqNrK6hd5OUAAfi4Zy4wzQQ1mOQrxxzTSLh7VQXjrnk54Kyz1WRtRHiHSmmgXXXbEEeuMxwUBEyRx0SRZoRb7vPbaX4zJfwWfoVI1SQ5eyQjIyPjj8Q7/mFKRP8DgL8A4D4z/zNu3wmAvw3gZQCvAfgVZj5775qZkZGR8d4ic11GRkbGe4wklxQASPiZWwmiZLt0v57WddxPBHNhJ3qoKoHrdrLp/IvXcPqKPeZwYlBs7ATE5r/8OP6Lr/+rAADz8BFY+Qn9DtfM79r9bDCZSc7IyHiueBop798E8Ocv7ftrAH6HmT8D4Hfc/zMyMjI+yPibyFyXkZGRkZGRkfFc8I4VU2b+h0T08qXdfxHAn3Lbvw7gHwD4j9/xWGQljABwpd7hZ+ZW9nR7PMbXdp8EAHx3cwN3d1ZOue5qdE5aqoYiSkpHEY0TGEneZiJjLNia0ABQitDNnBlPrTFbRAmUx7wa0c6t9KhnAu99fqQI5kdCGNxw8uNSaNzdHOBibaVVfV8GI50f9lfx6KGVSbFKQx1p6n6QZKX6TEs2FORQEIyqsW06PtjjT1x9EwBws77Aj/Z2pvDObgXlZHxaSWCw26ITMdtPMI4LawLyqfp+kLctaMAr9VsAgFdmd/BPlp/GDy6sHPl0P8Mw2OGhxgLG9bEsDGYuV7AudDAg2Wxn0CrKCQ+W9hw/d/Uu/vTxdwEAL5cPMTgJ7QO9wqmycsXX+xPcbo/s/m4ZumdWjLg1sxLkjzdneLE6BWANV7xk94XiDIO0x2zEGCRsR6INksmSDEYW6Ny5R5ZB2vvaeBU/7m1fPhgO8IDi+UNWIXEwOBHCoHISx5N6j5dmtk36KM7xNCJKswU4GFuVpFE7uV4qLx5Z4v5gx/y9/gCvXtj2vEXH0O4RFSOFWWMyPJXyaorDijHRZHqzk0roIJU8qfcYnJxuO9QY3ey1EAwqoksHq8S8xI9jydbwBICRMd9V9AR22mTSMReSzNTIxLeTFB5rlvQ88dPkOsBKGAHLdS/Utsj6ve4W/sH6FQDA67tj3N9bnrhoG/SB6+Q0D5Ijv02yhQsv7Y7yQ2WA1klLqZlynTdgWtQD+oUdVx0A3sUx5s1bCmnwgpMfGxZ4uF1gu7UyUKUE3pLWqOj7/TWcPXLPjBKPN+RK76+8tNzAPWMkI9ddXe3wz16xvHRS7nC7s9xwd7eCcpJ3PQqQ57pWhHElBePEGal9vHwUlgXMRY+fq28DsFz3uwefxHfO7DKR090cg1vGkb5nRGkwd/3XlAq9O/d+X9usawBFpXF4YLnuc1fu4peP/hAA8OnqXsi4PtVLPFD2+X6tu4I3Wmsc9ahbhOUci3LArZmtBH2sOcdLlV22cKXYBq67VqxxIu0yjIp0MM9b0IDG8c0fxXWvjyd4c7gCAPhxdxKWiRgzleOmXOffaTdmG9yqLRfr4yQfl3TgOkkmLB1IORCIZli9KfFwtOPlTneIPzy3XHfv3iGY3DKSgQL3evOjyCc0HWOJNNabuxXCYOsky6uqw2g811VQCdf5PFUWCdcxotmh4PAc8pBw3UChwiYTrhMqSn8pNTxK5bvvE67LyMjIeL/iJzU/usHMd9z2XQA3nvRBIvpVIvoaEX1N73ZP+lhGRkbG+xE/GdftM9dlZGRkZGRkZLwb/LFdeZn5kpn7237+FWb+EjN/SS4Wf9zTZWRkZDwXvCuum2euy8jIyMjIyMh4N/hJXXnvEdEtZr5DRLcA3H/aL3qHyTd2x7gYrVveG5sj3D+1Uie1L4JUkDSBUqdPv/5dA6b2clcrZQMA1Aw4eY4oTZT+XpSg1v0NPgjsnQSsayucllZ7Q8TB2VKUBrp0Eh5NNqMrAAAgAElEQVTNwY1xUEWQXv3iwav44vJ1vN5baZRhwtw5Un5/dx2vV3b7bD8LUlsAQTJlTCJXIg7bVaGwcA7CV2dbfMzJu65VG9worZRqb2oYJ40yTME50VQKuvbtpqB2JmKcORfWb7cv4MfdCQDrBryUVqo2FwOuVRv0B9Id9woeOQfIoaUgadJAkNOtmj5k6b14eBGusRIKV2pbMfrU7CF+1Ft58O9uPoXXd1bGdvviEJ0bC/2+BHfOGfJSDuM3vbRUMiontV7Ou9BHdaGwKGxfL8s+5AWWQk8yOtMM2VJo3Khsv85lH/rm4bDAg539g2K9mQcnXuoEhBsDIzHWzhH6SlPg3H23Eiq6kAqgc7aSkkzIHuxNMcmZ9M7Ara7C/k6XwQ1VlCbsFz1BDu5ZGKf5eSnIOLdK2DzeXtlruDW7CHmId/aryXdSGZ/PetUQIBGleDbrz0ouzegkpCyTNkwlbd40WoxTeVt0EuaQs/s+x0/MdV3Cdfda2+dvrA9xfmHvg9kX0dVaR6kgmSQ2lqMb84TrKgO4/aLSQVpK2wLUumdpENj3zgF1X+GR4zohONxzy3XufqrIdb2SQQr/5cPv45dWP8TrTgY6Gona3eDX2iv4Ye0MSNomcIPN00TYDs7PgsMYqwqFA5cnfNLsAtfdrC9wtbBLJi70PEgxGQiOw7oRGNw1Gx2deYk4SEV/z3wSd3vb7zM54tC5oy9lh2vVFrsDK682THikXXa2JrtUxO3Xjd2ezUfcWNo2ieP44FVC4Xpj5bU3qnVw/v3q+pN4Y2clyPe3S+w76wA87KvIdf6iHL7llw5IRjF3zt/zPmQuz8sRi9JxXdFH6SpFniiEhjLx+LVUgesOZBe47kG3xP2N7afdurHuzHBLQNxzPApgu3QuvrMK5zJynb//kkzgsd4UYdswBSdmwwTlpcVGQjmCGowMyyJEaYIsV/aA7BLXWzw+/5NS51slAs8u5IDz3v5usRkOJvymExd/4ZfPCIATrksR8rtZTNyf/fuAlOU4wP4bzPIZoPAA4H23bCEjIyPj/YqftGL6WwD+//bONday5Lrr/1W199nn3FfffsyjPePxjMeeISMgZDQyRkr8xYjYFsTBSMEGCSMiRUhEIgKEHFkCfyMBgZSgiMiARUAhtiMImUhGcUAowQIHT5xxbI9f8+7p6Xffvvee135ULT5U7arap++5c/p1zp3u9ZNave+++7F21d7/c0/Vf6/1Sb/8SQC/fXvCEQRBOFKI1gmCIAiCICyBRcrF/AZc8o9TRPQGgH8K4BcAfJGIfhrAawB+aqGzJRMkVydreGPXJdDY31kD/Cg/JcmCOGdwP84Q5AM/G9Zr8OAxN3q9nlfY7rnEPid64zCSu9/08erQzQy+fOlkGMkHE5QfElY6ZiWwyWiozgxo3Z23UbGJjFHY9TUvpzbHe4vzeLznEjhNOcfI15I7pif4M5su0cbQ19gEAMOqM2PW1lRLE0WkCXL6VIfEOVOb443KXc+Fcgs7PpFN1egwG5Flxo08A7C5DTNSzITLlRsdf218At+9fL/bt8qQ+RnJjX6JzaIM8e1PC0zbOn+TLMxcc6Yw8TMKFxuFiU+ict/6KIx+j5teGLW/NN3AmT03S7q7P0BTuu25jAmsqE7qxirujio38b6oh27fHRrgalpLtU32kltkflbIGBVnkZIEK+3PD5xwswh/avsiaj+Cf2W6juHY9aGZaKh9Xwe3igk3bF9jUrp2uTzZwDmfqGtaZ2h8QiFFHGZrrVXh3qqqLBniT+rZNgqZr3mZJsgyowz52G2fDwmq8k0US9j6fVw9RcDNIrQ1AMHAqPYzNTbDyb6bxX5l70SYOVDEnVmEENfsKH9ayta3JxNiH3JST3LObC6n9fxmZsaPAndK6y6P1zGcuPtqstsHTQ/SOtvRuqzvOrk/qHD/ppuR28xLHC+c1m1m06B1I1PgpT03U3dmZzskLUM6U5lqnVGhD3u9Bm3l34ai1jWNxjXvaJnaHI/3LuLRnqvLObJFSKjzQL6LpzZcoqKh6YeEP3amf1utS5Pl5KrBmr+pZ+sPn6t9wqNyC9cqF0dtdNDuLDOoez55TaNC4i1mwoXSJfW5Vq3hlatOM+taB63bHEyx1StReY3amxYox/6hmWgo76hhzZgYd+7zjcJ007XPicE4XE9lClS+FuurwxM4t+f0YH/Uh5n69qwpzozXKpTcTGsCp5B1zz4ADKnAUCXb+2umnoX212MNwfqa3dT2c5urLLM4fdK5WZ7cvhhmK3fKNUwnUd91q3VThOHqaqAwLl27XBxv4nWv42UdnR+p1jWNhjHtLHZMqJQa4JkpxE2pBo4y9Fqt2yd48xHogInMVEJte8vaqHUWhK2ec/K8vnc86Jud+Whpz28tdQvjht9jJlmhX58sI9lkdla3/fwlm258/XkEQRCEyCJZeT8x51cfvM2xCIIgrAzROkEQhDuA0iDdDp4oUM8PijBDbfj38ZmBzA9cDwrYDTcoZAcZRg+75YvPEP71xz4HwGV6/oN9l+H8i398HKe+4gZRTn5pCPWqy1dn94doqioeXxCEI88tJz8SBEEQBEEQBEEQhFvhZpMf3RRMQJE7y1ZtVKhRSpqRHXdJeJS26Hlb4wObQ5xeczakRwY7ON1zdfVO6iEezS8DaK1hzhrUJ4Pce2yu2h7+99oTAICv9x/BuOmFOFIb0n7lrLbDuhcSMxmrglVrkuWuPiic9WfX28r+z+7jeCF/B47nzlqnEm9PabOwvyYbrGuGu+MAbRyGVfjdPvfD+qEpgp3uWrWGK5O1sN+ab8eqyYJVtGl0sK+mVqPGqGD9vTxZD+1eT3KUft+RWcOFjIMVjC0BPnEK1RQtmw0Ab4Gr6z6u7jmL4rXBeqzFaqI1ik1MnNTa2XyDAe25NAF5tBrSAXYnWyZtx9FaS5bAdUz4VLfnaJSz0CVNEZqfGFcyN0r7en4cg8y15biOfY3E2t1x21mg8ba5K6O1UHuyHuUhaRcsupatYPWimLyGknqOhlBX7b4UajvCUCxhmcfjsO4en0yS8Cixt3Fhw7XvlGvBBnrf2ijcS7WJiVLSJFzIbLTiIbGpUfTDcW5Dn9ocSU1BCvGwTuqYJs7FtKbpEXT13jKskkQ9lsI9Q5mFOu7WK2VRFG75wa19PLLukvE8PNjB6dxp3YP5Lh7UTgNzMp16ldp3xBVb4CuDJwEA31x/CCOvdamd1jJh37+GMKx6qJrY732fSGyki5AAjohxaeLs/79PT+Ab+Ts7WtcmX0s1TZMNrx5Y1iH5jUZMzmOJMLY+PlPgMpztdtQUuFq5e/JaNcBeGV+BaJlUedC6usqirgDheTCWcM0f59J4PdhVm1Kj9PfqyK7hfKp1RoX6z9Qk2mIImLr1ddPHpX2ndVf6Gx196yynGtCiEJ5pzg04WFlnJnHa562aGS8OpYWjlrIlNFmsdUut7sMnilLtqyiMS7nrx0FWB62b1HmoTd1BJc8ju3YGgMvD+LlhhnmoEZ5CiS6z4u6wd/KwN+lrGEnyovZ1hPAqQrJb+Nl09YJbretFrdut+ljz13kq0bqq0UHTiDjY2TVmtC49X/uZphnsk4RZA6CYuQC4Z75jPQ7JGmO7CIIgCIcjM6aCIAiCIAiCIAjCSpEvpoIgCIIgCIIgCMJKWaqVFwrYHkzDj9pbad6zfRl/etNldnwg38XJzGehVJNgVzuhx5hytJ+19dIq7mFqvJ0SGjmcl2ZdlXi05+y+eoux77Pjjm0PE7+9Isa1nrP5vD48HmyNxio0PpMfM4WMlkXeIPf1zl7dO4nzV7eiLTix5ZXTHPWktSnbmMXUUrRrpW6m1JpUqWj7IQDB+unqFQIuU2dTtHZaHTIO2yZa0mAoZEk1TOi31qbBKNQeHdc9jGsX53DisvB27HHecsYqWnmhOMbECNY1O0z9V4h2VM3Iff3Rk9tD3LfmMsOeLEbo+ayiW9m0U2OwzdRZs8aur7t3vtrCRZ9t88p0HRMfd5NklDVWhZqxqY0xa2vltS5fo4LNsjY61PocTovO9QfrFiHU6IRC8JdZSzHjb8bg1nOX2Ptchlrf52RjVk3NUK2lDwD5bKPWKtePQLAoA75+pb9RWHftbJ37B9HKSz0TLN+TJsdVX2Px1GCIDV8PcYRex8bWLud5HWyTriZlvM+tb3OTGzS5O5kpFKzPNmunBD3x+9YE1V7GPDvb3Tg8RsDxNXdPK2L0/f32+LHLeGLdlUI9nV/Dtnb22G09wjq5PtlUVdC6mjWMv5kq7uFaa6PnKN1rqsS7i4thedfXH943fZQ2bne1cvb1N7CNXXZ6WBvVeYba+7CXmZC19KXdU7h4dSveA9og99mvyzKH8TUwSXN4NtimN+WM7bN9ljpW3MT6qRm6cMcv+hUyr79VlcWsr3W06qdWSWMV1nxd49PrBscSrZskWleOc2fhbWm1jqhT97JdDxsz69ph3tXoYH9n5Bvu3KnWbffGKPznxnY+xvHM9fmxbByyEU85x07j+ud8eQznp07rdsq1oE8meQ5ro8JzaJlCOJlKPcTu2W21rjQZJo1rg3GVd/rI9ryOdYQlPQ5CRmSTW4Cutyyn2XehONTZpSx+BlJi42VLodY41/EaOAPYW2K5jaP9GEy1InltgQqLfuYz8td9XPU1m7eLCTZ8TfExxc+oKvnTR/dsR+vC5RPDeLu0zQ1MW8t7oKPWTaLW6YqilXcmqzl1u0UQBEGYw934J6EgCIIgCIIgCILwNkK+mAqCIAiCIAiCIAgrZblWXgDr3kJ4erCLgXY2pqfW3sRD+VUAwNTG7LlXzAauNi6j4L7t42zpCnxfnG7iwsRZndKsopXR0N5udLw/wenBnjtnVoZtJiaH9Zkk17MS2z7TZLOuwr67ZR87Q2eHa2odbWKFwkbujmWsQtGvQ1ZWANFmlmYnrHS07dqZbLJJpkIOVdeRWNoslLe0ZbnB8U0X63qvwtTbu/I8i5lkE/sQMbkMk3D2rk3fBu9ZvxSyGwPA0FucL1abeGV0MligxnUebLFlnUWLZ2aw0Y/t2RYvH1d5sJbl2iDPXNz3rw/xQ1vnAQCP9i93rIstOvF4GlCwNE45R5W7a3tXMcBovQgxm8Rn1vanYRWyhQII2ZBDVmT/u9Lm2Kld/56dbuP8aCvs09qlrWJw5m2JlXJ2QQDILTJv7zu+PsH9a/txX9+fmbIhS3OmYppGTYzMe7pyZcI2A10Fy+XE9HDB2/hevnISo8zFaaYKehLtc7PWsFDkvYn3EmkOWTgB4Ny+z4Ba5cFaSgAKb4EjYjSJlTPNXt3S0wamzSZtFer2nrcKI58BtR73YIeu31SpoHwIZBHbMUnRezdm5QUBx3rOyvuOwR62vFX9icF5nNDuVYXUjnup2cJL1mXg3mnWcbbcBgC8OTmGK/6ZnNW63FvUTw7GOD1wmXs3dBmy4U5MDu37bl2XOFUMw/5F5nT16mQNU5+NPNU62yds9ZwN1rBCf1B1tK591tPO67yGwBSzyVrCQVmqQTF7LDKG6vvntddge8O11yCvg9ZZphCD9fuH4/lz6UTrHt7Ywak8Pp+t1p0tt/H66AR2fUb2aZOhrN050izns1rXtv+0zsL1a2Ux8FmNU617pLgStG5TTcIxdPLg5jDh55ozTHN3bQ/3rmJ/zd0LY9uDCpmO4zhynbzWAszXurEpsONfh3hzcqyjddpnQreqhvU6azINtJKlGZm3bN+3OcJ20VrT7YFal+qEIhvsy6nWFapG6dPuTkyOS6W7D1++chKj3GvdWAd7LHkbLCey0fa1qilk8lW5DX9bNFbh8tA9M/tlr6N1A7+sFcO0/ay7Ytr+HaCJg9bZxEZdNRqjifssqsY57NBbracKqs1AbtHJ7hya5i7Nzqt++IcAAJN3bOD1v+7a7x++78v42MZ3AAB9UpiyW7/PhGv+77yxLXC2cX/X1ayxb9x9f6Hewssj9+rJufEWzp1zHb31XB+//Mz7AQB2NAH8MZ/E8yEWtgzj1y+ldulBZQQO407H1Hm3Z0k3XHpO77cndXC7sOXQb+n2/pfJ8pzYVVsPV7v3s4y5/riz78rdaDvM69M21tn43+p6Zre5Tf1CRQHY5PUI074DYZfX93cAmTEVBEEQBEEQBEEQVspyZ0wtMKrdSJle4zC6+/zwEXyleQ8A4MJ4KyRoqIzGcOpHJsuZxBdp3UcPJctvZhYv+BH4rGdQ9N2otiIOyXA2iwon+qOwvkWnxdMYsL4GYVVnYRbp8a3LeGBtD3u+rikA9LQbLp2aPNbtm6klWNvuSHdLloyktyO2PWXCMRVxOFZjFaaNm/3iJPEF25ikiCqCKv2o+bSHc5MtfxwbRtuP6QlyP8T7UHENG1mJKwM3gn2x3MBu6WdwpoMwe7rWq8Ms4VpWo1AxvsaP6CtwmCk8mY9wKnczNaXN8WL9gGsLEKZ+5Hy/6WPoay/uVYOwr2UK9ehmR+Bju8XlQjWdfmzvL8vUWW+ZQhs0VoUEUGWVhWQcabITquPsM2uFpokzw21861kZ6ukWqg7nUzOF+NpEM7Nx5iFBE+Nk4ba5tj4IdRiNyWGNnwltfG3ZTg3EuBgm4huFvnclbGYl9v2M89XRGoZ+9qeXmXC/MVNISKaVBXysaUKVdD0pGxLkMNtQG9P0DGwR7//wXFISaMyTc3diEWonK7JhRvz/7j0e6olemm6EhDyV0RhN3fZlmXcTYPkkWGSiyyJNUvNmbvGt/jsAAFmvCUnYFHFIfHOsPw0zuJ17TyeFFxOtq5M6p09uXcTptd2O1rXPaGN1cClcl4iMDx737Gqtr0GtmwO1rjIZKrMetm/Xs1Hx+awozMSPJkVwHBSqCe2+ocugde/sX8XxfBxcE5fLDVyaOt3bnfaD1m0UFU4NnHatZXVw+ADoaF3h4z6Rj3Aic58nQ9PH5dq7eliHhHt7zSAsD+tQDNO1gdeSjGyYJc2T2clcmaB3hWqCvh3WvpYJjY1a184+l2WsCctGhbqkTuvcviaj8JmrlQ1JpdazKmhwTzXQfgozU/a6z7vZeBQxMlX67U24dzpa11Cop6tanUsTCR2gddZQ0Lr1ogp/Q8zTOndNUevaWHMV21srC5UkmGq3oZxRtQmpehqmaBPXUXwuCdExYHDXzpQKgiDcbmTGVBAEQRAEQRAEQVgp8sVUEARBEARBEARBWClLtfISA6+eOwkAuLC3GaxE01EPPPa18Exim0zqR5KNtp3MzrExqrjMmoA9d0y2wNjXVGONUIfzasZ4rU0UUxgon+xGaRMSCtlaB9vwdNjDK9dOuG2OM/q6xpa3x1lWwX61nlWhRqcmDokfLCgkU2j3aUkTXAQbMAil8TU2mwKXxs7SNq1yjL3tr55msd5lpaBKb70qo5W33Onju/WDAIDv0QOh3XVmsLHmbFUbRdWxOY2qHsalO8dk3AsW113N2Bk4S996v0LPWwG3immI27AK1/x9ez9GlTvOtM4wGTv7mkmTQpmkTmqn/iEOTpqSrs9tTPiUJzVjTdyRLYE0B9spaYb2CS+UYlRTn1xllIPS9vPOvbYdAcCUhNo6K+YZo3FWHwMA9HrRJqYTiysQa+MZq0KyqBRmZzNraesFTsoejH8u1ERBl63dDlANdRMgtfe9QijYylONN4cuvnds7OJkPyacGpauHxqjMPWJX5gpxN2YeB+m9wVR1qmf2Noe61qj8YlpeKpBFV3XjmQTu729u628ZIEXz90PADizs426tchOcmDikzc0lCSDQpIsCNCzz8HscvprTcBuq3VFR+v2vdZdyjjoHvVMSHyjtI1aV+lgG54OC7xyzWm1Os4Y6DokfgPQsZq2VtaUVOeAg7WuSV5rsKCOxXVn6jRmOC0wLX2d6jJzCZYAUKlBZbzHWvtptdPHt+vTAIAX1IOhVqfODNYH3opaVCiyJtzfkzoPr4xMklrOu5nFlYGz+64VNQa+JvBGrzxQ675jHwjP1aTKMfXPgyl1V9/aPk/qflKaAA9JkjAg9DVnNvZhR+uSesfcrRWqMob22kLEqNpndJR1Pytc0wSNAYCmJlTktO615iTOKJekJu81obZsVxs4aF1jY53V2Rwc7b6KuKt1o0Trpt4GXLvaoJ3kQembNm3bTHRI7PTg+l7QOkWMvam7hlmta8+d2tZL4o52p1rXandVvbXWIdE61SA+23ex5gmCINwOZMZUEARBEARBEARBWCnyxVQQBEEQBEEQBEFYKcu18jaAPuNsNZNBESw5uu7aEkOywdSupqKtt+MSUwg2mQ6WOtu11iiyDNTe6qgosQpn6a7BDqUosVWNFHaGzs70tTPboFqB8+Tk7df8zIL8etIM1Vqu0uCJg80sLZlkGwpWMtQq2ITA1GmXtsamSjJSkkG0n1Zx+2wng5lGu1JrJzIW2GNnVdtrQ2vLNKW2soaQOLYw1c6uNk3sh+dm+4Hi/51+SKyI6q1sTbZ7nEBq5apUvHfSuogc+1ZZb+FuMzgqwLbZZLPkOutoIdMlgqVcNXFfGhHIZ3lsShVceZPUYkaYuXc5rgeuX2ZgkgwRhcyrhoK1ztnt2nvY7ZM+Dx2rm1+fXdM4e8bZMd8sttHrx6yiNqk/alvrvEnqUKb9f0iZtmAjb6hjDWzvPV1Fm2WaUTON+W60t5EB1Ou+Tma/iM/ijAU7vXdb0lcYOs9P+oylQ4qGOvvfiNYZAtjbQzXHfXmkcGXfad2V1467rK1tXV/iuVpHyfMXA+pqXWvt5EYFay4aArXZh5MakEjiU0l2bKd1rd0zfmbwNQ3jHyameBxrgX2vdftoLe9+n1mtS2y0Ha1LXxk56DNndpiXD1g9k2E2nJddH82S3iusdHzmlb8GdNtLsV+fWPvr9rAZx+3q+KpHx25v4jmzEYHajL7TqHVm9trnaH18wOdsny4bpxUAoKcU+rbNaNt5TlK98NqSXctw5nVX9/KN4jjyItZmbrXOGBVe8Qifse0xW82do3XMAJob0DrEOIlnlo84RPQhAL8EQAP4d8z8C4dt/8SfHeMTX/g9AMBX99+Di//taQDA7/zSB/A7L7hs4VxVN1FXcQcA0MMO3ovXwtqDc1GvkKNWL3IV8aTn5Lau6KL73mCP2vb4C+53M+0xb5+DYp1dt8g2twkuy7fe6G2IzJgKgiAIgiDc4xCRBvArAD4M4CkAnyCip1YblSAI9xLyxVQQBEEQBEF4H4AXmfllZq4AfB7AR1cckyAI9xC3ZOW9UcuHMkD/is/a1yf4Gt1QVWLxS21IWbQrdextQMxUmGzfDW7G6tTSsZjFfZVBsOEQA+w9PcTRJmZzgmlrorO3eAY/WLKoZixNaVgHON3mbTOnfjpYx/1VHa9P1YAuvS2v4rD/dExo1g62iaXtbrW3vAJA0t7UJMs22SaJMV3X6Y85Fty07Smx7HJigyXrYrruWHOOk8aRWr/JuPUd66tftnnS3g2g2+yUU9eG7fr2XjU5YAYukGqT4nUnNtVwHf4aOB3+Se9bXL8+tcOpylmKAReXauOx3fsytSmnx6pHhGriskfaLANTEc/dnoMB3T6Hic20a8mbEyuSZ8a2zwOgymS5jvchJZY8qxGel3n3+VHiZrSu8FpnCoIOFvu4TXoPsE5+ntG6VAMPDu5gW2e3o+L6uVqX2Dhtj2B6yfWkdnbuPmdztS6xnYZ1fPA211lj0+ekfZaS5zDVulQDyxGhXo8BBQ1I7JSsXPzB0aziPZgeCzP22oOuZx7XWTYTrTtID5C26UGvPqCrW6mV90AtPeAzcVa3g7YkWqdqQBm3bHrxcyPVuu491r2GWUu6O+hMHAe8nuH608dTOt0I55r5nDro2ajHhGrsM+XqGa3z2+hEM2ezmh8U0+zPc7Wufbabg7Uu7avO5+TR5CEAZ5Kf3wDw51cUiyAI9yA3PWMqlg9BEO4FROsEQRAiRPQzRPQcET136crbYGRREIS3DbcyYxosHwBARK3l44V5O5AF8n039JmN4whpNuUwYmmzODLb9AncRpiMdrqD+dVzZuo6CQcsd2dNPVZTGB1OZ8vIcpxFs3Hk0+RAvRET0KSj62QZVlOIKYzGJzNgZJNrmEle0Rklb2dAk1mKcE1Ijo3uCHc2ZWRjX19uakCNX676KLevnxVJk0rYHGgGBOtnSVjFUWE94c4sbjpyns7apdcZRppt2kbdke9wXaY7+p+2Sztjms6ez86wtSPWs+0eft8AJo99zZTOgMb1qgLyiW/LiYWetu1nQz80axr1hvbnpdDn6T2Tzn6oJpn1TWcTNB2YLCptF1Uz8jF8DBxmiMi6kXnVuJ9tRi7RDdwMT3vuaa3jgS0dOFufjuxTkzwnyazLvNldVhRnNjjOBmYlhxkYXdo4i2AZ5Nva5go2S5JLHW1uXOsM0PNaxyOnd4CfmUq0rr1fTYHQHm42vL1XY2KjQ7UuzAxy5/kO50q0TtVxhiyd6SfTnS2rDtM6H6vNui6LEJNFJ1FT59lNZtjCDJbpXlNso7heJ7PvTuuMXzbhXOPTBcpj7c7xOU+vs3U9NP14rlTr0n1SLeo4FA5KYJRsk37+dGY0mcMMdap1qWbYLB6/o3Ucr2fW3RJcFsbt315PmmTMZvF4unRtCAD5yEJP3S90bQF/DzQbOaotr3XVwTOmxN2+DTODKjmvJlCSTCTMQndmTDk8I04/fEKtpvv5zTrpB/I6CmDSxE4hkzg/khn91GWiEq1L22h2+3g9h2hdcCkdrHWsKTwvzeDIvz11FsA7k58f9us6MPNnAXwWAJ754f4Ry/4jCMLbmVtRyYMsHw/NbpSOrNXl6BZOJwiCsBJE6wRBuBf4GoD3EtFjRNQD8HEAz644JkEQ7iHu+PAdM3+WmZ9h5mfyYv1On04QBGEliNYJgvB2hpkbAD8L4HcBfAfAF5n526uNSmHBpngAAAbuSURBVBCEewnim6x5RER/AcBnmPnH/c8/DwDM/M8O2ecSgBGAyzd10jvHKUhMiyAxLYbEtBhtTO9i5vtWHcw8ROvuOBLTYkhMi3HUYzrSenejiNbdEBLTYkhMi3HUY7oprbuVL6YZgO8D+CDcOwhfA/A33mp0jYieY+ZnbuqkdwiJaTEkpsWQmBbjKMZ0EKJ1dxaJaTEkpsWQmJbPUbw+iWkxJKbFkJgW43bEdNPJj5i5IaLW8qEBfE4sH4Ig3G2I1gmCIAiCINx5bqmOKTN/CcCXblMsgiAIRxLROkEQBEEQhDvLKnKXf3YF53wrJKbFkJgWQ2JajKMY0+3kKF6fxLQYEtNiSEyLcRRjup0cxeuTmBZDYloMiWkxbjmmm37HVBAEQRAEQRAEQRBuB0e+2rMgCIIgCIIgCIJwd7O0L6ZE9CEi+h4RvUhEn1rWeWdieCcR/S8ieoGIvk1Ef9+v/wwRnSWi5/2/jyw5rleJ6Jv+3M/5dSeI6PeI6Af+/+NLjOfJpC2eJ6I9Ivq5VbQTEX2OiC4S0beSdQe2DTl+2d9jf0JETy8xpn9BRN/15/0tItr26x8loknSZr+6xJjm9hcR/bxvp+8R0Y8vMaYvJPG8SkTP+/VLaadlIFp3aFyidfNjEa27+ZhE61aAaN2hcYnWzY9FtO7mY7r7tY6Z7/g/uEyWLwF4N4AegG8AeGoZ556J4zSAp/3yJlwJiKcAfAbAP1p2PElcrwI4NbPunwP4lF/+FIBfXFFsGsB5AO9aRTsB+ACApwF8663aBsBHAPx3AATg/QD+cIkx/SUAmV/+xSSmR9PtltxOB/aXv+e/AaAA8Jh/NvUyYpr5/b8E8E+W2U5L6AfRusPjEq2bf37RupuPSbRuyf9E694yLtG6+ecXrbv5mO56rVvWjOn7ALzIzC8zcwXg8wA+uqRzB5j5HDN/3S/vA/gOgIeWHceCfBTAr/nlXwPwkyuK44MAXmLm11Zxcmb+AwBXZ1bPa5uPAviP7PgqgG0iOr2MmJj5y8zc+B+/CuDh233eG43pED4K4PPMXDLzKwBehHtGlxYTERGAnwLwG7f7vCtGtO7GEa2DaN2txHQIonV3DtG6G0e0DqJ1txLTIdw1WresL6YPATiT/PwGViwcRPQogB8B8Id+1c/66frPLdNe4WEAXyaiPyKin/HrHmDmc375PIAHlhxTy8fRvclW2U4t89rmqNxnfwduhK/lMSL6YyL6fSL6sSXHclB/HYV2+jEAF5j5B8m6VbbT7eIotG0H0bqFEa27cUTr3hrRuiUhWrcwonU3jmjdW3NbtO6eTH5ERBsA/guAn2PmPQD/BsDjAP4cgHNwU9HL5EeZ+WkAHwbw94joA+kv2c2JLz19MhH1APwEgN/0q1bdTtexqraZBxF9GkAD4Nf9qnMAHmHmHwHwDwD8ZyLaWlI4R66/Ej6B7gfjKtvprkW0bjFE624c0bqFEa1bAqJ1iyFad+OI1i3MbdG6ZX0xPQvgncnPD/t1S4eIcjjx+nVm/q8AwMwXmNkwswXwb3EHpr8Pg5nP+v8vAvgtf/4LrV3B/39xmTF5Pgzg68x8wce30nZKmNc2K73PiOhvA/jLAP6mF1Z4W8UVv/xHcL7/J5YRzyH9tep2ygB8DMAXklhX1k63GdG6QxCtu2FE6xZAtG4liNYdgmjdDSNatwD3gtYt64vp1wC8l4ge86M1Hwfw7JLOHfD+538P4DvM/K+S9alf/a8C+NbsvncwpnUi2myX4V62/hZc+3zSb/ZJAL+9rJgSOqMfq2ynGea1zbMA/hY53g9gN7GG3FGI6EMA/jGAn2DmcbL+PiLSfvndAN4L4OUlxTSvv54F8HEiKojoMR/T/1tGTJ6/COC7zPxGu2KV7XSbEa2bH5No3Y0jWrdYTKJ1y0e0bn5MonU3jmjdYjHd/VrHdzirVPsPLrPW9+G+MX96WeedieFH4ewBfwLgef/vIwD+E4Bv+vXPAji9xJjeDZdJ6xsAvt22DYCTAP4ngB8A+B8ATiy5rdYBXAFwLFm39HaCE9BzAGo4z/xPz2sbuKxtv+LvsW8CeGaJMb0I5+9v76tf9dv+Nd+vzwP4OoC/ssSY5vYXgE/7dvoegA8vKya//j8A+Lsz2y6lnZbxT7RubkyidYfHIVp38zGJ1q3gn2jd3JhE6w6PQ7Tu5mO667WO/M6CIAiCIAiCIAiCsBLuyeRHgiAIgiAIgiAIwtFBvpgKgiAIgiAIgiAIK0W+mAqCIAiCIAiCIAgrRb6YCoIgCIIgCIIgCCtFvpgKgiAIgiAIgiAIK0W+mAqCIAiCIAiCIAgrRb6YCoIgCIIgCIIgCCtFvpgKgiAIgiAIgiAIK+X/A55yOiuBCW9UAAAAAElFTkSuQmCC\n", + "text/plain": [ + "

" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence)\n", + "plot_data((mel_outputs.float().data.cpu().numpy()[0],\n", + " mel_outputs_postnet.float().data.cpu().numpy()[0],\n", + " alignments.float().data.cpu().numpy()[0].T))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Synthesize audio from spectrogram using WaveGlow" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "with torch.no_grad():\n", + " audio = waveglow.infer(mel_outputs_postnet, sigma=0.666)\n", + "ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### (Optional) Remove WaveGlow bias" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "audio_denoised = denoiser(audio, strength=0.01)[:, 0]\n", + "ipd.Audio(audio_denoised.cpu().numpy(), rate=hparams.sampling_rate) " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/models/tacotron2/layers.py b/models/tacotron2/layers.py new file mode 100644 index 0000000000..615a64a439 --- /dev/null +++ b/models/tacotron2/layers.py @@ -0,0 +1,80 @@ +import torch +from librosa.filters import mel as librosa_mel_fn +from audio_processing import dynamic_range_compression +from audio_processing import dynamic_range_decompression +from stft import STFT + + +class LinearNorm(torch.nn.Module): + def __init__(self, in_dim, out_dim, bias=True, w_init_gain='linear'): + super(LinearNorm, self).__init__() + self.linear_layer = torch.nn.Linear(in_dim, out_dim, bias=bias) + + torch.nn.init.xavier_uniform_( + self.linear_layer.weight, + gain=torch.nn.init.calculate_gain(w_init_gain)) + + def forward(self, x): + return self.linear_layer(x) + + +class ConvNorm(torch.nn.Module): + def __init__(self, in_channels, out_channels, kernel_size=1, stride=1, + padding=None, dilation=1, bias=True, w_init_gain='linear'): + super(ConvNorm, self).__init__() + if padding is None: + assert(kernel_size % 2 == 1) + padding = int(dilation * (kernel_size - 1) / 2) + + self.conv = torch.nn.Conv1d(in_channels, out_channels, + kernel_size=kernel_size, stride=stride, + padding=padding, dilation=dilation, + bias=bias) + + torch.nn.init.xavier_uniform_( + self.conv.weight, gain=torch.nn.init.calculate_gain(w_init_gain)) + + def forward(self, signal): + conv_signal = self.conv(signal) + return conv_signal + + +class TacotronSTFT(torch.nn.Module): + def __init__(self, filter_length=1024, hop_length=256, win_length=1024, + n_mel_channels=80, sampling_rate=22050, mel_fmin=0.0, + mel_fmax=8000.0): + super(TacotronSTFT, self).__init__() + self.n_mel_channels = n_mel_channels + self.sampling_rate = sampling_rate + self.stft_fn = STFT(filter_length, hop_length, win_length) + mel_basis = librosa_mel_fn( + sampling_rate, filter_length, n_mel_channels, mel_fmin, mel_fmax) + mel_basis = torch.from_numpy(mel_basis).float() + self.register_buffer('mel_basis', mel_basis) + + def spectral_normalize(self, magnitudes): + output = dynamic_range_compression(magnitudes) + return output + + def spectral_de_normalize(self, magnitudes): + output = dynamic_range_decompression(magnitudes) + return output + + def mel_spectrogram(self, y): + """Computes mel-spectrograms from a batch of waves + PARAMS + ------ + y: Variable(torch.FloatTensor) with shape (B, T) in range [-1, 1] + + RETURNS + ------- + mel_output: torch.FloatTensor of shape (B, n_mel_channels, T) + """ + assert(torch.min(y.data) >= -1) + assert(torch.max(y.data) <= 1) + + magnitudes, phases = self.stft_fn.transform(y) + magnitudes = magnitudes.data + mel_output = torch.matmul(self.mel_basis, magnitudes) + mel_output = self.spectral_normalize(mel_output) + return mel_output diff --git a/models/tacotron2/logger.py b/models/tacotron2/logger.py new file mode 100644 index 0000000000..ad327383a2 --- /dev/null +++ b/models/tacotron2/logger.py @@ -0,0 +1,48 @@ +import random +import torch +from torch.utils.tensorboard import SummaryWriter +from plotting_utils import plot_alignment_to_numpy, plot_spectrogram_to_numpy +from plotting_utils import plot_gate_outputs_to_numpy + + +class Tacotron2Logger(SummaryWriter): + def __init__(self, logdir): + super(Tacotron2Logger, self).__init__(logdir) + + def log_training(self, reduced_loss, grad_norm, learning_rate, duration, + iteration): + self.add_scalar("training.loss", reduced_loss, iteration) + self.add_scalar("grad.norm", grad_norm, iteration) + self.add_scalar("learning.rate", learning_rate, iteration) + self.add_scalar("duration", duration, iteration) + + def log_validation(self, reduced_loss, model, y, y_pred, iteration): + self.add_scalar("validation.loss", reduced_loss, iteration) + _, mel_outputs, gate_outputs, alignments = y_pred + mel_targets, gate_targets = y + + # plot distribution of parameters + for tag, value in model.named_parameters(): + tag = tag.replace('.', '/') + self.add_histogram(tag, value.data.cpu().numpy(), iteration) + + # plot alignment, mel target and predicted, gate target and predicted + idx = random.randint(0, alignments.size(0) - 1) + self.add_image( + "alignment", + plot_alignment_to_numpy(alignments[idx].data.cpu().numpy().T), + iteration, dataformats='HWC') + self.add_image( + "mel_target", + plot_spectrogram_to_numpy(mel_targets[idx].data.cpu().numpy()), + iteration, dataformats='HWC') + self.add_image( + "mel_predicted", + plot_spectrogram_to_numpy(mel_outputs[idx].data.cpu().numpy()), + iteration, dataformats='HWC') + self.add_image( + "gate", + plot_gate_outputs_to_numpy( + gate_targets[idx].data.cpu().numpy(), + torch.sigmoid(gate_outputs[idx]).data.cpu().numpy()), + iteration, dataformats='HWC') diff --git a/models/tacotron2/loss_function.py b/models/tacotron2/loss_function.py new file mode 100644 index 0000000000..99cae952bb --- /dev/null +++ b/models/tacotron2/loss_function.py @@ -0,0 +1,19 @@ +from torch import nn + + +class Tacotron2Loss(nn.Module): + def __init__(self): + super(Tacotron2Loss, self).__init__() + + def forward(self, model_output, targets): + mel_target, gate_target = targets[0], targets[1] + mel_target.requires_grad = False + gate_target.requires_grad = False + gate_target = gate_target.view(-1, 1) + + mel_out, mel_out_postnet, gate_out, _ = model_output + gate_out = gate_out.view(-1, 1) + mel_loss = nn.MSELoss()(mel_out, mel_target) + \ + nn.MSELoss()(mel_out_postnet, mel_target) + gate_loss = nn.BCEWithLogitsLoss()(gate_out, gate_target) + return mel_loss + gate_loss diff --git a/models/tacotron2/loss_scaler.py b/models/tacotron2/loss_scaler.py new file mode 100644 index 0000000000..0662a60e2a --- /dev/null +++ b/models/tacotron2/loss_scaler.py @@ -0,0 +1,131 @@ +import torch + +class LossScaler: + + def __init__(self, scale=1): + self.cur_scale = scale + + # `params` is a list / generator of torch.Variable + def has_overflow(self, params): + return False + + # `x` is a torch.Tensor + def _has_inf_or_nan(x): + return False + + # `overflow` is boolean indicating whether we overflowed in gradient + def update_scale(self, overflow): + pass + + @property + def loss_scale(self): + return self.cur_scale + + def scale_gradient(self, module, grad_in, grad_out): + return tuple(self.loss_scale * g for g in grad_in) + + def backward(self, loss): + scaled_loss = loss*self.loss_scale + scaled_loss.backward() + +class DynamicLossScaler: + + def __init__(self, + init_scale=2**32, + scale_factor=2., + scale_window=1000): + self.cur_scale = init_scale + self.cur_iter = 0 + self.last_overflow_iter = -1 + self.scale_factor = scale_factor + self.scale_window = scale_window + + # `params` is a list / generator of torch.Variable + def has_overflow(self, params): +# return False + for p in params: + if p.grad is not None and DynamicLossScaler._has_inf_or_nan(p.grad.data): + return True + + return False + + # `x` is a torch.Tensor + def _has_inf_or_nan(x): + cpu_sum = float(x.float().sum()) + if cpu_sum == float('inf') or cpu_sum == -float('inf') or cpu_sum != cpu_sum: + return True + return False + + # `overflow` is boolean indicating whether we overflowed in gradient + def update_scale(self, overflow): + if overflow: + #self.cur_scale /= self.scale_factor + self.cur_scale = max(self.cur_scale/self.scale_factor, 1) + self.last_overflow_iter = self.cur_iter + else: + if (self.cur_iter - self.last_overflow_iter) % self.scale_window == 0: + self.cur_scale *= self.scale_factor +# self.cur_scale = 1 + self.cur_iter += 1 + + @property + def loss_scale(self): + return self.cur_scale + + def scale_gradient(self, module, grad_in, grad_out): + return tuple(self.loss_scale * g for g in grad_in) + + def backward(self, loss): + scaled_loss = loss*self.loss_scale + scaled_loss.backward() + +############################################################## +# Example usage below here -- assuming it's in a separate file +############################################################## +if __name__ == "__main__": + import torch + from torch.autograd import Variable + from dynamic_loss_scaler import DynamicLossScaler + + # N is batch size; D_in is input dimension; + # H is hidden dimension; D_out is output dimension. + N, D_in, H, D_out = 64, 1000, 100, 10 + + # Create random Tensors to hold inputs and outputs, and wrap them in Variables. + x = Variable(torch.randn(N, D_in), requires_grad=False) + y = Variable(torch.randn(N, D_out), requires_grad=False) + + w1 = Variable(torch.randn(D_in, H), requires_grad=True) + w2 = Variable(torch.randn(H, D_out), requires_grad=True) + parameters = [w1, w2] + + learning_rate = 1e-6 + optimizer = torch.optim.SGD(parameters, lr=learning_rate) + loss_scaler = DynamicLossScaler() + + for t in range(500): + y_pred = x.mm(w1).clamp(min=0).mm(w2) + loss = (y_pred - y).pow(2).sum() * loss_scaler.loss_scale + print('Iter {} loss scale: {}'.format(t, loss_scaler.loss_scale)) + print('Iter {} scaled loss: {}'.format(t, loss.data[0])) + print('Iter {} unscaled loss: {}'.format(t, loss.data[0] / loss_scaler.loss_scale)) + + # Run backprop + optimizer.zero_grad() + loss.backward() + + # Check for overflow + has_overflow = DynamicLossScaler.has_overflow(parameters) + + # If no overflow, unscale grad and update as usual + if not has_overflow: + for param in parameters: + param.grad.data.mul_(1. / loss_scaler.loss_scale) + optimizer.step() + # Otherwise, don't do anything -- ie, skip iteration + else: + print('OVERFLOW!') + + # Update loss scale for next iteration + loss_scaler.update_scale(has_overflow) + diff --git a/models/tacotron2/model.py b/models/tacotron2/model.py new file mode 100644 index 0000000000..ec0e9cea43 --- /dev/null +++ b/models/tacotron2/model.py @@ -0,0 +1,529 @@ +from math import sqrt +import torch +from torch.autograd import Variable +from torch import nn +from torch.nn import functional as F +from layers import ConvNorm, LinearNorm +from utils import to_gpu, get_mask_from_lengths + + +class LocationLayer(nn.Module): + def __init__(self, attention_n_filters, attention_kernel_size, + attention_dim): + super(LocationLayer, self).__init__() + padding = int((attention_kernel_size - 1) / 2) + self.location_conv = ConvNorm(2, attention_n_filters, + kernel_size=attention_kernel_size, + padding=padding, bias=False, stride=1, + dilation=1) + self.location_dense = LinearNorm(attention_n_filters, attention_dim, + bias=False, w_init_gain='tanh') + + def forward(self, attention_weights_cat): + processed_attention = self.location_conv(attention_weights_cat) + processed_attention = processed_attention.transpose(1, 2) + processed_attention = self.location_dense(processed_attention) + return processed_attention + + +class Attention(nn.Module): + def __init__(self, attention_rnn_dim, embedding_dim, attention_dim, + attention_location_n_filters, attention_location_kernel_size): + super(Attention, self).__init__() + self.query_layer = LinearNorm(attention_rnn_dim, attention_dim, + bias=False, w_init_gain='tanh') + self.memory_layer = LinearNorm(embedding_dim, attention_dim, bias=False, + w_init_gain='tanh') + self.v = LinearNorm(attention_dim, 1, bias=False) + self.location_layer = LocationLayer(attention_location_n_filters, + attention_location_kernel_size, + attention_dim) + self.score_mask_value = -float("inf") + + def get_alignment_energies(self, query, processed_memory, + attention_weights_cat): + """ + PARAMS + ------ + query: decoder output (batch, n_mel_channels * n_frames_per_step) + processed_memory: processed encoder outputs (B, T_in, attention_dim) + attention_weights_cat: cumulative and prev. att weights (B, 2, max_time) + + RETURNS + ------- + alignment (batch, max_time) + """ + + processed_query = self.query_layer(query.unsqueeze(1)) + processed_attention_weights = self.location_layer(attention_weights_cat) + energies = self.v(torch.tanh( + processed_query + processed_attention_weights + processed_memory)) + + energies = energies.squeeze(-1) + return energies + + def forward(self, attention_hidden_state, memory, processed_memory, + attention_weights_cat, mask): + """ + PARAMS + ------ + attention_hidden_state: attention rnn last output + memory: encoder outputs + processed_memory: processed encoder outputs + attention_weights_cat: previous and cummulative attention weights + mask: binary mask for padded data + """ + alignment = self.get_alignment_energies( + attention_hidden_state, processed_memory, attention_weights_cat) + + if mask is not None: + alignment.data.masked_fill_(mask, self.score_mask_value) + + attention_weights = F.softmax(alignment, dim=1) + attention_context = torch.bmm(attention_weights.unsqueeze(1), memory) + attention_context = attention_context.squeeze(1) + + return attention_context, attention_weights + + +class Prenet(nn.Module): + def __init__(self, in_dim, sizes): + super(Prenet, self).__init__() + in_sizes = [in_dim] + sizes[:-1] + self.layers = nn.ModuleList( + [LinearNorm(in_size, out_size, bias=False) + for (in_size, out_size) in zip(in_sizes, sizes)]) + + def forward(self, x): + for linear in self.layers: + x = F.dropout(F.relu(linear(x)), p=0.5, training=True) + return x + + +class Postnet(nn.Module): + """Postnet + - Five 1-d convolution with 512 channels and kernel size 5 + """ + + def __init__(self, hparams): + super(Postnet, self).__init__() + self.convolutions = nn.ModuleList() + + self.convolutions.append( + nn.Sequential( + ConvNorm(hparams.n_mel_channels, hparams.postnet_embedding_dim, + kernel_size=hparams.postnet_kernel_size, stride=1, + padding=int((hparams.postnet_kernel_size - 1) / 2), + dilation=1, w_init_gain='tanh'), + nn.BatchNorm1d(hparams.postnet_embedding_dim)) + ) + + for i in range(1, hparams.postnet_n_convolutions - 1): + self.convolutions.append( + nn.Sequential( + ConvNorm(hparams.postnet_embedding_dim, + hparams.postnet_embedding_dim, + kernel_size=hparams.postnet_kernel_size, stride=1, + padding=int((hparams.postnet_kernel_size - 1) / 2), + dilation=1, w_init_gain='tanh'), + nn.BatchNorm1d(hparams.postnet_embedding_dim)) + ) + + self.convolutions.append( + nn.Sequential( + ConvNorm(hparams.postnet_embedding_dim, hparams.n_mel_channels, + kernel_size=hparams.postnet_kernel_size, stride=1, + padding=int((hparams.postnet_kernel_size - 1) / 2), + dilation=1, w_init_gain='linear'), + nn.BatchNorm1d(hparams.n_mel_channels)) + ) + + def forward(self, x): + for i in range(len(self.convolutions) - 1): + x = F.dropout(torch.tanh(self.convolutions[i](x)), 0.5, self.training) + x = F.dropout(self.convolutions[-1](x), 0.5, self.training) + + return x + + +class Encoder(nn.Module): + """Encoder module: + - Three 1-d convolution banks + - Bidirectional LSTM + """ + def __init__(self, hparams): + super(Encoder, self).__init__() + + convolutions = [] + for _ in range(hparams.encoder_n_convolutions): + conv_layer = nn.Sequential( + ConvNorm(hparams.encoder_embedding_dim, + hparams.encoder_embedding_dim, + kernel_size=hparams.encoder_kernel_size, stride=1, + padding=int((hparams.encoder_kernel_size - 1) / 2), + dilation=1, w_init_gain='relu'), + nn.BatchNorm1d(hparams.encoder_embedding_dim)) + convolutions.append(conv_layer) + self.convolutions = nn.ModuleList(convolutions) + + self.lstm = nn.LSTM(hparams.encoder_embedding_dim, + int(hparams.encoder_embedding_dim / 2), 1, + batch_first=True, bidirectional=True) + + def forward(self, x, input_lengths): + for conv in self.convolutions: + x = F.dropout(F.relu(conv(x)), 0.5, self.training) + + x = x.transpose(1, 2) + + # pytorch tensor are not reversible, hence the conversion + input_lengths = input_lengths.cpu().numpy() + x = nn.utils.rnn.pack_padded_sequence( + x, input_lengths, batch_first=True) + + self.lstm.flatten_parameters() + outputs, _ = self.lstm(x) + + outputs, _ = nn.utils.rnn.pad_packed_sequence( + outputs, batch_first=True) + + return outputs + + def inference(self, x): + for conv in self.convolutions: + x = F.dropout(F.relu(conv(x)), 0.5, self.training) + + x = x.transpose(1, 2) + + self.lstm.flatten_parameters() + outputs, _ = self.lstm(x) + + return outputs + + +class Decoder(nn.Module): + def __init__(self, hparams): + super(Decoder, self).__init__() + self.n_mel_channels = hparams.n_mel_channels + self.n_frames_per_step = hparams.n_frames_per_step + self.encoder_embedding_dim = hparams.encoder_embedding_dim + self.attention_rnn_dim = hparams.attention_rnn_dim + self.decoder_rnn_dim = hparams.decoder_rnn_dim + self.prenet_dim = hparams.prenet_dim + self.max_decoder_steps = hparams.max_decoder_steps + self.gate_threshold = hparams.gate_threshold + self.p_attention_dropout = hparams.p_attention_dropout + self.p_decoder_dropout = hparams.p_decoder_dropout + + self.prenet = Prenet( + hparams.n_mel_channels * hparams.n_frames_per_step, + [hparams.prenet_dim, hparams.prenet_dim]) + + self.attention_rnn = nn.LSTMCell( + hparams.prenet_dim + hparams.encoder_embedding_dim, + hparams.attention_rnn_dim) + + self.attention_layer = Attention( + hparams.attention_rnn_dim, hparams.encoder_embedding_dim, + hparams.attention_dim, hparams.attention_location_n_filters, + hparams.attention_location_kernel_size) + + self.decoder_rnn = nn.LSTMCell( + hparams.attention_rnn_dim + hparams.encoder_embedding_dim, + hparams.decoder_rnn_dim, 1) + + self.linear_projection = LinearNorm( + hparams.decoder_rnn_dim + hparams.encoder_embedding_dim, + hparams.n_mel_channels * hparams.n_frames_per_step) + + self.gate_layer = LinearNorm( + hparams.decoder_rnn_dim + hparams.encoder_embedding_dim, 1, + bias=True, w_init_gain='sigmoid') + + def get_go_frame(self, memory): + """ Gets all zeros frames to use as first decoder input + PARAMS + ------ + memory: decoder outputs + + RETURNS + ------- + decoder_input: all zeros frames + """ + B = memory.size(0) + decoder_input = Variable(memory.data.new( + B, self.n_mel_channels * self.n_frames_per_step).zero_()) + return decoder_input + + def initialize_decoder_states(self, memory, mask): + """ Initializes attention rnn states, decoder rnn states, attention + weights, attention cumulative weights, attention context, stores memory + and stores processed memory + PARAMS + ------ + memory: Encoder outputs + mask: Mask for padded data if training, expects None for inference + """ + B = memory.size(0) + MAX_TIME = memory.size(1) + + self.attention_hidden = Variable(memory.data.new( + B, self.attention_rnn_dim).zero_()) + self.attention_cell = Variable(memory.data.new( + B, self.attention_rnn_dim).zero_()) + + self.decoder_hidden = Variable(memory.data.new( + B, self.decoder_rnn_dim).zero_()) + self.decoder_cell = Variable(memory.data.new( + B, self.decoder_rnn_dim).zero_()) + + self.attention_weights = Variable(memory.data.new( + B, MAX_TIME).zero_()) + self.attention_weights_cum = Variable(memory.data.new( + B, MAX_TIME).zero_()) + self.attention_context = Variable(memory.data.new( + B, self.encoder_embedding_dim).zero_()) + + self.memory = memory + self.processed_memory = self.attention_layer.memory_layer(memory) + self.mask = mask + + def parse_decoder_inputs(self, decoder_inputs): + """ Prepares decoder inputs, i.e. mel outputs + PARAMS + ------ + decoder_inputs: inputs used for teacher-forced training, i.e. mel-specs + + RETURNS + ------- + inputs: processed decoder inputs + + """ + # (B, n_mel_channels, T_out) -> (B, T_out, n_mel_channels) + decoder_inputs = decoder_inputs.transpose(1, 2) + decoder_inputs = decoder_inputs.view( + decoder_inputs.size(0), + int(decoder_inputs.size(1)/self.n_frames_per_step), -1) + # (B, T_out, n_mel_channels) -> (T_out, B, n_mel_channels) + decoder_inputs = decoder_inputs.transpose(0, 1) + return decoder_inputs + + def parse_decoder_outputs(self, mel_outputs, gate_outputs, alignments): + """ Prepares decoder outputs for output + PARAMS + ------ + mel_outputs: + gate_outputs: gate output energies + alignments: + + RETURNS + ------- + mel_outputs: + gate_outpust: gate output energies + alignments: + """ + # (T_out, B) -> (B, T_out) + alignments = torch.stack(alignments).transpose(0, 1) + # (T_out, B) -> (B, T_out) + gate_outputs = torch.stack(gate_outputs).transpose(0, 1) + gate_outputs = gate_outputs.contiguous() + # (T_out, B, n_mel_channels) -> (B, T_out, n_mel_channels) + mel_outputs = torch.stack(mel_outputs).transpose(0, 1).contiguous() + # decouple frames per step + mel_outputs = mel_outputs.view( + mel_outputs.size(0), -1, self.n_mel_channels) + # (B, T_out, n_mel_channels) -> (B, n_mel_channels, T_out) + mel_outputs = mel_outputs.transpose(1, 2) + + return mel_outputs, gate_outputs, alignments + + def decode(self, decoder_input): + """ Decoder step using stored states, attention and memory + PARAMS + ------ + decoder_input: previous mel output + + RETURNS + ------- + mel_output: + gate_output: gate output energies + attention_weights: + """ + cell_input = torch.cat((decoder_input, self.attention_context), -1) + self.attention_hidden, self.attention_cell = self.attention_rnn( + cell_input, (self.attention_hidden, self.attention_cell)) + self.attention_hidden = F.dropout( + self.attention_hidden, self.p_attention_dropout, self.training) + + attention_weights_cat = torch.cat( + (self.attention_weights.unsqueeze(1), + self.attention_weights_cum.unsqueeze(1)), dim=1) + self.attention_context, self.attention_weights = self.attention_layer( + self.attention_hidden, self.memory, self.processed_memory, + attention_weights_cat, self.mask) + + self.attention_weights_cum += self.attention_weights + decoder_input = torch.cat( + (self.attention_hidden, self.attention_context), -1) + self.decoder_hidden, self.decoder_cell = self.decoder_rnn( + decoder_input, (self.decoder_hidden, self.decoder_cell)) + self.decoder_hidden = F.dropout( + self.decoder_hidden, self.p_decoder_dropout, self.training) + + decoder_hidden_attention_context = torch.cat( + (self.decoder_hidden, self.attention_context), dim=1) + decoder_output = self.linear_projection( + decoder_hidden_attention_context) + + gate_prediction = self.gate_layer(decoder_hidden_attention_context) + return decoder_output, gate_prediction, self.attention_weights + + def forward(self, memory, decoder_inputs, memory_lengths): + """ Decoder forward pass for training + PARAMS + ------ + memory: Encoder outputs + decoder_inputs: Decoder inputs for teacher forcing. i.e. mel-specs + memory_lengths: Encoder output lengths for attention masking. + + RETURNS + ------- + mel_outputs: mel outputs from the decoder + gate_outputs: gate outputs from the decoder + alignments: sequence of attention weights from the decoder + """ + + decoder_input = self.get_go_frame(memory).unsqueeze(0) + decoder_inputs = self.parse_decoder_inputs(decoder_inputs) + decoder_inputs = torch.cat((decoder_input, decoder_inputs), dim=0) + decoder_inputs = self.prenet(decoder_inputs) + + self.initialize_decoder_states( + memory, mask=~get_mask_from_lengths(memory_lengths)) + + mel_outputs, gate_outputs, alignments = [], [], [] + while len(mel_outputs) < decoder_inputs.size(0) - 1: + decoder_input = decoder_inputs[len(mel_outputs)] + mel_output, gate_output, attention_weights = self.decode( + decoder_input) + mel_outputs += [mel_output.squeeze(1)] + gate_outputs += [gate_output.squeeze(1)] + alignments += [attention_weights] + + mel_outputs, gate_outputs, alignments = self.parse_decoder_outputs( + mel_outputs, gate_outputs, alignments) + + return mel_outputs, gate_outputs, alignments + + def inference(self, memory): + """ Decoder inference + PARAMS + ------ + memory: Encoder outputs + + RETURNS + ------- + mel_outputs: mel outputs from the decoder + gate_outputs: gate outputs from the decoder + alignments: sequence of attention weights from the decoder + """ + decoder_input = self.get_go_frame(memory) + + self.initialize_decoder_states(memory, mask=None) + + mel_outputs, gate_outputs, alignments = [], [], [] + while True: + decoder_input = self.prenet(decoder_input) + mel_output, gate_output, alignment = self.decode(decoder_input) + + mel_outputs += [mel_output.squeeze(1)] + gate_outputs += [gate_output] + alignments += [alignment] + + if torch.sigmoid(gate_output.data) > self.gate_threshold: + break + elif len(mel_outputs) == self.max_decoder_steps: + print("Warning! Reached max decoder steps") + break + + decoder_input = mel_output + + mel_outputs, gate_outputs, alignments = self.parse_decoder_outputs( + mel_outputs, gate_outputs, alignments) + + return mel_outputs, gate_outputs, alignments + + +class Tacotron2(nn.Module): + def __init__(self, hparams): + super(Tacotron2, self).__init__() + self.mask_padding = hparams.mask_padding + self.fp16_run = hparams.fp16_run + self.n_mel_channels = hparams.n_mel_channels + self.n_frames_per_step = hparams.n_frames_per_step + self.embedding = nn.Embedding( + hparams.n_symbols, hparams.symbols_embedding_dim) + std = sqrt(2.0 / (hparams.n_symbols + hparams.symbols_embedding_dim)) + val = sqrt(3.0) * std # uniform bounds for std + self.embedding.weight.data.uniform_(-val, val) + self.encoder = Encoder(hparams) + self.decoder = Decoder(hparams) + self.postnet = Postnet(hparams) + + def parse_batch(self, batch): + text_padded, input_lengths, mel_padded, gate_padded, \ + output_lengths = batch + text_padded = to_gpu(text_padded).long() + input_lengths = to_gpu(input_lengths).long() + max_len = torch.max(input_lengths.data).item() + mel_padded = to_gpu(mel_padded).float() + gate_padded = to_gpu(gate_padded).float() + output_lengths = to_gpu(output_lengths).long() + + return ( + (text_padded, input_lengths, mel_padded, max_len, output_lengths), + (mel_padded, gate_padded)) + + def parse_output(self, outputs, output_lengths=None): + if self.mask_padding and output_lengths is not None: + mask = ~get_mask_from_lengths(output_lengths) + mask = mask.expand(self.n_mel_channels, mask.size(0), mask.size(1)) + mask = mask.permute(1, 0, 2) + + outputs[0].data.masked_fill_(mask, 0.0) + outputs[1].data.masked_fill_(mask, 0.0) + outputs[2].data.masked_fill_(mask[:, 0, :], 1e3) # gate energies + + return outputs + + def forward(self, inputs): + text_inputs, text_lengths, mels, max_len, output_lengths = inputs + text_lengths, output_lengths = text_lengths.data, output_lengths.data + + embedded_inputs = self.embedding(text_inputs).transpose(1, 2) + + encoder_outputs = self.encoder(embedded_inputs, text_lengths) + + mel_outputs, gate_outputs, alignments = self.decoder( + encoder_outputs, mels, memory_lengths=text_lengths) + + mel_outputs_postnet = self.postnet(mel_outputs) + mel_outputs_postnet = mel_outputs + mel_outputs_postnet + + return self.parse_output( + [mel_outputs, mel_outputs_postnet, gate_outputs, alignments], + output_lengths) + + def inference(self, inputs): + embedded_inputs = self.embedding(inputs).transpose(1, 2) + encoder_outputs = self.encoder.inference(embedded_inputs) + mel_outputs, gate_outputs, alignments = self.decoder.inference( + encoder_outputs) + + mel_outputs_postnet = self.postnet(mel_outputs) + mel_outputs_postnet = mel_outputs + mel_outputs_postnet + + outputs = self.parse_output( + [mel_outputs, mel_outputs_postnet, gate_outputs, alignments]) + + return outputs diff --git a/models/tacotron2/multiproc.py b/models/tacotron2/multiproc.py new file mode 100644 index 0000000000..060ff937ac --- /dev/null +++ b/models/tacotron2/multiproc.py @@ -0,0 +1,23 @@ +import time +import torch +import sys +import subprocess + +argslist = list(sys.argv)[1:] +num_gpus = torch.cuda.device_count() +argslist.append('--n_gpus={}'.format(num_gpus)) +workers = [] +job_id = time.strftime("%Y_%m_%d-%H%M%S") +argslist.append("--group_name=group_{}".format(job_id)) + +for i in range(num_gpus): + argslist.append('--rank={}'.format(i)) + stdout = None if i == 0 else open("logs/{}_GPU_{}.log".format(job_id, i), + "w") + print(argslist) + p = subprocess.Popen([str(sys.executable)]+argslist, stdout=stdout) + workers.append(p) + argslist = argslist[:-1] + +for p in workers: + p.wait() diff --git a/models/tacotron2/plotting_utils.py b/models/tacotron2/plotting_utils.py new file mode 100644 index 0000000000..ca7e16880e --- /dev/null +++ b/models/tacotron2/plotting_utils.py @@ -0,0 +1,61 @@ +import matplotlib +matplotlib.use("Agg") +import matplotlib.pylab as plt +import numpy as np + + +def save_figure_to_numpy(fig): + # save it to a numpy array. + data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') + data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) + return data + + +def plot_alignment_to_numpy(alignment, info=None): + fig, ax = plt.subplots(figsize=(6, 4)) + im = ax.imshow(alignment, aspect='auto', origin='lower', + interpolation='none') + fig.colorbar(im, ax=ax) + xlabel = 'Decoder timestep' + if info is not None: + xlabel += '\n\n' + info + plt.xlabel(xlabel) + plt.ylabel('Encoder timestep') + plt.tight_layout() + + fig.canvas.draw() + data = save_figure_to_numpy(fig) + plt.close() + return data + + +def plot_spectrogram_to_numpy(spectrogram): + fig, ax = plt.subplots(figsize=(12, 3)) + im = ax.imshow(spectrogram, aspect="auto", origin="lower", + interpolation='none') + plt.colorbar(im, ax=ax) + plt.xlabel("Frames") + plt.ylabel("Channels") + plt.tight_layout() + + fig.canvas.draw() + data = save_figure_to_numpy(fig) + plt.close() + return data + + +def plot_gate_outputs_to_numpy(gate_targets, gate_outputs): + fig, ax = plt.subplots(figsize=(12, 3)) + ax.scatter(range(len(gate_targets)), gate_targets, alpha=0.5, + color='green', marker='+', s=1, label='target') + ax.scatter(range(len(gate_outputs)), gate_outputs, alpha=0.5, + color='red', marker='.', s=1, label='predicted') + + plt.xlabel("Frames (Green target, Red predicted)") + plt.ylabel("Gate State") + plt.tight_layout() + + fig.canvas.draw() + data = save_figure_to_numpy(fig) + plt.close() + return data diff --git a/models/tacotron2/requirements.txt b/models/tacotron2/requirements.txt new file mode 100644 index 0000000000..9b23dca725 --- /dev/null +++ b/models/tacotron2/requirements.txt @@ -0,0 +1,8 @@ +matplotlib==2.1.0 +tensorflow==1.15.2 +numpy==1.13.3 +inflect==0.2.5 +librosa==0.6.0 +scipy==1.0.0 +Unidecode==1.0.22 +pillow diff --git a/models/tacotron2/stft.py b/models/tacotron2/stft.py new file mode 100644 index 0000000000..edfc44ae8b --- /dev/null +++ b/models/tacotron2/stft.py @@ -0,0 +1,141 @@ +""" +BSD 3-Clause License + +Copyright (c) 2017, Prem Seetharaman +All rights reserved. + +* Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +import torch +import numpy as np +import torch.nn.functional as F +from torch.autograd import Variable +from scipy.signal import get_window +from librosa.util import pad_center, tiny +from audio_processing import window_sumsquare + + +class STFT(torch.nn.Module): + """adapted from Prem Seetharaman's https://github.com/pseeth/pytorch-stft""" + def __init__(self, filter_length=800, hop_length=200, win_length=800, + window='hann'): + super(STFT, self).__init__() + self.filter_length = filter_length + self.hop_length = hop_length + self.win_length = win_length + self.window = window + self.forward_transform = None + scale = self.filter_length / self.hop_length + fourier_basis = np.fft.fft(np.eye(self.filter_length)) + + cutoff = int((self.filter_length / 2 + 1)) + fourier_basis = np.vstack([np.real(fourier_basis[:cutoff, :]), + np.imag(fourier_basis[:cutoff, :])]) + + forward_basis = torch.FloatTensor(fourier_basis[:, None, :]) + inverse_basis = torch.FloatTensor( + np.linalg.pinv(scale * fourier_basis).T[:, None, :]) + + if window is not None: + assert(filter_length >= win_length) + # get window and zero center pad it to filter_length + fft_window = get_window(window, win_length, fftbins=True) + fft_window = pad_center(fft_window, filter_length) + fft_window = torch.from_numpy(fft_window).float() + + # window the bases + forward_basis *= fft_window + inverse_basis *= fft_window + + self.register_buffer('forward_basis', forward_basis.float()) + self.register_buffer('inverse_basis', inverse_basis.float()) + + def transform(self, input_data): + num_batches = input_data.size(0) + num_samples = input_data.size(1) + + self.num_samples = num_samples + + # similar to librosa, reflect-pad the input + input_data = input_data.view(num_batches, 1, num_samples) + input_data = F.pad( + input_data.unsqueeze(1), + (int(self.filter_length / 2), int(self.filter_length / 2), 0, 0), + mode='reflect') + input_data = input_data.squeeze(1) + + forward_transform = F.conv1d( + input_data, + Variable(self.forward_basis, requires_grad=False), + stride=self.hop_length, + padding=0) + + cutoff = int((self.filter_length / 2) + 1) + real_part = forward_transform[:, :cutoff, :] + imag_part = forward_transform[:, cutoff:, :] + + magnitude = torch.sqrt(real_part**2 + imag_part**2) + phase = torch.autograd.Variable( + torch.atan2(imag_part.data, real_part.data)) + + return magnitude, phase + + def inverse(self, magnitude, phase): + recombine_magnitude_phase = torch.cat( + [magnitude*torch.cos(phase), magnitude*torch.sin(phase)], dim=1) + + inverse_transform = F.conv_transpose1d( + recombine_magnitude_phase, + Variable(self.inverse_basis, requires_grad=False), + stride=self.hop_length, + padding=0) + + if self.window is not None: + window_sum = window_sumsquare( + self.window, magnitude.size(-1), hop_length=self.hop_length, + win_length=self.win_length, n_fft=self.filter_length, + dtype=np.float32) + # remove modulation effects + approx_nonzero_indices = torch.from_numpy( + np.where(window_sum > tiny(window_sum))[0]) + window_sum = torch.autograd.Variable( + torch.from_numpy(window_sum), requires_grad=False) + window_sum = window_sum.cuda() if magnitude.is_cuda else window_sum + inverse_transform[:, :, approx_nonzero_indices] /= window_sum[approx_nonzero_indices] + + # scale by hop ratio + inverse_transform *= float(self.filter_length) / self.hop_length + + inverse_transform = inverse_transform[:, :, int(self.filter_length/2):] + inverse_transform = inverse_transform[:, :, :-int(self.filter_length/2):] + + return inverse_transform + + def forward(self, input_data): + self.magnitude, self.phase = self.transform(input_data) + reconstruction = self.inverse(self.magnitude, self.phase) + return reconstruction diff --git a/models/tacotron2/tensorboard.png b/models/tacotron2/tensorboard.png new file mode 100644 index 0000000000000000000000000000000000000000..59223cf822ae6bf76c1b7ce2b1732059a68a10cc GIT binary patch literal 174241 zcma&Nb983Ux-}ZVF*>$7=-9UHj?uAg+wRy&$F^h! zt~sBlo~ku!6~h(e#1UX|U_n4Y5F{l;lt4fr8~>&|prQWW<>7=$K|o*|%!P#&B!z{E z6dY_#%&m+;KqSJGk^w3SqZoe2U-P)M$e=&+*iUKw?r}V9*@V6kB_g8+L?EEsL#m>o zn6v#-6zOdQsVfPCgp2$G_J~kdQgUt}sG_ok#ItwhdEDu`HQCC%-P-C1%z81MY6Xe& zHHCpVt-%Aqey7oIsRr<}(PhVYg8c#kg=PA!VHMxQ%ff<8#Kiuk!v{~)6}G{a74rc6 z`cjKOwiC4i$yEqKZ3sE9KwLE2N5_Vamhm02r#b_R#yx|3$t5ip_q1 zGa8~`+CbfX0@)d*NoK$XxpkuRVB^LP7cL^T>OH=GhL%5e)-SH%fVLHP6BH9WVsRz1 zN_|e@81;=zql_LkAd_}?p=S`{Z zX#&%I{H<8DRr<~$Z7!AbM?9lr=42#dwxOb?NRKO_e5np4V#-qH_xt!OM&bJu@0(Pm zuZlzo{!;>tZOi=#gF;rj7Tql+ZhomV(vjP^A8H`N=6k7mV)sOWT_y{2Q7xxMz zRWF|c9HM{z4u=^e{tizId{tmZE+G%dHwG}>0LE)j86o7LKuTnEC4UxT-Vunp0IWE; z0|ZE+T?M>E5Heve3HVBXHo;&87!??NKb+j0y!jdT6RHKkW?)FJ;tB0Nwpg!`I*MxO zgT7-bXnQ}X-S-X%xsa58PCJ4Q1iDbm9a1|uFM=+_r~YrxWIQool;0vk=7iGzU{T7e zKvNMU1%3Ooln1L|T!N#5T86(E-4^{jS2vF@uhxtzJ%&w`JMUvg(TvT5(*wT^S|^+@ z$VO04WLpd=Z+K>#0WAt=2x31>-@t%zG!;G7O9s1$z!BCFW{%mZbu8iQqgm zy1&Rk%`mdYvu1Kd&Wg|li#w)&SH+H;!&DnOCV0H>aG%Of-0s+}*^aa&*6o`vs%`L7 z@9_T1bM!OQyZ*bzJMBC3`**>ZTyeYaLC73IKZ6W}@bj|l0vUu-zmK6JgiQ@d7*N@R z*JiGWbclFKc!`t5H%KBDFno(ml%}Gj9cjM;D4KXs~AFMNZy~gJ5rHS`rACdmF!M4nW&uNoUUDkO}t6wG3Wwe zRm}l{)0fWV1^EU3h2*9GR_!o{dI#+uy#t*C)*ZGH785O` zl(GzQF=4F8=FgHMy`nf;Z3I+=%%IAk&|rQhC>q#0#8U$mrj zG5UPWJd|0^ap|;U&Tv+fh9Y5z(TuK_sc$7rT zamqtVs}wfNJtQ9%y#`<8?pE*MjtFyn^$=8XxRDx@EY9_2fttqbI zHfA&uo+3MAaY=F&a_MjxwxzcTwplviIs?y9?y)?6c(gyXJ&@lc+#B9EpERHGvn#Ou z!a2ZV!Qo*kz-q&xW3OQBFbrZfz$(PPWNb5s$6d!M#!|!HX7HE2kZ~jTlzGMYg~36y zLtmo7U?#=p%s$Cx&4SOAp9Yz3&0@*wX-G_y#Ek%Q69d6@tqkK(u z6QL0$lMuFO6g46i04ekV5g#!J?-H*bFP8iB)aWen$Z6&DMJOIBqLYIofnS*4 zm>-4zkbhs#BN4|4znSSw|F6zpK6aMS(E1@4@a||9$Zb6LDGXNXX6ipz=IzF>S$8*4 z7Q&2CLP_~VJVY?W`NEi@zKJ`Fj*HR9*o0+=-9-Edj~_@Mu8Qs?D8?C%w2T}c6dzt5 zj-|iVZLpjjF<2jbi3B8L(P*?DoGC39o=Q*CBPOqFheX>hn%M$W-}#&$G(uZhC7LVJW3CMx9Mj{ric0k=|MFOhi*+ zt+%1{KK1?x-b+kq$h-@TKiZd3nbgtPB&SCtjy{gYRlCdbC6z%yojyQBTEto-LSlcO zlnu=m1If7ENX^(+8>Mq^Kj;|C2CK^}G^Ys~g=8whCPnEg9YChXJO}+wAS|En#sJne{ns6ZKX*etTB8_pi|N z7-6zoGETC7?m(BO#5rj=scoQU)44^O>}uEMz~bhMAG25l2t`=hyzlg^WTR1VJB&vUr* zZh`~CL_Ao2r`q1Gyxp3$-951u#SIP*qnr2)@y?hg)-*PJvm#bZ0`TF1Vcqx4>${#C zzkMAFEQ)lwv#bg}QQn_3V2u?P*p{;9NezRI4_S}=n~PTqK4+NqdWbG&&MH}oT*_X_ zZfwAdh1N{R{j7i~wc~_oBv;yV4?E(=zN|=nOiy}9?KX$i^|m1-T^afCNqYj%8_rYG79ZqMq|X=A@2AAuhKsbL`jBJQX>KuZG21TndxL|*ox_9jcm9K~5+Cd5*Ml4z-^ZxI znCK3at~3v=d%45S-X}I-`;hQ3q3_5u1E=D(;^BUte3blVjv*g^hV~8QQ{j90?DC^6 z=r)0zv5YK?y3mpgwVx20oD|p&?Hq-|9!TyMgkOtUlU_E8S<`S1@pX<{Q*ky5H;4r{ zXxIcK5po)JoCcwy20MrlO0*MrO=F|V*N;EnEz4Kc3sehOe`KV4gx-g9kb-m}%E`($ z$;o#DLWJHWua>FW`r`(mi9o05+ikqy(pKS0Er@aONuNPU^BU zT!yyRboxfN2F7%5)^>mUw;&)qZd`wp*2YfyL~hnrHjZ3wyd?i(aQ#jH1Jjcb{fpvc z$xEUxt3V`d>tIa8O2nvg|5Z2tLjP6{A1n|3|5P*|ERrZy z0tkozh@^<1iW}%fr({&&s521LHQS-2VyvRH-GgYgKq3O?ghG%7RIpCX#|JsrR2^Bs z&Da*gj1+oH0!|t(!;ejKB!F!T`&(`>nV__AoH0%lNPqHHLbhZ$$*SWdr|FjaHPE#S zceS~$ysdn+%%t`_vMgh}a~o4{ARLu~L@x7rID9N1ZcM=;_dsCO|2H@pKngJtkCOBM z3-E#9Q20Q}t^Hp6|6e182mv^oCLu&bLYlw1v0qzRal;s;HmBmemT~yy=*aB75RKWt z%Md-hFP>byYE|UY{xNcV82IG4ljXYg`lV!r77!+osR*g^T++10KmF58BDle1K)G zlBD`IK}Yjzxu&!^pH}DHhxhc08>>q4&f{Rm^QGSD$TWXg(bJY2P)qKslN;$vtV60Y z8N>0G33&IyW5b0c|FP?uBbJUdecIW6WX_@2rJbWosOq!rj@HRJ8Kx|oB!=JG7ngCj z;|{rhY2YL`BvU81u_~UJPeXe2gd$YfM)M}_X3Bc@#rB~`IR|J6UmO_+qK|dGU8)h> z7PL5@*4ug=un?xLINl0m`&{IU3&q#}A>biRP9BYD~K z`>goK<@-L-B2qm6nXQ{MZQ~V!(&hDc+Ok)1qfRNew#>TkVl7ma216tm&=et=A*x!Mb}d=l>S?9pE6Ta@9=&;PxH{bQsP*nGSx>!!9RKdSE0Uj~*ZzN4N`Z|T^CRtz z+%k*(K6O4(Epbn`vqby{{@vb)dPCJ$1^bxm<{y>%I5zC_Tiy5tIbOS)3zgCHTP+#s ztYc9<8#q=4%RpVJ{STM2n6gsX>aqhJaTl%7$OR7Ff~l+Ttge9g;~hG5T$ zbeBjsR*`&9*JMOp!wcb<^(X4mPIih)(SydfLko+GpL300cPy8_T^Vw=l>670vbP*G z&&;~T+BIe|BMpz{^vp(H4Dty!v+)qWNcUSfv``b2! zFU3&i9t$#+8P0RXQQo(0ymHYW`ZH)Sq=f8W^Or2E1=_>mZH2Nc4Ws)9Nr`K&J0BiW z{g(3!Z92!iB5*fb?*vSC&?rG=hk;$9_35wNlMOfc-~MF2>8(giqGl5xzvYk$miao& zwgmu+MZ1^%$PX*^+#W<+2xyHd#72enQG<} zWQ+pV?ipFz57YDex1@%TSk%|_d6upPN!u@~>Eon@H>634`K~4cLkPn)oqM+X1aAK3 zG;-cGnb-+5=^>&A{^71ipS;MZWo*>6`O5CVEtj!ljKEK&YN->pNLA)Zc5Jm%ThzP2 zwNs9tv2S^f!B@uolH3{A4_JV2(EwGq_3T@DVycw$v4gx)968rv9)#eJ~PYpOP_Rmn-P^1 zvtAJ>9kK2y@tt_wr8LbuMb;*IoNZJTOfC;c`PMJ21pACJZJHHwZJTD`4=D|muz76t z5^a99UOL?>eaGdJ4(77HP^K{Yxu7Ve_W9%~%1P^aFwId?(X&2A9$46G563sszPecH z^eV*uu*GK#Z&9T6_3+*M^DOIL=$JuPu@4ySndcde`Ec*+d;HT1cY!v~rA#h#YY^*# zW3`i+dd$z}SLZ{u^W3!%t92KnRP@`4q4j;urs07h?Oz;}CD>;Axf6;-4AoSX_KI`gJocA6TIxjsl;ofFiSf0zHHAb$eOAw`%O%C$} zEYj&3@*eqqVEoG_)rEh%NQc<cI#ZpX5pxA)GV0EFt-Ie@w zhk24^r}K!;8o_?;#h2prYMLdrU4X)ify1}$8EJUrTf?^_odYHJ1-60>s+29grC<;H z3G5EZ%w5x}g62=hd&A%B?zyF z)sC;E@zXtig5m@KOCF_g{!^XFykJmJUS zmlZHlRCKHUuH+h zU}sYTj#6rV4}@X6pmV;?7fQ?YkS@MSRiDb-<6(JX_eQd#6-pLDw8=-F)=Oj z4wr;yDL7*4hEhY@(x^^`60c;KV^ahxFZN0fS>}QZbBtPrfovN3ZrzcoZ^atV z|KS*XT#q3&3u2rNvs|GppHPmKZ;Of)SOC>_(%fm$LQ<7LMPUbbpbq})4CAo8&duCM zA8jF3#$(kD_s}+nX0#$D4W6hD4ckT<)W{oHGG2dYlzL)VyyXoe5`Xxs-WCy@jAwvgkJI-7>L! zE*OdOVAlAts5GL2^G0nXEXf4mY!uZd{)sT=tcq zAeyb%<=)(P0P;$w?7JVZbf%@<9A3{VJCAB+1g18OT)Sdnk@8C@s8-OOcL-UQi7rSb z4s~5TIK083?U&%_h9Pm_X!51L%O)EFK6q4vbbcokB89{$80rHlh1iUQ^BUjqa_MlS zXe%m7BHbj(*BHUW*mq}{Zk;@9cB%y=<8HgwSQ^9d$;_ez*g=6M3trtAKU|+jdZr+}&k`?G)X`Tp&hum0^599wqze)9U3-&En0-*F%BAGx*i9Pg)v zg4S|Tdv2(I{?K$0%x^AO{^e0rn78ffm^f9Zbr)6SeXYBe3g=N^@ zsbcPW`jwmUISa|dJqJ2O#g+XLa$$j}Dz0|e5~d@}L#O$h6$ z3@ZlpRFsGpAsfyq4+r+Q7p!pl&;W76!-=81A{#y>n4Lfc-H?tiZb$Q0cSmB58t{MiEIzEkjxX;W#}CY{qj`9cJC znW;loA1;D3Dirwe;7jy)FZaX#4(9_ZKEXRIZ_mw^Tx{KF&drnwPUnmS{?N;)Dz8qr z55>nq(xwx($`~}+?V44PqG#$Bo>eh!&RMZ5&`J?gI$QV%p0kpG7%8zj0Mx#2mDJZQ zRIa9{gf~!JR_)KOd||}Ys!65^`z79oJ!~pbB`L*-veTCHJ))6bR#n|F0EhYn0j`UF zL1PJS?YO|2Tb-ttjd^cE#g}wJ*~@IBYf*;% zhi-7yU~;7N2y@$?S0St@jS+^R&oFJkPS5j{G`!0Di5UIjETPcymN4Cqb-XM~hv+5| zw$1j01%vn=Hd7N+OeDVOt!@Mz>$6;>oVLVmHdsQ|gIJViMmWrrE>5|5%Z4V)j~VA7 z$~15%e1ClN{Dh?~V-(ocGRb(?IT`cP~j7*OG`o_9|W)E4J6Y^D3PG_whC+*?0k-ybd|AlpXMp# zh<3TGh2^Qm86z9;-sJ%mun;A4%Z<4|H_PXYeC*|(>bSL1Nv(>Tg8*FI;%2h`8OQ}lmt#~$-7*5TxOo;_mNzv|rzoZpO0Q-G?Y%_ZZDdm8=1;60dFhcJcN_j79UQdS9v7oGX%DPIbdE<9AJyTm8T9`v== z206xqpJ`|>a=ExWNkbo8#$hm!r)fFPHUvY_;mG0gyE&+}3|8gO@l8DV`c7HS?)scB zlU}5SBYN3WUdy+_$`BMPFYQl`Gds+QR0n5D3}7V79UX?dteyWsp+w5Gbw( z+nCH>1m9N_k?%2|`oiS1BAnm~SWz7i;^Y`uKc`ET<5{V8$F_E$9*AU#C$uH?0$s9P z6buQyd{BcaY+2>P&amr|Sq^<>TSk)G@U35N z>$)nDj#!gP=W=C#Eyv!9smgrTSY78`p+Oo4zSa<&&uAG6>1*_>OLbx!Axw`1+x9&Pj#GovGKr_zUN(_v8yz^F!Ac?i&Uh0P! z_;m>_9QxwezVyq0MrX!W;VQB#7)XTLt&7gN&YeZ>;RxMtqX#Oe&^qbr~van|@7DQ|k;7wzXQhBT$ zx+e8~mK?C|9JP>}SM(od@nkS33oW%^&){v0cO%q}N1{z6gH$vd+Wo&4~cSEm-bSO((!`@XDz?YQ*n(efU{^Fa}s}gQ|C_=WNsc90uH_S!RAJ>p?IhTsDN2bJDJP#iVG-!MxCdDKB_k9 z`$B!0?S|ByS{3>SGlFiurv)@aoC5q6R-8x=4!aZlTsWi&0H# zMFpJqv^@fjmwbV764N-dGj+7#89K@cDg5RD99-syb7(L0$GCzZMJAI=un*jqD55+g z6b_!6q0Dc+lVRo|PIDVPKZsz{JWN<^J03rb7bU`)3}j&g@GRln5MYjhnRT3rb?(iy zjx!oOA2)#+4Lc#G4mI--GJJJ3Dg0Ix434PffUO9xepI2`l|n{7>Tp%vOe9>Z;_&tU zRR0X1Fp_Kj89Nu=x2#=B*OgrCShkb13t&dOt z@P>c(KEuO~kX!7^5e0D_2`5)|wM%vEuqGITLpUE)r{P}|ya-$F7H`D{0Mz0j_qfBr zAwt=x)rb1EBviJ}>Hav%>yE!+!(kbkU2Tc!i4j5UJ@c%=I_KhvYknuQ&`kSj^Y`Z< zkrBzhNC3S@mDY!3j`m=Ub9K)oN92*hm2-~{x9>&F2|4ePl#R8`*w+>*#Mh9i)-s|p zXh+~7lQ+*cgKJNBrBixO#Fob_ZC@{x6h95l?Uus)6BX= zf79HsdIep#dVL@f!nsK)VLE|}_Cbh;*i)T0oYT`Js|zlXrQsOybP~TSh^Ft@NFwKC zMR1IlnvXK(cUprjB3ohpqimH&u6!pB;4R-}Ku6L(&Q3C?nsBeymGD<4WVP9G2kzYj z>Q?WF6?YnVm3eMxY3#V??89xdJBZon)AJP`^%2#P>?$ekT6;UE!AU4vX2F|;i?xz>p=DF7PtLoh4B^-T6+?My#CdCv< zhBEd)O9^ghfRml+jV>OlDZqwrj{d8nJ)ud#RQ;lO=Uvd%w1W27wGpR+G2|ww9%)$~ zD2_UrAlYpCxwA46PsrDNhCt1%0OnB8hMji3#8o)ILb#u&0Ma_*5&p-os46 zyPBG*aY5gPK2k+;qXvWEehc|O@S#>K%x%DqIVKI)3pf;M>Hh|+L5Iw_9cQ*FyOxx` z<{>W`^xPE-ep?w;P-5gV&|Y>cSJX5#l;N5bAe*V|Hj!?-*C z1wth$-zoG~hC}K9F3!e6wKt2Fgnk(=!2Lu=MSC9R#Uwb8l>s4XJR~GX<4f4a*%w7u zQ)H1ByayT}JQ{M~R2E0SQeMV*nD;QxeziF>0PXBi5;(V9zk$gfmG0?@YuIFxBv7uJ z3Tlvb%WhQ4Fj#11Lq;Oh(|Or}`E{h(oM&sv?NTHtZ`w#?B)7*+$B8x@{RW&|Vnn>e zjY@#0&E?`nGTtWV#<%tc$z$7(SCzOl+i>qj1%sxZ9888cUQ(ab!p8$(uFEJ}8F=QQ z#V~(xJ1;>j^7S5$m<6_hgp=?`;oi7S!bCjiIWs{jS5WgeO&<{_**Z~VHlV(si2X8X zV`waqj9S1He5v_p+_7EYheE}q55P>UhkaE)H=04jUm6V?wKa?#q!_bo&9w7Ga3{J! z3BjNs((qwf6Un|FmDFnxey##t=@8~)2)eh~4>%QRm)O{#R8|mSI{Iu^B1vU4o1I@4Ls=C)I7ekL+>Vj#%avj#&5e z+B&O7%~RmlCvaY^`*pFQdwNGZL>>`fhFps}oW=fVK1`)MLJt=XhA_wX@f{NrBLizP z9YB*eM<~0rR@+jH_{|&5Otg8vx_irTH;v7YFRydj-nAaCE@5i>zK;xCOzn*k^Sw_% zYze2&*}hP4g07`kC~rA!$R;52cbgHG!FS-rn(v$sYmFszzW`12*)O`YX?s)!@4wrx z@|RXdL%F{_d^d}M6s^Kn7?p)@j!{H80m$|v8^X4w??6N!K|Q2l*@peO`UA?(ej=ev zFvN}J#Cjwo4*7%>IrZ&BR&arrs*SZ)!N>JD(|@~dd48o?q4*b%tKh1=`JbElsR18I z;Z#HS79T-V%M!veWu_sV1^UZ*XVh69zwSKG-E=SlOL$kAS~>$7n10gWJh{G^HWmX- z;O*j#RR4=nd35&Sv#yiXt=m+eh!R~cYy0&KL-w$|yJ`ym?ZC}8B;*5)UQ@l6OUPLv z%qK)mKS>$pjaPH*I2#697yF8!Nm*&=7V9C`&>udWKK>!5;Wa2a&_$qD3&ePp&;n_J z&iDGZ;5!I@6MQdM?P8uYl@OS=tpKOQH>{JEl6;j_$(m1cE=(XB5*+8~!(q#Xdgt8} z#py}GFc{1$4U`Hf6sgO8tCP4!{qk-z)a&aHZX9l$Y4|u9kbpR?2c%Iob|s1$8l1=ezoS#j0Fq^*Do<8SMGWVejk#SgKn~SM*jY$h zkz;WBkc)8|pWs$$h8Sx}Mh{>##pZB0xQ@$F(D@Tq%-R)FtP^F}Etoj1V<$9HqW>*x z;r0GUJ8{XAp;JxRu~+GdUKqAlpm8ExcByoH>kjDp*sIn>!LM4&fUC+3c2GV#mr=h* zZvpfH7!fYuO7+U)VG2uDBlnT!SfjC`D-wf$4<;77X!FB<%rw_WT1WU(t4zs0C2E_V zrQTsZ{4^YZpwJU$gUJYFX)=R-e!1(hu_dMYW*_D=FFYmhpXKZ0G;nxp87RU}_v{y? z}WY9P8`6OL@UDxFmnG3A_x|jcA@!o~gljy#e?!LdktSG4YPX#6CHN3-4 z$JU{3;LwHD%;VGJqf49Pag5${)jvLeBwOeF3NTh81Mn&0xnTvoqCk3CsXU&&#Y*XY z?;S?l;8F^q7zag3w4p{tm}#9ZUb{$Xcg2ee^QFyO?o zK9--?8$1H5wt`@1ZsW3$xa$h`Ao?)Eh?1L<<~FQnu;IJ%J?alTLu|E^VhPMsqHHb0 zO3c|R$U10EL6_mBUbo3GuaPerYv2BAoG{c`?l#(5nS2z0nmwfvX7{Bhsn@<@wI_!d zPcP@rLjO~Vt0Ly@55xy>^SxfCv7ftCeXrmj-WM1v25`0uks8uahp2hvb8BvkFrQSr z!Dg|$WMgpMqHk53cmNC029*V3phor$xlf)Tpz?#cMDkpIYsYcZ;iVk8XO%-%!o(sf6UAB1LdwHjiLo$8G|@ zP&YCf1X>5_!pUxB(N+T3@^YpG!Xrt*lz=YF7yx@Z0T6`Yd;H%eGJVY(pWwuYZi1e? z)jVga1IQz7ABPxfTh69|+#Eu3f2U~(b4l*=FX@IkU%ul8x{hZd#HaeU0R3A?W*tk~ z@l$t*wfwsZu0(*f)U+OPKPXx9d}P^`$xc+Gwy{x@WaVdBOGRg zqlE6`Cr8~E{OYu+JEzJa_w;SiLv{zfXHe|TZeM<#RtGYJgL8lgJ>f@~eNSy)EzwLh zk^ec+FRnv{r2}?kw&>}(qMLLs?#+qEanpxpJ=44SDz1#uF2*ur8{aWC<0IdjyhQQ! zsPTvrAKo6nwQ-rU+UdkJkjBORCr69D3+NL%4_rfy)4VSi(5~XLixpO+oYL`4Q*ruFmLYqp@wd*i zDx9HWsQuIr2MCEfB0F;zp8AQH-ZPwF8x*viJ52~`r&Km~H4aZQ`?x~HdN^0S@nj?q ztq(T;mCU_J4i~baDw6mM(c~EJpDx9XE3FAvzh$ zXmw$X@;Zn*1w}SunOM}={rKe-fp++F^g}<-Ct?p>@nAQ>K7MvqV6Eqdb1|-XtLGHu zS=Q_ovBWofX*|*w5tp%pbubM0pIldZ1HL!C(q$Ev9Lx(+U&eBuj=d^!{qpnpxL2~+ zkFX--mANmEO34{SlsW_RmVZId8fVkz*^I-?azN-T=YSyK+ly@*>P)&^hj6yWGBM97 z+<>Y0sEAyr5-aR$Ug6lKFd@9No#X-5BI9#mP`J_7aGe7x2710`L>*cBt)XsL|FdR&0;WTR6oi|0k z*BMK#0d zTZ~{#4~b?`RIveO480w4b#}IOlK58o>x$$C;Sn?CtGaV96ky(_)zAE!4wPEFYOz#! zIGA8P@UFuyC1@+{t+QjD)FmQ70=PrCuJa>nh{2i-uq*GG00dk=BKq_T^XKbDy99TP z+RmODyZLZjx3sC%ugY}4mUrW{AAH`@f8CZcoOEjRDNAKnrtV#EHa!dY__FZJz=MhTOk#HNCT zLhoge$;aZXRWSd#J40C>fCI&f&e`WCg-K8!Z_qb$K!lkDkJN7ON!U7U*z8xvAF8-< zp`h9)aCi_9t96fK131_(AbaQAFB2eJ$t6}TR2B! zW=S%9I@9YJdyjdO3n3?LBL!nKaNuojDriD-UOhz^>rd7Owdq&|aKOc?d8?1#-Fnt! z#2$+&cij^Tv#fvu_aM67r#qEl8JHu0yA9k3^lj;Bbx8_4=+9HrPlVARw?1X&g+^R; z`~&+n74)0t4{y)T1?E<`V}`?!i$N1-EW@KEL|Bl-1RLrka!8)III&RFOIYrN}<;|J`Zm!$;QS04on_;x8J?d--PTDKg;`8D0 z`nuem-F2H(UBLg6z%SYTiTP}jb{b3jD8>gB)2>jrB0XK_N#oH+PS2DM{O-9 zaxkWTpBu}M0Uo0b)P5Guz&^_j0lwyOyshas!JDWze>ug8RNhUie6iCUy_pg~`Kb>f zB=Yca`=k6PVqt$vNQ{Y>Ao%NGw(SmV3NKN%W;w%Ex#w;Tms=jA3g)+%^|rz_FK~9X6qd4hq}jfwM#JYv z4AX488DjTPZphmbLarVZfUt{fKOwP`_z>Z=UyGeQJKSK4T5og!)WRuwJRU%Te}iU5 zuq=lw8A_hT;~P&g*?nrEBI^)m1Z^jQ-=!NNolG30vFPWZkbg%=GtPS;ze@;KR(=;l z?ir$dez(n$X0|uz52%MqHoh?4?t)+;hsH|#_Jk^Dr1^|fB|&D7NPzP)R=bmF{)Ky3 ziTDs27ctX?240KCd@k*bK)Mjn{^DTYK=keF+2qWoqL`D1_k{+a3Ra+4?XabA{L9iA z#T})J!l^#+YPP!%H1IieDO%i`uoEHQ`eBQ1pZdl3GqM+n6SHm90c6IEe>YBY+C6RW ztk1t_s;~aoBX{l^x2?6MWu{ZQpU5oA8gI`ty*@w_Ft``t7c|rFPT{(#0bm9lwmbg8 z2wl#B0(K&}sg@_tOLrCkq@zl8IYFBt>&dnI80uYZb0iIf5;HgEnEIL(|5U=DK2f>d z;IiE3GHd|kTrZfe|D;QAQzWk98%(0bRB0P`89S{tuVuKBFzjj zyjsp2E2KBLy^)6sIH>cqsXcGLZE2NyJTQeTi@YYP57d7Ui7VPj=J7n@)?$UwhE*jv0Qiruv>yZ;1Lc@$lohC0;!+AfCvtUpH)YaTX;0 zYy4I`;6E&6%MV-mk!Z%1bKzD>li98JN%#NUz|3=w*F#DLnBdzodUecs)1Iq-Wbe9( zHtkd=^Zs8uo=ty)6N)~0tU*#nugPch8^2D|0tm))Kj7_#2hW&zG!VmFD%!kd2EgE5 zGFF$m{Yq9*c6%NdfdqwPrwrCKijy|23zLH#s}cPgng9|(4N=Te@IQa-EpxzSm-c$l z)h+GERSTxXxj_;iB9_D9R?Z*PFw9WN06)f-Hh%EU1AXRs0xhjP9hShFE&&+-+-se9 zQUAV!PJx_Ak4I#ZIC_#O5$FL`CEy*eg5V6vD;Z)JakYf)%S9AMI0%^$GA$aVd{Yk; zY@jEJOc@dF5H1n-ipT1zWpVE!SQZZbOSUm%wQ<^>g~khoHAymiqRi zQ9a0bAp&t!O7Ejkql=2EVdMzT7zzBI9(r1iWjkrip%pA;`fQb|s5L<*D3W_mO994y>fqp`httLqQ^VJH-%O`o)^%gBy1n)X`kzAK!k6`GbB2fN@ zZd=b*#5}PjxZZBj6^#rx%h{F7myqT>06om~NZ0>9kaUZkh&k)$jr4)0V~EBL%||kO zs6piYoVDIrG5{_5ZFi^;G=~&|cyM{K+h3Q3FXy?7rr;EbUa^SVhBFjteNw=XHLm{* zWPlBj`BM-_6-i#XkRmbs5%ts#LH7=Pl>yCR+}kIjB*H;~4T-YOCP@uNzfEt<+4?Oz z{8eb`cqFJnqq;74nwy|2HSlTRvo{&qq#LIseWPKosoOXSNYma}fAX>kCau0PMQP!S zy-nBe<|D#5KepF?3PmeD_wLPiaIXdwZ=~J!Ub$Or@tHnFoW9vr1RgzeX5HOP-SupT zyl+ueM^X?@Zxf^IBI&>&7+Lb*{r`M@f zvsPZpO!KU%>MW!%g;VBDjS!c_urf*wiM{j9j(-%?!*^%#ZKV6M59{2Xl##p z{Xyb;yRH25C`LXvGZk&K7vB5X3Glg+0pj0RXhO|G)ruHA2kPJVQDCE%4PC0DSHH1} zZAS=X!d~GlY>OVdT;;VhRAF-0&Wze86n(O5V1iD^$Q~x%hPA}JOAatHJDE8Po_ZXs zo`tV;4^<6}YtOz=;6dR#Tl21(Q*$r)UiAI3c;mi+ehyzwZ59~G078waZ7O3sHml&f zGEPBOFwf491^4P$zLGe~5cr3=*u-z*wi%Ya40%IY-n-T&w(VB;Ik z(p(IrDFyENNj#{F5Zrsvg&r@XZ}Pg{24AYY-4`yrT-hWN+*Qyd@W^uY6RJtT7>U#w+$6PU~@9=YO;l#SUFIjOYEa@1d$Alcy=Z)co$9 zdmgDa-u?G13}3|6L&K^dGp+DHCwK@4K%?i89BFU8u`7KQR37s;lDrf^`3_e5n=X7t zs1Ns%_Wi~}J-`VOY^acNhV9rxevcS|2fFlKb9IH~2qnhfkZXY{k~`CW`$P{fsjcA> z*%$QZ{CH1dFbh=@PT$nZ7lF-_@IZh=fDIc{vH~2uL4Z4AHD)U4pZp)HzB#zAsBgEi zZL3YvG`4N4vC-JZIkD}ev28YNtR^|JZJyY;dA~dFeE0spXU(3y*38-uf4{x@_=e$tH z8r+n`vVkQragk71(Q=S_09Vt}%@b zzR^{5KGQFpRx30yL|xIJ&{ISRKjGsHMUN#%?MPij1ule7+E3%nEb;BRB9r!}<-#;a zlVws6T`9Ogik-Hg*imfPGGe??yK=qj0lU95gst*lpg323IYwZW9?u6(ESDswn9@s` z&XLYVvstw_B;j^w z**@u;ZkMabv$9`e{C zjgn*%jtj-e;VB0R>zD?X$nz%r(=ATvSh7ZBs;~`eQ8+|rbl<}{f1df^l9w|4>bH^) z#8kj*4NfRMJ{X zR^X`YxehkDF2B#nL65Tlih-Wg$27?Tv8E$nNM4_fIh1+mRCc`E!1QFV)DTz;?$5*C zMFR}z;Po`zkiKC7nCyStwh}!t5*1D`tWi$Gx-o^`p{$ZTu`I}gz;yI8ZkH}=zBWuT zhJPngC0(zv*Nk?fOXLhq{Y=|WWN@Nylg4fGE6e$o5tdqQzNqzBT{M(McOUU>!>x2D z!4d+>00IDyBeU_&ss?5QzBznGX^L zOsn^^OqJES5TS?*Bh@MFQB2Q~ z3wHwJz~eR*TtijUFup09fy4faMRHWuyQEQ#Gcxhrh_LS=hvMj+?csSBS@Bf;uX!2W zfDh;PCU-)lMy#P@UZjRM7XMHx`Xdkl(xu+~ZVJ&I;~iJbAJ2t~&y+$Sny-|oc@(=LeSF!7zh2H=~u%p#}e^E{CoIt{2U#fL|lo>GYojBJ#=|tfXD^5SD8sMZ4v$xPUs8kiDE^ zS3!^0liK5QskBC}IUAF>W(NH7);oMgOBa%41#_utD=H^w^7ABE^J68W-<8QpohUBf zw0gT4G?CxM>%YLx=ImtZDMukOY?r_TGi&_|FMrhbc#WAk=2QJ;PfcMTCW>kLAlQ%d zkgmKk)tv5NDM4duR2JPY7be#Y?VCV5mlV1iN$|b?b|bQci$a<0* z);57by!$Fb1w6+$<*v1U^-$@%w>4birM>?CAxK8-nd(|wum})*G)QgO`-_$fbr>@f zXR{~xv4KU8AQjEuJ2zGo2z3IlSOLO=U8Uv8V>{u_R7-H#BI`mHwXDpEH588!b+R&* zuGBjo2oOr*0MzF%4>nvZFH1&#dstzq5k^{eNml%0wm+&{!IXEwi9lUTc_*8OBV} ziYsQZGovyV%I8BLQzaIeQsS-c}bl>r5Ph z~|1d<}+j4jjJhy_K*pH~KyN7|+d@w{%ylaH%}Oa)B*z zAg%Nwn1qRXhP*jS`;W4wL1lr?y-)L}7YEOtNU*BBbJ2v4-faM_LP0yd#f@ixDiURe z3a~eLN@iGQLgza*d1*8C2qC%{g?fuR9*)XSC+pjDsK;0P6#@|e+vqOsj?Z`AwwG5*XuA@SFPd@(|eX<|U#3$FopDr|=KEOwf=gioikPG#B+f$eLt)Jk)Eo{;?a_ne9j^)NPuJ z8HK?gD@@4vH>8)i+mk#Qux<(G%_s~v6?X4%YfhPMJv^;PG|-sx^7M4l zOnVn(RU_z^1wP2W;h#ziOF8!ajSZ4>yw7Ff;5De+(P;5x8Z~;9)OIE+soXJA(nvv?JSs!;#9RS;m+PsBHaKP!S)DQBY|2NfQM zY@6&CZ0~$a$S$lHB$h^$cPc3oc z6676gn%dJ9^zq*=fGeq0e4amqbbH{yQ=`6RG(8SW@B?4@_#bpaW*hvOl-Wy`jnJth zO+xr=CM}wU0Nz}yN;r)*4*a`OF(}h4z72j0D*g2c-b7sb+VZDKgLdadi0s=|?sUw= zIT~M~qo~aVGUqEu=TNjGgwR)tvh!UcE-mQ5{L4O;pE+~Wr}GbcF4_IUB!~h7b?)*n zRr~dv z2Ui`2G2*>jE+i^)2^{MSxG5WJ#*IQFf6r&Y4enIs8dHf{HyCE*&&OL)-<8K36Sz|L zP>hNTn%dsPc9g*R);$I**%NiHhQNXU>Tb;5?^Igsw$MK;5@FuZR;G2Fl z;{tH_R5l11Mfm2~+fXg+k!O$rKvClt4?SX0(T5{V_Y|xW(5Mo{nF9+Uxs=s@o za$nLP8-#V&z`o};hH`j}deSsBS;^|5iBMC_aOygwp);vw zV4;h{1c^DVH||_1>dPP@zM;`4u74?bNeqg#LRqeh>mXbR1yEWXRGpFUMRG|c^%>bxxM54iD8>+#d- z47|I6Kl84? z3p!{9h@jH01ahaYG&O}Vd6aE`o~>mh?771G=)rY zYaF`?DVy$h1M*XOJJdkTwx>`$s%^Qn2G|fd;KnzW*RO#~$aA4J6yj~+b+?twqG<*s z)YpI;=z0_4LYSR?C$k7brDp|d#Y@7bkX5KdO0(q|kLwDnelBz*K|B_0!6K1f=s>Z7 z1h3yDs{o1b3L=8KL$jxO*1{lVjyy^68p0^7!zbNRKu5JFES0$aK&lz(5!Z4j^&=5i z^ty>`PV7?>N6TQ}6zm9YejZZ%_h(pK?PQFZdJ?vcJv8@6niygrveiE#^v5Ly zGce^aD;7Wu5c=_C`v?_Hg)a5mFUAo=Lh)Zw?+ww{ssl_h-^P+d)S6m=>Qdz0ae^_O zhOl;!JAcX-_0;;X{;wz^|Ykh9ze#l&U2Dr{q&#~MaD z7I^F|?3%)61g`t09T?S`Fd>3eT^Kb^w{L{HjuT140T+!7SM|~M%eV}CQeHE)#g<_0 zMOP%VXnS47s_{vBP>^Z-!#!4u4~7lpjD(BkmZ?aqm99mR@cRt^2K{S$DaV*wErK%w z!I}JwnK)$LawG8M=b~i6%qe*fV9`5gx3viu{!~`S4=(xba0|w~S|yHcl@v&-&z?QQ zG$DN2d@&|B(wH3|`aT`<3hON2zAb?|?KjGc9!nB$SJ1o;tGhoqaX=1onC{6ya}!v! zL$Vuje8qetUo61g(RjUTop%xxZ#3L^=y19gcv*o=y4}#FOfo)y3AD5)mdO?fvI8UW zyfwlVrC3cg?VoBBckexDm3YBj29dzj2pLx^fhqgXhW4a$szI8uFMa zjF1FM@4^+`-jc>#@~MsYJ>NEbS9?$AzI;0|xK9fBElExCV*fEL5FhSa>Lh>V@A_(m zzNATfL<=uH&?_(m4J)vY8SRwM*(q&^5UBb96IL4t4ak^`WNC=yE1tC(FqN84SwI(R zoV}Iw5qQ3*W(1Vl9N{g758GFn=oOH}`#}PhwaJP(-xkBy+27t!!)2f8hVa&f%ejzU zGeyc~XzZsyqJVZ6DE<2xyOrHQx5jvk{%um!b$ayu*svqa7a>%_RF>8NpQZz2D!a}I zA!{O96gH~*7jm89+x~X=qqDv*Zr`u5a2PAxwwK9Ie0^Qfa4t4{Yif!WLt=}SHbFge zf98Z7u@Y{Jt4|r;ID(d1)=o6$QC=BAeSJ>SJ;1kX%Wz#h^JQv?N4lTSz=KiR5`XH&CLw`Yrw;4JeZZPf4 zx-`HKojoc#?Y>`$10TsL-}rFkgXK=pA%NcE%$eSn3lt^tP?0pnm`}t4oKiHx-|cwT z44A|bh?XNb5%HVZbu&xRcg)=U*(3E(U@t}%mV;13Bbk}^vCM$WS3T0`6UUonTQte= zX2kv8uq_J)6u6vk>rvDxE)eVMn!TSf6|Zi$@XtQtbIeZJJc9iL@UyY2whBd`Mz-~Cnzg*CPcRK?rl>VGGf`BnBp!xzz|;F!DYPbm~F}nv0Nm+YaZ(F zMTIJx5s&zVAxZC-r{PXrh=vD`aaj!{07n(-Q<`;WDk*wZZsX}vD;2}9ZR}Mc6o0)b zE08pBkXJaO8~^EVLD|y8Mc$%kNe`qvc3+rDKB+1ekUHY2cZ3`FM=(*~TBUOn)JCW* zv+|Tje%Z__OIIwCAkkQMCiIH;$hoC+X-BiV%n<3pvOfJ4hyjSb(#oH^Y%qevH!swQ zarh{5(oK_@z8_(yZm}I-bTsIlPYjKg#q#!4f%0fGpy(oSw#?LNpnW8BmLt5D^GalV zfgi_iGO2UFDpoK&6tDi-uO-~Ba4*=?P@iVsa=9DiL^=Il#I~22`^lrb=9+wSoW1J6={=)|E&Efp+^c*LoV=vB3wYxm zf0LOyEj%&6TGKg#(%R++#t23awj=K9xNNczYnr7=TE(#Z3g(MMu$Lm1oAY2d;0YaW zi%={<)5Rm}AnIrDGi-vpLm?JBYCZSS({ib5iE||m7mOMMaU0-=dDhggIQ`She+2)0;fl`#iP*cy6zz2323AQN{zwTubwjux>QE(D98d)O$ag!UVnj51I;B z5#jNq$+`2VjC&p@?+Y`zVnFj^sBRZ-lm@Ql;`$(PZ8e%&Pn^=e?m7D(Nu@Fyl;{lD zn`jt~VW{U9=2RCF2V-AJGG_Ufm2$uNE7*0B>`+aqw0tL|pqp>{KjL9wZBBXrLPs;j zrV%dg_`*DH_to&vWQVjjhVtG;6NakZeZ}M5^BWL>buHh9m~FR6U%YDm7YKMHUARQh zC*`cUG6c*vYTP!G0DInICiU1yG3UA>PRHVB+|g3(p7!~WHCGA zzl^S)!#9W*D-|bJm*~xmkDGo8hg_j77ZVV7(7D@yBd9Nooz(ibFujL$m;AbnFR$My{ zE8tLwux@^F4o84~!6WMRoX7}*uA*He_Zk%x44Mc!o;`J!`|@qc~MzA+MPFNob8!fEkv`n~x4T0mK-x!QYm zF>c#BAiW>`BnzJPIc}#Fxx@b$NIzlkciGl?px>WZkhs$(xL38}Jr_$XIM*j|P7QwHR;IDYQ$WLYu zexCqUvFCNX=k<#iCnslOF6;kSQAkDxN3F#)=_lB|HlCFO3-<|ktsI->8~-<%!B2pi zwb=3c){m&fZ}Dd4cgVLXQ`__I?R{F^I=L2Q2?FlMa1~@3zF()!lD+nl6&XII7<%Qc zRcris`u?e^T7*JT;%e(pjT{e)>Iv0A@Du$bQ(YP!vv@KnIj0!qUtm~RB>S;(LwM}r z_u!F~^dqQ4n$e&$9k`7Lq5{C$VlPH9Ad$JFhn6Wr%My zdMu&*qL*bMVaVP$g5woXO!~EtQ(dJG!e)D?WkuMo?vAO2>G#uhy)+iyp@1W39*_V} zi87hz1Ny3?X$Bmmg@B``1V>$<_p)g->6wJzB4}dF>}~r~5(>|SpOyJ5miu2jMpfuC zOH_q3W0GJ#1;T69wK1893F7~gecm~7NPBu>f)tr+x}Z1FrLclENeVVOtr}6dfN{H*P5OP3&9oVD#8(w>)>vmvg#n{$@6cExpbB=PH-eKH5|%!w37UzLNgROyz5GXHRTt8mZGUyZ zE#_#ETmR~CcZJBF$`Rt_yB_k_=&rz)LdS~$dU#8tLRZAV%9_=Ms#b7heaXtL5 zoqqI~<84!m`t*%c##dn5|MS2|Dk0O2OuIz{U;{RLvCboSzT<4qwO~+@Lip-L=dg2OPIvrn~q(nV$L3 z=Um(itHD{pr`0McL3^AuRyH^6&q=+)Fd^MhLzY68J;z{{u$34QtVP%}$@WHRsc7Q@ zP1EQ+L)Xr{ut=bEu;tyzd%DNy0tvqudUFBec@ks*Qlb+gqWW#Q<@yQ2F1ZPzw81mu zP42MPP7M)w_I^$PR7qURD%;%baLO3ip--&K-!$D2nqyJ##l;oiXb1EAXSTv3B^zp= zu8+?E#Gt|&0f;LR?5Y6GSDYUpwTJ|%7wm0bEuw4KsNmeS8<7n zztd|1+o2&(`;obXT_5Pv0N=;^bFe zyu^<<9{-L>h*$>quk&+U_zxvV)3IsyHwDv527g0R!<;i2IZG zn{iAd>Zs)Z_(x*Sb}KB6)~%fn^sr?bgJu%x^x^JgudtyPr^gCYr+oDFw#!M-NleASvlOyUZ3R5VJ z)Wg4Ac{hTBww#LoBtzWIZhx~BPP5v&E}XRI(?{hw%qZV%gFRP_7JxpBZOi|WS@C{MB9g_Q8J zX`!}q#LmVc*fB`(ISH~u{r+I&{ABjCY%`2hPG`Ghr28)rs%vtKU+uwR*mMQLdx?eb z>H=kPB)O4J5ZY;~f8AtZ0CMK@@C&feet{+4*|IyM_Ffym2+Lr={ zg@Qx)hWHubpsk=ZKAJC!X{ewsvVp)E${%8*wDeyv%jw)&lV=gLqt$gKt zh=@wub-|$(!-rA{Rj=xQgTF0ba|7c}nJ8ucW|2*ih=mHB2dQ^Q9?O7YQwLFYWQ4{_ zU@y`w3DP3c-$}C9=C6vmRPujO7M(`cDc1k3lt}WT3Xsi-UFR`QDfZDaHAk<2CO~$SGfW$DV_*Mf%*{DhaXud20{!i4YIqJhz<{t z=aubY%$ts*edphY%xN&%caJv`+#Ae9H7qDGBwhBSWwc=4@i`aZj+15b^1Y;2n63jQ zA!k0puxc^i80`6&CQ31HN!l15`;QkEgaKtCw;5@^sECw7Nm7$Oi!P92!Q|*lgH=`b z4r7@)4DKE_M&7rvy%&%T$yCM&$Cik`Sjahkf6KYcznuiESvor@(}8=Ty1P4zCgvpA zYc7d{lpubU%(i*_0FZRmYq})p6iHwVwae|KI)*>)do3UgNmeCp1Xw*R8XCeZ>>|(F zMM71-FW029Y`ppR$1by?n*ENX09*MuvI=UJEdhN87V8VnP53p>=za$h7TZeE zTgrt|$z#3zEt`diiW>$D&or&sr{HcWUsBdlke!737P%i=076@$v2JoVDeiqY`v<*o zzB-}8YwV1pYwPuG4}zB=^HwKzVzJb!QAck0CJ>c2(ul25+MO*`=)&cWYo4&@o5$VD zaz|o6TW)RW>za?d+wztjzmVY6BD2Y?Bu6}v#%@(f(q@G8;k!7$r8Y|HxY6=;$DdEC zRVdYJo*4|jq1YVP%sm8Wcs!AeCt6MJ-#quj+XvVMyIqAP_+%W;ZvA~I^pTOmcegXR zom%$0g1EruCT8o$49ChKw1CGKC02`NVw}e)ubOk z*fv{F?JU(UGM<$l;Ds45#&L>|$dx?!)vOtb1Egh=}g4*~*a$C$j$ucIr9q zdkH~Q{}*_+*dO@8#PJ(J@D>IIeE9n)bweRZ6+X2NaO|Tqtk9 zrIa7o9jBbyA`Pwa@b_HWe7@%-l7DOuM!)XRXHvm}ib;9<>qwEv@TM~yCCcR5cuc{s z{(V$xRJfY$58DV{mYoYAL1SQEM=}fJ!3p3b^tp`XS1cnXf7+#2w2F6ML2FP$)VCF% z{&|w1R&x)7g7FklG#Y(e=3nC>7&1A9a&cB{&aAjp{6x@rwyB)1P^S{9BmK;ezf(8= zs4C4<=i$MUyTJ%W#(aTUUkUNSKUo-z=4EBFG)08ZGpg#8P@JEJXFG=e!VCldwQT@B zO6f@AfQ;QyiIm-AIX_4-kN-R0$+vCP^Z4+uwG^JQa%gX&m4`aRPJ&3$*n) zt+2wuMAv=#U-uE5K|%{Y{jR)!s|jBTc3PxO?MG;j7g<&u+@9N9&x#N)woB-<=92Lq z`NjR}V}CG<0ptvp7cwuKgI-#&{vrqZeg5WMIn_G(e841|kLzOdam3-`7a#?Bw&jVy zIn<*!zlSnfMz&OAtjNv^ecbv(=ko{)q~htb-}*cS*`l!@ba)0HP=reX=e5!^+}!S@ zczx$4Oducv-Bwv8;*$;C>N?);T-hy4fx?(&I(PGpjA1#TMG3Mbb>UG3c;4Q7hu%Sb< zapC7gy^3BW0@Mi*2H}beLs+i&aIkYS73I~hnIH2qkHr?DZkzne(miX1V4T=Q0=Bhj z6MWqib5pvNH`Mb`4CX-G#D{qo|74z({Xzfug{NJMwL{pb`Ht)cp~aHFNQy+;Q^H8< zQidy=+)D-MLk>LSQShVqlLil=d4WRXT8#f)HJoPt@D4acv# zN4|_#8q3*nD-riiQL&!bU4-le_wt}}6C@;voT4Ba6MyRT(jjV8_jLDbmW>`(+M3IhVEsq&7X2ooT6gMu280;h-(n09u9i~GiLS&A*@j{r${R|eqG!e7P z*Rj{_%CSZ`a#)mu@x~`lE06|iK|=&|wc@}2+?EI5Q2x=lbY(Ukb@v8h>%E1ta@&a1 ztEg^1Dcv(}qlOES@PMx^(eitc`$(InS6|q(Tw^}dxicLLAWz=yS_#9q{`mCovz73@4Ff>C68_t0x*JKS+f}t?{ znN|bE{6npil3ZzmGI$?&)L#SEOfI z39J>P6AB+=^}LQM_>fG}pp)W*c2Scwa50j8cy{>^0|8 zr97?pIGumtIoG~e4Sy-SG|)4lw;Fc>P%|84&pu@A96N}CbbJL~GLNqT9eBI7q4P}| z;KR=7+@2TyYi_C46CbiekJe7_M<;?D7t|$cboCex#Ii6-wBsM_7j4`nJ5Jx7UQqr2b&KPpD=aLHLZz&}{Y>K6v&ZN|sxVd7X`tA}7HS_oK6d6Xc+w`$V} zQ{5xsTd8{d2$6+u4MIv#uk`?B#t}2NHXf2`v>9^thVz;k@qnY1UnR!#?YARSi{xte z^Xh`0>f=-G&B^=hfkw1B<)ie2WqQIBy|a0(QWb^d#@7*~^%*lm2j=W5@L&NCQgXiG z9gjTvoXuHLZ*5!IES+H2uCk0+ z>=Oj_7ETTE{uVz~=9PcoiyZv+F@6$TxCfsFnGQIlgTFr;Y@{UviZ+J+4 zEk(>|Mc7T#bzx*{fCn+b9V%#?GnP$jbeGQaZ0v%?&URN7i@W~~%?&%>czt_JeP#5fQNlIRHbIbu=CXT$FB@i;InEIl>>&3u%#9^*leVk@1Ov0bq~bnC(;gb3bEUkT&*%( zdC^z8Bd^)3JkS?P+rEDN1Ds3aF#yUdWf0tjwKCam-83CAYggIfcDgW)r|M0gPFH99 zYKZ7YxIVX2_MLxmNpN6fbHysS#{T8RS-SKP?VKXX}-Ftci8{tFQupZ*O&ig4C44!zf62%IjBq z2MeKd=OZAQi|I$iYA(JK>uHj*M7liwI%Mp0+`#;Cc}eKy>=Ar}0*ID+*`Y6`1{{>T zK@>8wzhDQg|G;lnl62RZqqxI&u0k&J=w-X>`=9V(sw>lPpZ{Jv?2>*xpW%?X_%7L5 zG|^#eUbGj+iiZEG<@|rBNiqr4=z3-K=EwgVRs3InjwXfGwkF z&5z|;GdX4zh|{tI-(O<1TEY?fKze(!o34_q_EqG34tNoUtAj6$Gp1lgY5kcvr7c;r zgmq94lk6+38?K%nL2W4N&e}kA5%d?nsN@1kSiJ37u#;9)+?DZ&1nBeZ(Tp=C>#^C(qRpwV_hgd=#Xh%e|x6m+Zn zAy0dM8)3SOpt|v|WYm(wA+QxMAwImW1oxXqfJG{|vU!8WhN-925OZN8h^^8()Th&| z-W9L-!jj1;W?eMilqV4nL1>hNsTUTI)waMyVStvwG+Q@Qp9JAv;}JC&rUHf)Zw+a4 zjGP&g!_FX?r{(3uT6{xwvj|^89(3kzcM3WmOXAQQ@IOPq4x#;eBBWS)D3I_$X=Gzt zujhZsaulTlo!N<*zzOryTmY8UfPk6L5)Y=*H&O!}pgzO2n?8H|Gnw-V{%aOFy3CXv$yC zUE{UzdcSSC&;|$wGXkaYB8ab*yw`nhT)~FhH}UMIn$i5%5|P;9f;FkrMgn2tU;11}%^p0OCK@YEhciEr(>{S~NU2wo$gvp=>1=?$agC9OOJ_edMTs2c^ z1=~~bPucQ5z|H#(fs9*5Wse zK9y%_zauB?0Uhmjue-fVCSV_18Nl)!@D@wEO-O6?GIT^Hp^%sZXrrC$wWo1w72)MA ziTGk}AumZ1W-sb`7j$GuV9_T)WU6xJzx{4GOhSu;o_b^ui{<(yfV?H(+VErGS<-zC zp_O3Vy&@_CekSCi*l>SAa2LWndflJ0?b|#F7kRA2FRuINw!SZSNSGbG0~{i?M{}oA z{cT_Yw+2U*z)CKUkqNH$@)i$qn!ot@idGf@jJ>U8aibCF2EfOFyAL5B2V86I)YS9I z`SUJ!GQQu@BUS&NXT`^N!k|b+5V5;dm^$l(p}Wd9F>!j4B%lXjr$w8TYoWU^#j%mP z&}ALEu4$7^JSG!dfOoBm8XCnJIOi9c^WYnPu%78+WHyF2!Lqod`M+KRpBj$C^ui~} zhZu*>U6R=S0)Cq=$cl2guA?6|hSY*Psl4W{GIM3Sn%P1d(Y~0H-9Mn*p*O@fm)+0y zct)>?QJ*HKvUdfY;eKR2-j$*zIh(A>QA#B6X$+!WqY@I{GwtC;VtBXeXjbjYER7ho z!2XPCPmgux^M$y*K^lN2iEAYXW8<4qtn zjqhe7Zc7sX@wkSup^hket9nQ1K}FIn{5p4B56%P%j?Fn1lV+&GUi``wU1y>CH*0f* zaF&>)say1<70g}hJ`I(Y&xNG|bf%BvHpU3O8}P0YT;3KK*3QqX*G{Ci+Y zZMmAX1Hr3jmtS1_;(0z?JDUipyBqU?RN4t|P4_lxOcz$N&hwY)=P^!J*518o&iv1Q zBFs$+WOqx&;#n0x3N+QUa8k0^j(J#i(U%d=7NqsOK`C#vC5qPN((P)oq@kCga{2YU ze8w4pV!`-dQ#Ke3c4WJ+^iEWj$3H##1A{fEzIDsvw$W}vs3&v4AUHRKTS-o0DLFJ% zR8JL(QGLu-UU*i&<%)z+ro>pDE3Zvtenejr@ktrsoq)2T)japnL`KyX`)U;dh$Q50 zOD*S<{b~o1Gb?WB3pOvh`C7~V&kFiC8yDX5JrWt2inmN2r0rBJWS;v(54X}=GBaH# z21myUfTUST$T$f|Ftw}?OK<3leb~ehYjPKyAMUPww@@JTTq?oY&+bC40+6?KE@I(x z?dtIx>0wBmWc^BbuzgkH-&C;}OTy*$%ke*Nav_qkkSWyJ$xz!&Uvp@mgQ;j*5sPoh z*l@n$nCBA3!Ls6Xn5K+*CnmpiPu}34fhs3ss`1%6h#d<|7(C3VAqdH?c0E$avcvY1 zJ=RqZAspx;6}i>5e$M=hl4NdmI?&&7Q7zsUOoB=NF&l3ITaj0^5FaZpg%8XE4a&%g zda~T6TB!9o_YLx?j2%kqSQ!>kIT6`n|D==!BG830n-vF4xdvYZvC|2mtT{tV)gR8| zcm65KnmaroOA$3E-Q|v~TTG_m51=bZ=qaq$rudvega3MFj80;j&R|Z4Dn1zb(XKe> zy!)9)VO5Sj)b;PG_A?jj&OYPbt~1y}m)Jr3poqM{wm;4Y1?;opHR($FDDQ9wfc&-8 z9PyQ^g=tXe`VVqleH>$wAhVN@z)WInRO{lXu(zRJ%@H`FfMNH_s^#V>lBrB z2$axIb9FfIqz<&Y-WO3}dqTm1J6K8k_U_K#-8NuF4A)xWCks#*y?C5mTTBKxOT!xYn#mEsideA~J(nFOaB}$(^h&&ow|#9>j*4fm)r1@tBz>rXa%ooMta3 zc7q>D)StMkZH;|MOW0)5T8?DK6Lm2%1Vi!tvA~DL!LjPjmSlYw!~pJ}h_Hm?E-aApR_H&a-lX zsMu|K$@RFDZJ*G{lFjL|#`B1M_e>Vd~)z+G)h z%$DmwzpE1FEwsiT7Wr_?5og2Y7S;1gZBe@PVgcfZ=>mlz3u+%IybGDEWs)b*d;MDf zl+ZkMaUTj7n!j_dp?@MT|G0Kmc*N)+22#{=x4+suKpdb3m~21x*=5+1(+yW{z0>_XJgf{(qMIU zEKp2)ApT;6_p?6Ch7|xoOJKB{gNV?nwtgxF+>@$EGX(x=2fjV?rE0E6_3Rq`{z8br zj;WRGPdj$4-T9%|opu=r@}+^$_tX?D3Ii~Dc7YE+qYPSrGgGn-aE?&-6;V&U_jU-L z@QksQr5;s)BxDTqf0nTP3X`onX-&};umahgE|@bxnuK{8&;V%=kk!3p6D`gU11auis?xac7#823B2 zl@LtXaJ=&n7Lw2MlA>u){vWZe=6PcCb;8BrKfO=dZ=VZD-JMj{YHZ%=Z@K|LtX?Q( zm+a#2WO*uYKygRn{oY6LQTrSfs^VWO4{KIB8kkg#2RLPoVMxCup;d0FqPi}Nrr}+k z_A==kpm#CwO5BRXME5dnB3W~fpd>T|rwzb6{be|xqm@??EyxHgna`vOjOdS0#bn44 zQ#e@+WFOg<$@45C0X0jBS{#%q0z0Y|gQa+QHpHZSNt5gGrX_%nx*Pkq?&z59Yj_dS zHdw$GS6Q||EDmgOnt0)9M3@xfuUrq;|2*>djld*5A`<@4m|fGXaqkgY5CDoqy8S*= z^Mi*S{;Db0Z}#?)bhXe7;U?d{Q~v$umsXurpeFtex>_FS9X4bry7Oc8O{c&&!h_Cv z$Q3=bB{gs84)wR$h}GVA+;=Z6^|1uAfdt&$pPW1}20YK-ClB{&9)ffZwa1)#mEvh! z#8nN@n;~Y*#&9Lf+0CNdvJmoXGKOrQergN}d?wlTI-!lBoJTgtfPWzha_q^L>tlWv zN!#Gu;ymFUpXKX~g{Bonu(!UjVkZ&AZvQtiZAyCz$c-u^GI_w!uy`TWjvp8I~>)Apr(9AuK>j2@ZQE6AE% z%TfP!oUL$qi3p&7F^vMdiY8oF6v_6ikyK&MQxp~pCysQ85y<9kH;ILNE*ow9(rDC7 zy~977>KC9TOO^QA-?<>AHahyZC3_vpuQfhu$7Qh(asm?PQ&)Y(CI~knH}1bC-u2`O zk4R<^u~Ckd;L;z(X_r2qBdrLr%x|#F-x2Wj6MZZZ4pGLH+7pW|yljkH>Cm-mt|_~D z8(`BQ#R}&5oMq0>0YfuuVMAURQ@CjL4qDI^B~mSq&qnk-?AGW z=l`h?_y@bC_&LW=#$~@#Deg2!pc0Uyw3#r8T{5w`FBEvP=-cs_9mx97b>aD%Mq`+p zRuw?9cN<9lbm)Tp9%IuEZXGDwEFK@dL(`erY3@2~_A~aY2H9Q_=h32UJg|*E^do~s zSprC?i32a;1OIrO%-Z?`9jM}-f2E1)!%&kQpOZ89xu6_Q@T#Sf6!Uzl7HSeH=<9H^ z`Jxr(4tp;}^3#fR$>JfC&e9W$F8~=utmeAM-yvM~ZJ+5Eb_KTAv~`=geVMT*Zp^Fa zP=mK`UEZ(t@0JEX#rNW^b^46@F(X}vsb&jwDpZjsABpo$zdk=$&JR^J+P>1veU8nD z7f_dh5&GvgYu_Q?n|qb`P=bkYon-~sJZaN^s2%G}G!YY`(0jbxW_u+Kz{6A!v<@KbH=5xaS

Y7E&zOX4BI{PFpD?Uno7W<$5_ZcwFpDjFi{eWq{ErLyQ`#9ZBe$X$b&2fik)aFkCUUYkzh} zN4DM_!2-rixb+#9t7($-`*0ADf2S?>d>d+!B1r>xo_Zdd$?2M* zW7&EAE^9qs5Cf{_eIw{DxvQSguH@@_#I11CDiND)=*^1^&tWGPxfJW2OZjnrW)ek7 ztIUd%hKdbtxBpJan}zQk7DVV{f5Rk}=y)Cbv^b=r=#-mK?P85uEoY7b8?x|rt_S88 zpNtS(+b>9o-r|OI(sCob?;wQzbQYAMz7dtHgWG5znCOmlHv@W$wmq9rKt&!W z-|5YPX11zb~c^KlPJO%yUHGV}zfBnfuGBm5BapAM1;F!28CtRFt^*o3qI1}paBLccax?r<7f`~g;> zzAcU~7-f#E_3I96QUxQ$$kcFyQVz^;OsRCV91Knj;G&(o_xi`m}!u<74mA+VmV<=3OS+`j2$M za+&4>wc*Y^cBh(2oSO>Huqf+=M^3`ivp4?)=kgSPWQuxy@oKTOH877c15+{4JYI8^ z#_6;rQ1D2ox%U+n_`;By$J(Z&%G>=`!*k8h89vozO97z=S;WvFf;?1lc=`1r|1sLc zZs$bxd#@lIFmgzz+cRoIIheIMeaG=mc%c7EEV3(|LC5N3BI`6x&votWSZ>c=uL_sM zrxQpxi5G9l*Sh~BHY}Q~Auby-*C)}IRS~1EjF*ckM(bQU^4Rd>PpGX0AeFxceNpb` zSB`59m60W+jp4^{2z@}Zv3r=BJ_la(idyDYKMGu08RbG6#TwgDydBQ!(TplMPbc1X z)$$+{oH4B!u|K( zEQ3{B;VBx=W^|f>ISjaQ?QN;-8-BC^B8Eb!u32Sg2hRdF)6S}Je`GlR*+p{yrwQMK z)i=|Ff#_?^9;?%)TG+mBgzkyjhNkgcqap+L``GO|F2T6V^r$nv4(ua~oH>Q3l&Cz8 zon3Kd$k*=GgJb(WqZP)X*tv`Lt3P42(>OK-OkBc)qNeZr5tp%?HZ&sX&8|%e@*;z~ zA-KyFL@LEr=Uf9bSd+-JG(k)Qx(4TyF^o=LPPzYXHmQ*=o~Kn@_It5gfupOG5 z9E#NxR|_^)P4O9K;+h0x1X82)-x$7TFYK_H>{V$M{9BDN9^8AykQo_%>bz-t?FLV7 z#ss{c-#qk3;rO7H-G|&}@F;B7zC&ttL>}Bk9xRQ1S*g^)_^?=N)Zw5K!@O8`}Z}f9iEtt2yTMtrj$1(3)dxNv!u0kb zKx(s2G{jk%9L5=f8-_<7@3+!sUt4VY*fOSbEt&;3vz?mw8LYswIZKw&9dh!AgfGrk zG5Hy#=nz5>=sM@HIZBf)?pp9{_-HLU7&bYe89P!h00}1~!}X2~N>QR=!rRvZ{F#dy zJ3Eyo0{#N8{)CY5SYYfJi+wh2g$|Lnv}zLb-?M7+>9B1QxW_{1RV}&st({6MOHjXH z!6qwz6*bi%H@BU+Vvw_8AhKE8*9mvW_@{!c{1@WgLZ|QAezn5_6NEu--iW(p7DXRR z5tC9j!{25vGI!dt!Y-Jvf9J~e(PmG%kp=F~(eoe&EBF@S!G7zhN=d-)Y;cZWoC^7O zR~u{Ga}vzdks_n z!R>rbA)^;hLZfQ6(=xmoy-?ua=_%RVE8ai(!*>-j$ z^JBOJi2siZ06=7MKiWN3K{??&ku3J2s)NMpgM8N+v;B=9KHUH0a#U#)&aB(#=h0P8 z%04m2vuW9ifL=vWah4qHL&XC*8G{mHpjimQ$4 z<@#AYg)#cA3C3iOF=@B-ICoP7Kg4vTU#CE)pTjjpW?JkGgyYdy1_g07`F*vOHwxZl z-;?WPS7+Yn_Q5&%A;K(Lqc9||C1W8|GqTLQ<&la)u^wCvJX2Nl zr+T%t2ybjt`;*gkGVx5maE~Redp+fGJ_5Gq?oixyGvwas-NS2{%hsiCn_;C7{?Eri z_mxNk^Hbj2QC@QPf3^RSi}#nwcfXH~Dy*EP^2_q|w&dC>ozCwC7jc!?D4vMw@KnvC zrLt;jxFm|;y>h&tnu0m0`&<0M8i*M(Oa1)H(#@Z-Hq7$yiLc^Yyk$brqSeF#_bN$m z(fI&|XHE{vmGf`q$7Wpu{ni45s&*Y(b-^Tq;PO633=LQgOa?4cS}l@dx`cf2kGO?W zv9eyeY`VQ|lGNHHxiNEO>Le?YCa-Hn0nAi#Yjm~G`^2;!;rJ32LwYOenB2P>W5^kl zn9LTK)fv8gZ0fyXX2~5gu(5p>Z$OZ;s8ur?%02hC1Qn1WUCj2

SVhvYJz4=6G*7 zA-Xsv=csgr#alAV&Gefe6qK#KW^Nl@1dSR{nK)e~J1RO{ag0~i;)(NHC}uiok{E?_ z;a)IXR38d||E3E=Qz&UkT9Z_OqHBtg7m<_}-KHYUh%ax;S&Zy)XhYoJgo|)O#tZ;D z?q2Luqho3mPC~M_2QZ$kLXZVlpz?hQPO>7;$fED+cM8Kx8X{uKx?*Z-<6UepZ<370N4C9}97Z|g zpvQcuSmJp8VRt89l>by9b`$8LCs^jR}@4$)Zj!|_* zX@Z%s37jfPrh11lRn;ay*S@K}hbr9Tij2HXCKPHI$_8xmA2!S6>{<=aH&}rFwiu!? z+p4`!?jZvg9}qF6kT(~~pi=B`!!7-|yauer15JP$L!*LUi3d^;c`z$XWg7yl$fH^o zEI#XBkdfyRP@M97*u{=%DPOXAuC8xhY|*iYPY#ufE_WJl;QgjI`*V++uClhlqs@Yh zi;yFoH`XCL!8ewR6=qdam;GFw3lSzkA!B>{J zC~FEt=a!qBWvF+~XKpg2rkmKn$nYP%xSl1c&!3&+{iTbh_-< zm&#LRfIxr`H1Re+a)`AhiFox&}p90+JQ4C_YU2AB|55dDyv#+)Kg%>(J zjkVSKF1JkmQRqbG><@7={tPbTP%!7AvrZ`Kw`5K^kYW1}C~W03mR%c}H}hb?se4fO zg=s4U)kt%C4tg$S@Aex?TtGbR(5|lF7E^RACP=ZVJ_5`))VZnvWBY;SvxqeLzd@?V z!rAA%dmq;1=EX>-RjXW9`Pt{$YSc2hKxwD(?2{Z(@$u+WgKighY}yAukG#LRtZ^BR zP+s5o!lO8#ZvFNoPwrktYASvP^O^gk_2q*!Xcqpqlh8)<0I9_)oZQ)eRL;WO#nfy! zrtcO%cy0Q%EfEv{n(+OJ9`G5#qD83YFU(woUY;dUv19Y%@cfO=gRWgUsxOuz8Zf-7 zTyIw6#BW5N*==a82-PK${tf^PVO-4Rt4XiT)t((=N)(xd_h;P|;8)h((992U14w~mjXLr7Ix~TMN|jr#JTG(1*ZBUJe?mIc#drDnig3wPa9bO9-qbzyARW+{VWKFlkmgF^2wH}iQPY#6(? zuG(f+HqW6aj_i%cf61hAC&pzWNCuGi^v|kN1X3hYDg4krsnz@b+-vNAos#$aKhH`V zWa5L6b63`>F{tSIFvmZrh4}Bh#E8D9e1wXAiaD5jGZ8>$e1z+wGO6-!1P(Ek;QylOxd)zbxVHj0#@%? z4ONq)UF0!DBvbSKW6;*QhM)_%4)X;O%)aP>s@=vM@dxtCzoN@7>(TkcfK=KE423c>zQZ_A)y2#!gqBZe zd9Go>xWcEl>Ay(Mx-W|Ubl*zd&7zy%Zh4E$h#~5|R_G9F^vB#A^|NzakcgxNkxe1* zGH>)JJfUm3ETrEg+>TfEf`9(yj^uA8_@#ZF>5!Q3VX$3`4lU?BxUi0BB<6V1VAsdk z+EB=-9%Z%sRw)9Uqm-W5kx-po^XUAcVB~U5r+$LaBukC_Pf)3lIXM6F^U4sHvY5C6 zJvm0H^kQP^llH=td>!`FDFGNy)exOFVSjTXT_M@-scqef0Tn3AlxS3#TN+AUYb?g5 zBzZR~6xCRh>7IlNtF^1u4qn43Ukm9G;||hIo&+%nKSmHKi}gj6se~eeaDm%YpLa3I zEfW1QoMyVlc4-MpBz#JItgmt{%=nwJ(XmW+P&X#&W$D!f9D&&T<~sNRvVHr=RdRRRk^7V6+L>V<(5@2M9&KzXP)(N=+TnGiFyK+(w;_fBdBYcsC{#_(RBU}ln;3*oj(|>w!H|1w$MUDA}i~|9lb%>CzrQ+ zO1zg9mSq0Uqalr8RP4%&A@&2%V#QScH!IR$OC)Us{@BecmA8-p(>nu zXG%6=5`TG|0BMA|%{b{|g6BdqG>U<%Y5VoS_Epn!NwjvLHbIKBwT2FJ)g1XMaG9ni zSo=6ww$+iUR&W*4mnf{wmxfl6;D?K#n~#(KdV zzTZhrR6Asmvv1_>$sS)TTO81V=XiB7|6JJN-x~ToH{()aqs#c(@P)}pfq$$!#wGny z-hetT+=gHS_0X}=gJEzr=2gklsJojiyLBT7W~1gjttaQp5$~}wwr5-1(%-_$?$Pr$ zH`)?$Ua!Mla#TqXdEX@V44)kKu;+N(0(n&t%F_-CEBTJ_*-Y)mL8b?yCMO5O~8B-KJmAzDeQTd$%GH7>wIGROjLov~F8p5u!M z58}yrZg6S1uw?h>jhr*_V-3J6v}8lens;vgv}pC1Y-|siJH5LOv_M~OVyz-uWkJR~fMwo*;Cn~oc@T+^<}(U( zx_^F4&t76TUkJHJ6Z&R`=MyV{CUcG$BvzyDbu?#Jnkoh-wTCyXvQOc!>&YT> zl6dPyY0lwT<#(xwaw!l2vO$#R=Zqps#RY9a?OVQncyUv`$2#JrH3MTHrSaf+krO%9 zl^mN!szKzgRf~@}Uj#J#u%wpgvFs^0VtStBCMh0|Yod88T=U;A6u7BKjOmi)jFddii z0BU8$Fru4kleZ{)BbNfwJ*SO}-KIj_kQ}m!y+TmVq*J>r={8pk^Yb;!TuLGIiJ_A5-`}W!c zVkLCk8NC~eJl#})l35Cd;2iS9231cWDK=~gs>LXxgMA(9JTiaPbs}WwDc4}WAN4yv zq6Tqfc{1xW$zBDiC#*-gr~VWp;Igh&LZWOU?nuX;b7Fl)<_QHWLx^{?Bt8YsqCr}q zWF5aI%>-#?g&5$Pq!ZCxW4T$rhGZq{e-X^`@DKB5dDBd07ac-))cguS!GU}an`uLB zg4w7zzjORDTw(gZ;#5pTED6{4o~XrV5^M=~#~!KOJAbHufr7SOs9$!+&&P?v2p>cd z4SEB*A-82BKr2$nR7B+m1m9cjxZa^9^nBXbE__MtxG*FO=YL_HLC&Il ztUSLoPZiqG#7HBhM;SXixe!5~pdt+>dU64}YRQtEmNfw=Bm4m`-g{k6-NCQFv1?i} z-E@xoYLKG@J1k1G-Hz_eaTL9`poAPiy)tF-BT7~Uwu~eex~R*0 zZOhP%v)dMxF}S7en6|}2=@!xCDf)r38{B5hj8aI(;vY;?wR?nR(hpZN(7p5n#6}y; zo>T+))9hM}^psg-%Zq8U*IIw8vwD?^`Q};(x*9|0TAUQme0`d7?(zF0(Q>>HK{WBi zd?)H-TF#}{^;qLA=f5qZQi+iFeX;EqD{$z56U!y;)o(0)h6p74Dr-ogpdO_7J%nWV0{+1X`O7bqy6a@j`Us`az>S}|>9!qFxw3u;7o8}exr zJXCNp(+pGyqFl2bj@ZYOmn;xvq084b(vg=`Y6i3-49oJ zvhdo%R=B5VSo%F?ta>!M=CYAvUwvsA=U0x|uKA)wW_GuQSyr}E8!v2A( z>yn~|JS(hIcjq=+h-vgRX2`8N?ndjlI@&`3L~g$rUYv~j?)y5jyf9wnisWTj zUF|#%rn#qWQrp)`^&r-Id6Cwb!BuM4i}#>j@u>B5Vt93S=nPFWbxpJ2bqXf@$Qo3) zsTR~-eY1uE<$GGF#<351@FCSi1hK?Kaa%usLK2$sXxg4=al2MTyb#cSz5}=Asd&Om z0Fi(CDll2v{zged@XsL&slO#wmG`RkC*(&RYWOUFIkkysRx?pQ1G~>zE`5qCq4a4m z-7E((^N0_Txc!B*`=LlMuJ*@gU~Yk|dMg^RQEgyL4wQ8qQ*#GsChc4ETE`Q)Wak^%)@Eu2}o?@`qwthyg9Joaoo`wDh~sMjdSv z_UqSe>~p?9CSKoIP{Zfgj+Qn|?FBfIW1H1$LTOes%_RWfR{>JNr z{UjOXozK2i^H+#OATWPq(L(2CO+4RV=7~ue2}2fSYzb_d00}p%g(QT(nBAt^5z6Ge zb4Jy;hvrhuPvA>?R&0d}FwV1o-6E*bIIOr0*BGK4(`1Opw_-s4-Kg^kbLzYlwCc+C z8|UT+@bzXT`7KMc3L&(To= zvv9TXhWA-?JPHcU3rd7?zW#;zb`H%NL@W2;O0d-S-(`-30t$IH!B(?pGV?XjIj?xt z3BKZ$s^q;}aV+osDQ~#^bti^Z`@f^hu={pKVg6&r_KxSnIyzrf+v8(AHf zbZ(OWtg`k$5y;7*{ah0f zzbbvRjpUWWj3nu6mX<6OQNBPqUab>j@Mcp*W7*TvgdjB8&y1bfC&9Xt&qsra)bc1& zm^&>{S1@6}F!!4)?BQI|#C07a9y^90>qTsdY3UMCm`e@7DShq}`x0JZ&6-n=KTTn% zN?q`b7N_V%;dTOZ4Rl574rzVd!+PQiZcUGQ+Na)2sd@t9=E|m^SjSD@nEy2V!#qk+ zUNi{&Ig}_87yvNQ1^f1SQCHS7Cf`W;s!=UBAW%q>w!A!%?8W(DU~9bu9V zgqe$IU1DmMVuKDf$*E{8^j#j9m&zHe@rJpODTrhpS6u{8n?Ef2HAa&U^Gv!u(#TfK zq0o4&QAp-Gv&WtuTMBlR(dJNlue3a|?2N*iYbMP{kp!?T{g3{tLgGW~8AZhZ=<|y~ zKVN_Z2YZ=ZeVrF`cl4YXxMZyr$ZnWj{Q9RI?d+<7O`a@zEF0yR4pf8au2H^1S<=QLP5u z@7jGV7pz9O{d5m;liSid=IZZ0e(y3G$M|_e{%lm2Er8mmr7hm30q*B%B2)F0RAg^*I@1@5s3m-r+O6N_34hX|j)4Xc-nx3iK`E4mYw?;6JBl`B}M$-K8_$ zh0SUJwc5%tu95zBGMQsy%`9N*-&F2AKk}rgT`Q`x93ki*^N$Ap7p=#mQ__$b%PwJZ ziew_-NfG%X_M4jPMahySb==#IiOqX2QP<(_}tdfhL~g zR|6zp?vr8!;8vgF7^igDg1r zjZolVw+%ebz3J{`Uo&sp%;+P2P?7@JPQ=~CZ`EoYEP^yt06aTLS#7i9h6vNWW93V?3%uhH(TDTpsPR8#wWTINf~0ysN)~ahJG^QekGOL&oeu1 zq;nicR{`aWEzpwxibDmb zXxnT&lYq!fm)F+8-)VW(C4Yt`w5A)}`&!MDvHs(s=GWspotl}Rav9(5V}x~u^~rW= z!lrJPPDN%0EZC$KL|mx7E%|^BH`9N949ee=yHZ)75^097z6<^HISfkgNOwc&IjGa{ z5K+vqGw(3b*Qw4pPi7Jz)>%s9FJ@W~B9vKeTX<^{3Ubz>** z&5L@-F|ZJ5_s2-w1khQE$!HjBU92;xS3(_)^JQzUJ$e+^=4NG6Ow3%Mlw}QcQ@zRD z`u$qa9lrQ)btf~DQOU20DZ3H&YHH?i^!{vPO_sxL2xc4??t2AYi(hzOUUe+pGP_4Y zQo=3e4YL;{P7Q7U+Y?6?!(K79CW3I@v&(8l&ctZlc)I9*6H+hm$z2^Rd6}~_;CZL#K33d5H#+uaBG@=EA|TA&c58h_&;$dKm0F&4q}~i;8o|9c#5jD0ZcbS8RI1+?FIG1p}z>1ia0r#l`IFM9?W5g0?vD-~VvI(OM*| z#sc>y*nZ#~hF17)Q6=kSE)6AvC1#IeQ+%qBq?sg}xQFK%Qc_aT*JA7^k?&y7#(g$D z8jAf=8EeB()8wL3LJ)qsXOZOX^Wf&CF_-jMZ44{6C|b3QW7In=a*smbQf9)KJK#!cCrZfkzhN7F=DakBDXsgXU{o^22>phFF!UB(BEC@!Me;m?AlsFAKVxmSW-( zbwgJavx*$$Wfd}yZada5>4C@LN2abcD*H@N{TdZNAXjEa?Y0-0%fhc>YgWZ0knLjM zE(?O#PosEy4n~}RO9g@UgFlOMKrw!&e9%9;%1AGTa^M{J6dg9zTL#o}*{QB1_)F|F zT1ZM0DEX1dlrP6MvIIM@?`IC-v3C-tmnu!LCYYWNWP)=xU!1iyfH)QTR))r zivIhl&09F^)jhh}9K{}E=!`qYYo!H@nM)0t0$xSZYVX{^4u{;5b5%QsFPjnt+ zymmlir--n3@>A{mL;td{p7zj?&^(57{doO`u&<=CwMd|Rpm|<#fjmPc_Z*gbq#_&B zN?F?5c)hHUWR2+&!KnsU zd5x0b<64Lzfw)EAiYK`c$~F)2Uz>emU4C&7Gd-(46@1f^Ih^5*XJ-hfH&-QWB(^$A zLnw3Oh>riJZ*h4*Ir!5Gr;USu{il!QDx=iLR-9ApJD3T2 zb2`$ngVbh7JR=r&xCYSz-G8+$a;Sdtp&|2MU-X@N0N|-tBZ4k3M^56 z;4fQoM3cT#$v7L)<8A(Za8KB6sI+s?LMb&6LHdF-^w4*x_FoCh)$3c0^qDq5a#=Tr z|A_x{rkZZ*I^rSXUxiGcgid_V+|y){TJ15AU46KrdnO{c6%m zb}#0jw10TLFde2BcIV|-Mbl`-Iy8XZvyN^^zKAC31@b(HbsgsoYjky?WU@=+mJ8kmSTjSi>R+bI{qb zO>2Q9v#en0AqKOaxTuy#HZq2UBKmHZN(I&!R=tEhJ}+rr(CILpwb7R3Wo}tID^y^L z$<*1P*If_j$T3ZkQ@q%Pk%#$rKuS24Ct_075}ruVR!Fxy+QqlcsEj(x^*ZP=)A?}& zA5eqbx9l#tOx?j7n2B<82s7Owg-=i#^EooAKdpQ+EA1-N=h%Ey5^!}AI^Usd(DSU@ zeq*p33jZ0fzt{AM89uiiEz$M1vOmw;oudoRc>9gr8gmW9VteQFe7^P@@DGOa@!gBo z*u5Yko|O)JfEQ>w^C4DauQP_LX4CxIsr0die9_hUuemw@_x!1)9JwzSlIM6t>Dxv` zZu#?Ad~=xM`_Z6(UmNa!P#UVLgqry#SFItIghyFzQTv}=s(TRGJLPD4dT?_*0U8d> zslv^U6A_5UM!8Cmo#QC;_Isp1dq4KeDf`R@9Bdp*AsSKe$q8J~9eM6dz~iEFEvVAE z;nBL*y`2{f(J zP~D`H-oOaoUS{ety48PqH1K3Ug*d*xn|5^B8MsqFw8IZ->@1o^HFg!hgIr1CCl!2d zQw~wvP?$Ig)V*sqce+~ftlEvw(|Ad49j~5^jHSv)_}1FW^gpv(ekVV&JJlf$G(gb2 zz^FcRtX4wNXiWGujf%P`N@+G1)gzJTevNd8aNDx`vwU<6k4PwH7i`b$U*N##lR|gO z0>6bG(J9Rj{#M9p=#GuQUgQLHoaU+jkrR*^An4C7H)fI?YcX~V&(s3h`WK&K_Z52o znq@b_hE`}=jHoFR6rdX7`@TS*vwRsu8Uk+Wl{Eem$T2i_YD<2>^2s*znmDfJycqFM zUw-6_bBG{~5JkLQ6So}MZ#);MFXbUhd{p(FlTgWEe*U=+X?TTce( zEMS<-;X#W(?KAawP2&2a3yu1McaIOLTMI?o(m>lmotwWWw-fL4k^O2cE8kugyAsuJ zqJQC4PH`hFP0>RWkCTscLOVrkhY{`vFVW~x{ZgJ}lXfQ|$Bzu*Z%@4T5N6bVkh07! z1-mlrBpX4(g6NRC$~VW0KXT`NvO?A$W!X!>W{l&`1T!vUx#|WZ+(^8%!wHHvPH|yw z3S`R<8Xw_zBr%?d8LF&>PA)&bm5X{f7(bm4@xzm-#@eF+PrMtq`maFaW67FbFJ?#S zWkvDmmf6lzDCaM=qoI!Rs)bCJm7PIn=sb2=(!84z9G;#GCDOX3n&INH36ajW-`jgR zaXR+ZXo&GeC7EIl{QMy`OA;e2p{IjBZ3{j?%)b?JRO+jrR03u`TM;nt zm-G%B(w}>mkTIbZqK0h)RPW#OT9P)C){P1o@bez;#d;=pukY;-_SfW|Pnihn8wfI= zMlwC;9M+n_;xR);o(UC;6?wPH6O|HE`&Qt{Qh~L5yja?~iqU=%Yz`Y9pvG)@S9L@VU*?{huPs^lEYm}d5F0W2uww;^Hy>qTDF(_0Fy6-syG0Nh{#( ziuAo6GLzAxNlZ$rBXYwjqTzqQnP>8CYF;wkpI}jTD}*X226P(ZeeJ#p zy>O!7d_++-BRwJH6>xKQKKx}$^)t+7n&TfSFP+DrCu-#+=;dJjS`a+n_H~ys7Tk0v z2J0`1NE~aF&`G3+pHPv>Xj-3t_hWmjt+1TM=px##b<5-HsD|%A zldMkc{)6Xb*o1Ng$<3K}^krOilH!Fg0S@5P1^^7cj>~Sr-x>C0i9GOZt}_t1GnmP> zHTxTj6o$XpuI+JxOjPywpT--1jZs=i%*g7-gVNg`3##o1w*(9Z~ zSmUcTaN9ZV8={e8uSQDQM2w-&h3bD=w=Kt%+on)Q&c1|`BkxjeyfnmeQWf_?rC9R& z;yItR0E&|04iP1Z@vMG+rjpD@ zZ$N(RcEV!EpARj948q)cS+-)iw!^#<>JCD^aExXbAlGvSMyqNVCY^8~bnF3dSCB;K zGZ)GFmaw{fSLt4$!9uwB{0BVwl|ZE6XWXbsNYiB5E~M4Qx{K ztBXna@ak%DtUB&RF{p_~rD%PoHlG}yv&xz`wDpFm3t7Ql1QD*dM>W0l0J8E9O;hnY zB1x>}NTg&12vUv9!D3GYKjAWFe55zJrDM=VvhXapC9stk{>ec%eNhl;H!VfTKk~?B zdMN-pOEE)?ahW)X-_uI|k@YCW9<%UCF=o+i!aHRDwar^#`K2$vfj%;TWnwFe`z+7; zdVP~3KBkd?l|OLR5XZ`E9~cv$kO`SjsCldnoPWQwQXvzHUfl6yUvp)(pmh7S zJaAPI!OeR}idft3zzBEMRm|x9>%H$iBGC@{J=OLY@G`N>9=HE}$6xdp#+BJ4tsa77 z&M=w==3oj_P;AlpPX08C5rJ05f(qw?yj{up-5Ml-8Xo+Y{+2RUtx(bGvH3kQSZ?oP zivvLpg6o}ETT7}fplU_LCM;{WrOf;M9q=XTw-j#5uucXsuR#&(pYf_5bQuwF+mUkK zsq6Luw-0c4FAAi1v3rV3tdV|3_@r-oxCH)U7P340@?;;zUh($*VRGY8TB*~7SM^h! zMg_c&IZ_(JGr#_*agSd`z0iRg?2;QbxUWO}Vb z!>>T0nn9E9pR57RG)iMec zqp!gIxi*hJe2c=fB)jh3KN*nGEjqgkO#^D=KvDQKc}hNQyFFrDVLFLfKTM0zoNinR z9hpm>UPNmP2RrH7!lTv4yB#BsGMMqJ!Qr1%UegR}mDdwlJ$`lgx=6N$Ovqvyq2kG2 zarkd-gmyio1DQqOx@{OC%g>TXQm5_h#qqB1!11p4Tv)>)xMy80FyFBm|AIJgpD-g@ zk~U1Txq_q^QWD{EF+1cw zE!8e>ms}9pFj7OXU$(JFrtS{i2X--v?4(EGE0ThFo`%@Y2wJpm&BQwdu|-J;bxk_L ztBt&p#AUMS5vG2dWSF6;+_>kp+-Po=)_2=>BYZ-7o@SNQwxSI^3J;gbbzC*Jis~r( zodg7c53Vc$Zk~-kS1Uprb~>%7i*7?P7I?>49;5V}&nQ5X@~TKv<;=&J0JG`slGCB4 zgybT~iywAC0s^N0hcPXPn>)a z0q>@qyl=*v3cPbicvTEcScfVL9eG8|-6ikbmVJgX`q1`k!>;y^8Xo)v`A5dnZg!u| zb=`U#kxsGaz{j_OdCC*Aib6#kQqfH_DLo}=;}qmKW3)v8hngv`#fzRs{x z&s6EuZs0yS^A<~xXv4$D?tH)mm!Wc7Zl+KCy2-5-(DV!yPl zi@EW3nbE(L^O^>E{!ghg!>ZRYgQtYL!;O)uN)B|4di<=+^h0Mt5aR^VCo^u~zjE4Eha}J$E`h;$ku zA;nOYV9;&&I+;D0?Dcj#HV>h;%H1}f8J2uS3R?~V-g16-aV;=O|HQuZOQ-eE=*J+N zTepg|voc-r72e-f@0IOWWEwwsMGDzaj=8Dx4=AvAZ0-ax1=~Z`vabT_uUyDi8cw8h z%o8>nw;5XaZcU+<4Etqs@r<)nx_1^jPzjJ3%tWSy7&u@)EqAsN?>l5i8RBg}i{sw% zJ?>Q*EpJuuPv&$2l(|zsHuA0*&R!>|!C8+sKF#D?veUdnk$uoOw^B>ViNSz#azXu? zgxR-X_j1Ywr3yoir)DEw(yd&5DT(av?tN&6jvr;(t+|v4uYSzXhMXFBwb1pL4%`6XN{XZDk6|xK%)D`OOFzOKF0#L#gC4u=}{XL?`<9 zO`X3Pj7g9$n;moW&7aUF$ri>Z$=$}VM)j90CeBLIQ-Kp7f@_wPJWjCHrHtn}hst7I zcQE;hxoEGlk$*5B0NX($l%|eK;iD%RRo(qff&n;3L6$z~#aQ&V zg@ErQ8J1mM5D~zGK&lRyWD(L&37_NLXqU}dhhIR);i z-M3X;zIqob%TCPjH>U=xO4IiUWuLU)yi*0p_*ko%&wr4q`JYa>ZnubSf&&#g&p1ZO z!mL#Ej7?5<+tdW3cr#C`z{YM;6W=CiZx z@oSa5C8D1?Ww2}d$VZNpue#eH(M7Ny0gr?v?ZUF@IHPaLH;f6W&W#6w{4rjhRR0ZY zaYctdTMs_^cN%_o8uUAaJhqvsx093J&2GLWPqQ-Hh)MohGrf%vVVHX($345NL72|2 z&p{7+oE90qWKyK@^M^nG$Jz0J>QTO@`orZl)PFwsu4tY3p-=tqMwITQ*8gVKQ(d0l zV~KB1Q+}P#y||qiZ3#KgWjW>lF=}J5#V}L~l88;$R^An!g@tCgu0v9*h1TW| zBke3?(9sgh6KIA#K}kVfVhaMJiAg7g8-|k*Kv@wXK`oM2Jo6Eu`(dH~H96P-wjW8t z$)g&WA$Q@q9(6C_9I$JfcZi9*?57HTh~0Q|#+30e^PFC*rXy)^#muHX{y z???t&ad}ejq<@_zAyK2sU%A}4=XGLWD%EJ|i2QRL*sQZjD-^sYu!H1Cp3`o;3QqRH zTy`l2!Xr^l1C%5IqubKXu4`NUOZQ5=pVhY0$f&x-#}lLvlhG${X+_9#SIMF!F>+)i z4M{vYa;aX)i-J9{ZFDm&W^)_)zK)9X6^&a&z`7#S1%%5W8HEc*_|_=$)EqG8_n(*8 zV*l`kP9`%kVR#;B4HA@;%FF(=Y^;lUV%ioIJy20C6&>{a=`*GY9Q*W>KzOYh_rBLK zK#Z(n)fym3(5l|35Ou6S^N_z3-7riY2CMS(d4o@Z z-s|nM6!1ZF{FQrPZ8$tsQpcZX3(mLV#W z6>zWV_`*|qO%Px~aOzwVN9Ns9q9I2SuGTB+MDO;7w`*;x@yO=hVQ{VgF8I5Dr+d4t z-%y%2*4Lddtpz`a*6Of%-E&gYu*}5Ee=n0xKw6`g&-H8X68*ITC2eH-E^;rN1nXC6 z;ycIsQwV*7G^XJK9+NbXUq6U}FXn9fo zxat7^q8wc_+=O`)@aH^yvPo6vDpRygmx?aqX5+#)16<;ACnI#3w6oW-W$9&8+GvNU zS;(JzlBZuCL4N4fL1v=n`YS?a(Gn$u`e541d!4vz(f$++`8%XuYt-e9#z_TQw7^VT z&?zz>U~YQXU}S~bl|MiY?VPZY(tJ$Gqwb~Dg&(P^CqTCB`HMj6VDt##TF`IY##;WA zy`}Ej_2-L*#q9iKYlG2h3ib_CxjCH3`6yMrG8;qRy@4nSizvZ_>;5j{4TPK5&nlb zx*CRxecb>lIEVGtHd2%O^Prg0!C2s? z#>JNjMzM_zBNE2+4IL`|j{p{e$YMdma?MSM{9evh*CL;>-yZ&*7L_+C`e?zA#6pUQ z1kT`oeK{HLrjPVHMB{#jKdrvWf_&H2&#YKBmyxV@#Fgzur=_2n86R7Lddg=uVOG`{ zbGJBms-Tw6#ws<<=-_b=6|d@If&0%i_aQiHW$xFj@{vl zk|iC-n%kdN9)P6?^ZOY*br;~bh|IT(UZJLPpwnr#)CqIs-_WKQNNt-(8>Xpv&`JO! z6HDd)>jJ|tDaOLc*Rh20V@S+KVo z{5!PX0q#f(SfWyMYI!S`ekI|lIJ-!9v~IR8?lI^7!}-|GSYCmDC5f6tN;sw5JaFzhWTzp{m~X920_S;K=zcl`lJj~a;F6^@b&e%L=f zLbxbGYtroZ1w_Pq&UFa;<0AE=Cu55dgv$O~tMr_4Lsf-c}>B_MyVH#hGC2}`QRkJ=hI360>Xe`CnXDzx3 zNE`;AZ8}#SY7NR}BbHc>m!!043%X48vC+rvuyInKHL9`#G!4Z+Y0WL^l%;A4>~SBoo8q+ z<^P@>D3K}&KL*$5pc}pqF9MRTV6zZ#aIUZ=Y{RJN@CiAIB>LVmkj#&NUaePIc8b1p z6G)V0PGBKZnplh{=}w<$ew{e*br+!n!BEpJk)D~^w(*2^{jkD2TCJedg+58DP%?Y9 z2~w(cp`3z8!|3)Rh^>#xV}morI7Z*dg?&EZy`~salIg0n+NnY7gQ(rdWmq!#2W?E? z4F-_JpXOPG@MPh=>^U%~1L~!(+DTCv*7{j}b97@%-G$SojWWQrsR6;UbcmW7?SR+) z$dac{449p}WjQ1t0ond(yfO>lY#Oab{)69e9OtAh|(L2T4QIZMmU>O|A>_QR{>lWjekFdS9>sLE6;il0I+Xp?eRHSh}!Y7axAtkPdtKp z+%YUuV(02ENo;V8b#Z~^USjI+HVfqj9`p-0}Pqcl5U;w}pkIFw%C$`ZSq zxAF&-CubCY+zxeJC7zCXh!OycHMlgELAy|OF6`aMrModpLNu&#MLP}E)aP80!G~52 zU~siLDD^yduOMMSFD*za+7XCFX5`)bIp{n%oy=ltpt!nFrHVu4WPHkbr&T+?fO?Ep z)*$x@(YYm-yWE)0mZ-t$C*s3up(Fv;4b|IUxN|w5QhPeZmsu}RCB^n2tSRJFHsamMXHf$s7Z2&tlFHgo%URXgm^yZaP=Pjl4T_6Hfx(XM7a zOPzM=1Mz|h%RaYo!AgT!;aDx zWq+*ZkH+kF;LOYtCU;vgdtLPC6P-Ens>rRK>W4iW#C$izUqK?Qm~!Ddjk@quG+k$B zY25$7_gn3m@-&KlWB)9p17_clxaVM;FnY!2Ru4Bqu;qlX86vUxpM8c0s=~U8LoqM# zc@sYhTV=86J>m15=c_f36Oj9yPVHM#$+d2$cFEFbO58|=TvbcmvbFhZk^_d^5Z?j8 zJG-p<1C9V z-XkL-bXJR~NxfGc?*oUo>|9-;vzyl0%fi8%GDRPxkWZ2+aki{@YX2vZ{*hJxC!a_} z#U(r)kT00c?!k9pfQ~7mP;HCKU9KJ<{VnU8FVPj&L-#q#1BYPRYDC`6_>J(`h== zX*;7TW6TdM++VXdVN65Xz37${{3+z3!fI7)oesDTg4p#)L-z~A$=-M!$d}n86m&X& zfA#L2iOHvrGv4R+3mqJ7@mR&FrYK2uOZQH^V1p;)D?_-BuAerrcvR4n6$CB>F|o-tAWQB&5%Zm#=VslU;(m6pqY=3;(r|&_){xFC$!+E$OU{ID@Fz^^oI&S7 z$P_G0jiU-nP%)NA=U{$}#qCC7a-rc7UJ~}8X4VC^If)z)BuZ)Yy7&%PU#YCIXk(h* zhjhB68rm#L8%?sg2bJf@bVw!+oOwbK?KV#cVs{G|>nmThwrZQlr$NgHR+<%uS%Jx? zINKO1@C-0!ujOer$b83*;zB5C14(2LdSbAVk%gy~7_T+W31zhpjy$|5GQW1P-$d0H zvD7lo!7*x}U>3do;x3k>Y=?u(`Ml5A)I}_PO!^cJ8y8cDsMYbL z6inwv6AmH33;KmM0trxuiH5q{QKSlM%s?}xyBx|BCB)I zvNZ6(8G9OCvOX2_)b(d;z$eNbJs)w!m+h~+}33etn_%?s9eDM$wUS43r=-u948XVDZA!>Q{Fk@>Xj z@fhdcjq0-BSA=F=Ud=%_;gSiz>wcVexxUarrjVTcNL*{o3XDB75i8`R*M(4W=#L;A}%?@-R(JjRN5p09$!g z;+jUq-Z1#s?*D5Z+H$;$S|~c08^^9}sBw4?{h;4Hk+dGCvx5~^l8de56P{0QSMfHV z3cxGLLEuxK3kgNnQ4ZsF4G0W+g1NFZaeHJPWD(4O^9 z@wE%*$?~^G;ErRTJKejVxEPs!c(L^;ETFp>bApbAmr+OuTDZ?xfmC~9CKDLX=I6## zuj`cr@#8H5#%IyM&91tjc*m+!GS30>>-E1s$5ImzPn-hGDw+EL4NzL+F08o8)-7PA zVg_XB!jbyu_aUDu ztG3S>P^0{a`@Udl?f&PI<}c`6j$#)4o(11mL7N=jd0Zy=(?hR=ZXI6&brnod;xIQcLHSRwLZ_-*xLSlfNWIw?(z5* zJbzq8FEzk+PQ?G^ZXxAFY!%<$JyO0o2HcrKy(n+lymr0(*LdQlUo0Q~KN9Em#-DD< zjtv8*Cxhw1@8F*AS47l?a~5kwvKJN^pa&sHF(v z@mZnEyIt=nq)#~NUD#qvQBhNO$mc3bUr=QD-iSoxzAjGvqjKe+md|O6ua%3tyZh#e z?`;8TmGH~T$432adpq}9Rc9Jj4PF_yBr}+~Wbac_AOeyl>ig-j%H2WM#BaNtB6kd^ z0pMTKeO3*=H=QCkIYT0gPpKngZQiFLnpoc|U^CI{e$n~)Rr(V*J6ACBSxTHxDI zxsHG5dRIe0t2It8A{SQ;Qr(xP*`2x7dr^K(4?00&dANIK-dH?4g`We*w2|4u++c%ygw_; z+lL<9cak+pq$-g-PY7j)KqRQpd?mBf2KP5i1SqhMFih-vvnYjrNXHDb*EOBpGN?#@ zRo?G^DVg#QhyJvlAcEc~mlqn260I2AXsM713ntwXl7HLJmHPRh;cxnDXj}vkWpqW1 z$;i}V&GFAZY{Q@el3-x^rN^qM|A)&9#jKct+P_qA~A6BI2)+vcdH5rP!#jG$jW5OfyY zimD>x9ItJ_v#pz?t+Hv3Ik$3du<9}KAQ2fMi-`X1D!V?Yrs`>7nlygkj((}!0OBwA0v`K5PMk7*`bas%UfP7)Nx#^dr*uwca@DN zB21l$>`YW$w;8|p3N?TGW;0uHD`|_W;MeQ4re$9G1GUt*BpL)O$2U>_hKu=Wk{SN0 zqi4lu0#u+QfO8wjFg40lCg4u-U!0oa z5w~0Mi73$V)@=&sRL8oTGB@sx16%a981Ui_aB}Z^rMl^%qj6&MQ4q~#TZTpWuqn6Q zG2c;tgVbN>Q}ecsvHJPR_1q18ri0rv=nSFF!@Od3D|JuPC;VLjDdAY49o}^y}Ho6_*`-pXTO8vc#4m6 z^^)X#y_W@kr{Tuej(%TFy#qD`qpk|^+wJ8^r#mcyE}iB9K4CuY`9`9q*roKj4#}pSaFhfI?%ljGRmQ49-_nvcz-qg`yN!&9eX=?TtRZY3r&uk=?@w zXDl@?o#8y@fa3mG!$zp8KAI;}TIGLS>{EF40y?PdLI)JPe|3cuuDB7Lp9)Tn7C;Ks z3nxxGr>vb70u?MWsXOjRttE3^0&c&-Tpw|~HqI|#*e0UeIj1GEQil_|SK`JuZ${Tj zWg7f{s7a)%v+*^DF&A9a$PbSk)%{^Qg-+>_YLd43sxEw9!K^|#W|xd0)*4YeDPYBacp=W1rN`VN1ztRMSj=;pB3fIyJ&bsWi?H~?<$wf8cIjOT5@G<=j$&b*@9B)_w1<0h6>q zP**lk44%q&Qj#1I%-g2Lef+0Udku2cu`nM5fkLnbx7XRowm0_bZ@*aNTL9I{v3t;0vA{Kn`51b>cBW)vPXGLC zro(*L7`iCp0)Nmx(ep=+QHII`ivW2vX#9JW;Kt5OQ0ms!y?DxZyU80nz1iZ|k4seS z|Mo;PuS$^6L~FO$s*Bu;i`>=@#W{CDB1lh?2c14(1~Skj2%1VOXzy*n1Z@dLz~v%3 z^XtO~GTb?`aCUeO!JllYhhAhBHMfh+D&hC5IE}9Vluc1g@2LlGMTj57DW}5s3fGS< zYy&5b$9A1gf6E@#21ZsZn)4gww7R>2bQoe%*hq5FP2r2ZiW359miES^nrjs|oz$^u zQ(zc(!G|FoqUP>$Q@-5mz9Pn^2DG1T>@SoajYl$FNA%a$u#yD%0;uUpU!|@zuEfa& z2P}@DzWKP{eE_3>6fO6}!PT*00ZC+&wDq|z%ePw(lsb=u6GQ&sBs=7=oa4F5;~J>k zWF8Bp4I=`+3e75KCc@DwIb!h#$a?J&P}^iHzDYf0v*4A{9ojE=XggCRSA~1n(aSoUl63SE@sFY^0N@ zRU~XurVv)Iin9oV>em!Fy5P(~-#n7+f;2`c%lo;SoL!-&Gqk-|X!3zBNzXxn`?RW0 zMU{@{U5WX{ZD?U9EZyXKtty`YrD1^$i#m{%8Enc{YxYMPIGp&jbLv*%WiM@K_$mac zJnHB#nn@O+4O6yFw-@;C9Jmu)>;~Jl$$!#as?C&j zUA8@uV*d*tAC!866wtNj$n#(A^Yw`tj{hI?&v}N}LCy7Kju@nh`hbYmvyvi_&LDcqW!zsi$rY9ax$&>PGl{>6NBRW4#v#iS0M*g4!b1 zxT;_s?NQ8CU}o9(mW%P15Tg6Y%*j;(9fUbVM8_&hN|esg5Rq24nw#(mFXmYjhDtrl zEdnJP@zFWX-4eVEA%uC?eTwkJ5C2U9;fc19Kr7Mfp^eDe>r47FKTzm!S_${=r{)jd zV$w#&M8&ZP-mGcMOfC_6sb`!Vx4xl^UVzXO1(!?4rO+742h8mVChLUj!&7*7{i)oO z#v5VPdRIt;*&$^iDATv=Z-ErXQ;<(b^241m`&KoB2l>mgxTn;PZ*yr3-4}ft&}DmImbE-Hnj&F|h_v<>olw;=i@7H^RMc0lGAv zs9XT#8sRwbe;nbq2V13=2Z!@2&+vm47mQ&=uYTPU@>mv)IACoy04YtV+ylLJ)8#Dh3R@= zGnFU2&rzcy!Su}jnC_~fffJFIi!wfm4f?~+f1-Nll)OzWH5yr2LBH>fszjDa&Zic# zILnFa<0}-X(#WFF1xI^1If=mq0|TwGdiiJqY8uNy&rolaPsP^f(<##eZPR66C+){{ zTVSapq&ji-Xi)q#4bpnR3H1~x00QDc5VL1cD2UBCL zP$>Vm3Q8KlwPe&0WYhW4_g&kr+PtDg)?pX==4n093p?oxA<^w)!k8O$it>~TU{idX zxJKHaeLrxKwQ8nCnpnk8TMCzzg?RF48Iq$WP|yU17*w?l01{_n7C?n~l!A8{E-cb3^+tO{up04DDc;F$wGqi1w#0LEBO2SEhJ{dix# z#85r&{5HyupyAHdP(#6c`@}7MK!X85b3A57uwKsCSlV2u>ddifw%Xjf@HhgcsjJmh zaDyTP|57%AeU4sY-8iR?v#+B6J;q^*67DflSPODtd8mXr^iY=Ki1SdBOvaRb`N^1x zp@06BELb2sv8@E6y`e&zmzg<7HivVbmwTmeo}lVi8Dx6fGQHtmiMh;gvBsRRRJx`T zHBBNGw(C?Tmk$B)b^{VpTHXmiMHHf~6nBB!mPLscnQ#TgXr-n|HK8~^=|A>q%URmG zi#W4P)jy42(;vnjpV1XqF+>LtMOtkjuMG9#FNE=BSf+HA$ZsHRkPt7KD=*pLeiKUa zVEKyG`$*nK0avSv@py|M>C?mEx8pO>6usT6#YYDaMN`p7d8N5XhTa=kqeRj7 zI#Gi$dh4ZQE9!{!u^WE(`M#w~s5+IfvF5fe{OdQEic2#1Uv+l4qH_MrDM*0dpA`v6 zk3I>YBdx*!x3waNeSJ}l=NC7DH-!C*EfGeCnjiNp+Uf`$j>e9r zNWO$0HjJEA>d*phzSUn4UJ*sByei5@C%zxXc$I!I0@5Skw>GQ{EJT#}yuFM-270&@Op`Rt%8#;D`QMH1~8GSA~)J%XTb<98S$b^ zwXkck_M!u`(vDWn)66#4w^T>GlSd0?imp)*-dJh7JGwE)e5zZms2kAyi@8sHmugql zKpZcgzb`SKfm_2r2d)m{J&WiS+FvtIHHS(GP%+eg=-&sNxneYZ;S|jvYkAV!|5KWb z&{vP$Y_2nJ+T!I&SDM5?!;3ab-lg*RD_c1SP!Z+rFxCpM39h{1pF2$f8^#O8;BW2} zP5#^|Ehwbi-A=cW?#kDb*a<)UB4nCf={K4y;Sm*G?~}o(ERZF$@# zwe-Ra4jNGr_Qo|eC%z%AcUtq|HJz!ru^qDWKXV*wbifYi(p=Exc@%9rsN zCEs#{K-t9(JUv^RQ(4szKSwfRYu+P*5mp#45fVs&p3eL;jB(kgVN_-lc}H_w9lPg3 zPovZfZ{z;4FzIWC=?ctB&r!JUs4-Sob9CbVfpPvrZapcu9P zD$FTDe-J&BrxS)`YoinVF~$Pbao7L_2A0Jc@{B;BqVc?7_@U#iT%%oWHuU(561Iz+w3TT*$l(3c7M6)Cwx< zXl$M&k9x$Ehv3oJNSnj{MjlLhgw=wXnY_0nF~S2~A(1iXfg{-h-vtTdWP{Oa6jQ(Bq94DWiNLrpt*)BL)<8jnx|j1H4!?G^GmU?EkjnB;uY)b# zx#iBU^3+yJXJ{KhcE6)H1SaN*iOpz9_sGAp-rp2!f-z30C>ZO%G!m5JlYnu3xM#wB ztFlDj`_CscX_83#a#v1~+Z&B{ruv2?>4)3{DuqgG5SSvvE)S+8mc9H7_o}z>vW@eM ze}$26v5mn*PUwH;3ui?Yig8ZIamz_>8RQ&3Bmj+@;XQgPnB4b>{j8Bdsz7hmzQ9n5 zR%49?rHJp*qXoj(<&f)+pd;AKj4$H4T$gebnj_r4D_F9h+r|_Z6;(AhNkhvwd;WdH zKVh@s52L*c9_}l@%3ZQu6eVMvus^Tg83ZP;qs3S zGo#v0UkxsH-JgwieraVc%gN0Nl))UnjAT7zQVU{i z6DuuY6uOTd6($S(5PZatX|zyHD+=@kis1 zYkD<|tnI-y{dIa!>_X!x-?c`axD)uz3fxcKpdAjB%+L>ES=Tx%dQ><+mf1FG~oPemt+E#zsuzFh7rUJgzOW4FVHTf z@4y4XYmu70&R^k|DH*dXG)0%Eq5$ZjlOpC+9LnU$wkAhSu}s=l2+i+-mjSh58PH$poh zoN)hq#^koYwJkLrz7(84 zUQ-1pc{Iv!bm!;3v(28}(#&9*DA06)v*J_5bkiXRWb=!m<+fuznYrCG8!O+eKhY?@ zLvUSLY9*@_X6%L0A8`RyMupmaDmBbm)(Xdc?q>T(WlZPO42`a+c0$-S{h+NGuMQ5) z@Q4&C#JyVLn-cbK`eyH0p|ZeNt?6$Tpg7iq5%Rfvw;wJACf*AcHZF0!SnbDKEXtdx!g)4Iqt~%151QDn;$Gm z8$FVx(hwnpcl($@-O+*4z?;ATn^)=v>!sv&3HA0^R%>_ zfW4j;WvS7*-N?&UEf$ZWorXwuh{TUri^H59q{f`gJ$vgQn{|$PQyOicp+GOY^($Wj zPfT)|6ak6t=MWiGy+N7GHNAuwWNAqcWj2*oi~_&0`D<>*KAQymCuA2}-Sb5{-gbJw zF_1j#HTpNX8|A!8;9h4c^Fd&ah4H#}v*bRJ6kNM2SKt~AoA-YGv?5*#qGGK1fsrmk z0ne0_#xZSW24lAcEXk^iS5EB1h?tdIZJJR2J?<@KzsDN6t?k08^BUzi6vFhUs;gg%Yw&p781WAZg9n40P$`yP z$l^cZd?i-G?&dEZH(#asJ+iNQKLi8Xnja{G<&E;NcD;`!1wFRZ`4%pI7(V%t$2Qd)l1LkibiQu0;mMVoq3^OthAePh=I}zufsEc$oo#bhCimIS{PH|Hws9%oOi~s6vHZJ#<~j?46bBN6F3Z2IAYkR zO%5J(@#rFsi`I{1a~Zi8L_WTGv2?Te*4sSJB&VAh)?%yI zU8BHJZ$J9vP*SU%-PkeuydQT*nTB9h`%(~ng0wJ|R3w#qn^~lRF9?$m3H@YqIwG(w z#&Age-)-GaIA;v0%+Xggp7N`a5|k_+PAH^cRs9nY&T86P*-T1P%P#l zGh(=*655=R=VeuAaJ#<}GGFwDT|CoN_+_R{nm$qe6v?n43eT6@?-<{@OvAA(b!KLM zK___lmbQ~NmX2XHeoMUw&5&evu+=7QwgL8lVpGSYB{oh3ptE1s^!bCMngxmQwVS5ilXPpXZUZcEu&6Q?I~9?BFaiT^R` zGbcE~M)q z*;HV6Cr4e@_>(dE8}YN4S{0lsy7J6G0FR&*8>J@EMu@nBFdBTM_@0uIgn6k!Xm5ZY zHbJrw&IpzNH63xKy`ag_GP#^IC_6 z(#-gNVHhvu5mZKUpD68IXYCLvU|@t!_HD;dDlifH%B@D?Q8)z$g>ueeV<7|q5LGNW zCx1LiCqOK36-Xp(!e-id0;Z>l(M-Q_s5=}8qqG6@`>Dhx4K^^Rn&D;OKI|`2g;6yT zb7%egqQr8PML4F_Py31kj9&#`58uUL-|rJ-iNDVt(_RnP_FY-KgwyIX5WkIL*Elh@(~?uAozrm?~(!j&QJe5d5&q zKbz;XcAVi89Sl$rg?K>v&F~7$58dVt`7_fL3ZJBZBL=-X z;XdUFr`hU~4UXv0T@d)YjrELmd=fi~O`rJHYH1%n4;Pl_s$(#1FS?(H=R~@NY0qd) zCZIf@FSj2TCyt9v#0<{AraWhAMUq^4l-={p#(DcZDwNFugvH!Qrz=`Xe^c{snE!sa z3E+zP)3u!m%QRoCkin8AW#Z!7hZ2%yAG*Uxj=WbBShDF$^qR=@sVLl;Ghdp>?`&s+F*kPAnV=TkSJNX|j~-rJOS zwT*P@(GaW4)5Y0R)lC=OIRklIJq}3mtl$hru!1V!{&tfCCwnFR z`-6PVpnEPRUu*r06;ig9QP&CHfXIt(Ogu>%A)PeJsyNY4a~NkH4MXTGOWek0RQN8E zVxfMceB*#UN5-=PmVKXkxOGg7syj<9?yF_}-}Xd$yfW~3YeuQ);d2yM5?w#UkvG>> z&(#L#)YFA!yvFu&Z4k5T`aawIJ8L*EIk+S?W32U4`!{VVf&6Ydo98g$#(#(85gsGm zY4I|{ji8kirNFYVGv+#kzGG2KqE%HV8k>OG=;|F;oqT`^evycI1NL+PAEmgXGfTF>+!tCDcjPc?c5K7H76bqxs~;tIAfw~p~2fdDc)#^_>r z8jV@!JF@sCYBck&!!tV4xVv7sZi4o8mc)^K!FT>%*9~)cXnoB`U_3?4Y{=r*ZJb?K zHWGOr=OIIk*bVct@zh3SDKa*;eA|M}D$0z0Ldzewd+l=X(|mlC=3R$gb#QL@R%z zn{KsJ+pu!}x%u7S#sa%RXtXdF&a8R!qb4W+RpV_lJt{pXRJ?5jj`;NaaXuW!N3+D2 zQxtwOc~1TS>)|r414RFS3Z?xG6$TbVG-f`^O;-l)3t91LLbvEc8;K?AE871Ek@i-y z+dpSJeu)#b2Ef`nA^MloO?2}t?5Ukuix<h1Z(gSyQMOh1CKj+rB01}xyIHM%HBS!`jbRFP#I0TR^ZwHo z>SgkPCwIEi%&GBS)QwPxY5Gk|^l!pWgw$_nObkPE3ND2>Qkb4@&~`Tt6cShrB8S)&8dVZlAiR7ce-}0MN3k7uG14xvh+{APJXRTF z*bi%N!Qk6RUG1kV>br(rMy@s7xF_+te@r(+`DACh!$Xc0PZ&_T-JxwDiQ-ffiDP1HYH)U;BeBSO6;8cdS7*BNk?Xg`A=v&b5! zE`Ahibr2jgCh)fPk_ZMskdA@NuChY7Z${6OBkp1#0lfWf=EUL@OV%kJ3iR9%bjgNv zX$i(#6+IeSXstYl##q7>d{&UCqWgC;mf z6fvk?xX@dG%++k@DYWaRVlD}*WE~zlUXd;l>DXefN^n2mYY{OgwPVf~XR->>II}zK zu71kr0q$SAp<93K`0aIYY~sT+uF&~eSam0tKNk+&*w z6fC2QoXQGMi&ms3`sSYU;*A=!StFaA*I)4I_GFN+7bS(grFReJh*E@M3l%3Xe!FAB znWhySj`f_2^D|U6>JiGn7&Z2l3_`{c(W;L#{uYPt3Br&j@w<2iXNb?f2ZJz?8S zft`=}gRvV~>HD=?=aeolPzVXV8uJcZO~Zv{Nch&!d*h-i!QluyKhzF zqprwBYnRKD+s9BA#p)y4`o!UP52_HlcyKC!*iko5$Bgc^$hPvRJUG5a3#Y%kyC;wG zuPQX0O(q=5!=CPA97)a7%)vx4oV-HRv;@K+=qduYur*|nHyk*lx$HuIhK~6u%`7O= zB1EN1gUMf#Fm2pg159v|PtnT$`rv>p&Z6c62*~2F_DZzhZFo?fx$fv&uA|VH6nTsv zM)3qHBvc`e)!)sbojJ@gP$KAPepsr~;BqMJzF{6x#Ii&Oe0CcKQii|lpc%o(^JWAo zo+X~EMos^gQD%pIaBd!p=0(Sw0@!&bH&^iqNoYO#zfbvcTfi^tnu>tai4S0|c~OKA zA5_wKV^3t8xJ&hCsefk>TNP#DScZ{6DcIh@zVUUQAdxl89a9l+_s+^RVtBX_J>}uH z0QAvE(#LsX1M0A3v+$253z>~$5f&Dmd18c-1F^wud|AYOQd2zJ>(#}pvr0-k1Mb+* z@e$+G&C0l9EK(%VYbP#(yN&COOU@SGIEpx zCw*Rh0H|XKUzwFeOG^nIgGuS>BnI=8agImj7|J8=Gw=?bADJ2W4W=q+S z!{H#o}6izlfob}O=xn;q;OHeZ5b&7rzh^fTHhyi*vOi>609^IpU& zrVo^7{7wRm#W}y{zS8gNSPasyEOPF&dZ(lP4f9@sh%0}pnV+?y8bMKjjwO$fWAGm& zSi@7ETf@jw2Y5^h4f=!8ed;yM)P)s^HQ8g&<{VLb)j_Z)WyN%c%qgWC|IPsEMK5_y zE}e61_t7A=mNwt>TC>@URSyr7*p?lPX~#4|TRQtXSOt4k+Wr3n;6NY0Hj8ol-68s} z^)b=D$H(iSt;a#zQ)b9czqa>W4;SgZG`kb@mZO|W^Ys2?+JvLs9g5C7Grs=Je|$r@ z_H%|GX%@&$`?rp!4dPZ8B+b5aF6k~5>F$EEhdR@wpQ^Hx!t!X1(_fU>VM#uv!)eY? zKm&K%NZ?k|y=WN&H;}+>BmoUXeSd`t3f(k&)SxpcjE|=R$lX9~_xuz*DDoOn+Wk@t zE@TFX7vkCDAwT$N-E)=mVJv1Jm9 zEZs}CE!S_9p-L>_D-Y6Pnw@k-9ON?;c#W)X?N!Fg62Ct8${gVmH?o#ZBBisLVItl- z(MnDJEc4i=i443CDwW=Sfi0lZ7;e9C?nL^YyB>{gs2;9(#N(;#h|Bz#MxFUm@pj@H zWypJ-cEQO?JuQy!#o~?6Tr5e{=g6aBkq6VA8C1DC>A-c6%~++GE~1R< z?m|m~;oHNHXhX`xGKqXDXAb9*24x41$?2(m;v{_QR!;_|L1*0!>JcU*))U{`_KzID z!^okm@EeOG6q9MREgiC`@!@~`0t-*nd+V3R3i(4l%*q7< z~LgWYIjOFWrOL^Th zO-}B0-rH72;03*F+zI+_x^H$Z-M_S&rlAtJ&Ql(XuIU99CXSz>K94#xm#wVPvNmaTbx&NxmbKR?3mwk1QX6@`Mt%J z-y3GX%b!bOlXp+P&!6Yl{F^-9Jk0!>zc-A_8d17;a+>bu6VHHICeWeHMfFJbDxT-~7UE+Fuz}xStsp? zokI>xQ0EEyG6O}&D$F$^uja+^)p#7=D)2ySIJKNK*11rv4D*AtbDF#&-#K2HH_Jz( za1)w@ox4XcL_Y~bcA`PtI;g(bINeIK z=tZV2C?0thdJG~CebPFEJguv`J>+KE)PJQV@-W$nXAjflvy_c`lE!GeSQe_{?@tkOZ9#i)dZ+%{bswotHQo^R{oqsqCqr&JSu!ln3=AZ->g-KBcx; z=8Dl8gSO7Rd04Cc$H8LO4s3@WqB(H4kpwQ+yYhegKm{d+9*9D=Y2r`e>9gMe z9H0PTv4~fJUNQ~*5OrxS!+Q}kIrV@L zQbOY)e}>Oh^cFRFU$_rcH?LxQ@Bt|CQ@~w8b+Pjo)Ro9+T!u6%4>Kp~!xP$9*&_Mu#!Mi!C&mwA z`=}@~Pn$1tFqU;BuiMtKsAq&=95chUG&j7Tnh4!=6hH@krS$P6vYv26cLmfmF zqMPBf2;U1hbT5~_6iU9YU-kLgYAIEf@3p%>A&!yu8${19cTz9>Rlov%F!P0E(7n4Nqe zN8Mo~mB7eq6I8jhAb@z#!D5^O;3M6L`h@eK${uJ?(aCfahlEwxZXsX5KXfQ1I)nmq zto9UC6X<8{3C3qW>n=9YBRB|<{Hbe9LM5?!ewsO1HZD1qw;O==k%8@uT6FRh8gdQ-$csO2~$ zl3jV;I|kNHWaF}3zX*Tz6a7cF3DW}^$1Yk{1WaexfIgzaL6Jet0zy1?sv-0CD0F>{ zyXu)0^eO9^`nuFHI`ukfXR>?|6y@7xydA!CFCClS354|Vp{?|#od;7*+Bhh4I=jdL zM6>m#>y9!8MHT@trHG$duXQ{nT=nxbI^7m zTIU>JdgP9@bmvC;uBE5b$?9errO}v<8VTQqcjL>uOX=alvDh8zSZ%gSfXVjnAS}Z4 zK-U*fUQ9z!@_4Xo{W!bH)S*|Zp=X-!gGJI#9!k_7sclA~J=xwI3w^bgICyfC@MNDOiD&+?6(s#%axhWB#-?u$k*GKQDZ!~4h6m$w$vv9X=Bu(X#R zEiI<44JJj%u{A>mxt65yp})#^1FL7%kCArcrw_A#$>%j0)d|_8?5$E#vUuR6ZQUKY z>SYJ>W%7^S#qWW;jU;dj>0VADuG#(Sul{O!=R4ntjj)qm_Oh3yFMQz(*Zl3k<3Iv8 zRRS8cF-#X{*;k`+dWuuSr`W#3wmK;eR8lItC<@$nx3=MeWwNdD8zZ(H+M#8U;BH&1 z3(9N%Z1C&I$k#MrUn$(H9AJ4Rug~|{?TXYsefo6rz@i)CP)-h=AfPy?Ji8sxZGFRV z=Xofnlm-h>G|2NSEkb82Q{+`3+=`}xoc(3Wu$sfzR6QN64Kbl0#V8Uj6sR~A9y$#t z10A+pwopbia5?ePs2hqA15RmNjhkmokb;vK&cBxW4GPULitkvpnKpOjUqe|zDKEhF z)@lx;XrWZ0|7h4eXtQMxV4?_xx)mn%#t;XC%kcjS18@lv`FxqzL3cF`$2I~Xn5 z5;%Mo131dO6G10c8sJohRk+;-E6;JzHYk544jTTQ$Qquri`YuZWX-LzLG^+VoOpDi zEGat;xBSy7YZc`%#K-SkHoWWkQHJ$dwkD&fTG}f+6RCKS#R-0>gt~pz`WWIPongSl zpQoeS2mBxRe<2K!r>3{knGGiV1{ewD8_SG3EpMT&%LFHQoO$JBEtGutGiN9F(<^4r zrq#Xi*iM}-C}iJjq*ON9q`~wXP%j#9c{e|_ABOc$tlt%MH#;ifI=3)W?@CPGJ9ja` z*=It{B2X*>5WIYso?2gK^3|q}mnl~b^XARGNbxf>RtqEbXV&hB#hZGI@xi)G-Zi*u zC4>^b{?{90Os+0q5b31PY|=iY2|@(&cw1K1<1kJ$`7CMx#yMGa0lW$k$jmrg(;7c`KjA8n9a%Ue96K&$(q5mmheCgKMK8i8hw8cUi?_%3LT4ERC zYqunuLnqm~h&S$dW_{wX$hP_I=15FlD-`enmgiyk9S@Lo1GyYyJMx7z1%Ec_=9$)E zlAOg$-VGz@M=l?VM${yPsly0Oele3SX@J?MLGy!p4LC%^Y< znthhX&+SnjOouM_AK~~nklzWpk;v{;O^&uT%D z5ewlOyfl8}mu?$fi18MG8kA1VZl_vvC|y{aPA&jX@wd8fJd9^_kDUn>tZo{i58Y^v zrKi?GD-f2(Qv2xYDskdmnAIpdw1H8TbjO%awCW+vX~t@kfU!yAQ_w{awqr{LgLios zN)V{uI#LD@bZe?g8Xdd{*et=DeK-ma@*)ZiE(~_iaX3B|T5{-mR)#$j@1P`!KAret zF&$kio+G2_;e)ip9IUl99&%Twx_eD25VSXKbLZ42HLmREj*qyXn(nOEaLUd9qG|J=&mlNs7oA@Uv;3$ zcoW@pEDWh_4fTW-6kK=J(gO=CVF>>FwU?&`)FlCQv2OBL-Ig*#{g3=oy);l-Pt0SC zk6tlBT}GBJCUy2OaHBIx;p3Sd*sD_)P>N@&2netpSJ9PP zhe}2 z(&3^|n|JE_}&$e%4WhwV*@oaQ8)>+}(w89t`|N%c5fYNd%8=77Z!Xx1b;5kc0>jApcRt>71ZQ7c;C2@tGM!FjlS$*~SVs$rdw#U*;c|XlzeSh`h zQ|Ui9Po5Rp0_DOtPDXTIMOubp?;MBfw~z7# zU2!NE*v4ZL)*!+@buBN1 zJIN$br{a6S0X5_Bnb*fEX|0rTWQt5*XOb|g5)WtH;`~m!c%GAZa4Lx+vd|G(peUuK z>D_bzB0~|SBETE>!}a$NbVW>LROT#i^W>x<@`kb=#!)h07+n14*ILkO`S_sCb$e<*dT4+zdx# zS}3V}GzA)YoXYmWiw~ynoIjHuyRgV1K2#nLC*q&}b%8h`oaPJ@kTsNI$~e;Gj6y{j z_ZVoQx$*r}r2@H7qU;+?K;%X5QV-+|jdatVg6o;9=`J&o6-B+l#Ofu}7t$BEjtBqw zAwsnN>ofqLUfN0*sH3%2gle4B2g^hQv8eMyf0O?Q;L@I}a?m4&^3QFaOl$jN=|2xn zMqloZiT}EECOx!yoH8XJy@dw5!Z@gh37`|o<^75D%+^f02X5#R;Tx_Kji~?Bb0@M| z{#-oqke;YtAg!9aOum>z$3g}&5+S6so|HiPh(NO}I(x_%G@;0H(PzrVHX8;O%S*Wv zMLef|jI1&N2Z0>M3Q#hcp(5Kopg$3UPPkl1(%BY7)|KisSi>f*PRfFxlRU?vV;BuQ z^sGQ#FA}GVE-`cwY0*vq=c30DemYF93?~p=EO?nN^Ko>g?zh&adIZT~Zyx$g3? zbHCZ!ue{IwnP$Ve+@Jn0f95y&cl~Jd+#l8AD6Cu-hU0Vg{djw1Jog*Bbn|T5^`D2s zZMfWz?+oJ`;;2Km*@3(DC2%8kFBd;G8p%nlacCQ3kOtx1HoH;iIPL?@`02B!)19X< zl+KK%J@~903pLEimmOm?V|N(L?0>me_X?MFhY@ZV(55i*@!nl;j-_P|Q>&dKoGtn; z$6k!jF>dTsQgsd^+79RT&{#Yuy2Bxp>RR&N9!#yL4UC0PUS!qW-ArVmjou}aboAJM zplF}o!5D}^s&bY?5iyiT-$$J4h~`&}6%9Ryu^vuj_IyV9d)T1s2JKbG*x3o{Nh^KZ(#5n%U8>CuVLX;f6Qk}t zI{%hR5}823aGaN~JisH=UjZN6AT%_DP6D23qjRkdaVX&)2!75Z+2^pv5)r-p)VcKK zk;Swt*nqsMhqM9Ov@z6*=5}m%!RR{{F-DKBq;rj#(BnR{d@ANTj`JQGdGGXE=mIOq zWti~m+m0|4C9+rlSlu2^cTQ}k|Jb-U-7`)*-DTbpfP$=Us3VC~*C@M)-lgtf{n=wd z=E5V3p;Ox~S{Sl5Bx{`KA7O)AyW4s}@_8`Ng4gmf+9tf-2lJE<^3=Fq2C(9xl;aq{ zosVeL9Y5D%hl6!43(NGns1aw?u=q559+WckozS01riUoD$J_yeE(2i4xq$U2(ASPr z$YJGR!fKXE7toKn#u-Jldlyk^1Vs>W(FJ%!hZ5 zr5|1TRGKUArT;eaSUT64O7{-0q$lc&=@Y8}MMmpsdh8$;2!<9q0X%7Ayq1$ajM0Q^ zzf@eIT@Zz7^MFhjjMQPXxJ)VK4j4xd$~5xI@6olakH~d3(>idsfdr_}w~Fp%*Kp15 zjcqxf0BOWM}CA3wVO_yIFVlRl9!|pe(;0g z>!*MEr_;au%fFdqD;$2F9FaTNsS@@lF)ib3k%8Jv2E z&O-(%?~e@E7_E=OB)(#m0S1G%WuBm?h`6pHzHio|a7F!gbpQ5m|Ca8)`|hKxKlRj8 zNB8|V4?OTd`tI-k?szmgImxNs&Gg74kED-({Nw3WuXDb5GY2+jOskp$Fy&5R&1}A7^q?TSpgLZL>{Wc)8MQF>Tt;-iIG-x00vHWj6Gw!d) z-ORfBJkoiKC+~UW=-NKHni0GIYx}IvfBy672Y%oO;<1Qr-GdK4n51LsH)`+fz_*b* zkr}-;r|wt>E@%FL&|avqD8nK`1+F;x6Qw_Bf{5+9?cZ&Ud?td%mF}ZIkO8p@3g@5* z3s=zNkCp%GI{!hl(E+;PvnHqnk&|tB1A}*%a5D6WA9>H8$PB8x+=0G30sV$_5yvNY z(gbMi23sNzHbw%uBmy8He|#&yoiqZbCD5e3H9QdzIg&+`lgI^;)zT{%si>AfJsH1G z1Oml3qD8WPExzwP3$(HeP?2Q*GZ^!MJL1r`x$-Q~;z9^1mkfy^631`ItY2wndQsCQ`)T!r`Z|MJMxlbj|e-XwQe#Bl8el8D>RJeWp!yxUO71Roz^209;kH$gdx<_($j(vqji zwC1voXUeQ6Pv7Ov?TVr@4km`#k-|937F_+6kcHm^pz4Ls;vOHJ-I9;CxA%ij=ZHJLV9^U zs^9z;!Q!p}5xx(OoKNSPv$5rRY`mUoMbJ)0#h+!lx;qZCzLn}w>PQFDl@DDnB3lT2 zoNxA_)cD-fcW_ifJN@FzK9cS|z5=gfJw4r6Om|gRKqZaEHqFOM3u%F}UFCC;eMwod zr8UZu@?RR;P5bSk^yJ2TaDGN2tg@t89zyf3T%@M4UAxWp{Pt98A}3Ky)*Y}t4qZ`S z`9I`7d{JZ|qLj9TG`GmR;Y2?06S5KZic`*khNsK|rA_+e`V0gSWh2_%9RhJMyQK)+ELU;%>6V*IgyG-cAFz zsJaz9WP)GOfBuP>(hbuQJ#uh7ZLs=f&^C0cpv2)pcIS{zdXL~Y28BtB3LUQla`*1}_4LgCOgcWg zo$i}hi*rjpckU#hUf`GNdZLUOKWETe&h1V^XF}M+pk^6I-OVGC*fA)^V^GZnbPTBy z_SqFR!dVL1rNI+zUfI#+x20O9=#kwt zv|EQ(MfwR4ndGSmZA%G6cW2Cx=6rg*J0H54x=R--jsiOe zMLIh+IOgO!kYyvHw0@Es-nmms3Xnk0lEwBkV5P}ss_$@K2XTpf&h?qLJjeLeDT9zv zudGMbKyyPUF|vZ*F$c1Dn{gW%h}?BYWdV8#bp{dth$(2!cA>4V z-GqKH(@ft!cRro1O{XVT=d*bQ<8_HMI~vg0r~`^hSNF26$Bs_Rld>0iJP8HHE61K> zd@iO>@7;swp}ZMk_age0x|TfOBcQN7Q|D!i7t+(ivuSm6BF6LPG*q6*R8H?A<-D>s znf~YAXd1bvmHxN!C(-9Fq&1Y`DT%9bGj3>>-rM(8K9Y^=Mm8@wOnx#x}4?VRDz0U@1(kd zb%Jt1TotBh?o`{o2l=m$qyu(7Na>N)eaerjDP4{ANQxLaw$-7A*a2%B@wXh{zL!IP z2u^M5q~F_jxLZT&jnIeDQAF5xu`OgYHGYTC(Noo<$7gG?14CJ~MTe7)GHI4}u(Ocq zwo;EcPbe`bQ1;d@>yB}m|IqhHgF2mY=%>LJ&1L3kXPS%Y{VOj`|7GTxG!FsxU6bqS z%h31iGZ(6@f#f$GRtR=d$gAz7KswfhY#rc5ejC0r=@M649TnYQ{n9!=v_~Bb9n$m+wY!Cd1tyF8fLCkj4R3fu`p}0y^qe1Gym&GEOifLt zzyJHcPftGiWcu}A|Mhq_KRy%EN5 ztarR;fD5BprtI`b9q%uB@D2XK;PZUk^F1i?uDk9^OG``X6QB6RCDH!UFa1*b^FROd zXzw{?(Ip;kksoh<^PAJ^>T2vHF-gDw`@jEHDf_9P`Y8-nZ2JSbUoOLQms8T<(*q{2 z>QoG*TW}`84cczS+&s^LrMy>J$&J}6NqTicrJ~?A%!Hy-#E92n z+h>(6vN|aQG>jG1elZHk-|PWuDj$_i=`AvaONs#%_?WCF^#0x>)tw>Cx7RDr)Q(vGqmTx%HjV!NjP z@NeEUa_ii(>ZD4A)4Zx<9Sx3p;jhSq!F0xQn}aZ2M9?V{C%sO(q)oWEJCUkvhitknm3Cj0N18z9l<8(ooCxYRk!m6R)AnGvRhSu`+f6V0W2aSk#pQ6DnQ!H(%Ck z|C}qE2L60!R5iF;S_pKdIgzG^YH7JL!GSv6;GLIN1c@Yhw#TLqV&c}eZRiY(xb&kN z!}z+pH5!KMcR%@GW6SHwiLG?^*m`=Z_F_t;kRIDwh;o-pz2(+i!y+BowrA=j3EGBN z{*Lj@wAaWC>*n9>%MlCm)P*sgwyb-4Z#M01jB=2olppxgKY6tsD215#5L9cLs=?%- z%eLFYIRwChz@dk8QRjLh51E8~_%4@^TVz{ItentzNRks;%RHRv=hp9?j9AXrVJm6S zq#=AA9}4LgEcWmS>$5V7Gv2^ zOkd@Eu7A6op3Nri^Qe5M%ey5F+WmgBw^D5DGiUc(yQm5N_SS*(k`@b9uO=q^@|pk(-Y4uhH=b3NL{H6AFfs0 zvCM_FCWbIwYqy$4W;i7KSq@N!&OnL+={53>TH|aBs*!`^_*bXGfN5W5I;t$N2E8AK z@zB5Mqw{`kXrmN;?|maH>8~DpRp=+i==mQ~RfNN+?U^9koc=h+nJgaS-J$PR$FlE_ zzL!d`G130i{&IMgyfC)N{T&D9I?bW8t8Qz&*07fKtBwik73yIA$*=8IzoC~&bA@rc zH7Pwq9m+8{jE7M;$)b8zdxUdfI4rWX$JnRg4xNik5-|ox47PGZdq~|La$nsar|&-q zZss+3&NIoT^j}UslO7mbNssIu3pr^3mU@ac?a&WAJn(QSk#~R=%-PPDCgNc8CI(Ie zv8;yIp@%tvT+w?&Pf~DpOq4-;jx#pY!5tLU$$iE78FD{&7<(;u4?mPHrhcju^3;#I zm2xr;0`5=Przb<&`%-VugT%03@Y_iNQOU96?2noDU9B5}{bkZNocUDz`> zJVd(0BE>^zPX=X2oAF#-xVAhN^d}m?UBGBSKXdHFYT9j9W3lFwD{n}r(7WW0oC(=2L?{cYc>_h*fzO+hNu;5dMZY36!&?O(*IG*mET1)5mSp>I^vq&U(pv8Am zbA%qtbLj)i=osqmn;YY){0uq@?acC%f<%L~2ZAc6mIroI9CBlCLe*1-FI;2wVVOCe zDjx5uv~hKfptMuY{M)^k@2MwJ3)<#N@R9po9(r}HL)vSulUW};OjN_RI#B>|2qW|| zDo15Xg`Bb_A>TFgbb4g@SQzWuJZr=KU4SBoRVjEsXn*QPy|UYfKG8Z9RE|9Ez>+wA zUcFI6GxFe>ZR``<_oN5MFQhXE^XcU1R{H4by~za@=_Pf$%xTv7cl z05{O?2|zIp>z+n?oeKfdNGenBHCk)nAKA#_SC^0ZUoWjL0u9{lBLS-SEu(v7J@hro z{?niSbb9GaUmCvMdVjr6BT&B~Xk%mJdV?7FA4uR*2>>PYK*+)bGxhoDbm4mzVi24A z*L4OCI(iJ(qOSvi9Y$>i4Q8Z4J;M9q7>9ptY{z>~@Rlw|=ljA42l(x9#Bc8W{u*3P zb8-z{ZwW7==dRa%*LQtadiv?7QA7{Z@BGg1q+k2BUyC5#@s4+-Klp<`NPqmte|)_` z-gN)(d*AzFSIO6kIIj<{fBoyzhd=z`^ozgvi?NvZwzs`4ee7c&yW~@;Nd|5pfq?`D z5*SEeAc27dZn6Y4Ao(|Nx4Z-fy4NjVM1uqlBruS`Kmr2^3?%T}61atQuj?{uzveZs z38UX{|MqXEzx>O;ykbW8_V&`_k3XK=UiQp0&xFD2pZ@8e(qH_=Uj+Tg@bGYYM|pAW?+s7E(7W`I>(YAS2?z4*f#$oFG`!= z$#N;Hk27E3lDq@s2j4)1n59LVqy@qLidpP6-NWuzB{k4oXq6=5DMoc-iQ-!Ts z)YSyqI}jEw?|vTpngRjgcC*LsyM6vu8!+mA$Nt}t0;gB(Fzg;1Q0vzGCw>I6bXJj zC~|a)ldYFU&FG&ka&D7lpz&L25P~wKhqe}t5QtGw2Z2%o&9M&z;q@mTi|uRwYvt}# z+ph-lq&-F+$j9hpJw39z6o*QBcwitM0OEpE*`;xCrC+3rf3Eb~ zc10$~@1S|e@=Pe)O!(>j!4|*?hsof35%#act$8^pgce@c{RO0J46o3 z$2v3x#A{`HBHM%_>RB0xddZRyv=K!R=4oAoc^0qvjrh5&HpbcBI26<;T~3&9vTfU> zwRfm&#OVH0Hyid{++Qiz%mS^4)NJni6rrG(>Sz| zT?ifqlMc!wi(0T1ko+P|tdVFUgow+!dR#zs|L)DWJhUmJ}9Oh(NJs{MB za&o6!BF}_#VT17nc^IDMRp005`T%50*52cR=0Z_D|7Noj7dEb`nJax+Tk_ z!VpOxNN3_HflTTYkqxNnpcb->@mq>GxToC&Dj)fa#PPtM91*14-0{$68+o)F!YB~1 z^(e}Q$g)FI$q%T1cQWV#$!}iBpSu%2y?SRlWDBYYA@dcxONcM(3e-7L=)^%}d?RAZ z{6_tu8btqM*@524WR}v(&P00A)C$$Un_f5fSeh^Iq-Pon(7lXui0ERV=1oVz?XJWh z2Y6Z6dUYZWP~~^DJ)Vhx^Ihy%h;(I@(Rv*X$FO{FpJ83}bqXL|e7sx16IrzG({f1gvTTpoV@J5ST)apZB`jBC_^Z)5^) zMkc>S;<(Yem;JGHAEHgyIe552XCk%QV;sylG~M(9f1reUbjrv53CCHUerC zq_z5&h*bMJ&dmjhj81iYFMWA?G2K0~ntuAePo-U;pZ~FXPuknV1L50XJ=og=s#j_g zF}5^opi~LVF~V>>1g`=OKpdh+KMmsmNY`Q-UMZ%DX9$2iIku_;b{r?rCp!H0fZ@=w z$kV|r)F>cNo6u{Foda3!A&@jCcMdhD9Pj9-$$aQ>?3(M0Fjh1;7;gt^ktt~E__H4t zsV%~}Gs-t8s-k(f8X(IVj3$SnEkXaiZ)P>EKMDm2h-KZv&|dn`$^+@#`gD5W*h+c` z`iN4{PLXh2XZ$ctfdG~sFLY+cAZfzNJ(>_$6+*v}CM#?I*|?#8Y8;hi^c2dp&B4|V zibc@2hv>9JP-+~IPs^nvT?}a5%9hk)ls)6KNOuvi!yNXf3p6&r1F@>=2y$B_ZX0C# z_*rEp@)g85-&+i`s3eqslSA7@*t%1%$QT<`7`$qE9y!%(cC3~bCic<+I{1aX z$)KxKU$h=t)^_aWHT0bmjdW~&n;mt-kyq1YopUU9x7peHbXtb4rggf1l7 zg=IoLE=;!L5bxmYMdh>Igq87m?{{DA*R+ zg>im7bV%!vx>E^#CJtb=4EVF%tKW<)+qTh_SK4XoKE6l)ijdK%^`mZO7%k|1%FvLM zIY`y;Y)3)E=*|areiYDQYtV$8sm}$nJ`SCwPPf@D7kcnGj7R3+lJ;oqj0tT%W3>CmP4n|9k$$G*iBvB?CD1 zg|u2_zGf!5H$HSbDFeGaQoo8_fhY#Gr3{JgfsieznS2-h3iaOn6;SMzS+ona_u@Rl^snj`@NEo@dcVa*~I)HO1Gr$x^QPQL>nPRKmx$~j0Wj#KT z_Qjp2;hX*Xd=83~k|THhyDy|8dFMOd8L#A@zLkyI8sl%23#HGxLfw-W?INdKyY*E* zV3jZV2Df)dQW0*zE)$OADe1Tv9IW0v(oky!t)R4cE$=XyK(TV$oBnhaw#;grXux@= z@yCgd7MbFmwba_@nG-gZSe07-1(MC+2w!F2N!C3x>%lkaHqp^?6YjN*w7ox)>I-;c z;#a_Uo4ZR;{e+h(G}+aBh5-!93&U5LR4M9Wpp!ehKzv2Ftuhhdi#*6|i!2i-oysNx z%8aT#F9rDd%NHyMWVN2oZZAYSOpA(}nHjG()5aRRK~OT?`so(o(WU*=hCeVSH_Czu z%^~`td6(66LSQu})t@8a#MxKS2m@UF+*<;9k7s?AGF#Xct+;L((UEG{ZZ6-%9xX1(9 z#zv7(lr7I~CUs{=rqc&WYBhMW-KzPe&0}ep$FLc`j0zY%h(BB}O@~aV!r6e3 zv-Q;Qt#lgjvv=8V??X=3Nh>+>xBh2sg)ABre>K7y=7-JLW|6y`lEdGd+1{}DX!_vtx23x#*3+@-PWs-ZFQ@;}d|BEmjna?O_tMu&4|nI(MmDYz zZWo^E?%Gf`hL9EwTK3QCUnLB6ro+E|3{m#U!%zeq>5F?9Dks#X=whKxFwYva2-HKb=CmOT_8Nh*Y@c~l-U7> zMaN5x<88-ZvZ*W_gGQkcv8;2MwV+y1j%Das4(7=(`na-lG1+lO8K?s~9v2wLJ>#Qv zqMqu^AvJ8fKz#_%!!b7ebLNaRHZ9LiK`*^?Vk6B$%k{~X(`mo15=kXQ=PqF6Jrq#E z@3u54+&7{3IHdetTz4@)3ooT57HXENyXik3yD#(^<=>Vvk3330SY+-nBrPEI#e6w_ z2a>_6Omr@)k7C80W;JI8{%dt54eiga&7{?>aTaJ$3P}HPbgwfk#F$5Q94SK%Stw|s zmq{m5gT_R|>Jt;k(#fIq^xuy^lAbx3O}o?=^`kP2Kqp4Fp(3fLk1XF8XLRfzL!Qe} z0_pm`98H}Kc(iv3p*pLnSShIkUC@G>~LyAQ8NvNjYe#pW#57eTi$V>N#is12qMBj z9sKFGFRSJzizf6D^d4boR&Qz(*P(pyI1RiN+R7mcVRb9xWLNhS6jX#3sf|os)KX0= zQ#QKl#d1)B1c+c4kP`ip3p>*JNh>3*WGreiw^IKJPJOPnSB2xELW6`n)0j;cnp5eK z-KF$@E`BDxY~)NTp6I6coq2fxdDPV#%z0Y9#g+h7kgh|^0R6c@eQSgQK$@6iT1TX< zbHPlx6o^0iK-;|fr*k^|C}*iqT;v(J+eZTSDS3bSd~$v2)TyL>`Gs)3>C3eRwX(93 z78h9x##KPLwlE&vcFrEvUdXG?`jve(#GQvnrNKZ_XnNQJb+k`Tfx32j$+JL-Da+pbZY!HP2mdxR$>K(4XpFe} z!pI{3*7^Urveu8#0NX$$zs__0p7*>bz3W}?3JQ)t`?EhwKl3v`lm7az|2n<<-S1An z{L8-_2P^*8Z~a#8{g%DDTg06ux_q(i1xajP5j6razMtLU;Jhl6Eff>cjtO*+%EJ&y zVSmfv7zUhN7x4RD7o#$!u}##pN^=>bpo*98RgyIH72uTf;F%7}O&G3FSbQG{5tVke z5~_Vz9eNpTix?(671-~bKa)Os@m_dp3uzS8g~-(N9HbHs4X%j+Y5^W&d5ZIfreb&G z2;O{pAYnxW6V1-a#?Thtd9E^~>!2v9{0FBt-{?fUhDC%lhJwwrL-OoFJUS0(iahlM zl5^A@@;uO)y}mfQX8zM~?xL2Qe{~dT54Q~CDzg?5^XGRalM^26y?-KGoH)w4{M6=L z970s^a2;gq6qZPhLnSsVe3gZ^IzWH`YVTJdBZ6B9jK<#kih1nBh>H-SkCH z_^BWiC&qiAdvqe8JQ#AOT1$-;nhfk-^WAwiN^u;)& z>i&sU785RVU}TenB1h7v*YC`b)fz8R%z2#xKP%onGA{r-5PI?97bVcUjU5V%`;Ql0R7?I&-)8bU;@%ZIrG zg3+`hwkT(dC7eN8hjBwr#LmKz4X3}8>;ic)U6}#hd+QdHJ^GI)cIKm=Z^D_W4DI_K zt{meqwAC~>zMsyn0#h?sWUmW!xHtJT(q{gnj*=JKyz$s~g!96m=``LvYmo-h8#@}1 z>ooac&UJ~?PzxzpJj;!~HUpcGQu2Mjao2Y~&zXGz55w1K?JRDe z)u~V#-F5w8<@bh}-{)cGzvH9cf2LiRzjKajyyj1rzt4VU|M%z7C;H9v)}Q7)|M@$w z$}jh0oQOK&_Su!<;_(f4`2yVK5nXAg+2wxx{bN7&V*$qb$AA3C^#1q1|4Lp4&t5Ew5Iac&AksSB1G0^G zgSYN~UwAzI>$#VtFBXo+L6(~w9@xehuX9n{#rV1Eejr?f7aCo#3HPoGeGHY6))M+m z+XEE&tZe#dqjoEqv6c3TF6F_8^=S-?7@g`f7>^kjMps?%G~XNyxXK}jvn(bH7C?_G z?}K2*kS(8d-rso@iLFyVD}mV7uZKkj$be3zSZ}Afsd}2KqA$X&EK*%prZbmAxd1nE z7lYmq`hr}@uUUE$-J_bGTAhpX(x{v3De77JYiMskCo8MRoT|sO_85m?lQa?4Aq$Mr zmZ7eTZd-Q86yFzT(YuF{4YE}CYGdRMv?_;HnjaDOWstos4p$!nU?B2UL;nGYUO9t3 z$3|4UbAHP>^jF7J1{S%Kl|k%k0%$Ml{X^fqG+WjCmI%OZuR|}eD9E!$+tHN_b!#0jz%M=vME&9Vsc0{A=veM% z(D-@ac_YMK03>l22KccDn(3L%nc%E%o*QE!1R0=X>x_4;GM@gQ)z_vI3tQA4HDX&BCKE2_G$-~`JvILIkC>&`rKbm84JyXfu?hbp@e z0tiChJP6anB-QW#=d<^w)*g#J9JpwCbdaNSXo5QKB8|I81PzFg7A#@?3UCIC2C*ZMxt?kAu+ELAIGA%9SyPvkmve^b zb6jxYPvpJ2)qifDii2mL-a!XPcmC_GG=W$|3!!mu8yH?bNcAOlvS9SDEm1dZ2itr&aJP*F zzFyty+_`f}P=SAc^hbY`W@iDNJx`a7PS^0U$jqEyR_`7JcU!uAvY20QmJqIH?7dm> z->i7-HLoCBX-R?`yJyfY-*spD?*H*{`tyH%b-MpgFQoQUXF$EO+Ur4n{%K)DQT1Bj z5QvP2jxoNsycZED$ zZxA=je;j0Uoss>-Py9rD@uoMuDXto{b$8rxM^FoS|3g3YLtbuK_uzvM21Sa8*Z$RC z{nag-&OyQk5*SEeAc27d1`-%Z;A=<#_0CEDwK#W?^}B~3emH)9_OqXj!|MO^PyaOi z!Y}+nzn8)N3n+ns?)3r+a}d-(0s{#QBruS`Kmy+=30#Nn^~fV^d+2jg!0LYVM}IW_ zJg-cO>*wpP*{I#oQ25Q{u3{YhX3D~CmsZi}8t<=Ac43rOAiOJ-D{1GI$J2-Yukm#M z&IPvOfz%F|C@4q`(>=jCaVbuYv;B&tS`FNO-bb6^Z>dL$OAG1n{6&Ap`{34wUpxGo zrW_d`KCbaU^Ki3Wp7HC9_j!JHow3|{-kX%M?gJnAKzi+KU;Bb4P!wcTL{TRo1|~pq zdNOzm1a%2?TSS*Z9KY`Pchgls^#?~j|_i8!<)#SIN zOEk>CFTW_lm>jJGP?^ZW(m!E{*ziwKtVo)YR5-k|jEz$nh=l7XXV6#G1<=&;kDCrI zCD1b>S_0W^+t6{zX;r-H+GP6UTld6vT&YatpdW&tQ<@k;qQ5StFRv`cP6D@nw?QQZ zvPonK>9I|@-FE84HWZS>L2>celgHCzl-nmj8jXSsYSISXu_N+(YN{6ejYYR}+y0P^ zC?9#tP1Ev)>EOtg1FV`vgin*jIfol>9CFmYLK?z@7U?klWm19;;m^5hWsPrKpgVZ za?u46i~AyN%59;d!G+dVMq0sr?hb{|JUa>ckMWbd>!Pk&PeK+Tkmx7msP%QA;`OAw zNT1I1MV;cCCP>L?QF`nEd2nGlEwU|o4X*W_)^K`qZ$5auT}bT)M4gZMCfa~uiBz+l ziEQ*Y@mcTNBHKVC>CAg<1CGj=`LRs|63w5W>Y_gJt;lIn%Q<4%TBIt_OE$lF$Dc{a zpCRI-` zU*Zd@R~ByXK9A2Z^7!(&^K|%4?(gz^@#-jUq<=|Tf>wpJ=Xvwm|GnXU!<`B0PgE~K zSK4XT575ceouB99IU$!Dy>mZ<>kA`+8>xE*_cFhY6YE1XJ6ccl0fG;E4_rg+T|5?0>R1bxfJDf^$q$WP@_ZasHuaF?tnyA z?^56T|Ji%D7)!IW?r&A?UHeeyb5&QLrh9tM_6#=W13|z}z=ZEh1RoApfg89%m+r1guJx>E{h$951zZG4De^MrS-ho!@7hEe*UHyQjPtx_ z6VPWcp1xF@P46AK8E9qCllU^6(t4P+`k6Ew3;Flx;|ueu6mO;% zR9xs1Qg^8Th%z0%mer#^>f6dAJSU?05UMT5YEjzdZR*9G5cB~IpeMQ%V`~F0-WY!w zFq=?ph|CtzEi&5c;wy_us>b>QQ>g@6vpEHd7@e$tk4y+szlk8el}6Lz{#bfwWF=ji zT20T^Pk|znWp!{)8|;^Iu%g2I)7>fGw zd0E;6Wx$-Ms1OYfSTTR*9zca9A`s#n6Lc_jw8B&a3IZGDmMYM|MU%fe=s-URV5O4~lxX1+eA>_upGT6hMOKyWvt*h7!B z401$o9uB5g3o~h90O$xa&%;907C%Dbd}nuG|xGwM)>Q1k4b@m-;hSysGKhnCUr>4$nyG;wyT ztn*vwMYK5+ZGQeu5l=bw7@v)`8)bdu>{5DQcs)H|ouf?jO~#uf4GHD$3toBaN!m~p zsskUnZou==^bP8uksh*l3~=tNLD6!Cvt2wp=>5~zsi3ysuC6pn9zC{v_WNy&K8Z$OdL72uic36J}akp z@NVdA@Iqk^U*jRe;#TVmn%QuPoo!{1759LG(gML0gFm)~c z?d2zMO=aPuQH{lvhnx(s$Whq}w?w+53Y!KS4O<>8qVlOCt#P%yilV@Gl~LC_E}V>~ zVp5)O;tE^E6%1tqh3Mb{Kd^A<(-l?2npOw(7oLm77S|f@TtK~l`bPTP&5LPcY%sX# zo}5}tFAvP5>tx>8=TIJ$AP=*-24#S*xP5ZMnXiigK6TZLVA}B5^}~5+nN?6-B!-(S zu8!IJn3Z3p@YpgD26lC&%oYs3#~GyvX+bsMGezBDSv05=#vzc>j&M&4OdJ?V8ike# zvS9H&k9Qs23~CBD5p%BYK_|&=UK|!Ko863)FQ-PjA zN|xFN;|61lN^XkJ8Ol zh6iX9v(oVCcNYLs(8%xY!8yZEZm>AX7G}pe=~4wH1TMBnl^ z&b$fAkDG8$-(2P(Ar?SIXj|Hh-O9NQk+ANgLB(O;A#XQ1^18}R*w4XIL$r;&+Gi^p z>5xj;cB32=wGY>1=2qU94mQ{jBGWj43s*_|vGVG7Awd7LK+>uF!Xy8RVXlxqb@j2d zyMh4&N|l<*ylqRl&s#<`F-zM48QxQ zsPpIj&5+0Sw>+30T_lF;{9gWE@qCv3wy{zjOtkv7E}rkk=#o zmi^zIPNO5Ale~zMUw1#_y*Ikw9p~-3j=pPszss<@yPG@{7R%2WbTJ1>XC?Qp<#pb5ZV+hTb-rQhoHww?4X77d@MC9` z8kCuFJvK#osofB5GANX`iR_#*KZR~b+ykELg#LZX0M*AjXT)5hPMc6e1gL;P$a%m` zYPq4?)XT#IpC6h}&%Jaem665N&E}6h#iAR%a=*VGumi&IN)rD0}E{v|Ud5 zQh=Y@xNfo@NweXgrty#SODd0Ur5gHGLf>|9t(@9VRXETOG1!jI@1_F|sMOFs0JkTZt%3NKXo6yk;QeomtIAnNf>B5^X4Hm zZVy#&VbCt63xn(Fb)Afmow0Ja@n!G1&Zg}vu`au|cdCq$kxU=1)sLgsd zj?@{faacDAJWx-D!6iNqSYv3swSCp6>f|YfqU~rK=hQ9LH54w9ER@?|e05njhtxx) zeRCb-yAyqYY8}t=4B;jR)Z3SvaKk%C%a`q8;c{*p+M{%K%3ow-Y-yRX0Qr?U__Dy} zTlYHAJS&^5q2`KdpEh5p?yzx>%#h0>hlf5jdL#6&=XdARzg{~VbaQb?H)F<;XZone zb6p*3q#A9}$2etKLvG|L-JQJr>Y3(i@b?^CpfTI`vQU?T1sY7K0evTW;oT#F?^XBu z!$17P5Y|us^iO}!LgQ7w^Dx1n@hiXbD@hl<-}sH+xbp|!j+iVOV{7m2Nc)Y_G`~E& z8FIIy_~+5yvQGxbKn;g=0YyWjjX@*u_glDKhH3@KJ5J**F?&xlFg(YL6KgL#6A%n&!0TYO${cZ!Ob^9p3lhx-(MHj_uUvoWVt&i zYDmL&FZ3kPlR!@bJqh$A(38MBPXe*Y@ja--o)fgZy!_6q_&cz~J>Baa*l4{X_axAh zKu-ca3G^gzn*_dB-RrjGzNfF>kx~24{_M}vCqD6s$nfJI|M>Tm@%Q}j-8Sgn?fkk0 zbeqw&`={@>>%#aLht1W(aIYc1Fh862o?1xzGljJC2)l5<=jDYdt@zhU6- z7KDaygPQai z_R{9T7zf>eBI2}cPk9%8ks(Y;3xugiB0wP#+~M_<@NRgYlp<0k_?)2)Y9bFu<<90J zaWa{(A?3ADL;uCI6ec$QKpBAa6;&eoxDKjle1HRR8bg6#@%cm0*#_xF7ADtIiHg*{ zCj|m^kApV(eRZ8fT0j8-1yB7T=54$36fU=W96CjPf}%szjqZXxT14i>Vdx!b`#q6d zo^;Q%WC@DmIFQF8llJSSRIB2FV%z#Caw&r335p0dn*tFgYQ+6IeiyZx)5nO$ioRj_ zL34+zqgA&`w1sUQ=n%e!tSFZ#ohUEYuM3D!Wbq6K$-I1Hj_3Jq8axYv)Y;iy)Ng3C+5Va$!%sg~&78#`ba0KJo4*c0h zZy1Q<;gEs;vYa8%%+1y|7s2G;F^8SmcG2P7$TfdZcR|ZR8ulT*2sZbt;o2@D+e5O9 zANBNUK}eKYPxv;nv4e{A7pY_((a(AJm4_iKJ9&|ShZGIs8S(tuT)b-&g9o%LhdK|h zLQhg)Y^nzRNwW{Hl@KC@GWZhs5%S|R`Yi3Q4n$p#c(1L)F(@j_8SS79c#hVm5b^lQ zbJAwm)jinIPlpMk<=2Ax^1~e+l5(;8dmi%Z5OwD!(ug$jd#3Ak3~PM)-L!qSL>WXn z=jr6>=Fgmb&o6HKy#2Y`zjsIXx9)T;lVuf*mzOpFw0wCv|unGR`^|g!c%@Q|IIsaXs*O`PyU}J9Cg87~M!OY|O=>Za;qZRdkS6ptPlY zQXhM1Wi~vdMD&XMwtUhxIPZ$09H6XHNr^;{`9|+NM4)b_GivCJR(**3{-Zn|6lh&4 zl+W?By2Al}(z&5~7Kk8yfCoi3M~P&9T#GghY9~I`&-#epnu5|rPr8Z0kVku@63>nd z?}qo%`W}=d6nFsC=`36tpIyEb9zrgul;Mjv=hLGP-Gqi=BYl12R3JwChM->oX*V1#ScMWWjD0PW3fbV*sR`IaY)9YvoPbSdW zdVK9L_dEoJ$qtRbGX)|VeXhY_hsVrq>Ra0nB1b-p;Gv|V5RqD4M>foJO?1^`4%-}D zszZOlSxjV?p;$%FL%-m>!~h#KCmsDuc{}GzDIp>}sK{7yICsVTg&g)deDUGI-E?i6 z1C;4gqM${hTNk~?Z1eGbv-tr4{3s$ z4%!~k)zrNSC8D01C3IUUFMPkgbn9$-@9Yw~_CfmL2QNn!>xVRjs2=uOhwjXb)UwxI;9tl%X9h06gM4lRiWrufL%b`gobT+h~4 zhn!^C+(9m^J(Np%l~2lOvDB4q2X&P?DB{W{2dDCG+upv9{+7^bT-(HZL-qVrX2;ZT zK+ihT8h=eiocwuDSV~ZTA|g+b&%WpNah@lQ0q9G--yES2gXXSHRPcn*+mCcv7aAO! zURs;Qo0fh9NiNAa6kG3DDCk6Fk&yfdn90&&bswIuQOft&Vz@;{pN4} zX8M_*`I+>=4}S2@AMC~KNuVcz`&I()R{D|myyrb{k3rj(cJfDIaDk;_{=)ivaQQm$ z%L`nrK;f|C0s!RzgJc2i%SDQ=WD)rA`ozG^%?1<*D#hrEp6PGy7l*YhC3a}YGZcIJa&S?ShV9jsSCu}qU8|;pO-_M_ z7tw`)3nUYB5$j@7x9Dh3+kyod-omkP!s{H)PySfM7FUTw8A8^!Q72@S8Y~_$gj-I3 zH=gfHE`5DzZiPxV2h8gNr}iZiAuAYS1T2 zaC-K^k7Ar-4LdCA4ACdOu>X;qeQvwIk~V9DJkLFZSvx4F<(-lAz~n~Iv|LV8!FL{W z4}C2kpQ1z2#<2*spE4#`C+imN!@aoX!$S8l@F4-(%Lv}%AMc0d7+%U=o}4W7S9fiAEes)V0v!vRC@pDwY1MJ|JBxH82Sd#p)8vQ?EwG< z*2B&BW`#kQ7qH0r6jGpQc$JbtngHn^+R~xTu?fq(>l|LW3{3?ClH*}xlsSv?)+f|M zsI~(G!^%~$^a2GBb2KRJxS2Oq4yN`Ij;nufz%rs zy46hvw(Bw8uUCiC5L6#>nV-jKJyzHc>Z$QjJ~32lfYWgLU(Y@hy2Epu^J#W^C+5=C z`WTy^eQBEC8?CX>LHG8d3PbkNo4x138|m4t2dqT!FBdkUJeXvzaL%IbJS#>$Zu#YD zgoGTKy9^pVr#WxqvC&)UQwSsm*%jQS5^DX-BFG!UYK2Cd2( zSwL0GbGx5&X~^RN*i^(n2oEriczZ_TX09-U67js=`4Xc>maFxE zpQr83YiOy-ADRY}r-tk{>zD`>Ilv3(JV8Z=uA%{WfK4amTiW3qpIInY=o`%;D#c?h z#)6TF{Zv7&8{0$Fm41s1t-5-!ec17+fUexcxa=BjbbL24{Dgkm;QW*$@;Kgf!-8wx zO~D66=~Q^%+(dHYU^NZHdEZK3y>>QzweL*&w+}y?{>{Xd^ncaf8?u(7=Fl^p*r2E# zRiT(cNYDcH0lc!$3b;^L^gNSRXG2VSI}QESHP3*dPrx&82+G5fk@v(CPo&R({`2YZ@GzY|eL8*ULmx_i z^;dtDNBibaH57shV2cGuAco=64xVk~9S7VI*YviMdw$oq{BU-14QtYn&+6_3^;P8|GT17TT_gbbvT8qNO==#9S zMi?^xdF6b1FI?K9E#y61t%1y*(hIm3EDL#6LnQ6m>1iw~nHwqUS-8r$Fdl8Q{I9T?R{yGzwCErnDD|cCArLlOwgX zxGijhoKW1YQ>olc`xyE2gCa$J?KIH#XkTTk3ZKmp)TP4x+DIOqf>I zx3cy(sTXQ?yNT?qC;jAPxF)?O%Cddgv`k-a>Q#_fmqynBiysBr zMxKpmqRcV{KC8ReE+7kJDuVqL z5P&`x25s{2)4H`;6e=ra;NnNrJJgl2fQro?7eUk^jOvu%?(TD81Xq5@CKhCtO*!(N z{iiVao-80-6#ATf!Drm)AuShau~0LA?uPo*7yd57J@_op*CyYIV?M@p!{!K{ z*cJ=7#27R{=HZ#pnQuhfTi`(5GiKRNr}6yo0NOJgLOETorB`>R({7`jURjw*jVTs& zj!}*i7Kw2RNqLp^J^H`ngNw=#I3JcN>ZVK>17dvOo_vmuk+y?I2IUpMQKZ_bLKs^~ zDuTm^wj7JCVK)!1zq-HY_esuTM92-netMtmzBzE;S{QupByQ*0`v&jZ6b9q_-C+5A z&U^luN1k4upTB#ZmyLD#NoXP<-h`9%{2|ZB_<25Yzw5b^1iCJaZ-{5OR&IG(-RT&| za$C;4&VJsDkdH<8BJWy2T|)obTp8gI z>cKQRau5h$=K{z2IQR_%RUFz!*=9#~)6CpbT0R&_U%}uclJe1!W!|y=^z!aZIzHP6 zO0}AX-~_)gww_*GpG{>BIo5lwf#;B8b75m3jIJ@Skmmg9&2;NE40{H~)Vk8zWq{sm*)O+@*0Y|6Vu*u{(+}Od~k* z(U>iwy9~E>7`*GP)L0x&OUO(9YGqSn2e-+9{^8Rvq`zN$B<3Cul>X4vOKGVwo}S%3 z73ZILZ^J{p-TY=VKwY%xy=Bn2G}SR1<04%>tKpb4Y5MRMvklv0T)Tn5V6FjpY-B&} zv7vD^PX2%^ng{5v926}o{L%II< z(CBLV$E%m(j2jKqZTjm`l}^dC4y?}o>KFnFYP+PX(b{q&6OHg1#2Y9Kr{^}~fX4a( z51azs&Zdj&1ii4@7;e#IqHk-AJebOKv+}n6pfBE&bHf}mNQRp&HV{xGhzA~DgX5%If*;Y3pgx+P{;HG!up|wpP z;vK4R0Bg+k)(2F!@8a6!i8Ok4Ck+Au*;jw;gwS8mIn)W%H73!qH$Z}WxTM|`dXv>b zH>*#TH^@s7hRl__?=|;0n?$`!qwVM`ZZx5{pjU{AlB{i z^z_n&ba7%m?eJdM?+?*cV}rtWvwcmL_H<1a-T}xUzzv->S{;uOJbwn)@9ET>8D=*d3j?7ang|byJc4#ah$?yD|O6QyX z-FYhSteBtv^rzF-)>bU0Tu?ps*kf;4fnWQzUrReXJ7Lh4e#z5(*Vot6@BQBIr7wKp z3vc*L?`BT|Jqh$A(33z<0zC=bMG3f$zWwhmX5Y)|&Lq&&z3xm-y;wa7^d!)eKu-ca z3EcM*xLxn^`#lQmU)_25r+@kcY+T03Y~X=tfLjT2cY4_2|wEmaML5ks;Ohe( z7pMS`0=jVp;*#q?bdYdi4}>J2qFp`7JCGWD1abi+M02GIuVW$YwV{3h#S#mAqp|ZJ zNDombBtQ}P&sPV8D(H^~;pzM0&2#CW>JJ1e=HbazkQkJKh%2k3sSG+-1Wh0{c}_%Z zP8|`*a*#|SCHsjpR0e@*`UH8r`*F85Gy&T9C+}pS3(Oxhg6L2{(&3sXI!a`eL6_D* zAo10Eq7gzid=Hci2uVo*T*S5Rsi0Z+w{Qyu^{%U+thPQF;EQCwv5l{PRGpmu=d5h2us%0t9$$e1#U2-dyWdMZcJK`(EC zqV*(p>RjL?_Dr7S&+|ex%}d0zbS@2$?Ah)-WfXA?s;WtUdJ%3=T`Kh!@UIC{UJ8l^ zeCGYTB%p9e*fuPnLXdL&C&I;ojL7dkxt+#@G!d5MXD7aGC2FfO*a**pnen}}vH?ww z<)?fiRU$F#Mt&7!KXHC5+@P(KNNnA4<<1VYIDNPbg^Gyb^){QP0SmT2f;f#`mb}~*-q^!{9wo}X_)E}4D z0`8*H4~h7T{A_c+hs!tD(ckF{;a^EO*8rtWod$zDvdx{giEpRw9%fH*K`;wOiR>GAc&~DoUR3A9TYbFX0WeWj%Zi= zXa?L0j#Sa^RiSQ#N1!({e7Wx|oco~W2&G{*oIxHs{~Y6Ld?XT_cgT zmoy{RMb39og5K}rzm|>0eOf7dp8gf?_I%Az$MQZ)MdBx0)F;Se9{`khLLFiW6ratN zm4;7#M>HpB>O4(uMew&y)lbsxj+0;KC~F_e^N0lA;J^GCf3ht3_x!oYk8)YYoVVM! zmepXH{LN2;&*%5DJK6u1N%3`;-~0@Dp85Cgdl7;E^7PHiyiM<3gcuw1!rY4j^zytT z3B299mlR(M3mY+C?BjVAo>q)C&I1Pn%mL_L1!y{?mf8nN+J;-#aU=#|KIPh#n!vY5 z-9rikk(y2Tms^b0l@-Sz$8YX4F4i|VUxK-<2`Mkb7>LUL1Ql}{aIC~bO&Sf;7rh#D z57&G3a*UDl&`x}A@nU+he=evverV!qAhvJ9CG5B;J;x-dOFjL9yL*obg6I1gvwdTm zjIkmnOYnM&(A=CEOietPM8Zbkr3@ZkloRI}^+p(^Vyvt#qa#dBT^#?lD<4_ex31w}0Bc z7-yPD(=~t|u>-RD3tJb`S8tpS&&6}6)W4(w|urhESc5Zf_lwv}Zs&kPtP%id~2HeK#@{5a; zsf71sP|1O|^-PhWtM(VD8y1h!{wzYbi-DrPwLTW@d$7b=FC!q%*9M>m$Fercy!(3CsgtU2*v^NbwDkn5^M3h`9O7+e`^j&e z>rJtq;WueO!kdrljyi%Tgq{xOg-7j5F;y0zK`I}lYkC?2)Z>>xzZV!s23SVw;i}#p za~gxn!A(2{(ceqp`Rr@^c%j$61ZO;4rGr>5dS zxryiD@SX2Z!`GNJ_8O`06)ueDiYZXojNTe#zFlBKdR+)3PSd}F=dsFu9tU5(tF1j)9LZ;baf4d5Tox) zX(o82WATCV;~{U$D6tVRx>^nmbt;zOs)o|%!s>_{V<^UtUR(@bZWm%TPFX4iwE=NZ zSUt>7!?uP{mC5=P2WhTgI1YmwiWLhc(+$Oq3{~9xuF@2W9AiKjtSt=pqW=7@(uxKi z_0&B!4wOXBQj83-cvWGOOL>?AZ{Y6gp-L)p4cztWhXb(yGs$wH6>{@B@#X6jb9!7ZuItCAjl1p>3 z53eWP|6GK)Sg|T7#9?^m+Mv8_L))DXE?6kXc$Vd-Ok(qcdzrzN2WhLae2a8-7aqAm z);^ZYP=#E|6X`@2q4aPeQFKED8W0Wj_Lq~sNgKyPo5kSOW!yb^F6wH1HH7OPOJ8rZ zh}xB^#Xjlt9oNj#4(ZwN`7llYeAGpQi$uSBkf#SkzIHI3rXZVMJsge$2RB&IIYdhx z;==9h>;`Vg7#VP1?ZZ{sdPt$snxuU!K7H?}&c2u`bh0mQoR33L<6ucL9E57cdOR|_ z9EV444Pe-i28hii3v{HrFpE&HThr~#7{Ox0b~e}!VeE~*NhIp-p;Oz?ZPe*k#~6Qo zCqr&L;zCUuz1*klV}$!_o<>p5LKauHk@s|2&HImwiTv7k#dpNbq%h!@MaB*LKg|@eD)(Kfg+FaXS8g7no;r4kC z53=qap2ogG1oI5)5Bi|j8d#(oanoiQ-gVy}dFda7Qf7h89cg>kYD4fpvf!p2-1Mm9 z!Y@65G6?z``-pv$`uQ#d&n@d>%kBH2yK9y?|8v>hpN((uc|;b$=(Kvr;F^DCzxlsa zaR@P(jm${r_%d*vsd;lEB-pdok}#Rx>)mb0Zt+`rZWe6AT{Au|3 z#`Nt{B{e1()0ijas+E2$XyBMHq;mWJwlAe$cY(TqFVk~w%EN`+=2Ze?76HlC4g663bdfm<%G#=#X zfGOX!yvUXjYU|Jm57E6eY-KGHZzoBYep`QZPancGhP zX8LOSf37|jG%M0d96(Ps#9_mR;<;Mra_!@C+TFnT3t(jU3X~A1C^JSl>y?kUw%ZBltJ~;|VrI z*jtd(-1O8zY>&!RJ=L$%)=qqKi&T;8AI{Uxt003}~6oDW$DMDkpdt+8r4JG7a8X!2@W z8X8MaU%3$H8VM?JeH0Xntd*qZ^W2*Poc)d|_E&>4lqN6^O?4B%_nz_4{;Tmm>Q0<8 zXArbbWqs~&ii-BayG8=`SO4yZK-awUFf%ige*M>foxwfp?7#cFzkBBoztu4Z(3^54x)Rd9M4qxu^lsUN=vE+P`1S z+1fMf{Xevib8W7mS@QR5@)CR_FG|P^IYb>iEGu%OEdBH}BuX}TCJLDojd6wD4|*N9 z62Z>&EuJAQq&*JFf%U;U`VR>B@20imV=998-LvKJvU1(0eMzvQ%cto4f1iGg1}12n zM30UBgZ?hx-!A78Kit%o_w5~zuGIS*qYJMQD}ob8_1SO z)}nu9Hd(-(&+2RLWYLTj@9gzLCPCNALFKk(Q`jZBxLNbC6o0+3RjGf2|1{t|24(Bb zzr(acb4cu`{@?|dKsUhH-_2vWoAva8HaR0RQ*o)XFD9`Mw64e|I$MNCDK3A1zn5P#q&6;$UdPJfYt65pm|qmn^)3cvt#7MjRr}k!elq8|B)bYauIqA&v2#*rzGG0HRv|Nq zuov~Pgz@fo$5x9!`q%S=nsgE2@1!Ed&p9pXg)p(FZ5PnU{RKG6XtoQT%}Gsden{%U z&EvI9fkMlhH-RE~zvIY$lsq?Ez>7~EjT0w=s9T*kVeK4=K3$1s9@?Bg94>*RL-~kL z?&!Nc7luBeMB{M21cD8DVJX5xIGIggDJzO|cvMMX)H-ysOSJwG)Np8V*gMAVKh;sY zHa7+|YY>?np?Pt%bP@TlUL#u+>N#<=91=4=Q|vgG7<5vp0E_{E{i9N-T2tpR>og?K zF;?nKM|xFIZB2-vrp#Gt4OZ_H;f!e~dBE^s6yH&3FzOspIQ8!6AhR~UAXbM?)t#3DN>&SV8E4_OJzsv41EqV ze@GQi;^O7tmw(x55dQ*QFE?L*bKMcpoLCy?@glEYsKyoZLgX zH}xgdLl_Xt3;$ew%aU%?=vgu$9TD;Sc^2H}=3~t$EY$vojJPYgys!yo{Ne|$59x20 z1E;UT-NNKyfn>p>Hpxgqu-ZmJ90>N@4NxH?<_zbqSKo-2euUuDi|pHM;ka8Y4+%d8 z`kcC)@0iXYh+SLJ|3k4MsmD=|n|8g+z5bf_1MR2}kmqkly_Mj_n|1PJWRke_xW&FJ z*N^e+v#mCaxw0ZEwfp6UN6J@U%RyIWLlPibP}P>)MtV!i3Y(z)cVQe`8t2C*FO%eO zvIxOJ@kXo_{WO+C8K3MwU;gCQ4q=$Cg&V{EH$8F}+vw=_<|As*r8@J~urIuE@;C63|O(>%Gn%H)GPhPjUHe#+Rf9 z2xRTs-?w*8)ZFUlch(jV5KuGz347dWHlOP7t>JJ|slRM#u*37Hv}x8iI*^q7mG9Rql%-(l_pco^74!3&?Z? zQ?FQ(C&y^SB~C&XK(D+DN}Ot}Oq*#+HRq}&xmo^{%rUXqdExh9Oq)~6#i*t_x6>Of zqAZxQ&KePk^eZ_RGF)Zdyw`#+!fumo8dLbf)ARhNB%90YL`a3Tdu@#b1-@fGwL1}9 z+i5D(25QGXpAhU4%?~I+1VcZbnlnu{WiY+U9T)o%u!*3A<#{$ zEYtyBT2geLKx6xOV$O1tG;c%CYMRlizF7m0-Dp;1HbnO=A|K*b-JTAd6-qQR%t4zH zFPlTb&U*?le{GtEK?o&M%cp=kToh>hIXlXbNsR_?f2G?mspx|?(`;cQNWj&_!-o(u z{?zp?CV?Mg=s~n=*Ut0ho0sm1bc9$^K8~5R4p;UZHQRggqPREfG^>ie>+~MZJi3q_ zofh6nHx2-W|E(j$YA&D@5ZFghMh9s7opraJ-ZP;1+kGA`l1CC-SpM?28NF^+>N(?R zrQC`XN|X*75?27sj$efck@cm`6IAXdfM$9tF}UeU4EtFm5Lo21SQYrcE`ZpEmdJWl zW1@c@;*n|${juQ4vLO6y*U`gW9kok&#GB=Q4l)En=DmM?lN=1nu0L$xdi)$-@*CO1 zC_YQx-7_9A+!^hgi^@I<08EkNgvIQhbe@LAiK*4VEZJ=4-%}Z==FnP!(HzPf|I0qNB3+n=6wrPG>A`Z z^pmLKT~-|pK>W1wU%2=*6(^+ceuiFFS_Fr!Rn2% za=r+}f*{%i}YsXK|HNLx51E!YP27~wOZpl`^PI;0qucR8V3f~-^6%7UD~^R@ zzK(!74i_5s=-*b)Rez#P^^7=Y7_y8oz2DLe>(h|K3Ba1*=w#8L8-J6c-_P;6QeM&K zOnA2XZ1VMC17`;P1m$oZz)f=cNkQShj1v*#hv3^KIX{w>yesl$rWix^WbiU)Y3 za$ZO?Le5|8e!zZ1jN$Y1Sd>rrFy50SZea#gD$bHr=UeHbh=_I*N68(F=*`+}V;%QW zJjr_><-3j@Bb}c~SOs)sAgLWEwO#gtN#-ds`xY9zkgJBEA~cGBcN%C}4MMMtgg_w) zMIpYZYGzoR?Dkfwkb`GNmU>%-In-3cbiATwJTYSgIe!b|bsyS?RE9-0mUbeOf)ycy z7!W5m!|L(g3SGrSCYRid&h*LeJhOV0DBGSNxtu}km>PSFe^y!WQ%ya^X4&jHchs!-3h)P)KWI}uAF|GSJMzxAfJ41KE}6h< zrKOOVty__A{YUm|gFm9T41^8jh3gel`b5t^JJVm$5X3{$7d3MLYBopS+O7kaAC%A-&&(L>;Up z9h=IG2Ta+5!&N5iETZ1G*NE-(|4M1hE=nD5iT zwlNLz?jAngwt5pn9NL)I&Tq!%T8VRG0W$%YGL~IJKr6&f`*wU%b?TVV_K`Y00`_xa z#9(D)n3>^mqgC@Pk3Zq3kRtx`< z*%dr^%pgFmT3P!+BB0$USTQIss4oA);?RMQW0PqAm5iBOg0MC*DR){hS_~Uk#t_9{xs-F?{gk*`~YOmJa5aVA>dwn{4# z@YZ)@Th3>&?qJ5p2pS=Vx?E2eBx*u|gC?Jx;4Ho)a0rQXCRwU$m8^&v65RMDyyZb( zgM)hR8_~Y*!cFcuG4Bpa?)qUL6jdMc6NSyw72~4}HjgdC4lXL+c+QanI1Vdf)zu&_ zpMrlTw!dEWF1U5^kcF!93Xz*iz6nc4+58O=QCa<swVoVJ|{q zbIro~cm_r9`xphS>t`gw+zYtV-u>Y3SR>jgTB2k}?WpMJlh3)gzx!QPG&7y5wK3n4 zY^wI_Dt0Xv6aF~rMAPAGEn&W2atq0-hW~OXeA_`$M1soS#>``*Fg2T`{1oTDUJvHW z<~d5C8x6n4<{2R2)IHq(<+*7pL)qB(EUl-tmfF#hs+HA zympE` z{m%+HDrf~o?5mMkGMX)>zWv~U ziA#7Q1Q-V=F4g@L%f}@B+Uw<5LS)d~f=9QD5fAIb=kaE^9PdCsM^9#tXLGTElOAI^ zO%E?2u&P4XKQ- z$qbunWN#)PJ1yN+G5iKAd_~h5B)i=TJ5o`n0@G}BAT*5cRFpmkk-2AIIN2w(1lXfd#dIS%eFiK^jT$8c0Il_w@p5;;iZ1M(xm!D14m~ zr~I5Pt3B=W-#TQJ{WR505FPKVE6MhcwX*Su!^yT5L%DhR?$9Rp>iWz{RLiVbRH^>lr-F^i&aOs*EILkMEE zC^MG&LyhSQBJlne#!_`WIgqc@o7EkkE%4mJs0Z$_ON`|8Cw|&_c!}brFpX^hu za_F-r+^NPIs`No#j%5H;+0}M%dJ~eo2d8LSHR}N^i@!M=N_S|HMQ7DAkMCS9G}rre zP!+5^1Gvt~;)+f+6P?N~h9r6=TRaxSG`tJR>K!9XU4tVcS4EkpFmv$qE3UZfp)jEv zro>I2eVK(WeuEk(T2#zvO&h?oi00j@Ut&;H4WCxMRZ{`C3U?< z;MKS1#-N>DM%xlIc9CG{BOG`~_3tmy#InN{qI?r0fq9q|pZ1~;BHkRj-EV(kxNpw^ zD<$j$J~A|PZPOK%R_iNn-V2Dv*%;z)Hh!opZVA|ujgMSvFG>bhs`ZUO@uXzV-Vfb#Qf#)~Hhe>0(>wkBr)q9pUWDWD(KiO(3*z=ATBDvH>f5_%FhG=Smn{Ti z+(uF3sC{Ii+b8|dmtS2u_W<3*L*sKq^$piKb-(K-*f*R9oLBpfV%fJ>s=x$^@N85s z`iM#8W|as_ZQk9y-v$LO=)2Z=GZ$OL_LDh=2rP7Zg`D}Hy&D&KfWsZJU zTZfF)q0{Gk!sSk_U`qh8uOap8ft*+K=02-je&O|%e*PJ42-8aAcxzOVcpDOIfr|MqFbCm-+FA?BEWcz6iVH6fF` zV%yh5zv`weaMdyAF|Y!A&R4Kynb6?c$b!VEfk&9ur` zUMVrOO6lJ*)%ctpc!bo^HoL8QyO;TrT;&349^KU*LKPyd1(9RgU>Ckljz2kVJzH10 zi*HBhLXO^jUdnHrR?WL=*UcdJ<@;&mG23B-IFa(sM-Hnkq3{dyx???HNwca~nX_u& z31xdbMGm{0N)My$)js~C_!TmqUzS zo?L}=iTE|&N1Cv&Mecw~#_3ku=)>G&f0MB0X%I)MRIyg)SQ~Ur_PZsFzC3JM`&BEI zH9zr;nNL3v%`7PgItR{-FvF|%yCa84>7LHF9A`t*Ef#|V(xY|!&l)SA^0#J4xm(v* z(Tl;7pts{T&>UBZcj|F`CJC5-3T%dS*L50WG5wT7>}KPzKqckd%D^@XpK0=2x%KVd z&3)tO`B(@IVq4t-hTS7LzdZbQ1YB1~U^=heEEun=R--xD>AI>;_y;NUZA0nYyZVn) zgZ$USh^X&bobG4R2W9)a};}fDcy;H z6;622F`n9-g55!0y|Y*p;&v-at`Js203f_011}r-cp;lVSRVS%Z(7~-r(Lz1Y^niC z#we>7WVLQQ^>TRk$fxE;M3qIm@enVVG6EjwwosqD&ek>{qO5)TSryoT|9_7EgYvtS zLhOfEQcd-y<1wrm|Np>$cQjXMN&lS)AS-je^$yumHGT&#sY<>5mkrzpPq6r3fc#ku zfgeuow<2S3(H(xkHNL#rQ%{e0xz^WK+`5T*D}239v*VPMp7PPJa{w6m8PoF#!h$6>X*NSq5GA!aC?%0~wA){{)oYLG0JJ48 z*!z8#*0%F z$??YLz8#**Z=kNw;a=(WGl;iInVe0-)j|z5-ys*sqC!w>rzy;&B#)Yrhw)A#P?(vm zwq4Jk=XYC(lCY+FVyKY3kny!_xMX&Ma7fK573N30VgR!gHkcJu=?d6k$Afs*e_8f< z`iB!*N|A$0lTf^2=x9(Ykq@f`9P$faMlX~;3|G{s&OafxwwgiVKtOn%qm-a$`PYmA z3`lQJA_kz(s=mto_#D(*m~j zE?NQK5wlp}%Ho6k7%T5Sy0)>raYe3jYB{3~erq&Du0O+DfAVN-Heao9-JNqiUc5fd z@=7Q67t$vStb};D{k5#Az$oTY`&`ExmdLk^+~tTyN3m~M^aACHk5-1dQMOKYDOZ66 za0#I(`+WFqc*DOtyKizPw!aHU(1{`AkE6C0+>geW2#$UYLouZGV)pKa;5>gT4L(|F ziq122QoYv{kr3vRf>{x{F1R{&zi2ZxNN@`xq8<$l|MGh$E%9%L>sJhmimcbM_ywv^ zuhcg3pcm=&f`be-&S#srJe&{;8TUY^?24)=ukJT4m?SzgAl&j zQjBc}!G2+lI}Ew`nHRFSlV&VTqB2J@@gRn}a6;H^MzQE7%A>|IH~!L?J7e6o^~}Ft zic)FYYI$R82zZg6p`WrS=-Dt;)BAiyNSd0Rn^uuZwdF(Qo21w zFE2xf_fT{0d2g>g=Z@#+*XRyL10FZemo`t)L|AWDIreSzXlooU`;h zIHR{O<2!1W|7K2eVSePb%*bbDe5H^g6_k<}=q8q$ja*LB=Q+e7(<}*RZ*H)&9|0(d z@HzyJE}TkV>{H61VzgUe#KxBO^eFAlrkKxvfrt;$e0`y|dgBIoK=SWS@Qi*OQ3QL@ z7!v5KYA?}hCio;Bj2>=2jA576It9K3JuZKn6r4GYbk>i{e|0zFCkdL%Y=}t9T|x_; z6NjrdJ#3{G`g9PU0$*JV_wBcqFj`-;m z$hdbb-Q=@Rkh}%ES5#YEv~*KND%iJ?9Wu!OwR2#2sk6^E`fffmJM)~8>NL9{Ypf63 zq!7~`$Oa_4^CTu|X4kRFo3qM^gUst+HsakUQtd}|;LoL|Oce>2cK-Z^apV`7<8Ns} zc^?@V8XtOJGuL;>t^z)9;qY^WT!KVaQ1*y?NYU4-nc8>t^h?U ziRj=?4&GW{HlThY&St%j-(2=tLVLHNSt>J41hROOV$zA5&Xl1ifNt>@q1Xt`N``lC z6$wn5ADQ|C8^^OkLdq>b{igKaRe`>v8yvU-B7C&Pgx{SG^hp)ak<=qIcVnAC=du`Q z)Y5Nkvc*;3noX7avZN{;S4L89OVL_H>%3kYbGkPl;_}t%Mb_bh9@Zx{CkB|k55cJsaJbYTKj*`FxyE6->*ve*jY1V z6D`XrPK=2CtXT--doNC@{s}v1e6eorwhU zQJ;I|FH<+`pL96keG2Sep85pK`)!)ej(@jt93|#@*vdFcKIOZFq|!#JS&F$}{uNha zrcEOB=WI=z{?;uVe0svMhpo`4;Ps*;j3eTW&z&c4k)SIg_sb7S!$1B~#-1&CQ2*FU zMff8G=f}qf_ITVmHVayiu_4XfROIaQJxK#KOj+Qd2^n>Br5S-*hpu zXXDWfjpRoe4>9j_R}6;&f7DOBA3c&PolYyw zug%E0&92ir@or3b^r^@dr&S|vMIwp*=0^v2-PNzwt%_t6WB&VpqmFyWzqb#BD(b_n zOZ&DaSlvYM@Y-)ypbNiZL2A=dc7%ux0cPa z?n|JgIv<&@vT&uaQ&9{Z*(MpP$lwAtj3vG4gx7A{lz9|%aHCCpN_g*;Q_6kxXVTHJW@xZ}nH0pbAG<_FJ3pgura@Dq>X@l3YwDeG%*jc;qgA?sEEBI0K;-0Z-Bu{bn31eOahYN2cB}fD(2`VKU zJR}nm`g2T4cQN~b`4#$!ICHudS=%MjrFoWM`&IRe=uK0L_^atJyG;GpM19sb(*qi zkFSWx5VrqRImH`9r>{T4Idkd&O^h*kp)#Xe`m_lUxJ}Gz@2(_QJBQR89eCxBQ-IPzITY?S0^Ka<^ z4^4{fAKgm2Ae}N6`edExu9OUs)9uksmR$jw~!ei<5WRYEGG=ZDN;&pHbnfbyw zgss`RSyS`rStXVFA;DK1NboV*TIfi4F`i+GLUDDIuj|XnK7Ak(E7Gtcs^g_Jl>5wl zB!YqnmO_WyCWz~CSw7>jD{*Rpl16XkLg*aPZB=`^z_b~1%dGV=MU|svl7owZe$7H# zFyFXu=JU&EXxs4HX?F(ZSkHLDd~_%=TfFC5#H!803wL09&RfgUCcePfof`9nA-tH{ z!1N0YHScVLnun0*sn(GKC$+B)2(M#%TzEDFB~OuY@5bl5vBjPc&BpIrpuxVwFJDI| zTuCyS3qGC1Az=btn65DMmSoN$-z8MqKa95e=$v36zT3}X3IAg( z+h5CRbpf-*3kQ`GcG_#LP=-Tr;TeuigER7066hTsYzg*Hf4jkG_TtLyasGa;Hh(14 z*@Dl9d?+RrcPgxWwM62p2w|zupow&0gc2F}q^{WaACA)Xj~Vzn>=mKE%)Z^DO9ZtT zp^|C{w>t*?5l*A=7H5P>uE8MCauh^E7Ie1rP!(SvD~K z40JBnQgz~}tehS4HtKUHus$BEtJav;Kr?L^MLAvJ%!j$I1ji4a@tOQ_s0K1uMRgPV z$sl1L_ynr2`kgbnRO6!+p_oEU0%}d>H)6{2^r-E}ezfOn*S|IR|H6+opM*p>uNO5pz`_ApVWOGFVnnU!Mzvk za2o*d!aS4m5^~*6+hg2st{IX|_pe55A?8DQ9!I_Jg>Z3pVABs(Y17U2y>mv6Hc3yr z#77>8r*E(fy|%n^rdH6~YiDhgo(N550uHBuE0-IOeWSNv5nN(|4QHTnjk=Og{{LnC z{vX)Si%x-pXB3D)`kWw3CLSZI($Udz{9kc3xGWc3=2<}CV{!5SqJRHG2wp_OUeIlv z#zwCa6{9ZvnC)0_I#eM`d{VV!G zue$3Qrb8l<7X8&Ca9AT=dGbTRY5BiyD_-D-QJFHlF}AIM=|F~7W@>IPn&k8R@ID8tZamZG>%M!YP_>8j-iTl-1y8C9KDE*5q3|9WG*hX?!n6=}IwKz?Lcmg3cbY3wG8Hzs$qg76d08@ZSPO^_#$9Gio zrXv>hwH!HCI7J$CBE};wrnt}KGt(4ovHXpiIB`c>39FKMOGT8rTf_>~6hkunZJT|0 z&1(yH;}6G`Ov4KE3dFEVA|#ggbt;P+0_ses;dT-%TI#x9tZz@@i5ti0F#p6)TA`vU zQE5KQ9gF>&tJwvB_`BCU+MQM%$XIRzScNf5CZcMQMw0^@6N2ueCPxt9)(*rp7*wSC zBi1kX;xj|)&Ku=l4HlG9mvjkAPj_X`I22zb+S<#0SHCUVM)te)h8~YHIh|}+{Sf& zzf=+Yi>=YX(W(m+x)}%{uiZUASwLpVK-B#a*Oci*x*Y`*P+;%E6+4{cR8b^A_1x4*{jcccBjlbASy#bku%^+sUHBTo+dNP%}KR zP1A0RajtI7?k(9$OU>61#;Z-T39QFJXNBKH^~1qaO$^``G@bpUH;}2CS)}2nD5*6ty7SdxMZZt+wA7RopG#i z;@esXYhm)CSGx;e2>zH>M&_!rYK#F$&`1hpJrSQbFTOUgbL%!ntReD;{Rw+t{m=La zaiY+G&R4Y~bZWJqOimazVMp?kb%Q&}K}w)2GX~1Y5_D57#Ixc>w8^8E8;`mhg2UDA`IL_!Tsnm-6PeCm}JnMNIHySr-hBF~39dPrv z!_AeMjcB|O_AL(igt}(otWX5>xevL567s_N)tNer@*Ube*6IEB>(5nz%Hl&k?&LIb zDKFmp&>t9IFD%=?OLnDl3{r+1!AuTm0Ly$&#J$X1g(hOpixI3AXI}K z+-}3SUd(=3P9g)^vnON3y4ZW57o_7S$k5`@PK3&pei)dGu;w#znS!Q?DbYk=k1I9{ zH>;{&4j@s-@hAi{LZRuR3qJAxf6*~#jC1;vnItgU*Eh}KbQODK40dT#S97BgGG$>3NFxh zoh1y!Jxq1&b6V)jHKFgoUavR0i{#X65_=S6bkyNtB{61V^}y{kbUo zUQKtY40$w#~PzRFAs&ZzaH5I+?9YZBeruZDZU6G7dFL3OUdZ!gJUE zCKdl}4EQZJ)So^Q1z7VGMA_VyACNsc@Gt1?b?W_H4VbciEdmk1r6=*>3i8S5Pa^&1mDC5ppI->Ir4kkq zICWfFF#t~A0nLJ;Q=nm|u`E^(CT+hdnM1bUL!mLFpQofCz&P3CCB!m)ldjWCy-yxr zaG1S)g^T=l);++hCk#naTok?RQBfQ+S{ol~kqUwqwWQDR zYU+YHx+iI)i8u}UqI>AU8saF1t12JTRq-VX<68x1K4vs!6E2H1y6XHmui}y)T;Lga{WM&6V+@YTp&Ph097?3)F7h7SXtwhIOeS#+`XQbWkMviV&m*xB9ZXrnKhCREOm zuEV|mu$#4;Wi_z(@?>_E4di*hE$ZaS85W59eztHZDes_kXs5r=8DzoFm0=t!p3w>} z3%zl&ABZg3B@N~K`Xjrqb>XLPKQN--TzzGR|6X5euE@_ZNER}=DS{kuN3 zocPVru^o&_>+fF(p!$n!4I3Ptr}iUV3ruc)4MoP*q`?n1!m6t>QHIx>Lo&iFJlr|9 zu+NaKa+;wlS2}5Zx&u3zOMA4miK|+)3t~c`Ww>sm)`V(S*E!pPIVSgLppBKdU z?m*r!nJr2)+!Xp9M=LtKIfybt)Vx2Hr6S3ugW%RVX6SwB=B4-QBEDpt3+3)$&C%QbtHdCUZ|}DigdF3_KDZXQ6+@(E@&pEQOPG|4DQI5c+OO;#-!E3;RhdxKlq~QiapJ| z-0|D&JMamZhyVPT`2NS=@5#HB3Sn+9IMIfH@gEdp5(sUw*tP@PR7bczPJ`|C>Up^` z55O8*H6U>jV%ECDZ`LzlOirpENe*X4&V6)J_ELOK_mSzl04nJ==R>}yPMqj1h8g0F zIUoNTcwq_UitSJvebxpH4%Ap&>n%<44C2Hrw(lP1%Y(C7aFKn+u%8d7+zVvh&xFi< zFpu^)uN-+o(dSkt*q_cP==(8{mDU(OwE#9f?39~-tr_#0U5o!H<27Dp4-=%}Fy>h>KZA;`Ox&2#0 zJ}va~ixW}NgTvF;h@)R~DYgtX{m#G2eMaSd^=+E%6aXU+oTNAtmQE&iY%(Jl5)IDX za=+0OdcPkhCnse5-4YJMQG)ONpYbQ&FNv1>1tmsp)s9g8G ze)oMlUl;2o1!AlCpx0GS!T`Y@tE0-mp)RtvKSTXwwyBD^s(O>-`tTu#E$C_2a&r>q zNH(=7nZ7Wgt&3~o86(v=yDzYx&yjqOe@G-I>|6A>EBa-><>_bQs zq31jlp`-4De82~BK~H}&6_5;rJs14kmK~6_%kk+afnIK@fueTUTfoC6mq7Jt`{OzR z5_f(%+Z2?1JdQMKl6g!Dj(23qgQ8tGem9@kVb`REx}y3o<^ZiN&&|Tqd)qSS}9`bC#0iyM|>8oUkp+8%#nXi zn0`Gjl1>U+ZMjaZSml01sJ=R#d6`bfvY@oF{0aj1o9CXgyy4B`^Nt-S``1wI(f@-P zRC4>=8jW8x#H3Qf6r2AYei4-rA+vufI-jeV5p>PMqB5j(W(B0sQ_-@K%F+ueOSvgB zejNA#kr^tuAb`TgJ~>O4;AffVuex#!e~TeNPS$A{{G_fo^0|Ws99^xo!09QS(%C$S ztKNZ|B~U2cj@#Z}T`k?wahe5sT<#?k1k^3{>UrsCLI|Qj1E`Z}8H>w1IMmJujAots z(d-Xjxkpj%pYk~^)g!@RVvXCdGkH9 z==H=q=cAR?_QXl|z>($zzlbEv9Nc8glA@ak&UB=V>_J)k`CI)H;o&8OU4~3(Yi0bS z2HFUPl3ZiMks`xhAqJU^#j!z2d{n21P1uCwB}fcQg87)H&YgZdrnNTlI#10zf5g~z z7Z`0-!|_QB{}MV48}tudL8@GN*3xUOFEg?;Gz1rZg?S$NA&rUN`6i=#87C+OvBC`1S8+I|)yjb>E2tt%G7PxL97`Nb^#qd+0SfqH{< z)8cajF%9}^->g9|T1)gYpv$UKRba>A+HDIl>Uo#9Z^M*dL-5ze}Nd|CnwG4nUFL(D5PH;2_D^L5168i|Gr$x>{_7( zQW66ar67Q7eHgM(dmuTz1e@1iWKKtu9a@2!0XRECpUdIu^|IUc628(nW7is5Jk&dT zZN2$idUQNeo^LEiHrB1vchjozj^^rGfdvgftW&Bmv3&GL?PkZy3wj>rs#SEufii zklB6vW$b0__kojh52feMMB_lINf6RFaT@DC``AhJ0eLB8BZ7U`6XXQ)dI`&;@`~(E zsASf%w>WT9nIBlW!${rmnC|4biW+)VAV6^LG~f4{-{?E|CbioxL_%B~w>+yb)qT#^ z8&f!NO3ZjH>})vGYLPaFJc)XFG3Q1w9Bik*(r%qEh?GPS|4sS zia)zFx~uYIH}B`E{X+tB;*2d*WB;9`iNaFC!uh z9~O2gk)v#%WJq0Ej(%izvCW;rPc}dNszNfA{3nd#oF`ahH7a>qxLZ- z$Egt^JUVRXaaF-+l`~P0VL8~4kk_YWSWI<0{doXQ{}lHpVqtML^hj)PWbY9+9D^)D zxm;}T;;rAWqHP+7@x|+}#-&Q5k7gIKPSP32v)su-H>d;PUO!+!k zICRvfyISlPB_QtS1eca-L}{VA6Km?}RoK*3is6z`?Sysp?E+H9W-2o$?Da`~Y>eDS z7rmu28jf;poYpV4OR}sdEjTU=w{0Rq*WXf&^**a&yd)7#?d;deO8ZX9Xm@vZLK zQeVug7bC+(k{}PtlXRvHjNd2kv>k>1MNg}Eac+-K!tHNo(T_U%_qvgEg~WvmJuxT? z@5hZq=iGPKVI^I4@{dmB4l}Ik%e~h!5#lJ%mAV{g&C~9xFK>5RVwY9vq^zd2#?4fuoJHoQy%XM}Cyauj>$4k}A zj1hDYDQc-a++*LZTxMR(Z4A6Uy!V`l+=L$e-)^}R04ob3W=#ZAxSb<$hE@Vw<=mz? zW2j}*RY6FElisB&2ko^1NGs`z-WJ3B{M-bx^VRB>}N8M!RlYMZFjJF^!vqc`E?VjJZklF#L1C&k8m@`4cB?lbNb4{`>(sxH! zvF_lm`GPDu*}MofUKEs&0>FAi7+OZZI3RfuN2*5jM*#f-A;?(Zj*a!)H+jQ7!abic z**fLe$KyXDEIF^|MpQoKSM|@_Dey8pssHOFV!CZQ2RiCmg@NJ z_q}TeJ>YO%c3o|1`8KwC&Or>2|KX(99Wb0}(7t6}EadQ{MPBe?GCM!9E-kzx7_^DO zCM{?-$dcF9)Z*KC0D}xwq}d1<0N?^1Tm`GC@wwZTy#1G(JHG#W>iBWB@11Lm2X!4Z#pSb~t%p&%`Q%{B{Ly&}Llg*#( zGq9gve|O*gNuE3@h1vzT??-9z(sRWDm~{D?}4@BNi8R460$l4Yd4(!dEL zajLzqPVWtqGV`LJ$epqMkkI8_v**6SiT0YF5}lHI-xGeC?qcTl#ogoZy!b+%cbm-k zsO7+E9x0Pv3cOxea$Rk}`<=LlBFYF_)y>YM55JnnJgH_7VP!VajT{;8;P*eCBV&D^ zO>k-atdJ8UJ;SpZpDcnT`SlaSPk-2y-w2sPpD578Lb#9p8`E7R?~O$PUVhW(rb)K+IsQ!~c(01ra_|KaK# zqbuveDBV=-q+;8w*tTukwq3DRv8{@2qhg-m#6B@9NhRrgeY<=JS+YlhrgTyiseMFB0QE zKFDrO)^K9%v->ybWjngA#-m}$+j@n?NCNpaI{O(j*K3%A8p7Pem|0%v7WuFrre}voK9Hdq`sHGcv%=E>OYbBHExMXz=enkMmu_qA{39_%d$PWQm?=al z@GkY~zjVr{m${&wz+3E~1?Il@r%fYQHBP;XkLNdv<_v?G={$p8tABM;@8?={J@qdD9= zmS;)xV^xk7Ml|8AT@5Gb5qUJvwGUzZdjSp9oO!nU#3 zEo3t_$Sw=$Et4AV^8YqTV5Lbg_-}pgvK;DULMGP9F&HX;sTIF<1UTzY3@Gri@*PUM z={xS(T}f+{EF7r9oKta83uznaSCKi12nA<=BMpdcL{ew+a+~V7hHf%?dt&xhT@ZA3 zlBPw>x423mMy}Stx}}>?be*1_s*rl^`aoLr(3C!w)RY*eSG6Q*i|%v6cOo*%in*#W zjrvSSv4%*WD-R4bBVK#RSEeU|_)LSkeJcj=_gNqtN!?NSlP>nxAYL}vpna+v{RIo6@)=>OZ{KL#PukvKu|%%{X<*Cj z#P)^6oc~~?!$Cbxm{BDpw4io}^Kg5JI}FWvDOWaNQvHnup-1 z;-`8@iP@cSk&>#aSMXhP9!?dG-`la>pDzD%8~OIb+RhN{8c*vP`8J!#0na^iJ^dH= zp9pdAqMaN4b5$LWgo73rW0Ar=>1DX^g)aS~UgQ)TbYQ=t z8?u&CpVd6a6;nO`uK!8k$-jEbgO^m|x!byr9DPNdDh&@+!tBHRnaT*USNtarr)B|laGfU*^^~sdW6;J%dS31a1Cvcl_DR1Yin3`_^yfgl48(|s-9c^Y~l4O_1$kD>|gD^Aa~YMwcG;3CtOqk{p;6WZ2|<({n@Ub6-f zC&|43+Y8{JYYob$>87Xl6hjk}+0^}!|~Ap_biBRNw-FM#5ttfD-Jbh zkv4x8aT)=XM$QpQF4wlj5(M2gLK00Es{>^4GZMxmNVomt}vF@1_htA$IL?Z!l8Y=dlnIs?0r}(RvdV<%WMC`aS?xXIdo>I zm&yPPI@RIR#A_iM1zBHA@EySr>7QtJU2Z%Sx=HXU#Ed=ttabk#!Xec-TTjKXy+sw1 z8S1xvM2)t(lGB=aEolEq`-XgrFV~Kl!ov_yOOaspY?!hGgZ2(`IJsP|Apse)&PP+AmaY&rR z^W$PyN|06Dc|OfgLS6Pk$!#1T?(Q|*!2K@F#z~J7p|YTPkn{oiV=SNin5k{8Z7MMATkulp6`WTm=j^s@+9uOE z{k--&0yu)24a?l?l~dE_-Po}wru+)SdzOL4#KBvSjaigW07Wz`fF8can_6jHwSMr~_JuTA+rcMA$5qDGcW?c?e}9TJ^vC}kg4 z#?=>6H$0tcDY_&o0&;-t(Wdf%rGER&9a7+LZt3CMm_ioqW3vEUAH_h6S1doKB`Zc% zeJAT2)z73Bn+4(+rheJ(lD4*@E{<{u9dRI;IA<63H9A0Me?Zv@;NouRTl2SVjr`S6 zgDCN&1elP3f`JFGqU7WYr?1epwrb_z7g9M zK1m;YRGi#!>QMTrf-+F4^e3k#4UQ{;Q7RyG#0NvT-SX~gT?^cT6p}+VdwGCP8q>!SQx@tBl; z*a*ZyPvoe(!`am(PkW2ZF4+~S>m*~w4<-*_`FXHP+mul~gu74*lwMYkh7hQ`yb5>| zUlTw7*#%zL<84LrhLfo^+V0b~SSMd5@79m_fD>Y4yWT~+WPK5u2mh$sjfKCndD{~o zn{T0$+Kf4A8}?Kdi$8)sX@0A1vyL224az$G{Rg935{T*Y_xnR4r$;HPz0V|n`6xb> zx3k^QLT}dUuy;9C=g&LDprTWzp37yWDh<4co?7@hEH61ht6IH^7wzEu`(JPiYLrUo)S1IKa_0_RAeX+@?M~#D}#LAWWH2p1uPU zovM#p%DoZ+y$WGe^44ysBo=l5BE+v)H6 zYTZXYTqW7Qu7XK6pjItg;P${v;Ki8&c?hw@v+e5o^uBu-7Uk#BkjNVlIdE7~6&{Bn zCgDSR)gz5Xg^0r=ZCm@49oaj#`nqVY)Cn0Yt(Q2B9d~Z=9*z;?gaf#DG3s5`r_GML zB_@<6uzd_o%WE+P1}ld4N@aKt;!zVF7ZBQYrBB)B?)Tf_=hVZL-WX(kg(>Y)yjboPfrB ziDUc}HTVpEM+sU5tJv)?eYO+|(TX;e*sAH&6%KCmFnV_`X%n|Hh-x=G=-Y8OufUMd@9bzdt$H%NG{76KJMx-!7} zZwHcWLrmZ5irW@7$$x%k)Sw61M1ee_HY+Y@0qdR0P(@a( z_q%}-qS92W7HK8t0Icmg5(9|?18H@!$YK^m;ocQd_l9}+aBcyd= z@h%pf$i9*`&xAAi7+ zsNO9CnQRn_Yi*Je)O9lSv+%7;R;#>fZe3&?RC#9xC&K@zrxYWf{YLR1@&Noe+@SvO zsWh9Quec6|g04U$B|a0GrLQ}l?U3t9oGQd>g*maO$-ni1VDW@qN5(X65^zF@k%Cye z-+n)_uV%D})Ozl?C5pjv7XujCb(kX!$8xf*+CY6fOL{K%K4YPZq!fBE&5!&LHoc{z z9J-YFrkwpqVCpMg#BTSKT>LfraFb@+<`4FlV)WdAMe4>ld5VmJ>2d`bamN?q9t_I3 zlmOE1VIWDZ7%BN5&3G38#-ZT8Pq-)$V*8g24loA)3xgBgn_bXFs=fU9LH`Jn+njk(L|OfF?Sa)^ctb9OeABaJNNlw3 zK+cH3X?TT^L~di%wC(Bitg}g+)rLQ^Ce%XjP*BjjStUL6MFdD# zhMh7J=;fhMUhCZ^hj33F&Ehnidb3|=; z$+dLQ&}F zRJGzB^gfR8JXbQTNYkxMSXD!Dg>7IUS9YQON8xgD#fn@-G18XN?x)-*VgCEu@OjbV zgzx*d4WaJYsDj_{a9_U1-B0cdlKNM>%+gthGXGZ`vz0biIqG zEZog?F-HDkU^{0p)jlcwfGtix>D{6#Y+hAN_kN=y6jDmoK|Ofo1zaL7P4DpRqO z`xCGenfr8dHdHCV-#Z8US4JRPM4-x$9Nyf6gu|}WdxXeV2`+$-6K-!7t7yg8Wh0D$&$w`ZT(+CGWPYZ-d z!C1qSbnBHZO4!m*D5+CvWS#0B!PO-O>5(`JSDXoOz*4T1?UqEpQ+~F|i7uvxFOMPR zw3SNv@pSgnbY;O-s5bMob1wnGp$ClK+c{b~s^=p9qJ%WDj6g`I?I zjazynv3jM(6ZBB$NrEzcc%I1+S}m_4);K|5Yb3c$PksPIjTUb;hRsFekEW=V=_cThco}O8ncfterIH8I851>oI@gm2X z?=9F*Yowg93jnw}yEx~*=beC4csU5_V^d5u-RJLmfjTq0g(gl~w-$;1)>LpsLI$dD zBdFW=C=B9qNbW`~FO)VK@A5r~oAgJpI|rIw`JNGp@>b6k5f5IBzsi2(IZ1n&{wi+d zQh+mIRObAKv?K>7)orO~dL^(H*PitH++odpde5dp%7g0%kW;K1`(=mgi_#n@ZX~%(oLh{JRn}Q-t)9u>qqZZTA$6rS8;CUU$qT!w!e^q z`O02nwcwS>yCTJHs&1`IIuFDWa!ULIYJ8EeQ7|}dVI$o3ydMa^z^=&bau62jD*E+z z=dRL&HkfP>18xcjobbG>XpGE5ao#RiJHjw_*V!}yO+XBD5*{>BeM+BuC2xlF-nfJj z{1=mB+*jULR5DyXj$XDGjPArS!Q0dO3K5EECmkUxwJ&ca#KOHdk@Q8(c`FlpN8&;Y6cU+o*fy{4;D;DcOf=03 zveD`6&BLOt*U23OR+_?WU>{;tEAd4iL!mK5=+FBnoee~L9^JeA4 zkXA$?Mr~4mh>%G~_1+VhaTw82I*>YF=C4FlY;lH9-yDkWmhI~~g>R2+xtcqr zGGEn{&ez6IGvDgow_KtVZobk~`n*MDAfM>4>||k-?$Gy|7tUG?eSxkKQ(tqg{))VU zee)q=Qc@8Nh5``(;_#CDQ2NkAr59mg8DhteJ71W5DrsT8G#`XsG3E`{FbdVdZ%<0b zYt6~p+W8#fOLE#MD0usG0L$KoNE2){5bAC{OD07J#0EPj4z`|@Hgb3Tf5+1=G(s*w zV&roh+qbDtYz9*7)60Epb@rbwdx6X z_O=iPdbP+{Pw(fh_U|Ko2cp(B!{~siTfFjD@f!}bE6pWG6l{yq*!6sE05uv;YoZcT zM=aJ@X~vUBqW4t`*%moe)wg_JTIBfG8D$1%YJ)TQ4b?1};QHjdQ)o^FB2EiLQJE>E z^jaNR#y>~HVsn5}#B!9h1__~Ryd!r#C`zpe2mkQEK)9h7Lm&+Yy+9QRhnU10Iu6NZW@(J=c3UhU!zal>1baPri3IZ%O~Cf zBOO^U?Vr!>NXEMw6dnqYk-S>=!e0%OG(Et65ktyN?gTcoSR7X+W8>AOZJwY*{t>v! zV{jB%YBX)l9ANP5Do`cjO3}oRuD95-MYo6OZ|>8N-;GfA`_X0*GnJk?chOmYNa}B0 zMA+(fezT8)xz-VH&T28FjS9S}2s+J{-rV5`R42D_O^mwJ1jewjkp~%}GaO>{S`$rR z;`_cx4-0?&Bqb|bj_w+T1~^|X(9Y}ynd_BJ#BCgh-E@ahZw0O_qpLGH0XO-n{n+U0 z>O@lMP!&@?9Y)!mKwi6_BWRw9sjZuI?d5A6^6<<&TpojVh`)FksANA_9A@`44NBCN z%;xFIkx;Ny?rAQIvd|*C+Mn(|*cR;OK8?lMiXF^6+1_;19DA_TTV?>WjgGbXvM#oi zU}RF?ttF}Y*W$F5x~37#AyH{H6lz_G{d~acj=y3d#L1;*S!hq zv9~d*q;L(|%3R@fbGRJnQW4p|1cDE96NVKmot<;&dc#GlN*|5fY}Hz+5hGqH$diR# z{K&ny-q(*@S~9oNtv%0?yy;sEbE#7~>(<@jOyFh3S(w*cpaxb>63xDNX;yNy-6M2{&o=Qz{qnk`OY zw|@NTR-`);A+$MIv=eS30)!{M2St`(H9^%If~PRM^?r5udZhEA4F6Ep)t$M{-Pb&y zGt8=jn!6a7qjjG@>3;(_ItL7+UsnrUY~bcI3!Q#T|J-r!+qz&k_NfcEl8tE~F+=_- ziRey&)o|8b^ce~*za3Bgit>7}U0frLy`OQm@a+*qG1f{IZ{Tu<#V~aQU6O$oKhPbi z=yMr-=9&{w=AY`ODdq&89{#tdC($-ay|#w!B!Kx~-pXpnq`h>^!k>A)@22Gf8kqY1 zZ`4?y53XBLen*|xR=xp_GjC0OQ5>V?e!Y{9Btp+VwjslKZC%x=|K1627Y49+jW#bo z=ejusF`%wdgKwF|h8g44pP|3V3ZTdo)h$+t0HZWeRE`nHj8>GnGH&O27Rk{XHU^KG zCZRo+?t$9~ZxU~RPB6uqU|PAI7Gf{P2_6isfH#AKq8P382FNgL(~s#Ibmc7g-Ts{S z+?d%IsGQNCHWk#jZZjMZThE5Zbzt7J7GuZA*}qjC;i{r zugZ9DBnFF^9V{9Pf-VMoO5jN?#%8eshwwB2K~kkSuGgq*Jj(=J)QX+7G&itB7+RzYmG$oZ%Mf!?g3xgbBTfX)4v6yk@ zh#FGUD1#g9ZMwny+xGu`(or_jobatz0@KKqJ$j6&cRcMF_g>fAxYkl$Dtbg1WummI z)=3sx`EPdeeK6qtbN<$#n}^eKS{^OUb-y9F{vvy#^oU}H1L-CJq@%ZNsv?rcJ%KYC zda{$lZQ~{e=TtGhf1EI85p}GWm-;~}fJGMLHSNGFn;vu|vyty|USLY=V=w*CS~~w)*`Q zEf!|P$K*q$y91fgj3)A`^kBwD96f45>yo7SNX?%J(_91kxe1^T;RB(K%KdG}2|xgX zG?0&JQZNVTKO0O+j(j$FLJDlw3Ofz*|99Qj{Rn3puV+uYu%{Hl<@1v<=6I+vty&eYxPQQP~>$rpCdh6D4Qvq$v{X zFN~Zg@hTo(-PK#9(&n5&qOI7M_f9%hP8m%*XxHD8byR%hvq*X}YT9a*0t)^p^&m$T zuW^5X7DTYrRW2gyAsTWjT%uNgatHC0Jfd>)lt%{(M_EGY+FSQno5Ux98!0;Lg0z?> z0}0DWrS5FP=JN-k@j_a)NII3VhO?1^J#Ok?s+nkx?O&=!z%O<)c8SI>SW$znR!P$e z6U;>?HGFB5=wwd32vWs}P8PkC6$f2~NaGoBjaIhKsH?Y4pUT+KeK;$aDuaqfT2!t1 z(08wD&sL+Nhgm&5O*85f76H*pue=*SF>g`j_gF|Ik@HfpNqqmZx`qHFP-!9SA+DH~ zY@Cu%p8h*4Di}3e6EZ(}ImPX+k^BDY&_%)AHJ^B*_S%#1!PR`F4;E2c5Yk#V^-hj& zSUEovjqzWMn^y4vy1ZMd$E2eTVe=b8_oaK$c(TECR|bX}kK=h>%dLJSBYP~pUaAU} zJ*EPzaz_Ab{l)RI>Qn2-Ccp8XGmXRWpI8mbejam|!E4d%vYGnUP4A*%o^h5wN4I&i zjwE@O({XnYpBnXpBR(kWScDlOwIcmD_}NYU!$1=juPQ`M!u%9MXG^V99}ZS3TdJA6 zUoQ=cZWX5j9}Mk?d8Gt;iAS#;=4Kd)M)(#q?efG(=5vjZVv|QnU5^l~@sv~2#>8~| zD(lW)^BOY+Nef(@WzW9}pVtY0&=}YnexHn=ghhJRo9#qbPQKkcg;jn>2S&N02Cd-9R}jnm&peM=#t)&AsDe zTmu+W9OM(GBE9y7Zw=q1XaFU5XOX5LboJ0V`{>WbUVQo|hOoP7ay}}lf#5py@VAG9xW19o(mt3p=#$7x_9OI~*&|X#z=bTiXf!P)ES7lM^wD!s+FHk_s_~!qaz*(7SfYOd2&kYeDnER0y&M7udD;f=LtUXPkOE=W zRH+d$zeWWAb}?V)dev!!1$xUUcgJu&U>`_N2|j;#O*~N^Z@)#lz(Q<%6zOzV1uPz8 zNZ&5pmn<3`Zd2sJr9q!p$@)YVHTyds`uP1WI}aEdq72!g%e$CmiBs|OnhVA6 z7m<@<_YP|!vN39MOIPHQy%*&y)xwY*d8ALQ^n077^WN)uXbBZa*|7r}0G*skjDS=E z4U|rw+=0EAD_U*Vxg%h`%)D0P_^&I((|^kZ`RJ~DLJ`YIalbD_-hlDx;Qh`iIdNzZ zhJ%!(_#X8=oY{lKnuN&p87Z&zDkFDn-gA07xV2n(4e}ThO!qZpjA8|s9E%tBo#~Cr zEZ=Xus(F{&4C>V3)$G-1_NAjVVPa11TYqaY06+OG>qN02>B{UPq`LHNF%e3?65rS4=l2k1QRDQ6{&tFiUOYkdZS-N(3}#>Tqj}WmT+-1d znJ}068NhLE+pQ`93m$!&9f5$Ube+i)eRZ)S0(eUahFR?VvD?HL%Pf$#u#Y7S)=SLK zDr$2!ZVPkcc7OhZ8(kPbj1w5cOlBS17q&+v)FyBFCMSA3g<($x_XY zW?9u%c=rSb?q7(qp|xuFTKJ_(FHXOT2quv+uu9m|S*{?jH69}xJajD~-{q16KYtf< zaUiklf$lz}$@1}A^wd2?O>i+17>jx!A_(UgF8juo4K!Uy_Zy3eOC=z*n&Z(ohK z{4PC(_lrNk{jZ77%myA+Fn{Gv1m2E~{S4TJAny2wf1%a4M>&t0a-pJ>$q3}I_v`DR z7%89{>I|j2syRp|O=5HMzEZ;c8I<%{(|dj7=6{w2@TWbnd+*5lh*Vb+ z(t#W3)E>iBM1Uq=keK!eAAQGiYeB8Ar@dtd?gR}6HeRTZ+J70AcT}LqME&9U5Izhp z)lS{CjorGMJ?V%)*MBwMIf>?eM<0WkQO*AK#Iti|BXJjT4dnlpKoP7n$J$Kl(VsfA3t2nf}F| z?ugDSYpt|893BX^L82TFe^y~6Zhp=`0G-6LV@Ro`yn*E3{(3UuxD6gt^zEf;j?|X_ zf$9tISgI9%Y@~Y_j|^Y^xcVn3RnWqPO*qNcPfocqGAze zxmz9%j;)uXU5->q_&Cg+bnxf|151|=vg%6arAxoEiOC9hM4AQ}g@#r?I8ma%2VTa= z?xHbfURWweNhTovxy%?gaD}=|UjFgYfAq9@FWMqTnVKxeW~1PRG|MKI?KbxLbF+`xOO50?W-5Q(D$(KzUL*w@oW3t!m`-xY z_e2yM!L)_c9MR_wDKB)on8{zh^fHr4$9QZW>Aci_UfpKc#^D zW96r?K@jMYlvLNTQT!sNbzGC+ySR?GmxE|kuF;iy1B|jRz78W`DvtnzPnsx3p_kaO zYfSRMW)suk_ltOq!b$3f!`=6g9vckV3Y9dW`{HG9zY`xR?B1jcOjT<0SkW@Epw!v( zZr-)?iGo&7Om!k;@GEX8ROw%1Eq(XMdzs!RuL^IT?yYSd&|#r2YdR`zwH{|V=;uT& z#Oe$Y#fQ|8JH9zr*rM@I#QxNJ!X%#xy*AiAUI0Y6>#%kYefRLyD2<{Omf>J)X1eFVR zK0}<}Cs=O|lK#!gAHl+#+g+zU*HAtZ17^ss)=-%a+wBXYd>OSOJu2+)d+9~fJ@vf% zm0a0u9^@{B^#d3C^%CI}FgNG^!J39kqw(i_!o=6|gB*N`Ff@k%{!XC@zmvy*V&kDrDVFCL`pTC^kvur%G*CfnO z5VJSCS-dTIFb>fNB0L@W>V6K76X|b21LGH(i!(;XZ`QNISI6T0H*JR<^nsz9ZZaoxeN#D|efrZp7N^G(0aCQrSS4!;ya`l~zE5OImM`?Hw7{uM(~`I8)b zC-d32btwdx+R8%A!op{%v#?mjmmgY+uN{60<7T{-_sbY_bJ>+7=kI(mSQ3&kapLIJl{bJpLX>^#^fu}cFvrG2M zp^?KH71>F`(lvOcZo!J{C)vdfUeA;@r7EQ(q_`CpzE^cRzcec~SCO^(hwh=E%FB;I zIRC0~ASx~CE)$nd(j7stlN3h$V0r=PLV*x(IN%|B_T@7gW5H%z_lL4O>FS2%`vd#L zc~9kCy#_x#;0KFo0(^Q8eXI4G--!c7n>qFxuR-8i1gmE|V5rdjcX(R&%jz$I=B`az zJ#TD3j5$f&1Z%{SyZjh{Je5t&9WNVcTM<$xwa?<_Lc1FmuWZh`kg|W_IEh&g-a;0! z0f9&2xAQE?Gst)*0TDGUmAY`yVC);{taAS>3WjL;NAxC9OK@>n@Yyb*|; zGk>+%C^BzFAl&^ozFe zF}P^q09N8~91II)0+f5B1|cJ2$iFgCLQV!b4BUJpMeD6rB9Um!T2)%e3O*;zP?p)( z(2XLT4#l?T&fj#OeULU){@fm;X`n1ni^W8y+O%(o4yoRWi$7nh*J1#$Cr{XyrCztQ z50@G8)_s~D4YeNa9}E(T`ipbgXlC+p-bebGQN$myDP58e1_uQmA6Aq96KYiH^tt*z zZCJH4zYYui69b;C7l}o-B^uol64Tc2sx>^^duqm?Ea_c|qh9x1jlc9ZC-@Q(0FW3N zH9L>+Vn5DG4gN8E>5QKgppoe1T_{34CcJTtHX_Y=wq|M#2Ie+hPrQv+9b0Vsl@)mD zM0EKwz3yZ8TT75W5c^%Syd4OAm~{q47dX*><#lZ|{F`c6A4U-(An=`b&@*>tqW=qb zmOYUcK=GLNKA!d5HNCN84){<2vb=4Ku=w?D9TE7xPu6zBW{l$N=Wv>9%h~QHqdxCU z9vPl^h70jAMZi;{lM{)M?U_??E1T6%_Ay-nIH17Dj8VtzFetekO*`iWN<-5+Bh>Z% zQw3CmGcO?m<~9)O-W{WerlO@O({gSNh*eD zIB49Dg1LQp8ifc<_{>K_>IlIpv66TE$tEY`_ql^!+&$50Is;isos}BW=(JfFIW~53 z0&P20OU4#$N;X&uFR>Kg#amV}ZT(!#l6asDZfqO|9zI#Dq@Ml_{_(vt2NX~@uKz-Q z^Cw~j*^j$dNpjVLk7v`&q_^FY)>m#?#!qo`J6htc^NJd@chd@g^UIAc-3hC~!@~JT z>!qw)p9)j3s(RMMV35x8VR>Ci^~AyJRCu-(soE4;=B;NUH@cIo;<|(YuoLb(mTc<| zihn?L=y}a&YgaIVKD2x11X<;024yU7_jJSIa5nWC>x&85A|Qv0n=tEC%m)Iv+eR>{ z6uH}exSgj<_?sM80Z+@!5Sc0%Qq!Ghs#1PqHj)pN`PO$5%jWniO^?23W+Ed3V~dWD zBf686=7~sGLQ?1zYVEntZOYTxTkbTaJr8KHrTAS9_e|Uui5D2b@V&AH$B9w=P`AXOJ zk3v+#f6g67$t=~8A1J2pL!0-SV==|3EF8#3SEz>M4{bxy$3Bg0#+Xm}t%5d(+23&d zl2Jw`<*gl~;$yq-zC2OqVjx=3RuxzWYS~P*h~&o#Z};+cdV+?CWG*p1X0k;T#BM4l zp%hs2#L^|;j%tR)n46ftH#bMvCMi)hcI#p1ay_not*rV^KykCTfmuI*J1!zznw$1K zfJ5|mdm2+7#)&$r5#lOgOMirkY%K_ngnk>48a7p>;{dRl*~i798JzcM8P}Ks`IM(4 zQ3CmAfsJO0g)07Zm@ufx$ zu-DUyLwG%rRh1+?gM;KBqv>GC53?g<|Cne=aHtc+w4EN4MJIGFPkbO=zGe)6}O zAj;ajU?;kN(d;LNa8zL-Iz@$JfGLB99%nY}S-IH2WL$5a4(t1YAPlbP99JULdmSF1 z$qR~pVB$(}V>!Wv2&kR+B8wSj1MyJWzz>vF=P~R-o6j`OyoGD#{bdf+6tt`qHz)$fWVxyS0i)MhQG#Gz*$< z*YjikZ$U;ucRF3l*kffAVXctlSDv#eT1HK!xmF*Np>V98yj7*x^;iqWT)F3=!aOYL zA~GIQuXEn5R&y9Nwm>8oqF-Z8g$a%=r0P*7c%U}2Ujs=}k-z1ZsH z^Mb};m~J_z0}pz>_c?S$ z9pDIV#PMxuVf$Eoqm7#3z}GMJ-RuK8lis(cN|DGMLRM6zl;eVsfo9^MrLR$KM(Fg* zyVtDeD$~nL^WP~>dD+{Iy?3e3jaQzL1I&-9r8Cm*e*Y4K)ND6Gc_o)#+WtaT1Zngv zr^dk^HKqyh8iN~bysIA9Wv44Y$xDeXi2g<^`HM=Hd%PV|3)v8V+(d%xjNSlGsX-U*fzy&w zQtd%&UjK`S{6Apink@M2Mbv0-{{Q=-3)0Z-f67KIa9h1VfFse)&yP3I?qHbXOoC*I z>-SNme{>2?#$xAoWDC`N}J{?7D3TR6yv7bmg93PU93GWA|P=*+9;ODe2WjT6PjSBP| z)mJs^NF1Zef);m?0jCD0Zvinj{Jm8Wz6d2+?H9b%^WiX9Bz9S44KmDc>|5n*cRb-DI z6TC#;KK_mL7aS_2F02;U{EtH6Vb`3VfWM*J#m2HhTyBw!GzA5Cf;GmWjGS0!^<=P^ zF8N^mioGGJd@Tl^wC()^o_1swv({w9*$B%Ga*j2=d45t1TN7^puYyQo0Yp4t=X zX)rT73%A2;TA zddQ6{Da`3hVKK|-VQDCZB^Js%uzc*VF9N#w%4U6@DcFBZeDDFrFMGfxFe!k|xB?X# z>|}Q#+hFk*pJ_sO%Wg4rOTd2T*lzaFCU8PXJYsk3w~OKvZH=68Ca8805bv+m+u~v# zkuz;~wK1EhcktyHpRs&Ydj)=Eoq&qFI95o$GmwGIDxD)=8&g&;ib$i9l=?6QmGR?4 zof+JRG_|GDx&As&6RdRDvuo=bma~s`-X0Q90iF{IEzW7MN+vv^gsHHW{uZgb-f!%p zM;~mZc(AOQ*?$^p|)B~qE9mFb0v9_EmcPii|GJaIxlRxQGbnQ zk*J@vMDGG+viw{tZzD(F{5tl)FfVH8m@ZU3A=Uwb?anM>pcO8j2)A%_qJec9TnRDp z4$aQj`dw7tyE_@S=dFC^wWNyi@W8Wii*fECH$?ale_PRz2eP=F401DTlnI|m-*p6y zU6!O@+NJ21tM8R>tT|4)UM+y}`Ygn4?R6DGaH{o1+kKv_SOowSaN=rJSdZ+^Z)+|S z%%Lq~AYa#u+`6qIW}52JsyI!aIhxdUIM#a{iVb;Hr6vaOGz2_bD*HSv1%<^8A&?5Q zaGmXIc+@AbuW}5!fE>v)ztr?Xh|%)sq{&7}ZC%B?6;2xLa|LevscVxgs!38eeBSwB z!OVsgerRY6|0s`6-L--iHSmXqKcb+e$Qe50$IXa*0kj-f!YVC88{(KF z2mfAk;0%H5bxlG>@gLe>MEMHJ>WHlt_EeefX`ZoTFqIXG43E`7Mc*6a$0k9lyFJjk z(T~$cLvA+fcu_Ma&+|g}CXe)x`-^uxlJKW`cWY;~#qZbnV<@3YiHz>+=0Q0~{{x^v zU%v`9exP;M5tUGu?G=q1!SzUEYhCuY7w>Yfrd!*`?PR#uPR#c@Vl|pn@Lr)!p*};W zObQ-3&??*!harzL116g1H(7SsoY7ptgj?J$J`O>xvut{&CaJB(cCAUHqx&^==qqPyx8XyW&hK_^ES}D zY!?+^J1=Zcyn$q~d5y9YJg}#tJ;{ItOS~ zAE))6X?tY}>>XXFjezY?@IBCct?Lj?k^|Hecx<7kGgvihtUNY7AgV&mx23~ew{Oxt zut4mWMblOcNq16N(8)-5MJ8L zr39#C^t`F{4(sk+u6l@}cPiVaW$86~|9B&S0>-QG?*4OjZ|tP~WA+x$i*Exj&2Oi# zb+0Sh>AJ@K%!G6X@=!ZzR-nVuP}zR=mZS23*sur2uk~`fZlHQ@C|~qeV-ve=S>9aCZ?dBM0gWG;)l1Mb@eHF~e}8F@ zeS7VIf~-#4cQ5R-(n`qE($$%xbjo8dE&U8+ce$Y8-a?Q^g!Cr}=gr*{{;tj|h$4DE zNPf~3!LZR8)fQ;;F_j5QAA*=eEr*9<$0ubYHC_qIcu zk(db{$()4u9$6NmT0vT$4DGYKlBaE09)aVb4xfkMby!-Wu}KbkCn3E%kn%SVP1rzh z#*Su(Y*F*3-E9lD7}MNFZ5NLKWM-N>ot^9QKal>=N0#PecC>xiy-05zx@h;cpR$kW z+c=--RS?;3r$M4#=v;VEqRZdVDLJ0or@HYz)W%T;T(?%SBzoHWrgTE+cX)3E;@r?{ zByc$2j!rl)H0=uWk0(y`5*p$EyBB8qsyh zX-!7LPMrdU!TUOn%T>0|-AG%*C3{EedHdS@trpV+ZKX*{CqA8&6PH%i&p$~Old3Y2 za08)6Efhu+u1&s2J=`fsbgM*Htsm!f6H(Ncm!(lE?TReg*^O>@(aTBU<5t{Up>lyU zx=nnDzc6yxT^Vs7g};yY@y`o~G%0Pc$Hg~2vyyOQ-EO&H#^rm;B*tBBC1YIo+_0$0 z4Kx)f{@>5;wSh>+9Y#qh zJ$Xavzyym$6DIXg)~tt`{ar9oVp0&*Bm(0pdD0$nZ~W)nA^TYJDSMk1C!P!UyAhZ6 zI?}!*V?e_4@VIM1W^P`F0kv~CD63c@xfO5f-G((0SntB zO>RVB5?0dWk+QK6+dOGv5*^=Snw&y5W{x|#SD|IRV}P% z;UZ;$2d;VQLD}>SZ@;u1&t3nqzcr2p=Wy{(X~DXvaeH-ZOeYI2t2GRBj1&xmfgv|A z^j6PSAI~+a=}S1Q$xC#SHH8W8sOIVUZx^(BEOp`m-+C6;lCPIp$AbECKPiwwi#kH&gP~eWJ969Cx31rTiWl4$l-cjobsUGj zAxJzc$U8W8CAU|T3+$Dwcl}u}l>m!s8TCH}(I&#Fe9zEB_VMH?JFoux8a2?lO3A*S zKjeJRxaUKI)E!l<2ra}zi8ZAKLr>}tH?&YUti|iMbzKx4!mx84v+zy46Z&@Ov2g9b zpgxgB<3=ss!gEaj_566!nzX=N5!HglG@h!zq~GmUTnXqe&g8o7M7r0Wtql00EnLil zGH|72bR|8OyT%uAmt@qtRovQl$4~0}s<;u91!NYT>B~1ndvRf<)4H@c`Tox5{9bg7 z#m-in5tlY;e{|0JwHW=?^sQEqaq*pfr|tU4xP7T|OSKPGe;<;uts;Z=Ce&^8``T}i zagJy3+YMIgj8GeeUvSC+#z{H?wp<7O2wg%+dM`3yAA@s32KPHV$LyiZH5L&S+N#RF zw>RxHX`?dILiP2lJrBDv^a~4jJGVF#8ULdAETr68#C2sTE&kV`f0A*w-HmXQIniHf z+-6*dYHwZ7g+^hw7B4<80K$m468OV#2ZuKcvd41+2E`tF0J?-NXe?vAR6Uvn&e%3M zdq*teq_BI?NuW=v4x!pFmn6#dbtkZ)5EPzKoCU6Bnk`1-DlE&Kx={nGSG>T%aNsGemyNfQni8DP; zS%D6TzIb}R+v}JRKl}$%NA2udw>^~^ahe)*=tjw?LzHUYom6a;c)@`;=t0pj>Al#d zMIVAO_7^%H^>N`){DS>o=icRuODv!_C5zIztK-Uso^f5*jGb=jvGla`J*YZl6x~!= z8JpjM)Bhf z{c4wM0=x+PAw8}#8?n6j%^NfqfLoj!qT?MFj)f^{wKf7vkpqQgl~07wu^UhCEu_?L3K#LZt-ASd3yQqG&)}DH%X9 zFiF8-QgCBDAOJ^o} zypGrJp0a<}`=mX;GVI9GgbcwL&_hBpt;yhoVU_?la6`sramGkR0)f;cR5CotAi0m^ zciW@0qprB$)_%d2-&2d-ZVcXldrwtzl(LFRa6F#~Y9A$$>Y^ahUit0U@}258Xozi8 zt4?9$Mm{-lKJ!`g`D0waJ$c@~wQ$fsX^fQj>^f&RCoXE@uwet5^aUK2x3-NthapCB zCP4o&{nL(Ap-vdC4@((D`D1cf-ZD(Dw~AXDgQSdrv33Q>(TwL@Lzg|18S(&w7>eES zt#=%t!$B|`0b(@yz!>Q~%6bNdInWVIqWt^RWz?TKGYbVV;n!&2XQga8hg`JJ1n23( zs81~6mjq4bhDg09my@m~Kk`6Ms|wiX2h?6Erw3h9**LeMa`8^sVs!q_+yO_s?~Tk0 zWwK{SSi=JAd}^!k}~c5ug)9GWoI4a%=ZR1%lLsLkSL3ezxY zJ#X_(dHY$d)2{Vs;w8br?ag8Xi!SZW1)=48ePM%uJbZ7++awAmwke_Oe>!!{eoEZ) z&lGmqbIbi+-?5yy>scHSmm!Ambxopf-+SDSHVwJlH&mY`$qwUoMfLNclgg<*6ST6R zbES+7|3bgRZ+fw@!+N^2&RfZ(+>PRVr%Jpd%_)3mSwUde(`wJ&N9c8H!Vt^F76u?u_b#CU%- z=$RhfA_F~rLOGMCVDFO4)kLdJxtM$w)yCP+EXM8r@Qt?I8`T1mI7l_A z8(-=0KxVWff{F6omDF#|InSx;qD;N5>bW4$*>{5q4^tiK18CPC$ct}9hd>qPWYES4 z%l-V(JV>?1*Weh(1H&!*J5zAomX@=9n!JKzfuVT&q;_`R$2An09x}L(>v^_s^|pE9(7imL#BmJTlqo`E+xpFRrz|KF$GA5rANxEL zgLrVie!ecrU0?$t{s<-blRx0Mf~TkUkW>V4prTYkfi#lU6l2pOTJuVW^{O zSX6*q_gHDxqH34M4c+()9go>~vco=m@jjba5+}44Ru?DR{MecHZudZ9p`CQgz=Zu= z`x937!Cl{}5T$qok+PE!tF%^ZUu@1kfAJ0vW(i8RU7jx^VFeM?A3=HnC_JV``BRHM z_K}fi?4H&aoTHjWdjif0&+A6TmW;~Stp(Ca6fQkL-#a)1D0($qRt>x|>gsd%Kgs}~ zZj)SU7&|xV7<3OhE5=wZVlMS$c8@s1pYoU_?y;p)*QbO?kEAtDpS5mT!#XstoY43~ zu);aRw~~L=w=P%Us$-y?ns<>eIMO{MU;c>v-aP2KdiAXYESg-^SCI;itDK_ zo&o585tW4mUp3?XM5s$fNg=GYa8XemcXckhx9Ge2PN{$2uItO8z3ws6Pe zVNj6gT2`PI_Y%{4gm*oh#waZvK!agX5b6UmyhjA&`L)r1RbC2ADROBRTKG1Y2f!zf zmlhzJxxSKBCMBO%308(@z$_r?KbUyN!u!{xBX!w)SvsZ~1y4 z!(|&uEqR;|_{>YvIngy5#gh$c+kq7-XTRn=)7Bak|vWe2HEy979a zESOgTK-rKk*`Ri_A-!xveF|;6L5mW;@F88vWmxS!y5bWAxQ$ z=HC-W;dRv$FUBq{q&+Ea`vYwX^14_B*)%U#^Jbw0?rO?&Q}?tz^!x#9$jWf7a*@X6 zg>ggAv?evpGJva{$jc%!-{A5DRZm+dRuQpo7K$+FT#LJfN zr=H$A68Ld-ub-r(=*}-Yg=PHv?ce@wyYtRF?STg#xa|9W>6d=Vpes3i_^{ve=}&*! zf1f^m`m*z=U-*-g0D&Ky+ER|pX4^bGU`<_}R(ab2Yn@uR)4!6iiT@*FJ@=n=gEmIs zjaBttn*3n=#dz(1LTrQD_G8ddFu~d}$=WXU%N^go;AM7#Jb#q?b1q4>zyEvt!WX^}9IkKc5~xd{E`hoP>Jq3+;PsM#FK}x1 zE$^}IzgJO~Klp<`@Wt!kmT!FH8}`}He%9vZ=H$jM%Va%WQ3=#_uPZ9<`oroHs7s(O zfw~0h5_t6z_%U^_Kl-CTvQK^LQ`^sl-uAukeXpT|{r>O&{`T?9+0oj{?KKD8Q<5G~0$)xxog$lmhJYL}uZ;jLNJ_ji<(qdfwVaXw4 zHP`4j2JKB5dN(jet06)o(jX*rLrlD4`I0q!|G4e@7flKTZ>;c~+Ghm$)gB9;A5eG% zzk_XXPw;!^_SKvZxUVnu{GIvo`<3#py(h@iw^|e4LZdr^yY{*_;XbRd^>ld>2>8PZ zp8J)re8o}R!JPs3IKSa%Zj!qnXk%PP|8?nFyG|}&q!(5P?VA$^ZAl0SxY(M7JcV~~ zRY(Ys98E%9F6I=KUC00;T9?Hw_w?#6yEAg$a>=Ok2|v3uXz7hAxAmN;C7vpd+E2Bf zu+GL6xhF2@dT|eY`m0{J+pKfWU20*@Xk~e0K{KbWFT_PmO7e&~rg2P4%Z7 zO*5#K|5;IY%q%B8V*M>`7j1uJ%8sp!*s@R`m4Z+#LZ*aN-)6b7fdmo=%3VA0#s#6> zRMw&z{gwJH5Fw3%7dq+~q3U}6xQ_&4XkEzfRiQR2OOmUOKRmn7eq+zL{dDIEJ136d zW^pgy+jdHvjf*xYR4_>1)8!ufM*0Tpi{xxqY|#$HW<(z|Y!6TD6`xOEe0B;Vy(iQovDI=}`YEOk<}IQxA`c=WkhVi;_LAi}0PpFFeq`Ebi-7 zJ9cr%sYl?Z1BL44*MD?hjtl`sq(84Chjd&33Pb@ zNSkt(ZV(!&S?$vE)8UFJ8g_{`q)%ue+FpxXgW;ow^RFaiVop4Un`)!bp%AgYP5h&v z@gF#UtQE=FL2L@|)kYqh`e3*my*89FG3f{=htCgoqguLZ_x^ z(os4f*^u$^nKt|0*dEUlh?s1#@}<194VLYoZ`t>py{hu0)4sq34l0PD2qb?($lY1c zH_&=W5DRQq6|W(1QBX4Q8iI)aQvdb#WcdzpG#6>-fOr_y=s^V&;1B2bljA$=ypVj} z?$!RBZbR~;KZFW~EvX~gW7$+RX0NGw-k+}r-94slx}c>c6Y3{bHfO!^Aph2uXN3-% z6B@1D%ZmqkM}=vN`=O&MuhOj(D(H3XCmqS&Do4LmAit>SfTg-hMc8s`z{_*-G74IXgv_va5c`xA#8L zw@otC_ELR=+~W^Q7f^aowcM!P41=u|_IyvXmDf!9*4K`TAv-PfqVF_On+DZnYDORl zZ*Mwn<=U}P!`^KLkXfL#`Xr~W$r`7r`Fht?zP0wwZKtGb2r;HimkL#uRYpN<`}XyG zLr-dl0fDW(>RLR1;Tr%UM0=*p@SUnF^YdA4nT1MYE? zA&_=RllzpLV?bYrcp;tTWD%}~wD?V$OKC_#TI+dqUPl`Ik}x(V`aA{Z9cu}U?{9tU zTm6xc##)|_<6c?`tXB8(w`zQ#k8Tv@*(ZnZaOyYhX|t!MciH`2SKEMg`lK}mSIe(R zWvUb!Q+kyhn`pM%>rbn%N?8A7rG0VmW;@ii z%Y9sAg~kt^VODj8E|931wP$DAg;F1I%C+O7(Ln!Y_aw;sipq=~*xP0GLiL_4sS}5C zla=%f-7i{*BI#i05J`28P~Snd!Z6HO2ptP3Zp!bdZH+m|1357Mz@t2euGwbx>8@L? zU;6YO`DZB=N*Z0bXh``}r@tVY4P*?$6)GD1*|^@Xc2X(&jB%m%L0M;o!1s12eRx`c z!c3!lX-wDHC&ylGcW?i$=;iwCy7n&L33`=wJI&=Lg?68?TPsdztVPF>ZZ4F5sgSo7 z8aVk)Kiwz3>HppHT5pr4b~$g#v-B|9%55#De23hTghS<=aKd2)Y*UCNO8nC{F`(nzQup@N3L- zT(_0ZrH7ny5hQu%LXCE)p0hpLDKWB8?&$HV;;Wsi1%x{F&a*dGh%37Kh{{@|vdZsB zbCLTo!l%vGyS1yZuhw;LDBeIt56_m_sQ$W!YMid7>zH<3h{C5>2!D2{c8yCOGunAx zsXWW)OO1Bd&6XD1WS8h2#WS4+$*oXQE*~JGPh-~}-y*t~w+qQ8mml2GpJ*;{P@|Wq z$@{m|EMEUQ@xwoMe3$Dv3DFbf)JEo;=B%VkzHF2Q71?PKUsrR?O0Ju?hdX!M4J}>n zpXafG?au#S(yY7{>!U+IA1?Bpgf~~7bv=|kG7Fj3#|(NNUqX+GPrq}p)*hJHZ%7(=MAINg2DUun{lw2qDr1L=)X{N3+8qTInn zWN1cZ)6C7wB?wh>EdKzln5XUw$jOOs)VHldjP{xEx+7%Nu z6suj0L-tH&r;JY81}9@92Dwa=w(UuqI<1M?lqPy?Q@pyO%Z-w3X+1Qx&yI~Z=Q#|e zJtZvgh>E_-#8{gOm6Mf5d*hVFsM8j;e5iyb>=zxGo?Ey z$AuX3Q7U@|YOHrhtdtN)f$eC&SV-qee&o=PRW@(X9mjb*mR2&?lbmq-EN!Fc0;Haa9%~79H{AY z1MZl-E*(y_+Dy49k5nEvN|Il?ER_X?*sVsYKA&WI`MDVV^dm++y_0gwhORBA;>Oi% zY+7~5w!`5;8J6pGgV`DBDj+Kr-zThWRYaLh$+6*^_Z+de);?+X=Bj;wgyP)7tNGY8AMJap#-LM6sOn6lwC8(L~t|_!yQo zp$H2~v=b(RY$t!>*e)fgNv^0xiexA*5kDc0h1g~eSK*kr66x+Sm?y`z4O?^-$cM7B zembv4^qHR9?7EsR+g>uHEwtr!c%s!QSeSq$G>M_UX2hL6poOK%+KlaxgRh=st(S#1 z&SY+9y3!sk?$9Jz3oVlMoidPjXtK*B?8!`rw-fkUX%|!CfP@nhM?|wS!erIvog-WI zjuWt+3L!=*vt+=Am!rO1${S6AfbYJZ~KFGzOm8k6{;Q>C@ z$+#>Xd^^aaB{gQ(i;n5292K3Bp(9n47FW2wNw!<-g>Y(@s561@pSDx1t?_f3P%}|t z;fOZw_0Ca2MT?Bw$}9Ee_qm~S$_TXvhSy-I^nur!5Z=X+{8J{*i5s2|_d;Bv6`u2(XK}b0p5YoH z#N#??hLG=ay=;OzxbdHSdj50_&+(h*(gpcmDvyxnx*XXp=#As0-*)84kqaX2*|SGZ zf;CPSgr(zq!L9i86_&u#x+&DX+z*NRFc!A!OD64$a^O3)Q)5tS+#Vae!A{a>#V91=s^BDh@?Ca9 zj-)aEGJeg-sF@Uq0ABI*khD~dS5?xrPS3R1wPok*_8q4kOffuNF5fhw4%8UBy?jIl z;eP3UO?FsZr&EO*uvNf7q{%X?zob%GSxI~uibgl+bdv% zPSJco1?FQg#zg5c=yTAC^vJR1r;A_hMrg*#bD4U(zj&7nxzVB>Iz4&^V@av=rqKVV zY*0>9p%p1l%s7RhFU)Juj9RDJ!OCtohC*BTz1|LAJSoy*wbSs(kc)wy@TC03@DTno z+^8urxp80eg4g`<{_S?(x$9k@gg+fR41&|6>3#5+L;Hg6wIDzUhH>Uf&|A2F89f7S z%RhJi4`0+dSa#m)lX7h@8Ft{!LYFwUv6JiF<1*UTOQ5uoNjW)zHiX40zOCWLa{IFM zeijwr>At)9L6-?e-p(C$nuEw;V|$15>Z6n6fc$fkuZdDFCrS<+KLXmof%S9eE9JXH zWl?(v7*a2zZ*Q?=p)#Qpmki0EtLDxeg?2*TWG{BTgl~xT{h225Q7axaDP=eY64!ZG zSG~tIwcUhtI`m5PcovqR0mR7sKN}ws4c<`)SWt(zih}0xiCw-(a4s$Sq6vXaMYRYZ zfc7_vMO*=Z1-dx0tCJxe-TG`(gFQRe=6xSdRG}vv)m#p}i}Fva+$?a6i0%&Ekh;xD z$DG&a3~df=(!YPc36+s=V_t1JP5J#>$j9*<#Q$9p5?Dd^3iGlsJ|l~8Ey(5Y;lpegTuYdh2lNU#3OUt5_ecQHe_TYmL8rv^N9?Hcv4CcfSaflz@ z6RvY_t!48BSNUJzdsg|#1<_PJ7ZZGtx|j#AfM+HqChYj}<96!QDLZ@itbO>yANGYk zwk5~(BUeBJh>I9YAeKNZfmi|?UIHtjg_c8TFP%hDIL3GWo_OL3+qZ9@u_gAdyY6zz z#SeVo0~@~lTP=gJ?zPowG%impfmi~u1Y!wnH3_Vwdlee9Lz$Pm7Vd@PPyh5!JMCL@ zbMu;a*saN^ea}7j*n8gd9)E9e3>Px7((yB&`HUO2J32b-BOm#QfrzN9t23N{oI7{! zg2zB|^B0FpCxJvt-gHZ}MMkT0*`7Y@dH1z8xg%PAXQjRQ@fj2Ef86|fkE?(9hkxidm+mro;fd%BRG|P7 zmX)XI^O-uW#pe_S4z$375qjDtk-wH?pfL1)~Xl0!cpQS7iAoD2=aYA`Q>5vc# z^)u&gu{||H$qG`zw&QzGUGI)>lA>6^xs*X`p)eu`J)gzhXvUjzA->xq|R;V>#u3OMm~mdjHb2l8&yBVEHb^|w zQvgTgvc3j77UZ)2DOnGoC6G7e<;XqC2{MA9=Hl0)>q5g&KIIQ;2++LHJ0Ld>pw<&0 z*^rywbA4@>qiFxvHHYlh@{^t(|KSLfZMy_V#6ydrz^($?>6ErePl{G$wyD_ZT*gO5 zRl<${@}{`8wUb`I!Uv2WvepADFdv-*N;$v0r5Gbqb^C~OlLnfeW=Gd03wD}IRYKtI9 zGeQjOl9zc#=qxBk*y4*ki$VQxkE61K6C9j1{ow3Qd#C} z^hdsv+O|r5|NYDz-nJ&ReVE-7+p9*5^U1jqYit1sAMy%b_;qC$3^AZ(ZP1r@}@~g@6?LMgzt&8&t%jODBu_^ ztF6uK?ttQszzzzKMaY)AM?NkiT?au3BC4WhT27MG2K5af1aC{C@$hygS)_$(|M=-Q z*n4(9><+5Bw8MhG6M8RO!ubOo1QLtyp?|E>KC8Y3iXu>hGZ~?;l?L^`O)`Pj=Z@+V z?%WJy(Sw8gB6OQj`k>W8D`u6ifS{z_=?i>4nsV@`d=WyqJg~T zqmTIw+L!=ZIsb^q3FqM$#d_g?y~5)cqLr768xZ28UwBSRj%v;)8S()o3V94^alRB` z&F@eS;?cDeRLwv1Y)FUn1KQdn=l=`uA|7QTkWWub&#+yc5c0=)VSYkBf|E$%g=@rJ z*C4RQ3zA#=n|JQqY54mo4ENuEzyHLz9f$Rmz{TobMMG-AYPRfl>kth>SxJYD&I#G6 z?OdmZdpxqT==V+R|0>_A1I zJI@1ASt&SYso>JkNJw+DBeU)Hx!zmsyr`naC(5j)LFp==psztGm-Z=-Dl59gt!*dm zfA0Rik1aUG?p40gHJ}n`N>A7=)n~NRuiQQ@dbQ5+TG76h*<40?m&RxMQ38~2pf35H z6rRZ)Ax=oot5#VxM9}ZbZ=cZHLT!#|tPyX>yc{FXRH=ND6;7XLgjP0mU%cb7i%%U? zRw(aJ9lq7-uAj8sHG@t&H8fLhfAq{d-H9UO$GoUo*sVA(-RtvbZm@y&O8dFGN32#E zIV}p28SVOEoMlWY*LX6o@r7|%pmx1OL<<@x8NVyF(+Qm$Dhj9@=BlL&!2_W{8(_>~ z_Yi30z5$_|wX@52^T~l9btIN7rK_Wu7jQOAxhOYd3PWvqr|v5r&N1gyZe+!n1A-gM zLF#vGrc~(QBG-NUMynjzo2}O_J@g|rDrj2f`2E7l$TH%*&`HnmAMj0y74mI)*y7`pQ^52>~;Cb)wsj<2y?Pk=@wgDo@K{dND zLHgqOa!m9c&-XRkOqueBQpH?F`2eZS`*5ZRE#X3^Ky>6^`e2gX5$v#2a@1R@dd70F zTWFV=fW$%ys}7}iEvT+Q4WAfn^miO+-eDs%WjzdMovjl)Mj(?6ya2 zk6o>K%jxL`N9MEht69`8P|sCYP20%18tn+vPFA7SL8{m4g}0>7*_?E*8%obu{{r8N zc9jVpU0ppbq_aSPnv*~ocvW?;e@7=;8?8a;Y<60J+`dZ4?yu5*6qv_IeG#IWHlw48 zsZAT&;h-Z?D04ng_w|BeyYAICVY6!Y=uQhbf|Ojcl0iaxE68pnci3%I3zXG{9={ zK-!hKkWzN!|EjWgzUG{r{Y=94{mn@!Um9RWM%|FT$epd{fT$H_@fyk^;A=!&qEJviKczT^A$U)|SBVM|zR#h7*MAF>~*dtOG* zaW`NeN{d@fpX+3)6hRsO*c!V4tpLh`8=HhA8>tgDQ<-z%c8^ut#DF;MG-)W&7JUqj zQzQC~F;rVV*)AqakMXIG- zTXh$9*pc4Wh`Ue}D9MUBD=nXm6lp0xZoriCiUAf11Qhx24)60Vl}x5o3fo)J?-Plu zt9qPY_q6CbzB9PjCnr#2pv={X<8YvtZG6h77TO3tQOJ)@7;f3(ijKwHgh|r8Xc^e@ z$pi!?O4AXY5C>&Y6AbY@`XywiEt_mb-ruH0J<09yZgDMZ@`Rk>xrCp20ZX1HsTjZ$ zOirbI&rX)u32nvv`^h(0i|9&F@+P#!Z*sWI24n?os+_P%(M61Ff{A6_4Zn(mk(D^} zO-cJ!|9(l=OC$^!r5m zaAvzbGa2#q^@#5oLn(?a23!>G5%GQA(As5#T3ErWIuq?-@$Rv$mUwKT#3GlIl0Q75 z#TMm{3Cu)F8h*YU&TUyU>2=S2eN*n;Q!5vA|kjCwZ>9(oZB*FHhjdvj`|ITkk*T&~DXKjAtpmce+= z%P8Z6zFXQGvSd+)Nh9SRq>O4WruILsHsDk)B&N7Un#jRL54{as_xD=xh5LBG|{L@O*>EGkT@y_H4&K<4FWv)@PlHMrfzqL_Z&m; zO;?}wZMAH*6%mW`lmFYoYpk|(+P}Xpwd)U8A z9kE|NaNfQ)AfDN&3L6#;3H3AARqE|)UK5!nO`3)#%I)uuy}`HQ4op{A?@+Z>l+U<< zAMVs~Ep+9ym6p2a`92v`Fq-=$R(Y5f*XVRnx%DRN>^m$)6=b4a}&kL1y;kD@33vt5l z!n}odgffS`u9Q!nkX^US&i2PYuqI9$mRpdY}{ zvQRs3udO&@b$ci5k+Ho_omL@(=S}-M?XC4sitl&Yy5>dQFnFCM_jlXr-UjcVOR8qI zSg-yD1GoCploDjT##r~&lUG}%jD-W;8r#&*RJX%3fbCW^P@-AGxR;XwuQ#dj_PnfD z>OWKBL}rX;p|n`v2?pHW$tvgM{_a4PeK)z!^(q$qYSfQI*MLzE+KM491W&Q(t#EKh zkN1&XGxg39tU=EyKa?w-a$_elfQ*j(G3rFta`UpUb78b27j zztMM{E57JaPh&-u7jmf8Bqh=(B$;1{eO=n5Da zT?R7nGsa{@K@t{F8J9Bhqe7dg5%mRz^J}l`_V|Dc`nJhu2fBoKniu3l1nzXFCsabT z!#m5Lx2I=z+Bb#{dfnH{f6HWpsN=+;jsAp;&>!E>e%wap%I%5LZ8oLF;u-nl06;JW zT?SMyjF-{@X8F^-;V}Ly4%hML^3c#8U!Zqd5sj;SE4kXN{XldyEb5dy`CYpo za~hm)4av~mDr36jcc&;}_NBY+@#)=e-0qZr4E%!3YHqw2bpJYIh4dpF>qt8B-(r-70QzDffUXFxFx`FA@z$r$8bs804ha zIjs%qM*X}!c7BKJT)lPj*8$z3bk=0Im}w>O*? zCDEAEsX^Dl_t7PJLr-VJqK}j=I4@d2Xw$AMJ!{ure_r%y!=eBg*I2h;C+D=fbI_a) z=j@<-&}^5&rLxSb9YCcB*k@ul02NX4^w7vuM%Cp^Q`RYl4wiP>?yLLl(6|6T;|+FV zp~+P>>SL}ZTF9p@9~o@6zn=SXZx>zCsR38KrQ5MRQ5R)=Kkt_&sVX^gU*F<`}xUc&CjaThW(c3 zpGAjac*ept&As%97Ijp2<*HlV5`&WKU{e3QZAVYX|6dUjc$vBveAzF!i^F6)p2yrnTAW#w~&JlBpyJEN)nZMLCtG!P=@`*)a zwe`N*w3ilGvP($baQ|di;S6=@PcY_rRS%EUMvvtUk4ad91Z!JeKxng}g3gioe5WLW=-eXwCXbw7jW~5ZQ zB8qU0okOrt#bQ{Fyk>WbQL}cc)OUr81&PP8i4~TP7DfcMRC>ib4D8 z;6X#-#URQ=#(C4pJQFy@WkQmypRt7^)savqAe@wC4@}N6;OXE>BPbApmr)mUlwZN) ztE`3H9Y~G0wm##=Y8)N@b=QyE^9k{xD<8Av5h7&1R}P}|P7qzk#p?}H=fQJ(eNs57 za{Jb}AHgqTi%*pPC~L=oi4f+kQ*a$Lxi zm9ms+u|VfVYNKhj3ED9-a3iGJIQ7j$#vNa24_ zzN+k$jX<=hFp@2|J~`n8S;WL=PzLE)@)(d%L{hR|7M;>Ai}dZg!NPybn5^%=O&qZ| zE8VI35_>SW&v%sQ)3b~w@EQeW-Bepu!1?3oScN?$4u5aE$}{`821~@psD%x8a4A{1 z0a^`mL6h91CbhmOfR;%|rSFDCkE|BR9zVI$j#Rcdiu?K<=e_OVP>Z_KSLA&K&Wl*E z69CTTz&DPu+P^Wq%KHP7m4(&60WWX#-RlWW3^m#3S%sAhy4+J$d-gCatHnme%Rk5c z5Qvv=8sXBSF*;l?H1-mS?kE_nd7eM@@Qa3Ro+HlEz;*w2@;6AAIb~s9m%ghojg^ei zZam6IBwwUQAT58V&I|KI9G-Ebw=%>InT$Tz3I;a??iW56(({|^{hzOTEGNzN&Pe>e zS5g9i0dnf581r$6c#Dvj>^OTYyUU$D?vNv69M3(K5s*Ng2o8G*YRWb}C_f7_{9!bs z-!=8)wQ|CDRYRZiFHa=WT3nUih{j2q=&Z8WZ96IbW!CP^TyJIK0)|_+O};kh!;a5O zhf<%#A~s_v4(6D$VX&^#Shk==Z=6o!Op-;=j26*H%F1j;;|cvP{U_sPS>=rFtRJ@5 z)StBHCfaQOt{&?W=W<4iu^A!Iq2>jZdfzkG`mQ&2MzN?{quo3hyD`SIdj{u|_;BHV zs}SFB-}aOpQF@&|4?QcLgB^(IL7=yx?u8$^ zRGiymyA+oWR0njF892qIKQZ5^5&Lf0Rqm|2U;ZLk@Vs@$IqQ=l5?!~cdcyW- z_aVCAF*#yBHzWh5#+;{5ZFkT?X~VRm%c;}19DGjv;b+7<{RSHzR($|M!0&>_7I(rZ z*EiK!-~aIN*tR@tL{yR?%_pa;67nLRhv1*|2Tb?z!Dmk}t}E#T(#+-iyhoTwE9 zW-%7kD;~|xqpKx>V*tC0Mp(iq_XNSaMBU;$!Ljy2w;B7g3ru2| z;rUH}Fz1GveyLW0B@|CjYvxfRs`&j&7Gtt)erGwK>AA>(&pX%AV(AA;1$i~?wVJ*S%vg~B!74W^V(J^bW%{x zixrH1TlD`^jS9!GF9!$n``lD)5!j2@PzQerD7w)nrgjwHl2pM1q)0> zEP8(Whx_^A)aX(6hnIRsm&>yjkjtDZ*o&gpLV!9eD3rOf1$B#qmrK*sB%TaKBMzD) z?qW!(`IMQNx$d+rm7k1!2Me9~`#>0QXJ|Z|(-dbBfpm#Lx%4O1y1j^spmtdHiaM!wl#E%wW!@g-xpTi=N+e*u=@G2(4Qio4!9m1F&MHKl{&`hS`~gXrOuUp~4>e-7e4=@n+^j(CKJrt=pS6=lMt73l0j*Bld~6LnL`TCKqKbYEJs z$ezBinq4YG=Y7)X7SUE%>wowrn{(O_Vg$tb@hN^%#k&C7l;fwA`T=FW__vFD}eR1ux^k5Dk3DGBzG$ACC)^8D#R>g z1V3qvd#a>g-!Jb<6q{^AtS%Qk2( zfKMKY*@QLo9II7!7$JY&8i=09x*@R~&e%iV~(_k_pj0t_F8_g}i!qfQ zxn{AiR9Z^ zI&@wt0O#x4h8=7sai!H(Tr_ed!@2PJ_?0>%#|(cmW2IHxBNve(%-%dy#YCL3hcDKK z77;zhQQ}uJt_yK;--si;uvdO8_c_s0DhX}?SXGv%wb-0_pRbgis_2h4DjAkd#`!a8 z^bqKqCogs`zVHLEbqr`Y8}hsSrX2FUR*d;d@yQWIzR@V3A{L?QunEs5;KXT+z9^-B zkzc%bI~9|qFk~&0qZn$1uS&9O$jK$jchqjw#-qRgA1#1~`+%SVXl}dcK+BC)k8y>R za^zChC6XMW{V)Pg&aP8vQ$)yMZ;};Q@kR(T98+W^<^2OG@9g<+D?Od94F+}{%YMww zBShz0NAOn)`SV`Gm4q9xtwtuRMPL5TFF}=dDe(@`B1-M!UHfn*!GqM6m`s1ScvsX% zjl1&R3#XyGlWj>9&j=MK$UoXfJ8Tia7}Ws(?jvaB<`EwY z7)q0RHvlB+Su=bHU)ygWwR5x4atV6oJJ=SdIyX2`0P;z~BKxSQ2+2^dh@)97 z&7zhrmQ>aln+m~MEMuXUL&`f5pT}2Q=TUV{w&**?AAWy3UJwc@1U;#%?D`cG9u{fT zQG%A1oF7b$Gmn|nVnqZ4AZ>A)C(C-G(UJq~9v8@UfQ_e#HCyh(6H|;>rk_rIJ=Aqh zj>JU^I>Ig;MF`H78nXzi8y`+tB7k-@jwO%(QAQhDDN$C?PoG6W?L~rORN~Rt- zF3z#ei1zSrCE6OFA>!q_F`ft^r{hsZ;zCO+x5As4e_1jj4Kj(zJ2UYW0`70GYOYXr zuXNj{FZ*5EOUc0OIi$fpjy2X--nzFgH=dBu^6;0De&X*;^0}3ycBUdc0s2KESbE{; zsE=Gd1%1NswH}Q6%4ma3VI%5W-5s;#p~1dhcO0D`ZePu%p;c0&Hp9i_Y<9cle#vM0 zw=pi5=<$S3cf{v%K`3&gI^aL)m4`*Gu7GMRs_NdJVFjBk0q)}75Q$j|xsDtd9H)at z5o;L@!z`gKlfNdM>+rtL$<$Q1RNql3_P3ymj$%0oo1N7}&>}G&&(rR>4xq%oSuUuY zNS?dUVigN@9}}R2cRqGRFy8xWb(W|kVFO0zHFV7QzapX_pKENTHL*s^VzJ{bZ{Q-D z%Z7k2upMCUFnk_G&ToWEgTXy^BY10!+4T-BfGNd@Q`yc?6)2?72^(L)sRfmZc`tr7 zK*pKYgC^&2Qp|lPE1YEtreuVq2h~Q)tA!A@8capRY}(HWNo5__3KNPcZf5z9`gC_L zE!;+1Q$XjhY%Za>tIuuEnnrNg5Ub37F^l64k@{Ck#;wklL8??m?tgWf!+WQvKl`2H z4DPJ!SW|x8+I|e_TtDe^sa5DPjv2fBE0TMs46Mjd&Ou)knYYPu_#$(kGJGq*$j?Ioi76c`ORhzN+DY0&G% zv_tO(c0@3 z@E6~QmyH1Zt|}> zTQ*hI_6t0;VFdAhZ(UrcLq-OhgU2hVV6WtpswF^`M9pOvY1v>?}mwUD)uXAs{w7^%yxjxEre;;4M zivr%JC)a|ig_|9r!Ka~DkC#*YgqpCl+5E2pf!z-%B|aE1N{<7zst+ww_h5V3A_oF@ zH!W!q$JPoe*d0E-ci!Hb=hCb3vJ9&s>ryp0viSL_?S-Y&WX;N(uX*x*h~q?>q~e*q zmX~J^9r$VC%!-QAi-}k0VF{&8FfD@b0L3Yf|4v?#SLHZ$2G1L8a~=SKTGD1We|1i- zDVy)aP?okzT?5e+UX}{qBe^AVY;Y<9FaA#KLLvKz2KNLEZe()9l{*{$$I<5;k`?~-({ai@4hMTTS`1Rr^ zia^V_Is2IOIKpq+h#acT>!3|SH_0=+BccTf0gWg)o1qiBAd`BeRNR_XiQnjQ+nk~6 zQ7voH5#$7g^B~=7d>a-^mh@k|FKY4@z%H(V9rK`4Qv7|NDkg&(9U~B%t3I3KScxRU2n*1+a1BAV}c%pyBkOp@>Ag zNr8Ev)H7%bpB*p7XP>@8Ob5mpHzIkg=!Va&XfuYRX@%YCZ{Xi;(Lv1#-XpUsQ<0H) zgs%(qlzL*yxNJ;2nbn*IbHAj~$XhUB>#3Kg4hA{GM z1Vn=K61x)WKcjWHZ-^n%$|hOq30ZluN^i^s;Aa<97#(L;MSKCJmJR_&^HzSHd3)Z> zuys$RMr__dC^-O;gdEZ+eO>UM{=QYKS&w4@45p{`^!571yLc&^|mU2n7NJsQF}$Z>3sXS`4#dIc^f5Y0KH;@&=$j_IU6QD(tA0l1L2dY2MG> z*gr$f>1-`l;}nnQ&ZpH^bO2QCVys;4-F+Ql*0!l_^^cL$bt)Jr{hsn2haUNmum(um z$Qe<@ME3PHV-nn+i|$1ZXSV*?*}zKo7#~hmY5|NZX#&0RF9t}i(SeV69YxdARBLjf z8)-oILYoXGuUAFugx7^Zw_G82!Y`?&iXprmWIg!J_w)pDgC>D4l9k7b3vhL5YNZSy z+^5{dhx>V7({CLKMkGw=meUJr!UuB1&lc8$$p8wIw*J*?o?Ijo zQ^}SqZpN}0-w9SbvB`zA;6oXFNd}{-k$AVi!7`V{fA_P=r^Byc=lh9cG8 zpWnDo-17rCv&f|{F#0{8CfSiXlmevqBJ;1_cEz*l*)2`VmO3(fHMF|7{iK2{`1Wr* z?%b)N9Q)a_m?qEW+<){8HF=N_i*%{<5ZvFw0~@ zix~2EQZwr;r(#J{dB>Ohbx$q18_bwEmHc-ld~=X%%zLI!>hVf7GaK{L{%WExO`XiQ zNVDl1?ksajukYi$n-4k<;%z;$355%`SEt%I6J(Mp{ep%y0UuNLHC#3?`J2%R)G?%2 zv6@TdW{tensk`>DHMPX{XbSRN^kP&b+wLM`$_JW`>|sYPjgKjLD{PQ0(H%;+Pd3u? z<mANLvo;^`ns{**6lP1kdZ9L|2=N} ze5>zg`p5WtWQPI2KXZj>fYAD7@abn?yzEd%8?ieg1LB?~UyimP1EH+Mmn>Bsf0O#a z2{fEJg^3hqc`jeMPWYn;j}^-iP7n&l;So!sDn~8UV@S1Z#04p56t1=k&OsiC_mf7T zd~Obp)F{yILkh~A?l(5JPn#O<0#JBN7UnQy~bo`Czu8kFh@?Ge=CF zI!GLp_-p(?_xN@adJ4R;4HtMZfNEXY(_70%A0f=Kw#{|iOfJgi*0U<5)vk8Sv~)9_ zQIMW?3(HSHegV9vcMVz3QAb-VTH6dIaDuVH_L}>WN8G&nmwpDrJANrBP^i3l(5BZL z`pT7~5T4G~x?V-cpd|ls3E%PRiXUm)Nv`@{kYKUF=xj6w^&*XH`=0WcIWOMs>b@gl zA6a*H{t<>~=bPtz@oZ^dPEl=JNsdXU5*gxatdDV9qfdbhA|zx3W+kI@li&g6MlH2!_HNx1)QpJ7=Xbp#2Ws-3s{;Em1ORvOUD!u=YMsmNY|bzgwKx1xjHyGQ_=4Zenvx<31!Dr8=W zuxb!y|MhS3>}D~4R7yx2ow;TN4#Oy3~Z#MkxIwXaU}_fpT23|$F?*FT0&yxzmqpbeWD`K5=tf^ zo)Y~e{fRY_p+PvOoaPIx2qXN#XGlny(2;~r;Cc&>BtsBpSixSqZDn)}Vmax8q{4yB zFLibBfCzb1ORBb-)dbn3_+^n&GsrFyYCt40hWdRRuA#WCodQqY+y!|b^cZv5(gzgj zt(X$3feU74{e%_^CDH=k>H4WYm7gGTv|WVY^2Nqhf2wXf>78(lB1zZLCEG@NeERub zaoZukVR6_})Jchr6#DH`6Ly}_6BI&?JBjF<5RCA>&~v0|`FBc(x$9qDAPicou1XnJ zH6I{47t5S^DJ_0=iTOC%EEjc%TZziaU5P>^MZIxA|oWuqY8L7XeXI(bxwV&RD z<8ULB+VEv1(AEkI7cQlsm=u{n(K9CrJDeO4nW}b)ds&SsKI5vup0MPebhws`K^C`~ z_hF&>3yBV$3>be)_+Z+x(L1R{njm!lWBR3j0^;udyp4cxXQ83y@2PC*i%viSCCQ_3 zLI6?j?Cm_s`Hkxk$}etiNA@_rh4{@&;bmtZ-n=H7&^Jh#QM8E;x!1s_olep=`ncS{ zRs%vuW4E^=>Ty+~U0!1;Ne9GJro&(|u=%Rr8chXkjzT221a=s~EG{XWVU{dMck`T;YyZcmtRH zH*qNbq?jf6HPy>#xyW9ygL?87gK}@rVv~spa4lOwTi?&Lm8(J&oLD(PIm7a{+m>i z{*8rJ&%+B4?qywh_0UbV^EEpjH4?`v_a^1~+&yMpYVU&{<6N#_pTzr^&vq5p^%9+B zUnjmHQMJ!{#H;0^agU2j8a|>Z{Cb|Yn#i5AkW+Lo4K~KQDiHx^gDqS-EW!X{W=jka zT*|p+7*(?hbzZ&UeKl`VkK`l`%bf2U)pcaYZAZQPr-Z{`PvBEJb#*h z&Hu2muFpHOY13SQRH!@szZ?ygeKV7r*PTvEIK;Vfa|t^Mpv!nc%= zca9(*nOeh&G;zW7E+*JgTuvS?$xWy_$e4;q%FR+*=y1+Tfla-VU(QOG$a10P?=H41 zs?7VmCJ-fak;unAKEZ86mUrT3jOJW9YlKKhz1Im>&Q}fak!*0<{pw)j}p{N&6q$OHwFH8VR-$q7j+l}~r^bPLk*Jc%}je2cZr*TvZ_|C)aE zReYh%l&uEzwWGao^p*!D$w%vizs9>v6X0?>>V4?zL#IW9m<08`N9cAlWrcen3BKAlR!& zVFvV*#M`fk!}@G4KaFV@7klKG-7Fyl983%B5al>08U0R23?L0Nk-lr|2t~+Dlkaq1 z^vtTOco!8kpk9taPR79G?^Ik5sJf9P0!f$`i_u#?1F|xZJ*80@T0YfwKJ`{J1Vk5k zTfuAU;Sv%q?l%uhNJvz*v@F%MAAa?|Xg^Yayi)(I7Jk^hZf5))-HV4*RD23;;En`b zO~lGsE3D*6V{g-ZkqZE3nRmNe*{tF$?pR$_gKA3&^0Nu?BcJldYy>WGlp=yze_lGt z!=@yzelR}uQ-~1vuydHATh}j}hf%Gh9wx?V*0+o$k)Ryg?$Cu;#92b;V;OYPbDSpf z!f$ELZY9tprG*gSv7wWoPX{W+rkw*JljCn#6OLB#WtLZ2X!@wbJi3Uc!_gu3s=w2< z+T8G6Vk?@e>+dM%u}B#|Aq*2EdEDw*DFaWLwCwozA;|Hjjb20{76mP3MRH7uZ+v*V z`8T02eQK)L?n49nIb*rw9JXuOTdms z{t}TQ=@srfYQ;akiD=qdW@8le+lQL2Lq;J7(O)VFz{>^0#;|J&y8aQL)w=R)Lu5aS zG%a+J?scUNDX-T~ys+Jf$gNa|#uM1c>|#t%ypiMf$Qh&cGzpa4_{J_Q6Mh65{Npyk zK0aH!zvHT)ABTLILK@dK5BnX?b%6J?9sC;#4Oky_G|VF2Tc6|Qf#?2O_=w;)*$#`* zz4HwheIYk6>an6LY1Kqxw(vXi0`@2?uK12l)Oh*Aokx8Om&ADYW{{&M(4mP*yAMG^ z{?wi$pQHAUMl??FzyOJrg4t1s^pTaCF;NC-Y;RxO4nO@yVgC%M0!*~ol%NjVirkH3 zyhNi5J(Q1_mJk!UzX3$OhMy0$Lu@AM}bfoM=_P&X7J3aHjU)DDrj&BCre%h^d zU6LxbYBBqpFOqSA^@vVh&2nHBEY3{^cN5%I*18b)&8V2>?3}SxVJM_^wwEoLj?|^o^x|%F zs6`_4@oZOPkrP5q>aOHU%w;ewvTUFmhiw!+)uZo|?vt)j#g4#8oIa@@yk2U?1X=Ba z)3g&q5ynL#_DNh=oPC={TJ28;g49E{eaih>W_g3e`5?YqW6~mF0TKA+l*5nPbSn8r ze=%mj)zj!DA(}cM9sA@f>+vj=$BcU-Ml`$&I{m1*nb;kRTqs492{ahR;-^yP)c5hL zqPO_#JWG+SwK1nN^bzNHB>!m*bA}DR16ZpMHGg$Ybzkbp!=m44seXb~qPP;fhF3#M z0vjdQq9zicA75oN%uE+5sTLn|Ys0BUkDJgBZULV7_xD4hvrl{9;gt}>-SG=c@q6!b z&`%t!W`8hWxM$nc)iffjYg1{nDVrFj4Lm{^DfG_vnScAPU_;8i!4%P`f+uaJY9fqr z`YFokn15i|FK$~lO)`40({-*4=^4i`OF!zl4NgOhc0z#Xzp*R)&8>eV&tvqCmf z+HquuWQH^Q9DdRf%wqm??6&77mFM5zP8-La(|3eN=_Vm&r4 zsy;5mMNuKnSmT+e0(9L;jw*j~;-f|d>?BXohn*Caq)nPQ+&X{fI~4w|){O#13(8RA9qLv1rfkku;m|W=b;960xeIrRktcK?RV6#9 zxbg1Ncz?R)?9V=`d+E4ZSQ+QA=z48fkJMA(f)Nh$8r_Z(DI)EH+Wcc#@Drqwtp+|# zh=Qoq#74wi@|Ejq8OCIEc&?b=_s`uX3a(=`W^su4*@daszT(r3X$QWLpgCf@=sQbq zrHO-x!iOlo@cJ}R2K6dCobrt}CjLC_hRKH7M-jaGiqMpobsP@(11R0F^VOC2+YEwp ze~9&oM)IjFG2TQuIgEJU)Lu+uiBj26?}JyVb8b!G z2A%@FN(}drmqM=Aenh_MxMYm-?=oPNl7NQ=)ZVh~LmDvDX@3}T2iX=-6}+c*s(ZXA z^D$)&9z?bup6_QaX5@B?5I=SHKJz|zZ$0xrKeYDR)%x96rY1uM@zJeNS5R|aX?Ge| zDUM@tjS--5U>xv+8*dQ6+C9 zBlK2v;mA*2zs{5~i|7%y%Q7YhPn}+TzK;bun?A_zibwrsr0Wjd zgtWjn;=B9o%vb1)l$nOlH;5aKjyVf6L`hHP zvX~;3$QO@u+A8H_eluQhHV(Nn=_g$;DJtUE9y#v0>NiuBFmT z+NS@~Q)69P9#Z)ZL>RT2gVK;{d+V|YM(c(;Bff{Xj`BLv5ruffAx-u1lY+WK!a6%~ z%H|^@zZp{eh6XktfHqP;0lVVqZZPN6_8Lp3Ri=TS1Q6nNv+%RH5^_=3_O-0$Q}~F% zzw`fK`P~e2Ry?hb&`>xwuznBr3R1)vDIG;xJ+h2tygW`9!ApbH!5dx}yFocle{!V2Z4Ye7}2VGU+JEKVp!=HpUeYSs%L+fA`oESpA zq?aqa0>~VNTn~U`%#a@k;=hZo|3rFJ=Lz*`nenM0)f=}67qLNR;Mo-K zumO-iU|jpyH>c+zKr_hXOWmK8QQuF@&o>R~=&G7)V82&5G6cEP@QOhekC~ zEZ+=MCV2go?bsFcmZHS*$2Hk33FnGYZ-L=b$LzZ#idoR@=JZr5!m$ulNg1E~>db(7 zWn?wxUCS`GebEr1yuqXgieRn#E6LsgE_akTF1KfDtc@2eipktJhkdkeN|@U3tn>F( z!}zD=xC?c68pO3TaLr_45DAYd52H_XT5Lque9Jyjv(8tsR*aoh1CC!V2~K8fY|*F2sGpiqvj$rZX}zN9718P_xAmA%;4E#iuM zIe$$fxUXj2wq>eh!Cv4!5Izd(%yg`^RT_Risq(87rbX#&CEz0g6)iIA!o|IKguc|X zF^x{U$Cvx`q!m8p+2N?Z4*~lAWpg1S;X_ZmDZI`*^d$2HhNLNme);M1F?`z;=Y633;WS z^>zqU@;z}@bV+#S%_(ghL@Yc|=s*HP9|TKT3)nz>QpZQMg9Jl?+s8889WeIo{zZf%qjtmgu;ZZ+i-ztgU$rOxjd>W9?@HD9*}?oHW*D6$B>snk z3spasj z_fGc?ZcUg+=FX@Sa002X?TR(A-Iz%5LmAbK74JjYA=A~(O`qi;2J!3v2qt_KW{aS= zU0rO}LjUSzmCKR@Xg+p8i$o>bDK|E#JON*LTPE zsC+uCqM2>Hs64u%Zp>w~tf`n78+=evi;dO2uY#-zNnJCK&N91Ydju=pd%smag6Uw6 zi55R`O8f+j>eq<_y=8soz)mZ^A~#U%#KDnD7klmnj2YDij<>;1x6+m_U6&utO-tV# zd=m^ly$(Jrub!UX!-aeA+AVerX<-PWpGQ3E@sg*q3)dq_UcFUVg9}~iD9>EB z380ttn~WYFJ1t|OeaYuAILYt~=X!N%ID%)n6kizPSXVwt%}dg9M!rnr(Jn@$?pM=K zF(`t08I*-TK6uA6b_`HhRi9MIs%pZ%qk}96_3~fh4Q{^r1+hAc1v9GMSyyImPMrF; zO&4T3>_ATzbOk0HycSK6eLlvXMmK_~{ghL7ldA-x~FvR%k)Bd*XPm54+%PqMTl!5 zm*Q_L5%j^@#FtSwrCem|8g}F2sLempgj@8+SQ_ne0Mp(i}0RT?bK7>WpUqz8dgz z5z&E*f>$#YI6^KP{FyYO3~qC0t2-v&9wfho)ogV7>r(YbiJ%9e2mf>-cT!MtRdmf_ zt;rvQ7ODX;Zs)i&nLuKZNy)w#3Y>8{Jwks|a4AsEimf=V)k3XZK#1ItA6G8B78k5t zJX}EU)evrF1s}dwyo~6NGmLOlBuz5;agjXYym@En#@==EKI!g1>h~P=ev)t00y^{k zcoOideem36#Oc2!K#;v{xbs!6nt1nD=W*gVwo83?y~bbCf~;KZmhleRb=8ycO_=$e zM|qODDn7)B<^u0#7+??W3V0F=RxjQvs+EhSPJlGovn_slT|Amm6cz8Ue{rd|qzxv) zwhMW*>Cjywyq9KcKFz&@$b*_;`nJ+n;9e}b&nxUpYH~A(a$g$s6)x$+uRB;RTf;z2 zaSO45O~i@~cJ|#&*zhxi!n+g?CEx#~9;L)9eP@$e^^!+r7s4-tai;@zu77H$@^9~!@L)~q;#y@2R*nqBkG zyLq895iH2r4zS;w#gMc4{Fh8K-b#?oO= zSVU4)7>6gNziiTSn(&$IW-0Y_(N)(6^N2I*N}^Kg-)jwPzgk5>J5D|IYxZ-gv+~rs z4L+&Rf3By=+|;Kel+Ic_5Lb6Io-uD-Za+tdOv(3F;AH2yAzYw6`888}tRkSMq0GG) z3NE-~YyL_X5+|7pyAz|oV<%6&S4wDlI zJIG{*kI5hIa7au!4rTX+Ft{Ff&|~qoP`w0YfOTdkD5|(x7D_5;F{|eQ^P*cx)iBRVr=`h$>72P;dyJnIq)c{6Iv6*aRyWc z5~a(Zqwnfo9-@B6kDF@3?-9SF+AZ+U_qNCg=1HL!3hRt1;Mm`a-T%(?74;_CM@YAk zZt`CFgw;pU2PYj?Px5lkxhsrAtOMa>typyRI8)X8fQR6CGqntfBrV^w+}m%nOtNRz zdJ&IDy8e(=e7A{Fqg}{>O)%fCIPA#wgOn+-0J|C>gfEw)2Is)++( z()S{3+IHus+L`uJfPIC=2e=|r)o@fhsqEc{CRDKe_>YIC?_`iCaU;bbT`+G%_shG@ z?@pi3I!C8i6o{B%TvZB?A0GE^$1|j{>uYP?eWF+E7|iy>;QdR3pG-fl6u27Vx& z>Dryu>WWfdZ)8?@WkGxe{hd9|Nx<6;)|BWA*fsu-#C*5dxZOcHl2%AM^5eis~Hr|^$}S%yJ`CM?5t|H zpY62Wy*c&%metG%JGQ>_5C4Jd(0oZkbOUdu_EmTZr;Ba0^r{>5NI5Yvntb+_jW7ze*5sW7rWN|SW_hr1NC^w2)t=0PK44Hac!2`Xfa$lJt!R4 z^jICYbE!;qsBE3B*6*R)!PlM6&u?mSk$to{qnU^X*JwT_khf|+0?HpC{n=%8#9t(s&^lP9q2rkas!|}P<3Yr@{=lRp zY1*JE@+b|g_hyT*m_PJvhP{{7`ZHmx6oyETlBxiyg9j0r)Iux5+w`kiUtqj_aqU>0 zX-nOqP^a5Rqn{HOJpY6%YuHWkIPFHDC2EL|J) zphdmF&H(XS-RpzHdrR;6@`v_+wMy5lEMavu4b_ZymBqI2i_w+tN3GQ_y!HEjEmm{0 zW=qMO_WJ=#<@0`_`u&L{188QR!@p%7d$2ltC@=dc&@CgOgGV+cS-YWto+skWQ!B`A zhgUR5lt`p!;PRde`_O`aQ{xF`)3+XWM?*wC#M5Bf*|ceH%j?i2va7PNaKOIz^?^Y- zqqX-HczU6KJR{Lzx$Rc<2b>?AouA)1T!7DNA8!KgtZi%>IlE8fBk%A2A^ML;J7Z(x zYk)bo5g)v()P~)n!_USA$*k#WI*6X;k3&UpkAJ3bo~hM}R87Cs%~5NQcc!nihmW*n z?33iFjuAegAn-?QKVFikp2_bz@*yA9ceLlBxpsB>F)bjMrSowJRr-p z2;sP4SXEE4WWMF+tr`b-WS--@u2JVZsf~Cnp_+@M<^)<1enYA%f`px<1$p%LRs3!8 z=0G=Bl|1BV12Z&SqU>n_^SA`>`{lYt7hyG}fn3Yw6G+hDI$l!X{@F&sd1$ca7rtCs z&xb0D7Dr}ttj|9AgQWkRWM-ex*-q< z%xvh&C*Uf~$j)vimyB5%9Sy%N*>aj-SelzNW@l&T;Nj_@nUE}bKQMS3@&hjW*-a#0 z7T-mZ&AKs!x115bUlRj;FTHb>M>1*FUFPK<1~C*pQmp)C?@K53dzO3O?*D20F{&fL z0xvG?-QY8yBJ@BF!Xx0l&*yIY@3v3B=i+BrOL|aUhSI59PVBScw9ys!>>rOWg$RoEyj3o zcz=ZO0)Ck!yfz$5GS=!c3b-o!8D>rG%%G9eNF=C1>(OkB1t7k_{Y# zEvcu~skf}}B4^aO?T0zQje^gO`erlxaqq?8d2^;Mrk8%El}XMmTjJMaNzr*-K9$XW zo}dd$woo>UsTKdXBY&@@R%7Buyfcca+ogn$O=Z^u{QXE!meHa)w?7{<%kl90qPvB;fb8echaF?sxX1 zqfnt7pIe<;pBxG8)?2*g)Y;a$2G{G<1Bc#%+I@9=jSJf|IM|`~g zju7TRJ|Yq;<`yxK>6`9PvmftL`-E}!ry{m-fBknca_c_&?;}KB$E43sBZ57`POjIh z@J(<8`NCxwt?&PWu{ik^q5HZKNs&@aj{h4d(tnojjv}&BDf$=6kjF!1FSzpI4-_iy zvF7O)Kt$(7$AsUbyZ^GlRRC(2NPE}Qa)L$FD)IBmh))c4*58|-rPz$OKg4vqq_dE9v5r!~%3Uha`JJfyLFB^X$x-li;4 z&1Lf~pGM?&K+)o>NpZ5@t<^+?b;#ZNPr|uD%WO_tDiv*qKP%LQtzHV8=Dd(yGi8&A zsXc=GbRXM+x@ur?yyzJ|j27%(|YCIs7B=#&-j9C}^D=bcQhUZ6;?LKg7ffmk65^XjW?auCO9OQYsWRaD7 zSm`3D3MkVrqEY6F90nS0562NAt$Pl(6gDAlfmA$cqP&Z945Few=Y9zOwZR1j6L5V=yx=-5Gd5~AB;0BjX3yQotK%^Id6opB zgepxb;0t!S_>IKXI=YF60ztF#AUtUfJw~)dTS{r<=w$bw_pjo9zu!|Q_GI~-RvjD` z_(B44ngV`WRT5&L={>8{sq5zaACT|=c)>s$Fp9r}&9Gifj}3c3kVMKV(CLUr{vCGO z@D525&kDmSV$5K6@frBE)dP^}QL!U;=r*6tlkef{HZ0O-EX4Ijkiii+KVL{X*dL7I zGyJRHD9oFiQraH>hUm0r=}BgQ&*G3OtiTnadvZCXzdSo@)Vk&O45R#ZlKVF1nYwjX z!m0O@{|)W`6M6lIRJyGqO~PSqKr5Yq5^CvDA5z2!*tF;7J~(|k@0#o_9jp+1PQx$F zPR#W9>EQiBd9(vC_UbUeZBv9FIbJpc5f8j%3~1jrFFl!BR!?Jo%)X$86f(lEBl^!K z<9tZP;O|KJr9LBi>Yq6Zs{@nY)TIzjIASJ<{GN>5&#?+#Khk9bGAioY>*JQg`$u{w zVjLOyKi2dAXbi+Nq=Oh?P@T7u8$iSak(7Fy~|ne9DbxX-5mzO z_sOjndGFLeUo04ltF8gnJ`T^3I zrB!7;OA)Qjt7DJ$tr+ghT4UQyX3wxU)OO0hw_nh-BZUmeOdny4xW^-fLeOIP>U8;P z4Eh5s;s-KaH1O5Omu04F*#%Cj?$i69K8A)A1E#aQcbL}qI9;3K-$XVUXVE0%>0Ybv zA_}^0RW4dvTJ*mFr{7Pssj59~U*jk6YU2*>Xu~>AzI`z}tJv>&HY6H~E2^+(ro9pDurt zDsZdKxi0HHIyK@5692DTW<}*+$YS{LHZ}Ut{Ix>Rh3*|pNg0X6K?$nI=T5ECJpHOl zXz{+S#)^_*_AsS=Z(=U5D5q7R18b*p(B5xyC%b&B5AJ_G*bQ#0#*~M`CCmQ@=*=|bw{xkx-=)836h!=dF7i{)@epr0~ zu6^mET^|wqe&h{Y`|fpp{n-2d`cSUuYkmFr52*dP z5mr`HGuzwS!+v@_i9EdZqyGoz0Tu$cc%?GhKFk8Y|MxKS6JfAJWBtp&6YinbJ8sD$p9WF0SifAukV&2Uk2;Gd(gbk~IF-e`<$`) zh6BdTREZ=vBs?bB&)Z4PHR7>#&T;@&n=b>y8a!Trup{AyL@~&kOT2vO=t&h}TrB`$m%m){D9yM;WC3Ec9YeXgc z(h_|GPb=-hk>TE_N=o>*UAjrF}c8q_iqWGw!NNw_*GF9lX;Lz->s6oEgN8ml7jD@3`UmEY#zce z?9(Wz}5<`hc(wO*RGuOOutaD*|O}qg--Uw2+1DjmE_uY zsfn-QP1Mtt6O0Uq(u1XwG2`vld?|6VUTI5)SZoZjdxC?xE^x|!)##}1Wo@J5y#|cq zur}^oGqf5rjz_v&Z`Ph)#<^-5lwIP^*ZFq zn7l3&gepnw_i5pJdc`z21PNE16RwI0YU-F-o#ilI&2V)r9UA~g`Tt*m_-7|nGzolq z9V*T&cNVS^BtjQciN7y6+FwR>)43xMxyQv!BfdAT)Amb=BZo^TrKz38sGV7?>iaeS zN)|c#QOOkOug)UwxFL(4QkJ4-sH39dx~i_K@_cnl+4o$s7qv+lC(B3#ieCG`xrWN| zi>O6bQ0H!VOjvjrE!N*Q-I*~>b>SLhgDsm>tL%lI(OYm}M_+_rUYh&lSCs6+*|sZL z$a8{yU~8K5D!6A{H$rH!S9=N&C~CU@{kS4bGtM>UeK#l4|3X%X!|8%((%L&Ne*2lL zd%pHQZI~MQ zGMmdOCK~nLe!%G>9hzDXn&Pb3dX^MOv&4R6eMRC?@1@lkhRA-OM+U=Ie|ds0JkPJa z^q}d*?L3{(YJHynqza&V<;?GAGOLKlYj@sov+O<|)IKLMW2YaToatGbW124l;xpiB zHh!HZd63|I=d1_hbgk=`J<>Toz<9Y|--U%HHDOAcP@WN^8lF+K=7M>+*2tXwq$^=b zM*__9g`r5>Gofb6odJ>)dbH?QgL>F`vB|a25|cmCZJYk6G7qzzvLZd^p^JHX%Hz;v z@(5?3$z$ppm=9^@lE%L`4dr#I#>R}o3LFsZTPhT|TBslTlxFc~2H|F`!VO_zVRtgA znlTz>QmK)OKoBOsTB;b&FpB6cGkR||4H}}Ud!H1gi9xYRK{~9+g9gvj{0@d_=O2?O zTQH1Fty_HuCsCW!I3S|#<*8BJ*anZ|;fYi5EB_KwwR@f=x9+z!7Qu&%Y2{j!35f49 zjoo6ZilD7T4r-C=9fuF6mWifo#0L!rt}4wm zaF@gMK?VHd<2Q?%MlE2Ob~M@28>B-wXGf*HNeEe%2r7jpIf%5$eZS6WAhGvy{Busa zyrzt9Gg1*u7E|Jz9G^x;Yv5P*@!Zo2%u^+gs&ucD^f)L=l=zuX!pGmE*ze8*6V7uK z$g35bz|aP~j`PXUL3qAz=X}Y8E*Ol-+_cc~d|ZKZ3NamxOm6XN@XAKRJihX$T8OX3 z=R$VJlmz^?edXR16+Jx)@z$~&^^aw8U&$|eYLOaep^S?rm-qtR{w#1de<3R?FBHQ= z=*C#prGxkt|C+i97S~6WTu87TFB1RI=y4?Z?tIOMueILqYP;|&L?U)5$Txl0?@&m6 z``VsUrCFX!$4Lo&u;N#%7>*R9<%X*q{|{vTv4kzjmu_q71?()3Zvu%F6LV-9{%?J6 zbXC>mBsl+U*F2-BWo6|B*K4AU%%VxVCCmI%Ru;)#g7<<?fAXP}iR*E*g!NP0k6`ZCnh?b3s|xJsup*vppr&tvL`)2gljgnQN;* z)csV90*T}|2d-Z&POm5K%E3Y2Dyxfu-*&Pw0}Blo!U%b(xPuxuQ3@-kqA{k^xHe{1 z>z;QH)&xmSP0dB2bT5mV-2*ZkpEzKF&#+i514~I(mdduoJp==@RM4C51s;UtFkQu* z_?n%D!=UeooA`B>)747{n3w7iG*}h?xFIS@l>0qo$ycu(7d!SUhAIJcwu~oR6j=CwEG^rg7=EpI!jJV408_gN;_)O&F)>+snFG>41?2 zamp;a%MIYryA{h_?KY@wBZyzuXbZSKpOOzW1HV z-swJ-=M&p1S3^8(ANL_U>1v?JqAc55#c|icKDLhGo5MmMR_ev%Y!iaUES;OC1Zu+m zCqMtSg8OIbfX#x=Zd|!RV|cwqgH-JMUW?{GsdD8mO^$pak8T0S@Hn>aLV&prB2XNq zzKnZp)8PJmKK^3d>hf=c5ABQvY;P6+PBP)I6Njwi#SLRW#|lvI;fL7!*J>^-xJH1- zI``^tOS?Rz01H>HQeRx+X*4Yzi0wxF@a`Aj=lC{Tf0BbIJ4UbwObDQGJ&xY+A_?h*NI5wx1pJ+E8qSB0{6UcPLDg*?9|Mb z75EJz3s}=kr61Vw;hfDBshzWT*@Xd9YvTTcE3nZ)b{o5U7eh!CDzP7w}v6gf(8dEs*86ppj$FnMNR7_E?`=sXU+!&*=lDPr0asmB z;FVM2pRy6*P;=g~R;PXml0z+j26g-kD44=~4died-cC}>B>iFq}tBXli;8QIaSUoX#_E()y+uM@$-K)hlercoN8W%;& ziNmI6Ly0Rgfz63aL?iF|fDr)e>T1b*oy*JpY|OOb+S83lWIb+YWngQnz6$Mv2KfXl zdO`4uGw-{JM%A)k``Ku1Mhz#u>Ai2Sdo^UW&RfWA^?BlVc+G@d9f|uF9hB6x$;lqN`h5HjZD! zClt)$BI{Lk(m31ajeObec($qw_gzx%FcWQa)^B=Rp@mTO*|vy(n}h7i(XM{W=sj&MQT?MKEGv(Ff`IR{CFoIQ;P!C3f)7aSR61A1*pDY+0&q| zkg?CpNn??`II1e-x0sY9DJ=S?4ZBZYDKW98`zs!w27fO2;A%;k#NCD!egf)8jEpqH zl0X=r?}KFf80TV^_#XZmpoU8<-E9#=7_ zPS#$~^JwK)-}x-A#Fs27;!--bHV%hUO^t(a?*2~0?#_21!~wk7`kuj?$(9mw)(wS- zjboEps02YSDOp|kLLU^oSMqwQp-E9*4TKc4chV0cKF&F`cp6+`58i%(^B}HSRFZNc z#-){r;S?C9XGnAN@>sf*V zr&-l!6euXzfQ%TceMqvVKG*P(I5!N0Yb{kCvw~zBMW$)g1c3+f3)pw7WqBuGer(zm zXz9%==yKJ_q;s|#k@+%7d*`Lk+{0JIZa-r$__^+DsP#sjqCKf-c|%vpaqk5md2c=G zQ#YX=e(Y~tVbd?$nf`W4C%6Q>%v!Dat@Lsg!j!4>mf&qJI&@pB_UQxe@7ih7C2k(h zKc&&gbc9{;LJAcU7bXY)F`2M3Q+C4}5Q(D1!9>0nnt;t7m zkOh2hi(C2Bw5S)=gRP$-SPW{aABv5!~RITTdi!BnFwFm!lBkt z2>h^%Zp$$Yd4fr@IBbPNkJkjy7Iw_lTz9VRK9ZUE+x0;}xXa9xqZ>U%YtnQ6e%9Qk zD=@*WI;~S1`yrZ3hJp5hrM{h~m)F#wK=Z*<2c-x6MY=kFv0ia>d#1!0D6Ez#X26@$ z6K4FmOQi@4ioI^wUbe83y%&ic8YgjnXF+|B8Bdw@k(?wMeTiIVs}E&BRs=;fR!p-L z12NcIeGx_Y1M*={tn$30^WhuS_M`c=ZRWwh+L@*TkBWW-ey4^0_-x;6lwP#vM1X zZ0fE`z)|Ni#M?e`bnu0Dq;cdcgb<4?V% zkmFBnTFrWd#R7;;+X?R(mKzk^h)|lF7sdt%dPqKVS(;QijQh5MNQL{30WjbzK)Uv% z{+ArZMUC_6eN-Vigjlmu^pO7uYMbc&8=_I%C-y@(DEMtSW45#1l78fCE#c;!6#|Kubq%Uf$zB?JGZK3`>mE zJv=J7<-Pk7fzA@W&SN$A>$cjq5=ut7C=z zAK|;Tc6I0Rl9#mH5Fx zqC^#hwkTEfumpBo%q{EAUugQh^-o@+zTPxFa8uoHE?OqHqQc;^un9j11QK_~O5(!Mj?vso@M(s6l6llVU?4dCh$*(K5*XQpW%(b(?7`|!HjmG_ zF;irg;dQ;2DkKuF+H}%aL4-@jLJ=j4Hc=ZG*dnEQFWH`n*{g| z>axs?{!WgPh24jfcd(=EW0jV&$#i)iWYEbWPt9WXyVCdob!#y@CGynZd=q&s{#Ls0 zL-_Fu49BajvBTpF9Bp5X<;)YmYjZM92`oaCE6VO1R*9Kx>Gj`ypy-siEaPi3__ z++I>KIR|AbMda5!$bf@UX2P+GhlS3XxTVF#4-*p;nY>96s(T%~IVHVbpE=zol%0hR zeV6i*tOnCT(!S{$dGq67n6SE%%#3eNCAQA1WKm;pW7%uR-p=j|W^gZ-p_3P=2;#I% zO%V+TM1%mDbsDHh#c2tWwuCz>WMRx}Y7_!3Wphd9=MX(I&=*h!RWDJ|ff~v~aP{j= zj);-t2EQ$NT-8uubCf%-#X3^*p!aG6uN+s5TV{A(P}@0RAu+Cj(66? zzLd>h??j^*^u0#S7Y5Sg2|yX3LP#oV>8Sx?aUT|Z?Bz$Q#Au8}66t+pOQi~SlbVkR zaobzryhqC0ks<4W3-X(*6@%E8gG6CdD}0?-_AK>I-OmWFdGhRO6N^8`T;5*8#y4i^ z!m4MH z*m2!~zbHETf1Lj)cD_PrDL3Vp{4su?pdp#wQ{7AFD51-r?=oLdhhaN zPtl)oNJFxG&+$$Rm$_?{ z_jDzPhOSa=DZH6fg|jj@S2hdUF)S)7`dYCFNSu&C{&#?B;l}+LN-&XzjfVm7&3W|{ zN&C2I-?`}49Ziv1)wP+?o`Pw6sled*le#bheFbujj?il%Pv@(XvF|u->6UI0X ze3tcAa-C5at}+57f=k-z17P5=?#CWLCGz_#ri)g9C{#HOXFuKz?26$jx9B*FORNee z_Fc7ibd0BGzC7vAxd8_RebtwhN6xBupQ2h((O8;hdOPVh%h@ys9=OYYOp&nFu(d4! z!GFEIr5gTiy3-?s*pL;)l6DV&iFyW*0!yN0hPY%h&&_DKxV z?KK{4_O^b#259--I`8u;pLH`?ujNnojLTKwCtDf4Y{C1FjGN(&5RZPT%}LAHZUbGA zKoSgIyNavUZ33#K-GF?q9s|NFN~%pCo0tI1`4_?--Y0`4SvNxwAEOq&DCbQ%!asKf zM^ie63|l~j+Tnr9q|?v~+Ahr`kpwoyee%53viZcH>v`-hYG->3z;cY) zNCL60HnSe|^PghjzxlZDjRAqo*&cQ*j%kZz>uqCGasunDNZJgh3-bnTf3!N`)wa5JFpz@a5CH5o!fX<(`$!-8`ZUH6rO(5so zCXJ$Y0|92)MG^muLIm?i7mFc$Hu`cOMzeZ&H|^5W^L2-x3s6p`@r8t3G|#JwH;?_d zo&OJ4SMT>9p6si&Q~iHc|1;Iy$~3wF%9;jPk0_4);NPTvWa_Q0LMq^+{#Zv9`OrM< FzW~z}6E^?= literal 0 HcmV?d00001 diff --git a/models/tacotron2/text/LICENSE b/models/tacotron2/text/LICENSE new file mode 100644 index 0000000000..4ad4ed1d5e --- /dev/null +++ b/models/tacotron2/text/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2017 Keith Ito + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/models/tacotron2/text/__init__.py b/models/tacotron2/text/__init__.py new file mode 100644 index 0000000000..02ecf0e741 --- /dev/null +++ b/models/tacotron2/text/__init__.py @@ -0,0 +1,74 @@ +""" from https://github.com/keithito/tacotron """ +import re +from text import cleaners +from text.symbols import symbols + + +# Mappings from symbol to numeric ID and vice versa: +_symbol_to_id = {s: i for i, s in enumerate(symbols)} +_id_to_symbol = {i: s for i, s in enumerate(symbols)} + +# Regular expression matching text enclosed in curly braces: +_curly_re = re.compile(r'(.*?)\{(.+?)\}(.*)') + + +def text_to_sequence(text, cleaner_names): + '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text. + + The text can optionally have ARPAbet sequences enclosed in curly braces embedded + in it. For example, "Turn left on {HH AW1 S S T AH0 N} Street." + + Args: + text: string to convert to a sequence + cleaner_names: names of the cleaner functions to run the text through + + Returns: + List of integers corresponding to the symbols in the text + ''' + sequence = [] + + # Check for curly braces and treat their contents as ARPAbet: + while len(text): + m = _curly_re.match(text) + if not m: + sequence += _symbols_to_sequence(_clean_text(text, cleaner_names)) + break + sequence += _symbols_to_sequence(_clean_text(m.group(1), cleaner_names)) + sequence += _arpabet_to_sequence(m.group(2)) + text = m.group(3) + + return sequence + + +def sequence_to_text(sequence): + '''Converts a sequence of IDs back to a string''' + result = '' + for symbol_id in sequence: + if symbol_id in _id_to_symbol: + s = _id_to_symbol[symbol_id] + # Enclose ARPAbet back in curly braces: + if len(s) > 1 and s[0] == '@': + s = '{%s}' % s[1:] + result += s + return result.replace('}{', ' ') + + +def _clean_text(text, cleaner_names): + for name in cleaner_names: + cleaner = getattr(cleaners, name) + if not cleaner: + raise Exception('Unknown cleaner: %s' % name) + text = cleaner(text) + return text + + +def _symbols_to_sequence(symbols): + return [_symbol_to_id[s] for s in symbols if _should_keep_symbol(s)] + + +def _arpabet_to_sequence(text): + return _symbols_to_sequence(['@' + s for s in text.split()]) + + +def _should_keep_symbol(s): + return s in _symbol_to_id and s is not '_' and s is not '~' diff --git a/models/tacotron2/text/cleaners.py b/models/tacotron2/text/cleaners.py new file mode 100644 index 0000000000..e2e35c1a8c --- /dev/null +++ b/models/tacotron2/text/cleaners.py @@ -0,0 +1,90 @@ +""" from https://github.com/keithito/tacotron """ + +''' +Cleaners are transformations that run over the input text at both training and eval time. + +Cleaners can be selected by passing a comma-delimited list of cleaner names as the "cleaners" +hyperparameter. Some cleaners are English-specific. You'll typically want to use: + 1. "english_cleaners" for English text + 2. "transliteration_cleaners" for non-English text that can be transliterated to ASCII using + the Unidecode library (https://pypi.python.org/pypi/Unidecode) + 3. "basic_cleaners" if you do not want to transliterate (in this case, you should also update + the symbols in symbols.py to match your data). +''' + +import re +from unidecode import unidecode +from .numbers import normalize_numbers + + +# Regular expression matching whitespace: +_whitespace_re = re.compile(r'\s+') + +# List of (regular expression, replacement) pairs for abbreviations: +_abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in [ + ('mrs', 'misess'), + ('mr', 'mister'), + ('dr', 'doctor'), + ('st', 'saint'), + ('co', 'company'), + ('jr', 'junior'), + ('maj', 'major'), + ('gen', 'general'), + ('drs', 'doctors'), + ('rev', 'reverend'), + ('lt', 'lieutenant'), + ('hon', 'honorable'), + ('sgt', 'sergeant'), + ('capt', 'captain'), + ('esq', 'esquire'), + ('ltd', 'limited'), + ('col', 'colonel'), + ('ft', 'fort'), +]] + + +def expand_abbreviations(text): + for regex, replacement in _abbreviations: + text = re.sub(regex, replacement, text) + return text + + +def expand_numbers(text): + return normalize_numbers(text) + + +def lowercase(text): + return text.lower() + + +def collapse_whitespace(text): + return re.sub(_whitespace_re, ' ', text) + + +def convert_to_ascii(text): + return unidecode(text) + + +def basic_cleaners(text): + '''Basic pipeline that lowercases and collapses whitespace without transliteration.''' + text = lowercase(text) + text = collapse_whitespace(text) + return text + + +def transliteration_cleaners(text): + '''Pipeline for non-English text that transliterates to ASCII.''' + text = convert_to_ascii(text) + text = lowercase(text) + text = collapse_whitespace(text) + return text + + +def english_cleaners(text): + '''Pipeline for English text, including number and abbreviation expansion.''' + text = convert_to_ascii(text) + text = lowercase(text) + text = expand_numbers(text) + text = expand_abbreviations(text) + text = collapse_whitespace(text) + return text diff --git a/models/tacotron2/text/cmudict.py b/models/tacotron2/text/cmudict.py new file mode 100644 index 0000000000..62bfef745c --- /dev/null +++ b/models/tacotron2/text/cmudict.py @@ -0,0 +1,65 @@ +""" from https://github.com/keithito/tacotron """ + +import re + + +valid_symbols = [ + 'AA', 'AA0', 'AA1', 'AA2', 'AE', 'AE0', 'AE1', 'AE2', 'AH', 'AH0', 'AH1', 'AH2', + 'AO', 'AO0', 'AO1', 'AO2', 'AW', 'AW0', 'AW1', 'AW2', 'AY', 'AY0', 'AY1', 'AY2', + 'B', 'CH', 'D', 'DH', 'EH', 'EH0', 'EH1', 'EH2', 'ER', 'ER0', 'ER1', 'ER2', 'EY', + 'EY0', 'EY1', 'EY2', 'F', 'G', 'HH', 'IH', 'IH0', 'IH1', 'IH2', 'IY', 'IY0', 'IY1', + 'IY2', 'JH', 'K', 'L', 'M', 'N', 'NG', 'OW', 'OW0', 'OW1', 'OW2', 'OY', 'OY0', + 'OY1', 'OY2', 'P', 'R', 'S', 'SH', 'T', 'TH', 'UH', 'UH0', 'UH1', 'UH2', 'UW', + 'UW0', 'UW1', 'UW2', 'V', 'W', 'Y', 'Z', 'ZH' +] + +_valid_symbol_set = set(valid_symbols) + + +class CMUDict: + '''Thin wrapper around CMUDict data. http://www.speech.cs.cmu.edu/cgi-bin/cmudict''' + def __init__(self, file_or_path, keep_ambiguous=True): + if isinstance(file_or_path, str): + with open(file_or_path, encoding='latin-1') as f: + entries = _parse_cmudict(f) + else: + entries = _parse_cmudict(file_or_path) + if not keep_ambiguous: + entries = {word: pron for word, pron in entries.items() if len(pron) == 1} + self._entries = entries + + + def __len__(self): + return len(self._entries) + + + def lookup(self, word): + '''Returns list of ARPAbet pronunciations of the given word.''' + return self._entries.get(word.upper()) + + + +_alt_re = re.compile(r'\([0-9]+\)') + + +def _parse_cmudict(file): + cmudict = {} + for line in file: + if len(line) and (line[0] >= 'A' and line[0] <= 'Z' or line[0] == "'"): + parts = line.split(' ') + word = re.sub(_alt_re, '', parts[0]) + pronunciation = _get_pronunciation(parts[1]) + if pronunciation: + if word in cmudict: + cmudict[word].append(pronunciation) + else: + cmudict[word] = [pronunciation] + return cmudict + + +def _get_pronunciation(s): + parts = s.strip().split(' ') + for part in parts: + if part not in _valid_symbol_set: + return None + return ' '.join(parts) diff --git a/models/tacotron2/text/numbers.py b/models/tacotron2/text/numbers.py new file mode 100644 index 0000000000..0d5f7fa818 --- /dev/null +++ b/models/tacotron2/text/numbers.py @@ -0,0 +1,71 @@ +""" from https://github.com/keithito/tacotron """ + +import inflect +import re + + +_inflect = inflect.engine() +_comma_number_re = re.compile(r'([0-9][0-9\,]+[0-9])') +_decimal_number_re = re.compile(r'([0-9]+\.[0-9]+)') +_pounds_re = re.compile(r'£([0-9\,]*[0-9]+)') +_dollars_re = re.compile(r'\$([0-9\.\,]*[0-9]+)') +_ordinal_re = re.compile(r'[0-9]+(st|nd|rd|th)') +_number_re = re.compile(r'[0-9]+') + + +def _remove_commas(m): + return m.group(1).replace(',', '') + + +def _expand_decimal_point(m): + return m.group(1).replace('.', ' point ') + + +def _expand_dollars(m): + match = m.group(1) + parts = match.split('.') + if len(parts) > 2: + return match + ' dollars' # Unexpected format + dollars = int(parts[0]) if parts[0] else 0 + cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0 + if dollars and cents: + dollar_unit = 'dollar' if dollars == 1 else 'dollars' + cent_unit = 'cent' if cents == 1 else 'cents' + return '%s %s, %s %s' % (dollars, dollar_unit, cents, cent_unit) + elif dollars: + dollar_unit = 'dollar' if dollars == 1 else 'dollars' + return '%s %s' % (dollars, dollar_unit) + elif cents: + cent_unit = 'cent' if cents == 1 else 'cents' + return '%s %s' % (cents, cent_unit) + else: + return 'zero dollars' + + +def _expand_ordinal(m): + return _inflect.number_to_words(m.group(0)) + + +def _expand_number(m): + num = int(m.group(0)) + if num > 1000 and num < 3000: + if num == 2000: + return 'two thousand' + elif num > 2000 and num < 2010: + return 'two thousand ' + _inflect.number_to_words(num % 100) + elif num % 100 == 0: + return _inflect.number_to_words(num // 100) + ' hundred' + else: + return _inflect.number_to_words(num, andword='', zero='oh', group=2).replace(', ', ' ') + else: + return _inflect.number_to_words(num, andword='') + + +def normalize_numbers(text): + text = re.sub(_comma_number_re, _remove_commas, text) + text = re.sub(_pounds_re, r'\1 pounds', text) + text = re.sub(_dollars_re, _expand_dollars, text) + text = re.sub(_decimal_number_re, _expand_decimal_point, text) + text = re.sub(_ordinal_re, _expand_ordinal, text) + text = re.sub(_number_re, _expand_number, text) + return text diff --git a/models/tacotron2/text/symbols.py b/models/tacotron2/text/symbols.py new file mode 100644 index 0000000000..1be47bfc75 --- /dev/null +++ b/models/tacotron2/text/symbols.py @@ -0,0 +1,18 @@ +""" from https://github.com/keithito/tacotron """ + +''' +Defines the set of symbols used in text input to the model. + +The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. ''' +from text import cmudict + +_pad = '_' +_punctuation = '!\'(),.:;? ' +_special = '-' +_letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' + +# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters): +_arpabet = ['@' + s for s in cmudict.valid_symbols] + +# Export all symbols: +symbols = [_pad] + list(_special) + list(_punctuation) + list(_letters) + _arpabet diff --git a/models/tacotron2/train.py b/models/tacotron2/train.py new file mode 100644 index 0000000000..88b09494ce --- /dev/null +++ b/models/tacotron2/train.py @@ -0,0 +1,290 @@ +import os +import time +import argparse +import math +from numpy import finfo + +import torch +from distributed import apply_gradient_allreduce +import torch.distributed as dist +from torch.utils.data.distributed import DistributedSampler +from torch.utils.data import DataLoader + +from model import Tacotron2 +from data_utils import TextMelLoader, TextMelCollate +from loss_function import Tacotron2Loss +from logger import Tacotron2Logger +from hparams import create_hparams + + +def reduce_tensor(tensor, n_gpus): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.reduce_op.SUM) + rt /= n_gpus + return rt + + +def init_distributed(hparams, n_gpus, rank, group_name): + assert torch.cuda.is_available(), "Distributed mode requires CUDA." + print("Initializing Distributed") + + # Set cuda device so everything is done on the right GPU. + torch.cuda.set_device(rank % torch.cuda.device_count()) + + # Initialize distributed communication + dist.init_process_group( + backend=hparams.dist_backend, init_method=hparams.dist_url, + world_size=n_gpus, rank=rank, group_name=group_name) + + print("Done initializing distributed") + + +def prepare_dataloaders(hparams): + # Get data, data loaders and collate function ready + trainset = TextMelLoader(hparams.training_files, hparams) + valset = TextMelLoader(hparams.validation_files, hparams) + collate_fn = TextMelCollate(hparams.n_frames_per_step) + + if hparams.distributed_run: + train_sampler = DistributedSampler(trainset) + shuffle = False + else: + train_sampler = None + shuffle = True + + train_loader = DataLoader(trainset, num_workers=1, shuffle=shuffle, + sampler=train_sampler, + batch_size=hparams.batch_size, pin_memory=False, + drop_last=True, collate_fn=collate_fn) + return train_loader, valset, collate_fn + + +def prepare_directories_and_logger(output_directory, log_directory, rank): + if rank == 0: + if not os.path.isdir(output_directory): + os.makedirs(output_directory) + os.chmod(output_directory, 0o775) + logger = Tacotron2Logger(os.path.join(output_directory, log_directory)) + else: + logger = None + return logger + + +def load_model(hparams): + model = Tacotron2(hparams).cuda() + if hparams.fp16_run: + model.decoder.attention_layer.score_mask_value = finfo('float16').min + + if hparams.distributed_run: + model = apply_gradient_allreduce(model) + + return model + + +def warm_start_model(checkpoint_path, model, ignore_layers): + assert os.path.isfile(checkpoint_path) + print("Warm starting model from checkpoint '{}'".format(checkpoint_path)) + checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') + model_dict = checkpoint_dict['state_dict'] + if len(ignore_layers) > 0: + model_dict = {k: v for k, v in model_dict.items() + if k not in ignore_layers} + dummy_dict = model.state_dict() + dummy_dict.update(model_dict) + model_dict = dummy_dict + model.load_state_dict(model_dict) + return model + + +def load_checkpoint(checkpoint_path, model, optimizer): + assert os.path.isfile(checkpoint_path) + print("Loading checkpoint '{}'".format(checkpoint_path)) + checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') + model.load_state_dict(checkpoint_dict['state_dict']) + optimizer.load_state_dict(checkpoint_dict['optimizer']) + learning_rate = checkpoint_dict['learning_rate'] + iteration = checkpoint_dict['iteration'] + print("Loaded checkpoint '{}' from iteration {}" .format( + checkpoint_path, iteration)) + return model, optimizer, learning_rate, iteration + + +def save_checkpoint(model, optimizer, learning_rate, iteration, filepath): + print("Saving model and optimizer state at iteration {} to {}".format( + iteration, filepath)) + torch.save({'iteration': iteration, + 'state_dict': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'learning_rate': learning_rate}, filepath) + + +def validate(model, criterion, valset, iteration, batch_size, n_gpus, + collate_fn, logger, distributed_run, rank): + """Handles all the validation scoring and printing""" + model.eval() + with torch.no_grad(): + val_sampler = DistributedSampler(valset) if distributed_run else None + val_loader = DataLoader(valset, sampler=val_sampler, num_workers=1, + shuffle=False, batch_size=batch_size, + pin_memory=False, collate_fn=collate_fn) + + val_loss = 0.0 + for i, batch in enumerate(val_loader): + x, y = model.parse_batch(batch) + y_pred = model(x) + loss = criterion(y_pred, y) + if distributed_run: + reduced_val_loss = reduce_tensor(loss.data, n_gpus).item() + else: + reduced_val_loss = loss.item() + val_loss += reduced_val_loss + val_loss = val_loss / (i + 1) + + model.train() + if rank == 0: + print("Validation loss {}: {:9f} ".format(iteration, val_loss)) + logger.log_validation(val_loss, model, y, y_pred, iteration) + + +def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, + rank, group_name, hparams): + """Training and validation logging results to tensorboard and stdout + + Params + ------ + output_directory (string): directory to save checkpoints + log_directory (string) directory to save tensorboard logs + checkpoint_path(string): checkpoint path + n_gpus (int): number of gpus + rank (int): rank of current gpu + hparams (object): comma separated list of "name=value" pairs. + """ + if hparams.distributed_run: + init_distributed(hparams, n_gpus, rank, group_name) + + torch.manual_seed(hparams.seed) + torch.cuda.manual_seed(hparams.seed) + + model = load_model(hparams) + learning_rate = hparams.learning_rate + optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, + weight_decay=hparams.weight_decay) + + if hparams.fp16_run: + from apex import amp + model, optimizer = amp.initialize( + model, optimizer, opt_level='O2') + + if hparams.distributed_run: + model = apply_gradient_allreduce(model) + + criterion = Tacotron2Loss() + + logger = prepare_directories_and_logger( + output_directory, log_directory, rank) + + train_loader, valset, collate_fn = prepare_dataloaders(hparams) + + # Load checkpoint if one exists + iteration = 0 + epoch_offset = 0 + if checkpoint_path is not None: + if warm_start: + model = warm_start_model( + checkpoint_path, model, hparams.ignore_layers) + else: + model, optimizer, _learning_rate, iteration = load_checkpoint( + checkpoint_path, model, optimizer) + if hparams.use_saved_learning_rate: + learning_rate = _learning_rate + iteration += 1 # next iteration is iteration + 1 + epoch_offset = max(0, int(iteration / len(train_loader))) + + model.train() + is_overflow = False + # ================ MAIN TRAINNIG LOOP! =================== + for epoch in range(epoch_offset, hparams.epochs): + print("Epoch: {}".format(epoch)) + for i, batch in enumerate(train_loader): + start = time.perf_counter() + for param_group in optimizer.param_groups: + param_group['lr'] = learning_rate + + model.zero_grad() + x, y = model.parse_batch(batch) + y_pred = model(x) + + loss = criterion(y_pred, y) + if hparams.distributed_run: + reduced_loss = reduce_tensor(loss.data, n_gpus).item() + else: + reduced_loss = loss.item() + if hparams.fp16_run: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + if hparams.fp16_run: + grad_norm = torch.nn.utils.clip_grad_norm_( + amp.master_params(optimizer), hparams.grad_clip_thresh) + is_overflow = math.isnan(grad_norm) + else: + grad_norm = torch.nn.utils.clip_grad_norm_( + model.parameters(), hparams.grad_clip_thresh) + + optimizer.step() + + if not is_overflow and rank == 0: + duration = time.perf_counter() - start + print("Train loss {} {:.6f} Grad Norm {:.6f} {:.2f}s/it".format( + iteration, reduced_loss, grad_norm, duration)) + logger.log_training( + reduced_loss, grad_norm, learning_rate, duration, iteration) + + if not is_overflow and (iteration % hparams.iters_per_checkpoint == 0): + validate(model, criterion, valset, iteration, + hparams.batch_size, n_gpus, collate_fn, logger, + hparams.distributed_run, rank) + if rank == 0: + checkpoint_path = os.path.join( + output_directory, "checkpoint_{}".format(iteration)) + save_checkpoint(model, optimizer, learning_rate, iteration, + checkpoint_path) + + iteration += 1 + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-o', '--output_directory', type=str, + help='directory to save checkpoints') + parser.add_argument('-l', '--log_directory', type=str, + help='directory to save tensorboard logs') + parser.add_argument('-c', '--checkpoint_path', type=str, default=None, + required=False, help='checkpoint path') + parser.add_argument('--warm_start', action='store_true', + help='load model weights only, ignore specified layers') + parser.add_argument('--n_gpus', type=int, default=1, + required=False, help='number of gpus') + parser.add_argument('--rank', type=int, default=0, + required=False, help='rank of current gpu') + parser.add_argument('--group_name', type=str, default='group_name', + required=False, help='Distributed group name') + parser.add_argument('--hparams', type=str, + required=False, help='comma separated name=value pairs') + + args = parser.parse_args() + hparams = create_hparams(args.hparams) + + torch.backends.cudnn.enabled = hparams.cudnn_enabled + torch.backends.cudnn.benchmark = hparams.cudnn_benchmark + + print("FP16 Run:", hparams.fp16_run) + print("Dynamic Loss Scaling:", hparams.dynamic_loss_scaling) + print("Distributed Run:", hparams.distributed_run) + print("cuDNN Enabled:", hparams.cudnn_enabled) + print("cuDNN Benchmark:", hparams.cudnn_benchmark) + + train(args.output_directory, args.log_directory, args.checkpoint_path, + args.warm_start, args.n_gpus, args.rank, args.group_name, hparams) diff --git a/models/tacotron2/utils.py b/models/tacotron2/utils.py new file mode 100644 index 0000000000..4395201844 --- /dev/null +++ b/models/tacotron2/utils.py @@ -0,0 +1,29 @@ +import numpy as np +from scipy.io.wavfile import read +import torch + + +def get_mask_from_lengths(lengths): + max_len = torch.max(lengths).item() + ids = torch.arange(0, max_len, out=torch.cuda.LongTensor(max_len)) + mask = (ids < lengths.unsqueeze(1)).bool() + return mask + + +def load_wav_to_torch(full_path): + sampling_rate, data = read(full_path) + return torch.FloatTensor(data.astype(np.float32)), sampling_rate + + +def load_filepaths_and_text(filename, split="|"): + with open(filename, encoding='utf-8') as f: + filepaths_and_text = [line.strip().split(split) for line in f] + return filepaths_and_text + + +def to_gpu(x): + x = x.contiguous() + + if torch.cuda.is_available(): + x = x.cuda(non_blocking=True) + return torch.autograd.Variable(x) From 79a21570e9c0bfb0da630f1bfd9d5f55523546b6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 04:50:06 -0400 Subject: [PATCH 0211/1042] re-codegen queries --- sqlx-data.json | 2013 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 1917 insertions(+), 96 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index aa3790e4be..d3a9e49cc4 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -20,13 +20,43 @@ "nullable": [] } }, - "293b56f5ef46fa7a860ff47b64b9784038883f742cb0c23bf9775cee78d4acef": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.deleted_at IS NULL\n ", + "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "2b04dd7db52ef3a97a6f420a22a4dbee7416ae58697438df69e87ffd47bbf350": { + "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "2bfb24d71e1fccb98b0c27912214dca2e2c2833355fdfa9f70e4d7a4d99a038b": { + "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 11 + }, + "nullable": [] + } + }, + "32a54bab859ecca83cf42280a38ba9965d6c029624acd976eb024bc28090fd3b": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND w2l_results.is_mod_hidden_from_public IS FALSE\n AND w2l_templates.is_mod_approved IS TRUE\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "template_token", + "name": "w2l_result_token", "type_info": { "type": "VarString", "flags": { @@ -38,23 +68,23 @@ }, { "ordinal": 1, - "name": "template_type", + "name": "maybe_tts_inference_result_token", "type_info": { - "type": "String", + "type": "VarString", "flags": { - "bits": 393 + "bits": 136 }, "char_set": 224, - "max_size": 28 + "max_size": 128 } }, { "ordinal": 2, - "name": "creator_user_token", + "name": "maybe_w2l_template_token", "type_info": { "type": "VarString", "flags": { - "bits": 4233 + "bits": 4228 }, "char_set": 224, "max_size": 128 @@ -62,67 +92,67 @@ }, { "ordinal": 3, - "name": "creator_username", + "name": "template_type", "type_info": { - "type": "VarString", + "type": "String", "flags": { - "bits": 4229 + "bits": 392 }, "char_set": 224, - "max_size": 80 + "max_size": 28 } }, { "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, - "name": "title", + "name": "template_title", "type_info": { "type": "String", "flags": { - "bits": 4225 + "bits": 4224 }, "char_set": 224, "max_size": 1020 } }, { - "ordinal": 7, - "name": "frame_width", + "ordinal": 5, + "name": "maybe_creator_user_token", "type_info": { - "type": "Long", + "type": "VarString", "flags": { - "bits": 1 + "bits": 4228 }, - "char_set": 63, - "max_size": 10 + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 } }, { "ordinal": 8, - "name": "frame_height", + "name": "file_size_bytes", "type_info": { "type": "Long", "flags": { @@ -134,7 +164,7 @@ }, { "ordinal": 9, - "name": "duration_millis", + "name": "frame_width", "type_info": { "type": "Long", "flags": { @@ -146,26 +176,26 @@ }, { "ordinal": 10, - "name": "maybe_public_bucket_preview_image_object_name", + "name": "frame_height", "type_info": { - "type": "VarString", + "type": "Long", "flags": { - "bits": 128 + "bits": 1 }, - "char_set": 224, - "max_size": 1020 + "char_set": 63, + "max_size": 10 } }, { "ordinal": 11, - "name": "maybe_public_bucket_preview_video_object_name", + "name": "duration_millis", "type_info": { - "type": "VarString", + "type": "Long", "flags": { - "bits": 128 + "bits": 1 }, - "char_set": 224, - "max_size": 1020 + "char_set": 63, + "max_size": 10 } }, { @@ -197,53 +227,23 @@ "Right": 0 }, "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, false, true, true, + true, + true, + true, + true, + true, + false, + false, + false, + false, false, false ] } }, - "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { - "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "2b04dd7db52ef3a97a6f420a22a4dbee7416ae58697438df69e87ffd47bbf350": { - "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "2bfb24d71e1fccb98b0c27912214dca2e2c2833355fdfa9f70e4d7a4d99a038b": { - "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 11 - }, - "nullable": [] - } - }, "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", "describe": { @@ -471,6 +471,213 @@ ] } }, + "3f40b37731f6049ef9167a8912e411123053ccdfebd1564e0a8f257c3a2db0b9": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND users.username = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "is_mod_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false, + false + ] + } + }, "4995b1688dab1a25fa16ea59d6282b052be5c4ed6bf740a0840e24c9ad67e55a": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.updatable_slug = ?\nAND w2l.deleted_at IS NULL\n ", "describe": { @@ -739,6 +946,210 @@ ] } }, + "685a1d5bb03668be301bab35d677e270c94661c53f370ecd6ce8f1b697c4a01d": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "w2l_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 392 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 4, + "name": "template_title", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 11, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ] + } + }, + "6df889b77bffbd3f0678ad7c0db88982fa79295b569072ed6bd424be9218a595": { + "query": "\nINSERT INTO firehose_entries\nSET\n token = ?,\n event_type = ?,\n maybe_target_user_token = ?,\n maybe_target_entity_token = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 4 + }, + "nullable": [] + } + }, "746526e2e150150ef804d717f743de85010972d98cbb137e44d8348cfdb4dddc": { "query": "\nINSERT INTO w2l_results\nSET\n token = ?,\n maybe_w2l_template_token = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_video_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n duration_millis = ?\n ", "describe": { @@ -1025,6 +1436,135 @@ "nullable": [] } }, + "88328e04ce947d3f541951aada9cee25135aa950ab478cf0dc6de453bb079e17": { + "query": "\nSELECT\n events.token as event_token,\n events.event_type,\n events.maybe_target_user_token,\n users.username as maybe_target_username,\n users.display_name as maybe_target_display_name,\n users.email_gravatar_hash as maybe_target_user_gravatar_hash,\n events.maybe_target_entity_token,\n events.created_at,\n events.updated_at\nFROM firehose_entries as events\nLEFT OUTER JOIN users\nON events.maybe_target_user_token = users.token\nORDER BY events.id DESC\nLIMIT 25\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "event_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "event_type", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "maybe_target_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "maybe_target_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "maybe_target_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "maybe_target_user_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_target_entity_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 8, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + true, + true, + true, + true, + true, + false, + false + ] + } + }, "8d61bb77449a78044135a3d6e5009a3a27b4671b7e40501d318b05ae401f02a8": { "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE email_address = ?\nLIMIT 1\n ", "describe": { @@ -1089,6 +1629,200 @@ ] } }, + "94847f448294888db6c800cc9558738b0c0de023225961d070b84b3906a2e493": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND users.username = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "w2l_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 392 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 4, + "name": "template_title", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 11, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ] + } + }, "9d4932baa4023accc2d28af98b58a47149d37c4937cd0171e6b955c09855c5f8": { "query": "\nINSERT INTO w2l_inference_jobs\nSET\n maybe_w2l_template_token = ?,\n maybe_tts_inference_result_token = ?,\n maybe_public_audio_bucket_location = NULL,\n maybe_public_image_bucket_location = NULL,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", "describe": { @@ -1385,6 +2119,420 @@ "nullable": [] } }, + "c294456f67ddbc875cc5c09fc66451172c03a78d1b83ceb75563205aae36e2e0": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND w2l.is_mod_approved IS TRUE\n AND users.username = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "is_mod_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false, + false + ] + } + }, + "c5aa964dead590345796e5c587c6abe47a6f3fcc201efe67b6134f4997f0ac7b": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "is_mod_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false, + false + ] + } + }, "ca5efc3be0f09649df2add5270c06544133288a95ffcb9a0589efd4fe345efe7": { "query": "\nINSERT INTO badges ( slug, title, description, image_url )\nVALUES ( ?, ?, ?, ? )\n ", "describe": { @@ -1395,6 +2543,200 @@ "nullable": [] } }, + "cfea55416f30c179080ae1fd9385a4201e5accfd9ace79d1a7f08cd34dcb0f01": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND w2l_results.is_mod_hidden_from_public IS FALSE\n AND w2l_templates.is_mod_approved IS TRUE\n AND users.username = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "w2l_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 392 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 4, + "name": "template_title", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 11, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ] + } + }, "d14d0e03acc71081726ef0e31bade27110f1ac8eff9ccfe155df5dbd47d06560": { "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n maybe_w2l_template_token,\n maybe_public_image_bucket_location,\n maybe_tts_inference_result_token,\n maybe_public_audio_bucket_hash,\n maybe_public_audio_bucket_location,\n\n maybe_original_audio_filename,\n maybe_original_audio_download_url,\n maybe_audio_mime_type,\n\n creator_ip_address,\n maybe_creator_user_token,\n\n creator_set_visibility,\n disable_end_bump,\n disable_watermark,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM w2l_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { @@ -1822,6 +3164,213 @@ ] } }, + "de3d708b18ad5acc2769cf8783d76c1b37840aabb87d141b6eba5ddd2aca593b": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND w2l.is_mod_approved IS TRUE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "is_mod_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false, + false + ] + } + }, "e568e1c13676e5cd06b2136808e71d10ddd87f6ef4a39848c03531ca65833a15": { "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_address,\n email_confirmed,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n banned,\n dark_mode_preference,\n avatar_public_bucket_hash,\n disable_gravatar,\n hide_results_preference,\n auto_play_audio_preference,\n auto_play_video_preference,\n maybe_preferred_tts_model_token,\n maybe_preferred_w2l_template_token,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username\nFROM users\nLEFT OUTER JOIN user_sessions\nON users.token = user_sessions.user_token\nWHERE user_sessions.token = ?\nAND user_sessions.deleted_at IS NULL\nAND users.deleted_at IS NULL\n ", "describe": { @@ -2417,6 +3966,278 @@ ] } }, + "f00402e757b7f5d8ded60e78ac47d93ccf4c71a2acba9dddcedfed0ac4d2e3a5": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND w2l_results.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "w2l_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "public_bucket_video_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 392 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 5, + "name": "template_title", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 9, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "maybe_template_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "maybe_template_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 12, + "name": "maybe_template_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 13, + "name": "maybe_template_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 14, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 15, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 17, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 18, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ] + } + }, "f6e625dde66da14091591f321eb65b0137fceb7168305bf17806f1876a644954": { "query": "\nSELECT\n token as template_token\nFROM w2l_templates\nWHERE token = ?\nAND deleted_at IS NULL\n ", "describe": { From ecc2c3bec6a60a3f2bc8bfb40dcce82d040be21b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 04:55:49 -0400 Subject: [PATCH 0212/1042] I installed my own requirements.txt somehow not realizing it was there what --- .gitignore | 1 + Dockerfile | 14 ++++++++++++++ models/tacotron2/requirements.txt | 16 +++++++++++----- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index b768de628d..a81f2dfe7b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ __pycache__/ _ignored_schema.rs models/Wav2Lip/python/ +models/tacotron2/python/ production_database_secret_connect.sh secrets.toml secrets.yaml diff --git a/Dockerfile b/Dockerfile index 3be99a7629..b14fda1445 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,14 @@ FROM nvidia/cuda:10.1-devel-ubuntu18.04 as pybuild-base # NB: https://github.com/NVIDIA/nvidia-docker/issues/864#issuecomment-439848887 # NB: We do not install ffmpeg, since the version is 3.8.* series and we need 4.2.* +# +# NB(2021-05-30): We need to install `cython3` for building modern Tacotron 2 +# dependencies (ie. protoc) on old Ubuntu RUN apt-get update \ && apt-get install -y \ build-essential \ curl \ + cython3 \ g++-7 \ gcc-7 \ git \ @@ -57,6 +61,16 @@ RUN python3.6 -m venv python RUN . python/bin/activate \ && pip install --upgrade pip \ && pip install -r requirements.txt + && deactivate + +COPY models/tacotron2 ./models/tacotron2 +WORKDIR models/tacotron2 + +RUN python3.6 -m venv python +RUN . python/bin/activate \ + && pip install --upgrade pip \ + && pip install -r requirements.txt + && deactivate # ==================== Rust Build Base ==================== diff --git a/models/tacotron2/requirements.txt b/models/tacotron2/requirements.txt index 9b23dca725..6eb9914bb1 100644 --- a/models/tacotron2/requirements.txt +++ b/models/tacotron2/requirements.txt @@ -1,8 +1,14 @@ -matplotlib==2.1.0 -tensorflow==1.15.2 -numpy==1.13.3 -inflect==0.2.5 +#numpy +scipy +matplotlib librosa==0.6.0 -scipy==1.0.0 +tensorflow +tensorboardX +inflect==0.2.5 Unidecode==1.0.22 pillow +jupyter + +# Note: numpy@latest doesn't build in Python<3.7, and numba causes it +numba==0.46.0 +numpy==1.14.3 From 943047e59cc13997507043eeaf215bafde479bfc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 06:09:33 -0400 Subject: [PATCH 0213/1042] an old "inference" script I found seems to work :whatever: --- models/tacotron2/inference.py | 112 ++++++++++++++++++++++++++++++ models/tacotron2/requirements.txt | 16 ++++- src/main.rs | 10 +++ 3 files changed, 136 insertions(+), 2 deletions(-) create mode 100755 models/tacotron2/inference.py diff --git a/models/tacotron2/inference.py b/models/tacotron2/inference.py new file mode 100755 index 0000000000..76b708a50e --- /dev/null +++ b/models/tacotron2/inference.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +# 1 +#import matplotlib +#import matplotlib.pylab as plt + +#import IPython.display as ipd + +import sys +sys.path.append('waveglow/') +import numpy as np +import torch + +from hparams import create_hparams +from model import Tacotron2 +from layers import TacotronSTFT, STFT +from audio_processing import griffin_lim +from train import load_model +from text import text_to_sequence +#from denoiser import Denoiser + +#from wav_images import render_histogram + +# 2 +def plot_data(data, figsize=(16, 4)): + fig, axes = plt.subplots(1, len(data), figsize=figsize) + for i in range(len(data)): + axes[i].imshow(data[i], aspect='auto', origin='bottom', + interpolation='none') + +# 3 +hparams = create_hparams() +hparams.sampling_rate = 22050 + +# Fix bug running on CPU? +#torch.set_default_dtype(torch.float16) +torch.set_default_tensor_type('torch.DoubleTensor') + +# 4 +checkpoint_path = "/home/bt/models/tacotron2-nvidia/tacotron2_statedict.pt" +#checkpoint_path = "/home/bt/models/tigger_tt_model.pt" +checkpoint_path = "/home/bt/models/uber_test_2.pt" +model = load_model(hparams) +model.load_state_dict(torch.load(checkpoint_path, map_location=torch.device('cpu'))['state_dict']) +#model.load_state_dict(torch.jit.load(checkpoint_path, map_location=torch.device('cpu'))['state_dict']) +#_ = model.cuda().eval().half() +#_ = model.eval().half() +_ = model.eval() + +# 5 +#waveglow_path = 'waveglow_256channels.pt' +#waveglow = torch.load(waveglow_path)['model'] +#waveglow.cuda().eval().half() +#for k in waveglow.convinv: +# k.float() +#denoiser = Denoiser(waveglow) + +# 6 +print("\n\n===TEXT===\n") +text = "This is the song that never ends. It goes on and on my friend." +#text = "hi you" +print("Text: {}".format(text)) +print("Text len: {}".format(len(text))) +seq = text_to_sequence(text, ['english_cleaners']) +print("Seq: {}".format(type(seq))) +print("Seq: {}".format(seq)) +sequence = np.array(seq)[None, :] +print("Sequence np.array: {}".format(type(sequence))) +print("Sequence np.array: {}".format(sequence)) +sequence = torch.from_numpy(sequence) +print("Sequence from_numpy: {}".format(type(sequence))) +print("Sequence from_numpy: {}".format(sequence)) +print("Sequence from_numpy dtype: {}".format(sequence.dtype)) +sequence = torch.autograd.Variable(sequence).long() + #torch.from_numpy(sequence)).cuda().long() + #torch.from_numpy(sequence)) + +print("Sequence torch autograd: {}".format(sequence)) + +print('Saving text sequence tensor') +torch.save(sequence, 'text_sequence.pt') + +print("\n\n===END TEXT===\n") + +# 7 +mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence) +#plot_data((mel_outputs.float().data.cpu().numpy()[0], +# mel_outputs_postnet.float().data.cpu().numpy()[0], +# alignments.float().data.cpu().numpy()[0].T)) + +# Per melgan preprocess.py, need to output: +# mel_output: torch.FloatTensor of shape (B, n_mel_channels, T) +print("Mel outputs:") +print(mel_outputs) +print(mel_outputs.shape) +print(mel_outputs_postnet) +print(mel_outputs_postnet.shape) + +print('Saving mels') +torch.save(mel_outputs, 'mel_outputs.mel') +torch.save(mel_outputs_postnet, 'mel_outputs_postnet.mel') + +#print('Rendering histograms') +#render_histogram(mel_outputs, 'mel_outputs.png') +#render_histogram(mel_outputs_postnet, 'mel_outputs_postnet.png') + +# 8 +with torch.no_grad(): + #audio = waveglow.infer(mel_outputs_postnet, sigma=0.666) + pass +#ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate) + diff --git a/models/tacotron2/requirements.txt b/models/tacotron2/requirements.txt index 6eb9914bb1..be6b21a3c2 100644 --- a/models/tacotron2/requirements.txt +++ b/models/tacotron2/requirements.txt @@ -2,8 +2,8 @@ scipy matplotlib librosa==0.6.0 -tensorflow -tensorboardX +#tensorflow +#tensorboardX inflect==0.2.5 Unidecode==1.0.22 pillow @@ -12,3 +12,15 @@ jupyter # Note: numpy@latest doesn't build in Python<3.7, and numba causes it numba==0.46.0 numpy==1.14.3 + + +#torch==1.8.1 +#tensorboardX==1.1 +#tensorflow==1.13.1 +#tensorflow-estimator==1.13.0 + +# Fix for +# TypeError: create_target_machine() got an unexpected keyword argument 'jitdebug' +llvmlite==0.32.1 + +tensorboardX==1.15 diff --git a/src/main.rs b/src/main.rs index 1e6c16ecea..3aa3640461 100644 --- a/src/main.rs +++ b/src/main.rs @@ -56,6 +56,16 @@ use crate::http_server::endpoints::users::list_user_w2l_inference_results::list_ use crate::http_server::endpoints::w2l::get_w2l_result::get_w2l_inference_result_handler; use crate::common_queries::firehose_publisher::FirehosePublisher; +// TODO TODO TODO TODO +// TODO TODO TODO TODO +// TODO TODO TODO TODO +// https://github.com/TensorSpeech/TensorFlowTTS (MAYBE USE THIS) +// TODO TODO TODO TODO +// TODO TODO TODO TODO +// TODO TODO TODO TODO + + + const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; // NB: sqlx::query is spammy and logs all queries as "info"-level From 8655e99e43c8190550ffc1d7675ec7e52c572b03 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 06:19:16 -0400 Subject: [PATCH 0214/1042] fix dockerfile && --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b14fda1445..cf36b2acd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,7 +60,7 @@ RUN python3.6 --version RUN python3.6 -m venv python RUN . python/bin/activate \ && pip install --upgrade pip \ - && pip install -r requirements.txt + && pip install -r requirements.txt \ && deactivate COPY models/tacotron2 ./models/tacotron2 @@ -69,7 +69,7 @@ WORKDIR models/tacotron2 RUN python3.6 -m venv python RUN . python/bin/activate \ && pip install --upgrade pip \ - && pip install -r requirements.txt + && pip install -r requirements.txt \ && deactivate # ==================== Rust Build Base ==================== From 41fc0ee1ecab77f6d8943bb4c4c13190515b62d9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 06:45:57 -0400 Subject: [PATCH 0215/1042] fix requirements? --- models/tacotron2/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/tacotron2/requirements.txt b/models/tacotron2/requirements.txt index be6b21a3c2..f3144498e1 100644 --- a/models/tacotron2/requirements.txt +++ b/models/tacotron2/requirements.txt @@ -3,7 +3,7 @@ scipy matplotlib librosa==0.6.0 #tensorflow -#tensorboardX +tensorboardX inflect==0.2.5 Unidecode==1.0.22 pillow @@ -23,4 +23,4 @@ numpy==1.14.3 # TypeError: create_target_machine() got an unexpected keyword argument 'jitdebug' llvmlite==0.32.1 -tensorboardX==1.15 +#tensorboardX==1.15 From 5e0a1d2e7d09af3201ee67a47354d6563b6ddda0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 07:04:42 -0400 Subject: [PATCH 0216/1042] Fix dockerfile again --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index cf36b2acd1..1cc6c02e23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,11 @@ FROM nvidia/cuda:10.1-devel-ubuntu18.04 as pybuild-base +# NB(2021-05-30): This is the image that Tacotron 2's Dockerfile specifies. +# I have not tried this, but it might be necessary. +# FROM pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 +# ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} + # NB: https://github.com/NVIDIA/nvidia-docker/issues/864#issuecomment-439848887 # NB: We do not install ffmpeg, since the version is 3.8.* series and we need 4.2.* # @@ -84,6 +89,7 @@ COPY Cargo.lock . COPY Cargo.toml . COPY sqlx-data.json . COPY src/ ./src +COPY db/ ./db COPY migrations/ ./migrations RUN $HOME/.cargo/bin/cargo fetch From e0479aa1da65a50760b73b1f517f6f2b7a8553a8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 13:14:55 -0400 Subject: [PATCH 0217/1042] Change dockerfile base image --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1cc6c02e23..5779e17d71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ # ==================== Python Build Base ==================== -FROM nvidia/cuda:10.1-devel-ubuntu18.04 as pybuild-base +#FROM nvidia/cuda:10.1-devel-ubuntu18.04 as pybuild-base # NB(2021-05-30): This is the image that Tacotron 2's Dockerfile specifies. # I have not tried this, but it might be necessary. -# FROM pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 -# ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +FROM pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 +ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} # NB: https://github.com/NVIDIA/nvidia-docker/issues/864#issuecomment-439848887 # NB: We do not install ffmpeg, since the version is 3.8.* series and we need 4.2.* From 18830fea616231ebd697c471619b18943ef43c0d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 13:22:57 -0400 Subject: [PATCH 0218/1042] I needed to actually re-specify .gitignore to not pass to docker ctx --- .dockerignore | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..0f84aa1211 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +*secret*sh +*~ +.env +.env-secrets +.git +.idea/ +__pycache__/ +_ignored_schema.rs +models/Wav2Lip/python/ +models/tacotron2/python/ +production_database_secret_connect.sh +secrets.toml +secrets.yaml +target/ From d52bdc6a10a661467676fb236768d94771ada6e5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 13:34:07 -0400 Subject: [PATCH 0219/1042] update dockerfile --- Dockerfile | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5779e17d71..d3ca3a993c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,28 @@ # ==================== Python Build Base ==================== + #FROM nvidia/cuda:10.1-devel-ubuntu18.04 as pybuild-base +FROM nvidia/cuda:11.3.0-devel-ubuntu20.04 as pybuild-base + +# tzdata install hangs without presetting the time zone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # NB(2021-05-30): This is the image that Tacotron 2's Dockerfile specifies. # I have not tried this, but it might be necessary. -FROM pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 -ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +#FROM pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 +#ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} # NB: https://github.com/NVIDIA/nvidia-docker/issues/864#issuecomment-439848887 # NB: We do not install ffmpeg, since the version is 3.8.* series and we need 4.2.* # # NB(2021-05-30): We need to install `cython3` for building modern Tacotron 2 # dependencies (ie. protoc) on old Ubuntu -RUN apt-get update \ - && apt-get install -y \ +# RUN apt-get update \ +# && apt-get install -y \ +RUN apt-get update +#RUN apt-cache search python3.8 +RUN apt-get install -y \ build-essential \ curl \ cython3 \ @@ -34,9 +43,9 @@ RUN apt-get update \ netcat \ python-dev \ python3-pip \ - python3.6 \ - python3.6-dev \ - python3.6-venv\ + python3.8 \ + python3.8-dev \ + python3.8-venv\ redis-tools \ silversearcher-ag \ sox \ @@ -61,8 +70,9 @@ FROM pybuild-base as pybuild-requirements COPY models/Wav2Lip ./models/Wav2Lip WORKDIR models/Wav2Lip -RUN python3.6 --version -RUN python3.6 -m venv python +RUN python3.8 --version +RUN python3.8 -m venv python + RUN . python/bin/activate \ && pip install --upgrade pip \ && pip install -r requirements.txt \ @@ -71,7 +81,7 @@ RUN . python/bin/activate \ COPY models/tacotron2 ./models/tacotron2 WORKDIR models/tacotron2 -RUN python3.6 -m venv python +RUN python3.8 -m venv python RUN . python/bin/activate \ && pip install --upgrade pip \ && pip install -r requirements.txt \ From 1d097c1a64940ddb405087adf980265957074f39 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 13:38:54 -0400 Subject: [PATCH 0220/1042] update dockerfile --- Dockerfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index d3ca3a993c..ee65c51a95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,10 +22,17 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # && apt-get install -y \ RUN apt-get update #RUN apt-cache search python3.8 + +# ffmpeg version 4.2.4-1ubuntu0.1 Copyright (c) 2000-2020 the FFmpeg developers +#RUN apt-get install -y ffmpeg +#RUN ffmpeg -version +#RUN sleep 100 + RUN apt-get install -y \ build-essential \ curl \ cython3 \ + ffmpeg \ g++-7 \ gcc-7 \ git \ @@ -56,12 +63,11 @@ RUN apt-get install -y \ # We need ffmpeg version >= 4.2 (https://superuser.com/a/579110) # http://ubuntuhandbook.org/index.php/2020/06/install-ffmpeg-4-3-via-ppa-ubuntu-18-04-16-04/ -RUN apt-get install -y software-properties-common \ - && add-apt-repository ppa:jonathonf/ffmpeg-4 \ - && apt-get update \ - && apt-get install -y ffmpeg - -RUN ffmpeg -version +#RUN apt-get install -y software-properties-common \ +# && add-apt-repository ppa:jonathonf/ffmpeg-4 \ +# && apt-get update \ +# && apt-get install -y ffmpeg +# # ==================== Python Build Step 2: Wav2Lip Requirements ==================== From eee60e3cdde231af9f763992c5a5a4633e2e4ff9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 13:44:43 -0400 Subject: [PATCH 0221/1042] upgrade python package --- models/Wav2Lip/requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/models/Wav2Lip/requirements.txt b/models/Wav2Lip/requirements.txt index de89d8f04b..25bf66c50a 100644 --- a/models/Wav2Lip/requirements.txt +++ b/models/Wav2Lip/requirements.txt @@ -2,9 +2,12 @@ librosa==0.7.0 numba==0.48 numpy==1.17.1 opencv-contrib-python==4.2.0.34 -opencv-python==4.1.0.25 python-magic==0.4.22 tensorflow==1.12.0 torch==1.1.0 torchvision==0.3.0 tqdm==4.45.0 + +# NB(bt, 2021-05-30): After upgrading ubuntu, the old opencv package isn't available +#opencv-python==4.1.0.25 +opencv-python==4.1.2.30 From a05b1ed55129da2c8e776f971f1b1c04482fee84 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 14:41:39 -0400 Subject: [PATCH 0222/1042] revert Wav2Lip using Python3.6 with a PPA hack --- Dockerfile | 24 +++++++++++++++++++++--- models/Wav2Lip/requirements.txt | 5 +---- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index ee65c51a95..35102f6c4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,8 @@ RUN apt-get update #RUN ffmpeg -version #RUN sleep 100 +# Notes on packages: +# * software-properties-common - needed for `apt-add-repository` RUN apt-get install -y \ build-essential \ curl \ @@ -48,6 +50,7 @@ RUN apt-get install -y \ libxext6 \ libxrender-dev \ netcat \ + software-properties-common \ python-dev \ python3-pip \ python3.8 \ @@ -60,6 +63,21 @@ RUN apt-get install -y \ vim \ wget + +# Wav2Lip needs Python3.6 +# NB: we install python3-virtualenv as `python3.6 -m venv` won't work anymore +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get install -y python3.6 +RUN apt-get install -y python3-virtualenv +RUN apt-get install -y python3.6-venv +RUN apt-get install -y python3.6-dev +#RUN apt-get install python3-virtualenv + + +RUN python3.6 --version +RUN python3.8 --version + # We need ffmpeg version >= 4.2 (https://superuser.com/a/579110) # http://ubuntuhandbook.org/index.php/2020/06/install-ffmpeg-4-3-via-ppa-ubuntu-18-04-16-04/ @@ -76,9 +94,9 @@ FROM pybuild-base as pybuild-requirements COPY models/Wav2Lip ./models/Wav2Lip WORKDIR models/Wav2Lip -RUN python3.8 --version -RUN python3.8 -m venv python - +# NB: We need Python3.6 +RUN python3.6 -m venv python +#RUN virtualenv -p /usr/bin/python3.6 ... wait RUN . python/bin/activate \ && pip install --upgrade pip \ && pip install -r requirements.txt \ diff --git a/models/Wav2Lip/requirements.txt b/models/Wav2Lip/requirements.txt index 25bf66c50a..de89d8f04b 100644 --- a/models/Wav2Lip/requirements.txt +++ b/models/Wav2Lip/requirements.txt @@ -2,12 +2,9 @@ librosa==0.7.0 numba==0.48 numpy==1.17.1 opencv-contrib-python==4.2.0.34 +opencv-python==4.1.0.25 python-magic==0.4.22 tensorflow==1.12.0 torch==1.1.0 torchvision==0.3.0 tqdm==4.45.0 - -# NB(bt, 2021-05-30): After upgrading ubuntu, the old opencv package isn't available -#opencv-python==4.1.0.25 -opencv-python==4.1.2.30 From 8abe0eee6279bef3b9c3d5f0bd105edf063681a3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 15:24:26 -0400 Subject: [PATCH 0223/1042] notes --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 35102f6c4e..9fb1c9cb24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,11 @@ # ==================== Python Build Base ==================== +# Authenticating with GHCR locally: +# https://docs.github.com/en/packages/working-with-a-github-packages-registry/migrating-to-the-container-registry-from-the-docker-registry +# ``` +# export CR_PAT=YOUR_TOKEN +# echo $CR_PAT | docker login ghcr.io -u USERNAME --password-stdin +# ``` #FROM nvidia/cuda:10.1-devel-ubuntu18.04 as pybuild-base FROM nvidia/cuda:11.3.0-devel-ubuntu20.04 as pybuild-base From 3b5c0b70eb1c730f8029a8ab5ca4387d02b19539 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 15:30:04 -0400 Subject: [PATCH 0224/1042] add pciutils --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9fb1c9cb24..dbee07533d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,7 @@ RUN apt-get update # Notes on packages: # * software-properties-common - needed for `apt-add-repository` +# * pciutils - needed for `lspci | grep -i nvidia` RUN apt-get install -y \ build-essential \ curl \ @@ -56,7 +57,7 @@ RUN apt-get install -y \ libxext6 \ libxrender-dev \ netcat \ - software-properties-common \ + pciutils \ python-dev \ python3-pip \ python3.8 \ @@ -64,6 +65,7 @@ RUN apt-get install -y \ python3.8-venv\ redis-tools \ silversearcher-ag \ + software-properties-common \ sox \ tmux \ vim \ From 6201f5c49a9ac9de948b0c4ab5240efe26c913ad Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 16:03:44 -0400 Subject: [PATCH 0225/1042] install nvidia drivers --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index dbee07533d..ca42ff60ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,10 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN apt-get update #RUN apt-cache search python3.8 +# NB: Here we install nvidia drivers +# NB: Avoid keyboard prompt +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nvidia-384 --no-install-recommends + # ffmpeg version 4.2.4-1ubuntu0.1 Copyright (c) 2000-2020 the FFmpeg developers #RUN apt-get install -y ffmpeg #RUN ffmpeg -version From ccd9a69ad71fbee9b3c8068296d8751340fca29c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 16:43:35 -0400 Subject: [PATCH 0226/1042] build with cached base image (test) let's see if this goes faster --- Dockerfile | 176 +++++++++++++++++++++++++++-------------------------- 1 file changed, 89 insertions(+), 87 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca42ff60ba..7fe26f7fa6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,94 +1,96 @@ # ==================== Python Build Base ==================== -# Authenticating with GHCR locally: -# https://docs.github.com/en/packages/working-with-a-github-packages-registry/migrating-to-the-container-registry-from-the-docker-registry -# ``` -# export CR_PAT=YOUR_TOKEN -# echo $CR_PAT | docker login ghcr.io -u USERNAME --password-stdin -# ``` +FROM ghcr.io/storytold/docker-base-images-nvidia-cuda-experimental:080f96bc7087 as pybuild-base -#FROM nvidia/cuda:10.1-devel-ubuntu18.04 as pybuild-base -FROM nvidia/cuda:11.3.0-devel-ubuntu20.04 as pybuild-base - -# tzdata install hangs without presetting the time zone -ENV TZ=UTC -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -# NB(2021-05-30): This is the image that Tacotron 2's Dockerfile specifies. -# I have not tried this, but it might be necessary. -#FROM pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 -#ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} - -# NB: https://github.com/NVIDIA/nvidia-docker/issues/864#issuecomment-439848887 -# NB: We do not install ffmpeg, since the version is 3.8.* series and we need 4.2.* +## Authenticating with GHCR locally: +## https://docs.github.com/en/packages/working-with-a-github-packages-registry/migrating-to-the-container-registry-from-the-docker-registry +## ``` +## export CR_PAT=YOUR_TOKEN +## echo $CR_PAT | docker login ghcr.io -u USERNAME --password-stdin +## ``` # -# NB(2021-05-30): We need to install `cython3` for building modern Tacotron 2 -# dependencies (ie. protoc) on old Ubuntu -# RUN apt-get update \ -# && apt-get install -y \ -RUN apt-get update -#RUN apt-cache search python3.8 - -# NB: Here we install nvidia drivers -# NB: Avoid keyboard prompt -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nvidia-384 --no-install-recommends - -# ffmpeg version 4.2.4-1ubuntu0.1 Copyright (c) 2000-2020 the FFmpeg developers -#RUN apt-get install -y ffmpeg -#RUN ffmpeg -version -#RUN sleep 100 - -# Notes on packages: -# * software-properties-common - needed for `apt-add-repository` -# * pciutils - needed for `lspci | grep -i nvidia` -RUN apt-get install -y \ - build-essential \ - curl \ - cython3 \ - ffmpeg \ - g++-7 \ - gcc-7 \ - git \ - htop \ - imagemagick \ - libc++-7-dev \ - libffi-dev \ - libgcc-7-dev \ - libmagic1 \ - libsm6 \ - libsndfile1 \ - libssl-dev \ - libxext6 \ - libxrender-dev \ - netcat \ - pciutils \ - python-dev \ - python3-pip \ - python3.8 \ - python3.8-dev \ - python3.8-venv\ - redis-tools \ - silversearcher-ag \ - software-properties-common \ - sox \ - tmux \ - vim \ - wget - - -# Wav2Lip needs Python3.6 -# NB: we install python3-virtualenv as `python3.6 -m venv` won't work anymore -RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get update -RUN apt-get install -y python3.6 -RUN apt-get install -y python3-virtualenv -RUN apt-get install -y python3.6-venv -RUN apt-get install -y python3.6-dev -#RUN apt-get install python3-virtualenv - - -RUN python3.6 --version -RUN python3.8 --version +##FROM nvidia/cuda:10.1-devel-ubuntu18.04 as pybuild-base +#FROM nvidia/cuda:11.3.0-devel-ubuntu20.04 as pybuild-base +# +## tzdata install hangs without presetting the time zone +#ENV TZ=UTC +#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +# +## NB(2021-05-30): This is the image that Tacotron 2's Dockerfile specifies. +## I have not tried this, but it might be necessary. +##FROM pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 +##ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +# +## NB: https://github.com/NVIDIA/nvidia-docker/issues/864#issuecomment-439848887 +## NB: We do not install ffmpeg, since the version is 3.8.* series and we need 4.2.* +## +## NB(2021-05-30): We need to install `cython3` for building modern Tacotron 2 +## dependencies (ie. protoc) on old Ubuntu +## RUN apt-get update \ +## && apt-get install -y \ +#RUN apt-get update +##RUN apt-cache search python3.8 +# +## NB: Here we install nvidia drivers +## NB: Avoid keyboard prompt +#RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nvidia-384 --no-install-recommends +# +## ffmpeg version 4.2.4-1ubuntu0.1 Copyright (c) 2000-2020 the FFmpeg developers +##RUN apt-get install -y ffmpeg +##RUN ffmpeg -version +##RUN sleep 100 +# +## Notes on packages: +## * software-properties-common - needed for `apt-add-repository` +## * pciutils - needed for `lspci | grep -i nvidia` +#RUN apt-get install -y \ +# build-essential \ +# curl \ +# cython3 \ +# ffmpeg \ +# g++-7 \ +# gcc-7 \ +# git \ +# htop \ +# imagemagick \ +# libc++-7-dev \ +# libffi-dev \ +# libgcc-7-dev \ +# libmagic1 \ +# libsm6 \ +# libsndfile1 \ +# libssl-dev \ +# libxext6 \ +# libxrender-dev \ +# netcat \ +# pciutils \ +# python-dev \ +# python3-pip \ +# python3.8 \ +# python3.8-dev \ +# python3.8-venv\ +# redis-tools \ +# silversearcher-ag \ +# software-properties-common \ +# sox \ +# tmux \ +# vim \ +# wget +# +# +## Wav2Lip needs Python3.6 +## NB: we install python3-virtualenv as `python3.6 -m venv` won't work anymore +#RUN add-apt-repository ppa:deadsnakes/ppa +#RUN apt-get update +#RUN apt-get install -y python3.6 +#RUN apt-get install -y python3-virtualenv +#RUN apt-get install -y python3.6-venv +#RUN apt-get install -y python3.6-dev +##RUN apt-get install python3-virtualenv +# +# +#RUN python3.6 --version +#RUN python3.8 --version # We need ffmpeg version >= 4.2 (https://superuser.com/a/579110) # http://ubuntuhandbook.org/index.php/2020/06/install-ffmpeg-4-3-via-ppa-ubuntu-18-04-16-04/ From 7bca17ad73be7935ce3ac80d574d659d342e8584 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 May 2021 16:47:54 -0400 Subject: [PATCH 0227/1042] put tacotron in the correct place --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7fe26f7fa6..9bcf4f8150 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,6 +105,8 @@ FROM ghcr.io/storytold/docker-base-images-nvidia-cuda-experimental:080f96bc7087 FROM pybuild-base as pybuild-requirements +WORKDIR / + COPY models/Wav2Lip ./models/Wav2Lip WORKDIR models/Wav2Lip @@ -116,6 +118,8 @@ RUN . python/bin/activate \ && pip install -r requirements.txt \ && deactivate +WORKDIR / + COPY models/tacotron2 ./models/tacotron2 WORKDIR models/tacotron2 From a9aa9c692f808e3dbafda5881b2d39b44684f954 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 14:42:10 -0400 Subject: [PATCH 0228/1042] improve download script --- scripts/download_gdrive.py | 58 ++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/scripts/download_gdrive.py b/scripts/download_gdrive.py index 0651a6cd9f..48d8536536 100755 --- a/scripts/download_gdrive.py +++ b/scripts/download_gdrive.py @@ -1,12 +1,19 @@ #!/usr/bin/env python3 +# For youtube-dl +from __future__ import unicode_literals + import argparse import gdown +import glob +import os import re +import urllib +import youtube_dl -parser = argparse.ArgumentParser(description='Download Google Drive files') +parser = argparse.ArgumentParser(description='Download Google Drive files, Youtube videos, or web urls') -parser.add_argument('--url', type=str, help='the google drive link') +parser.add_argument('--url', type=str, help='the web url, google drive link, or youtube url') parser.add_argument('--output_filename', type=str, help='the output filename') # This URL format gets an OAuth screen!! UGH!! @@ -14,7 +21,7 @@ parser.add_argument('--output_filename', type=str, help='the output filename') # Fixed URL: # 'https://drive.google.com/uc?id=15-tkgblTZpa0ifmvvyr_kgWy2cguTpAe' -def normalize_url(url): +def _normalize_google_drive_url(url): """ URLs must be in a certain form, or they present the user with an OAuth wall. This attempts to fix URLs in the wrong format. @@ -33,12 +40,47 @@ def normalize_url(url): return 'https://drive.google.com/uc?id={}'.format(document_token) +def download_url_or_gdrive(url, output_filename): + url = _normalize_google_drive_url(url) + print('Normalized URL: {}'.format(url)) + + gdown.download(url, output_filename, quiet=False) + +def download_youtube(youtube_url, output_filename): + + filename_expected = "ytdl_download/filename" + + ydl_opts = { + 'outtmpl': filename_expected, + 'noplaylist': True, + } + + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download([youtube_url]) + + # ytdl forcefully appends the extension + filename_search = '{}*'.format(filename_expected) + + found_filename = None + for file in glob.glob(filename_search): + found_filename = file + break + print('Downloaded as: {}'.format(found_filename)) + + os.rename(found_filename, output_filename) + args = parser.parse_args() +url = args.url +output_filename = args.output_filename -print('URL: {}'.format(args.url)) +print('URL: {}'.format(url)) +print('Output filename: {}'.format(output_filename)) -url = normalize_url(args.url) -print('Normalized URL: {}'.format(url)) - -gdown.download(url, args.output_filename, quiet=False) +parsed_url = urllib.parse.urlparse(url) +if 'youtube' in parsed_url.hostname: + print('Downloading from YouTube...') + download_youtube(url, output_filename) +else: + print('Downloading from Gdrive or Web...') + download_url_or_gdrive(url, output_filename) From 81ab8797d3edf2cd229669c2e3fd09a62ecab6a9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 14:43:18 -0400 Subject: [PATCH 0229/1042] add python's youtube-dl --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 9bcf4f8150..b01f555508 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,6 +101,9 @@ FROM ghcr.io/storytold/docker-base-images-nvidia-cuda-experimental:080f96bc7087 # && apt-get install -y ffmpeg # +# The downloader needs youtube-dl as a python library +RUN pip3 install --upgrade youtube-dl + # ==================== Python Build Step 2: Wav2Lip Requirements ==================== FROM pybuild-base as pybuild-requirements From 9b0f6badce5a0a18259cb95549988fd19a3539c5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 14:54:21 -0400 Subject: [PATCH 0230/1042] rename download script --- scripts/{download_gdrive.py => download_internet_file.py} | 4 ++-- src/script_execution/google_drive_download_command.rs | 4 ++-- src/tts_download_job.rs | 2 +- src/w2l_download_job.rs | 2 +- src/w2l_inference_job.rs | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) rename scripts/{download_gdrive.py => download_internet_file.py} (96%) diff --git a/scripts/download_gdrive.py b/scripts/download_internet_file.py similarity index 96% rename from scripts/download_gdrive.py rename to scripts/download_internet_file.py index 48d8536536..054be80aa5 100755 --- a/scripts/download_gdrive.py +++ b/scripts/download_internet_file.py @@ -13,8 +13,8 @@ import youtube_dl parser = argparse.ArgumentParser(description='Download Google Drive files, Youtube videos, or web urls') -parser.add_argument('--url', type=str, help='the web url, google drive link, or youtube url') -parser.add_argument('--output_filename', type=str, help='the output filename') +parser.add_argument('--url', type=str, help='the web url, google drive link, or youtube url', required=True) +parser.add_argument('--output_filename', type=str, help='the output filename', required=True) # This URL format gets an OAuth screen!! UGH!! # 'https://drive.google.com/file/d/15-tkgblTZpa0ifmvvyr_kgWy2cguTpAe/view?usp=sharing' diff --git a/src/script_execution/google_drive_download_command.rs b/src/script_execution/google_drive_download_command.rs index a3dd54f4da..4b4e6ac059 100644 --- a/src/script_execution/google_drive_download_command.rs +++ b/src/script_execution/google_drive_download_command.rs @@ -11,9 +11,9 @@ use tempdir::TempDir; /// We're using this because it's a hack that gets around OAuth gateways. All the Rust /// crates require OAuth permissions. Ugh. /// -/// Filename: `download_gdrive.py` +/// Filename: `download_internet_file.py` /// Arguments: -/// --url (google drive url) +/// --url (google drive, web, or youtube url) /// --output_file (local download filename) pub struct GoogleDriveDownloadCommand { command: String, diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 4999e596c6..92f7256aa8 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -95,7 +95,7 @@ async fn main() -> AnyhowResult<()> { let download_script = easyenv::get_env_string_or_default( "DOWNLOAD_SCRIPT", - "./scripts/download_gdrive.py"); + "./scripts/download_internet_file.py"); let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index b731c82efd..0b172cabfd 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -150,7 +150,7 @@ async fn main() -> AnyhowResult<()> { let download_script = easyenv::get_env_string_or_default( "DOWNLOAD_SCRIPT", - "./scripts/download_gdrive.py"); + "./scripts/download_internet_file.py"); let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 2bc7e220e1..3e91204e89 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -155,7 +155,7 @@ async fn main() -> AnyhowResult<()> { // TODO: In the future, we may want to enable downloading images or audio files. let download_script = easyenv::get_env_string_or_default( "DOWNLOAD_SCRIPT", - "./scripts/download_gdrive.py"); + "./scripts/download_internet_file.py"); let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); let temp_directory = PathBuf::from(temp_directory); From b98f250c07d8c23df9db48caed6ae84e2d572d03 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 15:45:40 -0400 Subject: [PATCH 0231/1042] debug sleep for w2l download job --- src/w2l_download_job.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 0b172cabfd..a66588505a 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -44,6 +44,7 @@ use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; use tempdir::TempDir; +use std::thread; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -87,6 +88,12 @@ struct Downloader { // Root to store W2L templates (public and private) pub bucket_root_w2l_template_uploads: String, + + // Temporary for debugging + // Arbitrary timeouts can be inserted so we can exec in and poke around. + pub debug_download_sleep_millis: u64, + pub debug_face_detect_sleep_millis: u64, + pub debug_job_end_sleep_millis: u64, } #[tokio::main] @@ -187,6 +194,9 @@ async fn main() -> AnyhowResult<()> { imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, w2l_processor: w2l_preprecess_command, bucket_root_w2l_template_uploads: bucket_root.to_string(), + debug_download_sleep_millis: easyenv::get_env_num("DEBUG_DOWNLOAD_SLEEP_MILLIS", 0)?, + debug_face_detect_sleep_millis: easyenv::get_env_num("DEBUG_FACE_DETECT_SLEEP_MILLIS", 0)?, + debug_job_end_sleep_millis: easyenv::get_env_num("DEBUG_JOB_END_SLEEP_MILLIS", 0)?, }; main_loop(downloader).await; @@ -304,6 +314,11 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) let download_filename = downloader.google_drive_downloader .download_file(&download_url, &temp_dir).await?; + if downloader.debug_download_sleep_millis != 0 { + warn!("Debug sleep after download: {} ms", downloader.debug_download_sleep_millis); + thread::sleep(Duration::from_millis(downloader.debug_download_sleep_millis)); + } + // ==================== PROCESS FACES ==================== // // This is the Python Pickle file with all the face frames. @@ -323,6 +338,11 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) is_image, spawn_process)?; + if downloader.debug_face_detect_sleep_millis != 0 { + warn!("Debug sleep after face detect: {} ms", downloader.debug_face_detect_sleep_millis); + thread::sleep(Duration::from_millis(downloader.debug_face_detect_sleep_millis)); + } + // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // let video_or_image_path = PathBuf::from(&download_filename); @@ -491,8 +511,12 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) anyhow!("error publishing event") })?; - mark_w2l_template_upload_job_done(&downloader.mysql_pool, job, true).await?; + if downloader.debug_job_end_sleep_millis != 0 { + warn!("Debug sleep after job end: {} ms", downloader.debug_job_end_sleep_millis); + thread::sleep(Duration::from_millis(downloader.debug_job_end_sleep_millis)); + } + Ok(()) } From a423cf6de92e070f691728794b6ab27d88df1ff4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 16:52:41 -0400 Subject: [PATCH 0232/1042] a bunch of alternative requirements files --- models/tacotron2/requirements-custom.txt | 26 ++++++++++++++++++ .../requirements-github-issue-2020-11-06.txt | 13 +++++++++ .../requirements-nvidia-original.txt | 9 +++++++ models/tacotron2/requirements.txt | 27 ++++--------------- 4 files changed, 53 insertions(+), 22 deletions(-) create mode 100644 models/tacotron2/requirements-custom.txt create mode 100644 models/tacotron2/requirements-github-issue-2020-11-06.txt create mode 100644 models/tacotron2/requirements-nvidia-original.txt diff --git a/models/tacotron2/requirements-custom.txt b/models/tacotron2/requirements-custom.txt new file mode 100644 index 0000000000..f3144498e1 --- /dev/null +++ b/models/tacotron2/requirements-custom.txt @@ -0,0 +1,26 @@ +#numpy +scipy +matplotlib +librosa==0.6.0 +#tensorflow +tensorboardX +inflect==0.2.5 +Unidecode==1.0.22 +pillow +jupyter + +# Note: numpy@latest doesn't build in Python<3.7, and numba causes it +numba==0.46.0 +numpy==1.14.3 + + +#torch==1.8.1 +#tensorboardX==1.1 +#tensorflow==1.13.1 +#tensorflow-estimator==1.13.0 + +# Fix for +# TypeError: create_target_machine() got an unexpected keyword argument 'jitdebug' +llvmlite==0.32.1 + +#tensorboardX==1.15 diff --git a/models/tacotron2/requirements-github-issue-2020-11-06.txt b/models/tacotron2/requirements-github-issue-2020-11-06.txt new file mode 100644 index 0000000000..514e774cbc --- /dev/null +++ b/models/tacotron2/requirements-github-issue-2020-11-06.txt @@ -0,0 +1,13 @@ +# A Github issue posted Nov 6, 2020 +# The file is unaltered from the issue contents +# https://github.com/NVIDIA/tacotron2/issues/366#issuecomment-723391198 +matplotlib==2.1.0 +tensorflow==1.15.2 +numpy==1.16.0 +numba==0.48.0 +inflect==0.2.5 +librosa==0.6.0 +scipy==1.0.0 +Unidecode==1.0.22 +pillow +future diff --git a/models/tacotron2/requirements-nvidia-original.txt b/models/tacotron2/requirements-nvidia-original.txt new file mode 100644 index 0000000000..58e16c5a32 --- /dev/null +++ b/models/tacotron2/requirements-nvidia-original.txt @@ -0,0 +1,9 @@ +matplotlib==2.1.0 +tensorflow==1.15.2 +numpy==1.13.3 +inflect==0.2.5 +librosa==0.6.0 +scipy==1.0.0 +Unidecode==1.0.22 +pillow + diff --git a/models/tacotron2/requirements.txt b/models/tacotron2/requirements.txt index f3144498e1..58e16c5a32 100644 --- a/models/tacotron2/requirements.txt +++ b/models/tacotron2/requirements.txt @@ -1,26 +1,9 @@ -#numpy -scipy -matplotlib -librosa==0.6.0 -#tensorflow -tensorboardX +matplotlib==2.1.0 +tensorflow==1.15.2 +numpy==1.13.3 inflect==0.2.5 +librosa==0.6.0 +scipy==1.0.0 Unidecode==1.0.22 pillow -jupyter -# Note: numpy@latest doesn't build in Python<3.7, and numba causes it -numba==0.46.0 -numpy==1.14.3 - - -#torch==1.8.1 -#tensorboardX==1.1 -#tensorflow==1.13.1 -#tensorflow-estimator==1.13.0 - -# Fix for -# TypeError: create_target_machine() got an unexpected keyword argument 'jitdebug' -llvmlite==0.32.1 - -#tensorboardX==1.15 From 21c2194828b6437436cb8f0f6a0014cd370e4bad Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 16:53:25 -0400 Subject: [PATCH 0233/1042] additional comment --- models/tacotron2/requirements-github-issue-2020-11-06.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/tacotron2/requirements-github-issue-2020-11-06.txt b/models/tacotron2/requirements-github-issue-2020-11-06.txt index 514e774cbc..413de89bff 100644 --- a/models/tacotron2/requirements-github-issue-2020-11-06.txt +++ b/models/tacotron2/requirements-github-issue-2020-11-06.txt @@ -1,6 +1,8 @@ # A Github issue posted Nov 6, 2020 # The file is unaltered from the issue contents +# This is purported to work under "python 3.6" # https://github.com/NVIDIA/tacotron2/issues/366#issuecomment-723391198 + matplotlib==2.1.0 tensorflow==1.15.2 numpy==1.16.0 From 14f1fd327cb6a05ae93d9a35e54e09a737597f88 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 17:21:55 -0400 Subject: [PATCH 0234/1042] requirements from halide that seem to be almost working --- .../requirements-halide-python36.txt | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 models/tacotron2/requirements-halide-python36.txt diff --git a/models/tacotron2/requirements-halide-python36.txt b/models/tacotron2/requirements-halide-python36.txt new file mode 100644 index 0000000000..091352bc0e --- /dev/null +++ b/models/tacotron2/requirements-halide-python36.txt @@ -0,0 +1,44 @@ +absl-py==0.12.0 +astor==0.8.1 +audioread==2.1.9 +cached-property==1.5.2 +cycler==0.10.0 +dataclasses==0.8 +decorator==5.0.9 +future==0.18.2 +gast==0.2.2 +google-pasta==0.2.0 +grpcio==1.38.0 +h5py==3.1.0 +importlib-metadata==4.4.0 +inflect==0.2.5 +joblib==1.0.1 +Keras-Applications==1.0.8 +Keras-Preprocessing==1.1.2 +librosa==0.6.0 +llvmlite==0.31.0 +Markdown==3.3.4 +matplotlib==2.1.0 +numba==0.48.0 +numpy==1.16.0 +opt-einsum==3.3.0 +Pillow==8.2.0 +protobuf==3.17.1 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +resampy==0.2.2 +scikit-learn==0.24.2 +scipy==1.0.0 +six==1.16.0 +tensorboard==1.15.0 +tensorflow==1.15.2 +tensorflow-estimator==1.15.1 +termcolor==1.1.0 +threadpoolctl==2.1.0 +torch==1.8.1 +typing-extensions==3.10.0.0 +Unidecode==1.0.22 +Werkzeug==2.0.1 +wrapt==1.12.1 +zipp==3.4.1 From 34940f75b128965730e905392b6fecc79406c979 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 17:26:32 -0400 Subject: [PATCH 0235/1042] inference wip --- models/tacotron2/inference.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/models/tacotron2/inference.py b/models/tacotron2/inference.py index 76b708a50e..8e79ba9f55 100755 --- a/models/tacotron2/inference.py +++ b/models/tacotron2/inference.py @@ -39,13 +39,15 @@ torch.set_default_tensor_type('torch.DoubleTensor') # 4 checkpoint_path = "/home/bt/models/tacotron2-nvidia/tacotron2_statedict.pt" #checkpoint_path = "/home/bt/models/tigger_tt_model.pt" -checkpoint_path = "/home/bt/models/uber_test_2.pt" +#checkpoint_path = "/home/bt/models/uber_test_2.pt" +checkpoint_path = "/home/bt/models/tacotron2/tacotron2_uberduck_JorgenVonStrangle" model = load_model(hparams) -model.load_state_dict(torch.load(checkpoint_path, map_location=torch.device('cpu'))['state_dict']) +#model.load_state_dict(torch.load(checkpoint_path, map_location=torch.device('cpu'))['state_dict']) +model.load_state_dict(torch.load(checkpoint_path)['state_dict']) #model.load_state_dict(torch.jit.load(checkpoint_path, map_location=torch.device('cpu'))['state_dict']) #_ = model.cuda().eval().half() #_ = model.eval().half() -_ = model.eval() +_ = model.cuda().eval() # 5 #waveglow_path = 'waveglow_256channels.pt' From 04bdfa77fbecd1c7ad899aba7612dfbbf6939fd7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 17:33:09 -0400 Subject: [PATCH 0236/1042] spectrogram inference works ! --- models/tacotron2/inference.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/models/tacotron2/inference.py b/models/tacotron2/inference.py index 8e79ba9f55..3cd2084fee 100755 --- a/models/tacotron2/inference.py +++ b/models/tacotron2/inference.py @@ -19,6 +19,9 @@ from train import load_model from text import text_to_sequence #from denoiser import Denoiser +# NB(bt, 2021-05-31): Trying to get everything on the same device +torch.device("cuda" if torch.cuda.is_available() else "cpu") + #from wav_images import render_histogram # 2 @@ -47,6 +50,13 @@ model.load_state_dict(torch.load(checkpoint_path)['state_dict']) #model.load_state_dict(torch.jit.load(checkpoint_path, map_location=torch.device('cpu'))['state_dict']) #_ = model.cuda().eval().half() #_ = model.eval().half() + + +# NB(bt, 2021-05-31): Trying to get everything on the same device +device = "cuda:0" +model = model.to(device) + + _ = model.cuda().eval() # 5 @@ -79,11 +89,16 @@ sequence = torch.autograd.Variable(sequence).long() print("Sequence torch autograd: {}".format(sequence)) -print('Saving text sequence tensor') -torch.save(sequence, 'text_sequence.pt') +# NB(bt, 2021-05-31): This works. Don't need to do it. +#print('Saving text sequence tensor') +#torch.save(sequence, 'text_sequence.pt') print("\n\n===END TEXT===\n") + +# NB(bt, 2021-05-31): Trying to get everything to the same device +sequence = sequence.to(device) + # 7 mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence) #plot_data((mel_outputs.float().data.cpu().numpy()[0], From 49dde0a2ff5a78b5615e6970ff7069bc4b7fbad9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 17:36:43 -0400 Subject: [PATCH 0237/1042] Add utility from my old experimentation (details) ~/dev/3rd/nvidia-tacotron2 commit f6be28d1f89b5e0bae89a771c5d116722f8661cb Author: Brandon Thomas Date: Sat Jan 11 01:24:26 2020 -0500 Running on CPU, also wav images --- models/tacotron2/wav_images.py | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 models/tacotron2/wav_images.py diff --git a/models/tacotron2/wav_images.py b/models/tacotron2/wav_images.py new file mode 100644 index 0000000000..ec5949805d --- /dev/null +++ b/models/tacotron2/wav_images.py @@ -0,0 +1,49 @@ +from PIL import Image, ImageDraw + +def render_histogram(wav, filename='histograms/output.png', row_index=None, col_index=None): + dimensions = wav.shape + print('>>> Tensor Image input size: {}'.format(dimensions)) + + wav_signal = None + rows = -1 + cols = -1 + if row_index is not None and col_index is not None: + rows = dimensions[row_index] + cols = dimensions[col_index] + wav_signal = wav + else: + # Remove 1-sized dimensions + wav_signal = wav.squeeze() + rows = wav_signal.shape[0] + cols = wav_signal.shape[1] + + print('>>> Generating image (rows: {} cols: {})'.format(rows, cols)) + image = Image.new('RGB', (rows, cols)) + pixels = image.load() + + minimum = wav_signal[0,0] + maximum = wav_signal[0,0] + + for x in range(0, rows): + for y in range(0, cols): + if wav_signal[x,y] > maximum: + maximum = wav_signal[x,y] + if wav_signal[x,y] < minimum: + minimum = wav_signal[x,y] + + print('Minimum: ' + str(minimum)) + print('Maximum: ' + str(maximum)) + + if maximum == minimum: + maximum = 1 # nb: to prevent division by zero below + + for x in range(0, rows): + for y in range(0, cols): + v = wav_signal[x,y] + scaled = int((v - minimum) / (maximum - minimum) * 255) + pixels[x, y] = (scaled, scaled, scaled) + + # Image Show only works on local X server: + #image.show() + image.save(filename) + From 79f7c6c82da7115942af4b03e3920fd4a2322d36 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 17:38:56 -0400 Subject: [PATCH 0238/1042] mel histograms look good! (this is just a test) --- models/tacotron2/inference.py | 6 +++--- models/tacotron2/{wav_images.py => vocodes_wav_images.py} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename models/tacotron2/{wav_images.py => vocodes_wav_images.py} (100%) diff --git a/models/tacotron2/inference.py b/models/tacotron2/inference.py index 3cd2084fee..abe6f3d5b8 100755 --- a/models/tacotron2/inference.py +++ b/models/tacotron2/inference.py @@ -22,7 +22,7 @@ from text import text_to_sequence # NB(bt, 2021-05-31): Trying to get everything on the same device torch.device("cuda" if torch.cuda.is_available() else "cpu") -#from wav_images import render_histogram +from vocodes_wav_images import render_histogram # 2 def plot_data(data, figsize=(16, 4)): @@ -118,8 +118,8 @@ torch.save(mel_outputs, 'mel_outputs.mel') torch.save(mel_outputs_postnet, 'mel_outputs_postnet.mel') #print('Rendering histograms') -#render_histogram(mel_outputs, 'mel_outputs.png') -#render_histogram(mel_outputs_postnet, 'mel_outputs_postnet.png') +render_histogram(mel_outputs, 'mel_outputs.png') +render_histogram(mel_outputs_postnet, 'mel_outputs_postnet.png') # 8 with torch.no_grad(): diff --git a/models/tacotron2/wav_images.py b/models/tacotron2/vocodes_wav_images.py similarity index 100% rename from models/tacotron2/wav_images.py rename to models/tacotron2/vocodes_wav_images.py From f23ceb22bfa11a516a5241b3f80cf3a61a21aeb8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 17:53:42 -0400 Subject: [PATCH 0239/1042] add waveglow @ master --- models/tacotron2/waveglow/.gitmodules | 3 + models/tacotron2/waveglow/LICENSE | 29 ++ models/tacotron2/waveglow/README.md | 91 ++++++ models/tacotron2/waveglow/config.json | 41 +++ models/tacotron2/waveglow/convert_model.py | 75 +++++ models/tacotron2/waveglow/denoiser.py | 40 +++ models/tacotron2/waveglow/distributed.py | 184 ++++++++++++ models/tacotron2/waveglow/glow.py | 311 ++++++++++++++++++++ models/tacotron2/waveglow/glow_old.py | 233 +++++++++++++++ models/tacotron2/waveglow/inference.py | 84 ++++++ models/tacotron2/waveglow/mel2samp.py | 142 +++++++++ models/tacotron2/waveglow/requirements.txt | 10 + models/tacotron2/waveglow/train.py | 188 ++++++++++++ models/tacotron2/waveglow/waveglow_logo.png | Bin 0 -> 391167 bytes 14 files changed, 1431 insertions(+) create mode 100644 models/tacotron2/waveglow/.gitmodules create mode 100644 models/tacotron2/waveglow/LICENSE create mode 100755 models/tacotron2/waveglow/README.md create mode 100755 models/tacotron2/waveglow/config.json create mode 100644 models/tacotron2/waveglow/convert_model.py create mode 100644 models/tacotron2/waveglow/denoiser.py create mode 100644 models/tacotron2/waveglow/distributed.py create mode 100644 models/tacotron2/waveglow/glow.py create mode 100644 models/tacotron2/waveglow/glow_old.py create mode 100644 models/tacotron2/waveglow/inference.py create mode 100644 models/tacotron2/waveglow/mel2samp.py create mode 100644 models/tacotron2/waveglow/requirements.txt create mode 100644 models/tacotron2/waveglow/train.py create mode 100755 models/tacotron2/waveglow/waveglow_logo.png diff --git a/models/tacotron2/waveglow/.gitmodules b/models/tacotron2/waveglow/.gitmodules new file mode 100644 index 0000000000..e3bf8bf069 --- /dev/null +++ b/models/tacotron2/waveglow/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tacotron2"] + path = tacotron2 + url = http://github.com/NVIDIA/tacotron2 diff --git a/models/tacotron2/waveglow/LICENSE b/models/tacotron2/waveglow/LICENSE new file mode 100644 index 0000000000..2a718d63da --- /dev/null +++ b/models/tacotron2/waveglow/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2018, NVIDIA Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/models/tacotron2/waveglow/README.md b/models/tacotron2/waveglow/README.md new file mode 100755 index 0000000000..b94648cf8e --- /dev/null +++ b/models/tacotron2/waveglow/README.md @@ -0,0 +1,91 @@ +![WaveGlow](waveglow_logo.png "WaveGLow") + +## WaveGlow: a Flow-based Generative Network for Speech Synthesis + +### Ryan Prenger, Rafael Valle, and Bryan Catanzaro + +In our recent [paper], we propose WaveGlow: a flow-based network capable of +generating high quality speech from mel-spectrograms. WaveGlow combines insights +from [Glow] and [WaveNet] in order to provide fast, efficient and high-quality +audio synthesis, without the need for auto-regression. WaveGlow is implemented +using only a single network, trained using only a single cost function: +maximizing the likelihood of the training data, which makes the training +procedure simple and stable. + +Our [PyTorch] implementation produces audio samples at a rate of 1200 +kHz on an NVIDIA V100 GPU. Mean Opinion Scores show that it delivers audio +quality as good as the best publicly available WaveNet implementation. + +Visit our [website] for audio samples. + +## Setup + +1. Clone our repo and initialize submodule + + ```command + git clone https://github.com/NVIDIA/waveglow.git + cd waveglow + git submodule init + git submodule update + ``` + +2. Install requirements `pip3 install -r requirements.txt` + +3. Install [Apex] + + +## Generate audio with our pre-existing model + +1. Download our [published model] +2. Download [mel-spectrograms] +3. Generate audio `python3 inference.py -f <(ls mel_spectrograms/*.pt) -w waveglow_256channels.pt -o . --is_fp16 -s 0.6` + +N.b. use `convert_model.py` to convert your older models to the current model +with fused residual and skip connections. + +## Train your own model + +1. Download [LJ Speech Data]. In this example it's in `data/` + +2. Make a list of the file names to use for training/testing + + ```command + ls data/*.wav | tail -n+10 > train_files.txt + ls data/*.wav | head -n10 > test_files.txt + ``` + +3. Train your WaveGlow networks + + ```command + mkdir checkpoints + python train.py -c config.json + ``` + + For multi-GPU training replace `train.py` with `distributed.py`. Only tested with single node and NCCL. + + For mixed precision training set `"fp16_run": true` on `config.json`. + +4. Make test set mel-spectrograms + + `python mel2samp.py -f test_files.txt -o . -c config.json` + +5. Do inference with your network + + ```command + ls *.pt > mel_files.txt + python3 inference.py -f mel_files.txt -w checkpoints/waveglow_10000 -o . --is_fp16 -s 0.6 + ``` + +[//]: # (TODO) +[//]: # (PROVIDE INSTRUCTIONS FOR DOWNLOADING LJS) +[pytorch 1.0]: https://github.com/pytorch/pytorch#installation +[website]: https://nv-adlr.github.io/WaveGlow +[paper]: https://arxiv.org/abs/1811.00002 +[WaveNet implementation]: https://github.com/r9y9/wavenet_vocoder +[Glow]: https://blog.openai.com/glow/ +[WaveNet]: https://deepmind.com/blog/wavenet-generative-model-raw-audio/ +[PyTorch]: http://pytorch.org +[published model]: https://drive.google.com/open?id=1rpK8CzAAirq9sWZhe9nlfvxMF1dRgFbF +[mel-spectrograms]: https://drive.google.com/file/d/1g_VXK2lpP9J25dQFhQwx7doWl_p20fXA/view?usp=sharing +[LJ Speech Data]: https://keithito.com/LJ-Speech-Dataset +[Apex]: https://github.com/nvidia/apex diff --git a/models/tacotron2/waveglow/config.json b/models/tacotron2/waveglow/config.json new file mode 100755 index 0000000000..7160474748 --- /dev/null +++ b/models/tacotron2/waveglow/config.json @@ -0,0 +1,41 @@ +{ + "train_config": { + "fp16_run": true, + "output_directory": "checkpoints", + "epochs": 100000, + "learning_rate": 1e-4, + "sigma": 1.0, + "iters_per_checkpoint": 2000, + "batch_size": 12, + "seed": 1234, + "checkpoint_path": "", + "with_tensorboard": false + }, + "data_config": { + "training_files": "train_files.txt", + "segment_length": 16000, + "sampling_rate": 22050, + "filter_length": 1024, + "hop_length": 256, + "win_length": 1024, + "mel_fmin": 0.0, + "mel_fmax": 8000.0 + }, + "dist_config": { + "dist_backend": "nccl", + "dist_url": "tcp://localhost:54321" + }, + + "waveglow_config": { + "n_mel_channels": 80, + "n_flows": 12, + "n_group": 8, + "n_early_every": 4, + "n_early_size": 2, + "WN_config": { + "n_layers": 8, + "n_channels": 256, + "kernel_size": 3 + } + } +} diff --git a/models/tacotron2/waveglow/convert_model.py b/models/tacotron2/waveglow/convert_model.py new file mode 100644 index 0000000000..6fdc41e871 --- /dev/null +++ b/models/tacotron2/waveglow/convert_model.py @@ -0,0 +1,75 @@ +import sys +import copy +import torch + +def _check_model_old_version(model): + if hasattr(model.WN[0], 'res_layers') or hasattr(model.WN[0], 'cond_layers'): + return True + else: + return False + + +def _update_model_res_skip(old_model, new_model): + for idx in range(0, len(new_model.WN)): + wavenet = new_model.WN[idx] + n_channels = wavenet.n_channels + n_layers = wavenet.n_layers + wavenet.res_skip_layers = torch.nn.ModuleList() + for i in range(0, n_layers): + if i < n_layers - 1: + res_skip_channels = 2*n_channels + else: + res_skip_channels = n_channels + res_skip_layer = torch.nn.Conv1d(n_channels, res_skip_channels, 1) + skip_layer = torch.nn.utils.remove_weight_norm(wavenet.skip_layers[i]) + if i < n_layers - 1: + res_layer = torch.nn.utils.remove_weight_norm(wavenet.res_layers[i]) + res_skip_layer.weight = torch.nn.Parameter(torch.cat([res_layer.weight, skip_layer.weight])) + res_skip_layer.bias = torch.nn.Parameter(torch.cat([res_layer.bias, skip_layer.bias])) + else: + res_skip_layer.weight = torch.nn.Parameter(skip_layer.weight) + res_skip_layer.bias = torch.nn.Parameter(skip_layer.bias) + res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight') + wavenet.res_skip_layers.append(res_skip_layer) + del wavenet.res_layers + del wavenet.skip_layers + +def _update_model_cond(old_model, new_model): + for idx in range(0, len(new_model.WN)): + wavenet = new_model.WN[idx] + n_channels = wavenet.n_channels + n_layers = wavenet.n_layers + n_mel_channels = wavenet.cond_layers[0].weight.shape[1] + cond_layer = torch.nn.Conv1d(n_mel_channels, 2*n_channels*n_layers, 1) + cond_layer_weight = [] + cond_layer_bias = [] + for i in range(0, n_layers): + _cond_layer = torch.nn.utils.remove_weight_norm(wavenet.cond_layers[i]) + cond_layer_weight.append(_cond_layer.weight) + cond_layer_bias.append(_cond_layer.bias) + cond_layer.weight = torch.nn.Parameter(torch.cat(cond_layer_weight)) + cond_layer.bias = torch.nn.Parameter(torch.cat(cond_layer_bias)) + cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') + wavenet.cond_layer = cond_layer + del wavenet.cond_layers + +def update_model(old_model): + if not _check_model_old_version(old_model): + return old_model + new_model = copy.deepcopy(old_model) + if hasattr(old_model.WN[0], 'res_layers'): + _update_model_res_skip(old_model, new_model) + if hasattr(old_model.WN[0], 'cond_layers'): + _update_model_cond(old_model, new_model) + for m in new_model.modules(): + if 'Conv' in str(type(m)) and not hasattr(m, 'padding_mode'): + setattr(m, 'padding_mode', 'zeros') + return new_model + +if __name__ == '__main__': + old_model_path = sys.argv[1] + new_model_path = sys.argv[2] + model = torch.load(old_model_path, map_location='cpu') + model['model'] = update_model(model['model']) + torch.save(model, new_model_path) + diff --git a/models/tacotron2/waveglow/denoiser.py b/models/tacotron2/waveglow/denoiser.py new file mode 100644 index 0000000000..0e73584d54 --- /dev/null +++ b/models/tacotron2/waveglow/denoiser.py @@ -0,0 +1,40 @@ +import sys +sys.path.append('tacotron2') +import torch +from layers import STFT + + +class Denoiser(torch.nn.Module): + """ Removes model bias from audio produced with waveglow """ + + def __init__(self, waveglow, filter_length=1024, n_overlap=4, + win_length=1024, mode='zeros'): + super(Denoiser, self).__init__() + self.stft = STFT(filter_length=filter_length, + hop_length=int(filter_length/n_overlap), + win_length=win_length).cuda() + if mode == 'zeros': + mel_input = torch.zeros( + (1, 80, 88), + dtype=waveglow.upsample.weight.dtype, + device=waveglow.upsample.weight.device) + elif mode == 'normal': + mel_input = torch.randn( + (1, 80, 88), + dtype=waveglow.upsample.weight.dtype, + device=waveglow.upsample.weight.device) + else: + raise Exception("Mode {} if not supported".format(mode)) + + with torch.no_grad(): + bias_audio = waveglow.infer(mel_input, sigma=0.0).float() + bias_spec, _ = self.stft.transform(bias_audio) + + self.register_buffer('bias_spec', bias_spec[:, :, 0][:, :, None]) + + def forward(self, audio, strength=0.1): + audio_spec, audio_angles = self.stft.transform(audio.cuda().float()) + audio_spec_denoised = audio_spec - self.bias_spec * strength + audio_spec_denoised = torch.clamp(audio_spec_denoised, 0.0) + audio_denoised = self.stft.inverse(audio_spec_denoised, audio_angles) + return audio_denoised diff --git a/models/tacotron2/waveglow/distributed.py b/models/tacotron2/waveglow/distributed.py new file mode 100644 index 0000000000..19cbfd2cca --- /dev/null +++ b/models/tacotron2/waveglow/distributed.py @@ -0,0 +1,184 @@ +# ***************************************************************************** +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the NVIDIA CORPORATION nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# ***************************************************************************** +import os +import sys +import time +import subprocess +import argparse + +import torch +import torch.distributed as dist +from torch.autograd import Variable + +def reduce_tensor(tensor, num_gpus): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.reduce_op.SUM) + rt /= num_gpus + return rt + +def init_distributed(rank, num_gpus, group_name, dist_backend, dist_url): + assert torch.cuda.is_available(), "Distributed mode requires CUDA." + print("Initializing Distributed") + + # Set cuda device so everything is done on the right GPU. + torch.cuda.set_device(rank % torch.cuda.device_count()) + + # Initialize distributed communication + dist.init_process_group(dist_backend, init_method=dist_url, + world_size=num_gpus, rank=rank, + group_name=group_name) + +def _flatten_dense_tensors(tensors): + """Flatten dense tensors into a contiguous 1D buffer. Assume tensors are of + same dense type. + Since inputs are dense, the resulting tensor will be a concatenated 1D + buffer. Element-wise operation on this buffer will be equivalent to + operating individually. + Arguments: + tensors (Iterable[Tensor]): dense tensors to flatten. + Returns: + A contiguous 1D buffer containing input tensors. + """ + if len(tensors) == 1: + return tensors[0].contiguous().view(-1) + flat = torch.cat([t.contiguous().view(-1) for t in tensors], dim=0) + return flat + +def _unflatten_dense_tensors(flat, tensors): + """View a flat buffer using the sizes of tensors. Assume that tensors are of + same dense type, and that flat is given by _flatten_dense_tensors. + Arguments: + flat (Tensor): flattened dense tensors to unflatten. + tensors (Iterable[Tensor]): dense tensors whose sizes will be used to + unflatten flat. + Returns: + Unflattened dense tensors with sizes same as tensors and values from + flat. + """ + outputs = [] + offset = 0 + for tensor in tensors: + numel = tensor.numel() + outputs.append(flat.narrow(0, offset, numel).view_as(tensor)) + offset += numel + return tuple(outputs) + +def apply_gradient_allreduce(module): + """ + Modifies existing model to do gradient allreduce, but doesn't change class + so you don't need "module" + """ + if not hasattr(dist, '_backend'): + module.warn_on_half = True + else: + module.warn_on_half = True if dist._backend == dist.dist_backend.GLOO else False + + for p in module.state_dict().values(): + if not torch.is_tensor(p): + continue + dist.broadcast(p, 0) + + def allreduce_params(): + if(module.needs_reduction): + module.needs_reduction = False + buckets = {} + for param in module.parameters(): + if param.requires_grad and param.grad is not None: + tp = type(param.data) + if tp not in buckets: + buckets[tp] = [] + buckets[tp].append(param) + if module.warn_on_half: + if torch.cuda.HalfTensor in buckets: + print("WARNING: gloo dist backend for half parameters may be extremely slow." + + " It is recommended to use the NCCL backend in this case. This currently requires" + + "PyTorch built from top of tree master.") + module.warn_on_half = False + + for tp in buckets: + bucket = buckets[tp] + grads = [param.grad.data for param in bucket] + coalesced = _flatten_dense_tensors(grads) + dist.all_reduce(coalesced) + coalesced /= dist.get_world_size() + for buf, synced in zip(grads, _unflatten_dense_tensors(coalesced, grads)): + buf.copy_(synced) + + for param in list(module.parameters()): + def allreduce_hook(*unused): + Variable._execution_engine.queue_callback(allreduce_params) + if param.requires_grad: + param.register_hook(allreduce_hook) + dir(param) + + def set_needs_reduction(self, input, output): + self.needs_reduction = True + + module.register_forward_hook(set_needs_reduction) + return module + + +def main(config, stdout_dir, args_str): + args_list = ['train.py'] + args_list += args_str.split(' ') if len(args_str) > 0 else [] + + args_list.append('--config={}'.format(config)) + + num_gpus = torch.cuda.device_count() + args_list.append('--num_gpus={}'.format(num_gpus)) + args_list.append("--group_name=group_{}".format(time.strftime("%Y_%m_%d-%H%M%S"))) + + if not os.path.isdir(stdout_dir): + os.makedirs(stdout_dir) + os.chmod(stdout_dir, 0o775) + + workers = [] + + for i in range(num_gpus): + args_list[-2] = '--rank={}'.format(i) + stdout = None if i == 0 else open( + os.path.join(stdout_dir, "GPU_{}.log".format(i)), "w") + print(args_list) + p = subprocess.Popen([str(sys.executable)]+args_list, stdout=stdout) + workers.append(p) + + for p in workers: + p.wait() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, required=True, + help='JSON file for configuration') + parser.add_argument('-s', '--stdout_dir', type=str, default=".", + help='directory to save stoud logs') + parser.add_argument( + '-a', '--args_str', type=str, default='', + help='double quoted string with space separated key value pairs') + + args = parser.parse_args() + main(args.config, args.stdout_dir, args.args_str) diff --git a/models/tacotron2/waveglow/glow.py b/models/tacotron2/waveglow/glow.py new file mode 100644 index 0000000000..7a7696403d --- /dev/null +++ b/models/tacotron2/waveglow/glow.py @@ -0,0 +1,311 @@ +# ***************************************************************************** +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the NVIDIA CORPORATION nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# ***************************************************************************** +import copy +import torch +from torch.autograd import Variable +import torch.nn.functional as F + + +@torch.jit.script +def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): + n_channels_int = n_channels[0] + in_act = input_a+input_b + t_act = torch.tanh(in_act[:, :n_channels_int, :]) + s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) + acts = t_act * s_act + return acts + + +class WaveGlowLoss(torch.nn.Module): + def __init__(self, sigma=1.0): + super(WaveGlowLoss, self).__init__() + self.sigma = sigma + + def forward(self, model_output): + z, log_s_list, log_det_W_list = model_output + for i, log_s in enumerate(log_s_list): + if i == 0: + log_s_total = torch.sum(log_s) + log_det_W_total = log_det_W_list[i] + else: + log_s_total = log_s_total + torch.sum(log_s) + log_det_W_total += log_det_W_list[i] + + loss = torch.sum(z*z)/(2*self.sigma*self.sigma) - log_s_total - log_det_W_total + return loss/(z.size(0)*z.size(1)*z.size(2)) + + +class Invertible1x1Conv(torch.nn.Module): + """ + The layer outputs both the convolution, and the log determinant + of its weight matrix. If reverse=True it does convolution with + inverse + """ + def __init__(self, c): + super(Invertible1x1Conv, self).__init__() + self.conv = torch.nn.Conv1d(c, c, kernel_size=1, stride=1, padding=0, + bias=False) + + # Sample a random orthonormal matrix to initialize weights + W = torch.qr(torch.FloatTensor(c, c).normal_())[0] + + # Ensure determinant is 1.0 not -1.0 + if torch.det(W) < 0: + W[:,0] = -1*W[:,0] + W = W.view(c, c, 1) + self.conv.weight.data = W + + def forward(self, z, reverse=False): + # shape + batch_size, group_size, n_of_groups = z.size() + + W = self.conv.weight.squeeze() + + if reverse: + if not hasattr(self, 'W_inverse'): + # Reverse computation + W_inverse = W.float().inverse() + W_inverse = Variable(W_inverse[..., None]) + if z.type() == 'torch.cuda.HalfTensor': + W_inverse = W_inverse.half() + self.W_inverse = W_inverse + z = F.conv1d(z, self.W_inverse, bias=None, stride=1, padding=0) + return z + else: + # Forward computation + log_det_W = batch_size * n_of_groups * torch.logdet(W) + z = self.conv(z) + return z, log_det_W + + +class WN(torch.nn.Module): + """ + This is the WaveNet like layer for the affine coupling. The primary difference + from WaveNet is the convolutions need not be causal. There is also no dilation + size reset. The dilation only doubles on each layer + """ + def __init__(self, n_in_channels, n_mel_channels, n_layers, n_channels, + kernel_size): + super(WN, self).__init__() + assert(kernel_size % 2 == 1) + assert(n_channels % 2 == 0) + self.n_layers = n_layers + self.n_channels = n_channels + self.in_layers = torch.nn.ModuleList() + self.res_skip_layers = torch.nn.ModuleList() + + start = torch.nn.Conv1d(n_in_channels, n_channels, 1) + start = torch.nn.utils.weight_norm(start, name='weight') + self.start = start + + # Initializing last layer to 0 makes the affine coupling layers + # do nothing at first. This helps with training stability + end = torch.nn.Conv1d(n_channels, 2*n_in_channels, 1) + end.weight.data.zero_() + end.bias.data.zero_() + self.end = end + + cond_layer = torch.nn.Conv1d(n_mel_channels, 2*n_channels*n_layers, 1) + self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') + + for i in range(n_layers): + dilation = 2 ** i + padding = int((kernel_size*dilation - dilation)/2) + in_layer = torch.nn.Conv1d(n_channels, 2*n_channels, kernel_size, + dilation=dilation, padding=padding) + in_layer = torch.nn.utils.weight_norm(in_layer, name='weight') + self.in_layers.append(in_layer) + + + # last one is not necessary + if i < n_layers - 1: + res_skip_channels = 2*n_channels + else: + res_skip_channels = n_channels + res_skip_layer = torch.nn.Conv1d(n_channels, res_skip_channels, 1) + res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight') + self.res_skip_layers.append(res_skip_layer) + + def forward(self, forward_input): + audio, spect = forward_input + audio = self.start(audio) + output = torch.zeros_like(audio) + n_channels_tensor = torch.IntTensor([self.n_channels]) + + spect = self.cond_layer(spect) + + for i in range(self.n_layers): + spect_offset = i*2*self.n_channels + acts = fused_add_tanh_sigmoid_multiply( + self.in_layers[i](audio), + spect[:,spect_offset:spect_offset+2*self.n_channels,:], + n_channels_tensor) + + res_skip_acts = self.res_skip_layers[i](acts) + if i < self.n_layers - 1: + audio = audio + res_skip_acts[:,:self.n_channels,:] + output = output + res_skip_acts[:,self.n_channels:,:] + else: + output = output + res_skip_acts + + return self.end(output) + + +class WaveGlow(torch.nn.Module): + def __init__(self, n_mel_channels, n_flows, n_group, n_early_every, + n_early_size, WN_config): + super(WaveGlow, self).__init__() + + self.upsample = torch.nn.ConvTranspose1d(n_mel_channels, + n_mel_channels, + 1024, stride=256) + assert(n_group % 2 == 0) + self.n_flows = n_flows + self.n_group = n_group + self.n_early_every = n_early_every + self.n_early_size = n_early_size + self.WN = torch.nn.ModuleList() + self.convinv = torch.nn.ModuleList() + + n_half = int(n_group/2) + + # Set up layers with the right sizes based on how many dimensions + # have been output already + n_remaining_channels = n_group + for k in range(n_flows): + if k % self.n_early_every == 0 and k > 0: + n_half = n_half - int(self.n_early_size/2) + n_remaining_channels = n_remaining_channels - self.n_early_size + self.convinv.append(Invertible1x1Conv(n_remaining_channels)) + self.WN.append(WN(n_half, n_mel_channels*n_group, **WN_config)) + self.n_remaining_channels = n_remaining_channels # Useful during inference + + def forward(self, forward_input): + """ + forward_input[0] = mel_spectrogram: batch x n_mel_channels x frames + forward_input[1] = audio: batch x time + """ + spect, audio = forward_input + + # Upsample spectrogram to size of audio + spect = self.upsample(spect) + assert(spect.size(2) >= audio.size(1)) + if spect.size(2) > audio.size(1): + spect = spect[:, :, :audio.size(1)] + + spect = spect.unfold(2, self.n_group, self.n_group).permute(0, 2, 1, 3) + spect = spect.contiguous().view(spect.size(0), spect.size(1), -1).permute(0, 2, 1) + + audio = audio.unfold(1, self.n_group, self.n_group).permute(0, 2, 1) + output_audio = [] + log_s_list = [] + log_det_W_list = [] + + for k in range(self.n_flows): + if k % self.n_early_every == 0 and k > 0: + output_audio.append(audio[:,:self.n_early_size,:]) + audio = audio[:,self.n_early_size:,:] + + audio, log_det_W = self.convinv[k](audio) + log_det_W_list.append(log_det_W) + + n_half = int(audio.size(1)/2) + audio_0 = audio[:,:n_half,:] + audio_1 = audio[:,n_half:,:] + + output = self.WN[k]((audio_0, spect)) + log_s = output[:, n_half:, :] + b = output[:, :n_half, :] + audio_1 = torch.exp(log_s)*audio_1 + b + log_s_list.append(log_s) + + audio = torch.cat([audio_0, audio_1],1) + + output_audio.append(audio) + return torch.cat(output_audio,1), log_s_list, log_det_W_list + + def infer(self, spect, sigma=1.0): + spect = self.upsample(spect) + # trim conv artifacts. maybe pad spec to kernel multiple + time_cutoff = self.upsample.kernel_size[0] - self.upsample.stride[0] + spect = spect[:, :, :-time_cutoff] + + spect = spect.unfold(2, self.n_group, self.n_group).permute(0, 2, 1, 3) + spect = spect.contiguous().view(spect.size(0), spect.size(1), -1).permute(0, 2, 1) + + if spect.type() == 'torch.cuda.HalfTensor': + audio = torch.cuda.HalfTensor(spect.size(0), + self.n_remaining_channels, + spect.size(2)).normal_() + else: + audio = torch.cuda.FloatTensor(spect.size(0), + self.n_remaining_channels, + spect.size(2)).normal_() + + audio = torch.autograd.Variable(sigma*audio) + + for k in reversed(range(self.n_flows)): + n_half = int(audio.size(1)/2) + audio_0 = audio[:,:n_half,:] + audio_1 = audio[:,n_half:,:] + + output = self.WN[k]((audio_0, spect)) + + s = output[:, n_half:, :] + b = output[:, :n_half, :] + audio_1 = (audio_1 - b)/torch.exp(s) + audio = torch.cat([audio_0, audio_1],1) + + audio = self.convinv[k](audio, reverse=True) + + if k % self.n_early_every == 0 and k > 0: + if spect.type() == 'torch.cuda.HalfTensor': + z = torch.cuda.HalfTensor(spect.size(0), self.n_early_size, spect.size(2)).normal_() + else: + z = torch.cuda.FloatTensor(spect.size(0), self.n_early_size, spect.size(2)).normal_() + audio = torch.cat((sigma*z, audio),1) + + audio = audio.permute(0,2,1).contiguous().view(audio.size(0), -1).data + return audio + + @staticmethod + def remove_weightnorm(model): + waveglow = model + for WN in waveglow.WN: + WN.start = torch.nn.utils.remove_weight_norm(WN.start) + WN.in_layers = remove(WN.in_layers) + WN.cond_layer = torch.nn.utils.remove_weight_norm(WN.cond_layer) + WN.res_skip_layers = remove(WN.res_skip_layers) + return waveglow + + +def remove(conv_list): + new_conv_list = torch.nn.ModuleList() + for old_conv in conv_list: + old_conv = torch.nn.utils.remove_weight_norm(old_conv) + new_conv_list.append(old_conv) + return new_conv_list diff --git a/models/tacotron2/waveglow/glow_old.py b/models/tacotron2/waveglow/glow_old.py new file mode 100644 index 0000000000..0de2375cc5 --- /dev/null +++ b/models/tacotron2/waveglow/glow_old.py @@ -0,0 +1,233 @@ +import copy +import torch +from glow import Invertible1x1Conv, remove + + +@torch.jit.script +def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): + n_channels_int = n_channels[0] + in_act = input_a+input_b + t_act = torch.tanh(in_act[:, :n_channels_int, :]) + s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) + acts = t_act * s_act + return acts + + +class WN(torch.nn.Module): + """ + This is the WaveNet like layer for the affine coupling. The primary difference + from WaveNet is the convolutions need not be causal. There is also no dilation + size reset. The dilation only doubles on each layer + """ + def __init__(self, n_in_channels, n_mel_channels, n_layers, n_channels, + kernel_size): + super(WN, self).__init__() + assert(kernel_size % 2 == 1) + assert(n_channels % 2 == 0) + self.n_layers = n_layers + self.n_channels = n_channels + self.in_layers = torch.nn.ModuleList() + self.res_skip_layers = torch.nn.ModuleList() + self.cond_layers = torch.nn.ModuleList() + + start = torch.nn.Conv1d(n_in_channels, n_channels, 1) + start = torch.nn.utils.weight_norm(start, name='weight') + self.start = start + + # Initializing last layer to 0 makes the affine coupling layers + # do nothing at first. This helps with training stability + end = torch.nn.Conv1d(n_channels, 2*n_in_channels, 1) + end.weight.data.zero_() + end.bias.data.zero_() + self.end = end + + for i in range(n_layers): + dilation = 2 ** i + padding = int((kernel_size*dilation - dilation)/2) + in_layer = torch.nn.Conv1d(n_channels, 2*n_channels, kernel_size, + dilation=dilation, padding=padding) + in_layer = torch.nn.utils.weight_norm(in_layer, name='weight') + self.in_layers.append(in_layer) + + cond_layer = torch.nn.Conv1d(n_mel_channels, 2*n_channels, 1) + cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') + self.cond_layers.append(cond_layer) + + # last one is not necessary + if i < n_layers - 1: + res_skip_channels = 2*n_channels + else: + res_skip_channels = n_channels + res_skip_layer = torch.nn.Conv1d(n_channels, res_skip_channels, 1) + res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight') + self.res_skip_layers.append(res_skip_layer) + + def forward(self, forward_input): + audio, spect = forward_input + audio = self.start(audio) + + for i in range(self.n_layers): + acts = fused_add_tanh_sigmoid_multiply( + self.in_layers[i](audio), + self.cond_layers[i](spect), + torch.IntTensor([self.n_channels])) + + res_skip_acts = self.res_skip_layers[i](acts) + if i < self.n_layers - 1: + audio = res_skip_acts[:,:self.n_channels,:] + audio + skip_acts = res_skip_acts[:,self.n_channels:,:] + else: + skip_acts = res_skip_acts + + if i == 0: + output = skip_acts + else: + output = skip_acts + output + return self.end(output) + + +class WaveGlow(torch.nn.Module): + def __init__(self, n_mel_channels, n_flows, n_group, n_early_every, + n_early_size, WN_config): + super(WaveGlow, self).__init__() + + self.upsample = torch.nn.ConvTranspose1d(n_mel_channels, + n_mel_channels, + 1024, stride=256) + assert(n_group % 2 == 0) + self.n_flows = n_flows + self.n_group = n_group + self.n_early_every = n_early_every + self.n_early_size = n_early_size + self.WN = torch.nn.ModuleList() + self.convinv = torch.nn.ModuleList() + + n_half = int(n_group/2) + + # Set up layers with the right sizes based on how many dimensions + # have been output already + n_remaining_channels = n_group + for k in range(n_flows): + if k % self.n_early_every == 0 and k > 0: + n_half = n_half - int(self.n_early_size/2) + n_remaining_channels = n_remaining_channels - self.n_early_size + self.convinv.append(Invertible1x1Conv(n_remaining_channels)) + self.WN.append(WN(n_half, n_mel_channels*n_group, **WN_config)) + self.n_remaining_channels = n_remaining_channels # Useful during inference + + def forward(self, forward_input): + return None + """ + forward_input[0] = audio: batch x time + forward_input[1] = upsamp_spectrogram: batch x n_cond_channels x time + """ + """ + spect, audio = forward_input + + # Upsample spectrogram to size of audio + spect = self.upsample(spect) + assert(spect.size(2) >= audio.size(1)) + if spect.size(2) > audio.size(1): + spect = spect[:, :, :audio.size(1)] + + spect = spect.unfold(2, self.n_group, self.n_group).permute(0, 2, 1, 3) + spect = spect.contiguous().view(spect.size(0), spect.size(1), -1).permute(0, 2, 1) + + audio = audio.unfold(1, self.n_group, self.n_group).permute(0, 2, 1) + output_audio = [] + s_list = [] + s_conv_list = [] + + for k in range(self.n_flows): + if k%4 == 0 and k > 0: + output_audio.append(audio[:,:self.n_multi,:]) + audio = audio[:,self.n_multi:,:] + + # project to new basis + audio, s = self.convinv[k](audio) + s_conv_list.append(s) + + n_half = int(audio.size(1)/2) + if k%2 == 0: + audio_0 = audio[:,:n_half,:] + audio_1 = audio[:,n_half:,:] + else: + audio_1 = audio[:,:n_half,:] + audio_0 = audio[:,n_half:,:] + + output = self.nn[k]((audio_0, spect)) + s = output[:, n_half:, :] + b = output[:, :n_half, :] + audio_1 = torch.exp(s)*audio_1 + b + s_list.append(s) + + if k%2 == 0: + audio = torch.cat([audio[:,:n_half,:], audio_1],1) + else: + audio = torch.cat([audio_1, audio[:,n_half:,:]], 1) + output_audio.append(audio) + return torch.cat(output_audio,1), s_list, s_conv_list + """ + + def infer(self, spect, sigma=1.0): + spect = self.upsample(spect) + # trim conv artifacts. maybe pad spec to kernel multiple + time_cutoff = self.upsample.kernel_size[0] - self.upsample.stride[0] + spect = spect[:, :, :-time_cutoff] + + spect = spect.unfold(2, self.n_group, self.n_group).permute(0, 2, 1, 3) + spect = spect.contiguous().view(spect.size(0), spect.size(1), -1).permute(0, 2, 1) + + if spect.type() == 'torch.cuda.HalfTensor': + audio = torch.cuda.HalfTensor(spect.size(0), + self.n_remaining_channels, + spect.size(2)).normal_() + else: + audio = torch.cuda.FloatTensor(spect.size(0), + self.n_remaining_channels, + spect.size(2)).normal_() + + audio = torch.autograd.Variable(sigma*audio) + + for k in reversed(range(self.n_flows)): + n_half = int(audio.size(1)/2) + if k%2 == 0: + audio_0 = audio[:,:n_half,:] + audio_1 = audio[:,n_half:,:] + else: + audio_1 = audio[:,:n_half,:] + audio_0 = audio[:,n_half:,:] + + output = self.WN[k]((audio_0, spect)) + s = output[:, n_half:, :] + b = output[:, :n_half, :] + audio_1 = (audio_1 - b)/torch.exp(s) + if k%2 == 0: + audio = torch.cat([audio[:,:n_half,:], audio_1],1) + else: + audio = torch.cat([audio_1, audio[:,n_half:,:]], 1) + + audio = self.convinv[k](audio, reverse=True) + + if k%4 == 0 and k > 0: + if spect.type() == 'torch.cuda.HalfTensor': + z = torch.cuda.HalfTensor(spect.size(0), + self.n_early_size, + spect.size(2)).normal_() + else: + z = torch.cuda.FloatTensor(spect.size(0), + self.n_early_size, + spect.size(2)).normal_() + audio = torch.cat((sigma*z, audio),1) + + return audio.permute(0,2,1).contiguous().view(audio.size(0), -1).data + + @staticmethod + def remove_weightnorm(model): + waveglow = model + for WN in waveglow.WN: + WN.start = torch.nn.utils.remove_weight_norm(WN.start) + WN.in_layers = remove(WN.in_layers) + WN.cond_layers = remove(WN.cond_layers) + WN.res_skip_layers = remove(WN.res_skip_layers) + return waveglow diff --git a/models/tacotron2/waveglow/inference.py b/models/tacotron2/waveglow/inference.py new file mode 100644 index 0000000000..39e7e222d3 --- /dev/null +++ b/models/tacotron2/waveglow/inference.py @@ -0,0 +1,84 @@ +# ***************************************************************************** +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the NVIDIA CORPORATION nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# ***************************************************************************** +import os +from scipy.io.wavfile import write +import torch +from mel2samp import files_to_list, MAX_WAV_VALUE +from denoiser import Denoiser + + +def main(mel_files, waveglow_path, sigma, output_dir, sampling_rate, is_fp16, + denoiser_strength): + mel_files = files_to_list(mel_files) + waveglow = torch.load(waveglow_path)['model'] + waveglow = waveglow.remove_weightnorm(waveglow) + waveglow.cuda().eval() + if is_fp16: + from apex import amp + waveglow, _ = amp.initialize(waveglow, [], opt_level="O3") + + if denoiser_strength > 0: + denoiser = Denoiser(waveglow).cuda() + + for i, file_path in enumerate(mel_files): + file_name = os.path.splitext(os.path.basename(file_path))[0] + mel = torch.load(file_path) + mel = torch.autograd.Variable(mel.cuda()) + mel = torch.unsqueeze(mel, 0) + mel = mel.half() if is_fp16 else mel + with torch.no_grad(): + audio = waveglow.infer(mel, sigma=sigma) + if denoiser_strength > 0: + audio = denoiser(audio, denoiser_strength) + audio = audio * MAX_WAV_VALUE + audio = audio.squeeze() + audio = audio.cpu().numpy() + audio = audio.astype('int16') + audio_path = os.path.join( + output_dir, "{}_synthesis.wav".format(file_name)) + write(audio_path, sampling_rate, audio) + print(audio_path) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument('-f', "--filelist_path", required=True) + parser.add_argument('-w', '--waveglow_path', + help='Path to waveglow decoder checkpoint with model') + parser.add_argument('-o', "--output_dir", required=True) + parser.add_argument("-s", "--sigma", default=1.0, type=float) + parser.add_argument("--sampling_rate", default=22050, type=int) + parser.add_argument("--is_fp16", action="store_true") + parser.add_argument("-d", "--denoiser_strength", default=0.0, type=float, + help='Removes model bias. Start with 0.1 and adjust') + + args = parser.parse_args() + + main(args.filelist_path, args.waveglow_path, args.sigma, args.output_dir, + args.sampling_rate, args.is_fp16, args.denoiser_strength) diff --git a/models/tacotron2/waveglow/mel2samp.py b/models/tacotron2/waveglow/mel2samp.py new file mode 100644 index 0000000000..f13f4af8a7 --- /dev/null +++ b/models/tacotron2/waveglow/mel2samp.py @@ -0,0 +1,142 @@ +# ***************************************************************************** +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the NVIDIA CORPORATION nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# *****************************************************************************\ +import os +import random +import argparse +import json +import torch +import torch.utils.data +import sys +from scipy.io.wavfile import read + +# We're using the audio processing from TacoTron2 to make sure it matches +sys.path.insert(0, 'tacotron2') +from tacotron2.layers import TacotronSTFT + +MAX_WAV_VALUE = 32768.0 + +def files_to_list(filename): + """ + Takes a text file of filenames and makes a list of filenames + """ + with open(filename, encoding='utf-8') as f: + files = f.readlines() + + files = [f.rstrip() for f in files] + return files + +def load_wav_to_torch(full_path): + """ + Loads wavdata into torch array + """ + sampling_rate, data = read(full_path) + return torch.from_numpy(data).float(), sampling_rate + + +class Mel2Samp(torch.utils.data.Dataset): + """ + This is the main class that calculates the spectrogram and returns the + spectrogram, audio pair. + """ + def __init__(self, training_files, segment_length, filter_length, + hop_length, win_length, sampling_rate, mel_fmin, mel_fmax): + self.audio_files = files_to_list(training_files) + random.seed(1234) + random.shuffle(self.audio_files) + self.stft = TacotronSTFT(filter_length=filter_length, + hop_length=hop_length, + win_length=win_length, + sampling_rate=sampling_rate, + mel_fmin=mel_fmin, mel_fmax=mel_fmax) + self.segment_length = segment_length + self.sampling_rate = sampling_rate + + def get_mel(self, audio): + audio_norm = audio / MAX_WAV_VALUE + audio_norm = audio_norm.unsqueeze(0) + audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False) + melspec = self.stft.mel_spectrogram(audio_norm) + melspec = torch.squeeze(melspec, 0) + return melspec + + def __getitem__(self, index): + # Read audio + filename = self.audio_files[index] + audio, sampling_rate = load_wav_to_torch(filename) + if sampling_rate != self.sampling_rate: + raise ValueError("{} SR doesn't match target {} SR".format( + sampling_rate, self.sampling_rate)) + + # Take segment + if audio.size(0) >= self.segment_length: + max_audio_start = audio.size(0) - self.segment_length + audio_start = random.randint(0, max_audio_start) + audio = audio[audio_start:audio_start+self.segment_length] + else: + audio = torch.nn.functional.pad(audio, (0, self.segment_length - audio.size(0)), 'constant').data + + mel = self.get_mel(audio) + audio = audio / MAX_WAV_VALUE + + return (mel, audio) + + def __len__(self): + return len(self.audio_files) + +# =================================================================== +# Takes directory of clean audio and makes directory of spectrograms +# Useful for making test sets +# =================================================================== +if __name__ == "__main__": + # Get defaults so it can work with no Sacred + parser = argparse.ArgumentParser() + parser.add_argument('-f', "--filelist_path", required=True) + parser.add_argument('-c', '--config', type=str, + help='JSON file for configuration') + parser.add_argument('-o', '--output_dir', type=str, + help='Output directory') + args = parser.parse_args() + + with open(args.config) as f: + data = f.read() + data_config = json.loads(data)["data_config"] + mel2samp = Mel2Samp(**data_config) + + filepaths = files_to_list(args.filelist_path) + + # Make directory if it doesn't exist + if not os.path.isdir(args.output_dir): + os.makedirs(args.output_dir) + os.chmod(args.output_dir, 0o775) + + for filepath in filepaths: + audio, sr = load_wav_to_torch(filepath) + melspectrogram = mel2samp.get_mel(audio) + filename = os.path.basename(filepath) + new_filepath = args.output_dir + '/' + filename + '.pt' + print(new_filepath) + torch.save(melspectrogram, new_filepath) diff --git a/models/tacotron2/waveglow/requirements.txt b/models/tacotron2/waveglow/requirements.txt new file mode 100644 index 0000000000..20c8b3eefb --- /dev/null +++ b/models/tacotron2/waveglow/requirements.txt @@ -0,0 +1,10 @@ +torch==1.0 +matplotlib==2.1.0 +tensorflow +numpy==1.13.3 +inflect==0.2.5 +librosa==0.6.0 +scipy==1.0.0 +tensorboardX==1.1 +Unidecode==1.0.22 +pillow diff --git a/models/tacotron2/waveglow/train.py b/models/tacotron2/waveglow/train.py new file mode 100644 index 0000000000..e8035c1528 --- /dev/null +++ b/models/tacotron2/waveglow/train.py @@ -0,0 +1,188 @@ +# ***************************************************************************** +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the NVIDIA CORPORATION nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# ***************************************************************************** +import argparse +import json +import os +import torch + +#=====START: ADDED FOR DISTRIBUTED====== +from distributed import init_distributed, apply_gradient_allreduce, reduce_tensor +from torch.utils.data.distributed import DistributedSampler +#=====END: ADDED FOR DISTRIBUTED====== + +from torch.utils.data import DataLoader +from glow import WaveGlow, WaveGlowLoss +from mel2samp import Mel2Samp + +def load_checkpoint(checkpoint_path, model, optimizer): + assert os.path.isfile(checkpoint_path) + checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') + iteration = checkpoint_dict['iteration'] + optimizer.load_state_dict(checkpoint_dict['optimizer']) + model_for_loading = checkpoint_dict['model'] + model.load_state_dict(model_for_loading.state_dict()) + print("Loaded checkpoint '{}' (iteration {})" .format( + checkpoint_path, iteration)) + return model, optimizer, iteration + +def save_checkpoint(model, optimizer, learning_rate, iteration, filepath): + print("Saving model and optimizer state at iteration {} to {}".format( + iteration, filepath)) + model_for_saving = WaveGlow(**waveglow_config).cuda() + model_for_saving.load_state_dict(model.state_dict()) + torch.save({'model': model_for_saving, + 'iteration': iteration, + 'optimizer': optimizer.state_dict(), + 'learning_rate': learning_rate}, filepath) + +def train(num_gpus, rank, group_name, output_directory, epochs, learning_rate, + sigma, iters_per_checkpoint, batch_size, seed, fp16_run, + checkpoint_path, with_tensorboard): + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + #=====START: ADDED FOR DISTRIBUTED====== + if num_gpus > 1: + init_distributed(rank, num_gpus, group_name, **dist_config) + #=====END: ADDED FOR DISTRIBUTED====== + + criterion = WaveGlowLoss(sigma) + model = WaveGlow(**waveglow_config).cuda() + + #=====START: ADDED FOR DISTRIBUTED====== + if num_gpus > 1: + model = apply_gradient_allreduce(model) + #=====END: ADDED FOR DISTRIBUTED====== + + optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) + + if fp16_run: + from apex import amp + model, optimizer = amp.initialize(model, optimizer, opt_level='O1') + + # Load checkpoint if one exists + iteration = 0 + if checkpoint_path != "": + model, optimizer, iteration = load_checkpoint(checkpoint_path, model, + optimizer) + iteration += 1 # next iteration is iteration + 1 + + trainset = Mel2Samp(**data_config) + # =====START: ADDED FOR DISTRIBUTED====== + train_sampler = DistributedSampler(trainset) if num_gpus > 1 else None + # =====END: ADDED FOR DISTRIBUTED====== + train_loader = DataLoader(trainset, num_workers=1, shuffle=False, + sampler=train_sampler, + batch_size=batch_size, + pin_memory=False, + drop_last=True) + + # Get shared output_directory ready + if rank == 0: + if not os.path.isdir(output_directory): + os.makedirs(output_directory) + os.chmod(output_directory, 0o775) + print("output directory", output_directory) + + if with_tensorboard and rank == 0: + from tensorboardX import SummaryWriter + logger = SummaryWriter(os.path.join(output_directory, 'logs')) + + model.train() + epoch_offset = max(0, int(iteration / len(train_loader))) + # ================ MAIN TRAINNIG LOOP! =================== + for epoch in range(epoch_offset, epochs): + print("Epoch: {}".format(epoch)) + for i, batch in enumerate(train_loader): + model.zero_grad() + + mel, audio = batch + mel = torch.autograd.Variable(mel.cuda()) + audio = torch.autograd.Variable(audio.cuda()) + outputs = model((mel, audio)) + + loss = criterion(outputs) + if num_gpus > 1: + reduced_loss = reduce_tensor(loss.data, num_gpus).item() + else: + reduced_loss = loss.item() + + if fp16_run: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + optimizer.step() + + print("{}:\t{:.9f}".format(iteration, reduced_loss)) + if with_tensorboard and rank == 0: + logger.add_scalar('training_loss', reduced_loss, i + len(train_loader) * epoch) + + if (iteration % iters_per_checkpoint == 0): + if rank == 0: + checkpoint_path = "{}/waveglow_{}".format( + output_directory, iteration) + save_checkpoint(model, optimizer, learning_rate, iteration, + checkpoint_path) + + iteration += 1 + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, + help='JSON file for configuration') + parser.add_argument('-r', '--rank', type=int, default=0, + help='rank of process for distributed') + parser.add_argument('-g', '--group_name', type=str, default='', + help='name of group for distributed') + args = parser.parse_args() + + # Parse configs. Globals nicer in this case + with open(args.config) as f: + data = f.read() + config = json.loads(data) + train_config = config["train_config"] + global data_config + data_config = config["data_config"] + global dist_config + dist_config = config["dist_config"] + global waveglow_config + waveglow_config = config["waveglow_config"] + + num_gpus = torch.cuda.device_count() + if num_gpus > 1: + if args.group_name == '': + print("WARNING: Multiple GPUs detected but no distributed group set") + print("Only running 1 GPU. Use distributed.py for multiple GPUs") + num_gpus = 1 + + if num_gpus == 1 and args.rank != 0: + raise Exception("Doing single GPU training on rank > 0") + + torch.backends.cudnn.enabled = True + torch.backends.cudnn.benchmark = False + train(num_gpus, args.rank, args.group_name, **train_config) diff --git a/models/tacotron2/waveglow/waveglow_logo.png b/models/tacotron2/waveglow/waveglow_logo.png new file mode 100755 index 0000000000000000000000000000000000000000..d15bed6048048fcb01514e19f4e7d28cbe2ce043 GIT binary patch literal 391167 zcma&NcT^JY`!{aGtSo7&<>1{g%|$lcVr6M*re>+-K*f=JPllD*a+SN>nJc%rahsAj zFgGA#D&j&+6huTHdVjvp@AsVZ{PB72a}?pg;kxhp8n4&ux_oc^Q19sBvxoWk_>SJc zcjplw-@%`Je0w4e9^k#QSJs-ud)VXmNbfdZX}|aa@5z4WTZXsz_$p!rx9#|O&kuRu zd+Nu>C)DxxXAj1!z=8MTIsdyB{yM&ncK$A2p8q^{@o?n3dPPZ9T}es(%GH1V{^tFC zrS{#a{d|18!T0andK_%C%rzYH=<_}Ora5lvpxymr_gWLL=qd}H)BESFu#lL$`){$A z6MR8Y8gsY_?R^Rpx2_*K3f_K6J2-p5_jN?O=%Mz(&U?pSl$V41Klw#rKe;{b|6V=H zSazldg$);AuU$pCjut=tjGD7rddkrvGe^uJT(h5T|8+UP_c8*j{I~TVOZ1-4W6vQ#spS-^j4Pph?D z1aFF(glrE22b|S28+TN_nK|J>ZAGk?L7m+_;jO7uNZzmR{P41qfXp5Zcs6Hxj47_J z8hn6S?_1tl#9|wrhRib-Y`cmw$DDy$s;a?IxMC*H$lCtq;8juj*7oUszs;$oA0e^Z zn%>jFo&Rq}wt7W?Q!}}6kCvO@me(4=I`Hc)-?T%x4^=t8+BmIxDD7qMH9D$G0M`<4 zB;fTf)^m6JCxep{HYqONDAmuHLv^87oC6wH1L%gGW%?TfyYScqYs`|jS`onJh->%f^&7PSUENkjn^-7XtOgGeTCUre-j1nJ>L6fCv(dAk@>eV$JFtrD+DW zNH*dbtP;P+!DE^yD>>aLoS0VS~^*NX9|H<|z&Hma9p%(wKl2?WbZ+kDoh3jT`SN8h#z zwRY9gHHgh;EG4poS6yaCIL;UWtn+Uj>W0DC{ou$ALX401>Ve>2O@B&RIjg!XN*nrj zI%~JnCw2=?;^3vIHgQAAI>1>kcp9OGN&qzk0JDnmFU!qe0Ny@3(EHSo0r{yf;YV1n^C^*`1 zk<`~M&0zYm+h-J}4>pF#i}awQ?)#-97G_!_8Lq#)u!R~k?tb(=8F+ygLEUIZ=lt8w z9FZmGHQ&!hWd;>;NrhFoGLIOrtM;b+YAL&sBu0jU_a!$rrU2HMxh~w~W;_YJzFFlZxYIAqTUx$9)hnvHoxV-E%DUMr~le1!e>HLFR(W6h$qsvL*H7kV1 zq`F$rVbCONF#)8xV*B2ivHOI*W7{hN{roUh^ZAM%+qXHH+qgR2XtR!9oehSq^YJ$eC6KsEt^->7P)!|Nq(z7 z07n8I7d!?>Mry8D%n>eeYeLu#lNrFD7p0SwLi3EK(x(HJfIp;CflQ5H=|Ck1+uGqir&%$GWo$j01h@^#?V#) znv0ApUcl>+)mf>=;3i?NO@obq0R3bS%*Ahad8Nmhj(N)!vU%;yOg$|rQA^5cZ*nKH zkB%T)h>T#zhZVci9}{=4GIN*`l|%z-Xn9wTRcJ#K^fxkLto>?DsyBUY(GICer1Nrl z-yeeWxFeuGC3Q~+_jy@pm9~~TOUJ6_JC&J13NqJ3Ui;w*Ic3vGcVwX;%|~UEC1(e6 z($Hf&GMPZT4;v^p!@cDP+lI+#k7P7#5a){PGy9gD{f2$i=IkeNV@6t;5OM9_Ch(+E z0qOhlV{lY&to@Es7oceMH;%9>vx;VlNbg;J9`z&AvkT^N^GocBsRg6}P)d?gu^I1MD+Ng}l+LN4bR=A|U|$x?6C<$#&VYKnxh0lh2XhRRPgh zW_pY>r)-PH{7mz!eluPrtw>{`AwKvo8AP`XME_wUTBHPydD4ck`yzOgo8qm+qM>W_ z*xLtB6NRUrAP*(OgNGHrab}0|-(#x{u{47Dj>n|gm?5kq$7;!fvf;+qFlAe<$#}MR zY#y=$f5=>2RQ1%Ue131tGs**g5_mqwB7zl`Bv6-p&@}5(z{&*Blhw6<^2VNq76CU` zvbbLxFXF)Znpicv5qdRy+g*~r>W?I!jVng@J@{?H(R&lC76Y=L2_hs*Rs$Okdc~RF z)xom``}Qj$&L;htoNZTbAskybH+6demj#t(%9=s=9@JkcbUD?BVFvy-@ziQJV7M)| ztf}!n4nHxD_#{>A%Yb%Mt%bmhv;&h@gFs_WbB+kfMvVKI%glkv01trB3pDFT-ogT> ztl+4_9CI5D1+09fE3mxrI*N14<60Vu9AY?ED5YzXP38+?I48f=VoBoPqjBe^?3LgC zZp5`5DH8Y7>}_kk79!aABnI4@52MAa16Aw-mU(b1b?R61ObRpl?ru->BOTn12;Fop z?2mZHk87}FQ(Q${-F1!y2-)IqF+@VPUxP|ol_OM^+1GQ)6HJ*hNNTD?^0=q{P8_S= zqxnA|hp&LJlof~4&S z#<$5RJ+@Q<*|paioK7YC&8v9>mSG@i*MdiJb4awGO&9E3sG??5Sb&%XTUa%O>0_S^ zZEs>u)>NNFSA@1CPshPh)8L`N06jO@*}(hBOGS<;+}KHI%~F+j%hDG~@#69cstW6; z0llSXTyD42t0QFc$hwxpTNqan*4>#VK@9R?FhrM>quVZ__BFK(a84_)w4*{^Gw&v^ z1fjc@1MOkJgCZ!k7v2h74Tmui3C0=Q@Xya;^x92YoeRUXo!a7u&K+0Uqr5lhz5py<7 zSYGV!#xS8`Um*)NsQ$6F({4^7wm`VnS^Qf%b!> zWb-nA?W!>|53xYrI4xNnX1km*RCGf`vTG?9M+|Dc<^~D$BLB~wK=u^>Bt|nYW*EM> zY@6NCB)HkR`@vyfAm}(+AZEL&q-XG#@E-(H3Pb8L*CY+{d$~Cx0QORVs6EnJf7_ zP9;?wPZ1$nk|Ueix>2zwwICX1G_*J}04yE2BM~Ze7_CXB+?c9Ay4;048*-Ud%B^17 z=vDWL!jOc7G6cLnVN`ys?4W-cuaIJ2kmJ!xf%8yAVL{Vun@#!B80&lrfF)fmQbJQN zWeFO$d^RPH(#PurakD1ZO9(+Bu&77lQzo^uS=3UEKcIjKoCkyDFAj zD{P`4-M935t=bk}?AmQ+6Wku?HW}^H#=D;Ia6y{}vZ^BIPCz<0nL9tKw2&D1VyiEW zV0Ai=sZocHb!UDdfz7x!&x2Z42LKapmRa;}h-_%-ArhJK({DSD?n2f$FwaDdcdo3Z z^@-PQ!X|~JT%bXJ3&gXE9yhyG@~y${g#(l3HeL*NlEZx+_KCHS+BaIB?PF(JVqkyv zNOpdrjb87(I087MSKLL6u4z*&ZC0w3tmwx^;1 z1eDQN@}zVF-XuQ6<$)H?Q_aM(DZ>`WFu#z8tmX)R`r0mH2CuqzJwU8D*khU@6Vu)g zS|0U2BT&A57Zvu5F-o=PnYi0tfdyu9%(;poBU}D0)T0O$R5=55{G4^?2qTK-@|`XP zNOaaBe<UVOy|F0VaHDZMdLYg0C|b*P8| z6mL8vzdZ~!shk8%edMMC{a8>+l>&o z?O1#_^Uwm54ZqiDDt(8Or?hLOLvKUJIK;pxeGZ@lJH%9og2j~y{b z^j5WBy)|a9U#-^pQ#BYXJ{0*coPK7y>)9`VeNa9K(4@A`=a<`#9O6xjXh=1%sT){<2EJjac+2}0=ILu z7A%6cOFbq4Je6S?W6*z8%0)&itY@_1ZYqd~9)I@0(uATVZP+~Ih$qrc zg_%|F#RGz3ZDXejHd2!Uu3}ew7Iq5hSKc(g%gNe_9T*#=Jqu&RQ;?^$=`ho{NBAtv zj=^1=T=ul3KPs06wL`5pF81c(FEvPv_r<2Q+W7+ zB(;M`<=9UDqm~wgy{Wy*57^fW?aKRUl_L+ix4?oTmiyejKl^=IYjJ3L;R+}wfNHY0 zm*&F5lh2uk1pzj>{13R}*9jEXPV5Q>DgVip-Jr0%op}ac=556kxCW$2A$rqGW@@o;fZML?JEcJP!vZ1;ZC7H3>z-48B1 z&pscV@j&g=q7?Pn%18xoQ(Xg`@nRK-9aZHD*mV4j61%V;=~bpoF?SBw|40E+dot4% z%%ctbiY(fkaw#%-MI{W_!vg5ByjZz+@V7>sMfveWW}r}ZX3@u$$*{N0vF@yNCkRu~ z%egwR8ee}YOg6dRomrPpJyHy3tq$%0&&aL-+R2=d$-)xfGKwv$!`h=m=BF9wbwt={ zjVw-(vRZ0rOcKc=C8p`}eLr%^?o`v)HOEg6Z+Zp6M!MDJ|%byH#42?0YB2Vq$tcWwbn2x77X%XysfLoxW7+U9R48lDQRS;pwQ z7?Z=xM6g9MPnz#TC^Rh_NjXnsHc>7(@JfT!12io z=Sc?M5F|-OfG`>fB@;8+azAUjd zo_Sc7slcyTFJ>|4NHA}B+&FI2X8~%F3jk)j&bK2P=_#w7N;12qo$82To5ofzfha0yGj&8LFZNAbz9CwXq5nbXD5Oqbu&kw-aqsceic?ckBAeblMORSl< zyce*xhldz;q7fot{SMm4w^8Kc}8ay zd^5AG^mIz8$&uZhfKdA*)1MIDt6c+4=jCZXC`fdNY|hGOb%6e$s4w40Cw$Ka_pu3! zIVTLMW^r3t!nV#Fi{VcH8)%v(cy5?=9JD290Fyb1-R0AQgqHsNMkYwZFr%?s=gHlR zOHAYUF1_n4=zjLy_L|HbA?qq0OKx$p@(unZh6Pq{XlD8*-}u0ZGe`36m_$c-f?BJq zDTy>UWOpWiQB)|UN9nmJ4*~7jSQuvmgunuuh^E`5o*{+Fm_kLJ1?Z@x?~>|5c%>21 zD-c;%EBmm3sv0R_j^GuVRQC6DdE$1fBfw4GokOvR*LvJeQ};pz z?L^Y*#t!iC`_HKmiUlPKuH|grxY$!UzjS1E{Q8ZUgEA2NXG_L(JQ04??94_N5jtQe zbH8x^a6Xi(9V}M7*5%W!;*Gm6wy&WUveiDeb%g&0qC+NiKCqSHY}Rql5>cb(QhaqW zu?Qrtb7q)(Ge9Zx7N+sEtHjA_v!N5d#|1h(m3YJ#F{QJxqbYpHTHYVFr*x$1bqDjt zdN3{Ju5ygy4T!V2)yBC(Y5 ztCpy#%=HAR9d=!qaz|%vz|5CNooi#D6dh+nvs5JmlwQ-H&P>GH^ZJxMu3C3IJla<4 z6_Q$FeEPxD2Eglo1LVNQ?x67>H#@6NG$#S*8EN2s#0_OCuq2ABIWJS^vah&?>T`C*B$y()>i|dtXW}K4)@s8rR%^yEbS(}eb zjOT2JHaL&7xwb)>EN10PW6}x?OBtUdZv<&0io5X4Q?)xoq8&2+md3k!DZUqmGvr=x zkAm^hFwciaS_r`PeMyKmm!hT4OmE8>L8RpQzDNiU@!!Z{(64?sQ+q-!xWdsfOn`W$-eB8Y2Z zT!{AK1X-BOT9 z^AM>*;`roN;q;?b1Ly51VW-5MlOkx~3W7tViLs;Ly5J)Zsl2;KPBUfiS?+!{e8gNF zmnnR(mG$_9`rH&Z%lF44{&{3=E=xSkeI7qXY>Kt;sO*%<5d!|A@u%n{DOaY0)LbTr z$<0oYCb|;MgFO#&Z$TS{pVbPn+_d;15Mc;x-Zfk+D%Atg=^Ec;&&NIdh_;4^cC~&r zMR%jsuzZ{%JP}WOFXhY_UqTrFrDmVM+B;?MGxKRm-Y4&AFc{nJ+8kK|#ovtoXo%b! zB~kPT+tQQ1JOl2|EyEX7nMz9!v-WDwU3)Vv_gFsCv&J9yzF%IDW0To0lhZFhVABdc z`rwZzT*bLpuSMd58ToxqR+3PE&22aBU`v@;^@)~V>CxYhh1NtLqfB%S#Sm+v{~nI6 z1o$gaFxX~KrFmmt-_&I3)y3!$z4^et+UGJsWojFfbAiqYCIW)-JVLea7VBMkLr@@c zQm3GSzLxRDolYbUW~lxf{YyObqCBkCR9|gQ8v-A5rC;DwVl1>5?duwG*i6olV=q6$ zVpJmy(cNbx?{2q7e(C#-dgQljPub}lTt5zqIO5GJ+p0JV+NSKIR~`BJGnL%;r}BCXoNbdT&FIr;cU8i7F%zxuB#3Z=YZn1>L_NSWrW*Gr7Nc`S@sTLy97bM8-UadJyB?~etE8OufNujAM>3Q*IxBRJ_aX6jmQ2Ul5qdPpNm*v{sT5us z(nD4R$9(+)bR4cKDTDv(J;IlECA_0OK09idyNmqmJwdu>4!ADc^yKy)%aE3+@U@>`qW1nF1wru^Xsmp*Uj?a9T7ap$1iO&LC zmOX5ZEiLIfmgieGXc!@kYE7L^X{Y`J!O2%w`Q?cEMzF_>h+(CLWUY$oh%X8HUkQLo zL0R)>SpSygjQEvQ_xD>)Xin~!l8J6Q_Z1=Y_P1#2psaw_?xoa#d8v*NhV;_-TyYj-RGnDkw!4yhy6vcA@c1uvLX zY@$nb1`+UGp)I(6?Zt_e9R*&h_dAkJ1o8WI4%#`Bp|eBRkYkySiz#n?(SB}5j8O4q z_k9y)rdjGjE`JytK@?4v|XKE~Rv?{)+ zYZ76&_SSN|lZtuzMZuL>(tj?|6^bZ6c%?Ka6 zrH1NB#ug@dVRc65Qr+)wfkN+X81i&N?fs?;Nf#2G70ObAE<5J@jNK`43Wq`P%S&L@ zR+s#zJ28ta{q>EuXL)h{mU@c?H8t{rso>F6g>)n@3rf^aVR>XY?)}3mRyH805 z1q+#~5tuK1YgvX!-~aDh*(t^>PjrO&&I%D<9VTD5_2Q|Q6|DdbeVQ}z2i^AttnJi2 z&y&A+@pBg!o!(@hB)z>B9SWtRpou0TQAkg~*g%`dgY50veR0KPZ0$`T2ce^O?u4eu z_}hTZfiC-y$8pHor>QvFS)Co397T&O8+##cqrX$8lacfG)rmi2U72?Qh0q~Ts<`8@ z-O0j-VB5c3`rQ>$Wa}KV7&%m;!EtZ{9fNxvGlyJR%6KXa_18VKfsG|Z`V~vn2LArk z6wW+t=L}+HI#e|c<0@8GD_htfev_+)#xVVFJWm!Tl@T>~jeo6~xP71HR2uLZdh$~V zYX*&48(FEiZJC@riS|Fe5c!nQmt_h|-oaLvM-@{M#(hca&?dv0tp|V2+UF|wZ2=)< z6T>S#NSRu`jS~1Zr#<-zkU>InvrNt;{OntI8o%iwgSQ!a@#(!*cRcGokTt(8P){YR ze(_419a)A)X#3zVwm1<{&2FU_5krRg1@jb-_hfB+4+!WF z{ChM7T8^$>9+e}`-6s3l>VZfBMXpcT-%7MMu*Lnr{x*H<-f3R_QzT2?Rew{Wypijt zrTCP#naK~gd-;SlQf?6l*u&I#J~b@Dh1 z+ng`f?rSu161(wveAsX*Pv#a)Z*`$(P2eeGQoFXiOt&G(HaoQ-H9F=IjhR6Mq7HZy zBs6w~5DHbgDz0wA`+>{P%;(oa#9Yhidhr$z9zlSgB|o*@<=;B1=4P0S!YXTTj&M#K z9sZ0r*AMU7nizmyk_jvlMIF7dQP}5D3rc7)_&2bRR>GELcUDo87WJ)%t?SQ`zOBg< z%|k@|t1-JD7ly1_T_7JlH#k-=f=mBtu~+T1_E`9v)~qu^TD}l4yGQ!}bh`+ynY%wL z&J#slx&Zn|W>94967C!?sfi3e=Cq-2)erpg%irQ}!|Vlq-#XAk_YF}5ZT-Z3QMZ_R zSd84v)EK9n@E8@#iKcm`k|Uruy|{j+ zmg(5!!$$GI^7BPZprs}o5hd<-{V4DQaYVq~qkyu{8F;ff+5SUVfhXPf=jTptamU-M zc%$``FtVo+&mH-l(OY>-qcvxW)_Ia~<*oFYec98b_LYzha6yvk-$YBVqu8}ddfz542#jX zr(L3=Gu=JXK?ttss*`-1hXkq<#htvSDe~e&m`Q?R^CwfR*V=dbX$NZX$^2d2!t0tJ zL`1<`q?Mgv$Ohtb$_QbKFW>r}DNUeE2lpu^!ywVCAjwPK_Wef47X4^ouft5(n&xbk z*YyN!M3R`PV#TDWhZbF5nHdRA*7y&n`Rkg`QiUMbN>6@=*LPJ7fzaa)JX9MKE*~=z zaw+EaLmn5>ebyqVry;1iv6c#V4>Rf-(OwzJ>O6UfWJC<`J`nNh>x()vp<#k!`_o?h zF|%OCijSx=(0(F&P&b-CH%sPV+Ypd@5lXf%lP6W+P0vv{G&Z;~nbk4^!FFarm2V*so-IppS1}>&TEnWpk{!cE3QbnqqUX4<9 zRnViJsD1;D{cm5IbB@o{sJ@Z1p#Bn-DcDOp>Wc6!flC++wj{WtYn=lM|DddyKTM1b zAU6@(0gvI&QYJu%4?$N2H)D#K5j&OYc(7A~$(a0r@&g~JDqH&`Y=ZY$Y~dPdVJNo? z2Ys%0CSd#pfyaK0_)jORX&UUFvK&tmTxU&uEQX&0g|xhSt!WztAX6{dMt2sZ9rq zL3V1_DUW9yC2N#%K`{c>wC-CGi^&iBET?WLE8g;1ncf)=$MO)osuAJUHV^h4qG+{Q zcB}~CJE|M=bs2MN@rZ_!W;-)W_VwnAvDujd7X=F+Hou5rHyE?r&>po7bv+e-!Nd)F=%;O99X6Vs^HZ3k6r{%anuXPC_In_EzmQKZbH;?D@v7dTKsA-<7oJN2J{|OkN7ENh zNO;Y>)<>(h)xh3K6^$Q(+Q$xE#E&|1nwuBNaCN|6aYu)zFJ?n|u<(**18Wgn?Mag6 zbu}%u=U0c4mEE=!F{N8y>%x58YdhmAc?t!^LnFq}a-MdlHuD8t>V-S)LVT0mib*9A z*=b)>$LVou`pe$9>bpr?8MQd!8(fxf^@)7u^q*PC!{(8pl+!B_Q(^J>*S+U3+j0yY_O2_mv| z0rnW0P259?xTo zAs6E&8nY+eMMEU@YOBj%f&BURpAVZa65FsOf7wUcS0*+dK@InW29UkqJvz?Av-w%# z#}8pYW$5(B$HajTF-Ep=vnaT#0s^d-z*h94Ook@!;mEm7Vi&mT zUMVPm(BG2Pl2v`o)$qd`-=})R>z|}Z$|g~gT|-?1^&J6>j$?!L`LW`S>1P#r3Jc8I z1LSDSgh`>_?8@|-bR&eGLrjp3knjLW+=9ST6KLAfj3=da%;k%<72r56@k;!^UGH?ZOKRaVq>Svl= z;P+zM829vmsyt;=|2DT-aUEtpJ~?y?83tj#T@PS_tA5QfujprP+CXYe2F)y4sAs^; zOkzJD!KARrTBrj{t@{BC?1{b{WKmX}1~qJo38&HUU4gZ4eX3C)!I*BPPKtAr58V_g zv(Vzryey`frh?!JS^k=?VS2%ee5a{AcAl`zOx z(pLCy^tjjp#}OYm;b`OmUE8+R03y8qDx|NsB*ftlQdZjd7c=A=@LX`7LW?M}nksn( zKY%JDJT38JJJ4gEju(eSagw=FAetJ)`6faih&2p5F##e>D$x{*`@9}hAlGPw&Yl^E zna#O;BPg~XY+qfEP`gBf9Z3N3uDlj3w=UY?6Q4uh<1RjQ8nr?Dh5sOd6N;iaEV9-UBpuGsYvgEk)+c4mqUu*3Ypdr(PYK5FhNtEyfO#$MwqcMfo4-5bTZWq4+rGOgmFi+h{ z7+%;NooGH%hhMJ-{+id6cY(T==K}PlOb~#Ky$oBKlN0p%y@*8f#JBjBHAS|(7&YH$ zX6U7Q3ghGX>n97p!q4TLhYZr$qZAvVmT%Egs?Tc>54$NaepiJp!v6cL4SDqa*td~{ zMw}v2zD;$*ff~ZUX(IjH@IQ+RV$PHcuokimTQ1gLKclkvVNg z-Tg6Du~)Q=*H_%!*u<{*c1v)|16(ZE?YhB=$Vrp8vGaj1d@2Th9x(D*iKAh!Q3hmj zp}^#|1$cqy%(nj}>0z^bLmari@$%~rroZ}$~U2E-qX??e_s#=}Z`w}97xdUIJZ zJM@8E|2tUx<+m%F4J4xe8vI0t%DBJBo~r61}`wS|P!2*H;_qGuCE01jIX z71v?I8uI>}sj2AJ#@b&GgXyF7ssFh|!&9 z6Bcz&nMU(8z%);-B6MnkM=o8qq!l=c81$AxgOCi`cJ}15UGdX#_C6Bw_2tQIGvtw( z%Vy1nfrl|z=35?D`{MPRj;ZvZ7+Drybv;#;{T*<`TRSPezzlc`!GjEAU#AJ}mTHc) z!9)nQsQY2a=E~qU8mtN86+a2Jc``;7d z{S!;0ipUf~HQ_d^dsMVjP#aNDXykF(v8ZVy$SGn)`2)K(wridFx7`A#MgCjJg-b=it<0nLI_ytjmllih;s`2Nd>88*WCUx zNwoz~^z|r*hTiU86D{qPlZMg{D=&rub8);O})?+*oF!*7VJUNAr{qS^b-(`xq2YbGei{*vJZCQZO7 z+t>DC#G0J5Z9As%Y*zv=I2Ys3x2y%N(bhBrY6HN+&~(U)Fae4_MZ|r$SIKwGR1*TT zf$?U(kn#rJ0}exP)dF-e=2@fJ^EGC}CL&Aom64rNHiYCAnvptSX~4u`X?wPczhuwn z4eYZHmy%TLKXoXOA#%PR^ha}*QvcilGf{aovF6m@5X}-jbh_xDSn4VAaJS({;NPN- z2z#Nman$BRf05r8+RfDR4Jhx7l?FGcqHrl&bcsw8p>pD6f=9Tp!sfY=iE7 z-236p)y^9MmEVZxk|!#t#}Or395ccTrxC@_ka&%wPLL$yCYfsOs5kp2UE!6y=PAi{@uM5w(W@?lD z0*%CpRQ6imPx3ioB73MWB)~&1r0yYiyy^xsE=LucK@t!EbzGu4&PYDJ`hl#onsb1~ z>7#{a6oYrGf}RU6<@}-KD`-*g{7VMODw3qsCTr$ehI!^{g$1zMw3fF&X2F~rjpSrq z#b!fhlC^vi9(#t~q`j&o>BLr3_w#0wtkfJPxRp`>Zi&KhDgvX8U(8V&(%v}JHS#8= zS=FrhbPkClHOp%!Km=o(a;OK~7YhcYWIXey>WiG|l+VSumqv}`DAUR9LA89DLf)j2 z_fu!6u=kj0o|Y1g#@U+o%bs1JpZnkRO8_G64+wOpcmJ~xup3wtxL9mrn`4BMmY@X| zb4WrPPJkH#j=LaMnmy!6^@R=NTO9nwv4(&;$ZDW1L4Yth!1^edlYT>OTfRFdFTH)H zm^|z(8bs_aTSXaC`)f?RYFPE&%v`OgN*;&>mNQzELW@B)zWSm1c$qmm&J-blUfXC^ zj0aHl0YG||TI&{Qcr*%^MxavJibULg}W!@Q{O<@+$MX8l&-#qabs>nqwkVs$sWeA0q_ z*K?3J9k<2NaoPCj+L*}00@{1#4lPB0mMh5cx@@}h0md7AZrGk2(zaMwRknN{tsmlj5k9 zi9yMLO-~-y!@yl{_KKg*|C%i`nVa(sFvz&AcEJwVlvM){Wp)UoAtT&iXt6Du zA?#JV@v?Av#NgrFX~IOyl3+UjgcNsZVpaQ4_36QNFz}jaE&r@@WyBDSE*$QCQ%OcDLykAm5ND=(YuALsS|^EIq5 ze1N$9^i5yEDK*!$2=RJ4Czo;wv8y z7eNCKf66qj%3{15D~+G-258aHyk0lZOC^110qNyzsPP$GOBBt?qk)$U-ke-kUMl|| z*FX02s9xvi-_+eBZ+aL-_rva==E!oKHvUm>>|?OF<(F3+qf9Fb6Jl_T{WaG_iF}$iONo;oIZ=pO<T}n{!^NuZ zq4_VVEVO+k?X>}9ZQBO3!fi#gy%J2E=fra_4(v`TC5YUnM9BEL49k=cnK!(rLMiTZ zbj21(WA?%XuVE4&L6=N8iA}tmFOSg&{LyT7cW_BAqp)WgGp{%ys&Cgf>VGAz@jZ(a zBaP;b_KaP*i*V&Cm@T3RZ(HM@XfBOc1xN~`q+_NBsyT#rPg@PQLgRtU2G8l3doA}e zL`>stzz-k(Z8vHw$td6MEO`DSey_CaA_efLan&kYhU(t33Rdr8c%SWY?>`ip0jc2& zduyL7dO5=}U+8^r9UhHnTl-MM85m<7h_<7qUF6P+w!gK~?p{yA#eoe8PaW8mh z-N)1W#gK3VqFk49+XQtjSwA^TewlDV!CX#{K8xDA2Nz#m*m*3Q(=-etLBZ#I- z(S9AU@ig^ z+w?9|<-^?dQzo)U5rzn^fpKpJ^Co+{Nt30!TIH38i%kTo3tHIM%w6|6^_C}u0rv(N zm3WmHpov^?6K1*RtdxiEn0es6x+LU^bdHawDa7X>M{^+`=YbZgm5#Z(*sjeBjMq-W z1hcHBuXG7wQ)f!2M|T$_uqHCxspSVvrnpEsd4)WFt5%uy(l5h7UBKwC*@Ia-UPmb} z2K@Fa^YeQYtmTG<1}ZiijcbLOev?U|%X`gkU zq_2Luzu;*>Q=u;z(>-0G_j!hN+EZAnW*phPHm+t$9Q;Q9Z*ivKef`cCMJ=H!^N~*? zQz8$q@hjd^Lhl`UrDv_;RI9bB|DnoFTqBlnC4CK$6D*)G9Z|h0Q)up;<$TQiWig-S zRDLtwEOhnmT4NfNkRwEnesq66CHzb;U?0O?=6^^$P$)RXiat@3a1@oMhXtKrGO^|95)Jni|cU zLvZ1)-0RHN@@{B~%Eydso==IN8|Bqym8HaLn~%?o7CyXRa5aT1`h;hE4K~5~NdIET z`&Cx`hiYs4KGu9@aZIPOT8FinBss{i#pVtNY0@r_>S}OLj1O78QDR73d04Ds27jtU zuWz-gnD~>1Ugn>^o7z?66BXboJRVt}gv)9M%E?M9$gj{VOs%w59BZk%lw;TYBB|Xtsg|F>(6T9A(beca3$+A)sYv6#uc=ZF@vq zqr36OmgfxVezt$SDq3dsj-m9__|40ykXkLh%bzhk!Oop!IPKS6@AI&u4$zZ$Imp-& zSs$#(Y%I3&1kPSQOAOi5C!uNWb`KK%OqyYe;{bDkITavxs_-m*p4VYltc#y{wZgxL zxn@%VySKFnUZ+O3NSmYaJ4Suro)`;^Y#*#{30?@D=99Mxs^a0&#%c^x0WDFvc;{7 zd{Hnjnl@&KY=4lwfm=!`tM8FUs4-NO94Gf)C@l{atZ|iO{{O9_h`TydEnx z{Aj*uhgCax$XSoAabBay?xKe$yjz)WHE)H0!mlj+X^*|??rZL9mzSk&I-Ik|3q(tQ zKd>vZVB)`^*u~pml6~}p3&2vbtD@j9@TIL@ecEAkDzI9%GjcpkHgd8!Q7FzOam?@& z`V;asc*upT@T??l!xnU;h|g zgJf7t3iZ~y@8O(SEl{Exq}QfOuK!ave(7}>yk5aV?feZC?8=9PBU zMVHx!&k5uOt?gDoBZcCwdih$j+GQkKUyHMI3ftZhfbS(@+mA|jaf-%@m4`b*0a0w1{XF6rHa|EjF+UCuUP7NXQ%>199D zH&jhvw9MV};||OtDhwshA}oOeVqmbM^CXZzdsGc}`va^lR;&*13e6Ukng|-gaaWjh zNISD0{!^w^7Uoi_b5Q4NH4DDSA0E?D=HDRQ;}&?hpP(#F`lPxF5L?Z=D_@c=+jy)l z^HtYb+1XZE{B82d{LjtR`sK9r-}ikNb;;B`N&s3+2K>FP=v7pFqerQ&r%rmAAYxFt za%Sf6x9S-mp7tO3K6;wF!ed}8KeP(=QdBFcWABI0>llw0_aXp6t)ZvWEyL&$ys5Uq6uTiapN7ev-Y%iq%~mR}k%~c!5J#ZF zPG_fz8keMM7oe6Ac__05lb!;70cbzy^T1LzNbhe22OEnD-_GuGzBa5i6gKtIy*Jqf zFx;*WQ^mV}`~Q5}67-G>V6$~dkVGZUJ8?I>BiC+W?@VL2|C3>S)0eIRQ`Wv(4;_x1f{w4|YMz$@0~!Gjf3)8+ z|0RFuXYg8#55*u|W!k;@c!K(reAHT8kA5DRmD|a&2lB)_3gE40fyOv<*{9vrtz{;b zfx*~~ys2BXyg#0>)3Vd{W+Ezm3Kk+k6AI6iA6Zl8+5<2oqO6P}Ft4CgJ#jGj0x(8t5SRE!=%4Y8XTEld{Du*hF z=N9UN6q3G?fW!CEc&KnDMnfwj@dUu0lWWVZ%G-1fDg_cA)=#lc@~MnDdFJoRWfpiTPyC7QJ?+63-9%0w@kaNA~)S3*B1Q#z&8e|vlFm2INUP-=v zB%Q1zwEq6SL?oM7=VnM=%8e9#5J!Vy zsM}TFf!9~dV=K(VE9}ZDIvyWYOS;W6%f{l;2~BY=oC9LodiCD>eA7p!@2-QcXJuco zxL^_`K1O{r*Pn);-6GG*Fm#Dq=;$n4vD6HL3|UCo@_S=dM=EHLGXF_>^&$OB!Uk*M zOxG_&4^R(p#zly@uE$*nkJs>Q{u$0~8hDKu>G^C?z3(b9#pPf`J}>T0+B%@-*=IN2 z%9!JM0XktUWo9{ere#BQT%llbZkzWjhHb)2wFH619d$LL#CMiFqJ*7$K*LmvpOPd1?eq`j-YRwZqIajd%LZZCwCG(nm`cePy_qCKlzi&-;WSvr-n zpsYZw)dXBrz*!5CKe17D4X6yw<1KR26*uA4h69$`u#&?s%)mI)jT_=YJd1pw(4@Af#XCcrIYkPMn*_uN)W<>RsNun{sb|7wf0Cv7x?`>_v@b+0 z8m-*IrtGXBAM_b+ey8>N_M5kx+>;JBQ1KJiTS1WONn(ezZ64kifuAS+g~&|X^wi~; zzJ^B@K|ftBDlK9lFX90~Ocd&xN0pVc9;vU+0(A4c~0p zkP$n+=XQmbQEn?WF9Op`lBxOw3>&8#Je%Nad{&oxUgrxgFVdc{x3Oji=YGkLt7om7 zWWy$ikkS0wcn-Ul^DMQFkBcBfkg?YbNn)4I59RULrCB-D!?V-9erUF$JH`2eB?1ti^b)yKeNM#tR zBT=Lse-0oqT#I0=m!y+A6x9lRT(=V1=d5MY(1NyTPd=U;xMast=gNl-fJ3rnnVUCo zS@6T27HZtKO1$%CI0gLC6ZsWwIi6au^!9d()jjJ!R_mtj{i*4q9Nm;^>Bd>qO&&Qy zQ|c+eM}_bCUtx$&&bGPRyI(BeO4Z_|>9{FRbort^&UY@8vK`q0akJTH*UiR=YXmXe zt$voplsr<*BoDZ2d++zn*;MK|`*=w{ICt*X_@$2H(36NiAIl)A+h>5jV~Y3yNz&~> zy~=pAdn&C*4%?~!DOwD`h|?%ZrK6r@I;4lMwAqD>U&VTk& z-7k<(y1v&XA3R_EUxDK^GQ+%bWAX%}nmucRIU{Wzj4R3}TE+)5EMMR{7*8mSZrnZ< zBQ(Gc-;)g-DG9WO1Whi^`z4UUJQk*-m9w%=nesucb{Lu2AvGiqtO$e`}VeIsLnYvj`Ro&`vU z-{;*m7->zaIFNm6Z@3S8_ERY{yJF7@tgnSXWt0Q}KgPh=%|3IYvE8@`*-vSRDVJel z$$NJP8d_;a0h4!-YZR)7()l&KJFdBW)H2$)VL(5%Y>ya>eCNtqEiywlbS1>At`2m~ zmj$kpa8d(M#w>pJ{n40?4#m~2!NC_Z=Zk$~5jOGT*O@K=NOv+gcF4x8qRhgeM*7{TycM3X-=f4_mQ|zy#PayK?OtD+oe%qrajq%=<_4Ale zZ)o=xls5ibNkDg<>bLbk-Fw{V-{y zX=$!rV0t?1u6WRD%2rHQk{k82KAfdcvN5J4%bdID$(0WEgkiM|*+@xWmfOMedqUyG zM1eCxGllejEgKLx&T-i;mWnT-eXsrm=UDwT7+#u_c6+hP{{vsV)+N)Y`ukFJMO9{t z2Np9G(?=Skx8L6x)2x4XqxpLIvU%_o2x3$&6PueTO#`UO&t< zZVlOHDZ&^^Wk2KCLfPs)b$cJ^nj2=dIDd?b%aH8$~d#amBB!<+~5t=H}L>Q6F0v@jFQqDc>lh zAyi(R;zN>@(7PjnhxgTM}oSt?{~5ZZD223%bqHH=BQjYq~;Qa+!i?dyYvo7Qd-);WG3@k;a{!b3VCM%jcxavY0_@HO!>AI|%94$l=SKj56&wcr<_@Q{~M(Kv_HNLGpe9vUtn5SX_`3(&~#L7&+&9R62 zrVoP*9tL&a>ID;%!=^ut$lT;6IoE%IotR|m7z8sEUC0Yj3ktHh@oe^l%tr!$6`EJK z`OUvl;_yO<*{PP}&-yL_ZAwTU<{={S?_0SHfLobqaGaP{v2p~O<9sW3*oz_tDASGy zSmEvG?|YDMj4vU(YbzEUs-k918SMwUu9uYB88FU@V#dBVxTcZ ztp0CqfX8hIewn`Br?M@N$*Z$W?(zLQ!^ERj?&daX#e`@Eb+om{LJkBJRAIoj8oS|k ztM$i>*LSLBbpT#HOChhlLVDH}!dkd754{Vl0&+9oM*E2duo=5ZC!B+k7XUCOPJn?k zp<3>O7b6f7qW`a>0)GAzG^2Yu`)&ESMe$~Oam~|rnSsBAXYUTQH`5l^T)Eo=rL`wJ zH@aR-H2J~FN7d#^z_;k!Y(RhW#@zP@kM_zPIbvSzDA-007ftfgoZCgn{_)4j=hE(s~6lA3zIzH1yW+66_TRHpF z{yLh*%=eS!h3xg|dxGuK(ywkEetpR^VN<$-3e>xWDFGI#X5Z4d#?dEhTvx=Eb_V#X zW1{FO9K!QBQgVz9Ty12t;=a(2pRi}N!<76^T;1B+I zE7<}XPq49|9{YIYEN9o?HNSH%k)&{)gCwJ5^FPofvxX1`{l37st#c=zm)Z5}cSd={ z3$Apuk$!{20`tDd!L$)2mT>Ug;&EG}|4?I8dF=rdl|(@<0y1a`dyj@9v4}FBH1b zPZcw3lp*6+=w`v7@*U41p4BT}M^Ug+pYXaS*sPfdK;0db;9XhVnY z-5+W8yiXKn+S5awgM>JP ztdluL*RZNQ2vH$G3Tc-t13g(bO75j^|WM<@UYSRk`w(GN}oCfJV#quOiOd zkIkXOoO8?mA3v4%^fZLtmfzLHX~w}`+w>SESL@%DzohWn(TjtMT+)v{Pcg6!^DCKr z_3afU?%|!-ol8G@fI88<^!)GI^YyEa0S&*z_3iK}p1s=IH)}T64shrE)G($s@4f4h zn>k?2#FJLLlJ_Awl^e|M9?M#4^R%43`Ho16#$O>IzZxU>*)roy_+L+8CVbXfi(l3? zh8)-mrM2Vk}po2N*Bn+g6oFS&y%!X$i7wIK9z zE-9f_Xq3x>9LIufzgJLhwy~vbxDY2k9>zqApl^WhOdxxh1eoxwbk^G?96wBu{4=Gb z>Dhgpe~SLcK(8CTsK2a2SfKLeVZ9JpC|%3R8a3sDB>45GRgWgW@K#oSFUMHxD)lBo zCFA@MWYAPQtqff*GS54>hO(z2ev%vuq?$(W62n}YM-2(i!xH}8{?NQ7Rgo0e+ktJs5=Vx?5h!tXvrK3X zz05!2EF+1{iKd*equjC}tgtJVrQzt1g#=0OQ6q#f|0$eby@FV&QXCHG-dGFw;exfP zoZ$!4Oftp~O~KFxS%HWpt=Qm+pA{{mB8P*FW=~gmV8>3zwC$Yl%FXXS0@6EZOOF8~ z_>bd3v57H(bJwG?FZ5pjC-SOoxD6`84{02c3{45N`Bh+R5^$D~csqld1nw;gq1^s< zze8f&_JScMikohTc|c|vms=G4aacU7BwbkQn^Eyu6&M4h?sI8qIoI2S=O_SXfp9Qr zUNQq@3#vX8T&Ulb=EF2+k#2(o@UpHWCw1YbDB~@0mHfG~scwwuMnE{}uAoot zR{A8@px?!aC0-U{#J@i&I`6&IipIn_5>)&Ilc5Kn$Gi-t;G^)@2SOD*4Emdzw?+z! zctZofRIghv7th20`L(V2jn}1_lt0lLn7|uX_*}}PxuJ=%+cfg}Em@}3XGzVpZP0HyZ?DyW50Dv){z4KvSs54C z1@}|a;@_2X&}>jz<%iJ-z0$L*s$4Z3FwQZlfrbhB?x?nWQ^sZ{j^@q151UDl>vGl) z8XVCiFZ^t@hjA~6l+Nm8^Ii(jyJQbw%5iU^eswl4i_79DC_fh=k8yb9Uwkm)Orqjz`5F+n;_cV7jRrji=X>=#9kBFUS)frzzg14 zVP4AMF1!~T+#E6*bM_U?to2eyC#{8#p*leuS>&)wki})iv0xq)xCB)HU8Wp9BMU=l zLk8!E1JBem^lALE4cUbjKoA23KEA+|@YxE1XPMRu`K2*kEzv48W_5=%&o70^o)vxD zv6lk^AG0daN>)(O|DTGpUfi}-OSxV`w#iM80n(l_Ml4w))Rm4^7sxXel*Fr(R3F98 zov0l|e%V^aU$gYz9~4?N7MY|uWE3C;#ik4y83(SjoS9czUdz(D_$j zr$GTh0HocDT7r}7uKp|ch2K~9p6LDzP9$6ZVZMBuF&k)u)#Ozj zc$WkE;DKM!$o~r@L+^d$dDDRvyZi*bK=7E^K8xdA;`;$7!2g}R7dHT2teS9<*xKm( zdS$@9L+QTI)Ad5;ChwK7=ZMj5WLn9sjVzp?n=5b@7QALS2Zrgvt>{t?U2|~{Xcunj z&9HRUDkEOtR8^JKR$?YxEaJg`9c7d6OhY5NPqd7F*ll(jo)FgeypU#-c%;)>8Wa(8 zZiU1Gv)mCOEhNhAX$D8W6P+uofDUZ@h^qdo_KK*0WTH9gJx7BJG7uZ!A0#M8EQc)g?v%~{pg zTkw-;!Bc-Q*(By0{byGLagTTQAIg7O$=dGB+ZKnrrbXQf1g|%(gLDIBM!je6>IZ{r zZvOS~Sv#yJv!+SsKCNQD^T#J=f|^|AGp^W;C_AZBJ7ptUc6^rCh8czir5Ei+c)o+x z$bOYB?LpmNNs*xB2<)uR4&zZh{p%tBDvbxRi<@$t?`a5V|5??8MuX*b?vo}=mwh#- zb;w66=<#M~=J<#7gAx00ZMP=-&UB$$YU?be6<&Zz|o>Ow= z^#IqO@Wr1JjP%sR0yziuD8rVAhs*PO_l1$;6?M8p%;Y}lDxdcJFZSj$;)(@uq8Lz5 zN+kK#PXAr~5mzEB<}5)YEJCJD92?N!;RZxjlLuI?>&+33sX5WXb^Rz4bLR)`Ekiorke8*?UuD(e7 z%{a+;*^s@D8!|i0YzeA1B3++xJCX`)A66Zk?1>^IZcvcRcKW($>#7yCAw^LueLlBx z|E9{cUQ&@(c5n7SQYSuy)0wWbDde%Pv~}N#?ma|K_Y!7!BXv~xJ z{ZEY?8A`fB=(M@w%bU;!)b}L!OUYr(SiNrUz^)IeJq+5k;DqAxU zVsg^qr(D@fSwYgRqoR*tD~hA07XcihfNtKiJG}t(iIn&g>fhFyHHDOGk7hTH%Yb;rQw; zu;1rYaZRglg{_`}z0$6ZX>r&LsZAPcX3HD3inf#~i~s=#ao?BtpKQVmv1;_809T}D zkeOvibO7)0y#G%s@0&&41Jf_XH6J#B76KCNfP=K?4vEzN!_1-}{DVyT`@4kDj%ZOdRSD)npI9L7gJ-FJ5Y7XwU6mqbe(C;;j=fcM7NWrEKUb#zKE<3XsWFC_#k{ z*Wb*=H&{IzkR>Vb=DfV}>nJq43+Hvax9AS^N3Zm1w@i=}PsMhs>okq8$M?BF1%8o4 zY<=Nyahh}_p=_ck?$-?2mW1OP{3g5dvHc*CRmF8hpTh(kzg+`b2o(ExUY{4hf&YdW z{1(if0+8<_^zCU8kIJV#jAug>@b+pH24(!nibm><%x9SXM#m8mu2`E|-Din`Cw(X` zZrL*%w1Np^nhcxi5cwk#*;5-=+txFdgzkjp)6E4`P(->_+eEm6$irF~( znnu1H#;0{)k(ldDe9!QE*~y7LTKvHSnH5q+E9YxiA7 zMYDAZcvzhK4_g^&s(ZPmyWLH6_Zw4c7e<&lX>dr>Clv)s)q-&67OqNclUz z9`0q#Lco4Kas@S=6cPtn&fOMop?$Io6=CIe{f)Rts`R$q#@l%JH-t58)YlSTKx{C1 zGdJ$q3sRH=3+ZGJ{h5Hz=AD=QX<;b|#9=vHWK<=eVmP_Ko8lau zBKKr|9%2nw7276446SO|gVgLhwN1-3xuF_meBVQ@23ZrCK#ni_Z$z&*)E3+E3WY0y zY{84htB0)8+#3SjuY)2NO+2wJKS(Lf<<_B@#~SZEZ)gu00P=}koyUP53vU~}ZZB9K zJ)4pio1}`TlDb=0=}*44nsK+Duu0S&3Qz9RsqU}n51mVLMZ5Yd;#P*65`~6&Jumz6 zu>(liTZ~7yartx19HPEgt><1)X7MBkAjO|Nq-rdAHZYV!iu#T(0OC_t!VS83&CX=D@v;_ zu%JPK@gL)zD*kpz@NCqsnbH$mlNK2}nXVoIAoRc|W-cFC6+6G(nXIHz?srT)I!-$C z9ee%g$x*l&b~EMijbEshEEg&bf*XrJ-pJYrW1jRmxc!UVQ3aSQ-MyYW?R(+TF(W_! z_HyMf_kd20zemgTJAM{y<4Q}lUHRKcg@&0)aZhN}#I>&hc4fJFb2eLsQ+IJ$ddPwm zao?ZUOW@`2iMKiz00~6z*ROm#jHe!89~lUmlD&ic$rOi9cOBkHrKwE5aHNr>^T_A8 zc`)-(=8A2+n{WH4YIBAKpt%eLZ*|M9GvhY=>S6BPDyKd|yYzh>YA12uhiKV5fLes- z32sM+!kF?p9)@$G*VD3x>q(AvE#UlEj=idmD-DO~QF=LssL$l6177T{%oeqYD-%%y-8;7b;wV|112T5cRitlgEw6pM z**t16M8pu>_5(1%LM^_PZw_!qQ*)-lSCk778MKGLcVXweVobT$^6P%w?Aw?v@gRpB zQq+x@(fK;Pc$osYaLW#8SdRr=`l#7`@MOek$%Ufa%eYrxIvlB1hO;aR#|Dk$L|Y%r zfoy;mg!|0cR2fJ#6GAJ zxxAG&$35qn@7g5DLul7wACD5tA2*tPqaCZeL2|2K5_^g`9mkscY3s!WaqO*dC7(%w zA_u+uCO`gUCOX18R;k7<$Y>CRGW z@UZ0FmmqFVu_b;19TLb!lzRugRrbzlz$Je zYs&sS6l64N`U9$X9i%aPFGW4coiLFkk^A&q)^}2W$rcRupsep(dlGL{;h1NV&`_Hz zSfTOM(xJhII_+^)bTl5o9lYvybFev#5PGFG@Sv2ZqTkf zBY}1eDJa^8*`9UGN#w1ns2T4`0p}oB%Ee(E3|Y0l8m1a$n;#;S*p>kUT(7DPe#GZs zUaK`ydP3v@yr55cYYrWw5K1TBl?6Y(u{vEZBcJo{z zn*}u#cn?|zUt5D3g?mb5L;XX?2kpWWMog>)&m!8U`QjJAdUhKE@0J<(J%X|eVJ=EY| zNIpc)Dn1QN5Ssn~DZfJ&a!|6qcfXEB`AEh6OMlJBUO&$n@i$sy{=P5P8KHYkZh+{Y|8WKfqf3$x6>IcX3OL$MFk>_3RLejplcw{k-~dOOcT>G-7?!BQF5cxXu(`r zTB<<<;cR02E{oB>TzKN_IS+ISCSyta?-z}n)Xlbph4^Tj=ZvY;Nbk1uq|7j#6XOD6 zd*G2&NIKhj*fK`?Lgl&@6GNvN5WAeUSRcN=RLf_5uzNWdU-Qaa~fdvK5z{)XL1 zT1w@wB9_NJPx8S&;asg_!FP-YXOhLP599L@Nn?;(7W6xDw@w8>Mm})&fZR*KZj!N{` z;Nl;RZ%)GQubM{9h57>uJ=dxw{j5p^F~103sTE7fxYmz(F^96*bAsxvUsg61 zw3YG6tk|o!RI-X1q@Vf`G-C|sZ%~%r0TG6Of=-vW+r-Q8$VPSz0pUCErNN)B9;Qrp zL~&RqygCjjKHr8b7#`YXJe-%2q zQGmOP1ZQXl$Uo3F4HH-F0uisjP;fLNu8Xx&knH<@ulQ#yK-jqsWRx z#TNd#tz-*=-u{5pMwYGIY+=)8O+4xCw#d3uY)>opP4;A#(!Gr*8fvlOW=UY(RM-(d z1?1mD?6(dIN*dgtBu7z_kd}TtMjU`^@j8mNo8_f-vVethmuNh)HaXMaNl7pU?zFpJ7j2 za8~vHEv}cfupK2=6(h9A{Z-2=)@I-rf>rY4eaXCaPN?^4Kz8@%=W3`gEv!F(pxKud z+-p7AL7L}CH|<6;Dn1G`P@T(L<;vme^{pO;map(yR^lSM6_8h|4HpOEZqcedaNVSp zsh)&%(!C05fbX(rIj@W{mXmf-(krIPVVw?Inds-@nO-DfN-j#06EKM(M31T(q(0VrnUFb$V=dh~XR94cCKbU&bQDM!_nTTgj8nN?8lF5pKPtv%;GBL- zPZ;-e0oBaUhQ5b&A}LtgEy$*LA!LI2Z;-=3fD^kQ+pD2_SQ^$ zUS^-!H-tK&sn_>2nLmIZvfj_LeJ3Iq0;yNaaePiv*LwxugJt|c1?qmvIqJCyB#jpF z?SQ{swP42y*4A38(bT(EXMSgfFf@nOn@JNSdVtBjIRE1*doi{vm#== zkfGrpJ^stY^>^hVLKLA@#`VZ`G zM~Tf|NJW=NIlA>={Hov9BL@Q@qxjKUg?yWsF||uzt+6ACz$JL}L}oE#)*WD1_2H2Z zR#TNu92-1ts3+V=*%z1o$Xa^GB+}@k!<&WI8!P>qedGvvN?amGX?sfpimPHt@K0+t z31aey$0MSmBU6Z)?bZ2%meT7mFD4@Co}D7GZ5rsk?!2zb+OMRH5q|{zQV~=q_j-89 zHAaO$AIt-`jqK2sBmGH$UxAe`h|+!&*_RJ8%`n`A7~GN(giB`I-}soDsbWSdSZMth zN^(3{RSG@|`rfR@cGn}@fSXOs`u?R8$Le?AoKKAiUJuY3+bNkgZO)tYacc;X?F!Jk zut!f>*wtE9Tr{aty|fn5J8X&^2vm^9*5v~KcapJE{fP@$ za}`aYe#6C@#X8zGhD`oE2Uaml2ivkyv9pURa3fY;OVX53xjXU|QmNK)h8NCVM|<n{^SD?h---c)a60N>P)ik;L;& z&zzW6Uv5&mCp9CWc+S=35g(op%(gVcwmGx~zU$m}GPTg9jHfB0%h2<~2#4LAMl5Pa z+UvaHx`=i7IE*|lLtN4^YeRlgOYxDx1TJKflZWIiDrh_%v& zXte1USV*k+o+Y~ek&dWW)US~)MSM^hFgd!TITN;ngxQsNPoBTWa*1cG>nA_YAvGE+ zzKdEptU2LxB;uK-lisLW9a-`HsYhJ-8ydZK4?-TM^PkMjQJI)9@vVv*XL9!EkLPC$ zz+`6VJ_ER*ucV`)VSS}I$(FkII>7=R@Z1^>NWz$w)@ol0YSN>V0}55^Ep=M7!1|Bv zYMszAT`cS;_0gL37kz$ibldN(Sy1?#T~*_pX2h`OgHcBJ!UU$IJBp>d-dZtzW4A*^ z+y4+-6^J{q8FMG1-&Vu7n@2WMoX)Ekx!cJZ6Ws5P+TsTp7L}^xuV1;68!iI5BYo`l z|IA))pY_+*UZ45ZW>HGYM;(Hk16wVQoJwPY5>g2Z?$r6Im|v>&?<7VaULR5177yGD zd-c7y6Rok}u*ian_>r(pdfgOvWzFC~U21-fe(S2lpB2o4crf`&BT#^Osv+|` zyBJ1)2XK^K8@CT5v^Uj~G_hvCb$&04z`$s&IgE~>xz}PJ+NhJv-QA=&j?5$t|79LJj|=!pW;)A#f`CB9;Lh6*v-TfCV&BHK66n<|F0S15 zy=I%;=gfo@8&djX*Pavwbpm4cn#HWjC;ltWo?mxQ$2PG=aqDboe%oNAIJ~QY#{T|s zu^Qni>K5kl4j%6`nS&hUe93Wu=O~}Il^9R*l^O_1pk}O#rc(?a7yrAX+wKD7dPJf# zA6v9XRzo#ch`PIMBJ+LHIqTeRiQu&A{U;+keMOuDesir-YDa`?k;o47H%GlGoQwbZ z^ntd7GzUy1%yxAPexz3VSmM=qUg5)pQmL+Tm+zqnpXa2jnO2^1*60=P#k3F=`Lv2p z1iM1_-HMQIGPtILsz$@$L7>(2sEOuK#~2azrXN4%M#RlkfuwIFiJG-kZjm71gLc^gN1hmjP$rg-+62IQeAYGeh zcFLNpkOPJ!j>CfIZoh;`xN&l>+zJPcZ^~py)_XyES@EZLwkNMa#9R1w2?ISnQ4D0l2>&$ zF)M`I0@9C3IeEHH)VbE&=O5E(*D3N}153?3;&LWbAU1L|ZHB~aWMP|tzraR~TLvM` z*3|{fF^-3NIpF{1K0WVOHN7fx^KaNQ`u0c<`ykdLMU`?YW&hNlRrF$%0eyIT8JndD zvsL=Qf_ZV2)V(}jkIK4EQ(rp%WJDrG*RDNu+)L|x0rl8FLdHXk-#pTqFN*PS3um&|p|H~9HH)f8 z{gt%PX`1?G+hHwjtLk%<%973brGCYS74rpm^Sw5osMErp{STYz%Ti5r%6izMK+~k7+9iwyRG@#LJHL+3w{(VpcaB;me*%N?nfZyD zXhl>G+k@I`wgpd;y1BcBi1Ei_yDDke+=F*F7VuQ#J#Uw2iY)^qokJIbPxMHKR@}G& zOP{N^d1Txw7;fF^x%+&VROYfbt{hg}K|a51PC<_cB}ej#STd;H#Jy&ZwOmW%dAu;=McT+yC^Rbo4xywrwOO-8EqF1qQ=lP0d$SpZ#( z-Qz`E2W!RUHWZf26thxMUw7tPU8Vm1lgwOY8uX(U^0mQLDC&%G$i|C*`h&!7S+Y#u zu#JcF#+1QVm_nq6+i4f^=^C4*^t^lrFd3S_kHif zQC$%YiV>9_K1lkvjL-Q#n?n8^Ou98ul>6@Cp>qn9&hoDFbL>1Ef;G3*_QsEP(alzf zlE_zM!g!n$SBb^)-W}ycTk+N%<*cjsxFRBL%g@a>T_R*27=&R?iU%bZZhpn35DM1D zZo1z|*Ie1mI9w-%W-cF8aVBOeUmHuX?}`>~iCR)2bD#s=8en@5}gMx+IH(mgZgE`|nhmpKtCptpeG3`BI?hT=IPb#+ zN!*XGRmt-nzl9^r)c7&#W|XAlym&vrwf(k5NZmv50r^%vW?0sPFM}e_AAz!^6 zO6UPvtA(#dJrhb&QE)7pauoCGu~-0|-`Cn=cqMEyu%8*0XssZ$ar=(RaZG!se4rgn zFCo2U?PnN6&$eEJfEG939>mX^_8;R;-@@&$xS*4)=p`zLR~EH)Vf6=H&QfMC);6Dc z9qRmxhHBD1`I9TVPP2k4Ci>X@m3GcO@PXVzD?@xRO(HL;1|F?Z6_>pi9ak=?I3!cz zuq|Rfv%4fIJPV%L{%o(9p}r^ngcu`qmLR$$4dje53d{`J$qyeMmXOMO&l{O5ky6+@ zeh~A!^d;2TuZefv4h!H9z}ZA$7EfzkRJU(F7}@VvQCCu z50bGl;$S=RENWna>u)7W;_9gp`W1|LWK~7{X!#hH@G(+1&Lk?I_RBxN@ZwLph!gxb z#K^n9CfKeJOKgUO7>_uJYAG7W;`XW&Row1vYxtNSsEi=44s*P5o=XQf)?g#4e+F|h zM6J#*7;|ZTk@0#yu9$+SP?v-2iO(#gy@8v<8qkVT)7<8Rp4_5nl~^bDQ&UrIv;W3@ zl%Wprs@{1nb8b0(C9J%z8?6FWru_f}dVL#o7((KwC$G^EUX6VP?seR9zJf2nvVHW4 z1US&c;rOSufoE9Z>>ZURxoYRL&&Z^okOV_*julv*?5%Yp>Vy@VKt)enm^-#(FCH^T8=>1(;erYVms``;U%yHQ-tQ zF+)eC;X>X=TX*gs$l+2|;BON+?`>H<%A0(uN6~xkIJ~Qtb^+PI_xVIoFv}{lBG;)H zfodq%->kfcA|F^v7CL(5LS6DV-BL^Jjb0n+@;I7XWKGZRLw@xIu(!q~sWWeqAp85;J$H=(YtEj>efFN+P7|s}BEO-%qd^T=x*qvL z4cyav$Zbq2cPL*oO7A1j@|o-x+tMbVC#@^}NMHu@caTR&AAngWf?SG9uRXiP=c;_< z(v39DnBISt;OjZ%>be4EYS;v!S z!NC8%?f9-k^F=kyjAE`xeju$l=9b7v_?1N3pth&_1liv+FS^&6QM{dHOin+j$=A8c=;Vkc13Po# z-&F8-T4!bU4~e(!x^~e}T>t0Uy;n+NXX>t#!5O%C$(!?h+wYIkGIhR=&R19a0B$;H zzA8KHM+mkPIrJ$sUT!7~Ufk4P7DXHQ9K=ul(nWgi1&%-_oYyzm@YDcEJfqU%BgmL| z)M!k_*jqFEjn08i!(5!E{bfvzLd3*(bKj0v8a}%@RaYzhkjO)T(epC$7%Qgjh+3Wv z?*~5q?qkQz1lZE0*3n_I&7076kL#mjwIBYP>_1X+WTjCXhuMMj2u9jgCY4$p_n);B z5e|)V?tpjxg!z#PH!LT8`({4Eg#~b%=3-?#w?2(pr9ETEA|CQ5`oc`#i~o_WKQWvr zv|=8-u6qOvW-Tv9y?D8fd#^d;8c>X`S@B5p3+W=a}sBVLg5GZO>*kLa;BtqtVYk zc_X)lUcPZRE8g!Lc}bu8ndM-4`y!}2S-%C6dwGvDH|%Cxz>^;pS25wUf!O8R2EOO!gt16VU1f%FwR!B@J{z)TIsoYFt`e15G;5yzAA;yU*VC;^K}=0(hMW5((MdCaGk3)qy^7=?;HH%6`6 zg~IAY(YKV=-GAI;Cs)JffqEsM9@b3A%cdtx?%rfxhYDL|F=nx6FOVv}u1c;cu5-&) zk)@jlA(FX+)m(>_O%H6otijFdj^6=l+|xSHUNWxY3E@^5jNXMllqdpzNm;7G9Vcas zj_q3<0+b^vR6JxJaef2Tp&v*7m`nIpS}3SwU0#m4k=%N`!#zp%5(X1*T2?a%z-`Y2 z%r6w84p3!tdl|)mQ~$zZ53EDNn|q!mx5E?t2&7qyreLrJN!4jKkYgDZv3UncHx>PX z;9D-0dzTxW?|GvkU7o=l-r;KT<_Fm8y)>!o?dSilyLe0ATMa;MkzRIMEr?`O`r;{H zhedr{-`6^aA2c=OGyj(n+B^rzqx0W$GE5g_`&<&yiIHJF9f_oljch>hv+%*yZ{XdT z7Ua}=;x89kR_&W0;+HvhQW&Rdk7 zllW6|D}&JMA_(1o)=lv#dFR`SXR-%7kLuNonN1`a2Nd;v%3>Uwd3KjSU~%n}AZIOb zBH5)A?|Zx}Zn^z>O-=Yydgc!1RSn+07ZO9CO3czSoZ!5}{p1WCx6kh4W_js5?fW!q z!V;6QIMQs{4VO)CL?5sJs}71CCvvKhb7y@gWE>6t3*kjSupBl9Rc1`no zoqaoFckQRG!FRpjvlYYRZiZjo48ytGYG(@r%9=RH{Xml}I2n=FBD5=TOPv2QhBA5CdpgXqfp2C&R zfq_4>A&m5|#r%DA+v@zpO{7L$4t3;8WF^IRL;sLu&41{oW4GR@Nyjqw`ymE+So_>D zO+Gd73n`}$zQW9x?&uzBF2QUK<)N?!Ju1}U)2s{Dj-w2A+u1@*-x7WF$GOeCtR)kP zG5tF_2AR*(YL_&6DfsXByQs^Ps$a+KN=a==t25W2KC@PFwKt!`Cbg*fZDX(52@4kg zSaXf%YY8w?;23#RZ4S5f) zpE?0lU(mV9*yOw`Sqgkd@IyR!M1;hBta9&`G>ugg<`UffI*p2k@<(#%?9dxEg=nqa z+(7%qYw(8TWdjY*O3L>)pxt_&<$bN*1499#TDFo?x&>+$TGqL0tn*Pptw_gLjmzER zee{oc;1vGZc=CHQRoa&StwTA5j(4B=_dKq>Cg^aH%>Z05_-FgKs`&~WqUSt$TJ_Vh zq!ZN0*xe6CUKsy0Iym9)E8L>Log#6E=(KWQ3X3)yX1$`FG&-odz`XAhJ2A9XY=Z+S zQ-%9;o=oslrJk`+sq(DtvSOCJdAa)*ZZmO0`O8S;L!D7etfHXXSBZU*9H#wVA?-SI!TT}7L)KvCmX0FlX5aXi4UHd+_yMeBa=Irf z=B6R>?R)lr<04k^EGvv|XIp^Z=T?fL3&L`@dAJ|JBQK3B5nWLQC(*5@&bdoqv z;qKGMq*u%-+XogG3~7<;rk`9FYQ#hQ%Y0J58$So1{6YiQoGXeX|Ym48l<_f|cd%V$t~^dGGUb z`p(423thxJw@03SSXN(7zGLd^+cnmcJ+Hs<<@ont+dtdeqgG_hrgC9GDv21^KLi#* zrUmKU#xEs&RVF1ReEWMl0*;htysGx)?kq;kJZ!XSD`C2;$=7$>{$`lJe(x*(cCqax zsem&*dr1g0w;qPSHM{i$qT(}fD`E&Clzi&z>RFU7p??BNP3)hsE#3gvnpw~c&coRU zTbF0nkK4_K>7O%;6@UNBs7#Ew=eI0!2fgMxb?`Ll-DTyV16*mP3_4u2?0G>_x zRrdUfaKMt}O9TUkxOUct+)eeS8t6Ss$q9F;Q}xhK#1)f!XoC_GKQ*Xg`vu^q_E#t+ zVG(&-fLcEDGfe-52Iy8Se0C2tVr|^`n&M{_pJO`Q>$-kxHm}{5gour9_zlwj;lnaZ z@(++mh1M;TwYtR<)&umHlPo{FaWE*t8J*$2Tvtt>d^Wf=yt&2t992@63mMJFQZMvd z`!QrQ-#PF=A+e5$gv@9ep!Tr8X=i z_~pHIuQk(V_RcFv10&%BjX7!OlFOuzj`;tehX%sD{I-6j{2Z*K7bB1TzDAM}y;t%x zFE6GmE=xVKw~UcJ-Z1$DpP)9vGI#wg8rZB912Pec3Pl^ozBTqag!WDcn=A@Gvyo|G z+*H;Zp=i+c%bJ6~9uDjR#pn zYjfE>7_oYm-6%sDm)Lqp&=+PSf^S*w_k2?!cXu`-XQWG;+-e8{8tER>$1pTcA}9eU zs0x^#Ip$kz$^`f>694rc&`9a4mWa~mtX$%lCFS&@E|fBcgT65&Gncj}`A5)%2{8qh>vib{NL}rtg9wgVjg*kW*iXS5zxtDu|40WV6MK` zekO$T442`eU-Na@DTL0QIxn59|U0#dObC34_iI`XMhvUc*z9-|4s zQ+c3~?Y<$Hpg=Jtf1@gFNxUzs?&U={N2{ask4@9XT>V^H+{VA?g1<%0$CHcJurvp-AEuY=*dHyv3FqFfLsi z6cLz$%I>}a%{Ncct~Zzj8sx;XytOC`W=NGUT(_Lq-lfE1-xQmh{59yBYtWaVa~d7d z2bPEO_>amMhVVzdFZzQcJb}U3Z+bPB4TNEz^mdUQVxuBr)vW4g2ap&5jJi9lVb9_o z>6v*N**OH(UTyD4UvbP-rWG!w=;15I;ynqevw`T0EF1a_aNEXp^g{0tI8tp4cJhSl zVW7#jp}FCUa=PX#oPmc6DB=OaVdH-Pr$51`usFvz=-SUvI_?jvd~v2I=tdFT?3K9( zihjD$X&#*CUOf2Y(NoYVfAp0uoe~;;M16&0-)Ur=d(RI?PFZ*!ixWEv`Z}F#KCE)C z0Or#J2mN699PXxkun&q!fR0WvS~_T8t^8Bk_~*&P-0MvsX3w_hkIJ-iot$o>Rq9qo$HkQ-yT? zA3A@LE8wtxvD^rJa`W~UMX&_pchPx?c+_7mI`n>2#_P1Gt`jP}&%X&<#a;(QEi|QH z!wH;XFKL6VcrT-$@um6eKEWB*wB~>MK9T#`a50~gi8)|kit6Fdzf;Fn{$@trqRyO# zC6Xqf-tRHStq678PaWXxp7D0#0FEBmeWTyk(E0B`8r zWcEQ7h4$fy)uS0W_{X-Tp%czcmSJul8ODK-vs|&I03aq_bCzRu)`0fU#br&~UzStC zJ__5sV}JQa3W#3<+g_9<2xWgdgiF+#pBphZ$eA9Rk@URWe(?=013evg!h*JgRtFlG zTjG8F%08!&`BDB=$C0Ar*3!yaCYxerVG8>SkXd6RCORF{dd}S3vc9i|U9{GPQJ%uY z?l>UM3I%F+Z0bt9ShZ=?7Nod{<&us#hL}F|SLIt9xEeOLmWe%TauK zec#Vryme?6P5BmZ^LFm^qto`JGw-Qasfo0djdPU3yxLI|U*{Y%tRy667@*Rw+` z-XhC#*X#&LSOPoiUo?H@xwrJSqMz@PBz!O`0DI0WolYHj(`blk(a)c|AJ34=D&ZK^ z^9!ee4ga5P823!o{{{Vq6N?Y8igcRB zA?|_i&sqKk7BL^DX-&EKZn`ay=ZoUIy;l+rlt*~7xI!{PbJEzKl*$ro|H;Y(iFI{@ z4iPpS&kzLdbe^m~+`|`C7IJrSWB`1Fv=8lR30R8R)+t-wR|p2Ajt zIUg&%!JOv*VwrC9;A>}+#@zr&eauL@)kofE9U;oG{6Cg;sn=)ItH}SWQ9m<}v}ffp zmDys!ec?PPfiFC@R6!Z+{QX{Cu4}hpsw7Z;oaav> zehjXA^!yPGe>EGv(zFj=%#V|`n#{XI(h~BU-aes&27~HH3R%GS9XZ3`MQnN93zS)P z(DixJvIrDbdTX}Yd@BB2vhO4H>j%&Le99HnaVGy+(T^=+ZF$8oZs&nhUo_7)V&z2| zrES(~xSv5MopnIeWD{gR;co$ffmo|%JSir9@0;eA=6C1Gys@o+0vz4l_Y-QQhRpwm z!$5~GvTpbrnw4RS-&`d>(Ixg^F8O6>0LF>BonIl!b|8jxBl&nsSi)zb*>_X(cb8TEm!m;DC2_~ ztvRit8F^!uf`-oH*uds5eWaNrLOIvFlA#4AdVu#YOr6kfJ;5ZL}NgK zrWQ&cr}gZRuJ;v{0}`ZOSK--({=gqy`;!jbYX9;~K}L&QCCHG$OwdApdhxk>`m*3) zCDIqy%a5MegxZxlh*D=4mn(-$wJs&MxK2Sb`Qj8R06at?e#M5e7^1R~fBWGjU1jHq&-$?abBJe4JdHiNh2A??aN2MEqk2D@$BnW~3Bqx$= ziD#3i(ovH=G$WIdrwzNElgXX7^IdL`Z=0w7<_p&>tkxvT`x&8iAz#2&oFiG|8zK!6 zXlTI=e?h~=l0bSVQJ$uKht{?p?=7j)exv|7I~H(p`?Wfdyb_&u!w;9d{B{4fB4OR& zI03qW5$u_H3spMfckt>uw-vU1ZFEgI4&aSj3%_FofJ(6v$t4@$cFVWcQOWe6ab+c4 zGXL-3%rMu&Wf6ifPe;dVu1hgad>1rn@V=?^LpaX%OeTM9oU=i9$nb zUr*QGLQ>fbfVTl;FG>m)0I%^^%{j@t!<=l;OaOn`S~%U{$qf%l@qO+sN+MHyvS`t? zfough|1#=*-%7M7H8f}bZt$d#8}{UTO6apQe(}+#W<2YvdbPT(HQhM1*(BY5B&?w_@;9L75j5^o z1|zewPG4$5@IaH=V!8X2h2>>fjx|c6+o=h59@>iWNl~~Q^XbkyV=KK6A#tE%{Bg+O z?kQs>w2;6S?TTy~I;F0u+ieGe4~MehFn$k^m%2F~!n-PS!DOJWJHydn{MDIy#VPFx zjr?U9mhZWE^e?Bt9rW`Dn=q>(+HFA>uv&p+gxVA@ux}#mduy`T>?nvg31ctj3yX1i zj;zCaUNYkrqApW+ZahH?cgQ)&a~^(RWom_nD+MQE5|&9v$p*6xkqrNP#9={BRWM~ zyutOIQ9ea{n=RWN2NO)pR6G%W|0&VseO_vF{NLu4gB%cl&`A^XI3`|aOd#{ui!OOPoY$q#e? zbRb&-E6{7nxFDy+R6a|jzQRcEMZdl=^grG6P^>Cx?u)+Gso^;as*Cg!Vbm9V(}h>h zrqBjBGO)d>A78#2d$9CoH;6r@ zNTY-spMc3G7TevD9b0WXx>wUz!gxp8{&2zndQc)rCPwzgn+{vF!NJDp{sHtY?zV=Z|!lEoh@2=X+gnlp-w3bst{QYtBBqLV^W>8iIds? zUqY4yxe%oP&9VONGj5gGG4s6kj`_C|xFWv6=Ex2<$vle@qYG1xl zT1#~^46zdo$tviVXsR$|_2vAI4y7NVUA!f(=P|p56Tk#lbg#_PC_-a2E=?|^@f|Gf zM2Mhs1<0qMKO7C z5%un<8O1v`4m9(uuTFgrh*wCab?Xgr3}e9{b6q*ql!@RMCU{E~y=^fR1!AgnfJXD# zRH~pKBse(3BBJLo6+owRwV-E`#oa&d89s04wBHl9WTzyKf3U7Hv#_6e;Hf+_ez-Sx zuI`B=TT(P&5})<9zfID-1aF7;ZY*y!nx$gV=exAXsn-0cgP;2~pY@qlUn4P#nR9XG=(bV4l6!7Sdtu2#$iy6tf(kxP7?&(rv# zIx>f8+Q%IL2(y8CKy+m*W*0B~Pzc0W2}OZ&n*+s&N=$#$Cj9~Sp+mIXnW00_e=zGu z(cK@;dnsd0P9?ey^|@5wIuf1eZxE|dvk+~xNe%SjCS=Do>k`YJ=jK!a710B!3~rot zGbK?z7{3XomU@fS4Ye1S@^R_aYm%r1z#2HF@eg3nr!g#f7_=S=M55+pQSXCFpxKJ7 z4Kak);@Xx%yzrL%Smx5{`zz#8vhyNQL=-f>yWnl5?oqbEzM zhpu?2h|GSvqSC&*URPsl1B_Up@9lxOPfqluG#2xN@s+$)kSk8=23(oebTVkmi4pJW zCnSd3oGD+f_Or8wxwc=U7(ZBc$Gu^a!mDu4$KE)M8kP*0`FKu$srfo4iIaQ-egF0k z|GJ{Kug{SQbR+O9*EQo0|CNA>oLD))f*XKxMPkP%ouyvFtUKr}VqN7;kF>ADLFTa) zFgtb!NpEB)Q^jy>bALPh5h@evnk%j+i7hS(7n=|4TjY$A0GqBCnSIwiWrwX+A6Maj zYa*kclb!yoINz?hw>XkC9hAL1n(VKR7y?Ky%y2nKX_a%W_0m2^GdGSzp8yZRm00gq z)eH^J2PBQctDtC=O5fmNWMe(KtvMa2k^;Ze0g;?A0+A`oE8JbwtPPnKtrNs~FI~}U z*iYV_Q^nX~9Dgqp6^NRvsJKBQ2bYvIW16@u`8^^vjdUOB`31lH_5FR$jnIJn_$#ys z#w*G#R4bjMno+keDW{m=`lsRPwl$H%o?hTx#G5EGQ)*w#ZkY+sa)JaS^a4T^m!ceG zbk5>d732}>Jo&T#sh(d+_$O2kVQ}BfZxELWf30!;3nsH_p{)wN2H4sDky4^~IVPcp z@wD-2#jHXO92+ARtcpSICF%SCa0i;$>t6Be>8t33U9~z>4rz93Me?R4xFIXI!dlU_ zMT3H`)a$~#ECTED?$BP9J>5G5DZA{#47IydPm}oFwhef?m=~6_D&}(OVrik|FP%fz zgU93dH#YPG`*%Rx#d5}sUhGK@Pb{r7kwjQK=~g*tg8;=fRc#eqced-vszqNsPg z_b?=Z&moj}Ls>QGr}{gM^kgLRgI8h4FGfkbAUwjf{@Cwlt%@?)!VxiFts1Q1wImPnU=KykbJfQ~(x; z68zRCe)tqHRS7++QTFk=bc56kp(`>@)MM|$Dc4_7Y=V94xL`;m*JWs%+>wR*Q80Wj zXh*WeI0a3^@D(TNoj{2ACs($$-u6kR8G)NzYq6vAqC%9>FpNEL zxCYK@Tvs?8)O->AgJ?%hvw=TJ#KQgGaEn8RMMEKPQ%Uy^M4P%x3#SSp#^zg)gk@&! z%jKsyakEQ#w>unO?!Tq+H}^zcSkoO)Cy(SN+BK$7kvg+-SVQ(@Gvr!WOw)d(AWd{Q zE&zQwD(-r$!mvcfAA|n5@dKW>yy?iiywIwUjSMBhp9H6q7-nJPPZWE=|FvuwX`YjO zdtSArQ*N*6+Gpz|*DgYr*#{pTO7`zbk1`B(hSEz*SIcwMiJtZ1&FQjZBbxeI7PHzu zO0b*@cri%$R%kgi8@)*36f_8LYFw-K$sP^w(v40t`WzNqpR~7eJPmCpSvvZ0&hPXC z4%by3-7y7tZ38S$Xx_02`n)c^X(D7nyO_dt3?H*a?KI1*>cB8AX>=@*z`cK`7?D`5 z5LC|IJ2(VN-G~uPUx6NQU@0;^_nMY6S)=p}<2+i#X557A>&0w;`IukC2_x%cYuw6R z66xq~;ZBXiXQKR@GYCsVG@sRWhqJ%Zt$@D6_01~TXY@q4kg$$s$rT`b=Ru0hV_^t> z(&fH@N??NO=r7QNWusm{)*cgfS#0A)ev{)umT8Cp)Hd~<7RWBVNILKAj+`VasXEej zZH@>xA|a>lze={Idi8FcQB^Fji9W2~?we3gX(MZco$!#(bjopwo_g2=gyXj`O>z5e*_zQE$>$0kc~>y6v*6j79h0v9v}V%3b0z7Mg; zWjR#qD7g_|yUW9h!0hxZCdVowfP5%wvk*;3b6K^*CehtNCdifg@kZT4#2iBQGBuyN zMrG9b)NSF*mLd9Hq=&8KyKPk%S9E*p^kwgVR_+xVozkC_o2ISMTtyb~^fFM>7Df4x zRk75N)CI7|v7+qae4<6!>kRMPBkg6|;QSZvqIvCXu)FyewMVW!dX`bQSdH)F?KTC6 z-(MZUXCHk*rW?K2k9_SA<3;<+(Hj1eFn7KqhJzEbgy8VE&NI({7|#>Yi$sv_qV%52 z&BGmpquU>Z996xcO_!bOPcMsTR{fi~{T)%EJHtYj02&uVtema3jSE;by{}yMKt6al zUC-;z@&CUSNTw1`Ql)x(LB$#^$Rtf zmYw6}uyys7d>T-7f~5K@ecP^V7xx4iyLlr$Nug~fAyG9Eq~~SYeh=Oz+%LVA%WeM$&p;R~JcWnJfjZSQFxte4^&e0gF8N0sM(q*Co zAN&;y9}xxsxKBy7yxn4&^@^}SRvwe~Uk}^E!{JukdVxyPVa<~Zbk!uthqy)Wn zwPY(fLmHx12Ax7g0)NhvTRY3T%`9Bn2Yr9QikjNsPY1Lx7iz|aDg&h9DuwQs;x6}D^=R91kC$DV5wrid~c-z zf>XnT4$-f4N{(YX4+*VPLg-|bd$j}>R3T*^d@sy3($Hd4V-N@0fJgrUT+$`U^l8!= zC|sBb5?=+pFIfX}1gY;$<1Vxjp1P6ioXH5;#-k2LmP&mc&2E{z9*s=|^lP3Ls|#&h z1OC`Cj9Wv6L+@XWoX5D$i?V_O!d2KDka@}EYF#J`*&{)LJ$CwX zgxR@(3E*`0d#jat{Bj^xYF@CgZyvaHQzJV>HftQWTIWY!aI!6XTfGW)lWdq5FyJs{ zd~2vecIGZ?C!*mTCGulFE4h~&!{hUBwFTXYXAQ5SrWLe)Jtiz^$#Zj|73;A%{!BaI zJH46c!)}s>kG|C7;q51VF^AfjX6v5N480u8#gc&fS;#$0;_aPWIgiG$IIL7Vv;xQ@C4Ol$e+zh%iLU0OKk#gqE4$|EoYx)iXUl$eRcx3La z;W7BGQo^(z&(fYmb7dZ(L@EqtcfL|LC9FIX5vK`<4H=bCAG|yFdD*P3P<)jpVey1~ zWHGwOaL#Mo->F$u6qbQ+duu@eI@uJvRCcc`9Qx#PF-MMQpRIQ*z`Z9yYhe83ckmo; z9~aeT$&0|scFD++3eVG@szGJP8Hf?k4CCO<15El7 zNU&Tuldghtfv!0x`HYrj4f?AI#7)~n12Iz5RUfih z=N*af)*3rwZbGy{NzO(6UXR!Jd^&bedZ?^6&MsKO_*&PlZ!I^SLAtL@(FeEWM~J^_ z`@Texq}sreXZnRy3J=6VtAwUW>+FmL?N=Z$xG0)6g^l&HZ05L8c#CATic=WMTkz+S z@Ddxy(-!=d3^lpTPZZAM-O1Kv#WRBkI7A_XT_*7W0lK5!yr6fTuFIl(= z6D;BcCFm2^u2-=y=1gz>06hKSKXX>p?)P^lRFsB%yd^G|#t`K!8(kmt94kwhcPfUw zK>T7zjb$3-nqxZ^2O&dP0; zyMF=|Y_IsJ=5Clh0T%~5l=)aUQ?YVc-extBcdJFDEFH+CRuV{8KU;^to|Y zz%AjHa0<^LsAogZE`;Y~j?V{t)=HkSYp~Qj{J+(H>Gyh*a50Fm4Hr(}X0&$GPx0cF zCyh(}KoSCaVyWa5pbpI>a4UG1<(8OxhFt~AXs#?6sKW1tL4^x2?j?!UX*-B(xw_t1H{YZF+N_CnX_5lD#$d*fFy?l+hduLuw8tLgYW`*i-j%QcRl8SMxQ%QkD)SeV%xuZ!-kivn>u_A)lx zxlR5g(K3(M7j&I|rjO6S5oYetmmjnI6})e!S5RQJDpvp^vK0lohL9s!8-ZZlo<8V{ zDM4${j%K%ZXy(Dn=TqfYkZLY7|=M5S=UokHLJhR`zM8iK(mbh^r5YeYK)?WjQQqSsb55;nz~0gs_`_DyB%6iPTY zyUZ$adn%5NVVIe*%_bA)_^$wY80~%Tw`+_mRhIytH;S2k(;4`&CJQgA1++5N;TYpq zMkX(s*Dwz7#&2@n0#_Fs-hLK9@+L4A&Hlp2=FS9sGs~L5DufDkg}gNW-NPg=9)F6& zvp_%R-QBR@Cw8L`C8(ryKi452A9p#FMa#o}JsMPPj(gl@P5L{j`-8Ydn-Lv@5BC}F zv-R7^f7cwjuWK*0^V=EglUR#RZIS^yQ_I!AXj~h+{q~N_yR~3&b07LsdiI=jFwG04 zD%1(#A3($y9zgIl&7Q#|A+k0tXA_B{TV&(@Nu&nOr_MX(B}dB8s>7ZdfMdj0Z*gPG|qP8 zuNYc#h%flGPRMZJyd_KW-V$-fZuIN(tEuf$x7^*lg9xEcJ0+IawJPtVZ=90yi2TE$h`x^48-sn$@f&%m& zqWg?Rojao+qT0uq0jRSHX4jm=122hnQYfr!0=xOc>Vf+~a z1CtM$i5u+n;EPSbDjy+E-!np#AOtHuTjd3ze`KeLk!Z_74V ziJTenYb?V(SuPpcMl+EqR8;2^_>QSF`FtE-nx)oI^!Ov0vfTQ`c!XB8y(vcRsveEk6M|DeT+DT-m^@C>~HPqe>dhovE z{E#34i0U#g;DimDSMccaGP+T?Y1{Z_5$x(C7M*CE3L913ZY~=K#Uq19AYQ9SkVffV zm#{(X!=jS7Q>t;U&O117`C=sKvJ7EGo)Cj-T%O9^!twCJx9U^551}C~6@V5qfTg#+ zC~*3=tQyc_NdBQxm~JGk)(C)e=3&IU3Wr3w%Dl;q5X6(Z z%hda(RGU(!4)do#LA9=fWH?B|Beo#lg}L!q*{i}*i~`pYy4jH85>U50E~v8@&4df1 zNvF}R9g=F0$hiDYQH1fq)%DlU#nK*ldTh8S>+(|{AnlWs+>dbo7lT-5Kd z|9WzRpL~P7kMBC!edGh(+N$KH(#OPv3=p_!6lA4W9aG3~nES*S3^tfSL<5OCpqzYE zJZV8Vn=eRsDdb^+qT>^a0^}om+=88dGq$Z!f34YHfjFSTw#(sFz;gRAbp7#tOYECB zgfXktJBz{lOZ>x9o9-^1d}H5|e0tmZbgd9ghYFi`S$&jeJh|__T|qz=WEYD7buCqb za)87@VFjIrdAjypf|u{_-%=+!MsjK2mTFz+Uavj7T36iwTn|?tCkMado1k+=jhfti z=92SmLo0VeauHU>x#6#Yvw(-Zo==k;fb8416ll%Vdjq~A2>)*L_ca;kA`xp zah#pmq>|j?Fe3fR?~671dl!Z;hAwR*-2X8tJq36)jNS%3;Ssflg;h1<;>_dA+S`}e z1_T2NavHoILl^9pYYD2sDoP_gg}pEgZ%G^K8>YLBtAi zV=`%CwdU}h%4?hp>kyWI!jncKgbn$|@}2K9l9r%#L4P~0;?rem$CoKli%oMEu_N0W z;uL6+Aj(0#^;zTt2)q}`SBbvYQNW-Yh;XIXMJ^vzHzgz4y|h&Cq>CbLcW~$GhX-dx zju07XhMz$0ys#x5hb_h-{7^xvL(0VSN_%Gt_VDy@KsQ%&` zDDxHhLH`EzNITRk6jvC3Em&MYNabzMZcIwT`XP z)OU2ik8#tP^@7N8lNJ%A-45a}CVD(F(3D@?T|+y#RH|*-STyr%YSi}X_XaV{ zY#G}yjy~5K0u5Sk3@#+xp2qy@#0#grc(S~-9KpTRmMY_4+Y-cm0O1h|@gs=xecM%b z7!)8_Y)lWoT^SFy;92xhI9B2!&)i-^=ml5N(T()eH)MpcJ_l+|Nii z0-C1bj|R3&W3M0V2Jl3opYO|4my9M7#UZEjlPNJ`gO)M{lso(k^~PUhNSyR=w%(OPaOx*xE)#4frnHhEMFfg(qhPVqGw19*r*UMWD2{IPb1ka@3xGg zN}vfS+pxRwmVr(kMQDPC!WBSvf?Q3ffls94KvoxQTMPZ6jE=|}{}g~ak5dE4a_ou@ z6p!dTwRsyaiXi$Sg)(j(D&@Yj+hgK6)7dHR(`Uyp34JLM^N^3KE=g_RpL01K3ZX!b z3A9Z(y_0#c1+5T{+`Il=vhE?TU*Jd)I>Pv^&~?c=;NQ@<(92Q*sV$O4gf+5Yr_d{D zt!&*gAU3f-l3sTFm1uRIj7*{oah>Gn;#0~ ziW$wU5gmem3EYGmhOJ$9eR0$&g##>r1oXNSWN9zHNnCfyU``#xSSHo{0UjImUuakJ(7lHRV z%jjU?yDECXUT5;1W*)m|sLIiphr`2zwI#A3z>{6^59km3(p%&{a2_Zp4n`4v zr@#<%!0q*jLnufHD1J@q4S!qVn~j{Ph)`>;tJo2H-oj2=lQK}VG&92^ztVB&d7|(jR3x8+VxY}SW)EhyM#~~gu)ntZF*KtMd9Kz5GUB0=gOB7-?azCu z7E^Yr4m~4ZeP2JoDGN2V2|PFYHLOPoN2O#7D;1laT ztwQ4Zz8A#gUi21JP{>QpYak(vB!x4Rtu(3GBnoE7Mh~I06@7Czl()rQC!@yj>w>>c z?@p)}7%Fj%HhEcO+>v0TVMZ1f$kEU;bbCr;lvdgtG zUyP6bmWlH}Nb;VPJr!v33pq>I4XhbKDt(3#IeLw>*G58fQMrik)Cm11jM38(q~Fn@ zB}eh%;VAxGLf}ySy$bC=^^Y7bHpibg{~2h3H~dW*OQ1L7Rk;eU%u`k2Iz(pn0nfhl0*Z3Rwy#BF%UD=^$mHKO_jaWS8jyKz**FYp#ZkX2FWPB>2e#Dz z=mqxeWFJxsp$`~r7D>3EdDetYJT`D>QBsR7R)WoTh_u1v8@6e{;R2k`ils+WM3)daBuuclBKu%?&MB z42t}`z*%CwTw9zxLQc<=Oz;-HUDI%ows zhUk|w|2dY-74B;unBxr2Dco-}9%{^p@I)_L@+4+&T(ClgoHE3Gb}i( qK$^y0Ey z8bolK!x_o6*q}&w7dYw0uV67A*<@4qHqSbQ%WCoB=>A*)q9l_8%vpv%-~pd-ThcFa z-Ts_qmka)_wKb3W^&hFrm_&)+)X~d%lG^O4=ERr0%y*Efh?d!k&R;ttWScAo>13kM z28i~|uyu6JTbr0K>6)8c$~EjQy$hZbw-nUg4XqNHdSU!=(MD%e+yIUHm1TNi&mPm! zeTTzSLe;BJlEn)QQ}SHyUvj#AHW^1yrb+Vbg+g=`y0$eYLa2TuEIZ^(99vsnaET2U zo$c6dZ{;7r{vQCRKv=(SAn*t@e$j?DBnKW${>2)89P^ic;x93&@JAxtwD{#F781as zl6|357>o?U6r>Ojo(Hpkq1OGf`|&k?o4x}1&o?yOJAdon{aYhPP%KYm?{XQU90W6WGe4y(pS!Ie zR!B59&nvO{)S96Nb6vI0$sPyl7N;IJ**s3igmAnRKaW3@IG^BE9Pv{87ykNR;6oq( zmpA%S9zZ;P|BwEDelhhA{`x=Q1L^yI`1=@F#76rPULFw7KlmsAAlnhopZi`Wep=g!GiGAnTlDaeUhzX8^Qr$dR(_GG^ZI<^tYpp8Ii{MF%6kH0 zmdc<-JQZh`$_?e3at68Nc_lmc3ySX_>YuD}G;Zl3(>!usm@_$m|GOJJ|M0J)aWCuT z66pH`atir`4`u*t0J(+z0WYn{FXS139K#=xkhjQR>=*jKV-q}nEZ`L7nCr|9%Qvxd zsOMDWTWjZW@`rG2&+QPO%SAH3pbtg#wt1&G39>z&62)10aLEhhRNu?=Zk4$PzvrC# zalB-W@9cA-lH2DyMQy8z^vKJ4SY>6ich$Q{_oC%ji7&#oRwS9$t~Av zo+xPx-Tv5t_&a1HH^|Br6XjCnR*HXdrJSrB*8bS{&z8gFkkz=5J_kzXvF7WH-1ZOr z{`mp&**s3dZXUN-|3gqzY~1D zl4wk0juE3^K=BbDASe*rUd*xz^fy?Wgb8{i!w9sHa&~vuf0H%{h@P z*Q~~LivjYXLitb;ejB6x=gAP}ShbH~er03JmPbGO;*YvMMq^PeD65|V)PMCqED-Z# z9Fuhx(m(m9pEUaneToTxkG|+pdy~v2X`rl>j#HeyP+uZhLnfI2GxtZ6gabpKB=8jMUv+l!qQ`66Hut9FJ%5EUs8)y$`tJtv9T#F4m zHmMl9FrLL;PjqXOY6RL0=F_YbF}Kd`na$5yTl3Uo)8-kQX4vJ$M1gCV-?1SFXpcZ{ zlO(%rzyX;nxK^8o*0x=EUbm^ob)*M+?Ec5rZsh5}3>%vCP4tCVJ#dxf-{J;dbUowF z2YD6DPx3Ff%w?a+>OmRTnLTx8a6qP8L!h2jeYR_a|1OL)SB>~x4^^1_CJOoYIuTa99p04V|L~O$jYiGMooEM<}x*2 z^j&!+zSyP~cZCt*<#h;^Ym==mUCx!>jA}l`nE~pAYdWZY$X)fvey&$N%9QI|)x)En zR&&x1x%&^2x$vBZAt~J@{dSK{O~1*!sBh3ULiPt}LpX5Cfq89=>QD^hD*WTl8)E2d zm-<+>H@rcE{TSn^{h^(qKZA5IWmO+*JoU5K=>QDSe2r^4L}U79SHlA8X9)xSCVee@ z6qZ1H$-z{OL->Tb#wxk5We!STEG)G@*`D@`U)GP>G`Q?D)&5RqYeT(lmc7hDGADa$ zf7kgNaJ9>(_q3R5e7b>&30clnH>fM4dRF_|Q9msEr7+;5I88OKH0fcoBiobogZ@By zl{qT%RO2e;dm{Cl@TLA$RNXt?$imEVq%zHVqg2M?LN%XsNDc&K!+4U9N+G-6){D<% z&}B5st>n1c%WPcNI~PTq#NXz0rc$~ry(RtmiwK9gC&}S2+RVRu``?*;mJ^ZWXeMio zB!^ixC9`=9DSh^rdBjopl&FbCUNvrc>v^hjtn+{%$}5$h+NP!Qi|{g0cEsyEr^S?; zLEe+(Z7;01X`Xzbdy-8)STBq{$x^>Xx%|W}&kKigIpuOR`{Hib6dv;7JjWm8=_JRV z%NYko87qoHD21egQz2S`mx?){ickC%c$GG# zOKFUINRa(W2W6juOt@y!I}e$0GTBNpoaIfjnB_AMpwxhMGF<$_Yq;Wyze|&+(UR^Q zyzbxHAgjr9eftRVd+p8FnryNW!$5wOT+^U&QWJky|H;&W-Fxfw8hbTp4A2Rmx5#iZ zIb_4-#kiB?Da%=wLCIw_SjAjBDP5DVm+TsO{2`{4Ys$B-ESp~CfZ56{4H#Jnh?L?L>WKLvmE_;>da83S*n?Y+%ho=i7ojeuDM2vc@#md_M zAuCL-sQ(;ffM-}c(qxK3t0r%`??c8oZLGo6M{oM5*+;oiG5Ixf)~mcT51B|ZlDWxN zR*x_uj9(*A~5)6!C_lci#C6Hi<#CCi_f!^yFy5P0yZe~}%0ENq;7c+r*-DfT$c*GO+Q{ObW!|or{FZQY{4;9G(%bZz(`GXM`P3hM z%HAZS3IC?;XYHdIi?INE&!_HjA3GK{V>g>LHrZz4zZNDsV_6CNb)q!gVnBH?g7RYo z(haDi=ryTU9%)R%{g3hpkU!)jZ3N?=3of}}=k8D49rd4sFMQMgmcO~h=5qB@OE>+) zO~!e94Ep_~;S|3aKn{=xWI+8>o%0|+jGNimrQcC|tu|aVYv<_)p7Va8^$}(s^dIV5 zs#*Ps$oi5@sULwr(;1KY+nBXzYTKNw%~iWw+TS{6H9K!djo0?NnWLR8VJfctuz<{= z_cWQ3KAG({DsRfTn}yEYwDKoVmb@uOZNh#I#V}sU2V#&N+IHs4Y=E&zXFP4tagA-B zwYfLf(e`MQ4nCUG?FGa`dr6!e#(|GMwM6`+mo&ihqBP@p{H{AN!#JPMVd$_7`9!|e zr`_uYUZ9DnQx%A#Q0p}8mByd&*meIu7t{mSbWptj=Er`MU9CSOin^B$hr(+g8Rn#4 z$Bj~$nOofA%`T_OD5pE=bzjDtGVW$K&g~s$-&@(H7kw;inSef)x0=&$03C{mcEVt6b|Vi!9n)3Kz*vxgBc=CYVSbqYiT>x{{y|DnLc+f zeJ%M#zP;t!Z!taYxG{d2p*2UeR~gZExqWTRMqF(kWNo7BvpL#KZ$ssl+tg0)ytcbR zE3T0~$KMh6HYGnGSr5VdOiSfAD@UiLa@(XF z6@KVdo+3{lcm01T;{;`F&d?xV_{gM!Q6c35RH1eoegk`5YvD9=#G{24>{48&8gI<_ z>mPW%Eod`2d(BT?v-7rl-)3o71%2r5hjxB?&rf$=|GwAnyy_jVvcc365DEr6KmO8> zjlc43uiW{!ul}34UH0&0Heh+|MUU;=@QoWx&y&ui2Z-;o-W(`xJcVK0)%4rtD!9&E zc)nit{LJ3n14>U#*BJNniP{@+&p{@}sG?U%z_nhAEj68hlHMxrPpc+?t zGyda0|HqvlUiQPCZ(jAyop*or-8&aOHoZ0+xn^R(V2O>$tKNOp&hsyLzPXL?qRU=n zvd!zV{{1`uZuNjh?w3FL<<2?Jo?{){SFic1HJl92*}PdwlgXBIQ^*@K#xL>g2DPSW z;%W@tTq(k-@MKt?6^cA=G%ClXax=4+-{`s^FE8|pPjN?@QaJ+h@jYLAkG*Z-i5EPv z^T+@E$ClT0IF#QbKYGO4nJa(yN-Gc2F#`6K9fW_%je#;{`RgZsZSl}1v1m!V#AJ(s zc_8H*uzXh8q%u;zhn@2%H*u9;El2m?SjSCIyjZ~9d=ik$E0=*t-J(63dz;G!4U z$zj?Q20C#3W*K|xyr=9fNPLFmnRA}m`N_+7UyEpvob75hLdb=y>DLspQ_aXuaSwkE z^J5xkG7BYLk={cD<&+_FF2}`kZX`F84+uF*a@LuhV%$~a&BB3~aaN4CJg%HRUa83e znJ>Kk3ue#vX4RJQ!~ly&^hGbc=!HAK{hxk&=OwRx$*?^`CNI7AQd>CUsa4vvmtOf& z8)yR-@Hy3vyuJP2x9|MRHUDDvAg8Ir%-CDFl!P3}JpG&_LoP=hFC@Qk!OZY%nNZVn zCZgtfl@H^EBuCN*A&X8knO!hf`xz0Jb>jFbiz1uuH^qL}9q2658QF|l8lhEx1zfH@DJ^kcsB;MX6F zu@4J)n)4uQ+U(ILPt*}-YS*Kkr!T;cpX;;R;&|E3BZ{!l50Kk0^iXZ%qJ4#G`{ z&VBaWovVLz_0I3#_50?|_~L>mcMkzDetGGuU%GSYn=iGt9T~yQJwE=E$E{6)e)9F7 zSer%qkgJ!y_GLDnR-eF_hIWiYDaaUY59>^$G0&ZZ2{2*Oc*U zcD@ACktxf$^j&?(b!0(ry!nN9;Utc~8VB4|<4PNw)$h3b@cV4mMb&4s7*4$dIW3cB zVw*ta({brD$u7G!Wj{Bg_dq9x9XuQ+kbkE*I}l}AiZo=>9Wa}MuOX{@BCog7=enHD zoAQk)$GjQB7VkfANrSIVhb^4zRFS~H(*N^{t=i%=>Z0i6Vz#H|k^sQIE<4U_$eJo+&e$f3a zAT0Vt^|Q2f_BoK&{?)$r=_j7HxgF_-SYxB_<;`TZuf6aU7h2!Es;?!_=xb?9=~plK z)df4(f8%<~bKK~Ab6*=}|r0t^gi-P*asV8kJ_o>zFeXOaob&xXEGws9l0z_W$q*y z%PO7n+Qe)(B$r*q3$N*v*?3cb{N+DBvOP)Ge`8Ow%|AML7E6vg`D=uA zXXkurUM{oz1#gtY&rlxa{DPv~;L=x0<+D_dL(1W$cxpNZQ=mF`= zP@YEKdbw3@s_~_A+oT*jz06IGu%`nuDgY~xL{_O<0g0P^lOahV3M6<@Ma4B8y znq9qT4S8i;dFRySbD~@x#=RVCIdy&#H%jG(K68>|X>xAqaZY(LS^S5${lm`h{p;`T zyyD+qVQFW8OF4Y&|NpI6`hm0z*9>zWdmntr?C9yh#{B7aN|r%aujf)~oNAQ@5rw`spu4=v=ds$Gpg$kWGL ziXMFVgEqP5;DR;)UiZG&*&+n}A8iDE8S=jhT6o|Z+8GuEnEV4KV_$jmS9V_bs~6fK z85YfnpR6M-AUO67D)UvSemePRTFH;}*=%sGaKe2uOl+OZk=!ZC2G3nLZ4c##AT!() zd_a5+Yev#upN;3 z)>YrK1!y{-js`U{>lU z8lR1qi&@I9tGu}fR3&Kx2KCq4Jhgk7+)nMN$l6xsKceD3vtRPpNa1HbBX5f%olbEN zx~Fn)cFQZwES|FAPg_J=$^sm3pZeN0U$eC!+KN?&VrZvc^X}K!9Ex!+U~!O5F6PH< z{?mTH;AJl`ciM6Ma$Vk_TQ}A@3TRU=Z|G@bjzgZbtF*J+)2ahA*x^^G8+hdFcdz`e z%`;j1;@}ZuH2hE|wg5^|A*-pZz{s4rT?fOD1+Z%e%%A;|Ze~5Gsk zc~75b{U>XeaQfIUz2!@`2Jn-Me`5VE?Fw$JbFx1o(C^R>Q6l%Xw1+x4L;uMk7!JU2Uyu&WuxZRas;_0-0O*H> z+}B?G$i`|DiN2OGEAt`RTFNNO3`0X#hHjR>Fc)qLUMQ|}jtae- zSMkOsy_@A0@yc&f{N!%N#7&>wP;OsIb))v5QK!7l$h@>3Td$wRlj~X|rS+)yBzbU= zwSU`_tPd`(Lh2JUW__ozra>>W_rKMCGEOQ({<3}bL;eMq9#r!QhsgrmmC|lsJcaLEZZq<9@|K2gU-@n894K@^VbhU(_p{%% z!3>))djZ!_QCTE;)$hJ4DlHuy6%`(vgiQg`GHJvn&2zuCTR0ssHc|oQf-=AE7KO+U zy_++)Ymzc^x~_w#?`%NixkKhGAHkhW#^70AGhtV#%5Jq^q#S1BIsTfi49rYC@KXm_ zbUvK8WTApha0W}8f&so7G&a8(tm}1soI=*dq+Uxsg5(RBge5sMS#&On#%!TcGX-~w z>(OMiEUI}XT2*$?Qu&FnU-aYQ=D1AnX|bdgbJN-r6?ebxZX2+2YLme%dCw*e6GSFc zoFL!J1dy_3(#1fTPE3nbgw0@rH(JGFe1BWNEqU}dp^ zjiDcW;Rkkbfj(?gAfw2Xy>;g8ue42K7B-ME`dHcso@}wh3orfIZr+J|+T1utDgG=E zrnld&vY*S5Dd}aFYUT*Tx#=fkK<{q||l^u@ROIxOiQGF^#18d}&NfHOY zI(Sx+KGztc++7aL*vQQGo9rj4NOsO8Q`KzbO4l2AdpAzkW_Y^J21pK+=V4ACkw$kAZKZ4V^m8+8xQ9+?5CeQ#cu|vk1XUco~d)5U%dDiCSxq( zFy~?XyacuHQ>#C<`_VsmUy%81yj;e~B83w?Bh^=H%LZ#AUE3$UQ*J-e*LHQ4`;5wZ z%`DzRIepSYb}!r*Kbf5iOJ)wn&_*-2Vxf->BsT4MYsE_*e~G>Bn|6XR#HvFvtnX-y z%d4W<(8A1t7-WPth4ze1F#Mjhjk-;sjl6XSW_aRH{gnK1ulN%fi)lfeGGMMrKbD_# z=j|c1-Fi|`SSriPCy}tfFi&jKRNiWjaW_u&Cn8>v2FXWZ@NNA;`=C-LeQ$6{X zw;fueP3_)b>mYruF+_eVpW#kw2d#9xoXm{9yGQv+KQMQGAy*L=+|5Fy^Au+ss*DM; z&-KF;d_c9{nZDIK)xSzzJzm8Jo0@PVg+1bpYIKCOF03-szSD>D_Hz1I z#znlfk;A9-srI&*JKk$=#LRsxZObLsUSe$u2k3Z941LItZvT0+&v~-X95)em88Vwc z;OPo}219a?^Bld&h5i zxue{ZwE20Tq(GHF`xQI*Z2zk6W66Iex0@lQ#q!K4=kYNA3z?rb=Mo?J36gBUQ#l5q z8dsV!&wG-*{YP_x!`zdk4dgKGQoBUn3ZBzbUVHh8%YJbmvT@TFe=>I@JLe;8Kigjr zR6rqR$Xs~3t5)B%!3`edp?-kSy7Rm?%y>P`( zcTWxKdztHCsU6+Z`l9H8|3((pD7ch|U@{ejA19ShE~mY?R}z}Zbq%f!cR4jq{?RV+ z4SYU-LwSOizV@Z|twnVvo1C((U@*eKm;rJwKjP%j0w->vO~Cz_;vSTKknTnIEM5`z zW9P-|juj*+Co7v*B$s=!`{9@5!^rI?=6?7~*O?-(C(Dk9ea65l(A-o{`8)uhq!8$@ z=)36uF2DYAx7{4b7z0;6S&^S|xc;8&?cf}PRZb(%Q0}Qh_aA(LYTeA-$Onu0nw8X) zFw@(wBp9RH&)kj^Ym%b09%#B-F!Vued@#n!Z5d-D+M)flW%iWWH5c0k71wHtu5q>@;T=bc52d?9Oj%uF5Brl9#rR)-YP-rInOknV#SS9@-ca(X zKl+rl*L;TKzy7!XYGsu%D%Z+U{Ic12BGrQ-m9au5+k8mw!|O#-NpC+lTl$<2fs^|W zxONJ`RSti0ob)nsJ-^I1fqSE`KB_*OwW?n~|JNo17yj}>vjdUlLDF^K$^%p7o<*2D zg;`-Y$Q(-MLZ>)8(Ms18KO;w!J3lY~kOQ-J<^FQ}pmrnrJKS8q*k7*Wer@ISt7cXl znJ0>Ld3r^5JDKy#-0ekXPWKIHBUs;|k7c7>eeBoY@OA5dZ~nWRt&i0qNkALG`V##o z!~p;{$L{&mJ<-Pk^|26d37`$(+OrkR{}@N)z7{j>CVlN!KlWAYYtOszJadyD4%P^a zM>%-<;afj!{VH>!Ti<=F5qdr=$+*YX04YK*Yn3N3z5OOTxsA9fc%zUxWN-zU>}ENx z)MVahow?dO)%YS!sSZS1p{DYERGMyfVxZZ&ZVtJv^0NfW?kM*p852`a(c5~N((d;h z_nu&W_@(?=M=FOfdpRiovwb#qq_hZtrQ?Z|pE+MUzSr?>;}&mI#@!5L zmx25FPoICX_as+mabR?RL%27~FI37Skn?h;U!Z8iM5#Wh+;aKJpUgp0O}{C$md>1j z_CkI5Ved(@*174Pr0z?|`ihZTXE(TX3Vk`ujP(a(2xo(^YNS?Z_H%oj+-&ZNOAo3! zfK{U^96%iq6EhV!VBo>TpGvpi#g4qFmNyAG*-N2A3@9sRw1~@30MKVo>cwu4+7?cel)?KTd3# zjwOGaT)P)=ScU$8)BAY?3b|sTeVew7r_D65&ixtVDmKHBYfX%FU%^$zwOo_A+8qdG z)yz|3oFzX(HJ|a|RvOt|`Tgs@^KI(B<+c}^)%hHbl*~@eYByfUmD#6sWzbKn&!k-g zw4KPaCVTst@B^-8oQ8a>e$alZ%^tR$ClneRUikR#<`;{(^f9L2XC86gqxU~*`tvXB z$DBMpl-rxbO2a9BGk~n9zk&e$8ZuS;w!L7trI}Q~+nsE0-0dsmFWN4(V@}Z)`k1F% zY?Iy>+(g=B$eRs$10qv(m${3>%@Kd2E^Vu*+?(#%xSJ~|6E=s~2tv=Rkh@>&IZ=mdLU*U=##kjym6_+GTOi+KXq6Jclt(waT$kX z*f=HsIVkgy|MC%g+W?!Wqk}RTAp?m^HiEC9EPp|(<^!uJE4Pa@xw{~J_`|sCZ0_MF ztL)dblR3vR<4q}?C4@h`7;CCsPk%|fmd6=~-e+I%XtyZ{bT9lS=4LZa+#vUK@1`&( zYF4}cihY<}O~FpwkhG>x*PP<)MaY6=Lvpe}(Z{LpgS@>NQSL`KC0$p;Qd~J5HC;o} zeP!O1?mI*FfcXf0>7W1IpWDHhJFmXe4#wm@mbV4u@e%C^{Q-SwJrtt{UYOheruVS| zeXWf-ZfW^()07{y!`)$-*0+*Z^u={;pSdZAljv)i|L{4<;X#=JLI$Kpva*6C|J9sj zb|lAH*0cM_Ed7uapT}wG*yJ#Pt5x@&-Gpq;_5rL$IyYAedx~z?YCRNWCxO((l7Hmg#R^Aen>5$ zy32g9?~7e=b2)Os?A*HEZ1ykxC=8H0pcnsgw_tcp)02-qIh(-k6$nh-B&4BW(})wB zoTvl3hqxCdxA923>qB`#i$;~BR-9T%s(F!`+7s};$4ibCp5E?02kBQ{^9rWb7AA=yiAjy})GbflD5+LF*0o++cR3m82@YE7P6tQQS%OG?UrW z2B%4J*eA2>hUCiPcWxLYbLKwqa}?;(u?ExKE%=vpc5Zs}zi>&0+7h0KVZbZU|FGFa z|FPe;OyepB=!8f62w3c)A7YVCGPVx+{)aL!pej3=OFtsZI`pW-R{=l#j2wQYn@%7~ zde zG~V=P@sX4f6)4?O`$Z08q-JPV_Lj_sZd};FJyHLuv)j~v@)~;KB`-8P#!nFai`pBN z&0(bB6u%kZjd+ZSfaW|}v|*mLZp=KyLVX3p`qU8ijdkr;((^MfvU5^98I;FGO>a-4 z>y`1o@sRl<4W8f~U4vavO2<&;GG`fFMJCr8 zM`m8fm&qL0t)PfQX-VnCtn>zzduzstK;rlt*&9pOS$LCXEpmT*lzWntJ8x99JxNdZ zBK)3AoAXP&P{@JuJn79$8Km`D*;gXUri*xuyV;PmLUu>r`9EQpr4Ox)3(2-C$RF{2 z(5yVxl!KIz_0r@r=GULW)imzc#RohQ1Dvt25jK4c|*K; z@-J>_PRX8Znv!6NmUsrw8xEdLq7-r*!*J5ppbI&{1ICTXq)ujMQUQYHh{GKPR8Fup| zsfwI$M(!uf`=E{M5OH~uhVG4)pw4I2N!x8U-T-Y93;ry233(8)bZFwB&~m_aXAiii z?!M4AlAV>gJBM9XMLy(z#^pEjrUFR|c705WpHYFzew#r~2H_yHyn2s8lm@?=jPVLy z+9wX_083lprntvM<_vRl*M<&AZq?7?HC4HN({@~b`5B#|uzQ#^!~3wi^n1;UpL-Zy zH$t7HXhO3M7+2MHkv@n?7=0l6JkYMl_Y>us?>F16pd~r$D23OwyZPok+lRli9J181(SY#?;Rk6d|FQW{UZ0x8f#@7b{~CE zp#Fg;&wDcTfJF?(j-z|q6k7bPhlprrMvEdf4UB&UZfh9rME>8Qio% z0~n+6HlM%zZ+~fT9N?hL$8P$V9j0efk22MPv<%6_h;K@(#toZ}DFSHS817gkFJQ8D zH_U`#zhoz+IIwoY-hSyiakER1o9X3$5T5yKh(5cGG4i4I!|t<*Z>b%P^i4@L#C^hW zYG&EVKgumSKSro*&h|}Bm+k5e9>%2IwbymY>J(= z?%nS<-+%B0vb^7eeUMZ(H{Muh$Is|&g58$iqf^|c-)qi;PNi%1O>4qw&Oyy)vNPkv zwI^wNS|@2watTwi*TU`(zsANYfS;UaSS2oMt&^!Rr+Trj?!ed9BM8;FrAf-&lpf%bEWORy80Z4$Q)7_dN`*hGLd_~V2%h0m$$ zEN~U0`;}P&oyjUls$5I7aYYi9G<0ZAa-V$e`mN@Zg2i+#Bvt;E7qk3vTFTe3Uo$An z)S|Jt6o2qZ8{GUR&Snj=HQTtk>Y{kHEw9De1}r3Mp@YT|y0AcJTD@z zK(Z$vm6zCwYWZdJ$Uc+YOcg$Y*?at9Zdi5Fmd=E6Vw5So|0y;ndM_YVDe6Q zv=Qh9`ae!10|v|$ac={}{T1r!U?I6_lAP^VT~|r*SA-pGIq7E{M)#V5`#L|8tECD5 zwqq=!boLi{(p)#)|nc0in7@#!v!UbXG=5Iv#a)Ei$m8hj8d*A&hLnz) zR#TZf#XY;SANK8c#m{y(?esJ|oHxesDIc}bfI6Q)@$leX5ANLksk`k-cAjk4Q|^pe zw`s$F(?LG#X?He20f%JRAk~xZxUqqn!3HY_Wf;Tp=_NK`IgH9788-G_^2AGKGO@%> zj}hkP5p{BzNN&|Xv=}Z5_k!%6A+z!9gg%+4D<>s$aQSl{ad|rMWB$${MB1H?VE0(= z&HQ6@%6>YvV9(E33dj7q|5KdVm`ZWCY=y9Qz5S4Z$-?YpZ_H()wikBzleJ#ir#=vh z{xbEa*=-fiWN)CszQ&<=3n{&(64$-hZ8`_$$K4^uIrOi*X@oKLXWso8dt(gilpL(! zfFphY{pP5TrTxc^KK9z1uRZN!4+zXR^P6LsgX&P~Q;$7mZuL+qV@mIG@jU`{7w=M=M6F&;JI}xN~;Qd0lYSd zPXGwK9+u97Q%%}1+%NPBJOoOotSEF&(cORP{oRSW3=`f-6M~;9+^QP*-4j)n+?ug7 z8)0cl@id*`nC_^lbdPpxetWWOP0vP$kupVNwSpAa%uNXZ=54Y!qjSKZmXp9hi-I&f zG-kXC3$Bi2zo1Sa%eii_fQ1}GD>rWr3Cf?~$}^0{cJ0lKyX(zf%I!0)EVrs;mw6^v ziZgTaj`m>bq%G~pwkK_MV1dm%+Bt#tk9Lrg zL{nn;&E1S}bU&w((RhX zfZ3|dx1p-CedCSo+9kKM1=HO$mF^8CRQph}Poz4%RJS|(qWCF3Gr}>ul70mB&GgfZ zw={{=0rHjc@Ikaer#QPoo#vAu zSynV=qAkdBRiW`|j8hxSc3EYZxwi_;+9o%u{ZhL}8<^WcnT@nrP)HjJn{p$CH{dyL z>4TKk!tGw13o~<5dKy}rcDyl$r`Z`-)~DGS8}KwWhhHwd;zAoU@QDZBa!7m4h8-J8 zw7aJ`SkO1>%`wa^>=Ra>yvydatdWgA38BwHP{tgTx$XCEvo($UlzWwx%q2@5$xf=e z?}c=|^TsAP4?AOWbjaTQRCf<`QXWS}T$iply*W-y$k$FG6%QvE-90r@`lUKI^2z=0 z`q=doc_lvm4efPZ)_1_F_&vqh0hOo97P;7ztlv}Lw}$9HE6IMujZ(h~w>ZVUe--aY z_u<@c(ElSl`Tzj`^hrcPRGHte>w*99#!uG%wG6 zEPXFTAFIvzQyeVxRvG$R@|eChA4(k^l;MyJ{VnSyeBuy}nM0{sBg~MjWZ98yE%HQJ zRwc{9T^|R)WS3}V9T}10S)iIek;|+d z&pVRJ=EkSMCki*Yo8{!?Ohw#>^!XkWxtGb$3V)fD_my{4!%ODIcJ1A)G>q^Yu_vip zuUSC$4D!rW=7lJeP3y?WpX{b2Om>Y;?m14mEr7hEJy`l8$x-Y{GOz!QI3P1x1lk0; zrt&3Bv8<1WjOS$3my$ce(@yl?+0txCkCVSdF{DN`NaQ)WHj> za;cJJF8#>7l9E~kPAi;`83Gz3}7vED&{R?^XR!V0p%`y2z!Uj zSsoTCyGt#Yo7O0svp*s0N6-mf+6)GhKz$Yc7V@#}#vzk2!Xq4B?+J*LjSD`rgg%~B zaWzA-l6%~*bNhLIsGeq$eRUuCS2#f_9;sEReyaIQ!MeKFwF3*#;I@vjkS!)gjI(Hy zwgG*bHZC=A2aKz#j4h#-kIHN)2ToOHMqK)h$kh^7xt%1;?gMS3Vfx+5t{COX9%xVb z?eQt@%ni~ieeM_VL%-3bIk{WksQ!_D7CY~6)qhGa^Ab0G z>1Qoo@n)BKrtUu6Y;H31lfB!?%$l;^%*I?=Z}ZZdN3>zaIC^w2hR@CBKKAtR>Cu7y zR0m>oD0S6A8QuNsa0Ub!qasf>5vJ-N%jwdoXb~3Zec}L9+nXiPKy?rUa z-LSe#PaHW+QHpyLYPT>^#ghF#aH94kH@L5sGZQC&oSPlLly}N&%Wqf8`w^FZNGS29 z)}i{V=HrI$OEA5RgJdfGq5S$vH6JDSa?5-~E$F$O$3KHx$&ZltB#|l2$&O-AlC_@M zp5zKlGxNcm9INH2`Z7{hf z+oRr-WSl`e$k#Ls8JIY>ijgy6{@<;$ivDM@UVlQMc}^0 z^Sx&>FUDNSt(p_bd<%gOzG2-zf7&n9ZO3B)6~|*)Ng&y0WmHZ-RWC z%9_6D%WwX&=>c-VBnnxh-Pi^ShqfXgUSKjtyTL0xbN^S--E$+7Zrt7MR*d!jDlN6e z8@J6Kc4OoCT%EC?stu}}>S0y<+`~%XmoW|-8I-pU;2~r5N65#6Uw_c-_6zk#EXom{ zAQ@BNv=Wr5%F@d@mut9Hvs0C?P35Ws#uBd}eYHQ+5ci%H`A~1k&2kUEElEkA+?sKh zfLXblg9$$UT>PitJI{&gE@;Q(2tp3qkV9YSWofeGs=HitbK*V+Ku104n> zkI8Em?$K}iWZo&xK2Y}_xZ?qp?xX^qWrw!!EX2ukZTR9vfK-c zm%p;PGKXj0`<3^)uU>YHq_Vm4tVVv8sHy4a`T>^QX7*@TghhO2Eq`vV{;N4qtmIGX zq$AQvQb%OW{Up82u*kENcT4tCGhpS`L-$jhu~2!M95`P^>d(~I8IAfvr`&H=s_vN& z$>XZ&P}*jMYFzgrOtT zHjKmVx^$tO+dWCiEvbV9>s80OC;7e0zqhz18+V-ZDUoG`u}O8;po)LxdF4ygf|v-T zctXjqt1}w9zjQw+bMm~X_uHh{RM-z_WnN3UxoIajmtFW5`85?MdQb8Y;W-j2+qq$$ zha1wmIs#cViUXD1%xO|?m`rkW4o>Wzq_2mrG|gI6=uun+Y-gDxB4F6yUWLfw2y!O6MQ#LbF^KEl2nR#7*D26FV%yHG&3jC zTo49hIJrubX1zqLe7=+6S}s@`zjXbZ56qG%=ZS7|4e}Q|xZj$Hb^Bh-okg^reK4nw zh_`!D7abtDe2#Pxvu}FJdrM3B@gDfo6XXgPXn_SJVLAQq1$fCWKc6sW66VbHuO3Dx zvc_?%Wz{fA&;NvE@a8L;A1!>GS6Pbb22sOCm`)91Qr0bGh|3(KK{9DQ)T2ti9 zR3AMoYOK%11s~AcBI)fI%B`qjIuvAH&+P00Bz%(`HKy@mPfNJ@qc_@#582C?WNfS5 zR%0e4yXLL9+r^S}dl&^XxAFs`ZKP+<9SG*=yRBAVQ;G)#h_fPV?bXbaofi!j_@izZ z$KLx_s2%eK&N8I@p%^0F+AI1Ne^0lX?0$oSn#I+_;n>?R=!WxNZVa74jc-`x%XuXd zBK^LII%utSo+kHjkJyJkeZrqr-n*Z5bNS$N53;UhsS@gS(H}Wx585{0j5nFaw0(fm z39tXy>v&7tS@tw-@2sZ<7634Afkn1u-x$nGCM-j#y)_i?g12uf=}zW@E`s zTx`0xtN*QnX)D*pOxPrN>;1;0%=@eO1(o`L7H*O_I;aBmewh2gWKN|H&4n{V>Iu8< zh%myx@o+&t@J%h?J;TWJL)@o-BU0gOgV-G9^>f#x3y%2vgE~l+jIDIuZK7u=y?**y zW>f1y_|L*>mP4MV^`+zmpR1a<+Mj7+^MMQZE*JmYePK)}OYF-}oY$|*+bxuQb2j>l zq;Mws(cD>AxZo!(?{U{t;_Lq4qEqG zZl8&}Q7FdJUgLk+k<*{5aLO>qLrwI^S3BmS+T@-jC3PJ0=0_5U0NgE-&F)uIOsdT{ z1>6X__)BpESyN%!FADZUUMmmSR87fM<6 zy(({x#odQ6@T+Fg3Pq9x*dRH6iYWJGVjZTK%_gQX0zt9<1DHTfhR&0Z-$j2RWF35m zDs$f;0_adC=Om^K`G5cN{*sbCA5rNv;^}!YJBA{cmD3JjjHiGO1~G|z(k*=Kq>`~>EiKU}=5(7U;!lz2_sSNabRtR?Gt7He>r zeYbYvb8!1-(atAo+4F9dIDM$BG()_9NQp)is!XcpaGra8| zq5^piliFjr@AXiNgDRtan;X{`QPLhv4uR#v%&(tIq~6^xV^nrI4hgK1P3{Wg1Ep9A zq{G}xbv;Y(*A09b8*?+0Q;{xHAp0xyZF~DSd`Z^SBvq)z^wK(BH z96GGlnS|bt2hm;54nIF)X?fdvQsd&j)mlvMGIhz`p5f%jxcHvW0eDefHt;f4v7Yiy zGVuy)aELk0g?*Fj#07@MI&XT(JB~tfsN41krhh^1PL=_*b}VP{pqu`jW$esy!{s4# zaq+-ECV};sOJh{w!kIg8RDRbxHcR>jaUd35_>IZX{FlaUV&^~c@DD-_0p&&#j`3`5 z&rw7ME%(mu6mL%g>qm#>&hz`dp?6iw0Dr=obaVghk@$|wf}Ch`pTx`ylEJ7FtNk7q zr+z4=h&c;OZgHi&8?$sWb+H$4GYFcuXPABoW@t+q6ilTw zd7o!}__r8X0;=br?xVLgLZVZL`>=GgQ@3Njas9SB3j>o>7pvJaNM6ufw7+@a=*av5 zBnys3b9dHvM79~8sEdao6vM6%ScB=WhQQtF_C zVrZ-kHdHs`Jhe72wT>V279IW<@db}33)hO-#c+aerbYv8*ZJm(1MfyUvz+&+2WYZ* zzp6>mS#PgUV|JL$fI9s=xVxrpWxn>CT-5IrWWvUPl(BVtYswWeVB#Q*mE@&bSjh&+ zPf?YisJEzV!ck_{-%osa1E>U#9oif;c%QgsWUMYDZg1WtwW|TTN% zB#~4GLQwM-@F^|<%3D|fmRr)LAO~{=nJi3fyed1%t-b8LoEA3D0&}eda|{0rV8c?& z`y~n*-@B%ibcbQCk|q0Q$^R)cwQRAr=K9_J#h+}7>bwoBHZgk2d`x^QD8pjC{NeVO zXIEt77NQ+>_LEV?+Cqi)+hot?ZU~`kuN_@j{}rVMBHzu~R5TY>8q+<2oqt0tD>nHnGm& z&NQI zHWJUcuOHY@~qO@nokrz7O#_-4CcOFFC?7^B8 zZfc^Zt5|J$0z;bIOTSVEe1zf)wd4LUwe?=>WtZw@(#$QggMY}3(=oj?;Uj> z*+E2xy*{=ufpK(h>}L9Pu3S;eEzeo?;q5&go5?SM72OrVQ08CALfGl=+=n|eGT}Uy z!-nn;I1Fk{);3_Gm=-aQ_b#F}emPDmGtpV@KG430{qQ}lnIX7>TJ?2Ce}g+f-AmW* z3b9(r5x+b^3R1*vRJ%Zugxbv~2&vk)4AdHYqwz*_{KF z`p~k+@BTr$W}X`|Vi4IFI^Cnlb*32}fq~SqSpZq-Ye2v{p^0a4-Ap$x!9d|riO=01 zLQ3J5y*GB9%&zj$RGyBHUwIl6uj2ez*C11npSGsgQDMKll!-vL-Sg0hK>|J@V-cNS zQa*t1Ee5Y;be=daeKEbwjOI{fwNL#i{DEFDGyJ^Zh!_`+oob%Ql+Jn~!R5U0vnwx_ zo=%L4&}gCP>D*>LGYPwdW%c!<176Y_vZJ$#&W@A1Xs8`2kIuPnx|n8h5=CKp0PTz3 z{2DrLr{>z(Gdq{t-jlNlk`0qEbR7~@Qlwo{X8_iB_aSM`wBH|(Li?ZTm;*^5u+H0) z^A9alg#%7pws+?Zf#O(6l5YaV6Pu=I2Nm8-wv*JpZFWeV-yD|9nxwq4>0C(sqhnWg9hi76fjbMGP8CB4q}NUpPg7-{3>lLA6CtVg zDGFS)8&8qG?>?0Od|gmdJaHy4;6`x{Pk2tF4{T4YJ%LBtS0`OP?B)c%D-bP3A(^9O$#&-Tvw1cWLh47$h^fT&3CE z3Jk<9ADZ4^oqi}RK^OONR>+2Z9>8QSJ?P;@->wv(?-w{AW_D54#){%)aD#1ZzM016 z99LA@-1v$Ni@2kOhsoS&L33;yxPyI0B%iQFD$G(c+77G@{v$^z=mn^>S*!D@dOHQyT z4}StyHb%%UvV(D$)8zrFqF;z$@ZHgK#RYKx;^9vj;~7?t_vOML!{3dldLMnn=dSW= zu{?Nd)y(u&liN_9CSj`VeP^)jl#I;j@@Fb<4{>?E*<`uBu31kdLZ?*xfEosR0OS~u zAJIR>JMuUqn|r|7OKEWTcf1jzSeS_=p-YWl6oWEa;`rQ9iK;}cA zhN74K`~82!qwew1zdx-xY}?IoyO#=<2E~A{=*?ZQ&z%NOsjI;kPL+p?cEEo;O07ls zrJa}Krz?27#;1ZBM7rDb+sSWs0B*mR|J^HRGzPkDKiPqA!Tj_4nAzVo*6-I3ZxDDG zU7PjkUn!MOM;djm;+7beAvz4TWeAgSKD*k7P0KyMN1oPh57d6Eo^84bsB&%B7p!)Xp7hb3P8@ilGkZ_2xcP5U;Oh@dq z?G+vlppX-VuO+=J{gbxwUgJO=Y}ANc&mF@rR&i^#eoMmEJNJ;4N(o!xqAM!>+Z%Ez~}k72WJE6Ra+CHO&@1+SZZ| zV$f3(=r|gb<`X9&2`F)+KjM>>-|q2A!;4;KZr!#%CsrmbefTwjB*eWm)H;;-;#DtZ zsBZKWlWDJccPg%P=)OlMkKHRIZ{1`v{WJ8n;HdX`^~5Ke$m|J+?fwLon}V=GqrAP? z5_G@17;7d%><>b}AYqXs%gQ^mmjO-U<6~CgDxlfBntdsYuIJ53JRr=CvJJ^R{3=#6 zZn^UL09?$TgET# z0}t!YWi%S(+p1Cm0&$kcHLr{t0HqmmAY9@wb zc*FrdS8_axxJVXvgYeN<30i!jL7U3Fj+?C4Y1`FNl&Z69GDW9m*PEiAG;I|!u~vy&tAYD*(*4o zYi1|Po|&|zdC_mUa)j*MErGXWflBA&SqeKHH|`C!8{s%l8`~~tNTzbbeh@B3WKXNP zP~~(jB$o~*Elvr08(uOwu}`-kRt{$TPheNHw_0H%jEu73x#mQq&3j?>y=NMrSk28I zc}8^u?X7Ty8&;EmbyAuE`Z1d58Hq_nAiMRcS4@RDcEn(QL&I7iSIS^MDz}$0&;2J_ z+17)dfAqr$9HXUTsx6|C`^sg71eh~K+<)mIWT72p;YO6|Y4_noR~aSnl|kYKmdogg z+D8u6%+sDzO~N`_NSc9<1!o2E%$k==%7mQ*6)1|?(!7iExd7AmEs^hYZ#UB4e7c^& z-oHX6vp9#Ot#`RaPK5i7CQ=nTdpu>?}9-WRFO~r=cwPD^7&38nw zfKxU4tRv5&4hJ$ro<^M;4Ca$R+W``EUE8k3)zt3@dov#9bj;asyrcX|=T2k|p|eTV z5g+mg7Q0whFM)j?uAvMdDy$~)TW-FU{$8c zO*3?~hI3Ugz2#n)?-xJEa7|;dE~pxV<^RD1d%&V|z4sKzSR2Z?ZQX|dJ#e*G7`Mhc zP@9pz9b4~9eAAV@9~2AN>xGAPsNx*5qnlLcTV}qwp*dT3U)8=ji^se9vLj?}2UI6) zbuPfo{f`FEe^o1f-Y$$U3bbJie(^|YaYIj|a^0-2L9wHjY z?-4e#2Yqh7+4CqEKv0Tb4)|rVNE>TKbMm-=I_a7RaNWJIN%tH-p)msKTz)6>p54do8%TN%s6nwTM??lwI@{EYvsz^=+m2a|2q^xc2GSlHuPFK69a zIDKk8!0wvXCj&-!tvekpy`pqCSiG(UNWWx6pjL_xFnaTkkD(s-_wJcCUg15{8-F zLNu?3E^^!qGPrBZ1=ksFqt~#SDLwzyN6#w|>Xb5A$8WMR45e)H{GWF&LKi8Uv6uKL z1sR6WN&C)W!s@R^*n2`#Lk}3tIjb*ja{Xqs`s?{&gmR&g!-sFb)aH@{&qu zu2J4S@Yxr+wj(?U8%Ds#KyOGgWKJlWyO*DboZW&KS={8VhI)$nfE?e~H)jjAUO=p= z*FKZrk#+ra&S3(}mqVHswlHM}fNbBv2W=%sbWorDE&yw++-yTkpoDaB_OMlY8!l2s z;d_Z6YLU+ciJ!$mz72)Ko97$mV!Q#N)asiA{u(apTZ)hlhDqpT{Kgvy_cw6C=twl%t$~5$T=8K^-_w*&`!A#2OZ1#Rut#2VL z#_nKcn*E9&lP_StPb^f7iHC=N#xnx#5nd%n`DPZmu))Tqrq-t!H!}Nz9!}fTK46w+(JQU4 zQAmxlTb=dDeIaf0na4#-^Q0XJR@7_5dl7jEDtu7zDRJHd2Cgi z^fiA6#dN#3wSXG>69lmHBd-L=mS3^xA}i7g-0DRZh}EdC{?NVg$pizP}@}+A!H{#n{cba{Z zf}-=E?gf4jNdVKaq(>YK(;(q{EwE^)O*h}X8cCR2suXrcSg28zV`HFGD>6MNOa&{w z%n>&7cRpS|0-JQn=(#2>vEUELDpdDmJU^-@PIgb_wxS>d5<#9bctDd*Q%1Bu#es+f zI~3Z7L#|r7Fj2j93bIEXflwImiK`OV-0`0(`&uqJI0D3YIPK=)3YR5kTY3CKzuqb` zpLA&d`)>N&IHya>-23Le;LA`A$nvt!0g# zhe=ny0?h03hesfCiS9S+vpFrQYp~dPbC+*OQx=PI0M{dMzSh?MYie`1)e%lmTaRQ8 z4eZ~rP|O!YlC`pPj05K!(?$Y!Giz>8_~|UsM)Emv9Ib!EIq@3Z8W+Qaat4D-SMKiV z1F?N>56$T!LqJ!qt1o4|y#8{ZEtLhm$+64{Hg2>LZ@X}mC2+RTR{NEoQ7P(n+x~Q; zuAQ`{VLRZqaxh~jcIVxnqW3zzCZyTEW#LmD1r&V`%gIw6+oP_z%I!*IM*p1sI=O_r z&Q)A#US?5l%Lr^J+VBllSJ-EN$<4!pFM3Crw=f+u+`GHCb+i0 z21H24{U%Dd)>#X>Ec|}u1R;a9d_UamLTXg`MxK3tvd?i`s`x*P8eHypjB=Wsk_l3C z?WWX``yhdNgLM1*p*`l8VKfHcu{x3Yyu$G0j=A$h(OU?oS>uuof<#0myfK)p-uhAq zXc{u>;|}p)%|tD)%+ZnC=?QA4V8>)pe#L&UeJ4l?EcNWqb46L@HExm(QJOm-2`=cWO`p0 zs~Ccn2&?YX=z2s2_cDY&E($IQ(*O4hW&g#5fiD2cWS`I)nPSfMC*UN0R8GXGz?y>Q zNTk$CBrIr;`hyLjil?gtm8WJ<+?eFU`Fp84PydNiMrR}eL+c+rH+^Q!x6rp{oH;y# zQ*EbQh3L$!5G!wQTh1z$t>fkck-7I3ESRapb$6d{d* ztbL}TPjscVET^NOD|-{uF*Pkomjv+_ujZW>MAY)|CQBX|4&V#e{Cg2S_t_BFJjM~| z7}OFrfY>u;x7+;F>~;GtlzxD(WMEX~cYsa`vS!Ypm8IL^PcM5NXArCvPJ47JhHtS{ ze$)tRA0DLq8KK_)vh}1zF<|jyh-Ge@^@_&T`0eNSD!4pt_k9|WzV9_yCkxBxD-?gC zn3^nw;_R+UgYAWT=NEsi9GwdZN<|jBO{YJ%1<+=_I`sbSexZQeU^^BtnHc5Cp$nTo z)7V%wZd>D&SJCTktta^_%Cj0oj6WD|aecIxnk%3bghLzZRlok8lztqS8H7((AV$-A zbWEbfw9DU^YJRLuC?+$`1Ffyu9?r2y7G_17{B5=PK-5HCyEHLtfBZ2c)j)j9UE+i| zNiL(Zxh*5draogPuIWhQZ*Ev>%ET2;9z_AG4fvNIzBhHLB&ITdilKJ7m^0LdSNj76 zYDc=V>hX$n650nPd>y0H7B~QXSGLmKI%iwu%3^(I_`U}%+n1t9Re-(p!=<w?UIK)d`>RI%2FEe8(Q6%U>$TJn#kffTEYUx>`8+d1L%B|I;KW#c>~*df zk&Me7%mn~{V|P&t_M~u~ARD-MvoXg>{&${G=x!S&Yk`4vdOkHZ^J$`tTG6H>- zYiNU9i8PczW&VTMs1soIIK~GJ+WcnL6tE@%P7W>}B z4rs~+y7;3oVXiU{X9cKR{Sp-NWm2_tAy#zM=uK5|N2h z9Z7Qk=t15;In z$5sx*W`~gBHJc^1&p;)!I ziWaZj+ZmE1w`4Ft`J;iDju-OnS=ElXz9Q2{G9OM(FD}Leyu?G&!IP)4gEeSJ`wjI0 z?|KR!cft;-m%7|erf-2h#NOU#q{_^Gxp`Z!GrLSfxeCeI zzF^Rg?RaoipfK4#5kA&G5kJ=Neq@o?xwh}z#*6w-pC=on9@xd5+GrHLK(*s!L_PI| zhOBCgd3?EcT1#d&!j>{#fm#kH6Ag#y=L_aD+vF$_={h9uSbdQC%6RZ(=HES1D+;D? zHIG1T+GB6JIa$V6w!pPu4WYl_^GtgmChTg#Ih?pk-!uq%DfMz${hPxf!XBNYBjhF$ z;{SUnBLtczfA89CpMb@J;b%hIcclz^L^j-?B)3|5{`fHf=ipK@Jsgas?2icS@E283iRQ_nMXcsDao6fC7tl>lW>tE4D&Tbl?(bZGiSKTE2KOxI*&vaA zX7REBTD%KNx?v;TfV1nAOpwR<>!1-QxeLBJVm5uY{2Ky#-{Q&p!$}HV`BQYHEmTou zXaq`vuX>NE@F?3vPQTcF_C+sw+u1y*L0;|rfrCe!pH5Qv)Ia5jz++4Em&jwDP9xef zj17RNKduB#1@I?VJmGYPDya}d8hwl_12h;Ds9U_=9|(I~t)#4aS_irv%EinI)3EGDQWL80tYnPM6RrviFMx+bsxyI4?oVugBh!6b zRA~buD0}q=PHR7Oj)W4niB78ex(Lf^n5fOD1*iH13_Bmo+^5p_7# z@n7^Clg&1o;7sE|Y!qyrYQqb7r;@#}ZUX~oyCqeBd!)STYgigHP)&%`K+l}W=eDU! z2nf$sWM>NqoDQ4$^^QNqO_eXD%99%f{Az>bWE^f*kjx8LBdL?SBh#!~ZX^Dws;H0V z|8l}m+kWEwLz-&G;xZ53sZ_Z&I!U&lnC89{^gx6p?p9>8YH;cNF!#2*a=Te!>()q_ zFNsT6%Uo1heRX1Wmigro&TOL zk^9jLa;K}TSza4W_lDQQX8WIJ6v*}}{8gDg_P|b6ICC=Dt4}8x8ZC3XHt2tfgvM;E zrTzBJU#kFN${V8FiqpI^z0sQ8YH}?2h5?17EWm)l(9man%1^A>$=GXFZiY^BYy72p zXUY2*VnS)->j;m8XE4!cdNGkMm=gN^qXWgxZCH4?b$d#3mS=k#uRVrTN?J#N=sOJ%P}`P_knnW;D=wm$|J2)^vuc zZn~>(G$fKrxV*k-Us~$!UZt4R*ioPd_a9HwNRrOdSSOQ>T%Oy4d@~uB#=p?aaZ-7B zv4ovRc{^^i#MmTUv8-EC1$=LX-UvA%6Eqt1PD$&%TxZ8sSK^%lJ!j*c&HIy6;LDan z`oEg9N#!Nq(uj+OeR3X=KW@C&hm9G}$odq|4BTNnv!cB=T4Nk$KP6Nrt2;&{TtXof z(o>w1=`Bnf_+d@+*v@=@6ofN0;*u2L?zrfPfN(7*R=l@rSbzi;1z zm)Y^C1|xZZl4Ng!rIz^e&ZwtYXFLx#GOkZncnJnatrMYvrnVCqYONazGa;|b0cP@) zIubj8H#yjDdYc&I=2_GA(aoo`z(|1Y!ti}Nrfypy<@teM7T168JZ}P{yw>OXcg8xF z=sXxKg+_7t^Q`~mdef@gvCc#3QbFqe`M2lPir1l6?RWfo*V{->%TTCG*`fTyCqk>q zWNYl^Yuxg#XIPn_`{Px$Bgikb(|a2?C8=3))g|eJowA)?rBMRfF^|Nk{mtxopwB*5 zasLApmBJzoTFaZ7t=Q`v1ZazAayOg<2Pw?n)(x<+P3mCR=YSj2}lUdbCn<49N zq_W&A`07Q~sU38|KWVPD)IW(mqwQ~X8HW=c$@Eq8dG(6d;ou%;JaGlDlCgM7yjh2f zVXhMb30z0(&1D~-r-=h~CZH)MdVGrOa10#^ZDJd@o1HI5VsYQ-DqGAj;;{|^OI5H~ zJGE(5^LJ3lz`@Put1JiCJ1XIWS(w=}U-q#8O{{(0#N^*9T4ns|bJTom0C_q$iL^ec zlyvX5MaOvmr^nH&Hw27NbZC#UFSK<miZ7T0>c?sS8L1jqRGYs(qInp2&Nz9oo#8zG^AQ)m5P8JC0FH_^){02wBn`yc z1Yk{*N~8H*SFIc1@B>-j*{L+@HJpP9mA>zVE9($e=I7_meBQY&Pa34fFc@DV#iIf7 zIYjRYH{l&E!dm9%=+Tf)%cmDhCWj3mMKKX8QR;=`V%f^a|Jk5?6iyCC|0dVsx+L}E z*w6-EosEwgOh1?(;e3b1h{d6nM%}OB9Rs9{8E%O5l1X2jW~q&rgZ|U26l${bUF18v z!R6+`P6ob*nY07nGP4w#p1bchCu2G4-niol7PQ`|aS=CyQW zk-J1{vyGA;&xW=;hqTi(h(4V~?2sun#8BMiuUg+K6?g5{F+CX9F{I_j6NV?X9A;~6 zTn&27&V$(zmmx&peM^qZ6{?H5XY^Ci_Yl-{CE9E0#@T*oy%f&b#cc$ zGszYLYxl2(snjCkeP>qxiwO*-N>f|2aqT$s{3bQ|$UQZQ14%4(Y2dcM)09R3(Izk8 z4!7sCR1MYaafAB}a~_i=-ttY@oY@<^?O`)WurGTrfx$xX99;A78Id33)Wa>Ivrva%F~K{+5+pZ4X60+Qi)Zx12fibD(9|&{J_mBy3J_ zj(HCApHxp$RXLs!n7ELxBpW9%KJxP*1lNaJ3fbyNRDLK=(y=A)W&RI5&!#lW`lM>I z!Ot-XAKLs>!41_BLMqtpdn}w)spx(@@?0^8Oi1`ktls85YBMigh6hERYvxza2rOe) zYMV?Or~U%;^p(mS@(MGnB(jEo6;HW0|H^Oz8-8lyHROVh1gKL?k6gQl@ z_;Zjc3D>+!$`XfpZM+3EHh!mY=lyhrxvh)ea`?kK4987?+1D*B@U&u$S5QU0DS0uu zdGYIsO)+q*F0}r}I|04Z>U^N3SmA8}*_+gn2cEqL#eRnrVHMTKIuvimPN%6i(Txtv z6RL_N6+;z!-N>NRLqO} zkP3K?*IEk`hja_^6nqw}gRN!T07UTH+E1q8Jf>+04RL6ickA7DamTFQ4q5>y{fL42I9p_6qCbydpFE~tOm2V2<)tGD$BGOO3)1@!O_@=cd{7Tdu+^gkr zS!IbHNXPm~)2CJpvhk!Q5B@aS(!6zt{}5D67iA&)vy`18uV#6*Ws2f@f9s9@a zTfujp=tlz0Yv`pov!oMYxO3yr%yeqUb9}yw&!1rA3`ftN;2TbrWFJpu6TDs^PQLF6 zNuY*mq9<(>vwl~f&=o(hx25^BX z$)!EZ(GKnvyKusGG7ruxhQaa@WRjT_C}0JlB6(|QI)LjlE8mv-{IwBWoB2{Cq{In? z1L#!i^Iobx_^h@PVz&K(eY*sQ^Z!k_ff}qS_(W6?+HRI@M ziQwA?sk`g=+c^y1sKDKOxISAv;;stwUU~|>Ba`o3jsDuEDX7`4acx>W!JotUUp*9n zz;@#s8NBU7+-AB(ZG|leJ$9Nzg3uf=P3D}nhY!};)oxuM?S3`-(`#T$j+WR_kXiZ= zZfB#^`sa>SV}KkDC#}_-P;szgXSP)0Ql%7g({trUL~e-Uj(i2DISVi72Cc<*TWFaF zY>TqZU!G^N4XQc6%l&a08(kZ-A<4J#kk*9G{<+7ckAJAOmHJ3x`?S3ZrT{?mqBrkG z_!idrbB0?i^7`mUa*V_MUYckvDlWyF&_2VM$pkaOHIZU5xQm_`_Z2T=#9ykLIrE=`{lyl*O z5wl15zHdvT`moM1E43}S9f*@#f*{3Q%Z(1ReFia+pOL<#S1%4>=IpH&pI3uIRk`j4 zL`dQoj+(cbU22E5W8otlBC879L^k)fdlZS3&wAu6ld1^NdfAJ4pq35lbQ+>RovK2< zJyb4ImiHX9vfy(YRv>)#b|o^=s#hqY5o2TY-05vda*Z&wi}VpXcv(3%A>~V|Q|6r_wg7*l$cNvy`E7gO6L;LzR9R3$_;3(MNRs&bZ@De-`}$PZ9M5mS&4^r5CftT?|+m z&pok|R1j|Y(?2mfUNuo=adl(BboJ%&S&PPji2(`$`vwyV5a@{#o;Om6oc(zA?}obu!_ zUUa8jmFBrWcL(DNF~l;H7dU8tlR+47epy%kvsTs3I8HB3)+eX0y7chFz3gby;m0mI zwP(V{hthHK2vzgTh|DjDmx;Q%1EtN#)Qu{1f<}`#+*!_|;)snY-h`z(U(N$Q?VgzF zjzn#vIDIB%4+phOQCsLD2^%u)xm*tY58&`x*j5=DAO-GRV=)IBqz$1#3y48ITjM1$ z<%P5l6DWrqg_@1$9DiBXhshOBwUeimls^2Fksc0FV&G4D!eUOI5H4p&O2j-w z`Qgdq(@;4XKC8pirvH0%&8)3cBN7sH<$Lcf}g(3EO z7CXDe503Alhtb-8DO=cb_hD3X8X*e6E-Z4o_UGgx;7tCUPSl)axMJuYB?tPOfxE?N z%eF%#Hn?^zwNP?a=8l11ql>>OF;fvU$b9g>;J~ZxVwOPmL#&8tLOQaNy(|T(2W@K` zURoD{tkX6NehQuBGlcu*+~)&Ku3L5@MtFAV-?V#y-?f`Nl5{JOLS=*;H~q9Yd#il~ zwKySSBoo%l-)W7WtUQm@>~p_Ur4JH7liFW*13MHEN|`SOSC1S62-||1Gb3tPkip8g z2Z93^B0aCJG;#}lVrN6!Hw0$0l4Qw7ik5BLVL?grzQcD77h+1dLlD8<>UX(?yiYTQ zHc8JBamX@+mLqjhx_@Jnt1EKV!2B%sfjRE7d(5k5UZLOz6tJ!YTF`6U5utG?kQ)1p zKv>PA5nm@tPn2<#@5l4l@8q=Z(zO<1qqqWH6jNM6%v7u#Rr;;elVg916e@n7n-{4z zzib{IxZ9edH?K7vZ$9F>m%LV=>O=ubh-^FaiRuGm>MnDS-^82Q0NE`QlMMV${N7zV zbp&=={QRfkKd{?)p5%(FTMp#OqPWIhb!Td?C zF_oW@T6ezf-5g-Z?%{-5(v*)^r}lX9@F2=YzTNKa%0I^J5Xn zGZjw3UuqT2M&J52m4^J6j@AKxzvypr%CY*N#g_m)MAK0oZ~h!!s}&Vo@7<&SZ~-|` zxcN%q&~&+cW9)R}0CHsq2B?2If19^>xyoMxrb72LO~Dlvq?PCg7BWdWq^$FhOt}i| zp^uk*dLifPSXmtNi(uVctz$Du;~(USo1Vojdaf{?u?Ll;3mWc0B`fz={fIHm6}u zS)pjtjyWo}w{Q%LI(NHob1E2=v8Sd$O9|h?V3o7vvP&;mRTBSf;sIlmgZM z#_0doJY>v_VBIm0!w}z`K8AI!`uW%KK;X>ikRy$S)7I&d+w&PgGlsw4EjXQf*~hrV z(pD(hoY^MACk|dQd|l~4;78mZhElh?Rx~uX2ZMzmvQB)3i4Ca#kY33B=-c#Bw<9p; zMxwMhOZAuOOs%l!>^Ld(!ZGK7Kk?*~Uz+N6ZYQJFZtOlv%TuyGdW=P8BJNFgvTmJr zolDT#Su)NWVuVktvEFb@WCC}=A3#JHxuOiBss-7W+q43Ud;S}Q;W{F4+64$e7JA*lc!P>4VDw?3EItW9g;?2Oq) z=lVf)pY>+33X2A;ZYcFpaZGHh|hkVl;nB9!Sae<~{it(XBU}O@TWEBs9NG6{DK| z8{i^w-SZl?tL^;J{C8@CQJWelC3FC8^Qfud>!4KgMl#xmmT_?Ftu_%_(!YfpHkQ&} zCA=`OzCE$Jy)zMM)qqCO<`b~_5dHB6FN8~*$K#svpIQJ} ze2s4(!Y+r32;PSC(IrsY^q;EbCWx-%!|4Z$^aj2I?2O0+@91zdLvGyQcmC@4&^}6| zOj~0ECB%paykv=p+el3=Ivec3+l%t22%A`I_Al6E)*$e4T^O zwNU?NU^i ztT1Sf^Z&7nTV{V#-M~PT)XVA4Uy=pQH#;xvaoc`8Q-v)|ZClP-oHJl{KEeZrUF;Vb zao6~g}I>*<65qu3Rigt=jOzVcK98nI`sAU4xLz;%ZXPm6}z8i`BPuFe(GGQ5Km=b*!>#w zIZ^g)j(bhyu|Sxcs`s5Bg44WF=ROvjC0_pL{X0%5g%|mS@Km@+DAoJZANUcjYvVM@ zm-CX5OZ#b@I3&_$EGe4HPqi}`=2uBTQr!v)14g?JYt5-?;#Ct&VR|Lk$DPl|+)=Ua z4N(hP{_7%rOQAPpP`!q4_P*Geyt#t_2&|IC2mru5ufa_&j9QWprt!nBe{HD@;~6VtfrklQG~swo3F^ zz>mDp=wtgSa3Konqto9m!tXC~*vk<64OYpkicW=wgnDzPC-RQ{mD|(KovjWuVm9ud zWOQ^wUku~?mJT2qDdlnWguy%uv#{~GP?ot`T=Qz$Wrn`5f5v=}bF^4wtXlgXu`yjA zDG^fd{6>fTB-UrvLq3n zMsCGEyET1Ork>#sCInzoLiEx}NvhdeQh6UzJ#fbjHfQYKu?PZgfZNX<9ejqSlQnUH z@X{&l=d|cRz8r|;CIOxPesr)0N}DTg9A*xY;a6-y7t%T@(agSocv|1P(H(g*me_js zEJ$*aL~6N#ZIz;JiOCNaPsi8MNx2+?52qfCE#P8_w^Bu58|Rb8J;q5&+?u(cxf{XxG!jZws5yd@SCheXOnsP`4wxqM z{mRM{b9b$kTOmZ75Qt&vQJxiTmY}CX4JZo`2#+19A&(neokOetO=I?q@j{L#^L$*; z{Pf*ah++(1c)I3UD_N0YG5X0dB- zMH8ddsM9m@r~PP? z!VI1L!erl|CHeRU?S3g{g57L7S@iG3`JG2*A671~8<$3D*p^OqK4E!)PBm>&=n=pY zo+24>f1Xh)UcN&+@Q6M62k|a$jhS|)q-}6Cx5WN6_C4UxpcQwN{LE#Hy1IHTLr_0! zj+omoe;a!-ZTYiQc3WM^T|dBNeF-de7XxWopesNJ*T^yR?Ri2FGOew&~nF#<*3e7#bEDJrw(#8k?5m{Cq+s+%QE8 zg(JX3zqlX_{muKs`0gf0$4v;@?K3bV=}x$V?=iT?imc4@W55Vz?ofPYXZ|95>4uXx zHJ=uBEkcrHgrH_9AnBnbkp%nKfj^9tOYQx8hE&hTNGmI5jpwu_47G5t59+@ougMgV6x^86p<^ba09qwwe0 zDk-gB4=Ks}+hMBu%eo#eQ+pJH+m={7ru&vc^gHehiedZ}cgHvnUmo1FFz!rJHsqj- z@mp4wVF5Rpz8BQnjd~=H*fExh37rT^tCPv*H?l5VQ!TqG9<9rETi?^xCdLu<)`3)B zrA;u7GWEzBV^>L(vAPG-Wylh?_g}UXY5kRR>EFs^KuzWrb~Ybo;*)_yoifW586yG| z-l`&XqboNL4--6)BqeRa_%6$_!(@jOivWeqp02^%2Q>RnpO6w)dj5Xr?kkw+vk~E8 zq$mqS;0u>~W+bbA|NFFWP3`@xoNM+^#1+ilW4%j`y<067Kb4=&H~6md7` zn;J5A%Tm!xb!<8XI5B;InC0ziTS8}%1bKAbIe%#di=bxs(mR)oTnr4>eE*uZH~Ql+ zJQ{rc4Zc?1BMak-z+R2H)0lP`&lo`SL#Y2w_9v?CGtQ!k+^wTi^n=AeH6Wu<{bex? zx<4N|I2OIZM&#*7sl@6#4X;7s3b@PT3$aik)Y50$piYN|okg##iYro{5Wsmw#t3c zvHfbUK~QFk5FBU?ebfwyMqEkqGH|NxY;OKpa+%CWwc!x=Y&pm3qaU6cX4iqmSHBrrVF%6~feA2|8kbv{ z#n~d#Qqg5bgUHY(5I{q7U(DgouZO&}{iqHvrgrP^GG%Q*$ z#}SBqN+G(e3Yy-c*VN7Ap(qjqe+^DG-V|7B`U#D{|tG<)5>u*-uNct#+h_OX2N@4H1^-|4dmPq3pW({Y4 z<0%tgz6c>-SD@CyQ3rE(56Slum&cgY@Zo;DqJNV3sAA$;b5AIH25ix||0HVsu-v*H z{wHWT(_ss4*+@~zO5;D`3ln*kq>z!Tspk~K10g*GsmPHqSy2-9KKMb<&Uw+7QsVg; zt4Gnceu=xm9FcW=&6Tg_jnewit1#+4TlmJ&wBDRN6O~Z47)Qkf-kq78qWQ`q*qb@d z7Pt&=u$y{H4cN_aw$Z@-KdPzN=aYx^r78{s9DHrL)8=c)$KRcT|C~zeHvtvvjljEf z8~neU*wZ+lseLYVZ3|$^NiEQF-*4DHh9JZjdTJLJZzN3dYnzrzDSR{@q*t1-@nvcb zr@PZ%CZ>nAuE_XPS)7Nl9dR8>5&T!9kDe9Umu6A#bwn=}ZyI|ML)z_YV?T%)A#*&6T zZ^z?&LJW3Gcr{IcJ)`irouhEm8h_$M{hCg(QAemZ?GLCGM-J|m(iTJ}R=5-Rq-?v8 z4+9ze1neY=INdH`D|HUm{z4oD6WjFyJ=h5_{d%d} zNq`tuAzRnG06$(ZA7;eO`$+1yRe$?N`@EJBT7?{ciEf!LiL~Sv6T|R1A#1Uk6jrCJ zCoQ)%QRBq>@z686%OTfuH*|3y>x#n=-`_Z0!-8Uez`i^4Hae|p1HoK))m_}3o_fg|*w>HXY zV;tSxF}HLJWERk|HR^XzKAclOp(qqE;F-#O+f2HMa@|YP^EU%I@BG_@^+5DGMPO#$ z6RNwfiSt>Ew_dB6-2w5u)s;&r&mbKJ@;Zljx3PVVFUp9)F`~xgS4%%$-Imd$7wS7^ z0RXa{D@Tl_C!MSl2l4QDZCdTqnagb)R;V$LZm-GJIFM(GDh%EJo3aYwJuo-f|?TfAMAO3wW`#Toes5Ies$T4+BS#WTYKqPuw`AouP$(O{f z(pKsnn;f^DAiS*m5M?PandD&D2BFDzaiam!C|JCnCu`ELJ7k>oW8dz!SPI+H`SpK7hF^KJH*`h_F`s)5mH5KNiNQ&e9U*JqJAg3a_X5BgG1Smj6s0m-%`~{KV(j z=;i@V&eU!XYe7ZzWV>hICl$Y{aslSsGNe=2)Z;5f=)359fBQ?_$mxN-%eix{Fsf+#^d3Qe1rMn$BhCU^6h7CCy2eX#{SKRbC(&1h<<`np ze22lID5+sQs9hS(xztm9OUZbxreOaG(k&etgA$qaHsg!<+`v4H0gemfphA!3d2)aw zpQUF%om15=a`JO+LCqN+u050Qbf_PVQJdxB_^(gcX95pM53FO&*uK=++B_l>O=*OR zs?W~a_B?~5l8p|<<*PREUV~EvHkKtSE&>%Pf!N3z3#358N9jxfCCjGQDb%o1bd$#WJ;W8;X8P4~3C5KTKjww{aI z{FZ+)wXgMmGC2dk9NTb!YPW3oJ)O|3Osxm>-%=NKK)ZbEeyY&6#Mo;lv+N11t6cKm?MY2URrH*sL6%4UI!*ir3oOxnjDxoQ@Y zn@sJga@DnMBFxGR-1Df^Yi@3b_H4FNrac&Klf~BW_|%mZ>D_f9iPWY|XJa>s`oMVG zn9W!eLsqEmAwED6X)e|NwOa8Q_W-hxzNJmh> z7Pgh6Taizku~28dHdlF+AMJpz9`9Fnm%kh0tZ(D<;*HFWQ% zB7~rhU^Uj6m70DY9I<7__z=Gg5jdkVSNv4=T=v$85ZXoKarhEx@a|X34od0>#_lbLw!NJ?DdrQdxsYpZ z|H-qLhW8W_mGg;5U>vw6S`ZiN3)?9;_&{OUlUG{d@|oqmFM1i=UW+3Q(TLn~7xbhn zA>$_`J+MkxMy_1Op)lD&Ji&SGLWsB;oEm*4TjZWW)O+&4C2Zu%YG@=BT#> z^vk^aW{gj%tDn_*n7)8RDMtsl_UxymZk2CgKzfrLdGHTdy)&Oh8)^{VC_2G;$o9+N z)9W1^Qudq+lI#OI`9m}dGTWE0Br8NK%OGgynmGV@MqfH{VkmL@THbhMD^+-$&+da2- zqO%Bt$V4h#jc^~h#9G9@MPhT=kE#B(-bQhM5(u*U=W{0PZOcb{m9ZAct}WXN)Ai{_brjy^^p_ z+W};h{Y0nW5P_lB{k~%(AG6+<_UrO-n+uK)P_Hx|whu3qczWAQJYUy;QXy;BkQ)_# z<>Vzc3v+Wcj#e>2OSmz{;U2kH^!6AULDvmwEqZ3Uh$}uA!0B9UKS6>(pEI*__gT_~ z+?lC&LMSknPY0odooS_y&EllzllD_>;&&MXai`O{SeA!BYczIKn-e_Nlt)(~*DY1a zJAs_PZz9)A6JV|y$Lw9+-3ifoh$0yht=XgN!X-~tzmeJa^{l@GxhWWaVe<>&55CAI z6N7r@loO8mw&c@jOTF=%hBV2|sA=5-$ouqa$eD53eg*YDwGnujZ2#h<3EV)fv&_~Rv`QykL-4{BJc-0`P%-B3MI-<6%kXqVVT zBJ2kbB42gomK#fvJv*x`%Dk>7PFZ0l_ELmP=V@)4qQ6jEi9VPp^)7ahdCM>T%eGJYMma}}%wxHY&R7bMn4fchLJy;aVeeISwUcrwqU7k7j)+vjQKY_c^5WoWe z8g51=_z>zEX+MqHv5Y`Czauwt7q&%{w)QSvc&P{dCz)N+VzhR7T7rj6kL^8HZe{=X zYW{f=dj=5XD2OLoJQ3E%N5B|4iqdH1ep(iEYEIip23&mu=vY+veTU=X+)g= zSwkBZzZQA({Et|FN{~r8PI~i=&9DXxkGn@F-0>1RyP4)jHn~YKlx|J-XByo4uMJb* zY#R+@yw0dpTtj44(UW&N5p&4?;yW_%!b$0r!&KlqliO@enW5_%+%aS^K%R)3xhVi2 z*|g>jG6_@VX>c8nE=RL(NTsc6N(N06(@?46REL3+TDQxeKH@BIO` ztZdkGh1Tz7Y140h%ZyES_J??>xmz&jh?tMsSqt#-`(JWCi^Ap=;@#JVzxINXz8!wJ z^X?2y-xMI{U!A^m^M(N43KlcV4<#+E(p&7^e|sPSA3OW~7eb-%bI-tLPu7s@ncP0D zOjY5rd;ix!F-rT#w2dFrrGIl_CsCCS#Br7B_O=t1yn7mQl&|_I22Fq7v;T-}3e(d= z5B!qfLJ`544y2*c@rdb&V7JRSInfcJ8~Ef0f6JGjq?z-*s`xH?7&Jqp(8O(9n1^vN zLkKd2_B=3#SUH3>(kBhWHbN1cCi{ORZV&W~Oy!{LdZO|Nv$WPKJx3|PF$%_(&5J>^SGH4-9ZF|Z||jYhS*Z&rc{S~9NRn>pr>_Vrs0NbEHLr{5n4 zwY08?n__6Fg3x^-2F(V+*qt=ec-%jIwS55@S>r~Op%5`?mM52_q+lAs?mcFHUx}ED z{R4~<)#08V`mdjpu+vW2Q-u;&KJLXs^53a;*YMPStNwXV-MHHVb?+qUcPu_i?WDe!+!99CMK zo{im9mhH)pRol*TerPgm!MXsBJ~Da1p3%u;%B0(_62WKe5?Mn2#8WxVGsWc zWPRwG*s4QnsZ>~jRt(6)?uX9+c{+rX-5^3L09|@^#yoTHjykTd#+TqfShT-u%BXQ^ z&jNsErnI}e=@#GkMdgNI7V8thUs|%lf+Pv!iOy}uC1lM;5ry)rj(QElzyMdI^tD?i zc3zUlJ_z#4JMg?)^(jKTw|U&M>Lj4F*t>+0laB0RtjLPbQf9l7`U-yRE+#a%`!e(Z9b|FAT)q9m+cz)W~0-uy_h{`9y9duPOvCK=xJT!=r zciUm!jw4kanKcSPNX0|aBwq%prG@8{Y67`6Tf;M`9mFLJJYST?G@wlU`uce;2gUN( zGLyDpWlYX!Dvd_(E7*6Sbfm-;ho3`T$be7uXTRu(#`uP zG)-lFdSQ+4OGVJr=a>`BK;31#fBJvG*}?PO-LL2Ef$Zd6H~q~@lREH?e`u!n51l?y z2=8{m&YLRGrUW6ZQzQ?l*;Gn4%&e;JE;%<@uM2X;pMB97`IwW{@J7YFU}UoR31;tx z-KJPIyG^oV`=9bLY`Jr*r#|MC9g}ldRF~AOGA^6R_%f0i*iF6Sc84{(W;sWCwQW4E zwYb{Fcr0e0A7Kn%fJhfK%>fo(hfit9Z%3FUfStvIHS1AtLz{nhY)1Og~CjI59xq# zttYiy8IOYro9oV`QKR2N*_R}jtX&COWuYeb&m}{_BTYLbCd;W1H}K08?ff5WvbLu- zgPO{0gOP3R?@68DW_mD@pM3N^rM)W)%p)@T;^viLA`ZrYNGf$9Dkfd=LN`t&bUjrr z)aK?WB@}jjF+Qru{OqyXihuJaU-L1wtQbp`pzdbA%I-X_^n(dURbXA6lYLkpdU+A- zm0+lzi!{%0l(VRE__rdb?Ph0qZHEj^aces78Mt-9Aj(0_lVz~ ztm+GHa{mCqB1r4(E313r^b)LO|4fAY%uPY^J_JL7hC5tEQLS+C-OGm$UFEu>9h7M0 z_XY)82-@W#+mA3?87n^|G&6>b*LN0uW~JP^o4jJv#7X6uk;#$eRbr&ROdcn1QWCa2 zE`qTP_bVdBqh8b$W6Y^$4=yx zE2KNzMN&Lkz$NV%Zjoro2(^w}y|+F6(K^nR4si!tiE>WW{JkR)AJlnOFbvRyZvA<| zL>!xa{z>3V;e;~r9|IV;@9hX-9vO23`uEe_lICPAt1MNET9JT$`eM57|IhE(X?)G| zA~+-33h1#VYGt)N)`)&O+O6nyhTFo={KsFmAV*TfU%GMrHFN-Q1aPIbzsS0o-w#&)1p;9yv{wl$|#8II1tD4w0kNDN)C z(xviC<_0$u+-q@8Otiv|>>TqlMOFJWaYf+^rZ*dZ0n1Ip%kYR^o@1P0geI%+jdqQp zw2%kb*IL~F$>;)gl?K9P_F_`GQ-YlzZ||#SgipG7B|?qJI*vtGdqfq5eWnecduHcILcVyf^FenC z6(=!<>ewXt?0lxK>8N0@FQv$Ps4<9L@3n&PRhVka-)CByh9`nKRq2i=z2P_$>0%52 zfCN*DKxqsokME!ob}^gw;G2zBAt97r;KA_v+aKHYQIhY(E}mdg!aRc=r%y?QwtP5Z z9)A@n0T1?3umR>T3>Z}+6o3t7jSVPT*lFWEOMEJKtzjP@pL#KK+6P!DIz*_Dkk1*Y z)z^I7+dFt_d`(y|{tXdyFQ)R6GQ!qiiyfI^whhnDT+TAt#oLMJn{4A1z=!0C=qg{PIneaj$)hsCJoA=o6CS+^u0vu# zP1axIT_~385Vd>?zEGz)*}M+{_zHGrB;&_(P;JOX-@l_fXUP5gd&_M6C3tyWy7xe+ zkIYB3svS{+vfkSEY zLn56iV`vjhskL-0SF~@qNxPO)q5p~!gr+T+;yKzU-%`pp)vi7R?;1rSA1KsLv!!79 ziQK_Rv0$B1MWk&;!M}MZ-H*T(TAr6%-!EFa-eWq)r8a#BH>wxfcohYev0v#IgC|Dp z3aL0XR5%QqyqCj!+pZU%QuS{$mB=vXFXP9Jrh3sxlX!~73e9r7fKm>I;=XIvwz_vG zJCfc%<}tm)xAC{&^hXW82Zz#c; zwWp}{lD&O^JPA|Rp*iF~+=0?0 zp_eno)lBSTVq;Y@QkaaH#$Rp->Gr8O1386jD5t^O7<<3=b<99({R@KM!`qXrPVGI! zFU=3*x37{Sh(lpJ56w={5F$w^2xF2pkB)6puS@kj_h6_c_p^1a}l!Nr7X z9t-`*CW`=E$oVpp>q|ivYFiiO6Yn6M!XMs(+|M0vhmVy=9A>+VOjs{lagD%4(Diax zTrj`3aM_`Qsf8_YJM97Y%4MT-5+l!%;F7`a6?`(mM)Gr0Ugwd76u2@Mh-AZR0|+HN zG~o3I?NWtgIlxQH;QX&y43v!K+hOE=+$zN*r}G6eNC#JWq1e>$ukLRf{S`12^|*>zhj3 zx5HD;gfml5Ql&15?CAfkq-PVy{_psSauUWw3(`f7Rsk%=CmlZ!$^y4hM?BC*WS6T8 zLUU0eL8ZH!B*H7vGfJfRB$vA~yZbzBmg z0F?~M@+foZPVU5N7_IjWcUYzyfwLzJCq*2YH0Ye~{Ckm-bH6W2kHLaHpg;u1aSr=` z10@*LaB9EHJ`qHNme=I?3DF%jcFP)@DI+X-YVM@6sCIqtNs$F`Z7qStnVeT+kbEn? z{LhSc15M65*Ed841AF`$XW`8F4k=Wmoq;v+o&I6ZC0ISjQ*I1_6+f4)|2Yhv}RGpX_Jhdw&#{7uU-diOMQv zt6F?w;Ro5tQ0We8%^Jycl=r_wpg9soR&5MGjbp+4Ztm2aZTR)AP$-U1Y@?D-%#TN` zW+S`~U>oy{rwUT=Sli__&3FitahG3h7X$KA9g2tw64HVQiRYWFX|DX+&{K{UafJu5 zvkY$cZE>S2HBSl+{_aWp{zlYVMM|JUzn_<+)EOA<%_^R9moHvO%^cU`)kq!_{HxR@ zp^78?RnrMjxdUh?$7dwPD3(8D@8k^w%?O^P%J9fnexTlS-89%)R6T(4CZ$FdRd@Os zEpu-pJ7D1*;$^eG+IvNZ>$+yfe5>@5T@pFoW|@V1L^x*2@2ot)mivRAZ=JUA(ZATZoGeX>rm zWQM23CrvWy7=otxTz&@%5FS?|s6^3{ELZ@?Fan9hDb2 z6TJEXoxMD?g9b{(soUPOq}8OY{@3$a`n*mmZZH_z#h`q8NAGgXQz*o7g;;-#s~-+1 z9V{zSJ0y2lxI8d8Y~cx#T1~|qw(p4(W9xtwfW~#g=B(s1nVn&YzS-Ovz8Y4sM`pYP z2-iUFy=T6G_@{t~6$y!Utqg_b%1(4x>A6MrvK`Wh6GQuq3)7O>x~7%uwUDzajM^n_ z`m?Z$k{w6ezB82&9_zpcO%v-A5SJ?on%50Yd?9>v-!Y#5Lddh%lt<4Ab}{==VO8Ol zUg6VKlQ1opsJA6nMyqitk$|8w62sstfk1MX^Ubajc3~n_W&X+gmZTQ0;rCRmMc7tv zz!7P7M~8wfY$-=a3!dF0qu#TKcD55aHsPJ6B|GVhJ-G-JgFjZI@la$xN@^LaZ=g>( zT9pI6#hBzKXPz!zSLO+*answ~{nY;%@bjD13k}I}&Q2e`G$RYglt--Iq?wuFVkm>& zKId@@hv}QVmb-kcI?&jbn;*bpREWTibq|M!9xhTlpynu?U(s%!?e#UXw45-vvAx4T z(c(jbJ`42@xcd}I;WbG_s9Kz6WRLrDRa$VzoADSsgE;xuOj`wRNjx96a>MC!MPEq! zDoa9*3z?==<&*O)q{Hsh4}2dp*vzc=#drLJlT%z$rqk=3Kq9ZyB5v2mv$1Q-9`kWd zCAO!9i=E!T+=drumvr!bv;OI#Xxi5?K&!|=7TlKR-h)NdFZ1yKZPuwq8GjUc}6_czqI_z*|2Z;om_t~o?aB; z({R-9ARJ*w`uq_vfwe1!B{yhT=O|~o)Z5YS=qCp4{|xF{?Hd`%!jlL3(sSKu=@wLF zXdQCx?bVTsEFw)adttJE`9&d<*}sD}%}rs<7?D>h3_*g8%KLP>Uhs(F8p1x?{oE!y z1yFUh(@3_`ZTv$`x`I8W0$6O-+?K1qX~@Ru+-OW#ZQhunRFLasZ%OJo`J}BGln9jR zlITV}KZYT%t5R4XEOCABBG`s8==wg#u~17Oe=)GcwlJGlwA7W%t+*FW5~YjK6(W^H zB_91=NCk@5b1{J0Ra}O-1;sa&p7$vAkL`&H$XIoSVpW4f!T8df{4MU8+K76n2Q=y4 zVz9XtDbT1Rclh8*G>2cRCjFmPpWS7*iN<=TE64(0@8yaQnR=uBmN=0Crc#mX+P?*I z)d?L=cOG;(Pm%5e>hR}p*;_IX%zzsMLd9Z5-iC%h&R8(otv&S${(eap2iw(*Ub=V4 z^L@WWo&LcEU0_o-^dwTPReDKZlSUp zBXetp$OK8OGhsQ1W^Z{}x8Y9}c7B*-YFYdB5R+x^ECeAEt0yz;#TP&q0w>PRLQrQG5~%G_RHY-Dn8B#$8ecu_dJ>PfiAJ=yHEe&^EH`?LAFZb(JNF z^UKI18HCkAip05%)epy}CbnR^hO?L81P>K@nN4D=vmw*JpI%+7n~_YZ6|@MM4pEmW zLQqnuHC+Cfpl4roI7I<%Ke~M~qm~7DDi)51!XF8{kQ0JSeXg*PK%GmNjy!V+D_`;g z0FfsGO+5o>_8ZBW&}Wo{wGkVUCMNuiBmuT|R|GB(hi+v5GW-hSnK4M*v1+MfHUu@g z#;suxVLx>ZBKXXS$XWBNL+q`9n|@4#O!pj_NCQhj$bXO`oBQV_s^2G4__pLW?!XHE5`$t7%8Kz!F3~w+JD9rwQCVfjdgdo z#plwaK{C6`G6Lh&2>P$-8c*UCP)_^WT)Rq#UbCJ58WK@|KrqPFhaEs@f;&HEW|7~k zn;)4XC>FMPa2-p}FY>T?w+--xEDRoP5)@ON!RJW3rkim=zY?XD<{qO;xWZ?t^Zaj> zL#(aHwY^|=DJ!6I9Z+qcx{>xI$+11P)?_sGN0Lh1V0zG z0%Xnz1VOpAMr<1C(ouJX99W8uZ@qowV=L+Yw%;WNba#sQ&?6hyybax4?r}q3V*v7) zt?&yg_XD2|I|c+&1Iv<5jr=cMq(WMczYwnf3$=EVLh|?vk5;D&kZ>`_74N)mOFOo_ zS&fGkhnZWcyduiz5mx1L(PlK)or{DDeK|$zt_-xomR)k7JzMawXjh2i6K!rEet8+3 zh56Z5zV6hyrs%Uwrpi*JnVTv!(_*{Bvgjfek})|)pEV#TJ>Rg>1!e?Xr5tJ9n0<3< zPSs}TpAgB}^RA(9k2c?#n)tY(>DvF<%dL{<2nM6}S#ehUthrwDY_r1bXG$)!_mWUa zcR}Yn+tyCFrf(UPjBKK=M>`T*p40<88HZ)R22bG6+s4Rfj1$Ql++IGP@F8dZUMZUm zmq&{Dpe$qoLcgYi=cDAJMKA>K=EwI{17@Rau8%Xz`ETbP~b|2 z$s~R88r~#VAq+{C=7nCug>JD)n&VJXm4-MOxQIg~2eM3OiX5d*7N;KdDvmdZM`R$w zsvaVAf=!Peh#pQMz97cWO*2P$eqv1j&AV#&fm?%)DF5xJ=)>*KCljt{2f=0=Ng#L1 zQK{t2H0y_gktm7R#Fw8WM(5y}#qq?U^l@}3%g!;Hd7=Y0MLVPG(oWf7zzP~Se_W@{ zQx6G-rzkRw`IGYzsK2jsNsnQJ-Hm9s{e)rawZcO}i_1x5W!!0{5>zk(S|kAQ0*8uz z`UINPG&YmfQr?xercwYL)+zf5LMO20-8pLt>b>rbLiB%lo@)zVphhV+PZmrh6tj)lzdLuUM}_m4;VNbIh1pdX>HAJ#>n1;+2^y}A@=_Yj^pLAp%(^NUBz5(nyqu#*mqn_ zx@8c*oLas8YvNZFzG<|n`HOjlk{$Ryvh=@^J03T2p!Le-12Pjho|!vp>Dw>n3D^Lh zh-9f1hLw;vh#!L=Au&bR4<)6Cd5aZ5-jcWTK0D;?az?o(Wetjt)-~So&-z5iO_qN@)ha zzz%I@MQ$&>qFVw>qM>pVY3iffzKrqk+UaxY)W)fS4zW~wDKcM`pWyF)wkn31(s~Kw zP-1VPiTsb%fEoQ`IQme0-kyFVvK8BvYi0h@TXoyS8oCwjr_+onOYK(4vBY0U%gn2& zQC32pvy$5rHc`XQ1e&4rw6YkZtSq~59NX*|b%v$vf>NyKR2Udc$8k}6b5;GU%wGQz zkgcmEGK}R*0@@6{l{Cg}*fYrH=XqO?O!(fL0co73LbnrEKQrYgOO;k+r+1EI}DLU9R8$~ku%&OH76-^hu{e*7~HgNI33|?J|ttu!s zPt6wLr>~2LdY}0aob7KAIuB2?;3)SiETFjH35&2NYk90%EZP$?F1zf!PUQB63kl2xH-NdyU+0fr{Bw!1GaYv~FfOYafq+0+Q< zSsLhFn4Sx2Kek{owW5}6a7}*lIC@wOK`x=}W{_5EVDSC8=&s3A-J_h_I6-W;a-#B# zn=;Imo5>E?Z83FmbQ3PA*DFXSof4tT3D*kB_VblMMq|)R_Ml9z9CwK$Or*n;)MfFy z0&E52*y=_Km)cvwgt+CLS}P4NU&L~IMSVTkr9^z=+XD{}oOl||_&VjD#NLJ}J$%O` zk?VWv%>as&20MGMnbUP>5~^wrEeED4UMkNRpH-L$X=}ygiAQdARMhINbR7`Y=M09_ zM(|%UF+RYV1T@Eor*OC5qN2YZ?i1LdjNqe6MhN$VKc1Odh?hdgRn;1ZM<=yCDw6sF zlS0CylOZvm+yCGfJD2X5JSnlCC+qIo%d_OH>2|-zfUEDR90iD+&6Qe0yV2u}M#+qs z4HZ>&+m^9$WtDue+dNy4WPR^G4=w*TZ$X0Kcn2JP31#iib{3($^0ueUWThn|$IJ9&wL#2xK2Ah0kQYN#|Ql{xb>s zi(#ZbZ+-TX$pTRVd%4eJpX#T7m;^Ch;;j{zw<~=Kd|eaOLOIWs`S#OIXu;w(SSK z+}uo)LJ*i8vBmtc(qC1VwaqPl#`wSw>pVM*n6F{^B;06iArxRH9i1EycfLh&5iVHT z6FbCb!kCDu*`c*O_378z^?zeq1vookySmzMGCF0iYR5g^6>+>Deb$XgVvHu`-em3{ z5@kBLBouFs;f~BU9<*c)UQIBLxn=(cCPCT0a~f+m6QZpQHfeG@D~dj=^ijF*GOFfv z_In}1SUBKpVS5~lX7}H&?Chnwdr#&OM)u@8eJNud4lD6C8NeZ#yq-__^4TT!p7`Vv zhne~0(za_ZBM^rU$uO_xV2fw#(FC#X)0>WnFPBGtLzs;|kun&FG& zN|3z}<)or;9%Tc=+0Dk?Y?M{@lQ`)1B-1AIaqdZGL_U!0J7^U+lZ1|?SwKPk*=FjQ$p4ol5z4mPy4pm zB;4#LryZ+ohT7f6WJ>qlaVhsdBzISQ#+zC~c6;M}Z?ugQ7W#NC7C>I;IFJ`)W!uJ# z28=KWON|+FN4&&I+z?Pbis}=_-8vOK)xAo&{%%CBI%FeR*2r_a&wnS$uF7u}Sy(cA zcovPjb#Sx%WE}NLTeU^B8QSb4eqor88O@3OHm&kK<$sjN0_C=NzvU}#7$miGUjjvu zi^I)cnv>lCQRWMj>wZURv1G^-<#3FAL8haun%UCUlyNs_=sJM~Q&Ys#n80muS=a%3@CTM^y}W#p7J@^Xrh!P5;!nyPuMz zpV7t@Z5eGNeba+qpT1|hUuayVLpMT}=_`J^V)L#PNiL^Z*^8RWasq|SB>5W3&k~^; zUkkisZ`HgdJSA>&%#2BS`kU4$?q`8=zO2e0g=OIqGaL`8NA>n1%jz z8))HR_(a9ecjNckV{RF$V}i_4&$5|cGVh5LzlR}Yd!vGIKS@Z*sp-@G3esn8s_@!Q z+Tyqt$fYmlI6#pPQlQV~b@r!p_@sPK{OM<&-g)}jr_C%LesGtan~kd3sgn;;ktb0T zwIOJ;)b>>xwV5m1&>WU4i=&w{!oG0u*6+M^dXvZXpR<1S{r~X(7-O;#!u*EK7dBlE za`TS%lW^m8{jIH*S#2vDEo{;;7G`sZww+Bzo~|aX2J)Hg&UuejGMmcgHAJ~~+{-x1 zyK%kMNODnSBR#ax~k?9)0C_MB9iqk+}i= z>#v{xYdd_R{*{eu=CbNvIc#uH{VPx(%cdD?GxXo+IUK`$2GEa^HsV&GIsVIVNwD0gQuKL;LJIK5K`X1lHa3$))RVxz6NDpVK;M=#UJu#atM<(w-Nx z&f9K~DboFcFMME{k9&OxNMCt@>R%|C3+4K2`ZfNcRj_hE<+5=lvvD_faI1W6=77Ah zwX1WFcu%s8p{eu4nfD}Hj!b+-zF59xmGhFzU~Z0bNPaR^_eKAT$CZFxMs?pBem@zj z!dLuV+E-A%-x1I5L8I`S+AEHGPg3i|wkJuI*sT^r{abzMfj6;PXk# zZGcb`9s;Gtg*Xu=;wNq`+zV6;@vQ7uD-a6(3Q~hw`8w+0ahtvLu8tZJv!_Q&kewj+ zP`a+sMtS2(VTt2rdhzV$dQi=3z~~gTi35-obsXEC)I@d}q!Nx!Z2~o5EPnEo)(BBw zQkAhf#xK>mjY+DQr+_`oMw8L)6D;@0W3tegE0xb4i7*V#&0Z?|b#p~N>3R}R+FBN7 zGB#-<|1+=py^@rhJ9rvq_6=MkF8#JpnJtpslfT)fHV0cUuUkxkuOj?3inuFz8XB9p zr@sREV$#G@mMko@Si!%ajOWQl8#EkOqkXpb}y z+AnfAmtA+6+fIu_nv~@e&&mVOAFlV=_WS^)m+^iaRwIFs0IG0=VH7WVWt*OlOrzADu*SLxW71}uU zP5VWDZEQY(MV!bqV=cAOBSgu0IZ}ki9bxZ~xl$)z#cfC>-cH}B87Ddhkd`if>1A$+ zG)twlucKiejlyxW;%mm>qRx+5PS8Pg##!AH6<1vpZ5sXSHXs}(pp4!0jF!;?8SPuyN|GLt1&0kn;*eaqyEwsF)x*8XMKhOQ!%cvGNqK!43R zk~e|Sb}`me|4E+>@#!PxCRK2F(HuBO!DjO_}Sy{v@K66QpWgYzpM1hNa@HM;@i1rn2r1zA zZQAN~ic_58z>)iW zb!ii-^-}t-u2xF*QkZP#_7zc9sT|FSJwGqcAXlq!689vP7dg*Nxmj-JEoEEvw{IG$ zX77<0$!}+8n!|&qB=mypEj?J5WGJnxsLJk8hNavb@HjiakgE|WfA%5y6xwm^Niy$T z+LJU;b*yUb)}yXI757lJ9#$GCenAB}Pc1t=OO?Mhzq9|2isEOixp{!QFlYBSz=_|>e^MiV3Bv<2x>(5i(B?3kUe}oNNQcyv z1{auzH@b=xe&MU&zD!Ri4YB!{7g*RFWYLGtLksEq1z`{t;pEp4=clTPe=kr4>MGkB z8a24XMt6x)#mstP^0B%7oDOaqr~H*WMwEjqY7pt|ph`=ay98{&(gK6{-~-Lbm1isWZQzk>AHF2l;>AP80X zI_O%!Nn4$CJ3^e+$HkFIh!=b`MEn!$TF$qyZk)SOW8BI{Mbz&+xf`<_K}^By?^4L z{L@dGy*6cU{@a^v(IroEmY_b64F|4A57h4=gWEoBqP|r9ss?Hrl+zBvExp|Obo+}S zpE_0YHj~Q8%zh{)?AYF06ElOk)%$SsbmgwYV*T*?WLQx}qK?&iCvr6! zXYF^~W$`vuA-l|(@X&72&#|#3AYb|IFWVq59OOyB_21YX*D+}TlwEKyj|$mEnXjNK zGb5f!*DX>?Pu81up49D8!(9{4l)|Pwt+VtyZF;{e{dxEnuXs}{k_@JpQ|OaU$;o^S7c$^g{CpR%yI{^&(~$h z{iI-)9<nO`Z z&{Ch8h1B6KUElUO)%H>P#WqvqszWoHZF4uH_~c;37>rLp=ymvPxah$3i!Xn%tzody zLjP@_S!(U+L4dYYPaYF4ZL<#bh^-5cqiR`HE#MIJ)medTwJ zzVYgB*jr-YKfd_KyN6;R52LY(tGt{qD%=)``xw!ZKQlLFKexN!HU?(WtXIRta-dv3LLB;Fdw+=YIZc}&-qdAk?wphImUeof&nzJhZ5 z8@%;bnTNmJk9ofqZ+I!@Y9FNQo!0uR*A#Q4ui1CJxGz!5qDi0gtSS59?#TX)9DJIP zgEFkM+;ZD3=Jv@OKiRGEa|nv^XN}~Vo363>>R}v`k=zpYIp?2aaS6yfb8)=^kWVlI z>d;hmrpgl>`vlcLLFQ00FHqFwCI3w=!OFuSeOFeP3*RfW9(yc%l1+AIdy-_wC`*zZ z`DR?o^AbOiucG2d?dq_NT}2AR)go|O;k;iA;~ZMKs_8GZ=lgde&Y8WdMGTy@C&>XR z+J51v_9QQV=yKDmy~=fJ3AwJRT{Js>GG~&%$_q8?$qmgKL~bvYMWMCjOdy1t^h?O@ zu_t-a$@n5kWL@PAIbW5}qjFe9>Qgh%{j5a+bF(Sw-Rub%!>4z+K0*&E|Dtc+J?BQ;@+%!J1mY8M3?V7rOV(v+@*4wwHQbyiVpI*<~($HcQD~ZrLomrbf`FKK1sboV0DZH{)5^kNlk+ zcY)sBMl@`LE1{C!f-P=lUE?te^S(85dDXJSI`d4k!HS+ha{E0ks-PFSU5=|GRN|KI zrZVRca`zy1nd)29LLevre-lQK4u@4s|viO8Xk4W+);7=> zi*WLx2PS$N*D-+*R7S0|Ca$uR%H^1fGyE-()S(%}KIu_M3X9&A(%^(!rf`!= ze)mPL_yO#K58@j^<7+10NAW~OmSH=nsPX;MTiy~>cWND4G&^ejGezA@kQq1EbO`&i z7`n&EE9Kh)G2V$WkDg{tq|NHtMs?lbUS~d~10lURrsi+h%TDf%&koCoenM-V~u4wL@iTE26gwPSz{ zkE?II+Qx)zv@nLF&pimxhO(JLyL|1<*IHa`+~s*IV_Mp9J|#fA$_5LYj3Zb+CmyVo zi*dPAPN>YC>YQg*u9UTxugYDN<-&NAna+N|R}p?hGLN{(%#z2N)@*LP-do2jx#Bmm zq>_7871=E-esVY7!W1{9bU1MjBfCu1J=03}tlZAh;%bDjhb1qot~tdiPO*#JAF02p zWL{vhbAK`K^8Qg2edGdx#F>F@bBgzkDoLdEr9-u!>7!Ft$+RY(e$r?K9)We z@Kzc=-K4`ZfIgOotiE~GH|-Du#CoIN_Ax?j({dYWT2&Hg~w(9kS1$3_o$>CyX;rV*JLNe;|Q* zFK?5%;F1e=-hA_$5ABc)vV8uF&$qBylR@SL)+{+3gKQ!Hssq%cAs^JWRkT=7m&jFx z+$*ds?B``5R_x)OYVJm^mSBezTAm#Fp5*p`@?eRfpy3X|Npi{8hfz{0{Ses40FEbF-tGPr{z$=!+yZU$a^ID5G2! z*)Ea#&5bFmX0FI?Bln8UKe6N>@Kwl>Z%Q`cr~`>X#&qEqzJ!q*1&n~vDCE<&bRKjf zG!l97dq3c{bL!yo<{+IQo1Vn4C!7T;$)uTBhAM|Dk}4I}(rb_~8!Y%Fp!y5<6yA)8 zfOwsEkQF>TSM#=55tgOU(>Y7~9%+^@nLK=pQ@km0=|@llmN!w%-a8H&pKT{b+5qBH z0)&U`0UA)^BVHy#!0OG4Is>9^dHsrO_$Bj*a(#xlCayEX=CHgbYhfx25vuW4YEgEJ zP>`RU5OP?ndVg~gN^cUlTJxT`5Kp8n=$#=>v-#tl0+{*N1N zqk?Zf(?@YohBwLZIVIkVf$Sj*a1I~vdR0EN#N-q3id80BEOg+u3dj-zQwGU-Lst_h zWIKO8WIxE^Flf_YJ0HxUZqXNR3gsvHru@v5%SGj6xGYB7zeNb?PWeVwGHyZf!g0ZuA|?5 z$>T3E`wpYBXs}YFjK%jzlAS6?Yvsz7(oiw7I|!_c7s$^r-3yfOx8s|HlZv>X1%$hMky-hq@l8-#^j<|Qpm$UcQe^Wf9W4#YBi<0x%pI@g zC%x>QROVF-a(v#jHQ!|Y#y`q*>NJJ&v|q{A^<7iNC^ znfH8V=gIS)w6_m4cjE8@Z;xc4Mkqa*&hC%FXW7x`yg`B|uyz^;Cw zJfggOAE&w#h3QpRuGS&^;dQJY*1{$Hb1}9i{Yn0pIRWt?6Z&E_!b5<&cZ7C_i^fE5+leQ-ba-LPlP5Eqj zj!OAoY1UsB_l+s3B`mq^`GAaI(Rl(0JL@xKzcr=!Lf;g>VLYxz&?!v>E1zJ~B07Tg z;MX5CyZwUG;Vh!7gA?>rH3OH+e>%NlDkCaQtw>Ijr-{Kwehz{H9!JC_<$8Wc0grWx zPigUV8DuZz?i-V?$vi237EN;t&d<5oNmGEFA!Eo|e!Bx5hZ-SuO#210i_Gctr$7Tn zoZ7^hfhuCv$d%V0xpj34D*o0|Qo}H_TZ83&rX^MVZAk6XJ|0(QKQ6vFX=?`x7Eaod z_uGK-Q#&DsEirL81qBLjD;TS%r#KB3uhV;+>?$2%Wf=NpTa#un7 zVuxs2-(s)JefO>Qq!JV5RiJIapY?}6(9yvKCQ}S>wP~?m z7Pl>+m=aAT=*r8p;~UbVT#2f3 z`=;%95EhoiUHw6$izCO~$efJ1@-&4i{xhwtYvtO2Yr~IQWc>)VBYL7?>4^&3qEY*~ zDQFv!**pmatDdN+P+bX5Ju;2hu6?doqdF(|2*V-s%)R#(?q>tyo*Op>7gckk+y@6W>(L5H`2Mg^`cNoLXr`~R``FVFUDSA8dXjF1(^5;%f`6-R(2 zLYRj`QdC$p3MvSw4(@PUw{@#^Tlep(*8KypJ9Y<%ia>548bWbh<`EYapcBAxg#{`c zLWX(f<(p*qq7d`Cs29Wj+?!;$*JhIp3*$WN@qZb{JvnWFXOztA^skn{p$$n} zTf2qcj=Ntm<5T>Cn+s&QXM3~n%8FxXadsYMT<3)22Kon{BfO1kk}E`haVX!y+GP0M z!YVI6PkL||ca-~qDX$b6eWYY*<6hTb*Ak*|Zf@*#H?&0I+@3nF1s%sG9GgmeST_yc z>NQCRZKk!UFsB^CNMygpl&kRk^m35kcE%?SsN?;2#4GX<@t&?)nCzN*b9>))H&dmowlF;+^55QFSt?R zhJrrgDLZW|D<@g1e4ds;z%*i zn8js&f}6{jT7H^W{o&{TF!~n3H?#Odp?jd4M9tfT~;p3O0g^~9-#JMMK@$kQBvxgeWi7_*tTG9UiU zum0vuZ?u=@Fg;_mb!+cC^Mtc{5L7*JP2F1b3%mMt1AAE6E1pbKb+@joarT>v`t7&q z3Q1Dbc?XE&uC$%+Nf*UQFSXdklzbl@Pg+-AD6&3qBT2X-Gn}79(cp^ z9L3S&b;hNHO8a(En@W4RCw%iM3rT!EgE1E{wsH@_zg1$h0^{o=&wm}-^zf-q=O%~` zw#_m=#(wR~Uwh}vfB5AX^Z9ZFUmyAFANuR)>(2T+ndKftQhR^u{cDlF*ZbiG`?wY3 zA8xM2ZhsE;w1$rH9Q%a*+TGq6d4oCgK$|w&k5=_n`GYq`ulO8OggIHtVQsAhIt;~S*O~U(A${%foO*_ z8RrGNy}5HlJi|>|=37BtBVktAq+RItKswFNdH=`We`9NspZRawzw6hUB#6A)40~-* zn9ASG%QJevyX-CRUEk0{?XP=q{ObMsWAk5;xGFxypXrv~ zjODfiPU*}&mY&JJ)-}n`ov%rD%-U2)_I)#|KRxuTXEZ5fennm*Ru;GLw0?^zj+vX| zZCjI69v)y$%Gb(k*U4+;CbyNpNK$3TV(BgsryaxG{RNo<@e#RQkPF8aEJfia^bF{J zofR+kNxb22VM5DcfHys9oD_8b7XVt+dvwPpyTa8`_``XtTp*y z;P6i95zv7#2=T83KKO$V(sf&`p(9oLGteeNdi!z6I&_7R*f~>nF?T_E82Lz6qDDS% z))X(BGd3xBykRtd+}&NxTEO&OEq{%OxyUWK416lQ)is*Dch)iyVXBUIhY zIL9R&+z^iEM711ido-hJgRY;0O$x1T?+}+XWSk$?)cRMFlym<#Ud)GLZj_r2?gjOU zdZqd*XkJJ9m^-%88j@wVD>s?LIm1X)*2%LrzvJTk0%@f^!bu1>gd_MW5YcB^#&DN8 z;r=Un8|!|ZLfCZ;?OM2wA_%8V1pj}=vo|UjU*RL557SMk#{Qe%sF<{wP}XK#ILf{G zAws!ixNwsmGSeSr@Eb#vZ?M~)|FG$LXf5Ug?fx*lIC-G%3)r~%rVA~am;Hs0@rn6) zC?CpgMK5RLp^jKR8MWvlR==vB%C1Uw@84!T^quXzVchyb593Txnyvv~4&*B>yv@r! zB5%3_o_jE#zTK5&#TYyW7mT>=D&>B%mbN^^6->R+ssQHx^4&eoIJ~6vjSrpW33h;d7+8>)W7)s zzli6y{F|lc7+--l$8ewX#eeg~7{3pjWAfkdFy1r1;?~Dl5PhLRePC$`{iO5>qyH7v zM-MK?-9CQF+{d)EYuwa%HIAMz&dYIE@r8SyU-8I4(>iEl{HA6P+HarbPqF7|O62*l zyB6f{VAB%|;B1ni?(tTP5iM|~3um11)((Rb-Y{xm;tGhgj!J(rp!HhPQc5vgdQ#2*S>b$xOh5TbWI`mZ3r=!g*_*#64>G&HO2su~2K;t9 z!%@HtNPL1HvmR_yF=*Hf9*-{G!~^;awSTfE-nH%(W zyrN^cq-hW3U1$!KL9*&^b3bKD$IQ*a1cG)7bdy^H$LJp(-DwLbzuDJ+c&mfH;@VDI z^0DWo?6WcudwI$1+Wmuw+ubc?3UjV-xP?ml0;{s?P?%o!-tS3=Sr1frIC4e%c~Eih z*n}gP*W{WT#Q*8{|EG7p`$ON2jRf2zY9XKc#TR3EvxvcH$$hua7p@?>yIJMAnUrPHrd)bG6Zd$0Kil}_pv@Gs==q5q=qr2o>*Kkzpy z7+)WO9vsa6pYo^w_S4%qt*^Y%1gxn_j0)>|NW|?L<7WNP zK-Yg4_{o+Zm0wyY^+i|X>Y;aMJ!I#BdYauH57Lbu=7mq^DnDoTGp&UZnX62PKYCUl z%ARxMK0l_e&7gj5M(;zf_LELY+IL9n!F}MzKlt(Zm!8c17$1Fp$c7jeG?@SK)~b)G zkAV2-H(4n8?|=8-V}V9}=+}Sl>#=ahn9btdH@^6d-4_i7`OgZW!B43 z_9wE}7uBb-t6ovxyq@8I1HG<>_P7)ue!7kYXV-q9@}cR-x2}hu>ql5|SsIacrZ3V* zmtWVNc}|&UdeE<9$01R#Qo=mk1%-RT3tn&?eSBif>X`YXL-PC?``8(@=QgK0Ig2;g zBrj^S4S(4MoyV7BxCi;|ul{!27ku@9|7!Si&&%dh?tl5Z28-2tX7eZ*TNzUsuh|^K zv&ldGoj*N{v5fUR(`0<*xuW{F0LH=S+u}&czRvZODe7N~>W`nu!&t_+C@psT6R2^u z$Km7b&cEr^HP76Oo*$V%%D-+GN0x3n^Ilrw{tmvr#|9|A#>Y!a{OffsYEmz#FU-Z* zBm;a-_8efYr+?GQa~kH8fajBJ++_S2!Z7wrx+R-?{aQqt@kU>A09 zg~;nfXf%Dw24c3s0}*c6Z-(D~q@FUV|3HX&l-PP%(gW{KQy zWfKnI{$Wo!1D_OQV1|9irth?T1U#ho0S+)hV8X%!eHJG5K!1j4P>40CXgq`7&~D|f z;oiVX3rTzVZyx3U>`;^z^_J=ozc$S7hv*vbVy^91N9)1}(PnFBD$YPFwX2tFQl1Z+xe%^0M;LvT}17LKuz=GZp-drvpvr5Fu>C zEiIRH8=nhVn)|YKk(=v5^?TL-+EWAif6AAeSn8Pm z%@7kC-YNpzurlFcEaFQt+;H-Mg|Ei&uaYU${0peqE;}P(i6c1*#8QK$hz&MP4(m2Rr`CIyoDr0jCyUM(A7gl2a zJ9iUZ@3SD=lu+$SZcPv0EsP|?L$&d)Pnp6I@ejAKcK-#aQ~GSH`Z3L`G-lENJp$?$ z?ZoRAbqO-BBmEVs)UrII*s|zkpO$lm7gbkI`vN!vbObTJn^$~>{p>P2j<8ZvB`I+aqOt5TmflittW!4b_Czq@DL9gM?lLZPo>@q~Gy z?rZd~A9(ARg$?GN&jB`bL*}kO^zIM6^R3_dRxQLZ_k&yqn|7G5`m-2;ybZ0ql~SIc z=?PbD{lMPld;O4`+55uU%&xjx%=+qf!egGrvE<4h_Y=l%Rh~2*`L)u7Q8?ekaz;D1 z$0`3^_@u%dzI1cO48ys#Au`S7?p)NF7rfvF&kv1PJ2QuA$bHrB-imv6yKk5O)lD(^ zJm+v;(*36NaW$2L=NP=~#7j?r#eYFx`eow^W34s}JPMxwvI&9vU|y=?UYKVs8e`dn zpfQ$bnmlvT0$a!E?RL6U_JPqKBGxx9Ngr$d_l~3QosC0>F>7M4u`$QQ(l(BkeZ@%} zer~0GO^etE{ddLqGvh2j2B&$LRNlM36j8d%!Jw}MQTEy-N;8kpr;TaG0 zlg-!qm+(BBVdD;SJr>JX&nIbbJ8f<7*_d(9K%RQOR(aqQYV(s@=`MMmB+h9Xysc}J z%A+l}hD`GCpj|xdBTUJQDfe&+W&W=MnU1haU-p?~xHH}~{!BQcFD(wA%`LA4 z-pHC{_B*VlX?;@gb^*%Hp?0G-bPpxXy@1iI4)U^)#>ihJ%izH7-|Rw|Ia;_Cv?fVj zzR5L7)ay1@a8z*-%>_Y{yJ7mw0VV>c1gUssWmAfGS!^Jno z#K0deHTH?aS>`BR*Vn+n%z_|0Hod{}HZqGXbQm=BN1y>n1DwE)O?J>~M;1`Ss0K7? zC%Z<(p@wDb6Mi@MaA*D%KO#M33B269+gCUV=jrvd<7(_W!#mB+@#{jmyV!rf$2VLy z_tZCmV1vA=2k2Pz7CWE=>l2iFp&i8xaKtH)=6fH!S7zYtW7S3~jJGeu?K3LOEUdY0 zktYSq)}G%FGviU*2TYsP=X1Q8=rBgGIKd(U;Kr2)DU3%Q)IGi&L;c|flsc)0Fo1_Me5r<; zWEOGzO^?8hnEZ897B^zKIp;ZU$~+z)2T7OuO+N6MPlxjCkf#IWcWCllc`x0la;tKr ze65V#Utvz^GVBEzU7PpK$aZqpHlpS9wx{;k=e;X%p*a*DU^MI7hWd?pk|Vy=bYVj}xvNVlQRh;F>eJ}EQ1o*roc5vSUoy-QpUeyBJitb5 z#zQuu^sg2&j`FV`c+mD-|N4P`lf}AN92xzv{>>8LZB-WUnD@;V?mTa0CtEA8Q*+3F z#wxE~o>w@l6V9kZsz+IuRG-{E>nljWZsH$7?)XTccy@z7G<05qZH2UT^}2qV;eU<5}n&w?bqd z^l>q0n_sSJKhNWwF)pKDr`boSkT^zo##NrVa9{AvFMji#U;O=FtmitfYV#>$DbIdb z^w#E>KmFR}%Q1}eEV^@D8eq`=K#%%N-#gPX-^7$iKefaC+`rpz z#>puWqpqo6=d0ilLFUc%{CDa3*c(}sHGQ-N< z@z57!KwUy!RHP|_H1}{bdd76WB`w}a4$a~I^24KnL5K(3Q~*sH=p-P62bJ!T$%5W$ zrqXGE0}Mb+o=IO{4*+T`(SWj%q|*gBTLS}IXp{86HsBomXENL~kyJOlY+&Exu3TJZId=i`}Cc*8aT*n+Vvw3k?B{M>=Gw8 zx_WS|i7}ff{Odn6R6b|9g{W*!c&)F!wGYmVsC2_De=N_8Dm#hFk)oWG?-{-CtI4@G z#pt0*hVihSN;j^-uMr~n8K_HaMq#|+%PrI|ptp*6h@!VTZs{M`6hj^2JnG|nU)?sw zQHNQ4Af34Bu?ew5{`6P=wIcuG3vU2F0*{CGqj8XaQ5$pgr4fhnZkL~#r)8J<2JXu7 zY~lV!9Uv7$W;MHi^UihL&3)oHprTUd5spV-aSJMSe5( z9A7A+J`tz-XP|i<=~K5ISz2i`JBOF$oxG*rfTR}$VQpbjID*dwkI&-ubT5z%N^b@E!LJx@p zzlc&I4akrj&)Q|rhe@TZyG-`zM-}>y3z@+{sluf@z54Y)^T;YQbl*%t&ll&6u;=*;t-p77Al z2V*0J?Af-Zuaf#P7K;Cy+w|u*Gxu^^5t*3>XCMsVS&p7%#OBj{-lEN?+~+bTKG)_`?n&y) zF~4}(920)yzL(80jH&+Rn6oh!sIRL&j{Xms=?T40?R_wPv{-f3MW)Y|6FgaY)*GKVJR$ ziYbEJadKVEu+qd6&QG?|FX1Z87&7OzwVY+K7~ldqv5I&04B3k5-N-y>f8$| zEfq}zoWLN+#D(;uvRmUggBnvvYlvs`!RYFMao)Iotg!; z3EGdhiviiaeT5qb?JG5+b}R1AaU#+~gv`~Do!#^OQD!`fza!i@6ehI>pD4dqywjF= zypPeso6M1Amu#Ct`AR;AM!XD+Dy!0Yvo`a0%D<>IQY+oU&S;hevb3t~4^wPIjI*DwID>f+m@$S6<^{Gh{!!eZ(ngsi>fgUGsrl@eJ{t=Zv}GQ6 zJY}MzFUN2b${15OJz00(|FQS$i&8UC&$zC*DWlE}F!nL&VsS&`*CX(Fd>rJ)nEYTI zWK2~4E|Ff5n^_KJHuf^?=CWJ)SDlbw>M1kLLc3)ArDX=)-jOXTZ4KV0W^IrId#A$5 zZ1!dQo-sw~r*RJa2B}X>=s)v+tv?$}oqq)W;6VK{^@%Z%&p~P1A*9 zx)RKXu;rfh%g>oo=?qz7;+1EnZ34X9+stR@?XGFY6#Z`8qz~cJ&;5t<#XrL+-46U` z+TW<~O;CEHLO1`9fIg5qrJGPdKgo^$;r}TcqJ5lbwH@ps=;tDl6ThXg88&s$A4Z(~ zhWP!iGf#GU(EC00gT;J+J$uMKWj-MH2knOk>o0sQoNXWoS~nf!BIk4a^i{v|^GoKWsBc)<$_eO&7H85AZ& z<6RpAH9po@>CbVl=zhN93`&N%t|iR^ZgE%`!2KmJw6UJR__o@7%2!^VWAiEZxon2{ z!GHXN5hro`<```_WpfPIcJ}2MLFUjLrk&B3dVebW39C;|^BHTb(%9DQH6~89%x+)N z&z+rrmAWqHaPPlE_E}y`+4D8+RR)OZLz!U@OBf2&P3i@kWWM_U`RY5r^{;;`=3c@d z{JZT-GC%w4KO5@`JWG5o@O+XlC$XW6Ii^3O;S1Wo_w&CO?Pm6`ma46_wkSk9&2|W6 zC;X003d89Bw<7Y`wdegcD99w}ZcVbiut>hW)-_23d9)ce<*59d=^N?&@}X~DaY&?} zdIve7-MiZxwph0Sr|~BgzF%9Mf|6TG{XWe3pDkpsaIPDN^A5u`P3L&yYm)!hNdMc5 zHOU1IZENTd^ol8melt0|HhlumQ_Qn+@Nx{9gn`5B=$_#f{~2_SuL-`%HA$79^04xh zKL+e_Q{D%WG^>o96)s<9klVrE;@L6Xm%wZ0J#l~d3o<=YU8LOVg1cn6yV+&9DZODW z9d6+ehA@Xc!DNzwnNL4y(Guw4I+c&f=ObW3z{7Puljaqe2l`6Czu_!oQ@S*)ByB3p z6TdS?gPyprNIF3fR`c)XA&;WRQ}GYt?0PB|ensWMVNjBJa2WFf_FDdua90?E%9e+b zUbg@2^Z%?C16jm*1k_P(wy0lv_z39CJpEIsU1Yl|bBe;0pG0Buyw2gZ5p+F4S%<+x^O%=k+ivR@%Cr2JQUi zv$srA_4OFWg-?9y6EQ}xK=IUx4s|m=u{-LKPn5XPAr0!1K5sY!*Aq8oquy~7EihrC zo-=+u`lSz#hjFzw@r-<+ui|Fizos@r<*o9YBhB3}NZI9sF*qyha`SR-=M^*l?P566 z-TY+Uf@Ii75Y1|9XD;EDZec}Xzmdzf4x1D#oz0s5a!-*T@P5=E()nj&ef+uy)OS5N zV56D9jfCcP)U$$$P-SJs6fRkg;gw;glHq3cYUX}mizhGOV#g`eCa(e!v!y4qyB|dE z#;d;-w)2N^$8ZbdCLFafPUXK2|P1nsfH{kwo+By9D!)!h54(0>(qTF03 z`vdiz`wQ=9&rjJi{}oPttMU<5#!)}A4tV*CQJ0)42hfG7?|YO;x2J4Lk=T}I2b-Tk zyxl&`>MK9<-e=+gIdigjyVw3j=e?lMWFdqF5&yRjfO#I9W9Yltu*1d}pKo?3Pj{MQ zdJ=!}ovnG#qYW3*W zV%orJ@g7}44wFw;V`;pI`l{%2WOD#XB#^vL`Ct3lcK!P_`ksUKA~~^eE#A~t zK99~7zmLdW0#o;^YAprE;m&>JLuZ?`zJ-Qja;qw(1rPPz#5UD3oY^zIDcUfpxZ4PU zJc_v4rNooRcl+E`!FFNilKoO+0Qj(UrQ#mR!}eGU=cMgkVwoU0XwfIdOQW0E@Md!_ zMhQKwy4iUog!P>C{1B%ysQ=@!kYn5ES%A*Fy@#TH39?qoWe^e~gS4z{Ls)GmS79%+ z`C!EBa*vT*$)MOCf-KB9kH!XHC)r1kJ#ZP3J;x`;^DVU>*ob=@F8^_h{l_y*wH-_S-rhCe$!hh+(z@JkQ_pru^t#zU%FQ~VD-pI`kxts2MU5tP2EW}_s)H?5Po#@to$7MEHy7Ega36Up$2v*MUI9_SD@6#=Hl-rQG~A65G%D6uaTD}s_wOo+fv zQlGx(+RDy6fsfu?=7XNqqSU{v8!0yMn9Pq31(c>VR|U7>C_|Mr&+KYnkj72o~=Him;GntVzZVFFv?_XF0oslU*8XWisLZJM6?%{ zYTmQ#miphe_81!D9diEPOafgGDZJ;8)E>N!d+4JxhbPC>$_I8AhgZ?7o(qI?b;3zLKGHM6jNbgR%;(P;d|M<``alz^ zoJ>&-de#2#D3D2;-1l|j0TwuF`O*d=Y=W9|cWLo%f_-)!nHc7F6tgsR>IbDq;uEm@ zTgLTKOj{6o7J!<~Fg!?9gye^@ba83@b)4dV7@-1PK#0^##d|(aBi{%+TreepI5gLD z`zW90+!uS#Md%a)Yurgdc_aXr|R&k)w zJGDvgjbyYkyesz@uetfrJBh9|?oUVpv{y6_4ZIkJgzriMNs{&|TfaJBSNmMa#956f zeL58iDoYBON3G(HJP;%As;_AT5*|>Q;x7*+DIPg+wi>oQkAHSIBj)Z@Q+Vgy-Q@bF zVU?+R%jk`X<8sM6_G%NN&)bv*T}(NSP)Q2es+pI_KoUL|gc~Nwk;SOTSFLKu#`|pO!akr7B5^ z)!S`hB}k@m3)y5q79@frW5Y_r?)t(O*fQn5s!d8xS9R^Kr(~?`&qGVt5b`ecGoR^a zRTO2GH-}3#?fb4jNYJ&?tb1m2(bDWr{5kSjpXAj!zLo{i(q2pZqVdIGbApnK_0!&? zuUa%&Yq}s*aoNWi zM1Awh@_1?QF}$i!2!SXWxu#S_>ok8#`0|a4w4jl~rB1z*J1#JOjWHI!^bt<=t?B8) zH~=PGMt8U{#@J|=YSZA~>G>QTs153^`@D-T;PS9zrtjr!;z|}i$*pWVxmZ$mhnw0b zD+#e&qsR|HyQX}0$X?xorjelu$+a*le#~-DszQ`%<)~n{mx^dLzU6S3TQKljB+UYq zhV)@~_%7r^_?$b!pDyd~;YZ*{?T4Np#?|1QSNuN?K9Cnp)O*!9Vb?X>&7K(HN7{jS zoWEiI#lem{*p$6clvd%UBOwNIF|S+H%+%nX2`;kR_i7hN{xs zU1&D9t?a}tcLSs|4!~Hz*Y{dCJtHPsrZ2Wuo*~dO#UD@aTajfYIl#-^aLY+fu_p4= z=qaS8#>T`=;&%)|vLnB+hV#XCHb55$MztsPr{@E%f+#R5f$0iVe>GOTgkPNa^$k4A zQIjE%rd6C<i4DIwXVv*(mMTIwp(qWPmTe>zY5|I#Jsv^&hIO-oElSUq*R zsHEm7jUW_%;(4<`7}ZZ|9;pP-o*b}sjTNlAtfcYM^hLm(-&(Ft@Q{C1a*p%zOW140 zi)y;bu}Lof2*v7mbA{Hys^L2xs%kZZb*!V)!}snb?*K{nLQ@OU z1W-P!-qW1oVb8cH z9S9}N?65aBt|m0`(9cIFxOGW)5{gJ}f+RL+A~Y?C2s-xVriSqb^B`ADS7SptiF65l zrgCiRt6bf!LmVxT1Erq2KiG+}iPACb2KP+=F9FUom>#Pn4ggSTL;Y3qHe8=T- zy|iS1=*U7J*}5KIsSQL_3u$AGjn1imS$U18dl|%o^Szqo{GX@9*~Sc56m1`;%x`c! z7pyAxvxxExvY5S}2FZhN)uY5nfg5&Io*c-4!_%Ukb)wLsqX+h`#{%CbQ-p_&#`^w~ zbAif}@{}+6!&O??y_(@3&!m|CZh?T$4*w`TAJWF>+$yZkM^G{#9+#1&ag}$%`YSKa zBQ>-hv%ASZDj9dRoUa;dXyFKYD0}oEYIc%$7loE`gyfWXG1(V!J2kVmECc&2K%k!a zsg!VEB}(p0P`XY&MIpZKEAqx(he_i9=brYNBEFaM4unXIKc6%zlM{H^oISo+D!ZZC z7^>iL+C-m}JG42n%5nU&o&&O>#+Q4iZ(Z5F?wdxe`p@sG`Dg zXtxd0VWtL(?#ZH4k>r@hS+CU5oMZA32@!T3B|iONMeOz?4k`$?*{~RXW1nYBDeDeK z;YOYtmx)T|;|IHU$!Y>`Vmz(hj)MS&nd0D>W}a>c$K#(LT7O4Oe+tWg!|+c^atz=# zaJyTVXz_w3cEZs8=$)I@L6ot(XKnZR4X1=@nQqrrmv(Tno%N4k7wg`+A|U-Gr`Wq~9EzvKuqyZ`gBI)}j#P_X~)KzOFK^M&m8m3C{i0l9m3X9N^A z%<&-Q;4=Z;(0YIX*_ITdiXA12rOM&F>|mQ+P8R=ez`iFQu>Boz7=NoS)|J*pi`s)Fy=O zA5S{&crW=xGz)cHp*kKCS*30A;bHrD?x$%qC%c~-Wlv$d- zne-*7>W9rkH50=m^%+hmAO(Bf>`Lr2srkBr^l3#FgHsWSde~f-$k^A_$#bb~&p5Zt zIf36+8I{JrFJtXv-mW>{^?xy2`8NU?UIeunw&xrm}={arzM?d(nVowAg- zZ#oWoaFmjl3ZK1o9A59qJ-^B1Dzj7Z*1;*E(EHfK;b-*sc^E)SP*P}S%ideZxvz!+ zxD^ouggrM=b(?5^6GA#T?tlB0@KUO+dOa0q+2_{hfiS!R3o(1Kdp8K!)&2_(c9?Fz zIaBHgke`u)P`}}Ib@=JKKwWAvfpZ&if_e)ryXonm)zs3gPM5TMy-OmFl->a>LD`#@ z$bRdccd&Y6i9X4+bpr{x^kVi5|Bky#Z0WCzD$DF7c!ZoO#rfz=j z`W>sA{3f%TQKO4%va1W8D2L7k#X7S~K3$J8xQRxAHJX=##w)qOCU;ofYo}hN?U3g~ zE>ph1RlTzEup%3ph$vS*SIY<9dU%B=>q7TD+M~%2@ky%y{;r~Q%malS`M5Ote=X_N zx@RE>C+38&piB2CeabH7?#D)XdQx*&^}>JXeWFU(`vSbsa!_w#`x)Cac#r;i%M9%t z(^&r2-VxsjxWBmKt@5kpCLQe>Hkmg2E~Mlcd*SxMjD}YA$>vFevt1WdL)mJE$G>~n zo>RW8)<%4Pm1bkyzyixQ#9YSk3acIrSLzwIm1+@=t9n5LI5Kv&yrES_yX%xoUBvt3 zj$Jw@Iovmee_nBO9%HjUGpJu$T<^k- z*Zo;bWmMag!uUo`6)t0;Z`sjvcZbt=yEl+mZIz+heublEdTzsIX z-$L>^{dls+CDpENz`@Wx!`F~B;#D&hi*Ixp|CJ zR(qGb?>>CCZWx3Go66u*5mo8w7)Buto6ga@1WNhUE-)T6BwHqKvgk#)K1qhYsA;}I z?9^K=m8gxBt#*5w;q&m@zJthEu^agA0Z(!IOHJ3%x8b%?&D#ZNeTH6}Alc(FL(%%N zm)L0R_cXW>th#<7OLuP=Vap%ieZCP5zEMP{V(a|77=oQ8!EH+2#yZ>TM-MI+76j0R z*C1oFPMjM@v4Ji8BdRQ52kb^}K7QFHVFbtWpnE*k!Ywg6q(Cz9eWzizqpe|FSxt!{M#VZJBol-*3J-Wa&g^#@nH(z|{aV09qYTA9*Z9T1`n!%iGb zB+Pdd)4UX{m-6VgTURhAF$!t z$KBI`YROk(JGfL3_oe(E2v#t^>&#j)--Emy9z-4 zp7#U0gRDQvRQJQb0)&q~k%gm9!xK95ye4=^O~B3xVbHtYq=~6eNmKn@Ac3%}KRKfz zZ-%s<2iZjtVYmLM+&@sU@mz=36f(aF38u^Yi{AptpW$ONx<}|60V5LdK52bnN2&_d z@sct_sK35UKmWh`X3Q+ptd`MlLvOVPBk%#KBlsMvs72OuAueO4=+(Dan=4N)#Nr3nquIR#_9;|8Cy+!O)0SYr4NORRO+1xYPnUt>Q?DAndhspn zLSYc=vqz%Jf!)8)eDmsXLBn66QEwa0+)&5}9nS&U^)czL)NpnAyC@nl^Y_Wj$^rfQ z_fm$nR%Ps8>5$G60ffX=O!Kj|++Cv#5USmSS)?yhc5hgEI{a8OZ0Zxw$P!IrwA_`e zDesqQW#}I!rbefgmQN|J**zJ5E;%e>(dx3}39$=of1Y^Up$EDZTp--}x5jWxK1liO zLQvJc^vC_9mA1fzF0MVGrpL8Ts`L4`LuO#;mGFFze_<-k)FEE#$|t~VPGCSsikf`v zM<0R3H1sc-uako^D`~+#uMOWxxDyhlU=QeDE*Mgkve(av@yH@_WS6_^#P-_dAGr?z zAvkE6pXRZ|As6%@{#N@<%Iwl#YNIPd=fx12N-#yT`6fn2)-S2oS$;?=!{uif zE{Ii&hFPc3rWbf!-?%EVxj$Z|@#RF@R16yp4=M|G8ldc}xKm|wnI_ruBWA^;7_?P` zDH=UL>8s9zqQcVV{lB^u4pj`jb_6@6krLX}{bPCJBE@T2VYMB7KiE7tD&OxKEKmHB zs=}430!?xCPn@Uy#||r%hr>@phg;#o-`(yiP3m%chbmoqi|h1vg>WgmFU{Os2wYjM zeEY&7kfG-HEo1usM%Kr^-`AekROU;2Jm&?tEsc!IIC`#=-|l4oy%Lr5Ij%hPEkXcu z_kjZkZ2!Nm9x?7y+_$AfnLf2!a?B@T(WAHUi$c6Mm<=kJ{+v<1WS!!)=Y(nGI5g|O zoYSW3?w)S94PieYJHK+A9zJ6j3W}e9OHgz8gbm4Dcl(e8J_wx~d%g5s%^|TA>@sXf z+?FdlVJTg-GTaL;Rgnncc-vUE)FblBFTH$`f_-#oHmaf2F@@G2Q{RTQfFlj?p@MR% zq!B1dZ;7FM_lo-fceyeiZ^ezI7 zfoJI_0)&!qL!jssKH_f;Ung~rmW0QHo~`H#9bOUWL&-GZJo)b5VKf|erYfkZe~@U> zM)3Tn|0M_yu^ydbIZJn7>mfNiHtm4}Qfc4=K^DV(7AK&mpT18K+>JLMTl5=W6YvTs z@Zm;?#xadEp&;eGrt?yk-p7_BrKgVLZ?_dsm8CXcwp2pNDTdXa?5*8)0*3tAq=(01 z3|bP$b&CSRYgOw$B@AG_dFm^HYQzpsiGIO$C z`=lhbpZMQII~AeVPkNE)2Ukxc(z;S}J*lJ42@Qqc1I#+>DRv-z(~e~?r1TZ;Vb@Kw z3nL*Kv87wQ6io|)5o+#fus&6^fdy#zHaR#&>Bz4H$8dGfy=()6c6~ZdqrcWDyn5K` z7wC%tT#+o5D#x1#J0*$J+OJoh{!^2BSK=twT-p109NYrz)=-_iPrWB}Y8^z{aF_v+ zJ+d>|edK<{GQIo8H*YE7x~6aT($5?=1Z=rbe_pSVR$T=md6;x1X~4(8ev_thj;^wa|nETnC& ze;Bh{+8(uKMbwu6iD+4N*={6{pz;i>`PL8A;Y>f@)4VZ1QJTAnH2K>Q7}5#MojSN^ z=BM5l74C4W@%|Muav{ji;Ni&Jg>5)L+8 zhY-dfss}s9eQrc&BEw3nD9H*b149V3(zd;^I!4UiYW{&o5IhOS({NwrN0fQ9uhJrf?$gnP?= zcQBU9z|9Phz&;X6@@(gUy{IKVQ{7w@J=Oc(G!n(S6G~3T8_QA`d!V5MWG?A41h+hg z>h-T+Jd#~}59d_)%(ILP>kYkCJzo7?B|c0aZN*UpKJ2{1r%C~fBh0g$?3QdT3#?)0 z>K<#8>bHj2woPFrUV;pYsim@=$6riwXRL-oK<)a-&7fQRDp}$NIaZLET;cutquMSV znL{a$_y2RxGo$K5^ksel>tgRC3c6Dq_>WpBLL2xk$OI+LEkPw@+ZpDK+=HTt?nqTf z%8H08p|g^I_tS_Re{z-q$daWpWw9S8Q8h!~>1mA;|BE*5cVteFm%mKBk|I1~6^`Lw=+x$9*MB$r zrio|&_Rh^X-;FKGROfF4+atdCha}F|3H-l&MjGx}yWBf^A&eF{7Lgr-1-d?j4&t-| z7hIp{<|c9cf2zBQ&a!0b-@-ufBmn$Tu6=_0y2 zJla~%DbeS-rDRY=+i$#p>Ya^M_kh8mirx+w5K|)!nYwn%S3$1#1e(@{9y}IjJ0`=Zjf2gDMI9YK8l}4iUN&;~ zJz@6G7gu5NW%@PZY`#-#$TqKXxMs&RQWu@p>bosjk4k*js^_M)R4XDMV&je^{VaC0o9@~ z*d77-J24$M)Sg-W1jGD}`vZ-DUe}<_sXo;4WdA%S#(~uots5Ri%i=T?-_)Ib8>U*w zMZGeey+!{Mo3iuLU3jsphEZE~qExqtdEmoq;zMZn*!P|0vWBlB>pL5#lM5*ffz zB`5!Bwoa%V#o(4qBLdL1Xp3fkIGo>CQJbOcuyjZ~jAJ?#Y~>#L>9u_1Ww2q^#JLh1J3=5!u~)l@zE#>Ac3){q~`sWlV&6IfR3nFrQC>RuSEJj(f_ z&BYAT2)>K?6-JIZIi@!>FUEHVPxT0g-s&%z^iWuK!By=w6&{M& zuS0qQ5LG-Iy%k+Wzw?Rl;Kv~F(yDE+A2JP~a3X(M1}ico{4>B)h};uxerSp*$hMhp zUNoK)Tuxi@T8P}9Gg*M#%7rRuYa%8ub;WY#7KeV09?tdRqw1DSjY#T>KAZnm%koFE zOl3V*Y4+&}4+ygI`Q|6D0YTX3l&=BVPya@IxHY4ODDq(KHOm}(!Q-oA8U55wt!Vss z?S{2Rq~U-yCI$1-NxKPl*0SW&>AwpJh=kh_I?-C`oALlo@fA&CotG#{ggVd@sBDg5 zV4+1`+pn1h1-AzIPQ84c9;08vhN*}yID#Nmy39v(`uO|_44%r0zt@hqL8Lso*+{LnvX%p_Y7 zVv1XL{~FD9x2{o2HmGl=)g(GP27JeTX~rcIxr7^=yopPYq2IDci3h1p&~-&?>daoQ;F%@%1cso9;U6&|Omz9;us zmVTVWj3*&(ANB&yk44S7ltUdXAy(kSWr9ZtR>rUaXC)x!6z|)7nM}aqvm>&0TlL13 zo#ta6ZjNg@ndcjjVKsE4)8+rg2LA5ffx#QIgRRGy#yKIHzanxMa71Wt#{mp+ZE44Mjyva^|dg(Qww~aDfN-EmI%W^+!e!*8=8HX&l@-EX({UO=&2lq?wuCV zr!L$-Ob7Nawg}xUL%Yn?(~$qaPs}n)ja)1v&*XDvdEDTf4u{()uvRG68;dNde+pQG~r(bS*An_MrSg>e;Z<#kBnAzyKF16$9y#+Upw zni?2!FLNDkRpIJ&r5_B$9fW8bIt=Jntrn7I>B`vE8Ajk&YF}L5_^L(0<&>XmjvcyE z7Uf4dQFLSR{4?!yHNL^6W$5GLFl;`TQDmhzKD-{fuN7ySnkU?*#9`;Z9 zxZZEF`>CK=O1P3UrK@ZeK6~E?_l_~xLD^mS1RuXNM)nT&EYl!MZ;1CvFYf(<8cO(Qq12w3;c-%j_T`YIUSy(oEl<|C8U_Gred%orr=Qex2LK6cJjUED} zu{C*{wuKyw3@PtF4dv9|eK7K=XjppWyBcd}oGr~V7j?BG_kw<7X=C^Dv=gksbvL<~ z6!+l2w&2jrF#CJ!-fJs`=Q<4ZU$tyhuGOkelCP%a1g7==*1MkOod%sZMTOi6*iV@$ z@BoZYar#8ja#vrsc3w%}Irxw1@L-gSy8kCAnGtR(-)4`eXN4M~TOc9f?w@yywj?^C)E)tn98cfOo9a0+&ZM6`s%q!r9sg zarX>cuMtwv4$dZkV?ua6@7^iHw@1;6@x~%54q(<3Dcm(4q~?xSCI=_)C%GP}K!=Gh z@}DB1;oEM1IyA7WmmBgc7;^YVCH%MQQwr&dmJB1F$%sEvJv*g`4B?$3Snb*gw*1fL z9oh4yO@B(g1Kx08GHm@<59Q}Tk@i_K1}Ooji(OQr*7qtBs^YDCveQ2iRaxMtT< zpW!S5m_I44N&&2-+#!i!{}^;!mXPZz4ri2)<>IO_(-7yS*tY`gn~%Mv$)^7d=(3l@ zh^`59iO|0u9(GRD-SH}A)|HG*DT5Tje(U$SMAxXvmjSJIEbxY1XwqSMVC{1OCg45TkfK4asjP&5aajcSNgqBu7*`00_pa%yVY+l9oz8Bkp#k_r zk{y!W7O1sM+<=Bl=gpR>sa?yPsd@>#;)a#Nj%v{g&xz!>&u-GI7&kqmc{N9SKDK*-~ zV3x(7HEg-YoGA`#n1oyYi6)!Sx0Kt(+Uwo*UtxmjzBku|*(9CnneQNu4~ep7_CZ#3 z4nY@^15caM1FZC#cn^sJ1TDb#CvyU4bYS-zb~2-U4`IlY77m@>we@Bq<&}Kqe0#P2 zu6&Z{)z<;!=9}{q>@WGBQ-zB#zn>QGj+|UJ>+;{PyT7?hgOT1s> zgW8ah0{bH~xqpWaMgq5?OMlhjvEEP3`5dsod}ynwf(~N~&IfeyZ|7w!9oD8lvEz5jQW-Xqhc@ zV>#X*7Sj7_p5L$+=!OyBFK3p6-raj)(|3;R8GpHL>v#ts7X+PiE}9qv4H3^PB7eaBlAw zCW}4YpkDscYTKhi{x~3p*CN)am8~@lz>7un2Lyy;K2)@#w4sKmxzz%8UR}DMk2g~ZiX;c zUi}(_+Uw$DxqX}A)J0}=%L0zG-xojW+o4O66Rx@2d>7w|@Ej{$e4t;_G2i;m@LwaB zKwH9D|I(Bk_E@1AG3E7jUSL!&lv?qzV4qu&dEJ&-l%qTlSb>5<;F(ZC#Fh*J7rJRqO&Q}HEKRI4t!%EhB( zOWV%N!PbFX6IM%I^e7cKlN*-vJ-{>%Qo0K-4;*=k#ska`=YmdsnF~suFeWvq$$#zTztU6y; z1rmHLYmw959=^fTbAQ*u!b;7m=qI(=mh=;P?-OK z890KB%ElGnAPaRG$cL&Q;?z7l62kdk($i%t4vi z4q+R~<`|f+ycD%t1UlekS)u=U*nBZnCI`=nWY+o`W13{u#g7*Lv}UNXEf7h>MJt|h zOIy7U7Hl8)CNslg6s&^`-t7=`Mjn|6y>_B0JAMUVU1rxIm{+4gr;n9(?(Rq;KXHJH z*lvC>-+5xbJr?DJcreC#62wm2yT;?IP5=vSOb3n54Yhnf2tq3#w;{ncgweaJX{(eR z#%LAO%Q1649qw4|HpL?GH%F{hTWU4Ua>)rO{aUxF@Ha1s!YcR`zXR`M6y0IF?lhbE z=MT0n7q_Bq1Tfp=)p!&8h-=-NAHZR6ciGMxN4Ybf1>fj5YyA(ORMqDf z<~p89b*Jrqf14fcP@8=`G#J!|@Oxd{=Nm^@;k!{W@eQzGv1CvwsRt`Hi;aROgq0`6)aX0ca(2?w`xWxU2@MUSChvf z%a_kb!nHoiZq;hiXX)|4}%dG;Dc4$o|&-> z6-v*U>>|7DXj=tSwPqf%cFv$WF6zM3;vT`Z3=w2+c^9Qrwtti}>9d|yr9_*wgow$| zeZ5kd1M?CMySa-~25+=%Rn@>2x=KpnVak8tB&_K*zgH%H9o=G+VPr7ps`!-=BYlH- zAKxYUA?6DzN!&~VUu1XisrF!1z52Qea6|#^=g6z6`vS8%%nD`P(Z+&zORkH5krgp= zO64}9zQ_e&48%56&Wv7hv(u=X_guXjHfJ?ru*qU_+*;9rYPs}(`L;gLUCcJ;QO%Gc z%!UmKx_yK%2msCrxO3daA1J?9HTg%TzU69cF^&IoR(7AiF16^QyAqYRB-YLJeM3ERh2oVZH zId5?@d@%qVXO&fJ1|sP!U@1W{_g z#c?6NZo*R%35dW86phJ7)jyNAEL~T^k?R_A+6&R%8BvEU)_PeQ@|cleR27v9nq+(d zHqr=JDB}cbP7{7__Id#LH&5p?mk7{Ew#|hoF9lJ(nDHY|WkjoWuJAVjBu#4Qk45e? znm=J_kOMXqy1a~pQU8WbmWM}Ad^2t+hnA17t#`fux1aq=%X0=-C7h&&+|NuxJ?iFi zW$p;S?(h&MmKpjapS|_;U14lHR zjVz2Yh(qR|?V_$41LwBl_a5UCQl1I_8FPy@j&4|Bv+!F%EgWL$FUG!sKI|v0X7P{- zuYP~@J;o4u7t2)XkB*nZjfa`%Hs7>#&OBB^)}vN!Fft0i43(V-DQcQg3)nBn{QpGd zFI>>BxjtFO0KW{{PENnB6k z$`7VPFGYeR=t-9V{HdJQ=htg<6jbBM4e zddu_+fgdl^d4}jGgJ8#Q$yv&=M@j@-Rz>tz#vs>542ph;)h#7;H+1ZO9BaT#uwwl zUvp|z7*2h>0zmc6v^b+^FHv%f4(t9lWr-hIcP1~$ryRT@IIJ-}cb+OOrcjjc#*W~= zRiEpPYLDud8`$e2Tkq@ifB+3%yfcSlSiQW)q}n!P=cu#oy`e6?+?W8bs}GrSX~M{W z+}so@1)KYhiRXy;usiA7zN|WE%{(HtdalHKMmechNM)1RFFYxsHKe24Q^?Qpvt1Q#xII2`{qjuJeixNI!bNaB^nkQ~;LrTPC;)6{JZWtpb37E<>8RS+#dPH4 z0nhm)MHy#cyTYB-B~73_CRiz#a3vXhfuCR+u*Rz+pZ`%chia%Sec9jPMd7Zhirg_N zZC#)q;c>qfT6U+d*u&E@VKDc>lX?J4VjfAzYwG-V5v8&!G}5$91ZJ9K0~g&kAxgHK z_;2Z!NSkDu?uEZWz4WF%ZL63#*6Iejk+SZ6aVN*HI}`>~;(;+l*S{aBr2$PJ*}w)! z{a{w)Ba#CbX$=z2=t=OAER+;}o^Jm4xivW5CN1(hZtW-OSL6@NN0U#IS<_s-e;NaM z44ex44=yPD>15sG+zi5>Gql&B>w6~*^GScryl@IUM55&s@5#_xeft6ij8xk8) z?KTJbKx0yF=SU4l8niHmvitB{k|5~pMY8B>U!&wT+fp#uyNGi4tC6M~EjX2gZ#vGB zR}J1W)W+N--9V;NSEdsJ=KGI$&W?E9c)T_rJCgYEPig}ylAf$2!#anb6wzVib)#`f zhoLP1-{t0#{avL+1>m@(y6Q(N96r}_w4e3r6X~DO`PZO$` zR+g0ReVO!H9*qFDS}Ch8eG+<->1g%7;wF*oKY7)cocH~^PyOcTS{dXg){|hmTiI^$ zWnNlvBaH_^zx^Osf?$FeVw7&zDJ;O7vxgI$&v~^w3VrDoFs}Nq;4o{d5wV;fXNyRX zMq0ta!`W})gQ+a^z_--UTD|G|p`ygOV=Q+Rr_IUwfSM-YZkyEpwTnuBNepmsHdk z*sQ@f%!>p`_NDG^Sz|pAIn2+%g~|UG9_7?o8He9^v`>f1x<18docM}eOOgAXC~^Hn zjct?d496IHk{_?}FylPzGL^_TIrxYH_~d$XLn9<$t&bvYsq9*rsk%IW;y=;$tclE? zo+xS2{qym;V@d|cAMtT&R5j|LBJYCE#=6cs{2z9#H`X0JaxPf(Rp@>f zOp#IV93rOf8$W=qhd=12gx(8>$P%oB2qw)G*|QTQAOb4EVE=LG#j`t(F&t?-|M+TswW)!YU!+)o|Az&q|z1gt4 zC@B+dqguM_}56QYl|>cBa_{+fp5^Gz`$- z1{Sn*Fz-GoBt`mU0%wG9$i!jVKion~15@N+VlP(Nj6a^45t0uWXZ!ZiwR}ar1+~Yl zN!fXS#;mHk-T{6+>!gq!SywWw9jSk{8^mQy@)oztkPC!kxf?h{J;Msdjw+kMS#{r^ zAr_`58Z}A(x-Y9oH--O;a6Sn9mIwc{UnhiQ~Z( zQM7&EFH}?=UQR24Rj42%x4{`TT8bBagWwD*iN?IFn*WGx?k6Jm+D(_=o^`A5N!mW2)OmpYp{M=!U z+o2G=^|bN~NIiUaev=J&PV-WWH=OE5(3@^mJcKv~N#`K5&CP^hyx|_onzgI^TyP`z zrAD9CKlc=`n#pArbwSPgL@5ya$>p(j1Vvk4IT^K7Pa9T4` zkuP$#mltRxLxtnoa{fP_&cdtd|BL@3Dh(>q2qH>%_Xq_6DM3LRDUt3NFdCH<>Cr7M zAy@eY9oP?C5svLXb|lB!^F-SBKwlv|2=7-FUnM!_PJ4%$g73 zqqIbSRe8ujr@UmPi}%w6rclKEDK|_`Ghm|aS=PM4o8^=Fz?dk!n!p5Xix9mvjnq;p zPn!e7dx~|(BSsbY(*985_m#8%$OZT3M;hP!<2@Yc3bUJur8x^+ExNs9L;oWSU=FNi zJ5KlF50}lIy*BZwGyT=Ji(uAHnc7Z}yP&hNe%4?jS(UneUSe;XIs5GmMcO9`GsOez z`xKNDlNwBw{1`3oBeZ-^w&8BJ!uDOqg73E;r^?|@w}2dE#91+ziPRU(?bCLI_`)s3 zUm`Zg31gC*+j#nTH{EPb7Zhgub^ZOP55?q829v6Rxl|5CCnIRu=fJR!@Y-+1c&2sK7>^HMtY#01{I-3KCw&6F1S|uyxD-iS11}jj*`th)miivM%5!-40uOH}1 zk9e7A_$(!ov>j?K!t&1Ltf-yvPAA3>b^kA)Xu@=A?Ruun^yj8zx!8)*KNeiLMU_8v zS9yuy>Df9-#wllzCeGM#f)oP|c61mEb?)W!3hi^Cyjj zDm~7C92wH5|75gW<784yj}3hfZ+I&$cxo}6hxnUeVn;HF0U+?dX`^&duRDw|c- z`6;iz4%qfQ$%RKGKRxo`C4x=G0*9>nBing9>49ho4y7dKUAy0--V0o#iNynE-{75% zAC)d$tj%2JQ@m6!m85!y?SF2|C#DV7M z`V8;ycycYcAX{h1vEUB;C+lmvt)==rI|+Pt?~45onrWH^*Bd@h+&i--5B%pI9i9^x zeooaeuIyhkms=yBGC#<}D0Uz2+*>@AM%VC)Z}*8n`gxPKqmIUVM8S=%Or}1WCibNe z#o;cpv(QWh%iXV|kKKQ@=lx4Ru05@L8+5)Z&tN4#w6C0*S25Ge-NjKq!RJPzF<%r0 zsIFRSXN$|u)ee7r4Sv>EE1d zOGm8?IGc2g6*5hK?%bKtCi^#9`}|jB%s7Gg;hBrA*ZQ&}^0To?_E2Re6Q7R;_4d;P zFB92}A&Xh(K2+2$6KwtpSvTwtxfk3!?lXO#ds+M~V*uI?Ie3wBvsMp?b@vJ+AVzYM7vc>@nvQY@H zMia$DB&lcY<_AnDqmJXQQEOvG=g_Ksh#~lnb65IiD7}jUheR|H~ z^s>oNa(br8REi61*zcMN?t4?P(tTUFbP+*rc zfIrJKbja)DVF>vQ%QcWPYHUG@E{k@!C9siFGcvTv?JRikdrX3BX9c@@adhX-&%1|} z1vOu`;j-F=vv*Xi?-%sH2vRE@vH7lwd6$oR(EL0!Zf+&j8v?Q&c)Lw6WR;l`3#~zU zuNAPL?f;xVtgI56)IoD|ME|{0(Eq*tNiKPGY~<2Eq)q6i0*WghFrQ+_Gyj}X zoKYrbU(%ehZ_YXuJKCfJ?6(Q+KtkWFO}(^~t)4E^Wv(|3rzsr9#W(Myj-e7Xdotg* z%0E-za%m-e|b>EPj>HfL(Y=M&PZyB_S|K(Pkcmn?;B>U zvh|AB%G`wsKG)R9C>fQvoMSb=*;tG3RfGwCV0djwQ)vMu|LcfYcCDxwXR_7=*X!slv_ay%fXy5Y7352ykwAWqJ6$bp%O}dIR7>Bs zQ^^_DV+N9?hj$iU?oLv;D;?{V$bi!}%4?1SXMO5TIftL9G+H$tyXCovI{>gEDu^YF zsli*;@>N`C>>G-29^=rHCjd!^<~9dm@Ynf@IUqkoBPwTB65Vns=i{snxln#hE4HQ0 zKscW^b#XI>k(5{AHf)e)My${f%GP=!7I}cvS{#i}>MIxMl6&u&WE` z0TG9B&#W)PyLaAqx5z`fGe*fGVg}!S*~J#wA3^umcGkBm`Byj=uV*TT*X_=W<_JO zYa384)@I+ry^$l~O6x8S9u%rAl}&+K#es|8X7Y+o?Tb+>svMvwKjFZWh}M=%3G1wP zvj{J5b?_(!`Ig15!~~Aqhau*V{l$L_p&+HUe-b_wyjMQ@ep`sdA$SWkHSh=W;Vd4V z?qHnliH!G{;T@aRjhvylXIWr?d-t1Tcs8hB!3?LlYlbU^U`Gf*rwaa=74j+8OUY=_ zruVR!gFTvQ6(o8z7mVA-y6ueH&cYtPTgk@yRL^@>E*96>yST$XnBbs!X6j>A*sv4K zH9Bf5ytb1EZW5W+=uoR;oGo^?&}(Y4T^;_)gcb>9dA~5&wk8fkLv#NpEpV5Ce87iY z*_%ag|1fk2xMyqjgT+KF#4?uci-y5my}h}kVW8~L8lzICO1x(^e&VcR0|8>b?Nx>9 z=>b=^O6xCC)$t*e}04tr1+I6uh9%NU*(u5#;J;`6;!A_pq%|EiW zzMzg%Sg@?6H*hTNCxs2ClRrC@7RSdIq)OK8rD33x*|DOPFENh`g1pSZb84`32Lg}O zbeu;ktL+m<9C8Y)w4ch)5{$e;%9g_;-wI}Qd54jdWftqo#6Bcpy|8aXGf*>*A9-l! z2|SSEm!jNzU>MZ^$aFaIBOm+#1LX(Z`<(pbE*n=?AWK;duqk$&KL0}U^Ww47r-o4u zRg5SHBz$1o-eK#RIC&4{f8ZeP7DGp?XDiH0cKf*74rivy!#XOPYi?95fatNy=0sm} z8in{i?UEfL4#lQDD%`Vv+vN5}eRJiIGV6kXWbNzz?%Ge_j>bsO?^BKEvrizv5<|fC zpW%HT@2!j<_Z$s(p6ONyZXVjIbzi*az@aG4lI20S#=*`{+J;Vta|}R`jgGpzE9AR@ zkZ_77%NJ2B!m*s99VcCnt+E)r4A|IoG= z$0wut-I9&^52{FP+`4g>K0*{uY`4-xcN0S#@LJ7s4}zUR0g4=~He;!C0S!f%lW%_B zEXo1DvMc%_o2cVqU0X$o&ydWN@{^6Kc4yKls`VXR4!bdvZbZ@)F^P$u`rSX%da`I*Zs3XeC# zu3Z(^2ZU)}A))AEd(kE7IZVUh&kH-pceG zrmLp*H;5)Aqy6g(Oq!|C8nd=)M+II&m=@__YDTpQH~ zeSihz+3OC?BWNeJShlZxne?~${7o5VO@~tCR!7=|QEaFD=l{Wh^a&$M2L<%KdMj)4 z{ma1*{G?bajsB;Ty*G@d+c2G;wfq_>nDje|tims=>n`hMcl55m0>ZofRC*1WqqQ$( zHE|?4tzU{Gvsu|YyS+?LZgNMMLZw?*K7izI_6^LR>T9}=`?9MQxUaQ- z%{F>)bs9x&eDr$A1cE^8TNz+>v=`q?UwDD0W_l|q8V4kHrG(hIA*@Fdm2=^4T0{*~ zu}kR%!}Vss;mNlO9833ykwNH`uLe0IlOp21M1n*TX}vLYnG*v)KR{5gMGnQYi1+(! z$dF{U(?HUeczq|G;|oj(g;huzI*ppx>`LgJ4)0J4&~rz3kL*U@BGnpW2lThxX;82< z->h}!nKm;tcl3s%*-yqMjxISfYppdxYYV2l!Rmc_G0z_%bj%&xi_hF6(30r28*_4N z@(LTaX=^wM-0@_w>tqf62f1sp`)s z$L)SSfAS2rkWl`rD0EhFm2~}xZa2!#7kUlbt{TQ~7lz#rpHXi&r_7U|J zE^f2qfjDIl5jh2@1R!*7FbSxB|As8lZ?)TuqM9l*&}KpuPnFrPUVaW<@TS?8pDJep(9N)2%QQd|J@~K)i(R3KjfWiPgJTug?|pcR77$0jxLeG zhW9^Y$0RK$jRqZ@mI(nkpzr$v>V)jF8^BHR4#a|_sh^k$5g%K7J-P?R#v=G)IsS+U z=1=1Bx+_>r%HuwY-#|Q@*7VcdLT@v0S|7!8g9xN&bRG#A(nocyj9i%+bR7m)nx0*R#$diB;7uu zj^z+q<$LBr!eLryvYY<&LlXsEk()cHIGbVOjMrSQJ4FC@jDpZwrXrx}mJ^`Umvp@o zU`U(r6s%|w?PmsK|2s~l8~&6DJ*_Rc?C)QOj2`43 zMlEZGOKSs!au%iVZo$r~hq*GhBWKL|?=K<3rfp976NiKi{pVjaPO;uAkVE?f8CDc9 z0EssQp(~H2Q8|~k!loz~RaUXBOa-`a>`}JJ4`Q1q9(-fMcE`X7Q_tr2$4kUbv|HyQ z*T(wcTi-evrGiMF;1pjoGEhg^(laF+mJ3S(3Q=sXbO*LcB8-*KZcecKZFR#b1Yg5qeWOg9kB`ZxV{ ziw}06tH>+sKplZdNi*{>whc^W%_Z?`d@giGdhlmM_mkl(yuH8en8PDw_OrfF)jhiN zko(Wf2xs%MCcO39G!rvf7ZvOA>d87S+CyFa+_snFC7ZsTPWPbT^Vp>>Y~; z!m`XizkgV4f*s`h{`m0M&Go_{28VSfysa_D@b1&P)A#f0Sz4VkuT&S3Mv9(y)P%YS z2T4Y26-~{ z0zbsQBnIDHUu~0_)`(ucBteOTL^6Y_wU zSN)ZeYQLj`(#Q{~+^v4PjpibpmW{;+W91h5SJRTr4|w)8IN zL>AcKr{b_s>K5#Vav)6NOs^HgXY9Rmmk;>_&}XP5*{owIyr$neaLL*kz4;aXg*f`+pQR~RY1 z{d$-xiByzZIYG0Xk!D|l?<@C;_tpdR%ra*0JbWEsp*VDRwMBw9tlwHYSUVhzy6pZLeNTRVI^-T94gsXR}yK5fX8sX)&ibAr#)}F`t`f(6>o1 zg-t()Zhr6@WEfTQ-oF*TsP0KtF5u0aeLUr?COcImO!48WqU!d8jPM#AqnSuvz@03m zcX2R4^z-!(9&>Qk{8?4svBS?ZI|?{-zbU|CdU>z|{-A2Ui&t}TlQb#&>((zHn-zr_ z!kTZE4Q!G_f3v$z;RoZ})mk5aX5^l7p=w1mH|^2RMJ#Ev+!x?de5lJkycuezh)i%6 zOucIa^oHW{RV;@Xu^k_%qccWWJTzD=cYD)$F#Sb|R4mtq;%7>enoMKs6py14|9{a1 zSR1}D*2GRy6EAP*DgZ}}4YX=z`&Bd0n)K9xPpQw0JlrnO79OK2F2GvV*ahzl!w9eW ziuJk30)Dq_=^~SyCk)Ky#FN|sVhSpXWA?Oz@Dhg^{ZJI ze+MUDl3bkX`dg^D3`!`e1W|5Y2gTeY-Kpc3HF)s*jo1V7;~h<-1>t~uCnYy^O@AjL z#9S~FexG`H!k|mmNIYkaoMW$VBTq#gUAas_8&6rr%Q^OPzXT|vbvexU&++x0+-cjz`!N$M|{;}{(??Tsm=&|$A(Y64TRV>pwX_MUErW%*Q9f3Xd@ zd4e3PWB_CMTLPAS7C=L(yPR0vRoN2A%U{jhJ7mcvsCjn4Q<=cTST4s9MdwiH)h7R3 zFZo1M+_a2;2`6&%uy9F`3^XT5vXNmjIsn;@?1T4hkxwQE?#a_rIN4ceY}R(l=RsOzf9($cPKaX( zn-x@*db|HpFCZ(SDHKWoI>FDL8(*^aYS~|9PpE;weNHj9*Wtb@{4~2nw$=dyNkg6A zl`4yhCEihZ{)*e3Img?50qV4i?f(>vBnU=C@EDF&$x5wE%~}SDSxt&3@LjgIZXr<$ zN7;tGBcDZ=8GYSQJ>3$Frwt#SVa46%PR&C+*D6+JA*b z+UbjM9jMdQ$|%vOZjg7?MysLLWujJKq9Fv+@xN_!I2o_l1(@RCvh&}JRscfxH{BhV zs+0fLM}kG}T0_eB;P=kK%PZj{fjZitubAin|8m#5-_G5~Wc0Nf^pOzm#FJSYMlH%q zFOzF`G55Vs7}>C;Yf;FXRJ#8j8^J9An|k=5r|0_9R@W+WvZ| zxquFna1lI!c~;Vjz(kRR#_4NX;Z|U8^Ky?=m_2*HN5jJ#r=XE>+1155Hliu9))=3F z4;aqpOO7tHRdlu%uj63m^QHcLj(XRg7cK-vE*vpN)kZ(-fGpBLXDf|&AH~L5RlE_8 z2f?oIHQibY8AHE=&XZgDzQiSAbWav;dlGQkl_nrn_8^pF)w4x!2ran;6_Bp z7s#5HD4|HjI}h$*kkT#$DNP@gZ%-YT9+hG=`-es)tvqar+xlIYzo@gm{%U+Cqp2si zGQij6SfXL;Ujx3PN`(vd+dS@oIo+vK7>GH|XEf{AT)g3VF5Yf$F0ZB>sfyrkP!H zzs3LgF2Hn*T>=i*p}C?_j|)5u5>!07T5Mgw8+xHa;hDvd{>Js^tEV$;jC+5?n8vg$ z0`GNiY;fmf?pgd;$?(+s_zRw(UvUevy;80~_rEN@NAQXR^a}_1EFhczC3r1)} zNaiyQ1%@zkGd#>>!8+VT25UU4waJU(>F8Q*C5dCO;E@@N=yDD~8s&O+ji#!t07Q5K z+e>*YDV7uEb6#BV>nvb0S=-a>Ik*^5Wc98Y{aY_P6L}}+m9zKaefC{D{iTJsN_se{ zbC*GJs|;MAO4ZQ3CFT3#gweZMi5Gz?d)vy%4hu71wM$va%23YVXpAv2n@XQ=1S8b? zZ<>>31hz|V%3R!rU`tpnA zrNAijm#(*i{&%t5+wF6G*axGitc_($(Lblg$Dz#EIn?=UkK4@jaPvC>&Msnd^!2J; z;-5WQBC~6XB_&$^Jbl?!zCtM#-u1|-icr?cgk;X7YDoMzp1syz{i5~5ZT9@Qv<V&l(>>BR#vCDi+TvukwKtEJ6(vK*7+3mLMkGYSUzI zfd7Eg>EdRKFnm$CMj(p$M1Tk$emsg&i~MI3NbI>Xf20*^G7)2`&Q}WAjG zA_GkY94L(46eaQ;=qpfvFEOPaI8sS{d;;~oKO;hmvD6R}-)u*U_P2W#_)`s33MuM6 z5LUAZY#6hiLp>J@H<=yp1kq$!hrWfC*wb*}&UFYr~ zQmh6i7xX^^f*C zzQQQ}^5LIIN>AC$)5*}o_XcrgKu33dkMzXbN#@5l+-WES{kH~T5Ah&)FV2zsU(Q9M_&(buhS52oM93AxNY~(ZZ%yz_m zFGIv*zeZ|pWA2GAKc69A?GHh^2YZyX+$ax)v|n@_q0oU8BDas?Acvs@%IVRV2QT;| zsWw*zCCG;;KV(|GwL9m4>snA5OKZQM?p=uuQT?br=sNJwKYV?rYVgc%9$Z+Nk+m)4FxC zUnxK*K@{>ND7_^*#Z;d&i0t6`%1ND$@a3MJ-65pk_N9$;>w z(Os`TCch^5Z=`ut8t?)x)5TX_&tKngOkVE|I{psn=7&r&5)?p3C>vMImsfC}xmlC7 zp9{C2Tn2#`=Tqisjhbkzb;&OaTOYnCS>g79MXJFnY#D9JInGlkk-ftsj4GI3l9l*J zL;Ssc2NpeZ@P+k?{rEZ0!2 zEA8rH+DmYhcGSpbjAv4tox<*x9R_(m9bFJLKNk^56P(WSOgP$+V3VtIGgd(<us6~?(y&b2u%*HkiD+4{gU3N-E z+!rnhbUtrJI$0^g>U{8h7}8XQDet~-2P-IjRX7CUQkL?bIlmXyVGrPKmYU+oUkg_Q zqr-KOVN@@XHmwW4y;UzQ;JZSoVwRg9q{Y>1?1C^VjJXFYr{KzVx4_6)eKI{>hvufj z9iRf@;S&If6vy|UW$zr@)J~XTt6=dTV80e}S>vZ6-wwpzJx%+^?p|$?lFhd%nlW$P zuGquLiz05mF4O>0LVVX2;Q9R6Q^Xj7KPHOdOZ<%E@NpA52|3AHM>7}mzgVp)Fo7{G{86AN9pn0wl?MG@^ z);@F*!yjmQ9W8+FnPPDQtvSe?xhDLcnqQJV2xijn1=iTUyUDQpAti}*@BXoBHwgLA z`f_C{U+F>1ifi})G;ez?Y`{AyI7O*}d2S-rWHk2L=ufuzY!KLrx)-v`0DI!8XU1gS z-iJbNqra1m-ZrkxlK6y}CU4N69e5HrUxv<4P}JN{o12akambBDhMek@`1kO}Muov% ztpO+k(J*(0WrV0|j!dcl<{MNj=OgO0aJp!)UqC0cQuMIP zz%Ej)!TRcf8KwW?%!}Z?^GaT;wn}uSQ=In6!P&h{YkIx~6!^8OJHK8X=an_Q8eYBN z3y_cjD0{3&74QXLdUgGFnX6^kbnOLQ`%ghTM0?)q#aJ4$tP|xFpJse;B{|3(WcKEE z$9989v#BtUi#BZMUWvA0Sixj4DbwwpU8c`0@!!kv$d8YsdhSkaL-<3K@(*%YN*}37 z*uG)(ZQltU{aD<7y3+IQij%NGjsX64&=JYI`{8a z=jl=Rb7|cZ-=&Kr6l#G+hk0rktryMW!ta`|<}CdSAAssHVCWjjj~(Ls+wE5s)K`xs3>B6zTTt7G!||9fjc(FLpdKZ zxc3fAP)a%_;D02a%@N`~5fWE!TXPKm2EH~5*t@_N<~32uHE{<@GFi^IUul^l4JP3B z;ALlz0B+W$M5CO@r&E?g=_R_iQ~148+jBmzp#@qOg3Soed{dpO`$;_?)AX*NntQ50*Z2oO_nF=d7_N6u$M|GxuYHWd*=%um^?CszAc1(z*!erJbG z@0iTzR=I~xZ?p_KN&awFd}48BsNG79v5U{x|8!yKMkjnv$F3}j`y$ZT{p(!f4+i0k zHIPO4F4USDYAbg-b!3VSbdS4XunG96e7!3*K(xep$)l3i7OmBk@NtePsd8uu=Kks_ z;~xzd@iYZw^hX&=b3qN<^W@0S@dOK_t`r0Qz|S_h;!+Hm%IkJ&S50wKYRJ3Y9s`P3 zHgtE@YP|Sh*vwfgB_P%Nf{oENfOUu*>-`I^mrVK1{1meEDgq&-$i4|714L;lZo-$D zi)EjRgr{Dsr(^E~ezEGhfC2=*Hn?!B0a9N;@#!XMLl}_UE;sNM?d({dii9>d9(H#k z>uRsxr?ZbW!JT(La+8eRwPdm6_SrO}+56akurDa1^gN*5xYjS35}Re`88BDI%&3O_ z)Tl+fVa^QNLWo>W4%4plklXea+s~Qkn;vOPV`395nD%yXZ_68B3SWL}(}n57aMeTx zfxN~6sy2zjvhhdRTGb)5(sGG24(H~lmEo@-lNgtnx195qdp%YMppB{P; zC^~4ZyxmI~7T5eOA+05i#ECI5m$y8}T_#qDp~|ER!{5b;oce)`523rCNm9qwC;6Yi zR^w@{Xg`(S<@>6(rxiXk*&2Z8o72h^c^aUY1Wj3aZ{x3qUtqrw>yvMvaBDsgY| z0EII%&g9*!8@|I27>OB)3|1H$`S9R5q%V--?AYvovB&pbk=aFj30oGZ&5a-@hAgGn1PGO!r`bGrT$PGc$m*GjS3=LFI==pQ_W0Ord^8Jw7A5!$Ix?gIeQB`gll^XY%EU6OdJLnfZSNTAi&f)1@V?zMF0;H_L%vEqoQ55f31-v?P#OJhMPB-vl$u%z>PY z5s^*IaGN@pmYSfZG|mCLvtOD6OP=0fKIFP?=G$+UbA{IAHdvloQhd+bpx!r3C4#1Q z?fpt3!(^g=mPpVB0TmC0Pww{iT{TeVL#za=M#ZPM$HY}5eX>Pa+@q#%^5EE&`+d

o@jPH@(1b&c^^8k1h@OZSVg)e(idQ! zcq_TD@_`uXY1Mrz|GJZzm4A|n87^*`>uc+4GYFLJHLY&m9(35%-k@Ljrw{|NxX_?d zs%n4r{4xpBGworl^jN$>E3NZ4HhM5;718hrYp7%cb5-2Z*jk~g9LQg!coAp`i~VGZ z(S|MuPCc2@%F0LO(O>o(ns0r4dAK>>{v1`ixIpZRZl-nD4p4o~Kln?dh1^~2{0pxC zIDL+_6;tSPPybm}LRK~b$~^o>Hl{d3!9L_Snbh;r5M@Z+G{l5@YGtw#o17m(#Dg+(QLC`w33`~&kbQR5TCa^Jo6$y zusGH)T9htug4%}bN;XMJB81!O1M*ZHdF`a6((2Y#S!eb6zdDkQ`5ah?p(+2yu|N(KN4;V+ua65eG8hHO}Q&C+Bk#Eb1ky8 zv;V7`o-O4zTRRec&Bq1P;Xh|y`f|r$i~y&)g{QUG@SN4*gN}MiR3T2n3N)q>rn)*4X`NMXrIgEfFt_*^^Q1Dld{(xKy-}8Ki)xzWx z6}zg!v(|F+;YO)cShP*E@WhU5F_!J2gEoapodxmT^V~B?T)R$gxD}dq=>{LITUf<# zYnp9aBBg22U4LTOl7?2FKoUx;>1vP%4xy-*rxB zB}^yB0fO56J5}3%580>v?O(FqQTbD*gi-vgB`k)sfPYnMxfvv@kY_hvrns>d%jw}l zG6<}YAZ6*aip}EaEV)PZ=7P{M*|*A#_%Zgg)eI==1IukE<7pe>-z|f38^>maq7Kq) zW9P|;RhjNXx?UFV{xZeFu}VwC@}azw-TZ-)a^-<(!q?VXGFVozx6U>@_fuhlmo_W0 zBHNxv9$iRoA&@xk`w+$q0x~%YotZrYD1z^DhVRHUG@V;F*=whkS+FlB+2UtGlCrnz z=(vV-JM2QwC}pbFNfP|L?wekw7a$c%LG5^(Rh>rwi&;z57HXH~oL2x{TEV_RdMm=R zDs35cOg(z5T3rG9-`!r$fncfC%h1GGZnK{^G?0g8aXRRr_U?Lpl<8#|^E+k{i=%(Z zZ@>BIs^9Oc>FBe6F}U`@@4EA=`Yqi`@=%fY8}@Kl?ZA3AJ2RXwK-JsawE?#1gU+)t zT;fTY_G#W;s?E%)k7wikBEL}{qq4rwUfie4#qBgYwOpM3G?=53t<~o#_>Cdc{W>JP zQ{l;!Qzkr?_blm3lKTGiFCYz%=b^x(XIR5lRjWxjy*WpHx2pjwIlJg)+Ny19VJL;z zH_Jw6BGZW{Px!hw9zj>8&j_5`KWx0os~oM0V*RQHB51+-Ke{jABL9F~N^}5e`YM2a z>&O}ZE-%wW@LsJ!&g{HWrWp7EQ6IKHQRLG8tG|j6E_;BBvOh^c_;c~TLou)U>apso zY|hX#X~$m_iR?ZznSVa(0RD%Dvv1PP0G)zC`U6tyZ@GVw^*1$4YI>L+tyssFawBo= z4uK}-e!sxJR3e`gypx60;)6ksP)vB}xcFs{aqk1B2pKrLm5e+9EtZ(#Zc_-tdgd;h z(*EQgx*I?fz(OhI^qD!>Y2<69SgGpsfNHkwbLZs8Iv;j;7V9GBu>xE}ddCd@1F1EZ zQMT$n8$68v+|8#e%Z7{O!dgIn4#71!yYGQhT}w8cRz$P>#wIN(M{a2(d3bjf)3@owZH)?hk7hK{k5r9xbQ4 z@sOfo-?+uF$kU{zl%~#?-kyl})#}X@meXsB62*`OJS?R%_*#|&ON&hd5w_4o1N`qH z@m*)kEOCI+74bCtxE4#r3h|F*$$3miW@h*cqqlB2lg4QG7Q$a=KQZ!~Ra>q}KL%d1 zfNiTl1b!yalNpy|;A7a@eY~JI_rd@|TAhxP*;3r&SPkn!ecCXW%bK3wub=*Kw((Zy zMUl1!?Ts4ZT}B(0`<A@jiG!MlO}Zs_O+3rY7pb)+vc9c zL)$_&{}8`s%=fo)wQIg)hg)q`?fRq6OnB}(>p^m`t)`f)*BAAI|8j>dpOy!{!m2bo zD)6~+bHOAMAz3M#CTP_o4p+?BqlUAGz(fMnl(W`UaL9Ha#z)z$hRyenSay?-hHX+Q5X)$fH~N6W`sUH^D{ojN$&2FfekMEm6y4yxL1xWq+#n!HC?JY%Qcf1 z;~igxEI&OKpQ2-5?d_N(_+eWldz9EvC=64wH64Uwq|@}V=H78Hc-r6aZ+%plH@Gf2 zAh(uSwLa8aTpnlg`c&o8XgNz$$QBS@BBF!=I^VX9`R}0Dp#Xpn##C~LO+t|FdU+$N z={HBl$(0fn8t=dIvXt7+*Zm-3oPD>~0P4Pj-^4O$6QSZfBEwX#fi^i!Z_pQEBH+kV z74)6&H*TH|r7P3afzY~_7=4JeIzkc@l|O?iNkj)4{+U%=7+tWhvsq}U`+Z5U)Sj;Y zv{O>}X(2DKdnEk4if0A0+D5+}v}`^;jAMPqG_4!(TKRkjiaUi`%)|^-Xs7)u=d3cB zXDG|6b64XzNTgHVC608V>s7dlS2Y;lKd}pF5B|Z1_Tj_}cOA>Ni`L6FwEQA>HQrkU zfL_kVr`U&^cDkN9cgF`p-c+Dv$~pm<1sS}6bx^#3 z`Utp2ZfXwTOhaz6`;*V#*1&5=7~zx7^h_r%+~kj}mW|`i8Fo6(FrbTDB-2^;%?@Z@ zmJoce)l&_k4#K3Yw6+GLhxr4A(KeMGun%(>c7=ecnRDYCszBv)?>$ViLzIlkQK}mm zSDX@^EaiJGDg9dl@9DMsRXvlwh|7g5ViaaRZMOM+vbuB<^FM^Ba|l?^ z;_;U^_LkP|HYvU?2K-Xk#H@XJdhVgI=t4uDiCWWDXHy8;J)0cWwwh+nV-UDU>B&48 z@BnA3Y8H3$d%D8_ffgtr!#Kf((N~T6z2lYEUSLZZnqv{=x!Q9Y1;l?4z~8^>NZg!H zTAlRJzOstmab>ffpR{lzCZ_4otOmx;^HYQQmKGgXBw#loz54g#mdWXoiq428zfw4E z(-b?NvL(YLOK{I$BmJUF_zj}y@8TIS6o}*O=gz70bNo><-)*P7Yio2Zq{`#L)@3ef*564%J7sQl zaT2#hMJDV~hdm}_Xf02W{Isj9l(g?zOSGN;yP}b=q-`&4cQ>e$hfTb~$DKZ6?TNUk zbM6WSY5!o(B&?p)3Qd0?@pr^)!ir^zAt6V($-YiRF;--7c_}L?`A@B7qh^r&&datoO9mHPMWdo@t!BK?t4t{2fO_& zKgb-0L>)Xfi#&zZyt{@XU)FdR5xUWI%6vN_ef!MxT60myCZL!%lsVwO_`_K-AuGNV z$-1qE9G3x{ed~fP`J?Lp!_rweH1&XMSU?0MCL$n>f`ot|-Kdm^l%zD$(p|EFlt_1X zcXy2Ll8_oLHDDt~j2QR)?!EuR&e?aq?|q*qgnX`tfF46iJ5rOd+ubqK`I7Gl^sh>Z zx1x&CZqHt9N1(4kZk@1L{xEabKUlEz!L^-|+-xEP79oY2(2X>Cr*0lWq8*E;0i!R= z;1mP=N{RCb4wx=s#RS(Q_;&#L?XjlC*9o2xb?SU{xKM-uU!h}QYLGvvA7ff4W&LRM zs{Ne)9}~Oj9_e$c4rH?zCgFsn!Mb}IIo1AKB=TD6ogQEammG%TrE04c-P$ovGfON$ zKhM@AEJSlk{pLdLc(Kp#e)|-)Hoon^1~q$FW4!#860#e2aBbY5Aqt!NB%OOqjFAg~ zs&yxi1hd%xJ$LW#$c8m+M7H)d6Bj#VV%Tey!_1SC*SKb-$5Yk-%(KWm1D{X?QXL1kl@KXM7C@lZQw;--F> zs{Q5!a=@E{w;paWPfuZKjrQPK+rmV7f$kAsrzF_6s9pMfa_}}#HK>?mto`>A&x}?A z3K5vH>B^p(pY~k>$u31ln{FOhg$wxPqLiD@wHKi{2VL%OKMSlby2$jOdTDM zikR(12ggDv?6>7}#!gdURZ;K~b=q*yw? z3x)vulGy5tCt6F2e=on2*E0!?zB%Ip%52s|(CR`8rf2q#`LbnKB^zruW+@4fxTYl( zoS{)CGJ*qp>;H)Vq3m=D2!nw6bhW-InaTOq_Tml^@ZF}06T!Xorjf#l(MV?HZ z4PsuAIV^WbMW=#(EAODl^9iaIdM|)Q=omHXm zC?4{3Ql1?)R&}^?L~j@>&n5&C1G1|yxx(*YiLacBVGyWzWuWRC&E{lZHK zXhi8ye=$Y(7c&i}e6Kek819CdFLC=mIZZoQI8U>&YV@K|z8mY*?M_;G2G#T=`6v?T z?NK`Wt6uS0emfXbI5tsWMNoi*0i?Lb*@$;W)q9$k@G(;{&Xyu65VDs9SO*f=647R- zA%6n_uBG+&Q<45k>NN9p)q7dy@$cLjZO5v+M8LZ3E0u^1Ix1@!S+I$~Z~Dx1SmLl~ zeY4k-X74@ezJ!5Dc`Pd{ZL^{|>2i7Z#81?zBB$FAha~cbsq6{(f)YS`+rJF90mF2F zGI~z>AoQMH-Q7xSP#d$yk@J;jYUlYoiQG4PCPPfep6(z`XX(d6a@v$GMx@ElZ1}L| z*M%&d$ZqPE?}@<==~a@C29m+e-1*4t+O+tbk(QB;`5*$#5dEw57|{8>Rm}39Sq5fyAuz-Vu6?g`n2AN%y+_=h^4-955)M zB#@rI4^_$MvhY7brzPj5%ttYmiy4s{wM5$KS2yRbzS$Eh6|RIgijVj7*9p5YQ$cUf z|8sPXMXesIM*(-+z5x1x{m=K4Yj3*K-M8(0N;YLy)rlWe}PJb?@RH~wrs|A1e z9L)(p@WcY4&Z`^VGz*0(LoohUb;0hm%l)`+ zjrYtZ-Ed~HWuJ0C@9cVNBN6j>YwBl8=Mv*p*U8W_t{-0P?2DCX0$aGM*G&>Nt)^w& z8^ud+gXudMxQEm=Om8=z_8rCV;2~4yGjvxF2p3@K;Yb%DoA}p4E5#|l zC7HL7fzI4|0R2dU5jLUm65`d|&p6K1xa>RAH`o#(0jk zoy1Xk;DaC;7gsuuyHiE0h>xo{%Ja&o?}IG~HJOCnbjr)H>JfoZpS@$)v%L+`s>qPo zATjy;nOD7z;zTx?bM8)rbBFszV<85JM!4d{&exTU`gd0*YS-+iIM=qEN28Qmdqs25 zYqNWLfI+GJ*4~GvAnUL8z^w`4_Dq9fb3r0@(i3)3kX^fO6}9}cZWFNuy1RBpUYE2M z8jtyf9(mo`A`&5f`Cu0)RsDm{kX*n#g7(1t6RU9?ZT976bAc;E@-~;H8tkgCU{V0KAUA65uZb#vUMFG016aCv)*F)7dCn-ua zc!f9iaf6O&_K4`at$}`|wj|x@Br0xU)H+!+QaJ%x3B&Wb^Gq%cP--k~rSwR959&_2 zLElMTW~0o*&7&|ruXc38Lr^uTO@Fw*_r9byJg~`)`cPgK+j7%gcDHV3xEapR6vJb$ zyW>AJGp04dR4%Q4p{Zqkd0o9(Nz&r;w@GiM9|2MHblQPLL}fw^~1}a`yh-k4@K1FIuo$5s5 z5^3#w%kgr2d{8r@t_>I{Lhhd%fi8%OPPEQ(oZ)G2>tNz+)SQC6u;4r>V9c4cZVrJUxKs8-{5n~b^iCV7fQ^Uc{kq25)6eDC zVdQd=nv$}Z71f=8RR_lZO-}TGl?Ycuj46_#9Fz3Ej}*ZNREO$+nK|xxjYd1Y>;*?X z(AxKzv#_#*S8ZcG&znYD^h1ORZ=R$*W7!|}p`19@s#0k5_9;Gi9Jjae$qcw-4)z%> zIE$Ulp3a@!#K$58UHe{1$cT>V2o>S1UI-|tilm@FB?hmGbEEHI@tlXf%_rnr3AWLH zqS&vZzTUD)74e#PQ;)f1dk7JqPPZLfgI-#65}?JNKLKw@0V&YG`^iKPB(hI45-)$s zePA6VQGG4=@Xw2gfkIT!_xOq$Y&5|iTWeT`Gs9ERV}#f9sr~8OVD+4vioN;md2Pg} zq5^qcXLawKlP6h*cv!!cG%?Dcd_luDs1;X7T<~ZE z?zUFk=w!vf0Tfl4q~4*ay&}ydEBCg&6R6hIvojj&)8>?y%0^Ms{D1}dX&IdKKRhM?UNJox)0cIa7IsYPq7+H9*`LdT;0KF zow&sHpWYtOcGm0y$bt4h1>i*R72mzxNq>~^h4~JfR2fc2nJE8^vXs$IZb3ql;{%hz zF0T!8Ve6#D*2_&h$Oojbc+?%(>y4BDO7*Kc6&VgLzSj7jm*0wNj<<4Wqj6>1~kZB;)mm3OeoPOB@~p zS7-M*FNp`;G5m1F(m5+#q#@6O+@B>#s`?(KmX=;W&b4QnJtI?IR@jl-OX(XRH!b90+-Xxc7?NJLc7$l!Tii)KR=kw{ zn(V6u?&q|D`*|K0MTWPMf=jub=e?dTicx|spTB4n&Df~E!Qr|i#9s-RPTIG-^vM|y zxTBW?t?Xy*)09mPW2{0^zYPa(xXw++2cvs0-O9h@oye-dg3SCkv3?5m9bu>nS-?LSn|R-oGNZj2RYA{ zr2Gs~%7^xyxbznQ<$fKvinR1{KHKrdcfRjJe+{eUJwXAG4<-cK7BjI$el-a>Jc=jH7n_afftImwT5WpjeSw* zlg15?2g*CN+@mL-?P1b4qt3vWNY%|{*Ze#n_uhS&UlOsEj10PtD7s*XF-l}eFxqGj zWjy1YjNvQ4P072d<2YUdP_oyl|Jd8J8Zn|{9*#l`bzV#U>>n-%%i1#3_BC7W?s7sX zi}v$r8H+-bst7ujY+`}$ZM6n{xBC|1e_ArimLgr!Utu5m-#?os-H-+NG)%pnb~dr< z^_(ltUM}g0?@#~M?4&~eOdGtoCHU5DFM~#`4+)UOnwclcG<^%B^!pVh`mi16oTj+A ztt2l~S2Mv^I|~Vv2di8u7^P+$hmG>LFDDT82LA+66h|zy-DFnBp{G8wVs`hNm-jFE zane=f7Of9}vjIH^Gy(b*te34c%*7USXIRLFQ6g5jcT~KT-`FNENHV8$pW$1b-XY&9 z#9+ax^UVc)44SOF3ZE|(BGWCMuNS!?jgm?#3>bWrYLKUU*1z$7APOslK7Z~{qkhU{ zKT#WLLzEw}FL(6cQsS84Kj``#r4)*sVQQ3!F4vi2duOS?m*hg!z1XHDS?e$s(aI8O z{1WzM^arT)E|p5nC$U4uMA=hx^7{$VR&b%um814^9Jk-#v*T*>S@!$uQKtW_trbh9 zJDOt$_TCGz+uGZ&U4Hir-7Cnnx}Sw+J-MKIMFwR(&r=ePM_8=Tj@&jQyy}^YW=(sUIRz{oJSa>*YUgB1x~?PA!~n-d@Xb)|-Re-;M&%EY1*OSmR32@xmdJ?2SZ~#c zHj00 z1}d5>38K3#qCcz!B%ff)&wo;l-qXu5Q)x?Ym-HRx@%O?;F2dQ9f5)fO0x)YzSx(5O3nWpTI$wJPh zeum)0F#HQJm)%ymW}ocCUxg)p-oXD^O+tm~p)rWNmD;h1jT{l0R2*G{&(A1rD2_n} z_Gu3vJ??F6aZ_lL+)Y!%sI*cH^MKqOC&R?n?eybZi&yF4Cp%v+P8~K#5F8Udp#9<> zM?emWCA>M+<%R?`B3Ejakbd}jv)#atS=oLi3k7u5VPHcG5`FqK45|s zd&v#V{XE?BmrDOdWaKcvwY0mfmT5A3n>b~PrX$`r_W<#ht^ z5W0W6z^*tCNU7+fqw2=+KBeBwfmJ`ZP>!Cl$121Syq+}to3YxNmsxE}<09Fvk?gE3 zEM~M5UDy=1pwWGljJI|-(e}3cxC!ES{Q&-qY)+oOcrV!ELN#a{XVw(O1=2JW@N{9e zkC&5}zwg3DMby7t`kQBTd+fx&2n!rcJU7C0Sgs^xrr%o9OZ*}@>U)hjT04<=3MQpTtkv|jUIl1R~ zX`{W=*qV1u7X7wizBii4Xbh}>cPUU?OifspwXnQcu0|v1II2W7Q8!M{;D!J+PL)erd?qKh6Nd>}`n%1@c6$#v833t4|f;!x70k;mY)wSZW z3K*3&J79t^rN{EkyDi0ZzuC7Jw^<2g zBkJcp2P#dY?WTVRS=-tm9%3#=PUQC;8b3HzpS(Hcgf4u!Kydrq)P{w$4QP6LTzX2! z{bjp6)uvPRwK_cSKTxfnYp8h2LL9gpsWkZn3>-uB9>V`s8h#TmDSW)PJiuc(L3psb zW-j(wIBSMCa~cs6uupZoWr%Z8Gj~s1me~u#S+S{J@4vG#vie8w(WIKNwaiugM`g=6 zGL%}#0FRP}oupOm#C_s6uzI<{3Gh_>ge{{P1W^a`NLP|axE&!=fXwZ*@=hNjYhEAQ;-XIxwXGN24cf6>8{6yQ%ewCflX_nz%<{-_^ zxTMUgK*G&|Zm*IR2TKP138+PSI)k%CRx`Vj%KTBK7;sVAmIeG1mYIG+9W49%@&@`E z^O07R8SJEZrD&h5hD{QxNuqzIlQ-nr+l}P1siErBQ8Xuwl?s(S<=_URsYF<7X}C zbDh-}TN2UnpIEo4U&9p8Qshb=6Du{70V4Ya`sap;{Gz2;+>^d)~dA+pKEmC zR*l7i`6AAB+jbv4;yPS*qym{X87`_@A;ARDM@zr$-~Cy#F7uZ@Iwws(s@J>*@?5y1 zlQF$%ELpwPEl2qYgK;T6qz|p#AxT`nu-E+ATJr+66HRo~3>yjMjN)m?d63t|jiDy} zpD~-kLlSqQSGvDd_#bk$pK>SCuiPD+RFgHspAim;YfzDCtoU_W_l+N08QF4XL#GSr zBK!fuc-~gd<_n`)3scjHWaIB~6W>3>61Zb8#$D7awKr7u&+VKH`?J6%2n=XlS5E?k zW$*r9W|Xr=vDJxoy6Jd>RRV@}$0`Jye&V{8NO4aq`@cP*pO#Bp#jT3x@+F zkfPUjbFA`Zl@=WZrKOplLY6V^zYmA`rkkJQEgn_JCAbU?_)n#&uYf2!L^VC?uVH1rR|(d>KLyNOq{YA$E``5|ZT3VkZU zTJrND$ftrB-F^CK<;P6Nj2Z7mL|^+j=7;*J1S5k0d#MADT8sQ{e&|Pe{=c~G?~(Om z9@URxLX8~Bn*dsHPSMuVO469?OVF3;T6VNR1xfhi7q!3tL_8~-2>s|kI!hNqCb8Rb zd=WxvmKtKG7xktCzJ&2>mgUZRfd#g~)^$Z%Fw5dIyoIjpG^4e#uCAPm{k9w=@Q!lc z{Erh~1R;zc{MY3AE2)=Em}^kEKa02DF%h8r4-8v>{cdf;W68ZXJ+Rt&9zA{clk9R4 zEj!Fp%$cc><}+j}WPUD~Wt{XiBqa%JnxCf zIgrY@f5>IrO?8?eIb9T`nIX}Yp4*vQ(dkW>c~bAoT-A^jK+g&A-Uo?G9XPlOyxYwt zn6zfU*~H+Q)hrt|TaISwl*{j0Bv0=c65b$$Hra&|0yf>-)Bxa8-QG6x+&FoEMt@Y? zbe7MOjgaZZVT4YH(||ADWp=n}l;7Lvt8XMWrlk(_xBuynuQQ6?ZTMuJvxT}Rf1v$m zL;VKbku9ijeqF=tHoCZCVKM;|b0EGHJk>QE1Jl?eR*y1Hp#yVXWIpjCqC#^eB%TJU zKFIQ_s9{NXJStpGpkf#_&o4oInF%1_#R+vKIv>Gp@Q>cLHigVmeTyT~&OK}N6Kir5 zYU27B>^3yafB!|nj-tG5;1bpxg-@cP&jj(<@kzy}x5J*_# zdo%Y=erRN`&21n~d_*{5iG+H|>rZ>_Xu_2#|j1QfAVk*Juq%c3jW3yzRnv$Z7DK zPp~6`!+;Mbe?1-Wi>`|HU3Gt4kN0{z$-^f3r({&Z*!-MLs&rG$wz?d=8Z^;0xc7yR|OqTSAKRbl;As4MpLqR_vW|sWS#PZ(wip!dJa5$EPlhHLVfvVO;m}=2~~5U zxqT~-wB787-|Tn3O1BmYrnJBo|7kUG<4$WQvMi*_2K02vE;@I@KP;$hX84Qdi}sI0 zuT{Ar=S}}13^z}dw5dH3FtwbHg|zB|tc1(Ain=LMEry|7Z@O2~`ygBqX_W;8@&#c; zbQ4^+MD+$sb`H>Z2XSM1-}OA9+xRMaI1hr1JDWlKx~{5SlP&@9YM6C!-J^US3n+*6 zx-ldz18%p_!bcnJNl_dcgCtAv?e{9h?oVf5vV|3*=VEka=3bU9+L#Eec%X2XqL^-w z4DM^kU)*P8L2+s3&q6|+2B5vul;+h@yeVI{(|3CHXYR&277pS^t3d)mAtz^|dwHH> z6cw15&d+ySd6(Jk>_~@dtQC*zoo1g9Y75I8`5#~9 zO_zKArcz$=r;$%S2{U~u%1v$x2e`}K_O^GZUCBGnOL}i`-fsT~@wUmJ0lm!a*^k0B zOqA&>S@XWZ%Pzfv4qe4b#*V$P37TP}feU2=y@%f20vg~9RWw7IeyzS)rSnT$xMNYf znZq_ELX`Dxw8{X~pIm7|l6x#Y`l7ps+bu1~$CoO!QRdDvbEHm5Vx}EWyvn2^uyH$W ze)Xp8@>}ExU)o%2@+H;`Mw)tS)#vec*y#ty%QD&?(h*pj1gvpruqHHM4Fh}|XsyN~ zSvFW!Zrx+I&&?#yZ{y#l5ysq@fGzb~kpIOb1|1l_@N5xv0&kN_1?;)W(x$ITxR+uD zGCQtoFSg`J7`Y{EIyhU=TCdv=UXVglw5L)Tja}Y&a@oY9fy!Fw1Y$RcO9(*cDs?us zuYQ)U)fe^H8X@L14*xeTaRe+kZjD|U?mtSoa`L$K6D~7{g;S$sTijMk&N5=`;d;vB){+NUZ#Bv$*WT5F;8JW0iJ?bJM+onN(^rOm0VQG&ZzpXNn>t6#9 za7|yJ=j>Ormr~?y0=}fUQ~uWLo~|1M4n3}XxU>FWIr%n6I62|NLxD$g1dw zOBL8^g3K959Da>d9>h{eqWX@8*Pn@TDRS81Jjbg_NJ^@kS@vCo7*B)#^$yX4M?TA2 zhYVyUu{F^-QqLl;*6zRAYoNF6>WF-?e0z+*XWUy6kD%6V0PXZ1Il1r_Q(@=P2`MPZ zKJ_%PT_?+erJtWfuXXLp>b+=S~8hNf_LznVZpdw za*G;Ps&)*^j$V%Q@2Bcim(EBaF3&obmm)^X8#Z_}>WbLYCMDVK=2{OUa|vV z-N@wH)Li4;|5jVniZWF(?hA=EV6=IZ0MSz>8U9hw{=w2~VuTL9W}lEYPm_XY8qhD} zaW8*lb$M`sSY7OupsXs`Laf}EAe_@$)_Uf=Bs!*TOWzupJ6x-($OtN4fM1qe7m13P zYOOX zZNb!d$E4|=AWRmDkAH5Wi%rpDl<23)%^aEJWn;*F;}gN0mQL@S5)-uzxd_ z{tVf*Tnm((-mE>|uDjgLNS$o+-9cc)JVJxKUQ2ZThMcC4^@iT|)$Y3WISiQkBjhFH zptC1HB*Xc)b;Am6S*6e>okia{7j3O<=4Ve#uudn#7%Oe}8~2R3CI2)=Xyu%ssgiT} zeHGxrGJF>&A3G1z5z#jXNij36 zSWE3LzW<}sQUY@#MAjz_Np2Y2`DGteSErq=iU+Vz*2@c;SvgBXjxX0n6}AJi2J%$Y z0F90z3;*mZ_*)D3!t$fjpp$4DRMp!nN~s^clK|_~(^0k!=Wyh&QVs@$BvPA*uOfGf z7juuF3!{5AepgR2u?HB(?DjwLru!{-sES2VAnnqmb$@1|IM#o8w(YYs(p>ooBnKFNEX93*XMql)EGD zE%fWT^j`I9R?tsZkN%V1g-;%v*fViw`l~pN(3is|^X=CMN8v%UWH-OtaOa~KEumEI zkQ4*t5cYZasm%t{YJZX`tP)bUg^Bms$1ThJXy$+@b+K;WBR_U9*(2M^{D#tQf5pLb zpCfhVH6@W9RRB4<%x*<$E=BHT?HTQTYftMQzNrd@1nz&^eW>)99>l2U)5+?wEs_K8 zkJqYlr(p|Db126r@u^rh`!MdY=|D(fHg89_^K70$3gpS?x~Ykrg-K1uUcEe7+%Fpf~fOHGllj z3tZ!W62JQzhG%6vr!yqc%Vb(cLH)JMGCkPOGQbbAn?L^W=YZJ!3%ehEZ37C{l!SyH zmo%(l8oGV4zEt-8FIo|luwQHwiWs`n*_2%8uZc>DCZN0sUUK{Hn7$x3bV0g5I=7Sg zL)%Zk3z_B3Rm>kt!CSGX5_KCkD%g{@Rz~8A>t5>uD6pI`ih)Cie*8&`IU4>?d_F_{ z@>!&D?{Yg!i}X&}F64Z72OQA`h%|uzh~C?yM~1{%`+X|1H^_wijx9UGxQA*&LcNu% zi5;P;Zys+gu`9IsEfbBbF@AK}z7!DLaP(ftV0bJvzy=eoAsvwADWHe4rvFkZ$>_uyP>H!ePBO z41RoL3`n^_OlG-3mKr~9aP*O+j@ley7oTS|H8g~|8g6P0c*Qd z)@_a)`7WVT`n(>V``&H-0QGxLnFXv~=qN*7D`Zw_G?IN%-~Nc5Z2uRCPn7OM(b^{) zwS4ok)n@%!ZiXxE{h{o`8QDQxu&X~5N_yLH^Jnr)Cge<|iRQ^W$nk%7 zQg^tdXv>RCl^@^RLQ&()ts7?ZSEz%x4%l(mUC%#C_A5Cn=eqI9iseM`2ufl{4c>Dp z+l~zfx%;kI;&E84pEG_olq&B;U2C|7<894$qx6E}GV;wxQq!@K3emPLZSKU|l{|Ea zB9}=|yL8VH;apcS3wu9rnH2cIw~Oj+?#>op^M8g`h6PGOKX=^1yRV#0s*tOWaeXwe+aT!Gy|`ho#HZXX&kO*i!0%H z%!bU|J+F^g^#!8`SMtR;c};fe;e~oF{*Btsp^)@h)GhT+rG&Ws=nu^W8*f!7woW?D z39Z#qnha8l4Lr^HT*QlJAQ0K#yFdo2A7F2d`jgz>K0=9)!1Cq1tPd}04JncB1+9kF zQ{mxWXO361hg`Djff8~hIc~U8fO}7pz4IEn2wwt}VDy?SA*%SI)LS**UPt1qUG_2Y z^J!kGB!iuVj<)e)$0%nCp>%ZH%3a$9kG

nR6RRL0cNMMe+GxqrF@j z*8H3RyLqiDm>_{Wg~5D4lrRZjZ(ut;!H8$Ht;cgwubJLR)jFk`?ZYluM$z-B|8Yn6hnCVCiOm^a!)@UqR<*Jj7Pa41|39%*?2I}U-;OTw zyZ`v*e;FFrz1p2DXl{(lKjQs)2=j|77Tw43S*OdV&?pApHU!3As$g2FBhbgR^XYAg zCzDaMlV@-|Q2k7Uwj9aqQaOy=vfvg7^%x^|34r#2SXjbRMQJyFaonHHb*AhyCdo&_ zT@@W(9Ts{I<_fkHPoAMR46^~6;V?{>7az-rDpc@h&ELBOsKmK#p-wxpu~Md(h^u~JtOA=_6k@BB7pFcCdH4-vhW~QiSY2-5 zBPo6ep8rdp(<&*ODu*njVcTVA=7evSi{iY8!9h71?poK@@zZ zE2^N$U=p%Z6Gn6L$Kyz2laYf><DVCJM-!H_3>+Yri~a>tRET^3v7zl)~02w!fUpe}n`U7#SbuM9JK(1jjY5 zc^*1XqZb0A&WYz*PyUVyY+#~)=t|_jbmc2OcViP1fSwX;IDNWRCq0+JRg=drTV4-D z>2Zpw9rf2-r7N9$_sTe<6z7Tc% zP5PUu^h8AlIp}IGbK!t)Gt9?G8IRWq?&{gh_N3wqMN3M=MG1SVpUyn>t$DSp1Lr`A z*rM6PxcuB@ATzyEQ|-zw4{ovDo;l~Oe%P%4UVcKdU4bTR=WB1=&^Ie~(tmW2$x#rg z@Ym+J4tV#cTR_3^S$U6VEw{!6({3c^d<|MrfY zPX%*3v?(Hbs1Hok^m~I+I!PG@^9`n{2yBvGsFNO%@qV=v4l|1FM2G6PyUCOvR}jxq zK0z+2CU%ZeQd-Z8&D#Vt)Y!#3t}>JHdO6X7;amrmbqyi1%|YYB>`TdJGes_M)RGQl zNOn@kZafR8p7-TEr+X^7y$e{6l=iXiDBpPfwM-_3Ap%E_v`pWde2C)9=8{QynX{wi z+nhe9T|4hp-9)w3@Bf>ao&MU0pRQGhI4LWD>PS+n7Z0O1+;+O;=5S(gS{-`ebCo&Y zhG1-T4oSd#bf<5?gAFuDgfgYSeSh@Et3LE8XDNoc;fQadIWGHxR!Lj@UA=RIPm)&! zEW@Tf{!>_#se48Y(?C;^;S8iz)$~9S3E58wuMmei-fgzzMO4%z#!X5{TU|-BT-KaE z0o!k0hyS^wLYO>`63%@pX&O`{zUAn-c`zf9;}cls03f3pOh z89Zt6+PMT7RMRb~HkB>{u-6&3ab~SAQesGw`h&m8`9|@VpFGIs&)YV zIP=E!2<)7&Ch2 z-bh<4$8{>2BJoxkEbe-*N5fqVH&2X?#~wLtbq?$IG@O(Z?;un}HmUJixNqE~CvMWnt@%gfvG$b7mo+@Yb%7-2ID)I0=xe?>v7_y`Q!m-Jig(ogyedi1kZu2} zz?DV3-%ZgLbrzq6)t(e)%~E9^*W$cJ>S~utwfJrp>iF_Nfw_4a0wnxL+r;|8)uB3t?%MTI1Z@n8@P~h;s$e?K4N_ z!$_c*pkKOgI>c2AymjooRw$z2YP4<+=N%N#LMkEws>_@-7)}kzl?U{RlwNRJ!@Dk8 zzH7}i==G1Y=Tkx{P;4U{vrgKx`NOrIo?sX@=ZHg?th2W7(<}j*X67t+33FnbqfgCT zKCvoem&|Pvt9fH9!6B~nd+^&&n2~epOR=cUD4nx51o!+fl&>P+*J+3lw#7oqdu7?(t5a-mq+C zY9=Nmt01DL_fTXYj5SyLauR$(ctVg9#`Nnad+D;yZTy)}<3GxGn_FA7P-Y}u=F;~CJ&X#_6U9XC0D~)TkZ6{V)YQ#fSG~KHvzJI-2o`)}OX>tRweYCAc*pPWaDyy$f zL?}ub{uR>)$l74A9G3hA8eP+pgEr=UTc3IE_0Svd0@Hj9pxN5!~-C5IWJy3sYJa7{9^*kd zC!p6YR$IO)C(>u4^gMr650pgHZQQ8i-N4aYHlJ@v=N+LtY;P{v?Mq)VeG-q629JMD zfnH-b3~0bFX7Mmr!>vE?&+)*RyboxbOrd}Vg@)y?X-XfVjQf{n*U>IBkCalFLwN4! zh50dOxM70wX|kWV)lePdf+d)|&idosmy0j;Y4VvGo4I8SxfpstCqfqyaO%eXrP<7m z6O$*P9k35w8l+pS_*+hITQ~(Cp1{tDi#+yZ?!rgriz6LlGKGMXiQV+OXh_`E{F4j9 zPh!C(1PVHyoFtinn?BF$e{*J;EDBYgvs?&Hf5oE zTT2MhBHw*GO^sW1vLb&=JxUlc?~fS6X(~(DL)1kv3)RHv1y#->xJczX?vDcZ;&Pip z!JNs&15BwR>^|b=#-frW>$`78gG44(WDwLpxL5GQZTqx((!E0Gqlh5MH^Cf(V;Zhw z12b3$R2@=A@yCDC6M8$Jv(Tlq9QW{h9Kb%(Z&Yx|>KOn1Z>^sm8Z&*+b%c@Op5*+3_VKE6 zifGz!_?ELvrvJp}XT@!$xk~N1?6%P*vu2_CzToR^hHk^oeWa6&JMrOJ1oh5m&pw4O z^p8C}uE-=1TmyC@jJ-)G-2cGy+H8i;BL6+Q>``tSrS2RFdg!5T%n!8NR8JME1Rqkp zQ(o!XD0;}>HAj|q9fhvIVt;by0q=-qY;6?1rU;uh!i)Oz?x~YueJ+DO$6dHf&3mUy zi?%iCu3To&XWLxW&Qvp#E*O3W_S%DQWb($m(yP@h3H6GpSxf81To*hICJr+bdL{|v zLhREjRM|RpR+SILG)A zkL$0Xa}8dV_dm@N36ji6Tby4wEER_>A)Zre2EvqPhKw4_5 zOJ^uQy5{6rG>K_0{ghc$3jR0fa8d}ql8i_9a$oU^F!$2(I?pa!X7ivwCx+q|P1D|= zX@>p4BE|z#Ep1%2I154&lO`d)HudT6tNW7Qbsqkc$oC-@b#+5}APw4O2i7pokFApO z<=0GCAN?>KO#Wfz1D}R_QO>(>+6|uHIQBW%x!90s%^YFm@{V!CQhkU4%wWmD$2)mk z&{e#ITd4T9Kf{kR#kKT~pA>siudftX|9!*JQF5AM_DR#EskHO%g0oKKdEPkx%ep-I ziLGSS5o+*qtHk&5M&x9Bhr7gyI)u~VB95;rF$%?0vI|XVJGPqfj*y@CU;*vU&fU0b z!Ai4gmNc0pZ+_l)ZA*=}L=sD6<8Xzm+w!MlXIh@?mx)*|{If9phI6u=WLmnMMfLer zo5rgiyMG_dg4At2MUz?YE7oXFD=^U#ANbVj4ktpx^$W%l`NA@kE7Ed$>f?v`vRH-vm6iW+yGMmDmGRFl;sb5M<$%quMTtEAiEe98m# zM~AiBjlX`Bk6Q+gR%0n;DR}=tGMv0Hpq;LGH;?s~;JKa`@aR*~{WcLtDP2l$90Zgf z;EVfQb*FP(J!kVsqGq+bE!WQy+JrV|(MUygfiqxyZ)W{x+R0ln;WF&p+>>8heg-wg zz(1VF>L2h4-@6#uU*0s7rK8y#lKfpSgxRqU3TnWo3pAS%Z?Jv0<{ZFo<%?63kwGLD z$1JkvdX7jB#L>ZH)X6Y!`Enpvc;)DqeS3NCrV$E&&oG6|&rIQ|gnH_-63c}fGU8-ff-Pk#2v&>agcifMoo*n}j1@?11DW{;hbJor4rs$!QmD{JyC@g6xUuGOMjFdJ{Fo+F~-jx_-$V2^MN5$Z&YE{V@ObRM{yo@ z@p;oO_W9K@*Uh+6>YH==iZ!0X3wYS(02}|&;0M`w$fr+Jmsjz?&*A|Oq5Ovp=>po1 z=RZPkgV{#C-G)196lWh^dsq~@&htsg-$Pg8ov{RM(xJAStu@?}NLp#nV((MxFI6|= z+%$c`b19c<*M>X__cTvn-X;@ol&N7QzgoZE7MxpCiT2VVr%U4KnxqPB6k{LuQ8x0W z_rEl3*{Pd&T(1Su*?+m);bLE0cU1c%f0&@;@GEX8QEyX5TENTJc~M`<_3%P>yUsWc zM!4S&&2-tWRlJd$aUQC0-7vnz3w`xlyvX0eN6a-d=KJo$<+@SY!I?kT_|;z+p-6a9&bD>f@|HAYMl;^c7x~D&v`6l;Nvf;x(rd3MOpwP88nMhd;xFYul4t zdHYW+T^x*JCXUAk`vuJ?cp-*e-;sxVYQDzwT}#;8JE zzi)-LY^J<2zb*7@8g#Oc$+k9dw0XxuKaCbX1sE?GLv0?gflHojNYV*)vTy|NykO?1 zbEhG28I-aP6yi1gF7DUZVCLy4-O4pw{uw(pe}|kY6-GQj`LPKen<(F!Hc@~Ej;PN! z-rpx<$G|s$co>60w42tP$e*8-SU}Bj&@5@MA!(CPw#_YG>o*HdQu>0*45*yN1i622 zuIU&E6T+Q&IC$uVdHWA_=O=CHfApVIH{M8tv~@JJ&GR)!xW!a;a zdtdJLYUwyfn*7Q9LZ&}pPC(cq9H!{4_^A`Gqctc_uVX&x9&zvIX(znjkNW>i2NgMj z^da+pfHn&L^r1hEgBNN~w4on7oBcT7P;e|Xp&=b?f&SwNd%%D2G9kvT$;}!T^;DsH zTjK+Z3M4{uhtJRb=U>SY=mUk<<<27L)_=K&?b$%HQbFOyW{LlxV16q@MoKgc9=FJIQ^qFt|2E)pGi&)Kjj%`TmyQ$tXOH6im!G}yqP;G zBX_sUpuDDEOPFLMKIxf6hOL2Xc(Vj!D{qj|8ZThOoi|d1`^e{ySl?~qX>8sC#TTmoj0saoPGvJIADoQ| z3pYsrjPklQ_BtKa>zaOxr&~#x{#_tGZJuXsZL|Ze)3B#<>Kb6oU2`*7b7A@yAaDA4 z*oAGA&=L0JK&V`VT6QY0k$#g!&Fw(94{pMqB>8L2T-Yk(u<_7HuCV4)G0!q~UBu1( zgL++O&Ci&txjuQ6?jnsHtnV1I(LSL3Sa(!=zU_OG4Xb;SD^Q!AwPR7X@8%3%^H=U+ z5pju8d7`UzAYRnpM%*CDMv->&r*jO#8vCB$ZQqmpyI;6?#W!CO`RnaJz5G;+R-9h8 zQPNr7UJq-$HqWT>R@w3s7H+%O+kqSL2ATfFLo~_)3%?hf-0mOnL6tJ#mi)vB>61dA zVGfK3lK=*U`V#;;9d_!8P5+O1jH96cs;1-lhM+^88sQzk``5_g?!&W10W&@eV^nxo z_)bomzG9=M!y8gM6q9H2S(qRLrcD$9112NHeAhA@?TKugw%#U)Z!4A@dQ8VS~uPpIG98n``|06|Iw!pJ!vcS z^2IHw^0i`~Vl~S0Kbi~0r|FdkpWsA~cLcm%dVP-=a<}{U;Gw;!RKwgaU8A_yA_0eD zu&p|U3ur^UB2=#-bX$B3qFvrSo$B`o#;`#xPbN2a(%WMMCS;VKHl+dMmnMWv!cTob zBOMl_NXOs$LEqPFM*XJ;dmXpBpB%{{8fCi>YCGv9eX`o(zGGM$?PiA|f|_h1xa%|Q zN)X+=QLzURly*Xpe;@VY8)n%qL`0=tHF6@X0rQGU}59P@3!m{Rd0TjdE-~ z7PKeYQf*Jzb>%NRFv3(@jk4~jx{y<#LMMNN{JpMsaGfW~-}UVwe{23#^?UQ^J{$1~ z(fs~>_yn(jKFPxy<*5gL7L*xNu@BVesSo_L1%CxCU|~8A?zJ|g|L6#0Q&XGurWcyp zc9W;rMj3BkUOYr50sJA;zlpq`Gq$02DnwhI94V5B$0yr)$TMFlS3A`0Q!Y@tl)lGV z;~Ez!F8X+5Omj;v;y0cxsN&E2l&+GXunx7arpK+YDb{5?r3+$lzk%lo{k#&m>)yL& zhg*(CY~tm!kQPMAi}~8Uyjx@~ji2n1Ucm6OUxb+A@%I<(N4gA9Ja zp>Q08z@6dpZ^{-|e&y5s;-c4e!4|LTXMWRnu3@)}?H=kq>N{5d@VjLitKsH9lywM3 zY0U0pAQ$?)wldB!-uYW5*korcL>~M6;T!fER{;lIdDhWd z;Sf3QX?)dlLFn{6fNqU4e&y9Y5#BNa7kx`=%5yEvBtB8y= zY8!A~qZZagxM%mHuobXw$um9&Cj6g9V_&R2@AI4v9}D^z*Rx*p(g$7|I`U)9^xHo& zy}f1&vI`>V_6A=)%f-v0m(Phz+BH3N&2>rWGLGXW?@9LZrl@Oir$5F3NK)Lq33Aj1~U;rdfHXPWHT*aXOs#AA^um@}K81Y(M_q&{N9h^$I^lRZ_ zeb45b9EOoDRH=UBV{hym;xcA3VALh$f2uJf9Ze=MY$45nD*t^zJDo|j<28m-#w7^X zH=cqByp%&&QBtm>y4d8=HLh#44-}t4oQ*fij|sUpGJW8)$-}Q6fHnBAu#6qp zEH}Z+xzvs7Vv`$T%iJI2an`s|l3cXiR0}R`&2%Xpx$SP}x*~J@EVkwQrcD-3e-kv$kxk z{^0#$EqoTXmTu}i?=uWEsX=@5HaWtuEkjk0yXen+ecWwRZZ12=BBtz?o+AmSXD+}#DEvI4X6Y~1Un)aM#=AR-P=5Mb3 z6*R#sXohObv-VxudL=%c5#?H!JJF&%_|${_`B7slYr8!AF&6UkNXLTlRB7pXlxJ7Q zRu1DJ@h`A3y>BDsBpEZI96L8T55MSnqd$9?e(@~gocTn)jaEUB2Uq_Y+sN|D65m;V zisyB-#a_?D`YvfTF_S$EMdsgO&Ji?!WzEEY&X_kKYX17&r!EhX9SWrp^Wg_C=fkYq z`ds#CY?u=-6GxO?${CAB{yn7NZaIllfsMIIdy-+F({67;wmjUj>3s5faepr3dC#8z zy58gXb()J+ea0Q>J$&`~QsH^s?4BWXd?x}bQl&VK*`DNI+<-kvn%c6t&7d|qwcJYz z9c`7*T05VjO)}$2M)(>^(W%U%O!=g{*3({`X(?SvxjTwidy@3SYuS@*#ijiBfyy_^ zJY^7ag?(%pjW5zy?s1fU$}JFwPq0w^SBM+&1{sjWlfH)h($Yo2rGG{$xKj^yUVYqkck#a_2viPXWeD8$cK}Elg5( zBY+W0h`kfoYFE5jyK#=#=@Z-pl}RZq`R?lL(Plua;%9^sjs*|?#J=%kyleWrVh_ZB9$~`_R=kPXCoDE(KGHYC@L;0rclx{vp zBdW8J)fW^&O^37Ar~F|f25x8R*TP3kp2|NsDfAgu!NlGti305@Z1A%`hz;0N;ORX1 z=#ydBfowT;XrGhYHxd4T#N=9se6WAmJ;p@7zii;KP&%NFiHyUzH8HK|EuprHDew~6 zPHMdFPd>P9CMl0$95~}h9)sxDiEbav21o{1lKxm8rPGMd?0%9$pJ5K>SkxtT%0nCQ zLuT0cQ`-}52s^JsHvqd1)1GEO+tuo(N~ic_7;Yir9l~XPrFoS6nm4(B7GCsyj+N;y z@>}C1eHV1$MGB#slY<9-Y%2EPp~oKzU6gg3XVH(Z;(?#`KpKlh>Q_#-oV;a^KyP=6 z?&qU+sWxiubf!CicaGmmJEf4CnOlA^yWaF?j8WUB(Dfx)?YUF-i{=cLzw?>KRLe=c zE4(l^t;%la%-+rXDLsBq&*LYL>|?#I-SSlbNqlCo@bKyyA;TRU1eJ}_Cf?mIyE|+p z2eH7m1Kr*vcQj}B<4zLXJp{#*W$UbRpVg81M~4*_%mIhW z(jOwl)xX9q-r9Xuo#JijN8z`EM|p~Cyc*y2HQvy_Z&~i*wL038;_9#L;kX;?`--L4 z&%5<@tG4R?#@>E#+w%|uzf;Daad}RQc~!1&t;CwSZUBpnu^x$p;TAa?1`F)TWvB&f4LYIR~_ov$jABB0G5!ZNI9N#Iw z+rH3AR~}~kRc074)!S8Uh-(ALCnjtp%Yg2)Fe%VZL#nMd5$+W^W60cmLdZXWmnrT} zgF5J(v{|3fK(1jFwt3R_v8y|2t29)9L6<3`nnxt&v- zp!7Fqa1YNok#EWwq{{0Spy;jmv$oP#;PJA~qJFK8YkOGubF!+iE>|Y0W;zMn8<5BDL=;L(VNv~?V-Ti z%r36&wd<|jhJPhq=Rymt-|i+x_2N*N;(0yG?Yyuzhp^Hwat$?%lkj@@_6Po0)Xe^P zYJ1Yc0qLlX0?q5lpHumVpx1AybLzC}g6%iOZhYVt`~Rz8n(ioZ=h|JW(U+BMQa$f?a+&A_?d4M%SOdm>-Mh1>-JCf zQt@+~4jx<=>1Eh3-4WMz(`bnGW*PR8`HL?w^PlaEO*-6iEc_>$SR56YE9sTY&;IbU z&5Wd-+0&j7Uljgkr^3w#ck-l~(;A}myey4--3+)NQrL(6;KPcHH04*A`*-Pg@Y+4^ z`Y!suYM(lkaVjg_QJ&5oug0JGTbaD*#T$OpZ*q;l#`T(AQk?$T8)f{>LmnS@<)&Ae zj5yk^Y|#!@`ya_O&bY~-od&5b^GO-*aHLGXsOu$hiV@Z~oTS?Y##9c2zV@ltrlvic zw6Ty4E5=vW3XcV8@I1q&z2M`x)(7{2#>GBLu8gP7de(61&mYx4WHKP%<_?+ufH}bw z`%+o`XKbUE*Qo4N4?y9nr(S1;qn`J9qwhYP`9!{r)(~NQGA^LK$8&yDJ@up0&(do@ zg*mLxKaT-(Vn1*L$d7eY=COBu@3b}A3YE#^;WLro<>jn?w^#owO!Cb&rJr&OaZSUu z?n&C4o;MFE8M^ImLN;CF#~N$iTiL_HtvQ(DADP<6GRvvu4~YqI$AZqq!ap6sCj=NzHXq3UZJ&Qz=zuzE@6?EN2AW%O0vS$x^gEdceS6sV0mKcC z`$y6_t|{p*I7#UKR1qh{%a>SFN-xv<(g%jfgLqoC;1QMMl&d1)amwki}-t@)>HUYr>L?n`GipY z?!j&^YFMYLYqE(+D*tqcK56o%zCjaH`le4zRrj=mS3UBo)E);<$J>(*lJGM@8bj_> z=mCwxD)fz*s5LMS##r{o`%nIvCDP%)7)VFsBfti*C`Ea?O>n3^bW;1^BfW=! zn2No>iuS}fPnx8sF<cd@cuGDpm~@#(*P9IBH18Q#1KRK*h&DeUKAq24z{?8aGl`_MBfHpboaBsU z6RWI@zJz%`p2v)n_yqEMbjV-zp!xgt2Iecg@yh2dJdf&?^rvj%G4Ir(8qYOAb2WZ` zqW+V{!uF)y>9#AWhau8#;qPIInSUM>SS#ONFIi6-xB9^jJGc;sFY4iiHP&H=eWbDO zqx!e>qj<;YZz7MrtLt|8t9~tf7uW0es$20Pe%!`u`S*6KwwvzSzWGcYAVmucb&=wJ z-5F=xM(k>Ly`35qzhBEo?YjC+20Ia!=~U=Cv958jCIA@k>e%WBM~?wxDbFyxL8iW$ zP5J7Oz!o&FGB(z6)W=)JJDMw{zBwmJw&sUImvqLhftT;BoN+epVOd`bl755xM&{_S zMj<}RFQgQE;OqB&J=VwRYxFt5S`B@Le#UeEDTqThnTF=X792Ak4oy!eg z+hiWG@^C3`F@i?8wmr!w#(R<@bUV#Q_FlIC7;eu5%(LW|TXU{1rLWS$#RsjAiKKJs zoV12(+mn3F?mfwX+MwY6fsD57B>#%si7dJ(#Vk{FvohX?x315y(DR4ZBzD(kY-bXdFeW4=a*~7nn^Hj0$B4FkqB3 zb*7EiV?i3Ewf1u=@Yd@$*AT6=wbk@Y?v$NFF-4J7nEN9d@&KsJ*SOc;N<6A*rV|mi z0mTFQkA6Sr5(1`co)CvBOG(2srNjR8>kf8~L%gq%UeIVjes7i8AOf}wQpnoKXk z?b-!5T7$x-SnKgRsDh}*9rdNUb-BUb@2m81*LBYHBi}|L(+{K@LAEpwsqp3YYv{c36_d%ytz;BQ@71y$6z)|dRM^9XXTNd2sxz)3>aWO#+b z1;!-CL*62z2Q+LS-&t?#@cwiRJRJ_p2-F>U>W~h|))DSC^fn{1wpH7l+%9Xl`zw=G z_v0a7^jp{p&U#!$3-kJ^?pfWBSd{)%o1{1?XX#6(`w(Dn87q0<&|74p?(;ubaG$#9 zqJ;yX2O8`G4~&{4_~I()bvCFQuh$^ex3SvE7JFR3c2hr?XRW0U z!NDSJzUn_Jd`v~2`oOQ>^#y7NEX#uO2GdSQw1DE5Hh&UBBmJ2VX185#v${-M7Ymz&Z^^=*K5q9{{g7Acd1_X3 zxdp~IP&e^dfabqtv5}|PN!Ab8#M3$gz%DYzkslj+f^1QhBgl5iPIX(x2e#X7K14=W7W4?b*3x!r%De(j#N^m~7g#)^L`kJ7Fj@$7KYzSqrot=!gF z@ptj)T3BIK*6@A&H7wc7alx`9J}(nPCZqi~a(TuX*NKDLVT&4Vd2&o`n#)r^MVWLn zf67jFU4*4yVbCUYT_cUBydmxWZ(OeF#^KX89-eaWRBy82rx4k=5*WwH3)wdt8S^&A zJun+9A(Ih=$Z>Sn^TpoJAENX}{*458&ye?X#?@i9tQ!=DoYmhB_I6OEGs$+?zukkQvzuhAUz7%&f38v2hD*wZ1C*8)b-WWx58B6NDr_pLF~9A%B%M-2Ig!KcfYt zLV9rZ-vYe2@&7tGZ$E0oLxj7uId6oxd*fZ<+6m;ZGT07g%)*pU$_|{|VunqhDIDDo zUi^dJG5*6F{zEjq=WyTi zU+=m2>`#2Q=Q%oTp?j4Odc!P@E*fxrl4#UU@@GS@=?YO^#*}ksEgHC7W533@r>za3 z)V0vdu*SLd(q7`@$>)j=P$ncimzUL z`PE;(_@#gUOBXMGZu)J~*Z%x#qul8;r}R1Sr9W(whBmm}^T0h9U;g-)FaG8Se-ncb zeeUP}IPpT#{i8TY#)U-#A5g+2JgxBX$r*FE{V&~xzOrT4uw+SA|p znZI+=LJH%6(gC;u?T7aC&j0V7aoFpNKla7YUn{0!i8>uYA% zeD3w1yZGrZ{B-EQ^VRQM*_eo3#zueV$?shJouB?YF+MWcXM)dUk-VwPPru>QdBq&{ zJE%|e{!zM!^zWg=yo$9taP&AOYH#NmfJ7I1REYF4(7RL)ih5!U>YK^e!#*VXZNWjvOwyg=5o%2oAO{hj3>$;P^F zm9zbjFK*?1&$IVLeNm40J$+wa?0WT^Uma!s>^Gi0+I}++e(Lj2#q~TVJ^jS=pSO&I zv>E2&pZ|NG58WYPLFv_h_3Aj(#`uGspx;);mYn(aGXEL(>i_cUXjine@4jSui%GV? zsSTz-WVX%eGf78}Cz;pxc=8tg;yQ>pd@|3Fh1IXZz0u!#oaP=$apY~1viXU>{E1lH z=Gl;b0DkP}e(d6}zW7&hUpfSMDCnp83R@P6jF*1dxS6~0?3#*|dQkaxH)Y;@&GaQ~rWafKYrGj=^1gB|<-f-L9tZX1`ubh)i(Nko-GI;aulYnp=(Z$RTI? z$}eQ2Gsfuct5Ha6Z)%5Y{UfBc_u%I5b~%62ZPiZen!K*3tNaG}2J61k4OrjNTKOYi zd?dy^Hko+#;Mwtq@A=`QpB?$9AJzqVreRHo=THvC@XSbG#LYUPp!Sxnao#>@lYG=R z(Pf<8zPFw?#9R8tkU`{fzO0{VSgTk6llF(GR^S(xA`&(A_`O zpZUM}vTw$7K6&$xhOFHPvwJtt=0j$i_dYOlyMIle?ltHhX5}hQe=YZj-zX%0oFOLS zGl@nSUwe}1w03nJdy+hlYk#DFmz3P|2)+CejlDdl(w+RXa!(4qxublFB>z!Apxw{L zpr+e=c35N5w6O_YxHE5VC1l=D$n-N_^gYS{_;a!D0&dHmRKC5vC-*Sb zM@o%3eJrYO*EOD9pKeE#S(c&L%RK5b>vPD|YtB97ckb!5u%3426AU#O_fjNljZePf z$&0^y^AJnR-k(_FJ4e|} zG^v&a;kt-9C`jjC2{kNfHQX60at3*qw;LxWSpbXy;lZhK#zeUdAPnLK|L{n zpspxCz51Ur=D|t-kxW@13+kU!gQ|ZR=yV?wh(CYu&%-Uy?p%)G?NaWpLp1K!OQ=12 zczja7f?BRw+P&;+J?&LrVx^N5={J8m#~`dR?ViOSy~?bfw58my@}wdTqUNgvppTJuqcul12 zd{nhHg#}uex1AtsQ>)BmI>$5oy`7rgd2zki^J^rab11%0<1di)xCPaj>a^D_pQ0Pt zo$xnmAv67gOCD<=UVJi+q}mgkczP?5+7NY5yE+A$S7Ga!1HgcJmF9JNZEUTF5mYY` z7jjv+wv%AyOHD=`+LOmCVO^hL5jUk+;qI>N@jK@|>$`+(v=3hY`0K-_Qg;3mRoFnv z8Jzl7F6y0!P|{JF0Goqd#s2tzR7CsQfVaCKWMTD#U~Bg?ePh>$J4py%)6bCJnI30} zcE$w8P3*$^|DX3?{Kmihjf?kw?7iXVzjvShk*6oV=P;(tp80Ay{(HOC6$?kd_6NUq z@lXG+e|qsNfAlLCZ~K$Cb$b%_B|b@MfB4LRA2s0_CrJJk4x3oU>mIgd{~GtOU3oN` zZf`F40nee7)AxV)`xp0q_ukNH9Ae(2@#q+^unI_H?Z4c=bvpOM z*)vHYbZ*ZQwS0zU+F0XV$PHfIY#p)EOW8MUwRnnNT>QJ{cZ>M;T_Su%FWwb?oqWjm z34JSzncsM=9FJzo-OJwn19|8M2=Rko+^*y2ytdk#nPs5z=$xeP5bhn|x--tW z73l48PbhA`4sTxW>3F;~DR%&8dTU%Ti#tMI|C+D8rXT)foCSB^dv{2lAy4_oC&uP` zUvqCrt!FTf1J(%nxfy|R(d7s|j*X)+cJgsum2$g(mtLH-3I!o8+Rp3F*o4Zz!^?M2 ztSaj$b*n$rlHpdkNO7IyAIK(VIzEShYk1~ozR&YF40u*&orQVmv)>*Z#@-X80W+`l zzI)gArk_{>>gQL9sWciHBkc17Z-UBil=wF`ikFdgZ%nEviRf$`b zS&2_iAF{YTOpXE$I?y37Y77j5j1dOz)D0Xqf#?vVLz==PfA&ZiH5u57S;vRcPY0JF zz%BH6B3)m2G2qleM&Zd_Z$?Vj#X?Dj`=UwL!Ih+I((jv3@CGm>3_=(d-?-7nEqvcY zr)`L9vxd6zx+hQ1|6tGr)eNI)P9CJUPjAQW--TH@;`cfj@S>h}u}JPFo0y~!s)-SpaO%bz6G?s!$NO$<{I>zf^Y6%_?d6O=IdsWgpO|7BS=^I2)5=UH zf9jl1h3d)cQSJy&|AM49cDvwmCo8>@Zu4F;2-QS_CKjB))j<^?8%sOjzk^Pln39er zNNP{fhT7zmHjE9&|9k=CAY&4fZXXkQprUWmuO9!D^`9f)>1dnm_A?pFZwJo1Z&HQU& zNDEV%;r98~3b$LEkTUd1lOlNo^nL$rT)=`BtaA5+$R9G}zfeJ&di!VIe(}D4{k|AK8Am_z z3m>`oz(4rF#qYfDcP>8kiyyl9_**`H<{QEved|XrKKRZLUi`Cv_|HP(XVZ!UGXLcN z`X?blw=aoqcg!{|`~#N#bN`yGZuy5Sk~<}zNlf>TKKyx|!q>n-6>Y3(jJc{!JT@bA zy%t58Ymy)H-Pv=@8Zb2@J{DOXDxY3nqSW=e$T7?7$hjU?h=UBHg9{tndQ24XGY_Z~v zlm;$wqt}p>yCZD#y2fT%1?T+>TPFzRef?tI!AHfDR2^w zYu%G z@u3ky*%%`{KBjZ`s3SCvj1S{K%n@0dcu7R?jVE z#F11yrIT)lmtF5C)LWK&mm9oRpTS;#nZ9JpdX|6WJt+hH7b>rA zS5>dN6&C)QPE9t;K!`FjL^&B2e#IZ8qcM>Ov`@eB(=ma=&VKLd=>#;d8)Qu6=WI?X zTVAnBI=tMNc%71ZDwu}2M_pS1B>18<7BO3R9raDMnFeGWzAt%?k z-eLF3_lWO;u6&0;(W&0l4?-5^n&ujYRaQS=lj2!_9Hbq27F`fwdB2i+Z7dJnYd>Yl zZ(JYyGZcVVk23!M&|NFBkw}-ycj=t;HMsl{?3?+jl~We3w|zy*Z%VgT7HH^ zul>iC-uLu+Y*PULP5y}`7IXf`kN%I*|G9?O12Y%X&r0wH84gAwvHyqi z&wSx$&V0qp0T}G}1K)fg>?vb1Z_MM_RX>G>onT&%t&cXj0)_QHBrLo2CcM2z`)^YH zLH)=3kldsGR>R#t<4nKjo03DlEO7Cf__zMaTSL!+8jP)^uN?bRbo^W(i)u}@3w_ZA72Po2Fje_$yLs|Deh?V_-X&N`INHPLuui0 zO*hSqdl59=q{!|$@O(hQPpv<8LsIOi^NrJ!A=V-ltPhToWY z;m$anp&U!#-bc`Rzs!CmUGHnPUzPNmcRkF2pRtJfKKoJh1%5&lu=dCJ!J6nP`(fYx ztLY6Wj7QoFW8IIwN55mQWuJrJy=}LC5VFy%k2G$*X6{#bUK^r2n*+BMiQ;9c4a;xU z-zYw#IHrsAP2RLU$r)5Ok_EGJE$Cql?gsMr@}@laP)_13FWAQuq{g4<2g&Hfu3u%Y z_dZlP^Y{A$*SaV9Gr1>u@05Qn*L}b>LDo;YzwSpG*S;2<*C|~vD?_-E%DI<&)`8cV zbFZTj>w1%mJYgyXYouboh~yU z2RfmL?Y{k6h8z0^nJARTQHW40tSB&*S^8(>euW>%?b0b<{F#pOlU{Mc4`rOtC1|8P zC^9JM#3qfJQ+qmT$?45g;6Ejx&ZbkMFK%VMjwf<^3yI=08lLL7+z?87gO1n{JIFp2hJA}nM5~`1= z8P&7ZW3S5)6ZL*>@hdBQSG>-R_&AT^KtNKU?eGA>Mh$j=LmYs%#AX7v8+poq7y+fj zA`FvVz`+>uB~K=)9K1ROwN8(zn0a_|He_jW59a*D)X+A7=$NM=4h}e)tJ3 zp|=M$1!VG~?HKP0KeX+(-q07?;lLn|s|Ix%^-9$u(uRcJAbltK59CNUM4=3;@c`i? zFt4J&0`(W_o;E}~(A!^50qM{_c{tS@pJelC!^(dRt=586m!U(ZeJ^7Lee$SY`2S4=qL!qNg*{q?y_zy5_;wiwEQeW6-x2r6S z=|3v8@vAoFGRWFr$SLp=xHBwRB>$E|a# zs5Z#$QV&yB5BUl&R}p3*33?esenHsVoSIKzkE zXT*8kq3!a@IP)Fen=x_2?cWtH@jBOdSH(5$k>7M`7u}!jW7cLY&YB!@%SYv?e0#a3 z&i%onqR&p!!_PS5T9Ed6U#Pu`_=KQMyNrOjU%DBjLJqiI$M~i(PHVk@@z8&I@EG{m z%or>1%)w@~o;^m;I9liZ8Q5jSaVqgE$L7@5TA^5JrEFpzo?(2>xc!()|FZDq@48Yi zs;@;IZZK&y*W`C;B=q=sZKVnwPNDDhe@fr-A@}wmYMA$JSoD8ywQ7-X`(g1sXPUyJs*U_qxn_moPUXXyfQzl@n&I- zb=_W^_Zs&-3L{_u-xH+1G)}2JZ|R<7%QN!-+U`BctAN_8XcyMXZBf3bFf@|RNs)Jv zH+4@ExQz%(yV0DJj7ZCRQM^v_AIeF}CxnHEF7=?ty(ZkO12S1STZKYcFK8>+A&W*~ zjr*Xq5htigLll*sW-~?K~rqfT( zq%;EYF~gui!sTM%$c39exa0t%a9yJ?UDNf(itpDconX7yE|b=vYk^+*rcD$<0|({H zDd$sw0i(XDFRyb9nCE{8ye*0<)NXqaav3eIjq7W87NN>ZWfn3%NzL!|qI#;V@DcZP zk&Ps4SO#^;#+&ywK>wnwuWaKj4>W{3>yP}%mjO`u3!}Oa6xMBu*_d6d3A9VOH(BYY z+yZg<1n&ZMQ0l``>v6=R_ThFR^g9-1Y)ZEj67&?sG&ckdHZcpdzo6!Wq|xawT(4iv`1^N3LnLD_VhY8 z+QJlL5C@g3sgh;2jCF~qv8>;nEGqhLpH{+~>e8Q7G13WoZ7iFt{8x-Xd&2%R=K90K zF<@TB9Dsi6;|OC7KiNh88pEkGmtF^|ryd7CAJuiT+JZ}Gj~nfbI61!x)_f{?WJh~g zrq{16e#KjGewUpfT4NTf=yTXr`XFu)hXh+6WlTNwfnSR|^mpo9c9l8;)EDgq^tP96 zYpdP$wyAb{1h0VGj4*``vq|fCkFz)J;5%W%LJstvx^e5Obo2t&Kde zuuTopb-T7Ejn*(LQ{!1Z4Oj;m7WzF8FD#@u{rx^X%=DFag+HJ9D#}IiPUTs8qq;qk zYrAq8tix9|uj2Rb9GoF%NbZQ*n z?lt%r$IloFjE6sX_6H|z)-#?m*JNzg^9Nv^bc$TXAm}FUNrG-K24&st1-50xP`X#wGn03PSKfsdD@fS)x~ssAeS?e&r59u& zJ{d8%$Mg7Am%O=Hoz#&|;^07X~KjfOO`I}hi*PxB=&piB@@Uu|^==-!MUQ5sK zJ)SzT@pL!|E-*gmfR4XiiO!Nkq?07KX z(&eZTi;l9&yGdoy_#7oK&Q5i_dB7Q3@O5P8&b`y0^0S1&d-{>9g=~#et#6#=zKjt` zR&WKXXwkzG#g9mw4QBK0^5sVwJ3Z<~y_dUKOXU)VkAAOj&tB{(#{f6u3=3wUq;c?R zIlFsG+*9kvU23e>aWHMSIDO# z_j$T1wMC1*OX#MA;;?**E(f#;QVt)OlUtsII0sy8X`cs9_oWAJk8=7bK~tUYtDw)q zMb^;A&h6gEDqvt^v0$*gKd!?!P$(aapjT$`#28`^44;r2=~?M6acae`NRiJ;RlBXO zG0H?Vk_#S=G(OCx9<5Nc2FIBYT5XKr{&OgBr+bUsj_vy6$;{C9+nH;#`uwh%sGNq` z18TRvk+nwLuZiKO!*6w)yuBkW1|?Ls8L%wlUmR*0nU4RJnE^rr)_Tgt8?CH9H116w%fo<<9O$)Zts_yb`G}JB)XJ}yixI$2yv5ELAV}0*?dqzA z(b*gOHKy?T=$het62VJqQRkVOKYPEHSgbS83*dnC(?GvqL zMcbz*i_giUuD^t)XV9d7A71o`8gFF$I76n-MK`kgMH}>#OvAXBgFH?7WpkoJ70ole zl5zG+?fh(z2e&2%Xv|SS5+!c4rK3LCqd@^WT`S4rwAA9KVIV!e{^`1>DQN*wIVryi?e>sFx z(zP9>!8`T2payzZUTEse_-^rb?{nA~w{q9rf2xX;9^2}yv5O75k1FVG6SR$w3>Jv9 zWB<@NLm12bO`mw!dB#88Kf7Y-jTn)1iD-XLl%1&{HKC8*$4H7;jJQ~rVO;wIw@ygZe&E~OC#sacY~E!tx9amAs2gvOGQ27FBZIR|1fEHx%ty&kurwUy zKO@}OI#qGU#Mz0OG;mh)TU*X8IR0L#ak0(eEpZXVB(m-)-qUl`_Ndi2{MAr&)!kN> zuM}GzW{9B^#QkFX{azP`n-0gi->@Hu{q&b!*SUa9JE%7BKV#K)y>puUvsmZm3T1X= z?@XQ4JO-@(n~ZXxZ&|1aLEPhQ22(t3&kLNNGsAGic0l{6DDlq!xVGJ{7gV$?LHW^x zURR^`Av|UEJB|)fv!+siVfE>!E#XPWVGc49$dUDKUJcj2G;kjH-YiJhj`=KnMOeQ{ z=KoLtJU>|t)@O~rN>*GpbY$Fp#!_TLa6-tO+A5p^E@|pf)>XPcixOU0j$ZmupP4Ik zhB&uEb^wQdaqTKYgiBWja0W!9Gv@g&! z#W%(}wszl`9_yg82}jbNEv-$R&qu(7Hs*a4k+1Y@QhM(KeboTlkrGaKZKLP7ZBAD> zOp;hf->Ss`0L=kuI3sp!9!$t6i{ehEo4lOc%#{N3GD zeOmbMbWo%)d#lU7pZX5iOx)G3Awoq^ST&>$m48=X*}{BY zBVB1;VBtzq{I>Fj+b4l)e7xInF=0eTX1m6T)yGQggTZ!;4VOk0#V$Iru9y%u9Q{_x>zC)^uhSr z`8x{|?stQ)Ir@ZNb{(GI?EM5XO499JeNWjFVc=l8T7Rz|Fg){U^AQW-}%M8uo=Yh4X8y#r=Ft1js2x+|^| zUO1mg>4F6m*1e}YbK5}=Uh`6F)@t4(G(Py8*_BcIG!O<1QX)CLA$X(GRZXv?*wei3 z4y*@O1Q&tu(P4?{NPptqechSnw74U=5?w5jrgQ7uzM*4q<3sjc8J`4k9V)@tD2OQv z2sOp*8G0E#_md~e@3~8Uh~T7VV;V>pJmhzdVdOlzc|HT&F?)L)t9T%&wSx*2+j*IR zeQf#h7>J4q)1y|gNTRZHXeV7jqNZiJW<0k~3yVd11d|X>f-R<=AanMfzcgM?+x|AY zy9qYF%lhFY5&!3nT50SS`w$GZlFGl_< z_GGfkgf;6a31}s5wz1YqPwl^un^R924v?L`0zMg?;a}(cPbDfu_5dxE6t`v3!OK)M zhn(4Bun#T2+JcZKW>eFIDjsU!I!9UG)Qg9J0+&IOjQ@uZl@s2G%7fLY$ z5lK%oOb5}yFmmb@u`2f$G%?}NoN`(#XI;rw=`8$Ku=!>;Z_XHZ%4jy6O2toh&eU5V zyrIPNpz^A%MivkiEq^H@U83>Wu!w7$jLqidxQtNy{Hbuvh;x~;ks#><_?z1UR7QoL z;z~et!OcF{rkr^MDf0=Q_J7-{Bg0U6gQdeKwSjYYgZ5JKuxABK&qme6 zrgXNT8;6e1>D5W4h`JSx_^>79k1E44Vk|;76rsgfP}!JETM<+zFtp8csu|6pP~oF1 z(6ZjsD+Yg)RhGQFB6uxT-p>zeX*pu@YtQrm@NfT)gQn_z*R(bkT}~KLdgj6f=LU3I zX=>p|3h)aA8NAuH^=}32!?u$+(v9rLUd-)lyeH_|CdFm@hI>@!wS1^>!Mz{4HO~wo zZ)B{--zGwx9ys+9q`u1CSmDbP8ak98B)H9yVDh(byj4i{PYmooP(H;XsWj*gdhTFW z?cz)+(#QU2x9UBYXWDQf#It|vaKF!4Z>s~nOk~LIM4O?n*%(PM00immvf)k zRwr-1c5<1CdUkW|D^+fBIWBvJcX2m6+_UC&tSJj$yHDxZ%}T~Yb*;VhJ-fDGEU{zl z^PR8s);GC1{h;Mo;r-(B zgjqV@me{sRPnxq@Qt;?W^EI(xc49NJGH{~KG>u`+%XUE<{Fg;OJ0y?m8y^(_FF%^ zq5;s5X005&!LVb&tjSWB&un?n*0V zS(Yi+9}QHHCNGh|BAG0j%_o1Gok2#It;UI8c-=XW0c{+Gx6k+M;i%|fb(XoIgj1Wj z?;SO-n*XybiLBiqhZ~rMj?6zjRnnx0usZrEg8O2vJiD)f4^!Lm~^5lU9LpOsv$kF z);is70dpk{Cdvyk`=w+= zpI6nOP@C^XG9!~@z~9ce_2r*$L)q`{rOIEF?SuAmMZ?>D(nt8sqfb=a?hgPkn15jF z!C~I~As&8reuB-UaNu1?OLs_GL2pH_+xq1J0^^`vUnc z7Ck_4+bXd42`N-*PBn~7L4%aW8HA&hVmK0-_tiS<0sz%@Mr#~|d@7J4yp)l8OKo_! zz5Ox4x(I7>b=6QMabaNB#PbT@>8lOVk(ILw&^K=8L-3T`eT%3H4}f^!)KC9gl~2Pg z`H5{V5|fVMi;rx+m7H0%7ga7X5rLcH^VCef+Y4v|t`q{7x{SE2EG zM(Vp?Gi4B79j@R}+DO)PLX@-5M&TeH{Rz#cnJ~Y%rD9`5)yF5`|ne&bRp(-Noy`(N!%b#(CafXP3 zB`6f3_ne(I6MwXLX4Yz#P}e_^_29IUSpol;iY|qk(%KZx2(JH4>)WNpvvAU?+o#s$ z9>NY2VQ;h9wB@--vi;pKXj3a(xrW z-ch91O^z(Q<_&J%4~}o&Bj$}Ju>84%?>foq5S<;f%Dk3e?y3#m$bh?N?x|Lye+k~m zZZT;bOpV#4D@j*WOc~k%@2j302SduGV{>*Nk6||jQwYMQ?$1z45zJ|+hVWtWrPnfsEQ7ptC z?cTOVeGi?IX@^JE*yKt$Nf)WC38~?Bb4u4l2`f2gYH*T&PU5siM|7+h@A&?l?;Mx^8W0tW({hcEoYj5p?uiSW1W_hr0DX1^Au zBdM4v7W zDteT4Rb)3AnMRgHmMGWXKqbN_&dh?ln}x>7{Hpo^X*B6C2Otr^7jm!UwtXjoU(3M1 zB6a&D7GH{D5&2^nCFNgQ!@Ie-RS&xKYd_H(8{)#ms}=~+3ss{G@b~wX z2=7Z9Ox?$1yBE)NXx;d~+y8lp&!vB@BfK2WqUL6%IpbD^3JeSDFMj&4qW34^`B?co zZ$hZ^U8fy_)OVXR-RW@;lKFmI8~&v+6m;*bsTF76?FeJyWpb8Kc4uI!;32hNanI7j z8D$Yr%moeuzvX0C<@8MX+yu*akgKH-lE2+_a>)=={%(=;VEDBMTQ6z7d$`Y!t9JL8 zXmag!29{sB#zj=7Z;ANuw3_CwJDCn6Y42w6nDB${p!UCv6Vh_;unpvog6Y+u^VP-! z-A1s<cR>c7&r zw8aN%pJN#lM6&my(NK~-#m6&0gA{Ca$H}Byf>gLYzvg)1VwqoIM_I)&csgrkVlBJ0 z$%6l+>f`6Px#$0=fHiv`-Hlw}1;NbA&r2mgMSI@Y~q z-#+qA0fl8eLKeEavwmT0eLiDZ+;~K6I^U*!p$TVdQTm*qMknR96OoZ&Nj;a+{Jq!^ zg(pj0vkS=|cs`KKij8h~deUcxIWU8v3IBzZ#=Of6E;e9}I`Z#q%QR<)-nQ~D&wc~q zFQV0dJBBp<-Pa|&i2gx*H*izH!ga_jVMk0c`q2`zNZdOtsQq@mnTz|Nw(jYEi-+W0#vup<87vPqzAqfu`7Eb$~7fQKXwV|LKFh0Er0Ta~i zHxsy!d#hjK<_c6+F>Qw(Kq=2}mD=H3FKML(oQEU^x*2+!vzrnj5>Zjo_oLC{8qw8E z2Qix^TF1!lnwzhuqss=_W)zKuHNdkZN2RpDH%FE{%;MHwBBHvPAV^mZYv#mnI>+f& zHiKo?Bv$VTdo$52SB?+kPlBL{*@kx`qxw55l9>-q9eP0emb=kn5?N%3My0*EwS*Un<^f@YWz8q^omQshh%#z+-yDEgt=ELirZz{2E zUw{kW@`2))&NGu4jA_5*MCCu1&I76sRLn5>yZjRX(t(`2^8chsYClzlIdQl|jJOD) zqgFRcDWpvdn;^w2Teellx=syC9sMNY=80rvXX4gO{zdeW6a29R(|Z|5$qC>%-27iO z{JxnL+9PbE<@i=3mE%>S-uq`>NfxtyZBgAGnLPBX9U-B6>ETbKCS=w=Bg-6HGf@H( z_n&k^(s`mebe62KGX*IBBu*xP0Vm6<=LTJK+hSKb;rNeNhtUAHw_(xfh!y%|sl*+R z`Nm>v;~ot~poZUpp&Wk>I+u>_Ut$pM1+kV8nmQMCs_(GnE59^BB+t-|4|9~8zMcqS zq|&wagOnSY6@XLWw_pq^XZk)Q97uoRn!&Eh9Z0lxcRReT`?9 z7h)IEKaH4wFGMm#e9o%Lu0EN4e>8HOg4zl3qrNY)_trJ$nh*?K_XWAc6gCQ%r1#&g z;2X z>CflyM+u|(LnAZW>DfNM{Nohxu$$h$5`L6!v*KeOc@wH1y#Ev>b#?EgwbI(}MW*rT zhlTR|3w(C(pLE&J@*3H(hv6!{V*GFtqs>}JX{f*H7H(I^Naa2gRTI_^U=|&kc-*$$ zCbI&J$OC#79wYN9mdX5K5a2HL()+O_#*%upT7ag^<%>(r4r&ln+s>`XPqj~x=Zel) z{FZK~*G&4iaH_N~pXGS8#a^{2oDVMi*U=F=+;Mvu8tvIcbskdul{nFga`nOWOwAac zISn6bSSPNoT%vXRJF)AFr^If|R*t;&iTS#=bl2Dq!svCkx}dOvpb%6GL*z7Gn>}90 z2t;XQWnor^^c}`-VKBKXtn*5TH`fYXJwz|63%erfXU)M7hZ z+PQQOzjsmbd8Y-A>3IImYD9-i6rcN++Py#Sk4a5Q5LXGsb1V*?NupAp?mY|gIc;}K z2+-}=AIY>L9Uq$Hhb?`e!;QLWQyn+s`zRCvn$RaYTT6Jz%*tBA$@cd}FytGNnzlIn z7t-)dWw0aR6a3$a1)E_rB;q#c%2ZlzyBBW9tt&j!Uw5v#w{%z*)b9Mdo2R&yUvSXn z{2vz;z^gVq@lB!Yuz2H<@5eQ-Ml~(&qGx8ySzpW5^pd&EiKiwP9m7s~wRr0*@hf-i zbMDom!xoioQoWP6IFD@zu6N47f?w!C3V_aYR~bQ$*2nYpw+3XyDfSt@OIcs(llf(c z(#nqZf61Y{2MER`26?^2RoGt=A1zEGwG9SH?pa8#C~erM$jg{G%(U8Wy=nxpuBap3 zb(p@qq7XIbxPMs9e};?me;{f;BS@Gb?ev^MckOe~?XUZJpD7b5)?WFXZm~T7!}jK5 z1?`a?cY>0Wwq?@knjlHV#NdAOe|UeBJ_vJ#4==@$E=+@cYW#u&AJJ%d!G6P`oFEqK zeqH%RL;d7N6D!qrn>i@V?ZY#e86E2MD+y1|dOTTMs$)Ht|G8tZ&ZJT0tIk(CRAk4X z`U{w$=+7>?3tEoe?VL9UW$TyeQ-$?U0pXrcMN=mUSOX)5^et=x&VC94!t(i%FX#-I+8<*$rHZuBhF#uOC{ z(me6U>(qK2j5=1bXiVRE7UI74xIJXKAbFl1zu`7H+EFIXLUXDzmqk)xbwT7Q?6#XZ z?W@mK77LpC+I8EmQK7A4nU4OZEvnMr|h+FC4MCVtaA$P-%7{Ti*u zj*$9~JOeNhHM{a*{vqcHq?-xFY4kBw*J3 zqKy_a%g8Mo0Nd2^pdWeU$A$mKhnDiY3|Qjc{8w90DJ5B#T-?CvMFY+ z+*=byw|K`Y7pFD9h0C277l8WjF#DoqoY98oLz^i98})B$WvUThS~b;WlCV2jFXQ{j zJOhS!d}Ytfsn>q;RA?&ui~>8bOA!s2!DV6Wz6v5+1Uret#0Y~=slcaON`co|5goXL z^$zd;yz>-2)rbvJS6;v-Wk|RDho$Ft{U(-&I8*V&3c0*&iWejsDM# zakKS5BiD~{uRlWmUEU(PPYvjAFINKtyeb;ZFftuX9`FwII&Qd61=E4GyKn~Y$cTV< zW57?l<+NtX(~3-(7oQ|KA35{F{TMWzwBi9rgt9hkKgVRZ&PUfse~%a{W4ns}IG@L=HLgGps%GRR?X zAy9hW-3DxVIDcl?N_3wCV<^UKA9VS}g03gz1Co_2`_aKpHv7>=hk0?b+GM-QgfO2@ za-ybx{2L#b_+K!0lnqodx^H%4_t0RE?Jx72xMntud<2oA14MIqF6$V=91!HH;;|j2 zEGKVQ@{`-AJagtm`uW!zHBSQffEU;yrW8*~-K4bgpRZ)I&AV2^gZ{mPd1I_HLzj=L z63`HPiMrDh`?r}At31}eXpo5OBi$JJj9|x^C&I9?5CezydQB|Y;%a5(9$m0L?X;M z?Dak)ZEcsT*)+4roq%}Ui+hE_FET)w4PA1wl8d+O-gXB?UXGH;z`%0~ExnZTrp%)O zmt08dqN;=W3+N*@(`|5OH~hH+$4>6!&40Mz%%-B}fm@&Kfgs1ng-gXk@t>vZR>JVE z0WO1;#5j#S0Y({Fk{$xGz0k-*cK>~1k^fwqxUK9Ccy`65%56B#!5MoRh*-`H+;8h} zqsFcIfzQpcog#r79M}m36?f6V9eQPPORONWm_0ziK|1`atZYwksu8G>qNS51Qtbr` z!=JT}Hb?*!rZ6I=V|SaL9UF>I*k6~ipN^k9Be#sUI%$b*YsEGlpDf|ag0P>M<69<| z6zy_FHq)^@T+V~)TzJ66xY0x}@u(t7!LSB*x^!oy3=u-F8|>(__ezGw-jNA^lE`!|dppMD~~ zj|-)(ixdcuqJMz4l z1Fm56(LY?B@`P+W3eUV)%mUrzqfs92 zS+u2i2r2A4$KGcYjUELoKbbK-sf-?YNcH0oy?E&Jx69E5(H4nCjX4zzX_i% z@HS|#ms>t2`jp!IPRT~;P~N2FX$WTmC_74ig#YV_3sBcUL(6>IOc<9S%>U4lK~*rjHpY!dy#OP8Wqb!RL~ouc`EcG|uf$e$ZSGqsMYqwZ z!L|{?vDKaVXuer^;c__rBS-gy-$xj4D_8?|()=UUlL$64#$B=W^xdoio_B(;ob&F? zWAA4X#7`O(Tc@fsxMIhjV6Eku|JqTbZ-x=y*HtsMqcuNhNi7W=FJcDm>DyQt`_nDO zja5n>U6raD*FEd2KzW_=zz-7! z(rGjnb)%z8TMtUg<~=)>s3N%lZ_Tg+3U=&xtsi*@R^sa&X>qFHZv;0Etq$@L8^%!ID+_dGB z(P!A3>Rl4^RL30C|*Gq+_%yzB^p;C`oFE^vW0L6=!*IbvrsH{RuuzH(o&QCP} z%Lej4CJ9J43{3C%lH3vL{M5#jgPR@BD{hU#Pb1bAhPS1gt}*!yw^DpqCP^DKmI{CV zS*yqW-;pa+B{&uQ?^`a`msI| zXS)vEsyv|d<09Ozo)f^9(IHFF%0mPcVb6qK0}G& zJ`Jh&lC-TFOJ^3FKsAdie=NG^c}A2UU%6;SNql{WN|*4ayU&k)>AtM%pn;%$*yA|# zU>-R;=40?RC6!wpl=IO0d5-eo_}XttHnUeom)R$m;Rd6}w!^;WprUq!))W3?$EaDS z8PgS&?MVi5Kko!7lO_!mv&-}C0vKF!sDj0)lwj2O zeB4Fc8b_XTUm)yEzr0$+$Wt9vcMv<`J#*+%tn7@ z`I93gd+#cp9Xl$!&^>-BYPOTAI>O+_cRVq~sns4{$zCM)eI1+Hz}^d$^mP}birLllmPXaCJ^<)043gSDcH>puJSHccu5 z(YC`Oh+le7UA$fa0IdYlhc=eSt0K|brF!$elG4Z7qFH#vg4!3L|Jl)mqUp>#$owmB zH3V|Ut#{;{X3K}GxSy4vjtD7bv%(^DR4|iHT~&Ja%KruV(i>yLr7Q zWM1$1ZuWcRrNH^vC*qpwQex{2j6Bltx<)c#`I#16zbC%lg%-|zczbx@$vJRJfVviW zjbd%JuMsFvu72gaR!rR6rrG$8_$wca2nW2S{mGY477G2vh5x3_uZ`VQWI~-Obp(YZ z&6?|TM;Ao3d_Kqgtkb!%Xq0SHzRxs+;3rr)O57RbCLnCB_o)eeD4n7A}F8X*EF`dTP`GW*8?8kX`Sru&N!&0yxXo=sd!lz5e9t*o#dOV2nz z)Y@cCpc|1Z(HrsT6T<(@rD+UBUlj6(QVwgAMWx8DQ+jO4X(e5sXn z)=@yNDg6s@MOIFwjIaL)T08SpeODczz6={)S*`SH<_)^GqGd2*t)7OJ(S`?ku(8%Y zlr)ojG+nAfJi)UPaG1g3H#jJVN0pB=`*BUnZO?uA$0~TBH01{$@u+XKzIrg(uBRgfedq>!wWHIwCzDaUND z7wOr8(qK}Bz)QP>+g~57{VL~GE06vi2?|CJL-G{AVb+*NV@6KKs79LGV5yt z;bOl=$|Ct6&L7z{(Xf#&Bx#kyhhF9dK>+>xcFt*)?niz~XwR)^?~oA^(}(|pa-?uo z_>y*#2#G7TgC|SeL|8pmu7ST9%-nc)iaNEL=L2-%J$w<66Gx!S>v&r0aellAZyO$_isSv#hmxqGla9KHQ3oah%?^V-(S${8bJiTLJRN)#$n%(#C z6ANO9qlNo(4Sn$u$apB=4%wzNx0doW=g!D6+INZSl%(7_E_W@DVj=jKWv3>1YWm4t z{71cXNit*dde*AAU#k4{<>?={=q@#b#=aQIw6W>mcw%H0vtw}6V>goxDm0T`4^I}N z6Tm9&oJaYUnIZXbc;srJL-;#!HRjr2_Y^QXkl!tM&XSx^s4+4u8&^Md%E&0*`sLSX z=%$bAH`Mn-qxRJCK=P@-oyFU|jPq2BBpc51xo606=av5F7{`V4=w4wXZ$7ljGR$AV z9IjKHezV>LQ}Gz|vK1lw8ve4syFcj{gN69Y>cc&Gs^R>d-|cRYQxMbj9>Y*-1LO{+ zwo8uMhVK6Vk9h!aa{%@cXH37d(hm(g2WWaAX4Fq&eUlp8r>u~lcuf4x_qNoCr4cq5!t5ZLW<%OwZEBtPcx4>Yu zi3%IMSMjBJ7Nm=H>K4x@VRNQ>!n#Lg&vo~dmz5uaU7Higt*}TXcIvlZ!>7CmD;L=w z*U~&)!hGH!d8x#KqI23^XeRR*o9YLI;pf50+2b8vzHEsXVMp?di-l<`-2Vu=qL$UHMGL4ltFA9oJB- zAv~nYY*HBP5TP2lBISMWV}%epqv#%0-GJC{GMNwIez)$)&myws{9!wU2(o+D;Qc(0kZcGB^JayF%%>59nNlhn(t86`7#kzxMl_$aKDTA z=q)c#K<<9$yUbq}CH}GtlwFS2i}%9N)Yq{cw7W&s4!t%s$>1aqLfe!GUrX~6!Z(y3 z9ieD0mWAi`haLA2&_BZ(EN=3oXR<}(;_&6B)(bn|UZHCf? zjL{#ciE~`4s>eGk)v|pC%i+ywBBS(AmEFTqah}uj={;%z`@>%($y;a5J-&h+_;G{Mq8Z+bTw+Hn=&6GO|B zyNY_Qm0J|&erl0$fC|=qgtPskOXOrn1iTYL3{6%>6^#AI67>>?b6U6AMliDXjTm~0 zt?P{}r`M;*c>S~kj9r^i7P|ZhCzMPq?96>cmmtU8V|>_#L$xthGmlhx>zG^OwFAn9 z1IB2vAjNewvvahsrJ{LzKW4?_+@xwOuSZ3hIG6Q8nt5smnU!t+h4SI4;PRsZ1L?yp zJ#;SsXX-HDJw-+qh_@7dW5c8CMBGc=2}c=;93c94nAJXG?~`B3yBo8PN$EEETFGx+ z{T6ttesMh3C;`&Nz<5Ak+uR1@MnjQ*_v(CU^Y`km*-?|lKiX}bL*zZ3T525ZUd!pyyU|FR(EuGw zDtwT>A5o7hj)5%u%_D+#_gm(lt=Rd+KVP>yZt+39}CV_DZ(9tT8C`7Hlsn<8pA1 zL6~jn6ZOoWjVg2MsXK+CcX&v52A|w<27FAHbnltuR`0}wyRd(qyUz@wlut@h*KVj8 zz-U9fY(I3t`+}gfDYh98vVeo@hKBBKeRXy zcsS=heRI;4>YZG3!REqU3eI@)qj^j1BPmt}w{6W~H{8m1Y_BZYE) z#}l(KnxdtQs4huOw3ctT(=eKHE=MiDFK2ppp9i3-eJn6lYHs{xGDG=qh#C`p#nC9f z*|Y!HSDvN$p|`GO{NQ(5nIE2^rRz;U*XDk3bqsXZe{}SpmX`l71;<53Wdc#4b7Ehy z{+`9rz?+n>k~m!VB5b5*OJb5P#Zat&le4m2`gi?z$NoSe$%-$VcVy}b5v^l5vVi;g zfHxKRIB#`Q7n7I9NL>j2!nTZG5LCn>1rK@@PpPM884w;phqDo` zLQYq|g)20U(rl#noPhy;_nY6jUP~cq7ikrsq^7DdUJ<$bX();JpD*B{I5kz|mk8Dc z?ah?za`{GM262Z4bH(C`lqi&u3>u4w+A?2&06GkIh zbJb6vpj~{<%U|eBBG7!q=a5i@Rop^!{dYz5J6U z>lh>JTCm)~RVY=!=9XRurkGEKa>9jgx2r)1mqO&NN9KtR=|FT9k&U)AJjb+RB%EXR zc~T@l4igBy#?9OY3Y^@Bt7mq}2lZn7QC`Zd#3`~XJb0+WX5`bI%FhlqqmvqKGN9iG0y>epEH11!YL*!e0V<-X6MsF7?g`3{f!!vi%Qk~T zCA$m#H}3%dstdM86&k-;V3eW41NTc0^edhycL4`nI_}FqZb5bNhj&U9ereyi5~SRj zIG^1I1Sf~m;^UO8o$2GZ*pc(*ypNI>8mK+^#l!z~h{hcC@55Bn)+1cAIv^fVZ;!1u zD{wP_l$PN-e9v#>)!Nn?b5tkXYE3hB9bXUj9AA86H#5i`Z}0QS0O{#ur95527Js{$ z{O8!h=r&$ox*T4!k^gL^S4k5FS`5CBzQJ}>64I?{dUE-kmi7u5zQH7B?&7t^g7k09 zKF@g~Bm*d9=O2i|YKHi=3Vi#T*R5t-0=sk*;=I;g+V`wlz=>sZy10WOqn#WR2(d_^ z%1RQ@1Anzi87@SCwLC3rDnKo{cZ|%U8aYOOpKXhTwHaJO;-n%vhzm*o^;30 z(;*}sm2(u+KJ=|s8oLoN;rPwr`(mV2 zd3|BHQnC=C@}A-tUfA&LUn>x%rJgMk?f& z#B1^;JYfAWnXgWDSL^l1uS!I3G?+c*VG*ah)b|}3;l&X=&?Kf?FLJLVK{_kLiJtfu z$>O?GFz?%(ena`K`y$LLvc2D0*#7ARDBL+T=kkW-VoL}_n*xwtW^CbKC3Qw(;YHP; zf%ReG#pSEWY14nJ8X0k{tSq)4UEc3x(@l;IX5`pAdtQ8@^e~ryaa90h z{$E7&xcKV^FppG=|)YsI3vFuSz`m=ciJ{_WV5 zp~1(W^mYO|s7%(Cai&LRS3{G89grD4c821X%E@P}+&7PHEyCq(qFEQj+DU6mY=QA( zT&oX&E2fd_UkaYO2E6aceqp0EaYc3|#?G6Cn4z_p_6fPe{AXWgaOG2unV$2t-P#It zwXl8JQS$1p3OX#%jAhA7u^qMV5!95s_cy4C%*y?7(V@5UJOR7g2IJH)f_C$jfY4WH zOity0ZikrqUbj2MT;4o)Mi%81RhPG2h?u2W@;A!3QN?TyBR@zLzH2(0?RAcEmdX^I z^yFn*E^VPC07=MVzAY##0fxystJmwHcLI63;YhMKogQmxuxzglMY#O5Jf&y$`3Mhe zU-_b`NP(^x=>%5U8CCr5w|Oz5M72T-y0^yNgWjcDLlKJwS_l*WX=5bQXa8@}N>|V! zXS4Y&oH!cZ+uC#k&Jzjk?_TXZn*j$>55W&&g?sWOiN|kDypoKkwQ-z{ zb9ezT#`07hsF5=kDDb%mU($Nn-28VI(~i}j>w02()V#cXoCf(Nf>+kM)7n+nlRd~s ztq6cdGM)e$hXAu!i#C6nwiSW>&t8eoj<*Eh1VEK3s{R?-%$hMeBt9H17t;PV!Vrq; zH(Q6_bH+Cn2dI3#!Nsv&$YyfF>E}NG9Ug^gTUL{$a*W-5&-tve5ES*_Y`|4?@iO5u zUPLYa(R!l++BBn=IEPH+J&8UT0j-QNiU<(@vDB$R^c>Oud#&Qlcsb3^&21Cl@qiK?D0}*HDaEL|38+_I;_d}{of!8N~$y?1Qk%EB_uXN zK$Mj9jWh^KjBYkkx?w z`pc9(>U!`OPf?ETSuoY2Q}3*k*(YLprKC1C7Nih?IeWa&*Y<%mCJ@LF83_W8np!t* z>Wi!fmR#j6(E=Uc#g|er48rCe7*E=EJ`i%&hVvsdd4PZ`Mf^_3WTTD?!cue72HumHptn@Vp0XudAxGS@KT^3DzThT z1{sZc24$1Ey`_qi!thRLDEybfd`9>Ap&0|PQNzJ|_d1hs2a5TUxpNXft*dC|Blz-K zvTON2vNa~a{KTi=o2~{R6PU}KR`=pm)RCJBmj1Z9THUz16oMQ+X}_N0fRqYm{-S}kt=>s?fpP{lJ=lb>o_NCn7VW)DMrnL`agq)dF>J-`wqZaUCV5x4jW5_p zAMZV58e^H8W1Gj=7I>R9wWzk|Oz}a?4JY`BBFw@qjnrrpoVdUFHba6M%I!+#b0c%> zOAcY8P~3HVSpMf3Y8lbif@TUkCYG8Bl5Hdhn8E^_k;tp%__^z5%gmKm?6g)F8LIB8 zHmy}owr$Az-!AIvnvVNQr61+MPPg3KRq@lAc)C9_b7&EN|CbH2w|~RM0=gNUPa_|o zTAhTVB2D%^41rNqxAz+h?dO+H>-ciT06Xdp_L}E}Yf++vEPOHPb`92FqY{NBr#O`mg0}@e_QmVAjFUv(xx!u z|J=S{`!$$uTd@Q^7xfZ(RK<&3A!M?fBVD$*b9T>TUY6>I@goujfW&CY zzVD%p_)FWITT|hKe-x*!6li4`PJMAGR~oxL81l`YPYRd^MIm@#7mDCc*EV!A0S!H# zKPiCOG)r%>`C;dPh{gB@mrVymp~E2`3#Rt02+~Sn&Mac`G1z%6X%4>OSdjRo(d!#G z`#6o~5c5XOu%x0EV@#=F*P%Mvq<_FFXqg5H86_qQ@qYy09_pWhEaO z<2iT2%HO-KpTgq{ZolY#A(WFoFOcdf`}L;N#9ClZPNp;>ki+-6Z+TCcxzPpaIYusd zF=}zVWYjD3(H3=uLxv3*6v9;VgTK+#F?_sG4QTmQvvr$xvOkUDJc;SQwLAwW0;+*S zT>i5!x-?D&3ZI;pYnx7oBmK;%1+{tNa3=-A43>cF2s#3ZVyW;5?0znMInDY5IFUOhR>K?aLu)z}ig$;VFt zzMpW)z6DDDr=?_@Eh3+;^D%L4*#3=CrCOEMyFKVmy%a13b2NV5lDbyp(}hm)cQUh{ zSGhkFwCXB9`ReZr+nHoE!_jS$)3t~WmNGr3Hsm##TDz-Xe^ocP}+)FUYg`C~vwFHq+5()o{^=Xi1icp-#&iq9xdas@24knK;F zIi~9OXOEArp13wYQcV zxFZ^k>=`d9O(1&9;(a8)r1g6rUR|K)Ei1uN9`tu~M#q^_(zs5pTNNilJ|;|&UH>I< z`8MM*<@+0%#J@Yp>XfdO_Z-NOElX#V+O7Wec>f1qA;pxiurP7QWDRtR>4gV#GbP-a zZlZJ0=O7tRjU?E5r8h}8L;Lps*P=c{1@E{ILBq=!2n)j>n5^@UegL=ECP~K5i@Vk4b;7$B5A-}K6`;Ogov+EMRF^~FvGLX? z)9itBvF$tK@OP1t%-s0u&e(gH{uayHd!BbMP0!y1zDhkWBXgCDC6eS`Wqb_XtyZU~a4y+ej^8vP*3HPKJbNXG?DU1YIi5ubbm6maA6X!wSr2vr~_(?-!k< zrIC5yI0|@vg73pl0;{pCP7l$p`tN7~3OB1{e|oNjip@sW`M`OOTo`yZg*EXG&$R}M^^a3~jaD2@@ji+M_ z@V1lmKRmAwT0r|no3WCQrYT$tWaw>_P0V9Tm$mP-toHmso^l#N>>e$51$6GHGL%G}3W`1Bo@=2OpV(!#&;gC7vXpE7f1$};JYbC}o)L6@AE1|6PF5XEE{pH~Ntl{E&yKCZ!WuW`T%O2f#`k;%S2U@Kr4JLDa zZT^%PJD=TeZmH|k{qISqn9mYQJ9qZ$4xF`~@@&+y9T8y-0K3I<`=?dKIvw*~=f+vd zE>XWLg(h!3HBmWmS5F35Su>j79`5K$AbrwNq58gY4NWwb-Vg$oLnwlf!Bjq*6#hC% zSj#-OYp_04YZGKx%E6rn33#iu5PcseG;C*D;V*aGfu@J{xIULXtHVEGf zd6phrG`AFaXEhX*{L+>!{5@wqTRCwESZJ)L=re1QUH;%h1D^|@s+8gBf70xh`%V^? zpPP7HR>yvRV-q2llJnLbErO+TyvCpYNISc}B zSms}Aqr#W_p3sdIPS7b0@cuE4Q0mQ;N(UX7-t6MC!WSMHqbsg8?kAe&oT?}?Z*dSF zEX=Agv77oDK3~JPBIaly7_ z-U$_^MkG4dEI005Z*`K<15(yHaZfw5a0}U6bB+_)g*_*Xd%m8O-HNj{Z%SIfH1(mM z3cfp)RT6d{1udo8n`2Y?-^w%=e@3ZxBXA_=#nE1O2#D!)ylq=OPZLEA%`f|PyTf#K zUsM!BBzSIgdEc~`_wGvH(*|14Wq_5gm-bQlsp2+$-sz?1`%GD{9Ho<=;Gey1a(UxI zWO(NO+}ydSdztfvu9KX5OvM3R;TlE6)bTuh>7y#kQbIM67QEe!a-Xd{s-tzpOe#2x z8~TNfd2ESzdGg}xW5w%hvllvl7Y&Kxx zM{3(ydziVOQ;;kN627tEX-kLW>YDA#7(&nB)p)Yn&%R}NKO*hahY}uUiyiyt2Y5ZH zK@HX+BSJ4bO|@FC)iT%T`(Mt#$$H!FuO3I4qT;VaHop0#ZZL@KcT210Qt5K9x8HpS zE5=!xV-ltr`efPcZ+S2}?U6XqNb~&Q$%pO=?BM*@wn;rul@D1p%x{ecKeP@7Xw}OI zRe3wplL1ORrxE~W#pVZ5)e5wxDqv{-rOuNto{dj^G4%QzCI03Huy7fLLQpHA=h)>0 zm~?pa{x)Z%`u-Vdt9TB}0mWH_%zyJ+Z6gM_^f9@c(y;Q&wE@4o4Sgs79@M#ETsutj zOnrVk2CK0?ZFHAXFTL+hn*<%E886O_d8Bx^EEiK%CmeUgP`oU94=e_rk}&9}HJBUd z?aaH{-@YIaOUgV-9A@gDvW;oC9KJT(9JL9uY)p9MIq(i~e>z2)t4*#m!?Fa9#2yj` z^`6jePn%6XhY&?Z%*aE~?!`U~b(s#ne^JCiEv&5~6GeP*u?a?9_l7~&dxS|lHbvp# zyS&r`MCe$n_?yar0=v+(@bM=_rw_*C&w4YQ$xa*kUWngc^m6}6RbFy+u2rjT2`pM? zjbWVP5v<@d3^`mBvudoMjp_6nL0&n3pqLJ*K8Q9V*?GgNR?KD>{2FkZHNd=G{8pGn zI|Mdo>G;FPcONTA6>+=<5|1NB7QD(mFt&T$N^qD%2xl-_nxZ(?-DqA+V7O@6!&bnh zk6D6pJX8Gm4$+SbPEHR(v_T49g+LG|7!f^*pnql3jB1%<>}f9Lat%P})|h zcZbu!tCnHs&}s~_!oUYk#a1iUtf6Xn_HBO%tMWxXDr`i@*fk-WPtgg|3rIdo({pf} zmzeDSGRMc0&&{a)8#ibd;fhQteV>hE{7(&)BSkYv*jQ{|S*G?~Bv^#j=#iy6;ZoA$ zTtG8W!WF`6$eH!qN$(qPrRnB>`{S~6m>7C)Y(P+`Izj>}q3Sw=*oqH^8EvyHc>S6ytdH7q{w1%#w8MBU&z zub+dQZTnZCB3{Xje7;fp!yzc6C!o#6$N&4e>754qhEdNn|G06(0mbA)QExXfQo_bPcp@b%R6=xsmVQj&V7Hta6C{h$moPOY==tMYNc z+NIWgv!eLeb+PS$zv7TUev*nU3-Uh-+PEEKAMgCz1Y3x*&$jk(_yi+wcp-41Iwf1^ zO4-eFK_oAGE*B}wSmNq1k+~RK>m(qa}D&qLsQIP}C<3Q8nz9qF zV{7oQbx6g&IR(<=BbJNPkKFFX%4K@t@(Q%?*2v)H#alQYy2|0|zxtwQ$qJJ@!Ysit z^Yr3348IMFLy3bWww)+XCfCyj6S~n_myE$~lXD(ge&G^k-62Ky9&*K^G?irRm?}2Q z1sf=Uu}8O@WpSKyH%g&3(i2%YG)VC1*E0`EhnRbLK!o9HA;lF>}x3>>pp)&^L1Yul4J8)I0M#S(EjH*|C(32*A`3c~cjId#nW@U+mQ0?Ub8(b;%H9 zDKCWpZ7-;;b>(38j~=@PYg6{WA#GAy710p0r{j^F)-@4buzhl8CD)WjCJzi}VUJt;w z?#|4WuX;^q02z23UX?DKml2|`sm7$%P!%>8^48?R6#=?$<*(d?zIRQ*PX?IaKW?;2 zVwKfor6i7D2RCkvUL`E|JWug*(jof}cA=Yyb(;@Tr1_%qFa9R(bP!?sqsKgIvheGu zw0-wM;SPgkxxFJ->j*|}@?Qs!&sXa)Tc0v_t(s$<3?8MXpZFcuH}w^@tPg}X9=d)ZyLR4%`qJh9X+;2iiN|> zfrzA>=IcL3KKD)`ksm;2k!|hxCb_2{v zGlrP%#9}~Nq$qTA^NU`E%<>pk^wycaccAfQOgCK_f23OCHb4{n3lHRcqKhY8r zBr0=7L8C6zlvm7J_|oENW6S&bg&5a%Q{L!YR48`!NqSu^jbHhaY$-x~^koPa&=Y$z z(=X8WpAs4a-~LLNISo)}Hp_P~k@0D;GLAQjRxMI%l;f9XARY)3j^H?X*7tn=@y;Ke zkirwK5OE|p-o!~lqc`R-QzUCl&lH<7R}!iOiFqRZtnAIdOQe5XMh(xsnX8s%ODev* z3-fbb%7^B5hgS0H&qvh>|6}6qwMpQw@^nWtUl)5i1Sn z`S#E#^p}txsYxa4a~PoW8w{cNRqf?#ZpeeG`~hL0nv6L0FT+{D$)1*+wig%AhG>Vnw|XBIwn| z|La+9^ZC_NliG|erR8M|jCp+iBHMf$NE~KS?}Vs!W<(?H6<|i&x*r@`>rqu%vmBquETbROEDt^2 zW(e=$@nRHW6yX@H&{g|Zq`jD#PW6!H(#}H-UFWp>&A!n1fw_QatO;N^I0i&FWx9Xi za}ByvSm06EFc8lZRt~(O&7o=lA6@#W4X`D~-9+gco8EyB@maklzz3a*QsGzM1lza2 z0Z(r*DG(ZxnBhz9ovENjIS20Adv#H^n#yPLHmt|OumD|z?DC==!jtSyO_hE zdK*qqv%SHZvR-GaacvUReWyEV|A6dKcz4YC*>=y0z zr!SgG+vC(LaT#%;qi=jHWs7g0Mdy1xg1iqQep7`KXtY)1_)IDI-&s=Mw)wf+2TFLeu_Le~D!C%0WX2;Gjnc(g@s$M)Kz_$y7i z-*sWj6vI`7tFf6C+LUK^RLP<2{FktSmt`FFy1kUUfX&>XY-EGkkEMYG&Ib&TL1mEF z8I8qe2yK#mp^%(VVE>!;^I{Y4GptQJM@Bhxw-~SS(UsEVa)VKc$5M`qt;zeZ5XmFd zZ@H#VLQUi8weSbc*M+rkZs>gPBMGNDaWLb@>37gAW9tI9KTq~MZlL%UNwL9Te)oG!l6wGCbdO+f zVUP=m;gaMaTKv?sFwBJUaGfRuvuw_Yzc}?l>$8TFC}&fxe(9JLnTCAUu~#aPUb*c_ZfG7Am!Ml54zOA zm%?-Bm!#7&xIX6uoTzTJIMxr@u{6IRyf{{RUDGpy(9Ile*HO?5#>rSG4vNKusV3xK zqzR_CA7^1sMzDQ@Lt;7vO6;!`BL1t?L+R^czhv`$={GPv`z!e<82ib7YMhOjT3k3b z3oUe(cw;g(%{f}>e`2Q%W&yylCv9iB^@!hw=ve4QLrrk`r6{`S-D`Md((Zeo9Fqiv zFPqH|(Ee-z+MtJbqnv>193gZD(9J~yV+nGK+3Wnv$fHi?2G>&0LDncKt0Vj+0JySi z_zx3Pbu%N25A_=Y6wbuXn{j60G5dH{pFW|{!!#QQ?3^+I%b{=qf5dF=C*z0?o*3@v zV+Jc1#koBNgsOF{;pY4i?%M6p>se5%*R>$W2?bFlwrTE4I0ZFJ1H0(S1CeU_pOi`l zTuw-HM7luG5uYt-m)0{mw#xG?{;%IN9gDH+SKsm3Ht-0SPXfNM zX(Ym@&(Xzfe*r85xTvx0bfrEHIs7d8(-cj;iS&oH!#6IVU?eL#JynL*pR$O*^`O#( zep413ceYvFaXM|#4Y~-;d38ZAbiB`lJLeHF!5WlPwQaR9=G^$^v|I=0V1uIsJJ7@C z4f2=FRvs*fygkkMVJdxSnacoyPfO8eYKXs^4_lLv9lOi;zqFz8+!O2p7=&#o$k7H~ zO55}J2LrneU5`Y^P5sv^Ehdjw4OR#v<8h6!#&nymOB!5HJg_!mUtgNHGX8^x@uycq z&}UQCfxI-p=YmL%fwzJss*ChwQt5HiJGRh(9$}gO-vXT$yLzn6l9u;YaaUQDkll?s z5Pp$qxEdKi5~&3XcK#w?Pt8VPdfs86X(3gbb6$0y0&m!nS(lv0%Nxu z6e?Sp6dnJ%0vyiYRwb%&{9f`jr))d6yn1kGQbV%>8Q^AQUvNF+^&3cUKc`sOi+hSIwNE!;042yj0v@5#=Os=3tfTH?>^}+BD_jw9 z<|YYQIV2`iiE!J}H#ynWS9Hly^p}mFF>HgBvv-FB5&R{Nq3ke97b0*6^{LGpLPgglUO_rgs zU@(ZCXzz$JXluMM6cZ}M`sqnGTaGH@`e(!D|Gl*tr20j&uX%vhZcecqg`bkpkj1;3 zm(MGErS71?J2&jjf>Q+*MAj@Q(CRt3@6ympwGfPvB`JNMRuOfb7{e^{=_5~%09o1K z!i$Q2&G!&lcwuW()4zeXL!vkf!ZS)^5s2DL;$)tf0TERSPFqYlB^C()_YMx`Rs?jv zPPGkJ99es=73y7M9bda%qFL5zlmVp`5{(UU_JUJQ%Y*5u=aqNaIC@nFH*O~>LOQaQ z9t7qnzF34ON!-Fi7#HPpa=o4X*B@;bQ2Ubzgij(a|5 zI5*Nb`TU5zIPK6`!XYBhC_6!21K ztT|bik?D;27YV%P0`T7&ieXS=V4<%@Fk~s z6Yqs!L1`4|qLfG0Es^^Xh@isT=1{lz^`W*($tbbK+0~>^SRZhF-0)RV3BFJHK%4t} zYwCNqK4-Jm;&JF+7C{jF4=cjhfy{O^H*XF<$hi;=-9CSMoI?`M1eZB+g^M6TC7 z=;RO~tm}s$r1=ZheXl^6uIoblPwVO{-9L>9;b(s^((*GgdnWa3^Dfq9W+oL9@x(hs z$uwl)1bb7XSZ=1+H#=hrUn)Whs{q$?<6)^HU9JTxaTB3#jUyPw|4O=-Em2riYl;KT z7{JwykqufYb$4eURNPw8KiY#k8vLXb-Vn`}I7O44P?ya#ZIln{C9nOxE+Lb5Z}&d< zFMHN0tsk|tPvq}>N%OMd_9st^zVGA2dW=56{fbufvbN+@`MwqVkmman{?kw6^GTR7 zu_IL=kPxzQ$39y=OLu!f-PB5-`I*qoiP=bkbY64_^R$nv198Q%>H|3iF%b`v3PKFOapQLxMr+*4x>z&sIHsu0;^<%rqG*) zFln|ReIt1niYZ@XQl7e5VGwIw*JP|3Y1EeH$r~FQK05P6Z?+`n)v;cHoSW7iGn&uR zrg%-$xT}yDLS8paiNRrQ3OSf6^I1_%3$9!85iFq#Ry#uElFoPTS^md&^KT~)lc~KY zl@=^}dZy^`&S4^o`{e@gG4CejvrRqrs~CQbA;2D5E41pA)Dzt05VmxGzgn!MW%Suq z=?!wN&3^TbRI$K`u(i&e%SFS!2Y9fVJrJRZPsM^5Mt<3<$UHHpxV@v7dCdcbIvla+ zf{Z`06?;}Rpy4=rhGv0hKytW+%bSWo$e=>sED31am)vXKM(6mSpN>!)WTHVOKSLch zq_8b;6^Y;H!s@JT6=Wr!e(-g{x7*KLu>8r8U(E+@et9H4=65n85tf6VEM@>~8~`qJ z9--e6FrH;J4%Oquz3aP9(V+%c*LKbfls~pa@3Br3G^(4Q=I>%`z8{qg^+8D$uHa0q zmw~B$OiEHhonJ)^LeuToD8GjHxo0LoTppUA3-;waP;28A%q-Y=*R~O*oO>*6a+c*T z8GM#gV@*B4ikqPd5dCI`Da!PhXRcCKklq4?qg1|%5(fEW-QU_)UN0Vxg>3{JGHF(e z5-rg%(GYLu5z2_0pJi=-BiE-{IixGiT$gY2zuBzX*k?h7Jqou=8f#o1K>w$7&wyZa zyv?JqnN#ra2_d&{Rl_9p(uz0X(g~ zI_#SOEfOWzRIPSOF`n|r6%C+>DF|YjbD(1=`0%v2AWVJrVS5lG0qOJKf}8)hLx$g^ z6>a?oGy=HrBo_iq{J`X6Mc{c0p?Y)B(+1j9Kk?^?czZ?sjG?KbdKTzW5g1*vpON+i zwV}HAUCa0<_{GT{iHMH2;5!1niVJCGN(T^&XOKeW&FIC-h(ZUAbEnT-A2JDM5!TYm z6ygWG{XrTd`L{D7C>!C8=T zA1;%=(uFECZYCwWFCm!mKO2iboZt6py+vIeR89toS?CXPPTmjX{a*|a};_m!r7Sx;opYMN{QxXI?$7h=LyAVO#3 zMVquGHSPJV;U?*?gdN9mnB`H7aEdJMdE9vl=-EljfPd$Fp?YgEJRMU1dYpT{OGo@H zV5iu#!+x=9v*We=)=k^5n>bI~f0Un}a>h<(->Ba0xP31U{^vgu#&4F}m|_MPb$t8u z0z8I)!JZlIsI+^H=P_YDx8~r|$DKD0chr$msAy4>-1%2Uj<@an=r!1M?!=Ve+Kifo zHq6qn0@)p73Z^Ot`Ms7EE2KAfx(~+>|IVcD$uoe-KAkz8crM&RLN{>`EhVg$-P6`* zlQhGu-{E(wE#LyV?#Emohf*GyN>6z*4QW<%40W$&2F&?kV`19RDX~?~Q z?Fq{oIiAyLe>W+DmyrAoS#ZSqOoaOR3{;N~Z`^5E__%i-MwP%DqhV9L1GY%}s)#l= zbETct+_vTVO58)FO}%(mmSK@{*G4ZJ-F+$T958r0N;df^;VqW#XPB{jUd_Js(`Q&Y zQcemxqBET|Mg2?}HJxeZ>ho;X6}yBW@ufWqsXupKItBc?{*VcL^$Y;s8h@ey00$n` zOv29$8M9x;OtJHh#ov|w#wTXH+G*YX^kg#kEsKtXHuxlmmH6)kXIcI4-&FE?+=O~o zp#4$-&8~ckVEd~lw9fXN?qe4G2q?KsmXAe7k8mgB`nV)z>#MK0aM_tHtxZmvF9yQm z=Qovw!Rn{ZoYXrneaCu24{Ep;H@4F1j8Q7)H>W24J*6QMf0v zcbu3$wj;ZqPO#|hAaM@AWFYLc55j<|SlgpHsJ0A2YkYPp>9r9x^qzoY}WzwOKp>ve9Ib3MeppXIaBs*~E~gQ`c(uC<@5vBx zH7+V9E#>(ZyVJyLU+QaTV!J2xC6JM=ss^{`=(d@|zCf&?W)- zp1GhH!Rs4n-S2fLa;E4szlnR;4xfFmEh@NFb!pY$-@7EVIUD1R*k%4bZx|eDPJ?i zpH`CLH}Mx7WF5P(HlBkbYOQiQZq*Ne`Z0VWcNe@~P8{_BnH-EOWfTuq!19d#n~$3p0g-e!rGzCs+6z73N|w@sw* zk^bUcH#M8g=XlWJ@a|C^0; zAlioF(5IuK@PYF?2*kFpCn%5JxHhM-Z89H%-JbsV)jMn)@M^wWW_zJ{7vLKfAgm#- z>&m)tTnydXUjdOgKRWQLk?^2PADS8V23yUNkDyedaeukJTzwLX*(&A0*lggFx)s<>fO8eBiVzm`Xp$$b;s7|c~=5iSxjUKs5X zRMYCnY{PCPo#aiE6g>VqNdj?)0H#SloE?JN_Izn@gdjHe@bSZsn?9s3yb9OTh{Wi` zuAMnP&z1nkN4PkaDHvGiMXkN}&MK6cHT(T)k1kRsmw~~N&=F^ARCH%%Jk2?GyG8jO zvKXq-Lt2(oD??Zz5mQS{D2woj4?-D#5`RE2N(Zlg{wlxh#!4A29ZdXHDTB9@j9R>s zH`p@I$2>7Wx2idN9Z&)o3xpD(y2rKmuFt3|M zzfE7NAF%ElxtxK4|{s4J0G&y#*aMx!|!3adpVb3D6za2cE5NVFPB~l@7 zTv5OKdg^s5D{+0^qNyb)zSzKf*_r}G{@E~s|HPX&iAsjyZP;K!hwRr-6MLd#+uA_i zK3wOO`Mc_bM!jrNfZrX<$27Ktvp;>8kSrDPusw`d1IL9HcdCvoI;@Av%En8JpQi=! zNHMGk(EHMP+*N?_~Mu=HMeA{5y(LOCM=9cR_L>d zN;b~P@BZ5qgw&CR8ndM09M4ZE$C1z4ZcEM=Q*5BAdx6U(MrDuRdzbf4U=mNs1E!lgpN* zx<-T6P50WOd0VdOI`*ZlKadrh&YgsOD6Z$We%3sm|{4aN(Mt`yJ_G>*&IZ~XYElD3G?U?bbvL949<8j4ejHALtD5ac^T>|eb zf1c)O94KDW_Y0C4h2q_6?z0E__0n7Q2qUJ z?R(N9!;fzFYduDq>3wf`Ze?$QuH~FolD0TdhuK?qpZC<&^!DB6-1fk;%z3H$aZi@) zKaSJV@Fw6#?~+%Z%E}*T)F1v6|AI)mO`w-6=`XNSz=OGG3f#!+P-|b0#oV_Q4PK*5 zh-vE@zeOm@e_vDH+kd zbP|ea97?K(VWuP$`vWwyLAaD>_)3}6l!;(-yjV0^CV%PnZZt(pO!@+I0%lO%o!QZa z;?Fv5E9uVwDf0f153&(8?VlVi#Zg%Z?&yL|ul zlND>cNuiE+h^^1ysox*Jjw3boKCLUBnm1-w5Y#*qcz%}g#fyX9`IvVjd#;igz-&2f z3BKDPhdSK48U*8Q5hZT6JYN7uwfRWw)!BC09mg3lTV5je?cfjRytbzc!Tl3sxGi~&d3 z@_&)8lNDQSz#gw?Bq`s;vLpTah<<{dyQ~K_XA;iZVig>4V-s(nJ4@XMrUEzXDM!$( z9|mbR0Q_1L$4AT>7*fTu`fB>ZSIdt8Kjg7#**|{%nd-qehljSbal3LmY`5@i! z0zlliIHcn5v|iw(d&AJ25tpWw*7-i-VJ{xw=p6$ z$|72!|0&NS2JS!FmW5)D=l^uXFb(vm4i=$JV(@nD43{_)YOpXFm1n6l-BEZ^`}@)O z&Kc@hn81C&bdRu0lRnRFRBUn?9Ha#0PxO8Taxj@(`3_;jPj`Eg3ovsi4j#wrx zYpqG1XgKG`HgIl=7GIyLrJBcvCye`w9Yx#S)^nC~e=VyZ(?M{ezDNUEdLX~uHZl(j zc*Pwyjd2JqX2*D(Uc3zuBgt(qesu!uTphT|yq(zTw(S~lALurs#GmnO)_Pu`Vw_Ku z{l~N|Rfqw>e&qm)La>!1mMdgcgYHtuasZb~aSHQC!}}WuE16tpt|Z#*WKgBBFCKq^Gc0>dEwv2C&imf* zjlQ`+u3q>u=3oXj`=IS@a7%d9OSb1NgS8S3KNU$r-UO?-thga!4Cg$bLkP&NEwom# zJ(~a2*lp_Dz0k#c8rEcA<4oz>YJG$cRBO4X7KQ%yEwRkR{q2x`JhN*}Yo?7h;He$_ z?bPC^JPNW-vv`c_8t|dCnH9#F6qg=8>1f2rL|>G#u?_J5Y7+Q-dzHQ5UZ_3MTCzAc zB}|xGf>7)9KA+^d=_Wr8zxUW-jz%imS%xViZCR&sUT#63J4NPWRq1_1g1gg;Wa=xW zD`Og4nU`isZcK6xEpV}qZQA$-$$j9JcPd8P2OUXqfO%DfyS?K z^E?}3Wv-m^>m?sK#%4w@slG@@moHM1jD+QnT?8WNg!B^__9wD$%Wm=Fb)0RnuQZw% zCdeG;NI?5uS@FfYf+ZCN>LHdJtFbdCbl*zWQ|ti74!=|h7`c8sXN45!J2cXpvpvJ? z(U`nHWEMJUFBNr637nIy`@FNMtano4 zh5$v43J*_k$-l4jH{$GcL>QpgyCBx_%oit+YewKQ>dO1%kbEx9?WNbLw z%rpbc9n5F5nYxOf$|_S7hz@H>vw7$&YbI*JHbP*=lpc{ml*59}rZ#|SWT!eM|C*-S z-=6bO_Y38G0@Aoj?qjafM8~|?iKeB}=a`pqx9rb(O;l9-D?MMiz)8wE@0JM$SwcU3 zKK9AhF+BVW`mARwLwBqY_j&x{dNUXVbnS5CGNwJDq{;fRyC3}Q>IXjhuxrdmOY3cK zPC`SJTS44)FMQ8ghH_Wr;Qr;E&equ*5?3xZA_I^VY1wwtS4Fb*v^kOfHW9Ngq&v#4 z>Uk|0d#>8{rd$>+CX>_faaKhJ`nIWipHEmcU*1Ao`pqZF&=<5j-Tma7=-nCQ;iE!$ z=jXVP9n{2b73(9Pu(iEtNh?7!Zfc?pUPX*Qnc`p+#NGtK6;Q^*1Qm`pAD*-3@G3hK z%i=%q`WXJVV|uKt`a#YB#n9=5@oL|)KiOKW)FV;79_xFa1Uk9;9N*OT0~sSaK!^mw z+K|8`abcYQcpQaqYI?hC{y09Aa`}Z6Vf51#FqO9-PxSb-#-<4Wb7!lFTST;ALoRfZ znW^-dWa(GBHM#%lOF2^oIDjaj`i-mbTRxgesvRanDlty{Btr?HzsMZAwGP{oF7Ugy8nfch-TsR z#nL`iEAr;X*k^`beoC*VqrC5u??4enyQjP_S@A&83SE%kSF=#{-Of=jMNW>NrAbN)+lp zvTTj=$C7?V;(fj!%mKliS<6<1P_G6j+EBh@s>+66K+M+fIZt(1{Z5ljq!o1+=ET9uL8YT5h)@JchsPA=RZPmkYJUoVIn++BM^?VbDY)>g^?N1qq9 zrupF`vXwOTY75mQgMR`BDo$W8leFE)rRWac8QQF9YworjD!WRVdNUQm{tP!Nq@P<) ziT7?NtYDqE&V+K5lfLU1l=0w7v#~WT6V56*^_5`0Gz_wz;l|n>_?Wod#B=G3?o70o zf1YBHZM1`F0NplI4+~QChtf_wV^_j>lmh z@CLj~R2d%bgvNbu;j=LCYF`!~6*0^z!aqW_E<}O;pKE+9ljAxKCS}eM+S7Av{o~8i z+z?DD7cA$vFY+a$4CodS=tT&nCxX7<9@)O}x8}zluXQk4Jcp@yd}0MFSt8)hJ7Vt&r$}3oM`Tc;VafG+_gLv}M^#Kg70+#5;@&-+}+8A?xnrjj_?^EwKLB*%++h!c7s-POP#EqOq~57Dp40BrywzmfeYFD)GkGl%?+f4KJNzGp1s zr)2qi;L1~gr9WU}K1+U89YO=aMpZO=PF~gDcFOa09U_Q3XXigycoyKwSLt37@7bc1 z<*Rn|l%k=Oix=tUt6*){7O2Np6He*%U{sE#?=racWOM#;wzALo(i?i&f+Fz4yD3XT zF1GiQ?@m7v&eq&?_+#IjXIEk zvotXJ;;X_?(X%7U8dj47P?h~=NFEu}MEWySVkJCdC8@3+q2QwO&`P@RpNRN0hW7>K z`zTOHzJ`kaHUxg6F7PoA?WGn{{KaM$d#tArD~jTTT#N@_g*b#R{6^U?Joke-$s-Mc zk}K+DYdwE>e8Iwmk^he%p{7LwAM3nC?cU2LaCV@a%B1tme@qv{fv?JkUMXKNCQg*Z z1WFgf}O;A!$@O^7U$1}vgmp}nu`z!eJg$n(1mN{cjZ7qg5qZBBCVSJc+8f6~$ z%cAp<((F!q+dMf^BI!p>sMt^b%aERqQ~n8tG3=MudCe{;`vpS>5+9X(_4F@%n&(DA z6fz-l`z32nIzYjbgMKg}3yPG9&4SIY#-JLH9wk~4aF$IUgC2VoZWy?MI+`MSKRju* zy8XTI0F(yi+1aoSVU+MM>{Ccgq%o6qN7gc^z zF?2n!hx4{$W-maq3Btj z8Ifl3C&$I^RT!;X@jSoZj8AgZ=MWio3B`My+|ch69}+Q5r`?y5Kc&F}Rd|0)Nnr#M z-dl^b?U$to+iuGFPIu(9wqF-Hhc7pM89_-?ugzq)Bv=6V9i6gpUPCO?WIgUZV)#qL zQc7{yJfr3OBE}Y%nTr@UN6>7%sW*p$o#|ii{5pCa0Bz%j5PSp~CVWf#!C{A~bZ6oc zu-g2T-un5`4eGm@!@F~)dX}-e6YM!)#muSaeT;WgCxx`+g%17EX>*E-RYwQDwx9tk z3$RQg;(0r>qD7@1BkY*+`ih#WL^$_TeOgTSo6SsQQ!w{)mYlONcA%CU96|$7h9+{@ zz{P>}>U1RLb;mVtAm{E&1OT{Jj@UQ~^f|o^Jimn@mxC^yw>t6N9SKKizP2ZJ=U@IF zVNBFXJ3sq9(z;DzodqOx=N%u&qj+*a$;w)b?th!XKZPw6MLzePmC@A|I&EuK z*zf}*kL#T;%|lsl(p$^BK}QA+^lNZ6_s15rx^!Dn z$AGf){igK~nsw&w|JWuVDRa>P_}772bHBaZI1EZtixHQhbDPGu8idBGyL-laZ_3=0 zV#PA_l-*UQucVls-{)9T6G2h2x}O6Jq*Y&DPurqrQ7LD8tG})=PW*PatzesH{p_oT z0MQ~eTw3*JLdicd#E?$4o2wFMVWi>!a=%dWUl&Dh;Y2-utEW_{^2LXp1y>Xzj4rX#GY(9H#X$J+B)jxn ze_;J?Ys>Fg5->z9@4bf}I5hQz@JaAv-^rfCRfj(WYqQ9Z{GpPx%V$sp*9oTueG0yi zPnUY?y{v{8%Kh}FY2WZAyb=+l?Tt2TqH<`$cnicCKhJrB`hRs%TAQ*{nVix(4DzZ? zhddqb*7iC~bOki0YcN~_CVaO})6Igc|7}2`G&(n_oIBXpUTNMDdSdj!hk#guPdvLD z%GpzlBkD`LvKXaXXxk5Oa|!R(-+jqb&s@V1U9+u;osHWe6RAISSZ2BzE4e3V(ry;r z`sj6w_wsFKC=LO`3XRv#aLHa}$6eMWY~$5t%CPAalf;|&HErMbIGP~ah8ZT{!MZUU z9d?2Ww&?mqut)#SnUqrM3$v%=d5c^+>O*)_x4WDaP2lEwl7{Kk4uwaXM+(!S1yhA; zA+7z~MjiGxl>Ml|=kF+u!v{(|7J>Md4C-u*t8ommaH{u-Jcdluy1F<{hY)#V#_fjJ zfnnr+Vagg1Pwe)^xW5T|cVa{OVs> z^FFNWrQR&&{rLGXwwwi@C92uC=eyBOdo>Q@r$h`{23@E4u598?Z|YGo!LAt%td>__ zC}zKZ-s~^os-b`R$bE+*tkLE^;;tT2qm_KGCWv;EeR?7P^oZVOxR(IT%FXK@O2`9! zaHo)VsP?}Tz6uzVY9WMK<@%@vKCpHGPoW)8Q0T*|8Dcf{cH5{~Goy>H|$j_Hj7#hcR_hICxTw|gWYWeJ|#4k>OzWfoMU_pD+% zS(k&50EFLDvLwjj?0=F z$$lhvKSiAjNNF$85XU|Q97)N=WUWTu(u*OU-7(YsM=(%pd~`DiFuZiwBnck z8xgnTr@>SaDM+9zQ7lym`KZhuK0KpzlQ}mZd~}qu>-!T@g1Lhw;ba zw~TgKn(}kjAT3b1`x65_=!YWgmsBR`6VEL!Y&UR}g!vNpA!pk+H?BWtdt8cM+r1g6 zaltp|F~HxrZQ6fq)d{lHsA(`-;yanmC~r6!dPT^87Fwk9M`|EpuwIBxS1J+5QhoG_ z%r9{z6{~3ogwU`o%8aiH?#9JuST{pl@ZOA42iHuoFvK3deu?AyDqbBIL&QZtWZhAjKS_*2vqRE%M^EJOE5S3t-P?)+2s(&W5 zKx&Qog3P zlLzDnEVuU6FbpP~bv(8o@5Ew4^RZ$hA&6nPQr?QVDA{VioY$bdI!N4l*U2lsvSz}; zE+dn(RCNC%S!NI{9E>NrG{2nVyIhD#mwC1KR#VDT_Ohs?gG3@2oiQ|%8q1(6hoFq% zbS$sHb=RF1&x89!c+J|@dZ$)oYRk-YCzBvGSQWha8(F{-U7#Q!KVHl7Dc;}dmjujs zxi|msLaJpJX?yJye>zdM?A?aU8Sw;T|Ng7+sEOhh>y?4)tPv+~@vn(?#e+H82TpRg zm;c1BK^K-iN)wjhfxA=V5N+o=mvsqH^9S0sKMFcgCjYw;27tR7qmOqupai5^bK7y2kOFf$%!l;!mERpL ztK1I508QesrRIBHD94r=T2W9HZN5vx7ovu~7vL2lpB3%7(+ZGIcZ{UD0KIXX4~fWK z%A2yFC7bfPOE}qMtnQrA%2Dz%SdGn-Sm`{7$<+v=Y|kwxSbKQrWt+SLrIl0EtMh}N@>6Z!q((aCGkmJ}}aebYD1 z)xMp9$%ipK3L}zpwMLVS`4;{5hmPgh5^|UBfW;NdIau+Th>slw8&$4cm;}?vo_sbH zD+Yhr&jL-#k5wT(%QEpVG>5W;cA9(x193F_LtcNKPJ;UJ3q#8c_?wq3-Kf^<)@Ll45Gqw9 z{RO9v{-s3jY~BY?v=a?att1TkXF8y&(?&{{T)F#p=VZ~l;OnU#g5=80yvNcp)J8A# z^9a+GM24q$-x6Psj5~lV`4d7STD6G4PUfCmXF9k=%!zXPOYfx4hTr#<{QSs_IXhZZMy9Q3IE4DDymPrN#|g^itO7ej9PR>JK~EgCWNN583`5Na&LnO&Z+$si)%z(ec>vIRE7l zYnmN#DzgBiN99tViGmPOL;;(YkJ}xHZB@j9+2hsVNJdZO`4*G|xeHpl6otf|6C&^8 zXeD82nnlIxng0k5Rf3vSalFY!WLnHhH;96p)=HhN*7^~ON6mY|;?eY$o(aDQ+rICM z9^lk+W;y6Fi*~JkzX-hU*xa2d8p)PM*?meR>9t=hoLA+he*9&j%&w3J`g`%xMyyZE z>dw{b?kuBKuPcMg;v=m-daFaJh{A(7IUT6Md?~x(ni}LWsbW)9acD7K#ZqLngQ9wx z37Oo%rMA(MoC^1xtifD*<2res*ZM6>x6DM}CCDGpHIDrgOJJ%p_&&KPn(GK^CMT@d za;{`-vK?n%IVGqsUsRl*yDu5u-?=T8z#7zAlx1@Jm?_V zHwCg)5Qk3HuKRi1fF9Ts0nL*hMlErs+Rs}x6%je1GfLMt*dK30$OiUgL-Hbziiuy~ zD?*$DhILYb;_^?;p#%)${@8dq?U>%dlG%yLT)rAo9y|fX#|K1GKo=L4Jw?{5>v5p8 zM0dl};Vf&%{`UT{&#y_yU+iione8BaJiJ>j`n;wwm0Uq9G!&t{U<+-&`n*bDv^?v9Lq;HKTj=3%erv76SomgMD-kw(RPv`# zR}Jo|ij1hqiS}(*$mn*QceLNG>7?b5zL!LX z9ZJ9A#q!*m>)Y>E0PE)JQKmfk-YEPC2mk^QM_L{2JGu); zBmFM75w|VI)P=`*`29}~4Y_gN9k70~SITt8Q$mJ6i$Lw|S@ZlVC+qVbKFF&p4xi+M zNO*o%WXCP!bTp0sl+oHD;DydjiGcgwE5UeH^}y->ijpt`@?jcHk?2J;j(9V`c(UO8 zCcB*S?dRMEG?1?8{dsCPzQ>oV#GaC?vJvxjRKLAyoWtYzcdvp->mge28o=UJ6()c^OL%EG4$}Z{L=4W$*0PXc_m(L@AA$xJG0Szgww|*i@v}l>9QP zm^*q9%_~DEP1-B>z(lwN?$nPIR(i}H`7Ndd4ptO1_!;}4&!>8Jp7(Hj#<|kpHX`R3 z=!UP{94lWe{RKj}GKI`rL@n-+r52$aVzaxMC4|x-eh_u(shRIXJL_Oh;odUU^Sq>r z5T>CrTgi;m7zvX0bA@_VyOW7LFMPkuUxjT7b2mkz&5wNEx2X@4VSeJf_?vn&ZM`shFMbbIm}ZkDe~y zJ9J|cgGhogEVd$S<{$)c659Sty{D~{*9o^ntd3bt7`pd_E9d+WgOx2~~x5Rw8K0Zfs5@;sBE2$^Oe)rqwnh3C2oei)K zz&j8D5n1TlCYBc=&~ckZz#R@f{(}QX9Y%^E`~XC~Pt5MV)WSlvCx7lDvBqkjOuvI~ zK~|ixbXfGK_>fcm&#(d5#;x}o=Z?$Bn~~3%?>ApmZxW#o>M*vE{OP~WZPdONl`&-f zLG5}VE??ZUo%u)!k%Rsf@K`-{u9~nh#yEVL%QMD;Py^T*=bjJs5Q1Iip93rk($4ly zrO7k4r=jC42yV5Dj|cI5oTu+*qPU*UMkS_4nfy0u@Ap_T)0%WYR(!anO_lSdvI_Q) z)cd8(n~DBl9`!9X29tZhZ9v|O7IGGJ&6ER&3XfGbH3D^hlmWNB*y13jN=y}==BWQQ zhs2Y|3a#MKY|A3JeJj>zqsY0PWXk~Zw*9#aM`UOZh%4+x04m^HkVAKC! zA+k0*eb~(CO^Hi1FqbHDa)ea5N7=xl8d(QV?9vw|k8Ak|u3Y(!a7H?w(NYwdS@F0N z)Kb3|bbP_~i5kPCwk-UoajHHyp0`TF8%?+!h3QgU9x*>)$3t3H|7yG2e@$dgzm07Cu27g4f>^#~j_+@0 z^?l6Q?g<8&YfPD8lUSdS`2T~tT!;`9L1Pr{Dw{24+g4adj^U^@_CjUS_$%~u#a)YJ zPfeB=)ZSKd5+Kp9>*QQi@V~ARdd;M6Wr>i%Vzd&+`VfVJsmx~7YW%rv*5P{LLgC1Q z&%0{-FM_>8U-lv92k%1;VjU>1; zg)u}-#p`_!I}@I6mG5UlS)mV>&FUo%!Y=S4-Y4zhoGqOCTJ(!AnytB1GQI?(qP8Ks z+tg-(`LmL7M-p~r_dE&3>i3UuouHfw%w+2iC@F6JKW|qif={9KcR9{<7K9D=EN=G% z;&!B{k?z(lm#s!jX{XpF`@MMd17q6Cz>?$q zJn%+(H0R*N_j{s&gwe*U{`r1*2pT7c5`(^qQCt*v*h#b<5X#8iMVURkdm@3GY{etv z>7$_u^CU6{no72g2 zO^~wYs_B5GGt=7Ve1hMxU7;da{>DtEvg$nj@iH61fv#c2R0mb)Eo)^{i#@t=LT{Cc z#9idgG$TH?&j#5W_l5lw2^v%?JR|@LThmS`C~G-SpwDdR(s0B-;DTKqdQsCaH<+S7 zOXAn>T;#V&S>nJc3u9v9xzDDPbL_)%zFtCrO1!q5#-{VnKva4JoAI8~bc!`2*k+^s zyG75cF4qMPlW&&Owr8qw;+qt#n(Cnj`ec_{6n`X7fxH%zGG%_noA7ax6Ogf8_94nl z#dXQkdHl%`n}@nGg;qd+$85N5QCC@M<5_f2q?H=SeB;i z&)%TUCyX&_wdjvi#$o+ubnv}`yfI_#pi`{YIt_ggFruF%-3~it`-}rTItseGZ>PjU zbnZ9(w~{KaG{7yVoT#k%M|p%*lIxi|Y2!BA)cVXR0Zwm8#Opp3Sv!fl5tr(_mu203qiJEi7Y<(r~rx@AoD(5v3K=G*!lC6 z$6E07mKj$zC~wFT{#mRjwr_2$Xy}qlola@p@>t3r@G`ako(|GFtCtjis+H`Kq*pUf z&7->|!uLv77MOl(pDKHv3lK8Ebmc2ilbN98@R#$fo7bO( zZl8%%JZ4W-{0qJEs+%j;zlsbeYBDg!`(?&dVZms!C4rJ>{fYbYR77(QQt9TNe(oKv zMzwKyia^X@U!H+-MfLLuci!z9YdWWC@`n`4JfAT+~y|jeCpnuvAV~tVlDV3#hAwCLOOOCuU5W)F&1v(P9L! zdXmVX6B(>X*XeVe!ijnYLTMz9H=Mk`4}u`b+Y;bz2TPyzDUSVduv(hnN8U<&4e_Q2 z4x6U&!_Y;^rqz{#vA+Y{AHB`OEgV8Bc9O2-9a^PD99vB;Dj6vH^PNL>k^4SrO%!Gp zQe>VVUv1&ie@K%?5K9N-KRu?DAHt=-q>O)V8ia8H;ChtSKf6D_zqQ7P!Bk0wbKG4? zdtB>KWlB5$gLwAGQ31nr_QIBc=wgpCELA$@fXJhR4Y3VQl!*(f?rp~8s)FGjnYCwO z5-zqPriN>;SqPt%(wiA&S`orfPfGWa?L?3%wGARsZ3LO*ErrIm%>7`q^Ex~Q_U1A( zT>2-e4Jc*+BJGg(LS)x-!3YPt?$x;d)fA>cuUtT20O+0K00{S(w&Ej?%q zhTbx#YG$Hy08Ui3Q5e#o?BUNf?b_7{YHjn^!Yh5ooAwY@^{DGQ(z#VS#g#Zr0Rvo} zpENwNxG7j%f(^Q&lZH66%@u=0-D*?ftI047Hx5&eeFPLBvd&jlJyEqF6*&sU6Ll^e z(Ol}?`w=Syy!0j(*Kc;kpYlD-)VPi%Zd*L21wB=ukP!#8$&duSU?Fzmsl6j6Pqbn+ z<`rQT%xyT4*}b{cbklhKeFv2hYi})Ir0gJ`j7vD7JiHa6{8hLuro_3m-cN~+9o)Lr zI2hYN|FBFS9oz7DO|g_)-?L__>_ia#FUJ_CzGir|Xe1Mr9&M;_%NmJww37gDH)M&D z*_hn6>0GLsS3-`AmRO57`731+I&M9Q3qXY9zGyAY%TlPO)0%V?ipMIv8~D@zf_^G!)cZou*vPdQJcUYX*PFzFhlTNz zdE(M;CXKu2`V|B{y}h%sXse|Ap0CgQ?tDN`0 z)3?5T8rI43D5V?dz&eR(vE&OgD((o!r?p;^oq0MnL$;7UZQ>m{?8&=MtOXd4=zq5R zon4uC_%n}(pH?8M%mo!*Sy2s2l8&(oAjOYei3P6s&85vw5vAYguCG$iLUIZ#O+s+m zSyf5h+wEvn_Oup~k|`7RL{VB2f2I*uq6&91l_2ByWfaLw<8X#sW+r4Y=R-~*>R=aU zJG%ynHPLW5uiluzGFSg`UoB?)gG&6w+rzGqt|3(FT&g z-+x_tX@eR0G^s3uzC&;TKtZ}2yJwGuBdbu5g^pAb7;eDx=Uq-GI@crUMRy<920TP(2 zuby6%5y%VQNs-f3XiHu5y}|Ki8~&>x5Pd&fMN>%?tboU8Z1XJho_Wd6N z?nOUP#oslKJ+y;f$!zF>PlyAzJwM(JQiIQGbFq^ESCgLI^k*00rA(j4U}Riw4ekq@ zcAGCAxn4dptvi@|8xsp~>hcUhO|1ve3*4L6C>g5%9#+QeR-&jtdHUNr z_aZCeX7c}Snk6v~Pt&|R?{7o*HYes6XG$u&@B3v~^#IGyoCNdfD{89ea75;x$25MI z(-=b#3+Ua_ieirM@zuNa9BRYrKZb4xa*!XYY>g0n;a11+1lJQ1mfQAFRylR1a-vx* zx#v|vUV<7-AftuRGZf8V3vI|QC|+W@zID}AaSm??&(@L|7cb{OcHxqY1kfqg+zs*o zP(5=6($3mc!@77lih^7=W>Avoe}f@11K*e`=n@Q#tja zr_ajkvy+F%cc;+srIdWmb?5vwC5^01kjgTZ?Vq>c8c&I;($P}@WMhZ_Bq*68_5;>=LDGvv#0I}$do<+YiU zQ;-OV?YiJc^PfJgJlNn^P<(XTydS)k@;%L_58P|B2}Gv*cbrX{HM%2UP#1r}f`=|<2ds)T2_mprm|L@K(B!`x1Zzs)Z;_>)gIA9k_&`t<>#C0;q)_S zm{d+sUkK8SSr>kI;S%ffYcHo`R@(RPadX=*{|sf%XP>3DufJ6!Jy=q@M-IN+m~O>( z-0bQ>VWdCzzQ;-tJ__VarbX}zSy1hk?ce{PVC}7+G7!3QjVOdri(~(MU}UQd9a(u_ zyli77@|Cln&WofyOba18_PDIR;eINak`x9@eB7wliuJ!Mq?PoN@)N5^a+Jf2_VHZ5 z^GyqjHy?kxiaovvZ6pkFIn+ouvOOWHntWWyq-)tfV?QFi8tvUUQl}BS<~0mnQjTj@ zA|ej+_+?a}kF$Qn55pM@95Bi9z?WKKzddREeqlnfTVSv=UzEB^#;PbiC|sDdu+R3g zyqeIcR%xii@HIu~MIZT`e!$7;r``@aN5@z(#F0jww7|AxK=XI<)>DKi_)Jh6lfd02ad@0|-naZ)Mbz2MnTIc_-&Y`)S zZOjdHx~m-K1A7gDCA)te3Yz|eUYEhj&2L63MS@HTlU8ArZUs4B8P3#BC*+o4v+^n8 z!=A5T#-;_kN&Z`hrX$a zHU7>7O)K^1pV~->K@zL`y&_U30>{yQuDDft?uOmXKS|9)H0YQ6q~~YG3ii%k>6KRj z#|*TaKP)qh^nWfSg?)>cm|5ZlzO_aLd{E!VeIPQhGgiz(U4SFj>;|a5^5*%in);o% z5Gvn;!%PJ%F!?oRXv`{MK8ouB|Fs(l1il~W7dxe)c_;iaZ>wv!b&HGb)CT^;WanRZ z{R(hesxkd2B>D^Oz(5a;f*p?3n*cZOj=q5X+eGgi4U>!qvZEZ`&*Hjs9l958zr;`e z5RLue&9?ORzKn0y+h)Q2+BMM>T#ePN`GuHB+V|j%5HMkrL8g;}Wf$Sg=lTpnL>Nnb zf5+HQYGR!>$QMQFBIvzSk}Ks{#K(y<#4*BkXwJJsdc1Skx>@2w?po;k%gL>t-ecV7 z`wysA01OCfoJfWfw{#RNc^@yhiZV*S=IEN0*LpR@Br0tAqA~PK#p#eqhr~ZlaL>|*U&8$tcC`eYJ&dK@qjq}r<%iBx)xu)#B&X|6t zxMg422)9KxYSOG1t}JD6To@i3=dJKwj;V@x@aBA*cHm<+n8=FSjAK9zqccj4ijv@C z=8HY~q0h853~C}SSHIec4(YKc4DvfQS)^FUaa*^1hIRCu?ZKEN6b$O)7J9i%RxGV| zNxWfKX<=*)#ZB8+KCyd;Cqb7nfYvFU^vDj-rpUs&TU7K+Y zK3eeEO7>gz*|H}FZdIxo`b(N9TweU6?v;++9D3mFe`A+3fZuWzhKL5H2)1F3MDdjk zAWoCgw8^@^PU+}pUSwb5k*)G}Sa)Fq>*pIK6cS@M#Yc=8y_X)w+s@pzeo^V89TM%R z(dIvvnq=W^GOD9y(CgbdO0yk4gH$<~_)!c!2jQC;0I_ zG+^)VdJE)A)a+?xo{7HwteQ70T~5SoR;x*?V%s7hwxoXkHsD-(gQpZR>HCQF@|31aa{}Q<>+C7RynWIowv-^~__vPRHB8Tc80ERg`IotC6umsw zzLD?~fuHk-J)y$gjDxBPi)O${c2cwHNRVd@uISfN)go$v6DBtNyJ#>x`ZpnRRF_t} zEw7c|Ou+S-p%~iChv0Tu5MM<>j=lU%Rq)<)SfZ0^Rwt`WRDZ*JU-QQ{{< z`ncV~a^s{XBUXi!xSyH1QSvGe`gM|8yib8&_o!Z+`b* ziw&Uow?X=D!2vo&V4}fHg3X@jUy`rq(GiaOLNwe2{YbR@ZR;aQ&&&pMM7nM=aBzw1 zNMp%^B(fJ=8rOxc z*Rb*}If;{&X}Ip1DXco;6Tp;d4ZMgR~5@Icpv<) zxCayDQlwbHCQYUtsK?mj`h4XqvHhDscf{Khh@iLDbEdd+YibLxh6^m*YUs51R_r!2 zHErl3N=pa~1!}lKlL9Pnm~qaKSvb|KgYy=3tOt=dl=H1XYtEdJZ}WWHH`{8!iMs@p z@676}Gcf+3d_4f8an>y1Tt4Pa(RxjSb-1nehJ?B4#^}(e-%W}Lm5SsI7KPAmJRv$N z?xFZtl96C+&hW7BEzj7{*p){xP%%pYhaYO$g5wci(YHYogd`J1Loq%iAd!BP5DwH3 ztfl1?TqPSS=9p#&`bi%}>m!+{@8-|3ULJPN10JrOumkx(q8x~(FjHsU5?v@`y2iS}a^mWCMy6N} zrq=4~ja@>VmrJbI#4fa?-`LxfBsd4U!Mz0!(`&uVDHRU{xvsjc2VX!52bb*55T(PX zaV+FZ;Op}<#W!8h3K!`czE0K^nK&2V{#Kv6Y$i^?PbF*FjBSDg`(vTdAFtC_hL?s~ zhV8AV@W|fZ<2}*d;-sHy5hkQW)y9uomyg0&fEL*b)F3fJdzF9wHXzA4LjM&nnX?uf z_*{1r(y%c_!3F7tUw+|qe2wv|t#;s~FQ}R2;S(vVIHvlp)T-}J?p;~|-@lCs>($-@ zjqfIAIWHiEKQL=M(;}|Enp%S_If}Ku6VX(NK}gJd-K>a$l2&QI2$fyD^DPTFS3^5) zqrod%(QR*|i>#ITox|Z7-ZehRt3cogUs+w;$3xG^ul-5djuRd<{2=G59dFEHCs|!x_PtMx`srgFs zGztL$@S=m@L-F+;`g`3`+K_u~5w`j@@$dJuX`pFN8tzhQ|LW_6v)T*=;s{i2O^WR zbGK2UH=Gm2s`0MngnPjXe&*QURc}csQ?H1!(!!IPej-?Ubj9R0?tx=Ww=_5wb9b6< z1FG`g_S#;p)Ac%cMIJ+#Wvvri1`uuT5h|ev(*>|MSmNeb?L*3k7)n@{aP3zzH;!1y%duhqJ@ZQ%pae6cZ zK_M7rXwjC=cR|bUGxukl#vO&f4XEG|Hu&cueOrk)XKt z3Fy>ln_M>rS)pNa5oliXf+tEe+D?Cq(Ok}s2M8ZBfvhGt+xcBn<(ga*cEEvmV8|2N z-pg0(P@d=eLOYJg)?)gsb2-3HKZD8chg_epxmif2b%VQZtZ`jdhkDqbK&Aqiq?J*r zqvDCOl>3$=!;-1%?L%3U`xI3-(djh<&h=4cAX7hVlO?DJ)HuWVDxQlS+*Uh`;E9NTcCzzQ0{~A=huHsGy`W5 zUTP{7_DWARR%82hHBQkAME!`RQ(ELNMv#S?jn7@a&cu~v zF14U4SLrwF$j*;K<%@RQy^V8?#ATKYQ6T3LBy0ytjkbHsmwDUT>%pThsBLVRYBl=f zI%+Ry7d{$VLV8F(tNaZ{mpZ-|`#^brK&u(3 zWk1PDB^^4-sPKlmm=U+~)R6vvRqK;erC z=}XXQNVT=BUyM};it^|Qtd2+@>)h|M5)q#gUJck03OSq7Vz+^%DDH|-Y5Z6{3dJ3IKlJ&>lQ{;pGyWHIfL&R` z0Oj$0{)spP-6n%+3O&3M1(V`%#8b}5D1K`=VD;a7t%uiZ0cTQ{XDDDDzcH$Bwk_`1mXMmy{qI<6!cmsE$?yqW)hvI zvGExlk#-4iHqK5j2$lArOpPOjIj-3DHTRE5|Ch+|C^wYb+fb@@X_DkG1~W0KZFtdO zEzuW~JZNR~%ZO6KVZ{Q)!-SK&IY~N|W{OXGYF=`i;UFHPr{~$yGPCghnoKqPF~hq0JL34&DTTVUKli32NqBG< z_J1s$1zS}8_w}WvOS%O~5u}DrDQS_8p}V^~rMpW&K)QsXhVGV-?ijj<8u-usd!F}j zzSr4%ueCmFvbZ_4=*cRtdq>HYa7}3n&%yYgv-;s)VTV?iR=b?BEr>6_nkcJaG1|Cu z#49JWL@eW3vpbve__^sb4iHl?H*QXMSzr?w&lT_ir+Nl)8LIbsRDC({i>!p^O9hh? z=DIkzOih0^<4oTLleJCO3BXi?_aIzWH|VY5;4DktXP@}8JHne(cP>9!e=cuGm+!cd zqE9=B#E!EKXVG4|+%D&vNjds1$$YRpZ8Ekgl_4H{U^bX3pLieIo9?h?x8=a*u!ul5 zIx==74pM8SqvN>f{r=OGH8oPNPQ6jKZ8*+yu9KM;b-@l*p$CDSlIkr+^hCEpp#&vT$`?A~J_Xb#`(8p4e?R4M!qJWZBO2Ixk+G zLh7I!oLD&dR@I?oh}$O;R*0V>rM&#dvXaSOA{i!J^HxcPPx67?jf%EE^umFME8)JV0X`~rI_>K=W_~8t~mtCYybtr+&Nc!*1 z&Hn;1oh0C^?aoC}KL5Q<0x#h?`TvP4%Y^QWDo@)odQu#m_wW9_wXC}z9jpDKw^olm zn#6H79bw&~((z>TO^G@IhamBsQuC^db#(=H=D*rh=ayV%`sNsyLIpge}jX^ ze~K3`89UA*cd41ri4vAo-DCL5|5$Dj>+h)OzF~5$+0wjKL-J8jB?X1&jk)XZ{6<10B|ieLkKL>63`bvSNnAF8pIug*5r?}Zh<(-*M(Ku2 zThj!-?bS@!ZE(_1^-gc$P~FjvXQwXdoVDc1f;}hJ%|{l`292}7;cN%s8OK%Ina8^T zUUfP-&*oWTA?e+pJZs!KBrjv0%ykf6mitYs9`aiBA+m$Cb{(J3L#M*`TVgFc#7v$a z;l{fGfNpXV4f1kB#eY3QKOe=@_e=HBFST8HZ{Nd3C)`qq^;8#b>!ajhYQE|@Gr1bKAO z)LD9iUpZq^wBuu9cUMh(%a4NRdV%o3cE4X(z~!nGm#%_k{_e5r?Xg`5G|5&~OyF1y zbMXOOi))qVK>P_;e(p(-Ew4mlbG|h~U07&edpBiC%B0CIC6Fmi#W<~alxy7Gf8m7M zBaK@`M0U7|25DiLH~!QXW`FX#IiPF74c+8puD#-Enh5za8N`?v2NE$&J^MjEBq#U; zgX!xK+fv?F(jpy$Kqcky3rGE-I0s+~n!t`$8|E-eX>TGJ9(r?!=iuOf%W(LF8$B;F zvRbLVmcnCZd%FJ#WYf;)Awu`0xbLU&GIW9Ez;0N$hJVAny$~qDI9lrb4nN%X-TJ#f zX?a>9Qu=Jm#Iocj$PU9LPPj>1m~&ogHiGrh1V8KL>GFgMNuna}-(XO=?OpaS{=|O@ zSqcq$M;b}x^+_bbld>jtD)^nYn}eN0i&rq4D&Hu))iyEG?QOtpx|h4pBm4#&`Zhns zGF=j_gd{X`l|MkcFg_{1M@ash-{GdjK44Gc^_XMXFiB#;!`JtFqn*pUeFB-tFlzZD z2v%=zS)q!Z;vfT4joUT4TJijdy7F(?l;gy!jcpnIMmNmbq84!AO{w;(>QhG7Y+mJY+ew_Y3x*6Z`B1|J zmNp`hvEBaHQ=(f5??6ac++(3vx#Mrs?7Gj!`hT0NHC2)yPxmps7w(*jVN^?~Ny4MU z=kIvpcv28z4{O~p^V06(+G1-IV0|ilP$$-Lo&>?I7AN}A*l~#GYvv%X@l?IkKm3NW4YSOC zNgO=q(DV=a8!U$tCerQ7mW*OzvaSoX}pkBNGaFq!B3jQiQ9(uMcoWH+-H zmMcvUCOR{1mCC*3n=wDYvu2(SAbsdKmp}7O4tW&LUs^<$=U5nl=hNM&&gDa_B3XQ& z6zBes8`s20TQj{F(M?otIg&ZdBajiLZ(0B_t`ge6Z^~X&${W{UVjwYQamJVp*1pgC zUBIFJU_wR#VrC$x*;f$Xk-nS_y5)Ei#1}^IuTG1LSTf~$A?!0o3J-Sc3Ja6eBh0-M z&I=c`d?!RvTT&Z$W!mk}yzU9h@C?^U`E36uY3l2a{>4h^@ZyU*5*4)XO@4GoF_!x* zpWvLP#jX%on(7EdOHGp-rpLz34^NB_>N(~t9WuyX@U83-KlQ%1AZE*n=l7Y$V9LYZ z=g98+_eFLZnTb&}Q7I(%c-N-qF5*YKFJ)d@{0M$;US&AC?^izC=IF0E1pll`u`f!~ zk4eYhY(FfxZs=mk%plwb`p*vDFeal8&)F7Y-%$;PKF=PgrzJSPPupjDJAOFe_`M_`!vx9gzxdG5VOoL8Q84n%7(bH#p~LEt83qUI)C4x zu;s~cf2<1S6v7GpNMV+9mE2K)m2y#+zmfE0=zx@;=IcP-7%XD){J|V2z&zRcFjR{6 z-@p~v)^GQmdSAl zEkNTKMVG_uG=q29b?dsrv}=+t%su>w1j6)T5pM7>Pxo#1WOw34*+xN=DvfXymqiNgS~>OcF*FS zANKC|a{qd*(0AIA{?F}y$vN#iObyr-b^C7RMsIS*J-{dG<Ul@HmxZjDvmuB^U zXPyjDXT9vo*Tj_6{LTFz@E>L`Tr@@Y&tJIz?x4G}3T1$I3^DEdCI{Ueep6Dl=N*Uq zTXA?4Xs0G!5`tg2U3FWd7w`hSYSFw)oQ}b?>X>tw6qX~C=xYaCD%;gh6wcCu`^0Ay zg@m&!N6UrtBRcS&g-@y&JSM4v7!qG4sPL$|Lq#7V5v4%wA*Fxeu~4oZ&sY|WOZT<-`v;Q&sFiazb#AT+_5zLjI$~s{ES0LK_Y5D{MSgqW07iuv%mXP6V-{MdY zHt@!)Z^tpJO^&ML3)7l(BGBh)^FN{No43M?uPEjr(Gf~O09XEN#aHi${0a0ikdwbs z0b6U$JP9D8kd?`|^ZX1Qig^bJgBMv(P_5l1|N4Roi6YwnQkWpA5 z`wC3YyA=s5;;*pT`ag%?KBsvUSD``pYbhxF+`u2H)m(ukwoN-PMF&3VNJ|@I$iBU6 zn_U9q!7Lkg%20e;S3$nqW5(8TmS%(mk&<4bcq!hNG&HbYz?yv`@C=SLIcONEZO}{Q zuwc`A;AovIR(4P!)8d{oE2^0NfCKB@i zY#LOzo#r6 zvX7KO&Yq?sdj69~q754W>L$WfGwOy>hQ04*uAu*xO$10d3aX%R>oVIIc$jg&JKlAj z>FMpb(N(5eiK)o4POB3bWRc%p}wMhnc`!lK$n)q}QpWan?Z zp8;`)*f|K*AgGOJm8G|EEB}nmZ4We{x0!|cda<2ves!U7y6!SDnmXOw;IZA&ayM`d z7_82yp_>+MCf)+FZW#e~G=Y8i-(1~bwB(XL3|QW^5F${FrSn%t6oz-7`rH7+&lfIF zbfWw+p_uMQ<2P`PbLY&X&khb>{qz5hvKuKKh~7>M|E-mt3pYRGT+H?II#T8)aqHX$ z6fN&zX|tVVLy!+^e5)^k^-TywRvd;zcNKnz0Z@*scYgoLqaPIMWdtz3^=7d0iVFuM zy`ed{g4JAK#!w|5(el>TS-*nr9KD`;2lAYRTvZFLHolSrG_p@y~ z8zpAUNQ0Ju$r?~#mVFALm!qbVv)D;*I*d`9!PmGTEL>niCTb)DoW+Tevrney16QO( zo`Ct-@(*em$VWo(C-(~RRC;aLfwNM!&78uhksn(`ur6dyU+cknIL=OTgFSRPi6mHw z=kE=nkBjVAHT(7`KGHtvT)kM+?6WZ$)K03x|3Otq+U}UTUI~kAAI#qSXu0#7>07HD zb*r2r>S&)@^;DleAj;j=UVh{+Fy>mie(NHM(?KGRf;~5_s6s?_pnq%JNOfB(`t!V@ zq{xkJSMvjmn~-=^1(-C`I=X?;f$Sp{`X~j)er!Fvbi8vO3=Lsi6%J*57G@$`{||HV z+53BrFxoC1cw@*uXhS)t6y~JL^21EW4O7)aZT&slEB<0z*o}|-@VbsQFFBnQB}~Yp z%9K4}vG8K_ReFi6(l6A%nBTj8q^q~Dk7~r3wEx62U*RS81O{9f76YYE5&M#HjIKVC z@E^tWin|nawCU;gijpb)@_ggk9GMBP*yQ`@-b$-M}hD1jSrPUyyzVg z_$F94sG7pNuU_)!<4Dcr;4j5%M%_OZ`o!jf%z>7*Qq9W733ruJde$@`j*`|^wu$^d zk=%x1Y-eeumGScmCTp@C+#XQI+kwop_?#t?sV(MoUNRI9b6g{yV|cjCfywGMFB>PQ zl~L{F-V#m^6MARpQxY?xWW4Kv<)#`A@-<1Ri+bw(0J$aZ&NpYva_oC z+46-ra%c%8nqxeX@R`k2&ai1l6xk@Hz{8!2VybGS4k-RK+eHHLKabmp;aD&dCFbr- zc|>*rH=*_poLkS)(Q|tTO!S#Er77ftw^|RN@a1<&Aw*o zn_QuSAeah5G4K4}5Zk3Zc@3XsLo*Pn4X|gP=lD9SC7mnSNor0DybU|>(jBg9w^)7K z^!ORai$jE5#VypVNGW7{hIpBZ+EYgneNkKmOu;Sq8K%U1ed1dg-wM^-OBcxJD?~9zO#;x9WQGlx?JWi6yT!!fc~T7j(ons}59iJEB54ZkSf^`Ge`RAt=Bc z%w?&KF1DFxZ7xus&nv=)TLTu*VNouoW6lY?YZy*l!1XZh*@Mspg);}?sFJysxhXq2 zTkfcTkGy`d&+J6pw#|N4*t^L?+P6QS`nV)wChhnxEUJRmJK~juyYId0lQLX2ZtC`J z;we9rOmrvq3=vspZ2Vdkhlc|+_6J#2(p(SRXy&ulspl=$0>ow3sv90OJoAf2ISCWA zUnWjw``Xsl2IyaTHSg!2QWIF{6I_c?08`LWR{9LhI3)I6#WVOuITP7(P(zs9mcjeL zwjqN7?q!#A^872pLe=Ulz-7jS2GbkA)de2pw6BvqKJR{?u5}UfZ(1~$WoM9aCxf|qkOHz{CUI#8C+rDN5HAewDZm}CX370=Zl3$kL0UPlTa_sNc zg8fTeCC|;YTgE6EvUpIGi=4=09O&qvBwFO*@(bFh)0y`D>_-rR-Sl^7S1Cmjf^KkE zE|h@{{wMA8O-?PS<*u}!_gI9B`pSDXe{4?O_bvS5m4iqiqa2isEh|xQw-F_h35b2R;|jF-1QC;q)8Qo+uh}YXDW&*(-N?*#7m~+!Q6#72;o(Lzrmw%2yb37 zcJ);?`->*4Ke=5|+!uu6VhE$%9y~DRgX>fN?0(?4*g3`Xd9r=4`+J!!W7)AST-1u8 zB}tjToP;%H8f4T(aGkx^FC7JmV-$41N~s!|w}botXm2?8s#D2ToaAexIX$tcHK2$P zn$hG?*56`tWXmk(=Q$Gx-8&LCoW%&t?wLKQj0iZ-4*&)6m){9Z(nt64JC~s(fzum& zN!0sz$#r)gQOf{xX&{SNpa=Zwb7l53p{DdTsm8pHLh#ou{#1bb<8qReySMh@-WGY$5DJzxsZ2@S-lE@%1{!O7cSSYrsYOX3=aMBW1Oe<0W-&&1 zQtB2;t_3;f$O!T&|BRYazpSMmN%I|eO6ZV}cG2!b?O zfE)%G-nBaGhy-ap?WIjz+rTlkh%S=ZP@8+IMbtgVjOtuQsr^89sU2oVdpw~7)CNaI zM|(afXDx%4uld7Ksep*Wv_q=7)5w-*n2Of?35fOH+Zh;+D z?ah7OLXw#QM_A>9-r0?8_VuOX`6mR^_Y-7#I)x*p5)-rviVn`JH>6xf#fyA_H+jB8yJhffb|teAKT%Reu$3Pzu3;& zj~#-qEw{mSJ7V|QQP1&PKh7dH$x_0apB#ja&d(k~)CCi+q@nJ~{@smS-$C!IV|}Td z0PQV<=tHhNx&uE*Y@eIh?Gcfx={XOe?g_4i6TST_0kgtI@h|lV4~p! zr~lT?bNTqf2bPDz`=A8@_(kY613|X@)UCe;?e|mqD=Qm2y=)8p0u3ywKktb|p-~dB zjiG$wMBUYvHPd$|R{mUU5am1xF7Q01mfeG%P}CIio-P zq{#Yk5#3M~#j{8s<@MYtciR`6UAsxd;+MQA=$5HU?uN`A)C)(dpQ5?Pb#nXn^!w29 z@31~Hie~>Ssd~YrrVPkhTErQO`mL9u^l?p7yq!#`mA_&F72!2r7U{3+)}`0e>+IjZ zZ*nE-j`h)Qk1o%ZA7uq41#blT=FbUu_~%y`5xuKac;E@#O8I(vM(gT?Sp>8+l09+# zXUv-8%(_d)P`8#>Tg01OQK8XZz3|1`;?{4MS~+@po7#H>1^LwiO1Y}7>cROK1{yay zEisBi255u;t3T9Y4#k|dg|#KmepiTAeJ{5e3PJI7qPMV33Fv%DEjr*73)M$-kSRm? z{-HA|!MwLNoQRl=bmvo{OS8*&od_(I8qjOKpHpup!!T7D`KM?0llYv-4B3gtitja} zABIQSF7Dsa1-nTK^ozk7CGwZ z^nh_=jx+A1a;d>oEjmW>9>8Vh*c-R9eea9_485KGHrjGt)WOwQTZ>`3-~_4=7hyU^ z{o%K%Te;WyWIuR=^mg*6cB?;|E4)Y>;92J3sV5MhL@H#@(Ph{G(?a_`4>v#gw$Z=t zYxcRfYeVC`=vU!fmyO;LF$OftVeY-6<}xX)UaZo*tLzc@X=jzyEJkI8NX9{YD^w8} zHo3_qu#2^U-&1>nL83fKH#^XK1CT{DPeC4cg_-LkAYxJFy4kIuJbx~hszJX6*w zJvn0)JCbS}Svxf$+Un(CdDi+NuzyEsfinBB=6Ru7FT-g#~ zw)*?e`bx)*xXU6T`XV+$uXtF>JHU^aC-u?JF@^ym$t^C$)-0?m&baWiKLif_eV2OQ z&}2Sut9teQ@8Q+_m;7{9;aV|q^j^Z*1K8#`p)+Z&i22}Xm6tKG8iPU{SseBdvMLU% z=Hi*ZtZ`Q|8+hvT=H4JP`I5(_fK#Y`Z-Yd}R1r^3U2oDD<}H;srMBKi_o z*~YRv5VC|F8Xcy*R!I+Mf*kY*phwAKhAn5NAxgoWdaga-jp?8nu!Hy##MNvaJs;zW zcH1g1+j03TzPc85f7#(7UPbL3MF6;aWv+*DZ#{jcq_h1H2^R|t#ah#D+*Q_~x zQuP(vrAc+cBy;guKP{BHv5sdHHYX?<`M!5nHln3Gxe7Ek%XcQo$loC?pX`^v{N;HR zrf?hCGXW}GKEqA9q2F1ep6X^90tSc!Po{I29=M2Z*6EA`M5Wm z?}#x!lzp=4$@-Nqkx22@vx<)KcP6Bi=u1@6B)99|xY`C z>)bWe>}_bn&U9bdTBNots@&8)cq*R+PFum{527DschnxVT8uV?*G+IYGUE=q9_*@= zj#ef`#F7_R(>Rh|HTYglcj>5$9u!MB^;!B(Qk^r$zWfl9mV>)8h)}neL(XOpFLGWj zUcmbhp@03R<0TK10;&ar9rXb-v2GBgsnZx9<-;8;xg8v5cZwy7>Y@XC<0Hfqcc(O< z_uf&RLdy10K$UQWETrw7)>EmlP?);8X){=3(vPTRHJ|y~pEdQ}z`A+g$LIeR$q#J~ zcr*LfcP4nsS}%~{KGwMP4kvB0PEE8y}UP-HF9myP9pX08IvYGbQDd@K79 z?{r7J-(ZTOyTDH1cN%N`gV#Kr%D+Bq*>LcY&FTaXLk(o>%DcWy-~@fZeMK$MatiI! z^Xx}vKc*$j#VbgM#-CEZEpkUo74Qs}b((M5CcI^LDKVD9A8zNH5=7jYS6IZqw~6iI ztZqhMFwg;PaSB*8ySpI{O8NBkjo>&-lb%3`8J>na@wMJU9UI=SZf8fat(p&2@v{Ps z;C2fcxK`)a*;k0G%nF4f3eMDIsfWmM76_1U(g!Y%aTY6XDKQI$`WM_8&h z0Y`Z7l+Q$Gh7=LpqBFHd%tLLcAdDK9s)9z)=qATVJ4XJ=`uz;vul zOe3dlDpX6foDd$!43}=@Dflt4J4EKvxPm5}^Lqr&6hC-i16K%$_pG7aX33DP%-j+9iyGBZ#9*6@jD8qbqnH$`ZMf>nKzqfQIAzSEf1T(ga(8M}GBdU%EdqIv?QT_e zwzb>M84rrfyBa%AlSaG@XYc1noF#of zM7Wi4=R|1Us%%{VcW;P9c4CqHQn@V9C6cXdC2~^KQW-+N0RzbH6Ivrje2Jks{`(td zPHlvmTai^xr-op$lhxx{ft^zisw1{j2(4vSd8~{J9CXm~wpvHFJ@OmwCh^A-P$c{} zS>^~VqC(uX}tOk3)TktsL8 z>2LL)pxOHL(_=gyiSN5OO*ocre8&S4DV$G9>8~ug!n`x#>>9H2D*XtWy^I%p!!zPV z>D(zHkr#ao(FAC%zb#d8{r-pJw^et)Ee+I)wGU;M8O5BF zOc2B!xxU%pzRtq%vv2z=VJrUPGAKDb0&$pX#x~MRCP1EYF73-SuN?P%rm8G9KciDP z^6e*<@TP)_-6NPFaTe`T(_Otpk?}0U#I#%5huIaoHAoOAivt0iGJ;nHH}04|D_^Y$ zCAvybc2tNgd#PKxjfjWD>7GS4A&lsP)ec4n@*!@d9MK@C& z_F3M4@0F*~7`AY3katsBJ*{c|y4c{n^1rM}GAehI+VXh)X~CW1O2COD&9P6`A6S}- z|A*}WIE-`s>G7b88kh_!iQABwwisc$*;v_t-uXKEJ;y=2E}^dML*+%O`OQ|7_W#YPMh95!u&Yz8~-QZ#L-7 zH!P;(%lB5dM2Aq(RZiM+J*QZ{U?7#kto$*U*b|k*vanJTcT)pz(yGs|9)AI}55b?Z z_z@p%ZefiBe>I1iS-ioIHU&<9CwaCG)Q5#>!1F`F2SI$w`i|be74KKPG$F-~#aQ2| z(Wb}#jrqka8EJu^ZBneo+C-zT5k6RI`@X>0^+Q|3&pBV2D$+Cr?|l2>*v9J9!7Jt- zfk%q>X%kT18en`eXZ`5F-iRH8TLuANh(5}(13n+iYLW}n z_->|&%Q4r4ZF#_<9Pwwe6dUSdZyZ?CVAVkgKdDq?ev5bU`t%J*55*KM11FBjhZKJ& z&QxbBB`jNZRO?7lu9DlS*7J900O=QtMmRAw!<*AwW8ot7$;`vBA5Psp14Ug!D>}nF z>raMUYKKh9EE2e^n?6vr#;u;CWkOHkhn~07UpliipUAls4^h_$2W4!1kVh{ZZ~-x6 z=Fq+(No5xFK5&TfS@06TGpmyG6=aq_B%IWxK@p}~W)eT}9Dfr483XeiFCss7jBuhy z_HsPRnl&`z$qsq z&34}n%Th0y^Nmz$XINJ;U6-GFg3D?aHiBv=Y*U`~@gnbVpEuZG15?k^9pXb?8mC?n zMS^k$!RwsdgEg|hm7|YGKW)Q}Khdu18+}AyR*7qVZd7<_YVYW0RlYK^1*GU>mQWZ| z?gCtq4e{2X3{VmI|F~ehT9&{~4w~07Id$LUnFaEnN&~!(dhy13>>ouo+q!Ri=$=AS z7V7pr54&U$JdZHBsC3Xn8wkSmMmN9IG8{CNB23MyJ1HR5(UdE*P({ zMFD(~S3$+y1w8o4FC{UN$4-M-fZw(5srr;wC=ij%j<6^khi8J8geF|YDQB{1mc3Sj zmW#$or6Q2D2)6jV8jHy;u(B+iL?>_PW*hSnsc7%@S9&Tb%*LMF|YYk9j`H4ochK1dWw+X8M?Z){cuRxT6=cG262H7=|os zVRo^|Z@7zfpEHOor>oq1$MkN5wIVlp2XJ5BlSbajKZ>Sn=#+o#%N`*7q%#Sbrh~sw zcgUe&+B<8ySpyHc-ji16n=5`)HAVSU9e#r)Kvo~8Al~1j(V20^hFya_^}tu=j=h5u z&YZNEjJOk`3En8SFA)?j$l07r$+KWGON%8b$$O%uempW0jLM5R*27Rz*>2tEMU&6| zXr>?H^x$^PX_5zR8EdZetwKT9MSMRj1ZYxd0=?N?U38YOR2b76W%vhspl3WEF73}+ z)kh~4u$T5=oU#AMBh&oWNfD$HYaomfDWdHWuEl>{;QSb~cjj=3m23i4i5LDRYVkzi zOO2~@n_0P3)|>q$L$>!=zdBItaVz1h^X#@(VFf-dQ!`twEObesDMrYbd}^J=i&F%; z!ID;ULn_;~1AeF<2JHVq?V=4eG_H{rW8;+dUg7?`i0jwo!g{C$$uaJHE8*kQL)170 zqMYH7ToR);OFDV^rkAqDek!%34Rx=$k2fyeuxjHIyl8*BCqF9vLRKj(SgAO^u5*Xv zqWNdSKJ_dlr8PvWdzs0InuGhIw)0+hNWf^kQiP^iVL{jUF0v_t6Zwc1bdukqB{gEL)l7wxhMO2wlZ=36r5XdhE+4sFnE4+YKHZG zZ>av^F}xu#n|c#TA;RPUiGMFuG1F?G_H)>3Radwjd3iW0B0rRzX_K1%lEtF7{mZ{Qa1^&S=9WrgwGmrN zX69xIrE&=Q5X0{}mE=BlCcVaJU6(k=LHT$aPgP3+ql?wQT4)+?LfMZL6;qe{V_d*+ ze|sb!GEM74w=?z}H);;nvlXzwd&;sGyReM^?bW{=ns(^8Y<_egsQ-L35Rxov|02N| zDMbCwwOjmE;T}#`u%wLN+AxPfm|mM&Ox<=Z3t)DsNTFZ57+gOx$NUD*bBI2yGu4(R zd|4z=!TMcI^CMKC`gPh2bsDw&?Duu&nI7TAU7eL!`26}$k>5&fzSb%8%HO!qDB`AJ z7t|>7olT{EB|9PdQ0#xVR%zUyoOzGaQh~|&#@J`F1@mGQdPfOhB4_kzICW_9vDi89 zsCG~o5Ka1**I!mQ`{LT}u>5}lYJ96(@6>q}VZb2$hVKs4#I9S*cRHgpm${Eq5pk~J z*gQtxc)qx~ZTH1xW%mk;o2QoL3n}Be^9Cnd&iJb?akDT=TJ>r|HHrIQf3E_XGiK-p z-`$4j_Ke8Ia3I_BQq_;dOnt_dM13+3i+sc{nQuuni`81)a-0bkS%1*@5Ugo~$>FVS zJHUdcT%LG6ee2t@!KOzqdU>}7-vAu1ZZ|GDD-?EI0bn$BhMO6Vx9d2B!; zQ7b!#hTu|)UTF}4FoRN(3`CPGIGs$;QhhPMaXLTYJ;tF8{v3EN8Tdg9JZ(G-uedg~ zELsYgZs{&)Gkkn|C6qhrW8#{2?MMtz9s75H*V<8;raKgwIEm08BU zc;znndNlUIzj9x#e+zcx3%*~M%jwdvMHYvx(%s}wKMk4zk=|wvbcVOdV+0Kvgwku zH^g)p>H71iH&AVJ*<91s0^gObDpJS%FDj}JzdQ+Sm5Yp|F&@}Av}o|YIWJfaXGEjdRq6dHdMYVp zjMgt^H8W;9@>ak&{Lk4hEiCweqh0b>elSP0J{s{aWV5X6;d|?);t@a3J-m&A`yH8c z1(GMq@0(gox3^fkhj{lienIha4D@Te~&l*>h=6Y7U5b!KhXLiKqK!ACj+x zNk%JY1}VTPS5I&{U+fj7@yxoDlux@lDJH;GD4Y!hhZ^XEO55}0_g(P_$&{?Nzp|$M zbO2B*m&f}*b#L37Y*A08H%QulLDS&DGrt&llO@4GF`og~W$qqRPQToGR=<3|sKW9S zy+NbFcq$9nQ->KIF#+SZDp}^dU-k$hCHXn8F==4oLPO6EID>`P%xGN=eHmsv5a|P# z;loR?kMN@aqtW56X;DBD*NxJLbNX9>aaW0gV9{$)Ne{piEIsdESwbW}kN0Z0t_yLu=YVY0imp!bu>cKFU%Q=$KlOaGp|`#X zn>sI??<*3HMX4mrFr5%_jAXK3?{>l&K7J^PwDm z8{~?O=JgOE>6a>qsF8s_X-Dwrx!3nlHt;Ul#mQ ztPq~E23kDkRCdFme%dm%&gR>qq$~%U!HoXn{~O71N=!l|Ph!u8AYax#N(>LPAyUoV8bb-v`s4Jc!$v9xfH9 z`W3jYTibcF8MJV$ZEI>E>GfinpK#M-RlEOtPlcf+v4X8m6YS)=Mt4mfGrJnBrj~x; zLbxl6$wVCYtZY1>7G--)HmhW;&&3q%UTF9a%AoS!DDCPs?VB9SeU#5|I{M*fi@**$ znB6d^*qylgUygIW1OIBk>Jbu``saMj#5N|mG~;l8=fba+xGkHCvevclJB6`+d?c#cb01Y#jI zgV@Xc>3k{JJC^d}60k|;N@nwlbVwXf1pFcUpIl$KuJjaxVtGeq!`N2sB#8o;lPfA9(-$guWgXi>n{sM=d2d)XHGO^BbZl8jB-mK28?34uds7+@qxuxt68B|JE$JD?YnNNO38y zlA6x^3LZo!H#I=K$r2S)^b;NYmRGvbCC@py)zb5f>nH(MNf}ZQf4Xp{*n~P~CT{!) z(DyIy-P~(tHZ`5ct|gi88m!!k^;iweO@$U4xxfYdD8Q%dVW$|My1}1G8$g+opIRWWpFqmM-nByf%j=(Uvm>G zoEeEM7b`Q143IPYw<=GiDWLwNUJC%yYDsE8aW+D23{WwqiqPSN746D^nKy|ETG$R6 zi`P$$IBRVOK3QJp4&lyavJT zj#o5;tG)B$a9X;RZH=MEiw~65ud;=*RQc2w|JX4>+|P^N2v0P9@qYanm^L(cb$(*5 zHRi}b!In?%=`;(vX!BD)Zf=qtD7q_eM!MrkeW8|WG)N)B8b!J%`u{k@_D(opkUBzMzI>JO^KjOEC>T^1235&OJ z9VzC|{mOgXN;egU{mn4FPsTdkOnE&@fd-tffm<>=W<<$S*QC<}u9T>jS-l60Nm0iI zDR+Pu{3tcY-dMYahJW>XS&s+qYAF z1?1Q#P} z+w`C2Y36lsM2j&?UmXxBe>_a;{o1%#0B5_=MVd@kKl8nyEPp7ByMFRpMYST?JTSZm z@kjB5j7sl)#lPiJ_HET@;G`G*2V`m~XGGP8mqqv%QYE}=P?S;*v@jRs78-#`{(4!H zp@uE|rw^0#*5WVLM_r_BVPr-RZl7-vsDDif%m~l9{+K`dV<^}k|IAnwl=0tP-?<|yBqcB(BYo<%4V%A#N)UqTs-D`0DgQ3QV zRzQ50&CDB1R-_tDB3U+4X0@7zZ;MuOd98D7e*w3wYJ`09gxle)XaAQMYrC&-Mk{D) z+yQXcarm%1+q%zIct~gGQu}#Rm{8iWcI_>;9bZv%e0OH5GXmskU-a;F zb@?S$3TxPXg5?AXIMPBsH7L! zdw}$N`snvwUAh)li~b6?+7_L8U%s=1fiF-Ham6={2QN0z^AwWQf)bvWwW*aKOx9R59IfRIY zKRDZr-GdYlXns3NR70gy_*CrPyP<@Iae#mgs~z^`MT`z?kBHTLO8a(r0Cs8_HVVs2 z(WK)r+X>yG$XtD!y?KM4pb`{&Y&LRgyzpi0IoBOnH!DvD0c`7ro}u`OM0niy?r@BY zb`EW7pN2?Ge7NxHx?OE~l)y%fwEJ4D<)0XcVP#5rW7IoHhnuMsr}Dhabmsq$1-B9*kx_p_STnh@{mi)PNMT^Ulm8#fGp;knK5s|Znf z1414N7ggWno}hDgjh5x(tu`_5HSs%~^2x%`j)FUYHV-^Ea}&C}dJ|2*!x|^X^SKLs z8jun})4!#PHT8wUIM@BZA^A?nY`J{Y8k3bI{CF=L8sB<1WOozy!c_ZAQgxrQFbzW| zFmc#$4lMNZB<~$%9v;)XgTvi+fjZw;0mpNbi+!PfB!JqA)#UA7ejihg7;k5(2%n1aQtAVRhgbJv&0{-xV0En<9o&n`DL8W%FfEr4 ztPx-CbEHLpUUBK2vt5vO>x&f;oB$_0j{dYh?i zV6ibER5@JdER%qun%|H53+H@nNfI80Qzb=-MP)BL8;^5~;jK>aw+;GlhJN8P za@4XhWbB-f)_XVaVRLdSdNRI*dYqd^6wni=X>5oASgUuQqK^`KKq1=VKys0-Q)eN=6zPMQ^;mzzTQ8ak2FW`tV&9fGA+wHU3P zIEHP*-~or|!of)V!#OR&yf9IvfMIoF1c=$x8m$Q9DCtwmEjWyy{M6RC6nIqZ~?5X zc{rL-u!vl_4)p}$(IS^?`xC#s7>hx>PE`nb&JvY-&KmpACj>`cJSNPl-(pP)IL*_1 zRw+#r3f)`rQ_t5^U-a&W1MlT1Q?~&49FY=6I1|tNkpb6}pTT;-h3J5)JpnXR(fdf6 z*l_Ev-W{O70Lcpx&Lf)ylKTz5`xe18N5#gi6ixsqT4Uj#O)lmYc}oz;?WzvR(7sZ> zRz4d9SnB_nt1eudl@NTBeM8!YV=8x&R>GmOhXe)GvWS2_g!?{faMlRkyh3xuzIP;;Q^Wa%p3n)Y_uU9|ex%ue+oJvhuEsMx+e=aszQ8}QL-QT; z^av+~3Qk*nrl1B#Wvo2z7nYuAl9w%hlqKhaM04kl9Ul!`!maK`vR&SZ_H{#+*Kl31 z3*7XaZeuSIuf2;+`Cv{0XBoSRaHFE|T?^l*&)ltMt9wQ`hmq!)fpbm&P0{bImJfsNcDSiNzlX$_f>aY) z9!ft9IaU`##Tov&Y?}o!V#jA!Xf% zcC&%b=bmEMZf13DF2e73l#Wnb;V(8^HGt~*%(QiI=Fn5g! zkjqF&XKReGymb6UKrGd2!8tcm0jyEZaq-&3l(4jH!nSVKQ3%jv-dq>=Nd9`H=#%L? zY3yn^%SfP>2y~i#y*tqD#kx12zYo`?(FD9@?1)W1p{8Ws6QyTLa(VbQec!$M!XoiKQ>OPLw~JOD zT}sBNW2uf_0zzpm&AN`%oH(3fD=C_LW<1v|uy#Oxams_{H>?*>3S~3=XMA4o>>)$~ z^~hdqXd>xZ(@!A};=16kvN<1oJe2jA5OhiIKTE z=n#6ahIA2|;H1;(LqDd$&N1G-=cI!c|EGqr0dAsiViQ>IL&#RucLa%{=s9#(MR>YQ zQ7iBQ02FSk`j-avYumCpiyV*ipYw0WKS(9LryBbQD_dBDk?pwlZzOj>>$B3|!^QTT zJ`i!rW-gU6@!ki6sT07yBwb7jGj@~V5*mZVi1QTg6*4z%9REa2Rp^7)e-Z}rBmwLC zR!Q!PfpmeuWbx|cHn~&36`iarTwDa>Xo}#7FB30t7%?$*FW?S2okHK> zEx$MuB0s-1!V4Vvg}!! z$40oufh9PhLR3Ag7_Fd5v$GBx-vTgsEev)fZ#lxXSdcIvq0FsF!|WwFD!V*|!HX z(V|MF9!_0A!Nv-b`DMT%_KM6`Z+}mgn>o);I!%{)vW)myzx$R)K0hMo*W~4c(?M5b-O&X|Kzp?{l zqXmf~Cdk2a<@s1zBFFP_Euzc|<{3h%KtV2NV^_36ZCFiov3US=dp+p7ek$m=>~fAp z#&WKZ9@hyOr0N@mx>vHg9M=Vq)JrSOuz5x6zWv%ey#)7D?+Pnopzz^XOHrR%;o-=}v|oZ_8d zHbRUwnD^JvVtbTcTMb%c?ffP; z{7{(bMP=W-)tKGPFHd$_bwupdpBv-tC zE}~}$@j`)bc<4QLz|)8h5U<@euSrDRho$hJk?ObJ@9qvZ@P`%a6Qwh!H)&qWA9X|! zA38fGJbX3w%*qRs?#Ayk)U-`D6c!=|;Jgqm|HSx)0WS0dIj?a@>i2}nP+tCzy6-eo zMlyga0#aWPSh|UBYKN7|QN@&qI$yeR`_&0Zh|20r40R0(3{vC`4h@!p2i-%p4ZfNF3V@43X)FD?tUcqMJEUQJYH%g-`9y^!HSN&n`;Ra$mgjv0l0VbpC1dF7 zCHn9t9L0Kt2V`589i(FKC7)}sM{^z64uR5u&*w>#G6&uE^3yzNxAoyZag`YHSq1vz zb$q~2lasHG806g3Id#_iNQ9tm1^S?W4(MUD!zfrcA@#jik{U={_d1Stxqoka2;{J)b$}hXB=XckqC}ijhrWuyxH*@F7cVV$EXfo znH72RBQGZmtyT6=8P)yEEubr4t;mDsLA22IkkCxqKiaueI+)pWPkb1diXO%CU&W2T zsQ;Uq;(p`OGjXpnHZBQCg^&vahdETn|C4qsVAP*LQahIXSD|d)WIjai1W^53`(vTw zo7F5i{bw}T^3(oz9wWB>XewvL@*_fs=nJ9H^z>M}^=B)wH+o@oj(C?I!NM`sy-o(C zjXJ7@>S+T%AHIeUlYNe>`ZuXU(rq-BVA|zZE-TsA45~434yIbB-U6@CZrbPS>qIOw zvsoGedci;9ioZj78yN7dtG<%3A-;hsNdGXhi-BI+l;!ZRK&wv#zLK!Z(=(Ag^(qK^ zc0u9)e_D#Y1cfiQ;svqT7OouI?mSmsiE4WOSiYn>7D4FffzxOAm-b&w zVH}@WCkEq4v8u(KYv+H2w{0?>5wX=8# zx-YQ~Z@A`nf`jzz_br-Sx0()i-|5~LS(vYR9kAX7T<49Pp0buzVWdXHHC;?6-ooR4 zUopucf@JOd*=6yC&tV;T2h8mokDUbY8UTTX^?{4h*P&jAT=AHgf-s0bTTbu2x-Fc!+0R`49hr7*1Cs z@(b!Q8D_4;H8Gc)id=(|#QdJR0{<)UNF^zGf@+-<*o_bahLK3+gnq(FrK z^DTC4$#YcZTXXSo-}`bJ*u|%&_f`pgt1{OsyK##BEXbXn@6c@+V)dr=Cij^WdV8*w zal0ytFQh>~EVT9BA0>!(1=TigqqkQ62;@&q5e`|Cro)5iIw+eYPJGtpUsBwb4aC)K zluJFcCC01wmsl_;3bqDyO9HdcxrdcaKYvRC_H48%(yD5UOvRt|&OoPr21dtzcc`#P zOqNsJw8oMhS#q2cg4rFtVac!P|BTTvN`^P@lL-;U&@~OaDbDMeM{=;ZG_1_`3A6Js z$2n5kO}-Q^@%Nn;e)kdjp^-)GH+&X$!hFrVj#N$hq9Rne`gn&1973l@wejfM+3-U0^#9{p}&4usypc*s{#{A zV93$!vgYuL>r>2&8W6#3)eERRo2yP+8qU_g{IxVtKiu43?{Rut-}^dqX-Y^wRMl}i7rl^1Z_;y80O%-?!)_-n{t&Q0+)-a zvvQr;W_d-RU$oe@HNjv{pEA=wlqaQVDSkWU?kD(J_5i(>D0Xa|A|aPptuKxbL%O0w zEMO2}I*w(Z6P!eOkDf7`!y`TKrojUqWSnO__GG+gP9A@8rup7;`d9mZKZzTXA2dVM zX_>II0I#ICZ(raLNI3!7Z+9RdJ$$vsmFbNAMW4%nt5|WTv@F7Hf)EO+nzr*F*9im1 zzp$$7xzbgSop!tkxZH37AVIiNv%4w{{!&CQzS-s$EYF2}Wbr4caaEA{c@{!DSKAhQRX|Ze zaW%Fl6N~={suA6|3nUdMg}e@oJ%P9UVme^W=UjUuP{bFHamaTG(D{2ZRB{GqP;_n? zr>U*Ij1_$TYQ$Az@!VBuo8zE;k1yZYrnO;gBGW&MH3D}^_K$!x^RwiyNv)~i+uuha zRheTu?{Lth*<4-yq-hzsMRY@gA7^D$!viN4QkmhnSE?~q<15-*D$G7Gn$N81t`Pep*~u{K}3%HFw=*}1jrZ9E$ZqT>`uf$Hp_-iytC`n3DC zVT*RwGR48$dw9;K;_D6o6c3#0Z>_xe^{iN!mA$I+Bo)Qja$T`Xk8coA9pZ+&>D4}e zX!_68Heve+T|Us^ zl-xcI%-Ok09lcB=`vu*rm}`5k1Uu0IPzM?fvMx97opvVd3?2wn1FqYqd~cMKJiHq% z;>Ab0c3s0J`TApEdDya!Ao#^nUBhn1Zq}~j`hWbY)kRZ2SA1tzIEApsnzN0Rd-29( z#&>?hXjB3E?!PuTs%*FLI=m^aou2vdNxs<}xl~owR9$P9cR$RNfCKgQB^QC^s3_-V zW}V=r0Kz@=a*vMm?2D;k<3A*^Ol$t@e-82x&rJ+Q?Z>hDj`=KxZ|J05HC6LQg|(a) zis4Dk6pAmw@rYKk_s1j{Mb?-)vLskVsLBkmtDF5pJ72wgckzjQ-H5q|r#`tTIIO>y zfn;199iXGlE=Tj9&(whTt9RpWIj(yN2RRRnpD7}4Xm7?Bu$fIC>L*=P0fb1?q6v%V9>jOngIop~;L`Wn z_AJ5SzW2@Xs{VFCy+Jyjx*^<)9R*G-OvOHrj#)mR4YU)RMWKwW)xD&ir0hCv60Q3- zC=)TJw~-c)LvNsCx`8<$NkcAzC{-|lVhG418L3{IZXhb)qL+0c=jIeITrDW<&#r!A zo4OQHq?+~_uR4qc{>(ZIXPctn06DR6PrARdBYk_w-0W@ZeAKR;07%_0s+xk<>3|B| zQZOIJLFQB4A2ylV(d9D@C!D1HIfns^_@e94Ao*G3;=j!1I*@N~*WS==uV|%0h!`5u z18!SO0QlmxnwCegE6}}dB(cUEo+Utup%v6z=TKlTN4toqdw)%cYjah3=jnnQi((xl z@w&C;pmn5Nirn&y>;ul|#33m{3Z}c!q-DNkvi+_Hs&pP|$s{}aEMCGH2GCE7a~6zmS$cRU&41!%T%nzpkyEp$ca#Fr)YKo?{b@up|C z)#F&U*Ud+73+_=-(&v|TsTt=E=VzimVWELLtRZd&+jzyB|n9GSZZ797$}UZ>{pMgW9(SE3HWr^BGLs<<%L z>3|)^n{X*ly%5PxB$N1PCIDxLjNcA zmBy{dY6ug*2G@rw8Ku!hLHCTVq%nbL!w;y-@_`utuykU>hbC>)%dX4MKPpnlw(dH! z{}I>*zTH`?u}Qao{y1B?$5P+MoYxdc@M)}XHVA9K%B$THw~W2G)um&}m+NFbsjc|S zdYI{bhTA$c?1_Ly%+iqTw%6_0q(t!RRif%x1Zfw!4B6~kIBq5c5ymPb9Tl8(WVFYfXQ_tbnQV>jf5*n>D{p9zQ8}3ha1ZFaF!( zO~3XD37xQ0xKPGT`%`d+@Nc@XkU!1mu1U2jBij3-htFGdj)T-cCYv#I-|#rq2DV2Y zV{W_MIh^==HQpXH$7{fvz?9Rn=QF1&Wb7XO3fHA%_Zf1!5HEWcf}1b-oD@iJi-U~o zQ}S8lz$$;KeetW%>P0;p%#mm7xMWtdJ;yO*zG2^M)3h$w2#GRRGTr@PmNbQk;xpa; zbuuHny=qI6Kb8Vu_l98y(ry=G%#9DFDYZx=PLtI3!yHUcVJzk_?TZ^Dzi;g}#V53cs;H z^JzKg4dfiizhEHH4tU-megZppWgn{lVOVR}51ot)r)d2?-3z8}|LsOHcZ|b)@H4;R zXJAwlxve)?R=wCOaDrI&08NOw;`$llzY^%7I0Zhxl614AUPU(|`e>u%eglTme><=K(g7*uOLsSX>;7_sIYSZ#%xBZ=hHW21nr0O#C02T*6sjHO z6jNJH6IzRgE-C#3SJy85n_55Id#SrOL&lAErCn}EzbtpnBk^Ikiqq~+F_}2MFPZ>W zln+@HIe|iU^XE^WUk`Y8d3MJgcIC@*$}UQ>wdM+n$a?VI1=>>owaN-5!zl|p$%W9&dQG-gffQ-7e#KWnY#vYw#WF!#2R*>M>8vuy-Mj?fpM?Pq4p$4crYi%2J$?|k?V@nl!k%=Mfv+xxNiVG!~5fz2U2Mc6QOFQsQ>pz?_&Up>Du zM@Kv^kx40S)rJel@bHKWhkXq376hXUsr>w!ewwAZqY{rYoqE;uw_B6${+;@UdyaM_ zdFtv9cwC5h_?pJqN=0BjL~SCupso0By5Z`EH0 z^Zq``h@Zdyk}?BIVIXi`#YPd@glB(cQj{!-PtlU3`4iHehaMw(J~9hYLC(pz zO=H@?k!iJS-R=R$h&F1!)Q2CD?0}?&pR||DfZOTmIr-acE~{ZI9L|V<;~=!jsLA3% zH4j;o!uGh$4}%-F5lxwwb}7*NPs3$GKIco;{>rwd6D+D> z|9wk^9^f=RvP&;uXp&sGey8?}l23GIt7mw+R zenH@;@?k@3L6OsEvlp$+R05%7FRjR8HK{YL($+Q%Pp@ybPT+A$2BD=c@97?=T*&-z zgt^D8tju;cms$oB^=Lbs1cx_2X-y-u=3Wi?n`UDtl+mPx0&et_;QSY(Zj=T*(wr)q z7{hfFq`4A`MN4x{p7dW-qRY@rY~8f4vqhLl8Nf=I!t^tu-kwcNN75QR;U^S(iV=W?ZH)X>^XfA%kGd%p4?9 z>~DBu2V(;mCFse#&^YG}GI&jfn_|7L(-b2gFsq^$n4j(Jd?-htSp&pB2@rsco^YlX zz&JH5E_sckhoj%caR1 z9OXzx%T&-$RfWWJ=%~&kPo4N@O;^(_tjx(lQW9$najDL$uI=HYL9iTULQ3KN^ZWbw zdJP4}Gm~sT4Q@pq5lSx1@1(KjP%F&pva!?Qt1rgG@wzRXi@Pf5l%b zwEWVhVNkW!ewU+MUDV+uv0N`q9{0Tib$+ByW&OUM4#E!Gfb&B&!H-54H;qqz)JH$e zcRlnlIy8bDMLxeF&nHr0RXgDGe%X#Zw!z2_z7^Lb~(NY!F&n+Il^yeC38k@5vryqjgNMq#9EReMBVT~&-S&sHSJ{?lj9PjL zU&rE1zWssy3G6!#UabG=ECjy3^h#Hrwf|YJlW`YZgS1e(y#g~MnJwv}} zdym%2K(EWhD``h0A}a?Nm_bSnX4o56rOyT76Hs^LAqy0g@l3j$-1t-<`c{?w6y~O> zFq1#}+Yf(*LrweDR^+mCbCkelds!HT!RM*0kq_5pL5)w{g*|!;0v%CNF*|<4@q2S1 z^a_+^zspf4EAD~tB`UQ6?q4nE7J|y@jc~jfl#cWJ)5~s> z%M9UPsC{i8g#CnJg4yr$3YQ=S4`KQHcFw!Jj5{cEgL9!U^`1vhH`1#rkjq#W%tatjJ)DrfP3-M;fYgMG4(Wxirb#uVq#; zfnc7l6}IKyIKqyhEe2;zMTW)f^E3i~Fc_H8nq;s38l1-1_8ewhrSL6?Ss~lh9$gS1 zTvsWbTMO$&TvcMQG!6IL2Uv|<2(Fqc)4u0@G!>_~hbz#Js$jr9Pob98UA_`-gOiPr zqLv;|lJe-`I?LGSuCWhTS*t&5-b?UV8W&&f2Wfq&+8fM00U<}wlsHU-KVc<)h2nF* zCP(Q25GQBKs;^L@^=7Q5o!lV;Gf&2v^Akx|JRVsx-(EDgnSEZkyn@!b3rl%;c}DLm z0-9j7wc@AHV}?AYtKXjtnC!-`SqKz`^Y?&kaIrw0(x)UtVl_-n@!!ZnX$3&R*0WOobR?{i_vZ@G)#PF8L!AKDbG83N`_*;R{aNVjB;bz z$w-p`Vy1VT9s*Vs+Ni;z$zAgR*`O|gwRDWx5_OFcaw%SQyyrNN78hK+*}_jwNMtrh+;h7()B__(o{7B^MBG{pAh?OZ7bV*q{x(iG z9{zDA0qmc_hlS+$A;+$+-T=081vxuqfj<16goUE;pWTuXup^_}6$MN7B4K%DRV#KN zY0qEVoLcg;u=+MO49fgf+E^CRNw_`leKY5`%JF~LNI~>9!-L=8kRs4_wCEUFNsk`9 zJfgMGsTeJTG!@NVN{^=I&bjdK^UpZBAM$-|9%$c_lh`-P!I8LG0EG3EyxS*UrMAls zzctPAPJ;r;rp^+30$L2?g(fOSCrA!94CK4pQLgRen3s))4eMlt4P`2FJ6 zH$c8ygb2!~Fu4R{JfCch#E{39Pe&T4Wxk4wqe($TFN3e=8%<@OfC(Y?f0f=2mySEr7(xg5JFb-~jJ#Qs}!)hB(ggzC_e2(zojKPR2Wr%@jvlZ~;DD4Ik z|D^6E1P4eC!~FYfXmf%-&ttIVcP}hh-xh^^;*dMa(dN+o4Z(R8 z^fZ2W?b^i;)tcINtD8>*A>#4S3Ye_ie=D5D?W-I<7!-YVDtpB)=^A}^>A1hHJMssT_YS%-UbRvuXT}L3;kS+`7 zWiG|{uEPK@8v?Mop6?KvqT+c7&wo`+dSESp^PfUiyw@OGh#mi(AMv@4_Z7kTD%>6X z4NIo!aq7Q*^xZRBjCzPljld5lc&<41dypf7a9TMI(d8 z?#IRfY5)b%!Z*oG;V;;zpf-yb_7$x15(wT^$TZDWid-|H8u)4ra}-SSuOZOqcbnrI zMWEZMeUaAfXZV)?LCOTWTkgbHnNVWkwrL!4sBcB=e1TFFn!P4La_J$;NQY-=0ASB1 z$(Iq1rB^o^E$Da2EI+(o{uh-^%!M|X6>%fE_mcT9O>~C5>80n&@8a|LBWY&l=?n7d zx*0f=0siC0B_5dufcA>7D`d#Ymv&*o-9rnW(XjA1*|6rF2EL=g}*&hajIW z6tzuTz%SKgjy0jqc(X`L--w~c@i{jXwyp)3@9CSSET=huW{-1qA68CI*g@i4;zg0# z3J=H{vyK(jhc{%ZPd886uwISEi*coOVfD`Mz5&Ft#j&8yR4wsfzC+%*T8`@hee92C zDyd3mBx^l?&5b<^IDTN_dgE8Tmp5*rGhtRz`jeKQ_}%yR!DtT=)9V`M&yx!uJwjipzg5%+*jl&#@T4S7uqUA% z=c-;E`{fD%?K_Fnd(`cUhNPngeZMNZn2LB?d|5W#ecPZCoFOekAD^M&HSsgBo!({C zy0zk|A&9gwROx~+Efq6FI~>nEC_hnu7QWH^%0Fq^+k^79P*%N`W-;D6VbNrrM<>j2 zc{h~Ww!7A;qTHL)O*1#~);k|<*0UnRx|#Axn6dORxh~bv%V(v7yeB|o$(0op&3YC% z*(*GThv4v;a_`ztaf_PUB4yz$q*e9iv#0UYD>dJ`6x7j#@ z|KlBp%Fqy3>3qt(2{`8!C_MD&54PiiWrH?uJzCKb;E6INogz|hf$R*dqScr3G%rCL z#SAXf8EM4Xk_q$JA@))+2+#>?LQ-g-f|l`2fMJj0Hd#Ew`Afq&r_?aiGsxFmia!nZ zDB?0c882eRs_BO9(t-jNxlO~BI{|&0z1Q=sqQR~Q$d_Rynh@&PQ4@tvd)#IDUhJ7`dirZE@rWY@yEN8$~-PV1~K1h-8?(G(+2 zihNL}9}cRaQ%w{@g`rDc#4<@)zvq!p(dLSkywDJ!2T!{Veo^Z8Id@?aUgk4T$8TwG zxBqTlh)DnaENLo)gQjL&ksZo*v+}T%A$fuEepJwv4hV)>Fm}@1JtnOY)KWFClKdAZ z6QXy!p|MU2Ps&CUsi}_j-F=_Aa%}>|pZDj~Zsg#qT-Od*8Y#t(oi0^&6Wmfh`R$W9hX~voX`QO_6r2nJjf0omqs4MdgVw4w%BCvXq>?Tf&RmE>Ee}mFUM2A z>cmS<3!oMRu3ul*0^d3RsUX*y^eqF7C{dv_gp(?qE@~MjNnLN&V;xv3vvFoGJ)56+ z4|JaUuZ-Hj*Ixfw@7|y5o+K+rL`|+jATY6;k#$&pFHg3%lPZ=`5SdH$XZ{l{oqb}O=q zx`Uza&KZ$@G4%_d?;M74o~kshHAy5X^=qDUmAygOq|pNuCfD5HqT)t}%xw=sNh>Ua z{?bJmlf(;qw4b$g!(8fJ+zhGRpABufVr-qrPRjq$^;l$q1I68CTrJ z&$bj=8R9{)B-D+x!#8#a>p`0=F~BJ%Dn@u*Qa} z!T1;|zeTp+`tN;@uOjl?f@^x=LD)3OV>0!4SeSlZFWy4m?@!y-q~g^k z^3Qlp$91#UV?m~WtiRi59-HLmKa$WK{f5~?dtRx>4gVZ=E%5D4!L$4%?AhO{+eFl$ zjtLC;@n(kGJ?dXG!1SaeW-JKIl+BdAvxq!WHn1F&{218qtTYX?FMD$a_4jP%ly!g9 z9(=ppS=%Pxt_JCil;jT&Q`)QP-wBOlrTlR;(9Y))__kpgT%km&fbZ_1X7^oLI!SWL zglmE9@8a2i3>EJio|sJ9_Zl}?Fvfo$bpN=jVoE325H9`A01@V?Gd-M0EIdWfe77zX zB*!r1gWvk%=4K-aNPo7I%o%6yO>1~7vEHQEdVf_BAhm(sE0Uu}Zv+jnAE924^_&*8 zw!cpfOvGtLnxHT0pI6+xm56qP1nyKL+HS6Bx?q=}AG`Chsx}czG1ojFn4&B@)#f|$ zD}PEyPp-yvR0}qo`s54Q^G+GkW`)-9wtU74yvR-q!Z#PC1D!I}X&x^`A6L02T)+@(8#ZjDXXQP_6qK*D*{K$hltgt|D8;s2A`p3+1!?8J--jWrrm$ zne}zZblmCG{hE!+KKw2x^DWetXoFB1=r&?cQfg2o*CR!RDWYZU{5&*k(rQ;>cf|ct zv(Nfc;q?56^x@;Z=WVN%c-x?9Gya*t)ZW&LA>ioU8=vExFb%uqZbxU!kA9+SM-gm9`aIMjT{M7sw(=Qp~|8{<<>fhsi3dQ z5AnNdaU|^D1w30>yzh$XKP$@VBXW?jr`ZCEDcKs{pA>m?w(8ud+sDP-WNf` z`bUPi<{16@`-GQcgnH_#RB&3vXtq|Ko&uLX&2Wv{4%e$O+${RR#Bb5io;=bBQhTRO z7a~8a-Aa~Gd6yw~Fewvj0BJ+KUc5}bmJ@c?n+L8jG}z|^p8#WqP`L;4I1Wbd7U48{ z2=8ob2PiPHdMa}TRxsSx{kuLkH^aM*DWLGkg?x&$FCOpTD_9Oso25wKUNoOR ztoVcAWN5wv39yRPb1AMx!Y7!D&J3_gP517l=~CYCN><$OFKq*iE!5t>z#aE5cErRI z)9ETWrR4UFH`CN!lU8fyeq3>ye7&XO7iXo=V22*6o?<;a->*)`KGwg!l6vUlVr5?a zs$-U0_5nuMU}esm`d?h|@C=$VdnUuA7jamxOs8P${T)KC11lb=^LT}`dU(L~4XKWN zkIMKvUhmt0ZKlDLsW?(pvy_nRMa(Q z+7_}9=yE6zx00DYu4)|tQh+qCbI}LsSjyiY`HS?J5K{(jk|%* z49^nx7Fn;AYJ9&R5^omTAX7O?P8O!6T?;q&N7rTpNYRE*({QP?UduykWyJCsVdGe%tXY?2s86+OXfVdVh)7fP0+PZg?2!c zxuY?Xf_Dp8Xb`X30GKNgpy~t zpH~dAg=n0ggQo$9hrd)$^`4v>;#6jLm|@#stQqTxiBQQ@&a~rgN!eT8cE&hEI6rvB zh43eW5o@-G4MmYfVT-;kAf#-Iof&WHBqz5-Nb}8Yq zV>a3gp(!MMW?BGFI;qtHw_}DG0RK(x07gUgAycG$ECIeyP2)?^^5Y|j+P{uG9_aY@ zqIhd=QI=md;IFo~1H70b)>O&^5>o30xAUotsbK#+5>-oZ!vI04#JooFKHcqvGIuo? z9MoiJ@PYf|!RhS$ z-Zej8ZlL6R(L}-Yj_K1eA?BuXm(8Jbs~6Ud|4Ggtir_FYu!+4~mQ|nqj!E&vXW!5Z}GJj}mA9x$n^akH&RAGa5 zJaL>2bsdlP^ia!uW0!N6!GavQDU*BTrbClf{PoFBJ?OT6lMa2&@^F~OS7j+K%?NhN z)mUzvww(rQZ7qgj#sTV6U0d#H{zV?r^uqvp$4Lqbdj8>^S#>y8g{H9GiOIR>2I0h! zu=eP_{3EA0KUJoj5xr9^&*Tm+&usXgj*~`CE zc2;<;219mat`qpargPC)nDz75O$#&jXLL2&GPZHbjhGC4XU)ze!yX|!jrEq`PXJFB z-_lvNFeEXQX6#^uKK{g|n<{sK9YYdO&uCSG9^j<3ear~p`#+W{1?(Sg;rg54ow>Pl%+zb4N8=Q6i zVqz8H;#%-u|2v_$SZ+J9@b~3TtOnjk;hm%RBCWN%l6%=00_yAGPVs(b;qv4kCv4?o zFPee7+{2uP@5IY}W-%WrFA+?^JHd>)=b(!2UupkFXfcsBoF1gKklUTdUT2-!t>e3o z+wf{P{oD*FIG_D#&AoFsW;%p#oOn7b(v{O;EQnvaU$0bw^=t=+wN}-=24!Md!3g4o zrGi(-3uL{@uvFcXlB&cFHLIJ@6v0t-cEKXcEW%b-JwT!&YH;ewrK5YU7W@%VmXKcLocDqh(MLQN=v6nqmgk8XJaW9E)V@g%` z(rw4$x&>CYXHUEM-yu!}{%ekx5HY(QlI<*<_EQG|@9Bq1Va(7*6ug1VOXMM4J7s(K zYj3)_4Jry>sS~{*~-JSHmvs<~q|57C$ z*`F=9d>h-%(l*<(!uqag%I(hJ@#|i#2o!cqGPR^6-mA)*oriK3gWg|#&iv!-tB`%^ zKN>m+xz`C$$Z*R4tyeFOb&*-mW~BCl=^++}%Vs}W?p8paF2h-^^Nd=`C9nE6EZtx7 zm%3{A^CR?wzykvFNwWn}&cbKg6qeRaK4xDQ92&WAk^K`msF|f-p8@-jSwq;OvN{Y1 zAjLf21N&oNo^O>#g0*vPohA3bBItn~80zWkr-g4QUg24o?-oO9KHxR~TZ@)3VAi5( zE%t2sO(96Uv>TxX`jqHF&9kcHRt^43fK`P+-z*DeYc(NDhQs35Rt;ft(Dhv%V*Ta% zA>VM6iB5^wH4lQEx?Qroik$?{Rh||22u`_R%cpha(7V?pHWb!XEoML2baDyx&nP_p z(ZhT@utH-m%hfgo`(x3G?2x}-YlYo_(7mny#;sQC za29!Vyi$C%D`w%?M%%?ec{RouV_9_rGe^LlmuJth+b>^t=ExZAWL3+{aUA+IW15GB zDj92tX;)gZ<+@EQ1*y-h1$}?yh?+yeM}B|zl>4@XS6Dd7nlD3kF-a#`A}1<`-K>0p zsMXJlHgLRy7e}A@?HzWm4uw&)s)At(s&`OS9@(dRHzrhKyUArt$$AkC?a{Ov-J_hn zj=&syJsG@fj7y^z{hURrwhdo&;W`}InDPF5x4^E&| zbUP9;o!lOsK^~)@ROX^hcDpZ9TOwsN7h2-N4=U$&IrGYkj;_tUYAFbgOdV9I@h*83 z$o|4&2q$HX$;QN3#(?f!t7h!Ay+v0Yj_ECXft(C(m^_y0M3#(#3qPOk`Q4c%j`rAWB15 z-yc4}!?_-je1j85T?=q_qwIBdg`{9_7s+S&7rkldtuj=5`-}N|zhRghwbi>o*~0qs zFJrA{CYX+)n>q_6-18BDYkq^iypJsXNRYk%Dfo_Wepz;yHzFZ>C+Ir!p~b*YVj(

<script>alert();</script>

\n"); + } + +} From f21bac526c9210b630c53127a825ae2cad8a5257 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 01:58:12 -0400 Subject: [PATCH 0394/1042] add model title in tts results list --- src/common_queries/list_tts_inference_results.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index f2bf3440f6..1074dfff3c 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -22,6 +22,7 @@ pub struct TtsInferenceRecordForList { pub tts_result_token: String, pub tts_model_token: String, + pub tts_model_title: String, pub raw_inference_text: String, pub maybe_creator_user_token: Option, @@ -114,6 +115,7 @@ impl ListTtsResultsQueryBuilder { TtsInferenceRecordForList { tts_result_token: r.tts_result_token.clone(), tts_model_token: r.tts_model_token.clone(), + tts_model_title: r.tts_model_title.clone(), raw_inference_text: r.raw_inference_text.clone(), maybe_creator_user_token: r.maybe_creator_user_token.clone(), maybe_creator_username: r.maybe_creator_username.clone(), @@ -174,6 +176,7 @@ SELECT tts_results.token as tts_result_token, tts_results.model_token as tts_model_token, + tts_models.title as tts_model_title, tts_results.raw_inference_text as raw_inference_text, users.token as maybe_creator_user_token, @@ -269,6 +272,7 @@ pub struct RawInternalTtsRecord { pub tts_result_token: String, pub tts_model_token: String, + pub tts_model_title: String, pub raw_inference_text: String, pub maybe_creator_user_token : Option, From c416069f697f4395ec8232a9cd06dccf804b18af Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 02:56:24 -0400 Subject: [PATCH 0395/1042] write redis with tts use count (needs connection config) --- Cargo.lock | 62 +++++++++++++++++++ Cargo.toml | 5 +- .../endpoints/tts/enqueue_infer_tts.rs | 21 ++++++- src/main.rs | 9 +++ src/server_state.rs | 3 + src/util/mod.rs | 1 + src/util/redis_keys.rs | 12 ++++ 7 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 src/util/redis_keys.rs diff --git a/Cargo.lock b/Cargo.lock index 9a72fa1aa2..236ed8af02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -886,6 +886,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "combine" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4369b5e4c0cddf64ad8981c0111e7df4f7078f4d6ba98fb31f2e17c4c57b7e" +dependencies = [ + "bytes 1.0.1", + "memchr", +] + [[package]] name = "concurrent-queue" version = "1.2.2" @@ -1145,6 +1155,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + [[package]] name = "easyenv" version = "0.0.5" @@ -2394,6 +2410,27 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r2d2" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +dependencies = [ + "log", + "parking_lot", + "scheduled-thread-pool", +] + +[[package]] +name = "r2d2_redis" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "182473b876b0b93e353682ec58e207dd1cb4a62278bbe0045fe52b86b74363bb" +dependencies = [ + "r2d2", + "redis", +] + [[package]] name = "radium" version = "0.5.3" @@ -2518,6 +2555,21 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "redis" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a32cb439c4e89c1e6415e5b3b23d9d8cc6dc1bf5a8cade19adbd5418de803be" +dependencies = [ + "async-trait", + "combine", + "dtoa", + "itoa", + "percent-encoding 2.1.0", + "sha1", + "url 2.2.2", +] + [[package]] name = "redox_syscall" version = "0.2.8" @@ -2711,6 +2763,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -3128,6 +3189,7 @@ dependencies = [ "magic-crypt", "md-5", "pulldown-cmark", + "r2d2_redis", "rand 0.8.3", "regex", "ring", diff --git a/Cargo.toml b/Cargo.toml index 3b45184ee3..7b91fead08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ uuid = { version = "0.8.2", features = ["v4"] } #tokio = { version = "1.6.0", features = ["macros"] } tokio = { version = "0.2.25", features = ["macros"] } -# Database +# MySql Database # "offline" supports typesafe builds without a live database # "rustls" doesn't appear to work with IP addresses, per: # https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 @@ -64,6 +64,9 @@ tokio = { version = "0.2.25", features = ["macros"] } #sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } +# Redis Database +r2d2_redis = "0.14.0" + # Files data-encoding = "2.3.2" ring = "0.17.0-alpha.10" diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 0ae039be73..86639b3c4a 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -7,20 +7,22 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::util::redis_keys::RedisKeys; +use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; +use r2d2_redis::redis::Commands; use regex::Regex; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::validations::check_for_slurs::contains_slurs; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct InferTtsRequest { @@ -91,6 +93,21 @@ pub async fn infer_tts_handler( // TODO(bt): CHECK DATABASE! let model_token = request.tts_model_token.to_string(); + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + warn!("redis error: {:?}", e); + InferTtsError::ServerError + })?; + + let redis_count_key = RedisKeys::tts_model_usage_count(&model_token); + + redis.incr(&redis_count_key, 1) + .map_err(|e| { + warn!("redis error: {:?}", e); + InferTtsError::ServerError + })?; + let ip_address = get_request_ip(&http_request); let creator_set_visibility = "public".to_string(); diff --git a/src/main.rs b/src/main.rs index f1a7a1225a..40ad895615 100644 --- a/src/main.rs +++ b/src/main.rs @@ -81,6 +81,9 @@ use crate::server_state::{ServerState, EnvConfig}; use crate::shared_constants::{DEFAULT_RUST_LOG, DEFAULT_MYSQL_PASSWORD}; use crate::util::encrypted_sort_id::SortKeyCrypto; use log::{info}; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2; +use r2d2_redis::redis::Commands; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; @@ -149,6 +152,11 @@ async fn main() -> AnyhowResult<()> { mysql_pool: pool.clone(), // NB: Pool is clone/sync/send-safe }; + let redis_manager = RedisConnectionManager::new("redis://localhost")?; + + let redis_pool = r2d2::Pool::builder() + .build(redis_manager)?; + info!("Reading env vars and setting up utils..."); let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); @@ -204,6 +212,7 @@ async fn main() -> AnyhowResult<()> { }, hostname: server_hostname, mysql_pool: pool, + redis_pool, firehose_publisher, cookie_manager, session_checker, diff --git a/src/server_state.rs b/src/server_state.rs index 1968307d7a..73b6658904 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -3,6 +3,7 @@ use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::util::encrypted_sort_id::SortKeyCrypto; +use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; /// State that is injected into every endpoint. @@ -15,6 +16,8 @@ pub struct ServerState { pub mysql_pool: MySqlPool, + pub redis_pool: r2d2::Pool, + pub cookie_manager: CookieManager, pub session_checker: SessionChecker, diff --git a/src/util/mod.rs b/src/util/mod.rs index 6ac0370ea4..46340aa50f 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -7,4 +7,5 @@ pub mod markdown_to_html; pub mod random_crockford_token; pub mod random_prefix_crockford_token; pub mod random_uuid; +pub mod redis_keys; pub mod semi_persistent_cache_dir; \ No newline at end of file diff --git a/src/util/redis_keys.rs b/src/util/redis_keys.rs new file mode 100644 index 0000000000..5cc2ea353c --- /dev/null +++ b/src/util/redis_keys.rs @@ -0,0 +1,12 @@ +/// Centralize all of the Redis key naming and dereferencing. +pub struct RedisKeys {} + +impl RedisKeys { + + /// This is a counter incremented with TTS inference. + /// We can use a job to read MySql and fix counts if they get out of sync. + pub fn tts_model_usage_count(model_token: &str) -> String { + // "model use count" + format!("ttsUseCount:{}", model_token) + } +} From 30dfe649aef5a5cfed333f21b36ea8b7bb88d00a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 03:02:35 -0400 Subject: [PATCH 0396/1042] configure redis with env var --- src/main.rs | 13 ++++++++++--- src/shared_constants.rs | 8 ++++++-- src/tts_download_job.rs | 4 ++-- src/tts_inference_job.rs | 4 ++-- src/w2l_download_job.rs | 4 ++-- src/w2l_inference_job.rs | 4 ++-- 6 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/main.rs b/src/main.rs index 40ad895615..964e449103 100644 --- a/src/main.rs +++ b/src/main.rs @@ -78,7 +78,9 @@ use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_t use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::server_state::{ServerState, EnvConfig}; -use crate::shared_constants::{DEFAULT_RUST_LOG, DEFAULT_MYSQL_PASSWORD}; +use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::encrypted_sort_id::SortKeyCrypto; use log::{info}; use r2d2_redis::RedisConnectionManager; @@ -141,7 +143,12 @@ async fn main() -> AnyhowResult<()> { let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", - DEFAULT_MYSQL_PASSWORD); + DEFAULT_MYSQL_CONNECTION_STRING); + + let redis_connection_string = + easyenv::get_env_string_or_default( + "REDIS_URL", + DEFAULT_REDIS_CONNECTION_STRING); let pool = MySqlPoolOptions::new() .max_connections(5) @@ -152,7 +159,7 @@ async fn main() -> AnyhowResult<()> { mysql_pool: pool.clone(), // NB: Pool is clone/sync/send-safe }; - let redis_manager = RedisConnectionManager::new("redis://localhost")?; + let redis_manager = RedisConnectionManager::new(redis_connection_string)?; let redis_pool = r2d2::Pool::builder() .build(redis_manager)?; diff --git a/src/shared_constants.rs b/src/shared_constants.rs index fdf33c4087..a562d9e317 100644 --- a/src/shared_constants.rs +++ b/src/shared_constants.rs @@ -1,12 +1,16 @@ //! Shared constants that all binaries use. +//! Or at least that deserve extremely high visibility. /// NB: `sqlx::query` is spammy and dumps all queries as "info"-level log lines. /// NB: `hyper::proto::h1::io` is incredibly spammy and logs every chunk of bytes in very large files being downloaded. pub const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn,hyper::proto::h1::io=warn"; //const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; // but sometimes we want to debug -/// The default MySql password for use in development -pub const DEFAULT_MYSQL_PASSWORD: &'static str = "mysql://storyteller:password@localhost/storyteller"; +/// The default Redis connection string for use in development +pub const DEFAULT_REDIS_CONNECTION_STRING: &'static str = "redis://localhost"; + +/// The default MySql connection string for use in development +pub const DEFAULT_MYSQL_CONNECTION_STRING: &'static str = "mysql://storyteller:password@localhost/storyteller"; /// The number of results we return by default for paginated queries. pub const DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE: u16 = 25; diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 4776c6b04f..9de093df57 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -41,7 +41,7 @@ use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_PASSWORD; +use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; @@ -136,7 +136,7 @@ async fn main() -> AnyhowResult<()> { let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", - DEFAULT_MYSQL_PASSWORD); + DEFAULT_MYSQL_CONNECTION_STRING); info!("Connecting to database..."); diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index f0a3ab747f..9d848ed5a2 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -44,7 +44,7 @@ use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_PASSWORD; +use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; @@ -160,7 +160,7 @@ async fn main() -> AnyhowResult<()> { let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", - DEFAULT_MYSQL_PASSWORD); + DEFAULT_MYSQL_CONNECTION_STRING); info!("Connecting to database..."); diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 6d95c6c410..fe56a03218 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -44,7 +44,7 @@ use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_PASSWORD; +use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; @@ -183,7 +183,7 @@ async fn main() -> AnyhowResult<()> { let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", - DEFAULT_MYSQL_PASSWORD); + DEFAULT_MYSQL_CONNECTION_STRING); info!("Connecting to database..."); diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 4bb423073b..082b237b26 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -46,7 +46,7 @@ use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_PASSWORD; +use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; @@ -179,7 +179,7 @@ async fn main() -> AnyhowResult<()> { let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", - DEFAULT_MYSQL_PASSWORD); + DEFAULT_MYSQL_CONNECTION_STRING); info!("Connecting to database..."); From 323bf9ea8d7c9e508e3fb613ce1da4190b6d5f8d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 03:12:50 -0400 Subject: [PATCH 0397/1042] move bucket utils under ./utils so it's with similar stuff --- .../endpoints/tts/enqueue_upload_tts_model.rs | 4 ++-- .../w2l/enqueue_infer_w2l_with_uploads.rs | 9 ++++----- src/main.rs | 3 +-- src/server_state.rs | 2 +- src/tts_download_job.rs | 11 +++++------ src/tts_inference_job.rs | 11 +++++------ src/{ => util}/buckets/bucket_client.rs | 0 src/{ => util}/buckets/bucket_path_unifier.rs | 4 ++-- src/{ => util}/buckets/bucket_paths.rs | 0 src/{ => util}/buckets/mod.rs | 0 src/util/mod.rs | 1 + src/w2l_download_job.rs | 9 ++++----- src/w2l_inference_job.rs | 15 +++++++-------- 13 files changed, 32 insertions(+), 37 deletions(-) rename src/{ => util}/buckets/bucket_client.rs (100%) rename src/{ => util}/buckets/bucket_path_unifier.rs (99%) rename src/{ => util}/buckets/bucket_paths.rs (100%) rename src/{ => util}/buckets/mod.rs (100%) diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index a1ee1d2265..25445fde53 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -7,8 +7,10 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; @@ -19,8 +21,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct UploadTtsModelRequest { diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index a396e216a3..d67c666c3f 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -1,4 +1,3 @@ -//use bytes::{Bytes, BytesMut}; use actix_http::http::header; use actix_multipart::{Multipart, Field}; use actix_web::body::ResponseBody::Body; @@ -8,17 +7,18 @@ use actix_web::http::header::ContentDisposition; use actix_web::web::{Data, Json, BytesMut}; use actix_web::{middleware, web, App, Error, HttpResponse, HttpServer, HttpRequest, Either, ResponseError}; use anyhow::anyhow; -use crate::buckets::bucket_client::BucketClient; -use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::read_multipart_field_bytes::checked_read_multipart_bytes; use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_boolean; use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_text; use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_bytes; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::anyhow_result::AnyhowResult; -use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::util::buckets::bucket_client::BucketClient; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::util::random_uuid::generate_random_uuid; use derive_more::{Display, Error}; use futures::{StreamExt, TryStreamExt}; @@ -28,7 +28,6 @@ use sqlx::error::Error::Database; use std::io::Write; use std::sync::Arc; use uuid::Uuid; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; const BUCKET_AUDIO_FILE_NAME : &'static str = "input_audio_file"; const BUCKET_IMAGE_FILE_NAME: &'static str = "input_image_file"; diff --git a/src/main.rs b/src/main.rs index 964e449103..282bcf4292 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,6 @@ pub const RESERVED_USERNAMES : &'static str = include_str!("../db/reserved_usern pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../db/reserved_usernames_including.txt"); pub const BANNED_SLURS : &'static str = include_str!("../db/banned_slurs.txt"); -pub mod buckets; pub mod common_queries; pub mod database_helpers; pub mod http_server; @@ -32,7 +31,6 @@ use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; -use crate::buckets::bucket_client::BucketClient; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::default_route_404::default_route_404; use crate::http_server::endpoints::events::list_events::list_events_handler; @@ -81,6 +79,7 @@ use crate::server_state::{ServerState, EnvConfig}; use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use crate::shared_constants::DEFAULT_RUST_LOG; +use crate::util::buckets::bucket_client::BucketClient; use crate::util::encrypted_sort_id::SortKeyCrypto; use log::{info}; use r2d2_redis::RedisConnectionManager; diff --git a/src/server_state.rs b/src/server_state.rs index 73b6658904..d62cf9a7c6 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,7 +1,7 @@ -use crate::buckets::bucket_client::BucketClient; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; +use crate::util::buckets::bucket_client::BucketClient; use crate::util::encrypted_sort_id::SortKeyCrypto; use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 9de093df57..5013688834 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -10,7 +10,6 @@ #[macro_use] extern crate serde_derive; -pub mod buckets; pub mod common_env; pub mod common_queries; pub mod database_helpers; @@ -21,9 +20,6 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; -use crate::buckets::bucket_client::BucketClient; -use crate::buckets::bucket_path_unifier::BucketPathUnifier; -use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; @@ -34,15 +30,18 @@ use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_failure; use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::tacotron_model_check_command::TacotronModelCheckCommand; +use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; +use crate::util::buckets::bucket_client::BucketClient; +use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::{check_directory_exists, check_file_exists}; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::File; diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 9d848ed5a2..00d5a9b8f5 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -10,7 +10,6 @@ #[macro_use] extern crate serde_derive; -pub mod buckets; pub mod common_env; pub mod common_queries; pub mod database_helpers; @@ -21,9 +20,6 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; -use crate::buckets::bucket_client::BucketClient; -use crate::buckets::bucket_path_unifier::BucketPathUnifier; -use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_inference_job_queries::TtsInferenceJobRecord; @@ -34,7 +30,12 @@ use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_done; use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_failure; use crate::job_queries::tts_inference_job_queries::query_tts_inference_job_records; use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; +use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; +use crate::util::buckets::bucket_client::BucketClient; +use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; @@ -44,8 +45,6 @@ use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; diff --git a/src/buckets/bucket_client.rs b/src/util/buckets/bucket_client.rs similarity index 100% rename from src/buckets/bucket_client.rs rename to src/util/buckets/bucket_client.rs diff --git a/src/buckets/bucket_path_unifier.rs b/src/util/buckets/bucket_path_unifier.rs similarity index 99% rename from src/buckets/bucket_path_unifier.rs rename to src/util/buckets/bucket_path_unifier.rs index 3fb2c9ec29..0e8fa3b161 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/util/buckets/bucket_path_unifier.rs @@ -1,6 +1,6 @@ -use crate::buckets::bucket_paths::hash_to_bucket_path; -use std::path::PathBuf; use crate::util::anyhow_result::AnyhowResult; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; +use std::path::PathBuf; /// This is designed to make it centrally configurable where /// different types of objects are stored. diff --git a/src/buckets/bucket_paths.rs b/src/util/buckets/bucket_paths.rs similarity index 100% rename from src/buckets/bucket_paths.rs rename to src/util/buckets/bucket_paths.rs diff --git a/src/buckets/mod.rs b/src/util/buckets/mod.rs similarity index 100% rename from src/buckets/mod.rs rename to src/util/buckets/mod.rs diff --git a/src/util/mod.rs b/src/util/mod.rs index 46340aa50f..0617f941a8 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,4 +1,5 @@ pub mod anyhow_result; +pub mod buckets; pub mod email_to_gravatar; pub mod encrypted_sort_id; pub mod filesystem; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index fe56a03218..68af39d48b 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -10,7 +10,6 @@ #[macro_use] extern crate serde_derive; -pub mod buckets; pub mod common_env; pub mod common_queries; pub mod database_helpers; @@ -21,8 +20,6 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; -use crate::buckets::bucket_client::BucketClient; -use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; @@ -36,7 +33,11 @@ use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGenera use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_process_upload_command::Wav2LipPreprocessClient; +use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; +use crate::util::buckets::bucket_client::BucketClient; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; @@ -44,8 +45,6 @@ use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 082b237b26..c67e250471 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -10,7 +10,6 @@ #[macro_use] extern crate serde_derive; -pub mod buckets; pub mod common_env; pub mod common_queries; pub mod database_helpers; @@ -21,10 +20,7 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; -use crate::buckets::bucket_client::BucketClient; -use crate::buckets::bucket_path_unifier::BucketPathUnifier; -use crate::buckets::bucket_paths::hash_to_bucket_path; -use util::hashing::hash_file_sha2::hash_file_sha2; +use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; @@ -38,16 +34,20 @@ use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGenera use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand; +use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; +use crate::util::buckets::bucket_client::BucketClient; +use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; +use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::random_crockford_token::random_crockford_token; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; @@ -57,7 +57,6 @@ use std::process::Command; use std::thread; use std::time::Duration; use tempdir::TempDir; -use crate::common_env::CommonEnv; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; From 3d8da4598f1138a485cfdbaaf7a02580590f6d94 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 03:39:09 -0400 Subject: [PATCH 0398/1042] add model use counts --- .../endpoints/tts/get_tts_model.rs | 2 +- .../endpoints/tts/get_tts_model_use_count.rs | 97 +++++++++++++++++++ src/http_server/endpoints/tts/mod.rs | 1 + src/main.rs | 6 ++ 4 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 src/http_server/endpoints/tts/get_tts_model_use_count.rs diff --git a/src/http_server/endpoints/tts/get_tts_model.rs b/src/http_server/endpoints/tts/get_tts_model.rs index 84de136aae..fffe744405 100644 --- a/src/http_server/endpoints/tts/get_tts_model.rs +++ b/src/http_server/endpoints/tts/get_tts_model.rs @@ -12,6 +12,7 @@ use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -25,7 +26,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/tts/get_tts_model_use_count.rs b/src/http_server/endpoints/tts/get_tts_model_use_count.rs new file mode 100644 index 0000000000..1610c235b4 --- /dev/null +++ b/src/http_server/endpoints/tts/get_tts_model_use_count.rs @@ -0,0 +1,97 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::redis_keys::RedisKeys; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use r2d2_redis::redis::{Commands, RedisError}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetTtsModelUseCountPathInfo { + model_token: String, +} + +#[derive(Serialize)] +pub struct GetTtsModelUseCountSuccessResponse { + pub success: bool, + pub count: Option, +} + +#[derive(Debug, Display)] +pub enum GetTtsModelUseCountError { + ServerError, + NotFoundError, +} + +impl ResponseError for GetTtsModelUseCountError { + fn status_code(&self) -> StatusCode { + match *self { + GetTtsModelUseCountError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + GetTtsModelUseCountError::NotFoundError => StatusCode::NOT_FOUND, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetTtsModelUseCountError::ServerError => "server error".to_string(), + GetTtsModelUseCountError::NotFoundError => "not found".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn get_tts_model_use_count_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + warn!("redis error: {:?}", e); + GetTtsModelUseCountError::ServerError + })?; + + let redis_count_key = RedisKeys::tts_model_usage_count(&path.model_token); + + let count : Option = redis.get(&redis_count_key) + .map_err(|e| { + warn!("redis error: {:?}", e); + GetTtsModelUseCountError::ServerError + })?; + + let response = GetTtsModelUseCountSuccessResponse { + success: true, + count, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetTtsModelUseCountError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/tts/mod.rs b/src/http_server/endpoints/tts/mod.rs index 32a08e8399..b7fbe06278 100644 --- a/src/http_server/endpoints/tts/mod.rs +++ b/src/http_server/endpoints/tts/mod.rs @@ -4,6 +4,7 @@ pub mod enqueue_infer_tts; pub mod enqueue_upload_tts_model; pub mod get_tts_inference_job_status; pub mod get_tts_model; +pub mod get_tts_model_use_count; pub mod get_tts_result; pub mod get_tts_upload_model_job_status; pub mod list_tts_models; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 282bcf4292..c4bedf0156 100644 --- a/src/main.rs +++ b/src/main.rs @@ -88,6 +88,7 @@ use r2d2_redis::redis::Commands; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; +use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_use_count_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -370,6 +371,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_tts_model_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/model/{model_token}/count") + .route(web::get().to(get_tts_model_use_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/model/{model_token}/edit") .route(web::post().to(edit_tts_model_handler)) From 5c766239075dcf042e40af93cc114777a57d4f13 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 03:56:26 -0400 Subject: [PATCH 0399/1042] redis-based counts for w2l --- .../endpoints/w2l/enqueue_infer_w2l.rs | 19 +++- .../w2l/enqueue_infer_w2l_with_uploads.rs | 17 ++++ .../w2l/get_w2l_template_use_count.rs | 97 +++++++++++++++++++ src/http_server/endpoints/w2l/mod.rs | 1 + src/main.rs | 6 ++ src/util/redis_keys.rs | 7 +- 6 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 src/http_server/endpoints/w2l/get_w2l_template_use_count.rs diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index 310c0db496..8f764b44e8 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -7,19 +7,21 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::redis_keys::RedisKeys; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; +use r2d2_redis::redis::Commands; use regex::Regex; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct InferW2lRequest { @@ -99,6 +101,21 @@ pub async fn infer_w2l_handler( }, }; + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + warn!("redis error: {:?}", e); + InferW2lError::ServerError + })?; + + let redis_count_key = RedisKeys::w2l_template_usage_count(&w2l_template_token); + + redis.incr(&redis_count_key, 1) + .map_err(|e| { + warn!("redis error: {:?}", e); + InferW2lError::ServerError + })?; + let ip_address = get_request_ip(&http_request); let creator_set_visibility = "public".to_string(); diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index d67c666c3f..ceb1e29007 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -20,9 +20,11 @@ use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::util::random_uuid::generate_random_uuid; +use crate::util::redis_keys::RedisKeys; use derive_more::{Display, Error}; use futures::{StreamExt, TryStreamExt}; use log::{warn, info}; +use r2d2_redis::redis::Commands; use sqlx::MySqlPool; use sqlx::error::Error::Database; use std::io::Write; @@ -214,6 +216,21 @@ pub async fn enqueue_infer_w2l_with_uploads( return Err(InferW2lWithUploadError::BadInput("Template does not exist".to_string())); } + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + warn!("redis error: {:?}", e); + InferW2lWithUploadError::ServerError + })?; + + let redis_count_key = RedisKeys::w2l_template_usage_count(&template_token); + + redis.incr(&redis_count_key, 1) + .map_err(|e| { + warn!("redis error: {:?}", e); + InferW2lWithUploadError::ServerError + })?; + // ==================== ANALYZE AND UPLOAD AUDIO FILE ==================== // let mut audio_type = "application/octet-stream".to_string(); diff --git a/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs b/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs new file mode 100644 index 0000000000..d5d22188e1 --- /dev/null +++ b/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs @@ -0,0 +1,97 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::redis_keys::RedisKeys; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use r2d2_redis::redis::{Commands, RedisError}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetW2lTemplateUseCountPathInfo { + template_token: String, +} + +#[derive(Serialize)] +pub struct GetW2lTemplateUseCountSuccessResponse { + pub success: bool, + pub count: Option, +} + +#[derive(Debug, Display)] +pub enum GetW2lTemplateUseCountError { + ServerError, + NotFoundError, +} + +impl ResponseError for GetW2lTemplateUseCountError { + fn status_code(&self) -> StatusCode { + match *self { + GetW2lTemplateUseCountError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + GetW2lTemplateUseCountError::NotFoundError => StatusCode::NOT_FOUND, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetW2lTemplateUseCountError::ServerError => "server error".to_string(), + GetW2lTemplateUseCountError::NotFoundError => "not found".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn get_w2l_template_use_count_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + warn!("redis error: {:?}", e); + GetW2lTemplateUseCountError::ServerError + })?; + + let redis_count_key = RedisKeys::w2l_template_usage_count(&path.template_token); + + let count : Option = redis.get(&redis_count_key) + .map_err(|e| { + warn!("redis error: {:?}", e); + GetW2lTemplateUseCountError::ServerError + })?; + + let response = GetW2lTemplateUseCountSuccessResponse { + success: true, + count, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetW2lTemplateUseCountError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index 2d7ed379de..79eeecaa10 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -7,6 +7,7 @@ pub mod enqueue_upload_w2l_template; pub mod get_w2l_inference_job_status; pub mod get_w2l_result; pub mod get_w2l_template; +pub mod get_w2l_template_use_count; pub mod get_w2l_upload_template_job_status; pub mod list_w2l_templates; pub mod set_w2l_template_mod_approval; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index c4bedf0156..8409492c8f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -89,6 +89,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_use_count_handler; +use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_template_use_count_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -425,6 +426,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_w2l_template_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/template/{template_token}/count") + .route(web::get().to(get_w2l_template_use_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/template/{template_token}/edit") .route(web::post().to(edit_w2l_template_handler)) diff --git a/src/util/redis_keys.rs b/src/util/redis_keys.rs index 5cc2ea353c..f93fddf321 100644 --- a/src/util/redis_keys.rs +++ b/src/util/redis_keys.rs @@ -6,7 +6,12 @@ impl RedisKeys { /// This is a counter incremented with TTS inference. /// We can use a job to read MySql and fix counts if they get out of sync. pub fn tts_model_usage_count(model_token: &str) -> String { - // "model use count" format!("ttsUseCount:{}", model_token) } + + /// This is a counter incremented with TTS inference. + /// We can use a job to read MySql and fix counts if they get out of sync. + pub fn w2l_template_usage_count(template_token: &str) -> String { + format!("w2lUseCount:{}", template_token) + } } From 31effd2ea86893d533b6ce836f68e8aa50a81c7a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 06:11:18 -0400 Subject: [PATCH 0400/1042] synthetic ids work (but need finishing and cleanup) --- .../up.sql | 10 + .../2021-06-13-010101_synthetic_ids/down.sql | 6 + .../2021-06-13-010101_synthetic_ids/up.sql | 47 +++++ src/job_queries/tts_inference_job_queries.rs | 175 ++++++++++++++++-- 4 files changed, 225 insertions(+), 13 deletions(-) create mode 100644 migrations/2021-06-13-010101_synthetic_ids/down.sql create mode 100644 migrations/2021-06-13-010101_synthetic_ids/up.sql diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 6a32694575..2ab9571dff 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -50,6 +50,11 @@ CREATE TABLE tts_results ( 'private' ) NOT NULL DEFAULT 'public', + -- The synthetic id associated with this result. + -- These ids are incremented on a per-user basis to help users + -- sequence their own work. They serve no other purpose. + maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL, + -- ========== BUCKET STORAGE ========== -- SHA2. @@ -133,6 +138,11 @@ CREATE TABLE w2l_results ( 'private' ) NOT NULL DEFAULT 'public', + -- The synthetic id associated with this result. + -- These ids are incremented on a per-user basis to help users + -- sequence their own work. They serve no other purpose. + maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL, + -- ========== BUCKET STORAGE ========== -- Where the wav, spectrogram, and etc. are located. diff --git a/migrations/2021-06-13-010101_synthetic_ids/down.sql b/migrations/2021-06-13-010101_synthetic_ids/down.sql new file mode 100644 index 0000000000..74c48f3c51 --- /dev/null +++ b/migrations/2021-06-13-010101_synthetic_ids/down.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +drop table tts_result_synthetic_ids; +drop table w2l_result_synthetic_ids; diff --git a/migrations/2021-06-13-010101_synthetic_ids/up.sql b/migrations/2021-06-13-010101_synthetic_ids/up.sql new file mode 100644 index 0000000000..f96901a2ff --- /dev/null +++ b/migrations/2021-06-13-010101_synthetic_ids/up.sql @@ -0,0 +1,47 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- It gets unwieldy to show results without a numeric sequence. +-- We create a synthetic sequence for each user to make revisiting results easier. +CREATE TABLE tts_result_synthetic_ids ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + user_token VARCHAR(32) NOT NULL, + + -- The next result id. + -- After we use an id, we increment this counter. + next_id BIGINT(20) NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (user_token) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- It gets unwieldy to show results without a numeric sequence. +-- We create a synthetic sequence for each user to make revisiting results easier. +CREATE TABLE w2l_result_synthetic_ids ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + user_token VARCHAR(32) NOT NULL, + + -- The next result id. + -- After we use an id, we increment this counter. + next_id BIGINT(20) NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (user_token) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index b06bb8ca98..65fe50cab5 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -7,7 +7,7 @@ use chrono::{Utc, DateTime}; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn, info}; -use sqlx::MySqlPool; +use sqlx::{MySqlPool, Transaction, MySql}; use std::path::Path; /// table: tts_inference_jobs @@ -241,7 +241,78 @@ pub async fn insert_tts_result>( let normalized_inference_text = job.raw_inference_text.clone(); // TODO - let query_result = sqlx::query!( + let maybe_creator_user_token = job.maybe_creator_user_token.clone(); + let mut maybe_creator_synthetic_id : Option = None; + + let mut transaction = pool.begin().await?; + + if let Some(creator_user_token) = maybe_creator_user_token.as_deref() { + + info!(">>>>> Creating synthetic for {}", creator_user_token); + + //let synthetic_id = { + // let id = get_incremented_synthetic_id(creator_user_token, &mut transaction).await?; + // id + //}; + //maybe_creator_synthetic_id = Some(synthetic_id); + + let query_result = sqlx::query!( + r#" +INSERT INTO tts_result_synthetic_ids +SET + user_token = ?, + next_id = 1 +ON DUPLICATE KEY UPDATE + user_token = ?, + next_id = next_id + 1 + "#, + creator_user_token, + creator_user_token + ) + .execute(&mut transaction) + .await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Transaction failure: {:?}", err); + } + } + + let query_result = sqlx::query_as!( + SyntheticIdRecord, + r#" +SELECT + next_id +FROM + tts_result_synthetic_ids +WHERE + user_token = ? +LIMIT 1 + "#, + creator_user_token, + ) + .fetch_one(&mut transaction) + .await; + + let record : SyntheticIdRecord = match query_result { + Ok(record) => record, + Err(err) => { + warn!("Transaction failure: {:?}", err); + transaction.rollback().await?; + return Err(anyhow!("Transaction failure: {:?}", err)); + } + }; + + let next_id = record.next_id as u64; + + info!(">>>>> synthetic is {}", next_id); + + maybe_creator_synthetic_id = Some(next_id); + } + + let record_id = { + let query_result = sqlx::query!( r#" INSERT INTO tts_results SET @@ -253,6 +324,8 @@ SET normalized_inference_text = ?, maybe_creator_user_token = ?, + maybe_creator_synthetic_id = ?, + creator_ip_address = ?, creator_set_visibility = 'public', @@ -270,7 +343,9 @@ SET normalized_inference_text, text_hash, - job.maybe_creator_user_token.clone(), + maybe_creator_user_token, + maybe_creator_synthetic_id, + job.creator_ip_address.clone(), bucket_audio_result_path, @@ -279,22 +354,96 @@ SET file_size_bytes, duration_millis ) - .execute(pool) - .await; + .execute(&mut transaction) + .await; - let record_id = match query_result { - Ok(res) => { - res.last_insert_id() - }, - Err(err) => { - // TODO: handle better - return Err(anyhow!("Mysql error: {:?}", err)); - } + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + record_id }; + transaction.commit().await?; + Ok((record_id, inference_result_token.clone())) } +pub struct SyntheticIdRecord { + pub next_id: i64, +} + +/*async fn get_incremented_synthetic_id<'a>( + user_token: &str, + transaction: &mut Transaction<'a, MySql> +) -> AnyhowResult { + + let do_rollback = { + let query_result = sqlx::query!( + r#" +INSERT INTO tts_result_synthetic_ids +SET + user_token = ?, + next_id = 1 +ON DUPLICATE KEY UPDATE + user_token = ?, + next_id = next_id + 1 + "#, + user_token, + user_token + ) + .execute(transaction) + .await; + + match query_result { + Ok(_) => false, + Err(err) => { + warn!("Transaction failure: {:?}", err); + true + } + } + }; + + if do_rollback { + transaction.rollback().await?; + return Err(anyhow!("Transaction failure")); + } + + let query_result = sqlx::query_as!( + SyntheticIdRecord, + r#" +SELECT + next_id +FROM + tts_result_synthetic_ids +WHERE + user_token = ? +LIMIT 1 + "#, + user_token, + ) + .fetch_one(transaction) + .await; + + let record : SyntheticIdRecord = match query_result { + Ok(record) => record, + Err(err) => { + warn!("Transaction failure: {:?}", err); + transaction.rollback().await?; + return Err(anyhow!("Transaction failure: {:?}", err)); + } + }; + + let next_id = record.next_id as u64; + Ok(next_id) +}*/ + pub struct TtsModelRecord2 { pub model_token: String, pub tts_model_type: String, From 76c2fe446c2a574a5f5f25f310d23c6d3ee9236e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 06:17:43 -0400 Subject: [PATCH 0401/1042] synthetic IDs for TTS --- debug/mysql/last_tts_inference_results.sh | 45 +++++++++++ debug/mysql/tts_results_synthetic_ids.sh | 7 ++ .../list_tts_inference_results.rs | 7 ++ src/job_queries/tts_inference_job_queries.rs | 76 ------------------- 4 files changed, 59 insertions(+), 76 deletions(-) create mode 100755 debug/mysql/last_tts_inference_results.sh create mode 100755 debug/mysql/tts_results_synthetic_ids.sh diff --git a/debug/mysql/last_tts_inference_results.sh b/debug/mysql/last_tts_inference_results.sh new file mode 100755 index 0000000000..fea637e52e --- /dev/null +++ b/debug/mysql/last_tts_inference_results.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV + +mysql -u storyteller \ + -ppassword \ + -h localhost \ + -D storyteller \ + -e "select * from tts_results\G" + + +#echo '===============================================' +# +#read -r -d '' query << EOF +# +# SELECT +# w2l_results.token as w2l_result_token, +# w2l_results.maybe_tts_inference_result_token, +# w2l_templates.token as maybe_w2l_template_token, +# w2l_templates.template_type, +# users.token as maybe_creator_user_token, +# users.username as maybe_creator_username, +# users.display_name as maybe_creator_display_name, +# w2l_results.file_size_bytes, +# w2l_results.frame_width, +# w2l_results.frame_height, +# w2l_results.duration_millis, +# w2l_results.created_at, +# w2l_results.updated_at +# FROM +# w2l_results +# LEFT OUTER JOIN w2l_templates ON w2l_results.maybe_w2l_template_token = w2l_templates.token +# LEFT OUTER JOIN users ON w2l_results.maybe_creator_user_token = users.token +# WHERE +# w2l_results.deleted_at IS NULL +# AND users.username = 'echelon'\G +#EOF +# +#echo $query +# +#mysql -u root \ +# -proot \ +# -h localhost \ +# -D storyteller \ +# -e "${query}" + diff --git a/debug/mysql/tts_results_synthetic_ids.sh b/debug/mysql/tts_results_synthetic_ids.sh new file mode 100755 index 0000000000..4bfbd85f23 --- /dev/null +++ b/debug/mysql/tts_results_synthetic_ids.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u storyteller \ + -ppassword \ + -h localhost \ + -D storyteller \ + -e "select * from tts_result_synthetic_ids\G" diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index 1074dfff3c..3c06bf9698 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -29,6 +29,8 @@ pub struct TtsInferenceRecordForList { pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, + pub maybe_creator_result_id: Option, + pub file_size_bytes: u32, pub duration_millis: u32, @@ -120,6 +122,7 @@ impl ListTtsResultsQueryBuilder { maybe_creator_user_token: r.maybe_creator_user_token.clone(), maybe_creator_username: r.maybe_creator_username.clone(), maybe_creator_display_name: r.maybe_creator_display_name.clone(), + maybe_creator_result_id: r.maybe_creator_result_id.map(|v| v as u64).clone(), file_size_bytes: if r.file_size_bytes > 0 { r.file_size_bytes as u32 } else { 0 }, duration_millis: if r.duration_millis > 0 { r.duration_millis as u32 } else { 0 }, created_at: r.created_at, @@ -183,6 +186,8 @@ SELECT users.username as maybe_creator_username, users.display_name as maybe_creator_display_name, + tts_results.maybe_creator_synthetic_id as maybe_creator_result_id, + tts_results.file_size_bytes, tts_results.duration_millis, tts_results.created_at, @@ -279,6 +284,8 @@ pub struct RawInternalTtsRecord { pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, + pub maybe_creator_result_id: Option, + pub file_size_bytes : i64, pub duration_millis : i64, pub created_at: DateTime, diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 65fe50cab5..82263fde48 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -247,15 +247,6 @@ pub async fn insert_tts_result>( let mut transaction = pool.begin().await?; if let Some(creator_user_token) = maybe_creator_user_token.as_deref() { - - info!(">>>>> Creating synthetic for {}", creator_user_token); - - //let synthetic_id = { - // let id = get_incremented_synthetic_id(creator_user_token, &mut transaction).await?; - // id - //}; - //maybe_creator_synthetic_id = Some(synthetic_id); - let query_result = sqlx::query!( r#" INSERT INTO tts_result_synthetic_ids @@ -306,8 +297,6 @@ LIMIT 1 let next_id = record.next_id as u64; - info!(">>>>> synthetic is {}", next_id); - maybe_creator_synthetic_id = Some(next_id); } @@ -379,71 +368,6 @@ pub struct SyntheticIdRecord { pub next_id: i64, } -/*async fn get_incremented_synthetic_id<'a>( - user_token: &str, - transaction: &mut Transaction<'a, MySql> -) -> AnyhowResult { - - let do_rollback = { - let query_result = sqlx::query!( - r#" -INSERT INTO tts_result_synthetic_ids -SET - user_token = ?, - next_id = 1 -ON DUPLICATE KEY UPDATE - user_token = ?, - next_id = next_id + 1 - "#, - user_token, - user_token - ) - .execute(transaction) - .await; - - match query_result { - Ok(_) => false, - Err(err) => { - warn!("Transaction failure: {:?}", err); - true - } - } - }; - - if do_rollback { - transaction.rollback().await?; - return Err(anyhow!("Transaction failure")); - } - - let query_result = sqlx::query_as!( - SyntheticIdRecord, - r#" -SELECT - next_id -FROM - tts_result_synthetic_ids -WHERE - user_token = ? -LIMIT 1 - "#, - user_token, - ) - .fetch_one(transaction) - .await; - - let record : SyntheticIdRecord = match query_result { - Ok(record) => record, - Err(err) => { - warn!("Transaction failure: {:?}", err); - transaction.rollback().await?; - return Err(anyhow!("Transaction failure: {:?}", err)); - } - }; - - let next_id = record.next_id as u64; - Ok(next_id) -}*/ - pub struct TtsModelRecord2 { pub model_token: String, pub tts_model_type: String, From 2b1cf957c72b33aa2f7508c36728f68be6304fc4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 06:30:28 -0400 Subject: [PATCH 0402/1042] synthetic ids for w2l --- .../list_w2l_inference_results.rs | 7 ++ src/job_queries/tts_inference_job_queries.rs | 11 ++- src/job_queries/w2l_inference_job_queries.rs | 97 ++++++++++++++++--- 3 files changed, 99 insertions(+), 16 deletions(-) diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/common_queries/list_w2l_inference_results.rs index 910a43df11..d8e28f6376 100644 --- a/src/common_queries/list_w2l_inference_results.rs +++ b/src/common_queries/list_w2l_inference_results.rs @@ -29,6 +29,8 @@ pub struct W2lInferenceRecordForList { pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, + + pub maybe_creator_result_id: Option, pub file_size_bytes: u32, pub frame_width: u32, @@ -138,6 +140,7 @@ impl ListW2lResultsQueryBuilder { maybe_creator_user_token: r.maybe_creator_user_token.clone(), maybe_creator_username: r.maybe_creator_username.clone(), maybe_creator_display_name: r.maybe_creator_display_name.clone(), + maybe_creator_result_id: r.maybe_creator_result_id.map(|v| v as u64).clone(), file_size_bytes: if r.file_size_bytes > 0 { r.file_size_bytes as u32 } else { 0 }, frame_width: if r.frame_width > 0 { r.frame_width as u32 } else { 0 }, frame_height: if r.frame_height > 0 { r.frame_height as u32 } else { 0 }, @@ -204,6 +207,8 @@ SELECT users.username as maybe_creator_username, users.display_name as maybe_creator_display_name, + w2l_results.maybe_creator_synthetic_id as maybe_creator_result_id, + w2l_results.file_size_bytes, w2l_results.frame_width, w2l_results.frame_height, @@ -320,6 +325,8 @@ pub struct RawInternalTtsRecord { pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, + pub maybe_creator_result_id: Option, + pub file_size_bytes: i32, pub frame_width: i32, pub frame_height: i32, diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 82263fde48..3cf40b921c 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -217,6 +217,10 @@ WHERE id = ? Ok(()) } +pub struct SyntheticIdRecord { + pub next_id: i64, +} + pub async fn insert_tts_result>( pool: &MySqlPool, job: &TtsInferenceJobRecord, @@ -266,6 +270,7 @@ ON DUPLICATE KEY UPDATE match query_result { Ok(_) => {}, Err(err) => { + //transaction.rollback().await?; warn!("Transaction failure: {:?}", err); } } @@ -296,7 +301,6 @@ LIMIT 1 }; let next_id = record.next_id as u64; - maybe_creator_synthetic_id = Some(next_id); } @@ -352,6 +356,7 @@ SET }, Err(err) => { // TODO: handle better + //transaction.rollback().await?; return Err(anyhow!("Mysql error: {:?}", err)); } }; @@ -364,10 +369,6 @@ SET Ok((record_id, inference_result_token.clone())) } -pub struct SyntheticIdRecord { - pub next_id: i64, -} - pub struct TtsModelRecord2 { pub model_token: String, pub tts_model_type: String, diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index d917bb270d..600cf56e87 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -213,6 +213,7 @@ WHERE id = ? Ok(()) } + pub async fn mark_w2l_inference_job_done( pool: &MySqlPool, job: &W2lInferenceJobRecord, @@ -241,6 +242,10 @@ WHERE id = ? Ok(()) } +pub struct SyntheticIdRecord { + pub next_id: i64, +} + pub async fn insert_w2l_result>( pool: &MySqlPool, job: &W2lInferenceJobRecord, @@ -259,7 +264,67 @@ pub async fn insert_w2l_result>( .display() .to_string(); - let query_result = sqlx::query!( + let maybe_creator_user_token = job.maybe_creator_user_token.clone(); + let mut maybe_creator_synthetic_id : Option = None; + + let mut transaction = pool.begin().await?; + + if let Some(creator_user_token) = maybe_creator_user_token.as_deref() { + let query_result = sqlx::query!( + r#" +INSERT INTO w2l_result_synthetic_ids +SET + user_token = ?, + next_id = 1 +ON DUPLICATE KEY UPDATE + user_token = ?, + next_id = next_id + 1 + "#, + creator_user_token, + creator_user_token + ) + .execute(&mut transaction) + .await; + + match query_result { + Ok(_) => {}, + Err(err) => { + //transaction.rollback().await?; + warn!("Transaction failure: {:?}", err); + } + } + + let query_result = sqlx::query_as!( + SyntheticIdRecord, + r#" +SELECT + next_id +FROM + w2l_result_synthetic_ids +WHERE + user_token = ? +LIMIT 1 + "#, + creator_user_token, + ) + .fetch_one(&mut transaction) + .await; + + let record : SyntheticIdRecord = match query_result { + Ok(record) => record, + Err(err) => { + warn!("Transaction failure: {:?}", err); + //transaction.rollback().await?; + return Err(anyhow!("Transaction failure: {:?}", err)); + } + }; + + let next_id = record.next_id as u64; + maybe_creator_synthetic_id = Some(next_id); + } + + let record_id = { + let query_result = sqlx::query!( r#" INSERT INTO w2l_results SET @@ -269,6 +334,8 @@ SET creator_ip_address = ?, creator_set_visibility = 'public', + maybe_creator_synthetic_id = ?, + public_bucket_video_path = ?, file_size_bytes = ?, @@ -282,6 +349,8 @@ SET job.maybe_creator_user_token.clone(), job.creator_ip_address.clone(), + maybe_creator_synthetic_id, + bucket_video_result_path, file_size_bytes, @@ -290,19 +359,25 @@ SET frame_height, duration_millis ) - .execute(pool) - .await; + .execute(&mut transaction) + .await; - let record_id = match query_result { - Ok(res) => { - res.last_insert_id() - }, - Err(err) => { - // TODO: handle better - return Err(anyhow!("Mysql error: {:?}", err)); - } + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + transaction.rollback().await?; + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + record_id }; + transaction.commit().await?; + Ok((record_id, inference_result_token.clone())) } From 4db289c3c09faf0d03dfd44cbc6d60d1e32fa592 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 06:31:07 -0400 Subject: [PATCH 0403/1042] codegen db --- sqlx-data.json | 8956 +++++++++++++++++++++++++----------------------- 1 file changed, 4581 insertions(+), 4375 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 92209881f3..6250cf2002 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -38,13 +38,13 @@ ] } }, - "0c32927e9292e0cf4aeb169ee3ed0ec122712a66a6850eb9e9da31d5a97e6e72": { - "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_video_path as maybe_public_bucket_video_path,\n\n jobs.maybe_w2l_template_token,\n w2l.template_type as w2l_template_type,\n w2l.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_inference_jobs as jobs\nJOIN w2l_templates as w2l\n ON w2l.token = jobs.maybe_w2l_template_token\nLEFT OUTER JOIN w2l_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", + "05c3ebf54cd8a5374d6f4bbe21b3c6027ce6314917af8acd0ed136d375d53342": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "job_token", + "name": "template_token", "type_info": { "type": "VarString", "flags": { @@ -56,55 +56,7 @@ }, { "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 2, - "name": "maybe_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "maybe_public_bucket_video_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 5, - "name": "w2l_template_type", + "name": "template_type", "type_info": { "type": "String", "flags": { @@ -115,7 +67,43 @@ } }, { - "ordinal": 6, + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, "name": "title", "type_info": { "type": "String", @@ -126,8 +114,80 @@ "max_size": 1020 } }, + { + "ordinal": 6, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, { "ordinal": 7, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, "name": "created_at", "type_info": { "type": "Timestamp", @@ -139,7 +199,7 @@ } }, { - "ordinal": 8, + "ordinal": 13, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -157,12 +217,17 @@ "nullable": [ false, false, - true, - true, - true, false, false, false, + false, + false, + false, + false, + true, + true, + true, + false, false ] } @@ -205,85 +270,8 @@ ] } }, - "0fc26cb6ab34ee6cd9d0cd8fac8bae7bd6f25bbf7c785cd5dc8888d08099b88b": { - "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_template_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_template_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 2, - "name": "maybe_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 4, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - true, - false, - false - ] - } - }, - "10d2e4544e7ee74c3c394f1305e281cbc4003df207183ceaa18384c2e77baa61": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.is_mod_disabled,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n tts.deleted_at IS NULL\n AND users.username = ?\n ", + "10baa66f954e30d5da4bdb0fe6f912b01c6ac7ec803801797a0a5fd81ae0947e": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n tts.created_at,\n tts.updated_at,\n tts.user_deleted_at,\n tts.mod_deleted_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", "describe": { "columns": [ { @@ -348,18 +336,6 @@ }, { "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, "name": "title", "type_info": { "type": "VarString", @@ -370,9 +346,33 @@ "max_size": 1020 } }, + { + "ordinal": 6, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, { "ordinal": 7, - "name": "is_mod_disabled", + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 8, + "name": "is_locked_from_use", "type_info": { "type": "Tiny", "flags": { @@ -383,7 +383,19 @@ } }, { - "ordinal": 8, + "ordinal": 9, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, "name": "created_at", "type_info": { "type": "Timestamp", @@ -395,7 +407,7 @@ } }, { - "ordinal": 9, + "ordinal": 11, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -405,6 +417,30 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 12, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { @@ -420,174 +456,16 @@ false, false, false, - false + false, + false, + false, + true, + true ] } }, - "12250420b0a3ff3bf2dd7be76daa2920777e349642245211706f32add3df442c": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.updatable_slug,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\nAND tts.deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 7, - "name": "private_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, - "215f82f4034ccb47bfa000c3fead66c3c9f7d5caa905aa99a702d396aeeebbc6": { - "query": "\nINSERT INTO tts_model_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 11 - }, - "nullable": [] - } - }, - "22a52ac04d4a53d48d5134fc929d39d8b13617bf62a2267be7bde0ac54ec38e2": { - "query": "\nUPDATE w2l_templates\nSET\n deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "2537264be5f06db65bb9faa4b1a3e3fe97a18e8247d85cb8b10ea6f3b72b942b": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND users.username = ?\n ", + "11554bff2bf06cf0f843b716c99c61b540639207d716eeabe3e2afb189290ab6": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.token = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { @@ -652,18 +530,6 @@ }, { "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, "name": "title", "type_info": { "type": "String", @@ -675,7 +541,7 @@ } }, { - "ordinal": 7, + "ordinal": 6, "name": "frame_width", "type_info": { "type": "Long", @@ -687,7 +553,7 @@ } }, { - "ordinal": 8, + "ordinal": 7, "name": "frame_height", "type_info": { "type": "Long", @@ -699,7 +565,7 @@ } }, { - "ordinal": 9, + "ordinal": 8, "name": "duration_millis", "type_info": { "type": "Long", @@ -711,7 +577,7 @@ } }, { - "ordinal": 10, + "ordinal": 9, "name": "maybe_public_bucket_preview_image_object_name", "type_info": { "type": "VarString", @@ -723,7 +589,7 @@ } }, { - "ordinal": 11, + "ordinal": 10, "name": "maybe_public_bucket_preview_video_object_name", "type_info": { "type": "VarString", @@ -735,12 +601,24 @@ } }, { - "ordinal": 12, - "name": "is_mod_public_listing_approved", + "ordinal": 11, + "name": "is_public_listing_approved", "type_info": { "type": "Tiny", "flags": { - "bits": 0 + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 }, "char_set": 63, "max_size": 1 @@ -748,6 +626,18 @@ }, { "ordinal": 13, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, "name": "created_at", "type_info": { "type": "Timestamp", @@ -759,7 +649,7 @@ } }, { - "ordinal": 14, + "ordinal": 15, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -769,6 +659,30 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 16, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 17, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { @@ -784,27 +698,80 @@ false, false, false, - false, true, true, true, false, - false + false, + false, + false, + true, + true ] } }, - "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { - "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "1168f39a5ec892d06e941b0dba0e955a27ef976567eeafd6e550bc42ea4c4331": { + "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", "describe": { "columns": [], "parameters": { - "Right": 3 + "Right": 7 }, "nullable": [] } }, - "2a00159789d38ecb28249096980d591606af8b55d60ce719d68309eb73d42bb0": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.is_mod_disabled,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n tts.deleted_at IS NULL\n AND tts.is_mod_disabled IS FALSE\n AND users.username = ?\n ", + "1236640a30d0d5177e13c31b61b791874f7c70026003a58ba6e07e7af46c30c1": { + "query": "\nSELECT\n next_id\nFROM\n w2l_result_synthetic_ids\nWHERE\n user_token = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "next_id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 20 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + } + }, + "150c98c00f4cd6954428736583eae913be578c4e69dac317d4445bc0e97743c9": { + "query": "\nSELECT\n next_id\nFROM\n tts_result_synthetic_ids\nWHERE\n user_token = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "next_id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 20 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + } + }, + "1513af6ee028edfea33311b64d5d71f784ea058bbde114cfbc9b1785ef98c7e8": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n tts.created_at,\n tts.updated_at,\n tts.user_deleted_at,\n tts.mod_deleted_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { @@ -869,18 +836,6 @@ }, { "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, "name": "title", "type_info": { "type": "VarString", @@ -891,9 +846,33 @@ "max_size": 1020 } }, + { + "ordinal": 6, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, { "ordinal": 7, - "name": "is_mod_disabled", + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 8, + "name": "is_locked_from_use", "type_info": { "type": "Tiny", "flags": { @@ -904,7 +883,19 @@ } }, { - "ordinal": 8, + "ordinal": 9, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, "name": "created_at", "type_info": { "type": "Timestamp", @@ -916,7 +907,7 @@ } }, { - "ordinal": 9, + "ordinal": 11, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -926,6 +917,30 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 12, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { @@ -941,17 +956,21 @@ false, false, false, - false + false, + false, + false, + true, + true ] } }, - "2afb1d98fe7716b378778f706703dbaaa3ebb458e08ba7668a745484bce70895": { - "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.model_token as tts_model_token,\n tts_results.inference_text as inference_text,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nWHERE\n tts_results.deleted_at IS NULL\n ", + "1556b7ed6b0cab002b52dfa8a5f81743a38967631ff24fd9555a168823b1dfc1": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "tts_result_token", + "name": "model_token", "type_info": { "type": "VarString", "flags": { @@ -963,7 +982,19 @@ }, { "ordinal": 1, - "name": "tts_model_token", + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", "type_info": { "type": "VarString", "flags": { @@ -973,37 +1004,25 @@ "max_size": 128 } }, - { - "ordinal": 2, - "name": "inference_text", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, { "ordinal": 3, - "name": "maybe_creator_user_token", + "name": "creator_username", "type_info": { "type": "VarString", "flags": { - "bits": 4228 + "bits": 4229 }, "char_set": 224, - "max_size": 128 + "max_size": 80 } }, { "ordinal": 4, - "name": "maybe_creator_username", + "name": "creator_display_name", "type_info": { "type": "VarString", "flags": { - "bits": 4228 + "bits": 4225 }, "char_set": 224, "max_size": 80 @@ -1011,42 +1030,30 @@ }, { "ordinal": 5, - "name": "maybe_creator_display_name", + "name": "title", "type_info": { "type": "VarString", "flags": { - "bits": 4224 + "bits": 4225 }, "char_set": 224, - "max_size": 80 + "max_size": 1020 } }, { "ordinal": 6, - "name": "file_size_bytes", + "name": "is_locked_from_use", "type_info": { - "type": "Long", + "type": "Tiny", "flags": { - "bits": 1 + "bits": 9 }, "char_set": 63, - "max_size": 10 + "max_size": 1 } }, { "ordinal": 7, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, "name": "created_at", "type_info": { "type": "Timestamp", @@ -1058,7 +1065,7 @@ } }, { - "ordinal": 9, + "ordinal": 8, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -1077,9 +1084,8 @@ false, false, false, - true, - true, - true, + false, + false, false, false, false, @@ -1087,33 +1093,53 @@ ] } }, - "2b04dd7db52ef3a97a6f420a22a4dbee7416ae58697438df69e87ffd47bbf350": { - "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "17c8d210d93ee9c0d8e8422eed605c9150f19f9bf72affcf111b15a67a5b9e70": { + "query": "\nUPDATE w2l_templates\nSET\n is_public_listing_approved = ?,\n is_locked_from_user_modification = ?,\n is_locked_from_use = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { "columns": [], "parameters": { - "Right": 3 + "Right": 6 }, "nullable": [] } }, - "2bfb24d71e1fccb98b0c27912214dca2e2c2833355fdfa9f70e4d7a4d99a038b": { - "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", + "18ca4c90132288c1dc3d43c6b0e7ee6aa12e990408e314e0038d4a376c9ddfbf": { + "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { "columns": [], "parameters": { - "Right": 11 + "Right": 5 }, "nullable": [] } }, - "2cebfec1341bb1066357b77440f36b3b7b7d292c64354912f21f7af6975de6fa": { - "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_confirmed,\n users.email_gravatar_hash,\n users.avatar_public_bucket_hash,\n\n users.dark_mode_preference,\n users.disable_gravatar,\n users.hide_results_preference,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n users.maybe_preferred_tts_model_token,\n users.maybe_preferred_w2l_template_token,\n\n users.user_role_slug,\n users.banned,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n user_roles.can_delete_own_tts_models,\n user_roles.can_delete_own_w2l_templates,\n\n user_roles.can_approve_w2l_templates,\n user_roles.can_edit_other_users_profiles,\n user_roles.can_edit_other_users_tts_models,\n user_roles.can_edit_other_users_w2l_templates,\n user_roles.can_delete_other_users_tts_models,\n user_roles.can_delete_other_users_w2l_templates,\n user_roles.can_ban_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.deleted_at IS NULL\n ", + "1cc62e5cdc8de18969d7e8018d91c38794e5a7d23f117e5b0a4126845f300e60": { + "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE\n token = ?\n AND deleted_at IS NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "25e7858d370ac1359deb42ef5292e85ccd1fe7debf3a1ba73303e5882523a4d0": { + "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n download_url = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 8 + }, + "nullable": [] + } + }, + "27aa92743ce8b2221feeac96de6eb8b378b427912de09779fee938b757de4bdc": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\nAND tts.user_deleted_at IS NULL\nAND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "user_token", + "name": "model_token", "type_info": { "type": "VarString", "flags": { @@ -1125,338 +1151,98 @@ }, { "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "email_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "email_confirmed", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 5, - "name": "email_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "avatar_public_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "dark_mode_preference", + "name": "tts_model_type", "type_info": { "type": "String", "flags": { "bits": 385 }, "char_set": 224, - "max_size": 40 + "max_size": 60 } }, { - "ordinal": 8, - "name": "disable_gravatar", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "hide_results_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "auto_play_audio_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 11, - "name": "auto_play_video_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "maybe_preferred_tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 13, - "name": "maybe_preferred_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 14, - "name": "user_role_slug", + "ordinal": 2, + "name": "creator_user_token", "type_info": { "type": "VarString", "flags": { "bits": 4233 }, "char_set": 224, - "max_size": 64 + "max_size": 128 } }, { - "ordinal": 15, - "name": "banned", + "ordinal": 3, + "name": "creator_username", "type_info": { - "type": "Tiny", + "type": "VarString", "flags": { - "bits": 1 + "bits": 4229 }, - "char_set": 63, - "max_size": 1 + "char_set": 224, + "max_size": 80 } }, { - "ordinal": 16, - "name": "can_use_tts", + "ordinal": 4, + "name": "creator_display_name", "type_info": { - "type": "Tiny", + "type": "VarString", "flags": { - "bits": 0 + "bits": 4225 }, - "char_set": 63, - "max_size": 1 + "char_set": 224, + "max_size": 80 } }, { - "ordinal": 17, - "name": "can_use_w2l", + "ordinal": 5, + "name": "title", "type_info": { - "type": "Tiny", + "type": "VarString", "flags": { - "bits": 0 + "bits": 4225 }, - "char_set": 63, - "max_size": 1 + "char_set": 224, + "max_size": 1020 } }, { - "ordinal": 18, - "name": "can_upload_tts_models", + "ordinal": 6, + "name": "private_bucket_hash", "type_info": { - "type": "Tiny", + "type": "String", "flags": { - "bits": 0 + "bits": 4233 }, - "char_set": 63, - "max_size": 1 + "char_set": 224, + "max_size": 256 } }, { - "ordinal": 19, - "name": "can_upload_w2l_templates", + "ordinal": 7, + "name": "created_at", "type_info": { - "type": "Tiny", + "type": "Timestamp", "flags": { - "bits": 0 + "bits": 1153 }, "char_set": 63, - "max_size": 1 + "max_size": 19 } }, { - "ordinal": 20, - "name": "can_delete_own_tts_models", + "ordinal": 8, + "name": "updated_at", "type_info": { - "type": "Tiny", + "type": "Timestamp", "flags": { - "bits": 0 + "bits": 9345 }, "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 21, - "name": "can_delete_own_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 22, - "name": "can_approve_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 23, - "name": "can_edit_other_users_profiles", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 24, - "name": "can_edit_other_users_tts_models", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 25, - "name": "can_edit_other_users_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 26, - "name": "can_delete_other_users_tts_models", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 27, - "name": "can_delete_other_users_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 28, - "name": "can_ban_users", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 + "max_size": 19 } } ], @@ -1470,32 +1256,52 @@ false, false, false, - true, false, false, - false, - false, - false, - true, - true, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true + false ] } }, + "28cf5843d68d6cb0e67eb316ef9e7e5fb05a6be2433f86c081576b279167508d": { + "query": "\nINSERT INTO tts_result_synthetic_ids\nSET\n user_token = ?,\n next_id = 1\nON DUPLICATE KEY UPDATE\n user_token = ?,\n next_id = next_id + 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "2b04dd7db52ef3a97a6f420a22a4dbee7416ae58697438df69e87ffd47bbf350": { + "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "2d485a1232d1b5bd1ad6a05ed744fe280121903fcebac4b7e9d12b732e0caa21": { + "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_hash = 'TODO',\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n inference_public_bucket_uuid = 'TODO',\n\n file_size_bytes = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 12 + }, + "nullable": [] + } + }, "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", "describe": { @@ -1516,405 +1322,34 @@ "nullable": [] } }, - "36ff2cf2bdea42da6649923163013cdb5eaf719532d99305410284c69bedd2c0": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.private_bucket_hash,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\nAND w2l.deleted_at IS NULL\n ", + "34642399c3e91960768cb659b6006db291574d9cd07a5bace3db573eff20c3fb": { + "query": "\nINSERT INTO w2l_result_synthetic_ids\nSET\n user_token = ?,\n next_id = 1\nON DUPLICATE KEY UPDATE\n user_token = ?,\n next_id = next_id + 1\n ", "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "private_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 11, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 12, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 13, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 14, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], + "columns": [], "parameters": { - "Right": 1 + "Right": 2 }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - false - ] + "nullable": [] } }, - "372e5a9d04fd56166ef38596b52b1707a5e6a78784f72395b24e91a15a523b2f": { - "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n model_token,\n inference_text,\n\n creator_ip_address,\n maybe_creator_user_token,\n creator_set_visibility,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM tts_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "346c02b215cb35efaa35fe96d866a8c9056ac46e987b4e01f64222259dccdcbf": { + "query": "\nUPDATE\n ip_address_bans\nSET\n mod_user_token = ?,\n deleted_at = NULL\nWHERE\n ip_address = ?\nLIMIT 1\n ", "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "inference_job_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "uuid_idempotency_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 3, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "inference_text", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 5, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_user_token", - "type_info": { - "type": "String", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 7, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 8, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 9, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 3 - } - }, - { - "ordinal": 10, - "name": "failure_reason", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 11, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 12, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "retry_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], + "columns": [], "parameters": { - "Right": 1 + "Right": 2 }, - "nullable": [ - false, - false, - false, - false, - false, - false, - true, - false, - false, - false, - true, - false, - false, - true - ] + "nullable": [] + } + }, + "376e36b1357952599866262c7356ba35f15ccdfb338d8ccdbdc06ccd69331fc3": { + "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] } }, "3a2dc0a7e41dcbdf21f91d1a1e54e813f74b0bac7aea052302089cae195eeb53": { @@ -1927,847 +1362,8 @@ "nullable": [] } }, - "3bc397a1157c4564562138dbef80cf89d2185943f5f925228936f0b5c67e1b95": { - "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, - "3de0e9c442ae0d4ca2f1da19357ac7685084e2843216313d7836ea60bc3dcfe0": { - "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_model_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_model_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 2, - "name": "maybe_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 4, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - true, - false, - false - ] - } - }, - "4b4b6c2b8306cd5b6b1fe28fd492de5e346f0a53c0efc4f0eec57c4238860764": { - "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.model_token as tts_model_token,\n tts_results.inference_text as inference_text,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND tts_results.is_mod_hidden_from_public IS FALSE\n AND tts_models.is_mod_disabled IS FALSE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tts_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "inference_text", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 3, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 7, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - true, - true, - true, - false, - false, - false, - false - ] - } - }, - "4f61f621682ba255814f8a55455ea2b7fba9b6c85affaf87abfe2f81e11322f0": { - "query": "\nSELECT\n id,\n status\nFROM tts_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false - ] - } - }, - "4fec6d4e2b33411835c679942860c328e20a4276fb39c3512f0608261c5467f4": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.is_mod_disabled,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE\n tts.deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_mod_disabled", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, - "53d77ee58e9b1f7fb900914895c25bcccb9297378ff018c1e553b187d1956e92": { - "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE username = ?\nLIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "email_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 3, - "name": "password_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 63, - "max_size": 60 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false - ] - } - }, - "55a75db2d060bd49951db22f6b4ffd1545777b61c256030d59662543adc06ab2": { - "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n ip_address_last_update = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 10 - }, - "nullable": [] - } - }, - "65ce50a3aafe979e77a541d576228d7fcfe76e33400427d01382da36c1585edb": { - "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "685a1d5bb03668be301bab35d677e270c94661c53f370ecd6ce8f1b697c4a01d": { - "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "w2l_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "maybe_tts_inference_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 392 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 4, - "name": "template_title", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 5, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 8, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 11, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 12, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false - ] - } - }, - "68932d5d72489d06972510fb218ceeccc5c925bcd250a804c8b857502e373f18": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.is_mod_disabled,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE\n tts.deleted_at IS NULL\n AND tts.is_mod_disabled IS FALSE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_mod_disabled", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, - "73558104936aab6add2d9c789b368b5f155f1fe01d32e7314f2f120422866f54": { - "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "7408744db4c97b12e50585e2cdbc421a6c467316dea6589e3a317686cfe0deb4": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n ", + "3c6615a453cf4ab41ff77616369a8aa1ed83f835ad35a5b52ca61fa77dfb8c77": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", "describe": { "columns": [ { @@ -2832,18 +1428,6 @@ }, { "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, "name": "title", "type_info": { "type": "String", @@ -2855,7 +1439,7 @@ } }, { - "ordinal": 7, + "ordinal": 6, "name": "frame_width", "type_info": { "type": "Long", @@ -2867,7 +1451,7 @@ } }, { - "ordinal": 8, + "ordinal": 7, "name": "frame_height", "type_info": { "type": "Long", @@ -2879,7 +1463,7 @@ } }, { - "ordinal": 9, + "ordinal": 8, "name": "duration_millis", "type_info": { "type": "Long", @@ -2891,7 +1475,7 @@ } }, { - "ordinal": 10, + "ordinal": 9, "name": "maybe_public_bucket_preview_image_object_name", "type_info": { "type": "VarString", @@ -2903,7 +1487,7 @@ } }, { - "ordinal": 11, + "ordinal": 10, "name": "maybe_public_bucket_preview_video_object_name", "type_info": { "type": "VarString", @@ -2915,12 +1499,24 @@ } }, { - "ordinal": 12, - "name": "is_mod_public_listing_approved", + "ordinal": 11, + "name": "is_public_listing_approved", "type_info": { "type": "Tiny", "flags": { - "bits": 0 + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 }, "char_set": 63, "max_size": 1 @@ -2928,6 +1524,18 @@ }, { "ordinal": 13, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, "name": "created_at", "type_info": { "type": "Timestamp", @@ -2939,7 +1547,7 @@ } }, { - "ordinal": 14, + "ordinal": 15, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -2949,253 +1557,14 @@ "char_set": 63, "max_size": 19 } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false - ] - } - }, - "746526e2e150150ef804d717f743de85010972d98cbb137e44d8348cfdb4dddc": { - "query": "\nINSERT INTO w2l_results\nSET\n token = ?,\n maybe_w2l_template_token = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_video_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n duration_millis = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 10 - }, - "nullable": [] - } - }, - "79e6cc7829521ba8b26b5a97d3c8ab4e371945fa581e1632440c05658a2731ef": { - "query": "\nUPDATE w2l_inference_jobs\nSET\n status = 'started',\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, - "7ab54466ca3021d90e0feb234acba56bf53d415dde69bc1c35120ed0c8dd1858": { - "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "uuid_idempotency_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 3, - "name": "on_success_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 5, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 6, - "name": "maybe_subject_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "maybe_actor_subject_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, - "name": "download_url", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 9, - "name": "download_url_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 48 - } - }, - { - "ordinal": 10, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 11, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 12, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 13, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 14, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 3 - } - }, - { - "ordinal": 15, - "name": "failure_reason", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } }, { "ordinal": 16, - "name": "created_at", + "name": "user_deleted_at", "type_info": { "type": "Timestamp", "flags": { - "bits": 1153 + "bits": 128 }, "char_set": 63, "max_size": 19 @@ -3203,19 +1572,7 @@ }, { "ordinal": 17, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 18, - "name": "retry_at", + "name": "mod_deleted_at", "type_info": { "type": "Timestamp", "flags": { @@ -3233,12 +1590,6 @@ false, false, false, - true, - false, - false, - true, - true, - true, false, false, false, @@ -3246,1179 +1597,19 @@ false, false, true, + true, + true, false, false, + false, + false, + true, true ] } }, - "8347b64356502ae32979d48680e6b2097e7f8c4cdc492b7285044856b58ba017": { - "query": "\nINSERT INTO w2l_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n maybe_w2l_template_token = ?,\n maybe_public_audio_bucket_hash = ?,\n maybe_public_audio_bucket_location = ?,\n\n maybe_original_audio_filename = ?,\n maybe_audio_mime_type = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n disable_end_bump = false,\n creator_set_visibility = \"public\",\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 9 - }, - "nullable": [] - } - }, - "865ce236444a36d597939660b14eb4b0cee05c70fdb73f31c2fb8cddddfef42c": { - "query": "\nINSERT INTO firehose_entries\nSET\n token = ?,\n event_type = ?,\n maybe_target_user_token = ?,\n maybe_target_entity_token = ?,\n maybe_created_entity_token = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - } - }, - "86f9a76cc97b471a23fff5b9ea78f7fc7ecf7603215f5db5a1ca4e063568b054": { - "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE token = ?\nAND deleted_at IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, - "88328e04ce947d3f541951aada9cee25135aa950ab478cf0dc6de453bb079e17": { - "query": "\nSELECT\n events.token as event_token,\n events.event_type,\n events.maybe_target_user_token,\n users.username as maybe_target_username,\n users.display_name as maybe_target_display_name,\n users.email_gravatar_hash as maybe_target_user_gravatar_hash,\n events.maybe_target_entity_token,\n events.created_at,\n events.updated_at\nFROM firehose_entries as events\nLEFT OUTER JOIN users\nON events.maybe_target_user_token = users.token\nORDER BY events.id DESC\nLIMIT 25\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "event_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "event_type", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "maybe_target_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "maybe_target_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "maybe_target_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "maybe_target_user_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "maybe_target_entity_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 8, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - true, - true, - true, - true, - true, - false, - false - ] - } - }, - "8d61bb77449a78044135a3d6e5009a3a27b4671b7e40501d318b05ae401f02a8": { - "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE email_address = ?\nLIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "email_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 3, - "name": "password_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 63, - "max_size": 60 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false - ] - } - }, - "8f19eee8d83541363f3a1797ddfa9647d2e01e8bb6c6d640e85886ae2b9567fa": { - "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n model_token = ?,\n inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 7 - }, - "nullable": [] - } - }, - "9009e90e61f6672b072a1d28a18bffb6adca4ff804378a21b2611c8b62c747f0": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.is_mod_disabled,\n w2l.created_at,\n w2l.updated_at,\n w2l.deleted_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\nAND w2l.deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 12, - "name": "is_mod_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_mod_disabled", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 15, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 16, - "name": "deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false, - false, - true - ] - } - }, - "916f732e506073fd54596a730906eaa1dae1699b59fac9ac87b1476e4524c877": { - "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n banned,\n avatar_public_bucket_hash,\n disable_gravatar,\n maybe_preferred_tts_model_token,\n maybe_preferred_w2l_template_token,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n cashapp_username,\n website_url,\n created_at\nFROM users\nWHERE users.username = ?\nAND users.deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "email_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "profile_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 5, - "name": "profile_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 6, - "name": "user_role_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 7, - "name": "banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "avatar_public_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 9, - "name": "disable_gravatar", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "maybe_preferred_tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 11, - "name": "maybe_preferred_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 12, - "name": "discord_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 13, - "name": "twitch_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 14, - "name": "twitter_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 15, - "name": "patreon_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 16, - "name": "github_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 17, - "name": "cashapp_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 18, - "name": "website_url", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 19, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - true, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false - ] - } - }, - "94847f448294888db6c800cc9558738b0c0de023225961d070b84b3906a2e493": { - "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND users.username = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "w2l_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "maybe_tts_inference_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 392 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 4, - "name": "template_title", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 5, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 8, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 11, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 12, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false - ] - } - }, - "95db7007b9e184d64d7b8983822cb635e5b142deb8e2bea59b04a95430b62d4d": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.is_mod_disabled,\n w2l.created_at,\n w2l.updated_at,\n w2l.deleted_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 12, - "name": "is_mod_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_mod_disabled", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 15, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 16, - "name": "deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false, - false, - true - ] - } - }, - "9710f22f2579903571f71726585bb7de66b8a7fa9540b851c6ace30c51d29c69": { - "query": "\nUPDATE w2l_templates\nSET\n deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c": { + "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { "columns": [], "parameters": { @@ -4427,8 +1618,202 @@ "nullable": [] } }, - "9c352e6fe6166ee18326908ef02dd61866d926ec63599aedf9f91a9d05d5c630": { - "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND tts_results.token = ?\n ", + "417dd070beeaf6ccd86852b7e99bf7fda5c5b15da4a3bcc066a0e2bb90ca7df1": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.is_public_listing_approved IS TRUE\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] + } + }, + "453e6252ea537f5f3fddae361b53f21c86fe829930cf7415d43aa899e4390bd9": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n ", "describe": { "columns": [ { @@ -4445,7 +1830,7 @@ }, { "ordinal": 1, - "name": "inference_text", + "name": "raw_inference_text", "type_info": { "type": "Blob", "flags": { @@ -4646,6 +2031,30 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 18, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { @@ -4669,47 +2078,57 @@ false, false, false, - false + false, + true, + true ] } }, - "9d4932baa4023accc2d28af98b58a47149d37c4937cd0171e6b955c09855c5f8": { - "query": "\nINSERT INTO w2l_inference_jobs\nSET\n maybe_w2l_template_token = ?,\n maybe_tts_inference_result_token = ?,\n maybe_public_audio_bucket_location = NULL,\n maybe_public_image_bucket_location = NULL,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - } - }, - "9e2a2897058a40d0b27b3afddf515fd9e458a7bec824de902a673362617e8987": { - "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n inference_text = ?,\n inference_text_hash_sha2 = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_hash = 'TODO',\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n inference_public_bucket_uuid = 'TODO',\n\n file_size_bytes = ?,\n duration_millis = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 10 - }, - "nullable": [] - } - }, - "a125f183f7ae5627fc2118e7302cea73d967200ba1cbb12550f5880187b7db5e": { - "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "a9ff4b16669e8fb8efa9660c951ec545451fa2515ebd5d18308d088dec987121": { - "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND w2l_results.is_mod_hidden_from_public IS FALSE\n AND w2l_templates.is_mod_public_listing_approved IS TRUE\n AND users.username = ?\n ", + "4f61f621682ba255814f8a55455ea2b7fba9b6c85affaf87abfe2f81e11322f0": { + "query": "\nSELECT\n id,\n status\nFROM tts_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "w2l_result_token", + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + }, + "52447dfed3f294970e4c589062342ede87ab3a1f8c483751294fcb91cadff386": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n AND tts_results.user_deleted_at IS NULL\n AND tts_results.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", "type_info": { "type": "VarString", "flags": { @@ -4721,23 +2140,23 @@ }, { "ordinal": 1, - "name": "maybe_tts_inference_result_token", + "name": "raw_inference_text", "type_info": { - "type": "VarString", + "type": "Blob", "flags": { - "bits": 136 + "bits": 4241 }, "char_set": 224, - "max_size": 128 + "max_size": 262140 } }, { "ordinal": 2, - "name": "maybe_w2l_template_token", + "name": "tts_model_token", "type_info": { "type": "VarString", "flags": { - "bits": 4228 + "bits": 4233 }, "char_set": 224, "max_size": 128 @@ -4745,21 +2164,9 @@ }, { "ordinal": 3, - "name": "template_type", + "name": "tts_model_title", "type_info": { - "type": "String", - "flags": { - "bits": 392 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 4, - "name": "template_title", - "type_info": { - "type": "String", + "type": "VarString", "flags": { "bits": 4224 }, @@ -4768,7 +2175,7 @@ } }, { - "ordinal": 5, + "ordinal": 4, "name": "maybe_creator_user_token", "type_info": { "type": "VarString", @@ -4780,7 +2187,7 @@ } }, { - "ordinal": 6, + "ordinal": 5, "name": "maybe_creator_username", "type_info": { "type": "VarString", @@ -4792,7 +2199,7 @@ } }, { - "ordinal": 7, + "ordinal": 6, "name": "maybe_creator_display_name", "type_info": { "type": "VarString", @@ -4803,8 +2210,92 @@ "max_size": 80 } }, + { + "ordinal": 7, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, { "ordinal": 8, + "name": "maybe_model_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "maybe_model_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 10, + "name": "maybe_model_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 11, + "name": "maybe_model_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 12, + "name": "public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "public_bucket_spectrogram_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, "name": "file_size_bytes", "type_info": { "type": "Long", @@ -4816,31 +2307,7 @@ } }, { - "ordinal": 9, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 11, + "ordinal": 15, "name": "duration_millis", "type_info": { "type": "Long", @@ -4852,7 +2319,7 @@ } }, { - "ordinal": 12, + "ordinal": 16, "name": "created_at", "type_info": { "type": "Timestamp", @@ -4864,7 +2331,7 @@ } }, { - "ordinal": 13, + "ordinal": 17, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -4874,6 +2341,30 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 18, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { @@ -4881,6 +2372,10 @@ }, "nullable": [ false, + false, + false, + true, + true, true, true, true, @@ -4890,6 +2385,72 @@ true, false, false, + false, + false, + false, + false, + true, + true + ] + } + }, + "53d77ee58e9b1f7fb900914895c25bcccb9297378ff018c1e553b187d1956e92": { + "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE username = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "password_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 63, + "max_size": 60 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ false, false, false, @@ -4897,8 +2458,18 @@ ] } }, - "ab4913d3a45a2ee16110021823a0786a22aa561a55aafaee27851da1e0a67e12": { - "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,\n\n jobs.model_token,\n tts.tts_model_type,\n tts.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_inference_jobs as jobs\nJOIN tts_models as tts\n ON tts.token = jobs.model_token\nLEFT OUTER JOIN tts_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", + "55a75db2d060bd49951db22f6b4ffd1545777b61c256030d59662543adc06ab2": { + "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n ip_address_last_update = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, + "56a0a2bfc3c73d6845ca9cf1d285b175a40b50a250b7effb9ce6474e58113d6e": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_model_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_model_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", "describe": { "columns": [ { @@ -4927,6 +2498,108 @@ }, { "ordinal": 2, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 3, + "name": "maybe_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 5, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + false, + false + ] + } + }, + "57928b1001a10a88ac5a07f43b1dd30f6dee6edbc0808be99fe1233f9e276c56": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_video_path as maybe_public_bucket_video_path,\n\n jobs.maybe_w2l_template_token,\n w2l.template_type as w2l_template_type,\n w2l.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_inference_jobs as jobs\nJOIN w2l_templates as w2l\n ON w2l.token = jobs.maybe_w2l_template_token\nLEFT OUTER JOIN w2l_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 3, "name": "maybe_result_token", "type_info": { "type": "VarString", @@ -4938,8 +2611,8 @@ } }, { - "ordinal": 3, - "name": "maybe_public_bucket_wav_audio_path", + "ordinal": 4, + "name": "maybe_public_bucket_video_path", "type_info": { "type": "VarString", "flags": { @@ -4949,9 +2622,424 @@ "max_size": 1020 } }, + { + "ordinal": 5, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "w2l_template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 7, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + false, + false, + false, + false + ] + } + }, + "5c13a3969006332b5910fcfd4cd58eecbcd5866059b76afff610a2ad3900d094": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_template_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_template_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 3, + "name": "maybe_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, { "ordinal": 4, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 5, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + false, + false + ] + } + }, + "5e54fd8569b2d6d3a0d11acee6bd03b75400b6325952eb16c8dac68a7933b13a": { + "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n is_banned,\n disable_gravatar,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "profile_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "profile_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 6, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 7, + "name": "is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "discord_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 10, + "name": "twitch_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 11, + "name": "twitter_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 12, + "name": "patreon_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 13, + "name": "github_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 14, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + false + ] + } + }, + "6105a7fd1cf7a4d61d1b00c262c44f0370f7a9b8827354b2e3161d78493a0f5b": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "creator_user_token", "type_info": { "type": "VarString", "flags": { @@ -4962,15 +3050,27 @@ } }, { - "ordinal": 5, - "name": "tts_model_type", + "ordinal": 4, + "name": "creator_username", "type_info": { - "type": "String", + "type": "VarString", "flags": { - "bits": 385 + "bits": 4229 }, "char_set": 224, - "max_size": 60 + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 } }, { @@ -5016,8 +3116,8 @@ "nullable": [ false, false, - true, - true, + false, + false, false, false, false, @@ -5026,23 +3126,43 @@ ] } }, - "ab78a81f85310750668e2c7de920c067290f2b004561ab706a5edfd893fd06fe": { - "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n maybe_public_bucket_preview_image_object_name = ?,\n maybe_public_bucket_preview_video_object_name = ?,\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", + "641253d4c621f75037877651bba044a21033f50a5e3f8fed9b06ec967a508853": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { "columns": [], "parameters": { - "Right": 19 + "Right": 1 }, "nullable": [] } }, - "ae2c639b0b9d4efd8734c23d5fd52178df985b4258ba7f3786fd609bfd647761": { - "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.model_token as tts_model_token,\n tts_results.inference_text as inference_text,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND users.username = ?\n ", + "65ce50a3aafe979e77a541d576228d7fcfe76e33400427d01382da36c1585edb": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "663506988f146de6f559e5df4f4a2b607e9421a69bb57db1f656774bf8b375db": { + "query": "\nUPDATE w2l_results\nSET\n mod_deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "66442f357c878d55cf99a388861c6d7434f4e543f76088a8d70d73b489b11784": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "tts_result_token", + "name": "w2l_result_token", "type_info": { "type": "VarString", "flags": { @@ -5054,11 +3174,11 @@ }, { "ordinal": 1, - "name": "tts_model_token", + "name": "maybe_tts_inference_result_token", "type_info": { "type": "VarString", "flags": { - "bits": 4233 + "bits": 136 }, "char_set": 224, "max_size": 128 @@ -5066,19 +3186,19 @@ }, { "ordinal": 2, - "name": "inference_text", + "name": "public_bucket_video_path", "type_info": { - "type": "Blob", + "type": "VarString", "flags": { - "bits": 4241 + "bits": 4225 }, "char_set": 224, - "max_size": 262140 + "max_size": 1020 } }, { "ordinal": 3, - "name": "maybe_creator_user_token", + "name": "maybe_w2l_template_token", "type_info": { "type": "VarString", "flags": { @@ -5090,6 +3210,42 @@ }, { "ordinal": 4, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 392 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 5, + "name": "template_title", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, "name": "maybe_creator_username", "type_info": { "type": "VarString", @@ -5101,7 +3257,7 @@ } }, { - "ordinal": 5, + "ordinal": 8, "name": "maybe_creator_display_name", "type_info": { "type": "VarString", @@ -5113,7 +3269,67 @@ } }, { - "ordinal": 6, + "ordinal": 9, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "maybe_template_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "maybe_template_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 12, + "name": "maybe_template_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 13, + "name": "maybe_template_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 14, "name": "file_size_bytes", "type_info": { "type": "Long", @@ -5125,7 +3341,31 @@ } }, { - "ordinal": 7, + "ordinal": 15, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 17, "name": "duration_millis", "type_info": { "type": "Long", @@ -5137,7 +3377,7 @@ } }, { - "ordinal": 8, + "ordinal": 18, "name": "created_at", "type_info": { "type": "Timestamp", @@ -5149,7 +3389,7 @@ } }, { - "ordinal": 9, + "ordinal": 19, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -5159,6 +3399,30 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 20, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { @@ -5166,20 +3430,52 @@ }, "nullable": [ false, - false, + true, false, true, true, true, + true, + true, + true, + true, + true, + true, + true, + true, false, false, false, - false + false, + false, + false, + true, + true ] } }, - "b1ad806aa93f66cab358d7cdb405d57c643e2ff131cfec9f1ced55c2e6ca9511": { - "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "68376f750b8421c10d319fa1045a2f3ce8fb1d35a5fe6ec92b2022babafbde80": { + "query": "\nUPDATE users\nSET\n user_role_slug = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "6b66db376952ff4cfcea6ffe1f9ecd4462722843a3d19fcd4df16f0fb993decc": { + "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "73558104936aab6add2d9c789b368b5f155f1fe01d32e7314f2f120422866f54": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { "columns": [], "parameters": { @@ -5188,8 +3484,8 @@ "nullable": [] } }, - "b3f987707efca036012cb6cbdc3a2e104ccf8d41f2e1608bcca6e05b308b5696": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND w2l.is_mod_public_listing_approved IS TRUE\n ", + "7622292c05fe1b5832136762042780a12a02048166bb1b4492dfd627c48c8968": { + "query": "\nSELECT\n token as template_token\nFROM w2l_templates\nWHERE\n token = ?\n AND user_deleted_at IS NULL\n AND mod_deleted_at IS NULL\n ", "describe": { "columns": [ { @@ -5203,146 +3499,171 @@ "char_set": 224, "max_size": 128 } - }, + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + } + }, + "78006a7a9388557d53c9a9cc8aeec84496e87355501c5b412f788a2291077473": { + "query": "\nSELECT\n slug,\n name,\n\n can_use_tts,\n can_use_w2l,\n can_delete_own_tts_results,\n can_delete_own_w2l_results,\n can_delete_own_account,\n\n can_upload_tts_models,\n can_upload_w2l_templates,\n can_delete_own_tts_models,\n can_delete_own_w2l_templates,\n\n can_approve_w2l_templates,\n can_edit_other_users_profiles,\n can_edit_other_users_tts_models,\n can_edit_other_users_w2l_templates,\n can_delete_other_users_tts_models,\n can_delete_other_users_tts_results,\n can_delete_other_users_w2l_templates,\n can_delete_other_users_w2l_results,\n can_ban_users,\n can_delete_users,\n\n created_at,\n updated_at\n\nFROM user_roles\n ", + "describe": { + "columns": [ { - "ordinal": 1, - "name": "template_type", + "ordinal": 0, + "name": "slug", "type_info": { "type": "String", "flags": { - "bits": 393 + "bits": 4229 }, "char_set": 224, - "max_size": 28 + "max_size": 64 + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 } }, { "ordinal": 2, - "name": "creator_user_token", + "name": "can_use_tts", "type_info": { - "type": "VarString", + "type": "Tiny", "flags": { - "bits": 4233 + "bits": 1 }, - "char_set": 224, - "max_size": 128 + "char_set": 63, + "max_size": 1 } }, { "ordinal": 3, - "name": "creator_username", + "name": "can_use_w2l", "type_info": { - "type": "VarString", + "type": "Tiny", "flags": { - "bits": 4229 + "bits": 1 }, - "char_set": 224, - "max_size": 80 + "char_set": 63, + "max_size": 1 } }, { "ordinal": 4, - "name": "creator_display_name", + "name": "can_delete_own_tts_results", "type_info": { - "type": "VarString", + "type": "Tiny", "flags": { - "bits": 4225 + "bits": 1 }, - "char_set": 224, - "max_size": 80 + "char_set": 63, + "max_size": 1 } }, { "ordinal": 5, - "name": "updatable_slug", + "name": "can_delete_own_w2l_results", "type_info": { - "type": "VarString", + "type": "Tiny", "flags": { - "bits": 4229 + "bits": 1 }, - "char_set": 224, - "max_size": 256 + "char_set": 63, + "max_size": 1 } }, { "ordinal": 6, - "name": "title", + "name": "can_delete_own_account", "type_info": { - "type": "String", + "type": "Tiny", "flags": { - "bits": 4225 + "bits": 1 }, - "char_set": 224, - "max_size": 1020 + "char_set": 63, + "max_size": 1 } }, { "ordinal": 7, - "name": "frame_width", + "name": "can_upload_tts_models", "type_info": { - "type": "Long", + "type": "Tiny", "flags": { "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 1 } }, { "ordinal": 8, - "name": "frame_height", + "name": "can_upload_w2l_templates", "type_info": { - "type": "Long", + "type": "Tiny", "flags": { "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 1 } }, { "ordinal": 9, - "name": "duration_millis", + "name": "can_delete_own_tts_models", "type_info": { - "type": "Long", + "type": "Tiny", "flags": { "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 1 } }, { "ordinal": 10, - "name": "maybe_public_bucket_preview_image_object_name", + "name": "can_delete_own_w2l_templates", "type_info": { - "type": "VarString", + "type": "Tiny", "flags": { - "bits": 128 + "bits": 1 }, - "char_set": 224, - "max_size": 1020 + "char_set": 63, + "max_size": 1 } }, { "ordinal": 11, - "name": "maybe_public_bucket_preview_video_object_name", + "name": "can_approve_w2l_templates", "type_info": { - "type": "VarString", + "type": "Tiny", "flags": { - "bits": 128 + "bits": 1 }, - "char_set": 224, - "max_size": 1020 + "char_set": 63, + "max_size": 1 } }, { "ordinal": 12, - "name": "is_mod_public_listing_approved", + "name": "can_edit_other_users_profiles", "type_info": { "type": "Tiny", "flags": { - "bits": 0 + "bits": 1 }, "char_set": 63, "max_size": 1 @@ -5350,6 +3671,102 @@ }, { "ordinal": 13, + "name": "can_edit_other_users_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "can_edit_other_users_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "can_delete_other_users_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "can_delete_other_users_tts_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "can_delete_other_users_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 18, + "name": "can_delete_other_users_w2l_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 19, + "name": "can_ban_users", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 20, + "name": "can_delete_users", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 21, "name": "created_at", "type_info": { "type": "Timestamp", @@ -5361,7 +3778,7 @@ } }, { - "ordinal": 14, + "ordinal": 22, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -5387,26 +3804,244 @@ false, false, false, - true, - true, - true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, false, false ] } }, - "b4582bc4d8865cc82bf1e0fe15209db07d1922a29c274b872cd5f5eaee4b261d": { - "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = 'started',\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "7ab54466ca3021d90e0feb234acba56bf53d415dde69bc1c35120ed0c8dd1858": { + "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "on_success_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 6, + "name": "download_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 7, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 9, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 10, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 11, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 12, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 15, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], "parameters": { "Right": 1 }, - "nullable": [] + "nullable": [ + false, + false, + false, + true, + false, + false, + true, + false, + false, + false, + false, + false, + true, + false, + false, + true + ] } }, - "b62d4ca915206b8f956a2664c72e02832d38ab34fb3be50d9a77cf8145297689": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND w2l.is_mod_public_listing_approved IS TRUE\n AND users.username = ?\n ", + "7cd0ad5d7b9a2c393f799beadc1e0a6bc85d952e07a1b82a52115b99be40ff6d": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { @@ -5471,18 +4106,6 @@ }, { "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, "name": "title", "type_info": { "type": "String", @@ -5494,7 +4117,7 @@ } }, { - "ordinal": 7, + "ordinal": 6, "name": "frame_width", "type_info": { "type": "Long", @@ -5506,7 +4129,7 @@ } }, { - "ordinal": 8, + "ordinal": 7, "name": "frame_height", "type_info": { "type": "Long", @@ -5518,7 +4141,7 @@ } }, { - "ordinal": 9, + "ordinal": 8, "name": "duration_millis", "type_info": { "type": "Long", @@ -5530,7 +4153,7 @@ } }, { - "ordinal": 10, + "ordinal": 9, "name": "maybe_public_bucket_preview_image_object_name", "type_info": { "type": "VarString", @@ -5542,7 +4165,7 @@ } }, { - "ordinal": 11, + "ordinal": 10, "name": "maybe_public_bucket_preview_video_object_name", "type_info": { "type": "VarString", @@ -5554,19 +4177,19 @@ } }, { - "ordinal": 12, - "name": "is_mod_public_listing_approved", + "ordinal": 11, + "name": "is_public_listing_approved", "type_info": { "type": "Tiny", "flags": { - "bits": 0 + "bits": 8 }, "char_set": 63, "max_size": 1 } }, { - "ordinal": 13, + "ordinal": 12, "name": "created_at", "type_info": { "type": "Timestamp", @@ -5578,7 +4201,394 @@ } }, { - "ordinal": 14, + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] + } + }, + "8347b64356502ae32979d48680e6b2097e7f8c4cdc492b7285044856b58ba017": { + "query": "\nINSERT INTO w2l_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n maybe_w2l_template_token = ?,\n maybe_public_audio_bucket_hash = ?,\n maybe_public_audio_bucket_location = ?,\n\n maybe_original_audio_filename = ?,\n maybe_audio_mime_type = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n disable_end_bump = false,\n creator_set_visibility = \"public\",\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, + "865ce236444a36d597939660b14eb4b0cee05c70fdb73f31c2fb8cddddfef42c": { + "query": "\nINSERT INTO firehose_entries\nSET\n token = ?,\n event_type = ?,\n maybe_target_user_token = ?,\n maybe_target_entity_token = ?,\n maybe_created_entity_token = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + } + }, + "88328e04ce947d3f541951aada9cee25135aa950ab478cf0dc6de453bb079e17": { + "query": "\nSELECT\n events.token as event_token,\n events.event_type,\n events.maybe_target_user_token,\n users.username as maybe_target_username,\n users.display_name as maybe_target_display_name,\n users.email_gravatar_hash as maybe_target_user_gravatar_hash,\n events.maybe_target_entity_token,\n events.created_at,\n events.updated_at\nFROM firehose_entries as events\nLEFT OUTER JOIN users\nON events.maybe_target_user_token = users.token\nORDER BY events.id DESC\nLIMIT 25\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "event_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "event_type", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "maybe_target_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "maybe_target_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "maybe_target_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "maybe_target_user_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_target_entity_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 8, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + true, + true, + true, + true, + true, + false, + false + ] + } + }, + "886861e91c7e8af53eb856c949389a172b8d9e079f1c2070d115423907c47c50": { + "query": "\nINSERT INTO tts_model_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n download_url = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 8 + }, + "nullable": [] + } + }, + "8ba1925f85324f4bc73f809a825453196a95b07008d23b0ee66e8a73f73a3510": { + "query": "\nUPDATE w2l_templates\nSET\n user_deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "8c0c0ad0fafef712bb2eb269bcef8658efcd84154d6f6eeda26b0399abd641c4": { + "query": "\nUPDATE w2l_templates\nSET\n user_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "8d61bb77449a78044135a3d6e5009a3a27b4671b7e40501d318b05ae401f02a8": { + "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE email_address = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "password_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 63, + "max_size": 60 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false + ] + } + }, + "961f792c24bee6009e3e8a215561db932871a53c48839b8cb80292de00e7b2e1": { + "query": "\nUPDATE tts_results\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "99f1869443b8331b3099af44c53d6db4573fdfb2df5af56668f3256bf446b096": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 8, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -5602,16 +4612,1156 @@ false, false, false, + false + ] + } + }, + "9a879f76345ef5682b995f99026fac8b2f4c64c8ec4bb078187d1e53effcfbaf": { + "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address_creation = ?,\n creator_ip_address_last_update = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n maybe_public_bucket_preview_image_object_name = ?,\n maybe_public_bucket_preview_video_object_name = ?,\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 19 + }, + "nullable": [] + } + }, + "9aded23dc849b7b416458afc8976207553e864690e36a8ee7066569ff489dc17": { + "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_confirmed,\n users.email_gravatar_hash,\n\n users.disable_gravatar,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n\n users.user_role_slug,\n users.is_banned,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l,\n user_roles.can_delete_own_tts_results,\n user_roles.can_delete_own_w2l_results,\n user_roles.can_delete_own_account,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n user_roles.can_delete_own_tts_models,\n user_roles.can_delete_own_w2l_templates,\n\n user_roles.can_approve_w2l_templates,\n user_roles.can_edit_other_users_profiles,\n user_roles.can_edit_other_users_tts_models,\n user_roles.can_edit_other_users_w2l_templates,\n user_roles.can_delete_other_users_tts_models,\n user_roles.can_delete_other_users_tts_results,\n user_roles.can_delete_other_users_w2l_templates,\n user_roles.can_delete_other_users_w2l_results,\n user_roles.can_ban_users,\n user_roles.can_delete_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "email_confirmed", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 5, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 7, + "name": "auto_play_audio_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "auto_play_video_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 10, + "name": "is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 11, + "name": "can_use_tts", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "can_use_w2l", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "can_delete_own_tts_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "can_delete_own_w2l_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "can_delete_own_account", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "can_upload_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "can_upload_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 18, + "name": "can_delete_own_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 19, + "name": "can_delete_own_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 20, + "name": "can_approve_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 21, + "name": "can_edit_other_users_profiles", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 22, + "name": "can_edit_other_users_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 23, + "name": "can_edit_other_users_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 24, + "name": "can_delete_other_users_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 25, + "name": "can_delete_other_users_tts_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 26, + "name": "can_delete_other_users_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 27, + "name": "can_delete_other_users_w2l_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 28, + "name": "can_ban_users", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 29, + "name": "can_delete_users", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, false, false, true, true, true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + } + }, + "9b607b814cd7c445c2c46ec5c13dc23f3a6a771a3469a786ed46ec099577d95e": { + "query": "\nUPDATE\n ip_address_bans\nSET\n mod_user_token = ?,\n deleted_at = CURRENT_TIMESTAMP\nWHERE\n ip_address = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "9cef4989671fb55d8e732f6f3d5c16aea7a9b3e01fc6a3bc7a56d0b066f07730": { + "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address_creation = ?,\n creator_ip_address_last_update = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n file_size_bytes = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, + "9d4932baa4023accc2d28af98b58a47149d37c4937cd0171e6b955c09855c5f8": { + "query": "\nINSERT INTO w2l_inference_jobs\nSET\n maybe_w2l_template_token = ?,\n maybe_tts_inference_result_token = ?,\n maybe_public_audio_bucket_location = NULL,\n maybe_public_image_bucket_location = NULL,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + } + }, + "a125f183f7ae5627fc2118e7302cea73d967200ba1cbb12550f5880187b7db5e": { + "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "a22a487d4651ec72b04fad3747dc1c25e47382e5481223a6a4fcf8172a737fb3": { + "query": "\nINSERT INTO w2l_results\nSET\n token = ?,\n maybe_w2l_template_token = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n maybe_creator_synthetic_id = ?,\n\n public_bucket_video_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 11 + }, + "nullable": [] + } + }, + "a88f0e74e4d30a8defdcafeed432451f1e4b32b2f8fbfa55d815b0c7cef40761": { + "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n is_banned,\n disable_gravatar,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n cashapp_username,\n website_url,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "profile_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "profile_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 6, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 7, + "name": "is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "discord_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 10, + "name": "twitch_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 11, + "name": "twitter_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 12, + "name": "patreon_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 13, + "name": "github_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 14, + "name": "cashapp_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 15, + "name": "website_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 16, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + false + ] + } + }, + "ab6f25eb0f6340d49aa705835ed428434d8b6ab14b0740f6a5ebe7841b9ece20": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "adf2d5eb967fa2bc00fad1c8fcc2e1bf4384d39dc54e8240d78bb4ca6fcc3409": { + "query": "\nUPDATE w2l_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "af6e3f4b8b29a9e26ae422bee2e24e1d926990c5111149d26dbe4ea97dceb87f": { + "query": "\nSELECT\n ip_bans.ip_address,\n ip_bans.maybe_target_user_token,\n users.username as maybe_target_username,\n ip_bans.mod_user_token,\n ip_bans.mod_notes,\n ip_bans.created_at,\n ip_bans.updated_at\nFROM\n ip_address_bans AS ip_bans\nLEFT OUTER JOIN users\n ON ip_bans.maybe_target_user_token = users.token\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 1, + "name": "maybe_target_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "maybe_target_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "mod_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "mod_notes", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + true, + true, + false, + false, false, false ] } }, + "b1ad806aa93f66cab358d7cdb405d57c643e2ff131cfec9f1ced55c2e6ca9511": { + "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "b1d7e1c405a33c391954167a3527580ec7fefac0055297150108a92f3945941f": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n AND w2l_results.user_deleted_at IS NULL\n AND w2l_results.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "w2l_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "public_bucket_video_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 392 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 5, + "name": "template_title", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 9, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "maybe_template_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "maybe_template_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 12, + "name": "maybe_template_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 13, + "name": "maybe_template_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 14, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 15, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 17, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 18, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, + "b257c267dfe063995374786d3004f215e77e8a32843c024645da6fa435507dce": { + "query": "\nINSERT INTO\n ip_address_bans\nSET\n ip_address = ?,\n maybe_target_user_token = ?,\n mod_user_token = ?,\n mod_notes = ?\nON DUPLICATE KEY UPDATE\n expires_at = NULL,\n deleted_at = NULL,\n ip_address = ?,\n maybe_target_user_token = ?,\n mod_user_token = ?,\n mod_notes = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 8 + }, + "nullable": [] + } + }, + "b6a53361eda0f0f0517cc39b9bc5b7ab867cc8a1bcc7bc81c7d29c51193e3bbf": { + "query": "\nUPDATE tts_models\nSET\n is_public_listing_approved = ?,\n is_locked_from_user_modification = ?,\n is_locked_from_use = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + } + }, "b7535ecf2e63b7f02b5b96f833fd8cd28285b24473df6f89795ff360c1723215": { "query": "\nSELECT *\nFROM w2l_template_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { @@ -5690,30 +5840,6 @@ }, { "ordinal": 6, - "name": "maybe_subject_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "maybe_actor_subject_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, "name": "download_url", "type_info": { "type": "VarString", @@ -5725,19 +5851,7 @@ } }, { - "ordinal": 9, - "name": "download_url_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 48 - } - }, - { - "ordinal": 10, + "ordinal": 7, "name": "creator_user_token", "type_info": { "type": "VarString", @@ -5749,7 +5863,7 @@ } }, { - "ordinal": 11, + "ordinal": 8, "name": "creator_ip_address", "type_info": { "type": "VarString", @@ -5761,7 +5875,7 @@ } }, { - "ordinal": 12, + "ordinal": 9, "name": "creator_set_visibility", "type_info": { "type": "String", @@ -5773,7 +5887,7 @@ } }, { - "ordinal": 13, + "ordinal": 10, "name": "status", "type_info": { "type": "String", @@ -5785,7 +5899,7 @@ } }, { - "ordinal": 14, + "ordinal": 11, "name": "attempt_count", "type_info": { "type": "Long", @@ -5797,7 +5911,7 @@ } }, { - "ordinal": 15, + "ordinal": 12, "name": "failure_reason", "type_info": { "type": "VarString", @@ -5809,7 +5923,7 @@ } }, { - "ordinal": 16, + "ordinal": 13, "name": "created_at", "type_info": { "type": "Timestamp", @@ -5821,7 +5935,7 @@ } }, { - "ordinal": 17, + "ordinal": 14, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -5833,7 +5947,7 @@ } }, { - "ordinal": 18, + "ordinal": 15, "name": "retry_at", "type_info": { "type": "Timestamp", @@ -5856,9 +5970,6 @@ false, false, true, - true, - true, - false, false, false, false, @@ -5871,12 +5982,216 @@ ] } }, - "bd151e17d1dd65c25ebc689a1d321ac14b8d5d8f1c9e8426c0c92ed278595de2": { - "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n file_size_bytes = ?\n ", + "baf058c93336325a642ab07a617c69eae9d2c7467bf3a8248e814063a6c36001": { + "query": "\nUPDATE w2l_templates\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { "columns": [], "parameters": { - "Right": 9 + "Right": 2 + }, + "nullable": [] + } + }, + "bc9c6b57b2578b8163956d11a5f6a9a77ddab6098d2c4207e1a70502db30c9d1": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.private_bucket_hash,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\nAND w2l.user_deleted_at IS NULL\nAND w2l.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "private_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + } + }, + "bee13160674414391587f56e8fb0aa3629e3dfcead0afa5759bff1e161f1bb6e": { + "query": "\nUPDATE w2l_templates\nSET\n is_public_listing_approved = ?,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 }, "nullable": [] } @@ -5891,8 +6206,38 @@ "nullable": [] } }, - "c1a4f3667de265a6f501b141cc35c44ce7b3703627c7d8336d0ef8445962ed34": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.updatable_slug,\n tts.title,\n\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.updatable_slug = ?\nAND tts.deleted_at IS NULL\n ", + "c63726afcf8d53a24597aad18bb44285eae8f5d056011811b066226f7db0198a": { + "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + } + }, + "ca5efc3be0f09649df2add5270c06544133288a95ffcb9a0589efd4fe345efe7": { + "query": "\nINSERT INTO badges ( slug, title, description, image_url )\nVALUES ( ?, ?, ?, ? )\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 4 + }, + "nullable": [] + } + }, + "cae002e97a3309e0e8361d2a08158c5a1dc1e5b5123c9c5e2c5f8e5e93d07387": { + "query": "\nUPDATE w2l_templates\nSET\n mod_deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "cc51b77e88c2ee5c046ea2f2287b9e9770743d50b0db90481662e5c1118885c5": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { @@ -5921,18 +6266,6 @@ }, { "ordinal": 2, - "name": "text_preprocessing_algorithm", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 72 - } - }, - { - "ordinal": 3, "name": "creator_user_token", "type_info": { "type": "VarString", @@ -5944,7 +6277,7 @@ } }, { - "ordinal": 4, + "ordinal": 3, "name": "creator_username", "type_info": { "type": "VarString", @@ -5956,7 +6289,7 @@ } }, { - "ordinal": 5, + "ordinal": 4, "name": "creator_display_name", "type_info": { "type": "VarString", @@ -5968,19 +6301,7 @@ } }, { - "ordinal": 6, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 7, + "ordinal": 5, "name": "title", "type_info": { "type": "VarString", @@ -5992,7 +6313,19 @@ } }, { - "ordinal": 8, + "ordinal": 6, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 7, "name": "created_at", "type_info": { "type": "Timestamp", @@ -6004,7 +6337,7 @@ } }, { - "ordinal": 9, + "ordinal": 8, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -6028,28 +6361,29 @@ false, false, false, - false, false ] } }, - "ca5efc3be0f09649df2add5270c06544133288a95ffcb9a0589efd4fe345efe7": { - "query": "\nINSERT INTO badges ( slug, title, description, image_url )\nVALUES ( ?, ?, ?, ? )\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 4 - }, - "nullable": [] - } - }, - "caa1b771bc360614445df502055125051fed57e758ea2bd81cf303bfb03c2c9f": { - "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND w2l_results.is_mod_hidden_from_public IS FALSE\n AND w2l_templates.is_mod_public_listing_approved IS TRUE\n ", + "ccd9df6bff43a6c7554e18b4866bccd56a464117477edfa30bb0fc58309eed93": { + "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n model_token,\n raw_inference_text,\n\n creator_ip_address,\n maybe_creator_user_token,\n creator_set_visibility,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM tts_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "w2l_result_token", + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "inference_job_token", "type_info": { "type": "VarString", "flags": { @@ -6060,8 +6394,56 @@ } }, { - "ordinal": 1, - "name": "maybe_tts_inference_result_token", + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", "type_info": { "type": "VarString", "flags": { @@ -6072,127 +6454,55 @@ } }, { - "ordinal": 2, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "template_type", + "ordinal": 7, + "name": "creator_set_visibility", "type_info": { "type": "String", "flags": { - "bits": 392 + "bits": 385 }, "char_set": 224, "max_size": 28 } }, { - "ordinal": 4, - "name": "template_title", + "ordinal": 8, + "name": "status", "type_info": { "type": "String", "flags": { - "bits": 4224 + "bits": 393 }, "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 5, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 8, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 + "max_size": 64 } }, { "ordinal": 9, - "name": "frame_width", + "name": "attempt_count", "type_info": { "type": "Long", "flags": { "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 3 } }, { "ordinal": 10, - "name": "frame_height", + "name": "failure_reason", "type_info": { - "type": "Long", + "type": "VarString", "flags": { - "bits": 1 + "bits": 128 }, - "char_set": 63, - "max_size": 10 + "char_set": 224, + "max_size": 2048 } }, { "ordinal": 11, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 12, "name": "created_at", "type_info": { "type": "Timestamp", @@ -6204,7 +6514,7 @@ } }, { - "ordinal": 13, + "ordinal": 12, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -6214,29 +6524,51 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 13, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { - "Right": 0 + "Right": 1 }, "nullable": [ false, - true, - true, - true, - true, - true, - true, - true, false, false, false, false, false, - false + true, + false, + false, + false, + true, + false, + false, + true ] } }, + "cf0b037e789ee09022b25f3381b0263c5cc6b59f3f328de4aa211bdb23a62629": { + "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + } + }, "d12757abef811cd6951e0e1427fdb496a8931b547519d320f5ade56007d3eb5d": { "query": "\nSELECT\n id,\n status\nFROM w2l_template_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", "describe": { @@ -6573,8 +6905,95 @@ ] } }, - "d15d602c1089dd5aaa75bc0f97697e9a51a058496efbcaa1a59ffef72dfcd2fa": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.deleted_at IS NULL\n ", + "dbce51eb1c80dded902c9509e50713483c7131f115e8ec10646e99e88110278b": { + "query": "\nUPDATE tts_results\nSET\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "de2d38b505ecff7b8b73996888ff6a904c5a0fab38e1c0b6156929c812b94302": { + "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n user_roles.slug as user_role_slug,\n user_roles.name as user_role_name\nFROM\n users\nJOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE\n user_roles.slug != 'user'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "user_role_slug", + "type_info": { + "type": "String", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 4, + "name": "user_role_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false + ] + } + }, + "dfc7d0ff70e135e1c4d50c7655b6064e5034f16eacd9bb94070933a68815d270": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { @@ -6639,18 +7058,6 @@ }, { "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, "name": "title", "type_info": { "type": "VarString", @@ -6661,6 +7068,18 @@ "max_size": 1020 } }, + { + "ordinal": 6, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, { "ordinal": 7, "name": "created_at", @@ -6702,14 +7121,198 @@ ] } }, - "d5d3e1bd935d32639ad7398936f4c8ee9f3caff6c0789db29b45a81555d1bdce": { - "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = 'started',\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "e3a0bd8073c3c0d84b9ab0e2e1ad8371f14b10801b7e4658b73c42ce1b27fa0b": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.is_public_listing_approved IS TRUE\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], "parameters": { "Right": 1 }, - "nullable": [] + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] } }, "e73192a3e763a287d5737821443c5a81649f17e39978497b99c52e1e896a75f6": { @@ -6750,259 +7353,33 @@ ] } }, - "e76b9212a263b286e5a012bcf725c33580f9c4a8da800f0da124a1ceff3086b0": { - "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n banned,\n avatar_public_bucket_hash,\n disable_gravatar,\n maybe_preferred_tts_model_token,\n maybe_preferred_w2l_template_token,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n created_at\nFROM users\nWHERE users.username = ?\nAND users.deleted_at IS NULL\n ", + "e9b73703961516622f748fa597b62af6b991a7a36acf9901ca6d315ed2440674": { + "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { - "columns": [ - { - "ordinal": 0, - "name": "user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "email_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "profile_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 5, - "name": "profile_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 6, - "name": "user_role_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 7, - "name": "banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "avatar_public_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 9, - "name": "disable_gravatar", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "maybe_preferred_tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 11, - "name": "maybe_preferred_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 12, - "name": "discord_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 13, - "name": "twitch_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 14, - "name": "twitter_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 15, - "name": "patreon_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 16, - "name": "github_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 17, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - } - ], + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + } + }, + "e9ec861d6eb135370f9baf3a866282ec5686d4b299fc4ec10900ab32ea8577d8": { + "query": "\nUPDATE w2l_results\nSET\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], "parameters": { "Right": 1 }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - true, - false, - true, - true, - true, - true, - true, - true, - true, - false - ] + "nullable": [] } }, - "ee3925235a4f3b0a5d6ec0053b15fb9a2d6af28ca1039e2563d7d53430078986": { - "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.model_token as tts_model_token,\n tts_results.inference_text as inference_text,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND tts_results.is_mod_hidden_from_public IS FALSE\n AND tts_models.is_mod_disabled IS FALSE\n AND users.username = ?\n ", + "ed67238afe48c159443d215cd1fe2f0264ad840768f789d068527ae542b08194": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "tts_result_token", + "name": "model_token", "type_info": { "type": "VarString", "flags": { @@ -7014,7 +7391,19 @@ }, { "ordinal": 1, - "name": "tts_model_token", + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", "type_info": { "type": "VarString", "flags": { @@ -7024,37 +7413,25 @@ "max_size": 128 } }, - { - "ordinal": 2, - "name": "inference_text", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, { "ordinal": 3, - "name": "maybe_creator_user_token", + "name": "creator_username", "type_info": { "type": "VarString", "flags": { - "bits": 4228 + "bits": 4229 }, "char_set": 224, - "max_size": 128 + "max_size": 80 } }, { "ordinal": 4, - "name": "maybe_creator_username", + "name": "creator_display_name", "type_info": { "type": "VarString", "flags": { - "bits": 4228 + "bits": 4225 }, "char_set": 224, "max_size": 80 @@ -7062,42 +7439,18 @@ }, { "ordinal": 5, - "name": "maybe_creator_display_name", + "name": "title", "type_info": { "type": "VarString", "flags": { - "bits": 4224 + "bits": 4225 }, "char_set": 224, - "max_size": 80 + "max_size": 1020 } }, { "ordinal": 6, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 7, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, "name": "created_at", "type_info": { "type": "Timestamp", @@ -7109,7 +7462,7 @@ } }, { - "ordinal": 9, + "ordinal": 7, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -7122,15 +7475,13 @@ } ], "parameters": { - "Right": 1 + "Right": 0 }, "nullable": [ false, false, false, - true, - true, - true, + false, false, false, false, @@ -7138,276 +7489,14 @@ ] } }, - "f00402e757b7f5d8ded60e78ac47d93ccf4c71a2acba9dddcedfed0ac4d2e3a5": { - "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND w2l_results.token = ?\n ", + "ee05f95e45f7aaa5811d70b1f00d3f40a361536dfc6451affcad97586dc3338f": { + "query": "\nUPDATE w2l_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { - "columns": [ - { - "ordinal": 0, - "name": "w2l_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "maybe_tts_inference_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "public_bucket_video_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 3, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 392 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 5, - "name": "template_title", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 8, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 9, - "name": "maybe_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 10, - "name": "maybe_template_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 11, - "name": "maybe_template_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 12, - "name": "maybe_template_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 13, - "name": "maybe_template_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 14, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 15, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 16, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 17, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 18, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], + "columns": [], "parameters": { "Right": 1 }, - "nullable": [ - false, - true, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false - ] + "nullable": [] } }, "f19300f4e8434c2669dac062e5a79c5ca71767c7380aab0ab40b51be8ff449d8": { @@ -7420,13 +7509,13 @@ "nullable": [] } }, - "f6e625dde66da14091591f321eb65b0137fceb7168305bf17806f1876a644954": { - "query": "\nSELECT\n token as template_token\nFROM w2l_templates\nWHERE token = ?\nAND deleted_at IS NULL\n ", + "fc0045b6fe338a903fa9f35307457d8dbb32c5838a4783fddf0c9beef7fdb4a5": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,\n\n jobs.model_token,\n tts.tts_model_type,\n tts.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_inference_jobs as jobs\nJOIN tts_models as tts\n ON tts.token = jobs.model_token\nLEFT OUTER JOIN tts_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "template_token", + "name": "job_token", "type_info": { "type": "VarString", "flags": { @@ -7435,22 +7524,139 @@ "char_set": 224, "max_size": 128 } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 3, + "name": "maybe_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 7, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { "Right": 1 }, "nullable": [ + false, + false, + false, + true, + true, + false, + false, + false, + false, false ] } }, - "fc8641968fc13589daceebeded8a8d0446309f8c2961066450f913c87156bbc6": { - "query": "\nUPDATE w2l_templates\nSET\n is_mod_public_listing_approved = ?,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "ffa85f7c65101a43653c724479ee080e0a74577d2ae37863b039f3a452493d1e": { + "query": "\nUPDATE w2l_results\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { "columns": [], "parameters": { - "Right": 3 + "Right": 1 }, "nullable": [] } From af11a1eddfd2667b0406bd27ae571d7bcb3c9677 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 14 Jun 2021 02:46:25 -0400 Subject: [PATCH 0404/1042] add mod details to ip address ban list --- .../endpoints/moderation/ip_bans/list_ip_bans.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs index 36aaef6854..b9f97fac1f 100644 --- a/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs +++ b/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs @@ -37,6 +37,8 @@ pub struct IpBanRecordForList { pub maybe_target_user_token: Option, pub maybe_target_username: Option, pub mod_user_token: String, + pub mod_username: String, + pub mod_display_name: String, pub mod_notes: String, pub created_at: DateTime, pub updated_at: DateTime, @@ -102,16 +104,23 @@ pub async fn list_ip_bans_handler( r#" SELECT ip_bans.ip_address, + ip_bans.maybe_target_user_token, - users.username as maybe_target_username, + banned_users.username as maybe_target_username, + ip_bans.mod_user_token, + mod_users.username as mod_username, + mod_users.display_name as mod_display_name, + ip_bans.mod_notes, ip_bans.created_at, ip_bans.updated_at FROM ip_address_bans AS ip_bans -LEFT OUTER JOIN users - ON ip_bans.maybe_target_user_token = users.token +LEFT OUTER JOIN users as banned_users + ON ip_bans.maybe_target_user_token = banned_users.token +JOIN users as mod_users + ON ip_bans.mod_user_token = mod_users.token "#, ) .fetch_all(&server_state.mysql_pool) From 92fda195ec6d3517a11f052c8962268e9102bdda Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 14 Jun 2021 03:12:16 -0400 Subject: [PATCH 0405/1042] get single IP ban info --- .../moderation/ip_bans/get_ip_ban.rs | 173 ++++++++++++++++++ .../moderation/ip_bans/list_ip_bans.rs | 2 +- .../endpoints/moderation/ip_bans/mod.rs | 1 + src/main.rs | 6 + 4 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs diff --git a/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs b/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs new file mode 100644 index 0000000000..9ed864f63e --- /dev/null +++ b/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs @@ -0,0 +1,173 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetIpBanPathInfo { + ip_address: String, +} + +#[derive(Serialize)] +pub struct GetIpBanResponse { + pub success: bool, + pub ip_address_ban: IpBanRecord, +} + +#[derive(Serialize)] +pub struct IpBanRecord { + pub ip_address: String, + pub maybe_target_user_token: Option, + pub maybe_target_username: Option, + pub mod_user_token: String, + pub mod_username: String, + pub mod_display_name: String, + pub mod_notes: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug, Display)] +pub enum GetIpBanError { + BadInput(String), + ServerError, + NotFound, + Unauthorized, +} + +impl ResponseError for GetIpBanError { + fn status_code(&self) -> StatusCode { + match *self { + GetIpBanError::BadInput(_) => StatusCode::BAD_REQUEST, + GetIpBanError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + GetIpBanError::NotFound => StatusCode::NOT_FOUND, + GetIpBanError::Unauthorized => StatusCode::UNAUTHORIZED, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetIpBanError::BadInput(reason) => reason.to_string(), + GetIpBanError::ServerError => "server error".to_string(), + GetIpBanError::NotFound => "not found".to_string(), + GetIpBanError::Unauthorized => "unauthorized".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn get_ip_ban_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetIpBanError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(GetIpBanError::Unauthorized); + } + }; + + if !user_session.can_ban_users { + warn!("user is not allowed to view bans: {}", user_session.user_token); + return Err(GetIpBanError::Unauthorized); + } + + let ip_address = path.ip_address.trim(); + + // NB: Lookup failure is Err(RowNotFound). + let maybe_result = sqlx::query_as!( + IpBanRecord, + r#" +SELECT + ip_bans.ip_address, + + ip_bans.maybe_target_user_token, + banned_users.username as maybe_target_username, + + ip_bans.mod_user_token, + mod_users.username as mod_username, + mod_users.display_name as mod_display_name, + + ip_bans.mod_notes, + ip_bans.created_at, + ip_bans.updated_at +FROM + ip_address_bans AS ip_bans +LEFT OUTER JOIN users as banned_users + ON ip_bans.maybe_target_user_token = banned_users.token +JOIN users as mod_users + ON ip_bans.mod_user_token = mod_users.token +WHERE + ip_bans.ip_address = ? +LIMIT 1 + "#, + ip_address + ) + .fetch_one(&server_state.mysql_pool) + .await; + + let result : IpBanRecord = match maybe_result { + Ok(result) => { + result + }, + Err(err) => { + match err { + RowNotFound => { + return Err(GetIpBanError::NotFound); + }, + _ => { + warn!("get ip ban db error: {:?}", err); + return Err(GetIpBanError::ServerError); + } + } + } + }; + + let response = GetIpBanResponse { + success: true, + ip_address_ban: result, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetIpBanError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs index b9f97fac1f..d1f146b48b 100644 --- a/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs +++ b/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs @@ -94,7 +94,7 @@ pub async fn list_ip_bans_handler( }; if !user_session.can_ban_users { - warn!("user is not allowed to delete bans: {}", user_session.user_token); + warn!("user is not allowed to see bans: {}", user_session.user_token); return Err(ListIpBansError::Unauthorized); } diff --git a/src/http_server/endpoints/moderation/ip_bans/mod.rs b/src/http_server/endpoints/moderation/ip_bans/mod.rs index 99c0855390..cc016c5c2b 100644 --- a/src/http_server/endpoints/moderation/ip_bans/mod.rs +++ b/src/http_server/endpoints/moderation/ip_bans/mod.rs @@ -1,3 +1,4 @@ pub mod add_ip_ban; pub mod delete_ip_ban; +pub mod get_ip_ban; pub mod list_ip_bans; diff --git a/src/main.rs b/src/main.rs index 8409492c8f..9bfcd83e2a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -90,6 +90,7 @@ use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_use_count_handler; use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_template_use_count_handler; +use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -329,6 +330,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::post().to(add_ip_ban_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/{ip_address}") + .route(web::get().to(get_ip_ban_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/{ip_address}/delete") .route(web::post().to(delete_ip_ban_handler)) From f25d731fc468ad669595c298ff0439ec85f1fb83 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 14 Jun 2021 03:33:31 -0400 Subject: [PATCH 0406/1042] don't show deleted bans --- debug/mysql/ip_bans.sh | 7 +++++++ .../endpoints/moderation/ip_bans/list_ip_bans.rs | 2 ++ 2 files changed, 9 insertions(+) create mode 100755 debug/mysql/ip_bans.sh diff --git a/debug/mysql/ip_bans.sh b/debug/mysql/ip_bans.sh new file mode 100755 index 0000000000..b543e80a6f --- /dev/null +++ b/debug/mysql/ip_bans.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u storyteller \ + -ppassword \ + -h localhost \ + -D storyteller \ + -e "select * from ip_address_bans\G" diff --git a/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs index d1f146b48b..59cabe5bfd 100644 --- a/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs +++ b/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs @@ -121,6 +121,8 @@ LEFT OUTER JOIN users as banned_users ON ip_bans.maybe_target_user_token = banned_users.token JOIN users as mod_users ON ip_bans.mod_user_token = mod_users.token +WHERE + ip_bans.deleted_at IS NULL "#, ) .fetch_all(&server_state.mysql_pool) From 8776893774d3a6de1fd645596c97126f51190755 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 14 Jun 2021 04:20:33 -0400 Subject: [PATCH 0407/1042] ip address validation --- .../endpoints/moderation/ip_bans/add_ip_ban.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs b/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs index da0480e9a3..1788d140e3 100644 --- a/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs +++ b/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs @@ -22,6 +22,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::validations::ip_addresses::validate_moderator_provided_ip_address; #[derive(Deserialize)] pub struct AddIpBanRequest { @@ -85,6 +86,13 @@ pub async fn add_ip_ban_handler( return Err(AddIpBanError::Unauthorized); } + let ip_address = request.ip_address.trim(); + + if let Err(e) = validate_moderator_provided_ip_address(&ip_address) { + warn!("Bad ip address: {}", e); + return Err(AddIpBanError::BadInput(e.to_string())); + } + info!("Creating ban..."); let query_result = sqlx::query!( @@ -110,7 +118,7 @@ ON DUPLICATE KEY UPDATE &user_session.user_token, &request.mod_notes, // Update - &request.ip_address, + &ip_address, &request.maybe_target_user_token, &user_session.user_token, &request.mod_notes, From c8818e7e3b05720cc28701365010c829ed4e8215 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 15 Jun 2021 00:12:54 -0400 Subject: [PATCH 0408/1042] add ip ban polling thread (not yet wired up to executor) --- src/common_queries/list_ip_bans.rs | 66 +++++++++++++++++++++++++++ src/common_queries/mod.rs | 1 + src/main.rs | 1 + src/threads/ip_banlist_set.rs | 39 ++++++++++++++++ src/threads/mod.rs | 2 + src/threads/poll_ip_banlist_thread.rs | 33 ++++++++++++++ 6 files changed, 142 insertions(+) create mode 100644 src/common_queries/list_ip_bans.rs create mode 100644 src/threads/ip_banlist_set.rs create mode 100644 src/threads/mod.rs create mode 100644 src/threads/poll_ip_banlist_thread.rs diff --git a/src/common_queries/list_ip_bans.rs b/src/common_queries/list_ip_bans.rs new file mode 100644 index 0000000000..98e12b8368 --- /dev/null +++ b/src/common_queries/list_ip_bans.rs @@ -0,0 +1,66 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::util::anyhow_result::AnyhowResult; +use log::warn; +use sqlx::MySqlPool; + +#[derive(Serialize)] +pub struct IpBanRecordForList { + pub ip_address: String, + pub maybe_target_user_token: Option, + pub maybe_target_username: Option, + pub mod_user_token: String, + pub mod_username: String, + pub mod_display_name: String, + pub mod_notes: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +// NB: This is unconstrained because I don't anticipate a huge number of IP bans at launch +// We can reengineer with growth. +pub async fn list_ip_bans(mysql_pool: &MySqlPool) -> AnyhowResult> { + let maybe_results = sqlx::query_as!( + IpBanRecordForList, + r#" +SELECT + ip_bans.ip_address, + + ip_bans.maybe_target_user_token, + banned_users.username as maybe_target_username, + + ip_bans.mod_user_token, + mod_users.username as mod_username, + mod_users.display_name as mod_display_name, + + ip_bans.mod_notes, + ip_bans.created_at, + ip_bans.updated_at +FROM + ip_address_bans AS ip_bans +LEFT OUTER JOIN users as banned_users + ON ip_bans.maybe_target_user_token = banned_users.token +JOIN users as mod_users + ON ip_bans.mod_user_token = mod_users.token +WHERE + ip_bans.deleted_at IS NULL + "#, + ) + .fetch_all(mysql_pool) + .await; + + match maybe_results { + Err(err) => { + match err { + RowNotFound => { + Ok(Vec::new()) + }, + _ => { + warn!("list ip bans db error: {:?}", err); + Err(anyhow!("error with query: {:?}", err)) + } + } + }, + Ok(results) => Ok(results), + } +} diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index 794cb9bf6d..a47c1dfc9f 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -1,5 +1,6 @@ pub mod badges; pub mod firehose_publisher; +pub mod list_ip_bans; pub mod list_tts_inference_results; pub mod list_tts_models; pub mod list_user_roles; diff --git a/src/main.rs b/src/main.rs index 9bfcd83e2a..ccef2a34e4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,6 +21,7 @@ pub mod database_helpers; pub mod http_server; pub mod server_state; pub mod shared_constants; +pub mod threads; pub mod util; pub mod validations; diff --git a/src/threads/ip_banlist_set.rs b/src/threads/ip_banlist_set.rs new file mode 100644 index 0000000000..a585c1e64a --- /dev/null +++ b/src/threads/ip_banlist_set.rs @@ -0,0 +1,39 @@ +use anyhow::anyhow; +use crate::AnyhowResult; +use std::collections::HashSet; +use std::collections::hash_map::RandomState; +use std::sync::{RwLock, PoisonError, RwLockWriteGuard, RwLockReadGuard, Arc}; + +#[derive(Clone)] +pub struct IpBanlistSet { + pub ip_banlist: Arc>>, +} + +impl IpBanlistSet { + + pub fn new() -> Self { + Self { + ip_banlist: Arc::new(RwLock::new(HashSet::new())), + } + } + + pub fn replace_list(&self, ip_banlist: HashSet) -> AnyhowResult<()> { + match self.ip_banlist.write() { + Err(_) => Err(anyhow!("Can't write lock")), + Ok(mut lock) => { + *lock = ip_banlist; + Ok(()) + }, + } + } + + pub fn is_banned(&self, ip_address: String) -> AnyhowResult { + match self.ip_banlist.read() { + Err(_) => Err(anyhow!("Can't read lock")), + Ok(read) => { + let result = read.contains(&ip_address); + Ok(result) + }, + } + } +} \ No newline at end of file diff --git a/src/threads/mod.rs b/src/threads/mod.rs new file mode 100644 index 0000000000..0fdb205acb --- /dev/null +++ b/src/threads/mod.rs @@ -0,0 +1,2 @@ +pub mod ip_banlist_set; +pub mod poll_ip_banlist_thread; \ No newline at end of file diff --git a/src/threads/poll_ip_banlist_thread.rs b/src/threads/poll_ip_banlist_thread.rs new file mode 100644 index 0000000000..9203834a08 --- /dev/null +++ b/src/threads/poll_ip_banlist_thread.rs @@ -0,0 +1,33 @@ +use crate::common_queries::list_ip_bans::list_ip_bans; +use crate::threads::ip_banlist_set::IpBanlistSet; +use log::warn; +use sqlx::MySqlPool; +use std::collections::HashSet; +use std::thread; +use std::time::Duration; +use anyhow::Error; + +pub async fn poll_ip_bans(ip_banlist_set: IpBanlistSet, mysql_pool: &MySqlPool) { + + loop { + let bans = match list_ip_bans(mysql_pool).await { + Ok(bans) => bans, + Err(e) => { + warn!("Error polling IP bans: {:?}", e); + thread::sleep(Duration::from_millis(30_000)); + continue; + } + }; + + let ip_addresses = bans.iter() + .map(|record| record.ip_address.clone()) + .collect::>(); + + match ip_banlist_set.replace_list(ip_addresses) { + Ok(_) => {} + Err(e) => { + warn!("error replacing banlist: {:?}", e); + } + } + } +} From e5d652878866c0bec17cfa6c822132fbc48a7f77 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 15 Jun 2021 01:38:04 -0400 Subject: [PATCH 0409/1042] this works, but takes 30 seconds! --- src/main.rs | 56 ++++++++++++++++++++ src/server_state.rs | 3 ++ src/threads/poll_ip_banlist_thread.rs | 74 +++++++++++++++++++++++++-- 3 files changed, 130 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index ccef2a34e4..90fcf67152 100644 --- a/src/main.rs +++ b/src/main.rs @@ -92,6 +92,11 @@ use std::sync::Arc; use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_use_count_handler; use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_template_use_count_handler; use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_handler; +use crate::threads::ip_banlist_set::IpBanlistSet; +use crate::threads::poll_ip_banlist_thread::poll_ip_bans; +use tokio::signal::unix::{Signal, signal}; +use tokio::signal::unix::SignalKind; +use std::sync::atomic::{AtomicBool, Ordering}; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -212,6 +217,43 @@ async fn main() -> AnyhowResult<()> { easyenv::get_env_string_or_default("SORT_KEY_SECRET", "webscale"); let sort_key_crypto = SortKeyCrypto::new(&sort_key_crypto_secret); + let ip_banlist = IpBanlistSet::new(); + + // Set up signal handlers + //let ctrl_c = tokio::task::spawn(tokio::signal::ctrl_c()); + //let (sighup, sigterm, sigquit) = ( + // tokio::task::spawn(hangup()), + // tokio::task::spawn(terminate()), + // tokio::task::spawn(quit()), + //); + + //let sigint = SignalKind::interrupt(); + //let sigterm = SignalKind::terminate(); + //let sigint = Signal::new(SIGINT).flatten_stream(); + //let sigterm = Signal::new(SIGTERM).flatten_stream(); + + + //let stream = sigint.select(sigterm); + + let shutdown = Arc::new(AtomicBool::new(false)); + + // Start polling thread. + let pool2 = pool.clone(); + let ip_banlist2 = ip_banlist.clone(); + let shutdown2 = shutdown.clone(); + + let join = tokio::task::spawn(async { + //poll_ip_bans(ip_banlist2, pool2, ctrl_c, sighup, sigterm, sigquit).await; + shutdown_signal(shutdown2).await + }); + + + let join2 = tokio::task::spawn(async { + //poll_ip_bans(ip_banlist2, pool2, ctrl_c, sighup, sigterm, sigquit).await; + poll_ip_bans(ip_banlist2, pool2, shutdown).await; + }); + + let server_state = ServerState { env_config: EnvConfig { num_workers, @@ -230,6 +272,7 @@ async fn main() -> AnyhowResult<()> { public_bucket_client, audio_uploads_bucket_root, sort_key_crypto, + ip_banlist, }; serve(server_state) @@ -237,6 +280,19 @@ async fn main() -> AnyhowResult<()> { Ok(()) } +async fn shutdown_signal(mut shutdown: Arc) { + // Wait for the CTRL+C signal + info!("Waiting for Ctrl+C"); + //let mut stream = signal(SignalKind::terminate())?; + //stream.recv().await; + tokio::signal::ctrl_c() + .await + .expect("failed to install CTRL+C signal handler"); + + info!("Shutdown recieved!"); + shutdown.store(true, Ordering::SeqCst); +} + pub async fn serve(server_state: ServerState) -> AnyhowResult<()> { let bind_address = server_state.env_config.bind_address.clone(); diff --git a/src/server_state.rs b/src/server_state.rs index d62cf9a7c6..8f80606658 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -5,6 +5,7 @@ use crate::util::buckets::bucket_client::BucketClient; use crate::util::encrypted_sort_id::SortKeyCrypto; use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; +use crate::threads::ip_banlist_set::IpBanlistSet; /// State that is injected into every endpoint. #[derive(Clone)] @@ -31,6 +32,8 @@ pub struct ServerState { pub audio_uploads_bucket_root: String, pub sort_key_crypto: SortKeyCrypto, + + pub ip_banlist: IpBanlistSet, } #[derive(Clone)] diff --git a/src/threads/poll_ip_banlist_thread.rs b/src/threads/poll_ip_banlist_thread.rs index 9203834a08..bf64d388a5 100644 --- a/src/threads/poll_ip_banlist_thread.rs +++ b/src/threads/poll_ip_banlist_thread.rs @@ -1,16 +1,70 @@ +use anyhow::Error; use crate::common_queries::list_ip_bans::list_ip_bans; use crate::threads::ip_banlist_set::IpBanlistSet; +use log::info; use log::warn; use sqlx::MySqlPool; use std::collections::HashSet; use std::thread; use std::time::Duration; -use anyhow::Error; +use tokio::signal::unix::{SignalKind, Signal}; +use tokio::signal::unix::signal; +use tokio::signal::ctrl_c; +use tokio::task::JoinHandle; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; -pub async fn poll_ip_bans(ip_banlist_set: IpBanlistSet, mysql_pool: &MySqlPool) { +pub async fn poll_ip_bans( + ip_banlist_set: IpBanlistSet, + mysql_pool: MySqlPool, + shutdown: Arc, +// ctrl_c: JoinHandle, +// sigterm: JoinHandle, +// sighup: JoinHandle, +// sigquit: JoinHandle, +) { + + //let mut stream = signal(SignalKind::interrupt()).unwrap(); loop { - let bans = match list_ip_bans(mysql_pool).await { + info!("Loop start..."); + + // NB: So we can exit on ctrl-c and other signals. + //tokio::select! { + // _ = background => println!(">>>>>> FOO"), + // _ = foreground => println!(">>>>>> BAR"), + if shutdown.load(Ordering::SeqCst) { + info!("Shutdown received"); + return; + } + + //}; +// tokio::select! { +// res = ctrl_c => { +// res??; +// info!(root_logger, "Received signal, shutting down"; "signal" => "SIGINT"); +// } +// _ = sigterm => { +// info!(root_logger, "Received signal, shutting down"; "signal" => "SIGTERM"); +// } +// _ = sighup => { +// info!(root_logger, "Received signal, shutting down"; "signal" => "SIGHUP"); +// } +// _ = sigquit => { +// info!(root_logger, "Received signal, shutting down"; "signal" => "SIGQUIT"); +// } +// }; + +// signal.poll_recv(); + + //info!("tokioselect (ip bans)"); + //tokio::select! { + // _ = shutdown_signal() => { + // info!("Sigint!!!") + // } + //} + + let bans = match list_ip_bans(&mysql_pool).await { Ok(bans) => bans, Err(e) => { warn!("Error polling IP bans: {:?}", e); @@ -19,15 +73,29 @@ pub async fn poll_ip_bans(ip_banlist_set: IpBanlistSet, mysql_pool: &MySqlPool) } }; + info!("Job fetching IP Address Bans..."); let ip_addresses = bans.iter() .map(|record| record.ip_address.clone()) .collect::>(); + info!("Job found {} IP Address Bans.", ip_addresses.len()); + match ip_banlist_set.replace_list(ip_addresses) { Ok(_) => {} Err(e) => { warn!("error replacing banlist: {:?}", e); } } + + info!("Sleeping..."); + for _ in 0..60 { + thread::sleep(Duration::from_millis(1_000)); + if shutdown.load(Ordering::SeqCst) { + info!("Shutdown received"); + return; + } + } + + info!("Loop end..."); } } From 38ed9f6804005015ffab543db9859a76fb2aece7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 16 Jun 2021 00:17:41 -0400 Subject: [PATCH 0410/1042] this works! ip ban polling async that doesn't stop interrupts Need to clean this up now... --- Cargo.lock | 1 + Cargo.toml | 2 +- src/main.rs | 77 ++++++++++++++++++--------- src/threads/poll_ip_banlist_thread.rs | 31 +++++++---- 4 files changed, 75 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 236ed8af02..f67cb88333 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1341,6 +1341,7 @@ dependencies = [ "futures-core", "futures-task", "futures-util", + "num_cpus", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7b91fead08..117e26bfb3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ chrono = { version = "0.4.19", features = ["serde"] } derive_more = "0.99.14" dotenv = "0.15.0" easyenv = "0.0.5" -futures = "0.3.15" +futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool hex = "0.4.3" hostname = "0.3.1" lazy_static = "1.4.0" diff --git a/src/main.rs b/src/main.rs index 90fcf67152..398fd8c4bc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -97,6 +97,11 @@ use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use tokio::signal::unix::{Signal, signal}; use tokio::signal::unix::SignalKind; use std::sync::atomic::{AtomicBool, Ordering}; +use tokio::sync::oneshot::{channel, Sender}; +use std::time::Duration; +use futures::Future; +use std::thread; +use futures::executor::ThreadPool; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -175,7 +180,7 @@ async fn main() -> AnyhowResult<()> { info!("Reading env vars and setting up utils..."); let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); - let num_workers = easyenv::get_env_num("NUM_WORKERS", 4)?; + let num_workers = easyenv::get_env_num("NUM_WORKERS", 8)?; let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); let cookie_secure = easyenv::get_env_bool_or_default("COOKIE_SECURE", true); @@ -218,6 +223,20 @@ async fn main() -> AnyhowResult<()> { let sort_key_crypto = SortKeyCrypto::new(&sort_key_crypto_secret); let ip_banlist = IpBanlistSet::new(); + let ip_banlist2 = ip_banlist.clone(); + let pool3 = pool.clone(); + + // Necessary to run our background work. + // I tried getting this running with actix, tokio, by other means, + // but didn't have much luck. The task didn't respond to ctrl-c + // interrupts. + let thread_pool = ThreadPool::new()?; + + let shutdown = Arc::new(AtomicBool::new(false)); + + thread_pool.spawn_ok(async { + poll_ip_bans("".to_string(), ip_banlist2, pool3, shutdown).await; + }); // Set up signal handlers //let ctrl_c = tokio::task::spawn(tokio::signal::ctrl_c()); @@ -227,31 +246,30 @@ async fn main() -> AnyhowResult<()> { // tokio::task::spawn(quit()), //); - //let sigint = SignalKind::interrupt(); - //let sigterm = SignalKind::terminate(); - //let sigint = Signal::new(SIGINT).flatten_stream(); - //let sigterm = Signal::new(SIGTERM).flatten_stream(); + + ////// Start polling thread. + ////let pool2 = pool.clone(); + ////let ip_banlist2 = ip_banlist.clone(); + //let shutdown2 = shutdown.clone(); + + ////let ip_banlist3 = ip_banlist.clone(); + ////let shutdown3 = shutdown.clone(); + + //let join = actix_rt::task::spawn_blocking(async { + // //poll_ip_bans(ip_banlist2, pool2, ctrl_c, sighup, sigterm, sigquit).await; + // shutdown_signal(shutdown2, tx).await + //}); - //let stream = sigint.select(sigterm); + //let join2 = tokio::task::spawn(async { + // //poll_ip_bans(ip_banlist2, pool2, ctrl_c, sighup, sigterm, sigquit).await; + // poll_ip_bans("foo".to_string(), ip_banlist2, pool2, shutdown).await; + //}); - let shutdown = Arc::new(AtomicBool::new(false)); - - // Start polling thread. - let pool2 = pool.clone(); - let ip_banlist2 = ip_banlist.clone(); - let shutdown2 = shutdown.clone(); - - let join = tokio::task::spawn(async { - //poll_ip_bans(ip_banlist2, pool2, ctrl_c, sighup, sigterm, sigquit).await; - shutdown_signal(shutdown2).await - }); - - - let join2 = tokio::task::spawn(async { - //poll_ip_bans(ip_banlist2, pool2, ctrl_c, sighup, sigterm, sigquit).await; - poll_ip_bans(ip_banlist2, pool2, shutdown).await; - }); + //let join3 = tokio::task::spawn(async { + // //poll_ip_bans(ip_banlist2, pool2, ctrl_c, sighup, sigterm, sigquit).await; + // poll_ip_bans("bar".to_string(), ip_banlist3, pool3, shutdown3).await; + //}); let server_state = ServerState { @@ -280,7 +298,14 @@ async fn main() -> AnyhowResult<()> { Ok(()) } -async fn shutdown_signal(mut shutdown: Arc) { +async fn test_future() { + loop { + info!("Thread 2"); + thread::sleep(Duration::from_millis(1000)); + } +} + +async fn shutdown_signal(mut shutdown: Arc, tx: Sender) { // Wait for the CTRL+C signal info!("Waiting for Ctrl+C"); //let mut stream = signal(SignalKind::terminate())?; @@ -289,8 +314,10 @@ async fn shutdown_signal(mut shutdown: Arc) { .await .expect("failed to install CTRL+C signal handler"); + tx.send(true).unwrap(); + info!("Shutdown recieved!"); - shutdown.store(true, Ordering::SeqCst); + shutdown.store(true, Ordering::Relaxed); } pub async fn serve(server_state: ServerState) -> AnyhowResult<()> diff --git a/src/threads/poll_ip_banlist_thread.rs b/src/threads/poll_ip_banlist_thread.rs index bf64d388a5..5422246820 100644 --- a/src/threads/poll_ip_banlist_thread.rs +++ b/src/threads/poll_ip_banlist_thread.rs @@ -13,11 +13,14 @@ use tokio::signal::ctrl_c; use tokio::task::JoinHandle; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use tokio::sync::oneshot::{Sender, Receiver}; pub async fn poll_ip_bans( + name: String, ip_banlist_set: IpBanlistSet, mysql_pool: MySqlPool, shutdown: Arc, +// mut rx: Receiver, // ctrl_c: JoinHandle, // sigterm: JoinHandle, // sighup: JoinHandle, @@ -27,16 +30,18 @@ pub async fn poll_ip_bans( //let mut stream = signal(SignalKind::interrupt()).unwrap(); loop { - info!("Loop start..."); + info!("[{}] Loop start...", &name); + //let signal = rx.try_recv().unwrap_or(false); + //info!("[{}] Signal: {}", &name, signal); // NB: So we can exit on ctrl-c and other signals. //tokio::select! { // _ = background => println!(">>>>>> FOO"), // _ = foreground => println!(">>>>>> BAR"), - if shutdown.load(Ordering::SeqCst) { - info!("Shutdown received"); - return; - } + //if shutdown.load(Ordering::Relaxed) { + // info!("[{}] Shutdown received", &name); + // return; + //} //}; // tokio::select! { @@ -87,15 +92,21 @@ pub async fn poll_ip_bans( } } - info!("Sleeping..."); - for _ in 0..60 { + info!("[{}] Sleeping...", &name); + for _ in 0..20 { + //std::thread::yield_now(); + tokio::task::yield_now().await; + + //let signal = rx.try_recv().unwrap_or(false); + //info!("[{}] Signal: {}", &name, signal); + info!("[{}] Sleep.", &name); thread::sleep(Duration::from_millis(1_000)); - if shutdown.load(Ordering::SeqCst) { - info!("Shutdown received"); + if shutdown.load(Ordering::Relaxed) { + info!("[{}], Shutdown received", &name); return; } } - info!("Loop end..."); + info!("[{}] Loop end...", &name); } } From b783fe5bcb3da2f2f4685315bdf43ed29a31ab52 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 16 Jun 2021 00:23:59 -0400 Subject: [PATCH 0411/1042] cleanup --- src/main.rs | 81 +++------------------------ src/threads/poll_ip_banlist_thread.rs | 81 +++------------------------ 2 files changed, 18 insertions(+), 144 deletions(-) diff --git a/src/main.rs b/src/main.rs index 398fd8c4bc..a436cc29e0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,6 +38,7 @@ use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::misc::enable_alpha::enable_alpha; use crate::http_server::endpoints::moderation::ip_bans::add_ip_ban::add_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::delete_ip_ban::delete_ip_ban_handler; +use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::list_ip_bans::list_ip_bans_handler; use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user_roles_handler; use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler; @@ -49,6 +50,7 @@ use crate::http_server::endpoints::tts::enqueue_infer_tts::infer_tts_handler; use crate::http_server::endpoints::tts::enqueue_upload_tts_model::upload_tts_model_handler; use crate::http_server::endpoints::tts::get_tts_inference_job_status::get_tts_inference_job_status_handler; use crate::http_server::endpoints::tts::get_tts_model::get_tts_model_handler; +use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_use_count_handler; use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; @@ -71,6 +73,7 @@ use crate::http_server::endpoints::w2l::enqueue_upload_w2l_template::upload_w2l_ use crate::http_server::endpoints::w2l::get_w2l_inference_job_status::get_w2l_inference_job_status_handler; use crate::http_server::endpoints::w2l::get_w2l_result::get_w2l_inference_result_handler; use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handler; +use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_template_use_count_handler; use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; @@ -80,8 +83,12 @@ use crate::server_state::{ServerState, EnvConfig}; use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use crate::shared_constants::DEFAULT_RUST_LOG; +use crate::threads::ip_banlist_set::IpBanlistSet; +use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; use crate::util::encrypted_sort_id::SortKeyCrypto; +use futures::Future; +use futures::executor::ThreadPool; use log::{info}; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2; @@ -89,19 +96,7 @@ use r2d2_redis::redis::Commands; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; -use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_use_count_handler; -use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_template_use_count_handler; -use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_handler; -use crate::threads::ip_banlist_set::IpBanlistSet; -use crate::threads::poll_ip_banlist_thread::poll_ip_bans; -use tokio::signal::unix::{Signal, signal}; -use tokio::signal::unix::SignalKind; -use std::sync::atomic::{AtomicBool, Ordering}; -use tokio::sync::oneshot::{channel, Sender}; use std::time::Duration; -use futures::Future; -use std::thread; -use futures::executor::ThreadPool; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -224,7 +219,7 @@ async fn main() -> AnyhowResult<()> { let ip_banlist = IpBanlistSet::new(); let ip_banlist2 = ip_banlist.clone(); - let pool3 = pool.clone(); + let mysql_pool3 = pool.clone(); // Necessary to run our background work. // I tried getting this running with actix, tokio, by other means, @@ -232,46 +227,10 @@ async fn main() -> AnyhowResult<()> { // interrupts. let thread_pool = ThreadPool::new()?; - let shutdown = Arc::new(AtomicBool::new(false)); - thread_pool.spawn_ok(async { - poll_ip_bans("".to_string(), ip_banlist2, pool3, shutdown).await; + poll_ip_bans(ip_banlist2, mysql_pool3).await; }); - // Set up signal handlers - //let ctrl_c = tokio::task::spawn(tokio::signal::ctrl_c()); - //let (sighup, sigterm, sigquit) = ( - // tokio::task::spawn(hangup()), - // tokio::task::spawn(terminate()), - // tokio::task::spawn(quit()), - //); - - - ////// Start polling thread. - ////let pool2 = pool.clone(); - ////let ip_banlist2 = ip_banlist.clone(); - //let shutdown2 = shutdown.clone(); - - ////let ip_banlist3 = ip_banlist.clone(); - ////let shutdown3 = shutdown.clone(); - - //let join = actix_rt::task::spawn_blocking(async { - // //poll_ip_bans(ip_banlist2, pool2, ctrl_c, sighup, sigterm, sigquit).await; - // shutdown_signal(shutdown2, tx).await - //}); - - - //let join2 = tokio::task::spawn(async { - // //poll_ip_bans(ip_banlist2, pool2, ctrl_c, sighup, sigterm, sigquit).await; - // poll_ip_bans("foo".to_string(), ip_banlist2, pool2, shutdown).await; - //}); - - //let join3 = tokio::task::spawn(async { - // //poll_ip_bans(ip_banlist2, pool2, ctrl_c, sighup, sigterm, sigquit).await; - // poll_ip_bans("bar".to_string(), ip_banlist3, pool3, shutdown3).await; - //}); - - let server_state = ServerState { env_config: EnvConfig { num_workers, @@ -298,28 +257,6 @@ async fn main() -> AnyhowResult<()> { Ok(()) } -async fn test_future() { - loop { - info!("Thread 2"); - thread::sleep(Duration::from_millis(1000)); - } -} - -async fn shutdown_signal(mut shutdown: Arc, tx: Sender) { - // Wait for the CTRL+C signal - info!("Waiting for Ctrl+C"); - //let mut stream = signal(SignalKind::terminate())?; - //stream.recv().await; - tokio::signal::ctrl_c() - .await - .expect("failed to install CTRL+C signal handler"); - - tx.send(true).unwrap(); - - info!("Shutdown recieved!"); - shutdown.store(true, Ordering::Relaxed); -} - pub async fn serve(server_state: ServerState) -> AnyhowResult<()> { let bind_address = server_state.env_config.bind_address.clone(); diff --git a/src/threads/poll_ip_banlist_thread.rs b/src/threads/poll_ip_banlist_thread.rs index 5422246820..0dcf47bcab 100644 --- a/src/threads/poll_ip_banlist_thread.rs +++ b/src/threads/poll_ip_banlist_thread.rs @@ -5,69 +5,22 @@ use log::info; use log::warn; use sqlx::MySqlPool; use std::collections::HashSet; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::thread; use std::time::Duration; -use tokio::signal::unix::{SignalKind, Signal}; -use tokio::signal::unix::signal; use tokio::signal::ctrl_c; -use tokio::task::JoinHandle; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; +use tokio::signal::unix::signal; +use tokio::signal::unix::{SignalKind, Signal}; use tokio::sync::oneshot::{Sender, Receiver}; +use tokio::task::JoinHandle; pub async fn poll_ip_bans( - name: String, ip_banlist_set: IpBanlistSet, mysql_pool: MySqlPool, - shutdown: Arc, -// mut rx: Receiver, -// ctrl_c: JoinHandle, -// sigterm: JoinHandle, -// sighup: JoinHandle, -// sigquit: JoinHandle, ) { - - //let mut stream = signal(SignalKind::interrupt()).unwrap(); - loop { - info!("[{}] Loop start...", &name); - //let signal = rx.try_recv().unwrap_or(false); - //info!("[{}] Signal: {}", &name, signal); - - // NB: So we can exit on ctrl-c and other signals. - //tokio::select! { - // _ = background => println!(">>>>>> FOO"), - // _ = foreground => println!(">>>>>> BAR"), - //if shutdown.load(Ordering::Relaxed) { - // info!("[{}] Shutdown received", &name); - // return; - //} - - //}; -// tokio::select! { -// res = ctrl_c => { -// res??; -// info!(root_logger, "Received signal, shutting down"; "signal" => "SIGINT"); -// } -// _ = sigterm => { -// info!(root_logger, "Received signal, shutting down"; "signal" => "SIGTERM"); -// } -// _ = sighup => { -// info!(root_logger, "Received signal, shutting down"; "signal" => "SIGHUP"); -// } -// _ = sigquit => { -// info!(root_logger, "Received signal, shutting down"; "signal" => "SIGQUIT"); -// } -// }; - -// signal.poll_recv(); - - //info!("tokioselect (ip bans)"); - //tokio::select! { - // _ = shutdown_signal() => { - // info!("Sigint!!!") - // } - //} + info!("Job fetching IP Address Bans..."); let bans = match list_ip_bans(&mysql_pool).await { Ok(bans) => bans, @@ -78,7 +31,6 @@ pub async fn poll_ip_bans( } }; - info!("Job fetching IP Address Bans..."); let ip_addresses = bans.iter() .map(|record| record.ip_address.clone()) .collect::>(); @@ -86,27 +38,12 @@ pub async fn poll_ip_bans( info!("Job found {} IP Address Bans.", ip_addresses.len()); match ip_banlist_set.replace_list(ip_addresses) { - Ok(_) => {} + Ok(_) => info!("internal banlist updated!"), Err(e) => { warn!("error replacing banlist: {:?}", e); - } + }, } - info!("[{}] Sleeping...", &name); - for _ in 0..20 { - //std::thread::yield_now(); - tokio::task::yield_now().await; - - //let signal = rx.try_recv().unwrap_or(false); - //info!("[{}] Signal: {}", &name, signal); - info!("[{}] Sleep.", &name); - thread::sleep(Duration::from_millis(1_000)); - if shutdown.load(Ordering::Relaxed) { - info!("[{}], Shutdown received", &name); - return; - } - } - - info!("[{}] Loop end...", &name); + thread::sleep(Duration::from_millis(20_000)); } } From 8b95ea3abd7f5016991bd4b9518d1b4fe1220149 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 16 Jun 2021 00:45:51 -0400 Subject: [PATCH 0412/1042] ip filter middleware (not rejecting yet) --- Cargo.lock | 1 + Cargo.toml | 1 + .../middleware/ip_filter_middleware.rs | 84 +++++++++++++++++++ src/http_server/middleware/mod.rs | 1 + src/http_server/mod.rs | 1 + src/main.rs | 5 ++ src/threads/ip_banlist_set.rs | 10 +++ src/threads/poll_ip_banlist_thread.rs | 7 -- 8 files changed, 103 insertions(+), 7 deletions(-) create mode 100644 src/http_server/middleware/ip_filter_middleware.rs create mode 100644 src/http_server/middleware/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f67cb88333..b396b54398 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3168,6 +3168,7 @@ dependencies = [ "actix-http", "actix-multipart", "actix-rt 2.2.0", + "actix-service", "actix-utils 3.0.0", "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 117e26bfb3..33dc410197 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,6 +86,7 @@ actix-files= "0.5" actix-http = "2.2.0" actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade actix-rt = "2.2.0" +actix-service = "1.0.6" actix-utils = "3.0.0" actix-web = "3.3.2" jwt = "0.13.0" diff --git a/src/http_server/middleware/ip_filter_middleware.rs b/src/http_server/middleware/ip_filter_middleware.rs new file mode 100644 index 0000000000..7544d98dfa --- /dev/null +++ b/src/http_server/middleware/ip_filter_middleware.rs @@ -0,0 +1,84 @@ +use std::pin::Pin; +use std::task::{Context, Poll}; + +use actix_service::{Service, Transform}; +use actix_web::{dev::ServiceRequest, dev::ServiceResponse, Error}; +use futures::future::{ok, Ready}; +use futures::Future; +use crate::threads::ip_banlist_set::IpBanlistSet; +use log::warn; +use log::info; + +// There are two steps in middleware processing. +// 1. Middleware initialization, middleware factory gets called with +// next service in chain as parameter. +// 2. Middleware's call method gets called with normal request. +#[derive(Clone)] // NB: Clone is safe because IpBanlist is clone-safe (internal Arc) +pub struct IpFilter { + ip_banlist: IpBanlistSet, +} + +impl IpFilter { + pub fn new(ip_banlist: IpBanlistSet) -> Self { + Self { + ip_banlist, + } + } +} + +// Middleware factory is `Transform` trait from actix-service crate +// `S` - type of the next service +// `B` - type of response's body +impl Transform for IpFilter + where + S: Service, Error = Error>, + S::Future: 'static, + B: 'static, +{ + type Request = ServiceRequest; + type Response = ServiceResponse; + type Error = Error; + type InitError = (); + type Transform = IpFilterMiddleware; + type Future = Ready>; + + fn new_transform(&self, service: S) -> Self::Future { + // NB: IpBanlist is clone-safe due to internal Arc. + ok(IpFilterMiddleware { service, ip_banlist: self.ip_banlist.clone() }) + } +} + +pub struct IpFilterMiddleware { + service: S, + ip_banlist: IpBanlistSet, +} + +impl Service for IpFilterMiddleware + where + S: Service, Error = Error>, + S::Future: 'static, + B: 'static, +{ + type Request = ServiceRequest; + type Response = ServiceResponse; + type Error = Error; + type Future = Pin>>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.service.poll_ready(cx) + } + + fn call(&mut self, req: ServiceRequest) -> Self::Future { + println!("Hi from start. You requested: {}", req.path()); + info!("Checking IP. There are {} bans currently.", self.ip_banlist.num_bans().unwrap_or(0)); + + let fut = self.service.call(req); + + Box::pin(async move { + let res = fut.await?; + + println!("Hi from response"); + Ok(res) + }) + } +} \ No newline at end of file diff --git a/src/http_server/middleware/mod.rs b/src/http_server/middleware/mod.rs new file mode 100644 index 0000000000..a2369086a3 --- /dev/null +++ b/src/http_server/middleware/mod.rs @@ -0,0 +1 @@ +pub mod ip_filter_middleware; \ No newline at end of file diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs index 9b6c9f4477..43daf6085b 100644 --- a/src/http_server/mod.rs +++ b/src/http_server/mod.rs @@ -1,2 +1,3 @@ pub mod endpoints; +pub mod middleware; pub mod web_utils; diff --git a/src/main.rs b/src/main.rs index a436cc29e0..8e3eaa376b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -97,6 +97,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; +use crate::http_server::middleware::ip_filter_middleware::IpFilter; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -270,6 +271,9 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> let log_format = "[%{HOSTNAME}e] %a \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T"; HttpServer::new(move || { + // NB: Safe to clone due to internal arc + let ip_banlist = server_state_arc.ip_banlist.clone(); + App::new() .app_data(server_state_arc.clone()) .wrap(Cors::default() @@ -310,6 +314,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .wrap(DefaultHeaders::new() .header("X-Backend-Hostname", &hostname) .header("X-Build-Sha", "")) + .wrap(IpFilter::new(ip_banlist)) // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== .service( web::resource("/create_account") diff --git a/src/threads/ip_banlist_set.rs b/src/threads/ip_banlist_set.rs index a585c1e64a..40023f83db 100644 --- a/src/threads/ip_banlist_set.rs +++ b/src/threads/ip_banlist_set.rs @@ -36,4 +36,14 @@ impl IpBanlistSet { }, } } + + pub fn num_bans(&self) -> AnyhowResult { + match self.ip_banlist.read() { + Err(_) => Err(anyhow!("Can't read lock")), + Ok(read) => { + let result = read.len(); + Ok(result) + }, + } + } } \ No newline at end of file diff --git a/src/threads/poll_ip_banlist_thread.rs b/src/threads/poll_ip_banlist_thread.rs index 0dcf47bcab..81ff51d471 100644 --- a/src/threads/poll_ip_banlist_thread.rs +++ b/src/threads/poll_ip_banlist_thread.rs @@ -5,15 +5,8 @@ use log::info; use log::warn; use sqlx::MySqlPool; use std::collections::HashSet; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; use std::thread; use std::time::Duration; -use tokio::signal::ctrl_c; -use tokio::signal::unix::signal; -use tokio::signal::unix::{SignalKind, Signal}; -use tokio::sync::oneshot::{Sender, Receiver}; -use tokio::task::JoinHandle; pub async fn poll_ip_bans( ip_banlist_set: IpBanlistSet, From 98b71277ac9ec82cbdf3a2c7c4fd74dc6b33711c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 16 Jun 2021 01:30:51 -0400 Subject: [PATCH 0413/1042] IP bans work! --- .../middleware/ip_filter_middleware.rs | 13 ++++++-- src/http_server/web_utils/ip_address.rs | 31 +++++++++++++++++++ src/threads/ip_banlist_set.rs | 4 +-- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/http_server/middleware/ip_filter_middleware.rs b/src/http_server/middleware/ip_filter_middleware.rs index 7544d98dfa..5815dfeb8e 100644 --- a/src/http_server/middleware/ip_filter_middleware.rs +++ b/src/http_server/middleware/ip_filter_middleware.rs @@ -8,6 +8,8 @@ use futures::Future; use crate::threads::ip_banlist_set::IpBanlistSet; use log::warn; use log::info; +use crate::http_server::web_utils::ip_address::get_service_request_ip; +use actix_web::error::ErrorForbidden; // There are two steps in middleware processing. // 1. Middleware initialization, middleware factory gets called with @@ -69,8 +71,15 @@ impl Service for IpFilterMiddleware } fn call(&mut self, req: ServiceRequest) -> Self::Future { - println!("Hi from start. You requested: {}", req.path()); - info!("Checking IP. There are {} bans currently.", self.ip_banlist.num_bans().unwrap_or(0)); + let ip_address = get_service_request_ip(&req); + + // NB: Fail open. + let is_banned = self.ip_banlist.is_banned(&ip_address).unwrap_or(false); + + if is_banned { + warn!("Ip is banned: {}", &ip_address); + return Box::pin(ok(req.error_response(ErrorForbidden("Forbidden")))) + } let fut = self.service.call(req); diff --git a/src/http_server/web_utils/ip_address.rs b/src/http_server/web_utils/ip_address.rs index bb500e129b..5372c5f26e 100644 --- a/src/http_server/web_utils/ip_address.rs +++ b/src/http_server/web_utils/ip_address.rs @@ -1,6 +1,7 @@ use actix_web::HttpRequest; use actix_web::http::HeaderName; use log::{info}; +use actix_web::dev::ServiceRequest; pub fn get_request_ip(request: &HttpRequest) -> String { match request.headers().get(HeaderName::from_static("x-voder-proxy-for")) { @@ -30,3 +31,33 @@ pub fn get_request_ip(request: &HttpRequest) -> String { }, } } + +// TODO: De-duplicate +pub fn get_service_request_ip(request: &ServiceRequest) -> String { + match request.headers().get(HeaderName::from_static("x-voder-proxy-for")) { + Some(ip_address) => { + // Unfortunately the upstream Rust proxy is replacing the `forwarded` and `x-forwarded-for` + // headers, so we populate this custom header as a workaround. + info!("Proxied IP address: {:?}", ip_address); + ip_address.to_str() + .unwrap_or("") + .to_string() + }, + None => { + // If we're running without the upstream Rust proxy, we can grab 'x-forarded-for', which is + // populated by the DigitalOcean load balancer. + let ip_address_and_port = request.connection_info() + .remote_addr() + .unwrap_or("") + .to_string(); + let ip_address = ip_address_and_port.split(":") + .collect::>() + .get(0) + .copied() + .unwrap_or("") + .to_string(); + info!("Forwarded IP address: {}", &ip_address); + ip_address + }, + } +} diff --git a/src/threads/ip_banlist_set.rs b/src/threads/ip_banlist_set.rs index 40023f83db..f0ec2611bf 100644 --- a/src/threads/ip_banlist_set.rs +++ b/src/threads/ip_banlist_set.rs @@ -27,11 +27,11 @@ impl IpBanlistSet { } } - pub fn is_banned(&self, ip_address: String) -> AnyhowResult { + pub fn is_banned>(&self, ip_address: S) -> AnyhowResult { match self.ip_banlist.read() { Err(_) => Err(anyhow!("Can't read lock")), Ok(read) => { - let result = read.contains(&ip_address); + let result = read.contains(ip_address.as_ref()); Ok(result) }, } From 38e1c4d692a4fd06c3238216ed6caa07aa74fef3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 16 Jun 2021 23:43:53 -0400 Subject: [PATCH 0414/1042] fix routes --- src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 8e3eaa376b..a41789fa43 100644 --- a/src/main.rs +++ b/src/main.rs @@ -421,7 +421,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> ) .service( web::resource("/result/{token}/delete") - .route(web::get().to(delete_tts_inference_result_handler)) + .route(web::post().to(delete_tts_inference_result_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) .service( @@ -485,7 +485,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> ) .service( web::resource("/result/{token}/delete") - .route(web::get().to(delete_w2l_inference_result_handler)) + .route(web::post().to(delete_w2l_inference_result_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) .service( From caac64c567af236d4ff86839ba3f1f5b66660ab9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Jun 2021 01:05:06 -0400 Subject: [PATCH 0415/1042] moderator-only fields for W2L templates --- src/common_queries/query_w2l_template.rs | 23 +++++++++++++++++-- .../endpoints/w2l/get_w2l_template.rs | 10 +++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/common_queries/query_w2l_template.rs b/src/common_queries/query_w2l_template.rs index 8a1ed4e0ce..82cf75ded1 100644 --- a/src/common_queries/query_w2l_template.rs +++ b/src/common_queries/query_w2l_template.rs @@ -30,6 +30,15 @@ pub struct W2lTemplateRecordForResponse { pub is_locked_from_user_modification: bool, pub created_at: DateTime, pub updated_at: DateTime, + pub moderator_fields: Option, +} + +/// "Moderator-only fields" that we wouldn't want to expose to ordinary users. +/// It's the web endpoint controller's responsibility to clear these for non-mods. +#[derive(Serialize)] +pub struct W2lTemplateModeratorFields { + pub creator_ip_address_creation: String, + pub creator_ip_address_last_update: String, pub user_deleted_at: Option>, pub mod_deleted_at: Option>, } @@ -50,6 +59,8 @@ pub struct W2lTemplateRecordRaw { pub is_public_listing_approved: Option, pub is_locked_from_use: i8, pub is_locked_from_user_modification: i8, + pub creator_ip_address_creation: String, + pub creator_ip_address_last_update: String, pub created_at: DateTime, pub updated_at: DateTime, pub user_deleted_at: Option>, @@ -101,8 +112,12 @@ pub async fn select_w2l_template_by_token( is_locked_from_user_modification: i8_to_bool(template.is_locked_from_user_modification), created_at: template.created_at.clone(), updated_at: template.updated_at.clone(), - user_deleted_at: template.user_deleted_at.clone(), - mod_deleted_at: template.mod_deleted_at.clone(), + moderator_fields: Some(W2lTemplateModeratorFields { + creator_ip_address_creation: template.creator_ip_address_creation.clone(), + creator_ip_address_last_update: template.creator_ip_address_last_update.clone(), + user_deleted_at: template.user_deleted_at.clone(), + mod_deleted_at: template.mod_deleted_at.clone(), + }), }; Ok(Some(template_for_response)) @@ -130,6 +145,8 @@ SELECT w2l.is_public_listing_approved, w2l.is_locked_from_use, w2l.is_locked_from_user_modification, + w2l.creator_ip_address_creation, + w2l.creator_ip_address_last_update, w2l.created_at, w2l.updated_at, w2l.user_deleted_at, @@ -167,6 +184,8 @@ SELECT w2l.is_public_listing_approved, w2l.is_locked_from_use, w2l.is_locked_from_user_modification, + w2l.creator_ip_address_creation, + w2l.creator_ip_address_last_update, w2l.created_at, w2l.updated_at, w2l.user_deleted_at, diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index 4dafbfad82..adada09ba7 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -78,11 +78,15 @@ pub async fn get_w2l_template_handler( })?; let mut show_deleted_templates = false; + let mut is_moderator = false; if let Some(user_session) = maybe_user_session { // NB: Moderators can see deleted templates. // Original creators cannot see them (unless they're moderators!) show_deleted_templates = user_session.can_delete_other_users_w2l_templates; + // Moderators get to see all the fields. + is_moderator = user_session.can_delete_other_users_w2l_results + || user_session.can_edit_other_users_w2l_templates; } let template_query_result = select_w2l_template_by_token( @@ -91,7 +95,7 @@ pub async fn get_w2l_template_handler( &server_state.mysql_pool ).await; - let template = match template_query_result { + let mut template = match template_query_result { Err(e) => { warn!("query error: {:?}", e); return Err(GetW2lTemplateError::ServerError); @@ -100,6 +104,10 @@ pub async fn get_w2l_template_handler( Ok(Some(template)) => template, }; + if !is_moderator { + template.moderator_fields = None; + } + let response = GetW2lTemplateSuccessResponse { success: true, template, From 12466896e08772f4b309a96c5741eddc3211e839 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Jun 2021 01:37:30 -0400 Subject: [PATCH 0416/1042] GET TTS model cleanup, add mod-only fields --- src/common_queries/query_tts_model.rs | 59 +++++++- src/common_queries/query_w2l_template.rs | 4 +- .../endpoints/tts/get_tts_model.rs | 127 ++++++------------ .../endpoints/w2l/get_w2l_template.rs | 2 +- 4 files changed, 97 insertions(+), 95 deletions(-) diff --git a/src/common_queries/query_tts_model.rs b/src/common_queries/query_tts_model.rs index 1ba4db4a2b..32db56d7bf 100644 --- a/src/common_queries/query_tts_model.rs +++ b/src/common_queries/query_tts_model.rs @@ -16,16 +16,31 @@ use std::sync::Arc; pub struct TtsModelRecordForResponse { pub model_token: String, pub tts_model_type: String, + pub text_preprocessing_algorithm: String, + pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, + pub title: String, pub description_markdown: String, pub description_rendered_html: String, + pub is_locked_from_use: bool, pub is_locked_from_user_modification: bool, + pub created_at: DateTime, pub updated_at: DateTime, + + pub maybe_moderator_fields: Option, +} + +/// "Moderator-only fields" that we wouldn't want to expose to ordinary users. +/// It's the web endpoint controller's responsibility to clear these for non-mods. +#[derive(Serialize)] +pub struct TtsModelModeratorFields { + pub creator_ip_address_creation: String, + pub creator_ip_address_last_update: String, pub user_deleted_at: Option>, pub mod_deleted_at: Option>, } @@ -34,16 +49,25 @@ pub struct TtsModelRecordForResponse { pub struct TtsModelRecordRaw { pub model_token: String, pub tts_model_type: String, + pub text_preprocessing_algorithm: String, + pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, + pub title: String, pub description_markdown: String, pub description_rendered_html: String, + pub is_locked_from_use: i8, pub is_locked_from_user_modification: i8, + pub created_at: DateTime, pub updated_at: DateTime, + + // Moderator fields + pub creator_ip_address_creation: String, + pub creator_ip_address_last_update: String, pub user_deleted_at: Option>, pub mod_deleted_at: Option>, } @@ -79,6 +103,7 @@ pub async fn select_tts_model_by_token( let model_for_response = TtsModelRecordForResponse { model_token: model.model_token.clone(), tts_model_type: model.tts_model_type.clone(), + text_preprocessing_algorithm: model.text_preprocessing_algorithm.clone(), creator_user_token: model.creator_user_token.clone(), creator_username: model.creator_username.clone(), creator_display_name: model.creator_display_name.clone(), @@ -89,8 +114,12 @@ pub async fn select_tts_model_by_token( is_locked_from_user_modification: i8_to_bool(model.is_locked_from_user_modification), created_at: model.created_at.clone(), updated_at: model.updated_at.clone(), - user_deleted_at: model.user_deleted_at.clone(), - mod_deleted_at: model.mod_deleted_at.clone(), + maybe_moderator_fields: Some(TtsModelModeratorFields { + creator_ip_address_creation: model.creator_ip_address_creation.clone(), + creator_ip_address_last_update: model.creator_ip_address_last_update.clone(), + user_deleted_at: model.user_deleted_at.clone(), + mod_deleted_at: model.mod_deleted_at.clone(), + }), }; Ok(Some(model_for_response)) @@ -106,18 +135,27 @@ async fn select_including_deleted( SELECT tts.token as model_token, tts.tts_model_type, + tts.text_preprocessing_algorithm, + tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, + tts.title, tts.description_markdown, tts.description_rendered_html, + tts.is_locked_from_use, tts.is_locked_from_user_modification, + tts.created_at, tts.updated_at, - tts.user_deleted_at, - tts.mod_deleted_at + + tts.creator_ip_address_creation, + tts.creator_ip_address_last_update, + tts.mod_deleted_at, + tts.user_deleted_at + FROM tts_models as tts JOIN users ON users.token = tts.creator_user_token @@ -139,18 +177,27 @@ async fn select_without_deleted( SELECT tts.token as model_token, tts.tts_model_type, + tts.text_preprocessing_algorithm, + tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, + tts.title, tts.description_markdown, tts.description_rendered_html, + tts.is_locked_from_use, tts.is_locked_from_user_modification, + tts.created_at, tts.updated_at, - tts.user_deleted_at, - tts.mod_deleted_at + + tts.creator_ip_address_creation, + tts.creator_ip_address_last_update, + tts.mod_deleted_at, + tts.user_deleted_at + FROM tts_models as tts JOIN users ON users.token = tts.creator_user_token diff --git a/src/common_queries/query_w2l_template.rs b/src/common_queries/query_w2l_template.rs index 82cf75ded1..f544bc5d9b 100644 --- a/src/common_queries/query_w2l_template.rs +++ b/src/common_queries/query_w2l_template.rs @@ -30,7 +30,7 @@ pub struct W2lTemplateRecordForResponse { pub is_locked_from_user_modification: bool, pub created_at: DateTime, pub updated_at: DateTime, - pub moderator_fields: Option, + pub maybe_moderator_fields: Option, } /// "Moderator-only fields" that we wouldn't want to expose to ordinary users. @@ -112,7 +112,7 @@ pub async fn select_w2l_template_by_token( is_locked_from_user_modification: i8_to_bool(template.is_locked_from_user_modification), created_at: template.created_at.clone(), updated_at: template.updated_at.clone(), - moderator_fields: Some(W2lTemplateModeratorFields { + maybe_moderator_fields: Some(W2lTemplateModeratorFields { creator_ip_address_creation: template.creator_ip_address_creation.clone(), creator_ip_address_last_update: template.creator_ip_address_last_update.clone(), user_deleted_at: template.user_deleted_at.clone(), diff --git a/src/http_server/endpoints/tts/get_tts_model.rs b/src/http_server/endpoints/tts/get_tts_model.rs index fffe744405..bbaf138d5b 100644 --- a/src/http_server/endpoints/tts/get_tts_model.rs +++ b/src/http_server/endpoints/tts/get_tts_model.rs @@ -8,6 +8,8 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::common_queries::query_tts_model::TtsModelRecordForResponse; +use crate::common_queries::query_tts_model::select_tts_model_by_token; use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; @@ -33,22 +35,6 @@ pub struct GetTtsModelPathInfo { token: String, } -#[derive(Serialize)] -pub struct TtsModelRecordForResponse { - pub model_token: String, - pub tts_model_type: String, - pub text_preprocessing_algorithm: String, - - pub creator_user_token: String, - pub creator_username: String, - pub creator_display_name: String, - - pub title: String, - - pub created_at: DateTime, - pub updated_at: DateTime, -} - #[derive(Serialize)] pub struct GetTtsModelSuccessResponse { pub success: bool, @@ -58,34 +44,21 @@ pub struct GetTtsModelSuccessResponse { #[derive(Debug, Display)] pub enum GetTtsModelError { ServerError, -} - -#[derive(Serialize)] -pub struct TtsModelRecord { - pub model_token: String, - pub tts_model_type: String, - pub text_preprocessing_algorithm: String, - - pub creator_user_token: String, - pub creator_username: String, - pub creator_display_name: String, - - pub title: String, - - pub created_at: DateTime, - pub updated_at: DateTime, + NotFound, } impl ResponseError for GetTtsModelError { fn status_code(&self) -> StatusCode { match *self { GetTtsModelError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + GetTtsModelError::NotFound => StatusCode::NOT_FOUND, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { GetTtsModelError::ServerError => "server error".to_string(), + GetTtsModelError::NotFound => "not found".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -97,73 +70,55 @@ pub async fn get_tts_model_handler( path: Path, server_state: web::Data>) -> Result { - // NB: Lookup failure is Err(RowNotFound). - // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_models = sqlx::query_as!( - TtsModelRecord, - r#" -SELECT - tts.token as model_token, - tts.tts_model_type, - tts.text_preprocessing_algorithm, + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetTtsModelError::ServerError + })?; - tts.creator_user_token, - users.username as creator_username, - users.display_name as creator_display_name, + let mut show_deleted_models = false; + let mut is_moderator = false; - tts.title, + if let Some(user_session) = maybe_user_session { + // NB: Moderators can see deleted models + // Original creators cannot see them (unless they're moderators!) + show_deleted_models = user_session.can_delete_other_users_tts_models; + // Moderators get to see all the fields. + is_moderator = user_session.can_delete_other_users_tts_results + || user_session.can_edit_other_users_tts_models; + } - tts.created_at, - tts.updated_at -FROM tts_models as tts -JOIN users -ON users.token = tts.creator_user_token -WHERE - tts.token = ? - AND tts.user_deleted_at IS NULL - AND tts.mod_deleted_at IS NULL - "#, - &path.token - ) - .fetch_one(&server_state.mysql_pool) - .await; // TODO: This will return error if it doesn't exist + let model_query_result = select_tts_model_by_token( + &path.token, + show_deleted_models, + &server_state.mysql_pool + ).await; - let template : TtsModelRecord = match maybe_models { - Ok(models) => models, - Err(err) => { - match err { - RowNotFound => { - return Err(GetTtsModelError::ServerError); - }, - _ => { - warn!("tts template query error: {:?}", err); - return Err(GetTtsModelError::ServerError); - } - } + let mut model = match model_query_result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(GetTtsModelError::ServerError); } + Ok(None) => return Err(GetTtsModelError::NotFound), + Ok(Some(model)) => model, }; - let model_for_response = TtsModelRecordForResponse { - model_token: template.model_token.clone(), - tts_model_type: template.tts_model_type.clone(), - text_preprocessing_algorithm: template.text_preprocessing_algorithm.clone(), - creator_user_token: template.creator_user_token.clone(), - creator_username: template.creator_username.clone(), - creator_display_name: template.creator_display_name.clone(), - title: template.title.clone(), - created_at: template.created_at.clone(), - updated_at: template.updated_at.clone(), - }; + if !is_moderator { + model.maybe_moderator_fields = None; + } let response = GetTtsModelSuccessResponse { success: true, - model: model_for_response, + model, }; let body = serde_json::to_string(&response) - .map_err(|e| GetTtsModelError::ServerError)?; + .map_err(|e| GetTtsModelError::ServerError)?; Ok(HttpResponse::Ok() - .content_type("application/json") - .body(body)) + .content_type("application/json") + .body(body)) } diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index adada09ba7..03e04703ec 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -105,7 +105,7 @@ pub async fn get_w2l_template_handler( }; if !is_moderator { - template.moderator_fields = None; + template.maybe_moderator_fields = None; } let response = GetW2lTemplateSuccessResponse { From 02c1442e6c5a89d97367ba093d3c169c4277b9ae Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Jun 2021 01:43:56 -0400 Subject: [PATCH 0417/1042] mod fields for TTS results --- src/common_queries/query_tts_result.rs | 22 +++++++++++++++++-- .../endpoints/tts/get_tts_result.rs | 10 ++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/common_queries/query_tts_result.rs b/src/common_queries/query_tts_result.rs index 632b2d6ac8..e6636a2d14 100644 --- a/src/common_queries/query_tts_result.rs +++ b/src/common_queries/query_tts_result.rs @@ -42,6 +42,14 @@ pub struct TtsResultRecordForResponse { pub created_at: DateTime, pub updated_at: DateTime, + pub maybe_moderator_fields: Option, +} + +/// "Moderator-only fields" that we wouldn't want to expose to ordinary users. +/// It's the web endpoint controller's responsibility to clear these for non-mods. +#[derive(Serialize)] +pub struct TtsResultModeratorFields { + pub creator_ip_address: String, pub user_deleted_at: Option>, pub mod_deleted_at: Option>, } @@ -76,6 +84,8 @@ pub struct TtsResultRecordRaw { pub created_at: DateTime, pub updated_at: DateTime, + // Moderator fields + pub creator_ip_address: String, pub user_deleted_at: Option>, pub mod_deleted_at: Option>, } @@ -136,8 +146,12 @@ pub async fn select_tts_result_by_token( created_at: ir.created_at.clone(), updated_at: ir.updated_at.clone(), - user_deleted_at: ir.user_deleted_at.clone(), - mod_deleted_at: ir.mod_deleted_at.clone(), + + maybe_moderator_fields: Some(TtsResultModeratorFields { + creator_ip_address: ir.creator_ip_address.clone(), + user_deleted_at: ir.user_deleted_at.clone(), + mod_deleted_at: ir.mod_deleted_at.clone(), + }), }; Ok(Some(ir_for_response)) @@ -175,6 +189,8 @@ SELECT tts_results.duration_millis, tts_results.created_at, tts_results.updated_at, + + tts_results.creator_ip_address, tts_results.user_deleted_at, tts_results.mod_deleted_at @@ -226,6 +242,8 @@ SELECT tts_results.duration_millis, tts_results.created_at, tts_results.updated_at, + + tts_results.creator_ip_address, tts_results.user_deleted_at, tts_results.mod_deleted_at diff --git a/src/http_server/endpoints/tts/get_tts_result.rs b/src/http_server/endpoints/tts/get_tts_result.rs index b762ed20cb..fdf6597276 100644 --- a/src/http_server/endpoints/tts/get_tts_result.rs +++ b/src/http_server/endpoints/tts/get_tts_result.rs @@ -78,11 +78,15 @@ pub async fn get_tts_inference_result_handler( })?; let mut show_deleted_results = false; + let mut is_moderator = false; if let Some(user_session) = maybe_user_session { // NB: Moderators can see deleted results. // Original creators cannot see them (unless they're moderators!) show_deleted_results = user_session.can_delete_other_users_tts_results; + // Moderators get to see all the fields. + is_moderator = user_session.can_delete_other_users_tts_results + || user_session.can_edit_other_users_tts_models; } let inference_result_query_result = select_tts_result_by_token( @@ -91,7 +95,7 @@ pub async fn get_tts_inference_result_handler( &server_state.mysql_pool ).await; - let inference_result = match inference_result_query_result { + let mut inference_result = match inference_result_query_result { Err(e) => { warn!("query error: {:?}", e); return Err(GetTtsResultError::ServerError); @@ -100,6 +104,10 @@ pub async fn get_tts_inference_result_handler( Ok(Some(inference_result)) => inference_result, }; + if !is_moderator { + inference_result.maybe_moderator_fields = None; + } + let response = GetTtsResultSuccessResponse { success: true, result: inference_result, From 8463a16fe1d916aa9acca79035ee44b9ea26972c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Jun 2021 01:47:31 -0400 Subject: [PATCH 0418/1042] remove extraneous logging --- src/http_server/middleware/ip_filter_middleware.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/http_server/middleware/ip_filter_middleware.rs b/src/http_server/middleware/ip_filter_middleware.rs index 5815dfeb8e..5b90e6f349 100644 --- a/src/http_server/middleware/ip_filter_middleware.rs +++ b/src/http_server/middleware/ip_filter_middleware.rs @@ -81,12 +81,10 @@ impl Service for IpFilterMiddleware return Box::pin(ok(req.error_response(ErrorForbidden("Forbidden")))) } + // Call endpoint and return results let fut = self.service.call(req); - Box::pin(async move { let res = fut.await?; - - println!("Hi from response"); Ok(res) }) } From 34c66465ce1d9645212d3ccc4afc5144fd977819 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Jun 2021 01:53:12 -0400 Subject: [PATCH 0419/1042] mod-only response for w2l result --- src/common_queries/query_w2l_result.rs | 21 +++++++++++++++++-- .../endpoints/w2l/get_w2l_result.rs | 10 ++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/common_queries/query_w2l_result.rs b/src/common_queries/query_w2l_result.rs index 3d49bec025..8183d0d8f8 100644 --- a/src/common_queries/query_w2l_result.rs +++ b/src/common_queries/query_w2l_result.rs @@ -44,6 +44,14 @@ pub struct W2lResultRecordForResponse { pub created_at: DateTime, pub updated_at: DateTime, + pub maybe_moderator_fields: Option, +} + +/// "Moderator-only fields" that we wouldn't want to expose to ordinary users. +/// It's the web endpoint controller's responsibility to clear these for non-mods. +#[derive(Serialize)] +pub struct W2lResultModeratorFields { + pub creator_ip_address: String, pub user_deleted_at: Option>, pub mod_deleted_at: Option>, } @@ -81,6 +89,7 @@ pub struct W2lResultRecordRaw { pub created_at: DateTime, pub updated_at: DateTime, + pub creator_ip_address: String, pub user_deleted_at: Option>, pub mod_deleted_at: Option>, } @@ -142,8 +151,12 @@ pub async fn select_w2l_result_by_token( created_at: ir.created_at.clone(), updated_at: ir.updated_at.clone(), - user_deleted_at: ir.user_deleted_at.clone(), - mod_deleted_at: ir.mod_deleted_at.clone(), + + maybe_moderator_fields: Some(W2lResultModeratorFields { + creator_ip_address: ir.creator_ip_address.clone(), + user_deleted_at: ir.user_deleted_at.clone(), + mod_deleted_at: ir.mod_deleted_at.clone(), + }), }; Ok(Some(ir_for_response)) @@ -182,6 +195,8 @@ SELECT w2l_results.duration_millis, w2l_results.created_at, w2l_results.updated_at, + + w2l_results.creator_ip_address, w2l_results.user_deleted_at, w2l_results.mod_deleted_at @@ -234,6 +249,8 @@ SELECT w2l_results.duration_millis, w2l_results.created_at, w2l_results.updated_at, + + w2l_results.creator_ip_address, w2l_results.user_deleted_at, w2l_results.mod_deleted_at diff --git a/src/http_server/endpoints/w2l/get_w2l_result.rs b/src/http_server/endpoints/w2l/get_w2l_result.rs index a66dc82f66..80797ec7a4 100644 --- a/src/http_server/endpoints/w2l/get_w2l_result.rs +++ b/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -80,11 +80,15 @@ pub async fn get_w2l_inference_result_handler( })?; let mut show_deleted_results = false; + let mut is_moderator = false; if let Some(user_session) = maybe_user_session { // NB: Moderators can see deleted results. // Original creators cannot see them (unless they're moderators!) show_deleted_results = user_session.can_delete_other_users_w2l_results; + // Moderators get to see all the fields. + is_moderator = user_session.can_delete_other_users_w2l_results + || user_session.can_edit_other_users_w2l_templates; } let inference_result_query_result = select_w2l_result_by_token( @@ -93,7 +97,7 @@ pub async fn get_w2l_inference_result_handler( &server_state.mysql_pool ).await; - let inference_result = match inference_result_query_result { + let mut inference_result = match inference_result_query_result { Err(e) => { warn!("query error: {:?}", e); return Err(GetW2lResultError::ServerError); @@ -102,6 +106,10 @@ pub async fn get_w2l_inference_result_handler( Ok(Some(inference_result)) => inference_result, }; + if !is_moderator { + inference_result.maybe_moderator_fields = None; + } + let response = GetW2lResultSuccessResponse { success: true, result: inference_result, From ceee06773440ff7539a54c7f9a1c611b421419d6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Jun 2021 02:18:47 -0400 Subject: [PATCH 0420/1042] re-codegen db --- sqlx-data.json | 4416 +++++++++++++++++++++++++----------------------- 1 file changed, 2312 insertions(+), 2104 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 6250cf2002..4ef971e535 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -270,446 +270,6 @@ ] } }, - "10baa66f954e30d5da4bdb0fe6f912b01c6ac7ec803801797a0a5fd81ae0947e": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n tts.created_at,\n tts.updated_at,\n tts.user_deleted_at,\n tts.mod_deleted_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "description_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 7, - "name": "description_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 8, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 11, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 12, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, - "11554bff2bf06cf0f843b716c99c61b540639207d716eeabe3e2afb189290ab6": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.token = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 7, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "is_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 8 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 15, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 16, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 17, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false, - false, - false, - true, - true - ] - } - }, "1168f39a5ec892d06e941b0dba0e955a27ef976567eeafd6e550bc42ea4c4331": { "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", "describe": { @@ -770,200 +330,6 @@ ] } }, - "1513af6ee028edfea33311b64d5d71f784ea058bbde114cfbc9b1785ef98c7e8": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n tts.created_at,\n tts.updated_at,\n tts.user_deleted_at,\n tts.mod_deleted_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "description_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 7, - "name": "description_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 8, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 11, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 12, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "1556b7ed6b0cab002b52dfa8a5f81743a38967631ff24fd9555a168823b1dfc1": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -1123,6 +489,524 @@ "nullable": [] } }, + "1dedb79cc9ba347610bf37d3bf80145c12e90493741eb961f83aba13b61160b9": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n\n tts_results.creator_ip_address,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n AND tts_results.user_deleted_at IS NULL\n AND tts_results.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 2, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "tts_model_title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "maybe_model_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "maybe_model_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 10, + "name": "maybe_model_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 11, + "name": "maybe_model_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 12, + "name": "public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "public_bucket_spectrogram_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 15, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 17, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 19, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, + "1f1605f78557a12e61e8e2f9a7d2888b71a95227fc91db2d1c2ef2a6dc857c34": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 8, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 9, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 11, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 12, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 14, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 15, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 16, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "25e7858d370ac1359deb42ef5292e85ccd1fe7debf3a1ba73303e5882523a4d0": { "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n download_url = ?,\n status = \"pending\"\n ", "describe": { @@ -1362,252 +1246,6 @@ "nullable": [] } }, - "3c6615a453cf4ab41ff77616369a8aa1ed83f835ad35a5b52ca61fa77dfb8c77": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 7, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "is_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 8 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 15, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 16, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 17, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false, - false, - false, - true, - true - ] - } - }, "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c": { "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -1812,13 +1450,13 @@ ] } }, - "453e6252ea537f5f3fddae361b53f21c86fe829930cf7415d43aa899e4390bd9": { - "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n ", + "493917c2c4ee367c6fcad90ed1f6a114a8755e3f23287c3a6dbe3196299d5729": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.token = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "tts_result_token", + "name": "template_token", "type_info": { "type": "VarString", "flags": { @@ -1830,19 +1468,19 @@ }, { "ordinal": 1, - "name": "raw_inference_text", + "name": "template_type", "type_info": { - "type": "Blob", + "type": "String", "flags": { - "bits": 4241 + "bits": 393 }, "char_set": 224, - "max_size": 262140 + "max_size": 28 } }, { "ordinal": 2, - "name": "tts_model_token", + "name": "creator_user_token", "type_info": { "type": "VarString", "flags": { @@ -1854,139 +1492,43 @@ }, { "ordinal": 3, - "name": "tts_model_title", + "name": "creator_username", "type_info": { "type": "VarString", "flags": { - "bits": 4224 + "bits": 4229 }, "char_set": 224, - "max_size": 1020 + "max_size": 80 } }, { "ordinal": 4, - "name": "maybe_creator_user_token", + "name": "creator_display_name", "type_info": { "type": "VarString", "flags": { - "bits": 4228 + "bits": 4225 }, "char_set": 224, - "max_size": 128 + "max_size": 80 } }, { "ordinal": 5, - "name": "maybe_creator_username", + "name": "title", "type_info": { - "type": "VarString", + "type": "String", "flags": { - "bits": 4228 + "bits": 4225 }, "char_set": 224, - "max_size": 80 + "max_size": 1020 } }, { "ordinal": 6, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, - "name": "maybe_model_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 9, - "name": "maybe_model_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 10, - "name": "maybe_model_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 11, - "name": "maybe_model_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 12, - "name": "public_bucket_wav_audio_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 13, - "name": "public_bucket_spectrogram_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 14, - "name": "file_size_bytes", + "name": "frame_width", "type_info": { "type": "Long", "flags": { @@ -1997,7 +1539,19 @@ } }, { - "ordinal": 15, + "ordinal": 7, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, "name": "duration_millis", "type_info": { "type": "Long", @@ -2008,6 +1562,90 @@ "max_size": 10 } }, + { + "ordinal": 9, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 15, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, { "ordinal": 16, "name": "created_at", @@ -2038,7 +1676,7 @@ "type_info": { "type": "Timestamp", "flags": { - "bits": 136 + "bits": 128 }, "char_set": 63, "max_size": 19 @@ -2050,7 +1688,7 @@ "type_info": { "type": "Timestamp", "flags": { - "bits": 136 + "bits": 128 }, "char_set": 63, "max_size": 19 @@ -2064,12 +1702,12 @@ false, false, false, - true, - true, - true, - true, - true, - true, + false, + false, + false, + false, + false, + false, true, true, true, @@ -2122,278 +1760,6 @@ ] } }, - "52447dfed3f294970e4c589062342ede87ab3a1f8c483751294fcb91cadff386": { - "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n AND tts_results.user_deleted_at IS NULL\n AND tts_results.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tts_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "raw_inference_text", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 2, - "name": "tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "tts_model_title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 5, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, - "name": "maybe_model_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 9, - "name": "maybe_model_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 10, - "name": "maybe_model_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 11, - "name": "maybe_model_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 12, - "name": "public_bucket_wav_audio_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 13, - "name": "public_bucket_spectrogram_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 14, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 15, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 16, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 17, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 18, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "53d77ee58e9b1f7fb900914895c25bcccb9297378ff018c1e553b187d1956e92": { "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE username = ?\nLIMIT 1\n ", "describe": { @@ -2458,6 +1824,317 @@ ] } }, + "54d3d299b6f1f4068f8f79cc9fbff1484ed6c9dd27389cba0e40a99093164ea8": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n\n w2l_results.creator_ip_address,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "w2l_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "public_bucket_video_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 392 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 5, + "name": "template_title", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 9, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "maybe_template_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "maybe_template_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 12, + "name": "maybe_template_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 13, + "name": "maybe_template_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 14, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 15, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 17, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 18, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 21, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "55a75db2d060bd49951db22f6b4ffd1545777b61c256030d59662543adc06ab2": { "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n ip_address_last_update = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", "describe": { @@ -2997,135 +2674,6 @@ ] } }, - "6105a7fd1cf7a4d61d1b00c262c44f0370f7a9b8827354b2e3161d78493a0f5b": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "text_preprocessing_algorithm", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 3, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 8, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "641253d4c621f75037877651bba044a21033f50a5e3f8fed9b06ec967a508853": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -3156,304 +2704,6 @@ "nullable": [] } }, - "66442f357c878d55cf99a388861c6d7434f4e543f76088a8d70d73b489b11784": { - "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "w2l_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "maybe_tts_inference_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "public_bucket_video_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 3, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 392 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 5, - "name": "template_title", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 8, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 9, - "name": "maybe_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 10, - "name": "maybe_template_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 11, - "name": "maybe_template_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 12, - "name": "maybe_template_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 13, - "name": "maybe_template_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 14, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 15, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 16, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 17, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 18, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 21, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - true, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "68376f750b8421c10d319fa1045a2f3ce8fb1d35a5fe6ec92b2022babafbde80": { "query": "\nUPDATE users\nSET\n user_role_slug = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -5331,109 +4581,6 @@ "nullable": [] } }, - "af6e3f4b8b29a9e26ae422bee2e24e1d926990c5111149d26dbe4ea97dceb87f": { - "query": "\nSELECT\n ip_bans.ip_address,\n ip_bans.maybe_target_user_token,\n users.username as maybe_target_username,\n ip_bans.mod_user_token,\n ip_bans.mod_notes,\n ip_bans.created_at,\n ip_bans.updated_at\nFROM\n ip_address_bans AS ip_bans\nLEFT OUTER JOIN users\n ON ip_bans.maybe_target_user_token = users.token\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 1, - "name": "maybe_target_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "maybe_target_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "mod_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "mod_notes", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 5, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 6, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - true, - true, - false, - false, - false, - false - ] - } - }, "b1ad806aa93f66cab358d7cdb405d57c643e2ff131cfec9f1ced55c2e6ca9511": { "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { @@ -5444,304 +4591,6 @@ "nullable": [] } }, - "b1d7e1c405a33c391954167a3527580ec7fefac0055297150108a92f3945941f": { - "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n AND w2l_results.user_deleted_at IS NULL\n AND w2l_results.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "w2l_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "maybe_tts_inference_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "public_bucket_video_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 3, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 392 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 5, - "name": "template_title", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 8, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 9, - "name": "maybe_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 10, - "name": "maybe_template_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 11, - "name": "maybe_template_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 12, - "name": "maybe_template_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 13, - "name": "maybe_template_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 14, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 15, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 16, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 17, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 18, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 21, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - true, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "b257c267dfe063995374786d3004f215e77e8a32843c024645da6fa435507dce": { "query": "\nINSERT INTO\n ip_address_bans\nSET\n ip_address = ?,\n maybe_target_user_token = ?,\n mod_user_token = ?,\n mod_notes = ?\nON DUPLICATE KEY UPDATE\n expires_at = NULL,\n deleted_at = NULL,\n ip_address = ?,\n maybe_target_user_token = ?,\n mod_user_token = ?,\n mod_notes = ?\n ", "describe": { @@ -6905,6 +5754,822 @@ ] } }, + "d58f3bca0ba61fc48f8239f88b1bf4f7ba631ff3e5c02381da1dcccb73e7f20e": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 8, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 9, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 11, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 12, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 14, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 15, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 16, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, + "d648f89461eb557770cbea6e18af4af2f2a3e2d34b536bf2f6b8b40c447a453c": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 15, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 16, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 17, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, + "d862b69c0ded636bb491c6442958c8baf3457d80791eadf880445335e5fcda0b": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n\n w2l_results.creator_ip_address,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n AND w2l_results.user_deleted_at IS NULL\n AND w2l_results.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "w2l_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "public_bucket_video_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 392 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 5, + "name": "template_title", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 9, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "maybe_template_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "maybe_template_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 12, + "name": "maybe_template_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 13, + "name": "maybe_template_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 14, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 15, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 17, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 18, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 21, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "dbce51eb1c80dded902c9509e50713483c7131f115e8ec10646e99e88110278b": { "query": "\nUPDATE tts_results\nSET\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -7373,6 +7038,135 @@ "nullable": [] } }, + "eadbe83c69c09f3e049ec7c760d1af0313cec8f3d82c0993e526738aa4d0a6dc": { + "query": "\nSELECT\n ip_bans.ip_address,\n\n ip_bans.maybe_target_user_token,\n banned_users.username as maybe_target_username,\n\n ip_bans.mod_user_token,\n mod_users.username as mod_username,\n mod_users.display_name as mod_display_name,\n\n ip_bans.mod_notes,\n ip_bans.created_at,\n ip_bans.updated_at\nFROM\n ip_address_bans AS ip_bans\nLEFT OUTER JOIN users as banned_users\n ON ip_bans.maybe_target_user_token = banned_users.token\nJOIN users as mod_users\n ON ip_bans.mod_user_token = mod_users.token\nWHERE\n ip_bans.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 1, + "name": "maybe_target_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "maybe_target_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "mod_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "mod_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "mod_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "mod_notes", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 8, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + true, + true, + false, + false, + false, + false, + false, + false + ] + } + }, "ed67238afe48c159443d215cd1fe2f0264ad840768f789d068527ae542b08194": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -7499,6 +7293,135 @@ "nullable": [] } }, + "ef828549a221fe6fad3105fede0b38a439a685d6ea7f83a0657df9cdbb80c589": { + "query": "\nSELECT\n ip_bans.ip_address,\n\n ip_bans.maybe_target_user_token,\n banned_users.username as maybe_target_username,\n\n ip_bans.mod_user_token,\n mod_users.username as mod_username,\n mod_users.display_name as mod_display_name,\n\n ip_bans.mod_notes,\n ip_bans.created_at,\n ip_bans.updated_at\nFROM\n ip_address_bans AS ip_bans\nLEFT OUTER JOIN users as banned_users\n ON ip_bans.maybe_target_user_token = banned_users.token\nJOIN users as mod_users\n ON ip_bans.mod_user_token = mod_users.token\nWHERE\n ip_bans.ip_address = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 1, + "name": "maybe_target_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "maybe_target_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "mod_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "mod_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "mod_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "mod_notes", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 8, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + true, + false, + false, + false, + false, + false, + false + ] + } + }, "f19300f4e8434c2669dac062e5a79c5ca71767c7380aab0ab40b51be8ff449d8": { "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 1 minute\nWHERE id = ?\n ", "describe": { @@ -7509,6 +7432,291 @@ "nullable": [] } }, + "fb0d98378c747ade55cb68e7e32403c77a4e41878aea586a31e11bfd82a07627": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n\n tts_results.creator_ip_address,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 2, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "tts_model_title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "maybe_model_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "maybe_model_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 10, + "name": "maybe_model_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 11, + "name": "maybe_model_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 12, + "name": "public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "public_bucket_spectrogram_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 15, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 17, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 19, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "fc0045b6fe338a903fa9f35307457d8dbb32c5838a4783fddf0c9beef7fdb4a5": { "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,\n\n jobs.model_token,\n tts.tts_model_type,\n tts.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_inference_jobs as jobs\nJOIN tts_models as tts\n ON tts.token = jobs.model_token\nLEFT OUTER JOIN tts_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", "describe": { From a82572ee5b301c1fddf26bfdc544023eec6353b1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Jun 2021 02:57:31 -0400 Subject: [PATCH 0421/1042] normalize twitter username --- .../endpoints/users/edit_profile.rs | 5 ++-- src/util/buckets/bucket_path_unifier.rs | 2 +- src/util/buckets/bucket_paths.rs | 2 +- src/validations/twitter_username.rs | 30 ++++++++++++++++--- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs index 76f45f1c44..661def0586 100644 --- a/src/http_server/endpoints/users/edit_profile.rs +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -19,7 +19,7 @@ use crate::validations::discord_username::validate_discord_username; use crate::validations::github_username::validate_github_username; use crate::validations::passwords::validate_passwords; use crate::validations::twitch_username::validate_twitch_username; -use crate::validations::twitter_username::validate_twitter_username; +use crate::validations::twitter_username::{validate_twitter_username, normalize_twitter_username_for_storage}; use crate::validations::username::validate_username; use crate::validations::username_reservations::is_reserved_username; use crate::validations::website_url::validate_website_url; @@ -160,7 +160,8 @@ pub async fn edit_profile_handler( if let Err(reason) = validate_twitter_username(trimmed) { return Err(EditProfileError::BadInput(reason)); } - twitter_username = Some(trimmed); + let normalized = normalize_twitter_username_for_storage(trimmed); + twitter_username = Some(normalized); } } diff --git a/src/util/buckets/bucket_path_unifier.rs b/src/util/buckets/bucket_path_unifier.rs index 0e8fa3b161..803355b021 100644 --- a/src/util/buckets/bucket_path_unifier.rs +++ b/src/util/buckets/bucket_path_unifier.rs @@ -163,7 +163,7 @@ impl BucketPathUnifier { #[cfg(test)] mod tests { - use crate::buckets::bucket_path_unifier::BucketPathUnifier; + use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use std::path::PathBuf; fn get_instance() -> BucketPathUnifier { diff --git a/src/util/buckets/bucket_paths.rs b/src/util/buckets/bucket_paths.rs index 6848e4176d..670a0a98e5 100644 --- a/src/util/buckets/bucket_paths.rs +++ b/src/util/buckets/bucket_paths.rs @@ -33,7 +33,7 @@ pub fn hash_to_bucket_path(file_hash: &str, #[cfg(test)] mod tests { - use crate::buckets::bucket_paths::hash_to_bucket_path; + use crate::util::buckets::bucket_paths::hash_to_bucket_path; #[test] fn test_bucket_without_root() { diff --git a/src/validations/twitter_username.rs b/src/validations/twitter_username.rs index a12d96695a..427b73b5e0 100644 --- a/src/validations/twitter_username.rs +++ b/src/validations/twitter_username.rs @@ -5,7 +5,7 @@ use regex::Regex; pub fn validate_twitter_username(username: &str) -> Result<(), String> { lazy_static! { static ref TWITTER_USERNAME_REGEX: Regex = { - Regex::new(r"^[A-Za-z0-9_]{4,15}$").expect("should be valid regex") + Regex::new("^@?[A-Za-z0-9_]{4,15}$").expect("should be valid regex") }; } @@ -13,8 +13,14 @@ pub fn validate_twitter_username(username: &str) -> Result<(), String> { return Err("twitter username is too short".to_string()); } - if username.len() > 15 { - return Err("twitter username is too long".to_string()); + if username.starts_with("@") { + if username.len() > 16 { + return Err("twitter username is too long".to_string()); + } + } else { + if username.len() > 15 { + return Err("twitter username is too long".to_string()); + } } if !TWITTER_USERNAME_REGEX.is_match(username) { @@ -24,24 +30,40 @@ pub fn validate_twitter_username(username: &str) -> Result<(), String> { Ok(()) } +/// Remove the leading '@' for consistency and better internal use. +pub fn normalize_twitter_username_for_storage(username: &str) -> String { + username.replace("@", "") +} + #[cfg(test)] mod tests { - use crate::validations::twitter_username::validate_twitter_username; + use crate::validations::twitter_username::{validate_twitter_username, normalize_twitter_username_for_storage}; #[test] fn valid_cases() { + assert!(validate_twitter_username("@echelon").is_ok()); assert!(validate_twitter_username("echelon").is_ok()); assert!(validate_twitter_username("four").is_ok()); assert!(validate_twitter_username("123456789012345").is_ok()); + assert!(validate_twitter_username("@123456789012345").is_ok()); assert!(validate_twitter_username("a_A_b_B_c_C_d_D").is_ok()); + assert!(validate_twitter_username("@a_A_b_B_c_C_d_D").is_ok()); } #[test] fn invalid_cases() { assert!(validate_twitter_username("").is_err()); + assert!(validate_twitter_username("@").is_err()); assert!(validate_twitter_username(" ").is_err()); assert!(validate_twitter_username("!!!!!!!!!!!").is_err()); assert!(validate_twitter_username("a").is_err()); + assert!(validate_twitter_username("@a").is_err()); assert!(validate_twitter_username("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").is_err()); } + + #[test] + fn test_normalize_twitter_username_for_storage() { + assert_eq!(normalize_twitter_username_for_storage("echelon"), "echelon".to_string()); + assert_eq!(normalize_twitter_username_for_storage("@echelon"), "echelon".to_string()); + } } \ No newline at end of file From 6af385dd4805837818a0269589430dcb3cc5a1f0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Jun 2021 03:10:18 -0400 Subject: [PATCH 0422/1042] normalize and correct cashapp --- .../endpoints/users/edit_profile.rs | 5 ++- src/validations/cashapp_username.rs | 38 +++++++++++++++---- src/validations/twitter_username.rs | 11 ++++-- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs index 661def0586..2ece9f5c35 100644 --- a/src/http_server/endpoints/users/edit_profile.rs +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -13,7 +13,7 @@ use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; -use crate::validations::cashapp_username::validate_cashapp_username; +use crate::validations::cashapp_username::{validate_cashapp_username, normalize_cashapp_username_for_storage}; use crate::validations::check_for_slurs::contains_slurs; use crate::validations::discord_username::validate_discord_username; use crate::validations::github_username::validate_github_username; @@ -209,7 +209,8 @@ pub async fn edit_profile_handler( if let Err(reason) = validate_cashapp_username(trimmed) { return Err(EditProfileError::BadInput(reason)); } - cashapp_username = Some(trimmed); + let normalized = normalize_cashapp_username_for_storage(trimmed); + cashapp_username = Some(normalized); } } diff --git a/src/validations/cashapp_username.rs b/src/validations/cashapp_username.rs index e51096a0cb..960b087ed9 100644 --- a/src/validations/cashapp_username.rs +++ b/src/validations/cashapp_username.rs @@ -4,16 +4,24 @@ use regex::Regex; pub fn validate_cashapp_username(username: &str) -> Result<(), String> { lazy_static! { static ref CASHAPP_USERNAME_REGEX: Regex = { - Regex::new(r"^.{1,20}$").expect("should be valid regex") + Regex::new(r"^\$?.{1,20}$").expect("should be valid regex") }; } - if username.len() < 1 { - return Err("cashapp username is too short".to_string()); - } - - if username.len() > 20 { - return Err("cashapp username is too long".to_string()); + if username.starts_with("$") { + if username.len() < 2 { + return Err("cashapp username is too short".to_string()); + } + if username.len() > 21 { + return Err("cashapp username is too long".to_string()); + } + } else { + if username.len() < 1 { + return Err("cashapp username is too short".to_string()); + } + if username.len() > 20 { + return Err("cashapp username is too long".to_string()); + } } if !CASHAPP_USERNAME_REGEX.is_match(username) { @@ -23,20 +31,34 @@ pub fn validate_cashapp_username(username: &str) -> Result<(), String> { Ok(()) } +/// Remove the leading '$' for consistency and better internal use. +pub fn normalize_cashapp_username_for_storage(username: &str) -> String { + username.replace("$", "") +} + #[cfg(test)] mod tests { - use crate::validations::cashapp_username::validate_cashapp_username; + use crate::validations::cashapp_username::{validate_cashapp_username, normalize_cashapp_username_for_storage}; #[test] fn valid_cases() { assert!(validate_cashapp_username("echelon").is_ok()); + assert!(validate_cashapp_username("$echelon").is_ok()); assert!(validate_cashapp_username("a").is_ok()); assert!(validate_cashapp_username("12345678901234567890").is_ok()); + assert!(validate_cashapp_username("$12345678901234567890").is_ok()); } #[test] fn invalid_cases() { assert!(validate_cashapp_username("").is_err()); + assert!(validate_cashapp_username("$").is_err()); assert!(validate_cashapp_username("123456789012345678901").is_err()); } + + #[test] + fn test_normalize_twitter_username_for_storage() { + assert_eq!(normalize_cashapp_username_for_storage("echelon"), "echelon".to_string()); + assert_eq!(normalize_cashapp_username_for_storage("$echelon"), "echelon".to_string()); + } } \ No newline at end of file diff --git a/src/validations/twitter_username.rs b/src/validations/twitter_username.rs index 427b73b5e0..3deed67fa8 100644 --- a/src/validations/twitter_username.rs +++ b/src/validations/twitter_username.rs @@ -9,15 +9,17 @@ pub fn validate_twitter_username(username: &str) -> Result<(), String> { }; } - if username.len() < 4 { - return Err("twitter username is too short".to_string()); - } - if username.starts_with("@") { + if username.len() < 5 { + return Err("twitter username is too short".to_string()); + } if username.len() > 16 { return Err("twitter username is too long".to_string()); } } else { + if username.len() < 4 { + return Err("twitter username is too short".to_string()); + } if username.len() > 15 { return Err("twitter username is too long".to_string()); } @@ -44,6 +46,7 @@ mod tests { assert!(validate_twitter_username("@echelon").is_ok()); assert!(validate_twitter_username("echelon").is_ok()); assert!(validate_twitter_username("four").is_ok()); + assert!(validate_twitter_username("@four").is_ok()); assert!(validate_twitter_username("123456789012345").is_ok()); assert!(validate_twitter_username("@123456789012345").is_ok()); assert!(validate_twitter_username("a_A_b_B_c_C_d_D").is_ok()); From 2f49daae7828bff9b3f6257a3c1890134c4646c9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Jun 2021 02:33:41 -0400 Subject: [PATCH 0423/1042] fix column swap bug --- debug/mysql/last_w2l_inference_results.sh | 4 ++-- src/job_queries/tts_inference_job_queries.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/mysql/last_w2l_inference_results.sh b/debug/mysql/last_w2l_inference_results.sh index ebc1e305b8..75db2a2c62 100755 --- a/debug/mysql/last_w2l_inference_results.sh +++ b/debug/mysql/last_w2l_inference_results.sh @@ -1,8 +1,8 @@ #!/bin/bash # ONLY FOR LOCAL DEV -mysql -u root \ - -proot \ +mysql -u storyteller \ + -ppassword \ -h localhost \ -D storyteller \ -e "select * from w2l_results\G" diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 3cf40b921c..3e6920e562 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -333,8 +333,8 @@ SET inference_result_token, job.model_token.clone(), job.raw_inference_text.clone(), - normalized_inference_text, text_hash, + normalized_inference_text, maybe_creator_user_token, maybe_creator_synthetic_id, From 4ded82b3c77773469b50e62ab31cd7e39868f561 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Jun 2021 02:38:40 -0400 Subject: [PATCH 0424/1042] fix path bug --- src/job_queries/tts_inference_job_queries.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 3e6920e562..9beed40a5e 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -238,7 +238,7 @@ pub async fn insert_tts_result>( .display() .to_string(); - let bucket_spectrogram_result_path = &bucket_audio_results_path + let bucket_spectrogram_result_path = &bucket_spectrogram_results_path .as_ref() .display() .to_string(); From c406972175a146f3cb28daed7ba0c5ab345fce65 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Jun 2021 02:50:23 -0400 Subject: [PATCH 0425/1042] store bucket uuid --- debug/mysql/last_w2l_inference_results.sh | 4 ++-- src/job_queries/tts_inference_job_queries.rs | 4 +++- src/tts_inference_job.rs | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/debug/mysql/last_w2l_inference_results.sh b/debug/mysql/last_w2l_inference_results.sh index 75db2a2c62..28478a985b 100755 --- a/debug/mysql/last_w2l_inference_results.sh +++ b/debug/mysql/last_w2l_inference_results.sh @@ -37,8 +37,8 @@ EOF echo $query -mysql -u root \ - -proot \ +mysql -u storyteller \ + -ppassword \ -h localhost \ -D storyteller \ -e "${query}" diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 9beed40a5e..393b92744a 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -227,6 +227,7 @@ pub async fn insert_tts_result>( text_hash: &str, bucket_audio_results_path: P, bucket_spectrogram_results_path: P, + bucket_uuid: &str, file_size_bytes: u64, duration_millis: u64 ) -> AnyhowResult<(u64, String)> @@ -325,7 +326,7 @@ SET public_bucket_hash = 'TODO', public_bucket_wav_audio_path = ?, public_bucket_spectrogram_path = ?, - inference_public_bucket_uuid = 'TODO', + inference_public_bucket_uuid = ?, file_size_bytes = ?, duration_millis = ? @@ -343,6 +344,7 @@ SET bucket_audio_result_path, bucket_spectrogram_result_path, + bucket_uuid, file_size_bytes, duration_millis diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 00d5a9b8f5..040b44f1dd 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -462,6 +462,7 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An &text_hash, &audio_result_object_path, &spectrogram_result_object_path, + &job.uuid_idempotency_token, file_metadata.file_size_bytes, file_metadata.duration_millis.unwrap_or(0)) .await?; From 8d8209d1f86b1c70e6da00e37559f0b6aa2e9ec1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Jun 2021 05:07:28 -0400 Subject: [PATCH 0426/1042] get rid of a few columns (tts results) --- .../up.sql | 7 ------- sqlx-data.json | 20 +++++++++---------- src/job_queries/tts_inference_job_queries.rs | 4 ---- src/tts_inference_job.rs | 1 - 4 files changed, 10 insertions(+), 22 deletions(-) diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 2ab9571dff..3b69328c24 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -57,17 +57,10 @@ CREATE TABLE tts_results ( -- ========== BUCKET STORAGE ========== - -- SHA2. - -- Where the wav, spectrogram, and etc. are located. - public_bucket_hash CHAR(64) NOT NULL, - -- Where the wav, spectrogram, and etc. are located. public_bucket_wav_audio_path VARCHAR(255) NOT NULL, public_bucket_spectrogram_path VARCHAR(255) NOT NULL, - -- Where the wav, spectrogram, and etc. are located. - inference_public_bucket_uuid VARCHAR(36) NOT NULL, - -- ========== METADATA ========== file_size_bytes INT(10) NOT NULL DEFAULT 0, diff --git a/sqlx-data.json b/sqlx-data.json index 4ef971e535..07627b4bc9 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -38,6 +38,16 @@ ] } }, + "04e4b27791b94ee7232d364089a7a110984e18cf10fd60e3dea7449d53c40c00": { + "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n\n file_size_bytes = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 12 + }, + "nullable": [] + } + }, "05c3ebf54cd8a5374d6f4bbe21b3c6027ce6314917af8acd0ed136d375d53342": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -1176,16 +1186,6 @@ "nullable": [] } }, - "2d485a1232d1b5bd1ad6a05ed744fe280121903fcebac4b7e9d12b732e0caa21": { - "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_hash = 'TODO',\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n inference_public_bucket_uuid = 'TODO',\n\n file_size_bytes = ?,\n duration_millis = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 12 - }, - "nullable": [] - } - }, "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", "describe": { diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 393b92744a..6b28fcf945 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -227,7 +227,6 @@ pub async fn insert_tts_result>( text_hash: &str, bucket_audio_results_path: P, bucket_spectrogram_results_path: P, - bucket_uuid: &str, file_size_bytes: u64, duration_millis: u64 ) -> AnyhowResult<(u64, String)> @@ -323,10 +322,8 @@ SET creator_ip_address = ?, creator_set_visibility = 'public', - public_bucket_hash = 'TODO', public_bucket_wav_audio_path = ?, public_bucket_spectrogram_path = ?, - inference_public_bucket_uuid = ?, file_size_bytes = ?, duration_millis = ? @@ -344,7 +341,6 @@ SET bucket_audio_result_path, bucket_spectrogram_result_path, - bucket_uuid, file_size_bytes, duration_millis diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 040b44f1dd..00d5a9b8f5 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -462,7 +462,6 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An &text_hash, &audio_result_object_path, &spectrogram_result_object_path, - &job.uuid_idempotency_token, file_metadata.file_size_bytes, file_metadata.duration_millis.unwrap_or(0)) .await?; From 84444c338044e3a6f7670599bbb33668f3b0a98d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Jun 2021 05:25:45 -0400 Subject: [PATCH 0427/1042] work on better tokens --- src/common_queries/mod.rs | 3 +- src/common_queries/tokens.rs | 44 +++++++++++++++++++ .../endpoints/users/create_account.rs | 3 +- src/job_queries/tts_download_job_queries.rs | 3 +- src/job_queries/tts_inference_job_queries.rs | 3 +- src/job_queries/w2l_download_job_queries.rs | 5 ++- src/job_queries/w2l_inference_job_queries.rs | 3 +- 7 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 src/common_queries/tokens.rs diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index a47c1dfc9f..fcff225d6a 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -11,4 +11,5 @@ pub mod query_tts_result; pub mod query_user_profile; pub mod query_w2l_result; pub mod query_w2l_template; -pub mod sessions; \ No newline at end of file +pub mod sessions; +pub mod tokens; \ No newline at end of file diff --git a/src/common_queries/tokens.rs b/src/common_queries/tokens.rs new file mode 100644 index 0000000000..e350784ee1 --- /dev/null +++ b/src/common_queries/tokens.rs @@ -0,0 +1,44 @@ +use crate::util::anyhow_result::AnyhowResult; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; + +/// Central place to define all the token types. +// TODO: Make tokens strongly typed. +pub struct Tokens {} + +TODO TODO TODO + +impl Tokens { + + pub fn new_user() -> AnyhowResult { + random_prefix_crockford_token("U:", 15) + } + + pub fn new_tts_model() -> AnyhowResult { + random_prefix_crockford_token("TM:", 15) + } + + pub fn new_tts_model_upload_job() -> AnyhowResult { + random_prefix_crockford_token("JTUP:", 32) + } + + pub fn new_tts_result() -> AnyhowResult { + random_prefix_crockford_token("TR:", 32) + } + + pub fn new_tts_inference_job() -> AnyhowResult { + random_prefix_crockford_token("JTINF:", 32) + } + + pub fn new_w2l_template() -> AnyhowResult { + random_prefix_crockford_token("WT:", 32) + } + + pub fn new_w2l_template_upload_job() -> AnyhowResult { + random_prefix_crockford_token("JTUP:", 32) + } + + pub fn new_w2l_result() -> AnyhowResult { + random_prefix_crockford_token("WR:", 32) + TODO TODO TODO + } +} diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index 4f5a658d77..812edc2cce 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -24,6 +24,7 @@ use crate::validations::username_reservations::is_reserved_username; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::validations::check_for_slurs::contains_slurs; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::common_queries::tokens::Tokens; const NEW_USER_ROLE: &'static str = "user"; @@ -98,7 +99,7 @@ pub async fn create_account_handler( return Err(CreateAccountError::BadInput("invalid email address".to_string())); } - let user_token = random_prefix_crockford_token("U:", 15) + let user_token = Tokens::new_user() .map_err(|e| { warn!("Bad crockford token: {:?}", e); CreateAccountError::ServerError diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index 478726e345..bdf6c950da 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -9,6 +9,7 @@ use crate::util::random_crockford_token::random_crockford_token; use sqlx::MySqlPool; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use std::path::Path; +use crate::common_queries::tokens::Tokens; /// table: tts_model_upload_jobs #[derive(Debug)] @@ -206,7 +207,7 @@ pub async fn insert_tts_model>( file_size_bytes: u64 ) -> AnyhowResult<(u64, String)> { - let model_token = random_prefix_crockford_token("TTS_MDL:", 32)?; + let model_token = Tokens::new_tts_model()?; let private_bucket_object_name = &private_bucket_object_name .as_ref() diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 6b28fcf945..7cd1f12fe2 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -9,6 +9,7 @@ use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn, info}; use sqlx::{MySqlPool, Transaction, MySql}; use std::path::Path; +use crate::common_queries::tokens::Tokens; /// table: tts_inference_jobs #[derive(Debug)] @@ -231,7 +232,7 @@ pub async fn insert_tts_result>( duration_millis: u64 ) -> AnyhowResult<(u64, String)> { - let inference_result_token = random_prefix_crockford_token("TTS_RES:", 32)?; + let inference_result_token = Tokens::new_tts_result()?; let bucket_audio_result_path = &bucket_audio_results_path .as_ref() diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 7aff182411..cfe9940180 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -4,10 +4,11 @@ use anyhow::anyhow; use chrono::Utc; +use crate::common_queries::tokens::Tokens; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; -use sqlx::MySqlPool; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use sqlx::MySqlPool; /// table: w2l_template_upload_jobs #[derive(Debug)] @@ -214,7 +215,7 @@ pub async fn insert_w2l_template( duration_millis: u64 ) -> AnyhowResult<(u64, String)> { - let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?; + let model_token = Tokens::new_w2l_template()?; let query_result = sqlx::query!( r#" diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index 600cf56e87..0d7987a631 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -10,6 +10,7 @@ use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn, info}; use sqlx::MySqlPool; use std::path::Path; +use crate::common_queries::tokens::Tokens; /// table: w2l_inference_jobs #[derive(Debug)] @@ -257,7 +258,7 @@ pub async fn insert_w2l_result>( duration_millis: u64 ) -> AnyhowResult<(u64, String)> { - let inference_result_token = random_prefix_crockford_token("W2L_RES:", 32)?; + let inference_result_token = Tokens::new_w2l_result()?; let bucket_video_result_path = &bucket_video_results_path .as_ref() From 51bbdf61ba244bd9d0086572d51e40ae690525ac Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Jun 2021 23:58:38 -0400 Subject: [PATCH 0428/1042] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b4e03839ee..38a1b26072 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ storyteller-web =============== -This is our main user account monolith that we'll also bake other pieces into. +This is the main user account monolith that we'll also bake other pieces into. Local development ----------------- From 6b20a1159517efc790e10e4812b8574b343bbb02 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 20 Jun 2021 11:55:22 -0400 Subject: [PATCH 0429/1042] install instructions for mac --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 38a1b26072..733d44e76f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,18 @@ If MySql in local dev can't be connected to, reset the accounts: https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux +### Install Diesel + +``` +cargo install diesel_cli +``` + +Mac [has issues](https://github.com/diesel-rs/diesel/issues/2605) and requires a few dependencies: + +``` +brew install libpq +``` + ### Database migrations To reset the entire database (drop, migrate), run: From ee4be7dc94e296b50eae41dcd5dc0fbbc0cd3b34 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 20 Jun 2021 23:55:13 -0400 Subject: [PATCH 0430/1042] instructions --- README.md | 10 ++++++++++ src/common_queries/tokens.rs | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 733d44e76f..623e6ee939 100644 --- a/README.md +++ b/README.md @@ -230,4 +230,14 @@ See: * https://stackoverflow.com/a/56354633 * https://cloud.google.com/storage/docs/access-control/making-data-public#buckets +### Generating Access Key and Secret Key + +https://cloud.telestream.net/tutorials/how-to-setting-up-google-cloud-storage/ + +1. Go to the GCS page +2. Click "settings" +3. Click "interoperability" tab +4. (enable interoperable access if not already set) +5. Click "create new key" + diff --git a/src/common_queries/tokens.rs b/src/common_queries/tokens.rs index e350784ee1..a6a6eeb1a2 100644 --- a/src/common_queries/tokens.rs +++ b/src/common_queries/tokens.rs @@ -5,7 +5,7 @@ use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; // TODO: Make tokens strongly typed. pub struct Tokens {} -TODO TODO TODO +//TODO TODO TODO impl Tokens { @@ -39,6 +39,6 @@ impl Tokens { pub fn new_w2l_result() -> AnyhowResult { random_prefix_crockford_token("WR:", 32) - TODO TODO TODO + //TODO TODO TODO } } From 847bbb935c70531bdaf4ada4ef6c65bc223ef6a8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 22 Jun 2021 02:35:13 -0400 Subject: [PATCH 0431/1042] python3.6 on apple m1 mac --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 623e6ee939..7bb73c3c48 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,17 @@ If MySql in local dev can't be connected to, reset the accounts: https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux +### Python 3.6 on Apple M1 Mac + +Python3.6 isn't supported on Apple silicon, and it's not in homebrew. It can be installed with +[nix using Rosetta](https://stackoverflow.com/a/65980989): + +Download: https://nixos.org/download.html#nix-quick-install + +``` +nix run nixpkgs.python36 -c python +``` + ### Install Diesel ``` From 3b8aa0a7c6e64677213c14d38bb505e337ebdcd4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 23 Jun 2021 01:35:29 -0400 Subject: [PATCH 0432/1042] more install instructions --- README.md | 12 ++++++++++++ debug/curl/seed.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7bb73c3c48..6805bc9f4b 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,18 @@ Download: https://nixos.org/download.html#nix-quick-install nix run nixpkgs.python36 -c python ``` +Install venv: + +``` +nix run nixpkgs.python36 -c python -m venv python +``` + +Install other packages on Mac that aren't used in venv: + +``` +python3 -m pip install --user requests gdown youtube_dl +``` + ### Install Diesel ``` diff --git a/debug/curl/seed.py b/debug/curl/seed.py index 95826c4460..46152614ee 100755 --- a/debug/curl/seed.py +++ b/debug/curl/seed.py @@ -105,7 +105,7 @@ def upload_tts_model(content_url, title, session_cookie): # Copied into Storyteller Google Drive upload_tts_model('https://drive.google.com/file/d/1d8KPwYqbRsGiYx1c4TCW4xCtcRLdIU1s/view?usp=sharing', 'Zephyr', session_echelon) -# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1yhoamPz1NOMy403gu2v8sIx5hPj3LQWs/view?usp=sharing', 'Wakko Warner', session_other) +upload_tts_model('https://drive.google.com/file/d/1yhoamPz1NOMy403gu2v8sIx5hPj3LQWs/view?usp=sharing', 'Wakko Warner', session_other) # TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1I6KGQZFMMB2ZvCzWEqy9xTgGl0L5cmfP/view?usp=sharing', 'Turret-192', session_echelon) # TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1T_hvgGcNOfofhNmDWUqpVtYDdSXQrWdE/view?usp=sharing', 'technoblade-324', session_brandon) # TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1NKyRiex3K1J0QyIv2np17D_H0rE98HyK/view?usp=sharing', 'Noire', session_other) From 68f355846f75c2f2c9f8f1018a3f089f53aa4d83 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 24 Jun 2021 23:59:22 -0400 Subject: [PATCH 0433/1042] didn't want to lose my github streak. ugh, work :( --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 6805bc9f4b..42fd6138d2 100644 --- a/README.md +++ b/README.md @@ -263,4 +263,3 @@ https://cloud.telestream.net/tutorials/how-to-setting-up-google-cloud-storage/ 4. (enable interoperable access if not already set) 5. Click "create new key" - From 10f86bf5135d915f4c077af20e9aac731de792fe Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 26 Jun 2021 13:09:29 -0400 Subject: [PATCH 0434/1042] revize tokens in the system --- src/common_queries/firehose_publisher.rs | 7 ++- src/common_queries/sessions.rs | 7 ++- src/common_queries/tokens.rs | 51 +++++++++------ .../endpoints/tts/enqueue_infer_tts.rs | 3 +- .../endpoints/tts/enqueue_upload_tts_model.rs | 4 +- .../endpoints/w2l/enqueue_infer_w2l.rs | 10 +++ .../w2l/enqueue_infer_w2l_with_uploads.rs | 3 +- .../w2l/enqueue_upload_w2l_template.rs | 9 +-- src/util/random_prefix_crockford_token.rs | 63 ++++++++++++------- 9 files changed, 104 insertions(+), 53 deletions(-) diff --git a/src/common_queries/firehose_publisher.rs b/src/common_queries/firehose_publisher.rs index 98e4e00885..4574d47c62 100644 --- a/src/common_queries/firehose_publisher.rs +++ b/src/common_queries/firehose_publisher.rs @@ -1,11 +1,12 @@ use anyhow::anyhow; +use crate::common_queries::tokens::Tokens; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn,info}; -use sqlx::{MySqlPool}; -use std::sync::Arc; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDone; +use sqlx::{MySqlPool}; +use std::sync::Arc; #[derive(Debug, Clone, Copy)] enum FirehoseEvent { @@ -167,7 +168,7 @@ impl FirehosePublisher { entity_token: Option<&str>, created_entity_token: Option<&str> ) -> AnyhowResult { - let token = random_prefix_crockford_token("EV", 32)?; + let token = Tokens::new_firehose_event()?; let query_result = sqlx::query!( r#" diff --git a/src/common_queries/sessions.rs b/src/common_queries/sessions.rs index ce3acbe725..991338c84f 100644 --- a/src/common_queries/sessions.rs +++ b/src/common_queries/sessions.rs @@ -1,14 +1,15 @@ use anyhow::anyhow; -use log::{info, warn}; use crate::AnyhowResult; +use crate::common_queries::tokens::Tokens; use crate::util::random_crockford_token::random_crockford_token; -use sqlx::MySqlPool; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use log::{info, warn}; +use sqlx::MySqlPool; pub async fn create_session_for_user(user_token: &str, ip_address: &str, mysql_pool: &MySqlPool) -> AnyhowResult { - let session_token = random_prefix_crockford_token("SESSION:", 32)?; + let session_token = Tokens::new_session()?; let query_result = sqlx::query!( r#" diff --git a/src/common_queries/tokens.rs b/src/common_queries/tokens.rs index a6a6eeb1a2..a2926eba47 100644 --- a/src/common_queries/tokens.rs +++ b/src/common_queries/tokens.rs @@ -9,36 +9,51 @@ pub struct Tokens {} impl Tokens { + // ========== Tokens with high visibility (make these nice) ========== + pub fn new_user() -> AnyhowResult { - random_prefix_crockford_token("U:", 15) + random_prefix_crockford_token("U:", 15, false) } pub fn new_tts_model() -> AnyhowResult { - random_prefix_crockford_token("TM:", 15) - } - - pub fn new_tts_model_upload_job() -> AnyhowResult { - random_prefix_crockford_token("JTUP:", 32) + random_prefix_crockford_token("TM:", 15, false) } pub fn new_tts_result() -> AnyhowResult { - random_prefix_crockford_token("TR:", 32) - } - - pub fn new_tts_inference_job() -> AnyhowResult { - random_prefix_crockford_token("JTINF:", 32) + random_prefix_crockford_token("TR:", 32, false) } pub fn new_w2l_template() -> AnyhowResult { - random_prefix_crockford_token("WT:", 32) - } - - pub fn new_w2l_template_upload_job() -> AnyhowResult { - random_prefix_crockford_token("JTUP:", 32) + random_prefix_crockford_token("WT:", 32, false) } pub fn new_w2l_result() -> AnyhowResult { - random_prefix_crockford_token("WR:", 32) - //TODO TODO TODO + random_prefix_crockford_token("WR:", 32, false) + } + + // ========== Tokens not typically visible to users ========== + + pub fn new_tts_model_upload_job() -> AnyhowResult { + random_prefix_crockford_token("JTUP:", 32, false) + } + + pub fn new_tts_inference_job() -> AnyhowResult { + random_prefix_crockford_token("JTINF:", 32, false) + } + + pub fn new_w2l_template_upload_job() -> AnyhowResult { + random_prefix_crockford_token("JWUP:", 32, false) + } + + pub fn new_w2l_inference_job() -> AnyhowResult { + random_prefix_crockford_token("JWINF:", 32, false) + } + + pub fn new_firehose_event() -> AnyhowResult { + random_prefix_crockford_token("EV:", 32, false) + } + + pub fn new_session() -> AnyhowResult { + random_prefix_crockford_token("SESSION:", 32, false) } } diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 86639b3c4a..07297108a5 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -5,6 +5,7 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::tokens::Tokens; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -112,7 +113,7 @@ pub async fn infer_tts_handler( let creator_set_visibility = "public".to_string(); // This token is returned to the client. - let job_token = random_prefix_crockford_token("TTS_INF:", 32) + let job_token = Tokens::new_tts_inference_job() .map_err(|e| { warn!("Error creating token"); InferTtsError::ServerError diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index 25445fde53..6614dcfbb7 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -5,6 +5,7 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::tokens::Tokens; use crate::database_helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -105,13 +106,12 @@ pub async fn upload_tts_model_handler( let creator_set_visibility = "public".to_string(); // This token is returned to the client. - let job_token = random_prefix_crockford_token("TTS_UP:", 32) + let job_token = Tokens::new_tts_model_upload_job() .map_err(|e| { warn!("Error creating token"); UploadTtsModelError::ServerError })?; - let query_result = sqlx::query!( r#" INSERT INTO tts_model_upload_jobs diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index 8f764b44e8..d2da24e8a9 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -64,6 +64,12 @@ pub async fn infer_w2l_handler( request: web::Json, server_state: web::Data>) -> Result { + + if true { + unimplemented!("this isn't finished"); + } + + let maybe_session = server_state .session_checker .maybe_get_session(&http_request, &server_state.mysql_pool) @@ -119,6 +125,10 @@ pub async fn infer_w2l_handler( let ip_address = get_request_ip(&http_request); let creator_set_visibility = "public".to_string(); + if true { + unimplemented!("this isn't finished"); + } + let query_result = sqlx::query!( r#" INSERT INTO w2l_inference_jobs diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index ceb1e29007..460b19c280 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -7,6 +7,7 @@ use actix_web::http::header::ContentDisposition; use actix_web::web::{Data, Json, BytesMut}; use actix_web::{middleware, web, App, Error, HttpResponse, HttpServer, HttpRequest, Either, ResponseError}; use anyhow::anyhow; +use crate::common_queries::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::read_multipart_field_bytes::checked_read_multipart_bytes; use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_boolean; @@ -265,7 +266,7 @@ pub async fn enqueue_infer_w2l_with_uploads( // ==================== SAVE JOB RECORD ==================== // // This token is returned to the client. - let job_token = random_prefix_crockford_token("W2L_INF:", 32) + let job_token = Tokens::new_w2l_inference_job() .map_err(|e| { warn!("Error creating token"); InferW2lWithUploadError::ServerError diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index 5bb5b29128..ddd45089c6 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -5,10 +5,14 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::tokens::Tokens; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; @@ -19,9 +23,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; -use crate::validations::check_for_slurs::contains_slurs; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct UploadW2lTemplateRequest { @@ -110,7 +111,7 @@ pub async fn upload_w2l_template_handler( let creator_set_visibility = "public".to_string(); // This token is returned to the client. - let job_token = random_prefix_crockford_token("W2L_UP:", 32) + let job_token = Tokens::new_w2l_template_upload_job() .map_err(|e| { warn!("Error creating token"); UploadW2lTemplateError::ServerError diff --git a/src/util/random_prefix_crockford_token.rs b/src/util/random_prefix_crockford_token.rs index 9ff0588d26..7026a6ca84 100644 --- a/src/util/random_prefix_crockford_token.rs +++ b/src/util/random_prefix_crockford_token.rs @@ -3,12 +3,14 @@ use anyhow::anyhow; use crate::util::anyhow_result::AnyhowResult; // Crockford characters -const CROCKFORD_CHARSET: &[u8] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; +const CROCKFORD_UPPERCASE_CHARSET: &[u8] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + +const CROCKFORD_LOWERCASE_CHARSET: &[u8] = b"0123456789abcdefghjkmnpqrstvwxyz"; // Random part can't be any less than this. const MIN_ENTROPY_LENGTH : usize = 8; -pub fn random_prefix_crockford_token(prefix: &str, length: usize) -> AnyhowResult { +pub fn random_prefix_crockford_token(prefix: &str, length: usize, uppercase: bool) -> AnyhowResult { let mut prefix = prefix.trim().to_string(); if prefix.is_empty() { @@ -27,10 +29,12 @@ pub fn random_prefix_crockford_token(prefix: &str, length: usize) -> AnyhowResul let mut rng = rand::thread_rng(); + let charset = if uppercase { CROCKFORD_UPPERCASE_CHARSET } else { CROCKFORD_LOWERCASE_CHARSET }; + let entropy_part: String = (0..entropy_length) .map(|_| { - let idx = rng.gen_range(0..CROCKFORD_CHARSET.len()); - CROCKFORD_CHARSET[idx] as char + let idx = rng.gen_range(0..charset.len()); + charset[idx] as char }) .collect(); @@ -44,34 +48,51 @@ mod tests { #[test] fn no_prefix_error() { - assert!(random_prefix_crockford_token("", 0).is_err()); - assert!(random_prefix_crockford_token("", 10).is_err()); - assert!(random_prefix_crockford_token("", 32).is_err()); + assert!(random_prefix_crockford_token("", 0, false).is_err()); + assert!(random_prefix_crockford_token("", 10, false).is_err()); + assert!(random_prefix_crockford_token("", 32, false).is_err()); + assert!(random_prefix_crockford_token("", 0, true).is_err()); + assert!(random_prefix_crockford_token("", 10, true).is_err()); + assert!(random_prefix_crockford_token("", 32, true).is_err()); } #[test] fn not_enough_entropy_error() { - assert!(random_prefix_crockford_token("foo", 0).is_err()); - assert!(random_prefix_crockford_token("foo", 5).is_err()); - assert!(random_prefix_crockford_token("foo", 11).is_err()); - assert!(random_prefix_crockford_token("foo:", 11).is_err()); + assert!(random_prefix_crockford_token("foo", 0, false).is_err()); + assert!(random_prefix_crockford_token("foo", 5, false).is_err()); + assert!(random_prefix_crockford_token("foo", 11, false).is_err()); + assert!(random_prefix_crockford_token("foo:", 11, false).is_err()); + assert!(random_prefix_crockford_token("foo", 0, true).is_err()); + assert!(random_prefix_crockford_token("foo", 5, true).is_err()); + assert!(random_prefix_crockford_token("foo", 11, true).is_err()); + assert!(random_prefix_crockford_token("foo:", 11, true).is_err()); } #[test] fn starts_with() { - assert!(random_prefix_crockford_token("w", 10).unwrap().starts_with("w:")); - assert!(random_prefix_crockford_token("w", 10).unwrap().starts_with("w:")); - assert!(random_prefix_crockford_token("prefix", 16).unwrap().starts_with("prefix:")); - assert!(random_prefix_crockford_token("prefix:", 16).unwrap().starts_with("prefix:")); + assert!(random_prefix_crockford_token("w", 10, false).unwrap().starts_with("w:")); + assert!(random_prefix_crockford_token("w", 10, false).unwrap().starts_with("w:")); + assert!(random_prefix_crockford_token("prefix", 16, false).unwrap().starts_with("prefix:")); + assert!(random_prefix_crockford_token("prefix:", 16, false).unwrap().starts_with("prefix:")); + assert!(random_prefix_crockford_token("w", 10, true).unwrap().starts_with("w:")); + assert!(random_prefix_crockford_token("w", 10, true).unwrap().starts_with("w:")); + assert!(random_prefix_crockford_token("prefix", 16, true).unwrap().starts_with("prefix:")); + assert!(random_prefix_crockford_token("prefix:", 16, true).unwrap().starts_with("prefix:")); } #[test] fn length() { - assert_eq!(random_prefix_crockford_token("w", 10).unwrap().len(), 10); - assert_eq!(random_prefix_crockford_token("w:", 10).unwrap().len(), 10); - assert_eq!(random_prefix_crockford_token("prefix", 16).unwrap().len(), 16); - assert_eq!(random_prefix_crockford_token("prefix:", 16).unwrap().len(), 16); - assert_eq!(random_prefix_crockford_token("asdf:", 32).unwrap().len(), 32); - assert_eq!(random_prefix_crockford_token("asdf", 32).unwrap().len(), 32); + assert_eq!(random_prefix_crockford_token("w", 10, false).unwrap().len(), 10); + assert_eq!(random_prefix_crockford_token("w:", 10, false).unwrap().len(), 10); + assert_eq!(random_prefix_crockford_token("prefix", 16, false).unwrap().len(), 16); + assert_eq!(random_prefix_crockford_token("prefix:", 16, false).unwrap().len(), 16); + assert_eq!(random_prefix_crockford_token("asdf:", 32, false).unwrap().len(), 32); + assert_eq!(random_prefix_crockford_token("asdf", 32, false).unwrap().len(), 32); + assert_eq!(random_prefix_crockford_token("w", 10, true).unwrap().len(), 10); + assert_eq!(random_prefix_crockford_token("w:", 10, true).unwrap().len(), 10); + assert_eq!(random_prefix_crockford_token("prefix", 16, true).unwrap().len(), 16); + assert_eq!(random_prefix_crockford_token("prefix:", 16, true).unwrap().len(), 16); + assert_eq!(random_prefix_crockford_token("asdf:", 32, true).unwrap().len(), 32); + assert_eq!(random_prefix_crockford_token("asdf", 32, true).unwrap().len(), 32); } } From c3464c4fe392345463d8ebf6305d5d58a39a103f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 26 Jun 2021 13:46:38 -0400 Subject: [PATCH 0435/1042] uppercase user tokens --- debug/curl/seed.py | 1 + src/common_queries/tokens.rs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/debug/curl/seed.py b/debug/curl/seed.py index 46152614ee..218e71e563 100755 --- a/debug/curl/seed.py +++ b/debug/curl/seed.py @@ -75,6 +75,7 @@ upload_w2l_template('https://drive.google.com/file/d/1yoBSb6nwFUZVI4CD-nM2BSofGd upload_w2l_template('https://drive.google.com/file/d/1ysBAdRjlSmfWQPFjGlMJpv_zcGmVw7RC/view?usp=sharing', 'short bill clinton clip', session_brandon) upload_w2l_template('https://i.imgur.com/NneE7Eq.jpg', 'itsa me mario!', session_echelon) upload_w2l_template('https://i.imgur.com/W1EYiPU.jpg', 'Creepy Sonic does a gasp', session_echelon) +upload_w2l_template('https://i.imgur.com/8U3IdUa.png', 'Creepy Sonic actually talks', session_echelon) upload_w2l_template('https://i.imgur.com/lKaQ4Er.jpg', 'hasta la vista', session_brandon) upload_w2l_template('https://i.imgur.com/uoGuTJo.jpeg', 'what do you mean vern?', session_brandon) upload_w2l_template('https://i.imgur.com/v8k9yau.jpeg', 'Miyamoto-san', session_brandon) diff --git a/src/common_queries/tokens.rs b/src/common_queries/tokens.rs index a2926eba47..9f906ebe1c 100644 --- a/src/common_queries/tokens.rs +++ b/src/common_queries/tokens.rs @@ -11,10 +11,6 @@ impl Tokens { // ========== Tokens with high visibility (make these nice) ========== - pub fn new_user() -> AnyhowResult { - random_prefix_crockford_token("U:", 15, false) - } - pub fn new_tts_model() -> AnyhowResult { random_prefix_crockford_token("TM:", 15, false) } @@ -33,6 +29,14 @@ impl Tokens { // ========== Tokens not typically visible to users ========== + pub fn new_user() -> AnyhowResult { + random_prefix_crockford_token("U:", 15, true) + } + + pub fn new_session() -> AnyhowResult { + random_prefix_crockford_token("SESSION:", 32, false) + } + pub fn new_tts_model_upload_job() -> AnyhowResult { random_prefix_crockford_token("JTUP:", 32, false) } @@ -52,8 +56,4 @@ impl Tokens { pub fn new_firehose_event() -> AnyhowResult { random_prefix_crockford_token("EV:", 32, false) } - - pub fn new_session() -> AnyhowResult { - random_prefix_crockford_token("SESSION:", 32, false) - } } From 949e707c7e0046613edce192ca299410f7cf37c3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 26 Jun 2021 14:19:57 -0400 Subject: [PATCH 0436/1042] update 404, index, and alpha controllers --- src/http_server/endpoints/default_route_404.rs | 2 +- src/http_server/endpoints/misc/enable_alpha.rs | 10 +++++----- src/http_server/endpoints/root_index.rs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/http_server/endpoints/default_route_404.rs b/src/http_server/endpoints/default_route_404.rs index ad5f379a43..6bc7403c15 100644 --- a/src/http_server/endpoints/default_route_404.rs +++ b/src/http_server/endpoints/default_route_404.rs @@ -6,5 +6,5 @@ pub async fn default_route_404() -> impl Responder { warn!("404 not found"); HttpResponse::NotFound() .content_type("text/html; charset=utf-8") - .body("

nope

") + .body("

not found

") } diff --git a/src/http_server/endpoints/misc/enable_alpha.rs b/src/http_server/endpoints/misc/enable_alpha.rs index 9f48d5134e..783088435e 100644 --- a/src/http_server/endpoints/misc/enable_alpha.rs +++ b/src/http_server/endpoints/misc/enable_alpha.rs @@ -9,8 +9,8 @@ use std::sync::Arc; const ALPHA_COOKIE_NAME : &'static str = "enable-alpha"; const CONTENT_TYPE : &'static str = "text/html; charset=utf-8"; -const ENABLE_LINK : &'static str = "enable"; -const DISABLE_LINK : &'static str = "disable"; +const ENABLE_LINK : &'static str = "I want Vocodes 2.0"; +const DISABLE_LINK : &'static str = "I don't want Vocodes 2.0"; const STATUS_LINK : &'static str = "status"; @@ -39,21 +39,21 @@ pub async fn enable_alpha(http_request: HttpRequest, HttpResponse::build(StatusCode::OK) .content_type(CONTENT_TYPE) - .body(format!("

Alpha mode.

Current `{}` cookie state = {}

{} | {} | {}", + .body(format!("

Vocodes 2.0

Current `{}` cookie state = {}

{} | {} | {}", ALPHA_COOKIE_NAME, cookie_exists, ENABLE_LINK, DISABLE_LINK, STATUS_LINK)) } Some(true) => { HttpResponse::build(StatusCode::OK) .content_type(CONTENT_TYPE) .cookie(cookie) - .body(format!("

Alpha mode

setting `{}` cookie

{} | {} | {}", + .body(format!("

Vocodes 2.0

setting `{}` cookie

{} | {} | {}", ALPHA_COOKIE_NAME, ENABLE_LINK, DISABLE_LINK, STATUS_LINK)) } Some(false) => { HttpResponse::build(StatusCode::OK) .content_type(CONTENT_TYPE) .del_cookie(&cookie) - .body(format!("

Alpha mode

unsetting `{}` cookie

{} | {} | {}", + .body(format!("

Vocodes 2.0

unsetting `{}` cookie

{} | {} | {}", ALPHA_COOKIE_NAME, ENABLE_LINK, DISABLE_LINK, STATUS_LINK)) } } diff --git a/src/http_server/endpoints/root_index.rs b/src/http_server/endpoints/root_index.rs index 3f49e31b9f..bfab8f441f 100644 --- a/src/http_server/endpoints/root_index.rs +++ b/src/http_server/endpoints/root_index.rs @@ -7,5 +7,5 @@ pub async fn get_root_index() -> impl Responder { debug!("GET /"); // NB: Google load balancer hits this a lot, and it spams. HttpResponse::build(StatusCode::OK) .content_type("text/html; charset=utf-8") - .body("

hello friend

") + .body("

hello!

Are you looking for an API? Join our Discord!

Maybe you want to work with us? We can pay! Get in touch!

") } From 0ca7ceadabf0db73f51d2d625cc8b62b2c021329 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 26 Jun 2021 20:21:20 -0400 Subject: [PATCH 0437/1042] completely reorganize readme --- README.md | 329 +++++++++++++++++++++++++++--------------------------- 1 file changed, 166 insertions(+), 163 deletions(-) diff --git a/README.md b/README.md index 42fd6138d2..2e47d24bc2 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This is the main user account monolith that we'll also bake other pieces into. Local development ----------------- -### Setup development +### Database Setup Install the following libraries, and see the notes further below about MySQL on Ubuntu 20.04. @@ -25,176 +25,19 @@ cargo install sqlx-cli --no-default-features --features mysql cargo install diesel_cli --no-default-features --features mysql ``` +#### Linux database notes + If MySql in local dev can't be connected to, reset the accounts: https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux -### Python 3.6 on Apple M1 Mac - -Python3.6 isn't supported on Apple silicon, and it's not in homebrew. It can be installed with -[nix using Rosetta](https://stackoverflow.com/a/65980989): - -Download: https://nixos.org/download.html#nix-quick-install - -``` -nix run nixpkgs.python36 -c python -``` - -Install venv: - -``` -nix run nixpkgs.python36 -c python -m venv python -``` - -Install other packages on Mac that aren't used in venv: - -``` -python3 -m pip install --user requests gdown youtube_dl -``` - -### Install Diesel - -``` -cargo install diesel_cli -``` - -Mac [has issues](https://github.com/diesel-rs/diesel/issues/2605) and requires a few dependencies: - -``` -brew install libpq -``` - -### Database migrations - -To reset the entire database (drop, migrate), run: - -``` -diesel database reset -``` - -To migrate at the current step and beyond: - -``` -diesel migration run -``` - -To undo migrations at the current step: - -``` -diesel migration redo -``` - -To save sqlx database state to cache file (necessary for builds): - -``` -SQLX_OFFLINE=true cargo sqlx prepare -``` - -Now that we have multiple binaries, it's required to include all the queries in the main -binary so we can generate the cached queries as a single target. That's then executed -with: - -``` -SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web -``` - -Production ----------- - -### Database migrations - -1. Set `DATABASE_URL` in `.env` to the production secrets (DO NOT COMMIT!) -2. Run `diesel migration run` - -Actix notes ------------ - -json request -``` -async fn handler(request: web::Json) -> impl Responder { "whatever" } -``` - -form-multipart request -``` -async fn handler(request: web::Form) -> impl Responder { "whatever" } -``` - -route parameters -``` -#[get("/{name}")] -async fn hello(name: web::Path) -> impl Responder { format!("Hi {}", name) } -``` - - -TODO ----- - -* Make all "token" types correct -* Use correct collation for all tables -* Install indexes on all foreign keys -* Make sure unique indexes are correct - -Notes / TODOs: - -* Examples for good Actix+Sqlx Tests: - https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr - -* Actix/sqlx runtime compat: - https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734 - - -Docker builds -------------- - -The repository needs to be given read access to the base docker image: - -https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings - -Local Nginx Proxy ------------------ -Set up a local nginx to proxy to the frontend and backend so cookie issues aren't annoying - -Edit, `/etc/nginx/sites-enabled/storyteller` - -``` -server { - listen 80; - server_name api.jungle.horse; - location / { - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:12345; - proxy_redirect off; - } -} -server { - listen 80; - server_name jungle.horse; - location / { - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:7000; - proxy_redirect off; - } -} - -``` - -And in /etc/hosts, - -``` -127.0.0.1 jungle.horse -127.0.0.1 api.jungle.horse - -``` - -Fixing dev MySql on Ubuntu 20.04 --------------------------------- +#### Fixing dev MySql on Ubuntu 20.04 For some reason, the MySql default install on 20.04 gave me a bunch of trouble. In retrospect, I _think_ this is because 'root@localhost' requires sudo to access, but if this gives any trouble in the future, here's how I got around it (two full hours of distraction!) - ``` sudo apt-get install mysql-server @@ -239,10 +82,132 @@ CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password'; GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost'; ``` +### Install Diesel (migrations only) + +We use Diesel to manage migrations, but we don't use it in server code. +Actual server code uses SQLx. To install the CLI tool, run the following: + +``` +cargo install diesel_cli +``` + +Mac [has issues](https://github.com/diesel-rs/diesel/issues/2605) and requires a few dependencies: + +``` +brew install libpq +``` + +### Database migrations + +To reset the entire database (drop, migrate), run: + +``` +diesel database reset +``` + +To migrate at the current step and beyond: + +``` +diesel migration run +``` + +To undo migrations at the current step: + +``` +diesel migration redo +``` + +### Server Query Codegen + +We use SQLx instead of Diesel in the production server. It's typesafe +SQL instead of an ORM like Diesel. + +SQLx connects to a database to derive type information, but obviously +cannot do this for builds in CI. In order to cache the types, we build +and check in a cache file (necessary for builds): + +``` +SQLX_OFFLINE=true cargo sqlx prepare +``` + +Now that we have multiple binaries, it's required to include all the queries in the main +binary so we can generate the cached queries as a single target. That's then executed +with: + +``` +SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web +``` + +### Setting Up a Local Nginx Proxy + +Set up a local nginx to proxy to the frontend and backend so cookie issues aren't annoying + +Edit, `/etc/nginx/sites-enabled/storyteller` + +``` +server { + listen 80; + server_name api.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:12345; + proxy_redirect off; + } +} +server { + listen 80; + server_name jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:7000; + proxy_redirect off; + } +} + +``` + +And in /etc/hosts, + +``` +127.0.0.1 jungle.horse +127.0.0.1 api.jungle.horse + +``` + +### Python 3.6 on Apple M1 Mac + +Python3.6 isn't supported on Apple silicon, and it's not in homebrew. It can be installed with +[nix using Rosetta](https://stackoverflow.com/a/65980989): + +Download: https://nixos.org/download.html#nix-quick-install + +``` +nix run nixpkgs.python36 -c python +``` + +Install venv: + +``` +nix run nixpkgs.python36 -c python -m venv python +``` + +Install other packages on Mac that aren't used in venv: + +``` +python3 -m pip install --user requests gdown youtube_dl +``` + Production ---------- -### Public buckets without list permission +These instructions assume running on GCP. + +### Database migrations + +1. Set `DATABASE_URL` in `.env` to the production secrets (DO NOT COMMIT!) +2. Run `diesel migration run` + +### Setting up public buckets without list permission Public buckets that deny the `list` action should use the following Role: @@ -253,7 +218,7 @@ See: * https://stackoverflow.com/a/56354633 * https://cloud.google.com/storage/docs/access-control/making-data-public#buckets -### Generating Access Key and Secret Key +### Generating Bucket Access Key and Secret Key https://cloud.telestream.net/tutorials/how-to-setting-up-google-cloud-storage/ @@ -263,3 +228,41 @@ https://cloud.telestream.net/tutorials/how-to-setting-up-google-cloud-storage/ 4. (enable interoperable access if not already set) 5. Click "create new key" +Actix notes +----------- + +json request +``` +async fn handler(request: web::Json) -> impl Responder { "whatever" } +``` + +form-multipart request +``` +async fn handler(request: web::Form) -> impl Responder { "whatever" } +``` + +route parameters +``` +#[get("/{name}")] +async fn hello(name: web::Path) -> impl Responder { format!("Hi {}", name) } +``` + +TODO +---- + +Notes / TODOs: + +* Examples for good Actix+Sqlx Tests: + https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr + +* Actix/sqlx runtime compat: + https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734 + + +Docker builds +------------- + +The repository needs to be given read access to the base docker image: + +https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings + From 6b745fd3b06fdf647d4ac9bfa298c2a397af0ab8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 27 Jun 2021 00:03:48 -0400 Subject: [PATCH 0438/1042] user badges + user badge granter + integrated into grant call sites --- debug/mysql/badges.sh | 7 + migrations/2021-05-15-212920_badges/up.sql | 3 + seed/bootstrap_inserts_roles_etc.sh | 10 +- seed/sql/user_badges.sql | 146 +++++++ src/common_queries/badge_granter.rs | 366 ++++++++++++++++++ src/common_queries/firehose_publisher.rs | 14 + src/common_queries/mod.rs | 1 + .../endpoints/users/create_account.rs | 8 + src/main.rs | 9 +- src/server_state.rs | 4 +- src/tts_download_job.rs | 15 + src/w2l_download_job.rs | 15 + 12 files changed, 595 insertions(+), 3 deletions(-) create mode 100755 debug/mysql/badges.sh create mode 100644 seed/sql/user_badges.sql create mode 100644 src/common_queries/badge_granter.rs diff --git a/debug/mysql/badges.sh b/debug/mysql/badges.sh new file mode 100755 index 0000000000..ecc8cad037 --- /dev/null +++ b/debug/mysql/badges.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u storyteller \ + -ppassword \ + -h localhost \ + -D storyteller \ + -e "select * from badges\G" diff --git a/migrations/2021-05-15-212920_badges/up.sql b/migrations/2021-05-15-212920_badges/up.sql index 31e2ccf155..0b6e499cea 100644 --- a/migrations/2021-05-15-212920_badges/up.sql +++ b/migrations/2021-05-15-212920_badges/up.sql @@ -31,6 +31,9 @@ CREATE TABLE user_badges ( user_token VARCHAR(32) NOT NULL, badge_slug VARCHAR(32) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + -- INDICES -- PRIMARY KEY (id), KEY fk_user_token (user_token), diff --git a/seed/bootstrap_inserts_roles_etc.sh b/seed/bootstrap_inserts_roles_etc.sh index 109a1213ad..0c30f7959b 100755 --- a/seed/bootstrap_inserts_roles_etc.sh +++ b/seed/bootstrap_inserts_roles_etc.sh @@ -1,12 +1,20 @@ #!/bin/bash # ONLY FOR LOCAL DEV - # --default-character-set=UTF8mb4 \ + +echo 'Inserting System User Roles...' mysql -u storyteller \ -ppassword \ -h localhost \ -D storyteller \ -e "source ./seed/sql/system_roles.sql" +echo 'Inserting Badges...' +mysql -u storyteller \ + -ppassword \ + -h localhost \ + -D storyteller \ + -e "source ./seed/sql/user_badges.sql" + diff --git a/seed/sql/user_badges.sql b/seed/sql/user_badges.sql new file mode 100644 index 0000000000..5e05455939 --- /dev/null +++ b/seed/sql/user_badges.sql @@ -0,0 +1,146 @@ +-- USER SIGNUP / TIME LIMITED + +INSERT INTO badges +SET + slug = 'early_user', + title = 'Early Adopter of Vocodes', + description = 'Early Adopter of Vocodes', + image_url = ''; + +-- TTS MODEL UPLOADS + +INSERT INTO badges +SET + slug = 'tts_model_uploader_1', + title = 'Uploaded 1 TTS Model', + description = 'Uploaded 1 TTS model to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'tts_model_uploader_5', + title = 'Uploaded 5 TTS Models', + description = 'Uploaded 5 TTS models to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'tts_model_uploader_10', + title = 'Uploaded 10 TTS Models', + description = 'Uploaded 10 TTS models to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'tts_model_uploader_20', + title = 'Uploaded 20 TTS Models', + description = 'Uploaded 20 TTS models to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'tts_model_uploader_50', + title = 'Uploaded 50 TTS Models', + description = 'Uploaded 50 TTS models to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'tts_model_uploader_100', + title = 'Uploaded 100 TTS Models', + description = 'Uploaded 100 TTS models to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'tts_model_uploader_150', + title = 'Uploaded 150 TTS Models', + description = 'Uploaded 150 TTS models to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'tts_model_uploader_200', + title = 'Uploaded 200 TTS Models', + description = 'Uploaded 200 TTS models to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'tts_model_uploader_250', + title = 'Uploaded 250 TTS Models', + description = 'Uploaded 250 TTS models to Vocodes', + image_url = ''; + +-- W2L TEMPLATE UPLOADS + +INSERT INTO badges +SET + slug = 'w2l_template_uploader_1', + title = 'Uploaded 1 W2L Template', + description = 'Uploaded 1 W2L Template to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'w2l_template_uploader_10', + title = 'Uploaded 10 W2L Templates', + description = 'Uploaded 10 W2L Templates to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'w2l_template_uploader_50', + title = 'Uploaded 50 W2L Templates', + description = 'Uploaded 50 W2L Templates to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'w2l_template_uploader_100', + title = 'Uploaded 100 W2L Templates', + description = 'Uploaded 100 W2L Templates to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'w2l_template_uploader_200', + title = 'Uploaded 200 W2L Templates', + description = 'Uploaded 200 W2L Templates to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'w2l_template_uploader_500', + title = 'Uploaded 500 W2L Templates', + description = 'Uploaded 500 W2L Templates to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'w2l_template_uploader_1000', + title = 'Uploaded 1000 W2L Templates', + description = 'Uploaded 1000 W2L Templates to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'w2l_template_uploader_2000', + title = 'Uploaded 2000 W2L Templates', + description = 'Uploaded 2000 W2L Templates to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'w2l_template_uploader_5000', + title = 'Uploaded 5000 W2L Templates', + description = 'Uploaded 5000 W2L Templates to Vocodes', + image_url = ''; + +INSERT INTO badges +SET + slug = 'w2l_template_uploader_10000', + title = 'Uploaded 10000 W2L Templates', + description = 'Uploaded 10000 W2L Templates to Vocodes', + image_url = ''; + diff --git a/src/common_queries/badge_granter.rs b/src/common_queries/badge_granter.rs new file mode 100644 index 0000000000..7c19b458eb --- /dev/null +++ b/src/common_queries/badge_granter.rs @@ -0,0 +1,366 @@ +use anyhow::anyhow; +use crate::common_queries::tokens::Tokens; +use crate::util::anyhow_result::AnyhowResult; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use log::{warn,info}; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDone; +use sqlx::{MySqlPool}; +use std::sync::Arc; +use crate::common_queries::firehose_publisher::FirehosePublisher; + +#[derive(Debug, Clone, Copy)] +pub enum UserBadge { + // Granted for early vocodes users. + EarlyUser, + + // Granted for uploading models + TtsModelUploader1, + TtsModelUploader5, + TtsModelUploader10, + TtsModelUploader20, + TtsModelUploader50, + TtsModelUploader100, + TtsModelUploader150, + TtsModelUploader200, + TtsModelUploader250, + + // Granted for uploading templates + W2lTemplateUploader1, + W2lTemplateUploader10, + W2lTemplateUploader50, + W2lTemplateUploader100, + W2lTemplateUploader200, + W2lTemplateUploader500, + W2lTemplateUploader1000, + W2lTemplateUploader2000, + W2lTemplateUploader5000, + W2lTemplateUploader10000, + + //// Granted for using models + //TtsInferenceUser100, + //TtsInferenceUser500, + //TtsInferenceUser1000, + //TtsInferenceUser10000, + //TtsInferenceUser100000, + + //// Granted for using templates + //W2lInferenceUser100, + //W2lInferenceUser500, + //W2lInferenceUser1000, + //W2lInferenceUser10000, + //W2lInferenceUser100000, +} + +impl UserBadge { + pub fn to_db_value(&self) -> &'static str { + match self { + UserBadge::EarlyUser => "early_user", + UserBadge::TtsModelUploader1 => "tts_model_uploader_1", + UserBadge::TtsModelUploader5 => "tts_model_uploader_5", + UserBadge::TtsModelUploader10 => "tts_model_uploader_10", + UserBadge::TtsModelUploader20 => "tts_model_uploader_20", + UserBadge::TtsModelUploader50 => "tts_model_uploader_50", + UserBadge::TtsModelUploader100 => "tts_model_uploader_100", + UserBadge::TtsModelUploader150 => "tts_model_uploader_150", + UserBadge::TtsModelUploader200 => "tts_model_uploader_200", + UserBadge::TtsModelUploader250 => "tts_model_uploader_250", + UserBadge::W2lTemplateUploader1 => "w2l_template_uploader_1", + UserBadge::W2lTemplateUploader10 => "w2l_template_uploader_10", + UserBadge::W2lTemplateUploader50 => "w2l_template_uploader_50", + UserBadge::W2lTemplateUploader100 => "w2l_template_uploader_100", + UserBadge::W2lTemplateUploader200 => "w2l_template_uploader_200", + UserBadge::W2lTemplateUploader500 => "w2l_template_uploader_500", + UserBadge::W2lTemplateUploader1000 => "w2l_template_uploader_1000", + UserBadge::W2lTemplateUploader2000 => "w2l_template_uploader_2000", + UserBadge::W2lTemplateUploader5000 => "w2l_template_uploader_5000", + UserBadge::W2lTemplateUploader10000 => "w2l_template_uploader_10000", + //UserBadge::TtsInferenceUser100 => "tts_inference_100", + //UserBadge::TtsInferenceUser500 => "tts_inference_500", + //UserBadge::TtsInferenceUser1000 => "tts_inference_1000", + //UserBadge::TtsInferenceUser10000 =>"tts_inference_10000", + //UserBadge::TtsInferenceUser100000 => "tts_inference_100000", + //UserBadge::W2lInferenceUser100 => "w2l_inference_100", + //UserBadge::W2lInferenceUser500 => "w2l_inference_500", + //UserBadge::W2lInferenceUser1000 => "w2l_inference_1000", + //UserBadge::W2lInferenceUser10000 => "w2l_inference_10000", + //UserBadge::W2lInferenceUser100000 => "w2l_inference_100000", + } + } +} + +#[derive(Clone)] +pub struct BadgeGranter { + pub mysql_pool: MySqlPool, + pub firehose_publisher: FirehosePublisher, // NB: Type is Copy/Clone safe due to internal Arc. +} + +struct ExistenceRecord { + pub does_exist: i64, +} + +struct CountRecord { + pub count: i64, +} + +impl BadgeGranter { + + pub async fn grant_early_user_badge(&self, user_token: &str) -> AnyhowResult<()> { + let _record_id = self.insert( + UserBadge::EarlyUser, + user_token, + ).await?; + Ok(()) + } + + /// This needs to be called *after* successful upload. + pub async fn maybe_grant_tts_model_uploads_badge(&self, user_token: &str) -> AnyhowResult<()> { + let count = self.count_tts_models_uploaded(user_token).await?; + + let mut maybe_badge = None; + + if count >= 250 { + maybe_badge = Some(UserBadge::TtsModelUploader250); + } else if count >= 200 { + maybe_badge = Some(UserBadge::TtsModelUploader200); + } else if count >= 150 { + maybe_badge = Some(UserBadge::TtsModelUploader150); + } else if count >= 100 { + maybe_badge = Some(UserBadge::TtsModelUploader100); + } else if count >= 50 { + maybe_badge = Some(UserBadge::TtsModelUploader50); + } else if count >= 20 { + maybe_badge = Some(UserBadge::TtsModelUploader20); + } else if count >= 10 { + maybe_badge = Some(UserBadge::TtsModelUploader10); + } else if count >= 5 { + maybe_badge = Some(UserBadge::TtsModelUploader5); + } else if count >= 1 { + maybe_badge = Some(UserBadge::TtsModelUploader1); + } + + let badge = match maybe_badge { + Some(badge) => badge, + None => return Ok(()), + }; + + if self.has_badge(user_token, badge).await? { + return Ok(()) + } + + let _record_id = self.insert( + badge, + user_token, + ).await?; + + self.firehose_publisher.publish_user_badge_granted(user_token, badge.to_db_value()) + .await?; + + Ok(()) + } + + + /// This needs to be called *after* successful upload. + pub async fn maybe_grant_w2l_template_uploads_badge(&self, user_token: &str) -> AnyhowResult<()> { + let count = self.count_w2l_templates_uploaded(user_token).await?; + + let mut maybe_badge = None; + + if count >= 10000 { + maybe_badge = Some(UserBadge::W2lTemplateUploader10000); + } else if count >= 5000 { + maybe_badge = Some(UserBadge::W2lTemplateUploader5000); + } else if count >= 2000 { + maybe_badge = Some(UserBadge::W2lTemplateUploader2000); + } else if count >= 1000 { + maybe_badge = Some(UserBadge::W2lTemplateUploader1000); + } else if count >= 500 { + maybe_badge = Some(UserBadge::W2lTemplateUploader500); + } else if count >= 200 { + maybe_badge = Some(UserBadge::W2lTemplateUploader200); + } else if count >= 100 { + maybe_badge = Some(UserBadge::W2lTemplateUploader100); + } else if count >= 50 { + maybe_badge = Some(UserBadge::W2lTemplateUploader50); + } else if count >= 10 { + maybe_badge = Some(UserBadge::W2lTemplateUploader10); + } else if count >= 1 { + maybe_badge = Some(UserBadge::W2lTemplateUploader1); + } + + let badge = match maybe_badge { + Some(badge) => badge, + None => return Ok(()), + }; + + if self.has_badge(user_token, badge).await? { + return Ok(()) + } + + let _record_id = self.insert( + badge, + user_token, + ).await?; + + self.firehose_publisher.publish_user_badge_granted(user_token, badge.to_db_value()) + .await?; + + Ok(()) + } + + + // ======================================================================= + + pub async fn has_badge(&self, user_token: &str, user_badge: UserBadge) -> AnyhowResult { + let maybe_result = sqlx::query_as!( + ExistenceRecord, + r#" +SELECT 1 as does_exist +FROM user_badges +WHERE + user_token = ? +AND + badge_slug = ? +LIMIT 1 + "#, + user_token, + user_badge.to_db_value() + ) + .fetch_one(&self.mysql_pool) + .await; + + let exists = match maybe_result { + Ok(_record) => true, + Err(err) => { + match err { + RowNotFound => false, + _ => { + warn!("query error: {:?}", err); + return Err(anyhow!("error querying: {:?}", err)); + } + } + } + }; + + Ok(exists) + } + + // ======================================================================= + + async fn count_tts_models_uploaded( + &self, + user_token: &str, + ) -> AnyhowResult { + // NB: This could get expensive! + let maybe_result = sqlx::query_as!( + CountRecord, + r#" +SELECT count(*) as count +FROM tts_models +WHERE + creator_user_token = ? +LIMIT 1 + "#, + user_token + ) + .fetch_one(&self.mysql_pool) + .await; + + self.handle_count_query(maybe_result) + } + + async fn count_w2l_templates_uploaded( + &self, + user_token: &str, + ) -> AnyhowResult { + // NB: This could get expensive! + // Especially at the scale we'll likely have W2L templates. + let maybe_result = sqlx::query_as!( + CountRecord, + r#" +SELECT count(*) as count +FROM w2l_templates +WHERE + creator_user_token = ? +LIMIT 1 + "#, + user_token + ) + .fetch_one(&self.mysql_pool) + .await; + + self.handle_count_query(maybe_result) + } + + async fn insert( + &self, + user_badge: UserBadge, + user_token: &str, + ) -> AnyhowResult { + let query_result = sqlx::query!( + r#" +INSERT INTO user_badges +SET + user_token = ?, + badge_slug = ? + "#, + user_token, + user_badge.to_db_value(), + ) + .execute(&self.mysql_pool) + .await; + + let record_id = Self::handle_results(query_result)?; + Ok(record_id) + } + + fn handle_results(query_result: Result) -> AnyhowResult { + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + warn!("Insert badge record DB error: {:?}", err); + + // NB: SQLSTATE[23000]: Integrity constraint violation + // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access) + match err { + Database(err) => { + let maybe_code = err.code().map(|c| c.into_owned()); + /*match maybe_code.as_deref() { + Some("23000") => { + if err.message().contains("username") { + return Err(UsernameTaken); + } else if err.message().contains("email_address") { + return Err(EmailTaken); + } + } + _ => {}, + }*/ + }, + _ => {}, + } + return Err(anyhow!("Error inserting record")); + } + }; + + Ok(record_id) + } + + fn handle_count_query(&self, query_result: Result) -> AnyhowResult { + let count = match query_result { + Ok(record) => record.count as u64, + Err(err) => { + match err { + RowNotFound => 0, + _ => { + warn!("query error: {:?}", err); + return Err(anyhow!("error querying: {:?}", err)); + } + } + } + }; + + Ok(count) + } +} diff --git a/src/common_queries/firehose_publisher.rs b/src/common_queries/firehose_publisher.rs index 4574d47c62..512e2d7d05 100644 --- a/src/common_queries/firehose_publisher.rs +++ b/src/common_queries/firehose_publisher.rs @@ -12,6 +12,9 @@ use std::sync::Arc; enum FirehoseEvent { UserSignUp, + // We don't publish for all badges (eg. early user signup) + UserBadgeGranted, + TtsModelUploadStarted, TtsModelUploadCompleted, TtsInferenceStarted, @@ -34,6 +37,7 @@ impl FirehoseEvent { pub fn to_db_value(&self) -> &'static str { match self { FirehoseEvent::UserSignUp => "user_sign_up", + FirehoseEvent::UserBadgeGranted=> "user_badge_granted", FirehoseEvent::TtsModelUploadStarted => "tts_model_upload_started", FirehoseEvent::TtsModelUploadCompleted => "tts_model_upload_completed", FirehoseEvent::TtsInferenceStarted => "tts_inference_started", @@ -69,6 +73,16 @@ impl FirehosePublisher { Ok(()) } + pub async fn publish_user_badge_granted(&self, user_token: &str, badge_slug: &str) -> AnyhowResult<()> { + let _record_id = self.insert( + FirehoseEvent::UserBadgeGranted, + Some(user_token), + Some(badge_slug), + None, + ).await?; + Ok(()) + } + pub async fn enqueue_tts_model_upload(&self, user_token: &str, job_token: &str) -> AnyhowResult<()> { let _record_id = self.insert( FirehoseEvent::TtsModelUploadStarted, diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index fcff225d6a..b35e0e5e98 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -1,3 +1,4 @@ +pub mod badge_granter; pub mod badges; pub mod firehose_publisher; pub mod list_ip_bans; diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index 812edc2cce..e077105def 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -190,6 +190,14 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) info!("new user id: {}", record_id); + // TODO: Remove me in the future. Too lazy to check the date, plus this will + // vary with adoption rate. + server_state.badge_granter.grant_early_user_badge(&user_token) + .await + .map_err(|e| { + warn!("error creating badge: {:?}", e); + CreateAccountError::ServerError + })?; let create_session_result = create_session_for_user(&user_token, &ip_address, &server_state.mysql_pool).await; diff --git a/src/main.rs b/src/main.rs index a41789fa43..349fa4605a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,6 +32,7 @@ use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; +use crate::common_queries::badge_granter::BadgeGranter; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::default_route_404::default_route_404; use crate::http_server::endpoints::events::list_events::list_events_handler; @@ -77,6 +78,7 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; +use crate::http_server::middleware::ip_filter_middleware::IpFilter; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::server_state::{ServerState, EnvConfig}; @@ -97,7 +99,6 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; -use crate::http_server::middleware::ip_filter_middleware::IpFilter; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -168,6 +169,11 @@ async fn main() -> AnyhowResult<()> { mysql_pool: pool.clone(), // NB: Pool is clone/sync/send-safe }; + let badge_granter = BadgeGranter { + mysql_pool: pool.clone(), // NB: Pool is clone/sync/send-safe + firehose_publisher: firehose_publisher.clone(), // NB: Also safe + }; + let redis_manager = RedisConnectionManager::new(redis_connection_string)?; let redis_pool = r2d2::Pool::builder() @@ -244,6 +250,7 @@ async fn main() -> AnyhowResult<()> { mysql_pool: pool, redis_pool, firehose_publisher, + badge_granter, cookie_manager, session_checker, private_bucket_client, diff --git a/src/server_state.rs b/src/server_state.rs index 8f80606658..c40e7bd2c9 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,11 +1,12 @@ +use crate::common_queries::badge_granter::BadgeGranter; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; +use crate::threads::ip_banlist_set::IpBanlistSet; use crate::util::buckets::bucket_client::BucketClient; use crate::util::encrypted_sort_id::SortKeyCrypto; use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; -use crate::threads::ip_banlist_set::IpBanlistSet; /// State that is injected into every endpoint. #[derive(Clone)] @@ -24,6 +25,7 @@ pub struct ServerState { pub session_checker: SessionChecker, pub firehose_publisher: FirehosePublisher, + pub badge_granter: BadgeGranter, pub private_bucket_client: BucketClient, pub public_bucket_client: BucketClient, diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 5013688834..e5bdf69be3 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -21,6 +21,7 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; use crate::common_env::CommonEnv; +use crate::common_queries::badge_granter::BadgeGranter; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; use crate::job_queries::tts_download_job_queries::grab_job_lock_and_mark_pending; @@ -69,6 +70,7 @@ struct Downloader { pub mysql_pool: MySqlPool, pub bucket_client: BucketClient, pub firehose_publisher: FirehosePublisher, + pub badge_granter: BadgeGranter, pub google_drive_downloader: GoogleDriveDownloadCommand, pub bucket_path_unifier: BucketPathUnifier, @@ -148,6 +150,11 @@ async fn main() -> AnyhowResult<()> { mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe }; + let badge_granter = BadgeGranter { + mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe + firehose_publisher: firehose_publisher.clone(), // NB: Also safe + }; + let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?; let py_script_name = easyenv::get_env_string_required(ENV_MODEL_CHECK_SCRIPT_NAME)?; @@ -167,6 +174,7 @@ async fn main() -> AnyhowResult<()> { bucket_path_unifier: BucketPathUnifier::default_paths(), bucket_root_tts_model_uploads: bucket_root.to_string(), firehose_publisher, + badge_granter, tts_check: tts_model_check_command, job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as i32, @@ -351,6 +359,13 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho anyhow!("error publishing event") })?; + downloader.badge_granter.maybe_grant_tts_model_uploads_badge(&job.creator_user_token) + .await + .map_err(|e| { + warn!("error maybe awarding badge: {:?}", e); + anyhow!("error maybe awarding badge") + })?; + info!("Job done: {}", job.id); Ok(()) diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 68af39d48b..e18dda4e37 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -21,6 +21,7 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; use crate::common_env::CommonEnv; +use crate::common_queries::badge_granter::BadgeGranter; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; use crate::job_queries::w2l_download_job_queries::grab_job_lock_and_mark_pending; @@ -83,6 +84,7 @@ struct Downloader { pub public_bucket_client: BucketClient, pub firehose_publisher: FirehosePublisher, + pub badge_granter: BadgeGranter, pub google_drive_downloader: GoogleDriveDownloadCommand, pub w2l_processor: Wav2LipPreprocessClient, @@ -195,6 +197,11 @@ async fn main() -> AnyhowResult<()> { mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe }; + let badge_granter = BadgeGranter { + mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe + firehose_publisher: firehose_publisher.clone(), // NB: Also safe + }; + let common_env = CommonEnv::read_from_env()?; let downloader = Downloader { @@ -205,6 +212,7 @@ async fn main() -> AnyhowResult<()> { download_script, google_drive_downloader, firehose_publisher, + badge_granter, ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, @@ -551,6 +559,13 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) anyhow!("error publishing event") })?; + downloader.badge_granter.maybe_grant_w2l_template_uploads_badge(&job.creator_user_token) + .await + .map_err(|e| { + warn!("error maybe awarding badge: {:?}", e); + anyhow!("error maybe awarding badge") + })?; + if downloader.debug_job_end_sleep_millis != 0 { warn!("Debug sleep after job end: {} ms", downloader.debug_job_end_sleep_millis); thread::sleep(Duration::from_millis(downloader.debug_job_end_sleep_millis)); From 679be230669665443821d4673a50013e0bcb5ad6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 27 Jun 2021 00:04:44 -0400 Subject: [PATCH 0439/1042] re-codegen db --- sqlx-data.json | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/sqlx-data.json b/sqlx-data.json index 07627b4bc9..14eed4e67b 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -242,6 +242,31 @@ ] } }, + "09e655ea4b3504cbb3bc18b92c6c25861c941ba073e3aead414e560b05bd7638": { + "query": "\nSELECT count(*) as count\nFROM tts_models\nWHERE\n creator_user_token = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + } + }, "0caed749627ff22ae1cea52621a375156259a731e8070459593082b80d463491": { "query": "\nSELECT\n id,\n status\nFROM tts_model_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", "describe": { @@ -1186,6 +1211,31 @@ "nullable": [] } }, + "314904656a333caa75a8644e73f4d338b7bd6f68c7ea9e37b5bc7f7b257bf34d": { + "query": "\nSELECT 1 as does_exist\nFROM user_badges\nWHERE\n user_token = ?\nAND\n badge_slug = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "does_exist", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 1 + } + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false + ] + } + }, "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", "describe": { @@ -5055,6 +5105,31 @@ "nullable": [] } }, + "c0f56e2656b566d3a8c869d39b0b56d1fbb75d7d9e817a24b10add44d348092d": { + "query": "\nSELECT count(*) as count\nFROM w2l_templates\nWHERE\n creator_user_token = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + } + }, "c63726afcf8d53a24597aad18bb44285eae8f5d056011811b066226f7db0198a": { "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -6570,6 +6645,16 @@ ] } }, + "da3197d279b94817c6c8a6277aff7a0cf44d66e6777f46680b1b07de48875dd5": { + "query": "\nINSERT INTO user_badges\nSET\n user_token = ?,\n badge_slug = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "dbce51eb1c80dded902c9509e50713483c7131f115e8ec10646e99e88110278b": { "query": "\nUPDATE tts_results\nSET\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { From ca5a9cf0ef66101140724b5ce53dca9da6d58c17 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 27 Jun 2021 00:23:03 -0400 Subject: [PATCH 0440/1042] add badges to profile endpoint --- src/common_queries/list_user_badges.rs | 76 +++++++++++++++++++ src/common_queries/mod.rs | 1 + .../endpoints/users/get_profile.rs | 58 ++++++++------ 3 files changed, 112 insertions(+), 23 deletions(-) create mode 100644 src/common_queries/list_user_badges.rs diff --git a/src/common_queries/list_user_badges.rs b/src/common_queries/list_user_badges.rs new file mode 100644 index 0000000000..b333920a64 --- /dev/null +++ b/src/common_queries/list_user_badges.rs @@ -0,0 +1,76 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::util::anyhow_result::AnyhowResult; +use log::{warn, info}; +use sqlx::MySqlPool; + +#[derive(Serialize)] +pub struct UserBadgeForList { + pub slug: String, + pub title: String, + pub description: String, + pub image_url: String, + pub granted_at: DateTime, +} + +pub struct RawDbUserBadgeForList { + slug: String, + title: String, + description: String, + image_url: String, + user_created_at : DateTime, +} + +pub async fn list_user_badges( + mysql_pool: &MySqlPool, + user_token: &str, +) -> AnyhowResult> { + info!("listing user badges"); + let maybe_user_badges = sqlx::query_as!( + RawDbUserBadgeForList, + r#" +SELECT + badges.slug, + badges.title, + badges.description, + badges.image_url, + user_badges.created_at as user_created_at + +FROM badges +JOIN user_badges +ON + badges.slug = user_badges.badge_slug +WHERE + user_badges.user_token = ? + "#, + user_token + ) + .fetch_all(mysql_pool) + .await; + + let user_badges : Vec = match maybe_user_badges { + Ok(badges) => badges, + Err(err) => { + warn!("Error: {:?}", err); + match err { + RowNotFound => Vec::new(), + _ => { + warn!("user badges query error: {:?}", err); + return Err(anyhow!("error querying user badges")); + } + } + } + }; + + Ok(user_badges.into_iter() + .map(|badge| { + UserBadgeForList { + slug: badge.slug.clone(), + title: badge.title.clone(), + description: badge.description.clone(), + image_url: badge.image_url.clone(), + granted_at: badge.user_created_at.clone(), + } + }) + .collect::>()) +} diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index b35e0e5e98..8be1dadc91 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -4,6 +4,7 @@ pub mod firehose_publisher; pub mod list_ip_bans; pub mod list_tts_inference_results; pub mod list_tts_models; +pub mod list_user_badges; pub mod list_user_roles; pub mod list_w2l_inference_results; pub mod list_w2l_templates; diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index bfd65a5506..318ee98f62 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -9,11 +9,14 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::common_queries::list_user_badges::UserBadgeForList; +use crate::common_queries::list_user_badges::list_user_badges; use crate::common_queries::sessions::create_session_for_user; use crate::database_helpers::boolean_converters::i8_to_bool; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -28,30 +31,9 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; // TODO: This is duplicated in query_user_profile -#[derive(Serialize)] -pub struct UserProfileRecord { - pub user_token: String, - pub username: String, - pub email_gravatar_hash: String, - pub display_name: String, - pub profile_markdown: String, - pub profile_rendered_html: String, - pub user_role_slug: String, - pub is_banned: i8, - pub disable_gravatar: i8, - pub discord_username: Option, - pub twitch_username: Option, - pub twitter_username: Option, - pub patreon_username: Option, - pub github_username: Option, - pub cashapp_username: Option, - pub website_url: Option, - pub created_at: DateTime, -} /// This changes the record: /// - changes banned to bool @@ -75,6 +57,7 @@ pub struct UserProfileRecordForResponse { pub github_username: Option, pub cashapp_username: Option, pub website_url: Option, + pub badges: Vec, pub created_at: DateTime, } @@ -84,6 +67,27 @@ pub struct ProfileSuccessResponse { pub user: Option, } +#[derive(Serialize)] +pub struct RawUserProfileRecord { + pub user_token: String, + pub username: String, + pub email_gravatar_hash: String, + pub display_name: String, + pub profile_markdown: String, + pub profile_rendered_html: String, + pub user_role_slug: String, + pub is_banned: i8, + pub disable_gravatar: i8, + pub discord_username: Option, + pub twitch_username: Option, + pub twitter_username: Option, + pub patreon_username: Option, + pub github_username: Option, + pub cashapp_username: Option, + pub website_url: Option, + pub created_at: DateTime, +} + #[derive(Debug, Display)] pub enum ProfileError { ServerError, @@ -122,7 +126,7 @@ pub async fn get_profile_handler( // NB: Lookup failure is Err(RowNotFound). // NB: Since this is publicly exposed, we don't query sensitive data. let maybe_profile_record = sqlx::query_as!( - UserProfileRecord, + RawUserProfileRecord, r#" SELECT users.token as user_token, @@ -153,7 +157,7 @@ WHERE .fetch_one(&server_state.mysql_pool) .await; // TODO: This will return error if it doesn't exist - let profile_record : UserProfileRecord = match maybe_profile_record { + let profile_record : RawUserProfileRecord = match maybe_profile_record { Ok(profile_record) => profile_record, Err(err) => { match err { @@ -169,6 +173,13 @@ WHERE } }; + let badges = list_user_badges(&server_state.mysql_pool, &profile_record.user_token) + .await + .unwrap_or_else(|err| { + warn!("Error querying badges: {:?}", err); + return Vec::new(); // NB: Fine if this fails. Not sure why it would. + }); + let profile_for_response = UserProfileRecordForResponse { user_token: profile_record.user_token.clone(), username: profile_record.username.clone(), @@ -187,6 +198,7 @@ WHERE cashapp_username: profile_record.cashapp_username.clone(), website_url: profile_record.website_url.clone(), created_at: profile_record.created_at.clone(), + badges, }; let response = ProfileSuccessResponse { From 9515a8c3cc12feec6c244aa49b2ffebca86e3b6b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 27 Jun 2021 00:23:39 -0400 Subject: [PATCH 0441/1042] re-codegen db for badges in profile --- sqlx-data.json | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/sqlx-data.json b/sqlx-data.json index 14eed4e67b..b85ca0649f 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -305,6 +305,83 @@ ] } }, + "10d93ef1350115742d8dddb951f98dfebabf7df3e44577b97b52343f0650eba1": { + "query": "\nSELECT\n badges.slug,\n badges.title,\n badges.description,\n badges.image_url,\n user_badges.created_at as user_created_at\n\nFROM badges\nJOIN user_badges\nON\n badges.slug = user_badges.badge_slug\nWHERE\n user_badges.user_token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 2, + "name": "description", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 3, + "name": "image_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 4, + "name": "user_created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false + ] + } + }, "1168f39a5ec892d06e941b0dba0e955a27ef976567eeafd6e550bc42ea4c4331": { "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", "describe": { From e302fe834c46477cf0dcfccc97d31f099f3eeb1d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 27 Jun 2021 18:07:53 -0400 Subject: [PATCH 0442/1042] make easier to reconfigure --- seed/bootstrap_inserts_roles_etc.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/seed/bootstrap_inserts_roles_etc.sh b/seed/bootstrap_inserts_roles_etc.sh index 0c30f7959b..fdf2fc31ed 100755 --- a/seed/bootstrap_inserts_roles_etc.sh +++ b/seed/bootstrap_inserts_roles_etc.sh @@ -1,19 +1,23 @@ #!/bin/bash # ONLY FOR LOCAL DEV +USERNAME='storyteller' +PASSWORD='password' +HOST='localhost' + # --default-character-set=UTF8mb4 \ echo 'Inserting System User Roles...' -mysql -u storyteller \ - -ppassword \ - -h localhost \ +mysql -u "${USERNAME}" \ + -p${PASSWORD} \ + -h $HOST \ -D storyteller \ -e "source ./seed/sql/system_roles.sql" echo 'Inserting Badges...' -mysql -u storyteller \ - -ppassword \ - -h localhost \ +mysql -u $USERNAME \ + -p${PASSWORD} \ + -h $HOST \ -D storyteller \ -e "source ./seed/sql/user_badges.sql" From 196efc52c15f57ec12673c657c1064618af43ee9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 30 Jun 2021 04:58:03 -0400 Subject: [PATCH 0443/1042] "no-op logger" to reduce log spam --- src/common_env.rs | 8 ++++++++ src/tts_inference_job.rs | 18 +++++++++++++++--- src/util/mod.rs | 1 + src/util/noop_logger.rs | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 src/util/noop_logger.rs diff --git a/src/common_env.rs b/src/common_env.rs index 36499cf53a..706d9fdfed 100644 --- a/src/common_env.rs +++ b/src/common_env.rs @@ -15,6 +15,12 @@ pub struct CommonEnv { /// The maximum number of retries a job will get. /// After a job exhausts these attempts, it will become "dead". pub job_max_attempts: u8, + + /// Number of job records to query at once. + pub job_batch_size: u32, + + /// Time to wait between no-op logger log lines. + pub no_op_logger_millis: u64, } impl CommonEnv { @@ -23,6 +29,8 @@ impl CommonEnv { Ok(Self { job_batch_wait_millis: easyenv::get_env_num("JOB_BATCH_WAIT_MILLIS", 100)?, job_max_attempts: easyenv::get_env_num("JOB_MAX_ATTEMPTS", 3)?, + job_batch_size: easyenv::get_env_num("JOB_BATCH_SIZE", 10)?, + no_op_logger_millis: easyenv::get_env_num("NO_OP_LOGGER_MILLIS", 15_000)?, }) } } \ No newline at end of file diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 00d5a9b8f5..630eef2373 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -19,7 +19,7 @@ pub mod shared_constants; pub mod util; use anyhow::anyhow; -use chrono::Utc; +use chrono::{Utc, DateTime, TimeZone}; use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_inference_job_queries::TtsInferenceJobRecord; @@ -40,6 +40,7 @@ use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::hashing::hash_string_sha2::hash_string_sha2; +use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; @@ -96,6 +97,12 @@ struct Inferencer { // Max job attempts before failure. // NB: This is an i32 so we don't need to convert to db column type. pub job_max_attempts: i32, + + // Number of jobs to dequeue at once. + pub job_batch_size: u32, + + // How long to wait between log lines + pub no_op_logger_millis: u64, } #[tokio::main] @@ -204,6 +211,8 @@ async fn main() -> AnyhowResult<()> { tts_vocoder_model_filename, job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as i32, + job_batch_size: common_env.job_batch_size, + no_op_logger_millis: common_env.no_op_logger_millis, }; main_loop(inferencer).await; @@ -217,8 +226,10 @@ const INCREASE_TIMEOUT_MILLIS : u64 = 1000; async fn main_loop(inferencer: Inferencer) { let mut error_timeout_millis = START_TIMEOUT_MILLIS; + let mut noop_logger = NoOpLogger::new(15_000); + loop { - let num_records = 1; + let num_records = inferencer.job_batch_size; let query_result = query_tts_inference_job_records( &inferencer.mysql_pool, @@ -236,7 +247,8 @@ async fn main_loop(inferencer: Inferencer) { }; if jobs.is_empty() { - info!("No jobs!"); + noop_logger.log_after_awhile(); + std::thread::sleep(Duration::from_millis(inferencer.job_batch_wait_millis)); continue; } diff --git a/src/util/mod.rs b/src/util/mod.rs index 0617f941a8..6e0c90bede 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -5,6 +5,7 @@ pub mod encrypted_sort_id; pub mod filesystem; pub mod hashing; pub mod markdown_to_html; +pub mod noop_logger; pub mod random_crockford_token; pub mod random_prefix_crockford_token; pub mod random_uuid; diff --git a/src/util/noop_logger.rs b/src/util/noop_logger.rs new file mode 100644 index 0000000000..17dcc1da93 --- /dev/null +++ b/src/util/noop_logger.rs @@ -0,0 +1,32 @@ +use chrono::{DateTime, Utc, TimeZone, Duration}; +use log::info; + +/// Log after periods of no work, but prevent spamming stdout. +pub struct NoOpLogger { + last_log_time: DateTime, + logless_duration: Duration, + op_count: u64, +} + +impl NoOpLogger { + pub fn new(log_every_millis: i64) -> Self { + Self { + last_log_time: Utc.timestamp(0, 0), + logless_duration: Duration::milliseconds(log_every_millis), + op_count: 0 + } + } + + pub fn log_after_awhile(&mut self) { + let now = Utc::now(); + let delta = now - self.last_log_time; + + if delta > self.logless_duration { + info!("No operations for {:?}. {} operations.", delta, self.op_count); + self.op_count = 0; + self.last_log_time = now; + } + + self.op_count += 1; + } +} \ No newline at end of file From 00d95f9e88acb0e753754cbbe9fc78c1ee2ccd05 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 30 Jun 2021 05:00:53 -0400 Subject: [PATCH 0444/1042] no-op logger --- src/tts_inference_job.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 630eef2373..3215d8c152 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -226,7 +226,7 @@ const INCREASE_TIMEOUT_MILLIS : u64 = 1000; async fn main_loop(inferencer: Inferencer) { let mut error_timeout_millis = START_TIMEOUT_MILLIS; - let mut noop_logger = NoOpLogger::new(15_000); + let mut noop_logger = NoOpLogger::new(inferencer.no_op_logger_millis as i64); loop { let num_records = inferencer.job_batch_size; From fada5e6b294bae3a74d21cdc4e07b9bc17aaf68a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 30 Jun 2021 17:09:18 -0400 Subject: [PATCH 0445/1042] some cold caching logic - skip worker once if cache is cold --- src/tts_inference_job.rs | 149 ++++++++++++++++++++++++++++++++------- 1 file changed, 122 insertions(+), 27 deletions(-) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 3215d8c152..e8da1aa631 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -22,7 +22,7 @@ use anyhow::anyhow; use chrono::{Utc, DateTime, TimeZone}; use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; -use crate::job_queries::tts_inference_job_queries::TtsInferenceJobRecord; +use crate::job_queries::tts_inference_job_queries::{TtsInferenceJobRecord, TtsModelRecord2}; use crate::job_queries::tts_inference_job_queries::get_tts_model_by_token; use crate::job_queries::tts_inference_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::tts_inference_job_queries::insert_tts_result; @@ -228,6 +228,8 @@ async fn main_loop(inferencer: Inferencer) { let mut noop_logger = NoOpLogger::new(inferencer.no_op_logger_millis as i64); + let mut cold_cache_okay = false; + loop { let num_records = inferencer.job_batch_size; @@ -253,27 +255,76 @@ async fn main_loop(inferencer: Inferencer) { continue; } - let result = process_jobs(&inferencer, jobs).await; + let batch_result = process_jobs( + &inferencer, + jobs, + cold_cache_okay).await; - match result { - Ok(_) => {}, + let cold_cache_skipped_count = match batch_result { + Ok(cold_cache_skipped_count) => cold_cache_skipped_count, Err(e) => { warn!("Error querying jobs: {:?}", e); std::thread::sleep(Duration::from_millis(error_timeout_millis)); error_timeout_millis += INCREASE_TIMEOUT_MILLIS; continue; } - } + }; error_timeout_millis = START_TIMEOUT_MILLIS; // reset + // We mostly want to ignore a cold cache unless other workers + // aren't picking up the slack. + if cold_cache_okay { + cold_cache_okay = false; + } else if cold_cache_skipped_count > 0 { + cold_cache_okay = true; + } + std::thread::sleep(Duration::from_millis(inferencer.job_batch_wait_millis)); } } -async fn process_jobs(inferencer: &Inferencer, jobs: Vec) -> AnyhowResult<()> { +/// Process a batch of jobs, returning the count of cold-cache skipped jobs. +async fn process_jobs( + inferencer: &Inferencer, + jobs: Vec, + cold_cache_okay: bool +) -> AnyhowResult { + let mut skipped_for_cold_cache = 0; + for job in jobs.into_iter() { - let result = process_job(inferencer, &job).await; + let model_state_result = ModelState::query_model_and_check_filesystem( + &job, + &inferencer.mysql_pool, + &inferencer.semi_persistent_cache).await; + + let model_state = match model_state_result { + Ok(model_state) => model_state, + Err(e) => { + warn!("Failure to check model state: {:?}", e); + + let failure_reason = ""; + let _r = mark_tts_inference_job_failure( + &inferencer.mysql_pool, + &job, + failure_reason, + inferencer.job_max_attempts + ).await; + continue; + } + }; + + if !model_state.is_downloaded_to_filesystem && !cold_cache_okay { + // We're going to skip this for now. (Maybe another worker has a warm cache and can continue.) + info!("Skipping TTS due to cold cache: {} ({})", + model_state.model_record.model_token, + model_state.model_record.title); + + skipped_for_cold_cache += 1; + continue; + } + + let result = process_job(inferencer, &job, &model_state.model_record).await; match result { Ok(_) => {}, Err(e) => { @@ -289,7 +340,7 @@ async fn process_jobs(inferencer: &Inferencer, jobs: Vec) } } - Ok(()) + Ok(skipped_for_cold_cache) } #[derive(Deserialize, Default)] @@ -306,7 +357,51 @@ fn read_metadata_file(filename: &PathBuf) -> AnyhowResult { Ok(serde_json::from_str(&buffer)?) } -async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> AnyhowResult<()> { +/// We check both of these in one go so that we can reuse the ModelRecord later +/// without another DB query. +struct ModelState { + pub model_record: TtsModelRecord2, + pub is_downloaded_to_filesystem: bool, +} + +impl ModelState { + /// Query the model details and see if the model file is on the filesystem in one go. + pub async fn query_model_and_check_filesystem( + job: &TtsInferenceJobRecord, + mysql_pool: &MySqlPool, + semi_persistent_cache: &SemiPersistentCacheDir, + ) -> AnyhowResult { + info!("Looking up TTS model by token: {}", &job.model_token); + + let query_result = get_tts_model_by_token( + &mysql_pool, + &job.model_token).await?; + + let model_record = match query_result { + Some(model) => model, + None => { + warn!("TTS model not found: {}", &job.model_token); + return Err(anyhow!("Model not found!")) + }, + }; + + let tts_synthesizer_fs_path = semi_persistent_cache.tts_synthesizer_model_path( + &model_record.model_token); + + let is_downloaded_to_filesystem = tts_synthesizer_fs_path.exists(); + + Ok(Self { + model_record, + is_downloaded_to_filesystem, + }) + } +} + +async fn process_job( + inferencer: &Inferencer, + job: &TtsInferenceJobRecord, + model_record: &TtsModelRecord2, +) -> AnyhowResult<()> { // TODO 1. Mark processing (DONE) @@ -352,21 +447,21 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An info!("Downloaded tts vocoder model from bucket!"); } - // ==================== LOOK UP TTS SYNTHESIZER RECORD (WHICH CONTAINS ITS BUCKET PATH) ==================== // - - info!("Looking up TTS model by token: {}", &job.model_token); - - let query_result = get_tts_model_by_token( - &inferencer.mysql_pool, - &job.model_token).await?; - - let tts_model = match query_result { - Some(model) => model, - None => { - warn!("TTS model not found: {}", &job.model_token); - return Err(anyhow!("Model not found!")) - }, - }; +// // ==================== LOOK UP TTS SYNTHESIZER RECORD (WHICH CONTAINS ITS BUCKET PATH) ==================== // +// +// info!("Looking up TTS model by token: {}", &job.model_token); +// +// let query_result = get_tts_model_by_token( +// &inferencer.mysql_pool, +// &job.model_token).await?; +// +// let tts_model = match query_result { +// Some(model) => model, +// None => { +// warn!("TTS model not found: {}", &job.model_token); +// return Err(anyhow!("Model not found!")) +// }, +// }; // ==================== CONFIRM OR DOWNLOAD TTS SYNTHESIZER MODEL ==================== // @@ -374,13 +469,13 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An // TODO: We'll probably need to LRU cache these. let tts_synthesizer_fs_path = inferencer.semi_persistent_cache.tts_synthesizer_model_path( - &tts_model.model_token); + &model_record.model_token); if !tts_synthesizer_fs_path.exists() { info!("TTS synthesizer model file does not exist: {:?}", &tts_synthesizer_fs_path); let tts_synthesizer_object_path = inferencer.bucket_path_unifier - .tts_synthesizer_path(&tts_model.private_bucket_hash); + .tts_synthesizer_path(&model_record.private_bucket_hash); info!("Download from template media path: {:?}", &tts_synthesizer_object_path); @@ -488,7 +583,7 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An inferencer.firehose_publisher.tts_inference_finished( job.maybe_creator_user_token.as_deref(), - &tts_model.model_token, + &model_record.model_token, &inference_result_token) .await .map_err(|e| { From 3bbab7232cfaa55fda8d2ab90e8198aeb78ed67d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Jul 2021 03:13:48 -0400 Subject: [PATCH 0446/1042] Working on porting HTTP server from old project https://github.com/ml-applications/voder/blob/7e33d9c7ca45bc4cd1d05011d5be4a4d035e9c82/cycle_gan/demo_server.py --- .../requirements-working-2021-05-31.txt | 7 + models/tacotron2/server.py | 251 ++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 models/tacotron2/server.py diff --git a/models/tacotron2/requirements-working-2021-05-31.txt b/models/tacotron2/requirements-working-2021-05-31.txt index 987c0f9960..bd2e6596f9 100644 --- a/models/tacotron2/requirements-working-2021-05-31.txt +++ b/models/tacotron2/requirements-working-2021-05-31.txt @@ -45,3 +45,10 @@ zipp==3.4.1 # For metadata python-magic==0.4.22 + +# For serving HTTP +falcon==3.0.1 +falcon-multipart==0.2.0 + +# For HTTP inference +SoundFile==0.10.3.post1 diff --git a/models/tacotron2/server.py b/models/tacotron2/server.py new file mode 100644 index 0000000000..2175f23413 --- /dev/null +++ b/models/tacotron2/server.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 + +""" +Starts a demo HTTP server to capture and transform audio +as a live demonstration of the trained model. +Brandon Thomas 2019-07-29 +""" + +import argparse +import falcon +import io +import librosa +import numpy as np +import os +import scipy +import soundfile +import tensorflow as tf +import subprocess +import tempfile + +from falcon_multipart.middleware import MultipartMiddleware +from model import CycleGAN +from preprocess import * +from wsgiref import simple_server + +print("TensorFlow version: {}".format(tf.version.VERSION)) + +INDEX_HTML = ''' + + + +

Voice Demo

+ + + +
+
+ +
+
+ + + +''' + + +class Converter(): + def __init__(self, model_dir, model_name): + self.num_features = 24 + self.sampling_rate = 16000 + self.frame_period = 5.0 + + self.model = CycleGAN(num_features = self.num_features, mode = 'test') + + self.model.load(filepath = os.path.join(model_dir, model_name)) + + """ + # NB: Save the graph + definition = self.model.sess.graph_def + directory = 'saved_model_2' + tf.train.write_graph(definition, directory, 'saved_model_2.pb', as_text=True) + # https://github.com/tensorflow/models/issues/3530#issuecomment-395968881 + output_dir = './saved_model/' + builder = tf.saved_model.builder.SavedModelBuilder(output_dir) + builder.add_meta_graph_and_variables( + self.model.sess, + [tf.saved_model.tag_constants.SERVING], + main_op=tf.tables_initializer(), + ) + builder.save() + """ + + """ + builder.add_meta_graph_and_variables( + self.model.sess, + [tf.saved_model.tag_constants.SERVING], + signature_def_map={ + 'predict_images': + prediction_signature, + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + classification_signature, + }, + main_op=tf.tables_initializer()) + """ + + self.mcep_normalization_params = np.load(os.path.join(model_dir, 'mcep_normalization.npz')) + self.mcep_mean_A = self.mcep_normalization_params['mean_A'] + self.mcep_std_A = self.mcep_normalization_params['std_A'] + self.mcep_mean_B = self.mcep_normalization_params['mean_B'] + self.mcep_std_B = self.mcep_normalization_params['std_B'] + + self.logf0s_normalization_params = np.load(os.path.join(model_dir, 'logf0s_normalization.npz')) + self.logf0s_mean_A = self.logf0s_normalization_params['mean_A'] + self.logf0s_std_A = self.logf0s_normalization_params['std_A'] + self.logf0s_mean_B = self.logf0s_normalization_params['mean_B'] + self.logf0s_std_B = self.logf0s_normalization_params['std_B'] + + def convert(self, wav, conversion_direction='A2B'): + wav = wav_padding(wav = wav, sr = self.sampling_rate, frame_period = self.frame_period, multiple = 4) + f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = self.sampling_rate, frame_period = self.frame_period) + coded_sp = world_encode_spectral_envelop(sp = sp, fs = self.sampling_rate, dim = self.num_features) + coded_sp_transposed = coded_sp.T + + if conversion_direction == 'A2B': + f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_A, std_log_src = self.logf0s_std_A, mean_log_target = self.logf0s_mean_B, std_log_target = self.logf0s_std_B) + coded_sp_norm = (coded_sp_transposed - self.mcep_mean_A) / self.mcep_std_A + coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0] + coded_sp_converted = coded_sp_converted_norm * self.mcep_std_B + self.mcep_mean_B + else: + f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_B, std_log_src = self.logf0s_std_B, mean_log_target = self.logf0s_mean_A, std_log_target = self.logf0s_std_A) + coded_sp_norm = (coded_sp_transposed - self.mcep_mean_B) / self.mcep_std_B + coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0] + coded_sp_converted = coded_sp_converted_norm * self.mcep_std_A + self.mcep_mean_A + + coded_sp_converted = coded_sp_converted.T + coded_sp_converted = np.ascontiguousarray(coded_sp_converted) + decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, fs = self.sampling_rate) + wav_transformed = world_speech_synthesis(f0 = f0_converted, decoded_sp = decoded_sp_converted, ap = ap, fs = self.sampling_rate, frame_period = self.frame_period) + + # For debugging model output, uncomment the following line: + # librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate) + + # TODO: Perhaps ditch this. It's probably unnecessary work. + upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000) + pcm_data = upsampled.astype(np.float64) + stereo_pcm_data = np.tile(pcm_data, (2,1)).T + + buf = io.BytesIO() + scipy.io.wavfile.write(buf, 48000, stereo_pcm_data.astype(np.float32)) + return buf + +# Set up model +# This should live long in memory, so we do it up front. +model_dir_default = './model/sf1_tm1' +model_name_default = 'sf1_tm1.ckpt' +converter = Converter(model_dir_default, model_name_default) + +class IndexHandler(): + def on_get(self, request, response): + response.content_type = 'text/html' + response.body = INDEX_HTML + +class ApiHandler(): + def on_post(self, request, response): + # NB: uses middleware to pull out data. + form_data = request.params['audio_data'].file + data, samplerate = soundfile.read(form_data) + + # For debugging browser input, uncomment the following line: + # scipy.io.wavfile.write('browser_input_audio.wav', samplerate, data) + + # NB: Convert the input stereo signal into mono. + # In the future the frontend should be responsible for sampling details. + mono = data[:, 0] + + # NB: We must downsample to the rate that the network is trained on. + downsampled = librosa.resample(mono, samplerate, 16000) + + # Evaluate the model + print(">>> Converting...") + results = converter.convert(downsampled, conversion_direction = 'A2B') + + temp_dir = tempfile.TemporaryDirectory(prefix='tmp_ml_audio') + temp_file = tempfile.NamedTemporaryFile(suffix='.wav') + + temp_file.write(results.read()) + + out_file = temp_dir.name + '/output.ogg' + + # NB: Browsers have a great deal of trouble decoding WAV files unless they are in the + # narrow slice of the WAV spec expected. None of the {librosa, scipy, soundfile} python + # tools do a good job of this, so here we shell out to ffmpeg and generate OGG. + # It's lazy and messy, but it works for now. + # See https://github.com/librosa/librosa/issues/361 for a survey of the library landscape + # See https://bugzilla.mozilla.org/show_bug.cgi?id=523837 for one of dozens of browser codec bugs + _stdout = subprocess.check_output(['ffmpeg', '-i', temp_file.name, '-acodec', 'libvorbis', out_file]) + + response.content_type = 'audio/ogg' + with open(out_file, mode='rb') as f: + response.data = f.read() + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--port', type=int, default=8000) + args = parser.parse_args() + + api = falcon.API(middleware=[MultipartMiddleware()]) + api.add_route('/', IndexHandler()) + api.add_route('/upload', ApiHandler()) + api.add_static_route('/script', os.path.abspath('./script')) + api.add_static_route('/sound', os.path.abspath('./sound')) + print('Serving on 0.0.0.0:%d' % args.port) + simple_server.make_server('0.0.0.0', args.port, api).serve_forever() + +if __name__ == '__main__': + main() From 21ac34b8a5880bfa2ac5030c1c0a5e7a6ad6c99c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Jul 2021 04:21:31 -0400 Subject: [PATCH 0447/1042] wip --- models/tacotron2/server.py | 251 -------------------------- models/tacotron2/vocodes_server.py | 272 +++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+), 251 deletions(-) delete mode 100644 models/tacotron2/server.py create mode 100644 models/tacotron2/vocodes_server.py diff --git a/models/tacotron2/server.py b/models/tacotron2/server.py deleted file mode 100644 index 2175f23413..0000000000 --- a/models/tacotron2/server.py +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env python3 - -""" -Starts a demo HTTP server to capture and transform audio -as a live demonstration of the trained model. -Brandon Thomas 2019-07-29 -""" - -import argparse -import falcon -import io -import librosa -import numpy as np -import os -import scipy -import soundfile -import tensorflow as tf -import subprocess -import tempfile - -from falcon_multipart.middleware import MultipartMiddleware -from model import CycleGAN -from preprocess import * -from wsgiref import simple_server - -print("TensorFlow version: {}".format(tf.version.VERSION)) - -INDEX_HTML = ''' - - - -

Voice Demo

- - - -
-
- -
-
- - - -''' - - -class Converter(): - def __init__(self, model_dir, model_name): - self.num_features = 24 - self.sampling_rate = 16000 - self.frame_period = 5.0 - - self.model = CycleGAN(num_features = self.num_features, mode = 'test') - - self.model.load(filepath = os.path.join(model_dir, model_name)) - - """ - # NB: Save the graph - definition = self.model.sess.graph_def - directory = 'saved_model_2' - tf.train.write_graph(definition, directory, 'saved_model_2.pb', as_text=True) - # https://github.com/tensorflow/models/issues/3530#issuecomment-395968881 - output_dir = './saved_model/' - builder = tf.saved_model.builder.SavedModelBuilder(output_dir) - builder.add_meta_graph_and_variables( - self.model.sess, - [tf.saved_model.tag_constants.SERVING], - main_op=tf.tables_initializer(), - ) - builder.save() - """ - - """ - builder.add_meta_graph_and_variables( - self.model.sess, - [tf.saved_model.tag_constants.SERVING], - signature_def_map={ - 'predict_images': - prediction_signature, - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: - classification_signature, - }, - main_op=tf.tables_initializer()) - """ - - self.mcep_normalization_params = np.load(os.path.join(model_dir, 'mcep_normalization.npz')) - self.mcep_mean_A = self.mcep_normalization_params['mean_A'] - self.mcep_std_A = self.mcep_normalization_params['std_A'] - self.mcep_mean_B = self.mcep_normalization_params['mean_B'] - self.mcep_std_B = self.mcep_normalization_params['std_B'] - - self.logf0s_normalization_params = np.load(os.path.join(model_dir, 'logf0s_normalization.npz')) - self.logf0s_mean_A = self.logf0s_normalization_params['mean_A'] - self.logf0s_std_A = self.logf0s_normalization_params['std_A'] - self.logf0s_mean_B = self.logf0s_normalization_params['mean_B'] - self.logf0s_std_B = self.logf0s_normalization_params['std_B'] - - def convert(self, wav, conversion_direction='A2B'): - wav = wav_padding(wav = wav, sr = self.sampling_rate, frame_period = self.frame_period, multiple = 4) - f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = self.sampling_rate, frame_period = self.frame_period) - coded_sp = world_encode_spectral_envelop(sp = sp, fs = self.sampling_rate, dim = self.num_features) - coded_sp_transposed = coded_sp.T - - if conversion_direction == 'A2B': - f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_A, std_log_src = self.logf0s_std_A, mean_log_target = self.logf0s_mean_B, std_log_target = self.logf0s_std_B) - coded_sp_norm = (coded_sp_transposed - self.mcep_mean_A) / self.mcep_std_A - coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0] - coded_sp_converted = coded_sp_converted_norm * self.mcep_std_B + self.mcep_mean_B - else: - f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_B, std_log_src = self.logf0s_std_B, mean_log_target = self.logf0s_mean_A, std_log_target = self.logf0s_std_A) - coded_sp_norm = (coded_sp_transposed - self.mcep_mean_B) / self.mcep_std_B - coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0] - coded_sp_converted = coded_sp_converted_norm * self.mcep_std_A + self.mcep_mean_A - - coded_sp_converted = coded_sp_converted.T - coded_sp_converted = np.ascontiguousarray(coded_sp_converted) - decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, fs = self.sampling_rate) - wav_transformed = world_speech_synthesis(f0 = f0_converted, decoded_sp = decoded_sp_converted, ap = ap, fs = self.sampling_rate, frame_period = self.frame_period) - - # For debugging model output, uncomment the following line: - # librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate) - - # TODO: Perhaps ditch this. It's probably unnecessary work. - upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000) - pcm_data = upsampled.astype(np.float64) - stereo_pcm_data = np.tile(pcm_data, (2,1)).T - - buf = io.BytesIO() - scipy.io.wavfile.write(buf, 48000, stereo_pcm_data.astype(np.float32)) - return buf - -# Set up model -# This should live long in memory, so we do it up front. -model_dir_default = './model/sf1_tm1' -model_name_default = 'sf1_tm1.ckpt' -converter = Converter(model_dir_default, model_name_default) - -class IndexHandler(): - def on_get(self, request, response): - response.content_type = 'text/html' - response.body = INDEX_HTML - -class ApiHandler(): - def on_post(self, request, response): - # NB: uses middleware to pull out data. - form_data = request.params['audio_data'].file - data, samplerate = soundfile.read(form_data) - - # For debugging browser input, uncomment the following line: - # scipy.io.wavfile.write('browser_input_audio.wav', samplerate, data) - - # NB: Convert the input stereo signal into mono. - # In the future the frontend should be responsible for sampling details. - mono = data[:, 0] - - # NB: We must downsample to the rate that the network is trained on. - downsampled = librosa.resample(mono, samplerate, 16000) - - # Evaluate the model - print(">>> Converting...") - results = converter.convert(downsampled, conversion_direction = 'A2B') - - temp_dir = tempfile.TemporaryDirectory(prefix='tmp_ml_audio') - temp_file = tempfile.NamedTemporaryFile(suffix='.wav') - - temp_file.write(results.read()) - - out_file = temp_dir.name + '/output.ogg' - - # NB: Browsers have a great deal of trouble decoding WAV files unless they are in the - # narrow slice of the WAV spec expected. None of the {librosa, scipy, soundfile} python - # tools do a good job of this, so here we shell out to ffmpeg and generate OGG. - # It's lazy and messy, but it works for now. - # See https://github.com/librosa/librosa/issues/361 for a survey of the library landscape - # See https://bugzilla.mozilla.org/show_bug.cgi?id=523837 for one of dozens of browser codec bugs - _stdout = subprocess.check_output(['ffmpeg', '-i', temp_file.name, '-acodec', 'libvorbis', out_file]) - - response.content_type = 'audio/ogg' - with open(out_file, mode='rb') as f: - response.data = f.read() - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--port', type=int, default=8000) - args = parser.parse_args() - - api = falcon.API(middleware=[MultipartMiddleware()]) - api.add_route('/', IndexHandler()) - api.add_route('/upload', ApiHandler()) - api.add_static_route('/script', os.path.abspath('./script')) - api.add_static_route('/sound', os.path.abspath('./sound')) - print('Serving on 0.0.0.0:%d' % args.port) - simple_server.make_server('0.0.0.0', args.port, api).serve_forever() - -if __name__ == '__main__': - main() diff --git a/models/tacotron2/vocodes_server.py b/models/tacotron2/vocodes_server.py new file mode 100644 index 0000000000..de5068866a --- /dev/null +++ b/models/tacotron2/vocodes_server.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 + +""" +Starts a demo HTTP server to capture and transform audio +as a live demonstration of the trained model. +Brandon Thomas 2019-07-29 +""" + +import argparse +import falcon +import io +import librosa +import numpy as np +import os +import scipy +import soundfile +import tensorflow as tf +import subprocess +import tempfile +import json + +from falcon_multipart.middleware import MultipartMiddleware +#from model import CycleGAN +#from preprocess import * +from wsgiref import simple_server + +print("TensorFlow version: {}".format(tf.version.VERSION)) + +INDEX_HTML = ''' + + + + +

TTS Inference

+ + + +
+
+ +
+
+ + +
+
+ + +
+ + +''' + + +#class Converter(): +# def __init__(self, model_dir, model_name): +# self.num_features = 24 +# self.sampling_rate = 16000 +# self.frame_period = 5.0 +# +# self.model = CycleGAN(num_features = self.num_features, mode = 'test') +# +# self.model.load(filepath = os.path.join(model_dir, model_name)) +# +# """ +# # NB: Save the graph +# definition = self.model.sess.graph_def +# directory = 'saved_model_2' +# tf.train.write_graph(definition, directory, 'saved_model_2.pb', as_text=True) +# # https://github.com/tensorflow/models/issues/3530#issuecomment-395968881 +# output_dir = './saved_model/' +# builder = tf.saved_model.builder.SavedModelBuilder(output_dir) +# builder.add_meta_graph_and_variables( +# self.model.sess, +# [tf.saved_model.tag_constants.SERVING], +# main_op=tf.tables_initializer(), +# ) +# builder.save() +# """ +# +# """ +# builder.add_meta_graph_and_variables( +# self.model.sess, +# [tf.saved_model.tag_constants.SERVING], +# signature_def_map={ +# 'predict_images': +# prediction_signature, +# signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: +# classification_signature, +# }, +# main_op=tf.tables_initializer()) +# """ +# +# self.mcep_normalization_params = np.load(os.path.join(model_dir, 'mcep_normalization.npz')) +# self.mcep_mean_A = self.mcep_normalization_params['mean_A'] +# self.mcep_std_A = self.mcep_normalization_params['std_A'] +# self.mcep_mean_B = self.mcep_normalization_params['mean_B'] +# self.mcep_std_B = self.mcep_normalization_params['std_B'] +# +# self.logf0s_normalization_params = np.load(os.path.join(model_dir, 'logf0s_normalization.npz')) +# self.logf0s_mean_A = self.logf0s_normalization_params['mean_A'] +# self.logf0s_std_A = self.logf0s_normalization_params['std_A'] +# self.logf0s_mean_B = self.logf0s_normalization_params['mean_B'] +# self.logf0s_std_B = self.logf0s_normalization_params['std_B'] +# +# def convert(self, wav, conversion_direction='A2B'): +# wav = wav_padding(wav = wav, sr = self.sampling_rate, frame_period = self.frame_period, multiple = 4) +# f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = self.sampling_rate, frame_period = self.frame_period) +# coded_sp = world_encode_spectral_envelop(sp = sp, fs = self.sampling_rate, dim = self.num_features) +# coded_sp_transposed = coded_sp.T +# +# if conversion_direction == 'A2B': +# f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_A, std_log_src = self.logf0s_std_A, mean_log_target = self.logf0s_mean_B, std_log_target = self.logf0s_std_B) +# coded_sp_norm = (coded_sp_transposed - self.mcep_mean_A) / self.mcep_std_A +# coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0] +# coded_sp_converted = coded_sp_converted_norm * self.mcep_std_B + self.mcep_mean_B +# else: +# f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_B, std_log_src = self.logf0s_std_B, mean_log_target = self.logf0s_mean_A, std_log_target = self.logf0s_std_A) +# coded_sp_norm = (coded_sp_transposed - self.mcep_mean_B) / self.mcep_std_B +# coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0] +# coded_sp_converted = coded_sp_converted_norm * self.mcep_std_A + self.mcep_mean_A +# +# coded_sp_converted = coded_sp_converted.T +# coded_sp_converted = np.ascontiguousarray(coded_sp_converted) +# decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, fs = self.sampling_rate) +# wav_transformed = world_speech_synthesis(f0 = f0_converted, decoded_sp = decoded_sp_converted, ap = ap, fs = self.sampling_rate, frame_period = self.frame_period) +# +# # For debugging model output, uncomment the following line: +# # librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate) +# +# # TODO: Perhaps ditch this. It's probably unnecessary work. +# upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000) +# pcm_data = upsampled.astype(np.float64) +# stereo_pcm_data = np.tile(pcm_data, (2,1)).T +# +# buf = io.BytesIO() +# scipy.io.wavfile.write(buf, 48000, stereo_pcm_data.astype(np.float32)) +# return buf + +## Set up model +## This should live long in memory, so we do it up front. +#model_dir_default = './model/sf1_tm1' +#model_name_default = 'sf1_tm1.ckpt' +#converter = Converter(model_dir_default, model_name_default) + +class IndexHandler(): + def on_get(self, request, response): + response.content_type = 'text/html' + response.body = INDEX_HTML + +class ApiHandler(): + def on_post(self, request, response): + raw_data = json.load(request.bounded_stream) + inference_text = raw_data.get('inference_text') + print(inference_text) + + ## NB: uses middleware to pull out data. + #form_data = request.params['audio_data'].file + #data, samplerate = soundfile.read(form_data) + + ## For debugging browser input, uncomment the following line: + ## scipy.io.wavfile.write('browser_input_audio.wav', samplerate, data) + + ## NB: Convert the input stereo signal into mono. + ## In the future the frontend should be responsible for sampling details. + #mono = data[:, 0] + + ## NB: We must downsample to the rate that the network is trained on. + #downsampled = librosa.resample(mono, samplerate, 16000) + + ## Evaluate the model + #print(">>> Converting...") + #results = converter.convert(downsampled, conversion_direction = 'A2B') + + #temp_dir = tempfile.TemporaryDirectory(prefix='tmp_ml_audio') + #temp_file = tempfile.NamedTemporaryFile(suffix='.wav') + + #temp_file.write(results.read()) + + #out_file = temp_dir.name + '/output.ogg' + + ## NB: Browsers have a great deal of trouble decoding WAV files unless they are in the + ## narrow slice of the WAV spec expected. None of the {librosa, scipy, soundfile} python + ## tools do a good job of this, so here we shell out to ffmpeg and generate OGG. + ## It's lazy and messy, but it works for now. + ## See https://github.com/librosa/librosa/issues/361 for a survey of the library landscape + ## See https://bugzilla.mozilla.org/show_bug.cgi?id=523837 for one of dozens of browser codec bugs + #_stdout = subprocess.check_output(['ffmpeg', '-i', temp_file.name, '-acodec', 'libvorbis', out_file]) + + #response.content_type = 'audio/ogg' + #with open(out_file, mode='rb') as f: + # response.data = f.read() + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--port', type=int, default=8000) + args = parser.parse_args() + + api = falcon.API(middleware=[MultipartMiddleware()]) + api.add_route('/', IndexHandler()) + api.add_route('/infer', ApiHandler()) + #api.add_static_route('/script', os.path.abspath('./script')) + #api.add_static_route('/sound', os.path.abspath('./sound')) + print('Serving on 0.0.0.0:%d' % args.port) + simple_server.make_server('0.0.0.0', args.port, api).serve_forever() + +if __name__ == '__main__': + main() From 321322c0a5619e777dd587b56bf0c43147b5732c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Jul 2021 16:39:27 -0400 Subject: [PATCH 0448/1042] server is working and fast, but need to wire it up for API --- models/tacotron2/vocodes_common.py | 96 ++++++++++++++++++++++++++++++ models/tacotron2/vocodes_server.py | 10 ++++ 2 files changed, 106 insertions(+) create mode 100644 models/tacotron2/vocodes_common.py diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py new file mode 100644 index 0000000000..b1a93a425b --- /dev/null +++ b/models/tacotron2/vocodes_common.py @@ -0,0 +1,96 @@ + +import sys +sys.path.append('waveglow/') +import shutil +import json + +import numpy as np +import torch + +from hparams import create_hparams +from model import Tacotron2 +from layers import TacotronSTFT +from audio_processing import griffin_lim +from text import text_to_sequence +from denoiser import Denoiser + +from scipy.io.wavfile import write as write_wav + +# For metadata +import subprocess +import magic +import os + +def print_info(): + print('========================================') + print('Python interpreter', sys.executable) + print('PyTorch version', torch.__version__) + print('CUDA Available?', torch.cuda.is_available()) + print('CUDA Device count', torch.cuda.device_count()) + print('========================================', flush=True) + +def load_tacotron_model(checkpoint_path): + hparams = create_hparams() + hparams.sampling_rate = 22050 # Don't change this + hparams.max_decoder_steps = 1000 # How long the audio will be before it cuts off (1000 is about 11 seconds) + hparams.gate_threshold = 0.1 # Model must be 90% sure the clip is over before ending generation (the higher this number is, the more likely that the AI will keep generating until it reaches the Max Decoder Steps) + model = Tacotron2(hparams) + print('Loading synthesizer at path: {}'.format(checkpoint_path)) + model.load_state_dict(torch.load(checkpoint_path)['state_dict']) + _ = model.cuda().eval().half() + # NB(bt, 2021-05-31): Trying to get everything on the same device + device = "cuda:0" + model = model.to(device) + return model + +def load_waveglow_model(checkpoint_path): + waveglow = torch.load(checkpoint_path)['model'] + waveglow.cuda().eval().half() + for k in waveglow.convinv: + k.float() + denoiser = Denoiser(waveglow) + return waveglow + +def preprocess_text(text, raw_input=True): + sequence = None + for i in text.split("\n"): + if len(i) < 1: + continue + print(i) + if raw_input: + if i[-1] != ";": + i = i+";" + else: + #i = ARPA(i) + pass + print(i) + with torch.no_grad(): + # save VRAM by not including gradients + sequence = np.array(text_to_sequence(i, ['english_cleaners']))[None, :] + sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long() + break # TODO: Handle multiple lines. + return sequence + +class TacotronWaveglowPipeline: + + def __init__(self, tacotron: Tacotron2, waveglow: Denoiser): + self.tacotron = tacotron + self.waveglow = waveglow + + def infer(self, raw_text): + sequence = preprocess_text(raw_text) + + with torch.no_grad(): + mel_outputs, mel_outputs_postnet, _, alignments = self.tacotron.inference(sequence) + + with torch.no_grad(): + sigma = 0.8 + audio = self.waveglow.infer(mel_outputs_postnet, sigma=sigma) + + + output_audio = audio[0].data.cpu().numpy().astype(np.float32) + + print(output_audio) + + #write_wav(args.output_audio_filename, hparams.sampling_rate, output_audio) + diff --git a/models/tacotron2/vocodes_server.py b/models/tacotron2/vocodes_server.py index de5068866a..b62afacb80 100644 --- a/models/tacotron2/vocodes_server.py +++ b/models/tacotron2/vocodes_server.py @@ -24,6 +24,10 @@ from falcon_multipart.middleware import MultipartMiddleware #from preprocess import * from wsgiref import simple_server +from vocodes_common import TacotronWaveglowPipeline +from vocodes_common import load_tacotron_model +from vocodes_common import load_waveglow_model + print("TensorFlow version: {}".format(tf.version.VERSION)) INDEX_HTML = ''' @@ -207,6 +211,10 @@ INDEX_HTML = ''' #model_name_default = 'sf1_tm1.ckpt' #converter = Converter(model_dir_default, model_name_default) +tacotron = load_tacotron_model('/home/bt/models/tacotron2/tacotron2_uberduck_Noire.pt') +waveglow = load_waveglow_model('/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt') +pipeline = TacotronWaveglowPipeline(tacotron, waveglow) + class IndexHandler(): def on_get(self, request, response): response.content_type = 'text/html' @@ -218,6 +226,8 @@ class ApiHandler(): inference_text = raw_data.get('inference_text') print(inference_text) + pipeline.infer(inference_text) + ## NB: uses middleware to pull out data. #form_data = request.params['audio_data'].file #data, samplerate = soundfile.read(form_data) From 560beb07739813893d1101ba20e6e836c75f3e94 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Jul 2021 17:27:02 -0400 Subject: [PATCH 0449/1042] progress on loading models from multiple args --- models/tacotron2/vocodes_common.py | 113 ++++++++++++++++++++++++++--- models/tacotron2/vocodes_server.py | 63 +++++++--------- 2 files changed, 131 insertions(+), 45 deletions(-) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index b1a93a425b..9f43b92753 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -1,4 +1,3 @@ - import sys sys.path.append('waveglow/') import shutil @@ -21,7 +20,7 @@ import subprocess import magic import os -def print_info(): +def print_gpu_info(): print('========================================') print('Python interpreter', sys.executable) print('PyTorch version', torch.__version__) @@ -52,6 +51,7 @@ def load_waveglow_model(checkpoint_path): return waveglow def preprocess_text(text, raw_input=True): + # TODO: Handle ARPAbet sequence = None for i in text.split("\n"): if len(i) < 1: @@ -71,14 +71,104 @@ def preprocess_text(text, raw_input=True): break # TODO: Handle multiple lines. return sequence +class NumpyEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, np.ndarray): + return obj.tolist() + return json.JSONEncoder.default(self, obj) + +def save_spectorgram_json_file(mel_outputs_postnet, output_spectrogram_filename): + # squeeze(0) -> 3D to 2D (by removing the singular 1-length "wrapper" dimension) + # transpose() -> originally 80x{N}, we turn to {N}x80 + mel_for_json = mel_outputs_postnet.cpu().numpy().squeeze(0).transpose() + mel_for_scaling = mel_for_json.copy() + + max_value = np.amax(mel_for_scaling) + min_value = np.amin(mel_for_scaling) + mel_range = max_value - min_value + + mel_for_scaling -= min_value + mel_for_scaling *= (255.0 / mel_range) + mel_for_scaling = mel_for_scaling.astype('int32') + + json_data = { + 'mel': mel_for_json, + 'mel_scaled': mel_for_scaling, + } + + with open(output_spectrogram_filename, 'w') as outfile: + json.dump(json_data, outfile, cls=NumpyEncoder) + +def save_wav_audio_file(audio, output_audio_filename, sampling_rate=22050): + output_audio = audio[0].data.cpu().numpy().astype(np.float32) + write_wav(output_audio_filename, sampling_rate, output_audio) + +def generate_metadata_file(output_audio_filename, output_metadata_filename): + # ==== METADATA (1) ==== + command = [ + "ffprobe", + "-loglevel", "quiet", + "-print_format", "json", + "-show_format", + "-show_streams", + output_audio_filename + ] + + pipe = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + out, err = pipe.communicate() + ffmpeg_metadata = json.loads(out) + + print(ffmpeg_metadata) + + # ==== METADATA (2) ==== + mime_type = magic.from_file(output_audio_filename, mime=True) + file_size_bytes = os.path.getsize(output_audio_filename) + + duration_millis = int(float(ffmpeg_metadata['format']['duration']) * 1000) + + metadata = { + 'duration_millis': duration_millis, + 'mimetype': mime_type, + 'file_size_bytes': file_size_bytes, + } + + with open(output_metadata_filename, 'w') as json_file: + json.dump(metadata, json_file) + + class TacotronWaveglowPipeline: - def __init__(self, tacotron: Tacotron2, waveglow: Denoiser): - self.tacotron = tacotron - self.waveglow = waveglow + def __init__(self): + self.tacotron = None + self.tacotron_filename = None + self.waveglow = None + self.waveglow_filename = None - def infer(self, raw_text): - sequence = preprocess_text(raw_text) + def maybe_load_waveglow_model(self, checkpoint_path): + if self.waveglow_filename == checkpoint_path: + print('Waveglow model already loaded into memory: {}'.format(checkpoint_path)) + return + print('Dumping previous waveglow model from memory: {}'.format(self.waveglow_filename)) + print('Loading waveglow model into memory: {}'.format(checkpoint_path), flush=True) + self.waveglow = load_waveglow_model(checkpoint_path) + self.waveglow_filename = checkpoint_path + + def maybe_load_tacotron_model(self, checkpoint_path): + if self.tacotron_filename == checkpoint_path: + print('Tacotron model already loaded into memory: {}'.format(checkpoint_path)) + return + print('Dumping previous tacotron model from memory: {}'.format(self.tacotron_filename)) + print('Loading tacotron model into memory: {}'.format(checkpoint_path), flush=True) + self.tacotron = load_tacotron_model(checkpoint_path) + self.tacotron_filename = checkpoint_path + + def infer(self, args): + assert('raw_text' in args) + assert('output_audio_filename' in args) + assert('output_spectrogram_filename' in args) + assert('output_metadata_filename' in args) + + sequence = preprocess_text(args['raw_text']) with torch.no_grad(): mel_outputs, mel_outputs_postnet, _, alignments = self.tacotron.inference(sequence) @@ -87,10 +177,13 @@ class TacotronWaveglowPipeline: sigma = 0.8 audio = self.waveglow.infer(mel_outputs_postnet, sigma=sigma) - output_audio = audio[0].data.cpu().numpy().astype(np.float32) - print(output_audio) + print('Saving spectrogram as JSON...') + save_spectorgram_json_file(mel_outputs_postnet, args['output_spectrogram_filename']) - #write_wav(args.output_audio_filename, hparams.sampling_rate, output_audio) + print('Encoding and saving audio...') + save_wav_audio_file(output_audio, args['output_audio_filename']) + print('Generating metadata file...') + generate_metadata_file(args['output_audio_filename'], args['output_metadata_filename']) diff --git a/models/tacotron2/vocodes_server.py b/models/tacotron2/vocodes_server.py index b62afacb80..dd6980a690 100644 --- a/models/tacotron2/vocodes_server.py +++ b/models/tacotron2/vocodes_server.py @@ -25,11 +25,14 @@ from falcon_multipart.middleware import MultipartMiddleware from wsgiref import simple_server from vocodes_common import TacotronWaveglowPipeline +from vocodes_common import print_gpu_info from vocodes_common import load_tacotron_model from vocodes_common import load_waveglow_model print("TensorFlow version: {}".format(tf.version.VERSION)) +print_gpu_info() + INDEX_HTML = ''' @@ -211,9 +214,9 @@ INDEX_HTML = ''' #model_name_default = 'sf1_tm1.ckpt' #converter = Converter(model_dir_default, model_name_default) -tacotron = load_tacotron_model('/home/bt/models/tacotron2/tacotron2_uberduck_Noire.pt') -waveglow = load_waveglow_model('/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt') -pipeline = TacotronWaveglowPipeline(tacotron, waveglow) +#tacotron = load_tacotron_model('/home/bt/models/tacotron2/tacotron2_uberduck_Noire.pt') +#waveglow = load_waveglow_model('/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt') +pipeline = TacotronWaveglowPipeline() class IndexHandler(): def on_get(self, request, response): @@ -223,43 +226,33 @@ class IndexHandler(): class ApiHandler(): def on_post(self, request, response): raw_data = json.load(request.bounded_stream) + + # Request parameters + vocoder_checkpoint_path = raw_data.get('vocoder_checkpoint_path') + synthesizer_checkpoint_path = raw_data.get('synthesizer_checkpoint_path') inference_text = raw_data.get('inference_text') - print(inference_text) + output_audio_filename = raw_data.get('output_audio_filename') + output_spectrogram_filename = raw_data.get('output_spectrogram_filename') + output_metadata_filename = raw_data.get('output_metadata_filename') - pipeline.infer(inference_text) + print('vocoder_checkpoint_path: {}'.format(vocoder_checkpoint_path)) + print('synthesizer_checkpoint_path: {}'.format(synthesizer_checkpoint_path)) + print('inference_text: {}'.format(inference_text)) + print('output_audio_filename: {}'.format(output_audio_filename)) + print('output_spectrogram_filename: {}'.format(output_spectrogram_filename)) + print('output_metadata_filename: {}'.format(output_metadata_filename)) - ## NB: uses middleware to pull out data. - #form_data = request.params['audio_data'].file - #data, samplerate = soundfile.read(form_data) + pipeline.maybe_load_waveglow_model(vocoder_checkpoint_path) + pipeline.maybe_load_tacotron_model(synthesizer_checkpoint_path) - ## For debugging browser input, uncomment the following line: - ## scipy.io.wavfile.write('browser_input_audio.wav', samplerate, data) + inference_args = { + 'raw_text': inference_text, + 'output_audio_filename': output_audio_filename, + 'output_spectrogram_filename': output_spectrogram_filename, + 'output_metadata_filename': output_metadata_filename, + } - ## NB: Convert the input stereo signal into mono. - ## In the future the frontend should be responsible for sampling details. - #mono = data[:, 0] - - ## NB: We must downsample to the rate that the network is trained on. - #downsampled = librosa.resample(mono, samplerate, 16000) - - ## Evaluate the model - #print(">>> Converting...") - #results = converter.convert(downsampled, conversion_direction = 'A2B') - - #temp_dir = tempfile.TemporaryDirectory(prefix='tmp_ml_audio') - #temp_file = tempfile.NamedTemporaryFile(suffix='.wav') - - #temp_file.write(results.read()) - - #out_file = temp_dir.name + '/output.ogg' - - ## NB: Browsers have a great deal of trouble decoding WAV files unless they are in the - ## narrow slice of the WAV spec expected. None of the {librosa, scipy, soundfile} python - ## tools do a good job of this, so here we shell out to ffmpeg and generate OGG. - ## It's lazy and messy, but it works for now. - ## See https://github.com/librosa/librosa/issues/361 for a survey of the library landscape - ## See https://bugzilla.mozilla.org/show_bug.cgi?id=523837 for one of dozens of browser codec bugs - #_stdout = subprocess.check_output(['ffmpeg', '-i', temp_file.name, '-acodec', 'libvorbis', out_file]) + pipeline.infer(inference_args) #response.content_type = 'audio/ogg' #with open(out_file, mode='rb') as f: From 5bc943a9f5268537e6186ad122e56488cb30dc6d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Jul 2021 17:38:58 -0400 Subject: [PATCH 0450/1042] server seems to work! --- models/tacotron2/vocodes_common.py | 4 +- models/tacotron2/vocodes_server.py | 94 ++++++++++++------------------ 2 files changed, 38 insertions(+), 60 deletions(-) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index 9f43b92753..b92e08817d 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -177,13 +177,11 @@ class TacotronWaveglowPipeline: sigma = 0.8 audio = self.waveglow.infer(mel_outputs_postnet, sigma=sigma) - output_audio = audio[0].data.cpu().numpy().astype(np.float32) - print('Saving spectrogram as JSON...') save_spectorgram_json_file(mel_outputs_postnet, args['output_spectrogram_filename']) print('Encoding and saving audio...') - save_wav_audio_file(output_audio, args['output_audio_filename']) + save_wav_audio_file(audio, args['output_audio_filename']) print('Generating metadata file...') generate_metadata_file(args['output_audio_filename'], args['output_metadata_filename']) diff --git a/models/tacotron2/vocodes_server.py b/models/tacotron2/vocodes_server.py index dd6980a690..3cd934f649 100644 --- a/models/tacotron2/vocodes_server.py +++ b/models/tacotron2/vocodes_server.py @@ -36,61 +36,41 @@ print_gpu_info() INDEX_HTML = ''' + + +

TTS Inference

- -
-
- -
-
- -
- + + +
+ + + +
+ + + +
+
From 13c4f22e4965101182d2730a6a9b0bab10370824 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Jul 2021 01:35:36 -0400 Subject: [PATCH 0451/1042] Perform TTS using a sidecar HTTP server --- .env | 2 + Cargo.lock | 1 + Cargo.toml | 4 + src/clients/mod.rs | 1 + src/clients/tts_inference_sidecar_client.rs | 111 ++++++++++++++++++++ src/main.rs | 1 + src/tts_inference_job.rs | 34 ++++-- 7 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 src/clients/mod.rs create mode 100644 src/clients/tts_inference_sidecar_client.rs diff --git a/.env b/.env index a8b1b87739..26d3ebce9c 100644 --- a/.env +++ b/.env @@ -42,6 +42,8 @@ TTS_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/tacotron2 TTS_INFERENCE_SCRIPT_NAME=vocodes_inference.py TTS_MODEL_CHECK_SCRIPT_NAME=vocodes_model_check.py +TTS_INFERENCE_SIDECAR_HOSTNAME=localhost:8000 + #TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt # THIS SOUNDS HORRID #TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_v4.pt diff --git a/Cargo.lock b/Cargo.lock index b396b54398..9084d78cc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3184,6 +3184,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", + "hyper", "infer", "jwt", "lazy_static", diff --git a/Cargo.toml b/Cargo.toml index 33dc410197..235d33f247 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,10 @@ actix-utils = "3.0.0" actix-web = "3.3.2" jwt = "0.13.0" +# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime +# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. +hyper = { version = "0.13.10" } + # Crypto bcrypt = "0.8" # password hashes hmac = "0.10.0" # cookies diff --git a/src/clients/mod.rs b/src/clients/mod.rs new file mode 100644 index 0000000000..8f7ae3bad7 --- /dev/null +++ b/src/clients/mod.rs @@ -0,0 +1 @@ +pub mod tts_inference_sidecar_client; \ No newline at end of file diff --git a/src/clients/tts_inference_sidecar_client.rs b/src/clients/tts_inference_sidecar_client.rs new file mode 100644 index 0000000000..3f5c8d0dc9 --- /dev/null +++ b/src/clients/tts_inference_sidecar_client.rs @@ -0,0 +1,111 @@ +//use actix_web::client::Client; +use actix_web::http::header; +use anyhow::anyhow; +use crate::util::anyhow_result::AnyhowResult; +use hyper::client::Client; +use log::info; +use std::path::Path; +use hyper::{Body, Request}; + +pub struct TtsInferenceSidecarClient { + hostname: String, + //client: Client, +} + +#[derive(Serialize)] +struct InferenceRequest { + pub vocoder_checkpoint_path : String, + pub synthesizer_checkpoint_path : String, + pub inference_text : String, + pub output_audio_filename : String, + pub output_spectrogram_filename : String, + pub output_metadata_filename : String, +} + +impl TtsInferenceSidecarClient { + pub fn new(hostname: &str) -> Self { + //let client = Client::builder() + // .header("User-Agent", "actix/tts_inference_job") + // .finish(); + Self { + hostname: hostname.to_string(), + // client, + } + } + + pub async fn request_inference>( + &self, + raw_text: &str, + synthesizer_checkpoint_path: P, + vocoder_checkpoint_path: P, + output_audio_filename: P, + output_spectrogram_filename: P, + output_metadata_filename: P, + ) -> AnyhowResult<()> { + + let vocoder_checkpoint_path = vocoder_checkpoint_path + .as_ref() + .to_str() + .map(|s| s.to_string()) + .ok_or(anyhow!("bad vocoder path"))?; + + let synthesizer_checkpoint_path = synthesizer_checkpoint_path + .as_ref() + .to_str() + .map(|s| s.to_string()) + .ok_or(anyhow!("bad synthesizer path"))?; + + let output_audio_filename = output_audio_filename + .as_ref() + .to_str() + .map(|s| s.to_string()) + .ok_or(anyhow!("bad output audio path"))?; + + let output_spectrogram_filename = output_spectrogram_filename + .as_ref() + .to_str() + .map(|s| s.to_string()) + .ok_or(anyhow!("bad output spectrogram path"))?; + + let output_metadata_filename = output_metadata_filename + .as_ref() + .to_str() + .map(|s| s.to_string()) + .ok_or(anyhow!("bad output metadata path"))?; + + let request = InferenceRequest { + inference_text: raw_text.to_string(), + vocoder_checkpoint_path, + synthesizer_checkpoint_path, + output_audio_filename, + output_spectrogram_filename, + output_metadata_filename, + }; + + let url = format!("http://{}/infer", self.hostname); + info!("Requesting {}", url); + + //let maybe_response = self.client.get(&url) + // .header(header::CONTENT_TYPE, "application/json") + // .send_json(&request) + // .await; + + let request = serde_json::to_string(&request)?; + + let req = Request::builder() + .method(hyper::Method::POST) + .uri(url) + .header("content-type", "application/json") + .body(Body::from(request))?; + + let client = Client::new(); + + let maybe_response = client.request(req).await?; + + //match maybe_response { + // Err(e) => Err(anyhow!("Error talking to sidecar: {:?}", e)), + // Ok(_) => Ok(()), + //} + Ok(()) + } +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 349fa4605a..91fb0d5f04 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,7 @@ pub const RESERVED_USERNAMES : &'static str = include_str!("../db/reserved_usern pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../db/reserved_usernames_including.txt"); pub const BANNED_SLURS : &'static str = include_str!("../db/banned_slurs.txt"); +pub mod clients; pub mod common_queries; pub mod database_helpers; pub mod http_server; diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index e8da1aa631..94ae4c4c3d 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -10,6 +10,7 @@ #[macro_use] extern crate serde_derive; +pub mod clients; pub mod common_env; pub mod common_queries; pub mod database_helpers; @@ -55,6 +56,7 @@ use std::process::Command; use std::thread; use std::time::Duration; use tempdir::TempDir; +use crate::clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -72,6 +74,9 @@ const ENV_SEMIPERSISTENT_CACHE_DIR : &'static str = "SEMIPERSISTENT_CACHE_DIR"; const ENV_CODE_DIRECTORY : &'static str = "TTS_CODE_DIRECTORY"; const ENV_INFERENCE_SCRIPT_NAME : &'static str = "TTS_INFERENCE_SCRIPT_NAME"; +// HTTP sidecar +const ENV_TTS_INFERENCE_SIDECAR_HOSTNAME: &'static str = "TTS_INFERENCE_SIDECAR_HOSTNAME"; + const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Inferencer { @@ -86,7 +91,8 @@ struct Inferencer { pub bucket_path_unifier: BucketPathUnifier, pub semi_persistent_cache: SemiPersistentCacheDir, - pub tts_inference: TacotronInferenceCommand, + pub tts_inference_command: TacotronInferenceCommand, + pub tts_inference_sidecar_client: TtsInferenceSidecarClient, // Command to run pub tts_vocoder_model_filename: String, @@ -155,6 +161,12 @@ async fn main() -> AnyhowResult<()> { &py_script_name, ); + let sidecar_hostname = + easyenv::get_env_string_required(ENV_TTS_INFERENCE_SIDECAR_HOSTNAME)?; + + let tts_inference_sidecar_client = + TtsInferenceSidecarClient::new(&sidecar_hostname); + let temp_directory = easyenv::get_env_string_or_default( "DOWNLOAD_TEMP_DIR", DEFAULT_TEMP_DIR); @@ -204,7 +216,8 @@ async fn main() -> AnyhowResult<()> { //ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, //ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, //imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, - tts_inference: tts_inference_command, + tts_inference_command, + tts_inference_sidecar_client, bucket_path_unifier: BucketPathUnifier::default_paths(), semi_persistent_cache, firehose_publisher, @@ -508,15 +521,24 @@ async fn process_job( info!("Expected output spectrogram filename: {:?}", &output_spectrogram_fs_path); info!("Expected output metadata filename: {:?}", &output_metadata_fs_path); - inferencer.tts_inference.execute( + //inferencer.tts_inference_command.execute( + // &tts_synthesizer_fs_path, + // &tts_vocoder_model_fs_path, + // &text_input_fs_path, + // &output_audio_fs_path, + // &output_spectrogram_fs_path, + // &output_metadata_fs_path, + // false, + //)?; + + inferencer.tts_inference_sidecar_client.request_inference( + &job.raw_inference_text, &tts_synthesizer_fs_path, &tts_vocoder_model_fs_path, - &text_input_fs_path, &output_audio_fs_path, &output_spectrogram_fs_path, &output_metadata_fs_path, - false, - )?; + ).await?; // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // From aced214213d314ef212808eacb53b1f22b4b8017 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 4 Jul 2021 23:35:29 -0400 Subject: [PATCH 0452/1042] lightweight LFU cache that returns the discarded key --- Cargo.lock | 19 ++++++ Cargo.toml | 1 + src/util/mod.rs | 3 +- src/util/virtual_lfu_cache.rs | 123 ++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 src/util/virtual_lfu_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 9084d78cc9..a93c002336 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1810,6 +1810,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "lfu" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "256290473df5a92218b86e619ab5fe8a831a167b70af720726b565c68b5360d3" +dependencies = [ + "linked_hash_set", +] + [[package]] name = "libc" version = "0.2.94" @@ -1828,6 +1837,15 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +[[package]] +name = "linked_hash_set" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47186c6da4d81ca383c7c47c1bfc80f4b95f4720514d860a5407aaf4233f9588" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "local-waker" version = "0.1.1" @@ -3188,6 +3206,7 @@ dependencies = [ "infer", "jwt", "lazy_static", + "lfu", "log", "magic-crypt", "md-5", diff --git a/Cargo.toml b/Cargo.toml index 235d33f247..617394b102 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ futures = { version = "0.3.15", features = ["thread-pool"] } # feature for Threa hex = "0.4.3" hostname = "0.3.1" lazy_static = "1.4.0" +lfu = "0.2.5" log = "0.4.14" rand = "0.8.3" regex = "1.5.4" diff --git a/src/util/mod.rs b/src/util/mod.rs index 6e0c90bede..c9c3a663ce 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -10,4 +10,5 @@ pub mod random_crockford_token; pub mod random_prefix_crockford_token; pub mod random_uuid; pub mod redis_keys; -pub mod semi_persistent_cache_dir; \ No newline at end of file +pub mod semi_persistent_cache_dir; +pub mod virtual_lfu_cache; \ No newline at end of file diff --git a/src/util/virtual_lfu_cache.rs b/src/util/virtual_lfu_cache.rs new file mode 100644 index 0000000000..9f48ff8dfa --- /dev/null +++ b/src/util/virtual_lfu_cache.rs @@ -0,0 +1,123 @@ +use anyhow::anyhow; +use crate::util::anyhow_result::AnyhowResult; +use lfu::LFUCache; +use std::sync::{Arc, RwLock}; +use std::collections::HashSet; + +/// This stands in front of the python HTTP sidecar and controls which +/// models get to remain in memory. The Python sidecar keeps multiple +/// models around, and Rust dictates which ones to keep and which to +/// purge. +pub struct VirtualLfuCache { + cache: LFUCache, +} + +impl VirtualLfuCache { + + pub fn new(capacity: usize) -> AnyhowResult { + let cache = LFUCache::with_capacity(capacity) + .map_err(|e| anyhow!("Error creating cache (probably capacity): {:?}", e))?; + Ok(Self { + cache, + }) + } + + pub fn in_cache>(&self, model: S) -> bool { + self.cache.contains(model.as_ref()) + } + + /// Returns the evicted entry. + pub fn insert(&mut self, path: &str) -> Option { + let initial_keys = self.get_keyset(); + + self.cache.set(path.to_string(), ()); + + let after_keys = self.get_keyset(); + + let difference = initial_keys.difference(&after_keys); + + difference.last() + .map(|item| item.to_string()) + } + + pub fn size(&self) -> usize { + self.cache.len() + } + + fn get_keyset(&self) -> HashSet { + // NB: This only makes sense for our use while we're using small caches. + let mut key_set = HashSet::with_capacity(self.size()); + for (k, _) in self.cache.iter() { + key_set.insert(k.to_string()); + } + key_set + } +} + +#[cfg(test)] +pub mod tests { + use crate::util::virtual_lfu_cache::VirtualLfuCache; + + #[test] + fn insert_beyond_capacity() { + let mut cache = VirtualLfuCache::new(3).unwrap(); + assert_eq!(cache.size(), 0); + cache.insert("foo"); + assert_eq!(cache.size(), 1); + cache.insert("bar"); + assert_eq!(cache.size(), 2); + cache.insert("baz"); + assert_eq!(cache.size(), 3); + cache.insert("bin"); + assert_eq!(cache.size(), 3); + cache.insert("bin"); + assert_eq!(cache.size(), 3); + cache.insert("111111111111111111"); + assert_eq!(cache.size(), 3); + } + + fn repeated_insert_single_key() { + let mut cache = VirtualLfuCache::new(3).unwrap(); + assert_eq!(cache.size(), 0); + for _ in 0..10 { + cache.insert("foo"); + } + assert_eq!(cache.size(), 1); + } + + #[test] + fn returns_first_for_eviction_when_all_used_once() { + let mut cache = VirtualLfuCache::new(3).unwrap(); + cache.insert("foo"); + cache.insert("bar"); + cache.insert("baz"); + let discarded = cache.insert("bin"); + assert_eq!(discarded, Some("foo".to_string())); + } + + #[test] + fn retains_frequent_value() { + let mut cache = VirtualLfuCache::new(3).unwrap(); + for _ in 0..10 { + cache.insert("foo"); + } + cache.insert("bar"); + cache.insert("baz"); + let discarded = cache.insert("bin"); + assert_eq!(discarded, Some("bar".to_string())); + } + + #[test] + fn retains_second_frequent_value() { + let mut cache = VirtualLfuCache::new(3).unwrap(); + for _ in 0..10 { + cache.insert("foo"); + } + for _ in 0..2 { + cache.insert("bar"); + } + cache.insert("baz"); + let discarded = cache.insert("bin"); + assert_eq!(discarded, Some("baz".to_string())); + } +} From 304312ce6f59df3f0bf70a7a6f04285b59d67797 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Jul 2021 01:15:16 -0400 Subject: [PATCH 0453/1042] stupid string reference. Ugh. --- src/util/virtual_lfu_cache.rs | 60 ++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/src/util/virtual_lfu_cache.rs b/src/util/virtual_lfu_cache.rs index 9f48ff8dfa..2b57e19369 100644 --- a/src/util/virtual_lfu_cache.rs +++ b/src/util/virtual_lfu_cache.rs @@ -3,6 +3,7 @@ use crate::util::anyhow_result::AnyhowResult; use lfu::LFUCache; use std::sync::{Arc, RwLock}; use std::collections::HashSet; +use std::borrow::Borrow; /// This stands in front of the python HTTP sidecar and controls which /// models get to remain in memory. The Python sidecar keeps multiple @@ -22,12 +23,12 @@ impl VirtualLfuCache { }) } - pub fn in_cache>(&self, model: S) -> bool { - self.cache.contains(model.as_ref()) + pub fn in_cache(&self, model: &String) -> bool { + self.cache.contains(model) } /// Returns the evicted entry. - pub fn insert(&mut self, path: &str) -> Option { + pub fn insert_returning_replaced(&mut self, path: &str) -> Option { let initial_keys = self.get_keyset(); self.cache.set(path.to_string(), ()); @@ -62,17 +63,17 @@ pub mod tests { fn insert_beyond_capacity() { let mut cache = VirtualLfuCache::new(3).unwrap(); assert_eq!(cache.size(), 0); - cache.insert("foo"); + cache.insert_returning_replaced("foo"); assert_eq!(cache.size(), 1); - cache.insert("bar"); + cache.insert_returning_replaced("bar"); assert_eq!(cache.size(), 2); - cache.insert("baz"); + cache.insert_returning_replaced("baz"); assert_eq!(cache.size(), 3); - cache.insert("bin"); + cache.insert_returning_replaced("bin"); assert_eq!(cache.size(), 3); - cache.insert("bin"); + cache.insert_returning_replaced("bin"); assert_eq!(cache.size(), 3); - cache.insert("111111111111111111"); + cache.insert_returning_replaced("111111111111111111"); assert_eq!(cache.size(), 3); } @@ -80,7 +81,7 @@ pub mod tests { let mut cache = VirtualLfuCache::new(3).unwrap(); assert_eq!(cache.size(), 0); for _ in 0..10 { - cache.insert("foo"); + cache.insert_returning_replaced("foo"); } assert_eq!(cache.size(), 1); } @@ -88,36 +89,51 @@ pub mod tests { #[test] fn returns_first_for_eviction_when_all_used_once() { let mut cache = VirtualLfuCache::new(3).unwrap(); - cache.insert("foo"); - cache.insert("bar"); - cache.insert("baz"); - let discarded = cache.insert("bin"); + cache.insert_returning_replaced("foo"); + cache.insert_returning_replaced("bar"); + cache.insert_returning_replaced("baz"); + let discarded = cache.insert_returning_replaced("bin"); assert_eq!(discarded, Some("foo".to_string())); + + assert_eq!(false, cache.in_cache(&"foo".to_string())); + assert_eq!(true, cache.in_cache(&"bar".to_string())); + assert_eq!(true, cache.in_cache(&"baz".to_string())); + assert_eq!(true, cache.in_cache(&"bin".to_string())); } #[test] fn retains_frequent_value() { let mut cache = VirtualLfuCache::new(3).unwrap(); for _ in 0..10 { - cache.insert("foo"); + cache.insert_returning_replaced("foo"); } - cache.insert("bar"); - cache.insert("baz"); - let discarded = cache.insert("bin"); + cache.insert_returning_replaced("bar"); + cache.insert_returning_replaced("baz"); + let discarded = cache.insert_returning_replaced("bin"); assert_eq!(discarded, Some("bar".to_string())); + + assert_eq!(true, cache.in_cache(&"foo".to_string())); + assert_eq!(false, cache.in_cache(&"bar".to_string())); + assert_eq!(true, cache.in_cache(&"baz".to_string())); + assert_eq!(true, cache.in_cache(&"bin".to_string())); } #[test] fn retains_second_frequent_value() { let mut cache = VirtualLfuCache::new(3).unwrap(); for _ in 0..10 { - cache.insert("foo"); + cache.insert_returning_replaced("foo"); } for _ in 0..2 { - cache.insert("bar"); + cache.insert_returning_replaced("bar"); } - cache.insert("baz"); - let discarded = cache.insert("bin"); + cache.insert_returning_replaced("baz"); + let discarded = cache.insert_returning_replaced("bin"); assert_eq!(discarded, Some("baz".to_string())); + + assert_eq!(true, cache.in_cache(&"foo".to_string())); + assert_eq!(true, cache.in_cache(&"bar".to_string())); + assert_eq!(false, cache.in_cache(&"baz".to_string())); + assert_eq!(true, cache.in_cache(&"bin".to_string())); } } From 4da28003f21f573804b5af303a2143d665a1691d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Jul 2021 02:18:20 -0400 Subject: [PATCH 0454/1042] attempt at running LRU for the sidecar --- models/tacotron2/vocodes_common.py | 21 +++++---- models/tacotron2/vocodes_server.py | 17 +++++-- src/clients/tts_inference_sidecar_client.rs | 3 ++ src/tts_inference_job.rs | 14 ++++++ src/util/virtual_lfu_cache.rs | 49 +++++++++++++++++++++ 5 files changed, 93 insertions(+), 11 deletions(-) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index b92e08817d..be3ca7c16a 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -139,8 +139,8 @@ def generate_metadata_file(output_audio_filename, output_metadata_filename): class TacotronWaveglowPipeline: def __init__(self): - self.tacotron = None - self.tacotron_filename = None + # Cache of "path" => loaded model + self.tacotron_model_cache = {} self.waveglow = None self.waveglow_filename = None @@ -153,16 +153,19 @@ class TacotronWaveglowPipeline: self.waveglow = load_waveglow_model(checkpoint_path) self.waveglow_filename = checkpoint_path - def maybe_load_tacotron_model(self, checkpoint_path): - if self.tacotron_filename == checkpoint_path: + def maybe_load_tacotron_model_to_cache(self, checkpoint_path): + if checkpoint_path in self.tacotron_model_cache: print('Tacotron model already loaded into memory: {}'.format(checkpoint_path)) return - print('Dumping previous tacotron model from memory: {}'.format(self.tacotron_filename)) print('Loading tacotron model into memory: {}'.format(checkpoint_path), flush=True) - self.tacotron = load_tacotron_model(checkpoint_path) - self.tacotron_filename = checkpoint_path + model = load_tacotron_model(checkpoint_path) + self.tacotron_model_cache[checkpoint_path] = model + + def uncache_tacotron_model(self, checkpoint_path): + self.tacotron_model_cache.pop(checkpoint_path, None) def infer(self, args): + assert('synthesizer_checkpoint_path' in args) assert('raw_text' in args) assert('output_audio_filename' in args) assert('output_spectrogram_filename' in args) @@ -170,8 +173,10 @@ class TacotronWaveglowPipeline: sequence = preprocess_text(args['raw_text']) + tacotron = self.tacotron_model_cache[args['synthesizer_checkpoint_path']] + with torch.no_grad(): - mel_outputs, mel_outputs_postnet, _, alignments = self.tacotron.inference(sequence) + mel_outputs, mel_outputs_postnet, _, alignments = tacotron.inference(sequence) with torch.no_grad(): sigma = 0.8 diff --git a/models/tacotron2/vocodes_server.py b/models/tacotron2/vocodes_server.py index 3cd934f649..bd7ceec03b 100644 --- a/models/tacotron2/vocodes_server.py +++ b/models/tacotron2/vocodes_server.py @@ -208,25 +208,36 @@ class ApiHandler(): raw_data = json.load(request.bounded_stream) # Request parameters - vocoder_checkpoint_path = raw_data.get('vocoder_checkpoint_path') synthesizer_checkpoint_path = raw_data.get('synthesizer_checkpoint_path') + vocoder_checkpoint_path = raw_data.get('vocoder_checkpoint_path') inference_text = raw_data.get('inference_text') output_audio_filename = raw_data.get('output_audio_filename') output_spectrogram_filename = raw_data.get('output_spectrogram_filename') output_metadata_filename = raw_data.get('output_metadata_filename') - print('vocoder_checkpoint_path: {}'.format(vocoder_checkpoint_path)) + # Optional request parameters + maybe_clear_synthesizer_checkpoint_path = \ + raw_data.get('maybe_clear_synthesizer_checkpoint_path') + print('synthesizer_checkpoint_path: {}'.format(synthesizer_checkpoint_path)) + print('vocoder_checkpoint_path: {}'.format(vocoder_checkpoint_path)) print('inference_text: {}'.format(inference_text)) print('output_audio_filename: {}'.format(output_audio_filename)) print('output_spectrogram_filename: {}'.format(output_spectrogram_filename)) print('output_metadata_filename: {}'.format(output_metadata_filename)) + print('maybe_clear_synthesizer_checkpoint_path: {}'.format( + maybe_clear_synthesizer_checkpoint_path)) + + if maybe_clear_synthesizer_checkpoint_path: + print('Clearing synthesizer: {}'.format(maybe_clear_synthesizer_checkpoint_path)) + pipeline.uncache_tacotron_model(maybe_clear_synthesizer_checkpoint_path) pipeline.maybe_load_waveglow_model(vocoder_checkpoint_path) - pipeline.maybe_load_tacotron_model(synthesizer_checkpoint_path) + pipeline.maybe_load_tacotron_model_to_cache(synthesizer_checkpoint_path) inference_args = { 'raw_text': inference_text, + 'synthesizer_checkpoint_path': synthesizer_checkpoint_path, 'output_audio_filename': output_audio_filename, 'output_spectrogram_filename': output_spectrogram_filename, 'output_metadata_filename': output_metadata_filename, diff --git a/src/clients/tts_inference_sidecar_client.rs b/src/clients/tts_inference_sidecar_client.rs index 3f5c8d0dc9..7f33574a92 100644 --- a/src/clients/tts_inference_sidecar_client.rs +++ b/src/clients/tts_inference_sidecar_client.rs @@ -20,6 +20,7 @@ struct InferenceRequest { pub output_audio_filename : String, pub output_spectrogram_filename : String, pub output_metadata_filename : String, + pub maybe_unload_model_path: Option, } impl TtsInferenceSidecarClient { @@ -41,6 +42,7 @@ impl TtsInferenceSidecarClient { output_audio_filename: P, output_spectrogram_filename: P, output_metadata_filename: P, + maybe_unload_model_path: Option, ) -> AnyhowResult<()> { let vocoder_checkpoint_path = vocoder_checkpoint_path @@ -80,6 +82,7 @@ impl TtsInferenceSidecarClient { output_audio_filename, output_spectrogram_filename, output_metadata_filename, + maybe_unload_model_path, }; let url = format!("http://{}/infer", self.hostname); diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 94ae4c4c3d..10b0d71901 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -44,6 +44,7 @@ use crate::util::hashing::hash_string_sha2::hash_string_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; +use crate::util::virtual_lfu_cache::SyncVirtualLfuCache; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; @@ -94,6 +95,9 @@ struct Inferencer { pub tts_inference_command: TacotronInferenceCommand, pub tts_inference_sidecar_client: TtsInferenceSidecarClient, + // Keep tabs of which models to hold in the sidecar memory with this virtual LRU cache + pub virtual_model_lfu: SyncVirtualLfuCache, + // Command to run pub tts_vocoder_model_filename: String, @@ -208,6 +212,8 @@ async fn main() -> AnyhowResult<()> { let common_env = CommonEnv::read_from_env()?; + let virtual_lfu_cache = SyncVirtualLfuCache::new(2)?; + let inferencer = Inferencer { download_temp_directory: temp_directory, mysql_pool, @@ -218,6 +224,7 @@ async fn main() -> AnyhowResult<()> { //imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, tts_inference_command, tts_inference_sidecar_client, + virtual_model_lfu: virtual_lfu_cache, bucket_path_unifier: BucketPathUnifier::default_paths(), semi_persistent_cache, firehose_publisher, @@ -511,6 +518,11 @@ async fn process_job( // ==================== RUN INFERENCE ==================== // + // TODO: Fix this. + let maybe_unload_model_path = inferencer + .virtual_model_lfu + .insert_returning_replaced(tts_synthesizer_fs_path.to_str().unwrap_or(""))?; + let output_audio_fs_path = temp_dir.path().join("output.wav"); let output_metadata_fs_path = temp_dir.path().join("metadata.json"); let output_spectrogram_fs_path = temp_dir.path().join("spectrogram.json"); @@ -520,6 +532,7 @@ async fn process_job( info!("Expected output audio filename: {:?}", &output_audio_fs_path); info!("Expected output spectrogram filename: {:?}", &output_spectrogram_fs_path); info!("Expected output metadata filename: {:?}", &output_metadata_fs_path); + info!("Maybe unload model: {:?}", &maybe_unload_model_path); //inferencer.tts_inference_command.execute( // &tts_synthesizer_fs_path, @@ -538,6 +551,7 @@ async fn process_job( &output_audio_fs_path, &output_spectrogram_fs_path, &output_metadata_fs_path, + maybe_unload_model_path, ).await?; // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // diff --git a/src/util/virtual_lfu_cache.rs b/src/util/virtual_lfu_cache.rs index 2b57e19369..8584ffbeb5 100644 --- a/src/util/virtual_lfu_cache.rs +++ b/src/util/virtual_lfu_cache.rs @@ -13,6 +13,12 @@ pub struct VirtualLfuCache { cache: LFUCache, } +/// And the Sync/Send threadsafe + interior mutability version. +#[derive(Clone)] +pub struct SyncVirtualLfuCache { + cache: Arc::> +} + impl VirtualLfuCache { pub fn new(capacity: usize) -> AnyhowResult { @@ -55,6 +61,49 @@ impl VirtualLfuCache { } } +impl SyncVirtualLfuCache { + + pub fn new(capacity: usize) -> AnyhowResult { + let cache = VirtualLfuCache::new(capacity)?; + Ok(Self { + cache: Arc::new(RwLock::new(cache)), + }) + } + + pub fn in_cache(&self, model: &String) -> AnyhowResult { + let exists = self.cache + .read() + .map(|cache| cache.in_cache(model)) + .map_err(|err| anyhow!("mutex error: {:?}", err))?; + Ok(exists) + } + + /// Returns the evicted entry. + pub fn insert_returning_replaced(&self, path: &str) -> AnyhowResult> { + let maybe_replaced = self.cache + .write() + .map(|mut cache| cache.insert_returning_replaced(path)) + .map_err(|err| anyhow!("mutex error: {:?}", err))?; + Ok(maybe_replaced) + } + + pub fn size(&self) -> AnyhowResult { + let size = self.cache + .read() + .map(|cache| cache.size()) + .map_err(|err| anyhow!("mutex error: {:?}", err))?; + Ok(size) + } + + fn get_keyset(&self) -> AnyhowResult> { + let key_set = self.cache + .read() + .map(|cache| cache.get_keyset()) + .map_err(|err| anyhow!("mutex error: {:?}", err))?; + Ok(key_set) + } +} + #[cfg(test)] pub mod tests { use crate::util::virtual_lfu_cache::VirtualLfuCache; From b476485cd72de32c252d7851b76702ff3d0f199a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Jul 2021 10:33:48 -0400 Subject: [PATCH 0455/1042] model loading/unloading works --- models/tacotron2/vocodes_common.py | 6 ++++++ models/tacotron2/vocodes_server.py | 1 - src/clients/tts_inference_sidecar_client.rs | 6 ++++-- src/tts_inference_job.rs | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index be3ca7c16a..13960873e0 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -160,9 +160,15 @@ class TacotronWaveglowPipeline: print('Loading tacotron model into memory: {}'.format(checkpoint_path), flush=True) model = load_tacotron_model(checkpoint_path) self.tacotron_model_cache[checkpoint_path] = model + print('Now {} synthesizer models in memory'.format(len(self.tacotron_model_cache))) def uncache_tacotron_model(self, checkpoint_path): + print('Currently {} synthesizer models in memory'.format( + len(self.tacotron_model_cache))) + print('Clearing synthesizer from memory: {}'.format(checkpoint_path)) self.tacotron_model_cache.pop(checkpoint_path, None) + print('Now reduced to {} synthesizer models in memory'.format( + len(self.tacotron_model_cache))) def infer(self, args): assert('synthesizer_checkpoint_path' in args) diff --git a/models/tacotron2/vocodes_server.py b/models/tacotron2/vocodes_server.py index bd7ceec03b..ce5c39cb30 100644 --- a/models/tacotron2/vocodes_server.py +++ b/models/tacotron2/vocodes_server.py @@ -229,7 +229,6 @@ class ApiHandler(): maybe_clear_synthesizer_checkpoint_path)) if maybe_clear_synthesizer_checkpoint_path: - print('Clearing synthesizer: {}'.format(maybe_clear_synthesizer_checkpoint_path)) pipeline.uncache_tacotron_model(maybe_clear_synthesizer_checkpoint_path) pipeline.maybe_load_waveglow_model(vocoder_checkpoint_path) diff --git a/src/clients/tts_inference_sidecar_client.rs b/src/clients/tts_inference_sidecar_client.rs index 7f33574a92..a302d081f2 100644 --- a/src/clients/tts_inference_sidecar_client.rs +++ b/src/clients/tts_inference_sidecar_client.rs @@ -20,7 +20,9 @@ struct InferenceRequest { pub output_audio_filename : String, pub output_spectrogram_filename : String, pub output_metadata_filename : String, - pub maybe_unload_model_path: Option, + + /// To instruct the sidecar to unload the model from memory + pub maybe_clear_synthesizer_checkpoint_path: Option, } impl TtsInferenceSidecarClient { @@ -82,7 +84,7 @@ impl TtsInferenceSidecarClient { output_audio_filename, output_spectrogram_filename, output_metadata_filename, - maybe_unload_model_path, + maybe_clear_synthesizer_checkpoint_path: maybe_unload_model_path, }; let url = format!("http://{}/infer", self.hostname); diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 10b0d71901..884f6d7213 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -532,7 +532,7 @@ async fn process_job( info!("Expected output audio filename: {:?}", &output_audio_fs_path); info!("Expected output spectrogram filename: {:?}", &output_spectrogram_fs_path); info!("Expected output metadata filename: {:?}", &output_metadata_fs_path); - info!("Maybe unload model: {:?}", &maybe_unload_model_path); + warn!("Maybe unload model: {:?}", &maybe_unload_model_path); //inferencer.tts_inference_command.execute( // &tts_synthesizer_fs_path, From 26be0594e8ca1d602f5e4bcc41545c0bd6b2e679 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Jul 2021 10:41:14 -0400 Subject: [PATCH 0456/1042] control number of sidecar models --- src/tts_inference_job.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 884f6d7213..72bb1f7a38 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -113,6 +113,9 @@ struct Inferencer { // How long to wait between log lines pub no_op_logger_millis: u64, + + // Maximum number of synthesizer models to hold in memory. + pub sidecar_max_synthesizer_models: u32, } #[tokio::main] @@ -205,6 +208,8 @@ async fn main() -> AnyhowResult<()> { let tts_vocoder_model_filename = easyenv::get_env_string_or_default( "TTS_VOCODER_MODEL_FILENAME", "waveglow.pth"); + let sidecar_max_synthesizer_models = easyenv::get_env_num( + "SIDECAR_MAX_SYNTHESIZER_MODELS", 3)?; let firehose_publisher = FirehosePublisher { mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe @@ -233,6 +238,7 @@ async fn main() -> AnyhowResult<()> { job_max_attempts: common_env.job_max_attempts as i32, job_batch_size: common_env.job_batch_size, no_op_logger_millis: common_env.no_op_logger_millis, + sidecar_max_synthesizer_models, }; main_loop(inferencer).await; From bd289a06b35a21a55bbab2ab7a958124e265e4b4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Jul 2021 11:41:18 -0400 Subject: [PATCH 0457/1042] wait on cold memory --- .env | 1 + src/tts_inference_job.rs | 39 ++++++++++++++++++++++++++++++--------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/.env b/.env index 26d3ebce9c..995c16971e 100644 --- a/.env +++ b/.env @@ -43,6 +43,7 @@ TTS_INFERENCE_SCRIPT_NAME=vocodes_inference.py TTS_MODEL_CHECK_SCRIPT_NAME=vocodes_model_check.py TTS_INFERENCE_SIDECAR_HOSTNAME=localhost:8000 +SIDECAR_MAX_SYNTHESIZER_MODELS=2 #TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt # THIS SOUNDS HORRID #TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 72bb1f7a38..9d57a15d80 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -322,7 +322,9 @@ async fn process_jobs( let model_state_result = ModelState::query_model_and_check_filesystem( &job, &inferencer.mysql_pool, - &inferencer.semi_persistent_cache).await; + &inferencer.semi_persistent_cache, + &inferencer.virtual_model_lfu, + ).await; let model_state = match model_state_result { Ok(model_state) => model_state, @@ -340,14 +342,20 @@ async fn process_jobs( } }; - if !model_state.is_downloaded_to_filesystem && !cold_cache_okay { - // We're going to skip this for now. (Maybe another worker has a warm cache and can continue.) - info!("Skipping TTS due to cold cache: {} ({})", - model_state.model_record.model_token, - model_state.model_record.title); + if !model_state.is_downloaded_to_filesystem || !model_state.is_in_memory_cache { + warn!("Model isn't in cache (downloaded = {}), (in memory = {})", + model_state.is_downloaded_to_filesystem, + model_state.is_in_memory_cache); - skipped_for_cold_cache += 1; - continue; + if !cold_cache_okay { + // We're going to skip this for now. (Maybe another worker has a warm cache and can continue.) + warn!("Skipping TTS due to cold cache: {} ({})", + model_state.model_record.model_token, + model_state.model_record.title); + + skipped_for_cold_cache += 1; + continue; + } } let result = process_job(inferencer, &job, &model_state.model_record).await; @@ -388,6 +396,7 @@ fn read_metadata_file(filename: &PathBuf) -> AnyhowResult { struct ModelState { pub model_record: TtsModelRecord2, pub is_downloaded_to_filesystem: bool, + pub is_in_memory_cache: bool, } impl ModelState { @@ -396,6 +405,7 @@ impl ModelState { job: &TtsInferenceJobRecord, mysql_pool: &MySqlPool, semi_persistent_cache: &SemiPersistentCacheDir, + virtual_cache: &SyncVirtualLfuCache, ) -> AnyhowResult { info!("Looking up TTS model by token: {}", &job.model_token); @@ -416,9 +426,17 @@ impl ModelState { let is_downloaded_to_filesystem = tts_synthesizer_fs_path.exists(); + let path = tts_synthesizer_fs_path + .to_str() + .ok_or(anyhow!("could not make path"))? + .to_string(); + + let is_in_memory_cache = virtual_cache.in_cache(&path)?; + Ok(Self { model_record, is_downloaded_to_filesystem, + is_in_memory_cache, }) } } @@ -538,7 +556,10 @@ async fn process_job( info!("Expected output audio filename: {:?}", &output_audio_fs_path); info!("Expected output spectrogram filename: {:?}", &output_spectrogram_fs_path); info!("Expected output metadata filename: {:?}", &output_metadata_fs_path); - warn!("Maybe unload model: {:?}", &maybe_unload_model_path); + + if let Some(model_path) = maybe_unload_model_path.as_deref() { + warn!("Unload model from sidecar: {:?}", &model_path); + } //inferencer.tts_inference_command.execute( // &tts_synthesizer_fs_path, From 51ba9517d9e827788935f1938dafa0ced89b3de1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Jul 2021 11:46:29 -0400 Subject: [PATCH 0458/1042] sort model names alphabetically --- src/http_server/endpoints/tts/list_tts_models.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/http_server/endpoints/tts/list_tts_models.rs b/src/http_server/endpoints/tts/list_tts_models.rs index 276992520d..9e947487e6 100644 --- a/src/http_server/endpoints/tts/list_tts_models.rs +++ b/src/http_server/endpoints/tts/list_tts_models.rs @@ -106,7 +106,7 @@ WHERE .fetch_all(&server_state.mysql_pool) .await; // TODO: This will return error if it doesn't exist - let models : Vec = match maybe_models { + let mut models : Vec = match maybe_models { Ok(models) => models, Err(err) => { match err { @@ -121,6 +121,8 @@ WHERE } }; + models.sort_by(|a, b| (&a.title).cmp(&b.title)); + let models_for_response = models.into_iter() .map(|model| { TtsModelRecordForResponse { From 453991f6d89b40987c6a9aa890606966283bf7e2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Jul 2021 15:01:27 -0400 Subject: [PATCH 0459/1042] WIP cold cache strategy --- src/tts_inference_job.rs | 7 +- src/util/jobs/cold_cache_strategy.rs | 86 ++++++++++++++++++++++++ src/util/jobs/mod.rs | 2 + src/util/{ => jobs}/virtual_lfu_cache.rs | 2 +- src/util/mod.rs | 4 +- 5 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 src/util/jobs/cold_cache_strategy.rs create mode 100644 src/util/jobs/mod.rs rename src/util/{ => jobs}/virtual_lfu_cache.rs (98%) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 9d57a15d80..aded992df9 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -21,15 +21,16 @@ pub mod util; use anyhow::anyhow; use chrono::{Utc, DateTime, TimeZone}; +use crate::clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; -use crate::job_queries::tts_inference_job_queries::{TtsInferenceJobRecord, TtsModelRecord2}; use crate::job_queries::tts_inference_job_queries::get_tts_model_by_token; use crate::job_queries::tts_inference_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::tts_inference_job_queries::insert_tts_result; use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_done; use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_failure; use crate::job_queries::tts_inference_job_queries::query_tts_inference_job_records; +use crate::job_queries::tts_inference_job_queries::{TtsInferenceJobRecord, TtsModelRecord2}; use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use crate::shared_constants::DEFAULT_RUST_LOG; @@ -41,15 +42,16 @@ use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::hashing::hash_string_sha2::hash_string_sha2; +use crate::util::jobs::virtual_lfu_cache::SyncVirtualLfuCache; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; -use crate::util::virtual_lfu_cache::SyncVirtualLfuCache; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; +use std::collections::HashMap; use std::fs::{File, metadata}; use std::io::{BufReader, Read}; use std::path::{PathBuf, Path}; @@ -57,7 +59,6 @@ use std::process::Command; use std::thread; use std::time::Duration; use tempdir::TempDir; -use crate::clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; diff --git a/src/util/jobs/cold_cache_strategy.rs b/src/util/jobs/cold_cache_strategy.rs new file mode 100644 index 0000000000..3031ee8e08 --- /dev/null +++ b/src/util/jobs/cold_cache_strategy.rs @@ -0,0 +1,86 @@ +use std::collections::HashMap; +use chrono::{DateTime, Utc, Duration}; + +#[derive(Eq, PartialEq, Copy, Clone, Debug)] +pub enum ColdCacheStrategy { + /// Don't handle the miss right now. Wait or skip. + WaitOrSkip, + /// Handle the miss now. + Proceed, +} + +pub struct ColdCacheLog { + max_cold_duration: Duration, + cache_miss_log: HashMap>, + //get_time_function: Box DateTime>, + get_time_function: fn() -> DateTime, + + //pub not_downloaded_model_ids_versus_miss_counts: HashMap>, + + // There should be less of a penalty for cold memory. + //pub not_in_memory_model_ids_versus_miss_counts: HashMap>, +} + +impl ColdCacheLog { + + pub fn new(max_cold_duration: Duration) -> Self { + Self { + max_cold_duration, + cache_miss_log: HashMap::new(), + get_time_function: || Utc::now(), + } + } + + pub fn new_for_testing( + max_cold_duration: Duration, + //get_time_function: Box DateTime> + get_time_function: fn() -> DateTime, + ) -> Self { + Self { + max_cold_duration, + cache_miss_log: HashMap::new(), + get_time_function, + } + } + + + // NB: Not threadsafe! + #[must_use] + pub fn cache_miss(&mut self, id: i64) -> ColdCacheStrategy { + let now : DateTime = (self.get_time_function)(); + + if let Some((_id, wait_start)) = self.cache_miss_log.get_key_value(&id) { + let duration = now.signed_duration_since(wait_start.clone()); + + if duration > self.max_cold_duration { + self.cache_miss_log.remove(&id); + return ColdCacheStrategy::Proceed; + } + } else { + self.cache_miss_log.insert(id, now); + } + + ColdCacheStrategy::WaitOrSkip + } + + // NB: For testing + fn set_time_function(&mut self, get_time_function: fn() -> DateTime) { + self.get_time_function = get_time_function; + } +} + +#[cfg(test)] +mod tests { + use crate::util::jobs::cold_cache_strategy::ColdCacheLog; + use crate::util::jobs::cold_cache_strategy::ColdCacheStrategy; + use chrono::{Duration, Utc}; + + #[test] + fn cold_cache_log_cache_miss() { + let mut cold_cache_log = ColdCacheLog::new(Duration::seconds(10)); + + cold_cache_log.set_time_function(|| Utc::now()); + + assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); + } +} \ No newline at end of file diff --git a/src/util/jobs/mod.rs b/src/util/jobs/mod.rs new file mode 100644 index 0000000000..e67cfe8380 --- /dev/null +++ b/src/util/jobs/mod.rs @@ -0,0 +1,2 @@ +pub mod cold_cache_strategy; +pub mod virtual_lfu_cache; diff --git a/src/util/virtual_lfu_cache.rs b/src/util/jobs/virtual_lfu_cache.rs similarity index 98% rename from src/util/virtual_lfu_cache.rs rename to src/util/jobs/virtual_lfu_cache.rs index 8584ffbeb5..0a25f23fd0 100644 --- a/src/util/virtual_lfu_cache.rs +++ b/src/util/jobs/virtual_lfu_cache.rs @@ -106,7 +106,7 @@ impl SyncVirtualLfuCache { #[cfg(test)] pub mod tests { - use crate::util::virtual_lfu_cache::VirtualLfuCache; + use crate::util::jobs::virtual_lfu_cache::VirtualLfuCache; #[test] fn insert_beyond_capacity() { diff --git a/src/util/mod.rs b/src/util/mod.rs index c9c3a663ce..933401bacb 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -4,11 +4,11 @@ pub mod email_to_gravatar; pub mod encrypted_sort_id; pub mod filesystem; pub mod hashing; +pub mod jobs; pub mod markdown_to_html; pub mod noop_logger; pub mod random_crockford_token; pub mod random_prefix_crockford_token; pub mod random_uuid; pub mod redis_keys; -pub mod semi_persistent_cache_dir; -pub mod virtual_lfu_cache; \ No newline at end of file +pub mod semi_persistent_cache_dir; \ No newline at end of file From cd508ed946eefb65386fb70ffe364719906d944e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Jul 2021 22:16:36 -0400 Subject: [PATCH 0460/1042] cold cache tracking algorithm works --- src/util/jobs/cold_cache_strategy.rs | 53 ++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/src/util/jobs/cold_cache_strategy.rs b/src/util/jobs/cold_cache_strategy.rs index 3031ee8e08..e6a990bfd1 100644 --- a/src/util/jobs/cold_cache_strategy.rs +++ b/src/util/jobs/cold_cache_strategy.rs @@ -12,8 +12,8 @@ pub enum ColdCacheStrategy { pub struct ColdCacheLog { max_cold_duration: Duration, cache_miss_log: HashMap>, - //get_time_function: Box DateTime>, - get_time_function: fn() -> DateTime, + get_time_function: Box DateTime>, + //get_time_function: fn() -> DateTime, //pub not_downloaded_model_ids_versus_miss_counts: HashMap>, @@ -27,14 +27,15 @@ impl ColdCacheLog { Self { max_cold_duration, cache_miss_log: HashMap::new(), - get_time_function: || Utc::now(), + //get_time_function: || Utc::now(), + get_time_function: Box::new(|| Utc::now()), } } pub fn new_for_testing( max_cold_duration: Duration, - //get_time_function: Box DateTime> - get_time_function: fn() -> DateTime, + get_time_function: Box DateTime> + //get_time_function: fn() -> DateTime, ) -> Self { Self { max_cold_duration, @@ -64,7 +65,8 @@ impl ColdCacheLog { } // NB: For testing - fn set_time_function(&mut self, get_time_function: fn() -> DateTime) { + //fn set_time_function(&mut self, get_time_function: fn() -> DateTime) { + fn set_time_function(&mut self, get_time_function: Box DateTime>) { self.get_time_function = get_time_function; } } @@ -73,14 +75,45 @@ impl ColdCacheLog { mod tests { use crate::util::jobs::cold_cache_strategy::ColdCacheLog; use crate::util::jobs::cold_cache_strategy::ColdCacheStrategy; - use chrono::{Duration, Utc}; + use chrono::{Duration, Utc, DateTime, TimeZone}; + + fn get_date(datetime: &str) -> DateTime { + let datetime = DateTime::parse_from_rfc3339(datetime).unwrap(); + let utc : DateTime = DateTime::from(datetime); + utc + } #[test] - fn cold_cache_log_cache_miss() { + fn cold_cache_cache_miss_algorithm() { let mut cold_cache_log = ColdCacheLog::new(Duration::seconds(10)); - cold_cache_log.set_time_function(|| Utc::now()); - + // First invocation + cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:00+00:00"))); assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); + cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:01+00:00"))); + assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); + cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:05+00:00"))); + assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); + cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:10+00:00"))); + assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); + // Final invocation after time expires. Proceed. + cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:11+00:00"))); + assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::Proceed); + + // New invocation. + cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:15+00:00"))); + assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); + cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:25+00:00"))); + // New ID. + assert_eq!(cold_cache_log.cache_miss(20), ColdCacheStrategy::WaitOrSkip); + // Old ID (still wait) + assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); + // Old ID is done, new ID is still waiting. + cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:30+00:00"))); + assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::Proceed); + assert_eq!(cold_cache_log.cache_miss(20), ColdCacheStrategy::WaitOrSkip); + // Now the new ID is also done. + cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:41+00:00"))); + assert_eq!(cold_cache_log.cache_miss(20), ColdCacheStrategy::Proceed); } } \ No newline at end of file From 9c447e0442b89f643831232c3a614bbe2d5f9fee Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Jul 2021 22:57:43 -0400 Subject: [PATCH 0461/1042] multithreaded version --- src/util/jobs/cold_cache_strategy.rs | 83 ++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 10 deletions(-) diff --git a/src/util/jobs/cold_cache_strategy.rs b/src/util/jobs/cold_cache_strategy.rs index e6a990bfd1..06611d4c91 100644 --- a/src/util/jobs/cold_cache_strategy.rs +++ b/src/util/jobs/cold_cache_strategy.rs @@ -1,28 +1,92 @@ -use std::collections::HashMap; +use anyhow::anyhow; use chrono::{DateTime, Utc, Duration}; +use crate::util::anyhow_result::AnyhowResult; +use std::collections::HashMap; +use std::sync::{RwLock, Arc}; #[derive(Eq, PartialEq, Copy, Clone, Debug)] pub enum ColdCacheStrategy { - /// Don't handle the miss right now. Wait or skip. + /// Don't handle the cache miss right now. + /// Wait or skip until told otherwise. WaitOrSkip, - /// Handle the miss now. + /// Handle the cache miss now. + /// Do the download, calculation, whatever immediately. Proceed, } +/// Keep track of multiple caches. +/// This tells the caller when to ignore work or when to pick it up. +pub struct MultiColdCacheLog { + in_memory_log: ColdCacheLog, + on_disk_log: ColdCacheLog, + // ... future caches ? +} + +/// Keep track of a cache and tell the caller when to proceed with work. +/// If it isn't time yet, skip or wait and let another consumer handle it. pub struct ColdCacheLog { max_cold_duration: Duration, + //history_expire_duration: Duration, cache_miss_log: HashMap>, get_time_function: Box DateTime>, - //get_time_function: fn() -> DateTime, +} - //pub not_downloaded_model_ids_versus_miss_counts: HashMap>, +/// And the Sync/Send threadsafe + interior mutability version. +#[derive(Clone)] +pub struct SyncMultiColdCacheLog { + multi_cache_log: Arc>, +} - // There should be less of a penalty for cold memory. - //pub not_in_memory_model_ids_versus_miss_counts: HashMap>, +impl SyncMultiColdCacheLog { + pub fn new(memory_max_cold_duration: Duration, disk_max_cold_duration: Duration) -> Self { + let multi_cache_log = MultiColdCacheLog::new( + memory_max_cold_duration, + disk_max_cold_duration, + ); + Self { + multi_cache_log: Arc::new(RwLock::new(multi_cache_log)), + } + } + + #[must_use] + pub fn memory_cache_miss(&self, id: i64) -> AnyhowResult { + let result = self.multi_cache_log + .write() + .map(|mut cache_logs| cache_logs.memory_cache_miss(id)) + .map_err(|err| anyhow!("mutex error: {:?}", err))?; + Ok(result) + } + + #[must_use] + pub fn disk_cache_miss(&self, id: i64) -> AnyhowResult { + let result = self.multi_cache_log + .write() + .map(|mut cache_logs| cache_logs.disk_cache_miss(id)) + .map_err(|err| anyhow!("mutex error: {:?}", err))?; + Ok(result) + } +} + +impl MultiColdCacheLog { + pub fn new(memory_max_cold_duration: Duration, disk_max_cold_duration: Duration) -> Self { + Self { + in_memory_log: ColdCacheLog::new(memory_max_cold_duration), + on_disk_log: ColdCacheLog::new(disk_max_cold_duration), + } + } + + #[must_use] + pub fn memory_cache_miss(&mut self, id: i64) -> ColdCacheStrategy { + self.in_memory_log.cache_miss(id) + } + + #[must_use] + pub fn disk_cache_miss(&mut self, id: i64) -> ColdCacheStrategy { + self.on_disk_log.cache_miss(id) + } } impl ColdCacheLog { - pub fn new(max_cold_duration: Duration) -> Self { Self { max_cold_duration, @@ -44,8 +108,7 @@ impl ColdCacheLog { } } - - // NB: Not threadsafe! + // NB: Not threadsafe due to multiple operations against hashes! #[must_use] pub fn cache_miss(&mut self, id: i64) -> ColdCacheStrategy { let now : DateTime = (self.get_time_function)(); From 424b98bb9d1ca2a3a5530c8158702897694394ce Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Jul 2021 23:24:53 -0400 Subject: [PATCH 0462/1042] rename cache strategizer, add forget duration (untested) --- src/util/jobs/cache_miss_strategizer.rs | 170 ++++++++++++++++ src/util/jobs/cache_miss_strategizer_multi.rs | 68 +++++++ src/util/jobs/cold_cache_strategy.rs | 182 ------------------ src/util/jobs/mod.rs | 3 +- 4 files changed, 240 insertions(+), 183 deletions(-) create mode 100644 src/util/jobs/cache_miss_strategizer.rs create mode 100644 src/util/jobs/cache_miss_strategizer_multi.rs delete mode 100644 src/util/jobs/cold_cache_strategy.rs diff --git a/src/util/jobs/cache_miss_strategizer.rs b/src/util/jobs/cache_miss_strategizer.rs new file mode 100644 index 0000000000..7b965fcc0d --- /dev/null +++ b/src/util/jobs/cache_miss_strategizer.rs @@ -0,0 +1,170 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc, Duration}; +use std::collections::HashMap; + +/// How to handle cache miss events, as dictated by the `CacheMissStrategizer`. +#[derive(Eq, PartialEq, Copy, Clone, Debug)] +pub enum CacheMissStrategy { + /// Don't handle the cache miss right now. + /// Wait or skip until told otherwise. + WaitOrSkip, + /// Handle the cache miss now. + /// Do the download, calculation, whatever immediately. + Proceed, +} + +struct CacheMissLog { + pub first_miss: DateTime, + pub most_recent_miss: DateTime, +} + +/// CacheMissStrategizer +/// +/// This is the core component of the cold cache system. +/// +/// Keep track of a cache and tell the caller when to proceed with work. +/// If it isn't time yet, skip or wait and let another consumer handle it. +pub struct CacheMissStrategizer { + /// How long it takes after the first cache miss to do work. + max_cold_duration: Duration, + + /// How long it takes to reset the cold cache timer. + /// eg. If we see a cache miss for X and then another one a day later, + /// that shouldn't be cause to do work now. + forget_duration: Duration, + + /// Log of all cache misses. + cache_miss_log: HashMap, + + /// Clock can be replaced for testing. + get_time_function: Box DateTime>, +} + +impl CacheMissLog { + fn new(first_miss: DateTime) -> Self { + Self { + first_miss: first_miss.clone(), + most_recent_miss: first_miss, + } + } + + fn with_most_recent_miss(&self, most_recent_miss: DateTime) -> Self { + Self { + first_miss: self.first_miss.clone(), + most_recent_miss, + } + } +} + +impl CacheMissStrategizer { + pub fn new(max_cold_duration: Duration, forget_duration: Duration) -> Self { + Self { + max_cold_duration, + forget_duration, + cache_miss_log: HashMap::new(), + get_time_function: Box::new(|| Utc::now()), + } + } + + pub fn new_for_testing( + max_cold_duration: Duration, + forget_duration: Duration, + get_time_function: Box DateTime> + ) -> Self { + Self { + max_cold_duration, + forget_duration, + cache_miss_log: HashMap::new(), + get_time_function, + } + } + + // NB: Not threadsafe due to multiple operations against hashes! + #[must_use] + pub fn cache_miss(&mut self, id: i64) -> CacheMissStrategy { + let now : DateTime = (self.get_time_function)(); + + if let Some((_id, cache_miss_log)) = self.cache_miss_log.get_key_value(&id) { + let admit_duration = now.signed_duration_since(cache_miss_log.first_miss.clone()); + let forget_duration = now.signed_duration_since(cache_miss_log.most_recent_miss.clone()); + + if forget_duration > self.forget_duration { + // Too much time elapsed. + // Treat this as essentially seeing the miss for the first time. + let cache_miss_log = CacheMissLog::new(now); + self.cache_miss_log.insert(id, cache_miss_log); + CacheMissStrategy::WaitOrSkip + } + else if admit_duration > self.max_cold_duration { + // We're done waiting. Do the work. + self.cache_miss_log.remove(&id); + CacheMissStrategy::Proceed + } + else { + // Still waiting. + let cache_miss_log = cache_miss_log.with_most_recent_miss(now); + self.cache_miss_log.insert(id, cache_miss_log); + CacheMissStrategy::WaitOrSkip + } + } else { + let cache_miss_log = CacheMissLog::new(now); + self.cache_miss_log.insert(id, cache_miss_log); + CacheMissStrategy::WaitOrSkip + } + } + + // NB: Private; for testing only + fn set_time_function(&mut self, get_time_function: Box DateTime>) { + self.get_time_function = get_time_function; + } +} + +#[cfg(test)] +mod tests { + use chrono::{Duration, Utc, DateTime, TimeZone}; + use crate::util::jobs::cache_miss_strategizer::CacheMissStrategizer; + use crate::util::jobs::cache_miss_strategizer::CacheMissStrategy; + + fn get_date(datetime: &str) -> DateTime { + let datetime = DateTime::parse_from_rfc3339(datetime).unwrap(); + let utc : DateTime = DateTime::from(datetime); + utc + } + + #[test] + fn cold_cache_cache_miss_algorithm() { + let mut cache_miss_strategizer = CacheMissStrategizer::new( + Duration::seconds(10), + Duration::seconds(60), + ); + + // First invocation + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:00:00+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(10), CacheMissStrategy::WaitOrSkip); + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:00:01+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(10), CacheMissStrategy::WaitOrSkip); + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:00:05+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(10), CacheMissStrategy::WaitOrSkip); + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:00:10+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(10), CacheMissStrategy::WaitOrSkip); + // Final invocation after time expires. Proceed. + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:00:11+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(10), CacheMissStrategy::Proceed); + + // New invocation. + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:00:15+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(10), CacheMissStrategy::WaitOrSkip); + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:00:25+00:00"))); + // New ID. + assert_eq!(cache_miss_strategizer.cache_miss(20), CacheMissStrategy::WaitOrSkip); + // Old ID (still wait) + assert_eq!(cache_miss_strategizer.cache_miss(10), CacheMissStrategy::WaitOrSkip); + // Old ID is done, new ID is still waiting. + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:00:30+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(10), CacheMissStrategy::Proceed); + assert_eq!(cache_miss_strategizer.cache_miss(20), CacheMissStrategy::WaitOrSkip); + // Now the new ID is also done. + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:00:41+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(20), CacheMissStrategy::Proceed); + } +} \ No newline at end of file diff --git a/src/util/jobs/cache_miss_strategizer_multi.rs b/src/util/jobs/cache_miss_strategizer_multi.rs new file mode 100644 index 0000000000..4c91f051a7 --- /dev/null +++ b/src/util/jobs/cache_miss_strategizer_multi.rs @@ -0,0 +1,68 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc, Duration}; +use crate::util::anyhow_result::AnyhowResult; +use crate::util::jobs::cache_miss_strategizer::{CacheMissStrategizer, CacheMissStrategy}; +use std::sync::{RwLock, Arc}; + +/// Keep track of multiple caches, each with different time penalties. +/// (This isn't strictly necessary) +pub struct MultiCacheMissStrategizer { + in_memory_log: CacheMissStrategizer, + on_disk_log: CacheMissStrategizer, + // ... future caches ? +} + +/// And the Sync/Send threadsafe + interior mutability version. +#[derive(Clone)] +pub struct SyncMultiCacheMissStrategizer { + multi_cache_log: Arc>, +} + +impl SyncMultiCacheMissStrategizer { + pub fn new(in_memory_log: CacheMissStrategizer, on_disk_log: CacheMissStrategizer) -> Self { + let multi_cache_log = MultiCacheMissStrategizer::new( + in_memory_log, + on_disk_log, + ); + Self { + multi_cache_log: Arc::new(RwLock::new(multi_cache_log)), + } + } + + #[must_use] + pub fn memory_cache_miss(&self, id: i64) -> AnyhowResult { + let result = self.multi_cache_log + .write() + .map(|mut cache_logs| cache_logs.memory_cache_miss(id)) + .map_err(|err| anyhow!("mutex error: {:?}", err))?; + Ok(result) + } + + #[must_use] + pub fn disk_cache_miss(&self, id: i64) -> AnyhowResult { + let result = self.multi_cache_log + .write() + .map(|mut cache_logs| cache_logs.disk_cache_miss(id)) + .map_err(|err| anyhow!("mutex error: {:?}", err))?; + Ok(result) + } +} + +impl MultiCacheMissStrategizer { + pub fn new(in_memory_log: CacheMissStrategizer, on_disk_log: CacheMissStrategizer) -> Self { + Self { + in_memory_log, + on_disk_log, + } + } + + #[must_use] + pub fn memory_cache_miss(&mut self, id: i64) -> CacheMissStrategy { + self.in_memory_log.cache_miss(id) + } + + #[must_use] + pub fn disk_cache_miss(&mut self, id: i64) -> CacheMissStrategy { + self.on_disk_log.cache_miss(id) + } +} diff --git a/src/util/jobs/cold_cache_strategy.rs b/src/util/jobs/cold_cache_strategy.rs deleted file mode 100644 index 06611d4c91..0000000000 --- a/src/util/jobs/cold_cache_strategy.rs +++ /dev/null @@ -1,182 +0,0 @@ -use anyhow::anyhow; -use chrono::{DateTime, Utc, Duration}; -use crate::util::anyhow_result::AnyhowResult; -use std::collections::HashMap; -use std::sync::{RwLock, Arc}; - -#[derive(Eq, PartialEq, Copy, Clone, Debug)] -pub enum ColdCacheStrategy { - /// Don't handle the cache miss right now. - /// Wait or skip until told otherwise. - WaitOrSkip, - /// Handle the cache miss now. - /// Do the download, calculation, whatever immediately. - Proceed, -} - -/// Keep track of multiple caches. -/// This tells the caller when to ignore work or when to pick it up. -pub struct MultiColdCacheLog { - in_memory_log: ColdCacheLog, - on_disk_log: ColdCacheLog, - // ... future caches ? -} - -/// Keep track of a cache and tell the caller when to proceed with work. -/// If it isn't time yet, skip or wait and let another consumer handle it. -pub struct ColdCacheLog { - max_cold_duration: Duration, - //history_expire_duration: Duration, - cache_miss_log: HashMap>, - get_time_function: Box DateTime>, -} - -/// And the Sync/Send threadsafe + interior mutability version. -#[derive(Clone)] -pub struct SyncMultiColdCacheLog { - multi_cache_log: Arc>, -} - -impl SyncMultiColdCacheLog { - pub fn new(memory_max_cold_duration: Duration, disk_max_cold_duration: Duration) -> Self { - let multi_cache_log = MultiColdCacheLog::new( - memory_max_cold_duration, - disk_max_cold_duration, - ); - Self { - multi_cache_log: Arc::new(RwLock::new(multi_cache_log)), - } - } - - #[must_use] - pub fn memory_cache_miss(&self, id: i64) -> AnyhowResult { - let result = self.multi_cache_log - .write() - .map(|mut cache_logs| cache_logs.memory_cache_miss(id)) - .map_err(|err| anyhow!("mutex error: {:?}", err))?; - Ok(result) - } - - #[must_use] - pub fn disk_cache_miss(&self, id: i64) -> AnyhowResult { - let result = self.multi_cache_log - .write() - .map(|mut cache_logs| cache_logs.disk_cache_miss(id)) - .map_err(|err| anyhow!("mutex error: {:?}", err))?; - Ok(result) - } -} - -impl MultiColdCacheLog { - pub fn new(memory_max_cold_duration: Duration, disk_max_cold_duration: Duration) -> Self { - Self { - in_memory_log: ColdCacheLog::new(memory_max_cold_duration), - on_disk_log: ColdCacheLog::new(disk_max_cold_duration), - } - } - - #[must_use] - pub fn memory_cache_miss(&mut self, id: i64) -> ColdCacheStrategy { - self.in_memory_log.cache_miss(id) - } - - #[must_use] - pub fn disk_cache_miss(&mut self, id: i64) -> ColdCacheStrategy { - self.on_disk_log.cache_miss(id) - } -} - -impl ColdCacheLog { - pub fn new(max_cold_duration: Duration) -> Self { - Self { - max_cold_duration, - cache_miss_log: HashMap::new(), - //get_time_function: || Utc::now(), - get_time_function: Box::new(|| Utc::now()), - } - } - - pub fn new_for_testing( - max_cold_duration: Duration, - get_time_function: Box DateTime> - //get_time_function: fn() -> DateTime, - ) -> Self { - Self { - max_cold_duration, - cache_miss_log: HashMap::new(), - get_time_function, - } - } - - // NB: Not threadsafe due to multiple operations against hashes! - #[must_use] - pub fn cache_miss(&mut self, id: i64) -> ColdCacheStrategy { - let now : DateTime = (self.get_time_function)(); - - if let Some((_id, wait_start)) = self.cache_miss_log.get_key_value(&id) { - let duration = now.signed_duration_since(wait_start.clone()); - - if duration > self.max_cold_duration { - self.cache_miss_log.remove(&id); - return ColdCacheStrategy::Proceed; - } - } else { - self.cache_miss_log.insert(id, now); - } - - ColdCacheStrategy::WaitOrSkip - } - - // NB: For testing - //fn set_time_function(&mut self, get_time_function: fn() -> DateTime) { - fn set_time_function(&mut self, get_time_function: Box DateTime>) { - self.get_time_function = get_time_function; - } -} - -#[cfg(test)] -mod tests { - use crate::util::jobs::cold_cache_strategy::ColdCacheLog; - use crate::util::jobs::cold_cache_strategy::ColdCacheStrategy; - use chrono::{Duration, Utc, DateTime, TimeZone}; - - fn get_date(datetime: &str) -> DateTime { - let datetime = DateTime::parse_from_rfc3339(datetime).unwrap(); - let utc : DateTime = DateTime::from(datetime); - utc - } - - #[test] - fn cold_cache_cache_miss_algorithm() { - let mut cold_cache_log = ColdCacheLog::new(Duration::seconds(10)); - - // First invocation - cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:00+00:00"))); - assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); - cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:01+00:00"))); - assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); - cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:05+00:00"))); - assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); - cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:10+00:00"))); - assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); - // Final invocation after time expires. Proceed. - cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:11+00:00"))); - assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::Proceed); - - // New invocation. - cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:15+00:00"))); - assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); - cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:25+00:00"))); - // New ID. - assert_eq!(cold_cache_log.cache_miss(20), ColdCacheStrategy::WaitOrSkip); - // Old ID (still wait) - assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::WaitOrSkip); - // Old ID is done, new ID is still waiting. - cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:30+00:00"))); - assert_eq!(cold_cache_log.cache_miss(10), ColdCacheStrategy::Proceed); - assert_eq!(cold_cache_log.cache_miss(20), ColdCacheStrategy::WaitOrSkip); - // Now the new ID is also done. - cold_cache_log.set_time_function(Box::new(|| get_date("2021-07-01T13:00:41+00:00"))); - assert_eq!(cold_cache_log.cache_miss(20), ColdCacheStrategy::Proceed); - } -} \ No newline at end of file diff --git a/src/util/jobs/mod.rs b/src/util/jobs/mod.rs index e67cfe8380..b871837a3e 100644 --- a/src/util/jobs/mod.rs +++ b/src/util/jobs/mod.rs @@ -1,2 +1,3 @@ -pub mod cold_cache_strategy; +pub mod cache_miss_strategizer; +pub mod cache_miss_strategizer_multi; pub mod virtual_lfu_cache; From f28e405f6de553e6f7463fc623f48e5c277125aa Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Jul 2021 23:30:57 -0400 Subject: [PATCH 0463/1042] tests on expiry --- src/util/jobs/cache_miss_strategizer.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/util/jobs/cache_miss_strategizer.rs b/src/util/jobs/cache_miss_strategizer.rs index 7b965fcc0d..b8ce6d1ecd 100644 --- a/src/util/jobs/cache_miss_strategizer.rs +++ b/src/util/jobs/cache_miss_strategizer.rs @@ -166,5 +166,21 @@ mod tests { // Now the new ID is also done. cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:00:41+00:00"))); assert_eq!(cache_miss_strategizer.cache_miss(20), CacheMissStrategy::Proceed); + + // New Invocation + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:01:00+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(30), CacheMissStrategy::WaitOrSkip); + // We waited too long... + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:02:01+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(30), CacheMissStrategy::WaitOrSkip); + + // New Invocation + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:04:00+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(30), CacheMissStrategy::WaitOrSkip); // previously seen + assert_eq!(cache_miss_strategizer.cache_miss(40), CacheMissStrategy::WaitOrSkip); // new + // We waited just long enough before expiry + cache_miss_strategizer.set_time_function(Box::new(|| get_date("2021-07-01T13:04:59+00:00"))); + assert_eq!(cache_miss_strategizer.cache_miss(30), CacheMissStrategy::Proceed); + assert_eq!(cache_miss_strategizer.cache_miss(40), CacheMissStrategy::Proceed); } } \ No newline at end of file From d2f0710097bb0960d2543b9ef4f7e280d46f3b67 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 6 Jul 2021 02:22:13 -0400 Subject: [PATCH 0464/1042] cache strategies seem to work (too much logging though) --- src/tts_inference_job.rs | 83 +++++++++++++------ src/util/jobs/cache_miss_strategizer.rs | 9 +- src/util/jobs/cache_miss_strategizer_multi.rs | 30 ++++--- 3 files changed, 79 insertions(+), 43 deletions(-) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index aded992df9..20178b84ad 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -19,7 +19,7 @@ pub mod script_execution; pub mod shared_constants; pub mod util; -use anyhow::anyhow; +use anyhow::{anyhow, Error}; use chrono::{Utc, DateTime, TimeZone}; use crate::clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; use crate::common_env::CommonEnv; @@ -42,6 +42,9 @@ use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::hashing::hash_string_sha2::hash_string_sha2; +use crate::util::jobs::cache_miss_strategizer::CacheMissStrategizer; +use crate::util::jobs::cache_miss_strategizer::CacheMissStrategy; +use crate::util::jobs::cache_miss_strategizer_multi::SyncMultiCacheMissStrategizer; use crate::util::jobs::virtual_lfu_cache::SyncVirtualLfuCache; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; @@ -98,6 +101,7 @@ struct Inferencer { // Keep tabs of which models to hold in the sidecar memory with this virtual LRU cache pub virtual_model_lfu: SyncVirtualLfuCache, + pub cache_miss_strategizers: SyncMultiCacheMissStrategizer, // Command to run pub tts_vocoder_model_filename: String, @@ -220,6 +224,23 @@ async fn main() -> AnyhowResult<()> { let virtual_lfu_cache = SyncVirtualLfuCache::new(2)?; + let cache_miss_strategizers = { + let in_memory_strategizer = CacheMissStrategizer::new( + chrono::Duration::milliseconds(5_000), + chrono::Duration::milliseconds(60_000), + ); + + let on_disk_strategizer = CacheMissStrategizer::new( + chrono::Duration::milliseconds(20_000), + chrono::Duration::milliseconds(120_000), + ); + + SyncMultiCacheMissStrategizer::new( + in_memory_strategizer, + on_disk_strategizer, + ) + }; + let inferencer = Inferencer { download_temp_directory: temp_directory, mysql_pool, @@ -231,6 +252,7 @@ async fn main() -> AnyhowResult<()> { tts_inference_command, tts_inference_sidecar_client, virtual_model_lfu: virtual_lfu_cache, + cache_miss_strategizers, bucket_path_unifier: BucketPathUnifier::default_paths(), semi_persistent_cache, firehose_publisher, @@ -255,8 +277,6 @@ async fn main_loop(inferencer: Inferencer) { let mut noop_logger = NoOpLogger::new(inferencer.no_op_logger_millis as i64); - let mut cold_cache_okay = false; - loop { let num_records = inferencer.job_batch_size; @@ -285,28 +305,20 @@ async fn main_loop(inferencer: Inferencer) { let batch_result = process_jobs( &inferencer, jobs, - cold_cache_okay).await; + ).await; - let cold_cache_skipped_count = match batch_result { - Ok(cold_cache_skipped_count) => cold_cache_skipped_count, + match batch_result { + Ok(_) => {}, Err(e) => { warn!("Error querying jobs: {:?}", e); std::thread::sleep(Duration::from_millis(error_timeout_millis)); error_timeout_millis += INCREASE_TIMEOUT_MILLIS; continue; } - }; + } error_timeout_millis = START_TIMEOUT_MILLIS; // reset - // We mostly want to ignore a cold cache unless other workers - // aren't picking up the slack. - if cold_cache_okay { - cold_cache_okay = false; - } else if cold_cache_skipped_count > 0 { - cold_cache_okay = true; - } - std::thread::sleep(Duration::from_millis(inferencer.job_batch_wait_millis)); } } @@ -315,9 +327,7 @@ async fn main_loop(inferencer: Inferencer) { async fn process_jobs( inferencer: &Inferencer, jobs: Vec, - cold_cache_okay: bool -) -> AnyhowResult { - let mut skipped_for_cold_cache = 0; +) -> AnyhowResult<()> { for job in jobs.into_iter() { let model_state_result = ModelState::query_model_and_check_filesystem( @@ -344,18 +354,37 @@ async fn process_jobs( }; if !model_state.is_downloaded_to_filesystem || !model_state.is_in_memory_cache { - warn!("Model isn't in cache (downloaded = {}), (in memory = {})", + warn!("Model isn't ready (downloaded = {}), (in memory = {})", model_state.is_downloaded_to_filesystem, model_state.is_in_memory_cache); - if !cold_cache_okay { - // We're going to skip this for now. (Maybe another worker has a warm cache and can continue.) - warn!("Skipping TTS due to cold cache: {} ({})", - model_state.model_record.model_token, - model_state.model_record.title); + let maybe_strategy = if !model_state.is_downloaded_to_filesystem { + inferencer.cache_miss_strategizers.disk_cache_miss(&model_state.model_record.model_token) + } else { + inferencer.cache_miss_strategizers.memory_cache_miss(&model_state.model_record.model_token) + }; - skipped_for_cold_cache += 1; - continue; + match maybe_strategy { + Err(err) => { + warn!("Failure to process job: {:?}", err); + let failure_reason = ""; + let _r = mark_tts_inference_job_failure( + &inferencer.mysql_pool, + &job, + failure_reason, + inferencer.job_max_attempts + ).await; + continue; + }, + Ok(CacheMissStrategy::WaitOrSkip) => { + // We're going to skip this for now. + // Maybe another worker has a warm cache and can continue. + warn!("Skipping TTS due to cold cache: {} ({})", + model_state.model_record.model_token, + model_state.model_record.title); + continue; + }, + Ok(CacheMissStrategy::Proceed) => {}, // We're going to go ahead... } } @@ -375,7 +404,7 @@ async fn process_jobs( } } - Ok(skipped_for_cold_cache) + Ok(()) } #[derive(Deserialize, Default)] diff --git a/src/util/jobs/cache_miss_strategizer.rs b/src/util/jobs/cache_miss_strategizer.rs index b8ce6d1ecd..333fdbbda7 100644 --- a/src/util/jobs/cache_miss_strategizer.rs +++ b/src/util/jobs/cache_miss_strategizer.rs @@ -1,6 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc, Duration}; use std::collections::HashMap; +use std::hash::Hash; /// How to handle cache miss events, as dictated by the `CacheMissStrategizer`. #[derive(Eq, PartialEq, Copy, Clone, Debug)] @@ -24,7 +25,7 @@ struct CacheMissLog { /// /// Keep track of a cache and tell the caller when to proceed with work. /// If it isn't time yet, skip or wait and let another consumer handle it. -pub struct CacheMissStrategizer { +pub struct CacheMissStrategizer { /// How long it takes after the first cache miss to do work. max_cold_duration: Duration, @@ -34,7 +35,7 @@ pub struct CacheMissStrategizer { forget_duration: Duration, /// Log of all cache misses. - cache_miss_log: HashMap, + cache_miss_log: HashMap, /// Clock can be replaced for testing. get_time_function: Box DateTime>, @@ -56,7 +57,7 @@ impl CacheMissLog { } } -impl CacheMissStrategizer { +impl CacheMissStrategizer { pub fn new(max_cold_duration: Duration, forget_duration: Duration) -> Self { Self { max_cold_duration, @@ -81,7 +82,7 @@ impl CacheMissStrategizer { // NB: Not threadsafe due to multiple operations against hashes! #[must_use] - pub fn cache_miss(&mut self, id: i64) -> CacheMissStrategy { + pub fn cache_miss(&mut self, id: T) -> CacheMissStrategy { let now : DateTime = (self.get_time_function)(); if let Some((_id, cache_miss_log)) = self.cache_miss_log.get_key_value(&id) { diff --git a/src/util/jobs/cache_miss_strategizer_multi.rs b/src/util/jobs/cache_miss_strategizer_multi.rs index 4c91f051a7..554be524cd 100644 --- a/src/util/jobs/cache_miss_strategizer_multi.rs +++ b/src/util/jobs/cache_miss_strategizer_multi.rs @@ -7,8 +7,8 @@ use std::sync::{RwLock, Arc}; /// Keep track of multiple caches, each with different time penalties. /// (This isn't strictly necessary) pub struct MultiCacheMissStrategizer { - in_memory_log: CacheMissStrategizer, - on_disk_log: CacheMissStrategizer, + in_memory_log: CacheMissStrategizer, + on_disk_log: CacheMissStrategizer, // ... future caches ? } @@ -19,7 +19,10 @@ pub struct SyncMultiCacheMissStrategizer { } impl SyncMultiCacheMissStrategizer { - pub fn new(in_memory_log: CacheMissStrategizer, on_disk_log: CacheMissStrategizer) -> Self { + pub fn new( + in_memory_log: CacheMissStrategizer, + on_disk_log: CacheMissStrategizer, + ) -> Self { let multi_cache_log = MultiCacheMissStrategizer::new( in_memory_log, on_disk_log, @@ -30,26 +33,29 @@ impl SyncMultiCacheMissStrategizer { } #[must_use] - pub fn memory_cache_miss(&self, id: i64) -> AnyhowResult { + pub fn memory_cache_miss(&self, token: &str) -> AnyhowResult { let result = self.multi_cache_log .write() - .map(|mut cache_logs| cache_logs.memory_cache_miss(id)) + .map(|mut cache_logs| cache_logs.memory_cache_miss(token)) .map_err(|err| anyhow!("mutex error: {:?}", err))?; Ok(result) } #[must_use] - pub fn disk_cache_miss(&self, id: i64) -> AnyhowResult { + pub fn disk_cache_miss(&self, token: &str) -> AnyhowResult { let result = self.multi_cache_log .write() - .map(|mut cache_logs| cache_logs.disk_cache_miss(id)) + .map(|mut cache_logs| cache_logs.disk_cache_miss(token)) .map_err(|err| anyhow!("mutex error: {:?}", err))?; Ok(result) } } impl MultiCacheMissStrategizer { - pub fn new(in_memory_log: CacheMissStrategizer, on_disk_log: CacheMissStrategizer) -> Self { + pub fn new( + in_memory_log: CacheMissStrategizer, + on_disk_log: CacheMissStrategizer, + ) -> Self { Self { in_memory_log, on_disk_log, @@ -57,12 +63,12 @@ impl MultiCacheMissStrategizer { } #[must_use] - pub fn memory_cache_miss(&mut self, id: i64) -> CacheMissStrategy { - self.in_memory_log.cache_miss(id) + pub fn memory_cache_miss(&mut self, token: &str) -> CacheMissStrategy { + self.in_memory_log.cache_miss(token.to_string()) } #[must_use] - pub fn disk_cache_miss(&mut self, id: i64) -> CacheMissStrategy { - self.on_disk_log.cache_miss(id) + pub fn disk_cache_miss(&mut self, token: &str) -> CacheMissStrategy { + self.on_disk_log.cache_miss(token.to_string()) } } From beb68b50515dbce588706d7d77e7fb4e88835c68 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 6 Jul 2021 02:26:58 -0400 Subject: [PATCH 0465/1042] env var controls --- src/tts_inference_job.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 20178b84ad..a1f330fa50 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -226,13 +226,21 @@ async fn main() -> AnyhowResult<()> { let cache_miss_strategizers = { let in_memory_strategizer = CacheMissStrategizer::new( - chrono::Duration::milliseconds(5_000), - chrono::Duration::milliseconds(60_000), + chrono::Duration::milliseconds( + easyenv::get_env_num("MEMORY_MAX_COLD_DURATION_MILLIS", 5_000)?, + ), + chrono::Duration::milliseconds( + easyenv::get_env_num("MEMORY_CACHE_FORGET_DURATION_MILLIS", 60_000)?, + ), ); let on_disk_strategizer = CacheMissStrategizer::new( - chrono::Duration::milliseconds(20_000), - chrono::Duration::milliseconds(120_000), + chrono::Duration::milliseconds( + easyenv::get_env_num("DISK_MAX_COLD_DURATION_MILLIS", 20_000)?, + ), + chrono::Duration::milliseconds( + easyenv::get_env_num("DISK_CACHE_FORGET_DURATION_MILLIS", 120_000)?, + ), ); SyncMultiCacheMissStrategizer::new( From f58a4353dd9493532213760922e72ed3d2668814 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 9 Jul 2021 22:20:36 -0400 Subject: [PATCH 0466/1042] delete tts model --- src/common_queries/delete_tts_model.rs | 88 ++++++++ src/common_queries/mod.rs | 1 + .../endpoints/tts/delete_tts_model.rs | 196 ++++++++++++++++++ src/http_server/endpoints/tts/mod.rs | 1 + src/main.rs | 6 + 5 files changed, 292 insertions(+) create mode 100644 src/common_queries/delete_tts_model.rs create mode 100644 src/http_server/endpoints/tts/delete_tts_model.rs diff --git a/src/common_queries/delete_tts_model.rs b/src/common_queries/delete_tts_model.rs new file mode 100644 index 0000000000..47507def96 --- /dev/null +++ b/src/common_queries/delete_tts_model.rs @@ -0,0 +1,88 @@ +use crate::util::anyhow_result::AnyhowResult; +use sqlx::MySqlPool; + +pub async fn delete_tts_model_as_user( + tts_model_token: &str, + mysql_pool: &MySqlPool, +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE tts_models +SET + user_deleted_at = CURRENT_TIMESTAMP +WHERE + token = ? +LIMIT 1 + "#, + tts_model_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn delete_tts_model_as_mod( + tts_model_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool, +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE tts_models +SET + maybe_mod_user_token = ?, + mod_deleted_at = CURRENT_TIMESTAMP +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + tts_model_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn undelete_tts_model_as_user( + tts_model_token: &str, + mysql_pool: &MySqlPool, +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE tts_models +SET + user_deleted_at = NULL +WHERE + token = ? +LIMIT 1 + "#, + tts_model_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn undelete_tts_model_as_mod( + tts_model_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool, +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE tts_models +SET + maybe_mod_user_token = ?, + mod_deleted_at = NULL +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + tts_model_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index 8be1dadc91..0396d24fca 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -1,5 +1,6 @@ pub mod badge_granter; pub mod badges; +pub mod delete_tts_model; pub mod firehose_publisher; pub mod list_ip_bans; pub mod list_tts_inference_results; diff --git a/src/http_server/endpoints/tts/delete_tts_model.rs b/src/http_server/endpoints/tts/delete_tts_model.rs new file mode 100644 index 0000000000..f254615c77 --- /dev/null +++ b/src/http_server/endpoints/tts/delete_tts_model.rs @@ -0,0 +1,196 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::delete_tts_model::delete_tts_model_as_mod; +use crate::common_queries::delete_tts_model::delete_tts_model_as_user; +use crate::common_queries::delete_tts_model::undelete_tts_model_as_mod; +use crate::common_queries::delete_tts_model::undelete_tts_model_as_user; +use crate::common_queries::query_tts_model::select_tts_model_by_token; +use crate::common_queries::query_tts_result::select_tts_result_by_token; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct DeleteTtsModelPathInfo { + token: String, +} + +#[derive(Deserialize)] +pub struct DeleteTtsModelRequest { + set_delete: bool, + /// NB: this is only to disambiguate when a user is both a mod and an author. + as_mod: Option, +} + +#[derive(Debug, Display)] +pub enum DeleteTtsModelError { + BadInput(String), + NotAuthorized, + NotFound, + ServerError, +} + +impl ResponseError for DeleteTtsModelError { + fn status_code(&self) -> StatusCode { + match *self { + DeleteTtsModelError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteTtsModelError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteTtsModelError::NotFound => StatusCode::NOT_FOUND, + DeleteTtsModelError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + DeleteTtsModelError::BadInput(reason) => reason.to_string(), + DeleteTtsModelError::NotAuthorized => "unauthorized".to_string(), + DeleteTtsModelError::NotFound => "not found".to_string(), + DeleteTtsModelError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn delete_tts_model_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data> +) -> Result { + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + DeleteTtsModelError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(DeleteTtsModelError::NotAuthorized); + } + }; + + // NB: First permission check. + // Only mods should see deleted models (both user_* and mod_* deleted). + let is_mod = user_session.can_delete_other_users_tts_models; + + let model_query_result = select_tts_model_by_token( + &path.token, + is_mod, + &server_state.mysql_pool, + ).await; + + let tts_model = match model_query_result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(DeleteTtsModelError::ServerError); + } + Ok(None) => return Err(DeleteTtsModelError::NotFound), + Ok(Some(model)) => model, + }; + + // NB: Second set of permission checks + let is_author = &tts_model.creator_user_token == &user_session.user_token; + + if !is_author && !is_mod { + warn!("user is not allowed to delete model: {}", user_session.user_token); + return Err(DeleteTtsModelError::NotAuthorized); + } + + // NB: I can't imagine we need to store this. + // let ip_address = get_request_ip(&http_request); + + let as_mod = delete_as_mod(is_mod, is_author, request.as_mod); + + let query_result = if request.set_delete { + if as_mod { + delete_tts_model_as_mod( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } else { + delete_tts_model_as_user( + &path.token, + &server_state.mysql_pool + ).await + } + } else { + if as_mod { + undelete_tts_model_as_mod( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } else { + // NB: Technically only mods can see their own inference_results here + undelete_tts_model_as_user( + &path.token, + &server_state.mysql_pool + ).await + } + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Delete tts model DB error: {:?}", err); + return Err(DeleteTtsModelError::ServerError); + } + }; + + Ok(simple_json_success()) +} + +fn delete_as_mod(user_is_mod: bool, user_is_author: bool, as_mod_flag: Option) -> bool { + // NB: Explored this with a truth table. + let as_mod_flag_value = as_mod_flag.unwrap_or(false); + user_is_mod && !(user_is_author && as_mod_flag.is_some() && !as_mod_flag_value) +} + +#[cfg(test)] +mod tests { + use crate::http_server::endpoints::tts::delete_tts_model::delete_as_mod; + + #[test] + fn test_delete_as_mod() { + // Deleted as a user + assert!(!delete_as_mod(false, false, None)); + assert!(!delete_as_mod(false, true, None)); + assert!(!delete_as_mod(false, true, Some(false))); + assert!(!delete_as_mod(false, true, Some(true))); + + // Deleted as a mod + assert!(delete_as_mod(true, false, None)); + assert!(delete_as_mod(true, false, Some(false))); + assert!(delete_as_mod(true, false, Some(true))); + + // Moderator + Author, deleting as a user + assert!(!delete_as_mod(true, true, Some(false))); + + // Moderator + Author, deleting as a mod + assert!(delete_as_mod(true, true, None)); + assert!(delete_as_mod(true, true, Some(true))); + } +} diff --git a/src/http_server/endpoints/tts/mod.rs b/src/http_server/endpoints/tts/mod.rs index b7fbe06278..6c8b4f674f 100644 --- a/src/http_server/endpoints/tts/mod.rs +++ b/src/http_server/endpoints/tts/mod.rs @@ -1,3 +1,4 @@ +pub mod delete_tts_model; pub mod delete_tts_result; pub mod edit_tts_model; pub mod enqueue_infer_tts; diff --git a/src/main.rs b/src/main.rs index 91fb0d5f04..0fe6f75802 100644 --- a/src/main.rs +++ b/src/main.rs @@ -100,6 +100,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; +use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -412,6 +413,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_tts_model_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/model/{token}/delete") + .route(web::post().to(delete_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/model/{model_token}/count") .route(web::get().to(get_tts_model_use_count_handler)) From 103ef34b06ca70f9ecb5016a4d993bc86af2a793 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Jul 2021 01:32:23 -0400 Subject: [PATCH 0467/1042] huge database module refactor, move under unified directory tree --- src/clients/tts_inference_sidecar_client.rs | 3 +-- .../helpers}/boolean_converters.rs | 0 .../helpers}/enums.rs | 0 src/database/helpers/mod.rs | 3 +++ src/{common_queries => database/helpers}/tokens.rs | 0 .../mediators}/badge_granter.rs | 4 ++-- .../mediators}/firehose_publisher.rs | 2 +- src/database/mediators/mod.rs | 2 ++ src/database/mod.rs | 3 +++ src/{common_queries => database/queries}/badges.rs | 0 .../queries/create_session.rs} | 2 +- .../queries}/delete_tts_model.rs | 0 .../queries}/list_ip_bans.rs | 0 .../queries}/list_tts_inference_results.rs | 0 .../queries}/list_tts_models.rs | 2 +- .../queries}/list_user_badges.rs | 0 .../queries}/list_user_roles.rs | 2 +- .../queries}/list_w2l_inference_results.rs | 0 .../queries}/list_w2l_templates.rs | 2 +- src/{common_queries => database/queries}/mod.rs | 7 ++----- .../queries}/query_tts_model.rs | 4 ++-- .../queries}/query_tts_result.rs | 4 ++-- .../queries}/query_user_profile.rs | 6 +++--- .../queries}/query_w2l_result.rs | 4 ++-- .../queries}/query_w2l_template.rs | 4 ++-- src/database_helpers/mod.rs | 2 -- src/http_server/endpoints/events/list_events.rs | 5 ++--- .../endpoints/moderation/ip_bans/add_ip_ban.rs | 4 ++-- .../endpoints/moderation/ip_bans/delete_ip_ban.rs | 4 ++-- .../endpoints/moderation/ip_bans/get_ip_ban.rs | 2 +- .../endpoints/moderation/ip_bans/list_ip_bans.rs | 2 +- .../endpoints/moderation/user_roles/list_roles.rs | 6 +++--- .../endpoints/moderation/user_roles/list_staff.rs | 2 +- .../moderation/user_roles/set_user_role.rs | 10 +++++----- src/http_server/endpoints/tts/delete_tts_model.rs | 12 ++++++------ src/http_server/endpoints/tts/delete_tts_result.rs | 2 +- src/http_server/endpoints/tts/edit_tts_model.rs | 5 ++--- src/http_server/endpoints/tts/enqueue_infer_tts.rs | 4 ++-- .../endpoints/tts/enqueue_upload_tts_model.rs | 4 ++-- .../endpoints/tts/get_tts_inference_job_status.rs | 3 +-- src/http_server/endpoints/tts/get_tts_model.rs | 5 ++--- .../endpoints/tts/get_tts_model_use_count.rs | 1 - src/http_server/endpoints/tts/get_tts_result.rs | 7 +++---- .../tts/get_tts_upload_model_job_status.rs | 3 +-- src/http_server/endpoints/tts/list_tts_models.rs | 5 ++--- src/http_server/endpoints/users/create_account.rs | 14 +++++++------- src/http_server/endpoints/users/edit_profile.rs | 6 +++--- src/http_server/endpoints/users/get_profile.rs | 7 +++---- .../users/list_user_tts_inference_results.rs | 6 +++--- .../endpoints/users/list_user_tts_models.rs | 5 ++--- .../users/list_user_w2l_inference_results.rs | 9 ++++----- .../endpoints/users/list_user_w2l_templates.rs | 7 +++---- src/http_server/endpoints/users/login.rs | 5 ++--- src/http_server/endpoints/users/logout.rs | 3 +-- src/http_server/endpoints/users/session_info.rs | 3 +-- src/http_server/endpoints/w2l/delete_w2l_result.rs | 3 +-- .../endpoints/w2l/delete_w2l_template.rs | 10 +++++----- src/http_server/endpoints/w2l/edit_w2l_template.rs | 3 +-- src/http_server/endpoints/w2l/enqueue_infer_w2l.rs | 2 +- .../w2l/enqueue_infer_w2l_with_uploads.rs | 2 +- .../endpoints/w2l/enqueue_upload_w2l_template.rs | 4 ++-- .../endpoints/w2l/get_w2l_inference_job_status.rs | 3 +-- src/http_server/endpoints/w2l/get_w2l_result.rs | 7 +++---- src/http_server/endpoints/w2l/get_w2l_template.rs | 9 ++++----- .../endpoints/w2l/get_w2l_template_use_count.rs | 1 - .../w2l/get_w2l_upload_template_job_status.rs | 3 +-- .../endpoints/w2l/list_w2l_templates.rs | 7 +++---- .../endpoints/w2l/set_w2l_template_mod_approval.rs | 6 +++--- src/http_server/web_utils/ip_address.rs | 2 +- src/http_server/web_utils/session_checker.rs | 2 +- src/job_queries/tts_download_job_queries.rs | 4 ++-- src/job_queries/tts_inference_job_queries.rs | 2 +- src/job_queries/w2l_download_job_queries.rs | 2 +- src/job_queries/w2l_inference_job_queries.rs | 2 +- src/main.rs | 9 ++++----- src/server_state.rs | 4 ++-- src/threads/poll_ip_banlist_thread.rs | 2 +- 77 files changed, 137 insertions(+), 159 deletions(-) rename src/{database_helpers => database/helpers}/boolean_converters.rs (100%) rename src/{database_helpers => database/helpers}/enums.rs (100%) create mode 100644 src/database/helpers/mod.rs rename src/{common_queries => database/helpers}/tokens.rs (100%) rename src/{common_queries => database/mediators}/badge_granter.rs (98%) rename src/{common_queries => database/mediators}/firehose_publisher.rs (99%) create mode 100644 src/database/mediators/mod.rs create mode 100644 src/database/mod.rs rename src/{common_queries => database/queries}/badges.rs (100%) rename src/{common_queries/sessions.rs => database/queries/create_session.rs} (95%) rename src/{common_queries => database/queries}/delete_tts_model.rs (100%) rename src/{common_queries => database/queries}/list_ip_bans.rs (100%) rename src/{common_queries => database/queries}/list_tts_inference_results.rs (100%) rename src/{common_queries => database/queries}/list_tts_models.rs (98%) rename src/{common_queries => database/queries}/list_user_badges.rs (100%) rename src/{common_queries => database/queries}/list_user_roles.rs (98%) rename src/{common_queries => database/queries}/list_w2l_inference_results.rs (100%) rename src/{common_queries => database/queries}/list_w2l_templates.rs (98%) rename src/{common_queries => database/queries}/mod.rs (75%) rename src/{common_queries => database/queries}/query_tts_model.rs (97%) rename src/{common_queries => database/queries}/query_tts_result.rs (98%) rename src/{common_queries => database/queries}/query_user_profile.rs (98%) rename src/{common_queries => database/queries}/query_w2l_result.rs (98%) rename src/{common_queries => database/queries}/query_w2l_template.rs (97%) delete mode 100644 src/database_helpers/mod.rs diff --git a/src/clients/tts_inference_sidecar_client.rs b/src/clients/tts_inference_sidecar_client.rs index a302d081f2..2d762df964 100644 --- a/src/clients/tts_inference_sidecar_client.rs +++ b/src/clients/tts_inference_sidecar_client.rs @@ -1,11 +1,10 @@ -//use actix_web::client::Client; use actix_web::http::header; use anyhow::anyhow; use crate::util::anyhow_result::AnyhowResult; use hyper::client::Client; +use hyper::{Body, Request}; use log::info; use std::path::Path; -use hyper::{Body, Request}; pub struct TtsInferenceSidecarClient { hostname: String, diff --git a/src/database_helpers/boolean_converters.rs b/src/database/helpers/boolean_converters.rs similarity index 100% rename from src/database_helpers/boolean_converters.rs rename to src/database/helpers/boolean_converters.rs diff --git a/src/database_helpers/enums.rs b/src/database/helpers/enums.rs similarity index 100% rename from src/database_helpers/enums.rs rename to src/database/helpers/enums.rs diff --git a/src/database/helpers/mod.rs b/src/database/helpers/mod.rs new file mode 100644 index 0000000000..81606ff216 --- /dev/null +++ b/src/database/helpers/mod.rs @@ -0,0 +1,3 @@ +pub mod boolean_converters; +pub mod enums; +pub mod tokens; \ No newline at end of file diff --git a/src/common_queries/tokens.rs b/src/database/helpers/tokens.rs similarity index 100% rename from src/common_queries/tokens.rs rename to src/database/helpers/tokens.rs diff --git a/src/common_queries/badge_granter.rs b/src/database/mediators/badge_granter.rs similarity index 98% rename from src/common_queries/badge_granter.rs rename to src/database/mediators/badge_granter.rs index 7c19b458eb..1d8bd4734b 100644 --- a/src/common_queries/badge_granter.rs +++ b/src/database/mediators/badge_granter.rs @@ -1,5 +1,6 @@ use anyhow::anyhow; -use crate::common_queries::tokens::Tokens; +use crate::database::helpers::tokens::Tokens; +use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn,info}; @@ -7,7 +8,6 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDone; use sqlx::{MySqlPool}; use std::sync::Arc; -use crate::common_queries::firehose_publisher::FirehosePublisher; #[derive(Debug, Clone, Copy)] pub enum UserBadge { diff --git a/src/common_queries/firehose_publisher.rs b/src/database/mediators/firehose_publisher.rs similarity index 99% rename from src/common_queries/firehose_publisher.rs rename to src/database/mediators/firehose_publisher.rs index 512e2d7d05..850f4a8a73 100644 --- a/src/common_queries/firehose_publisher.rs +++ b/src/database/mediators/firehose_publisher.rs @@ -1,5 +1,5 @@ use anyhow::anyhow; -use crate::common_queries::tokens::Tokens; +use crate::database::helpers::tokens::Tokens; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn,info}; diff --git a/src/database/mediators/mod.rs b/src/database/mediators/mod.rs new file mode 100644 index 0000000000..5cc9617e88 --- /dev/null +++ b/src/database/mediators/mod.rs @@ -0,0 +1,2 @@ +pub mod badge_granter; +pub mod firehose_publisher; \ No newline at end of file diff --git a/src/database/mod.rs b/src/database/mod.rs new file mode 100644 index 0000000000..d160ea03e0 --- /dev/null +++ b/src/database/mod.rs @@ -0,0 +1,3 @@ +pub mod helpers; +pub mod mediators; +pub mod queries; \ No newline at end of file diff --git a/src/common_queries/badges.rs b/src/database/queries/badges.rs similarity index 100% rename from src/common_queries/badges.rs rename to src/database/queries/badges.rs diff --git a/src/common_queries/sessions.rs b/src/database/queries/create_session.rs similarity index 95% rename from src/common_queries/sessions.rs rename to src/database/queries/create_session.rs index 991338c84f..14ea5d8446 100644 --- a/src/common_queries/sessions.rs +++ b/src/database/queries/create_session.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use crate::AnyhowResult; -use crate::common_queries::tokens::Tokens; +use crate::database::helpers::tokens::Tokens; use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{info, warn}; diff --git a/src/common_queries/delete_tts_model.rs b/src/database/queries/delete_tts_model.rs similarity index 100% rename from src/common_queries/delete_tts_model.rs rename to src/database/queries/delete_tts_model.rs diff --git a/src/common_queries/list_ip_bans.rs b/src/database/queries/list_ip_bans.rs similarity index 100% rename from src/common_queries/list_ip_bans.rs rename to src/database/queries/list_ip_bans.rs diff --git a/src/common_queries/list_tts_inference_results.rs b/src/database/queries/list_tts_inference_results.rs similarity index 100% rename from src/common_queries/list_tts_inference_results.rs rename to src/database/queries/list_tts_inference_results.rs diff --git a/src/common_queries/list_tts_models.rs b/src/database/queries/list_tts_models.rs similarity index 98% rename from src/common_queries/list_tts_models.rs rename to src/database/queries/list_tts_models.rs index a9468463d7..420a632c5a 100644 --- a/src/common_queries/list_tts_models.rs +++ b/src/database/queries/list_tts_models.rs @@ -1,9 +1,9 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use crate::database::helpers::boolean_converters::i8_to_bool; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; -use crate::database_helpers::boolean_converters::i8_to_bool; #[derive(Serialize)] pub struct TtsModelRecordForList { diff --git a/src/common_queries/list_user_badges.rs b/src/database/queries/list_user_badges.rs similarity index 100% rename from src/common_queries/list_user_badges.rs rename to src/database/queries/list_user_badges.rs diff --git a/src/common_queries/list_user_roles.rs b/src/database/queries/list_user_roles.rs similarity index 98% rename from src/common_queries/list_user_roles.rs rename to src/database/queries/list_user_roles.rs index a606f8b9fa..da4619f772 100644 --- a/src/common_queries/list_user_roles.rs +++ b/src/database/queries/list_user_roles.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use crate::database_helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; +use crate::database::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/database/queries/list_w2l_inference_results.rs similarity index 100% rename from src/common_queries/list_w2l_inference_results.rs rename to src/database/queries/list_w2l_inference_results.rs diff --git a/src/common_queries/list_w2l_templates.rs b/src/database/queries/list_w2l_templates.rs similarity index 98% rename from src/common_queries/list_w2l_templates.rs rename to src/database/queries/list_w2l_templates.rs index e294aa8f47..c403b8c244 100644 --- a/src/common_queries/list_w2l_templates.rs +++ b/src/database/queries/list_w2l_templates.rs @@ -1,9 +1,9 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; -use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; #[derive(Serialize)] pub struct W2lTemplateRecordForList { diff --git a/src/common_queries/mod.rs b/src/database/queries/mod.rs similarity index 75% rename from src/common_queries/mod.rs rename to src/database/queries/mod.rs index 0396d24fca..a74deffd40 100644 --- a/src/common_queries/mod.rs +++ b/src/database/queries/mod.rs @@ -1,7 +1,6 @@ -pub mod badge_granter; pub mod badges; +pub mod create_session; pub mod delete_tts_model; -pub mod firehose_publisher; pub mod list_ip_bans; pub mod list_tts_inference_results; pub mod list_tts_models; @@ -13,6 +12,4 @@ pub mod query_tts_model; pub mod query_tts_result; pub mod query_user_profile; pub mod query_w2l_result; -pub mod query_w2l_template; -pub mod sessions; -pub mod tokens; \ No newline at end of file +pub mod query_w2l_template; \ No newline at end of file diff --git a/src/common_queries/query_tts_model.rs b/src/database/queries/query_tts_model.rs similarity index 97% rename from src/common_queries/query_tts_model.rs rename to src/database/queries/query_tts_model.rs index 32db56d7bf..5eb2ab758d 100644 --- a/src/common_queries/query_tts_model.rs +++ b/src/database/queries/query_tts_model.rs @@ -1,8 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database_helpers::boolean_converters::i8_to_bool; -use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; +use crate::database::helpers::boolean_converters::i8_to_bool; +use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/src/common_queries/query_tts_result.rs b/src/database/queries/query_tts_result.rs similarity index 98% rename from src/common_queries/query_tts_result.rs rename to src/database/queries/query_tts_result.rs index e6636a2d14..e1a82097b3 100644 --- a/src/common_queries/query_tts_result.rs +++ b/src/database/queries/query_tts_result.rs @@ -1,8 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database_helpers::boolean_converters::i8_to_bool; -use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; +use crate::database::helpers::boolean_converters::i8_to_bool; +use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/src/common_queries/query_user_profile.rs b/src/database/queries/query_user_profile.rs similarity index 98% rename from src/common_queries/query_user_profile.rs rename to src/database/queries/query_user_profile.rs index 530b302158..b848460a56 100644 --- a/src/common_queries/query_user_profile.rs +++ b/src/database/queries/query_user_profile.rs @@ -1,16 +1,16 @@ use anyhow::anyhow; +use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::database::helpers::boolean_converters::i8_to_bool; use derive_more::{Display, Error}; use log::{info, warn, log}; +use md5::{Md5, Digest}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use chrono::{DateTime, Utc}; -use md5::{Md5, Digest}; -use crate::database_helpers::boolean_converters::i8_to_bool; // TODO: This duplicates the get_profile_handler. diff --git a/src/common_queries/query_w2l_result.rs b/src/database/queries/query_w2l_result.rs similarity index 98% rename from src/common_queries/query_w2l_result.rs rename to src/database/queries/query_w2l_result.rs index 8183d0d8f8..c3497301fd 100644 --- a/src/common_queries/query_w2l_result.rs +++ b/src/database/queries/query_w2l_result.rs @@ -1,8 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database_helpers::boolean_converters::i8_to_bool; -use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; +use crate::database::helpers::boolean_converters::i8_to_bool; +use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/src/common_queries/query_w2l_template.rs b/src/database/queries/query_w2l_template.rs similarity index 97% rename from src/common_queries/query_w2l_template.rs rename to src/database/queries/query_w2l_template.rs index f544bc5d9b..cba7f16fde 100644 --- a/src/common_queries/query_w2l_template.rs +++ b/src/database/queries/query_w2l_template.rs @@ -1,8 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; -use crate::database_helpers::boolean_converters::i8_to_bool; +use crate::database::helpers::boolean_converters::i8_to_bool; +use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/src/database_helpers/mod.rs b/src/database_helpers/mod.rs deleted file mode 100644 index b5f78fd29f..0000000000 --- a/src/database_helpers/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod boolean_converters; -pub mod enums; \ No newline at end of file diff --git a/src/http_server/endpoints/events/list_events.rs b/src/http_server/endpoints/events/list_events.rs index 3f1bdad007..847b1fff8b 100644 --- a/src/http_server/endpoints/events/list_events.rs +++ b/src/http_server/endpoints/events/list_events.rs @@ -7,11 +7,11 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::list_w2l_templates::{list_w2l_templates, W2lTemplateRecordForList}; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::list_w2l_templates::{list_w2l_templates, W2lTemplateRecordForList}; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -25,7 +25,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Serialize)] pub struct EventRecord { diff --git a/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs b/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs index 1788d140e3..99ea56639c 100644 --- a/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs +++ b/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs @@ -5,13 +5,14 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::validations::check_for_slurs::contains_slurs; +use crate::validations::ip_addresses::validate_moderator_provided_ip_address; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; @@ -22,7 +23,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::validations::ip_addresses::validate_moderator_provided_ip_address; #[derive(Deserialize)] pub struct AddIpBanRequest { diff --git a/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs b/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs index 1aef770a33..348a922964 100644 --- a/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs +++ b/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs @@ -4,8 +4,9 @@ use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; +use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -22,7 +23,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use actix_web::web::Path; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs b/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs index 9ed864f63e..5ba8b6bf2e 100644 --- a/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs +++ b/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs @@ -7,7 +7,7 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs index 59cabe5bfd..ca2cebdca1 100644 --- a/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs +++ b/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs @@ -7,7 +7,7 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/src/http_server/endpoints/moderation/user_roles/list_roles.rs b/src/http_server/endpoints/moderation/user_roles/list_roles.rs index 836f0b91bc..81c1262c5b 100644 --- a/src/http_server/endpoints/moderation/user_roles/list_roles.rs +++ b/src/http_server/endpoints/moderation/user_roles/list_roles.rs @@ -7,9 +7,9 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::common_queries::list_user_roles::UserRoleForList; -use crate::common_queries::list_user_roles::list_user_roles; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::queries::list_user_roles::UserRoleForList; +use crate::database::queries::list_user_roles::list_user_roles; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/src/http_server/endpoints/moderation/user_roles/list_staff.rs b/src/http_server/endpoints/moderation/user_roles/list_staff.rs index 3c21a63977..9a43ba5afd 100644 --- a/src/http_server/endpoints/moderation/user_roles/list_staff.rs +++ b/src/http_server/endpoints/moderation/user_roles/list_staff.rs @@ -7,7 +7,7 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/src/http_server/endpoints/moderation/user_roles/set_user_role.rs b/src/http_server/endpoints/moderation/user_roles/set_user_role.rs index d07942f284..0fdf59f97f 100644 --- a/src/http_server/endpoints/moderation/user_roles/set_user_role.rs +++ b/src/http_server/endpoints/moderation/user_roles/set_user_role.rs @@ -6,10 +6,12 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::query_user_profile::select_user_profile_by_username; -use crate::common_queries::list_user_roles::list_user_roles; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::queries::list_user_roles::list_user_roles; +use crate::database::queries::query_user_profile::select_user_profile_by_username; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; use crate::validations::model_uploads::validate_model_title; @@ -22,8 +24,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; -use crate::http_server::web_utils::response_success_helpers::simple_json_success; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/tts/delete_tts_model.rs b/src/http_server/endpoints/tts/delete_tts_model.rs index f254615c77..4e3d77d9d3 100644 --- a/src/http_server/endpoints/tts/delete_tts_model.rs +++ b/src/http_server/endpoints/tts/delete_tts_model.rs @@ -6,12 +6,12 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::delete_tts_model::delete_tts_model_as_mod; -use crate::common_queries::delete_tts_model::delete_tts_model_as_user; -use crate::common_queries::delete_tts_model::undelete_tts_model_as_mod; -use crate::common_queries::delete_tts_model::undelete_tts_model_as_user; -use crate::common_queries::query_tts_model::select_tts_model_by_token; -use crate::common_queries::query_tts_result::select_tts_result_by_token; +use crate::database::queries::delete_tts_model::delete_tts_model_as_mod; +use crate::database::queries::delete_tts_model::delete_tts_model_as_user; +use crate::database::queries::delete_tts_model::undelete_tts_model_as_mod; +use crate::database::queries::delete_tts_model::undelete_tts_model_as_user; +use crate::database::queries::query_tts_model::select_tts_model_by_token; +use crate::database::queries::query_tts_result::select_tts_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/src/http_server/endpoints/tts/delete_tts_result.rs b/src/http_server/endpoints/tts/delete_tts_result.rs index 3dc15cf2e7..6111a0665f 100644 --- a/src/http_server/endpoints/tts/delete_tts_result.rs +++ b/src/http_server/endpoints/tts/delete_tts_result.rs @@ -6,7 +6,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::query_tts_result::select_tts_result_by_token; +use crate::database::queries::query_tts_result::select_tts_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/src/http_server/endpoints/tts/edit_tts_model.rs b/src/http_server/endpoints/tts/edit_tts_model.rs index 47c733a147..68029a2f5d 100644 --- a/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/src/http_server/endpoints/tts/edit_tts_model.rs @@ -6,8 +6,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::query_tts_model::select_tts_model_by_token; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::query_tts_model::select_tts_model_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -20,11 +19,11 @@ use crate::validations::check_for_slurs::contains_slurs; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; +use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use sqlx::MySqlPool; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 07297108a5..ea221ff696 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -5,8 +5,8 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::tokens::Tokens; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index 6614dcfbb7..0747e85529 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -5,8 +5,8 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::tokens::Tokens; -use crate::database_helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; +use crate::database::helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; +use crate::database::helpers::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; diff --git a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index 33de8929c4..c74a66dc34 100644 --- a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -8,10 +8,10 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -25,7 +25,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/tts/get_tts_model.rs b/src/http_server/endpoints/tts/get_tts_model.rs index bbaf138d5b..f20effb37e 100644 --- a/src/http_server/endpoints/tts/get_tts_model.rs +++ b/src/http_server/endpoints/tts/get_tts_model.rs @@ -8,9 +8,8 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::query_tts_model::TtsModelRecordForResponse; -use crate::common_queries::query_tts_model::select_tts_model_by_token; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::query_tts_model::TtsModelRecordForResponse; +use crate::database::queries::query_tts_model::select_tts_model_by_token; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; diff --git a/src/http_server/endpoints/tts/get_tts_model_use_count.rs b/src/http_server/endpoints/tts/get_tts_model_use_count.rs index 1610c235b4..00d5109827 100644 --- a/src/http_server/endpoints/tts/get_tts_model_use_count.rs +++ b/src/http_server/endpoints/tts/get_tts_model_use_count.rs @@ -8,7 +8,6 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; diff --git a/src/http_server/endpoints/tts/get_tts_result.rs b/src/http_server/endpoints/tts/get_tts_result.rs index fdf6597276..7124f7c737 100644 --- a/src/http_server/endpoints/tts/get_tts_result.rs +++ b/src/http_server/endpoints/tts/get_tts_result.rs @@ -8,10 +8,10 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::query_tts_result::TtsResultRecordForResponse; -use crate::common_queries::query_tts_result::select_tts_result_by_token; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::query_tts_result::TtsResultRecordForResponse; +use crate::database::queries::query_tts_result::select_tts_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -25,7 +25,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs index cbde835a1a..877698df2b 100644 --- a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs @@ -8,10 +8,10 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -25,7 +25,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/tts/list_tts_models.rs b/src/http_server/endpoints/tts/list_tts_models.rs index 9e947487e6..4ad973581d 100644 --- a/src/http_server/endpoints/tts/list_tts_models.rs +++ b/src/http_server/endpoints/tts/list_tts_models.rs @@ -5,11 +5,12 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -23,8 +24,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use chrono::{DateTime, Utc}; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Serialize)] pub struct TtsModelRecordForResponse { diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index e077105def..6729cfe7d0 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -5,13 +5,19 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::helpers::tokens::Tokens; +use crate::database::queries::create_session::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; +use crate::validations::username_reservations::is_reserved_username; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; @@ -19,12 +25,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::util::email_to_gravatar::email_to_gravatar; -use crate::validations::username_reservations::is_reserved_username; -use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; -use crate::validations::check_for_slurs::contains_slurs; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; -use crate::common_queries::tokens::Tokens; const NEW_USER_ROLE: &'static str = "user"; diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs index 2ece9f5c35..b0b9e55a0a 100644 --- a/src/http_server/endpoints/users/edit_profile.rs +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -6,8 +6,10 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::create_session::create_session_for_user; +use crate::database::queries::query_user_profile::select_user_profile_by_username; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; @@ -30,8 +32,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::common_queries::query_user_profile::select_user_profile_by_username; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index 318ee98f62..4981f8c812 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -9,10 +9,9 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::list_user_badges::UserBadgeForList; -use crate::common_queries::list_user_badges::list_user_badges; -use crate::common_queries::sessions::create_session_for_user; -use crate::database_helpers::boolean_converters::i8_to_bool; +use crate::database::helpers::boolean_converters::i8_to_bool; +use crate::database::queries::list_user_badges::UserBadgeForList; +use crate::database::queries::list_user_badges::list_user_badges; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; diff --git a/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/src/http_server/endpoints/users/list_user_tts_inference_results.rs index cfc3de9c07..4086f10b20 100644 --- a/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -8,9 +8,9 @@ use actix_web::web::{Path, Query}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::list_tts_inference_results::ListTtsResultsQueryBuilder; -use crate::common_queries::list_tts_inference_results::TtsInferenceRecordForList; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::create_session::create_session_for_user; +use crate::database::queries::list_tts_inference_results::ListTtsResultsQueryBuilder; +use crate::database::queries::list_tts_inference_results::TtsInferenceRecordForList; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; diff --git a/src/http_server/endpoints/users/list_user_tts_models.rs b/src/http_server/endpoints/users/list_user_tts_models.rs index 3024060668..17cf9544e0 100644 --- a/src/http_server/endpoints/users/list_user_tts_models.rs +++ b/src/http_server/endpoints/users/list_user_tts_models.rs @@ -8,11 +8,11 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::list_tts_models::{TtsModelRecordForList, list_tts_models}; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::list_tts_models::{TtsModelRecordForList, list_tts_models}; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -26,7 +26,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs index d8adb03969..5413459842 100644 --- a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs @@ -4,15 +4,16 @@ use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; +use actix_web::web::{Path, Query}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::list_w2l_inference_results::W2lInferenceRecordForList; -use crate::common_queries::list_w2l_inference_results::ListW2lResultsQueryBuilder; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::list_w2l_inference_results::ListW2lResultsQueryBuilder; +use crate::database::queries::list_w2l_inference_results::W2lInferenceRecordForList; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -26,8 +27,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use actix_web::web::{Path, Query}; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/users/list_user_w2l_templates.rs b/src/http_server/endpoints/users/list_user_w2l_templates.rs index 30b13dd5c0..f4f0e78931 100644 --- a/src/http_server/endpoints/users/list_user_w2l_templates.rs +++ b/src/http_server/endpoints/users/list_user_w2l_templates.rs @@ -4,14 +4,15 @@ use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; +use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::list_w2l_templates::{W2lTemplateRecordForList, list_w2l_templates}; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::list_w2l_templates::{W2lTemplateRecordForList, list_w2l_templates}; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -25,8 +26,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use actix_web::web::Path; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/users/login.rs b/src/http_server/endpoints/users/login.rs index 71d8258a93..848e551dbb 100644 --- a/src/http_server/endpoints/users/login.rs +++ b/src/http_server/endpoints/users/login.rs @@ -6,11 +6,11 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::create_session::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; @@ -21,7 +21,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct LoginRequest { diff --git a/src/http_server/endpoints/users/logout.rs b/src/http_server/endpoints/users/logout.rs index 32ee04d7de..18fb79d47f 100644 --- a/src/http_server/endpoints/users/logout.rs +++ b/src/http_server/endpoints/users/logout.rs @@ -6,10 +6,10 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; @@ -22,7 +22,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Serialize)] pub struct LogoutSuccessResponse { diff --git a/src/http_server/endpoints/users/session_info.rs b/src/http_server/endpoints/users/session_info.rs index 76697f4678..712cf9a46b 100644 --- a/src/http_server/endpoints/users/session_info.rs +++ b/src/http_server/endpoints/users/session_info.rs @@ -6,10 +6,10 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -23,7 +23,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Serialize)] pub struct UserInfo { diff --git a/src/http_server/endpoints/w2l/delete_w2l_result.rs b/src/http_server/endpoints/w2l/delete_w2l_result.rs index 7fa7b9216d..7315b7f14a 100644 --- a/src/http_server/endpoints/w2l/delete_w2l_result.rs +++ b/src/http_server/endpoints/w2l/delete_w2l_result.rs @@ -6,12 +6,11 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::query_w2l_result::select_w2l_result_by_token; +use crate::database::queries::query_w2l_result::select_w2l_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; -use crate::util::random_crockford_token::random_crockford_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/src/http_server/endpoints/w2l/delete_w2l_template.rs b/src/http_server/endpoints/w2l/delete_w2l_template.rs index 5033aa2193..b111eb2198 100644 --- a/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -6,9 +6,11 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::query_w2l_template::select_w2l_template_by_token; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; use crate::validations::model_uploads::validate_model_title; @@ -17,13 +19,11 @@ use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; +use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use sqlx::MySqlPool; -use crate::http_server::web_utils::response_success_helpers::simple_json_success; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/w2l/edit_w2l_template.rs b/src/http_server/endpoints/w2l/edit_w2l_template.rs index 86ebc0fbb2..a71e9ad0f8 100644 --- a/src/http_server/endpoints/w2l/edit_w2l_template.rs +++ b/src/http_server/endpoints/w2l/edit_w2l_template.rs @@ -6,8 +6,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::query_w2l_template::select_w2l_template_by_token; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index d2da24e8a9..b1a7a7d8a8 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -5,7 +5,7 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 460b19c280..53161eb7a9 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -7,7 +7,7 @@ use actix_web::http::header::ContentDisposition; use actix_web::web::{Data, Json, BytesMut}; use actix_web::{middleware, web, App, Error, HttpResponse, HttpServer, HttpRequest, Either, ResponseError}; use anyhow::anyhow; -use crate::common_queries::tokens::Tokens; +use crate::database::helpers::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::read_multipart_field_bytes::checked_read_multipart_bytes; use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_boolean; diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index ddd45089c6..9d3a9b0cfb 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -5,8 +5,8 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::tokens::Tokens; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; diff --git a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs index 6e0ef10c9e..3ba632c06e 100644 --- a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs +++ b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs @@ -8,10 +8,10 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -25,7 +25,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/w2l/get_w2l_result.rs b/src/http_server/endpoints/w2l/get_w2l_result.rs index 80797ec7a4..07d4532219 100644 --- a/src/http_server/endpoints/w2l/get_w2l_result.rs +++ b/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -8,12 +8,12 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::query_w2l_result::W2lResultRecordForResponse; -use crate::common_queries::query_w2l_result::select_w2l_result_by_token; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::query_w2l_result::W2lResultRecordForResponse; +use crate::database::queries::query_w2l_result::select_w2l_result_by_token; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -27,7 +27,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index 03e04703ec..66819823e1 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -8,11 +8,11 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::query_w2l_template::W2lTemplateRecordForResponse; -use crate::common_queries::query_w2l_template::select_w2l_template_by_token; -use crate::common_queries::sessions::create_session_for_user; -use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; +use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; +use crate::database::queries::query_w2l_template::W2lTemplateRecordForResponse; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; @@ -25,7 +25,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs b/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs index d5d22188e1..48ea0387fd 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs @@ -8,7 +8,6 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; diff --git a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs index cc3b996759..0674d2ccdc 100644 --- a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs +++ b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs @@ -8,10 +8,10 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -25,7 +25,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/w2l/list_w2l_templates.rs b/src/http_server/endpoints/w2l/list_w2l_templates.rs index 197a8a12d4..f7621f464a 100644 --- a/src/http_server/endpoints/w2l/list_w2l_templates.rs +++ b/src/http_server/endpoints/w2l/list_w2l_templates.rs @@ -5,11 +5,13 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::common_queries::sessions::create_session_for_user; +use crate::database::queries::list_w2l_templates::{list_w2l_templates, W2lTemplateRecordForList}; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -23,9 +25,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use chrono::{DateTime, Utc}; -use crate::common_queries::list_w2l_templates::{list_w2l_templates, W2lTemplateRecordForList}; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Serialize)] pub struct ListW2lTemplatesSuccessResponse { diff --git a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs index 46cf707786..d2612a1d6d 100644 --- a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs +++ b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs @@ -6,9 +6,10 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::common_queries::query_w2l_template::select_w2l_template_by_token; -use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; use crate::validations::model_uploads::validate_model_title; @@ -21,7 +22,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/web_utils/ip_address.rs b/src/http_server/web_utils/ip_address.rs index 5372c5f26e..93d82ce294 100644 --- a/src/http_server/web_utils/ip_address.rs +++ b/src/http_server/web_utils/ip_address.rs @@ -1,7 +1,7 @@ use actix_web::HttpRequest; +use actix_web::dev::ServiceRequest; use actix_web::http::HeaderName; use log::{info}; -use actix_web::dev::ServiceRequest; pub fn get_request_ip(request: &HttpRequest) -> String { match request.headers().get(HeaderName::from_static("x-voder-proxy-for")) { diff --git a/src/http_server/web_utils/session_checker.rs b/src/http_server/web_utils/session_checker.rs index 4d4f2e6a41..5488b6b2b9 100644 --- a/src/http_server/web_utils/session_checker.rs +++ b/src/http_server/web_utils/session_checker.rs @@ -1,7 +1,7 @@ use actix_web::HttpRequest; use anyhow::anyhow; use crate::AnyhowResult; -use crate::database_helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; +use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; use crate::http_server::web_utils::cookie_manager::CookieManager; use log::{info, warn}; use sqlx::MySqlPool; diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index bdf6c950da..712b6c4a57 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -4,12 +4,12 @@ use anyhow::anyhow; use chrono::Utc; +use crate::database::helpers::tokens::Tokens; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; -use sqlx::MySqlPool; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use sqlx::MySqlPool; use std::path::Path; -use crate::common_queries::tokens::Tokens; /// table: tts_model_upload_jobs #[derive(Debug)] diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 7cd1f12fe2..040ca6e8a5 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -4,12 +4,12 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; +use crate::database::helpers::tokens::Tokens; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn, info}; use sqlx::{MySqlPool, Transaction, MySql}; use std::path::Path; -use crate::common_queries::tokens::Tokens; /// table: tts_inference_jobs #[derive(Debug)] diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index cfe9940180..6f8d8705d9 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -4,7 +4,7 @@ use anyhow::anyhow; use chrono::Utc; -use crate::common_queries::tokens::Tokens; +use crate::database::helpers::tokens::Tokens; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index 0d7987a631..33702c7a83 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -4,13 +4,13 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; +use crate::database::helpers::tokens::Tokens; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn, info}; use sqlx::MySqlPool; use std::path::Path; -use crate::common_queries::tokens::Tokens; /// table: w2l_inference_jobs #[derive(Debug)] diff --git a/src/main.rs b/src/main.rs index 0fe6f75802..89f16e8264 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,8 +17,7 @@ pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../db/reserved_user pub const BANNED_SLURS : &'static str = include_str!("../db/banned_slurs.txt"); pub mod clients; -pub mod common_queries; -pub mod database_helpers; +pub mod database; pub mod http_server; pub mod server_state; pub mod shared_constants; @@ -33,8 +32,8 @@ use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; -use crate::common_queries::badge_granter::BadgeGranter; -use crate::common_queries::firehose_publisher::FirehosePublisher; +use crate::database::mediators::badge_granter::BadgeGranter; +use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::default_route_404::default_route_404; use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::misc::enable_alpha::enable_alpha; @@ -46,6 +45,7 @@ use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler; use crate::http_server::endpoints::moderation::user_roles::set_user_role::set_user_role_handler; use crate::http_server::endpoints::root_index::get_root_index; +use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handler; use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; use crate::http_server::endpoints::tts::enqueue_infer_tts::infer_tts_handler; @@ -100,7 +100,6 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; -use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO diff --git a/src/server_state.rs b/src/server_state.rs index c40e7bd2c9..19301e9ef3 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,5 +1,5 @@ -use crate::common_queries::badge_granter::BadgeGranter; -use crate::common_queries::firehose_publisher::FirehosePublisher; +use crate::database::mediators::badge_granter::BadgeGranter; +use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::threads::ip_banlist_set::IpBanlistSet; diff --git a/src/threads/poll_ip_banlist_thread.rs b/src/threads/poll_ip_banlist_thread.rs index 81ff51d471..d71c04ae8b 100644 --- a/src/threads/poll_ip_banlist_thread.rs +++ b/src/threads/poll_ip_banlist_thread.rs @@ -1,5 +1,5 @@ use anyhow::Error; -use crate::common_queries::list_ip_bans::list_ip_bans; +use crate::database::queries::list_ip_bans::list_ip_bans; use crate::threads::ip_banlist_set::IpBanlistSet; use log::info; use log::warn; From db2b69fe538812c1f25b25df762a4e9dde115fb0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Jul 2021 01:37:38 -0400 Subject: [PATCH 0468/1042] fixup jobs from refactor --- src/tts_download_job.rs | 7 +++---- src/tts_inference_job.rs | 5 ++--- src/w2l_download_job.rs | 7 +++---- src/w2l_inference_job.rs | 5 ++--- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index e5bdf69be3..b3194aac7f 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -11,8 +11,7 @@ #[macro_use] extern crate serde_derive; pub mod common_env; -pub mod common_queries; -pub mod database_helpers; +pub mod database; pub mod job_queries; pub mod script_execution; pub mod shared_constants; @@ -21,8 +20,8 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; use crate::common_env::CommonEnv; -use crate::common_queries::badge_granter::BadgeGranter; -use crate::common_queries::firehose_publisher::FirehosePublisher; +use crate::database::mediators::badge_granter::BadgeGranter; +use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; use crate::job_queries::tts_download_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::tts_download_job_queries::insert_tts_model; diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index a1f330fa50..6bcfd18f4d 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -12,8 +12,7 @@ pub mod clients; pub mod common_env; -pub mod common_queries; -pub mod database_helpers; +pub mod database; pub mod job_queries; pub mod script_execution; pub mod shared_constants; @@ -23,7 +22,7 @@ use anyhow::{anyhow, Error}; use chrono::{Utc, DateTime, TimeZone}; use crate::clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; use crate::common_env::CommonEnv; -use crate::common_queries::firehose_publisher::FirehosePublisher; +use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_inference_job_queries::get_tts_model_by_token; use crate::job_queries::tts_inference_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::tts_inference_job_queries::insert_tts_result; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index e18dda4e37..bcc2028fb8 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -11,8 +11,7 @@ #[macro_use] extern crate serde_derive; pub mod common_env; -pub mod common_queries; -pub mod database_helpers; +pub mod database; pub mod job_queries; pub mod script_execution; pub mod shared_constants; @@ -21,8 +20,8 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; use crate::common_env::CommonEnv; -use crate::common_queries::badge_granter::BadgeGranter; -use crate::common_queries::firehose_publisher::FirehosePublisher; +use crate::database::mediators::badge_granter::BadgeGranter; +use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; use crate::job_queries::w2l_download_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::w2l_download_job_queries::insert_w2l_template; diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index c67e250471..55ed7d4a70 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -11,8 +11,7 @@ #[macro_use] extern crate serde_derive; pub mod common_env; -pub mod common_queries; -pub mod database_helpers; +pub mod database; pub mod job_queries; pub mod script_execution; pub mod shared_constants; @@ -21,7 +20,7 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; use crate::common_env::CommonEnv; -use crate::common_queries::firehose_publisher::FirehosePublisher; +use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; use crate::job_queries::w2l_inference_job_queries::grab_job_lock_and_mark_pending; From b8e4dadaf3659c96cce163804b1b7470473756b6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Jul 2021 01:38:13 -0400 Subject: [PATCH 0469/1042] re-codegen after refactor --- sqlx-data.json | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/sqlx-data.json b/sqlx-data.json index b85ca0649f..10a14ec23c 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1288,6 +1288,16 @@ "nullable": [] } }, + "2b80162b8957ac044f245af93d678aee976535d58fe0de0b39087323aa720792": { + "query": "\nUPDATE tts_models\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "314904656a333caa75a8644e73f4d338b7bd6f68c7ea9e37b5bc7f7b257bf34d": { "query": "\nSELECT 1 as does_exist\nFROM user_badges\nWHERE\n user_token = ?\nAND\n badge_slug = ?\nLIMIT 1\n ", "describe": { @@ -2841,6 +2851,16 @@ "nullable": [] } }, + "6aaa7c04e6984434bd2feb6f42a9014cd45e995b94e4d5e29f2d502f09136691": { + "query": "\nUPDATE tts_models\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "6b66db376952ff4cfcea6ffe1f9ecd4462722843a3d19fcd4df16f0fb993decc": { "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -5560,6 +5580,16 @@ ] } }, + "cd8541805f58293d33c30e108dba73a385407dd5546c5b7380da7397b4f6de34": { + "query": "\nUPDATE tts_models\nSET\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, "cf0b037e789ee09022b25f3381b0263c5cc6b59f3f328de4aa211bdb23a62629": { "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -7329,6 +7359,16 @@ ] } }, + "ebd41740a814e311e36a0e0b3ea7ecc829687c7d14ba854c81efa2404bdd77d6": { + "query": "\nUPDATE tts_models\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, "ed67238afe48c159443d215cd1fe2f0264ad840768f789d068527ae542b08194": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { From f24cdb6d38974ad489e701e80a67496ac6ca6e53 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Jul 2021 13:08:17 -0400 Subject: [PATCH 0470/1042] store delete model IP address --- src/database/queries/delete_tts_model.rs | 6 ++++++ src/http_server/endpoints/tts/delete_tts_model.rs | 7 ++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/database/queries/delete_tts_model.rs b/src/database/queries/delete_tts_model.rs index 47507def96..7ec6622aed 100644 --- a/src/database/queries/delete_tts_model.rs +++ b/src/database/queries/delete_tts_model.rs @@ -3,17 +3,20 @@ use sqlx::MySqlPool; pub async fn delete_tts_model_as_user( tts_model_token: &str, + creator_ip_address: &str, mysql_pool: &MySqlPool, ) -> AnyhowResult<()> { let _r = sqlx::query!( r#" UPDATE tts_models SET + creator_ip_address_last_update = ?, user_deleted_at = CURRENT_TIMESTAMP WHERE token = ? LIMIT 1 "#, + creator_ip_address, tts_model_token, ) .execute(mysql_pool) @@ -46,17 +49,20 @@ LIMIT 1 pub async fn undelete_tts_model_as_user( tts_model_token: &str, + creator_ip_address: &str, mysql_pool: &MySqlPool, ) -> AnyhowResult<()> { let _r = sqlx::query!( r#" UPDATE tts_models SET + creator_ip_address_last_update = ?, user_deleted_at = NULL WHERE token = ? LIMIT 1 "#, + creator_ip_address, tts_model_token, ) .execute(mysql_pool) diff --git a/src/http_server/endpoints/tts/delete_tts_model.rs b/src/http_server/endpoints/tts/delete_tts_model.rs index 4e3d77d9d3..5beaf9c408 100644 --- a/src/http_server/endpoints/tts/delete_tts_model.rs +++ b/src/http_server/endpoints/tts/delete_tts_model.rs @@ -118,8 +118,7 @@ pub async fn delete_tts_model_handler( return Err(DeleteTtsModelError::NotAuthorized); } - // NB: I can't imagine we need to store this. - // let ip_address = get_request_ip(&http_request); + let ip_address = get_request_ip(&http_request); let as_mod = delete_as_mod(is_mod, is_author, request.as_mod); @@ -133,7 +132,8 @@ pub async fn delete_tts_model_handler( } else { delete_tts_model_as_user( &path.token, - &server_state.mysql_pool + &ip_address, + &server_state.mysql_pool, ).await } } else { @@ -147,6 +147,7 @@ pub async fn delete_tts_model_handler( // NB: Technically only mods can see their own inference_results here undelete_tts_model_as_user( &path.token, + &ip_address, &server_state.mysql_pool ).await } From b0e09cf9c55f26d143cd4f55919df0f37fa25e06 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Jul 2021 13:22:08 -0400 Subject: [PATCH 0471/1042] extracting delete role util (not yet used) --- .../delete_request_role_disambiguation.rs | 59 +++++++++++++++++++ src/util/mod.rs | 1 + 2 files changed, 60 insertions(+) create mode 100644 src/util/delete_request_role_disambiguation.rs diff --git a/src/util/delete_request_role_disambiguation.rs b/src/util/delete_request_role_disambiguation.rs new file mode 100644 index 0000000000..0aca876160 --- /dev/null +++ b/src/util/delete_request_role_disambiguation.rs @@ -0,0 +1,59 @@ +//! This is meant to disambiguate whether the deletion should occur as a user or moderator +//! for soft-deleted records. All such requests have a similar shape, and this was tested +//! with a truth table. + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum DeleteRole { + ErrorDoNotDelete, + AsUser, + AsMod, +} + +pub fn delete_role_disambiguation( + user_is_mod: bool, + user_is_author: bool, + as_mod_flag: Option +) -> DeleteRole { + if !user_is_mod && !user_is_author { + return DeleteRole::ErrorDoNotDelete; + } + // NB: Explored this with a truth table. + let as_mod_flag_value = as_mod_flag.unwrap_or(false); + let delete_as_mod = user_is_mod && !(user_is_author && as_mod_flag.is_some() && !as_mod_flag_value); + if delete_as_mod { DeleteRole::AsMod } else { DeleteRole::AsUser } +} + +#[cfg(test)] +mod tests { + use crate::util::delete_request_role_disambiguation::{delete_role_disambiguation, DeleteRole}; + + #[test] + fn test_delete_as_mod() { + // User is neither author nor mod. + // Do not delete! + assert_eq!(DeleteRole::ErrorDoNotDelete, delete_role_disambiguation(false, false, None)); + assert_eq!(DeleteRole::ErrorDoNotDelete, delete_role_disambiguation(false, false, Some(false))); + assert_eq!(DeleteRole::ErrorDoNotDelete, delete_role_disambiguation(false, false, Some(true))); + + // User is author. + // Deleted as a user. + assert_eq!(DeleteRole::AsUser, delete_role_disambiguation(false, true, None)); + assert_eq!(DeleteRole::AsUser, delete_role_disambiguation(false, true, Some(false))); + assert_eq!(DeleteRole::AsUser, delete_role_disambiguation(false, true, Some(true))); + + // User is mod. + // Deleted as a mod. + assert_eq!(DeleteRole::AsMod, delete_role_disambiguation(true, false, None)); + assert_eq!(DeleteRole::AsMod, delete_role_disambiguation(true, false, Some(false))); + assert_eq!(DeleteRole::AsMod, delete_role_disambiguation(true, false, Some(true))); + + // User is mod and author. + // Deleting as a user. + assert_eq!(DeleteRole::AsUser, delete_role_disambiguation(true, true, Some(false))); + + // User is mod and author. + // Deleting as a mod. + assert_eq!(DeleteRole::AsMod, delete_role_disambiguation(true, true, None)); + assert_eq!(DeleteRole::AsMod, delete_role_disambiguation(true, true, Some(true))); + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 933401bacb..9af7627c74 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,5 +1,6 @@ pub mod anyhow_result; pub mod buckets; +pub mod delete_request_role_disambiguation; pub mod email_to_gravatar; pub mod encrypted_sort_id; pub mod filesystem; From 2c342ac1d95261adbf1432cb500068e94fd0c081 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Jul 2021 13:25:28 -0400 Subject: [PATCH 0472/1042] fixup tests --- src/database/helpers/boolean_converters.rs | 2 +- src/database/queries/list_tts_inference_results.rs | 2 +- src/database/queries/list_w2l_inference_results.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/database/helpers/boolean_converters.rs b/src/database/helpers/boolean_converters.rs index 43d395a1e8..33e6898f0c 100644 --- a/src/database/helpers/boolean_converters.rs +++ b/src/database/helpers/boolean_converters.rs @@ -21,7 +21,7 @@ pub fn nullable_i8_to_optional_bool(value: Option) -> Option { #[cfg(test)] mod tests { - use crate::database_helpers::boolean_converters::{ + use crate::database::helpers::boolean_converters::{ i8_to_bool, nullable_i8_to_bool, nullable_i8_to_optional_bool diff --git a/src/database/queries/list_tts_inference_results.rs b/src/database/queries/list_tts_inference_results.rs index 3c06bf9698..fc39026961 100644 --- a/src/database/queries/list_tts_inference_results.rs +++ b/src/database/queries/list_tts_inference_results.rs @@ -294,7 +294,7 @@ pub struct RawInternalTtsRecord { #[cfg(test)] mod tests { - use crate::common_queries::list_tts_inference_results::ListTtsResultsQueryBuilder; + use crate::database::queries::list_tts_inference_results::ListTtsResultsQueryBuilder; #[test] fn predicates_without_scoping() { diff --git a/src/database/queries/list_w2l_inference_results.rs b/src/database/queries/list_w2l_inference_results.rs index d8e28f6376..5506d2513f 100644 --- a/src/database/queries/list_w2l_inference_results.rs +++ b/src/database/queries/list_w2l_inference_results.rs @@ -341,7 +341,7 @@ pub struct RawInternalTtsRecord { #[cfg(test)] mod tests { - use crate::common_queries::list_w2l_inference_results::ListW2lResultsQueryBuilder; + use crate::database::queries::list_w2l_inference_results::ListW2lResultsQueryBuilder; #[test] fn predicates_without_scoping() { From c306730c0bc653acb8775332fbdd816c8b0eaeb0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Jul 2021 13:30:31 -0400 Subject: [PATCH 0473/1042] use common delete permission check --- .../endpoints/tts/delete_tts_model.rs | 110 +++++++----------- ...ation.rs => delete_role_disambiguation.rs} | 2 +- src/util/mod.rs | 2 +- 3 files changed, 46 insertions(+), 68 deletions(-) rename src/util/{delete_request_role_disambiguation.rs => delete_role_disambiguation.rs} (95%) diff --git a/src/http_server/endpoints/tts/delete_tts_model.rs b/src/http_server/endpoints/tts/delete_tts_model.rs index 5beaf9c408..86e3a0af5f 100644 --- a/src/http_server/endpoints/tts/delete_tts_model.rs +++ b/src/http_server/endpoints/tts/delete_tts_model.rs @@ -16,6 +16,8 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; +use crate::util::delete_role_disambiguation::DeleteRole; +use crate::util::delete_role_disambiguation::delete_role_disambiguation; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; @@ -110,46 +112,54 @@ pub async fn delete_tts_model_handler( Ok(Some(model)) => model, }; - // NB: Second set of permission checks - let is_author = &tts_model.creator_user_token == &user_session.user_token; - - if !is_author && !is_mod { - warn!("user is not allowed to delete model: {}", user_session.user_token); - return Err(DeleteTtsModelError::NotAuthorized); - } - let ip_address = get_request_ip(&http_request); - let as_mod = delete_as_mod(is_mod, is_author, request.as_mod); + // NB: Second set of permission checks + let is_author = &tts_model.creator_user_token == &user_session.user_token; + let delete_role = delete_role_disambiguation(is_mod, is_author, request.as_mod); let query_result = if request.set_delete { - if as_mod { - delete_tts_model_as_mod( - &path.token, - &user_session.user_token, - &server_state.mysql_pool - ).await - } else { - delete_tts_model_as_user( - &path.token, - &ip_address, - &server_state.mysql_pool, - ).await + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to delete model: {}", user_session.user_token); + return Err(DeleteTtsModelError::NotAuthorized); + } + DeleteRole::AsUser => { + delete_tts_model_as_user( + &path.token, + &ip_address, + &server_state.mysql_pool, + ).await + } + DeleteRole::AsMod => { + delete_tts_model_as_mod( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } } } else { - if as_mod { - undelete_tts_model_as_mod( - &path.token, - &user_session.user_token, - &server_state.mysql_pool - ).await - } else { - // NB: Technically only mods can see their own inference_results here - undelete_tts_model_as_user( - &path.token, - &ip_address, - &server_state.mysql_pool - ).await + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to delete model: {}", user_session.user_token); + return Err(DeleteTtsModelError::NotAuthorized); + } + DeleteRole::AsUser => { + // NB: Technically only mods can see their own inference_results here + undelete_tts_model_as_user( + &path.token, + &ip_address, + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + undelete_tts_model_as_mod( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } } }; @@ -163,35 +173,3 @@ pub async fn delete_tts_model_handler( Ok(simple_json_success()) } - -fn delete_as_mod(user_is_mod: bool, user_is_author: bool, as_mod_flag: Option) -> bool { - // NB: Explored this with a truth table. - let as_mod_flag_value = as_mod_flag.unwrap_or(false); - user_is_mod && !(user_is_author && as_mod_flag.is_some() && !as_mod_flag_value) -} - -#[cfg(test)] -mod tests { - use crate::http_server::endpoints::tts::delete_tts_model::delete_as_mod; - - #[test] - fn test_delete_as_mod() { - // Deleted as a user - assert!(!delete_as_mod(false, false, None)); - assert!(!delete_as_mod(false, true, None)); - assert!(!delete_as_mod(false, true, Some(false))); - assert!(!delete_as_mod(false, true, Some(true))); - - // Deleted as a mod - assert!(delete_as_mod(true, false, None)); - assert!(delete_as_mod(true, false, Some(false))); - assert!(delete_as_mod(true, false, Some(true))); - - // Moderator + Author, deleting as a user - assert!(!delete_as_mod(true, true, Some(false))); - - // Moderator + Author, deleting as a mod - assert!(delete_as_mod(true, true, None)); - assert!(delete_as_mod(true, true, Some(true))); - } -} diff --git a/src/util/delete_request_role_disambiguation.rs b/src/util/delete_role_disambiguation.rs similarity index 95% rename from src/util/delete_request_role_disambiguation.rs rename to src/util/delete_role_disambiguation.rs index 0aca876160..7519a71373 100644 --- a/src/util/delete_request_role_disambiguation.rs +++ b/src/util/delete_role_disambiguation.rs @@ -25,7 +25,7 @@ pub fn delete_role_disambiguation( #[cfg(test)] mod tests { - use crate::util::delete_request_role_disambiguation::{delete_role_disambiguation, DeleteRole}; + use crate::util::delete_role_disambiguation::{delete_role_disambiguation, DeleteRole}; #[test] fn test_delete_as_mod() { diff --git a/src/util/mod.rs b/src/util/mod.rs index 9af7627c74..12d47f5db4 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,6 +1,6 @@ pub mod anyhow_result; pub mod buckets; -pub mod delete_request_role_disambiguation; +pub mod delete_role_disambiguation; pub mod email_to_gravatar; pub mod encrypted_sort_id; pub mod filesystem; From 0e3059f90616711668f96edc2280039d79229e81 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Jul 2021 21:12:19 -0400 Subject: [PATCH 0474/1042] better deletion for w2l templates --- src/database/queries/delete_w2l_template.rs | 94 ++++++++++ src/database/queries/mod.rs | 1 + .../endpoints/tts/delete_tts_model.rs | 19 +- .../endpoints/w2l/delete_w2l_template.rs | 168 ++++++------------ 4 files changed, 161 insertions(+), 121 deletions(-) create mode 100644 src/database/queries/delete_w2l_template.rs diff --git a/src/database/queries/delete_w2l_template.rs b/src/database/queries/delete_w2l_template.rs new file mode 100644 index 0000000000..8c29c56aa1 --- /dev/null +++ b/src/database/queries/delete_w2l_template.rs @@ -0,0 +1,94 @@ +use crate::util::anyhow_result::AnyhowResult; +use sqlx::MySqlPool; + +pub async fn delete_w2l_template_as_user( + template_token: &str, + creator_ip_address: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE w2l_templates +SET + creator_ip_address_last_update = ?, + user_deleted_at = CURRENT_TIMESTAMP +WHERE + token = ? +LIMIT 1 + "#, + creator_ip_address, + template_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn delete_w2l_template_as_mod( + template_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE w2l_templates +SET + maybe_mod_user_token = ?, + mod_deleted_at = CURRENT_TIMESTAMP +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + template_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn undelete_w2l_template_as_user( + template_token: &str, + creator_ip_address: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE w2l_templates +SET + creator_ip_address_last_update = ?, + user_deleted_at = NULL +WHERE + token = ? +LIMIT 1 + "#, + creator_ip_address, + template_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn undelete_w2l_template_as_mod( + template_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE w2l_templates +SET + maybe_mod_user_token = ?, + mod_deleted_at = NULL +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + template_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} diff --git a/src/database/queries/mod.rs b/src/database/queries/mod.rs index a74deffd40..ed49db0788 100644 --- a/src/database/queries/mod.rs +++ b/src/database/queries/mod.rs @@ -1,6 +1,7 @@ pub mod badges; pub mod create_session; pub mod delete_tts_model; +pub mod delete_w2l_template; pub mod list_ip_bans; pub mod list_tts_inference_results; pub mod list_tts_models; diff --git a/src/http_server/endpoints/tts/delete_tts_model.rs b/src/http_server/endpoints/tts/delete_tts_model.rs index 86e3a0af5f..8197488110 100644 --- a/src/http_server/endpoints/tts/delete_tts_model.rs +++ b/src/http_server/endpoints/tts/delete_tts_model.rs @@ -112,10 +112,23 @@ pub async fn delete_tts_model_handler( Ok(Some(model)) => model, }; - let ip_address = get_request_ip(&http_request); - // NB: Second set of permission checks let is_author = &tts_model.creator_user_token == &user_session.user_token; + + if !is_author && !is_mod { + warn!("user is not allowed to delete models: {}", user_session.user_token); + return Err(DeleteTtsModelError::NotAuthorized); + } + + if !is_mod { + if tts_model.is_locked_from_user_modification || tts_model.is_locked_from_use { + warn!("user is not allowed to delete models (locked): {}", user_session.user_token); + return Err(DeleteTtsModelError::NotAuthorized); + } + } + + let ip_address = get_request_ip(&http_request); + let delete_role = delete_role_disambiguation(is_mod, is_author, request.as_mod); let query_result = if request.set_delete { @@ -142,7 +155,7 @@ pub async fn delete_tts_model_handler( } else { match delete_role { DeleteRole::ErrorDoNotDelete => { - warn!("user is not allowed to delete model: {}", user_session.user_token); + warn!("user is not allowed to undelete model: {}", user_session.user_token); return Err(DeleteTtsModelError::NotAuthorized); } DeleteRole::AsUser => { diff --git a/src/http_server/endpoints/w2l/delete_w2l_template.rs b/src/http_server/endpoints/w2l/delete_w2l_template.rs index b111eb2198..bd4f9c2017 100644 --- a/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -7,11 +7,17 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::queries::delete_w2l_template::delete_w2l_template_as_mod; +use crate::database::queries::delete_w2l_template::delete_w2l_template_as_user; +use crate::database::queries::delete_w2l_template::undelete_w2l_template_as_mod; +use crate::database::queries::delete_w2l_template::undelete_w2l_template_as_user; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; +use crate::util::delete_role_disambiguation::DeleteRole; +use crate::util::delete_role_disambiguation::delete_role_disambiguation; use crate::util::random_crockford_token::random_crockford_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; @@ -34,6 +40,8 @@ pub struct DeleteW2lTemplatePathInfo { #[derive(Deserialize)] pub struct DeleteW2lTemplateRequest { set_delete: bool, + /// NB: this is only to disambiguate when a user is both a mod and an author. + as_mod: Option, } #[derive(Debug, Display)] @@ -119,40 +127,57 @@ pub async fn delete_w2l_template_handler( if !is_mod { if w2l_template.is_locked_from_user_modification || w2l_template.is_locked_from_use { + warn!("user is not allowed to delete templates (locked): {}", user_session.user_token); return Err(DeleteW2lTemplateError::NotAuthorized); } } let ip_address = get_request_ip(&http_request); + let delete_role = delete_role_disambiguation(is_mod, is_author, request.as_mod); + let query_result = if request.set_delete { - if is_author { - user_delete_template( - &path.token, - &user_session.user_token, - &server_state.mysql_pool - ).await - } else { - mod_delete_template( - &path.token, - &user_session.user_token, - &server_state.mysql_pool - ).await + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to delete template: {}", user_session.user_token); + return Err(DeleteW2lTemplateError::NotAuthorized); + } + DeleteRole::AsUser => { + delete_w2l_template_as_user( + &path.token, + &ip_address, + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + delete_w2l_template_as_mod( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } } } else { - if is_author { - // NB: Technically only mods can see their own templates here - user_undelete_template( - &path.token, - &user_session.user_token, - &server_state.mysql_pool - ).await - } else { - mod_undelete_template( - &path.token, - &user_session.user_token, - &server_state.mysql_pool - ).await + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to delete template: {}", user_session.user_token); + return Err(DeleteW2lTemplateError::NotAuthorized); + } + DeleteRole::AsUser => { + // NB: Technically only mods can see their own templates here + undelete_w2l_template_as_user( + &path.token, + &ip_address, + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + undelete_w2l_template_as_mod( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } } }; @@ -167,96 +192,3 @@ pub async fn delete_w2l_template_handler( Ok(simple_json_success()) } -async fn user_delete_template( - template_token: &str, - mod_user_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - let _r = sqlx::query!( - r#" -UPDATE w2l_templates -SET - user_deleted_at = CURRENT_TIMESTAMP, - maybe_mod_user_token = ? -WHERE - token = ? -LIMIT 1 - "#, - mod_user_token, - template_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} - -async fn mod_delete_template( - template_token: &str, - mod_user_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - let _r = sqlx::query!( - r#" -UPDATE w2l_templates -SET - mod_deleted_at = CURRENT_TIMESTAMP, - maybe_mod_user_token = ? -WHERE - token = ? -LIMIT 1 - "#, - mod_user_token, - template_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} - -async fn user_undelete_template( - template_token: &str, - mod_user_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - - let _r = sqlx::query!( - r#" -UPDATE w2l_templates -SET - user_deleted_at = NULL, - maybe_mod_user_token = ? -WHERE - token = ? -LIMIT 1 - "#, - mod_user_token, - template_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} - -async fn mod_undelete_template( - template_token: &str, - mod_user_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - - let _r = sqlx::query!( - r#" -UPDATE w2l_templates -SET - mod_deleted_at = NULL, - maybe_mod_user_token = ? -WHERE - token = ? -LIMIT 1 - "#, - mod_user_token, - template_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} From 5586abe7a257fe5ed97685c891e7c40e803a2992 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Jul 2021 21:38:47 -0400 Subject: [PATCH 0475/1042] better delete for w2l results --- src/database/queries/delete_w2l_result.rs | 88 ++++++++++ src/database/queries/mod.rs | 1 + .../endpoints/w2l/delete_w2l_result.rs | 158 +++++------------- 3 files changed, 135 insertions(+), 112 deletions(-) create mode 100644 src/database/queries/delete_w2l_result.rs diff --git a/src/database/queries/delete_w2l_result.rs b/src/database/queries/delete_w2l_result.rs new file mode 100644 index 0000000000..5218d3a3f2 --- /dev/null +++ b/src/database/queries/delete_w2l_result.rs @@ -0,0 +1,88 @@ +use crate::util::anyhow_result::AnyhowResult; +use sqlx::MySqlPool; + +pub async fn delete_w2l_inference_result_as_user( + inference_result_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE w2l_results +SET + user_deleted_at = CURRENT_TIMESTAMP +WHERE + token = ? +LIMIT 1 + "#, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn delete_w2l_inference_result_as_mod( + inference_result_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE w2l_results +SET + mod_deleted_at = CURRENT_TIMESTAMP, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn undelete_w2l_inference_result_as_user( + inference_result_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE w2l_results +SET + user_deleted_at = NULL +WHERE + token = ? +LIMIT 1 + "#, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn undelete_w2l_inference_result_as_mod( + inference_result_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE w2l_results +SET + mod_deleted_at = NULL, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} diff --git a/src/database/queries/mod.rs b/src/database/queries/mod.rs index ed49db0788..a19283cd5d 100644 --- a/src/database/queries/mod.rs +++ b/src/database/queries/mod.rs @@ -1,6 +1,7 @@ pub mod badges; pub mod create_session; pub mod delete_tts_model; +pub mod delete_w2l_result; pub mod delete_w2l_template; pub mod list_ip_bans; pub mod list_tts_inference_results; diff --git a/src/http_server/endpoints/w2l/delete_w2l_result.rs b/src/http_server/endpoints/w2l/delete_w2l_result.rs index 7315b7f14a..7864714374 100644 --- a/src/http_server/endpoints/w2l/delete_w2l_result.rs +++ b/src/http_server/endpoints/w2l/delete_w2l_result.rs @@ -6,11 +6,17 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::queries::delete_w2l_result::delete_w2l_inference_result_as_mod; +use crate::database::queries::delete_w2l_result::delete_w2l_inference_result_as_user; +use crate::database::queries::delete_w2l_result::undelete_w2l_inference_result_as_mod; +use crate::database::queries::delete_w2l_result::undelete_w2l_inference_result_as_user; use crate::database::queries::query_w2l_result::select_w2l_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; +use crate::util::delete_role_disambiguation::DeleteRole; +use crate::util::delete_role_disambiguation::delete_role_disambiguation; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; @@ -32,6 +38,8 @@ pub struct DeleteW2lInferenceResultPathInfo { #[derive(Deserialize)] pub struct DeleteW2lInferenceResultRequest { set_delete: bool, + /// NB: this is only to disambiguate when a user is both a mod and an author. + as_mod: Option, } #[derive(Debug, Display)] @@ -119,35 +127,48 @@ pub async fn delete_w2l_inference_result_handler( return Err(DeleteW2lInferenceResultError::NotAuthorized); } - // NB: I can't imagine we need to store this. - // let ip_address = get_request_ip(&http_request); + let delete_role = delete_role_disambiguation(is_mod, is_author, request.as_mod); let query_result = if request.set_delete { - if is_author { - user_delete_inference_result( - &path.token, - &server_state.mysql_pool - ).await - } else { - mod_delete_inference_result( - &path.token, - &user_session.user_token, - &server_state.mysql_pool - ).await + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to delete inference results: {}", user_session.user_token); + return Err(DeleteW2lInferenceResultError::NotAuthorized); + } + DeleteRole::AsUser => { + delete_w2l_inference_result_as_user( + &path.token, + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + delete_w2l_inference_result_as_mod( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } } } else { - if is_author { - // NB: Technically only mods can see their own inference_results here - user_undelete_inference_result( - &path.token, - &server_state.mysql_pool - ).await - } else { - mod_undelete_inference_result( - &path.token, - &user_session.user_token, - &server_state.mysql_pool - ).await + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to delete inference results: {}", user_session.user_token); + return Err(DeleteW2lInferenceResultError::NotAuthorized); + } + DeleteRole::AsUser => { + // NB: Technically only mods can see their own inference_results here + undelete_w2l_inference_result_as_user( + &path.token, + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + undelete_w2l_inference_result_as_mod( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } } }; @@ -162,90 +183,3 @@ pub async fn delete_w2l_inference_result_handler( Ok(simple_json_success()) } -async fn user_delete_inference_result( - inference_result_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - let _r = sqlx::query!( - r#" -UPDATE w2l_results -SET - user_deleted_at = CURRENT_TIMESTAMP -WHERE - token = ? -LIMIT 1 - "#, - inference_result_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} - -async fn mod_delete_inference_result( - inference_result_token: &str, - mod_user_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - let _r = sqlx::query!( - r#" -UPDATE w2l_results -SET - mod_deleted_at = CURRENT_TIMESTAMP, - maybe_mod_user_token = ? -WHERE - token = ? -LIMIT 1 - "#, - mod_user_token, - inference_result_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} - -async fn user_undelete_inference_result( - inference_result_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - - let _r = sqlx::query!( - r#" -UPDATE w2l_results -SET - user_deleted_at = NULL -WHERE - token = ? -LIMIT 1 - "#, - inference_result_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} - -async fn mod_undelete_inference_result( - inference_result_token: &str, - mod_user_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - - let _r = sqlx::query!( - r#" -UPDATE w2l_results -SET - mod_deleted_at = NULL, - maybe_mod_user_token = ? -WHERE - token = ? -LIMIT 1 - "#, - mod_user_token, - inference_result_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} From a875e138eb33974e438e8a7b5432b56e36260626 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Jul 2021 22:17:00 -0400 Subject: [PATCH 0476/1042] model deletion improvement for tts results --- src/database/queries/delete_tts_result.rs | 88 ++++++++++ src/database/queries/mod.rs | 1 + .../endpoints/tts/delete_tts_result.rs | 159 +++++------------- 3 files changed, 135 insertions(+), 113 deletions(-) create mode 100644 src/database/queries/delete_tts_result.rs diff --git a/src/database/queries/delete_tts_result.rs b/src/database/queries/delete_tts_result.rs new file mode 100644 index 0000000000..bc9812c095 --- /dev/null +++ b/src/database/queries/delete_tts_result.rs @@ -0,0 +1,88 @@ +use crate::util::anyhow_result::AnyhowResult; +use sqlx::MySqlPool; + +pub async fn delete_tts_inference_result_as_user( + inference_result_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE tts_results +SET + user_deleted_at = CURRENT_TIMESTAMP +WHERE + token = ? +LIMIT 1 + "#, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn delete_tts_inference_result_as_mod( + inference_result_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE tts_results +SET + mod_deleted_at = CURRENT_TIMESTAMP, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn undelete_tts_inference_result_as_user( + inference_result_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE tts_results +SET + user_deleted_at = NULL +WHERE + token = ? +LIMIT 1 + "#, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +pub async fn undelete_tts_inference_result_as_mod( + inference_result_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult<()> { + let _r = sqlx::query!( + r#" +UPDATE tts_results +SET + mod_deleted_at = NULL, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} diff --git a/src/database/queries/mod.rs b/src/database/queries/mod.rs index a19283cd5d..3c6092a765 100644 --- a/src/database/queries/mod.rs +++ b/src/database/queries/mod.rs @@ -1,6 +1,7 @@ pub mod badges; pub mod create_session; pub mod delete_tts_model; +pub mod delete_tts_result; pub mod delete_w2l_result; pub mod delete_w2l_template; pub mod list_ip_bans; diff --git a/src/http_server/endpoints/tts/delete_tts_result.rs b/src/http_server/endpoints/tts/delete_tts_result.rs index 6111a0665f..6e8d1d3b37 100644 --- a/src/http_server/endpoints/tts/delete_tts_result.rs +++ b/src/http_server/endpoints/tts/delete_tts_result.rs @@ -6,11 +6,17 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::queries::delete_tts_result::delete_tts_inference_result_as_mod; +use crate::database::queries::delete_tts_result::delete_tts_inference_result_as_user; +use crate::database::queries::delete_tts_result::undelete_tts_inference_result_as_mod; +use crate::database::queries::delete_tts_result::undelete_tts_inference_result_as_user; use crate::database::queries::query_tts_result::select_tts_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; +use crate::util::delete_role_disambiguation::DeleteRole; +use crate::util::delete_role_disambiguation::delete_role_disambiguation; use crate::util::random_crockford_token::random_crockford_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; @@ -33,6 +39,8 @@ pub struct DeleteTtsInferenceResultPathInfo { #[derive(Deserialize)] pub struct DeleteTtsInferenceResultRequest { set_delete: bool, + /// NB: this is only to disambiguate when a user is both a mod and an author. + as_mod: Option, } #[derive(Debug, Display)] @@ -120,35 +128,48 @@ pub async fn delete_tts_inference_result_handler( return Err(DeleteTtsInferenceResultError::NotAuthorized); } - // NB: I can't imagine we need to store this. - // let ip_address = get_request_ip(&http_request); + let delete_role = delete_role_disambiguation(is_mod, is_author, request.as_mod); let query_result = if request.set_delete { - if is_author { - user_delete_inference_result( - &path.token, - &server_state.mysql_pool - ).await - } else { - mod_delete_inference_result( - &path.token, - &user_session.user_token, - &server_state.mysql_pool - ).await + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to delete inference results: {}", user_session.user_token); + return Err(DeleteTtsInferenceResultError::NotAuthorized); + } + DeleteRole::AsUser => { + delete_tts_inference_result_as_user( + &path.token, + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + delete_tts_inference_result_as_mod( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } } } else { - if is_author { - // NB: Technically only mods can see their own inference_results here - user_undelete_inference_result( - &path.token, - &server_state.mysql_pool - ).await - } else { - mod_undelete_inference_result( - &path.token, - &user_session.user_token, - &server_state.mysql_pool - ).await + match delete_role { + DeleteRole::ErrorDoNotDelete => { + warn!("user is not allowed to undelete inference results: {}", user_session.user_token); + return Err(DeleteTtsInferenceResultError::NotAuthorized); + } + DeleteRole::AsUser => { + // NB: Technically only mods can see their own inference_results here + undelete_tts_inference_result_as_user( + &path.token, + &server_state.mysql_pool + ).await + } + DeleteRole::AsMod => { + undelete_tts_inference_result_as_mod( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } } }; @@ -162,91 +183,3 @@ pub async fn delete_tts_inference_result_handler( Ok(simple_json_success()) } - -async fn user_delete_inference_result( - inference_result_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - let _r = sqlx::query!( - r#" -UPDATE tts_results -SET - user_deleted_at = CURRENT_TIMESTAMP -WHERE - token = ? -LIMIT 1 - "#, - inference_result_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} - -async fn mod_delete_inference_result( - inference_result_token: &str, - mod_user_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - let _r = sqlx::query!( - r#" -UPDATE tts_results -SET - mod_deleted_at = CURRENT_TIMESTAMP, - maybe_mod_user_token = ? -WHERE - token = ? -LIMIT 1 - "#, - mod_user_token, - inference_result_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} - -async fn user_undelete_inference_result( - inference_result_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - - let _r = sqlx::query!( - r#" -UPDATE tts_results -SET - user_deleted_at = NULL -WHERE - token = ? -LIMIT 1 - "#, - inference_result_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} - -async fn mod_undelete_inference_result( - inference_result_token: &str, - mod_user_token: &str, - mysql_pool: &MySqlPool -) -> Result<(), sqlx::Error> { - - let _r = sqlx::query!( - r#" -UPDATE tts_results -SET - mod_deleted_at = NULL, - maybe_mod_user_token = ? -WHERE - token = ? -LIMIT 1 - "#, - mod_user_token, - inference_result_token, - ) - .execute(mysql_pool) - .await?; - Ok(()) -} From e7c0c5b6f02b8b982f191c8ff2f9c0e4f799d293 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Jul 2021 22:17:52 -0400 Subject: [PATCH 0477/1042] codegen --- sqlx-data.json | 120 ++++++++++++++++++++++++------------------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 10a14ec23c..fc0d476d12 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -2272,6 +2272,16 @@ ] } }, + "551749bc153f9c8683a3a1ca5bb50744322aac046db34327501cf991485c4dee": { + "query": "\nUPDATE tts_models\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "55a75db2d060bd49951db22f6b4ffd1545777b61c256030d59662543adc06ab2": { "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n ip_address_last_update = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", "describe": { @@ -3790,26 +3800,6 @@ "nullable": [] } }, - "8ba1925f85324f4bc73f809a825453196a95b07008d23b0ee66e8a73f73a3510": { - "query": "\nUPDATE w2l_templates\nSET\n user_deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "8c0c0ad0fafef712bb2eb269bcef8658efcd84154d6f6eeda26b0399abd641c4": { - "query": "\nUPDATE w2l_templates\nSET\n user_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, "8d61bb77449a78044135a3d6e5009a3a27b4671b7e40501d318b05ae401f02a8": { "query": "\nSELECT token, username, email_address, password_hash\nFROM users\nWHERE email_address = ?\nLIMIT 1\n ", "describe": { @@ -3874,6 +3864,16 @@ ] } }, + "8ff27fc5583db97f157f1b808f593a72c77dc5dcf5a633cd9c1162af6b97c011": { + "query": "\nUPDATE w2l_templates\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "961f792c24bee6009e3e8a215561db932871a53c48839b8cb80292de00e7b2e1": { "query": "\nUPDATE tts_results\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -4978,16 +4978,6 @@ ] } }, - "baf058c93336325a642ab07a617c69eae9d2c7467bf3a8248e814063a6c36001": { - "query": "\nUPDATE w2l_templates\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, "bc9c6b57b2578b8163956d11a5f6a9a77ddab6098d2c4207e1a70502db30c9d1": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.private_bucket_hash,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\nAND w2l.user_deleted_at IS NULL\nAND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -5227,6 +5217,16 @@ ] } }, + "c4640a618e1d179fa95adce67a329818f8b1626ac88fa76dbb871b8788136191": { + "query": "\nUPDATE w2l_templates\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "c63726afcf8d53a24597aad18bb44285eae8f5d056011811b066226f7db0198a": { "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -5247,16 +5247,6 @@ "nullable": [] } }, - "cae002e97a3309e0e8361d2a08158c5a1dc1e5b5123c9c5e2c5f8e5e93d07387": { - "query": "\nUPDATE w2l_templates\nSET\n mod_deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, "cc51b77e88c2ee5c046ea2f2287b9e9770743d50b0db90481662e5c1118885c5": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -5580,16 +5570,6 @@ ] } }, - "cd8541805f58293d33c30e108dba73a385407dd5546c5b7380da7397b4f6de34": { - "query": "\nUPDATE tts_models\nSET\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, "cf0b037e789ee09022b25f3381b0263c5cc6b59f3f328de4aa211bdb23a62629": { "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -6441,6 +6421,16 @@ ] } }, + "d71ef2cfed337f9e3f9e1c2255bdec6347763910fd589482f0b0e56b794d7bb7": { + "query": "\nUPDATE tts_models\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "d862b69c0ded636bb491c6442958c8baf3457d80791eadf880445335e5fcda0b": { "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n\n w2l_results.creator_ip_address,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n AND w2l_results.user_deleted_at IS NULL\n AND w2l_results.mod_deleted_at IS NULL\n ", "describe": { @@ -6978,6 +6968,16 @@ ] } }, + "e0ad2a7fe4ae795e4f097f604d5cfcf9605298b58cde5ca23bdf9c8ec7383b00": { + "query": "\nUPDATE w2l_templates\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "e3a0bd8073c3c0d84b9ab0e2e1ad8371f14b10801b7e4658b73c42ce1b27fa0b": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.is_public_listing_approved IS TRUE\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -7172,6 +7172,16 @@ ] } }, + "e4e5be790f087b9aa6bee87e2315140bf621e4919362ebe72ab070163f18e9c8": { + "query": "\nUPDATE w2l_templates\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "e73192a3e763a287d5737821443c5a81649f17e39978497b99c52e1e896a75f6": { "query": "\nSELECT\n token as session_token,\n user_token\nFROM user_sessions\nWHERE token = ?\nAND deleted_at IS NULL\n ", "describe": { @@ -7359,16 +7369,6 @@ ] } }, - "ebd41740a814e311e36a0e0b3ea7ecc829687c7d14ba854c81efa2404bdd77d6": { - "query": "\nUPDATE tts_models\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, "ed67238afe48c159443d215cd1fe2f0264ad840768f789d068527ae542b08194": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { From bdf549cc1dd2b729c1b6ae747749e3efad78052a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 11 Jul 2021 16:15:07 -0400 Subject: [PATCH 0478/1042] enums work (not yet finished) --- src/database/enums/mod.rs | 1 + src/database/enums/record_visibility.rs | 24 +++++++++++++++++++ src/database/mod.rs | 1 + .../endpoints/users/edit_profile.rs | 12 ++++++---- .../endpoints/users/get_profile.rs | 10 ++++++++ src/threads/poll_ip_banlist_thread.rs | 2 ++ 6 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 src/database/enums/mod.rs create mode 100644 src/database/enums/record_visibility.rs diff --git a/src/database/enums/mod.rs b/src/database/enums/mod.rs new file mode 100644 index 0000000000..fa9a069908 --- /dev/null +++ b/src/database/enums/mod.rs @@ -0,0 +1 @@ +pub mod record_visibility; \ No newline at end of file diff --git a/src/database/enums/record_visibility.rs b/src/database/enums/record_visibility.rs new file mode 100644 index 0000000000..f9f51d8f16 --- /dev/null +++ b/src/database/enums/record_visibility.rs @@ -0,0 +1,24 @@ +//! These are columns where users can control the visibility of their data. + +//#[sqlx(rename_all = "lowercase", rename = "preferred_tts_result_visibility")] + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] +#[sqlx(rename_all = "lowercase")] +pub enum RecordVisibility { + #[sqlx(rename = "public")] + Public, + #[sqlx(rename = "hidden")] + Hidden, + #[sqlx(rename = "private")] + Private, +} + +impl RecordVisibility { + pub fn to_str(&self) -> &'static str { + match self { + RecordVisibility::Public => "public", + RecordVisibility::Hidden => "hidden", + RecordVisibility::Private => "private", + } + } +} diff --git a/src/database/mod.rs b/src/database/mod.rs index d160ea03e0..e86d45f8df 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -1,3 +1,4 @@ +pub mod enums; pub mod helpers; pub mod mediators; pub mod queries; \ No newline at end of file diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs index b0b9e55a0a..f8bb1f7553 100644 --- a/src/http_server/endpoints/users/edit_profile.rs +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -6,6 +6,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::create_session::create_session_for_user; use crate::database::queries::query_user_profile::select_user_profile_by_username; use crate::http_server::web_utils::ip_address::get_request_ip; @@ -52,6 +53,9 @@ pub struct EditProfileRequest { pub github_username: Option, pub cashapp_username: Option, pub website_url: Option, + + pub preferred_tts_result_visibility: Option, + pub preferred_w2l_result_visibility: Option, } #[derive(Serialize)] @@ -138,6 +142,10 @@ pub async fn edit_profile_handler( editor_is_moderator = true; } + if !editor_is_original_user && !editor_is_moderator { + return Err(EditProfileError::NotAuthorized); + } + // Fields to set let mut twitter_username = None; let mut twitch_username = None; @@ -148,10 +156,6 @@ pub async fn edit_profile_handler( let mut profile_markdown = None; let mut profile_html = None; - if !editor_is_original_user && !editor_is_moderator { - return Err(EditProfileError::NotAuthorized); - } - if let Some(twitter) = request.twitter_username.as_deref() { let trimmed = twitter.trim(); if trimmed.is_empty() { diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index 4981f8c812..ea1f17e527 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -9,6 +9,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::boolean_converters::i8_to_bool; use crate::database::queries::list_user_badges::UserBadgeForList; use crate::database::queries::list_user_badges::list_user_badges; @@ -49,6 +50,9 @@ pub struct UserProfileRecordForResponse { pub user_role_slug: String, pub is_banned: bool, pub disable_gravatar: bool, + pub preferred_tts_result_visibility: RecordVisibility, + //pub preferred_tts_result_visibility: String, + //pub preferred_w2l_result_visibility: RecordVisibility, pub discord_username: Option, pub twitch_username: Option, pub twitter_username: Option, @@ -77,6 +81,9 @@ pub struct RawUserProfileRecord { pub user_role_slug: String, pub is_banned: i8, pub disable_gravatar: i8, + pub preferred_tts_result_visibility: RecordVisibility, + //pub preferred_tts_result_visibility: String, + //pub preferred_w2l_result_visibility: RecordVisibility, pub discord_username: Option, pub twitch_username: Option, pub twitter_username: Option, @@ -137,6 +144,7 @@ SELECT user_role_slug, is_banned, disable_gravatar, + preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility`, discord_username, twitch_username, twitter_username, @@ -189,6 +197,8 @@ WHERE user_role_slug: profile_record.user_role_slug.clone(), is_banned: i8_to_bool(profile_record.is_banned), disable_gravatar: i8_to_bool(profile_record.disable_gravatar), + preferred_tts_result_visibility: profile_record.preferred_tts_result_visibility.clone(), + //preferred_w2l_result_visibility: profile_record.preferred_w2l_result_visibility, discord_username: profile_record.discord_username.clone(), twitch_username: profile_record.twitch_username.clone(), twitter_username: profile_record.twitter_username.clone(), diff --git a/src/threads/poll_ip_banlist_thread.rs b/src/threads/poll_ip_banlist_thread.rs index d71c04ae8b..89b59aa32c 100644 --- a/src/threads/poll_ip_banlist_thread.rs +++ b/src/threads/poll_ip_banlist_thread.rs @@ -39,4 +39,6 @@ pub async fn poll_ip_bans( thread::sleep(Duration::from_millis(20_000)); } + + warn!("Should never happen: IP poller exits"); } From baffa751d630dfa2c3dab0c882b95616f1ebb5f0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 11 Jul 2021 16:19:33 -0400 Subject: [PATCH 0479/1042] visibility enums --- src/database/enums/record_visibility.rs | 9 ++++----- src/http_server/endpoints/users/get_profile.rs | 11 +++++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/database/enums/record_visibility.rs b/src/database/enums/record_visibility.rs index f9f51d8f16..d589f4cfe5 100644 --- a/src/database/enums/record_visibility.rs +++ b/src/database/enums/record_visibility.rs @@ -1,15 +1,14 @@ //! These are columns where users can control the visibility of their data. -//#[sqlx(rename_all = "lowercase", rename = "preferred_tts_result_visibility")] - +/// To use this in a query, the query must have type annotations. +/// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375 +/// eg. preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility` #[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] +#[serde(rename_all = "lowercase")] #[sqlx(rename_all = "lowercase")] pub enum RecordVisibility { - #[sqlx(rename = "public")] Public, - #[sqlx(rename = "hidden")] Hidden, - #[sqlx(rename = "private")] Private, } diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index ea1f17e527..ac5bc31dbe 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -51,8 +51,7 @@ pub struct UserProfileRecordForResponse { pub is_banned: bool, pub disable_gravatar: bool, pub preferred_tts_result_visibility: RecordVisibility, - //pub preferred_tts_result_visibility: String, - //pub preferred_w2l_result_visibility: RecordVisibility, + pub preferred_w2l_result_visibility: RecordVisibility, pub discord_username: Option, pub twitch_username: Option, pub twitter_username: Option, @@ -82,8 +81,7 @@ pub struct RawUserProfileRecord { pub is_banned: i8, pub disable_gravatar: i8, pub preferred_tts_result_visibility: RecordVisibility, - //pub preferred_tts_result_visibility: String, - //pub preferred_w2l_result_visibility: RecordVisibility, + pub preferred_w2l_result_visibility: RecordVisibility, pub discord_username: Option, pub twitch_username: Option, pub twitter_username: Option, @@ -145,6 +143,7 @@ SELECT is_banned, disable_gravatar, preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility`, + preferred_w2l_result_visibility as `preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility`, discord_username, twitch_username, twitter_username, @@ -197,8 +196,8 @@ WHERE user_role_slug: profile_record.user_role_slug.clone(), is_banned: i8_to_bool(profile_record.is_banned), disable_gravatar: i8_to_bool(profile_record.disable_gravatar), - preferred_tts_result_visibility: profile_record.preferred_tts_result_visibility.clone(), - //preferred_w2l_result_visibility: profile_record.preferred_w2l_result_visibility, + preferred_tts_result_visibility: profile_record.preferred_tts_result_visibility, + preferred_w2l_result_visibility: profile_record.preferred_w2l_result_visibility, discord_username: profile_record.discord_username.clone(), twitch_username: profile_record.twitch_username.clone(), twitter_username: profile_record.twitter_username.clone(), From 444511d79eab5d6318c6d8342df0f9b6bb4640b8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 11 Jul 2021 16:38:26 -0400 Subject: [PATCH 0480/1042] propagate visibility to enqueue tts infer --- .../endpoints/tts/enqueue_infer_tts.rs | 16 ++++++++++++---- src/http_server/endpoints/users/edit_profile.rs | 8 ++++++-- src/http_server/web_utils/session_checker.rs | 11 +++++++++++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index ea221ff696..ab985402f9 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -5,7 +5,8 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::helpers::enums::{DownloadUrlType, W2lTemplateType}; use crate::database::helpers::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -30,7 +31,7 @@ pub struct InferTtsRequest { uuid_idempotency_token: String, tts_model_token: String, inference_text: String, - creator_set_visibility: Option, + creator_set_visibility: Option, } #[derive(Serialize)] @@ -110,7 +111,14 @@ pub async fn infer_tts_handler( })?; let ip_address = get_request_ip(&http_request); - let creator_set_visibility = "public".to_string(); + + let maybe_user_preferred_visibility : Option = maybe_user_session + .as_ref() + .map(|user_session| user_session.preferred_tts_result_visibility); + + let set_visibility = request.creator_set_visibility + .or(maybe_user_preferred_visibility) + .unwrap_or(RecordVisibility::Public); // This token is returned to the client. let job_token = Tokens::new_tts_inference_job() @@ -141,7 +149,7 @@ SET inference_text, maybe_user_token, ip_address, - creator_set_visibility + set_visibility ) .execute(&server_state.mysql_pool) .await; diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs index f8bb1f7553..ebd85b241d 100644 --- a/src/http_server/endpoints/users/edit_profile.rs +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -54,8 +54,8 @@ pub struct EditProfileRequest { pub cashapp_username: Option, pub website_url: Option, - pub preferred_tts_result_visibility: Option, - pub preferred_w2l_result_visibility: Option, + pub preferred_tts_result_visibility: RecordVisibility, + pub preferred_w2l_result_visibility: RecordVisibility, } #[derive(Serialize)] @@ -285,6 +285,8 @@ UPDATE users SET profile_markdown = ?, profile_rendered_html = ?, + preferred_tts_result_visibility = ?, + preferred_w2l_result_visibility = ?, discord_username = ?, twitter_username = ?, twitch_username = ?, @@ -298,6 +300,8 @@ LIMIT 1 "#, profile_markdown, profile_html, + request.preferred_tts_result_visibility, + request.preferred_w2l_result_visibility, discord_username, twitter_username, twitch_username, diff --git a/src/http_server/web_utils/session_checker.rs b/src/http_server/web_utils/session_checker.rs index 5488b6b2b9..a7691877f4 100644 --- a/src/http_server/web_utils/session_checker.rs +++ b/src/http_server/web_utils/session_checker.rs @@ -1,6 +1,7 @@ use actix_web::HttpRequest; use anyhow::anyhow; use crate::AnyhowResult; +use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; use crate::http_server::web_utils::cookie_manager::CookieManager; use log::{info, warn}; @@ -30,6 +31,8 @@ pub struct SessionUserRecord { pub disable_gravatar: bool, pub auto_play_audio_preference: Option, + pub preferred_tts_result_visibility: RecordVisibility, + pub preferred_w2l_result_visibility: RecordVisibility, pub auto_play_video_preference: Option, // ===== ROLE ===== // @@ -77,6 +80,8 @@ pub struct SessionUserRawDbRecord { pub disable_gravatar: i8, pub auto_play_audio_preference: Option, pub auto_play_video_preference: Option, + pub preferred_tts_result_visibility: RecordVisibility, + pub preferred_w2l_result_visibility: RecordVisibility, pub user_role_slug: String, pub is_banned: i8, @@ -184,6 +189,8 @@ SELECT users.disable_gravatar, users.auto_play_audio_preference, users.auto_play_video_preference, + users.preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility`, + users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility`, users.user_role_slug, users.is_banned, @@ -234,10 +241,14 @@ WHERE user_sessions.token = ? email_address: raw_user_record.email_address.clone(), email_confirmed: i8_to_bool(raw_user_record.email_confirmed), email_gravatar_hash: raw_user_record.email_gravatar_hash.clone(), + // Preference disable_gravatar: i8_to_bool(raw_user_record.disable_gravatar), auto_play_audio_preference: nullable_i8_to_optional_bool(raw_user_record.auto_play_audio_preference), auto_play_video_preference: nullable_i8_to_optional_bool(raw_user_record.auto_play_video_preference), user_role_slug: raw_user_record.user_role_slug.clone(), + preferred_tts_result_visibility: raw_user_record.preferred_tts_result_visibility, + preferred_w2l_result_visibility: raw_user_record.preferred_w2l_result_visibility, + is_banned: i8_to_bool(raw_user_record.is_banned), // Usage From 04ea0d7d8cde704fe55ae43fe9e4520a0c9a2d61 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 11 Jul 2021 16:49:03 -0400 Subject: [PATCH 0481/1042] more propagation of visibility --- .../endpoints/w2l/enqueue_infer_w2l.rs | 5 +++-- .../w2l/enqueue_infer_w2l_with_uploads.rs | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index b1a7a7d8a8..0b259e7877 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -5,7 +5,8 @@ use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::helpers::enums::{DownloadUrlType, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -27,7 +28,7 @@ use std::sync::Arc; pub struct InferW2lRequest { w2l_template_token: Option, tts_inference_result_token: Option, - creator_set_visibility: Option, + creator_set_visibility: Option, } #[derive(Serialize)] diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 53161eb7a9..11efc44746 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -31,6 +31,7 @@ use sqlx::error::Error::Database; use std::io::Write; use std::sync::Arc; use uuid::Uuid; +use crate::database::enums::record_visibility::RecordVisibility; const BUCKET_AUDIO_FILE_NAME : &'static str = "input_audio_file"; const BUCKET_IMAGE_FILE_NAME: &'static str = "input_image_file"; @@ -39,8 +40,6 @@ const BUCKET_VIDEO_FILE_NAME : &'static str = "input_video_file"; const MIN_BYTES : usize = 10; const MAX_BYTES : usize = 1024 * 1024 * 20; - - /// Just to query for existence #[derive(Serialize)] pub struct W2lTemplateExistenceRecord { @@ -115,7 +114,7 @@ pub async fn enqueue_infer_w2l_with_uploads( let maybe_session = server_state .session_checker - .maybe_get_session(&http_request, &server_state.mysql_pool) + .maybe_get_user_session(&http_request, &server_state.mysql_pool) .await .map_err(|e| { warn!("Session checker error: {:?}", e); @@ -126,6 +125,13 @@ pub async fn enqueue_infer_w2l_with_uploads( .as_ref() .map(|user_session| user_session.user_token.to_string()); + let maybe_user_preferred_visibility : Option = maybe_session + .as_ref() + .map(|user_session| user_session.preferred_tts_result_visibility); + + let set_visibility = maybe_user_preferred_visibility + .unwrap_or(RecordVisibility::Public); + info!("Enqueue infer w2l by user token: {:?}", maybe_user_token); // ==================== READ MULTIPART REQUEST ==================== // @@ -291,7 +297,7 @@ SET maybe_creator_user_token = ?, creator_ip_address = ?, disable_end_bump = false, - creator_set_visibility = "public", + creator_set_visibility = ?, status = "pending" "#, job_token.to_string(), @@ -302,7 +308,8 @@ SET maybe_audio_file_name.clone(), Some(audio_type.clone()), maybe_user_token.clone(), - ip_address.to_string() + ip_address.to_string(), + set_visibility, ) .execute(&server_state.mysql_pool) .await; From 6bcd1033a15907eb9c4302ceb13b6a3693edc186 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 11 Jul 2021 17:12:35 -0400 Subject: [PATCH 0482/1042] jobs: creator visibility propagation --- src/job_queries/tts_inference_job_queries.rs | 8 +++++--- src/job_queries/w2l_inference_job_queries.rs | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 040ca6e8a5..7355efd09c 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -4,6 +4,7 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; +use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::tokens::Tokens; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; @@ -23,7 +24,7 @@ pub struct TtsInferenceJobRecord { pub creator_ip_address: String, pub maybe_creator_user_token: Option, - pub creator_set_visibility: String, // TODO + pub creator_set_visibility: RecordVisibility, pub status: String, // TODO pub attempt_count: i32, @@ -51,7 +52,7 @@ SELECT creator_ip_address, maybe_creator_user_token, - creator_set_visibility, + creator_set_visibility as `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, status, attempt_count, @@ -321,7 +322,7 @@ SET maybe_creator_synthetic_id = ?, creator_ip_address = ?, - creator_set_visibility = 'public', + creator_set_visibility = ?, public_bucket_wav_audio_path = ?, public_bucket_spectrogram_path = ?, @@ -339,6 +340,7 @@ SET maybe_creator_synthetic_id, job.creator_ip_address.clone(), + job.creator_set_visibility, bucket_audio_result_path, bucket_spectrogram_result_path, diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index 33702c7a83..d33ce20625 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -4,6 +4,7 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; +use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::tokens::Tokens; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; @@ -35,7 +36,7 @@ pub struct W2lInferenceJobRecord { pub creator_ip_address: String, pub maybe_creator_user_token: Option, - pub creator_set_visibility: String, // TODO + pub creator_set_visibility: RecordVisibility, pub disable_end_bump: i8, // bool pub disable_watermark: i8, // bool @@ -73,7 +74,7 @@ SELECT creator_ip_address, maybe_creator_user_token, - creator_set_visibility, + creator_set_visibility as `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, disable_end_bump, disable_watermark, @@ -333,7 +334,7 @@ SET maybe_w2l_template_token = ?, maybe_creator_user_token = ?, creator_ip_address = ?, - creator_set_visibility = 'public', + creator_set_visibility = ?, maybe_creator_synthetic_id = ?, @@ -349,6 +350,7 @@ SET job.maybe_w2l_template_token.clone(), job.maybe_creator_user_token.clone(), job.creator_ip_address.clone(), + job.creator_set_visibility, maybe_creator_synthetic_id, From cc959f88fd4ba332ae768670949f9cc6af02bf8a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 11 Jul 2021 19:14:26 -0400 Subject: [PATCH 0483/1042] hack record visibility --- src/database/enums/record_visibility.rs | 12 ++++++++++++ .../queries/list_tts_inference_results.rs | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/database/enums/record_visibility.rs b/src/database/enums/record_visibility.rs index d589f4cfe5..54cfe94006 100644 --- a/src/database/enums/record_visibility.rs +++ b/src/database/enums/record_visibility.rs @@ -1,5 +1,8 @@ //! These are columns where users can control the visibility of their data. +use anyhow::anyhow; +use crate::util::anyhow_result::AnyhowResult; + /// To use this in a query, the query must have type annotations. /// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375 /// eg. preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility` @@ -20,4 +23,13 @@ impl RecordVisibility { RecordVisibility::Private => "private", } } + + pub fn from_str(record_visibility: &str) -> AnyhowResult { + match record_visibility { + "public" => Ok(RecordVisibility::Public), + "hidden" => Ok(RecordVisibility::Hidden), + "private" => Ok(RecordVisibility::Private), + _ => Err(anyhow!("invalid value: {:?}", record_visibility)), + } + } } diff --git a/src/database/queries/list_tts_inference_results.rs b/src/database/queries/list_tts_inference_results.rs index fc39026961..fe7b94d47e 100644 --- a/src/database/queries/list_tts_inference_results.rs +++ b/src/database/queries/list_tts_inference_results.rs @@ -1,5 +1,6 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use crate::database::enums::record_visibility::RecordVisibility; use crate::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; @@ -34,6 +35,8 @@ pub struct TtsInferenceRecordForList { pub file_size_bytes: u32, pub duration_millis: u32, + pub visibility: RecordVisibility, + //pub model_is_mod_approved: bool, // converted //pub maybe_mod_user_token: Option, @@ -125,6 +128,7 @@ impl ListTtsResultsQueryBuilder { maybe_creator_result_id: r.maybe_creator_result_id.map(|v| v as u64).clone(), file_size_bytes: if r.file_size_bytes > 0 { r.file_size_bytes as u32 } else { 0 }, duration_millis: if r.duration_millis > 0 { r.duration_millis as u32 } else { 0 }, + visibility: RecordVisibility::from_str(&r.creator_set_visibility).unwrap_or(RecordVisibility::Public), created_at: r.created_at, updated_at: r.updated_at, } @@ -172,6 +176,12 @@ impl ListTtsResultsQueryBuilder { } pub fn build_query_string(&self) -> String { + // TODO: I haven't figured out how to get field name disambiguation and type coercion working here. + // (1) tts_results.creator_set_visibility `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, + // Query error: no column found for name: creator_set_visibility + // (2) creator_set_visibility `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, + // Column 'creator_set_visibility' in field list is ambiguous + // TODO/NB: Unfortunately SQLx can't statically typecheck this query let mut query = r#" SELECT @@ -190,6 +200,9 @@ SELECT tts_results.file_size_bytes, tts_results.duration_millis, + + tts_results.creator_set_visibility, + tts_results.created_at, tts_results.updated_at @@ -288,6 +301,9 @@ pub struct RawInternalTtsRecord { pub file_size_bytes : i64, pub duration_millis : i64, + + pub creator_set_visibility: String, + pub created_at: DateTime, pub updated_at: DateTime, } From cde4f33489180a2e0ed1821583421d0c40ca3aa5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 11 Jul 2021 19:19:37 -0400 Subject: [PATCH 0484/1042] added to w2l --- .../queries/list_w2l_inference_results.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/database/queries/list_w2l_inference_results.rs b/src/database/queries/list_w2l_inference_results.rs index 5506d2513f..327e098918 100644 --- a/src/database/queries/list_w2l_inference_results.rs +++ b/src/database/queries/list_w2l_inference_results.rs @@ -1,5 +1,6 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use crate::database::enums::record_visibility::RecordVisibility; use crate::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; @@ -37,6 +38,8 @@ pub struct W2lInferenceRecordForList { pub frame_height: u32, pub duration_millis: u32, + pub visibility: RecordVisibility, + //pub template_is_mod_approved: bool, // converted //pub maybe_mod_user_token: Option, @@ -145,6 +148,7 @@ impl ListW2lResultsQueryBuilder { frame_width: if r.frame_width > 0 { r.frame_width as u32 } else { 0 }, frame_height: if r.frame_height > 0 { r.frame_height as u32 } else { 0 }, duration_millis: if r.duration_millis > 0 { r.duration_millis as u32 } else { 0 }, + visibility: RecordVisibility::from_str(&r.creator_set_visibility).unwrap_or(RecordVisibility::Public), created_at: r.created_at, updated_at: r.updated_at, } @@ -192,6 +196,12 @@ impl ListW2lResultsQueryBuilder { } pub fn build_query_string(&self) -> String { + // TODO: I haven't figured out how to get field name disambiguation and type coercion working here. + // (1) w2l_results.creator_set_visibility `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, + // Query error: no column found for name: creator_set_visibility + // (2) creator_set_visibility `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, + // Column 'creator_set_visibility' in field list is ambiguous + // TODO/NB: Unfortunately SQLx can't statically typecheck this query let mut query = r#" SELECT @@ -213,6 +223,9 @@ SELECT w2l_results.frame_width, w2l_results.frame_height, w2l_results.duration_millis, + + w2l_results.creator_set_visibility, + w2l_results.created_at, w2l_results.updated_at @@ -332,6 +345,8 @@ pub struct RawInternalTtsRecord { pub frame_height: i32, pub duration_millis: i32, + pub creator_set_visibility: String, + //pub template_is_mod_approved: i8, // needs convert //pub maybe_mod_user_token: Option, From cd41803b7c26e7f2b1790585a4b75b851c148c9b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 11 Jul 2021 19:20:07 -0400 Subject: [PATCH 0485/1042] re-codegen (need to update FE to respect profile edit) --- sqlx-data.json | 2386 +++++++++++++++++++++++++----------------------- 1 file changed, 1219 insertions(+), 1167 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index fc0d476d12..5fcc2c5f25 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -38,16 +38,6 @@ ] } }, - "04e4b27791b94ee7232d364089a7a110984e18cf10fd60e3dea7449d53c40c00": { - "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n\n file_size_bytes = ?,\n duration_millis = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 12 - }, - "nullable": [] - } - }, "05c3ebf54cd8a5374d6f4bbe21b3c6027ce6314917af8acd0ed136d375d53342": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -242,6 +232,304 @@ ] } }, + "05e0a883bcf6c0563bbb7cad4a5bb357b502b7134d6633a04c993d9f74c8923c": { + "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n maybe_w2l_template_token,\n maybe_public_image_bucket_location,\n maybe_tts_inference_result_token,\n maybe_public_audio_bucket_hash,\n maybe_public_audio_bucket_location,\n\n maybe_original_audio_filename,\n maybe_original_audio_download_url,\n maybe_audio_mime_type,\n\n creator_ip_address,\n maybe_creator_user_token,\n\n creator_set_visibility as `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n disable_end_bump,\n disable_watermark,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM w2l_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "inference_job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_public_image_bucket_location", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_public_audio_bucket_hash", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 7, + "name": "maybe_public_audio_bucket_location", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "maybe_original_audio_filename", + "type_info": { + "type": "String", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 9, + "name": "maybe_original_audio_download_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 10, + "name": "maybe_audio_mime_type", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 12, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 13, + "name": "creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 14, + "name": "disable_end_bump", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "disable_watermark", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 17, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 18, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 19, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + false, + false, + false, + false, + false, + true, + false, + false, + true + ] + } + }, "09e655ea4b3504cbb3bc18b92c6c25861c941ba073e3aead414e560b05bd7638": { "query": "\nSELECT count(*) as count\nFROM tts_models\nWHERE\n creator_user_token = ?\nLIMIT 1\n ", "describe": { @@ -305,6 +593,16 @@ ] } }, + "0ede38519f58e65cceec9204b5a28780b3f32f7ba6efc68f5d33f46a5865d987": { + "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n\n file_size_bytes = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 13 + }, + "nullable": [] + } + }, "10d93ef1350115742d8dddb951f98dfebabf7df3e44577b97b52343f0650eba1": { "query": "\nSELECT\n badges.slug,\n badges.title,\n badges.description,\n badges.image_url,\n user_badges.created_at as user_created_at\n\nFROM badges\nJOIN user_badges\nON\n badges.slug = user_badges.badge_slug\nWHERE\n user_badges.user_token = ?\n ", "describe": { @@ -1298,6 +1596,200 @@ "nullable": [] } }, + "2c7795f817e475290e32a2bf1061cdbbf9f08c9bcdf8b03d92b860a5ce15985e": { + "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n model_token,\n raw_inference_text,\n\n creator_ip_address,\n maybe_creator_user_token,\n creator_set_visibility as `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM tts_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "inference_job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 8, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 9, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 10, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 11, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 12, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + true, + false, + false, + true + ] + } + }, "314904656a333caa75a8644e73f4d338b7bd6f68c7ea9e37b5bc7f7b257bf34d": { "query": "\nSELECT 1 as does_exist\nFROM user_badges\nWHERE\n user_token = ?\nAND\n badge_slug = ?\nLIMIT 1\n ", "describe": { @@ -1363,6 +1855,16 @@ "nullable": [] } }, + "369a7576edc128cf2b310648d6fc45a2191c1263d433fe6d1461d15dbf26690e": { + "query": "\nINSERT INTO w2l_results\nSET\n token = ?,\n maybe_w2l_template_token = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n maybe_creator_synthetic_id = ?,\n\n public_bucket_video_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 12 + }, + "nullable": [] + } + }, "376e36b1357952599866262c7356ba35f15ccdfb338d8ccdbdc06ccd69331fc3": { "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -1373,16 +1875,6 @@ "nullable": [] } }, - "3a2dc0a7e41dcbdf21f91d1a1e54e813f74b0bac7aea052302089cae195eeb53": { - "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 9 - }, - "nullable": [] - } - }, "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c": { "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -1587,6 +2079,434 @@ ] } }, + "45c86917f422dc22026ba4ba31251cbd784ff1f13709e533048c8a4e5c438485": { + "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_confirmed,\n users.email_gravatar_hash,\n\n users.disable_gravatar,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n users.preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n\n users.user_role_slug,\n users.is_banned,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l,\n user_roles.can_delete_own_tts_results,\n user_roles.can_delete_own_w2l_results,\n user_roles.can_delete_own_account,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n user_roles.can_delete_own_tts_models,\n user_roles.can_delete_own_w2l_templates,\n\n user_roles.can_approve_w2l_templates,\n user_roles.can_edit_other_users_profiles,\n user_roles.can_edit_other_users_tts_models,\n user_roles.can_edit_other_users_w2l_templates,\n user_roles.can_delete_other_users_tts_models,\n user_roles.can_delete_other_users_tts_results,\n user_roles.can_delete_other_users_w2l_templates,\n user_roles.can_delete_other_users_w2l_results,\n user_roles.can_ban_users,\n user_roles.can_delete_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "email_confirmed", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 5, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 7, + "name": "auto_play_audio_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "auto_play_video_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 10, + "name": "preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 11, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 12, + "name": "is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "can_use_tts", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "can_use_w2l", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "can_delete_own_tts_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "can_delete_own_w2l_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "can_delete_own_account", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 18, + "name": "can_upload_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 19, + "name": "can_upload_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 20, + "name": "can_delete_own_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 21, + "name": "can_delete_own_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 22, + "name": "can_approve_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 23, + "name": "can_edit_other_users_profiles", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 24, + "name": "can_edit_other_users_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 25, + "name": "can_edit_other_users_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 26, + "name": "can_delete_other_users_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 27, + "name": "can_delete_other_users_tts_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 28, + "name": "can_delete_other_users_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 29, + "name": "can_delete_other_users_w2l_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 30, + "name": "can_ban_users", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 31, + "name": "can_delete_users", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + } + }, "493917c2c4ee367c6fcad90ed1f6a114a8755e3f23287c3a6dbe3196299d5729": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.token = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -3641,16 +4561,6 @@ ] } }, - "8347b64356502ae32979d48680e6b2097e7f8c4cdc492b7285044856b58ba017": { - "query": "\nINSERT INTO w2l_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n maybe_w2l_template_token = ?,\n maybe_public_audio_bucket_hash = ?,\n maybe_public_audio_bucket_location = ?,\n\n maybe_original_audio_filename = ?,\n maybe_audio_mime_type = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n disable_end_bump = false,\n creator_set_visibility = \"public\",\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 9 - }, - "nullable": [] - } - }, "865ce236444a36d597939660b14eb4b0cee05c70fdb73f31c2fb8cddddfef42c": { "query": "\nINSERT INTO firehose_entries\nSET\n token = ?,\n event_type = ?,\n maybe_target_user_token = ?,\n maybe_target_entity_token = ?,\n maybe_created_entity_token = ?\n ", "describe": { @@ -4023,408 +4933,6 @@ "nullable": [] } }, - "9aded23dc849b7b416458afc8976207553e864690e36a8ee7066569ff489dc17": { - "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_confirmed,\n users.email_gravatar_hash,\n\n users.disable_gravatar,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n\n users.user_role_slug,\n users.is_banned,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l,\n user_roles.can_delete_own_tts_results,\n user_roles.can_delete_own_w2l_results,\n user_roles.can_delete_own_account,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n user_roles.can_delete_own_tts_models,\n user_roles.can_delete_own_w2l_templates,\n\n user_roles.can_approve_w2l_templates,\n user_roles.can_edit_other_users_profiles,\n user_roles.can_edit_other_users_tts_models,\n user_roles.can_edit_other_users_w2l_templates,\n user_roles.can_delete_other_users_tts_models,\n user_roles.can_delete_other_users_tts_results,\n user_roles.can_delete_other_users_w2l_templates,\n user_roles.can_delete_other_users_w2l_results,\n user_roles.can_ban_users,\n user_roles.can_delete_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "email_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "email_confirmed", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 5, - "name": "email_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "disable_gravatar", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 7, - "name": "auto_play_audio_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "auto_play_video_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "user_role_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 10, - "name": "is_banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 11, - "name": "can_use_tts", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "can_use_w2l", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "can_delete_own_tts_results", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "can_delete_own_w2l_results", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 15, - "name": "can_delete_own_account", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 16, - "name": "can_upload_tts_models", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 17, - "name": "can_upload_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 18, - "name": "can_delete_own_tts_models", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 19, - "name": "can_delete_own_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 20, - "name": "can_approve_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 21, - "name": "can_edit_other_users_profiles", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 22, - "name": "can_edit_other_users_tts_models", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 23, - "name": "can_edit_other_users_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 24, - "name": "can_delete_other_users_tts_models", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 25, - "name": "can_delete_other_users_tts_results", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 26, - "name": "can_delete_other_users_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 27, - "name": "can_delete_other_users_w2l_results", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 28, - "name": "can_ban_users", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 29, - "name": "can_delete_users", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true - ] - } - }, "9b607b814cd7c445c2c46ec5c13dc23f3a6a771a3469a786ed46ec099577d95e": { "query": "\nUPDATE\n ip_address_bans\nSET\n mod_user_token = ?,\n deleted_at = CURRENT_TIMESTAMP\nWHERE\n ip_address = ?\nLIMIT 1\n ", "describe": { @@ -4465,249 +4973,6 @@ "nullable": [] } }, - "a22a487d4651ec72b04fad3747dc1c25e47382e5481223a6a4fcf8172a737fb3": { - "query": "\nINSERT INTO w2l_results\nSET\n token = ?,\n maybe_w2l_template_token = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n maybe_creator_synthetic_id = ?,\n\n public_bucket_video_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n duration_millis = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 11 - }, - "nullable": [] - } - }, - "a88f0e74e4d30a8defdcafeed432451f1e4b32b2f8fbfa55d815b0c7cef40761": { - "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n is_banned,\n disable_gravatar,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n cashapp_username,\n website_url,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "email_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "profile_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 5, - "name": "profile_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 6, - "name": "user_role_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 7, - "name": "is_banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "disable_gravatar", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "discord_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 10, - "name": "twitch_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 11, - "name": "twitter_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 12, - "name": "patreon_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 13, - "name": "github_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 14, - "name": "cashapp_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 15, - "name": "website_url", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 16, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - false - ] - } - }, "ab6f25eb0f6340d49aa705835ed428434d8b6ab14b0740f6a5ebe7841b9ece20": { "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -5237,6 +5502,265 @@ "nullable": [] } }, + "c909600af05572fd5febb6aca20c3ae983fc75187587b197717f334f0139527e": { + "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n is_banned,\n disable_gravatar,\n preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n preferred_w2l_result_visibility as `preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n cashapp_username,\n website_url,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "profile_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "profile_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 6, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 7, + "name": "is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 10, + "name": "preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 11, + "name": "discord_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 12, + "name": "twitch_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 13, + "name": "twitter_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 14, + "name": "patreon_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 15, + "name": "github_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 16, + "name": "cashapp_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 17, + "name": "website_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 18, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + false + ] + } + }, "ca5efc3be0f09649df2add5270c06544133288a95ffcb9a0589efd4fe345efe7": { "query": "\nINSERT INTO badges ( slug, title, description, image_url )\nVALUES ( ?, ?, ?, ? )\n ", "describe": { @@ -5376,200 +5900,6 @@ ] } }, - "ccd9df6bff43a6c7554e18b4866bccd56a464117477edfa30bb0fc58309eed93": { - "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n model_token,\n raw_inference_text,\n\n creator_ip_address,\n maybe_creator_user_token,\n creator_set_visibility,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM tts_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "inference_job_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "uuid_idempotency_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 3, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "raw_inference_text", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 5, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 8, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 9, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 3 - } - }, - { - "ordinal": 10, - "name": "failure_reason", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 11, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 12, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "retry_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - true, - false, - false, - false, - true, - false, - false, - true - ] - } - }, "cf0b037e789ee09022b25f3381b0263c5cc6b59f3f328de4aa211bdb23a62629": { "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -5618,304 +5948,6 @@ ] } }, - "d14d0e03acc71081726ef0e31bade27110f1ac8eff9ccfe155df5dbd47d06560": { - "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n maybe_w2l_template_token,\n maybe_public_image_bucket_location,\n maybe_tts_inference_result_token,\n maybe_public_audio_bucket_hash,\n maybe_public_audio_bucket_location,\n\n maybe_original_audio_filename,\n maybe_original_audio_download_url,\n maybe_audio_mime_type,\n\n creator_ip_address,\n maybe_creator_user_token,\n\n creator_set_visibility,\n disable_end_bump,\n disable_watermark,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM w2l_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "inference_job_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "uuid_idempotency_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 3, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "maybe_public_image_bucket_location", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 5, - "name": "maybe_tts_inference_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "maybe_public_audio_bucket_hash", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 7, - "name": "maybe_public_audio_bucket_location", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 8, - "name": "maybe_original_audio_filename", - "type_info": { - "type": "String", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 9, - "name": "maybe_original_audio_download_url", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 10, - "name": "maybe_audio_mime_type", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 11, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 12, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 13, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 14, - "name": "disable_end_bump", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 15, - "name": "disable_watermark", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 16, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 17, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 3 - } - }, - { - "ordinal": 18, - "name": "failure_reason", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 19, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 21, - "name": "retry_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - false, - true, - false, - false, - false, - false, - false, - true, - false, - false, - true - ] - } - }, "d58f3bca0ba61fc48f8239f88b1bf4f7ba631ff3e5c02381da1dcccb73e7f20e": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", "describe": { @@ -6431,6 +6463,16 @@ "nullable": [] } }, + "d850784ca167e686b91fdd4d3cce20c7a00e4e38e4da629733114a1c55ba4ab3": { + "query": "\nINSERT INTO w2l_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n maybe_w2l_template_token = ?,\n maybe_public_audio_bucket_hash = ?,\n maybe_public_audio_bucket_location = ?,\n\n maybe_original_audio_filename = ?,\n maybe_audio_mime_type = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n disable_end_bump = false,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, "d862b69c0ded636bb491c6442958c8baf3457d80791eadf880445335e5fcda0b": { "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n\n w2l_results.creator_ip_address,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n AND w2l_results.user_deleted_at IS NULL\n AND w2l_results.mod_deleted_at IS NULL\n ", "describe": { @@ -7172,6 +7214,16 @@ ] } }, + "e49c75cab8eadb662ffc67e7cd0ebeec7157c97734a2aa8b238cf2b5e92d9351": { + "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n preferred_tts_result_visibility = ?,\n preferred_w2l_result_visibility = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 11 + }, + "nullable": [] + } + }, "e4e5be790f087b9aa6bee87e2315140bf621e4919362ebe72ab070163f18e9c8": { "query": "\nUPDATE w2l_templates\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { From 92949a719b85314f05ef60223b4b3ef7e4531993 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 7 Aug 2021 18:51:09 -0400 Subject: [PATCH 0486/1042] fix visibility db serialization bug --- models/tacotron2/vocodes_server.py | 0 src/http_server/endpoints/tts/enqueue_infer_tts.rs | 2 +- src/job_queries/tts_inference_job_queries.rs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 models/tacotron2/vocodes_server.py diff --git a/models/tacotron2/vocodes_server.py b/models/tacotron2/vocodes_server.py old mode 100644 new mode 100755 diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index ab985402f9..8c9059f446 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -149,7 +149,7 @@ SET inference_text, maybe_user_token, ip_address, - set_visibility + set_visibility.to_str(), ) .execute(&server_state.mysql_pool) .await; diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 7355efd09c..086539714b 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -340,7 +340,7 @@ SET maybe_creator_synthetic_id, job.creator_ip_address.clone(), - job.creator_set_visibility, + job.creator_set_visibility.to_str(), bucket_audio_result_path, bucket_spectrogram_result_path, From bf1067668021736b0ed5e708e4fc84d0301ffab8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 7 Aug 2021 18:51:57 -0400 Subject: [PATCH 0487/1042] re-codegen --- sqlx-data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlx-data.json b/sqlx-data.json index 5fcc2c5f25..c5e436d6d1 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1803,7 +1803,7 @@ "bits": 129 }, "char_set": 63, - "max_size": 1 + "max_size": 2 } } ], From e4aa009750788bb8a54b01a4cc74fdd97f28ad90 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 11 Aug 2021 17:44:15 -0400 Subject: [PATCH 0488/1042] result visibility editing fix --- .../endpoints/users/edit_profile.rs | 81 ++++++++++--------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs index ebd85b241d..0d8f823040 100644 --- a/src/http_server/endpoints/users/edit_profile.rs +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -54,8 +54,8 @@ pub struct EditProfileRequest { pub cashapp_username: Option, pub website_url: Option, - pub preferred_tts_result_visibility: RecordVisibility, - pub preferred_w2l_result_visibility: RecordVisibility, + pub preferred_tts_result_visibility: Option, + pub preferred_w2l_result_visibility: Option, } #[derive(Serialize)] @@ -244,44 +244,17 @@ pub async fn edit_profile_handler( let ip_address = get_request_ip(&http_request); + let preferred_tts_result_visibility = request.preferred_tts_result_visibility + .unwrap_or(RecordVisibility::Hidden); + + let preferred_w2l_result_visibility = request.preferred_w2l_result_visibility + .unwrap_or(RecordVisibility::Hidden); + let query_result = if editor_is_original_user { // We need to store the IP address details. sqlx::query!( r#" UPDATE users -SET - profile_markdown = ?, - profile_rendered_html = ?, - discord_username = ?, - twitter_username = ?, - twitch_username = ?, - github_username = ?, - cashapp_username = ?, - website_url = ?, - ip_address_last_update = ?, - version = version + 1 - -WHERE users.token = ? -LIMIT 1 - "#, - profile_markdown, - profile_html, - discord_username, - twitter_username, - twitch_username, - github_username, - cashapp_username, - website_url, - ip_address.clone(), - user_record.user_token.clone(), - ) - .execute(&server_state.mysql_pool) - .await - } else { - // We need to store the moderator details. - sqlx::query!( - r#" -UPDATE users SET profile_markdown = ?, profile_rendered_html = ?, @@ -293,6 +266,42 @@ SET github_username = ?, cashapp_username = ?, website_url = ?, + ip_address_last_update = ?, + version = version + 1 + +WHERE users.token = ? +LIMIT 1 + "#, + profile_markdown, + profile_html, + preferred_tts_result_visibility.to_str(), + preferred_w2l_result_visibility.to_str(), + discord_username, + twitter_username, + twitch_username, + github_username, + cashapp_username, + website_url, + ip_address.clone(), + user_record.user_token.clone(), + ) + .execute(&server_state.mysql_pool) + .await + } else { + // We need to store the moderator details. + // Also, mods shouldn't change user preferences. + sqlx::query!( + r#" +UPDATE users +SET + profile_markdown = ?, + profile_rendered_html = ?, + discord_username = ?, + twitter_username = ?, + twitch_username = ?, + github_username = ?, + cashapp_username = ?, + website_url = ?, version = version + 1 WHERE users.token = ? @@ -300,8 +309,6 @@ LIMIT 1 "#, profile_markdown, profile_html, - request.preferred_tts_result_visibility, - request.preferred_w2l_result_visibility, discord_username, twitter_username, twitch_username, From 3967e1236cbc24be9652f8862b620a37be53ecc9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 11 Aug 2021 17:45:16 -0400 Subject: [PATCH 0489/1042] fix codegen --- sqlx-data.json | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index c5e436d6d1..84b1a6bfed 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1875,6 +1875,16 @@ "nullable": [] } }, + "3a2dc0a7e41dcbdf21f91d1a1e54e813f74b0bac7aea052302089cae195eeb53": { + "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c": { "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -3202,16 +3212,6 @@ "nullable": [] } }, - "55a75db2d060bd49951db22f6b4ffd1545777b61c256030d59662543adc06ab2": { - "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n ip_address_last_update = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 10 - }, - "nullable": [] - } - }, "56a0a2bfc3c73d6845ca9cf1d285b175a40b50a250b7effb9ce6474e58113d6e": { "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_model_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_model_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", "describe": { @@ -7214,16 +7214,6 @@ ] } }, - "e49c75cab8eadb662ffc67e7cd0ebeec7157c97734a2aa8b238cf2b5e92d9351": { - "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n preferred_tts_result_visibility = ?,\n preferred_w2l_result_visibility = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 11 - }, - "nullable": [] - } - }, "e4e5be790f087b9aa6bee87e2315140bf621e4919362ebe72ab070163f18e9c8": { "query": "\nUPDATE w2l_templates\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -7686,6 +7676,16 @@ "nullable": [] } }, + "f75b713d166398ebea2bdbde4e416dc8eb2c48dba4aa9bd164bcf5bc77237ce1": { + "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n preferred_tts_result_visibility = ?,\n preferred_w2l_result_visibility = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n ip_address_last_update = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 12 + }, + "nullable": [] + } + }, "fb0d98378c747ade55cb68e7e32403c77a4e41878aea586a31e11bfd82a07627": { "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n\n tts_results.creator_ip_address,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n ", "describe": { From ce21551109ed0cf1bc41605189cb12df68e9c28e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 11 Aug 2021 19:26:28 -0400 Subject: [PATCH 0490/1042] fix types --- src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs | 2 +- src/job_queries/w2l_inference_job_queries.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 11efc44746..6dee654e52 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -309,7 +309,7 @@ SET Some(audio_type.clone()), maybe_user_token.clone(), ip_address.to_string(), - set_visibility, + set_visibility.to_str(), ) .execute(&server_state.mysql_pool) .await; diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index d33ce20625..0e77bd6c33 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -350,7 +350,7 @@ SET job.maybe_w2l_template_token.clone(), job.maybe_creator_user_token.clone(), job.creator_ip_address.clone(), - job.creator_set_visibility, + job.creator_set_visibility.to_str(), maybe_creator_synthetic_id, From 7738d45be048e6298d54526bd9cbd3f514cc25ac Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 11 Aug 2021 19:52:50 -0400 Subject: [PATCH 0491/1042] use no-op logger in other 3 jobs (already in TTS inference) --- src/tts_download_job.rs | 10 +++++++++- src/w2l_download_job.rs | 10 +++++++++- src/w2l_inference_job.rs | 10 +++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index b3194aac7f..838355823e 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -38,6 +38,7 @@ use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::{check_directory_exists, check_file_exists}; use crate::util::hashing::hash_file_sha2::hash_file_sha2; +use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; @@ -84,6 +85,9 @@ struct Downloader { // Sleep between batches pub job_batch_wait_millis: u64, + // How long to wait between log lines + pub no_op_logger_millis: u64, + // Max job attempts before failure. // NB: This is an i32 so we don't need to convert to db column type. pub job_max_attempts: i32, @@ -177,6 +181,7 @@ async fn main() -> AnyhowResult<()> { tts_check: tts_model_check_command, job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as i32, + no_op_logger_millis: common_env.no_op_logger_millis, }; main_loop(downloader).await; @@ -190,6 +195,8 @@ const INCREASE_TIMEOUT_MILLIS : u64 = 1000; async fn main_loop(downloader: Downloader) { let mut error_timeout_millis = START_TIMEOUT_MILLIS; + let mut noop_logger = NoOpLogger::new(downloader.no_op_logger_millis as i64); + loop { let num_records = 1; let query_result = query_tts_upload_job_records(&downloader.mysql_pool, num_records).await; @@ -205,7 +212,8 @@ async fn main_loop(downloader: Downloader) { }; if jobs.is_empty() { - info!("No jobs!"); + noop_logger.log_after_awhile(); + std::thread::sleep(Duration::from_millis(downloader.job_batch_wait_millis)); continue; } diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index bcc2028fb8..243a9a809e 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -41,6 +41,7 @@ use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; +use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; @@ -106,6 +107,9 @@ struct Downloader { // Sleep between batches pub job_batch_wait_millis: u64, + // How long to wait between log lines + pub no_op_logger_millis: u64, + // Max job attempts before failure. // NB: This is an i32 so we don't need to convert to db column type. pub job_max_attempts: i32, @@ -222,6 +226,7 @@ async fn main() -> AnyhowResult<()> { debug_job_end_sleep_millis: easyenv::get_env_num("DEBUG_JOB_END_SLEEP_MILLIS", 0)?, job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as i32, + no_op_logger_millis: common_env.no_op_logger_millis, }; main_loop(downloader).await; @@ -235,6 +240,8 @@ const INCREASE_TIMEOUT_MILLIS : u64 = 1000; async fn main_loop(downloader: Downloader) { let mut error_timeout_millis = START_TIMEOUT_MILLIS; + let mut noop_logger = NoOpLogger::new(downloader.no_op_logger_millis as i64); + loop { let num_records = 1; @@ -254,7 +261,8 @@ async fn main_loop(downloader: Downloader) { }; if jobs.is_empty() { - info!("No jobs!"); + noop_logger.log_after_awhile(); + std::thread::sleep(Duration::from_millis(downloader.job_batch_wait_millis)); continue; } diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 55ed7d4a70..2b0148358e 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -42,6 +42,7 @@ use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; +use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; @@ -103,6 +104,9 @@ struct Inferencer { // Sleep between batches pub job_batch_wait_millis: u64, + // How long to wait between log lines + pub no_op_logger_millis: u64, + // Max job attempts before failure. // NB: This is an i32 so we don't need to convert to db column type. pub job_max_attempts: i32, @@ -230,6 +234,7 @@ async fn main() -> AnyhowResult<()> { w2l_end_bump_filename, job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as i32, + no_op_logger_millis: common_env.no_op_logger_millis, }; main_loop(inferencer).await; @@ -243,6 +248,8 @@ const INCREASE_TIMEOUT_MILLIS : u64 = 1000; async fn main_loop(inferencer: Inferencer) { let mut error_timeout_millis = START_TIMEOUT_MILLIS; + let mut noop_logger = NoOpLogger::new(inferencer.no_op_logger_millis as i64); + loop { let num_records = 1; @@ -262,7 +269,8 @@ async fn main_loop(inferencer: Inferencer) { }; if jobs.is_empty() { - info!("No jobs!"); + noop_logger.log_after_awhile(); + std::thread::sleep(Duration::from_millis(inferencer.job_batch_wait_millis)); continue; } From a2f2763759fd944ab60da9086ba5f2ddb6c6f2fa Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 11 Aug 2021 20:28:57 -0400 Subject: [PATCH 0492/1042] Visibility in endpoints: don't list hidden results --- .../queries/list_tts_inference_results.rs | 69 +++++++++++++++-- .../queries/list_w2l_inference_results.rs | 75 ++++++++++++++++++- .../users/list_user_tts_inference_results.rs | 6 ++ .../users/list_user_w2l_inference_results.rs | 2 + src/tts_download_job.rs | 2 +- 5 files changed, 144 insertions(+), 10 deletions(-) diff --git a/src/database/queries/list_tts_inference_results.rs b/src/database/queries/list_tts_inference_results.rs index fe7b94d47e..cda1765345 100644 --- a/src/database/queries/list_tts_inference_results.rs +++ b/src/database/queries/list_tts_inference_results.rs @@ -49,6 +49,7 @@ pub struct TtsInferenceRecordForList { /// testability, construction, and correctness. pub struct ListTtsResultsQueryBuilder { scope_creator_username: Option, + include_user_hidden: bool, include_mod_disabled_results: bool, sort_ascending: bool, offset: Option, @@ -60,6 +61,7 @@ impl ListTtsResultsQueryBuilder { pub fn new() -> Self { Self { scope_creator_username: None, + include_user_hidden: false, include_mod_disabled_results: false, sort_ascending: false, offset: None, @@ -73,6 +75,11 @@ impl ListTtsResultsQueryBuilder { self } + pub fn include_user_hidden(mut self, include_user_hidden: bool) -> Self { + self.include_user_hidden = include_user_hidden; + self + } + pub fn include_mod_disabled_results(mut self, include_mod_disabled_results: bool) -> Self { self.include_mod_disabled_results = include_mod_disabled_results; self @@ -261,6 +268,15 @@ LEFT OUTER JOIN users } } + if !self.include_user_hidden { + if !first_predicate_added { + query.push_str(" WHERE tts_results.creator_set_visibility = 'public'"); + first_predicate_added = true; + } else { + query.push_str(" AND tts_results.creator_set_visibility = 'public'"); + } + } + if !self.include_mod_disabled_results { if !first_predicate_added { query.push_str(" WHERE tts_results.user_deleted_at IS NULL"); @@ -317,7 +333,8 @@ mod tests { let query_builder = ListTtsResultsQueryBuilder::new(); assert_eq!(&query_builder.build_predicates(), - " WHERE tts_results.user_deleted_at IS NULL \ + " WHERE tts_results.creator_set_visibility = 'public' \ + AND tts_results.user_deleted_at IS NULL \ AND tts_results.mod_deleted_at IS NULL \ ORDER BY tts_results.id DESC \ LIMIT ?"); @@ -330,19 +347,33 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE users.username = ? \ + AND tts_results.creator_set_visibility = 'public' \ AND tts_results.user_deleted_at IS NULL \ AND tts_results.mod_deleted_at IS NULL \ ORDER BY tts_results.id DESC \ LIMIT ?"); } + #[test] + fn predicates_including_user_hidden() { + let query_builder = ListTtsResultsQueryBuilder::new() + .include_user_hidden(true); + + assert_eq!(&query_builder.build_predicates(), + " WHERE tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id DESC \ + LIMIT ?"); + } + #[test] fn predicates_including_deleted() { let query_builder = ListTtsResultsQueryBuilder::new() .include_mod_disabled_results(true); assert_eq!(&query_builder.build_predicates(), - " ORDER BY tts_results.id DESC \ + " WHERE tts_results.creator_set_visibility = 'public' \ + ORDER BY tts_results.id DESC \ LIMIT ?"); } @@ -352,7 +383,8 @@ mod tests { .sort_ascending(true); assert_eq!(&query_builder.build_predicates(), - " WHERE tts_results.user_deleted_at IS NULL \ + " WHERE tts_results.creator_set_visibility = 'public' \ + AND tts_results.user_deleted_at IS NULL \ AND tts_results.mod_deleted_at IS NULL \ ORDER BY tts_results.id ASC \ LIMIT ?"); @@ -365,6 +397,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE tts_results.id < ? \ + AND tts_results.creator_set_visibility = 'public' \ AND tts_results.user_deleted_at IS NULL \ AND tts_results.mod_deleted_at IS NULL \ ORDER BY tts_results.id DESC \ @@ -379,6 +412,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE tts_results.id > ? \ + AND tts_results.creator_set_visibility = 'public' \ AND tts_results.user_deleted_at IS NULL \ AND tts_results.mod_deleted_at IS NULL \ ORDER BY tts_results.id ASC \ @@ -392,7 +426,8 @@ mod tests { // NB: Does not change the query itself! Just the downstream binding. assert_eq!(&query_builder.build_predicates(), - " WHERE tts_results.user_deleted_at IS NULL \ + " WHERE tts_results.creator_set_visibility = 'public' \ + AND tts_results.user_deleted_at IS NULL \ AND tts_results.mod_deleted_at IS NULL \ ORDER BY tts_results.id DESC \ LIMIT ?"); @@ -405,7 +440,8 @@ mod tests { // NB: Without a cursor, nothing happens. assert_eq!(&query_builder.build_predicates(), - " WHERE tts_results.user_deleted_at IS NULL \ + " WHERE tts_results.creator_set_visibility = 'public' \ + AND tts_results.user_deleted_at IS NULL \ AND tts_results.mod_deleted_at IS NULL \ ORDER BY tts_results.id DESC \ LIMIT ?"); @@ -420,6 +456,7 @@ mod tests { // NB: This will change the sort order and greater/less than direction! assert_eq!(&query_builder.build_predicates(), " WHERE tts_results.id > ? \ + AND tts_results.creator_set_visibility = 'public' \ AND tts_results.user_deleted_at IS NULL \ AND tts_results.mod_deleted_at IS NULL \ ORDER BY tts_results.id ASC \ @@ -436,6 +473,7 @@ mod tests { // NB: This will change the sort order and greater/less than direction! assert_eq!(&query_builder.build_predicates(), " WHERE tts_results.id < ? \ + AND tts_results.creator_set_visibility = 'public' \ AND tts_results.user_deleted_at IS NULL \ AND tts_results.mod_deleted_at IS NULL \ ORDER BY tts_results.id DESC \ @@ -451,6 +489,27 @@ mod tests { .cursor_is_reversed(true) .sort_ascending(true); + // NB: This will change the sort order and greater/less than direction! + assert_eq!(&query_builder.build_predicates(), + " WHERE tts_results.id < ? \ + AND users.username = ? \ + AND tts_results.creator_set_visibility = 'public' \ + AND tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_limit_scope_user_show_hidden_offset_cursor_is_reversed_sort_ascending() { + let query_builder = ListTtsResultsQueryBuilder::new() + .limit(1000) + .include_user_hidden(true) + .scope_creator_username(Some("pikachu")) + .offset(Some(100)) + .cursor_is_reversed(true) + .sort_ascending(true); + // NB: This will change the sort order and greater/less than direction! assert_eq!(&query_builder.build_predicates(), " WHERE tts_results.id < ? \ diff --git a/src/database/queries/list_w2l_inference_results.rs b/src/database/queries/list_w2l_inference_results.rs index 327e098918..ed60b82df0 100644 --- a/src/database/queries/list_w2l_inference_results.rs +++ b/src/database/queries/list_w2l_inference_results.rs @@ -52,6 +52,7 @@ pub struct W2lInferenceRecordForList { /// testability, construction, and correctness. pub struct ListW2lResultsQueryBuilder { scope_creator_username: Option, + include_user_hidden: bool, include_mod_deleted_results: bool, include_user_deleted_results: bool, include_templates_not_approved_for_public_listing: bool, @@ -65,6 +66,7 @@ impl ListW2lResultsQueryBuilder { pub fn new() -> Self { Self { scope_creator_username: None, + include_user_hidden: false, include_mod_deleted_results: false, include_user_deleted_results: false, include_templates_not_approved_for_public_listing: false, @@ -80,6 +82,11 @@ impl ListW2lResultsQueryBuilder { self } + pub fn include_user_hidden(mut self, include_user_hidden: bool) -> Self { + self.include_user_hidden = include_user_hidden; + self + } + pub fn include_mod_deleted_results(mut self, include_mod_deleted_results: bool) -> Self { self.include_mod_deleted_results = include_mod_deleted_results; self @@ -293,6 +300,15 @@ LEFT OUTER JOIN users } } + if !self.include_user_hidden { + if !first_predicate_added { + query.push_str(" WHERE w2l_results.creator_set_visibility = 'public'"); + first_predicate_added = true; + } else { + query.push_str(" AND w2l_results.creator_set_visibility = 'public'"); + } + } + if !self.include_mod_deleted_results { if !first_predicate_added { query.push_str(" WHERE w2l_results.mod_deleted_at IS NULL"); @@ -364,6 +380,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ AND w2l_results.mod_deleted_at IS NULL \ AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id DESC \ @@ -378,6 +395,20 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE users.username = ? \ AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_including_user_hidden() { + let query_builder = ListW2lResultsQueryBuilder::new() + .include_user_hidden(true); + + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ AND w2l_results.mod_deleted_at IS NULL \ AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id DESC \ @@ -391,6 +422,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id DESC \ LIMIT ?"); @@ -403,6 +435,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ AND w2l_results.mod_deleted_at IS NULL \ ORDER BY w2l_results.id DESC \ LIMIT ?"); @@ -416,6 +449,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ ORDER BY w2l_results.id DESC \ LIMIT ?"); } @@ -426,7 +460,8 @@ mod tests { .include_templates_not_approved_for_public_listing(true); assert_eq!(&query_builder.build_predicates(), - " WHERE w2l_results.mod_deleted_at IS NULL \ + " WHERE w2l_results.creator_set_visibility = 'public' \ + AND w2l_results.mod_deleted_at IS NULL \ AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id DESC \ LIMIT ?"); @@ -439,7 +474,8 @@ mod tests { .include_mod_deleted_results(true); assert_eq!(&query_builder.build_predicates(), - " WHERE w2l_results.user_deleted_at IS NULL \ + " WHERE w2l_results.creator_set_visibility = 'public' \ + AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id DESC \ LIMIT ?"); } @@ -451,7 +487,8 @@ mod tests { .include_user_deleted_results(true); assert_eq!(&query_builder.build_predicates(), - " WHERE w2l_results.mod_deleted_at IS NULL \ + " WHERE w2l_results.creator_set_visibility = 'public' \ + AND w2l_results.mod_deleted_at IS NULL \ ORDER BY w2l_results.id DESC \ LIMIT ?"); } @@ -464,7 +501,8 @@ mod tests { .include_templates_not_approved_for_public_listing(true); assert_eq!(&query_builder.build_predicates(), - " ORDER BY w2l_results.id DESC \ + " WHERE w2l_results.creator_set_visibility = 'public' \ + ORDER BY w2l_results.id DESC \ LIMIT ?"); } @@ -475,6 +513,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ AND w2l_results.mod_deleted_at IS NULL \ AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id ASC \ @@ -489,6 +528,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE w2l_results.id < ? \ AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ AND w2l_results.mod_deleted_at IS NULL \ AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id DESC \ @@ -504,6 +544,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE w2l_results.id > ? \ AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ AND w2l_results.mod_deleted_at IS NULL \ AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id ASC \ @@ -518,6 +559,7 @@ mod tests { // NB: Does not change the query itself! Just the downstream binding. assert_eq!(&query_builder.build_predicates(), " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ AND w2l_results.mod_deleted_at IS NULL \ AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id DESC \ @@ -532,6 +574,7 @@ mod tests { // NB: Without a cursor, nothing happens. assert_eq!(&query_builder.build_predicates(), " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ AND w2l_results.mod_deleted_at IS NULL \ AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id DESC \ @@ -548,6 +591,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE w2l_results.id > ? \ AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ AND w2l_results.mod_deleted_at IS NULL \ AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id ASC \ @@ -565,6 +609,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE w2l_results.id < ? \ AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ AND w2l_results.mod_deleted_at IS NULL \ AND w2l_results.user_deleted_at IS NULL \ ORDER BY w2l_results.id DESC \ @@ -580,6 +625,28 @@ mod tests { .cursor_is_reversed(true) .sort_ascending(true); + // NB: This will change the sort order and greater/less than direction! + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.id < ? \ + AND users.username = ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.creator_set_visibility = 'public' \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_limit_scope_user_show_hidden_offset_cursor_is_reversed_sort_ascending() { + let query_builder = ListW2lResultsQueryBuilder::new() + .limit(1000) + .include_user_hidden(true) + .scope_creator_username(Some("pikachu")) + .offset(Some(100)) + .cursor_is_reversed(true) + .sort_ascending(true); + // NB: This will change the sort order and greater/less than direction! assert_eq!(&query_builder.build_predicates(), " WHERE w2l_results.id < ? \ diff --git a/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/src/http_server/endpoints/users/list_user_tts_inference_results.rs index 4086f10b20..fe05114958 100644 --- a/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -90,11 +90,14 @@ pub async fn list_user_tts_inference_results_handler( ListTtsInferenceResultsForUserError::ServerError })?; + // Permissions & ACLs + let mut viewer_is_user = false; let mut is_mod_that_can_see_deleted = false; match maybe_user_session { None => {}, Some(session) => { + viewer_is_user = path.username == session.username; is_mod_that_can_see_deleted = session.can_delete_other_users_tts_results; }, }; @@ -117,9 +120,12 @@ pub async fn list_user_tts_inference_results_handler( None }; + let include_user_hidden = viewer_is_user || is_mod_that_can_see_deleted; + let mut query_builder = ListTtsResultsQueryBuilder::new() .sort_ascending(sort_ascending) .scope_creator_username(Some(path.username.as_ref())) + .include_user_hidden(include_user_hidden) .include_mod_disabled_results(is_mod_that_can_see_deleted) .limit(limit) .cursor_is_reversed(cursor_is_reversed) diff --git a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs index 5413459842..38dc924606 100644 --- a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs @@ -120,11 +120,13 @@ pub async fn list_user_w2l_inference_results_handler( None }; + let include_user_hidden = is_author || is_mod_that_can_see_deleted; let include_unapproved = is_author || is_mod_that_can_approve_w2l_templates; let mut query_builder = ListW2lResultsQueryBuilder::new() .sort_ascending(sort_ascending) .scope_creator_username(Some(path.username.as_ref())) + .include_user_hidden(include_user_hidden) .include_user_deleted_results(is_mod_that_can_see_deleted) .include_mod_deleted_results(is_mod_that_can_see_deleted) .include_templates_not_approved_for_public_listing(include_unapproved) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 838355823e..f5c7594071 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -213,7 +213,7 @@ async fn main_loop(downloader: Downloader) { if jobs.is_empty() { noop_logger.log_after_awhile(); - + std::thread::sleep(Duration::from_millis(downloader.job_batch_wait_millis)); continue; } From cf28558c274ed5465458d34631acb059e211a79a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 15 Aug 2021 22:26:32 -0400 Subject: [PATCH 0493/1042] return privacy on tts results --- src/database/helpers/enums.rs | 4 ++++ src/database/queries/query_tts_result.rs | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/database/helpers/enums.rs b/src/database/helpers/enums.rs index 876d9dc2d3..6e49c85ab7 100644 --- a/src/database/helpers/enums.rs +++ b/src/database/helpers/enums.rs @@ -4,13 +4,17 @@ #![allow(non_camel_case_types)] +#[deprecated(note = "Use `RecordVisibility` instead!")] #[derive(Deserialize)] pub enum CreatorSetVisibility { /// public + #[deprecated(note = "Use `RecordVisibility` instead!")] Public, /// hidden + #[deprecated(note = "Use `RecordVisibility` instead!")] Hidden, /// private + #[deprecated(note = "Use `RecordVisibility` instead!")] Private, } diff --git a/src/database/queries/query_tts_result.rs b/src/database/queries/query_tts_result.rs index e1a82097b3..0655c43430 100644 --- a/src/database/queries/query_tts_result.rs +++ b/src/database/queries/query_tts_result.rs @@ -11,6 +11,8 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::database::helpers::enums::CreatorSetVisibility; +use crate::database::enums::record_visibility::RecordVisibility; #[derive(Serialize)] pub struct TtsResultRecordForResponse { @@ -33,6 +35,8 @@ pub struct TtsResultRecordForResponse { pub public_bucket_wav_audio_path: String, pub public_bucket_spectrogram_path: String, + pub creator_set_visibility: RecordVisibility, + pub file_size_bytes: u32, pub duration_millis: u32, @@ -75,6 +79,8 @@ pub struct TtsResultRecordRaw { pub public_bucket_wav_audio_path: String, pub public_bucket_spectrogram_path: String, + pub creator_set_visibility: String, + pub file_size_bytes: i32, pub duration_millis: i32, @@ -141,6 +147,10 @@ pub async fn select_tts_result_by_token( public_bucket_wav_audio_path: ir.public_bucket_wav_audio_path.clone(), public_bucket_spectrogram_path: ir.public_bucket_spectrogram_path.clone(), + // NB: Fail open/public since we're already looking at it + creator_set_visibility: RecordVisibility::from_str(&ir.creator_set_visibility) + .unwrap_or(RecordVisibility::Public), + file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, @@ -185,6 +195,8 @@ SELECT tts_results.public_bucket_wav_audio_path, tts_results.public_bucket_spectrogram_path, + tts_results.creator_set_visibility, + tts_results.file_size_bytes, tts_results.duration_millis, tts_results.created_at, @@ -238,6 +250,8 @@ SELECT tts_results.public_bucket_wav_audio_path, tts_results.public_bucket_spectrogram_path, + tts_results.creator_set_visibility, + tts_results.file_size_bytes, tts_results.duration_millis, tts_results.created_at, From 20668c267b5de34ef01ea62474d2e98de3466fe7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 15 Aug 2021 22:39:20 -0400 Subject: [PATCH 0494/1042] show visibility for W2L results --- src/database/queries/query_tts_result.rs | 3 +-- src/database/queries/query_w2l_result.rs | 13 +++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/database/queries/query_tts_result.rs b/src/database/queries/query_tts_result.rs index 0655c43430..0b1b3b6bde 100644 --- a/src/database/queries/query_tts_result.rs +++ b/src/database/queries/query_tts_result.rs @@ -1,6 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::boolean_converters::i8_to_bool; use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use derive_more::{Display, Error}; @@ -11,8 +12,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::database::helpers::enums::CreatorSetVisibility; -use crate::database::enums::record_visibility::RecordVisibility; #[derive(Serialize)] pub struct TtsResultRecordForResponse { diff --git a/src/database/queries/query_w2l_result.rs b/src/database/queries/query_w2l_result.rs index c3497301fd..3f4bbe6cc7 100644 --- a/src/database/queries/query_w2l_result.rs +++ b/src/database/queries/query_w2l_result.rs @@ -11,6 +11,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::database::enums::record_visibility::RecordVisibility; #[derive(Serialize)] pub struct W2lResultRecordForResponse { @@ -33,6 +34,8 @@ pub struct W2lResultRecordForResponse { pub maybe_template_creator_display_name: Option, pub maybe_template_creator_gravatar_hash: Option, + pub creator_set_visibility: RecordVisibility, + pub file_size_bytes: u32, pub frame_width: u32, pub frame_height: u32, @@ -78,6 +81,8 @@ pub struct W2lResultRecordRaw { pub maybe_template_creator_display_name: Option, pub maybe_template_creator_gravatar_hash: Option, + pub creator_set_visibility: String, + pub file_size_bytes: i32, pub frame_width: i32, pub frame_height: i32, @@ -142,6 +147,10 @@ pub async fn select_w2l_result_by_token( maybe_template_creator_display_name: ir.maybe_template_creator_display_name.clone(), maybe_template_creator_gravatar_hash: ir.maybe_template_creator_gravatar_hash.clone(), + // NB: Fail open/public since we're already looking at it + creator_set_visibility: RecordVisibility::from_str(&ir.creator_set_visibility) + .unwrap_or(RecordVisibility::Public), + //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, @@ -189,6 +198,8 @@ SELECT template_users.display_name as maybe_template_creator_display_name, template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash, + w2l_results.creator_set_visibility, + w2l_results.file_size_bytes, w2l_results.frame_width, w2l_results.frame_height, @@ -243,6 +254,8 @@ SELECT template_users.display_name as maybe_template_creator_display_name, template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash, + w2l_results.creator_set_visibility, + w2l_results.file_size_bytes, w2l_results.frame_width, w2l_results.frame_height, From 94cfaba606b5c6a6450f94a895475e9b5536c53e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 16 Aug 2021 22:59:14 -0400 Subject: [PATCH 0495/1042] Add creator_set_visibility to TTS model and W2L template --- src/database/queries/query_tts_model.rs | 12 ++++++++++++ src/database/queries/query_w2l_template.rs | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/src/database/queries/query_tts_model.rs b/src/database/queries/query_tts_model.rs index 5eb2ab758d..9358b34796 100644 --- a/src/database/queries/query_tts_model.rs +++ b/src/database/queries/query_tts_model.rs @@ -1,6 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::boolean_converters::i8_to_bool; use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use derive_more::{Display, Error}; @@ -26,6 +27,8 @@ pub struct TtsModelRecordForResponse { pub description_markdown: String, pub description_rendered_html: String, + pub creator_set_visibility: RecordVisibility, + pub is_locked_from_use: bool, pub is_locked_from_user_modification: bool, @@ -59,6 +62,8 @@ pub struct TtsModelRecordRaw { pub description_markdown: String, pub description_rendered_html: String, + pub creator_set_visibility: String, + pub is_locked_from_use: i8, pub is_locked_from_user_modification: i8, @@ -110,6 +115,9 @@ pub async fn select_tts_model_by_token( title: model.title.clone(), description_markdown: model.description_markdown.clone(), description_rendered_html: model.description_rendered_html.clone(), + // NB: Fail open/public with creator_set_visibility since we're already looking at it + creator_set_visibility: RecordVisibility::from_str(&model.creator_set_visibility) + .unwrap_or(RecordVisibility::Public), is_locked_from_use: i8_to_bool(model.is_locked_from_use), is_locked_from_user_modification: i8_to_bool(model.is_locked_from_user_modification), created_at: model.created_at.clone(), @@ -145,6 +153,8 @@ SELECT tts.description_markdown, tts.description_rendered_html, + tts.creator_set_visibility, + tts.is_locked_from_use, tts.is_locked_from_user_modification, @@ -187,6 +197,8 @@ SELECT tts.description_markdown, tts.description_rendered_html, + tts.creator_set_visibility, + tts.is_locked_from_use, tts.is_locked_from_user_modification, diff --git a/src/database/queries/query_w2l_template.rs b/src/database/queries/query_w2l_template.rs index cba7f16fde..5b2f9b10d5 100644 --- a/src/database/queries/query_w2l_template.rs +++ b/src/database/queries/query_w2l_template.rs @@ -1,6 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::boolean_converters::i8_to_bool; use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use derive_more::{Display, Error}; @@ -25,6 +26,7 @@ pub struct W2lTemplateRecordForResponse { pub duration_millis: u32, pub maybe_image_object_name: Option, pub maybe_video_object_name: Option, + pub creator_set_visibility: RecordVisibility, pub is_public_listing_approved: Option, pub is_locked_from_use: bool, pub is_locked_from_user_modification: bool, @@ -56,6 +58,7 @@ pub struct W2lTemplateRecordRaw { pub duration_millis: i32, pub maybe_public_bucket_preview_image_object_name: Option, pub maybe_public_bucket_preview_video_object_name: Option, + pub creator_set_visibility: String, pub is_public_listing_approved: Option, pub is_locked_from_use: i8, pub is_locked_from_user_modification: i8, @@ -107,6 +110,9 @@ pub async fn select_w2l_template_by_token( duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), + // NB: Fail open/public with creator_set_visibility since we're already looking at it + creator_set_visibility: RecordVisibility::from_str(&template.creator_set_visibility) + .unwrap_or(RecordVisibility::Public), is_public_listing_approved: nullable_i8_to_optional_bool(template.is_public_listing_approved), is_locked_from_use: i8_to_bool(template.is_locked_from_use), is_locked_from_user_modification: i8_to_bool(template.is_locked_from_user_modification), @@ -142,6 +148,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, + w2l.creator_set_visibility, w2l.is_public_listing_approved, w2l.is_locked_from_use, w2l.is_locked_from_user_modification, @@ -181,6 +188,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, + w2l.creator_set_visibility, w2l.is_public_listing_approved, w2l.is_locked_from_use, w2l.is_locked_from_user_modification, From 5f71e30e2d90975e51a7df6786eb987eab4be1f2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 16 Aug 2021 22:59:56 -0400 Subject: [PATCH 0496/1042] re-codegen --- sqlx-data.json | 4170 +++++++++++++++++++++++++----------------------- 1 file changed, 2137 insertions(+), 2033 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 84b1a6bfed..21ac78b48d 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -530,6 +530,291 @@ ] } }, + "07d7053be9e10371e7ef83a307bf283392a388d4709c53dca7ad9b3411657766": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 12, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 16, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 17, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + true, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "09e655ea4b3504cbb3bc18b92c6c25861c941ba073e3aead414e560b05bd7638": { "query": "\nSELECT count(*) as count\nFROM tts_models\nWHERE\n creator_user_token = ?\nLIMIT 1\n ", "describe": { @@ -680,6 +965,304 @@ ] } }, + "1150dfa30863056960462f868b148241648fb6dfd94e8f3a8ae6b5da4f6f48fd": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.creator_set_visibility,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n\n tts_results.creator_ip_address,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n AND tts_results.user_deleted_at IS NULL\n AND tts_results.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 2, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "tts_model_title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "maybe_model_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "maybe_model_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 10, + "name": "maybe_model_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 11, + "name": "maybe_model_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 12, + "name": "public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "public_bucket_spectrogram_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 15, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 17, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 20, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "1168f39a5ec892d06e941b0dba0e955a27ef976567eeafd6e550bc42ea4c4331": { "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", "describe": { @@ -899,524 +1482,6 @@ "nullable": [] } }, - "1dedb79cc9ba347610bf37d3bf80145c12e90493741eb961f83aba13b61160b9": { - "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n\n tts_results.creator_ip_address,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n AND tts_results.user_deleted_at IS NULL\n AND tts_results.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tts_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "raw_inference_text", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 2, - "name": "tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "tts_model_title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 5, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, - "name": "maybe_model_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 9, - "name": "maybe_model_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 10, - "name": "maybe_model_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 11, - "name": "maybe_model_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 12, - "name": "public_bucket_wav_audio_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 13, - "name": "public_bucket_spectrogram_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 14, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 15, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 16, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 17, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 18, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 19, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, - "1f1605f78557a12e61e8e2f9a7d2888b71a95227fc91db2d1c2ef2a6dc857c34": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "text_preprocessing_algorithm", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 3, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "description_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 8, - "name": "description_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 9, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 11, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 12, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 14, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 15, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 16, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "25e7858d370ac1359deb42ef5292e85ccd1fe7debf3a1ba73303e5882523a4d0": { "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n download_url = ?,\n status = \"pending\"\n ", "describe": { @@ -1815,6 +1880,304 @@ ] } }, + "32d1eb129b7fd640c4f17c08eb3da1bb1dd8b057c9860cf1cb4949800b002dcd": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.creator_set_visibility,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n\n tts_results.creator_ip_address,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 2, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "tts_model_title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "maybe_model_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "maybe_model_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 10, + "name": "maybe_model_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 11, + "name": "maybe_model_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 12, + "name": "public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "public_bucket_spectrogram_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 15, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 17, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 20, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", "describe": { @@ -2517,278 +2880,6 @@ ] } }, - "493917c2c4ee367c6fcad90ed1f6a114a8755e3f23287c3a6dbe3196299d5729": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.token = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 7, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "is_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 8 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 15, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 16, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 17, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 18, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "4f61f621682ba255814f8a55455ea2b7fba9b6c85affaf87abfe2f81e11322f0": { "query": "\nSELECT\n id,\n status\nFROM tts_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", "describe": { @@ -2891,317 +2982,6 @@ ] } }, - "54d3d299b6f1f4068f8f79cc9fbff1484ed6c9dd27389cba0e40a99093164ea8": { - "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n\n w2l_results.creator_ip_address,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "w2l_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "maybe_tts_inference_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "public_bucket_video_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 3, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 392 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 5, - "name": "template_title", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 8, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 9, - "name": "maybe_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 10, - "name": "maybe_template_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 11, - "name": "maybe_template_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 12, - "name": "maybe_template_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 13, - "name": "maybe_template_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 14, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 15, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 16, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 17, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 18, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 21, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 22, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - true, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "551749bc153f9c8683a3a1ca5bb50744322aac046db34327501cf991485c4dee": { "query": "\nUPDATE tts_models\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -4561,6 +4341,330 @@ ] } }, + "7f2844bf439eb8499b5f92022cb60d6cf3a9abeba44540dbb7ea6b58876132f4": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.creator_set_visibility,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n\n w2l_results.creator_ip_address,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n AND w2l_results.user_deleted_at IS NULL\n AND w2l_results.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "w2l_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "public_bucket_video_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 392 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 5, + "name": "template_title", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 9, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "maybe_template_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "maybe_template_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 12, + "name": "maybe_template_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 13, + "name": "maybe_template_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 14, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 15, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 17, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 18, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 19, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 22, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 23, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "865ce236444a36d597939660b14eb4b0cee05c70fdb73f31c2fb8cddddfef42c": { "query": "\nINSERT INTO firehose_entries\nSET\n token = ?,\n event_type = ?,\n maybe_target_user_token = ?,\n maybe_target_entity_token = ?,\n maybe_created_entity_token = ?\n ", "describe": { @@ -4973,6 +5077,330 @@ "nullable": [] } }, + "a41f4fd69021a353de77b99a9c76fc6181a607abf94cbd9ac509a5a8226c1a24": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.creator_set_visibility,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n\n w2l_results.creator_ip_address,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "w2l_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "public_bucket_video_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 392 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 5, + "name": "template_title", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 9, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "maybe_template_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "maybe_template_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 12, + "name": "maybe_template_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 13, + "name": "maybe_template_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 14, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 15, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 17, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 18, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 19, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 22, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 23, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "ab6f25eb0f6340d49aa705835ed428434d8b6ab14b0740f6a5ebe7841b9ece20": { "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -5948,511 +6376,6 @@ ] } }, - "d58f3bca0ba61fc48f8239f88b1bf4f7ba631ff3e5c02381da1dcccb73e7f20e": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "text_preprocessing_algorithm", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 3, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "description_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 8, - "name": "description_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 9, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 11, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 12, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 14, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 15, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 16, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, - "d648f89461eb557770cbea6e18af4af2f2a3e2d34b536bf2f6b8b40c447a453c": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 7, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "is_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 8 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 15, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 16, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 17, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 18, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "d71ef2cfed337f9e3f9e1c2255bdec6347763910fd589482f0b0e56b794d7bb7": { "query": "\nUPDATE tts_models\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -6473,317 +6396,6 @@ "nullable": [] } }, - "d862b69c0ded636bb491c6442958c8baf3457d80791eadf880445335e5fcda0b": { - "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n\n w2l_results.creator_ip_address,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n AND w2l_results.user_deleted_at IS NULL\n AND w2l_results.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "w2l_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "maybe_tts_inference_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "public_bucket_video_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 3, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 392 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 5, - "name": "template_title", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 8, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 9, - "name": "maybe_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 10, - "name": "maybe_template_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 11, - "name": "maybe_template_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 12, - "name": "maybe_template_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 13, - "name": "maybe_template_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 14, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 15, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 16, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 17, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 18, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 21, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 22, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - true, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "da3197d279b94817c6c8a6277aff7a0cf44d66e6777f46680b1b07de48875dd5": { "query": "\nINSERT INTO user_badges\nSET\n user_token = ?,\n badge_slug = ?\n ", "describe": { @@ -7262,6 +6874,252 @@ ] } }, + "e79731cdcae40dd4ad37d8668a86811df45eb5e05cd27743a3758f91bf8f293c": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 8, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 9, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 10, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 11, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 15, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 16, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 17, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "e9b73703961516622f748fa597b62af6b991a7a36acf9901ca6d315ed2440674": { "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -7282,6 +7140,291 @@ "nullable": [] } }, + "ea262e5cea1bdb43faf69a9ebbec8d82a7f6ffd4a8c5f1254dc05263a59b3a9b": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.token = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 12, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 16, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 17, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + true, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "eadbe83c69c09f3e049ec7c760d1af0313cec8f3d82c0993e526738aa4d0a6dc": { "query": "\nSELECT\n ip_bans.ip_address,\n\n ip_bans.maybe_target_user_token,\n banned_users.username as maybe_target_username,\n\n ip_bans.mod_user_token,\n mod_users.username as mod_username,\n mod_users.display_name as mod_display_name,\n\n ip_bans.mod_notes,\n ip_bans.created_at,\n ip_bans.updated_at\nFROM\n ip_address_bans AS ip_bans\nLEFT OUTER JOIN users as banned_users\n ON ip_bans.maybe_target_user_token = banned_users.token\nJOIN users as mod_users\n ON ip_bans.mod_user_token = mod_users.token\nWHERE\n ip_bans.deleted_at IS NULL\n ", "describe": { @@ -7686,13 +7829,13 @@ "nullable": [] } }, - "fb0d98378c747ade55cb68e7e32403c77a4e41878aea586a31e11bfd82a07627": { - "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n\n tts_results.creator_ip_address,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n ", + "faafd5b65bb75e2fe993d11a6e12aefe6121702b40aea641aa7aab56799364b5": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "tts_result_token", + "name": "model_token", "type_info": { "type": "VarString", "flags": { @@ -7704,19 +7847,31 @@ }, { "ordinal": 1, - "name": "raw_inference_text", + "name": "tts_model_type", "type_info": { - "type": "Blob", + "type": "String", "flags": { - "bits": 4241 + "bits": 385 }, "char_set": 224, - "max_size": 262140 + "max_size": 60 } }, { "ordinal": 2, - "name": "tts_model_token", + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "creator_user_token", "type_info": { "type": "VarString", "flags": { @@ -7726,37 +7881,25 @@ "max_size": 128 } }, - { - "ordinal": 3, - "name": "tts_model_title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, { "ordinal": 4, - "name": "maybe_creator_user_token", + "name": "creator_username", "type_info": { "type": "VarString", "flags": { - "bits": 4228 + "bits": 4229 }, "char_set": 224, - "max_size": 128 + "max_size": 80 } }, { "ordinal": 5, - "name": "maybe_creator_username", + "name": "creator_display_name", "type_info": { "type": "VarString", "flags": { - "bits": 4228 + "bits": 4225 }, "char_set": 224, "max_size": 80 @@ -7764,126 +7907,78 @@ }, { "ordinal": 6, - "name": "maybe_creator_display_name", + "name": "title", "type_info": { "type": "VarString", "flags": { - "bits": 4224 + "bits": 4225 }, "char_set": 224, - "max_size": 80 + "max_size": 1020 } }, { "ordinal": 7, - "name": "maybe_creator_gravatar_hash", + "name": "description_markdown", "type_info": { - "type": "String", + "type": "Blob", "flags": { - "bits": 4224 + "bits": 4241 }, "char_set": 224, - "max_size": 128 + "max_size": 262140 } }, { "ordinal": 8, - "name": "maybe_model_creator_user_token", + "name": "description_rendered_html", "type_info": { - "type": "VarString", + "type": "Blob", "flags": { - "bits": 4228 + "bits": 4241 }, "char_set": 224, - "max_size": 128 + "max_size": 262140 } }, { "ordinal": 9, - "name": "maybe_model_creator_username", + "name": "creator_set_visibility", "type_info": { - "type": "VarString", + "type": "String", "flags": { - "bits": 4228 + "bits": 393 }, "char_set": 224, - "max_size": 80 + "max_size": 28 } }, { "ordinal": 10, - "name": "maybe_model_creator_display_name", + "name": "is_locked_from_use", "type_info": { - "type": "VarString", + "type": "Tiny", "flags": { - "bits": 4224 + "bits": 9 }, - "char_set": 224, - "max_size": 80 + "char_set": 63, + "max_size": 1 } }, { "ordinal": 11, - "name": "maybe_model_creator_gravatar_hash", + "name": "is_locked_from_user_modification", "type_info": { - "type": "String", + "type": "Tiny", "flags": { - "bits": 4224 + "bits": 9 }, - "char_set": 224, - "max_size": 128 + "char_set": 63, + "max_size": 1 } }, { "ordinal": 12, - "name": "public_bucket_wav_audio_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 13, - "name": "public_bucket_spectrogram_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 14, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 15, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 16, "name": "created_at", "type_info": { "type": "Timestamp", @@ -7895,7 +7990,7 @@ } }, { - "ordinal": 17, + "ordinal": 13, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -7907,8 +8002,8 @@ } }, { - "ordinal": 18, - "name": "creator_ip_address", + "ordinal": 14, + "name": "creator_ip_address_creation", "type_info": { "type": "VarString", "flags": { @@ -7919,24 +8014,36 @@ } }, { - "ordinal": 19, - "name": "user_deleted_at", + "ordinal": 15, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 16, + "name": "mod_deleted_at", "type_info": { "type": "Timestamp", "flags": { - "bits": 136 + "bits": 128 }, "char_set": 63, "max_size": 19 } }, { - "ordinal": 20, - "name": "mod_deleted_at", + "ordinal": 17, + "name": "user_deleted_at", "type_info": { "type": "Timestamp", "flags": { - "bits": 136 + "bits": 128 }, "char_set": 63, "max_size": 19 @@ -7950,15 +8057,12 @@ false, false, false, - true, - true, - true, - true, - true, - true, - true, - true, - true, + false, + false, + false, + false, + false, + false, false, false, false, From 2522bfca9156f1d8bf2e51c6f864a75932f5a42f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 17 Aug 2021 00:58:23 -0400 Subject: [PATCH 0497/1042] update creator set visibility in models/templates (untested) --- .../endpoints/tts/edit_tts_model.rs | 19 +++++++++++++++---- .../endpoints/w2l/edit_w2l_template.rs | 19 +++++++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/http_server/endpoints/tts/edit_tts_model.rs b/src/http_server/endpoints/tts/edit_tts_model.rs index 68029a2f5d..c02f3afb83 100644 --- a/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/src/http_server/endpoints/tts/edit_tts_model.rs @@ -6,6 +6,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::query_tts_model::select_tts_model_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -37,8 +38,8 @@ pub struct EditTtsModelRequest { pub title: Option, pub description_markdown: Option, + pub creator_set_visibility: Option, //pub updatable_slug: Option, - //pub creator_set_visibility: Option, //pub tts_model_type: Option, //pub text_preprocessing_algorithm: Option, //pub vocoder_token: Option, @@ -148,6 +149,7 @@ pub async fn edit_tts_model_handler( let mut title = None; let mut description_markdown = None; let mut description_html = None; + let mut creator_set_visibility = RecordVisibility::Public; if let Some(payload) = request.title.as_deref() { if contains_slurs(payload) { @@ -169,6 +171,11 @@ pub async fn edit_tts_model_handler( description_html = Some(html); } + if let Some(visibility) = request.creator_set_visibility.as_deref() { + creator_set_visibility = RecordVisibility::from_str(visibility) + .map_err(|_| EditTtsModelError::BadInput("bad record visibility".to_string()))?; + } + let ip_address = get_request_ip(&http_request); let query_result = if is_author { @@ -180,6 +187,7 @@ SET title = ?, description_markdown = ?, description_rendered_html = ?, + creator_set_visibility = ?, creator_ip_address_last_update = ?, version = version + 1 WHERE token = ? @@ -188,6 +196,7 @@ LIMIT 1 &title, &description_markdown, &description_html, + &creator_set_visibility.to_str(), &ip_address, &model_record.model_token, ) @@ -202,6 +211,7 @@ SET title = ?, description_markdown = ?, description_rendered_html = ?, + creator_set_visibility = ?, maybe_mod_user_token = ?, version = version + 1 WHERE token = ? @@ -210,6 +220,7 @@ LIMIT 1 &title, &description_markdown, &description_html, + &creator_set_visibility.to_str(), &user_session.user_token, &model_record.model_token, ) @@ -217,10 +228,10 @@ LIMIT 1 .await }; - // TODO: This is lazy and suboptimal af to query again + // TODO: This is lazy and suboptimal af to UPDATE again. // The reason we're doing this is because `sqlx` only does static type checking of queries - // with string literals. It does not support dynamic query building, thus the predicates - // must be held constant. :( + // with string literals. It does not support dynamic query building, thus the PREDICATES + // MUST BE HELD CONSTANT (at least in type signature). :( if is_mod { update_mod_details( &request, diff --git a/src/http_server/endpoints/w2l/edit_w2l_template.rs b/src/http_server/endpoints/w2l/edit_w2l_template.rs index a71e9ad0f8..5d5929e2b7 100644 --- a/src/http_server/endpoints/w2l/edit_w2l_template.rs +++ b/src/http_server/endpoints/w2l/edit_w2l_template.rs @@ -6,6 +6,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -36,8 +37,8 @@ pub struct EditW2lTemplateRequest { // ========== Author + Moderator options ========== pub title: Option, pub description_markdown: Option, + pub creator_set_visibility: Option, //pub updatable_slug: Option, - //pub creator_set_visibility: Option, // ========== Moderator options ========== @@ -143,6 +144,7 @@ pub async fn edit_w2l_template_handler( let mut title = None; let mut description_markdown = None; let mut description_html = None; + let mut creator_set_visibility = RecordVisibility::Public; if let Some(payload) = request.title.as_deref() { if contains_slurs(payload) { @@ -164,6 +166,11 @@ pub async fn edit_w2l_template_handler( description_html = Some(html); } + if let Some(visibility) = request.creator_set_visibility.as_deref() { + creator_set_visibility = RecordVisibility::from_str(visibility) + .map_err(|_| EditW2lTemplateError::BadInput("bad record visibility".to_string()))?; + } + let ip_address = get_request_ip(&http_request); let query_result = if is_author { @@ -175,6 +182,7 @@ SET title = ?, description_markdown = ?, description_rendered_html = ?, + creator_set_visibility = ?, creator_ip_address_last_update = ?, version = version + 1 WHERE token = ? @@ -183,6 +191,7 @@ LIMIT 1 &title, &description_markdown, &description_html, + &creator_set_visibility.to_str(), &ip_address, &template_record.template_token, ) @@ -197,6 +206,7 @@ SET title = ?, description_markdown = ?, description_rendered_html = ?, + creator_set_visibility = ?, maybe_mod_user_token = ?, version = version + 1 WHERE token = ? @@ -205,6 +215,7 @@ LIMIT 1 &title, &description_markdown, &description_html, + &creator_set_visibility.to_str(), &user_session.user_token, &template_record.template_token, ) @@ -220,10 +231,10 @@ LIMIT 1 } }; - // TODO: This is lazy and suboptimal af to query again + // TODO: This is lazy and suboptimal af to UPDATE again. // The reason we're doing this is because `sqlx` only does static type checking of queries - // with string literals. It does not support dynamic query building, thus the predicates - // must be held constant. :( + // with string literals. It does not support dynamic query building, thus the PREDICATES + // MUST BE HELD CONSTANT (at least in type signature). :( if is_mod { update_mod_details( &request, From b3ee7bb1f72c5b4886b6e9eb0b7de69bc25293a1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 17 Aug 2021 03:00:55 -0400 Subject: [PATCH 0498/1042] Update result visibility (new endpoint) --- .../endpoints/tts/edit_tts_model.rs | 10 +- .../endpoints/tts/edit_tts_result.rs | 186 ++++++++++++++++++ src/http_server/endpoints/tts/mod.rs | 1 + src/main.rs | 6 + 4 files changed, 198 insertions(+), 5 deletions(-) create mode 100644 src/http_server/endpoints/tts/edit_tts_result.rs diff --git a/src/http_server/endpoints/tts/edit_tts_model.rs b/src/http_server/endpoints/tts/edit_tts_model.rs index c02f3afb83..644c4b434f 100644 --- a/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/src/http_server/endpoints/tts/edit_tts_model.rs @@ -56,7 +56,7 @@ pub struct EditTtsModelRequest { pub enum EditTtsModelError { BadInput(String), NotAuthorized, - TemplateNotFound, + ModelNotFound, ServerError, } @@ -65,7 +65,7 @@ impl ResponseError for EditTtsModelError { match *self { EditTtsModelError::BadInput(_) => StatusCode::BAD_REQUEST, EditTtsModelError::NotAuthorized => StatusCode::UNAUTHORIZED, - EditTtsModelError::TemplateNotFound => StatusCode::NOT_FOUND, + EditTtsModelError::ModelNotFound => StatusCode::NOT_FOUND, EditTtsModelError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } } @@ -74,7 +74,7 @@ impl ResponseError for EditTtsModelError { let error_reason = match self { EditTtsModelError::BadInput(reason) => reason.to_string(), EditTtsModelError::NotAuthorized=> "unauthorized".to_string(), - EditTtsModelError::TemplateNotFound => "not found".to_string(), + EditTtsModelError::ModelNotFound => "not found".to_string(), EditTtsModelError::ServerError => "server error".to_string(), }; @@ -121,11 +121,11 @@ pub async fn edit_tts_model_handler( }, Ok(None) => { warn!("could not find model"); - return Err(EditTtsModelError::TemplateNotFound); + return Err(EditTtsModelError::ModelNotFound); }, Err(err) => { warn!("error looking up model: {:?}", err); - return Err(EditTtsModelError::TemplateNotFound); + return Err(EditTtsModelError::ModelNotFound); }, }; diff --git a/src/http_server/endpoints/tts/edit_tts_result.rs b/src/http_server/endpoints/tts/edit_tts_result.rs new file mode 100644 index 0000000000..cd57d8db75 --- /dev/null +++ b/src/http_server/endpoints/tts/edit_tts_result.rs @@ -0,0 +1,186 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::queries::query_tts_model::select_tts_model_by_token; +use crate::database::queries::query_tts_result::select_tts_result_by_token; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct EditTtsResultPathInfo { + token: String, +} + +#[derive(Deserialize)] +pub struct EditTtsResultRequest { + // ========== Author + Moderator options ========== + pub creator_set_visibility: Option, +} + +#[derive(Debug, Display)] +pub enum EditTtsResultError { + BadInput(String), + NotAuthorized, + ResultNotFound, + ServerError, +} + +impl ResponseError for EditTtsResultError { + fn status_code(&self) -> StatusCode { + match *self { + EditTtsResultError::BadInput(_) => StatusCode::BAD_REQUEST, + EditTtsResultError::NotAuthorized => StatusCode::UNAUTHORIZED, + EditTtsResultError::ResultNotFound => StatusCode::NOT_FOUND, + EditTtsResultError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + EditTtsResultError::BadInput(reason) => reason.to_string(), + EditTtsResultError::NotAuthorized=> "unauthorized".to_string(), + EditTtsResultError::ResultNotFound => "not found".to_string(), + EditTtsResultError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn edit_tts_inference_result_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EditTtsResultError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(EditTtsResultError::NotAuthorized); + } + }; + + // NB: Moderators can see deleted results. + // Original creators cannot see them (unless they're moderators!) + let show_deleted_results = user_session.can_delete_other_users_tts_results; + + // Moderators get to see all the fields. + let is_moderator = user_session.can_delete_other_users_tts_results + || user_session.can_edit_other_users_tts_models; // TODO: Not an exact permission fit + + let inference_result_query_result = select_tts_result_by_token( + &path.token, + show_deleted_results, + &server_state.mysql_pool + ).await; + + let mut inference_result = match inference_result_query_result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(EditTtsResultError::ServerError); + } + Ok(None) => return Err(EditTtsResultError::ResultNotFound), + Ok(Some(inference_result)) => inference_result, + }; + + // NB: Second set of permission checks + let mut is_author = false; + if let Some(creator_user_token) = inference_result.maybe_creator_user_token.as_deref() { + is_author = creator_user_token == &user_session.user_token; + } + + if !is_author && !is_moderator { + warn!("user is not allowed to edit result: {}", user_session.user_token); + return Err(EditTtsResultError::NotAuthorized); + } + + // Author + Mod fields. + // These fields must be present on all requests. + let mut creator_set_visibility = RecordVisibility::Public; + + if let Some(visibility) = request.creator_set_visibility.as_deref() { + creator_set_visibility = RecordVisibility::from_str(visibility) + .map_err(|_| EditTtsResultError::BadInput("bad record visibility".to_string()))?; + } + + let ip_address = get_request_ip(&http_request); + + let query_result = if is_author { + // TODO: Don't update the original IP address. Create a new field. + // We need to store the IP address details. + sqlx::query!( + r#" +UPDATE tts_results +SET + creator_set_visibility = ?, + creator_ip_address = ? +WHERE token = ? +LIMIT 1 + "#, + &creator_set_visibility.to_str(), + &ip_address, + &inference_result.tts_result_token, + ) + .execute(&server_state.mysql_pool) + .await + } else { + // We need to store the moderator details. + sqlx::query!( + r#" +UPDATE tts_results +SET + creator_set_visibility = ?, + maybe_mod_user_token = ? +WHERE token = ? +LIMIT 1 + "#, + &creator_set_visibility.to_str(), + &user_session.user_token, + &inference_result.tts_result_token, + ) + .execute(&server_state.mysql_pool) + .await + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update TTS result DB error: {:?}", err); + return Err(EditTtsResultError::ServerError); + } + }; + + Ok(simple_json_success()) +} diff --git a/src/http_server/endpoints/tts/mod.rs b/src/http_server/endpoints/tts/mod.rs index 6c8b4f674f..5d7e38d8c1 100644 --- a/src/http_server/endpoints/tts/mod.rs +++ b/src/http_server/endpoints/tts/mod.rs @@ -1,6 +1,7 @@ pub mod delete_tts_model; pub mod delete_tts_result; pub mod edit_tts_model; +pub mod edit_tts_result; pub mod enqueue_infer_tts; pub mod enqueue_upload_tts_model; pub mod get_tts_inference_job_status; diff --git a/src/main.rs b/src/main.rs index 89f16e8264..58c3bcd4ac 100644 --- a/src/main.rs +++ b/src/main.rs @@ -100,6 +100,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; +use crate::http_server::endpoints::tts::edit_tts_result::edit_tts_inference_result_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -432,6 +433,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_tts_inference_result_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/result/{token}/edit") + .route(web::post().to(edit_tts_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/result/{token}/delete") .route(web::post().to(delete_tts_inference_result_handler)) From fd0dcb95a417c944de628ab5c79d4184fae93114 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 17 Aug 2021 03:52:01 -0400 Subject: [PATCH 0499/1042] new endpoint to edit w2l result visibility (untested) --- .../endpoints/w2l/edit_w2l_result.rs | 185 ++++++++++++++++++ src/http_server/endpoints/w2l/mod.rs | 1 + src/main.rs | 8 +- 3 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 src/http_server/endpoints/w2l/edit_w2l_result.rs diff --git a/src/http_server/endpoints/w2l/edit_w2l_result.rs b/src/http_server/endpoints/w2l/edit_w2l_result.rs new file mode 100644 index 0000000000..cbda727582 --- /dev/null +++ b/src/http_server/endpoints/w2l/edit_w2l_result.rs @@ -0,0 +1,185 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::queries::query_w2l_result::select_w2l_result_by_token; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct EditW2lResultPathInfo { + token: String, +} + +#[derive(Deserialize)] +pub struct EditW2lResultRequest { + // ========== Author + Moderator options ========== + pub creator_set_visibility: Option, +} + +#[derive(Debug, Display)] +pub enum EditW2lResultError { + BadInput(String), + NotAuthorized, + ResultNotFound, + ServerError, +} + +impl ResponseError for EditW2lResultError { + fn status_code(&self) -> StatusCode { + match *self { + EditW2lResultError::BadInput(_) => StatusCode::BAD_REQUEST, + EditW2lResultError::NotAuthorized => StatusCode::UNAUTHORIZED, + EditW2lResultError::ResultNotFound => StatusCode::NOT_FOUND, + EditW2lResultError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + EditW2lResultError::BadInput(reason) => reason.to_string(), + EditW2lResultError::NotAuthorized=> "unauthorized".to_string(), + EditW2lResultError::ResultNotFound => "not found".to_string(), + EditW2lResultError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn edit_w2l_inference_result_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EditW2lResultError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(EditW2lResultError::NotAuthorized); + } + }; + + // NB: Moderators can see deleted results. + // Original creators cannot see them (unless they're moderators!) + let show_deleted_results = user_session.can_delete_other_users_w2l_results; + + // Moderators get to see all the fields. + let is_moderator = user_session.can_delete_other_users_w2l_results + || user_session.can_edit_other_users_w2l_templates; // TODO: Not an exact permission fit + + let inference_result_query_result = select_w2l_result_by_token( + &path.token, + show_deleted_results, + &server_state.mysql_pool + ).await; + + let mut inference_result = match inference_result_query_result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(EditW2lResultError::ServerError); + } + Ok(None) => return Err(EditW2lResultError::ResultNotFound), + Ok(Some(inference_result)) => inference_result, + }; + + // NB: Second set of permission checks + let mut is_author = false; + if let Some(creator_user_token) = inference_result.maybe_creator_user_token.as_deref() { + is_author = creator_user_token == &user_session.user_token; + } + + if !is_author && !is_moderator { + warn!("user is not allowed to edit result: {}", user_session.user_token); + return Err(EditW2lResultError::NotAuthorized); + } + + // Author + Mod fields. + // These fields must be present on all requests. + let mut creator_set_visibility = RecordVisibility::Public; + + if let Some(visibility) = request.creator_set_visibility.as_deref() { + creator_set_visibility = RecordVisibility::from_str(visibility) + .map_err(|_| EditW2lResultError::BadInput("bad record visibility".to_string()))?; + } + + let ip_address = get_request_ip(&http_request); + + let query_result = if is_author { + // TODO: Don't update the original IP address. Create a new field. + // We need to store the IP address details. + sqlx::query!( + r#" +UPDATE w2l_results +SET + creator_set_visibility = ?, + creator_ip_address = ? +WHERE token = ? +LIMIT 1 + "#, + &creator_set_visibility.to_str(), + &ip_address, + &inference_result.w2l_result_token, + ) + .execute(&server_state.mysql_pool) + .await + } else { + // We need to store the moderator details. + sqlx::query!( + r#" +UPDATE w2l_results +SET + creator_set_visibility = ?, + maybe_mod_user_token = ? +WHERE token = ? +LIMIT 1 + "#, + &creator_set_visibility.to_str(), + &user_session.user_token, + &inference_result.w2l_result_token, + ) + .execute(&server_state.mysql_pool) + .await + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update W2L result DB error: {:?}", err); + return Err(EditW2lResultError::ServerError); + } + }; + + Ok(simple_json_success()) +} diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index 79eeecaa10..0f55ac2244 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -1,5 +1,6 @@ pub mod delete_w2l_result; pub mod delete_w2l_template; +pub mod edit_w2l_result; pub mod edit_w2l_template; pub mod enqueue_infer_w2l; pub mod enqueue_infer_w2l_with_uploads; diff --git a/src/main.rs b/src/main.rs index 58c3bcd4ac..0a9c854b75 100644 --- a/src/main.rs +++ b/src/main.rs @@ -48,6 +48,7 @@ use crate::http_server::endpoints::root_index::get_root_index; use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handler; use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; +use crate::http_server::endpoints::tts::edit_tts_result::edit_tts_inference_result_handler; use crate::http_server::endpoints::tts::enqueue_infer_tts::infer_tts_handler; use crate::http_server::endpoints::tts::enqueue_upload_tts_model::upload_tts_model_handler; use crate::http_server::endpoints::tts::get_tts_inference_job_status::get_tts_inference_job_status_handler; @@ -68,6 +69,7 @@ use crate::http_server::endpoints::users::logout::logout_handler; use crate::http_server::endpoints::users::session_info::session_info_handler; use crate::http_server::endpoints::w2l::delete_w2l_result::delete_w2l_inference_result_handler; use crate::http_server::endpoints::w2l::delete_w2l_template::delete_w2l_template_handler; +use crate::http_server::endpoints::w2l::edit_w2l_result::edit_w2l_inference_result_handler; use crate::http_server::endpoints::w2l::edit_w2l_template::edit_w2l_template_handler; use crate::http_server::endpoints::w2l::enqueue_infer_w2l::infer_w2l_handler; use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; @@ -100,7 +102,6 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; -use crate::http_server::endpoints::tts::edit_tts_result::edit_tts_inference_result_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -502,6 +503,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_w2l_inference_result_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/result/{token}/edit") + .route(web::post().to(edit_w2l_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/result/{token}/delete") .route(web::post().to(delete_w2l_inference_result_handler)) From 7719262904e33b88de5ba88577082de554dcaaed Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 17 Aug 2021 03:52:44 -0400 Subject: [PATCH 0500/1042] codegen sql --- sqlx-data.json | 106 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 33 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 21ac78b48d..622e24584b 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -888,6 +888,26 @@ "nullable": [] } }, + "0efe809b1ce59d68c5e8fb43d42951b02ad7c17da8e7ef3d0812b75db9c73174": { + "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + } + }, + "0ff13b84956c2625050211a5bb2352a489ef8435571c364c8b2d1d21d936cbd4": { + "query": "\nUPDATE tts_results\nSET\n creator_set_visibility = ?,\n maybe_mod_user_token = ?\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, "10d93ef1350115742d8dddb951f98dfebabf7df3e44577b97b52343f0650eba1": { "query": "\nSELECT\n badges.slug,\n badges.title,\n badges.description,\n badges.image_url,\n user_badges.created_at as user_created_at\n\nFROM badges\nJOIN user_badges\nON\n badges.slug = user_badges.badge_slug\nWHERE\n user_badges.user_token = ?\n ", "describe": { @@ -1462,12 +1482,12 @@ "nullable": [] } }, - "18ca4c90132288c1dc3d43c6b0e7ee6aa12e990408e314e0038d4a376c9ddfbf": { - "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "1b1a5d54d20a834c73c7afa3d80681bdf7e0b8ca9767fe7ea0861cc25f10af54": { + "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { "columns": [], "parameters": { - "Right": 5 + "Right": 6 }, "nullable": [] } @@ -2178,6 +2198,16 @@ ] } }, + "33cd0c0b8eaaf494ef97232ac89775b8eb6a22d310824b7aa7db310dd4fd41b1": { + "query": "\nUPDATE w2l_results\nSET\n creator_set_visibility = ?,\n maybe_mod_user_token = ?\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", "describe": { @@ -3521,6 +3551,16 @@ ] } }, + "60c06a507349919addb5d26b1c4a283a04f66fce86e84ed0a51db6001c9fc2b5": { + "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + } + }, "641253d4c621f75037877651bba044a21033f50a5e3f8fed9b06ec967a508853": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -5047,6 +5087,16 @@ "nullable": [] } }, + "9cc5f697350814199344d3d9a0ff026e0d1e09e9097648400d4b63cfcbd31178": { + "query": "\nUPDATE w2l_results\nSET\n creator_set_visibility = ?,\n creator_ip_address = ?\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, "9cef4989671fb55d8e732f6f3d5c16aea7a9b3e01fc6a3bc7a56d0b066f07730": { "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address_creation = ?,\n creator_ip_address_last_update = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n file_size_bytes = ?\n ", "describe": { @@ -5920,16 +5970,6 @@ "nullable": [] } }, - "c63726afcf8d53a24597aad18bb44285eae8f5d056011811b066226f7db0198a": { - "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - } - }, "c909600af05572fd5febb6aca20c3ae983fc75187587b197717f334f0139527e": { "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n is_banned,\n disable_gravatar,\n preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n preferred_w2l_result_visibility as `preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n cashapp_username,\n website_url,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", "describe": { @@ -6199,6 +6239,16 @@ "nullable": [] } }, + "cc4372cdd1a65999cbaf008f200683eedc93ab83e17256e47e011667ba8384d8": { + "query": "\nUPDATE tts_results\nSET\n creator_set_visibility = ?,\n creator_ip_address = ?\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, "cc51b77e88c2ee5c046ea2f2287b9e9770743d50b0db90481662e5c1118885c5": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -6328,16 +6378,6 @@ ] } }, - "cf0b037e789ee09022b25f3381b0263c5cc6b59f3f328de4aa211bdb23a62629": { - "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - } - }, "d12757abef811cd6951e0e1427fdb496a8931b547519d320f5ade56007d3eb5d": { "query": "\nSELECT\n id,\n status\nFROM w2l_template_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", "describe": { @@ -6376,6 +6416,16 @@ ] } }, + "d2ef33027abc15b6ec00260f7d30a8c66a3190229ea103ecb571d4957af63525": { + "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + } + }, "d71ef2cfed337f9e3f9e1c2255bdec6347763910fd589482f0b0e56b794d7bb7": { "query": "\nUPDATE tts_models\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -7120,16 +7170,6 @@ ] } }, - "e9b73703961516622f748fa597b62af6b991a7a36acf9901ca6d315ed2440674": { - "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - } - }, "e9ec861d6eb135370f9baf3a866282ec5686d4b299fc4ec10900ab32ea8577d8": { "query": "\nUPDATE w2l_results\nSET\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { From c071bcfe1ebac4ec17e5e0e5009ff7bdfcf7b76a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 18 Aug 2021 23:27:26 -0400 Subject: [PATCH 0501/1042] return description for w2l templates --- src/database/queries/query_w2l_template.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/database/queries/query_w2l_template.rs b/src/database/queries/query_w2l_template.rs index 5b2f9b10d5..6c25a42d67 100644 --- a/src/database/queries/query_w2l_template.rs +++ b/src/database/queries/query_w2l_template.rs @@ -21,6 +21,8 @@ pub struct W2lTemplateRecordForResponse { pub creator_username: String, pub creator_display_name: String, pub title: String, + pub description_markdown: String, + pub description_rendered_html: String, pub frame_width: u32, pub frame_height: u32, pub duration_millis: u32, @@ -53,6 +55,8 @@ pub struct W2lTemplateRecordRaw { pub creator_username: String, pub creator_display_name: String, pub title: String, + pub description_markdown: String, + pub description_rendered_html: String, pub frame_width: i32, pub frame_height: i32, pub duration_millis: i32, @@ -105,6 +109,8 @@ pub async fn select_w2l_template_by_token( creator_username: template.creator_username.clone(), creator_display_name: template.creator_display_name.clone(), title: template.title.clone(), + description_markdown: template.description_markdown.clone(), + description_rendered_html: template.description_rendered_html.clone(), frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, @@ -143,6 +149,8 @@ SELECT users.username as creator_username, users.display_name as creator_display_name, w2l.title, + w2l.description_markdown, + w2l.description_rendered_html, w2l.frame_width, w2l.frame_height, w2l.duration_millis, @@ -183,6 +191,8 @@ SELECT users.username as creator_username, users.display_name as creator_display_name, w2l.title, + w2l.description_markdown, + w2l.description_rendered_html, w2l.frame_width, w2l.frame_height, w2l.duration_millis, From d1e5cd992397b32512225cf87a7cf939301e02a3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 18 Aug 2021 23:28:03 -0400 Subject: [PATCH 0502/1042] codegen db --- sqlx-data.json | 1192 +++++++++++++++++++++++++----------------------- 1 file changed, 622 insertions(+), 570 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 622e24584b..8055203f77 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -530,291 +530,6 @@ ] } }, - "07d7053be9e10371e7ef83a307bf283392a388d4709c53dca7ad9b3411657766": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 7, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 12, - "name": "is_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 8 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 15, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 16, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 17, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 18, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "09e655ea4b3504cbb3bc18b92c6c25861c941ba073e3aead414e560b05bd7638": { "query": "\nSELECT count(*) as count\nFROM tts_models\nWHERE\n creator_user_token = ?\nLIMIT 1\n ", "describe": { @@ -3621,6 +3336,317 @@ "nullable": [] } }, + "6d1293d70935baabc6eb84bae5297342aaf1f1814040a1ecd6a2c3b1aea5db61": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.description_markdown,\n w2l.description_rendered_html,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.token = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 7, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 8, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 14, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 18, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 19, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + true, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "73558104936aab6add2d9c789b368b5f155f1fe01d32e7314f2f120422866f54": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { @@ -7180,291 +7206,6 @@ "nullable": [] } }, - "ea262e5cea1bdb43faf69a9ebbec8d82a7f6ffd4a8c5f1254dc05263a59b3a9b": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.token = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 7, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 12, - "name": "is_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 8 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 15, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 16, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 17, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 18, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "eadbe83c69c09f3e049ec7c760d1af0313cec8f3d82c0993e526738aa4d0a6dc": { "query": "\nSELECT\n ip_bans.ip_address,\n\n ip_bans.maybe_target_user_token,\n banned_users.username as maybe_target_username,\n\n ip_bans.mod_user_token,\n mod_users.username as mod_username,\n mod_users.display_name as mod_display_name,\n\n ip_bans.mod_notes,\n ip_bans.created_at,\n ip_bans.updated_at\nFROM\n ip_address_bans AS ip_bans\nLEFT OUTER JOIN users as banned_users\n ON ip_bans.maybe_target_user_token = banned_users.token\nJOIN users as mod_users\n ON ip_bans.mod_user_token = mod_users.token\nWHERE\n ip_bans.deleted_at IS NULL\n ", "describe": { @@ -7859,6 +7600,317 @@ "nullable": [] } }, + "f5b9de16a3e6ca4caa4bc122d3f20f35f5c0cf648881488c0ae75391ac0cb8ab": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.description_markdown,\n w2l.description_rendered_html,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 7, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 8, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 14, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 18, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 19, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + true, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "f75b713d166398ebea2bdbde4e416dc8eb2c48dba4aa9bd164bcf5bc77237ce1": { "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n preferred_tts_result_visibility = ?,\n preferred_w2l_result_visibility = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n ip_address_last_update = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", "describe": { From 1a7fc2ab7277946d59af0376fc07a709f185092b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 20 Aug 2021 01:13:51 -0400 Subject: [PATCH 0503/1042] Add HifiGan model (https://github.com/SortAnon/hifi-gan.git) --- models/hifi-gan/LICENSE | 21 + models/hifi-gan/LJSpeech-1.1/training.txt | 12950 ++++++++++++++++++ models/hifi-gan/LJSpeech-1.1/validation.txt | 150 + models/hifi-gan/README.md | 105 + models/hifi-gan/audio_processing.py | 93 + models/hifi-gan/config_32k.json | 37 + models/hifi-gan/config_v1.json | 37 + models/hifi-gan/config_v1b.json | 37 + models/hifi-gan/config_v2.json | 37 + models/hifi-gan/config_v3.json | 37 + models/hifi-gan/denoiser.py | 36 + models/hifi-gan/env.py | 15 + models/hifi-gan/hifiutils.py | 58 + models/hifi-gan/inference.py | 95 + models/hifi-gan/inference_e2e.py | 93 + models/hifi-gan/meldataset.py | 168 + models/hifi-gan/models.py | 283 + models/hifi-gan/requirements.txt | 7 + models/hifi-gan/stft.py | 141 + models/hifi-gan/train.py | 280 + models/hifi-gan/validation_loss.png | Bin 0 -> 10995 bytes 21 files changed, 14680 insertions(+) create mode 100644 models/hifi-gan/LICENSE create mode 100644 models/hifi-gan/LJSpeech-1.1/training.txt create mode 100644 models/hifi-gan/LJSpeech-1.1/validation.txt create mode 100644 models/hifi-gan/README.md create mode 100644 models/hifi-gan/audio_processing.py create mode 100644 models/hifi-gan/config_32k.json create mode 100644 models/hifi-gan/config_v1.json create mode 100644 models/hifi-gan/config_v1b.json create mode 100644 models/hifi-gan/config_v2.json create mode 100644 models/hifi-gan/config_v3.json create mode 100644 models/hifi-gan/denoiser.py create mode 100644 models/hifi-gan/env.py create mode 100644 models/hifi-gan/hifiutils.py create mode 100644 models/hifi-gan/inference.py create mode 100644 models/hifi-gan/inference_e2e.py create mode 100644 models/hifi-gan/meldataset.py create mode 100644 models/hifi-gan/models.py create mode 100644 models/hifi-gan/requirements.txt create mode 100644 models/hifi-gan/stft.py create mode 100644 models/hifi-gan/train.py create mode 100644 models/hifi-gan/validation_loss.png diff --git a/models/hifi-gan/LICENSE b/models/hifi-gan/LICENSE new file mode 100644 index 0000000000..5afae394d6 --- /dev/null +++ b/models/hifi-gan/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Jungil Kong + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/models/hifi-gan/LJSpeech-1.1/training.txt b/models/hifi-gan/LJSpeech-1.1/training.txt new file mode 100644 index 0000000000..a61c09e730 --- /dev/null +++ b/models/hifi-gan/LJSpeech-1.1/training.txt @@ -0,0 +1,12950 @@ +LJ001-0001|Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition|Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition +LJ001-0002|in being comparatively modern.|in being comparatively modern. +LJ001-0003|For although the Chinese took impressions from wood blocks engraved in relief for centuries before the woodcutters of the Netherlands, by a similar process|For although the Chinese took impressions from wood blocks engraved in relief for centuries before the woodcutters of the Netherlands, by a similar process +LJ001-0004|produced the block books, which were the immediate predecessors of the true printed book,|produced the block books, which were the immediate predecessors of the true printed book, +LJ001-0005|the invention of movable metal letters in the middle of the fifteenth century may justly be considered as the invention of the art of printing.|the invention of movable metal letters in the middle of the fifteenth century may justly be considered as the invention of the art of printing. +LJ001-0006|And it is worth mention in passing that, as an example of fine typography,|And it is worth mention in passing that, as an example of fine typography, +LJ001-0007|the earliest book printed with movable types, the Gutenberg, or "forty-two line Bible" of about 1455,|the earliest book printed with movable types, the Gutenberg, or "forty-two line Bible" of about fourteen fifty-five, +LJ001-0008|has never been surpassed.|has never been surpassed. +LJ001-0009|Printing, then, for our purpose, may be considered as the art of making books by means of movable types.|Printing, then, for our purpose, may be considered as the art of making books by means of movable types. +LJ001-0010|Now, as all books not primarily intended as picture-books consist principally of types composed to form letterpress,|Now, as all books not primarily intended as picture-books consist principally of types composed to form letterpress, +LJ001-0011|it is of the first importance that the letter used should be fine in form;|it is of the first importance that the letter used should be fine in form; +LJ001-0012|especially as no more time is occupied, or cost incurred, in casting, setting, or printing beautiful letters|especially as no more time is occupied, or cost incurred, in casting, setting, or printing beautiful letters +LJ001-0013|than in the same operations with ugly ones.|than in the same operations with ugly ones. +LJ001-0014|And it was a matter of course that in the Middle Ages, when the craftsmen took care that beautiful form should always be a part of their productions whatever they were,|And it was a matter of course that in the Middle Ages, when the craftsmen took care that beautiful form should always be a part of their productions whatever they were, +LJ001-0015|the forms of printed letters should be beautiful, and that their arrangement on the page should be reasonable and a help to the shapeliness of the letters themselves.|the forms of printed letters should be beautiful, and that their arrangement on the page should be reasonable and a help to the shapeliness of the letters themselves. +LJ001-0016|The Middle Ages brought calligraphy to perfection, and it was natural therefore|The Middle Ages brought calligraphy to perfection, and it was natural therefore +LJ001-0017|that the forms of printed letters should follow more or less closely those of the written character, and they followed them very closely.|that the forms of printed letters should follow more or less closely those of the written character, and they followed them very closely. +LJ001-0018|The first books were printed in black letter, i.e. the letter which was a Gothic development of the ancient Roman character,|The first books were printed in black letter, i.e. the letter which was a Gothic development of the ancient Roman character, +LJ001-0019|and which developed more completely and satisfactorily on the side of the "lower-case" than the capital letters;|and which developed more completely and satisfactorily on the side of the "lower-case" than the capital letters; +LJ001-0020|the "lower-case" being in fact invented in the early Middle Ages.|the "lower-case" being in fact invented in the early Middle Ages. +LJ001-0021|The earliest book printed with movable type, the aforesaid Gutenberg Bible, is printed in letters which are an exact imitation|The earliest book printed with movable type, the aforesaid Gutenberg Bible, is printed in letters which are an exact imitation +LJ001-0022|of the more formal ecclesiastical writing which obtained at that time; this has since been called "missal type,"|of the more formal ecclesiastical writing which obtained at that time; this has since been called "missal type," +LJ001-0023|and was in fact the kind of letter used in the many splendid missals, psalters, etc., produced by printing in the fifteenth century.|and was in fact the kind of letter used in the many splendid missals, psalters, etc., produced by printing in the fifteenth century. +LJ001-0024|But the first Bible actually dated (which also was printed at Maintz by Peter Schoeffer in the year 1462)|But the first Bible actually dated (which also was printed at Maintz by Peter Schoeffer in the year fourteen sixty-two) +LJ001-0025|imitates a much freer hand, simpler, rounder, and less spiky, and therefore far pleasanter and easier to read.|imitates a much freer hand, simpler, rounder, and less spiky, and therefore far pleasanter and easier to read. +LJ001-0026|On the whole the type of this book may be considered the ne-plus-ultra of Gothic type,|On the whole the type of this book may be considered the ne-plus-ultra of Gothic type, +LJ001-0027|especially as regards the lower-case letters; and type very similar was used during the next fifteen or twenty years not only by Schoeffer,|especially as regards the lower-case letters; and type very similar was used during the next fifteen or twenty years not only by Schoeffer, +LJ001-0029|But though on the whole, except in Italy, Gothic letter was most often used|But though on the whole, except in Italy, Gothic letter was most often used +LJ001-0030|a very few years saw the birth of Roman character not only in Italy, but in Germany and France.|a very few years saw the birth of Roman character not only in Italy, but in Germany and France. +LJ001-0031|In 1465 Sweynheim and Pannartz began printing in the monastery of Subiaco near Rome,|In fourteen sixty-five Sweynheim and Pannartz began printing in the monastery of Subiaco near Rome, +LJ001-0032|and used an exceedingly beautiful type, which is indeed to look at a transition between Gothic and Roman,|and used an exceedingly beautiful type, which is indeed to look at a transition between Gothic and Roman, +LJ001-0033|but which must certainly have come from the study of the twelfth or even the eleventh century MSS.|but which must certainly have come from the study of the twelfth or even the eleventh century MSS. +LJ001-0034|They printed very few books in this type, three only; but in their very first books in Rome, beginning with the year 1468,|They printed very few books in this type, three only; but in their very first books in Rome, beginning with the year fourteen sixty-eight, +LJ001-0035|they discarded this for a more completely Roman and far less beautiful letter.|they discarded this for a more completely Roman and far less beautiful letter. +LJ001-0036|But about the same year Mentelin at Strasburg began to print in a type which is distinctly Roman;|But about the same year Mentelin at Strasburg began to print in a type which is distinctly Roman; +LJ001-0037|and the next year Gunther Zeiner at Augsburg followed suit;|and the next year Gunther Zeiner at Augsburg followed suit; +LJ001-0038|while in 1470 at Paris Udalric Gering and his associates turned out the first books printed in France, also in Roman character.|while in fourteen seventy at Paris Udalric Gering and his associates turned out the first books printed in France, also in Roman character. +LJ001-0039|The Roman type of all these printers is similar in character,|The Roman type of all these printers is similar in character, +LJ001-0040|and is very simple and legible, and unaffectedly designed for use; but it is by no means without beauty.|and is very simple and legible, and unaffectedly designed for use; but it is by no means without beauty. +LJ001-0041|It must be said that it is in no way like the transition type of Subiaco,|It must be said that it is in no way like the transition type of Subiaco, +LJ001-0042|and though more Roman than that, yet scarcely more like the complete Roman type of the earliest printers of Rome.|and though more Roman than that, yet scarcely more like the complete Roman type of the earliest printers of Rome. +LJ001-0043|A further development of the Roman letter took place at Venice.|A further development of the Roman letter took place at Venice. +LJ001-0044|John of Spires and his brother Vindelin, followed by Nicholas Jenson, began to print in that city,|John of Spires and his brother Vindelin, followed by Nicholas Jenson, began to print in that city, +LJ001-0045|1469, 1470;|fourteen sixty-nine, fourteen seventy; +LJ001-0046|their type is on the lines of the German and French rather than of the Roman printers.|their type is on the lines of the German and French rather than of the Roman printers. +LJ001-0047|Of Jenson it must be said that he carried the development of Roman type as far as it can go:|Of Jenson it must be said that he carried the development of Roman type as far as it can go: +LJ001-0048|his letter is admirably clear and regular, but at least as beautiful as any other Roman type.|his letter is admirably clear and regular, but at least as beautiful as any other Roman type. +LJ001-0049|After his death in the "fourteen eighties," or at least by 1490, printing in Venice had declined very much;|After his death in the "fourteen eighties," or at least by fourteen ninety, printing in Venice had declined very much; +LJ001-0050|and though the famous family of Aldus restored its technical excellence, rejecting battered letters,|and though the famous family of Aldus restored its technical excellence, rejecting battered letters, +LJ001-0051|and paying great attention to the "press work" or actual process of printing,|and paying great attention to the "press work" or actual process of printing, +LJ001-0052|yet their type is artistically on a much lower level than Jenson's, and in fact|yet their type is artistically on a much lower level than Jenson's, and in fact +LJ001-0053|they must be considered to have ended the age of fine printing in Italy.|they must be considered to have ended the age of fine printing in Italy. +LJ001-0054|Jenson, however, had many contemporaries who used beautiful type,|Jenson, however, had many contemporaries who used beautiful type, +LJ001-0055|some of which -- as, e.g., that of Jacobus Rubeus or Jacques le Rouge -- is scarcely distinguishable from his.|some of which -- as, e.g., that of Jacobus Rubeus or Jacques le Rouge -- is scarcely distinguishable from his. +LJ001-0056|It was these great Venetian printers, together with their brethren of Rome, Milan,|It was these great Venetian printers, together with their brethren of Rome, Milan, +LJ001-0057|Parma, and one or two other cities, who produced the splendid editions of the Classics, which are one of the great glories of the printer's art,|Parma, and one or two other cities, who produced the splendid editions of the Classics, which are one of the great glories of the printer's art, +LJ001-0058|and are worthy representatives of the eager enthusiasm for the revived learning of that epoch. By far,|and are worthy representatives of the eager enthusiasm for the revived learning of that epoch. By far, +LJ001-0059|the greater part of these Italian printers, it should be mentioned, were Germans or Frenchmen, working under the influence of Italian opinion and aims.|the greater part of these Italian printers, it should be mentioned, were Germans or Frenchmen, working under the influence of Italian opinion and aims. +LJ001-0060|It must be understood that through the whole of the fifteenth and the first quarter of the sixteenth centuries|It must be understood that through the whole of the fifteenth and the first quarter of the sixteenth centuries +LJ001-0061|the Roman letter was used side by side with the Gothic.|the Roman letter was used side by side with the Gothic. +LJ001-0062|Even in Italy most of the theological and law books were printed in Gothic letter,|Even in Italy most of the theological and law books were printed in Gothic letter, +LJ001-0063|which was generally more formally Gothic than the printing of the German workmen,|which was generally more formally Gothic than the printing of the German workmen, +LJ001-0064|many of whose types, indeed, like that of the Subiaco works, are of a transitional character.|many of whose types, indeed, like that of the Subiaco works, are of a transitional character. +LJ001-0065|This was notably the case with the early works printed at Ulm, and in a somewhat lesser degree at Augsburg.|This was notably the case with the early works printed at Ulm, and in a somewhat lesser degree at Augsburg. +LJ001-0066|In fact Gunther Zeiner's first type (afterwards used by Schussler) is remarkably like the type of the before-mentioned Subiaco books.|In fact Gunther Zeiner's first type (afterwards used by Schussler) is remarkably like the type of the before-mentioned Subiaco books. +LJ001-0067|In the Low Countries and Cologne, which were very fertile of printed books, Gothic was the favorite.|In the Low Countries and Cologne, which were very fertile of printed books, Gothic was the favorite. +LJ001-0069|This type was introduced into England by Wynkyn de Worde, Caxton's successor,|This type was introduced into England by Wynkyn de Worde, Caxton's successor, +LJ001-0070|and was used there with very little variation all through the sixteenth and seventeenth centuries, and indeed into the eighteenth.|and was used there with very little variation all through the sixteenth and seventeenth centuries, and indeed into the eighteenth. +LJ001-0071|Most of Caxton's own types are of an earlier character, though they also much resemble Flemish or Cologne letter.|Most of Caxton's own types are of an earlier character, though they also much resemble Flemish or Cologne letter. +LJ001-0072|After the end of the fifteenth century the degradation of printing, especially in Germany and Italy,|After the end of the fifteenth century the degradation of printing, especially in Germany and Italy, +LJ001-0073|went on apace; and by the end of the sixteenth century there was no really beautiful printing done:|went on apace; and by the end of the sixteenth century there was no really beautiful printing done: +LJ001-0074|the best, mostly French or Low-Country, was neat and clear, but without any distinction;|the best, mostly French or Low-Country, was neat and clear, but without any distinction; +LJ001-0075|the worst, which perhaps was the English, was a terrible falling-off from the work of the earlier presses;|the worst, which perhaps was the English, was a terrible falling-off from the work of the earlier presses; +LJ001-0076|and things got worse and worse through the whole of the seventeenth century, so that in the eighteenth printing was very miserably performed.|and things got worse and worse through the whole of the seventeenth century, so that in the eighteenth printing was very miserably performed. +LJ001-0077|In England about this time, an attempt was made (notably by Caslon, who started business in London as a type-founder in 1720)|In England about this time, an attempt was made (notably by Caslon, who started business in London as a type-founder in seventeen twenty) +LJ001-0078|to improve the letter in form.|to improve the letter in form. +LJ001-0079|Caslon's type is clear and neat, and fairly well designed;|Caslon's type is clear and neat, and fairly well designed; +LJ001-0080|he seems to have taken the letter of the Elzevirs of the seventeenth century for his model:|he seems to have taken the letter of the Elzevirs of the seventeenth century for his model: +LJ001-0081|type cast from his matrices is still in everyday use.|type cast from his matrices is still in everyday use. +LJ001-0082|In spite, however, of his praiseworthy efforts, printing had still one last degradation to undergo.|In spite, however, of his praiseworthy efforts, printing had still one last degradation to undergo. +LJ001-0083|The seventeenth century founts were bad rather negatively than positively.|The seventeenth century founts were bad rather negatively than positively. +LJ001-0084|But for the beauty of the earlier work they might have seemed tolerable.|But for the beauty of the earlier work they might have seemed tolerable. +LJ001-0085|It was reserved for the founders of the later eighteenth century to produce letters which are positively ugly, and which, it may be added,|It was reserved for the founders of the later eighteenth century to produce letters which are positively ugly, and which, it may be added, +LJ001-0086|are dazzling and unpleasant to the eye owing to the clumsy thickening and vulgar thinning of the lines:|are dazzling and unpleasant to the eye owing to the clumsy thickening and vulgar thinning of the lines: +LJ001-0087|for the seventeenth-century letters are at least pure and simple in line. The Italian, Bodoni, and the Frenchman, Didot,|for the seventeenth-century letters are at least pure and simple in line. The Italian, Bodoni, and the Frenchman, Didot, +LJ001-0088|were the leaders in this luckless change, though our own Baskerville, who was at work some years before them, went much on the same lines;|were the leaders in this luckless change, though our own Baskerville, who was at work some years before them, went much on the same lines; +LJ001-0089|but his letters, though uninteresting and poor, are not nearly so gross and vulgar as those of either the Italian or the Frenchman.|but his letters, though uninteresting and poor, are not nearly so gross and vulgar as those of either the Italian or the Frenchman. +LJ001-0090|With this change the art of printing touched bottom,|With this change the art of printing touched bottom, +LJ001-0091|so far as fine printing is concerned, though paper did not get to its worst till about 1840.|so far as fine printing is concerned, though paper did not get to its worst till about eighteen forty. +LJ001-0092|The Chiswick press in 1844 revived Caslon's founts, printing for Messrs. Longman the Diary of Lady Willoughby.|The Chiswick press in eighteen forty-four revived Caslon's founts, printing for Messrs. Longman the Diary of Lady Willoughby. +LJ001-0093|This experiment was so far successful that about 1850 Messrs. Miller and Richard of Edinburgh|This experiment was so far successful that about eighteen fifty Messrs. Miller and Richard of Edinburgh +LJ001-0094|were induced to cut punches for a series of "old style" letters.|were induced to cut punches for a series of "old style" letters. +LJ001-0095|These and similar founts, cast by the above firm and others,|These and similar founts, cast by the above firm and others, +LJ001-0096|have now come into general use and are obviously a great improvement on the ordinary "modern style" in use in England, which is in fact the Bodoni type|have now come into general use and are obviously a great improvement on the ordinary "modern style" in use in England, which is in fact the Bodoni type +LJ001-0097|a little reduced in ugliness. The design of the letters of this modern "old style" leaves a good deal to be desired,|a little reduced in ugliness. The design of the letters of this modern "old style" leaves a good deal to be desired, +LJ001-0098|and the whole effect is a little too gray, owing to the thinness of the letters.|and the whole effect is a little too gray, owing to the thinness of the letters. +LJ001-0099|It must be remembered, however, that most modern printing is done by machinery on soft paper, and not by the hand press,|It must be remembered, however, that most modern printing is done by machinery on soft paper, and not by the hand press, +LJ001-0100|and these somewhat wiry letters are suitable for the machine process, which would not do justice to letters of more generous design.|and these somewhat wiry letters are suitable for the machine process, which would not do justice to letters of more generous design. +LJ001-0101|It is discouraging to note that the improvement of the last fifty years is almost wholly confined to Great Britain.|It is discouraging to note that the improvement of the last fifty years is almost wholly confined to Great Britain. +LJ001-0102|Here and there a book is printed in France or Germany with some pretension to good taste,|Here and there a book is printed in France or Germany with some pretension to good taste, +LJ001-0103|but the general revival of the old forms has made no way in those countries.|but the general revival of the old forms has made no way in those countries. +LJ001-0104|Italy is contentedly stagnant.|Italy is contentedly stagnant. +LJ001-0105|America has produced a good many showy books, the typography, paper, and illustrations of which are, however, all wrong,|America has produced a good many showy books, the typography, paper, and illustrations of which are, however, all wrong, +LJ001-0106|oddity rather than rational beauty and meaning being apparently the thing sought for both in the letters and the illustrations.|oddity rather than rational beauty and meaning being apparently the thing sought for both in the letters and the illustrations. +LJ001-0107|To say a few words on the principles of design in typography:|To say a few words on the principles of design in typography: +LJ001-0108|it is obvious that legibility is the first thing to be aimed at in the forms of the letters;|it is obvious that legibility is the first thing to be aimed at in the forms of the letters; +LJ001-0109|this is best furthered by the avoidance of irrational swellings and spiky projections, and by the using of careful purity of line.|this is best furthered by the avoidance of irrational swellings and spiky projections, and by the using of careful purity of line. +LJ001-0110|Even the Caslon type when enlarged shows great shortcomings in this respect:|Even the Caslon type when enlarged shows great shortcomings in this respect: +LJ001-0111|the ends of many of the letters such as the t and e are hooked up in a vulgar and meaningless way,|the ends of many of the letters such as the t and e are hooked up in a vulgar and meaningless way, +LJ001-0112|instead of ending in the sharp and clear stroke of Jenson's letters;|instead of ending in the sharp and clear stroke of Jenson's letters; +LJ001-0113|there is a grossness in the upper finishings of letters like the c, the a, and so on,|there is a grossness in the upper finishings of letters like the c, the a, and so on, +LJ001-0114|an ugly pear-shaped swelling defacing the form of the letter:|an ugly pear-shaped swelling defacing the form of the letter: +LJ001-0115|in short, it happens to this craft, as to others, that the utilitarian practice, though it professes to avoid ornament,|in short, it happens to this craft, as to others, that the utilitarian practice, though it professes to avoid ornament, +LJ001-0116|still clings to a foolish, because misunderstood conventionality, deduced from what was once ornament, and is by no means useful;|still clings to a foolish, because misunderstood conventionality, deduced from what was once ornament, and is by no means useful; +LJ001-0117|which title can only be claimed by artistic practice, whether the art in it be conscious or unconscious.|which title can only be claimed by artistic practice, whether the art in it be conscious or unconscious. +LJ001-0118|In no characters is the contrast between the ugly and vulgar illegibility of the modern type|In no characters is the contrast between the ugly and vulgar illegibility of the modern type +LJ001-0119|and the elegance and legibility of the ancient more striking than in the Arabic numerals.|and the elegance and legibility of the ancient more striking than in the Arabic numerals. +LJ001-0120|In the old print each figure has its definite individuality, and one cannot be mistaken for the other;|In the old print each figure has its definite individuality, and one cannot be mistaken for the other; +LJ001-0121|in reading the modern figures the eyes must be strained before the reader can have any reasonable assurance|in reading the modern figures the eyes must be strained before the reader can have any reasonable assurance +LJ001-0122|that he has a 5, an 8, or a 3 before him, unless the press work is of the best:|that he has a five, an eight, or a three before him, unless the press work is of the best: +LJ001-0123|this is awkward if you have to read Bradshaw's Guide in a hurry.|this is awkward if you have to read Bradshaw's Guide in a hurry. +LJ001-0124|One of the differences between the fine type and the utilitarian must probably be put down to a misapprehension of a commercial necessity:|One of the differences between the fine type and the utilitarian must probably be put down to a misapprehension of a commercial necessity: +LJ001-0125|this is the narrowing of the modern letters.|this is the narrowing of the modern letters. +LJ001-0126|Most of Jenson's letters are designed within a square,|Most of Jenson's letters are designed within a square, +LJ001-0127|the modern letters are narrowed by a third or thereabout; but while this gain of space very much hampers the possibility of beauty of design,|the modern letters are narrowed by a third or thereabout; but while this gain of space very much hampers the possibility of beauty of design, +LJ001-0128|it is not a real gain, for the modern printer throws the gain away by putting inordinately wide spaces between his lines, which, probably,|it is not a real gain, for the modern printer throws the gain away by putting inordinately wide spaces between his lines, which, probably, +LJ001-0129|the lateral compression of his letters renders necessary.|the lateral compression of his letters renders necessary. +LJ001-0130|Commercialism again compels the use of type too small in size to be comfortable reading:|Commercialism again compels the use of type too small in size to be comfortable reading: +LJ001-0131|the size known as "Long primer" ought to be the smallest size used in a book meant to be read.|the size known as "Long primer" ought to be the smallest size used in a book meant to be read. +LJ001-0132|Here, again, if the practice of "leading" were retrenched larger type could be used without enhancing the price of a book.|Here, again, if the practice of "leading" were retrenched larger type could be used without enhancing the price of a book. +LJ001-0133|One very important matter in "setting up" for fine printing is the "spacing," that is, the lateral distance of words from one another.|One very important matter in "setting up" for fine printing is the "spacing," that is, the lateral distance of words from one another. +LJ001-0134|In good printing the spaces between the words should be as near as possible equal|In good printing the spaces between the words should be as near as possible equal +LJ001-0135|it is impossible that they should be quite equal except in lines of poetry|it is impossible that they should be quite equal except in lines of poetry +LJ001-0136|modern printers understand this, but it is only practiced in the very best establishments.|modern printers understand this, but it is only practiced in the very best establishments. +LJ001-0137|But another point which they should attend to they almost always disregard;|But another point which they should attend to they almost always disregard; +LJ001-0138|this is the tendency to the formation of ugly meandering white lines or "rivers" in the page|this is the tendency to the formation of ugly meandering white lines or "rivers" in the page +LJ001-0139|a blemish which can be nearly, though not wholly, avoided by care and forethought|a blemish which can be nearly, though not wholly, avoided by care and forethought +LJ001-0140|the desirable thing being "the breaking of the line" as in bonding masonry or brickwork|the desirable thing being "the breaking of the line" as in bonding masonry or brickwork +LJ001-0141|The general solidity of a page is much to be sought for|The general solidity of a page is much to be sought for +LJ001-0142|modern printers generally overdo the "whites" in the spacing, a defect probably forced on them by the characterless quality of the letters.|modern printers generally overdo the "whites" in the spacing, a defect probably forced on them by the characterless quality of the letters. +LJ001-0143|For where these are boldly and carefully designed, and each letter is thoroughly individual in form,|For where these are boldly and carefully designed, and each letter is thoroughly individual in form, +LJ001-0144|the words may be set much closer together, without loss of clearness.|the words may be set much closer together, without loss of clearness. +LJ001-0145|No definite rules, however, except the avoidance of "rivers" and excess of white, can be given for the spacing,|No definite rules, however, except the avoidance of "rivers" and excess of white, can be given for the spacing, +LJ001-0146|which requires the constant exercise of judgment and taste on the part of the printer.|which requires the constant exercise of judgment and taste on the part of the printer. +LJ001-0147|The position of the page on the paper should be considered if the book is to have a satisfactory look.|The position of the page on the paper should be considered if the book is to have a satisfactory look. +LJ001-0148|Here once more the almost invariable modern practice is in opposition to a natural sense of proportion.|Here once more the almost invariable modern practice is in opposition to a natural sense of proportion. +LJ001-0149|From the time when books first took their present shape till the end of the sixteenth century, or indeed later,|From the time when books first took their present shape till the end of the sixteenth century, or indeed later, +LJ001-0150|the page so lay on the paper that there was more space allowed to the bottom and fore margin than to the top and back of the paper,|the page so lay on the paper that there was more space allowed to the bottom and fore margin than to the top and back of the paper, +LJ001-0151|the unit of the book being looked on as the two pages forming an opening.|the unit of the book being looked on as the two pages forming an opening. +LJ001-0152|The modern printer, in the teeth of the evidence given by his own eyes, considers the single page as the unit, and prints the page in the middle of his paper|The modern printer, in the teeth of the evidence given by his own eyes, considers the single page as the unit, and prints the page in the middle of his paper +LJ001-0153|only nominally so, however, in many cases, since when he uses a headline he counts that in,|only nominally so, however, in many cases, since when he uses a headline he counts that in, +LJ001-0154|the result as measured by the eye being that the lower margin is less than the top one, and that the whole opening has an upside-down look vertically|the result as measured by the eye being that the lower margin is less than the top one, and that the whole opening has an upside-down look vertically +LJ001-0155|and that laterally the page looks as if it were being driven off the paper.|and that laterally the page looks as if it were being driven off the paper. +LJ001-0156|The paper on which the printing is to be done is a necessary part of our subject:|The paper on which the printing is to be done is a necessary part of our subject: +LJ001-0157|of this it may be said that though there is some good paper made now,|of this it may be said that though there is some good paper made now, +LJ001-0158|it is never used except for very expensive books, although it would not materially increase the cost in all but the very cheapest.|it is never used except for very expensive books, although it would not materially increase the cost in all but the very cheapest. +LJ001-0159|The paper that is used for ordinary books is exceedingly bad even in this country, but is beaten in the race for vileness|The paper that is used for ordinary books is exceedingly bad even in this country, but is beaten in the race for vileness +LJ001-0160|by that made in America, which is the worst conceivable.|by that made in America, which is the worst conceivable. +LJ001-0161|There seems to be no reason why ordinary paper should not be better made,|There seems to be no reason why ordinary paper should not be better made, +LJ001-0162|even allowing the necessity for a very low price; but any improvement must be based on showing openly that the cheap article is cheap,|even allowing the necessity for a very low price; but any improvement must be based on showing openly that the cheap article is cheap, +LJ001-0163|e.g. the cheap paper should not sacrifice toughness and durability to a smooth and white surface,|e.g. the cheap paper should not sacrifice toughness and durability to a smooth and white surface, +LJ001-0164|which should be indications of a delicacy of material and manufacture which would of necessity increase its cost.|which should be indications of a delicacy of material and manufacture which would of necessity increase its cost. +LJ001-0165|One fruitful source of badness in paper|One fruitful source of badness in paper +LJ001-0166|is the habit that publishers have of eking out a thin volume by printing it on thick paper almost of the substance of cardboard,|is the habit that publishers have of eking out a thin volume by printing it on thick paper almost of the substance of cardboard, +LJ001-0167|a device which deceives nobody, and makes a book very unpleasant to read.|a device which deceives nobody, and makes a book very unpleasant to read. +LJ001-0168|On the whole, a small book should be printed on paper which is as thin as may be without being transparent.|On the whole, a small book should be printed on paper which is as thin as may be without being transparent. +LJ001-0169|The paper used for printing the small highly ornamented French service-books about the beginning of the sixteenth century is a model in this respect,|The paper used for printing the small highly ornamented French service-books about the beginning of the sixteenth century is a model in this respect, +LJ001-0170|being thin, tough, and opaque.|being thin, tough, and opaque. +LJ001-0171|However, the fact must not be blinked that machine-made paper cannot in the nature of things be made of so good a texture as that made by hand.|However, the fact must not be blinked that machine-made paper cannot in the nature of things be made of so good a texture as that made by hand. +LJ001-0172|The ornamentation of printed books is too wide a subject to be dealt with fully here; but one thing must be said on it.|The ornamentation of printed books is too wide a subject to be dealt with fully here; but one thing must be said on it. +LJ001-0173|The essential point to be remembered is that the ornament, whatever it is, whether picture or pattern-work, should form part of the page,|The essential point to be remembered is that the ornament, whatever it is, whether picture or pattern-work, should form part of the page, +LJ001-0174|should be a part of the whole scheme of the book.|should be a part of the whole scheme of the book. +LJ001-0175|Simple as this proposition is, it is necessary to be stated,|Simple as this proposition is, it is necessary to be stated, +LJ001-0176|because the modern practice is to disregard the relation between the printing and the ornament altogether,|because the modern practice is to disregard the relation between the printing and the ornament altogether, +LJ001-0177|so that if the two are helpful to one another it is a mere matter of accident.|so that if the two are helpful to one another it is a mere matter of accident. +LJ001-0178|The due relation of letter to pictures and other ornament was thoroughly understood by the old printers; so that|The due relation of letter to pictures and other ornament was thoroughly understood by the old printers; so that +LJ001-0179|even when the woodcuts are very rude indeed,|even when the woodcuts are very rude indeed, +LJ001-0180|the proportions of the page still give pleasure by the sense of richness that the cuts and letter together convey.|the proportions of the page still give pleasure by the sense of richness that the cuts and letter together convey. +LJ001-0181|When, as is most often the case, there is actual beauty in the cuts,|When, as is most often the case, there is actual beauty in the cuts, +LJ001-0182|the books so ornamented are amongst the most delightful works of art that have ever been produced.|the books so ornamented are amongst the most delightful works of art that have ever been produced. +LJ001-0183|Therefore, granted well-designed type, due spacing of the lines and words, and proper position of the page on the paper,|Therefore, granted well-designed type, due spacing of the lines and words, and proper position of the page on the paper, +LJ001-0184|all books might be at least comely and well-looking: and if to these good qualities were added really beautiful ornament and pictures,|all books might be at least comely and well-looking: and if to these good qualities were added really beautiful ornament and pictures, +LJ001-0185|printed books might once again illustrate to the full|printed books might once again illustrate to the full +LJ001-0186|the position of our Society that a work of utility might be also a work of art, if we cared to make it so.|the position of our Society that a work of utility might be also a work of art, if we cared to make it so. +LJ002-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 4: Newgate down to 1818.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section four: Newgate down to eighteen eighteen. +LJ002-0002|Under the conditions referred to in the previous chapter,|Under the conditions referred to in the previous chapter, +LJ002-0003|with criminals and misdemeanants of all shades crowding perpetually into its narrow limits, the latter state of Newgate was worse than the first.|with criminals and misdemeanants of all shades crowding perpetually into its narrow limits, the latter state of Newgate was worse than the first. +LJ002-0004|The new jail fell as far short of the demands made on it as did the old.|The new jail fell as far short of the demands made on it as did the old. +LJ002-0005|The prison population fluctuated a great deal,|The prison population fluctuated a great deal, +LJ002-0006|but it was almost always in excess of the accommodation available, and there were times when the place was full to overflowing.|but it was almost always in excess of the accommodation available, and there were times when the place was full to overflowing. +LJ002-0007|Neild gives some figures which well illustrate this.|Neild gives some figures which well illustrate this. +LJ002-0008|On the 14th June, 1800, there were 199 debtors and 289 felons in the prison.|On the fourteenth June, eighteen hundred, there were one hundred ninety-nine debtors and two hundred eighty-nine felons in the prison. +LJ002-0009|On the 27th April, in the following year,|On the twenty-seventh April, in the following year, +LJ002-0010|these numbers had risen to 275 and 375 respectively, or 650 in all.|these numbers had risen to two hundred seventy-five and three hundred seventy-five respectively, or six hundred fifty in all. +LJ002-0011|For two more years these high figures were steadily maintained, and in 1803 the total rose to 710.|For two more years these high figures were steadily maintained, and in eighteen oh three the total rose to seven hundred ten. +LJ002-0012|After that they fell as steadily,|After that they fell as steadily, +LJ002-0013|till, 1808, the lowest point was touched of 197 debtors and 182 felons, or 379 in all.|till, eighteen oh eight, the lowest point was touched of one hundred ninety-seven debtors and one hundred eighty-two felons, or three hundred seventy-nine in all. +LJ002-0014|The numbers soon increased, however, and by 1811 had again risen to 629; and Mr. Neild was told that there had been at one time|The numbers soon increased, however, and by eighteen eleven had again risen to six hundred twenty-nine; and Mr. Neild was told that there had been at one time +LJ002-0015|300 debtors and 900 criminals in Newgate, or twelve hundred prisoners in all.|three hundred debtors and nine hundred criminals in Newgate, or twelve hundred prisoners in all. +LJ002-0016|Previous to that date there had been 700 or 800 frequently, and once, in Mr. Akerman's time, one thousand.|Previous to that date there had been seven hundred or eight hundred frequently, and once, in Mr. Akerman's time, one thousand. +LJ002-0017|Trustworthy evidence is forthcoming to the effect that these high figures were constantly maintained for many months at a time.|Trustworthy evidence is forthcoming to the effect that these high figures were constantly maintained for many months at a time. +LJ002-0018|The inadequacy of the jail was noticed and reported upon again and again by the grand juries of the city of London,|The inadequacy of the jail was noticed and reported upon again and again by the grand juries of the city of London, +LJ002-0019|who seldom let a session go by without visiting Newgate.|who seldom let a session go by without visiting Newgate. +LJ002-0020|In 1813|In eighteen thirteen +LJ002-0021|the grand jury made a special presentment to the Court of Common Council, pointing out that on the debtors' side, which was intended for only 100,|the grand jury made a special presentment to the Court of Common Council, pointing out that on the debtors' side, which was intended for only one hundred, +LJ002-0022|no less than 340 were crowded, to the great inconvenience and danger of the inmates.|no less than three hundred forty were crowded, to the great inconvenience and danger of the inmates. +LJ002-0023|On the female side matters were much worse;|On the female side matters were much worse; +LJ002-0024|Quote. the apartments set apart for them, being built to accommodate 60 persons, now contain about 120. End quote.|Quote. the apartments set apart for them, being built to accommodate sixty persons, now contain about one hundred twenty. End quote. +LJ002-0025|Returns laid before the House of Commons showed that 6439 persons had been committed to Newgate|Returns laid before the House of Commons showed that six thousand, four hundred thirty-nine persons had been committed to Newgate +LJ002-0026|in the three years between 1813 and 1816,|in the three years between eighteen thirteen and eighteen sixteen, +LJ002-0027|and this number did not include the debtors, a numerous class, who were still committed to Newgate pending the completion of the White Cross Street prison.|and this number did not include the debtors, a numerous class, who were still committed to Newgate pending the completion of the White Cross Street prison. +LJ002-0028|In order to realize the evils entailed by incarceration in Newgate in these days, it is necessary to give some account of its interior|In order to realize the evils entailed by incarceration in Newgate in these days, it is necessary to give some account of its interior +LJ002-0029|as it was occupied and appropriated in 1810.|as it was occupied and appropriated in eighteen ten. +LJ002-0030|Full details of the arrangements are to be found in Mr. Neild's "State of Prisons in England, Scotland, and Wales," published in 1812.|Full details of the arrangements are to be found in Mr. Neild's "State of Prisons in England, Scotland, and Wales," published in eighteen twelve. +LJ002-0031|The jail at that date was divided into eight separate and more or less distinct departments, each of which had its own wards and yard.|The jail at that date was divided into eight separate and more or less distinct departments, each of which had its own wards and yard. +LJ002-0032|These were: 1. The male debtors' side.|These were: one. The male debtors' side. +LJ002-0033|2. The female debtors' side. 3. The chapel yard. 4. The middle yard.|two. The female debtors' side. three. The chapel yard. four. The middle yard. +LJ002-0034|5. The master felons' side. 6. The female felons' side. 7. The state side.|five. The master felons' side. six. The female felons' side. seven. The state side. +LJ002-0035|8. The press yard.|eight. The press yard. +LJ002-0036|1. The male debtors' side consisted of a yard forty-nine feet by thirty-one,|one. The male debtors' side consisted of a yard forty-nine feet by thirty-one, +LJ002-0037|leading to thirteen wards on various floors, and a day room.|leading to thirteen wards on various floors, and a day room. +LJ002-0038|Of these wards, three were appropriated to the "cabin side," so called because|Of these wards, three were appropriated to the "cabin side," so called because +LJ002-0039|they each contained four small rooms or "cabins" seven feet square,|they each contained four small rooms or "cabins" seven feet square, +LJ002-0040|intended to accommodate a couple of prisoners apiece, but often much more crowded.|intended to accommodate a couple of prisoners apiece, but often much more crowded. +LJ002-0041|Two other wards were appropriated to the master's side debtors; they were each twenty-three feet by fourteen and a half,|Two other wards were appropriated to the master's side debtors; they were each twenty-three feet by fourteen and a half, +LJ002-0042|and supposed to accommodate twenty persons. The eight remaining wards were for the common side debtors,|and supposed to accommodate twenty persons. The eight remaining wards were for the common side debtors, +LJ002-0043|long narrow rooms -- one thirty-six feet, six twenty-three feet, and the eighth eighteen,|long narrow rooms -- one thirty-six feet, six twenty-three feet, and the eighth eighteen, +LJ002-0044|the whole about fifteen feet wide.|the whole about fifteen feet wide. +LJ002-0045|The various wards were all about eleven feet in height,|The various wards were all about eleven feet in height, +LJ002-0046|and were occupied as a rule by ten to fifteen people when the prison was not crowded, but double the number was occasionally placed in them.|and were occupied as a rule by ten to fifteen people when the prison was not crowded, but double the number was occasionally placed in them. +LJ002-0047|The day room was fitted with benches and settles after the manner of the tap in a public-house.|The day room was fitted with benches and settles after the manner of the tap in a public-house. +LJ002-0048|2. The female debtors' side consisted of a court-yard forty-nine by sixteen feet,|two. The female debtors' side consisted of a court-yard forty-nine by sixteen feet, +LJ002-0049|leading to two wards, one of which was thirty-six feet by fifteen,|leading to two wards, one of which was thirty-six feet by fifteen, +LJ002-0050|and the other eighteen by fifteen; and they nominally held twenty-two persons.|and the other eighteen by fifteen; and they nominally held twenty-two persons. +LJ002-0051|A high wall fifteen feet in height divided the females' court-yard from the men's.|A high wall fifteen feet in height divided the females' court-yard from the men's. +LJ002-0052|3. The chapel yard was about forty-three feet by twenty-five.|three. The chapel yard was about forty-three feet by twenty-five. +LJ002-0053|It had been for some time devoted principally to felons of the worst types,|It had been for some time devoted principally to felons of the worst types, +LJ002-0054|those who were the oldest offenders, sentenced to transportation, and who had narrowly escaped the penalty of death.|those who were the oldest offenders, sentenced to transportation, and who had narrowly escaped the penalty of death. +LJ002-0055|This arrangement was, however, modified after 1811, and the chapel yard was allotted to misdemeanants and prisoners awaiting trial.|This arrangement was, however, modified after eighteen eleven, and the chapel yard was allotted to misdemeanants and prisoners awaiting trial. +LJ002-0056|The wards in this part were five in number, all in dimensions twenty feet by fifteen, with a sixth ward fifteen feet square.|The wards in this part were five in number, all in dimensions twenty feet by fifteen, with a sixth ward fifteen feet square. +LJ002-0057|These wards were all fitted with barrack-beds, but no bedding was supplied.|These wards were all fitted with barrack-beds, but no bedding was supplied. +LJ002-0058|The chapel yard led to the chapel, and on the staircase were two rooms frequently set apart for the king's witnesses,|The chapel yard led to the chapel, and on the staircase were two rooms frequently set apart for the king's witnesses, +LJ002-0059|those who had turned king's evidence, whose safety might have been imperiled had they been lodged with the men against whom they had informed.|those who had turned king's evidence, whose safety might have been imperiled had they been lodged with the men against whom they had informed. +LJ002-0060|But these king's witnesses were also put at times into the press yard among the capital convicts, seemingly a very dangerous proceeding,|But these king's witnesses were also put at times into the press yard among the capital convicts, seemingly a very dangerous proceeding, +LJ002-0061|or they lodged with the gatesmen, the prisoner officers who had charge of the inner gates.|or they lodged with the gatesmen, the prisoner officers who had charge of the inner gates. +LJ002-0062|The middle yard was at first given up to the least heinous offenders. After 1812 it changed functions with the chapel yard.|The middle yard was at first given up to the least heinous offenders. After eighteen twelve it changed functions with the chapel yard. +LJ002-0063|It was fifty feet by twenty-five, and had five wards each thirty-eight by fifteen. At one end of the yard was an arcade,|It was fifty feet by twenty-five, and had five wards each thirty-eight by fifteen. At one end of the yard was an arcade, +LJ002-0064|directly under the chapel, in which there were three cells, used either for the confinement of disorderly and refractory prisoners,|directly under the chapel, in which there were three cells, used either for the confinement of disorderly and refractory prisoners, +LJ002-0065|or female convicts ordered for execution.|or female convicts ordered for execution. +LJ002-0066|The master felons' side consisted of a yard the same size as the preceding, appropriated nominally to the most decent and better-behaved prisoners,|The master felons' side consisted of a yard the same size as the preceding, appropriated nominally to the most decent and better-behaved prisoners, +LJ002-0067|but really kept for the few who had funds sufficient to gain them admission to these more comfortable quarters.|but really kept for the few who had funds sufficient to gain them admission to these more comfortable quarters. +LJ002-0068|Here were also lodged the gatesmen, the prisoners who had charge of the inner gates, and who were entrusted with the duty of escorting visitors from the gates|Here were also lodged the gatesmen, the prisoners who had charge of the inner gates, and who were entrusted with the duty of escorting visitors from the gates +LJ002-0069|to the various wards their friends occupied.|to the various wards their friends occupied. +LJ002-0070|The state side was the part stolen from the female felons' side.|The state side was the part stolen from the female felons' side. +LJ002-0071|It was large and comparatively commodious, being maintained on a better footing than any other part of the prison.|It was large and comparatively commodious, being maintained on a better footing than any other part of the prison. +LJ002-0072|The inmates were privileged, either by antecedents or the fortunate possession of sufficient funds to pay the charges of the place.|The inmates were privileged, either by antecedents or the fortunate possession of sufficient funds to pay the charges of the place. +LJ002-0073|Neild takes it for granted that the former rather than the latter prevailed in the selection,|Neild takes it for granted that the former rather than the latter prevailed in the selection, +LJ002-0074|and tells us that in the state side, quote, such prisoners were safely associated whose manners and conduct evince a more liberal style of education,|and tells us that in the state side, quote, such prisoners were safely associated whose manners and conduct evince a more liberal style of education, +LJ002-0075|and who are therefore lodged apart from all other districts of the jail. End quote.|and who are therefore lodged apart from all other districts of the jail. End quote. +LJ002-0076|The state side contained twelve good-sized rooms,|The state side contained twelve good-sized rooms, +LJ002-0077|from twenty-one by eighteen feet to fifteen feet square, which were furnished with bedsteads and bedding.|from twenty-one by eighteen feet to fifteen feet square, which were furnished with bedsteads and bedding. +LJ002-0078|7. The press yard was that part set aside for the condemned.|seven. The press yard was that part set aside for the condemned. +LJ002-0079|Its name and its situation were the same as those of the old place of carrying out the terrible sentence inflicted on accused persons who stood mute.|Its name and its situation were the same as those of the old place of carrying out the terrible sentence inflicted on accused persons who stood mute. +LJ002-0080|The long narrow yard still remained as we saw it in Jacobite times,|The long narrow yard still remained as we saw it in Jacobite times, +LJ002-0081|and beyond it was now a day room for the capital convicts or those awaiting execution.|and beyond it was now a day room for the capital convicts or those awaiting execution. +LJ002-0082|Beyond the press yard were three stories, condemned cells, fifteen in all, with vaulted ceilings nine feet high to the crown of the arch.|Beyond the press yard were three stories, condemned cells, fifteen in all, with vaulted ceilings nine feet high to the crown of the arch. +LJ002-0083|The ground floor cells were nine feet by six;|The ground floor cells were nine feet by six; +LJ002-0084|those on the first floor were rather larger on account of a set-off in the wall; and the uppermost were the largest, for the same reason.|those on the first floor were rather larger on account of a set-off in the wall; and the uppermost were the largest, for the same reason. +LJ002-0085|Security was provided for in these condemned cells by lining the substantial stone walls with planks studded with broad-headed nails;|Security was provided for in these condemned cells by lining the substantial stone walls with planks studded with broad-headed nails; +LJ002-0086|they were lighted by a double-grated window two feet nine inches by fourteen inches; and in the doors, which were four inches thick,|they were lighted by a double-grated window two feet nine inches by fourteen inches; and in the doors, which were four inches thick, +LJ002-0087|a circular aperture had been let in to give ventilation and secure a free current of air.|a circular aperture had been let in to give ventilation and secure a free current of air. +LJ002-0088|In each cell there was a barrack bedstead on the floor without bedding.|In each cell there was a barrack bedstead on the floor without bedding. +LJ002-0089|8. The female felons were deprived of part of the space which the architect had intended for them.|eight. The female felons were deprived of part of the space which the architect had intended for them. +LJ002-0090|More than half their quadrangle had been partitioned off for another purpose,|More than half their quadrangle had been partitioned off for another purpose, +LJ002-0091|and what remained was divided into a master's and a common side for female felons.|and what remained was divided into a master's and a common side for female felons. +LJ002-0092|The two yards were adjoining, that for the common side much the largest.|The two yards were adjoining, that for the common side much the largest. +LJ002-0093|There were nine wards in all on the female side, one of them in the attic,|There were nine wards in all on the female side, one of them in the attic, +LJ002-0094|with four casements and two fireplaces, being allotted for a female infirmary|with four casements and two fireplaces, being allotted for a female infirmary +LJ002-0095|and the rest being provided with barrack beds, and in dimensions varying from thirty feet by fifteen to fifteen feet by ten.|and the rest being provided with barrack beds, and in dimensions varying from thirty feet by fifteen to fifteen feet by ten. +LJ002-0096|The eight courts above enumerated were well supplied with water;|The eight courts above enumerated were well supplied with water; +LJ002-0097|they had dust-bins, sewers, and so forth, "properly disposed," and the city scavenger paid periodical visits to the prison.|they had dust-bins, sewers, and so forth, "properly disposed," and the city scavenger paid periodical visits to the prison. +LJ002-0098|The prisoners had few comforts, beyond the occasional use of a bath at some distance, situated in the press yard,|The prisoners had few comforts, beyond the occasional use of a bath at some distance, situated in the press yard, +LJ002-0099|to which access was granted rarely and as a great favor. But they were allowed the luxury of drink -- if they could pay for it.|to which access was granted rarely and as a great favor. But they were allowed the luxury of drink -- if they could pay for it. +LJ002-0100|A recent reform had closed the tap kept by the jailer within the precincts, but|A recent reform had closed the tap kept by the jailer within the precincts, but +LJ002-0101|there was still a "convenient room" which served, and, quote,|there was still a "convenient room" which served, and, quote, +LJ002-0102|near it a grating through which the debtors receive their beer from the neighboring public-houses.|near it a grating through which the debtors receive their beer from the neighboring public-houses. +LJ002-0103|The felons' side has a similar accommodation, and this mode of introducing the beverage is adopted because no publican as such|The felons' side has a similar accommodation, and this mode of introducing the beverage is adopted because no publican as such +LJ002-0104|can be permitted to enter the interior of this prison. End quote. The tap-room and bar were just behind the felons' entrance lodge,|can be permitted to enter the interior of this prison. End quote. The tap-room and bar were just behind the felons' entrance lodge, +LJ002-0105|and beyond it was a room called the "wine room," because formerly used for the sale of wine, but|and beyond it was a room called the "wine room," because formerly used for the sale of wine, but +LJ002-0106|in which latterly a copper had been fixed for the cooking of provisions sent in by charitable persons.|in which latterly a copper had been fixed for the cooking of provisions sent in by charitable persons. +LJ002-0107|Quote, On the top of the jail, continues Neild, are a watch-house and a sentry-box, where two or more guards, with dogs and firearms,|Quote, On the top of the jail, continues Neild, are a watch-house and a sentry-box, where two or more guards, with dogs and firearms, +LJ002-0108|watch all night. Adjoining the felons' side lodge is the keeper's office, where the prison books are kept, and his clerk,|watch all night. Adjoining the felons' side lodge is the keeper's office, where the prison books are kept, and his clerk, +LJ002-0109|called the clerk of the papers, attends daily. End quote.|called the clerk of the papers, attends daily. End quote. +LJ002-0110|Having thus briefly described the plan and appropriation of the prison, I propose to deal now with the general condition of the inmates, and the manner of their life.|Having thus briefly described the plan and appropriation of the prison, I propose to deal now with the general condition of the inmates, and the manner of their life. +LJ002-0111|Of these the debtors, male and female, formed a large proportion.|Of these the debtors, male and female, formed a large proportion. +LJ002-0112|The frequency and extent of processes against debtors seventy or eighty years ago will appear almost incredible|The frequency and extent of processes against debtors seventy or eighty years ago will appear almost incredible +LJ002-0113|in an age when insolvent acts and bankruptcy courts do so much to relieve the impecunious,|in an age when insolvent acts and bankruptcy courts do so much to relieve the impecunious, +LJ002-0114|and imprisonment for debt has almost entirely disappeared.|and imprisonment for debt has almost entirely disappeared. +LJ002-0116|The number of processes against debtors annually was extraordinary.|The number of processes against debtors annually was extraordinary. +LJ002-0117|Neild gives, on the authority of Mr. Burchell, the under sheriff of Middlesex,|Neild gives, on the authority of Mr. Burchell, the under sheriff of Middlesex, +LJ002-0118|a table showing the figures for the year ending Michaelmas 1802.|a table showing the figures for the year ending Michaelmas eighteen oh two. +LJ002-0119|In that period upwards of 200,000 writs|In that period upwards of two hundred thousand writs +LJ002-0120|had been issued for the arrests of debtors in the kingdom, for sums varying from fourpence to £500 and upwards.|had been issued for the arrests of debtors in the kingdom, for sums varying from fourpence to five hundred pounds and upwards. +LJ002-0121|Fifteen thousand of these were issued in Middlesex alone, which at that time was reckoned as only a fifteenth of Great Britain.|Fifteen thousand of these were issued in Middlesex alone, which at that time was reckoned as only a fifteenth of Great Britain. +LJ002-0122|The number of arrests actually made was 114,300 for the kingdom, and 7020 for Middlesex.|The number of arrests actually made was one hundred fourteen thousand, three hundred for the kingdom, and seven thousand twenty for Middlesex. +LJ002-0123|Barely half of these gave bail bonds on arrests, and the remainder went to prison.|Barely half of these gave bail bonds on arrests, and the remainder went to prison. +LJ002-0124|Quite half of the foregoing writs and arrests applied to sums under £30.|Quite half of the foregoing writs and arrests applied to sums under thirty pounds. +LJ002-0125|Neild also says that in 1793,|Neild also says that in seventeen ninety-three, +LJ002-0126|5719 writs and executions for debts between £10 and £20 were issued in Middlesex,|five thousand, seven hundred nineteen writs and executions for debts between ten pounds and twenty pounds were issued in Middlesex, +LJ002-0127|and the aggregate amount of debts sued for was £81,791.|and the aggregate amount of debts sued for was eighty-one thousand, seven hundred ninety-one pounds. +LJ002-0128|He also makes the curious calculation that the costs of these actions if undefended|He also makes the curious calculation that the costs of these actions if undefended +LJ002-0129|would have amounted to £68,728, and if defended,|would have amounted to sixty-eight thousand, seven hundred twenty-eight pounds, and if defended, +LJ002-0130|£285,950; in other words, that to recover eighty odd thousand pounds,|two hundred eighty-five thousand, nine hundred fifty pounds; in other words, that to recover eighty odd thousand pounds, +LJ002-0131|three times the amount would be expended.|three times the amount would be expended. +LJ002-0132|An elaborate machinery planned for the protection of the trader, and altogether on his side, had long existed for the recovery of debts.|An elaborate machinery planned for the protection of the trader, and altogether on his side, had long existed for the recovery of debts. +LJ002-0133|Alfred the Great established the Court Baron, the Hundred Court, and the County Court, which among other matters entertained pleas for debt.|Alfred the Great established the Court Baron, the Hundred Court, and the County Court, which among other matters entertained pleas for debt. +LJ002-0134|The County Court was the sheriff's, who sat there surrounded by the bishop and the magnates of the county;|The County Court was the sheriff's, who sat there surrounded by the bishop and the magnates of the county; +LJ002-0135|but as time passed, difficulties and delays in obtaining judgment led to the removal of causes to the great Court of King's Bench,|but as time passed, difficulties and delays in obtaining judgment led to the removal of causes to the great Court of King's Bench, +LJ002-0136|and the disuse of the inferior courts.|and the disuse of the inferior courts. +LJ002-0137|So much inconvenience ensued, that in 1518 the Corporation obtained from Parliament an act empowering two aldermen|So much inconvenience ensued, that in fifteen eighteen the Corporation obtained from Parliament an act empowering two aldermen +LJ002-0138|and four common councilmen to hold Courts of Requests, or Courts of Conscience, to hear and determine all causes of debt|and four common councilmen to hold Courts of Requests, or Courts of Conscience, to hear and determine all causes of debt +LJ002-0139|under 40 shillings arising within the city.|under forty shillings arising within the city. +LJ002-0140|These courts were extended two centuries later to several large provincial towns, and all were in full activity when Neild wrote,|These courts were extended two centuries later to several large provincial towns, and all were in full activity when Neild wrote, +LJ002-0141|and indeed supplied the bulk of the poor debtors committed to prison.|and indeed supplied the bulk of the poor debtors committed to prison. +LJ002-0142|These courts were open to many and grave objections.|These courts were open to many and grave objections. +LJ002-0143|The commissioners who presided were, quote, little otherwise than self-elected|The commissioners who presided were, quote, little otherwise than self-elected +LJ002-0144|and when once appointed continued to serve sine die; they were generally near in rank to the parties whose causes they decided.|and when once appointed continued to serve sine die; they were generally near in rank to the parties whose causes they decided. +LJ002-0145|Often a commissioner had to leave the bench because he was himself a party to the suit that was sub judice.|Often a commissioner had to leave the bench because he was himself a party to the suit that was sub judice. +LJ002-0146|The activity as well as the futility of these courts may be estimated from the statement given by Neild|The activity as well as the futility of these courts may be estimated from the statement given by Neild +LJ002-0147|that thirteen hundred and twelve debtors were committed by them to Newgate between 1797 and 1808,|that thirteen hundred and twelve debtors were committed by them to Newgate between seventeen ninety-seven and eighteen oh eight, +LJ002-0148|and that no more than 197 creditors recovered debts and costs.|and that no more than one hundred ninety-seven creditors recovered debts and costs. +LJ002-0150|Costs were the gallons of sack to the pennyworth of debt.|Costs were the gallons of sack to the pennyworth of debt. +LJ002-0151|Neild found at his visit to Newgate in 1810,|Neild found at his visit to Newgate in eighteen ten, +LJ002-0152|fourteen men and women who had lain there ten, eleven, and thirteen years for debts of a few shillings,|fourteen men and women who had lain there ten, eleven, and thirteen years for debts of a few shillings, +LJ002-0153|weighted by treble the amount of costs.|weighted by treble the amount of costs. +LJ002-0154|Thus, amongst others, Thomas Blackburn had been committed on October 15th for a debt of 1 shilling 5 pence.|Thus, amongst others, Thomas Blackburn had been committed on October fifteenth for a debt of one shilling five pence. +LJ002-0155|for which the costs were 6 shillings 10 pence.|for which the costs were six shillings ten pence. +LJ002-0156|Thomas Dobson, on 22nd August, 1799, for 1 shilling, with costs of 8 shillings, 10 pence.|Thomas Dobson, on twenty-second August, seventeen ninety-nine, for one shilling, with costs of eight shillings, ten pence. +LJ002-0158|Other cases are recorded elsewhere, as at the Giltspur Street Compter, where in 1805 Mr. Neild found a man named William Grant|Other cases are recorded elsewhere, as at the Giltspur Street Compter, where in eighteen oh five Mr. Neild found a man named William Grant +LJ002-0159|detained for 1 shilling 9 pence, with costs of 5 shillings.|detained for one shilling nine pence, with costs of five shillings. +LJ002-0160|and John Lancaster for 1 shilling, 8 pence, with costs of 7 shillings, 6 pence. Quote.|and John Lancaster for one shilling, eight pence, with costs of seven shillings, six pence. Quote. +LJ002-0161|These surely, I thought," says Mr. Neild, "were bad enough! But it was not so. End quote.|These surely, I thought," says Mr. Neild, "were bad enough! But it was not so. End quote. +LJ002-0162|He recites another most outrageous and extraordinary case, in which one John Bird,|He recites another most outrageous and extraordinary case, in which one John Bird, +LJ002-0163|a market porter, was arrested and committed at the suit of a publican|a market porter, was arrested and committed at the suit of a publican +LJ002-0164|for the paltry sum of 4 pence, with costs of 7 shillings, 6 pence.|for the paltry sum of four pence, with costs of seven shillings, six pence. +LJ002-0165|Bird was, however, discharged within three days by a subscription raised among his fellow-prisoners.|Bird was, however, discharged within three days by a subscription raised among his fellow-prisoners. +LJ002-0166|Mr. Buxton, in his "Inquiry into the System of Prison Discipline,"|Mr. Buxton, in his "Inquiry into the System of Prison Discipline," +LJ002-0168|The lad in question was found in Coldbath Fields prison, to which he had been sent for a month in default of paying a fine of forty shillings.|The lad in question was found in Coldbath Fields prison, to which he had been sent for a month in default of paying a fine of forty shillings. +LJ002-0169|He had been in the employ of a corn-chandler at Islington, and went into London with his master's cart and horse.|He had been in the employ of a corn-chandler at Islington, and went into London with his master's cart and horse. +LJ002-0170|There was in the City Road a temporary bar, with a collector of tolls who was sometimes on the spot and sometimes not.|There was in the City Road a temporary bar, with a collector of tolls who was sometimes on the spot and sometimes not. +LJ002-0171|The boy declared he saw no one, and accordingly passed through without paying the toll of a penny.|The boy declared he saw no one, and accordingly passed through without paying the toll of a penny. +LJ002-0172|For this he was summoned before a magistrate, and sentenced as already stated.|For this he was summoned before a magistrate, and sentenced as already stated. +LJ002-0173|The lad was proved to be of good character and the son of respectable parents.|The lad was proved to be of good character and the son of respectable parents. +LJ002-0174|Mr. Buxton's friends at once paid the forty shillings, and the boy was released.|Mr. Buxton's friends at once paid the forty shillings, and the boy was released. +LJ002-0175|The costs in heavier debts always doubled the sum; if the arrest was made in the country it trebled it.|The costs in heavier debts always doubled the sum; if the arrest was made in the country it trebled it. +LJ002-0176|Neild gives a list of the various items charged upon a debt of £10, which included instructions to sue,|Neild gives a list of the various items charged upon a debt of ten pounds, which included instructions to sue, +LJ002-0177|affidavit of debt, drawing praecipe (1 pound, 5 shillings), capias, fee to officer on arrest,|affidavit of debt, drawing praecipe (one pound, five shillings), capias, fee to officer on arrest, +LJ002-0178|affidavit of service, and many more, amounting in all to twenty-seven,|affidavit of service, and many more, amounting in all to twenty-seven, +LJ002-0179|and costing 11 pounds, 15 shillings, 8 pence, within ten days.|and costing eleven pounds, fifteen shillings, eight pence, within ten days. +LJ002-0180|Before dealing with the debtors in Newgate, I may refer incidentally|Before dealing with the debtors in Newgate, I may refer incidentally +LJ002-0181|to those in other London prisons, for Newgate was not the only place of durance for these unfortunate people. There were also the King's Bench,|to those in other London prisons, for Newgate was not the only place of durance for these unfortunate people. There were also the King's Bench, +LJ002-0182|the Fleet, and the Marshalsea prisons especially devoted to them,|the Fleet, and the Marshalsea prisons especially devoted to them, +LJ002-0183|whilst Ludgate, the Giltspur Street, and Borough Compters also received them|whilst Ludgate, the Giltspur Street, and Borough Compters also received them +LJ002-0184|the latter two being also a prison for felons and vagrants arrested within certain limits.|the latter two being also a prison for felons and vagrants arrested within certain limits. +LJ002-0185|The King's Bench was a national prison, in which were confined all debtors arrested for debt or for contempt of the court of the King's Bench.|The King's Bench was a national prison, in which were confined all debtors arrested for debt or for contempt of the court of the King's Bench. +LJ002-0186|The population generally amounted to from five hundred to seven hundred, the accommodation being calculated for two hundred.|The population generally amounted to from five hundred to seven hundred, the accommodation being calculated for two hundred. +LJ002-0187|Every new-comer was entitled to a "chummage" ticket, but did not always get it,|Every new-comer was entitled to a "chummage" ticket, but did not always get it, +LJ002-0188|being often obliged to pay a high rent for a bed at the coffee-house or in some room which was vacated by its regular occupant.|being often obliged to pay a high rent for a bed at the coffee-house or in some room which was vacated by its regular occupant. +LJ002-0189|No fixed rates or rules governed the hiring out of rooms or parts of a room, and all sorts of imposition was practiced.|No fixed rates or rules governed the hiring out of rooms or parts of a room, and all sorts of imposition was practiced. +LJ002-0190|The best, or at least the most influential prisoners, got lodging in the State House, which contained "eight large handsome rooms."|The best, or at least the most influential prisoners, got lodging in the State House, which contained "eight large handsome rooms." +LJ002-0191|Besides those actually resident within the walls,|Besides those actually resident within the walls, +LJ002-0192|another two hundred more or less took advantage of "the rules," and lived outside within a circumference of two miles and a half.|another two hundred more or less took advantage of "the rules," and lived outside within a circumference of two miles and a half. +LJ002-0193|In these cases security was given for the amount of the debt,|In these cases security was given for the amount of the debt, +LJ002-0194|and a heavy fee at the rate of £8 per £100, with £4 for every additional hundred.|and a heavy fee at the rate of eight pounds per one hundred pounds, with four pounds for every additional hundred. +LJ002-0195|Besides these, a number had the privilege of a "run on the key," which allowed a prisoner to go into the rules for the day.|Besides these, a number had the privilege of a "run on the key," which allowed a prisoner to go into the rules for the day. +LJ002-0196|The foregoing rentals and payments for privileges, together with fees exacted on commitment and discharge, went to the marshal or keeper of the prison,|The foregoing rentals and payments for privileges, together with fees exacted on commitment and discharge, went to the marshal or keeper of the prison, +LJ002-0197|whose net annual income thus entirely derived from the impecunious amounted to between three and four thousand pounds.|whose net annual income thus entirely derived from the impecunious amounted to between three and four thousand pounds. +LJ002-0198|The office of marshal had been hereditary,|The office of marshal had been hereditary, +LJ002-0199|but in the 27th George II the right of presentation was bought by the Crown for £10,500.|but in the twenty-seventh George the second the right of presentation was bought by the Crown for ten thousand, five hundred pounds. +LJ002-0200|The marshal was supposed to be resident either within the prison or the rules.|The marshal was supposed to be resident either within the prison or the rules. +LJ002-0201|He seems to have felt no responsibility as to the welfare or comfort of those in charge, and out of whom he made all his money.|He seems to have felt no responsibility as to the welfare or comfort of those in charge, and out of whom he made all his money. +LJ002-0202|The prison was always in "the most filthy state imaginable."|The prison was always in "the most filthy state imaginable." +LJ002-0203|The half or wholly starved prisoners fished for alms or food at the gratings.|The half or wholly starved prisoners fished for alms or food at the gratings. +LJ002-0204|When they were sick no more notice was taken of them than of a dog.|When they were sick no more notice was taken of them than of a dog. +LJ002-0205|A man dying of liver complaint lay on the cold stones without a bed or food to eat.|A man dying of liver complaint lay on the cold stones without a bed or food to eat. +LJ002-0206|Dissolute habits prevailed on all sides; drunkenness was universal, gambling perpetual.|Dissolute habits prevailed on all sides; drunkenness was universal, gambling perpetual. +LJ002-0207|The yards were taken up with rackets and five courts, and here and there were "bumble puppy grounds," a game in which the players rolled iron balls|The yards were taken up with rackets and five courts, and here and there were "bumble puppy grounds," a game in which the players rolled iron balls +LJ002-0208|into holes marked with numbers.|into holes marked with numbers. +LJ002-0209|How to make most profit out of the wretched denizens of the jail was the marshal's only care. He got a rent for the coffee-house and the bake-house;|How to make most profit out of the wretched denizens of the jail was the marshal's only care. He got a rent for the coffee-house and the bake-house; +LJ002-0210|the keeper of the large tap-room called the Brace, because it was once kept by two brothers named Partridge, also paid him toll.|the keeper of the large tap-room called the Brace, because it was once kept by two brothers named Partridge, also paid him toll. +LJ002-0211|The sale of spirits was forbidden, but gin could always be had at the whistling shops, where it was known as Moonshine, Sky Blue,|The sale of spirits was forbidden, but gin could always be had at the whistling shops, where it was known as Moonshine, Sky Blue, +LJ002-0212|Mexico, and was consumed at the rate of a hogshead per week.|Mexico, and was consumed at the rate of a hogshead per week. +LJ002-0213|The Fleet, which stood in Farringdon Street,|The Fleet, which stood in Farringdon Street, +LJ002-0214|was a prison for debtors and persons committed for contempt by the courts of Chancery, Exchequer, and Common Pleas.|was a prison for debtors and persons committed for contempt by the courts of Chancery, Exchequer, and Common Pleas. +LJ002-0215|It was so used for the date of the abolition of the Star Chamber in the 16th Charles I.|It was so used for the date of the abolition of the Star Chamber in the sixteenth Charles the first +LJ002-0216|The shameful malpractices of Bambridge,|The shameful malpractices of Bambridge, +LJ002-0217|the warden of the Fleet at the commencement of the eighteenth century, are too well known to need more than a passing reference.|the warden of the Fleet at the commencement of the eighteenth century, are too well known to need more than a passing reference. +LJ002-0218|A committee of the House of Commons investigated the charges against Bambridge, who was proved to have connived at the escape of some debtors,|A committee of the House of Commons investigated the charges against Bambridge, who was proved to have connived at the escape of some debtors, +LJ002-0219|and to have been guilty of extortion to others. One Sir William Rich, Bart., he had loaded with heavy irons|and to have been guilty of extortion to others. One Sir William Rich, Bart., he had loaded with heavy irons +LJ002-0220|In consequence of these disclosures, both Bambridge and Huggin, his predecessor in the office, were committed to Newgate,|In consequence of these disclosures, both Bambridge and Huggin, his predecessor in the office, were committed to Newgate, +LJ002-0221|and many reforms instituted. But the condition of the prison and its inmates remained unsatisfactory to the last.|and many reforms instituted. But the condition of the prison and its inmates remained unsatisfactory to the last. +LJ002-0222|It contained generally from six to seven hundred inmates, while another hundred more or less resided in the rules outside.|It contained generally from six to seven hundred inmates, while another hundred more or less resided in the rules outside. +LJ002-0223|The principle of "chummage" prevailed as in the King's Bench,|The principle of "chummage" prevailed as in the King's Bench, +LJ002-0224|but a number of rooms, fifteen more or less, were reserved for poor debtors under the name of Bartholomew Fair.|but a number of rooms, fifteen more or less, were reserved for poor debtors under the name of Bartholomew Fair. +LJ002-0225|The rentals of rooms and fees went to the warden, whose income was £2372.|The rentals of rooms and fees went to the warden, whose income was two thousand three hundred seventy-two pounds. +LJ002-0226|The same evils of overcrowding, uncleanliness, want of medical attendance,|The same evils of overcrowding, uncleanliness, want of medical attendance, +LJ002-0227|absence or neglect of divine service, were present as in the King's Bench, but in an exaggerated form.|absence or neglect of divine service, were present as in the King's Bench, but in an exaggerated form. +LJ002-0228|The Committee on Jails reported that, quote, although the house of the warden looked into the court,|The Committee on Jails reported that, quote, although the house of the warden looked into the court, +LJ002-0229|and the turnkeys slept in the prison, yet scenes of riot, drunkenness, and disorder were most prevalent, end quote.|and the turnkeys slept in the prison, yet scenes of riot, drunkenness, and disorder were most prevalent, end quote. +LJ002-0230|The state of morals was disgraceful. Any woman obtained admission if sober, and if she got drunk she was not turned out.|The state of morals was disgraceful. Any woman obtained admission if sober, and if she got drunk she was not turned out. +LJ002-0231|There was no distinct place for the female debtors, who lived in the same galleries as the men.|There was no distinct place for the female debtors, who lived in the same galleries as the men. +LJ002-0232|Disturbances were frequent, owing to the riotous conduct of intoxicated women.|Disturbances were frequent, owing to the riotous conduct of intoxicated women. +LJ002-0233|Twice a week there was a wine and beer club held at night, which lasted till two or three in the morning.|Twice a week there was a wine and beer club held at night, which lasted till two or three in the morning. +LJ002-0234|In the yard behind the prison|In the yard behind the prison +LJ002-0235|were places set apart for skittles, fives, and tennis, which strangers frequented as any other place of public amusement.|were places set apart for skittles, fives, and tennis, which strangers frequented as any other place of public amusement. +LJ002-0236|Matters were rather better at the Marshalsea.|Matters were rather better at the Marshalsea. +LJ002-0237|This very ancient prison, which stood in the High Street, Southwark,|This very ancient prison, which stood in the High Street, Southwark, +LJ002-0238|was used for debtors arrested for the lowest sums within twelve miles of the palace of Whitehall;|was used for debtors arrested for the lowest sums within twelve miles of the palace of Whitehall; +LJ002-0239|also for prisoners committed by the Admiralty Court.|also for prisoners committed by the Admiralty Court. +LJ002-0240|At one time the Marshalsea was the receptacle of pirates, but none were committed to it after 1789.|At one time the Marshalsea was the receptacle of pirates, but none were committed to it after seventeen eighty-nine. +LJ002-0241|The court of the Marshalsea was instituted by Charles I. in the sixth year of his reign,|The court of the Marshalsea was instituted by Charles the first in the sixth year of his reign, +LJ002-0242|to be held before the steward of the royal household, the knight marshal, and the steward of the court,|to be held before the steward of the royal household, the knight marshal, and the steward of the court, +LJ002-0243|with jurisdiction to hold pleas in all actions within the prescribed limits. The court was chiefly used for the recovery of small debts under £10|with jurisdiction to hold pleas in all actions within the prescribed limits. The court was chiefly used for the recovery of small debts under ten pounds +LJ002-0244|but its business was much reduced by the extension of the Courts of Conscience.|but its business was much reduced by the extension of the Courts of Conscience. +LJ002-0245|The prison was a nest of abuses, like its neighbor the King's Bench|The prison was a nest of abuses, like its neighbor the King's Bench +LJ002-0246|and came under the strong animadversion of the Jail Committee of 1729.|and came under the strong animadversion of the Jail Committee of seventeen twenty-nine. +LJ002-0247|As the business of the Marshalsea Court declined, the numbers in its prison diminished.|As the business of the Marshalsea Court declined, the numbers in its prison diminished. +LJ002-0248|The population, as reported by the committee in 1814, averaged about sixty,|The population, as reported by the committee in eighteen fourteen, averaged about sixty, +LJ002-0249|and the prison, although wives and children resided within the walls, was not overcrowded.|and the prison, although wives and children resided within the walls, was not overcrowded. +LJ002-0250|Their conduct too was orderly on the whole.|Their conduct too was orderly on the whole. +LJ002-0251|Drunkenness was not common, chiefly because liquor was not to be had freely, although the tapster paid a rent of two guineas a week for permission to sell it.|Drunkenness was not common, chiefly because liquor was not to be had freely, although the tapster paid a rent of two guineas a week for permission to sell it. +LJ002-0252|The inmates, who euphemistically styled themselves "collegians,"|The inmates, who euphemistically styled themselves "collegians," +LJ002-0253|were governed by rules which they themselves had framed, and under which subscriptions were levied|were governed by rules which they themselves had framed, and under which subscriptions were levied +LJ002-0254|and fines imposed for conduct disapproved of by the "college."|and fines imposed for conduct disapproved of by the "college." +LJ002-0255|A court of the collegians was held every Monday to manage its affairs, at which all prisoners were required to attend.|A court of the collegians was held every Monday to manage its affairs, at which all prisoners were required to attend. +LJ002-0256|A committee of collegians was elected to act as the executive, also a secretary or accountant to receive monies and keep books,|A committee of collegians was elected to act as the executive, also a secretary or accountant to receive monies and keep books, +LJ002-0257|and a master of the ale-room, who kept this the scene of their revels clean, and saw that boiling water was provided for grog.|and a master of the ale-room, who kept this the scene of their revels clean, and saw that boiling water was provided for grog. +LJ002-0258|Bad language, quarreling, throwing water over one another was forbidden on pain of fine and being sent to Coventry;|Bad language, quarreling, throwing water over one another was forbidden on pain of fine and being sent to Coventry; +LJ002-0259|but the prevailing moral tone may be guessed from the penalty inflicted upon persons singing obscene songs before nine p.m.|but the prevailing moral tone may be guessed from the penalty inflicted upon persons singing obscene songs before nine p.m. +LJ002-0260|Yet the public opinion of the whole body seems to have checked dissipation.|Yet the public opinion of the whole body seems to have checked dissipation. +LJ002-0261|The poorer prisoners were not in abject want, as in other prisons,|The poorer prisoners were not in abject want, as in other prisons, +LJ002-0262|owing to many charitable gifts and bequests, which included annual donations from the Archbishop of Canterbury,|owing to many charitable gifts and bequests, which included annual donations from the Archbishop of Canterbury, +LJ002-0263|the Lord Steward of the Household, the steward and officers of the Marshalsea Court, and others.|the Lord Steward of the Household, the steward and officers of the Marshalsea Court, and others. +LJ002-0264|Legacies had also been left to free a certain number of debtors, notably that of £100 per annum|Legacies had also been left to free a certain number of debtors, notably that of one hundred pounds per annum +LJ002-0265|left by a Mr. Henry Allnutt, who was long a prisoner in the Marshalsea, and came into a fortune while there.|left by a Mr. Henry Allnutt, who was long a prisoner in the Marshalsea, and came into a fortune while there. +LJ002-0266|His bequest, which was charged upon his manor at Goring, Oxon, and hence called the Oxford Charity,|His bequest, which was charged upon his manor at Goring, Oxon, and hence called the Oxford Charity, +LJ002-0267|was applied only to the release of poor debtors whom £4 each could free.|was applied only to the release of poor debtors whom four pounds each could free. +LJ002-0268|The supreme control of the Marshalsea was vested in the marshal of the royal household; but although he drew a salary of £500 a year,|The supreme control of the Marshalsea was vested in the marshal of the royal household; but although he drew a salary of five hundred pounds a year, +LJ002-0269|he did nothing beyond visiting the prison occasionally, and left the administration to the deputy marshal.|he did nothing beyond visiting the prison occasionally, and left the administration to the deputy marshal. +LJ002-0270|The latter's salary, with fees, the rent of the tap and of the chandler's shop, amounted to about £600 a year.|The latter's salary, with fees, the rent of the tap and of the chandler's shop, amounted to about six hundred pounds a year. +LJ002-0271|The compters of Ludgate, Giltspur Street, and the Borough were discontinued as debtors' prisons (as was Newgate also)|The compters of Ludgate, Giltspur Street, and the Borough were discontinued as debtors' prisons (as was Newgate also) +LJ002-0272|on the opening of Whitecross prison for debtors in 1815.|on the opening of Whitecross prison for debtors in eighteen fifteen. +LJ002-0273|Ludgate to the last was the debtors' prison for freemen of the city of London,|Ludgate to the last was the debtors' prison for freemen of the city of London, +LJ002-0274|clergymen, proctors, attorneys, and persons specially selected by the Corporation.|clergymen, proctors, attorneys, and persons specially selected by the Corporation. +LJ002-0275|At one time the Ludgate debtors, accompanied by the keeper,|At one time the Ludgate debtors, accompanied by the keeper, +LJ002-0276|went outside and beyond the prison to call on their creditors, and try to arrange their debts, but this practice was discontinued.|went outside and beyond the prison to call on their creditors, and try to arrange their debts, but this practice was discontinued. +LJ002-0277|There were fifteen rooms of various sizes, and as the numbers imprisoned rarely exceeded five-and-twenty, the place was never overcrowded,|There were fifteen rooms of various sizes, and as the numbers imprisoned rarely exceeded five-and-twenty, the place was never overcrowded, +LJ002-0278|while the funds of several bequests and charities were applied in adding to the material comfort of the prisoners.|while the funds of several bequests and charities were applied in adding to the material comfort of the prisoners. +LJ002-0279|The Giltspur Street Compter received sheriffs' debtors, also felons, vagrants, and night charges.|The Giltspur Street Compter received sheriffs' debtors, also felons, vagrants, and night charges. +LJ002-0280|It was generally crowded, as debtors who would have gone to the Poultry Compter were sent to Giltspur Street when the former was condemned as unfit to receive prisoners.|It was generally crowded, as debtors who would have gone to the Poultry Compter were sent to Giltspur Street when the former was condemned as unfit to receive prisoners. +LJ002-0281|The demands for fees were excessive in Giltspur Street.|The demands for fees were excessive in Giltspur Street. +LJ002-0282|Those who could not pay were thrown into the wards with the night charges,|Those who could not pay were thrown into the wards with the night charges, +LJ002-0283|and denied admission to the "charity wards," which partook of all the benefits of bequests and donations to poor debtors.|and denied admission to the "charity wards," which partook of all the benefits of bequests and donations to poor debtors. +LJ002-0284|The Borough Compter was in a disgraceful state to the last. The men's ward had an earth, or rather a mud, floor,|The Borough Compter was in a disgraceful state to the last. The men's ward had an earth, or rather a mud, floor, +LJ002-0285|and was so unfit to sleep on that it had not been used for many years, so that the men and women associated together indiscriminately.|and was so unfit to sleep on that it had not been used for many years, so that the men and women associated together indiscriminately. +LJ002-0286|The rooms had no fireplaces, so it mattered little that no coals were allowed.|The rooms had no fireplaces, so it mattered little that no coals were allowed. +LJ002-0287|There were no beds or bedding, no straw even.|There were no beds or bedding, no straw even. +LJ002-0288|In one room Mr. Neild found a woman ill of a flux shut up with three men;|In one room Mr. Neild found a woman ill of a flux shut up with three men; +LJ002-0289|the latter raised eighteen pence among them to pay for a truss of straw for the poor woman to lie on.|the latter raised eighteen pence among them to pay for a truss of straw for the poor woman to lie on. +LJ002-0290|Neild found the prisoners in the Borough Compter ragged, starving, and dirty.|Neild found the prisoners in the Borough Compter ragged, starving, and dirty. +LJ002-0291|I come now to the debtors in Newgate. The quarters they occupied were divided, as I have said, into three principal divisions|I come now to the debtors in Newgate. The quarters they occupied were divided, as I have said, into three principal divisions +LJ002-0292|the master's side, the cabin side, and the common side. Payment of a fee of 3 shillings gained the debtor admission to the two first named;|the master's side, the cabin side, and the common side. Payment of a fee of three shillings gained the debtor admission to the two first named; +LJ002-0293|those who could pay nothing went, as a matter of course, to the common side;|those who could pay nothing went, as a matter of course, to the common side; +LJ002-0294|a further fee was, however, demanded from the new-comer before he was made free of either the master's or the cabin side.|a further fee was, however, demanded from the new-comer before he was made free of either the master's or the cabin side. +LJ002-0295|This was the reprehensible claim for "garnish," which had already been abolished in all well-conducted prisons, but which still was demanded in Newgate.|This was the reprehensible claim for "garnish," which had already been abolished in all well-conducted prisons, but which still was demanded in Newgate. +LJ002-0296|Garnish on the cabin side was a guinea at entrance for coals, candles, brooms, etc., and a gallon of beer on discharge;|Garnish on the cabin side was a guinea at entrance for coals, candles, brooms, etc., and a gallon of beer on discharge; +LJ002-0297|on the master's side it was thirteen and fourpence, and a gallon of beer on entrance, although Mr. Newman,|on the master's side it was thirteen and fourpence, and a gallon of beer on entrance, although Mr. Newman, +LJ002-0298|in his evidence in 1814, said it was more,|in his evidence in eighteen fourteen, said it was more, +LJ002-0299|and gave the garnish for the common side at that sum, which is five shillings more than Mr. Neild says was extorted on the common side.|and gave the garnish for the common side at that sum, which is five shillings more than Mr. Neild says was extorted on the common side. +LJ002-0300|Numerous tyrannies were practiced on all who would not and could not pay the garnish.|Numerous tyrannies were practiced on all who would not and could not pay the garnish. +LJ002-0301|They were made to wash and swab the ward, or they were shut out from the ward fireplace, and forbidden to pass a chalked line drawn on the floor,|They were made to wash and swab the ward, or they were shut out from the ward fireplace, and forbidden to pass a chalked line drawn on the floor, +LJ002-0302|and so were unable either to warm themselves or to cook their food.|and so were unable either to warm themselves or to cook their food. +LJ002-0303|Besides these fees, legitimate and illegitimate, there were others which must be paid before release.|Besides these fees, legitimate and illegitimate, there were others which must be paid before release. +LJ002-0304|The sheriff demanded 4 shillings, 6 pence for his liberate, the jailer 6 shillings, 10 pence more, and the turnkey 2 shillings;|The sheriff demanded four shillings, six pence for his liberate, the jailer six shillings, ten pence more, and the turnkey two shillings; +LJ002-0305|and thus when the debtor's debt had been actually paid, or when he had abandoned his property to the creditors, and, almost destitute,|and thus when the debtor's debt had been actually paid, or when he had abandoned his property to the creditors, and, almost destitute, +LJ002-0306|looked forward to his liberty, he was still delayed until he had paid a new debt arising, quote,|looked forward to his liberty, he was still delayed until he had paid a new debt arising, quote, +LJ002-0307|only out of a satisfaction of all his former debts, end quote. The fees were not always extorted, it is true;|only out of a satisfaction of all his former debts, end quote. The fees were not always extorted, it is true; +LJ002-0308|nor was non-payment made a pretext for further imprisonment, thanks to the humanity of the jailer, or the funds provided by various charities.|nor was non-payment made a pretext for further imprisonment, thanks to the humanity of the jailer, or the funds provided by various charities. +LJ002-0309|There was this much honest forbearance in Newgate in these days,|There was this much honest forbearance in Newgate in these days, +LJ002-0310|that debtors who could afford the cabin and master's side were not permitted to share in the prison charities.|that debtors who could afford the cabin and master's side were not permitted to share in the prison charities. +LJ002-0311|These were lumped together into a general fund,|These were lumped together into a general fund, +LJ002-0312|and a calculation made as to the amount that might be expended per week from the whole sum, so that the latter might last out the year.|and a calculation made as to the amount that might be expended per week from the whole sum, so that the latter might last out the year. +LJ002-0313|It generally ran to about six pounds per week. The money, which at one time had been distributed quarterly, and all went in drink,|It generally ran to about six pounds per week. The money, which at one time had been distributed quarterly, and all went in drink, +LJ002-0314|was after 1807, through the exertions of the keeper of the jail, spent in the purchase of necessaries.|was after eighteen oh seven, through the exertions of the keeper of the jail, spent in the purchase of necessaries. +LJ002-0315|But this weekly pittance did not go far when the debtors' side was crowded, as it often was;|But this weekly pittance did not go far when the debtors' side was crowded, as it often was; +LJ002-0316|notably as when numbers filled Newgate in anticipation of Lord Redesdale's bill for insolvent debtors,|notably as when numbers filled Newgate in anticipation of Lord Redesdale's bill for insolvent debtors, +LJ002-0317|and there were as many as three hundred and fifty prisoners in at one time.|and there were as many as three hundred and fifty prisoners in at one time. +LJ002-0318|The city also allowed the poor debtors fourteen ounces of bread daily, and their share of eight stone of meat, an allowance which never varied,|The city also allowed the poor debtors fourteen ounces of bread daily, and their share of eight stone of meat, an allowance which never varied, +LJ002-0319|issued once a week, and divided as far as it would go -- a very precarious and uncertain ration.|issued once a week, and divided as far as it would go -- a very precarious and uncertain ration. +LJ002-0320|The bread was issued every alternate day; and while some prisoners often ate their whole allowance at once,|The bread was issued every alternate day; and while some prisoners often ate their whole allowance at once, +LJ002-0321|others who arrived just after the time of distribution were often forty-eight hours without food. The latter might also be six days without meat.|others who arrived just after the time of distribution were often forty-eight hours without food. The latter might also be six days without meat. +LJ002-0322|Share in the weekly allowance of meat might also be denied to debtors who had not paid "garnish," as well as in the weekly grant from the charitable fund.|Share in the weekly allowance of meat might also be denied to debtors who had not paid "garnish," as well as in the weekly grant from the charitable fund. +LJ002-0323|Hence starvation stared many in the face, unless friends from outside came to their assistance,|Hence starvation stared many in the face, unless friends from outside came to their assistance, +LJ002-0324|or the keeper made them a special grant of 6 pence per diem out of the common stock;|or the keeper made them a special grant of six pence per diem out of the common stock; +LJ002-0325|or the sixpenny allowance was claimed for the creditors, which seldom happened, owing to the expense the process entailed.|or the sixpenny allowance was claimed for the creditors, which seldom happened, owing to the expense the process entailed. +LJ002-0326|The poor debtors were not supplied with beds. Those who could pay the price might hire them from each other,|The poor debtors were not supplied with beds. Those who could pay the price might hire them from each other, +LJ002-0327|or from persons who made a trade of it, or they might bring their beds with them into the prison.|or from persons who made a trade of it, or they might bring their beds with them into the prison. +LJ002-0328|Failing any of these methods, seeing that straw was forbidden for fear of fire, they had to be satisfied with a couple of the rugs provided by the city|Failing any of these methods, seeing that straw was forbidden for fear of fire, they had to be satisfied with a couple of the rugs provided by the city +LJ002-0329|the supply of which was, however, limited, and there were not always enough to give bedding to all. The stock was diminished by theft;|the supply of which was, however, limited, and there were not always enough to give bedding to all. The stock was diminished by theft; +LJ002-0330|female visitors carried them out of the prisons, or the debtors destroyed them when the weather was warm,|female visitors carried them out of the prisons, or the debtors destroyed them when the weather was warm, +LJ002-0331|and they were not in great demand, in order to convert them into mop-heads or cleaning-rags.|and they were not in great demand, in order to convert them into mop-heads or cleaning-rags. +LJ002-0332|Sometimes rugs were urgently required and not forthcoming;|Sometimes rugs were urgently required and not forthcoming; +LJ002-0333|a severe winter set in, the new stock had not been supplied by the contractors, and the poor debtors perished of cold.|a severe winter set in, the new stock had not been supplied by the contractors, and the poor debtors perished of cold. +LJ002-0334|Again, there was no regular allowance of fuel. Coals were purchased out of the garnish money and the charitable fund;|Again, there was no regular allowance of fuel. Coals were purchased out of the garnish money and the charitable fund; +LJ002-0335|so were candles, salt, pepper, mops and brooms. But the latter could have been of little service. Dirt prevailed everywhere;|so were candles, salt, pepper, mops and brooms. But the latter could have been of little service. Dirt prevailed everywhere; +LJ002-0336|indeed the place, with its oak floors caulked with pitch, and smoked ceilings,|indeed the place, with its oak floors caulked with pitch, and smoked ceilings, +LJ002-0337|could not be made even to look clean while there was no obligation of personal cleanliness on individuals, who often came into the prison in filthy rags.|could not be made even to look clean while there was no obligation of personal cleanliness on individuals, who often came into the prison in filthy rags. +LJ002-0338|Only now and again, in extreme cases, an unusually nasty companion was stripped, haled to the pump,|Only now and again, in extreme cases, an unusually nasty companion was stripped, haled to the pump, +LJ003-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 5: Newgate down to 1818, part 2.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section five: Newgate down to eighteen eighteen, part two. +LJ003-0002|The squalor and uncleanness of the debtors' side was intensified by constant overcrowding.|The squalor and uncleanness of the debtors' side was intensified by constant overcrowding. +LJ003-0003|Prisoners were committed to it quite without reference to its capacity.|Prisoners were committed to it quite without reference to its capacity. +LJ003-0004|No remonstrance was attended to,|No remonstrance was attended to, +LJ003-0005|no steps taken to reduce the number of committals, and the governor was obliged to utilize the chapel as a day and night room.|no steps taken to reduce the number of committals, and the governor was obliged to utilize the chapel as a day and night room. +LJ003-0006|Besides this, although the families of debtors were no longer permitted to live with them inside the jail,|Besides this, although the families of debtors were no longer permitted to live with them inside the jail, +LJ003-0007|hundreds of women and children came in every morning to spend the day there, and there was no limitation whatever to the numbers of visitors admitted to the debtors' side.|hundreds of women and children came in every morning to spend the day there, and there was no limitation whatever to the numbers of visitors admitted to the debtors' side. +LJ003-0008|Friends arrived about nine a.m., and went out at nine p.m., when as many as two hundred visitors have been observed leaving the debtors' yards at one time.|Friends arrived about nine a.m., and went out at nine p.m., when as many as two hundred visitors have been observed leaving the debtors' yards at one time. +LJ003-0009|The day passed in revelry and drunkenness. Although spirituous liquors were forbidden,|The day passed in revelry and drunkenness. Although spirituous liquors were forbidden, +LJ003-0010|wine and beer might be had in any quantity, the only limitation being|wine and beer might be had in any quantity, the only limitation being +LJ003-0011|that not more than one bottle of wine or one quart of beer could be issued at one time. No account was taken of the amount of liquors admitted in one day,|that not more than one bottle of wine or one quart of beer could be issued at one time. No account was taken of the amount of liquors admitted in one day, +LJ003-0012|and debtors might practically have as much as they liked, if they could only pay for it.|and debtors might practically have as much as they liked, if they could only pay for it. +LJ003-0013|No attempt was made to check drunkenness, beyond the penalty of shutting out friends from any ward in which a prisoner exceeded.|No attempt was made to check drunkenness, beyond the penalty of shutting out friends from any ward in which a prisoner exceeded. +LJ003-0014|Quarreling among the debtors was not unfrequent. Blows were struck, and fights often ensued.|Quarreling among the debtors was not unfrequent. Blows were struck, and fights often ensued. +LJ003-0015|For this and other acts of misconduct there was the discipline of the refractory ward, or "strong room" on the debtors' side.|For this and other acts of misconduct there was the discipline of the refractory ward, or "strong room" on the debtors' side. +LJ003-0016|Bad cases were removed to a cell on the felons' side, and here they were locked in solitary confinement for three days at a time.|Bad cases were removed to a cell on the felons' side, and here they were locked in solitary confinement for three days at a time. +LJ003-0017|Order throughout the debtors' side was preserved|Order throughout the debtors' side was preserved +LJ003-0018|and discipline maintained by a system open to grave abuses, and which had the prescription of long usage,|and discipline maintained by a system open to grave abuses, and which had the prescription of long usage, +LJ003-0019|and which was never wholly rooted out for many years to come.|and which was never wholly rooted out for many years to come. +LJ003-0020|This was the pernicious plan of governing by prisoners, or of setting a favored few in authority over the many.|This was the pernicious plan of governing by prisoners, or of setting a favored few in authority over the many. +LJ003-0021|The head of the debtors' prison was a prisoner called the steward, who was chosen by the whole body from six whom the keeper nominated.|The head of the debtors' prison was a prisoner called the steward, who was chosen by the whole body from six whom the keeper nominated. +LJ003-0022|This steward was practically supreme.|This steward was practically supreme. +LJ003-0023|All the allowances of food passed through his hands; he had the control of the poor-box for chance charities,|All the allowances of food passed through his hands; he had the control of the poor-box for chance charities, +LJ003-0024|he collected the garnish money, and distributed the weekly grant from the prison charitable fund.|he collected the garnish money, and distributed the weekly grant from the prison charitable fund. +LJ003-0025|In the latter duties he was, however, supervised by three auditors, freely chosen by the prisoners among themselves.|In the latter duties he was, however, supervised by three auditors, freely chosen by the prisoners among themselves. +LJ003-0026|The auditors were paid a shilling each for their services each time the poor-box was opened. The steward was also remunerated for his trouble.|The auditors were paid a shilling each for their services each time the poor-box was opened. The steward was also remunerated for his trouble. +LJ003-0027|He had a double allowance of bread, deducted, of course, from the already too limited portion of the rest,|He had a double allowance of bread, deducted, of course, from the already too limited portion of the rest, +LJ003-0028|and no doubt made the meat also pay toll.|and no doubt made the meat also pay toll. +LJ003-0029|Under the steward there were captains of wards, chosen in the same way, and performing analogous duties.|Under the steward there were captains of wards, chosen in the same way, and performing analogous duties. +LJ003-0030|These subordinate chiefs were also rewarded out of the scanty prison rations.|These subordinate chiefs were also rewarded out of the scanty prison rations. +LJ003-0031|The same system was extended to the criminal side, and cases were on record of the place of wardsman being sold for considerable sums.|The same system was extended to the criminal side, and cases were on record of the place of wardsman being sold for considerable sums. +LJ003-0032|So valuable were they deemed, that as much as fifty guineas was offered to the keeper for the post.|So valuable were they deemed, that as much as fifty guineas was offered to the keeper for the post. +LJ003-0033|Enough has been said, probably, to prove that there was room for improvement in the condition and treatment of debtors in the prisons of the city of London.|Enough has been said, probably, to prove that there was room for improvement in the condition and treatment of debtors in the prisons of the city of London. +LJ003-0034|This gradually was forced upon the consciousness of the Corporation,|This gradually was forced upon the consciousness of the Corporation, +LJ003-0035|and about 1812 application was made to Parliament for funds to build a new debtors' prison.|and about eighteen twelve application was made to Parliament for funds to build a new debtors' prison. +LJ003-0036|Authority was given to raise money on the Orphans' Fund to the extent of £90,000.|Authority was given to raise money on the Orphans' Fund to the extent of ninety thousand pounds. +LJ003-0037|A site was purchased between Red Lion and White Cross streets, and a new prison planned,|A site was purchased between Red Lion and White Cross streets, and a new prison planned, +LJ003-0038|which would accommodate the inmates of Newgate and of the three compters, Ludgate,|which would accommodate the inmates of Newgate and of the three compters, Ludgate, +LJ003-0039|Giltspur Street, and the Poultry, or about four hundred and seventy-six in all.|Giltspur Street, and the Poultry, or about four hundred and seventy-six in all. +LJ003-0040|The evils of association for these debtors were perpetuated, although the plan provided for the separation of the various contingents committed to it.|The evils of association for these debtors were perpetuated, although the plan provided for the separation of the various contingents committed to it. +LJ003-0041|There was no lack of air and light for the new jail, and several exercising yards.|There was no lack of air and light for the new jail, and several exercising yards. +LJ003-0043|and it was not ready to relieve Newgate till late in 1815.|and it was not ready to relieve Newgate till late in eighteen fifteen. +LJ003-0044|The reforms which were to be attempted in that prison|The reforms which were to be attempted in that prison +LJ003-0045|more particularly as regarded the classification of prisoners, and which were dependent on the space to be gained by the removal of the debtors,|more particularly as regarded the classification of prisoners, and which were dependent on the space to be gained by the removal of the debtors, +LJ003-0046|could not be carried out till then. It is to be feared that long after the opening of White Cross Street prison,|could not be carried out till then. It is to be feared that long after the opening of White Cross Street prison, +LJ003-0047|Newgate continued to be a reproach to those responsible for its management.|Newgate continued to be a reproach to those responsible for its management. +LJ003-0048|I pass now to the criminal side of Newgate, which consisted of the six quarters or yards already enumerated and described.|I pass now to the criminal side of Newgate, which consisted of the six quarters or yards already enumerated and described. +LJ003-0049|The inmates of this part, as distinguished from the debtors, were comprised in four classes:|The inmates of this part, as distinguished from the debtors, were comprised in four classes: +LJ003-0050|(1) those awaiting trial;|(one) those awaiting trial; +LJ003-0051|(2) persons under sentence of imprisonment for a fixed period, or until they shall have paid certain fines;|(two) persons under sentence of imprisonment for a fixed period, or until they shall have paid certain fines; +LJ003-0052|(3) transports awaiting removal to the colonies, and (4) capital convicts, condemned to death and awaiting execution.|(three) transports awaiting removal to the colonies, and (four) capital convicts, condemned to death and awaiting execution. +LJ003-0053|At one time the whole of these different categories were thrown together pell-mell, young and old, the untried with the convicted.|At one time the whole of these different categories were thrown together pell-mell, young and old, the untried with the convicted. +LJ003-0054|An imperfect attempt at classification was, however, made in 1812, and a yard was as far as possible set apart for the untried,|An imperfect attempt at classification was, however, made in eighteen twelve, and a yard was as far as possible set apart for the untried, +LJ003-0055|or class (1), with whom, under the imperious demand for accommodation, were also associated the misdemeanants, or class (2).|or class (one), with whom, under the imperious demand for accommodation, were also associated the misdemeanants, or class (two). +LJ003-0056|This was the chapel yard, with its five wards, which were calculated to hold seventy prisoners, but often held many more.|This was the chapel yard, with its five wards, which were calculated to hold seventy prisoners, but often held many more. +LJ003-0057|A further sub-classification was attempted by separating at night those charged with misdemeanors from those charged with felony,|A further sub-classification was attempted by separating at night those charged with misdemeanors from those charged with felony, +LJ003-0058|but all mingled freely during the day in the yard.|but all mingled freely during the day in the yard. +LJ003-0059|The sleeping accommodation in the chapel-yard wards, and indeed throughout the prison, consisted of a barrack bed,|The sleeping accommodation in the chapel-yard wards, and indeed throughout the prison, consisted of a barrack bed, +LJ003-0060|which was a wooden flooring on a slightly inclined plane, with a beam running across the top to serve as a pillow.|which was a wooden flooring on a slightly inclined plane, with a beam running across the top to serve as a pillow. +LJ003-0061|No beds were issued, only two rugs per prisoner.|No beds were issued, only two rugs per prisoner. +LJ003-0062|When each sleeper had the full lateral space allotted to him, it amounted to one foot and a half on the barrack bed;|When each sleeper had the full lateral space allotted to him, it amounted to one foot and a half on the barrack bed; +LJ003-0063|but when the ward was obliged to accommodate double the ordinary number, as was frequently the case,|but when the ward was obliged to accommodate double the ordinary number, as was frequently the case, +LJ003-0064|the sleepers covered the entire floor, with the exception of a passage in the middle.|the sleepers covered the entire floor, with the exception of a passage in the middle. +LJ003-0065|All the misdemeanants, whatever their offense, were lodged in this chapel ward.|All the misdemeanants, whatever their offense, were lodged in this chapel ward. +LJ003-0066|As many various and, according to our ideas, heinous crimes came under this head,|As many various and, according to our ideas, heinous crimes came under this head, +LJ003-0067|in the then existing state of the law,|in the then existing state of the law, +LJ003-0068|the man guilty of a common assault found himself side by side with the fraudulent, or others who had attempted abominable crimes.|the man guilty of a common assault found himself side by side with the fraudulent, or others who had attempted abominable crimes. +LJ003-0069|In this heterogeneous society were also thrown the unfortunate journalists to whom I have already referred, and on whom imprisonment in Newgate|In this heterogeneous society were also thrown the unfortunate journalists to whom I have already referred, and on whom imprisonment in Newgate +LJ003-0070|was frequently adjudged for so-called libels, or too out-spoken comments in print.|was frequently adjudged for so-called libels, or too out-spoken comments in print. +LJ003-0071|It was particularly recommended by the Committee on Jails in 1814|It was particularly recommended by the Committee on Jails in eighteen fourteen +LJ003-0072|that some other and less mixed prison should be used for the confinement of persons convicted of libels. But this suggestion was ignored.|that some other and less mixed prison should be used for the confinement of persons convicted of libels. But this suggestion was ignored. +LJ003-0073|Indeed the partial classification attempted seems to have been abandoned within a year or two.|Indeed the partial classification attempted seems to have been abandoned within a year or two. +LJ003-0074|The Hon. H. G. Bennet, who visited Newgate in 1817, saw in one yard, in a total of seventy-two prisoners,|The Hon. H. G. Bennet, who visited Newgate in eighteen seventeen, saw in one yard, in a total of seventy-two prisoners, +LJ003-0075|thirty-five tried and thirty-seven untried. Of the former, three were transports for life,|thirty-five tried and thirty-seven untried. Of the former, three were transports for life, +LJ003-0076|four for fourteen years, and three of them persons sentenced to fines or short imprisonment -- one for little more than a month.|four for fourteen years, and three of them persons sentenced to fines or short imprisonment -- one for little more than a month. +LJ003-0077|Two of the untried were for murder, and several for house-breaking and highway robbery.|Two of the untried were for murder, and several for house-breaking and highway robbery. +LJ003-0078|Nor were the misdemeanants and bail prisoners any longer separated from those whose crimes were of a more serious character.|Nor were the misdemeanants and bail prisoners any longer separated from those whose crimes were of a more serious character. +LJ003-0079|Mr. Bennet refers to a gentleman confined for want of bail, who occupied a room with five others|Mr. Bennet refers to a gentleman confined for want of bail, who occupied a room with five others +LJ003-0080|two committed by the Bankruptcy Commissioner, one for perjury, and two transports.|two committed by the Bankruptcy Commissioner, one for perjury, and two transports. +LJ003-0081|Persons convicted of publishing libels were still immured in the same rooms with transports and felons.|Persons convicted of publishing libels were still immured in the same rooms with transports and felons. +LJ003-0082|The middle yard, as far as its limits would permit, was appropriated to felons and transports. The wards here were generally very crowded.|The middle yard, as far as its limits would permit, was appropriated to felons and transports. The wards here were generally very crowded. +LJ003-0083|Each ward was calculated to hold twenty-four, allowing each individual one foot and a half;|Each ward was calculated to hold twenty-four, allowing each individual one foot and a half; +LJ003-0084|Quote, a common-sized man, says the keeper, Mr. Newman, can turn in nineteen inches, end quote.|Quote, a common-sized man, says the keeper, Mr. Newman, can turn in nineteen inches, end quote. +LJ003-0085|These twenty-four could just sleep on the barrack bed; when the number was higher, and it often rose to forty, the surplus had to sleep on the floor.|These twenty-four could just sleep on the barrack bed; when the number was higher, and it often rose to forty, the surplus had to sleep on the floor. +LJ003-0086|The crowding was in consequence of the delay in removing transports.|The crowding was in consequence of the delay in removing transports. +LJ003-0087|These often remained in Newgate for six months, sometimes a year, in some cases longer;|These often remained in Newgate for six months, sometimes a year, in some cases longer; +LJ003-0088|in one, for seven years -- that of a man sentenced to death, for whom great interest had been made, but whom it was not thought right to pardon.|in one, for seven years -- that of a man sentenced to death, for whom great interest had been made, but whom it was not thought right to pardon. +LJ003-0089|Occasionally the transports made themselves so useful in the jail that they were passed over.|Occasionally the transports made themselves so useful in the jail that they were passed over. +LJ003-0090|Mr. Newman admitted that he had petitioned that certain "trusty men" might be left in the jail.|Mr. Newman admitted that he had petitioned that certain "trusty men" might be left in the jail. +LJ003-0091|Constantly associated with these convicted felons were numbers of juveniles, infants of tender years.|Constantly associated with these convicted felons were numbers of juveniles, infants of tender years. +LJ003-0092|There were frequently in the middle yard seven or eight children, the youngest barely nine,|There were frequently in the middle yard seven or eight children, the youngest barely nine, +LJ003-0093|the oldest only twelve or thirteen, exposed to all the contaminating influences of the place.|the oldest only twelve or thirteen, exposed to all the contaminating influences of the place. +LJ003-0094|Mr. Bennet mentions also the case of young men of better stamp, clerks in city offices, and youths of good parentage,|Mr. Bennet mentions also the case of young men of better stamp, clerks in city offices, and youths of good parentage, +LJ003-0095|Quote, in this dreadful situation, end quote. who had been rescued from the hulks through the kindness and attention of the Secretary of State.|Quote, in this dreadful situation, end quote. who had been rescued from the hulks through the kindness and attention of the Secretary of State. +LJ003-0096|Quote, yet they had been long enough, he goes on to say, in the prison associated with the lowest and vilest criminals,|Quote, yet they had been long enough, he goes on to say, in the prison associated with the lowest and vilest criminals, +LJ003-0097|with convicts of all ages and characters, to render it next to impossible but that, with the obliteration of all sense of self-respect,|with convicts of all ages and characters, to render it next to impossible but that, with the obliteration of all sense of self-respect, +LJ003-0098|the inevitable consequence of such a situation, their morals must have been destroyed;|the inevitable consequence of such a situation, their morals must have been destroyed; +LJ003-0099|and though distress or the seduction of others might have led to the commission of this their first offense,|and though distress or the seduction of others might have led to the commission of this their first offense, +LJ003-0100|yet the society they were driven to live in, the language they daily heard, and the lessons they were taught in this academy,|yet the society they were driven to live in, the language they daily heard, and the lessons they were taught in this academy, +LJ003-0101|must have had a tendency to turn them into the world hardened and accomplished in the ways of vice and crime. End quote.|must have had a tendency to turn them into the world hardened and accomplished in the ways of vice and crime. End quote. +LJ003-0102|Mr. Buxton, in the work already quoted, instances another grievous case of the horrors of indiscriminate association in Newgate.|Mr. Buxton, in the work already quoted, instances another grievous case of the horrors of indiscriminate association in Newgate. +LJ003-0103|It was that of a person, quote, who practiced in the law, and who was connected by marriage with some very respectable families.|It was that of a person, quote, who practiced in the law, and who was connected by marriage with some very respectable families. +LJ003-0104|Having been committed to Clerkenwell,|Having been committed to Clerkenwell, +LJ003-0105|he was sent on to Newgate in a coach, handcuffed to a noted house-breaker, who was afterwards cast for death.|he was sent on to Newgate in a coach, handcuffed to a noted house-breaker, who was afterwards cast for death. +LJ003-0106|The first night in Newgate, and for the subsequent fortnight,|The first night in Newgate, and for the subsequent fortnight, +LJ003-0107|he slept in the same bed with a highwayman on one side, and a man charged with murder on the other.|he slept in the same bed with a highwayman on one side, and a man charged with murder on the other. +LJ003-0108|Spirits were freely introduced, and although he at first abstained,|Spirits were freely introduced, and although he at first abstained, +LJ003-0109|he found he must adopt the manners of his companions, or that his life would be in danger.|he found he must adopt the manners of his companions, or that his life would be in danger. +LJ003-0110|They viewed him with some suspicion, as one of whom they knew nothing.|They viewed him with some suspicion, as one of whom they knew nothing. +LJ003-0111|He was in consequence put out of the protection of their internal law, end quote. Their code was a subject of some curiosity.|He was in consequence put out of the protection of their internal law, end quote. Their code was a subject of some curiosity. +LJ003-0112|When any prisoner committed an offense against the community or against an individual, he was tried by a court in the jail.|When any prisoner committed an offense against the community or against an individual, he was tried by a court in the jail. +LJ003-0113|A prisoner, generally the oldest and most dexterous thief,|A prisoner, generally the oldest and most dexterous thief, +LJ003-0114|was appointed judge, and a towel tied in knots was hung on each side in imitation of a wig.|was appointed judge, and a towel tied in knots was hung on each side in imitation of a wig. +LJ003-0115|The judge sat in proper form; he was punctiliously styled "my lord."|The judge sat in proper form; he was punctiliously styled "my lord." +LJ003-0116|A jury having been selected and duly sworn, the culprit was then arraigned. Justice, however, was not administered with absolute integrity.|A jury having been selected and duly sworn, the culprit was then arraigned. Justice, however, was not administered with absolute integrity. +LJ003-0117|A bribe to the judge was certain to secure acquittal, and the neglect of the formality was as certainly followed by condemnation.|A bribe to the judge was certain to secure acquittal, and the neglect of the formality was as certainly followed by condemnation. +LJ003-0118|Various punishments were inflicted, the heaviest of which was standing in the pillory.|Various punishments were inflicted, the heaviest of which was standing in the pillory. +LJ003-0119|This was carried out by putting the criminal's head through the legs of a chair, and stretching out his arms and tying them to the legs.|This was carried out by putting the criminal's head through the legs of a chair, and stretching out his arms and tying them to the legs. +LJ003-0120|The culprit was then compelled to carry the chair about with him.|The culprit was then compelled to carry the chair about with him. +LJ003-0121|But all punishments might readily be commuted into a fine to be spent in gin for judge and jury.|But all punishments might readily be commuted into a fine to be spent in gin for judge and jury. +LJ003-0122|The prisoner mentioned above was continually persecuted by trials of this kind.|The prisoner mentioned above was continually persecuted by trials of this kind. +LJ003-0123|The most trifling acts were magnified into offenses.|The most trifling acts were magnified into offenses. +LJ003-0124|He was charged with moving something which should not be touched, with leaving a door open, or coughing maliciously to the disturbance of his companions.|He was charged with moving something which should not be touched, with leaving a door open, or coughing maliciously to the disturbance of his companions. +LJ003-0125|The evidence was invariably sufficient to convict, and the judge never hesitated to inflict the heaviest penalties.|The evidence was invariably sufficient to convict, and the judge never hesitated to inflict the heaviest penalties. +LJ003-0126|The unfortunate man was compelled at length to adopt the habits of his associates;|The unfortunate man was compelled at length to adopt the habits of his associates; +LJ003-0127|Quote, by insensible degrees he began to lose his repugnance to their society,|Quote, by insensible degrees he began to lose his repugnance to their society, +LJ003-0128|caught their flash terms and sung their songs, was admitted to their revels, and acquired, in place of habits of perfect sobriety,|caught their flash terms and sung their songs, was admitted to their revels, and acquired, in place of habits of perfect sobriety, +LJ003-0129|a taste for spirits. End quote.|a taste for spirits. End quote. +LJ003-0130|His wife visited him in Newgate, and wrote a pitiable account of the state in which she found her husband.|His wife visited him in Newgate, and wrote a pitiable account of the state in which she found her husband. +LJ003-0131|He was an inmate of the same ward with others of the most dreadful sort, quote,|He was an inmate of the same ward with others of the most dreadful sort, quote, +LJ003-0132|whose language and manners, whose female associates of the most abandoned description, and the scenes consequent with such lost wretches|whose language and manners, whose female associates of the most abandoned description, and the scenes consequent with such lost wretches +LJ003-0133|prevented me from going inside but seldom, and I used to communicate with him through the bars from the passage. End quote.|prevented me from going inside but seldom, and I used to communicate with him through the bars from the passage. End quote. +LJ003-0134|One day he was too ill to come down and meet her.|One day he was too ill to come down and meet her. +LJ003-0135|She went up to the ward and found him lying down, quote,|She went up to the ward and found him lying down, quote, +LJ003-0136|pale as death, very ill, and in a dreadfully dirty state, the wretches making game of him, and enjoying my distress;|pale as death, very ill, and in a dreadfully dirty state, the wretches making game of him, and enjoying my distress; +LJ003-0137|and I learned he had been up with the others the whole night.|and I learned he had been up with the others the whole night. +LJ003-0138|Though they could not force him to gamble, he was compelled to drink,|Though they could not force him to gamble, he was compelled to drink, +LJ003-0139|and I was obliged afterwards to let him have five shillings to pay his share,|and I was obliged afterwards to let him have five shillings to pay his share, +LJ003-0140|otherwise he would have been stripped of his clothes. End quote.|otherwise he would have been stripped of his clothes. End quote. +LJ003-0141|Felons who could pay the price were permitted, irrespective of their character or offenses,|Felons who could pay the price were permitted, irrespective of their character or offenses, +LJ003-0142|to purchase the greater ease and comfort of the master's side.|to purchase the greater ease and comfort of the master's side. +LJ003-0143|The entrance fee was at least 13 shillings, 6 pence a head, with half-a-crown a week more for bed and bedding,|The entrance fee was at least thirteen shillings, six pence a head, with half-a-crown a week more for bed and bedding, +LJ003-0144|the wards being furnished with barrack bedsteads, upon which each prisoner had the regulation allowance of sleeping room|the wards being furnished with barrack bedsteads, upon which each prisoner had the regulation allowance of sleeping room +LJ003-0145|or about a foot and a half laterally. These fees were in reality a substantial contribution towards the expenses of the jail;|or about a foot and a half laterally. These fees were in reality a substantial contribution towards the expenses of the jail; +LJ003-0146|without them the keeper declared that he could not pay the salaries of turnkeys and servants, nor keep the prison going at all.|without them the keeper declared that he could not pay the salaries of turnkeys and servants, nor keep the prison going at all. +LJ003-0147|Besides the jail fees, there was garnish of half-a-guinea, collected by the steward,|Besides the jail fees, there was garnish of half-a-guinea, collected by the steward, +LJ003-0148|and spent in providing coals, candles, plates, knives, and forks; while all the occupants of this part of the prison|and spent in providing coals, candles, plates, knives, and forks; while all the occupants of this part of the prison +LJ003-0149|supported themselves; they had the ration of prison bread only,|supported themselves; they had the ration of prison bread only, +LJ003-0150|but they had no share in the prison meat or other charities, and they or their friends found them in food.|but they had no share in the prison meat or other charities, and they or their friends found them in food. +LJ003-0151|All who could scrape together the cash seem to have gladly availed themselves of the privilege of entering the master's side.|All who could scrape together the cash seem to have gladly availed themselves of the privilege of entering the master's side. +LJ003-0152|It was the only way to escape the horrors, the distress, penury, and rags of the common yards.|It was the only way to escape the horrors, the distress, penury, and rags of the common yards. +LJ003-0153|Idleness was not so universally the rule in this part of the jail.|Idleness was not so universally the rule in this part of the jail. +LJ003-0154|Artisans and others were at liberty to work at their trades, provided they were not dangerous.|Artisans and others were at liberty to work at their trades, provided they were not dangerous. +LJ003-0155|Tailoring and shoemaking was permitted, but it was deemed unsafe to allow a carpenter or blacksmith to have his tools.|Tailoring and shoemaking was permitted, but it was deemed unsafe to allow a carpenter or blacksmith to have his tools. +LJ003-0156|All the money earned by prisoners was at their own disposal, and was spent almost habitually in drink, chambering, and wantonness.|All the money earned by prisoners was at their own disposal, and was spent almost habitually in drink, chambering, and wantonness. +LJ003-0157|The best accommodation the jail could offer was reserved for the prisoners on the state side,|The best accommodation the jail could offer was reserved for the prisoners on the state side, +LJ003-0158|from whom still higher fees were exacted, with the same discreditable idea of swelling the revenues of the prison.|from whom still higher fees were exacted, with the same discreditable idea of swelling the revenues of the prison. +LJ003-0159|To constitute this the aristocratic quarter, unwarrantable demands were made upon the space properly allotted to the female felons,|To constitute this the aristocratic quarter, unwarrantable demands were made upon the space properly allotted to the female felons, +LJ003-0160|and no lodger was rejected, whatever his status, who offered himself and could bring grist to the mill.|and no lodger was rejected, whatever his status, who offered himself and could bring grist to the mill. +LJ003-0161|The luxury of the state side was for a long time open to all who could pay|The luxury of the state side was for a long time open to all who could pay +LJ003-0162|the convicted felon, the transport awaiting removal, the lunatic whose case was still undecided,|the convicted felon, the transport awaiting removal, the lunatic whose case was still undecided, +LJ003-0163|the misdemeanant tried or untried, the debtor who wished to avoid the discomfort of the crowded debtors' side, the outspoken newspaper editor,|the misdemeanant tried or untried, the debtor who wished to avoid the discomfort of the crowded debtors' side, the outspoken newspaper editor, +LJ003-0164|or the daring reporter of parliamentary debates.|or the daring reporter of parliamentary debates. +LJ003-0165|The better class of inmate complained bitterly of this enforced companionship with the vile,|The better class of inmate complained bitterly of this enforced companionship with the vile, +LJ003-0166|association at one time forbidden by custom, but which greed and rapacity long made the rule.|association at one time forbidden by custom, but which greed and rapacity long made the rule. +LJ003-0167|The fee for admission to the state side, as fixed by the table of fees, was three guineas, but Mr. Newman declared that he never took more than two.|The fee for admission to the state side, as fixed by the table of fees, was three guineas, but Mr. Newman declared that he never took more than two. +LJ003-0168|Ten and sixpence a week more was charged as rent for a single bed; where two or more slept in a bed the rent was seven shillings a week each.|Ten and sixpence a week more was charged as rent for a single bed; where two or more slept in a bed the rent was seven shillings a week each. +LJ003-0169|Prisoners who could afford it sometimes paid for four beds, at the rate of twenty-eight shillings, and so secured the luxury of a private room.|Prisoners who could afford it sometimes paid for four beds, at the rate of twenty-eight shillings, and so secured the luxury of a private room. +LJ003-0170|A Mr. Lundy, charged with forgery, was thus accommodated on the state side for upwards of five years.|A Mr. Lundy, charged with forgery, was thus accommodated on the state side for upwards of five years. +LJ003-0171|But the keeper protested that no single prisoner could thus monopolize space if the state side was crowded.|But the keeper protested that no single prisoner could thus monopolize space if the state side was crowded. +LJ003-0172|The keeper went still further in his efforts to make money.|The keeper went still further in his efforts to make money. +LJ003-0173|He continued the ancient practice of letting out a portion of his own house, and by a poetical fiction treated it as an annex of the state side.|He continued the ancient practice of letting out a portion of his own house, and by a poetical fiction treated it as an annex of the state side. +LJ003-0174|Mr. Davison, sent to Newgate for embezzlement, and whose case is given in the preceding chapter,|Mr. Davison, sent to Newgate for embezzlement, and whose case is given in the preceding chapter, +LJ003-0175|was accommodated with a room in Mr. Newman's house at the extravagant rental of thirty guineas per week;|was accommodated with a room in Mr. Newman's house at the extravagant rental of thirty guineas per week; +LJ003-0176|Mr. Cobbett was also a lodger of Mr. Newman's; and so were any members of the aristocracy,|Mr. Cobbett was also a lodger of Mr. Newman's; and so were any members of the aristocracy, +LJ003-0177|if they happened to be in funds -- among whom was the Marquis of Sligo in 1811.|if they happened to be in funds -- among whom was the Marquis of Sligo in eighteen eleven. +LJ003-0178|The female felons' wards I shall describe at length in the next chapter,|The female felons' wards I shall describe at length in the next chapter, +LJ003-0179|which will deal with Mrs. Fry's philanthropic exertions at this period in this particular part of the prison.|which will deal with Mrs. Fry's philanthropic exertions at this period in this particular part of the prison. +LJ003-0180|These wards were always full to overflowing; sometimes double the number the rooms could accommodate were crowded into them.|These wards were always full to overflowing; sometimes double the number the rooms could accommodate were crowded into them. +LJ003-0181|There was a master's side for females who could pay the usual fees, but they associated with the rest in the one narrow yard common to all.|There was a master's side for females who could pay the usual fees, but they associated with the rest in the one narrow yard common to all. +LJ003-0182|The tried and the untried, young and old, were herded together|The tried and the untried, young and old, were herded together +LJ003-0183|sometimes girls of thirteen, twelve, even ten or nine years of age, were exposed to, quote,|sometimes girls of thirteen, twelve, even ten or nine years of age, were exposed to, quote, +LJ003-0184|all the contagion and profligacy which prevailed in this part of the prison, end quote.|all the contagion and profligacy which prevailed in this part of the prison, end quote. +LJ003-0185|There was no separation even for the women under sentence of death, who lived in a common and perpetually crowded ward.|There was no separation even for the women under sentence of death, who lived in a common and perpetually crowded ward. +LJ003-0186|Only when the order of execution came down were those about to suffer placed apart in one of the rooms in the arcade of the middle ward.|Only when the order of execution came down were those about to suffer placed apart in one of the rooms in the arcade of the middle ward. +LJ003-0187|I have kept till the last that part of the prison which was usually the last resting-place of so many.|I have kept till the last that part of the prison which was usually the last resting-place of so many. +LJ003-0188|The old press yard has been fully described in a previous chapter.|The old press yard has been fully described in a previous chapter. +LJ003-0189|The name still survived in the new press yard, which was the receptacle of the male condemned prisoners. It was generally crowded, like the rest of the prison.|The name still survived in the new press yard, which was the receptacle of the male condemned prisoners. It was generally crowded, like the rest of the prison. +LJ003-0190|Except in murder cases, where the execution was generally very promptly performed,|Except in murder cases, where the execution was generally very promptly performed, +LJ003-0191|strange and inconceivable delay occurred in carrying out the extreme sentence.|strange and inconceivable delay occurred in carrying out the extreme sentence. +LJ003-0192|Hence there was a terrible accumulation of prisoners in the condemned cells.|Hence there was a terrible accumulation of prisoners in the condemned cells. +LJ003-0193|Once, during the long illness of George III., as many as one hundred were there waiting the "Report," as it was called.|Once, during the long illness of George the third, as many as one hundred were there waiting the "Report," as it was called. +LJ003-0194|At another time there were fifty, one of whom had been under sentence a couple of years.|At another time there were fifty, one of whom had been under sentence a couple of years. +LJ003-0195|Mr. Bennet speaks of thirty-eight capital convicts he found in the press yard in February 1817,|Mr. Bennet speaks of thirty-eight capital convicts he found in the press yard in February eighteen seventeen, +LJ003-0196|five of whom had been condemned the previous July, four in September, and twenty-nine in October.|five of whom had been condemned the previous July, four in September, and twenty-nine in October. +LJ003-0197|This procrastination bred certain callousness.|This procrastination bred certain callousness. +LJ003-0198|Few realizing that the dreadful fate would overtake them, dismissed the prospect of death,|Few realizing that the dreadful fate would overtake them, dismissed the prospect of death, +LJ003-0199|and until the day was actually fixed, spent the time in roistering, swearing, gambling, or playing at ball.|and until the day was actually fixed, spent the time in roistering, swearing, gambling, or playing at ball. +LJ003-0200|Visitors were permitted access to them without stint;|Visitors were permitted access to them without stint; +LJ003-0201|unlimited drink was not denied them provided it was obtained in regulated quantities at one time.|unlimited drink was not denied them provided it was obtained in regulated quantities at one time. +LJ003-0202|These capital convicts, says Mr. Bennet, quote, lessened the ennui and despair of their situation by unbecoming merriment|These capital convicts, says Mr. Bennet, quote, lessened the ennui and despair of their situation by unbecoming merriment +LJ003-0203|or sought relief in the constant application of intoxicating stimulants.|or sought relief in the constant application of intoxicating stimulants. +LJ003-0204|I saw Cashman a few hours before his execution, smoking and drinking with the utmost unconcern and indifference, end quote.|I saw Cashman a few hours before his execution, smoking and drinking with the utmost unconcern and indifference, end quote. +LJ003-0205|Those who were thus reckless reacted upon the penitent who knew their days were numbered,|Those who were thus reckless reacted upon the penitent who knew their days were numbered, +LJ003-0206|and their gibes and jollity counteracted the ordinary's counsels or the independent preacher's earnest prayers.|and their gibes and jollity counteracted the ordinary's counsels or the independent preacher's earnest prayers. +LJ003-0207|For while Roman Catholics and Dissenters were encouraged to see ministers of their own persuasion,|For while Roman Catholics and Dissenters were encouraged to see ministers of their own persuasion, +LJ003-0208|a number of amateurs were ever ready to give their gratuitous ministrations to the condemned.|a number of amateurs were ever ready to give their gratuitous ministrations to the condemned. +LJ003-0209|The prisoners in the press yard had free access during the day to the yard and large day room;|The prisoners in the press yard had free access during the day to the yard and large day room; +LJ003-0210|at night they were placed in the fifteen cells, two, three, or more together, according to the total number to be accommodated.|at night they were placed in the fifteen cells, two, three, or more together, according to the total number to be accommodated. +LJ003-0211|They were never left quite alone for fear of suicide, and for the same reason they were searched for weapons or poisons.|They were never left quite alone for fear of suicide, and for the same reason they were searched for weapons or poisons. +LJ003-0212|But they nevertheless frequently managed to secrete the means of making away with themselves, and accomplished their purpose.|But they nevertheless frequently managed to secrete the means of making away with themselves, and accomplished their purpose. +LJ003-0213|Convicted murderers were kept continuously in the cells on bread and water,|Convicted murderers were kept continuously in the cells on bread and water, +LJ003-0214|in couples, from the time of sentence to that of execution, which was about three or four days generally,|in couples, from the time of sentence to that of execution, which was about three or four days generally, +LJ003-0215|from Friday to Monday, so as to include one Sunday, on which day there was a special service for the condemned in the prison chapel.|from Friday to Monday, so as to include one Sunday, on which day there was a special service for the condemned in the prison chapel. +LJ003-0216|This latter was an ordeal which all dreaded, and many avoided by denying their faith.|This latter was an ordeal which all dreaded, and many avoided by denying their faith. +LJ003-0217|The condemned occupied an open pew in the center of the chapel, hung with black; in front of them, upon a table, was a black coffin in full view.|The condemned occupied an open pew in the center of the chapel, hung with black; in front of them, upon a table, was a black coffin in full view. +LJ003-0218|The chapel was filled with a curious but callous congregation, who came to stare at the miserable people thus publicly exposed.|The chapel was filled with a curious but callous congregation, who came to stare at the miserable people thus publicly exposed. +LJ003-0219|Well might Mr. Bennet write that the condition of the condemned side was the most prominent of the manifold evils in the present system of Newgate,|Well might Mr. Bennet write that the condition of the condemned side was the most prominent of the manifold evils in the present system of Newgate, +LJ003-0220|quote, so discreditable to the metropolis, end quote.|quote, so discreditable to the metropolis, end quote. +LJ003-0221|Yet it must have been abundantly plain to the reader that the other evils existing were great and glaring. A brief summary of them will best prove this.|Yet it must have been abundantly plain to the reader that the other evils existing were great and glaring. A brief summary of them will best prove this. +LJ003-0222|The jail was neither suitable nor sufficiently large. It was not even kept weather-tight.|The jail was neither suitable nor sufficiently large. It was not even kept weather-tight. +LJ003-0223|The roof of the female prison, says the grand jury in their presentment in 1813, let in the rain.|The roof of the female prison, says the grand jury in their presentment in eighteen thirteen, let in the rain. +LJ003-0224|Supplies of common necessaries, such as have now been part of the furniture of every British jail for many years,|Supplies of common necessaries, such as have now been part of the furniture of every British jail for many years, +LJ003-0225|were meager or altogether absent.|were meager or altogether absent. +LJ003-0226|The rations of food were notoriously inadequate, and so carelessly distributed, that many were left to starve.|The rations of food were notoriously inadequate, and so carelessly distributed, that many were left to starve. +LJ003-0227|So unjust and unequal was the system, that the allowance to convicted criminals was better than that of the innocent debtor,|So unjust and unequal was the system, that the allowance to convicted criminals was better than that of the innocent debtor, +LJ003-0228|and the general insufficiency was such|and the general insufficiency was such +LJ003-0229|that it multiplied beyond all reason the number of visitors, many of whom came merely as the purveyors of food to their friends.|that it multiplied beyond all reason the number of visitors, many of whom came merely as the purveyors of food to their friends. +LJ003-0230|The prison allowances were eked out by the broken victuals generously given by several eating-house keepers in the city,|The prison allowances were eked out by the broken victuals generously given by several eating-house keepers in the city, +LJ003-0231|such as Messrs. Birch of Cornhill and Messrs. Leach and Dollimore of Ludgate Hill.|such as Messrs. Birch of Cornhill and Messrs. Leach and Dollimore of Ludgate Hill. +LJ003-0232|These were fetched away in a large tub on a truck by a turnkey.|These were fetched away in a large tub on a truck by a turnkey. +LJ003-0233|Amongst the heap was often the meat that had made turtle soup, which, when heated and stirred together in a saucepan, was said to be very good eating.|Amongst the heap was often the meat that had made turtle soup, which, when heated and stirred together in a saucepan, was said to be very good eating. +LJ003-0234|The bedding was scanty; fuel and light had to be purchased out of prisoners' private means; clothing was issued but rarely,|The bedding was scanty; fuel and light had to be purchased out of prisoners' private means; clothing was issued but rarely, +LJ003-0235|even to prisoners almost in nakedness, and as a special charitable gift. Extortion was practiced right and left.|even to prisoners almost in nakedness, and as a special charitable gift. Extortion was practiced right and left. +LJ003-0236|Garnish continued to be demanded long after it had disappeared in other and better-regulated prisons.|Garnish continued to be demanded long after it had disappeared in other and better-regulated prisons. +LJ003-0237|The fees on reception and discharge must be deemed exorbitant, when it is remembered the impoverished class who usually crowded the jail;|The fees on reception and discharge must be deemed exorbitant, when it is remembered the impoverished class who usually crowded the jail; +LJ003-0238|and they were exacted to relieve a rich corporation from paying for the maintenance of their own prison.|and they were exacted to relieve a rich corporation from paying for the maintenance of their own prison. +LJ003-0239|This imposition of fees left prisoners destitute on their discharge, without funds to support them in their first struggle to recommence life,|This imposition of fees left prisoners destitute on their discharge, without funds to support them in their first struggle to recommence life, +LJ003-0240|with ruined character, bad habits, and often bad health contracted in the jail.|with ruined character, bad habits, and often bad health contracted in the jail. +LJ003-0241|A further and a more iniquitous method of extorting money|A further and a more iniquitous method of extorting money +LJ003-0242|was still practiced, that of loading newly-arrived prisoners until they paid certain fees.|was still practiced, that of loading newly-arrived prisoners until they paid certain fees. +LJ003-0243|Ironing was still the rule, not only for the convicted, but for those charged with felonies; only the misdemeanants escaped.|Ironing was still the rule, not only for the convicted, but for those charged with felonies; only the misdemeanants escaped. +LJ003-0244|At the commencement of every sessions, such of the untried as had purchased "easement" of irons were called up and re-fettered,|At the commencement of every sessions, such of the untried as had purchased "easement" of irons were called up and re-fettered, +LJ003-0245|preparatory to their appearance in the Old Bailey. Irons were seldom removed from the convicted until discharge;|preparatory to their appearance in the Old Bailey. Irons were seldom removed from the convicted until discharge; +LJ003-0246|sometimes the wearer was declared medically unfit, or he obtained release by long good conduct,|sometimes the wearer was declared medically unfit, or he obtained release by long good conduct, +LJ003-0247|or the faithful discharge of some petty office, such as gatesman or captain of a ward.|or the faithful discharge of some petty office, such as gatesman or captain of a ward. +LJ003-0248|The irons weighed from three to four pounds, but heavier irons, seven or eight pounds' weight,|The irons weighed from three to four pounds, but heavier irons, seven or eight pounds' weight, +LJ003-0249|were imposed in case of misconduct; and when there had been an attempt at escape,|were imposed in case of misconduct; and when there had been an attempt at escape, +LJ003-0250|the culprit was chained down to the floor by running a chain through his irons which prevented him from climbing to the window of his cell.|the culprit was chained down to the floor by running a chain through his irons which prevented him from climbing to the window of his cell. +LJ003-0251|Among other excuses offered for thus manacling all almost without exception, was that it was the best and safest method|Among other excuses offered for thus manacling all almost without exception, was that it was the best and safest method +LJ003-0252|of distinguishing a prisoner from a stranger and temporary visitor.|of distinguishing a prisoner from a stranger and temporary visitor. +LJ003-0253|Clothes or prison uniform would not have served the purpose, for a disguise can be rapidly and secretly put on,|Clothes or prison uniform would not have served the purpose, for a disguise can be rapidly and secretly put on, +LJ003-0254|whereas irons cannot well be exchanged without loss of time and attracting much attention.|whereas irons cannot well be exchanged without loss of time and attracting much attention. +LJ003-0255|The unchecked admission of crowds of visitors to the felons' as well as the debtors' side was another unmixed evil.|The unchecked admission of crowds of visitors to the felons' as well as the debtors' side was another unmixed evil. +LJ003-0256|By this means spirits, otherwise unattainable and strictly prohibited, were smuggled into the jail.|By this means spirits, otherwise unattainable and strictly prohibited, were smuggled into the jail. +LJ003-0257|Searches were made certainly, but they were too often superficial, or they might be evaded by a trifling bribe.|Searches were made certainly, but they were too often superficial, or they might be evaded by a trifling bribe. +LJ003-0258|Hence the frequent cases of drunkenness, of which no notice was taken, unless people grew riotous in their cups|Hence the frequent cases of drunkenness, of which no notice was taken, unless people grew riotous in their cups +LJ003-0259|and attracted attention by their disorderly behavior.|and attracted attention by their disorderly behavior. +LJ003-0260|Another frightful consequence of this indiscriminate admission was the influx of numbers of abandoned women,|Another frightful consequence of this indiscriminate admission was the influx of numbers of abandoned women, +LJ003-0261|only a few of whom had the commendable prudery to pass themselves off as the wives of prisoners.|only a few of whom had the commendable prudery to pass themselves off as the wives of prisoners. +LJ003-0262|Any reputed, and indeed any real, wife might spend the night in Newgate if she would pay the shilling fee, commonly known as the "bad money,"|Any reputed, and indeed any real, wife might spend the night in Newgate if she would pay the shilling fee, commonly known as the "bad money," +LJ003-0263|which might have done something towards increasing the prison receipts, had it not been appropriated by the turnkey who winked at this evasion of the rules.|which might have done something towards increasing the prison receipts, had it not been appropriated by the turnkey who winked at this evasion of the rules. +LJ003-0264|Among the daily visitors were members of the criminal classes still at large,|Among the daily visitors were members of the criminal classes still at large, +LJ003-0265|the thieves and burglars who carried on the active business of their profession, from which their confederates were temporarily debarred.|the thieves and burglars who carried on the active business of their profession, from which their confederates were temporarily debarred. +LJ003-0266|One notorious character, while a prisoner awaiting transfer to the hulks,|One notorious character, while a prisoner awaiting transfer to the hulks, +LJ003-0267|kept open house, so to speak, and entertained daily within the walls a select party of the most noted thieves in London.|kept open house, so to speak, and entertained daily within the walls a select party of the most noted thieves in London. +LJ003-0268|This delectable society enticed into their set a clerk who had been imprisoned for fraud,|This delectable society enticed into their set a clerk who had been imprisoned for fraud, +LJ003-0269|and offered him half the booty if he would give full information as to the transactions and correspondence of his late employers.|and offered him half the booty if he would give full information as to the transactions and correspondence of his late employers. +LJ003-0270|Owing to the facility of intercourse between inside and outside, many crimes were doubtless hatched in Newgate.|Owing to the facility of intercourse between inside and outside, many crimes were doubtless hatched in Newgate. +LJ003-0271|Some of the worst and most extensive burglaries were planned there.|Some of the worst and most extensive burglaries were planned there. +LJ003-0273|"I believe," says Mr. Bennet in the letter already largely quoted,|"I believe," says Mr. Bennet in the letter already largely quoted, +LJ003-0274|"that there is no place in the metropolis where more crimes are projected or where stolen property is more secreted than in Newgate."|"that there is no place in the metropolis where more crimes are projected or where stolen property is more secreted than in Newgate." +LJ003-0275|These malpractices were fostered by the absence of all supervision and the generally unbroken idleness.|These malpractices were fostered by the absence of all supervision and the generally unbroken idleness. +LJ003-0276|Although attempted partially at Bridewell, and more systematically at the new Millbank penitentiary,|Although attempted partially at Bridewell, and more systematically at the new Millbank penitentiary, +LJ003-0277|but just open (1816), the regular employment of prisoners had never yet been accepted as a principle in the metropolitan prisons.|but just open (eighteen sixteen), the regular employment of prisoners had never yet been accepted as a principle in the metropolitan prisons. +LJ003-0278|Insuperable difficulties were still supposed to stand in the way of any general employment of prisoners at their trades.|Insuperable difficulties were still supposed to stand in the way of any general employment of prisoners at their trades. +LJ003-0279|There was fear as to the unrestricted use of tools,|There was fear as to the unrestricted use of tools, +LJ003-0280|limits of space, the interference of the ill-disposed, who would neither work nor let others do so,|limits of space, the interference of the ill-disposed, who would neither work nor let others do so, +LJ003-0281|and the danger of losing material, raw or manufactured.|and the danger of losing material, raw or manufactured. +LJ003-0282|Many years were to elapse before these objections should be fairly met and universally overcome.|Many years were to elapse before these objections should be fairly met and universally overcome. +LJ003-0283|It was not strange, therefore, that the inmates of Newgate should turn their unoccupied brains and idle hands to all manner of mischief;|It was not strange, therefore, that the inmates of Newgate should turn their unoccupied brains and idle hands to all manner of mischief; +LJ003-0284|that when they were not carousing, plotting, or scheming,|that when they were not carousing, plotting, or scheming, +LJ003-0285|they should gamble with dice or cards, and play at bumble puppy or some other disreputable game of chance.|they should gamble with dice or cards, and play at bumble puppy or some other disreputable game of chance. +LJ003-0286|The report of the Committee of the House of Commons painted so black a picture of Newgate as then conducted, that the Corporation were roused in very shame|The report of the Committee of the House of Commons painted so black a picture of Newgate as then conducted, that the Corporation were roused in very shame +LJ003-0287|to undertake some kind of reform.|to undertake some kind of reform. +LJ003-0288|The above-mentioned report was ordered to be printed upon the 9th May.|The above-mentioned report was ordered to be printed upon the ninth May. +LJ003-0289|Upon the 29th July the same year,|Upon the twenty-ninth July the same year, +LJ003-0290|the court of aldermen appointed a committee of its own body, assisted by the town clerk, Mr. Dance, city surveyor, son to the architect of Newgate,|the court of aldermen appointed a committee of its own body, assisted by the town clerk, Mr. Dance, city surveyor, son to the architect of Newgate, +LJ003-0291|and Mr. Addison, keeper of Newgate, to make a visitation of the jails supposed to be the best managed, including those of Petworth and Gloucester.|and Mr. Addison, keeper of Newgate, to make a visitation of the jails supposed to be the best managed, including those of Petworth and Gloucester. +LJ003-0292|This committee was to compare allowances, examine rules, and certify as to the condition of prisoners;|This committee was to compare allowances, examine rules, and certify as to the condition of prisoners; +LJ003-0293|also to make such proposals as might appear salutary, and calculated to improve Newgate and the rest of the city jails.|also to make such proposals as might appear salutary, and calculated to improve Newgate and the rest of the city jails. +LJ003-0294|This committee made its report in September the following year, and an excellent report it is, so far as its recommendations are concerned.|This committee made its report in September the following year, and an excellent report it is, so far as its recommendations are concerned. +LJ003-0295|The committee seems to have fully realized, even at this early date (1815),|The committee seems to have fully realized, even at this early date (eighteen fifteen), +LJ003-0296|many of the indispensable conditions of a model prison according to modern ideas.|many of the indispensable conditions of a model prison according to modern ideas. +LJ003-0297|It admitted the paramount necessity|It admitted the paramount necessity +LJ003-0298|for giving every prisoner a sleeping cell to himself, an amount of enlightenment which is hardly general among European nations at this|for giving every prisoner a sleeping cell to himself, an amount of enlightenment which is hardly general among European nations at this +LJ003-0299|the latter end of the nineteenth century, several of which still fall far short of our English ideal,|the latter end of the nineteenth century, several of which still fall far short of our English ideal, +LJ003-0300|that all prisoners should always be in separate cells by night, and those of short sentences by day.|that all prisoners should always be in separate cells by night, and those of short sentences by day. +LJ003-0301|It recommended day cells or rooms for regular labor, which should be compulsory upon all transports and prisoners sentenced to hard labor,|It recommended day cells or rooms for regular labor, which should be compulsory upon all transports and prisoners sentenced to hard labor, +LJ003-0302|the work being constant and suitable, with certain hours of relaxation and for food and exercise.|the work being constant and suitable, with certain hours of relaxation and for food and exercise. +LJ003-0303|The personal cleanliness of all prisoners was to be insisted upon; they should be made to wash at least once a day,|The personal cleanliness of all prisoners was to be insisted upon; they should be made to wash at least once a day, +LJ003-0304|with the penalty of forfeiting the day's allowance of food, an increase of which the committee had recommended.|with the penalty of forfeiting the day's allowance of food, an increase of which the committee had recommended. +LJ003-0305|The provision of more baths was also suggested, and the daily sweeping out of the prison.|The provision of more baths was also suggested, and the daily sweeping out of the prison. +LJ003-0306|The clothes of prisoners arriving dirty, or in rags, should be fumigated before worn in the jail,|The clothes of prisoners arriving dirty, or in rags, should be fumigated before worn in the jail, +LJ003-0308|A laundry should be established, and a matron appointed on the female side, where all the prisoners' washing could be performed.|A laundry should be established, and a matron appointed on the female side, where all the prisoners' washing could be performed. +LJ003-0309|Proper hours for locking and unlocking prisoners should be insisted upon;|Proper hours for locking and unlocking prisoners should be insisted upon; +LJ003-0310|a bell should give notice thereof, and of meal-hours, working-hours, or of escapes.|a bell should give notice thereof, and of meal-hours, working-hours, or of escapes. +LJ003-0311|The committee took upon itself to lay down stringent rules for the discipline of the prison.|The committee took upon itself to lay down stringent rules for the discipline of the prison. +LJ003-0312|The jailer should be required to visit every part and see every prisoner daily; the chaplain should perform service, visit the sick,|The jailer should be required to visit every part and see every prisoner daily; the chaplain should perform service, visit the sick, +LJ003-0313|instruct the prisoners, quote, give spiritual advice and administer religious consolation, end quote. to all who might need them;|instruct the prisoners, quote, give spiritual advice and administer religious consolation, end quote. to all who might need them; +LJ003-0314|the surgeon should see all prisoners, whether ill or well, once a week, and take general charge of the infirmaries.|the surgeon should see all prisoners, whether ill or well, once a week, and take general charge of the infirmaries. +LJ003-0315|All three, governor, chaplain, and surgeon, should keep journals, which should be inspected periodically by the visiting magistrates.|All three, governor, chaplain, and surgeon, should keep journals, which should be inspected periodically by the visiting magistrates. +LJ003-0316|It should be peremptorily forbidden to the keeper or any officer to make a pecuniary profit out of the supplies of food, fuel, or other necessaries.|It should be peremptorily forbidden to the keeper or any officer to make a pecuniary profit out of the supplies of food, fuel, or other necessaries. +LJ003-0317|No prisoner should be allowed to obtain superior accommodation on the payment of any fees. Fees indeed should be generally abolished, garnish also.|No prisoner should be allowed to obtain superior accommodation on the payment of any fees. Fees indeed should be generally abolished, garnish also. +LJ003-0318|No prisoners should in future be ironed, except in cases of misconduct,|No prisoners should in future be ironed, except in cases of misconduct, +LJ003-0319|provided only that their security was not jeopardized, and dependent upon the enforcement of another new rule,|provided only that their security was not jeopardized, and dependent upon the enforcement of another new rule, +LJ003-0320|which recommended restrictions upon the number of visitors admitted.|which recommended restrictions upon the number of visitors admitted. +LJ003-0321|No wine or beer should be in future admitted into or sold in the jail,|No wine or beer should be in future admitted into or sold in the jail, +LJ003-0322|except for the use of the debtors, or as medical comforts for the infirmary.|except for the use of the debtors, or as medical comforts for the infirmary. +LJ003-0323|Drunkenness, if it ever occurred, should be visited with severe punishment;|Drunkenness, if it ever occurred, should be visited with severe punishment; +LJ003-0324|gaming of all sorts should be peremptorily forbidden under heavy pains and penalties.|gaming of all sorts should be peremptorily forbidden under heavy pains and penalties. +LJ003-0325|The feelings of the condemned prisoners should no longer be outraged by their exposure in the chapel, and the chapel should be rearranged,|The feelings of the condemned prisoners should no longer be outraged by their exposure in the chapel, and the chapel should be rearranged, +LJ003-0326|so that the various classes might be seated separately, and so as not to see each other.|so that the various classes might be seated separately, and so as not to see each other. +LJ003-0327|It will hardly be denied that these proposals went to the root of the matter.|It will hardly be denied that these proposals went to the root of the matter. +LJ003-0328|Had they been accepted in their entirety, little fault could in future have been found with the managers of Newgate.|Had they been accepted in their entirety, little fault could in future have been found with the managers of Newgate. +LJ003-0329|In common justice to them, it must be admitted that immediate effect was given to all that could be easily carried out.|In common justice to them, it must be admitted that immediate effect was given to all that could be easily carried out. +LJ003-0330|The state side ceased to exist, and the female prisoners thus regained the space of which their quadrangle had been robbed.|The state side ceased to exist, and the female prisoners thus regained the space of which their quadrangle had been robbed. +LJ003-0331|The privileges of the master's side also disappeared; fees were nominally abolished, and garnish was scotched, although not yet killed outright.|The privileges of the master's side also disappeared; fees were nominally abolished, and garnish was scotched, although not yet killed outright. +LJ003-0332|A certain number of bedsteads were provided, and there was a slight increase in the ration of bread.|A certain number of bedsteads were provided, and there was a slight increase in the ration of bread. +LJ003-0333|But here the recommendations touched at once upon the delicate subject of expense, and it is clear that the committee hesitated on this score.|But here the recommendations touched at once upon the delicate subject of expense, and it is clear that the committee hesitated on this score. +LJ003-0334|It made this too the excuse for begging the most important issue of the whole question.|It made this too the excuse for begging the most important issue of the whole question. +LJ003-0335|The committee did not deny the superior advantages offered by such prisons as Gloucester and Petworth,|The committee did not deny the superior advantages offered by such prisons as Gloucester and Petworth, +LJ003-0336|but it at once deprecated the idea that the city could follow the laudable example thus set in the provinces. Quote,|but it at once deprecated the idea that the city could follow the laudable example thus set in the provinces. Quote, +LJ003-0337|Were a metropolitan prison erected on the same lines, with all the space not only for air and exercise, but for day rooms and sleeping cells|Were a metropolitan prison erected on the same lines, with all the space not only for air and exercise, but for day rooms and sleeping cells +LJ003-0338|End quote. it would cover some thirty acres, and cost a great deal more than the city, with the example of Whitecross Street prison before it,|End quote. it would cover some thirty acres, and cost a great deal more than the city, with the example of Whitecross Street prison before it, +LJ003-0339|could possibly afford.|could possibly afford. +LJ003-0340|The committee does not seem to have yet understood that Newgate could be only and properly replaced|The committee does not seem to have yet understood that Newgate could be only and properly replaced +LJ003-0341|by a new jail built on the outskirts, as Holloway eventually was, and permitted itself to be altogether countered|by a new jail built on the outskirts, as Holloway eventually was, and permitted itself to be altogether countered +LJ003-0342|and checked in its efforts towards reform by the prohibitory costliness of the land about Newgate.|and checked in its efforts towards reform by the prohibitory costliness of the land about Newgate. +LJ003-0343|With the seeming impossibility of extending the limits of the prison as it then stood,|With the seeming impossibility of extending the limits of the prison as it then stood, +LJ003-0344|all chances of classification and separation vanished, and the greatest evils remained untouched.|all chances of classification and separation vanished, and the greatest evils remained untouched. +LJ003-0345|All the committee could do in this respect was to throw the responsibility on others.|All the committee could do in this respect was to throw the responsibility on others. +LJ003-0346|It pointed out that the Government was to blame for the overcrowding, and might diminish it if it chose.|It pointed out that the Government was to blame for the overcrowding, and might diminish it if it chose. +LJ003-0347|It was very desirable that there should be a more speedy removal of transports from Newgate to the ships.|It was very desirable that there should be a more speedy removal of transports from Newgate to the ships. +LJ003-0348|Again, there was the new Millbank penitentiary now ready for occupation.|Again, there was the new Millbank penitentiary now ready for occupation. +LJ003-0349|Why not relieve Newgate by drawing more largely upon the superior accommodation which Millbank offered?|Why not relieve Newgate by drawing more largely upon the superior accommodation which Millbank offered? +LJ004-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 7: The beginnings of prison reform.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section seven: The beginnings of prison reform. +LJ004-0002|While Mrs. Fry was diligently engaged upon her self-imposed task in Newgate,|While Mrs. Fry was diligently engaged upon her self-imposed task in Newgate, +LJ004-0003|other earnest people, inspired doubtless by her noble example, were stirred up to activity in the same great work.|other earnest people, inspired doubtless by her noble example, were stirred up to activity in the same great work. +LJ004-0004|It began to be understood that prison reform could only be compassed by continuous and combined effort.|It began to be understood that prison reform could only be compassed by continuous and combined effort. +LJ004-0005|The pleadings, however eloquent, of a single individual were unable to more than partially remedy the widespread and colossal evils of British prisons.|The pleadings, however eloquent, of a single individual were unable to more than partially remedy the widespread and colossal evils of British prisons. +LJ004-0006|Howard's energy and devotion were rewarded by lively sympathy, but the desire to improve which followed his exposures was but short-lived.|Howard's energy and devotion were rewarded by lively sympathy, but the desire to improve which followed his exposures was but short-lived. +LJ004-0007|It was so powerless against the persistent neglect of those intrusted with prison management, that, five-and-twenty years later,|It was so powerless against the persistent neglect of those intrusted with prison management, that, five-and-twenty years later, +LJ004-0008|Mr. Neild, a second Howard,|Mr. Neild, a second Howard, +LJ004-0009|as indefatigable and self-sacrificing, found by personal visitation that the condition of jails throughout the kingdom was,|as indefatigable and self-sacrificing, found by personal visitation that the condition of jails throughout the kingdom was, +LJ004-0010|with a few bright exceptions, still deplorable and disgraceful.|with a few bright exceptions, still deplorable and disgraceful. +LJ004-0011|Mr. Neild was compelled to admit in 1812 that "the great reformation produced by Howard|Mr. Neild was compelled to admit in eighteen twelve that "the great reformation produced by Howard +LJ004-0012|was in several places merely temporary:|was in several places merely temporary: +LJ004-0013|some prisons that had been ameliorated under the persuasive influence of his kind advice were relapsing into their former horrid state of privation,|some prisons that had been ameliorated under the persuasive influence of his kind advice were relapsing into their former horrid state of privation, +LJ004-0014|filthiness, severity, or neglect; many new dungeons had aggravated the evils against which his sagacity could not but remonstrate;|filthiness, severity, or neglect; many new dungeons had aggravated the evils against which his sagacity could not but remonstrate; +LJ004-0015|the motives for a transient amendment were becoming paralyzed, and the effect had ceased with the cause."|the motives for a transient amendment were becoming paralyzed, and the effect had ceased with the cause." +LJ004-0016|I have shown in a previous chapter what Newgate was at this period, despite a vast expenditure and boasted efforts to introduce reforms.|I have shown in a previous chapter what Newgate was at this period, despite a vast expenditure and boasted efforts to introduce reforms. +LJ004-0017|Some of the county jails, and one or two borough jails, had been rebuilt,|Some of the county jails, and one or two borough jails, had been rebuilt, +LJ004-0018|generally through the personal activity of influential and benevolent local magnates, but the true principles of prison construction|generally through the personal activity of influential and benevolent local magnates, but the true principles of prison construction +LJ004-0019|were as yet but imperfectly understood, and such portions of the "improved" jails of that period as were still extant a few years back,|were as yet but imperfectly understood, and such portions of the "improved" jails of that period as were still extant a few years back, +LJ004-0020|contrast ludicrously with the prison architecture based upon a century's experience of our own age.|contrast ludicrously with the prison architecture based upon a century's experience of our own age. +LJ004-0021|The neglect of prison reform in those days was not to be visited upon the legislature.|The neglect of prison reform in those days was not to be visited upon the legislature. +LJ004-0022|The executive, although harassed by internal commotion and foreign war, was not entirely callous to the crying need for amelioration in jails.|The executive, although harassed by internal commotion and foreign war, was not entirely callous to the crying need for amelioration in jails. +LJ004-0023|Measures remedial, although at best partial and incomplete, were introduced from time to time.|Measures remedial, although at best partial and incomplete, were introduced from time to time. +LJ004-0024|Thus in 1813 the exaction of jail fees had been forbidden by law,|Thus in eighteen thirteen the exaction of jail fees had been forbidden by law, +LJ004-0025|and two other acts more peremptory and precise followed on the same subject in succeeding years.|and two other acts more peremptory and precise followed on the same subject in succeeding years. +LJ004-0026|In 1814 a bill was brought in to insist upon the appointment of chaplains in jails, and when this had passed into law,|In eighteen fourteen a bill was brought in to insist upon the appointment of chaplains in jails, and when this had passed into law, +LJ004-0027|it was subsequently amplified, and the rates of salaries fixed.|it was subsequently amplified, and the rates of salaries fixed. +LJ004-0028|Various acts were also passed to consolidate and amend previous jail acts.|Various acts were also passed to consolidate and amend previous jail acts. +LJ004-0029|The erection of new prison buildings was made imperative under certain conditions and following certain rules|The erection of new prison buildings was made imperative under certain conditions and following certain rules +LJ004-0030|the principle of classification was freshly enunciated; prison regulations were framed for general observance.|the principle of classification was freshly enunciated; prison regulations were framed for general observance. +LJ004-0031|But the effect of this legislation was rather weakened by the remoteness of the pressure exercised.|But the effect of this legislation was rather weakened by the remoteness of the pressure exercised. +LJ004-0032|The onus of improvement lay upon the magistracy, the local authorities administering local funds,|The onus of improvement lay upon the magistracy, the local authorities administering local funds, +LJ004-0033|and they were not threatened with any particular penalties if they evaded or ignored the new acts.|and they were not threatened with any particular penalties if they evaded or ignored the new acts. +LJ004-0034|Moreover, the laws applied more particularly to county jurisdictions.|Moreover, the laws applied more particularly to county jurisdictions. +LJ004-0035|The borough jails, those in fact under corporate management, were not included in the new measures;|The borough jails, those in fact under corporate management, were not included in the new measures; +LJ004-0036|it was hoped that their rulers would hire accommodation in the county prisons, and that the inferior establishments would in course of time disappear.|it was hoped that their rulers would hire accommodation in the county prisons, and that the inferior establishments would in course of time disappear. +LJ004-0037|Yet the borough jails were destined to survive many years, and to exhibit for a long time to come all the worst features of jail mismanagement.|Yet the borough jails were destined to survive many years, and to exhibit for a long time to come all the worst features of jail mismanagement. +LJ004-0038|It was in 1817 that a small band of philanthropists resolved to form themselves into an association for the improvement of prison discipline.|It was in eighteen seventeen that a small band of philanthropists resolved to form themselves into an association for the improvement of prison discipline. +LJ004-0039|They were hopeless of any general reform by the action of the executive alone.|They were hopeless of any general reform by the action of the executive alone. +LJ004-0040|They felt that private enterprise might|They felt that private enterprise might +LJ004-0041|with advantage step in, and by the collection and diffusion of information, and the reiteration of sound advice, greatly assist the good work.|with advantage step in, and by the collection and diffusion of information, and the reiteration of sound advice, greatly assist the good work. +LJ004-0042|The association was organized under the most promising auspices.|The association was organized under the most promising auspices. +LJ004-0043|A king's son, the Duke of Gloucester, was the patron; among the vice-presidents were many great peers of the realm|A king's son, the Duke of Gloucester, was the patron; among the vice-presidents were many great peers of the realm +LJ004-0044|several bishops, and a number of members of the House of Commons, including Mr. Manners Sutton,|several bishops, and a number of members of the House of Commons, including Mr. Manners Sutton, +LJ004-0045|Mr. Sturges Bourne, Sir James Mackintosh, Sir James Scarlett, and William Wilberforce.|Mr. Sturges Bourne, Sir James Mackintosh, Sir James Scarlett, and William Wilberforce. +LJ004-0046|An active committee was appointed, comprising many names already well known, some of them destined to become famous in the annals of philanthropy.|An active committee was appointed, comprising many names already well known, some of them destined to become famous in the annals of philanthropy. +LJ004-0047|One of the moving spirits was the Honorable H. G. Bennet, M.P., whose vigorous protests against the lamentable condition of Newgate have already been recorded.|One of the moving spirits was the Honorable H. G. Bennet, M.P., whose vigorous protests against the lamentable condition of Newgate have already been recorded. +LJ004-0048|Mrs. Fry's brother, Mr. Samuel Hoare, Junior, was chairman of the committee, on which also served many noted members of the Society of Friends|Mrs. Fry's brother, Mr. Samuel Hoare, Junior, was chairman of the committee, on which also served many noted members of the Society of Friends +LJ004-0049|Mr. Gurney, Mr. Fry, Messrs. Forster, and Mr. T. F. Buxton, the coadjutor of Wilberforce in the great anti-slavery struggle.|Mr. Gurney, Mr. Fry, Messrs. Forster, and Mr. T. F. Buxton, the coadjutor of Wilberforce in the great anti-slavery struggle. +LJ004-0050|Mr. Buxton had already been associated with Mrs. Fry in the Newgate visitation, and his attention had thus been drawn to the neglected state of English prisons.|Mr. Buxton had already been associated with Mrs. Fry in the Newgate visitation, and his attention had thus been drawn to the neglected state of English prisons. +LJ004-0051|When in Belgium he had examined with great satisfaction the admirable management of the great "Maison de Force" at Ghent,|When in Belgium he had examined with great satisfaction the admirable management of the great "Maison de Force" at Ghent, +LJ004-0052|which Howard had eulogized some forty years before.|which Howard had eulogized some forty years before. +LJ004-0054|In order to give greater value to the pamphlet,|In order to give greater value to the pamphlet, +LJ004-0055|he personally visited several English jails, and pointed his observations by drawing forcible contrasts between the good and bad.|he personally visited several English jails, and pointed his observations by drawing forcible contrasts between the good and bad. +LJ004-0056|Mr. Buxton's small work on prison discipline gave a new aspect to the question he had so much at heart.|Mr. Buxton's small work on prison discipline gave a new aspect to the question he had so much at heart. +LJ004-0057|For the first time the doctrine was enunciated that prisoners had rights of their own.|For the first time the doctrine was enunciated that prisoners had rights of their own. +LJ004-0058|The untried, and in the eyes of the law still innocent, could claim pure air, wholesome and sufficient food, and opportunities for exercise.|The untried, and in the eyes of the law still innocent, could claim pure air, wholesome and sufficient food, and opportunities for exercise. +LJ004-0059|They had a right, Mr. Buxton affirmed, to be employed in their own crafts, provided it could be safely followed in prison.|They had a right, Mr. Buxton affirmed, to be employed in their own crafts, provided it could be safely followed in prison. +LJ004-0060|"You have no right," he says, addressing the authorities, "to subject a prisoner to suffering from cold,|"You have no right," he says, addressing the authorities, "to subject a prisoner to suffering from cold, +LJ004-0061|by want of bed-clothing by night or firing by day|by want of bed-clothing by night or firing by day +LJ004-0062|and the reason is plain: you have taken him from his home, and have deprived him of the means of providing himself with the necessaries or comforts of life,|and the reason is plain: you have taken him from his home, and have deprived him of the means of providing himself with the necessaries or comforts of life, +LJ004-0063|and therefore you are bound to furnish him with moderate indeed but suitable accommodation."|and therefore you are bound to furnish him with moderate indeed but suitable accommodation." +LJ004-0064|"You have for the same reason," he goes on, "no right to ruin his habits by compelling him to be idle,|"You have for the same reason," he goes on, "no right to ruin his habits by compelling him to be idle, +LJ004-0065|his morals by compelling him to mix with a promiscuous assemblage of hardened and convicted criminals,|his morals by compelling him to mix with a promiscuous assemblage of hardened and convicted criminals, +LJ004-0066|or his health by forcing him at night into a damp, unventilated cell, with such crowds of companions as very speedily render the air foul and putrid;|or his health by forcing him at night into a damp, unventilated cell, with such crowds of companions as very speedily render the air foul and putrid; +LJ004-0067|or to make him sleep in close contact with the victims of contagious and loathsome disease,|or to make him sleep in close contact with the victims of contagious and loathsome disease, +LJ004-0068|or amidst the noxious effluvia of dirt and corruption.|or amidst the noxious effluvia of dirt and corruption. +LJ004-0069|In short, attention to his feelings, mental and bodily, a supply of every necessary, abstraction from evil society,|In short, attention to his feelings, mental and bodily, a supply of every necessary, abstraction from evil society, +LJ004-0070|the conservation of his health and industrious habits, are the clear, evident, undeniable rights of an unconvicted prisoner."|the conservation of his health and industrious habits, are the clear, evident, undeniable rights of an unconvicted prisoner." +LJ004-0071|Nor even when found guilty and his liberty forfeited did his privileges cease. The law appointed a suitable punishment for the offense;|Nor even when found guilty and his liberty forfeited did his privileges cease. The law appointed a suitable punishment for the offense; +LJ004-0072|it was for those charged with the administration of the law to guard carefully against any aggravation of that punishment,|it was for those charged with the administration of the law to guard carefully against any aggravation of that punishment, +LJ004-0073|to see that "no circumstances of severity are found in his treatment which are not found in his sentence."|to see that "no circumstances of severity are found in his treatment which are not found in his sentence." +LJ004-0074|No judge ever condemned a man to be half-perished with cold by day, or half-suffocated with heat by night.|No judge ever condemned a man to be half-perished with cold by day, or half-suffocated with heat by night. +LJ004-0075|"Who ever heard of a criminal being sentenced to catch the rheumatism or the typhus fever?"|"Who ever heard of a criminal being sentenced to catch the rheumatism or the typhus fever?" +LJ004-0076|"Disease, cold, famine, nakedness, and contagious and polluted air are not lawful punishments in the hands of the civil magistrates;|"Disease, cold, famine, nakedness, and contagious and polluted air are not lawful punishments in the hands of the civil magistrates; +LJ004-0077|nor has he a right to poison or starve his fellow-creatures."|nor has he a right to poison or starve his fellow-creatures." +LJ004-0078|"The convicted delinquent has his rights," said Mr. Buxton authoritatively.|"The convicted delinquent has his rights," said Mr. Buxton authoritatively. +LJ004-0079|"All measures and practices in prison which may injure him in any way are illegal,|"All measures and practices in prison which may injure him in any way are illegal, +LJ004-0080|because they are not specified in his sentence; he is therefore entitled to a wholesome atmosphere,|because they are not specified in his sentence; he is therefore entitled to a wholesome atmosphere, +LJ004-0081|decent clothing and bedding, and a diet sufficient to support him."|decent clothing and bedding, and a diet sufficient to support him." +LJ004-0082|These somewhat novel but undoubtedly indisputable propositions were backed up, not by sound arguments only, but by the letter of the law.|These somewhat novel but undoubtedly indisputable propositions were backed up, not by sound arguments only, but by the letter of the law. +LJ004-0083|As Mr. Buxton pointed out, many old acts of parliament designed to protect the prisoner were still in full force.|As Mr. Buxton pointed out, many old acts of parliament designed to protect the prisoner were still in full force. +LJ004-0084|Some might be in abeyance, but they had never been repealed, and some were quite freshly imported upon the Statute Book.|Some might be in abeyance, but they had never been repealed, and some were quite freshly imported upon the Statute Book. +LJ004-0085|As far back as the reign of Charles II., a law was passed declaring that sufficient provision should be made for the relief and setting on work|As far back as the reign of Charles the second, a law was passed declaring that sufficient provision should be made for the relief and setting on work +LJ004-0086|of "poor and needy prisoners committed to the common jail for felony and other misdemeanors, who many times perish before their trial;|of "poor and needy prisoners committed to the common jail for felony and other misdemeanors, who many times perish before their trial; +LJ004-0087|and the poor there living idle and unemployed become debauched, and come forth instructed in the practice of thievery and lewdness."|and the poor there living idle and unemployed become debauched, and come forth instructed in the practice of thievery and lewdness." +LJ004-0088|As a remedy, justices of the peace were empowered to provide materials for the setting of poor prisoners to work,|As a remedy, justices of the peace were empowered to provide materials for the setting of poor prisoners to work, +LJ004-0089|and to pay overseers or instructors out of the county rates.|and to pay overseers or instructors out of the county rates. +LJ004-0090|Again, the 22 Charles II. c 20 ordered the jailer to keep felons and debtors "separate and apart from one another,|Again, the twenty-two Charles the second c twenty ordered the jailer to keep felons and debtors "separate and apart from one another, +LJ004-0091|in distinct rooms, on pain of forfeiting his office and treble damages to the party aggrieved."|in distinct rooms, on pain of forfeiting his office and treble damages to the party aggrieved." +LJ004-0092|A much later act, the 14 George III. c. 59 (1774),|A much later act, the fourteen George the third c. fifty-nine (seventeen seventy-four), +LJ004-0093|which was contemporaneous with Howard's first journeys, laid down precise rules as regards cleanliness, and the proper supply of space and air.|which was contemporaneous with Howard's first journeys, laid down precise rules as regards cleanliness, and the proper supply of space and air. +LJ004-0094|This act set forth that "whereas the malignant fever commonly called the jail distemper|This act set forth that "whereas the malignant fever commonly called the jail distemper +LJ004-0095|is found to be owing to want of cleanliness and fresh air in the several jails,|is found to be owing to want of cleanliness and fresh air in the several jails, +LJ004-0096|the fatal consequences whereof might be prevented if the justices of the peace were duly authorized|the fatal consequences whereof might be prevented if the justices of the peace were duly authorized +LJ004-0097|to provide such accommodations in jails as may be necessary to answer this salutary purpose,|to provide such accommodations in jails as may be necessary to answer this salutary purpose, +LJ004-0098|it is enacted that the justices shall order the walls of every room to be scraped and white-washed once every year."|it is enacted that the justices shall order the walls of every room to be scraped and white-washed once every year." +LJ004-0099|Ventilators, hand and others, were to be supplied.|Ventilators, hand and others, were to be supplied. +LJ004-0100|An infirmary, consisting of two distinct rooms, one for males and one for females, should be provided for the separate accommodation of the sick.|An infirmary, consisting of two distinct rooms, one for males and one for females, should be provided for the separate accommodation of the sick. +LJ004-0101|Warm and cold baths, or "commodious bathing tubs,"|Warm and cold baths, or "commodious bathing tubs," +LJ004-0102|were to be kept in every jail, and the prisoners directed to wash in them before release. These provisions were almost a dead letter.|were to be kept in every jail, and the prisoners directed to wash in them before release. These provisions were almost a dead letter. +LJ004-0103|Yet another act passed in 1791, if properly observed, should have insured proper attention to them.|Yet another act passed in seventeen ninety-one, if properly observed, should have insured proper attention to them. +LJ004-0104|By the 31 George III. c. 46, s. 5,|By the thirty-one George the third c. forty-six, s. five, +LJ004-0105|two or more justices were appointed visitors of prisons, and directed to visit and inspect three times every quarter.|two or more justices were appointed visitors of prisons, and directed to visit and inspect three times every quarter. +LJ004-0106|They were to report in writing to quarter sessions as to the state of the jail, and as to all abuses which they might observe therein.|They were to report in writing to quarter sessions as to the state of the jail, and as to all abuses which they might observe therein. +LJ004-0107|The most important jail act of that early period, however, was the 24 George III. c. 54, s. 4 (1784)|The most important jail act of that early period, however, was the twenty-four George the third c. fifty-four, s. four (seventeen eighty-four) +LJ004-0108|which was the first legislative attempt to compel the classification of prisoners, or their separation into classes|which was the first legislative attempt to compel the classification of prisoners, or their separation into classes +LJ004-0109|according to their categories or crimes.|according to their categories or crimes. +LJ004-0110|It was made incumbent upon the justices to provide distinct places of confinement for five classes of prisoners, viz.|It was made incumbent upon the justices to provide distinct places of confinement for five classes of prisoners, viz. +LJ004-0111|1. Prisoners convicted of felony. 2. Prisoners committed on a charge or suspicion of felony.|one. Prisoners convicted of felony. two. Prisoners committed on a charge or suspicion of felony. +LJ004-0112|3. Prisoners guilty of misdemeanors. 4. Prisoners charged with misdemeanors. 5. Debtors.|three. Prisoners guilty of misdemeanors. four. Prisoners charged with misdemeanors. five. Debtors. +LJ004-0113|It was further ordered that male prisoners should be kept perfectly distinct from the females.|It was further ordered that male prisoners should be kept perfectly distinct from the females. +LJ004-0114|King's evidences were also to be lodged apart.|King's evidences were also to be lodged apart. +LJ004-0115|Infirmaries separating the sexes were also to be provided, a chapel too, and warm and cold baths.|Infirmaries separating the sexes were also to be provided, a chapel too, and warm and cold baths. +LJ004-0116|"Care also was to be taken that the prisoners shall not be kept in any apartment underground."|"Care also was to be taken that the prisoners shall not be kept in any apartment underground." +LJ004-0117|In an early report of the Prison Discipline Improvement Society,|In an early report of the Prison Discipline Improvement Society, +LJ004-0118|published some six-and-thirty years after the promulgation of this act, the flagrant and persistent violations of it and others|published some six-and-thirty years after the promulgation of this act, the flagrant and persistent violations of it and others +LJ004-0119|which had continued through that long period, are forcibly pointed out.|which had continued through that long period, are forcibly pointed out. +LJ004-0120|In 1818, out of five hundred and eighteen prisons in the United Kingdom,|In eighteen eighteen, out of five hundred and eighteen prisons in the United Kingdom, +LJ004-0121|to which a total of upwards of one hundred thousand prisoners had been committed in the year, only twenty-three prisons were divided according to law;|to which a total of upwards of one hundred thousand prisoners had been committed in the year, only twenty-three prisons were divided according to law; +LJ004-0122|fifty-nine had no division whatever to separate males and females; one hundred and thirty-six had only one division for the purpose;|fifty-nine had no division whatever to separate males and females; one hundred and thirty-six had only one division for the purpose; +LJ004-0123|sixty-eight had only two divisions, and so on.|sixty-eight had only two divisions, and so on. +LJ004-0124|In four hundred and forty-five prisons no work of any description had been introduced for the employment of prisoners;|In four hundred and forty-five prisons no work of any description had been introduced for the employment of prisoners; +LJ004-0125|in the balance some work was done, but with the most meager results.|in the balance some work was done, but with the most meager results. +LJ004-0126|The want of room was still a crying evil.|The want of room was still a crying evil. +LJ004-0127|In one hundred jails,|In one hundred jails, +LJ004-0128|capable of accommodating only eight thousand five hundred and forty-five persons, as many as thirteen thousand and fifty-seven were crowded.|capable of accommodating only eight thousand five hundred and forty-five persons, as many as thirteen thousand and fifty-seven were crowded. +LJ004-0129|Many of the jails were in the most deplorable condition:|Many of the jails were in the most deplorable condition: +LJ004-0130|incommodious, as has been stated, insecure, unhealthy, and unprovided with the printed or written regulations required by law.|incommodious, as has been stated, insecure, unhealthy, and unprovided with the printed or written regulations required by law. +LJ004-0131|To specify more particularly one or two of the worst, it may be mentioned that in the Borough Compter|To specify more particularly one or two of the worst, it may be mentioned that in the Borough Compter +LJ004-0132|the old evils of indiscriminate association still continued unchecked.|the old evils of indiscriminate association still continued unchecked. +LJ004-0133|All prisoners passed their time in absolute idleness, or killed it by gambling and loose conversation.|All prisoners passed their time in absolute idleness, or killed it by gambling and loose conversation. +LJ004-0134|The debtors were crowded almost inconceivably. In a space twenty feet long by six wide,|The debtors were crowded almost inconceivably. In a space twenty feet long by six wide, +LJ004-0135|twenty men slept on eight straw beds, with sixteen rugs amongst them, and a piece of timber for a bolster.|twenty men slept on eight straw beds, with sixteen rugs amongst them, and a piece of timber for a bolster. +LJ004-0136|Mr. Buxton, who found this, declared that it seemed physically impossible, but he was assured that it was true,|Mr. Buxton, who found this, declared that it seemed physically impossible, but he was assured that it was true, +LJ004-0137|and that it was accomplished by "sleeping edgewise."|and that it was accomplished by "sleeping edgewise." +LJ004-0138|One poor wretch, who had slept next the wall, said he had been literally unable to move for the pressure.|One poor wretch, who had slept next the wall, said he had been literally unable to move for the pressure. +LJ004-0139|"In the morning the stench and heat were so oppressive that he and every one else on waking rushed unclothed into the yard;"|"In the morning the stench and heat were so oppressive that he and every one else on waking rushed unclothed into the yard;" +LJ004-0140|and the turnkey told Mr. Buxton that the "smell on first opening the door was enough to knock down a horse.|and the turnkey told Mr. Buxton that the "smell on first opening the door was enough to knock down a horse. +LJ004-0141|The hospital was filled with infectious cases, and in one room, seven feet by nine, with closed windows,|The hospital was filled with infectious cases, and in one room, seven feet by nine, with closed windows, +LJ004-0142|where a lad lay ill with fever, three other prisoners, at first perfectly healthy, were lodged. Of course they were seized with the fever;|where a lad lay ill with fever, three other prisoners, at first perfectly healthy, were lodged. Of course they were seized with the fever; +LJ004-0143|so that the culprit, in addition to his sentence,|so that the culprit, in addition to his sentence, +LJ004-0144|had to endure by "the regulations of the city a disease very dangerous in its nature," and ran the risk of a lingering and painful death.|had to endure by "the regulations of the city a disease very dangerous in its nature," and ran the risk of a lingering and painful death. +LJ004-0145|At Guildford prison, which Mr. Buxton also visited in 1818,|At Guildford prison, which Mr. Buxton also visited in eighteen eighteen, +LJ004-0146|there was no infirmary, no chapel, no work, no classification.|there was no infirmary, no chapel, no work, no classification. +LJ004-0147|The irons, which nearly every one wore, were remarkably heavy; those double ironed could not take off their small clothes.|The irons, which nearly every one wore, were remarkably heavy; those double ironed could not take off their small clothes. +LJ004-0148|No prison dress was allowed, and half the inmates were without shirts or shoes or stockings.|No prison dress was allowed, and half the inmates were without shirts or shoes or stockings. +LJ004-0149|The diet was limited to dry bread, which was of the best certainly, and a pound and a half in weight.|The diet was limited to dry bread, which was of the best certainly, and a pound and a half in weight. +LJ004-0150|Matters were on much the same footing at St. Albans.|Matters were on much the same footing at St. Albans. +LJ004-0151|They were far worse at Bristol,|They were far worse at Bristol, +LJ004-0152|although at Mr. Buxton's visit a new jail was in process of erection, the first step towards reform since Howard's visitation in 1774.|although at Mr. Buxton's visit a new jail was in process of erection, the first step towards reform since Howard's visitation in seventeen seventy-four. +LJ004-0153|In 1818 the old jail was so densely packed that it was nearly impossible to pass through the yards for the throng.|In eighteen eighteen the old jail was so densely packed that it was nearly impossible to pass through the yards for the throng. +LJ004-0154|One hundred and fifty were lodged in a prison just capable of holding fifty-two.|One hundred and fifty were lodged in a prison just capable of holding fifty-two. +LJ004-0155|In the crowd, all of them persons who had "no other avocation or mode of livelihood but thieving," Mr. Buxton counted eleven children|In the crowd, all of them persons who had "no other avocation or mode of livelihood but thieving," Mr. Buxton counted eleven children +LJ004-0156|children hardly old enough to be released from the nursery.|children hardly old enough to be released from the nursery. +LJ004-0157|All charged with felony were in heavy irons, without distinction of age.|All charged with felony were in heavy irons, without distinction of age. +LJ004-0158|All were in ill health; almost all were in rags; almost all were filthy in the extreme.|All were in ill health; almost all were in rags; almost all were filthy in the extreme. +LJ004-0159|The state of the prison, the desperation of the prisoners, broadly hinted in their conversation and plainly expressed in their conduct,|The state of the prison, the desperation of the prisoners, broadly hinted in their conversation and plainly expressed in their conduct, +LJ004-0160|the uproar of oaths, complaints, and obscenity,|the uproar of oaths, complaints, and obscenity, +LJ004-0161|"the indescribable stench," presented together a concentration of the utmost misery and the utmost guilt.|"the indescribable stench," presented together a concentration of the utmost misery and the utmost guilt. +LJ004-0162|It was "a scene of infernal passions and distresses," says Buxton, "which few have imagination sufficient to picture,|It was "a scene of infernal passions and distresses," says Buxton, "which few have imagination sufficient to picture, +LJ004-0163|and of which fewer still would believe that the original is to be found in this enlightened and happy country."|and of which fewer still would believe that the original is to be found in this enlightened and happy country." +LJ004-0164|There was still worse to come. Having explored the yards and adjacent day rooms, and sleeping cells, a door was unlocked,|There was still worse to come. Having explored the yards and adjacent day rooms, and sleeping cells, a door was unlocked, +LJ004-0165|the visitors were furnished with candles, and they descended eighteen long steps into a vault.|the visitors were furnished with candles, and they descended eighteen long steps into a vault. +LJ004-0166|At the bottom was a circular space, through which ran a narrow passage, and the sides of which were fitted with barrack bedsteads.|At the bottom was a circular space, through which ran a narrow passage, and the sides of which were fitted with barrack bedsteads. +LJ004-0167|The floor was on the level of the river, and very damp.|The floor was on the level of the river, and very damp. +LJ004-0168|The smell at one o'clock of the day "was something more than can be expressed by the term disgusting."|The smell at one o'clock of the day "was something more than can be expressed by the term disgusting." +LJ004-0170|The only ventilation of this pit, this "dark, cheerless, damp, unwholesome cavern -- a dungeon in its worst sense"|The only ventilation of this pit, this "dark, cheerless, damp, unwholesome cavern -- a dungeon in its worst sense" +LJ004-0171|was by a kind of chimney, which the prisoners kept hermetically sealed, and which had never been opened in the memory of the turnkey.|was by a kind of chimney, which the prisoners kept hermetically sealed, and which had never been opened in the memory of the turnkey. +LJ004-0172|Untried persons were often lodged in this nauseous underground den,|Untried persons were often lodged in this nauseous underground den, +LJ004-0173|and sometimes slept in "the pit," loaded with heavy irons for a whole year, waiting the jail delivery.|and sometimes slept in "the pit," loaded with heavy irons for a whole year, waiting the jail delivery. +LJ004-0174|Confinement for twelve months in the Bristol jail was counted a punishment equivalent to seven years' transportation.|Confinement for twelve months in the Bristol jail was counted a punishment equivalent to seven years' transportation. +LJ004-0175|In this prison there was no female infirmary.|In this prison there was no female infirmary. +LJ004-0176|Sick women and their children remained in the ordinary wards, and propagated disease.|Sick women and their children remained in the ordinary wards, and propagated disease. +LJ004-0177|No prison dress was allowed; no reception-room was provided, no soap, towels, or baths.|No prison dress was allowed; no reception-room was provided, no soap, towels, or baths. +LJ004-0178|The bedclothes consisted only of a single "very slight" rug.|The bedclothes consisted only of a single "very slight" rug. +LJ004-0179|The allowance of food daily to felons was a fourpenny loaf,|The allowance of food daily to felons was a fourpenny loaf, +LJ004-0180|a price which in those days fluctuated enormously -- as much as a hundred percent in a couple of years;|a price which in those days fluctuated enormously -- as much as a hundred percent in a couple of years; +LJ004-0181|but as no similar variation occurred in the prisoner's appetite, his ration was somewhat precarious.|but as no similar variation occurred in the prisoner's appetite, his ration was somewhat precarious. +LJ004-0182|As for the debtors, they had no allowance whatever, and were often in imminent danger of starvation.|As for the debtors, they had no allowance whatever, and were often in imminent danger of starvation. +LJ004-0183|With all this, the inmates were crowded together at night to such a degree as to excite surprise that they should escape suffocation.|With all this, the inmates were crowded together at night to such a degree as to excite surprise that they should escape suffocation. +LJ004-0184|There reigned through the whole edifice a chilly, damp, unwholesome atmosphere, and the effluvia from the prisoners was so nauseous|There reigned through the whole edifice a chilly, damp, unwholesome atmosphere, and the effluvia from the prisoners was so nauseous +LJ004-0185|that the chaplain found it necessary to take his place before they entered chapel, as he could not otherwise have faced the smell.|that the chaplain found it necessary to take his place before they entered chapel, as he could not otherwise have faced the smell. +LJ004-0186|It is consoling to know that there were a few brilliant exceptions to this cruel, callous neglect.|It is consoling to know that there were a few brilliant exceptions to this cruel, callous neglect. +LJ004-0187|Already, as early as 1818, a prison existed at Bury St. Edmunds which was a model for imitation to others at that time,|Already, as early as eighteen eighteen, a prison existed at Bury St. Edmunds which was a model for imitation to others at that time, +LJ004-0188|and which even fulfilled many of the exacting requirements of modern days.|and which even fulfilled many of the exacting requirements of modern days. +LJ004-0189|The great principles of classification, cleanliness, and employment were closely observed.|The great principles of classification, cleanliness, and employment were closely observed. +LJ004-0190|There were eighty-four separate sleeping-cells, and unless the jail was overcrowded, every inmate passed the night alone,|There were eighty-four separate sleeping-cells, and unless the jail was overcrowded, every inmate passed the night alone, +LJ004-0191|and in comparative comfort, with a bed and proper bedding.|and in comparative comfort, with a bed and proper bedding. +LJ004-0192|The prison stood on a dry, airy situation outside the town.|The prison stood on a dry, airy situation outside the town. +LJ004-0193|Prisoners on reception were treated as they are now-a-days -- bathed, dressed in prison clothes, and inspected by the surgeon.|Prisoners on reception were treated as they are now-a-days -- bathed, dressed in prison clothes, and inspected by the surgeon. +LJ004-0194|No irons were worn except as a punishment.|No irons were worn except as a punishment. +LJ004-0195|Personal cleanliness was insisted upon, and all parts of the prison were kept scrupulously clean.|Personal cleanliness was insisted upon, and all parts of the prison were kept scrupulously clean. +LJ004-0196|There was an infirmary, properly found and duly looked after.|There was an infirmary, properly found and duly looked after. +LJ004-0197|No idleness was permitted among the inmates. Trades were taught, or prisoners were allowed to follow their own if suitable.|No idleness was permitted among the inmates. Trades were taught, or prisoners were allowed to follow their own if suitable. +LJ004-0198|There was, besides, a mill for grinding corn, somewhat similar to a turn-spit, which prisoners turned by walking in rows.|There was, besides, a mill for grinding corn, somewhat similar to a turn-spit, which prisoners turned by walking in rows. +LJ004-0199|This made exertion compulsory, and imposed hard labor as a proper punishment.|This made exertion compulsory, and imposed hard labor as a proper punishment. +LJ004-0200|Another jail, that of Ilchester, was also worthy of all commendation. It exhibited all the good points of that at Bury.|Another jail, that of Ilchester, was also worthy of all commendation. It exhibited all the good points of that at Bury. +LJ004-0201|At Ilchester the rule of employment had been carried further.|At Ilchester the rule of employment had been carried further. +LJ004-0202|A system not adopted generally till nearly half a century later had already prevailed at Ilchester.|A system not adopted generally till nearly half a century later had already prevailed at Ilchester. +LJ004-0203|The new jail had been in a great measure constructed by the prisoners themselves.|The new jail had been in a great measure constructed by the prisoners themselves. +LJ004-0204|Masons, bricklayers, carpenters, painters had been employed upon the buildings, and the work was pronounced excellent by competent judges.|Masons, bricklayers, carpenters, painters had been employed upon the buildings, and the work was pronounced excellent by competent judges. +LJ004-0205|Industrial labor had also been introduced with satisfactory results.|Industrial labor had also been introduced with satisfactory results. +LJ004-0206|Blanket weaving and cloth spinning was carried on prosperously,|Blanket weaving and cloth spinning was carried on prosperously, +LJ004-0207|and all the material for prisoners' apparel was manufactured in the jail.|and all the material for prisoners' apparel was manufactured in the jail. +LJ004-0208|There were work-rooms for wool-washing, dyeing, carding, and spinning.|There were work-rooms for wool-washing, dyeing, carding, and spinning. +LJ004-0209|The looms were constantly busy. Tailors were always at work, and every article of clothing and bedding was made up within the walls.|The looms were constantly busy. Tailors were always at work, and every article of clothing and bedding was made up within the walls. +LJ004-0210|There was a prison laundry too, where all the prisoners' linen was regularly washed.|There was a prison laundry too, where all the prisoners' linen was regularly washed. +LJ004-0211|The moral welfare of the inmates was as closely looked after as the physical.|The moral welfare of the inmates was as closely looked after as the physical. +LJ004-0212|There was an attentive chaplain, a schoolmaster, and regular religious and other instruction.|There was an attentive chaplain, a schoolmaster, and regular religious and other instruction. +LJ004-0213|Compared with those highly meritorious institutions Newgate still showed but badly.|Compared with those highly meritorious institutions Newgate still showed but badly. +LJ004-0214|Its evils were inherent and irremediable, but some ameliorating measures had been introduced,|Its evils were inherent and irremediable, but some ameliorating measures had been introduced, +LJ004-0215|mainly through the exertions of a new governor, Mr. Brown, who succeeded Mr. Newman at Newgate in 1817.|mainly through the exertions of a new governor, Mr. Brown, who succeeded Mr. Newman at Newgate in eighteen seventeen. +LJ004-0216|The most noticeable of the improvements introduced was a better regulation of dietaries within the prison.|The most noticeable of the improvements introduced was a better regulation of dietaries within the prison. +LJ004-0217|The old haphazard system, by which meat was issued in bulk,|The old haphazard system, by which meat was issued in bulk, +LJ004-0218|a week's allowance at a time, was abolished, and there was a regular scale of daily rations adopted.|a week's allowance at a time, was abolished, and there was a regular scale of daily rations adopted. +LJ004-0219|The diet was now ample. It consisted of a pound and a half of bread per diem;|The diet was now ample. It consisted of a pound and a half of bread per diem; +LJ004-0220|for breakfast a pint of gruel; for dinner half a pound of boiled meat, or a quart of soup with vegetables, on alternate days.|for breakfast a pint of gruel; for dinner half a pound of boiled meat, or a quart of soup with vegetables, on alternate days. +LJ004-0221|The food was properly prepared in the prison kitchen.|The food was properly prepared in the prison kitchen. +LJ004-0222|Meat was no longer issued raw, to be imperfectly cooked before a ward fire and bolted gluttonously, the whole two pounds at one sitting.|Meat was no longer issued raw, to be imperfectly cooked before a ward fire and bolted gluttonously, the whole two pounds at one sitting. +LJ004-0223|Mr. Brown confidently asserted that no jail in England now fed its inmates so well as did Newgate.|Mr. Brown confidently asserted that no jail in England now fed its inmates so well as did Newgate. +LJ004-0224|So plentiful was this dietary, that although the old permission remained in force of allowing the friends of prisoners to bring them supplies from outside,|So plentiful was this dietary, that although the old permission remained in force of allowing the friends of prisoners to bring them supplies from outside, +LJ004-0225|the practice was falling into abeyance, and the prisoners seldom required private assistance to eke out their meals.|the practice was falling into abeyance, and the prisoners seldom required private assistance to eke out their meals. +LJ004-0226|It was also claimed for the more ample and more orderly distribution of victuals, that the general health of the prisoners had greatly improved.|It was also claimed for the more ample and more orderly distribution of victuals, that the general health of the prisoners had greatly improved. +LJ004-0227|Mr. Brown also, much to his own credit, brought about the abandonment of the practice of ironing all prisoners as a matter of course.|Mr. Brown also, much to his own credit, brought about the abandonment of the practice of ironing all prisoners as a matter of course. +LJ004-0228|In 1818 prisoners awaiting trial in Newgate, were at length relieved from this illegal infliction.|In eighteen eighteen prisoners awaiting trial in Newgate, were at length relieved from this illegal infliction. +LJ004-0229|Convicts were not even compelled to wear irons, providing they behaved well.|Convicts were not even compelled to wear irons, providing they behaved well. +LJ004-0230|It was found that shackles might be safely dispensed with, even in the case of the most desperate characters.|It was found that shackles might be safely dispensed with, even in the case of the most desperate characters. +LJ004-0231|This was effected by stopping the nearly indiscriminate admission of visitors, which had hitherto prevailed all over the jail.|This was effected by stopping the nearly indiscriminate admission of visitors, which had hitherto prevailed all over the jail. +LJ004-0232|Ironing it will be remembered, was a distinguishing badge, so that when the jail was cleared the free might be readily known from the captive, and escapes prevented.|Ironing it will be remembered, was a distinguishing badge, so that when the jail was cleared the free might be readily known from the captive, and escapes prevented. +LJ004-0234|This change led to some discontent, until it was found that the much greater boon of relief from irons accompanied it, and the reform was quietly accepted.|This change led to some discontent, until it was found that the much greater boon of relief from irons accompanied it, and the reform was quietly accepted. +LJ004-0235|Indeed the best consequences followed from the removal of irons. The prisoners were much better disposed; there were no riots, and fewer disturbances.|Indeed the best consequences followed from the removal of irons. The prisoners were much better disposed; there were no riots, and fewer disturbances. +LJ004-0236|But nothing short of radical reform and complete reconstruction could touch the deep-seated evils of association, overcrowding, and idleness.|But nothing short of radical reform and complete reconstruction could touch the deep-seated evils of association, overcrowding, and idleness. +LJ004-0237|The first still produced deplorable results -- results to be observable for many years to come.|The first still produced deplorable results -- results to be observable for many years to come. +LJ004-0238|Mr. Buxton mentions the case of a boy whose apparent innocence and artlessness had attracted his attention.|Mr. Buxton mentions the case of a boy whose apparent innocence and artlessness had attracted his attention. +LJ004-0239|He had been committed for an offense for which he was acquitted.|He had been committed for an offense for which he was acquitted. +LJ004-0240|He left Newgate utterly corrupted, and after lapsing into crime, soon returned with a very different character.|He left Newgate utterly corrupted, and after lapsing into crime, soon returned with a very different character. +LJ004-0241|Other cases of moral deterioration have already been recorded.|Other cases of moral deterioration have already been recorded. +LJ004-0242|Some attempt was made to reduce the overcrowding, on the recommendation of the House of Commons Committee of 1818, but this applied only a partial remedy.|Some attempt was made to reduce the overcrowding, on the recommendation of the House of Commons Committee of eighteen eighteen, but this applied only a partial remedy. +LJ004-0243|The bulk of the prisoners were still left in idleness.|The bulk of the prisoners were still left in idleness. +LJ004-0244|A few fortunate criminals, many of them kept back from transportation on purpose, who were skilled in trades, were employed at them.|A few fortunate criminals, many of them kept back from transportation on purpose, who were skilled in trades, were employed at them. +LJ004-0245|Painters, plasterers, and carpenters were allowed to follow their handicrafts, with the reward of sixpence per diem and a double allowance of food.|Painters, plasterers, and carpenters were allowed to follow their handicrafts, with the reward of sixpence per diem and a double allowance of food. +LJ004-0246|They used their own tools, and this without any dangerous consequences as regards facilitating the escape of others,|They used their own tools, and this without any dangerous consequences as regards facilitating the escape of others, +LJ004-0247|thus disposing of the objection so long raised against the industrial employment of prisoners in Newgate.|thus disposing of the objection so long raised against the industrial employment of prisoners in Newgate. +LJ004-0248|But this boon of toil was denied to all but a very limited number.|But this boon of toil was denied to all but a very limited number. +LJ004-0249|As the Prison Discipline Society pertinently observed in a report dated 1820,|As the Prison Discipline Society pertinently observed in a report dated eighteen twenty, +LJ004-0250|"It is obvious that reformation must be materially impeded, and in some cases utterly defeated, when the prisoners are defectively classed,|"It is obvious that reformation must be materially impeded, and in some cases utterly defeated, when the prisoners are defectively classed, +LJ005-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 8: The beginnings of prison reform.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section eight: The beginnings of prison reform. +LJ005-0002|Newgate prisoners were the victims to another most objectionable practice which obtained all over London.|Newgate prisoners were the victims to another most objectionable practice which obtained all over London. +LJ005-0003|Persons committed to a metropolitan jail at that time were taken in gangs, men and women handcuffed together, or linked on to a long chain,|Persons committed to a metropolitan jail at that time were taken in gangs, men and women handcuffed together, or linked on to a long chain, +LJ005-0004|unless they could afford to pay for a vehicle out of their own funds.|unless they could afford to pay for a vehicle out of their own funds. +LJ005-0005|Even then they were not certain of the favor, for I find a reference to a decent and respectable woman sent to Newgate|Even then they were not certain of the favor, for I find a reference to a decent and respectable woman sent to Newgate +LJ005-0006|who handed a shilling to the escort warder to provide her with a hackney coach; but this functionary pocketed the cash, and obliged the woman to walk|who handed a shilling to the escort warder to provide her with a hackney coach; but this functionary pocketed the cash, and obliged the woman to walk +LJ005-0007|chained to the rest. As the miserable crew filed through the public streets, exposed to the scornful gaze of every passenger,|chained to the rest. As the miserable crew filed through the public streets, exposed to the scornful gaze of every passenger, +LJ005-0008|they were followed by a crowd of reckless boys, who jeered at and insulted them.|they were followed by a crowd of reckless boys, who jeered at and insulted them. +LJ005-0009|Many thus led in procession were in a shocking condition of dirt and misery, frequently nearly naked, and often bearing upon them|Many thus led in procession were in a shocking condition of dirt and misery, frequently nearly naked, and often bearing upon them +LJ005-0010|the germs, more or less developed, of contagious disease. "Caravans," the forerunners of the prison vans,|the germs, more or less developed, of contagious disease. "Caravans," the forerunners of the prison vans, +LJ005-0011|were first made use of about 1827. That the need for prison reform was imperative may be gathered from the few out of many instances I have adduced,|were first made use of about eighteen twenty-seven. That the need for prison reform was imperative may be gathered from the few out of many instances I have adduced, +LJ005-0012|yet there were those who, wedded to ancient ideas, were intolerant of change; they would not admit the existence of any evils.|yet there were those who, wedded to ancient ideas, were intolerant of change; they would not admit the existence of any evils. +LJ005-0013|One smug alderman, a member of the House of Commons, sneered at the ultra philanthropy of the champions of prison improvement.|One smug alderman, a member of the House of Commons, sneered at the ultra philanthropy of the champions of prison improvement. +LJ005-0014|Speaking on a debate on prison matters, he declared that|Speaking on a debate on prison matters, he declared that +LJ005-0015|our prisoners have all that prisoners ought to have, without gentlemen think they ought to be indulged with Turkey carpets.|our prisoners have all that prisoners ought to have, without gentlemen think they ought to be indulged with Turkey carpets. +LJ005-0016|The Society for the Improvement of Prison Discipline was taxed with a desire to introduce a system|The Society for the Improvement of Prison Discipline was taxed with a desire to introduce a system +LJ005-0017|tending to divest punishment of its just and salutary terrors;|tending to divest punishment of its just and salutary terrors; +LJ005-0018|an imputation which the Society indignantly and very justly repudiated, the statement being, as they said,|an imputation which the Society indignantly and very justly repudiated, the statement being, as they said, +LJ005-0019|refuted by abundant evidence, and having no foundation whatever in truth.|refuted by abundant evidence, and having no foundation whatever in truth. +LJ005-0020|Among those whom the Society found arrayed against it was Sydney Smith,|Among those whom the Society found arrayed against it was Sydney Smith, +LJ005-0021|who, in a caustic article contributed to the 'Edinburgh Review,' protested against the pampering of criminals|who, in a caustic article contributed to the 'Edinburgh Review,' protested against the pampering of criminals +LJ005-0022|While fully admitting the good intentions of the Society, he condemned their ultra humanitarianism as misplaced.|While fully admitting the good intentions of the Society, he condemned their ultra humanitarianism as misplaced. +LJ005-0023|He took exception to various of the proposals of the Society. He thought they leant too much to a system of indulgence and education in jails.|He took exception to various of the proposals of the Society. He thought they leant too much to a system of indulgence and education in jails. +LJ005-0024|He objected to the instruction of prisoners in reading and writing.|He objected to the instruction of prisoners in reading and writing. +LJ005-0025|"A poor man who is lucky enough," he said, "to have his son committed for a felony|"A poor man who is lucky enough," he said, "to have his son committed for a felony +LJ005-0026|educates him under such a system for nothing, while the virtuous simpleton who is on the other side of the wall is paying by the quarter for these attainments."|educates him under such a system for nothing, while the virtuous simpleton who is on the other side of the wall is paying by the quarter for these attainments." +LJ005-0027|He was altogether against too liberal a diet; he disapproved of industrial occupations in jails, as not calculated to render prisons terrible.|He was altogether against too liberal a diet; he disapproved of industrial occupations in jails, as not calculated to render prisons terrible. +LJ005-0028|"There should be no tea and sugar, no assemblage of female felons around the washing-tub,|"There should be no tea and sugar, no assemblage of female felons around the washing-tub, +LJ005-0029|nothing but beating hemp and pulling oakum and pounding bricks -- no work but what was tedious, unusual."|nothing but beating hemp and pulling oakum and pounding bricks -- no work but what was tedious, unusual." +LJ005-0030|"In prisons, which are really meant to keep the multitude in order, and to be a terror to evil-doers, there must be no sharings of profits,|"In prisons, which are really meant to keep the multitude in order, and to be a terror to evil-doers, there must be no sharings of profits, +LJ005-0031|no visiting of friends, no education but religious education, no freedom of diet,|no visiting of friends, no education but religious education, no freedom of diet, +LJ005-0032|no weavers' looms or carpenters' benches. There must be a great deal of solitude, coarse food, a dress of shame,|no weavers' looms or carpenters' benches. There must be a great deal of solitude, coarse food, a dress of shame, +LJ005-0033|hard, incessant, irksome, eternal labor, a planned and regulated and unrelenting exclusion of happiness and comfort.|hard, incessant, irksome, eternal labor, a planned and regulated and unrelenting exclusion of happiness and comfort. +LJ005-0034|Undeterred by these sarcasms and misrepresentations,|Undeterred by these sarcasms and misrepresentations, +LJ005-0035|the Society pursued its laudable undertaking with remarkable energy and great singleness of purpose.|the Society pursued its laudable undertaking with remarkable energy and great singleness of purpose. +LJ005-0036|The objects it had in view were set forth in one of its earliest meetings.|The objects it had in view were set forth in one of its earliest meetings. +LJ005-0037|It sought to obtain and diffuse useful information|It sought to obtain and diffuse useful information +LJ005-0038|to suggest beneficial regulations, and circulate tracts demonstrating the advantages of classification,|to suggest beneficial regulations, and circulate tracts demonstrating the advantages of classification, +LJ005-0039|constant inspection, regular employment, and humane treatment generally, with religious and moral instruction.|constant inspection, regular employment, and humane treatment generally, with religious and moral instruction. +LJ005-0040|It earnestly advocated the appointment of female officers to take exclusive charge of female prisoners,|It earnestly advocated the appointment of female officers to take exclusive charge of female prisoners, +LJ005-0041|a much-needed and, according to our ideas, indispensable reform, already initiated by the Ladies' Committee at Newgate.|a much-needed and, according to our ideas, indispensable reform, already initiated by the Ladies' Committee at Newgate. +LJ005-0042|It made the subject of the newly-invented tread-wheels, or stepping-wheels, as they were at first called, its peculiar affair,|It made the subject of the newly-invented tread-wheels, or stepping-wheels, as they were at first called, its peculiar affair, +LJ005-0043|and obtained full details, from places where they had been adopted, of the nature of these new machines|and obtained full details, from places where they had been adopted, of the nature of these new machines +LJ005-0044|the method by which they were worked, and the dietaries of the prisoners employed upon them.|the method by which they were worked, and the dietaries of the prisoners employed upon them. +LJ005-0045|Nor did it confine itself to mere verbal recommendations.|Nor did it confine itself to mere verbal recommendations. +LJ005-0046|The good it tried to do took active shape in the establishment of temporary refuges -- at Hoxton for males, and in the Hackney Road for females|The good it tried to do took active shape in the establishment of temporary refuges -- at Hoxton for males, and in the Hackney Road for females +LJ005-0047|for the reception of deserving cases discharged from prison. The governor of Newgate and other metropolitan prisons had orders of admission to this refuge|for the reception of deserving cases discharged from prison. The governor of Newgate and other metropolitan prisons had orders of admission to this refuge +LJ005-0048|which he could bestow on prisoners on release, and so save the better-disposed or the completely destitute from lapsing at once into crime.|which he could bestow on prisoners on release, and so save the better-disposed or the completely destitute from lapsing at once into crime. +LJ005-0049|The refuge, which had for its object the training of its inmates in habits of industry,|The refuge, which had for its object the training of its inmates in habits of industry, +LJ005-0050|and in moral and religious duty, and which after a time sought to provide them with suitable situations, was supported entirely out of the funds of the Society.|and in moral and religious duty, and which after a time sought to provide them with suitable situations, was supported entirely out of the funds of the Society. +LJ005-0051|At the time of its greatest prosperity, its annual income from donations and subscriptions was about one thousand six hundred pounds.|At the time of its greatest prosperity, its annual income from donations and subscriptions was about one thousand six hundred pounds. +LJ005-0052|Another point to which the Society devoted infinite pains was the preparation of plans for the guidance of architects in the construction of prisons.|Another point to which the Society devoted infinite pains was the preparation of plans for the guidance of architects in the construction of prisons. +LJ005-0053|A very valuable volume published by the Society|A very valuable volume published by the Society +LJ005-0054|traced the progress of prison architecture from the days when the jail was the mere annexe of the baronial or episcopal castle|traced the progress of prison architecture from the days when the jail was the mere annexe of the baronial or episcopal castle +LJ005-0055|or a dungeon above or below the gate of a town, to the first attempts at systematic reconstruction carried out under the advice and supervision of Howard.|or a dungeon above or below the gate of a town, to the first attempts at systematic reconstruction carried out under the advice and supervision of Howard. +LJ005-0056|It is interesting to observe that the plan of "radiation," by which the prison blocks radiated from a central hall, like spokes in a wheel|It is interesting to observe that the plan of "radiation," by which the prison blocks radiated from a central hall, like spokes in a wheel +LJ005-0057|was introduced as early as 1790 by Mr. Blackburn|was introduced as early as seventeen ninety by Mr. Blackburn +LJ005-0058|an architect of eminence who was very largely employed in the erection of prison buildings at the close of the last century.|an architect of eminence who was very largely employed in the erection of prison buildings at the close of the last century. +LJ005-0059|With some important modifications this principle of radiation is still the rule.|With some important modifications this principle of radiation is still the rule. +LJ005-0060|The Society did not limit its remarks to the description of what had already been done|The Society did not limit its remarks to the description of what had already been done +LJ005-0061|but it offered suggestions for future buildings, with numerous carefully-executed drawings and designs of the model it recommended for imitation.|but it offered suggestions for future buildings, with numerous carefully-executed drawings and designs of the model it recommended for imitation. +LJ005-0062|Experience has since shown that in some respects these plans are defective, especially in the placing of the governor's residence in the center of the prison.|Experience has since shown that in some respects these plans are defective, especially in the placing of the governor's residence in the center of the prison. +LJ005-0063|It was thought that this would guarantee constant supervision and inspection, but it did nothing of the kind, and only the presence of warders on duty|It was thought that this would guarantee constant supervision and inspection, but it did nothing of the kind, and only the presence of warders on duty +LJ005-0064|is found now-a-days to be really efficacious. The main recommendations, however, are based upon common sense|is found now-a-days to be really efficacious. The main recommendations, however, are based upon common sense +LJ005-0065|and none are more commendable than that which deprecates the excessive ornamentation of the external parts of the edifice.|and none are more commendable than that which deprecates the excessive ornamentation of the external parts of the edifice. +LJ005-0066|"The new jails," as Howard says, "having pompous fronts, appear like palaces to the lower class of people, and many persons are against them on this account."|"The new jails," as Howard says, "having pompous fronts, appear like palaces to the lower class of people, and many persons are against them on this account." +LJ005-0067|The Prison Society reproves the misdirected efforts of ambitious architects, who by a lavish and improvident expenditure of public money|The Prison Society reproves the misdirected efforts of ambitious architects, who by a lavish and improvident expenditure of public money +LJ005-0068|sought to "rank the prisons they built among the most splendid buildings of the city or town."|sought to "rank the prisons they built among the most splendid buildings of the city or town." +LJ005-0069|Absence of embellishment is in perfect unison with the character of the establishment.|Absence of embellishment is in perfect unison with the character of the establishment. +LJ005-0070|These are principles fully recognized now-a-days, and it may fairly be conceded that the Prison Discipline Society's ideal|These are principles fully recognized now-a-days, and it may fairly be conceded that the Prison Discipline Society's ideal +LJ005-0071|differed little from that kept in view in the construction of the latest and best modern jails.|differed little from that kept in view in the construction of the latest and best modern jails. +LJ005-0072|After a few years of active exertion the Society was rewarded by fresh legislation.|After a few years of active exertion the Society was rewarded by fresh legislation. +LJ005-0073|To its efforts, and their effect upon Parliament and the public mind, we must attribute the new Jail Acts of 4 George IV|To its efforts, and their effect upon Parliament and the public mind, we must attribute the new Jail Acts of four George the fourth +LJ005-0074|cap. 64, and 5 George IV. cap. 85|cap. sixty-four, and five George the fourth cap. eighty-five +LJ005-0075|which having gone through several sessions, at last became law in 1823 to 4|which having gone through several sessions, at last became law in eighteen twenty-three to four +LJ005-0076|By the preamble of the first-named act it was declared|By the preamble of the first-named act it was declared +LJ005-0077|"expedient to introduce such measures and arrangements as shall not only provide for the safe custody,|"expedient to introduce such measures and arrangements as shall not only provide for the safe custody, +LJ005-0078|but shall also tend more effectually to preserve the health|but shall also tend more effectually to preserve the health +LJ005-0079|and improve the morals of the prisoners, and shall insure the proper measure of punishment to convicted offenders.|and improve the morals of the prisoners, and shall insure the proper measure of punishment to convicted offenders. +LJ005-0080|Accordingly due provision was made for the enforcement of hard labor on all prisoners sentenced to it, and for the employment of all others.|Accordingly due provision was made for the enforcement of hard labor on all prisoners sentenced to it, and for the employment of all others. +LJ005-0082|unless such ability (to work) should cease by reason of sickness, infirmity, the want of sufficient work, or from any other cause.|unless such ability (to work) should cease by reason of sickness, infirmity, the want of sufficient work, or from any other cause. +LJ005-0083|It was distinctly laid down that male and female prisoners should be confined in separate buildings or parts of the prison,|It was distinctly laid down that male and female prisoners should be confined in separate buildings or parts of the prison, +LJ005-0084|"so as to prevent them from seeing, conversing, or holding any intercourse with each other."|"so as to prevent them from seeing, conversing, or holding any intercourse with each other." +LJ005-0085|Classification was insisted upon, in the manner laid down by the 24 George III. cap. 54,|Classification was insisted upon, in the manner laid down by the twenty-four George the third cap. fifty-four, +LJ005-0086|with such further separation as the justices should deem conducive to good order and discipline.|with such further separation as the justices should deem conducive to good order and discipline. +LJ005-0087|Female prisoners were in all cases to be under the charge of female officers.|Female prisoners were in all cases to be under the charge of female officers. +LJ005-0088|Every prison containing female prisoners was to have a matron who was to reside constantly in the prison.|Every prison containing female prisoners was to have a matron who was to reside constantly in the prison. +LJ005-0089|The religious and moral welfare of the prisoners were to be attended to,|The religious and moral welfare of the prisoners were to be attended to, +LJ005-0090|the first by daily services, the latter by the appointment of schoolmasters and instruction in reading and writing.|the first by daily services, the latter by the appointment of schoolmasters and instruction in reading and writing. +LJ005-0091|Last, but not least, the use of irons was strictly forbidden, "except in cases of urgent and absolute necessity,"|Last, but not least, the use of irons was strictly forbidden, "except in cases of urgent and absolute necessity," +LJ005-0092|and every prisoner was to be provided with a hammock or cot to himself, suitable bedding, and, if possible, a separate cell.|and every prisoner was to be provided with a hammock or cot to himself, suitable bedding, and, if possible, a separate cell. +LJ005-0093|The second act, passed in the following year, enlarged and amended the first, and at the same time gave powers to the House|The second act, passed in the following year, enlarged and amended the first, and at the same time gave powers to the House +LJ005-0094|to call for information as to the observance of its provisions.|to call for information as to the observance of its provisions. +LJ005-0095|The promulgation of these two Jail Acts strengthened the hands of the Prison Discipline Society enormously.|The promulgation of these two Jail Acts strengthened the hands of the Prison Discipline Society enormously. +LJ005-0096|It had now a legal and authoritative standard of efficiency to apply,|It had now a legal and authoritative standard of efficiency to apply, +LJ005-0097|and could expose all the local authorities that still lagged behind, or neglected to comply with the provisions of the new laws.|and could expose all the local authorities that still lagged behind, or neglected to comply with the provisions of the new laws. +LJ005-0098|The Society did not shrink from its self-imposed duty, but continued year after year, with unflagging energy and unflinching spirit, to watch closely|The Society did not shrink from its self-imposed duty, but continued year after year, with unflagging energy and unflinching spirit, to watch closely +LJ005-0099|and report at length upon the condition of the prisons of the country.|and report at length upon the condition of the prisons of the country. +LJ005-0100|For this purpose it kept up an extensive correspondence with all parts of the kingdom, and circulated queries to be answered in detail,|For this purpose it kept up an extensive correspondence with all parts of the kingdom, and circulated queries to be answered in detail, +LJ005-0102|Upon these and the private visitations made by various members the Society obtained the facts,|Upon these and the private visitations made by various members the Society obtained the facts, +LJ005-0103|often highly damnatory, which were embodied in its annual reports.|often highly damnatory, which were embodied in its annual reports. +LJ005-0104|The progress of improvement was certainly extremely slow.|The progress of improvement was certainly extremely slow. +LJ005-0105|It was long before the many jurisdictions imitated the few.|It was long before the many jurisdictions imitated the few. +LJ005-0106|Jails, of which the old prison at Reading was a specimen, were still left intact.|Jails, of which the old prison at Reading was a specimen, were still left intact. +LJ005-0107|In that prison, with its cells and yards arranged within the shell of an ancient abbey chapel,|In that prison, with its cells and yards arranged within the shell of an ancient abbey chapel, +LJ005-0109|or contriving some means of escape by climbing the fluted columns which supported the Gothic arches of the aisles,|or contriving some means of escape by climbing the fluted columns which supported the Gothic arches of the aisles, +LJ005-0110|and so passing by the roof down into the garden and on to freedom.|and so passing by the roof down into the garden and on to freedom. +LJ005-0111|In a county prison adjoining the metropolis, the separation between the male and female quarters was supposed to be accomplished by the erection of an iron railing;|In a county prison adjoining the metropolis, the separation between the male and female quarters was supposed to be accomplished by the erection of an iron railing; +LJ005-0112|in this same prison capital convicts were chained to the floor until execution.|in this same prison capital convicts were chained to the floor until execution. +LJ005-0113|In another jail not far off male and female felons still occupied the same room -- underground, and reached by a ladder of ten steps.|In another jail not far off male and female felons still occupied the same room -- underground, and reached by a ladder of ten steps. +LJ005-0114|In others the separation between the sexes consisted in a hanging curtain|In others the separation between the sexes consisted in a hanging curtain +LJ005-0115|or an imaginary boundary line, and nothing prevented parties from passing to either side|or an imaginary boundary line, and nothing prevented parties from passing to either side +LJ005-0116|but an empty regulation which all so disposed could defy.|but an empty regulation which all so disposed could defy. +LJ005-0117|Numbers of the jails were still unprovided with chaplains, and the prisoners never heard Divine service.|Numbers of the jails were still unprovided with chaplains, and the prisoners never heard Divine service. +LJ005-0118|In many others there were no infirmaries, no places set apart for the confinement of prisoners afflicted with dangerous and infectious disorders.|In many others there were no infirmaries, no places set apart for the confinement of prisoners afflicted with dangerous and infectious disorders. +LJ005-0119|No attempt was made to maintain discipline.|No attempt was made to maintain discipline. +LJ005-0120|Half the jails had no code of rules properly prepared and sanctioned by the judges, according to law.|Half the jails had no code of rules properly prepared and sanctioned by the judges, according to law. +LJ005-0121|By degrees, however,|By degrees, however, +LJ005-0122|the changes necessary to bring the prisons into conformity with the recent acts were attempted, if not actually introduced into the county prisons, to which,|the changes necessary to bring the prisons into conformity with the recent acts were attempted, if not actually introduced into the county prisons, to which, +LJ005-0123|with a few of the more important city or borough prisons, these acts more especially applied.|with a few of the more important city or borough prisons, these acts more especially applied. +LJ005-0124|Most of the local authorities embarked into considerable expenditure, determined to rebuild their jails de novo on the most approved pattern,|Most of the local authorities embarked into considerable expenditure, determined to rebuild their jails de novo on the most approved pattern, +LJ005-0125|or to reappropriate, reconstruct, and patch up the existing prisons till they were more in accordance with the growing requirements of the times.|or to reappropriate, reconstruct, and patch up the existing prisons till they were more in accordance with the growing requirements of the times. +LJ005-0126|Religious worship became more generally the rule; chaplains were appointed, and chapels provided for them; surgeons and hospitals also.|Religious worship became more generally the rule; chaplains were appointed, and chapels provided for them; surgeons and hospitals also. +LJ005-0127|Workshops were built at many prisons, various kinds of manufactures and trades were set on foot, including weaving, matting, shoe-making, and tailoring.|Workshops were built at many prisons, various kinds of manufactures and trades were set on foot, including weaving, matting, shoe-making, and tailoring. +LJ005-0128|The interior of one prison was illuminated throughout with gas, -- still a novelty, which had been generally adopted in London only four years previously,|The interior of one prison was illuminated throughout with gas, -- still a novelty, which had been generally adopted in London only four years previously, +LJ005-0129|"a measure which must greatly tend to discourage attempts to escape."|"a measure which must greatly tend to discourage attempts to escape." +LJ005-0130|There were tread-wheels at most of the prisons, and regular employment thereon or at some other kind of hard labor.|There were tread-wheels at most of the prisons, and regular employment thereon or at some other kind of hard labor. +LJ005-0131|In many places too where the prisoners earned money by their work, they were granted a portion of it for their own use after proper deduction for maintenance.|In many places too where the prisoners earned money by their work, they were granted a portion of it for their own use after proper deduction for maintenance. +LJ005-0132|Only a few glaring evils still demanded a remedy.|Only a few glaring evils still demanded a remedy. +LJ005-0133|The provision of separate sleeping cells was still quite inadequate. For instance,|The provision of separate sleeping cells was still quite inadequate. For instance, +LJ005-0134|in twenty-two county jails there were 1063 sleeping cells in all (in 1823)|in twenty-two county jails there were one thousand sixty-three sleeping cells in all (in eighteen twenty-three) +LJ005-0135|and the average daily number committed that year amounted to 3985.|and the average daily number committed that year amounted to three thousand, nine hundred eighty-five. +LJ005-0136|The want of sleeping cells long continued a crying need.|The want of sleeping cells long continued a crying need. +LJ005-0137|Four years later the Prison Society reported|Four years later the Prison Society reported +LJ005-0138|that in four prisons, which at one time of the year contained 1308 prisoners, there were only sixty-eight sleeping rooms or cells,|that in four prisons, which at one time of the year contained one thousand three hundred eight prisoners, there were only sixty-eight sleeping rooms or cells, +LJ005-0139|making an average of nineteen persons occupying each room.|making an average of nineteen persons occupying each room. +LJ005-0140|At the New Prison, Clerkenwell, which had become the principal reception jail of Middlesex, and so took all the untried,|At the New Prison, Clerkenwell, which had become the principal reception jail of Middlesex, and so took all the untried, +LJ005-0141|the sleeping space per head was only sixteen inches, and often as many as 293 men had to be accommodated on barrack beds|the sleeping space per head was only sixteen inches, and often as many as two hundred ninety-three men had to be accommodated on barrack beds +LJ005-0142|occupying barely 390 feet lineal|occupying barely three hundred ninety feet lineal +LJ005-0143|The "scenes of tumult and obscenity" in these night rooms are said to have been beyond description; a prisoner in one nocturnal riot lost an eye.|The "scenes of tumult and obscenity" in these night rooms are said to have been beyond description; a prisoner in one nocturnal riot lost an eye. +LJ005-0144|Yet to Clerkenwell were now committed the juveniles, and all who were inexperienced in crime.|Yet to Clerkenwell were now committed the juveniles, and all who were inexperienced in crime. +LJ005-0145|Great want of uniformity in treatment in the various prisons was still noticeable|Great want of uniformity in treatment in the various prisons was still noticeable +LJ005-0146|and was indeed destined to continue for another half century, in other words, until the introduction of the Prison Act of 1877.|and was indeed destined to continue for another half century, in other words, until the introduction of the Prison Act of eighteen seventy-seven. +LJ005-0147|At the time of which I am writing there was great diversity of practice as regards the hours of labor.|At the time of which I am writing there was great diversity of practice as regards the hours of labor. +LJ005-0148|In some prisons the prisoners worked seven hours a day, in others ten and ten and a half.|In some prisons the prisoners worked seven hours a day, in others ten and ten and a half. +LJ005-0149|The nature of the employment varied greatly in severity, especially the tread-wheel labor.|The nature of the employment varied greatly in severity, especially the tread-wheel labor. +LJ005-0150|In some county jails, as I have already said, female prisoners were placed upon the tread-wheel;|In some county jails, as I have already said, female prisoners were placed upon the tread-wheel; +LJ005-0151|in others women were very properly exempted from it, and also from all severe labor.|in others women were very properly exempted from it, and also from all severe labor. +LJ005-0152|Earnings were very differently appropriated. Here the prisoners were given the whole amount, there a half or a third.|Earnings were very differently appropriated. Here the prisoners were given the whole amount, there a half or a third. +LJ005-0153|Sometimes this money might be expended in the purchase of extra articles of food.|Sometimes this money might be expended in the purchase of extra articles of food. +LJ005-0154|The rations varied considerably everywhere.|The rations varied considerably everywhere. +LJ005-0155|It was still limited to bread in some places, the allowance of which varied from one to three pounds;|It was still limited to bread in some places, the allowance of which varied from one to three pounds; +LJ005-0156|in others meat, soup, gruel, beer were given.|in others meat, soup, gruel, beer were given. +LJ005-0157|Here and there food was not issued in kind, but a money allowance which the prisoner might expend himself.|Here and there food was not issued in kind, but a money allowance which the prisoner might expend himself. +LJ005-0158|Bedding and clothing was still denied, but only in a few jails;|Bedding and clothing was still denied, but only in a few jails; +LJ005-0159|in others both were supplied in ample quantities, the cost varying per prisoner from twenty shillings to five pounds.|in others both were supplied in ample quantities, the cost varying per prisoner from twenty shillings to five pounds. +LJ005-0160|It was plain that although the law had defined general principles of prison government,|It was plain that although the law had defined general principles of prison government, +LJ005-0161|too much discretion was still left to the magistracy to fill in the details. The legislature only recommended,|too much discretion was still left to the magistracy to fill in the details. The legislature only recommended, +LJ005-0162|it did not peremptorily insist. Too often the letter of the law was observed, but not its spirit.|it did not peremptorily insist. Too often the letter of the law was observed, but not its spirit. +LJ005-0163|One great impediment to wide amelioration was that a vast number of small jails lay out of reach of the law.|One great impediment to wide amelioration was that a vast number of small jails lay out of reach of the law. +LJ005-0164|When the new acts were introduced, numerous prisons under local jurisdiction were exempted from the operation of the law.|When the new acts were introduced, numerous prisons under local jurisdiction were exempted from the operation of the law. +LJ005-0165|They were so radically bad that reform seemed hopeless, and it was thought wiser not to bring them under provisions which clearly could not be enforced.|They were so radically bad that reform seemed hopeless, and it was thought wiser not to bring them under provisions which clearly could not be enforced. +LJ005-0166|Mr. Peel, who as Home Secretary had charge of the bill,|Mr. Peel, who as Home Secretary had charge of the bill, +LJ005-0167|which became the 4 George IV. cap. 64, said that he had abstained from legislating for these small jurisdictions "on mature deliberation."|which became the four George the fourth cap. sixty-four, said that he had abstained from legislating for these small jurisdictions "on mature deliberation." +LJ005-0168|"It is not," he said, "that I am insensible of the lamentable and disgraceful situation in which many of them are,|"It is not," he said, "that I am insensible of the lamentable and disgraceful situation in which many of them are, +LJ005-0169|but I indulge a hope that many of them will contract with the counties,|but I indulge a hope that many of them will contract with the counties, +LJ005-0170|that many of them will build new jails, and that when in a year or two we come to examine their situation,|that many of them will build new jails, and that when in a year or two we come to examine their situation, +LJ005-0171|we shall find but few which have not in one or other of these ways removed the grievance of which such just complaint is made.|we shall find but few which have not in one or other of these ways removed the grievance of which such just complaint is made. +LJ005-0172|When that time arrives|When that time arrives +LJ005-0173|I shall not hesitate to ask Parliament for powers to compel them to make the necessary alterations, for it is not to be endured that these local jurisdictions should remain|I shall not hesitate to ask Parliament for powers to compel them to make the necessary alterations, for it is not to be endured that these local jurisdictions should remain +LJ005-0174|in the deplorable situation in which many of them now are."|in the deplorable situation in which many of them now are." +LJ005-0175|At this time there were in England one hundred and seventy boroughs, cities, towns, and liberties|At this time there were in England one hundred and seventy boroughs, cities, towns, and liberties +LJ005-0176|which possessed the right of trying criminals for various offenses.|which possessed the right of trying criminals for various offenses. +LJ005-0177|Nearly every one of these jurisdictions had its own prison, and there were one hundred and sixty such jails in all.|Nearly every one of these jurisdictions had its own prison, and there were one hundred and sixty such jails in all. +LJ005-0178|Many of them consisted of one or two rooms at most.|Many of them consisted of one or two rooms at most. +LJ005-0179|he total number of prisoners they received during the year varied from two persons to many hundreds.|he total number of prisoners they received during the year varied from two persons to many hundreds. +LJ005-0180|It was in these jails, withdrawn from the pressure of authority, that the new rules were invariably ignored.|It was in these jails, withdrawn from the pressure of authority, that the new rules were invariably ignored. +LJ005-0181|The right and privilege of the borough to maintain its own place of confinement was so "ancient and indisputable,"|The right and privilege of the borough to maintain its own place of confinement was so "ancient and indisputable," +LJ005-0182|that for long no idea of interfering with them was entertained.|that for long no idea of interfering with them was entertained. +LJ005-0183|All that was urged was that the borough magistracy had no right to govern their jails|All that was urged was that the borough magistracy had no right to govern their jails +LJ005-0184|so as to corrupt those committed, "to the injury of the peace and morals of the public."|so as to corrupt those committed, "to the injury of the peace and morals of the public." +LJ005-0185|As time passed, however, these magistrates made no effort at reform.|As time passed, however, these magistrates made no effort at reform. +LJ005-0186|They neither built new jails nor contracted with the counties, as had been expected, for the transfer of their prisoners.|They neither built new jails nor contracted with the counties, as had been expected, for the transfer of their prisoners. +LJ005-0187|As the Society put it in 1827, "the friends to the improvement of prison discipline will regret to learn|As the Society put it in eighteen twenty-seven, "the friends to the improvement of prison discipline will regret to learn +LJ005-0188|that the jails attached to corporate jurisdictions continue to be the fruitful sources|that the jails attached to corporate jurisdictions continue to be the fruitful sources +LJ005-0189|of vice and misery, debasing all who are confined within their walls, and disseminating through their respective communities|of vice and misery, debasing all who are confined within their walls, and disseminating through their respective communities +LJ005-0190|the knowledge and practice of every species of criminality.|the knowledge and practice of every species of criminality. +LJ005-0191|The Society proceeded to support this indictment by facts. It is much the old story.|The Society proceeded to support this indictment by facts. It is much the old story. +LJ005-0192|The prisoners were lodged in rooms whence they could converse with passengers in the streets, and freely obtain spirits and other prohibited articles.|The prisoners were lodged in rooms whence they could converse with passengers in the streets, and freely obtain spirits and other prohibited articles. +LJ005-0193|All descriptions of offenders congregated together in the felons' wards.|All descriptions of offenders congregated together in the felons' wards. +LJ005-0194|The keeper and his officers resided at a distance from the jail, and left its inmates to their own devices.|The keeper and his officers resided at a distance from the jail, and left its inmates to their own devices. +LJ005-0195|There was no decency whatever in the internal arrangements;|There was no decency whatever in the internal arrangements; +LJ005-0196|still no separation of the sexes, no means of ablution or other necessary services.|still no separation of the sexes, no means of ablution or other necessary services. +LJ005-0197|One borough prison consisted of nothing more than a couple of cells, about ten yards square, and absolutely nothing more.|One borough prison consisted of nothing more than a couple of cells, about ten yards square, and absolutely nothing more. +LJ005-0198|In another borough, with a population of ten thousand, the prison was of the same dimensions.|In another borough, with a population of ten thousand, the prison was of the same dimensions. +LJ005-0199|One cell was a dungeon, and the other an "improper and unhealthy abode for any human being," with a watercourse running through it.|One cell was a dungeon, and the other an "improper and unhealthy abode for any human being," with a watercourse running through it. +LJ005-0200|Most of these small jails were still in existence and in much the same state eight years later,|Most of these small jails were still in existence and in much the same state eight years later, +LJ005-0201|as is shown by the report of the Commissioners to inquire into the state of the municipal corporations in 1835.|as is shown by the report of the Commissioners to inquire into the state of the municipal corporations in eighteen thirty-five. +LJ005-0203|Thus Dinas Mwddy, in Merionethshire, had, "besides the pinfold and the stocks or crib, a little prison."|Thus Dinas Mwddy, in Merionethshire, had, "besides the pinfold and the stocks or crib, a little prison." +LJ005-0204|Clun, in Shropshire, had a lock-up under the town hall.|Clun, in Shropshire, had a lock-up under the town hall. +LJ005-0205|At Eye, in Suffolk, the jail was part of the poor-house; so it was at Richmond, in Yorkshire, where the master of the workhouse was also keeper of the jail.|At Eye, in Suffolk, the jail was part of the poor-house; so it was at Richmond, in Yorkshire, where the master of the workhouse was also keeper of the jail. +LJ005-0206|At Godmanchester there was no jail, but a cage to secure prisoners till they could be taken before a magistrate.|At Godmanchester there was no jail, but a cage to secure prisoners till they could be taken before a magistrate. +LJ005-0207|Kidderminster had a prison, one damp chill room,|Kidderminster had a prison, one damp chill room, +LJ005-0208|the only aperture through which air could be admitted being an iron grating level with the street,|the only aperture through which air could be admitted being an iron grating level with the street, +LJ005-0209|through the bars of which quills or reeds were inserted, and drink conveyed to the prisoners.|through the bars of which quills or reeds were inserted, and drink conveyed to the prisoners. +LJ005-0210|At Walsall, in Staffordshire,|At Walsall, in Staffordshire, +LJ005-0211|the jail consisted of six cells, frequently so damp that the moisture trickled down the walls; there was not space for air or exercise,|the jail consisted of six cells, frequently so damp that the moisture trickled down the walls; there was not space for air or exercise, +LJ005-0212|and the prison allowance was still limited to bread and water.|and the prison allowance was still limited to bread and water. +LJ005-0213|Newgate through all these years continued a bye-word with the Society.|Newgate through all these years continued a bye-word with the Society. +LJ005-0214|Some reforms had certainly been introduced, such as the abolition of irons, already referred to, and the establishment of male and female infirmaries.|Some reforms had certainly been introduced, such as the abolition of irons, already referred to, and the establishment of male and female infirmaries. +LJ005-0215|The regular daily visitation of the chaplain was also insisted upon.|The regular daily visitation of the chaplain was also insisted upon. +LJ005-0216|But it was pointed out in 1823 that defective construction must always bar the way to any radical improvement in Newgate.|But it was pointed out in eighteen twenty-three that defective construction must always bar the way to any radical improvement in Newgate. +LJ005-0217|Without enlargement no material change in discipline or interior economy could possibly be introduced.|Without enlargement no material change in discipline or interior economy could possibly be introduced. +LJ005-0218|The chapel still continued incommodious and insufficient|The chapel still continued incommodious and insufficient +LJ005-0219|female prisoners were still exposed to the full view of the males, the netting in front of the gallery being perfectly useless as a screen.|female prisoners were still exposed to the full view of the males, the netting in front of the gallery being perfectly useless as a screen. +LJ005-0220|In 1824 Newgate had no glass in its windows, except in the infirmary and one ward of the chapel yard;|In eighteen twenty-four Newgate had no glass in its windows, except in the infirmary and one ward of the chapel yard; +LJ005-0221|and the panes were filled in with oiled paper, an insufficient protection against the weather;|and the panes were filled in with oiled paper, an insufficient protection against the weather; +LJ005-0222|and as the window-frames would not shut tight, the prisoners complained much of the cold, especially at night.|and as the window-frames would not shut tight, the prisoners complained much of the cold, especially at night. +LJ005-0223|There was a diminution in the numbers in custody, due to the adoption of the practice of not committing at once to Newgate every offender for trial at the Old Bailey|There was a diminution in the numbers in custody, due to the adoption of the practice of not committing at once to Newgate every offender for trial at the Old Bailey +LJ005-0224|but nothing had been done to improve the prison buildings.|but nothing had been done to improve the prison buildings. +LJ005-0225|In 1827 the Society was compelled to report that "no material change had taken place in Newgate since the passing of the prison laws,|In eighteen twenty-seven the Society was compelled to report that "no material change had taken place in Newgate since the passing of the prison laws, +LJ005-0226|and that consequently the observance of their most important provisions was habitually neglected.|and that consequently the observance of their most important provisions was habitually neglected. +LJ005-0227|It was enacted that the court of aldermen should make rules for the government of the prison, and that these should be posted publicly within the walls.|It was enacted that the court of aldermen should make rules for the government of the prison, and that these should be posted publicly within the walls. +LJ005-0228|As yet no rules or regulations had been printed or prepared.|As yet no rules or regulations had been printed or prepared. +LJ005-0229|By another clause of the Jail Act, two justices were to be appointed to visit the prison at least thrice in every quarter, and "oftener if occasion required."|By another clause of the Jail Act, two justices were to be appointed to visit the prison at least thrice in every quarter, and "oftener if occasion required." +LJ005-0230|These justices were to inspect every part of the prison, and examine into the state and condition of prisoners.|These justices were to inspect every part of the prison, and examine into the state and condition of prisoners. +LJ005-0231|The city justices had not fulfilled this obligation.|The city justices had not fulfilled this obligation. +LJ005-0232|Idleness was still the general rule for all prisoners in Newgate, in defiance of the law.|Idleness was still the general rule for all prisoners in Newgate, in defiance of the law. +LJ005-0233|There was no instruction of adult prisoners, in accordance with the law. The sleeping accommodation was still altogether contrary to the latest ideas.|There was no instruction of adult prisoners, in accordance with the law. The sleeping accommodation was still altogether contrary to the latest ideas. +LJ005-0235|Last, and worst of all, the arrangements for keeping the condemned prisoners between sentence and execution were more than unsatisfactory.|Last, and worst of all, the arrangements for keeping the condemned prisoners between sentence and execution were more than unsatisfactory. +LJ005-0236|They were not confined apart from each other, but were crowded thirty or forty together in the press yard,|They were not confined apart from each other, but were crowded thirty or forty together in the press yard, +LJ005-0237|so that "corrupt conversation obliterated from the mind of him who is doomed to suffer every serious feeling and valuable impression."|so that "corrupt conversation obliterated from the mind of him who is doomed to suffer every serious feeling and valuable impression." +LJ005-0238|I shall have more to say on this subject, and upon the state of Newgate generally, in the following chapter.|I shall have more to say on this subject, and upon the state of Newgate generally, in the following chapter. +LJ005-0239|The Prison Society did not relax its efforts as time passed, but its leading members had other and more pressing claims upon their energies.|The Prison Society did not relax its efforts as time passed, but its leading members had other and more pressing claims upon their energies. +LJ005-0240|Mr. Buxton had succeeded to the great work which William Wilberforce had commenced, and led the repeated attacks upon slavery in British colonies|Mr. Buxton had succeeded to the great work which William Wilberforce had commenced, and led the repeated attacks upon slavery in British colonies +LJ005-0241|till the whole body of the slaves were manumitted in 1833.|till the whole body of the slaves were manumitted in eighteen thirty-three. +LJ005-0242|In the year immediately preceding this, Parliament was too busy with the great question of its own reform to spare much time for domestic legislation.|In the year immediately preceding this, Parliament was too busy with the great question of its own reform to spare much time for domestic legislation. +LJ005-0243|Nevertheless a committee of the House of Commons was appointed in 1831 to report upon the whole system of secondary punishments,|Nevertheless a committee of the House of Commons was appointed in eighteen thirty-one to report upon the whole system of secondary punishments, +LJ005-0244|which dealt with jails of all classes, as well as transportation.|which dealt with jails of all classes, as well as transportation. +LJ005-0245|This committee animadverted strongly upon the system in force at the metropolitan jails, and more especially upon the condition of Newgate|This committee animadverted strongly upon the system in force at the metropolitan jails, and more especially upon the condition of Newgate +LJ005-0246|where "prisoners before and after trial are under no efficient superintendence," and where "there was no restraint, or attempt at restraint."|where "prisoners before and after trial are under no efficient superintendence," and where "there was no restraint, or attempt at restraint." +LJ005-0247|Mr. Samuel Hoare was examined by this committee|Mr. Samuel Hoare was examined by this committee +LJ005-0249|From the moment of a person's committal he was certain to be plunged deeper and deeper in guilt.|From the moment of a person's committal he was certain to be plunged deeper and deeper in guilt. +LJ005-0250|The prisoners were crowded together in the jail, contrary to the requirements of the 4 George IV.|The prisoners were crowded together in the jail, contrary to the requirements of the four George the fourth +LJ005-0251|Again in 1835 prisons and their inmates became once more the care of the senate, and the subject was taken up this time by the House of Lords.|Again in eighteen thirty-five prisons and their inmates became once more the care of the senate, and the subject was taken up this time by the House of Lords. +LJ005-0252|A committee was appointed, under the presidency of the Duke of Richmond|A committee was appointed, under the presidency of the Duke of Richmond +LJ005-0253|"to inquire into and report upon the several jails and houses of correction in the counties, cities, and corporate towns within England and Wales|"to inquire into and report upon the several jails and houses of correction in the counties, cities, and corporate towns within England and Wales +LJ005-0254|upon the rules and discipline therein established with regard to the treatment of unconvicted as well as convicted persons."|upon the rules and discipline therein established with regard to the treatment of unconvicted as well as convicted persons." +LJ005-0255|The committee was also to report upon the manner in which sentences were carried out, and to recommend any alterations necessary in the rules|The committee was also to report upon the manner in which sentences were carried out, and to recommend any alterations necessary in the rules +LJ005-0256|in order to insure uniformity of discipline. It met on the 31st March, 1835, and continued its sittings well into July|in order to insure uniformity of discipline. It met on the thirty-first March, eighteen thirty-five, and continued its sittings well into July +LJ005-0257|during which time a host of witnesses were examined, and the committee presented three separate reports,|during which time a host of witnesses were examined, and the committee presented three separate reports, +LJ005-0258|embodying recommendations which may be said to have formed the basis of modern prison management.|embodying recommendations which may be said to have formed the basis of modern prison management. +LJ005-0259|It was laid down as a first and indispensable principle that uniformity of discipline should prevail everywhere,|It was laid down as a first and indispensable principle that uniformity of discipline should prevail everywhere, +LJ005-0260|a theory which did not become a practical fact for forty more years.|a theory which did not become a practical fact for forty more years. +LJ005-0261|As a means of securing this uniformity,|As a means of securing this uniformity, +LJ005-0262|it was suggested that the rules framed for prison government should be subjected to the Secretary of State for approval,|it was suggested that the rules framed for prison government should be subjected to the Secretary of State for approval, +LJ005-0263|and not, as heretofore, to the judges of assize; that, both to check abuses and watch the progress of improvement,|and not, as heretofore, to the judges of assize; that, both to check abuses and watch the progress of improvement, +LJ005-0264|inspectors of prisons should be appointed, who should visit all the prisons from time to time and report to the Secretary of State.|inspectors of prisons should be appointed, who should visit all the prisons from time to time and report to the Secretary of State. +LJ005-0265|It was recommended that the dietaries should be submitted and approved like the rules; that convicted prisoners should not receive any food but the jail allowance;|It was recommended that the dietaries should be submitted and approved like the rules; that convicted prisoners should not receive any food but the jail allowance; +LJ005-0266|that food and fuel should be issued in kind, and never provided by the prisoners themselves out of monies granted them.|that food and fuel should be issued in kind, and never provided by the prisoners themselves out of monies granted them. +LJ005-0267|The use of tobacco, hitherto pretty generally indulged in both by men and women,|The use of tobacco, hitherto pretty generally indulged in both by men and women, +LJ005-0268|should be strictly prohibited, "as a stimulating luxury inconsistent with any notion of strict discipline and the due pressure of just punishment."|should be strictly prohibited, "as a stimulating luxury inconsistent with any notion of strict discipline and the due pressure of just punishment." +LJ005-0269|Prison officers should not have any share in prisoners' earnings,|Prison officers should not have any share in prisoners' earnings, +LJ005-0270|which should be paid into general prison funds, and no part of them handed over to the prisoners themselves.|which should be paid into general prison funds, and no part of them handed over to the prisoners themselves. +LJ005-0271|As a means of increasing the severity of imprisonment, letters and visits from outside should not be permitted during the first six months of an imprisonment.|As a means of increasing the severity of imprisonment, letters and visits from outside should not be permitted during the first six months of an imprisonment. +LJ005-0272|Various other recommendations were made as regards the appointment of chaplain and schoolmasters; the limitation of the powers of wardsmen,|Various other recommendations were made as regards the appointment of chaplain and schoolmasters; the limitation of the powers of wardsmen, +LJ005-0273|or prisoners employed in positions of trust, who should not be permitted to traffic with their fellow-prisoners in any way.|or prisoners employed in positions of trust, who should not be permitted to traffic with their fellow-prisoners in any way. +LJ005-0274|The committee most of all insisted upon the entire individual separation of prisoners, except during the hours of labor,|The committee most of all insisted upon the entire individual separation of prisoners, except during the hours of labor, +LJ005-0275|religious worship, and instruction, as "absolutely necessary for preventing contamination,|religious worship, and instruction, as "absolutely necessary for preventing contamination, +LJ005-0276|and for securing a proper system of prison discipline.|and for securing a proper system of prison discipline. +LJ005-0277|This was the first enunciation of the system of separate confinement,|This was the first enunciation of the system of separate confinement, +LJ005-0278|which was eventually to replace the attempted arrangement of prisoners by classes according to antecedents and crimes,|which was eventually to replace the attempted arrangement of prisoners by classes according to antecedents and crimes, +LJ005-0279|an incomplete and fallacious method of preventing contamination.|an incomplete and fallacious method of preventing contamination. +LJ005-0280|The Lords' Committee fully recognized the painful fact|The Lords' Committee fully recognized the painful fact +LJ005-0281|that the greatest mischief followed from the intercourse which was still permitted in so many prisons; to use its words,|that the greatest mischief followed from the intercourse which was still permitted in so many prisons; to use its words, +LJ005-0282|"the comparatively innocent are seduced, the unwary are entrapped,|"the comparatively innocent are seduced, the unwary are entrapped, +LJ005-0283|and the tendency to crime in offenders not entirely hardened is confirmed by the language, the suggestions, and the example|and the tendency to crime in offenders not entirely hardened is confirmed by the language, the suggestions, and the example +LJ005-0284|of more depraved and systematic criminals.|of more depraved and systematic criminals. +LJ005-0285|This committee, as well as the one preceding it, also reported in terms of strong reprobation on the small prisons and jails|This committee, as well as the one preceding it, also reported in terms of strong reprobation on the small prisons and jails +LJ005-0286|still under the borough corporations. The Commons' Committee gave it as their opinion that they were in a deplorable state.|still under the borough corporations. The Commons' Committee gave it as their opinion that they were in a deplorable state. +LJ005-0287|The same language was used by the commissioners appointed to inquire into the municipal corporations in 1835,|The same language was used by the commissioners appointed to inquire into the municipal corporations in eighteen thirty-five, +LJ005-0288|when speaking more particularly of the borough jails.|when speaking more particularly of the borough jails. +LJ005-0289|In these the commissioners found "additional proof of the evils of continuing the present constitution of the local tribunals.|In these the commissioners found "additional proof of the evils of continuing the present constitution of the local tribunals. +LJ005-0290|Instances rarely occur in which the borough jails admit of any proper classification of the prisoners.|Instances rarely occur in which the borough jails admit of any proper classification of the prisoners. +LJ005-0291|In some large towns, as at Berwick on Tweed, Southampton, and Southwark, they (the prisons) are in a very discreditable condition.|In some large towns, as at Berwick on Tweed, Southampton, and Southwark, they (the prisons) are in a very discreditable condition. +LJ005-0292|In many of the smaller boroughs they are totally unfit for the confinement of human beings.|In many of the smaller boroughs they are totally unfit for the confinement of human beings. +LJ005-0293|In these places the prisoners are often without a proper supply of air and light; frequently the jails are mere dungeons under the town hall.|In these places the prisoners are often without a proper supply of air and light; frequently the jails are mere dungeons under the town hall. +LJ005-0294|It was frequently stated in evidence that the jail of the borough was in so unfit a state for the reception of prisoners,|It was frequently stated in evidence that the jail of the borough was in so unfit a state for the reception of prisoners, +LJ005-0295|that plaintiffs were unwilling to consign the defendants against whom they had obtained execution to confinement within its walls."|that plaintiffs were unwilling to consign the defendants against whom they had obtained execution to confinement within its walls." +LJ005-0296|The Lords' Committee on Jails were of the same opinion, and considered the prisons under corporate or peculiar jurisdiction in a very unsatisfactory condition.|The Lords' Committee on Jails were of the same opinion, and considered the prisons under corporate or peculiar jurisdiction in a very unsatisfactory condition. +LJ005-0297|They therefore recommended that the prisoners should be removed|They therefore recommended that the prisoners should be removed +LJ005-0298|to the county jails from such prisons as were past improvement, and that the borough funds should be charged for the accommodation.|to the county jails from such prisons as were past improvement, and that the borough funds should be charged for the accommodation. +LJ005-0299|The whole question was again dealt with in Lord John Russell's bill for the reform of the municipal corporations, and with a more liberal election of town councilors,|The whole question was again dealt with in Lord John Russell's bill for the reform of the municipal corporations, and with a more liberal election of town councilors, +LJ005-0300|and the establishment of municipal institutions upon a proper footing,|and the establishment of municipal institutions upon a proper footing, +LJ006-0002|In the preceding chapter I have been tempted by the importance of the general question to give it prominence and precedence over the particular branch of which I am treating.|In the preceding chapter I have been tempted by the importance of the general question to give it prominence and precedence over the particular branch of which I am treating. +LJ006-0003|Newgate has remained rather in the background while the whole of the jails as a body were under discussion.|Newgate has remained rather in the background while the whole of the jails as a body were under discussion. +LJ006-0004|But this digression was necessary in order to present a more complete picture of the state of jails in the early part of the present century,|But this digression was necessary in order to present a more complete picture of the state of jails in the early part of the present century, +LJ006-0005|just before the public mind was first awakened to the need for thorough reform.|just before the public mind was first awakened to the need for thorough reform. +LJ006-0006|I shall now return to the great jail of the city of London, and give a more detailed account of its condition and inner life|I shall now return to the great jail of the city of London, and give a more detailed account of its condition and inner life +LJ006-0007|as the inspectors of prisons found them in 1835 to 6.|as the inspectors of prisons found them in eighteen thirty-five to six. +LJ006-0008|These gentlemen were appointed in October 1835, owing to the strong representations of the Lords' Committee|These gentlemen were appointed in October eighteen thirty-five, owing to the strong representations of the Lords' Committee +LJ006-0009|backed up by the evidence of several influential witnesses.|backed up by the evidence of several influential witnesses. +LJ006-0010|Mr. Samuel Hoare, when examined, considered it indispensably necessary, to carry out whatever system might be established,|Mr. Samuel Hoare, when examined, considered it indispensably necessary, to carry out whatever system might be established, +LJ006-0011|that inspectors should watch over the observance of the law. He saw no objection on the score of their probable interference with the local jurisdiction,|that inspectors should watch over the observance of the law. He saw no objection on the score of their probable interference with the local jurisdiction, +LJ006-0012|but he would not arm them with any authority lest their cooperation might be offensive.|but he would not arm them with any authority lest their cooperation might be offensive. +LJ006-0013|Sir Frederick Roe was of the same opinion as regards the appointment, but he would give the inspectors the power of acting as well as reporting.|Sir Frederick Roe was of the same opinion as regards the appointment, but he would give the inspectors the power of acting as well as reporting. +LJ006-0014|They should be persons, he thought, selected from the highest class; the duty was most important,|They should be persons, he thought, selected from the highest class; the duty was most important, +LJ006-0015|one which required discretion, judgment, and knowledge of law, with sufficient insight and experience to discover defects in prison discipline.|one which required discretion, judgment, and knowledge of law, with sufficient insight and experience to discover defects in prison discipline. +LJ006-0016|These considerations no doubt had weight|These considerations no doubt had weight +LJ006-0017|with those who made the selection of the first inspectors, and the two gentlemen appointed were probably the most fitted in England to be so employed.|with those who made the selection of the first inspectors, and the two gentlemen appointed were probably the most fitted in England to be so employed. +LJ006-0019|The first named had long been an active philanthropist, devoting himself more particularly to the reformation of juvenile criminals.|The first named had long been an active philanthropist, devoting himself more particularly to the reformation of juvenile criminals. +LJ006-0020|William Crawford had been one of the promoters and managers of the Philanthropic Society's farm school.|William Crawford had been one of the promoters and managers of the Philanthropic Society's farm school. +LJ006-0021|Later on he had devoted himself to the personal investigation of the prisons of the United States.|Later on he had devoted himself to the personal investigation of the prisons of the United States. +LJ006-0022|At that time the mild and intelligent prison discipline in force in Pennsylvania, the legacy of the old Quaker immigrants,|At that time the mild and intelligent prison discipline in force in Pennsylvania, the legacy of the old Quaker immigrants, +LJ006-0023|had made such prisons as Auburn a model for imitation.|had made such prisons as Auburn a model for imitation. +LJ006-0024|Several European states had dispatched emissaries to examine and report upon them.|Several European states had dispatched emissaries to examine and report upon them. +LJ006-0025|France had sent Misseurs Beaumont and De Tocqueville, who subsequently published several interesting works on the subject.|France had sent Misseurs Beaumont and De Tocqueville, who subsequently published several interesting works on the subject. +LJ006-0026|England was represented by Mr. Crawford, and the result of his inquiry was given to the public as an appendix to the House of Commons' Report on Secondary Punishments.|England was represented by Mr. Crawford, and the result of his inquiry was given to the public as an appendix to the House of Commons' Report on Secondary Punishments. +LJ006-0027|It is an able and exhaustive state paper, testifying to the keenness of the writer's perception, and his unremitting labor in pursuing his researches.|It is an able and exhaustive state paper, testifying to the keenness of the writer's perception, and his unremitting labor in pursuing his researches. +LJ006-0028|Mr. Crawford was thoroughly versed in the still imperfectly understood science of prison management, and fully qualified for his new duties.|Mr. Crawford was thoroughly versed in the still imperfectly understood science of prison management, and fully qualified for his new duties. +LJ006-0029|The second inspector, the Rev. Whitworth Russell, was the chaplain of Millbank penitentiary,|The second inspector, the Rev. Whitworth Russell, was the chaplain of Millbank penitentiary, +LJ006-0030|the great architectural experiment which grew out of the strong representations of Jeremy Bentham and others, and was the first national recognition of the principle|the great architectural experiment which grew out of the strong representations of Jeremy Bentham and others, and was the first national recognition of the principle +LJ006-0031|that punishment must be reformatory as well as deterrent.|that punishment must be reformatory as well as deterrent. +LJ006-0032|Messrs. Crawford and Russell proceeded to carry out their new functions with commendable energy, and without a moment's loss of time.|Messrs. Crawford and Russell proceeded to carry out their new functions with commendable energy, and without a moment's loss of time. +LJ006-0033|The ink was barely dry upon their letters of appointment before they appeared at Newgate, and commenced a searching investigation.|The ink was barely dry upon their letters of appointment before they appeared at Newgate, and commenced a searching investigation. +LJ006-0035|took voluminous evidence from all classes of individuals, from the governor down to the convict in the condemned cells.|took voluminous evidence from all classes of individuals, from the governor down to the convict in the condemned cells. +LJ006-0036|They visited the wards after locking-up time, and saw with their own eyes what went on.|They visited the wards after locking-up time, and saw with their own eyes what went on. +LJ006-0038|"a subject of magnitude and importance sufficient to exclude other jails," they soon narrowed their inquiry still further,|"a subject of magnitude and importance sufficient to exclude other jails," they soon narrowed their inquiry still further, +LJ006-0039|and limited it to Newgate alone. Newgate indeed became the sole theme of their first report.|and limited it to Newgate alone. Newgate indeed became the sole theme of their first report. +LJ006-0040|The fact was that the years as they passed, nearly twenty in all, had worked but little permanent improvement in this detestable prison.|The fact was that the years as they passed, nearly twenty in all, had worked but little permanent improvement in this detestable prison. +LJ006-0041|Changes introduced under pressure had been only skin deep.|Changes introduced under pressure had been only skin deep. +LJ006-0042|Relapse was rapid and inevitable, so that the latter state of the prison was worse than the first.|Relapse was rapid and inevitable, so that the latter state of the prison was worse than the first. +LJ006-0043|The disgraceful overcrowding had been partially ended, but the same evils of indiscriminate association were still present; there was the old neglect of decency,|The disgraceful overcrowding had been partially ended, but the same evils of indiscriminate association were still present; there was the old neglect of decency, +LJ006-0044|the same callous indifference to the moral well-being of the prisoners, the same want of employment and of all disciplinary control.|the same callous indifference to the moral well-being of the prisoners, the same want of employment and of all disciplinary control. +LJ006-0045|All these evils were set forth at length in the inspectors' first report.|All these evils were set forth at length in the inspectors' first report. +LJ006-0046|There was no longer the faintest possible excuse for overcrowding. The numbers now committed to Newgate had sensibly diminished.|There was no longer the faintest possible excuse for overcrowding. The numbers now committed to Newgate had sensibly diminished. +LJ006-0047|The prison had become more or less a place of detention only, harboring mainly those awaiting trial.|The prison had become more or less a place of detention only, harboring mainly those awaiting trial. +LJ006-0048|To these were still added an average of about fifty expecting the last penalty of the law; a certain number of transports awaiting removal to the colonies;|To these were still added an average of about fifty expecting the last penalty of the law; a certain number of transports awaiting removal to the colonies; +LJ006-0049|an occasional prisoner or two committed by the Houses of Parliament, the Courts of King's Bench, Common Pleas,|an occasional prisoner or two committed by the Houses of Parliament, the Courts of King's Bench, Common Pleas, +LJ006-0050|the Exchequer, the Commissioners of bankruptcy and of taxes; smugglers, and a larger number sentenced for very short terms,|the Exchequer, the Commissioners of bankruptcy and of taxes; smugglers, and a larger number sentenced for very short terms, +LJ006-0051|and for offenses of the most varying description, by the Central Criminal Court.|and for offenses of the most varying description, by the Central Criminal Court. +LJ006-0052|The sum total thus produced was inconsiderable compared with the hundreds that had formerly filled the jail,|The sum total thus produced was inconsiderable compared with the hundreds that had formerly filled the jail, +LJ006-0053|and the whole by proper management might have been so accommodated as to prevent overcrowding.|and the whole by proper management might have been so accommodated as to prevent overcrowding. +LJ006-0054|But incredible as it may appear, the authorities of Newgate declined to avail themselves of the advantages offered them,|But incredible as it may appear, the authorities of Newgate declined to avail themselves of the advantages offered them, +LJ006-0055|and when the population fell they shut up one half the jail and crowded up the other.|and when the population fell they shut up one half the jail and crowded up the other. +LJ006-0056|Some rooms remained quite empty and unoccupied, while others were full to overflowing.|Some rooms remained quite empty and unoccupied, while others were full to overflowing. +LJ006-0057|Not only were the wards thus needlessly crammed, and for no reason but the niggardliness of the corporation which refused a proper supply of bedding|Not only were the wards thus needlessly crammed, and for no reason but the niggardliness of the corporation which refused a proper supply of bedding +LJ006-0058|but the occupants of each were huddled together indiscriminately. The inspectors found in the same wards in the chapel yard the convicted and the untried,|but the occupants of each were huddled together indiscriminately. The inspectors found in the same wards in the chapel yard the convicted and the untried, +LJ006-0059|the felon and the misdemeanant, the sane and the insane, the old and young offender.|the felon and the misdemeanant, the sane and the insane, the old and young offender. +LJ006-0060|The classification prescribed by the Jail Act, which laid down that certain prisoners should not intermix, was openly neglected,|The classification prescribed by the Jail Act, which laid down that certain prisoners should not intermix, was openly neglected, +LJ006-0061|and "the greatest contempt shown for the law."|and "the greatest contempt shown for the law." +LJ006-0062|In another part there were men charged with and convicted of unnatural offenses shut up with lads of tender years;|In another part there were men charged with and convicted of unnatural offenses shut up with lads of tender years; +LJ006-0063|minor offenders charged with small thefts or non-payment of small sums were cheek by jowl with convicts sentenced to long terms of transportation.|minor offenders charged with small thefts or non-payment of small sums were cheek by jowl with convicts sentenced to long terms of transportation. +LJ006-0064|In the master's side yard, which had only one washing place, as many as seventy-eight prisoners, frequently more,|In the master's side yard, which had only one washing place, as many as seventy-eight prisoners, frequently more, +LJ006-0065|were associated together, "of every variety of age, habit, and delinquency, without employment, oversight, or control."|were associated together, "of every variety of age, habit, and delinquency, without employment, oversight, or control." +LJ006-0066|In the middle yard it was still worse.|In the middle yard it was still worse. +LJ006-0067|"Here," say the inspectors, "are herded together the very worst class of prisoners; certainly a more wretched combination of human beings can hardly be imagined.|"Here," say the inspectors, "are herded together the very worst class of prisoners; certainly a more wretched combination of human beings can hardly be imagined. +LJ006-0068|We have reason to fear that poverty, ragged clothes, and an inability to pay the ward dues, elsewhere exacted for better accommodation,|We have reason to fear that poverty, ragged clothes, and an inability to pay the ward dues, elsewhere exacted for better accommodation, +LJ006-0069|consign many of the more petty and unpracticed offenders to this place,|consign many of the more petty and unpracticed offenders to this place, +LJ006-0070|where they inevitably meet with further contamination from the society of the most abandoned and incorrigible inmates of the jail."|where they inevitably meet with further contamination from the society of the most abandoned and incorrigible inmates of the jail." +LJ006-0071|No doubt the governor for the time being, Mr. Cope, was in a great measure to blame for all this, and for the want of proper classification.|No doubt the governor for the time being, Mr. Cope, was in a great measure to blame for all this, and for the want of proper classification. +LJ006-0072|I shall have occasion to speak again, and more at length, of Mr. Cope's careless and perfunctory discharge|I shall have occasion to speak again, and more at length, of Mr. Cope's careless and perfunctory discharge +LJ006-0073|of his many manifest duties, but I shall here confine myself to animadverting on his neglect as regards the appropriation of his prison.|of his many manifest duties, but I shall here confine myself to animadverting on his neglect as regards the appropriation of his prison. +LJ006-0074|He was unable to give any reason whatever for not utilizing the whole of the wards.|He was unable to give any reason whatever for not utilizing the whole of the wards. +LJ006-0075|He saw certain rooms fill up, and yet took no steps to open others that were locked up and empty.|He saw certain rooms fill up, and yet took no steps to open others that were locked up and empty. +LJ006-0076|He blamed the construction of Newgate for the neglect of classification, and was yet compelled to confess that he had made no attempt whatever to carry it out.|He blamed the construction of Newgate for the neglect of classification, and was yet compelled to confess that he had made no attempt whatever to carry it out. +LJ006-0077|The fact was, he did not keep the classification of prisoners on first arrival in his own hands, nor even in that of his officers.|The fact was, he did not keep the classification of prisoners on first arrival in his own hands, nor even in that of his officers. +LJ006-0079|The inspectors found that prisoners had their places assigned to them|The inspectors found that prisoners had their places assigned to them +LJ006-0080|by the inner gatesman, himself a convicted prisoner, and a "wardsman" or responsible head of a room.|by the inner gatesman, himself a convicted prisoner, and a "wardsman" or responsible head of a room. +LJ006-0081|The wardsman still exacted dues, of which more directly,|The wardsman still exacted dues, of which more directly, +LJ006-0082|and this particular official took excellent care to select as residents for his own ward those most suitable from his own point of view.|and this particular official took excellent care to select as residents for his own ward those most suitable from his own point of view. +LJ006-0083|"So great is the authority exercised by him,|"So great is the authority exercised by him, +LJ006-0084|and so numerous were his opportunities of showing favoritism, that all the prisoners may be said to be in his power.|and so numerous were his opportunities of showing favoritism, that all the prisoners may be said to be in his power. +LJ006-0085|If a man is poor and ragged, however inexperienced in crime, or however trifling may be the offense for which he has been committed|If a man is poor and ragged, however inexperienced in crime, or however trifling may be the offense for which he has been committed +LJ006-0086|his place is assigned among the most depraved, the most experienced, and the most incorrigible offenders in the middle yard.|his place is assigned among the most depraved, the most experienced, and the most incorrigible offenders in the middle yard. +LJ006-0087|It must be admitted that so far but little effort had been made to counteract the evils of indiscriminate association.|It must be admitted that so far but little effort had been made to counteract the evils of indiscriminate association. +LJ006-0088|It was not likely that a system which left innocent men -- for the great bulk of new arrivals were still untried|It was not likely that a system which left innocent men -- for the great bulk of new arrivals were still untried +LJ006-0089|to be pitchforked by chance anywhere, into any sort of company,|to be pitchforked by chance anywhere, into any sort of company, +LJ006-0090|within this the greatest nursery of crime in London, should exercise even the commonest care for the personal decency or comfort of the prisoners.|within this the greatest nursery of crime in London, should exercise even the commonest care for the personal decency or comfort of the prisoners. +LJ006-0091|Their treatment was also a matter of chance. They still slept on rope mats on the floor, herded together in companies of four or more to keep one another warm|Their treatment was also a matter of chance. They still slept on rope mats on the floor, herded together in companies of four or more to keep one another warm +LJ006-0092|and under the scanty covering of a couple of dirty stable-rugs apiece.|and under the scanty covering of a couple of dirty stable-rugs apiece. +LJ006-0093|So closely did they lie together, that the inspectors at their night visits found it difficult in stepping across the room to avoid treading on them.|So closely did they lie together, that the inspectors at their night visits found it difficult in stepping across the room to avoid treading on them. +LJ006-0094|Sometimes two mats were allotted to three sleepers. Sometimes four slept under the same bedding, and left their mats unoccupied.|Sometimes two mats were allotted to three sleepers. Sometimes four slept under the same bedding, and left their mats unoccupied. +LJ006-0095|The rugs used were never washed; an order existed that the bedding should be taken into the yards to be aired, but it was not very punctually obeyed.|The rugs used were never washed; an order existed that the bedding should be taken into the yards to be aired, but it was not very punctually obeyed. +LJ006-0096|The only convenience for personal ablutions were the pumps in the yards, and the far-off baths in the condemned or press-yard.|The only convenience for personal ablutions were the pumps in the yards, and the far-off baths in the condemned or press-yard. +LJ006-0097|Water might not be taken into the ward for washing purposes.|Water might not be taken into the ward for washing purposes. +LJ006-0098|There was some provision of clothing, but it was quite insufficient, and nothing at all was given if prisoners had enough of their own to cover their nakedness.|There was some provision of clothing, but it was quite insufficient, and nothing at all was given if prisoners had enough of their own to cover their nakedness. +LJ006-0099|The inspectors paraded the prisoners, and found them generally ragged and ill-clad, squalid and filthy in the extreme;|The inspectors paraded the prisoners, and found them generally ragged and ill-clad, squalid and filthy in the extreme; +LJ006-0100|many without stockings, and with hardly shoes to their feet;|many without stockings, and with hardly shoes to their feet; +LJ006-0101|some, who had the semblance of covering on the upper part of their feet, had no soles to the shoes, and their bare feet were on the ground.|some, who had the semblance of covering on the upper part of their feet, had no soles to the shoes, and their bare feet were on the ground. +LJ006-0102|This, too, was in the depth of the winter, and during a most inclement season.|This, too, was in the depth of the winter, and during a most inclement season. +LJ006-0103|The allowance of food was not illiberal,|The allowance of food was not illiberal, +LJ006-0104|but its issue was precarious, and dependent on the good will of the wardsmen, who measured out the portions to each according to his eye,|but its issue was precarious, and dependent on the good will of the wardsmen, who measured out the portions to each according to his eye, +LJ006-0105|and not with weights and measures, no turnkey being present.|and not with weights and measures, no turnkey being present. +LJ006-0106|Too much was left to the wardsman. It was he who could issue small luxuries;|Too much was left to the wardsman. It was he who could issue small luxuries; +LJ006-0107|he sold tea, coffee, sugar, tobacco, although prohibited, and extra beer.|he sold tea, coffee, sugar, tobacco, although prohibited, and extra beer. +LJ006-0108|He charged a weekly sum as ward dues for the use of knives, forks, and plates|He charged a weekly sum as ward dues for the use of knives, forks, and plates +LJ006-0109|a perpetuation under another form of the old detestable custom of garnish.|a perpetuation under another form of the old detestable custom of garnish. +LJ006-0110|He had power where his exactions were resisted of making the ward most uncomfortable for the defaulter.|He had power where his exactions were resisted of making the ward most uncomfortable for the defaulter. +LJ006-0111|He could trump up a false complaint against his fellow-prisoner, and so get him punished;|He could trump up a false complaint against his fellow-prisoner, and so get him punished; +LJ006-0112|he might keep him from the fire, or give him his soup or gruel in a pail instead of a basin.|he might keep him from the fire, or give him his soup or gruel in a pail instead of a basin. +LJ006-0113|The authority of these wardsmen so improperly exalted, and so entirely unchecked, degenerated into a baneful despotism.|The authority of these wardsmen so improperly exalted, and so entirely unchecked, degenerated into a baneful despotism. +LJ006-0114|They bought their offices from one another, and were thus considered to have a vested interest in them.|They bought their offices from one another, and were thus considered to have a vested interest in them. +LJ006-0115|Their original capital had been a few shillings, and for this they purchased the right to tax their fellows to the extent of pounds per week.|Their original capital had been a few shillings, and for this they purchased the right to tax their fellows to the extent of pounds per week. +LJ006-0116|The wardsman had a monopoly in supplying provisions, gave dinner and breakfast at his own price, and was such complete master of the ward|The wardsman had a monopoly in supplying provisions, gave dinner and breakfast at his own price, and was such complete master of the ward +LJ006-0117|that none of its inmates were suffered to make tea or coffee for themselves lest it should interfere with his sales.|that none of its inmates were suffered to make tea or coffee for themselves lest it should interfere with his sales. +LJ006-0118|He made collections when it suited him for ward purposes, to be spent as he chose, in candles and so forth.|He made collections when it suited him for ward purposes, to be spent as he chose, in candles and so forth. +LJ006-0119|When the wardsman was a man of some education, with some knowledge of legal chicanery gained by personal experience, he might add considerably to his emoluments|When the wardsman was a man of some education, with some knowledge of legal chicanery gained by personal experience, he might add considerably to his emoluments +LJ006-0120|by drawing briefs and petitions for his fellows. There was a recognized charge of 5 shillings per brief,|by drawing briefs and petitions for his fellows. There was a recognized charge of five shillings per brief, +LJ006-0121|for a petition of from 1 shilling, half pence to 8 shillings, according to its length,|for a petition of from one shilling, half pence to eight shillings, according to its length, +LJ006-0122|and by these payments a wardsman had been known to amass as much as £40.|and by these payments a wardsman had been known to amass as much as forty pounds. +LJ006-0123|The man intrusted with this privilege was often the inner gatesman,|The man intrusted with this privilege was often the inner gatesman, +LJ006-0124|the prisoner official already mentioned, who held the fate of new arrivals as regards location in his hands.|the prisoner official already mentioned, who held the fate of new arrivals as regards location in his hands. +LJ006-0125|It was not strange that he should sometimes misuse his power, and when prisoners were not to be cajoled into securing his legal services,|It was not strange that he should sometimes misuse his power, and when prisoners were not to be cajoled into securing his legal services, +LJ006-0126|had been known to employ threats, declaring that he was often consulted by the governor as to a prisoner's character,|had been known to employ threats, declaring that he was often consulted by the governor as to a prisoner's character, +LJ006-0127|in view of speaking to it at the trial, and he could easily do them a good turn -- or a very bad one.|in view of speaking to it at the trial, and he could easily do them a good turn -- or a very bad one. +LJ006-0128|The brief-drawing gatesman and wardsman at the time of the inspectors' first visit must have been a particularly powerful personage.|The brief-drawing gatesman and wardsman at the time of the inspectors' first visit must have been a particularly powerful personage. +LJ006-0129|He was on the most intimate and improperly familiar terms with the turnkeys,|He was on the most intimate and improperly familiar terms with the turnkeys, +LJ006-0130|had a key of both the master's side and middle side yards, was the only person present at the distribution of beer, and was trusted to examine,|had a key of both the master's side and middle side yards, was the only person present at the distribution of beer, and was trusted to examine, +LJ006-0131|and, if he chose, pass in, all provisions, money, clothes, and letters brought for prisoners by their friends.|and, if he chose, pass in, all provisions, money, clothes, and letters brought for prisoners by their friends. +LJ006-0132|All the wardsmen alike were more or less irresponsible.|All the wardsmen alike were more or less irresponsible. +LJ006-0133|The turnkeys complained bitterly that these old prisoners had more power than they themselves.|The turnkeys complained bitterly that these old prisoners had more power than they themselves. +LJ006-0134|The governor himself admitted that a prisoner of weak intellect who had been severely beaten and much injured by a wardsman did not dare complain|The governor himself admitted that a prisoner of weak intellect who had been severely beaten and much injured by a wardsman did not dare complain +LJ006-0135|the victim of this cruel ill-usage having "more fear of the power of the wardsman to injure him, than confidence in the governor's power to protect him."|the victim of this cruel ill-usage having "more fear of the power of the wardsman to injure him, than confidence in the governor's power to protect him." +LJ006-0136|These wardsmen, besides thus ruling the roast, had numerous special privileges, if such they can be called.|These wardsmen, besides thus ruling the roast, had numerous special privileges, if such they can be called. +LJ006-0137|They were not obliged to attend chapel, and seldom if ever went; "prisoners," said one of them under examination, "did not like the trouble of going to chapel."|They were not obliged to attend chapel, and seldom if ever went; "prisoners," said one of them under examination, "did not like the trouble of going to chapel." +LJ006-0138|They had a standing bedstead to sleep on, and a good flock mattress; double allowance of provisions, filched from the common stock.|They had a standing bedstead to sleep on, and a good flock mattress; double allowance of provisions, filched from the common stock. +LJ006-0139|Nobody interfered with them or regulated their conduct. They might get drunk when so disposed, and did so frequently, alone or in company.|Nobody interfered with them or regulated their conduct. They might get drunk when so disposed, and did so frequently, alone or in company. +LJ006-0140|Evidence was given before the inspectors of eight or ten prisoners seen "giddy drunk, not able to sit upon forms."|Evidence was given before the inspectors of eight or ten prisoners seen "giddy drunk, not able to sit upon forms." +LJ006-0141|The female wards-women were also given to intemperance.|The female wards-women were also given to intemperance. +LJ006-0142|The matron deposed to having seen the gates-woman "exceedingly drunk," and having been insulted by her.|The matron deposed to having seen the gates-woman "exceedingly drunk," and having been insulted by her. +LJ006-0143|There was no penalty attached to drunkenness.|There was no penalty attached to drunkenness. +LJ006-0144|A wardsman did not necessarily lose his situation for it. Nor was drink the only creature comfort he might enjoy.|A wardsman did not necessarily lose his situation for it. Nor was drink the only creature comfort he might enjoy. +LJ006-0145|He could indulge in snuff if a snuff-taker,|He could indulge in snuff if a snuff-taker, +LJ006-0146|and might always smoke his pipe undisturbed; for although the use of tobacco had been prohibited since the report of the Lords Committee,|and might always smoke his pipe undisturbed; for although the use of tobacco had been prohibited since the report of the Lords Committee, +LJ006-0147|it was still freely introduced into the prison.|it was still freely introduced into the prison. +LJ006-0148|Probably authority would not have been so recklessly usurped by the wardsmen had not the proper officials too readily surrendered it.|Probably authority would not have been so recklessly usurped by the wardsmen had not the proper officials too readily surrendered it. +LJ006-0149|The turnkeys left the prisoners very much to themselves, never entering the wards after locking-up time, at dusk, till unlocking next morning,|The turnkeys left the prisoners very much to themselves, never entering the wards after locking-up time, at dusk, till unlocking next morning, +LJ006-0150|and then only went round to count the number.|and then only went round to count the number. +LJ006-0151|Many of them were otherwise and improperly occupied for hours every day in menial services for the governor, cleaning his windows or grooming his horse.|Many of them were otherwise and improperly occupied for hours every day in menial services for the governor, cleaning his windows or grooming his horse. +LJ006-0152|One turnkey had been so employed several hours daily for nearly eleven years.|One turnkey had been so employed several hours daily for nearly eleven years. +LJ006-0153|It was not strange that subordinates should neglect their duty when superiors set the example.|It was not strange that subordinates should neglect their duty when superiors set the example. +LJ006-0154|Nothing was more prominently brought out by the inspectors than the inefficiency of the governor at that time, Mr. Cope.|Nothing was more prominently brought out by the inspectors than the inefficiency of the governor at that time, Mr. Cope. +LJ006-0155|He may have erred in some points through ignorance, but in others he was clearly guilty of culpable neglect.|He may have erred in some points through ignorance, but in others he was clearly guilty of culpable neglect. +LJ006-0156|We have seen that he took no pains to classify and separate prisoners on reception.|We have seen that he took no pains to classify and separate prisoners on reception. +LJ006-0157|This was only one of many grave omissions on his part. He did not feel it incumbent on himself to visit his prison often or see his prisoners.|This was only one of many grave omissions on his part. He did not feel it incumbent on himself to visit his prison often or see his prisoners. +LJ006-0158|The act prescribed that he should do both every twenty-four hours, but days passed without his entering the wards.|The act prescribed that he should do both every twenty-four hours, but days passed without his entering the wards. +LJ006-0159|The prisoners declared that they did not see him oftener than twice a week;|The prisoners declared that they did not see him oftener than twice a week; +LJ006-0160|one man who had been in the condemned ward for two months, said the governor only came there four times.|one man who had been in the condemned ward for two months, said the governor only came there four times. +LJ006-0161|Again, a turnkey deposed that his chief did not enter the wards more than once a fortnight.|Again, a turnkey deposed that his chief did not enter the wards more than once a fortnight. +LJ006-0162|But it is only fair to Mr. Cope to state that he himself said he went whenever he could find time|But it is only fair to Mr. Cope to state that he himself said he went whenever he could find time +LJ006-0163|and that he was constantly engaged attending sessions and going with drafts to the hulks.|and that he was constantly engaged attending sessions and going with drafts to the hulks. +LJ006-0164|But when he did visit, his inspections were of the most superficial character|But when he did visit, his inspections were of the most superficial character +LJ006-0165|sometimes he looked at his bolts and bars, but he never examined the cupboards, coal-boxes, or other possible hiding-places for cards|sometimes he looked at his bolts and bars, but he never examined the cupboards, coal-boxes, or other possible hiding-places for cards +LJ006-0166|dice, dangerous implements, or other prohibited articles.|dice, dangerous implements, or other prohibited articles. +LJ006-0167|He only attended chapel once on Sunday, never on the week-day, and generally devoted the time service was in progress|He only attended chapel once on Sunday, never on the week-day, and generally devoted the time service was in progress +LJ006-0168|to taking the descriptions of newly-arrived prisoners.|to taking the descriptions of newly-arrived prisoners. +LJ006-0169|He really did not know what passed in his jail|He really did not know what passed in his jail +LJ006-0170|and was surprised when the inspectors proved to him that practices of which he was ignorant, and which he admitted that he reprehended, went on without hindrance.|and was surprised when the inspectors proved to him that practices of which he was ignorant, and which he admitted that he reprehended, went on without hindrance. +LJ006-0171|He was satisfied to let matters run on as in the old times, he said in his own justification; with him what was, was right,|He was satisfied to let matters run on as in the old times, he said in his own justification; with him what was, was right, +LJ006-0172|and evils that should have been speedily rooted out remained because they had the prescription of long usage.|and evils that should have been speedily rooted out remained because they had the prescription of long usage. +LJ006-0173|He kept no daily journal of occurrences, and nothing, however important, was recorded at the time.|He kept no daily journal of occurrences, and nothing, however important, was recorded at the time. +LJ006-0174|The aldermen never called upon him to report, and left him nearly unsupervised and uncontrolled.|The aldermen never called upon him to report, and left him nearly unsupervised and uncontrolled. +LJ006-0175|In his administration of discipline he was quite uncertain;|In his administration of discipline he was quite uncertain; +LJ006-0176|the punishments he inflicted were unequal,|the punishments he inflicted were unequal, +LJ006-0177|and it was not the least part of the blame imputed to him that he made special favorites of particular prisoners, retaining of his own accord in Newgate,|and it was not the least part of the blame imputed to him that he made special favorites of particular prisoners, retaining of his own accord in Newgate, +LJ006-0178|and for years, felons who should have been sent beyond the seas.|and for years, felons who should have been sent beyond the seas. +LJ006-0179|But, indeed, his whole rule was far too mild, and under this mistaken leniency|But, indeed, his whole rule was far too mild, and under this mistaken leniency +LJ006-0180|the interior of the jail was more like a bear-garden or the noisy purlieus of a public-house than a prison.|the interior of the jail was more like a bear-garden or the noisy purlieus of a public-house than a prison. +LJ006-0181|It was the same old story -- evil constantly in the ascendant, the least criminal at the mercy of the most depraved.|It was the same old story -- evil constantly in the ascendant, the least criminal at the mercy of the most depraved. +LJ006-0182|Under the reckless contempt for regulations,|Under the reckless contempt for regulations, +LJ006-0183|the apathy of the authorities, and the undue ascendancy of those who, as convicted felons, should have been most sternly repressed,|the apathy of the authorities, and the undue ascendancy of those who, as convicted felons, should have been most sternly repressed, +LJ006-0184|the most hardened and the oldest in vice had the best of it, while the inexperienced beginner went to the wall.|the most hardened and the oldest in vice had the best of it, while the inexperienced beginner went to the wall. +LJ006-0185|Edward Gibbon Wakefield, who spent three years in Newgate a little before the time of the inspectors' first report,|Edward Gibbon Wakefield, who spent three years in Newgate a little before the time of the inspectors' first report, +LJ006-0186|said with justice that "incredible scenes of horror occur in Newgate."|said with justice that "incredible scenes of horror occur in Newgate." +LJ006-0187|It was, moreover, in his opinion undoubtedly the greatest nursery of crime in London.|It was, moreover, in his opinion undoubtedly the greatest nursery of crime in London. +LJ006-0188|The days were passed in idleness, debauchery, riotous quarreling, immoral conversation,|The days were passed in idleness, debauchery, riotous quarreling, immoral conversation, +LJ006-0189|gambling, indirect contravention of parliamentary rules, instruction in all nefarious processes,|gambling, indirect contravention of parliamentary rules, instruction in all nefarious processes, +LJ006-0190|lively discourse upon past criminal exploits, elaborate discussion of others to be perpetrated after release.|lively discourse upon past criminal exploits, elaborate discussion of others to be perpetrated after release. +LJ006-0191|No provision whatever was made for the employment of prisoners, no materials were purchased, no trade instructors appointed.|No provision whatever was made for the employment of prisoners, no materials were purchased, no trade instructors appointed. +LJ006-0192|There was no school for adults; only the boys were taught anything, and their instructor, with his assistant, were convicted prisoners.|There was no school for adults; only the boys were taught anything, and their instructor, with his assistant, were convicted prisoners. +LJ006-0193|Idle hands and unoccupied brains found in mischief the only means of whiling away the long hours of incarceration.|Idle hands and unoccupied brains found in mischief the only means of whiling away the long hours of incarceration. +LJ006-0194|Gaming of all kinds, although forbidden by the Jail Acts, was habitually practiced.|Gaming of all kinds, although forbidden by the Jail Acts, was habitually practiced. +LJ006-0195|This was admitted in evidence by the turnkeys, and was proved by the appearance of the prison tables, which bore the marks of gaming-boards deeply cut into them.|This was admitted in evidence by the turnkeys, and was proved by the appearance of the prison tables, which bore the marks of gaming-boards deeply cut into them. +LJ006-0196|Prisoners confessed that it was a favorite occupation, the chief games being "shoving halfpence" on the table,|Prisoners confessed that it was a favorite occupation, the chief games being "shoving halfpence" on the table, +LJ006-0197|pitch in the hole, cribbage, dominoes, and common tossing, at which as much as four or five shillings would change hands in an hour.|pitch in the hole, cribbage, dominoes, and common tossing, at which as much as four or five shillings would change hands in an hour. +LJ006-0198|But this was not the only amusement. Most of the wards took in the daily papers,|But this was not the only amusement. Most of the wards took in the daily papers, +LJ006-0199|the most popular being the "Times," "Morning Herald," and "Morning Chronicle"; on Sunday the "Weekly Dispatch," "Bell's Life," and the "Weekly Messenger."|the most popular being the "Times," "Morning Herald," and "Morning Chronicle"; on Sunday the "Weekly Dispatch," "Bell's Life," and the "Weekly Messenger." +LJ006-0200|The newsman had free access to the prison; he passed in unsearched and unexamined, and, unaccompanied by an officer,|The newsman had free access to the prison; he passed in unsearched and unexamined, and, unaccompanied by an officer, +LJ006-0201|went at once to his customers, who bought their paper and paid for it themselves.|went at once to his customers, who bought their paper and paid for it themselves. +LJ006-0202|The news-vendor was also a tobacconist,|The news-vendor was also a tobacconist, +LJ006-0203|and he had thus ample means of introducing to the prisoners the prohibited but always much-coveted and generally procurable weed.|and he had thus ample means of introducing to the prisoners the prohibited but always much-coveted and generally procurable weed. +LJ006-0204|In the same way the wardsman laid in his stock to be retailed. Other light literature besides the daily journals were in circulation:|In the same way the wardsman laid in his stock to be retailed. Other light literature besides the daily journals were in circulation: +LJ006-0205|novels, flash songs, play-books, such as "Jane Shore," "Grimm's German Tales," with Cruikshank's illustrations,|novels, flash songs, play-books, such as "Jane Shore," "Grimm's German Tales," with Cruikshank's illustrations, +LJ006-0206|and publications which in these days would have been made the subject of a criminal prosecution.|and publications which in these days would have been made the subject of a criminal prosecution. +LJ006-0207|One of these, published by Stockdale, the inspectors styled "a book of the most disgusting nature."|One of these, published by Stockdale, the inspectors styled "a book of the most disgusting nature." +LJ006-0208|There was also a good supply of Bibles and prayers,|There was also a good supply of Bibles and prayers, +LJ006-0209|the donation of a philanthropic gentleman, Captain Brown, but these, particularly the Bibles, bore little appearance of having been used.|the donation of a philanthropic gentleman, Captain Brown, but these, particularly the Bibles, bore little appearance of having been used. +LJ006-0210|Drink, in more or less unlimited quantities, was still to be had.|Drink, in more or less unlimited quantities, was still to be had. +LJ006-0211|Spirits certainly were now excluded; but a potman, with full permission of the sheriffs,|Spirits certainly were now excluded; but a potman, with full permission of the sheriffs, +LJ006-0212|brought in beer for sale from a neighboring public-house, and visited all the wards with no other escort than the prisoner gatesman.|brought in beer for sale from a neighboring public-house, and visited all the wards with no other escort than the prisoner gatesman. +LJ006-0213|The quantity to be issued per head was limited by the prison regulations to one pint|The quantity to be issued per head was limited by the prison regulations to one pint +LJ006-0214|but no steps were taken to prevent any prisoner from obtaining more if he could pay for it.|but no steps were taken to prevent any prisoner from obtaining more if he could pay for it. +LJ006-0215|The beer-man brought in as much as he pleased; he sold it without the controlling presence of an officer.|The beer-man brought in as much as he pleased; he sold it without the controlling presence of an officer. +LJ006-0216|Not only did prisoners come again and again for a "pint," but large quantities were carried off to the wards to be drunk later in the day.|Not only did prisoners come again and again for a "pint," but large quantities were carried off to the wards to be drunk later in the day. +LJ006-0217|There were more varied, and at times, especially when beer had circulated freely, more uproarious diversions.|There were more varied, and at times, especially when beer had circulated freely, more uproarious diversions. +LJ006-0218|Wrestling, in which legs were occasionally broken, was freely indulged in; also such low games as "cobham,"|Wrestling, in which legs were occasionally broken, was freely indulged in; also such low games as "cobham," +LJ006-0219|leap-frog, puss in the corner, and "fly the garter," for which purpose the rugs were spread out to prevent feet slipping on the floor.|leap-frog, puss in the corner, and "fly the garter," for which purpose the rugs were spread out to prevent feet slipping on the floor. +LJ006-0220|Feasting alternated with fighting.|Feasting alternated with fighting. +LJ006-0221|The weekly introduction of food, to which I shall presently refer, formed the basis of luxurious banquets, washed down by liquor|The weekly introduction of food, to which I shall presently refer, formed the basis of luxurious banquets, washed down by liquor +LJ006-0222|and enlivened by flash songs and thrilling long-winded descriptions of robberies and other "plants."|and enlivened by flash songs and thrilling long-winded descriptions of robberies and other "plants." +LJ006-0223|There was much swearing and bad language, the very worst that could be used, from the first thing in the morning to the last thing at night.|There was much swearing and bad language, the very worst that could be used, from the first thing in the morning to the last thing at night. +LJ006-0224|New arrivals, especially the innocent and still guileless debutant, were tormented with rude horse-play, and assailed by the most insulting "chaff."|New arrivals, especially the innocent and still guileless debutant, were tormented with rude horse-play, and assailed by the most insulting "chaff." +LJ006-0225|If any man presumed to turn in too early|If any man presumed to turn in too early +LJ006-0226|he was "toed," that is to say, a string was fastened to his big toe while he was asleep, and he was dragged from off his mat,|he was "toed," that is to say, a string was fastened to his big toe while he was asleep, and he was dragged from off his mat, +LJ006-0227|or his bedclothes were drawn away across the room.|or his bedclothes were drawn away across the room. +LJ006-0228|The ragged part of the prisoners were very anxious to destroy the clothes of the better dressed, and often lighted small pieces of cloth,|The ragged part of the prisoners were very anxious to destroy the clothes of the better dressed, and often lighted small pieces of cloth, +LJ006-0229|which they dropped smoldering into their fellow-prisoners' pockets.|which they dropped smoldering into their fellow-prisoners' pockets. +LJ006-0230|Often the victim, goaded to madness, attacked his tormentors; a fight was then certain to follow.|Often the victim, goaded to madness, attacked his tormentors; a fight was then certain to follow. +LJ006-0231|These fights sometimes took place in the daytime, when a ring was regularly formed, and two or three stood by the door to watch for the officer's approach.|These fights sometimes took place in the daytime, when a ring was regularly formed, and two or three stood by the door to watch for the officer's approach. +LJ006-0232|More often they occurred at night, and were continued to the bitter end.|More often they occurred at night, and were continued to the bitter end. +LJ006-0233|The prisoners in this way administered serious punishment on one another. Black eyes and broken noses were always to be seen.|The prisoners in this way administered serious punishment on one another. Black eyes and broken noses were always to be seen. +LJ006-0234|More cruel injuries were common enough, which did not result from honest hand-to-hand fights.|More cruel injuries were common enough, which did not result from honest hand-to-hand fights. +LJ006-0235|The surgeon's journal produced to the inspectors contained numerous entries of terrible wounds inflicted in a cowardly way.|The surgeon's journal produced to the inspectors contained numerous entries of terrible wounds inflicted in a cowardly way. +LJ006-0236|"A serious accident: one of the prisoners had a hot poker run into his eye."|"A serious accident: one of the prisoners had a hot poker run into his eye." +LJ006-0237|"A lad named Matthew White has had a wound in his eye by a bone thrown at him, which very nearly destroyed vision."|"A lad named Matthew White has had a wound in his eye by a bone thrown at him, which very nearly destroyed vision." +LJ006-0238|"There was a disturbance in the transport yard yesterday evening, and the police were called in.|"There was a disturbance in the transport yard yesterday evening, and the police were called in. +LJ006-0239|During the tumult a prisoner, who was one of the worst of the rioters, was bruised about the head and body.|During the tumult a prisoner, who was one of the worst of the rioters, was bruised about the head and body. +LJ006-0240|Watkins' knee-joint is very severely injured.|Watkins' knee-joint is very severely injured. +LJ006-0241|A prisoner Baxter is in the infirmary in consequence of a severe injury to his wrist-joint.|A prisoner Baxter is in the infirmary in consequence of a severe injury to his wrist-joint. +LJ006-0242|Watkins' case, referred to above, is made the subject of another and a special report from the surgeon.|Watkins' case, referred to above, is made the subject of another and a special report from the surgeon. +LJ006-0243|He was in the transport side, when one of his fellows, in endeavoring to strike another prisoner with a large poker, missed his aim, and struck Watkins' knee|He was in the transport side, when one of his fellows, in endeavoring to strike another prisoner with a large poker, missed his aim, and struck Watkins' knee +LJ006-0244|Violent inflammation and extensive suppuration ensued, and for a considerable time amputation seemed inevitable.|Violent inflammation and extensive suppuration ensued, and for a considerable time amputation seemed inevitable. +LJ006-0245|After severe suffering prolonged for many months, the inflammation was subdued, but the cartilage of the knee-joint was destroyed, and he was crippled for life.|After severe suffering prolonged for many months, the inflammation was subdued, but the cartilage of the knee-joint was destroyed, and he was crippled for life. +LJ006-0246|On another occasion a young man, who was being violently teased, seized a knife and stabbed his tormentor in the back.|On another occasion a young man, who was being violently teased, seized a knife and stabbed his tormentor in the back. +LJ006-0247|The prisoner who used the knife was secured, but it was the wardsman, and not the officers, to whom the report was made, and no official inquiry or punishment followed.|The prisoner who used the knife was secured, but it was the wardsman, and not the officers, to whom the report was made, and no official inquiry or punishment followed. +LJ006-0248|Matters were at times still worse, and the rioting went on to such dangerous lengths as to endanger the safety of the building.|Matters were at times still worse, and the rioting went on to such dangerous lengths as to endanger the safety of the building. +LJ006-0249|On one occasion a disturbance was raised which was not quelled until windows had been broken and forms and tables burnt.|On one occasion a disturbance was raised which was not quelled until windows had been broken and forms and tables burnt. +LJ006-0250|The officers were obliged to go in among the prisoners to restore order with drawn cutlasses,|The officers were obliged to go in among the prisoners to restore order with drawn cutlasses, +LJ006-0251|but the presence and authority of the governor himself became indispensable.|but the presence and authority of the governor himself became indispensable. +LJ006-0252|The worst fights occurred on Sunday afternoons; but nearly every night the act of locking up became, from the consequent removal of all supervision,|The worst fights occurred on Sunday afternoons; but nearly every night the act of locking up became, from the consequent removal of all supervision, +LJ006-0253|the signal for the commencement of obscene talk, revelry, and violence.|the signal for the commencement of obscene talk, revelry, and violence. +LJ006-0254|Other regulations laid down by the Jail Acts were still defied. One of these was that prisoners should be restricted to the jail allowance of food;|Other regulations laid down by the Jail Acts were still defied. One of these was that prisoners should be restricted to the jail allowance of food; +LJ006-0255|but all could still obtain as much extra, and of a luxurious kind, as their friends chose to bring them in.|but all could still obtain as much extra, and of a luxurious kind, as their friends chose to bring them in. +LJ006-0256|Visitors were still permitted to come with supplies on given days of the week, about the only limitation being that the food should be cooked, and cold;|Visitors were still permitted to come with supplies on given days of the week, about the only limitation being that the food should be cooked, and cold; +LJ006-0257|hot meat, poultry, and fish were also forbidden.|hot meat, poultry, and fish were also forbidden. +LJ006-0258|But the inspectors found in the ward cupboards mince-pies and other pasties, cold joints, hams, and so forth.|But the inspectors found in the ward cupboards mince-pies and other pasties, cold joints, hams, and so forth. +LJ006-0259|Many other articles were introduced by visitors, including money, tobacco, pipes, and snuff.|Many other articles were introduced by visitors, including money, tobacco, pipes, and snuff. +LJ006-0260|From the same source came the two or three strong files which the inspectors found in one ward,|From the same source came the two or three strong files which the inspectors found in one ward, +LJ006-0261|together with four bradawls, several large iron spikes, screws, nails, and knives;|together with four bradawls, several large iron spikes, screws, nails, and knives; +LJ006-0262|all of them instruments calculated to facilitate attempts at breaking out of prison,|all of them instruments calculated to facilitate attempts at breaking out of prison, +LJ006-0263|and capable of becoming most dangerous weapons in the hands of desperate and determined men.|and capable of becoming most dangerous weapons in the hands of desperate and determined men. +LJ006-0264|The nearly indiscriminate admission of visitors, although restricted to certain days, continued to be an unmixed evil.|The nearly indiscriminate admission of visitors, although restricted to certain days, continued to be an unmixed evil. +LJ006-0265|The untried might see their friends three times a week, the convicted only once.|The untried might see their friends three times a week, the convicted only once. +LJ006-0266|On these occasions precautions were supposed to be taken to exclude bad characters,|On these occasions precautions were supposed to be taken to exclude bad characters, +LJ006-0267|yet many persons of notoriously loose life continually obtained egress.|yet many persons of notoriously loose life continually obtained egress. +LJ006-0268|Women saw men if they merely pretended to be wives; even boys were visited by their sweethearts.|Women saw men if they merely pretended to be wives; even boys were visited by their sweethearts. +LJ006-0269|Decency was, however, insured by a line of demarcation, and visitors were kept upon each side of a separated double iron railing.|Decency was, however, insured by a line of demarcation, and visitors were kept upon each side of a separated double iron railing. +LJ006-0270|But no search was made to intercept prohibited articles at the gate, and there was no permanent gate-keeper,|But no search was made to intercept prohibited articles at the gate, and there was no permanent gate-keeper, +LJ006-0271|which would have greatly helped to keep out bad characters. Some idea of the difficulty and inconvenience of these lax regulations as regards visiting,|which would have greatly helped to keep out bad characters. Some idea of the difficulty and inconvenience of these lax regulations as regards visiting, +LJ006-0272|may be gathered from the statement that as many as three hundred were often admitted on the same day|may be gathered from the statement that as many as three hundred were often admitted on the same day +LJ006-0273|enough to altogether upset what small show of decorum and discipline was still preserved in the prison.|enough to altogether upset what small show of decorum and discipline was still preserved in the prison. +LJ006-0274|Perhaps the worst feature of the visiting system was the permission accorded to male prisoners "under the name of husbands, brothers, and sons"|Perhaps the worst feature of the visiting system was the permission accorded to male prisoners "under the name of husbands, brothers, and sons" +LJ006-0275|to have access to the female side on Sundays and Wednesdays, in order to visit their supposed relations there.|to have access to the female side on Sundays and Wednesdays, in order to visit their supposed relations there. +LJ006-0276|On this female side, where the Ladies' Association still reigned supreme, more system and a greater semblance of decorum was maintained.|On this female side, where the Ladies' Association still reigned supreme, more system and a greater semblance of decorum was maintained. +LJ006-0277|But there were evils akin to those on the male side, prominent amongst which was the undue influence accorded to prisoners.|But there were evils akin to those on the male side, prominent amongst which was the undue influence accorded to prisoners. +LJ006-0278|A female prisoner kept the registers.|A female prisoner kept the registers. +LJ006-0279|Wards-women were allowed much the same authority, with the same temptations to excess, and intoxication was not unknown among them and others.|Wards-women were allowed much the same authority, with the same temptations to excess, and intoxication was not unknown among them and others. +LJ006-0280|The clothing was still meager and ragged: the washing places insufficient, and wanting in decency;|The clothing was still meager and ragged: the washing places insufficient, and wanting in decency; +LJ006-0281|in some yards|in some yards +LJ006-0282|the pump was the only provision, and this in a place within sight of visitors, of the windows of the male turnkeys, and unprotected from the weather.|the pump was the only provision, and this in a place within sight of visitors, of the windows of the male turnkeys, and unprotected from the weather. +LJ006-0283|There was the same crowding in the sleeping arrangements as on the male side; the same scarcity of bedding.|There was the same crowding in the sleeping arrangements as on the male side; the same scarcity of bedding. +LJ006-0284|It was a special evil of this part of the prison, that the devotional exercises, originally so profitable, had grown into a kind of edifying spectacle,|It was a special evil of this part of the prison, that the devotional exercises, originally so profitable, had grown into a kind of edifying spectacle, +LJ006-0285|which numbers of well-meaning but inquisitive people were anxious to witness.|which numbers of well-meaning but inquisitive people were anxious to witness. +LJ006-0286|Thus, when the inspectors visited there were twenty-three strangers, and only twenty-eight prisoners.|Thus, when the inspectors visited there were twenty-three strangers, and only twenty-eight prisoners. +LJ006-0287|The presence of so many strangers, many of them gentlemen, distracted the prisoners' attention, and could not be productive of much good.|The presence of so many strangers, many of them gentlemen, distracted the prisoners' attention, and could not be productive of much good. +LJ006-0288|The separation of the sexes was not indeed rigidly carried out in Newgate as yet.|The separation of the sexes was not indeed rigidly carried out in Newgate as yet. +LJ006-0289|We have seen that male prisoners visited their female relations and friends on the female side. Besides this,|We have seen that male prisoners visited their female relations and friends on the female side. Besides this, +LJ006-0290|the gatesman who prepared the briefs had interviews with female prisoners alone while taking their instructions; a female came alone and unaccompanied by a matron|the gatesman who prepared the briefs had interviews with female prisoners alone while taking their instructions; a female came alone and unaccompanied by a matron +LJ006-0291|to clean the governor's office in the male prison;|to clean the governor's office in the male prison; +LJ006-0292|male prisoners carried coal into the female prison, when they saw and could speak or pass letters to the female prisoners;|male prisoners carried coal into the female prison, when they saw and could speak or pass letters to the female prisoners; +LJ006-0293|and the men could also at any time go for tea, coffee, and sugar to Mrs. Brown's shop, which was inside the female gate.|and the men could also at any time go for tea, coffee, and sugar to Mrs. Brown's shop, which was inside the female gate. +LJ006-0294|In the bail-dock, where most improper general association was permitted, the female prisoners were often altogether in the charge of male turnkeys.|In the bail-dock, where most improper general association was permitted, the female prisoners were often altogether in the charge of male turnkeys. +LJ006-0295|The governor was also personally responsible for gross contravention of this rule of separation,|The governor was also personally responsible for gross contravention of this rule of separation, +LJ006-0296|and was in the habit of drawing frequently upon the female prison for prisoners to act as domestic servants in his own private dwelling.|and was in the habit of drawing frequently upon the female prison for prisoners to act as domestic servants in his own private dwelling. +LJ006-0297|Some member of the Ladies' Association observed and commented upon the fact that a "young rosy-cheeked girl" had been kept by the governor from transportation,|Some member of the Ladies' Association observed and commented upon the fact that a "young rosy-cheeked girl" had been kept by the governor from transportation, +LJ006-0298|while older women in infirm health were sent across the seas.|while older women in infirm health were sent across the seas. +LJ006-0299|His excuse was that he had given the girl his promise that she should not go, an assumption of prerogative which by no means rested with him;|His excuse was that he had given the girl his promise that she should not go, an assumption of prerogative which by no means rested with him; +LJ006-0300|but he afterwards admitted that the girl had been recommended to him by the principal turnkey, who knew something of her friends.|but he afterwards admitted that the girl had been recommended to him by the principal turnkey, who knew something of her friends. +LJ006-0301|This woman was really his servant, employed to help in cleaning, and taken on whenever there was extra work to be done.|This woman was really his servant, employed to help in cleaning, and taken on whenever there was extra work to be done. +LJ006-0302|The governor had a great dislike, he said, to seeing strangers in his house.|The governor had a great dislike, he said, to seeing strangers in his house. +LJ006-0303|This girl had been first engaged on account of the extra work entailed by certain prisoners|This girl had been first engaged on account of the extra work entailed by certain prisoners +LJ006-0304|committed by the House of Commons, who had been lodged in the governor's own house.|committed by the House of Commons, who had been lodged in the governor's own house. +LJ006-0305|The house at this time was full of men and visitors; waiters came in from the taverns with meals.|The house at this time was full of men and visitors; waiters came in from the taverns with meals. +LJ006-0306|Some of the prisoners had their valets, and all these were constantly in and out of the kitchen where this female prisoner was employed.|Some of the prisoners had their valets, and all these were constantly in and out of the kitchen where this female prisoner was employed. +LJ006-0307|There was reveling and roistering, as usual, with "high life below-stairs.|There was reveling and roistering, as usual, with "high life below-stairs. +LJ006-0308|The governor sent down wine on festive occasions, of which no doubt the prisoner housemaid had her share.|The governor sent down wine on festive occasions, of which no doubt the prisoner housemaid had her share. +LJ007-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 10: The first report of the inspector of prisons.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section ten: The first report of the inspector of prisons. +LJ007-0002|Eighteen years had elapsed since the formation of the "Ladies' Association,"|Eighteen years had elapsed since the formation of the "Ladies' Association," +LJ007-0003|and Mrs. Fry with her colleagues still labored assiduously in Newgate, devoting themselves mainly to the female prison,|and Mrs. Fry with her colleagues still labored assiduously in Newgate, devoting themselves mainly to the female prison, +LJ007-0004|although their ministrations were occasionally extended to the male side.|although their ministrations were occasionally extended to the male side. +LJ007-0005|The inspectors paid tribute to the excellence of the motives of these philanthropic ladies, and recognized the good they did.|The inspectors paid tribute to the excellence of the motives of these philanthropic ladies, and recognized the good they did. +LJ007-0006|They had introduced "much order and cleanliness,"|They had introduced "much order and cleanliness," +LJ007-0007|had provided work for those who had hitherto passed their time in total idleness, and had made the treatment of female transports on the way to New South Wales|had provided work for those who had hitherto passed their time in total idleness, and had made the treatment of female transports on the way to New South Wales +LJ007-0008|their especial care.|their especial care. +LJ007-0009|They had tried, moreover,|They had tried, moreover, +LJ007-0010|by their presence and their pious, disinterested efforts, to restrain the dissolute manners and vicious language of the unhappy and depraved inmates.|by their presence and their pious, disinterested efforts, to restrain the dissolute manners and vicious language of the unhappy and depraved inmates. +LJ007-0011|But it was already plain that they constituted an independent authority within the jails; they were frequently in conflict with the chaplain,|But it was already plain that they constituted an independent authority within the jails; they were frequently in conflict with the chaplain, +LJ007-0012|who not strangely resented the orders issued by the aldermen|who not strangely resented the orders issued by the aldermen +LJ007-0013|that women should be frequently kept from chapel in order that they might attend the ladies' lectures and exhortations.|that women should be frequently kept from chapel in order that they might attend the ladies' lectures and exhortations. +LJ007-0014|The admission of a crowd of visitors to assist in these lay services has already been remarked upon; as the inspectors pointed out,|The admission of a crowd of visitors to assist in these lay services has already been remarked upon; as the inspectors pointed out, +LJ007-0015|it had the bad effect of distracting attention,|it had the bad effect of distracting attention, +LJ007-0016|it tended to "dissipate reflection, diminish the gloom of the prison, and mitigate the punishment which the law has sentenced the prisoner to undergo."|it tended to "dissipate reflection, diminish the gloom of the prison, and mitigate the punishment which the law has sentenced the prisoner to undergo." +LJ007-0017|It is to be feared too that although the surface was thus whitewashed and decorous, much that was vicious still festered and rankled beneath,|It is to be feared too that although the surface was thus whitewashed and decorous, much that was vicious still festered and rankled beneath, +LJ007-0018|and that when the restraining influences of the ladies were absent, the female prisoners relapsed into immoral and uncleanly discourse.|and that when the restraining influences of the ladies were absent, the female prisoners relapsed into immoral and uncleanly discourse. +LJ007-0019|Even in the daytime, when supervision was withdrawn, "the language used to be dreadful," says one of the women when under examination;|Even in the daytime, when supervision was withdrawn, "the language used to be dreadful," says one of the women when under examination; +LJ007-0020|"swearing and talking of what crimes they had committed, and how they had done it." Another witness declared she had heard the most shocking language in the yard; she said|"swearing and talking of what crimes they had committed, and how they had done it." Another witness declared she had heard the most shocking language in the yard; she said +LJ007-0021|"she had never witnessed such scenes before, and hopes she never shall again -- it was dreadful!"|"she had never witnessed such scenes before, and hopes she never shall again -- it was dreadful!" +LJ007-0022|After locking-up time, which varied, as on the male side, according to the daylight, the scenes were often riotous and disgraceful.|After locking-up time, which varied, as on the male side, according to the daylight, the scenes were often riotous and disgraceful. +LJ007-0023|The poor, who could afford no luxuries, went to bed early, but were kept awake by the revelries of the rich,|The poor, who could afford no luxuries, went to bed early, but were kept awake by the revelries of the rich, +LJ007-0024|who supped royally on the supplies provided from outside, and kept it up till ten or eleven o'clock.|who supped royally on the supplies provided from outside, and kept it up till ten or eleven o'clock. +LJ007-0025|There were frequent quarrels and fights; shoes and other missiles were freely bandied about;|There were frequent quarrels and fights; shoes and other missiles were freely bandied about; +LJ007-0026|and with all this "the most dreadful oaths, the worst language, too bad to be repeated," were made use of every night.|and with all this "the most dreadful oaths, the worst language, too bad to be repeated," were made use of every night. +LJ007-0027|Bad as were the various parts of the jail already dealt with,|Bad as were the various parts of the jail already dealt with, +LJ007-0028|there still remained one where the general callous indifference and mismanagement culminated in cruel culpable neglect.|there still remained one where the general callous indifference and mismanagement culminated in cruel culpable neglect. +LJ007-0029|The condition of the capitally-convicted prisoners after sentence was still very disgraceful. The side they occupied, still known as the press-yard,|The condition of the capitally-convicted prisoners after sentence was still very disgraceful. The side they occupied, still known as the press-yard, +LJ007-0030|consisted of two dozen rooms and fifteen cells. In these various chambers, until just before the inspectors made their report,|consisted of two dozen rooms and fifteen cells. In these various chambers, until just before the inspectors made their report, +LJ007-0031|all classes of the condemned, those certain to suffer, and the larger number who were nearly certain of a reprieve,|all classes of the condemned, those certain to suffer, and the larger number who were nearly certain of a reprieve, +LJ007-0032|were jumbled up together, higgledy-piggledy, the old and the young, the murderer and the child who had broken into a dwelling.|were jumbled up together, higgledy-piggledy, the old and the young, the murderer and the child who had broken into a dwelling. +LJ007-0033|All privacy was impossible under the circumstances.|All privacy was impossible under the circumstances. +LJ007-0034|At times the numbers congregated together were very great; as many as fifty and sixty, even more, were crowded indiscriminately into the press-yard.|At times the numbers congregated together were very great; as many as fifty and sixty, even more, were crowded indiscriminately into the press-yard. +LJ007-0035|The better-disposed complained bitterly of what they had to endure;|The better-disposed complained bitterly of what they had to endure; +LJ007-0036|one man declared that the language of the condemned rooms was disgusting, that he was dying a death every day in being compelled to associate with such characters.|one man declared that the language of the condemned rooms was disgusting, that he was dying a death every day in being compelled to associate with such characters. +LJ007-0037|In the midst of the noisy and blasphemous talk no one could pursue his meditations; any who tried to pray became the sport and ridicule of his brutal fellows.|In the midst of the noisy and blasphemous talk no one could pursue his meditations; any who tried to pray became the sport and ridicule of his brutal fellows. +LJ007-0038|Owing to the repeated entreaties of the criminals who could hardly hope to escape the gallows, some show of classification was carried out,|Owing to the repeated entreaties of the criminals who could hardly hope to escape the gallows, some show of classification was carried out, +LJ007-0039|and when the inspectors visited Newgate they found the three certain to die in a day-room by themselves;|and when the inspectors visited Newgate they found the three certain to die in a day-room by themselves; +LJ007-0040|in a second room were fourteen more who had every hope of a reprieve.|in a second room were fourteen more who had every hope of a reprieve. +LJ007-0041|The whole of these seventeen had, however, a common airing-yard, and took their exercise there at the same time,|The whole of these seventeen had, however, a common airing-yard, and took their exercise there at the same time, +LJ007-0042|so that men in the most awful situation, daily expecting to be hanged,|so that men in the most awful situation, daily expecting to be hanged, +LJ007-0043|were associated continually with a number of those who could look with certainty on a mitigation of punishment.|were associated continually with a number of those who could look with certainty on a mitigation of punishment. +LJ007-0044|The latter, light-hearted and reckless, conducted themselves in the most unseemly fashion, and "with as much indifference as the inmates of the other parts of the prison."|The latter, light-hearted and reckless, conducted themselves in the most unseemly fashion, and "with as much indifference as the inmates of the other parts of the prison." +LJ007-0045|They amused themselves after their own fashion; played all day long at blind-man's-buff and leap-frog, or beat each other with a knotted handkerchief,|They amused themselves after their own fashion; played all day long at blind-man's-buff and leap-frog, or beat each other with a knotted handkerchief, +LJ007-0046|laughing and uproarious, utterly unmindful of the companionship of men upon whom lay the shadow of an impending shameful death.|laughing and uproarious, utterly unmindful of the companionship of men upon whom lay the shadow of an impending shameful death. +LJ007-0047|"Men whose cases were dangerous, and those most seriously inclined, complained of these annoyances,"|"Men whose cases were dangerous, and those most seriously inclined, complained of these annoyances," +LJ007-0048|so subversive of meditation, so disturbing to the thoughts;|so subversive of meditation, so disturbing to the thoughts; +LJ007-0049|they suffered sickening anxiety, and wished to be locked up alone. This indiscriminate association lasted for months,|they suffered sickening anxiety, and wished to be locked up alone. This indiscriminate association lasted for months, +LJ007-0050|during the whole of which time the unhappy convicts who had but little hope of commutation were exposed to the mockery of their reckless associates.|during the whole of which time the unhappy convicts who had but little hope of commutation were exposed to the mockery of their reckless associates. +LJ007-0051|The brutal callousness of the bulk of the inmates of the press-yard may be gathered from the prison punishment-book, which frequently recorded such entries as the following:|The brutal callousness of the bulk of the inmates of the press-yard may be gathered from the prison punishment-book, which frequently recorded such entries as the following: +LJ007-0052|Benjamin Vines and Daniel Ward put in irons for two days for breaking the windows of the day room in the condemned cells.|Benjamin Vines and Daniel Ward put in irons for two days for breaking the windows of the day room in the condemned cells. +LJ007-0053|"Joseph Coleman put in irons for three days for striking one of the prisoners," in the same place.|"Joseph Coleman put in irons for three days for striking one of the prisoners," in the same place. +LJ007-0054|There were disputes and quarrels constantly among these doomed men; it was a word and blow, an argument clenched always with a fight.|There were disputes and quarrels constantly among these doomed men; it was a word and blow, an argument clenched always with a fight. +LJ007-0055|The more peaceably disposed found some occupation in making Newgate tokens,|The more peaceably disposed found some occupation in making Newgate tokens, +LJ007-0056|leaden hearts, and "grinding the impressions off penny-pieces, then pricking figures or words on them to give to their friends as memorials.|leaden hearts, and "grinding the impressions off penny-pieces, then pricking figures or words on them to give to their friends as memorials. +LJ007-0057|Turnkeys occasionally visited the press-yard, but its occupants were under little or no control.|Turnkeys occasionally visited the press-yard, but its occupants were under little or no control. +LJ007-0058|The chaplain, who might have been expected to make these men his peculiar care, and who at one time had visited them frequently, often several times a week,|The chaplain, who might have been expected to make these men his peculiar care, and who at one time had visited them frequently, often several times a week, +LJ007-0059|had relaxed his efforts, because, according to his own account, he was so frequently stopped in the performance of his duties.|had relaxed his efforts, because, according to his own account, he was so frequently stopped in the performance of his duties. +LJ007-0060|In his evidence before the inspectors he declared that "for years he gave his whole time to his duties, from an early hour in the morning till late in the afternoon.|In his evidence before the inspectors he declared that "for years he gave his whole time to his duties, from an early hour in the morning till late in the afternoon. +LJ007-0061|He left off because he was so much interfered with and laughed at, and from seeing that no success attended his efforts, owing to the evils arising from association.|He left off because he was so much interfered with and laughed at, and from seeing that no success attended his efforts, owing to the evils arising from association. +LJ007-0062|Latterly his ministrations to the condemned had been restricted to a visit on Sunday afternoons, and occasionally about once a fortnight on a week-day.|Latterly his ministrations to the condemned had been restricted to a visit on Sunday afternoons, and occasionally about once a fortnight on a week-day. +LJ007-0063|It is only fair to Mr. Cotton to add that, according to his own journal, he was unremitting in his attentions to convicts who were actually cast for death,|It is only fair to Mr. Cotton to add that, according to his own journal, he was unremitting in his attentions to convicts who were actually cast for death, +LJ007-0064|and the day of whose execution was fixed. He had no doubt a difficult mission to discharge;|and the day of whose execution was fixed. He had no doubt a difficult mission to discharge; +LJ007-0065|on the one hand, the Ladies' Association, supported and encouraged by public approval, trenched upon his peculiar province;|on the one hand, the Ladies' Association, supported and encouraged by public approval, trenched upon his peculiar province; +LJ007-0066|on the other, the governor of the jail sneered at his zeal|on the other, the governor of the jail sneered at his zeal +LJ007-0067|stigmatized his often most just strictures on abuses as "a bundle of nonsense," and the aldermen, when he appealed to them for protection and countenance,|stigmatized his often most just strictures on abuses as "a bundle of nonsense," and the aldermen, when he appealed to them for protection and countenance, +LJ007-0068|generally sided with his opponents. Nevertheless the inspectors summed up against him.|generally sided with his opponents. Nevertheless the inspectors summed up against him. +LJ007-0069|While admitting that he had had many difficulties to contend with,|While admitting that he had had many difficulties to contend with, +LJ007-0070|and that he had again and again protested against the obstacles thrown in his way, the inspectors "cannot forbear expressing their opinion that he might have shown greater perseverance,|and that he had again and again protested against the obstacles thrown in his way, the inspectors "cannot forbear expressing their opinion that he might have shown greater perseverance, +LJ007-0071|in the face of impediments confessedly discouraging|in the face of impediments confessedly discouraging +LJ007-0072|as regards the private teaching of prisoners; and they went on to say that "a resolved adherence, in spite of discouragements the most disheartening,|as regards the private teaching of prisoners; and they went on to say that "a resolved adherence, in spite of discouragements the most disheartening, +LJ007-0073|to that line of conduct which his duty imposed on him|to that line of conduct which his duty imposed on him +LJ007-0074|would, it is probable, have eventually overcome the reluctance of some of the prisoners at least, and would have possessed so much moral dignity|would, it is probable, have eventually overcome the reluctance of some of the prisoners at least, and would have possessed so much moral dignity +LJ007-0075|as effectually to rebuke and abash the profane spirit of the more insolent and daring of the criminals.|as effectually to rebuke and abash the profane spirit of the more insolent and daring of the criminals. +LJ007-0076|The lax discipline maintained in Newgate was still further deteriorated by the presence of two other classes of prisoners who ought never to have been inmates of such a jail.|The lax discipline maintained in Newgate was still further deteriorated by the presence of two other classes of prisoners who ought never to have been inmates of such a jail. +LJ007-0077|One of these were the criminal lunatics, who were at this time and for long previous continuously imprisoned there.|One of these were the criminal lunatics, who were at this time and for long previous continuously imprisoned there. +LJ007-0078|As the law stood since the passing of the 9th George IV. c. 40, any two justices might remove a prisoner found to be insane, either on commitment|As the law stood since the passing of the ninth George the fourth c. forty, any two justices might remove a prisoner found to be insane, either on commitment +LJ007-0079|or arraignment, to an asylum, and the Secretary of State had the same power as regards any who became insane while undergoing sentence.|or arraignment, to an asylum, and the Secretary of State had the same power as regards any who became insane while undergoing sentence. +LJ007-0080|These powers were not invariably put in force, and there were in consequence many unhappy lunatics in Newgate and other jails,|These powers were not invariably put in force, and there were in consequence many unhappy lunatics in Newgate and other jails, +LJ007-0081|whose proper place was the asylum.|whose proper place was the asylum. +LJ007-0082|At the time the Lords' Committee sat there were eight thus retained in Newgate, and a return in the appendix of the Lords' report|At the time the Lords' Committee sat there were eight thus retained in Newgate, and a return in the appendix of the Lords' report +LJ007-0083|gives a total of thirty-nine lunatics confined in various jails, many of them guilty of murder and other serious crimes.|gives a total of thirty-nine lunatics confined in various jails, many of them guilty of murder and other serious crimes. +LJ007-0084|The inspectors in the following year, on examining the facts, found that some of these poor creatures had been in confinement for long periods:|The inspectors in the following year, on examining the facts, found that some of these poor creatures had been in confinement for long periods: +LJ007-0085|at Newgate and York Castle as long as five years; "at Ilchester and Morpeth for seven years; at Warwick for eight years,|at Newgate and York Castle as long as five years; "at Ilchester and Morpeth for seven years; at Warwick for eight years, +LJ007-0086|at Buckingham and Hereford for eleven years|at Buckingham and Hereford for eleven years +LJ007-0087|at Appleby for thirteen years, at Anglesea for fifteen years, at Exeter for sixteen years, and at Pembroke|at Appleby for thirteen years, at Anglesea for fifteen years, at Exeter for sixteen years, and at Pembroke +LJ007-0088|for not less a period than twenty-four years.|for not less a period than twenty-four years. +LJ007-0089|It was manifestly wrong that such persons, "visited by the most awful of calamities," should be detained in a common prison.|It was manifestly wrong that such persons, "visited by the most awful of calamities," should be detained in a common prison. +LJ007-0090|Not only did their presence tend greatly to interfere with the discipline of the prison, but their condition was deplorable in the extreme.|Not only did their presence tend greatly to interfere with the discipline of the prison, but their condition was deplorable in the extreme. +LJ007-0091|The lunatic became the sport of the idle and the depraved. His cure was out of the question;|The lunatic became the sport of the idle and the depraved. His cure was out of the question; +LJ007-0092|he was placed in a situation "beyond all others calculated to confirm his malady and prolong his sufferings."|he was placed in a situation "beyond all others calculated to confirm his malady and prolong his sufferings." +LJ007-0093|The matter was still further complicated at Newgate by the presence within the walls of sham lunatics. Some of those included in the category|The matter was still further complicated at Newgate by the presence within the walls of sham lunatics. Some of those included in the category +LJ007-0094|had actually been returned as sane from the asylum to which they had been sent, and there was always some uncertainty as to who was mad and who not.|had actually been returned as sane from the asylum to which they had been sent, and there was always some uncertainty as to who was mad and who not. +LJ007-0095|Prisoners indeed were known to boast that they had saved their necks by feigning insanity.|Prisoners indeed were known to boast that they had saved their necks by feigning insanity. +LJ007-0096|It was high time that the unsatisfactory state of the law as regards the treatment of criminal lunatics should be remedied|It was high time that the unsatisfactory state of the law as regards the treatment of criminal lunatics should be remedied +LJ007-0097|and not the least of the good services rendered by the new inspectors was their inquiry into the status of these unfortunate people, and their recommendation to improve it.|and not the least of the good services rendered by the new inspectors was their inquiry into the status of these unfortunate people, and their recommendation to improve it. +LJ007-0098|The other inmates of the prison of an exceptional character, and exempted from the regular discipline, such as it was,|The other inmates of the prison of an exceptional character, and exempted from the regular discipline, such as it was, +LJ007-0099|were the ten persons committed to Newgate by the House of Commons in 1835.|were the ten persons committed to Newgate by the House of Commons in eighteen thirty-five. +LJ007-0100|These were the gentlemen concerned in the bribery case at Ipswich in 1835,|These were the gentlemen concerned in the bribery case at Ipswich in eighteen thirty-five, +LJ007-0101|when a petition was presented against the return of Messrs. Adam Dundas and Fitzroy Kelly. Various witnesses, including Messrs. J. B. Dasent,|when a petition was presented against the return of Messrs. Adam Dundas and Fitzroy Kelly. Various witnesses, including Messrs. J. B. Dasent, +LJ007-0102|Pilgrim, Bond, and Clamp, had refused to give evidence before the House of Commons' Committee; a Speaker's warrant was issued for their arrest when they absconded.|Pilgrim, Bond, and Clamp, had refused to give evidence before the House of Commons' Committee; a Speaker's warrant was issued for their arrest when they absconded. +LJ007-0103|Mr. J. E. Sparrow and Mr. Clipperton|Mr. J. E. Sparrow and Mr. Clipperton +LJ007-0104|the parliamentary agents of the members whose election was impugned, were implicated in aiding and abetting the others to abscond, and a Mr. O'Mally,|the parliamentary agents of the members whose election was impugned, were implicated in aiding and abetting the others to abscond, and a Mr. O'Mally, +LJ007-0105|counsel for the two M.P.'s, was also concerned.|counsel for the two M.P.'s, was also concerned. +LJ007-0106|Pilgrim and Dasent were caught and given into the custody of the sergeant-at-arms, and the rest were either arrested or they surrendered.|Pilgrim and Dasent were caught and given into the custody of the sergeant-at-arms, and the rest were either arrested or they surrendered. +LJ007-0107|A resolution at once passed the House without division to commit the whole to Newgate, where they remained for various terms.|A resolution at once passed the House without division to commit the whole to Newgate, where they remained for various terms. +LJ007-0108|Dasent and Pilgrim were released in ten days, on making due submission.|Dasent and Pilgrim were released in ten days, on making due submission. +LJ007-0109|O'Mally sent in a medical certificate, declaring that the imprisonment was endangering his life, and after some question he was also released.|O'Mally sent in a medical certificate, declaring that the imprisonment was endangering his life, and after some question he was also released. +LJ007-0110|The rest were detained for more than a month, it being considered that they were the most guilty, as being either professional agents, who advised the others to abscond,|The rest were detained for more than a month, it being considered that they were the most guilty, as being either professional agents, who advised the others to abscond, +LJ007-0111|or witnesses who did not voluntarily come forward when the chance was given them.|or witnesses who did not voluntarily come forward when the chance was given them. +LJ007-0112|Many of the old customs once prevalent in the State Side, so properly condemned and abolished,|Many of the old customs once prevalent in the State Side, so properly condemned and abolished, +LJ007-0113|were revived for the convenience of these gentlemen, whose incarceration was thus rendered as little like imprisonment as possible.|were revived for the convenience of these gentlemen, whose incarceration was thus rendered as little like imprisonment as possible. +LJ007-0114|A certain number, who could afford the high rate of a guinea per diem, fixed by the under sheriff, were lodged in the governor's house,|A certain number, who could afford the high rate of a guinea per diem, fixed by the under sheriff, were lodged in the governor's house, +LJ007-0115|slept there, and had their meals provided for them from the Sessions House or London Coffee-House.|slept there, and had their meals provided for them from the Sessions House or London Coffee-House. +LJ007-0116|A few others, who could not afford a payment of more than half a guinea, were permitted to monopolize a part of the prison infirmary,|A few others, who could not afford a payment of more than half a guinea, were permitted to monopolize a part of the prison infirmary, +LJ007-0117|where the upper ward was exclusively appropriated to their use. They also had their meals sent in, and, with the food, wine almost ad libitum.|where the upper ward was exclusively appropriated to their use. They also had their meals sent in, and, with the food, wine almost ad libitum. +LJ007-0118|A prisoner, one of the wardsmen, waited on those in the infirmary; the occupants of the governor's house had their own servants, or the governor's.|A prisoner, one of the wardsmen, waited on those in the infirmary; the occupants of the governor's house had their own servants, or the governor's. +LJ007-0119|As a rule, visitors, many of them persons of good position, came and went all day long, and as late as nine at night;|As a rule, visitors, many of them persons of good position, came and went all day long, and as late as nine at night; +LJ007-0120|some to the infirmary, many more to the governor's house.|some to the infirmary, many more to the governor's house. +LJ007-0121|There were no restraints, cards and backgammon were played, and the time passed in feasting and revelry.|There were no restraints, cards and backgammon were played, and the time passed in feasting and revelry. +LJ007-0122|Even Mr. Cope admitted that the committal of this class of prisoners to Newgate was most inconvenient,|Even Mr. Cope admitted that the committal of this class of prisoners to Newgate was most inconvenient, +LJ007-0123|and the inspectors expressed themselves still more strongly in reprehension of the practice.|and the inspectors expressed themselves still more strongly in reprehension of the practice. +LJ007-0124|The infirmary at this particular period epitomized the condition of the jail at large.|The infirmary at this particular period epitomized the condition of the jail at large. +LJ007-0125|It was diverted from its proper uses, and, as the "place of the greatest comfort," was allotted to persons who should not have been sent to Newgate at all.|It was diverted from its proper uses, and, as the "place of the greatest comfort," was allotted to persons who should not have been sent to Newgate at all. +LJ007-0126|All the evils of indiscriminate association were strongly accentuated by the crowd collected within its narrow limits.|All the evils of indiscriminate association were strongly accentuated by the crowd collected within its narrow limits. +LJ007-0127|"It may easily be imagined," say the inspectors, in speaking of the prison generally,|"It may easily be imagined," say the inspectors, in speaking of the prison generally, +LJ007-0128|"what must be the state of discipline in a place filled with characters so various as were assembled there, where the tried and the untried, the sick and the healthy,|"what must be the state of discipline in a place filled with characters so various as were assembled there, where the tried and the untried, the sick and the healthy, +LJ007-0129|the sane and the insane, the young and the old, the trivial offender and the man about to suffer the extreme penalty of the law,|the sane and the insane, the young and the old, the trivial offender and the man about to suffer the extreme penalty of the law, +LJ007-0130|are all huddled together without discrimination, oversight, or control."|are all huddled together without discrimination, oversight, or control." +LJ007-0131|Enough has probably been extracted from this most damnatory report to give a complete picture of the disgraceful state in which Newgate still remained in 1835.|Enough has probably been extracted from this most damnatory report to give a complete picture of the disgraceful state in which Newgate still remained in eighteen thirty-five. +LJ007-0132|The inspectors, however, honestly admitted that although the site of the prison was convenient, its construction was as bad as bad could be.|The inspectors, however, honestly admitted that although the site of the prison was convenient, its construction was as bad as bad could be. +LJ007-0133|Valuable space was cumbered with many long and winding passages, numerous staircases, and unnecessarily thick and cumbrous inner walls.|Valuable space was cumbered with many long and winding passages, numerous staircases, and unnecessarily thick and cumbrous inner walls. +LJ007-0134|The wards were in some cases spacious, but they were entirely unsuited for separation or the inspection of prisoners.|The wards were in some cases spacious, but they were entirely unsuited for separation or the inspection of prisoners. +LJ007-0135|The yards were narrow and confined, mainly because the ground plan was radically vicious. These were evils inseparable from the place.|The yards were narrow and confined, mainly because the ground plan was radically vicious. These were evils inseparable from the place. +LJ007-0136|But there were others remediable under a better system of management.|But there were others remediable under a better system of management. +LJ007-0137|More attention to ventilation, which was altogether neglected and inadequate, would have secured a better atmosphere for the unhappy inmates|More attention to ventilation, which was altogether neglected and inadequate, would have secured a better atmosphere for the unhappy inmates +LJ007-0138|who constantly breathed an air heavy, and, when the wards were first opened in the morning, particularly offensive.|who constantly breathed an air heavy, and, when the wards were first opened in the morning, particularly offensive. +LJ007-0139|Again, the discipline commonly deemed inseparable from every place of durance was entirely wanting.|Again, the discipline commonly deemed inseparable from every place of durance was entirely wanting. +LJ007-0140|The primary object of committing a prisoner to jail, as the inspectors pointed out, was to deter not only the criminal himself, but others from crime,|The primary object of committing a prisoner to jail, as the inspectors pointed out, was to deter not only the criminal himself, but others from crime, +LJ007-0141|and "to dispose him, by meditation and seclusion, to return to an honest life."|and "to dispose him, by meditation and seclusion, to return to an honest life." +LJ007-0142|But at Newgate the convicted prisoner, instead of privation and hard fare,|But at Newgate the convicted prisoner, instead of privation and hard fare, +LJ007-0143|"is permitted to purchase whatever his own means or the means of his friends in or out of prison can afford,|"is permitted to purchase whatever his own means or the means of his friends in or out of prison can afford, +LJ007-0144|and he can almost invariably procure the luxuries of his class of life, beer and tobacco, in abundance.|and he can almost invariably procure the luxuries of his class of life, beer and tobacco, in abundance. +LJ007-0145|Instead of seclusion and meditation, his time is passed in the midst of a body of criminals of every class and degree, in riot, debauchery, and gaming,|Instead of seclusion and meditation, his time is passed in the midst of a body of criminals of every class and degree, in riot, debauchery, and gaming, +LJ007-0146|vaunting his own adventures, or listening to those of others;|vaunting his own adventures, or listening to those of others; +LJ007-0147|communicating his own skill and aptitude in crime, or acquiring the lessons of greater adepts. He has access to newspapers, and of course|communicating his own skill and aptitude in crime, or acquiring the lessons of greater adepts. He has access to newspapers, and of course +LJ007-0148|prefers that description which are expressly prepared for his own class, and which abound in vulgar adventure in criminal enterprise, and in the histories of the police,|prefers that description which are expressly prepared for his own class, and which abound in vulgar adventure in criminal enterprise, and in the histories of the police, +LJ007-0149|the jail, and the scaffold.|the jail, and the scaffold. +LJ007-0150|He is allowed intercourse with prostitutes who, in nine cases out of ten, have originally conduced to his ruin;|He is allowed intercourse with prostitutes who, in nine cases out of ten, have originally conduced to his ruin; +LJ007-0151|and his connection with them is confirmed by that devotion and generosity towards their paramours in adversity for which these otherwise degraded women are remarkable.|and his connection with them is confirmed by that devotion and generosity towards their paramours in adversity for which these otherwise degraded women are remarkable. +LJ007-0152|Having thus passed his time, he returns a greater adept in crime, with a wider acquaintance among criminals, and, what perhaps is even more injurious to him,|Having thus passed his time, he returns a greater adept in crime, with a wider acquaintance among criminals, and, what perhaps is even more injurious to him, +LJ007-0153|is generally known to all the worst men in the country; not only without the inclination, but almost without the ability of returning to an honest life."|is generally known to all the worst men in the country; not only without the inclination, but almost without the ability of returning to an honest life." +LJ007-0154|These pungent and well-grounded strictures applied with still greater force to the unconvicted prisoner, the man who came to the prison innocent, and still uncontaminated,|These pungent and well-grounded strictures applied with still greater force to the unconvicted prisoner, the man who came to the prison innocent, and still uncontaminated, +LJ007-0155|to be subjected to the same baneful influences, and to suffer the same moral deterioration, whether ultimately convicted or set free.|to be subjected to the same baneful influences, and to suffer the same moral deterioration, whether ultimately convicted or set free. +LJ007-0156|The whole system, or more correctly the want of system, was baneful and pernicious to the last degree.|The whole system, or more correctly the want of system, was baneful and pernicious to the last degree. +LJ007-0157|The evils of such association were aggravated by the unbroken idleness; one "evil inflamed the other;" reformation|The evils of such association were aggravated by the unbroken idleness; one "evil inflamed the other;" reformation +LJ007-0158|or any kind of moral improvement was impossible; the prisoner's career was inevitably downward, till he struck the lowest depths.|or any kind of moral improvement was impossible; the prisoner's career was inevitably downward, till he struck the lowest depths. +LJ007-0159|Forced and constant intercourse with the most depraved individuals of his own class;|Forced and constant intercourse with the most depraved individuals of his own class; +LJ007-0160|the employment of those means and agents by which the lowest passions and the most vulgar propensities of man are perpetually kept in the highest state of excitement|the employment of those means and agents by which the lowest passions and the most vulgar propensities of man are perpetually kept in the highest state of excitement +LJ007-0161|drink, gaming, obscene and blasphemous language; utter idleness, the almost unrestricted admission of money and luxuries;|drink, gaming, obscene and blasphemous language; utter idleness, the almost unrestricted admission of money and luxuries; +LJ007-0162|uncontrolled conversation with visitors of the very worst description -- prostitutes, thieves, receivers of stolen goods|uncontrolled conversation with visitors of the very worst description -- prostitutes, thieves, receivers of stolen goods +LJ007-0163|all the tumultuous and diversified passions and emotions which circumstances like these must necessarily generate|all the tumultuous and diversified passions and emotions which circumstances like these must necessarily generate +LJ007-0164|forbid the faintest shadow of a hope that in a soil so unfavorable for moral culture|forbid the faintest shadow of a hope that in a soil so unfavorable for moral culture +LJ007-0165|any awakening truth, salutary exhortation, or imperfect resolutions of amendment can take root or grow.|any awakening truth, salutary exhortation, or imperfect resolutions of amendment can take root or grow. +LJ007-0166|Strong as were the foregoing remarks, the inspectors wound up their report in still more trenchant language|Strong as were the foregoing remarks, the inspectors wound up their report in still more trenchant language +LJ007-0167|framing a terrible indictment against those responsible for the condition of Newgate. Their words deserve to be quoted in full.|framing a terrible indictment against those responsible for the condition of Newgate. Their words deserve to be quoted in full. +LJ007-0168|"We cannot close these remarks," say the inspectors, "without an expression of the painful feelings with which we submit to your Lordship|"We cannot close these remarks," say the inspectors, "without an expression of the painful feelings with which we submit to your Lordship +LJ007-0169|this picture of the existing state of Newgate|this picture of the existing state of Newgate +LJ007-0170|That in this vast metropolis, the center of wealth, civilization, and information;|That in this vast metropolis, the center of wealth, civilization, and information; +LJ007-0171|distinguished as the seat of religion, worth, and philanthropy,|distinguished as the seat of religion, worth, and philanthropy, +LJ007-0172|where is to be found in operation every expedient by which Ignorance may be superseded by Knowledge, Idleness by Industry, and Suffering by Benevolence;|where is to be found in operation every expedient by which Ignorance may be superseded by Knowledge, Idleness by Industry, and Suffering by Benevolence; +LJ007-0173|that in the metropolis of this highly-favored country, to which the eyes of other lands turn for example, a system of prison discipline such as that enforced in Newgate|that in the metropolis of this highly-favored country, to which the eyes of other lands turn for example, a system of prison discipline such as that enforced in Newgate +LJ007-0174|should be for a number of years in undisturbed operation, not only in contempt of religion and humanity,|should be for a number of years in undisturbed operation, not only in contempt of religion and humanity, +LJ007-0175|but in opposition to the recorded denunciations of authority, and in defiance of the express enactments of the law,|but in opposition to the recorded denunciations of authority, and in defiance of the express enactments of the law, +LJ007-0176|is indeed a subject which cannot but impress every considerate mind with humiliation and sorrow.|is indeed a subject which cannot but impress every considerate mind with humiliation and sorrow. +LJ007-0177|We trust, however, that the day is at hand when this stain will be removed from the character of the city of London,|We trust, however, that the day is at hand when this stain will be removed from the character of the city of London, +LJ007-0178|and when the first municipal authority of our land will be no longer subjected to the reproach of fostering an institution which outrages the rights and feelings of humanity,|and when the first municipal authority of our land will be no longer subjected to the reproach of fostering an institution which outrages the rights and feelings of humanity, +LJ007-0179|defeats the ends of justice, and disgraces the profession of a Christian country.|defeats the ends of justice, and disgraces the profession of a Christian country. +LJ007-0180|The publication of this report raised a storm in the city, and the corporation was roused to make an immediate protest.|The publication of this report raised a storm in the city, and the corporation was roused to make an immediate protest. +LJ007-0181|A committee of aldermen was forthwith appointed to report upon the inspectors' report,|A committee of aldermen was forthwith appointed to report upon the inspectors' report, +LJ007-0182|and the result was another lengthy blue book, printed in the parliamentary papers, 1836,|and the result was another lengthy blue book, printed in the parliamentary papers, eighteen thirty-six, +LJ007-0183|traversing where it was possible the statements of the inspectors, and offering explanation and palliation of such evils as could not be denied.|traversing where it was possible the statements of the inspectors, and offering explanation and palliation of such evils as could not be denied. +LJ007-0184|The inspectors retorted without loss of time, reiterating their charges, and pointing out that the committee of aldermen by its own admission|The inspectors retorted without loss of time, reiterating their charges, and pointing out that the committee of aldermen by its own admission +LJ007-0185|justified the original allegations. It was impossible to deny the indiscriminate association; the gambling, drinking, smoking, quarreling in the jail;|justified the original allegations. It was impossible to deny the indiscriminate association; the gambling, drinking, smoking, quarreling in the jail; +LJ007-0186|the undue authority given to prisoners, the levying of garnish under another name|the undue authority given to prisoners, the levying of garnish under another name +LJ007-0187|the neglect of the condemned convicts, the filthy condition of the wards, the insufficiency of bedding and clothing,|the neglect of the condemned convicts, the filthy condition of the wards, the insufficiency of bedding and clothing, +LJ007-0188|the misemployment of officers and prisoners by the governor.|the misemployment of officers and prisoners by the governor. +LJ007-0189|The corporation evidently had the worst of it, and began to feel the necessity for undertaking the great work of reform.|The corporation evidently had the worst of it, and began to feel the necessity for undertaking the great work of reform. +LJ007-0190|Next year we find the inspectors expressing their satisfaction that "the full and faithful exposure which we felt it our duty to make of Newgate|Next year we find the inspectors expressing their satisfaction that "the full and faithful exposure which we felt it our duty to make of Newgate +LJ007-0191|has been productive of at least some advantage,|has been productive of at least some advantage, +LJ007-0192|inasmuch as it has aroused the attention of those upon whom parliamentary reports and grand jury presentments had hitherto failed to make the slightest impression.|inasmuch as it has aroused the attention of those upon whom parliamentary reports and grand jury presentments had hitherto failed to make the slightest impression. +LJ007-0193|The measures of improvement introduced were mainly as follows:|The measures of improvement introduced were mainly as follows: +LJ007-0194|the fixing of "inspection holes" in the doors and walls, so as to insure more supervision; of windows opening into the well-holes,|the fixing of "inspection holes" in the doors and walls, so as to insure more supervision; of windows opening into the well-holes, +LJ007-0195|to give better light and ventilation; the construction of bed-places, three tiers high alongside the walls for males, two tiers for females;|to give better light and ventilation; the construction of bed-places, three tiers high alongside the walls for males, two tiers for females; +LJ007-0196|the provision of dining-rooms and dining-tables.|the provision of dining-rooms and dining-tables. +LJ007-0197|The infirmary was enlarged, the admission of visitors limited, and the passing of articles prevented by a wire screen.|The infirmary was enlarged, the admission of visitors limited, and the passing of articles prevented by a wire screen. +LJ007-0198|The windows were to be glazed and painted to prevent prisoners from looking out;|The windows were to be glazed and painted to prevent prisoners from looking out; +LJ007-0199|baths, fumigating places for clothing, wash-house, and the removal of dust-bins, completed the new arrangements in the main prison.|baths, fumigating places for clothing, wash-house, and the removal of dust-bins, completed the new arrangements in the main prison. +LJ007-0200|In the press-yard, the press-room and ward above it were parceled out into nine separate sleeping cells;|In the press-yard, the press-room and ward above it were parceled out into nine separate sleeping cells; +LJ007-0201|each was provided with an iron bedstead, and a small desk at which the condemned man might read or write.|each was provided with an iron bedstead, and a small desk at which the condemned man might read or write. +LJ007-0202|But the one great and most crying evil remained unremedied.|But the one great and most crying evil remained unremedied. +LJ007-0203|"The mischief of jail associations," say the inspectors,|"The mischief of jail associations," say the inspectors, +LJ007-0204|"which has been demonstrably proved to be the fruitful source of all the abuses and irregularities which have so long disgraced Newgate,|"which has been demonstrably proved to be the fruitful source of all the abuses and irregularities which have so long disgraced Newgate, +LJ007-0205|is not only permitted still to exist in the prison, but is rendered more powerful than before.|is not only permitted still to exist in the prison, but is rendered more powerful than before. +LJ007-0206|In endeavoring to arrest contamination, prisoners were more closely confined, and associated in smaller numbers;|In endeavoring to arrest contamination, prisoners were more closely confined, and associated in smaller numbers; +LJ007-0207|but this had the effect of throwing them into closer contact, and of making them more intimately acquainted with, more directly influential upon, one another.|but this had the effect of throwing them into closer contact, and of making them more intimately acquainted with, more directly influential upon, one another. +LJ007-0208|In the inspectors' fourth report, dated 1839,|In the inspectors' fourth report, dated eighteen thirty-nine, +LJ007-0209|they return to the charge, and again call the corporation to task for their mismanagement of Newgate.|they return to the charge, and again call the corporation to task for their mismanagement of Newgate. +LJ007-0210|Abuses and irregularities, which had been partially remedied by the reform introduced in 1837, were once more in the ascendant.|Abuses and irregularities, which had been partially remedied by the reform introduced in eighteen thirty-seven, were once more in the ascendant. +LJ007-0211|"In our late visits," they say, "we have seen manifest indications of a retrograde movement in this respect,|"In our late visits," they say, "we have seen manifest indications of a retrograde movement in this respect, +LJ007-0212|and a tendency to return to much of that laxity and remissness which formerly marked the management of this prison."|and a tendency to return to much of that laxity and remissness which formerly marked the management of this prison." +LJ007-0213|Again the following year the inspectors repeat their charge.|Again the following year the inspectors repeat their charge. +LJ007-0214|"The prominent evils of this prison (Newgate) -- evils which the alterations made within the last four years have failed to remove|"The prominent evils of this prison (Newgate) -- evils which the alterations made within the last four years have failed to remove +LJ007-0215|are the association of prisoners, and the unusual contamination to which such association gives rise.|are the association of prisoners, and the unusual contamination to which such association gives rise. +LJ007-0216|For nearly twenty-two hours out of the twenty-four the prisoners are locked up, during which time no officer is stationed in the ward with them.|For nearly twenty-two hours out of the twenty-four the prisoners are locked up, during which time no officer is stationed in the ward with them. +LJ007-0218|Newgate is only less extensively injurious than formerly because it is less crowded.|Newgate is only less extensively injurious than formerly because it is less crowded. +LJ007-0219|The effects of the imprisonment are to vitiate its inmates, to extend their acquaintanceship with each other,|The effects of the imprisonment are to vitiate its inmates, to extend their acquaintanceship with each other, +LJ007-0220|to corrupt the prisoner charged with an offense of which he may be innocent, and to confirm in guilt the young and inexperienced offender.|to corrupt the prisoner charged with an offense of which he may be innocent, and to confirm in guilt the young and inexperienced offender. +LJ007-0221|The reports as the years flow on reiterate the same complaints.|The reports as the years flow on reiterate the same complaints. +LJ007-0222|Much bitterness of feeling is evidently engendered, and the corporation grows more and more angry with the inspectors.|Much bitterness of feeling is evidently engendered, and the corporation grows more and more angry with the inspectors. +LJ007-0223|The prison officials appear to be on the side of the inspectors, to the great dissatisfaction of the corporation, who claimed the full allegiance and support of its servants.|The prison officials appear to be on the side of the inspectors, to the great dissatisfaction of the corporation, who claimed the full allegiance and support of its servants. +LJ007-0224|In a resolution passed by the Court of Aldermen on 18th March, 1842,|In a resolution passed by the Court of Aldermen on eighteenth March, eighteen forty-two, +LJ007-0225|I find it ordered "that the ordinary of Newgate be restricted from making any communications to the Home Office|I find it ordered "that the ordinary of Newgate be restricted from making any communications to the Home Office +LJ007-0226|or the Inspectors of Prisons, and that he be required wholly to confine himself to the performance of his duty as prescribed by Act of Parliament.|or the Inspectors of Prisons, and that he be required wholly to confine himself to the performance of his duty as prescribed by Act of Parliament. +LJ007-0227|The inspectors were not to be deterred, however, by any opposition from the earnest discharge of their functions, and continued to report against Newgate.|The inspectors were not to be deterred, however, by any opposition from the earnest discharge of their functions, and continued to report against Newgate. +LJ007-0228|In their tenth report|In their tenth report +LJ007-0229|they state that they are compelled by an imperative sense of duty to advert in terms of decided condemnation to the lamentable condition of the prisons of the city of London,|they state that they are compelled by an imperative sense of duty to advert in terms of decided condemnation to the lamentable condition of the prisons of the city of London, +LJ007-0230|Newgate, Giltspur St. Compter, and the City Bridewell,|Newgate, Giltspur St. Compter, and the City Bridewell, +LJ007-0231|in which the master evil of jail association and consequent contamination still continues to operate directly to the encouragement of crime.|in which the master evil of jail association and consequent contamination still continues to operate directly to the encouragement of crime. +LJ007-0232|The plan adopted for ventilating the dining-room on the 'master's side' and that of the middle yard is very inefficient;|The plan adopted for ventilating the dining-room on the 'master's side' and that of the middle yard is very inefficient; +LJ007-0233|it consists of several circular perforations, about two inches in diameter,|it consists of several circular perforations, about two inches in diameter, +LJ007-0234|slanting downwards from the top of the walls to the outside adjoining the slaughterhouses of Newgate market; and occasionally, in hot weather,|slanting downwards from the top of the walls to the outside adjoining the slaughterhouses of Newgate market; and occasionally, in hot weather, +LJ007-0235|instead of ventilating the apartments, they only serve to convey the offensive effluvia arising from the decaying animal matter into the dining-rooms.|instead of ventilating the apartments, they only serve to convey the offensive effluvia arising from the decaying animal matter into the dining-rooms. +LJ007-0236|Sometimes the stench in hot weather is said to be very bad.|Sometimes the stench in hot weather is said to be very bad. +LJ007-0237|Many rats also come through these so-called ventilators, as they open close to the ground at the back of the prison.|Many rats also come through these so-called ventilators, as they open close to the ground at the back of the prison. +LJ007-0238|At the same time the inspectors animadvert strongly upon the misconduct of prisoners and the frequency of prison punishments,|At the same time the inspectors animadvert strongly upon the misconduct of prisoners and the frequency of prison punishments, +LJ007-0239|both offenses and punishments affording a sufficient index to the practices going forward; and they wind up by declaring|both offenses and punishments affording a sufficient index to the practices going forward; and they wind up by declaring +LJ007-0240|that a strict compliance with their duties gave them no choice "but to report matters as we found them,|that a strict compliance with their duties gave them no choice "but to report matters as we found them, +LJ007-0241|and again and again to protest against Newgate as it at present exists.|and again and again to protest against Newgate as it at present exists. +LJ007-0242|No complete and permanent improvement was indeed possible while Newgate remained unchanged.|No complete and permanent improvement was indeed possible while Newgate remained unchanged. +LJ008-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 11: Executions, part one.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section eleven: Executions, part one. +LJ008-0002|I propose to return now to the subject of Newgate executions,|I propose to return now to the subject of Newgate executions, +LJ008-0003|which we left at the time of the discontinuance of the long-practiced procession to Tyburn.|which we left at the time of the discontinuance of the long-practiced procession to Tyburn. +LJ008-0004|The reasons for this change were fully set forth in a previous chapter.|The reasons for this change were fully set forth in a previous chapter. +LJ008-0005|The terrible spectacle was as demoralizing to the public, for whose admonition it was intended,|The terrible spectacle was as demoralizing to the public, for whose admonition it was intended, +LJ008-0006|as the exposure was brutal and cruel towards the principal actors.|as the exposure was brutal and cruel towards the principal actors. +LJ008-0007|The decision to remove the scene of action to the immediate front of the jail itself|The decision to remove the scene of action to the immediate front of the jail itself +LJ008-0008|was in the right direction, as making the performance shorter and diminishing the area of display.|was in the right direction, as making the performance shorter and diminishing the area of display. +LJ008-0009|But the Old Bailey was not exclusively used;|But the Old Bailey was not exclusively used; +LJ008-0010|at first, and for some few years after 1784, executions took place occasionally at a distance from Newgate.|at first, and for some few years after seventeen eighty-four, executions took place occasionally at a distance from Newgate. +LJ008-0011|This was partly due to the survival of the old notion that the scene of the crime ought also to witness the retribution;|This was partly due to the survival of the old notion that the scene of the crime ought also to witness the retribution; +LJ008-0012|partly perhaps because residents in and about the Old Bailey raised a loud protest against the constant erection of the scaffold in their neighborhood.|partly perhaps because residents in and about the Old Bailey raised a loud protest against the constant erection of the scaffold in their neighborhood. +LJ008-0013|As regards the first, I find that in 1786|As regards the first, I find that in seventeen eighty-six +LJ008-0014|John Hogan, the murderer of a Mr. Odell, an attorney who resided in Charlotte Street, Rathbone Place,|John Hogan, the murderer of a Mr. Odell, an attorney who resided in Charlotte Street, Rathbone Place, +LJ008-0015|was executed on a gibbet in front of his victim's house.|was executed on a gibbet in front of his victim's house. +LJ008-0016|Lawrence Jones, a burglar, was in 1793 ordered for execution in Hatton Garden, near the house he had robbed;|Lawrence Jones, a burglar, was in seventeen ninety-three ordered for execution in Hatton Garden, near the house he had robbed; +LJ008-0017|and when he evaded the sentence by suicide, his body was exhibited in the same neighborhood,|and when he evaded the sentence by suicide, his body was exhibited in the same neighborhood, +LJ008-0018|extended upon a plank on the top of an open cart, in his clothes, and fettered.|extended upon a plank on the top of an open cart, in his clothes, and fettered. +LJ008-0019|Again, as late as 1809 and 1812, Execution Dock, on the banks of the Thames, was still retained.|Again, as late as eighteen oh nine and eighteen twelve, Execution Dock, on the banks of the Thames, was still retained. +LJ008-0020|Here John Sutherland, commander of the British armed transport 'The Friends,' suffered on the 29th June, 1809,|Here John Sutherland, commander of the British armed transport 'The Friends,' suffered on the twenty-ninth June, eighteen oh nine, +LJ008-0021|for the murder of his cabin-boy, whom he stabbed after much ill-usage on board the ship as it lay in the Tagus.|for the murder of his cabin-boy, whom he stabbed after much ill-usage on board the ship as it lay in the Tagus. +LJ008-0022|On the 18th December, 1812,|On the eighteenth December, eighteen twelve, +LJ008-0023|two sailors, Charles Palm and Sam Tilling, were hanged at the same place for the murder of their captain, James Keith|two sailors, Charles Palm and Sam Tilling, were hanged at the same place for the murder of their captain, James Keith +LJ008-0024|of the trading vessel 'Adventure,' upon the high seas.|of the trading vessel 'Adventure,' upon the high seas. +LJ008-0025|They were taken in a cart to the place of execution, amidst a vast concourse of people.|They were taken in a cart to the place of execution, amidst a vast concourse of people. +LJ008-0026|Palm, as soon as he was seated in the cart,|Palm, as soon as he was seated in the cart, +LJ008-0027|put a quid of tobacco into his mouth, and offered another to his companion, who refused it with indignation|put a quid of tobacco into his mouth, and offered another to his companion, who refused it with indignation +LJ008-0028|Some indications of pity were offered for the fate of Tilling; Palm, execration alone.|Some indications of pity were offered for the fate of Tilling; Palm, execration alone. +LJ008-0029|But the Old Bailey gradually, and in spite of all objections urged, monopolized the dread business of execution.|But the Old Bailey gradually, and in spite of all objections urged, monopolized the dread business of execution. +LJ008-0030|The first affair of the kind on this spot was on the 3rd December, 1783,|The first affair of the kind on this spot was on the third December, seventeen eighty-three, +LJ008-0031|when, in pursuance of an order issued by the Recorder to the sheriffs of Middlesex and the keeper of His Majesty's jail, Newgate,|when, in pursuance of an order issued by the Recorder to the sheriffs of Middlesex and the keeper of His Majesty's jail, Newgate, +LJ008-0032|a scaffold was erected in front of that prison for the execution of several convicts named by the Recorder.|a scaffold was erected in front of that prison for the execution of several convicts named by the Recorder. +LJ008-0033|Ten were executed|Ten were executed +LJ008-0034|the scaffold hung with black; and the inhabitants of the neighborhood, having petitioned the sheriffs to remove the scene of execution to the old place,|the scaffold hung with black; and the inhabitants of the neighborhood, having petitioned the sheriffs to remove the scene of execution to the old place, +LJ008-0035|were told that the plan had been well considered, and would be persevered in.|were told that the plan had been well considered, and would be persevered in. +LJ008-0036|The following 23rd April, it is stated that the malefactors ordered for execution on the 18th instead|The following twenty-third April, it is stated that the malefactors ordered for execution on the eighteenth instead +LJ008-0037|were brought out of Newgate about eight in the morning, and suspended on a gallows of a new construction.|were brought out of Newgate about eight in the morning, and suspended on a gallows of a new construction. +LJ008-0038|After hanging the usual time they were taken down, and the machine cleared away in half-an-hour.|After hanging the usual time they were taken down, and the machine cleared away in half-an-hour. +LJ008-0039|By practice the art is much improved, and there is no part of the world in which villains are hanged in so neat a manner, and with so little ceremony.|By practice the art is much improved, and there is no part of the world in which villains are hanged in so neat a manner, and with so little ceremony. +LJ008-0040|A full description of this new gallows, which was erected in front of the debtors' door, is to be found in contemporary records.|A full description of this new gallows, which was erected in front of the debtors' door, is to be found in contemporary records. +LJ008-0041|The criminals are not exposed to view till they mount the fatal stage. The last part of the stage, or that next to the jail,|The criminals are not exposed to view till they mount the fatal stage. The last part of the stage, or that next to the jail, +LJ008-0042|is enclosed by a temporary roof, under which are placed two seats for the reception of the sheriffs, one on each side of the stairs leading to the scaffold.|is enclosed by a temporary roof, under which are placed two seats for the reception of the sheriffs, one on each side of the stairs leading to the scaffold. +LJ008-0043|Round the north, west, and south sides are erected galleries for the reception of officers, attendants, etc.|Round the north, west, and south sides are erected galleries for the reception of officers, attendants, etc. +LJ008-0044|and at the distance of five feet from the same is fixed a strong railing all round the scaffold to enclose a place for the constables.|and at the distance of five feet from the same is fixed a strong railing all round the scaffold to enclose a place for the constables. +LJ008-0045|In the middle of this machinery is placed a movable platform, in form of a trap-door, ten feet long by eight wide,|In the middle of this machinery is placed a movable platform, in form of a trap-door, ten feet long by eight wide, +LJ008-0046|on the middle of which is placed the gibbet, extending from the jail across the Old Bailey.|on the middle of which is placed the gibbet, extending from the jail across the Old Bailey. +LJ008-0047|This movable platform is raised six inches higher than the rest of the scaffold, and on it the convicts stand;|This movable platform is raised six inches higher than the rest of the scaffold, and on it the convicts stand; +LJ008-0048|it is supported by two beams, which are held in their place by bolts. The movement of the lever withdraws the bolts, the platform falls in;|it is supported by two beams, which are held in their place by bolts. The movement of the lever withdraws the bolts, the platform falls in; +LJ008-0049|and this, being much more sudden and regular than that of a cart being drawn away, has the effect of immediate death.|and this, being much more sudden and regular than that of a cart being drawn away, has the effect of immediate death. +LJ008-0050|A broadsheet dated April 24th, 1787, describing an execution on the newly-invented scaffold before the debtors' door,|A broadsheet dated April twenty-fourth, seventeen eighty-seven, describing an execution on the newly-invented scaffold before the debtors' door, +LJ008-0051|says, "The scaffold on which these miserable people suffered is a temporary machine which was drawn out of the yard of the sessions house by horses;|says, "The scaffold on which these miserable people suffered is a temporary machine which was drawn out of the yard of the sessions house by horses; +LJ008-0052|it is supported by strong posts fixed into grooves made in the street;|it is supported by strong posts fixed into grooves made in the street; +LJ008-0053|the whole is temporary, being all calculated to take to pieces, which are preserved within the prison."|the whole is temporary, being all calculated to take to pieces, which are preserved within the prison." +LJ008-0054|This contrivance appears to have been copied with improvements from that which had been used in Dublin at a still earlier date,|This contrivance appears to have been copied with improvements from that which had been used in Dublin at a still earlier date, +LJ008-0055|for that city claims the priority in establishing the custom of hanging criminals at the jail itself.|for that city claims the priority in establishing the custom of hanging criminals at the jail itself. +LJ008-0056|The Dublin "engine of death," as the gallows are styled in the account from which the following description is taken, consisted of an iron bar|The Dublin "engine of death," as the gallows are styled in the account from which the following description is taken, consisted of an iron bar +LJ008-0057|parallel to the prison wall, and about four feet from it, but strongly affixed thereto with iron scroll clamps.|parallel to the prison wall, and about four feet from it, but strongly affixed thereto with iron scroll clamps. +LJ008-0058|From this bar hang several iron loops, in which the halters are tied.|From this bar hang several iron loops, in which the halters are tied. +LJ008-0059|Under this bar at a proper distance is a piece of flooring or platform,|Under this bar at a proper distance is a piece of flooring or platform, +LJ008-0060|projecting somewhat beyond the range of the iron bar, and swinging upon hinges affixed to the wall.|projecting somewhat beyond the range of the iron bar, and swinging upon hinges affixed to the wall. +LJ008-0061|The entrance upon this floor or leaf is from the middle window over the gate of the prison;|The entrance upon this floor or leaf is from the middle window over the gate of the prison; +LJ008-0062|and this floor is supported below,|and this floor is supported below, +LJ008-0063|while the criminals stand upon it, by two pieces of timber, which are made to slide in and out of the prison wall through apertures made for that purpose.|while the criminals stand upon it, by two pieces of timber, which are made to slide in and out of the prison wall through apertures made for that purpose. +LJ008-0064|When the criminals are tied up and prepared for their fate, this floor suddenly falls down, upon withdrawing the supporters inwards.|When the criminals are tied up and prepared for their fate, this floor suddenly falls down, upon withdrawing the supporters inwards. +LJ008-0065|They are both drawn at once by a windlass, and the unhappy culprits remain suspended.|They are both drawn at once by a windlass, and the unhappy culprits remain suspended. +LJ008-0066|This mode of execution, it is alleged, gave rise to the old vulgar "chaff," "Take care, or you'll die at the fall of the leaf."|This mode of execution, it is alleged, gave rise to the old vulgar "chaff," "Take care, or you'll die at the fall of the leaf." +LJ008-0067|The machinery in use in Dublin is much the same as that employed at many jails now-a-days.|The machinery in use in Dublin is much the same as that employed at many jails now-a-days. +LJ008-0068|But the fall apart and inwards of two leaves is considered superior.|But the fall apart and inwards of two leaves is considered superior. +LJ008-0069|The latter is the method still followed at Newgate.|The latter is the method still followed at Newgate. +LJ008-0070|The sentences inflicted in front of Newgate were not limited to hanging.|The sentences inflicted in front of Newgate were not limited to hanging. +LJ008-0071|In the few years which elapsed between the establishment of the gallows at Newgate|In the few years which elapsed between the establishment of the gallows at Newgate +LJ008-0072|and the abolition of the practice of burning females for petty treason, more than one woman suffered this penalty at the Old Bailey.|and the abolition of the practice of burning females for petty treason, more than one woman suffered this penalty at the Old Bailey. +LJ008-0073|One case is preserved by Catnach,|One case is preserved by Catnach, +LJ008-0074|that of Phoebe Harris, who in 1788 was "barbariously" executed and burnt before Newgate for coining.|that of Phoebe Harris, who in seventeen eighty-eight was "barbariously" executed and burnt before Newgate for coining. +LJ008-0075|She is described as a well-made little woman, something more than thirty years of age, of a pale complexion and not disagreeable features.|She is described as a well-made little woman, something more than thirty years of age, of a pale complexion and not disagreeable features. +LJ008-0076|When she came out of prison she appeared languid and terrified, and trembled greatly as she advanced to the stake,|When she came out of prison she appeared languid and terrified, and trembled greatly as she advanced to the stake, +LJ008-0077|where the apparatus for the punishment she was about to experience|where the apparatus for the punishment she was about to experience +LJ008-0078|seemed to strike her mind with horror and consternation, to the exclusion of all power of recollectedness in preparation for the approaching awful moment.|seemed to strike her mind with horror and consternation, to the exclusion of all power of recollectedness in preparation for the approaching awful moment. +LJ008-0079|She walked from the debtors' door to a stake fixed in the ground about half-way between the scaffold and Newgate Street.|She walked from the debtors' door to a stake fixed in the ground about half-way between the scaffold and Newgate Street. +LJ008-0080|She was immediately tied by the neck to an iron bolt fixed near the top of the stake,|She was immediately tied by the neck to an iron bolt fixed near the top of the stake, +LJ008-0081|and after praying fervently for a few minutes, the steps on which she stood were drawn away, and she was left suspended.|and after praying fervently for a few minutes, the steps on which she stood were drawn away, and she was left suspended. +LJ008-0082|A chain fastened by nails to the stake was then put round her body by the executioner with his assistants.|A chain fastened by nails to the stake was then put round her body by the executioner with his assistants. +LJ008-0083|Two cart-loads of faggots were piled about her, and after she had hung for half-an-hour the fire was kindled.|Two cart-loads of faggots were piled about her, and after she had hung for half-an-hour the fire was kindled. +LJ008-0084|The flames presently burned the halter, the body fell a few inches, and hung then by the iron chain.|The flames presently burned the halter, the body fell a few inches, and hung then by the iron chain. +LJ008-0085|The fire had not quite burnt out at twelve, in nearly four hours, that is to say.|The fire had not quite burnt out at twelve, in nearly four hours, that is to say. +LJ008-0086|A great concourse of people attended on this melancholy occasion.|A great concourse of people attended on this melancholy occasion. +LJ008-0088|As many spectators as ever thronged to see the dreadful show,|As many spectators as ever thronged to see the dreadful show, +LJ008-0089|and they were packed into a more limited space, disporting themselves as heretofore by brutal horse-play, coarse jests, and frantic yells.|and they were packed into a more limited space, disporting themselves as heretofore by brutal horse-play, coarse jests, and frantic yells. +LJ008-0090|It was still the custom to offer warm encouragement or bitter disapproval, according to the character and antecedents of the sufferer.|It was still the custom to offer warm encouragement or bitter disapproval, according to the character and antecedents of the sufferer. +LJ008-0091|The highwayman, whose exploits many in the crowd admired or emulated, was cheered and bidden to die game;|The highwayman, whose exploits many in the crowd admired or emulated, was cheered and bidden to die game; +LJ008-0092|the man of better birth could hope for no sympathy, whatever his crime.|the man of better birth could hope for no sympathy, whatever his crime. +LJ008-0093|At the execution of Governor Wall, in 1802, the furious hatred of the mob was plainly apparent in their appalling cries.|At the execution of Governor Wall, in eighteen oh two, the furious hatred of the mob was plainly apparent in their appalling cries. +LJ008-0094|His appearance on the scaffold was the signal for three prolonged shouts from an innumerable populace, "the brutal effusion of one common sentiment."|His appearance on the scaffold was the signal for three prolonged shouts from an innumerable populace, "the brutal effusion of one common sentiment." +LJ008-0095|It was said that so large a crowd had never collected since the execution of Mrs. Brownrigg, nor had the public indignation risen so high.|It was said that so large a crowd had never collected since the execution of Mrs. Brownrigg, nor had the public indignation risen so high. +LJ008-0096|Pieman and ballad-monger did their usual roaring trade amidst the dense throng.|Pieman and ballad-monger did their usual roaring trade amidst the dense throng. +LJ008-0097|No sooner was the "job" finished than half-a-dozen competitors appeared, each offering the identical rope for sale at a shilling an inch.|No sooner was the "job" finished than half-a-dozen competitors appeared, each offering the identical rope for sale at a shilling an inch. +LJ008-0098|One was the "yeoman of the halter," a Newgate official, the executioner's assistant, whom Mr. J. T. Smith, who was present at the execution,|One was the "yeoman of the halter," a Newgate official, the executioner's assistant, whom Mr. J. T. Smith, who was present at the execution, +LJ008-0099|describes as "a most diabolical-looking little wretch -- Jack Ketch's head man.|describes as "a most diabolical-looking little wretch -- Jack Ketch's head man. +LJ008-0100|The yeoman was, however, under-sold by his wife, "Rosy Emma,"|The yeoman was, however, under-sold by his wife, "Rosy Emma," +LJ008-0101|exuberant in talk and hissing hot from Pie Corner, where she had taken her morning dose of gin-and-bitters.|exuberant in talk and hissing hot from Pie Corner, where she had taken her morning dose of gin-and-bitters. +LJ008-0102|A little further off, says Mr. Smith, was "a lath of a fellow past three-score years and ten,|A little further off, says Mr. Smith, was "a lath of a fellow past three-score years and ten, +LJ008-0103|who had just arrived from the purlieus of Black Boy Alley, woebegone as Romeo's apothecary, exclaiming,|who had just arrived from the purlieus of Black Boy Alley, woebegone as Romeo's apothecary, exclaiming, +LJ008-0104|Here's the identical rope at sixpence an inch.|Here's the identical rope at sixpence an inch. +LJ008-0105|Mr. Smith's account of the condemned convict, whose cell he was permitted to enter, may be inserted here.|Mr. Smith's account of the condemned convict, whose cell he was permitted to enter, may be inserted here. +LJ008-0106|He was introduced by the ordinary, Dr. Forde, a name familiar to the reader, who met him at the felons' door|He was introduced by the ordinary, Dr. Forde, a name familiar to the reader, who met him at the felons' door +LJ008-0107|in his canonicals, and with his head as stiffly erect as a sheriff's coachman.|in his canonicals, and with his head as stiffly erect as a sheriff's coachman. +LJ008-0108|The ordinary "gravely uttered, 'Come this way, Mr. Smith.'|The ordinary "gravely uttered, 'Come this way, Mr. Smith.' +LJ008-0109|As we crossed the press yard a cock crew, and the solitary clanking of a restless chain was dreadfully horrible.|As we crossed the press yard a cock crew, and the solitary clanking of a restless chain was dreadfully horrible. +LJ008-0110|The prisoners had not risen.|The prisoners had not risen. +LJ008-0111|They entered a "stone cold room," and were presently joined by the prisoner.|They entered a "stone cold room," and were presently joined by the prisoner. +LJ008-0112|He was death's counterfeit, tall, shriveled, and pale;|He was death's counterfeit, tall, shriveled, and pale; +LJ008-0113|and his soul shot out so piercingly through the port-holes of his head, that the first glance of him nearly petrified me|and his soul shot out so piercingly through the port-holes of his head, that the first glance of him nearly petrified me +LJ008-0114|His hands were clasped, and he was truly penitent.|His hands were clasped, and he was truly penitent. +LJ008-0115|After the yeoman had requested him to stand up, 'he pinioned him,' as the Newgate phrase is|After the yeoman had requested him to stand up, 'he pinioned him,' as the Newgate phrase is +LJ008-0116|and tied the cord with so little feeling that the governor (Wall), who had not given the wretch his accustomed fee, observed,|and tied the cord with so little feeling that the governor (Wall), who had not given the wretch his accustomed fee, observed, +LJ008-0117|"You have tied me very tight," upon which Dr. Forde ordered him to slacken the cord, which he did, but not without muttering.|"You have tied me very tight," upon which Dr. Forde ordered him to slacken the cord, which he did, but not without muttering. +LJ008-0118|"Thank you, sir," said the governor to the doctor, "it is of little moment."|"Thank you, sir," said the governor to the doctor, "it is of little moment." +LJ008-0119|He then made some observations to the attendant about the fire, and turning to the doctor, questioned him.|He then made some observations to the attendant about the fire, and turning to the doctor, questioned him. +LJ008-0120|"Do tell me, sir; I am informed I shall go down with great force; is it so?"|"Do tell me, sir; I am informed I shall go down with great force; is it so?" +LJ008-0121|After the construction and action of the machine had been explained, the doctor asked the governor what kind of men he had commanded at Goree,|After the construction and action of the machine had been explained, the doctor asked the governor what kind of men he had commanded at Goree, +LJ008-0122|where the murder for which he was condemned had been committed.|where the murder for which he was condemned had been committed. +LJ008-0123|"Sir," he answered, "they sent me the very riff-raff."|"Sir," he answered, "they sent me the very riff-raff." +LJ008-0124|The poor soul then joined the doctor in prayer, and never did I witness more contrition at any condemned sermon than he then evinced.|The poor soul then joined the doctor in prayer, and never did I witness more contrition at any condemned sermon than he then evinced. +LJ008-0125|The sheriff arrived, attended by his officers, to receive the prisoner from the keeper.|The sheriff arrived, attended by his officers, to receive the prisoner from the keeper. +LJ008-0126|A new hat was partly flattened on his head, for, owing to its being too small in the crown, it stood many inches too high behind.|A new hat was partly flattened on his head, for, owing to its being too small in the crown, it stood many inches too high behind. +LJ008-0127|As we were crossing the press yard,|As we were crossing the press yard, +LJ008-0128|the dreadful execrations of some of the felons so shook his frame that he observed "the clock had struck;" and quickening his pace,|the dreadful execrations of some of the felons so shook his frame that he observed "the clock had struck;" and quickening his pace, +LJ008-0129|he soon arrived at the room where the sheriff was to give a receipt for his body, according to the usual custom.|he soon arrived at the room where the sheriff was to give a receipt for his body, according to the usual custom. +LJ008-0130|Before the colonel had been pinioned he had pulled out two white handkerchiefs, one of which he bound over his temples so as nearly to conceal his eyes,|Before the colonel had been pinioned he had pulled out two white handkerchiefs, one of which he bound over his temples so as nearly to conceal his eyes, +LJ008-0132|Over the handkerchief around his brows he placed a white cap, the new hat being on top of all.|Over the handkerchief around his brows he placed a white cap, the new hat being on top of all. +LJ008-0133|He was dressed in a mixed-colored loose coat with a black collar, swandown waistcoat, blue pantaloons, and white silk stockings.|He was dressed in a mixed-colored loose coat with a black collar, swandown waistcoat, blue pantaloons, and white silk stockings. +LJ008-0134|Thus appareled he ascended the stairs at the debtors' door,|Thus appareled he ascended the stairs at the debtors' door, +LJ008-0135|and stepped out on to the platform, to be received, as has been said, by prolonged yells.|and stepped out on to the platform, to be received, as has been said, by prolonged yells. +LJ008-0136|These evidently deprived him of the small portion of fortitude he had summoned up.|These evidently deprived him of the small portion of fortitude he had summoned up. +LJ008-0137|He bowed his head under extreme pressure of ignominy,|He bowed his head under extreme pressure of ignominy, +LJ008-0138|and at his request the ordinary drew the cap further down over his face, when in an instant,|and at his request the ordinary drew the cap further down over his face, when in an instant, +LJ008-0139|without waiting for any signal, the platform dropped, and he was launched into eternity.|without waiting for any signal, the platform dropped, and he was launched into eternity. +LJ008-0141|the doubtfulness of the issue, or the superior position of the perpetrator,|the doubtfulness of the issue, or the superior position of the perpetrator, +LJ008-0142|the attendance at the execution was certain to be tumultuous, and the conduct of the mob disorderly.|the attendance at the execution was certain to be tumultuous, and the conduct of the mob disorderly. +LJ008-0143|This was notably the case at the execution of Holloway and Haggerty|This was notably the case at the execution of Holloway and Haggerty +LJ008-0144|in 1807, an event long remembered from the fatal and disastrous consequences which followed it.|in eighteen oh seven, an event long remembered from the fatal and disastrous consequences which followed it. +LJ008-0145|They were accused by a confederate, who, goaded by conscience, had turned approver, of the murder of a Mr. Steele,|They were accused by a confederate, who, goaded by conscience, had turned approver, of the murder of a Mr. Steele, +LJ008-0146|who kept a lavender warehouse in the city, and who had gardens at Feltham,|who kept a lavender warehouse in the city, and who had gardens at Feltham, +LJ008-0147|whither he often went to distill the lavender, returning to London the same evening.|whither he often went to distill the lavender, returning to London the same evening. +LJ008-0148|One night he was missing|One night he was missing +LJ008-0149|and after a long interval his dead body was discovered, shockingly disfigured, in a ditch. This was in 1802.|and after a long interval his dead body was discovered, shockingly disfigured, in a ditch. This was in eighteen oh two. +LJ008-0150|Four years passed without the detection of the murderers,|Four years passed without the detection of the murderers, +LJ008-0151|but in the beginning of 1807 one of them, at that time just sentenced to transportation,|but in the beginning of eighteen oh seven one of them, at that time just sentenced to transportation, +LJ008-0152|made a full confession, and implicated Holloway and Haggerty.|made a full confession, and implicated Holloway and Haggerty. +LJ008-0153|They were accordingly apprehended and brought to trial, the informer, Hanfield by name, being accepted as king's evidence.|They were accordingly apprehended and brought to trial, the informer, Hanfield by name, being accepted as king's evidence. +LJ008-0154|Conviction followed mainly on his testimony; but the two men, especially Holloway, stoutly maintained their innocence to the last.|Conviction followed mainly on his testimony; but the two men, especially Holloway, stoutly maintained their innocence to the last. +LJ008-0155|Very great excitement prevailed in the town throughout the trial, and this greatly increased when the verdict was known.|Very great excitement prevailed in the town throughout the trial, and this greatly increased when the verdict was known. +LJ008-0156|An enormous crowd assembled to witness the execution, amounting, it was said, to the hitherto unparalleled number of 40,000.|An enormous crowd assembled to witness the execution, amounting, it was said, to the hitherto unparalleled number of forty thousand. +LJ008-0157|By eight o'clock not an inch of ground in front of the platform was unoccupied.|By eight o'clock not an inch of ground in front of the platform was unoccupied. +LJ008-0159|Very soon women began to scream with terror;|Very soon women began to scream with terror; +LJ008-0160|some, especially of low stature, found it difficult to remain standing, and several, although held up for some time by the men nearest them,|some, especially of low stature, found it difficult to remain standing, and several, although held up for some time by the men nearest them, +LJ008-0161|presently fell, and were at once trampled to death.|presently fell, and were at once trampled to death. +LJ008-0162|Cries of Murder! murder! were now raised, and added greatly to the horrors of the scene.|Cries of Murder! murder! were now raised, and added greatly to the horrors of the scene. +LJ008-0163|Panic became general. More women, children, and many men were borne down, to perish beneath the feet of the rest.|Panic became general. More women, children, and many men were borne down, to perish beneath the feet of the rest. +LJ008-0164|The most affecting and distressing scene was at Green Arbor Lane, just opposite the debtors' door of the prison.|The most affecting and distressing scene was at Green Arbor Lane, just opposite the debtors' door of the prison. +LJ008-0165|Here a couple of piemen had been selling their wares; the basket of one of them, which was raised upon a four-legged stool, was upset.|Here a couple of piemen had been selling their wares; the basket of one of them, which was raised upon a four-legged stool, was upset. +LJ008-0166|The pieman stooped down to pick up his scattered stock, and some of the mob, not seeing what had happened, stumbled over him.|The pieman stooped down to pick up his scattered stock, and some of the mob, not seeing what had happened, stumbled over him. +LJ008-0167|No one who fell ever rose again.|No one who fell ever rose again. +LJ008-0168|Among the rest was a woman with an infant at the breast.|Among the rest was a woman with an infant at the breast. +LJ008-0169|She was killed, but in the act of falling she forced her child into the arms of a man near her, and implored him in God's name to save it;|She was killed, but in the act of falling she forced her child into the arms of a man near her, and implored him in God's name to save it; +LJ008-0170|the man, needing all his care for his own life, threw the child from him,|the man, needing all his care for his own life, threw the child from him, +LJ008-0171|and it passed along the heads of the crowd, to be caught at last by a person who struggled with it to a cart and deposited it there in safety.|and it passed along the heads of the crowd, to be caught at last by a person who struggled with it to a cart and deposited it there in safety. +LJ008-0172|In another part seven persons met their death by suffocation.|In another part seven persons met their death by suffocation. +LJ008-0173|In this convulsive struggle for bare existence people fought fiercely with one another, and the weakest, of course the women, went under.|In this convulsive struggle for bare existence people fought fiercely with one another, and the weakest, of course the women, went under. +LJ008-0175|This went on for more than an hour, and until the malefactors were cut down and the gallows removed;|This went on for more than an hour, and until the malefactors were cut down and the gallows removed; +LJ008-0176|then the mob began to thin, and the streets were cleared by the city marshals and a number of constables.|then the mob began to thin, and the streets were cleared by the city marshals and a number of constables. +LJ008-0177|The catastrophe exceeded the worst anticipations. Nearly one hundred dead and dying lay about; and after all had been removed,|The catastrophe exceeded the worst anticipations. Nearly one hundred dead and dying lay about; and after all had been removed, +LJ008-0178|the bodies for identification, the wounded to hospitals, a cart-load of shoes, hats, petticoats, and fragments of wearing apparel were picked up.|the bodies for identification, the wounded to hospitals, a cart-load of shoes, hats, petticoats, and fragments of wearing apparel were picked up. +LJ008-0180|Among the dead was a sailor lad whom no one knew;|Among the dead was a sailor lad whom no one knew; +LJ008-0181|he had his pockets filled with bread and cheese, and it was generally supposed that he had come a long distance to see the fatal show.|he had his pockets filled with bread and cheese, and it was generally supposed that he had come a long distance to see the fatal show. +LJ008-0182|A tremendous crowd assembled when Bellingham was executed in 1812 for the murder of Spencer Percival, at that time prime minister;|A tremendous crowd assembled when Bellingham was executed in eighteen twelve for the murder of Spencer Percival, at that time prime minister; +LJ008-0183|but there were no serious accidents, beyond those caused by the goring of a maddened, over-driven ox which forced its way through the crowd.|but there were no serious accidents, beyond those caused by the goring of a maddened, over-driven ox which forced its way through the crowd. +LJ008-0184|Precautions had been taken by the erection of barriers, and the posting of placards at all the avenues to the Old Bailey, on which was printed,|Precautions had been taken by the erection of barriers, and the posting of placards at all the avenues to the Old Bailey, on which was printed, +LJ008-0185|Beware of entering the crowd! Remember thirty poor persons were pressed to death by the crowd when Haggerty and Holloway were executed!|Beware of entering the crowd! Remember thirty poor persons were pressed to death by the crowd when Haggerty and Holloway were executed! +LJ008-0186|The concourse was very great, notwithstanding these warnings.|The concourse was very great, notwithstanding these warnings. +LJ008-0187|It was still greater at Fauntleroy's execution in 1824, when no less than 100,000 persons assembled, it was said.|It was still greater at Fauntleroy's execution in eighteen twenty-four, when no less than one hundred thousand persons assembled, it was said. +LJ008-0188|Every window and roof which could command a view of the horrible performance was occupied.|Every window and roof which could command a view of the horrible performance was occupied. +LJ008-0189|All the avenues and approaches, places even whence nothing whatever could be seen of the scaffold,|All the avenues and approaches, places even whence nothing whatever could be seen of the scaffold, +LJ008-0190|were blocked by persons who had overflowed from the area in front of the jail.|were blocked by persons who had overflowed from the area in front of the jail. +LJ008-0191|At Courvoisier's execution in 1840 it was the same, or worse.|At Courvoisier's execution in eighteen forty it was the same, or worse. +LJ008-0192|As early as six a.m. the number assembled already exceeded that seen on ordinary occasions;|As early as six a.m. the number assembled already exceeded that seen on ordinary occasions; +LJ008-0193|by seven a.m. the whole space was so thronged that it was impossible to move one way or the other.|by seven a.m. the whole space was so thronged that it was impossible to move one way or the other. +LJ008-0194|Some persons were kept for more than five hours standing against the barriers, and many nearly fainted from exhaustion.|Some persons were kept for more than five hours standing against the barriers, and many nearly fainted from exhaustion. +LJ008-0195|Every window had its party of occupants; the adjoining roofs were equally crowded.|Every window had its party of occupants; the adjoining roofs were equally crowded. +LJ008-0196|High prices were asked and paid for front seats or good standing room. As much as £5 was given for the attic story|High prices were asked and paid for front seats or good standing room. As much as five pounds was given for the attic story +LJ008-0197|of the Lamb's Coffee House;|of the Lamb's Coffee House; +LJ008-0198|£2 was a common price for a window.|two pounds was a common price for a window. +LJ008-0199|At the George public-house to the south of the drop, Sir W. Watkin Wynn, Baronet,|At the George public-house to the south of the drop, Sir W. Watkin Wynn, Baronet, +LJ008-0200|hired a room for the night and morning, which he and a large party of friends occupied before and during the execution;|hired a room for the night and morning, which he and a large party of friends occupied before and during the execution; +LJ008-0201|in an adjoining house, that of an undertaker, was Lord Alfred Paget, also with several friends.|in an adjoining house, that of an undertaker, was Lord Alfred Paget, also with several friends. +LJ008-0202|Those who had hired apartments spent the night in them, keeping up their courage with liquids and cigars.|Those who had hired apartments spent the night in them, keeping up their courage with liquids and cigars. +LJ008-0203|Numbers of ladies were present, although the public feeling was much against their attendance.|Numbers of ladies were present, although the public feeling was much against their attendance. +LJ008-0204|One well-dressed woman fell out of a first-floor window on to the shoulders of the crowd below, but neither she nor any one else was greatly hurt.|One well-dressed woman fell out of a first-floor window on to the shoulders of the crowd below, but neither she nor any one else was greatly hurt. +LJ008-0205|The city authorities had endeavored to take all precautions against panic and excitement among the crowd,|The city authorities had endeavored to take all precautions against panic and excitement among the crowd, +LJ008-0206|and caused a number of stout additional barriers to be erected in front of the scaffold,|and caused a number of stout additional barriers to be erected in front of the scaffold, +LJ008-0207|and although one of these gave way owing to the extraordinary pressure, no serious accident occurred.|and although one of these gave way owing to the extraordinary pressure, no serious accident occurred. +LJ008-0208|Some years later an eye-witness published a graphic account of one of these scenes.|Some years later an eye-witness published a graphic account of one of these scenes. +LJ008-0209|Soon after midnight on the Sunday night, for by this time the present practice of executing on Monday morning had been pretty generally introduced,|Soon after midnight on the Sunday night, for by this time the present practice of executing on Monday morning had been pretty generally introduced, +LJ008-0210|the crowd began to congregate in and about the Old Bailey.|the crowd began to congregate in and about the Old Bailey. +LJ008-0211|Gin-shops and coffee-houses were the first to open doors, and touts began to bid for tenants for the various rooms upstairs.|Gin-shops and coffee-houses were the first to open doors, and touts began to bid for tenants for the various rooms upstairs. +LJ008-0212|Cries of "Comfortable room!" "Excellent situation!" "Beautiful prospect!" "Splendid view!" resounded on every side.|Cries of "Comfortable room!" "Excellent situation!" "Beautiful prospect!" "Splendid view!" resounded on every side. +LJ008-0213|By this time the workmen might be heard busily erecting the gallows;|By this time the workmen might be heard busily erecting the gallows; +LJ008-0214|the sounds of hammer and saw intermingled with the broad jeers and coarse jests of the rapidly increasing mob.|the sounds of hammer and saw intermingled with the broad jeers and coarse jests of the rapidly increasing mob. +LJ008-0215|One by one the huge uprights of black timber were fitted together,|One by one the huge uprights of black timber were fitted together, +LJ008-0216|until presently the huge stage loomed dark above the crowd which was now ranged round the barriers;|until presently the huge stage loomed dark above the crowd which was now ranged round the barriers; +LJ008-0217|a throng of people whom neither rain, snow, storm, nor darkness ever hindered from attending the show.|a throng of people whom neither rain, snow, storm, nor darkness ever hindered from attending the show. +LJ008-0218|They were mainly members of the criminal classes.|They were mainly members of the criminal classes. +LJ008-0219|their conversation was of companions and associates of former years, long ago imprisoned, transported, hanged, while they,|their conversation was of companions and associates of former years, long ago imprisoned, transported, hanged, while they, +LJ008-0220|hoary-headed and hardened in guilt, were still at large.|hoary-headed and hardened in guilt, were still at large. +LJ008-0221|They talked of the days when the convicts were hung up a dozen or more in a row;|They talked of the days when the convicts were hung up a dozen or more in a row; +LJ008-0222|of those who had shown the white, and those who had died game.|of those who had shown the white, and those who had died game. +LJ008-0223|The approaching ceremony had evidently no terrors for these "idolaters of the gallows."|The approaching ceremony had evidently no terrors for these "idolaters of the gallows." +LJ008-0224|With them were younger men and women:|With them were younger men and women: +LJ008-0225|the former already vowed to the same criminal career, and looking up to their elders with the respect due to successful practitioners;|the former already vowed to the same criminal career, and looking up to their elders with the respect due to successful practitioners; +LJ008-0226|the latter unsexed and brutalized by dissipation,|the latter unsexed and brutalized by dissipation, +LJ008-0227|slipshod and slovenly, in crushed bonnet and dirty shawl, the gown fastened by a single hook,|slipshod and slovenly, in crushed bonnet and dirty shawl, the gown fastened by a single hook, +LJ008-0228|their harsh and half-cracked voices full of maudlin, besotted sympathy for those about to die.|their harsh and half-cracked voices full of maudlin, besotted sympathy for those about to die. +LJ008-0229|Above the murmur and tumult of that noisy assembly, the lowing and bleating of cattle as they were driven into the stalls and pens of Smithfield|Above the murmur and tumult of that noisy assembly, the lowing and bleating of cattle as they were driven into the stalls and pens of Smithfield +LJ008-0230|fell with a strange unnatural sound upon the ear|fell with a strange unnatural sound upon the ear +LJ008-0231|Hush! the unceasing murmur of the mob now breaks into a loud deep roar,|Hush! the unceasing murmur of the mob now breaks into a loud deep roar, +LJ008-0232|a sound as if the ocean had suddenly broken through some ancient boundary, against which its ever restless billows had for ages battered;|a sound as if the ocean had suddenly broken through some ancient boundary, against which its ever restless billows had for ages battered; +LJ008-0233|the wide dark sea of heads is all at once in motion;|the wide dark sea of heads is all at once in motion; +LJ008-0234|each wave seems trying to overleap the other as they are drawn onwards towards this outlet.|each wave seems trying to overleap the other as they are drawn onwards towards this outlet. +LJ008-0235|Every link in that great human chain is shaken, along the whole lengthened line has the motion jarred, and each in turn sees,|Every link in that great human chain is shaken, along the whole lengthened line has the motion jarred, and each in turn sees, +LJ008-0236|coiled up on the floor of the scaffold like a serpent, the hangman's rope!|coiled up on the floor of the scaffold like a serpent, the hangman's rope! +LJ008-0237|The human hand that placed it there was only seen for a moment,|The human hand that placed it there was only seen for a moment, +LJ008-0238|as it lay, white and ghastly, upon the black boards, and then again was as suddenly withdrawn, as if ashamed of the deed it had done.|as it lay, white and ghastly, upon the black boards, and then again was as suddenly withdrawn, as if ashamed of the deed it had done. +LJ008-0239|The loud shout of the multitude once more subsided, or only fell upon the abstracted ear like the dreamy murmur of an ocean shell.|The loud shout of the multitude once more subsided, or only fell upon the abstracted ear like the dreamy murmur of an ocean shell. +LJ008-0240|Then followed sounds more distinct and audible, in which ginger-beer, pies, fried fish, sandwiches, and fruit|Then followed sounds more distinct and audible, in which ginger-beer, pies, fried fish, sandwiches, and fruit +LJ008-0241|were vended under the names of notorious murderers, highwaymen, and criminals,|were vended under the names of notorious murderers, highwaymen, and criminals, +LJ008-0242|famous in the annals of Newgate for the hardihood they had displayed in the hour of execution, when they terminated their career of crime at the gallows.|famous in the annals of Newgate for the hardihood they had displayed in the hour of execution, when they terminated their career of crime at the gallows. +LJ008-0243|Threading his way among these itinerant vendors was seen the meek-faced deliverer of tracts, the man of good intentions, now bonneted,|Threading his way among these itinerant vendors was seen the meek-faced deliverer of tracts, the man of good intentions, now bonneted, +LJ008-0244|now laughed at, the skirt of his seedy black coat torn across; yet,|now laughed at, the skirt of his seedy black coat torn across; yet, +LJ008-0245|though pulled right and left, or sent headlong into the crowd by the swing of some brutal and muscular arm,|though pulled right and left, or sent headlong into the crowd by the swing of some brutal and muscular arm, +LJ008-0246|Never once from that pale face passed away its benign and patient expression.|Never once from that pale face passed away its benign and patient expression. +LJ008-0247|but ever the same form moved along in the fulfillment of his mission, in spite of all persecution.|but ever the same form moved along in the fulfillment of his mission, in spite of all persecution. +LJ008-0248|Another fight followed the score which had already taken place; this time two women were the combatants.|Another fight followed the score which had already taken place; this time two women were the combatants. +LJ008-0249|Blinded with their long hair, they tore at each other like two furies; their bonnets and caps were trodden underfoot in the kennel,|Blinded with their long hair, they tore at each other like two furies; their bonnets and caps were trodden underfoot in the kennel, +LJ008-0250|and lay disregarded beside the body of the poor dog which, while searching for its master in the crowd,|and lay disregarded beside the body of the poor dog which, while searching for its master in the crowd, +LJ008-0251|was an hour before kicked to death by the savage and brutal mob.|was an hour before kicked to death by the savage and brutal mob. +LJ008-0252|Another deep roar, louder than any which had preceded it, broke from the multitude.|Another deep roar, louder than any which had preceded it, broke from the multitude. +LJ008-0253|Then came the cry of 'Hats off!' and 'Down in front!' as at a theatre.|Then came the cry of 'Hats off!' and 'Down in front!' as at a theatre. +LJ008-0254|It was followed by the deep and solemn booming of the death-bell from the church of St. Sepulchre|It was followed by the deep and solemn booming of the death-bell from the church of St. Sepulchre +LJ008-0255|the iron knell that rang upon the beating heart of the living man who was about to die;|the iron knell that rang upon the beating heart of the living man who was about to die; +LJ008-0256|and with blanched cheek, and sinking, we turned away from the scene.|and with blanched cheek, and sinking, we turned away from the scene. +LJ008-0257|In thus describing the saturnalia before the gallows I have been drawn on somewhat beyond the period with which I am at present dealing.|In thus describing the saturnalia before the gallows I have been drawn on somewhat beyond the period with which I am at present dealing. +LJ008-0258|Let me retrace my steps, and speak more in detail of the treatment of the condemned in those bloodthirsty and brutally indifferent days,|Let me retrace my steps, and speak more in detail of the treatment of the condemned in those bloodthirsty and brutally indifferent days, +LJ008-0259|and of their demeanor after sentence until the last penalty was paid.|and of their demeanor after sentence until the last penalty was paid. +LJ008-0260|One of the worst evils was the terrible and long-protracted uncertainty as to the result.|One of the worst evils was the terrible and long-protracted uncertainty as to the result. +LJ008-0261|In the case of convicted murderers only was prompt punishment inflicted,|In the case of convicted murderers only was prompt punishment inflicted, +LJ008-0262|and with them indeed this dispatch amounted to undue precipitancy.|and with them indeed this dispatch amounted to undue precipitancy. +LJ008-0263|Forty-eight hours was the limit of time allowed to the unhappy man to make his peace, and during that time he was still kept on a bare allowance of bread and water.|Forty-eight hours was the limit of time allowed to the unhappy man to make his peace, and during that time he was still kept on a bare allowance of bread and water. +LJ008-0264|But the murderers formed only a small proportion of the total number sentenced to death, and for the rest there was a long period of anxious suspense,|But the murderers formed only a small proportion of the total number sentenced to death, and for the rest there was a long period of anxious suspense, +LJ008-0265|although in the long run mercy generally prevailed, and very few capitally convicted for crimes less than murder actually suffered.|although in the long run mercy generally prevailed, and very few capitally convicted for crimes less than murder actually suffered. +LJ008-0266|Thus in the years between May 1st, 1827, and 30th April, 1831,|Thus in the years between May first, eighteen twenty-seven, and thirtieth April, eighteen thirty-one, +LJ008-0267|no less than four hundred and fifty-one sentences of death for capital crimes were passed at the Old Bailey;|no less than four hundred and fifty-one sentences of death for capital crimes were passed at the Old Bailey; +LJ008-0268|but of these three hundred and ninety-six were reversed by the king in council, and only fifty-two were really executed.|but of these three hundred and ninety-six were reversed by the king in council, and only fifty-two were really executed. +LJ008-0269|Already the severity of our criminal code, and the number of capital felonies upon the statute book, had brought a reaction;|Already the severity of our criminal code, and the number of capital felonies upon the statute book, had brought a reaction; +LJ008-0270|and while the courts adhered to the letter of the law, appeals were constantly made to the royal prerogative of mercy.|and while the courts adhered to the letter of the law, appeals were constantly made to the royal prerogative of mercy. +LJ008-0271|This was more particularly the practice in London.|This was more particularly the practice in London. +LJ008-0272|Judges on assize were satisfied with simply recording a sentence of death against offenders whom they did not think deserved the extreme penalty.|Judges on assize were satisfied with simply recording a sentence of death against offenders whom they did not think deserved the extreme penalty. +LJ008-0273|At the Old Bailey almost every one capitally convicted by a jury was sentenced to be hanged.|At the Old Bailey almost every one capitally convicted by a jury was sentenced to be hanged. +LJ008-0274|The result in the latter case was left in the first place to the king in council,|The result in the latter case was left in the first place to the king in council, +LJ008-0275|but there was a further appeal then, as now, to the king himself, or practically to the Home Secretary.|but there was a further appeal then, as now, to the king himself, or practically to the Home Secretary. +LJ008-0276|Neither in town or country were cases entirely taken on their own merits.|Neither in town or country were cases entirely taken on their own merits. +LJ008-0277|Convicted offenders might have good or bad luck; they might be arraigned when their particular crime was uncommon, and were then nearly certain to escape;|Convicted offenders might have good or bad luck; they might be arraigned when their particular crime was uncommon, and were then nearly certain to escape; +LJ008-0278|or theirs might be one of many, and it might be considered necessary to "make an example."|or theirs might be one of many, and it might be considered necessary to "make an example." +LJ008-0279|In this latter it might fairly be said that a man was put to death less for his own sins than for the crimes of others.|In this latter it might fairly be said that a man was put to death less for his own sins than for the crimes of others. +LJ008-0280|The absurdity of the system, its irregularity and cruelty, were fully touched upon by the inspectors of prisons in their first report.|The absurdity of the system, its irregularity and cruelty, were fully touched upon by the inspectors of prisons in their first report. +LJ008-0281|They found at Newgate, under disgraceful conditions as already described,|They found at Newgate, under disgraceful conditions as already described, +LJ008-0282|seventeen capital convicts, upon all of whom the sentence of death had been passed. Eventually two only of the whole number suffered;|seventeen capital convicts, upon all of whom the sentence of death had been passed. Eventually two only of the whole number suffered; +LJ008-0283|two others were sentenced to three months' imprisonment, and the balance to varying terms.|two others were sentenced to three months' imprisonment, and the balance to varying terms. +LJ008-0284|Nothing could be more strongly marked than the contrast between the ultimate destiny of different individuals all abiding the same awful doom:|Nothing could be more strongly marked than the contrast between the ultimate destiny of different individuals all abiding the same awful doom: +LJ008-0285|on the one hand the gallows, on the other a short imprisonment.|on the one hand the gallows, on the other a short imprisonment. +LJ008-0286|The inspectors very properly desired to call attention to the inevitable tendency in this mode of dealing with "the most awful sanctions of the law,"|The inspectors very properly desired to call attention to the inevitable tendency in this mode of dealing with "the most awful sanctions of the law," +LJ008-0287|to make those sanctions an object of contemptuous mockery.|to make those sanctions an object of contemptuous mockery. +LJ008-0288|The consequences were plainly proved to the inspectors.|The consequences were plainly proved to the inspectors. +LJ008-0289|Capitally convicted prisoners did, as a matter of fact, "treat with habitual and inexpressible levity the sentence of death."|Capitally convicted prisoners did, as a matter of fact, "treat with habitual and inexpressible levity the sentence of death." +LJ008-0290|Of this I have treated at length in the last chapter.|Of this I have treated at length in the last chapter. +LJ008-0291|The time thus spent varied considerably,|The time thus spent varied considerably, +LJ008-0292|but it was seldom less than six weeks. It all depended upon the sovereign's disposition to do business.|but it was seldom less than six weeks. It all depended upon the sovereign's disposition to do business. +LJ008-0293|Sometimes the Privy Council did not meet for months, and during all that time the convicts languished with hope|Sometimes the Privy Council did not meet for months, and during all that time the convicts languished with hope +LJ008-0294|nearly indefinitely deferred.|nearly indefinitely deferred. +LJ008-0295|When the council had decided, the news was conveyed to Newgate by the Recorder, who made his "report," as it was called.|When the council had decided, the news was conveyed to Newgate by the Recorder, who made his "report," as it was called. +LJ008-0296|The time of the arrival of this report was generally known at Newgate,|The time of the arrival of this report was generally known at Newgate, +LJ008-0297|and its contents were anxiously awaited by both convicts in the press-yard and their friends collected in a crowd outside the gates.|and its contents were anxiously awaited by both convicts in the press-yard and their friends collected in a crowd outside the gates. +LJ008-0298|Sometimes the report was delayed.|Sometimes the report was delayed. +LJ008-0299|On one occasion, Mr. Wakefield tells us, the Recorder, who had attended the council at Windsor, did not deliver the report till the following day.|On one occasion, Mr. Wakefield tells us, the Recorder, who had attended the council at Windsor, did not deliver the report till the following day. +LJ008-0300|The prisoners and their friends, therefore, were kept in a state of the most violent suspense for many hours,|The prisoners and their friends, therefore, were kept in a state of the most violent suspense for many hours, +LJ008-0301|during which they counted the moments -- the prisoners in their cells as usual, and their friends in the street in front of Newgate, where they passed the night.|during which they counted the moments -- the prisoners in their cells as usual, and their friends in the street in front of Newgate, where they passed the night. +LJ008-0302|I have heard the protracted agony of both classes described by those who witnessed it in terms so strong, that I am unwilling to repeat them.|I have heard the protracted agony of both classes described by those who witnessed it in terms so strong, that I am unwilling to repeat them. +LJ008-0303|The crowd of men and women who passed the night in front of Newgate, began, as soon as the hour was passed when they had expected the report,|The crowd of men and women who passed the night in front of Newgate, began, as soon as the hour was passed when they had expected the report, +LJ008-0304|to utter imprecations against the Recorder, the Secretary of State, the Council, and the King;|to utter imprecations against the Recorder, the Secretary of State, the Council, and the King; +LJ008-0305|they never ceased cursing until the passion of anger so excited was exchanged for joy in some and grief in others.|they never ceased cursing until the passion of anger so excited was exchanged for joy in some and grief in others. +LJ008-0306|I myself heard more than one of those whose lives were spared by that decision of the council,|I myself heard more than one of those whose lives were spared by that decision of the council, +LJ008-0307|afterwards express a wish to murder the Recorder for having kept them so long in suspense.|afterwards express a wish to murder the Recorder for having kept them so long in suspense. +LJ008-0308|The Recorder's report generally reached Newgate late at night.|The Recorder's report generally reached Newgate late at night. +LJ008-0309|Its receipt was immediately followed by the promulgation of its contents to the persons most closely concerned,|Its receipt was immediately followed by the promulgation of its contents to the persons most closely concerned, +LJ008-0310|which was done with a sort of ceremony intended to be impressive.|which was done with a sort of ceremony intended to be impressive. +LJ008-0311|The whole of the convicts were assembled together in one ward, and made to kneel down.|The whole of the convicts were assembled together in one ward, and made to kneel down. +LJ008-0312|To them entered the chaplain or ordinary of Newgate in full canonicals,|To them entered the chaplain or ordinary of Newgate in full canonicals, +LJ008-0313|who in solemn tones communicated to each in turn the fate in store for him.|who in solemn tones communicated to each in turn the fate in store for him. +LJ008-0314|The form of imparting the intelligence was generally the same.|The form of imparting the intelligence was generally the same. +LJ008-0315|"So-and-so, I am sorry to tell you that it is all against you;" or,|"So-and-so, I am sorry to tell you that it is all against you;" or, +LJ008-0316|"A. B., your case has been taken into consideration by the king in council, and His Majesty has been mercifully pleased to spare your life."|"A. B., your case has been taken into consideration by the king in council, and His Majesty has been mercifully pleased to spare your life." +LJ008-0317|The fatal news was not always received in the same way.|The fatal news was not always received in the same way. +LJ008-0318|The men who were doomed often fell down in convulsions upon the floor.|The men who were doomed often fell down in convulsions upon the floor. +LJ008-0319|Sometimes any who had had a narrow escape fainted, but the bulk of those respited looked on with unfeeling indifference.|Sometimes any who had had a narrow escape fainted, but the bulk of those respited looked on with unfeeling indifference. +LJ009-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 12: Executions, part two.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section twelve: Executions, part two. +LJ009-0002|It is satisfactory to be able to record that some consideration was shown the capital convict actually awaiting execution.|It is satisfactory to be able to record that some consideration was shown the capital convict actually awaiting execution. +LJ009-0003|Even so severe a critic as Mr. Wakefield states that "a stranger to the scene|Even so severe a critic as Mr. Wakefield states that "a stranger to the scene +LJ009-0004|would be astonished to observe the peculiar tenderness, I was going to add respect,|would be astonished to observe the peculiar tenderness, I was going to add respect, +LJ009-0005|which persons under sentence of death obtain from all the officers of the prison.|which persons under sentence of death obtain from all the officers of the prison. +LJ009-0006|Before sentence a prisoner has only to observe the regulations of the jail in order to remain neglected and unnoticed.|Before sentence a prisoner has only to observe the regulations of the jail in order to remain neglected and unnoticed. +LJ009-0007|Once ordered to the cells, friends of all classes suddenly rise up; his fellow-prisoners, the turnkeys, the chaplain,|Once ordered to the cells, friends of all classes suddenly rise up; his fellow-prisoners, the turnkeys, the chaplain, +LJ009-0008|the keepers, and the sheriffs all seem interested in his fate, and he can make no reasonable request that is not at once granted by whomsoever he may address.|the keepers, and the sheriffs all seem interested in his fate, and he can make no reasonable request that is not at once granted by whomsoever he may address. +LJ009-0009|This rule has some, but very few, exceptions; such as where a hardened offender behaves with great levity and brutality, as if he cared nought for his life,|This rule has some, but very few, exceptions; such as where a hardened offender behaves with great levity and brutality, as if he cared nought for his life, +LJ009-0010|and thought every one anxious to promote his death.|and thought every one anxious to promote his death. +LJ009-0011|Mr. Wakefield goes on to remark that persons convicted of forgery "excited an extraordinary degree of interest in all who approached them."|Mr. Wakefield goes on to remark that persons convicted of forgery "excited an extraordinary degree of interest in all who approached them." +LJ009-0012|This was noticeable with Fauntleroy, who, on account of his birth and antecedents, was allowed to occupy a turnkey's room,|This was noticeable with Fauntleroy, who, on account of his birth and antecedents, was allowed to occupy a turnkey's room, +LJ009-0013|and kept altogether separate from the other prisoners until the day of his death.|and kept altogether separate from the other prisoners until the day of his death. +LJ009-0014|It cannot be denied, however, that the ordinary's treatment was somewhat unfeeling, and in proof thereof|It cannot be denied, however, that the ordinary's treatment was somewhat unfeeling, and in proof thereof +LJ009-0015|I will quote an extract from the reverend gentleman's own journal.|I will quote an extract from the reverend gentleman's own journal. +LJ009-0016|He seems to have improved the occasion when preaching the condemned sermon before Fauntleroy, by pointing a moral from that unhappy man's own case.|He seems to have improved the occasion when preaching the condemned sermon before Fauntleroy, by pointing a moral from that unhappy man's own case. +LJ009-0017|For this the chaplain was a few days later summoned before the jail committee of aldermen,|For this the chaplain was a few days later summoned before the jail committee of aldermen, +LJ009-0018|and informed that the public would not in future be admitted to hear the condemned sermon. "I was also informed," writes Mr. Cotton,|and informed that the public would not in future be admitted to hear the condemned sermon. "I was also informed," writes Mr. Cotton, +LJ009-0019|that this resolution was in consequence of their (the aldermen's) disapproving of the last discourse delivered by me,|that this resolution was in consequence of their (the aldermen's) disapproving of the last discourse delivered by me, +LJ009-0020|previous to the execution of Henry Fauntleroy for uttering a forged security|previous to the execution of Henry Fauntleroy for uttering a forged security +LJ009-0021|in which it was said I had enlarged upon the heinous nature of his crime, and warned the public to avoid such conduct.|in which it was said I had enlarged upon the heinous nature of his crime, and warned the public to avoid such conduct. +LJ009-0022|I was informed that this unnecessarily harassed his feelings, and that the object of such sermons was solely to console the prisoner,|I was informed that this unnecessarily harassed his feelings, and that the object of such sermons was solely to console the prisoner, +LJ009-0023|and that from the time of his conviction nothing but what is consolatory should be addressed to a criminal.|and that from the time of his conviction nothing but what is consolatory should be addressed to a criminal. +LJ009-0024|One of the aldermen, moreover, informed me that the whole court of aldermen were unanimous in their opinion on this subject.|One of the aldermen, moreover, informed me that the whole court of aldermen were unanimous in their opinion on this subject. +LJ009-0025|As to the exclusion of strangers on these occasions,|As to the exclusion of strangers on these occasions, +LJ009-0026|the experience I have had convinces me that one, and perhaps the only, good of an execution, i. e. the solemn admonition to the public,|the experience I have had convinces me that one, and perhaps the only, good of an execution, i. e. the solemn admonition to the public, +LJ009-0027|will thereby be lost.|will thereby be lost. +LJ009-0028|Probably the reader will side with the aldermen against the ordinary.|Probably the reader will side with the aldermen against the ordinary. +LJ009-0029|This episode throws some doubt upon the tenderness and proper feeling exhibited by the chaplain towards the most deserving members of his criminal flock;|This episode throws some doubt upon the tenderness and proper feeling exhibited by the chaplain towards the most deserving members of his criminal flock; +LJ009-0030|and the idea will be strengthened by the following account of the Sunday service in the prison chapel on the occasion when the condemned sermon was preached.|and the idea will be strengthened by the following account of the Sunday service in the prison chapel on the occasion when the condemned sermon was preached. +LJ009-0031|The extract is from Mr. E. Gibbon Wakefield's brochure, the date 1828, just three years after Fauntleroy's death.|The extract is from Mr. E. Gibbon Wakefield's brochure, the date eighteen twenty-eight, just three years after Fauntleroy's death. +LJ009-0032|Strangers were now excluded, but the sheriffs attended in state, wearing their gold chains,|Strangers were now excluded, but the sheriffs attended in state, wearing their gold chains, +LJ009-0033|while behind their pew stood a couple of tall footmen in state liveries.|while behind their pew stood a couple of tall footmen in state liveries. +LJ009-0034|The sheriffs were in one gallery;|The sheriffs were in one gallery; +LJ009-0035|in the other opposite were the convicts capitally convicted who had been respited.|in the other opposite were the convicts capitally convicted who had been respited. +LJ009-0036|Down below between the galleries was the mass of the prison population;|Down below between the galleries was the mass of the prison population; +LJ009-0037|the schoolmaster and the juvenile prisoners being seated round the communion-table, opposite the pulpit.|the schoolmaster and the juvenile prisoners being seated round the communion-table, opposite the pulpit. +LJ009-0038|In the center of the chapel was the condemned pew, a large dock-like erection painted black.|In the center of the chapel was the condemned pew, a large dock-like erection painted black. +LJ009-0039|Those who sat in it were visible to the whole congregation, and still more to the ordinary,|Those who sat in it were visible to the whole congregation, and still more to the ordinary, +LJ009-0040|whose desk and pulpit were just in front of the condemned pew, and within a couple of yards of it.|whose desk and pulpit were just in front of the condemned pew, and within a couple of yards of it. +LJ009-0041|The occupants of this terrible black pew were the last always to enter the chapel.|The occupants of this terrible black pew were the last always to enter the chapel. +LJ009-0042|Upon the occasion which I am describing they were four in number; and here I will continue the narrative in Mr. Wakefield's own words:|Upon the occasion which I am describing they were four in number; and here I will continue the narrative in Mr. Wakefield's own words: +LJ009-0043|First is a youth of eighteen, condemned for stealing in a dwelling-house goods valued above five pounds.|First is a youth of eighteen, condemned for stealing in a dwelling-house goods valued above five pounds. +LJ009-0044|His features have no felonious cast;|His features have no felonious cast; +LJ009-0045|he steps boldly with head upright, looks to the women's gallery, and smiles. His intention is to pass for a brave fellow,|he steps boldly with head upright, looks to the women's gallery, and smiles. His intention is to pass for a brave fellow, +LJ009-0046|but the attempt fails; he trembles, his knees knock together, and his head droops as he enters the condemned pew.|but the attempt fails; he trembles, his knees knock together, and his head droops as he enters the condemned pew. +LJ009-0047|The next convict is clearly and unmistakably a villain.|The next convict is clearly and unmistakably a villain. +LJ009-0048|He is a hardened offender, previously cast for life, reprieved, transported to Australia, and since returned without pardon.|He is a hardened offender, previously cast for life, reprieved, transported to Australia, and since returned without pardon. +LJ009-0049|For this offense the punishment is death.|For this offense the punishment is death. +LJ009-0050|He has, however, doubly earned his sentence, and is actually condemned for burglary committed since his arrival in England.|He has, however, doubly earned his sentence, and is actually condemned for burglary committed since his arrival in England. +LJ009-0051|His look at the sheriffs and the ordinary is full of scorn and defiance.|His look at the sheriffs and the ordinary is full of scorn and defiance. +LJ009-0052|The third convict is a sheep-stealer, a poor ignorant fellow in whose crime are mitigating circumstances,|The third convict is a sheep-stealer, a poor ignorant fellow in whose crime are mitigating circumstances, +LJ009-0053|but who is left to die on the supposition that this is not his first conviction, and still more because a good many sheep have of late been stolen by other people.|but who is left to die on the supposition that this is not his first conviction, and still more because a good many sheep have of late been stolen by other people. +LJ009-0054|He is quite content to die;|He is quite content to die; +LJ009-0055|indeed the chaplain and others have brought him firmly to believe that his situation is enviable, and that the gates of heaven are open to receive him.|indeed the chaplain and others have brought him firmly to believe that his situation is enviable, and that the gates of heaven are open to receive him. +LJ009-0056|The last of the four is said to have been a clergyman of the Church of England, condemned for forgery, "a miserable old man in a tattered suit of black.|The last of the four is said to have been a clergyman of the Church of England, condemned for forgery, "a miserable old man in a tattered suit of black. +LJ009-0057|Already he is half dead.|Already he is half dead. +LJ009-0058|Great efforts have been made to save his life.|Great efforts have been made to save his life. +LJ009-0059|Friends, even utter strangers, have interceded for him, and to the last he has buoyed himself up by hope of reprieve.|Friends, even utter strangers, have interceded for him, and to the last he has buoyed himself up by hope of reprieve. +LJ009-0060|Now his doom is sealed irrevocably, and he has given himself up to despair.|Now his doom is sealed irrevocably, and he has given himself up to despair. +LJ009-0061|He staggers towards the pew, reels into it, stumbles forward, flings himself on the ground, and, by a curious twist of the spine,|He staggers towards the pew, reels into it, stumbles forward, flings himself on the ground, and, by a curious twist of the spine, +LJ009-0062|buries his head under his body.|buries his head under his body. +LJ009-0063|The sheriffs shudder, their inquisitive friends crane forward, the keeper frowns on the excited congregation,|The sheriffs shudder, their inquisitive friends crane forward, the keeper frowns on the excited congregation, +LJ009-0064|the lately smirking footmen close their eyes and forget their liveries, the ordinary clasps his hands, the turnkeys cry 'Hush!'|the lately smirking footmen close their eyes and forget their liveries, the ordinary clasps his hands, the turnkeys cry 'Hush!' +LJ009-0065|and the old clerk lifts up his cracked voice, saying, 'Let us sing to the praise and glory of God.'|and the old clerk lifts up his cracked voice, saying, 'Let us sing to the praise and glory of God.' +LJ009-0066|The morning hymn is sung first, as if to remind the condemned that next morning at eight a.m. they are to die.|The morning hymn is sung first, as if to remind the condemned that next morning at eight a.m. they are to die. +LJ009-0067|The service proceeds. At last the burial service is reached.|The service proceeds. At last the burial service is reached. +LJ009-0068|The youth alone is able to read, but from long want of practice he is at a loss to find the place in his prayer-book.|The youth alone is able to read, but from long want of practice he is at a loss to find the place in his prayer-book. +LJ009-0069|The ordinary observes him, looks to the sheriffs, and says aloud, 'The service for the dead!'|The ordinary observes him, looks to the sheriffs, and says aloud, 'The service for the dead!' +LJ009-0070|The youth's hands tremble as they hold the book upside down.|The youth's hands tremble as they hold the book upside down. +LJ009-0071|The burglar is heard to mutter an angry oath.|The burglar is heard to mutter an angry oath. +LJ009-0072|The sheep-stealer smiles, and, extending his arms upwards, looks with a glad expression to the roof of the chapel.|The sheep-stealer smiles, and, extending his arms upwards, looks with a glad expression to the roof of the chapel. +LJ009-0073|The forger has never moved.|The forger has never moved. +LJ009-0074|Let us pass on.|Let us pass on. +LJ009-0075|All have sung 'the Lamentation of a Sinner,' and have seemed to pray 'especially for those now awaiting the awful execution of the law.'|All have sung 'the Lamentation of a Sinner,' and have seemed to pray 'especially for those now awaiting the awful execution of the law.' +LJ009-0076|We come to the sermon.|We come to the sermon. +LJ009-0077|The ordinary of Newgate is an orthodox, unaffected, Church of England divine,|The ordinary of Newgate is an orthodox, unaffected, Church of England divine, +LJ009-0078|who preaches plain, homely discourses, as fit as any religious discourse can be fit for the irritated audience.|who preaches plain, homely discourses, as fit as any religious discourse can be fit for the irritated audience. +LJ009-0079|The sermon of this day, whether eloquent or plain, useful or useless, must produce a striking effect at the moment of its delivery.|The sermon of this day, whether eloquent or plain, useful or useless, must produce a striking effect at the moment of its delivery. +LJ009-0080|The text, without another word, is enough to raise the wildest passions of the audience|The text, without another word, is enough to raise the wildest passions of the audience +LJ009-0081|For a while the preacher addresses himself to the congregation at large, who listen attentively|For a while the preacher addresses himself to the congregation at large, who listen attentively +LJ009-0082|except the clergyman and the burglar, the former of whom is still rolled up at the bottom of the condemned pew,|except the clergyman and the burglar, the former of whom is still rolled up at the bottom of the condemned pew, +LJ009-0083|while the eyes of the latter are wandering round the chapel, and one of them is occasionally winked impudently at some acquaintance amongst the prisoners for trial.|while the eyes of the latter are wandering round the chapel, and one of them is occasionally winked impudently at some acquaintance amongst the prisoners for trial. +LJ009-0084|At length the ordinary pauses, and then, in a deep tone, which, though hardly above a whisper, is audible to all, says,|At length the ordinary pauses, and then, in a deep tone, which, though hardly above a whisper, is audible to all, says, +LJ009-0085|'Now for you, my poor fellow mortals, who are about to suffer the last penalty of the law.'|'Now for you, my poor fellow mortals, who are about to suffer the last penalty of the law.' +LJ009-0086|But why should I repeat the whole?|But why should I repeat the whole? +LJ009-0087|It is enough to say that in the same solemn tone he talks about the minutest of crimes, punishments, bonds, shame,|It is enough to say that in the same solemn tone he talks about the minutest of crimes, punishments, bonds, shame, +LJ009-0088|ignominy, sorrow, sufferings, wretchedness, pangs,|ignominy, sorrow, sufferings, wretchedness, pangs, +LJ009-0089|childless parents, widows and helpless orphans, broken and contrite hearts, and death tomorrow morning for the benefit of society.|childless parents, widows and helpless orphans, broken and contrite hearts, and death tomorrow morning for the benefit of society. +LJ009-0090|The dying men are dreadfully agitated.|The dying men are dreadfully agitated. +LJ009-0091|The young stealer in a dwelling-house no longer has the least pretense to bravery. He grasps the back of the pew,|The young stealer in a dwelling-house no longer has the least pretense to bravery. He grasps the back of the pew, +LJ009-0092|his legs give way, he utters a faint groan, and sinks on the floor.|his legs give way, he utters a faint groan, and sinks on the floor. +LJ009-0093|Why does no one stir to help him? Where would be the use? The hardened burglar moves not, nor does he speak;|Why does no one stir to help him? Where would be the use? The hardened burglar moves not, nor does he speak; +LJ009-0094|but his face is of an ashy paleness; and if you look carefully you may see the blood trickling from his lip,|but his face is of an ashy paleness; and if you look carefully you may see the blood trickling from his lip, +LJ009-0095|which he has bitten unconsciously, or from rage, or to rouse his fainting courage.|which he has bitten unconsciously, or from rage, or to rouse his fainting courage. +LJ009-0096|The poor sheep-stealer is in a frenzy.|The poor sheep-stealer is in a frenzy. +LJ009-0097|He throws his hands far from him, and shouts aloud, 'Mercy, good Lord! mercy is all I ask. The Lord in His mercy come!|He throws his hands far from him, and shouts aloud, 'Mercy, good Lord! mercy is all I ask. The Lord in His mercy come! +LJ009-0098|There! there! I see the Lamb of God! Oh! how happy! Oh! this is happy!|There! there! I see the Lamb of God! Oh! how happy! Oh! this is happy! +LJ009-0099|Meanwhile the clergyman, still bent into the form of a sleeping dog,|Meanwhile the clergyman, still bent into the form of a sleeping dog, +LJ009-0100|struggles violently; his feet, legs, hands, and arms, even the muscles of his back,|struggles violently; his feet, legs, hands, and arms, even the muscles of his back, +LJ009-0101|move with a quick, jerking motion, not naturally, but, as it were, like the affected parts of a galvanized corpse.|move with a quick, jerking motion, not naturally, but, as it were, like the affected parts of a galvanized corpse. +LJ009-0102|Suddenly he utters a short sharp scream, and all is still.|Suddenly he utters a short sharp scream, and all is still. +LJ009-0103|The silence is short. As the ordinary proceeds 'to conclude,'|The silence is short. As the ordinary proceeds 'to conclude,' +LJ009-0104|the women set up a yell, which is mixed with a rustling noise, occasioned by the removal of those whose hysterics have ended in fainting.|the women set up a yell, which is mixed with a rustling noise, occasioned by the removal of those whose hysterics have ended in fainting. +LJ009-0105|The sheriffs cover their faces, and one of their inquisitive friends blows his nose with his glove.|The sheriffs cover their faces, and one of their inquisitive friends blows his nose with his glove. +LJ009-0106|The keeper tries to appear unmoved, but his eye wanders anxiously over the combustible assembly.|The keeper tries to appear unmoved, but his eye wanders anxiously over the combustible assembly. +LJ009-0107|The children round the communion-table stare and gape with childish wonder.|The children round the communion-table stare and gape with childish wonder. +LJ009-0108|The two masses of prisoners for trial undulate and slightly murmur,|The two masses of prisoners for trial undulate and slightly murmur, +LJ009-0109|while the capital convicts who were lately in that black pew appear faint with emotion.|while the capital convicts who were lately in that black pew appear faint with emotion. +LJ009-0110|This exhibition lasts for some minutes, and then the congregation disperses,|This exhibition lasts for some minutes, and then the congregation disperses, +LJ009-0111|the condemned returning to the cells: the forger carried by turnkeys; the youth sobbing aloud convulsively, as a passionate child;|the condemned returning to the cells: the forger carried by turnkeys; the youth sobbing aloud convulsively, as a passionate child; +LJ009-0112|the burglar muttering curses and savage expressions of defiance; whilst the poor sheep-stealer shakes hands with the turnkeys,|the burglar muttering curses and savage expressions of defiance; whilst the poor sheep-stealer shakes hands with the turnkeys, +LJ009-0113|whistles merrily, and points upwards with madness in his look.|whistles merrily, and points upwards with madness in his look. +LJ009-0114|Mr. Wakefield winds up his graphic but somewhat sensational account by describing another religious service, which may appropriately be inserted here.|Mr. Wakefield winds up his graphic but somewhat sensational account by describing another religious service, which may appropriately be inserted here. +LJ009-0115|He says, "On the day of execution there is no service in the chapel of Newgate.|He says, "On the day of execution there is no service in the chapel of Newgate. +LJ009-0116|On the following day the capital convicts, whose companions have been hanged, are required to return thanks for their narrow escape.|On the following day the capital convicts, whose companions have been hanged, are required to return thanks for their narrow escape. +LJ009-0117|The firmest disbeliever in religion, if he had not lately been irritated by taking part in such a scene as the condemned service in Newgate,|The firmest disbeliever in religion, if he had not lately been irritated by taking part in such a scene as the condemned service in Newgate, +LJ009-0118|could hardly witness this ceremony without being affected. The men, who were so lately snatched from the jaws of death,|could hardly witness this ceremony without being affected. The men, who were so lately snatched from the jaws of death, +LJ009-0119|kneel, whilst the rest of the congregation sit, and the ordinary, in a tone of peculiar solemnity, says,|kneel, whilst the rest of the congregation sit, and the ordinary, in a tone of peculiar solemnity, says, +LJ009-0120|Almighty God, Father of all mercies, we thine unworthy servants do give thee most humble and hearty thanks|Almighty God, Father of all mercies, we thine unworthy servants do give thee most humble and hearty thanks +LJ009-0121|for all thy goodness and loving-kindness to us, and to all men;|for all thy goodness and loving-kindness to us, and to all men; +LJ009-0122|particularly to those who desire now to offer up their praises and thanksgivings for thy late mercies vouchsafed unto them.|particularly to those who desire now to offer up their praises and thanksgivings for thy late mercies vouchsafed unto them. +LJ009-0123|Could any one, knowing the late situation of the kneeling men, looking as they do at the empty pew,|Could any one, knowing the late situation of the kneeling men, looking as they do at the empty pew, +LJ009-0124|occupied when they saw it last, but a few hours ago, by their comrades who are now dead;|occupied when they saw it last, but a few hours ago, by their comrades who are now dead; +LJ009-0125|could any one, not disgusted with the religious ceremonials of Newgate, witness this scene without emotion?|could any one, not disgusted with the religious ceremonials of Newgate, witness this scene without emotion? +LJ009-0126|Hardly any one.|Hardly any one. +LJ009-0127|But what are the feelings of those who take part in it?|But what are the feelings of those who take part in it? +LJ009-0128|I have been present at the scene not less than twenty times, and have invariably observed|I have been present at the scene not less than twenty times, and have invariably observed +LJ009-0129|that many of the kneeling men or boys laughed while they knelt, pinched each other, and, when they could do so without fear of being seen by any officer of the prison,|that many of the kneeling men or boys laughed while they knelt, pinched each other, and, when they could do so without fear of being seen by any officer of the prison, +LJ009-0130|winked at other prisoners in derision of what was taking place; and I have frequently heard men and lads who had been of the kneeling party|winked at other prisoners in derision of what was taking place; and I have frequently heard men and lads who had been of the kneeling party +LJ009-0131|boast to their companions after the service that they had wiped their eyes during the thanksgiving, to make the ordinary believe they had been crying.|boast to their companions after the service that they had wiped their eyes during the thanksgiving, to make the ordinary believe they had been crying. +LJ009-0132|Although this misapplication of religious services still went on,|Although this misapplication of religious services still went on, +LJ009-0133|the outside public continued to be excluded from the Newgate chapel on the day the condemned sermon was preached.|the outside public continued to be excluded from the Newgate chapel on the day the condemned sermon was preached. +LJ009-0134|This very proper rule was, however, set aside on the Sunday preceding Courvoisier's execution.|This very proper rule was, however, set aside on the Sunday preceding Courvoisier's execution. +LJ009-0135|So many applications for admission were made to the sheriffs,|So many applications for admission were made to the sheriffs, +LJ009-0136|that they reluctantly agreed to open the gallery which had formerly been occupied by strangers on these occasions.|that they reluctantly agreed to open the gallery which had formerly been occupied by strangers on these occasions. +LJ009-0137|Cards were issued, and to such an extent, that although the service was not to commence till half-past ten, by nine a.m.|Cards were issued, and to such an extent, that although the service was not to commence till half-past ten, by nine a.m. +LJ009-0138|all the avenues to the prison gates were blocked by ticket-holders.|all the avenues to the prison gates were blocked by ticket-holders. +LJ009-0139|In spite of the throng, owing to the excellent arrangements made by the sheriffs,|In spite of the throng, owing to the excellent arrangements made by the sheriffs, +LJ009-0140|no inconvenience was suffered by the congregation, among whom were Lord Adolphus Fitz Clarence, Lord Coventry,|no inconvenience was suffered by the congregation, among whom were Lord Adolphus Fitz Clarence, Lord Coventry, +LJ009-0141|Lord Paget, Lord Bruce, several members of the House of Commons, and a few ladies.|Lord Paget, Lord Bruce, several members of the House of Commons, and a few ladies. +LJ009-0142|Contemporary accounts give a minute description of the demeanor of the convict upon this solemn occasion.|Contemporary accounts give a minute description of the demeanor of the convict upon this solemn occasion. +LJ009-0143|He sat on a bench before the pulpit, -- the hideous condemned pew had been swept away, -- and never once raised his eyes during the service.|He sat on a bench before the pulpit, -- the hideous condemned pew had been swept away, -- and never once raised his eyes during the service. +LJ009-0144|In fact his looks denoted extreme sorrow and contrition,|In fact his looks denoted extreme sorrow and contrition, +LJ009-0145|and he seemed to suffer great inward agitation when the ordinary particularly alluded to the crime for the perpetration of which he stood condemned.|and he seemed to suffer great inward agitation when the ordinary particularly alluded to the crime for the perpetration of which he stood condemned. +LJ009-0146|Mr. Carver, the ordinary, appears to have addressed himself directly to Courvoisier,|Mr. Carver, the ordinary, appears to have addressed himself directly to Courvoisier, +LJ009-0147|and to have dwelt with more emphasis than good taste upon the nature of the crime, and the necessity for repentance.|and to have dwelt with more emphasis than good taste upon the nature of the crime, and the necessity for repentance. +LJ009-0148|But the chaplain admitted that the solitude of the convict's cell|But the chaplain admitted that the solitude of the convict's cell +LJ009-0149|was more appropriate for serious reflection and profitable ministration than this exciting occasion before a large and public assembly.|was more appropriate for serious reflection and profitable ministration than this exciting occasion before a large and public assembly. +LJ009-0150|So far as I can find, Courvoisier was the last condemned criminal who was thus exhibited to a crowd of morbidly curious spectators.|So far as I can find, Courvoisier was the last condemned criminal who was thus exhibited to a crowd of morbidly curious spectators. +LJ009-0151|The atrocity of the murder no doubt attracted extraordinary attention to it.|The atrocity of the murder no doubt attracted extraordinary attention to it. +LJ009-0152|The crowd outside Newgate on the day of execution has already been described; but there was also a select gathering of distinguished visitors within the jail.|The crowd outside Newgate on the day of execution has already been described; but there was also a select gathering of distinguished visitors within the jail. +LJ009-0153|First came the sheriffs, the under-sheriffs, and several aldermen and city officials, then Lord Powerscourt and several other peers of the realm.|First came the sheriffs, the under-sheriffs, and several aldermen and city officials, then Lord Powerscourt and several other peers of the realm. +LJ009-0154|Mr. Charles Kean the tragedian was also present,|Mr. Charles Kean the tragedian was also present, +LJ009-0155|drawn to this terrible exhibition by the example of his father, the more celebrated Edmund Kean, who had witnessed the execution of Thistlewood|drawn to this terrible exhibition by the example of his father, the more celebrated Edmund Kean, who had witnessed the execution of Thistlewood +LJ009-0156|"with a view," as he himself said, "to his professional studies."|"with a view," as he himself said, "to his professional studies." +LJ009-0157|But there is little doubt that as executions became more rare they made more impression on the public mind.|But there is little doubt that as executions became more rare they made more impression on the public mind. +LJ009-0158|Already a strong dislike to the reckless and almost indiscriminate application of the extreme penalty was apparent in all classes,|Already a strong dislike to the reckless and almost indiscriminate application of the extreme penalty was apparent in all classes, +LJ009-0159|and the mitigation of the criminal code, for which Romilly had so strenuously labored, was daily more and more of an accomplished fact.|and the mitigation of the criminal code, for which Romilly had so strenuously labored, was daily more and more of an accomplished fact. +LJ009-0160|In 1832|In eighteen thirty-two +LJ009-0161|capital punishment was abolished for forgery, except in cases of forging or altering wills or powers of attorney to transfer stock.|capital punishment was abolished for forgery, except in cases of forging or altering wills or powers of attorney to transfer stock. +LJ009-0162|Nevertheless, after that date no person whatever was executed for this offense.|Nevertheless, after that date no person whatever was executed for this offense. +LJ009-0163|In the same year capital punishment was further restricted, and ceased to be the legal sentence for coining,|In the same year capital punishment was further restricted, and ceased to be the legal sentence for coining, +LJ009-0164|sheep or horse stealing, and stealing in a dwelling-house.|sheep or horse stealing, and stealing in a dwelling-house. +LJ009-0165|House-breaking, as distinguished from burglary, was similarly exempted in the following year;|House-breaking, as distinguished from burglary, was similarly exempted in the following year; +LJ009-0166|next, the offenses of returning from transportation, stealing post-office letters, and sacrilege were no longer punishable with death.|next, the offenses of returning from transportation, stealing post-office letters, and sacrilege were no longer punishable with death. +LJ009-0167|In 1837 Lord John Russell's acts swept away a number of capital offenses, including cutting and maiming, rick-burning,|In eighteen thirty-seven Lord John Russell's acts swept away a number of capital offenses, including cutting and maiming, rick-burning, +LJ009-0168|robbery, burglary, and arson.|robbery, burglary, and arson. +LJ009-0169|Within a couple of years the number of persons sentenced to death in England had fallen from four hundred and thirty-eight in 1837|Within a couple of years the number of persons sentenced to death in England had fallen from four hundred and thirty-eight in eighteen thirty-seven +LJ009-0170|to fifty-six in 1839.|to fifty-six in eighteen thirty-nine. +LJ009-0171|Gradually the application of capital punishment became more and more restricted, and was soon the penalty for murder alone.|Gradually the application of capital punishment became more and more restricted, and was soon the penalty for murder alone. +LJ009-0172|While in London, for instance, in 1829, twenty-four persons had been executed for crimes other than murder,|While in London, for instance, in eighteen twenty-nine, twenty-four persons had been executed for crimes other than murder, +LJ009-0173|from 1832 to 1844 not a single person had been executed in the metropolis except for this the gravest crime.|from eighteen thirty-two to eighteen forty-four not a single person had been executed in the metropolis except for this the gravest crime. +LJ009-0174|In 1837 the death penalty was practically limited to murder or attempts to murder,|In eighteen thirty-seven the death penalty was practically limited to murder or attempts to murder, +LJ009-0175|and in 1841 this was accepted as the almost universally established rule.|and in eighteen forty-one this was accepted as the almost universally established rule. +LJ009-0176|Seven other crimes, however, were still capital by law, and so continued till the passing of the Criminal Consolidation Acts of 1861.|Seven other crimes, however, were still capital by law, and so continued till the passing of the Criminal Consolidation Acts of eighteen sixty-one. +LJ009-0177|With the amelioration of the criminal code, other cruel concomitants of execution also disappeared.|With the amelioration of the criminal code, other cruel concomitants of execution also disappeared. +LJ009-0178|In 1832 the dissection of bodies cut down from the gallows, which had been decreed centuries previously, was abolished;|In eighteen thirty-two the dissection of bodies cut down from the gallows, which had been decreed centuries previously, was abolished; +LJ009-0179|the most recent enactment in force was the 9th George IV. cap. 31, which directed the dissection of all bodies of executed murderers,|the most recent enactment in force was the ninth George the fourth cap. thirty-one, which directed the dissection of all bodies of executed murderers, +LJ009-0180|the idea being to intensify the dread of capital punishment. That such dread was not universal or deep-seated may be gathered from the fact|the idea being to intensify the dread of capital punishment. That such dread was not universal or deep-seated may be gathered from the fact +LJ009-0181|that authentic cases were known previous to the first cited act of criminals selling their own bodies to surgeons for dissection.|that authentic cases were known previous to the first cited act of criminals selling their own bodies to surgeons for dissection. +LJ009-0182|This dissection was carried out for Newgate prisoners in Surgeons' Hall, adjoining Newgate,|This dissection was carried out for Newgate prisoners in Surgeons' Hall, adjoining Newgate, +LJ009-0183|the site of the present Sessions House of the Old Bailey, and the operation was witnessed by students and a number of curious spectators.|the site of the present Sessions House of the Old Bailey, and the operation was witnessed by students and a number of curious spectators. +LJ009-0184|Lord Ferrers' body was brought to Surgeons' Hall after execution in his own carriage and six;|Lord Ferrers' body was brought to Surgeons' Hall after execution in his own carriage and six; +LJ009-0185|after the post mortem had been carried out, the corpse was exposed to view in a first-floor room.|after the post mortem had been carried out, the corpse was exposed to view in a first-floor room. +LJ009-0186|Pennant speaks of Surgeons' Hall as a handsome building, ornamented with Ionic pilasters, and with a double flight of steps to the first floor.|Pennant speaks of Surgeons' Hall as a handsome building, ornamented with Ionic pilasters, and with a double flight of steps to the first floor. +LJ009-0187|Beneath is a door for the admission of the bodies of murderers and other felons. There were other public dissecting rooms for criminals.|Beneath is a door for the admission of the bodies of murderers and other felons. There were other public dissecting rooms for criminals. +LJ009-0188|One was attached to Hicks' Hall, the Clerkenwell Sessions House, built out of monies provided by Sir Baptist Hicks, a wealthy alderman of the reign of James I.|One was attached to Hicks' Hall, the Clerkenwell Sessions House, built out of monies provided by Sir Baptist Hicks, a wealthy alderman of the reign of James the first +LJ009-0189|Persons were still living in 1855 who had witnessed dissections at Hicks' Hall, and|Persons were still living in eighteen fifty-five who had witnessed dissections at Hicks' Hall, and +LJ009-0190|whom the horrid scene, with the additional effect of some noted criminals hanging on the walls, drove out again sick and faint,|whom the horrid scene, with the additional effect of some noted criminals hanging on the walls, drove out again sick and faint, +LJ009-0191|as we have heard some relate, and with pale and terrified features, to get a breath of air.|as we have heard some relate, and with pale and terrified features, to get a breath of air. +LJ009-0192|The dissection of executed criminals was abolished soon after the discovery of the crime of burking,|The dissection of executed criminals was abolished soon after the discovery of the crime of burking, +LJ009-0193|with the idea that ignominy would no longer attach to an operation which ceased to be compulsory for the most degraded beings;|with the idea that ignominy would no longer attach to an operation which ceased to be compulsory for the most degraded beings; +LJ009-0194|and that executors or persons having lawful possession of the bodies|and that executors or persons having lawful possession of the bodies +LJ009-0195|of people who had died friendless, would voluntarily surrender them for the advancement of medical science.|of people who had died friendless, would voluntarily surrender them for the advancement of medical science. +LJ009-0196|Another brutal practice had nearly disappeared about the time of the abolition of dissection. This was the public exhibition of the body,|Another brutal practice had nearly disappeared about the time of the abolition of dissection. This was the public exhibition of the body, +LJ009-0197|as was done in the case of Mrs. Phipoe, the murderess, who was executed in front of Newgate in 1798,|as was done in the case of Mrs. Phipoe, the murderess, who was executed in front of Newgate in seventeen ninety-eight, +LJ009-0198|and her body publicly exhibited in a place built for the purpose in the Old Bailey.|and her body publicly exhibited in a place built for the purpose in the Old Bailey. +LJ009-0199|About this time I find that the bodies of two murderers, Clench and Mackay,|About this time I find that the bodies of two murderers, Clench and Mackay, +LJ009-0200|were publicly exposed in a stable in Little Bridge Street, near Apothecaries' Hall,|were publicly exposed in a stable in Little Bridge Street, near Apothecaries' Hall, +LJ009-0201|Surgeons' Hall being let to the lieutenancy of the county for the accommodation of the militia.|Surgeons' Hall being let to the lieutenancy of the county for the accommodation of the militia. +LJ009-0202|In 1811 Williams, who murdered the Marrs in Ratcliffe Highway, having committed suicide in jail to escape hanging,|In eighteen eleven Williams, who murdered the Marrs in Ratcliffe Highway, having committed suicide in jail to escape hanging, +LJ009-0203|it was determined that a public exhibition should be made of the body through the neighborhood which had been the scene of the monster's crimes.|it was determined that a public exhibition should be made of the body through the neighborhood which had been the scene of the monster's crimes. +LJ009-0204|A long procession was formed, headed by constables, who cleared the way with their staves.|A long procession was formed, headed by constables, who cleared the way with their staves. +LJ009-0205|Then came the newly-formed horse patrol, with drawn cutlasses,|Then came the newly-formed horse patrol, with drawn cutlasses, +LJ009-0206|parish officers, peace officers, the high constable of the county of Middlesex on horseback, and then the body of Williams,|parish officers, peace officers, the high constable of the county of Middlesex on horseback, and then the body of Williams, +LJ009-0207|extended at full length on an inclined platform|extended at full length on an inclined platform +LJ009-0208|erected on the cart, about four feet high at the head, and gradually sloping towards the horse, giving a full view of the body,|erected on the cart, about four feet high at the head, and gradually sloping towards the horse, giving a full view of the body, +LJ009-0209|which was dressed in blue trousers and a blue-and-white striped waistcoat, but without a coat, as when found in the cell.|which was dressed in blue trousers and a blue-and-white striped waistcoat, but without a coat, as when found in the cell. +LJ009-0210|On the left side of the head the fatal mall,|On the left side of the head the fatal mall, +LJ009-0211|and on the right the ripping chisel, with which the murders had been committed, were exposed to view.|and on the right the ripping chisel, with which the murders had been committed, were exposed to view. +LJ009-0212|The countenance of Williams was ghastly in the extreme, and the whole had an appearance too horrible for description.|The countenance of Williams was ghastly in the extreme, and the whole had an appearance too horrible for description. +LJ009-0213|The procession traversed Ratcliffe twice, halting for a quarter of an hour in front of the victims' dwelling,|The procession traversed Ratcliffe twice, halting for a quarter of an hour in front of the victims' dwelling, +LJ009-0214|and was accompanied throughout by "an immense concourse of persons, eager to get a sight of the murderer's remains.|and was accompanied throughout by "an immense concourse of persons, eager to get a sight of the murderer's remains. +LJ009-0215|All the shops in the neighborhood were shut, and the windows and tops of the houses were crowded with spectators.|All the shops in the neighborhood were shut, and the windows and tops of the houses were crowded with spectators. +LJ009-0216|Hanging in chains upon the gibbet which had served for the execution,|Hanging in chains upon the gibbet which had served for the execution, +LJ009-0217|or on another specially erected on some commanding spot, had fallen into disuse by 1832.|or on another specially erected on some commanding spot, had fallen into disuse by eighteen thirty-two. +LJ009-0218|But there was an attempt to revive it at that date, when the act for dispensing with the dissection of criminals was passed.|But there was an attempt to revive it at that date, when the act for dispensing with the dissection of criminals was passed. +LJ009-0219|A clause was inserted to the effect that|A clause was inserted to the effect that +LJ009-0220|the bodies of all prisoners convicted of murder should either be hung in chains, or buried under the gallows on which they had been executed,|the bodies of all prisoners convicted of murder should either be hung in chains, or buried under the gallows on which they had been executed, +LJ009-0221|according to the discretion of the court before whom the prisoners might be tried.|according to the discretion of the court before whom the prisoners might be tried. +LJ009-0222|The revival of this barbarous practice caused much indignation in certain quarters,|The revival of this barbarous practice caused much indignation in certain quarters, +LJ009-0223|but it was actually tried in two provincial towns, Leicester and Durham.|but it was actually tried in two provincial towns, Leicester and Durham. +LJ009-0224|At the first-named the exhibition nearly created a tumult, and the body was taken down and buried,|At the first-named the exhibition nearly created a tumult, and the body was taken down and buried, +LJ009-0225|but not before the greatest scandal had been caused by the unseemly proceedings of the crowd that flocked to see the sight. A sort of fair was held,|but not before the greatest scandal had been caused by the unseemly proceedings of the crowd that flocked to see the sight. A sort of fair was held, +LJ009-0226|gaming-tables were set up, cards were played under the gibbet, to the disturbance of the public peace and the annoyance of all decent people.|gaming-tables were set up, cards were played under the gibbet, to the disturbance of the public peace and the annoyance of all decent people. +LJ009-0227|At Jarrow Stake, where the Durham murderer's body was exposed, there were similar scenes, mingled with compassion for the culprit's family,|At Jarrow Stake, where the Durham murderer's body was exposed, there were similar scenes, mingled with compassion for the culprit's family, +LJ009-0228|and a subscription was set on foot for them then and there at the foot of the gibbet.|and a subscription was set on foot for them then and there at the foot of the gibbet. +LJ009-0229|Later on, after dark, some friends of the deceased stole the body and buried it in the sand, and this was the end of hanging in chains.|Later on, after dark, some friends of the deceased stole the body and buried it in the sand, and this was the end of hanging in chains. +LJ009-0230|After this a law was passed which prescribed that the bodies of all executed murderers should be buried within the walls of the jail.|After this a law was passed which prescribed that the bodies of all executed murderers should be buried within the walls of the jail. +LJ009-0231|Although these objectionable practices had disappeared,|Although these objectionable practices had disappeared, +LJ009-0232|there were still many shocking incidents at executions, owing to the bungling and unskilful way in which the operation was performed.|there were still many shocking incidents at executions, owing to the bungling and unskilful way in which the operation was performed. +LJ009-0233|The rope still broke sometimes, although it was not often that the horrid scene seen at Jersey at the beginning of the century was repeated.|The rope still broke sometimes, although it was not often that the horrid scene seen at Jersey at the beginning of the century was repeated. +LJ009-0234|There the hangman added his weight to that of the suspended culprit,|There the hangman added his weight to that of the suspended culprit, +LJ009-0235|and having first pulled him sideways, then got upon his shoulders, so that the rope broke.|and having first pulled him sideways, then got upon his shoulders, so that the rope broke. +LJ009-0236|To the great surprise of all who witnessed this dreadful scene,|To the great surprise of all who witnessed this dreadful scene, +LJ009-0237|the poor criminal rose straight upon his feet, with the hangman on his shoulders, and immediately loosened the rope with his fingers.|the poor criminal rose straight upon his feet, with the hangman on his shoulders, and immediately loosened the rope with his fingers. +LJ009-0238|After this the sheriffs sent for another rope, but the spectators interfered, and the man was carried back to jail.|After this the sheriffs sent for another rope, but the spectators interfered, and the man was carried back to jail. +LJ009-0239|The whole case was referred to the king, and the poor wretch, whose crime had been a military one, was eventually pardoned.|The whole case was referred to the king, and the poor wretch, whose crime had been a military one, was eventually pardoned. +LJ009-0240|A somewhat similar event happened at Chester not long afterwards; the ropes by which two offenders were turned off broke a few inches from their necks.|A somewhat similar event happened at Chester not long afterwards; the ropes by which two offenders were turned off broke a few inches from their necks. +LJ009-0241|They were taken back to jail, and were again brought out in the afternoon, by which time fresh and stronger ropes had been procured,|They were taken back to jail, and were again brought out in the afternoon, by which time fresh and stronger ropes had been procured, +LJ009-0242|and the sentence was properly and completely carried out.|and the sentence was properly and completely carried out. +LJ009-0243|Other cases might be quoted,|Other cases might be quoted, +LJ009-0244|especially that of William Snow, alias Sketch, who slipped from the gallows at Exeter and fell to the ground.|especially that of William Snow, alias Sketch, who slipped from the gallows at Exeter and fell to the ground. +LJ009-0245|He soon rose to his feet, and hearing the sorrowful exclamations of the populace, coolly said,|He soon rose to his feet, and hearing the sorrowful exclamations of the populace, coolly said, +LJ009-0246|Good people, do not be hurried; I am not, I can wait.|Good people, do not be hurried; I am not, I can wait. +LJ009-0247|Similar cases were not wanting as regards the executions before Newgate. Others were not less horrible, although there was no failure of apparatus.|Similar cases were not wanting as regards the executions before Newgate. Others were not less horrible, although there was no failure of apparatus. +LJ009-0248|Sometimes the condemned man made a hard fight for life.|Sometimes the condemned man made a hard fight for life. +LJ009-0249|When Charles White was executed in 1823 for arson, he arranged a handkerchief|When Charles White was executed in eighteen twenty-three for arson, he arranged a handkerchief +LJ009-0250|in such a way that the executioner found a difficulty in pinioning his hands.|in such a way that the executioner found a difficulty in pinioning his hands. +LJ009-0251|White managed to keep his wrists asunder, and continued to struggle with the officials for some time. Eventually he was pinioned with a cord in the usual manner.|White managed to keep his wrists asunder, and continued to struggle with the officials for some time. Eventually he was pinioned with a cord in the usual manner. +LJ009-0252|On the scaffold he made a violent attempt to loosen his bonds, and succeeded in getting his hands free.|On the scaffold he made a violent attempt to loosen his bonds, and succeeded in getting his hands free. +LJ009-0253|Then with a strong effort he pushed off the white cap, and tried to liberate his neck from the halter, which by this time had been adjusted.|Then with a strong effort he pushed off the white cap, and tried to liberate his neck from the halter, which by this time had been adjusted. +LJ009-0254|The hangman summoned assistance, and with help tied the cap over White's face with a handkerchief.|The hangman summoned assistance, and with help tied the cap over White's face with a handkerchief. +LJ009-0255|The miserable wretch during the whole of this time was struggling with the most determined violence, to the great horror of the spectators.|The miserable wretch during the whole of this time was struggling with the most determined violence, to the great horror of the spectators. +LJ009-0256|Still he resisted.|Still he resisted. +LJ009-0257|and having got from the falling drop to the firm part of the platform, he nearly succeeded in tearing the handkerchief from his eyes.|and having got from the falling drop to the firm part of the platform, he nearly succeeded in tearing the handkerchief from his eyes. +LJ009-0258|However, the ceremony went forward, and when the signal was given the drop sank.|However, the ceremony went forward, and when the signal was given the drop sank. +LJ009-0259|The wretched man did not fall with it, but jumped on to the platform, and seizing the rope with his hands, tried to avoid strangulation.|The wretched man did not fall with it, but jumped on to the platform, and seizing the rope with his hands, tried to avoid strangulation. +LJ009-0260|The spectacle was horrible;|The spectacle was horrible; +LJ009-0261|the convict was half on the platform, half hanging, and the convulsions of his body were appalling.|the convict was half on the platform, half hanging, and the convulsions of his body were appalling. +LJ009-0262|The crowd vociferously yelled their disapproval, and at length|The crowd vociferously yelled their disapproval, and at length +LJ009-0263|the executioner forced the struggling criminal from the platform, so that the rope sustained his whole weight.|the executioner forced the struggling criminal from the platform, so that the rope sustained his whole weight. +LJ009-0264|His face was visible to the whole crowd, and was fearful to behold.|His face was visible to the whole crowd, and was fearful to behold. +LJ009-0265|Even now his sufferings were not at an end,|Even now his sufferings were not at an end, +LJ009-0266|and his death was not compassed until the executioner terminated his sufferings by hanging on to his legs.|and his death was not compassed until the executioner terminated his sufferings by hanging on to his legs. +LJ009-0267|When Luigi Buranelli was executed in 1855,|When Luigi Buranelli was executed in eighteen fifty-five, +LJ009-0268|through the improper adjustment of the rope his sufferings were prolonged for five minutes;|through the improper adjustment of the rope his sufferings were prolonged for five minutes; +LJ009-0269|his chest heaved, and it was evident that his struggle was a fearful one.|his chest heaved, and it was evident that his struggle was a fearful one. +LJ009-0270|A worse case still was that of William Bousfield, who, when awaiting execution for murder, about the same date,|A worse case still was that of William Bousfield, who, when awaiting execution for murder, about the same date, +LJ009-0271|had attempted to throw himself upon the fire in his condemned cell.|had attempted to throw himself upon the fire in his condemned cell. +LJ009-0272|He was in consequence so weak when brought out for execution, that he had to be carried by four men,|He was in consequence so weak when brought out for execution, that he had to be carried by four men, +LJ009-0273|two supporting his body and two his legs.|two supporting his body and two his legs. +LJ009-0274|His wretched, abject condition, seated in a chair under the drop, was such as almost to unnerve the executioner Calcraft,|His wretched, abject condition, seated in a chair under the drop, was such as almost to unnerve the executioner Calcraft, +LJ009-0275|who bad been further upset by a letter threatening to shoot him when he appeared to perform his task.|who bad been further upset by a letter threatening to shoot him when he appeared to perform his task. +LJ009-0276|Calcraft, the moment he had adjusted the cap and rope, ran down the steps, drew the bolt, and disappeared.|Calcraft, the moment he had adjusted the cap and rope, ran down the steps, drew the bolt, and disappeared. +LJ009-0277|For a second or two the body hung motionless, then, with a strength that astonished the attendant officials,|For a second or two the body hung motionless, then, with a strength that astonished the attendant officials, +LJ009-0278|Bousfield slowly drew himself up, and rested with his feet on the right side of the drop.|Bousfield slowly drew himself up, and rested with his feet on the right side of the drop. +LJ009-0279|One of the turnkeys rushed forward and pushed him off.|One of the turnkeys rushed forward and pushed him off. +LJ009-0280|Again the wretched creature succeeded in obtaining foothold, but this time on the left side of the drop.|Again the wretched creature succeeded in obtaining foothold, but this time on the left side of the drop. +LJ009-0281|Calcraft was forced to return, and he once more pushed Bousfield off,|Calcraft was forced to return, and he once more pushed Bousfield off, +LJ009-0282|who for the fourth time regained his foothold. Again he was repelled,|who for the fourth time regained his foothold. Again he was repelled, +LJ009-0283|this time Calcraft adding his weight to the body, and the strangulation was completed.|this time Calcraft adding his weight to the body, and the strangulation was completed. +LJ009-0284|It was stated in evidence before the Commission on Capital Punishment in 1864,|It was stated in evidence before the Commission on Capital Punishment in eighteen sixty-four, +LJ009-0285|that Calcraft's method of hanging was very rough, much the same as if he had been hanging a dog.|that Calcraft's method of hanging was very rough, much the same as if he had been hanging a dog. +LJ009-0286|There has never been much science in the system of carrying out the extreme penalty in this country; the "finisher of the law"|There has never been much science in the system of carrying out the extreme penalty in this country; the "finisher of the law" +LJ009-0287|has come more by chance than fitness or special education to exercise his loathsome office.|has come more by chance than fitness or special education to exercise his loathsome office. +LJ009-0288|Calcraft, of whom mention has just been made, was by trade a lady's shoemaker,|Calcraft, of whom mention has just been made, was by trade a lady's shoemaker, +LJ009-0289|and before he took to hanging he was employed as a watchman at Reid's brewery in Liquorpond Street.|and before he took to hanging he was employed as a watchman at Reid's brewery in Liquorpond Street. +LJ009-0290|He was at first engaged as assistant to the executioner Tom Cheshire, but in due course rose to be chief.|He was at first engaged as assistant to the executioner Tom Cheshire, but in due course rose to be chief. +LJ009-0291|He was always known as a mild-mannered man of simple tastes, much given to angling in the New River, and a devoted rabbit fancier.|He was always known as a mild-mannered man of simple tastes, much given to angling in the New River, and a devoted rabbit fancier. +LJ009-0292|He was well known in the neighborhood where he resided, and the street gamins cried "Jack Ketch" as he went along the street.|He was well known in the neighborhood where he resided, and the street gamins cried "Jack Ketch" as he went along the street. +LJ009-0293|While Calcraft was in office other aspirants to fame appeared in the field.|While Calcraft was in office other aspirants to fame appeared in the field. +LJ009-0294|One was Askern,|One was Askern, +LJ009-0295|who had been a convicted prisoner at York, but who consented to act as hangman when Calcraft was engaged, and no other functionary could be obtained.|who had been a convicted prisoner at York, but who consented to act as hangman when Calcraft was engaged, and no other functionary could be obtained. +LJ009-0296|It was not always easy to hire a hangman.|It was not always easy to hire a hangman. +LJ009-0297|There is still extant a curious petition presented to the Treasury by Ralph Griffith, Esq., high sheriff of Flintshire,|There is still extant a curious petition presented to the Treasury by Ralph Griffith, Esq., high sheriff of Flintshire, +LJ009-0298|which sets forth that the petitioner had been at great expense by sending clerks and agents to Liverpool and Shrewsbury to hire an executioner.|which sets forth that the petitioner had been at great expense by sending clerks and agents to Liverpool and Shrewsbury to hire an executioner. +LJ009-0299|The man to be hanged belonged to Wales, and no Welshman would do the job.|The man to be hanged belonged to Wales, and no Welshman would do the job. +LJ009-0300|Traveling expenses of these agents cost £15, and another £10 were spent in the hire of a Shropshire man,|Traveling expenses of these agents cost fifteen pounds, and another ten pounds were spent in the hire of a Shropshire man, +LJ009-0301|who deserted, and was pursued, but without success.|who deserted, and was pursued, but without success. +LJ009-0302|Another man was hired, himself a convict, whose fees for self and wife were twelve guineas.|Another man was hired, himself a convict, whose fees for self and wife were twelve guineas. +LJ009-0303|Then came the cost of the gallows,|Then came the cost of the gallows, +LJ009-0304|4 pounds, 12 shillings; and finally the funeral, cart, coffin, and other petty expenses, amounting to seven pounds 10|four pounds, twelve shillings; and finally the funeral, cart, coffin, and other petty expenses, amounting to seven pounds ten +LJ010-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 13: Newgate notorieties, part 1.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section thirteen: Newgate notorieties, part one. +LJ010-0002|In chapter two of the present volume I brought down the record of crime to the second decade of the present century.|In chapter two of the present volume I brought down the record of crime to the second decade of the present century. +LJ010-0003|I propose now to continue the subject, and to devote a couple of chapters to criminal occurrences of a more recent date,|I propose now to continue the subject, and to devote a couple of chapters to criminal occurrences of a more recent date, +LJ010-0004|only premising that as accounts become more voluminous I shall be compelled to deal with fewer cases,|only premising that as accounts become more voluminous I shall be compelled to deal with fewer cases, +LJ010-0005|taking in preference those which are typical and invested with peculiar interest. It is somewhat remarkable that a marked change soon comes over the Calendar.|taking in preference those which are typical and invested with peculiar interest. It is somewhat remarkable that a marked change soon comes over the Calendar. +LJ010-0006|Certain crimes, those against the person especially, diminished gradually. They became less easy or remunerative.|Certain crimes, those against the person especially, diminished gradually. They became less easy or remunerative. +LJ010-0007|Police protection was better and more effective;|Police protection was better and more effective; +LJ010-0008|the streets of London were well lighted, the suburbs were more populous and regularly patrolled.|the streets of London were well lighted, the suburbs were more populous and regularly patrolled. +LJ010-0009|People, too, were getting into the habit of carrying but little cash about them, and no valuables but their watches or personal jewelery.|People, too, were getting into the habit of carrying but little cash about them, and no valuables but their watches or personal jewelery. +LJ010-0010|Street robberies offered fewer inducements to depredators, and evil-doers were compelled to adopt other methods of preying upon their fellows.|Street robberies offered fewer inducements to depredators, and evil-doers were compelled to adopt other methods of preying upon their fellows. +LJ010-0011|This led to a rapid and marked increase in all kinds of fraud;|This led to a rapid and marked increase in all kinds of fraud; +LJ010-0012|and prominent in the criminal annals of Newgate in these later years will be found numerous remarkable instances of this class of offense --|and prominent in the criminal annals of Newgate in these later years will be found numerous remarkable instances of this class of offense -- +LJ010-0013|forgeries committed systematically, and for long periods, as in the case of Fauntleroy,|forgeries committed systematically, and for long periods, as in the case of Fauntleroy, +LJ010-0014|to cover enormous defalcations; the fabrication of deeds, wills, and false securities|to cover enormous defalcations; the fabrication of deeds, wills, and false securities +LJ010-0015|for the purpose of misappropriating funds or feloniously obtaining cash;|for the purpose of misappropriating funds or feloniously obtaining cash; +LJ010-0016|thefts of bullion, bank-notes, specie, and gold-dust, planned with consummate ingenuity,|thefts of bullion, bank-notes, specie, and gold-dust, planned with consummate ingenuity, +LJ010-0017|eluding the keenest vigilance, and carried out with reckless daring;|eluding the keenest vigilance, and carried out with reckless daring; +LJ010-0018|jewel-boxes cleverly stolen under the very noses of owners or care-takers.|jewel-boxes cleverly stolen under the very noses of owners or care-takers. +LJ010-0019|As time passed,|As time passed, +LJ010-0020|the extraordinary extension of all commercial operations led to many entirely novel and often gigantic financial frauds.|the extraordinary extension of all commercial operations led to many entirely novel and often gigantic financial frauds. +LJ010-0021|The credulity of investors, the unscrupulous dishonesty of bankers,|The credulity of investors, the unscrupulous dishonesty of bankers, +LJ010-0022|the slackness of supervision over wholly irresponsible agents, produced many terrible monetary catastrophes,|the slackness of supervision over wholly irresponsible agents, produced many terrible monetary catastrophes, +LJ010-0023|and lodged men like Cole, Robson, and Redpath in Newgate.|and lodged men like Cole, Robson, and Redpath in Newgate. +LJ010-0024|While the varying conditions of social life thus brought about many changes in the character of offenses against property,|While the varying conditions of social life thus brought about many changes in the character of offenses against property, +LJ010-0025|those against the person became more and more limited to the most heinous, or those which menaced or destroyed life.|those against the person became more and more limited to the most heinous, or those which menaced or destroyed life. +LJ010-0026|There was no increase in murder or manslaughter; the number of such crimes remained pretty constant proportionately to population.|There was no increase in murder or manslaughter; the number of such crimes remained pretty constant proportionately to population. +LJ010-0027|Nor did the methods by which they were perpetrated greatly vary from those in times past.|Nor did the methods by which they were perpetrated greatly vary from those in times past. +LJ010-0028|The causes also continued much the same. Passion, revenge, cupidity, sudden ebullitions of homicidal rage,|The causes also continued much the same. Passion, revenge, cupidity, sudden ebullitions of homicidal rage, +LJ010-0029|the cold-blooded, calculating atrocity born of self-interest, were still the irresistible incentives to kill.|the cold-blooded, calculating atrocity born of self-interest, were still the irresistible incentives to kill. +LJ010-0030|The brutal ferocity of the wild beast once aroused, the same means, the same weapons were employed to do the dreadful deed,|The brutal ferocity of the wild beast once aroused, the same means, the same weapons were employed to do the dreadful deed, +LJ010-0031|the same and happily often futile precautions taken to conceal the crime.|the same and happily often futile precautions taken to conceal the crime. +LJ010-0032|Pegsworth, and Greenacre, and Daniel Good merely reproduced types that had gone before, and that have since reappeared.|Pegsworth, and Greenacre, and Daniel Good merely reproduced types that had gone before, and that have since reappeared. +LJ010-0033|Esther Hibner was as inhuman in her ill-usage of the parish apprentice she killed as Martha Brownrigg had been.|Esther Hibner was as inhuman in her ill-usage of the parish apprentice she killed as Martha Brownrigg had been. +LJ010-0034|Thurtell and Hunt followed in the footsteps of Billings, Wood, and Catherine Hayes.|Thurtell and Hunt followed in the footsteps of Billings, Wood, and Catherine Hayes. +LJ010-0035|Courvoisier might have lived a century earlier.|Courvoisier might have lived a century earlier. +LJ010-0036|Hocker was found upon the scene of his crime, irresistibly attracted thither, as was Theodore Gardelle.|Hocker was found upon the scene of his crime, irresistibly attracted thither, as was Theodore Gardelle. +LJ010-0037|Now and again there seemed to be a recurrence of a murder epidemic,|Now and again there seemed to be a recurrence of a murder epidemic, +LJ010-0038|as there had been before; as in the year 1849, a year memorable for the Rush murders at Norwich,|as there had been before; as in the year eighteen forty-nine, a year memorable for the Rush murders at Norwich, +LJ010-0039|the Gleeson Wilson murder at Liverpool, that of the Mannings in London, and of many more.|the Gleeson Wilson murder at Liverpool, that of the Mannings in London, and of many more. +LJ010-0040|Men like Mobbs, the miscreant known as "General Haynau" on account of his blood-thirstiness, still murdered their wives;|Men like Mobbs, the miscreant known as "General Haynau" on account of his blood-thirstiness, still murdered their wives; +LJ010-0041|or like Cannon the chimney-sweeper, who savagely killed the policeman.|or like Cannon the chimney-sweeper, who savagely killed the policeman. +LJ010-0042|A not altogether new crime, however, akin to murder,|A not altogether new crime, however, akin to murder, +LJ010-0043|although happily never passing beyond dastardly attempts, cropped up in these times, and was often frequently repeated within a short interval.|although happily never passing beyond dastardly attempts, cropped up in these times, and was often frequently repeated within a short interval. +LJ010-0044|The present Queen very soon after her accession|The present Queen very soon after her accession +LJ010-0045|became the victim of the most cowardly and unmanly outrages, and the attempted murder of the sovereign by Oxford in 1840|became the victim of the most cowardly and unmanly outrages, and the attempted murder of the sovereign by Oxford in eighteen forty +LJ010-0046|was followed in the very next year by those of Francis and of Bean in two consecutive months,|was followed in the very next year by those of Francis and of Bean in two consecutive months, +LJ010-0048|These crimes had their origin too often in the disordered brains of lunatics at large, like Captain Goode.|These crimes had their origin too often in the disordered brains of lunatics at large, like Captain Goode. +LJ010-0049|Their perpetrators were charged with high treason, but met with merciful clemency as irresponsible beings.|Their perpetrators were charged with high treason, but met with merciful clemency as irresponsible beings. +LJ010-0050|But at various dates treason more distinct and tangible came to the front: attempts to levy war against the State.|But at various dates treason more distinct and tangible came to the front: attempts to levy war against the State. +LJ010-0051|The well-known Cato Street conspiracy,|The well-known Cato Street conspiracy, +LJ010-0052|which grew out of disturbed social conditions after the last French war, amidst general distress,|which grew out of disturbed social conditions after the last French war, amidst general distress, +LJ010-0053|and when the people were beginning to agitate for a larger share of political power, was among the earliest, and to some extent the most desperate, of these.|and when the people were beginning to agitate for a larger share of political power, was among the earliest, and to some extent the most desperate, of these. +LJ010-0054|Its ringleaders, Thistlewood and the rest, were after capture honored by committal as State prisoners to the Tower,|Its ringleaders, Thistlewood and the rest, were after capture honored by committal as State prisoners to the Tower, +LJ010-0055|but they came one and all to Newgate for trial at the Old Bailey, and remained there after conviction till they were hanged.|but they came one and all to Newgate for trial at the Old Bailey, and remained there after conviction till they were hanged. +LJ010-0056|Later on, the Chartists agitated persistently for the concession embraced in the so-called People's Charter, many of which|Later on, the Chartists agitated persistently for the concession embraced in the so-called People's Charter, many of which +LJ010-0057|are by this time actually, and by more legitimate efforts, engrafted upon our Constitution.|are by this time actually, and by more legitimate efforts, engrafted upon our Constitution. +LJ010-0058|But the Chartists sought their ends by riot and rebellion, and gained only imprisonment for their pains.|But the Chartists sought their ends by riot and rebellion, and gained only imprisonment for their pains. +LJ010-0059|Some five hundred in all were arrested, but as only three of these were lodged in Newgate, I shall not recur to them in my narrative.|Some five hundred in all were arrested, but as only three of these were lodged in Newgate, I shall not recur to them in my narrative. +LJ010-0060|The Cato Street conspiracy would have been simply ridiculous but for the recklessness of the desperadoes who planned it.|The Cato Street conspiracy would have been simply ridiculous but for the recklessness of the desperadoes who planned it. +LJ010-0062|But they proceeded in all seriousness, and would have shrunk from no outrage or atrocity in furtherance of their foolhardy enterprise.|But they proceeded in all seriousness, and would have shrunk from no outrage or atrocity in furtherance of their foolhardy enterprise. +LJ010-0063|The massacre of the whole of the Cabinet Ministers at one stroke was to be followed by an attack|The massacre of the whole of the Cabinet Ministers at one stroke was to be followed by an attack +LJ010-0064|upon "the old man and the old woman," as they styled the Mansion House and the Bank of England.|upon "the old man and the old woman," as they styled the Mansion House and the Bank of England. +LJ010-0065|At the former the "Provisional Government" was to be established,|At the former the "Provisional Government" was to be established, +LJ010-0066|which under Thistlewood as dictator was to rule the nation, by first handing over its capital to fire and pillage.|which under Thistlewood as dictator was to rule the nation, by first handing over its capital to fire and pillage. +LJ010-0067|This Thistlewood had seen many vicissitudes throughout his strange, adventurous career. The son of a respectable Lincolnshire farmer,|This Thistlewood had seen many vicissitudes throughout his strange, adventurous career. The son of a respectable Lincolnshire farmer, +LJ010-0068|he became a militia officer, and married a woman with £10,000, in which, however, she had only a life interest.|he became a militia officer, and married a woman with ten thousand pounds, in which, however, she had only a life interest. +LJ010-0069|She died early, and Thistlewood, left to his own resources,|She died early, and Thistlewood, left to his own resources, +LJ010-0070|followed the profession of arms, first in the British service, and then in that of the French revolutionary Government.|followed the profession of arms, first in the British service, and then in that of the French revolutionary Government. +LJ010-0071|It was during this period that he was said to have imbibed his revolutionary ideas.|It was during this period that he was said to have imbibed his revolutionary ideas. +LJ010-0072|Returning to England,|Returning to England, +LJ010-0073|he found himself rich in a small landed property, which he presently sold to a man who became bankrupt before he had paid over the purchase money.|he found himself rich in a small landed property, which he presently sold to a man who became bankrupt before he had paid over the purchase money. +LJ010-0074|After this he tried farming, but failed.|After this he tried farming, but failed. +LJ010-0075|He married again and came to London, where he soon became notorious as a reckless gambler and a politician holding the most extreme views.|He married again and came to London, where he soon became notorious as a reckless gambler and a politician holding the most extreme views. +LJ010-0076|In this way he formed the acquaintance of Watson and others, with whom he was arraigned for treasonable practices, and imprisoned.|In this way he formed the acquaintance of Watson and others, with whom he was arraigned for treasonable practices, and imprisoned. +LJ010-0077|On his release he sent a challenge to Lord Sidmouth, the Home Secretary, and was again arrested and imprisoned.|On his release he sent a challenge to Lord Sidmouth, the Home Secretary, and was again arrested and imprisoned. +LJ010-0078|On his second release, goaded by his fancied wrongs, he began to plot a dark and dreadful revenge,|On his second release, goaded by his fancied wrongs, he began to plot a dark and dreadful revenge, +LJ010-0079|and thus the conspiracy in which he was the prime mover took shape, and came to a head.|and thus the conspiracy in which he was the prime mover took shape, and came to a head. +LJ010-0080|The Government obtained early and full information of the nefarious scheme.|The Government obtained early and full information of the nefarious scheme. +LJ010-0081|One of the conspirators, by name Edwards,|One of the conspirators, by name Edwards, +LJ010-0082|made a voluntary confession to Sir Herbert Taylor one morning at Windsor; after which Thistlewood and his accomplices were closely watched,|made a voluntary confession to Sir Herbert Taylor one morning at Windsor; after which Thistlewood and his accomplices were closely watched, +LJ010-0083|and measures taken to arrest them when their plans were so far developed that no doubt could remain as to their guilt.|and measures taken to arrest them when their plans were so far developed that no doubt could remain as to their guilt. +LJ010-0084|The day appointed for the murder and rising actually arrived before the authorities interfered.|The day appointed for the murder and rising actually arrived before the authorities interfered. +LJ010-0085|It was the day on which Lord Harrowby was to entertain his colleagues at dinner in Grosvenor Square.|It was the day on which Lord Harrowby was to entertain his colleagues at dinner in Grosvenor Square. +LJ010-0086|The occasion was considered excellent by the conspirators for disposal of the whole Cabinet at one blow,|The occasion was considered excellent by the conspirators for disposal of the whole Cabinet at one blow, +LJ010-0087|and it was arranged that one of their number should knock at Lord Harrowby's door on the pretense of leaving a parcel,|and it was arranged that one of their number should knock at Lord Harrowby's door on the pretense of leaving a parcel, +LJ010-0088|and that when it was opened the whole band should rush in.|and that when it was opened the whole band should rush in. +LJ010-0089|While a few secured the servants, the rest were to fall upon Lord Harrowby and his guests.|While a few secured the servants, the rest were to fall upon Lord Harrowby and his guests. +LJ010-0090|Hand-grenades were to be thrown into the dining-room, and during the noise and confusion the assassination of the ministers was to be completed,|Hand-grenades were to be thrown into the dining-room, and during the noise and confusion the assassination of the ministers was to be completed, +LJ010-0091|the heads of Lord Castlereagh and Lord Sidmouth being carried away in a bag.|the heads of Lord Castlereagh and Lord Sidmouth being carried away in a bag. +LJ010-0092|Lord Harrowby's dinner-party was postponed, but the conspirators knew nothing of it,|Lord Harrowby's dinner-party was postponed, but the conspirators knew nothing of it, +LJ010-0093|and those who watched his house were further encouraged in their mistake by the arrival of many carriages,|and those who watched his house were further encouraged in their mistake by the arrival of many carriages, +LJ010-0094|bound, as it happened, to the Archbishop of York's.|bound, as it happened, to the Archbishop of York's. +LJ010-0095|Meanwhile the main body remained at their headquarters, a ruined stable in Cato Street,|Meanwhile the main body remained at their headquarters, a ruined stable in Cato Street, +LJ010-0096|Edgeware Road, completing their dispositions for assuming supreme power after the blow had been struck.|Edgeware Road, completing their dispositions for assuming supreme power after the blow had been struck. +LJ010-0097|Here they were surprised by the police, headed by a magistrate, and supported by a strong detachment of Her Majesty's Guards.|Here they were surprised by the police, headed by a magistrate, and supported by a strong detachment of Her Majesty's Guards. +LJ010-0098|The police were the first to arrive on the spot, the Guards having entered the street at the wrong end.|The police were the first to arrive on the spot, the Guards having entered the street at the wrong end. +LJ010-0099|The conspirators were in a loft, approached by a ladder and a trap-door, access through which could only be obtained one by one.|The conspirators were in a loft, approached by a ladder and a trap-door, access through which could only be obtained one by one. +LJ010-0100|The first constable who entered Thistlewood ran through the body with a sword, but others quickly followed,|The first constable who entered Thistlewood ran through the body with a sword, but others quickly followed, +LJ010-0101|the lights were extinguished, and a desperate conflict ensued.|the lights were extinguished, and a desperate conflict ensued. +LJ010-0102|The Guards, headed by Lord Adolphus Fitz Clarence, now reinforced the police, and the conspirators gave way.|The Guards, headed by Lord Adolphus Fitz Clarence, now reinforced the police, and the conspirators gave way. +LJ010-0103|Nine of the latter were captured, with all the war material, cutlasses, pistols, hand-grenades, and ammunition.|Nine of the latter were captured, with all the war material, cutlasses, pistols, hand-grenades, and ammunition. +LJ010-0104|Thistlewood and fourteen more succeeded for the moment in making their escape, but most of them were subsequently taken.|Thistlewood and fourteen more succeeded for the moment in making their escape, but most of them were subsequently taken. +LJ010-0106|He was in bed with his breeches on (in the pockets of which were found a number of cartridges), the black belt he had worn at Cato Street,|He was in bed with his breeches on (in the pockets of which were found a number of cartridges), the black belt he had worn at Cato Street, +LJ010-0107|and a military sash.|and a military sash. +LJ010-0108|The trial of the conspirators came on some six weeks later, at the Old Bailey.|The trial of the conspirators came on some six weeks later, at the Old Bailey. +LJ010-0109|Thistlewood made a long and rambling defense, the chief features of which were abuse of Lord Sidmouth, and the vilification of the informer Edwards.|Thistlewood made a long and rambling defense, the chief features of which were abuse of Lord Sidmouth, and the vilification of the informer Edwards. +LJ010-0110|Several of the other prisoners took the same line as regards Edwards,|Several of the other prisoners took the same line as regards Edwards, +LJ010-0111|and there seems to have been good reason for supposing that he was a greater villain than any of those arraigned.|and there seems to have been good reason for supposing that he was a greater villain than any of those arraigned. +LJ010-0112|He had been in a state of abject misery, and when he first joined "the reformers," as the Cato Street conspirators called themselves,|He had been in a state of abject misery, and when he first joined "the reformers," as the Cato Street conspirators called themselves, +LJ010-0113|he had neither a bed to lie upon nor a coat to his back.|he had neither a bed to lie upon nor a coat to his back. +LJ010-0114|His sudden access to means unlimited was no doubt due to the profitable role he soon adopted of Government informer and spy,|His sudden access to means unlimited was no doubt due to the profitable role he soon adopted of Government informer and spy, +LJ010-0115|and it is pretty certain that for some time he served both sides;|and it is pretty certain that for some time he served both sides; +LJ010-0116|on the one inveigling silly enthusiasts to join in the plot, and denouncing them on the other.|on the one inveigling silly enthusiasts to join in the plot, and denouncing them on the other. +LJ010-0117|The employment of Edwards, and the manner in which the conspirators were allowed to commit themselves further and further before the law was set in motion against them,|The employment of Edwards, and the manner in which the conspirators were allowed to commit themselves further and further before the law was set in motion against them, +LJ010-0118|were not altogether creditable to the Government.|were not altogether creditable to the Government. +LJ010-0119|It was asserted, not without foundation, at these trials, that Edwards repeatedly incited the associates he was betraying|It was asserted, not without foundation, at these trials, that Edwards repeatedly incited the associates he was betraying +LJ010-0120|to commit outrage, to set fire to houses, throw hand-grenades into the carriages of ministers;|to commit outrage, to set fire to houses, throw hand-grenades into the carriages of ministers; +LJ010-0121|that he was, to use Thistlewood's words, "a contriver, instigator, and entrapper."|that he was, to use Thistlewood's words, "a contriver, instigator, and entrapper." +LJ010-0122|The Government were probably not proud of their agent, for Edwards, after the conviction had been assured, went abroad to enjoy, it was said,|The Government were probably not proud of their agent, for Edwards, after the conviction had been assured, went abroad to enjoy, it was said, +LJ010-0123|an ample pension, so long as he did not return to England.|an ample pension, so long as he did not return to England. +LJ010-0124|Five of the conspirators, Thistlewood, Ings, Brunt, Davidson, and Tidd, were sentenced to death,|Five of the conspirators, Thistlewood, Ings, Brunt, Davidson, and Tidd, were sentenced to death, +LJ010-0125|and suffered in the usual way in front of Newgate, with the additional penalty of decapitation, as traitors, after they had been hanged.|and suffered in the usual way in front of Newgate, with the additional penalty of decapitation, as traitors, after they had been hanged. +LJ010-0126|A crowd as great as any known collected in the Old Bailey to see the ceremony, about which there were some peculiar features worth recording.|A crowd as great as any known collected in the Old Bailey to see the ceremony, about which there were some peculiar features worth recording. +LJ010-0127|The reckless demeanor of all the convicts except Davidson was most marked. Thistlewood and Ings sucked oranges on the scaffold;|The reckless demeanor of all the convicts except Davidson was most marked. Thistlewood and Ings sucked oranges on the scaffold; +LJ010-0128|they with Brunt and Tidd scorned the ordinary's ministrations,|they with Brunt and Tidd scorned the ordinary's ministrations, +LJ010-0129|but Ings said he hoped God would be more merciful to him than men had been.|but Ings said he hoped God would be more merciful to him than men had been. +LJ010-0130|Ings was especially defiant. He sought to cheer Davidson, who seemed affected, crying out,|Ings was especially defiant. He sought to cheer Davidson, who seemed affected, crying out, +LJ010-0131|Come, old cock-of-wax, it will soon be over.|Come, old cock-of-wax, it will soon be over. +LJ010-0132|As the executioner fastened the noose, he nodded to a friend he saw in the crowd;|As the executioner fastened the noose, he nodded to a friend he saw in the crowd; +LJ010-0133|and catching sight of the coffins ranged around the gallows, he smiled at the show with contemptuous indifference.|and catching sight of the coffins ranged around the gallows, he smiled at the show with contemptuous indifference. +LJ010-0134|He roared out snatches of a song about Death or Liberty, and just before he was turned off,|He roared out snatches of a song about Death or Liberty, and just before he was turned off, +LJ010-0135|yelled out three cheers to the populace whom he faced.|yelled out three cheers to the populace whom he faced. +LJ010-0136|He told the executioner to "do it tidy," to pull it tight, and was in a state of hysterical exaltation up to the very last.|He told the executioner to "do it tidy," to pull it tight, and was in a state of hysterical exaltation up to the very last. +LJ010-0137|Davidson, who was the only one who seemed to realize his awful situation, listened patiently and with thankfulness to the chaplain,|Davidson, who was the only one who seemed to realize his awful situation, listened patiently and with thankfulness to the chaplain, +LJ010-0138|and died in a manner strongly contrasting with that of his fellows.|and died in a manner strongly contrasting with that of his fellows. +LJ010-0139|After the five bodies had hung for half-an-hour, a man in a mask came forward to complete the sentence.|After the five bodies had hung for half-an-hour, a man in a mask came forward to complete the sentence. +LJ010-0140|Contemporary reports state that from the skillful manner in which he performed the decapitation,|Contemporary reports state that from the skillful manner in which he performed the decapitation, +LJ010-0141|he was generally supposed to be a surgeon.|he was generally supposed to be a surgeon. +LJ010-0142|Be this as it may, the weapon used was only an ordinary axe, which rather indicates that force, not skill, was employed.|Be this as it may, the weapon used was only an ordinary axe, which rather indicates that force, not skill, was employed. +LJ010-0143|This axe is still in existence, and is preserved at Newgate with various other unpleasant curiosities,|This axe is still in existence, and is preserved at Newgate with various other unpleasant curiosities, +LJ010-0144|but is only an ordinary commonplace tool.|but is only an ordinary commonplace tool. +LJ010-0145|These were the last executions for high treason, but not the last prisoners by many who passed through Newgate charged with sedition.|These were the last executions for high treason, but not the last prisoners by many who passed through Newgate charged with sedition. +LJ010-0146|Attacks upon the sovereign, as I have said, became more common after the accession of the young Queen Victoria in 1838.|Attacks upon the sovereign, as I have said, became more common after the accession of the young Queen Victoria in eighteen thirty-eight. +LJ010-0147|It was a form of high treason not unknown in earlier reigns. In 1786 a mad woman, Margaret Nicholson,|It was a form of high treason not unknown in earlier reigns. In seventeen eighty-six a mad woman, Margaret Nicholson, +LJ010-0148|tried to stab George III. as he was alighting from his carriage at the gate of St. James's Palace.|tried to stab George the third as he was alighting from his carriage at the gate of St. James's Palace. +LJ010-0149|She was seized before she could do any mischief,|She was seized before she could do any mischief, +LJ010-0150|and eventually lodged in Bethlehem Hospital, where she died after forty years' detention, at the advanced age of one hundred.|and eventually lodged in Bethlehem Hospital, where she died after forty years' detention, at the advanced age of one hundred. +LJ010-0151|Again, a soldier, by name Hatfield, who had been wounded in the head, and discharged from the army for unsoundness of mind,|Again, a soldier, by name Hatfield, who had been wounded in the head, and discharged from the army for unsoundness of mind, +LJ010-0152|fired a pistol at George III. from the pit of Drury Lane theatre in 1800.|fired a pistol at George the third from the pit of Drury Lane theatre in eighteen hundred. +LJ010-0153|William IV. was also the victim of a murderous outrage on Ascot race-course in 1832,|William the fourth was also the victim of a murderous outrage on Ascot race-course in eighteen thirty-two, +LJ010-0154|when John Collins, "a person in the garb of a sailor, of wretched appearance, and having a wooden leg,"|when John Collins, "a person in the garb of a sailor, of wretched appearance, and having a wooden leg," +LJ010-0155|threw a stone at the king, which hit him on the forehead, but did no serious injury.|threw a stone at the king, which hit him on the forehead, but did no serious injury. +LJ010-0156|Collins, when charged, pleaded that he had lost his leg in action, that he had petitioned without success for a pension,|Collins, when charged, pleaded that he had lost his leg in action, that he had petitioned without success for a pension, +LJ010-0157|and that, as he was starving, he had resolved on this desperate deed,|and that, as he was starving, he had resolved on this desperate deed, +LJ010-0158|feeling, as he said, that he might as well be shot or hanged as remain in such a state.|feeling, as he said, that he might as well be shot or hanged as remain in such a state. +LJ010-0159|He was eventually sentenced to death, but the plea of lunacy was allowed, and he was confined for life.|He was eventually sentenced to death, but the plea of lunacy was allowed, and he was confined for life. +LJ010-0160|None of the foregoing attempts were, however, so dastardly or determined as that made by Oxford upon our present gracious Queen|None of the foregoing attempts were, however, so dastardly or determined as that made by Oxford upon our present gracious Queen +LJ010-0161|two years after she ascended the throne.|two years after she ascended the throne. +LJ010-0162|The cowardly crime was probably encouraged by the fearless and confiding manner in which the Queen,|The cowardly crime was probably encouraged by the fearless and confiding manner in which the Queen, +LJ010-0163|secure as it seemed in the affections of her loyal people, freely appeared in public.|secure as it seemed in the affections of her loyal people, freely appeared in public. +LJ010-0164|Oxford, who was only nineteen at the time his offense was committed, had been born at Birmingham,|Oxford, who was only nineteen at the time his offense was committed, had been born at Birmingham, +LJ010-0165|but he came as a lad to London, and took service as a pot-boy to a publican.|but he came as a lad to London, and took service as a pot-boy to a publican. +LJ010-0166|From this he was promoted to barman, and as such had charge of the business in various public-houses.|From this he was promoted to barman, and as such had charge of the business in various public-houses. +LJ010-0167|He left his last situation in April 1840, and established himself in lodgings in Lambeth,|He left his last situation in April eighteen forty, and established himself in lodgings in Lambeth, +LJ010-0168|after which he devoted himself to pistol practice in shooting-galleries, sometimes in Leicester Square,|after which he devoted himself to pistol practice in shooting-galleries, sometimes in Leicester Square, +LJ010-0169|sometimes in the Strand, or the West End.|sometimes in the Strand, or the West End. +LJ010-0170|His acquaintances often asked his object in this, but he kept his own counsel till the 10th June.|His acquaintances often asked his object in this, but he kept his own counsel till the tenth June. +LJ010-0171|On that day Oxford was on the watch at Buckingham Palace.|On that day Oxford was on the watch at Buckingham Palace. +LJ010-0172|He saw Prince Albert return there from a visit to Woolwich, and then passed on to Constitution Hill,|He saw Prince Albert return there from a visit to Woolwich, and then passed on to Constitution Hill, +LJ010-0173|where he waited till four p.m., the time at which the Queen and Prince Consort usually took an afternoon drive.|where he waited till four p.m., the time at which the Queen and Prince Consort usually took an afternoon drive. +LJ010-0174|About six p.m. the royal carriage, a low open vehicle drawn by four horses, ridden by postilions, left the palace.|About six p.m. the royal carriage, a low open vehicle drawn by four horses, ridden by postilions, left the palace. +LJ010-0175|Oxford, who had been pacing backwards and forwards with his hands under the lapels of his coat, saw the carriage approach.|Oxford, who had been pacing backwards and forwards with his hands under the lapels of his coat, saw the carriage approach. +LJ010-0176|He was on the right or north side of the road. Prince Albert occupied the same side of the carriage, the Queen the left.|He was on the right or north side of the road. Prince Albert occupied the same side of the carriage, the Queen the left. +LJ010-0177|As the carriage came up to him Oxford turned, put his hand into his breast, drew a pistol, and fired at the Queen.|As the carriage came up to him Oxford turned, put his hand into his breast, drew a pistol, and fired at the Queen. +LJ010-0178|The shot missed, and as the carriage passed on, Oxford drew a second pistol and fired again.|The shot missed, and as the carriage passed on, Oxford drew a second pistol and fired again. +LJ010-0179|The Queen saw this second movement, and stooped to avoid the shot;|The Queen saw this second movement, and stooped to avoid the shot; +LJ010-0180|the Prince too rose to shield her with his person. Again, providentially, the bullet went wide of the mark,|the Prince too rose to shield her with his person. Again, providentially, the bullet went wide of the mark, +LJ010-0181|and the royal party drove back to Clarence House, the Queen being anxious to give the first news of the outrage and of her safety to her mother,|and the royal party drove back to Clarence House, the Queen being anxious to give the first news of the outrage and of her safety to her mother, +LJ010-0182|the Duchess of Kent.|the Duchess of Kent. +LJ010-0183|Meanwhile the pistol-shots had attracted the attention of the bystanders, of whom there was a fair collection, as usual, waiting to see the Queen pass.|Meanwhile the pistol-shots had attracted the attention of the bystanders, of whom there was a fair collection, as usual, waiting to see the Queen pass. +LJ010-0184|Oxford was seized by a person named Lowe, who was at first mistaken for the assailant.|Oxford was seized by a person named Lowe, who was at first mistaken for the assailant. +LJ010-0185|But Oxford at once assumed the responsibility for his crime, saying, "It was I. I did it. I'll give myself up.|But Oxford at once assumed the responsibility for his crime, saying, "It was I. I did it. I'll give myself up. +LJ010-0186|There is no occasion to use violence. I will go with you.|There is no occasion to use violence. I will go with you. +LJ010-0187|He was taken into custody, and removed first to a police cell, thence committed to Newgate, after he had been examined before the Privy Council.|He was taken into custody, and removed first to a police cell, thence committed to Newgate, after he had been examined before the Privy Council. +LJ010-0188|Oxford expressed little anxiety or concern.|Oxford expressed little anxiety or concern. +LJ010-0189|He asked more than once whether the Queen was hurt, and acknowledged that the pistols were loaded with ball.|He asked more than once whether the Queen was hurt, and acknowledged that the pistols were loaded with ball. +LJ010-0190|A craze for notoriety, to be achieved at any cost, was the one absorbing idea in young Oxford's disordered brain.|A craze for notoriety, to be achieved at any cost, was the one absorbing idea in young Oxford's disordered brain. +LJ010-0191|After his arrest he thought only of the excitement his attempt had raised, nothing of its atrocity,|After his arrest he thought only of the excitement his attempt had raised, nothing of its atrocity, +LJ010-0192|or of the fatal consequences which might have ensued.|or of the fatal consequences which might have ensued. +LJ010-0193|When brought to trial he hardly realized his position,|When brought to trial he hardly realized his position, +LJ010-0194|but gazed with complacency around the crowded court, and eagerly inquired what persons of distinction were present.|but gazed with complacency around the crowded court, and eagerly inquired what persons of distinction were present. +LJ010-0195|He smiled continually, and when the indictment was read, burst into loud and discordant fits of laughter.|He smiled continually, and when the indictment was read, burst into loud and discordant fits of laughter. +LJ010-0196|These antics may have been assumed to bear out the plea of insanity set up in his defense,|These antics may have been assumed to bear out the plea of insanity set up in his defense, +LJ010-0197|but that there was madness in his family, and that he himself was of unsound mind, could not be well denied.|but that there was madness in his family, and that he himself was of unsound mind, could not be well denied. +LJ010-0198|His father, it was proved in evidence, had been at times quite mad; and Oxford's mental state might be inferred from his own proceedings.|His father, it was proved in evidence, had been at times quite mad; and Oxford's mental state might be inferred from his own proceedings. +LJ010-0199|Among his papers was found a curious document, purporting to be the rules of an association called|Among his papers was found a curious document, purporting to be the rules of an association called +LJ010-0200|"Young England," which Oxford had evolved out of his own inflated self-conceit, and which had never any real corporeal existence.|"Young England," which Oxford had evolved out of his own inflated self-conceit, and which had never any real corporeal existence. +LJ010-0201|"Young England" was a secret society, with no aim or object.|"Young England" was a secret society, with no aim or object. +LJ010-0202|Its sworn members, known only to Oxford, and all of them mere shadows,|Its sworn members, known only to Oxford, and all of them mere shadows, +LJ010-0203|were bound to provide themselves with sword, rifle, dagger, and a pair of pistols;|were bound to provide themselves with sword, rifle, dagger, and a pair of pistols; +LJ010-0204|to wear a black crape mask, to obey punctually the orders of their commander-in-chief,|to wear a black crape mask, to obey punctually the orders of their commander-in-chief, +LJ010-0205|and to assume any disguise, if required to go into the country on the business of the association.|and to assume any disguise, if required to go into the country on the business of the association. +LJ010-0206|The officers of the society were to be known only by "factitious names."|The officers of the society were to be known only by "factitious names." +LJ010-0207|Thus, among the presidents were those of Gowrie, Justinian, Aloman, Colsman, Kenneth, and Godfrey;|Thus, among the presidents were those of Gowrie, Justinian, Aloman, Colsman, Kenneth, and Godfrey; +LJ010-0208|Hannibal and Ethelred were on the council; Anthony, Augustus, and Frederic were among the generals;|Hannibal and Ethelred were on the council; Anthony, Augustus, and Frederic were among the generals; +LJ010-0209|Louis and Amadeus among the captains; and Hercules, Neptune, and Mars among the lieutenants of the association.|Louis and Amadeus among the captains; and Hercules, Neptune, and Mars among the lieutenants of the association. +LJ010-0210|The various grades were distinguished by cockades and bows of different colors.|The various grades were distinguished by cockades and bows of different colors. +LJ010-0211|The society was supposed to meet regularly, and its proceedings, together with the speeches made, were duly recorded.|The society was supposed to meet regularly, and its proceedings, together with the speeches made, were duly recorded. +LJ010-0212|With Oxford's other papers were found letters from the secretary, written as it seemed by Oxford to himself, after the manner of Mr. Toots,|With Oxford's other papers were found letters from the secretary, written as it seemed by Oxford to himself, after the manner of Mr. Toots, +LJ010-0213|all of which declared their approval of the commander-in-chief.|all of which declared their approval of the commander-in-chief. +LJ010-0214|One expressed pleasure that Oxford improved so much in speaking, and declared that his (Oxford's) speech the last time "was beautiful."|One expressed pleasure that Oxford improved so much in speaking, and declared that his (Oxford's) speech the last time "was beautiful." +LJ010-0215|This letter went on to say that a new member had been introduced by Lt. Mars, "a fine, tall, gentlemanly young man,|This letter went on to say that a new member had been introduced by Lt. Mars, "a fine, tall, gentlemanly young man, +LJ010-0216|and it is said that he is a military officer, but his name has not yet transpired.|and it is said that he is a military officer, but his name has not yet transpired. +LJ010-0217|Soon after he was introduced we were alarmed by a violent knocking at the door;|Soon after he was introduced we were alarmed by a violent knocking at the door; +LJ010-0218|in an instant our faces were covered, we cocked our pistols, and with drawn swords stood waiting to receive the enemy.|in an instant our faces were covered, we cocked our pistols, and with drawn swords stood waiting to receive the enemy. +LJ010-0219|While one stood over the fire with the papers, another stood with lighted torch to fire the house.|While one stood over the fire with the papers, another stood with lighted torch to fire the house. +LJ010-0220|We then sent the old woman to open the door, and it proved to be some little boys who had knocked and ran away.|We then sent the old woman to open the door, and it proved to be some little boys who had knocked and ran away. +LJ010-0221|Another letter directed Oxford to attend an extraordinary meeting of "Young England"|Another letter directed Oxford to attend an extraordinary meeting of "Young England" +LJ010-0222|in consequence of having received some information of an important nature from Hanover.|in consequence of having received some information of an important nature from Hanover. +LJ010-0223|You must attend; and if your master will not give you leave, you must come in defiance of him.|You must attend; and if your master will not give you leave, you must come in defiance of him. +LJ010-0224|No serious importance could be attached to these, the manifest inventions of a disordered intellect.|No serious importance could be attached to these, the manifest inventions of a disordered intellect. +LJ010-0225|The whole of the evidence pointed so strongly towards insanity, that the jury brought in a verdict of acquittal on that ground,|The whole of the evidence pointed so strongly towards insanity, that the jury brought in a verdict of acquittal on that ground, +LJ010-0226|and Oxford was ordered to be detained during Her Majesty's pleasure.|and Oxford was ordered to be detained during Her Majesty's pleasure. +LJ010-0227|He went from Newgate first to Bethlehem, from which he was removed to Broadmoor on the opening of the great criminal lunatic asylum at that place.|He went from Newgate first to Bethlehem, from which he was removed to Broadmoor on the opening of the great criminal lunatic asylum at that place. +LJ010-0228|He was released from Broadmoor in 1878, and went abroad.|He was released from Broadmoor in eighteen seventy-eight, and went abroad. +LJ010-0229|Within a couple of years a second attempt to assassinate the Queen was perpetrated in nearly the same spot, by a man named John Francis,|Within a couple of years a second attempt to assassinate the Queen was perpetrated in nearly the same spot, by a man named John Francis, +LJ010-0230|who was arrested in the very act, just as he had fired one shot.|who was arrested in the very act, just as he had fired one shot. +LJ010-0231|His motives for thus imitating the dastardly crime of Oxford are shrouded in obscurity.|His motives for thus imitating the dastardly crime of Oxford are shrouded in obscurity. +LJ010-0232|He could not plead insanity like his predecessor, and no attempt was made at his trial to prove him of unsound mind.|He could not plead insanity like his predecessor, and no attempt was made at his trial to prove him of unsound mind. +LJ010-0235|he would be in some way provided for, he having been for some time previously in abject circumstances.|he would be in some way provided for, he having been for some time previously in abject circumstances. +LJ010-0236|The deed was long premeditated, and would have been executed a day earlier had not his courage failed him at the last moment.|The deed was long premeditated, and would have been executed a day earlier had not his courage failed him at the last moment. +LJ010-0237|A youth named Pearson had seen him present a pistol at the Queen's carriage,|A youth named Pearson had seen him present a pistol at the Queen's carriage, +LJ010-0238|but draw it back again, exclaiming presently, "I wish I had done it."|but draw it back again, exclaiming presently, "I wish I had done it." +LJ010-0239|Pearson weakly allowed Francis to go off without securing his apprehension, but later he gave full information.|Pearson weakly allowed Francis to go off without securing his apprehension, but later he gave full information. +LJ010-0240|The Queen was apprised of the danger, and begged not to go abroad;|The Queen was apprised of the danger, and begged not to go abroad; +LJ010-0241|but she declared she would not remain a prisoner in her own palace, and next day drove out as usual in an open barouche.|but she declared she would not remain a prisoner in her own palace, and next day drove out as usual in an open barouche. +LJ010-0242|Nothing happened till Her Majesty returned to Buckingham Palace about six p.m., when, on descending Constitution Hill,|Nothing happened till Her Majesty returned to Buckingham Palace about six p.m., when, on descending Constitution Hill, +LJ010-0243|with an equerry riding close on each side of her carriage, a man who had been leaning against the palace garden wall suddenly advanced,|with an equerry riding close on each side of her carriage, a man who had been leaning against the palace garden wall suddenly advanced, +LJ010-0244|leveled a pistol at the Queen, and fired. He was so close to the carriage that the smoke of his pistol enveloped the face of Colonel Wylde,|leveled a pistol at the Queen, and fired. He was so close to the carriage that the smoke of his pistol enveloped the face of Colonel Wylde, +LJ010-0245|one of the equerries. The Queen was untouched, and at first, it is said, hardly realized the danger she had escaped.|one of the equerries. The Queen was untouched, and at first, it is said, hardly realized the danger she had escaped. +LJ010-0246|Francis had already been seized by a policeman named Trounce, who saw his movement with the pistol, but too late to prevent its discharge.|Francis had already been seized by a policeman named Trounce, who saw his movement with the pistol, but too late to prevent its discharge. +LJ010-0247|The prisoner was conveyed without delay to the Home Office, and there examined by the Privy Council, which had been hastily summoned for the purpose.|The prisoner was conveyed without delay to the Home Office, and there examined by the Privy Council, which had been hastily summoned for the purpose. +LJ010-0248|On searching him the pistol was found in his pocket, the barrel still warm; also some loose powder and a bullet.|On searching him the pistol was found in his pocket, the barrel still warm; also some loose powder and a bullet. +LJ010-0249|There was some doubt as to whether the pistol when fired was actually loaded with ball, but the jury brought in a verdict of guilty|There was some doubt as to whether the pistol when fired was actually loaded with ball, but the jury brought in a verdict of guilty +LJ010-0250|of the criminal intent to kill.|of the criminal intent to kill. +LJ010-0251|Francis was sentenced to be hanged, decapitated, and quartered,|Francis was sentenced to be hanged, decapitated, and quartered, +LJ010-0252|the old traitor's doom, but was spared, and subsequently transported for life.|the old traitor's doom, but was spared, and subsequently transported for life. +LJ010-0253|The enthusiasm of the people at the Queen's escape was uproarious, and her drive next day was one long triumphal progress.|The enthusiasm of the people at the Queen's escape was uproarious, and her drive next day was one long triumphal progress. +LJ010-0254|At the Italian Opera in the evening the audience, on the Queen's appearance, greeted her with loud cheers, and called for the national anthem.|At the Italian Opera in the evening the audience, on the Queen's appearance, greeted her with loud cheers, and called for the national anthem. +LJ010-0255|This was in May 1842.|This was in May eighteen forty-two. +LJ010-0256|Undeterred by the well-merited punishment which had overtaken Francis,|Undeterred by the well-merited punishment which had overtaken Francis, +LJ010-0257|a third miscreant made a similar but far less serious attempt in the month of July following.|a third miscreant made a similar but far less serious attempt in the month of July following. +LJ010-0259|a deformed lad among the crowd was seen to present a pistol at Her Majesty's carriage,|a deformed lad among the crowd was seen to present a pistol at Her Majesty's carriage, +LJ010-0260|in the Mall, about half-way between Buckingham and St. James's Palaces.|in the Mall, about half-way between Buckingham and St. James's Palaces. +LJ010-0261|Only one person saw the movement, a lad named Dasset, who at once collared the cripple, and taking him up to two policemen,|Only one person saw the movement, a lad named Dasset, who at once collared the cripple, and taking him up to two policemen, +LJ010-0263|The policemen treated the matter as a hoax, and allowed the culprit to make off.|The policemen treated the matter as a hoax, and allowed the culprit to make off. +LJ010-0264|Later on, however, Dasset was himself seized and interrogated,|Later on, however, Dasset was himself seized and interrogated, +LJ010-0265|and on his information handbills were circulated, giving the exact description of the deformed youth, who had a hump-back,|and on his information handbills were circulated, giving the exact description of the deformed youth, who had a hump-back, +LJ010-0266|and a long, sickly, pale face, with light hair;|and a long, sickly, pale face, with light hair; +LJ010-0267|his nose was marked with a scar or black patch, and he was altogether of a dirty appearance.|his nose was marked with a scar or black patch, and he was altogether of a dirty appearance. +LJ010-0268|It happened that a lad named Bean had absconded from his father's home some weeks before,|It happened that a lad named Bean had absconded from his father's home some weeks before, +LJ010-0269|whose description, as given by his father to the police,|whose description, as given by his father to the police, +LJ010-0271|A visit to the father's residence was followed by the arrest of the son, who had by this time returned.|A visit to the father's residence was followed by the arrest of the son, who had by this time returned. +LJ010-0272|This son, John William Bean, was fully identified by Dasset, and presently examined by the Privy Council.|This son, John William Bean, was fully identified by Dasset, and presently examined by the Privy Council. +LJ010-0273|He was eventually charged with a misdemeanor, the capital charge having been abandoned, and committed for trial.|He was eventually charged with a misdemeanor, the capital charge having been abandoned, and committed for trial. +LJ010-0274|Much the same motives of seeking notoriety seem to have impelled Bean, who was perfectly sane, to his rash act;|Much the same motives of seeking notoriety seem to have impelled Bean, who was perfectly sane, to his rash act; +LJ010-0275|but it was proved that the pistol was not loaded with ball,|but it was proved that the pistol was not loaded with ball, +LJ010-0276|and he was only convicted of an attempt "to harass, vex, and grieve the sovereign.|and he was only convicted of an attempt "to harass, vex, and grieve the sovereign. +LJ010-0277|Lord Abinger sentenced him to eighteen months' imprisonment in Newgate,|Lord Abinger sentenced him to eighteen months' imprisonment in Newgate, +LJ010-0278|but the place of durance was changed, to meet the existing law, to Millbank Penitentiary.|but the place of durance was changed, to meet the existing law, to Millbank Penitentiary. +LJ010-0279|I shall mention briefly one more case, in which, however, there was no murderous intent, before I pass on to other crimes.|I shall mention briefly one more case, in which, however, there was no murderous intent, before I pass on to other crimes. +LJ010-0280|On June 1850 the Queen was once more subjected to cowardly outrage, the offender being a Mr. Pate, a gentleman by birth,|On June eighteen fifty the Queen was once more subjected to cowardly outrage, the offender being a Mr. Pate, a gentleman by birth, +LJ010-0281|who had borne the Queen's commission, first as cornet, and then lieutenant, in the 10th Hussars.|who had borne the Queen's commission, first as cornet, and then lieutenant, in the tenth Hussars. +LJ010-0282|Pate was said to be an eccentric person, given to strange acts and antics, such as mixing whiskey and camphor with his morning bath-water,|Pate was said to be an eccentric person, given to strange acts and antics, such as mixing whiskey and camphor with his morning bath-water, +LJ010-0283|and walking for choice through prickly gorse bushes.|and walking for choice through prickly gorse bushes. +LJ010-0284|He always kept the blinds down at his chambers in Jermyn Street; and as the St. James's clock chimed quarter-past three,|He always kept the blinds down at his chambers in Jermyn Street; and as the St. James's clock chimed quarter-past three, +LJ010-0285|invariably went out in a cab, for which he always paid the same fare,|invariably went out in a cab, for which he always paid the same fare, +LJ010-0286|nine shillings, all in shillings, and no other coin.|nine shillings, all in shillings, and no other coin. +LJ010-0287|But this was not sufficient to constitute lunacy, nor was his plea of "momentary uncontrollable impulse"|But this was not sufficient to constitute lunacy, nor was his plea of "momentary uncontrollable impulse" +LJ010-0288|deemed valid as any palliation of his offense.|deemed valid as any palliation of his offense. +LJ010-0289|That offense was a brutal assault upon Her Majesty, whom he struck in the face with a small stick just as she was leaving Cambridge House.|That offense was a brutal assault upon Her Majesty, whom he struck in the face with a small stick just as she was leaving Cambridge House. +LJ010-0290|The blow crushed the bonnet and bruised the forehead of the Queen, who was happily not otherwise injured.|The blow crushed the bonnet and bruised the forehead of the Queen, who was happily not otherwise injured. +LJ010-0291|Pate was found guilty, and sentenced to seven years' transportation, the judge, Baron Alderson, abstaining from inflicting the penalty of whipping,|Pate was found guilty, and sentenced to seven years' transportation, the judge, Baron Alderson, abstaining from inflicting the penalty of whipping, +LJ010-0292|which was authorized by a recent act, on account of Mr. Pate's family and position in life.|which was authorized by a recent act, on account of Mr. Pate's family and position in life. +LJ010-0294|frauds indicating great boldness, extensive design, and ingenuity became more prevalent.|frauds indicating great boldness, extensive design, and ingenuity became more prevalent. +LJ010-0295|The increase of bank forgeries, and its cause, I referred to in a previous chapter.|The increase of bank forgeries, and its cause, I referred to in a previous chapter. +LJ010-0296|At one session of the Old Bailey, in 1821, no less than thirty-five true bills were found for passing forged notes.|At one session of the Old Bailey, in eighteen twenty-one, no less than thirty-five true bills were found for passing forged notes. +LJ010-0297|But there were other notorious cases of forgery.|But there were other notorious cases of forgery. +LJ010-0298|That of Fauntleroy the banker, in 1824,|That of Fauntleroy the banker, in eighteen twenty-four, +LJ010-0299|caused much excitement at the time on account of the magnitude of the fraud, and the seeming probity of the culprit.|caused much excitement at the time on account of the magnitude of the fraud, and the seeming probity of the culprit. +LJ010-0300|Mr. Fauntleroy was a member of a banking firm, which his father had established in conjunction with a gentleman of the name of Marsh, and others.|Mr. Fauntleroy was a member of a banking firm, which his father had established in conjunction with a gentleman of the name of Marsh, and others. +LJ010-0301|He had entered the house as clerk in 1800;|He had entered the house as clerk in eighteen hundred; +LJ010-0302|in 1807, and when only twenty-two, he succeeded to his father's share in the business.|in eighteen oh seven, and when only twenty-two, he succeeded to his father's share in the business. +LJ010-0303|According to Fauntleroy's own case, he found at once that the firm was heavily involved,|According to Fauntleroy's own case, he found at once that the firm was heavily involved, +LJ010-0304|through advances made to various builders, and that it could only maintain its credit by wholesale discounting.|through advances made to various builders, and that it could only maintain its credit by wholesale discounting. +LJ010-0305|Its embarrassments were greatly increased by the bankruptcy of two of its clients in the building trade,|Its embarrassments were greatly increased by the bankruptcy of two of its clients in the building trade, +LJ010-0306|and the bank became liable for a sum of £170,000.|and the bank became liable for a sum of one hundred seventy thousand pounds. +LJ010-0307|New liabilities were incurred to the extent of £100,000 by more failures, and in 1819,|New liabilities were incurred to the extent of one hundred thousand pounds by more failures, and in eighteen nineteen, +LJ010-0308|by the death of one of the partners, a large sum in cash had to be withdrawn from the bank to pay his heirs.|by the death of one of the partners, a large sum in cash had to be withdrawn from the bank to pay his heirs. +LJ010-0309|"During these numerous and trying difficulties" -- it is Mr. Fauntleroy who speaks --|"During these numerous and trying difficulties" -- it is Mr. Fauntleroy who speaks -- +LJ010-0310|the house was nearly without resources, and the whole burthen of management falling on me, I sought resources where I could;|the house was nearly without resources, and the whole burthen of management falling on me, I sought resources where I could; +LJ010-0311|in other words, he forged powers of attorney, and proceeded to realize securities lodged in his bank under various names.|in other words, he forged powers of attorney, and proceeded to realize securities lodged in his bank under various names. +LJ010-0312|Among the prisoner's private papers, one was found giving full details of the stock he had feloniously sold out,|Among the prisoner's private papers, one was found giving full details of the stock he had feloniously sold out, +LJ010-0313|the sum total amounting to some £170,000, with a declaration in his own handwriting to the following effect.|the sum total amounting to some one hundred seventy thousand pounds, with a declaration in his own handwriting to the following effect. +LJ010-0314|In order to keep up the credit of our house, I have forged powers of attorney for the above sums and parties,|In order to keep up the credit of our house, I have forged powers of attorney for the above sums and parties, +LJ010-0315|and sold out to the amount here stated, and without the knowledge of my partners.|and sold out to the amount here stated, and without the knowledge of my partners. +LJ010-0316|I kept up the payments of the dividends, but made no entries of such payments in my books.|I kept up the payments of the dividends, but made no entries of such payments in my books. +LJ010-0317|The bank began first to refuse our acceptances, and to destroy the credit of our house; the bank shall smart for it.|The bank began first to refuse our acceptances, and to destroy the credit of our house; the bank shall smart for it. +LJ011-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 14: Newgate notorieties, part 1|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section fourteen: Newgate notorieties, part one +LJ011-0002|Many stories were in circulation at the time of Fauntleroy's trial with regard to his forgeries. It was said that he had by means of them|Many stories were in circulation at the time of Fauntleroy's trial with regard to his forgeries. It was said that he had by means of them +LJ011-0003|sold out so large an amount of stock, that he paid £16,000 a year in dividends to escape detection.|sold out so large an amount of stock, that he paid sixteen thousand pounds a year in dividends to escape detection. +LJ011-0004|Once he ran a narrow risk of being found out.|Once he ran a narrow risk of being found out. +LJ011-0005|A lady in the country, who had £13,000 in the stocks, desired her London agent to sell them out.|A lady in the country, who had thirteen thousand pounds in the stocks, desired her London agent to sell them out. +LJ011-0006|He went to the bank, and found that no stocks stood in her name. He called at once upon Fauntleroy, his client's bankers, for an explanation,|He went to the bank, and found that no stocks stood in her name. He called at once upon Fauntleroy, his client's bankers, for an explanation, +LJ011-0007|and was told by Mr. Fauntleroy that the lady had desired him to sell out, "which I have done," added the fraudulent banker, "and here are the proceeds,"|and was told by Mr. Fauntleroy that the lady had desired him to sell out, "which I have done," added the fraudulent banker, "and here are the proceeds," +LJ011-0008|whereupon he produced exchequer bills to the amount.|whereupon he produced exchequer bills to the amount. +LJ011-0010|On another occasion the banker forged a gentleman's name while the latter was sitting with him in his private room,|On another occasion the banker forged a gentleman's name while the latter was sitting with him in his private room, +LJ011-0011|and took the instrument out to a clerk with the ink not dry.|and took the instrument out to a clerk with the ink not dry. +LJ011-0012|It must be added that the Bank of England, on discovering the forgeries,|It must be added that the Bank of England, on discovering the forgeries, +LJ011-0013|replaced the stock in the names of the original holders, who might otherwise have been completely ruined.|replaced the stock in the names of the original holders, who might otherwise have been completely ruined. +LJ011-0014|A newspaper report of the time describes Fauntleroy as a well-made man of middle stature.|A newspaper report of the time describes Fauntleroy as a well-made man of middle stature. +LJ011-0015|His hair, though gray, was thick, and lay smooth over his forehead.|His hair, though gray, was thick, and lay smooth over his forehead. +LJ011-0016|His countenance had an expression of most subdued resignation.|His countenance had an expression of most subdued resignation. +LJ011-0017|The impression which his appearance altogether was calculated to make was that of the profoundest commiseration.|The impression which his appearance altogether was calculated to make was that of the profoundest commiseration. +LJ011-0018|The crime, long carried on without detection, was first discovered in 1820,|The crime, long carried on without detection, was first discovered in eighteen twenty, +LJ011-0019|when it was found that a sum of £10,000, standing in the name of three trustees, of whom Fauntleroy was one,|when it was found that a sum of ten thousand pounds, standing in the name of three trustees, of whom Fauntleroy was one, +LJ011-0020|had been sold out under a forged power of attorney.|had been sold out under a forged power of attorney. +LJ011-0021|Further investigations brought other similar frauds to light,|Further investigations brought other similar frauds to light, +LJ011-0022|and fixed the whole sum misappropriated at £170,000, the first forgery dating back to 1814.|and fixed the whole sum misappropriated at one hundred seventy thousand pounds, the first forgery dating back to eighteen fourteen. +LJ011-0023|A run upon the bank immediately followed, which was only met by a suspension of payment and the closing of its doors.|A run upon the bank immediately followed, which was only met by a suspension of payment and the closing of its doors. +LJ011-0024|Meanwhile public gossip was busy with Fauntleroy's name,|Meanwhile public gossip was busy with Fauntleroy's name, +LJ011-0025|and it was openly stated in the press and in conversation that the proceeds of these frauds had been squandered in chambering, gambling, and debauchery.|and it was openly stated in the press and in conversation that the proceeds of these frauds had been squandered in chambering, gambling, and debauchery. +LJ011-0026|Fauntleroy was scouted as a licentious libertine, a deep and determined gamester, a spendthrift whose extravagance knew no bounds.|Fauntleroy was scouted as a licentious libertine, a deep and determined gamester, a spendthrift whose extravagance knew no bounds. +LJ011-0027|The veil was lifted from his private life, and he was accused of persistent immorality.|The veil was lifted from his private life, and he was accused of persistent immorality. +LJ011-0028|In his defense|In his defense +LJ011-0029|he sought to rebut these charges, which indeed were never clearly made out, and it is pretty certain that his own account of the causes which led him into dishonesty|he sought to rebut these charges, which indeed were never clearly made out, and it is pretty certain that his own account of the causes which led him into dishonesty +LJ011-0030|was substantially true|was substantially true +LJ011-0031|He called many witnesses, seventeen in all, to speak of him as they had found him; and these, all respectable city merchants and business men,|He called many witnesses, seventeen in all, to speak of him as they had found him; and these, all respectable city merchants and business men, +LJ011-0032|declared that they had hitherto formed a high opinion of his honor, integrity, and goodness of disposition,|declared that they had hitherto formed a high opinion of his honor, integrity, and goodness of disposition, +LJ011-0033|deeming him the last person capable of a dishonorable action.|deeming him the last person capable of a dishonorable action. +LJ011-0034|These arguments availed little with the jury, who after a short deliberation found Fauntleroy guilty, and he was sentenced to death.|These arguments availed little with the jury, who after a short deliberation found Fauntleroy guilty, and he was sentenced to death. +LJ011-0035|Every endeavor was used, however, to obtain a commutation of sentence. His case was twice argued before the judges on points of law,|Every endeavor was used, however, to obtain a commutation of sentence. His case was twice argued before the judges on points of law, +LJ011-0036|but the result in both cases was unfavorable.|but the result in both cases was unfavorable. +LJ011-0037|Appeals were made to the Home Secretary, and all possible political interest brought to bear, but without success.|Appeals were made to the Home Secretary, and all possible political interest brought to bear, but without success. +LJ011-0038|Fauntleroy meanwhile lay in Newgate, not herded with other condemned prisoners, as the custom was,|Fauntleroy meanwhile lay in Newgate, not herded with other condemned prisoners, as the custom was, +LJ011-0039|but in a separate chamber, that belonging to one of the warders of the jail.|but in a separate chamber, that belonging to one of the warders of the jail. +LJ011-0040|I find in the chaplain's journal, under date 1824, various entries relative to this prisoner.|I find in the chaplain's journal, under date eighteen twenty-four, various entries relative to this prisoner. +LJ011-0041|Visited Mr. Fauntleroy. My application for books for him not having been attended, I had no prayer-book to give him.|Visited Mr. Fauntleroy. My application for books for him not having been attended, I had no prayer-book to give him. +LJ011-0042|Visited Mr. Fauntleroy. The sheriffs have very kindly permitted him to remain in the turnkey's room where he was originally placed; nor can I omit expressing a hope|Visited Mr. Fauntleroy. The sheriffs have very kindly permitted him to remain in the turnkey's room where he was originally placed; nor can I omit expressing a hope +LJ011-0043|that this may prove the beginning of a better system of confinement, and that every description of persons who may be unfortunately under sentence of death|that this may prove the beginning of a better system of confinement, and that every description of persons who may be unfortunately under sentence of death +LJ011-0044|will no longer be herded indiscriminately together.|will no longer be herded indiscriminately together. +LJ011-0045|The kindliness of the city authorities to Fauntleroy was not limited to the assignment of a separate place of durance.|The kindliness of the city authorities to Fauntleroy was not limited to the assignment of a separate place of durance. +LJ011-0046|As I have already said, they took the chaplain seriously to task for the bad taste shown in the condemned sermon preached before Fauntleroy.|As I have already said, they took the chaplain seriously to task for the bad taste shown in the condemned sermon preached before Fauntleroy. +LJ011-0047|This was on the text,|This was on the text, +LJ011-0048|Wherefore let him that thinketh he standeth take heed lest he fall," and was full of the most pointed allusions to the culprit.|Wherefore let him that thinketh he standeth take heed lest he fall," and was full of the most pointed allusions to the culprit. +LJ011-0049|Fauntleroy constantly groaned aloud while the sermon proceeded, and contemporary reports declared|Fauntleroy constantly groaned aloud while the sermon proceeded, and contemporary reports declared +LJ011-0050|that he appeared to feel deeply the force of the reverend gentleman's observations, especially when the chaplain spoke of|that he appeared to feel deeply the force of the reverend gentleman's observations, especially when the chaplain spoke of +LJ011-0051|the great magnitude of our erring brother's offense, one of the most dangerous description in a trading community.|the great magnitude of our erring brother's offense, one of the most dangerous description in a trading community. +LJ011-0052|The sermon ended with an appeal to the dying man, exhorting him to penitence.|The sermon ended with an appeal to the dying man, exhorting him to penitence. +LJ011-0053|This "personality," and it can be called by no other name, is carefully excluded from prison pulpit utterances on the eve of an execution.|This "personality," and it can be called by no other name, is carefully excluded from prison pulpit utterances on the eve of an execution. +LJ011-0054|A very curious and, in its way, amusing circumstance in connection with this case was the offer of a certain Italian,|A very curious and, in its way, amusing circumstance in connection with this case was the offer of a certain Italian, +LJ011-0055|Edmund Angelini, to take Fauntleroy's place.|Edmund Angelini, to take Fauntleroy's place. +LJ011-0056|Angelini wrote to the Lord Mayor to this effect,|Angelini wrote to the Lord Mayor to this effect, +LJ011-0057|urging that Fauntleroy was a father, a citizen: "his life is useful, mine a burthen, to the State."|urging that Fauntleroy was a father, a citizen: "his life is useful, mine a burthen, to the State." +LJ011-0058|He was summoned to the Mansion House, where he repeated his request, crying, "Accordez moi cette grâce," with much urgency.|He was summoned to the Mansion House, where he repeated his request, crying, "Accordez moi cette grâce," with much urgency. +LJ011-0059|There were doubts of his sanity.|There were doubts of his sanity. +LJ011-0060|He wrote afterwards to the effect that the moment he had offered himself, an unknown assassin came to aim a blow at him.|He wrote afterwards to the effect that the moment he had offered himself, an unknown assassin came to aim a blow at him. +LJ011-0061|Let this monster give his name; I am ready to fight him. I am still determined to put myself in the place of Mr. Fauntleroy.|Let this monster give his name; I am ready to fight him. I am still determined to put myself in the place of Mr. Fauntleroy. +LJ011-0062|If the law of this country can receive such a sacrifice, my death will render to heaven an innocent man, and to earth a repentant sinner.|If the law of this country can receive such a sacrifice, my death will render to heaven an innocent man, and to earth a repentant sinner. +LJ011-0063|Fauntleroy was not entirely dependent upon the ordinary for ghostly counsel in his extremity.|Fauntleroy was not entirely dependent upon the ordinary for ghostly counsel in his extremity. +LJ011-0064|He was also attended by the Rev. Mr. Springett and the indefatigable Mr. Baker, whose name has already been mentioned.|He was also attended by the Rev. Mr. Springett and the indefatigable Mr. Baker, whose name has already been mentioned. +LJ011-0065|When led out on the morning of his execution, these two last-named gentlemen each took hold of one of his arms, and so accompanied him to the scaffold.|When led out on the morning of his execution, these two last-named gentlemen each took hold of one of his arms, and so accompanied him to the scaffold. +LJ011-0066|The concourse in front of Newgate was enormous, but much sympathy was evinced for this unfortunate victim to human weakness and ruthless laws.|The concourse in front of Newgate was enormous, but much sympathy was evinced for this unfortunate victim to human weakness and ruthless laws. +LJ011-0067|A report was, moreover, widely circulated, and the impression long prevailed, that he actually escaped death.|A report was, moreover, widely circulated, and the impression long prevailed, that he actually escaped death. +LJ011-0068|It was said that strangulation had been prevented by the insertion of a silver tube in his wind-pipe,|It was said that strangulation had been prevented by the insertion of a silver tube in his wind-pipe, +LJ011-0069|and that after hanging for the regulated time he was taken down and easily restored to consciousness.|and that after hanging for the regulated time he was taken down and easily restored to consciousness. +LJ011-0070|Afterwards, according to the common rumor, he went abroad and lived there for many years; but the story is not only wholly unsubstantiated,|Afterwards, according to the common rumor, he went abroad and lived there for many years; but the story is not only wholly unsubstantiated, +LJ011-0071|but there is good evidence to show that the body after execution was handed over to his friends and interred privately.|but there is good evidence to show that the body after execution was handed over to his friends and interred privately. +LJ011-0072|Some years were still to elapse before capital punishment ceased to be the penalty for forgery, and in the interval|Some years were still to elapse before capital punishment ceased to be the penalty for forgery, and in the interval +LJ011-0073|several persons were sentenced to or suffered death for this crime.|several persons were sentenced to or suffered death for this crime. +LJ011-0074|There were two notable capital convictions for forgery in 1828.|There were two notable capital convictions for forgery in eighteen twenty-eight. +LJ011-0075|One was that of Captain Montgomery, who assumed the aliases of Colonel Wallace and Colonel Morgan.|One was that of Captain Montgomery, who assumed the aliases of Colonel Wallace and Colonel Morgan. +LJ011-0076|His offense was uttering forged notes, and there was strong suspicion that he had long subsisted entirely by this fraud.|His offense was uttering forged notes, and there was strong suspicion that he had long subsisted entirely by this fraud. +LJ011-0077|The act for which he was taken into custody was the payment of a forged ten-pound note for half-a-dozen silver spoons.|The act for which he was taken into custody was the payment of a forged ten-pound note for half-a-dozen silver spoons. +LJ011-0078|Montgomery was an adept at forgery. He had gone wrong early. Although born of respectable parents, and gazetted to a commission in the army,|Montgomery was an adept at forgery. He had gone wrong early. Although born of respectable parents, and gazetted to a commission in the army, +LJ011-0079|he soon left the service and betook himself to dishonest ways.|he soon left the service and betook himself to dishonest ways. +LJ011-0080|His first forgery was the marvelous imitation of the signature of the Hon. Mr. Neville, M.P., who wrote an extremely cramped and curious hand.|His first forgery was the marvelous imitation of the signature of the Hon. Mr. Neville, M.P., who wrote an extremely cramped and curious hand. +LJ011-0081|He was not prosecuted for this fraud on account of the respectability of his family, and soon after this escape|He was not prosecuted for this fraud on account of the respectability of his family, and soon after this escape +LJ011-0082|he came to London, where he practiced as a professional swindler and cheat.|he came to London, where he practiced as a professional swindler and cheat. +LJ011-0083|For a long time justice did not overtake him for any criminal offense, but he was frequently in Newgate and in the King's Bench for debt.|For a long time justice did not overtake him for any criminal offense, but he was frequently in Newgate and in the King's Bench for debt. +LJ011-0084|After three years' confinement in the latter prison he passed himself off as his brother, Colonel Montgomery,|After three years' confinement in the latter prison he passed himself off as his brother, Colonel Montgomery, +LJ011-0085|a distinguished officer, and would have married an heiress had not the imposture been discovered in time.|a distinguished officer, and would have married an heiress had not the imposture been discovered in time. +LJ011-0086|He then took to forging bank-notes, and was arrested as I have described above.|He then took to forging bank-notes, and was arrested as I have described above. +LJ011-0087|Montgomery was duly sentenced to death, but he preferred suicide to the gallows. After sentence his demeanor was serious yet firm.|Montgomery was duly sentenced to death, but he preferred suicide to the gallows. After sentence his demeanor was serious yet firm. +LJ011-0088|The night previous to that fixed for his execution he wrote several letters, one of them being to Edward Gibbon Wakefield, a fellow-prisoner,|The night previous to that fixed for his execution he wrote several letters, one of them being to Edward Gibbon Wakefield, a fellow-prisoner, +LJ011-0089|and listened attentively to the ordinary, who read him the well-known address written and delivered by Dr. Dodd previous to his own execution for forgery.|and listened attentively to the ordinary, who read him the well-known address written and delivered by Dr. Dodd previous to his own execution for forgery. +LJ011-0090|But next morning he was found dead in his cell.|But next morning he was found dead in his cell. +LJ011-0091|In one corner after much search a phial was found labeled "Prussic acid,"|In one corner after much search a phial was found labeled "Prussic acid," +LJ011-0092|which it was asserted he had been in the habit of carrying about his person ever since he had taken to passing forged notes, as an antidote against disgrace.|which it was asserted he had been in the habit of carrying about his person ever since he had taken to passing forged notes, as an antidote against disgrace. +LJ011-0093|This phial he had managed to retain in his possession in spite of the frequent searches to which he was subjected in Newgate.|This phial he had managed to retain in his possession in spite of the frequent searches to which he was subjected in Newgate. +LJ011-0094|The second conviction for forgery in 1828 was that of the Quaker Joseph Hunton, a man of previously the highest repute in the city of London.|The second conviction for forgery in eighteen twenty-eight was that of the Quaker Joseph Hunton, a man of previously the highest repute in the city of London. +LJ011-0095|He had prospered in early life, was a slop-seller on a large scale at Bury St. Edmunds, and a sugar-baker in the metropolis.|He had prospered in early life, was a slop-seller on a large scale at Bury St. Edmunds, and a sugar-baker in the metropolis. +LJ011-0096|He married a lady also belonging to the Society of Friends, who brought him a large fortune, which, and his own money, he put into a city firm,|He married a lady also belonging to the Society of Friends, who brought him a large fortune, which, and his own money, he put into a city firm, +LJ011-0097|that of Dickson and Co.|that of Dickson and Co. +LJ011-0098|He soon, however, became deeply involved in Stock Exchange speculations,|He soon, however, became deeply involved in Stock Exchange speculations, +LJ011-0099|and losing heavily, to meet the claims upon him he put out a number of forged bills of exchange or acceptances,|and losing heavily, to meet the claims upon him he put out a number of forged bills of exchange or acceptances, +LJ011-0100|to which the signature of one Wilkins of Abingdon was found to be forged.|to which the signature of one Wilkins of Abingdon was found to be forged. +LJ011-0101|Hunton tried to fly the country on the detection of the fraud, but was arrested at Plymouth just as he was on the point of leaving England in the New York packet.|Hunton tried to fly the country on the detection of the fraud, but was arrested at Plymouth just as he was on the point of leaving England in the New York packet. +LJ011-0102|He had gone on board in his Quaker dress, but when captured was found in a light-green frock,|He had gone on board in his Quaker dress, but when captured was found in a light-green frock, +LJ011-0103|a pair of light gray pantaloons, a black stock and a foraging cap.|a pair of light gray pantaloons, a black stock and a foraging cap. +LJ011-0104|Hunton was put upon his trial at the Old Bailey, and in due course sentenced to death.|Hunton was put upon his trial at the Old Bailey, and in due course sentenced to death. +LJ011-0105|His defense was that the forged acceptances would have been met on coming to maturity, and that he had no real desire to defraud.|His defense was that the forged acceptances would have been met on coming to maturity, and that he had no real desire to defraud. +LJ011-0106|Hunton accepted his sentence with great resignation, although he protested against the inhumanity of the laws which condemned him to death.|Hunton accepted his sentence with great resignation, although he protested against the inhumanity of the laws which condemned him to death. +LJ011-0107|On entering Newgate he said,|On entering Newgate he said, +LJ011-0108|I wish after this day to have communication with nobody; let me take leave of my wife, and family, and friends. I have already suffered an execution;|I wish after this day to have communication with nobody; let me take leave of my wife, and family, and friends. I have already suffered an execution; +LJ011-0109|my heart has undergone that horrible penalty.|my heart has undergone that horrible penalty. +LJ011-0110|He was, however, visited by and received his wife, and several members of the Society of Friends.|He was, however, visited by and received his wife, and several members of the Society of Friends. +LJ011-0111|Two elders of the meeting sat up with him in the press yard the whole of the night previous to execution, and a third,|Two elders of the meeting sat up with him in the press yard the whole of the night previous to execution, and a third, +LJ011-0112|Mr. Sparks Moline, came to attend him to the scaffold.|Mr. Sparks Moline, came to attend him to the scaffold. +LJ011-0113|He met his death with unshaken firmness, only entreating that a certain blue handkerchief,|He met his death with unshaken firmness, only entreating that a certain blue handkerchief, +LJ011-0114|to which he seemed fondly attached, should be used to bandage his eyes, which request was readily granted.|to which he seemed fondly attached, should be used to bandage his eyes, which request was readily granted. +LJ011-0115|Hunton's execution no doubt aroused public attention to the cruelty and futility of the capital law against forgery.|Hunton's execution no doubt aroused public attention to the cruelty and futility of the capital law against forgery. +LJ011-0116|A society which had already been started against capital punishment|A society which had already been started against capital punishment +LJ011-0117|devoted its efforts first to a mitigation of the forgery statute, but could not immediately accomplish much.|devoted its efforts first to a mitigation of the forgery statute, but could not immediately accomplish much. +LJ011-0118|In 1829 the gallows claimed two more victims for this offense.|In eighteen twenty-nine the gallows claimed two more victims for this offense. +LJ011-0119|One was Richard Gifford, a well-educated youth who had been at Christ's Hospital, and afterwards in the National Debt Office.|One was Richard Gifford, a well-educated youth who had been at Christ's Hospital, and afterwards in the National Debt Office. +LJ011-0120|Unfortunately he took to drink, lost his appointment, and fell from bad to worse.|Unfortunately he took to drink, lost his appointment, and fell from bad to worse. +LJ011-0121|Suddenly, after being at the lowest depths, he emerged, and was found by his friends living in comfort in the Waterloo Road.|Suddenly, after being at the lowest depths, he emerged, and was found by his friends living in comfort in the Waterloo Road. +LJ011-0122|His funds, which he pretended came to him with a rich wife, were really the proceeds of frauds upon the Bank of England.|His funds, which he pretended came to him with a rich wife, were really the proceeds of frauds upon the Bank of England. +LJ011-0123|He forged the names of people who held stock on the Bank books, and got the value of the stock;|He forged the names of people who held stock on the Bank books, and got the value of the stock; +LJ011-0124|he also forged dividend receipts and got the dividends. He was only six-and-twenty when he was hanged.|he also forged dividend receipts and got the dividends. He was only six-and-twenty when he was hanged. +LJ011-0125|The other and the last criminal executed for forgery in this country was one Maynard, who was convicted of a fraud upon the Custom House.|The other and the last criminal executed for forgery in this country was one Maynard, who was convicted of a fraud upon the Custom House. +LJ011-0126|In conjunction with two others, one of whom was a clerk in the Custom House,|In conjunction with two others, one of whom was a clerk in the Custom House, +LJ011-0127|and had access to the official records, he forged a warrant for one thousand nine-hundred seventy-three pounds and was paid the money by the comptroller general.|and had access to the official records, he forged a warrant for one thousand nine-hundred seventy-three pounds and was paid the money by the comptroller general. +LJ011-0128|Maynard was convicted of uttering the forged document, Jones of being an accessory; the third prisoner was acquitted.|Maynard was convicted of uttering the forged document, Jones of being an accessory; the third prisoner was acquitted. +LJ011-0129|Maynard was the only one who suffered death.|Maynard was the only one who suffered death. +LJ011-0130|This was on the last day of 1829. In the following session Sir Robert Peel brought in a bill to consolidate the acts relating to forgery.|This was on the last day of eighteen twenty-nine. In the following session Sir Robert Peel brought in a bill to consolidate the acts relating to forgery. +LJ011-0131|Upon the third reading of this bill Sir James Macintosh moved as an amendment that capital punishment should be abolished for all crimes of forgery,|Upon the third reading of this bill Sir James Macintosh moved as an amendment that capital punishment should be abolished for all crimes of forgery, +LJ011-0132|except the forgery of wills and powers of attorney.|except the forgery of wills and powers of attorney. +LJ011-0133|This amendment was strongly supported outside the House, and a petition in favor of its passing was presented,|This amendment was strongly supported outside the House, and a petition in favor of its passing was presented, +LJ011-0134|signed by more than a thousand members of banking firms.|signed by more than a thousand members of banking firms. +LJ011-0135|Macintosh's amendment was carried in the Commons, but the new law did not pass the Lords, who re-enacted the capital penalty.|Macintosh's amendment was carried in the Commons, but the new law did not pass the Lords, who re-enacted the capital penalty. +LJ011-0136|Still no sentence of death was carried out for the offense, and in 1832|Still no sentence of death was carried out for the offense, and in eighteen thirty-two +LJ011-0137|the Attorney-General introduced a bill to abolish capital punishment entirely for forgery.|the Attorney-General introduced a bill to abolish capital punishment entirely for forgery. +LJ011-0138|It passed the Commons, but opposition was again encountered in the Lords.|It passed the Commons, but opposition was again encountered in the Lords. +LJ011-0139|This time they sent back the bill, re-enacting only the two penalties for will forging and the forging of powers of attorney;|This time they sent back the bill, re-enacting only the two penalties for will forging and the forging of powers of attorney; +LJ011-0140|in other words, they had advanced in 1832 to the point at which the Lower House had arrived in 1830.|in other words, they had advanced in eighteen thirty-two to the point at which the Lower House had arrived in eighteen thirty. +LJ011-0141|There were at the moment in Newgate six convicts sentenced to death for forging wills.|There were at the moment in Newgate six convicts sentenced to death for forging wills. +LJ011-0142|The question was whether the Government would dare to take their lives at the bidding of the House of Lords,|The question was whether the Government would dare to take their lives at the bidding of the House of Lords, +LJ011-0143|and in defiance of the vote of the assembly which more accurately represented public opinion. It was indeed announced that their fate was sealed;|and in defiance of the vote of the assembly which more accurately represented public opinion. It was indeed announced that their fate was sealed; +LJ011-0144|but Mr. Joseph Hume pressed the Government hard, and obtained an assurance that the men should not be executed.|but Mr. Joseph Hume pressed the Government hard, and obtained an assurance that the men should not be executed. +LJ011-0145|The new Forgery Act with the Lords' amendment passed into law, but the latter proved perfectly harmless,|The new Forgery Act with the Lords' amendment passed into law, but the latter proved perfectly harmless, +LJ011-0146|and no person ever after suffered death for any variety of this crime.|and no person ever after suffered death for any variety of this crime. +LJ011-0147|I will include in this part of the present chapter almost one of the last instances of a crime which in time past|I will include in this part of the present chapter almost one of the last instances of a crime which in time past +LJ011-0148|had invariably been visited with the death penalty, and which was of a distinctly fraudulent nature.|had invariably been visited with the death penalty, and which was of a distinctly fraudulent nature. +LJ011-0149|The abduction of Miss Turner by the brothers Wakefield bore a strong resemblance to the carrying off and forcible marrying of heiresses as already described.|The abduction of Miss Turner by the brothers Wakefield bore a strong resemblance to the carrying off and forcible marrying of heiresses as already described. +LJ011-0150|Miss Turner was a school-girl of barely fifteen,|Miss Turner was a school-girl of barely fifteen, +LJ011-0151|only child of a gentleman of large property in Cheshire, of which county he was actually high sheriff at the time of his daughter's abduction.|only child of a gentleman of large property in Cheshire, of which county he was actually high sheriff at the time of his daughter's abduction. +LJ011-0152|The elder brother,|The elder brother, +LJ011-0153|Edward Gibbon Wakefield, the prime mover in the abduction, was a barrister, not exactly briefless, but without a large practice.|Edward Gibbon Wakefield, the prime mover in the abduction, was a barrister, not exactly briefless, but without a large practice. +LJ011-0154|He had, it was said, a good private income, and was already a widower with two children at the time of his committing the offense for which he was subsequently tried.|He had, it was said, a good private income, and was already a widower with two children at the time of his committing the offense for which he was subsequently tried. +LJ011-0155|He had eloped with his first wife from school.|He had eloped with his first wife from school. +LJ011-0156|While on a visit to Macclesfield he heard by chance of Miss Turner, and that she would inherit all her father's possessions.|While on a visit to Macclesfield he heard by chance of Miss Turner, and that she would inherit all her father's possessions. +LJ011-0157|He thereupon conceived an idea of carrying her off and marrying her willy nilly at Gretna Green.|He thereupon conceived an idea of carrying her off and marrying her willy nilly at Gretna Green. +LJ011-0158|The two brothers started at once for Liverpool, where Miss Turner was at school with a Mrs. Daulby.|The two brothers started at once for Liverpool, where Miss Turner was at school with a Mrs. Daulby. +LJ011-0159|At Manchester, en route, a traveling carriage was purchased, which was driven up to Mrs. Daulby's door at eight in the morning,|At Manchester, en route, a traveling carriage was purchased, which was driven up to Mrs. Daulby's door at eight in the morning, +LJ011-0160|and a servant hurriedly alighted from it, bearing a letter for Miss Turner.|and a servant hurriedly alighted from it, bearing a letter for Miss Turner. +LJ011-0161|This purported to be from the medical attendant of Mr. Turner, written at Shrigley, Mr. Turner's place of residence;|This purported to be from the medical attendant of Mr. Turner, written at Shrigley, Mr. Turner's place of residence; +LJ011-0162|and it stated that Mrs. Turner had been stricken with paralysis.|and it stated that Mrs. Turner had been stricken with paralysis. +LJ011-0163|She was not in immediate danger, but she wished to see her daughter, "as it was possible she might soon become incapable of recognizing any one."|She was not in immediate danger, but she wished to see her daughter, "as it was possible she might soon become incapable of recognizing any one." +LJ011-0164|Miss Turner, greatly agitated, accompanied the messenger who had brought this news, a disguised servant of Wakefield's,|Miss Turner, greatly agitated, accompanied the messenger who had brought this news, a disguised servant of Wakefield's, +LJ011-0165|who had plausibly explained that he had only recently been engaged at Shrigley.|who had plausibly explained that he had only recently been engaged at Shrigley. +LJ011-0166|The road taken was via Manchester, where the servant said a Dr. Hull was to be picked up to go on with them to Shrigley.|The road taken was via Manchester, where the servant said a Dr. Hull was to be picked up to go on with them to Shrigley. +LJ011-0167|At Manchester, however, the carriage stopped at the Albion Hotel.|At Manchester, however, the carriage stopped at the Albion Hotel. +LJ011-0168|Miss Turner was shown into a private room, where Mr. Wakefield soon presented himself.|Miss Turner was shown into a private room, where Mr. Wakefield soon presented himself. +LJ011-0169|Miss Turner, not knowing him, would have left the room, but he said he came from her father, and she remained.|Miss Turner, not knowing him, would have left the room, but he said he came from her father, and she remained. +LJ011-0170|Wakefield, in reply to her inquiries, satisfied her that her mother was well, and that the real reason for summoning her from school|Wakefield, in reply to her inquiries, satisfied her that her mother was well, and that the real reason for summoning her from school +LJ011-0171|was the state of her father's affairs.|was the state of her father's affairs. +LJ011-0172|Mr. Turner was on the verge of bankruptcy. He was at that moment at Kendal, and wished her to join him there at once.|Mr. Turner was on the verge of bankruptcy. He was at that moment at Kendal, and wished her to join him there at once. +LJ011-0173|Miss Turner consented to go on, and they traveled night and day towards the north.|Miss Turner consented to go on, and they traveled night and day towards the north. +LJ011-0174|But at Kendal there was no Mr. Turner, and, to allay Miss Turner's growing anxiety,|But at Kendal there was no Mr. Turner, and, to allay Miss Turner's growing anxiety, +LJ011-0175|Wakefield found it necessary to become more explicit regarding her father's affairs.|Wakefield found it necessary to become more explicit regarding her father's affairs. +LJ011-0176|He now pretended that Mr. Turner was also on his way to the border, pursued by sheriffs' officers.|He now pretended that Mr. Turner was also on his way to the border, pursued by sheriffs' officers. +LJ011-0177|The fact was, Wakefield went on to say, an uncle of his had advanced Mr. Turner £60,000, which had temporarily staved off ruin.|The fact was, Wakefield went on to say, an uncle of his had advanced Mr. Turner sixty thousand pounds, which had temporarily staved off ruin. +LJ011-0178|But another bank had since failed, and nothing could save Mr. Turner but the transfer of some property to Miss Turner, and its settlement on her,|But another bank had since failed, and nothing could save Mr. Turner but the transfer of some property to Miss Turner, and its settlement on her, +LJ011-0179|so that it might become the exclusive property of her husband, "whoever he might be."|so that it might become the exclusive property of her husband, "whoever he might be." +LJ011-0180|Wakefield added that it had been suggested he should marry Miss Turner, but that he had laughed at the idea.|Wakefield added that it had been suggested he should marry Miss Turner, but that he had laughed at the idea. +LJ011-0181|Wakefield's uncle took the matter more seriously, and declared that unless the marriage came off Mr. Turner must be sold up.|Wakefield's uncle took the matter more seriously, and declared that unless the marriage came off Mr. Turner must be sold up. +LJ011-0182|Miss Turner, thus pressed, consented to go on to Gretna Green.|Miss Turner, thus pressed, consented to go on to Gretna Green. +LJ011-0183|Passing through Carlisle, she was told that Mr. Turner was in the town, but could not show himself.|Passing through Carlisle, she was told that Mr. Turner was in the town, but could not show himself. +LJ011-0184|Nothing could release him from his trouble but the arrival of the marriage certificate from Gretna Green.|Nothing could release him from his trouble but the arrival of the marriage certificate from Gretna Green. +LJ011-0185|Filial affection rose superior to all scruples, and Miss Turner, having crossed the border, was married to Wakefield|Filial affection rose superior to all scruples, and Miss Turner, having crossed the border, was married to Wakefield +LJ011-0186|by the blacksmith in the usual way.|by the blacksmith in the usual way. +LJ011-0187|Returning to Carlisle, she now heard that her father had been set free, and had gone home to Shrigley, whither they were to follow him.|Returning to Carlisle, she now heard that her father had been set free, and had gone home to Shrigley, whither they were to follow him. +LJ011-0188|They set out, but at Leeds Wakefield found himself called suddenly to Paris;|They set out, but at Leeds Wakefield found himself called suddenly to Paris; +LJ011-0189|the other brother was accordingly sent on a pretended mission to Shrigley to bring Mr. Turner on to London, whither Wakefield and Miss Turner also proceeded.|the other brother was accordingly sent on a pretended mission to Shrigley to bring Mr. Turner on to London, whither Wakefield and Miss Turner also proceeded. +LJ011-0190|On arrival, Wakefield pretended that they had missed Mr. Turner, and must follow him over to France.|On arrival, Wakefield pretended that they had missed Mr. Turner, and must follow him over to France. +LJ011-0191|The strangely-married couple thereupon pressed on to Dover, and crossed over to Calais.|The strangely-married couple thereupon pressed on to Dover, and crossed over to Calais. +LJ011-0192|The fact of the abduction did not transpire for some days.|The fact of the abduction did not transpire for some days. +LJ011-0193|Then Mrs. Daulby learnt that Miss Turner had not arrived at Shrigley, but that she had gone to Manchester.|Then Mrs. Daulby learnt that Miss Turner had not arrived at Shrigley, but that she had gone to Manchester. +LJ011-0194|Friends went in pursuit and traced her to Huddersfield and further north.|Friends went in pursuit and traced her to Huddersfield and further north. +LJ011-0195|The terror and dismay of her parents were soon intensified by the receipt of a letter from Wakefield, at Carlisle, announcing the marriage.|The terror and dismay of her parents were soon intensified by the receipt of a letter from Wakefield, at Carlisle, announcing the marriage. +LJ011-0196|Mr. Turner at once set off for London, where he sought the assistance of the police,|Mr. Turner at once set off for London, where he sought the assistance of the police, +LJ011-0197|and presently ascertained that Wakefield had gone to the Continent with his involuntary bride.|and presently ascertained that Wakefield had gone to the Continent with his involuntary bride. +LJ011-0198|An uncle of Miss Wakefield's, accompanied by his solicitor and a Bow Street runner, at once went in pursuit.|An uncle of Miss Wakefield's, accompanied by his solicitor and a Bow Street runner, at once went in pursuit. +LJ011-0199|Meanwhile, a second letter turned up from Wakefield at Calais, in which he assured Mrs. Turner that Miss Turner was fondly attached to him,|Meanwhile, a second letter turned up from Wakefield at Calais, in which he assured Mrs. Turner that Miss Turner was fondly attached to him, +LJ011-0200|and went on to say, "I do assure you, madam, that it shall be the anxious endeavor of my life to promote her happiness by every means in my power."|and went on to say, "I do assure you, madam, that it shall be the anxious endeavor of my life to promote her happiness by every means in my power." +LJ011-0201|The game, however, was nearly up. Miss Turner was met by her uncle on Calais pier as she was walking with Wakefield.|The game, however, was nearly up. Miss Turner was met by her uncle on Calais pier as she was walking with Wakefield. +LJ011-0202|The uncle claimed her. The husband resisted.|The uncle claimed her. The husband resisted. +LJ011-0203|Monsieur le Maire was appealed to, and decided to leave it to the young lady, who at once abandoned Wakefield.|Monsieur le Maire was appealed to, and decided to leave it to the young lady, who at once abandoned Wakefield. +LJ011-0204|As he still urged his rights over his wife, Miss Turner cried out in protest, "No, no, I am not his wife;|As he still urged his rights over his wife, Miss Turner cried out in protest, "No, no, I am not his wife; +LJ011-0205|he carried me away by fraud and stratagem, and forced me to accompany him to Gretna Green|he carried me away by fraud and stratagem, and forced me to accompany him to Gretna Green +LJ011-0206|By the same forcible means I was compelled to quit England, and to trust myself to the protection of this person,|By the same forcible means I was compelled to quit England, and to trust myself to the protection of this person, +LJ011-0207|whom I never saw until I was taken from Liverpool, and never want to see again.|whom I never saw until I was taken from Liverpool, and never want to see again. +LJ011-0208|On this Wakefield gave in.|On this Wakefield gave in. +LJ011-0209|He surrendered the bride who had never been a wife, and she returned to England with her friends, while Wakefield went on alone to Paris.|He surrendered the bride who had never been a wife, and she returned to England with her friends, while Wakefield went on alone to Paris. +LJ011-0210|Mr. William Wakefield was arrested at Dover, conveyed to Chester, and committed to Lancaster Jail for trial at the next assizes,|Mr. William Wakefield was arrested at Dover, conveyed to Chester, and committed to Lancaster Jail for trial at the next assizes, +LJ011-0211|when indictments were preferred against both brothers "for having carried away Ellen Turner, spinster,|when indictments were preferred against both brothers "for having carried away Ellen Turner, spinster, +LJ011-0212|then a maid and heir apparent unto her father, for the sake of the lucre of her substance; and for having afterwards unlawfully and against her will|then a maid and heir apparent unto her father, for the sake of the lucre of her substance; and for having afterwards unlawfully and against her will +LJ011-0213|married the said Ellen Turner.|married the said Ellen Turner. +LJ011-0214|They were tried in March of the following year, Edward Wakefield having apparently given himself up,|They were tried in March of the following year, Edward Wakefield having apparently given himself up, +LJ011-0215|and found guilty, remaining in Lancaster Jail for a couple of months, when they were brought up to the court of King's Bench for judgment.|and found guilty, remaining in Lancaster Jail for a couple of months, when they were brought up to the court of King's Bench for judgment. +LJ011-0216|The prosecution pressed for a severe penalty. Edward Wakefield pleaded that his trial had already cost him £3000.|The prosecution pressed for a severe penalty. Edward Wakefield pleaded that his trial had already cost him three thousand pounds. +LJ011-0217|Mr. Justice Bayley, in summing up,|Mr. Justice Bayley, in summing up, +LJ011-0218|spoke severely of the gross deception practiced upon an innocent girl, and sentenced the brothers each to three years' imprisonment,|spoke severely of the gross deception practiced upon an innocent girl, and sentenced the brothers each to three years' imprisonment, +LJ011-0219|William Wakefield in Lancaster Jail, and Edward Gibbon Wakefield in Newgate, which sentences were duly enforced.|William Wakefield in Lancaster Jail, and Edward Gibbon Wakefield in Newgate, which sentences were duly enforced. +LJ011-0220|The marriage was annulled by an Act of Parliament, although Wakefield petitioned against it,|The marriage was annulled by an Act of Parliament, although Wakefield petitioned against it, +LJ011-0221|and was brought from Newgate, at his own request, to oppose the second reading of the bill.|and was brought from Newgate, at his own request, to oppose the second reading of the bill. +LJ011-0222|He also wrote and published a pamphlet from the jail to show that Miss Turner had been a consenting party to the marriage, and was really his wife.|He also wrote and published a pamphlet from the jail to show that Miss Turner had been a consenting party to the marriage, and was really his wife. +LJ011-0223|Neither his address nor his pamphlet availed much, for the bill for the divorce passed both Houses.|Neither his address nor his pamphlet availed much, for the bill for the divorce passed both Houses. +LJ011-0224|That Mr. Wakefield was a shrewd critic and close observer of all that went on in the Newgate of those days,|That Mr. Wakefield was a shrewd critic and close observer of all that went on in the Newgate of those days, +LJ011-0225|will be admitted by those who have read his book on "the punishment of death,|will be admitted by those who have read his book on "the punishment of death, +LJ011-0226|which was based on his jail experiences, and of which I have availed myself in the last chapter.|which was based on his jail experiences, and of which I have availed myself in the last chapter. +LJ011-0227|After their release from Lancaster and Newgate respectively, both Wakefields went abroad.|After their release from Lancaster and Newgate respectively, both Wakefields went abroad. +LJ011-0228|Mr. W. Wakefield served in a continental army, and rose to the rank of colonel,|Mr. W. Wakefield served in a continental army, and rose to the rank of colonel, +LJ011-0229|after which he went to New Zealand, and held an important post in that colony.|after which he went to New Zealand, and held an important post in that colony. +LJ011-0230|Mr. E. G. Wakefield took part in the scheme for the colonization of North Australia, and for some years resided in that colony.|Mr. E. G. Wakefield took part in the scheme for the colonization of North Australia, and for some years resided in that colony. +LJ011-0231|Miss Turner subsequently married Mr. Legh of Lym Hall, Cheshire.|Miss Turner subsequently married Mr. Legh of Lym Hall, Cheshire. +LJ011-0232|It must not be imagined that although highway robbery was now nearly extinct, and felonious outrages in the streets were rare,|It must not be imagined that although highway robbery was now nearly extinct, and felonious outrages in the streets were rare, +LJ011-0233|that thieves or depredators were idle or entirely unsuccessful.|that thieves or depredators were idle or entirely unsuccessful. +LJ011-0234|Bigger "jobs" than ever were planned and attempted,|Bigger "jobs" than ever were planned and attempted, +LJ011-0235|as in the burglary at Lambeth Palace, when the thieves were fortunately disappointed, the archbishop having, before he left town,|as in the burglary at Lambeth Palace, when the thieves were fortunately disappointed, the archbishop having, before he left town, +LJ011-0236|sent his plate-chests, eight in number, to the silversmith's for greater security.|sent his plate-chests, eight in number, to the silversmith's for greater security. +LJ011-0237|The jewelers were always a favorite prey of the London thieves.|The jewelers were always a favorite prey of the London thieves. +LJ011-0238|Shops were broken into, as when that of Grimaldi and Johnson, in the Strand, was robbed of watches to the value of £6000.|Shops were broken into, as when that of Grimaldi and Johnson, in the Strand, was robbed of watches to the value of six thousand pounds. +LJ011-0239|Where robbery with violence was intended, the perpetrators had now to adopt various shifts and contrivances to secure their victim.|Where robbery with violence was intended, the perpetrators had now to adopt various shifts and contrivances to secure their victim. +LJ011-0240|No more curious instance of this ever occurred than the assault made by one Howard upon a Mr. Mullay, with intent to rob him.|No more curious instance of this ever occurred than the assault made by one Howard upon a Mr. Mullay, with intent to rob him. +LJ011-0241|The latter had advertised, offering a sum of £1000 to anyone who would introduce him to some mercantile equipment.|The latter had advertised, offering a sum of one thousand pounds to anyone who would introduce him to some mercantile equipment. +LJ011-0242|Howard replied, desiring Mr. Mullay to call upon him in a house in Red Lion Square.|Howard replied, desiring Mr. Mullay to call upon him in a house in Red Lion Square. +LJ011-0243|Mr. Mullay went, and a second interview was agreed upon, when a third person, Mr. Owen,|Mr. Mullay went, and a second interview was agreed upon, when a third person, Mr. Owen, +LJ011-0244|through whose interest an appointment under Government was to be obtained for Mullay, would be present.|through whose interest an appointment under Government was to be obtained for Mullay, would be present. +LJ011-0245|Mr. Mullay called again, taking with him £500 in cash. Howard discovered this, and his manner was very suspicious;|Mr. Mullay called again, taking with him five hundred pounds in cash. Howard discovered this, and his manner was very suspicious; +LJ011-0246|there were weapons in the room -- a long knife, a heavy trap-ball bat, and a poker.|there were weapons in the room -- a long knife, a heavy trap-ball bat, and a poker. +LJ011-0247|Mr. Mullay became alarmed, and as Mr. Owen did not appear, withdrew;|Mr. Mullay became alarmed, and as Mr. Owen did not appear, withdrew; +LJ011-0248|Howard, strange to say, making no attempt to detain him; probably because Mullay promised to return a few days later, and to bring more money.|Howard, strange to say, making no attempt to detain him; probably because Mullay promised to return a few days later, and to bring more money. +LJ011-0249|On this renewed visit Mr. Owen was still absent, and Mr. Mullay agreed to write him a note from a copy Howard gave him.|On this renewed visit Mr. Owen was still absent, and Mr. Mullay agreed to write him a note from a copy Howard gave him. +LJ011-0250|While thus engaged, Howard thrust the poker into the fire.|While thus engaged, Howard thrust the poker into the fire. +LJ011-0251|Mullay protested, and then Howard, under the influence of ungovernable rage, as it seemed, jumped up,|Mullay protested, and then Howard, under the influence of ungovernable rage, as it seemed, jumped up, +LJ011-0252|locked the door, and attacked Mullay violently with the trap-ball bat and knife.|locked the door, and attacked Mullay violently with the trap-ball bat and knife. +LJ011-0253|Mullay defended himself, and managed to break the knife, but not before he had cut himself severely.|Mullay defended himself, and managed to break the knife, but not before he had cut himself severely. +LJ011-0254|A life and death struggle ensued. Mullay cried "Murder!"|A life and death struggle ensued. Mullay cried "Murder!" +LJ011-0255|Howard swore he would finish him, but proved the weaker of the two, and Mullay got him down on the floor.|Howard swore he would finish him, but proved the weaker of the two, and Mullay got him down on the floor. +LJ011-0257|Howard was secured, given into custody, and committed to Newgate.|Howard was secured, given into custody, and committed to Newgate. +LJ011-0258|The defense he set up was, that Mullay had used epithets towards him while they were negotiating a business matter,|The defense he set up was, that Mullay had used epithets towards him while they were negotiating a business matter, +LJ011-0259|and that, being an irritable temper, he had struck Mullay, after which a violent scuffle took place.|and that, being an irritable temper, he had struck Mullay, after which a violent scuffle took place. +LJ011-0260|It was, however, proved that Howard was in needy circumstances, and that his proposals to Mr. Mullay could only have originated in a desire to rob him.|It was, however, proved that Howard was in needy circumstances, and that his proposals to Mr. Mullay could only have originated in a desire to rob him. +LJ011-0261|He was found guilty of an assault with intent, and sentenced to transportation for fourteen years.|He was found guilty of an assault with intent, and sentenced to transportation for fourteen years. +LJ011-0262|A more complicated and altogether most extraordinary case of assault, with intent to extort money, occurred a few years later.|A more complicated and altogether most extraordinary case of assault, with intent to extort money, occurred a few years later. +LJ011-0263|It was perpetrated upon a respectable country solicitor,|It was perpetrated upon a respectable country solicitor, +LJ011-0264|Mr. Gee, of Bishop Stortford, who administered the estate of a certain Mr. Canning, deceased.|Mr. Gee, of Bishop Stortford, who administered the estate of a certain Mr. Canning, deceased. +LJ011-0265|This Mr. Canning had left his widow a life interest in £2000 so long as she remained unmarried.|This Mr. Canning had left his widow a life interest in two thousand pounds so long as she remained unmarried. +LJ011-0266|The money went after her to her children.|The money went after her to her children. +LJ011-0267|Mr. Gee had invested £1200 of this, and was seeking how best to place the remaining £800,|Mr. Gee had invested one thousand two hundred pounds of this, and was seeking how best to place the remaining eight hundred pounds, +LJ011-0268|when he was asked to meet a Mr. Heath in London with regard to the sale of certain lands at Bishop Stortford.|when he was asked to meet a Mr. Heath in London with regard to the sale of certain lands at Bishop Stortford. +LJ011-0269|An appointment was made and kept by Mr. Gee,|An appointment was made and kept by Mr. Gee, +LJ011-0270|but on arrival he was met by a young sailor with a letter which begged Mr. Gee to go to Heath's house, as the latter was not well.|but on arrival he was met by a young sailor with a letter which begged Mr. Gee to go to Heath's house, as the latter was not well. +LJ011-0271|Mr. Gee went in the coach sent for him, and alighted at 27, York Street, West, Commercial Road.|Mr. Gee went in the coach sent for him, and alighted at twenty-seven, York Street, West, Commercial Road. +LJ011-0272|The coach immediately drove off; Mr. Gee entered the house, asked for Mr. Heath, was told he would find him in the back kitchen at breakfast.|The coach immediately drove off; Mr. Gee entered the house, asked for Mr. Heath, was told he would find him in the back kitchen at breakfast. +LJ011-0273|He was about to descend the stairs when three persons, one of them the young sailor,|He was about to descend the stairs when three persons, one of them the young sailor, +LJ011-0274|fell upon him, and in spite of his resistance carried him into a sort of den partitioned off at the end of the back kitchen.|fell upon him, and in spite of his resistance carried him into a sort of den partitioned off at the end of the back kitchen. +LJ011-0275|There he was seated on some sort of wooden bench and securely fastened.|There he was seated on some sort of wooden bench and securely fastened. +LJ011-0276|"A chain fixed to staples at his back passed round his chest under his arms, and was padlocked on the left side;"|"A chain fixed to staples at his back passed round his chest under his arms, and was padlocked on the left side;" +LJ011-0277|his feet were bound with cords and made fast to rings in the floor. Thus manacled, one of the party, who pretended to be Mrs. Canning's brother,|his feet were bound with cords and made fast to rings in the floor. Thus manacled, one of the party, who pretended to be Mrs. Canning's brother, +LJ011-0278|addressed him, insisting that he should forthwith sign a cheque for the £800 of the Canning inheritance still uninvested,|addressed him, insisting that he should forthwith sign a cheque for the eight hundred pounds of the Canning inheritance still uninvested, +LJ011-0279|and write an order sufficient to secure the surrender of the other £1200.|and write an order sufficient to secure the surrender of the other one thousand two hundred pounds. +LJ011-0280|Mr. Gee at first stoutly refused.|Mr. Gee at first stoutly refused. +LJ011-0281|Then, as they warned him that he would be kept a prisoner in total darkness in this horrible den until he agreed to their demands,|Then, as they warned him that he would be kept a prisoner in total darkness in this horrible den until he agreed to their demands, +LJ011-0282|he gave in, and signed the documents thus illegally extorted.|he gave in, and signed the documents thus illegally extorted. +LJ011-0283|One was a cheque for £800 on his bankers, the other an order to Mr. Bell of Newport, Essex, requesting the surrender of a deed.|One was a cheque for eight hundred pounds on his bankers, the other an order to Mr. Bell of Newport, Essex, requesting the surrender of a deed. +LJ011-0284|His captors having thus succeeded in their designs, left him, no doubt to realize the money.|His captors having thus succeeded in their designs, left him, no doubt to realize the money. +LJ011-0285|The door of his place of durance stood open, and Mr. Gee began to consider whether he might not escape.|The door of his place of durance stood open, and Mr. Gee began to consider whether he might not escape. +LJ011-0286|For three hours he struggled without success with his bonds,|For three hours he struggled without success with his bonds, +LJ011-0287|but at length managed to wriggle out of the chain which confined his body, and soon loosened the ropes round his feet.|but at length managed to wriggle out of the chain which confined his body, and soon loosened the ropes round his feet. +LJ011-0288|Thus free, he eluded the vigilance of two of the party, who were at dinner in the front kitchen,|Thus free, he eluded the vigilance of two of the party, who were at dinner in the front kitchen, +LJ011-0289|and creeping out into the garden at the back, climbed the wall, and got into the street.|and creeping out into the garden at the back, climbed the wall, and got into the street. +LJ011-0290|His first act was to send a messenger to stop the cheque and the order to Mr. Bell, his next to seek the help of the police.|His first act was to send a messenger to stop the cheque and the order to Mr. Bell, his next to seek the help of the police. +LJ011-0291|Two Bow Street runners were dispatched to the house in York Street, which had evidently been taken on purpose for the outrage.|Two Bow Street runners were dispatched to the house in York Street, which had evidently been taken on purpose for the outrage. +LJ011-0292|There was no furniture in the place, and the den in the kitchen had been recently and specially constructed of boards of immense strength and thickness.|There was no furniture in the place, and the den in the kitchen had been recently and specially constructed of boards of immense strength and thickness. +LJ011-0293|It was a cell five feet by three, within another, the intervening being filled with rammed earth to deaden the sound.|It was a cell five feet by three, within another, the intervening being filled with rammed earth to deaden the sound. +LJ012-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 15: Newgate notorieties, part 3|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section fifteen: Newgate notorieties, part three +LJ012-0002|On the arrival of the police the house was empty.|On the arrival of the police the house was empty. +LJ012-0003|The two men on guard had gone off immediately after Mr. Gee had escaped, but they returned later in the day, and were apprehended.|The two men on guard had gone off immediately after Mr. Gee had escaped, but they returned later in the day, and were apprehended. +LJ012-0004|Inquiries set on foot also elicited the suspicion that the person who had represented Mrs. Canning's brother|Inquiries set on foot also elicited the suspicion that the person who had represented Mrs. Canning's brother +LJ012-0005|was a blind man named Edwards, who had taken this house in York Street, and who was known to be a frequent visitor at Mrs. Canning's.|was a blind man named Edwards, who had taken this house in York Street, and who was known to be a frequent visitor at Mrs. Canning's. +LJ012-0006|A watch was set on him at her house, where he was soon afterwards arrested.|A watch was set on him at her house, where he was soon afterwards arrested. +LJ012-0007|Edwards, whom Mr. Gee easily identified with the others, at once admitted that he was the prime mover of the conspiracy.|Edwards, whom Mr. Gee easily identified with the others, at once admitted that he was the prime mover of the conspiracy. +LJ012-0008|He had sought by all legal means to obtain possession of the £2000, but had failed, and had had recourse to more violent means.|He had sought by all legal means to obtain possession of the two thousand pounds, but had failed, and had had recourse to more violent means. +LJ012-0009|It turned out that he was really married to Mrs. Canning,|It turned out that he was really married to Mrs. Canning, +LJ012-0010|both having been recognized by the clergyman who had performed the ceremony, and the assault had been committed to secure the money|both having been recognized by the clergyman who had performed the ceremony, and the assault had been committed to secure the money +LJ012-0011|which Mrs. Canning had lost by remarriage.|which Mrs. Canning had lost by remarriage. +LJ012-0012|All three men were committed for trial, although Edwards wished to exculpate the others as having only acted under his order.|All three men were committed for trial, although Edwards wished to exculpate the others as having only acted under his order. +LJ012-0013|At the trial the indictment charging them with felony could not be sustained, but they were found guilty of conspiracy and assault.|At the trial the indictment charging them with felony could not be sustained, but they were found guilty of conspiracy and assault. +LJ012-0014|Edwards was sentenced to two years' imprisonment in Newgate,|Edwards was sentenced to two years' imprisonment in Newgate, +LJ012-0015|Weedon and Lecasser to twelve and six months respectively in Coldbath Fields.|Weedon and Lecasser to twelve and six months respectively in Coldbath Fields. +LJ012-0016|At no period could thieves in London or elsewhere have prospered had they been unable to dispose of their ill-gotten goods.|At no period could thieves in London or elsewhere have prospered had they been unable to dispose of their ill-gotten goods. +LJ012-0017|The trade of fence, or receiver, therefore, is very nearly as old as the crimes which it so obviously fostered.|The trade of fence, or receiver, therefore, is very nearly as old as the crimes which it so obviously fostered. +LJ012-0018|One of the most notorious, and for a time most successful practitioners in this illicit trade, passed through Newgate in 1831.|One of the most notorious, and for a time most successful practitioners in this illicit trade, passed through Newgate in eighteen thirty-one. +LJ012-0019|The name of Ikey Solomons was long remembered by thief and thief-taker.|The name of Ikey Solomons was long remembered by thief and thief-taker. +LJ012-0020|He began as an itinerant street vendor at eight, at ten he passed bad money,|He began as an itinerant street vendor at eight, at ten he passed bad money, +LJ012-0021|at fourteen he was a pickpocket and a "duffer," or a seller of sham goods.|at fourteen he was a pickpocket and a "duffer," or a seller of sham goods. +LJ012-0022|He early saw the profits to be made out of purchasing stolen goods, but could not embark in it at first for want of capital.|He early saw the profits to be made out of purchasing stolen goods, but could not embark in it at first for want of capital. +LJ012-0023|He was taken up when still in his teens for stealing a pocketbook, and was sentenced to transportation, but did not get beyond the hulks at Chatham.|He was taken up when still in his teens for stealing a pocketbook, and was sentenced to transportation, but did not get beyond the hulks at Chatham. +LJ012-0024|On his release an uncle, a slop-seller in Chatham, gave him a situation as "barker," or salesman,|On his release an uncle, a slop-seller in Chatham, gave him a situation as "barker," or salesman, +LJ012-0025|at which he realized £150 within a couple of years.|at which he realized one hundred fifty pounds within a couple of years. +LJ012-0026|With this capital he returned to London and set up as a fence.|With this capital he returned to London and set up as a fence. +LJ012-0027|He had such great aptitude for business, and such a thorough knowledge of the real value of goods,|He had such great aptitude for business, and such a thorough knowledge of the real value of goods, +LJ012-0028|that he was soon admitted to be one of the best judges known of all kinds of property, from a glass bottle to a five hundred guinea chronometer.|that he was soon admitted to be one of the best judges known of all kinds of property, from a glass bottle to a five hundred guinea chronometer. +LJ012-0029|But he never paid more than a fixed price for all articles of the same class, whatever their intrinsic value.|But he never paid more than a fixed price for all articles of the same class, whatever their intrinsic value. +LJ012-0030|Thus, a watch was paid for as a watch, whether it was of gold or silver; a piece of linen as such, whether the stuff was coarse or fine.|Thus, a watch was paid for as a watch, whether it was of gold or silver; a piece of linen as such, whether the stuff was coarse or fine. +LJ012-0031|This rule in dealing with stolen goods continues to this day, and has made the fortune of many since Ikey.|This rule in dealing with stolen goods continues to this day, and has made the fortune of many since Ikey. +LJ012-0032|Solomons also established a system of provincial agency, by which stolen goods were passed on from London to the seaports, and so abroad.|Solomons also established a system of provincial agency, by which stolen goods were passed on from London to the seaports, and so abroad. +LJ012-0033|Jewels were re-set,|Jewels were re-set, +LJ012-0034|diamonds refaced; all marks by which other articles might be identified, the selvages of linen, the stamps on shoes,|diamonds refaced; all marks by which other articles might be identified, the selvages of linen, the stamps on shoes, +LJ012-0035|the number and names on watches, were carefully removed or obliterated after the goods passed out of his hands.|the number and names on watches, were carefully removed or obliterated after the goods passed out of his hands. +LJ012-0036|On one occasion the whole of the proceeds of a robbery from a boot shop was traced to Solomons';|On one occasion the whole of the proceeds of a robbery from a boot shop was traced to Solomons'; +LJ012-0037|the owner came with the police, and was morally convinced that it was his property,|the owner came with the police, and was morally convinced that it was his property, +LJ012-0038|but could not positively identify it, and Ikey defied them to remove a single shoe.|but could not positively identify it, and Ikey defied them to remove a single shoe. +LJ012-0039|In the end the injured bootmaker agreed to buy back his stolen stock|In the end the injured bootmaker agreed to buy back his stolen stock +LJ012-0040|at the price Solomons had paid for it, and it cost him about a hundred pounds to re-stock his shop with his own goods.|at the price Solomons had paid for it, and it cost him about a hundred pounds to re-stock his shop with his own goods. +LJ012-0041|As a general rule Ikey Solomons confined his purchases to small articles, mostly of jewelery and plate,|As a general rule Ikey Solomons confined his purchases to small articles, mostly of jewelery and plate, +LJ012-0042|which he kept concealed in a hiding-place with a trap-door just under his bed.|which he kept concealed in a hiding-place with a trap-door just under his bed. +LJ012-0043|He lived in Rosemary Lane, and sometimes he had as much as £20,000 worth of goods secreted on the premises.|He lived in Rosemary Lane, and sometimes he had as much as twenty thousand pounds worth of goods secreted on the premises. +LJ012-0045|he put another lady, with whom he was on intimate terms.|he put another lady, with whom he was on intimate terms. +LJ012-0046|The second house was in Lower Queen Street, Islington, and he used it for some time as a depot for valuables.|The second house was in Lower Queen Street, Islington, and he used it for some time as a depot for valuables. +LJ012-0047|But it was eventually discovered by Mrs. Solomons, a very jealous wife,|But it was eventually discovered by Mrs. Solomons, a very jealous wife, +LJ012-0048|and this, with the danger arising from an extensive robbery of watches in Cheapside, in which Ikey was implicated as a receiver,|and this, with the danger arising from an extensive robbery of watches in Cheapside, in which Ikey was implicated as a receiver, +LJ012-0049|led him to think seriously of trying his fortunes in another land.|led him to think seriously of trying his fortunes in another land. +LJ012-0050|He was about to emigrate to New South Wales, when he was arrested at Islington and committed to Newgate on a charge of receiving stolen goods.|He was about to emigrate to New South Wales, when he was arrested at Islington and committed to Newgate on a charge of receiving stolen goods. +LJ012-0051|While thus incarcerated he managed to escape from custody, but not actually from jail, by an ingenious contrivance which is worth mentioning.|While thus incarcerated he managed to escape from custody, but not actually from jail, by an ingenious contrivance which is worth mentioning. +LJ012-0052|He claimed to be admitted to bail, and was taken from Newgate on a writ of habeas before one of the judges sitting at Westminster.|He claimed to be admitted to bail, and was taken from Newgate on a writ of habeas before one of the judges sitting at Westminster. +LJ012-0053|He was conveyed in a coach driven by a confederate, and under the escort of a couple of turnkeys.|He was conveyed in a coach driven by a confederate, and under the escort of a couple of turnkeys. +LJ012-0054|Solomons, while waiting to appear in court, persuaded the turnkeys to take him to a public-house, where all might "refresh."|Solomons, while waiting to appear in court, persuaded the turnkeys to take him to a public-house, where all might "refresh." +LJ012-0055|While there he was joined by his wife and other friends.|While there he was joined by his wife and other friends. +LJ012-0056|After a short carouse the prisoner went into Westminster, his case was heard, bail refused, and he was ordered back to Newgate.|After a short carouse the prisoner went into Westminster, his case was heard, bail refused, and he was ordered back to Newgate. +LJ012-0057|But he once more persuaded the turnkeys to pause at the public, where more liquor was consumed.|But he once more persuaded the turnkeys to pause at the public, where more liquor was consumed. +LJ012-0058|When the journey was resumed, Mrs. Solomons accompanied her husband in the coach. Half-way to Newgate she was taken with a fit.|When the journey was resumed, Mrs. Solomons accompanied her husband in the coach. Half-way to Newgate she was taken with a fit. +LJ012-0059|One turnkey was stupidly drunk, and Ikey persuaded the other, who was not much better,|One turnkey was stupidly drunk, and Ikey persuaded the other, who was not much better, +LJ012-0060|to let the coach change and pass Petticoat Lane en route to the jail, where the suffering woman might be handed over to her friends.|to let the coach change and pass Petticoat Lane en route to the jail, where the suffering woman might be handed over to her friends. +LJ012-0061|On stopping at a door in this low street, Ikey jumped out, ran into the house, slamming the door behind him.|On stopping at a door in this low street, Ikey jumped out, ran into the house, slamming the door behind him. +LJ012-0062|He passed through and out at the back, and was soon beyond pursuit.|He passed through and out at the back, and was soon beyond pursuit. +LJ012-0063|By-and-by the turnkeys, sobered by their loss, returned to Newgate alone, and pleaded in excuse that they had been drugged.|By-and-by the turnkeys, sobered by their loss, returned to Newgate alone, and pleaded in excuse that they had been drugged. +LJ012-0064|Ikey left no traces, and the police could hear nothing of him. He had in fact gone out of the country, to Copenhagen, whence he passed on to New York.|Ikey left no traces, and the police could hear nothing of him. He had in fact gone out of the country, to Copenhagen, whence he passed on to New York. +LJ012-0065|There he devoted himself to the circulation of forged notes. He was also anxious to do business in watches,|There he devoted himself to the circulation of forged notes. He was also anxious to do business in watches, +LJ012-0066|and begged his wife to send him over a consignment of cheap "righteous" watches, or such as had been honestly obtained, and not "on the cross."|and begged his wife to send him over a consignment of cheap "righteous" watches, or such as had been honestly obtained, and not "on the cross." +LJ012-0067|But Mrs. Solomons could not resist the temptation to dabble in stolen goods, and she was found shipping watches of the wrong category to New York.|But Mrs. Solomons could not resist the temptation to dabble in stolen goods, and she was found shipping watches of the wrong category to New York. +LJ012-0068|For this she received a sentence of fourteen years' transportation, and was sent to Van Diemen's Land.|For this she received a sentence of fourteen years' transportation, and was sent to Van Diemen's Land. +LJ012-0069|Ikey joined her at Hobart Town, where they set up a general shop, and soon began to prosper.|Ikey joined her at Hobart Town, where they set up a general shop, and soon began to prosper. +LJ012-0070|He was, however, recognized, and ere long an order came out from home for his arrest and transfer to England,|He was, however, recognized, and ere long an order came out from home for his arrest and transfer to England, +LJ012-0071|which presently followed, and again found himself an inmate of Newgate, waiting trial as a receiver and a prison-breaker.|which presently followed, and again found himself an inmate of Newgate, waiting trial as a receiver and a prison-breaker. +LJ012-0072|He was indicted on eight charges, two only of which were substantiated, but on each of them he received a sentence of seven years' transportation.|He was indicted on eight charges, two only of which were substantiated, but on each of them he received a sentence of seven years' transportation. +LJ012-0073|At his own request he was reconveyed to Hobart Town, where his son had been carrying on the business.|At his own request he was reconveyed to Hobart Town, where his son had been carrying on the business. +LJ012-0074|Whether Ikey was "assigned" to his own family is not recorded, but no doubt he succeeded to his own property when the term of servitude had expired.|Whether Ikey was "assigned" to his own family is not recorded, but no doubt he succeeded to his own property when the term of servitude had expired. +LJ012-0075|No doubt, on the removal of Ikey Solomons from the scene, his mantle fell upon worthy successors.|No doubt, on the removal of Ikey Solomons from the scene, his mantle fell upon worthy successors. +LJ012-0076|There was an increase rather than an abatement in jewel and bullion robberies in the years immediately following,|There was an increase rather than an abatement in jewel and bullion robberies in the years immediately following, +LJ012-0077|and the thieves seem to have had no difficulty in disposing of their spoil.|and the thieves seem to have had no difficulty in disposing of their spoil. +LJ012-0078|One of the largest robberies of its class was that effected upon the Custom House in the winter of 1834.|One of the largest robberies of its class was that effected upon the Custom House in the winter of eighteen thirty-four. +LJ012-0079|A large amount of specie was nearly always retained here in the department of the Receiver of Fines.|A large amount of specie was nearly always retained here in the department of the Receiver of Fines. +LJ012-0080|This was known to some clerks in the office, who began to consider how they might lay hands on a lot of cash.|This was known to some clerks in the office, who began to consider how they might lay hands on a lot of cash. +LJ012-0081|Being inexperienced, they decided to call in the services of a couple of professional housebreakers,|Being inexperienced, they decided to call in the services of a couple of professional housebreakers, +LJ012-0082|Jordan and Sullivan, who at once set to work in a business-like way to obtain impressions of the keys of the strong room and chest.|Jordan and Sullivan, who at once set to work in a business-like way to obtain impressions of the keys of the strong room and chest. +LJ012-0083|But before committing themselves to an attempt on the latter, it was of importance to ascertain how much it usually contained.|But before committing themselves to an attempt on the latter, it was of importance to ascertain how much it usually contained. +LJ012-0084|For this purpose Jordan waited on the receiver to make a small payment, for which he tendered a fifty-pound note.|For this purpose Jordan waited on the receiver to make a small payment, for which he tendered a fifty-pound note. +LJ012-0085|The chest was opened to give change, and a heavy tray lifted out which plainly held some £4000 in cash.|The chest was opened to give change, and a heavy tray lifted out which plainly held some four thousand pounds in cash. +LJ012-0086|Some difficulty then arose as to gaining admission to the strong room, and it was arranged that a man, May, another Custom House clerk,|Some difficulty then arose as to gaining admission to the strong room, and it was arranged that a man, May, another Custom House clerk, +LJ012-0087|should be introduced into the building, and secreted there during the night to accomplish the robbery.|should be introduced into the building, and secreted there during the night to accomplish the robbery. +LJ012-0088|May was smuggled in through a window on the esplanade behind an opened umbrella. When the place was quite deserted|May was smuggled in through a window on the esplanade behind an opened umbrella. When the place was quite deserted +LJ012-0089|he broke open the chest and stole £4700 in notes, with a quantity of gold and some silver.|he broke open the chest and stole four thousand, seven hundred pounds in notes, with a quantity of gold and some silver. +LJ012-0090|He went out next morning with the booty when the doors were re-opened, and attracted no attention.|He went out next morning with the booty when the doors were re-opened, and attracted no attention. +LJ012-0091|The spoil was fairly divided; part of the notes were disposed of to a traveling "receiver," who passed over to the Continent and there cashed them easily.|The spoil was fairly divided; part of the notes were disposed of to a traveling "receiver," who passed over to the Continent and there cashed them easily. +LJ012-0092|This occurred in November 1834. The Custom House officials were in a state of consternation,|This occurred in November eighteen thirty-four. The Custom House officials were in a state of consternation, +LJ012-0093|and the police were unable at first to get on the track of the thieves.|and the police were unable at first to get on the track of the thieves. +LJ012-0094|While the excitement was still fresh, a new robbery of diamonds was committed at a bonded warehouse in the immediate neighborhood, on Custom House Quay.|While the excitement was still fresh, a new robbery of diamonds was committed at a bonded warehouse in the immediate neighborhood, on Custom House Quay. +LJ012-0095|The jewels had belonged to a Spanish countess recently deceased, who had sent them to England for greater security on the outbreak of the first Carlist war.|The jewels had belonged to a Spanish countess recently deceased, who had sent them to England for greater security on the outbreak of the first Carlist war. +LJ012-0096|At her death the diamonds were divided between her four daughters, but only half had been claimed,|At her death the diamonds were divided between her four daughters, but only half had been claimed, +LJ012-0097|and at the time of the robbery there were still £6000 worth in the warehouse.|and at the time of the robbery there were still six thousand pounds worth in the warehouse. +LJ012-0098|These were deposited in an iron chest of great strength on the second floor.|These were deposited in an iron chest of great strength on the second floor. +LJ012-0099|The thieves it was supposed had secreted themselves in the warehouse during business hours, and waited till night to carry out their plans.|The thieves it was supposed had secreted themselves in the warehouse during business hours, and waited till night to carry out their plans. +LJ012-0100|Some ham sandwiches, several cigar ends, and two empty champagne bottles were found on the premises next day, showing how they had passed their time.|Some ham sandwiches, several cigar ends, and two empty champagne bottles were found on the premises next day, showing how they had passed their time. +LJ012-0101|They had had serious work to get at the diamonds. It was necessary to force one heavy door from its hinges, and to cut through the thick panels of another.|They had had serious work to get at the diamonds. It was necessary to force one heavy door from its hinges, and to cut through the thick panels of another. +LJ012-0102|The lock and fastenings of the chest were forced by means of a "jack," an instrument known to housebreakers,|The lock and fastenings of the chest were forced by means of a "jack," an instrument known to housebreakers, +LJ012-0103|which if introduced into a keyhole, and worked like a bit and brace, will soon destroy the strongest lock.|which if introduced into a keyhole, and worked like a bit and brace, will soon destroy the strongest lock. +LJ012-0104|The thieves were satisfied with the diamonds; they broke open other cases containing gold watches and plate, but abstracted nothing.|The thieves were satisfied with the diamonds; they broke open other cases containing gold watches and plate, but abstracted nothing. +LJ012-0105|The police were of opinion that these robberies were both the work of the same hand.|The police were of opinion that these robberies were both the work of the same hand. +LJ012-0106|But it was not until the autumn that they traced some of the notes stolen from the Custom House to Jordan and Sullivan.|But it was not until the autumn that they traced some of the notes stolen from the Custom House to Jordan and Sullivan. +LJ012-0107|About this time also suspicion fell upon Huey, one of the clerks, who was arrested soon afterwards, and made a clean breast of the whole affair.|About this time also suspicion fell upon Huey, one of the clerks, who was arrested soon afterwards, and made a clean breast of the whole affair. +LJ012-0108|There was a hunt for the two well-known house-breakers, who were eventually heard of at a lodging in Kennington.|There was a hunt for the two well-known house-breakers, who were eventually heard of at a lodging in Kennington. +LJ012-0109|But they at once made tracks, and took up their residence under assumed names in a tavern in Bloomsbury.|But they at once made tracks, and took up their residence under assumed names in a tavern in Bloomsbury. +LJ012-0110|The police lost all trace of them for some days, but at length Sullivan's brother was followed from the house in Kennington to the above-mentioned tavern.|The police lost all trace of them for some days, but at length Sullivan's brother was followed from the house in Kennington to the above-mentioned tavern. +LJ012-0111|Both the thieves were now apprehended, but only a small portion of the lost property was recovered,|Both the thieves were now apprehended, but only a small portion of the lost property was recovered, +LJ012-0112|notwithstanding a minute search through the room they had occupied.|notwithstanding a minute search through the room they had occupied. +LJ012-0113|After their arrest, Jordan's wife and Sullivan's brother came to the inn, and begged to be allowed to visit this room;|After their arrest, Jordan's wife and Sullivan's brother came to the inn, and begged to be allowed to visit this room; +LJ012-0114|but their request, in spite of their earnest entreaties,|but their request, in spite of their earnest entreaties, +LJ012-0115|was refused, at the instigation of the police. A few days later a frequent guest at the tavern arrived, and had this same room allotted to him.|was refused, at the instigation of the police. A few days later a frequent guest at the tavern arrived, and had this same room allotted to him. +LJ012-0116|A fire was lit in it, and the maid in doing so threw a lot of rubbish, as it seemed, which had accumulated under the grate, on top of the burning coals.|A fire was lit in it, and the maid in doing so threw a lot of rubbish, as it seemed, which had accumulated under the grate, on top of the burning coals. +LJ012-0117|By-and-by the occupant of the room noticed something glittering in the center of the fire, which, to inspect more closely, he took out with the tongs.|By-and-by the occupant of the room noticed something glittering in the center of the fire, which, to inspect more closely, he took out with the tongs. +LJ012-0118|It was a large gold brooch set in pearls, but a portion of the mounting had melted with the heat.|It was a large gold brooch set in pearls, but a portion of the mounting had melted with the heat. +LJ012-0119|The fire was raked out, and in the ashes were found seven large and four dozen small brilliants, also seven emeralds,|The fire was raked out, and in the ashes were found seven large and four dozen small brilliants, also seven emeralds, +LJ012-0120|one of them of considerable size.|one of them of considerable size. +LJ012-0121|A part of the "swag" stolen from the bonded warehouse was thus recovered, but it was supposed that a number of the stolen notes had perished in the fire.|A part of the "swag" stolen from the bonded warehouse was thus recovered, but it was supposed that a number of the stolen notes had perished in the fire. +LJ012-0122|The condign punishment meted out to these Custom House robbers had no deterrent effect seemingly.|The condign punishment meted out to these Custom House robbers had no deterrent effect seemingly. +LJ012-0123|Within three months, three new and most mysterious burglaries were committed at the West End, all in houses adjoining each other.|Within three months, three new and most mysterious burglaries were committed at the West End, all in houses adjoining each other. +LJ012-0124|One was occupied by the Portuguese ambassador, who lost a quantity of jewelery from an escritoire, and his neighbors lost plate and cash.|One was occupied by the Portuguese ambassador, who lost a quantity of jewelery from an escritoire, and his neighbors lost plate and cash. +LJ012-0125|Not the slightest clue to these large affairs was ever obtained, but it is probable that they were "put up" jobs, or managed with the complicity of servants.|Not the slightest clue to these large affairs was ever obtained, but it is probable that they were "put up" jobs, or managed with the complicity of servants. +LJ012-0126|The next year twelve thousand sovereigns were cleverly stolen in the Mile End Road.|The next year twelve thousand sovereigns were cleverly stolen in the Mile End Road. +LJ012-0127|The gold-dust robbery of 1839, the first of its kind, was cleverly and carefully planned with the assistance of a dishonest employee.|The gold-dust robbery of eighteen thirty-nine, the first of its kind, was cleverly and carefully planned with the assistance of a dishonest employee. +LJ012-0128|A young man named Caspar, clerk to a steam-ship company,|A young man named Caspar, clerk to a steam-ship company, +LJ012-0129|learnt through the firm's correspondence that a quantity of gold-dust|learnt through the firm's correspondence that a quantity of gold-dust +LJ012-0130|brought in a man-of-war from Brazil had been transhipped at Falmouth for conveyance to London.|brought in a man-of-war from Brazil had been transhipped at Falmouth for conveyance to London. +LJ012-0131|The letter informed him of the marks and sizes of the cases containing the precious metal,|The letter informed him of the marks and sizes of the cases containing the precious metal, +LJ012-0132|and he with his father arranged that a messenger should call for the stuff with forged credentials, and anticipating the rightful owner.|and he with his father arranged that a messenger should call for the stuff with forged credentials, and anticipating the rightful owner. +LJ012-0133|The fraudulent messenger, by the help of young Caspar, established his claim to the boxes, paid the wharfage dues, and carried off the gold-dust.|The fraudulent messenger, by the help of young Caspar, established his claim to the boxes, paid the wharfage dues, and carried off the gold-dust. +LJ012-0134|Presently the proper person arrived from the consignees, but found the gold-dust gone.|Presently the proper person arrived from the consignees, but found the gold-dust gone. +LJ012-0135|The police were at once employed, and after infinite pains they discovered the person, one Moss, who had acted as the messenger.|The police were at once employed, and after infinite pains they discovered the person, one Moss, who had acted as the messenger. +LJ012-0136|Moss was known to be intimate with the elder Caspar,|Moss was known to be intimate with the elder Caspar, +LJ012-0137|father of the clerk to the steam-ship company, and these facts were deemed sufficient to justify the arrest of all three.|father of the clerk to the steam-ship company, and these facts were deemed sufficient to justify the arrest of all three. +LJ012-0138|They also ascertained that a gold-refiner,|They also ascertained that a gold-refiner, +LJ012-0139|Solomons, had sold bar gold to the value of £1200 to certain bullion dealers.|Solomons, had sold bar gold to the value of one thousand two hundred pounds to certain bullion dealers. +LJ012-0140|Solomons was not straightforward in his replies as to where he got the gold, and he was soon placed in the dock with the Caspars and Moss.|Solomons was not straightforward in his replies as to where he got the gold, and he was soon placed in the dock with the Caspars and Moss. +LJ012-0141|Moss presently turned approver,|Moss presently turned approver, +LJ012-0142|and implicated "Money Moses," another Jew, for the whole affair had been planned and executed by members of the Hebrew persuasion.|and implicated "Money Moses," another Jew, for the whole affair had been planned and executed by members of the Hebrew persuasion. +LJ012-0143|"Money Moses" had received the stolen gold-dust from Moss' father-in-law, Davis, or Isaacs, who was never arrested,|"Money Moses" had received the stolen gold-dust from Moss' father-in-law, Davis, or Isaacs, who was never arrested, +LJ012-0144|and passed it on to Solomons by his daughter, a widow named Abrahams.|and passed it on to Solomons by his daughter, a widow named Abrahams. +LJ012-0146|In the course of the trial it came out that almost every one concerned except the Caspars had endeavored to defraud his accomplices.|In the course of the trial it came out that almost every one concerned except the Caspars had endeavored to defraud his accomplices. +LJ012-0147|Moss peached because he declared he had been done out of the proper price of the gold-dust; but it was clear that he had tried to appropriate the whole of the stuff,|Moss peached because he declared he had been done out of the proper price of the gold-dust; but it was clear that he had tried to appropriate the whole of the stuff, +LJ012-0148|instead of handing it or the price of it back to the Caspars.|instead of handing it or the price of it back to the Caspars. +LJ012-0149|"Money Moses" and Mrs. Abrahams imposed upon Moss as to the price paid by Solomons;|"Money Moses" and Mrs. Abrahams imposed upon Moss as to the price paid by Solomons; +LJ012-0150|Mrs. Abrahams imposed upon her father by abstracting a portion of the dust and selling it on her own account;|Mrs. Abrahams imposed upon her father by abstracting a portion of the dust and selling it on her own account; +LJ012-0151|Solomons cheated the whole lot by retaining half the gold in his possession, and only giving an I. O. U. for it,|Solomons cheated the whole lot by retaining half the gold in his possession, and only giving an I. O. U. for it, +LJ012-0152|which he refused to redeem on account of the row about the robbery.|which he refused to redeem on account of the row about the robbery. +LJ012-0153|Moses, it may be added, was a direct descendant of Ikey Solomons.|Moses, it may be added, was a direct descendant of Ikey Solomons. +LJ012-0154|He was ostensibly a publican,|He was ostensibly a publican, +LJ012-0155|and kept the Black Lion in Vinegar Yard, Drury Lane, where secretly he did business as one of the most daring and successful fencers ever known in the metropolis.|and kept the Black Lion in Vinegar Yard, Drury Lane, where secretly he did business as one of the most daring and successful fencers ever known in the metropolis. +LJ012-0156|His arrest and conviction cast dismay over the whole gang of receivers, and for a time seriously checked the nefarious traffic.|His arrest and conviction cast dismay over the whole gang of receivers, and for a time seriously checked the nefarious traffic. +LJ012-0157|It may be added that prison life did not agree with "Money Moses"; a striking change came over his appearance while in Newgate.|It may be added that prison life did not agree with "Money Moses"; a striking change came over his appearance while in Newgate. +LJ012-0158|Before his confinement he had been a sleek round person, addicted obviously to the pleasures of the table.|Before his confinement he had been a sleek round person, addicted obviously to the pleasures of the table. +LJ012-0159|He did not thrive on prison fare,|He did not thrive on prison fare, +LJ012-0160|now more strictly meager, thanks to the inspectors and the more stringent discipline, and before he embarked for Australia to undergo his fourteen years,|now more strictly meager, thanks to the inspectors and the more stringent discipline, and before he embarked for Australia to undergo his fourteen years, +LJ012-0161|he was reported to have fallen away to a shadow.|he was reported to have fallen away to a shadow. +LJ012-0162|Having brought down the records of great frauds, forgeries, and thefts from about 1825 to 1840,|Having brought down the records of great frauds, forgeries, and thefts from about eighteen twenty-five to eighteen forty, +LJ012-0163|I will now retrace my steps and give some account of the more remarkable murders during that period.|I will now retrace my steps and give some account of the more remarkable murders during that period. +LJ012-0164|No murder has created greater sensation and horror throughout England than that of Mr. Weare by Thurtell, Hunt, and Probert.|No murder has created greater sensation and horror throughout England than that of Mr. Weare by Thurtell, Hunt, and Probert. +LJ012-0165|As this was accomplished beyond the limits of the metropolis, and its perpetrators arraigned at Hertford,|As this was accomplished beyond the limits of the metropolis, and its perpetrators arraigned at Hertford, +LJ012-0166|where the principal actor suffered death, the case hardly comes within the limits of my subject.|where the principal actor suffered death, the case hardly comes within the limits of my subject. +LJ012-0167|But Probert, who turned king's evidence, and materially assisted conviction,|But Probert, who turned king's evidence, and materially assisted conviction, +LJ012-0168|was tried at the Old Bailey the following year for horse-stealing, and hanged in front of Newgate.|was tried at the Old Bailey the following year for horse-stealing, and hanged in front of Newgate. +LJ012-0169|The murder was still fresh in the memory of the populace, and Probert was all but lynched on his way to jail.|The murder was still fresh in the memory of the populace, and Probert was all but lynched on his way to jail. +LJ012-0170|According to his statement, when sentenced to death, he had been driven to horse-stealing by the execration which had pursued him after the murder.|According to his statement, when sentenced to death, he had been driven to horse-stealing by the execration which had pursued him after the murder. +LJ012-0171|Every door had been closed against him, every hope of future support blasted.|Every door had been closed against him, every hope of future support blasted. +LJ012-0172|Since the calamitous event," he went on, "that happened at Hertford, I have been a lost man."|Since the calamitous event," he went on, "that happened at Hertford, I have been a lost man." +LJ012-0173|The event which he styles calamitous we may well characterize as one of the most deliberately atrocious murders on record.|The event which he styles calamitous we may well characterize as one of the most deliberately atrocious murders on record. +LJ012-0174|Thurtell was a gambler, and Weare had won a good deal of money from him.|Thurtell was a gambler, and Weare had won a good deal of money from him. +LJ012-0175|Weare was supposed to carry a "private bank" about with him in a pocket in his under waistcoat.|Weare was supposed to carry a "private bank" about with him in a pocket in his under waistcoat. +LJ012-0176|To obtain possession of this, Thurtell with his two associates resolved to kill him.|To obtain possession of this, Thurtell with his two associates resolved to kill him. +LJ012-0177|The victim was invited to visit Probert's cottage in the country near Elstree.|The victim was invited to visit Probert's cottage in the country near Elstree. +LJ012-0178|Thurtell drove him down in a gig, "to be killed as he traveled," in Thurtell's own words.|Thurtell drove him down in a gig, "to be killed as he traveled," in Thurtell's own words. +LJ012-0179|The others followed, and on overtaking Thurtell, found he had done the job alone in a retired part of the road known as Gill's Hill Lane.|The others followed, and on overtaking Thurtell, found he had done the job alone in a retired part of the road known as Gill's Hill Lane. +LJ012-0180|The murderer explained that he had first fired a pistol at Weare's head, but the shot glanced off his cheek.|The murderer explained that he had first fired a pistol at Weare's head, but the shot glanced off his cheek. +LJ012-0181|Then he attacked the other's throat with a penknife, and last of all drove the pistol barrel into his forehead.|Then he attacked the other's throat with a penknife, and last of all drove the pistol barrel into his forehead. +LJ012-0182|After the murder the villains divided the spoil, and went on to Probert's cottage, and supped off pork-chops brought down on purpose.|After the murder the villains divided the spoil, and went on to Probert's cottage, and supped off pork-chops brought down on purpose. +LJ012-0183|During the night they sought to dispose of the body by throwing it into a pond, but two days later had to throw it into another pond.|During the night they sought to dispose of the body by throwing it into a pond, but two days later had to throw it into another pond. +LJ012-0184|Meanwhile the discovery of pistol and knife spattered with human blood and brains|Meanwhile the discovery of pistol and knife spattered with human blood and brains +LJ012-0185|raised the alarm, and suspicion fell upon the three murderers, who were arrested.|raised the alarm, and suspicion fell upon the three murderers, who were arrested. +LJ012-0186|The crime was brought home to Thurtell by the confession of Hunt,|The crime was brought home to Thurtell by the confession of Hunt, +LJ012-0187|one of his accomplices, who took the police to the pond, where the remains of the unfortunate Mr. Weare were discovered, sunk in a sack weighted by stones.|one of his accomplices, who took the police to the pond, where the remains of the unfortunate Mr. Weare were discovered, sunk in a sack weighted by stones. +LJ012-0188|Probert was then admitted as a witness, and the case was fully proved against Thurtell, who was hanged in front of Hertford Jail.|Probert was then admitted as a witness, and the case was fully proved against Thurtell, who was hanged in front of Hertford Jail. +LJ012-0189|Hunt, in consideration of the information he had given, escaped death, and was sentenced to transportation for life.|Hunt, in consideration of the information he had given, escaped death, and was sentenced to transportation for life. +LJ012-0190|Widespread horror and indignation was evoked throughout the kingdom by the discovery of the series|Widespread horror and indignation was evoked throughout the kingdom by the discovery of the series +LJ012-0191|of atrocious murders perpetrated in Edinburgh by the miscreants Burke and Hare,|of atrocious murders perpetrated in Edinburgh by the miscreants Burke and Hare, +LJ012-0192|the first of whom has added to the British language a synonym for illegal suppression.|the first of whom has added to the British language a synonym for illegal suppression. +LJ012-0193|The crimes of these inhuman purveyors to medical science do not fall within the limits of this work.|The crimes of these inhuman purveyors to medical science do not fall within the limits of this work. +LJ012-0194|But Burke and Hare had their imitators further south,|But Burke and Hare had their imitators further south, +LJ012-0195|and of these Bishop and Williams, who were guilty of many peculiar atrocities, ended their murderous careers in front of the debtors' door at Newgate.|and of these Bishop and Williams, who were guilty of many peculiar atrocities, ended their murderous careers in front of the debtors' door at Newgate. +LJ012-0196|Bishop, whose real name was Head, married a half-sister of Williams'. Williams was a professional resurrectionist, or body-snatcher,|Bishop, whose real name was Head, married a half-sister of Williams'. Williams was a professional resurrectionist, or body-snatcher, +LJ012-0197|a trade almost openly countenanced when "subjects" for the anatomy schools were only to be got by rifling graves, or worse.|a trade almost openly countenanced when "subjects" for the anatomy schools were only to be got by rifling graves, or worse. +LJ012-0198|Bishop was a carpenter, but having been suddenly thrown out of work, he joined his brother-in-law in his line of business.|Bishop was a carpenter, but having been suddenly thrown out of work, he joined his brother-in-law in his line of business. +LJ012-0199|After a little|After a little +LJ012-0200|Bishop got weary of the dangers and fatigues of exhumation, and proposed to Williams that instead of disinterring they should murder their subjects.|Bishop got weary of the dangers and fatigues of exhumation, and proposed to Williams that instead of disinterring they should murder their subjects. +LJ012-0201|Bishop confessed that he was moved to this by the example of Burke and Hare.|Bishop confessed that he was moved to this by the example of Burke and Hare. +LJ012-0202|They pursued their terrible trade for five years without scruple and without detection.|They pursued their terrible trade for five years without scruple and without detection. +LJ012-0203|Eventually the law overtook them, but almost by accident.|Eventually the law overtook them, but almost by accident. +LJ012-0204|They presented themselves about noon one day at the dissecting room of King's College Hospital, accompanied by a third man,|They presented themselves about noon one day at the dissecting room of King's College Hospital, accompanied by a third man, +LJ012-0205|an avowed "snatcher" and habitué of the Fortune of War, a public-house in Smithfield frequented openly by men of this awful profession.|an avowed "snatcher" and habitué of the Fortune of War, a public-house in Smithfield frequented openly by men of this awful profession. +LJ012-0206|This man, May, asked the porter at King's College if "he wanted anything?" the euphemism for offering a body.|This man, May, asked the porter at King's College if "he wanted anything?" the euphemism for offering a body. +LJ012-0207|The porter asked what he had got, and the answer was, a male subject.|The porter asked what he had got, and the answer was, a male subject. +LJ012-0208|Reference was made to Mr. Partridge, the demonstrator in anatomy,|Reference was made to Mr. Partridge, the demonstrator in anatomy, +LJ012-0209|and after some haggling they agreed on a price, and in the afternoon the snatchers brought a hamper which contained a body in a sack.|and after some haggling they agreed on a price, and in the afternoon the snatchers brought a hamper which contained a body in a sack. +LJ012-0210|The porter received it, but from its freshness became suspicious of foul play.|The porter received it, but from its freshness became suspicious of foul play. +LJ012-0211|Mr. Partridge was sent for, and he with some of the students soon decided that the corpse had not died a natural death.|Mr. Partridge was sent for, and he with some of the students soon decided that the corpse had not died a natural death. +LJ012-0212|The snatchers were detained, the police sent for, and arrest followed as a matter of course.|The snatchers were detained, the police sent for, and arrest followed as a matter of course. +LJ012-0213|An inquest was held on the body, which was identified as that of an Italian boy,|An inquest was held on the body, which was identified as that of an Italian boy, +LJ012-0214|Carlo Ferrari, who made a living by exhibiting white mice about the streets,|Carlo Ferrari, who made a living by exhibiting white mice about the streets, +LJ012-0215|and the jury returned a verdict of willful murder against persons unknown, expressing a strong opinion that Bishop,|and the jury returned a verdict of willful murder against persons unknown, expressing a strong opinion that Bishop, +LJ012-0216|Williams, and May had been concerned in the transaction.|Williams, and May had been concerned in the transaction. +LJ012-0217|Meanwhile, a search had been made at Nova Scotia Gardens, Bethnal Green, where Bishop and Williams lived.|Meanwhile, a search had been made at Nova Scotia Gardens, Bethnal Green, where Bishop and Williams lived. +LJ012-0218|At first nothing peculiar was found; but at a second search the back-garden ground was dug up,|At first nothing peculiar was found; but at a second search the back-garden ground was dug up, +LJ012-0219|and in one corner, at some depth, a bundle of clothes were unearthed, which, with a hairy cap,|and in one corner, at some depth, a bundle of clothes were unearthed, which, with a hairy cap, +LJ012-0220|were known to be what Ferrari had worn when last seen.|were known to be what Ferrari had worn when last seen. +LJ012-0221|In another portion of the garden more clothing, partly male and partly female, was discovered,|In another portion of the garden more clothing, partly male and partly female, was discovered, +LJ012-0222|plainly pointing to the perpetration of other crimes.|plainly pointing to the perpetration of other crimes. +LJ012-0223|These facts were represented before the police magistrate|These facts were represented before the police magistrate +LJ012-0224|who examined Bishop and his fellows, and further incriminating evidence adduced, to the effect that the prisoners had bartered for a coach to carry "a stiff 'un";|who examined Bishop and his fellows, and further incriminating evidence adduced, to the effect that the prisoners had bartered for a coach to carry "a stiff 'un"; +LJ012-0225|they had also been seen to leave their cottage, carrying out a sack with something heavy inside. On this they were fully committed to Newgate for trial.|they had also been seen to leave their cottage, carrying out a sack with something heavy inside. On this they were fully committed to Newgate for trial. +LJ012-0226|This trial came off in due course at the Central Criminal Court, where the prisoners were charged on two counts,|This trial came off in due course at the Central Criminal Court, where the prisoners were charged on two counts, +LJ012-0227|one that of the murder of the Italian boy, the other that of a boy unknown.|one that of the murder of the Italian boy, the other that of a boy unknown. +LJ012-0228|The evidence from first to last was circumstantial,|The evidence from first to last was circumstantial, +LJ012-0229|but the jury, after a short deliberation, did not hesitate to bring in a verdict of guilty, and all three were condemned to death.|but the jury, after a short deliberation, did not hesitate to bring in a verdict of guilty, and all three were condemned to death. +LJ012-0230|Shortly before the day fixed for execution, Bishop made a full confession, the bulk of which bore the impress of truth,|Shortly before the day fixed for execution, Bishop made a full confession, the bulk of which bore the impress of truth, +LJ012-0231|although it included statements that were improbable and unsubstantiated.|although it included statements that were improbable and unsubstantiated. +LJ012-0232|He asserted that the victim was a Lincolnshire lad, and not an Italian boy, although the latter was fully proved.|He asserted that the victim was a Lincolnshire lad, and not an Italian boy, although the latter was fully proved. +LJ012-0233|According to the confession, death had been inflicted by drowning in a well, whereas the medical evidence all pointed to violence.|According to the confession, death had been inflicted by drowning in a well, whereas the medical evidence all pointed to violence. +LJ012-0234|It was, however, pretty clear that this victim, like preceding ones, had been lured to Nova Scotia Gardens, and there drugged with a large dose of laudanum.|It was, however, pretty clear that this victim, like preceding ones, had been lured to Nova Scotia Gardens, and there drugged with a large dose of laudanum. +LJ012-0235|While they were in a state of insensibility the murder was committed.|While they were in a state of insensibility the murder was committed. +LJ012-0236|Bishop's confession was endorsed by Williams, and the immediate result was the respite of May.|Bishop's confession was endorsed by Williams, and the immediate result was the respite of May. +LJ012-0237|A very painful scene occurred in Newgate when the news of his escape from death was imparted to May.|A very painful scene occurred in Newgate when the news of his escape from death was imparted to May. +LJ012-0238|He fainted, and the warrant of mercy nearly proved his death-blow. The other two looked on at his agitation with an indifference amounting to apathy.|He fainted, and the warrant of mercy nearly proved his death-blow. The other two looked on at his agitation with an indifference amounting to apathy. +LJ012-0239|The execution took place a week or two later, in the presence of such a crowd as had not been seen near Newgate for years.|The execution took place a week or two later, in the presence of such a crowd as had not been seen near Newgate for years. +LJ012-0240|I will close this chapter with a brief account of another murder,|I will close this chapter with a brief account of another murder, +LJ012-0241|the memory of which is still fresh in the minds of Londoners, although half a century has passed since it was committed.|the memory of which is still fresh in the minds of Londoners, although half a century has passed since it was committed. +LJ012-0242|The horror with which Greenacre's crime struck the town was unparalleled since the time when Catherine Hayes slew her husband.|The horror with which Greenacre's crime struck the town was unparalleled since the time when Catherine Hayes slew her husband. +LJ012-0243|There were many features of resemblance in these crimes.|There were many features of resemblance in these crimes. +LJ012-0244|The decapitation and dismemberment, the bestowal of the remains in various parts of the town, the preservation of the head in spirits of wine,|The decapitation and dismemberment, the bestowal of the remains in various parts of the town, the preservation of the head in spirits of wine, +LJ012-0245|in the hope that the features might some day be recognized, were alike in both.|in the hope that the features might some day be recognized, were alike in both. +LJ012-0246|The murder in both cases was long a profound mystery. In this which I am now describing,|The murder in both cases was long a profound mystery. In this which I am now describing, +LJ012-0247|a bricklayer found a human trunk near some new buildings in the Edgeware Road, one morning in the last week of 1836.|a bricklayer found a human trunk near some new buildings in the Edgeware Road, one morning in the last week of eighteen thirty-six. +LJ012-0248|The inquest on these remains, which medical examination showed to be those of a female,|The inquest on these remains, which medical examination showed to be those of a female, +LJ012-0249|returned a verdict of willful murder against some person unknown.|returned a verdict of willful murder against some person unknown. +LJ012-0250|On the 7th July, 1837,|On the seventh July, eighteen thirty-seven, +LJ012-0251|the lockman of "Ben Jonson lock," in Stepney Fields, found a human head jammed into the lock gates.|the lockman of "Ben Jonson lock," in Stepney Fields, found a human head jammed into the lock gates. +LJ012-0252|Closer investigation proved that it belonged to the trunk already discovered on the 2nd February.|Closer investigation proved that it belonged to the trunk already discovered on the second February. +LJ012-0253|A further discovery was made in an osier bed near Cold Harbor Lane, Camberwell,|A further discovery was made in an osier bed near Cold Harbor Lane, Camberwell, +LJ012-0254|where a workman found a bundle containing two human legs, in a drain.|where a workman found a bundle containing two human legs, in a drain. +LJ012-0255|These were the missing members of the same mutilated trunk,|These were the missing members of the same mutilated trunk, +LJ012-0256|and there was now evidence sufficient to establish conclusively that the woman thus collected piecemeal had been barbarously done to death.|and there was now evidence sufficient to establish conclusively that the woman thus collected piecemeal had been barbarously done to death. +LJ012-0257|But the affair still remained a profound mystery. No light was thrown upon it till, towards the end of March,|But the affair still remained a profound mystery. No light was thrown upon it till, towards the end of March, +LJ012-0258|a Mr. Gay of Goodge Street came to view the head, and immediately recognized it as that of a widowed sister, Hannah Brown,|a Mr. Gay of Goodge Street came to view the head, and immediately recognized it as that of a widowed sister, Hannah Brown, +LJ012-0259|who had been missing since the previous Christmas Day.|who had been missing since the previous Christmas Day. +LJ012-0260|The murdered individual was thus identified. The next step was to ascertain where and with whom she had last been seen.|The murdered individual was thus identified. The next step was to ascertain where and with whom she had last been seen. +LJ012-0261|This brought suspicion on to a certain James Greenacre,|This brought suspicion on to a certain James Greenacre, +LJ012-0262|whom she was to have married, and in whose company she had left her own lodgings to visit his in Camberwell.|whom she was to have married, and in whose company she had left her own lodgings to visit his in Camberwell. +LJ012-0263|The police wished to refer to Greenacre, but as he was not forthcoming,|The police wished to refer to Greenacre, but as he was not forthcoming, +LJ012-0264|a warrant was issued for his apprehension, which was effected at Kennington on the 24th March.|a warrant was issued for his apprehension, which was effected at Kennington on the twenty-fourth March. +LJ012-0265|A woman named Gale, who lived with him, was arrested at the same time. The prisoners were examined at the Marylebone police court.|A woman named Gale, who lived with him, was arrested at the same time. The prisoners were examined at the Marylebone police court. +LJ012-0266|Greenacre, a stout, middle-aged man, wrapped in a brown greatcoat, assumed an air of insolent bravado;|Greenacre, a stout, middle-aged man, wrapped in a brown greatcoat, assumed an air of insolent bravado; +LJ012-0267|but his despair must have been great, as was evident from his attempt to strangle himself in the station-house.|but his despair must have been great, as was evident from his attempt to strangle himself in the station-house. +LJ012-0268|Suspicion grew almost to certainty as the evidence was unfolded.|Suspicion grew almost to certainty as the evidence was unfolded. +LJ012-0269|Mrs. Brown was a washer-woman, supposed to be worth some money; hence Greenacre's offer of marriage.|Mrs. Brown was a washer-woman, supposed to be worth some money; hence Greenacre's offer of marriage. +LJ012-0270|She had realized all her effects, and brought them with her furniture to Greenacre's lodgings. The two when married were to emigrate to Hudson's Bay.|She had realized all her effects, and brought them with her furniture to Greenacre's lodgings. The two when married were to emigrate to Hudson's Bay. +LJ012-0271|Whether it was greed or a quarrel that drove Greenacre to the desperate deed remains obscure.|Whether it was greed or a quarrel that drove Greenacre to the desperate deed remains obscure. +LJ012-0272|They were apparently good friends when last seen together at a neighbor's, where they seemed "perfectly happy and sociable, and eager for the wedding day."|They were apparently good friends when last seen together at a neighbor's, where they seemed "perfectly happy and sociable, and eager for the wedding day." +LJ012-0273|But Greenacre in his confession pretended that he and his intended had quarreled over her property or the want of it,|But Greenacre in his confession pretended that he and his intended had quarreled over her property or the want of it, +LJ012-0274|and that in a moment of anger he knocked her down.|and that in a moment of anger he knocked her down. +LJ012-0275|He thought he had killed her, and in his terror began at once to consider how he might dispose of the body and escape arrest.|He thought he had killed her, and in his terror began at once to consider how he might dispose of the body and escape arrest. +LJ012-0276|While she was senseless, but really still alive, he cut off her head, and dismembered the body in the manner already described.|While she was senseless, but really still alive, he cut off her head, and dismembered the body in the manner already described. +LJ012-0277|It is scarcely probable that he would have gone to this extremity if he had had no previous evil intention,|It is scarcely probable that he would have gone to this extremity if he had had no previous evil intention, +LJ012-0278|and the most probable inference is that he inveigled Mrs. Brown to his lodgings with the set purpose of taking her life.|and the most probable inference is that he inveigled Mrs. Brown to his lodgings with the set purpose of taking her life. +LJ012-0279|His measures for the disposal of the corpus delicti remind us of those taken by Mrs. Hayes and her associates,|His measures for the disposal of the corpus delicti remind us of those taken by Mrs. Hayes and her associates, +LJ012-0280|or of Gardelle's frantic efforts to conceal his crime.|or of Gardelle's frantic efforts to conceal his crime. +LJ012-0281|The most ghastly part of the story is that which deals with his getting rid of the head.|The most ghastly part of the story is that which deals with his getting rid of the head. +LJ012-0282|This, wrapped up in a silk handkerchief,|This, wrapped up in a silk handkerchief, +LJ012-0283|he carried under his coat-flaps through the streets, and afterwards on his cap in a crowded city omnibus.|he carried under his coat-flaps through the streets, and afterwards on his cap in a crowded city omnibus. +LJ012-0284|It was not until he left the bus, and walked up by the Regent's Canal, that he conceived the idea of throwing the head into the water.|It was not until he left the bus, and walked up by the Regent's Canal, that he conceived the idea of throwing the head into the water. +LJ012-0285|Another day elapsed before he got rid of the rest of the body,|Another day elapsed before he got rid of the rest of the body, +LJ012-0286|all of which, according to his own confession, made no doubt with the idea of exonerating Mrs. Gale, he accomplished without her assistance.|all of which, according to his own confession, made no doubt with the idea of exonerating Mrs. Gale, he accomplished without her assistance. +LJ012-0287|On the other hand, it was adduced in evidence that Mrs. Gale had been at his lodgings the very day after the murder,|On the other hand, it was adduced in evidence that Mrs. Gale had been at his lodgings the very day after the murder, +LJ012-0288|and was seen to be busily engaged in washing down the house with bucket and mop.|and was seen to be busily engaged in washing down the house with bucket and mop. +LJ012-0289|Greenacre, when tried at the Old Bailey, admitted that he had been guilty of manslaughter.|Greenacre, when tried at the Old Bailey, admitted that he had been guilty of manslaughter. +LJ012-0290|While conversing with Mrs. Brown, he declared the unfortunate woman was rocking herself to and fro in a chair;|While conversing with Mrs. Brown, he declared the unfortunate woman was rocking herself to and fro in a chair; +LJ012-0291|as she leant back he put his foot against the chair, and so tilted it over.|as she leant back he put his foot against the chair, and so tilted it over. +LJ012-0292|Mrs. Brown fell with it, and Greenacre, to his horror, found that she was dead.|Mrs. Brown fell with it, and Greenacre, to his horror, found that she was dead. +LJ012-0293|But the medical evidence was clear that the decapitation had been effected during life, and the jury, after a short deliberation,|But the medical evidence was clear that the decapitation had been effected during life, and the jury, after a short deliberation, +LJ012-0294|without hesitation brought in a verdict of willful murder.|without hesitation brought in a verdict of willful murder. +LJ012-0295|The woman Gale was also found guilty, but sentence of death was only passed on Greenacre.|The woman Gale was also found guilty, but sentence of death was only passed on Greenacre. +LJ012-0296|The execution was, as usual, attended by an immense concourse, and Greenacre died amidst the loudest execrations.|The execution was, as usual, attended by an immense concourse, and Greenacre died amidst the loudest execrations. +LJ013-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 16: Newgate notorieties continued, part 1.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section sixteen: Newgate notorieties continued, part one. +LJ013-0002|As the century advanced crimes of fraud increased.|As the century advanced crimes of fraud increased. +LJ013-0003|They not only became more numerous, but they were on a wider scale.|They not only became more numerous, but they were on a wider scale. +LJ013-0004|The most extensive and systematic robberies were planned and carried out so as long to escape detection.|The most extensive and systematic robberies were planned and carried out so as long to escape detection. +LJ013-0005|One of the earliest of the big operators in fraudulent finance was Edward Beaumont Smith,|One of the earliest of the big operators in fraudulent finance was Edward Beaumont Smith, +LJ013-0006|who was convicted in 1841 of uttering false exchequer bills to an almost fabulous amount.|who was convicted in eighteen forty-one of uttering false exchequer bills to an almost fabulous amount. +LJ013-0007|A not entirely novel kind of fraud, but one carried out on a larger scale than heretofore,|A not entirely novel kind of fraud, but one carried out on a larger scale than heretofore, +LJ013-0008|came to light in this same year, 1841.|came to light in this same year, eighteen forty-one. +LJ013-0009|This was the willful shipwreck and casting away of a vessel which, with her supposed cargo, had been heavily insured.|This was the willful shipwreck and casting away of a vessel which, with her supposed cargo, had been heavily insured. +LJ013-0010|The 'Dryad' was a brig owned principally by two persons named Wallace, one a seaman, the other merchant.|The 'Dryad' was a brig owned principally by two persons named Wallace, one a seaman, the other merchant. +LJ013-0011|She was freighted by the firm of Zulueta and Co. for a voyage to Santa Cruz.|She was freighted by the firm of Zulueta and Co. for a voyage to Santa Cruz. +LJ013-0012|Her owners insured her for a full sum of £2000, after which the Wallaces insured her privily|Her owners insured her for a full sum of two thousand pounds, after which the Wallaces insured her privily +LJ013-0013|with other underwriters for a second sum of £2000.|with other underwriters for a second sum of two thousand pounds. +LJ013-0014|After this, on the faith of forced bills of lading, the captain, Loose by name,|After this, on the faith of forced bills of lading, the captain, Loose by name, +LJ013-0015|being a party to the intended fraud, they obtained further insurances on goods never shipped.|being a party to the intended fraud, they obtained further insurances on goods never shipped. +LJ013-0016|It was fully proved in evidence that when the Dryad sailed she carried nothing but the cargo belonging to Zulueta and Co.|It was fully proved in evidence that when the Dryad sailed she carried nothing but the cargo belonging to Zulueta and Co. +LJ013-0017|Yet the Wallaces pretended to have put on board quantities of flannels, cloths, cotton prints,|Yet the Wallaces pretended to have put on board quantities of flannels, cloths, cotton prints, +LJ013-0018|beef, pork, butter, and earthenwares, on all of which they effected insurances.|beef, pork, butter, and earthenwares, on all of which they effected insurances. +LJ013-0019|Loose had his instructions to cast away the ship on the first possible opportunity, and from the time of his leaving Liverpool|Loose had his instructions to cast away the ship on the first possible opportunity, and from the time of his leaving Liverpool +LJ013-0021|The larboard pump was suffered to remain choked up, and the long-boat was fitted with tackles and held ready for use at a moment's notice.|The larboard pump was suffered to remain choked up, and the long-boat was fitted with tackles and held ready for use at a moment's notice. +LJ013-0022|The ship, however, met with exasperatingly fine weather, and it was not until the captain reached the West India Islands|The ship, however, met with exasperatingly fine weather, and it was not until the captain reached the West India Islands +LJ013-0023|that he got a chance of accomplishing his crime.|that he got a chance of accomplishing his crime. +LJ013-0024|At a place called the Silver Keys he ran the ship on the reef.|At a place called the Silver Keys he ran the ship on the reef. +LJ013-0025|But another ship, concluding that he was acting in ignorance, rendered him assistance.|But another ship, concluding that he was acting in ignorance, rendered him assistance. +LJ013-0026|The 'Dryad' was got off, repaired, and her voyage renewed to Santa Cruz.|The 'Dryad' was got off, repaired, and her voyage renewed to Santa Cruz. +LJ013-0027|He crept along the coast close in shore, looking for a quiet spot to cast away the ship,|He crept along the coast close in shore, looking for a quiet spot to cast away the ship, +LJ013-0028|and at last, when within fifteen miles of port, with wind and weather perfectly fair, he ran her on to the rocks.|and at last, when within fifteen miles of port, with wind and weather perfectly fair, he ran her on to the rocks. +LJ013-0029|Even then she might have been saved, but the captain would not suffer the crew to act. Nearly the whole of the cargo was lost as well as the ship.|Even then she might have been saved, but the captain would not suffer the crew to act. Nearly the whole of the cargo was lost as well as the ship. +LJ013-0030|The captain and crew, however, got safely to Jamaica, and so to England; the captain dying on the voyage home.|The captain and crew, however, got safely to Jamaica, and so to England; the captain dying on the voyage home. +LJ013-0031|The crime soon became public.|The crime soon became public. +LJ013-0032|Mate, carpenter, and crew were eager to disavow complicity, and voluntarily gave information.|Mate, carpenter, and crew were eager to disavow complicity, and voluntarily gave information. +LJ013-0033|The Wallaces were arrested, committed to Newgate, and tried at the Old Bailey.|The Wallaces were arrested, committed to Newgate, and tried at the Old Bailey. +LJ013-0034|The case was clearly proved against them, and both were sentenced to transportation for life.|The case was clearly proved against them, and both were sentenced to transportation for life. +LJ013-0035|While lying in Newgate, awaiting removal to the convict ship, both prisoners made full confessions.|While lying in Newgate, awaiting removal to the convict ship, both prisoners made full confessions. +LJ013-0036|According to their own statements the loss of the 'Dryad' was only one of six intentional shipwrecks with which they had been concerned.|According to their own statements the loss of the 'Dryad' was only one of six intentional shipwrecks with which they had been concerned. +LJ013-0037|The crime of fraudulent insurance they declared was very common, and the underwriters must have lost great sums in this way.|The crime of fraudulent insurance they declared was very common, and the underwriters must have lost great sums in this way. +LJ013-0038|The merchant Wallace said he had been led into the crime by the advice and example of a city friend who had gone largely into this nefarious business;|The merchant Wallace said he had been led into the crime by the advice and example of a city friend who had gone largely into this nefarious business; +LJ013-0039|this Wallace added that his friend had made several voyages with the distinct intention of superintending the predetermined shipwrecks.|this Wallace added that his friend had made several voyages with the distinct intention of superintending the predetermined shipwrecks. +LJ013-0040|The other Wallace, the sailor, also traced his lapse into crime to evil counsel.|The other Wallace, the sailor, also traced his lapse into crime to evil counsel. +LJ013-0041|He was an honest sea-captain, he said, trading from Liverpool, where once he had the misfortune to be introduced to a man of wealth,|He was an honest sea-captain, he said, trading from Liverpool, where once he had the misfortune to be introduced to a man of wealth, +LJ013-0042|the foundations of which had been laid by buying old ships on purpose to cast them away.|the foundations of which had been laid by buying old ships on purpose to cast them away. +LJ013-0043|This person made much of Wallace, encouraged his attentions to his daughter,|This person made much of Wallace, encouraged his attentions to his daughter, +LJ013-0044|and tempted him to take to fraudulent insurance as a certain method of achieving fortune.|and tempted him to take to fraudulent insurance as a certain method of achieving fortune. +LJ013-0045|Wallace's relations warned him against his Liverpool friend,|Wallace's relations warned him against his Liverpool friend, +LJ013-0046|but he would not take their advice, and developing his transactions, ended as we have seen.|but he would not take their advice, and developing his transactions, ended as we have seen. +LJ013-0047|A clergyman nearly a century later followed in the steps of Dr. Dodd, but did not under more humane laws lose his life.|A clergyman nearly a century later followed in the steps of Dr. Dodd, but did not under more humane laws lose his life. +LJ013-0048|The Rev. W. Bailey, LL.D., was convicted at the Central Criminal Court, in February 1843, of forgery.|The Rev. W. Bailey, LL.D., was convicted at the Central Criminal Court, in February eighteen forty-three, of forgery. +LJ013-0049|A notorious miser, Robert Smith, had recently died in Seven Dials, where he had amassed a considerable fortune.|A notorious miser, Robert Smith, had recently died in Seven Dials, where he had amassed a considerable fortune. +LJ013-0050|But among the charges on the estate he left|But among the charges on the estate he left +LJ013-0051|was a promissory note for £2875, produced by Dr. Bailey, and purporting to be signed by Smith.|was a promissory note for two thousand eight hundred seventy-five pounds, produced by Dr. Bailey, and purporting to be signed by Smith. +LJ013-0052|The executors to the estate disputed the validity of this document.|The executors to the estate disputed the validity of this document. +LJ013-0053|Miss Bailey, the doctor's sister, in whose favor the note was said to have been given,|Miss Bailey, the doctor's sister, in whose favor the note was said to have been given, +LJ013-0054|then brought an action against the administrators, and at the trial Dr. Bailey swore that the note had been given him by Smith.|then brought an action against the administrators, and at the trial Dr. Bailey swore that the note had been given him by Smith. +LJ013-0055|The jury did not believe him, and the verdict was for the defendants.|The jury did not believe him, and the verdict was for the defendants. +LJ013-0056|Subsequently Bailey was arrested on a charge of forgery, and after a long trial found guilty. His sentence was transportation for life.|Subsequently Bailey was arrested on a charge of forgery, and after a long trial found guilty. His sentence was transportation for life. +LJ013-0057|A gigantic conspiracy to defraud was discovered in the following year, when a solicitor named William Henry Barber,|A gigantic conspiracy to defraud was discovered in the following year, when a solicitor named William Henry Barber, +LJ013-0058|Joshua Fletcher a surgeon, and three others were charged with forging wills for the purpose of obtaining unclaimed stock in the funds.|Joshua Fletcher a surgeon, and three others were charged with forging wills for the purpose of obtaining unclaimed stock in the funds. +LJ013-0059|There were two separate affairs.|There were two separate affairs. +LJ013-0060|In the first a maiden lady, Miss Slack,|In the first a maiden lady, Miss Slack, +LJ013-0061|who was the possessor of two separate sums in consols, neglected through strange carelessness on her own part and that of her friends|who was the possessor of two separate sums in consols, neglected through strange carelessness on her own part and that of her friends +LJ013-0062|to draw the dividends on more than one sum.|to draw the dividends on more than one sum. +LJ013-0063|The other, remaining unclaimed for ten years, was transferred at the end of that time to the commissioners for the reduction of the National Debt.|The other, remaining unclaimed for ten years, was transferred at the end of that time to the commissioners for the reduction of the National Debt. +LJ013-0064|Barber, it was said, became aware of this,|Barber, it was said, became aware of this, +LJ013-0065|and that he gained access to Miss Slack on pretense of conveying to her some funded property left her by an aunt.|and that he gained access to Miss Slack on pretense of conveying to her some funded property left her by an aunt. +LJ013-0066|By this means her signature was obtained; a forged will was prepared bequeathing the unclaimed stock to Miss Slack;|By this means her signature was obtained; a forged will was prepared bequeathing the unclaimed stock to Miss Slack; +LJ013-0067|a note purporting to be from Miss Slack was addressed to the governor of the Bank of England, begging that the said stock might be handed over to her,|a note purporting to be from Miss Slack was addressed to the governor of the Bank of England, begging that the said stock might be handed over to her, +LJ013-0068|and a person calling herself Miss Slack duly attended at the bank, where the money was handed over to her in proper form.|and a person calling herself Miss Slack duly attended at the bank, where the money was handed over to her in proper form. +LJ013-0069|A second will, also forged, was propounded at Doctors Commons as that of a Mrs. Hunt of Bristol.|A second will, also forged, was propounded at Doctors Commons as that of a Mrs. Hunt of Bristol. +LJ013-0070|Mrs. Hunt had left money in the funds which remained unclaimed, and had been transferred, as in Miss Slack's case.|Mrs. Hunt had left money in the funds which remained unclaimed, and had been transferred, as in Miss Slack's case. +LJ013-0071|Here again the money, with ten years' interest, was handed over to Barber and another calling himself Thomas Hunt, an executor of the will.|Here again the money, with ten years' interest, was handed over to Barber and another calling himself Thomas Hunt, an executor of the will. +LJ013-0072|It was shown that the will must be a forgery,|It was shown that the will must be a forgery, +LJ013-0073|as its signature was dated 1829, whereas Mrs. Hunt actually died in 1806.|as its signature was dated eighteen twenty-nine, whereas Mrs. Hunt actually died in eighteen oh six. +LJ013-0074|A third similar fraud to the amount of £2000 was also brought to light.|A third similar fraud to the amount of two thousand pounds was also brought to light. +LJ013-0075|Fletcher was the moving spirit of the whole business. It was he who had introduced Barber to Miss Slack,|Fletcher was the moving spirit of the whole business. It was he who had introduced Barber to Miss Slack, +LJ013-0076|and held all the threads of these intricate and nefarious transactions.|and held all the threads of these intricate and nefarious transactions. +LJ013-0078|Barber was subsequently pardoned, but was not replaced on the rolls as an attorney till 1855,|Barber was subsequently pardoned, but was not replaced on the rolls as an attorney till eighteen fifty-five, +LJ013-0079|when Lord Campbell delivered judgment on Barber's petition, to the effect that|when Lord Campbell delivered judgment on Barber's petition, to the effect that +LJ013-0080|the evidence to establish his (Barber's) connivance in the frauds was too doubtful for us to continue his exclusion any longer.|the evidence to establish his (Barber's) connivance in the frauds was too doubtful for us to continue his exclusion any longer. +LJ013-0081|Banks and bankers continued to be victimized.|Banks and bankers continued to be victimized. +LJ013-0082|In 1844|In eighteen forty-four +LJ013-0083|the Bank of England was defrauded of a sum of £8000 by one of its clerks, Burgess, in conjunction with an accomplice named Elder.|the Bank of England was defrauded of a sum of eight thousand pounds by one of its clerks, Burgess, in conjunction with an accomplice named Elder. +LJ013-0084|Burgess fraudulently transferred consols to the above amount, standing in the name of Mr. Oxenford, to another party.|Burgess fraudulently transferred consols to the above amount, standing in the name of Mr. Oxenford, to another party. +LJ013-0085|A person, Elder of course, who personated Oxenford, attended at the bank to complete the transfer and sell the stock.|A person, Elder of course, who personated Oxenford, attended at the bank to complete the transfer and sell the stock. +LJ013-0086|Burgess, who was purposely on leave from the bank, effected the sale, which was paid for with a cheque for nearly the whole amount on Lubbock's Bank.|Burgess, who was purposely on leave from the bank, effected the sale, which was paid for with a cheque for nearly the whole amount on Lubbock's Bank. +LJ013-0087|Burgess and Elder proceeded in company to cash this, but as they wanted all gold,|Burgess and Elder proceeded in company to cash this, but as they wanted all gold, +LJ013-0088|the cashier gave them eight Bank of England notes for £1000 each, saying that they could get so much specie nowhere else.|the cashier gave them eight Bank of England notes for one thousand pounds each, saying that they could get so much specie nowhere else. +LJ013-0089|Thither Elder went alone, provided with a number of canvas and one large carpet-bag.|Thither Elder went alone, provided with a number of canvas and one large carpet-bag. +LJ013-0090|But when the latter was filled with gold it was too heavy to lift,|But when the latter was filled with gold it was too heavy to lift, +LJ013-0091|and Elder had to be assisted by two bank porters, who carried it for him to a carriage waiting near the Mansion House.|and Elder had to be assisted by two bank porters, who carried it for him to a carriage waiting near the Mansion House. +LJ013-0092|The thieves, for Elder was soon joined by Burgess, drove together to Ben Caunt's, the pugilist's, public-house in St. Martin's Lane,|The thieves, for Elder was soon joined by Burgess, drove together to Ben Caunt's, the pugilist's, public-house in St. Martin's Lane, +LJ013-0093|where the cash was transferred from the carpet-bag to a portmanteau.|where the cash was transferred from the carpet-bag to a portmanteau. +LJ013-0094|The same evening both started for Liverpool, and embarking on board the mail steamer 'Britannia,' escaped to the United States.|The same evening both started for Liverpool, and embarking on board the mail steamer 'Britannia,' escaped to the United States. +LJ013-0095|Burgess' continued absence was soon noticed at the bank.|Burgess' continued absence was soon noticed at the bank. +LJ013-0096|Suspicions were aroused when it was found that he had been employed in selling stock for Mr. Oxenford, which developed into certainty|Suspicions were aroused when it was found that he had been employed in selling stock for Mr. Oxenford, which developed into certainty +LJ013-0097|as soon as that gentleman was referred to.|as soon as that gentleman was referred to. +LJ013-0098|Mr. Oxenford having denied that he had made any transfer of stock, the matter was at once put into the hands of the police.|Mr. Oxenford having denied that he had made any transfer of stock, the matter was at once put into the hands of the police. +LJ013-0099|A smart detective, Forrester, after a little inquiry,|A smart detective, Forrester, after a little inquiry, +LJ013-0100|established the fact that the man who had personated Mr. Oxenford was a horse-dealer named Joseph Elder, an intimate acquaintance of Burgess'.|established the fact that the man who had personated Mr. Oxenford was a horse-dealer named Joseph Elder, an intimate acquaintance of Burgess'. +LJ013-0101|Forrester next traced the fugitives to Liverpool,|Forrester next traced the fugitives to Liverpool, +LJ013-0102|and thence to Halifax, whither he followed them, accompanied by a confidential clerk from the bank.|and thence to Halifax, whither he followed them, accompanied by a confidential clerk from the bank. +LJ013-0103|At Halifax Forrester learnt that the men he wanted had gone on to Boston, thence to Buffalo and Canada, and back to Boston.|At Halifax Forrester learnt that the men he wanted had gone on to Boston, thence to Buffalo and Canada, and back to Boston. +LJ013-0104|He found them at length residing at the latter place, one as a landed proprietor, the other as a publican.|He found them at length residing at the latter place, one as a landed proprietor, the other as a publican. +LJ013-0105|Elder, the former, was soon apprehended at his house, but he evaded the law by hanging himself with his pocket-handkerchief.|Elder, the former, was soon apprehended at his house, but he evaded the law by hanging himself with his pocket-handkerchief. +LJ013-0106|The inn belonging to Burgess was surrounded|The inn belonging to Burgess was surrounded +LJ013-0107|but he escaped through a back door on to the river, and rowed off in a boat to a hiding-place in the woods.|but he escaped through a back door on to the river, and rowed off in a boat to a hiding-place in the woods. +LJ013-0108|Next day a person betrayed him for the reward, and he was soon captured.|Next day a person betrayed him for the reward, and he was soon captured. +LJ013-0109|The proceeds of the robbery were lodged in a Boston bank,|The proceeds of the robbery were lodged in a Boston bank, +LJ013-0110|but four hundred sovereigns were found on Elder, while two hundred more were found in Burgess' effects.|but four hundred sovereigns were found on Elder, while two hundred more were found in Burgess' effects. +LJ013-0111|Burgess was eventually brought back to England, tried at the Central Criminal Court, and sentenced to transportation for life.|Burgess was eventually brought back to England, tried at the Central Criminal Court, and sentenced to transportation for life. +LJ013-0112|Within a month or two the bank of Messrs. Rogers and Co., Clement's Lane, was broken into.|Within a month or two the bank of Messrs. Rogers and Co., Clement's Lane, was broken into. +LJ013-0113|Robberies as daring in conception as they were boldly executed were common enough.|Robberies as daring in conception as they were boldly executed were common enough. +LJ013-0114|One night a quantity of plate was stolen from Windsor Castle; another time Buckingham Palace was robbed.|One night a quantity of plate was stolen from Windsor Castle; another time Buckingham Palace was robbed. +LJ013-0115|Of this class was the ingenious yet peculiarly simple robbery effected at the house of Lord Fitzgerald in Belgrave Square.|Of this class was the ingenious yet peculiarly simple robbery effected at the house of Lord Fitzgerald in Belgrave Square. +LJ013-0116|The butler, on the occasion of a death in the family, when the house was in some confusion, arranged with a burglar to come in,|The butler, on the occasion of a death in the family, when the house was in some confusion, arranged with a burglar to come in, +LJ013-0117|and with another carry off the plate-chest in broad daylight, and as a matter of business. No one interfered or asked any questions.|and with another carry off the plate-chest in broad daylight, and as a matter of business. No one interfered or asked any questions. +LJ013-0118|The thief walked into the house in Belgrave Square, and openly carried off the plate-chest, deposited it in a light cart at the door, and drove away.|The thief walked into the house in Belgrave Square, and openly carried off the plate-chest, deposited it in a light cart at the door, and drove away. +LJ013-0119|Howse, the steward, accused the other servants, but they retorted, declaring that he had been visited by the thief the day previous,|Howse, the steward, accused the other servants, but they retorted, declaring that he had been visited by the thief the day previous, +LJ013-0120|whom he had shown over the plate closet.|whom he had shown over the plate closet. +LJ013-0121|Howse and his accomplice were arrested; the former was found guilty and sentenced to fifteen years, but the latter was acquitted.|Howse and his accomplice were arrested; the former was found guilty and sentenced to fifteen years, but the latter was acquitted. +LJ013-0122|Stealing plate was about this period the crime of a more aristocratic thief.|Stealing plate was about this period the crime of a more aristocratic thief. +LJ013-0123|The club spoons and other articles of plate were long a source of profitable income to a gentleman named Ashley,|The club spoons and other articles of plate were long a source of profitable income to a gentleman named Ashley, +LJ013-0124|who belonged to five good London clubs --|who belonged to five good London clubs -- +LJ013-0125|the Junior United Service, the Union, Reform, Colonial, and Erechtheum clubs.|the Junior United Service, the Union, Reform, Colonial, and Erechtheum clubs. +LJ013-0126|When one of these clubs was taken in at the Army and Navy, that establishment also suffered.|When one of these clubs was taken in at the Army and Navy, that establishment also suffered. +LJ013-0127|Suspicion fell at length upon Ashley, who was seen to handle the forks and spoons at table in a strange manner.|Suspicion fell at length upon Ashley, who was seen to handle the forks and spoons at table in a strange manner. +LJ013-0128|A watch was set on his house, in Allington Street, Pimlico,|A watch was set on his house, in Allington Street, Pimlico, +LJ013-0129|and one day a police constable tracked him to a silversmith's in Holborn Hill,|and one day a police constable tracked him to a silversmith's in Holborn Hill, +LJ013-0130|where Ashley produced four silver spoons, and begged that his initials might be engraved upon them.|where Ashley produced four silver spoons, and begged that his initials might be engraved upon them. +LJ013-0131|Ashley was arrested as he left the shop; the spoons were impounded, and it was found that the club monogram had been erased from them.|Ashley was arrested as he left the shop; the spoons were impounded, and it was found that the club monogram had been erased from them. +LJ013-0132|On a search of the prisoner's lodgings in Allington Street, a silver fork was found,|On a search of the prisoner's lodgings in Allington Street, a silver fork was found, +LJ013-0133|a number of pawnbrokers' duplicates, and three small files. It was proved at the trial that Ashley had asked his landlady for brick-dust and leather,|a number of pawnbrokers' duplicates, and three small files. It was proved at the trial that Ashley had asked his landlady for brick-dust and leather, +LJ013-0134|and it was contended that these with the files were used to alter the marks on the plate.|and it was contended that these with the files were used to alter the marks on the plate. +LJ013-0135|At most of the clubs the servants had been mulcted to make good lost plate, which had no doubt been stolen by the prisoner.|At most of the clubs the servants had been mulcted to make good lost plate, which had no doubt been stolen by the prisoner. +LJ013-0136|Several pawnbrokers were subpoenaed and obliged to surrender plate, to the extent in some cases of a couple of dozen of spoons or forks,|Several pawnbrokers were subpoenaed and obliged to surrender plate, to the extent in some cases of a couple of dozen of spoons or forks, +LJ013-0137|which the various club secretaries identified as the property of their respective clubs.|which the various club secretaries identified as the property of their respective clubs. +LJ013-0138|Ashley was the son of an army agent and banker,|Ashley was the son of an army agent and banker, +LJ013-0139|and many witnesses were brought to attest to his previous good character, but he was found guilty and sentenced to seven years' transportation.|and many witnesses were brought to attest to his previous good character, but he was found guilty and sentenced to seven years' transportation. +LJ013-0140|A robbery of a somewhat novel kind was executed in rather a bungling fashion by Ker, a sea-captain,|A robbery of a somewhat novel kind was executed in rather a bungling fashion by Ker, a sea-captain, +LJ013-0141|whose ship brought home a mixed cargo from Bahia and other ports.|whose ship brought home a mixed cargo from Bahia and other ports. +LJ013-0142|Part of the freight were four hundred rough diamonds valued at £4000.|Part of the freight were four hundred rough diamonds valued at four thousand pounds. +LJ013-0143|These packages were consigned to Messrs. Shroeder of London; and as it was known that they were to arrive in Ker's ship,|These packages were consigned to Messrs. Shroeder of London; and as it was known that they were to arrive in Ker's ship, +LJ013-0144|one of the owners had met her at Deal, but the captain had already absconded with the packages of precious stones in his pocket.|one of the owners had met her at Deal, but the captain had already absconded with the packages of precious stones in his pocket. +LJ013-0145|Ker came at once to London, and, by the help of the landlord of a public-house in Smithfield and others, disposed of the whole of the diamonds.|Ker came at once to London, and, by the help of the landlord of a public-house in Smithfield and others, disposed of the whole of the diamonds. +LJ013-0146|A Jew named Benjamin effected the sale to certain merchants named Blogg and Martin,|A Jew named Benjamin effected the sale to certain merchants named Blogg and Martin, +LJ013-0147|who declared that the rough diamond market was in such a depressed condition that they could only afford to give £1750 for stones worth|who declared that the rough diamond market was in such a depressed condition that they could only afford to give one thousand seven fifty pounds for stones worth +LJ013-0148|four thousand pounds|four thousand pounds +LJ013-0149|The circumstances of this purchase of brilliants from a stranger at such an inadequate price was strongly commented upon at Ker's trial.|The circumstances of this purchase of brilliants from a stranger at such an inadequate price was strongly commented upon at Ker's trial. +LJ013-0150|The moment it was discovered that the diamonds had disappeared, the affair was taken up by the police.|The moment it was discovered that the diamonds had disappeared, the affair was taken up by the police. +LJ013-0151|Forrester, the detective who had pursued and captured Burgess at Boston, tracked Ker to France, and following him there, eventually captured him|Forrester, the detective who had pursued and captured Burgess at Boston, tracked Ker to France, and following him there, eventually captured him +LJ013-0152|at Montreuil. He was arraigned at the Old Bailey, and the case fully proved. His sentence was seven years' transportation.|at Montreuil. He was arraigned at the Old Bailey, and the case fully proved. His sentence was seven years' transportation. +LJ013-0153|The gravest crimes continued at intervals to inspire the town with horror, and concentrate public attention upon the jail of Newgate,|The gravest crimes continued at intervals to inspire the town with horror, and concentrate public attention upon the jail of Newgate, +LJ013-0154|and the murderers immured within its walls.|and the murderers immured within its walls. +LJ013-0155|Courvoisier's case made a great stir. There was unusual atrocity in this murder of an aged, infirm gentleman,|Courvoisier's case made a great stir. There was unusual atrocity in this murder of an aged, infirm gentleman, +LJ013-0156|a scion of the ducal house of Bedford, by his confidential valet and personal attendant.|a scion of the ducal house of Bedford, by his confidential valet and personal attendant. +LJ013-0157|Lord William Russell lived alone in Norfolk Street, Park Lane. He was a widower, and seventy-three years of age.|Lord William Russell lived alone in Norfolk Street, Park Lane. He was a widower, and seventy-three years of age. +LJ013-0158|One morning in May his lordship was found dead in his bed with his throat cut.|One morning in May his lordship was found dead in his bed with his throat cut. +LJ013-0159|The fact of the murder was first discovered by the housemaid,|The fact of the murder was first discovered by the housemaid, +LJ013-0160|who, on going down early, was surprised to find the dining-room in a state of utter confusion;|who, on going down early, was surprised to find the dining-room in a state of utter confusion; +LJ013-0161|the furniture turned upside down, the drawers of the escritoire open and rifled,|the furniture turned upside down, the drawers of the escritoire open and rifled, +LJ013-0162|a bundle lying on the floor, as though thieves had been interrupted in the act.|a bundle lying on the floor, as though thieves had been interrupted in the act. +LJ013-0163|The housemaid summoned the cook, and both went to call the valet, Courvoisier,|The housemaid summoned the cook, and both went to call the valet, Courvoisier, +LJ013-0164|who came from his room ready dressed, a suspicious circumstance, as he was always late in the morning.|who came from his room ready dressed, a suspicious circumstance, as he was always late in the morning. +LJ013-0165|The housemaid suggested that they should see if his lordship was all right, and the three went to his bedroom.|The housemaid suggested that they should see if his lordship was all right, and the three went to his bedroom. +LJ013-0166|While Courvoisier opened the shutters, the housemaid, approaching the bed, saw that the pillow was saturated with blood.|While Courvoisier opened the shutters, the housemaid, approaching the bed, saw that the pillow was saturated with blood. +LJ013-0167|The discovery of the murdered man immediately followed. The neighborhood was alarmed, the police sent for, and a close inquiry forthwith commenced.|The discovery of the murdered man immediately followed. The neighborhood was alarmed, the police sent for, and a close inquiry forthwith commenced. +LJ013-0168|That Lord William Russell had committed suicide was at once declared impossible.|That Lord William Russell had committed suicide was at once declared impossible. +LJ013-0169|It was also clearly proved that no forcible entry had been made into the house;|It was also clearly proved that no forcible entry had been made into the house; +LJ013-0170|the fresh marks of violence upon the door had evidently been made inside, and not from outside;|the fresh marks of violence upon the door had evidently been made inside, and not from outside; +LJ013-0171|moreover, the instruments, poker and chisel, by which they had no doubt been effected,|moreover, the instruments, poker and chisel, by which they had no doubt been effected, +LJ013-0172|were found in the butler's pantry, used by Courvoisier.|were found in the butler's pantry, used by Courvoisier. +LJ013-0173|The researches of the police soon laid bare other suspicious facts.|The researches of the police soon laid bare other suspicious facts. +LJ013-0174|The bundle found in the dining-room contained, with clothes, various small articles of plate and jewelery which a thief would probably have put into his pocket.|The bundle found in the dining-room contained, with clothes, various small articles of plate and jewelery which a thief would probably have put into his pocket. +LJ013-0175|Upstairs in the bedroom a rouleaux box for sovereigns had been broken open,|Upstairs in the bedroom a rouleaux box for sovereigns had been broken open, +LJ013-0176|also the jewel-box and note-case, from the latter of which was abstracted a ten-pound note known to have been in the possession of the deceased.|also the jewel-box and note-case, from the latter of which was abstracted a ten-pound note known to have been in the possession of the deceased. +LJ013-0177|His lordship's watch was gone.|His lordship's watch was gone. +LJ013-0178|Further suspicion was caused by the position of a book and a wax candle by the bedside.|Further suspicion was caused by the position of a book and a wax candle by the bedside. +LJ013-0179|The latter was so placed that it could throw no light on the former, which was a 'Life of Sir Samuel Romilly.'|The latter was so placed that it could throw no light on the former, which was a 'Life of Sir Samuel Romilly.' +LJ013-0180|The intention of the real murderer to shift the crime to burglars was evident although futile,|The intention of the real murderer to shift the crime to burglars was evident although futile, +LJ013-0181|and the police, feeling convinced that the crime had been committed by some inmate of the house,|and the police, feeling convinced that the crime had been committed by some inmate of the house, +LJ013-0182|took Courvoisier into custody, and placed the two female servants under surveillance.|took Courvoisier into custody, and placed the two female servants under surveillance. +LJ013-0183|The valet's strange demeanor had attracted attention from the first.|The valet's strange demeanor had attracted attention from the first. +LJ013-0184|He had hung over the body in a state of dreadful agitation, answering no questions, and taking no part in the proceedings.|He had hung over the body in a state of dreadful agitation, answering no questions, and taking no part in the proceedings. +LJ013-0185|Three days later a close search of the butler's pantry produced fresh circumstantial evidence.|Three days later a close search of the butler's pantry produced fresh circumstantial evidence. +LJ013-0186|Behind the skirting board several of his lordship's rings were discovered;|Behind the skirting board several of his lordship's rings were discovered; +LJ013-0187|near it was his Waterloo medal, and the above-mentioned ten-pound note.|near it was his Waterloo medal, and the above-mentioned ten-pound note. +LJ013-0188|Further investigation was rewarded by the discovery in the pantry of a split gold ring, used by Lord William to carry his keys on;|Further investigation was rewarded by the discovery in the pantry of a split gold ring, used by Lord William to carry his keys on; +LJ013-0189|next, and in the same place, a chased gold key;|next, and in the same place, a chased gold key; +LJ013-0190|and at last his lordship's watch was found secreted under the leads of the sink.|and at last his lordship's watch was found secreted under the leads of the sink. +LJ013-0191|All this was evidence sufficient to warrant Courvoisier's committal for trial;|All this was evidence sufficient to warrant Courvoisier's committal for trial; +LJ013-0192|but still he found friends, and a liberal subscription was raised among the foreign servants in London to provide funds for his defense.|but still he found friends, and a liberal subscription was raised among the foreign servants in London to provide funds for his defense. +LJ013-0193|Courvoisier, when put on his trial, pleaded not guilty;|Courvoisier, when put on his trial, pleaded not guilty; +LJ013-0194|but on the second day the discovery of fresh evidence, more particularly the recovery of some of Lord William's stolen plate,|but on the second day the discovery of fresh evidence, more particularly the recovery of some of Lord William's stolen plate, +LJ013-0195|induced the prisoner to make a full confession of his crime to the lawyers who defended him.|induced the prisoner to make a full confession of his crime to the lawyers who defended him. +LJ013-0196|This placed them in a position of much embarrassment. To have thrown up their brief would have been to have secured Courvoisier's conviction.|This placed them in a position of much embarrassment. To have thrown up their brief would have been to have secured Courvoisier's conviction. +LJ013-0197|Mr. Phillips, who led in the case, went to the other extreme,|Mr. Phillips, who led in the case, went to the other extreme, +LJ013-0198|and in an impassioned address implored the jury not to send an innocent man to the gallows.|and in an impassioned address implored the jury not to send an innocent man to the gallows. +LJ013-0199|It will be remembered that the question whether Mr. Phillips had not exceeded the limits usually allowed to counsel was much debated at the time.|It will be remembered that the question whether Mr. Phillips had not exceeded the limits usually allowed to counsel was much debated at the time. +LJ013-0200|The jury without hesitation found Courvoisier guilty, and he was sentenced to death.|The jury without hesitation found Courvoisier guilty, and he was sentenced to death. +LJ013-0201|The prisoner's demeanor had greatly changed during the trial.|The prisoner's demeanor had greatly changed during the trial. +LJ013-0202|Coolness amounting almost to effrontery gave way to hopeless dejection.|Coolness amounting almost to effrontery gave way to hopeless dejection. +LJ013-0203|On his removal to Newgate after sentence,|On his removal to Newgate after sentence, +LJ013-0204|he admitted that he had been justly convicted, and expressed great anxiety that his fellow-servants should be relieved from all suspicion.|he admitted that he had been justly convicted, and expressed great anxiety that his fellow-servants should be relieved from all suspicion. +LJ013-0205|Later in the day he tried to commit suicide by cramming a towel down his throat, but was prevented.|Later in the day he tried to commit suicide by cramming a towel down his throat, but was prevented. +LJ013-0206|Next morning he made a full confession in presence of his attorney, and the governor, Mr. Cope.|Next morning he made a full confession in presence of his attorney, and the governor, Mr. Cope. +LJ013-0207|In this he gave as the motives of his crime a quarrel he had with his master, who threatened to discharge him without a character.|In this he gave as the motives of his crime a quarrel he had with his master, who threatened to discharge him without a character. +LJ013-0208|Lord William, according to the valet, was of a peevish, difficult temper;|Lord William, according to the valet, was of a peevish, difficult temper; +LJ013-0209|he was annoyed with his man for various small omissions and acts of forgetfulness, and on the night of the murder had taken Courvoisier to task rather sharply.|he was annoyed with his man for various small omissions and acts of forgetfulness, and on the night of the murder had taken Courvoisier to task rather sharply. +LJ013-0210|Finally, on coming downstairs after bed-time, Lord William had found Courvoisier in the dining-room.|Finally, on coming downstairs after bed-time, Lord William had found Courvoisier in the dining-room. +LJ013-0211|"What are you doing here?" asked his lordship.|"What are you doing here?" asked his lordship. +LJ013-0212|"You can have no good intentions; you must quit my service tomorrow morning."|"You can have no good intentions; you must quit my service tomorrow morning." +LJ013-0213|This seems to have decided Courvoisier,|This seems to have decided Courvoisier, +LJ013-0214|who took a carving-knife from the sideboard in the dining-room, went upstairs to Lord William's bedroom, and drew the knife across his throat.|who took a carving-knife from the sideboard in the dining-room, went upstairs to Lord William's bedroom, and drew the knife across his throat. +LJ013-0215|"He appeared to die instantly," said the murderer, in conclusion.|"He appeared to die instantly," said the murderer, in conclusion. +LJ013-0216|His account of his acts and movements after the deed|His account of his acts and movements after the deed +LJ013-0217|varied so considerably in the several documents he left behind, that too much reliance cannot be placed upon his confession.|varied so considerably in the several documents he left behind, that too much reliance cannot be placed upon his confession. +LJ013-0218|His last statement contains the words, "The public now think I am a liar, and they will not believe me when I say the truth."|His last statement contains the words, "The public now think I am a liar, and they will not believe me when I say the truth." +LJ013-0219|This was no doubt the case, but this much truth his confession may be taken to contain:|This was no doubt the case, but this much truth his confession may be taken to contain: +LJ013-0220|that Courvoisier was idle, discontented, ready to take offense, greedy of gain;|that Courvoisier was idle, discontented, ready to take offense, greedy of gain; +LJ013-0221|that he could not resist the opportunity for robbery offered him by his situation at Lord William Russell's; that when vexed with his master|that he could not resist the opportunity for robbery offered him by his situation at Lord William Russell's; that when vexed with his master +LJ013-0222|he did not shrink from murder, both for revenge and to conceal his other crimes.|he did not shrink from murder, both for revenge and to conceal his other crimes. +LJ013-0223|Courvoisier wished to commit suicide in Newgate, but was prevented by the vigilant supervision to which he was subjected while in jail.|Courvoisier wished to commit suicide in Newgate, but was prevented by the vigilant supervision to which he was subjected while in jail. +LJ013-0224|The attempt was to have been made by opening a vein and allowing himself to bleed to death.|The attempt was to have been made by opening a vein and allowing himself to bleed to death. +LJ013-0225|The Sunday night before his execution he would not go to bed when ordered.|The Sunday night before his execution he would not go to bed when ordered. +LJ013-0226|The governor insisted, but Courvoisier showed great reluctance to strip.|The governor insisted, but Courvoisier showed great reluctance to strip. +LJ013-0227|The order was, however, at length obeyed, and the whole of the prisoner's clothes were minutely searched.|The order was, however, at length obeyed, and the whole of the prisoner's clothes were minutely searched. +LJ013-0229|Courvoisier admitted that he had intended to bind it tightly round his arm and bleed himself to death in the night.|Courvoisier admitted that he had intended to bind it tightly round his arm and bleed himself to death in the night. +LJ013-0230|The next inquiry was how he hoped to open a vein. "With a bit of sharpened stick picked out of the ordinary firewood."|The next inquiry was how he hoped to open a vein. "With a bit of sharpened stick picked out of the ordinary firewood." +LJ013-0231|"Where is it?" asked the governor.|"Where is it?" asked the governor. +LJ013-0232|The prisoner replied that he had left it in the mattress of which he had just been deprived.|The prisoner replied that he had left it in the mattress of which he had just been deprived. +LJ013-0233|The bed was searched, but no piece of sharpened wood was found. It was thought that it might have been lost in changing the mattresses.|The bed was searched, but no piece of sharpened wood was found. It was thought that it might have been lost in changing the mattresses. +LJ013-0234|The cloth above referred to belonged to the inner seam of his trousers, which he had managed to tear out.|The cloth above referred to belonged to the inner seam of his trousers, which he had managed to tear out. +LJ013-0235|There is nothing to show that Courvoisier really contemplated self-destruction.|There is nothing to show that Courvoisier really contemplated self-destruction. +LJ013-0236|A murder which reproduced many of the features of that committed by Greenacre soon followed, and excited the public mind even more than that of Courvoisier's.|A murder which reproduced many of the features of that committed by Greenacre soon followed, and excited the public mind even more than that of Courvoisier's. +LJ013-0237|Daniel Good's crime might have remained long undiscovered but for his own careless stupidity.|Daniel Good's crime might have remained long undiscovered but for his own careless stupidity. +LJ013-0238|He was coachman to a gentleman at Roehampton. One day he went into a pawnbroker's at Wandsworth, and bought a pair of breeches on credit.|He was coachman to a gentleman at Roehampton. One day he went into a pawnbroker's at Wandsworth, and bought a pair of breeches on credit. +LJ013-0239|At the same time he was seen to steal and secrete a pair of trousers. The shop-boy gave information.|At the same time he was seen to steal and secrete a pair of trousers. The shop-boy gave information. +LJ013-0240|Good was followed to his stables by a policeman, but obstinately denied the theft.|Good was followed to his stables by a policeman, but obstinately denied the theft. +LJ013-0241|The policeman insisted on searching the premises, at which Good displayed some uneasiness.|The policeman insisted on searching the premises, at which Good displayed some uneasiness. +LJ013-0242|This increased when the officer, accompanied by two others, a neighbor and a bailiff, entered one of the stables.|This increased when the officer, accompanied by two others, a neighbor and a bailiff, entered one of the stables. +LJ013-0243|Good now offered to go to Wandsworth and satisfy the pawnbroker.|Good now offered to go to Wandsworth and satisfy the pawnbroker. +LJ013-0244|Just at this moment, however, the searchers found concealed under two trusses of hay a woman's headless and dismembered trunk.|Just at this moment, however, the searchers found concealed under two trusses of hay a woman's headless and dismembered trunk. +LJ013-0245|At the constable's cry of alarm Good rushed from the stable and locked the door behind him.|At the constable's cry of alarm Good rushed from the stable and locked the door behind him. +LJ013-0246|Some time elapsed before the imprisoned party could force open the doors, and by then the fugitive had escaped.|Some time elapsed before the imprisoned party could force open the doors, and by then the fugitive had escaped. +LJ013-0247|Medical assistance having been summoned, it was ascertained how the dismemberment had been effected.|Medical assistance having been summoned, it was ascertained how the dismemberment had been effected. +LJ013-0248|At the same time an overpowering odor attracted them to the adjoining harness-room, where the missing remains were raked out|At the same time an overpowering odor attracted them to the adjoining harness-room, where the missing remains were raked out +LJ013-0249|half consumed in the ashes of a wood fire.|half consumed in the ashes of a wood fire. +LJ013-0250|In the same room a large axe and saw were found covered with blood.|In the same room a large axe and saw were found covered with blood. +LJ013-0251|Inquiry into the character of Good exposed him as a loose liver, who "kept company" with several women.|Inquiry into the character of Good exposed him as a loose liver, who "kept company" with several women. +LJ013-0252|One called his sister, but supposed to be his wife, had occupied a room in South Street, Manchester Square,|One called his sister, but supposed to be his wife, had occupied a room in South Street, Manchester Square, +LJ013-0253|with a son of Good's by a former wife. Another wife, real or fictitious, existed in Spitalfields,|with a son of Good's by a former wife. Another wife, real or fictitious, existed in Spitalfields, +LJ013-0254|and evidence was given of close relation between Good and a third woman, a girl named Butcher, residing at Woolwich.|and evidence was given of close relation between Good and a third woman, a girl named Butcher, residing at Woolwich. +LJ013-0255|The victim was the first of these three.|The victim was the first of these three. +LJ013-0256|Good had told her, much to her perturbation, that she was to move from South Street to Roehampton, and one day he fetched her.|Good had told her, much to her perturbation, that she was to move from South Street to Roehampton, and one day he fetched her. +LJ013-0257|They were seen together on Barnes Common, and again in Putney Park Lane, where they were talking loud and angrily.|They were seen together on Barnes Common, and again in Putney Park Lane, where they were talking loud and angrily. +LJ013-0258|The poor creature was never seen again alive.|The poor creature was never seen again alive. +LJ013-0259|The actual method of the murder was never exactly ascertained.|The actual method of the murder was never exactly ascertained. +LJ013-0260|Good himself remained at large for some weeks. He had tramped as far as Tunbridge, where he obtained work as a bricklayer's laborer;|Good himself remained at large for some weeks. He had tramped as far as Tunbridge, where he obtained work as a bricklayer's laborer; +LJ013-0261|he there gave satisfaction for industry, but he was taciturn, and would hold no converse with his fellows.|he there gave satisfaction for industry, but he was taciturn, and would hold no converse with his fellows. +LJ013-0262|The woman where he lodged noticed that he was very restless at night, moaning and sighing much. Detection came unexpectedly.|The woman where he lodged noticed that he was very restless at night, moaning and sighing much. Detection came unexpectedly. +LJ013-0263|He was recognized by an ex-policeman who had known him at Roehampton, and immediately arrested.|He was recognized by an ex-policeman who had known him at Roehampton, and immediately arrested. +LJ013-0264|In his effects were found the clothes he had on at the time of his escape from the stables, and under the jacket he was wearing|In his effects were found the clothes he had on at the time of his escape from the stables, and under the jacket he was wearing +LJ013-0265|was a piece of a woman's calico apron stained with blood, which he had used to save the pressure on his shoulder by the hod.|was a piece of a woman's calico apron stained with blood, which he had used to save the pressure on his shoulder by the hod. +LJ013-0266|Good was committed to Newgate, and tried at the Central Criminal Court before a crowded court.|Good was committed to Newgate, and tried at the Central Criminal Court before a crowded court. +LJ013-0267|He made a rambling defense, ending by saying,|He made a rambling defense, ending by saying, +LJ013-0268|Good ladies and gentlemen all, I have a great deal more to say, but I am so bad I cannot say it.|Good ladies and gentlemen all, I have a great deal more to say, but I am so bad I cannot say it. +LJ014-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 17: Newgate notorieties continued, part 2.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section seventeen: Newgate notorieties continued, part two. +LJ014-0002|Hocker's murder is in its way interesting, as affording another proof of the extraordinary way in which the culprit returned to the scene of his guilt.|Hocker's murder is in its way interesting, as affording another proof of the extraordinary way in which the culprit returned to the scene of his guilt. +LJ014-0003|The cries of his victim, a Mr. Delarue,|The cries of his victim, a Mr. Delarue, +LJ014-0004|brought passers-by and policemen to the spot, a lonely place near a dead wall beyond Belsize Hall, Hampstead,|brought passers-by and policemen to the spot, a lonely place near a dead wall beyond Belsize Hall, Hampstead, +LJ014-0005|but too late to give substantial aid.|but too late to give substantial aid. +LJ014-0006|While the body lay there still warm, battered and bleeding from the cruel blows inflicted upon him by his cowardly assailant, a man came by singing.|While the body lay there still warm, battered and bleeding from the cruel blows inflicted upon him by his cowardly assailant, a man came by singing. +LJ014-0007|He entered into conversation with the policemen, and learnt, as it seemed for the first time, what had happened.|He entered into conversation with the policemen, and learnt, as it seemed for the first time, what had happened. +LJ014-0008|His remark was, "It is a nasty job;" he took hold of the dead hand, and confessed that he felt "queer" at the shocking sight.|His remark was, "It is a nasty job;" he took hold of the dead hand, and confessed that he felt "queer" at the shocking sight. +LJ014-0009|This sight was his own handiwork,|This sight was his own handiwork, +LJ014-0010|yet he could not overcome the strange fascination it had for him, and remained by the side of the corpse till the stretcher came.|yet he could not overcome the strange fascination it had for him, and remained by the side of the corpse till the stretcher came. +LJ014-0011|Even then he followed it as far as Belsize Lane.|Even then he followed it as far as Belsize Lane. +LJ014-0012|It was here that the others engaged in their dismal office in removing the dead first got a good look at the stranger's face.|It was here that the others engaged in their dismal office in removing the dead first got a good look at the stranger's face. +LJ014-0013|He wanted a light for a cigar, and got it from a lantern which was lifted up and fully betrayed his features.|He wanted a light for a cigar, and got it from a lantern which was lifted up and fully betrayed his features. +LJ014-0014|It was noticed that he wore a mackintosh. Next day the police, in making a careful search of the scene of the murder,|It was noticed that he wore a mackintosh. Next day the police, in making a careful search of the scene of the murder, +LJ014-0015|picked up a coat-button, which afterwards played an important part in the identification of the murderer.|picked up a coat-button, which afterwards played an important part in the identification of the murderer. +LJ014-0016|A letter, which afforded an additional clue, was also found in the pocket of the deceased. Still it was many weeks before any arrest was made.|A letter, which afforded an additional clue, was also found in the pocket of the deceased. Still it was many weeks before any arrest was made. +LJ014-0017|In the mean time the police were not idle.|In the mean time the police were not idle. +LJ014-0018|It came out by degrees that the person who had been seen in Belsize Lane on the night the body was found was a friend of the deceased.|It came out by degrees that the person who had been seen in Belsize Lane on the night the body was found was a friend of the deceased. +LJ014-0019|His name was Hocker; he was by trade a ladies' shoemaker; and it was also ascertained that after the day of the murder he was flush of money.|His name was Hocker; he was by trade a ladies' shoemaker; and it was also ascertained that after the day of the murder he was flush of money. +LJ014-0021|a coat among them much torn and stained, with three buttons missing, one of which corresponded with that picked up at Hampstead.|a coat among them much torn and stained, with three buttons missing, one of which corresponded with that picked up at Hampstead. +LJ014-0022|The letter found in the pocket of the deceased was sealed with a wafer marked F,|The letter found in the pocket of the deceased was sealed with a wafer marked F, +LJ014-0023|and many of the same sort were found in the possession of the accused. This was enough to obtain a committal,|and many of the same sort were found in the possession of the accused. This was enough to obtain a committal, +LJ014-0024|after several remands; but the case contained elements of doubt, and the evidence at the trial was entirely circumstantial.|after several remands; but the case contained elements of doubt, and the evidence at the trial was entirely circumstantial. +LJ014-0025|A witness deposed to meeting Hocker, soon after the cries of murder were heard,|A witness deposed to meeting Hocker, soon after the cries of murder were heard, +LJ014-0026|running at a dog-trot into London, and others swore that they plainly recognized him as the man seen soon afterwards in the lane.|running at a dog-trot into London, and others swore that they plainly recognized him as the man seen soon afterwards in the lane. +LJ014-0027|A woman whom he called on the same evening declared he had worn a mackintosh, his coat was much torn, there was a stain of blood on his shirt-cuff,|A woman whom he called on the same evening declared he had worn a mackintosh, his coat was much torn, there was a stain of blood on his shirt-cuff, +LJ014-0028|and he was in possession, the first time to her knowledge, of a watch.|and he was in possession, the first time to her knowledge, of a watch. +LJ014-0029|This was Delarue's watch, fully identified as such, which Hocker told his brother Delarue had given him the morning of the murder.|This was Delarue's watch, fully identified as such, which Hocker told his brother Delarue had given him the morning of the murder. +LJ014-0030|These were damnatory facts which well supported the prosecution.|These were damnatory facts which well supported the prosecution. +LJ014-0031|The prisoner made an elaborate defense, in which he sought to vilify the character of deceased|The prisoner made an elaborate defense, in which he sought to vilify the character of deceased +LJ014-0032|as the seducer of an innocent girl to whom he (Hocker) had been fondly attached.|as the seducer of an innocent girl to whom he (Hocker) had been fondly attached. +LJ014-0033|When her ruin was discovered her brother panted for revenge.|When her ruin was discovered her brother panted for revenge. +LJ014-0034|Hocker, whose skill in counterfeiting handwriting was known, was asked to fabricate a letter making an assignation with Delarue|Hocker, whose skill in counterfeiting handwriting was known, was asked to fabricate a letter making an assignation with Delarue +LJ014-0035|in a lonely part of Hampstead.|in a lonely part of Hampstead. +LJ014-0036|Hocker and the brother went to the spot, where the latter left him to meet his sister's seducer alone.|Hocker and the brother went to the spot, where the latter left him to meet his sister's seducer alone. +LJ014-0037|Soon afterwards Hocker heard cries of "murder,"|Soon afterwards Hocker heard cries of "murder," +LJ014-0038|and proceeding to where they came from, found Delarue dead, slain by the furious brother.|and proceeding to where they came from, found Delarue dead, slain by the furious brother. +LJ014-0039|Hocker was so overcome, feeling himself the principal cause of the tragedy,|Hocker was so overcome, feeling himself the principal cause of the tragedy, +LJ014-0040|that he rushed to a slaughterhouse in Hampstead and purposely stained his clothes with blood.|that he rushed to a slaughterhouse in Hampstead and purposely stained his clothes with blood. +LJ014-0041|Such an extravagant defense did not weigh with judge or jury;|Such an extravagant defense did not weigh with judge or jury; +LJ014-0042|the first summed up dead against the prisoner, and the latter, after retiring for ten minutes, found him guilty.|the first summed up dead against the prisoner, and the latter, after retiring for ten minutes, found him guilty. +LJ014-0043|Hocker's conduct in Newgate while under sentence of death was most extraordinary.|Hocker's conduct in Newgate while under sentence of death was most extraordinary. +LJ014-0044|He drew up several long statements, containing narratives purely fictitious, imputing crimes to his victim, and repeating his line of defense|He drew up several long statements, containing narratives purely fictitious, imputing crimes to his victim, and repeating his line of defense +LJ014-0045|that Delarue had suffered by the hands of imaginary outraged brothers acting as the avengers of females deeply injured by him.|that Delarue had suffered by the hands of imaginary outraged brothers acting as the avengers of females deeply injured by him. +LJ014-0046|Hocker made several pretended confessions and revelations, all of which were proved to be absolutely false by the police on inquiry.|Hocker made several pretended confessions and revelations, all of which were proved to be absolutely false by the police on inquiry. +LJ014-0047|His demeanor was a strange compound of wickedness, falsehood, and deceit.|His demeanor was a strange compound of wickedness, falsehood, and deceit. +LJ014-0048|But at the fatal hour his hardihood forsook him, and he was almost insensible when taken out of his cell for execution.|But at the fatal hour his hardihood forsook him, and he was almost insensible when taken out of his cell for execution. +LJ014-0049|Restoratives were applied, but he was in a fainting condition when tied, and had to be supported by the assistant executioner|Restoratives were applied, but he was in a fainting condition when tied, and had to be supported by the assistant executioner +LJ014-0050|while Calcraft adjusted the noose.|while Calcraft adjusted the noose. +LJ014-0051|There was an epidemic of murder in the United Kingdom about 1848 to 9.|There was an epidemic of murder in the United Kingdom about eighteen forty-eight to nine. +LJ014-0052|In November of the first-named year occurred the wholesale slaughter of the Jermys in their house, Stanfield Hall, by the miscreant Rush.|In November of the first-named year occurred the wholesale slaughter of the Jermys in their house, Stanfield Hall, by the miscreant Rush. +LJ014-0053|Soon afterwards, in Gloucestershire,|Soon afterwards, in Gloucestershire, +LJ014-0055|Next year John Gleeson Wilson, at Liverpool, murdered a woman, Ann Henrichson, also a maidservant and two children;|Next year John Gleeson Wilson, at Liverpool, murdered a woman, Ann Henrichson, also a maidservant and two children; +LJ014-0056|while in Ireland a wife dashed out her husband's brains with a hammer.|while in Ireland a wife dashed out her husband's brains with a hammer. +LJ014-0057|London did not escape the contagion, and prominent among the detestable crimes of the period stands that of the Mannings at Bermondsey.|London did not escape the contagion, and prominent among the detestable crimes of the period stands that of the Mannings at Bermondsey. +LJ014-0058|These great criminals suffered at Horsemonger Lane Jail, but they were tried at the Central Criminal Court, and were for some time inmates of Newgate.|These great criminals suffered at Horsemonger Lane Jail, but they were tried at the Central Criminal Court, and were for some time inmates of Newgate. +LJ014-0059|Their victim was a man named Patrick O'Connor, a Custom-House gauger, who had been a suitor of Marie de Roux before she became Mrs. Manning.|Their victim was a man named Patrick O'Connor, a Custom-House gauger, who had been a suitor of Marie de Roux before she became Mrs. Manning. +LJ014-0060|Marie de Roux up to the time of her marriage had been in service as lady's maid to Lady Blantyre, daughter of the Duchess of Sutherland,|Marie de Roux up to the time of her marriage had been in service as lady's maid to Lady Blantyre, daughter of the Duchess of Sutherland, +LJ014-0061|and Manning hoped to get some small Government appointment through his wife's interest.|and Manning hoped to get some small Government appointment through his wife's interest. +LJ014-0062|He had failed in this as well as in the business of a publican, which he had at one time adopted.|He had failed in this as well as in the business of a publican, which he had at one time adopted. +LJ014-0063|After the marriage a close intimacy was still maintained between O'Connor and the Mannings.|After the marriage a close intimacy was still maintained between O'Connor and the Mannings. +LJ014-0064|He lived at Mile End, whence he walked often to call at 3, Minver Place, Bermondsey, the residence of his old love.|He lived at Mile End, whence he walked often to call at three, Minver Place, Bermondsey, the residence of his old love. +LJ014-0065|O'Connor was a man of substance. He had long followed the profitable trade of a money-lender,|O'Connor was a man of substance. He had long followed the profitable trade of a money-lender, +LJ014-0066|and by dint of usurious interest on small sums advanced to needy neighbors, had amassed as much as eight thousand pounds or ten thousand pounds.|and by dint of usurious interest on small sums advanced to needy neighbors, had amassed as much as eight thousand pounds or ten thousand pounds. +LJ014-0067|His wealth was well known to "Maria," as he called Mrs. Manning, who made several ineffectual attempts to get money out of him.|His wealth was well known to "Maria," as he called Mrs. Manning, who made several ineffectual attempts to get money out of him. +LJ014-0068|At last this fiendish woman made up her mind to murder O'Connor and appropriate all his possessions.|At last this fiendish woman made up her mind to murder O'Connor and appropriate all his possessions. +LJ014-0069|Her husband, to whom she coolly confided her intention,|Her husband, to whom she coolly confided her intention, +LJ014-0070|a heavy brutish fellow, was yet aghast at his wife's resolve, and tried hard to dissuade her from bad purpose.|a heavy brutish fellow, was yet aghast at his wife's resolve, and tried hard to dissuade her from bad purpose. +LJ014-0071|In his confession after sentence he declared that she plied him well with brandy at this period,|In his confession after sentence he declared that she plied him well with brandy at this period, +LJ014-0072|and that during the whole time he was never in his right senses.|and that during the whole time he was never in his right senses. +LJ014-0073|Meanwhile this woman, unflinching in her cold, bloody determination, carefully laid all her plans for the consummation of the deed.|Meanwhile this woman, unflinching in her cold, bloody determination, carefully laid all her plans for the consummation of the deed. +LJ014-0074|One fine afternoon in August, O'Connor was met walking in the direction of Bermondsey.|One fine afternoon in August, O'Connor was met walking in the direction of Bermondsey. +LJ014-0075|He was dressed with particular care, as he was to dine at the Mannings and meet friends, one a young lady.|He was dressed with particular care, as he was to dine at the Mannings and meet friends, one a young lady. +LJ014-0076|He was seen afterwards smoking and talking with his hosts in their back parlor, and never seen again alive.|He was seen afterwards smoking and talking with his hosts in their back parlor, and never seen again alive. +LJ014-0077|It came out in the husband's confession that Mrs. Manning induced O'Connor to go down to the kitchen to wash his hands, that she followed him to the basement,|It came out in the husband's confession that Mrs. Manning induced O'Connor to go down to the kitchen to wash his hands, that she followed him to the basement, +LJ014-0078|that she stood behind him as he stood near the open grave she herself had dug for him,|that she stood behind him as he stood near the open grave she herself had dug for him, +LJ014-0079|and which he mistook for a drain, and that while he was speaking to her she put the muzzle of a pistol close to the back of his head and shot him down.|and which he mistook for a drain, and that while he was speaking to her she put the muzzle of a pistol close to the back of his head and shot him down. +LJ014-0080|She ran upstairs, told her husband, made him go down to look at her handiwork, and as O'Connor was not quite dead,|She ran upstairs, told her husband, made him go down to look at her handiwork, and as O'Connor was not quite dead, +LJ014-0081|Manning gave the coup de grace with a crowbar.|Manning gave the coup de grace with a crowbar. +LJ014-0082|After this Mrs. Manning changed her dress and went off in a cab to O'Connor's lodgings,|After this Mrs. Manning changed her dress and went off in a cab to O'Connor's lodgings, +LJ014-0083|which, having possessed herself of the murdered man's keys, she rifled from end to end.|which, having possessed herself of the murdered man's keys, she rifled from end to end. +LJ014-0084|Returning to her own home, where Manning meantime had been calmly smoking and talking to the neighbors over the basement wall,|Returning to her own home, where Manning meantime had been calmly smoking and talking to the neighbors over the basement wall, +LJ014-0085|the corpse lying just inside the kitchen all the while, the two set to work to strip the body and hide it under the stones of the floor.|the corpse lying just inside the kitchen all the while, the two set to work to strip the body and hide it under the stones of the floor. +LJ014-0086|This job was not completed till the following day, as the hole had to be enlarged, and the only tool they had was a dust-shovel.|This job was not completed till the following day, as the hole had to be enlarged, and the only tool they had was a dust-shovel. +LJ014-0087|A quantity of quicklime was thrown in with the body to destroy all identification.|A quantity of quicklime was thrown in with the body to destroy all identification. +LJ014-0088|This was on a Thursday evening.|This was on a Thursday evening. +LJ014-0089|For the remainder of that week and part of the next the murderers stayed in the house, and occupied the kitchen, close to the remains of their victim.|For the remainder of that week and part of the next the murderers stayed in the house, and occupied the kitchen, close to the remains of their victim. +LJ014-0090|On the Sunday Mrs. Manning roasted a goose at this same kitchen fire, and ate it with relish in the afternoon.|On the Sunday Mrs. Manning roasted a goose at this same kitchen fire, and ate it with relish in the afternoon. +LJ014-0091|This cold-blooded indifference after the event was only outdone by the premeditation of this horrible murder.|This cold-blooded indifference after the event was only outdone by the premeditation of this horrible murder. +LJ014-0092|The hole must have been excavated and the quicklime purchased quite three weeks before O'Connor met his death,|The hole must have been excavated and the quicklime purchased quite three weeks before O'Connor met his death, +LJ014-0093|and during that time he must frequently have stood or sat over his own grave.|and during that time he must frequently have stood or sat over his own grave. +LJ014-0094|Discovery of the murder came in this wise. O'Connor, a punctual and well-conducted official, was at once missed at the London Docks.|Discovery of the murder came in this wise. O'Connor, a punctual and well-conducted official, was at once missed at the London Docks. +LJ014-0095|On the third day his friends began to inquire for him,|On the third day his friends began to inquire for him, +LJ014-0096|and at their request two police officers were sent to Bermondsey to inquire for him at the Mannings, with whom it was well known that he was very intimate.|and at their request two police officers were sent to Bermondsey to inquire for him at the Mannings, with whom it was well known that he was very intimate. +LJ014-0097|The Mannings had seen or heard nothing of him, of course. As O'Connor still did not turn up, the police after a couple of days returned to Minver Place.|The Mannings had seen or heard nothing of him, of course. As O'Connor still did not turn up, the police after a couple of days returned to Minver Place. +LJ014-0098|The house was empty, bare and stripped of all its furniture, and its former occupants had decamped.|The house was empty, bare and stripped of all its furniture, and its former occupants had decamped. +LJ014-0099|The circumstance was suspicious, and a search was at once made of the whole premises.|The circumstance was suspicious, and a search was at once made of the whole premises. +LJ014-0100|In the back kitchen one of the detectives remarked that the cement between certain stones looked lighter than the rest, and on trying it with a knife,|In the back kitchen one of the detectives remarked that the cement between certain stones looked lighter than the rest, and on trying it with a knife, +LJ014-0102|The stones were at once taken up;|The stones were at once taken up; +LJ014-0104|Six inches lower the body of O'Connor was uncovered.|Six inches lower the body of O'Connor was uncovered. +LJ014-0105|He was lying on his face, his legs tied up to his hips so as to allow of the body fitting into the hole.|He was lying on his face, his legs tied up to his hips so as to allow of the body fitting into the hole. +LJ014-0106|The lime had done its work so rapidly that the features would have been indistinguishable but for the prominent chin and a set of false teeth.|The lime had done its work so rapidly that the features would have been indistinguishable but for the prominent chin and a set of false teeth. +LJ014-0107|The corpse settled all doubts, and the next point was to lay hands upon the Mannings.|The corpse settled all doubts, and the next point was to lay hands upon the Mannings. +LJ014-0108|It was soon ascertained that the wife had gone off in a cab with a quantity of luggage.|It was soon ascertained that the wife had gone off in a cab with a quantity of luggage. +LJ014-0109|Part of this she had deposited to be left till called for at one station, while she had gone herself to another, that at Euston Square.|Part of this she had deposited to be left till called for at one station, while she had gone herself to another, that at Euston Square. +LJ014-0110|At the first the boxes were impounded, opened, and found to contain many of O'Connor's effects.|At the first the boxes were impounded, opened, and found to contain many of O'Connor's effects. +LJ014-0111|At the second exact information was obtained of Mrs. Manning's movements. She had gone to Edinburgh.|At the second exact information was obtained of Mrs. Manning's movements. She had gone to Edinburgh. +LJ014-0112|A telegraphic message, then newly adapted to the purposes of criminal detection,|A telegraphic message, then newly adapted to the purposes of criminal detection, +LJ014-0113|advised the Edinburgh police of the whole affair, and within an hour an answer was telegraphed, stating that Mrs. Manning was in custody.|advised the Edinburgh police of the whole affair, and within an hour an answer was telegraphed, stating that Mrs. Manning was in custody. +LJ014-0114|She had been to brokers to negotiate the sale of certain foreign railway stock, with which they had been warned from London not to deal,|She had been to brokers to negotiate the sale of certain foreign railway stock, with which they had been warned from London not to deal, +LJ014-0115|and they had given information to the police.|and they had given information to the police. +LJ014-0116|Her arrest was planned, and, when the telegram arrived from London, completed.|Her arrest was planned, and, when the telegram arrived from London, completed. +LJ014-0117|An examination of her boxes disclosed a quantity of O'Connor's property.|An examination of her boxes disclosed a quantity of O'Connor's property. +LJ014-0118|Mrs. Manning was transferred to London and lodged in the Horsemonger Lane Jail, where her husband soon afterwards joined her.|Mrs. Manning was transferred to London and lodged in the Horsemonger Lane Jail, where her husband soon afterwards joined her. +LJ014-0119|He had fled to Jersey, where he was recognized and arrested.|He had fled to Jersey, where he was recognized and arrested. +LJ014-0120|Each tried to throw the blame on the other; Manning declared his wife had committed the murder, Mrs. Manning indignantly denied the charge.|Each tried to throw the blame on the other; Manning declared his wife had committed the murder, Mrs. Manning indignantly denied the charge. +LJ014-0121|The prisoners were in due course transferred to Newgate, to be put upon their trial at the Central Criminal Court.|The prisoners were in due course transferred to Newgate, to be put upon their trial at the Central Criminal Court. +LJ014-0122|A great number of distinguished people assembled as usual at the Old Bailey on the day of trial.|A great number of distinguished people assembled as usual at the Old Bailey on the day of trial. +LJ014-0123|The Mannings were arraigned together; the husband standing at one of the front corners of the dock, his wife at the other end.|The Mannings were arraigned together; the husband standing at one of the front corners of the dock, his wife at the other end. +LJ014-0124|Manning, who was dressed in black, appeared to be a heavy, bull-necked, repulsive-looking man, with a very fair complexion and light hair.|Manning, who was dressed in black, appeared to be a heavy, bull-necked, repulsive-looking man, with a very fair complexion and light hair. +LJ014-0125|Mrs. Manning was not without personal charms;|Mrs. Manning was not without personal charms; +LJ014-0126|her face was comely, she had dark hair and good eyes, and was above the middle height, yet inclined to be stout.|her face was comely, she had dark hair and good eyes, and was above the middle height, yet inclined to be stout. +LJ014-0127|She was smartly dressed in a plaid shawl, a white lace cap;|She was smartly dressed in a plaid shawl, a white lace cap; +LJ014-0128|her hair was dressed in long crepe bands. She had lace ruffles at her wrist, and wore primrose-colored kid gloves.|her hair was dressed in long crepe bands. She had lace ruffles at her wrist, and wore primrose-colored kid gloves. +LJ014-0129|The case rested upon the facts which have been already set forth, and was proved to the satisfaction of the jury, who brought in a verdict of guilty.|The case rested upon the facts which have been already set forth, and was proved to the satisfaction of the jury, who brought in a verdict of guilty. +LJ014-0130|Manning, when sentence of death was passed on him, said nothing;|Manning, when sentence of death was passed on him, said nothing; +LJ014-0131|but Mrs. Manning, speaking in a foreign accent, addressed the court with great fluency and vehemence.|but Mrs. Manning, speaking in a foreign accent, addressed the court with great fluency and vehemence. +LJ014-0132|She complained that she had no justice; there was no law for her, she had found no protection either from judges, the prosecutor, or her husband.|She complained that she had no justice; there was no law for her, she had found no protection either from judges, the prosecutor, or her husband. +LJ014-0133|She had not been treated like a Christian, but like a wild beast of the forest. She declared that the money found in her possession had been sent her from abroad;|She had not been treated like a Christian, but like a wild beast of the forest. She declared that the money found in her possession had been sent her from abroad; +LJ014-0134|that O'Connor had been more to her than her husband, that she ought to have married him.|that O'Connor had been more to her than her husband, that she ought to have married him. +LJ014-0135|It was against common sense to charge her with murdering the only friend she had in the world;|It was against common sense to charge her with murdering the only friend she had in the world; +LJ014-0136|the culprit was really her husband, who killed O'Connor out of jealousy and revengeful feelings.|the culprit was really her husband, who killed O'Connor out of jealousy and revengeful feelings. +LJ014-0137|When the judge assumed the black cap|When the judge assumed the black cap +LJ014-0138|Mrs. Manning became still more violent, shouting, "No, no, I will not stand it! You ought to be ashamed of yourselves!"|Mrs. Manning became still more violent, shouting, "No, no, I will not stand it! You ought to be ashamed of yourselves!" +LJ014-0139|and would have left the dock had not Mr. Cope, the governor of Newgate, restrained her.|and would have left the dock had not Mr. Cope, the governor of Newgate, restrained her. +LJ014-0140|After judgment was passed she repeatedly cried out Shame!|After judgment was passed she repeatedly cried out Shame! +LJ014-0141|and stretching out her hand, she gathered up a quantity of the rue which, following ancient custom dating from the days of the jail fever,|and stretching out her hand, she gathered up a quantity of the rue which, following ancient custom dating from the days of the jail fever, +LJ014-0142|was strewn in front of the dock, and sprinkled it towards the bench with a contemptuous gesture.|was strewn in front of the dock, and sprinkled it towards the bench with a contemptuous gesture. +LJ014-0143|On being removed to Newgate from the court Mrs. Manning became perfectly furious.|On being removed to Newgate from the court Mrs. Manning became perfectly furious. +LJ014-0144|She uttered loud imprecations, cursing judge, jury, barristers, witnesses, and all who stood around.|She uttered loud imprecations, cursing judge, jury, barristers, witnesses, and all who stood around. +LJ014-0146|They had to handcuff her by force against the most violent resistance, and still she raged and stormed,|They had to handcuff her by force against the most violent resistance, and still she raged and stormed, +LJ014-0147|shaking her clenched and manacled hands in the officers' faces.|shaking her clenched and manacled hands in the officers' faces. +LJ014-0148|From Newgate the Mannings were taken in separate cabs to Horsemonger Lane Jail.|From Newgate the Mannings were taken in separate cabs to Horsemonger Lane Jail. +LJ014-0149|On this journey her manner changed completely. She became flippant, joked with the officers, asked how they liked her "resolution" in the dock,|On this journey her manner changed completely. She became flippant, joked with the officers, asked how they liked her "resolution" in the dock, +LJ014-0150|and expressed the utmost contempt for her husband, whom she never intended to acknowledge or speak to again.|and expressed the utmost contempt for her husband, whom she never intended to acknowledge or speak to again. +LJ014-0151|Later her mood changed to abject despair.|Later her mood changed to abject despair. +LJ014-0152|On reaching the condemned cell she threw herself upon the floor and shrieked in an hysterical agony of tears.|On reaching the condemned cell she threw herself upon the floor and shrieked in an hysterical agony of tears. +LJ014-0153|After this, until the day of execution, she recovered her spirits, and displayed reckless effrontery,|After this, until the day of execution, she recovered her spirits, and displayed reckless effrontery, +LJ014-0154|mocking at the chaplain, and turning a deaf ear to the counsels of a benevolent lady who came to visit.|mocking at the chaplain, and turning a deaf ear to the counsels of a benevolent lady who came to visit. +LJ014-0155|Now she abused the jury, now called Manning a vagabond,|Now she abused the jury, now called Manning a vagabond, +LJ014-0156|and through all ate heartily at every meal, slept soundly at nights, and talked with cheerfulness on almost any subject.|and through all ate heartily at every meal, slept soundly at nights, and talked with cheerfulness on almost any subject. +LJ014-0157|Nevertheless, she attempted to commit suicide by driving her nails, purposely left long, into her throat.|Nevertheless, she attempted to commit suicide by driving her nails, purposely left long, into her throat. +LJ014-0158|She was discovered just as she was getting black in the face.|She was discovered just as she was getting black in the face. +LJ014-0159|Manning's demeanor was more in harmony with his situation, and the full confession he made|Manning's demeanor was more in harmony with his situation, and the full confession he made +LJ014-0160|elucidated all dark and uncertain points in connection with the crime.|elucidated all dark and uncertain points in connection with the crime. +LJ014-0161|The actual execution, which took place at another prison than Newgate, is rather beyond the scope of this work.|The actual execution, which took place at another prison than Newgate, is rather beyond the scope of this work. +LJ014-0162|But it may be mentioned that the concourse was so enormous that it drew down the well-merited and trenchant disapproval of Charles Dickens,|But it may be mentioned that the concourse was so enormous that it drew down the well-merited and trenchant disapproval of Charles Dickens, +LJ014-0163|who wrote to the 'Times,' saying that he believed "a sight so inconceivably awful|who wrote to the 'Times,' saying that he believed "a sight so inconceivably awful +LJ014-0164|as the wickedness and levity of the immense crowd collected at the execution this morning could be imagined by no man,|as the wickedness and levity of the immense crowd collected at the execution this morning could be imagined by no man, +LJ014-0165|and presented by no heathen land under the sun.|and presented by no heathen land under the sun. +LJ014-0166|faded in my mind before the atrocious bearing, looks, and language of the assembled spectators.|faded in my mind before the atrocious bearing, looks, and language of the assembled spectators. +LJ014-0167|When I came upon the scene at midnight, the shrillness of the cries and howls that were raised from time to time,|When I came upon the scene at midnight, the shrillness of the cries and howls that were raised from time to time, +LJ014-0168|denoting that they came from a concourse of boys and girls already assembled in the best places, made my blood run cold.|denoting that they came from a concourse of boys and girls already assembled in the best places, made my blood run cold. +LJ014-0169|It will be in the memory of many that Mrs. Manning appeared on the scaffold in a black satin dress, which was bound tightly round her waist.|It will be in the memory of many that Mrs. Manning appeared on the scaffold in a black satin dress, which was bound tightly round her waist. +LJ014-0170|This preference brought the costly stuff into disrepute, and its unpopularity lasted for nearly thirty years.|This preference brought the costly stuff into disrepute, and its unpopularity lasted for nearly thirty years. +LJ014-0171|I will briefly describe one or two of the more remarkable murders in the years immediately following, then pass on to another branch of crime.|I will briefly describe one or two of the more remarkable murders in the years immediately following, then pass on to another branch of crime. +LJ014-0172|Robert Marley at the time of his arrest called himself a surgical instrument maker.|Robert Marley at the time of his arrest called himself a surgical instrument maker. +LJ014-0173|It was understood also that he had served in the army as a private, and had, moreover, undergone a sentence of transportation.|It was understood also that he had served in the army as a private, and had, moreover, undergone a sentence of transportation. +LJ014-0174|But it was supposed that he had been once in a good position, well born, and well educated.|But it was supposed that he had been once in a good position, well born, and well educated. +LJ014-0175|When lying under sentence of death in Newgate, he was visited by a lady, a gentlewoman in every sense of the word, who was said to be his sister.|When lying under sentence of death in Newgate, he was visited by a lady, a gentlewoman in every sense of the word, who was said to be his sister. +LJ014-0176|His determined addiction to evil courses had led to his being cast off by his family,|His determined addiction to evil courses had led to his being cast off by his family, +LJ014-0177|and he must have been at the end of his resources when he committed the crime for which he suffered.|and he must have been at the end of his resources when he committed the crime for which he suffered. +LJ014-0178|His offense was the murder of Richard Cope,|His offense was the murder of Richard Cope, +LJ014-0179|a working jeweler, shopman to a Mr. Berry of Parliament Street. It was Cope's duty to stay in the shop till the last, close the shutters,|a working jeweler, shopman to a Mr. Berry of Parliament Street. It was Cope's duty to stay in the shop till the last, close the shutters, +LJ014-0180|secure the stock of watches and jewelry, then lock up the place and take on the keys to Mr. Berry's private house in Pimlico.|secure the stock of watches and jewelry, then lock up the place and take on the keys to Mr. Berry's private house in Pimlico. +LJ014-0181|Cope, a small man, crippled, and of weakly constitution, was alone in the shop about 9:30.|Cope, a small man, crippled, and of weakly constitution, was alone in the shop about nine:thirty. +LJ014-0182|the shutters were up, and he was preparing to close, when Marley entered and fell upon him with a life-preserver,|the shutters were up, and he was preparing to close, when Marley entered and fell upon him with a life-preserver, +LJ014-0183|meaning to kill him and rifle the shop.|meaning to kill him and rifle the shop. +LJ014-0184|The noise of the struggle was heard outside in the street,|The noise of the struggle was heard outside in the street, +LJ014-0185|and bystanders peeped in through the shutters, but no one entered or sought to interfere in what seemed only a domestic quarrel.|and bystanders peeped in through the shutters, but no one entered or sought to interfere in what seemed only a domestic quarrel. +LJ014-0186|A milliner's porter,|A milliner's porter, +LJ014-0187|Lerigo, was also attracted by the noise of the row, but after walking a few paces he felt dissatisfied, and returned to the spot.|Lerigo, was also attracted by the noise of the row, but after walking a few paces he felt dissatisfied, and returned to the spot. +LJ014-0188|Pushing the shop-door open, he saw Marley finishing his murderous assault.|Pushing the shop-door open, he saw Marley finishing his murderous assault. +LJ014-0189|Lerigo turned for assistance to take the man into custody.|Lerigo turned for assistance to take the man into custody. +LJ014-0190|Marley, disturbed, picked up a cigar and parcel from the counter, then ran out, pursued by Lerigo only.|Marley, disturbed, picked up a cigar and parcel from the counter, then ran out, pursued by Lerigo only. +LJ014-0191|Marley ran along the street, down into Cannon Row|Marley ran along the street, down into Cannon Row +LJ014-0192|then into Palace Yard, where the waterman of the cab-tank, in obedience to Lerigo's shouts, collared the fugitive.|then into Palace Yard, where the waterman of the cab-tank, in obedience to Lerigo's shouts, collared the fugitive. +LJ014-0193|Escorted by his two captors, Marley was taken back into Parliament Street to the jeweler's shop.|Escorted by his two captors, Marley was taken back into Parliament Street to the jeweler's shop. +LJ014-0194|The policemen were now in possession;|The policemen were now in possession; +LJ014-0195|two of them supported Cope, who was still alive, although insensible, and Marley was apprehended. The evidence against him was completed|two of them supported Cope, who was still alive, although insensible, and Marley was apprehended. The evidence against him was completed +LJ014-0196|by his identification by Cope in Westminster Hospital, who survived long enough to make a formal deposition before Mr. Jardine,|by his identification by Cope in Westminster Hospital, who survived long enough to make a formal deposition before Mr. Jardine, +LJ014-0197|the police magistrate, that Marley was the man who had beaten him to death.|the police magistrate, that Marley was the man who had beaten him to death. +LJ014-0198|Marley at his trial was undefended, and the sheriffs offered him counsel; but he declined. The witnesses against him all spoke the truth, he said;|Marley at his trial was undefended, and the sheriffs offered him counsel; but he declined. The witnesses against him all spoke the truth, he said; +LJ014-0199|there was no case to make out; why waste money on lawyers for the defense? His demeanor was cool and collected throughout;|there was no case to make out; why waste money on lawyers for the defense? His demeanor was cool and collected throughout; +LJ014-0200|he seemed while in Newgate to realize thoroughly that there was no hope for him, and was determined to face his fate bravely.|he seemed while in Newgate to realize thoroughly that there was no hope for him, and was determined to face his fate bravely. +LJ014-0201|After sentence, the Newgate officers who had special charge of him noticed that he slept well and ate well, enjoying all his meals.|After sentence, the Newgate officers who had special charge of him noticed that he slept well and ate well, enjoying all his meals. +LJ014-0202|One of them went into his cell just at dinner-time;|One of them went into his cell just at dinner-time; +LJ014-0203|the great clock of St. Sepulchre's close by was striking the hour, and Marley, who had his elbows on the table,|the great clock of St. Sepulchre's close by was striking the hour, and Marley, who had his elbows on the table, +LJ014-0204|with his head resting on his hands, looked up and observed calmly, "Go along, clock; come along, gallows."|with his head resting on his hands, looked up and observed calmly, "Go along, clock; come along, gallows." +LJ014-0205|On the dread morning he came out to execution quite gaily, and tripped up the stairs to the scaffold.|On the dread morning he came out to execution quite gaily, and tripped up the stairs to the scaffold. +LJ014-0206|His captors, it may be added (Lerigo and Allen), were warmly commended by the judge for their courage and activity.|His captors, it may be added (Lerigo and Allen), were warmly commended by the judge for their courage and activity. +LJ014-0207|The former was given a reward of twenty and the latter of ten pounds.|The former was given a reward of twenty and the latter of ten pounds. +LJ014-0208|A murderous assault on a police constable, which so nearly ended fatally that the culprit was sentenced to death, although not executed,|A murderous assault on a police constable, which so nearly ended fatally that the culprit was sentenced to death, although not executed, +LJ014-0209|was perpetrated in 1852. The case was accompanied with the most shocking brutality.|was perpetrated in eighteen fifty-two. The case was accompanied with the most shocking brutality. +LJ014-0210|Cannon, by trade a chimney-sweep, had long been characterized by the bitterest hatred of the police force,|Cannon, by trade a chimney-sweep, had long been characterized by the bitterest hatred of the police force, +LJ014-0211|and had been repeatedly sentenced to imprisonment for most desperate and ferocious attacks upon various constables. His last victim was Dwyer,|and had been repeatedly sentenced to imprisonment for most desperate and ferocious attacks upon various constables. His last victim was Dwyer, +LJ014-0212|a fine young officer who had been summoned to take Cannon into custody when the latter was drunk and riotous in front of a public-house.|a fine young officer who had been summoned to take Cannon into custody when the latter was drunk and riotous in front of a public-house. +LJ014-0213|Dwyer found Cannon bleeding profusely from a wound in the head, and persuaded him to go to a doctor's.|Dwyer found Cannon bleeding profusely from a wound in the head, and persuaded him to go to a doctor's. +LJ014-0214|They walked together quietly for some little distance, then Cannon, without the slightest warning,|They walked together quietly for some little distance, then Cannon, without the slightest warning, +LJ014-0215|threw the constable on his back, and violently assaulted him by jumping on his chest and stomach,|threw the constable on his back, and violently assaulted him by jumping on his chest and stomach, +LJ014-0216|and by getting his hand inside Dwyer's stock, with the idea of strangling him.|and by getting his hand inside Dwyer's stock, with the idea of strangling him. +LJ014-0217|Dwyer managed to overpower his assailant, and got to his feet; but Cannon butted at him with his head, and again threw him to the ground,|Dwyer managed to overpower his assailant, and got to his feet; but Cannon butted at him with his head, and again threw him to the ground, +LJ014-0218|after which he kicked his prostrate foe in the most brutal and cowardly manner, and until he was almost senseless, and bruised from head to foot.|after which he kicked his prostrate foe in the most brutal and cowardly manner, and until he was almost senseless, and bruised from head to foot. +LJ014-0219|Once more Dwyer got to his feet, and managed, by drawing his staff, to keep Cannon at bay until a second constable came to his aid.|Once more Dwyer got to his feet, and managed, by drawing his staff, to keep Cannon at bay until a second constable came to his aid. +LJ014-0220|All this time not one of a numerous body of bystanders offered to assist the policeman in his extremity.|All this time not one of a numerous body of bystanders offered to assist the policeman in his extremity. +LJ014-0221|On the contrary, many of them encouraged the brutal assailant in his savage attack.|On the contrary, many of them encouraged the brutal assailant in his savage attack. +LJ014-0222|To Cannon's infinite surprise, he was indicted for attempt to murder, and not for a simple assault, and found guilty.|To Cannon's infinite surprise, he was indicted for attempt to murder, and not for a simple assault, and found guilty. +LJ014-0223|The judge, in passing sentence of death, told him he richly deserved the punishment.|The judge, in passing sentence of death, told him he richly deserved the punishment. +LJ014-0224|As Dwyer survived, Cannon escaped the death sentence, which was commuted to penal servitude for life.|As Dwyer survived, Cannon escaped the death sentence, which was commuted to penal servitude for life. +LJ014-0225|A handsome sum was subscribed for the injured constable, who was disabled for life.|A handsome sum was subscribed for the injured constable, who was disabled for life. +LJ014-0226|Only a few have vied with Cannon in fiendish cruelty and brutality.|Only a few have vied with Cannon in fiendish cruelty and brutality. +LJ014-0227|One of these was Mobbs, who lived in the Minories,|One of these was Mobbs, who lived in the Minories, +LJ014-0228|generally known by the soubriquet of "General Haynau," a name execrated in England about this time.|generally known by the soubriquet of "General Haynau," a name execrated in England about this time. +LJ014-0229|Mobbs systematically ill-used his wife for a long space of time, and at last cut her throat.|Mobbs systematically ill-used his wife for a long space of time, and at last cut her throat. +LJ014-0230|For this he was executed in front of Newgate in 1833.|For this he was executed in front of Newgate in eighteen thirty-three. +LJ014-0231|Emmanuel Barthelemy again,|Emmanuel Barthelemy again, +LJ014-0232|the French refugee, was a murderer of the same description, who dispatched his victim with a loaded cane, after which, to secure his escape,|the French refugee, was a murderer of the same description, who dispatched his victim with a loaded cane, after which, to secure his escape, +LJ014-0233|he shot an old soldier who had attempted to detain him. He was convicted and executed.|he shot an old soldier who had attempted to detain him. He was convicted and executed. +LJ014-0234|He died impenitent, declaring that he had no belief, and that it was idle to ask forgiveness of God.|He died impenitent, declaring that he had no belief, and that it was idle to ask forgiveness of God. +LJ014-0235|I want forgiveness of man; I want those doors (of the prison) opened.|I want forgiveness of man; I want those doors (of the prison) opened. +LJ014-0236|Barthelemy was generally supposed to have been a secret agent of the French police.|Barthelemy was generally supposed to have been a secret agent of the French police. +LJ014-0237|I will now pass to grave but less atrocious crimes.|I will now pass to grave but less atrocious crimes. +LJ014-0238|In 1850 occurred the first of a series of gigantic frauds,|In eighteen fifty occurred the first of a series of gigantic frauds, +LJ014-0239|which followed each other at no long intervals, which had a strong family likeness, and originated all of them to make money easily,|which followed each other at no long intervals, which had a strong family likeness, and originated all of them to make money easily, +LJ014-0240|without capital, and at railroad speed.|without capital, and at railroad speed. +LJ014-0241|Walter Watts was an inventor, a creator, who struck an entirely new and original line of crime.|Walter Watts was an inventor, a creator, who struck an entirely new and original line of crime. +LJ014-0242|Employed as a clerk in the Globe Assurance,|Employed as a clerk in the Globe Assurance, +LJ014-0243|he with unusual quickness of apprehension discovered and promptly turned to account an inexcusably lax system of management,|he with unusual quickness of apprehension discovered and promptly turned to account an inexcusably lax system of management, +LJ014-0244|which offered peculiar chances of profit to an ingenious and unscrupulous man.|which offered peculiar chances of profit to an ingenious and unscrupulous man. +LJ014-0245|It was the custom in this office to make the banker's passbook the basis of the entries in the company's ledgers.|It was the custom in this office to make the banker's passbook the basis of the entries in the company's ledgers. +LJ014-0246|Thus, when a payment was made by the company, the amount disbursed was carried to account in the general books from its entry in the passbook,|Thus, when a payment was made by the company, the amount disbursed was carried to account in the general books from its entry in the passbook, +LJ014-0247|and without reference to or comparison with the documents in which the payment was claimed.|and without reference to or comparison with the documents in which the payment was claimed. +LJ014-0248|This passbook, when not at the bank, was in the exclusive custody of Watts.|This passbook, when not at the bank, was in the exclusive custody of Watts. +LJ014-0249|The cheques drawn by the directors also passed through his hands; to him too they came back to be verified and put by,|The cheques drawn by the directors also passed through his hands; to him too they came back to be verified and put by, +LJ014-0250|after they had been cashed by the bank.|after they had been cashed by the bank. +LJ014-0251|In this way Watts had complete control over the whole of the monetary transactions of the company.|In this way Watts had complete control over the whole of the monetary transactions of the company. +LJ014-0252|He could do what he liked with the passbook, and by its adoption, as described as the basis of all entries,|He could do what he liked with the passbook, and by its adoption, as described as the basis of all entries, +LJ014-0253|there was no independent check upon him if he chose to tamper with it.|there was no independent check upon him if he chose to tamper with it. +LJ014-0254|This he did to an enormous extent,|This he did to an enormous extent, +LJ014-0255|continually altering, erasing, and adding figures to correspond with and cover the abstractions he made of various cheques as they were drawn.|continually altering, erasing, and adding figures to correspond with and cover the abstractions he made of various cheques as they were drawn. +LJ014-0256|It seems incredible that this passbook, which when produced in court|It seems incredible that this passbook, which when produced in court +LJ014-0257|was a mass of blots and erasures, should not have created suspicion of foul play either at the bank or at the company's board.|was a mass of blots and erasures, should not have created suspicion of foul play either at the bank or at the company's board. +LJ014-0258|Implicit confidence appears to have been placed in Watts, who was the son of an old and trusted employee, and, moreover, a young man of plausible address.|Implicit confidence appears to have been placed in Watts, who was the son of an old and trusted employee, and, moreover, a young man of plausible address. +LJ014-0259|Watts led two lives.|Watts led two lives. +LJ014-0260|In the West End he was a man of fashion, with a town house, a house at Brighton, and a cellar full of good wine at both.|In the West End he was a man of fashion, with a town house, a house at Brighton, and a cellar full of good wine at both. +LJ014-0261|He rode a priceless hack in Rotten Row, or drove down to Richmond in a mail phaeton and pair.|He rode a priceless hack in Rotten Row, or drove down to Richmond in a mail phaeton and pair. +LJ014-0262|He played high, and spent his nights at the club, or in joyous and dissolute company.|He played high, and spent his nights at the club, or in joyous and dissolute company. +LJ014-0264|Under his auspices several "stars" appeared on the boards of the Marylebone theatre,|Under his auspices several "stars" appeared on the boards of the Marylebone theatre, +LJ014-0265|and later he became manager of the newly rebuilt Olympic at Wych Street.|and later he became manager of the newly rebuilt Olympic at Wych Street. +LJ014-0266|No one cared too closely to inquire into the sources of wealth. Some said he was a fortunate speculator in stocks,|No one cared too closely to inquire into the sources of wealth. Some said he was a fortunate speculator in stocks, +LJ014-0267|others that he had had extraordinary luck as a gold-digger. Had his West End and little-informed associates followed him into the city,|others that he had had extraordinary luck as a gold-digger. Had his West End and little-informed associates followed him into the city, +LJ014-0268|whither he was taken every morning in a smart brougham, they would have seen him alight from it in Cornhill,|whither he was taken every morning in a smart brougham, they would have seen him alight from it in Cornhill, +LJ014-0269|and walk forward on foot to enter as a humble and unpretending employee the doors of the Globe Assurance office.|and walk forward on foot to enter as a humble and unpretending employee the doors of the Globe Assurance office. +LJ014-0271|Nevertheless, in this position, through the culpable carelessness which left him unfettered, he managed between 1844|Nevertheless, in this position, through the culpable carelessness which left him unfettered, he managed between eighteen forty-four +LJ014-0273|The detection of these frauds came while he was still prominently before the world as the lessee of the Olympic.|The detection of these frauds came while he was still prominently before the world as the lessee of the Olympic. +LJ014-0274|Rumors were abroad that serious defalcations had been discovered in one of the insurance offices,|Rumors were abroad that serious defalcations had been discovered in one of the insurance offices, +LJ014-0275|but it was long before the public realized that the fraudulent clerk and the great theatrical manager were one and the same person.|but it was long before the public realized that the fraudulent clerk and the great theatrical manager were one and the same person. +LJ014-0276|Watts's crime was discovered by the secretary of the Globe Company, who came suddenly upon the extensive falsification of the passbook.|Watts's crime was discovered by the secretary of the Globe Company, who came suddenly upon the extensive falsification of the passbook. +LJ014-0277|An inquiry was at once set on foot, and the frauds were traced to Watts.|An inquiry was at once set on foot, and the frauds were traced to Watts. +LJ014-0278|The latter, when first taxed with his offense, protested his innocence boldly, and positively denied all knowledge of the affair;|The latter, when first taxed with his offense, protested his innocence boldly, and positively denied all knowledge of the affair; +LJ014-0279|and he had so cleverly destroyed all traces that it was not easy to bring home the charge.|and he had so cleverly destroyed all traces that it was not easy to bring home the charge. +LJ014-0280|But it was proved that Watts had appropriated one cheque for £1400,|But it was proved that Watts had appropriated one cheque for fourteen hundred pounds, +LJ014-0281|which he had paid into his own bankers, and on this he was committed to Newgate for trial.|which he had paid into his own bankers, and on this he was committed to Newgate for trial. +LJ014-0282|There were two counts in the indictment: one for stealing a cheque value £1400, the second for stealing a bit of paper value one penny.|There were two counts in the indictment: one for stealing a cheque value fourteen hundred pounds, the second for stealing a bit of paper value one penny. +LJ014-0283|The jury found him guilty of the latter only, with a point of law reserved. This was fully argued before three judges,|The jury found him guilty of the latter only, with a point of law reserved. This was fully argued before three judges, +LJ014-0285|and not for the slight offense as it appeared on the record.|and not for the slight offense as it appeared on the record. +LJ014-0286|The sentence of the court, one of ten years' transportation, struck the prisoner with dismay.|The sentence of the court, one of ten years' transportation, struck the prisoner with dismay. +LJ014-0287|He had been led to suppose that twelve months' imprisonment was the utmost the law could inflict, and he broke down utterly under the unexpected blow.|He had been led to suppose that twelve months' imprisonment was the utmost the law could inflict, and he broke down utterly under the unexpected blow. +LJ014-0288|That same evening he committed suicide in Newgate.|That same evening he committed suicide in Newgate. +LJ014-0289|The details of the suicide were given at the inquest. Watts had been in ill-health from the time of his first arrest.|The details of the suicide were given at the inquest. Watts had been in ill-health from the time of his first arrest. +LJ014-0290|In Giltspur Street Compter, where he was first lodged|In Giltspur Street Compter, where he was first lodged +LJ014-0291|he showed symptoms of delirium tremens, and admitted that he had been addicted to the excessive use of stimulants.|he showed symptoms of delirium tremens, and admitted that he had been addicted to the excessive use of stimulants. +LJ014-0292|His health improved, but was still indifferent when he was brought up for sentence, and he was an occupant of the Newgate infirmary.|His health improved, but was still indifferent when he was brought up for sentence, and he was an occupant of the Newgate infirmary. +LJ014-0293|He returned from court in a state of gloomy dejection,|He returned from court in a state of gloomy dejection, +LJ014-0294|and in the middle of the night one of the fellow-prisoners who slept in the same ward noticed that he was not in his bed.|and in the middle of the night one of the fellow-prisoners who slept in the same ward noticed that he was not in his bed. +LJ014-0295|This man got up to look for him, and found him hanging from the bars of a neighboring room.|This man got up to look for him, and found him hanging from the bars of a neighboring room. +LJ014-0296|He had made use of a piece of rope cut out from the sacking of his bedstead, and had tied his feet together with a silk pocket-handkerchief.|He had made use of a piece of rope cut out from the sacking of his bedstead, and had tied his feet together with a silk pocket-handkerchief. +LJ014-0297|The prison officers were called, but Watts was quite cold and stiff when he was cut down.|The prison officers were called, but Watts was quite cold and stiff when he was cut down. +LJ014-0298|Strange to say, a second suicide occurred in Newgate the same night,|Strange to say, a second suicide occurred in Newgate the same night, +LJ014-0299|that of a prize-fighter named Donovan, tried the same day, and convicted of manslaughter.|that of a prize-fighter named Donovan, tried the same day, and convicted of manslaughter. +LJ014-0300|Sentence of death had been recorded against Donovan, who, like Watts, had seemingly been overcome with sudden despair.|Sentence of death had been recorded against Donovan, who, like Watts, had seemingly been overcome with sudden despair. +LJ014-0301|In 1853 a second case of gigantic fraud alarmed and scandalized the financial world.|In eighteen fifty-three a second case of gigantic fraud alarmed and scandalized the financial world. +LJ014-0302|It outshone the defalcations of Watts.|It outshone the defalcations of Watts. +LJ014-0303|Nothing to equal the excitement caused by the forgeries of Robert Ferdinand Pries had been known before in the city of London.|Nothing to equal the excitement caused by the forgeries of Robert Ferdinand Pries had been known before in the city of London. +LJ014-0304|He was a corn merchant who operated largely in grain.|He was a corn merchant who operated largely in grain. +LJ014-0305|So enormous were his transactions, that they often affected the markets, and caused great fluctuations in prices.|So enormous were his transactions, that they often affected the markets, and caused great fluctuations in prices. +LJ014-0306|These had been attributed to political action; some thought that the large purchases in foreign grains, effected at losing prices,|These had been attributed to political action; some thought that the large purchases in foreign grains, effected at losing prices, +LJ014-0307|were intended by the protectionists to depress the wheat market, and secure the support of the farmers at the forthcoming election;|were intended by the protectionists to depress the wheat market, and secure the support of the farmers at the forthcoming election; +LJ014-0308|others, that Napoleon III., but recently proclaimed Emperor of the French, wished to gain the popularity necessary to secure the people.|others, that Napoleon the third, but recently proclaimed Emperor of the French, wished to gain the popularity necessary to secure the people. +LJ014-0309|Few realized that these mysterious operations were the "convulsive attempt" of a ruined and dishonest speculator to sustain his credit.|Few realized that these mysterious operations were the "convulsive attempt" of a ruined and dishonest speculator to sustain his credit. +LJ014-0310|Pries, although enjoying a high reputation in the city, had long been in a bad way.|Pries, although enjoying a high reputation in the city, had long been in a bad way. +LJ014-0312|His method was to obtain advances twice over on the same bills of lading or corn warrants. The duplicates were forged.|His method was to obtain advances twice over on the same bills of lading or corn warrants. The duplicates were forged. +LJ014-0313|In this way he obtained vast sums from several firms, and one to which he was indebted upwards of £50,000 subsequently stopped payment.|In this way he obtained vast sums from several firms, and one to which he was indebted upwards of fifty thousand pounds subsequently stopped payment. +LJ014-0314|Pries at length was discovered|Pries at length was discovered +LJ014-0315|through a dishonored cheque for £3000, paid over as an installment of £18,000 owing for an advance on warrants.|through a dishonored cheque for three thousand pounds, paid over as an installment of eighteen thousand pounds owing for an advance on warrants. +LJ014-0316|Inquiries were instituted when the cheque was protested, which led to the discovery of the forgeries.|Inquiries were instituted when the cheque was protested, which led to the discovery of the forgeries. +LJ014-0317|Pries was lodged in Newgate, tried at the Old Bailey, and transported for life.|Pries was lodged in Newgate, tried at the Old Bailey, and transported for life. +LJ014-0318|Another set of frauds, which resembled those of Pries in principle, although not in practice, were soon afterwards discovered.|Another set of frauds, which resembled those of Pries in principle, although not in practice, were soon afterwards discovered. +LJ014-0320|proposed to gain the capital he needed for business purposes by raising money on dock warrants for imported goods which had no real existence.|proposed to gain the capital he needed for business purposes by raising money on dock warrants for imported goods which had no real existence. +LJ014-0321|When such goods arrived they were frequently left at a wharf, paying rent until it suited the importer to remove them.|When such goods arrived they were frequently left at a wharf, paying rent until it suited the importer to remove them. +LJ014-0322|The dock warrant was issued by the wharfinger as certificate that he held the goods.|The dock warrant was issued by the wharfinger as certificate that he held the goods. +LJ014-0323|The warrant thus represented money, and was often used as such, being endorsed and passed from hand to hand as other negotiable bills.|The warrant thus represented money, and was often used as such, being endorsed and passed from hand to hand as other negotiable bills. +LJ014-0324|Cole's plan was to have a wharf of his own, nominally occupied by a creature trading as Maltby and Co.|Cole's plan was to have a wharf of his own, nominally occupied by a creature trading as Maltby and Co. +LJ014-0325|Goods would be landed at this wharf;|Goods would be landed at this wharf; +LJ014-0326|Maltby and Co. would issue warrants on them deliverable to the importer, and the goods were then passed to be stored in neighboring warehouses.|Maltby and Co. would issue warrants on them deliverable to the importer, and the goods were then passed to be stored in neighboring warehouses. +LJ014-0327|The owners of the latter would then issue a second set of warrants on these goods, in total ignorance of the fact that they were already pledged.|The owners of the latter would then issue a second set of warrants on these goods, in total ignorance of the fact that they were already pledged. +LJ014-0328|Cole quickly raised money on both sets of warrants. He carried on this game for some time with great success,|Cole quickly raised money on both sets of warrants. He carried on this game for some time with great success, +LJ014-0329|and so developed his business that in one year his transactions amounted to a couple of millions of pounds.|and so developed his business that in one year his transactions amounted to a couple of millions of pounds. +LJ014-0330|He had several narrow escapes.|He had several narrow escapes. +LJ014-0331|Once a warrant-holder sent down a clerk to view certain goods, and the clerk found that these goods had already a "stop" upon them, or were pledged.|Once a warrant-holder sent down a clerk to view certain goods, and the clerk found that these goods had already a "stop" upon them, or were pledged. +LJ014-0332|Cole escaped by throwing the blame on a careless partner, and at once removed the "stop."|Cole escaped by throwing the blame on a careless partner, and at once removed the "stop." +LJ014-0333|Again, some of the duplicate and fictitious warrants were held by a firm which suspended payment, and there was no knowing into whose hands they might fall.|Again, some of the duplicate and fictitious warrants were held by a firm which suspended payment, and there was no knowing into whose hands they might fall. +LJ014-0334|Cole found out where they were, and redeemed them at a heavy outlay, thus obtaining business relations with the firm that held them,|Cole found out where they were, and redeemed them at a heavy outlay, thus obtaining business relations with the firm that held them, +LJ014-0335|which were soon developed, much to that firm's subsequent anger and regret.|which were soon developed, much to that firm's subsequent anger and regret. +LJ014-0336|Last of all, the well-known bankers Overend and Gurney, whose own affairs created much excitement some years later,|Last of all, the well-known bankers Overend and Gurney, whose own affairs created much excitement some years later, +LJ014-0337|wishing to verify the value of warrants they held, and sending to Maltby and Co.'s wharf, found out half the truth.|wishing to verify the value of warrants they held, and sending to Maltby and Co.'s wharf, found out half the truth. +LJ014-0338|These bankers, wishing for more specific information,|These bankers, wishing for more specific information, +LJ014-0339|asked Davidson and Gordon, a firm with which Cole was closely allied, whether the warrants meant goods or nothing.|asked Davidson and Gordon, a firm with which Cole was closely allied, whether the warrants meant goods or nothing. +LJ014-0340|They could not deny that the latter was the truth, and were forthwith stigmatized by Mr. Chapman, Overend and Gurney's representative, as rogues.|They could not deny that the latter was the truth, and were forthwith stigmatized by Mr. Chapman, Overend and Gurney's representative, as rogues. +LJ015-0001|Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 18: Newgate notorieties continued, part 3.|Chronicles of Newgate, Volume two. By Arthur Griffiths. Section eighteen: Newgate notorieties continued, part three. +LJ015-0002|The course of the swindlers was by no means smooth, but it was not till 1854 that suspicion arose that anything was wrong.|The course of the swindlers was by no means smooth, but it was not till eighteen fifty-four that suspicion arose that anything was wrong. +LJ015-0003|A firm which held a lot of warrants suddenly demanded the delivery of the goods they covered.|A firm which held a lot of warrants suddenly demanded the delivery of the goods they covered. +LJ015-0004|The goods having no existence, Cole of course could not deliver them.|The goods having no existence, Cole of course could not deliver them. +LJ015-0005|About this time Davidson and Gordon, the people above-mentioned,|About this time Davidson and Gordon, the people above-mentioned, +LJ015-0006|who had fraudulent warrants out of their own to the extent of £150,000, suspended payment and absconded.|who had fraudulent warrants out of their own to the extent of one hundred fifty thousand pounds, suspended payment and absconded. +LJ015-0007|This affected Cole's credit, and ugly reports were in circulation charging him with the issue of simulated warrants.|This affected Cole's credit, and ugly reports were in circulation charging him with the issue of simulated warrants. +LJ015-0008|These indeed were out to the value of £367,800.|These indeed were out to the value of three hundred sixty-seven thousand, eight hundred pounds. +LJ015-0009|Cole's difficulties increased more and more; warrant-holders came down upon him demanding to realize their goods.|Cole's difficulties increased more and more; warrant-holders came down upon him demanding to realize their goods. +LJ015-0010|Cole now suspended payment.|Cole now suspended payment. +LJ015-0011|Maltby, who had bolted, was pursued and arrested, to end his life miserably by committing suicide in a Newgate cell.|Maltby, who had bolted, was pursued and arrested, to end his life miserably by committing suicide in a Newgate cell. +LJ015-0012|Cole too was apprehended, and in due course tried at the Central Criminal Court.|Cole too was apprehended, and in due course tried at the Central Criminal Court. +LJ015-0013|He was found guilty, and sentenced to the seemingly inadequate punishment of four years' transportation.|He was found guilty, and sentenced to the seemingly inadequate punishment of four years' transportation. +LJ015-0014|Davidson and Gordon were also sentenced to imprisonment.|Davidson and Gordon were also sentenced to imprisonment. +LJ015-0015|A more distressing case stands next on the criminal records --|A more distressing case stands next on the criminal records -- +LJ015-0016|the failure and subsequent sentence of the bankers Messrs. Strahan, Paul, and Bates,|the failure and subsequent sentence of the bankers Messrs. Strahan, Paul, and Bates, +LJ015-0017|for the fraudulent disposal of securities lodged in their hands. This firm was one of the oldest banking establishments in the kingdom,|for the fraudulent disposal of securities lodged in their hands. This firm was one of the oldest banking establishments in the kingdom, +LJ015-0018|and dated back to the Commonwealth, when, under the title of Snow and Walton, it carried on business as pawnbrokers.|and dated back to the Commonwealth, when, under the title of Snow and Walton, it carried on business as pawnbrokers. +LJ015-0019|The Strahan of the firm which came to grief was a Snow who changed his name for a fortune of £200,000;|The Strahan of the firm which came to grief was a Snow who changed his name for a fortune of two hundred thousand pounds; +LJ015-0020|he was a man esteemed and respected in society and the world of finance, incapable as it was thought of a dishonest deed.|he was a man esteemed and respected in society and the world of finance, incapable as it was thought of a dishonest deed. +LJ015-0021|Sir John Dean Paul had inherited a baronetcy from his father, together with an honored name;|Sir John Dean Paul had inherited a baronetcy from his father, together with an honored name; +LJ015-0022|he was himself a prominent member of the Low Church, of austere piety, active in all good works.|he was himself a prominent member of the Low Church, of austere piety, active in all good works. +LJ015-0023|Mr. Bates had been confidential managing clerk, and was taken into the firm not alone as a reward for long and faithful service|Mr. Bates had been confidential managing clerk, and was taken into the firm not alone as a reward for long and faithful service +LJ015-0024|but that he might strengthen it by his long experience and known business capacity.|but that he might strengthen it by his long experience and known business capacity. +LJ015-0025|The bank enjoyed an excellent reputation, it had a good connection, and was supposed to be perfectly sound.|The bank enjoyed an excellent reputation, it had a good connection, and was supposed to be perfectly sound. +LJ015-0026|Moreover, the partners were sober, steady men, who paid unremitting attention to business.|Moreover, the partners were sober, steady men, who paid unremitting attention to business. +LJ015-0027|Yet even so early as the death of the first Sir John Paul,|Yet even so early as the death of the first Sir John Paul, +LJ015-0028|the bank was insolvent, and instead of starting on a fresh life with a new name, it should then and there have closed its doors.|the bank was insolvent, and instead of starting on a fresh life with a new name, it should then and there have closed its doors. +LJ015-0029|In December 1851 the balance sheet showed a deficiency of upwards of £70,000.|In December eighteen fifty-one the balance sheet showed a deficiency of upwards of seventy thousand pounds. +LJ015-0030|The bank had been conducted on false principles;|The bank had been conducted on false principles; +LJ015-0031|it had assumed enormous responsibilities -- on one side by the ownership of the Mostyn collieries, a valueless property,|it had assumed enormous responsibilities -- on one side by the ownership of the Mostyn collieries, a valueless property, +LJ015-0032|and on the other by backing up|and on the other by backing up +LJ015-0033|an impecunious and rotten firm of contractors with vast liabilities and pledged to impossible works abroad.|an impecunious and rotten firm of contractors with vast liabilities and pledged to impossible works abroad. +LJ015-0034|The engagements of the bank on these two heads amounting to nearly half a million of money,|The engagements of the bank on these two heads amounting to nearly half a million of money, +LJ015-0035|produced immediate embarrassment and financial distress.|produced immediate embarrassment and financial distress. +LJ015-0036|The bank was already insolvent,|The bank was already insolvent, +LJ015-0037|and the partners had to decide between suspending payment or continuing to hold its head above water by flagitious processes.|and the partners had to decide between suspending payment or continuing to hold its head above water by flagitious processes. +LJ015-0038|They chose, unhappily for themselves, the latter alternative.|They chose, unhappily for themselves, the latter alternative. +LJ015-0039|Money they must have, and money they raised to meet their urgent necessities upon the balances and securities deposited with them by their customers.|Money they must have, and money they raised to meet their urgent necessities upon the balances and securities deposited with them by their customers. +LJ015-0040|This borrowing continued, and on such a scale that their paper was soon at a discount,|This borrowing continued, and on such a scale that their paper was soon at a discount, +LJ015-0041|and the various discount houses would not advance sufficient sums to relieve the necessities of the bank.|and the various discount houses would not advance sufficient sums to relieve the necessities of the bank. +LJ015-0042|Then it was that instead of merely pledging securities, the bank sold them outright, and thus passed the Rubicon of fraud.|Then it was that instead of merely pledging securities, the bank sold them outright, and thus passed the Rubicon of fraud. +LJ015-0043|This went on for some time, and might never have been discovered had some good stroke of luck provided any of the partners|This went on for some time, and might never have been discovered had some good stroke of luck provided any of the partners +LJ015-0044|with money enough to retrieve the position of the bank. But that passed from bad to worse;|with money enough to retrieve the position of the bank. But that passed from bad to worse; +LJ015-0045|the firm's paper went down further and further in value; an application to the Committee of Bankers for assistance was peremptorily refused,|the firm's paper went down further and further in value; an application to the Committee of Bankers for assistance was peremptorily refused, +LJ015-0046|then came a run on the bank, and it was compelled to stop payment.|then came a run on the bank, and it was compelled to stop payment. +LJ015-0047|Its debts amounted to three-quarters of a million, and the dividend it eventually paid was three and twopence in the pound.|Its debts amounted to three-quarters of a million, and the dividend it eventually paid was three and twopence in the pound. +LJ015-0048|But worse than the bankruptcy was the confession made by the partners in the court.|But worse than the bankruptcy was the confession made by the partners in the court. +LJ015-0049|They admitted that they had made away with many of the securities entrusted to their keeping.|They admitted that they had made away with many of the securities entrusted to their keeping. +LJ015-0050|Following this, warrants were issued for their arrest,|Following this, warrants were issued for their arrest, +LJ015-0051|the specific charge being the unlawful negotiation of Danish bonds and other shares belonging to the Rev. Dr. Griffiths of Rochester|the specific charge being the unlawful negotiation of Danish bonds and other shares belonging to the Rev. Dr. Griffiths of Rochester +LJ015-0052|to the value of £20,000.|to the value of twenty thousand pounds. +LJ015-0053|Bates was at once captured in Norfolk Street, Strand.|Bates was at once captured in Norfolk Street, Strand. +LJ015-0054|Police officers went down at night to Nutfield, near Reigate, and arrested Sir John Paul, but allowed the prisoner to sleep there.|Police officers went down at night to Nutfield, near Reigate, and arrested Sir John Paul, but allowed the prisoner to sleep there. +LJ015-0055|Next morning they only just saved the train to town, and left Sir John behind on the platform, but he subsequently surrendered himself.|Next morning they only just saved the train to town, and left Sir John behind on the platform, but he subsequently surrendered himself. +LJ015-0056|Mr. Strahan was arrested at a friend's house in Bryanston Square.|Mr. Strahan was arrested at a friend's house in Bryanston Square. +LJ015-0057|All three were tried at the Central Criminal Court, and sentenced to fourteen years' transportation, passing some time in Newgate en route.|All three were tried at the Central Criminal Court, and sentenced to fourteen years' transportation, passing some time in Newgate en route. +LJ015-0058|Bates, the least guilty, was pardoned in 1858.|Bates, the least guilty, was pardoned in eighteen fifty-eight. +LJ015-0059|Two cases of extensive embezzlement which were discovered almost simultaneously, those of Robson and Redpath,|Two cases of extensive embezzlement which were discovered almost simultaneously, those of Robson and Redpath, +LJ015-0060|will long be remembered both within and without the commercial world.|will long be remembered both within and without the commercial world. +LJ015-0061|They both reproduced many of the features of the case of Watts, already described,|They both reproduced many of the features of the case of Watts, already described, +LJ015-0062|but in neither did the sums misappropriated reach quite the same high figure.|but in neither did the sums misappropriated reach quite the same high figure. +LJ015-0063|But neither Robson nor Redpath would have been able to pursue their fraudulent designs with success had they not, like Watts,|But neither Robson nor Redpath would have been able to pursue their fraudulent designs with success had they not, like Watts, +LJ015-0064|been afforded peculiar facilities by the slackness of system and the want of methodical administration in the concerns by which they were employed.|been afforded peculiar facilities by the slackness of system and the want of methodical administration in the concerns by which they were employed. +LJ015-0065|Robson was of humble origin, but he was well educated, and he had some literary abilities.|Robson was of humble origin, but he was well educated, and he had some literary abilities. +LJ015-0066|His proclivities were theatrical, and he was the author of several plays,|His proclivities were theatrical, and he was the author of several plays, +LJ015-0067|one at least of which, 'Love and Loyalty,' with Wallack in a leading part, achieved a certain success.|one at least of which, 'Love and Loyalty,' with Wallack in a leading part, achieved a certain success. +LJ015-0068|He began life as a law-writer, earning thereby some fifteen or eighteen shillings a week;|He began life as a law-writer, earning thereby some fifteen or eighteen shillings a week; +LJ015-0069|but the firm he served got him a situation as clerk in the office of the Great Northern Railway,|but the firm he served got him a situation as clerk in the office of the Great Northern Railway, +LJ015-0070|whence he passed to a better position under the Crystal Palace Company.|whence he passed to a better position under the Crystal Palace Company. +LJ015-0071|He now married, although his salary was only a pound a week; but he soon got on.|He now married, although his salary was only a pound a week; but he soon got on. +LJ015-0072|He had a pleasant address, showed good business aptitudes, and quickly acquired the approval of his superiors.|He had a pleasant address, showed good business aptitudes, and quickly acquired the approval of his superiors. +LJ015-0073|Within a year he was advanced to the post of chief clerk in the transfer department, at a salary of £150 a year.|Within a year he was advanced to the post of chief clerk in the transfer department, at a salary of one hundred fifty pounds a year. +LJ015-0074|His immediate chief was a Mr. Fasson, upon whose confidence he gained so rapidly, through his activity, industry, and engaging manners,|His immediate chief was a Mr. Fasson, upon whose confidence he gained so rapidly, through his activity, industry, and engaging manners, +LJ015-0075|that ere long the whole management of the transfer department was entrusted to him.|that ere long the whole management of the transfer department was entrusted to him. +LJ015-0076|Some time elapsed before Robson succumbed to temptation.|Some time elapsed before Robson succumbed to temptation. +LJ015-0077|He was not the first man of loose morality and expensive tastes|He was not the first man of loose morality and expensive tastes +LJ015-0078|who preferred to risk his future reputation and liberty to the present discomfort of living upon narrow means.|who preferred to risk his future reputation and liberty to the present discomfort of living upon narrow means. +LJ015-0079|The temptation was all the greater because the chances of successful fraud lay ready to hand.|The temptation was all the greater because the chances of successful fraud lay ready to hand. +LJ015-0080|Shares in the company were represented by certificates, which often enough never left the company's, or more exactly Robson's, hands.|Shares in the company were represented by certificates, which often enough never left the company's, or more exactly Robson's, hands. +LJ015-0081|He conceived the idea of transferring shares, bogus shares from a person who held none, to any one who would buy them in the open market.|He conceived the idea of transferring shares, bogus shares from a person who held none, to any one who would buy them in the open market. +LJ015-0082|He took it for granted that the certificates representing these bogus shares, and which practically did not exist, would never be called for.|He took it for granted that the certificates representing these bogus shares, and which practically did not exist, would never be called for. +LJ015-0083|This ingenious method of raising funds he adopted and carried on without detection,|This ingenious method of raising funds he adopted and carried on without detection, +LJ015-0084|till the defalcations from fraudulent transfers and fraudulent issues combined amounted to £27,000.|till the defalcations from fraudulent transfers and fraudulent issues combined amounted to twenty-seven thousand pounds. +LJ015-0085|With the proceeds of these flagitious frauds Robson feasted and made merry.|With the proceeds of these flagitious frauds Robson feasted and made merry. +LJ015-0086|He kept open house at Kilburn Priory;|He kept open house at Kilburn Priory; +LJ015-0087|entertained literary, artistic, and dramatic celebrities; had a smart "turn out," attended all the race-meetings, and dressed in the latest fashion.|entertained literary, artistic, and dramatic celebrities; had a smart "turn out," attended all the race-meetings, and dressed in the latest fashion. +LJ015-0088|To his wife, poor soul, he made no pretense of fidelity, and she enjoyed only so much of his company as was necessarily spent|To his wife, poor soul, he made no pretense of fidelity, and she enjoyed only so much of his company as was necessarily spent +LJ015-0089|in receiving guests at home, or could be spared from two rival establishments in other parts of the town.|in receiving guests at home, or could be spared from two rival establishments in other parts of the town. +LJ015-0090|To account for his revenues he pretended to have been very lucky on the Stock Exchange, which was at one time true to a limited extent,|To account for his revenues he pretended to have been very lucky on the Stock Exchange, which was at one time true to a limited extent, +LJ015-0091|and to have succeeded in other speculations.|and to have succeeded in other speculations. +LJ015-0092|When his friends asked why he, a wealthy man of independent means, continued to slave on as a clerk on a pittance,|When his friends asked why he, a wealthy man of independent means, continued to slave on as a clerk on a pittance, +LJ015-0093|he replied gaily that his regular work at the Crystal Palace office was useful as a sort of discipline, and kept him steady.|he replied gaily that his regular work at the Crystal Palace office was useful as a sort of discipline, and kept him steady. +LJ015-0094|All this time his position was one of extreme insecurity. He was standing over a mine which at any moment might explode.|All this time his position was one of extreme insecurity. He was standing over a mine which at any moment might explode. +LJ015-0095|The blow fell suddenly, and when least expected. One morning Mr. Fasson asked casually for certain certificates,|The blow fell suddenly, and when least expected. One morning Mr. Fasson asked casually for certain certificates, +LJ015-0096|whether representing real or fictitious shares does not appear; but they were certificates connected in some way with Robson's long practiced frauds|whether representing real or fictitious shares does not appear; but they were certificates connected in some way with Robson's long practiced frauds +LJ015-0097|and he could not produce them. His chief asked sternly where they were.|and he could not produce them. His chief asked sternly where they were. +LJ015-0098|Robson said they were at Kilburn Priory.|Robson said they were at Kilburn Priory. +LJ015-0099|"Let us go to Kilburn for them together," said Mr. Fasson, growing suspicious.|"Let us go to Kilburn for them together," said Mr. Fasson, growing suspicious. +LJ015-0100|They drove there, and Robson on arrival did the honors of his house, rang for lunch to gain time,|They drove there, and Robson on arrival did the honors of his house, rang for lunch to gain time, +LJ015-0101|but at Mr. Fasson's pressing demands went upstairs to fetch the certificates.|but at Mr. Fasson's pressing demands went upstairs to fetch the certificates. +LJ015-0102|He came back to explain that he had mislaid them.|He came back to explain that he had mislaid them. +LJ015-0103|Mr. Fasson, more and more ill at ease, would not accept this subterfuge, and declared they must be found.|Mr. Fasson, more and more ill at ease, would not accept this subterfuge, and declared they must be found. +LJ015-0104|Robson again left him, but only to gather together hastily all the money and valuables on which he could lay his hands, with which he left the house.|Robson again left him, but only to gather together hastily all the money and valuables on which he could lay his hands, with which he left the house. +LJ015-0105|Mr. Fasson waited and waited for his subordinate to re-appear, and at last discovered his flight.|Mr. Fasson waited and waited for his subordinate to re-appear, and at last discovered his flight. +LJ015-0106|A reward was forthwith offered for Robson's apprehension.|A reward was forthwith offered for Robson's apprehension. +LJ015-0107|Meanwhile the absconding clerk had coolly driven to a favorite dining-place in the West End,|Meanwhile the absconding clerk had coolly driven to a favorite dining-place in the West End, +LJ015-0108|where a fish curry and a brace of partridges were set before him,|where a fish curry and a brace of partridges were set before him, +LJ015-0109|and he discussed the latter with appetite, but begged that they would never give him curry again, as he did not like it.|and he discussed the latter with appetite, but begged that they would never give him curry again, as he did not like it. +LJ015-0110|After dinner he went into hiding for a day or two,|After dinner he went into hiding for a day or two, +LJ015-0111|then, accompanied by a lady, not Mrs. Robson, he took steamer and started for Copenhagen.|then, accompanied by a lady, not Mrs. Robson, he took steamer and started for Copenhagen. +LJ015-0112|But the continental police had been warned to look out for him, and two Danish inspectors got upon his track,|But the continental police had been warned to look out for him, and two Danish inspectors got upon his track, +LJ015-0113|followed him over to Sweden, and arrested him at Helsingfors.|followed him over to Sweden, and arrested him at Helsingfors. +LJ015-0114|Thence he was transferred to Copenhagen and surrendered in due course to a London police officer.|Thence he was transferred to Copenhagen and surrendered in due course to a London police officer. +LJ015-0115|Little more remains to be said about Robson. He appears to have accepted his position, and to have at once resigned himself to his fate.|Little more remains to be said about Robson. He appears to have accepted his position, and to have at once resigned himself to his fate. +LJ015-0116|When brought to trial he took matters very coolly, and at first pleaded "Not Guilty," but subsequently withdrew the plea.|When brought to trial he took matters very coolly, and at first pleaded "Not Guilty," but subsequently withdrew the plea. +LJ015-0117|Sergeant Ballantine, who prosecuted,|Sergeant Ballantine, who prosecuted, +LJ015-0118|paid him the compliment of describing him as "a young man of great intelligence, considerable powers of mind,|paid him the compliment of describing him as "a young man of great intelligence, considerable powers of mind, +LJ015-0119|and possessed of an education very much beyond the rank of life to which he originally belonged."|and possessed of an education very much beyond the rank of life to which he originally belonged." +LJ015-0120|Robson was found guilty, and sentenced to two terms of transportation, one for twenty and one for fourteen years.|Robson was found guilty, and sentenced to two terms of transportation, one for twenty and one for fourteen years. +LJ015-0121|Newgate officers who remember Robson still describe him as a fine young man, who behaved well as a prisoner,|Newgate officers who remember Robson still describe him as a fine young man, who behaved well as a prisoner, +LJ015-0122|but who had all the appearance of a careless, thoughtless, happy-go-lucky fellow.|but who had all the appearance of a careless, thoughtless, happy-go-lucky fellow. +LJ015-0123|In many respects the embezzlement of which Leopold Redpath was guilty closely resembled that of Robson,|In many respects the embezzlement of which Leopold Redpath was guilty closely resembled that of Robson, +LJ015-0124|but it was based upon more extended and audacious forgeries.|but it was based upon more extended and audacious forgeries. +LJ015-0125|Redpath's crime arose from his peculiar and independent position as registrar of stock of the Great Northern Railway Company.|Redpath's crime arose from his peculiar and independent position as registrar of stock of the Great Northern Railway Company. +LJ015-0126|This offered him great facilities for the creation of artificial stock, its sale from a fictitious holder, and transfer to himself.|This offered him great facilities for the creation of artificial stock, its sale from a fictitious holder, and transfer to himself. +LJ015-0127|All the signatures in the transfer were forged. Not only did he thus transfer and realize "bogus" stock|All the signatures in the transfer were forged. Not only did he thus transfer and realize "bogus" stock +LJ015-0128|but he bought bona fide amounts, and increased their value by altering the figures,|but he bought bona fide amounts, and increased their value by altering the figures, +LJ015-0129|by inserting say 1 before 500, and thus making it|by inserting say one before five hundred, and thus making it +LJ015-0130|fifteen hundred pounds, which larger amount was duly carried to his credit on the register, and entered upon the certificates of transfer.|fifteen hundred pounds, which larger amount was duly carried to his credit on the register, and entered upon the certificates of transfer. +LJ015-0131|By these means Redpath misappropriated vast sums during a period extending over ten years.|By these means Redpath misappropriated vast sums during a period extending over ten years. +LJ015-0132|The total amount was never exactly made out, but the false stock created and issued by him was estimated at two hundred twenty thousand pounds.|The total amount was never exactly made out, but the false stock created and issued by him was estimated at two hundred twenty thousand pounds. +LJ015-0133|Even when the bubble burst Redpath, who had lived at the rate of twenty thousand a year,|Even when the bubble burst Redpath, who had lived at the rate of twenty thousand a year, +LJ015-0134|had assets in the shape of land, house, furniture, pictures, and objets d'art to the value of £50,000.|had assets in the shape of land, house, furniture, pictures, and objets d'art to the value of fifty thousand pounds. +LJ015-0135|He began in a very small way.|He began in a very small way. +LJ015-0136|First a lawyer's clerk, he then got an appointment in the Peninsular and Oriental Company's office;|First a lawyer's clerk, he then got an appointment in the Peninsular and Oriental Company's office; +LJ015-0137|afterwards he set up as an insurance broker on his own account, but presently failed.|afterwards he set up as an insurance broker on his own account, but presently failed. +LJ015-0138|His fault was generosity,|His fault was generosity, +LJ015-0139|an open-handed, unthinking charity which gave freely to the poor and needy the money which belonged to his creditors.|an open-handed, unthinking charity which gave freely to the poor and needy the money which belonged to his creditors. +LJ015-0140|After his bankruptcy he obtained a place as clerk in the Great Northern Railway office,|After his bankruptcy he obtained a place as clerk in the Great Northern Railway office, +LJ015-0141|from which he rose to be assistant registrar, with the special duties of transferring shares.|from which he rose to be assistant registrar, with the special duties of transferring shares. +LJ015-0142|He soon proved his ability, and by unremitting attention mastered the whole work of the office.|He soon proved his ability, and by unremitting attention mastered the whole work of the office. +LJ015-0143|Later on he became registrar, and in this more independent position|Later on he became registrar, and in this more independent position +LJ015-0144|developed to a colossal extent the frauds he had already practiced as a subordinate.|developed to a colossal extent the frauds he had already practiced as a subordinate. +LJ015-0145|Now he launched out into great expenditure, took a house in Chester Terrace, and became known as a Maecenas and patron of the arts.|Now he launched out into great expenditure, took a house in Chester Terrace, and became known as a Maecenas and patron of the arts. +LJ015-0146|He had a nice taste in bric-à-brac, and was considered a good judge of pictures.|He had a nice taste in bric-à-brac, and was considered a good judge of pictures. +LJ015-0147|Leading social and artistic personages were to be met with at his house, and his hospitality was far famed.|Leading social and artistic personages were to be met with at his house, and his hospitality was far famed. +LJ015-0148|The choicest wines, the finest fruits,|The choicest wines, the finest fruits, +LJ015-0149|peas at ten shillings a quart, five-guinea pines, and early asparagus were to be found on his table.|peas at ten shillings a quart, five-guinea pines, and early asparagus were to be found on his table. +LJ015-0150|But his chief extravagance, his favorite folly, was the exercise of an ostentatious benevolence.|But his chief extravagance, his favorite folly, was the exercise of an ostentatious benevolence. +LJ015-0151|The philanthropy he had displayed in a small way when less prosperous became now a passion.|The philanthropy he had displayed in a small way when less prosperous became now a passion. +LJ015-0152|His name headed every subscription list; his purse was always open.|His name headed every subscription list; his purse was always open. +LJ015-0153|Not content with giving where assistance was solicited, he himself sought out deserving cases and personally afforded relief.|Not content with giving where assistance was solicited, he himself sought out deserving cases and personally afforded relief. +LJ015-0154|When the crash came there were pensioners and other recipients of his bounty who could not believe|When the crash came there were pensioners and other recipients of his bounty who could not believe +LJ015-0155|that so good a man had really been for years a swindler and a rogue.|that so good a man had really been for years a swindler and a rogue. +LJ015-0156|Down at Weybridge, where he had a country place, his name was long remembered with gratitude by the poor.|Down at Weybridge, where he had a country place, his name was long remembered with gratitude by the poor. +LJ015-0157|During the days of his prosperity he was a governor of Christ's Hospital,|During the days of his prosperity he was a governor of Christ's Hospital, +LJ015-0158|of the St. Ann's Society, and one of the supporters and managers of the Patriotic Fund.|of the St. Ann's Society, and one of the supporters and managers of the Patriotic Fund. +LJ015-0159|In his person he was neat and fastidious;|In his person he was neat and fastidious; +LJ015-0160|he patronized the best tailors, and had a fashionable coiffeur from Hanover Square daily to curl his hair.|he patronized the best tailors, and had a fashionable coiffeur from Hanover Square daily to curl his hair. +LJ015-0161|There was something dramatic in Redpath's detection|There was something dramatic in Redpath's detection +LJ015-0162|Just after Robson's frauds had agitated the minds of all directors of companies, the chairman of the Great Northern (Mr. Denison)|Just after Robson's frauds had agitated the minds of all directors of companies, the chairman of the Great Northern (Mr. Denison) +LJ015-0163|was standing at a railway station talking to a certain well-known peer of the realm.|was standing at a railway station talking to a certain well-known peer of the realm. +LJ015-0164|Redpath passed and lifted his hat to his chairman; the latter acknowledged the salute.|Redpath passed and lifted his hat to his chairman; the latter acknowledged the salute. +LJ015-0165|But the peer rushed forward and shook Redpath warmly by the hand.|But the peer rushed forward and shook Redpath warmly by the hand. +LJ015-0166|"What do you know of our clerk?" asked Mr. Denison of his lordship. "Only that he is a capital fellow, who gives the best dinners and balls in town."|"What do you know of our clerk?" asked Mr. Denison of his lordship. "Only that he is a capital fellow, who gives the best dinners and balls in town." +LJ015-0167|Redpath had industriously circulated reports that he had prospered greatly in speculation;|Redpath had industriously circulated reports that he had prospered greatly in speculation; +LJ015-0168|but the chairman of the Great Northern could not realize that a clerk of the company could honestly be in the possession of unlimited wealth.|but the chairman of the Great Northern could not realize that a clerk of the company could honestly be in the possession of unlimited wealth. +LJ015-0169|It was at once decided at the board to make a thorough examination of all his books.|It was at once decided at the board to make a thorough examination of all his books. +LJ015-0170|Redpath was called in and informed of the intended investigation. He tried to stave off the evil hour by declaring that everything was perfectly right;|Redpath was called in and informed of the intended investigation. He tried to stave off the evil hour by declaring that everything was perfectly right; +LJ015-0171|but finding he could not escape, he said he would resign his post, and leaving the boardroom, disappeared.|but finding he could not escape, he said he would resign his post, and leaving the boardroom, disappeared. +LJ015-0172|The inquiry soon revealed the colossal character of the frauds.|The inquiry soon revealed the colossal character of the frauds. +LJ015-0173|Warrants were issued for Redpath's arrest, but he had flown to Paris.|Warrants were issued for Redpath's arrest, but he had flown to Paris. +LJ015-0174|Thither police officers followed, only to find that he had returned to London.|Thither police officers followed, only to find that he had returned to London. +LJ015-0175|A further search discovered him at breakfast at a small house in the New Road.|A further search discovered him at breakfast at a small house in the New Road. +LJ015-0176|He was arrested, examined before a police magistrate, and committed to Newgate.|He was arrested, examined before a police magistrate, and committed to Newgate. +LJ015-0177|Great excitement prevailed in the city and the West End when Redpath's defalcations were made public.|Great excitement prevailed in the city and the West End when Redpath's defalcations were made public. +LJ015-0178|The Stock Market was greatly affected, and society, more especially that which frequents Exeter Hall, was convulsed.|The Stock Market was greatly affected, and society, more especially that which frequents Exeter Hall, was convulsed. +LJ015-0179|The Central Criminal Court, when the trial came on,|The Central Criminal Court, when the trial came on, +LJ015-0180|was densely crowded, and many curious eyes were turned upon the somewhat remarkable man who occupied the dock.|was densely crowded, and many curious eyes were turned upon the somewhat remarkable man who occupied the dock. +LJ015-0181|He is described by a contemporary account as a fresh-looking man of forty years of age, slightly bald, inclined to embonpoint,|He is described by a contemporary account as a fresh-looking man of forty years of age, slightly bald, inclined to embonpoint, +LJ015-0182|and thoroughly embodying the idea of English respectability.|and thoroughly embodying the idea of English respectability. +LJ015-0183|His manner was generally self-possessed, but his face was marked with "uneasy earnestness,"|His manner was generally self-possessed, but his face was marked with "uneasy earnestness," +LJ015-0184|and he looked about him with wayward, furtive glances.|and he looked about him with wayward, furtive glances. +LJ015-0185|When the jury found a verdict of guilty he remained unmoved. He listened without emotion to the judge's well-merited censures,|When the jury found a verdict of guilty he remained unmoved. He listened without emotion to the judge's well-merited censures, +LJ015-0186|and received his sentence of transportation for life without much surprise.|and received his sentence of transportation for life without much surprise. +LJ015-0187|Redpath passed away into the outer darkness of a penal colony, where he was still living a year or two back|Redpath passed away into the outer darkness of a penal colony, where he was still living a year or two back +LJ015-0188|But his name lingers still in this country as that of the first swindler of his time,|But his name lingers still in this country as that of the first swindler of his time, +LJ015-0189|and the prototype of a class not uncommon in our later days|and the prototype of a class not uncommon in our later days +LJ015-0190|that of dishonest rogues who assume piety and philanthropy as a cloak for their misdeeds.|that of dishonest rogues who assume piety and philanthropy as a cloak for their misdeeds. +LJ015-0191|In Newgate Redpath is remembered by the prison officer as a difficult man to deal with.|In Newgate Redpath is remembered by the prison officer as a difficult man to deal with. +LJ015-0192|From the moment of his reception he gave himself great airs, as a martyr and a man heavily wronged.|From the moment of his reception he gave himself great airs, as a martyr and a man heavily wronged. +LJ015-0193|By-and-by, when escape seemed hopeless, and after sentence, he suddenly degenerated into the lowest stamp of criminal,|By-and-by, when escape seemed hopeless, and after sentence, he suddenly degenerated into the lowest stamp of criminal, +LJ015-0194|and behaved so as to justify a belief that he had been a jail-bird all his life.|and behaved so as to justify a belief that he had been a jail-bird all his life. +LJ015-0195|It has been already remarked in these pages that with changed social conditions came a great change in the character of crimes.|It has been already remarked in these pages that with changed social conditions came a great change in the character of crimes. +LJ015-0196|Highway robberies, for instance, had disappeared, if we except the spasmodic and severely repressed outbreak of "garotting,"|Highway robberies, for instance, had disappeared, if we except the spasmodic and severely repressed outbreak of "garotting," +LJ015-0198|It was no longer possible to stop a coach or carriage, or rob the postman who carried the mail.|It was no longer possible to stop a coach or carriage, or rob the postman who carried the mail. +LJ015-0199|The improved methods of locomotion had put a stop to these depredations. People traveled in company, as a rule;|The improved methods of locomotion had put a stop to these depredations. People traveled in company, as a rule; +LJ015-0200|only when single and unprotected were they in any danger of attack, and that but rarely.|only when single and unprotected were they in any danger of attack, and that but rarely. +LJ015-0201|There were still big prizes, however, to tempt the daring, and none appealed more to the thievish instinct than the custom of transmitting gold by rail.|There were still big prizes, however, to tempt the daring, and none appealed more to the thievish instinct than the custom of transmitting gold by rail. +LJ015-0202|The precious metal was sent from place to place carefully locked up and guarded, no doubt;|The precious metal was sent from place to place carefully locked up and guarded, no doubt; +LJ015-0203|but were the precautions too minute, the vigilance too close to be eluded or overcome?|but were the precautions too minute, the vigilance too close to be eluded or overcome? +LJ015-0204|This was the question which presented itself to the fertile brain of one Pierce,|This was the question which presented itself to the fertile brain of one Pierce, +LJ015-0205|who had been concerned in various "jobs" of a dishonest character, and who for the moment was a clerk in a betting office.|who had been concerned in various "jobs" of a dishonest character, and who for the moment was a clerk in a betting office. +LJ015-0206|He laid the suggestion before Agar, a professional thief, who was of opinion it contained elements of success.|He laid the suggestion before Agar, a professional thief, who was of opinion it contained elements of success. +LJ015-0207|But the collusion and active assistance of employees of the railway carriers were indispensable, and together|But the collusion and active assistance of employees of the railway carriers were indispensable, and together +LJ015-0208|they sounded one Burgess, a guard on the South-Eastern Railway, a line by which large quantities of bullion were sent to the Continent.|they sounded one Burgess, a guard on the South-Eastern Railway, a line by which large quantities of bullion were sent to the Continent. +LJ015-0209|Burgess detailed the whole system of transmission.|Burgess detailed the whole system of transmission. +LJ015-0210|The gold, packed in an iron-bound box, was securely lodged in safes locked with patent Chubbs.|The gold, packed in an iron-bound box, was securely lodged in safes locked with patent Chubbs. +LJ015-0211|Each safe had three sets of double keys, all held by confidential servants of the company.|Each safe had three sets of double keys, all held by confidential servants of the company. +LJ015-0212|One pair was with the traffic superintendent in London, another with an official in Folkestone,|One pair was with the traffic superintendent in London, another with an official in Folkestone, +LJ015-0213|a third with the captain of the Folkestone and Boulogne boat.|a third with the captain of the Folkestone and Boulogne boat. +LJ015-0214|At the other side of the Channel the French railway authorities took charge.|At the other side of the Channel the French railway authorities took charge. +LJ015-0215|The safes while on the line en route between London and Folkestone were in the guard's van.|The safes while on the line en route between London and Folkestone were in the guard's van. +LJ015-0216|This was an important step, and they might easily be robbed some day when Burgess was the guard, provided only that they could be opened.|This was an important step, and they might easily be robbed some day when Burgess was the guard, provided only that they could be opened. +LJ015-0217|The next step was to get impressions and fabricate false keys.|The next step was to get impressions and fabricate false keys. +LJ015-0218|A new accomplice was now needed within the company's establishment, and Pierce looked about long before he found the right person.|A new accomplice was now needed within the company's establishment, and Pierce looked about long before he found the right person. +LJ015-0219|At last he decided to enlist one Tester, a clerk in the traffic department, whom he thought would prove a likely tool.|At last he decided to enlist one Tester, a clerk in the traffic department, whom he thought would prove a likely tool. +LJ015-0220|The four waited patiently for their opportunity,|The four waited patiently for their opportunity, +LJ015-0221|which came when the safes were sent to Chubbs' to be repaired; and Chubbs sent them back, but only with one key,|which came when the safes were sent to Chubbs' to be repaired; and Chubbs sent them back, but only with one key, +LJ015-0222|in such a way that Tester had possession of this key for a time.|in such a way that Tester had possession of this key for a time. +LJ015-0223|He lent it to Agar for a brief space, who promptly took an impression on wax. But the safes had a double lock;|He lent it to Agar for a brief space, who promptly took an impression on wax. But the safes had a double lock; +LJ015-0224|the difficulty was to get a copy of the second key.|the difficulty was to get a copy of the second key. +LJ015-0225|This was at length effected by Agar and Pierce.|This was at length effected by Agar and Pierce. +LJ015-0226|After hanging about the Folkestone office for some time, they saw at last that the key was kept in a certain cupboard.|After hanging about the Folkestone office for some time, they saw at last that the key was kept in a certain cupboard. +LJ015-0227|Still watching and waiting for the first chance, they seized it when the clerks left the office empty for a moment.|Still watching and waiting for the first chance, they seized it when the clerks left the office empty for a moment. +LJ015-0228|Pierce boldly stepped in, found the cupboard unlocked; he removed the key, handed it to Agar outside,|Pierce boldly stepped in, found the cupboard unlocked; he removed the key, handed it to Agar outside, +LJ015-0229|who quickly took the wax impression, handed it back to Pierce; Pierce replaced it, left the office, and the thing was done.|who quickly took the wax impression, handed it back to Pierce; Pierce replaced it, left the office, and the thing was done. +LJ015-0230|After this nothing remained but to wait for some occasion when the amount transmitted would be sufficient to justify the risks of robbery.|After this nothing remained but to wait for some occasion when the amount transmitted would be sufficient to justify the risks of robbery. +LJ015-0231|It was Tester's business, who had access to the railway company's books, to watch for this.|It was Tester's business, who had access to the railway company's books, to watch for this. +LJ015-0232|Meanwhile the others completed their preparations with the utmost care.|Meanwhile the others completed their preparations with the utmost care. +LJ015-0233|A weight of shot was bought and stowed in carpet bags ready to replace exactly the abstracted gold.|A weight of shot was bought and stowed in carpet bags ready to replace exactly the abstracted gold. +LJ015-0234|Courier bags were bought to carry the "stuff" slung over the shoulders;|Courier bags were bought to carry the "stuff" slung over the shoulders; +LJ015-0235|and last, but not least, Agar frequently traveled up and down the line to test the false keys he had manufactured with Pierce's assistance.|and last, but not least, Agar frequently traveled up and down the line to test the false keys he had manufactured with Pierce's assistance. +LJ015-0236|Burgess admitted him into the guard's van, where he fitted and filed the keys till they worked easily and satisfactorily in the locks of the safe.|Burgess admitted him into the guard's van, where he fitted and filed the keys till they worked easily and satisfactorily in the locks of the safe. +LJ015-0237|One night Tester whispered to Agar and Pierce, "All right," as they cautiously lounged about London Bridge.|One night Tester whispered to Agar and Pierce, "All right," as they cautiously lounged about London Bridge. +LJ015-0238|The thieves took first-class tickets, handed their bags full of shot to the porters, who placed them in the guard's van.|The thieves took first-class tickets, handed their bags full of shot to the porters, who placed them in the guard's van. +LJ015-0239|Just as the train was starting Agar slipped into the van with Burgess, and Pierce got into a first-class carriage.|Just as the train was starting Agar slipped into the van with Burgess, and Pierce got into a first-class carriage. +LJ015-0240|Agar at once got to work on the first safe.|Agar at once got to work on the first safe. +LJ015-0241|opened it, took out and broke into the bullion box, removed the gold, substituted the shot from a carpet bag,|opened it, took out and broke into the bullion box, removed the gold, substituted the shot from a carpet bag, +LJ015-0242|re-fastened and re-sealed the bullion box, and replaced it in the safe.|re-fastened and re-sealed the bullion box, and replaced it in the safe. +LJ015-0243|At Redhill Tester met the train and relieved the thieves of a portion of the stolen gold.|At Redhill Tester met the train and relieved the thieves of a portion of the stolen gold. +LJ015-0244|At the same station Pierce joined Agar in the guard's van, and there were now three to carry on the robbery.|At the same station Pierce joined Agar in the guard's van, and there were now three to carry on the robbery. +LJ015-0245|The two remaining safes were attacked and nearly entirely despoiled in the same way as the first, and the contents transferred to the courier bags.|The two remaining safes were attacked and nearly entirely despoiled in the same way as the first, and the contents transferred to the courier bags. +LJ015-0246|The train was now approaching Folkestone, and Agar and Pierce hid themselves in a dark part of the van.|The train was now approaching Folkestone, and Agar and Pierce hid themselves in a dark part of the van. +LJ015-0247|At that station the safes were given out, heavy with shot, not gold; the thieves went on to Dover, and by-and-by,|At that station the safes were given out, heavy with shot, not gold; the thieves went on to Dover, and by-and-by, +LJ015-0248|with Ostend tickets previously procured, returned to London without mishap, and by degrees disposed of much of the stolen gold.|with Ostend tickets previously procured, returned to London without mishap, and by degrees disposed of much of the stolen gold. +LJ015-0249|The theft was discovered at Boulogne, when the boxes were found not to weigh exactly what they ought. But no clue was obtained to the thieves,|The theft was discovered at Boulogne, when the boxes were found not to weigh exactly what they ought. But no clue was obtained to the thieves, +LJ015-0250|and the theft might have remained a mystery but for the subsequent bad faith of Pierce to his accomplice Agar.|and the theft might have remained a mystery but for the subsequent bad faith of Pierce to his accomplice Agar. +LJ015-0251|The latter was ere long arrested on a charge of uttering forged cheques, convicted, and sentenced to transportation for life.|The latter was ere long arrested on a charge of uttering forged cheques, convicted, and sentenced to transportation for life. +LJ015-0252|When he knew that he could not escape his fate,|When he knew that he could not escape his fate, +LJ015-0253|he handed over to Pierce a sum of £3000, his own, whether rightly or wrongly acquired never came out,|he handed over to Pierce a sum of three thousand pounds, his own, whether rightly or wrongly acquired never came out, +LJ015-0254|together with the unrealized part of the bullion, amounting in all to some £15,000,|together with the unrealized part of the bullion, amounting in all to some fifteen thousand pounds, +LJ015-0255|and begged his accomplice to invest it as a settlement on a woman named Kay, by whom he had had a child.|and begged his accomplice to invest it as a settlement on a woman named Kay, by whom he had had a child. +LJ015-0256|Pierce made Kay only a few small payments, then appropriated the rest of the money.|Pierce made Kay only a few small payments, then appropriated the rest of the money. +LJ015-0257|Kay, who had been living with Agar at the time of the bullion robbery,|Kay, who had been living with Agar at the time of the bullion robbery, +LJ015-0258|went to the police in great fury and distress, and disclosed all she knew of the affair.|went to the police in great fury and distress, and disclosed all she knew of the affair. +LJ015-0259|Agar too, in Newgate, heard how Pierce had treated him, and at once readily turned approver.|Agar too, in Newgate, heard how Pierce had treated him, and at once readily turned approver. +LJ015-0260|As the evidence he gave incriminated Pierce, Burgess, and Tester, all three were arrested and committed to Newgate for trial.|As the evidence he gave incriminated Pierce, Burgess, and Tester, all three were arrested and committed to Newgate for trial. +LJ015-0261|The whole strange story, the long incubation and the elaborate accomplishment of the plot,|The whole strange story, the long incubation and the elaborate accomplishment of the plot, +LJ015-0262|came out at the Old Bailey, and was acknowledged to be one of the most extraordinary on record.|came out at the Old Bailey, and was acknowledged to be one of the most extraordinary on record. +LJ015-0263|Scarcely had the conviction of these daring and astute thieves been assured, than another gigantic fraud was brought to light.|Scarcely had the conviction of these daring and astute thieves been assured, than another gigantic fraud was brought to light. +LJ015-0264|The series of boldly-conceived and cleverly-executed forgeries in which James Townshend Saward, commonly called "Jem the Penman,"|The series of boldly-conceived and cleverly-executed forgeries in which James Townshend Saward, commonly called "Jem the Penman," +LJ015-0265|was the prime mover,|was the prime mover, +LJ015-0266|has probably no parallel in the annals of crime. Saward himself is a striking and in some respects an unique figure in criminal history.|has probably no parallel in the annals of crime. Saward himself is a striking and in some respects an unique figure in criminal history. +LJ015-0267|A man of birth and education, a member of the bar, and of acknowledged legal attainments, his proclivities were all downward.|A man of birth and education, a member of the bar, and of acknowledged legal attainments, his proclivities were all downward. +LJ015-0268|Instead of following an honorable profession, he preferred to turn his great natural talents and ready wits to the most nefarious practices.|Instead of following an honorable profession, he preferred to turn his great natural talents and ready wits to the most nefarious practices. +LJ015-0269|He was known to the whole criminal fraternity as a high-class receiver of stolen goods, a negotiator more especially of stolen paper,|He was known to the whole criminal fraternity as a high-class receiver of stolen goods, a negotiator more especially of stolen paper, +LJ015-0270|cheques and bills, of which he made a particular use.|cheques and bills, of which he made a particular use. +LJ015-0271|He dealt too in the precious metals, when they had been improperly acquired,|He dealt too in the precious metals, when they had been improperly acquired, +LJ015-0272|and it was to him that Agar, Pierce, and the rest applied when seeking to dispose of their stolen bullion.|and it was to him that Agar, Pierce, and the rest applied when seeking to dispose of their stolen bullion. +LJ015-0273|But Saward's operations were mainly directed to the fabrication and uttering of forged cheques.|But Saward's operations were mainly directed to the fabrication and uttering of forged cheques. +LJ015-0274|His method was comprehensive and deeply laid.|His method was comprehensive and deeply laid. +LJ015-0275|Burglars brought him the cheques they stole from houses, thieves what they got in pocketbooks.|Burglars brought him the cheques they stole from houses, thieves what they got in pocketbooks. +LJ015-0276|Cheques blank and canceled were his stock-in-trade. The former he filled up by exact imitation of the latter, signature and all.|Cheques blank and canceled were his stock-in-trade. The former he filled up by exact imitation of the latter, signature and all. +LJ015-0277|When he could get nothing but the blank cheque, he set in motion all sorts of schemes for obtaining signatures, such as|When he could get nothing but the blank cheque, he set in motion all sorts of schemes for obtaining signatures, such as +LJ015-0278|commencing sham actions, and addressing formal applications, merely for the reply.|commencing sham actions, and addressing formal applications, merely for the reply. +LJ015-0279|One stroke of luck which he turned to great account|One stroke of luck which he turned to great account +LJ015-0280|was the return from transportation of an old "pal" and confederate, who brought with him some bills of exchange.|was the return from transportation of an old "pal" and confederate, who brought with him some bills of exchange. +LJ015-0281|Saward's method of negotiating the cheques was equally well planned.|Saward's method of negotiating the cheques was equally well planned. +LJ015-0282|Like his great predecessor Old Patch, he never went to a bank himself, nor did any of his accomplices.|Like his great predecessor Old Patch, he never went to a bank himself, nor did any of his accomplices. +LJ015-0283|The bearer of the cheque was always innocent and ignorant of the fraudulent nature of the document he presented.|The bearer of the cheque was always innocent and ignorant of the fraudulent nature of the document he presented. +LJ015-0284|In order to obtain messengers of this sort, Saward answered advertisements of persons seeking employment,|In order to obtain messengers of this sort, Saward answered advertisements of persons seeking employment, +LJ015-0285|and when these presented themselves, entrusted them as a beginning with the duty of cashing cheques.|and when these presented themselves, entrusted them as a beginning with the duty of cashing cheques. +LJ015-0286|A confederate followed the emissary closely,|A confederate followed the emissary closely, +LJ015-0287|not only to ensure fair play and the surrender of the proceeds if the cheque was cashed, but to give timely notice if it was not,|not only to ensure fair play and the surrender of the proceeds if the cheque was cashed, but to give timely notice if it was not, +LJ015-0288|so that Saward and the rest might make themselves scarce.|so that Saward and the rest might make themselves scarce. +LJ015-0289|As each transaction was carried out from a different address, and a different messenger always employed,|As each transaction was carried out from a different address, and a different messenger always employed, +LJ015-0290|the forgers always escaped detection. But fate overtook two of the gang,|the forgers always escaped detection. But fate overtook two of the gang, +LJ015-0291|partly through their own carelessness, when transferring their operations to Yarmouth.|partly through their own carelessness, when transferring their operations to Yarmouth. +LJ015-0292|One named Hardwicke assumed the name of Ralph, and, to obtain commercial credit in Yarmouth, paid in £250 to a Yarmouth bank|One named Hardwicke assumed the name of Ralph, and, to obtain commercial credit in Yarmouth, paid in two hundred fifty pounds to a Yarmouth bank +LJ015-0293|as coming from a Mr. Whitney.|as coming from a Mr. Whitney. +LJ015-0294|He forgot to add that it was to be placed to Ralph's credit, and when he called as Ralph,|He forgot to add that it was to be placed to Ralph's credit, and when he called as Ralph, +LJ015-0295|he was told it was only at Mr. Whitney's disposal, and that it could be paid to no one else.|he was told it was only at Mr. Whitney's disposal, and that it could be paid to no one else. +LJ015-0296|Hardwicke, or "Ralph," appealed to Saward in his difficulty|Hardwicke, or "Ralph," appealed to Saward in his difficulty +LJ015-0297|and that clever schemer sent an elaborate letter of instructions how to ask for the money.|and that clever schemer sent an elaborate letter of instructions how to ask for the money. +LJ015-0298|But while Hardwicke was in communication with Saward, the bank was in communication with London|But while Hardwicke was in communication with Saward, the bank was in communication with London +LJ015-0299|and the circumstances were deemed sufficiently suspicious to warrant the arrest of the gentlemen at Yarmouth on a charge of forgery and conspiracy.|and the circumstances were deemed sufficiently suspicious to warrant the arrest of the gentlemen at Yarmouth on a charge of forgery and conspiracy. +LJ015-0300|Saward's letter to Hardwicke fell into the hands of the police and compromised him.|Saward's letter to Hardwicke fell into the hands of the police and compromised him. +LJ015-0301|While Hardwicke and Atwell were in Newgate awaiting trial, active search was made for Saward,|While Hardwicke and Atwell were in Newgate awaiting trial, active search was made for Saward, +LJ015-0302|who was at length taken in a coffee-shop near Oxford Street, under the name of Hopkins. He resisted at first, and denied his identity,|who was at length taken in a coffee-shop near Oxford Street, under the name of Hopkins. He resisted at first, and denied his identity, +LJ015-0303|but on being searched, two blank cheques of the London and Westminster Bank were found in his pocket.|but on being searched, two blank cheques of the London and Westminster Bank were found in his pocket. +LJ015-0304|He then confessed that he was the redoubtable Jem Saward, or Jem the Penman, and was conveyed to a police-court, and thence to Newgate.|He then confessed that he was the redoubtable Jem Saward, or Jem the Penman, and was conveyed to a police-court, and thence to Newgate. +LJ015-0305|At his trial Atwell and Hardwicke, two of his chief allies and accomplices, turned approvers,|At his trial Atwell and Hardwicke, two of his chief allies and accomplices, turned approvers, +LJ015-0306|and the whole scheme of systematic forgery was laid bare.|and the whole scheme of systematic forgery was laid bare. +LJ015-0307|The evidence was corroborated by that of many of the victims who had acted as messengers,|The evidence was corroborated by that of many of the victims who had acted as messengers, +LJ015-0308|and others who swore to the meetings of the conspirators and their movements. Saward was found guilty,|and others who swore to the meetings of the conspirators and their movements. Saward was found guilty, +LJ015-0309|and the judge, in passing sentence on him of transportation for life, expressed deep regret that "the ingenuity, skill, and talent,|and the judge, in passing sentence on him of transportation for life, expressed deep regret that "the ingenuity, skill, and talent, +LJ015-0310|which had received so perverted and mistaken direction,|which had received so perverted and mistaken direction, +LJ015-0311|had not been guided by a sense of virtue, and directed to more honorable and useful pursuits.|had not been guided by a sense of virtue, and directed to more honorable and useful pursuits. +LJ015-0312|The proceeds of these forgeries amounted, it was said, to some thousands per annum.|The proceeds of these forgeries amounted, it was said, to some thousands per annum. +LJ015-0313|Saward spent all his share at low gaming houses, and in all manner of debaucheries.|Saward spent all his share at low gaming houses, and in all manner of debaucheries. +LJ015-0314|He was in person a short, square-built man of gentlemanly address, sharp and shrewd in conversation and manner.|He was in person a short, square-built man of gentlemanly address, sharp and shrewd in conversation and manner. +LJ016-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 19: Later Records|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section nineteen: Later Records +LJ016-0002|The old notion always prevailed that Newgate was impregnable, so to speak, from within,|The old notion always prevailed that Newgate was impregnable, so to speak, from within, +LJ016-0003|and that none of its inmates could hope to escape from its secure precincts.|and that none of its inmates could hope to escape from its secure precincts. +LJ016-0004|Yet the jail, in spite of its fortress-like aspect, was by no means really safe.|Yet the jail, in spite of its fortress-like aspect, was by no means really safe. +LJ016-0005|Year after year prisoners determined to get free, and occasionally succeeded in their efforts.|Year after year prisoners determined to get free, and occasionally succeeded in their efforts. +LJ016-0006|The inspectors' reports mention many cases of evasion accomplished.|The inspectors' reports mention many cases of evasion accomplished. +LJ016-0007|There were others less successful.|There were others less successful. +LJ016-0008|Charles Thomas White, awaiting execution for arson, made a desperate effort to escape from Newgate in 1827.|Charles Thomas White, awaiting execution for arson, made a desperate effort to escape from Newgate in eighteen twenty-seven. +LJ016-0009|He had friends and auxiliaries inside the jail and out. The cell he occupied was near the outer wall,|He had friends and auxiliaries inside the jail and out. The cell he occupied was near the outer wall, +LJ016-0010|and had he but been able to remove its iron bars, he might have descended into Newgate Street by means of a rope ladder.|and had he but been able to remove its iron bars, he might have descended into Newgate Street by means of a rope ladder. +LJ016-0011|The ladder was actually made, of black sewing-thread firmly and closely interwoven. But White could not remove the bars;|The ladder was actually made, of black sewing-thread firmly and closely interwoven. But White could not remove the bars; +LJ016-0012|the instruments needed for the purpose never reached him.|the instruments needed for the purpose never reached him. +LJ016-0013|It was noticed that he was most anxious to receive a pair of shoes for which he had asked, and when they arrived they were closely examined.|It was noticed that he was most anxious to receive a pair of shoes for which he had asked, and when they arrived they were closely examined. +LJ016-0014|Sewn in between the upper and lower leathers several spring saws were found, which would have easily cut through any bars.|Sewn in between the upper and lower leathers several spring saws were found, which would have easily cut through any bars. +LJ016-0015|White, when taxed with his attempt, admitted that the accusation was true, and spoke "with pride and satisfaction of the practicability of his scheme."|White, when taxed with his attempt, admitted that the accusation was true, and spoke "with pride and satisfaction of the practicability of his scheme." +LJ016-0016|There is an attempt at escape mentioned in Mr. Wakefield's book, which might have been an intended suicide.|There is an attempt at escape mentioned in Mr. Wakefield's book, which might have been an intended suicide. +LJ016-0017|John Williams, a young fellow only twenty-three years of age, awaited execution in 1827 for stealing in a dwelling-house.|John Williams, a young fellow only twenty-three years of age, awaited execution in eighteen twenty-seven for stealing in a dwelling-house. +LJ016-0018|On the very morning on which he was to suffer he eluded the vigilance, such as it was, of his officers|On the very morning on which he was to suffer he eluded the vigilance, such as it was, of his officers +LJ016-0019|and climbed up the pipe of a cistern in the corner of the press yard; some thought with the idea of drowning himself.|and climbed up the pipe of a cistern in the corner of the press yard; some thought with the idea of drowning himself. +LJ016-0020|He never reached the cistern, but fell back into the yard, injuring his legs severely.|He never reached the cistern, but fell back into the yard, injuring his legs severely. +LJ016-0021|Although his execution was imminent, a surgeon attended to his wounds, and he was carried more dead than alive to the scaffold.|Although his execution was imminent, a surgeon attended to his wounds, and he was carried more dead than alive to the scaffold. +LJ016-0022|A harrowing scene followed;|A harrowing scene followed; +LJ016-0023|the wounds broke open and bled profusely while the last dread penalty was being performed, to the manifest excitement and indignation of the crowd.|the wounds broke open and bled profusely while the last dread penalty was being performed, to the manifest excitement and indignation of the crowd. +LJ016-0024|A more daring and skilful escape was effected in 1836 by the chimney-sweep Henry Williams,|A more daring and skilful escape was effected in eighteen thirty-six by the chimney-sweep Henry Williams, +LJ016-0025|who, while detained in the press-yard as a capital convict, under sentence of death for burglary,|who, while detained in the press-yard as a capital convict, under sentence of death for burglary, +LJ016-0026|managed to get away in the very same spot where his namesake had nine years before so miserably failed.|managed to get away in the very same spot where his namesake had nine years before so miserably failed. +LJ016-0027|Escape seemed absolutely hopeless,|Escape seemed absolutely hopeless, +LJ016-0028|and would certainly have been impossible to any one less nimble than a chimney-sweep, trained under the old system to ascend the most intricate flues.|and would certainly have been impossible to any one less nimble than a chimney-sweep, trained under the old system to ascend the most intricate flues. +LJ016-0029|Even after Williams had got out, persons were disposed to disbelieve that the escape had been accomplished in the manner indicated;|Even after Williams had got out, persons were disposed to disbelieve that the escape had been accomplished in the manner indicated; +LJ016-0030|they preferred to credit it to carelessness or collusion from officers of the jail.|they preferred to credit it to carelessness or collusion from officers of the jail. +LJ016-0031|Yet from the circumstantial account given by Williams after recapture, there can be little doubt that he got away as will be described.|Yet from the circumstantial account given by Williams after recapture, there can be little doubt that he got away as will be described. +LJ016-0032|Williams as a capital convict was lodged in the press-yard or condemned ward.|Williams as a capital convict was lodged in the press-yard or condemned ward. +LJ016-0033|He had access to the airing yard, and there was for hours no kind of supervision.|He had access to the airing yard, and there was for hours no kind of supervision. +LJ016-0034|In one corner of the airing yard stood a cistern at some height from the ground;|In one corner of the airing yard stood a cistern at some height from the ground; +LJ016-0035|the wall beneath and above it was "rusticated," in other words, the granite surface had become roughened, and offered a sort of foothold.|the wall beneath and above it was "rusticated," in other words, the granite surface had become roughened, and offered a sort of foothold. +LJ016-0036|About fifty feet from the ground level, and above the cistern, a revolving chevaux-de-frise of iron was fixed,|About fifty feet from the ground level, and above the cistern, a revolving chevaux-de-frise of iron was fixed, +LJ016-0037|with only a short interval between it and the wall, supported by a horizontal iron railing with upright points;|with only a short interval between it and the wall, supported by a horizontal iron railing with upright points; +LJ016-0038|in the wall above the chevaux-de-frise projected a series of iron spikes sharp enough to forbid further ascent.|in the wall above the chevaux-de-frise projected a series of iron spikes sharp enough to forbid further ascent. +LJ016-0039|Williams surveyed these formidable obstacles to evasion, and calmly proceeded to surmount them.|Williams surveyed these formidable obstacles to evasion, and calmly proceeded to surmount them. +LJ016-0040|His first task was to gain the top of the cistern; this he effected by keeping his back to one side of the angle,|His first task was to gain the top of the cistern; this he effected by keeping his back to one side of the angle, +LJ016-0041|and working with his hands behind him, while he used his bare feet like claws upon the other side of the wall angle.|and working with his hands behind him, while he used his bare feet like claws upon the other side of the wall angle. +LJ016-0042|The condition of the stone surface just mentioned assisted him in this, and he managed to get beyond the cistern to the railing below the chevaux-de-frise.|The condition of the stone surface just mentioned assisted him in this, and he managed to get beyond the cistern to the railing below the chevaux-de-frise. +LJ016-0043|The least slip now would have been fatal to him. But he could not thrust his body in through the narrow space left by the chevaux-de-frise,|The least slip now would have been fatal to him. But he could not thrust his body in through the narrow space left by the chevaux-de-frise, +LJ016-0044|and was compelled to work along the railing round three-quarters of the square of the yard,|and was compelled to work along the railing round three-quarters of the square of the yard, +LJ016-0045|and at length reached a point opposite the top of the building containing the condemned wards. This had been a perilous and painful task;|and at length reached a point opposite the top of the building containing the condemned wards. This had been a perilous and painful task; +LJ016-0046|the spikes of the railing penetrated his flesh and made progression slow and difficult.|the spikes of the railing penetrated his flesh and made progression slow and difficult. +LJ016-0047|But the worst part of the business was to jump from this irksome foothold of the iron grating on to the top of the building just mentioned,|But the worst part of the business was to jump from this irksome foothold of the iron grating on to the top of the building just mentioned, +LJ016-0048|a distance of eight or nine feet.|a distance of eight or nine feet. +LJ016-0049|He had here completed his ascent.|He had here completed his ascent. +LJ016-0050|His next job was to descend outside Newgate.|His next job was to descend outside Newgate. +LJ016-0051|Clambering along the roof,|Clambering along the roof, +LJ016-0052|he passed to the top of the ordinary's residence, hoping to find an open sky-light by which he might enter and so work downstairs.|he passed to the top of the ordinary's residence, hoping to find an open sky-light by which he might enter and so work downstairs. +LJ016-0053|If the worst came to the worst, he intended to have gone down some chimney, as he had often done before in the way of business.|If the worst came to the worst, he intended to have gone down some chimney, as he had often done before in the way of business. +LJ016-0054|But he did not like the risk of entering a room by the fireplace, and the chances of detection it offered.|But he did not like the risk of entering a room by the fireplace, and the chances of detection it offered. +LJ016-0055|He traversed vainly all the roofs in Newgate Street,|He traversed vainly all the roofs in Newgate Street, +LJ016-0056|running a great risk of discovery as he passed by a lot of workmen at Tyler's manufactory in Warwick Square, which had formerly been the College of Physicians.|running a great risk of discovery as he passed by a lot of workmen at Tyler's manufactory in Warwick Square, which had formerly been the College of Physicians. +LJ016-0057|As his coat was an encumbrance, he left it on the top of the third house in Newgate Street, and thus in shirt-sleeves, barefoot and bareheaded,|As his coat was an encumbrance, he left it on the top of the third house in Newgate Street, and thus in shirt-sleeves, barefoot and bareheaded, +LJ016-0058|he worked along to the roofs in Warwick Lane.|he worked along to the roofs in Warwick Lane. +LJ016-0059|Here he came upon a woman on the leads hanging out clothes to dry.|Here he came upon a woman on the leads hanging out clothes to dry. +LJ016-0060|Williams concealed himself behind a chimney till she had re-entered her garret,|Williams concealed himself behind a chimney till she had re-entered her garret, +LJ016-0061|and then following her down a step ladder into the house, told his story, appealed to and won her compassion.|and then following her down a step ladder into the house, told his story, appealed to and won her compassion. +LJ016-0062|She suffered him to pass downstairs.|She suffered him to pass downstairs. +LJ016-0063|Below he met another woman and a girl, both of whom were terrified at his appearance, but|Below he met another woman and a girl, both of whom were terrified at his appearance, but +LJ016-0064|when he explained that he was running away from the gallows they left him the road clear.|when he explained that he was running away from the gallows they left him the road clear. +LJ016-0065|To walk out into the street was an easy affair, and he was now free, with one and fourpence in his pocket and a shirt and trousers for all his clothing.|To walk out into the street was an easy affair, and he was now free, with one and fourpence in his pocket and a shirt and trousers for all his clothing. +LJ016-0066|Denied admission everywhere as a ragged, half-naked beggar,|Denied admission everywhere as a ragged, half-naked beggar, +LJ016-0067|he tramped across London Bridge to Wandsworth, where he refreshed himself with a pint of strong ale, the first sustenance he had taken since his escape,|he tramped across London Bridge to Wandsworth, where he refreshed himself with a pint of strong ale, the first sustenance he had taken since his escape, +LJ016-0068|and continued his march to Kingston, where he slept soundly under a hedge till next morning.|and continued his march to Kingston, where he slept soundly under a hedge till next morning. +LJ016-0069|Entering a town, he obtained employment at once as a chimney-sweep|Entering a town, he obtained employment at once as a chimney-sweep +LJ016-0070|from a widow woman, who gave him "bub and grub," or food and one-and-sixpence, for every nine days' work.|from a widow woman, who gave him "bub and grub," or food and one-and-sixpence, for every nine days' work. +LJ016-0071|Dissatisfied with this remuneration, he again took to the road, and tramped into Hampshire,|Dissatisfied with this remuneration, he again took to the road, and tramped into Hampshire, +LJ016-0072|where he presently committed a burglary at Lymington, was caught, and lodged in Winchester Jail.|where he presently committed a burglary at Lymington, was caught, and lodged in Winchester Jail. +LJ016-0073|Mr. Cope, the governor of Newgate, having been communicated with, proceeded to Winchester, where he at once identified Williams.|Mr. Cope, the governor of Newgate, having been communicated with, proceeded to Winchester, where he at once identified Williams. +LJ016-0074|The success, although very short-lived, which attended him, no doubt inspired other inmates of Newgate to follow his example.|The success, although very short-lived, which attended him, no doubt inspired other inmates of Newgate to follow his example. +LJ016-0075|It was for some time after this a constant practice to go up the chimneys in the hopes of escaping by the flue.|It was for some time after this a constant practice to go up the chimneys in the hopes of escaping by the flue. +LJ016-0076|Even then, however, irons across barred the ascent after a certain distance, and in no one case did a fugitive get clear away.|Even then, however, irons across barred the ascent after a certain distance, and in no one case did a fugitive get clear away. +LJ016-0077|A man named Lears, under sentence of transportation for an attempt at murder on board ship, got up part of the way,|A man named Lears, under sentence of transportation for an attempt at murder on board ship, got up part of the way, +LJ016-0078|but had to come down again covered with soot and filth just as the officers entered the ward.|but had to come down again covered with soot and filth just as the officers entered the ward. +LJ016-0079|Lears was rewarded by being obliged to wear cross irons on his legs, a punishment rarely inflicted in Newgate,|Lears was rewarded by being obliged to wear cross irons on his legs, a punishment rarely inflicted in Newgate, +LJ016-0080|and probably one of the few cases of a recurrence, but under proper safeguards and limitations, to the old system of chains.|and probably one of the few cases of a recurrence, but under proper safeguards and limitations, to the old system of chains. +LJ016-0081|On another occasion Mr. Cope the governor came in and missed a man.|On another occasion Mr. Cope the governor came in and missed a man. +LJ016-0082|The ward was one short of its number. What had become of the fellow?|The ward was one short of its number. What had become of the fellow? +LJ016-0084|the fugitive, uncomfortably ensconced in the flue, came down of his own accord, like Colonel Colt's raccoon.|the fugitive, uncomfortably ensconced in the flue, came down of his own accord, like Colonel Colt's raccoon. +LJ016-0085|After this great iron guards, just as are to be seen in lunatic asylums,|After this great iron guards, just as are to be seen in lunatic asylums, +LJ016-0086|were fixed over the fireplaces, and the prisoners had no longer access to the chimneys.|were fixed over the fireplaces, and the prisoners had no longer access to the chimneys. +LJ016-0087|Among the escapes still remembered was one in 1849, accomplished by a man who had been employed|Among the escapes still remembered was one in eighteen forty-nine, accomplished by a man who had been employed +LJ016-0088|working at the roof of the chapel on the female side.|working at the roof of the chapel on the female side. +LJ016-0090|Taking advantage of being unobserved, he got out through the roof on to the leads, and traveled along them towards No. 1, Newgate Street.|Taking advantage of being unobserved, he got out through the roof on to the leads, and traveled along them towards Number one, Newgate Street. +LJ016-0091|This was a public-house.|This was a public-house. +LJ016-0092|He stepped in at a garret window, coolly walked downstairs, and entered the bar.|He stepped in at a garret window, coolly walked downstairs, and entered the bar. +LJ016-0093|They asked him how he had cut his hand, which was bleeding, and he said he had done it while working up on the roof.|They asked him how he had cut his hand, which was bleeding, and he said he had done it while working up on the roof. +LJ016-0094|No further notice was taken of him; no one seemingly suspected that he was a prisoner, and he was suffered to walk off without let or hindrance.|No further notice was taken of him; no one seemingly suspected that he was a prisoner, and he was suffered to walk off without let or hindrance. +LJ016-0095|In 1853 three men escaped in company from one of the wards in the middle yard.|In eighteen fifty-three three men escaped in company from one of the wards in the middle yard. +LJ016-0096|They were penal servitude men, their names Bell, Brown, and Barry, and they were awaiting transfer to Leicester,|They were penal servitude men, their names Bell, Brown, and Barry, and they were awaiting transfer to Leicester, +LJ016-0097|which with Wakefield was utilized as a receptacle for convicts not going to Western Australia,|which with Wakefield was utilized as a receptacle for convicts not going to Western Australia, +LJ016-0098|or any of the new establishments at home, at Portland, Dartmoor, or elsewhere.|or any of the new establishments at home, at Portland, Dartmoor, or elsewhere. +LJ016-0099|These men managed to cut a hole in the ceiling of the ward near the iron cage on the landing, and so got access to the roof.|These men managed to cut a hole in the ceiling of the ward near the iron cage on the landing, and so got access to the roof. +LJ016-0100|At that time rope mats were still used as beds.|At that time rope mats were still used as beds. +LJ016-0101|One of the three, shamming ill, remained all day in his ward, where he employed himself unraveling the rope from the sleeping-mats.|One of the three, shamming ill, remained all day in his ward, where he employed himself unraveling the rope from the sleeping-mats. +LJ016-0102|By evening he manufactured a good long length,|By evening he manufactured a good long length, +LJ016-0103|and after all was quiet the three got on to the roof through the hole, and so on to Tyler's manufactory close by,|and after all was quiet the three got on to the roof through the hole, and so on to Tyler's manufactory close by, +LJ016-0104|whence they let themselves down into the street by the rope.|whence they let themselves down into the street by the rope. +LJ016-0105|These men were all in prison dress at the time of their escape, but one of their number, Bell, sent back his clothes a few days later by parcel's delivery,|These men were all in prison dress at the time of their escape, but one of their number, Bell, sent back his clothes a few days later by parcel's delivery, +LJ016-0106|with a civil note to the governor, saying he had no further use for them. All three fugitives were recaptured,|with a civil note to the governor, saying he had no further use for them. All three fugitives were recaptured, +LJ016-0107|Brown almost at once; then Barry, who was taken at the East End in a public-house where he had arranged to meet a pal.|Brown almost at once; then Barry, who was taken at the East End in a public-house where he had arranged to meet a pal. +LJ016-0108|The Newgate officers obtained information of this, and went to the spot, where they effected the capture,|The Newgate officers obtained information of this, and went to the spot, where they effected the capture, +LJ016-0109|but not till they had had an exciting chase down the street.|but not till they had had an exciting chase down the street. +LJ016-0110|The third, Bell, remained longest at large. He too was run into at a lodging in the Kingsland Road.|The third, Bell, remained longest at large. He too was run into at a lodging in the Kingsland Road. +LJ016-0111|The officers dropped on to him while he was still in bed, but as they came upstairs he jumped up and hid in a cupboard.|The officers dropped on to him while he was still in bed, but as they came upstairs he jumped up and hid in a cupboard. +LJ016-0112|All three after recapture passed on, as originally intended, to Leicester, where they did their "bit" and were released;|All three after recapture passed on, as originally intended, to Leicester, where they did their "bit" and were released; +LJ016-0113|but only to be taken soon afterwards for a fresh offense, and again pass through Newgate with sentences of penal servitude.|but only to be taken soon afterwards for a fresh offense, and again pass through Newgate with sentences of penal servitude. +LJ016-0114|A later case was still more remarkable, as it was effected after the alteration of the prison and its reconstruction on the newest lines.|A later case was still more remarkable, as it was effected after the alteration of the prison and its reconstruction on the newest lines. +LJ016-0115|A sailor, Krapps by name, occupied one of the upper cells in the new block.|A sailor, Krapps by name, occupied one of the upper cells in the new block. +LJ016-0116|The doors, through incomplete knowledge of prison needs, were not, as now, sheeted with iron.|The doors, through incomplete knowledge of prison needs, were not, as now, sheeted with iron. +LJ016-0117|The prisoner had nothing to deal with but wooden panels, and by dint of cutting and chopping he got both the lower panels out.|The prisoner had nothing to deal with but wooden panels, and by dint of cutting and chopping he got both the lower panels out. +LJ016-0118|Through the aperture he crept out on to the landing at the dead of night, and so down into the central space of the building.|Through the aperture he crept out on to the landing at the dead of night, and so down into the central space of the building. +LJ016-0119|Under superior orders all the doors and gates of this block were left open at night, to allow the night watchman to pass freely to all parts.|Under superior orders all the doors and gates of this block were left open at night, to allow the night watchman to pass freely to all parts. +LJ016-0120|This was considered safer than intrusting him with keys.|This was considered safer than intrusting him with keys. +LJ016-0121|Krapps walked at once into the yard and across to the female side, where he found some of the washing still hanging out to dry.|Krapps walked at once into the yard and across to the female side, where he found some of the washing still hanging out to dry. +LJ016-0122|He made a strong rope with several of the sheets; then, returning to the male yard,|He made a strong rope with several of the sheets; then, returning to the male yard, +LJ016-0123|got hold of the step ladder used in lighting the gas, and which under our more careful supervision would have been, as now-a-days, chained up.|got hold of the step ladder used in lighting the gas, and which under our more careful supervision would have been, as now-a-days, chained up. +LJ016-0124|Cutting the cord which fastened the two legs of the step ladder, he opened them out and made one long length;|Cutting the cord which fastened the two legs of the step ladder, he opened them out and made one long length; +LJ016-0125|with this, placed against the wall near the chevaux-de-frise, he made an escalade.|with this, placed against the wall near the chevaux-de-frise, he made an escalade. +LJ016-0126|The top of the wall was gained without difficulty.|The top of the wall was gained without difficulty. +LJ016-0127|Along this Krapps crawled, and then dropped down on to the cook-house.|Along this Krapps crawled, and then dropped down on to the cook-house. +LJ016-0128|He now put in requisition the rope made of the sheets, and with its help lowered himself into the street.|He now put in requisition the rope made of the sheets, and with its help lowered himself into the street. +LJ016-0129|Down below were market-carts waiting for daylight, and among them Krapps found a refuge and friends.|Down below were market-carts waiting for daylight, and among them Krapps found a refuge and friends. +LJ016-0130|The first intimation of his escape was afforded by the police, who informed the prison authorities next day that a rope was hanging down from the cook-house roof.|The first intimation of his escape was afforded by the police, who informed the prison authorities next day that a rope was hanging down from the cook-house roof. +LJ016-0131|Nothing more was heard of Krapps. The curious thing in his case was that his offense was a trifling one;|Nothing more was heard of Krapps. The curious thing in his case was that his offense was a trifling one; +LJ016-0132|he was still untried, but would almost certainly have escaped with a minor penalty, say of three or four months' imprisonment.|he was still untried, but would almost certainly have escaped with a minor penalty, say of three or four months' imprisonment. +LJ016-0133|There is, however, no explanation of the motives which prompt prisoners to attempt escapes.|There is, however, no explanation of the motives which prompt prisoners to attempt escapes. +LJ016-0134|Cases well authenticated have been known of men who had all but completed their sentences, and for whom the prison gates would open within a few days,|Cases well authenticated have been known of men who had all but completed their sentences, and for whom the prison gates would open within a few days, +LJ016-0135|who yet faced extraordinary risks to advance their enlargement by only a few hours.|who yet faced extraordinary risks to advance their enlargement by only a few hours. +LJ016-0136|On the other hand, at the great convict establishments, such is the moral restraint of a systematic discipline,|On the other hand, at the great convict establishments, such is the moral restraint of a systematic discipline, +LJ016-0137|that numbers of men, "lifers," and others with ten, fourteen, or twenty years to do, can be trusted to work out of doors without bolts and bars|that numbers of men, "lifers," and others with ten, fourteen, or twenty years to do, can be trusted to work out of doors without bolts and bars +LJ016-0138|at a distance from the prison.|at a distance from the prison. +LJ016-0139|The last escape from Newgate was only three years ago, and occurred just before the final closing of the prison.|The last escape from Newgate was only three years ago, and occurred just before the final closing of the prison. +LJ016-0140|No report of it was made public, as the man was almost immediately recaptured.|No report of it was made public, as the man was almost immediately recaptured. +LJ016-0141|He was at work under the supervision of the artisan warder of the prison,|He was at work under the supervision of the artisan warder of the prison, +LJ016-0142|who permitted him to go up on to the roof of the old wards, in order to throw water for flushing purposes down a shoot.|who permitted him to go up on to the roof of the old wards, in order to throw water for flushing purposes down a shoot. +LJ016-0143|He was out of sight while so employed, and remained so long absent that the warder, becoming uneasy, went in search of him.|He was out of sight while so employed, and remained so long absent that the warder, becoming uneasy, went in search of him. +LJ016-0144|He had disappeared.|He had disappeared. +LJ016-0145|Encouraged by the shouts and signals of some workmen employed on a building outside, the prisoner made one of the most marvellous jumps on record,|Encouraged by the shouts and signals of some workmen employed on a building outside, the prisoner made one of the most marvellous jumps on record, +LJ016-0146|from the building he was on to a distant wall, with a drop of sixty feet between.|from the building he was on to a distant wall, with a drop of sixty feet between. +LJ016-0147|Then he ran along the coping of the wall towards its angle with Tyler's manufactory, and dropped down on to the gridiron below.|Then he ran along the coping of the wall towards its angle with Tyler's manufactory, and dropped down on to the gridiron below. +LJ016-0148|This was not strong enough to carry him, and he fell through.|This was not strong enough to carry him, and he fell through. +LJ016-0149|Suicides and executions were, however, always the most effectual methods of making exit from durance.|Suicides and executions were, however, always the most effectual methods of making exit from durance. +LJ016-0150|Suicides at Newgate were numerous enough, but they seldom possessed any novel or unusual features;|Suicides at Newgate were numerous enough, but they seldom possessed any novel or unusual features; +LJ016-0151|prison suicides seldom do, except as regards ingenuity and determination.|prison suicides seldom do, except as regards ingenuity and determination. +LJ016-0152|Only great resolution indeed, persisted in to the bitter end,|Only great resolution indeed, persisted in to the bitter end, +LJ016-0153|would make death a certainty, so limited and imperfect are the means generally available.|would make death a certainty, so limited and imperfect are the means generally available. +LJ016-0154|When a bit of rope carefully secreted,|When a bit of rope carefully secreted, +LJ016-0155|braces, shoe-strings, shirt torn into strips are the only instruments, and a bar or small hook|braces, shoe-strings, shirt torn into strips are the only instruments, and a bar or small hook +LJ016-0156|at no elevation affords the only drop, strangulation would seldom supervene but for the resolution of the miserable felo de se.|at no elevation affords the only drop, strangulation would seldom supervene but for the resolution of the miserable felo de se. +LJ016-0157|One curious instance of a suicide carried out under the most adverse and extraordinary circumstances may be quoted.|One curious instance of a suicide carried out under the most adverse and extraordinary circumstances may be quoted. +LJ016-0158|It was that of a "Long Firm" swindler, by name Johnson,|It was that of a "Long Firm" swindler, by name Johnson, +LJ016-0159|who contrived to hang himself from a hammock hook only eighteen inches from the ground.|who contrived to hang himself from a hammock hook only eighteen inches from the ground. +LJ016-0160|The noose was one of his hammock straps, which he buckled round his throat.|The noose was one of his hammock straps, which he buckled round his throat. +LJ016-0161|Having carefully spread out a blanket on the floor just below the hammock as it lay suspended,|Having carefully spread out a blanket on the floor just below the hammock as it lay suspended, +LJ016-0162|he fastened one end of the strap above mentioned to the hook, and then fell down.|he fastened one end of the strap above mentioned to the hook, and then fell down. +LJ016-0163|He might have saved himself at any moment by merely extending an arm; but he lay there patiently till death supervened.|He might have saved himself at any moment by merely extending an arm; but he lay there patiently till death supervened. +LJ016-0164|When discovered next morning, quite dead, it was found that the strap actually did not touch his throat;|When discovered next morning, quite dead, it was found that the strap actually did not touch his throat; +LJ016-0165|three fingers might have been inserted between it and the flesh; the pressure was all on the arteries behind the ears,|three fingers might have been inserted between it and the flesh; the pressure was all on the arteries behind the ears, +LJ016-0166|and surgical opinion stated that the stoppage of circulation was the cause of death.|and surgical opinion stated that the stoppage of circulation was the cause of death. +LJ016-0167|Probably dissolution came as easily and almost without pain.|Probably dissolution came as easily and almost without pain. +LJ016-0168|A laudable desire to invest executions with more and more solemnity and decorum gained ground as they became more rare.|A laudable desire to invest executions with more and more solemnity and decorum gained ground as they became more rare. +LJ016-0169|As more humane principles were introduced into prison management,|As more humane principles were introduced into prison management, +LJ016-0170|greater attention was paid to the capital convicts, and the horrors of their situation while awaiting sentence|greater attention was paid to the capital convicts, and the horrors of their situation while awaiting sentence +LJ016-0171|were as far as possible mitigated and toned down. But there was little improvement in the ceremony itself.|were as far as possible mitigated and toned down. But there was little improvement in the ceremony itself. +LJ016-0172|There were still untoward accidents occasionally at executions, and even the chief practitioner of recent times, Calcraft,|There were still untoward accidents occasionally at executions, and even the chief practitioner of recent times, Calcraft, +LJ016-0173|was not always to be trusted to do his fell work efficiently.|was not always to be trusted to do his fell work efficiently. +LJ016-0174|Having mentioned Calcraft's name,|Having mentioned Calcraft's name, +LJ016-0175|I may be permitted to digress for a moment to give a few particulars concerning the last officially appointed hangman of the city of London.|I may be permitted to digress for a moment to give a few particulars concerning the last officially appointed hangman of the city of London. +LJ016-0176|After Calcraft's resignation no successor was really appointed.|After Calcraft's resignation no successor was really appointed. +LJ016-0177|Marwood, whose name is so familiar with the present generation,|Marwood, whose name is so familiar with the present generation, +LJ016-0178|had no official status, and was merely an operator selected by the Corporation, and who, on the strength of it,|had no official status, and was merely an operator selected by the Corporation, and who, on the strength of it, +LJ016-0179|contracted with sheriffs and conveners to work by the job.|contracted with sheriffs and conveners to work by the job. +LJ016-0180|But Calcraft regularly succeeded Foxen, who followed Botting, and Dennis, the actor in the 1780 riots.|But Calcraft regularly succeeded Foxen, who followed Botting, and Dennis, the actor in the seventeen eighty riots. +LJ016-0181|Calcraft was born at Baddow, in Essex, in 1800;|Calcraft was born at Baddow, in Essex, in eighteen hundred; +LJ016-0182|he was a shoemaker by trade, and settled in London after his marriage in 1825.|he was a shoemaker by trade, and settled in London after his marriage in eighteen twenty-five. +LJ016-0183|The story goes, that about 1828|The story goes, that about eighteen twenty-eight +LJ016-0184|his attention was drawn early one morning to a man who leant against a lamp-post in Finsbury Square, coughing violently.|his attention was drawn early one morning to a man who leant against a lamp-post in Finsbury Square, coughing violently. +LJ016-0185|Calcraft, who, in spite of the dreadful calling he subsequently followed, was always reputed a kindly man,|Calcraft, who, in spite of the dreadful calling he subsequently followed, was always reputed a kindly man, +LJ016-0186|invited the man with the cough to enter a neighboring house and try a little peppermint for it.|invited the man with the cough to enter a neighboring house and try a little peppermint for it. +LJ016-0187|The other accepted, and they got into conversation.|The other accepted, and they got into conversation. +LJ016-0188|He told Calcraft that he was Foxen the executioner, and that he was that moment on his way to Newgate to hang a man,|He told Calcraft that he was Foxen the executioner, and that he was that moment on his way to Newgate to hang a man, +LJ016-0189|but that his cough was getting so much the master of him that he feared he would not be able to carry on his duties much longer.|but that his cough was getting so much the master of him that he feared he would not be able to carry on his duties much longer. +LJ016-0190|"I have no idea who the sheriffs will get to do the work after me," said Foxen, adding that his assistant, Tom Cheshire,|"I have no idea who the sheriffs will get to do the work after me," said Foxen, adding that his assistant, Tom Cheshire, +LJ016-0191|was given to drink, and not to be trusted.|was given to drink, and not to be trusted. +LJ016-0192|"I think I could do that sort of job," said Calcraft, on the spur of the moment.|"I think I could do that sort of job," said Calcraft, on the spur of the moment. +LJ016-0193|Foxen asked him his name and address, and went away.|Foxen asked him his name and address, and went away. +LJ016-0194|Calcraft thought no more of what had occurred till the next sessions at the Old Bailey, when the sheriffs sent for him,|Calcraft thought no more of what had occurred till the next sessions at the Old Bailey, when the sheriffs sent for him, +LJ016-0195|and offered him the post of executioner for the city of London and Middlesex.|and offered him the post of executioner for the city of London and Middlesex. +LJ016-0196|He accepted, having at first Tom Cheshire as his assistant, then for a time, when Cheshire was dismissed for drunkenness, a man named Osborne.|He accepted, having at first Tom Cheshire as his assistant, then for a time, when Cheshire was dismissed for drunkenness, a man named Osborne. +LJ016-0197|After that he worked alone.|After that he worked alone. +LJ016-0198|I cannot find that Calcraft was sworn in when appointed, or any exact information when the old forbidding ceremony ceased to be practiced.|I cannot find that Calcraft was sworn in when appointed, or any exact information when the old forbidding ceremony ceased to be practiced. +LJ016-0199|It was customary to make the executioner take the Bible in his hand, and swear solemnly that he would dispatch every criminal condemned to die,|It was customary to make the executioner take the Bible in his hand, and swear solemnly that he would dispatch every criminal condemned to die, +LJ016-0200|without favoring father or mother or any other relation or friend.|without favoring father or mother or any other relation or friend. +LJ016-0201|When he had taken the oath he was dismissed with the words, "Get thee hence, wretch!"|When he had taken the oath he was dismissed with the words, "Get thee hence, wretch!" +LJ016-0202|Calcraft's emoluments were a guinea per week, and an extra guinea for every execution.|Calcraft's emoluments were a guinea per week, and an extra guinea for every execution. +LJ016-0203|He got besides half-a-crown for every man he flogged, and an allowance to provide cats or birch rods.|He got besides half-a-crown for every man he flogged, and an allowance to provide cats or birch rods. +LJ016-0204|For acting as executioner of Horsemonger Lane Jail|For acting as executioner of Horsemonger Lane Jail +LJ016-0205|he received a retaining fee of £5, 5 shillings, with the usual guinea for each job;|he received a retaining fee of five pounds, five shillings, with the usual guinea for each job; +LJ016-0206|he was also at liberty to engage himself in the country, where he demanded and was paid £10 on each occasion.|he was also at liberty to engage himself in the country, where he demanded and was paid ten pounds on each occasion. +LJ016-0207|It was not always easy to get a hangman so cheap, as I have already indicated on a previous page.|It was not always easy to get a hangman so cheap, as I have already indicated on a previous page. +LJ016-0208|The onus and responsibility of carrying out the sentence is personal to the sheriff. A good story is told illustrating this.|The onus and responsibility of carrying out the sentence is personal to the sheriff. A good story is told illustrating this. +LJ016-0209|Some wags in Scotland seized Calcraft and kept him in durance the night before the execution.|Some wags in Scotland seized Calcraft and kept him in durance the night before the execution. +LJ016-0210|Meanwhile the convener or sheriff was in despair, expecting that, failing the executioner, he would have to do the job himself.|Meanwhile the convener or sheriff was in despair, expecting that, failing the executioner, he would have to do the job himself. +LJ016-0211|But, fortunately for him, just at the last moment Calcraft was set free.|But, fortunately for him, just at the last moment Calcraft was set free. +LJ016-0212|Calcraft's salary was more than the proverbial "thirteenpence halfpenny -- hangman's wages."|Calcraft's salary was more than the proverbial "thirteenpence halfpenny -- hangman's wages." +LJ016-0213|The origin of this expression dates, it is said, from the time when the Scottish mark,|The origin of this expression dates, it is said, from the time when the Scottish mark, +LJ016-0214|a silver coin bearing the same relation to the Scottish pound that an English shilling does to an English pound, was made to pass current in England.|a silver coin bearing the same relation to the Scottish pound that an English shilling does to an English pound, was made to pass current in England. +LJ016-0215|The mark was valued at thirteenpence halfpenny, or rather more than the shilling, which from time immemorial had been the hangman's wages.|The mark was valued at thirteenpence halfpenny, or rather more than the shilling, which from time immemorial had been the hangman's wages. +LJ016-0216|That very ancient perquisite the convict's clothes was never claimed by Calcraft, and it may be doubted whether he was entitled to it.|That very ancient perquisite the convict's clothes was never claimed by Calcraft, and it may be doubted whether he was entitled to it. +LJ016-0217|On one particular occasion, however, he got them. A gentleman whose sins brought him to the gallows at Maidstone|On one particular occasion, however, he got them. A gentleman whose sins brought him to the gallows at Maidstone +LJ016-0218|wished to do Calcraft a good turn, and sent to his London tailor for a complete new suit, in which he appeared at his execution.|wished to do Calcraft a good turn, and sent to his London tailor for a complete new suit, in which he appeared at his execution. +LJ016-0219|He expressly bequeathed them to Calcraft, who was graciously pleased to accept them.|He expressly bequeathed them to Calcraft, who was graciously pleased to accept them. +LJ016-0220|On another occasion an importunate person begged Calcraft eagerly to claim his right to the clothes, and give them to him.|On another occasion an importunate person begged Calcraft eagerly to claim his right to the clothes, and give them to him. +LJ016-0221|Calcraft consented, got and bestowed the clothes, only to find that the person he had obliged exhibited them publicly.|Calcraft consented, got and bestowed the clothes, only to find that the person he had obliged exhibited them publicly. +LJ016-0222|It may be added that of late years the clothes in which a convict has suffered are invariably burnt.|It may be added that of late years the clothes in which a convict has suffered are invariably burnt. +LJ016-0223|Capital convicts go to the gallows in their own clothing, and not in prison dress, unless the former is quite unfit to be worn.|Capital convicts go to the gallows in their own clothing, and not in prison dress, unless the former is quite unfit to be worn. +LJ016-0224|Calcraft shared the odium which his office, not strangely, has always inspired. But he was admitted into the jail,|Calcraft shared the odium which his office, not strangely, has always inspired. But he was admitted into the jail, +LJ016-0225|which his predecessors were not, and who were paid their wages over the gate to obviate the necessity for letting them enter.|which his predecessors were not, and who were paid their wages over the gate to obviate the necessity for letting them enter. +LJ016-0226|To this curious etiquette was due the appointment of an official whose office has long since disappeared, "the yeoman of the halter,"|To this curious etiquette was due the appointment of an official whose office has long since disappeared, "the yeoman of the halter," +LJ016-0227|whose business it was to provide the rope and do the pinioning, and who was paid a fee of five shillings.|whose business it was to provide the rope and do the pinioning, and who was paid a fee of five shillings. +LJ016-0228|They did not dislike Calcraft, however, at Newgate. He was an illiterate, simple-minded man, who scarcely remembered what executions he had performed.|They did not dislike Calcraft, however, at Newgate. He was an illiterate, simple-minded man, who scarcely remembered what executions he had performed. +LJ016-0229|He kept no record of them, and when asked questions, referred to the officers of the jail.|He kept no record of them, and when asked questions, referred to the officers of the jail. +LJ016-0230|His nature must have been kindly.|His nature must have been kindly. +LJ016-0231|When he came to the prison for his wages his grandchildren often accompanied him, affectionately clinging to his hands;|When he came to the prison for his wages his grandchildren often accompanied him, affectionately clinging to his hands; +LJ016-0232|and he owned a pet pony which would follow him about like a dog.|and he owned a pet pony which would follow him about like a dog. +LJ016-0233|In his own profession|In his own profession +LJ016-0234|he was not unskilful, but he proceeded entirely by rule of thumb, leaving the result very much to chance and the strength of the rope.|he was not unskilful, but he proceeded entirely by rule of thumb, leaving the result very much to chance and the strength of the rope. +LJ016-0235|He was so much in favor of short drops that his immediate successor, Marwood, stigmatized him as "short-drop" man.|He was so much in favor of short drops that his immediate successor, Marwood, stigmatized him as "short-drop" man. +LJ016-0236|Marwood being, on the other hand, in favor of giving a man as much rope as possible.|Marwood being, on the other hand, in favor of giving a man as much rope as possible. +LJ016-0237|With Calcraft's method there were undoubtedly many failures, and it was a common custom for him to go below the gallows|With Calcraft's method there were undoubtedly many failures, and it was a common custom for him to go below the gallows +LJ016-0238|"just to steady their legs a little;" in other words, to add his weight to that of the hanging bodies.|"just to steady their legs a little;" in other words, to add his weight to that of the hanging bodies. +LJ016-0239|Marwood till latterly seemed to have done his work more effectually, and has been known to give as much as six feet fall.|Marwood till latterly seemed to have done his work more effectually, and has been known to give as much as six feet fall. +LJ016-0240|This generally produces instantaneous death, although cases where complete fracture of the spinal cord occurred are said to be rare.|This generally produces instantaneous death, although cases where complete fracture of the spinal cord occurred are said to be rare. +LJ016-0241|Calcraft served the city of London till 1874, when he was pensioned at the rate of twenty-five shillings per week.|Calcraft served the city of London till eighteen seventy-four, when he was pensioned at the rate of twenty-five shillings per week. +LJ016-0242|The last execution at which he acted was that of Godwin, on the 25th May, 1874.|The last execution at which he acted was that of Godwin, on the twenty-fifth May, eighteen seventy-four. +LJ016-0243|Marwood, who succeeded him, and who died while these sheets were in the press, was a Lincolnshire man, a native of Horncastle,|Marwood, who succeeded him, and who died while these sheets were in the press, was a Lincolnshire man, a native of Horncastle, +LJ016-0244|who first took to the work from predilection, and the idea of being useful in his generation, as he himself assured the writer of these pages.|who first took to the work from predilection, and the idea of being useful in his generation, as he himself assured the writer of these pages. +LJ016-0245|Until the time of his death he kept a small shop close to the church in Horncastle.|Until the time of his death he kept a small shop close to the church in Horncastle. +LJ016-0246|Over the door, in gilt letters, were the words "Crown Office"; in the window was a pile of official envelopes, ostentatiously displayed,|Over the door, in gilt letters, were the words "Crown Office"; in the window was a pile of official envelopes, ostentatiously displayed, +LJ016-0247|while round about were shoe-strings, boot-laces, and lasts. Marwood, strange to say, followed the same trade as Calcraft.|while round about were shoe-strings, boot-laces, and lasts. Marwood, strange to say, followed the same trade as Calcraft. +LJ016-0248|Marwood was proud of his calling, and when questioned as to whether his process was satisfactory, replied that he heard "no complaints."|Marwood was proud of his calling, and when questioned as to whether his process was satisfactory, replied that he heard "no complaints." +LJ016-0249|The strange competition amongst hundreds to succeed Marwood is a strange fact too recently before the public to need mention here.|The strange competition amongst hundreds to succeed Marwood is a strange fact too recently before the public to need mention here. +LJ016-0250|It may, however, be remarked that the wisdom of appointing any regular hangman is very open to question,|It may, however, be remarked that the wisdom of appointing any regular hangman is very open to question, +LJ016-0251|and must be strongly deprecated on moral grounds, as tending to the utter degradation of one individual.|and must be strongly deprecated on moral grounds, as tending to the utter degradation of one individual. +LJ016-0252|Possibly such changes may be introduced into the method of execution|Possibly such changes may be introduced into the method of execution +LJ016-0253|that the ceremony may be made more mechanical, thus rendering the personal intervention of a skilled functionary unnecessary.|that the ceremony may be made more mechanical, thus rendering the personal intervention of a skilled functionary unnecessary. +LJ016-0254|Executions long continued to be in public, in spite of remonstrance and reprobation.|Executions long continued to be in public, in spite of remonstrance and reprobation. +LJ016-0255|The old prejudices, such as that which enlisted Dr. Johnson on the side of the Tyburn procession, still lingered and prevented any change.|The old prejudices, such as that which enlisted Dr. Johnson on the side of the Tyburn procession, still lingered and prevented any change. +LJ016-0256|It was thought that capital punishment would lose its deterrent effect if it ceased to be public,|It was thought that capital punishment would lose its deterrent effect if it ceased to be public, +LJ016-0257|and the raison d'être of the penalty, which in principle so many opposed, would be gone.|and the raison d'être of the penalty, which in principle so many opposed, would be gone. +LJ016-0258|This line of argument prevailed over the manifest horrors of the spectacle. These increased as time passed.|This line of argument prevailed over the manifest horrors of the spectacle. These increased as time passed. +LJ016-0259|The graphic and terrible account given by Charles Dickens of the awful scene before Horsemonger Lane Jail, at the execution of the Mannings,|The graphic and terrible account given by Charles Dickens of the awful scene before Horsemonger Lane Jail, at the execution of the Mannings, +LJ016-0260|has already been quoted. Again, the concourse of people collected in front of Newgate to witness the execution,|has already been quoted. Again, the concourse of people collected in front of Newgate to witness the execution, +LJ016-0261|simultaneously, of the five pirates, part of the mutinous crew of the 'Flowery Land,' was greater than on any previous occasion.|simultaneously, of the five pirates, part of the mutinous crew of the 'Flowery Land,' was greater than on any previous occasion. +LJ016-0262|It was a callous, careless crowd of coarse-minded, semi-brutalized folk, who came to enjoy themselves.|It was a callous, careless crowd of coarse-minded, semi-brutalized folk, who came to enjoy themselves. +LJ016-0263|Few, if any, showed any feeling of terror, none were impressed with the solemnity, or realized the warning which the sight conveyed.|Few, if any, showed any feeling of terror, none were impressed with the solemnity, or realized the warning which the sight conveyed. +LJ016-0264|The upturned faces of the eager spectators resembled those of the 'gods' at Drury Lane on Boxing Night;|The upturned faces of the eager spectators resembled those of the 'gods' at Drury Lane on Boxing Night; +LJ016-0265|the crowd had come to witness a popular and gratuitous public performance -- better than a prize-fight or a play.|the crowd had come to witness a popular and gratuitous public performance -- better than a prize-fight or a play. +LJ016-0266|No notion that they were assisting at a vindication of the law filled the minds of those present with dread.|No notion that they were assisting at a vindication of the law filled the minds of those present with dread. +LJ016-0267|On the contrary, the prevailing sentiment was one of satisfaction at the success of the spectacle.|On the contrary, the prevailing sentiment was one of satisfaction at the success of the spectacle. +LJ016-0268|The remarks heard amongst the crowd were of coarse approval.|The remarks heard amongst the crowd were of coarse approval. +LJ016-0271|The reply evinced equal satisfaction, and the speaker, with a profane oath, declared that he would like to act as Jack Ketch to the whole lot.|The reply evinced equal satisfaction, and the speaker, with a profane oath, declared that he would like to act as Jack Ketch to the whole lot. +LJ016-0272|To the disgrace of the better-educated and better-bred public, executions could still command the attendance of curious aristocrats from the West End.|To the disgrace of the better-educated and better-bred public, executions could still command the attendance of curious aristocrats from the West End. +LJ016-0273|At Müller's execution there was great competition for front seats,|At Müller's execution there was great competition for front seats, +LJ016-0274|and the windows of the opposite houses, which commanded a good view, as usual fetched high prices.|and the windows of the opposite houses, which commanded a good view, as usual fetched high prices. +LJ016-0275|As much as £25 was paid for a first-floor front on this occasion.|As much as twenty-five pounds was paid for a first-floor front on this occasion. +LJ016-0276|Never, indeed, had an execution been more generally patronized.|Never, indeed, had an execution been more generally patronized. +LJ016-0277|This is proved by contemporary accounts, especially one graphic and realistic article which appeared in the 'Times,'|This is proved by contemporary accounts, especially one graphic and realistic article which appeared in the 'Times,' +LJ016-0278|and which contributed in no small degree to the introduction of private executions. A great crowd was expected, and a great crowd came.|and which contributed in no small degree to the introduction of private executions. A great crowd was expected, and a great crowd came. +LJ016-0279|They collected over night in the bright light of a November moon.|They collected over night in the bright light of a November moon. +LJ016-0280|"There were well-dressed and ill-dressed, old men and lads, women and girls."|"There were well-dressed and ill-dressed, old men and lads, women and girls." +LJ016-0281|Rain fell heavily at intervals, but did not thin the concourse.|Rain fell heavily at intervals, but did not thin the concourse. +LJ016-0282|"Till three o'clock it was one long revelry of songs and laughter, shouting, and often quarreling, though,|"Till three o'clock it was one long revelry of songs and laughter, shouting, and often quarreling, though, +LJ016-0283|to do them mere justice, there was at least till then a half-drunken ribald gaiety among the crowd that made them all akin."|to do them mere justice, there was at least till then a half-drunken ribald gaiety among the crowd that made them all akin." +LJ016-0284|There were preachers among the crowd, but they could not get a patient hearing.|There were preachers among the crowd, but they could not get a patient hearing. +LJ016-0285|Then one struck up the hymn of the Promised Land, and the refrain was at once taken up with a mighty chorus --|Then one struck up the hymn of the Promised Land, and the refrain was at once taken up with a mighty chorus -- +LJ016-0286|Oh, my! Think I've got to die.|Oh, my! Think I've got to die. +LJ016-0287|This was presently superseded by a fresh catch --|This was presently superseded by a fresh catch -- +LJ016-0288|"Müller, Müller, He's the man," till a diversion was created by the appearance of the gallows, which was received with continuous yells.|"Müller, Müller, He's the man," till a diversion was created by the appearance of the gallows, which was received with continuous yells. +LJ016-0289|As day broke the character of the crowd was betrayed.|As day broke the character of the crowd was betrayed. +LJ016-0290|There were but few women, except of the most degraded sort; the men were mostly young men --|There were but few women, except of the most degraded sort; the men were mostly young men -- +LJ016-0291|sharpers, thieves, gamblers, betting men, the outsiders of the boxing ring,|sharpers, thieves, gamblers, betting men, the outsiders of the boxing ring, +LJ016-0292|bricklayers' laborers, dock workmen, German artisans and sugar-bakers|bricklayers' laborers, dock workmen, German artisans and sugar-bakers +LJ016-0293|with the rakings of cheap singing-halls and billiard-rooms, the fast young men of London.|with the rakings of cheap singing-halls and billiard-rooms, the fast young men of London. +LJ016-0294|But all, whether young or old, men or women, seemed to know nothing, feel nothing, to have no object but the gallows, and to laugh,|But all, whether young or old, men or women, seemed to know nothing, feel nothing, to have no object but the gallows, and to laugh, +LJ016-0295|curse, or shout, as in this heaving and struggling forward they gained or lost in their strong efforts to get nearer where Müller was to die.|curse, or shout, as in this heaving and struggling forward they gained or lost in their strong efforts to get nearer where Müller was to die. +LJ016-0296|The actual execution made some impression.|The actual execution made some impression. +LJ016-0297|The crowd was for a moment awed and stilled by the quiet rapid passage from life to death!|The crowd was for a moment awed and stilled by the quiet rapid passage from life to death! +LJ016-0298|But before "the slight slow vibrations of the body had well ended,|But before "the slight slow vibrations of the body had well ended, +LJ016-0299|robbery and violence, loud laughing, oaths, fighting, obscene conduct, and still more filthy language reigned round the gallows far and near.|robbery and violence, loud laughing, oaths, fighting, obscene conduct, and still more filthy language reigned round the gallows far and near. +LJ016-0300|Such too the scene remained with little change or respite till the old hangman (Calcraft) slunk again along the drop,|Such too the scene remained with little change or respite till the old hangman (Calcraft) slunk again along the drop, +LJ016-0301|amid hisses and sneering inquiries of what he had had to drink that morning.|amid hisses and sneering inquiries of what he had had to drink that morning. +LJ016-0302|He, after failing once to cut the rope, made a second attempt more successfully, and the body of Müller disappeared from view.|He, after failing once to cut the rope, made a second attempt more successfully, and the body of Müller disappeared from view. +LJ016-0303|It was preposterous to claim for such a scene as this that it conveyed any great moral lesson, or had any deterring influence.|It was preposterous to claim for such a scene as this that it conveyed any great moral lesson, or had any deterring influence. +LJ016-0304|Numbers of humane and thoughtful persons had long been convinced of this.|Numbers of humane and thoughtful persons had long been convinced of this. +LJ016-0305|Already the urgent necessity for abolishing public executions had been brought before the House of Commons by Mr. Hibbert,|Already the urgent necessity for abolishing public executions had been brought before the House of Commons by Mr. Hibbert, +LJ016-0306|and the question, as part of the whole subject of capital punishment, had been referred to a royal commission in January 1864.|and the question, as part of the whole subject of capital punishment, had been referred to a royal commission in January eighteen sixty-four. +LJ016-0307|Full evidence was taken on all points, and on that regarding public executions there was a great preponderance of opinion towards their abolition,|Full evidence was taken on all points, and on that regarding public executions there was a great preponderance of opinion towards their abolition, +LJ016-0308|yet the witnesses were not unanimous.|yet the witnesses were not unanimous. +LJ016-0309|Some of the judges would have retained the public spectacle; the ordinary of Newgate was not certain that public executions were not the best.|Some of the judges would have retained the public spectacle; the ordinary of Newgate was not certain that public executions were not the best. +LJ016-0310|Another distinguished witness feared|Another distinguished witness feared +LJ016-0311|that any secrecy in the treatment of the condemned would invest them with a new and greater interest, which was much to be deprecated.|that any secrecy in the treatment of the condemned would invest them with a new and greater interest, which was much to be deprecated. +LJ016-0312|Foreign witnesses, too, were in favor of publicity.|Foreign witnesses, too, were in favor of publicity. +LJ016-0313|On the other hand, Lords Cranworth and Wensleydale recommended private executions; so did Mr. Spencer Walpole, M.P.|On the other hand, Lords Cranworth and Wensleydale recommended private executions; so did Mr. Spencer Walpole, M.P. +LJ016-0314|Sir George Grey thought there was a growing feeling in favor of executions within the prison precincts.|Sir George Grey thought there was a growing feeling in favor of executions within the prison precincts. +LJ016-0315|Colonel (now Sir Edmund) Henderson was strongly in favor of them,|Colonel (now Sir Edmund) Henderson was strongly in favor of them, +LJ016-0316|based on his experience of them in Western Australia. He not only thought them likely to be more deterrent,|based on his experience of them in Western Australia. He not only thought them likely to be more deterrent, +LJ016-0317|but believed that a public ceremony destroyed the whole value of an execution.|but believed that a public ceremony destroyed the whole value of an execution. +LJ016-0318|Other officials, great lawyers, governors of prisons, and chaplains supported this view.|Other officials, great lawyers, governors of prisons, and chaplains supported this view. +LJ016-0319|The only doubts expressed were as to the sufficiency of the safeguards, as to the certainty of death and its subsequent publication.|The only doubts expressed were as to the sufficiency of the safeguards, as to the certainty of death and its subsequent publication. +LJ016-0320|But these, it was thought, might be provided by the admission of the press and the holding of a coroner's inquest.|But these, it was thought, might be provided by the admission of the press and the holding of a coroner's inquest. +LJ016-0321|Duly impressed with the weight of evidence in favor of abolition,|Duly impressed with the weight of evidence in favor of abolition, +LJ016-0322|the commission recommended that death sentences should be carried out within the jail, under such regulations as might be considered necessary|the commission recommended that death sentences should be carried out within the jail, under such regulations as might be considered necessary +LJ016-0323|to prevent abuse, and satisfy the public that the law had been complied with.|to prevent abuse, and satisfy the public that the law had been complied with. +LJ016-0324|But it is curious to note that there were several dissentients among the commissioners to this paragraph of the report.|But it is curious to note that there were several dissentients among the commissioners to this paragraph of the report. +LJ016-0325|The judge of the Admiralty Court, the Right Hon. Stephen Lushington, the Right Hon. James Moncrieff,|The judge of the Admiralty Court, the Right Hon. Stephen Lushington, the Right Hon. James Moncrieff, +LJ016-0326|Lord Advocate, Mr. Charles Neate, Mr. William Ewart, and last, but not least, Mr. John Bright|Lord Advocate, Mr. Charles Neate, Mr. William Ewart, and last, but not least, Mr. John Bright +LJ016-0327|declared that they were not prepared to agree to the resolution respecting private executions.|declared that they were not prepared to agree to the resolution respecting private executions. +LJ016-0328|Nevertheless, in the very next session|Nevertheless, in the very next session +LJ016-0329|a bill was introduced by Mr. Hibbert, M.P., and accepted by the Government, providing for the future carrying out of executions within prisons.|a bill was introduced by Mr. Hibbert, M.P., and accepted by the Government, providing for the future carrying out of executions within prisons. +LJ016-0330|It was read for the first time in March 1866, but did not become law till 1868.|It was read for the first time in March eighteen sixty-six, but did not become law till eighteen sixty-eight. +LJ016-0331|The last public execution in front of Newgate was that of the Fenian Michael Barrett,|The last public execution in front of Newgate was that of the Fenian Michael Barrett, +LJ016-0332|who was convicted of complicity in the Clerkenwell explosion, intended to effect the release of Burke and Casey|who was convicted of complicity in the Clerkenwell explosion, intended to effect the release of Burke and Casey +LJ016-0333|from Clerkenwell prison, by which many persons lost their lives.|from Clerkenwell prison, by which many persons lost their lives. +LJ016-0334|Unusual precautions were taken upon this occasion, as some fresh outrage was apprehended.|Unusual precautions were taken upon this occasion, as some fresh outrage was apprehended. +LJ016-0335|There was no interference with the crowd, which collected as usual, although not to the customary extent.|There was no interference with the crowd, which collected as usual, although not to the customary extent. +LJ016-0336|But Newgate and its neighborhood was carefully held by the police, both city and metropolitan.|But Newgate and its neighborhood was carefully held by the police, both city and metropolitan. +LJ016-0337|In the houses opposite the prison numbers of detectives mixed with the spectators;|In the houses opposite the prison numbers of detectives mixed with the spectators; +LJ016-0338|inside the jail was Colonel Frazer, the chief commissioner of the city police, and at no great distance, although in the background,|inside the jail was Colonel Frazer, the chief commissioner of the city police, and at no great distance, although in the background, +LJ016-0339|troops were held in readiness to act if required. Everything passed off quite quietly, however,|troops were held in readiness to act if required. Everything passed off quite quietly, however, +LJ016-0340|and Calcraft, who had been threatened with summary retribution if he executed Barrett, carried out the sentence without mishap.|and Calcraft, who had been threatened with summary retribution if he executed Barrett, carried out the sentence without mishap. +LJ016-0341|The sufferer was stolid and reticent to the last.|The sufferer was stolid and reticent to the last. +LJ016-0342|The first private execution under the new law took place within the precincts of Maidstone Jail.|The first private execution under the new law took place within the precincts of Maidstone Jail. +LJ016-0343|The sufferer was a porter on the London, Chatham, and Dover railway, sentenced to death for shooting the station-master at Dover.|The sufferer was a porter on the London, Chatham, and Dover railway, sentenced to death for shooting the station-master at Dover. +LJ016-0344|The ceremony, which was witnessed by only a few officials and representatives of the press, was performed with the utmost decency and decorum.|The ceremony, which was witnessed by only a few officials and representatives of the press, was performed with the utmost decency and decorum. +LJ016-0346|a fact duly advertised as completed by the hoisting of the black flag over the jail,|a fact duly advertised as completed by the hoisting of the black flag over the jail, +LJ016-0347|had undoubtedly a solemn, impressive effect upon those outside.|had undoubtedly a solemn, impressive effect upon those outside. +LJ016-0348|The same was realized in the first private execution within Newgate,|The same was realized in the first private execution within Newgate, +LJ016-0349|that of Alexander Mackay, who murdered his mistress at Norton Folgate by beating her with a rolling-pin and furnace-rake,|that of Alexander Mackay, who murdered his mistress at Norton Folgate by beating her with a rolling-pin and furnace-rake, +LJ016-0350|and who expiated his crime on the 8th September, 1868.|and who expiated his crime on the eighth September, eighteen sixty-eight. +LJ016-0351|A more marked change from the old scene can hardly be conceived. Instead of the roar of the brutalized crowd,|A more marked change from the old scene can hardly be conceived. Instead of the roar of the brutalized crowd, +LJ016-0352|the officials spoke in whispers; there was but little moving to and fro.|the officials spoke in whispers; there was but little moving to and fro. +LJ016-0353|Almost absolute silence prevailed until the great bell began to toll its deep note, and broke the stillness with its regular and monotonous clangour,|Almost absolute silence prevailed until the great bell began to toll its deep note, and broke the stillness with its regular and monotonous clangour, +LJ016-0354|and the ordinary, in a voice trembling with emotion, read the burial service aloud.|and the ordinary, in a voice trembling with emotion, read the burial service aloud. +LJ016-0355|Mackay's fortitude, which had been great,|Mackay's fortitude, which had been great, +LJ016-0356|broke down at the supreme moment before the horror of the stillness, the awful impressiveness of the scene in which he was the principal actor.|broke down at the supreme moment before the horror of the stillness, the awful impressiveness of the scene in which he was the principal actor. +LJ016-0357|No time was lost in carrying out the dread ceremony; but it was not completed without some of the officials turning sick, and the moment it was over,|No time was lost in carrying out the dread ceremony; but it was not completed without some of the officials turning sick, and the moment it was over, +LJ016-0358|all who could were glad to escape from the last act of the ghastly drama at which they had assisted.|all who could were glad to escape from the last act of the ghastly drama at which they had assisted. +LJ016-0359|Private executions at their first introduction were not popular with the Newgate officials, and for intelligible reasons.|Private executions at their first introduction were not popular with the Newgate officials, and for intelligible reasons. +LJ016-0360|The change added greatly to the responsibilities of the governor and his subordinates. Hitherto the public had seemed to assist at the ceremony;|The change added greatly to the responsibilities of the governor and his subordinates. Hitherto the public had seemed to assist at the ceremony; +LJ016-0361|the moment too that the condemned man had passed through the debtors' door on to the scaffold the prison had done with him,|the moment too that the condemned man had passed through the debtors' door on to the scaffold the prison had done with him, +LJ016-0362|and the great outside world shared in the completion of the sacrifice.|and the great outside world shared in the completion of the sacrifice. +LJ016-0363|This feeling was the stronger because|This feeling was the stronger because +LJ016-0364|all the ghastly paraphernalia, the gallows itself and the process of erecting and removing it, rested with the city architect, and not with the prison officials.|all the ghastly paraphernalia, the gallows itself and the process of erecting and removing it, rested with the city architect, and not with the prison officials. +LJ016-0365|Moreover, after the execution, under the old system, the latter had only to receive the body for burial after it had been cut down by the hangman,|Moreover, after the execution, under the old system, the latter had only to receive the body for burial after it had been cut down by the hangman, +LJ016-0366|and placed decently in a shell by the workmen who removed the gallows.|and placed decently in a shell by the workmen who removed the gallows. +LJ016-0367|Under the new system the whole of the arrangements from first to last fell upon the officers.|Under the new system the whole of the arrangements from first to last fell upon the officers. +LJ016-0368|It was they who formed the chief part of the small select group of spectators;|It was they who formed the chief part of the small select group of spectators; +LJ016-0369|upon them devolved the painful duty of cutting down the body and preparing for the inquest.|upon them devolved the painful duty of cutting down the body and preparing for the inquest. +LJ016-0370|All that the hangman, whoever he may be, does under the new regime is to unhook the halter and remove the pinioning straps.|All that the hangman, whoever he may be, does under the new regime is to unhook the halter and remove the pinioning straps. +LJ016-0371|The interment in a shell filled with quicklime in the passage-way leading to the Old Bailey is also a part of the duty of the prison officials.|The interment in a shell filled with quicklime in the passage-way leading to the Old Bailey is also a part of the duty of the prison officials. +LJ016-0373|and for the greater security of prisoners it is roofed in with iron bars which gives it, at least overhead, the aspect of a huge cage.|and for the greater security of prisoners it is roofed in with iron bars which gives it, at least overhead, the aspect of a huge cage. +LJ016-0374|Underfoot and upon the walls roughly cut into the stones, are single initial letters, the brief epitaphs of those who lie below.|Underfoot and upon the walls roughly cut into the stones, are single initial letters, the brief epitaphs of those who lie below. +LJ016-0375|As this burial-ground leads to the adjacent Central Criminal Court, accused murderers, on going to and returning from trial,|As this burial-ground leads to the adjacent Central Criminal Court, accused murderers, on going to and returning from trial, +LJ016-0376|literally walked over what, in case of conviction, would be their own graves.|literally walked over what, in case of conviction, would be their own graves. +LJ016-0377|The older officers, with several of whom I have conversed, have thus had unusual opportunities of watching the demeanor of murderers both before trial|The older officers, with several of whom I have conversed, have thus had unusual opportunities of watching the demeanor of murderers both before trial +LJ016-0378|and after sentence.|and after sentence. +LJ016-0379|All as a rule, unless poignant remorse has brought a desire to court their richly-merited retribution, are buoyed up|All as a rule, unless poignant remorse has brought a desire to court their richly-merited retribution, are buoyed up +LJ016-0380|with hope to the last. There is always the chance of a flaw in the indictment, of a missing witness, or extenuating circumstances.|with hope to the last. There is always the chance of a flaw in the indictment, of a missing witness, or extenuating circumstances. +LJ016-0381|Even when in the condemned cell, with a shameful death within measurable distance,|Even when in the condemned cell, with a shameful death within measurable distance, +LJ016-0382|many cling still to life, expecting much from the intercession of friends or the humanitarianism of the age.|many cling still to life, expecting much from the intercession of friends or the humanitarianism of the age. +LJ016-0383|All almost without exception sleep soundly at night, except the first after sentence,|All almost without exception sleep soundly at night, except the first after sentence, +LJ016-0384|when the first shock of the verdict and the solemn notification of the impending blow keeps nearly all awake, or at least disturbs their night's rest.|when the first shock of the verdict and the solemn notification of the impending blow keeps nearly all awake, or at least disturbs their night's rest. +LJ016-0385|But the uneasiness soon wears off. The second night sleep comes readily, and is sound;|But the uneasiness soon wears off. The second night sleep comes readily, and is sound; +LJ016-0386|many of the most abandoned murderers snore peacefully their eight hours, even on the night immediately preceding execution.|many of the most abandoned murderers snore peacefully their eight hours, even on the night immediately preceding execution. +LJ016-0387|All too have a fairly good appetite, and eat with relish, up to the last moment.|All too have a fairly good appetite, and eat with relish, up to the last moment. +LJ016-0388|A few go further, and are almost gluttonous.|A few go further, and are almost gluttonous. +LJ016-0389|Giovanni Lanni, the Italian boy who murdered a Frenchwoman in the Haymarket,|Giovanni Lanni, the Italian boy who murdered a Frenchwoman in the Haymarket, +LJ016-0390|and was arrested on board ship just as he was about to leave the country, had a little spare cash, which he devoted entirely to the purchase of extra food.|and was arrested on board ship just as he was about to leave the country, had a little spare cash, which he devoted entirely to the purchase of extra food. +LJ016-0391|He ate constantly and voraciously after sentence, as though eager to cram as many meals as possible into the few hours still left him to live.|He ate constantly and voraciously after sentence, as though eager to cram as many meals as possible into the few hours still left him to live. +LJ016-0392|Jeffrey, who murdered his own child, an infant of six, by hanging him in a cellar in Seven Dials,|Jeffrey, who murdered his own child, an infant of six, by hanging him in a cellar in Seven Dials, +LJ016-0393|called for a roast duck directly he entered the condemned cell.|called for a roast duck directly he entered the condemned cell. +LJ016-0394|The request was not granted, as the old custom of allowing capital convicts whatever they asked for in the way of food has not been the rule in Newgate.|The request was not granted, as the old custom of allowing capital convicts whatever they asked for in the way of food has not been the rule in Newgate. +LJ016-0395|The diet of the condemned is the ordinary diet of the prison,|The diet of the condemned is the ordinary diet of the prison, +LJ016-0396|but to which additions are sometimes made, chiefly of stimulants, if deemed necessary, by the medical officer of the jail.|but to which additions are sometimes made, chiefly of stimulants, if deemed necessary, by the medical officer of the jail. +LJ016-0397|The craving for tobacco which so dominates the habitual smoker often leads the convicted to plead hard for a last smoke.|The craving for tobacco which so dominates the habitual smoker often leads the convicted to plead hard for a last smoke. +LJ016-0398|As a special favor|As a special favor +LJ016-0399|Wainwright was allowed a cigar the night before execution, which he smoked in the prison yard, walking up and down with the governor, Mr. Sydney Smith.|Wainwright was allowed a cigar the night before execution, which he smoked in the prison yard, walking up and down with the governor, Mr. Sydney Smith. +LJ016-0400|Wainwright's demeanor was one of reckless effrontery steadily maintained to the last.|Wainwright's demeanor was one of reckless effrontery steadily maintained to the last. +LJ016-0401|His conversation turned always upon his influence over the weaker sex, and the extraordinary success he had achieved.|His conversation turned always upon his influence over the weaker sex, and the extraordinary success he had achieved. +LJ016-0402|No woman could resist him, he calmly assured Mr. Smith that night as they walked together, and he recounted his villanies one by one.|No woman could resist him, he calmly assured Mr. Smith that night as they walked together, and he recounted his villanies one by one. +LJ016-0403|His effrontery was only outdone by his cool contempt for the consolations of religion.|His effrontery was only outdone by his cool contempt for the consolations of religion. +LJ016-0404|The man who had made a pious life a cloak for his misdeeds, the once exemplary young man and indefatigable Sunday School teacher,|The man who had made a pious life a cloak for his misdeeds, the once exemplary young man and indefatigable Sunday School teacher, +LJ016-0405|went impenitent to the gallows. The only sign of feeling he showed was in asking to be allowed to choose the hymns on the Sunday|went impenitent to the gallows. The only sign of feeling he showed was in asking to be allowed to choose the hymns on the Sunday +LJ016-0406|the condemned sermon was preached in the prison chapel, and this was probably only that he might hear the singing of a lady with a magnificent voice|the condemned sermon was preached in the prison chapel, and this was probably only that he might hear the singing of a lady with a magnificent voice +LJ016-0408|During the singing of these hymns Wainwright fainted, but whether from real emotion or the desire to make a sensation was never exactly known.|During the singing of these hymns Wainwright fainted, but whether from real emotion or the desire to make a sensation was never exactly known. +LJ016-0409|On the fatal morning he came gaily out of his cell,|On the fatal morning he came gaily out of his cell, +LJ016-0410|nodded pleasantly to the governor, who stood just opposite, and then walked briskly towards the execution shed, smiling as he went along.|nodded pleasantly to the governor, who stood just opposite, and then walked briskly towards the execution shed, smiling as he went along. +LJ016-0411|There was a smile on his face when it was last seen, and just as the terrible white cap was drawn over it.|There was a smile on his face when it was last seen, and just as the terrible white cap was drawn over it. +LJ016-0412|Wainwright's execution was within the jail, but only nominally private.|Wainwright's execution was within the jail, but only nominally private. +LJ016-0413|No less than sixty-seven persons were present, admitted by special permission of the sheriff.|No less than sixty-seven persons were present, admitted by special permission of the sheriff. +LJ016-0414|Rumour even went so far as to assert that among the spectators were several women, disguised in male habiliments;|Rumour even went so far as to assert that among the spectators were several women, disguised in male habiliments; +LJ016-0415|but the story was never substantiated, and we may hope that it rested only on the idle gossip of the day.|but the story was never substantiated, and we may hope that it rested only on the idle gossip of the day. +LJ016-0416|Many, like Wainwright, were calm and imperturbable throughout their trying ordeal.|Many, like Wainwright, were calm and imperturbable throughout their trying ordeal. +LJ016-0417|Catherine Wilson, the poisoner, was reserved and reticent to the last, expressing no contrition, but also no fear --|Catherine Wilson, the poisoner, was reserved and reticent to the last, expressing no contrition, but also no fear -- +LJ016-0418|a tall, gaunt, repulsive-looking woman, who no more shrank from cowardly, secret crimes than from the penalty they entailed.|a tall, gaunt, repulsive-looking woman, who no more shrank from cowardly, secret crimes than from the penalty they entailed. +LJ016-0419|Kate Webster, who was tried at the Central Criminal Court, and passed through Newgate, although she suffered at Wandsworth,|Kate Webster, who was tried at the Central Criminal Court, and passed through Newgate, although she suffered at Wandsworth, +LJ016-0420|is remembered at the former prison as a defiant,|is remembered at the former prison as a defiant, +LJ016-0421|brutal creature who showed no remorse, but was subject to fits of ungovernable passion, when she broke out into language the most appalling.|brutal creature who showed no remorse, but was subject to fits of ungovernable passion, when she broke out into language the most appalling. +LJ016-0422|The man Marley displayed fortitude of a less repulsive kind. He acknowledged his guilt from the first.|The man Marley displayed fortitude of a less repulsive kind. He acknowledged his guilt from the first. +LJ016-0423|When the sheriff offered him counsel for his defense, he declined, saying he wished to make none -- "the witnesses for the prosecution spoke the truth."|When the sheriff offered him counsel for his defense, he declined, saying he wished to make none -- "the witnesses for the prosecution spoke the truth." +LJ016-0424|During the trial and after sentence he remained perfectly cool and collected.|During the trial and after sentence he remained perfectly cool and collected. +LJ016-0425|When visited one day in the condemned cell, just as St. Sepulchre's clock was striking, he looked up and said laughingly, "Go along, clock;|When visited one day in the condemned cell, just as St. Sepulchre's clock was striking, he looked up and said laughingly, "Go along, clock; +LJ016-0426|come along, gallows.|come along, gallows. +LJ016-0427|He tripped up the chapel-stairs to hear the condemned sermon, and came out with cheerful alacrity on the morning he was to die.|He tripped up the chapel-stairs to hear the condemned sermon, and came out with cheerful alacrity on the morning he was to die. +LJ016-0428|Some condemned convicts converse but little with the warders who have them unceasingly in charge.|Some condemned convicts converse but little with the warders who have them unceasingly in charge. +LJ016-0429|Others talk freely enough on various topics, but principally upon their own cases.|Others talk freely enough on various topics, but principally upon their own cases. +LJ016-0430|When vanity is strongly developed there is the keen anxiety to hear what is being said about them outside.|When vanity is strongly developed there is the keen anxiety to hear what is being said about them outside. +LJ016-0431|One was vexed to think that his victims had a finer funeral than he would have.|One was vexed to think that his victims had a finer funeral than he would have. +LJ016-0432|The only subject another showed any interest in was the theatres and the new pieces that were being produced. A third, Christian Satler,|The only subject another showed any interest in was the theatres and the new pieces that were being produced. A third, Christian Satler, +LJ016-0433|laughed and jested with the officers about "Jack Ketch," who, through the postponement of the execution, would lose his Christmas dinner.|laughed and jested with the officers about "Jack Ketch," who, through the postponement of the execution, would lose his Christmas dinner. +LJ016-0434|When they brought in the two watchers to relieve guard one night, Sattler said,|When they brought in the two watchers to relieve guard one night, Sattler said, +LJ016-0435|"Two fresh men! May I speak to them? Yes! I must caution you," he went on to the warders, "not to go to sleep, or I shall be off through that little hole,"|"Two fresh men! May I speak to them? Yes! I must caution you," he went on to the warders, "not to go to sleep, or I shall be off through that little hole," +LJ016-0436|pointing to an aperture for ventilating the cell. On the morning of execution he asked how far it was to the gallows, and was told it was quite close.|pointing to an aperture for ventilating the cell. On the morning of execution he asked how far it was to the gallows, and was told it was quite close. +LJ016-0438|that the convict's clothes were still the executioner's perquisite.|that the convict's clothes were still the executioner's perquisite. +LJ016-0439|Often the convicts give way to despair. They are too closely watched to be allowed to do themselves much mischief, or suicides would probably be more frequent.|Often the convicts give way to despair. They are too closely watched to be allowed to do themselves much mischief, or suicides would probably be more frequent. +LJ016-0440|But it is neither easy to obtain the instruments of self-destruction nor to elude the vigilance of their guard.|But it is neither easy to obtain the instruments of self-destruction nor to elude the vigilance of their guard. +LJ016-0441|The man, Bousfield, however, whose execution was so sadly bungled,|The man, Bousfield, however, whose execution was so sadly bungled, +LJ016-0442|made a determined effort to burn himself to death by throwing himself bodily on to the fire in the condemned ward.|made a determined effort to burn himself to death by throwing himself bodily on to the fire in the condemned ward. +LJ016-0444|They were still much swollen when he was led out to execution.|They were still much swollen when he was led out to execution. +LJ016-0445|Miller, the Chelsea murderer, who packed his victim's body in a box, and tried to send it by parcels delivery, tried to kill himself,|Miller, the Chelsea murderer, who packed his victim's body in a box, and tried to send it by parcels delivery, tried to kill himself, +LJ016-0446|but ineffectively, by running his head against his cell wall.|but ineffectively, by running his head against his cell wall. +LJ017-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 20: Newgate Notorieties, part 1.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section twenty: Newgate Notorieties, part one. +LJ017-0002|As these records draw to a close, the crimes I chronicle become so much more recent in date that they will be fresh in the memory of most of my readers.|As these records draw to a close, the crimes I chronicle become so much more recent in date that they will be fresh in the memory of most of my readers. +LJ017-0003|Nevertheless, in order to give completeness to the picture|Nevertheless, in order to give completeness to the picture +LJ017-0004|I have attempted to draw of crime in connection with Newgate, from first to last, I must make some mention, in this my penultimate chapter,|I have attempted to draw of crime in connection with Newgate, from first to last, I must make some mention, in this my penultimate chapter, +LJ017-0005|of some of the most heinous offenses of modern times.|of some of the most heinous offenses of modern times. +LJ017-0006|The crime of poisoning has always been viewed with peculiar loathing and terror in this country.|The crime of poisoning has always been viewed with peculiar loathing and terror in this country. +LJ017-0007|It will be remembered that as far back as the reign of Henry VIII. a new and most cruel penalty was devised for the punishment of the Bishop of Rochester's cook,|It will be remembered that as far back as the reign of Henry the eighth a new and most cruel penalty was devised for the punishment of the Bishop of Rochester's cook, +LJ017-0008|who had poisoned his master and many of his dependents.|who had poisoned his master and many of his dependents. +LJ017-0009|Sir Thomas Overbury was undoubtedly poisoned by Lord Rochester in the reign of James I.,|Sir Thomas Overbury was undoubtedly poisoned by Lord Rochester in the reign of James the first, +LJ017-0010|and it is hinted that James himself nearly fell a victim to a nefarious attempt of the Duke of Buckingham.|and it is hinted that James himself nearly fell a victim to a nefarious attempt of the Duke of Buckingham. +LJ017-0011|But secret poisoning on a wholesale scale such as was practiced in Italy and France was happily never popularized in England.|But secret poisoning on a wholesale scale such as was practiced in Italy and France was happily never popularized in England. +LJ017-0012|The well-known and lethal aqua Toffania,|The well-known and lethal aqua Toffania, +LJ017-0013|so called after its inventress, a Roman woman named Toffana, and which was so widely adopted by ladies anxious to get rid of their husbands,|so called after its inventress, a Roman woman named Toffana, and which was so widely adopted by ladies anxious to get rid of their husbands, +LJ017-0014|was never introduced into this country. Its admission was probably checked by the increased vigilance at the custom houses,|was never introduced into this country. Its admission was probably checked by the increased vigilance at the custom houses, +LJ017-0015|the necessity for which was urged by Mr. Addison, when Secretary of State, in 1717.|the necessity for which was urged by Mr. Addison, when Secretary of State, in seventeen seventeen. +LJ017-0016|The cases of poisoning in the British calendars are rare, nor indeed was the guilt of the accused always clearly established.|The cases of poisoning in the British calendars are rare, nor indeed was the guilt of the accused always clearly established. +LJ017-0017|It is quite possible that Catherine Blandy, who poisoned her father at the instigation of her lover,|It is quite possible that Catherine Blandy, who poisoned her father at the instigation of her lover, +LJ017-0018|was ignorant of the destructive character of the powders, probably arsenic, which she administered.|was ignorant of the destructive character of the powders, probably arsenic, which she administered. +LJ017-0019|Captain Donellan, who was convicted of poisoning his brother-in-law, Sir Theodosius Broughton, and executed for it,|Captain Donellan, who was convicted of poisoning his brother-in-law, Sir Theodosius Broughton, and executed for it, +LJ017-0020|would probably have had the benefit in these days of the doubts raised at his trial.|would probably have had the benefit in these days of the doubts raised at his trial. +LJ017-0021|A third case, more especially interesting to us as having passed through Newgate,|A third case, more especially interesting to us as having passed through Newgate, +LJ017-0022|was that of Eliza Fenning, who was convicted of an attempt to poison a whole family|was that of Eliza Fenning, who was convicted of an attempt to poison a whole family +LJ017-0023|by putting arsenic in the dumplings she had prepared for them. The charge rested entirely on circumstantial evidence,|by putting arsenic in the dumplings she had prepared for them. The charge rested entirely on circumstantial evidence, +LJ017-0024|and as Fenning, although convicted and executed, protested her innocence in the most solemn manner to the last,|and as Fenning, although convicted and executed, protested her innocence in the most solemn manner to the last, +LJ017-0025|the justice of the sentence was doubted at the time.|the justice of the sentence was doubted at the time. +LJ017-0026|Yet it was clearly proved that the dumplings contained arsenic, that she, and she alone, had made the dough,|Yet it was clearly proved that the dumplings contained arsenic, that she, and she alone, had made the dough, +LJ017-0027|that arsenic was within her reach in the house,|that arsenic was within her reach in the house, +LJ017-0028|that she had had a quarrel with her mistress, and that the latter with all others who tasted the dumplings were similarly attacked, although no one died.|that she had had a quarrel with her mistress, and that the latter with all others who tasted the dumplings were similarly attacked, although no one died. +LJ017-0029|The crime of poisoning is essentially one which will be most prevalent in a high state of civilization,|The crime of poisoning is essentially one which will be most prevalent in a high state of civilization, +LJ017-0030|when the spread of scientific knowledge places nefarious means at the disposal of many,|when the spread of scientific knowledge places nefarious means at the disposal of many, +LJ017-0031|instead of limiting them, as in the days of the Borgias and Brinvilliers, to the specially informed and unscrupulously powerful few.|instead of limiting them, as in the days of the Borgias and Brinvilliers, to the specially informed and unscrupulously powerful few. +LJ017-0032|The first intimation conveyed to society of the new terror which threatened it was in the arrest and arraignment of William Palmer,|The first intimation conveyed to society of the new terror which threatened it was in the arrest and arraignment of William Palmer, +LJ017-0034|The case contained elements of much uncertainty, and yet it was so essential,|The case contained elements of much uncertainty, and yet it was so essential, +LJ017-0035|in the interests and for the due protection of the public, that the fullest and fairest inquiry should be made,|in the interests and for the due protection of the public, that the fullest and fairest inquiry should be made, +LJ017-0036|that the trial was transferred to the Central Criminal Court,|that the trial was transferred to the Central Criminal Court, +LJ017-0037|under the authority of an Act passed on purpose, known as the Trial of offenses Act, and sometimes as Lord Campbell's Act.|under the authority of an Act passed on purpose, known as the Trial of offenses Act, and sometimes as Lord Campbell's Act. +LJ017-0039|is obviously of paramount importance, and the powers granted by this Act have been frequently put in practice since.|is obviously of paramount importance, and the powers granted by this Act have been frequently put in practice since. +LJ017-0040|The trial of Catherine Winsor, the baby farmer,|The trial of Catherine Winsor, the baby farmer, +LJ017-0041|was thus brought to the Central Criminal Court from Exeter assizes, and that of the Stauntons from Maidstone.|was thus brought to the Central Criminal Court from Exeter assizes, and that of the Stauntons from Maidstone. +LJ017-0042|Palmer's trial caused the most intense excitement.|Palmer's trial caused the most intense excitement. +LJ017-0043|The direful suspicions which surrounded the case filled the whole country with uneasiness and misgiving,|The direful suspicions which surrounded the case filled the whole country with uneasiness and misgiving, +LJ017-0044|and the deepest anxiety was felt that the crime, if crime there had been, should be brought home to its perpetrator.|and the deepest anxiety was felt that the crime, if crime there had been, should be brought home to its perpetrator. +LJ017-0045|The Central Criminal Court was crowded to suffocation.|The Central Criminal Court was crowded to suffocation. +LJ017-0046|Great personages occupied seats upon the bench;|Great personages occupied seats upon the bench; +LJ017-0047|the rest of the available space was allotted by ticket, to secure which the greatest influence was necessary.|the rest of the available space was allotted by ticket, to secure which the greatest influence was necessary. +LJ017-0048|People came to stare at the supposed cold-blooded prisoner;|People came to stare at the supposed cold-blooded prisoner; +LJ017-0049|with morbid curiosity to scan his features and watch his demeanor through the shifting, nicely-balanced phases of his protracted trial.|with morbid curiosity to scan his features and watch his demeanor through the shifting, nicely-balanced phases of his protracted trial. +LJ017-0050|Palmer, who was only thirty-one at the time of his trial, was in appearance short and stout, with a round head|Palmer, who was only thirty-one at the time of his trial, was in appearance short and stout, with a round head +LJ017-0051|covered rather scantily with light sandy hair.|covered rather scantily with light sandy hair. +LJ017-0052|His skin was extraordinarily fair, his cheeks fresh and ruddy; altogether his face, though commonplace, was not exactly ugly;|His skin was extraordinarily fair, his cheeks fresh and ruddy; altogether his face, though commonplace, was not exactly ugly; +LJ017-0053|there was certainly nothing in it which indicated cruel cunning or deliberate truculence.|there was certainly nothing in it which indicated cruel cunning or deliberate truculence. +LJ017-0054|His features were not careworn, but rather set, and he looked older than his age.|His features were not careworn, but rather set, and he looked older than his age. +LJ017-0055|Throughout his trial he preserved an impassive countenance, but he clearly took a deep interest in all that passed.|Throughout his trial he preserved an impassive countenance, but he clearly took a deep interest in all that passed. +LJ017-0056|Although the strain lasted fourteen days, he showed no signs of exhaustion, either physical or mental.|Although the strain lasted fourteen days, he showed no signs of exhaustion, either physical or mental. +LJ017-0057|On returning to jail each day he talked freely and without reserve to the warders in charge of him, chiefly on incidents in the day's proceedings.|On returning to jail each day he talked freely and without reserve to the warders in charge of him, chiefly on incidents in the day's proceedings. +LJ017-0058|He was confident to the very last that it would be impossible to find him guilty;|He was confident to the very last that it would be impossible to find him guilty; +LJ017-0059|even after sentence, and until within a few hours of execution, he was buoyed up with the hope of reprieve.|even after sentence, and until within a few hours of execution, he was buoyed up with the hope of reprieve. +LJ017-0060|The conviction that he would escape had taken so firm a hold of him,|The conviction that he would escape had taken so firm a hold of him, +LJ017-0061|that he steadily refused to confess his guilt, lest it should militate against his chances.|that he steadily refused to confess his guilt, lest it should militate against his chances. +LJ017-0062|In the condemned cell he frequently repeated, quote, I go to my death a murdered man, end quote.|In the condemned cell he frequently repeated, quote, I go to my death a murdered man, end quote. +LJ017-0063|He made no distinct admissions even on the scaffold; but when the chaplain at the last moment exhorted him to confess,|He made no distinct admissions even on the scaffold; but when the chaplain at the last moment exhorted him to confess, +LJ017-0064|he made use of the remarkable words, quote, If it is necessary for my soul's sake to confess this murder,|he made use of the remarkable words, quote, If it is necessary for my soul's sake to confess this murder, +LJ017-0065|I ought also to confess the others: I mean my wife and my brother's. End quote.|I ought also to confess the others: I mean my wife and my brother's. End quote. +LJ017-0066|Yet he was silent when specifically pressed to confess that he had killed his wife and his brother.|Yet he was silent when specifically pressed to confess that he had killed his wife and his brother. +LJ017-0067|Palmer was ably defended, but the weight of evidence was clearly with the prosecution, led by Sir Alexander Cockburn,|Palmer was ably defended, but the weight of evidence was clearly with the prosecution, led by Sir Alexander Cockburn, +LJ017-0068|and public opinion at the termination of the trial coincided with the verdict of the jury.|and public opinion at the termination of the trial coincided with the verdict of the jury. +LJ017-0069|Originally a doctor in practice at Rugeley, in Staffordshire, he had gradually withdrawn from medicine, and devoted himself to the turf;|Originally a doctor in practice at Rugeley, in Staffordshire, he had gradually withdrawn from medicine, and devoted himself to the turf; +LJ017-0070|but his sporting operations did not prosper, and he became a needy man, always driven to desperate straits for cash.|but his sporting operations did not prosper, and he became a needy man, always driven to desperate straits for cash. +LJ017-0071|To meet his liabilities, he raised large sums on forged bills of acceptance drawn upon his mother, a woman of some means,|To meet his liabilities, he raised large sums on forged bills of acceptance drawn upon his mother, a woman of some means, +LJ017-0072|whose signature he counterfeited.|whose signature he counterfeited. +LJ017-0073|In 1854 he owed a very large sum of money, but he was temporarily relieved by the death of his wife,|In eighteen fifty-four he owed a very large sum of money, but he was temporarily relieved by the death of his wife, +LJ017-0074|whose life he had insured for £13,000.|whose life he had insured for thirteen thousand pounds. +LJ017-0075|There is every reason to suppose that he poisoned his wife to obtain possession of this sum upon her death.|There is every reason to suppose that he poisoned his wife to obtain possession of this sum upon her death. +LJ017-0076|His brother was supposed to have been his next victim, upon whose life he had also effected an insurance for another £13,000.|His brother was supposed to have been his next victim, upon whose life he had also effected an insurance for another thirteen thousand pounds. +LJ017-0077|The brother too died conveniently, but the life office took some exception to the manner of the death, and hesitated to disburse the funds claimed by Palmer.|The brother too died conveniently, but the life office took some exception to the manner of the death, and hesitated to disburse the funds claimed by Palmer. +LJ017-0078|Palmer tried to get a new insurance on the life of a hanger-on, one Bates, but no office would accept it, no doubt greatly to Bates's longevity.|Palmer tried to get a new insurance on the life of a hanger-on, one Bates, but no office would accept it, no doubt greatly to Bates's longevity. +LJ017-0079|Meanwhile the bill discounters who held the forged acceptances, with other promissory notes, began to clamor for payment, and talk of issuing writs.|Meanwhile the bill discounters who held the forged acceptances, with other promissory notes, began to clamor for payment, and talk of issuing writs. +LJ017-0080|Palmer, alive to the danger he ran of a prosecution for forgery, should the fraud he had committed be brought to light,|Palmer, alive to the danger he ran of a prosecution for forgery, should the fraud he had committed be brought to light, +LJ017-0081|sought about for a fresh victim to supply him with funds.|sought about for a fresh victim to supply him with funds. +LJ017-0082|He fixed upon a sporting friend, Mr. John Parsons Cook, who had been in luck at Shrewsbury races, both as a winner and a backer,|He fixed upon a sporting friend, Mr. John Parsons Cook, who had been in luck at Shrewsbury races, both as a winner and a backer, +LJ017-0083|whom he persuaded to go and stay at Rugeley in an hotel just opposite his own house.|whom he persuaded to go and stay at Rugeley in an hotel just opposite his own house. +LJ017-0084|It was there that Cook was first taken ill with violent retchings and vomitings, all dating from visits of Palmer, who brought him medicines and food.|It was there that Cook was first taken ill with violent retchings and vomitings, all dating from visits of Palmer, who brought him medicines and food. +LJ017-0085|Palmer's plan was to administer poison in quantities insufficient to cause death, but enough to produce illness which would account for death.|Palmer's plan was to administer poison in quantities insufficient to cause death, but enough to produce illness which would account for death. +LJ017-0086|For this purpose he gave, or there was the strongest presumption that he gave,|For this purpose he gave, or there was the strongest presumption that he gave, +LJ017-0087|antimony, which caused Cook's constant sickness.|antimony, which caused Cook's constant sickness. +LJ017-0088|Quantities of antimony were found in the body after death.|Quantities of antimony were found in the body after death. +LJ017-0089|While Cook lay ill, Palmer in his name pocketed the proceeds of the Shrewsbury settling,|While Cook lay ill, Palmer in his name pocketed the proceeds of the Shrewsbury settling, +LJ017-0090|and so got the money for which he was prepared to barter his soul.|and so got the money for which he was prepared to barter his soul. +LJ017-0091|The last act now approached, and in order to avoid the detection of this last fraud, Palmer laid his plans for disposing of Cook.|The last act now approached, and in order to avoid the detection of this last fraud, Palmer laid his plans for disposing of Cook. +LJ017-0092|He decided to use strychnia, or the vegetable poison otherwise known as nux vomica;|He decided to use strychnia, or the vegetable poison otherwise known as nux vomica; +LJ017-0093|and one of the many links in the long chain of evidence was an entry in a book of Palmer's to the effect that Quote.|and one of the many links in the long chain of evidence was an entry in a book of Palmer's to the effect that Quote. +LJ017-0094|strychnia kills by causing tetanic fixing of the respiratory muscles. End quote.|strychnia kills by causing tetanic fixing of the respiratory muscles. End quote. +LJ017-0095|The purchase by Palmer of strychnia was proved.|The purchase by Palmer of strychnia was proved. +LJ017-0096|The night he bought it, Cook, who had been taking certain pills under medical advice, not Palmer's, was seized with violent convulsions.|The night he bought it, Cook, who had been taking certain pills under medical advice, not Palmer's, was seized with violent convulsions. +LJ017-0097|He had swallowed his pills as usual, at least Palmer had administered them|He had swallowed his pills as usual, at least Palmer had administered them +LJ017-0098|whether the ordinary or his own pills will never be known, except as may be inferred from the results, which indicate that he had taken the latter.|whether the ordinary or his own pills will never be known, except as may be inferred from the results, which indicate that he had taken the latter. +LJ017-0099|Cook recovered this time; it was probably Palmer's intention that he should recover, wishing to encourage the supposition that Cook was in a bad way.|Cook recovered this time; it was probably Palmer's intention that he should recover, wishing to encourage the supposition that Cook was in a bad way. +LJ017-0100|Next night Cook had a second and a more violent attack.|Next night Cook had a second and a more violent attack. +LJ017-0101|That day Palmer had bought more strychnia, and had called in a fresh doctor.|That day Palmer had bought more strychnia, and had called in a fresh doctor. +LJ017-0102|The second attack was fatal, and ended in Cook's death from tetanus.|The second attack was fatal, and ended in Cook's death from tetanus. +LJ017-0103|This tetanus, according to the prosecution, was produced by strychnia, and followed the administration of pills by Palmer|This tetanus, according to the prosecution, was produced by strychnia, and followed the administration of pills by Palmer +LJ017-0104|prescribed nominally by the fresh doctor, for which Palmer had substituted his own.|prescribed nominally by the fresh doctor, for which Palmer had substituted his own. +LJ017-0105|Cook's death was horrible|Cook's death was horrible +LJ017-0106|fearful paroxysms and cramps, ending in suffocation by the tetanic rigor which caught the muscles of the chest.|fearful paroxysms and cramps, ending in suffocation by the tetanic rigor which caught the muscles of the chest. +LJ017-0107|After Cook's death his stepfather, who was much attached to him, came to Rugeley.|After Cook's death his stepfather, who was much attached to him, came to Rugeley. +LJ017-0108|He was struck with the appearance of the corpse, which was not emaciated, as after a long disease ending in death;|He was struck with the appearance of the corpse, which was not emaciated, as after a long disease ending in death; +LJ017-0109|while the muscles of the fingers were tightly clenched, not open, as usual in a corpse.|while the muscles of the fingers were tightly clenched, not open, as usual in a corpse. +LJ017-0110|He said nothing, but began to feel uneasy when he found that Cook's betting-book was missing, and that Palmer put it forward|He said nothing, but began to feel uneasy when he found that Cook's betting-book was missing, and that Palmer put it forward +LJ017-0111|that his friend had died greatly embarrassed, with bills to the amount of £4000 out in his name.|that his friend had died greatly embarrassed, with bills to the amount of four thousand pounds out in his name. +LJ017-0112|Palmer too showed an indecent haste in preparing the body for interment, and in obtaining the usual certificate.|Palmer too showed an indecent haste in preparing the body for interment, and in obtaining the usual certificate. +LJ017-0113|After this the step-father insisted upon a post-mortem, which was conducted somewhat carelessly.|After this the step-father insisted upon a post-mortem, which was conducted somewhat carelessly. +LJ017-0114|The intestines were, however, preserved and sent for analysis,|The intestines were, however, preserved and sent for analysis, +LJ017-0115|but it was proved that Palmer tried hard to get possession of the jar containing them,|but it was proved that Palmer tried hard to get possession of the jar containing them, +LJ017-0116|and even sought to upset the vehicle by which they were being conveyed a part of the way to London.|and even sought to upset the vehicle by which they were being conveyed a part of the way to London. +LJ017-0117|The examination of the stomach betrayed the presence of antimony in large quantities, but no strychnia,|The examination of the stomach betrayed the presence of antimony in large quantities, but no strychnia, +LJ017-0118|and it was on the entire absence of the latter that the defense was principally based when Palmer was brought to trial.|and it was on the entire absence of the latter that the defense was principally based when Palmer was brought to trial. +LJ017-0119|All the circumstances were so suspicious that he could not escape the criminal charge.|All the circumstances were so suspicious that he could not escape the criminal charge. +LJ017-0120|He had already been arrested on a writ issued at the instance of the money-lenders, and an action had been commenced against Mrs. Palmer on her acceptances.|He had already been arrested on a writ issued at the instance of the money-lenders, and an action had been commenced against Mrs. Palmer on her acceptances. +LJ017-0121|It came out at once that these had been forged, and the whole affair at once took the ugliest complexion.|It came out at once that these had been forged, and the whole affair at once took the ugliest complexion. +LJ017-0122|A government prosecution was instituted, and Palmer was brought to Newgate for trial at the Central Criminal Court.|A government prosecution was instituted, and Palmer was brought to Newgate for trial at the Central Criminal Court. +LJ017-0123|There was not much reserve about him when there.|There was not much reserve about him when there. +LJ017-0124|He frequently declared before and during the trial that it would be impossible to find him guilty.|He frequently declared before and during the trial that it would be impossible to find him guilty. +LJ017-0125|He never actually said that he was not guilty, but he was confident he would not be convicted.|He never actually said that he was not guilty, but he was confident he would not be convicted. +LJ017-0126|He relied on the absence of the strychnia.|He relied on the absence of the strychnia. +LJ017-0127|But the chain of circumstantial evidence was strong enough to satisfy the jury, who agreed to their verdict in an hour.|But the chain of circumstantial evidence was strong enough to satisfy the jury, who agreed to their verdict in an hour. +LJ017-0128|At the last moment Palmer tossed a bit of paper over to his counsel, on which he had written, quote, I think there will be a verdict of Not Guilty, end quote.|At the last moment Palmer tossed a bit of paper over to his counsel, on which he had written, quote, I think there will be a verdict of Not Guilty, end quote. +LJ017-0129|Even after the death sentence had been passed upon him he clung to the hope that the Government would grant him a reprieve.|Even after the death sentence had been passed upon him he clung to the hope that the Government would grant him a reprieve. +LJ017-0130|To the last, therefore, he played the part of a man wrongfully convicted, and did not abandon hope|To the last, therefore, he played the part of a man wrongfully convicted, and did not abandon hope +LJ017-0131|even when the high sheriff had told him there was no possibility of a reprieve, and within a few hours of execution.|even when the high sheriff had told him there was no possibility of a reprieve, and within a few hours of execution. +LJ017-0132|He suffered at Stafford in front of the jail.|He suffered at Stafford in front of the jail. +LJ017-0133|Palmer speedily found imitators.|Palmer speedily found imitators. +LJ017-0134|Within a few weeks occurred the Leeds poisoning case, in which the murderer undoubtedly was inspired by the facts made public at Palmer's trial.|Within a few weeks occurred the Leeds poisoning case, in which the murderer undoubtedly was inspired by the facts made public at Palmer's trial. +LJ017-0135|Dove, a fiendish brute, found from the evidence in that case that he could kill his wife, whom he hated,|Dove, a fiendish brute, found from the evidence in that case that he could kill his wife, whom he hated, +LJ017-0136|with exquisite torture, and with a poison that would leave, as he thought, no trace.|with exquisite torture, and with a poison that would leave, as he thought, no trace. +LJ017-0137|In the latter hope he was happily disappointed. But as this case is beyond my subject, I merely mention it as one of the group already referred to.|In the latter hope he was happily disappointed. But as this case is beyond my subject, I merely mention it as one of the group already referred to. +LJ017-0138|Three years later came the case of Dr. Smethurst,|Three years later came the case of Dr. Smethurst, +LJ017-0139|presenting still greater features of resemblance with Palmer's, for both were medical men, and both raised difficult questions of medical jurisprudence.|presenting still greater features of resemblance with Palmer's, for both were medical men, and both raised difficult questions of medical jurisprudence. +LJ017-0140|In both the jury had no doubt as to the guilt of the accused, only in Smethurst's case the then Home Secretary, Sir George Cornewall Lewis,|In both the jury had no doubt as to the guilt of the accused, only in Smethurst's case the then Home Secretary, Sir George Cornewall Lewis, +LJ017-0141|could not divest his mind of serious doubt, and of which the murderer got the benefit.|could not divest his mind of serious doubt, and of which the murderer got the benefit. +LJ017-0142|Smethurst's escape may have influenced the jury in the Poplar poisoning case,|Smethurst's escape may have influenced the jury in the Poplar poisoning case, +LJ017-0143|which followed close on its heels, although in that the verdict of "Not Guilty" was excusable, as the evidence was entirely circumstantial.|which followed close on its heels, although in that the verdict of "Not Guilty" was excusable, as the evidence was entirely circumstantial. +LJ017-0144|There was no convincing proof that the accused had administered the poison, although beyond question that poison had occasioned the death.|There was no convincing proof that the accused had administered the poison, although beyond question that poison had occasioned the death. +LJ017-0145|Dr. Smethurst was long an inmate of Newgate, and was tried at the Central Criminal Court.|Dr. Smethurst was long an inmate of Newgate, and was tried at the Central Criminal Court. +LJ017-0146|He had all the characteristics of the poisoner -- the calm deliberation,|He had all the characteristics of the poisoner -- the calm deliberation, +LJ017-0147|the protracted dissimulation, as with unshrinking, relentless wickedness the deadly work is carried on to the end.|the protracted dissimulation, as with unshrinking, relentless wickedness the deadly work is carried on to the end. +LJ017-0148|Smethurst's victim was a Miss Bankes, with whom he had contracted a bigamous marriage.|Smethurst's victim was a Miss Bankes, with whom he had contracted a bigamous marriage. +LJ017-0149|He had met her at a boarding-house, where he lived with his own wife, a person of "shady" antecedents,|He had met her at a boarding-house, where he lived with his own wife, a person of "shady" antecedents, +LJ017-0150|and whom he left without scruple to join Miss Bankes.|and whom he left without scruple to join Miss Bankes. +LJ017-0151|The latter seems to have succumbed only too willingly to his fascinations, and to have as readily agreed to marry him,|The latter seems to have succumbed only too willingly to his fascinations, and to have as readily agreed to marry him, +LJ017-0152|in spite of the existence of the other Mrs. Smethurst.|in spite of the existence of the other Mrs. Smethurst. +LJ017-0153|Probably the doctor had told her the story he brought forward when tried for bigamy, namely,|Probably the doctor had told her the story he brought forward when tried for bigamy, namely, +LJ017-0154|that Mrs. Smethurst had no right to the name, but had a husband of her own, one Johnson, alive -- a story subsequently disproved.|that Mrs. Smethurst had no right to the name, but had a husband of her own, one Johnson, alive -- a story subsequently disproved. +LJ017-0155|Miss Bankes seems to have counted upon some species of whitewashing, no less than the repudiation of the other marriage,|Miss Bankes seems to have counted upon some species of whitewashing, no less than the repudiation of the other marriage, +LJ017-0156|and told her sister as much when they last met.|and told her sister as much when they last met. +LJ017-0157|For some months Smethurst and Miss Bankes lived together as man and wife, first in London, and then at Richmond.|For some months Smethurst and Miss Bankes lived together as man and wife, first in London, and then at Richmond. +LJ017-0158|She had a little fortune of her own, some £1700 or £1800,|She had a little fortune of her own, some one thousand seven hundred pounds or one thousand eight hundred pounds, +LJ017-0159|and a life-interest in £5000, a fact on which Smethurst's counsel dwelt with much weight,|and a life-interest in five thousand pounds, a fact on which Smethurst's counsel dwelt with much weight, +LJ017-0160|as indicating a motive for keeping her alive rather than killing her.|as indicating a motive for keeping her alive rather than killing her. +LJ017-0161|But probably the lump sum was the bait, or perhaps Smethurst wished to return to his temporarily deserted first wife.|But probably the lump sum was the bait, or perhaps Smethurst wished to return to his temporarily deserted first wife. +LJ017-0162|Whatever the exact cause which impelled him to crime, it seems certain that he began to give her some poison,|Whatever the exact cause which impelled him to crime, it seems certain that he began to give her some poison, +LJ017-0163|either arsenic or antimony, or both, in small quantities,|either arsenic or antimony, or both, in small quantities, +LJ017-0164|with the idea of subjecting her to the irritant poison slowly but surely until the desired effect, death, was achieved.|with the idea of subjecting her to the irritant poison slowly but surely until the desired effect, death, was achieved. +LJ017-0165|As she became worse and worse, Smethurst called in the best medical advice in Richmond,|As she became worse and worse, Smethurst called in the best medical advice in Richmond, +LJ017-0166|but was careful to prime them with his facts and lead them if possible to accept his diagnosis of the case.|but was careful to prime them with his facts and lead them if possible to accept his diagnosis of the case. +LJ017-0167|Smethurst was found guilty by the jury, and sentenced to death.|Smethurst was found guilty by the jury, and sentenced to death. +LJ017-0168|But a long public discussion followed, and in consequence he was reprieved.|But a long public discussion followed, and in consequence he was reprieved. +LJ017-0169|The Home Secretary, in a letter to the Lord Chief Baron, stated that, quote, although the facts are full of suspicion against Smethurst,|The Home Secretary, in a letter to the Lord Chief Baron, stated that, quote, although the facts are full of suspicion against Smethurst, +LJ017-0170|there is not absolute and complete evidence of his guilt. End quote.|there is not absolute and complete evidence of his guilt. End quote. +LJ017-0171|Smethurst was therefore given a free pardon for the offense of murder,|Smethurst was therefore given a free pardon for the offense of murder, +LJ017-0172|but he was subsequently again tried for bigamy, and sentenced to twelve months' imprisonment.|but he was subsequently again tried for bigamy, and sentenced to twelve months' imprisonment. +LJ017-0173|Catherine Wilson was a female poisoner who did business wholesale.|Catherine Wilson was a female poisoner who did business wholesale. +LJ017-0174|She was tried in April 1862 on suspicion of having attempted to poison a neighbor with oil of vitriol.|She was tried in April eighteen sixty-two on suspicion of having attempted to poison a neighbor with oil of vitriol. +LJ017-0175|The circumstances were strange. Mrs. Wilson had gone to the chemist's for medicine,|The circumstances were strange. Mrs. Wilson had gone to the chemist's for medicine, +LJ017-0176|and on her return had administered a dose of something which burnt the mouth badly, but did not prove fatal.|and on her return had administered a dose of something which burnt the mouth badly, but did not prove fatal. +LJ017-0177|Wilson was acquitted on this charge, but other suspicious facts cropped up while she was in Newgate.|Wilson was acquitted on this charge, but other suspicious facts cropped up while she was in Newgate. +LJ017-0178|It appeared that several persons with whom she was intimate had succumbed suddenly.|It appeared that several persons with whom she was intimate had succumbed suddenly. +LJ017-0179|In all cases the symptoms were much the same,|In all cases the symptoms were much the same, +LJ017-0180|vomiting, violent retching, purging, such as are visible in cholera, and all dated from the time when she knew a young man named Dixon,|vomiting, violent retching, purging, such as are visible in cholera, and all dated from the time when she knew a young man named Dixon, +LJ017-0181|who had been in the habit of taking colchicum for rheumatism. Mrs. Wilson heard then casually from a medical man|who had been in the habit of taking colchicum for rheumatism. Mrs. Wilson heard then casually from a medical man +LJ017-0182|that it was a very dangerous medicine, and she profited by what she had heard.|that it was a very dangerous medicine, and she profited by what she had heard. +LJ017-0183|Soon afterwards Dixon died, showing all the symptoms already described.|Soon afterwards Dixon died, showing all the symptoms already described. +LJ017-0184|Soon afterwards a friend, Mrs. Atkinson, came to London from Westmoreland, and stayed in Mrs. Wilson's house.|Soon afterwards a friend, Mrs. Atkinson, came to London from Westmoreland, and stayed in Mrs. Wilson's house. +LJ017-0185|She was in good health on leaving home, and had with her a large sum of money.|She was in good health on leaving home, and had with her a large sum of money. +LJ017-0186|While with Mrs. Wilson she became suddenly and alarmingly ill, and died in great agony.|While with Mrs. Wilson she became suddenly and alarmingly ill, and died in great agony. +LJ017-0187|Her husband, who came up to town, would not allow a post-mortem, and again Mrs. Wilson escaped.|Her husband, who came up to town, would not allow a post-mortem, and again Mrs. Wilson escaped. +LJ017-0188|Mrs. Atkinson's symptoms had been the same as Dixon's. Then Mrs. Wilson went to live with a man named Taylor,|Mrs. Atkinson's symptoms had been the same as Dixon's. Then Mrs. Wilson went to live with a man named Taylor, +LJ017-0189|who was presently attacked in the same way as the others, but, but, thanks to the prompt administration of remedies, he recovered.|who was presently attacked in the same way as the others, but, but, thanks to the prompt administration of remedies, he recovered. +LJ017-0190|After this came the charge of administering oil of vitriol, which failed, as has been described.|After this came the charge of administering oil of vitriol, which failed, as has been described. +LJ017-0191|Last of all Mrs. Wilson poisoned her landlady, Mrs. Soames, under precisely the same conditions as the foregoing.|Last of all Mrs. Wilson poisoned her landlady, Mrs. Soames, under precisely the same conditions as the foregoing. +LJ017-0192|Here, however, the evidence was strong and sufficient.|Here, however, the evidence was strong and sufficient. +LJ017-0193|It was proved that Mrs. Wilson had given Mrs. Soames something peculiar to drink,|It was proved that Mrs. Wilson had given Mrs. Soames something peculiar to drink, +LJ017-0194|that immediately afterwards Mrs. Soames was taken ill with vomiting and purging,|that immediately afterwards Mrs. Soames was taken ill with vomiting and purging, +LJ017-0195|and that Mrs. Wilson administered the same medicine again and again.|and that Mrs. Wilson administered the same medicine again and again. +LJ017-0196|The last time Mrs. Soames showed great reluctance to take it, but Wilson said it would certainly do her good.|The last time Mrs. Soames showed great reluctance to take it, but Wilson said it would certainly do her good. +LJ017-0197|This mysterious medicine Wilson kept carefully locked up, and allowed no one to see it,|This mysterious medicine Wilson kept carefully locked up, and allowed no one to see it, +LJ017-0198|but its nature was betrayed when this last victim also died.|but its nature was betrayed when this last victim also died. +LJ017-0199|The first post-mortem indicated death from natural causes,|The first post-mortem indicated death from natural causes, +LJ017-0200|but a more careful investigation attributed it beyond doubt to over-doses of colchicum.|but a more careful investigation attributed it beyond doubt to over-doses of colchicum. +LJ017-0201|Dr. Alfred Taylor, the great authority and writer on medical jurisprudence, corroborated this, and in his evidence on the trial|Dr. Alfred Taylor, the great authority and writer on medical jurisprudence, corroborated this, and in his evidence on the trial +LJ017-0202|fairly electrified the court by declaring it his opinion that many deaths, supposed to be from cholera, were really due to poison.|fairly electrified the court by declaring it his opinion that many deaths, supposed to be from cholera, were really due to poison. +LJ017-0203|This fact was referred to by the judge in his summing up,|This fact was referred to by the judge in his summing up, +LJ017-0204|who said that he feared it was only too true that secret poisoning was at that time very rife in the metropolis.|who said that he feared it was only too true that secret poisoning was at that time very rife in the metropolis. +LJ017-0205|Wilson was duly sentenced to death, and suffered impenitent, hardened, and without any confession of her guilt.|Wilson was duly sentenced to death, and suffered impenitent, hardened, and without any confession of her guilt. +LJ017-0206|Although murder by insidious methods had become more common, cases where violence of the most deadly and determined kind was offered|Although murder by insidious methods had become more common, cases where violence of the most deadly and determined kind was offered +LJ017-0207|had not quite disappeared. I will mention two cases of this class, one accompanied with piracy on the high seas,|had not quite disappeared. I will mention two cases of this class, one accompanied with piracy on the high seas, +LJ017-0208|the other perpetrated in a railway-carriage, and showing the promptitude with which criminals accept and utilize altered conditions of life,|the other perpetrated in a railway-carriage, and showing the promptitude with which criminals accept and utilize altered conditions of life, +LJ017-0209|more particularly as regards locomotion.|more particularly as regards locomotion. +LJ017-0210|The first case was that of the 'Flowery Land,'|The first case was that of the 'Flowery Land,' +LJ017-0211|which left London for Singapore on the 28th July, 1863, with a cargo of wine and other goods.|which left London for Singapore on the twenty-eighth July, eighteen sixty-three, with a cargo of wine and other goods. +LJ017-0212|Her captain was John Smith;|Her captain was John Smith; +LJ017-0213|the first and second mates, Karswell and Taffir; there were two other Englishmen on board, and the rest of the crew were a polyglot lot,|the first and second mates, Karswell and Taffir; there were two other Englishmen on board, and the rest of the crew were a polyglot lot, +LJ017-0214|most of them, as was proved by their subsequent acts, blackguards of the deepest dye.|most of them, as was proved by their subsequent acts, blackguards of the deepest dye. +LJ017-0215|Six were Spaniards, or rather natives of Manilla, and men of color; one was a Greek, another a Turk;|Six were Spaniards, or rather natives of Manilla, and men of color; one was a Greek, another a Turk; +LJ017-0216|there were also a Frenchman, a Norwegian (the carpenter), three Chinamen, a "Sclavonian," and a black on board.|there were also a Frenchman, a Norwegian (the carpenter), three Chinamen, a "Sclavonian," and a black on board. +LJ017-0217|Navigation and discipline could not be easy with such a nondescript crew.|Navigation and discipline could not be easy with such a nondescript crew. +LJ017-0218|The captain was kindly but somewhat intemperate, the first mate a man of some determination,|The captain was kindly but somewhat intemperate, the first mate a man of some determination, +LJ017-0219|and punishment such as rope's-ending and tying to the bulwarks had to be applied to get the work properly done.|and punishment such as rope's-ending and tying to the bulwarks had to be applied to get the work properly done. +LJ017-0220|The six Spaniards, the Greek, and the Turk were in the same watch,|The six Spaniards, the Greek, and the Turk were in the same watch, +LJ017-0221|eight truculent and reckless scoundrels, who, brooding over their fancied wrongs, and burning for revenge,|eight truculent and reckless scoundrels, who, brooding over their fancied wrongs, and burning for revenge, +LJ017-0222|hatched amongst them a plot to murder their officers and seize the ship.|hatched amongst them a plot to murder their officers and seize the ship. +LJ017-0223|The mutiny was organized with great secrecy, and broke out most unexpectedly in the middle of the night.|The mutiny was organized with great secrecy, and broke out most unexpectedly in the middle of the night. +LJ017-0224|A simultaneous attack was made upon the captain and the first mate.|A simultaneous attack was made upon the captain and the first mate. +LJ017-0225|The latter had the watch on deck.|The latter had the watch on deck. +LJ017-0226|One half of the mutineers fell upon him unawares with handspikes and capstan-bars.|One half of the mutineers fell upon him unawares with handspikes and capstan-bars. +LJ017-0227|He was struck down, imploring mercy, but they beat him about the head and face|He was struck down, imploring mercy, but they beat him about the head and face +LJ017-0228|till every feature was obliterated, and then, still living, flung him into the sea.|till every feature was obliterated, and then, still living, flung him into the sea. +LJ017-0229|Meanwhile the captain, roused from his berth,|Meanwhile the captain, roused from his berth, +LJ017-0230|came out of the cabin, was caught near the 'companion' by the rest of the mutineers, and promptly dispatched with daggers.|came out of the cabin, was caught near the 'companion' by the rest of the mutineers, and promptly dispatched with daggers. +LJ017-0231|His body was found lying in a pool of blood in a night-dress, stabbed over and over again in the left side.|His body was found lying in a pool of blood in a night-dress, stabbed over and over again in the left side. +LJ017-0232|The captain's brother, a passenger on board the 'Flowery Land,' was also stabbed to death and his body thrown overboard.|The captain's brother, a passenger on board the 'Flowery Land,' was also stabbed to death and his body thrown overboard. +LJ017-0233|The second mate, who had heard the hammering of the capstan-bars and the handspikes, with the first mate's and captain's agonized cries, had come out,|The second mate, who had heard the hammering of the capstan-bars and the handspikes, with the first mate's and captain's agonized cries, had come out, +LJ017-0234|verified the murderers, and then shut himself up in his cabin.|verified the murderers, and then shut himself up in his cabin. +LJ017-0235|He was soon summoned on deck, but as he would not move, the mutineers came down and stood in a circle round his berth.|He was soon summoned on deck, but as he would not move, the mutineers came down and stood in a circle round his berth. +LJ017-0236|Leon, or Lyons, who spoke English,|Leon, or Lyons, who spoke English, +LJ017-0237|when asked said they would spare his life if he would navigate the ship for them to the River Plate or Buenos Aires.|when asked said they would spare his life if he would navigate the ship for them to the River Plate or Buenos Aires. +LJ017-0238|Taffir, the second mate, agreed, but constantly went in fear of his life for the remainder of the voyage;|Taffir, the second mate, agreed, but constantly went in fear of his life for the remainder of the voyage; +LJ017-0239|and although the mutineers spared him, they ill-treated the Chinamen, and cut one badly with knives.|and although the mutineers spared him, they ill-treated the Chinamen, and cut one badly with knives. +LJ017-0240|Immediately after the murder cases of champagne, which formed part of the cargo, were brought on deck and broached;|Immediately after the murder cases of champagne, which formed part of the cargo, were brought on deck and broached; +LJ017-0241|the captain's cabin ransacked, his money and clothes divided amongst the mutineers, as well as much of the merchandise on board.|the captain's cabin ransacked, his money and clothes divided amongst the mutineers, as well as much of the merchandise on board. +LJ017-0242|Leon wished to make every one on board share and share alike, so as to implicate the innocent with the guilty;|Leon wished to make every one on board share and share alike, so as to implicate the innocent with the guilty; +LJ017-0243|but Vartos, or Watto, the Turk, would not allow any but the eight mutineers to have anything.|but Vartos, or Watto, the Turk, would not allow any but the eight mutineers to have anything. +LJ017-0244|The murders were perpetrated on the 10th September, and the ship continued her voyage for nearly three weeks, meeting and speaking one ship only.|The murders were perpetrated on the tenth September, and the ship continued her voyage for nearly three weeks, meeting and speaking one ship only. +LJ017-0245|On the 2nd October they sighted land, ten miles distant;|On the second October they sighted land, ten miles distant; +LJ017-0246|the mutineers took command of the ship, put her about till night-fall, by which time they had scuttled her, got out the boats, and all left the ship.|the mutineers took command of the ship, put her about till night-fall, by which time they had scuttled her, got out the boats, and all left the ship. +LJ017-0247|The rest of the crew were also permitted to embark, except the Chinamen, one of whom was thrown into the water and drowned,|The rest of the crew were also permitted to embark, except the Chinamen, one of whom was thrown into the water and drowned, +LJ017-0248|while the other two were left to go down in the ship, and were seen clinging to the tops until the waters closed over them.|while the other two were left to go down in the ship, and were seen clinging to the tops until the waters closed over them. +LJ017-0249|The boats reached the shore on the 4th October. Leon had prepared a plausible tale to the effect that they belonged to an American ship|The boats reached the shore on the fourth October. Leon had prepared a plausible tale to the effect that they belonged to an American ship +LJ017-0250|from Peru bound to Bordeaux, which had foundered at sea;|from Peru bound to Bordeaux, which had foundered at sea; +LJ017-0251|that they had been in the boats five days and nights, but that the captain and others had been lost.|that they had been in the boats five days and nights, but that the captain and others had been lost. +LJ017-0252|The place at which they landed was not far from the entrance to the River Plate.|The place at which they landed was not far from the entrance to the River Plate. +LJ017-0253|A farmer took them in for the night, and drove them next day to Rocha, a place north of Maldonado.|A farmer took them in for the night, and drove them next day to Rocha, a place north of Maldonado. +LJ017-0254|Taffir, the mate, finding there was a man who could speak English at another place twenty miles off,|Taffir, the mate, finding there was a man who could speak English at another place twenty miles off, +LJ017-0255|repaired there secretly, and so gave information to the Brazilian authorities.|repaired there secretly, and so gave information to the Brazilian authorities. +LJ017-0256|The mutineers were arrested, the case inquired into by a naval court-martial,|The mutineers were arrested, the case inquired into by a naval court-martial, +LJ017-0257|and the prisoners eventually surrendered to the British authorities, brought to England, and lodged in Newgate.|and the prisoners eventually surrendered to the British authorities, brought to England, and lodged in Newgate. +LJ017-0258|Their trial followed at the Central Criminal Court.|Their trial followed at the Central Criminal Court. +LJ017-0259|Eight were arraigned at the same time: six Spaniards, Leon, Blanco, Duranno, Santos, and Marsolino;|Eight were arraigned at the same time: six Spaniards, Leon, Blanco, Duranno, Santos, and Marsolino; +LJ017-0260|Vartos the Turk, and Carlos the Greek.|Vartos the Turk, and Carlos the Greek. +LJ017-0261|Seven were found guilty of murder on the high seas, and one, Carlos, acquitted.|Seven were found guilty of murder on the high seas, and one, Carlos, acquitted. +LJ017-0262|Two of the seven, Santos and Marsolino, were reprieved, and their sentences commuted to penal servitude for life;|Two of the seven, Santos and Marsolino, were reprieved, and their sentences commuted to penal servitude for life; +LJ017-0263|the remaining five were executed in one batch.|the remaining five were executed in one batch. +LJ017-0264|They were an abject, miserable crew, cowards at heart; but some, especially Lopez, continued bloodthirsty to the last.|They were an abject, miserable crew, cowards at heart; but some, especially Lopez, continued bloodthirsty to the last. +LJ017-0265|Lopez took a violent dislike to the officer of the ward in charge of them, and often expressed a keen desire to do for him.|Lopez took a violent dislike to the officer of the ward in charge of them, and often expressed a keen desire to do for him. +LJ017-0266|They none of them spoke much English except Leon, commonly called Lyons.|They none of them spoke much English except Leon, commonly called Lyons. +LJ017-0267|After condemnation, as the rules now kept capital convicts strictly apart, they could not be lodged in the two condemned cells,|After condemnation, as the rules now kept capital convicts strictly apart, they could not be lodged in the two condemned cells, +LJ017-0268|and they were each kept in an ordinary separate cell of the newly-constructed block, with the "traps," or square openings in the cell door,|and they were each kept in an ordinary separate cell of the newly-constructed block, with the "traps," or square openings in the cell door, +LJ017-0269|let down. A full view of them was thus at all times obtainable by the officers who, without intermission, day and night patrolled the ward.|let down. A full view of them was thus at all times obtainable by the officers who, without intermission, day and night patrolled the ward. +LJ017-0270|On the morning of execution the noise of fixing the gallows in the street outside awoke one or two of them.|On the morning of execution the noise of fixing the gallows in the street outside awoke one or two of them. +LJ017-0271|Lyons asked the time, and was told it was only five.|Lyons asked the time, and was told it was only five. +LJ017-0272|"Ah!" he remarked, "they will have to wait for us then till eight."|"Ah!" he remarked, "they will have to wait for us then till eight." +LJ017-0273|Lopez was more talkative.|Lopez was more talkative. +LJ017-0274|When the warder went in to call him he asked for his clothes. He was told he would have to wear his own.|When the warder went in to call him he asked for his clothes. He was told he would have to wear his own. +LJ017-0276|Then he wanted to know when the policemen would arrive, and was told none would come.|Then he wanted to know when the policemen would arrive, and was told none would come. +LJ017-0277|The soldiers then?|The soldiers then? +LJ017-0278|No soldiers either.|No soldiers either. +LJ017-0280|The convicts were pinioned one by one and sent singly out to the gallows.|The convicts were pinioned one by one and sent singly out to the gallows. +LJ017-0281|As the first to appear would have some time to wait for his fellows, a difficult and painful ordeal,|As the first to appear would have some time to wait for his fellows, a difficult and painful ordeal, +LJ017-0282|the seemingly most courageous was selected to lead the way.|the seemingly most courageous was selected to lead the way. +LJ017-0283|This was Duranno; but the sight of the heaving mass of uplifted, impassioned faces|This was Duranno; but the sight of the heaving mass of uplifted, impassioned faces +LJ017-0284|was too much for his nerves, and he so nearly fainted that he had to be seated in a chair.|was too much for his nerves, and he so nearly fainted that he had to be seated in a chair. +LJ018-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 21: Newgate Notorieties, part 2.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section twenty-one: Newgate Notorieties, part two. +LJ018-0002|In July 1864 occurred the murder of Mr. Briggs, a gentleman advanced in years and chief clerk in Robarts' bank.|In July eighteen sixty-four occurred the murder of Mr. Briggs, a gentleman advanced in years and chief clerk in Robarts' bank. +LJ018-0003|As the circumstances under which it was perpetrated were somewhat novel,|As the circumstances under which it was perpetrated were somewhat novel, +LJ018-0004|and as some time elapsed before the discovery and apprehension of the supposed murderer,|and as some time elapsed before the discovery and apprehension of the supposed murderer, +LJ018-0005|the public mind was greatly agitated by the affair for several months. The story of the murder must be pretty familiar to most of my readers.|the public mind was greatly agitated by the affair for several months. The story of the murder must be pretty familiar to most of my readers. +LJ018-0006|Mr. Briggs left the bank one afternoon as usual, dined with his daughter at Peckham,|Mr. Briggs left the bank one afternoon as usual, dined with his daughter at Peckham, +LJ018-0007|then returned to the city to take the train from Fenchurch Street home, traveling by the North London Railway.|then returned to the city to take the train from Fenchurch Street home, traveling by the North London Railway. +LJ018-0008|He lived at Hackney, but he never reached it alive.|He lived at Hackney, but he never reached it alive. +LJ018-0009|When the train arrived at Hackney station, a passenger who was about to enter one of the carriages found the cushions soaked with blood.|When the train arrived at Hackney station, a passenger who was about to enter one of the carriages found the cushions soaked with blood. +LJ018-0010|Inside the carriage was a hat, a walking-stick, and a small black leather bag.|Inside the carriage was a hat, a walking-stick, and a small black leather bag. +LJ018-0011|About the same time a body was discovered on the line near the railway-bridge by Victoria Park.|About the same time a body was discovered on the line near the railway-bridge by Victoria Park. +LJ018-0012|It was that of an aged man, whose head had been battered in by a life-preserver.|It was that of an aged man, whose head had been battered in by a life-preserver. +LJ018-0013|There was a deep wound just over the ear, the skull was fractured, and there were several other blows and wounds on the head.|There was a deep wound just over the ear, the skull was fractured, and there were several other blows and wounds on the head. +LJ018-0014|Strange to say, the unfortunate man was not yet dead, and he actually survived more than four-and-twenty hours.|Strange to say, the unfortunate man was not yet dead, and he actually survived more than four-and-twenty hours. +LJ018-0015|His identity was established by a bundle of letters in his pocket, which bore his full address:|His identity was established by a bundle of letters in his pocket, which bore his full address: +LJ018-0016|T. Briggs, Esq., Robarts & Co., Lombard Street.|T. Briggs, Esq., Robarts and Co., Lombard Street. +LJ018-0017|The friends of Mr. Briggs were communicated with, and it was ascertained that when he left home the morning of the murderous attack,|The friends of Mr. Briggs were communicated with, and it was ascertained that when he left home the morning of the murderous attack, +LJ018-0019|The stick and bag were his, but not the hat.|The stick and bag were his, but not the hat. +LJ018-0020|A desperate and deadly struggle must have taken place in the carriage, and the stain of a bloody hand marked the door.|A desperate and deadly struggle must have taken place in the carriage, and the stain of a bloody hand marked the door. +LJ018-0021|The facts of the murder and its object, robbery, were thus conclusively proved.|The facts of the murder and its object, robbery, were thus conclusively proved. +LJ018-0022|It was also easily established that the hat found in the carriage had been bought at Walker's, a hatter's in Crawford Street,|It was also easily established that the hat found in the carriage had been bought at Walker's, a hatter's in Crawford Street, +LJ018-0023|Marylebone; while within a few days Mr. Briggs' gold chain was traced to a jeweler's in Cheapside,|Marylebone; while within a few days Mr. Briggs' gold chain was traced to a jeweler's in Cheapside, +LJ018-0024|Mr. Death, who had given another in exchange for it to a man supposed to be a foreigner.|Mr. Death, who had given another in exchange for it to a man supposed to be a foreigner. +LJ018-0025|More precise clues to the murderer were not long wanting; indeed the readiness with which they were produced and followed up|More precise clues to the murderer were not long wanting; indeed the readiness with which they were produced and followed up +LJ018-0026|showed how greatly the publicity and wide dissemination of the news regarding murder facilitate the detection of crime.|showed how greatly the publicity and wide dissemination of the news regarding murder facilitate the detection of crime. +LJ018-0027|In little more than a week a cabman came forward and voluntarily made a statement which at once drew suspicion to a German, Franz Müller,|In little more than a week a cabman came forward and voluntarily made a statement which at once drew suspicion to a German, Franz Müller, +LJ018-0028|who had been a lodger of his. Müller had given the cabman's little daughter a jeweler's cardboard box bearing the name of Mr. Death.|who had been a lodger of his. Müller had given the cabman's little daughter a jeweler's cardboard box bearing the name of Mr. Death. +LJ018-0029|A photograph of Müller shown the jeweler was identified as the likeness of the man who had exchanged Mr. Briggs' chain.|A photograph of Müller shown the jeweler was identified as the likeness of the man who had exchanged Mr. Briggs' chain. +LJ018-0030|Last of all, the cabman swore that he had bought the very hat found in the carriage for Müller at the hatter's, Walker's of Crawford Street.|Last of all, the cabman swore that he had bought the very hat found in the carriage for Müller at the hatter's, Walker's of Crawford Street. +LJ018-0031|This fixed the crime pretty certainly upon Müller, who had already left the country, thus increasing suspicion under which he lay.|This fixed the crime pretty certainly upon Müller, who had already left the country, thus increasing suspicion under which he lay. +LJ018-0032|There was no mystery about his departure; he had gone to Canada, by the 'Victoria' sailing ship, starting from the London docks, and bound to New York.|There was no mystery about his departure; he had gone to Canada, by the 'Victoria' sailing ship, starting from the London docks, and bound to New York. +LJ018-0033|Directly the foregoing facts were established, a couple of detective officers, armed with a warrant to arrest Müller,|Directly the foregoing facts were established, a couple of detective officers, armed with a warrant to arrest Müller, +LJ018-0034|and accompanied by Mr. Death the jeweler and the cabman, went down to Liverpool and took the first steamer across the Atlantic.|and accompanied by Mr. Death the jeweler and the cabman, went down to Liverpool and took the first steamer across the Atlantic. +LJ018-0035|This was the 'City of Manchester,' which was expected to arrive some days before the 'Victoria,' and did so.|This was the 'City of Manchester,' which was expected to arrive some days before the 'Victoria,' and did so. +LJ018-0036|The officers went on board the 'Victoria' at once, Müller was identified by Mr. Death, and the arrest was made.|The officers went on board the 'Victoria' at once, Müller was identified by Mr. Death, and the arrest was made. +LJ018-0037|In searching the prisoner's box, Mr. Briggs' watch was found wrapped up in a piece of leather,|In searching the prisoner's box, Mr. Briggs' watch was found wrapped up in a piece of leather, +LJ018-0038|and Müller at the time of his capture was actually wearing Mr. Briggs' hat, cut down and somewhat altered.|and Müller at the time of his capture was actually wearing Mr. Briggs' hat, cut down and somewhat altered. +LJ018-0039|The prisoner was forthwith extradited and sent back to England, which he reached with his escort on the 17th September the same year.|The prisoner was forthwith extradited and sent back to England, which he reached with his escort on the seventeenth September the same year. +LJ018-0040|His trial followed at the next sessions of the Central Criminal Court, and ended in his conviction.|His trial followed at the next sessions of the Central Criminal Court, and ended in his conviction. +LJ018-0041|The case was one of circumstantial evidence, but, as Sir Robert Collyer the Solicitor-General pointed out,|The case was one of circumstantial evidence, but, as Sir Robert Collyer the Solicitor-General pointed out, +LJ018-0042|it was the strongest circumstantial evidence which had ever been brought forward in a murder case.|it was the strongest circumstantial evidence which had ever been brought forward in a murder case. +LJ018-0043|It was really evidence of facts which could not be controverted or explained away.|It was really evidence of facts which could not be controverted or explained away. +LJ018-0044|There was the prisoner's poverty, his inability to account for himself on the night of the murder, and his possession of the property of the murdered man.|There was the prisoner's poverty, his inability to account for himself on the night of the murder, and his possession of the property of the murdered man. +LJ018-0045|An alibi was set up for the defense, but not well substantiated, and the jury without hesitation returned a verdict of guilty.|An alibi was set up for the defense, but not well substantiated, and the jury without hesitation returned a verdict of guilty. +LJ018-0046|Müller protested after sentence of death had been passed upon him that he had been convicted on a false statement of facts.|Müller protested after sentence of death had been passed upon him that he had been convicted on a false statement of facts. +LJ018-0047|He adhered to this almost to the very last. His case had been warmly espoused by the Society for the Protection of Germans in this country,|He adhered to this almost to the very last. His case had been warmly espoused by the Society for the Protection of Germans in this country, +LJ018-0048|and powerful influence was exerted both here and abroad to obtain a reprieve.|and powerful influence was exerted both here and abroad to obtain a reprieve. +LJ018-0049|Müller knew that any confession would ruin his chances of escape.|Müller knew that any confession would ruin his chances of escape. +LJ018-0050|His arguments were specious and evasive when pressed to confess. "Why should man confess to man?" he replied;|His arguments were specious and evasive when pressed to confess. "Why should man confess to man?" he replied; +LJ018-0051|man cannot forgive man, only God can do so. Man is therefore only accountable to God.|man cannot forgive man, only God can do so. Man is therefore only accountable to God. +LJ018-0052|But on the gallows, when the cap was over his eyes and the rope had been adjusted round his neck, and within a second of the moment when he would be launched into eternity,|But on the gallows, when the cap was over his eyes and the rope had been adjusted round his neck, and within a second of the moment when he would be launched into eternity, +LJ018-0053|he whispered in the ear of the German pastor who attended him on the scaffold,|he whispered in the ear of the German pastor who attended him on the scaffold, +LJ018-0054|While in the condemned cell he conversed freely with the warders in broken English or through an interpreter.|While in the condemned cell he conversed freely with the warders in broken English or through an interpreter. +LJ018-0055|He is described as not a bad-looking man, with a square German type of face,|He is described as not a bad-looking man, with a square German type of face, +LJ018-0056|blue eyes which were generally half closed, and very fair hair.|blue eyes which were generally half closed, and very fair hair. +LJ018-0057|He was short in stature, his legs were light for the upper part of his body, which was powerful, almost herculean.|He was short in stature, his legs were light for the upper part of his body, which was powerful, almost herculean. +LJ018-0058|It is generally supposed that he committed the murder under a sudden access of covetousness and greed.|It is generally supposed that he committed the murder under a sudden access of covetousness and greed. +LJ018-0059|He saw Mr. Briggs' watch-chain, and followed him instantly into the carriage, determined to have it at all costs.|He saw Mr. Briggs' watch-chain, and followed him instantly into the carriage, determined to have it at all costs. +LJ018-0060|His crime under this aspect of it was less premeditated, and less atrocious therefore, than that of Lefroy.|His crime under this aspect of it was less premeditated, and less atrocious therefore, than that of Lefroy. +LJ018-0061|One other curious murder may be added to the two foregoing.|One other curious murder may be added to the two foregoing. +LJ018-0062|Christian Sattler was by birth a German.|Christian Sattler was by birth a German. +LJ018-0063|He had led a wild life; had left his native land and enlisted first in the French army in Algeria,|He had led a wild life; had left his native land and enlisted first in the French army in Algeria, +LJ018-0064|afterwards in the British German Legion raised for the Crimean War.|afterwards in the British German Legion raised for the Crimean War. +LJ018-0065|At the disbandment of the force, as he was without resources, he turned his attention to hotel robberies, by which he lived for some years.|At the disbandment of the force, as he was without resources, he turned his attention to hotel robberies, by which he lived for some years. +LJ018-0066|He at length stole a carpet-bag containing valuables, and fled to Hamburgh.|He at length stole a carpet-bag containing valuables, and fled to Hamburgh. +LJ018-0067|Thither, he was pursued by a detective officer,|Thither, he was pursued by a detective officer, +LJ018-0068|Inspector Thain, who, being unable to obtain his extradition legally, had him inveigled on board an English steamer,|Inspector Thain, who, being unable to obtain his extradition legally, had him inveigled on board an English steamer, +LJ018-0069|where the arrest was made.|where the arrest was made. +LJ018-0070|Sattler was ironed for safe custody,|Sattler was ironed for safe custody, +LJ018-0071|a proceeding which he vehemently resented, and begged that they might be removed, as the handcuffs hurt his wrists.|a proceeding which he vehemently resented, and begged that they might be removed, as the handcuffs hurt his wrists. +LJ018-0072|The inspector said that they could not be removed till he reached England.|The inspector said that they could not be removed till he reached England. +LJ018-0073|This reply of his contained no promise of immediate release.|This reply of his contained no promise of immediate release. +LJ018-0074|Sattler probably misunderstood, and he declared that the police officer had broken faith with him, having, moreover, stated that|Sattler probably misunderstood, and he declared that the police officer had broken faith with him, having, moreover, stated that +LJ018-0075|while at sea the captain of the ship was responsible for the security of the prisoner.|while at sea the captain of the ship was responsible for the security of the prisoner. +LJ018-0076|As Sattler brooded over his wrongs, his rage got the upper hand, and he resolved to wreak it upon Thain.|As Sattler brooded over his wrongs, his rage got the upper hand, and he resolved to wreak it upon Thain. +LJ018-0077|Although manacled, he managed to get a pistol from his chest and load it.|Although manacled, he managed to get a pistol from his chest and load it. +LJ018-0078|The next time Thain entered his cabin he fired at him point-blank, and lodged three bullets in his breast.|The next time Thain entered his cabin he fired at him point-blank, and lodged three bullets in his breast. +LJ018-0079|The unfortunate man survived till he landed, but died in Guy's Hospital.|The unfortunate man survived till he landed, but died in Guy's Hospital. +LJ018-0080|Sattler was tried for murder and convicted;|Sattler was tried for murder and convicted; +LJ018-0081|his defense being that he had intended to commit suicide, but that, on the appearance of this officer who had wronged him,|his defense being that he had intended to commit suicide, but that, on the appearance of this officer who had wronged him, +LJ018-0082|he had yielded to an irresistible impulse to kill him.|he had yielded to an irresistible impulse to kill him. +LJ018-0083|Sattler was a very excitable although not an ill-tempered man.|Sattler was a very excitable although not an ill-tempered man. +LJ018-0084|While in Newgate awaiting trial he frequently tried to justify his murder by declaring that the police officer had broken faith with him.|While in Newgate awaiting trial he frequently tried to justify his murder by declaring that the police officer had broken faith with him. +LJ018-0085|He would shoot any man or any policeman like a dog, or any number of them, who had treated him in that way.|He would shoot any man or any policeman like a dog, or any number of them, who had treated him in that way. +LJ018-0086|His demeanor immediately preceding his execution I have referred to in the last chapter.|His demeanor immediately preceding his execution I have referred to in the last chapter. +LJ018-0087|Several cases of gigantic fraud, rivaling any already recorded, were brought to light between 1856 and 1873.|Several cases of gigantic fraud, rivaling any already recorded, were brought to light between eighteen fifty-six and eighteen seventy-three. +LJ018-0088|I propose next to describe the leading features of the most important of these.|I propose next to describe the leading features of the most important of these. +LJ018-0089|Another case of long-continued successful forgery was brought to light two years after the convictions of Saward and his accomplices.|Another case of long-continued successful forgery was brought to light two years after the convictions of Saward and his accomplices. +LJ018-0090|This conspiracy was cleverly planned, but had scarcely so many ramifications as that of Saward. Its originators were a couple of men,|This conspiracy was cleverly planned, but had scarcely so many ramifications as that of Saward. Its originators were a couple of men, +LJ018-0091|Wagner and Bateman, who had already been convicted of systematic forgery, and sentenced to transportation, but they had been released on ticket-of-leave|Wagner and Bateman, who had already been convicted of systematic forgery, and sentenced to transportation, but they had been released on ticket-of-leave +LJ018-0092|in 1856.|in eighteen fifty-six. +LJ018-0093|As a blind for their new frauds, they set up as law-stationers in York Buildings, Adelphi, and at once commenced their nefarious traffic.|As a blind for their new frauds, they set up as law-stationers in York Buildings, Adelphi, and at once commenced their nefarious traffic. +LJ018-0094|Forged cheques and bills were soon uttered in great numbers, as well as base coin.|Forged cheques and bills were soon uttered in great numbers, as well as base coin. +LJ018-0095|The police suspecting the house in York Buildings, put a watch on the premises, which they kept up for more than a year,|The police suspecting the house in York Buildings, put a watch on the premises, which they kept up for more than a year, +LJ018-0096|and thus obtained personal knowledge of all who passed in and out, but without obtaining any direct evidence.|and thus obtained personal knowledge of all who passed in and out, but without obtaining any direct evidence. +LJ018-0097|At length a man was caught in the act of passing a forged cheque at the Union Bank,|At length a man was caught in the act of passing a forged cheque at the Union Bank, +LJ018-0098|and recognized as one of the frequenters of the bogus law-stationers. His arrest led to that of others.|and recognized as one of the frequenters of the bogus law-stationers. His arrest led to that of others. +LJ018-0099|Among them was a man named Chandler, formerly a bill discounter by profession, who by degrees, to meet his extravagant expenditure,|Among them was a man named Chandler, formerly a bill discounter by profession, who by degrees, to meet his extravagant expenditure, +LJ018-0100|took to appropriating the bills intrusted to him, and so lost his business, after which he became a clerk to Messrs. Wagner and Bateman.|took to appropriating the bills intrusted to him, and so lost his business, after which he became a clerk to Messrs. Wagner and Bateman. +LJ018-0101|Chandler while in Newgate turned informer, and betrayed the whole conspiracy.|Chandler while in Newgate turned informer, and betrayed the whole conspiracy. +LJ018-0102|Besides his employers, a jeweler named Humphreys was in the "swim," at whose shop in Red Lion Square was discovered a quantity of base gold|Besides his employers, a jeweler named Humphreys was in the "swim," at whose shop in Red Lion Square was discovered a quantity of base gold +LJ018-0103|and silver coins, with all the latest appliances for coining, including those of electroplating;|and silver coins, with all the latest appliances for coining, including those of electroplating; +LJ018-0104|also a furniture dealer and one or two more commonplace rogues. The arch villain was never taken into custody.|also a furniture dealer and one or two more commonplace rogues. The arch villain was never taken into custody. +LJ018-0105|He, like Saward, was an artist in penmanship.|He, like Saward, was an artist in penmanship. +LJ018-0106|He was a German named Kerp,|He was a German named Kerp, +LJ018-0107|eighty years of age, who had spent his whole life in imitating other people's signatures, and had acquired the most consummate skill in the practice.|eighty years of age, who had spent his whole life in imitating other people's signatures, and had acquired the most consummate skill in the practice. +LJ018-0108|His copies were generally pronounced indistinguishable from and as good as the originals.|His copies were generally pronounced indistinguishable from and as good as the originals. +LJ018-0109|The aged but wary Kerp, the moment the plot was discovered, vanished, and was never more heard of.|The aged but wary Kerp, the moment the plot was discovered, vanished, and was never more heard of. +LJ018-0110|Much the same plan was adopted by these forgers as by Saward to get their cheques cashed.|Much the same plan was adopted by these forgers as by Saward to get their cheques cashed. +LJ018-0111|They advertised for clerks, and employed the most likely of the applicants by sending them to the bank.|They advertised for clerks, and employed the most likely of the applicants by sending them to the bank. +LJ018-0112|It was one of these, Glendinning, who had allowed himself to be utilized for some time in this way, whose capture led to the breaking up of the gang.|It was one of these, Glendinning, who had allowed himself to be utilized for some time in this way, whose capture led to the breaking up of the gang. +LJ018-0113|The principals in this conspiracy, Wagner and Bateman, were sentenced to penal servitude for life,|The principals in this conspiracy, Wagner and Bateman, were sentenced to penal servitude for life, +LJ018-0114|the others to twenty and ten years.|the others to twenty and ten years. +LJ018-0115|It was stated in evidence that the monies obtained by these forgeries amounted to £8000 or £10,000,|It was stated in evidence that the monies obtained by these forgeries amounted to eight thousand pounds or ten thousand pounds, +LJ018-0116|and that the forged cheques which had been presented, but refused, amounted to double the sum.|and that the forged cheques which had been presented, but refused, amounted to double the sum. +LJ018-0117|Wagner, after conviction, offered to reveal, for a reward of £3000|Wagner, after conviction, offered to reveal, for a reward of three thousand pounds +LJ018-0118|a system which had long been in practice of defrauding the Exchequer of vast sums by means of forged stamps.|a system which had long been in practice of defrauding the Exchequer of vast sums by means of forged stamps. +LJ018-0120|A more elaborate plot in many ways, more secretly, more patiently prepared than the preceding, or indeed than any in the calendar,|A more elaborate plot in many ways, more secretly, more patiently prepared than the preceding, or indeed than any in the calendar, +LJ018-0121|was the case of the forgeries upon the Bank of England discovered in 1863,|was the case of the forgeries upon the Bank of England discovered in eighteen sixty-three, +LJ018-0122|but not before the forged paper had been put in circulation for more than a couple of years. In 1861,|but not before the forged paper had been put in circulation for more than a couple of years. In eighteen sixty-one, +LJ018-0123|a man named Burnett came with his wife and took up his residence at Whitchurch, Hampshire, at no great distance from Laverstock,|a man named Burnett came with his wife and took up his residence at Whitchurch, Hampshire, at no great distance from Laverstock, +LJ018-0124|where are Messrs. Portal's mills for the manufacture of bank-note paper.|where are Messrs. Portal's mills for the manufacture of bank-note paper. +LJ018-0125|Burnett had only just come out of jail after completing a sentence of penal servitude.|Burnett had only just come out of jail after completing a sentence of penal servitude. +LJ018-0126|His object in visiting Whitchurch was to undermine the honesty of some workman in the mills;|His object in visiting Whitchurch was to undermine the honesty of some workman in the mills; +LJ018-0127|and he eventually succeeded, his wife making the first overtures, in persuading a lad named Brown to steal some of the bank paper.|and he eventually succeeded, his wife making the first overtures, in persuading a lad named Brown to steal some of the bank paper. +LJ018-0128|Brown took several sheets, and then was detected by Brewer, a fellow-workman of superior grade,|Brown took several sheets, and then was detected by Brewer, a fellow-workman of superior grade, +LJ018-0129|who threatened to betray the theft. But Brewer, either before or after this, succumbed to temptation,|who threatened to betray the theft. But Brewer, either before or after this, succumbed to temptation, +LJ018-0130|and stole paper on a much larger scale than Brown.|and stole paper on a much larger scale than Brown. +LJ018-0131|All that was taken was handed over to Burnett, or a "woman in black" whom Brown met by appointment at Waterloo station.|All that was taken was handed over to Burnett, or a "woman in black" whom Brown met by appointment at Waterloo station. +LJ018-0132|To facilitate his operations, Brewer obtained a false master key from Burnett,|To facilitate his operations, Brewer obtained a false master key from Burnett, +LJ018-0133|which gave him access to all parts of the mills, the packing-room included.|which gave him access to all parts of the mills, the packing-room included. +LJ018-0134|In this part of the mills a large quantity of bank-note paper was kept at the period of the robbery,|In this part of the mills a large quantity of bank-note paper was kept at the period of the robbery, +LJ018-0135|and in the states known as "water-leaf" and "sized," which are the penultimate processes of manufacture.|and in the states known as "water-leaf" and "sized," which are the penultimate processes of manufacture. +LJ018-0136|One more remains, that of "glazing," without which no paper is issued for engraving.|One more remains, that of "glazing," without which no paper is issued for engraving. +LJ018-0137|None of the stolen paper was glazed, and this was an important clue to the subsequent discovery of the crime.|None of the stolen paper was glazed, and this was an important clue to the subsequent discovery of the crime. +LJ018-0138|Some time in 1862, a large deficiency in stock of bank paper unglazed was discovered at the mills.|Some time in eighteen sixty-two, a large deficiency in stock of bank paper unglazed was discovered at the mills. +LJ018-0139|Soon afterwards the inspectors of bank-notes at the Bank of England detected the presentation at the bank of spurious notes on genuine paper.|Soon afterwards the inspectors of bank-notes at the Bank of England detected the presentation at the bank of spurious notes on genuine paper. +LJ018-0140|The two facts taken in conjunction|The two facts taken in conjunction +LJ018-0141|led to the employment of the police, and the offer of a reward of £1500 for the detection of the offenders.|led to the employment of the police, and the offer of a reward of fifteen hundred pounds for the detection of the offenders. +LJ018-0142|By this time Brown alone had stolen three or four hundred sheets,|By this time Brown alone had stolen three or four hundred sheets, +LJ018-0143|each containing two notes, many of the sheets suitable for engraving any kind of note from £1000 downwards.|each containing two notes, many of the sheets suitable for engraving any kind of note from one thousand pounds downwards. +LJ018-0144|The amount of Brewer's abstractions (who was eventually acquitted) was never exactly estimated.|The amount of Brewer's abstractions (who was eventually acquitted) was never exactly estimated. +LJ018-0145|Suspicion appears to have rested on Brown, who had left Laverstock,|Suspicion appears to have rested on Brown, who had left Laverstock, +LJ018-0146|and he was soon approached by the police. Almost directly he was questioned he made a clean breast of the whole affair.|and he was soon approached by the police. Almost directly he was questioned he made a clean breast of the whole affair. +LJ018-0147|The next step was to take the principals, and under such circumstances as would insure their conviction.|The next step was to take the principals, and under such circumstances as would insure their conviction. +LJ018-0148|A watch was set on Burnett, who was followed to the shop of one Buncher, a butcher in Strutton Ground.|A watch was set on Burnett, who was followed to the shop of one Buncher, a butcher in Strutton Ground. +LJ018-0149|Buncher was then tracked to North Kent Terrace, New Cross, where a Mr. and Mrs. Campbell resided,|Buncher was then tracked to North Kent Terrace, New Cross, where a Mr. and Mrs. Campbell resided, +LJ018-0150|with whom he did business in exchanging the false notes.|with whom he did business in exchanging the false notes. +LJ018-0151|The police officers now taxed Mrs. Campbell with complicity, and frightened her into collusion.|The police officers now taxed Mrs. Campbell with complicity, and frightened her into collusion. +LJ018-0152|With her assistance on a certain day a couple of bricks were taken out of the wall dividing her front and back parlors;|With her assistance on a certain day a couple of bricks were taken out of the wall dividing her front and back parlors; +LJ018-0153|the officers ensconced themselves in the latter, and waited for Buncher's expected visit.|the officers ensconced themselves in the latter, and waited for Buncher's expected visit. +LJ018-0154|He came to complete a sale of forged notes, and he wanted a couple of hundred pounds for what he had.|He came to complete a sale of forged notes, and he wanted a couple of hundred pounds for what he had. +LJ018-0155|Mrs. Campbell offered him less, and there was an altercation, in the course of which Buncher became very violent, and at length,|Mrs. Campbell offered him less, and there was an altercation, in the course of which Buncher became very violent, and at length, +LJ018-0156|after using much intemperate language, he left the place in a huff.|after using much intemperate language, he left the place in a huff. +LJ018-0157|In the course of his remarks, however, he said,|In the course of his remarks, however, he said, +LJ018-0158|I am the man that has got all the bank paper; I have £30,000 now, and the Bank of England cannot stop it.|I am the man that has got all the bank paper; I have thirty thousand pounds now, and the Bank of England cannot stop it. +LJ018-0159|This was all the police wanted to know.|This was all the police wanted to know. +LJ018-0160|They next watched Buncher, and found that he paid frequent visits to Birmingham.|They next watched Buncher, and found that he paid frequent visits to Birmingham. +LJ018-0161|They also discovered that through the intermediacy of one Robert Cummings, well known as a reputed coiner,|They also discovered that through the intermediacy of one Robert Cummings, well known as a reputed coiner, +LJ018-0162|he had been introduced to a man named Griffiths, an engraver and copper-plate printer.|he had been introduced to a man named Griffiths, an engraver and copper-plate printer. +LJ018-0163|Griffiths was an unusually clever and skilful workman,|Griffiths was an unusually clever and skilful workman, +LJ018-0164|who had devoted all his talent and all his energies for some seventeen years to the fabrication of false bank-notes.|who had devoted all his talent and all his energies for some seventeen years to the fabrication of false bank-notes. +LJ018-0165|On a certain day, the 27th October, 1862, the two were arrested simultaneously;|On a certain day, the twenty-seventh October, eighteen sixty-two, the two were arrested simultaneously; +LJ018-0166|Buncher in London, and Griffiths in Birmingham.|Buncher in London, and Griffiths in Birmingham. +LJ018-0167|Nothing was found in Buncher's premises in Strutton Ground, which were thoroughly searched,|Nothing was found in Buncher's premises in Strutton Ground, which were thoroughly searched, +LJ018-0168|but proofs of Griffiths' guilt were at once apparent on entering his work-room.|but proofs of Griffiths' guilt were at once apparent on entering his work-room. +LJ018-0169|In one corner was a printing-press actually in use, and on it were twenty-one forged Bank of England notes, without date or signature.|In one corner was a printing-press actually in use, and on it were twenty-one forged Bank of England notes, without date or signature. +LJ018-0170|On the bed were twenty forged ten-pound notes complete and ready for use, and twenty-five five-pound notes.|On the bed were twenty forged ten-pound notes complete and ready for use, and twenty-five five-pound notes. +LJ018-0171|"Mother plates" for engraving the body of the notes lay about, and other plates for various processes.|"Mother plates" for engraving the body of the notes lay about, and other plates for various processes. +LJ018-0172|More than this, Griffiths took the police to a field where, in a bank, a number of other plates were secreted.|More than this, Griffiths took the police to a field where, in a bank, a number of other plates were secreted. +LJ018-0173|Griffiths afterwards admitted that he had been employed in defrauding the bank since 1846,|Griffiths afterwards admitted that he had been employed in defrauding the bank since eighteen forty-six, +LJ018-0174|and the prominent part he played secured for him on conviction the heaviest sentence of the law.|and the prominent part he played secured for him on conviction the heaviest sentence of the law. +LJ018-0175|This was penal servitude for life, Buncher's sentence being twenty-five, and Burnett's twenty years.|This was penal servitude for life, Buncher's sentence being twenty-five, and Burnett's twenty years. +LJ018-0176|Cummings, who had introduced Buncher to Griffiths, was also tried for being in possession of stolen bank paper for improper purposes.|Cummings, who had introduced Buncher to Griffiths, was also tried for being in possession of stolen bank paper for improper purposes. +LJ018-0177|But as there was no independent corroboration of the informer's evidence, according to the custom of the British law,|But as there was no independent corroboration of the informer's evidence, according to the custom of the British law, +LJ018-0178|the case was considered not proved, and he was acquitted.|the case was considered not proved, and he was acquitted. +LJ018-0179|On his return to Newgate to be finally discharged, Cummings jumped up the stairs and fairly danced for joy.|On his return to Newgate to be finally discharged, Cummings jumped up the stairs and fairly danced for joy. +LJ018-0180|But he was not long at large; he was too active an evil-doer|But he was not long at large; he was too active an evil-doer +LJ018-0181|and was perpetually in trouble. Commencing life as a resurrection man, when that trade failed through the change in the law,|and was perpetually in trouble. Commencing life as a resurrection man, when that trade failed through the change in the law, +LJ018-0182|and no more bodies were to be bought|and no more bodies were to be bought +LJ018-0183|he devoted his energies to coining and forgery, and in the latter line was a friend and associate of Saward's.|he devoted his energies to coining and forgery, and in the latter line was a friend and associate of Saward's. +LJ018-0184|One narrow escape he had, however, before he abandoned his old business.|One narrow escape he had, however, before he abandoned his old business. +LJ018-0185|A Bow Street officer saw him leaving London in the evening by Camberwell Green, accompanied by two other men.|A Bow Street officer saw him leaving London in the evening by Camberwell Green, accompanied by two other men. +LJ018-0186|It was well known that they were resurrectionists, and a strict watch was kept at all the turnpike gates on the southern roads leading into London.|It was well known that they were resurrectionists, and a strict watch was kept at all the turnpike gates on the southern roads leading into London. +LJ018-0187|An officer was placed for this purpose at New Cross, Camberwell, and Kennington gates.|An officer was placed for this purpose at New Cross, Camberwell, and Kennington gates. +LJ018-0188|Presently "Old Bob" drove up to Camberwell Gate in the same cart in which he had been seen to start.|Presently "Old Bob" drove up to Camberwell Gate in the same cart in which he had been seen to start. +LJ018-0189|The officers rushed out to detain him. "What have you got here? We must search the cart," they cry.|The officers rushed out to detain him. "What have you got here? We must search the cart," they cry. +LJ018-0190|"By all means," replies Bob, and a close investigation follows, without any detection of the corpse concealed.|"By all means," replies Bob, and a close investigation follows, without any detection of the corpse concealed. +LJ018-0191|Bob was therefore allowed to pass on.|Bob was therefore allowed to pass on. +LJ018-0192|But they had the body, all the same; it had been dressed up in decent clothes and made to stand upright in the cart.|But they had the body, all the same; it had been dressed up in decent clothes and made to stand upright in the cart. +LJ018-0193|With the police officers it had passed muster as a living member of the party.|With the police officers it had passed muster as a living member of the party. +LJ018-0194|Cummings was repeatedly "run in" for the offense of coining and uttering bad money, whether coin or notes.|Cummings was repeatedly "run in" for the offense of coining and uttering bad money, whether coin or notes. +LJ018-0195|His regular trade, followed before he took to the life of resurrectionist, was that of an engraver.|His regular trade, followed before he took to the life of resurrectionist, was that of an engraver. +LJ018-0196|He was a notorious criminal,|He was a notorious criminal, +LJ018-0197|an habitual offender in his own particular line, one who would stick at no trifles to evade detection or escape capture.|an habitual offender in his own particular line, one who would stick at no trifles to evade detection or escape capture. +LJ018-0198|It is told of "Bob" Brennan, an official specially employed for years by the Mint|It is told of "Bob" Brennan, an official specially employed for years by the Mint +LJ018-0199|to watch and prosecute coiners, that he received information that coining was carried on by Cummings and others at a place in Westminster.|to watch and prosecute coiners, that he received information that coining was carried on by Cummings and others at a place in Westminster. +LJ018-0200|He went there with a posse of officers and forced his way upstairs to the first floor,|He went there with a posse of officers and forced his way upstairs to the first floor, +LJ018-0201|where the coiners, unexpectedly disturbed, fell an easy prey.|where the coiners, unexpectedly disturbed, fell an easy prey. +LJ018-0202|But the police nearly paid the penalty of capture with their lives.|But the police nearly paid the penalty of capture with their lives. +LJ018-0203|Proceeding cautiously down the stairs, they found that the flooring at the bottom had been taken up.|Proceeding cautiously down the stairs, they found that the flooring at the bottom had been taken up. +LJ018-0204|Where it had lain was a yawning gulf or trap sufficient to do for the whole body of police engaged in the capture.|Where it had lain was a yawning gulf or trap sufficient to do for the whole body of police engaged in the capture. +LJ018-0205|Cummings was caught shortly afterwards.|Cummings was caught shortly afterwards. +LJ018-0206|He was a tall, slender man, with a long face and iron-gray hair.|He was a tall, slender man, with a long face and iron-gray hair. +LJ018-0207|The community of coiners of which he was so notorious a member|The community of coiners of which he was so notorious a member +LJ018-0208|were a low lot, the lowest among criminals except, perhaps, the 'smashers,' or those who passed the counterfeit money.|were a low lot, the lowest among criminals except, perhaps, the 'smashers,' or those who passed the counterfeit money. +LJ018-0209|It was not easy to detect coiners, or bring home their guilt to them.|It was not easy to detect coiners, or bring home their guilt to them. +LJ018-0210|Those who manufactured and those who passed had no direct dealings with each other.|Those who manufactured and those who passed had no direct dealings with each other. +LJ018-0211|The false coin was bought by an agent from an agent, and dealings were carried on secretly at the "Clock House" in Seven Dials.|The false coin was bought by an agent from an agent, and dealings were carried on secretly at the "Clock House" in Seven Dials. +LJ018-0212|The annals of fraudulent crime probably contain nothing|The annals of fraudulent crime probably contain nothing +LJ018-0213|which in dramatic interest can compare with the conviction of William Roupell for forgery.|which in dramatic interest can compare with the conviction of William Roupell for forgery. +LJ018-0214|As the case must still be well remembered by the present generation, it will be necessary to give here only the briefest summary.|As the case must still be well remembered by the present generation, it will be necessary to give here only the briefest summary. +LJ018-0215|William Roupell was the eldest but illegitimate son of a wealthy man who subsequently married Roupell's mother, and had further legitimate issue.|William Roupell was the eldest but illegitimate son of a wealthy man who subsequently married Roupell's mother, and had further legitimate issue. +LJ018-0216|William was brought up as an attorney, and became in due course his father's man of business.|William was brought up as an attorney, and became in due course his father's man of business. +LJ018-0217|As such he had pretty general control over his father's estates and affairs.|As such he had pretty general control over his father's estates and affairs. +LJ018-0218|In 1855|In eighteen fifty-five +LJ018-0219|he instructed certain solicitors to prepare a deed of gift as from his father, conveying to him estates near Kingston.|he instructed certain solicitors to prepare a deed of gift as from his father, conveying to him estates near Kingston. +LJ018-0220|The old gentleman's signature to this deed of gift was a forgery,|The old gentleman's signature to this deed of gift was a forgery, +LJ018-0221|but upon this forged and false conveyance William Roupell, who had already embarked upon a career of wild extravagance,|but upon this forged and false conveyance William Roupell, who had already embarked upon a career of wild extravagance, +LJ018-0222|obtained a mortgage of £7000.|obtained a mortgage of seven thousand pounds. +LJ018-0223|In 1856 the father died.|In eighteen fifty-six the father died. +LJ018-0224|It had been supposed up to this date that he had willed his property, amounting in all to upwards of £200,000,|It had been supposed up to this date that he had willed his property, amounting in all to upwards of two hundred thousand pounds, +LJ018-0225|but after the funeral William Roupell produced another and a later will,|but after the funeral William Roupell produced another and a later will, +LJ018-0226|leaving everything to the widow, and constituting William sole executor.|leaving everything to the widow, and constituting William sole executor. +LJ018-0227|This will was a deliberate forgery.|This will was a deliberate forgery. +LJ018-0228|Five or six years later, William Roupell minutely described how he had effected the fraud.|Five or six years later, William Roupell minutely described how he had effected the fraud. +LJ018-0229|The day his father died he got the keys of his private bureau, opened it, and took out the authentic will.|The day his father died he got the keys of his private bureau, opened it, and took out the authentic will. +LJ018-0230|After reading it, and finding this unfavorable to himself, he resolved to carry out his deliberate plan,|After reading it, and finding this unfavorable to himself, he resolved to carry out his deliberate plan, +LJ018-0231|namely, to suppress it and substitute another.|namely, to suppress it and substitute another. +LJ018-0232|He himself prepared it on a blank form which he had brought with him on purpose.|He himself prepared it on a blank form which he had brought with him on purpose. +LJ018-0233|To this fraudulent instrument he appended forged signatures, and in due course obtained probate.|To this fraudulent instrument he appended forged signatures, and in due course obtained probate. +LJ018-0234|As he possessed nearly unbounded influence over his mother, her accession to the property meant that William could dispose of it as he pleased.|As he possessed nearly unbounded influence over his mother, her accession to the property meant that William could dispose of it as he pleased. +LJ018-0235|He embarked forthwith in a career of the wildest extravagance, and ere long he had parted in his mother's name with most of the landed estates.|He embarked forthwith in a career of the wildest extravagance, and ere long he had parted in his mother's name with most of the landed estates. +LJ018-0236|One large item of his expenditure was a contested election at Lambeth, which he gained at a cost of £10,000.|One large item of his expenditure was a contested election at Lambeth, which he gained at a cost of ten thousand pounds. +LJ018-0237|No fortune could stand the inroads he made into his mother's money,|No fortune could stand the inroads he made into his mother's money, +LJ018-0238|and in 1862 he was obliged to fly the country, hopelessly and irretrievably ruined.|and in eighteen sixty-two he was obliged to fly the country, hopelessly and irretrievably ruined. +LJ018-0239|His disappearance gave color and substance to evil reports already in circulation that the will and conveyance above referred to|His disappearance gave color and substance to evil reports already in circulation that the will and conveyance above referred to +LJ018-0240|were fictitious documents. His next brother, who should have inherited under the authentic will,|were fictitious documents. His next brother, who should have inherited under the authentic will, +LJ018-0241|forthwith brought an ejectment on the possessor of lands purchased on the authority of the forged conveyance and will.|forthwith brought an ejectment on the possessor of lands purchased on the authority of the forged conveyance and will. +LJ018-0242|The case was tried at Guildford Assizes, and caused intense excitement,|The case was tried at Guildford Assizes, and caused intense excitement, +LJ018-0243|the hardship to the holders of these lands being plain, should the allegations of invalidity be made good.|the hardship to the holders of these lands being plain, should the allegations of invalidity be made good. +LJ018-0244|The effect of establishing the forgeries would be to restore to the Roupell family lands for which a price had already been paid|The effect of establishing the forgeries would be to restore to the Roupell family lands for which a price had already been paid +LJ018-0245|in all good faith to another, but a criminal member of the family.|in all good faith to another, but a criminal member of the family. +LJ018-0246|At first the case was contested hotly, but, to the profound astonishment of every one inside and outside the court,|At first the case was contested hotly, but, to the profound astonishment of every one inside and outside the court, +LJ018-0247|William Roupell himself was brought as a principal witness to clench the case by a confession altogether against himself.|William Roupell himself was brought as a principal witness to clench the case by a confession altogether against himself. +LJ018-0248|He told his story with perfect coolness and self-possession, but in a grave and serious tone.|He told his story with perfect coolness and self-possession, but in a grave and serious tone. +LJ018-0249|Every word he uttered was said with consideration, and sometimes with a long pause,|Every word he uttered was said with consideration, and sometimes with a long pause, +LJ018-0250|but at the same time with an air of the most entire truthfulness and candor.|but at the same time with an air of the most entire truthfulness and candor. +LJ018-0251|He confessed himself a perjurer in having sworn to the false will, and a wholesale forger, having manufactured no less than ten false signatures|He confessed himself a perjurer in having sworn to the false will, and a wholesale forger, having manufactured no less than ten false signatures +LJ018-0252|to deeds involving on the whole some £350,000.|to deeds involving on the whole some three hundred fifty thousand pounds. +LJ018-0253|For these crimes William Roupell was tried at the Central Criminal Court on the 24th September, 1862.|For these crimes William Roupell was tried at the Central Criminal Court on the twenty-fourth September, eighteen sixty-two. +LJ018-0254|He declined to plead, but a plea of "Not Guilty" was recorded.|He declined to plead, but a plea of "Not Guilty" was recorded. +LJ018-0255|The case was easily and rapidly disposed of.|The case was easily and rapidly disposed of. +LJ018-0256|Roupell made a long statement more in exculpation than in his defense.|Roupell made a long statement more in exculpation than in his defense. +LJ018-0257|He complained that he had at first been the dupe of others, and admitted that he had too readily fallen astray.|He complained that he had at first been the dupe of others, and admitted that he had too readily fallen astray. +LJ018-0258|But while repudiating the charges made against him of systematic extravagance and immorality,|But while repudiating the charges made against him of systematic extravagance and immorality, +LJ018-0259|he confessed that his whole life had been a gigantic mistake, and he was ready to make what atonement he could.|he confessed that his whole life had been a gigantic mistake, and he was ready to make what atonement he could. +LJ018-0260|Mr. Justice Byles, in passing sentence, commented severely upon the commission of such crimes by a man in Roupell's position in life,|Mr. Justice Byles, in passing sentence, commented severely upon the commission of such crimes by a man in Roupell's position in life, +LJ018-0261|and passed the heaviest sentence of the law, transportation for life.|and passed the heaviest sentence of the law, transportation for life. +LJ018-0262|Roupell received the announcement with a cheerful countenance,|Roupell received the announcement with a cheerful countenance, +LJ018-0263|and left the dock with evident satisfaction and relief at the termination of a most painful ordeal.|and left the dock with evident satisfaction and relief at the termination of a most painful ordeal. +LJ018-0264|Roupell was quiet and submissive while in Newgate, unassuming in manner, and ready to make the best of his position.|Roupell was quiet and submissive while in Newgate, unassuming in manner, and ready to make the best of his position. +LJ018-0265|He carried this character with him into penal servitude, and after enduring the full severity of his punishment for several years,|He carried this character with him into penal servitude, and after enduring the full severity of his punishment for several years, +LJ018-0266|was at length advanced to the comparative ease of a post much coveted by convicts, that of hospital nurse.|was at length advanced to the comparative ease of a post much coveted by convicts, that of hospital nurse. +LJ018-0267|His uniform good conduct gained him release from Portland on ticket-of-leave in 1882, just twenty years after his conviction.|His uniform good conduct gained him release from Portland on ticket-of-leave in eighteen eighty-two, just twenty years after his conviction. +LJ018-0268|A daring and cleverly-planned robbery of diamonds was that of the Tarpeys, man and wife,|A daring and cleverly-planned robbery of diamonds was that of the Tarpeys, man and wife, +LJ018-0269|from an assistant of Loudon and Ryder's, the jewelers in Bond Street. The trick was an old one.|from an assistant of Loudon and Ryder's, the jewelers in Bond Street. The trick was an old one. +LJ018-0270|The assistant called with the jewels on approbation at a house specially hired for the purpose in the West End,|The assistant called with the jewels on approbation at a house specially hired for the purpose in the West End, +LJ018-0271|and was rendered insensible by chloroform, after which he was bound and the precious stones stolen.|and was rendered insensible by chloroform, after which he was bound and the precious stones stolen. +LJ018-0272|Mrs. Tarpey was almost immediately captured and put on her trial, but she was acquitted on the plea that she had acted under the coercion of her husband.|Mrs. Tarpey was almost immediately captured and put on her trial, but she was acquitted on the plea that she had acted under the coercion of her husband. +LJ018-0273|Tarpey was caught through his wife,|Tarpey was caught through his wife, +LJ018-0274|who was followed, disguised, and with her hair dyed black, to a house in the Marylebone Road, where she met her husband.|who was followed, disguised, and with her hair dyed black, to a house in the Marylebone Road, where she met her husband. +LJ018-0275|On Tarpey's defense it was stated that the idea of the theft had been suggested to him by a novel, at a time he had lost largely on the turf.|On Tarpey's defense it was stated that the idea of the theft had been suggested to him by a novel, at a time he had lost largely on the turf. +LJ018-0276|The first plot was against Mr. Harry Emmanuel, but he escaped, and the attempt was made upon Loudon and Ryder.|The first plot was against Mr. Harry Emmanuel, but he escaped, and the attempt was made upon Loudon and Ryder. +LJ018-0277|The last great case of fraud upon the Bank of England will fitly close this branch of the criminal records of Newgate.|The last great case of fraud upon the Bank of England will fitly close this branch of the criminal records of Newgate. +LJ018-0278|This was the well and astutely devised plot of the brothers Bidwell,|This was the well and astutely devised plot of the brothers Bidwell, +LJ018-0279|assisted by Macdonell and Noyes, all of them citizens of the United States, by which the bank lost upwards of £100,000.|assisted by Macdonell and Noyes, all of them citizens of the United States, by which the bank lost upwards of one hundred thousand pounds. +LJ018-0281|Their operations were no less worldwide.|Their operations were no less worldwide. +LJ018-0282|In 1871 they crossed the Atlantic,|In eighteen seventy-one they crossed the Atlantic, +LJ018-0283|and by means of forged letters of credit and introduction from London, obtained large sums from continental banks, in Berlin,|and by means of forged letters of credit and introduction from London, obtained large sums from continental banks, in Berlin, +LJ018-0284|Dresden, Bordeaux, Marseilles, and Lyons.|Dresden, Bordeaux, Marseilles, and Lyons. +LJ018-0285|With this as capital they came back to England via Buenos Aires,|With this as capital they came back to England via Buenos Aires, +LJ018-0286|and Austin Bidwell opened a bona fide credit in the Burlington or West End branch of the Bank of England,|and Austin Bidwell opened a bona fide credit in the Burlington or West End branch of the Bank of England, +LJ018-0287|to which he was introduced by a well-known tailor in Saville Row.|to which he was introduced by a well-known tailor in Saville Row. +LJ018-0288|After this the other conspirators traveled to obtain genuine bills and master the system of the leading houses at home and abroad.|After this the other conspirators traveled to obtain genuine bills and master the system of the leading houses at home and abroad. +LJ018-0289|When all was ready, Bidwell first "refreshed his credit" at the Bank of England, as well as disarmed suspicion,|When all was ready, Bidwell first "refreshed his credit" at the Bank of England, as well as disarmed suspicion, +LJ018-0290|by paying in a genuine bill of Messrs. Rothschilds' for £4500, which was duly discounted.|by paying in a genuine bill of Messrs. Rothschilds' for forty-five hundred pounds, which was duly discounted. +LJ018-0291|Then he explained to the bank manager|Then he explained to the bank manager +LJ018-0292|that his transactions at Birmingham would shortly be very large, owing to the development of his business there in the alleged manufacture of Pullman cars.|that his transactions at Birmingham would shortly be very large, owing to the development of his business there in the alleged manufacture of Pullman cars. +LJ018-0293|The ground thus cleared, the forgers poured in from Birmingham numbers of forged acceptances,|The ground thus cleared, the forgers poured in from Birmingham numbers of forged acceptances, +LJ018-0294|all of which were discounted to the value of £102,217.|all of which were discounted to the value of one hundred two thousand, two hundred seventeen pounds. +LJ018-0295|The fraud was rendered possible by the absence of a check usual in the United States.|The fraud was rendered possible by the absence of a check usual in the United States. +LJ018-0296|There such bills would be sent to the drawer to be initialed, and the forgery would have been at once detected.|There such bills would be sent to the drawer to be initialed, and the forgery would have been at once detected. +LJ018-0297|It was the discovery of this flaw in the banking system which had encouraged the Americans to attempt this crime.|It was the discovery of this flaw in the banking system which had encouraged the Americans to attempt this crime. +LJ018-0298|Time was clearly an important factor in the fraud, hence the bills were sent forward in quick succession.|Time was clearly an important factor in the fraud, hence the bills were sent forward in quick succession. +LJ018-0299|Long before they came to maturity the forgers hoped to be well beyond arrest. They had, moreover, sought to destroy all clue.|Long before they came to maturity the forgers hoped to be well beyond arrest. They had, moreover, sought to destroy all clue. +LJ018-0300|The sums obtained by Bidwell in the name of "Warren" at the Bank of England|The sums obtained by Bidwell in the name of "Warren" at the Bank of England +LJ018-0301|were lodged at once by drafts to "Horton," another alias, in the Continental Bank.|were lodged at once by drafts to "Horton," another alias, in the Continental Bank. +LJ018-0302|For these cash was obtained in notes; the notes were exchanged by one of the conspirators for gold at the Bank of England, and again the same day|For these cash was obtained in notes; the notes were exchanged by one of the conspirators for gold at the Bank of England, and again the same day +LJ018-0303|a second conspirator exchanged the gold for notes. But just as all promised well, the frauds were detected through the carelessness of the forgers.|a second conspirator exchanged the gold for notes. But just as all promised well, the frauds were detected through the carelessness of the forgers. +LJ018-0304|They had omitted to insert the dates in certain bills.|They had omitted to insert the dates in certain bills. +LJ018-0305|The bills were sent as a matter of form to the drawer to have the date added, and the forgery was at once detected.|The bills were sent as a matter of form to the drawer to have the date added, and the forgery was at once detected. +LJ018-0306|Noyes was seized without difficulty, as it was a part of the scheme that he should act as the dupe, and remain on the spot in London till all the money was obtained.|Noyes was seized without difficulty, as it was a part of the scheme that he should act as the dupe, and remain on the spot in London till all the money was obtained. +LJ018-0307|Through Noyes the rest of the conspirators were eventually apprehended. Very little if any of the ill-gotten proceeds, however, was ever recovered.|Through Noyes the rest of the conspirators were eventually apprehended. Very little if any of the ill-gotten proceeds, however, was ever recovered. +LJ018-0308|Large sums, as they were realized, were transmitted to the United States, and invested in various American securities,|Large sums, as they were realized, were transmitted to the United States, and invested in various American securities, +LJ018-0309|where probably the money still remains.|where probably the money still remains. +LJ018-0310|The prisoners, who were committed to Newgate for trial,|The prisoners, who were committed to Newgate for trial, +LJ018-0311|had undoubtedly the command of large funds while there, and would have readily disbursed it to effect their enlargement.|had undoubtedly the command of large funds while there, and would have readily disbursed it to effect their enlargement. +LJ018-0312|A plot was soon discovered,|A plot was soon discovered, +LJ018-0313|deep laid, and with many ramifications, by which some of the Newgate warders were to be bribed to allow the prisoners to escape from their cells at night.|deep laid, and with many ramifications, by which some of the Newgate warders were to be bribed to allow the prisoners to escape from their cells at night. +LJ018-0314|Certain friends of the prisoners were watched, and found to be in communication with these warders,|Certain friends of the prisoners were watched, and found to be in communication with these warders, +LJ018-0315|to whom it was said £100 apiece had been given down as the price of their infidelity.|to whom it was said one hundred pounds apiece had been given down as the price of their infidelity. +LJ018-0316|Further sums were to have been paid after the escape;|Further sums were to have been paid after the escape; +LJ018-0317|and one warder admitted that he was to have £1000 more paid to him, and to be provided with a passage to Australia.|and one warder admitted that he was to have one thousand pounds more paid to him, and to be provided with a passage to Australia. +LJ018-0318|The vigilance of the Newgate officials, assisted by the city police, completely frustrated this plot.|The vigilance of the Newgate officials, assisted by the city police, completely frustrated this plot. +LJ018-0319|A second was nevertheless set on foot,|A second was nevertheless set on foot, +LJ018-0320|in which the plan of action was changed, and the freedom of the prisoners was to be obtained by means of a rescue from the dock during the trial.|in which the plan of action was changed, and the freedom of the prisoners was to be obtained by means of a rescue from the dock during the trial. +LJ018-0321|An increase of policemen on duty sufficed to prevent any attempt of this kind.|An increase of policemen on duty sufficed to prevent any attempt of this kind. +LJ018-0322|Nor were these two abortive efforts all that were planned.|Nor were these two abortive efforts all that were planned. +LJ018-0323|A year or two after, when the prisoners were undergoing their life sentences of penal servitude,|A year or two after, when the prisoners were undergoing their life sentences of penal servitude, +LJ018-0324|much uneasiness was caused at one of the convict prisons by information that bribery on a large scale was again at work amongst the officials.|much uneasiness was caused at one of the convict prisons by information that bribery on a large scale was again at work amongst the officials. +LJ018-0325|But extra precautions and close supervision have so far proved effectual, and the prisoners are still in custody after a lapse of ten years.|But extra precautions and close supervision have so far proved effectual, and the prisoners are still in custody after a lapse of ten years. +LJ018-0326|I propose to end at this point the detailed account of the more prominent criminal cases which lodged their perpetrators in Newgate.|I propose to end at this point the detailed account of the more prominent criminal cases which lodged their perpetrators in Newgate. +LJ018-0327|The most recent affairs are still too fresh in the public mind to need more than a passing reference.|The most recent affairs are still too fresh in the public mind to need more than a passing reference. +LJ018-0328|Few of the Newgate notorieties of late years show any marked peculiarities;|Few of the Newgate notorieties of late years show any marked peculiarities; +LJ018-0329|their crimes follow in the lines of others already found, and often more than once, in the calendars.|their crimes follow in the lines of others already found, and often more than once, in the calendars. +LJ018-0330|Violent passions too easily aroused prompted the Frenchwoman Marguerite Dixblanc|Violent passions too easily aroused prompted the Frenchwoman Marguerite Dixblanc +LJ018-0331|to murder her mistress, Madame Riel, in Park Lane, as Courvoisier, the Swiss, had been tempted to murder Lord William Russell.|to murder her mistress, Madame Riel, in Park Lane, as Courvoisier, the Swiss, had been tempted to murder Lord William Russell. +LJ018-0332|Greed in the latter case was a secondary motive;|Greed in the latter case was a secondary motive; +LJ018-0333|it was the principal incentive with Kate Webster, that fierce and brutal female savage who took the life of her mistress at Richmond.|it was the principal incentive with Kate Webster, that fierce and brutal female savage who took the life of her mistress at Richmond. +LJ018-0334|Webster, it may be mentioned here, was one of the worst prisoners ever remembered in Newgate|Webster, it may be mentioned here, was one of the worst prisoners ever remembered in Newgate +LJ018-0335|most violent in temper, and addicted to the most frightful language.|most violent in temper, and addicted to the most frightful language. +LJ018-0336|Webster's devices for disposing of the body of her victim will call to mind those of Theodore Gardelle,|Webster's devices for disposing of the body of her victim will call to mind those of Theodore Gardelle, +LJ018-0337|of Good, and Greenacre, and Catherine Hayes.|of Good, and Greenacre, and Catherine Hayes. +LJ018-0338|Greed in another form led the Stauntons to make away with Mrs. Patrick Staunton, murdering her with devilish cruelty by slow degrees.|Greed in another form led the Stauntons to make away with Mrs. Patrick Staunton, murdering her with devilish cruelty by slow degrees. +LJ018-0339|The judge, Sir Henry Hawkins, in passing sentence|The judge, Sir Henry Hawkins, in passing sentence +LJ018-0340|characterized this as a crime more black and hideous than any in the criminal annals of the country.|characterized this as a crime more black and hideous than any in the criminal annals of the country. +LJ018-0341|But it was scarcely worse than that of Mrs. Brownrigg, or that of the Meteyards, both of whom did their helpless apprentices to death.|But it was scarcely worse than that of Mrs. Brownrigg, or that of the Meteyards, both of whom did their helpless apprentices to death. +LJ018-0342|It was to effect the rupture of an irksome tie that led Henry Wainwright to murder Harriet Lane deliberately and in cold blood.|It was to effect the rupture of an irksome tie that led Henry Wainwright to murder Harriet Lane deliberately and in cold blood. +LJ018-0343|In this case the tie was unsanctified, but it was not more inconvenient than that which urged Greenacre to a similar crime.|In this case the tie was unsanctified, but it was not more inconvenient than that which urged Greenacre to a similar crime. +LJ018-0344|In cold-blooded premeditation it rivaled that of the Mannings.|In cold-blooded premeditation it rivaled that of the Mannings. +LJ018-0345|As in that case, the grave had been dug long in anticipation, and the chloride of lime purchased to destroy the corpse.|As in that case, the grave had been dug long in anticipation, and the chloride of lime purchased to destroy the corpse. +LJ018-0346|Henry Wainwright's attempt to get rid of the body was ingenious, but not original,|Henry Wainwright's attempt to get rid of the body was ingenious, but not original, +LJ018-0347|and the circumstances which led to detection were scarcely novel proofs of the old adage that murder will out.|and the circumstances which led to detection were scarcely novel proofs of the old adage that murder will out. +LJ018-0348|Henry Wainwright's impassioned denial of his crime, even after it had been brought fully home to him, has many parallels in the criminal records.|Henry Wainwright's impassioned denial of his crime, even after it had been brought fully home to him, has many parallels in the criminal records. +LJ018-0349|His disclaimer, distinct and detailed on every point, was intended simply for effect.|His disclaimer, distinct and detailed on every point, was intended simply for effect. +LJ018-0350|He might swear he was not the murderer, that he never fired a pistol in his life,|He might swear he was not the murderer, that he never fired a pistol in his life, +LJ018-0351|and that, in spite of the verdict of the jury, "he left the dock with a calm and quiet conscience;"|and that, in spite of the verdict of the jury, "he left the dock with a calm and quiet conscience;" +LJ018-0352|but there was no doubt of his guilt, as the Lord Chief Justice told him, while expressing great regret at his rash assertion.|but there was no doubt of his guilt, as the Lord Chief Justice told him, while expressing great regret at his rash assertion. +LJ018-0353|Wainwright's demeanor after sentence has been described in the last chapter.|Wainwright's demeanor after sentence has been described in the last chapter. +LJ018-0354|Doubts were long entertained whether Thomas Wainwright,|Doubts were long entertained whether Thomas Wainwright, +LJ018-0355|who was convicted as an accessory after the fact, had not really taken an active part in the murder.|who was convicted as an accessory after the fact, had not really taken an active part in the murder. +LJ018-0356|But a conversation overheard between the two brothers in Newgate satisfactorily exonerated Thomas Wainwright.|But a conversation overheard between the two brothers in Newgate satisfactorily exonerated Thomas Wainwright. +LJ018-0357|Poisoning has still its victims.|Poisoning has still its victims. +LJ018-0358|Christina Edmunds had resort to strychnia, the same lethal drug that Palmer used;|Christina Edmunds had resort to strychnia, the same lethal drug that Palmer used; +LJ018-0359|her object being first to dispose of the wife of a man for whom she had conceived a guilty passion,|her object being first to dispose of the wife of a man for whom she had conceived a guilty passion, +LJ018-0360|then to divert suspicion from herself by throwing it on a confectioner, whose sweetmeats she bought,|then to divert suspicion from herself by throwing it on a confectioner, whose sweetmeats she bought, +LJ018-0361|tampered with, and returned to the shop.|tampered with, and returned to the shop. +LJ018-0362|The trial of Miss Edmunds was transferred to the Central Criminal Court under Lord Campbell's Act, already referred to.|The trial of Miss Edmunds was transferred to the Central Criminal Court under Lord Campbell's Act, already referred to. +LJ018-0363|She was found guilty.|She was found guilty. +LJ018-0364|It will be remembered that she made a statement which led to the empaneling of a jury of matrons, who decided that there was no cause for an arrest of judgment.|It will be remembered that she made a statement which led to the empaneling of a jury of matrons, who decided that there was no cause for an arrest of judgment. +LJ018-0365|Kate Webster followed the same course; but these pleas of pregnancy are not common now-a-days.|Kate Webster followed the same course; but these pleas of pregnancy are not common now-a-days. +LJ018-0366|Although sentence of death was passed on Edmunds, it was commuted to penal servitude for life;|Although sentence of death was passed on Edmunds, it was commuted to penal servitude for life; +LJ018-0367|but she eventually passed into Broadmoor Lunatic Asylum, where she busies herself with watercolor drawing.|but she eventually passed into Broadmoor Lunatic Asylum, where she busies herself with watercolor drawing. +LJ018-0368|The still more recent cases of poisoning which have occurred were not connected with Newgate.|The still more recent cases of poisoning which have occurred were not connected with Newgate. +LJ018-0369|The mysterious Bravo case, that of Dr. Lamson, and that of Kate Dover|The mysterious Bravo case, that of Dr. Lamson, and that of Kate Dover +LJ018-0370|unhappily show that society is more than ever at the mercy of the insidious and unscrupulous administration of poisonous drugs.|unhappily show that society is more than ever at the mercy of the insidious and unscrupulous administration of poisonous drugs. +LJ018-0371|A case reproducing many of the features of the 'Flowery Land' occurred twelve years later, when the crew of the 'Lennie'|A case reproducing many of the features of the 'Flowery Land' occurred twelve years later, when the crew of the 'Lennie' +LJ018-0372|mutinied, murdered the captain and mates, sparing the steward only on condition that he would navigate the ship to the Mediterranean.|mutinied, murdered the captain and mates, sparing the steward only on condition that he would navigate the ship to the Mediterranean. +LJ018-0373|The mutineers were of the same stamp as the crew of the 'Flowery Land'|The mutineers were of the same stamp as the crew of the 'Flowery Land' +LJ018-0374|foreigners, vindictive, reckless, and truculent ruffians, easily moved to murderous rage.|foreigners, vindictive, reckless, and truculent ruffians, easily moved to murderous rage. +LJ018-0375|The 'Lennie's' men were all Greeks, except one known as French Peter,|The 'Lennie's' men were all Greeks, except one known as French Peter, +LJ018-0376|who was the ringleader, and who had long been an habitual criminal, a reputed murderer, and certainly an inmate more than once of a French bagne.|who was the ringleader, and who had long been an habitual criminal, a reputed murderer, and certainly an inmate more than once of a French bagne. +LJ018-0377|Conviction was obtained through the evidence of the steward and two of the least culpable of the crew.|Conviction was obtained through the evidence of the steward and two of the least culpable of the crew. +LJ018-0378|In Newgate the 'Lennie' mutineers were extremely well behaved.|In Newgate the 'Lennie' mutineers were extremely well behaved. +LJ018-0379|Resolute, determined-looking men, their courage broke down in confinement.|Resolute, determined-looking men, their courage broke down in confinement. +LJ018-0380|They paid close attention to the counsels of the archimandrite, and died quite penitent. A story is told of one of them, "Big Harry,"|They paid close attention to the counsels of the archimandrite, and died quite penitent. A story is told of one of them, "Big Harry," +LJ018-0381|the wildest and most cut-throat looking of the lot, which proves that he could be grateful for kindness, and was not all bad.|the wildest and most cut-throat looking of the lot, which proves that he could be grateful for kindness, and was not all bad. +LJ018-0382|He had steadfastly refused to eat meat on some religions scruples, and for the same reason would not touch soup.|He had steadfastly refused to eat meat on some religions scruples, and for the same reason would not touch soup. +LJ018-0383|He was glad, therefore, to get an extra allowance of bread,|He was glad, therefore, to get an extra allowance of bread, +LJ018-0384|and to show his gratitude to the warder who procured this privilege for him, he made him a present.|and to show his gratitude to the warder who procured this privilege for him, he made him a present. +LJ018-0385|It was his own handiwork -- a bird pecking at a flower;|It was his own handiwork -- a bird pecking at a flower; +LJ018-0386|the whole manufactured while in the condemned cell of the crumb of bread made into paste.|the whole manufactured while in the condemned cell of the crumb of bread made into paste. +LJ018-0387|The flower had berries also of bread fixed on stems made from the fiber drawn from the stuffing of his mattress,|The flower had berries also of bread fixed on stems made from the fiber drawn from the stuffing of his mattress, +LJ018-0388|and the bird's legs were a couple of teeth broken off the prisoner's comb.|and the bird's legs were a couple of teeth broken off the prisoner's comb. +LJ018-0389|Of the lesser criminals, forgers, thieves, swindlers, Newgate continued to receive its full share up to the last.|Of the lesser criminals, forgers, thieves, swindlers, Newgate continued to receive its full share up to the last. +LJ018-0390|But there were few cases so remarkable as the great ones already recorded.|But there were few cases so remarkable as the great ones already recorded. +LJ018-0391|Mr. Bamell Oakley made a rich harvest for a time, and was said at the time of his trial|Mr. Bamell Oakley made a rich harvest for a time, and was said at the time of his trial +LJ018-0392|to have obtained as much as £40,000 by false and fraudulent pretenses.|to have obtained as much as forty thousand pounds by false and fraudulent pretenses. +LJ018-0393|Messrs. Swindlehurst, Saffery, and Langley cleared a large profit|Messrs. Swindlehurst, Saffery, and Langley cleared a large profit +LJ018-0394|by swindling the Artisans' Dwellings Company; and Madame Rachel passed through Newgate on her way to Millbank|by swindling the Artisans' Dwellings Company; and Madame Rachel passed through Newgate on her way to Millbank +LJ018-0395|convicted of obtaining jewelery under the false pretense of making silly women "beautiful for ever."|convicted of obtaining jewelery under the false pretense of making silly women "beautiful for ever." +LJ018-0396|The greatest causes célèbre, however, of recent times were the turf frauds by which the Comtesse de Goncourt was swindled|The greatest causes célèbre, however, of recent times were the turf frauds by which the Comtesse de Goncourt was swindled +LJ018-0397|out of large sums in sham sporting speculations. The conviction of the principals in this nefarious transaction,|out of large sums in sham sporting speculations. The conviction of the principals in this nefarious transaction, +LJ018-0398|Benson, the two Kurrs, Bale, and Murray, led to strange revelations of dishonest practices amongst the detective police,|Benson, the two Kurrs, Bale, and Murray, led to strange revelations of dishonest practices amongst the detective police, +LJ019-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 22. Newgate Reformed.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section twenty-two. Newgate Reformed. +LJ019-0002|The time at length approached when a radical and complete change was to come over the old city jail.|The time at length approached when a radical and complete change was to come over the old city jail. +LJ019-0003|It was impossible for Newgate to escape for ever the influences pressing so strongly towards prison reform.|It was impossible for Newgate to escape for ever the influences pressing so strongly towards prison reform. +LJ019-0004|Elsewhere the spirit had been more or less active, although not uniformly or always to the same extent.|Elsewhere the spirit had been more or less active, although not uniformly or always to the same extent. +LJ019-0005|There had been a pause in legislation, except of a permissive kind. The 2nd and 3rd Victoria, cap. 56|There had been a pause in legislation, except of a permissive kind. The second and third Victoria, cap. fifty-six +LJ019-0006|laid it down that individuals might be confined separately and apart in single cells.|laid it down that individuals might be confined separately and apart in single cells. +LJ019-0007|By other acts local authorities were empowered to construct new jails or hire accommodation in the district;|By other acts local authorities were empowered to construct new jails or hire accommodation in the district; +LJ019-0008|but no steps had been taken in Parliament to enforce a better system of discipline,|but no steps had been taken in Parliament to enforce a better system of discipline, +LJ019-0009|or to insist upon the construction of prisons on the most approved plan.|or to insist upon the construction of prisons on the most approved plan. +LJ019-0010|As regards the first, however, Sir James Graham, when Home Secretary in 1843,|As regards the first, however, Sir James Graham, when Home Secretary in eighteen forty-three, +LJ019-0011|had appointed a committee of prison inspectors, presided over by the Under Secretary of State, to draw up rules and dietaries,|had appointed a committee of prison inspectors, presided over by the Under Secretary of State, to draw up rules and dietaries, +LJ019-0012|which were then recommended to and generally adopted by the visiting justices all over the kingdom.|which were then recommended to and generally adopted by the visiting justices all over the kingdom. +LJ019-0013|As regards the second, the Government had set a good example, and in deciding upon the erection of Pentonville prison|As regards the second, the Government had set a good example, and in deciding upon the erection of Pentonville prison +LJ019-0014|had embarked on a considerable expenditure in order to provide a model prison for general imitation.|had embarked on a considerable expenditure in order to provide a model prison for general imitation. +LJ019-0015|The first stone of Pentonville prison was laid on the 10th April, 1840, by the Marquis of Normanby,|The first stone of Pentonville prison was laid on the tenth April, eighteen forty, by the Marquis of Normanby, +LJ019-0016|then Home Secretary, and the prison, which contained five hundred and twenty cells, was occupied on the 21st December, 1842.|then Home Secretary, and the prison, which contained five hundred and twenty cells, was occupied on the twenty-first December, eighteen forty-two. +LJ019-0017|This building was a costly affair. The site was uneven, and had to be leveled;|This building was a costly affair. The site was uneven, and had to be leveled; +LJ019-0018|moreover, the gross expenditure was increased "partly from its being considered necessary, as it was a national prison,|moreover, the gross expenditure was increased "partly from its being considered necessary, as it was a national prison, +LJ019-0019|to make a great archway, and to make the character of it more imposing than if it had been situated in the country, and had been an ordinary prison.|to make a great archway, and to make the character of it more imposing than if it had been situated in the country, and had been an ordinary prison. +LJ019-0020|Up to the 21st December, 1842,|Up to the twenty-first December, eighteen forty-two, +LJ019-0021|with the additions made to that date, the total expenditure amounted to nearly £90,000, or about £180 per cell.|with the additions made to that date, the total expenditure amounted to nearly ninety thousand pounds, or about one hundred eighty pounds per cell. +LJ019-0022|On the other hand, it must be admitted|On the other hand, it must be admitted +LJ019-0023|that this was an experimental construction, and that too strict a limitation of outlay would have militated seriously against the usefulness of the building.|that this was an experimental construction, and that too strict a limitation of outlay would have militated seriously against the usefulness of the building. +LJ019-0024|Nor must it be overlooked that this, the first model prison, although obtained at a considerable cost, became actually what its name implied.|Nor must it be overlooked that this, the first model prison, although obtained at a considerable cost, became actually what its name implied. +LJ019-0025|Pentonville has really been the model on which all subsequent prison construction has been based. All prisons at home and abroad are but variations,|Pentonville has really been the model on which all subsequent prison construction has been based. All prisons at home and abroad are but variations, +LJ019-0026|of course with the added improvements following longer experience, of the pattern originated by the architectural genius of Sir Joshua Jebb.|of course with the added improvements following longer experience, of the pattern originated by the architectural genius of Sir Joshua Jebb. +LJ019-0027|The internal arrangements of the new model were carefully supervised by a body of distinguished men, among which were many peers, Lord John Russell,|The internal arrangements of the new model were carefully supervised by a body of distinguished men, among which were many peers, Lord John Russell, +LJ019-0028|Mr. Shaw-Lefevre, the Speaker of the House of Commons, Sir Benjamin Brodie,|Mr. Shaw-Lefevre, the Speaker of the House of Commons, Sir Benjamin Brodie, +LJ019-0029|Major Jebb, R.E., and the two prison inspectors, Messrs. Crawford and Russell, with whose names the reader is already familiar.|Major Jebb, R.E., and the two prison inspectors, Messrs. Crawford and Russell, with whose names the reader is already familiar. +LJ019-0030|Major, afterwards Sir Joshua Jebb,|Major, afterwards Sir Joshua Jebb, +LJ019-0031|was the moving spirit among these commissioners, and he is now generally recognized as the originator of modern prison architecture.|was the moving spirit among these commissioners, and he is now generally recognized as the originator of modern prison architecture. +LJ019-0032|The movement thus laudably initiated by the Government soon spread to the provinces.|The movement thus laudably initiated by the Government soon spread to the provinces. +LJ019-0033|Some jurisdictions, greatly to their credit, strove at once to follow the lead of the central authority.|Some jurisdictions, greatly to their credit, strove at once to follow the lead of the central authority. +LJ019-0034|Within half-a-dozen years no less than fifty-four new prisons were built on the Pentonville plan, others were in progress,|Within half-a-dozen years no less than fifty-four new prisons were built on the Pentonville plan, others were in progress, +LJ019-0035|and the total number of separate cells provided amounted to eleven thousand odd.|and the total number of separate cells provided amounted to eleven thousand odd. +LJ019-0036|This list included Wakefield, Leeds, Kirkdale, Manchester, Birmingham, and Dublin.|This list included Wakefield, Leeds, Kirkdale, Manchester, Birmingham, and Dublin. +LJ019-0037|Liverpool was building a new prison with a thousand cells, the county of Surrey one with seven hundred.|Liverpool was building a new prison with a thousand cells, the county of Surrey one with seven hundred. +LJ019-0038|The cost in each varied considerably, the general average being from £120 to £130 per cell.|The cost in each varied considerably, the general average being from one hundred twenty pounds to one hundred thirty pounds per cell. +LJ019-0039|At Pentonville the rate was higher, but there the expense had been increased by the site,|At Pentonville the rate was higher, but there the expense had been increased by the site, +LJ019-0040|the difficulty of access, and the admitted necessity of giving architectural importance to this the national model prison.|the difficulty of access, and the admitted necessity of giving architectural importance to this the national model prison. +LJ019-0041|Other jurisdictions were less prompt to recognize their responsibilities, the city of London among the number, as I shall presently show at length.|Other jurisdictions were less prompt to recognize their responsibilities, the city of London among the number, as I shall presently show at length. +LJ019-0042|These were either satisfied with a makeshift, and modified existing buildings, without close regard to their suitability, or for a long time did nothing at all.|These were either satisfied with a makeshift, and modified existing buildings, without close regard to their suitability, or for a long time did nothing at all. +LJ019-0043|Among the latter were notably the counties of Cheshire, Lincolnshire, Norfolk, Suffolk, Nottinghamshire, the East and North Ridings of Yorkshire.|Among the latter were notably the counties of Cheshire, Lincolnshire, Norfolk, Suffolk, Nottinghamshire, the East and North Ridings of Yorkshire. +LJ019-0044|The south and west of England were also very laggard, and many years were still to elapse before the prisons in these parts were properly reconstituted.|The south and west of England were also very laggard, and many years were still to elapse before the prisons in these parts were properly reconstituted. +LJ019-0045|Not less remarkable than this diverse interpretation of a manifest duty|Not less remarkable than this diverse interpretation of a manifest duty +LJ019-0046|was the variety of views as regards the discipline to be introduced in these new prisons. The time was one|was the variety of views as regards the discipline to be introduced in these new prisons. The time was one +LJ019-0047|when thoughtful people who concerned themselves closely with social questions were greatly exercised as to the best system of treating the inmates of a jail.|when thoughtful people who concerned themselves closely with social questions were greatly exercised as to the best system of treating the inmates of a jail. +LJ019-0048|A new and still imperfectly understood science had arisen,|A new and still imperfectly understood science had arisen, +LJ019-0049|the principles of which were debated by disputants of widely opposite opinions with an earnestness that sometimes bordered upon acrimony.|the principles of which were debated by disputants of widely opposite opinions with an earnestness that sometimes bordered upon acrimony. +LJ019-0050|One school were strongly in favor of the continuous separation of prisoners,|One school were strongly in favor of the continuous separation of prisoners, +LJ019-0051|the other supported the theory of labor in association, but under a stringent rule of silence, with isolation only at night.|the other supported the theory of labor in association, but under a stringent rule of silence, with isolation only at night. +LJ019-0052|Both systems came to us from the United States. The difference was really more in degree than in principle,|Both systems came to us from the United States. The difference was really more in degree than in principle, +LJ019-0053|and our modern practice has prudently tried to steer between the two extremes, accepting as the best system a judicious combination of both.|and our modern practice has prudently tried to steer between the two extremes, accepting as the best system a judicious combination of both. +LJ019-0054|But about 1850 the two sides were distinctly hostile, and the controversy ran high.|But about eighteen fifty the two sides were distinctly hostile, and the controversy ran high. +LJ019-0055|High authorities were in favor of continuous separation.|High authorities were in favor of continuous separation. +LJ019-0056|Colonel Jebb preferred it; Messrs. Crawford and Whitworth Russell were convinced that the complete isolation of criminals from one another|Colonel Jebb preferred it; Messrs. Crawford and Whitworth Russell were convinced that the complete isolation of criminals from one another +LJ019-0057|was the true basis of a sound system of prison discipline.|was the true basis of a sound system of prison discipline. +LJ019-0058|Prison chaplains of experience and high repute, such as Messrs. Field, Clay, Kingsmill, Burt, and Osborne, also advocated it.|Prison chaplains of experience and high repute, such as Messrs. Field, Clay, Kingsmill, Burt, and Osborne, also advocated it. +LJ019-0059|It was claimed for it that it was more deterrent;|It was claimed for it that it was more deterrent; +LJ019-0060|that in districts where it was the rule, evil-doers especially dreaded coming under its irksome conditions.|that in districts where it was the rule, evil-doers especially dreaded coming under its irksome conditions. +LJ019-0061|Another argument was, that it afforded more hope of the reformation of criminals.|Another argument was, that it afforded more hope of the reformation of criminals. +LJ019-0062|The system of associated labor in silence had also its warm supporters,|The system of associated labor in silence had also its warm supporters, +LJ019-0063|who maintained that under this system prisoners were more industrious and more healthy|who maintained that under this system prisoners were more industrious and more healthy +LJ019-0064|that their condition was more natural, and approximated more nearly to that of daily life.|that their condition was more natural, and approximated more nearly to that of daily life. +LJ019-0065|Better industrial results were obtained from it, and instruction in trades was easier, and prisoners were more likely to leave jail|Better industrial results were obtained from it, and instruction in trades was easier, and prisoners were more likely to leave jail +LJ019-0066|with the means of earning an honest livelihood if so disposed.|with the means of earning an honest livelihood if so disposed. +LJ019-0067|The opposing champions were not slow to find faults and flaws in the system they condemned.|The opposing champions were not slow to find faults and flaws in the system they condemned. +LJ019-0068|Separation was injurious to health, mental or physical, said one side; men broke down when subjected to it for more than a certain period,|Separation was injurious to health, mental or physical, said one side; men broke down when subjected to it for more than a certain period, +LJ019-0069|and it was unsafe to fix this limit above twelve months, although some rash advocates were in favor of eighteen months, some indeed of two years.|and it was unsafe to fix this limit above twelve months, although some rash advocates were in favor of eighteen months, some indeed of two years. +LJ019-0070|The other side retorted that the system of associated labor was most costly, so many officers being required to maintain the discipline of silence;|The other side retorted that the system of associated labor was most costly, so many officers being required to maintain the discipline of silence; +LJ019-0071|moreover, it was nearly impossible to prevent communication and mutual contamination.|moreover, it was nearly impossible to prevent communication and mutual contamination. +LJ019-0072|It is scarcely necessary to follow the controversy further. I have only introduced the subject as showing how little as yet the State|It is scarcely necessary to follow the controversy further. I have only introduced the subject as showing how little as yet the State +LJ019-0073|was impressed with the necessity for authoritative interference.|was impressed with the necessity for authoritative interference. +LJ019-0074|The legislature was content to let local jurisdictions experimentalize for themselves; with the strange, anomalous result,|The legislature was content to let local jurisdictions experimentalize for themselves; with the strange, anomalous result, +LJ019-0075|that a thief or other criminal might be quite differently treated according as he was incarcerated on one side or another of a border line.|that a thief or other criminal might be quite differently treated according as he was incarcerated on one side or another of a border line. +LJ019-0076|This variety was often extended to all branches of prison economy.|This variety was often extended to all branches of prison economy. +LJ019-0077|There was an absolute want of uniformity in dietaries; in some prisons it was too liberal, in others too low.|There was an absolute want of uniformity in dietaries; in some prisons it was too liberal, in others too low. +LJ019-0078|The amount of exercise varied from one or two hours daily to half the working day.|The amount of exercise varied from one or two hours daily to half the working day. +LJ019-0079|The cells inhabited by prisoners were of very varying dimensions;|The cells inhabited by prisoners were of very varying dimensions; +LJ019-0080|some were not sufficiently ventilated, others were warmed artificially, and were unwholesomely close.|some were not sufficiently ventilated, others were warmed artificially, and were unwholesomely close. +LJ019-0081|The use of gas or some other means of lighting might be adopted, but more often was dispensed with.|The use of gas or some other means of lighting might be adopted, but more often was dispensed with. +LJ019-0082|In a great number of prisons no provision was made for the education of prisoners, in some others there was a sufficient staff of schoolmasters and instructors.|In a great number of prisons no provision was made for the education of prisoners, in some others there was a sufficient staff of schoolmasters and instructors. +LJ019-0083|The discipline also varied greatly, from the severely penal to the culpably lax.|The discipline also varied greatly, from the severely penal to the culpably lax. +LJ019-0084|The greatest pains might be taken to secure isolation,|The greatest pains might be taken to secure isolation, +LJ019-0085|the prisoners might be supervised and watched at every step, and made liable to punishment for a trifling breach of an irksome code of regulations,|the prisoners might be supervised and watched at every step, and made liable to punishment for a trifling breach of an irksome code of regulations, +LJ019-0086|or they might herd together or communicate freely as in the old worst days. They might see each other when they liked, and converse sotto voce,|or they might herd together or communicate freely as in the old worst days. They might see each other when they liked, and converse sotto voce, +LJ019-0087|or make signs; or the chances of recognizing or being recognized were reduced to a minimum by the use of a mask.|or make signs; or the chances of recognizing or being recognized were reduced to a minimum by the use of a mask. +LJ019-0088|There was no general rule of employment. Hard labor was often not insisted upon in separate confinement;|There was no general rule of employment. Hard labor was often not insisted upon in separate confinement; +LJ019-0089|sometimes it embraced the tread-wheel or the newly-invented instruments known as cranks, which ground air.|sometimes it embraced the tread-wheel or the newly-invented instruments known as cranks, which ground air. +LJ019-0090|The alternative between labor or idleness, or the selection of the form of labor,|The alternative between labor or idleness, or the selection of the form of labor, +LJ019-0091|were mere matters of chance, and decided according to the views of the local magistracy.|were mere matters of chance, and decided according to the views of the local magistracy. +LJ019-0092|They were approved of and employed at some prisons, at others objected to because they were unproductive,|They were approved of and employed at some prisons, at others objected to because they were unproductive, +LJ019-0093|and because the machine was often so imperfect that the amount of effort could not be exactly regulated.|and because the machine was often so imperfect that the amount of effort could not be exactly regulated. +LJ019-0094|Opinions differed greatly with regard to the tread-wheel; some authorities advocated it as a very severe and irksome punishment,|Opinions differed greatly with regard to the tread-wheel; some authorities advocated it as a very severe and irksome punishment, +LJ019-0095|which was yet under full control, and might be made to work corn-mills or prove otherwise productive;|which was yet under full control, and might be made to work corn-mills or prove otherwise productive; +LJ019-0096|other authorities as strongly condemned it as brutalizing, unequal in its operation, and altogether a "deplorable invention."|other authorities as strongly condemned it as brutalizing, unequal in its operation, and altogether a "deplorable invention." +LJ019-0097|This want of uniformity in prison discipline became ere long an acknowledged evil pressing for some remedy,|This want of uniformity in prison discipline became ere long an acknowledged evil pressing for some remedy, +LJ019-0098|and the question was once more taken up in the House of Commons. In 1849 Mr. Charles Pearson, M.P.,|and the question was once more taken up in the House of Commons. In eighteen forty-nine Mr. Charles Pearson, M.P., +LJ019-0099|moved for a committee to report upon the best means of securing some uniform system which should be "punitive, reformatory, and self-supporting;"|moved for a committee to report upon the best means of securing some uniform system which should be "punitive, reformatory, and self-supporting;" +LJ019-0100|but the session was far advanced, and the matter was relegated to the following year.|but the session was far advanced, and the matter was relegated to the following year. +LJ019-0101|In 1850 Sir George Grey brought forward a new motion to the same effect,|In eighteen fifty Sir George Grey brought forward a new motion to the same effect, +LJ019-0102|which was promptly carried, with the additional instruction to the committee to suggest any improvements.|which was promptly carried, with the additional instruction to the committee to suggest any improvements. +LJ019-0103|The latter had reference more especially to a proposal emanating from Mr. Charles Pearson himself.|The latter had reference more especially to a proposal emanating from Mr. Charles Pearson himself. +LJ019-0104|That gentleman had come to the conclusion that the ordinary and hackneyed methods of treatment were practically inefficacious,|That gentleman had come to the conclusion that the ordinary and hackneyed methods of treatment were practically inefficacious, +LJ019-0105|and that a new system of prison discipline should be introduced.|and that a new system of prison discipline should be introduced. +LJ019-0106|His plan was to devote the whole labor of prisoners sentenced to any term between three months and four years to agriculture.|His plan was to devote the whole labor of prisoners sentenced to any term between three months and four years to agriculture. +LJ019-0107|District prisons were to be established for this purpose, each of which would be in the heart of a farm of a thousand acres.|District prisons were to be established for this purpose, each of which would be in the heart of a farm of a thousand acres. +LJ019-0108|The prisoners were to cultivate the land and raise sufficient produce for their own support.|The prisoners were to cultivate the land and raise sufficient produce for their own support. +LJ019-0109|Mr. Pearson backed up his recommendations by many sound arguments.|Mr. Pearson backed up his recommendations by many sound arguments. +LJ019-0110|Field labor, he urged, and with reason, was a very suitable employment;|Field labor, he urged, and with reason, was a very suitable employment; +LJ019-0111|healthful, easily learnt, and well adapted to the circumstances of unskilled laborers.|healthful, easily learnt, and well adapted to the circumstances of unskilled laborers. +LJ019-0112|Such excellent returns might be counted upon, that a margin of profit would be left after the cost of the prisons had been defrayed.|Such excellent returns might be counted upon, that a margin of profit would be left after the cost of the prisons had been defrayed. +LJ019-0113|The scheme was no doubt fascinating, and in many respects feasible;|The scheme was no doubt fascinating, and in many respects feasible; +LJ019-0114|but Mr. Pearson overlooked some points in which a more practical mind would have foreseen difficulty, and perhaps forecasted failure.|but Mr. Pearson overlooked some points in which a more practical mind would have foreseen difficulty, and perhaps forecasted failure. +LJ019-0115|In his proposal he dwelt much upon the humanizing effects of healthful open-air toil,|In his proposal he dwelt much upon the humanizing effects of healthful open-air toil, +LJ019-0116|anticipating the best results from a system which made earnings, and indeed release, dependent upon the amount of work done.|anticipating the best results from a system which made earnings, and indeed release, dependent upon the amount of work done. +LJ019-0117|That industry might thus be stimulated and encouraged was probable enough,|That industry might thus be stimulated and encouraged was probable enough, +LJ019-0118|and later experience has fully proved the advantage of a judicious system of gratuities for labor;|and later experience has fully proved the advantage of a judicious system of gratuities for labor; +LJ019-0119|but Mr. Pearson hardly considered the converse sufficiently, and|but Mr. Pearson hardly considered the converse sufficiently, and +LJ019-0120|omitted the fact that he might have to deal with that persistent idleness which is not an unknown characteristic of the criminal class.|omitted the fact that he might have to deal with that persistent idleness which is not an unknown characteristic of the criminal class. +LJ019-0121|The hope of reward might do much, but no system of penal discipline is complete unless it can also count upon the fear of punishment.|The hope of reward might do much, but no system of penal discipline is complete unless it can also count upon the fear of punishment. +LJ019-0122|Mr. Pearson seems to have taken for granted that all prisoners would behave well in his district prisons.|Mr. Pearson seems to have taken for granted that all prisoners would behave well in his district prisons. +LJ019-0123|On that account he made no provision to insure safe custody,|On that account he made no provision to insure safe custody, +LJ019-0124|thinking perhaps that prisoners so well disposed would cheerfully remain in jail of their own accord.|thinking perhaps that prisoners so well disposed would cheerfully remain in jail of their own accord. +LJ019-0125|But an open farm of a thousand acres would have offered abundant chances of escape, which some at least would have attempted, probably with success.|But an open farm of a thousand acres would have offered abundant chances of escape, which some at least would have attempted, probably with success. +LJ019-0126|The creation of an expensive staff for supervision,|The creation of an expensive staff for supervision, +LJ019-0127|or the still more costly process of walling in the whole farm, would have greatly added to the charges of these establishments.|or the still more costly process of walling in the whole farm, would have greatly added to the charges of these establishments. +LJ019-0128|I have lingered too long perhaps over Mr. Pearson's proposal, but some reference was indispensable to a scheme|I have lingered too long perhaps over Mr. Pearson's proposal, but some reference was indispensable to a scheme +LJ019-0129|which marked the growth of public interest in prison affairs, and which was the germ of the new system|which marked the growth of public interest in prison affairs, and which was the germ of the new system +LJ019-0130|since admirably developed in the convict prisons of this country.|since admirably developed in the convict prisons of this country. +LJ019-0131|Mr. Pearson and the committee of 1850 have the more claim on our consideration, because,|Mr. Pearson and the committee of eighteen fifty have the more claim on our consideration, because, +LJ019-0132|in the inquiry which followed, attention was again attracted to Newgate.|in the inquiry which followed, attention was again attracted to Newgate. +LJ019-0133|The condition of that prison in 1850 may be gathered from the pages of the report. Not much had been done to remedy the old defects;|The condition of that prison in eighteen fifty may be gathered from the pages of the report. Not much had been done to remedy the old defects; +LJ019-0134|radical improvement was generally considered impossible. The great evil, however, had been sensibly diminished.|radical improvement was generally considered impossible. The great evil, however, had been sensibly diminished. +LJ019-0135|There was no longer, or at worst but rarely, and for short periods, the same overcrowding.|There was no longer, or at worst but rarely, and for short periods, the same overcrowding. +LJ019-0136|This was obviated by the frequent sessions of the Central Criminal Court, and the utilization of the two subsidiary prisons in Giltspur Street and Southwark.|This was obviated by the frequent sessions of the Central Criminal Court, and the utilization of the two subsidiary prisons in Giltspur Street and Southwark. +LJ019-0137|The prison population of Newgate was still subject to great fluctuations, but it seldom rose above two hundred and fifty or three hundred|The prison population of Newgate was still subject to great fluctuations, but it seldom rose above two hundred and fifty or three hundred +LJ019-0138|at the most crowded periods, or just before the sessional jail delivery; and at its lowest it fell sometimes to fifty or sixty.|at the most crowded periods, or just before the sessional jail delivery; and at its lowest it fell sometimes to fifty or sixty. +LJ019-0139|These numbers would have still further decreased, and the jail would have been almost empty, but for the misdemeanants who were still sent to Newgate|These numbers would have still further decreased, and the jail would have been almost empty, but for the misdemeanants who were still sent to Newgate +LJ019-0140|at times on long terms of imprisonment, and for the transports, whom the Home Office were often, as of old, slow to remove.|at times on long terms of imprisonment, and for the transports, whom the Home Office were often, as of old, slow to remove. +LJ019-0141|The old wards, day rooms and sleeping rooms combined, of which the reader has already heard so much,|The old wards, day rooms and sleeping rooms combined, of which the reader has already heard so much, +LJ019-0142|now seldom contained more than ten or a dozen each. Some sort of decorum was maintained among the occupants in the day-time.|now seldom contained more than ten or a dozen each. Some sort of decorum was maintained among the occupants in the day-time. +LJ019-0143|Drinking and gaming,|Drinking and gaming, +LJ019-0144|the indiscriminate visitation of friends, and the almost unlimited admission of extra food, these more glaring defects had disappeared.|the indiscriminate visitation of friends, and the almost unlimited admission of extra food, these more glaring defects had disappeared. +LJ019-0145|But reformation was only skin deep. Below the surface many of the old evils still rankled.|But reformation was only skin deep. Below the surface many of the old evils still rankled. +LJ019-0146|There was as yet no control over the prisoners after locking-up time;|There was as yet no control over the prisoners after locking-up time; +LJ019-0147|this occurred in summer at eight, but in the winter months it took place at dusk, and was often as early as four or five.|this occurred in summer at eight, but in the winter months it took place at dusk, and was often as early as four or five. +LJ019-0148|The prisoners were still left to themselves till next morning's unlocking,|The prisoners were still left to themselves till next morning's unlocking, +LJ019-0149|and they spent some fourteen or fifteen hours in total darkness, and almost without check or control.|and they spent some fourteen or fifteen hours in total darkness, and almost without check or control. +LJ019-0150|Captain Williams, who was the inspector of prisons for the home district in succession to Messrs. Crawford and Russell,|Captain Williams, who was the inspector of prisons for the home district in succession to Messrs. Crawford and Russell, +LJ019-0151|stated in evidence that he was visiting Newgate one night, when he heard a great disturbance in one of the day and sleeping rooms,|stated in evidence that he was visiting Newgate one night, when he heard a great disturbance in one of the day and sleeping rooms, +LJ019-0152|and on entering it found the prisoners engaged in kicking bundles of wood from one end of the ward to the other.|and on entering it found the prisoners engaged in kicking bundles of wood from one end of the ward to the other. +LJ019-0153|Some attempt at supervision was exercised by the night watchman stationed on the leads, who might hear what went on inside.|Some attempt at supervision was exercised by the night watchman stationed on the leads, who might hear what went on inside. +LJ019-0154|If any disturbance reached his ears, he reported the case to the governor, who next morning visited the ward in fault, and asked for the culprit.|If any disturbance reached his ears, he reported the case to the governor, who next morning visited the ward in fault, and asked for the culprit. +LJ019-0155|The enforcement of discipline depended upon the want of honor among thieves.|The enforcement of discipline depended upon the want of honor among thieves. +LJ019-0156|Unless the guilty prisoner was given up, the whole ward was punished, either by the exclusion of visitors or the deprivation of fire,|Unless the guilty prisoner was given up, the whole ward was punished, either by the exclusion of visitors or the deprivation of fire, +LJ019-0157|sharp tests which generally broke down the fidelity of the inmates of the ward to one another.|sharp tests which generally broke down the fidelity of the inmates of the ward to one another. +LJ019-0158|Later on a more efficacious but still imperfect method of supervision was introduced. Iron cages, which are still to be seen in Newgate,|Later on a more efficacious but still imperfect method of supervision was introduced. Iron cages, which are still to be seen in Newgate, +LJ019-0159|were constructed on the landings, ensconced in which warders spent the night, on duty, and alert to watch the sleepers below,|were constructed on the landings, ensconced in which warders spent the night, on duty, and alert to watch the sleepers below, +LJ019-0160|and check by remonstrance or threat of punishment all who broke the peace of the prison.|and check by remonstrance or threat of punishment all who broke the peace of the prison. +LJ019-0161|These disciplinary improvements were, however, only slowly and gradually introduced.|These disciplinary improvements were, however, only slowly and gradually introduced. +LJ019-0162|Other changes affecting the condition and proper treatment of prisoners were not made until the inspector had urged and recommended them.|Other changes affecting the condition and proper treatment of prisoners were not made until the inspector had urged and recommended them. +LJ019-0163|Thus the wards, which, as I have said, were left in complete darkness, were now to be lighted with gas; and after this most salutary addition,|Thus the wards, which, as I have said, were left in complete darkness, were now to be lighted with gas; and after this most salutary addition, +LJ019-0164|the personal superintendence of night officers, as already described, became possible.|the personal superintendence of night officers, as already described, became possible. +LJ019-0165|The rule became general as regards the prison dress; hitherto clothing had been issued only to such as were destitute or in rags, and all classes of prisoners,|The rule became general as regards the prison dress; hitherto clothing had been issued only to such as were destitute or in rags, and all classes of prisoners, +LJ019-0166|those for trial, and those sentenced for short terms or long|those for trial, and those sentenced for short terms or long +LJ019-0167|wore no distinguishing costume, although its use was admitted, not only for cleanliness, but as a badge of condition, and a security against escape.|wore no distinguishing costume, although its use was admitted, not only for cleanliness, but as a badge of condition, and a security against escape. +LJ019-0168|Renewed recommendations to provide employment resulted in the provision of a certain amount of oakum for picking,|Renewed recommendations to provide employment resulted in the provision of a certain amount of oakum for picking, +LJ019-0169|and one or two men were allowed to mend clothes and make shoes. The rules made by the Secretary of State were hung up in conspicuous parts of the prison;|and one or two men were allowed to mend clothes and make shoes. The rules made by the Secretary of State were hung up in conspicuous parts of the prison; +LJ019-0170|more officers were appointed, as the time of so many of those already on the staff was monopolized by attendance at the Central Criminal Court.|more officers were appointed, as the time of so many of those already on the staff was monopolized by attendance at the Central Criminal Court. +LJ019-0171|Another custom which had led to disorder was abolished;|Another custom which had led to disorder was abolished; +LJ019-0172|prisoners who had been acquitted were not permitted to return to the prison to show their joy and receive the congratulations of their unfortunate fellows.|prisoners who had been acquitted were not permitted to return to the prison to show their joy and receive the congratulations of their unfortunate fellows. +LJ019-0173|The Corporation seems to have introduced these salutary changes without hesitation.|The Corporation seems to have introduced these salutary changes without hesitation. +LJ019-0174|It was less prompt apparently in dealing with structural alterations and improvements.|It was less prompt apparently in dealing with structural alterations and improvements. +LJ019-0175|Well-founded complaints had been made of the want of heating appliances in the jail.|Well-founded complaints had been made of the want of heating appliances in the jail. +LJ019-0176|The wards had open fires, but the separate cells were not warmed at all.|The wards had open fires, but the separate cells were not warmed at all. +LJ019-0177|A scheme for heating the whole prison with hot-water pipes, after the system now generally adopted elsewhere, was considered,|A scheme for heating the whole prison with hot-water pipes, after the system now generally adopted elsewhere, was considered, +LJ019-0179|This, with a scheme for limiting the jail to untried prisoners, had been urgently recommended by Lord John Russell in 1830.|This, with a scheme for limiting the jail to untried prisoners, had been urgently recommended by Lord John Russell in eighteen thirty. +LJ019-0180|His letter to the Corporation, under date 4th June,|His letter to the Corporation, under date fourth June, +LJ019-0181|is an interesting document, and shows that even at that date the Government contemplated the erection of a model prison.|is an interesting document, and shows that even at that date the Government contemplated the erection of a model prison. +LJ019-0182|Lord John Russell, commenting upon the offer of the Corporation to improve Newgate,|Lord John Russell, commenting upon the offer of the Corporation to improve Newgate, +LJ019-0183|provided it was henceforth used only for untried prisoners, suggested that Newgate should be entirely reconstructed, and the new building adopted as a model.|provided it was henceforth used only for untried prisoners, suggested that Newgate should be entirely reconstructed, and the new building adopted as a model. +LJ019-0184|The Corporation had agreed to spend £20,000 on alterations, but £60,000 would suffice to reconstruct.|The Corporation had agreed to spend twenty thousand pounds on alterations, but sixty thousand pounds would suffice to reconstruct. +LJ019-0185|Lord John, with great fairness, admitted that the whole of this burthen could not be imposed upon the city|Lord John, with great fairness, admitted that the whole of this burthen could not be imposed upon the city +LJ019-0186|seeing that since the establishment of the Central Criminal Court, Newgate received prisoners for trial from several counties,|seeing that since the establishment of the Central Criminal Court, Newgate received prisoners for trial from several counties, +LJ019-0187|and he was therefore prepared to submit to Parliament a proposal that half the cost of reconstruction should be borne by public funds.|and he was therefore prepared to submit to Parliament a proposal that half the cost of reconstruction should be borne by public funds. +LJ019-0188|He forwarded plans prepared by the inspectors of prisons, not for blind adoption, but as a guide.|He forwarded plans prepared by the inspectors of prisons, not for blind adoption, but as a guide. +LJ019-0189|This plan was on the principle of cellular separation, a system, according to Lord John Russell, desirable in all prisons,|This plan was on the principle of cellular separation, a system, according to Lord John Russell, desirable in all prisons, +LJ019-0190|"but in a metropolitan prison absolutely essential." The Corporation in reply demurred rather to accepting strict separation as a rule,|"but in a metropolitan prison absolutely essential." The Corporation in reply demurred rather to accepting strict separation as a rule, +LJ019-0191|feeling that it approached too nearly to solitary confinement.|feeling that it approached too nearly to solitary confinement. +LJ019-0192|The court was, however, prepared to consider Lord John Russell's proposal with regard to the cost of rebuilding;|The court was, however, prepared to consider Lord John Russell's proposal with regard to the cost of rebuilding; +LJ019-0193|but as the plan was "confessedly experimental, for the benefit of the country generally, the amount for which the city should be responsible should be distinctly limited|but as the plan was "confessedly experimental, for the benefit of the country generally, the amount for which the city should be responsible should be distinctly limited +LJ019-0194|not to exceed a certain sum to be agreed upon.|not to exceed a certain sum to be agreed upon. +LJ019-0195|A proviso was also made that the magistrates should continue to exercise full control over the new jail,|A proviso was also made that the magistrates should continue to exercise full control over the new jail, +LJ019-0196|free from any other interference than that of the inspectors on the part of Government.|free from any other interference than that of the inspectors on the part of Government. +LJ019-0197|No doubt wiser counsels prevailed with Lord John Russell,|No doubt wiser counsels prevailed with Lord John Russell, +LJ019-0198|and on a more mature consideration he realized that the limited area of the existing Newgate site,|and on a more mature consideration he realized that the limited area of the existing Newgate site, +LJ019-0199|and the costliness of enlarging it, forbade all idea of entirely reconstructing the jail so as to constitute it a model prison.|and the costliness of enlarging it, forbade all idea of entirely reconstructing the jail so as to constitute it a model prison. +LJ019-0200|It would be far better to begin at the beginning,|It would be far better to begin at the beginning, +LJ019-0201|to select a sufficiently spacious piece of ground, and erect a prison which from foundations to roofs should be in conformity with the newest ideas.|to select a sufficiently spacious piece of ground, and erect a prison which from foundations to roofs should be in conformity with the newest ideas. +LJ019-0202|The preference given to the Pentonville system destroyed all hopes of a complete reformation of Newgate.|The preference given to the Pentonville system destroyed all hopes of a complete reformation of Newgate. +LJ019-0203|But the condition of the great city jail was evidently considered a reproach by the city authorities, and a year after the opening of the new "model" at Pentonville,|But the condition of the great city jail was evidently considered a reproach by the city authorities, and a year after the opening of the new "model" at Pentonville, +LJ019-0204|a serious effort was made to reconstruct Newgate.|a serious effort was made to reconstruct Newgate. +LJ019-0205|In 1845 the Jail Committee brought forward a definite proposal to purchase ground in the immediate vicinity for the erection of a new jail.|In eighteen forty-five the Jail Committee brought forward a definite proposal to purchase ground in the immediate vicinity for the erection of a new jail. +LJ019-0206|This jail was nominally to replace the Giltspur Street Compter,|This jail was nominally to replace the Giltspur Street Compter, +LJ019-0207|The site of which was to be sold to Christ's Hospital, but the intention was of course to embody and absorb old Newgate in the new construction.|The site of which was to be sold to Christ's Hospital, but the intention was of course to embody and absorb old Newgate in the new construction. +LJ019-0208|The proposal made was to purchase some fifty thousand square feet between Newgate, Warwick Lane, and the Sessions House,|The proposal made was to purchase some fifty thousand square feet between Newgate, Warwick Lane, and the Sessions House, +LJ019-0209|the situation having been proved by long experience to be salubrious.|the situation having been proved by long experience to be salubrious. +LJ019-0210|But when this suggestion was brought before the court of aldermen, various amendments were proposed.|But when this suggestion was brought before the court of aldermen, various amendments were proposed. +LJ019-0211|It was urged that the area selected for purchase must be excessively costly to acquire, and still quite inadequate for the city needs.|It was urged that the area selected for purchase must be excessively costly to acquire, and still quite inadequate for the city needs. +LJ019-0212|The Home Secretary had laid it down that at least five acres would be indispensable, and such an area it was impossible to obtain within the limits of the city.|The Home Secretary had laid it down that at least five acres would be indispensable, and such an area it was impossible to obtain within the limits of the city. +LJ019-0213|Now for the first time the Tuffnell estate in Holloway was mentioned.|Now for the first time the Tuffnell estate in Holloway was mentioned. +LJ019-0214|The Corporation owned lands there covering from nineteen to twenty acres.|The Corporation owned lands there covering from nineteen to twenty acres. +LJ019-0215|Why not move the city prison bodily into this more rural spot, with its purer air and greater breathing space?|Why not move the city prison bodily into this more rural spot, with its purer air and greater breathing space? +LJ019-0216|Eventually Holloway was decided upon as a site for the new city prison.|Eventually Holloway was decided upon as a site for the new city prison. +LJ019-0217|The necessary preliminaries took some time, but the contracts for the new building were completed in 1849, when the works were commenced.|The necessary preliminaries took some time, but the contracts for the new building were completed in eighteen forty-nine, when the works were commenced. +LJ019-0218|The prison was to contain four hundred and four prisoners, and the estimated expenditure was £79,000.|The prison was to contain four hundred and four prisoners, and the estimated expenditure was seventy-nine thousand pounds. +LJ019-0219|It was to accommodate only the convicted prisoners sentenced to terms short of penal servitude, and after its completion|It was to accommodate only the convicted prisoners sentenced to terms short of penal servitude, and after its completion +LJ019-0220|the uses of Newgate were narrowed almost entirely to those of a prison of detention.|the uses of Newgate were narrowed almost entirely to those of a prison of detention. +LJ019-0221|It was intended as far as possible that, except awaiting trial, no prisoner should find himself relegated to Newgate.|It was intended as far as possible that, except awaiting trial, no prisoner should find himself relegated to Newgate. +LJ019-0222|This principle became more and more generally the rule, although it has never been punctiliously observed. Now and again|This principle became more and more generally the rule, although it has never been punctiliously observed. Now and again +LJ019-0223|misdemeanants have found their way into Newgate, and within the last few years one offender against the privileges of the House of Commons.|misdemeanants have found their way into Newgate, and within the last few years one offender against the privileges of the House of Commons. +LJ019-0224|With the reduction of numbers to be accommodated, there was ample space in Newgate for its reconstruction on the most approved modern lines.|With the reduction of numbers to be accommodated, there was ample space in Newgate for its reconstruction on the most approved modern lines. +LJ019-0225|In 1857|In eighteen fifty-seven +LJ019-0226|the erection of a wing or large block of cells was commenced within the original walls of the prison, and upon the north or male side.|the erection of a wing or large block of cells was commenced within the original walls of the prison, and upon the north or male side. +LJ019-0227|This block contained one hundred and thirty cells, embracing every modern improvement;|This block contained one hundred and thirty cells, embracing every modern improvement; +LJ019-0228|it also contained eleven reception cells, six punishment cells, and a couple of cells for condemned criminals.|it also contained eleven reception cells, six punishment cells, and a couple of cells for condemned criminals. +LJ019-0229|This block was completed in 1859, after which the hitherto unavoidable and long-continued promiscuous association of prisoners|This block was completed in eighteen fifty-nine, after which the hitherto unavoidable and long-continued promiscuous association of prisoners +LJ019-0230|In 1861 a similar work was undertaken to provide separate cellular accommodation for the female inmates of Newgate,|In eighteen sixty-one a similar work was undertaken to provide separate cellular accommodation for the female inmates of Newgate, +LJ019-0231|and by the following year forty-seven new cells had been built on the most approved plan.|and by the following year forty-seven new cells had been built on the most approved plan. +LJ019-0232|During this reconstruction the female prisoners were lodged in Holloway,|During this reconstruction the female prisoners were lodged in Holloway, +LJ019-0233|and when it was completed, both sides of the prison were brought into harmony with modern ideas.|and when it was completed, both sides of the prison were brought into harmony with modern ideas. +LJ019-0234|The old buildings were entirely disused, and the whole of the inmates of Newgate were kept constantly in separate confinement.|The old buildings were entirely disused, and the whole of the inmates of Newgate were kept constantly in separate confinement. +LJ019-0235|With the last re-edification of Newgate, a work executed some seven centuries after the first stone of the old jail was laid,|With the last re-edification of Newgate, a work executed some seven centuries after the first stone of the old jail was laid, +LJ019-0236|the architectural records of the prison end. Nothing much was done at Newgate in the way of building, outside or in, after 1862.|the architectural records of the prison end. Nothing much was done at Newgate in the way of building, outside or in, after eighteen sixty-two. +LJ019-0237|The Act for private executions led to the erection of the gallows shed in the exercising yard, and at the flank of the passage from the condemned cells.|The Act for private executions led to the erection of the gallows shed in the exercising yard, and at the flank of the passage from the condemned cells. +LJ019-0238|The first "glass house," or room in which prisoners could talk in private with their attorneys, but yet be seen by the warder on the watch, had been constructed|The first "glass house," or room in which prisoners could talk in private with their attorneys, but yet be seen by the warder on the watch, had been constructed +LJ019-0239|and others were subsequently added.|and others were subsequently added. +LJ019-0241|But in the interval very comprehensive and, I think it must be admitted, salutary changes were successively introduced into the management of prisons.|But in the interval very comprehensive and, I think it must be admitted, salutary changes were successively introduced into the management of prisons. +LJ019-0242|Newgate naturally shared in any advantages due to these reforms. I propose, therefore, to refer to them in the concluding pages of this work,|Newgate naturally shared in any advantages due to these reforms. I propose, therefore, to refer to them in the concluding pages of this work, +LJ019-0243|and thus bring the history of prison discipline down to our own times.|and thus bring the history of prison discipline down to our own times. +LJ019-0244|The last inquiry into the condition and management of our jails and houses of correction was that made by the Lords' Committee in 1863.|The last inquiry into the condition and management of our jails and houses of correction was that made by the Lords' Committee in eighteen sixty-three. +LJ019-0245|The inquiry was most searching and complete, and the committee spoke plainly in its report.|The inquiry was most searching and complete, and the committee spoke plainly in its report. +LJ019-0246|It animadverted strongly on "the many and wide differences as regards construction, labor, diet, and general discipline"|It animadverted strongly on "the many and wide differences as regards construction, labor, diet, and general discipline" +LJ019-0247|which existed in the various prisons,|which existed in the various prisons, +LJ019-0248|leading to an inequality, uncertainty, and inefficiency of punishment productive of the most prejudicial results.|leading to an inequality, uncertainty, and inefficiency of punishment productive of the most prejudicial results. +LJ019-0249|The varieties in construction were still very marked.|The varieties in construction were still very marked. +LJ019-0250|In many prisons the prisoners were still associated, and, from the want of a sufficient number of cells, the principle of separation was still greatly neglected.|In many prisons the prisoners were still associated, and, from the want of a sufficient number of cells, the principle of separation was still greatly neglected. +LJ019-0251|Yet this principle, as the committee pointed out, "must now be accepted as the foundation of prison discipline,"|Yet this principle, as the committee pointed out, "must now be accepted as the foundation of prison discipline," +LJ019-0252|while its rigid maintenance was in its opinion vital to the efficiency of the jails.|while its rigid maintenance was in its opinion vital to the efficiency of the jails. +LJ019-0253|Even where cells had been built|Even where cells had been built +LJ019-0254|they were frequently below the standard size, and were therefore not certified for occupation as was required by law.|they were frequently below the standard size, and were therefore not certified for occupation as was required by law. +LJ019-0255|Great numbers were not lighted at night, and were without means by which their inmates could communicate, in case of urgent necessity, with their keepers.|Great numbers were not lighted at night, and were without means by which their inmates could communicate, in case of urgent necessity, with their keepers. +LJ019-0256|Still greater were the differences with regard to employment. The various authorities held widely different opinions as to what constituted hard labor.|Still greater were the differences with regard to employment. The various authorities held widely different opinions as to what constituted hard labor. +LJ019-0257|Here the tread-wheel was in use, there cellular cranks, or hard-labor machines.|Here the tread-wheel was in use, there cellular cranks, or hard-labor machines. +LJ019-0258|Both, however, varied greatly in mechanism and in the amount of energy they called forth,|Both, however, varied greatly in mechanism and in the amount of energy they called forth, +LJ019-0259|while the former was intended for the congregate labor of a number, and the latter, as its name implies, imposed continuous solitary toil.|while the former was intended for the congregate labor of a number, and the latter, as its name implies, imposed continuous solitary toil. +LJ019-0260|At other prisons "shot-drill," the lifting and carrying of heavy round shot, was the favorite method of inflicting penal labor.|At other prisons "shot-drill," the lifting and carrying of heavy round shot, was the favorite method of inflicting penal labor. +LJ019-0261|With these differences were others as opposed concerning industrial occupation.|With these differences were others as opposed concerning industrial occupation. +LJ019-0262|The jail authorities often gave the highest, possibly undue, importance to the value of remunerative employment,|The jail authorities often gave the highest, possibly undue, importance to the value of remunerative employment, +LJ019-0263|and sought to make profitable returns from prisoners' labor the test of prison efficiency. In this view the committee could not coincide,|and sought to make profitable returns from prisoners' labor the test of prison efficiency. In this view the committee could not coincide, +LJ019-0264|and it was decidedly of opinion that in all short sentences the hard labor of the tread-wheel, crank, and so forth should be the invariable rule.|and it was decidedly of opinion that in all short sentences the hard labor of the tread-wheel, crank, and so forth should be the invariable rule. +LJ019-0265|In dietaries, again, the same wide diversity of practice obtained.|In dietaries, again, the same wide diversity of practice obtained. +LJ019-0266|The efforts made by Sir James Graham years before to introduce uniformity in this particular had failed of effect.|The efforts made by Sir James Graham years before to introduce uniformity in this particular had failed of effect. +LJ019-0267|The Secretary of State's suggested scale of diet had seldom been closely followed.|The Secretary of State's suggested scale of diet had seldom been closely followed. +LJ019-0268|In some places the dietary was too full, in others too meager. Its constituents were not of the most suitable character.|In some places the dietary was too full, in others too meager. Its constituents were not of the most suitable character. +LJ019-0269|More animal food was given than was necessary.|More animal food was given than was necessary. +LJ019-0270|Vegetables, especially the potato, that most valuable anti-scorbutic, was too often omitted.|Vegetables, especially the potato, that most valuable anti-scorbutic, was too often omitted. +LJ019-0271|In a word, the value of diet as a part of penal discipline was still insufficiently recognized.|In a word, the value of diet as a part of penal discipline was still insufficiently recognized. +LJ019-0272|The prisons were still far from inflicting the three punishments, hard labor, hard fare, and a hard bed|The prisons were still far from inflicting the three punishments, hard labor, hard fare, and a hard bed +LJ019-0273|which Sir Joshua Jebb told the committee he considered the proper elements of penal discipline.|which Sir Joshua Jebb told the committee he considered the proper elements of penal discipline. +LJ019-0274|It is interesting to note here|It is interesting to note here +LJ019-0275|that the committee of 1863 fully endorsed Sir Joshua's recommendations as regards a "hard bed," and recommended that|that the committee of eighteen sixty-three fully endorsed Sir Joshua's recommendations as regards a "hard bed," and recommended that +LJ019-0276|during short sentences, or the earlier stages of a long confinement,|during short sentences, or the earlier stages of a long confinement, +LJ019-0277|the prisoners should be made to dispense with the use of a mattress, and should sleep on planks.|the prisoners should be made to dispense with the use of a mattress, and should sleep on planks. +LJ019-0278|This suggestion was adopted in the Act of 1865, which followed the committee's report, and of which more directly.|This suggestion was adopted in the Act of eighteen sixty-five, which followed the committee's report, and of which more directly. +LJ019-0279|Clause 92, Schedule 1 of that act authorized the use of plank beds, which were adopted in many prisons.|Clause ninety-two, Schedule one of that act authorized the use of plank beds, which were adopted in many prisons. +LJ019-0280|They are now the universal rule,|They are now the universal rule, +LJ019-0281|introduced, as was erroneously supposed, by the prison commissioners appointed under the Prison Act of 1877.|introduced, as was erroneously supposed, by the prison commissioners appointed under the Prison Act of eighteen seventy-seven. +LJ019-0282|Their origin it will be seen dates back much further than that.|Their origin it will be seen dates back much further than that. +LJ019-0283|Beds might well be made hard and their use strictly limited.|Beds might well be made hard and their use strictly limited. +LJ019-0284|According to this committee of 1863, beds in the smaller and most carelessly conducted prisons formed a large element in the life of a prisoner.|According to this committee of eighteen sixty-three, beds in the smaller and most carelessly conducted prisons formed a large element in the life of a prisoner. +LJ019-0285|In one jail fifteen hours were spent in bed out of the twenty-four. This was in keeping with other grave defects and omissions.|In one jail fifteen hours were spent in bed out of the twenty-four. This was in keeping with other grave defects and omissions. +LJ019-0286|The minor borough prisons were the worst blot on the still dark and imperfect system.|The minor borough prisons were the worst blot on the still dark and imperfect system. +LJ019-0287|They were very numerous, very imperfect in construction and management, and they were very little required.|They were very numerous, very imperfect in construction and management, and they were very little required. +LJ019-0288|In them, according to the committee, the old objectionable practices were still in full force.|In them, according to the committee, the old objectionable practices were still in full force. +LJ019-0289|There was unrestrained association of untried and convicted, juvenile with adult prisoners, vagrants, misdemeanants, felons.|There was unrestrained association of untried and convicted, juvenile with adult prisoners, vagrants, misdemeanants, felons. +LJ019-0290|There were dormitories without light, control, or regulation at night,|There were dormitories without light, control, or regulation at night, +LJ019-0291|which warders, dreading assault, were afraid to enter after dark, even to check rioting and disturbance.|which warders, dreading assault, were afraid to enter after dark, even to check rioting and disturbance. +LJ019-0292|Prisoners still slept two in a bed.|Prisoners still slept two in a bed. +LJ019-0293|In one prison the bedsteads had been removed lest the prisoners should break them up and convert them into weapons of offense.|In one prison the bedsteads had been removed lest the prisoners should break them up and convert them into weapons of offense. +LJ019-0294|The prison buildings were in many places out of repair; other houses often overlooked them.|The prison buildings were in many places out of repair; other houses often overlooked them. +LJ019-0295|A single officer was the only custodian and disciplinary authority in the jail.|A single officer was the only custodian and disciplinary authority in the jail. +LJ019-0296|Complete idleness was tolerated; there was neither penal labor nor light employment.|Complete idleness was tolerated; there was neither penal labor nor light employment. +LJ019-0297|The prisoners inter-communicated freely, and exercised the most injurious, corrupting influences upon one another.|The prisoners inter-communicated freely, and exercised the most injurious, corrupting influences upon one another. +LJ019-0298|The total want of administration was very marked,|The total want of administration was very marked, +LJ019-0299|but in one prison it was such that the prisoners' food was supplied daily from the neighboring inn, and the innkeeper's bill constituted the only accounts kept.|but in one prison it was such that the prisoners' food was supplied daily from the neighboring inn, and the innkeeper's bill constituted the only accounts kept. +LJ019-0300|The committee might well suggest the abolition of these jails, or their amalgamation with the larger county establishments in their immediate neighborhood.|The committee might well suggest the abolition of these jails, or their amalgamation with the larger county establishments in their immediate neighborhood. +LJ019-0301|Some idea of the comparative uselessness of these small borough prisons was conveyed by some figures quoted by the committee.|Some idea of the comparative uselessness of these small borough prisons was conveyed by some figures quoted by the committee. +LJ019-0302|In 1862 there were in all one hundred and ninety-three jails in England and Wales;|In eighteen sixty-two there were in all one hundred and ninety-three jails in England and Wales; +LJ019-0303|of these, sixty-three gave admittance during the entire year to less than twenty-five prisoners;|of these, sixty-three gave admittance during the entire year to less than twenty-five prisoners; +LJ019-0304|twenty-two others received between eleven and twenty-five;|twenty-two others received between eleven and twenty-five; +LJ019-0305|fourteen received less than eleven and more than six;|fourteen received less than eleven and more than six; +LJ019-0306|while twenty-seven received less than six prisoners, and were in some instances absolutely tenantless.|while twenty-seven received less than six prisoners, and were in some instances absolutely tenantless. +LJ019-0307|The result of the recommendation of the committee of 1862 was the Prison Act of 1865,|The result of the recommendation of the committee of eighteen sixty-two was the Prison Act of eighteen sixty-five, +LJ019-0308|the penultimate of such enactments, many of the provisions of which still remain in force.|the penultimate of such enactments, many of the provisions of which still remain in force. +LJ019-0309|The main object of this act was to compass that uniformity in discipline and treatment generally|The main object of this act was to compass that uniformity in discipline and treatment generally +LJ019-0310|which had long been admitted as indispensable, and had never as yet been properly obtained.|which had long been admitted as indispensable, and had never as yet been properly obtained. +LJ019-0311|The legislature was beginning to overcome its disinclination to interfere actively or authoritatively|The legislature was beginning to overcome its disinclination to interfere actively or authoritatively +LJ019-0312|with the local jurisdictions, although still very leniently disposed.|with the local jurisdictions, although still very leniently disposed. +LJ019-0313|However, it now laid down in plain language and with precise details the requirements of a good jail system.|However, it now laid down in plain language and with precise details the requirements of a good jail system. +LJ019-0314|The separation of prisoners in cells duly certified by the inspectors was insisted upon,|The separation of prisoners in cells duly certified by the inspectors was insisted upon, +LJ019-0315|also their constant employment in labor appropriate to their condition.|also their constant employment in labor appropriate to their condition. +LJ019-0316|Hard labor of the first and second class was carefully defined.|Hard labor of the first and second class was carefully defined. +LJ019-0317|The former, which consisted principally of the tread-wheel, cranks, capstans, shot-drill,|The former, which consisted principally of the tread-wheel, cranks, capstans, shot-drill, +LJ019-0318|was to be the rule for all convicted prisoners throughout the early stages of their detention;|was to be the rule for all convicted prisoners throughout the early stages of their detention; +LJ019-0319|while the latter, which included various forms of industrial employment,|while the latter, which included various forms of industrial employment, +LJ019-0320|was the boon to which willing industry extending over a long period established a certain claim.|was the boon to which willing industry extending over a long period established a certain claim. +LJ019-0321|The infliction of punishment more or less uniform was thus aimed at.|The infliction of punishment more or less uniform was thus aimed at. +LJ019-0322|On the other hand, new and careful regulations were framed to secure the moral and material well-being of the inmates of the jails.|On the other hand, new and careful regulations were framed to secure the moral and material well-being of the inmates of the jails. +LJ019-0323|The law made it imperative that every prison should have a prison chapel, and that daily and Sunday services should be held.|The law made it imperative that every prison should have a prison chapel, and that daily and Sunday services should be held. +LJ019-0324|The chaplain's duties were enlarged, and the principle of toleration accepted to the extent of securing to all prisoners|The chaplain's duties were enlarged, and the principle of toleration accepted to the extent of securing to all prisoners +LJ019-0325|the ministrations of ministers of their own form of belief.|the ministrations of ministers of their own form of belief. +LJ019-0326|Steps were taken to provide the illiterate with secular instruction.|Steps were taken to provide the illiterate with secular instruction. +LJ019-0327|No less close was the care as regards preservation of health.|No less close was the care as regards preservation of health. +LJ019-0328|Stringent rules were prescribed for the prison surgeons;|Stringent rules were prescribed for the prison surgeons; +LJ019-0329|every prison was ordered to keep up an infirmary, and the medical supervision was to be strict and continuous.|every prison was ordered to keep up an infirmary, and the medical supervision was to be strict and continuous. +LJ019-0330|Dietaries were drawn up for adoption on the recommendation of a committee of experts.|Dietaries were drawn up for adoption on the recommendation of a committee of experts. +LJ019-0331|Baths were provided, ablutions ordered, and all appliances to insure personal cleanliness.|Baths were provided, ablutions ordered, and all appliances to insure personal cleanliness. +LJ019-0332|The administration of good government was to be watched over by the local magistracy, certain of whom, styled visiting justices,|The administration of good government was to be watched over by the local magistracy, certain of whom, styled visiting justices, +LJ019-0333|were elected to inspect the prisons frequently, to examine the prisoners, hear complaints, and check abuses.|were elected to inspect the prisons frequently, to examine the prisoners, hear complaints, and check abuses. +LJ019-0334|Under them the governor or jailer was held strictly responsible.|Under them the governor or jailer was held strictly responsible. +LJ019-0335|The books and journals he was to keep were minutely specified, and his constant presence in or near the jail was insisted upon.|The books and journals he was to keep were minutely specified, and his constant presence in or near the jail was insisted upon. +LJ019-0336|His disciplinary powers were defined by the act, and his duties,|His disciplinary powers were defined by the act, and his duties, +LJ019-0337|both in controlling his subordinates and in protecting the prisoners from petty tyranny and oppression, every one of whom he was to see once every twenty-four hours.|both in controlling his subordinates and in protecting the prisoners from petty tyranny and oppression, every one of whom he was to see once every twenty-four hours. +LJ019-0338|But discipline was to be maintained if necessary by punishment,|But discipline was to be maintained if necessary by punishment, +LJ019-0339|while decency and good order were to be insured by the strict prohibition of gambling and drunkenness.|while decency and good order were to be insured by the strict prohibition of gambling and drunkenness. +LJ019-0340|The latter was rendered nearly impossible by the penalties imposed on persons bringing spirituous liquors into the jail.|The latter was rendered nearly impossible by the penalties imposed on persons bringing spirituous liquors into the jail. +LJ019-0341|The old custom, so fruitful of the worst evils, of keeping a tap inside the prison was made illegal.|The old custom, so fruitful of the worst evils, of keeping a tap inside the prison was made illegal. +LJ019-0342|So was the employment of prisoners in any position of trust or authority;|So was the employment of prisoners in any position of trust or authority; +LJ019-0343|they were not to be turnkeys or assistant turnkeys, neither wardsman nor yardsman, overseer,|they were not to be turnkeys or assistant turnkeys, neither wardsman nor yardsman, overseer, +LJ019-0344|monitor, or schoolmaster, nor to be engaged in the service of any officer of the prison.|monitor, or schoolmaster, nor to be engaged in the service of any officer of the prison. +LJ019-0345|The Act of 1865 also encouraged and empowered the local authorities to "alter, enlarge, or rebuild" their prisons.|The Act of eighteen sixty-five also encouraged and empowered the local authorities to "alter, enlarge, or rebuild" their prisons. +LJ019-0346|They might raise funds for this purpose, provided a certificate for the necessity for the new works was given, either by the recorder|They might raise funds for this purpose, provided a certificate for the necessity for the new works was given, either by the recorder +LJ019-0347|chairman of quarter sessions, or even by a couple of justices.|chairman of quarter sessions, or even by a couple of justices. +LJ019-0348|Every facility was promised. The sanction of the Secretary of State would not be withheld if plans and estimates were duly submitted,|Every facility was promised. The sanction of the Secretary of State would not be withheld if plans and estimates were duly submitted, +LJ019-0349|and they met with the approval of his professional adviser, the surveyor-general of prisons.|and they met with the approval of his professional adviser, the surveyor-general of prisons. +LJ019-0350|The funds necessary would be advanced by the Public Works Loan Commissioners, and the interest might be charged against the county or borough rates.|The funds necessary would be advanced by the Public Works Loan Commissioners, and the interest might be charged against the county or borough rates. +LJ019-0351|Nor were these the only inducements offered. Where local authorities were indisposed to set their prisons in order,|Nor were these the only inducements offered. Where local authorities were indisposed to set their prisons in order, +LJ019-0352|or hesitated to embark upon any considerable expenditure to alter or rebuild,|or hesitated to embark upon any considerable expenditure to alter or rebuild, +LJ019-0353|they were at liberty to hire suitable cell accommodation from any neighbors who might have it to spare; the only proviso,|they were at liberty to hire suitable cell accommodation from any neighbors who might have it to spare; the only proviso, +LJ019-0354|That no such contract was valid between one jurisdiction and another unless the Secretary of State was satisfied that the prison it was intended to use|That no such contract was valid between one jurisdiction and another unless the Secretary of State was satisfied that the prison it was intended to use +LJ019-0355|came up in all respects to modern requirements.|came up in all respects to modern requirements. +LJ019-0356|But the act was not limited to permissive legislation. Its provisions and enactments were backed up by certain penalties.|But the act was not limited to permissive legislation. Its provisions and enactments were backed up by certain penalties. +LJ019-0357|The Secretary of State was empowered to deal rather summarily with "inadequate" prisons, in other words,|The Secretary of State was empowered to deal rather summarily with "inadequate" prisons, in other words, +LJ019-0358|with those in which there was no separation, no proper enforcement of hard labor, no chapel, infirmary, and so forth.|with those in which there was no separation, no proper enforcement of hard labor, no chapel, infirmary, and so forth. +LJ019-0359|He could in the first place withhold the government grant in aid of prison funds by refusing the certificate to the Treasury upon which the allowance was paid.|He could in the first place withhold the government grant in aid of prison funds by refusing the certificate to the Treasury upon which the allowance was paid. +LJ019-0360|This he might do on the representation of the inspector of prisons,|This he might do on the representation of the inspector of prisons, +LJ019-0361|who was bound to report any deficiencies and abuses he might find at his periodical visits. The Secretary of State might go further.|who was bound to report any deficiencies and abuses he might find at his periodical visits. The Secretary of State might go further. +LJ019-0362|Where the local authority had neglected to comply with the provisions of the 1865 Act for four consecutive years,|Where the local authority had neglected to comply with the provisions of the eighteen sixty-five Act for four consecutive years, +LJ019-0363|he could close the "inadequate" prison, by declaring it unfit for the reception of prisoners.|he could close the "inadequate" prison, by declaring it unfit for the reception of prisoners. +LJ019-0364|His order would at the same time specify some neighboring and more satisfactory prison which the local authority would be compelled to utilize instead,|His order would at the same time specify some neighboring and more satisfactory prison which the local authority would be compelled to utilize instead, +LJ019-0365|and with the concurrence of the other authority, and on payment. A few provisos governed these rather extensive powers.|and with the concurrence of the other authority, and on payment. A few provisos governed these rather extensive powers. +LJ019-0366|It was necessary, for instance, to give due notice when the government grant was to be withdrawn,|It was necessary, for instance, to give due notice when the government grant was to be withdrawn, +LJ019-0367|and with the warning a copy of the particular defects and allegations was to be sent to the local authority.|and with the warning a copy of the particular defects and allegations was to be sent to the local authority. +LJ019-0368|The latter too was to be laid before the House of Commons.|The latter too was to be laid before the House of Commons. +LJ019-0369|In the same way, six months' notice was required in cases where the closing of a prison was contemplated;|In the same way, six months' notice was required in cases where the closing of a prison was contemplated; +LJ019-0370|but if these conditions were observed, the Secretary of State could deal sharply enough with the defaulting jurisdictions.|but if these conditions were observed, the Secretary of State could deal sharply enough with the defaulting jurisdictions. +LJ019-0371|Yet the law was seldom if ever enforced.|Yet the law was seldom if ever enforced. +LJ019-0372|It was practically inoperative as regards the penalties for neglect. It was no doubt as irksome and inconvenient to the Secretary of State|It was practically inoperative as regards the penalties for neglect. It was no doubt as irksome and inconvenient to the Secretary of State +LJ019-0373|to avail himself of his powers, as it was difficult to bring home the derelictions of duties and evasion of the acts. Too much was left to the inspectors.|to avail himself of his powers, as it was difficult to bring home the derelictions of duties and evasion of the acts. Too much was left to the inspectors. +LJ019-0374|It was nearly impossible for them to exercise a very close supervision over the whole of the prisons of the country.|It was nearly impossible for them to exercise a very close supervision over the whole of the prisons of the country. +LJ019-0375|There were only two of them, and they could not visit each prison more than once in each year, sometimes not oftener than once in eighteen months.|There were only two of them, and they could not visit each prison more than once in each year, sometimes not oftener than once in eighteen months. +LJ019-0376|The task imposed upon them, tending as it did to the imposition of a fine upon the local authorities, was not a pleasant one,|The task imposed upon them, tending as it did to the imposition of a fine upon the local authorities, was not a pleasant one, +LJ019-0377|and it is not strange if they did not very frequently hand up the offenders to the reproof and correction of the Secretary of State.|and it is not strange if they did not very frequently hand up the offenders to the reproof and correction of the Secretary of State. +LJ019-0378|As the almost inevitable consequence, while the more glaring defects in prison management disappeared,|As the almost inevitable consequence, while the more glaring defects in prison management disappeared, +LJ019-0379|matters went on after the 1865 Act much the same as they had done before. Districts differed greatly in the attention they paid to prison affairs.|matters went on after the eighteen sixty-five Act much the same as they had done before. Districts differed greatly in the attention they paid to prison affairs. +LJ019-0380|In one part the most praiseworthy activity prevailed,|In one part the most praiseworthy activity prevailed, +LJ019-0381|in another there was half-heartedness, even apathy and an almost complete contempt for the provisions of the act.|in another there was half-heartedness, even apathy and an almost complete contempt for the provisions of the act. +LJ019-0382|As the years passed, great want of uniformity continued to prevail throughout the prisons of the United Kingdom.|As the years passed, great want of uniformity continued to prevail throughout the prisons of the United Kingdom. +LJ019-0383|The whole question assumed sufficient importance to become a part of the Government program when Lord Beaconsfield took office in 1874.|The whole question assumed sufficient importance to become a part of the Government program when Lord Beaconsfield took office in eighteen seventy-four. +LJ019-0384|The Home Secretary in that administration,|The Home Secretary in that administration, +LJ019-0385|Mr. (now Sir Richard) Cross, having applied himself vigorously to the task of reorganizing the whole system, became convinced|Mr. (now Sir Richard) Cross, having applied himself vigorously to the task of reorganizing the whole system, became convinced +LJ019-0386|that no complete reform could be accomplished so long as the prisons were left under the jurisdiction of the local authorities.|that no complete reform could be accomplished so long as the prisons were left under the jurisdiction of the local authorities. +LJ019-0387|The Prisons Bill of 1876 contemplated the transfer of the prisons to Government.|The Prisons Bill of eighteen seventy-six contemplated the transfer of the prisons to Government. +LJ019-0388|This bill, reintroduced in 1877, became law that year, after which the whole of the prisons, including Newgate,|This bill, reintroduced in eighteen seventy-seven, became law that year, after which the whole of the prisons, including Newgate, +LJ019-0389|passed under the more direct control of the State.|passed under the more direct control of the State. +LJ019-0390|Since then a strong central authority has labored steadfastly to compass concentration,|Since then a strong central authority has labored steadfastly to compass concentration, +LJ019-0391|to close useless prisons, and to insure that uniformity of system which all thoughtful persons had long admitted to be of paramount importance|to close useless prisons, and to insure that uniformity of system which all thoughtful persons had long admitted to be of paramount importance +LJ019-0392|in the administration of prisons.|in the administration of prisons. +LJ019-0393|Three years after the advent of the prison commissioners, it was decided that Newgate was an excessively costly and redundant establishment.|Three years after the advent of the prison commissioners, it was decided that Newgate was an excessively costly and redundant establishment. +LJ019-0394|It was only filled at the periods when the sessions of the Central Criminal Court were in progress;|It was only filled at the periods when the sessions of the Central Criminal Court were in progress; +LJ019-0395|at others an expensive staff was maintained with little or nothing to do.|at others an expensive staff was maintained with little or nothing to do. +LJ019-0396|At a short distance stood another prison of detention, that of Clerkenwell,|At a short distance stood another prison of detention, that of Clerkenwell, +LJ019-0397|with spare accommodation sufficient to receive all prisoners who were then committed to Newgate. These arguments were unanswerable.|with spare accommodation sufficient to receive all prisoners who were then committed to Newgate. These arguments were unanswerable. +LJ019-0398|Accordingly, it was ordered by Sir William Harcourt, the present Secretary of State, that Newgate should cease to be used as a regular prison,|Accordingly, it was ordered by Sir William Harcourt, the present Secretary of State, that Newgate should cease to be used as a regular prison, +LJ019-0399|and it is now, except during sessions or when the gallows is in requisition, practically and for ever closed.|and it is now, except during sessions or when the gallows is in requisition, practically and for ever closed. +LJ020-0001|Marion Harland's Cookery for Beginners. Bread Sponge and Breakfast Breads.|Marion Harland's Cookery for Beginners. Bread Sponge and Breakfast Breads. +LJ020-0002|Bread raised with what is known to bakers as a “sponge,” requires more time and a trifle more work than the simpler form for which I have just already given directions.|Bread raised with what is known to bakers as a “sponge,” requires more time and a trifle more work than the simpler form for which I have just already given directions. +LJ020-0003|But it keeps fresh longer, is softer and more nutritious, and a second-rate brand of flour thus treated produces a better loaf|But it keeps fresh longer, is softer and more nutritious, and a second-rate brand of flour thus treated produces a better loaf +LJ020-0004|than when mixed up with yeast and water only.|than when mixed up with yeast and water only. +LJ020-0005|Sponge-making is, therefore, an important if not an essential accomplishment in a cook, be she novice or veteran.|Sponge-making is, therefore, an important if not an essential accomplishment in a cook, be she novice or veteran. +LJ020-0006|Three potatoes of fair size, peeled and boiled mealy.|Three potatoes of fair size, peeled and boiled mealy. +LJ020-0007|Five tablespoonfuls of yeast. One tablespoonful of white sugar.|Five tablespoonfuls of yeast. One tablespoonful of white sugar. +LJ020-0008|One tablespoonful of butter.|One tablespoonful of butter. +LJ020-0009|Three cups of lukewarm water in which the potatoes were boiled, strained through a coarse cloth.|Three cups of lukewarm water in which the potatoes were boiled, strained through a coarse cloth. +LJ020-0010|One heaping cup of sifted flour.|One heaping cup of sifted flour. +LJ020-0011|Put the potatoes into a large bowl or tray and mash them to powder with a potato beetle, or a wooden spoon.|Put the potatoes into a large bowl or tray and mash them to powder with a potato beetle, or a wooden spoon. +LJ020-0012|While still hot, mix in the sugar and butter, beating all to a lumpless cream.|While still hot, mix in the sugar and butter, beating all to a lumpless cream. +LJ020-0013|Add a few spoonfuls at a time, the potato-water alternately with the flour by the handful,|Add a few spoonfuls at a time, the potato-water alternately with the flour by the handful, +LJ020-0014|beating the batter smooth as you go on until all of the liquid and flour has gone in.|beating the batter smooth as you go on until all of the liquid and flour has gone in. +LJ020-0015|Beat hard one minute before pouring in the yeast.|Beat hard one minute before pouring in the yeast. +LJ020-0016|it is well to stir into the yeast a bit of soda no larger than a grain of corn already wet up in a teaspoonful of boiling water.|it is well to stir into the yeast a bit of soda no larger than a grain of corn already wet up in a teaspoonful of boiling water. +LJ020-0017|Now whip up the batter with a wooden spoon for another minute, and the sponge is made.|Now whip up the batter with a wooden spoon for another minute, and the sponge is made. +LJ020-0018|Throw a cloth over the bowl and set by for five or six hours to rise.|Throw a cloth over the bowl and set by for five or six hours to rise. +LJ020-0019|If you intend to bake in the forenoon, make the sponge at bedtime. If in the afternoon, early in the morning.|If you intend to bake in the forenoon, make the sponge at bedtime. If in the afternoon, early in the morning. +LJ020-0020|When the sponge is light sift a quart and a cup of flour into a bowl or tray with two teaspoonfuls of salt.|When the sponge is light sift a quart and a cup of flour into a bowl or tray with two teaspoonfuls of salt. +LJ020-0021|Into a hollow, like a crater in the middle of the flour, empty your sponge-bowl, and work the flour down into it.|Into a hollow, like a crater in the middle of the flour, empty your sponge-bowl, and work the flour down into it. +LJ020-0022|Wash out the bowl with a little lukewarm water and add this to the dough. If it should prove too soft, work in, cautiously, a little more flour.|Wash out the bowl with a little lukewarm water and add this to the dough. If it should prove too soft, work in, cautiously, a little more flour. +LJ020-0023|If too stiff, warm water, a spoonful at a time until you can handle the paste easily. The danger is in getting it too stiff. Now.|If too stiff, warm water, a spoonful at a time until you can handle the paste easily. The danger is in getting it too stiff. Now. +LJ020-0024|knead and set for risings first and second, as you have already been instructed. This sponge will be found especially useful in making Graham Bread.|knead and set for risings first and second, as you have already been instructed. This sponge will be found especially useful in making Graham Bread. +LJ020-0025|One quart of Graham flour, one cup of white flour.|One quart of Graham flour, one cup of white flour. +LJ020-0026|One half cup of Indian meal. One half cup of molasses. Two teaspoonfuls of salt. Soda, the size of a pea.|One half cup of Indian meal. One half cup of molasses. Two teaspoonfuls of salt. Soda, the size of a pea. +LJ020-0027|Half the quantity of sponge given in preceding receipt.|Half the quantity of sponge given in preceding receipt. +LJ020-0028|Warm water for rinsing bowl -- about half a cup.|Warm water for rinsing bowl -- about half a cup. +LJ020-0029|Put the brown or Graham flour unsifted into the bread-bowl.|Put the brown or Graham flour unsifted into the bread-bowl. +LJ020-0030|Sift into it white flour, meal and salt, and stir up well while dry.|Sift into it white flour, meal and salt, and stir up well while dry. +LJ020-0031|Into the “crater” dug out in the middle, pour the sponge, warm water, the molasses, and soda dissolved in hot water.|Into the “crater” dug out in the middle, pour the sponge, warm water, the molasses, and soda dissolved in hot water. +LJ020-0032|Knead as you would white bread, and set aside for the rising.|Knead as you would white bread, and set aside for the rising. +LJ020-0033|It will not swell so fast as the white, so give yourself more time for making it.|It will not swell so fast as the white, so give yourself more time for making it. +LJ020-0034|When light, knead well and long;|When light, knead well and long; +LJ020-0035|make into two loaves, then put into well-greased pans and leave for an hour, or until it becomes more than twice the original size of the dough.|make into two loaves, then put into well-greased pans and leave for an hour, or until it becomes more than twice the original size of the dough. +LJ020-0036|Take care that it does not burn in baking. The molasses renders it liable to scorching.|Take care that it does not burn in baking. The molasses renders it liable to scorching. +LJ020-0037|The oven must be steady, but not so hot as for white bread, nor will the Graham bread be done quite so soon as that made of bolted flour.|The oven must be steady, but not so hot as for white bread, nor will the Graham bread be done quite so soon as that made of bolted flour. +LJ020-0038|Turn the pans once while baking, moving them as gently as possible.|Turn the pans once while baking, moving them as gently as possible. +LJ020-0039|If rudely shaken or jarred, there will be heavy streaks in loaves.|If rudely shaken or jarred, there will be heavy streaks in loaves. +LJ020-0040|Graham bread is wholesome and sweet, and ought to be eaten frequently in every family, particularly by young people whose bones and teeth are in forming.|Graham bread is wholesome and sweet, and ought to be eaten frequently in every family, particularly by young people whose bones and teeth are in forming. +LJ020-0041|The phosphates which the process of “bolting” removes to a large extent from white flour, go directly to the manufacture of bone,|The phosphates which the process of “bolting” removes to a large extent from white flour, go directly to the manufacture of bone, +LJ020-0042|and these also tend to nourish and strengthen the brain.|and these also tend to nourish and strengthen the brain. +LJ020-0043|After mixing your bread in the morning either with sponge or with yeast, divide the kneaded dough into two portions.|After mixing your bread in the morning either with sponge or with yeast, divide the kneaded dough into two portions. +LJ020-0044|Mould one into a round ball, and set aside for a loaf as already directed.|Mould one into a round ball, and set aside for a loaf as already directed. +LJ020-0045|Make a hole in the middle of the other batch and pour into it a tablespoonful of butter, just melted, but not hot.|Make a hole in the middle of the other batch and pour into it a tablespoonful of butter, just melted, but not hot. +LJ020-0046|Close the dough over it, dust your hands and kneading-board with flour and work in the shortening until the dough is elastic and ceases to be sticky.|Close the dough over it, dust your hands and kneading-board with flour and work in the shortening until the dough is elastic and ceases to be sticky. +LJ020-0047|Put it into a floured bowl, cover with a cloth and set away out of draught and undue heat, for three hours.|Put it into a floured bowl, cover with a cloth and set away out of draught and undue heat, for three hours. +LJ020-0048|Knead it again, then, and wait upon its rising for another three hours. The dough should be as soft as can be handled.|Knead it again, then, and wait upon its rising for another three hours. The dough should be as soft as can be handled. +LJ020-0049|When it is light for the second time flour your board, rubbing in the flour and blowing lightly away what does not adhere to the surface.|When it is light for the second time flour your board, rubbing in the flour and blowing lightly away what does not adhere to the surface. +LJ020-0050|Toss the lump dough upon it and knead thoroughly for five minutes.|Toss the lump dough upon it and knead thoroughly for five minutes. +LJ020-0051|Flour a rolling-pin and roll the dough into a sheet not more than half an inch thick.|Flour a rolling-pin and roll the dough into a sheet not more than half an inch thick. +LJ020-0052|Cut this into round cakes with a biscuit-cutter or a sharp-edged tumbler and fold, not quite in the middle,|Cut this into round cakes with a biscuit-cutter or a sharp-edged tumbler and fold, not quite in the middle, +LJ020-0053|in the form of turnovers, pinching the corners of the fold pretty hard to hinder the flap of dough from flying up as the rising proceeds.|in the form of turnovers, pinching the corners of the fold pretty hard to hinder the flap of dough from flying up as the rising proceeds. +LJ020-0054|Rub the bottom and sides of a baking-pan with sweet lard or butter. Do this with a bit of clean soft rag or tissue-paper,|Rub the bottom and sides of a baking-pan with sweet lard or butter. Do this with a bit of clean soft rag or tissue-paper, +LJ020-0055|visiting every corner of the pan, but not leaving thick layers and streaks of grease after it.|visiting every corner of the pan, but not leaving thick layers and streaks of grease after it. +LJ020-0056|Arrange the rolls in regular rows in the pan about a quarter of an inch apart.|Arrange the rolls in regular rows in the pan about a quarter of an inch apart. +LJ020-0057|Cover with a cloth and set nearer the fire than you dared trust the dough, and let them rise for an hour.|Cover with a cloth and set nearer the fire than you dared trust the dough, and let them rise for an hour. +LJ020-0058|Peep under the cloth two or three times to see whether they rise evenly, and turn the pan around once that all may be equally exposed to the heat.|Peep under the cloth two or three times to see whether they rise evenly, and turn the pan around once that all may be equally exposed to the heat. +LJ020-0059|When the time is up and the rolls are puffy and promising, set them in a pretty quick oven and bake half an hour,|When the time is up and the rolls are puffy and promising, set them in a pretty quick oven and bake half an hour, +LJ020-0060|turning the pan once in this time, and covering with clean -- never printed -- paper, should they brown too fast.|turning the pan once in this time, and covering with clean -- never printed -- paper, should they brown too fast. +LJ020-0061|Break the rolls apart from one another and eat warm. They are also good cold, and if the directions be followed implicitly, very good always.|Break the rolls apart from one another and eat warm. They are also good cold, and if the directions be followed implicitly, very good always. +LJ020-0062|Graham Rolls are made by treating the dough mixed for Graham bread as above and following the foregoing receipt in every section, but allowing more time for rising and baking.|Graham Rolls are made by treating the dough mixed for Graham bread as above and following the foregoing receipt in every section, but allowing more time for rising and baking. +LJ020-0063|They are even better when cold than hot.|They are even better when cold than hot. +LJ020-0064|Breakfast Biscuit.|Breakfast Biscuit. +LJ020-0065|Two cups of fresh milk slightly warmed.|Two cups of fresh milk slightly warmed. +LJ020-0066|One quart and a cup of flour sifted.|One quart and a cup of flour sifted. +LJ020-0067|Five tablespoonfuls of yeast.|Five tablespoonfuls of yeast. +LJ020-0068|One even tablespoonful of white sugar.|One even tablespoonful of white sugar. +LJ020-0069|One even teaspoonful of salt.|One even teaspoonful of salt. +LJ020-0070|Bit of soda as large as a pea, dissolved in hot water.|Bit of soda as large as a pea, dissolved in hot water. +LJ020-0071|One tablespoonful of butter, just melted, not hot.|One tablespoonful of butter, just melted, not hot. +LJ020-0072|Yolk of one egg beaten light.|Yolk of one egg beaten light. +LJ020-0073|Sift the flour, salt and sugar into a bowl,|Sift the flour, salt and sugar into a bowl, +LJ020-0074|hollow the heap in the center and pour in the milk, working down the flour into the liquid with a spoon or your hands until it is thoroughly melted.|hollow the heap in the center and pour in the milk, working down the flour into the liquid with a spoon or your hands until it is thoroughly melted. +LJ020-0075|Into a second hollow pour the yeast and knead thoroughly for fifteen minutes.|Into a second hollow pour the yeast and knead thoroughly for fifteen minutes. +LJ020-0076|Wrap bowl and biscuit in a thick cloth and set to rise where it will neither become chilled nor sour over night.|Wrap bowl and biscuit in a thick cloth and set to rise where it will neither become chilled nor sour over night. +LJ020-0077|Study the temperature in different parts of the kitchen and kitchen closets to the end of finding the best places for raising dough and sponge.|Study the temperature in different parts of the kitchen and kitchen closets to the end of finding the best places for raising dough and sponge. +LJ020-0078|Do all this at bedtime. Early in the morning turn out the dough upon a floured board, work it for a minute into manageable shape;|Do all this at bedtime. Early in the morning turn out the dough upon a floured board, work it for a minute into manageable shape; +LJ020-0079|drill several finger-holes in it and fill them with the melted butter, the dissolved soda and the beaten yolk of egg.|drill several finger-holes in it and fill them with the melted butter, the dissolved soda and the beaten yolk of egg. +LJ020-0080|Pinch the dough hard to stop the mouths of these cavities,|Pinch the dough hard to stop the mouths of these cavities, +LJ020-0081|and knead for ten minutes, carefully at first, lest the liquids should be wasted, and more boldly when they are absorbed by the paste.|and knead for ten minutes, carefully at first, lest the liquids should be wasted, and more boldly when they are absorbed by the paste. +LJ020-0082|Roll out into a sheet half an inch thick with a floured rolling-pin; cut into round cakes, set these closely together in a well-greased pan;|Roll out into a sheet half an inch thick with a floured rolling-pin; cut into round cakes, set these closely together in a well-greased pan; +LJ020-0083|prick each with a fork and let them rise near the fire for half an hour, covered with a light cloth.|prick each with a fork and let them rise near the fire for half an hour, covered with a light cloth. +LJ020-0084|Bake from twenty to twenty-five minutes in a quick oven, turning the pan around once, quickly and lightly.|Bake from twenty to twenty-five minutes in a quick oven, turning the pan around once, quickly and lightly. +LJ020-0085|Break apart from one another and pile on a plate, throwing a clean doily or a small napkin over them. Break open at table.|Break apart from one another and pile on a plate, throwing a clean doily or a small napkin over them. Break open at table. +LJ020-0086|Hot rolls and muffins should never be cut.|Hot rolls and muffins should never be cut. +LJ020-0087|One word with regard to getting up early in order to give dough a chance for the second rising.|One word with regard to getting up early in order to give dough a chance for the second rising. +LJ020-0088|It is not a wholesome practice for any woman -- least of all a young girl to be out of bed two hours before she eats her breakfast.|It is not a wholesome practice for any woman -- least of all a young girl to be out of bed two hours before she eats her breakfast. +LJ020-0089|Studying upon an empty stomach provokes dyspepsia and injures the eyes.|Studying upon an empty stomach provokes dyspepsia and injures the eyes. +LJ020-0090|Active exercise in like circumstances tempts debility and disease.|Active exercise in like circumstances tempts debility and disease. +LJ020-0091|Yet our bread and rolls must be looked after at the proper time.|Yet our bread and rolls must be looked after at the proper time. +LJ020-0092|Have yourself called on biscuit mornings an hour earlier than usual.|Have yourself called on biscuit mornings an hour earlier than usual. +LJ020-0093|Rise, wash face and hands, rinse the mouth out and brush back the hair.|Rise, wash face and hands, rinse the mouth out and brush back the hair. +LJ020-0094|Put on stockings and slippers, such underclothing as may be needed to prevent cold, a wrapper and the kitchen apron.|Put on stockings and slippers, such underclothing as may be needed to prevent cold, a wrapper and the kitchen apron. +LJ020-0095|Cover your hair entirely with a handkerchief or sweeping cap.|Cover your hair entirely with a handkerchief or sweeping cap. +LJ020-0096|Before beginning operations downstairs eat a half-slice of dry bread or a biscuit.|Before beginning operations downstairs eat a half-slice of dry bread or a biscuit. +LJ020-0097|You will not relish it, but take it all the same to appease the empty, discontented stomach.|You will not relish it, but take it all the same to appease the empty, discontented stomach. +LJ020-0098|Having made out your rolls and tucked them up snugly for the final rise, return to your chamber for a comfortable bath and toilet.|Having made out your rolls and tucked them up snugly for the final rise, return to your chamber for a comfortable bath and toilet. +LJ020-0099|When habited for the day in all except the outer gown, collar, etc., slip on the wrapper again and run down to put the biscuits in the oven.|When habited for the day in all except the outer gown, collar, etc., slip on the wrapper again and run down to put the biscuits in the oven. +LJ020-0100|Unless it is too hot, they will get no harm while you finish dressing in ten minutes, just in season to turn the pan.|Unless it is too hot, they will get no harm while you finish dressing in ten minutes, just in season to turn the pan. +LJ020-0101|From the beginning of your apprenticeship in housewifery, learn how to "dovetail" your duties neatly into one another.|From the beginning of your apprenticeship in housewifery, learn how to "dovetail" your duties neatly into one another. +LJ020-0102|A wise accommodation of parts and angles, and compactness in the adjustment of "must-be-dones"|A wise accommodation of parts and angles, and compactness in the adjustment of "must-be-dones" +LJ020-0103|are better than mere personal strength in the accomplishment of such tasks as fall to women to perform.|are better than mere personal strength in the accomplishment of such tasks as fall to women to perform. +LJ020-0104|Master these, and do not let them master you.|Master these, and do not let them master you. +LJ020-0105|Weave the little duties in and under and among what seem to be the greater.|Weave the little duties in and under and among what seem to be the greater. +LJ020-0106|While your bread is taking a three hours’ rise, you are free in body and mind for other things.|While your bread is taking a three hours’ rise, you are free in body and mind for other things. +LJ020-0107|The grand secret of keeping house well and without worry, lies in the art of packing and fitting different kinds of work and in picking up the minutes.|The grand secret of keeping house well and without worry, lies in the art of packing and fitting different kinds of work and in picking up the minutes. +LJ020-0108|Other things besides rising dough get on quite as well without your standing by to watch them.|Other things besides rising dough get on quite as well without your standing by to watch them. +LJ021-0001|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt, Section 6.|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt, Section six. +LJ021-0002|September 30, 1934.|September thirty, nineteen thirty-four. +LJ021-0003|Three months have passed since I talked with you shortly after the adjournment of the Congress.|Three months have passed since I talked with you shortly after the adjournment of the Congress. +LJ021-0004|Tonight I continue that report, though, because of the shortness of time, I must defer a number of subjects to a later date.|Tonight I continue that report, though, because of the shortness of time, I must defer a number of subjects to a later date. +LJ021-0005|Recently the most notable public questions that have concerned us all|Recently the most notable public questions that have concerned us all +LJ021-0006|have had to do with industry and labor and with respect to these, certain developments have taken place which I consider of importance.|have had to do with industry and labor and with respect to these, certain developments have taken place which I consider of importance. +LJ021-0007|I am happy to report that after years of uncertainty, culminating in the collapse of the spring of 1933,|I am happy to report that after years of uncertainty, culminating in the collapse of the spring of nineteen thirty-three, +LJ021-0008|we are bringing order out of the old chaos|we are bringing order out of the old chaos +LJ021-0009|with a greater certainty of the employment of labor at a reasonable wage and of more business at a fair profit.|with a greater certainty of the employment of labor at a reasonable wage and of more business at a fair profit. +LJ021-0010|These governmental and industrial developments hold promise of new achievements for the nation.|These governmental and industrial developments hold promise of new achievements for the nation. +LJ021-0011|Men may differ as to the particular form of governmental activity|Men may differ as to the particular form of governmental activity +LJ021-0012|with respect to industry and business, but nearly all are agreed that private enterprise in times such as these|with respect to industry and business, but nearly all are agreed that private enterprise in times such as these +LJ021-0014|but also our processes of civilization.|but also our processes of civilization. +LJ021-0015|The underlying necessity for such activity|The underlying necessity for such activity +LJ021-0016|is indeed as strong now as it was years ago when Elihu Root said the following very significant words:|is indeed as strong now as it was years ago when Elihu Root said the following very significant words: +LJ021-0017|Instead of the give and take of free individual contract,|Instead of the give and take of free individual contract, +LJ021-0018|the tremendous power of organization has combined great aggregations of capital in enormous industrial establishments|the tremendous power of organization has combined great aggregations of capital in enormous industrial establishments +LJ021-0019|working through vast agencies of commerce and employing great masses of men in movements of production|working through vast agencies of commerce and employing great masses of men in movements of production +LJ021-0020|and transportation and trade, so great in the mass that each individual concerned in them is quite helpless by himself.|and transportation and trade, so great in the mass that each individual concerned in them is quite helpless by himself. +LJ021-0021|The relations between the employer and the employed, between the owners of aggregated capital and the units of organized labor,|The relations between the employer and the employed, between the owners of aggregated capital and the units of organized labor, +LJ021-0022|between the small producer, the small trader, the consumer,|between the small producer, the small trader, the consumer, +LJ021-0023|and the great transporting and manufacturing and distributing agencies,|and the great transporting and manufacturing and distributing agencies, +LJ021-0024|all present new questions for the solution of which the old reliance upon the free action of individual wills appears quite inadequate.|all present new questions for the solution of which the old reliance upon the free action of individual wills appears quite inadequate. +LJ021-0025|And in many directions, the intervention of that organized control which we call government|And in many directions, the intervention of that organized control which we call government +LJ021-0026|seems necessary to produce the same result of justice and right conduct|seems necessary to produce the same result of justice and right conduct +LJ021-0027|which obtained through the attrition of individuals before the new conditions arose.|which obtained through the attrition of individuals before the new conditions arose. +LJ021-0028|It was in this spirit thus described by Secretary Root|It was in this spirit thus described by Secretary Root +LJ021-0029|that we approached our task of reviving private enterprise in March, 1933.|that we approached our task of reviving private enterprise in March, nineteen thirty-three. +LJ021-0030|Our first problem was, of course, the banking situation because, as you know, the banks had collapsed.|Our first problem was, of course, the banking situation because, as you know, the banks had collapsed. +LJ021-0031|Some banks could not be saved but the great majority of them, either through their own resources or with government aid,|Some banks could not be saved but the great majority of them, either through their own resources or with government aid, +LJ021-0032|have been restored to complete public confidence.|have been restored to complete public confidence. +LJ021-0033|This has given safety to millions of depositors in these banks.|This has given safety to millions of depositors in these banks. +LJ021-0034|Closely following this great constructive effort we have, through various federal agencies,|Closely following this great constructive effort we have, through various federal agencies, +LJ021-0035|saved debtors and creditors alike in many other fields of enterprise, such as loans on farm mortgages and home mortgages;|saved debtors and creditors alike in many other fields of enterprise, such as loans on farm mortgages and home mortgages; +LJ021-0036|loans to the railroads and insurance companies and, finally, help for home owners and industry itself.|loans to the railroads and insurance companies and, finally, help for home owners and industry itself. +LJ021-0037|In all of these efforts the government has come to the assistance of business|In all of these efforts the government has come to the assistance of business +LJ021-0038|and with the full expectation that the money used to assist these enterprises will eventually be repaid.|and with the full expectation that the money used to assist these enterprises will eventually be repaid. +LJ021-0039|I believe it will be.|I believe it will be. +LJ021-0040|The second step we have taken in the restoration of normal business enterprise|The second step we have taken in the restoration of normal business enterprise +LJ021-0041|has been to clean up thoroughly unwholesome conditions in the field of investment.|has been to clean up thoroughly unwholesome conditions in the field of investment. +LJ021-0042|In this we have had assistance from many bankers and businessmen,|In this we have had assistance from many bankers and businessmen, +LJ021-0043|most of whom recognize the past evils in the banking system, in the sale of securities, in the deliberate encouragement of stock gambling,|most of whom recognize the past evils in the banking system, in the sale of securities, in the deliberate encouragement of stock gambling, +LJ021-0044|in the sale of unsound mortgages and in many other ways in which the public lost billions of dollars.|in the sale of unsound mortgages and in many other ways in which the public lost billions of dollars. +LJ021-0045|They saw that without changes in the policies and methods of investment|They saw that without changes in the policies and methods of investment +LJ021-0046|there could be no recovery of public confidence in the security of savings.|there could be no recovery of public confidence in the security of savings. +LJ021-0047|The country now enjoys the safety of bank savings under the new banking laws,|The country now enjoys the safety of bank savings under the new banking laws, +LJ021-0048|the careful checking of new securities under the Securities Act|the careful checking of new securities under the Securities Act +LJ021-0050|I sincerely hope that as a result|I sincerely hope that as a result +LJ021-0051|people will be discouraged in unhappy efforts to get rich quick by speculating in securities.|people will be discouraged in unhappy efforts to get rich quick by speculating in securities. +LJ021-0052|The average person almost always loses.|The average person almost always loses. +LJ021-0053|Only a very small minority of the people of this country believe in gambling as a substitute for the old philosophy of Benjamin Franklin|Only a very small minority of the people of this country believe in gambling as a substitute for the old philosophy of Benjamin Franklin +LJ021-0054|that the way to wealth is through work.|that the way to wealth is through work. +LJ021-0055|In meeting the problems of industrial recovery the chief agency of the government has been the National Recovery Administration.|In meeting the problems of industrial recovery the chief agency of the government has been the National Recovery Administration. +LJ021-0056|Under its guidance, trades and industries covering over 90 percent of all industrial employees|Under its guidance, trades and industries covering over ninety percent of all industrial employees +LJ021-0057|have adopted codes of fair competition, which have been approved by the President.|have adopted codes of fair competition, which have been approved by the President. +LJ021-0058|Under these codes, in the industries covered, child labor has been eliminated.|Under these codes, in the industries covered, child labor has been eliminated. +LJ021-0059|The work day and the work week have been shortened.|The work day and the work week have been shortened. +LJ021-0060|Minimum wages have been established and other wages adjusted toward a rising standard of living.|Minimum wages have been established and other wages adjusted toward a rising standard of living. +LJ021-0061|The emergency purpose of the N.R.A. was to put men to work and since its creation more than four million persons have been reemployed,|The emergency purpose of the N.R.A. was to put men to work and since its creation more than four million persons have been reemployed, +LJ021-0062|in great part through the cooperation of American business brought about under the codes.|in great part through the cooperation of American business brought about under the codes. +LJ021-0063|Benefits of the Industrial Recovery Program have come,|Benefits of the Industrial Recovery Program have come, +LJ021-0064|not only to labor in the form of new jobs, in relief from overwork and in relief from underpay,|not only to labor in the form of new jobs, in relief from overwork and in relief from underpay, +LJ021-0065|but also to the owners and managers of industry because,|but also to the owners and managers of industry because, +LJ021-0066|together with a great increase in the payrolls, there has come a substantial rise in the total of industrial profits|together with a great increase in the payrolls, there has come a substantial rise in the total of industrial profits +LJ021-0067|a rise from a deficit figure in the first quarter of 1933|a rise from a deficit figure in the first quarter of nineteen thirty-three +LJ021-0068|to a level of sustained profits within one year from the inauguration of N.R.A.|to a level of sustained profits within one year from the inauguration of N.R.A. +LJ021-0069|Now it should not be expected that even employed labor and capital would be completely satisfied with present conditions.|Now it should not be expected that even employed labor and capital would be completely satisfied with present conditions. +LJ021-0070|Employed workers have not by any means all enjoyed a return to the earnings of prosperous times,|Employed workers have not by any means all enjoyed a return to the earnings of prosperous times, +LJ021-0071|although millions of hitherto underprivileged workers are today far better paid than ever before.|although millions of hitherto underprivileged workers are today far better paid than ever before. +LJ021-0072|Also, billions of dollars of invested capital have today a greater security of present and future earning power than before.|Also, billions of dollars of invested capital have today a greater security of present and future earning power than before. +LJ021-0073|This is because of the establishment of fair, competitive standards and because of relief from unfair competition|This is because of the establishment of fair, competitive standards and because of relief from unfair competition +LJ021-0074|in wage cutting which depresses markets and destroys purchasing power.|in wage cutting which depresses markets and destroys purchasing power. +LJ021-0075|But it is an undeniable fact that the restoration of other billions of sound investments to a reasonable earning power|But it is an undeniable fact that the restoration of other billions of sound investments to a reasonable earning power +LJ021-0076|could not be brought about in one year.|could not be brought about in one year. +LJ021-0077|There is no magic formula,|There is no magic formula, +LJ021-0078|no economic panacea, which could simply revive over-night the heavy industries and the trades dependent upon them.|no economic panacea, which could simply revive over-night the heavy industries and the trades dependent upon them. +LJ021-0079|Nevertheless the gains of trade and industry, as a whole, have been substantial.|Nevertheless the gains of trade and industry, as a whole, have been substantial. +LJ021-0080|In these gains and in the policies of the administration there are assurances that hearten all forward- looking men and women|In these gains and in the policies of the administration there are assurances that hearten all forward- looking men and women +LJ021-0081|with the confidence that we are definitely rebuilding our political and economic system on the lines laid down by the New Deal|with the confidence that we are definitely rebuilding our political and economic system on the lines laid down by the New Deal +LJ021-0082|lines which as I have so often made clear, are in complete accord with the underlying principles of orderly popular government|lines which as I have so often made clear, are in complete accord with the underlying principles of orderly popular government +LJ021-0083|which Americans have demanded since the white man first came to these shores.|which Americans have demanded since the white man first came to these shores. +LJ021-0084|We count, in the future as in the past, on the driving power of individual initiative|We count, in the future as in the past, on the driving power of individual initiative +LJ021-0085|and the incentive of fair private profit,|and the incentive of fair private profit, +LJ021-0086|strengthened with the acceptance of those obligations to the public interest which rest upon us all.|strengthened with the acceptance of those obligations to the public interest which rest upon us all. +LJ021-0087|We have the right to expect that this driving power will be given patriotically and whole-heartedly to our nation.|We have the right to expect that this driving power will be given patriotically and whole-heartedly to our nation. +LJ021-0088|We have passed through the formative period of code making in the National Recovery Administration|We have passed through the formative period of code making in the National Recovery Administration +LJ021-0089|and have effected a reorganization of the N.R.A.|and have effected a reorganization of the N.R.A. +LJ021-0090|suited to the needs of the next phase, which is, in turn, a period of preparation for legislation which will determine its permanent form.|suited to the needs of the next phase, which is, in turn, a period of preparation for legislation which will determine its permanent form. +LJ021-0091|In this recent reorganization we have recognized three distinct functions:|In this recent reorganization we have recognized three distinct functions: +LJ021-0092|first, the legislative or policy making function;|first, the legislative or policy making function; +LJ021-0093|second, the administrative function of code making and revision; and, third, the judicial function, which includes enforcement,|second, the administrative function of code making and revision; and, third, the judicial function, which includes enforcement, +LJ021-0094|consumer complaints and the settlement of disputes between employers and employees and between one employer and another.|consumer complaints and the settlement of disputes between employers and employees and between one employer and another. +LJ021-0095|We are now prepared to move into this second phase, on the basis of our experience in the first phase|We are now prepared to move into this second phase, on the basis of our experience in the first phase +LJ021-0096|under the able and energetic leadership of General Johnson.|under the able and energetic leadership of General Johnson. +LJ021-0097|we shall watch carefully the working of this new machinery for the second phase of N.R.A.,|we shall watch carefully the working of this new machinery for the second phase of N.R.A., +LJ021-0098|modifying it where it needs modification and finally making recommendations to the Congress,|modifying it where it needs modification and finally making recommendations to the Congress, +LJ021-0099|in order that the functions of N.R.A. which have proved their worth may be made a part of the permanent machinery of government.|in order that the functions of N.R.A. which have proved their worth may be made a part of the permanent machinery of government. +LJ021-0100|Let me call your attention to the fact that the national Industrial Recovery Act|Let me call your attention to the fact that the national Industrial Recovery Act +LJ021-0101|gave businessmen the opportunity they had sought for years to improve business conditions through what has been called self-government in industry.|gave businessmen the opportunity they had sought for years to improve business conditions through what has been called self-government in industry. +LJ021-0102|If the codes which have been written have been too complicated,|If the codes which have been written have been too complicated, +LJ021-0103|if they have gone too far in such matters as price fixing and limitation of production,|if they have gone too far in such matters as price fixing and limitation of production, +LJ021-0104|let it be remembered that so far as possible, consistent with the immediate public interest of this past year|let it be remembered that so far as possible, consistent with the immediate public interest of this past year +LJ021-0105|and the vital necessity of improving labor conditions,|and the vital necessity of improving labor conditions, +LJ021-0106|the representatives of trade and industry were permitted to write their ideas into the codes.|the representatives of trade and industry were permitted to write their ideas into the codes. +LJ021-0107|It is now time to review these actions as a whole to determine through deliberative means in the light of experience,|It is now time to review these actions as a whole to determine through deliberative means in the light of experience, +LJ021-0108|from the standpoint of the good of the industries themselves, as well as the general public interest,|from the standpoint of the good of the industries themselves, as well as the general public interest, +LJ021-0109|whether the methods and policies adopted in the emergency|whether the methods and policies adopted in the emergency +LJ021-0110|have been best calculated to promote industrial recovery and a permanent improvement of business and labor conditions.|have been best calculated to promote industrial recovery and a permanent improvement of business and labor conditions. +LJ021-0111|There may be a serious question as to the wisdom of many of those devices to control production,|There may be a serious question as to the wisdom of many of those devices to control production, +LJ021-0112|or to prevent destructive price cutting which many business organizations have insisted were necessary,|or to prevent destructive price cutting which many business organizations have insisted were necessary, +LJ021-0113|or whether their effect may have been to prevent that volume of production which would make possible lower prices and increased employment.|or whether their effect may have been to prevent that volume of production which would make possible lower prices and increased employment. +LJ021-0114|Another question arises as to whether in fixing minimum wages on the basis of an hourly or weekly wage|Another question arises as to whether in fixing minimum wages on the basis of an hourly or weekly wage +LJ021-0115|we have reached into the heart of the problem which is to provide such annual earnings for the lowest paid worker as will meet his minimum needs.|we have reached into the heart of the problem which is to provide such annual earnings for the lowest paid worker as will meet his minimum needs. +LJ021-0116|We also question the wisdom of extending code requirements suited to the great industrial centers and to large employers,|We also question the wisdom of extending code requirements suited to the great industrial centers and to large employers, +LJ021-0117|to the great number of small employers in the smaller communities.|to the great number of small employers in the smaller communities. +LJ021-0118|During the last twelve months our industrial recovery has been to some extent retarded by strikes,|During the last twelve months our industrial recovery has been to some extent retarded by strikes, +LJ021-0119|including a few of major importance.|including a few of major importance. +LJ021-0120|I would not minimize the inevitable losses to employers and employees and to the general public through such conflicts.|I would not minimize the inevitable losses to employers and employees and to the general public through such conflicts. +LJ021-0121|But I would point out that the extent and severity of labor disputes during this period|But I would point out that the extent and severity of labor disputes during this period +LJ021-0122|has been far less than in any previous, comparable period.|has been far less than in any previous, comparable period. +LJ021-0123|When the businessmen of the country were demanding the right to organize themselves adequately to promote their legitimate interests;|When the businessmen of the country were demanding the right to organize themselves adequately to promote their legitimate interests; +LJ021-0124|when the farmers were demanding legislation which would give them opportunities and incentives to organize themselves for a common advance,|when the farmers were demanding legislation which would give them opportunities and incentives to organize themselves for a common advance, +LJ021-0125|it was natural that the workers should seek and obtain a statutory declaration of their constitutional right|it was natural that the workers should seek and obtain a statutory declaration of their constitutional right +LJ021-0126|to organize themselves for collective bargaining as embodied in Section 7 (a) of the national Industrial Recovery Act.|to organize themselves for collective bargaining as embodied in Section seven (a) of the national Industrial Recovery Act. +LJ021-0127|Machinery set up by the federal government has provided some new methods of adjustment.|Machinery set up by the federal government has provided some new methods of adjustment. +LJ021-0128|Both employers and employees must share the blame of not using them as fully as they should.|Both employers and employees must share the blame of not using them as fully as they should. +LJ021-0129|The employer who turns away from impartial agencies of peace,|The employer who turns away from impartial agencies of peace, +LJ021-0130|who denies freedom of organization to his employees, or fails to make every reasonable effort at a peaceful solution of their differences,|who denies freedom of organization to his employees, or fails to make every reasonable effort at a peaceful solution of their differences, +LJ021-0131|is not fully supporting the recovery effort of his government.|is not fully supporting the recovery effort of his government. +LJ021-0132|The workers who turn away from these same impartial agencies and decline to use their good offices to gain their ends|The workers who turn away from these same impartial agencies and decline to use their good offices to gain their ends +LJ021-0133|are likewise not fully cooperating with their government.|are likewise not fully cooperating with their government. +LJ021-0134|It is time that we made a clean-cut effort to bring about that united action of management and labor,|It is time that we made a clean-cut effort to bring about that united action of management and labor, +LJ021-0135|which is one of the high purposes of the Recovery Act.|which is one of the high purposes of the Recovery Act. +LJ021-0136|We have passed through more than a year of education.|We have passed through more than a year of education. +LJ021-0137|Step by step we have created all the government agencies necessary to insure, as a general rule, industrial peace,|Step by step we have created all the government agencies necessary to insure, as a general rule, industrial peace, +LJ021-0138|with justice for all those willing to use these agencies whenever their voluntary bargaining fails to produce a necessary agreement.|with justice for all those willing to use these agencies whenever their voluntary bargaining fails to produce a necessary agreement. +LJ021-0139|There should be at least a full and fair trial given to these means of ending industrial warfare;|There should be at least a full and fair trial given to these means of ending industrial warfare; +LJ021-0140|and in such an effort we should be able to secure for employers and employees and consumers|and in such an effort we should be able to secure for employers and employees and consumers +LJ021-0141|the benefits that all derive from the continuous, peaceful operation of our essential enterprises.|the benefits that all derive from the continuous, peaceful operation of our essential enterprises. +LJ021-0142|Accordingly, I propose to confer within the coming month|Accordingly, I propose to confer within the coming month +LJ021-0143|with small groups of those truly representative of large employers of labor and of large groups of organized labor,|with small groups of those truly representative of large employers of labor and of large groups of organized labor, +LJ021-0144|in order to seek their cooperation in establishing what I may describe as a specific trial period of industrial peace.|in order to seek their cooperation in establishing what I may describe as a specific trial period of industrial peace. +LJ021-0145|From those willing to join in establishing this hoped-for period of peace,|From those willing to join in establishing this hoped-for period of peace, +LJ021-0146|I shall seek assurances of the making and maintenance of agreements, which can be mutually relied upon,|I shall seek assurances of the making and maintenance of agreements, which can be mutually relied upon, +LJ021-0147|under which wages, hours and working conditions|under which wages, hours and working conditions +LJ021-0148|may be determined and any later adjustments shall be made either by agreement or, in case of disagreement,|may be determined and any later adjustments shall be made either by agreement or, in case of disagreement, +LJ021-0149|through the mediation or arbitration of state or federal agencies.|through the mediation or arbitration of state or federal agencies. +LJ021-0150|I shall not ask either employers or employees permanently to lay aside the weapons common to industrial war.|I shall not ask either employers or employees permanently to lay aside the weapons common to industrial war. +LJ021-0151|But I shall ask both groups to give a fair trial to peaceful methods of adjusting their conflicts of opinion and interest,|But I shall ask both groups to give a fair trial to peaceful methods of adjusting their conflicts of opinion and interest, +LJ021-0152|and to experiment for a reasonable time with measures suitable to civilize our industrial civilization.|and to experiment for a reasonable time with measures suitable to civilize our industrial civilization. +LJ021-0153|Closely allied to the N.R.A.|Closely allied to the N.R.A. +LJ021-0154|is the program of Public Works provided for in the same Act and designed to put more men back to work,|is the program of Public Works provided for in the same Act and designed to put more men back to work, +LJ021-0156|To those who say that our expenditures for public works and other means for recovery are a waste that we cannot afford,|To those who say that our expenditures for public works and other means for recovery are a waste that we cannot afford, +LJ021-0157|I answer that no country, however rich, can afford the waste of its human resources.|I answer that no country, however rich, can afford the waste of its human resources. +LJ021-0158|Demoralization caused by vast unemployment is our greatest extravagance.|Demoralization caused by vast unemployment is our greatest extravagance. +LJ021-0159|Morally, it is the greatest menace to our social order.|Morally, it is the greatest menace to our social order. +LJ021-0160|Some people try to tell me that we must make up our minds that for the future we shall permanently have millions of unemployed|Some people try to tell me that we must make up our minds that for the future we shall permanently have millions of unemployed +LJ021-0161|just as other countries have had them for over a decade.|just as other countries have had them for over a decade. +LJ021-0162|What may be necessary for those countries is not my responsibility to determine.|What may be necessary for those countries is not my responsibility to determine. +LJ021-0163|But as for this country, I stand or fall by my refusal to accept as a necessary condition of our future|But as for this country, I stand or fall by my refusal to accept as a necessary condition of our future +LJ021-0164|a permanent army of unemployed.|a permanent army of unemployed. +LJ021-0165|On the contrary,|On the contrary, +LJ021-0166|we must make it a national principle that we will not tolerate a large army of unemployed and that we will arrange our national economy|we must make it a national principle that we will not tolerate a large army of unemployed and that we will arrange our national economy +LJ021-0167|to end our present unemployment as soon as we can and then to take wise measures against its return.|to end our present unemployment as soon as we can and then to take wise measures against its return. +LJ021-0168|I do not want to think that it is the destiny of any American to remain permanently on relief rolls.|I do not want to think that it is the destiny of any American to remain permanently on relief rolls. +LJ021-0169|Those, fortunately few in number, who are frightened by boldness and cowed by the necessity for making decisions,|Those, fortunately few in number, who are frightened by boldness and cowed by the necessity for making decisions, +LJ021-0170|complain that all we have done is unnecessary and subject to great risks.|complain that all we have done is unnecessary and subject to great risks. +LJ021-0171|Now that these people are coming out of their storm cellars, they forget that there ever was a storm.|Now that these people are coming out of their storm cellars, they forget that there ever was a storm. +LJ021-0172|They point to England.|They point to England. +LJ021-0173|They would have you believe that England has made progress out of her depression by a do-nothing policy, by letting nature take her course.|They would have you believe that England has made progress out of her depression by a do-nothing policy, by letting nature take her course. +LJ021-0174|England has her peculiarities and we have ours|England has her peculiarities and we have ours +LJ021-0175|but I do not believe any intelligent observer can accuse England of undue orthodoxy in the present emergency.|but I do not believe any intelligent observer can accuse England of undue orthodoxy in the present emergency. +LJ021-0176|Did England let nature take her course? No.|Did England let nature take her course? No. +LJ021-0177|Did England hold to the gold standard when her reserves were threatened?|Did England hold to the gold standard when her reserves were threatened? +LJ021-0178|Has England gone back to the gold standard today?|Has England gone back to the gold standard today? +LJ021-0179|Did England hesitate to call in ten billion dollars of her war bonds bearing 5 percent interest,|Did England hesitate to call in ten billion dollars of her war bonds bearing five percent interest, +LJ021-0180|to issue new bonds therefore bearing only 3 and one half percent interest,|to issue new bonds therefore bearing only three and one half percent interest, +LJ021-0181|thereby saving the British treasury one hundred and fifty million dollars a year in interest alone?|thereby saving the British treasury one hundred and fifty million dollars a year in interest alone? +LJ021-0182|And let it be recorded that the British bankers helped.|And let it be recorded that the British bankers helped. +LJ021-0183|Is it not a fact that ever since the year 1909,|Is it not a fact that ever since the year nineteen oh nine, +LJ021-0184|Great Britain in many ways has advanced further along lines of social security than the United States?|Great Britain in many ways has advanced further along lines of social security than the United States? +LJ021-0185|Is it not a fact that relations between capital and labor on the basis of collective bargaining are much further advanced in Great Britain|Is it not a fact that relations between capital and labor on the basis of collective bargaining are much further advanced in Great Britain +LJ021-0186|than in the United States?|than in the United States? +LJ021-0187|It is perhaps not strange that the conservative British press has told us with pardonable irony|It is perhaps not strange that the conservative British press has told us with pardonable irony +LJ021-0188|that much of our New Deal program is only an attempt to catch up with English reforms that go back ten years or more.|that much of our New Deal program is only an attempt to catch up with English reforms that go back ten years or more. +LJ021-0189|Nearly all Americans are sensible and calm people.|Nearly all Americans are sensible and calm people. +LJ021-0190|We do not get greatly excited nor is our peace of mind disturbed, whether we be businessmen or workers or farmers,|We do not get greatly excited nor is our peace of mind disturbed, whether we be businessmen or workers or farmers, +LJ021-0191|by awesome pronouncements concerning the unconstitutionality of some of our measures of recovery and relief and reform.|by awesome pronouncements concerning the unconstitutionality of some of our measures of recovery and relief and reform. +LJ021-0192|We are not frightened by reactionary lawyers or political editors.|We are not frightened by reactionary lawyers or political editors. +LJ021-0193|All of these cries have been heard before.|All of these cries have been heard before. +LJ021-0194|More than twenty years ago, when Theodore Roosevelt and Woodrow Wilson were attempting to correct abuses in our national life,|More than twenty years ago, when Theodore Roosevelt and Woodrow Wilson were attempting to correct abuses in our national life, +LJ021-0195|the great Chief Justice White said:|the great Chief Justice White said: +LJ021-0196|There is great danger it seems to me to arise from the constant habit which prevails where anything is opposed or objected to,|There is great danger it seems to me to arise from the constant habit which prevails where anything is opposed or objected to, +LJ021-0197|of referring without rhyme or reason to the Constitution as a means of preventing its accomplishment, thus creating the general impression|of referring without rhyme or reason to the Constitution as a means of preventing its accomplishment, thus creating the general impression +LJ021-0198|that the Constitution is but a barrier to progress instead of being the broad highway through which alone true progress may be enjoyed.|that the Constitution is but a barrier to progress instead of being the broad highway through which alone true progress may be enjoyed. +LJ021-0199|In our efforts for recovery|In our efforts for recovery +LJ021-0200|we have avoided on the one hand the theory that business should and must be taken over into an all-embracing government.|we have avoided on the one hand the theory that business should and must be taken over into an all-embracing government. +LJ021-0201|We have avoided on the other hand|We have avoided on the other hand +LJ021-0202|the equally untenable theory that it is an interference with liberty to offer reasonable help when private enterprise is in need of help.|the equally untenable theory that it is an interference with liberty to offer reasonable help when private enterprise is in need of help. +LJ021-0203|The course we have followed fits the American practice of government -- a practice of taking action step by step,|The course we have followed fits the American practice of government -- a practice of taking action step by step, +LJ021-0204|of regulating only to meet concrete needs -- a practice of courageous recognition of change.|of regulating only to meet concrete needs -- a practice of courageous recognition of change. +LJ021-0205|I believe with Abraham Lincoln, that "The legitimate object of government is to do for a community of people|I believe with Abraham Lincoln, that "The legitimate object of government is to do for a community of people +LJ021-0206|whatever they need to have done but cannot do at all or cannot do so well for themselves in their separate and individual capacities."|whatever they need to have done but cannot do at all or cannot do so well for themselves in their separate and individual capacities." +LJ021-0207|I am not for a return to that definition of liberty under which for many years a free people were being gradually regimented|I am not for a return to that definition of liberty under which for many years a free people were being gradually regimented +LJ021-0208|into the service of the privileged few.|into the service of the privileged few. +LJ021-0209|I prefer and I am sure you prefer that broader definition of liberty under which we are moving forward to greater freedom,|I prefer and I am sure you prefer that broader definition of liberty under which we are moving forward to greater freedom, +LJ021-0210|to greater security for the average man than he has ever known before in the history of America.|to greater security for the average man than he has ever known before in the history of America. +LJ022-0001|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt, Section 7.|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt, Section seven. +LJ022-0002|April 28, 1935.|April twenty-eight, nineteen thirty-five. +LJ022-0003|Since my annual message to the Congress on January fourth, last, I have not addressed the general public over the air.|Since my annual message to the Congress on January fourth, last, I have not addressed the general public over the air. +LJ022-0004|In the many weeks since that time the Congress has devoted itself to the arduous task of formulating legislation necessary to the country's welfare.|In the many weeks since that time the Congress has devoted itself to the arduous task of formulating legislation necessary to the country's welfare. +LJ022-0005|It has made and is making distinct progress.|It has made and is making distinct progress. +LJ022-0006|Before I come to any of the specific measures, however, I want to leave in your minds one clear fact.|Before I come to any of the specific measures, however, I want to leave in your minds one clear fact. +LJ022-0007|The administration and the Congress are not proceeding in any haphazard fashion in this task of government.|The administration and the Congress are not proceeding in any haphazard fashion in this task of government. +LJ022-0008|Each of our steps has a definite relationship to every other step.|Each of our steps has a definite relationship to every other step. +LJ022-0009|The job of creating a program for the nation's welfare is, in some respects, like the building of a ship.|The job of creating a program for the nation's welfare is, in some respects, like the building of a ship. +LJ022-0010|At different points on the coast where I often visit they build great seagoing ships.|At different points on the coast where I often visit they build great seagoing ships. +LJ022-0011|When one of these ships is under construction and the steel frames have been set in the keel,|When one of these ships is under construction and the steel frames have been set in the keel, +LJ022-0012|it is difficult for a person who does not know ships to tell how it will finally look when it is sailing the high seas.|it is difficult for a person who does not know ships to tell how it will finally look when it is sailing the high seas. +LJ022-0013|It may seem confused to some, but out of the multitude of detailed parts that go into the making of the structure|It may seem confused to some, but out of the multitude of detailed parts that go into the making of the structure +LJ022-0014|the creation of a useful instrument for man ultimately comes.|the creation of a useful instrument for man ultimately comes. +LJ022-0015|It is that way with the making of a national policy. The objective of the nation has greatly changed in three years|It is that way with the making of a national policy. The objective of the nation has greatly changed in three years +LJ022-0016|Before that time individual self- interest and group selfishness were paramount in public thinking.|Before that time individual self- interest and group selfishness were paramount in public thinking. +LJ022-0017|The general good was at a discount.|The general good was at a discount. +LJ022-0018|Three years of hard thinking have changed the picture.|Three years of hard thinking have changed the picture. +LJ022-0019|More and more people,|More and more people, +LJ022-0020|cause of clearer thinking and a better understanding, are considering the whole rather than a mere part relating to one section or to one crop,|cause of clearer thinking and a better understanding, are considering the whole rather than a mere part relating to one section or to one crop, +LJ022-0021|or to one industry, or to an individual private occupation.|or to one industry, or to an individual private occupation. +LJ022-0022|That is a tremendous gain for the principles of democracy.|That is a tremendous gain for the principles of democracy. +LJ022-0023|The overwhelming majority of people in this country know how to sift the wheat from the chaff in what they hear and what they read.|The overwhelming majority of people in this country know how to sift the wheat from the chaff in what they hear and what they read. +LJ022-0024|They know that the process of the constructive rebuilding of America cannot be done in a day or a year,|They know that the process of the constructive rebuilding of America cannot be done in a day or a year, +LJ022-0025|but that it is being done in spite of the few who seek to confuse them and to profit by their confusion.|but that it is being done in spite of the few who seek to confuse them and to profit by their confusion. +LJ022-0026|Americans as a whole are feeling a lot better -- a lot more cheerful than for many, many years.|Americans as a whole are feeling a lot better -- a lot more cheerful than for many, many years. +LJ022-0027|The most difficult place in the world to get a clear open perspective of the country as a whole is Washington.|The most difficult place in the world to get a clear open perspective of the country as a whole is Washington. +LJ022-0028|I am reminded sometimes of what President Wilson once said:|I am reminded sometimes of what President Wilson once said: +LJ022-0029|So many people come to Washington who know things that are not so,|So many people come to Washington who know things that are not so, +LJ022-0030|and so few people who know anything about what the people of the United States are thinking about.|and so few people who know anything about what the people of the United States are thinking about. +LJ022-0031|That is why I occasionally leave this scene of action for a few days|That is why I occasionally leave this scene of action for a few days +LJ022-0032|to go fishing or back home to Hyde Park, so that I can have a chance to think quietly about the country as a whole.|to go fishing or back home to Hyde Park, so that I can have a chance to think quietly about the country as a whole. +LJ022-0033|"To get away from the trees", as they say, "and to look at the whole forest."|"To get away from the trees", as they say, "and to look at the whole forest." +LJ022-0034|This duty of seeing the country in a long-range perspective|This duty of seeing the country in a long-range perspective +LJ022-0035|is one which, in a very special manner, attaches to this office to which you have chosen me.|is one which, in a very special manner, attaches to this office to which you have chosen me. +LJ022-0036|Did you ever stop to think that there are, after all, only two positions in the nation|Did you ever stop to think that there are, after all, only two positions in the nation +LJ022-0037|that are filled by the vote of all of the voters -- the President and the Vice-President?|that are filled by the vote of all of the voters -- the President and the Vice-President? +LJ022-0038|That makes it particularly necessary for the Vice- President and for me to conceive of our duty toward the entire country.|That makes it particularly necessary for the Vice- President and for me to conceive of our duty toward the entire country. +LJ022-0039|I speak, therefore, tonight, to and of the American people as a whole.|I speak, therefore, tonight, to and of the American people as a whole. +LJ022-0040|My most immediate concern is in carrying out the purposes of the great work program just enacted by the Congress.|My most immediate concern is in carrying out the purposes of the great work program just enacted by the Congress. +LJ022-0041|Its first objective is to put men and women now on the relief rolls to work and, incidentally,|Its first objective is to put men and women now on the relief rolls to work and, incidentally, +LJ022-0042|to assist materially in our already unmistakable march toward recovery.|to assist materially in our already unmistakable march toward recovery. +LJ022-0043|I shall not confuse my discussion by a multitude of figures. So many figures are quoted to prove so many things.|I shall not confuse my discussion by a multitude of figures. So many figures are quoted to prove so many things. +LJ022-0044|Sometimes it depends upon what paper you read and what broadcast you hear.|Sometimes it depends upon what paper you read and what broadcast you hear. +LJ022-0045|Therefore, let us keep our minds on two or three simple, essential facts in connection with this problem of unemployment.|Therefore, let us keep our minds on two or three simple, essential facts in connection with this problem of unemployment. +LJ022-0047|However, for the first time in five years the relief rolls have declined instead of increased during the winter months.|However, for the first time in five years the relief rolls have declined instead of increased during the winter months. +LJ022-0048|They are still declining.|They are still declining. +LJ022-0049|The simple fact is that many million more people have private work today than two years ago today or one year ago today,|The simple fact is that many million more people have private work today than two years ago today or one year ago today, +LJ022-0050|and every day that passes offers more chances to work for those who want to work.|and every day that passes offers more chances to work for those who want to work. +LJ022-0051|In spite of the fact that unemployment remains a serious problem|In spite of the fact that unemployment remains a serious problem +LJ022-0052|here as in every other nation, we have come to recognize the possibility and the necessity of certain helpful remedial measures.|here as in every other nation, we have come to recognize the possibility and the necessity of certain helpful remedial measures. +LJ022-0053|These measures are of two kinds.|These measures are of two kinds. +LJ022-0054|The first is to make provisions intended to relieve, to minimize, and to prevent future unemployment;|The first is to make provisions intended to relieve, to minimize, and to prevent future unemployment; +LJ022-0055|the second is to establish the practical means to help those who are unemployed in this present emergency.|the second is to establish the practical means to help those who are unemployed in this present emergency. +LJ022-0056|Our social security legislation is an attempt to answer the first of these questions; our Works Relief program, the second.|Our social security legislation is an attempt to answer the first of these questions; our Works Relief program, the second. +LJ022-0057|The program for social security now pending before the Congress is a necessary part of the future unemployment policy of the government.|The program for social security now pending before the Congress is a necessary part of the future unemployment policy of the government. +LJ022-0058|While our present and projected expenditures for work relief are wholly within the reasonable limits of our national credit resources,|While our present and projected expenditures for work relief are wholly within the reasonable limits of our national credit resources, +LJ022-0059|it is obvious that we cannot continue to create governmental deficits for that purpose year after year.|it is obvious that we cannot continue to create governmental deficits for that purpose year after year. +LJ022-0060|We must begin now to make provision for the future.|We must begin now to make provision for the future. +LJ022-0061|That is why our social security program is an important part of the complete picture.|That is why our social security program is an important part of the complete picture. +LJ022-0062|It proposes, by means of old age pensions,|It proposes, by means of old age pensions, +LJ022-0063|to help those who have reached the age of retirement to give up their jobs and thus give to the younger generation greater opportunities for work|to help those who have reached the age of retirement to give up their jobs and thus give to the younger generation greater opportunities for work +LJ022-0064|and to give to all a feeling of security as they look toward old age.|and to give to all a feeling of security as they look toward old age. +LJ022-0065|The unemployment insurance part of the legislation|The unemployment insurance part of the legislation +LJ022-0066|will not only help to guard the individual in future periods of lay-off against dependence upon relief,|will not only help to guard the individual in future periods of lay-off against dependence upon relief, +LJ022-0067|but it will, by sustaining purchasing power, cushion the shock of economic distress.|but it will, by sustaining purchasing power, cushion the shock of economic distress. +LJ022-0068|Another helpful feature of unemployment insurance is the incentive it will give to employers to plan more carefully|Another helpful feature of unemployment insurance is the incentive it will give to employers to plan more carefully +LJ022-0069|in order that unemployment may be prevented by the stabilizing of employment itself.|in order that unemployment may be prevented by the stabilizing of employment itself. +LJ022-0070|Provisions for social security, however, are protections for the future.|Provisions for social security, however, are protections for the future. +LJ022-0071|Our responsibility for the immediate necessities of the unemployed has been met by the Congress|Our responsibility for the immediate necessities of the unemployed has been met by the Congress +LJ022-0072|through the most comprehensive work plan in the history of the nation.|through the most comprehensive work plan in the history of the nation. +LJ022-0073|Our problem is to put to work three and one-half million employable persons now on the relief rolls.|Our problem is to put to work three and one-half million employable persons now on the relief rolls. +LJ022-0074|It is a problem quite as much for private industry as for the government.|It is a problem quite as much for private industry as for the government. +LJ022-0075|We are losing no time getting the government's vast work relief program underway,|We are losing no time getting the government's vast work relief program underway, +LJ022-0076|and we have every reason to believe that it should be in full swing by autumn.|and we have every reason to believe that it should be in full swing by autumn. +LJ022-0077|In directing it, I shall recognize six fundamental principles:|In directing it, I shall recognize six fundamental principles: +LJ022-0078|1. The projects should be useful.|one. The projects should be useful. +LJ022-0079|Projects shall be of a nature that a considerable proportion of the money spent will go into wages for labor.|Projects shall be of a nature that a considerable proportion of the money spent will go into wages for labor. +LJ022-0080|Projects will be sought which promise ultimate return to the federal treasury of a considerable proportion of the costs.|Projects will be sought which promise ultimate return to the federal treasury of a considerable proportion of the costs. +LJ022-0081|Funds allotted for each project should be actually and promptly spent and not held over until later years.|Funds allotted for each project should be actually and promptly spent and not held over until later years. +LJ022-0082|In all cases projects must be of a character to give employment to those on the relief rolls.|In all cases projects must be of a character to give employment to those on the relief rolls. +LJ022-0083|6. Projects will be allocated to localities or relief areas in relation to the number of workers on relief rolls in those areas.|six. Projects will be allocated to localities or relief areas in relation to the number of workers on relief rolls in those areas. +LJ022-0084|I next want to make it clear exactly how we shall direct the work.|I next want to make it clear exactly how we shall direct the work. +LJ022-0085|I have set up a Division of Applications and Information|I have set up a Division of Applications and Information +LJ022-0086|to which all proposals for the expenditure of money must go for preliminary study and consideration.|to which all proposals for the expenditure of money must go for preliminary study and consideration. +LJ022-0087|After the Division of Applications and Information has sifted those projects,|After the Division of Applications and Information has sifted those projects, +LJ022-0088|they will be sent to an Allotment Division composed of representatives of the more important governmental agencies|they will be sent to an Allotment Division composed of representatives of the more important governmental agencies +LJ022-0089|charged with carrying on work relief projects.|charged with carrying on work relief projects. +LJ022-0090|The group will also include representatives of cities, and of labor, farming, banking and industry.|The group will also include representatives of cities, and of labor, farming, banking and industry. +LJ022-0091|This Allotment Division will consider all of the recommendations submitted to it|This Allotment Division will consider all of the recommendations submitted to it +LJ022-0092|and such projects as they approve will be next submitted to the President who under the Act is required to make final allocations.|and such projects as they approve will be next submitted to the President who under the Act is required to make final allocations. +LJ022-0093|The next step will be to notify the proper government agency|The next step will be to notify the proper government agency +LJ022-0094|in whose field the project falls, and also to notify another agency which I am creating -- a Progress Division.|in whose field the project falls, and also to notify another agency which I am creating -- a Progress Division. +LJ022-0095|This Division will have the duty of coordinating the purchases of materials and supplies|This Division will have the duty of coordinating the purchases of materials and supplies +LJ022-0096|and of making certain that people who are employed will be taken from the relief rolls.|and of making certain that people who are employed will be taken from the relief rolls. +LJ022-0097|It will also have the responsibility of determining work payments in various localities,|It will also have the responsibility of determining work payments in various localities, +LJ022-0098|of making full use of existing employment services and to assist people engaged in relief work|of making full use of existing employment services and to assist people engaged in relief work +LJ022-0099|to move as rapidly as possible back into private employment when such employment is available.|to move as rapidly as possible back into private employment when such employment is available. +LJ022-0100|Moreover, this Division will be charged with keeping projects moving on schedule.|Moreover, this Division will be charged with keeping projects moving on schedule. +LJ022-0101|I have felt it to be essentially wise and prudent to avoid, so far as possible,|I have felt it to be essentially wise and prudent to avoid, so far as possible, +LJ022-0102|the creation of new governmental machinery for supervising this work.|the creation of new governmental machinery for supervising this work. +LJ022-0103|The national government now has at least sixty different agencies with the staff|The national government now has at least sixty different agencies with the staff +LJ022-0104|and the experience and the competence necessary to carry on the two hundred and fifty or three hundred kinds of work that will be undertaken.|and the experience and the competence necessary to carry on the two hundred and fifty or three hundred kinds of work that will be undertaken. +LJ022-0105|These agencies, therefore, will simply be doing on a somewhat enlarged scale the same sort of things that they have been doing.|These agencies, therefore, will simply be doing on a somewhat enlarged scale the same sort of things that they have been doing. +LJ022-0106|This will make certain that the largest possible portion of the funds allotted|This will make certain that the largest possible portion of the funds allotted +LJ022-0107|will be spent for actually creating new work and not for building up expensive overhead organizations here in Washington.|will be spent for actually creating new work and not for building up expensive overhead organizations here in Washington. +LJ022-0108|For many months preparations have been under way.|For many months preparations have been under way. +LJ022-0109|The allotment of funds for desirable projects has already begun.|The allotment of funds for desirable projects has already begun. +LJ022-0110|The key men for the major responsibilities of this great task already have been selected.|The key men for the major responsibilities of this great task already have been selected. +LJ022-0111|I well realize that the country is expecting before this year is out to see the "dirt fly", as they say, in carrying on the work,|I well realize that the country is expecting before this year is out to see the "dirt fly", as they say, in carrying on the work, +LJ022-0112|and I assure my fellow citizens that no energy will be spared in using these funds effectively|and I assure my fellow citizens that no energy will be spared in using these funds effectively +LJ022-0113|to make a major attack upon the problem of unemployment.|to make a major attack upon the problem of unemployment. +LJ022-0114|Our responsibility is to all of the people in this country.|Our responsibility is to all of the people in this country. +LJ022-0115|This is a great national crusade to destroy enforced idleness which is an enemy of the human spirit|This is a great national crusade to destroy enforced idleness which is an enemy of the human spirit +LJ022-0116|generated by this depression|generated by this depression +LJ022-0117|Our attack upon these enemies must be without stint and without discrimination.|Our attack upon these enemies must be without stint and without discrimination. +LJ022-0118|No sectional, no political distinctions can be permitted.|No sectional, no political distinctions can be permitted. +LJ022-0119|It must, however, be recognized that when an enterprise of this character is extended over more than three thousand counties throughout the nation,|It must, however, be recognized that when an enterprise of this character is extended over more than three thousand counties throughout the nation, +LJ022-0120|there may be occasional instances of inefficiency, bad management, or misuse of funds.|there may be occasional instances of inefficiency, bad management, or misuse of funds. +LJ022-0121|When cases of this kind occur,|When cases of this kind occur, +LJ022-0122|there will be those, of course, who will try to tell you that the exceptional failure is characteristic of the entire endeavor.|there will be those, of course, who will try to tell you that the exceptional failure is characteristic of the entire endeavor. +LJ022-0123|It should be remembered that in every big job there are some imperfections.|It should be remembered that in every big job there are some imperfections. +LJ022-0124|There are chiselers in every walk of life; there are those in every industry who are guilty of unfair practices;|There are chiselers in every walk of life; there are those in every industry who are guilty of unfair practices; +LJ022-0125|every profession has its black sheep, but long experience in government has taught me|every profession has its black sheep, but long experience in government has taught me +LJ022-0126|that the exceptional instances of wrong-doing in government are probably less numerous than in almost every other line of endeavor.|that the exceptional instances of wrong-doing in government are probably less numerous than in almost every other line of endeavor. +LJ022-0127|The most effective means of preventing such evils in this Works Relief program will be the eternal vigilance of the American people themselves.|The most effective means of preventing such evils in this Works Relief program will be the eternal vigilance of the American people themselves. +LJ022-0128|I call upon my fellow citizens everywhere to cooperate with me|I call upon my fellow citizens everywhere to cooperate with me +LJ022-0129|in making this the most efficient and the cleanest example of public enterprise the world has ever seen.|in making this the most efficient and the cleanest example of public enterprise the world has ever seen. +LJ022-0130|It is time to provide a smashing answer for those cynical men who say that a democracy cannot be honest and efficient.|It is time to provide a smashing answer for those cynical men who say that a democracy cannot be honest and efficient. +LJ022-0131|If you will help, this can be done.|If you will help, this can be done. +LJ022-0132|I, therefore, hope you will watch the work in every corner of this Nation.|I, therefore, hope you will watch the work in every corner of this Nation. +LJ022-0133|Feel free to criticize. Tell me of instances where work can be done better, or where improper practices prevail.|Feel free to criticize. Tell me of instances where work can be done better, or where improper practices prevail. +LJ022-0134|Neither you nor I want criticism conceived in a purely fault-finding or partisan spirit,|Neither you nor I want criticism conceived in a purely fault-finding or partisan spirit, +LJ022-0135|but I am jealous of the right of every citizen to call to the attention of his or her government|but I am jealous of the right of every citizen to call to the attention of his or her government +LJ022-0136|examples of how the public money can be more effectively spent for the benefit of the American people.|examples of how the public money can be more effectively spent for the benefit of the American people. +LJ022-0137|I now come, my friends, to a part of the remaining business before the Congress.|I now come, my friends, to a part of the remaining business before the Congress. +LJ022-0138|It has under consideration many measures which provide for the rounding out of the program of economic and social reconstruction|It has under consideration many measures which provide for the rounding out of the program of economic and social reconstruction +LJ022-0139|with which we have been concerned for two years.|with which we have been concerned for two years. +LJ022-0140|I can mention only a few of them tonight, but I do not want my mention of specific measures|I can mention only a few of them tonight, but I do not want my mention of specific measures +LJ022-0141|to be interpreted as lack of interest in or disapproval of many other important proposals that are pending.|to be interpreted as lack of interest in or disapproval of many other important proposals that are pending. +LJ022-0142|The National Industrial Recovery Act expires on the sixteenth of June.|The National Industrial Recovery Act expires on the sixteenth of June. +LJ022-0143|After careful consideration, I have asked the Congress to extend the life of this useful agency of government.|After careful consideration, I have asked the Congress to extend the life of this useful agency of government. +LJ022-0144|As we have proceeded with the administration of this Act,|As we have proceeded with the administration of this Act, +LJ022-0145|we have found from time to time more and more useful ways of promoting its purposes.|we have found from time to time more and more useful ways of promoting its purposes. +LJ022-0146|No reasonable person wants to abandon our present gains|No reasonable person wants to abandon our present gains +LJ022-0147|we must continue to protect children,|we must continue to protect children, +LJ022-0148|to enforce minimum wages, to prevent excessive hours,|to enforce minimum wages, to prevent excessive hours, +LJ022-0149|to safeguard, define and enforce collective bargaining, and, while retaining fair competition,|to safeguard, define and enforce collective bargaining, and, while retaining fair competition, +LJ022-0150|to eliminate so far as humanly possible, the kinds of unfair practices by selfish minorities which unfortunately|to eliminate so far as humanly possible, the kinds of unfair practices by selfish minorities which unfortunately +LJ022-0151|did more than anything else to bring about the recent collapse of industries.|did more than anything else to bring about the recent collapse of industries. +LJ022-0152|There is likewise pending before the Congress|There is likewise pending before the Congress +LJ022-0153|legislation to provide for the elimination of unnecessary holding companies in the public utility field.|legislation to provide for the elimination of unnecessary holding companies in the public utility field. +LJ022-0154|I consider this legislation a positive recovery measure.|I consider this legislation a positive recovery measure. +LJ022-0155|Power production in this country is virtually back to the 1929 peak.|Power production in this country is virtually back to the nineteen twenty-nine peak. +LJ022-0156|The operating companies in the gas and electric utility field are by and large in good condition, but|The operating companies in the gas and electric utility field are by and large in good condition, but +LJ022-0157|under holding company domination the utility industry has long been hopelessly at war within itself and with public sentiment.|under holding company domination the utility industry has long been hopelessly at war within itself and with public sentiment. +LJ022-0158|By far the greater part of the general decline in utility securities had occurred before I was inaugurated.|By far the greater part of the general decline in utility securities had occurred before I was inaugurated. +LJ022-0159|The absentee management of unnecessary holding company control|The absentee management of unnecessary holding company control +LJ022-0160|has lost touch with, and has lost the sympathy of, the communities it pretends to serve.|has lost touch with, and has lost the sympathy of, the communities it pretends to serve. +LJ022-0161|Even more significantly it has given the country as a whole an uneasy apprehension of overconcentrated economic power.|Even more significantly it has given the country as a whole an uneasy apprehension of overconcentrated economic power. +LJ022-0162|A business that loses the confidence of its customers and the goodwill of the public cannot long continue to be a good risk for the investor.|A business that loses the confidence of its customers and the goodwill of the public cannot long continue to be a good risk for the investor. +LJ022-0163|This legislation will serve the investor by ending the conditions which have caused that lack of confidence and goodwill.|This legislation will serve the investor by ending the conditions which have caused that lack of confidence and goodwill. +LJ022-0164|It will put the public utility operating industry on a sound basis for the future,|It will put the public utility operating industry on a sound basis for the future, +LJ022-0165|both in its public relations and in its internal relations.|both in its public relations and in its internal relations. +LJ022-0166|This legislation will not only in the long run result in providing lower electric and gas rates to the consumer,|This legislation will not only in the long run result in providing lower electric and gas rates to the consumer, +LJ022-0167|but it will protect the actual value and earning power of properties now owned by thousands of investors|but it will protect the actual value and earning power of properties now owned by thousands of investors +LJ022-0168|who have little protection under the old laws against what used to be called frenzied finance.|who have little protection under the old laws against what used to be called frenzied finance. +LJ022-0169|It will not destroy values.|It will not destroy values. +LJ022-0170|Not only business recovery, but the general economic recovery of the nation will be greatly stimulated by the enactment of legislation|Not only business recovery, but the general economic recovery of the nation will be greatly stimulated by the enactment of legislation +LJ022-0171|designed to improve the status of our transportation agencies.|designed to improve the status of our transportation agencies. +LJ022-0172|There is need for legislation providing for the regulation of interstate transportation by buses and trucks,|There is need for legislation providing for the regulation of interstate transportation by buses and trucks, +LJ022-0174|for the strengthening of the Interstate Commerce Commission to enable it to carry out a rounded conception of the national transportation system|for the strengthening of the Interstate Commerce Commission to enable it to carry out a rounded conception of the national transportation system +LJ022-0175|in which the benefits of private ownership are retained while the public stake in these important services is protected by the public's government.|in which the benefits of private ownership are retained while the public stake in these important services is protected by the public's government. +LJ022-0176|Finally, the reestablishment of public confidence in the banks of the nation|Finally, the reestablishment of public confidence in the banks of the nation +LJ022-0177|is one of the most hopeful results of our efforts as a Nation to reestablish public confidence in private banking.|is one of the most hopeful results of our efforts as a Nation to reestablish public confidence in private banking. +LJ022-0178|We all know that private banking actually exists by virtue of the permission of and regulation by the people as a whole,|We all know that private banking actually exists by virtue of the permission of and regulation by the people as a whole, +LJ022-0179|speaking through their government.|speaking through their government. +LJ022-0180|Wise public policy, however, requires not only that banking be safe|Wise public policy, however, requires not only that banking be safe +LJ022-0181|but that its resources be most fully utilized in the economic life of the country|but that its resources be most fully utilized in the economic life of the country +LJ022-0182|To this end it was decided more than twenty years ago|To this end it was decided more than twenty years ago +LJ022-0183|that the government should assume the responsibility of providing a means by which the credit of the nation might be controlled,|that the government should assume the responsibility of providing a means by which the credit of the nation might be controlled, +LJ022-0184|not by a few private banking institutions, but by a body with public prestige and authority.|not by a few private banking institutions, but by a body with public prestige and authority. +LJ022-0185|The answer to this demand was the Federal Reserve System.|The answer to this demand was the Federal Reserve System. +LJ022-0186|Twenty years of experience with this system have justified the efforts made to create it,|Twenty years of experience with this system have justified the efforts made to create it, +LJ022-0187|but these twenty years have shown by experience definite possibilities for improvement.|but these twenty years have shown by experience definite possibilities for improvement. +LJ022-0188|Certain proposals made to amend the Federal Reserve Act deserve prompt and favorable action by the Congress.|Certain proposals made to amend the Federal Reserve Act deserve prompt and favorable action by the Congress. +LJ022-0189|They are a minimum of wise readjustments of our Federal Reserve System in the light of past experience and present needs.|They are a minimum of wise readjustments of our Federal Reserve System in the light of past experience and present needs. +LJ022-0190|These measures I have mentioned are, in large part, the program which under my constitutional duty I have recommended to the Congress.|These measures I have mentioned are, in large part, the program which under my constitutional duty I have recommended to the Congress. +LJ022-0191|They are essential factors in a rounded program for national recovery.|They are essential factors in a rounded program for national recovery. +LJ022-0192|They contemplate the enrichment of our national life|They contemplate the enrichment of our national life +LJ022-0193|by a sound and rational ordering of its various elements and wise provisions for the protection of the weak against the strong.|by a sound and rational ordering of its various elements and wise provisions for the protection of the weak against the strong. +LJ022-0194|Never since my inauguration in March, 1933, have I felt so unmistakably the atmosphere of recovery.|Never since my inauguration in March, nineteen thirty-three, have I felt so unmistakably the atmosphere of recovery. +LJ022-0195|But it is more than the recovery of the material basis of our individual lives.|But it is more than the recovery of the material basis of our individual lives. +LJ022-0196|It is the recovery of confidence in our democratic processes and institutions.|It is the recovery of confidence in our democratic processes and institutions. +LJ022-0197|We have survived all of the arduous burdens and the threatening dangers of a great economic calamity.|We have survived all of the arduous burdens and the threatening dangers of a great economic calamity. +LJ022-0198|We have in the darkest moments of our national trials retained our faith in our own ability to master our destiny.|We have in the darkest moments of our national trials retained our faith in our own ability to master our destiny. +LJ022-0199|Fear is vanishing and confidence is growing on every side,|Fear is vanishing and confidence is growing on every side, +LJ022-0200|renewed faith in the vast possibilities of human beings to improve their material and spiritual status|renewed faith in the vast possibilities of human beings to improve their material and spiritual status +LJ022-0201|through the instrumentality of the democratic form of government.|through the instrumentality of the democratic form of government. +LJ022-0202|That faith is receiving its just reward.|That faith is receiving its just reward. +LJ022-0203|For that we can be thankful to the God who watches over America.|For that we can be thankful to the God who watches over America. +LJ023-0001|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt, Section 9.|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt, Section nine. +LJ023-0002|March 9, 1937. Part one.|March nine, nineteen thirty-seven. Part one. +LJ023-0003|Last Thursday I described in detail certain economic problems which everyone admits now face the nation.|Last Thursday I described in detail certain economic problems which everyone admits now face the nation. +LJ023-0004|For the many messages which have come to me after that speech, and which it is physically impossible to answer individually,|For the many messages which have come to me after that speech, and which it is physically impossible to answer individually, +LJ023-0005|I take this means of saying "thank you."|I take this means of saying "thank you." +LJ023-0006|Tonight, sitting at my desk in the White House, I make my first radio report to the people in my second term of office.|Tonight, sitting at my desk in the White House, I make my first radio report to the people in my second term of office. +LJ023-0007|I am reminded of that evening in March, four years ago, when I made my first radio report to you.|I am reminded of that evening in March, four years ago, when I made my first radio report to you. +LJ023-0008|We were then in the midst of the great banking crisis.|We were then in the midst of the great banking crisis. +LJ023-0009|Soon after, with the authority of the Congress,|Soon after, with the authority of the Congress, +LJ023-0010|we asked the nation to turn over all of its privately held gold, dollar for dollar, to the government of the United States.|we asked the nation to turn over all of its privately held gold, dollar for dollar, to the government of the United States. +LJ023-0011|Today's recovery proves how right that policy was.|Today's recovery proves how right that policy was. +LJ023-0012|But when, almost two years later, it came before the Supreme Court its constitutionality was upheld only by a five-to-four vote.|But when, almost two years later, it came before the Supreme Court its constitutionality was upheld only by a five-to-four vote. +LJ023-0013|The change of one vote would have thrown all the affairs of this great Nation back into hopeless chaos.|The change of one vote would have thrown all the affairs of this great Nation back into hopeless chaos. +LJ023-0014|In effect, four Justices ruled that the right under a private contract|In effect, four Justices ruled that the right under a private contract +LJ023-0015|to exact a pound of flesh was more sacred than the main objectives of the Constitution to establish an enduring Nation.|to exact a pound of flesh was more sacred than the main objectives of the Constitution to establish an enduring Nation. +LJ023-0016|In 1933 you and I knew that we must never let our economic system get completely out of joint again|In nineteen thirty-three you and I knew that we must never let our economic system get completely out of joint again +LJ023-0017|that we could not afford to take the risk of another great depression.|that we could not afford to take the risk of another great depression. +LJ023-0018|We also became convinced that the only way to avoid a repetition of those dark days was to have a government with power to prevent|We also became convinced that the only way to avoid a repetition of those dark days was to have a government with power to prevent +LJ023-0019|and to cure the abuses and the inequalities which had thrown that system out of joint.|and to cure the abuses and the inequalities which had thrown that system out of joint. +LJ023-0020|We then began a program of remedying those abuses and inequalities -- to give balance and stability to our economic system|We then began a program of remedying those abuses and inequalities -- to give balance and stability to our economic system +LJ023-0021|to make it bomb-proof against the causes of 1929.|to make it bomb-proof against the causes of nineteen twenty-nine. +LJ023-0022|Today we are only part-way through that program|Today we are only part-way through that program +LJ023-0023|and recovery is speeding up to a point where the dangers of 1929 are again becoming possible,|and recovery is speeding up to a point where the dangers of nineteen twenty-nine are again becoming possible, +LJ023-0024|not this week or month perhaps, but within a year or two.|not this week or month perhaps, but within a year or two. +LJ023-0025|National laws are needed to complete that program.|National laws are needed to complete that program. +LJ023-0026|Individual or local or state effort alone cannot protect us in 1937 any better than ten years ago.|Individual or local or state effort alone cannot protect us in nineteen thirty-seven any better than ten years ago. +LJ023-0027|It will take time -- and plenty of time -- to work out our remedies administratively even after legislation is passed.|It will take time -- and plenty of time -- to work out our remedies administratively even after legislation is passed. +LJ023-0028|To complete our program of protection in time, therefore,|To complete our program of protection in time, therefore, +LJ023-0029|we cannot delay one moment in making certain that our national government has power to carry through.|we cannot delay one moment in making certain that our national government has power to carry through. +LJ023-0030|Four years ago action did not come until the eleventh hour.|Four years ago action did not come until the eleventh hour. +LJ023-0031|It was almost too late.|It was almost too late. +LJ023-0032|If we learned anything from the depression|If we learned anything from the depression +LJ023-0033|we will not allow ourselves to run around in new circles of futile discussion and debate, always postponing the day of decision.|we will not allow ourselves to run around in new circles of futile discussion and debate, always postponing the day of decision. +LJ023-0034|The American people have learned from the depression.|The American people have learned from the depression. +LJ023-0035|For in the last three national elections|For in the last three national elections +LJ023-0036|an overwhelming majority of them voted a mandate that the Congress and the President begin the task of providing that protection|an overwhelming majority of them voted a mandate that the Congress and the President begin the task of providing that protection +LJ023-0037|not after long years of debate, but now.|not after long years of debate, but now. +LJ023-0038|The courts, however, have cast doubts on the ability of the elected Congress to protect us against catastrophe|The courts, however, have cast doubts on the ability of the elected Congress to protect us against catastrophe +LJ023-0039|by meeting squarely our modern social and economic conditions.|by meeting squarely our modern social and economic conditions. +LJ023-0040|We are at a crisis in our ability to proceed with that protection.|We are at a crisis in our ability to proceed with that protection. +LJ023-0041|It is a quiet crisis. There are no lines of depositors outside closed banks.|It is a quiet crisis. There are no lines of depositors outside closed banks. +LJ023-0042|But to the far-sighted it is far-reaching in its possibilities of injury to America.|But to the far-sighted it is far-reaching in its possibilities of injury to America. +LJ023-0043|I want to talk with you very simply about the need for present action in this crisis|I want to talk with you very simply about the need for present action in this crisis +LJ023-0044|the need to meet the unanswered challenge of one-third of a Nation ill-nourished, ill-clad, ill-housed.|the need to meet the unanswered challenge of one-third of a Nation ill-nourished, ill-clad, ill-housed. +LJ023-0045|Last Thursday I described the American form of government as a three horse team provided by the Constitution to the American people|Last Thursday I described the American form of government as a three horse team provided by the Constitution to the American people +LJ023-0046|so that their field might be plowed.|so that their field might be plowed. +LJ023-0047|The three horses are, of course, the three branches of government -- the Congress, the Executive and the courts.|The three horses are, of course, the three branches of government -- the Congress, the Executive and the courts. +LJ023-0048|Two of the horses are pulling in unison today; the third is not.|Two of the horses are pulling in unison today; the third is not. +LJ023-0049|Those who have intimated that the President of the United States is trying to drive that team|Those who have intimated that the President of the United States is trying to drive that team +LJ023-0050|overlook the simple fact that the President, as Chief Executive, is himself one of the three horses.|overlook the simple fact that the President, as Chief Executive, is himself one of the three horses. +LJ023-0051|It is the American people themselves who are in the driver's seat.|It is the American people themselves who are in the driver's seat. +LJ023-0052|It is the American people themselves who want the furrow plowed.|It is the American people themselves who want the furrow plowed. +LJ023-0053|It is the American people themselves who expect the third horse to pull in unison with the other two.|It is the American people themselves who expect the third horse to pull in unison with the other two. +LJ023-0054|I hope that you have re-read the Constitution of the United States in these past few weeks.|I hope that you have re-read the Constitution of the United States in these past few weeks. +LJ023-0055|Like the Bible, it ought to be read again and again.|Like the Bible, it ought to be read again and again. +LJ023-0056|It is an easy document to understand when you remember that it was called into being|It is an easy document to understand when you remember that it was called into being +LJ023-0057|because the Articles of Confederation under which the original thirteen States tried to operate after the Revolution|because the Articles of Confederation under which the original thirteen States tried to operate after the Revolution +LJ023-0058|showed the need of a national government with power enough to handle national problems.|showed the need of a national government with power enough to handle national problems. +LJ023-0059|In its Preamble, the Constitution states that it was intended to form a more perfect Union and promote the general welfare;|In its Preamble, the Constitution states that it was intended to form a more perfect Union and promote the general welfare; +LJ023-0060|and the powers given to the Congress to carry out those purposes can be best described by saying|and the powers given to the Congress to carry out those purposes can be best described by saying +LJ023-0061|that they were all the powers needed to meet each and every problem which then had a national character|that they were all the powers needed to meet each and every problem which then had a national character +LJ023-0062|and which could not be met by merely local action.|and which could not be met by merely local action. +LJ023-0063|But the framers went further.|But the framers went further. +LJ023-0064|Having in mind that in succeeding generations many other problems then undreamed of would become national problems|Having in mind that in succeeding generations many other problems then undreamed of would become national problems +LJ023-0065|they gave to the Congress the ample broad powers "to levy taxes|they gave to the Congress the ample broad powers "to levy taxes +LJ023-0066|and provide for the common defense and general welfare of the United States.|and provide for the common defense and general welfare of the United States. +LJ023-0067|That, my friends, is what I honestly believe|That, my friends, is what I honestly believe +LJ023-0068|to have been the clear and underlying purpose of the patriots who wrote a federal constitution to create a national government|to have been the clear and underlying purpose of the patriots who wrote a federal constitution to create a national government +LJ023-0069|with national power, intended as they said, "to form a more perfect union|with national power, intended as they said, "to form a more perfect union +LJ023-0070|for ourselves and our posterity.|for ourselves and our posterity. +LJ023-0071|For nearly twenty years there was no conflict between the Congress and the Court.|For nearly twenty years there was no conflict between the Congress and the Court. +LJ023-0072|Congress passed a statute which, in 1803, the Court said violated an express provision of the Constitution.|Congress passed a statute which, in eighteen oh three, the Court said violated an express provision of the Constitution. +LJ023-0073|The Court claimed the power to declare it unconstitutional and did so declare it.|The Court claimed the power to declare it unconstitutional and did so declare it. +LJ023-0074|But a little later the Court itself admitted|But a little later the Court itself admitted +LJ023-0075|that it was an extraordinary power to exercise and through Mr. Justice Washington laid down this limitation upon it:|that it was an extraordinary power to exercise and through Mr. Justice Washington laid down this limitation upon it: +LJ023-0076|It is but a decent respect due to the wisdom, the integrity and the patriotism of the legislative body,|It is but a decent respect due to the wisdom, the integrity and the patriotism of the legislative body, +LJ023-0077|by which any law is passed,|by which any law is passed, +LJ023-0078|to presume in favor of its validity until its violation of the Constitution is proved beyond all reasonable doubt.|to presume in favor of its validity until its violation of the Constitution is proved beyond all reasonable doubt. +LJ023-0079|But since the rise of the modern movement for social and economic progress through legislation,|But since the rise of the modern movement for social and economic progress through legislation, +LJ023-0080|the Court has more and more often and more and more boldly|the Court has more and more often and more and more boldly +LJ023-0081|asserted a power to veto laws passed by the Congress and state legislatures in complete disregard of this original limitation.|asserted a power to veto laws passed by the Congress and state legislatures in complete disregard of this original limitation. +LJ023-0082|In the last four years the sound rule of giving statutes the benefit of all reasonable doubt has been cast aside.|In the last four years the sound rule of giving statutes the benefit of all reasonable doubt has been cast aside. +LJ023-0083|The Court has been acting not as a judicial body, but as a policy-making body.|The Court has been acting not as a judicial body, but as a policy-making body. +LJ023-0084|When the Congress has sought to stabilize national agriculture, to improve the conditions of labor,|When the Congress has sought to stabilize national agriculture, to improve the conditions of labor, +LJ023-0085|to safeguard business against unfair competition,|to safeguard business against unfair competition, +LJ023-0086|to protect our national resources, and in many other ways, to serve our clearly national needs,|to protect our national resources, and in many other ways, to serve our clearly national needs, +LJ023-0087|the majority of the Court has been assuming the power to pass on the wisdom of these acts of the Congress|the majority of the Court has been assuming the power to pass on the wisdom of these acts of the Congress +LJ023-0088|and to approve or disapprove the public policy written into these laws.|and to approve or disapprove the public policy written into these laws. +LJ023-0089|That is not only my accusation.|That is not only my accusation. +LJ023-0090|It is the accusation of most distinguished justices of the present Supreme Court.|It is the accusation of most distinguished justices of the present Supreme Court. +LJ023-0091|I have not the time to quote to you all the language used by dissenting justices in many of these cases.|I have not the time to quote to you all the language used by dissenting justices in many of these cases. +LJ023-0092|But in the case holding the Railroad Retirement Act unconstitutional, for instance,|But in the case holding the Railroad Retirement Act unconstitutional, for instance, +LJ023-0093|Chief Justice Hughes said in a dissenting opinion that the majority opinion was "a departure from sound principles,"|Chief Justice Hughes said in a dissenting opinion that the majority opinion was "a departure from sound principles," +LJ023-0094|and placed "an unwarranted limitation upon the commerce clause."|and placed "an unwarranted limitation upon the commerce clause." +LJ023-0095|And three other justices agreed with him.|And three other justices agreed with him. +LJ023-0096|In the case of holding the A.A.A. unconstitutional,|In the case of holding the A.A.A. unconstitutional, +LJ023-0097|Justice Stone said of the majority opinion that it was a "tortured construction of the Constitution."|Justice Stone said of the majority opinion that it was a "tortured construction of the Constitution." +LJ023-0098|And two other justices agreed with him.|And two other justices agreed with him. +LJ023-0099|In the case holding the New York Minimum Wage Law unconstitutional, Justice Stone said|In the case holding the New York Minimum Wage Law unconstitutional, Justice Stone said +LJ023-0100|that the majority were actually reading into the Constitution their own "personal economic predilections," and that|that the majority were actually reading into the Constitution their own "personal economic predilections," and that +LJ023-0101|if the legislative power is not left free to choose the methods of solving the problems of poverty,|if the legislative power is not left free to choose the methods of solving the problems of poverty, +LJ023-0102|subsistence, and health of large numbers in the community, then|subsistence, and health of large numbers in the community, then +LJ023-0103|government is to be rendered impotent.|government is to be rendered impotent. +LJ023-0104|And two other justices agreed with him.|And two other justices agreed with him. +LJ023-0105|In the face of these dissenting opinions, there is no basis for the claim made by some members of the Court|In the face of these dissenting opinions, there is no basis for the claim made by some members of the Court +LJ023-0106|that something in the Constitution has compelled them regretfully to thwart the will of the people.|that something in the Constitution has compelled them regretfully to thwart the will of the people. +LJ023-0107|In the face of such dissenting opinions, it is perfectly clear that, as Chief Justice Hughes has said,|In the face of such dissenting opinions, it is perfectly clear that, as Chief Justice Hughes has said, +LJ023-0108|We are under a Constitution, but the Constitution is what the judges say it is.|We are under a Constitution, but the Constitution is what the judges say it is. +LJ023-0109|The Court in addition to the proper use of its judicial functions has improperly set itself up as a third house of the Congress|The Court in addition to the proper use of its judicial functions has improperly set itself up as a third house of the Congress +LJ023-0110|a super-legislature, as one of the justices has called it|a super-legislature, as one of the justices has called it +LJ023-0111|reading into the Constitution words and implications which are not there, and which were never intended to be there.|reading into the Constitution words and implications which are not there, and which were never intended to be there. +LJ023-0112|We have, therefore,|We have, therefore, +LJ023-0113|reached the point as a nation where we must take action to save the Constitution from the Court and the Court from itself.|reached the point as a nation where we must take action to save the Constitution from the Court and the Court from itself. +LJ023-0114|We must find a way to take an appeal from the Supreme Court to the Constitution itself.|We must find a way to take an appeal from the Supreme Court to the Constitution itself. +LJ023-0115|We want a Supreme Court which will do justice under the Constitution -- not over it.|We want a Supreme Court which will do justice under the Constitution -- not over it. +LJ023-0116|In our courts we want a government of laws and not of men.|In our courts we want a government of laws and not of men. +LJ023-0117|I want -- as all Americans want -- an independent judiciary as proposed by the framers of the Constitution.|I want -- as all Americans want -- an independent judiciary as proposed by the framers of the Constitution. +LJ023-0118|That means a Supreme Court that will enforce the Constitution as written|That means a Supreme Court that will enforce the Constitution as written +LJ023-0119|that will refuse to amend the Constitution by the arbitrary exercise of judicial power -- amended by judicial say-so.|that will refuse to amend the Constitution by the arbitrary exercise of judicial power -- amended by judicial say-so. +LJ023-0120|It does not mean a judiciary so independent that it can deny the existence of facts which are universally recognized.|It does not mean a judiciary so independent that it can deny the existence of facts which are universally recognized. +LJ023-0121|How then could we proceed to perform the mandate given us?|How then could we proceed to perform the mandate given us? +LJ023-0122|It was said in last year's Democratic platform,|It was said in last year's Democratic platform, +LJ023-0123|If these problems cannot be effectively solved within the Constitution,|If these problems cannot be effectively solved within the Constitution, +LJ023-0124|we shall seek such clarifying amendment as will assure the power to enact those laws,|we shall seek such clarifying amendment as will assure the power to enact those laws, +LJ023-0125|adequately to regulate commerce, protect public health and safety, and safeguard economic security.|adequately to regulate commerce, protect public health and safety, and safeguard economic security. +LJ023-0126|In other words, we said we would seek an amendment only if every other possible means by legislation were to fail.|In other words, we said we would seek an amendment only if every other possible means by legislation were to fail. +LJ023-0127|When I commenced to review the situation with the problem squarely before me,|When I commenced to review the situation with the problem squarely before me, +LJ023-0128|I came by a process of elimination to the conclusion that, short of amendments,|I came by a process of elimination to the conclusion that, short of amendments, +LJ023-0129|the only method which was clearly constitutional, and would at the same time carry out other much needed reforms,|the only method which was clearly constitutional, and would at the same time carry out other much needed reforms, +LJ023-0130|was to infuse new blood into all our courts.|was to infuse new blood into all our courts. +LJ023-0131|We must have men worthy and equipped to carry out impartial justice.|We must have men worthy and equipped to carry out impartial justice. +LJ023-0132|But, at the same time, we must have judges who will bring to the courts a present-day sense of the Constitution|But, at the same time, we must have judges who will bring to the courts a present-day sense of the Constitution +LJ023-0133|judges who will retain in the courts the judicial functions of a court,|judges who will retain in the courts the judicial functions of a court, +LJ023-0134|and reject the legislative powers which the courts have today assumed.|and reject the legislative powers which the courts have today assumed. +LJ023-0135|In forty-five out of the forty-eight states of the Union, judges are chosen not for life but for a period of years.|In forty-five out of the forty-eight states of the Union, judges are chosen not for life but for a period of years. +LJ023-0136|In many states judges must retire at the age of seventy.|In many states judges must retire at the age of seventy. +LJ023-0137|Congress has provided financial security|Congress has provided financial security +LJ023-0138|by offering life pensions at full pay for federal judges on all courts who are willing to retire at seventy.|by offering life pensions at full pay for federal judges on all courts who are willing to retire at seventy. +LJ023-0139|In the case of Supreme Court justices, that pension is $20,000 a year.|In the case of Supreme Court justices, that pension is twenty thousand dollars a year. +LJ023-0140|But all federal judges, once appointed,|But all federal judges, once appointed, +LJ023-0141|can, if they choose, hold office for life, no matter how old they may get to be.|can, if they choose, hold office for life, no matter how old they may get to be. +LJ024-0001|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt|The Fireside Chats of Franklin Delano Roosevelt, by Franklin D Roosevelt +LJ024-0002|Section 10. March 9, 1937. Part 2.|Section ten. March nine, nineteen thirty-seven. Part two. +LJ024-0003|What is my proposal?|What is my proposal? +LJ024-0004|It is simply this: whenever a judge or justice of any federal court|It is simply this: whenever a judge or justice of any federal court +LJ024-0005|has reached the age of seventy and does not avail himself of the opportunity to retire on a pension,|has reached the age of seventy and does not avail himself of the opportunity to retire on a pension, +LJ024-0006|a new member shall be appointed by the President then in office,|a new member shall be appointed by the President then in office, +LJ024-0007|with the approval, as required by the Constitution, of the Senate of the United States.|with the approval, as required by the Constitution, of the Senate of the United States. +LJ024-0008|That plan has two chief purposes.|That plan has two chief purposes. +LJ024-0009|By bringing into the judicial system a steady and continuing stream of new and younger blood, I hope, first,|By bringing into the judicial system a steady and continuing stream of new and younger blood, I hope, first, +LJ024-0010|to make the administration of all federal justice speedier and, therefore, less costly;|to make the administration of all federal justice speedier and, therefore, less costly; +LJ024-0011|secondly, to bring to the decision of social and economic problems younger men who have had personal experience and contact|secondly, to bring to the decision of social and economic problems younger men who have had personal experience and contact +LJ024-0012|with modern facts and circumstances under which average men have to live and work.|with modern facts and circumstances under which average men have to live and work. +LJ024-0013|This plan will save our national Constitution from hardening of the judicial arteries.|This plan will save our national Constitution from hardening of the judicial arteries. +LJ024-0014|The number of judges to be appointed would depend wholly on the decision of present judges now over seventy,|The number of judges to be appointed would depend wholly on the decision of present judges now over seventy, +LJ024-0015|or those who would subsequently reach the age of seventy.|or those who would subsequently reach the age of seventy. +LJ024-0016|If, for instance, any one of the six justices of the Supreme Court now over the age of seventy should retire as provided under the plan,|If, for instance, any one of the six justices of the Supreme Court now over the age of seventy should retire as provided under the plan, +LJ024-0017|no additional place would be created.|no additional place would be created. +LJ024-0018|Consequently, although there never can be more than fifteen, there may be only fourteen, or thirteen, or twelve.|Consequently, although there never can be more than fifteen, there may be only fourteen, or thirteen, or twelve. +LJ024-0019|And there may be only nine.|And there may be only nine. +LJ024-0020|There is nothing novel or radical about this idea.|There is nothing novel or radical about this idea. +LJ024-0021|It seeks to maintain the federal bench in full vigor.|It seeks to maintain the federal bench in full vigor. +LJ024-0022|It has been discussed and approved by many persons of high authority|It has been discussed and approved by many persons of high authority +LJ024-0023|ever since a similar proposal passed the House of Representatives in 1869.|ever since a similar proposal passed the House of Representatives in eighteen sixty-nine. +LJ024-0024|Why was the age fixed at seventy?|Why was the age fixed at seventy? +LJ024-0025|Because the laws of many states, the practice of the Civil Service, the regulations of the Army and Navy,|Because the laws of many states, the practice of the Civil Service, the regulations of the Army and Navy, +LJ024-0026|and the rules of many of our universities and of almost every great private business enterprise,|and the rules of many of our universities and of almost every great private business enterprise, +LJ024-0027|commonly fix the retirement age at seventy years or less.|commonly fix the retirement age at seventy years or less. +LJ024-0028|The statute would apply to all the courts in the federal system.|The statute would apply to all the courts in the federal system. +LJ024-0029|There is general approval so far as the lower federal courts are concerned.|There is general approval so far as the lower federal courts are concerned. +LJ024-0030|The plan has met opposition only so far as the Supreme Court of the United States itself is concerned.|The plan has met opposition only so far as the Supreme Court of the United States itself is concerned. +LJ024-0031|If such a plan is good for the lower courts it certainly ought to be equally good for the highest court from which there is no appeal.|If such a plan is good for the lower courts it certainly ought to be equally good for the highest court from which there is no appeal. +LJ024-0032|Those opposing this plan have sought to arouse prejudice and fear by crying that I am seeking to "pack" the Supreme Court|Those opposing this plan have sought to arouse prejudice and fear by crying that I am seeking to "pack" the Supreme Court +LJ024-0033|and that a baneful precedent will be established.|and that a baneful precedent will be established. +LJ024-0034|What do they mean by the words "packing the Court"?|What do they mean by the words "packing the Court"? +LJ024-0035|Let me answer this question with a bluntness that will end all honest misunderstanding of my purposes.|Let me answer this question with a bluntness that will end all honest misunderstanding of my purposes. +LJ024-0036|If by that phrase "packing the Court" it is charged that I wish to place on the bench spineless puppets|If by that phrase "packing the Court" it is charged that I wish to place on the bench spineless puppets +LJ024-0037|who would disregard the law and would decide specific cases as I wished them to be decided, I make this answer:|who would disregard the law and would decide specific cases as I wished them to be decided, I make this answer: +LJ024-0038|that no President fit for his office would appoint,|that no President fit for his office would appoint, +LJ024-0039|and no Senate of honorable men fit for their office would confirm, that kind of appointees to the Supreme Court.|and no Senate of honorable men fit for their office would confirm, that kind of appointees to the Supreme Court. +LJ024-0040|But if by that phrase the charge is made|But if by that phrase the charge is made +LJ024-0041|that I would appoint and the Senate would confirm justices worthy to sit beside present members of the Court|that I would appoint and the Senate would confirm justices worthy to sit beside present members of the Court +LJ024-0042|who understand those modern conditions,|who understand those modern conditions, +LJ024-0043|that I will appoint justices who will not undertake to override the judgment of the Congress on legislative policy,|that I will appoint justices who will not undertake to override the judgment of the Congress on legislative policy, +LJ024-0044|that I will appoint justices who will act as justices and not as legislators|that I will appoint justices who will act as justices and not as legislators +LJ024-0045|if the appointment of such justices can be called "packing the Courts,"|if the appointment of such justices can be called "packing the Courts," +LJ024-0046|then I say that I and with me the vast majority of the American people favor doing just that thing -- now.|then I say that I and with me the vast majority of the American people favor doing just that thing -- now. +LJ024-0047|Is it a dangerous precedent for the Congress to change the number of the justices? The Congress has always had, and will have, that power.|Is it a dangerous precedent for the Congress to change the number of the justices? The Congress has always had, and will have, that power. +LJ024-0048|The number of justices has been changed several times before,|The number of justices has been changed several times before, +LJ024-0049|in the administration of John Adams and Thomas Jefferson -- both signers of the Declaration of Independence|in the administration of John Adams and Thomas Jefferson -- both signers of the Declaration of Independence +LJ024-0050|Andrew Jackson, Abraham Lincoln and Ulysses S. Grant.|Andrew Jackson, Abraham Lincoln and Ulysses S. Grant. +LJ024-0051|I suggest only the addition of justices to the bench|I suggest only the addition of justices to the bench +LJ024-0052|in accordance with a clearly defined principle relating to a clearly defined age limit.|in accordance with a clearly defined principle relating to a clearly defined age limit. +LJ024-0053|Fundamentally, if in the future, America cannot trust the Congress it elects to refrain from abuse of our Constitutional usages|Fundamentally, if in the future, America cannot trust the Congress it elects to refrain from abuse of our Constitutional usages +LJ024-0054|democracy will have failed far beyond the importance to it of any king of precedent concerning the judiciary.|democracy will have failed far beyond the importance to it of any king of precedent concerning the judiciary. +LJ024-0055|We think it so much in the public interest to maintain a vigorous judiciary|We think it so much in the public interest to maintain a vigorous judiciary +LJ024-0056|that we encourage the retirement of elderly judges by offering them a life pension at full salary.|that we encourage the retirement of elderly judges by offering them a life pension at full salary. +LJ024-0057|Why then should we leave the fulfillment of this public policy to chance|Why then should we leave the fulfillment of this public policy to chance +LJ024-0058|or make independent on upon the desire or prejudice of any individual justice?|or make independent on upon the desire or prejudice of any individual justice? +LJ024-0059|It is the clear intention of our public policy to provide for a constant flow of new and younger blood into the judiciary.|It is the clear intention of our public policy to provide for a constant flow of new and younger blood into the judiciary. +LJ024-0060|Normally every President appoints a large number of district and circuit court judges and a few members of the Supreme Court.|Normally every President appoints a large number of district and circuit court judges and a few members of the Supreme Court. +LJ024-0061|Until my first term practically every President of the United States has appointed at least one member of the Supreme Court.|Until my first term practically every President of the United States has appointed at least one member of the Supreme Court. +LJ024-0062|President Taft appointed five members and named a Chief Justice;|President Taft appointed five members and named a Chief Justice; +LJ024-0063|President Wilson, three; President Harding, four, including a Chief Justice; President Coolidge, one;|President Wilson, three; President Harding, four, including a Chief Justice; President Coolidge, one; +LJ024-0064|President Hoover, three, including a Chief Justice.|President Hoover, three, including a Chief Justice. +LJ024-0065|Such a succession of appointments should have provided a Court well-balanced as to age.|Such a succession of appointments should have provided a Court well-balanced as to age. +LJ024-0066|But chance and the disinclination of individuals to leave the Supreme bench|But chance and the disinclination of individuals to leave the Supreme bench +LJ024-0067|have now given us a Court in which five justices will be over seventy-five years of age before next June|have now given us a Court in which five justices will be over seventy-five years of age before next June +LJ024-0068|and one over seventy.|and one over seventy. +LJ024-0069|Thus a sound public policy has been defeated.|Thus a sound public policy has been defeated. +LJ024-0070|I now propose that we establish by law an assurance against any such ill-balanced court in the future.|I now propose that we establish by law an assurance against any such ill-balanced court in the future. +LJ024-0071|I propose that hereafter, when a judge reaches the age of seventy,|I propose that hereafter, when a judge reaches the age of seventy, +LJ024-0072|a new and younger judge shall be added to the court automatically.|a new and younger judge shall be added to the court automatically. +LJ024-0073|In this way I propose to enforce a sound public policy by law|In this way I propose to enforce a sound public policy by law +LJ024-0074|instead of leaving the composition of our federal courts, including the highest,|instead of leaving the composition of our federal courts, including the highest, +LJ024-0075|to be determined by chance or the personal indecision of individuals.|to be determined by chance or the personal indecision of individuals. +LJ024-0076|If such a law as I propose is regarded as establishing a new precedent,|If such a law as I propose is regarded as establishing a new precedent, +LJ024-0077|is it not a most desirable precedent?|is it not a most desirable precedent? +LJ024-0078|Like all lawyers, like all Americans, I regret the necessity of this controversy.|Like all lawyers, like all Americans, I regret the necessity of this controversy. +LJ024-0079|But the welfare of the United States, and indeed of the Constitution itself, is what we all must think about first.|But the welfare of the United States, and indeed of the Constitution itself, is what we all must think about first. +LJ024-0080|Our difficulty with the Court today rises not from the Court as an institution but from human beings within it.|Our difficulty with the Court today rises not from the Court as an institution but from human beings within it. +LJ024-0081|But we cannot yield our constitutional destiny to the personal judgment of a few men who, being fearful of the future,|But we cannot yield our constitutional destiny to the personal judgment of a few men who, being fearful of the future, +LJ024-0082|would deny us the necessary means of dealing with the present.|would deny us the necessary means of dealing with the present. +LJ024-0083|This plan of mine is no attack on the Court;|This plan of mine is no attack on the Court; +LJ024-0084|it seeks to restore the Court to its rightful and historic place in our constitutional government|it seeks to restore the Court to its rightful and historic place in our constitutional government +LJ024-0085|and to have it resume its high task of building anew on the Constitution "a system of living law."|and to have it resume its high task of building anew on the Constitution "a system of living law." +LJ024-0086|The Court itself can best undo what the Court has done.|The Court itself can best undo what the Court has done. +LJ024-0088|I hope that thereby the difficult process of constitutional amendment may be rendered unnecessary. But,|I hope that thereby the difficult process of constitutional amendment may be rendered unnecessary. But, +LJ024-0089|let us examine the process.|let us examine the process. +LJ024-0090|There are many types of amendment proposed.|There are many types of amendment proposed. +LJ024-0091|Each one is radically different from the other.|Each one is radically different from the other. +LJ024-0092|There is no substantial groups within the Congress or outside it who are agreed on any single amendment.|There is no substantial groups within the Congress or outside it who are agreed on any single amendment. +LJ024-0093|It would take months or years to get substantial agreement upon the type and language of the amendment.|It would take months or years to get substantial agreement upon the type and language of the amendment. +LJ024-0094|It would take months and years thereafter to get a two-thirds majority in favor of that amendment in both Houses of the Congress.|It would take months and years thereafter to get a two-thirds majority in favor of that amendment in both Houses of the Congress. +LJ024-0095|Then would come the long course of ratification by three-fourths of all the states.|Then would come the long course of ratification by three-fourths of all the states. +LJ024-0096|No amendment which any powerful economic interests or the leaders of any powerful political party have had reason to oppose|No amendment which any powerful economic interests or the leaders of any powerful political party have had reason to oppose +LJ024-0097|has ever been ratified within anything like a reasonable time.|has ever been ratified within anything like a reasonable time. +LJ024-0098|And thirteen states which contain only five percent of the voting population can block ratification|And thirteen states which contain only five percent of the voting population can block ratification +LJ024-0099|even though the thirty- five states with ninety-five percent of the population are in favor of it.|even though the thirty- five states with ninety-five percent of the population are in favor of it. +LJ024-0100|A very large percentage of newspaper publishers, Chambers of Commerce,|A very large percentage of newspaper publishers, Chambers of Commerce, +LJ024-0101|Bar Association, Manufacturers' Associations, who are trying to give the impression that they really do want a constitutional amendment|Bar Association, Manufacturers' Associations, who are trying to give the impression that they really do want a constitutional amendment +LJ024-0102|would be the first to exclaim as soon as an amendment was proposed|would be the first to exclaim as soon as an amendment was proposed +LJ024-0103|Oh! I was for an amendment all right, but this amendment you proposed is not the kind of amendment that I was thinking about.|Oh! I was for an amendment all right, but this amendment you proposed is not the kind of amendment that I was thinking about. +LJ024-0104|I am therefore, going to spend my time, my efforts and my money|I am therefore, going to spend my time, my efforts and my money +LJ024-0105|to block the amendment, although I would be awfully glad to help get some other kind of amendment ratified.|to block the amendment, although I would be awfully glad to help get some other kind of amendment ratified. +LJ024-0106|Two groups oppose my plan on the ground that they favor a constitutional amendment.|Two groups oppose my plan on the ground that they favor a constitutional amendment. +LJ024-0107|The first includes those who fundamentally object to social and economic legislation along modern lines.|The first includes those who fundamentally object to social and economic legislation along modern lines. +LJ024-0108|This is the same group who during the campaign last Fall tried to block the mandate of the people.|This is the same group who during the campaign last Fall tried to block the mandate of the people. +LJ024-0109|Now they are making a last stand.|Now they are making a last stand. +LJ024-0111|the legislation demanded by the mandate.|the legislation demanded by the mandate. +LJ024-0112|To them I say: I do not think you will be able long to fool the American people as to your purposes.|To them I say: I do not think you will be able long to fool the American people as to your purposes. +LJ024-0113|The other groups is composed of those who honestly believe the amendment process is the best|The other groups is composed of those who honestly believe the amendment process is the best +LJ024-0114|and who would be willing to support a reasonable amendment if they could agree on one.|and who would be willing to support a reasonable amendment if they could agree on one. +LJ024-0115|To them I say: we cannot rely on an amendment as the immediate or only answer to our present difficulties.|To them I say: we cannot rely on an amendment as the immediate or only answer to our present difficulties. +LJ024-0116|When the time comes for action,|When the time comes for action, +LJ024-0117|you will find that many of those who pretend to support you will sabotage any constructive amendment which is proposed.|you will find that many of those who pretend to support you will sabotage any constructive amendment which is proposed. +LJ024-0118|Look at these strange bed-fellows of yours.|Look at these strange bed-fellows of yours. +LJ024-0120|And remember one thing more.|And remember one thing more. +LJ024-0121|Even if an amendment were passed, and even if in the years to come it were to be ratified,|Even if an amendment were passed, and even if in the years to come it were to be ratified, +LJ024-0122|its meaning would depend upon the kind of justices who would be sitting on the Supreme Court bench.|its meaning would depend upon the kind of justices who would be sitting on the Supreme Court bench. +LJ024-0123|An amendment, like the rest of the Constitution,|An amendment, like the rest of the Constitution, +LJ024-0124|is what the justices say it is rather than what its framers or you might hope it is.|is what the justices say it is rather than what its framers or you might hope it is. +LJ024-0125|This proposal of mine will not infringe in the slightest upon the civil or religious liberties so dear to every American.|This proposal of mine will not infringe in the slightest upon the civil or religious liberties so dear to every American. +LJ024-0126|My record as Governor and President proves my devotion to those liberties.|My record as Governor and President proves my devotion to those liberties. +LJ024-0127|You who know me can have no fear that I would tolerate the destruction by any branch of government of any part of our heritage of freedom.|You who know me can have no fear that I would tolerate the destruction by any branch of government of any part of our heritage of freedom. +LJ024-0128|The present attempt by those opposed to progress to play upon the fears of danger to personal liberty|The present attempt by those opposed to progress to play upon the fears of danger to personal liberty +LJ024-0129|brings again to mind that crude and cruel strategy tried by the same opposition|brings again to mind that crude and cruel strategy tried by the same opposition +LJ024-0130|to frighten the workers of America in a pay-envelope propaganda against the Social Security Law.|to frighten the workers of America in a pay-envelope propaganda against the Social Security Law. +LJ024-0131|The workers were not fooled by that propaganda then. The people of America will not be fooled by such propaganda now.|The workers were not fooled by that propaganda then. The people of America will not be fooled by such propaganda now. +LJ024-0132|I am in favor of action through legislation:|I am in favor of action through legislation: +LJ024-0133|First, because I believe that it can be passed at this session of the Congress.|First, because I believe that it can be passed at this session of the Congress. +LJ024-0134|Second, because it will provide a reinvigorated|Second, because it will provide a reinvigorated +LJ024-0135|liberal-minded judiciary necessary to furnish quicker and cheaper justice from bottom to top.|liberal-minded judiciary necessary to furnish quicker and cheaper justice from bottom to top. +LJ024-0136|Third, because it will provide a series of federal courts willing to enforce the Constitution as written,|Third, because it will provide a series of federal courts willing to enforce the Constitution as written, +LJ024-0137|and unwilling to assert legislative powers by writing into it their own political and economic policies.|and unwilling to assert legislative powers by writing into it their own political and economic policies. +LJ024-0138|During the past half century the balance of power between the three great branches of the federal government,|During the past half century the balance of power between the three great branches of the federal government, +LJ024-0139|has been tipped out of balance by the courts in direct contradiction of the high purposes of the framers of the Constitution.|has been tipped out of balance by the courts in direct contradiction of the high purposes of the framers of the Constitution. +LJ024-0140|It is my purpose to restore that balance.|It is my purpose to restore that balance. +LJ024-0141|You who know me will accept my solemn assurance that in a world in which democracy is under attack,|You who know me will accept my solemn assurance that in a world in which democracy is under attack, +LJ024-0142|I seek to make American democracy succeed.|I seek to make American democracy succeed. +LJ024-0143|You and I will do our part.|You and I will do our part. +LJ025-0001|The Science: History of the Universe, Volume 5.|The Science: History of the Universe, Volume five. +LJ025-0002|Edited by Francis Rolt-Wheeler. Biology. Chapter 7. Organic Functions. Part One.|Edited by Francis Rolt-Wheeler. Biology. Chapter seven. Organic Functions. Part One. +LJ025-0003|The facts dealing with the physiology of organisms, the activities associated with that which we call life|The facts dealing with the physiology of organisms, the activities associated with that which we call life +LJ025-0004|are often designated Organic Functions.|are often designated Organic Functions. +LJ025-0005|The terms animal physiology, plant physiology and human physiology are in common use and often suggest to the lay reader|The terms animal physiology, plant physiology and human physiology are in common use and often suggest to the lay reader +LJ025-0006|that the functions or workings of the organs of plants, animals or man are quite distinct,|that the functions or workings of the organs of plants, animals or man are quite distinct, +LJ025-0007|so much so as to require discussion in different treatises.|so much so as to require discussion in different treatises. +LJ025-0008|This is true only as a matter of detail,|This is true only as a matter of detail, +LJ025-0009|for in the past fifty years it has been made evident that in general principles all living things are fundamentally similar.|for in the past fifty years it has been made evident that in general principles all living things are fundamentally similar. +LJ025-0010|One of the most important summaries of this similarity|One of the most important summaries of this similarity +LJ025-0011|is Huxley's famous essay, "The Border Territory Between the Animal and Vegetable Kingdoms," written in 1876,|is Huxley's famous essay, "The Border Territory Between the Animal and Vegetable Kingdoms," written in eighteen seventy-six, +LJ025-0012|extracts from which follow.|extracts from which follow. +LJ025-0013|In the second edition of the "Regne Animal," published in 1828,|In the second edition of the "Regne Animal," published in eighteen twenty-eight, +LJ025-0014|Cuvier devotes a special section to the Division of Organized Beings into Animals and Vegetables,|Cuvier devotes a special section to the Division of Organized Beings into Animals and Vegetables, +LJ025-0015|in which the question is treated with that comprehensiveness of knowledge and clear critical judgment which characterize his writings|in which the question is treated with that comprehensiveness of knowledge and clear critical judgment which characterize his writings +LJ025-0016|and justify biologists in regarding them as representative expressions of the most extensive, if not the profoundest, knowledge of his time.|and justify biologists in regarding them as representative expressions of the most extensive, if not the profoundest, knowledge of his time. +LJ025-0017|He affirms that living beings have been subdivided from the earliest times into animated beings, which possess sense and motion,|He affirms that living beings have been subdivided from the earliest times into animated beings, which possess sense and motion, +LJ025-0018|and inanimated beings, which are devoid of these functions and simply vegetable.|and inanimated beings, which are devoid of these functions and simply vegetable. +LJ025-0019|Although the roots of plants direct themselves toward moisture and their leaves toward air and light,|Although the roots of plants direct themselves toward moisture and their leaves toward air and light, +LJ025-0020|although the parts of some plants exhibit oscillating movements without any perceptible cause and the leaves of others retract when touched,|although the parts of some plants exhibit oscillating movements without any perceptible cause and the leaves of others retract when touched, +LJ025-0021|yet none of these movements justify the ascription to plants of perception of will.|yet none of these movements justify the ascription to plants of perception of will. +LJ025-0022|From the mobility of animals Cuvier, with his characteristic partiality for teleological reasoning,|From the mobility of animals Cuvier, with his characteristic partiality for teleological reasoning, +LJ025-0023|reduces the necessity of the existence in them of an alimentary cavity, or reservoir of food,|reduces the necessity of the existence in them of an alimentary cavity, or reservoir of food, +LJ025-0024|whence their nutrition may be drawn by vessels, which are a sort of internal roots; and, in the presence of this alimentary cavity|whence their nutrition may be drawn by vessels, which are a sort of internal roots; and, in the presence of this alimentary cavity +LJ025-0025|he naturally sees the primary and the most important distinction between animals and plants,|he naturally sees the primary and the most important distinction between animals and plants, +LJ025-0026|Following out his teleological argument, Cuvier remarks that the organization of this cavity|Following out his teleological argument, Cuvier remarks that the organization of this cavity +LJ025-0027|and its appurtenances must needs vary according to the nature of the aliment and the operations which it has to undergo|and its appurtenances must needs vary according to the nature of the aliment and the operations which it has to undergo +LJ025-0028|before it can be converted into substances fitted for absorption,|before it can be converted into substances fitted for absorption, +LJ025-0029|while the atmosphere and the earth supply plants with juices ready prepared and which can be absorbed immediately.|while the atmosphere and the earth supply plants with juices ready prepared and which can be absorbed immediately. +LJ025-0030|As the animal body required to be independent of heat and of the atmosphere,|As the animal body required to be independent of heat and of the atmosphere, +LJ025-0031|there were no means by which the motion of its fluids could be produced by internal causes.|there were no means by which the motion of its fluids could be produced by internal causes. +LJ025-0032|Hence arose the second great distinctive character of animals, or the circulatory system, which is less important than the digestive,|Hence arose the second great distinctive character of animals, or the circulatory system, which is less important than the digestive, +LJ025-0033|since it was unnecessary, and therefore is absent, in the more simple animals.|since it was unnecessary, and therefore is absent, in the more simple animals. +LJ025-0034|Animals further needed muscles for locomotion and nerves for sensibility.|Animals further needed muscles for locomotion and nerves for sensibility. +LJ025-0035|Hence, says Cuvier, it was necessary that the chemical composition of the animal body should be more complicated than that of the plant;|Hence, says Cuvier, it was necessary that the chemical composition of the animal body should be more complicated than that of the plant; +LJ025-0036|and it is so, inasmuch as an additional substance -- nitrogen -- enters into it as an essential element;|and it is so, inasmuch as an additional substance -- nitrogen -- enters into it as an essential element; +LJ025-0037|while in plants nitrogen is only accidentally joined with the three other fundamental constituents of organic beings|while in plants nitrogen is only accidentally joined with the three other fundamental constituents of organic beings +LJ025-0038|carbon, hydrogen and oxygen.|carbon, hydrogen and oxygen. +LJ025-0039|he afterward affirms that nitrogen is peculiar to animals, and herein he places the third distinction between the animal and the plant.|he afterward affirms that nitrogen is peculiar to animals, and herein he places the third distinction between the animal and the plant. +LJ025-0040|The soil and the atmosphere supply plants with water composed of hydrogen and oxygen|The soil and the atmosphere supply plants with water composed of hydrogen and oxygen +LJ025-0041|and carbonic acid containing carbon and oxygen.|and carbonic acid containing carbon and oxygen. +LJ025-0042|They retain the hydrogen and the carbon, exhale the superfluous oxygen and absorb little or no nitrogen.|They retain the hydrogen and the carbon, exhale the superfluous oxygen and absorb little or no nitrogen. +LJ025-0043|The essential character of vegetable life is the exhalation of oxygen, which is effected through the agency of light.|The essential character of vegetable life is the exhalation of oxygen, which is effected through the agency of light. +LJ025-0044|Animals, on the contrary, derive their nourishment either directly or indirectly from plants.|Animals, on the contrary, derive their nourishment either directly or indirectly from plants. +LJ025-0045|They get rid of the superfluous hydrogen and carbon and accumulate nitrogen.|They get rid of the superfluous hydrogen and carbon and accumulate nitrogen. +LJ025-0046|The relations of plants and animals to the atmosphere are therefore inverse.|The relations of plants and animals to the atmosphere are therefore inverse. +LJ025-0047|The plant withdraws water and carbonic acid from the atmosphere, the animal contributes both to it.|The plant withdraws water and carbonic acid from the atmosphere, the animal contributes both to it. +LJ025-0048|Respiration -- that is, the absorption of oxygen and the exhalation of carbonic acid|Respiration -- that is, the absorption of oxygen and the exhalation of carbonic acid +LJ025-0049|is the specially animal function of animals and constitutes their fourth distinctive character.|is the specially animal function of animals and constitutes their fourth distinctive character. +LJ025-0050|Thus wrote Cuvier in 1828.|Thus wrote Cuvier in eighteen twenty-eight. +LJ025-0051|But in the fourth and fifth decades of this century|But in the fourth and fifth decades of this century +LJ025-0052|the greatest and most rapid revolution which biological science has ever undergone was effected by the application of the modern microscope|the greatest and most rapid revolution which biological science has ever undergone was effected by the application of the modern microscope +LJ025-0053|to the investigation of organic structure,|to the investigation of organic structure, +LJ025-0054|by the introduction of exact and easily manageable methods of conducting the chemical analysis of organic compounds and finally|by the introduction of exact and easily manageable methods of conducting the chemical analysis of organic compounds and finally +LJ025-0055|by the employment of instruments of precision for the measurement of the physical forces which are at work in the living economy.|by the employment of instruments of precision for the measurement of the physical forces which are at work in the living economy. +LJ025-0056|That the semi-fluid contents (which we now term protoplasm) of the cells of certain plants,|That the semi-fluid contents (which we now term protoplasm) of the cells of certain plants, +LJ025-0057|such as the Charae, are in constant and regular motion was made out by Bonaventura Corti a century ago;|such as the Charae, are in constant and regular motion was made out by Bonaventura Corti a century ago; +LJ025-0058|but the fact, important as it was, fell into oblivion and had to be rediscovered by Treviranus in 1807.|but the fact, important as it was, fell into oblivion and had to be rediscovered by Treviranus in eighteen oh seven. +LJ025-0059|Robert Brown noted the more complex motions of the protoplasm in the cells of Tradescantia in 1831,|Robert Brown noted the more complex motions of the protoplasm in the cells of Tradescantia in eighteen thirty-one, +LJ025-0060|and now such movements of the living substance of plants are well known to be some of the most widely prevalent phenomena of vegetable life.|and now such movements of the living substance of plants are well known to be some of the most widely prevalent phenomena of vegetable life. +LJ025-0061|Agardh and other of the botanists of Cuvier's generation who occupied themselves with the lower plants had observed that,|Agardh and other of the botanists of Cuvier's generation who occupied themselves with the lower plants had observed that, +LJ025-0062|under particular circumstances, the contents of the cells of certain water-weeds were set free and moved about with considerable velocity|under particular circumstances, the contents of the cells of certain water-weeds were set free and moved about with considerable velocity +LJ025-0063|and with all the appearances of spontaneity as locomotive bodies,|and with all the appearances of spontaneity as locomotive bodies, +LJ025-0064|which, from their similarity to animals of simple organization, were called "zoospores."|which, from their similarity to animals of simple organization, were called "zoospores." +LJ025-0065|Even as late as 1845, however, a botanist of Schleiden's eminence dealt very skeptically with these statements,|Even as late as eighteen forty-five, however, a botanist of Schleiden's eminence dealt very skeptically with these statements, +LJ025-0066|and his skepticism was the more justified since Ehrenberg in his elaborate and comprehensive work on the infusoria,|and his skepticism was the more justified since Ehrenberg in his elaborate and comprehensive work on the infusoria, +LJ025-0067|had declared the greater number of what are now recognized as locomotive plants to be animals.|had declared the greater number of what are now recognized as locomotive plants to be animals. +LJ025-0068|"At the present day," writes Huxley,|"At the present day," writes Huxley, +LJ025-0069|innumerable plants and free plant cells are known to pass the whole or part of their lives in an actively locomotive condition,|innumerable plants and free plant cells are known to pass the whole or part of their lives in an actively locomotive condition, +LJ025-0070|in nowise distinguishable from that of one of the simpler animals, and|in nowise distinguishable from that of one of the simpler animals, and +LJ025-0071|while in this condition their movements are, to all appearances, as spontaneous -- as much the product of volition -- as those of such animals.|while in this condition their movements are, to all appearances, as spontaneous -- as much the product of volition -- as those of such animals. +LJ025-0072|Hence the teleological argument for Cuvier's first diagnostic character|Hence the teleological argument for Cuvier's first diagnostic character +LJ025-0073|the presence in animals of an alimentary cavity, or internal pocket, in which they can carry about their nutriment|the presence in animals of an alimentary cavity, or internal pocket, in which they can carry about their nutriment +LJ025-0074|has broken down, so far, at least, as his mode of stating it goes. And|has broken down, so far, at least, as his mode of stating it goes. And +LJ025-0075|with the advance of microscopic anatomy, the universality of the fact itself among animals has ceased to be predicable.|with the advance of microscopic anatomy, the universality of the fact itself among animals has ceased to be predicable. +LJ025-0076|Many animals of even complex structure which live parasitically within others are wholly devoid of an alimentary cavity.|Many animals of even complex structure which live parasitically within others are wholly devoid of an alimentary cavity. +LJ025-0077|Their food is provided for them,|Their food is provided for them, +LJ025-0078|not only ready cooked but ready digested, and the alimentary canal, become superfluous, has disappeared, and, again,|not only ready cooked but ready digested, and the alimentary canal, become superfluous, has disappeared, and, again, +LJ025-0079|the males of most Rotifers have no digestive apparatus.|the males of most Rotifers have no digestive apparatus. +LJ025-0080|Finally amid the lowest forms of animal life the speck of gelatinous protoplasm, which constitutes the whole body,|Finally amid the lowest forms of animal life the speck of gelatinous protoplasm, which constitutes the whole body, +LJ025-0081|has no permanent digestive cavity or mouth, but takes in its food anywhere and digests, so to speak, all over its body.|has no permanent digestive cavity or mouth, but takes in its food anywhere and digests, so to speak, all over its body. +LJ025-0082|But although Cuvier's leading diagnosis of the animal from the plant will not stand a strict test,|But although Cuvier's leading diagnosis of the animal from the plant will not stand a strict test, +LJ025-0083|it remains one of the most constant of the distinctive characters of animals.|it remains one of the most constant of the distinctive characters of animals. +LJ025-0084|And if we substitute for the possession of an alimentary cavity the power of taking solid nutriment into the body and there digesting it,|And if we substitute for the possession of an alimentary cavity the power of taking solid nutriment into the body and there digesting it, +LJ025-0085|the definition so changed will cover all animals, except certain parasites,|the definition so changed will cover all animals, except certain parasites, +LJ025-0086|and the few and exceptional cases of non-parasitic animals which do not feed at all.|and the few and exceptional cases of non-parasitic animals which do not feed at all. +LJ025-0087|On the other hand, the definition thus amended will exclude all ordinary vegetable organisms.|On the other hand, the definition thus amended will exclude all ordinary vegetable organisms. +LJ025-0088|Cuvier himself practically gives up his second distinctive mark when he admits that it is wanting in the simpler animals.|Cuvier himself practically gives up his second distinctive mark when he admits that it is wanting in the simpler animals. +LJ025-0089|The third distinction is based on a completely erroneous conception of the chemical differences|The third distinction is based on a completely erroneous conception of the chemical differences +LJ025-0090|and resemblances between the constituents of animal and vegetable organisms, for which Cuvier is not responsible,|and resemblances between the constituents of animal and vegetable organisms, for which Cuvier is not responsible, +LJ025-0092|It is now established that nitrogen is as essential a constituent of vegetable as of animal living matter|It is now established that nitrogen is as essential a constituent of vegetable as of animal living matter +LJ025-0093|and that the latter is, chemically speaking, just as complicated as the former.|and that the latter is, chemically speaking, just as complicated as the former. +LJ025-0094|Starchy substances, cellulose and sugar,|Starchy substances, cellulose and sugar, +LJ025-0095|once supposed to be exclusively confined to plants, are now known to be regular and normal products of animals.|once supposed to be exclusively confined to plants, are now known to be regular and normal products of animals. +LJ025-0096|Amylaceous and saccharine substances are largely manufactured, even by the highest animals.|Amylaceous and saccharine substances are largely manufactured, even by the highest animals. +LJ025-0097|Cellulose is widespread as a constituent of the skeletons of the lower animals|Cellulose is widespread as a constituent of the skeletons of the lower animals +LJ025-0098|and it is probable that amyloid substances are universally present in the animal organism, though not in the precise form of starch.|and it is probable that amyloid substances are universally present in the animal organism, though not in the precise form of starch. +LJ025-0099|Moreover, although it remains true that there is an inverse relation between the green plant in sunshine and the animal,|Moreover, although it remains true that there is an inverse relation between the green plant in sunshine and the animal, +LJ025-0100|in so far as under these circumstances the green plant|in so far as under these circumstances the green plant +LJ025-0101|decomposes carbonic acid and exhales oxygen while the animal absorbs oxygen and exhales carbonic acid,|decomposes carbonic acid and exhales oxygen while the animal absorbs oxygen and exhales carbonic acid, +LJ025-0102|yet the exact researches of the modern chemical investigators of the physiological processes of plants have clearly demonstrated the fallacy|yet the exact researches of the modern chemical investigators of the physiological processes of plants have clearly demonstrated the fallacy +LJ025-0103|of attempting to draw any general distinction between animals and vegetables on this ground. In fact, the difference vanishes with the sunshine,|of attempting to draw any general distinction between animals and vegetables on this ground. In fact, the difference vanishes with the sunshine, +LJ025-0104|even in the case of the green plant, which in the dark absorbs oxygen and gives out carbonic acid like any animal.|even in the case of the green plant, which in the dark absorbs oxygen and gives out carbonic acid like any animal. +LJ025-0105|On the other hand, those plants, such as the fungi,|On the other hand, those plants, such as the fungi, +LJ025-0106|which contain no chlorophyll and are not green, are always, so far as respiration is concerned, in the exact position of animals.|which contain no chlorophyll and are not green, are always, so far as respiration is concerned, in the exact position of animals. +LJ025-0107|They absorb oxygen and give out carbonic acid.|They absorb oxygen and give out carbonic acid. +LJ025-0108|Thus, by the progress of knowledge, Cuvier's fourth distinction between the animal and the plant|Thus, by the progress of knowledge, Cuvier's fourth distinction between the animal and the plant +LJ025-0109|has been as completely invalidated as the third and second, and even the first can be retained only in a modified form|has been as completely invalidated as the third and second, and even the first can be retained only in a modified form +LJ025-0110|and subject to exceptions.|and subject to exceptions. +LJ025-0111|But has the advance of biology simply tended to break down old distinctions without establishing new ones?|But has the advance of biology simply tended to break down old distinctions without establishing new ones? +LJ025-0112|With a qualification, to be considered presently, the answer to this question is undoubtedly in the affirmative.|With a qualification, to be considered presently, the answer to this question is undoubtedly in the affirmative. +LJ025-0113|The famous researches of Schwann and Schleiden in 1837 and the following years|The famous researches of Schwann and Schleiden in eighteen thirty-seven and the following years +LJ025-0114|founded the modern science of histology or that branch of anatomy which deals with the ultimate visible structure of organisms as revealed by the microscope,|founded the modern science of histology or that branch of anatomy which deals with the ultimate visible structure of organisms as revealed by the microscope, +LJ025-0115|and from that day to this the rapid improvement of methods of investigation and the energy of a host of accurate observers|and from that day to this the rapid improvement of methods of investigation and the energy of a host of accurate observers +LJ025-0116|have given greater and greater breadth and firmness to Schwann's great generalization|have given greater and greater breadth and firmness to Schwann's great generalization +LJ025-0117|that a fundamental unity of structure obtains in animals and plants,|that a fundamental unity of structure obtains in animals and plants, +LJ025-0118|and that, however diverse may be the fabrics or tissues of which their bodies are composed, all these varied structures result|and that, however diverse may be the fabrics or tissues of which their bodies are composed, all these varied structures result +LJ025-0119|from the metamorphosis of morphological units (termed cells in a more general sense than that in which the word "cells" was at first employed),|from the metamorphosis of morphological units (termed cells in a more general sense than that in which the word "cells" was at first employed), +LJ025-0120|which are not only similar in animals and in plants respectively,|which are not only similar in animals and in plants respectively, +LJ025-0121|but present a close resemblance when those of animals and those of plants are compared together.|but present a close resemblance when those of animals and those of plants are compared together. +LJ025-0122|The contractility which is the fundamental condition of locomotion," continues Huxley,|The contractility which is the fundamental condition of locomotion," continues Huxley, +LJ025-0123|has not only been discovered to exist far more widely among plants than was formerly imagined,|has not only been discovered to exist far more widely among plants than was formerly imagined, +LJ025-0124|but in the plants the act of contraction has been found to be accompanied,|but in the plants the act of contraction has been found to be accompanied, +LJ025-0125|as Dr. Burdon Sanderson's interesting investigations have shown, by a disturbance of the electrical state of the contractile substance|as Dr. Burdon Sanderson's interesting investigations have shown, by a disturbance of the electrical state of the contractile substance +LJ025-0126|comparable to that which was found by Du Bois Reymond to be a concomitant of the activity of ordinary muscle in animals. Again,|comparable to that which was found by Du Bois Reymond to be a concomitant of the activity of ordinary muscle in animals. Again, +LJ025-0127|I know of no test by which the reaction of the leaves of the Sundew and of other plants to stimuli, so fully and carefully studied by Mr. Darwin,|I know of no test by which the reaction of the leaves of the Sundew and of other plants to stimuli, so fully and carefully studied by Mr. Darwin, +LJ025-0128|can be distinguished from those acts of contraction following upon stimuli, which are called "reflex" in animals.|can be distinguished from those acts of contraction following upon stimuli, which are called "reflex" in animals. +LJ025-0129|On each lobe of the bi-lobed leaf of Venus flytrap are three delicate filaments which stand out at right angles from the surface of the leaf.|On each lobe of the bi-lobed leaf of Venus flytrap are three delicate filaments which stand out at right angles from the surface of the leaf. +LJ025-0130|Touch one of them with the end of a fine human hair|Touch one of them with the end of a fine human hair +LJ025-0131|and the lobes of the leaf instantly close together in virtue of an act of contraction of part of their substance,|and the lobes of the leaf instantly close together in virtue of an act of contraction of part of their substance, +LJ025-0132|just as the body of a snail contracts into its shell when one of its "horns" is irritated.|just as the body of a snail contracts into its shell when one of its "horns" is irritated. +LJ025-0133|The reflex action of the snail is the result of the presence of a nervous system in the animal.|The reflex action of the snail is the result of the presence of a nervous system in the animal. +LJ025-0134|A molecular change takes place in the nerve of the tentacle,|A molecular change takes place in the nerve of the tentacle, +LJ025-0135|is propagated to the muscles by which the body is retracted, and causing them to contract, the act of retraction is brought about.|is propagated to the muscles by which the body is retracted, and causing them to contract, the act of retraction is brought about. +LJ025-0136|Of course the similarity of the acts does not necessarily involve the conclusion that the mechanism by which they are effected is the same|Of course the similarity of the acts does not necessarily involve the conclusion that the mechanism by which they are effected is the same +LJ025-0137|but it suggests a suspicion of their identity which needs careful testing.|but it suggests a suspicion of their identity which needs careful testing. +LJ025-0138|The results of inquiries into the structure of the nervous system of animals|The results of inquiries into the structure of the nervous system of animals +LJ025-0139|converge toward the conclusion that the nerve fibers, which have been regarded as ultimate elements of nervous tissue,|converge toward the conclusion that the nerve fibers, which have been regarded as ultimate elements of nervous tissue, +LJ025-0140|are not such, but are simply the visible aggregations of vastly more attenuated filaments,|are not such, but are simply the visible aggregations of vastly more attenuated filaments, +LJ025-0141|the diameter of which dwindles down to the limits of our present microscopic vision,|the diameter of which dwindles down to the limits of our present microscopic vision, +LJ025-0142|greatly as these have been extended by modern improvements of the microscope, and that a nerve is, in its essence,|greatly as these have been extended by modern improvements of the microscope, and that a nerve is, in its essence, +LJ025-0143|nothing but a linear tract of specially modified protoplasm between two points of an organism|nothing but a linear tract of specially modified protoplasm between two points of an organism +LJ025-0144|one of which is able to affect the other by means of the communication so established.|one of which is able to affect the other by means of the communication so established. +LJ025-0145|Hence it is conceivable that even the simplest living being may possess a nervous system.|Hence it is conceivable that even the simplest living being may possess a nervous system. +LJ025-0146|And the question whether plants are provided with a nervous system or not thus acquires a new aspect and presents the histologist and physiologist|And the question whether plants are provided with a nervous system or not thus acquires a new aspect and presents the histologist and physiologist +LJ025-0147|with a problem of extreme difficulty, which must be attacked from a new point of view and by the aid of methods which have yet to be invented.|with a problem of extreme difficulty, which must be attacked from a new point of view and by the aid of methods which have yet to be invented. +LJ025-0148|"Thus it must be admitted," he says again, "that plants may be contractile and locomotive;|"Thus it must be admitted," he says again, "that plants may be contractile and locomotive; +LJ025-0149|that, while locomotive, their movements may have as much appearance of spontaneity as those of the lowest animals,|that, while locomotive, their movements may have as much appearance of spontaneity as those of the lowest animals, +LJ025-0150|and that many exhibit actions comparable to those which are brought about by the agency of a nervous system in animals.|and that many exhibit actions comparable to those which are brought about by the agency of a nervous system in animals. +LJ025-0151|And it must be allowed to be possible that further research may reveal the existence of something comparable to a nervous system in plants.|And it must be allowed to be possible that further research may reveal the existence of something comparable to a nervous system in plants. +LJ025-0152|So that I know not where we can hope to find any absolute distinction between animals and plants, unless we return to their mode of nutrition|So that I know not where we can hope to find any absolute distinction between animals and plants, unless we return to their mode of nutrition +LJ025-0153|and inquire whether certain differences of a more occult character than those imagined to exist by Cuvier,|and inquire whether certain differences of a more occult character than those imagined to exist by Cuvier, +LJ025-0154|and which certainly hold good for the vast majority of animals and plants, are of universal application.|and which certainly hold good for the vast majority of animals and plants, are of universal application. +LJ025-0155|A bean may be supplied with water in which salts of ammonia and certain other mineral salts are dissolved in due proportion,|A bean may be supplied with water in which salts of ammonia and certain other mineral salts are dissolved in due proportion, +LJ025-0156|with atmospheric air containing its ordinary minute dose of carbonic acid and with nothing else but sunlight and heat.|with atmospheric air containing its ordinary minute dose of carbonic acid and with nothing else but sunlight and heat. +LJ025-0157|Under these circumstances, unnatural as they are, with proper management, the bean will thrust forth its radicle and its plumule;|Under these circumstances, unnatural as they are, with proper management, the bean will thrust forth its radicle and its plumule; +LJ025-0158|the former will grow down into roots, the latter grow up into the stem and leaves of a vigorous bean-plant,|the former will grow down into roots, the latter grow up into the stem and leaves of a vigorous bean-plant, +LJ025-0159|and this plant will, in due time, flower and produce its crop of beans just as if it were grown in the garden or in the field.|and this plant will, in due time, flower and produce its crop of beans just as if it were grown in the garden or in the field. +LJ025-0160|The weight of the nitrogenous protein compounds,|The weight of the nitrogenous protein compounds, +LJ025-0161|of the oily, starchy, saccharine and woody substances contained in the full-grown plant and its seeds|of the oily, starchy, saccharine and woody substances contained in the full-grown plant and its seeds +LJ025-0162|will be vastly greater than the weight of the same substances contained in the bean from which it sprang.|will be vastly greater than the weight of the same substances contained in the bean from which it sprang. +LJ025-0163|But nothing has been supplied to the bean save water, carbonic acid, ammonia, potash, lime,|But nothing has been supplied to the bean save water, carbonic acid, ammonia, potash, lime, +LJ025-0164|iron and the like in combination with phosphoric, sulphuric and other acids.|iron and the like in combination with phosphoric, sulphuric and other acids. +LJ025-0165|Neither protein, nor fat, nor starch, nor sugar, nor any substance in the slightest degree resembling them has formed part of the food of the bean.|Neither protein, nor fat, nor starch, nor sugar, nor any substance in the slightest degree resembling them has formed part of the food of the bean. +LJ025-0166|But the weights of the carbon, hydrogen, oxygen, nitrogen, phosphorus, sulfur and other elementary bodies contained in the bean-plant|But the weights of the carbon, hydrogen, oxygen, nitrogen, phosphorus, sulfur and other elementary bodies contained in the bean-plant +LJ025-0167|and in the seeds which it produces are exactly equivalent to the weights of the same elements which have disappeared from the materials supplied to the bean during its growth.|and in the seeds which it produces are exactly equivalent to the weights of the same elements which have disappeared from the materials supplied to the bean during its growth. +LJ025-0168|Whence it follows that the bean has taken in only the raw materials of its fabric and has manufactured them into bean-stuffs.|Whence it follows that the bean has taken in only the raw materials of its fabric and has manufactured them into bean-stuffs. +LJ025-0169|The bean has been able to perform this great chemical feat by the help of its green coloring matter, or chlorophyll,|The bean has been able to perform this great chemical feat by the help of its green coloring matter, or chlorophyll, +LJ025-0170|for it is only the green parts of the plant which, under the influence of sunlight, have the marvelous power of decomposing carbonic acid,|for it is only the green parts of the plant which, under the influence of sunlight, have the marvelous power of decomposing carbonic acid, +LJ025-0171|setting free the oxygen and laying hold of the carbon which it contains.|setting free the oxygen and laying hold of the carbon which it contains. +LJ025-0172|In fact, the bean obtains two of the absolutely indispensable elements of its substance from two distinct sources.|In fact, the bean obtains two of the absolutely indispensable elements of its substance from two distinct sources. +LJ025-0173|The watery solution, in which its roots are plunged, contains nitrogen but no carbon;|The watery solution, in which its roots are plunged, contains nitrogen but no carbon; +LJ025-0174|the air, to which the leaves are exposed, contains carbon,|the air, to which the leaves are exposed, contains carbon, +LJ025-0175|but its nitrogen is in the state of a free gas, in which condition the bean can make no use of it,|but its nitrogen is in the state of a free gas, in which condition the bean can make no use of it, +LJ025-0176|and the chlorophyll is the apparatus by which the carbon is extracted from the atmospheric carbonic acid, the leaves being the chief laboratories|and the chlorophyll is the apparatus by which the carbon is extracted from the atmospheric carbonic acid, the leaves being the chief laboratories +LJ026-0001|The Science: History of the Universe, Volume 5. Edited by Francis Rolt-Wheeler. Biology. Chapter 8.|The Science: History of the Universe, Volume five. Edited by Francis Rolt-Wheeler. Biology. Chapter eight. +LJ026-0002|Life Processes. Part One.|Life Processes. Part One. +LJ026-0003|Nutrition thus, as has been pointed out, makes it possible to classify most organisms as animals or plants.|Nutrition thus, as has been pointed out, makes it possible to classify most organisms as animals or plants. +LJ026-0004|Yet there are many unicellular forms in which both kinds of nutrition go on at the same time;|Yet there are many unicellular forms in which both kinds of nutrition go on at the same time; +LJ026-0005|that is, the forms may possess a mouth for the ingestion of solid food and green coloring matter, chlorophyll,|that is, the forms may possess a mouth for the ingestion of solid food and green coloring matter, chlorophyll, +LJ026-0006|for the manufacture of starchy food from gaseous matter.|for the manufacture of starchy food from gaseous matter. +LJ026-0007|Many of the lowest forms of life have long been puzzles|Many of the lowest forms of life have long been puzzles +LJ026-0008|and the beginner in biological study is surprised to find them described in textbooks of both botany and zoology.|and the beginner in biological study is surprised to find them described in textbooks of both botany and zoology. +LJ026-0009|The fact is that they are on the border line, are neither plants nor animals but simply organisms.|The fact is that they are on the border line, are neither plants nor animals but simply organisms. +LJ026-0010|Since they cannot be classified, it is necessary that they be listed both under botany and zoology, in order to make sure that they will not be omitted entirely.|Since they cannot be classified, it is necessary that they be listed both under botany and zoology, in order to make sure that they will not be omitted entirely. +LJ026-0011|Because of these uncertain forms of life,|Because of these uncertain forms of life, +LJ026-0012|Haeckel proposed once to include all one-celled animals and plants in a third kingdom to be called Protista (meaning the first of all life).|Haeckel proposed once to include all one-celled animals and plants in a third kingdom to be called Protista (meaning the first of all life). +LJ026-0013|Parker's definition of animals and plants,|Parker's definition of animals and plants, +LJ026-0014|based on the foregoing considerations, is convenient for distinguishing between animals and plants in all cases except the doubtful unicellar forms.|based on the foregoing considerations, is convenient for distinguishing between animals and plants in all cases except the doubtful unicellar forms. +LJ026-0015|He says: Animals are organisms of fixed and definite form, in which the cell-body is not covered with a cellulose wall.|He says: Animals are organisms of fixed and definite form, in which the cell-body is not covered with a cellulose wall. +LJ026-0016|They ingest solid proteinaceous food,|They ingest solid proteinaceous food, +LJ026-0017|their nutritive processes result in oxidation, they have a definite organ of excretion and are capable of automatic movement|their nutritive processes result in oxidation, they have a definite organ of excretion and are capable of automatic movement +LJ026-0018|Plants are organisms of constantly varying form in which the cell body is surrounded by a cellulose wall;|Plants are organisms of constantly varying form in which the cell body is surrounded by a cellulose wall; +LJ026-0019|they cannot ingest solid food, but are nourished by a watery solution of nutrient materials.|they cannot ingest solid food, but are nourished by a watery solution of nutrient materials. +LJ026-0020|If chlorophyll is present, the carbon dioxide of the air serves as a source of carbon,|If chlorophyll is present, the carbon dioxide of the air serves as a source of carbon, +LJ026-0021|nitrogen is obtained from simple salts and the nutritive processes result in deoxidation;|nitrogen is obtained from simple salts and the nutritive processes result in deoxidation; +LJ026-0022|if chlorophyll is absent, carbon is obtained from sugar or some similar compound,|if chlorophyll is absent, carbon is obtained from sugar or some similar compound, +LJ026-0023|nitrogen either from simple salts or from proteids, and the process of nutrition is one of oxidation.|nitrogen either from simple salts or from proteids, and the process of nutrition is one of oxidation. +LJ026-0024|There is no special excretory organ, and, except in the case of certain reproductive bodies, there is usually no locomotion.|There is no special excretory organ, and, except in the case of certain reproductive bodies, there is usually no locomotion. +LJ026-0025|The important point to recognize is that these boundaries are artificial and that there are no scientific frontiers in Nature.|The important point to recognize is that these boundaries are artificial and that there are no scientific frontiers in Nature. +LJ026-0026|As in the liquefaction of gases, there is a "critical point" at which the substance under experiment is neither gaseous nor liquid.|As in the liquefaction of gases, there is a "critical point" at which the substance under experiment is neither gaseous nor liquid. +LJ026-0027|as in a mountainous country, it is impossible to say where mountain ends and valley begins; as in the development of an animal,|as in a mountainous country, it is impossible to say where mountain ends and valley begins; as in the development of an animal, +LJ026-0028|it is futile to argue about the exact period when, for instance, the egg becomes a tadpole or the tadpole a frog,|it is futile to argue about the exact period when, for instance, the egg becomes a tadpole or the tadpole a frog, +LJ026-0030|The distinction between the higher plants and animals is perfectly sharp and obvious,|The distinction between the higher plants and animals is perfectly sharp and obvious, +LJ026-0031|but when the two groups are traced downward they are found gradually to merge,|but when the two groups are traced downward they are found gradually to merge, +LJ026-0032|as it were, into an assemblage of organisms which partake of the characters of both kingdoms and cannot without a certain violence|as it were, into an assemblage of organisms which partake of the characters of both kingdoms and cannot without a certain violence +LJ026-0033|be either included in or excluded from either.|be either included in or excluded from either. +LJ026-0034|When any given "protist" has to be classified the case must be decided on its individual merits;|When any given "protist" has to be classified the case must be decided on its individual merits; +LJ026-0035|the organism must be compared in detail with all those which resemble it closely in structure, physiology and life history,|the organism must be compared in detail with all those which resemble it closely in structure, physiology and life history, +LJ026-0036|and then a balance must be struck and the doubtful form placed in the kingdom with which it has, on the whole, most points in common.|and then a balance must be struck and the doubtful form placed in the kingdom with which it has, on the whole, most points in common. +LJ026-0037|It will no doubt occur to the reader that, on the theory of evolution,|It will no doubt occur to the reader that, on the theory of evolution, +LJ026-0038|the fact of the animal and vegetable kingdoms being related to one another like two trees united at the roots may be accounted for by the hypothesis that|the fact of the animal and vegetable kingdoms being related to one another like two trees united at the roots may be accounted for by the hypothesis that +LJ026-0040|And in this connection the fact that some bacteria -- the simplest organisms known and devoid of chlorophyll|And in this connection the fact that some bacteria -- the simplest organisms known and devoid of chlorophyll +LJ026-0041|may flourish in solutions wholly devoid of organic matter is very significant.|may flourish in solutions wholly devoid of organic matter is very significant. +LJ026-0042|The lower plants and animals referred to above are so far from everyday observation|The lower plants and animals referred to above are so far from everyday observation +LJ026-0043|and hence so unfamiliar that to most people the comparison made will mean little in terms of ordinary green flowering plants and common vertebrate animals.|and hence so unfamiliar that to most people the comparison made will mean little in terms of ordinary green flowering plants and common vertebrate animals. +LJ026-0044|In order to emphasize the fundamental similarity of organic function in higher and lower animals and plants, let us compare any higher plant|In order to emphasize the fundamental similarity of organic function in higher and lower animals and plants, let us compare any higher plant +LJ026-0045|e.g., a bean plant with a higher animal, e.g., frog or even man.|e.g., a bean plant with a higher animal, e.g., frog or even man. +LJ026-0046|In each the life is the sum total of a series of definite processes -- nutrition or food supply,|In each the life is the sum total of a series of definite processes -- nutrition or food supply, +LJ026-0047|circulation, metabolism, excretion, oxygenation (part of respiration),|circulation, metabolism, excretion, oxygenation (part of respiration), +LJ026-0048|movement, irritability (nervous activity) and reproduction.|movement, irritability (nervous activity) and reproduction. +LJ026-0049|In turn these will be compared for the animal and the plant,|In turn these will be compared for the animal and the plant, +LJ026-0050|following in part the comparisons of certain animals and plants by Sedgwick and Wilson and others.|following in part the comparisons of certain animals and plants by Sedgwick and Wilson and others. +LJ026-0051|These comparisons will, however, be translated into terms applicable to any species of higher plants or animals.|These comparisons will, however, be translated into terms applicable to any species of higher plants or animals. +LJ026-0052|In the nutrition of the animal the most essential and characteristic part of the food supply is derived from vegetable|In the nutrition of the animal the most essential and characteristic part of the food supply is derived from vegetable +LJ026-0053|or animal matter in the form of various organic compounds, of which the most important are proteids (protoplasm, albumen, etc.),|or animal matter in the form of various organic compounds, of which the most important are proteids (protoplasm, albumen, etc.), +LJ026-0054|carbohydrates (starch, cellulose) and fats.|carbohydrates (starch, cellulose) and fats. +LJ026-0055|These materials are used by the animal in the manufacture of new protoplasm to take the place of that which has been used up.|These materials are used by the animal in the manufacture of new protoplasm to take the place of that which has been used up. +LJ026-0056|It is, however, impossible for the animal to build these materials directly into the substance of its own body.|It is, however, impossible for the animal to build these materials directly into the substance of its own body. +LJ026-0057|They must first undergo certain preparatory chemical changes known collectively as digestion|They must first undergo certain preparatory chemical changes known collectively as digestion +LJ026-0058|and only after the completion of this process can all the food be absorbed into the circulation.|and only after the completion of this process can all the food be absorbed into the circulation. +LJ026-0059|For this purpose the food is taken not into the body proper, but into a kind of tubular chemical laboratory; called the alimentary canal,|For this purpose the food is taken not into the body proper, but into a kind of tubular chemical laboratory; called the alimentary canal, +LJ026-0060|through which it slowly passes, being subjected meanwhile to the action of certain chemical substances or reagents, known as digestive ferments.|through which it slowly passes, being subjected meanwhile to the action of certain chemical substances or reagents, known as digestive ferments. +LJ026-0061|These substances, which are dissolved in a watery liquid to form the digestive fluid, are secreted by the walls of the alimentary tube.|These substances, which are dissolved in a watery liquid to form the digestive fluid, are secreted by the walls of the alimentary tube. +LJ026-0062|Through their action the solid portions are liquefied and the food is rendered capable of absorption into the body proper.|Through their action the solid portions are liquefied and the food is rendered capable of absorption into the body proper. +LJ026-0063|The food supply of the higher plant, like that of the animal, is the source of the required matter and energy,|The food supply of the higher plant, like that of the animal, is the source of the required matter and energy, +LJ026-0065|Matter enters the plant in the liquid or gaseous form by diffusion,|Matter enters the plant in the liquid or gaseous form by diffusion, +LJ026-0066|both from the soil through the roots (liquids) and from the atmosphere through the leaves (gases).|both from the soil through the roots (liquids) and from the atmosphere through the leaves (gases). +LJ026-0067|We have here the direct absorption into the body proper of food-stuffs precisely as the animal takes in water and oxygen.|We have here the direct absorption into the body proper of food-stuffs precisely as the animal takes in water and oxygen. +LJ026-0068|Energy enters the plant, to a small extent,|Energy enters the plant, to a small extent, +LJ026-0069|as the potential energy of foodstuffs, but comes in principally as the kinetic energy of sunlight absorbed in the leaves.|as the potential energy of foodstuffs, but comes in principally as the kinetic energy of sunlight absorbed in the leaves. +LJ026-0070|Of the substances the solids (salts, etc.) must be dissolved in water before they can be taken in.|Of the substances the solids (salts, etc.) must be dissolved in water before they can be taken in. +LJ026-0071|Water and dissolved salts continually pass by diffusion from the soil into the roots, where together they constitute the sap.|Water and dissolved salts continually pass by diffusion from the soil into the roots, where together they constitute the sap. +LJ026-0072|The sap travels throughout the whole plant, the main though not the only cause of movement|The sap travels throughout the whole plant, the main though not the only cause of movement +LJ026-0073|being the constant transpiration (evaporation) of watery vapor from the leaves, especially through the stomata.|being the constant transpiration (evaporation) of watery vapor from the leaves, especially through the stomata. +LJ026-0074|The gaseous matters (carbon dioxide, oxygen, nitrogen) enter the plant mainly by diffusion from the atmosphere,|The gaseous matters (carbon dioxide, oxygen, nitrogen) enter the plant mainly by diffusion from the atmosphere, +LJ026-0075|are dissolved by the sap in the leaves and elsewhere and thus may pass to every portion of the plant.|are dissolved by the sap in the leaves and elsewhere and thus may pass to every portion of the plant. +LJ026-0076|The green plant owes its power of absorbing the energy of sunlight|The green plant owes its power of absorbing the energy of sunlight +LJ026-0077|to the chlorophyll bodies or chromatophores, for plants which, like fungi, etc., are devoid of chlorophyll, are unable thus to acquire energy.|to the chlorophyll bodies or chromatophores, for plants which, like fungi, etc., are devoid of chlorophyll, are unable thus to acquire energy. +LJ026-0078|Entering the chlorophyll bodies, the kinetic energy of sunlight is applied to the decomposition of carbon dioxide and water.|Entering the chlorophyll bodies, the kinetic energy of sunlight is applied to the decomposition of carbon dioxide and water. +LJ026-0079|After passing through manifold but imperfectly known processes, the elements of these substances finally reappear as starch,|After passing through manifold but imperfectly known processes, the elements of these substances finally reappear as starch, +LJ026-0080|often in the form of granules embedded in the chlorophyll bodies and free oxygen, most of which is returned to the atmosphere.|often in the form of granules embedded in the chlorophyll bodies and free oxygen, most of which is returned to the atmosphere. +LJ026-0081|Thus the leaf of a green plant in the light is continually absorbing carbon dioxide and giving forth free oxygen.|Thus the leaf of a green plant in the light is continually absorbing carbon dioxide and giving forth free oxygen. +LJ026-0082|Carbon dioxide and water contain no potential energy, since the affinities of their constituent elements are completely satisfied.|Carbon dioxide and water contain no potential energy, since the affinities of their constituent elements are completely satisfied. +LJ026-0083|Starch, however, contains potential energy, since the molecule is relatively unstable|Starch, however, contains potential energy, since the molecule is relatively unstable +LJ026-0084|i.e., capable of decomposition into simpler, stabler molecules in which stronger affinities are satisfied.|i.e., capable of decomposition into simpler, stabler molecules in which stronger affinities are satisfied. +LJ026-0085|And this is due to the fact that in the manufacture of starch in the chlorophyll bodies|And this is due to the fact that in the manufacture of starch in the chlorophyll bodies +LJ026-0086|the kinetic energy of sunlight was expended in lifting the atoms into position of vantage, thus endowing them with energy of position.|the kinetic energy of sunlight was expended in lifting the atoms into position of vantage, thus endowing them with energy of position. +LJ026-0087|In this way some of the radiant and kinetic energy of the sun comes to be stored up as potential energy in the starch.|In this way some of the radiant and kinetic energy of the sun comes to be stored up as potential energy in the starch. +LJ026-0088|In short, the green plant is able by cooperation with sunlight to use simple raw materials (carbon dioxide, water, oxygen, etc.)|In short, the green plant is able by cooperation with sunlight to use simple raw materials (carbon dioxide, water, oxygen, etc.) +LJ026-0089|poor in energy or devoid of it, and out of them to manufacture food -- i.e. complex compounds rich in available potential energy.|poor in energy or devoid of it, and out of them to manufacture food -- i.e. complex compounds rich in available potential energy. +LJ026-0090|This power is possessed by green plants alone; all other organisms being dependent for energy upon the potential energy of ready-made food.|This power is possessed by green plants alone; all other organisms being dependent for energy upon the potential energy of ready-made food. +LJ026-0091|This must, in the first instance, be provided for them by green plants, and hence without chlorophyll-bearing plants,|This must, in the first instance, be provided for them by green plants, and hence without chlorophyll-bearing plants, +LJ026-0092|animals (and colorless plants as well) apparently could not long exist.|animals (and colorless plants as well) apparently could not long exist. +LJ026-0093|The plant absorbs also a small amount of kinetic energy, independently of the sunlight, in the form of heat.|The plant absorbs also a small amount of kinetic energy, independently of the sunlight, in the form of heat. +LJ026-0094|This, however, is probably not a source of vital energy, but only contributes to the maintenance of the body temperature.|This, however, is probably not a source of vital energy, but only contributes to the maintenance of the body temperature. +LJ026-0095|Food (starch) thus produced in the green leaves of higher plants and the inorganic foods|Food (starch) thus produced in the green leaves of higher plants and the inorganic foods +LJ026-0096|water, nitrites or nitrates and various mineral substances in solution in water|water, nitrites or nitrates and various mineral substances in solution in water +LJ026-0097|furnish the materials and energy required for the life and growth of the plant.|furnish the materials and energy required for the life and growth of the plant. +LJ026-0098|The circulatory system distributes these foods. In animals foods prepared for absorption in the stomach and intestine (by digestion)|The circulatory system distributes these foods. In animals foods prepared for absorption in the stomach and intestine (by digestion) +LJ026-0099|are absorbed by the circulating liquids (blood and lymph) and transported to all cells of the animal body.|are absorbed by the circulating liquids (blood and lymph) and transported to all cells of the animal body. +LJ026-0100|In the plant the inorganic matter in water from the soil are absorbed by the roots and carried up definite tubes in the woody part of the stem.|In the plant the inorganic matter in water from the soil are absorbed by the roots and carried up definite tubes in the woody part of the stem. +LJ026-0101|The causes of this ascent are not clear,|The causes of this ascent are not clear, +LJ026-0102|but root pressure due to osmosis, capillary action and evaporation from the leaves are factors.|but root pressure due to osmosis, capillary action and evaporation from the leaves are factors. +LJ026-0103|Just as the solid food of animals must be digested in preparation for absorption,|Just as the solid food of animals must be digested in preparation for absorption, +LJ026-0104|so starch manufactured in the leaves must be digested (dissolved) before it can be transported.|so starch manufactured in the leaves must be digested (dissolved) before it can be transported. +LJ026-0106|The change is from starch to a sugar capable of diffusion.|The change is from starch to a sugar capable of diffusion. +LJ026-0107|Dissolved in water, the sugar is transported down delicate tubes, chiefly in the growing bark region of the stem.|Dissolved in water, the sugar is transported down delicate tubes, chiefly in the growing bark region of the stem. +LJ026-0108|It is clear that there are upward and downward currents of water containing food (comparable to blood of an animal),|It is clear that there are upward and downward currents of water containing food (comparable to blood of an animal), +LJ026-0109|but no system of complete circulation as in the blood vessels of a higher animal.|but no system of complete circulation as in the blood vessels of a higher animal. +LJ026-0110|However, the result in distributed food is the same in the plant and in the animal.|However, the result in distributed food is the same in the plant and in the animal. +LJ026-0111|In the cells the foods undergo metabolic changes.|In the cells the foods undergo metabolic changes. +LJ026-0112|In an animal the foods in the circulating liquids, blood and lymph, are selected and absorbed by the cells.|In an animal the foods in the circulating liquids, blood and lymph, are selected and absorbed by the cells. +LJ026-0113|Only proteid foods form new protoplasm|Only proteid foods form new protoplasm +LJ026-0114|and even of proteids only a limited amount, 75 to one hundred grams a day for a man, is built into new protoplasm.|and even of proteids only a limited amount, seventy-five to one hundred grams a day for a man, is built into new protoplasm. +LJ026-0115|The excess undergoes oxidation and forms nitrogen excretions.|The excess undergoes oxidation and forms nitrogen excretions. +LJ026-0116|The foods containing only the elements carbon, hydrogen and oxygen (fats and carbohydrates)|The foods containing only the elements carbon, hydrogen and oxygen (fats and carbohydrates) +LJ026-0117|are directly oxidized to excretions and, lacking nitrogen, cannot serve for making new animal protoplasm.|are directly oxidized to excretions and, lacking nitrogen, cannot serve for making new animal protoplasm. +LJ026-0118|Fat and carbohydrate foods, then, never become living matter.|Fat and carbohydrate foods, then, never become living matter. +LJ026-0119|They may be stored, especially as fat, until needed for oxidation to supply energy.|They may be stored, especially as fat, until needed for oxidation to supply energy. +LJ026-0120|The building up of the protoplasm from proteids is anabolism, constructive metabolism.|The building up of the protoplasm from proteids is anabolism, constructive metabolism. +LJ026-0121|The destruction of protoplasm, excess proteids or the fat and carbohydrate foods is catabolism, destructive metabolism.|The destruction of protoplasm, excess proteids or the fat and carbohydrate foods is catabolism, destructive metabolism. +LJ026-0122|Catabolism is probably due to enzyme action, but the final result is chiefly carbon dioxide and water,|Catabolism is probably due to enzyme action, but the final result is chiefly carbon dioxide and water, +LJ026-0123|which could be derived by the ordinary chemical evolution of protoplasm, proteid, sugar, starch or fats.|which could be derived by the ordinary chemical evolution of protoplasm, proteid, sugar, starch or fats. +LJ026-0124|In the plant, starch, as has been seen, is first formed in the chlorophyll-bodies.|In the plant, starch, as has been seen, is first formed in the chlorophyll-bodies. +LJ026-0125|But the formation of starch, all important as it is, is after all only the manufacture of food|But the formation of starch, all important as it is, is after all only the manufacture of food +LJ026-0126|as a preliminary to the real processes of nutrition.|as a preliminary to the real processes of nutrition. +LJ026-0127|These processes must take place everywhere in ordinary protoplasm,|These processes must take place everywhere in ordinary protoplasm, +LJ026-0128|for it is here that oxidation occurs and the need for a renewal of matter and energy consequently arises.|for it is here that oxidation occurs and the need for a renewal of matter and energy consequently arises. +LJ026-0129|Sooner or later the starch grains are changed into a kind of sugar (glucose), which, unlike starch, dissolves in the sap|Sooner or later the starch grains are changed into a kind of sugar (glucose), which, unlike starch, dissolves in the sap +LJ026-0130|and may thus be easily transported to all parts of the plant.|and may thus be easily transported to all parts of the plant. +LJ026-0131|Wherever there is need for new protoplasm, whether to repair previous waste or to supply materials for growth,|Wherever there is need for new protoplasm, whether to repair previous waste or to supply materials for growth, +LJ026-0132|after absorption into the cells the elements of the starch (or glucose) are, by the living protoplasm, in some unknown way|after absorption into the cells the elements of the starch (or glucose) are, by the living protoplasm, in some unknown way +LJ026-0133|combined with nitrogen and sulphur (probably also with salts, water, etc.) to form proteid matter.|combined with nitrogen and sulphur (probably also with salts, water, etc.) to form proteid matter. +LJ026-0134|The particles of this newly formed compound are incorporated into the protoplasm.|The particles of this newly formed compound are incorporated into the protoplasm. +LJ026-0135|If a larger quantity of starch is formed in the chlorophyll bodies than is immediately needed by the protoplasm for purposes of repair or growth,|If a larger quantity of starch is formed in the chlorophyll bodies than is immediately needed by the protoplasm for purposes of repair or growth, +LJ026-0136|it may be reconverted into starch after journeying as glucose through the plant|it may be reconverted into starch after journeying as glucose through the plant +LJ026-0138|Apparently when this reserve supply is finally needed at any point in the plant, it is again changed to glucose and transported thither.|Apparently when this reserve supply is finally needed at any point in the plant, it is again changed to glucose and transported thither. +LJ026-0139|It is probable that new leaves and new tissues generally are always formed in part from this reserve starch.|It is probable that new leaves and new tissues generally are always formed in part from this reserve starch. +LJ026-0140|In the plant as in the animal metabolism must consist of anabolic and catabolic processes.|In the plant as in the animal metabolism must consist of anabolic and catabolic processes. +LJ026-0141|The construction in the cells of new proteid from the absorbed carbohydrate and the materials from the soil is true anabolism.|The construction in the cells of new proteid from the absorbed carbohydrate and the materials from the soil is true anabolism. +LJ026-0142|It is also clear that catabolism or oxidation for the liberation of energy occurs as in animals, but this process is slower.|It is also clear that catabolism or oxidation for the liberation of energy occurs as in animals, but this process is slower. +LJ026-0143|Probably foods containing carbon, hydrogen and oxygen are the sources of energy in the higher plants as in animals.|Probably foods containing carbon, hydrogen and oxygen are the sources of energy in the higher plants as in animals. +LJ026-0144|In both plants and animals simple waste substances result from the catabolic processes in the cells.|In both plants and animals simple waste substances result from the catabolic processes in the cells. +LJ026-0145|In the animal carbon dioxide, water and nitrogen compounds are the chief excretions.|In the animal carbon dioxide, water and nitrogen compounds are the chief excretions. +LJ026-0146|They are absorbed by the circulating liquids and carried to the eliminating organs, lungs and kidneys chiefly, for elimination.|They are absorbed by the circulating liquids and carried to the eliminating organs, lungs and kidneys chiefly, for elimination. +LJ026-0147|In the higher plants the excretions are carbon dioxide, which escapes through the epidermis of root, stem and leaf and through the stomata;|In the higher plants the excretions are carbon dioxide, which escapes through the epidermis of root, stem and leaf and through the stomata; +LJ026-0148|water which is lost by evaporation, especially from the leaf surface through the stomata;|water which is lost by evaporation, especially from the leaf surface through the stomata; +LJ026-0149|excretions which are lost by osmosis through the roots and the accumulated but useless mineral substances which are eliminated by leaf fall.|excretions which are lost by osmosis through the roots and the accumulated but useless mineral substances which are eliminated by leaf fall. +LJ026-0150|In both animals and plants oxygen is essential to the catabolic part of metabolism.|In both animals and plants oxygen is essential to the catabolic part of metabolism. +LJ026-0151|Hence oxygen must be supplied to the cells.|Hence oxygen must be supplied to the cells. +LJ026-0152|Oxygenation is the term used to denote the oxygen-supplying part of respiration;|Oxygenation is the term used to denote the oxygen-supplying part of respiration; +LJ026-0153|the other part of respiration, elimination of carbon dioxide, has been treated under excretions.|the other part of respiration, elimination of carbon dioxide, has been treated under excretions. +LJ026-0154|In the animal oxygen is absorbed by the blood, in excess by the hemoglobin of the red cells of the blood|In the animal oxygen is absorbed by the blood, in excess by the hemoglobin of the red cells of the blood +LJ026-0155|and later is absorbed from the blood and lymph by all the living cells.|and later is absorbed from the blood and lymph by all the living cells. +LJ026-0156|In the plant also oxygen is absorbed through the epidermis and stomata from the air.|In the plant also oxygen is absorbed through the epidermis and stomata from the air. +LJ026-0157|This process is, however, obscured during the day because of the oxygen freed in the manufacture of starch which goes on at that time.|This process is, however, obscured during the day because of the oxygen freed in the manufacture of starch which goes on at that time. +LJ026-0158|Probably this freed oxygen is used for the purpose of oxygenation, but more is freed in the photosynthetic process than is needed for oxygenation|Probably this freed oxygen is used for the purpose of oxygenation, but more is freed in the photosynthetic process than is needed for oxygenation +LJ026-0159|and hence the excess oxygen is eliminated while starch manufacture is in process.|and hence the excess oxygen is eliminated while starch manufacture is in process. +LJ026-0160|In comparing a higher animal and a green plant confusion must be avoided regarding the part played by oxygen and carbon dioxide in true respiration|In comparing a higher animal and a green plant confusion must be avoided regarding the part played by oxygen and carbon dioxide in true respiration +LJ026-0161|with the part played by the same substances in starch formation (photosynthesis).|with the part played by the same substances in starch formation (photosynthesis). +LJ026-0162|In non-green plants like the Indian pipe and mushrooms the breathing of oxygen and the excretion of carbon dioxide are as in the animal.|In non-green plants like the Indian pipe and mushrooms the breathing of oxygen and the excretion of carbon dioxide are as in the animal. +LJ026-0163|This is true also of green plants in darkness and even in the light of all parts of green plants except the chlorophyll-bodies.|This is true also of green plants in darkness and even in the light of all parts of green plants except the chlorophyll-bodies. +LJ026-0164|These constitute a sort of extra mechanism, enabling green plants to make their own carbohydrate food.|These constitute a sort of extra mechanism, enabling green plants to make their own carbohydrate food. +LJ026-0165|Imagine a higher animal with an attachment for turning the carbon dioxide and water excreted|Imagine a higher animal with an attachment for turning the carbon dioxide and water excreted +LJ026-0166|back to starch usable as food and the comparison of the green plant and the animal would be complete.|back to starch usable as food and the comparison of the green plant and the animal would be complete. +LJ027-0001|The Science: History of the Universe, Volume 5. Edited by Francis Rolt-Wheeler.|The Science: History of the Universe, Volume five. Edited by Francis Rolt-Wheeler. +LJ027-0002|Biology. Chapter 10. Morphology and Embryology, Part One.|Biology. Chapter ten. Morphology and Embryology, Part One. +LJ027-0003|The facts of biology which admit of adequate explanation only in connection with the theory of descent|The facts of biology which admit of adequate explanation only in connection with the theory of descent +LJ027-0004|are grouped by Romanes and other writers on organic evolution under the heads of morphology, embryology, classification,|are grouped by Romanes and other writers on organic evolution under the heads of morphology, embryology, classification, +LJ027-0005|paleontology, distribution and domestication.|paleontology, distribution and domestication. +LJ027-0007|There are numberless similarities and correlations and surprising uniformities.|There are numberless similarities and correlations and surprising uniformities. +LJ027-0008|The great variety of life as exhibited in the countless species of plants and animals has been referred to, and yet, great as this variety is,|The great variety of life as exhibited in the countless species of plants and animals has been referred to, and yet, great as this variety is, +LJ027-0009|there are, after all, only a few types of structure among all animals and plants, some three or four or eight or ten general modes of development,|there are, after all, only a few types of structure among all animals and plants, some three or four or eight or ten general modes of development, +LJ027-0010|and all the rest are modifications from these few types.|and all the rest are modifications from these few types. +LJ027-0011|It is, moreover, true that all living forms are but series of modifications and extensions of one single plan of structure.|It is, moreover, true that all living forms are but series of modifications and extensions of one single plan of structure. +LJ027-0012|All have the same ultimate substance|All have the same ultimate substance +LJ027-0013|the mysterious semi-fluid network of protoplasm, which is, so far as is known, the physical basis of all life;|the mysterious semi-fluid network of protoplasm, which is, so far as is known, the physical basis of all life; +LJ027-0014|and the equally mysterious nuclear substance or chromatin|and the equally mysterious nuclear substance or chromatin +LJ027-0015|which in some fashion presides over all the movements of the protoplasm and is the physical basis of the phenomena of heredity.|which in some fashion presides over all the movements of the protoplasm and is the physical basis of the phenomena of heredity. +LJ027-0016|The same laws of heredity, variability and of response to outside stimulus hold in all parts of the organic world.|The same laws of heredity, variability and of response to outside stimulus hold in all parts of the organic world. +LJ027-0017|All organisms have the same need of reproduction.|All organisms have the same need of reproduction. +LJ027-0018|All are forced to make concession after concession to their surroundings, and in these concessions all progress in life consists.|All are forced to make concession after concession to their surroundings, and in these concessions all progress in life consists. +LJ027-0019|And at last each organism or each alliance of organisms must come to the greatest concession of all, which is called death.|And at last each organism or each alliance of organisms must come to the greatest concession of all, which is called death. +LJ027-0020|The unity in life, then, is not less a fact than is life's great diversity.|The unity in life, then, is not less a fact than is life's great diversity. +LJ027-0021|Whatever emphasis is laid upon the diversity of life, the essential unity of all organisms must not be forgotten.|Whatever emphasis is laid upon the diversity of life, the essential unity of all organisms must not be forgotten. +LJ027-0022|An examination of the facts in each of the lines of evidence makes it clear|An examination of the facts in each of the lines of evidence makes it clear +LJ027-0023|that the only reasonable explanation for the existence of a fundamental unity in organic life|that the only reasonable explanation for the existence of a fundamental unity in organic life +LJ027-0024|is the theory of descent -- i.e., that similarities are due to blood relationship and that differences come from adaptive modifications.|is the theory of descent -- i.e., that similarities are due to blood relationship and that differences come from adaptive modifications. +LJ027-0025|The facts adduced from morphology, being the result of researches into the structure of adult animals and plants,|The facts adduced from morphology, being the result of researches into the structure of adult animals and plants, +LJ027-0026|lead to a preview of certain principles of adaptation, necessary for their interpretation.|lead to a preview of certain principles of adaptation, necessary for their interpretation. +LJ027-0027|First, it must be noted that some structures are not non-adaptive, that is, do not change to fit changed habits or conditions of life.|First, it must be noted that some structures are not non-adaptive, that is, do not change to fit changed habits or conditions of life. +LJ027-0028|Such structures or organs are most often found internally.|Such structures or organs are most often found internally. +LJ027-0029|For illustration: a change in the locomotive habit of a bird from that of flying to that of an ostrich|For illustration: a change in the locomotive habit of a bird from that of flying to that of an ostrich +LJ027-0030|is associated with an adaptive modification of locomotor structures, legs and wings,|is associated with an adaptive modification of locomotor structures, legs and wings, +LJ027-0031|but not in any striking way is there change in the internal organs.|but not in any striking way is there change in the internal organs. +LJ027-0032|Internal organs may persist unchanged and hence they offer good guides to classification.|Internal organs may persist unchanged and hence they offer good guides to classification. +LJ027-0033|On the other hand, external structures are likely to undergo adaptation when habits or conditions of life change.|On the other hand, external structures are likely to undergo adaptation when habits or conditions of life change. +LJ027-0034|Hence, as Jordan has said, "the inside of an animal tells the real history of its ancestry; the outside tells us only where its ancestors have been."|Hence, as Jordan has said, "the inside of an animal tells the real history of its ancestry; the outside tells us only where its ancestors have been." +LJ027-0035|In the second place, it must be noted that adaptations to similar conditions may result in superficial resemblances.|In the second place, it must be noted that adaptations to similar conditions may result in superficial resemblances. +LJ027-0036|For example, there is a superficial resemblance between the wing of an insect and the wing of a bird|For example, there is a superficial resemblance between the wing of an insect and the wing of a bird +LJ027-0037|both adaptations to an aerial environment;|both adaptations to an aerial environment; +LJ027-0038|between the heart of an insect and the heart of a vertebrate animal|between the heart of an insect and the heart of a vertebrate animal +LJ027-0039|both adaptations for pumping blood;|both adaptations for pumping blood; +LJ027-0040|between the fin of a fish and the paddle of a whale|between the fin of a fish and the paddle of a whale +LJ027-0041|both adaptive swimming organs, yet the resemblance in these cases does not go deeper than the surface -- it is one of function only.|both adaptive swimming organs, yet the resemblance in these cases does not go deeper than the surface -- it is one of function only. +LJ027-0042|All such cases of resemblance in function but not in detailed plan of structure are called "analogies,"|All such cases of resemblance in function but not in detailed plan of structure are called "analogies," +LJ027-0043|and mean nothing more than similarity of environment.|and mean nothing more than similarity of environment. +LJ027-0044|Turning to more fundamental resemblances, such as the wing of a bat and the wing of a bird,|Turning to more fundamental resemblances, such as the wing of a bat and the wing of a bird, +LJ027-0045|careful study shows detailed internal as well as external similarities of structure. Such cases are "homologies".|careful study shows detailed internal as well as external similarities of structure. Such cases are "homologies". +LJ027-0046|On the one hand, then, are found structures which are perfectly analogous and yet in no way homologous:|On the one hand, then, are found structures which are perfectly analogous and yet in no way homologous: +LJ027-0047|totally different structures are modified to perform the same functions.|totally different structures are modified to perform the same functions. +LJ027-0048|On the other hand are found structures which are perfectly homologous and yet in no way analogous:|On the other hand are found structures which are perfectly homologous and yet in no way analogous: +LJ027-0049|the structural elements remain, but are profoundly modified to perform totally different functions.|the structural elements remain, but are profoundly modified to perform totally different functions. +LJ027-0050|Homology thus means identity of structure which is the result of identity of parentage. It is the stamp of heredity.|Homology thus means identity of structure which is the result of identity of parentage. It is the stamp of heredity. +LJ027-0051|It means blood relationship.|It means blood relationship. +LJ027-0052|These principles of homology are essential to a correct interpretation of the facts of morphology.|These principles of homology are essential to a correct interpretation of the facts of morphology. +LJ027-0053|The most striking fact of similar structure among plants and among animals is the existence of a common general plan in any group.|The most striking fact of similar structure among plants and among animals is the existence of a common general plan in any group. +LJ027-0054|Since backboned animals are best known to most readers, they may be taken as an illustration.|Since backboned animals are best known to most readers, they may be taken as an illustration. +LJ027-0055|"All vertebrate animals, and none other," says Le Conte, "have an internal jointed skeleton worked by muscles on the outside.|"All vertebrate animals, and none other," says Le Conte, "have an internal jointed skeleton worked by muscles on the outside. +LJ027-0056|The relation of skeleton and muscle in arthropods is exactly the reverse.|The relation of skeleton and muscle in arthropods is exactly the reverse. +LJ027-0057|In all vertebrates, and in none other, the axis of this skeleton is a jointed backbone (vertebral column)|In all vertebrates, and in none other, the axis of this skeleton is a jointed backbone (vertebral column) +LJ027-0058|enclosing and protecting the nervous centers (cerebrospinal axis).|enclosing and protecting the nervous centers (cerebrospinal axis). +LJ027-0059|These, therefore, may well be called backboned animals.|These, therefore, may well be called backboned animals. +LJ027-0060|All vertebrates, and none other, have a number of their anterior vertebral joints enlarged and consolidated into a box to form the skull,|All vertebrates, and none other, have a number of their anterior vertebral joints enlarged and consolidated into a box to form the skull, +LJ027-0061|in order to enclose and protect a similar enlargement of the nervous center,|in order to enclose and protect a similar enlargement of the nervous center, +LJ027-0062|viz., the brain; and also usually, but not always, a number of posterior joints,|viz., the brain; and also usually, but not always, a number of posterior joints, +LJ027-0063|enlarged and consolidated to form the pelvis, to serve as a firm support to the hind-limbs.|enlarged and consolidated to form the pelvis, to serve as a firm support to the hind-limbs. +LJ027-0064|All vertebrates, and none other, have two cavities,|All vertebrates, and none other, have two cavities, +LJ027-0065|enclosed and protected by the skeleton, viz., the neural cavity above, and the visceral or body cavity below, the vertebral column.|enclosed and protected by the skeleton, viz., the neural cavity above, and the visceral or body cavity below, the vertebral column. +LJ027-0066|All vertebrates, with few exceptions, and no other animals, have two and only two pair of limbs.|All vertebrates, with few exceptions, and no other animals, have two and only two pair of limbs. +LJ027-0067|The exceptions are of two kinds, viz.: (a) some lowest fishes, amphioxus and lampreys,|The exceptions are of two kinds, viz.: (a) some lowest fishes, amphioxus and lampreys, +LJ027-0068|which probably represent the vertebrate condition before limbs were acquired;|which probably represent the vertebrate condition before limbs were acquired; +LJ027-0069|and (b) degenerate forms like snakes and some lizards, which have lost their limbs by disuse.|and (b) degenerate forms like snakes and some lizards, which have lost their limbs by disuse. +LJ027-0070|So much concerns the general plan of skeletal structures and is strongly suggestive of -- in fact it is inexplicable without -- common origin.|So much concerns the general plan of skeletal structures and is strongly suggestive of -- in fact it is inexplicable without -- common origin. +LJ027-0071|But much more remains which is not only suggestive, but demonstrative of such origin.|But much more remains which is not only suggestive, but demonstrative of such origin. +LJ027-0072|By extensive comparison in the taxonomic and ontogenic series, the whole vertebrate structure in all its details in different animals|By extensive comparison in the taxonomic and ontogenic series, the whole vertebrate structure in all its details in different animals +LJ027-0073|may be shown to be modifications one of another.|may be shown to be modifications one of another. +LJ027-0074|Sometimes a piece is enlarged, sometimes diminished, or even becomes obsolete; sometimes several pieces are consolidated into one;|Sometimes a piece is enlarged, sometimes diminished, or even becomes obsolete; sometimes several pieces are consolidated into one; +LJ027-0075|but, in spite of all these obscurations, corresponding parts usually may be made out.|but, in spite of all these obscurations, corresponding parts usually may be made out. +LJ027-0076|These remarkable similarities in the common general plan alone are convincing evidences of descent,|These remarkable similarities in the common general plan alone are convincing evidences of descent, +LJ027-0077|but attention may be called to a like similarity extending to the details of structure.|but attention may be called to a like similarity extending to the details of structure. +LJ027-0078|For example: the wings of a bat (a mammal), a bird and a fossil flying reptile all show the same bones adaptively modified;|For example: the wings of a bat (a mammal), a bird and a fossil flying reptile all show the same bones adaptively modified; +LJ027-0079|a series of either fore or hind limbs of a mammal with one toe (horse),|a series of either fore or hind limbs of a mammal with one toe (horse), +LJ027-0080|two toes (sheep), four toes (hog) and five toes (dog)|two toes (sheep), four toes (hog) and five toes (dog) +LJ027-0081|exhibit a remarkable series of homologies pointing to a five-toed ancestor,|exhibit a remarkable series of homologies pointing to a five-toed ancestor, +LJ027-0082|and any other series of organs of vertebrates would give the same evidence of fundamental resemblances (homologies).|and any other series of organs of vertebrates would give the same evidence of fundamental resemblances (homologies). +LJ027-0083|For such a series of facts the reader must be referred to special books like Wiedersheim's "Comparative Anatomy of the Vertebrates,"|For such a series of facts the reader must be referred to special books like Wiedersheim's "Comparative Anatomy of the Vertebrates," +LJ027-0084|Romanes's "Darwin and After Darwin", and Le Conte's "Evolution."|Romanes's "Darwin and After Darwin", and Le Conte's "Evolution." +LJ027-0085|The existence of great similarities in vertebrate structure is not always fully recognized.|The existence of great similarities in vertebrate structure is not always fully recognized. +LJ027-0086|To the superficial observer the bodies of animals of different classes seem to differ fundamentally in plan,|To the superficial observer the bodies of animals of different classes seem to differ fundamentally in plan, +LJ027-0087|to be entirely different machines, made each for its own purposes, at once, out of hand.|to be entirely different machines, made each for its own purposes, at once, out of hand. +LJ027-0088|Extensive comparison, on the contrary, shows them to be the same, although the essential identity is obscured by adaptive modifications.|Extensive comparison, on the contrary, shows them to be the same, although the essential identity is obscured by adaptive modifications. +LJ027-0089|The simplest, in fact the only scientific, explanation of the phenomena of vertebrate structure is the idea of a primal vertebrate,|The simplest, in fact the only scientific, explanation of the phenomena of vertebrate structure is the idea of a primal vertebrate, +LJ027-0090|modified more and more through successive generations by the necessities of different modes of life.|modified more and more through successive generations by the necessities of different modes of life. +LJ027-0091|See, then, the difference between man's mode of working and Nature's.|See, then, the difference between man's mode of working and Nature's. +LJ027-0092|A man having made a steam-engine, and desiring to use it for a different purpose from that for which it was first designed and used,|A man having made a steam-engine, and desiring to use it for a different purpose from that for which it was first designed and used, +LJ027-0093|will nearly always be compelled to add new parts not contemplated in the original machine.|will nearly always be compelled to add new parts not contemplated in the original machine. +LJ027-0094|Nature rarely makes new parts -- never, if she can avoid it -- but, on the contrary, adapts an old part to the new function.|Nature rarely makes new parts -- never, if she can avoid it -- but, on the contrary, adapts an old part to the new function. +LJ027-0095|It is as if Nature were not free to use any and every device to accomplish her end, but were conditioned by her own plans of structure;|It is as if Nature were not free to use any and every device to accomplish her end, but were conditioned by her own plans of structure; +LJ027-0096|as, indeed, she must be according to the derivation theory.|as, indeed, she must be according to the derivation theory. +LJ027-0097|Thus, in the fin of a fish, the fore-paw of a reptile or a mammal, the wing of a bird, and the arm and hand of a man|Thus, in the fin of a fish, the fore-paw of a reptile or a mammal, the wing of a bird, and the arm and hand of a man +LJ027-0098|is found the same part, variously modified for many purposes.|is found the same part, variously modified for many purposes. +LJ027-0099|Another striking class of the facts of morphology which admit of scientific explanation only along the line of homology|Another striking class of the facts of morphology which admit of scientific explanation only along the line of homology +LJ027-0100|are the thousands of cases of rudimentary or vestigial structures to be found.|are the thousands of cases of rudimentary or vestigial structures to be found. +LJ027-0101|Throughout both the animal and vegetable kingdoms dwarfed and useless representatives of organs are constantly met with,|Throughout both the animal and vegetable kingdoms dwarfed and useless representatives of organs are constantly met with, +LJ027-0102|which in other and allied kinds of animals and plants are of large size and functional utility.|which in other and allied kinds of animals and plants are of large size and functional utility. +LJ027-0103|Thus, for instance, the unborn whale has rudimentary teeth,|Thus, for instance, the unborn whale has rudimentary teeth, +LJ027-0104|which are never destined to cut the gums; and throughout its life this animal retains, in a similarly rudimentary condition,|which are never destined to cut the gums; and throughout its life this animal retains, in a similarly rudimentary condition, +LJ027-0105|a number of organs which never could have been of use to any kind of creature save a terrestrial quadruped.|a number of organs which never could have been of use to any kind of creature save a terrestrial quadruped. +LJ027-0106|Other well-known examples among vertebrates are: Vestiges of hind limbs in certain snakes, reduced wings in the Apteryx and ostriches,|Other well-known examples among vertebrates are: Vestiges of hind limbs in certain snakes, reduced wings in the Apteryx and ostriches, +LJ027-0107|rudiments of eyes in cave fishes, hind limbs beneath the skin of whales, the vermiform appendix in man,|rudiments of eyes in cave fishes, hind limbs beneath the skin of whales, the vermiform appendix in man, +LJ027-0108|as well as useless muscles to move the ears and the skin, and also a very much reduced hairy covering over the surface of the body.|as well as useless muscles to move the ears and the skin, and also a very much reduced hairy covering over the surface of the body. +LJ027-0109|Wiedersheim has recorded more than one hundred and eighty such structural reminiscences in man.|Wiedersheim has recorded more than one hundred and eighty such structural reminiscences in man. +LJ027-0110|Now, rudimentary organs of this kind are of such frequent occurrence, that almost every species of organism presents one or more of them|Now, rudimentary organs of this kind are of such frequent occurrence, that almost every species of organism presents one or more of them +LJ027-0111|usually, indeed, a considerable number.|usually, indeed, a considerable number. +LJ027-0112|How, then, are they to be accounted for?|How, then, are they to be accounted for? +LJ027-0113|Of course the theory of descent with adaptive modification has a simple answer to supply|Of course the theory of descent with adaptive modification has a simple answer to supply +LJ027-0114|Namely that when from changed conditions of life, an organ which was previously useful becomes useless,|Namely that when from changed conditions of life, an organ which was previously useful becomes useless, +LJ027-0115|it will be suffered to dwindle away in successive generations, under the influence of certain natural causes.|it will be suffered to dwindle away in successive generations, under the influence of certain natural causes. +LJ027-0116|On the other hand, the theory of special creation can only maintain that these rudiments are formed for the sake of adhering to an ideal type.|On the other hand, the theory of special creation can only maintain that these rudiments are formed for the sake of adhering to an ideal type. +LJ027-0117|"Now, here again the former theory appears to be triumphant over the latter," says Romanes,|"Now, here again the former theory appears to be triumphant over the latter," says Romanes, +LJ027-0118|"for, without waiting to dispute the wisdom of making dwarfed and useless structures merely for the whimsical motive assigned,|"for, without waiting to dispute the wisdom of making dwarfed and useless structures merely for the whimsical motive assigned, +LJ027-0119|surely if such a method were adopted in so many cases, we should expect that in consistency it would be adopted in all cases.|surely if such a method were adopted in so many cases, we should expect that in consistency it would be adopted in all cases. +LJ027-0120|This reasonable expectation, however, is far from being realized.|This reasonable expectation, however, is far from being realized. +LJ027-0121|In numberless cases, such as that of the fore-limbs of serpents, no vestige of a rudiment is present.|In numberless cases, such as that of the fore-limbs of serpents, no vestige of a rudiment is present. +LJ027-0122|But the vacillating policy in the matter of rudiments does not end here; for it is shown in a still more aggravated form|But the vacillating policy in the matter of rudiments does not end here; for it is shown in a still more aggravated form +LJ027-0123|where within the limits of the same natural group of organisms a rudiment is sometimes present and sometimes absent.|where within the limits of the same natural group of organisms a rudiment is sometimes present and sometimes absent. +LJ027-0124|For instance, although in nearly all the numerous species of snakes there are no vestiges of limbs,|For instance, although in nearly all the numerous species of snakes there are no vestiges of limbs, +LJ027-0125|in the Python we find very tiny rudiments of the hindlimbs. Now,|in the Python we find very tiny rudiments of the hindlimbs. Now, +LJ027-0126|is it a worthy conception of Deity that,|is it a worthy conception of Deity that, +LJ027-0127|while neglecting to maintain his unity of ideal in the case of nearly all the numerous species of snakes, he should have added a tiny rudiment in the case of the Python|while neglecting to maintain his unity of ideal in the case of nearly all the numerous species of snakes, he should have added a tiny rudiment in the case of the Python +LJ027-0128|and even in that case should have maintained his ideal very inefficiently, inasmuch as only two limbs, instead of four, are represented?|and even in that case should have maintained his ideal very inefficiently, inasmuch as only two limbs, instead of four, are represented? +LJ027-0129|Convincing as are the evidences of descent recorded in the structure of plants and animals,|Convincing as are the evidences of descent recorded in the structure of plants and animals, +LJ027-0130|these evidences have been in the past thirty years somewhat overshadowed by the far more surprising evidences|these evidences have been in the past thirty years somewhat overshadowed by the far more surprising evidences +LJ027-0131|of descent discovered in the development of plant and animal embryos.|of descent discovered in the development of plant and animal embryos. +LJ027-0132|A dozen volumes would be necessary|A dozen volumes would be necessary +LJ027-0133|to present the mass of embryological evidence, but a few salient facts will illustrate the kind of evidence to be deduced from embryology.|to present the mass of embryological evidence, but a few salient facts will illustrate the kind of evidence to be deduced from embryology. +LJ027-0134|Most remarkable of all the principles which have been discovered by embryologists is the "Recapitulation Doctrine"|Most remarkable of all the principles which have been discovered by embryologists is the "Recapitulation Doctrine" +LJ027-0135|which, briefly stated, is that individual development (ontogeny) recapitulates ancestral history (phylogeny).|which, briefly stated, is that individual development (ontogeny) recapitulates ancestral history (phylogeny). +LJ027-0136|Illustrations quoted from the works of Romanes and Le Conte will make this principle clear.|Illustrations quoted from the works of Romanes and Le Conte will make this principle clear. +LJ027-0137|"It is an observable fact," says Romanes,|"It is an observable fact," says Romanes, +LJ027-0138|that there is often a close correspondence between developmental changes as revealed by any chronological series of fossils which may happen to have been preserved,|that there is often a close correspondence between developmental changes as revealed by any chronological series of fossils which may happen to have been preserved, +LJ027-0139|and developmental changes which may be observed during the life history of now existing individuals belonging to the same group of animals.|and developmental changes which may be observed during the life history of now existing individuals belonging to the same group of animals. +LJ027-0141|is closely reproduced in the life-history of existing deer. Or, in other words,|is closely reproduced in the life-history of existing deer. Or, in other words, +LJ027-0142|the antlers of an existing deer furnish in their development a kind of "resume," or recapitulation, of the successive phases|the antlers of an existing deer furnish in their development a kind of "resume," or recapitulation, of the successive phases +LJ027-0143|whereby the primitive horn was gradually superseded by horns presenting a greater and greater number of prongs in successive species of extinct deer.|whereby the primitive horn was gradually superseded by horns presenting a greater and greater number of prongs in successive species of extinct deer. +LJ027-0144|Now, it must be obvious|Now, it must be obvious +LJ027-0145|that such a recapitulation in the life history of an existing animal of developmental changes successively distinctive of sundry allied,|that such a recapitulation in the life history of an existing animal of developmental changes successively distinctive of sundry allied, +LJ027-0146|though now extinct species, speaks strongly in favor of evolution.|though now extinct species, speaks strongly in favor of evolution. +LJ027-0147|For as it is of the essence of this theory that new forms arise from older forms by way of hereditary descent,|For as it is of the essence of this theory that new forms arise from older forms by way of hereditary descent, +LJ027-0148|we should antecedently expect, if the theory is true,|we should antecedently expect, if the theory is true, +LJ027-0149|that the phases of development presented by the individual organism would follow, in their main outlines, those phases of development|that the phases of development presented by the individual organism would follow, in their main outlines, those phases of development +LJ027-0150|through which their long line of ancestors had passed.|through which their long line of ancestors had passed. +LJ027-0151|The only alternative view is that as species of deer,|The only alternative view is that as species of deer, +LJ027-0152|for instance, were separately created, additional prongs were successively added to their antlers; and yet that,|for instance, were separately created, additional prongs were successively added to their antlers; and yet that, +LJ027-0153|in order to be so added to successive species, every individual deer belonging to later species was required to repeat in his own lifetime|in order to be so added to successive species, every individual deer belonging to later species was required to repeat in his own lifetime +LJ027-0154|the process of successive additions which had previously taken place in a remote series of extinct species.|the process of successive additions which had previously taken place in a remote series of extinct species. +LJ027-0155|Now I do not deny that this view is a possible view; but I do deny that it is a probable one.|Now I do not deny that this view is a possible view; but I do deny that it is a probable one. +LJ027-0156|According to the evolutionary interpretation of such facts, we can see a very good reason why the life-history of the individual|According to the evolutionary interpretation of such facts, we can see a very good reason why the life-history of the individual +LJ027-0157|is thus a condensed resume of the life history of its ancestral species.|is thus a condensed resume of the life history of its ancestral species. +LJ027-0158|But according to the opposite view no reason can be assigned why such should be the case.|But according to the opposite view no reason can be assigned why such should be the case. +LJ027-0159|"It is well known," likewise comments Le Conte,|"It is well known," likewise comments Le Conte, +LJ027-0160|that the embryo or larva of a frog or toad, when first hatched, is a legless, tail-swimming, water-breathing, gill-breathing animal.|that the embryo or larva of a frog or toad, when first hatched, is a legless, tail-swimming, water-breathing, gill-breathing animal. +LJ027-0161|It is essentially a fish, and would be so classed if it remained in this condition.|It is essentially a fish, and would be so classed if it remained in this condition. +LJ027-0162|The fish retains permanently this form, but the frog passes on.|The fish retains permanently this form, but the frog passes on. +LJ027-0163|Next, it forms first one pair and then another pair of legs; and meanwhile it begins to breathe also by lungs.|Next, it forms first one pair and then another pair of legs; and meanwhile it begins to breathe also by lungs. +LJ027-0164|At this stage it breathes equally by lungs and by gills -- i.e., both air and water.|At this stage it breathes equally by lungs and by gills -- i.e., both air and water. +LJ027-0165|Now, the lower forms of amphibians, such as siredon, menobranchus, siren, etc.,|Now, the lower forms of amphibians, such as siredon, menobranchus, siren, etc., +LJ027-0166|retain permanently this form, and are therefore called "perennibranchs," but the frog still passes on.|retain permanently this form, and are therefore called "perennibranchs," but the frog still passes on. +LJ027-0167|Then the gills gradually dry up, as the lungs develop, and they now breathe wholly by lungs, but still retain the tail.|Then the gills gradually dry up, as the lungs develop, and they now breathe wholly by lungs, but still retain the tail. +LJ027-0168|Now this is the permanent, mature condition of many amphibians,|Now this is the permanent, mature condition of many amphibians, +LJ027-0169|such as the triton, the salamander, etc., which are therefore called "caducibranchs," but the frog still passes on.|such as the triton, the salamander, etc., which are therefore called "caducibranchs," but the frog still passes on. +LJ027-0170|Finally, it loses the tail, or rather its tail is absorbed and its material used in further development, and it becomes a perfect frog,|Finally, it loses the tail, or rather its tail is absorbed and its material used in further development, and it becomes a perfect frog, +LJ027-0171|the highest order (anoura) of this class. Thus, then, in ontogeny the fish goes no further than the fish stages.|the highest order (anoura) of this class. Thus, then, in ontogeny the fish goes no further than the fish stages. +LJ027-0172|The perennibranch passes through the fish stage to the perennibranch amphibian.|The perennibranch passes through the fish stage to the perennibranch amphibian. +LJ027-0173|The caducibranch takes first the fish form, then the perennibranch form, and finally the caducibranch form, but goes no further.|The caducibranch takes first the fish form, then the perennibranch form, and finally the caducibranch form, but goes no further. +LJ027-0174|Last, the anoura takes first the fish-form, then that of the perennibranch, then that of the caducibranch, and finally becomes anoura.|Last, the anoura takes first the fish-form, then that of the perennibranch, then that of the caducibranch, and finally becomes anoura. +LJ027-0175|Now, this is undoubtedly the order of succession of forms in geological times -- i.e., in the phylogenic series.|Now, this is undoubtedly the order of succession of forms in geological times -- i.e., in the phylogenic series. +LJ027-0176|Fishes first appeared in the Devonian and Upper Silurian in very reptilian or rather amphibian forms.|Fishes first appeared in the Devonian and Upper Silurian in very reptilian or rather amphibian forms. +LJ027-0177|Then in the Carboniferous, fishes still continuing, there appeared the lowest -- i.e., most fish-like forms of amphibians.|Then in the Carboniferous, fishes still continuing, there appeared the lowest -- i.e., most fish-like forms of amphibians. +LJ027-0178|These were undoubtedly perennibranchs. In the Permian and Triassic higher forms appeared, which were certainly caducibranch.|These were undoubtedly perennibranchs. In the Permian and Triassic higher forms appeared, which were certainly caducibranch. +LJ027-0179|Finally, only in the Tertiary, so far as we yet know, do the highest form (anoura) appear.|Finally, only in the Tertiary, so far as we yet know, do the highest form (anoura) appear. +LJ027-0180|The general similarity of the three series is complete.|The general similarity of the three series is complete. +LJ028-0001|The Seven Wonders of the Ancient World. By Edgar J. Banks. Chapter 2. The Walls of Babylon.|The Seven Wonders of the Ancient World. By Edgar J. Banks. Chapter two. The Walls of Babylon. +LJ028-0002|In the old city of Damascus you climb to the hump of a tall fleet dromedary.|In the old city of Damascus you climb to the hump of a tall fleet dromedary. +LJ028-0003|With guides and guards about you, you ride through the covered bazaars crowded with dark-faced Arabs in strange costumes,|With guides and guards about you, you ride through the covered bazaars crowded with dark-faced Arabs in strange costumes, +LJ028-0004|and along the narrow winding lane which was once called the "Street Called Straight."|and along the narrow winding lane which was once called the "Street Called Straight." +LJ028-0005|Leaving the city by the eastern gate, and passing a small village or two,|Leaving the city by the eastern gate, and passing a small village or two, +LJ028-0006|you ascend the hill to the plateau, and before you, as far as the eye can reach, stretches the great Arabian Desert.|you ascend the hill to the plateau, and before you, as far as the eye can reach, stretches the great Arabian Desert. +LJ028-0007|With mingling fear and wonder at the mystery always lying beyond the desert horizon,|With mingling fear and wonder at the mystery always lying beyond the desert horizon, +LJ028-0008|you tap gently with your heel upon the shoulder of the dromedary to urge her on.|you tap gently with your heel upon the shoulder of the dromedary to urge her on. +LJ028-0009|At first, paying little heed to you, she hesitates and glances anxiously about the desert as if in search of an enemy.|At first, paying little heed to you, she hesitates and glances anxiously about the desert as if in search of an enemy. +LJ028-0010|Now and then she reaches down to graze the thorny argool along the way.|Now and then she reaches down to graze the thorny argool along the way. +LJ028-0011|As the taps upon her shoulder are repeated, she stretches out her long neck, and with long strides makes for the eastern horizon;|As the taps upon her shoulder are repeated, she stretches out her long neck, and with long strides makes for the eastern horizon; +LJ028-0012|she realizes that she is bound on the long journey across the desert. Hour after hour she bears you over the hard monotonous plain.|she realizes that she is bound on the long journey across the desert. Hour after hour she bears you over the hard monotonous plain. +LJ028-0013|The Damascus mosques and their minarets sink beneath the western sky.|The Damascus mosques and their minarets sink beneath the western sky. +LJ028-0014|The desert about you shows no signs of life;|The desert about you shows no signs of life; +LJ028-0015|only a tall column of whirling sand, rearing its head until it is lost in the blue above, moves majestically along.|only a tall column of whirling sand, rearing its head until it is lost in the blue above, moves majestically along. +LJ028-0016|In the distance your eyes detect a beautiful lake with shores fringed with trees,|In the distance your eyes detect a beautiful lake with shores fringed with trees, +LJ028-0017|but soon the phantom lake vanishes, while others, still farther beyond, appear and vanish in rapid succession.|but soon the phantom lake vanishes, while others, still farther beyond, appear and vanish in rapid succession. +LJ028-0018|Like a great ball of fire the sun sinks in the west.|Like a great ball of fire the sun sinks in the west. +LJ028-0019|The stars come out one by one and shine brighter than elsewhere as if to light you on your way.|The stars come out one by one and shine brighter than elsewhere as if to light you on your way. +LJ028-0020|Late at night the weary dromedary kneels, and on the ground, close beside her, you lie down to sleep.|Late at night the weary dromedary kneels, and on the ground, close beside her, you lie down to sleep. +LJ028-0021|Again, long before the stars have been scattered by the morning sun, you are on your way.|Again, long before the stars have been scattered by the morning sun, you are on your way. +LJ028-0022|Day after day you travel on, scorched by the heat of noon-day, shivering in the chill winds of the night.|Day after day you travel on, scorched by the heat of noon-day, shivering in the chill winds of the night. +LJ028-0023|Two weeks pass, and at last you stand on the eastern edge of the plateau|Two weeks pass, and at last you stand on the eastern edge of the plateau +LJ028-0024|gazing down upon the great Euphrates winding along the valley beneath.|gazing down upon the great Euphrates winding along the valley beneath. +LJ028-0025|You have crossed the Arabian Desert, the first stage of the long journey to the walls of Babylon.|You have crossed the Arabian Desert, the first stage of the long journey to the walls of Babylon. +LJ028-0026|Here in the valley the water is sweet and the food abundant.|Here in the valley the water is sweet and the food abundant. +LJ028-0027|For ten days you follow down the river, through little villages and black tent encampments, among scenes of strange Arab life which never lose their charm.|For ten days you follow down the river, through little villages and black tent encampments, among scenes of strange Arab life which never lose their charm. +LJ028-0028|Everywhere the valley is dotted with the mounds of buried cities carefully guarding the secrets of the centuries of long ago.|Everywhere the valley is dotted with the mounds of buried cities carefully guarding the secrets of the centuries of long ago. +LJ028-0029|At last you see before you a mound rising like a mountain from the level plain.|At last you see before you a mound rising like a mountain from the level plain. +LJ028-0030|Your journey is at an end. Before you is Babylon, the "Gate of God," as the old name means.|Your journey is at an end. Before you is Babylon, the "Gate of God," as the old name means. +LJ028-0031|About you is all that remains of the second of the Seven Wonders of the World.|About you is all that remains of the second of the Seven Wonders of the World. +LJ028-0032|Babylon, even in the days of Nebuchadnezzar, was an old, old city.|Babylon, even in the days of Nebuchadnezzar, was an old, old city. +LJ028-0033|There is a Hebrew tradition that it was the oldest of all cities, but now we know that great empires flourished and passed away before Babylon was built.|There is a Hebrew tradition that it was the oldest of all cities, but now we know that great empires flourished and passed away before Babylon was built. +LJ028-0034|Old King Sargon I., who may have lived as early as three thousand eight hundred B.C.,|Old King Sargon I., who may have lived as early as three thousand eight hundred B.C., +LJ028-0035|seems to have been the first to mention Babylon, and one of his inscriptions seems to say that he built the city and gave it its name.|seems to have been the first to mention Babylon, and one of his inscriptions seems to say that he built the city and gave it its name. +LJ028-0036|But in those very early days Babylon was little more than a shrine, surrounded with mud huts and date palms.|But in those very early days Babylon was little more than a shrine, surrounded with mud huts and date palms. +LJ028-0037|It was about 2250 B.C., when the great Hammurabi made it his capital, that it became the chief city of Babylonia.|It was about twenty-two fifty B.C., when the great Hammurabi made it his capital, that it became the chief city of Babylonia. +LJ028-0038|Its history for the next fifteen hundred years or more is obscure.|Its history for the next fifteen hundred years or more is obscure. +LJ028-0039|We know the names of its kings, and the records speak of long wars with the Assyrians.|We know the names of its kings, and the records speak of long wars with the Assyrians. +LJ028-0040|In the year six eighty-nine B.C., Sinacherib, King of Nineveh, captured Babylon,|In the year six eighty-nine B.C., Sinacherib, King of Nineveh, captured Babylon, +LJ028-0041|tore down its palaces and temples and walls, and scraped even the foundations of the city into the river.|tore down its palaces and temples and walls, and scraped even the foundations of the city into the river. +LJ028-0042|The place where the old city had stood for three thousand years again became a desert.|The place where the old city had stood for three thousand years again became a desert. +LJ028-0043|Esarhaddon, the son of Sinacherib, was the next King of Nineveh.|Esarhaddon, the son of Sinacherib, was the next King of Nineveh. +LJ028-0044|He rebuilt Babylon that in accordance with the ancient custom he might be crowned in the sacred city.|He rebuilt Babylon that in accordance with the ancient custom he might be crowned in the sacred city. +LJ028-0045|When Esarhaddon died, one of his sons, Samas-sum-yukin, was made King of Babylon.|When Esarhaddon died, one of his sons, Samas-sum-yukin, was made King of Babylon. +LJ028-0046|Another son, Assurbanipal, or the great Sardanapalus of the Greeks, became the King of Nineveh.|Another son, Assurbanipal, or the great Sardanapalus of the Greeks, became the King of Nineveh. +LJ028-0047|War broke out between the two brothers, and again Babylon was captured.|War broke out between the two brothers, and again Babylon was captured. +LJ028-0048|In 626 Assurbanipal died,|In six twenty-six Assurbanipal died, +LJ028-0049|and in that same year Nabopolassar, the father of the great Nebuchadnezzar, became the King of Babylon.|and in that same year Nabopolassar, the father of the great Nebuchadnezzar, became the King of Babylon. +LJ028-0050|The building of the Babylon so famous in history began with Nabopolassar.|The building of the Babylon so famous in history began with Nabopolassar. +LJ028-0051|He enlarged the old city, erected temples, and began the construction of its walls.|He enlarged the old city, erected temples, and began the construction of its walls. +LJ028-0052|In 606, Nineveh, the old enemy of Babylon, fell, never to rise again.|In six oh six, Nineveh, the old enemy of Babylon, fell, never to rise again. +LJ028-0053|The next year, in 605, Nabopolassar died, and Nebuchadnezzar succeeded him to the throne.|The next year, in six oh five, Nabopolassar died, and Nebuchadnezzar succeeded him to the throne. +LJ028-0054|He continued the building operations of his father, until Babylon became the greatest city of its age,|He continued the building operations of his father, until Babylon became the greatest city of its age, +LJ028-0055|and surrounded it with walls the like of which no other city has ever seen.|and surrounded it with walls the like of which no other city has ever seen. +LJ028-0056|Nebuchadnezzar, or Nebuchadrezzar, as his name should be spelled, was the greatest character in Babylonian history,|Nebuchadnezzar, or Nebuchadrezzar, as his name should be spelled, was the greatest character in Babylonian history, +LJ028-0057|but about his name so many legends have grown that it is sometimes difficult to learn the facts of his life.|but about his name so many legends have grown that it is sometimes difficult to learn the facts of his life. +LJ028-0058|Early he married Amuhia, a daughter of the Medean king.|Early he married Amuhia, a daughter of the Medean king. +LJ028-0059|His military career began while he was still the crown prince, and his father was on the throne.|His military career began while he was still the crown prince, and his father was on the throne. +LJ028-0060|In 605,|In six oh five, +LJ028-0061|at the head of the Babylonian army, he defeated the Egyptians in the famous battle of Carchemish, the old Hittite capital,|at the head of the Babylonian army, he defeated the Egyptians in the famous battle of Carchemish, the old Hittite capital, +LJ028-0062|and drove them from Asia.|and drove them from Asia. +LJ028-0063|Then Syria and Palestine were added to his future empire.|Then Syria and Palestine were added to his future empire. +LJ028-0064|In 597, when he sent his army to Jerusalem, he won the hatred of the Jews by taking Jehoiakin, the King, captive.|In five ninety-seven, when he sent his army to Jerusalem, he won the hatred of the Jews by taking Jehoiakin, the King, captive. +LJ028-0065|Eleven years later, in 586, he destroyed the sacred Hebrew city,|Eleven years later, in five eighty-six, he destroyed the sacred Hebrew city, +LJ028-0066|transported the Jews to Babylon, and brought the Hebrew kingdom to an end.|transported the Jews to Babylon, and brought the Hebrew kingdom to an end. +LJ028-0067|Centuries afterward, even to this day, Jewish mothers teach their children to hate his name.|Centuries afterward, even to this day, Jewish mothers teach their children to hate his name. +LJ028-0068|They tell how he forced the exiles to carry heavy bags of sand across the desert to increase their burdens;|They tell how he forced the exiles to carry heavy bags of sand across the desert to increase their burdens; +LJ028-0069|how he cast Hebrew lads into a fiery furnace and into the lions' den,|how he cast Hebrew lads into a fiery furnace and into the lions' den, +LJ028-0070|and how, in punishment for all his wickedness, he became a calf, and for seven years grazed the grass in the fields about the city.|and how, in punishment for all his wickedness, he became a calf, and for seven years grazed the grass in the fields about the city. +LJ028-0071|Late in his life, in 567, he invaded Egypt.|Late in his life, in five sixty-seven, he invaded Egypt. +LJ028-0072|During all his reign there was little peace in his great mixed turbulent empire.|During all his reign there was little peace in his great mixed turbulent empire. +LJ028-0073|The walls of the palaces of many of the Assyrian kings were lined with great stone slabs engraved with reliefs and sometimes with the portrait of a king.|The walls of the palaces of many of the Assyrian kings were lined with great stone slabs engraved with reliefs and sometimes with the portrait of a king. +LJ028-0074|But in Babylonia stone was difficult to obtain, and sculptures were very rare.|But in Babylonia stone was difficult to obtain, and sculptures were very rare. +LJ028-0075|Therefore it was useless to hope that Nebuchadnezzar's portrait would be found on his palace walls.|Therefore it was useless to hope that Nebuchadnezzar's portrait would be found on his palace walls. +LJ028-0076|However, several decades ago, an Oriental appeared at the Berlin Museum,|However, several decades ago, an Oriental appeared at the Berlin Museum, +LJ028-0077|offering for sale a small cameo engraved with a helmeted head of a Greek type.|offering for sale a small cameo engraved with a helmeted head of a Greek type. +LJ028-0078|About the head was an inscription in Greek characters saying that the face was that of Nebuchadnezzar.|About the head was an inscription in Greek characters saying that the face was that of Nebuchadnezzar. +LJ028-0079|The museum authorities believed that the cameo was one of the many spurious objects which the Eastern forgers were constantly sending to Europe,|The museum authorities believed that the cameo was one of the many spurious objects which the Eastern forgers were constantly sending to Europe, +LJ028-0080|yet they took an impression of it, and returned it to its owner.|yet they took an impression of it, and returned it to its owner. +LJ028-0081|Years later, when the archaeologists could readily distinguish the false from the true,|Years later, when the archaeologists could readily distinguish the false from the true, +LJ028-0082|it was recognized that the cameo was genuine, and that it bore the likeness of the great King.|it was recognized that the cameo was genuine, and that it bore the likeness of the great King. +LJ028-0083|Unfortunately, the little stone seal, perhaps the only one to preserve for us his features, appears to have been lost for ever.|Unfortunately, the little stone seal, perhaps the only one to preserve for us his features, appears to have been lost for ever. +LJ028-0084|Its impression shows the face of a beardless young man, intelligent and refined.|Its impression shows the face of a beardless young man, intelligent and refined. +LJ028-0085|The eyes are suggestive of the Semitic; the nose is of the Greek type;|The eyes are suggestive of the Semitic; the nose is of the Greek type; +LJ028-0086|the lips are thin, the chin prominent; the neck is that of a strong vigorous man.|the lips are thin, the chin prominent; the neck is that of a strong vigorous man. +LJ028-0087|Such was the appearance of the builder of the walls of Babylon.|Such was the appearance of the builder of the walls of Babylon. +LJ028-0088|Religion and cruelty frequently go hand in hand, and Nebuchadnezzar was exceedingly religious.|Religion and cruelty frequently go hand in hand, and Nebuchadnezzar was exceedingly religious. +LJ028-0089|Though a great warrior, it was not for his military deeds that he was best known.|Though a great warrior, it was not for his military deeds that he was best known. +LJ028-0090|He was fond of restoring the ruined temples of the old Babylonian cities,|He was fond of restoring the ruined temples of the old Babylonian cities, +LJ028-0091|and most of the records which have come from his time speak chiefly of his deeds of piety.|and most of the records which have come from his time speak chiefly of his deeds of piety. +LJ028-0092|Read the introduction to any of his inscriptions, of which the following is one, and you will call him vain and proud,|Read the introduction to any of his inscriptions, of which the following is one, and you will call him vain and proud, +LJ028-0093|but his scribe wrote it in the manner customary for the scribes of those days to write of their royal masters.|but his scribe wrote it in the manner customary for the scribes of those days to write of their royal masters. +LJ028-0094|Nebuchadnezzar, King of Babylon, the exalted prince, the favorite of Marduk, the lofty patesi,|Nebuchadnezzar, King of Babylon, the exalted prince, the favorite of Marduk, the lofty patesi, +LJ028-0095|the beloved of Nabu, the arbiter, the possessor of wisdom, who seeks out the path of their divinity,|the beloved of Nabu, the arbiter, the possessor of wisdom, who seeks out the path of their divinity, +LJ028-0096|who reverences their lordship; the untiring governor, who ponders daily concerning the maintenance of Esagil and Ezida,|who reverences their lordship; the untiring governor, who ponders daily concerning the maintenance of Esagil and Ezida, +LJ028-0097|and is continually anxious for the shrines of Babylon and Borsippa;|and is continually anxious for the shrines of Babylon and Borsippa; +LJ028-0098|the wise, the pious, the maintainer of Esagil and Ezida,|the wise, the pious, the maintainer of Esagil and Ezida, +LJ028-0099|the first-born son of Nabopolassar, King of Babylon, am I.|the first-born son of Nabopolassar, King of Babylon, am I. +LJ028-0100|However cruel and religiously intolerant Nebuchadnezzar may have been, he was undoubtedly the greatest builder the world has ever seen.|However cruel and religiously intolerant Nebuchadnezzar may have been, he was undoubtedly the greatest builder the world has ever seen. +LJ028-0101|There is scarcely one of the thousands of ruin mounds in Babylonia which does not contain bricks bearing his name.|There is scarcely one of the thousands of ruin mounds in Babylonia which does not contain bricks bearing his name. +LJ028-0102|There is scarcely a royal record from his reign which is not chiefly occupied with descriptions of his building operations.|There is scarcely a royal record from his reign which is not chiefly occupied with descriptions of his building operations. +LJ028-0103|He rebuilt scores of the ancient temples, surrounded many cities with walls,|He rebuilt scores of the ancient temples, surrounded many cities with walls, +LJ028-0104|lined the shores of the rivers with embankments, and spanned the rivers with bridges.|lined the shores of the rivers with embankments, and spanned the rivers with bridges. +LJ028-0105|Tradition says that to please his foreign wife from the mountainous country he built the famous hanging gardens, but that may be only a tradition.|Tradition says that to please his foreign wife from the mountainous country he built the famous hanging gardens, but that may be only a tradition. +LJ028-0106|His palace in Babylon was one of the world's largest buildings, but the walls with which he protected his palace and city were the wonder of the whole world.|His palace in Babylon was one of the world's largest buildings, but the walls with which he protected his palace and city were the wonder of the whole world. +LJ028-0107|The ancients never tired of describing them.|The ancients never tired of describing them. +LJ028-0108|Fortunately in several of his long inscriptions, recently discovered in the Babylonian mounds, Nebuchadnezzar speaks of the building of the walls.|Fortunately in several of his long inscriptions, recently discovered in the Babylonian mounds, Nebuchadnezzar speaks of the building of the walls. +LJ028-0109|In one of them he says:|In one of them he says: +LJ028-0110|I completed Imgur-Bel and Nimitti-Bel, the great walls of Babylon, the mighty city, the city of his exalted power.|I completed Imgur-Bel and Nimitti-Bel, the great walls of Babylon, the mighty city, the city of his exalted power. +LJ028-0111|At the entrance of the great gates I erected strong bulls of bronze, and terrible serpents standing upright.|At the entrance of the great gates I erected strong bulls of bronze, and terrible serpents standing upright. +LJ028-0112|My father did that which no previous king had done.|My father did that which no previous king had done. +LJ028-0113|With mortar and bricks he built two moat-walls about the city,|With mortar and bricks he built two moat-walls about the city, +LJ028-0114|and I, with mortar and bricks, built a third great moat-wall, and joined it and united it closely with the moat-walls of my father.|and I, with mortar and bricks, built a third great moat-wall, and joined it and united it closely with the moat-walls of my father. +LJ028-0115|I laid its foundation deep to the water level;|I laid its foundation deep to the water level; +LJ028-0116|I raised its summit mountain high. I constructed a moat-wall of burned bricks about the west wall of Babylon.|I raised its summit mountain high. I constructed a moat-wall of burned bricks about the west wall of Babylon. +LJ028-0117|My father built the moat-wall of the Arachtu canal securely with mortar and bricks.|My father built the moat-wall of the Arachtu canal securely with mortar and bricks. +LJ028-0118|He built well the quays along the opposite shore of the Euphrates, but he did not finish all his work,|He built well the quays along the opposite shore of the Euphrates, but he did not finish all his work, +LJ028-0119|but I, his first-born, the beloved of his heart,|but I, his first-born, the beloved of his heart, +LJ028-0120|built the moat-walls of Arachtu with mortar and bricks, and, joining them together with those of my father, made them very solid.|built the moat-walls of Arachtu with mortar and bricks, and, joining them together with those of my father, made them very solid. +LJ028-0121|A thing which no king before had ever done:|A thing which no king before had ever done: +LJ028-0122|To the west of Babylon, at a greater distance from the outer wall, I constructed an enclosing wall four thousand cubits in length about the city.|To the west of Babylon, at a greater distance from the outer wall, I constructed an enclosing wall four thousand cubits in length about the city. +LJ028-0123|I dug its moat to the water level.|I dug its moat to the water level. +LJ028-0124|I walled up its side with mortar and burned bricks, and I united it securely with the moat-walls of my father.|I walled up its side with mortar and burned bricks, and I united it securely with the moat-walls of my father. +LJ028-0125|Along its edge I built a great wall of mortar and burned bricks mountain high.|Along its edge I built a great wall of mortar and burned bricks mountain high. +LJ028-0126|Berossus, a priest of the temple of Bel at Babylon, writing about 250 B.C.,|Berossus, a priest of the temple of Bel at Babylon, writing about two fifty B.C., +LJ028-0127|was living in the city while the walls were still standing, though in a ruinous condition.|was living in the city while the walls were still standing, though in a ruinous condition. +LJ028-0128|His brief description of them should not be omitted. He says that Nebuchadnezzar|His brief description of them should not be omitted. He says that Nebuchadnezzar +LJ028-0129|built three walls round about the inner city, and three others about that which was the outer; and this he did with burnt brick.|built three walls round about the inner city, and three others about that which was the outer; and this he did with burnt brick. +LJ028-0130|And after he had walled the city, and adorned its gates, he built another palace before his father's palace; but so that they joined to it:|And after he had walled the city, and adorned its gates, he built another palace before his father's palace; but so that they joined to it: +LJ028-0131|to describe whose vast height and immense riches it would perhaps be too much for me to attempt.|to describe whose vast height and immense riches it would perhaps be too much for me to attempt. +LJ028-0132|Yet as large and lofty as they were, they were completed in fifteen days.|Yet as large and lofty as they were, they were completed in fifteen days. +LJ028-0133|He also erected elevated places for walking, of stone; and made it resemble mountains: and built it so that it might be planted with all sorts of trees.|He also erected elevated places for walking, of stone; and made it resemble mountains: and built it so that it might be planted with all sorts of trees. +LJ028-0136|Of all the ancient descriptions of the famous walls and the city they protected, that of Herodotus is the fullest.|Of all the ancient descriptions of the famous walls and the city they protected, that of Herodotus is the fullest. +LJ028-0137|Perhaps Herodotus had never been in Babylon;|Perhaps Herodotus had never been in Babylon; +LJ028-0139|yet he at least tried to be accurate. He says:|yet he at least tried to be accurate. He says: +LJ028-0140|The city stands on a broad plain, and is an exact square, a hundred and twenty furlongs in length each way,|The city stands on a broad plain, and is an exact square, a hundred and twenty furlongs in length each way, +LJ028-0141|so that the entire circuit is four hundred and eighty furlongs.|so that the entire circuit is four hundred and eighty furlongs. +LJ028-0142|While such is its size, in magnificence there is no other city that approaches to it.|While such is its size, in magnificence there is no other city that approaches to it. +LJ028-0143|It is surrounded, in the first place, by a broad and deep moat, full of water, behind which rises a wall|It is surrounded, in the first place, by a broad and deep moat, full of water, behind which rises a wall +LJ028-0144|fifty royal cubits in width, and two hundred in height.|fifty royal cubits in width, and two hundred in height. +LJ028-0145|And here I may not omit to tell the use to which the mould dug out of the great moat was turned, nor the manner wherein the wall was wrought.|And here I may not omit to tell the use to which the mould dug out of the great moat was turned, nor the manner wherein the wall was wrought. +LJ028-0146|As fast as they dug the moat, the soil which they got from the cutting was made into bricks,|As fast as they dug the moat, the soil which they got from the cutting was made into bricks, +LJ028-0147|and when a sufficient number were completed they baked the bricks in kilns.|and when a sufficient number were completed they baked the bricks in kilns. +LJ028-0148|Then they set to building, and began by bricking the borders of the moat, after which they proceeded to construct the wall itself,|Then they set to building, and began by bricking the borders of the moat, after which they proceeded to construct the wall itself, +LJ028-0149|using throughout for their cement hot bitumen, and interposing a layer of wattled reeds at every thirtieth course of the bricks.|using throughout for their cement hot bitumen, and interposing a layer of wattled reeds at every thirtieth course of the bricks. +LJ028-0150|On the top, along the edges of the wall, they constructed buildings of a single chamber facing one another,|On the top, along the edges of the wall, they constructed buildings of a single chamber facing one another, +LJ028-0151|leaving between them room for a four-horse chariot to turn.|leaving between them room for a four-horse chariot to turn. +LJ028-0152|In the circuit of the wall are a hundred gates, all of brass, with brazen lintels and sideposts.|In the circuit of the wall are a hundred gates, all of brass, with brazen lintels and sideposts. +LJ028-0153|The bitumen used in the work was brought to Babylon from Is, a small stream which flows into the Euphrates|The bitumen used in the work was brought to Babylon from Is, a small stream which flows into the Euphrates +LJ028-0154|at the point where the city of the same name stands, eight days' journey from Babylon.|at the point where the city of the same name stands, eight days' journey from Babylon. +LJ028-0155|Lumps of bitumen are found in great abundance in this river.|Lumps of bitumen are found in great abundance in this river. +LJ028-0156|The city is divided into two portions by the river which runs through the midst of it.|The city is divided into two portions by the river which runs through the midst of it. +LJ028-0157|This river is the Euphrates, a broad, deep, swift stream, which rises in Armenia, and empties itself into the Erythraean Sea.|This river is the Euphrates, a broad, deep, swift stream, which rises in Armenia, and empties itself into the Erythraean Sea. +LJ028-0158|The city wall is brought down on both sides to the edge of the stream,|The city wall is brought down on both sides to the edge of the stream, +LJ028-0159|thence from the corners of the wall there is carried along each bank of the river a fence of burned bricks.|thence from the corners of the wall there is carried along each bank of the river a fence of burned bricks. +LJ028-0160|The houses are mostly three and four stories high;|The houses are mostly three and four stories high; +LJ028-0161|the streets all run in straight lines, not only those parallel to the river, but also the cross streets which lead down to the waterside.|the streets all run in straight lines, not only those parallel to the river, but also the cross streets which lead down to the waterside. +LJ028-0162|At the river end of these cross streets are low gates in the fence that skirts the stream,|At the river end of these cross streets are low gates in the fence that skirts the stream, +LJ028-0163|which are, like the great gates in the outer wall, of brass, and open on the water.|which are, like the great gates in the outer wall, of brass, and open on the water. +LJ028-0164|The outer wall is the main defense of the city.|The outer wall is the main defense of the city. +LJ028-0165|There is, however, a second inner wall, of less thickness than the first, but very little inferior to it in strength.|There is, however, a second inner wall, of less thickness than the first, but very little inferior to it in strength. +LJ028-0166|The center of each division of the town is occupied by a fortress.|The center of each division of the town is occupied by a fortress. +LJ028-0167|In the one stood the palace of the kings, surrounded by a wall of great strength and size;|In the one stood the palace of the kings, surrounded by a wall of great strength and size; +LJ028-0168|in the other was the sacred precinct of Jupiter Belus,|in the other was the sacred precinct of Jupiter Belus, +LJ028-0169|a square enclosure two furlongs each way, with gates of solid brass; which was also remaining in my time.|a square enclosure two furlongs each way, with gates of solid brass; which was also remaining in my time. +LJ028-0170|In the middle of the precinct there was a tower of solid masonry, a furlong in length and breadth,|In the middle of the precinct there was a tower of solid masonry, a furlong in length and breadth, +LJ028-0171|upon which was raised a second tower, and on that a third, and so on up to eight.|upon which was raised a second tower, and on that a third, and so on up to eight. +LJ028-0172|The ascent to the top is on the outside, by a path which winds round all the towers.|The ascent to the top is on the outside, by a path which winds round all the towers. +LJ028-0173|When one is about halfway up, one finds a resting place and seats, where persons are wont to sit sometimes on their way to the summit.|When one is about halfway up, one finds a resting place and seats, where persons are wont to sit sometimes on their way to the summit. +LJ028-0174|Other ancient descriptions of the walls have been left us by Ctesias of the fifth century B.C., and by Strabo of the beginning of the Christian era,|Other ancient descriptions of the walls have been left us by Ctesias of the fifth century B.C., and by Strabo of the beginning of the Christian era, +LJ028-0175|but they add little to our knowledge.|but they add little to our knowledge. +LJ028-0176|Should we compare these ancient descriptions of the walls, we should find them hopelessly conflicting.|Should we compare these ancient descriptions of the walls, we should find them hopelessly conflicting. +LJ028-0177|However, they teach us that in those early days when most cities were surrounded by enormous walls,|However, they teach us that in those early days when most cities were surrounded by enormous walls, +LJ028-0178|the walls of Babylon were so long and wide and high that all who saw them were amazed.|the walls of Babylon were so long and wide and high that all who saw them were amazed. +LJ028-0179|It is only from their ruins that we may hope to obtain accurate information of the strongest fortifications in the ancient world.|It is only from their ruins that we may hope to obtain accurate information of the strongest fortifications in the ancient world. +LJ028-0180|In the year 562, after a long reign of forty-three years, Nebuchadnezzar died.|In the year five sixty-two, after a long reign of forty-three years, Nebuchadnezzar died. +LJ028-0181|He was followed by three kings whose reigns were short,|He was followed by three kings whose reigns were short, +LJ028-0182|and in 555 Nabonidus, the father of the Biblical Belshazzar, came to the throne.|and in five fifty-five Nabonidus, the father of the Biblical Belshazzar, came to the throne. +LJ028-0183|Cyrus, the King of Persia, was rising to power, and after he had defeated the Medes|Cyrus, the King of Persia, was rising to power, and after he had defeated the Medes +LJ028-0184|he extended his empire to the Mediterranean and even to Egypt.|he extended his empire to the Mediterranean and even to Egypt. +LJ028-0185|Perhaps Babylon was so strongly fortified that at first he made no attempt to add it to his empire,|Perhaps Babylon was so strongly fortified that at first he made no attempt to add it to his empire, +LJ028-0186|but when Nabonidus joined with the King of Egypt and with the wealthy Croesus of Lydia in an alliance against him,|but when Nabonidus joined with the King of Egypt and with the wealthy Croesus of Lydia in an alliance against him, +LJ028-0187|Cyrus decided that Babylon must be taken.|Cyrus decided that Babylon must be taken. +LJ028-0188|In 538 the city fell, and for a time it became the home of the Persian King.|In five thirty-eight the city fell, and for a time it became the home of the Persian King. +LJ028-0190|A great empire which had existed for more than three thousand years was brought to an end.|A great empire which had existed for more than three thousand years was brought to an end. +LJ028-0191|The old enemies of Babylon rejoiced.|The old enemies of Babylon rejoiced. +LJ028-0192|When the news came to the Hebrews, who were held there in exile, they excitedly rushed about the streets, crying: "Babylon is fallen,"|When the news came to the Hebrews, who were held there in exile, they excitedly rushed about the streets, crying: "Babylon is fallen," +LJ028-0193|and to them came hope of returning to Jerusalem.|and to them came hope of returning to Jerusalem. +LJ028-0194|But how did the "mighty city" fall? How could Cyrus take Babylon whose walls were strong enough to resist any army?|But how did the "mighty city" fall? How could Cyrus take Babylon whose walls were strong enough to resist any army? +LJ028-0195|It is a long story. Poets have sung it. Historians have written it. Prophets have preached it. Legends have gathered about it.|It is a long story. Poets have sung it. Historians have written it. Prophets have preached it. Legends have gathered about it. +LJ028-0196|Every child knows the story of "the writing of the hand on the wall." It was the night that Babylon fell.|Every child knows the story of "the writing of the hand on the wall." It was the night that Babylon fell. +LJ028-0197|Belshazzar, the King, he was really the King's son, gave a feast to a thousand of his nobles.|Belshazzar, the King, he was really the King's son, gave a feast to a thousand of his nobles. +LJ028-0198|In the great banquet hall of the palace, when the guests were drinking from the golden cups, and the revelry was at its highest,|In the great banquet hall of the palace, when the guests were drinking from the golden cups, and the revelry was at its highest, +LJ028-0199|there suddenly appeared upon the wall an armless hand.|there suddenly appeared upon the wall an armless hand. +LJ028-0200|High up, where all might see it, the armless hand wrote the King's fate.|High up, where all might see it, the armless hand wrote the King's fate. +LJ028-0201|"Thou art weighed in the balances and art found wanting."|"Thou art weighed in the balances and art found wanting." +LJ028-0202|"In that night," so the story ends, "Belshazzar, the Chaldean King, was slain."|"In that night," so the story ends, "Belshazzar, the Chaldean King, was slain." +LJ028-0203|Less picturesque than this Hebrew legend is the royal record of Babylon, which fortunately was inscribed upon a clay cylinder from the ruins of the city.|Less picturesque than this Hebrew legend is the royal record of Babylon, which fortunately was inscribed upon a clay cylinder from the ruins of the city. +LJ028-0204|It refers to the death of the King's son, possibly to Belshazzar of the Bible story.|It refers to the death of the King's son, possibly to Belshazzar of the Bible story. +LJ028-0205|In the month Tammuz, when Cyrus fought the troops of Akkad (Babylonia) at Opis on the river Salsallat,|In the month Tammuz, when Cyrus fought the troops of Akkad (Babylonia) at Opis on the river Salsallat, +LJ028-0206|he subdued the people, and wherever they collected, he slew them.|he subdued the people, and wherever they collected, he slew them. +LJ028-0207|On the fourteenth day Sippar was taken without a battle.|On the fourteenth day Sippar was taken without a battle. +LJ028-0208|Nabonidus fled.|Nabonidus fled. +LJ028-0209|On the sixteenth day the troops of Cyrus entered Babylon without a battle.|On the sixteenth day the troops of Cyrus entered Babylon without a battle. +LJ028-0210|Nabonidus was taken prisoner in Babylon. On the third of Marchesvan Cyrus entered Babylon and proclaimed peace to all the city.|Nabonidus was taken prisoner in Babylon. On the third of Marchesvan Cyrus entered Babylon and proclaimed peace to all the city. +LJ028-0211|He appointed Gobrias governor of Babylon.|He appointed Gobrias governor of Babylon. +LJ028-0212|On the night of the eleventh day Gobrias killed the son of the King.|On the night of the eleventh day Gobrias killed the son of the King. +LJ028-0213|Nor does the royal record of Babylon contain the only contemporary account of the fall of the city,|Nor does the royal record of Babylon contain the only contemporary account of the fall of the city, +LJ028-0214|for upon a barrel-shaped cylinder of clay bearing a long inscription we have Cyrus's account of his capture of Babylon. Extracts from it are as follows:|for upon a barrel-shaped cylinder of clay bearing a long inscription we have Cyrus's account of his capture of Babylon. Extracts from it are as follows: +LJ028-0215|Marduk, the great lord, looking with joy on his pious works and upright heart,|Marduk, the great lord, looking with joy on his pious works and upright heart, +LJ028-0216|commanded him (Cyrus) to go forth to his city Babylon, and he went by his side as a friend and companion.|commanded him (Cyrus) to go forth to his city Babylon, and he went by his side as a friend and companion. +LJ028-0217|His many troops, whose number, like the waters of the river, could not be counted, marched in full armor at his side,|His many troops, whose number, like the waters of the river, could not be counted, marched in full armor at his side, +LJ028-0218|Without a skirmish or a battle, he permitted them to enter Babylon, and, sparing the city, he delivered the King Nabonidus to him.|Without a skirmish or a battle, he permitted them to enter Babylon, and, sparing the city, he delivered the King Nabonidus to him. +LJ028-0219|All the people of Babylon prostrated themselves before him, and, kissing his feet, rejoiced in his sovereignty, while happiness shone on their faces.|All the people of Babylon prostrated themselves before him, and, kissing his feet, rejoiced in his sovereignty, while happiness shone on their faces. +LJ028-0220|The inscription continues: I am Cyrus, king of the world.|The inscription continues: I am Cyrus, king of the world. +LJ028-0221|When I made my gracious entry into Babylon, with exceeding joy I took up my abode in the royal palace.|When I made my gracious entry into Babylon, with exceeding joy I took up my abode in the royal palace. +LJ028-0222|My many troops marched peacefully into Babylon.|My many troops marched peacefully into Babylon. +LJ028-0223|I gave heed to the needs of Babylon and its cities, and the servitude of the Babylonians, whatever was oppressive, I removed from them.|I gave heed to the needs of Babylon and its cities, and the servitude of the Babylonians, whatever was oppressive, I removed from them. +LJ028-0224|I quieted their sighings and soothed their sorrows.|I quieted their sighings and soothed their sorrows. +LJ028-0225|A much longer account of the capture of Babylon by Cyrus appears in the writings of Herodotus.|A much longer account of the capture of Babylon by Cyrus appears in the writings of Herodotus. +LJ028-0226|Though Herodotus wrote nearly a hundred years after Babylon fell, his story seems to bear the stamp of truth.|Though Herodotus wrote nearly a hundred years after Babylon fell, his story seems to bear the stamp of truth. +LJ028-0227|He certainly mentions details which neither Nabonidus nor Cyrus would care to have appear in their royal records. His story is as follows:|He certainly mentions details which neither Nabonidus nor Cyrus would care to have appear in their royal records. His story is as follows: +LJ028-0228|Cyrus, with the first approach of the ensuing spring, marched forward against Babylon.|Cyrus, with the first approach of the ensuing spring, marched forward against Babylon. +LJ028-0229|The Babylonians, encamped without their walls, awaited his coming.|The Babylonians, encamped without their walls, awaited his coming. +LJ028-0230|A battle was fought at a short distance from the city, in which the Babylonians were defeated by the Persian King,|A battle was fought at a short distance from the city, in which the Babylonians were defeated by the Persian King, +LJ028-0231|whereupon they withdrew within their defenses.|whereupon they withdrew within their defenses. +LJ028-0232|Here they shut themselves up and made light of his siege, having laid in a store of provision for many years in preparation against this attack;|Here they shut themselves up and made light of his siege, having laid in a store of provision for many years in preparation against this attack; +LJ028-0233|for when they saw Cyrus conquering nation after nation, they were convinced that he would never stop, and their turn would come at last.|for when they saw Cyrus conquering nation after nation, they were convinced that he would never stop, and their turn would come at last. +LJ028-0234|Cyrus was now reduced to great perplexity, as time went on and he made no progress against the place.|Cyrus was now reduced to great perplexity, as time went on and he made no progress against the place. +LJ028-0235|In this distress either someone made this suggestion to him, or he bethought himself of a plan which he proceeded to put in execution.|In this distress either someone made this suggestion to him, or he bethought himself of a plan which he proceeded to put in execution. +LJ028-0236|He placed a portion of his army at the point where the river enters the city, and another body at the back of the place where it issues forth,|He placed a portion of his army at the point where the river enters the city, and another body at the back of the place where it issues forth, +LJ028-0237|with orders to march into the town by the bed of the stream, as soon as the water became shallow enough:|with orders to march into the town by the bed of the stream, as soon as the water became shallow enough: +LJ028-0238|he then himself drew off with the unwarlike portion of his host, and made for the place where Nitocris dug the basin for the river,|he then himself drew off with the unwarlike portion of his host, and made for the place where Nitocris dug the basin for the river, +LJ028-0239|where he did exactly what she had done formerly: he turned the Euphrates by a canal into the basin, which was then a marsh,|where he did exactly what she had done formerly: he turned the Euphrates by a canal into the basin, which was then a marsh, +LJ028-0240|on which the river sank to such an extent that the natural bed of the stream became fordable.|on which the river sank to such an extent that the natural bed of the stream became fordable. +LJ028-0241|Hereupon the Persians who had been left for the purpose at Babylon by the river side|Hereupon the Persians who had been left for the purpose at Babylon by the river side +LJ028-0242|entered the stream, which had now sunk so as to reach about midway up a man's thigh, and thus got into the town.|entered the stream, which had now sunk so as to reach about midway up a man's thigh, and thus got into the town. +LJ028-0243|Had the Babylonians been apprised of what Cyrus was about, or had they noticed their danger, they would never have allowed the Persians to enter the city,|Had the Babylonians been apprised of what Cyrus was about, or had they noticed their danger, they would never have allowed the Persians to enter the city, +LJ028-0244|but would have destroyed them utterly; for they would have made fast all the street gates which gave upon the river,|but would have destroyed them utterly; for they would have made fast all the street gates which gave upon the river, +LJ028-0245|and mounting upon the walls along both sides of the stream, would so have caught the enemy as it were in a trap.|and mounting upon the walls along both sides of the stream, would so have caught the enemy as it were in a trap. +LJ028-0246|But, as it was, the Persians came upon them by surprise and so took the city.|But, as it was, the Persians came upon them by surprise and so took the city. +LJ028-0247|Owing to the vast size of the place, the inhabitants of the central parts (as the residents of Babylon declare),|Owing to the vast size of the place, the inhabitants of the central parts (as the residents of Babylon declare), +LJ028-0248|long after the outer portions of the town were taken, knew nothing of what had chanced, but as they were engaged in a festival,|long after the outer portions of the town were taken, knew nothing of what had chanced, but as they were engaged in a festival, +LJ028-0249|continued dancing and reveling until they learned the capture but too certainly.|continued dancing and reveling until they learned the capture but too certainly. +LJ028-0250|Such, then, were the circumstances of the first taking of Babylon.|Such, then, were the circumstances of the first taking of Babylon. +LJ028-0251|When Cyrus took Babylon, little or no force was employed.|When Cyrus took Babylon, little or no force was employed. +LJ028-0252|Only the King's son, Belshazzar, was killed.|Only the King's son, Belshazzar, was killed. +LJ028-0253|The city was spared; the great walls were left standing; the daily sacrifices were continued in the temples, and Cyrus made his home in the royal palace.|The city was spared; the great walls were left standing; the daily sacrifices were continued in the temples, and Cyrus made his home in the royal palace. +LJ028-0254|The people, enjoying the greater freedom which Cyrus permitted them, were contented, and life in Babylon went on about as before.|The people, enjoying the greater freedom which Cyrus permitted them, were contented, and life in Babylon went on about as before. +LJ028-0255|In 529 Cyrus died.|In five twenty-nine Cyrus died. +LJ028-0256|During the reigns of the two following Persian kings Babylon was slowly regaining its independence,|During the reigns of the two following Persian kings Babylon was slowly regaining its independence, +LJ028-0257|and in 521 Nebuchadnezzar III., a native Babylonian, was placed on the throne.|and in five twenty-one Nebuchadnezzar the third, a native Babylonian, was placed on the throne. +LJ028-0258|Then the Babylonians secretly plotted to throw off the Persian yoke.|Then the Babylonians secretly plotted to throw off the Persian yoke. +LJ028-0259|That same year, when Darius Hystaspes came to the Persian throne, the Babylonians openly rebelled.|That same year, when Darius Hystaspes came to the Persian throne, the Babylonians openly rebelled. +LJ028-0260|The following story from Herodotus tells the results:|The following story from Herodotus tells the results: +LJ028-0261|At last when the time came for rebelling openly they did as follows:|At last when the time came for rebelling openly they did as follows: +LJ028-0262|having first set apart their mothers, each man chose besides out of his whole household one woman whomsoever he pleased;|having first set apart their mothers, each man chose besides out of his whole household one woman whomsoever he pleased; +LJ028-0263|these alone were allowed to live, while all the rest were brought to one place and strangled.|these alone were allowed to live, while all the rest were brought to one place and strangled. +LJ028-0264|The women chosen were kept to make bread for the men; while the others were strangled that they might not consume the stores.|The women chosen were kept to make bread for the men; while the others were strangled that they might not consume the stores. +LJ028-0265|When tidings reached Darius of what had happened,|When tidings reached Darius of what had happened, +LJ028-0266|he drew together all his power and began the war by marching straight upon Babylon and laying siege to the place.|he drew together all his power and began the war by marching straight upon Babylon and laying siege to the place. +LJ028-0267|The Babylonians, however, cared not a whit for his siege.|The Babylonians, however, cared not a whit for his siege. +LJ028-0268|Mounting upon the battlements that crowned their walls, they insulted and jeered at Darius and his mighty host.|Mounting upon the battlements that crowned their walls, they insulted and jeered at Darius and his mighty host. +LJ028-0269|One even shouted to them and said, "Why sit ye there, Persians? Why do ye not go back to your homes? Till mules foal ye will not take our city!"|One even shouted to them and said, "Why sit ye there, Persians? Why do ye not go back to your homes? Till mules foal ye will not take our city!" +LJ028-0270|This was said by a Babylonian who thought that a mule would never foal.|This was said by a Babylonian who thought that a mule would never foal. +LJ028-0271|Now when a year and seven months had passed, Darius and his army were quite wearied out, finding that they could not anyhow take the city.|Now when a year and seven months had passed, Darius and his army were quite wearied out, finding that they could not anyhow take the city. +LJ028-0272|All stratagems and all arts had been used, and yet the King could not prevail|All stratagems and all arts had been used, and yet the King could not prevail +LJ028-0273|not even when he tried the means by which Cyrus had made himself master of the place.|not even when he tried the means by which Cyrus had made himself master of the place. +LJ028-0274|The Babylonians were ever upon the watch, and he found no way of conquering them.|The Babylonians were ever upon the watch, and he found no way of conquering them. +LJ028-0275|At last, in the twentieth month,|At last, in the twentieth month, +LJ028-0276|a marvelous thing happened to Zopyrus, son of the Megabyzus who was among the seven men that overthrew the Magus.|a marvelous thing happened to Zopyrus, son of the Megabyzus who was among the seven men that overthrew the Magus. +LJ028-0277|One of his sumpter-mules gave birth to a foal.|One of his sumpter-mules gave birth to a foal. +LJ028-0278|Zopyrus, when they told him, not thinking that it could be true, went and saw the colt with his own eyes;|Zopyrus, when they told him, not thinking that it could be true, went and saw the colt with his own eyes; +LJ028-0279|after which he commanded his servants to tell no one what had come to pass, while he himself pondered the matter.|after which he commanded his servants to tell no one what had come to pass, while he himself pondered the matter. +LJ028-0280|Calling to mind then the words of the Babylonian at the beginning of the siege:|Calling to mind then the words of the Babylonian at the beginning of the siege: +LJ028-0282|for it seemed to him that there was a divine providence in the man having used the phrase, and then his mule having foaled.|for it seemed to him that there was a divine providence in the man having used the phrase, and then his mule having foaled. +LJ028-0283|As soon therefore as he felt within himself that Babylon was fated to be taken, he went to Darius and asked him if he set a very high value on its conquest.|As soon therefore as he felt within himself that Babylon was fated to be taken, he went to Darius and asked him if he set a very high value on its conquest. +LJ028-0284|When he found that Darius did indeed value it highly, he considered further with himself how he might make the deed his own, and be the man to take Babylon.|When he found that Darius did indeed value it highly, he considered further with himself how he might make the deed his own, and be the man to take Babylon. +LJ028-0285|Noble exploits in Persia are ever highly honored and bring their authors to greatness.|Noble exploits in Persia are ever highly honored and bring their authors to greatness. +LJ028-0286|He therefore reviewed all ways of bringing the city under,|He therefore reviewed all ways of bringing the city under, +LJ028-0287|but found none by which he could hope to prevail, unless he maimed himself and then went over to the enemy.|but found none by which he could hope to prevail, unless he maimed himself and then went over to the enemy. +LJ028-0288|To do this seeming to him a light matter, he mutilated himself in a way that was utterly without remedy.|To do this seeming to him a light matter, he mutilated himself in a way that was utterly without remedy. +LJ028-0289|For he cut off his own nose and ears, and then, clipping his hair close and flogging himself with a scourge,|For he cut off his own nose and ears, and then, clipping his hair close and flogging himself with a scourge, +LJ028-0290|he came in this plight before Darius.|he came in this plight before Darius. +LJ028-0291|Wrath stirred within the King at the sight of a man of his lofty rank in such a condition;|Wrath stirred within the King at the sight of a man of his lofty rank in such a condition; +LJ028-0292|leaping down from his throne he exclaimed aloud and asked Zopyrus who it was that had disfigured him, and what he had done to be so treated.|leaping down from his throne he exclaimed aloud and asked Zopyrus who it was that had disfigured him, and what he had done to be so treated. +LJ028-0293|Zopyrus answered, "There is not a man in the world, but thou, O King, that could reduce me to such a plight|Zopyrus answered, "There is not a man in the world, but thou, O King, that could reduce me to such a plight +LJ028-0294|no stranger's hands have wrought this work on me, but my own only.|no stranger's hands have wrought this work on me, but my own only. +LJ028-0295|I maimed myself because I could not endure that the Assyrians should laugh at the Persians. "Wretched man," said Darius,|I maimed myself because I could not endure that the Assyrians should laugh at the Persians. "Wretched man," said Darius, +LJ028-0296|thou coverest the foulest deeds with the fairest possible name, when thou sayest thy maiming is to help our siege forward.|thou coverest the foulest deeds with the fairest possible name, when thou sayest thy maiming is to help our siege forward. +LJ028-0297|How will thy disfigurement, thou simpleton, induce the enemy to yield one day sooner?|How will thy disfigurement, thou simpleton, induce the enemy to yield one day sooner? +LJ028-0298|Surely thou hadst gone out of thy mind when thou didst so misuse thyself.|Surely thou hadst gone out of thy mind when thou didst so misuse thyself. +LJ028-0299|"Had I told thee," rejoined the other, "what I was bent on doing, thou wouldst not have suffered it;|"Had I told thee," rejoined the other, "what I was bent on doing, thou wouldst not have suffered it; +LJ028-0300|as it is, I kept my own counsel, and so accomplished my plans.|as it is, I kept my own counsel, and so accomplished my plans. +LJ028-0301|Now, therefore, if there be no failure on thy part, we shall take Babylon.|Now, therefore, if there be no failure on thy part, we shall take Babylon. +LJ028-0302|I will desert to the enemy as I am, and when I get into their city I will tell them that it is by thee that I have been thus treated.|I will desert to the enemy as I am, and when I get into their city I will tell them that it is by thee that I have been thus treated. +LJ028-0303|I think they will believe my words and entrust me with a command of troops. Thou, on thy part, must wait|I think they will believe my words and entrust me with a command of troops. Thou, on thy part, must wait +LJ028-0304|till the tenth day after I am entered within the town, and then place near to the gates of Semiramis a detachment of thy army,|till the tenth day after I am entered within the town, and then place near to the gates of Semiramis a detachment of thy army, +LJ028-0305|troops for whose loss thou wilt care little, a thousand men.|troops for whose loss thou wilt care little, a thousand men. +LJ028-0306|Wait, after that, seven days, and post me another detachment, two thousand strong, at the Nineveh gates;|Wait, after that, seven days, and post me another detachment, two thousand strong, at the Nineveh gates; +LJ028-0307|then let twenty days pass, and at the end of that time station near the Chaldasan gates a body of four thousand.|then let twenty days pass, and at the end of that time station near the Chaldasan gates a body of four thousand. +LJ028-0308|Let neither these nor the former troops be armed with any weapons but their swords those thou mayest leave them.|Let neither these nor the former troops be armed with any weapons but their swords those thou mayest leave them. +LJ028-0309|After the twenty days are over, bid thy whole army attack the city on every side, and put me two bodies of Persians,|After the twenty days are over, bid thy whole army attack the city on every side, and put me two bodies of Persians, +LJ028-0310|one at the Belian, the other at the Cissian gates; for I expect that, on account of my successes,|one at the Belian, the other at the Cissian gates; for I expect that, on account of my successes, +LJ028-0311|the Babylonians will entrust everything, even the keys of their gates, to me. Then it will be for me and my Persians to do the rest.|the Babylonians will entrust everything, even the keys of their gates, to me. Then it will be for me and my Persians to do the rest. +LJ028-0312|Having left these instructions, Zopyrus fled towards the gates of the town, often looking back, to give himself the air of a deserter.|Having left these instructions, Zopyrus fled towards the gates of the town, often looking back, to give himself the air of a deserter. +LJ028-0313|The men upon the towers, whose business it was to keep a lookout,|The men upon the towers, whose business it was to keep a lookout, +LJ028-0314|observing him, hastened down, and setting one of the gates slightly ajar, questioned him who he was, and on what errand he had come.|observing him, hastened down, and setting one of the gates slightly ajar, questioned him who he was, and on what errand he had come. +LJ028-0315|He replied that he was Zopyrus, and deserted to them from the Persians.|He replied that he was Zopyrus, and deserted to them from the Persians. +LJ028-0316|Then the doorkeepers, when they heard this, carried him at once before the Magistrates.|Then the doorkeepers, when they heard this, carried him at once before the Magistrates. +LJ028-0317|Introduced into their assembly, he began to bewail his misfortunes, telling them that|Introduced into their assembly, he began to bewail his misfortunes, telling them that +LJ028-0318|Darius had maltreated him in the way they could see, only because he had given advice that the siege should be raised, since there seemed no hope of taking the city.|Darius had maltreated him in the way they could see, only because he had given advice that the siege should be raised, since there seemed no hope of taking the city. +LJ028-0319|"And now," he went on to say, "my coming to you, Babylonians,|"And now," he went on to say, "my coming to you, Babylonians, +LJ028-0320|will prove the greatest gain that you could possibly receive, while to Darius and the Persians it will be the severest loss.|will prove the greatest gain that you could possibly receive, while to Darius and the Persians it will be the severest loss. +LJ028-0321|Verily he by whom I have been so mutilated shall not escape unpunished. And truly all the paths of his counsels are known to me.|Verily he by whom I have been so mutilated shall not escape unpunished. And truly all the paths of his counsels are known to me. +LJ028-0322|Thus did Zopyrus speak.|Thus did Zopyrus speak. +LJ028-0323|The Babylonians, seeing a Persian of such exalted rank in so grievous a plight, his nose and ears cut off,|The Babylonians, seeing a Persian of such exalted rank in so grievous a plight, his nose and ears cut off, +LJ028-0324|his body red with marks of scourging and with blood, had no suspicion but that he spoke the truth, and was really come to be their friend and helper.|his body red with marks of scourging and with blood, had no suspicion but that he spoke the truth, and was really come to be their friend and helper. +LJ028-0325|They were ready, therefore, to grant him anything he asked;|They were ready, therefore, to grant him anything he asked; +LJ028-0326|and on his suing for a command, they entrusted to him a body of troops with the help of which he proceeded to do as he had arranged with Darius.|and on his suing for a command, they entrusted to him a body of troops with the help of which he proceeded to do as he had arranged with Darius. +LJ028-0327|On the tenth day after his flight he led out his detachment, and surrounding the thousand men,|On the tenth day after his flight he led out his detachment, and surrounding the thousand men, +LJ028-0328|whom Darius according to agreement had sent first, he fell upon them and slew them all.|whom Darius according to agreement had sent first, he fell upon them and slew them all. +LJ028-0329|Then the Babylonians, seeing that his deeds were as brave as his words, were beyond measure pleased, and set no bounds to their trust.|Then the Babylonians, seeing that his deeds were as brave as his words, were beyond measure pleased, and set no bounds to their trust. +LJ028-0330|and when the next period agreed on had elapsed, again with a band of picked men he sallied forth, and slaughtered the two thousand.|and when the next period agreed on had elapsed, again with a band of picked men he sallied forth, and slaughtered the two thousand. +LJ028-0331|After this second exploit, his praise was in all mouths.|After this second exploit, his praise was in all mouths. +LJ028-0332|Once more, however, he waited till the interval appointed had gone by, and then leading the troops to the place where the four thousand were,|Once more, however, he waited till the interval appointed had gone by, and then leading the troops to the place where the four thousand were, +LJ028-0333|he put them also to the sword.|he put them also to the sword. +LJ028-0334|This last victory gave him the finishing stroke to his power and made him all in all with the Babylonians:|This last victory gave him the finishing stroke to his power and made him all in all with the Babylonians: +LJ028-0335|accordingly they committed to him the command of their whole army, and put the keys of their city into his hands.|accordingly they committed to him the command of their whole army, and put the keys of their city into his hands. +LJ028-0336|Darius now, still keeping to the plan agreed upon,|Darius now, still keeping to the plan agreed upon, +LJ028-0337|attacked the walls on every side, whereupon Zopyrus played out the remainder of his stratagem.|attacked the walls on every side, whereupon Zopyrus played out the remainder of his stratagem. +LJ028-0338|While the Babylonians, crowding to the walls, did their best to resist the Persian assault,|While the Babylonians, crowding to the walls, did their best to resist the Persian assault, +LJ028-0339|he threw open the Cissian and Belian gates, and admitted the enemy.|he threw open the Cissian and Belian gates, and admitted the enemy. +LJ028-0340|Such of the Babylonians as witnessed the treachery took refuge in the temple of Jupiter Belus;|Such of the Babylonians as witnessed the treachery took refuge in the temple of Jupiter Belus; +LJ028-0341|the rest who did not see it kept at their posts, till at last they too learned that they were betrayed.|the rest who did not see it kept at their posts, till at last they too learned that they were betrayed. +LJ028-0342|Thus was Babylon taken for the second time.|Thus was Babylon taken for the second time. +LJ028-0343|Darius having become master of the place, destroyed the wall, and tore down all the gates;|Darius having become master of the place, destroyed the wall, and tore down all the gates; +LJ028-0344|for Cyrus had done neither the one nor the other when he took Babylon.|for Cyrus had done neither the one nor the other when he took Babylon. +LJ028-0345|He then chose out near three thousand of the leading citizens and caused them to be crucified, while he allowed the remainder still to inhabit the city.|He then chose out near three thousand of the leading citizens and caused them to be crucified, while he allowed the remainder still to inhabit the city. +LJ028-0346|Further, wishing to prevent the race of the Babylonians from becoming extinct,|Further, wishing to prevent the race of the Babylonians from becoming extinct, +LJ028-0347|he provided wives for them in the room of those whom (as I explained before) they strangled to save their stores.|he provided wives for them in the room of those whom (as I explained before) they strangled to save their stores. +LJ028-0348|These he levied from the nations bordering on Babylonia,|These he levied from the nations bordering on Babylonia, +LJ028-0349|who were each required to send so large a number to Babylon, that in all there were collected no fewer than fifty thousand.|who were each required to send so large a number to Babylon, that in all there were collected no fewer than fifty thousand. +LJ028-0350|It is from these women that the Babylonians of our times are sprung.|It is from these women that the Babylonians of our times are sprung. +LJ028-0351|As for Zopyrus he was considered by Darius to have surpassed, in the greatness of his achievements, all other Persians,|As for Zopyrus he was considered by Darius to have surpassed, in the greatness of his achievements, all other Persians, +LJ028-0352|whether of former or of later times, except only Cyrus with whom no person ever yet thought himself worthy to compare.|whether of former or of later times, except only Cyrus with whom no person ever yet thought himself worthy to compare. +LJ028-0353|Darius, as the story goes, would often say that "he had rather Zopyrus were unmaimed, than be master of twenty more Babylons."|Darius, as the story goes, would often say that "he had rather Zopyrus were unmaimed, than be master of twenty more Babylons." +LJ028-0354|And he honored Zopyrus greatly; year by year he presented him with all the gifts which are held in most esteem among the Persians;|And he honored Zopyrus greatly; year by year he presented him with all the gifts which are held in most esteem among the Persians; +LJ028-0355|he gave him likewise the government of Babylon for his life, free from tribute, and he also granted him many other favors.|he gave him likewise the government of Babylon for his life, free from tribute, and he also granted him many other favors. +LJ028-0356|How much truth there may be in this interesting tale of Herodotus, we may never know,|How much truth there may be in this interesting tale of Herodotus, we may never know, +LJ028-0357|yet we may be sure that Babylon was taken by Darius only by use of stratagem. Its walls were impregnable.|yet we may be sure that Babylon was taken by Darius only by use of stratagem. Its walls were impregnable. +LJ028-0358|Cyrus had permitted them to stand, and as long as he made Babylon his home, the city was as strongly protected as ever.|Cyrus had permitted them to stand, and as long as he made Babylon his home, the city was as strongly protected as ever. +LJ028-0359|Darius, who besieged the rebellious city twice, weakened it by destroying some of its walls.|Darius, who besieged the rebellious city twice, weakened it by destroying some of its walls. +LJ028-0360|During the reign of Xerxes again the city rebelled, and in 484 B.C. he captured it, and completely demolished its defenses.|During the reign of Xerxes again the city rebelled, and in four eighty-four B.C. he captured it, and completely demolished its defenses. +LJ028-0361|Yet Babylon continued to live,|Yet Babylon continued to live, +LJ028-0362|for history mentions the names of two of its later rulers. The palace of Nebuchadnezzar was occupied by Alexander the Great,|for history mentions the names of two of its later rulers. The palace of Nebuchadnezzar was occupied by Alexander the Great, +LJ028-0363|and there on June 13, 323 B.C., he met his death.|and there on June thirteen, three twenty-three B.C., he met his death. +LJ028-0364|The city then fell to Seleucus,|The city then fell to Seleucus, +LJ028-0365|one of Alexander's generals, who for a time made it his home, but he was a Greek and cared little for things Babylonian.|one of Alexander's generals, who for a time made it his home, but he was a Greek and cared little for things Babylonian. +LJ028-0366|Therefore, to destroy the power of the old capital, he planned to build Seleucia on the Tigris about fifty miles to the east.|Therefore, to destroy the power of the old capital, he planned to build Seleucia on the Tigris about fifty miles to the east. +LJ028-0367|The priests of the temple of Bel, so a story tells us,|The priests of the temple of Bel, so a story tells us, +LJ028-0368|learned of his purpose, and when they were consulted as to the most favorable time for beginning the work upon the new city,|learned of his purpose, and when they were consulted as to the most favorable time for beginning the work upon the new city, +LJ028-0369|they intentionally mentioned a most unfavorable hour.|they intentionally mentioned a most unfavorable hour. +LJ028-0370|The priests' deception was unavailing, and in 275 B.C., the inhabitants of Babylon were transported to Seleucia.|The priests' deception was unavailing, and in two seventy-five B.C., the inhabitants of Babylon were transported to Seleucia. +LJ028-0371|Then the world metropolis, stripped of most of its population, became a mere village.|Then the world metropolis, stripped of most of its population, became a mere village. +LJ028-0372|The poor of the surrounding country occupied its dismantled palaces.|The poor of the surrounding country occupied its dismantled palaces. +LJ028-0373|The Hebrew exiles, whose ancestors Nebuchadnezzar had brought from Jerusalem,|The Hebrew exiles, whose ancestors Nebuchadnezzar had brought from Jerusalem, +LJ028-0374|settled there, and finally the place was abandoned to the Arabs of the desert.|settled there, and finally the place was abandoned to the Arabs of the desert. +LJ028-0375|Slowly the few remaining walls fell, and were buried in their own ruins.|Slowly the few remaining walls fell, and were buried in their own ruins. +LJ028-0376|As the centuries passed the mounds into which the city had turned grew higher and higher with the ruins of the huts later built upon them,|As the centuries passed the mounds into which the city had turned grew higher and higher with the ruins of the huts later built upon them, +LJ028-0377|until at last the foundations of the temples and palaces were buried fully a hundred feet beneath the surface.|until at last the foundations of the temples and palaces were buried fully a hundred feet beneath the surface. +LJ028-0378|Even the shepherds ceased to graze their sheep there, and the wandering Arabs,|Even the shepherds ceased to graze their sheep there, and the wandering Arabs, +LJ028-0379|fearing the wild beasts and evil spirits which lurk among all old ruins, refused to pitch their tents there.|fearing the wild beasts and evil spirits which lurk among all old ruins, refused to pitch their tents there. +LJ028-0380|The prophecy of the Hebrew Isaiah was fulfilled:|The prophecy of the Hebrew Isaiah was fulfilled: +LJ028-0381|Wild beasts of the desert shall lie there;|Wild beasts of the desert shall lie there; +LJ028-0382|and their houses shall be full of doleful creatures, and owls shall dwell there, and satyrs shall dance there,|and their houses shall be full of doleful creatures, and owls shall dwell there, and satyrs shall dance there, +LJ028-0383|and the wild beasts of the islands shall cry in their desolate houses, and dragons in their pleasant palaces.|and the wild beasts of the islands shall cry in their desolate houses, and dragons in their pleasant palaces. +LJ028-0384|So Babylon was buried and forgotten.|So Babylon was buried and forgotten. +LJ028-0385|It had become, as Dio Cassius said, "Mounds and legends and ruins."|It had become, as Dio Cassius said, "Mounds and legends and ruins." +LJ028-0386|But the walls of the old city had not yet served their full purpose.|But the walls of the old city had not yet served their full purpose. +LJ028-0387|The Sassanian kings of Persia were fond of hunting, and Babylon, then overgrown with trees, was their game preserve.|The Sassanian kings of Persia were fond of hunting, and Babylon, then overgrown with trees, was their game preserve. +LJ028-0388|The old walls were restored to a height sufficient to prevent the escape of the animals, and among the ruins the kings enjoyed their favorite sport.|The old walls were restored to a height sufficient to prevent the escape of the animals, and among the ruins the kings enjoyed their favorite sport. +LJ028-0389|St. Jerome said:|St. Jerome said: +LJ028-0390|I was informed by a certain Elamite brother, who came from those regions, and now leads the life of a monk at Jerusalem,|I was informed by a certain Elamite brother, who came from those regions, and now leads the life of a monk at Jerusalem, +LJ028-0391|that there is a royal hunting ground at Babylon, and that wild game of every kind is contained within the circuit of its walls.|that there is a royal hunting ground at Babylon, and that wild game of every kind is contained within the circuit of its walls. +LJ028-0392|The statement of St. Jerome is confirmed by the following passage from Zosimus, a Greek writer of the fifth century A.D.:|The statement of St. Jerome is confirmed by the following passage from Zosimus, a Greek writer of the fifth century A.D.: +LJ028-0393|As the Emperor Julian was marching forward through Babylonia,|As the Emperor Julian was marching forward through Babylonia, +LJ028-0394|he passed other unimportant fortresses, and came at last to a walled enclosure, which the natives pointed out as a royal hunting ground.|he passed other unimportant fortresses, and came at last to a walled enclosure, which the natives pointed out as a royal hunting ground. +LJ028-0395|It was a low rampart, enclosing a wide space planted with trees of every sort, in which all kinds of beasts were shut up;|It was a low rampart, enclosing a wide space planted with trees of every sort, in which all kinds of beasts were shut up; +LJ028-0396|they were supplied with food by keepers, and gave the king the opportunity of hunting whenever he felt inclined.|they were supplied with food by keepers, and gave the king the opportunity of hunting whenever he felt inclined. +LJ028-0397|When Julian saw this, he caused a large part of the wall to be overthrown, and as the beasts escaped they were shot down by his soldiers.|When Julian saw this, he caused a large part of the wall to be overthrown, and as the beasts escaped they were shot down by his soldiers. +LJ028-0398|The walls of Babylon were destined to serve still another purpose.|The walls of Babylon were destined to serve still another purpose. +LJ028-0399|The spread of Mohammedanism caused new cities to be built, and Babylon was the quarry for their building material.|The spread of Mohammedanism caused new cities to be built, and Babylon was the quarry for their building material. +LJ028-0400|The walls of Babylon were transformed into the sacred cities of Kerbela and Nejef.|The walls of Babylon were transformed into the sacred cities of Kerbela and Nejef. +LJ028-0401|In the eleventh century, on the site of the southern part of Babylon, the city of Hillah was built.|In the eleventh century, on the site of the southern part of Babylon, the city of Hillah was built. +LJ028-0402|Hillah might be called a child of Babylon, for it is almost entirely constructed with Nebuchadnezzar's bricks.|Hillah might be called a child of Babylon, for it is almost entirely constructed with Nebuchadnezzar's bricks. +LJ028-0403|The walls of the houses are built of them.|The walls of the houses are built of them. +LJ028-0404|The courtyards and streets are paved with them, and as you walk about the city the name of Nebuchadnezzar everywhere meets your eye.|The courtyards and streets are paved with them, and as you walk about the city the name of Nebuchadnezzar everywhere meets your eye. +LJ028-0405|Many of the ten thousand people living in Hillah still gain their livelihood by digging the bricks from the ruins to sell to the modern builders.|Many of the ten thousand people living in Hillah still gain their livelihood by digging the bricks from the ruins to sell to the modern builders. +LJ028-0406|The great irrigating dams across the Euphrates are constructed entirely of them.|The great irrigating dams across the Euphrates are constructed entirely of them. +LJ028-0407|The people of Hillah, too, are a survival of Babylonian times.|The people of Hillah, too, are a survival of Babylonian times. +LJ028-0408|Some are Arabs of the same tribes which used to roam the desert in Nebuchadnezzar's days. Some are the children of the Hebrew exiles of old.|Some are Arabs of the same tribes which used to roam the desert in Nebuchadnezzar's days. Some are the children of the Hebrew exiles of old. +LJ028-0409|Some, calling themselves Christians, are the descendants of Babylonians, perhaps of Nebuchadnezzar himself.|Some, calling themselves Christians, are the descendants of Babylonians, perhaps of Nebuchadnezzar himself. +LJ028-0410|There among the ruins they still live in the same kind of houses,|There among the ruins they still live in the same kind of houses, +LJ028-0411|dressing the same, eating the same food as did their ancestors when Nebuchadnezzar built the walls of Babylon.|dressing the same, eating the same food as did their ancestors when Nebuchadnezzar built the walls of Babylon. +LJ028-0412|Among the first of the modern travelers to describe the ruins of Babylon was Anthony Shirley, an Englishman who visited Mesopotamia in 1599.|Among the first of the modern travelers to describe the ruins of Babylon was Anthony Shirley, an Englishman who visited Mesopotamia in fifteen ninety-nine. +LJ028-0413|In his quaint way he says:|In his quaint way he says: +LJ028-0414|All the ground on which Babylon was spread is left now desolate; nothing standing in that Peninsula between the Euphrates and the Tigris,|All the ground on which Babylon was spread is left now desolate; nothing standing in that Peninsula between the Euphrates and the Tigris, +LJ028-0415|but only part, and that a small part, of the great tower, which God hath suffered to stand|but only part, and that a small part, of the great tower, which God hath suffered to stand +LJ028-0416|(if man may speak so confidently of His great impenetrable counsels), for an eternal Testimony of His great work in the confusion of Man's pride,|(if man may speak so confidently of His great impenetrable counsels), for an eternal Testimony of His great work in the confusion of Man's pride, +LJ028-0417|and that Arke of Nebuchadnezzar for as perpetual a memory of his great idolatry and condigne punishment.|and that Arke of Nebuchadnezzar for as perpetual a memory of his great idolatry and condigne punishment. +LJ028-0418|About that same time Pietro della Valle, an Italian, visited Babylon,|About that same time Pietro della Valle, an Italian, visited Babylon, +LJ028-0419|and digging from the wall an inscribed square brick bearing the name of Nebuchadnezzar, he took it to Rome where it may still be seen.|and digging from the wall an inscribed square brick bearing the name of Nebuchadnezzar, he took it to Rome where it may still be seen. +LJ028-0420|That was the first object taken from Babylon to Europe;|That was the first object taken from Babylon to Europe; +LJ028-0421|it was the beginning of the great collections of Babylonian antiquities in the museums of the Western world.|it was the beginning of the great collections of Babylonian antiquities in the museums of the Western world. +LJ028-0422|Among the later visitors to Babylon was the great Niebuhr.|Among the later visitors to Babylon was the great Niebuhr. +LJ028-0423|In 1812, James Claudius Rich, the British Resident at Baghdad, made the first complete examination of the ruins.|In eighteen twelve, James Claudius Rich, the British Resident at Baghdad, made the first complete examination of the ruins. +LJ028-0424|Porter, Layard, and Rawlinson followed him, but the real scientific exploration of Babylon and its walls|Porter, Layard, and Rawlinson followed him, but the real scientific exploration of Babylon and its walls +LJ028-0425|was begun by the Deutsche Orientgesellschaft, in 1889, and continued till the summer of 1915.|was begun by the Deutsche Orientgesellschaft, in eighteen eighty-nine, and continued till the summer of nineteen fifteen. +LJ028-0426|For fifteen years Dr. Koldewey and his assistants, with a force of two hundred native workmen, have labored there winter and summer.|For fifteen years Dr. Koldewey and his assistants, with a force of two hundred native workmen, have labored there winter and summer. +LJ028-0427|The enormous amount of debris which buried the palaces and temples and walls of Nebuchadnezzar's city, in places to the depth of a hundred feet,|The enormous amount of debris which buried the palaces and temples and walls of Nebuchadnezzar's city, in places to the depth of a hundred feet, +LJ028-0428|has been removed, and the surrounding city walls have been traced.|has been removed, and the surrounding city walls have been traced. +LJ028-0429|The excavations have shown that Babylon, as the ancients told us, was nearly square.|The excavations have shown that Babylon, as the ancients told us, was nearly square. +LJ028-0430|The Euphrates flowed through it, but the greater part of the city was on the eastern shore.|The Euphrates flowed through it, but the greater part of the city was on the eastern shore. +LJ028-0431|The city walls, of which the ancients were so proud, appear here and there like low ridges far out on the plain;|The city walls, of which the ancients were so proud, appear here and there like low ridges far out on the plain; +LJ028-0432|other parts of them have disappeared entirely.|other parts of them have disappeared entirely. +LJ028-0433|In the northern part of the enclosure to the east of the river,|In the northern part of the enclosure to the east of the river, +LJ028-0434|the large high mound, which resembles a mountain from a distance, still bears the ancient name Babel.|the large high mound, which resembles a mountain from a distance, still bears the ancient name Babel. +LJ028-0435|Arabs, searching for bricks, have burrowed their way down deep into it, revealing massive walls and arches.|Arabs, searching for bricks, have burrowed their way down deep into it, revealing massive walls and arches. +LJ028-0436|The Germans maintain that it is the ruin of the Tower of Babel.|The Germans maintain that it is the ruin of the Tower of Babel. +LJ028-0437|Here, it has been suggested, were the famous hanging gardens which some ancient authors included among the Seven Wonders of the World.|Here, it has been suggested, were the famous hanging gardens which some ancient authors included among the Seven Wonders of the World. +LJ028-0438|However, it is possible that the hanging gardens existed only in the imagination of the Greek writers,|However, it is possible that the hanging gardens existed only in the imagination of the Greek writers, +LJ028-0439|for none of the many building inscriptions from Nebuchadnezzar mentions them.|for none of the many building inscriptions from Nebuchadnezzar mentions them. +LJ028-0440|Possibly along the terraces of the walls, or upon the stages of some lofty temple tower,|Possibly along the terraces of the walls, or upon the stages of some lofty temple tower, +LJ028-0441|trees and overhanging vines were planted, and thus the travelers' tales arose.|trees and overhanging vines were planted, and thus the travelers' tales arose. +LJ028-0442|At a distance of about two miles to the south of Babel is the larger and lower mound called the Kasr, or the Fortress,|At a distance of about two miles to the south of Babel is the larger and lower mound called the Kasr, or the Fortress, +LJ028-0443|because great masses of masonry used to project from its surface.|because great masses of masonry used to project from its surface. +LJ028-0444|Deep down in the mound the Germans discovered the palace of Nebuchadnezzar with its hundreds of small chambers and its huge surrounding walls.|Deep down in the mound the Germans discovered the palace of Nebuchadnezzar with its hundreds of small chambers and its huge surrounding walls. +LJ028-0445|The mound still farther south is called Amran, because upon its summit stands the tomb of a Mohammedan saint of that name.|The mound still farther south is called Amran, because upon its summit stands the tomb of a Mohammedan saint of that name. +LJ028-0446|There lie the ruins of the famous temple of Esagil, sacred to Marduk.|There lie the ruins of the famous temple of Esagil, sacred to Marduk. +LJ028-0447|Upon the little mound Jumjuma farther on, an Arab village has long stood.|Upon the little mound Jumjuma farther on, an Arab village has long stood. +LJ028-0448|All of the ancient writers agree in saying that Babylon was surrounded with both inner and outer walls, and the ruins confirm their statements.|All of the ancient writers agree in saying that Babylon was surrounded with both inner and outer walls, and the ruins confirm their statements. +LJ028-0449|Parts of the walls of Nineveh are still standing to the height of one hundred and twenty-five feet,|Parts of the walls of Nineveh are still standing to the height of one hundred and twenty-five feet, +LJ028-0450|but the walls of Babylon have so long been used to supply bricks to the builders of the neighboring cities that only their bases remain.|but the walls of Babylon have so long been used to supply bricks to the builders of the neighboring cities that only their bases remain. +LJ028-0451|In places even the bases have disappeared, and their moats have long been filled with the drifting sand.|In places even the bases have disappeared, and their moats have long been filled with the drifting sand. +LJ028-0452|The outer wall bore the name of Nimitti-Bel. Its direction was northeast and southwest, forming a triangle with the river.|The outer wall bore the name of Nimitti-Bel. Its direction was northeast and southwest, forming a triangle with the river. +LJ028-0453|The northeastern section may now be traced for a distance of less than three miles, and the southwestern "for more than a mile,|The northeastern section may now be traced for a distance of less than three miles, and the southwestern "for more than a mile, +LJ028-0454|but both sections originally reached the river.|but both sections originally reached the river. +LJ028-0455|It seems that the circuit of the outer wall was about eleven miles.|It seems that the circuit of the outer wall was about eleven miles. +LJ028-0456|The small portions of it which have been excavated suffice to show its construction.|The small portions of it which have been excavated suffice to show its construction. +LJ028-0457|The moat, ten feet deep, and of a width no longer known, ran close to its base. The wall was double.|The moat, ten feet deep, and of a width no longer known, ran close to its base. The wall was double. +LJ028-0458|Its outer part was about twenty-four feet in thickness, and its foundations, as Nebuchadnezzar said, were carried down to the water level.|Its outer part was about twenty-four feet in thickness, and its foundations, as Nebuchadnezzar said, were carried down to the water level. +LJ028-0459|Its bricks, measuring about thirteen inches square and three inches in thickness, were burned and stamped with the usual short inscription:|Its bricks, measuring about thirteen inches square and three inches in thickness, were burned and stamped with the usual short inscription: +LJ028-0460|Nebuchadnezzar, King of Babylon, the restorer of the temples Esagil and Ezida,|Nebuchadnezzar, King of Babylon, the restorer of the temples Esagil and Ezida, +LJ028-0461|the first-born son of Nabopolassar, King of Babylon.|the first-born son of Nabopolassar, King of Babylon. +LJ028-0462|They were laid in bitumen.|They were laid in bitumen. +LJ028-0463|The inner part of the wall was constructed of unburned bricks, and at a distance of about thirty-six feet from the outer part.|The inner part of the wall was constructed of unburned bricks, and at a distance of about thirty-six feet from the outer part. +LJ028-0464|The intervening space, which was filled with dirt probably to the upper inner edge of the outer part,|The intervening space, which was filled with dirt probably to the upper inner edge of the outer part, +LJ028-0465|served as an elevated road where several chariots might have been driven abreast.|served as an elevated road where several chariots might have been driven abreast. +LJ028-0466|This inner part was about twenty-four feet wide, and at intervals of about one hundred and forty feet it was surmounted with towers.|This inner part was about twenty-four feet wide, and at intervals of about one hundred and forty feet it was surmounted with towers. +LJ028-0467|The entire width of the outer defense, not including the moat, was therefore about eighty-two feet;|The entire width of the outer defense, not including the moat, was therefore about eighty-two feet; +LJ028-0468|its height was probably more than double its width, but that may never be determined.|its height was probably more than double its width, but that may never be determined. +LJ028-0469|The inner wall of Babylon was called Imgur-Bel, and like the outer wall, it was double.|The inner wall of Babylon was called Imgur-Bel, and like the outer wall, it was double. +LJ028-0470|Time has dealt even less kindly with it, for it may be traced only for the distance of about a mile along its eastern side.|Time has dealt even less kindly with it, for it may be traced only for the distance of about a mile along its eastern side. +LJ028-0471|Nebuchadnezzar says that he built it of burned bricks, but only sun-dried bricks laid in mud now appear.|Nebuchadnezzar says that he built it of burned bricks, but only sun-dried bricks laid in mud now appear. +LJ028-0472|Its outer part, about twelve feet in width, was protected with towers at intervals of sixty-five feet.|Its outer part, about twelve feet in width, was protected with towers at intervals of sixty-five feet. +LJ028-0473|A space of about twenty-three feet separated it from its inner part, which was about twenty feet in width.|A space of about twenty-three feet separated it from its inner part, which was about twenty feet in width. +LJ028-0474|It too was surmounted with towers.|It too was surmounted with towers. +LJ028-0475|No traces of its moat have appeared.|No traces of its moat have appeared. +LJ028-0476|The entire width of this inner defense was about fifty-five feet; its height is uncertain.|The entire width of this inner defense was about fifty-five feet; its height is uncertain. +LJ028-0477|To protect the sun-dried bricks of the inner wall from the winter rains|To protect the sun-dried bricks of the inner wall from the winter rains +LJ028-0478|there were drains of large burned bricks, some of which bore the following long inscription:|there were drains of large burned bricks, some of which bore the following long inscription: +LJ028-0479|Nebuchadnezzar, King of Babylon,|Nebuchadnezzar, King of Babylon, +LJ028-0480|the exalted prince, the protector of Esagil and Ezida, son of Nabopolassar, King of Babylon, am I.|the exalted prince, the protector of Esagil and Ezida, son of Nabopolassar, King of Babylon, am I. +LJ028-0481|Nabopolassar, the father, my begetter, built Imgur-Bel, the great wall of Babylon,|Nabopolassar, the father, my begetter, built Imgur-Bel, the great wall of Babylon, +LJ028-0482|but I, the devout petitioner, the worshipper of the gods, built the moat, and made its wall of burned brick and bitumen mountain high.|but I, the devout petitioner, the worshipper of the gods, built the moat, and made its wall of burned brick and bitumen mountain high. +LJ028-0483|O Marduk, great god, look joyfully upon the precious work of my hands. Be thou my protector.|O Marduk, great god, look joyfully upon the precious work of my hands. Be thou my protector. +LJ028-0484|Grant me as a gift a life of distant days.|Grant me as a gift a life of distant days. +LJ028-0485|The outer and inner defenses of Babylon were so strong and so high that no enemy could hope to take them,|The outer and inner defenses of Babylon were so strong and so high that no enemy could hope to take them, +LJ028-0486|yet the palace of Nebuchadnezzar was protected by a third defense far stronger.|yet the palace of Nebuchadnezzar was protected by a third defense far stronger. +LJ028-0487|Fortunately its walls have suffered less from the hands of the brick hunters, and the German excavators have been able to reconstruct their plan.|Fortunately its walls have suffered less from the hands of the brick hunters, and the German excavators have been able to reconstruct their plan. +LJ028-0488|They may best be described by means of the accompanying diagram representing a cross section.|They may best be described by means of the accompanying diagram representing a cross section. +LJ028-0489|Had the enemy of Babylon succeeded in breaking through the outer and inner defenses of the city the royal palace would have still been far from his reach.|Had the enemy of Babylon succeeded in breaking through the outer and inner defenses of the city the royal palace would have still been far from his reach. +LJ028-0490|He would have had to cross a deep moat, to scale a wall of burned bricks about twenty feet in thickness and perhaps three times as high,|He would have had to cross a deep moat, to scale a wall of burned bricks about twenty feet in thickness and perhaps three times as high, +LJ028-0491|then a second wall still higher, a third and fourth and a fifth, each stronger and higher than the others,|then a second wall still higher, a third and fourth and a fifth, each stronger and higher than the others, +LJ028-0492|and surmounted with towers, and then finally a sixth wall|and surmounted with towers, and then finally a sixth wall +LJ028-0493|whose summit reached into the sky as far, perhaps, as the tallest of the modern buildings.|whose summit reached into the sky as far, perhaps, as the tallest of the modern buildings. +LJ028-0494|Between the several sections were wide spaces where foot soldiers and charioteers might fight.|Between the several sections were wide spaces where foot soldiers and charioteers might fight. +LJ028-0495|It must have been an imposing sight to one standing without to have seen the walls, one after another,|It must have been an imposing sight to one standing without to have seen the walls, one after another, +LJ028-0496|rising higher and higher, like a great terraced, turreted mountain.|rising higher and higher, like a great terraced, turreted mountain. +LJ028-0497|We do not know their height, for the statements of the ancient writers disagree.|We do not know their height, for the statements of the ancient writers disagree. +LJ028-0498|Herodotus says that it was three hundred and thirty-five feet;|Herodotus says that it was three hundred and thirty-five feet; +LJ028-0499|Ctesias mentions three hundred feet; probably they were not far from the truth.|Ctesias mentions three hundred feet; probably they were not far from the truth. +LJ028-0500|The ruins reach the height of about forty feet.|The ruins reach the height of about forty feet. +LJ028-0501|Nor were the walls about the palace a great mass of dull brick masonry.|Nor were the walls about the palace a great mass of dull brick masonry. +LJ028-0502|The Ishtar gateway leading to the palace was encased with beautiful blue glazed bricks,|The Ishtar gateway leading to the palace was encased with beautiful blue glazed bricks, +LJ028-0503|and decorated here and there with large reliefs representing bulls and lions and dragons,|and decorated here and there with large reliefs representing bulls and lions and dragons, +LJ028-0504|designed in colors of white and blue and yellow and black.|designed in colors of white and blue and yellow and black. +LJ028-0505|It seems that the bricks of the reliefs were molded and glazed separately and so accurately that when built into the wall they fitted perfectly.|It seems that the bricks of the reliefs were molded and glazed separately and so accurately that when built into the wall they fitted perfectly. +LJ028-0506|A modern artist would have difficulty in doing such accurate work.|A modern artist would have difficulty in doing such accurate work. +LJ028-0507|Some of these decorations, the most valuable objects found in the ruins of the great city, still remain in their places on the walls;|Some of these decorations, the most valuable objects found in the ruins of the great city, still remain in their places on the walls; +LJ028-0508|others have been taken to the Berlin Museum.|others have been taken to the Berlin Museum. +LJ028-0509|Nebuchadnezzar speaks of great bronze gates and of images of bronze, but none have been discovered.|Nebuchadnezzar speaks of great bronze gates and of images of bronze, but none have been discovered. +LJ028-0510|Probably their metal was far too valuable for the enemy to leave behind.|Probably their metal was far too valuable for the enemy to leave behind. +LJ028-0511|Should you walk along the shore of the Euphrates at Babylon, you would still see the embankments which Nebuchadnezzar constructed of bricks bearing his name,|Should you walk along the shore of the Euphrates at Babylon, you would still see the embankments which Nebuchadnezzar constructed of bricks bearing his name, +LJ028-0512|but the river walls have disappeared, and the buttresses of the bridges have been torn or washed away.|but the river walls have disappeared, and the buttresses of the bridges have been torn or washed away. +LJ028-0513|Should you cross the river to search for the western inner wall, you would find but a small fragment of it.|Should you cross the river to search for the western inner wall, you would find but a small fragment of it. +LJ028-0514|The great outer wall seems to have disappeared completely beneath the desert surface.|The great outer wall seems to have disappeared completely beneath the desert surface. +LJ028-0515|Such were the walls of Babylon,|Such were the walls of Babylon, +LJ028-0516|the strongest, the thickest, the loftiest, the most intricate, perhaps the most beautiful that ever protected a city,|the strongest, the thickest, the loftiest, the most intricate, perhaps the most beautiful that ever protected a city, +LJ028-0517|walls which no ancient army was ever able to take by storm.|walls which no ancient army was ever able to take by storm. +LJ028-0518|It is not strange, then, that they were included among the Seven Wonders of the World,|It is not strange, then, that they were included among the Seven Wonders of the World, +LJ028-0519|or that the Babylonian soldier stood confidently upon their summit, and jeering at the Persian army encamped below, shouted:|or that the Babylonian soldier stood confidently upon their summit, and jeering at the Persian army encamped below, shouted: +LJ029-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ029-0002|Chapter 2. The Assassination: Part 1.|Chapter two. The Assassination: Part one. +LJ029-0003|This chapter describes President Kennedy's trip to Dallas, from its origin through its tragic conclusion.|This chapter describes President Kennedy's trip to Dallas, from its origin through its tragic conclusion. +LJ029-0004|The narrative of these events is based largely on the recollections of the participants,|The narrative of these events is based largely on the recollections of the participants, +LJ029-0005|although in many instances documentary or other evidence has also been used by the Commission.|although in many instances documentary or other evidence has also been used by the Commission. +LJ029-0006|Beginning with the advance plans and Secret Service preparations for the trip,|Beginning with the advance plans and Secret Service preparations for the trip, +LJ029-0007|this chapter reviews the motorcade through Dallas, the fleeting moments of the assassination,|this chapter reviews the motorcade through Dallas, the fleeting moments of the assassination, +LJ029-0008|the activities at Parkland Memorial Hospital, and the return of the Presidential party to Washington.|the activities at Parkland Memorial Hospital, and the return of the Presidential party to Washington. +LJ029-0009|An evaluation of the procedures employed to safeguard the President, with recommendations for improving these procedures, appears in Chapter 8 of the report.|An evaluation of the procedures employed to safeguard the President, with recommendations for improving these procedures, appears in Chapter eight of the report. +LJ029-0010|Planning the Texas Trip|Planning the Texas Trip +LJ029-0011|President Kennedy's visit to Texas in November 1963 had been under consideration for almost a year before it occurred.|President Kennedy's visit to Texas in November nineteen sixty-three had been under consideration for almost a year before it occurred. +LJ029-0012|He had made only a few brief visits to the State since the 1960 Presidential campaign and in 1962 he began to consider a formal visit.|He had made only a few brief visits to the State since the nineteen sixty Presidential campaign and in nineteen sixty-two he began to consider a formal visit. +LJ029-0013|During 1963, the reasons for making the trip became more persuasive.|During nineteen sixty-three, the reasons for making the trip became more persuasive. +LJ029-0014|As a political leader, the President wished to resolve the factional controversy within the Democratic Party in Texas before the election of 1964.|As a political leader, the President wished to resolve the factional controversy within the Democratic Party in Texas before the election of nineteen sixty-four. +LJ029-0015|The party itself saw an opportunity to raise funds by having the President speak at a political dinner eventually planned for Austin.|The party itself saw an opportunity to raise funds by having the President speak at a political dinner eventually planned for Austin. +LJ029-0016|As Chief of State, the President always welcomed the opportunity to learn, firsthand, about the problems which concerned the American people.|As Chief of State, the President always welcomed the opportunity to learn, firsthand, about the problems which concerned the American people. +LJ029-0017|Moreover, he looked forward to the public appearances which he personally enjoyed. The basic decision on the November trip to Texas was made at a meeting of President Kennedy,|Moreover, he looked forward to the public appearances which he personally enjoyed. The basic decision on the November trip to Texas was made at a meeting of President Kennedy, +LJ029-0018|Vice President Johnson, and Governor Connally on June 5th, 1963, at the Cortez Hotel in El Paso, Texas.|Vice President Johnson, and Governor Connally on June fifth, nineteen sixty-three, at the Cortez Hotel in El Paso, Texas. +LJ029-0019|The President had spoken earlier that day at the Air Force Academy in Colorado Springs, Colorado.|The President had spoken earlier that day at the Air Force Academy in Colorado Springs, Colorado. +LJ029-0020|and had stopped in El Paso to discuss the proposed visit and other matters with the Vice President and the Governor.|and had stopped in El Paso to discuss the proposed visit and other matters with the Vice President and the Governor. +LJ029-0021|The three agreed that the President would come to Texas in late November 1963.|The three agreed that the President would come to Texas in late November nineteen sixty-three. +LJ029-0022|The original plan called for the President to spend only 1 day in the State, making whirlwind visits to Dallas, Fort Worth, San Antonio, and Houston.|The original plan called for the President to spend only one day in the State, making whirlwind visits to Dallas, Fort Worth, San Antonio, and Houston. +LJ029-0023|In September, the White House decided to permit further visits by the President and extended the trip to run from the afternoon of November 21|In September, the White House decided to permit further visits by the President and extended the trip to run from the afternoon of November twenty-one +LJ029-0024|through the evening of Friday, November 22.|through the evening of Friday, November twenty-two. +LJ029-0025|When Governor Connally called at the White House on October 4 to discuss the details of the visit,|When Governor Connally called at the White House on October four to discuss the details of the visit, +LJ029-0026|it was agreed that the planning of events in Texas would be left largely to the Governor.|it was agreed that the planning of events in Texas would be left largely to the Governor. +LJ029-0027|At the White House, Kenneth O'Donnell, special assistant to the President, acted as coordinator for the trip.|At the White House, Kenneth O'Donnell, special assistant to the President, acted as coordinator for the trip. +LJ029-0028|Everyone agreed that, if there was sufficient time, a motorcade through downtown Dallas would be the best way for the people to see their President.|Everyone agreed that, if there was sufficient time, a motorcade through downtown Dallas would be the best way for the people to see their President. +LJ029-0029|When the trip was planned for only 1 day, Governor Connally had opposed the motorcade because there was not enough time. The Governor stated, however, that, quote,|When the trip was planned for only one day, Governor Connally had opposed the motorcade because there was not enough time. The Governor stated, however, that, quote, +LJ029-0030|once we got San Antonio moved from Friday to Thursday afternoon, where that was his initial stop in Texas,|once we got San Antonio moved from Friday to Thursday afternoon, where that was his initial stop in Texas, +LJ029-0031|then we had the time, and I withdrew my objections to a motorcade. End quote.|then we had the time, and I withdrew my objections to a motorcade. End quote. +LJ029-0032|According to O'Donnell, quote, we had a motorcade wherever we went, end quote.|According to O'Donnell, quote, we had a motorcade wherever we went, end quote. +LJ029-0033|particularly in large cities where the purpose was to let the President be seen by as many people as possible.|particularly in large cities where the purpose was to let the President be seen by as many people as possible. +LJ029-0034|In his experience, quote, it would be automatic, end quote, for the Secret Service to arrange a route which would, within the time allotted,|In his experience, quote, it would be automatic, end quote, for the Secret Service to arrange a route which would, within the time allotted, +LJ029-0035|bring the President, quote, through an area which exposes him to the greatest number of people, end quote.|bring the President, quote, through an area which exposes him to the greatest number of people, end quote. +LJ029-0036|Advance Preparations for the Dallas Trip|Advance Preparations for the Dallas Trip +LJ029-0037|Advance preparations for President Kennedy's visit to Dallas were primarily the responsibility of two Secret Service agents:|Advance preparations for President Kennedy's visit to Dallas were primarily the responsibility of two Secret Service agents: +LJ029-0038|Special Agent Winston G. Lawson, a member of the White House detail who acted as the advance agent, and Forrest V. Sorrels,|Special Agent Winston G. Lawson, a member of the White House detail who acted as the advance agent, and Forrest V. Sorrels, +LJ029-0039|special agent in charge of the Dallas office. Both agents were advised of the trip on November 4.|special agent in charge of the Dallas office. Both agents were advised of the trip on November four. +LJ029-0040|Lawson received a tentative schedule of the Texas trip on November 8 from Roy H. Kellerman, assistant special agent in charge of the White House detail,|Lawson received a tentative schedule of the Texas trip on November eight from Roy H. Kellerman, assistant special agent in charge of the White House detail, +LJ029-0041|who was the Secret Service official responsible for the entire Texas journey.|who was the Secret Service official responsible for the entire Texas journey. +LJ029-0042|As advance agent working closely with Sorrels, Lawson had responsibility for arranging the timetable for the President's visit to Dallas|As advance agent working closely with Sorrels, Lawson had responsibility for arranging the timetable for the President's visit to Dallas +LJ029-0043|and coordinating local activities with the White House staff, the organizations directly concerned with the visit, and local law enforcement officials.|and coordinating local activities with the White House staff, the organizations directly concerned with the visit, and local law enforcement officials. +LJ029-0044|Lawson's most important responsibilities were to take preventive action against anyone in Dallas considered a threat to the President,|Lawson's most important responsibilities were to take preventive action against anyone in Dallas considered a threat to the President, +LJ029-0045|to select the luncheon site and motorcade route, and to plan security measures for the luncheon and the motorcade.|to select the luncheon site and motorcade route, and to plan security measures for the luncheon and the motorcade. +LJ029-0046|Preventive Intelligence Activities. The Protective Research Section (PRS) of the Secret Service|Preventive Intelligence Activities. The Protective Research Section (PRS) of the Secret Service +LJ029-0047|maintains records of people who have threatened the President or so conducted themselves as to be deemed a potential danger to him.|maintains records of people who have threatened the President or so conducted themselves as to be deemed a potential danger to him. +LJ029-0048|On November 8, 1963,|On November eight, nineteen sixty-three, +LJ029-0049|after undertaking the responsibility for advance preparations for the visit to Dallas, Agent Lawson went to the PRS offices in Washington.|after undertaking the responsibility for advance preparations for the visit to Dallas, Agent Lawson went to the PRS offices in Washington. +LJ029-0050|A check of the geographic indexes there revealed no listing for any individual deemed to be a potential danger to the President|A check of the geographic indexes there revealed no listing for any individual deemed to be a potential danger to the President +LJ029-0051|in the territory of the Secret Service regional office which includes Dallas and Fort Worth.|in the territory of the Secret Service regional office which includes Dallas and Fort Worth. +LJ029-0052|To supplement the PRS files, the Secret Service depends largely on local police departments and local offices of other Federal agencies|To supplement the PRS files, the Secret Service depends largely on local police departments and local offices of other Federal agencies +LJ029-0053|which advise it of potential threats immediately before the visit of the President to their community.|which advise it of potential threats immediately before the visit of the President to their community. +LJ029-0054|Upon his arrival in Dallas on November 12|Upon his arrival in Dallas on November twelve +LJ029-0055|Lawson conferred with the local police and the local office of the Federal Bureau of Investigation about potential dangers to the President.|Lawson conferred with the local police and the local office of the Federal Bureau of Investigation about potential dangers to the President. +LJ029-0056|Although there was no mention in PRS files of the demonstration in Dallas against Ambassador Adlai Stevenson on October 24th,|Although there was no mention in PRS files of the demonstration in Dallas against Ambassador Adlai Stevenson on October twenty-fourth, +LJ029-0057|1963, Lawson inquired about the incident and obtained through the local police photographs of some of the persons involved.|nineteen sixty-three, Lawson inquired about the incident and obtained through the local police photographs of some of the persons involved. +LJ029-0058|On November 22 a Secret Service agent stood at the entrance to the Trade Mart, where the President was scheduled to speak, with copies of these photographs.|On November twenty-two a Secret Service agent stood at the entrance to the Trade Mart, where the President was scheduled to speak, with copies of these photographs. +LJ029-0059|Dallas detectives in the lobby of the Trade Mart and in the luncheon area also had copies of these photographs.|Dallas detectives in the lobby of the Trade Mart and in the luncheon area also had copies of these photographs. +LJ029-0060|A number of people who resembled some of those in the photographs were placed under surveillance at the Trade Mart.|A number of people who resembled some of those in the photographs were placed under surveillance at the Trade Mart. +LJ029-0061|The FBI office in Dallas gave the local Secret Service representatives the name of a possibly dangerous individual in the Dallas area who was investigated.|The FBI office in Dallas gave the local Secret Service representatives the name of a possibly dangerous individual in the Dallas area who was investigated. +LJ029-0062|It also advised the Secret Service of the circulation on November 21 of a handbill sharply critical of President Kennedy,|It also advised the Secret Service of the circulation on November twenty-one of a handbill sharply critical of President Kennedy, +LJ029-0063|discussed in chapter 6 of this report.|discussed in chapter six of this report. +LJ029-0064|Shortly before, the Dallas police had reported to the Secret Service that the handbill had appeared on the streets of Dallas.|Shortly before, the Dallas police had reported to the Secret Service that the handbill had appeared on the streets of Dallas. +LJ029-0065|Neither the Dallas police nor the FBI had yet learned the source of the handbill.|Neither the Dallas police nor the FBI had yet learned the source of the handbill. +LJ029-0066|No one else was identified to the Secret Service through local inquiry as potentially dangerous,|No one else was identified to the Secret Service through local inquiry as potentially dangerous, +LJ029-0067|nor did PRS develop any additional information between November 12, when Lawson left Washington, and November 22.|nor did PRS develop any additional information between November twelve, when Lawson left Washington, and November twenty-two. +LJ029-0068|The adequacy of the intelligence system maintained by the Secret Service at the time of the assassination,|The adequacy of the intelligence system maintained by the Secret Service at the time of the assassination, +LJ029-0069|including a detailed description of the available data on Lee Harvey Oswald and the reasons why his name had not been furnished to the Secret Service,|including a detailed description of the available data on Lee Harvey Oswald and the reasons why his name had not been furnished to the Secret Service, +LJ029-0070|is discussed in chapter 8.|is discussed in chapter eight. +LJ029-0071|An important purpose of the President's visit to Dallas was to speak at a luncheon given by business and civic leaders.|An important purpose of the President's visit to Dallas was to speak at a luncheon given by business and civic leaders. +LJ029-0072|The White House staff informed the Secret Service|The White House staff informed the Secret Service +LJ029-0073|that the President would arrive and depart from Dallas' Love Field; that a motorcade through the downtown area of Dallas to the luncheon site should be arranged;|that the President would arrive and depart from Dallas' Love Field; that a motorcade through the downtown area of Dallas to the luncheon site should be arranged; +LJ029-0074|and that following the luncheon the President would return to the airport by the most direct route.|and that following the luncheon the President would return to the airport by the most direct route. +LJ029-0075|Accordingly, it was important to determine the luncheon site as quickly as possible, so that security could be established at the site and the motorcade route selected.|Accordingly, it was important to determine the luncheon site as quickly as possible, so that security could be established at the site and the motorcade route selected. +LJ029-0076|On November 4, Gerald A. Behn, agent in charge of the White House detail, asked Sorrels to examine three potential sites for the luncheon.|On November four, Gerald A. Behn, agent in charge of the White House detail, asked Sorrels to examine three potential sites for the luncheon. +LJ029-0077|One building, Market Hall, was unavailable for November 22.|One building, Market Hall, was unavailable for November twenty-two. +LJ029-0078|The second, the Women's Building at the State Fair Grounds,|The second, the Women's Building at the State Fair Grounds, +LJ029-0079|was a one-story building with few entrances and easy to make secure, but it lacked necessary food-handling facilities|was a one-story building with few entrances and easy to make secure, but it lacked necessary food-handling facilities +LJ029-0080|and had certain unattractive features, including a low ceiling with exposed conduits and beams.|and had certain unattractive features, including a low ceiling with exposed conduits and beams. +LJ029-0081|The third possibility, the Trade Mart, a handsome new building with all the necessary facilities, presented security problems. It had numerous entrances,|The third possibility, the Trade Mart, a handsome new building with all the necessary facilities, presented security problems. It had numerous entrances, +LJ029-0082|several tiers of balconies surrounding the central court where the luncheon would be held, and several catwalks crossing the court at each level.|several tiers of balconies surrounding the central court where the luncheon would be held, and several catwalks crossing the court at each level. +LJ029-0083|On November 4, Sorrels told Behn he believed security difficulties at the Trade Mart could be overcome by special precautions.|On November four, Sorrels told Behn he believed security difficulties at the Trade Mart could be overcome by special precautions. +LJ029-0084|Lawson also evaluated the security hazards at the Trade Mart on November 13.|Lawson also evaluated the security hazards at the Trade Mart on November thirteen. +LJ029-0085|Kenneth O'Donnell made the final decision to hold the luncheon at the Trade Mart; Behn so notified Lawson on November 14.|Kenneth O'Donnell made the final decision to hold the luncheon at the Trade Mart; Behn so notified Lawson on November fourteen. +LJ029-0086|Once the Trade Mart had been selected, Sorrels and Lawson worked out detailed arrangements for security at the building.|Once the Trade Mart had been selected, Sorrels and Lawson worked out detailed arrangements for security at the building. +LJ029-0087|In addition to the preventive measures already mentioned, they provided for controlling access to the building, closing off and policing areas around it,|In addition to the preventive measures already mentioned, they provided for controlling access to the building, closing off and policing areas around it, +LJ029-0088|securing the roof and insuring the presence of numerous police officers inside and around the building.|securing the roof and insuring the presence of numerous police officers inside and around the building. +LJ029-0089|Ultimately more than 200 law enforcement officers, mainly Dallas police but including 8 Secret Service agents,|Ultimately more than two hundred law enforcement officers, mainly Dallas police but including eight Secret Service agents, +LJ029-0090|were deployed in and around the Trade Mart.|were deployed in and around the Trade Mart. +LJ029-0091|The Motorcade Route.|The Motorcade Route. +LJ029-0092|On November 8, when Lawson was briefed on the itinerary for the trip to Dallas,|On November eight, when Lawson was briefed on the itinerary for the trip to Dallas, +LJ029-0093|he was told that 45 minutes had been allotted for a motorcade procession from Love Field to the luncheon site.|he was told that forty-five minutes had been allotted for a motorcade procession from Love Field to the luncheon site. +LJ029-0094|Lawson was not specifically instructed to select the parade route, but he understood that this was one of his functions.|Lawson was not specifically instructed to select the parade route, but he understood that this was one of his functions. +LJ029-0095|Even before the Trade Mart had been definitely selected, Lawson and Sorrels began to consider the best motorcade route from Love Field to the Trade Mart.|Even before the Trade Mart had been definitely selected, Lawson and Sorrels began to consider the best motorcade route from Love Field to the Trade Mart. +LJ029-0096|On November 14, Lawson and Sorrels attended a meeting at Love Field|On November fourteen, Lawson and Sorrels attended a meeting at Love Field +LJ029-0097|and on their return to Dallas drove over the route which Sorrels believed best suited for the proposed motorcade.|and on their return to Dallas drove over the route which Sorrels believed best suited for the proposed motorcade. +LJ029-0098|This route, eventually selected for the motorcade from the airport to the Trade Mart, measured 10 miles and could be driven easily within the allotted 45 minutes.|This route, eventually selected for the motorcade from the airport to the Trade Mart, measured ten miles and could be driven easily within the allotted forty-five minutes. +LJ029-0099|From Love Field the route passed through a portion of suburban Dallas,|From Love Field the route passed through a portion of suburban Dallas, +LJ029-0100|through the downtown area along Main Street and then to the Trade Mart via Stemmons Freeway.|through the downtown area along Main Street and then to the Trade Mart via Stemmons Freeway. +LJ029-0101|For the President's return to Love Field following the luncheon, the agents selected the most direct route, which was approximately 4 miles.|For the President's return to Love Field following the luncheon, the agents selected the most direct route, which was approximately four miles. +LJ029-0102|After the selection of the Trade Mart as the luncheon site,|After the selection of the Trade Mart as the luncheon site, +LJ029-0103|Lawson and Sorrels met with Dallas Chief of Police Jesse E. Curry, Assistant Chief Charles Batchelor,|Lawson and Sorrels met with Dallas Chief of Police Jesse E. Curry, Assistant Chief Charles Batchelor, +LJ029-0104|Deputy Chief N. T. Fisher, and several other command officers to discuss details of the motorcade and possible routes.|Deputy Chief N. T. Fisher, and several other command officers to discuss details of the motorcade and possible routes. +LJ029-0105|The route was further reviewed by Lawson and Sorrels with Assistant Chief Batchelor and members of the local host committee on November 15.|The route was further reviewed by Lawson and Sorrels with Assistant Chief Batchelor and members of the local host committee on November fifteen. +LJ029-0106|The police officials agreed that the route recommended by Sorrels was the proper one and did not express a belief that any other route might be better.|The police officials agreed that the route recommended by Sorrels was the proper one and did not express a belief that any other route might be better. +LJ029-0107|On November 18,|On November eighteen, +LJ029-0108|Sorrels and Lawson drove over the selected route with Batchelor and other police officers, verifying that it could be traversed within 45 minutes.|Sorrels and Lawson drove over the selected route with Batchelor and other police officers, verifying that it could be traversed within forty-five minutes. +LJ029-0109|Representatives of the local host committee and the White House staff were advised by the Secret Service of the actual route on the afternoon of November 18.|Representatives of the local host committee and the White House staff were advised by the Secret Service of the actual route on the afternoon of November eighteen. +LJ029-0110|The route impressed the agents as a natural and desirable one.|The route impressed the agents as a natural and desirable one. +LJ029-0111|Sorrels, who had participated in Presidential protection assignments in Dallas since a visit by President Franklin D. Roosevelt|Sorrels, who had participated in Presidential protection assignments in Dallas since a visit by President Franklin D. Roosevelt +LJ029-0112|in 1936, as testified that the traditional parade route in Dallas was along Main Street, since the tall buildings along the street|in nineteen thirty-six, as testified that the traditional parade route in Dallas was along Main Street, since the tall buildings along the street +LJ029-0113|gave more people an opportunity to participate.|gave more people an opportunity to participate. +LJ029-0114|The route chosen from the airport to Main Street was the normal one, except where Harwood Street was selected as the means of access to Main Street|The route chosen from the airport to Main Street was the normal one, except where Harwood Street was selected as the means of access to Main Street +LJ029-0115|in preference to a short stretch of the Central Expressway, which presented a minor safety hazard|in preference to a short stretch of the Central Expressway, which presented a minor safety hazard +LJ029-0116|and could not accommodate spectators as conveniently as Harwood Street.|and could not accommodate spectators as conveniently as Harwood Street. +LJ029-0117|According to Lawson, the chosen route seemed to be the best.|According to Lawson, the chosen route seemed to be the best. +LJ029-0118|It afforded us wide streets most of the way, because of the buses that were in the motorcade.|It afforded us wide streets most of the way, because of the buses that were in the motorcade. +LJ029-0119|It afforded us a chance to have alternative routes if something happened on the motorcade route. It was the type of suburban area a good part of the way|It afforded us a chance to have alternative routes if something happened on the motorcade route. It was the type of suburban area a good part of the way +LJ029-0120|where the crowds would be able to be controlled for a great distance, and we figured that the largest crowds would be downtown, which they were,|where the crowds would be able to be controlled for a great distance, and we figured that the largest crowds would be downtown, which they were, +LJ029-0121|and that the wide streets that we would use downtown would be of sufficient width to keep the public out of our way.|and that the wide streets that we would use downtown would be of sufficient width to keep the public out of our way. +LJ029-0122|Elm Street, parallel to Main Street and one block north,|Elm Street, parallel to Main Street and one block north, +LJ029-0123|was not used for the main portion of the downtown part of the motorcade because Main Street offered better vantage points for spectators.|was not used for the main portion of the downtown part of the motorcade because Main Street offered better vantage points for spectators. +LJ029-0124|To reach the Trade Mart from Main Street the agents decided to use the Stemmons Freeway (Route No. 77), the most direct route.|To reach the Trade Mart from Main Street the agents decided to use the Stemmons Freeway (Route Number seventy-seven), the most direct route. +LJ029-0125|The only practical way for westbound traffic on Main Street|The only practical way for westbound traffic on Main Street +LJ029-0126|to reach the northbound lanes of the Stemmons Freeway is via Elm Street, which Route No. 77 traffic is instructed to follow in this part of the city.|to reach the northbound lanes of the Stemmons Freeway is via Elm Street, which Route Number seventy-seven traffic is instructed to follow in this part of the city. +LJ029-0127|Elm Street was to be reached from Main by turning right at Houston,|Elm Street was to be reached from Main by turning right at Houston, +LJ029-0128|going one block north and then turning left onto Elm.|going one block north and then turning left onto Elm. +LJ029-0129|On this last portion of the journey, only 5 minutes from the Trade Mart,|On this last portion of the journey, only five minutes from the Trade Mart, +LJ029-0130|the President's motorcade would pass the Texas School Book Depository Building on the northwest corner of Houston and Elm Streets.|the President's motorcade would pass the Texas School Book Depository Building on the northwest corner of Houston and Elm Streets. +LJ029-0131|The building overlooks Dealey Plaza, an attractively landscaped triangle of 3 acres.|The building overlooks Dealey Plaza, an attractively landscaped triangle of three acres. +LJ029-0132|From Houston Street, which forms the base of the triangle, three streets -- Commerce, Main, and Elm --|From Houston Street, which forms the base of the triangle, three streets -- Commerce, Main, and Elm -- +LJ029-0133|trisect the plaza, converging at the apex of the triangle to form a triple underpass beneath a multiple railroad bridge|trisect the plaza, converging at the apex of the triangle to form a triple underpass beneath a multiple railroad bridge +LJ029-0134|almost 500 feet from Houston Street.|almost five hundred feet from Houston Street. +LJ029-0135|Elm Street, the northernmost of the three, after intersecting Houston curves in a southwesterly arc|Elm Street, the northernmost of the three, after intersecting Houston curves in a southwesterly arc +LJ029-0136|through the underpass and leads into an access road,|through the underpass and leads into an access road, +LJ029-0137|which branches off to the right and is used by traffic going to the Stemmons Freeway and the Dallas-Fort Worth Turnpike.|which branches off to the right and is used by traffic going to the Stemmons Freeway and the Dallas-Fort Worth Turnpike. +LJ029-0138|The Elm Street approach to the Stemmons Freeway is necessary|The Elm Street approach to the Stemmons Freeway is necessary +LJ029-0139|in order to avoid the traffic hazards which would otherwise exist if right turns were permitted from both Main and Elm into the freeway.|in order to avoid the traffic hazards which would otherwise exist if right turns were permitted from both Main and Elm into the freeway. +LJ029-0140|To create this traffic pattern, a concrete barrier between Main and Elm Streets presents an obstacle to a right turn|To create this traffic pattern, a concrete barrier between Main and Elm Streets presents an obstacle to a right turn +LJ029-0141|from Main across Elm to the access road to Stemmons Freeway and the Dallas-Fort Worth Turnpike.|from Main across Elm to the access road to Stemmons Freeway and the Dallas-Fort Worth Turnpike. +LJ029-0142|This concrete barrier extends far enough beyond the access road to make it impracticable for vehicles to turn right from Main directly to the access road.|This concrete barrier extends far enough beyond the access road to make it impracticable for vehicles to turn right from Main directly to the access road. +LJ029-0143|A sign located on this barrier instructs Main Street traffic not to make any turns.|A sign located on this barrier instructs Main Street traffic not to make any turns. +LJ029-0144|In conformity with these arrangements, traffic proceeding west on Main is directed to turn right at Houston|In conformity with these arrangements, traffic proceeding west on Main is directed to turn right at Houston +LJ029-0145|in order to reach the Dallas-Fort Worth Turnpike, which has the same access road from Elm Street as does the Stemmons Freeway.|in order to reach the Dallas-Fort Worth Turnpike, which has the same access road from Elm Street as does the Stemmons Freeway. +LJ029-0146|The planning for the motorcade also included advance preparations for security arrangements along the route.|The planning for the motorcade also included advance preparations for security arrangements along the route. +LJ029-0147|Sorrels and Lawson reviewed the route in cooperation with Assistant Chief Bachelor and other Dallas police officials who took notes on the requirements|Sorrels and Lawson reviewed the route in cooperation with Assistant Chief Bachelor and other Dallas police officials who took notes on the requirements +LJ029-0148|for controlling the crowds and traffic, watching the overpasses, and providing motorcycle escort.|for controlling the crowds and traffic, watching the overpasses, and providing motorcycle escort. +LJ029-0149|To control traffic, arrangements were made for the deployment of foot patrolmen and motorcycle police at various positions along the route.|To control traffic, arrangements were made for the deployment of foot patrolmen and motorcycle police at various positions along the route. +LJ029-0150|Police were assigned to each overpass on the route and instructed to keep them clear of unauthorized persons.|Police were assigned to each overpass on the route and instructed to keep them clear of unauthorized persons. +LJ029-0151|No arrangements were made for police or building custodians|No arrangements were made for police or building custodians +LJ029-0152|to inspect buildings along the motorcade route since the Secret Service did not normally request or make such a check.|to inspect buildings along the motorcade route since the Secret Service did not normally request or make such a check. +LJ029-0153|Under standard procedures, the responsibility for watching the windows of buildings was shared by local police stationed along the route|Under standard procedures, the responsibility for watching the windows of buildings was shared by local police stationed along the route +LJ029-0154|and Secret Service agents riding in the motorcade.|and Secret Service agents riding in the motorcade. +LJ029-0155|As the date for the President's visit approached,|As the date for the President's visit approached, +LJ029-0156|the two Dallas newspapers carried several reports of his motorcade route.|the two Dallas newspapers carried several reports of his motorcade route. +LJ029-0157|The selection of the Trade Mart as the possible site for the luncheon first appeared in the Dallas Times-Herald on November 15, 1963.|The selection of the Trade Mart as the possible site for the luncheon first appeared in the Dallas Times-Herald on November fifteen, nineteen sixty-three. +LJ029-0158|The following day, the newspaper reported that the Presidential party|The following day, the newspaper reported that the Presidential party +LJ029-0159|quote, apparently will loop through the downtown area, probably on Main Street, en route from Dallas Love Field, end quote.|quote, apparently will loop through the downtown area, probably on Main Street, en route from Dallas Love Field, end quote. +LJ029-0160|on its way to the Trade Mart. On November 19, the Times-Herald afternoon paper detailed the precise route:|on its way to the Trade Mart. On November nineteen, the Times-Herald afternoon paper detailed the precise route: +LJ029-0161|From the airport, the President's party will proceed to Mockingbird Lane to Lemmon and then to Turtle Creek, turning south to Cedar Springs.|From the airport, the President's party will proceed to Mockingbird Lane to Lemmon and then to Turtle Creek, turning south to Cedar Springs. +LJ029-0162|The motorcade will then pass through downtown on Harwood|The motorcade will then pass through downtown on Harwood +LJ029-0163|and then west on Main, turning back to Elm at Houston and then out Stemmons Freeway to the Trade Mart.|and then west on Main, turning back to Elm at Houston and then out Stemmons Freeway to the Trade Mart. +LJ029-0164|Also on November 19, the Morning News reported that the President's motorcade would travel from Love Field along specified streets, then|Also on November nineteen, the Morning News reported that the President's motorcade would travel from Love Field along specified streets, then +LJ029-0165|Harwood to Main, Main to Houston, Houston to Elm, Elm under the Triple Underpass to Stemmons Freeway, and on to the Trade Mart.|Harwood to Main, Main to Houston, Houston to Elm, Elm under the Triple Underpass to Stemmons Freeway, and on to the Trade Mart. +LJ029-0166|On November 20 a front page story reported that the streets on which the Presidential motorcade would travel included "Main and Stemmons Freeway."|On November twenty a front page story reported that the streets on which the Presidential motorcade would travel included "Main and Stemmons Freeway." +LJ029-0167|On the morning of the President's arrival,|On the morning of the President's arrival, +LJ029-0168|the Morning News noted that the motorcade would travel through downtown Dallas onto the Stemmons Freeway, and reported that, quote, the motorcade will move slowly|the Morning News noted that the motorcade would travel through downtown Dallas onto the Stemmons Freeway, and reported that, quote, the motorcade will move slowly +LJ029-0169|so that crowds can get a good view of President Kennedy and his wife.|so that crowds can get a good view of President Kennedy and his wife. +LJ029-0170|Dallas Before the Visit|Dallas Before the Visit +LJ029-0171|The President's intention to pay a visit to Texas in the fall of 1963 aroused interest throughout the State.|The President's intention to pay a visit to Texas in the fall of nineteen sixty-three aroused interest throughout the State. +LJ029-0172|The two Dallas newspapers provided their readers with a steady stream of information and speculation about the trip,|The two Dallas newspapers provided their readers with a steady stream of information and speculation about the trip, +LJ029-0173|beginning on September 13, when the Times-Herald announced in a front page article that President Kennedy was planning a brief 1-day tour of four Texas cities|beginning on September thirteen, when the Times-Herald announced in a front page article that President Kennedy was planning a brief one-day tour of four Texas cities +LJ029-0174|Dallas, Fort Worth, San Antonio, and Houston|Dallas, Fort Worth, San Antonio, and Houston +LJ029-0175|Both Dallas papers cited White House sources on September 26 as confirming the President's intention to visit Texas on November 21 and 22,|Both Dallas papers cited White House sources on September twenty-six as confirming the President's intention to visit Texas on November twenty-one and twenty-two, +LJ029-0176|with Dallas scheduled as one of the stops.|with Dallas scheduled as one of the stops. +LJ029-0177|Articles, editorials, and letters to the editor in the Dallas Morning News and the Dallas Times-Herald after September 13|Articles, editorials, and letters to the editor in the Dallas Morning News and the Dallas Times-Herald after September thirteen +LJ029-0178|reflected the feeling in the community toward the forthcoming Presidential visit.|reflected the feeling in the community toward the forthcoming Presidential visit. +LJ029-0179|Although there were critical editorials and letters to the editors, the news stories reflected the desire of Dallas officials to welcome the President with dignity and courtesy.|Although there were critical editorials and letters to the editors, the news stories reflected the desire of Dallas officials to welcome the President with dignity and courtesy. +LJ029-0180|An editorial in the Times-Herald of September 17|An editorial in the Times-Herald of September seventeen +LJ029-0181|called on the people of Dallas to be "congenial hosts" even though "Dallas didn't vote for Mr. Kennedy in 1960,|called on the people of Dallas to be "congenial hosts" even though "Dallas didn't vote for Mr. Kennedy in nineteen sixty, +LJ029-0182|may not endorse him in '64.|may not endorse him in 'sixty-four. +LJ029-0183|On October 3 the Dallas Morning News quoted U.S. Representative Joe Pool's hope|On October three the Dallas Morning News quoted U.S. Representative Joe Pool's hope +LJ029-0184|that President Kennedy would receive a "good welcome" and would not face demonstrations like those encountered|that President Kennedy would receive a "good welcome" and would not face demonstrations like those encountered +LJ029-0185|by Vice President Johnson during the 1960 campaign.|by Vice President Johnson during the nineteen sixty campaign. +LJ029-0186|Increased concern about the President's visit was aroused by the incident involving the U.S. Ambassador to the United Nations, Adlai E. Stevenson.|Increased concern about the President's visit was aroused by the incident involving the U.S. Ambassador to the United Nations, Adlai E. Stevenson. +LJ029-0187|On the evening of October 24, 1963, after addressing a meeting in Dallas,|On the evening of October twenty-four, nineteen sixty-three, after addressing a meeting in Dallas, +LJ029-0189|The local, national, and international reaction to this incident evoked from Dallas officials and newspapers strong condemnations of the demonstrators.|The local, national, and international reaction to this incident evoked from Dallas officials and newspapers strong condemnations of the demonstrators. +LJ029-0190|Mayor Earle Cabell called on the city to redeem itself during President Kennedy's visit.|Mayor Earle Cabell called on the city to redeem itself during President Kennedy's visit. +LJ029-0191|He asserted that Dallas had shed its reputation of the twenties as the, quote, Southwest hate capital of Dixie, end quote|He asserted that Dallas had shed its reputation of the twenties as the, quote, Southwest hate capital of Dixie, end quote +LJ029-0192|On October 26 the press reported Chief of Police Curry's plans|On October twenty-six the press reported Chief of Police Curry's plans +LJ029-0193|to call in 100 extra off-duty officers to help protect President Kennedy.|to call in one hundred extra off-duty officers to help protect President Kennedy. +LJ029-0194|Any thought that the President might cancel his visit to Dallas was ended|Any thought that the President might cancel his visit to Dallas was ended +LJ029-0195|when Governor Connally confirmed on November 8 that the President would come to Texas on November 21 and 22,|when Governor Connally confirmed on November eight that the President would come to Texas on November twenty-one and twenty-two, +LJ029-0196|and that he would visit San Antonio, Houston, Fort Worth, Dallas, and Austin.|and that he would visit San Antonio, Houston, Fort Worth, Dallas, and Austin. +LJ029-0197|During November the Dallas papers reported frequently on the plans for protecting the President, stressing the thoroughness of the preparations.|During November the Dallas papers reported frequently on the plans for protecting the President, stressing the thoroughness of the preparations. +LJ029-0198|They conveyed the pleas of Dallas leaders that citizens not demonstrate or create disturbances during the President's visit.|They conveyed the pleas of Dallas leaders that citizens not demonstrate or create disturbances during the President's visit. +LJ029-0199|On November 18 the Dallas City Council adopted a new city ordinance prohibiting interference with attendance at lawful assemblies.|On November eighteen the Dallas City Council adopted a new city ordinance prohibiting interference with attendance at lawful assemblies. +LJ029-0200|Two days before the President's arrival Chief Curry warned that the Dallas police would not permit improper conduct during the President's visit.|Two days before the President's arrival Chief Curry warned that the Dallas police would not permit improper conduct during the President's visit. +LJ029-0201|Meanwhile, on November 17|Meanwhile, on November seventeen +LJ029-0202|the president of the Dallas Chamber of Commerce referred to the city's reputation for being the friendliest town in America and asserted that citizens would, quote,|the president of the Dallas Chamber of Commerce referred to the city's reputation for being the friendliest town in America and asserted that citizens would, quote, +LJ029-0203|greet the President of the United States with the warmth and pride that keep the Dallas spirit famous the world over, end quote,|greet the President of the United States with the warmth and pride that keep the Dallas spirit famous the world over, end quote, +LJ029-0204|Two days later, a local Republican leader called for a "civilized nonpartisan" welcome|Two days later, a local Republican leader called for a "civilized nonpartisan" welcome +LJ029-0205|for President Kennedy, stating that "in many respects Dallas County has isolated itself from the main stream of life in the world in this decade.|for President Kennedy, stating that "in many respects Dallas County has isolated itself from the main stream of life in the world in this decade. +LJ029-0206|Another reaction to the impending visit -- hostile to the President -- came to a head shortly before his arrival.|Another reaction to the impending visit -- hostile to the President -- came to a head shortly before his arrival. +LJ029-0207|On November 21 there appeared on the streets of Dallas the anonymous handbill mentioned above.|On November twenty-one there appeared on the streets of Dallas the anonymous handbill mentioned above. +LJ029-0208|It was fashioned after the "wanted" circulars issued by law enforcement agencies.|It was fashioned after the "wanted" circulars issued by law enforcement agencies. +LJ029-0209|Beneath two photographs of President Kennedy, one full- face and one profile, appeared the caption, quote, Wanted for Treason,|Beneath two photographs of President Kennedy, one full- face and one profile, appeared the caption, quote, Wanted for Treason, +LJ029-0210|end quote, followed by a scurrilous bill of particulars that constituted a vilification of the President.|end quote, followed by a scurrilous bill of particulars that constituted a vilification of the President. +LJ029-0211|And on the morning of the President's arrival, there appeared in the Morning News a full, black-bordered advertisement headed:|And on the morning of the President's arrival, there appeared in the Morning News a full, black-bordered advertisement headed: +LJ029-0212|"Welcome Mr. Kennedy to Dallas," sponsored by the American Fact-finding Committee, which the sponsor later testified was an ad hoc committee|"Welcome Mr. Kennedy to Dallas," sponsored by the American Fact-finding Committee, which the sponsor later testified was an ad hoc committee +LJ029-0213|quote, formed strictly for the purpose of having a name to put in the paper, end quote,|quote, formed strictly for the purpose of having a name to put in the paper, end quote, +LJ030-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ030-0002|Chapter 2. The Assassination: Part 2.|Chapter two. The Assassination: Part two. +LJ030-0003|Visits to Other Texas Cities|Visits to Other Texas Cities +LJ030-0004|The trip to Texas began with the departure of President and Mrs. Kennedy from the White House|The trip to Texas began with the departure of President and Mrs. Kennedy from the White House +LJ030-0005|by helicopter at 10:45 A.M., Eastern Standard Time, on November 21, 1963, for Andrews Air Force Base.|by helicopter at ten:forty-five A.M., Eastern Standard Time, on November twenty-one, nineteen sixty-three, for Andrews Air Force Base. +LJ030-0006|They took off in the Presidential plane, Air Force One, at 11 a.m., arriving at San Antonio at 1:30 p.m., Eastern Standard Time.|They took off in the Presidential plane, Air Force One, at eleven a.m., arriving at San Antonio at one:thirty p.m., Eastern Standard Time. +LJ030-0007|They were greeted by Vice President Johnson and Governor Connally, who joined the Presidential party in a motorcade through San Antonio.|They were greeted by Vice President Johnson and Governor Connally, who joined the Presidential party in a motorcade through San Antonio. +LJ030-0008|During the afternoon, President Kennedy dedicated the U.S. Air Force School of Aerospace Medicine at Brooks Air Force Base.|During the afternoon, President Kennedy dedicated the U.S. Air Force School of Aerospace Medicine at Brooks Air Force Base. +LJ030-0009|Late in the afternoon he flew to Houston where he rode through the city in a motorcade,|Late in the afternoon he flew to Houston where he rode through the city in a motorcade, +LJ030-0010|spoke at the Rice University Stadium, and attended a dinner in honor of U.S. Representative Albert Thomas.|spoke at the Rice University Stadium, and attended a dinner in honor of U.S. Representative Albert Thomas. +LJ030-0011|At Rice Stadium a very large, enthusiastic crowd greeted the President.|At Rice Stadium a very large, enthusiastic crowd greeted the President. +LJ030-0012|In Houston, as elsewhere during the trip, the crowds showed much interest in Mrs. Kennedy.|In Houston, as elsewhere during the trip, the crowds showed much interest in Mrs. Kennedy. +LJ030-0013|David F. Powers of the President's staff later stated that when the President asked for his assessment of the day's activities, Powers replied|David F. Powers of the President's staff later stated that when the President asked for his assessment of the day's activities, Powers replied +LJ030-0014|quote, that the crowd was about the same as the one which came to see him before but there were 100,000 extra people on hand who came to see Mrs. Kennedy.|quote, that the crowd was about the same as the one which came to see him before but there were one hundred thousand extra people on hand who came to see Mrs. Kennedy. +LJ030-0015|Late in the evening, the Presidential party flew to Fort Worth where they spent the night at the Texas Hotel.|Late in the evening, the Presidential party flew to Fort Worth where they spent the night at the Texas Hotel. +LJ030-0016|On the morning of November 22, President Kennedy attended a breakfast at the hotel and afterward addressed a crowd at an open parking lot.|On the morning of November twenty-two, President Kennedy attended a breakfast at the hotel and afterward addressed a crowd at an open parking lot. +LJ030-0017|The President liked outdoor appearances because more people could see and hear him.|The President liked outdoor appearances because more people could see and hear him. +LJ030-0018|Before leaving the hotel, the President, Mrs. Kennedy, and Kenneth O'Donnell talked about the risks inherent in Presidential public appearances.|Before leaving the hotel, the President, Mrs. Kennedy, and Kenneth O'Donnell talked about the risks inherent in Presidential public appearances. +LJ030-0019|According to O'Donnell, the President commented that, quote,|According to O'Donnell, the President commented that, quote, +LJ030-0020|if anybody really wanted to shoot the President of the United States, it was not a very difficult job|if anybody really wanted to shoot the President of the United States, it was not a very difficult job +LJ030-0021|all one had to do was get a high building someday with a telescopic rifle, and there was nothing anybody could do to defend against such an attempt.|all one had to do was get a high building someday with a telescopic rifle, and there was nothing anybody could do to defend against such an attempt. +LJ030-0022|Upon concluding the conversation, the President prepared to depart for Dallas.|Upon concluding the conversation, the President prepared to depart for Dallas. +LJ030-0023|Arrival at Love Field|Arrival at Love Field +LJ030-0024|In Dallas the rain had stopped, and by midmorning a gloomy overcast sky had given way to the bright sunshine that greeted the Presidential party|In Dallas the rain had stopped, and by midmorning a gloomy overcast sky had given way to the bright sunshine that greeted the Presidential party +LJ030-0025|when Air Force One touched down at Love Field at 11:40 a.m., Eastern Standard Time.|when Air Force One touched down at Love Field at eleven:forty a.m., Eastern Standard Time. +LJ030-0026|Governor and Mrs. Connally and Senator Ralph W. Yarborough had come with the President from Fort Worth.|Governor and Mrs. Connally and Senator Ralph W. Yarborough had come with the President from Fort Worth. +LJ030-0027|Vice President Johnson's airplane, Air Force Two, had arrived at Love Field at approximately 11:35 a.m.,|Vice President Johnson's airplane, Air Force Two, had arrived at Love Field at approximately eleven:thirty-five a.m., +LJ030-0028|and the Vice President and Mrs. Johnson were in the receiving line to greet President and Mrs. Kennedy.|and the Vice President and Mrs. Johnson were in the receiving line to greet President and Mrs. Kennedy. +LJ030-0029|After a welcome from the Dallas reception committee,|After a welcome from the Dallas reception committee, +LJ030-0030|President and Mrs. Kennedy walked along a chain-link fence at the reception area greeting a large crowd of spectators that had gathered behind it.|President and Mrs. Kennedy walked along a chain-link fence at the reception area greeting a large crowd of spectators that had gathered behind it. +LJ030-0031|Secret Service agents formed a cordon to keep the press and photographers from impeding their passage and scanned the crowd for threatening movements.|Secret Service agents formed a cordon to keep the press and photographers from impeding their passage and scanned the crowd for threatening movements. +LJ030-0032|Dallas police stood at intervals along the fence and Dallas plain clothes men mixed in the crowd.|Dallas police stood at intervals along the fence and Dallas plain clothes men mixed in the crowd. +LJ030-0033|Vice President and Mrs. Johnson followed along the fence, guarded by four members of the Vice-Presidential detail.|Vice President and Mrs. Johnson followed along the fence, guarded by four members of the Vice-Presidential detail. +LJ030-0034|Approximately 10 minutes after the arrival at Love Field, the President and Mrs. Kennedy went to the Presidential automobile to begin the motorcade.|Approximately ten minutes after the arrival at Love Field, the President and Mrs. Kennedy went to the Presidential automobile to begin the motorcade. +LJ030-0035|Organization of the Motorcade|Organization of the Motorcade +LJ030-0036|Secret Service arrangements for Presidential trips, which were followed in the Dallas motorcade,|Secret Service arrangements for Presidential trips, which were followed in the Dallas motorcade, +LJ030-0037|are designed to provide protection while permitting large numbers of people to see the President.|are designed to provide protection while permitting large numbers of people to see the President. +LJ030-0038|Every effort is made to prevent unscheduled stops, although the President may, and in Dallas did, order stops in order to greet the public.|Every effort is made to prevent unscheduled stops, although the President may, and in Dallas did, order stops in order to greet the public. +LJ030-0039|Men the motorcade slows or stops, agents take positions between the President and the crowd.|Men the motorcade slows or stops, agents take positions between the President and the crowd. +LJ030-0040|The order of vehicles in the Dallas motorcade was as follows:|The order of vehicles in the Dallas motorcade was as follows: +LJ030-0041|Motorcycles. -- Dallas police motorcycles preceded the pilot car.|Motorcycles. -- Dallas police motorcycles preceded the pilot car. +LJ030-0042|Manned by officers of the Dallas Police Department, this automobile preceded the main party by approximately quarter of a mile.|Manned by officers of the Dallas Police Department, this automobile preceded the main party by approximately quarter of a mile. +LJ030-0043|Its function was to alert police along the route that the motorcade was approaching and to check for signs of trouble.|Its function was to alert police along the route that the motorcade was approaching and to check for signs of trouble. +LJ030-0044|Motorcycles. -- Next came four to six motorcycle policemen whose main purpose was to keep the crowd back.|Motorcycles. -- Next came four to six motorcycle policemen whose main purpose was to keep the crowd back. +LJ030-0045|The lead car. -- Described as a "rolling command car," this was an unmarked Dallas police car, driven by Chief of Police Curry|The lead car. -- Described as a "rolling command car," this was an unmarked Dallas police car, driven by Chief of Police Curry +LJ030-0046|and occupied by Secret Service Agents Sorrels and Lawson and by Dallas County Sheriff J. E. Decker.|and occupied by Secret Service Agents Sorrels and Lawson and by Dallas County Sheriff J. E. Decker. +LJ030-0047|The occupants scanned the crowd and the buildings along the route.|The occupants scanned the crowd and the buildings along the route. +LJ030-0048|Their main function was to spot trouble in advance and to direct any necessary steps to meet the trouble.|Their main function was to spot trouble in advance and to direct any necessary steps to meet the trouble. +LJ030-0049|Following normal practice, the lead automobile stayed approximately 4 to 5 car lengths ahead of the President's limousine.|Following normal practice, the lead automobile stayed approximately four to five car lengths ahead of the President's limousine. +LJ030-0050|The Presidential limousine.|The Presidential limousine. +LJ030-0051|The President's automobile was specially designed 1961 Lincoln convertible|The President's automobile was specially designed nineteen sixty-one Lincoln convertible +LJ030-0052|with two collapsible jump seats between the front and rear seats.|with two collapsible jump seats between the front and rear seats. +LJ030-0053|It was outfitted with a clear plastic bubbletop which was neither bulletproof nor bullet resistant.|It was outfitted with a clear plastic bubbletop which was neither bulletproof nor bullet resistant. +LJ030-0054|Because the skies had cleared in Dallas, Lawson directed that the top not be used for the day's activities.|Because the skies had cleared in Dallas, Lawson directed that the top not be used for the day's activities. +LJ030-0055|He acted on instructions he had received earlier from Assistant Special Agent in Charge Roy H. Kellerman, who was in Fort Worth with the President.|He acted on instructions he had received earlier from Assistant Special Agent in Charge Roy H. Kellerman, who was in Fort Worth with the President. +LJ030-0056|Kellerman had discussed the matter with O'Donnell, whose instructions were, quote,|Kellerman had discussed the matter with O'Donnell, whose instructions were, quote, +LJ030-0057|If the weather is clear and it is not raining, have that bubbletop off, end quote,|If the weather is clear and it is not raining, have that bubbletop off, end quote, +LJ030-0058|Elevated approximately 15 inches above the back of the front seat|Elevated approximately fifteen inches above the back of the front seat +LJ030-0059|was a metallic frame with four handholds that riders in the car could grip while standing in the rear seat during parades.|was a metallic frame with four handholds that riders in the car could grip while standing in the rear seat during parades. +LJ030-0060|At the rear on each side of the automobile were small running boards, each designed to hold a Secret Service agent,|At the rear on each side of the automobile were small running boards, each designed to hold a Secret Service agent, +LJ030-0061|with a metallic handle for the rider to grasp.|with a metallic handle for the rider to grasp. +LJ030-0062|The President had frequently stated that he did not want agents to ride on these steps during a motorcade except when necessary.|The President had frequently stated that he did not want agents to ride on these steps during a motorcade except when necessary. +LJ030-0063|He had repeated this wish only a few days before, during his visit to Tampa, Florida.|He had repeated this wish only a few days before, during his visit to Tampa, Florida. +LJ030-0064|President Kennedy rode on the right-hand side of the rear seat with Mrs. Kennedy on his left.|President Kennedy rode on the right-hand side of the rear seat with Mrs. Kennedy on his left. +LJ030-0065|Governor Connally occupied the right jump seat, Mrs. Connally the left.|Governor Connally occupied the right jump seat, Mrs. Connally the left. +LJ030-0066|Driving the Presidential limousine was Special Agent William R. Greer of the Secret Service;|Driving the Presidential limousine was Special Agent William R. Greer of the Secret Service; +LJ030-0067|on his right sat Kellerman.|on his right sat Kellerman. +LJ030-0068|Kellerman's responsibilities included maintaining radio communications with the lead and follow-up cars,|Kellerman's responsibilities included maintaining radio communications with the lead and follow-up cars, +LJ030-0069|scanning the route, and getting out and standing near the President when the cars stopped.|scanning the route, and getting out and standing near the President when the cars stopped. +LJ030-0070|Motorcycles. -- Four motorcycles, two on each side, flanked the rear of the Presidential car.|Motorcycles. -- Four motorcycles, two on each side, flanked the rear of the Presidential car. +LJ030-0071|They provided some cover for the President, but their main purpose was to keep back the crowd.|They provided some cover for the President, but their main purpose was to keep back the crowd. +LJ030-0072|On previous occasions, the President had requested that, to the extent possible, these flanking motorcycles keep back from the sides of his car.|On previous occasions, the President had requested that, to the extent possible, these flanking motorcycles keep back from the sides of his car. +LJ030-0073|Presidential follow-up car.|Presidential follow-up car. +LJ030-0074|This vehicle, a 1955 Cadillac eight-passenger convertible especially outfitted for the Secret Service,|This vehicle, a nineteen fifty-five Cadillac eight-passenger convertible especially outfitted for the Secret Service, +LJ030-0075|followed closely behind the President's automobile.|followed closely behind the President's automobile. +LJ030-0076|It carried eight Secret Service agents -- two in the front seat, two in the rear, and two on each of the right and left running boards.|It carried eight Secret Service agents -- two in the front seat, two in the rear, and two on each of the right and left running boards. +LJ030-0077|Each agent carried a .38-caliber pistol, and a shotgun and automatic rifle were also available.|Each agent carried a thirty-eight-caliber pistol, and a shotgun and automatic rifle were also available. +LJ030-0078|Presidential Assistants David F. Powers and Kenneth O'Donnell sat in the right and left jump seats, respectively.|Presidential Assistants David F. Powers and Kenneth O'Donnell sat in the right and left jump seats, respectively. +LJ030-0079|The agents in this car, under established procedure, had instructions to watch the route for signs of trouble,|The agents in this car, under established procedure, had instructions to watch the route for signs of trouble, +LJ030-0080|scanning not only the crowds but the windows and roofs of buildings, overpasses, and crossings.|scanning not only the crowds but the windows and roofs of buildings, overpasses, and crossings. +LJ030-0081|They were instructed to watch particularly for thrown objects, sudden actions in the crowd, and any movements toward the Presidential car.|They were instructed to watch particularly for thrown objects, sudden actions in the crowd, and any movements toward the Presidential car. +LJ030-0082|The agents on the front of the running boards had directions to move immediately to positions just to the rear of the President and Mrs. Kennedy when the President's car slowed|The agents on the front of the running boards had directions to move immediately to positions just to the rear of the President and Mrs. Kennedy when the President's car slowed +LJ030-0083|to a walking pace or stopped,|to a walking pace or stopped, +LJ030-0084|or when the press of the crowd made it impossible for the escort motorcycles to stay in position on the car's rear flanks.|or when the press of the crowd made it impossible for the escort motorcycles to stay in position on the car's rear flanks. +LJ030-0085|The two agents on the rear of the running boards were to advance toward the front of the President's car whenever it stopped or slowed down sufficiently for them to do so.|The two agents on the rear of the running boards were to advance toward the front of the President's car whenever it stopped or slowed down sufficiently for them to do so. +LJ030-0086|Vice-Presidential car.|Vice-Presidential car. +LJ030-0087|The Vice-Presidential automobile, a four-door Lincoln convertible obtained locally for use in the motorcade,|The Vice-Presidential automobile, a four-door Lincoln convertible obtained locally for use in the motorcade, +LJ030-0088|proceeded approximately two to three car lengths behind the President's follow-up car.|proceeded approximately two to three car lengths behind the President's follow-up car. +LJ030-0089|This distance was maintained so that spectators would normally turn their gaze from the President's automobile by the time the Vice President came into view.|This distance was maintained so that spectators would normally turn their gaze from the President's automobile by the time the Vice President came into view. +LJ030-0090|Vice President Johnson sat on the right-hand side of the rear seat, Mrs. Johnson in the center, and Senator Yarborough on the left.|Vice President Johnson sat on the right-hand side of the rear seat, Mrs. Johnson in the center, and Senator Yarborough on the left. +LJ030-0091|Rufus W. Youngblood, special agent in charge of the Vice President's detail,|Rufus W. Youngblood, special agent in charge of the Vice President's detail, +LJ030-0092|occupied the right-hand side of the front seat, and Hurchel Jacks of the Texas State Highway patrol was the driver.|occupied the right-hand side of the front seat, and Hurchel Jacks of the Texas State Highway patrol was the driver. +LJ030-0093|Vice-Presidential follow-up car.|Vice-Presidential follow-up car. +LJ030-0094|Driven by an officer of the Dallas Police Department,|Driven by an officer of the Dallas Police Department, +LJ030-0095|this vehicle was occupied by three Secret Service agents and Clifton C. Garter, assistant to the Vice President.|this vehicle was occupied by three Secret Service agents and Clifton C. Garter, assistant to the Vice President. +LJ030-0096|These agents performed for the Vice President the same functions that the agents in the Presidential follow-up car performed for the President.|These agents performed for the Vice President the same functions that the agents in the Presidential follow-up car performed for the President. +LJ030-0097|Remainder of motorcade.|Remainder of motorcade. +LJ030-0099|telephone and Western Union vehicles, a White House communications car,|telephone and Western Union vehicles, a White House communications car, +LJ030-0100|three cars for press photographers, an official party bus for White House staff members and others, and two press buses.|three cars for press photographers, an official party bus for White House staff members and others, and two press buses. +LJ030-0101|Admiral George G. Burkley, physician to the President,|Admiral George G. Burkley, physician to the President, +LJ030-0102|was in a car following those, quote, containing the local and national representatives, end quote,|was in a car following those, quote, containing the local and national representatives, end quote, +LJ030-0103|Police car and motorcycles.|Police car and motorcycles. +LJ030-0104|A Dallas police car and several motorcycles at the rear kept the motorcade together and prevented unauthorized vehicles from joining the motorcade.|A Dallas police car and several motorcycles at the rear kept the motorcade together and prevented unauthorized vehicles from joining the motorcade. +LJ030-0105|Communications in the motorcade.|Communications in the motorcade. +LJ030-0106|A base station at a fixed location in Dallas operated a radio network which linked together the lead car,|A base station at a fixed location in Dallas operated a radio network which linked together the lead car, +LJ030-0107|Presidential car, Presidential follow-up car,|Presidential car, Presidential follow-up car, +LJ030-0108|White House communications car, Trade Mart, Love Field, and the Presidential and Vice-Presidential airplanes.|White House communications car, Trade Mart, Love Field, and the Presidential and Vice-Presidential airplanes. +LJ030-0109|The Vice-Presidential car|The Vice-Presidential car +LJ030-0110|and Vice-Presidential follow-up car used portable sets with a separate frequency for their own car-to-car communication.|and Vice-Presidential follow-up car used portable sets with a separate frequency for their own car-to-car communication. +LJ030-0111|The Drive through Dallas|The Drive through Dallas +LJ030-0112|The motorcade left Love Field shortly after 11:50 a.m. and drove at speeds up to 25 to 30 miles an hour|The motorcade left Love Field shortly after eleven:fifty a.m. and drove at speeds up to twenty-five to thirty miles an hour +LJ030-0113|through thinly populated areas on the outskirts of Dallas.|through thinly populated areas on the outskirts of Dallas. +LJ030-0114|At the President's direction, his automobile stopped twice, the first time to permit him to respond to a sign asking him to shake hands.|At the President's direction, his automobile stopped twice, the first time to permit him to respond to a sign asking him to shake hands. +LJ030-0115|During this brief stop, agents in the front positions on the running boards of the Presidential follow-up car came forward and stood beside the President's car,|During this brief stop, agents in the front positions on the running boards of the Presidential follow-up car came forward and stood beside the President's car, +LJ030-0116|looking out toward the crowd, and Special Agent Kellerman assumed his position next to the car.|looking out toward the crowd, and Special Agent Kellerman assumed his position next to the car. +LJ030-0117|On the other occasion, the President halted the motorcade to speak to a Catholic nun and a group of small children.|On the other occasion, the President halted the motorcade to speak to a Catholic nun and a group of small children. +LJ030-0118|In the downtown area, large crowds of spectators gave the President a tremendous reception.|In the downtown area, large crowds of spectators gave the President a tremendous reception. +LJ030-0119|The crowds were so dense that Special Agent Clinton J. Hill|The crowds were so dense that Special Agent Clinton J. Hill +LJ030-0120|had to leave the left front running board of the President's follow-up car four times to ride on the rear of the President's limousine.|had to leave the left front running board of the President's follow-up car four times to ride on the rear of the President's limousine. +LJ030-0121|Several times Special Agent John D. Ready came forward from the right front running board of the Presidential follow-up car|Several times Special Agent John D. Ready came forward from the right front running board of the Presidential follow-up car +LJ030-0122|to the right side of the President's car.|to the right side of the President's car. +LJ030-0123|Special Agent Glen A. Bennett once left his place inside the follow-up car to help keep the crowd away from the President's car.|Special Agent Glen A. Bennett once left his place inside the follow-up car to help keep the crowd away from the President's car. +LJ030-0124|When a teenage boy ran toward the rear of the President's car, Ready left the running board to chase the boy back into the crowd.|When a teenage boy ran toward the rear of the President's car, Ready left the running board to chase the boy back into the crowd. +LJ030-0125|On several occasions when the Vice President's car was slowed down by the throng, Special Agent Youngblood stepped out to hold the crowd back.|On several occasions when the Vice President's car was slowed down by the throng, Special Agent Youngblood stepped out to hold the crowd back. +LJ030-0126|According to plan, the President's motorcade proceeded west through downtown Dallas on Main Street|According to plan, the President's motorcade proceeded west through downtown Dallas on Main Street +LJ030-0127|to the intersection of Houston Street, which marks the beginning of Dealey Plaza.|to the intersection of Houston Street, which marks the beginning of Dealey Plaza. +LJ030-0128|From Main Street the motorcade turned right and went north on Houston Street, passing tall buildings on the right,|From Main Street the motorcade turned right and went north on Houston Street, passing tall buildings on the right, +LJ030-0129|and headed toward the Texas School Book Depository Building.|and headed toward the Texas School Book Depository Building. +LJ030-0130|The spectators were still thickly congregated in front of the buildings which lined the east side of Houston Street, but the crowd thinned abruptly along Elm Street,|The spectators were still thickly congregated in front of the buildings which lined the east side of Houston Street, but the crowd thinned abruptly along Elm Street, +LJ030-0131|which curves in a southwesterly direction as it proceeds downgrade toward the Triple Underpass and the Stemmons Freeway.|which curves in a southwesterly direction as it proceeds downgrade toward the Triple Underpass and the Stemmons Freeway. +LJ030-0132|As the motorcade approached the intersection of Houston and Elm Streets, there was general gratification in the Presidential party about the enthusiastic reception.|As the motorcade approached the intersection of Houston and Elm Streets, there was general gratification in the Presidential party about the enthusiastic reception. +LJ030-0133|Evaluating the political overtones, Kenneth O'Donnell was especially pleased|Evaluating the political overtones, Kenneth O'Donnell was especially pleased +LJ030-0134|because it convinced him that the average Dallas resident was like other American citizens in respecting and admiring the President.|because it convinced him that the average Dallas resident was like other American citizens in respecting and admiring the President. +LJ030-0135|Mrs. Connally, elated by the reception, turned to President Kennedy and said, quote, Mr. President, you can't say Dallas doesn't love you.|Mrs. Connally, elated by the reception, turned to President Kennedy and said, quote, Mr. President, you can't say Dallas doesn't love you. +LJ030-0136|end quote, the President replied, "That is very obvious."|end quote, the President replied, "That is very obvious." +LJ030-0137|The Assassination|The Assassination +LJ030-0138|At 12:30 p.m., Eastern Standard Time, as the President's open limousine proceeded at approximately 11 miles per hour along Elm Street|At twelve:thirty p.m., Eastern Standard Time, as the President's open limousine proceeded at approximately eleven miles per hour along Elm Street +LJ030-0139|toward the Triple Underpass, shots fired from a rifle mortally wounded President Kennedy and seriously injured Governor Connally.|toward the Triple Underpass, shots fired from a rifle mortally wounded President Kennedy and seriously injured Governor Connally. +LJ030-0140|One bullet passed through the President's neck;|One bullet passed through the President's neck; +LJ030-0141|a subsequent bullet, which was lethal, shattered the right side of his skull.|a subsequent bullet, which was lethal, shattered the right side of his skull. +LJ030-0142|Governor Connally sustained bullet wounds in his back, the right side of his chest, right wrist, and left thigh.|Governor Connally sustained bullet wounds in his back, the right side of his chest, right wrist, and left thigh. +LJ030-0143|The exact time of the assassination was fixed by the testimony of four witnesses.|The exact time of the assassination was fixed by the testimony of four witnesses. +LJ030-0144|Special Agent Rufus W. Youngblood observed that the large electric sign clock atop the Texas School Book Depository Building|Special Agent Rufus W. Youngblood observed that the large electric sign clock atop the Texas School Book Depository Building +LJ030-0145|showed the numerals 12:30 as the Vice-Presidential automobile proceeded north on Houston Street, a few seconds before the shots were fired.|showed the numerals twelve:thirty as the Vice-Presidential automobile proceeded north on Houston Street, a few seconds before the shots were fired. +LJ030-0146|Just prior to the shooting, David F. Powers, riding in the Secret Service follow-up car, remarked to Kenneth O'Donnell|Just prior to the shooting, David F. Powers, riding in the Secret Service follow-up car, remarked to Kenneth O'Donnell +LJ030-0147|that it was 12:30 p.m., the time they were due at the Trade Mart.|that it was twelve:thirty p.m., the time they were due at the Trade Mart. +LJ030-0148|Seconds after the shooting, Roy Kellerman, riding in the front seat of the Presidential limousine,|Seconds after the shooting, Roy Kellerman, riding in the front seat of the Presidential limousine, +LJ030-0149|looked at his watch and said "12:30" to the driver, Special Agent Greer.|looked at his watch and said "twelve:thirty" to the driver, Special Agent Greer. +LJ030-0150|The Dallas police radio log reflects that Chief of Police Curry reported the shooting of the President|The Dallas police radio log reflects that Chief of Police Curry reported the shooting of the President +LJ030-0151|and issued his initial orders at 12:30 p.m.|and issued his initial orders at twelve:thirty p.m. +LJ030-0152|Speed of the Limousine|Speed of the Limousine +LJ030-0153|William Greer, operator of the Presidential limousine, estimated the car's speed at the time of the first shot as 12 to 15 miles per hour.|William Greer, operator of the Presidential limousine, estimated the car's speed at the time of the first shot as twelve to fifteen miles per hour. +LJ030-0154|Other witnesses in the motorcade estimated the speed of the President's limousine from 7 to 22 miles per hour.|Other witnesses in the motorcade estimated the speed of the President's limousine from seven to twenty-two miles per hour. +LJ030-0155|A more precise determination has been made from motion pictures taken on the scene by an amateur photographer, Abraham Zapruder.|A more precise determination has been made from motion pictures taken on the scene by an amateur photographer, Abraham Zapruder. +LJ030-0156|Based on these films, the speed of the President's automobile is computed at an average speed of 11.2 miles per hour.|Based on these films, the speed of the President's automobile is computed at an average speed of eleven point two miles per hour. +LJ030-0157|The car maintained this average speed over a distance of approximately 186 feet immediately preceding the shot which struck the President in the head.|The car maintained this average speed over a distance of approximately one hundred eighty-six feet immediately preceding the shot which struck the President in the head. +LJ030-0158|While the car traveled this distance, the Zapruder camera ran 152 frames.|While the car traveled this distance, the Zapruder camera ran one hundred fifty-two frames. +LJ030-0159|Since the camera operates at a speed of 18.3 frames per second,|Since the camera operates at a speed of eighteen point three frames per second, +LJ030-0160|it was calculated that the car required 8.3 seconds to cover the 136 feet.|it was calculated that the car required eight point three seconds to cover the one hundred thirty-six feet. +LJ030-0161|This represents a speed of 11.2 miles per hour.|This represents a speed of eleven point two miles per hour. +LJ030-0162|In the Presidential Limousine|In the Presidential Limousine +LJ030-0163|Mrs. John F. Kennedy, on the left of the rear seat of the limousine, looked toward her left and waved to the crowds along the route.|Mrs. John F. Kennedy, on the left of the rear seat of the limousine, looked toward her left and waved to the crowds along the route. +LJ030-0164|Soon after the motorcade turned onto Elm Street, she heard a sound similar to a motorcycle noise and a cry from Governor Connally,|Soon after the motorcade turned onto Elm Street, she heard a sound similar to a motorcycle noise and a cry from Governor Connally, +LJ030-0165|which caused her to look to her right.|which caused her to look to her right. +LJ030-0166|On turning she saw a quizzical look on her husband's face as he raised his left hand to his throat.|On turning she saw a quizzical look on her husband's face as he raised his left hand to his throat. +LJ030-0167|Mrs. Kennedy then heard a second shot and saw the President's skull torn open under the impact of the bullet.|Mrs. Kennedy then heard a second shot and saw the President's skull torn open under the impact of the bullet. +LJ030-0168|As she cradled her mortally wounded husband, Mrs. Kennedy cried, quote,|As she cradled her mortally wounded husband, Mrs. Kennedy cried, quote, +LJ030-0169|Oh, my God, they have shot my husband. I love you, Jack.|Oh, my God, they have shot my husband. I love you, Jack. +LJ030-0170|Governor Connally testified that he recognized the first noise as a rifle shot|Governor Connally testified that he recognized the first noise as a rifle shot +LJ030-0171|and the thought immediately crossed his mind that it was an assassination attempt.|and the thought immediately crossed his mind that it was an assassination attempt. +LJ030-0172|From his position in the right jump seat immediately in front of the President,|From his position in the right jump seat immediately in front of the President, +LJ030-0173|he instinctively turned to his right because the shot appeared to come from over his right shoulder.|he instinctively turned to his right because the shot appeared to come from over his right shoulder. +LJ030-0174|Unable to see the President as he turned to the right,|Unable to see the President as he turned to the right, +LJ030-0175|the Governor started to look back over his left shoulder, but he never completed the turn because he felt something strike him in the back.|the Governor started to look back over his left shoulder, but he never completed the turn because he felt something strike him in the back. +LJ030-0176|In his testimony before the Commission, Governor Connally was certain that he was hit by the second shot, which he stated he did not hear.|In his testimony before the Commission, Governor Connally was certain that he was hit by the second shot, which he stated he did not hear. +LJ030-0177|Mrs. Connally, too, heard a frightening noise from her right.|Mrs. Connally, too, heard a frightening noise from her right. +LJ030-0178|Looking over her right shoulder, she saw that the President had both hands at his neck but she observed no blood and heard nothing.|Looking over her right shoulder, she saw that the President had both hands at his neck but she observed no blood and heard nothing. +LJ030-0179|She watched as he slumped down with an empty expression on his face.|She watched as he slumped down with an empty expression on his face. +LJ030-0180|Roy Kellerman, in the right front seat of the limousine, heard a report like a firecracker pop.|Roy Kellerman, in the right front seat of the limousine, heard a report like a firecracker pop. +LJ030-0181|Turning to his right in the direction of the noise, Kellerman heard the President say|Turning to his right in the direction of the noise, Kellerman heard the President say +LJ030-0182|"My God, I am hit," and saw both of the President's hands move up toward his neck.|"My God, I am hit," and saw both of the President's hands move up toward his neck. +LJ030-0183|As he told the driver, quote, Let's get out of here; we are hit, end quote,|As he told the driver, quote, Let's get out of here; we are hit, end quote, +LJ030-0184|Kellerman grabbed his microphone and radioed ahead to the lead car,|Kellerman grabbed his microphone and radioed ahead to the lead car, +LJ030-0185|quote, we are hit. Get us to the hospital immediately, end quote.|quote, we are hit. Get us to the hospital immediately, end quote. +LJ030-0186|The driver, William Greer, heard a noise which he took to be a backfire from one of the motorcycles flanking the Presidential car.|The driver, William Greer, heard a noise which he took to be a backfire from one of the motorcycles flanking the Presidential car. +LJ030-0187|When he heard the same noise again, Greer glanced over his shoulder and saw Governor Connally fall.|When he heard the same noise again, Greer glanced over his shoulder and saw Governor Connally fall. +LJ030-0188|At the sound of the second shot|At the sound of the second shot +LJ030-0189|he realized that something was wrong, and he pressed down on the accelerator as Kellerman said, quote, Get out of here fast, end quote.|he realized that something was wrong, and he pressed down on the accelerator as Kellerman said, quote, Get out of here fast, end quote. +LJ030-0190|As he issued his instructions to Greer and to the lead car, Kellerman heard a flurry of shots within 5 seconds of the first noise.|As he issued his instructions to Greer and to the lead car, Kellerman heard a flurry of shots within five seconds of the first noise. +LJ030-0191|According to Kellerman, Mrs. Kennedy then cried out, quote,|According to Kellerman, Mrs. Kennedy then cried out, quote, +LJ030-0192|What are they doing to you! end quote. Looking back from the front seat,|What are they doing to you! end quote. Looking back from the front seat, +LJ030-0193|Kellerman saw Governor Connally in his wife's lap and Special Agent Clinton J. Hill lying across the trunk of the car.|Kellerman saw Governor Connally in his wife's lap and Special Agent Clinton J. Hill lying across the trunk of the car. +LJ030-0194|Mrs. Connally heard a second shot fired and pulled her husband down into her lap.|Mrs. Connally heard a second shot fired and pulled her husband down into her lap. +LJ030-0195|Observing his blood-covered chest as he was pulled into his wife's lap, Governor Connally believed himself mortally wounded.|Observing his blood-covered chest as he was pulled into his wife's lap, Governor Connally believed himself mortally wounded. +LJ030-0196|He cried out, quote, Oh, no, no, no. My God, they are going to kill us all, end quote,|He cried out, quote, Oh, no, no, no. My God, they are going to kill us all, end quote, +LJ030-0197|At first Mrs. Connally thought that her husband had been killed,|At first Mrs. Connally thought that her husband had been killed, +LJ030-0198|but then she noticed an almost imperceptible movement and knew that he was still alive. She said, quote, It's all right. Be still, end quote,|but then she noticed an almost imperceptible movement and knew that he was still alive. She said, quote, It's all right. Be still, end quote, +LJ030-0199|The Governor was lying with his head on his wife's lap when he heard a shot hit the President.|The Governor was lying with his head on his wife's lap when he heard a shot hit the President. +LJ030-0200|At that point, both Governor and Mrs. Connally observed brain tissue splattered over the interior of the car.|At that point, both Governor and Mrs. Connally observed brain tissue splattered over the interior of the car. +LJ030-0201|According to Governor and Mrs. Connally, it was after this shot that Kellerman issued his emergency instructions and the car accelerated.|According to Governor and Mrs. Connally, it was after this shot that Kellerman issued his emergency instructions and the car accelerated. +LJ030-0202|Reaction by Secret Service Agents|Reaction by Secret Service Agents +LJ030-0203|From the left front running board of the President's follow-up car,|From the left front running board of the President's follow-up car, +LJ030-0204|Special Agent Hill was scanning the few people standing on the south side of Elm Street after the motorcade had turned off Houston Street.|Special Agent Hill was scanning the few people standing on the south side of Elm Street after the motorcade had turned off Houston Street. +LJ030-0205|He estimated that the motorcade had slowed down to approximately 9 or 10 miles per hour|He estimated that the motorcade had slowed down to approximately nine or ten miles per hour +LJ030-0206|on the turn at the intersection of Houston and Elm Streets and then proceeded at a rate of 12 to 15 miles per hour|on the turn at the intersection of Houston and Elm Streets and then proceeded at a rate of twelve to fifteen miles per hour +LJ030-0207|with the follow-up car trailing the President's automobile by approximately 5 feet.|with the follow-up car trailing the President's automobile by approximately five feet. +LJ030-0208|Hill heard a noise, which seemed to be a firecracker, coming from his right rear.|Hill heard a noise, which seemed to be a firecracker, coming from his right rear. +LJ030-0209|He immediately looked to his right, quote, and, in so doing, my eyes had to cross the Presidential limousine|He immediately looked to his right, quote, and, in so doing, my eyes had to cross the Presidential limousine +LJ030-0210|and I saw President Kennedy grab at himself and lurch forward and to the left, end quote.|and I saw President Kennedy grab at himself and lurch forward and to the left, end quote. +LJ030-0211|Hill jumped from the follow-up car and ran to the President's automobile.|Hill jumped from the follow-up car and ran to the President's automobile. +LJ030-0212|At about the time he reached the President's automobile,|At about the time he reached the President's automobile, +LJ030-0213|Hill heard a second shot, proximately 5 seconds after the first, which removed a portion of the President's head.|Hill heard a second shot, proximately five seconds after the first, which removed a portion of the President's head. +LJ030-0214|At the instant that Hill stepped onto the left rear step of the President's automobile and grasped the handhold,|At the instant that Hill stepped onto the left rear step of the President's automobile and grasped the handhold, +LJ030-0215|the car lurched forward, causing him to lose his footing. He ran three or four steps, regained his position and mounted the car.|the car lurched forward, causing him to lose his footing. He ran three or four steps, regained his position and mounted the car. +LJ030-0216|Between the time he originally seized the handhold and the time he mounted the car, Hill recalled:|Between the time he originally seized the handhold and the time he mounted the car, Hill recalled: +LJ030-0217|quote, Mrs. Kennedy had jumped up from the seat and was, it appeared to me, reaching for something coming off the fight rear bumper of the car,|quote, Mrs. Kennedy had jumped up from the seat and was, it appeared to me, reaching for something coming off the fight rear bumper of the car, +LJ030-0218|the right rear tail, when she noticed that I was trying to climb on the car.|the right rear tail, when she noticed that I was trying to climb on the car. +LJ030-0219|She turned toward me and I grabbed her and put her back in the back seat, crawled up on top of the back seat and lay there.|She turned toward me and I grabbed her and put her back in the back seat, crawled up on top of the back seat and lay there. +LJ030-0220|David Powers, who witnessed the scene from the President's follow-up car,|David Powers, who witnessed the scene from the President's follow-up car, +LJ030-0221|stated that Mrs. Kennedy would probably have fallen off the rear end of the car and been killed if Hill had not pushed her back into the Presidential automobile.|stated that Mrs. Kennedy would probably have fallen off the rear end of the car and been killed if Hill had not pushed her back into the Presidential automobile. +LJ030-0222|Mrs. Kennedy had no recollection of climbing onto the back of the car.|Mrs. Kennedy had no recollection of climbing onto the back of the car. +LJ030-0223|Special Agent Ready, on the right front running board of the Presidential follow-up car,|Special Agent Ready, on the right front running board of the Presidential follow-up car, +LJ030-0224|heard noises that sounded like firecrackers and ran toward the President's limousine.|heard noises that sounded like firecrackers and ran toward the President's limousine. +LJ030-0225|But he was immediately called back by Special Agent Emory P. Roberts, in charge of the follow-up car,|But he was immediately called back by Special Agent Emory P. Roberts, in charge of the follow-up car, +LJ030-0226|who did not believe that he could reach, the President's car at the speed it was then traveling.|who did not believe that he could reach, the President's car at the speed it was then traveling. +LJ030-0227|Special Agent George W. Hickey, Jr., in the rear seat of the Presidential follow-up car,|Special Agent George W. Hickey, Jr., in the rear seat of the Presidential follow-up car, +LJ030-0228|picked up and cocked an automatic rifle as he heard the last shot.|picked up and cocked an automatic rifle as he heard the last shot. +LJ030-0229|At this point the cars were speeding through the underpass and had left the scene of the shooting,|At this point the cars were speeding through the underpass and had left the scene of the shooting, +LJ030-0230|but Hickey kept the automatic weapon ready as the car raced to the hospital.|but Hickey kept the automatic weapon ready as the car raced to the hospital. +LJ030-0231|Most of the other Secret Service agents in the motorcade had drawn their sidearms.|Most of the other Secret Service agents in the motorcade had drawn their sidearms. +LJ030-0232|Roberts noticed that the Vice President's car was approximately one-half block|Roberts noticed that the Vice President's car was approximately one-half block +LJ030-0233|behind the Presidential follow-up car at the time of the shooting and signaled for it to move in closer.|behind the Presidential follow-up car at the time of the shooting and signaled for it to move in closer. +LJ030-0234|Directing the security detail for the Vice President from the right front seat of the Vice-Presidential car, Special Agent Youngblood recalled, quote,|Directing the security detail for the Vice President from the right front seat of the Vice-Presidential car, Special Agent Youngblood recalled, quote, +LJ030-0235|As we were beginning to go down this incline, all of a sudden there was an explosive noise.|As we were beginning to go down this incline, all of a sudden there was an explosive noise. +LJ030-0236|I quickly observed unnatural movement of crowds, like ducking or scattering, and quick movements in the Presidential follow-up car.|I quickly observed unnatural movement of crowds, like ducking or scattering, and quick movements in the Presidential follow-up car. +LJ030-0237|So I turned around and hit the Vice President on the shoulder and hollered, get down,|So I turned around and hit the Vice President on the shoulder and hollered, get down, +LJ030-0238|and then looked around again and saw more of this movement, and so I proceeded to go to the back seat and get on top of him.|and then looked around again and saw more of this movement, and so I proceeded to go to the back seat and get on top of him. +LJ030-0239|Youngblood was not positive that he was in the rear seat before the second shot, but thought it probable because of President Johnson's statement|Youngblood was not positive that he was in the rear seat before the second shot, but thought it probable because of President Johnson's statement +LJ030-0240|to that effect immediately after the assassination.|to that effect immediately after the assassination. +LJ030-0241|President Johnson emphasized Youngblood's instantaneous reaction after the first shot:|President Johnson emphasized Youngblood's instantaneous reaction after the first shot: +LJ030-0242|I was startled by the sharp report or explosion,|I was startled by the sharp report or explosion, +LJ030-0243|but I had no time to speculate as to its origin because Agent Youngblood turned in a flash, immediately after the first explosion,|but I had no time to speculate as to its origin because Agent Youngblood turned in a flash, immediately after the first explosion, +LJ030-0244|hitting me on the shoulder, and shouted to all of us in the back seat to get down.|hitting me on the shoulder, and shouted to all of us in the back seat to get down. +LJ030-0245|I was pushed down by Agent Youngblood.|I was pushed down by Agent Youngblood. +LJ030-0246|Almost in the same moment in which he hit or pushed me, he vaulted over the back seat and sat on me.|Almost in the same moment in which he hit or pushed me, he vaulted over the back seat and sat on me. +LJ030-0247|I was bent over under the weight of Agent Youngblood's body, toward Mrs. Johnson and Senator Yarborough, end quote,|I was bent over under the weight of Agent Youngblood's body, toward Mrs. Johnson and Senator Yarborough, end quote, +LJ030-0248|Clifton C. Carter, riding in the Vice President's follow-up car a short distance behind,|Clifton C. Carter, riding in the Vice President's follow-up car a short distance behind, +LJ030-0249|reported that Youngblood was in the rear seat using his body to shield the Vice President before the second and third shots were fired.|reported that Youngblood was in the rear seat using his body to shield the Vice President before the second and third shots were fired. +LJ030-0250|Other Secret Service agents assigned to the motorcade remained at their posts during the race to the hospital.|Other Secret Service agents assigned to the motorcade remained at their posts during the race to the hospital. +LJ030-0251|None stayed at the scene of the shooting, and none entered the Texas School Book Depository Building at or immediately after the shooting.|None stayed at the scene of the shooting, and none entered the Texas School Book Depository Building at or immediately after the shooting. +LJ030-0252|Secret Service procedure requires that each agent stay with the person being protected|Secret Service procedure requires that each agent stay with the person being protected +LJ030-0253|and not be diverted unless it is necessary to accomplish the protective assignment.|and not be diverted unless it is necessary to accomplish the protective assignment. +LJ030-0254|Forrest V. Sorrels, special agent in charge of the Dallas office, was the first Secret Service agent to return to the scene of the assassination,|Forrest V. Sorrels, special agent in charge of the Dallas office, was the first Secret Service agent to return to the scene of the assassination, +LJ030-0255|approximately 20 or 25 minutes after the shots were fired.|approximately twenty or twenty-five minutes after the shots were fired. +LJ031-0001|Report of the President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. +LJ031-0002|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter 2. The Assassination: Part 3.|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter two. The Assassination: Part three. +LJ031-0003|Parkland Memorial Hospital. The Race to the Hospital|Parkland Memorial Hospital. The Race to the Hospital +LJ031-0004|In the final instant of the assassination, the Presidential motorcade began a race to Parkland Memorial Hospital,|In the final instant of the assassination, the Presidential motorcade began a race to Parkland Memorial Hospital, +LJ031-0005|approximately 4 miles from the Texas School Book Depository Building.|approximately four miles from the Texas School Book Depository Building. +LJ031-0006|On receipt of the radio message from Kellerman to the lead car that the President had been hit,|On receipt of the radio message from Kellerman to the lead car that the President had been hit, +LJ031-0008|Meanwhile, Chief Curry ordered the police base station to notify Parkland Hospital that the wounded President was en route.|Meanwhile, Chief Curry ordered the police base station to notify Parkland Hospital that the wounded President was en route. +LJ031-0009|The radio log of the Dallas Police Department shows that at 12:30 p.m. on November 22|The radio log of the Dallas Police Department shows that at twelve:thirty p.m. on November twenty-two +LJ031-0010|Chief Curry radioed, quote, Go to the hospital -- Parkland Hospital. Have them stand by, end quote,|Chief Curry radioed, quote, Go to the hospital -- Parkland Hospital. Have them stand by, end quote, +LJ031-0011|A moment later Curry added, quote, Looks like the President has been hit. Have Parkland stand by, end quote,|A moment later Curry added, quote, Looks like the President has been hit. Have Parkland stand by, end quote, +LJ031-0012|The base station replied, quote, They have been notified, end quote,|The base station replied, quote, They have been notified, end quote, +LJ031-0013|Traveling at speeds estimated at times to be up to 70 or 80 miles per hour down the Stemmons Freeway and Harry Hines Boulevard|Traveling at speeds estimated at times to be up to seventy or eighty miles per hour down the Stemmons Freeway and Harry Hines Boulevard +LJ031-0014|the Presidential limousine arrived at the emergency entrance of the Parkland Hospital at about 12:35 p.m.|the Presidential limousine arrived at the emergency entrance of the Parkland Hospital at about twelve:thirty-five p.m. +LJ031-0015|Arriving almost simultaneously were the President's follow-up car, the Vice President's automobile, and the Vice President's follow-up car.|Arriving almost simultaneously were the President's follow-up car, the Vice President's automobile, and the Vice President's follow-up car. +LJ031-0016|Admiral Burkley, the President's physician, arrived at the hospital, quote, between 3 and 5 minutes following the arrival of the President, end quote.|Admiral Burkley, the President's physician, arrived at the hospital, quote, between three and five minutes following the arrival of the President, end quote. +LJ031-0017|since the riders in his car, quote, were not exactly aware what had happened, end quote, and the car went on to the Trade Mart first.|since the riders in his car, quote, were not exactly aware what had happened, end quote, and the car went on to the Trade Mart first. +LJ031-0018|When Parkland Hospital received the notification, the staff in the emergency area was alerted and trauma rooms 1 and 2 were prepared.|When Parkland Hospital received the notification, the staff in the emergency area was alerted and trauma rooms one and two were prepared. +LJ031-0019|These rooms were for the emergency treatment of acutely ill or injured patients. Although the first message mentioned an injury only to President Kennedy,|These rooms were for the emergency treatment of acutely ill or injured patients. Although the first message mentioned an injury only to President Kennedy, +LJ031-0020|two rooms were prepared.|two rooms were prepared. +LJ031-0021|As the President's limousine sped toward the hospital, 12 doctors to the emergency area: surgeons, Drs. Malcolm O. Perry,|As the President's limousine sped toward the hospital, twelve doctors to the emergency area: surgeons, Drs. Malcolm O. Perry, +LJ031-0022|Charles R. Baxter, Robert N. McClelland, Ronald C. Jones; the chief neurologist, Dr. William Kemp Clark;|Charles R. Baxter, Robert N. McClelland, Ronald C. Jones; the chief neurologist, Dr. William Kemp Clark; +LJ031-0023|4 anesthesiologists, Drs. Marion T. Jenkins, Adolph H. Giesecke, Jr., Jackie H. Hunt,|four anesthesiologists, Drs. Marion T. Jenkins, Adolph H. Giesecke, Jr., Jackie H. Hunt, +LJ031-0024|Gene C. Akin;|Gene C. Akin; +LJ031-0025|urological surgeon, Dr Paul C. Peters; an oral surgeon, Dr. Don T. Curtis; and a heart specialist,|urological surgeon, Dr Paul C. Peters; an oral surgeon, Dr. Don T. Curtis; and a heart specialist, +LJ031-0026|Dr. Fouad A. Bashour.|Dr. Fouad A. Bashour. +LJ031-0027|Upon arriving at Parkland Hospital,|Upon arriving at Parkland Hospital, +LJ031-0028|Lawson jumped from the lead car and rushed into the emergency entrance, where he was met by hospital staff members wheeling stretchers out to the automobile.|Lawson jumped from the lead car and rushed into the emergency entrance, where he was met by hospital staff members wheeling stretchers out to the automobile. +LJ031-0029|Special Agent Hill removed his suit jacket and covered the President's head and upper chest to prevent the taking of photographs.|Special Agent Hill removed his suit jacket and covered the President's head and upper chest to prevent the taking of photographs. +LJ031-0030|Governor Connally, who had lost consciousness on the ride to the hospital, regained consciousness when the limousine stopped abruptly at the emergency entrance.|Governor Connally, who had lost consciousness on the ride to the hospital, regained consciousness when the limousine stopped abruptly at the emergency entrance. +LJ031-0031|Despite his serious wounds, Governor Connally tried to get out of the way so that medical help could reach the President.|Despite his serious wounds, Governor Connally tried to get out of the way so that medical help could reach the President. +LJ031-0032|Although he was reclining in his wife's arms, he lurched forward in an effort to stand upright and get out of the car, but he collapsed again.|Although he was reclining in his wife's arms, he lurched forward in an effort to stand upright and get out of the car, but he collapsed again. +LJ031-0033|Then he experienced his first sensation of pain, which became excruciating.|Then he experienced his first sensation of pain, which became excruciating. +LJ031-0034|The Governor was lifted onto a stretcher and taken into trauma room 2.|The Governor was lifted onto a stretcher and taken into trauma room two. +LJ031-0035|For a moment, Mrs. Kennedy refused to release the President, whom she held in her lap,|For a moment, Mrs. Kennedy refused to release the President, whom she held in her lap, +LJ031-0036|but then Kellerman, Greer, and Lawson lifted the President onto a stretcher and pushed it into trauma room 1.|but then Kellerman, Greer, and Lawson lifted the President onto a stretcher and pushed it into trauma room one. +LJ031-0037|Treatment of President Kennedy|Treatment of President Kennedy +LJ031-0038|The first physician to see the President at Parkland Hospital was Dr. Charles J. Carrico, a resident in general surgery.|The first physician to see the President at Parkland Hospital was Dr. Charles J. Carrico, a resident in general surgery. +LJ031-0039|Dr. Carrico was in the emergency area, examining another patient, when he was notified that President Kennedy was en route to the hospital.|Dr. Carrico was in the emergency area, examining another patient, when he was notified that President Kennedy was en route to the hospital. +LJ031-0040|Approximately 2 minutes later, Dr. Carrico saw the President on his back, being wheeled into the emergency area.|Approximately two minutes later, Dr. Carrico saw the President on his back, being wheeled into the emergency area. +LJ031-0041|He noted that the President was blue-white or ashen in color; had slow, spasmodic, agonal respiration without any coordination;|He noted that the President was blue-white or ashen in color; had slow, spasmodic, agonal respiration without any coordination; +LJ031-0042|made no voluntary movements; had his eyes open with the pupils dilated without any reaction to light,|made no voluntary movements; had his eyes open with the pupils dilated without any reaction to light, +LJ031-0043|evidenced no palpable pulse; and had a few chest sounds which were thought to be heartbeats.|evidenced no palpable pulse; and had a few chest sounds which were thought to be heartbeats. +LJ031-0044|On the basis of these findings, Dr. Carrico concluded that President Kennedy was still alive.|On the basis of these findings, Dr. Carrico concluded that President Kennedy was still alive. +LJ031-0045|Dr. Carrico noted two wounds: a small bullet wound in the front lower neck,|Dr. Carrico noted two wounds: a small bullet wound in the front lower neck, +LJ031-0046|and an extensive wound in the President's head where a sizable portion of the skull was missing.|and an extensive wound in the President's head where a sizable portion of the skull was missing. +LJ031-0047|He observed shredded brain tissue and, quote, considerable slow oozing, end quote, from the latter wound,|He observed shredded brain tissue and, quote, considerable slow oozing, end quote, from the latter wound, +LJ031-0048|followed by, quote, more profuse bleeding, end quote, after some circulation was established.|followed by, quote, more profuse bleeding, end quote, after some circulation was established. +LJ031-0049|Dr. Carrico felt the President's back and determined that there was no large wound there which would be an immediate threat to life.|Dr. Carrico felt the President's back and determined that there was no large wound there which would be an immediate threat to life. +LJ031-0050|Observing the serious problems presented by the head wound and inadequate respiration, Dr. Carrico directed his attention to improving the President's breathing.|Observing the serious problems presented by the head wound and inadequate respiration, Dr. Carrico directed his attention to improving the President's breathing. +LJ031-0051|He noted contusions, hematoma to the right of the larynx, which was deviated slightly to the left,|He noted contusions, hematoma to the right of the larynx, which was deviated slightly to the left, +LJ031-0052|and also ragged tissue which indicated a tracheal injury.|and also ragged tissue which indicated a tracheal injury. +LJ031-0053|Dr. Carrico inserted a cuffed endotracheal tube past the injury, inflated the cuff, and connected it to a Bennett machine to assist in respiration.|Dr. Carrico inserted a cuffed endotracheal tube past the injury, inflated the cuff, and connected it to a Bennett machine to assist in respiration. +LJ031-0054|At that point, direction of the President's treatment was undertaken by Dr. Malcolm O. Perry, who arrived at trauma room 1 a few moments after the President.|At that point, direction of the President's treatment was undertaken by Dr. Malcolm O. Perry, who arrived at trauma room one a few moments after the President. +LJ031-0055|Dr. Perry noted the President's back brace as he felt for a femoral pulse, which he did not find.|Dr. Perry noted the President's back brace as he felt for a femoral pulse, which he did not find. +LJ031-0056|Observing that an effective airway had to be established if treatment was to be effective, Dr. Perry performed a tracheotomy, which required 3 to 5 minutes.|Observing that an effective airway had to be established if treatment was to be effective, Dr. Perry performed a tracheotomy, which required three to five minutes. +LJ031-0057|While Dr. Perry was performing the tracheotomy, Drs. Carrico and Ronald Jones made cutdowns on the President's right leg and left arm, respectively,|While Dr. Perry was performing the tracheotomy, Drs. Carrico and Ronald Jones made cutdowns on the President's right leg and left arm, respectively, +LJ031-0058|to infuse blood and fluids into the circulatory system.|to infuse blood and fluids into the circulatory system. +LJ031-0059|Dr. Carrico treated the President's known ad-renal insufficiency by administering hydrocortisone.|Dr. Carrico treated the President's known ad-renal insufficiency by administering hydrocortisone. +LJ031-0060|Dr. Robert N. McClelland entered at that point and assisted Dr. Perry with the tracheotomy.|Dr. Robert N. McClelland entered at that point and assisted Dr. Perry with the tracheotomy. +LJ031-0061|Dr. Fouad Bashour, chief of cardiology, Dr. M. T. Jenkins, chief of anesthesiology, and Dr. A. H. Giesecke, Jr.,|Dr. Fouad Bashour, chief of cardiology, Dr. M. T. Jenkins, chief of anesthesiology, and Dr. A. H. Giesecke, Jr., +LJ031-0062|then joined in the effort to revive the President.|then joined in the effort to revive the President. +LJ031-0063|When Dr. Perry noted free air and blood in the President's chest cavity, he asked that chest tubes be inserted to allow for drainage of blood and air.|When Dr. Perry noted free air and blood in the President's chest cavity, he asked that chest tubes be inserted to allow for drainage of blood and air. +LJ031-0064|Drs. Paul C. Peters and Charles R. Baxter initiated these procedures.|Drs. Paul C. Peters and Charles R. Baxter initiated these procedures. +LJ031-0065|As a result of the infusion of liquids through the cutdowns, the cardiac massage, and the airway,|As a result of the infusion of liquids through the cutdowns, the cardiac massage, and the airway, +LJ031-0066|the doctors were able to maintain peripheral circulation as monitored at the neck (carotid) artery and at the wrist (radial) pulse.|the doctors were able to maintain peripheral circulation as monitored at the neck (carotid) artery and at the wrist (radial) pulse. +LJ031-0067|A femoral pulse was also detected in the President's leg.|A femoral pulse was also detected in the President's leg. +LJ031-0068|While these medical efforts were in progress,|While these medical efforts were in progress, +LJ031-0069|Dr. Clark noted some electrical activity on the cardiotachyscope attached to monitor the President's heart responses.|Dr. Clark noted some electrical activity on the cardiotachyscope attached to monitor the President's heart responses. +LJ031-0070|Dr. Clark, who most closely observed the head wound,|Dr. Clark, who most closely observed the head wound, +LJ031-0071|described a large, gaping wound in the right rear part of the head, with substantial damage and exposure of brain tissue, and a considerable loss of blood.|described a large, gaping wound in the right rear part of the head, with substantial damage and exposure of brain tissue, and a considerable loss of blood. +LJ031-0072|Dr. Clark did not see any other hole or wound on the President's head.|Dr. Clark did not see any other hole or wound on the President's head. +LJ031-0073|According to Dr. Clark, the small bullet hole on the right rear of the President's head discovered during the subsequent autopsy|According to Dr. Clark, the small bullet hole on the right rear of the President's head discovered during the subsequent autopsy +LJ031-0074|quote, could have easily been hidden in the blood and hair, end quote,|quote, could have easily been hidden in the blood and hair, end quote, +LJ031-0075|In the absence of any neurological, muscular, or heart response, the doctors concluded that efforts to revive the President were hopeless.|In the absence of any neurological, muscular, or heart response, the doctors concluded that efforts to revive the President were hopeless. +LJ031-0076|This was verified by Admiral Burkley, the President's physician, who arrived at the hospital after emergency treatment was underway and concluded that, quote,|This was verified by Admiral Burkley, the President's physician, who arrived at the hospital after emergency treatment was underway and concluded that, quote, +LJ031-0077|my direct services to him at that moment would have interfered with the action of the team which was in progress, end quote,|my direct services to him at that moment would have interfered with the action of the team which was in progress, end quote, +LJ031-0078|At approximately 1 p.m., after last rites were administered to the President by Father Oscar L. Huber, Dr. Clark pronounced the President dead.|At approximately one p.m., after last rites were administered to the President by Father Oscar L. Huber, Dr. Clark pronounced the President dead. +LJ031-0079|He made the official determination because the ultimate cause of death, the severe head injury, was within his sphere of specialization.|He made the official determination because the ultimate cause of death, the severe head injury, was within his sphere of specialization. +LJ031-0080|The time was fixed at 1 p.m., as an approximation, since it was impossible to determine the precise moment when life left the President.|The time was fixed at one p.m., as an approximation, since it was impossible to determine the precise moment when life left the President. +LJ031-0081|President Kennedy could have survived the neck injury, but the head wound was fatal.|President Kennedy could have survived the neck injury, but the head wound was fatal. +LJ031-0082|From a medical viewpoint, President Kennedy was alive when he arrived at Parkland Hospital;|From a medical viewpoint, President Kennedy was alive when he arrived at Parkland Hospital; +LJ031-0083|the doctors observed that he had a heartbeat and was making some respiratory efforts.|the doctors observed that he had a heartbeat and was making some respiratory efforts. +LJ031-0084|But his condition was hopeless, and the extraordinary efforts of the doctors to save him could not help but to have been unavailing.|But his condition was hopeless, and the extraordinary efforts of the doctors to save him could not help but to have been unavailing. +LJ031-0085|Since the Dallas doctors directed all their efforts to controlling the massive bleeding caused by the head wound, and to reconstructing an airway to his lungs,|Since the Dallas doctors directed all their efforts to controlling the massive bleeding caused by the head wound, and to reconstructing an airway to his lungs, +LJ031-0086|the President remained on his back throughout his medical treatment at Parkland.|the President remained on his back throughout his medical treatment at Parkland. +LJ031-0087|When asked why he did not turn the President over, Dr. Carrico testified as follows:|When asked why he did not turn the President over, Dr. Carrico testified as follows: +LJ031-0088|This man was in obvious extreme distress and any more thorough inspection would have involved several minutes -- well, several|This man was in obvious extreme distress and any more thorough inspection would have involved several minutes -- well, several +LJ031-0089|considerable time which at this juncture was not available.|considerable time which at this juncture was not available. +LJ031-0090|A thorough inspection would have involved washing and cleansing the back, and this is not practical in treating an acutely injured patient.|A thorough inspection would have involved washing and cleansing the back, and this is not practical in treating an acutely injured patient. +LJ031-0092|Did you ever have occasion to look at the President's back?|Did you ever have occasion to look at the President's back? +LJ031-0093|Answer: No, sir. Before -- well, in trying to treat an acutely injured patient, you have to establish an airway, adequate ventilation|Answer: No, sir. Before -- well, in trying to treat an acutely injured patient, you have to establish an airway, adequate ventilation +LJ031-0094|and you have to establish adequate circulation.|and you have to establish adequate circulation. +LJ031-0095|Before this was accomplished the President's cardiac activity had ceased and closed cardiac massage was instituted, which made it impossible to inspect his back.|Before this was accomplished the President's cardiac activity had ceased and closed cardiac massage was instituted, which made it impossible to inspect his back. +LJ031-0096|Question: Was any effort made to inspect the President's back after he had expired? Answer: No, sir.|Question: Was any effort made to inspect the President's back after he had expired? Answer: No, sir. +LJ031-0097|Question: And why was no effort made at that time to inspect his back? Answer:|Question: And why was no effort made at that time to inspect his back? Answer: +LJ031-0098|I suppose nobody really had the heart to do it.|I suppose nobody really had the heart to do it. +LJ031-0099|Moreover, the Parkland doctors took no further action after the President had expired because they concluded that it was beyond the scope of their permissible duties.|Moreover, the Parkland doctors took no further action after the President had expired because they concluded that it was beyond the scope of their permissible duties. +LJ031-0100|Treatment of Governor Connally|Treatment of Governor Connally +LJ031-0101|While one medical team tried to revive President Kennedy, a second performed a series of operations on the bullet wounds sustained by Governor Connally.|While one medical team tried to revive President Kennedy, a second performed a series of operations on the bullet wounds sustained by Governor Connally. +LJ031-0102|Governor Connally was originally seen by Dr. Carrico and Dr. Richard Dulany.|Governor Connally was originally seen by Dr. Carrico and Dr. Richard Dulany. +LJ031-0103|While Dr. Carrico went on to attend the President, Dr. Dulany stayed with the Governor and was soon joined by several other doctors.|While Dr. Carrico went on to attend the President, Dr. Dulany stayed with the Governor and was soon joined by several other doctors. +LJ031-0104|At approximately 12:45 p.m., Dr. Robert Shaw,|At approximately twelve:forty-five p.m., Dr. Robert Shaw, +LJ031-0105|chief of thoracic surgery, arrived at trauma room 2, to take charge of the care of Governor Connally,|chief of thoracic surgery, arrived at trauma room two, to take charge of the care of Governor Connally, +LJ031-0106|whose major wound fell within Dr. Shaw's area of specialization.|whose major wound fell within Dr. Shaw's area of specialization. +LJ031-0107|Governor Connally had a large sucking wound in the front of the right chest which caused extreme pain and difficulty in breathing.|Governor Connally had a large sucking wound in the front of the right chest which caused extreme pain and difficulty in breathing. +LJ031-0108|Rubber tubes were inserted between the second and third ribs to reexpand the right lung, which had collapsed because of the opening in the chest wall.|Rubber tubes were inserted between the second and third ribs to reexpand the right lung, which had collapsed because of the opening in the chest wall. +LJ031-0109|At 1:35 p.m., after Governor Connally had been moved to the operating room, Dr. Shaw started the first operation|At one:thirty-five p.m., after Governor Connally had been moved to the operating room, Dr. Shaw started the first operation +LJ031-0110|by cutting away the edges of the wound on the front of the Governor's chest and suturing the damaged lung and lacerated muscles.|by cutting away the edges of the wound on the front of the Governor's chest and suturing the damaged lung and lacerated muscles. +LJ031-0111|The elliptical wound in the Governor's back, located slightly to the left of the Governor's right armpit approximately five-eighths inch (a centimeter and a half)|The elliptical wound in the Governor's back, located slightly to the left of the Governor's right armpit approximately five-eighths inch (a centimeter and a half) +LJ031-0112|in its greatest diameter, was treated by cutting away the damaged skin and suturing the back muscle and skin.|in its greatest diameter, was treated by cutting away the damaged skin and suturing the back muscle and skin. +LJ031-0113|This operation was concluded at 3:20 p.m.|This operation was concluded at three:twenty p.m. +LJ031-0114|Two additional operations were performed on Governor Connally for wounds which he had not realized he had sustained until he regained consciousness the following day.|Two additional operations were performed on Governor Connally for wounds which he had not realized he had sustained until he regained consciousness the following day. +LJ031-0115|From approximately 4 p.m. to 4:50 p.m. on November 22,|From approximately four p.m. to four:fifty p.m. on November twenty-two, +LJ031-0116|Dr. Charles F. Gregory, chief of orthopedic surgery, operated on the wounds of Governor Connally's right wrist,|Dr. Charles F. Gregory, chief of orthopedic surgery, operated on the wounds of Governor Connally's right wrist, +LJ031-0117|assisted by Drs. William Osborne and John Parker.|assisted by Drs. William Osborne and John Parker. +LJ031-0118|The wound on the back of the wrist was left partially open for draining, and the wound on the palm side was enlarged, cleansed, and closed.|The wound on the back of the wrist was left partially open for draining, and the wound on the palm side was enlarged, cleansed, and closed. +LJ031-0119|The fracture was set, and a cast was applied with some traction utilized.|The fracture was set, and a cast was applied with some traction utilized. +LJ031-0120|While the second operation was in progress,|While the second operation was in progress, +LJ031-0121|Dr. George T. Shires, assisted by Drs. Robert McClelland, Charles Baxter, and Ralph Don Patman,|Dr. George T. Shires, assisted by Drs. Robert McClelland, Charles Baxter, and Ralph Don Patman, +LJ031-0122|treated the gunshot wound in the left thigh.|treated the gunshot wound in the left thigh. +LJ031-0123|This punctuate missile wound, about two-fifths inch in diameter (1 centimeter) and located approximately 5 inches above the left knee,|This punctuate missile wound, about two-fifths inch in diameter (one centimeter) and located approximately five inches above the left knee, +LJ031-0124|was cleansed and closed with sutures; but a small metallic fragment remained in the Governor's leg.|was cleansed and closed with sutures; but a small metallic fragment remained in the Governor's leg. +LJ031-0125|Vice President Johnson at Parkland|Vice President Johnson at Parkland +LJ031-0126|As President Kennedy and Governor Connally were being removed from the limousine onto stretchers,|As President Kennedy and Governor Connally were being removed from the limousine onto stretchers, +LJ031-0127|a protective circle of Secret Service agents surrounded Vice President and Mrs. Johnson|a protective circle of Secret Service agents surrounded Vice President and Mrs. Johnson +LJ031-0128|and escorted them into Parkland Hospital through the emergency entrance.|and escorted them into Parkland Hospital through the emergency entrance. +LJ031-0129|The agents moved a nurse and patient out of a nearby room, lowered the shades, and took emergency security measures to protect the Vice President.|The agents moved a nurse and patient out of a nearby room, lowered the shades, and took emergency security measures to protect the Vice President. +LJ031-0130|Two men from the President's follow-up car were detailed to help protect the Vice President.|Two men from the President's follow-up car were detailed to help protect the Vice President. +LJ031-0131|An agent was stationed at the entrance to stop anyone who was not a member of the Presidential party.|An agent was stationed at the entrance to stop anyone who was not a member of the Presidential party. +LJ031-0132|U.S. Representatives Henry B. Gonzalez,|U.S. Representatives Henry B. Gonzalez, +LJ031-0133|Jack Brooks, Homer Thornberry, and Albert Thomas joined Clifton C. Carter and the group of special agents protecting the Vice President.|Jack Brooks, Homer Thornberry, and Albert Thomas joined Clifton C. Carter and the group of special agents protecting the Vice President. +LJ031-0134|On one occasion Mrs. Johnson, accompanied by two Secret Service agents, left the room to see Mrs. Kennedy and Mrs. Connally.|On one occasion Mrs. Johnson, accompanied by two Secret Service agents, left the room to see Mrs. Kennedy and Mrs. Connally. +LJ031-0135|Concern that the Vice President might also be a target for assassination prompted the Secret Service agents to urge him to leave the hospital and return to Washington immediately.|Concern that the Vice President might also be a target for assassination prompted the Secret Service agents to urge him to leave the hospital and return to Washington immediately. +LJ031-0136|The Vice President decided to wait until he received definitive word of the President's condition.|The Vice President decided to wait until he received definitive word of the President's condition. +LJ031-0137|At approximately 1:20 p.m., Vice President Johnson was notified by O'Donnell that President Kennedy was dead.|At approximately one:twenty p.m., Vice President Johnson was notified by O'Donnell that President Kennedy was dead. +LJ031-0138|Special Agent Youngblood learned from Mrs. Johnson the location of her two daughters|Special Agent Youngblood learned from Mrs. Johnson the location of her two daughters +LJ031-0139|and made arrangements through Secret Service headquarters in Washington to provide them with protection immediately.|and made arrangements through Secret Service headquarters in Washington to provide them with protection immediately. +LJ031-0140|When consulted by the Vice President, O'Donnell advised him to go to the airfield immediately and return to Washington.|When consulted by the Vice President, O'Donnell advised him to go to the airfield immediately and return to Washington. +LJ031-0141|It was decided that the Vice President should return on the Presidential plane rather than on the Vice-Presidential plane because it had better communication equipment.|It was decided that the Vice President should return on the Presidential plane rather than on the Vice-Presidential plane because it had better communication equipment. +LJ031-0142|The Vice President conferred with White House Assistant Press Secretary Malcolm Kilduff|The Vice President conferred with White House Assistant Press Secretary Malcolm Kilduff +LJ031-0143|and decided that there would be no release of the news of the President's death until the Vice President had left the hospital.|and decided that there would be no release of the news of the President's death until the Vice President had left the hospital. +LJ031-0144|When told that Mrs. Kennedy refused to leave without the President's body, the Vice President said that he would not leave Dallas without her.|When told that Mrs. Kennedy refused to leave without the President's body, the Vice President said that he would not leave Dallas without her. +LJ031-0145|On the recommendation of the Secret Service agents, Vice President Johnson decided to board the Presidential airplane, Air Force One,|On the recommendation of the Secret Service agents, Vice President Johnson decided to board the Presidential airplane, Air Force One, +LJ031-0146|and wait for Mrs. Kennedy and the President's body.|and wait for Mrs. Kennedy and the President's body. +LJ031-0147|Secret Service Emergency Security Arrangements|Secret Service Emergency Security Arrangements +LJ031-0148|Immediately after President Kennedy's stretcher was wheeled into trauma room 1, Secret Service agents took positions at the door of the small emergency room.|Immediately after President Kennedy's stretcher was wheeled into trauma room one, Secret Service agents took positions at the door of the small emergency room. +LJ031-0149|A nurse was asked to identify hospital personnel and to tell everyone, except necessary medical staff members, to leave the emergency room.|A nurse was asked to identify hospital personnel and to tell everyone, except necessary medical staff members, to leave the emergency room. +LJ031-0150|Other Secret Service agents posted themselves in the corridors and other areas near the emergency room.|Other Secret Service agents posted themselves in the corridors and other areas near the emergency room. +LJ031-0151|Special Agent Lawson made certain that the Dallas police kept the public and press away from the immediate area of the hospital.|Special Agent Lawson made certain that the Dallas police kept the public and press away from the immediate area of the hospital. +LJ031-0152|Agents Kellerman and Hill telephoned the head of the White House detail, Gerald A. Behn, to advise him of the assassination.|Agents Kellerman and Hill telephoned the head of the White House detail, Gerald A. Behn, to advise him of the assassination. +LJ031-0153|The telephone line to Washington was kept open throughout the remainder of the stay at the hospital.|The telephone line to Washington was kept open throughout the remainder of the stay at the hospital. +LJ031-0154|Secret Service agents stationed at later stops on the President's itinerary of November 22 were redeployed.|Secret Service agents stationed at later stops on the President's itinerary of November twenty-two were redeployed. +LJ031-0155|Men at the Trade Mart were driven to Parkland Hospital in Dallas police cars.|Men at the Trade Mart were driven to Parkland Hospital in Dallas police cars. +LJ031-0156|The Secret Service group awaiting the President in Austin were instructed to return to Washington.|The Secret Service group awaiting the President in Austin were instructed to return to Washington. +LJ031-0157|Meanwhile, the Secret Service agents in charge of security at Love Field started to make arrangements for departure.|Meanwhile, the Secret Service agents in charge of security at Love Field started to make arrangements for departure. +LJ031-0158|As soon as one of the agents learned of the shooting, he asked the officer in charge of the police detail at the airport|As soon as one of the agents learned of the shooting, he asked the officer in charge of the police detail at the airport +LJ031-0159|to institute strict security measures for the Presidential aircraft, the airport terminal, and the surrounding area.|to institute strict security measures for the Presidential aircraft, the airport terminal, and the surrounding area. +LJ031-0160|The police were cautioned to prevent picture taking.|The police were cautioned to prevent picture taking. +LJ031-0161|Secret Service agents working with police cleared the areas adjacent to the aircraft, including warehouses,|Secret Service agents working with police cleared the areas adjacent to the aircraft, including warehouses, +LJ031-0162|other terminal buildings and the neighboring parking lots, of all people.|other terminal buildings and the neighboring parking lots, of all people. +LJ031-0163|The agents decided not to shift the Presidential aircraft to the far side of the airport because the original landing area was secure|The agents decided not to shift the Presidential aircraft to the far side of the airport because the original landing area was secure +LJ031-0164|and a move would require new measures.|and a move would require new measures. +LJ031-0165|When security arrangements at the airport were complete, the Secret Service made the necessary arrangements for the Vice President to leave the hospital.|When security arrangements at the airport were complete, the Secret Service made the necessary arrangements for the Vice President to leave the hospital. +LJ031-0166|Unmarked police cars took the Vice President and Mrs. Johnson from Parkland Hospital to Love Field.|Unmarked police cars took the Vice President and Mrs. Johnson from Parkland Hospital to Love Field. +LJ031-0167|Chief Curry drove one automobile occupied by Vice President Johnson, U.S. Representatives Thomas and Thornberry, and Special Agent Youngblood.|Chief Curry drove one automobile occupied by Vice President Johnson, U.S. Representatives Thomas and Thornberry, and Special Agent Youngblood. +LJ031-0168|In another car Mrs. Johnson was driven to the airport accompanied by Secret Service agents and Representative Brooks.|In another car Mrs. Johnson was driven to the airport accompanied by Secret Service agents and Representative Brooks. +LJ031-0169|Motorcade policemen who escorted the automobiles were requested by the Vice President and Agent Youngblood not to use sirens.|Motorcade policemen who escorted the automobiles were requested by the Vice President and Agent Youngblood not to use sirens. +LJ031-0170|During the drive Vice President Johnson, at Youngblood's instruction, kept below window level.|During the drive Vice President Johnson, at Youngblood's instruction, kept below window level. +LJ031-0171|Removal of the President's Body|Removal of the President's Body +LJ031-0172|While the team of doctors at Parkland Hospital tried desperately to save the life of President Kennedy,|While the team of doctors at Parkland Hospital tried desperately to save the life of President Kennedy, +LJ031-0173|Mrs. Kennedy alternated between watching them and waiting outside.|Mrs. Kennedy alternated between watching them and waiting outside. +LJ031-0174|After the President was pronounced dead,|After the President was pronounced dead, +LJ031-0175|O'Donnell tried to persuade Mrs. Kennedy to leave the area, but she refused.|O'Donnell tried to persuade Mrs. Kennedy to leave the area, but she refused. +LJ031-0176|A casket was obtained and the President's body was prepared for removal.|A casket was obtained and the President's body was prepared for removal. +LJ031-0177|Before the body could be taken from the hospital,|Before the body could be taken from the hospital, +LJ031-0178|two Dallas officials informed members of the President's start that the body could not be removed from the city until an autopsy was performed.|two Dallas officials informed members of the President's start that the body could not be removed from the city until an autopsy was performed. +LJ031-0179|Despite the protests of these officials, the casket was wheeled out of the hospital, placed in an ambulance, and transported to the airport shortly after 2 p.m.|Despite the protests of these officials, the casket was wheeled out of the hospital, placed in an ambulance, and transported to the airport shortly after two p.m. +LJ031-0180|At approximately 2:15 p.m. the casket was loaded, with some difficulty because of the narrow airplane door,|At approximately two:fifteen p.m. the casket was loaded, with some difficulty because of the narrow airplane door, +LJ031-0181|onto the rear of the Presidential plane where seats had been removed to make room.|onto the rear of the Presidential plane where seats had been removed to make room. +LJ031-0182|Concerned that the local officials might try to prevent the plane's departure, O'Donnell asked that the pilot take off immediately.|Concerned that the local officials might try to prevent the plane's departure, O'Donnell asked that the pilot take off immediately. +LJ031-0183|He was informed that takeoff would be delayed until Vice President Johnson was sworn in.|He was informed that takeoff would be delayed until Vice President Johnson was sworn in. +LJ031-0184|Swearing in of the New President|Swearing in of the New President +LJ031-0185|From the Presidential airplane, the Vice President telephoned Attorney General Robert F. Kennedy,|From the Presidential airplane, the Vice President telephoned Attorney General Robert F. Kennedy, +LJ031-0186|who advised that Mr. Johnson take the Presidential oath of office before the plane left Dallas.|who advised that Mr. Johnson take the Presidential oath of office before the plane left Dallas. +LJ031-0187|Federal Judge Sarah T. Hughes hastened to the plane to administer the oath.|Federal Judge Sarah T. Hughes hastened to the plane to administer the oath. +LJ031-0188|Members of the Presidential and Vice-Presidential parties filled the central compartment of the plane to witness the swearing in.|Members of the Presidential and Vice-Presidential parties filled the central compartment of the plane to witness the swearing in. +LJ031-0189|At 2:38 p.m., Eastern Standard Time, Lyndon Baines Johnson took the oath of office as the 36th President of the United States.|At two:thirty-eight p.m., Eastern Standard Time, Lyndon Baines Johnson took the oath of office as the thirty-sixth President of the United States. +LJ031-0190|Mrs. Kennedy and Mrs. Johnson stood at the side of the new President as he took the oath of office.|Mrs. Kennedy and Mrs. Johnson stood at the side of the new President as he took the oath of office. +LJ031-0191|Nine minutes later, the Presidential airplane departed for Washington, D.C.|Nine minutes later, the Presidential airplane departed for Washington, D.C. +LJ031-0192|Return to Washington, D.C.|Return to Washington, D.C. +LJ031-0193|On the return flight, Mrs. Kennedy sat with David Powers, Kenneth O'Donnell, and Lawrence O'Brien.|On the return flight, Mrs. Kennedy sat with David Powers, Kenneth O'Donnell, and Lawrence O'Brien. +LJ031-0194|At 5:58 p.m. Eastern Standard Time,|At five:fifty-eight p.m. Eastern Standard Time, +LJ031-0195|Air Force One landed at Andrews Air Force Base, where President Kennedy had begun his last trip only 31 hours before.|Air Force One landed at Andrews Air Force Base, where President Kennedy had begun his last trip only thirty-one hours before. +LJ031-0196|Detailed security arrangements had been made by radio from the President's plane on the return flight.|Detailed security arrangements had been made by radio from the President's plane on the return flight. +LJ031-0197|The public had been excluded from the base, and only Government officials and the press were permitted near the landing area.|The public had been excluded from the base, and only Government officials and the press were permitted near the landing area. +LJ031-0198|Upon arrival, President Johnson made a brief statement over television and radio.|Upon arrival, President Johnson made a brief statement over television and radio. +LJ031-0199|President and Mrs. Johnson were flown by helicopter to the White House, from where Mrs. Johnson was driven to her residence under Secret Service escort.|President and Mrs. Johnson were flown by helicopter to the White House, from where Mrs. Johnson was driven to her residence under Secret Service escort. +LJ031-0200|The President then walked to the Executive Office Building, where he worked until 9 p.m.|The President then walked to the Executive Office Building, where he worked until nine p.m. +LJ031-0201|Given a choice between the National Naval Medical Center at Bethesda, Maryland, and the Army's Walter Reed Hospital,|Given a choice between the National Naval Medical Center at Bethesda, Maryland, and the Army's Walter Reed Hospital, +LJ031-0202|Mrs. Kennedy chose the hospital in Bethesda for the autopsy because the President had served in the Navy.|Mrs. Kennedy chose the hospital in Bethesda for the autopsy because the President had served in the Navy. +LJ031-0203|Mrs. Kennedy and the Attorney General,|Mrs. Kennedy and the Attorney General, +LJ031-0204|with three Secret Service agents, accompanied President Kennedy's body on the 45-minute automobile trip from Andrews Air Force Base to the hospital.|with three Secret Service agents, accompanied President Kennedy's body on the forty-five-minute automobile trip from Andrews Air Force Base to the hospital. +LJ031-0205|On the 17th floor of the Hospital, Mrs. Kennedy and the Attorney General joined other members of the Kennedy family to await the conclusion of the autopsy.|On the seventeenth floor of the Hospital, Mrs. Kennedy and the Attorney General joined other members of the Kennedy family to await the conclusion of the autopsy. +LJ031-0206|Mrs. Kennedy was guarded by Secret Service agents in quarters assigned to her in the naval hospital.|Mrs. Kennedy was guarded by Secret Service agents in quarters assigned to her in the naval hospital. +LJ031-0207|The Secret Service established a communication system with the White House and screened all telephone calls and visitors.|The Secret Service established a communication system with the White House and screened all telephone calls and visitors. +LJ031-0208|The hospital received the President's body for autopsy at approximately 7:35 p.m.|The hospital received the President's body for autopsy at approximately seven:thirty-five p.m. +LJ031-0209|X-rays and photographs were taken preliminarily and the pathological examination began at about 8 p.m.|X-rays and photographs were taken preliminarily and the pathological examination began at about eight p.m. +LJ031-0210|The autopsy report noted that President Kennedy was 46 years of age, 72 and one half inches tall,|The autopsy report noted that President Kennedy was forty-six years of age, seventy-two and one half inches tall, +LJ031-0211|weighed 170 pounds, had blue eyes and reddish-brown hair.|weighed one hundred seventy pounds, had blue eyes and reddish-brown hair. +LJ031-0212|The body was muscular and well developed with no gross skeletal abnormalities except for those caused by the gunshot wounds.|The body was muscular and well developed with no gross skeletal abnormalities except for those caused by the gunshot wounds. +LJ031-0213|Under "Pathological Diagnosis" the cause of death was set forth as "Gunshot wound, head."|Under "Pathological Diagnosis" the cause of death was set forth as "Gunshot wound, head." +LJ031-0214|The autopsy examination revealed two wounds in the President's head.|The autopsy examination revealed two wounds in the President's head. +LJ031-0215|One wound, approximately one-fourth of an inch by five-eighths of an inch (6 by 15 millimeters),|One wound, approximately one-fourth of an inch by five-eighths of an inch (six by fifteen millimeters), +LJ031-0216|was located about an inch (2.5 centimeters) to the right and slightly above the large bony protrusion|was located about an inch (two point five centimeters) to the right and slightly above the large bony protrusion +LJ031-0217|(external occipital protuberance) which juts out at the center of the lower part of the back of the skull.|(external occipital protuberance) which juts out at the center of the lower part of the back of the skull. +LJ031-0218|The second head wound measured approximately 5 inches (13 centimeters) in its greatest diameter,|The second head wound measured approximately five inches (thirteen centimeters) in its greatest diameter, +LJ031-0219|but it was difficult to measure accurately because multiple crisscross fractures radiated from the large defect.|but it was difficult to measure accurately because multiple crisscross fractures radiated from the large defect. +LJ031-0220|During the autopsy examination, Federal agents brought the surgeons three pieces of bone recovered from Elm Street and the Presidential automobile.|During the autopsy examination, Federal agents brought the surgeons three pieces of bone recovered from Elm Street and the Presidential automobile. +LJ031-0221|When put together, these fragments accounted for approximately three-quarters of the missing portion of the skull.|When put together, these fragments accounted for approximately three-quarters of the missing portion of the skull. +LJ031-0222|The surgeons observed, through X-ray analysis, 30 or 40 tiny dustlike fragments of metal|The surgeons observed, through X-ray analysis, thirty or forty tiny dustlike fragments of metal +LJ031-0223|running in a line from the wound in the rear of the President's head toward the front part of the skull,|running in a line from the wound in the rear of the President's head toward the front part of the skull, +LJ031-0224|with a sizable metal fragment lying just above the right eye.|with a sizable metal fragment lying just above the right eye. +LJ031-0225|From this head wound two small irregularly shaped fragments of metal were recovered and turned over to the FBI.|From this head wound two small irregularly shaped fragments of metal were recovered and turned over to the FBI. +LJ031-0226|The autopsy also disclosed a wound near the base of the back of President Kennedy's neck slightly to the right of his spine.|The autopsy also disclosed a wound near the base of the back of President Kennedy's neck slightly to the right of his spine. +LJ031-0228|concluded that the bullet had emerged from the front portion of the President's neck that had been cut away by the tracheotomy at Parkland.|concluded that the bullet had emerged from the front portion of the President's neck that had been cut away by the tracheotomy at Parkland. +LJ031-0229|The nature and characteristics of this neck wound are discussed fully in the next chapter.|The nature and characteristics of this neck wound are discussed fully in the next chapter. +LJ031-0230|After the autopsy was concluded at approximately 11 p.m., the President's body was prepared for burial.|After the autopsy was concluded at approximately eleven p.m., the President's body was prepared for burial. +LJ031-0231|This was finished at approximately 4 a.m.|This was finished at approximately four a.m. +LJ031-0232|Shortly thereafter, the President's wife, family and aides left Bethesda Naval Hospital.|Shortly thereafter, the President's wife, family and aides left Bethesda Naval Hospital. +LJ031-0233|The President's body was taken to the East Room of the White House where it was placed under ceremonial military guard.|The President's body was taken to the East Room of the White House where it was placed under ceremonial military guard. +LJ032-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ032-0002|Chapter 4. The Assassin: Part 1.|Chapter four. The Assassin: Part one. +LJ032-0003|The preceding chapter has established that the bullets which killed President Kennedy and wounded Governor Connally|The preceding chapter has established that the bullets which killed President Kennedy and wounded Governor Connally +LJ032-0004|were fired from the southeast corner window of the sixth floor of the Texas School Book Depository Building|were fired from the southeast corner window of the sixth floor of the Texas School Book Depository Building +LJ032-0005|and that the weapon which fired these bullets was a Mannlicher-Carcano|and that the weapon which fired these bullets was a Mannlicher-Carcano +LJ032-0006|6.5-millimeter Italian rifle bearing the serial number C2766.|six point five-millimeter Italian rifle bearing the serial number C two seven six six. +LJ032-0007|In this chapter the Commission evaluates the evidence upon which it has based its conclusion concerning the identity of the assassin.|In this chapter the Commission evaluates the evidence upon which it has based its conclusion concerning the identity of the assassin. +LJ032-0008|This evidence includes (1) the ownership and possession of the weapon used to commit the assassination,|This evidence includes (one) the ownership and possession of the weapon used to commit the assassination, +LJ032-0009|(2) the means by which the weapon was brought into the Depository Building,|(two) the means by which the weapon was brought into the Depository Building, +LJ032-0010|(3) the identity of the person present at the window from which the shots were fired,|(three) the identity of the person present at the window from which the shots were fired, +LJ032-0011|(4) the killing of Dallas Patrolman J. D. Tippit within 45 minutes after the assassination, (5)|(four) the killing of Dallas Patrolman J. D. Tippit within forty-five minutes after the assassination, (five) +LJ032-0012|the resistance to arrest and the attempted shooting of another police officer by the man (Lee Harvey Oswald) subsequently accused of assassinating President Kennedy|the resistance to arrest and the attempted shooting of another police officer by the man (Lee Harvey Oswald) subsequently accused of assassinating President Kennedy +LJ032-0013|and killing Patrolman Tippit,|and killing Patrolman Tippit, +LJ032-0014|(6) the lies told to the police by Oswald, (7)|(six) the lies told to the police by Oswald, (seven) +LJ032-0015|the evidence linking Oswald to the attempted killing of Maj. Gen. Edwin A. Walker (Resigned, U.S. Army) on April 10, 1963,|the evidence linking Oswald to the attempted killing of Maj. Gen. Edwin A. Walker (Resigned, U.S. Army) on April ten, nineteen sixty-three, +LJ032-0016|and (8) Oswald's capability with a rifle.|and (eight) Oswald's capability with a rifle. +LJ032-0017|Ownership And Possession Of Assassination Weapon|Ownership And Possession Of Assassination Weapon +LJ032-0018|Purchase of Rifle by Oswald|Purchase of Rifle by Oswald +LJ032-0019|Shortly after the Mannlicher-Carcano rifle was found on the sixth floor of the Texas School Book Depository Building, agents of the FBI|Shortly after the Mannlicher-Carcano rifle was found on the sixth floor of the Texas School Book Depository Building, agents of the FBI +LJ032-0020|learned from retail outlets in Dallas that Crescent Firearms, Inc., of New York City,|learned from retail outlets in Dallas that Crescent Firearms, Inc., of New York City, +LJ032-0021|was a distributor of surplus Italian 6.5-millimeter military rifles. During the evening of November 22, 1963,|was a distributor of surplus Italian six point five-millimeter military rifles. During the evening of November twenty-two, nineteen sixty-three, +LJ032-0022|a review of the records of Crescent Firearms revealed that the firm had shipped an Italian carbine, serial number C2766,|a review of the records of Crescent Firearms revealed that the firm had shipped an Italian carbine, serial number C two seven six six, +LJ032-0023|to Klein's Sporting Goods Co., of Chicago, Illinois.|to Klein's Sporting Goods Co., of Chicago, Illinois. +LJ032-0024|After searching their records from 10 p.m. to 4 a.m.|After searching their records from ten p.m. to four a.m. +LJ032-0025|the officers of Klein's discovered that a rifle bearing serial number C2766 had been shipped to one A. Hidell,|the officers of Klein's discovered that a rifle bearing serial number C two seven six six had been shipped to one A. Hidell, +LJ032-0026|Post Office Box 2915, Dallas, Texas, on March 20, 1963.|Post Office Box two nine one five, Dallas, Texas, on March twenty, nineteen sixty-three. +LJ032-0027|According to its microfilm records, Klein's received an order for a rifle on March 13, 1963,|According to its microfilm records, Klein's received an order for a rifle on March thirteen, nineteen sixty-three, +LJ032-0028|on a coupon clipped from the February 1963 issue of the American Rifleman magazine.|on a coupon clipped from the February nineteen sixty-three issue of the American Rifleman magazine. +LJ032-0029|The order coupon was signed, in handprinting, "A. Hidell, P.O. Box 2915, Dallas, Texas."|The order coupon was signed, in handprinting, "A. Hidell, P.O. Box two nine one five, Dallas, Texas." +LJ032-0030|It was sent in an envelope bearing the same name and return address in handwriting.|It was sent in an envelope bearing the same name and return address in handwriting. +LJ032-0031|Document examiners for the Treasury Department and the FBI testified unequivocally|Document examiners for the Treasury Department and the FBI testified unequivocally +LJ032-0032|that the bold printing on the face of the mail-order coupon was in the handprinting of Lee Harvey Oswald and that the writing on the envelope was also his.|that the bold printing on the face of the mail-order coupon was in the handprinting of Lee Harvey Oswald and that the writing on the envelope was also his. +LJ032-0033|Oswald's writing on these and other documents was identified by comparing the writing and printing on the documents in question|Oswald's writing on these and other documents was identified by comparing the writing and printing on the documents in question +LJ032-0034|with that appearing on documents known to have been written by Oswald, such as his letters, passport application, and endorsements of checks.|with that appearing on documents known to have been written by Oswald, such as his letters, passport application, and endorsements of checks. +LJ032-0035|In addition to the order coupon the envelope contained a. U.S. postal money order for $21.45,|In addition to the order coupon the envelope contained a. U.S. postal money order for twenty-one dollars, forty-five cents, +LJ032-0036|purchased as No. 2,202,130,462 in Dallas, Texas, on March 12, 1963.|purchased as Number two two zero two one three zero four six two in Dallas, Texas, on March twelve, nineteen sixty-three. +LJ032-0037|The canceled money order was obtained from the Post Office Department.|The canceled money order was obtained from the Post Office Department. +LJ032-0038|Opposite the printed words "Pay To" were written the words "Kleins Sporting Goods,"|Opposite the printed words "Pay To" were written the words "Kleins Sporting Goods," +LJ032-0039|and opposite the printed word "From"|and opposite the printed word "From" +LJ032-0040|were written the words "A. Hidell, P.O. Box 2915 Dallas, Texas."|were written the words "A. Hidell, P.O. Box two nine one five Dallas, Texas." +LJ032-0041|These words were also in the handwriting of Lee Harvey Oswald.|These words were also in the handwriting of Lee Harvey Oswald. +LJ032-0042|From Klein's records it was possible to trace the processing of the order after its receipt.|From Klein's records it was possible to trace the processing of the order after its receipt. +LJ032-0043|A bank deposit made on March 13, 1963, included an item of $21.45. Klein's shipping order form|A bank deposit made on March thirteen, nineteen sixty-three, included an item of twenty-one dollars, forty-five cents. Klein's shipping order form +LJ032-0044|shows an imprint made by the cash register which recorded the receipt of $21.45 on March 13, 1963.|shows an imprint made by the cash register which recorded the receipt of twenty-one dollars, forty-five cents on March thirteen, nineteen sixty-three. +LJ032-0045|This price included $19.95 for the rifle and the scope, and $1.50 for postage and handling.|This price included nineteen dollars, ninety-five cents for the rifle and the scope, and one dollar, fifty cents for postage and handling. +LJ032-0046|The rifle without the scope cost only $12.78.|The rifle without the scope cost only twelve dollars, seventy-eight cents. +LJ032-0047|According to the vice president of Klein's, William Waldman,|According to the vice president of Klein's, William Waldman, +LJ032-0048|the scope was mounted on the rifle by a gunsmith employed by Klein's, and the rifle was shipped fully assembled in accordance with customary company procedures.|the scope was mounted on the rifle by a gunsmith employed by Klein's, and the rifle was shipped fully assembled in accordance with customary company procedures. +LJ032-0049|The specific rifle shipped against the order had been received by Klein's from Crescent on February 21, 1963.|The specific rifle shipped against the order had been received by Klein's from Crescent on February twenty-one, nineteen sixty-three. +LJ032-0050|It bore the manufacturer's serial number C2766.|It bore the manufacturer's serial number C two seven six six. +LJ032-0051|On that date, Klein's placed an internal control number VC836 on this rifle.|On that date, Klein's placed an internal control number V C eight three six on this rifle. +LJ032-0052|According to Klein's shipping order form, one Italian carbine 6.5 x 4 x scope,|According to Klein's shipping order form, one Italian carbine six point five x four x scope, +LJ032-0053|control number VC836, serial number C2766, was shipped parcel post to|control number V C eight three six, serial number C two seven six six, was shipped parcel post to +LJ032-0054|"A. Hidell, P.O. Box 2915, Dallas, Texas," on March 20, 1963.|"A. Hidell, P.O. Box two nine one five, Dallas, Texas," on March twenty, nineteen sixty-three. +LJ032-0055|Information received from the Italian Armed Forces Intelligence Service has established that this particular rifle was the only rifle of its type|Information received from the Italian Armed Forces Intelligence Service has established that this particular rifle was the only rifle of its type +LJ032-0056|bearing serial number C2766.|bearing serial number C two seven six six. +LJ032-0057|The post office box to which the rifle was shipped was rented to "Lee H. Oswald" from October 9, 1962,|The post office box to which the rifle was shipped was rented to "Lee H. Oswald" from October nine, nineteen sixty-two, +LJ032-0058|to May 14, 1963.|to May fourteen, nineteen sixty-three. +LJ032-0059|Experts on handwriting identification from the Treasury Department and the FBI|Experts on handwriting identification from the Treasury Department and the FBI +LJ032-0060|testified that the signature and other writing on the application for that box were in the handwriting of Lee Harvey Oswald|testified that the signature and other writing on the application for that box were in the handwriting of Lee Harvey Oswald +LJ032-0061|as was a change-of-address card dated May 12, 1963,|as was a change-of-address card dated May twelve, nineteen sixty-three, +LJ032-0062|by which Oswald requested that mail addressed to that box be forwarded to him in New Orleans, where he had moved on April 24.|by which Oswald requested that mail addressed to that box be forwarded to him in New Orleans, where he had moved on April twenty-four. +LJ032-0063|Since the rifle was shipped from Chicago on March 20, 1963, it was received in Dallas during the period when Oswald rented and used the box.|Since the rifle was shipped from Chicago on March twenty, nineteen sixty-three, it was received in Dallas during the period when Oswald rented and used the box. +LJ032-0064|It is not known whether the application for post office box 2915 listed "A. Hidell" as a person entitled to receive mail at this box.|It is not known whether the application for post office box two nine one five listed "A. Hidell" as a person entitled to receive mail at this box. +LJ032-0065|In accordance with postal regulations, the portion of the application which lists names of persons, other than the applicant, entitled to receive mail|In accordance with postal regulations, the portion of the application which lists names of persons, other than the applicant, entitled to receive mail +LJ032-0066|was thrown away after the box was closed in May 1963.|was thrown away after the box was closed in May nineteen sixty-three. +LJ032-0067|Postal Inspector Harry D. Holmes of the Dallas Post Office testified, however, that when a package is received for a certain box,|Postal Inspector Harry D. Holmes of the Dallas Post Office testified, however, that when a package is received for a certain box, +LJ032-0068|a notice is placed in that box regardless of whether the name on the package is listed on the application as a person entitled to receive mail through that box.|a notice is placed in that box regardless of whether the name on the package is listed on the application as a person entitled to receive mail through that box. +LJ032-0069|The person having access to the box then takes the notice to the window and is given the package.|The person having access to the box then takes the notice to the window and is given the package. +LJ032-0070|Ordinarily, Inspector Holmes testified, identification is not requested because it is assumed that the person with the notice is entitled to the package.|Ordinarily, Inspector Holmes testified, identification is not requested because it is assumed that the person with the notice is entitled to the package. +LJ032-0071|Oswald's use of the name "Hidell" to purchase the assassination weapon was one of several instances in which he used this name as an alias.|Oswald's use of the name "Hidell" to purchase the assassination weapon was one of several instances in which he used this name as an alias. +LJ032-0072|When arrested on the day of the assassination, he had in his possession a Smith & Wesson 38 caliber revolver|When arrested on the day of the assassination, he had in his possession a Smith and Wesson thirty-eight caliber revolver +LJ032-0073|purchased by mail-order coupon from Seaport-Traders, Inc., a mail-order division of George Rose & Co., Los Angeles.|purchased by mail-order coupon from Seaport-Traders, Inc., a mail-order division of George Rose and Co., Los Angeles. +LJ032-0074|The mail-order coupon listed the purchaser as "A. J. Hidell Age 28"|The mail-order coupon listed the purchaser as "A. J. Hidell Age twenty-eight" +LJ032-0075|with the address of post office box 2915 in Dallas.|with the address of post office box two nine one five in Dallas. +LJ032-0076|Handwriting experts from the FBI and the Treasury Department testified that the writing on the mail-order form was that of Lee Harvey Oswald.|Handwriting experts from the FBI and the Treasury Department testified that the writing on the mail-order form was that of Lee Harvey Oswald. +LJ032-0077|Among other identification cards in Oswald's wallet at the time of his arrest were a Selective Service notice of classification,|Among other identification cards in Oswald's wallet at the time of his arrest were a Selective Service notice of classification, +LJ032-0078|a Selective Service registration certificate, and a certificate of service in the U.S. Marine Corps, all three cards being in his own name.|a Selective Service registration certificate, and a certificate of service in the U.S. Marine Corps, all three cards being in his own name. +LJ032-0079|Also in his wallet at that time|Also in his wallet at that time +LJ032-0080|were a Selective Service notice of classification and a Marine certificate of service in the name of Alek James Hidell.|were a Selective Service notice of classification and a Marine certificate of service in the name of Alek James Hidell. +LJ032-0081|On the Hidell Selective Service card there appeared a signature, "Alek J. Hidell,"|On the Hidell Selective Service card there appeared a signature, "Alek J. Hidell," +LJ032-0082|and the photograph of Lee Harvey Oswald.|and the photograph of Lee Harvey Oswald. +LJ032-0083|Experts on questioned documents from the Treasury Department and the FBI testified that the Hidell cards were counterfeit photographic reproductions|Experts on questioned documents from the Treasury Department and the FBI testified that the Hidell cards were counterfeit photographic reproductions +LJ032-0084|made by photographing the Oswald cards, retouching the resulting negatives, and producing prints from the retouched negatives.|made by photographing the Oswald cards, retouching the resulting negatives, and producing prints from the retouched negatives. +LJ032-0085|The Hidell signature on the notice of classification was in the handwriting of Oswald.|The Hidell signature on the notice of classification was in the handwriting of Oswald. +LJ032-0086|In Oswald's personal effects found in his room at 1026 North Beckley Avenue in Dallas|In Oswald's personal effects found in his room at ten twenty-six North Beckley Avenue in Dallas +LJ032-0087|was a purported international certificate of vaccination signed by "Dr. A. J. Hideel,|was a purported international certificate of vaccination signed by "Dr. A. J. Hideel, +LJ032-0088|Post Office Box 30016, New Orleans|Post Office Box three zero zero one six, New Orleans +LJ032-0089|It certified that Lee Harvey Oswald had been vaccinated for smallpox on June 8, 1963.|It certified that Lee Harvey Oswald had been vaccinated for smallpox on June eight, nineteen sixty-three. +LJ032-0090|This, too, was a forgery. The signature of "A. J. Hideel" was in the handwriting of Lee Harvey Oswald.|This, too, was a forgery. The signature of "A. J. Hideel" was in the handwriting of Lee Harvey Oswald. +LJ032-0091|There is no "Dr. Hideel" licensed to practice medicine in Louisiana.|There is no "Dr. Hideel" licensed to practice medicine in Louisiana. +LJ032-0092|There is no post office box 30016 in the New Orleans Post Office|There is no post office box three zero zero one six in the New Orleans Post Office +LJ032-0093|but Oswald had rented post office box 30061 in New Orleans on June 3, 1963,|but Oswald had rented post office box three zero zero six one in New Orleans on June three, nineteen sixty-three, +LJ032-0094|listing Marina Oswald and A. J. Hidell|listing Marina Oswald and A. J. Hidell +LJ032-0095|as additional persons entitled to receive mail in the box. The New Orleans postal authorities had not discarded the portion of the application|as additional persons entitled to receive mail in the box. The New Orleans postal authorities had not discarded the portion of the application +LJ032-0096|listing the names of those, other than the owner of the box, entitled to receive mail through the box.|listing the names of those, other than the owner of the box, entitled to receive mail through the box. +LJ032-0097|Expert testimony confirmed that the writing on this application was that of Lee Harvey Oswald.|Expert testimony confirmed that the writing on this application was that of Lee Harvey Oswald. +LJ032-0098|Hidell's name on the post office box application was part of Oswald's use of a nonexistent Hidell|Hidell's name on the post office box application was part of Oswald's use of a nonexistent Hidell +LJ032-0099|to serve as president of the so-called New Orleans Chapter of the Fair Play for Cuba Committee.|to serve as president of the so-called New Orleans Chapter of the Fair Play for Cuba Committee. +LJ032-0101|testified that she first learned of Oswald's use of the fictitious name "Hidell" in connection with his pro-Castro activities in New Orleans.|testified that she first learned of Oswald's use of the fictitious name "Hidell" in connection with his pro-Castro activities in New Orleans. +LJ032-0102|According to her testimony, he compelled her to write the name "Hidell" on membership cards in the space designated for the signature of the "Chapter President."|According to her testimony, he compelled her to write the name "Hidell" on membership cards in the space designated for the signature of the "Chapter President." +LJ032-0103|The name "Hidell" was stamped on some of the "Chapter's" printed literature and on the membership application blanks.|The name "Hidell" was stamped on some of the "Chapter's" printed literature and on the membership application blanks. +LJ032-0104|Marina Oswald testified,|Marina Oswald testified, +LJ032-0105|quote, I knew there was no such organization. And I know Hidell is merely an altered Fidel, and I laughed at such foolishness.|quote, I knew there was no such organization. And I know Hidell is merely an altered Fidel, and I laughed at such foolishness. +LJ032-0106|end quote. Hidell was a fictitious president of an organization of which Oswald was the only member.|end quote. Hidell was a fictitious president of an organization of which Oswald was the only member. +LJ032-0107|When seeking employment in New Orleans,|When seeking employment in New Orleans, +LJ032-0108|Oswald listed a "Sgt. Robert Hidell" as a reference on one job application and "George Hidell" as a reference on another.|Oswald listed a "Sgt. Robert Hidell" as a reference on one job application and "George Hidell" as a reference on another. +LJ032-0109|Both names were found to be fictitious.|Both names were found to be fictitious. +LJ032-0110|Moreover, the use of "Alek" as a first name for Hidell is a further link to Oswald because "Alek" was Oswald's nickname in Russia.|Moreover, the use of "Alek" as a first name for Hidell is a further link to Oswald because "Alek" was Oswald's nickname in Russia. +LJ032-0111|Letters received by Marina Oswald from her husband signed "Alek" were given to the Commission.|Letters received by Marina Oswald from her husband signed "Alek" were given to the Commission. +LJ032-0112|Oswald's Palmprint on Rifle Barrel|Oswald's Palmprint on Rifle Barrel +LJ032-0113|Based on the above evidence, the Commission concluded that Oswald purchased the rifle found on the sixth floor of the Depository Building.|Based on the above evidence, the Commission concluded that Oswald purchased the rifle found on the sixth floor of the Depository Building. +LJ032-0114|Additional evidence of ownership was provided in the form of palmprint identification which indicated that Oswald had possession of the rifle he had purchased.|Additional evidence of ownership was provided in the form of palmprint identification which indicated that Oswald had possession of the rifle he had purchased. +LJ032-0115|A few minutes after the rifle was discovered on the sixth floor of the Depository Building|A few minutes after the rifle was discovered on the sixth floor of the Depository Building +LJ032-0116|it was examined by Lt. J. C. Day of the identification bureau of the Dallas police.|it was examined by Lt. J. C. Day of the identification bureau of the Dallas police. +LJ032-0117|He lifted the rifle by the wooden stock after his examination convinced him that the wood was too rough to take fingerprints.|He lifted the rifle by the wooden stock after his examination convinced him that the wood was too rough to take fingerprints. +LJ032-0118|Capt. J. W. Fritz then ejected a cartridge by operating the bolt,|Capt. J. W. Fritz then ejected a cartridge by operating the bolt, +LJ032-0119|but only after Day viewed the knob on the bolt through a magnifying glass and found no prints.|but only after Day viewed the knob on the bolt through a magnifying glass and found no prints. +LJ032-0120|Day continued to examine the rifle with the magnifying glass, looking for possible fingerprints.|Day continued to examine the rifle with the magnifying glass, looking for possible fingerprints. +LJ032-0121|He applied fingerprint powder to the side of the metal housing near the trigger, and noticed traces of two prints.|He applied fingerprint powder to the side of the metal housing near the trigger, and noticed traces of two prints. +LJ032-0122|At 11:45 p.m. on November 22,|At eleven:forty-five p.m. on November twenty-two, +LJ032-0123|the rifle was released to the FBI and forwarded to Washington where it was examined on the morning of November 23|the rifle was released to the FBI and forwarded to Washington where it was examined on the morning of November twenty-three +LJ032-0124|by Sebastian F. Latona, supervisor of the Latent Fingerprint Section of the FBI's Identification Division.|by Sebastian F. Latona, supervisor of the Latent Fingerprint Section of the FBI's Identification Division. +LJ032-0125|In his testimony before the Commission, Latona stated that when he received the rifle, the area where prints were visible was protected by cellophane.|In his testimony before the Commission, Latona stated that when he received the rifle, the area where prints were visible was protected by cellophane. +LJ032-0126|He examined these prints, as well as photographs of them which the Dallas police had made, and concluded that:|He examined these prints, as well as photographs of them which the Dallas police had made, and concluded that: +LJ032-0127|the formations, the ridge formations and characteristics, were insufficient for purposes of either effecting identification|the formations, the ridge formations and characteristics, were insufficient for purposes of either effecting identification +LJ032-0128|or a determination that the print was not identical with the prints of people.|or a determination that the print was not identical with the prints of people. +LJ032-0129|Accordingly, my opinion simply was that the latent prints which were there were of no value, end quote.|Accordingly, my opinion simply was that the latent prints which were there were of no value, end quote. +LJ032-0130|Latona then processed the complete weapon but developed no identifiable prints.|Latona then processed the complete weapon but developed no identifiable prints. +LJ032-0131|He stated that the poor quality of the wood and the metal would cause the rifle to absorb moisture from the skin, thereby making a clear print unlikely.|He stated that the poor quality of the wood and the metal would cause the rifle to absorb moisture from the skin, thereby making a clear print unlikely. +LJ032-0132|On November 22, however, before surrendering possession of the rifle to the FBI Laboratory,|On November twenty-two, however, before surrendering possession of the rifle to the FBI Laboratory, +LJ032-0133|Lieutenant Day of the Dallas Police Department had "lifted" a palmprint from the underside of the gun barrel|Lieutenant Day of the Dallas Police Department had "lifted" a palmprint from the underside of the gun barrel +LJ032-0134|quote, near the firing end of the barrel about 3 inches under the woodstock when I took the woodstock loose, end quote,|quote, near the firing end of the barrel about three inches under the woodstock when I took the woodstock loose, end quote, +LJ032-0135|"Lifting" a print involves the use of adhesive material to remove the fingerprint powder which adheres to the original print.|"Lifting" a print involves the use of adhesive material to remove the fingerprint powder which adheres to the original print. +LJ032-0136|In this way the powdered impression is actually removed from the object.|In this way the powdered impression is actually removed from the object. +LJ032-0137|The lifting had been so complete in this case that there was no trace of the print on the rifle itself when it was examined by Latona.|The lifting had been so complete in this case that there was no trace of the print on the rifle itself when it was examined by Latona. +LJ032-0138|Nor was there any indication that the lift had been performed.|Nor was there any indication that the lift had been performed. +LJ032-0139|Day, on the other hand, believed that sufficient traces of the print had been left on the rifle barrel,|Day, on the other hand, believed that sufficient traces of the print had been left on the rifle barrel, +LJ032-0140|because he did not release the lifted print until November 26, when he received instructions to send "everything that we had" to the FBI.|because he did not release the lifted print until November twenty-six, when he received instructions to send "everything that we had" to the FBI. +LJ032-0141|The print arrived in the FBI Laboratory in Washington on November 29, mounted on a card on which Lieutenant Day had written the words|The print arrived in the FBI Laboratory in Washington on November twenty-nine, mounted on a card on which Lieutenant Day had written the words +LJ032-0142|quote, off underside gun barrel near end of grip C2766, end quote,|quote, off underside gun barrel near end of grip C two seven six six, end quote, +LJ032-0143|The print's positive identity as having been lifted from the rifle|The print's positive identity as having been lifted from the rifle +LJ032-0144|was confirmed by FBI Laboratory tests which established that the adhesive material bearing the print|was confirmed by FBI Laboratory tests which established that the adhesive material bearing the print +LJ032-0145|also bore impressions of the same irregularities that appeared on the barrel of the rifle.|also bore impressions of the same irregularities that appeared on the barrel of the rifle. +LJ032-0146|Latona testified that this palmprint was the right palmprint of Lee Harvey Oswald.|Latona testified that this palmprint was the right palmprint of Lee Harvey Oswald. +LJ032-0147|At the request of the Commission, Arthur Mandella,|At the request of the Commission, Arthur Mandella, +LJ032-0148|fingerprint expert with the New York City Police Department, conducted an independent examination and also determined that this was the right palmprint of Oswald.|fingerprint expert with the New York City Police Department, conducted an independent examination and also determined that this was the right palmprint of Oswald. +LJ032-0149|Latona's findings were also confirmed by Ronald G. Wittmus, another FBI fingerprint expert.|Latona's findings were also confirmed by Ronald G. Wittmus, another FBI fingerprint expert. +LJ032-0150|In the opinion of these experts, it was not possible to estimate the time which elapsed between the placing of the print on the rifle and the date of the lift.|In the opinion of these experts, it was not possible to estimate the time which elapsed between the placing of the print on the rifle and the date of the lift. +LJ032-0151|Experts testifying before the Commission agreed that palmprints are as unique as fingerprints for purposes of establishing identification.|Experts testifying before the Commission agreed that palmprints are as unique as fingerprints for purposes of establishing identification. +LJ032-0152|Oswald's palmprint on the underside of the barrel demonstrates that he handled the rifle when it was disassembled.|Oswald's palmprint on the underside of the barrel demonstrates that he handled the rifle when it was disassembled. +LJ032-0153|A palmprint could not be placed on this portion of the rifle, when assembled, because the wooden foregrip covers the barrel at this point.|A palmprint could not be placed on this portion of the rifle, when assembled, because the wooden foregrip covers the barrel at this point. +LJ032-0154|The print is additional proof that the rifle was in Oswald's possession.|The print is additional proof that the rifle was in Oswald's possession. +LJ032-0155|Fibers on Rifle|Fibers on Rifle +LJ032-0156|In a crevice between the butt plate of the rifle and the wooden stock|In a crevice between the butt plate of the rifle and the wooden stock +LJ032-0157|was a tuft of several cotton fibers of dark blue, gray-black, and orange-yellow shades.|was a tuft of several cotton fibers of dark blue, gray-black, and orange-yellow shades. +LJ032-0158|On November 23, 1963,|On November twenty-three, nineteen sixty-three, +LJ032-0159|these fibers were examined by Paul M. Stombaugh, a special agent assigned to the Hair and Fiber Unit of the FBI Laboratory.|these fibers were examined by Paul M. Stombaugh, a special agent assigned to the Hair and Fiber Unit of the FBI Laboratory. +LJ032-0160|He compared them with the fibers found in the shirt which Oswald was wearing when arrested in the Texas Theatre.|He compared them with the fibers found in the shirt which Oswald was wearing when arrested in the Texas Theatre. +LJ032-0161|This shirt was also composed of dark blue, gray- black and orange-yellow cotton fibers.|This shirt was also composed of dark blue, gray- black and orange-yellow cotton fibers. +LJ032-0162|Stombaugh testified that the colors, shades, and twist of the fibers found in the tuft on the rifle matched those in Oswald's shirt.|Stombaugh testified that the colors, shades, and twist of the fibers found in the tuft on the rifle matched those in Oswald's shirt. +LJ032-0163|Stombaugh explained in his testimony that in fiber analysis, as distinct from fingerprint or firearms identification,|Stombaugh explained in his testimony that in fiber analysis, as distinct from fingerprint or firearms identification, +LJ032-0164|it is not possible to state with scientific certainty that a particular small group of fibers come from a certain piece of clothing|it is not possible to state with scientific certainty that a particular small group of fibers come from a certain piece of clothing +LJ032-0166|Judgments as to probability will depend on the number and types of matches.|Judgments as to probability will depend on the number and types of matches. +LJ032-0167|He concluded, quote, There is no doubt in my mind that these fibers could have come from this shirt.|He concluded, quote, There is no doubt in my mind that these fibers could have come from this shirt. +LJ032-0168|There is no way, however, to eliminate the possibility of the fibers having come from another identical shirt, end quote,|There is no way, however, to eliminate the possibility of the fibers having come from another identical shirt, end quote, +LJ032-0169|Having considered the probabilities as explained in Stombaugh's testimony,|Having considered the probabilities as explained in Stombaugh's testimony, +LJ032-0170|the Commission has concluded that the fibers in the tuft on the rifle most probably came from the shirt worn by Oswald when he was arrested,|the Commission has concluded that the fibers in the tuft on the rifle most probably came from the shirt worn by Oswald when he was arrested, +LJ032-0171|and that this was the same shirt which Oswald wore on the morning of the assassination.|and that this was the same shirt which Oswald wore on the morning of the assassination. +LJ032-0172|Marina Oswald testified that she thought her husband wore this shirt to work on that day.|Marina Oswald testified that she thought her husband wore this shirt to work on that day. +LJ032-0173|The testimony of those who saw him after the assassination was inconclusive about the color of Oswald's shirt,|The testimony of those who saw him after the assassination was inconclusive about the color of Oswald's shirt, +LJ032-0174|but Mary Bledsoe, a former landlady of Oswald, saw him on a bus approximately 10 minutes after the assassination|but Mary Bledsoe, a former landlady of Oswald, saw him on a bus approximately ten minutes after the assassination +LJ032-0175|and identified the shirt as being the one worn by Oswald primarily because of a distinctive hole in the shirt's right elbow.|and identified the shirt as being the one worn by Oswald primarily because of a distinctive hole in the shirt's right elbow. +LJ032-0176|Moreover, the bus transfer which he obtained as he left the bus was still in the pocket when he was arrested.|Moreover, the bus transfer which he obtained as he left the bus was still in the pocket when he was arrested. +LJ032-0177|Although Oswald returned to his roominghouse after the assassination and when questioned by the police, claimed to have changed his shirt,|Although Oswald returned to his roominghouse after the assassination and when questioned by the police, claimed to have changed his shirt, +LJ032-0178|the evidence indicates that he continued wearing the same shirt which he was wearing all morning and which he was still wearing when arrested.|the evidence indicates that he continued wearing the same shirt which he was wearing all morning and which he was still wearing when arrested. +LJ032-0179|In light of these findings the Commission evaluated the additional testimony of Stombaugh|In light of these findings the Commission evaluated the additional testimony of Stombaugh +LJ032-0180|that the fibers were caught in the crevice of the rifle's butt plate, quote, in the recent past, end quote,|that the fibers were caught in the crevice of the rifle's butt plate, quote, in the recent past, end quote, +LJ032-0181|Although Stombaugh was unable to estimate the period of time the fibers were on the rifle he said that the fibers, quote,|Although Stombaugh was unable to estimate the period of time the fibers were on the rifle he said that the fibers, quote, +LJ032-0182|were clean, they had good color to them, there was no grease on them and they were not fragmented. They looked as if they had just been picked up, end quote.|were clean, they had good color to them, there was no grease on them and they were not fragmented. They looked as if they had just been picked up, end quote. +LJ032-0183|The relative freshness of the fibers is strong evidence that they were caught on the rifle on the morning of the assassination or during the preceding evening.|The relative freshness of the fibers is strong evidence that they were caught on the rifle on the morning of the assassination or during the preceding evening. +LJ032-0184|For 10 days prior to the eve of the assassination Oswald had not been present at Ruth Paine's house in Irving, Texas, where the rifle was kept.|For ten days prior to the eve of the assassination Oswald had not been present at Ruth Paine's house in Irving, Texas, where the rifle was kept. +LJ032-0185|Moreover, the Commission found no reliable evidence that Oswald used the rifle at any time between September 23,|Moreover, the Commission found no reliable evidence that Oswald used the rifle at any time between September twenty-three, +LJ032-0186|when it was transported from New Orleans, and November 22, the day of the assassination.|when it was transported from New Orleans, and November twenty-two, the day of the assassination. +LJ032-0187|The fact that on the morning of the assassination Oswald was wearing the shirt from which these relatively fresh fibers most probably originated,|The fact that on the morning of the assassination Oswald was wearing the shirt from which these relatively fresh fibers most probably originated, +LJ032-0188|provides some evidence that they were placed on the rifle that day|provides some evidence that they were placed on the rifle that day +LJ032-0189|since there was limited, if any, opportunity for Oswald to handle the weapon during the 2 months prior to November 22.|since there was limited, if any, opportunity for Oswald to handle the weapon during the two months prior to November twenty-two. +LJ032-0190|On the other hand Stombaugh pointed out that fibers might retain their freshness if the rifle had been|On the other hand Stombaugh pointed out that fibers might retain their freshness if the rifle had been +LJ032-0191|quote, put aside, end quote, after catching the fibers.|quote, put aside, end quote, after catching the fibers. +LJ032-0192|The rifle used in the assassination probably had been wrapped in a blanket for about 8 weeks prior to November 22.|The rifle used in the assassination probably had been wrapped in a blanket for about eight weeks prior to November twenty-two. +LJ032-0193|Because the relative freshness of these fibers might be explained by the continuous storage of the rifle in the blanket,|Because the relative freshness of these fibers might be explained by the continuous storage of the rifle in the blanket, +LJ032-0194|the Commission was unable to reach any firm conclusion as to when the fibers were caught in the rifle.|the Commission was unable to reach any firm conclusion as to when the fibers were caught in the rifle. +LJ032-0195|The Commission was able to conclude, however, that the fibers most probably came from Oswald's shirt.|The Commission was able to conclude, however, that the fibers most probably came from Oswald's shirt. +LJ032-0196|This adds to the conviction of the Commission that Oswald owned and handled the weapon used in the assassination.|This adds to the conviction of the Commission that Oswald owned and handled the weapon used in the assassination. +LJ032-0197|Photograph of Oswald With Rifle|Photograph of Oswald With Rifle +LJ032-0199|the Oswalds lived on Neely Street in Dallas in a rented house which had a small back yard.|the Oswalds lived on Neely Street in Dallas in a rented house which had a small back yard. +LJ032-0200|One Sunday, while his wife was hanging diapers, Oswald asked her to take a picture of him holding a rifle, a pistol|One Sunday, while his wife was hanging diapers, Oswald asked her to take a picture of him holding a rifle, a pistol +LJ032-0201|and issues of two newspapers later identified as the Worker and the Militant.|and issues of two newspapers later identified as the Worker and the Militant. +LJ032-0202|Two pictures were taken. The Commission has concluded that the rifle shown in these pictures is the same rifle which was found on the sixth floor of the Depository Building|Two pictures were taken. The Commission has concluded that the rifle shown in these pictures is the same rifle which was found on the sixth floor of the Depository Building +LJ032-0203|on November 22, 1963. One of these pictures, Exhibit No. 133-A, shows most of the rifle's configuration.|on November twenty-two, nineteen sixty-three. One of these pictures, Exhibit Number one thirty-three A, shows most of the rifle's configuration. +LJ032-0204|Special Agent Lyndal L. Shaneyfelt, a photography expert with the FBI,|Special Agent Lyndal L. Shaneyfelt, a photography expert with the FBI, +LJ032-0205|photographed the rifle used in the assassination, attempting to duplicate the position of the rifle and the lighting in Exhibit No. 133-A.|photographed the rifle used in the assassination, attempting to duplicate the position of the rifle and the lighting in Exhibit Number one thirty-three A. +LJ032-0206|After comparing the rifle in the simulated photograph with the rifle in Exhibit No. 133-A, Shaneyfelt testified, quote,|After comparing the rifle in the simulated photograph with the rifle in Exhibit Number one thirty-three A, Shaneyfelt testified, quote, +LJ032-0207|I found it to be the same general configuration. All appearances were the same, end quote,|I found it to be the same general configuration. All appearances were the same, end quote, +LJ032-0208|He found, quote, one notch in the stock at this point that appears very faintly in the photograph, end quote.|He found, quote, one notch in the stock at this point that appears very faintly in the photograph, end quote. +LJ032-0209|He stated, however, that while he, quote, found no differences, end quote, between the rifles in the two photographs, he could not make a, quote, positive identification|He stated, however, that while he, quote, found no differences, end quote, between the rifles in the two photographs, he could not make a, quote, positive identification +LJ032-0210|to the exclusion of all other rifles of the same general configuration, end quote.|to the exclusion of all other rifles of the same general configuration, end quote. +LJ032-0211|The authenticity of these pictures has been established by expert testimony which links the second picture,|The authenticity of these pictures has been established by expert testimony which links the second picture, +LJ032-0212|Commission Exhibit No. 133-B, to Oswald's Imperial Reflex camera, with which Marina Oswald testified she took the pictures.|Commission Exhibit Number one thirty-three B, to Oswald's Imperial Reflex camera, with which Marina Oswald testified she took the pictures. +LJ032-0213|The negative of that picture, Commission Exhibit No. 133-B, was found among Oswald's possessions.|The negative of that picture, Commission Exhibit Number one thirty-three B, was found among Oswald's possessions. +LJ032-0214|Using a recognized technique of determining whether a picture was taken with a particular camera,|Using a recognized technique of determining whether a picture was taken with a particular camera, +LJ032-0215|Shaneyfelt compared this negative with a negative which he made by taking a new picture with Oswald's camera.|Shaneyfelt compared this negative with a negative which he made by taking a new picture with Oswald's camera. +LJ032-0216|He concluded that the negative of Exhibit No. 133-B was exposed in Oswald's Imperial Reflex camera to the exclusion of all other cameras.|He concluded that the negative of Exhibit Number one thirty-three B was exposed in Oswald's Imperial Reflex camera to the exclusion of all other cameras. +LJ032-0217|He could not test Exhibit No. 133-A in the same way because the negative was never recovered.|He could not test Exhibit Number one thirty-three A in the same way because the negative was never recovered. +LJ032-0218|Both pictures, however, have identical backgrounds and lighting and, judging from the shadows, were taken at the same angle.|Both pictures, however, have identical backgrounds and lighting and, judging from the shadows, were taken at the same angle. +LJ032-0219|They are photographs of the same scene.|They are photographs of the same scene. +LJ032-0220|Since Exhibit No. 133-B was taken with Oswald's camera,|Since Exhibit Number one thirty-three B was taken with Oswald's camera, +LJ032-0221|it is reasonably certain that Exhibit No. 133-A was taken by the same camera at the same time, as Marina Oswald testified.|it is reasonably certain that Exhibit Number one thirty-three A was taken by the same camera at the same time, as Marina Oswald testified. +LJ032-0222|Moreover, Shaneyfelt testified that in his opinion the photographs were not composites of two different photographs|Moreover, Shaneyfelt testified that in his opinion the photographs were not composites of two different photographs +LJ032-0223|and that Oswald's face had not been superimposed on another body.|and that Oswald's face had not been superimposed on another body. +LJ032-0224|One of the photographs taken by Marina Oswald was widely published in newspapers and magazines,|One of the photographs taken by Marina Oswald was widely published in newspapers and magazines, +LJ032-0225|and in many instances the details of these pictures differed from the original, and even from each other, particularly as to the configuration of the rifle.|and in many instances the details of these pictures differed from the original, and even from each other, particularly as to the configuration of the rifle. +LJ032-0226|The Commission sought to determine whether these photographs were touched prior to publication.|The Commission sought to determine whether these photographs were touched prior to publication. +LJ032-0227|Shaneyfelt testified that the published photographs appeared to be based on a copy of the original which the publications had each retouched differently.|Shaneyfelt testified that the published photographs appeared to be based on a copy of the original which the publications had each retouched differently. +LJ032-0228|Several of the publications furnished the Commission with the prints they had used, or described by correspondence the retouching they had done.|Several of the publications furnished the Commission with the prints they had used, or described by correspondence the retouching they had done. +LJ032-0229|This information enabled the Commission to conclude|This information enabled the Commission to conclude +LJ032-0230|that the published pictures were the same as the original except for retouching done by these publications, apparently for the purpose of clarifying the lines of the rifle|that the published pictures were the same as the original except for retouching done by these publications, apparently for the purpose of clarifying the lines of the rifle +LJ032-0231|and other details in the picture.|and other details in the picture. +LJ032-0232|The dates surrounding the taking of this picture and the purchase of the rifle|The dates surrounding the taking of this picture and the purchase of the rifle +LJ032-0233|reinforce the belief that the rifle in the photograph is the rifle which Oswald bought from Klein's.|reinforce the belief that the rifle in the photograph is the rifle which Oswald bought from Klein's. +LJ032-0234|The rifle was shipped from Klein's in Chicago on March 20, 1963, at a time when the Oswalds were living on Neely Street.|The rifle was shipped from Klein's in Chicago on March twenty, nineteen sixty-three, at a time when the Oswalds were living on Neely Street. +LJ032-0235|From an examination of one of the photographs, the Commission determined the dates of the issues of the Militant and the Worker which Oswald was holding in his hand.|From an examination of one of the photographs, the Commission determined the dates of the issues of the Militant and the Worker which Oswald was holding in his hand. +LJ032-0236|By checking the actual mailing dates of these issues and the time it usually takes to effect delivery to Dallas,|By checking the actual mailing dates of these issues and the time it usually takes to effect delivery to Dallas, +LJ032-0237|it was established that the photographs must have been taken sometime after March 27.|it was established that the photographs must have been taken sometime after March twenty-seven. +LJ032-0238|Marina Oswald testified that the photographs were taken on a Sunday about 2 weeks before the attempted shooting of Maj. Gen. Edwin A. Walker|Marina Oswald testified that the photographs were taken on a Sunday about two weeks before the attempted shooting of Maj. Gen. Edwin A. Walker +LJ032-0239|on April 10, 1963.|on April ten, nineteen sixty-three. +LJ032-0240|By Sunday, March 31, 1963,|By Sunday, March thirty-one, nineteen sixty-three, +LJ032-0241|10 days prior to the Walker attempt, Oswald had undoubtedly received the rifle shipped from Chicago on March 20,|ten days prior to the Walker attempt, Oswald had undoubtedly received the rifle shipped from Chicago on March twenty, +LJ032-0242|the revolver shipped from Los Angeles on the same date, and the two newspapers which he was holding in the picture.|the revolver shipped from Los Angeles on the same date, and the two newspapers which he was holding in the picture. +LJ032-0243|Rifle Among Oswald's Possessions|Rifle Among Oswald's Possessions +LJ032-0244|Marina Oswald testified that the rifle found on the sixth floor of the Depository Building was the "fateful rifle of Lee Oswald."|Marina Oswald testified that the rifle found on the sixth floor of the Depository Building was the "fateful rifle of Lee Oswald." +LJ032-0245|Moreover, it was the only rifle owned by her husband following his return from the Soviet Union in June 1962.|Moreover, it was the only rifle owned by her husband following his return from the Soviet Union in June nineteen sixty-two. +LJ032-0246|It had been purchased in March 1963, and taken to New Orleans where Marina Oswald saw it in their rented apartment|It had been purchased in March nineteen sixty-three, and taken to New Orleans where Marina Oswald saw it in their rented apartment +LJ032-0247|during the summer of 1963.|during the summer of nineteen sixty-three. +LJ032-0248|It appears from his wife's testimony|It appears from his wife's testimony +LJ032-0249|that Oswald may have sat on the screened-in porch at night practicing with the rifle by looking through the telescopic sight and operating the bolt.|that Oswald may have sat on the screened-in porch at night practicing with the rifle by looking through the telescopic sight and operating the bolt. +LJ032-0250|In September 1963,|In September nineteen sixty-three, +LJ032-0251|Oswald loaded their possessions into a station wagon owned by Ruth Paine, who had invited Marina Oswald and the baby to live at her home in Irving, Texas.|Oswald loaded their possessions into a station wagon owned by Ruth Paine, who had invited Marina Oswald and the baby to live at her home in Irving, Texas. +LJ032-0252|Marina Oswald has stated that the rifle was among these possessions, although Ruth Paine testified that she was not aware of it.|Marina Oswald has stated that the rifle was among these possessions, although Ruth Paine testified that she was not aware of it. +LJ032-0253|From September 24, 1963, when Marina Oswald arrived in Irving from New Orleans, until the morning of the assassination,|From September twenty-four, nineteen sixty-three, when Marina Oswald arrived in Irving from New Orleans, until the morning of the assassination, +LJ032-0254|the rifle was, according to the evidence, stored in a green and brown blanket in the Paines' garage among the Oswalds' other possessions.|the rifle was, according to the evidence, stored in a green and brown blanket in the Paines' garage among the Oswalds' other possessions. +LJ032-0255|About 1 week after the return from New Orleans,|About one week after the return from New Orleans, +LJ032-0256|Marina Oswald was looking in the garage for parts to the baby's crib and thought that the parts might be in the blanket.|Marina Oswald was looking in the garage for parts to the baby's crib and thought that the parts might be in the blanket. +LJ032-0257|When she started to open the blanket, she saw the stock of the rifle.|When she started to open the blanket, she saw the stock of the rifle. +LJ032-0258|Ruth and Michael Paine both noticed the rolled-up blanket in the garage during the time that Marina Oswald was living in their home.|Ruth and Michael Paine both noticed the rolled-up blanket in the garage during the time that Marina Oswald was living in their home. +LJ032-0259|On several occasions, Michael Paine moved the blanket in the garage.|On several occasions, Michael Paine moved the blanket in the garage. +LJ032-0260|He thought it contained tent poles, or possibly other camping equipment such as a folding shovel.|He thought it contained tent poles, or possibly other camping equipment such as a folding shovel. +LJ032-0261|When he appeared before the Commission, Michael Paine lifted the blanket|When he appeared before the Commission, Michael Paine lifted the blanket +LJ032-0262|with the rifle wrapped inside and testified that it appeared to be the same approximate weight and shape as the package in his garage.|with the rifle wrapped inside and testified that it appeared to be the same approximate weight and shape as the package in his garage. +LJ032-0263|About 3 hours after the assassination, a detective and deputy sheriff saw the blanket-roll,|About three hours after the assassination, a detective and deputy sheriff saw the blanket-roll, +LJ032-0264|tied with a string, lying on the floor of the Paines' garage.|tied with a string, lying on the floor of the Paines' garage. +LJ032-0265|Each man testified that he thought he could detect the outline of a rifle in the blanket, even though the blanket was empty.|Each man testified that he thought he could detect the outline of a rifle in the blanket, even though the blanket was empty. +LJ032-0266|Paul M. Stombaugh, of the FBI Laboratory,|Paul M. Stombaugh, of the FBI Laboratory, +LJ032-0267|examined the blanket and discovered a bulge approximately 10 inches long midway in the blanket.|examined the blanket and discovered a bulge approximately ten inches long midway in the blanket. +LJ032-0268|This bulge was apparently caused by a hard protruding object which had stretched the blanket's fibers.|This bulge was apparently caused by a hard protruding object which had stretched the blanket's fibers. +LJ032-0269|It could have been caused by the telescopic sight of the rifle which was approximately 11 inches long.|It could have been caused by the telescopic sight of the rifle which was approximately eleven inches long. +LJ032-0270|Having reviewed the evidence that (1) Lee Harvey Oswald purchased the rifle used in the assassination,|Having reviewed the evidence that (one) Lee Harvey Oswald purchased the rifle used in the assassination, +LJ032-0271|(2) Oswald's palmprint was on the rifle in a position which shows that he had handled it while it was disassembled,|(two) Oswald's palmprint was on the rifle in a position which shows that he had handled it while it was disassembled, +LJ032-0272|(3) fibers found on the rifle most probably came from the shirt Oswald was wearing on the day of the assassination,|(three) fibers found on the rifle most probably came from the shirt Oswald was wearing on the day of the assassination, +LJ032-0273|(4) a photograph taken in the yard of Oswald's apartment showed him holding this rifle, and (5)|(four) a photograph taken in the yard of Oswald's apartment showed him holding this rifle, and (five) +LJ032-0274|the rifle was kept among Oswald's possessions from the time of its purchase until the day of the assassination,|the rifle was kept among Oswald's possessions from the time of its purchase until the day of the assassination, +LJ032-0275|the Commission concluded that the rifle used to assassinate President Kennedy and wound Governor Connally was owned and possessed by Lee Harvey Oswald.|the Commission concluded that the rifle used to assassinate President Kennedy and wound Governor Connally was owned and possessed by Lee Harvey Oswald. +LJ033-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ033-0002|Chapter 4. The Assassin: Part 2.|Chapter four. The Assassin: Part two. +LJ033-0003|The rifle in the building.|The rifle in the building. +LJ033-0004|The Commission has evaluated the evidence tending to show how Lee Harvey Oswald's Mannlicher-Carcano rifle, serial number C2766,|The Commission has evaluated the evidence tending to show how Lee Harvey Oswald's Mannlicher-Carcano rifle, serial number C two seven six six, +LJ033-0005|was brought into the Depository Building, where it was found on the sixth floor shortly after the assassination.|was brought into the Depository Building, where it was found on the sixth floor shortly after the assassination. +LJ033-0006|In this connection the Commission considered (1)|In this connection the Commission considered (one) +LJ033-0007|the circumstances surrounding Oswald's return to Irving, Texas, on Thursday, November 21, 1963,|the circumstances surrounding Oswald's return to Irving, Texas, on Thursday, November twenty-one, nineteen sixty-three, +LJ033-0008|(2) the disappearance of the rifle from its normal place of storage, (3)|(two) the disappearance of the rifle from its normal place of storage, (three) +LJ033-0009|Oswald's arrival at the Depository Building on November 22, carrying a long and bulky brown paper package,|Oswald's arrival at the Depository Building on November twenty-two, carrying a long and bulky brown paper package, +LJ033-0010|the presence of a long handmade brown paper bag near the point from which the shots were fired, and (5) the palmprint,|the presence of a long handmade brown paper bag near the point from which the shots were fired, and (five) the palmprint, +LJ033-0011|fiber, and paper analyses linking Oswald and the assassination weapon to this bag.|fiber, and paper analyses linking Oswald and the assassination weapon to this bag. +LJ033-0012|The Curtain Rod Story|The Curtain Rod Story +LJ033-0013|During October and November of 1963,|During October and November of nineteen sixty-three, +LJ033-0014|Lee Harvey Oswald lived in a roominghouse in Dallas while his wife and children lived in Irving, at the home of Ruth Paine,|Lee Harvey Oswald lived in a roominghouse in Dallas while his wife and children lived in Irving, at the home of Ruth Paine, +LJ033-0015|approximately 15 miles from Oswald's place of work at the Texas School Book Depository.|approximately fifteen miles from Oswald's place of work at the Texas School Book Depository. +LJ033-0016|Oswald traveled between Dallas and Irving on weekends in a car driven by a neighbor of the Paines, Buell Wesley Frazier,|Oswald traveled between Dallas and Irving on weekends in a car driven by a neighbor of the Paines, Buell Wesley Frazier, +LJ033-0017|who also worked at the Depository.|who also worked at the Depository. +LJ033-0018|Oswald generally would go to Irving on Friday afternoon and return to Dallas Monday morning.|Oswald generally would go to Irving on Friday afternoon and return to Dallas Monday morning. +LJ033-0019|According to the testimony of Frazier, Marina Oswald, and Ruth Paine, it appears that Oswald never returned to Irving in midweek|According to the testimony of Frazier, Marina Oswald, and Ruth Paine, it appears that Oswald never returned to Irving in midweek +LJ033-0020|prior to November 21, 1963, except on Monday, October 21, when he visited his wife in the hospital|prior to November twenty-one, nineteen sixty-three, except on Monday, October twenty-one, when he visited his wife in the hospital +LJ033-0021|after the birth of their second child.|after the birth of their second child. +LJ033-0022|During the morning of November 21, Oswald asked Frazier whether he could ride home with him that afternoon.|During the morning of November twenty-one, Oswald asked Frazier whether he could ride home with him that afternoon. +LJ033-0023|Frazier, surprised, asked him why he was going to Irving on Thursday night rather than Friday.|Frazier, surprised, asked him why he was going to Irving on Thursday night rather than Friday. +LJ033-0024|Oswald replied, quote, I'm going home to get some curtain rods to put in an apartment, end quote.|Oswald replied, quote, I'm going home to get some curtain rods to put in an apartment, end quote. +LJ033-0025|The two men left work at 4:40 p.m. and drove to Irving.|The two men left work at four:forty p.m. and drove to Irving. +LJ033-0026|There was little conversation between them on the way home.|There was little conversation between them on the way home. +LJ033-0027|Mrs. Linnie Mae Randle, Frazier's sister, commented to her brother about Oswald's unusual midweek return to Irving.|Mrs. Linnie Mae Randle, Frazier's sister, commented to her brother about Oswald's unusual midweek return to Irving. +LJ033-0028|Frazier told her that Oswald had come home to get curtain rods.|Frazier told her that Oswald had come home to get curtain rods. +LJ033-0029|It would appear, however, that obtaining curtain rods was not the purpose of Oswald's trip to Irving on November 21.|It would appear, however, that obtaining curtain rods was not the purpose of Oswald's trip to Irving on November twenty-one. +LJ033-0030|Mrs. A. C. Johnson, his landlady, testified that Oswald's room at 1026 North Beckley Avenue|Mrs. A. C. Johnson, his landlady, testified that Oswald's room at ten twenty-six North Beckley Avenue +LJ033-0031|had curtains and curtain rods, and that Oswald had never discussed the subject with her.|had curtains and curtain rods, and that Oswald had never discussed the subject with her. +LJ033-0032|In the Paines' garage, along with many other objects of a household character,|In the Paines' garage, along with many other objects of a household character, +LJ033-0033|there were two flat lightweight curtain rods belonging to Ruth Paine but they were still there on Friday afternoon after Oswald's arrest.|there were two flat lightweight curtain rods belonging to Ruth Paine but they were still there on Friday afternoon after Oswald's arrest. +LJ033-0034|Oswald never asked Mrs. Paine about the use of curtain rods,|Oswald never asked Mrs. Paine about the use of curtain rods, +LJ033-0035|and Marina Oswald testified that Oswald did not say anything about curtain rods on the day before the assassination.|and Marina Oswald testified that Oswald did not say anything about curtain rods on the day before the assassination. +LJ033-0036|No curtain rods were known to have been discovered in the Depository Building after the assassination.|No curtain rods were known to have been discovered in the Depository Building after the assassination. +LJ033-0037|In deciding whether Oswald carried a rifle to work in a long paper bag on November 22,|In deciding whether Oswald carried a rifle to work in a long paper bag on November twenty-two, +LJ033-0038|the Commission gave weight to the fact that Oswald gave a false reason for returning home on November 21,|the Commission gave weight to the fact that Oswald gave a false reason for returning home on November twenty-one, +LJ033-0039|and one which provided an excuse for the carrying of a bulky package the following morning.|and one which provided an excuse for the carrying of a bulky package the following morning. +LJ033-0040|Before dinner on November 21, Oswald played on the lawn of the Paines' home with his daughter June.|Before dinner on November twenty-one, Oswald played on the lawn of the Paines' home with his daughter June. +LJ033-0041|After dinner Ruth Paine and Marina Oswald were busy cleaning house and preparing their children for bed.|After dinner Ruth Paine and Marina Oswald were busy cleaning house and preparing their children for bed. +LJ033-0042|Between the hours of 8 and 9 p.m. they were occupied with the children in the bedrooms located at the extreme east end of the house.|Between the hours of eight and nine p.m. they were occupied with the children in the bedrooms located at the extreme east end of the house. +LJ033-0043|On the west end of the house is the attached garage, which can be reached from the kitchen or from the outside.|On the west end of the house is the attached garage, which can be reached from the kitchen or from the outside. +LJ033-0044|In the garage were the personal belongings of the Oswald family including, as the evidence has shown, the rifle wrapped in the old brown and green blanket.|In the garage were the personal belongings of the Oswald family including, as the evidence has shown, the rifle wrapped in the old brown and green blanket. +LJ033-0045|At approximately 9 p.m., after the children had been put to bed, Mrs. Paine, according to her testimony before the Commission, quote,|At approximately nine p.m., after the children had been put to bed, Mrs. Paine, according to her testimony before the Commission, quote, +LJ033-0047|I noticed when I went out that the light was on, end quote,|I noticed when I went out that the light was on, end quote, +LJ033-0048|Mrs. Paine was certain that she had not left the light on in the garage after dinner.|Mrs. Paine was certain that she had not left the light on in the garage after dinner. +LJ033-0049|According to Mrs. Paine, Oswald had gone to bed by 9 p.m.;|According to Mrs. Paine, Oswald had gone to bed by nine p.m.; +LJ033-0050|Marina Oswald testified that it was between 9 and 10 p.m.|Marina Oswald testified that it was between nine and ten p.m. +LJ033-0051|Neither Marina Oswald nor Ruth Paine saw Oswald in the garage.|Neither Marina Oswald nor Ruth Paine saw Oswald in the garage. +LJ033-0052|The period between 8 and 9 p.m., however, provided ample opportunity for Oswald to prepare the rifle for his departure the next morning.|The period between eight and nine p.m., however, provided ample opportunity for Oswald to prepare the rifle for his departure the next morning. +LJ033-0053|Only if disassembled could the rifle fit into the paper bag found near the window from which the shots were fired.|Only if disassembled could the rifle fit into the paper bag found near the window from which the shots were fired. +LJ033-0054|A firearms expert with the FBI assembled the rifle in 6 minutes using a 10-cent coin as a tool,|A firearms expert with the FBI assembled the rifle in six minutes using a ten-cent coin as a tool, +LJ033-0055|and he could disassemble it more rapidly.|and he could disassemble it more rapidly. +LJ033-0056|While the rifle may have already been disassembled when Oswald arrived home on Thursday, he had ample time that evening to disassemble the rifle|While the rifle may have already been disassembled when Oswald arrived home on Thursday, he had ample time that evening to disassemble the rifle +LJ033-0057|and insert it into the paper bag.|and insert it into the paper bag. +LJ033-0058|On the day of the assassination, Marina Oswald was watching television when she learned of the shooting.|On the day of the assassination, Marina Oswald was watching television when she learned of the shooting. +LJ033-0059|A short time later Mrs. Paine told her that someone had shot the President, quote, from the building in which Lee is working, end quote.|A short time later Mrs. Paine told her that someone had shot the President, quote, from the building in which Lee is working, end quote. +LJ033-0060|Marina Oswald testified that at that time, quote, My heart dropped.|Marina Oswald testified that at that time, quote, My heart dropped. +LJ033-0061|I then went to the garage to see whether the rifle was there and I saw that the blanket was still there and I said Thank God, end quote.|I then went to the garage to see whether the rifle was there and I saw that the blanket was still there and I said Thank God, end quote. +LJ033-0062|She did not unroll the blanket. She saw that it was in its usual position and it appeared to her to have something inside.|She did not unroll the blanket. She saw that it was in its usual position and it appeared to her to have something inside. +LJ033-0063|Soon afterward, at about 3 p.m., police officers arrived and searched the house.|Soon afterward, at about three p.m., police officers arrived and searched the house. +LJ033-0064|Mrs. Paine pointed out that most of the Oswalds' possessions were in the garage.|Mrs. Paine pointed out that most of the Oswalds' possessions were in the garage. +LJ033-0065|With Ruth Paine acting as an interpreter, Detective Rose asked Marina whether her husband had a rifle.|With Ruth Paine acting as an interpreter, Detective Rose asked Marina whether her husband had a rifle. +LJ033-0066|Mrs. Paine, who had no knowledge of the rifle, first said "No," but when the question was translated, Marina Oswald replied "Yes."|Mrs. Paine, who had no knowledge of the rifle, first said "No," but when the question was translated, Marina Oswald replied "Yes." +LJ033-0067|She pointed to the blanket which was on the floor very close to where Ruth Paine was standing.|She pointed to the blanket which was on the floor very close to where Ruth Paine was standing. +LJ033-0068|Mrs. Paine testified, quote,|Mrs. Paine testified, quote, +LJ033-0069|As she [Marina] told me about it I stepped onto the blanket roll|As she [Marina] told me about it I stepped onto the blanket roll +LJ033-0070|And she indicated to me that she had peered into this roll and saw a portion of what she took to be a gun she knew her husband to have, a rifle.|And she indicated to me that she had peered into this roll and saw a portion of what she took to be a gun she knew her husband to have, a rifle. +LJ033-0071|And I then translated this to the officers that she knew that her husband had a gun that he had stored in here.|And I then translated this to the officers that she knew that her husband had a gun that he had stored in here. +LJ033-0073|Mrs. Paine had the actual blanket before her as she testified and she indicated that the blanket hung limp in the officer's hand.|Mrs. Paine had the actual blanket before her as she testified and she indicated that the blanket hung limp in the officer's hand. +LJ033-0074|Marina Oswald testified that this was her first knowledge that the rifle was not in its accustomed place.|Marina Oswald testified that this was her first knowledge that the rifle was not in its accustomed place. +LJ033-0075|The Long and Bulky Package|The Long and Bulky Package +LJ033-0076|On the morning of November 22, 1963,|On the morning of November twenty-two, nineteen sixty-three, +LJ033-0077|Lee Harvey Oswald left the Paine house in Irving at approximately 7:15 a.m., while Marina Oswald was still in bed.|Lee Harvey Oswald left the Paine house in Irving at approximately seven:fifteen a.m., while Marina Oswald was still in bed. +LJ033-0078|Neither she nor Mrs. Paine saw him leave the house. About half-a-block away from the Paine house was the residence of Mrs. Linnie Mae Randle,|Neither she nor Mrs. Paine saw him leave the house. About half-a-block away from the Paine house was the residence of Mrs. Linnie Mae Randle, +LJ033-0079|sister of the man with whom Oswald drove to work -- Buell Wesley Frazier.|sister of the man with whom Oswald drove to work -- Buell Wesley Frazier. +LJ033-0080|Mrs. Randle stated that on the morning of November 22, while her brother was eating breakfast,|Mrs. Randle stated that on the morning of November twenty-two, while her brother was eating breakfast, +LJ033-0081|she looked out the breakfast-room window and saw Oswald cross the street and walk toward the driveway where her brother parked his car near the carport.|she looked out the breakfast-room window and saw Oswald cross the street and walk toward the driveway where her brother parked his car near the carport. +LJ033-0082|He carried a, quote, heavy brown bag, end quote.|He carried a, quote, heavy brown bag, end quote. +LJ033-0083|Oswald gripped the bag in his right hand near the top, quote, It tapered like this as he hugged it in his hand.|Oswald gripped the bag in his right hand near the top, quote, It tapered like this as he hugged it in his hand. +LJ033-0084|It was more bulky toward the bottom, end quote, than toward the top.|It was more bulky toward the bottom, end quote, than toward the top. +LJ033-0085|She then opened the kitchen door and saw Oswald open the right rear door of her brother's car and place the package in the back of the car.|She then opened the kitchen door and saw Oswald open the right rear door of her brother's car and place the package in the back of the car. +LJ033-0086|Mrs. Randle estimated that the package was approximately 28 inches long and about 8 inches wide.|Mrs. Randle estimated that the package was approximately twenty-eight inches long and about eight inches wide. +LJ033-0087|She thought that its color was similar to that of the bag found on the sixth floor of the School Book Depository after the assassination.|She thought that its color was similar to that of the bag found on the sixth floor of the School Book Depository after the assassination. +LJ033-0088|Frazier met Oswald at the kitchen door and together they walked to the car.|Frazier met Oswald at the kitchen door and together they walked to the car. +LJ033-0089|After entering the car, Frazier glanced over his shoulder and noticed a brown paper package on the back seat.|After entering the car, Frazier glanced over his shoulder and noticed a brown paper package on the back seat. +LJ033-0090|He asked, quote, What's the package, Lee? End quote. Oswald replied, quote, curtain rods, end quote.|He asked, quote, What's the package, Lee? End quote. Oswald replied, quote, curtain rods, end quote. +LJ033-0091|Frazier told the Commission, quote,|Frazier told the Commission, quote, +LJ033-0092|the main reason he was going over there that Thursday afternoon when he was to bring back some curtain rods, so I didn't think any more about it when he told me that, end quote,|the main reason he was going over there that Thursday afternoon when he was to bring back some curtain rods, so I didn't think any more about it when he told me that, end quote, +LJ033-0093|Frazier estimated that the bag was 2 feet long, quote, give and take a few inches, end quote, and about 5 or 6 inches wide.|Frazier estimated that the bag was two feet long, quote, give and take a few inches, end quote, and about five or six inches wide. +LJ033-0094|As they sat in the car, Frazier asked Oswald where his lunch was, and Oswald replied that he was going to buy his lunch that day.|As they sat in the car, Frazier asked Oswald where his lunch was, and Oswald replied that he was going to buy his lunch that day. +LJ033-0095|Frazier testified that Oswald carried no lunch bag that day.|Frazier testified that Oswald carried no lunch bag that day. +LJ033-0096|quote, when he rode with me, I say he always brought lunch except that one day on November 22 he didn't bring his lunch that day, end quote,|quote, when he rode with me, I say he always brought lunch except that one day on November twenty-two he didn't bring his lunch that day, end quote, +LJ033-0097|Frazier parked the car in the company parking lot about 2 blocks north of the Depository Building.|Frazier parked the car in the company parking lot about two blocks north of the Depository Building. +LJ033-0098|Oswald left the car first, picked up the brown paper bag, and proceeded toward the building ahead of Frazier.|Oswald left the car first, picked up the brown paper bag, and proceeded toward the building ahead of Frazier. +LJ033-0099|Frazier walked behind and as they crossed the railroad tracks he watched the switching of the cars.|Frazier walked behind and as they crossed the railroad tracks he watched the switching of the cars. +LJ033-0100|Frazier recalled that one end of the package was under Oswald's armpit and the lower part was held with his right hand|Frazier recalled that one end of the package was under Oswald's armpit and the lower part was held with his right hand +LJ033-0101|so that it was carried straight and parallel to his body.|so that it was carried straight and parallel to his body. +LJ033-0102|When Oswald entered the rear door of the Depository Building, he was about 50 feet ahead of Frazier.|When Oswald entered the rear door of the Depository Building, he was about fifty feet ahead of Frazier. +LJ033-0103|It was the first time that Oswald had not walked with Frazier from the parking lot to the building entrance.|It was the first time that Oswald had not walked with Frazier from the parking lot to the building entrance. +LJ033-0104|When Frazier entered the building, he did not see Oswald.|When Frazier entered the building, he did not see Oswald. +LJ033-0105|One employee, Jack Dougherty,|One employee, Jack Dougherty, +LJ033-0106|believed that he saw Oswald coming to work, but he does not remember that Oswald had anything in his hands as he entered the door.|believed that he saw Oswald coming to work, but he does not remember that Oswald had anything in his hands as he entered the door. +LJ033-0107|No other employee has been found who saw Oswald enter that morning.|No other employee has been found who saw Oswald enter that morning. +LJ033-0108|In deciding whether Oswald carried the assassination weapon in the bag which Frazier and Mrs. Randle saw,|In deciding whether Oswald carried the assassination weapon in the bag which Frazier and Mrs. Randle saw, +LJ033-0109|the Commission has carefully considered the testimony of these two witnesses with regard to the length of the bag.|the Commission has carefully considered the testimony of these two witnesses with regard to the length of the bag. +LJ033-0110|Frazier and Mrs. Randle testified|Frazier and Mrs. Randle testified +LJ033-0111|that the bag which Oswald was carrying was approximately 27 or 28 inches long, whereas the wooden stock of the rifle, which is its largest component,|that the bag which Oswald was carrying was approximately twenty-seven or twenty-eight inches long, whereas the wooden stock of the rifle, which is its largest component, +LJ033-0112|measured 34.8 inches.|measured thirty-four point eight inches. +LJ033-0113|The bag found on the sixth floor was 88 inches long.|The bag found on the sixth floor was eighty-eight inches long. +LJ033-0114|When Frazier appeared before the Commission and was asked to demonstrate how Oswald carried the package, he said, quote, Like I said,|When Frazier appeared before the Commission and was asked to demonstrate how Oswald carried the package, he said, quote, Like I said, +LJ033-0115|I remember that I didn't look at the package very much but when I did look at it he did have his hands on the package like that, end quote, and at this point|I remember that I didn't look at the package very much but when I did look at it he did have his hands on the package like that, end quote, and at this point +LJ033-0116|Frazier placed the upper part of the package under his armpit and attempted to cup his right hand beneath the bottom of the bag.|Frazier placed the upper part of the package under his armpit and attempted to cup his right hand beneath the bottom of the bag. +LJ033-0117|The disassembled rifle was too long to be carried in this manner.|The disassembled rifle was too long to be carried in this manner. +LJ033-0118|Similarly, when the butt of the rifle was placed in Frazier's hand, it extended above his shoulder to ear level.|Similarly, when the butt of the rifle was placed in Frazier's hand, it extended above his shoulder to ear level. +LJ033-0119|Moreover, in an interview on December 1, 1963, with agents of the FBI, Frazier had marked the point on the back seat of his car|Moreover, in an interview on December one, nineteen sixty-three, with agents of the FBI, Frazier had marked the point on the back seat of his car +LJ033-0120|which he believed was where the bag reached when it was laid on the seat with one edge against the door.|which he believed was where the bag reached when it was laid on the seat with one edge against the door. +LJ033-0121|The distance between the point on the seat and the door was 27 inches.|The distance between the point on the seat and the door was twenty-seven inches. +LJ033-0122|Mrs. Randle said, when shown the paper bag,|Mrs. Randle said, when shown the paper bag, +LJ033-0123|that the bag she saw Oswald carrying, quote, wasn't that long, I mean it was folded down at the top as I told you. It definitely wasn't that long, end quote,|that the bag she saw Oswald carrying, quote, wasn't that long, I mean it was folded down at the top as I told you. It definitely wasn't that long, end quote, +LJ033-0124|And she folded the bag to length of about 28 and a half inches.|And she folded the bag to length of about twenty-eight and a half inches. +LJ033-0125|Frazier doubted whether the bag that Oswald carried was as wide as the bag found on the sixth floor, although Mrs. Randle testified|Frazier doubted whether the bag that Oswald carried was as wide as the bag found on the sixth floor, although Mrs. Randle testified +LJ033-0126|that the width was approximately the same.|that the width was approximately the same. +LJ033-0127|The Commission has weighed the visual recollection of Frazier and Mrs. Randle against the evidence here presented|The Commission has weighed the visual recollection of Frazier and Mrs. Randle against the evidence here presented +LJ033-0128|that the bag Oswald carried contained the assassination weapon and has concluded that Frazier and Randle are mistaken as to the length of the bag.|that the bag Oswald carried contained the assassination weapon and has concluded that Frazier and Randle are mistaken as to the length of the bag. +LJ033-0129|Mrs. Randle saw the bag fleetingly|Mrs. Randle saw the bag fleetingly +LJ033-0130|and her first remembrance is that it was held in Oswald's right hand, quote, and it almost touched the ground as he carried it, end quote.|and her first remembrance is that it was held in Oswald's right hand, quote, and it almost touched the ground as he carried it, end quote. +LJ033-0131|Frazier's view of the bag was from the rear. He continually advised that he was not paying close attention. For example, he said, quote,|Frazier's view of the bag was from the rear. He continually advised that he was not paying close attention. For example, he said, quote, +LJ033-0132|I didn't pay too much attention the way he was walking because I was walking along there looking at the railroad cars and watching the men on the diesel switch them cars|I didn't pay too much attention the way he was walking because I was walking along there looking at the railroad cars and watching the men on the diesel switch them cars +LJ033-0133|and I didn't pay too much attention on how he carried the package at all, end quote.|and I didn't pay too much attention on how he carried the package at all, end quote. +LJ033-0134|Frazier could easily have been mistaken when he stated that Oswald held the bottom of the bag cupped in his hand with the upper end tucked into his armpit.|Frazier could easily have been mistaken when he stated that Oswald held the bottom of the bag cupped in his hand with the upper end tucked into his armpit. +LJ033-0136|A handmade bag of wrapping paper and tape was found in the southeast corner of the sixth floor alongside the window from which the shots were fired.|A handmade bag of wrapping paper and tape was found in the southeast corner of the sixth floor alongside the window from which the shots were fired. +LJ033-0137|It was not a standard type bag which could be obtained in a store and it was presumably made for a particular purpose.|It was not a standard type bag which could be obtained in a store and it was presumably made for a particular purpose. +LJ033-0138|It was the appropriate size to contain, in disassembled form,|It was the appropriate size to contain, in disassembled form, +LJ033-0139|Oswald's Mannlicher-Carcano rifle, serial No. C2766, which was also found on the sixth floor.|Oswald's Mannlicher-Carcano rifle, serial Number C two seven six six, which was also found on the sixth floor. +LJ033-0140|Three cartons had been placed at the window apparently to act as a gun rest and a fourth carton was placed behind those at the window.|Three cartons had been placed at the window apparently to act as a gun rest and a fourth carton was placed behind those at the window. +LJ033-0141|A person seated on the fourth carton could assemble the rifle without being seen from the rest of the sixth floor|A person seated on the fourth carton could assemble the rifle without being seen from the rest of the sixth floor +LJ033-0142|because the cartons stacked around the southeast corner would shield him.|because the cartons stacked around the southeast corner would shield him. +LJ033-0143|The presence of the bag in this corner is cogent evidence that it was used as the container for the rifle.|The presence of the bag in this corner is cogent evidence that it was used as the container for the rifle. +LJ033-0144|At the time the bag was found, Lieutenant Day of the Dallas police wrote on it, quote, Found next to the sixth floor window gun fired from.|At the time the bag was found, Lieutenant Day of the Dallas police wrote on it, quote, Found next to the sixth floor window gun fired from. +LJ033-0145|May have been used to carry gun. Lt. J. C. Day, end quote.|May have been used to carry gun. Lt. J. C. Day, end quote. +LJ033-0146|Scientific Evidence Linking Rifle and Oswald to Paper Bag|Scientific Evidence Linking Rifle and Oswald to Paper Bag +LJ033-0147|Oswald's fingerprint and palmprint found on bag.|Oswald's fingerprint and palmprint found on bag. +LJ033-0148|Using a standard chemical method involving silver nitrates|Using a standard chemical method involving silver nitrates +LJ033-0149|the FBI Laboratory developed a latent palmprint and latent fingerprint on the bag.|the FBI Laboratory developed a latent palmprint and latent fingerprint on the bag. +LJ033-0150|Sebastian F. Latona, supervisor of the FBI's Latent Fingerprint Section,|Sebastian F. Latona, supervisor of the FBI's Latent Fingerprint Section, +LJ033-0151|identified these prints as the left index fingerprint and right palmprint of Lee Harvey Oswald|identified these prints as the left index fingerprint and right palmprint of Lee Harvey Oswald +LJ033-0152|The portion of the palm which was identified was the heel of the right palm, i.e., the area near the wrist, on the little finger side.|The portion of the palm which was identified was the heel of the right palm, i.e., the area near the wrist, on the little finger side. +LJ033-0153|These prints were examined independently by Ronald G. Wittmus of the FBI,|These prints were examined independently by Ronald G. Wittmus of the FBI, +LJ033-0154|and by Arthur Mandella, a fingerprint expert with the New York City Police Department.|and by Arthur Mandella, a fingerprint expert with the New York City Police Department. +LJ033-0155|Both concluded that the prints were the right palm and left index finger of Lee Oswald. No other identifiable prints were found on the bag.|Both concluded that the prints were the right palm and left index finger of Lee Oswald. No other identifiable prints were found on the bag. +LJ033-0156|Oswald's palmprint on the bottom of the paper bag indicated, of course, that he had handled the bag.|Oswald's palmprint on the bottom of the paper bag indicated, of course, that he had handled the bag. +LJ033-0157|Furthermore, it was consistent with the bag having contained a heavy or bulky object when he handled it since a light object is usually held by the fingers.|Furthermore, it was consistent with the bag having contained a heavy or bulky object when he handled it since a light object is usually held by the fingers. +LJ033-0158|The palmprint was found on the closed end of the bag.|The palmprint was found on the closed end of the bag. +LJ033-0159|It was from Oswald's right hand, in which he carried the long package as he walked from Frazier's car to the building.|It was from Oswald's right hand, in which he carried the long package as he walked from Frazier's car to the building. +LJ033-0160|Materials used to make bag.|Materials used to make bag. +LJ033-0161|On the day of the assassination, the Dallas police obtained a sample of wrapping paper and tape|On the day of the assassination, the Dallas police obtained a sample of wrapping paper and tape +LJ033-0162|from the shipping room of the Depository and forwarded it to the FBI Laboratory in Washington.|from the shipping room of the Depository and forwarded it to the FBI Laboratory in Washington. +LJ033-0163|James C. Cadigan, a questioned-documents expert with the Bureau, compared the samples with the paper and tape in the actual bag.|James C. Cadigan, a questioned-documents expert with the Bureau, compared the samples with the paper and tape in the actual bag. +LJ033-0164|He testified, quote,|He testified, quote, +LJ033-0165|In all of the observations and physical tests that I made I found the bag and the paper sample were the same, end quote.|In all of the observations and physical tests that I made I found the bag and the paper sample were the same, end quote. +LJ033-0166|Among other tests, the paper and tape were submitted to fiber analysis and spectrographic examination.|Among other tests, the paper and tape were submitted to fiber analysis and spectrographic examination. +LJ033-0167|In addition the tape was compared to determine whether the sample tape and the tape on the bag had been taken from the tape dispensing machine at the Depository.|In addition the tape was compared to determine whether the sample tape and the tape on the bag had been taken from the tape dispensing machine at the Depository. +LJ033-0168|When asked to explain the similarity of characteristics, Cadigan stated, quote, well briefly|When asked to explain the similarity of characteristics, Cadigan stated, quote, well briefly +LJ033-0169|it would be the thickness of both the paper and the tape, the color under various lighting conditions of both the paper and the tape,|it would be the thickness of both the paper and the tape, the color under various lighting conditions of both the paper and the tape, +LJ033-0170|the width of the tape, the knurled markings on the surface of the fiber, the texture of the fiber, the letting pattern|the width of the tape, the knurled markings on the surface of the fiber, the texture of the fiber, the letting pattern +LJ033-0171|I found that the paper sack found on the sixth floor and the sample|I found that the paper sack found on the sixth floor and the sample +LJ033-0172|had the same observable characteristics both under the microscope and all the visual tests that I could conduct.|had the same observable characteristics both under the microscope and all the visual tests that I could conduct. +LJ033-0173|The papers I also found were similar in fiber composition, therefore, in addition to the visual characteristics,|The papers I also found were similar in fiber composition, therefore, in addition to the visual characteristics, +LJ033-0174|microscopic and UV (ultra violet) characteristics, end quote.|microscopic and UV (ultra violet) characteristics, end quote. +LJ033-0175|Mr. Cadigan concluded that the paper and tape from the bag were identical in all respects to the sample paper and tape|Mr. Cadigan concluded that the paper and tape from the bag were identical in all respects to the sample paper and tape +LJ033-0176|taken from the Texas School Book Depository shipping room on November 22, 1963.|taken from the Texas School Book Depository shipping room on November twenty-two, nineteen sixty-three. +LJ033-0177|On December 1, 1963, a replica bag was made from materials found on that date in the shipping room.|On December one, nineteen sixty-three, a replica bag was made from materials found on that date in the shipping room. +LJ033-0178|This was done as an investigatory aid|This was done as an investigatory aid +LJ033-0179|since the original bag had been discolored during various laboratory examinations and could not be used for valid identification by witnesses.|since the original bag had been discolored during various laboratory examinations and could not be used for valid identification by witnesses. +LJ033-0180|Cadigan found that the paper used to make this replica sack had different characteristics from the paper in the original bag.|Cadigan found that the paper used to make this replica sack had different characteristics from the paper in the original bag. +LJ033-0181|The science of paper analysis enabled him to distinguish between different rolls of paper even though they were produced by the same manufacturer.|The science of paper analysis enabled him to distinguish between different rolls of paper even though they were produced by the same manufacturer. +LJ033-0182|Since the Depository normally used approximately one roll of paper every 3 working days,|Since the Depository normally used approximately one roll of paper every three working days, +LJ033-0183|it was not surprising that the replica sack made on December 1, 1963,|it was not surprising that the replica sack made on December one, nineteen sixty-three, +LJ033-0184|had different characteristics from both the actual bag and the sample taken on November 22.|had different characteristics from both the actual bag and the sample taken on November twenty-two. +LJ033-0185|On the other hand, since two rolls could be made from the same batch of paper,|On the other hand, since two rolls could be made from the same batch of paper, +LJ033-0186|one cannot estimate when, prior to November 22, Oswald made the paper bag.|one cannot estimate when, prior to November twenty-two, Oswald made the paper bag. +LJ033-0187|However, the complete identity of characteristics between the paper and tape in the bag found on the sixth floor|However, the complete identity of characteristics between the paper and tape in the bag found on the sixth floor +LJ033-0188|and the paper and tape found in the shipping room of the Depository on November 22|and the paper and tape found in the shipping room of the Depository on November twenty-two +LJ033-0189|enabled the Commission to conclude that the bag was made from these materials.|enabled the Commission to conclude that the bag was made from these materials. +LJ033-0190|The Depository shipping department was on the first floor to which Oswald had access in the normal performance of his duties filling orders.|The Depository shipping department was on the first floor to which Oswald had access in the normal performance of his duties filling orders. +LJ033-0191|Fibers in paper bag matched fibers in blanket|Fibers in paper bag matched fibers in blanket +LJ033-0192|When Paul M. Stombaugh of the FBI Laboratory examined the paper bag,|When Paul M. Stombaugh of the FBI Laboratory examined the paper bag, +LJ033-0193|he found, on the inside, a single brown delustered viscose fiber and several light green cotton fibers.|he found, on the inside, a single brown delustered viscose fiber and several light green cotton fibers. +LJ033-0194|The blanket in which the rifle was stored was composed of brown and green cotton, viscose and woolen fibers.|The blanket in which the rifle was stored was composed of brown and green cotton, viscose and woolen fibers. +LJ033-0195|The single brown viscose fiber found in the bag matched some of the brown viscose fibers from the blanket in all observable characteristics.|The single brown viscose fiber found in the bag matched some of the brown viscose fibers from the blanket in all observable characteristics. +LJ033-0196|The green cotton fibers found in the paper bag matched some of the green cotton fibers in the blanket|The green cotton fibers found in the paper bag matched some of the green cotton fibers in the blanket +LJ033-0197|quote, in all observable microscopic characteristics, end quote.|quote, in all observable microscopic characteristics, end quote. +LJ033-0198|Despite these matches, however, Stombaugh was unable to render on opinion that the fibers which he found in the bag had probably come from the blanket,|Despite these matches, however, Stombaugh was unable to render on opinion that the fibers which he found in the bag had probably come from the blanket, +LJ033-0199|because other types of fibers present in the blanket were not found in the bag.|because other types of fibers present in the blanket were not found in the bag. +LJ033-0200|He concluded, quote,|He concluded, quote, +LJ033-0201|All I would say here is that it is possible that these fibers could have come from this blanket.|All I would say here is that it is possible that these fibers could have come from this blanket. +LJ033-0202|because this blanket is composed of brown and green woolen fibers, brown and green delustered viscose fibers, and brown and green cotton fibers.|because this blanket is composed of brown and green woolen fibers, brown and green delustered viscose fibers, and brown and green cotton fibers. +LJ033-0203|We found no brown cotton fibers, no green viscose fibers, and no woolen fibers.|We found no brown cotton fibers, no green viscose fibers, and no woolen fibers. +LJ033-0204|So if I found all of these then I would have been able to say these fibers probably had come from this blanket. But since I found so few,|So if I found all of these then I would have been able to say these fibers probably had come from this blanket. But since I found so few, +LJ033-0205|then I would say the possibility exists, these fibers could have come from this blanket, end quote.|then I would say the possibility exists, these fibers could have come from this blanket, end quote. +LJ033-0206|Stombaugh confirmed that the rifle could have picked up fibers from the blanket and transferred them to the paper bag.|Stombaugh confirmed that the rifle could have picked up fibers from the blanket and transferred them to the paper bag. +LJ033-0207|In light of the other evidence linking Lee Harvey Oswald, the blanket, and the rifle to the paper bag found on the sixth floor,|In light of the other evidence linking Lee Harvey Oswald, the blanket, and the rifle to the paper bag found on the sixth floor, +LJ033-0208|the Commission considered Stombaugh's testimony of probative value in deciding whether Oswald carried the rifle into the building in the paper bag.|the Commission considered Stombaugh's testimony of probative value in deciding whether Oswald carried the rifle into the building in the paper bag. +LJ033-0209|The preponderance of the evidence supports the conclusion that Lee Harvey Oswald (1)|The preponderance of the evidence supports the conclusion that Lee Harvey Oswald (one) +LJ033-0210|told the curtain rod story to Frazier to explain both the return to Irving on a Thursday|told the curtain rod story to Frazier to explain both the return to Irving on a Thursday +LJ033-0211|and the obvious bulk of the package which he intended to bring to work the next day;|and the obvious bulk of the package which he intended to bring to work the next day; +LJ033-0212|(2) took paper and tape from the wrapping bench of the Depository and fashioned a bag large enough to carry the disassembled rifle;|(two) took paper and tape from the wrapping bench of the Depository and fashioned a bag large enough to carry the disassembled rifle; +LJ033-0213|(3) removed the rifle from the blanket in the Paines' garage on Thursday evening;|(three) removed the rifle from the blanket in the Paines' garage on Thursday evening; +LJ033-0214|(4) carried the rifle into the Depository Building, concealed in the bag;|(four) carried the rifle into the Depository Building, concealed in the bag; +LJ034-0001|Report of the President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. +LJ034-0002|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter 4. The Assassin: Part 3.|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter four. The Assassin: Part three. +LJ034-0003|Oswald at Window|Oswald at Window +LJ034-0004|Lee Harvey Oswald was hired on October 15, 1963, by the Texas School Book Depository as an "order filler."|Lee Harvey Oswald was hired on October fifteen, nineteen sixty-three, by the Texas School Book Depository as an "order filler." +LJ034-0005|He worked principally on the first and sixth floors of the building, gathering books listed on orders and delivering them to the shipping room on the first floor.|He worked principally on the first and sixth floors of the building, gathering books listed on orders and delivering them to the shipping room on the first floor. +LJ034-0006|He had ready access to the sixth floor, from the southeast corner window of which the shots were fired.|He had ready access to the sixth floor, from the southeast corner window of which the shots were fired. +LJ034-0007|The Commission evaluated the physical evidence found near the window after the assassination and the testimony of eyewitnesses|The Commission evaluated the physical evidence found near the window after the assassination and the testimony of eyewitnesses +LJ034-0008|in deciding whether Lee Harvey Oswald was present at this window at the time of the assassination.|in deciding whether Lee Harvey Oswald was present at this window at the time of the assassination. +LJ034-0009|Palmprints and Fingerprints on Cartons and Paper Bag|Palmprints and Fingerprints on Cartons and Paper Bag +LJ034-0010|Below the southeast corner window on the sixth floor was a large carton of books|Below the southeast corner window on the sixth floor was a large carton of books +LJ034-0011|measuring approximately 18 by 12 by 14 inches which had been moved from a stack along the south wall.|measuring approximately eighteen by twelve by fourteen inches which had been moved from a stack along the south wall. +LJ034-0012|Atop this carton was a small carton marked "Rolling Readers," measuring approximately 13 by 9 by 8 inches.|Atop this carton was a small carton marked "Rolling Readers," measuring approximately thirteen by nine by eight inches. +LJ034-0013|In front of this small carton and resting partially on the windowsill was another small "Rolling Readers" carton.|In front of this small carton and resting partially on the windowsill was another small "Rolling Readers" carton. +LJ034-0014|These two small cartons had been moved from a stack about three aisles away.|These two small cartons had been moved from a stack about three aisles away. +LJ034-0015|The boxes in the window appeared to have been arranged as a convenient gun rest.|The boxes in the window appeared to have been arranged as a convenient gun rest. +LJ034-0016|Behind these boxes was another carton placed on the floor on which a man sitting|Behind these boxes was another carton placed on the floor on which a man sitting +LJ034-0017|could look southwesterly down Elm Street over the top of the "Rolling Readers" cartons.|could look southwesterly down Elm Street over the top of the "Rolling Readers" cartons. +LJ034-0018|Next to these cartons was the handmade paper bag, previously discussed,|Next to these cartons was the handmade paper bag, previously discussed, +LJ034-0019|on which appeared the print of the left index finger and right palm of Lee Harvey Oswald.|on which appeared the print of the left index finger and right palm of Lee Harvey Oswald. +LJ034-0020|The cartons were forwarded to the FBI in Washington.|The cartons were forwarded to the FBI in Washington. +LJ034-0021|Sebastian F. Latona, supervisor of the Latent Fingerprint Section,|Sebastian F. Latona, supervisor of the Latent Fingerprint Section, +LJ034-0022|testified that 20 identifiable fingerprints and 8 palmprints were developed on these cartons.|testified that twenty identifiable fingerprints and eight palmprints were developed on these cartons. +LJ034-0023|The carton on the windowsill and the large carton below the window contained no prints which could be identified as being those of Lee Harvey Oswald.|The carton on the windowsill and the large carton below the window contained no prints which could be identified as being those of Lee Harvey Oswald. +LJ034-0024|The other "Rolling Readers" carton, however, contained a palmprint and a fingerprint which were identified by Latona|The other "Rolling Readers" carton, however, contained a palmprint and a fingerprint which were identified by Latona +LJ034-0025|as being the left palmprint and right index fingerprint of Lee Harvey Oswald.|as being the left palmprint and right index fingerprint of Lee Harvey Oswald. +LJ034-0026|The Commission has considered the possibility that the cartons might have been moved in connection with the work that was being performed on the sixth floor on November 22.|The Commission has considered the possibility that the cartons might have been moved in connection with the work that was being performed on the sixth floor on November twenty-two. +LJ034-0027|Depository employees were laying a new floor at the west end and transferring books from the west to the east end of the building.|Depository employees were laying a new floor at the west end and transferring books from the west to the east end of the building. +LJ034-0028|The "Rolling Readers" cartons, however, had not been moved by the floor layers and had apparently been taken to the window from their regular position for some particular purpose.|The "Rolling Readers" cartons, however, had not been moved by the floor layers and had apparently been taken to the window from their regular position for some particular purpose. +LJ034-0029|The "Rolling Readers" boxes contained, instead of books, light blocks used as reading aids.|The "Rolling Readers" boxes contained, instead of books, light blocks used as reading aids. +LJ034-0030|They could be easily adjusted and were still solid enough to serve as a gun rest.|They could be easily adjusted and were still solid enough to serve as a gun rest. +LJ034-0031|The box on the floor, behind the three near the window,|The box on the floor, behind the three near the window, +LJ034-0032|had been one of these moved by the floor layers from the west wall to near the east side of the building in preparation for the laying of the floor.|had been one of these moved by the floor layers from the west wall to near the east side of the building in preparation for the laying of the floor. +LJ034-0033|During the afternoon of November 22,|During the afternoon of November twenty-two, +LJ034-0034|Lieutenant Day of the Dallas police dusted this carton with powder and developed a palmprint on the top edge of the carton on the side nearest the window.|Lieutenant Day of the Dallas police dusted this carton with powder and developed a palmprint on the top edge of the carton on the side nearest the window. +LJ034-0035|The position of this palmprint on the carton was parallel with the long axis of the box, and at right angles with the short axis;|The position of this palmprint on the carton was parallel with the long axis of the box, and at right angles with the short axis; +LJ034-0036|the bottom of the palm rested on the box.|the bottom of the palm rested on the box. +LJ034-0037|Someone sitting on the box facing the window would have his palm in this position if he placed his hand alongside his right hip.|Someone sitting on the box facing the window would have his palm in this position if he placed his hand alongside his right hip. +LJ034-0038|This print which had been cut out of the box was also forwarded to the FBI and Latona identified it as Oswald's right palmprint.|This print which had been cut out of the box was also forwarded to the FBI and Latona identified it as Oswald's right palmprint. +LJ034-0039|In Latona's opinion, quote,|In Latona's opinion, quote, +LJ034-0040|not too long, end quote, a time had elapsed between the time that the print was placed on the carton and the time that it had been developed by the Dallas police.|not too long, end quote, a time had elapsed between the time that the print was placed on the carton and the time that it had been developed by the Dallas police. +LJ034-0041|Although Bureau experiments had shown that 24 hours was a likely maximum time, Latona stated|Although Bureau experiments had shown that twenty-four hours was a likely maximum time, Latona stated +LJ034-0042|that he could only testify with certainty that the print was less than 3 days old.|that he could only testify with certainty that the print was less than three days old. +LJ034-0043|The print, therefore, could have been placed on the carton at any time within this period.|The print, therefore, could have been placed on the carton at any time within this period. +LJ034-0044|The freshness of this print could be estimated only because the Dallas police developed it through the use of powder.|The freshness of this print could be estimated only because the Dallas police developed it through the use of powder. +LJ034-0045|Since cartons absorb perspiration, powder can successfully develop a print on such material only within a limited time.|Since cartons absorb perspiration, powder can successfully develop a print on such material only within a limited time. +LJ034-0046|When the FBI in Washington received the cartons, the remaining prints, including Oswald's on the Rolling Readers carton,|When the FBI in Washington received the cartons, the remaining prints, including Oswald's on the Rolling Readers carton, +LJ034-0047|were developed by chemical processes.|were developed by chemical processes. +LJ034-0048|The freshness of prints developed in this manner cannot be estimated,|The freshness of prints developed in this manner cannot be estimated, +LJ034-0049|so no conclusions can be drawn as to whether these remaining prints preceded or followed the print developed in Dallas by powder.|so no conclusions can be drawn as to whether these remaining prints preceded or followed the print developed in Dallas by powder. +LJ034-0050|Most of the prints were found to have been placed on the cartons by an FBI clerk|Most of the prints were found to have been placed on the cartons by an FBI clerk +LJ034-0051|and a Dallas police officer after the cartons had been processed with powder by the Dallas Police.|and a Dallas police officer after the cartons had been processed with powder by the Dallas Police. +LJ034-0052|In his independent investigation, Arthur Mandella of the New York City Police Department|In his independent investigation, Arthur Mandella of the New York City Police Department +LJ034-0053|reached the same conclusion as Latona that the prints found on the cartons were those of Lee Harvey Oswald.|reached the same conclusion as Latona that the prints found on the cartons were those of Lee Harvey Oswald. +LJ034-0054|In addition, Mandella was of the opinion that the print taken from the carton on the floor|In addition, Mandella was of the opinion that the print taken from the carton on the floor +LJ034-0055|was probably made within a day or a day and a half of the examination on November 22.|was probably made within a day or a day and a half of the examination on November twenty-two. +LJ034-0056|Another expert with the FBI, Ronald G. Wittmus, conducted a separate examination and also agreed with Latona that the prints were Oswald's.|Another expert with the FBI, Ronald G. Wittmus, conducted a separate examination and also agreed with Latona that the prints were Oswald's. +LJ034-0057|In evaluating the significance of these fingerprint and palmprint identifications,|In evaluating the significance of these fingerprint and palmprint identifications, +LJ034-0058|the Commission considered the possibility that Oswald handled these cartons as part of his normal duties.|the Commission considered the possibility that Oswald handled these cartons as part of his normal duties. +LJ034-0059|Since other identifiable prints were developed on the cartons, the Commission requested that they be compared with the prints of the 12 warehouse employees|Since other identifiable prints were developed on the cartons, the Commission requested that they be compared with the prints of the twelve warehouse employees +LJ034-0060|who, like Oswald, might have handled the cartons. They were also compared with the prints of those law enforcement officials who might have handled the cartons.|who, like Oswald, might have handled the cartons. They were also compared with the prints of those law enforcement officials who might have handled the cartons. +LJ034-0061|The results of this investigation are fully discussed in chapter 6, page 249.|The results of this investigation are fully discussed in chapter six, page two forty-nine. +LJ034-0062|Although a person could handle a carton and not leave identifiable prints,|Although a person could handle a carton and not leave identifiable prints, +LJ034-0063|none of these employees except Oswald left identifiable prints on the cartons.|none of these employees except Oswald left identifiable prints on the cartons. +LJ034-0064|This finding, in addition to the freshness of one of the prints and the presence of Oswald's prints on two of the four cartons and the paper bag|This finding, in addition to the freshness of one of the prints and the presence of Oswald's prints on two of the four cartons and the paper bag +LJ034-0065|led the Commission to attach some probative value to the fingerprint and palmprint identifications in reaching the conclusion|led the Commission to attach some probative value to the fingerprint and palmprint identifications in reaching the conclusion +LJ034-0066|that Oswald was at the window from which the shots were fired, although the prints do not establish the exact time he was there.|that Oswald was at the window from which the shots were fired, although the prints do not establish the exact time he was there. +LJ034-0067|Oswald's Presence on Sixth Floor Approximately 35 Minutes Before the Assassination|Oswald's Presence on Sixth Floor Approximately thirty-five Minutes Before the Assassination +LJ034-0068|Additional testimony linking Oswald with the point from which the shots were fired|Additional testimony linking Oswald with the point from which the shots were fired +LJ034-0069|was provided by the testimony of Charles Givens, who was the last known employee to see Oswald inside the building prior to the assassination.|was provided by the testimony of Charles Givens, who was the last known employee to see Oswald inside the building prior to the assassination. +LJ034-0070|During the morning of November 22, Givens was working with the floor-laying crew in the southwest section of the sixth floor.|During the morning of November twenty-two, Givens was working with the floor-laying crew in the southwest section of the sixth floor. +LJ034-0071|At about 11:45 a.m. the floor-laying crew used both elevators to come down from the sixth floor.|At about eleven:forty-five a.m. the floor-laying crew used both elevators to come down from the sixth floor. +LJ034-0072|The employees raced the elevators to the first floor. Givens saw Oswald standing at the gate on the fifth floor as the elevator went by.|The employees raced the elevators to the first floor. Givens saw Oswald standing at the gate on the fifth floor as the elevator went by. +LJ034-0073|Givens testified that after reaching the first floor, quote,|Givens testified that after reaching the first floor, quote, +LJ034-0074|I discovered I left my cigarettes in my jacket pocket upstairs, and I took the elevator back upstairs to get my jacket with my cigarettes in it, end quote.|I discovered I left my cigarettes in my jacket pocket upstairs, and I took the elevator back upstairs to get my jacket with my cigarettes in it, end quote. +LJ034-0075|He saw Oswald, a clipboard in hand, walking from the southeast corner of the sixth floor toward the elevator.|He saw Oswald, a clipboard in hand, walking from the southeast corner of the sixth floor toward the elevator. +LJ034-0076|Givens said to Oswald, quote, Boy are you going downstairs? It's near lunch time, end quote.|Givens said to Oswald, quote, Boy are you going downstairs? It's near lunch time, end quote. +LJ034-0077|Oswald said, quote, No, sir. When you get downstairs, close the gate to the elevator, end quote.|Oswald said, quote, No, sir. When you get downstairs, close the gate to the elevator, end quote. +LJ034-0078|Oswald was referring to the west elevator which operates by pushbutton and only with the gate closed. Givens said, "Okay,"|Oswald was referring to the west elevator which operates by pushbutton and only with the gate closed. Givens said, "Okay," +LJ034-0079|and rode down in the east elevator.|and rode down in the east elevator. +LJ034-0080|When he reached the first floor, the west elevator -- the one with the gate was not there.|When he reached the first floor, the west elevator -- the one with the gate was not there. +LJ034-0081|Givens thought this was about 11:55 a.m.|Givens thought this was about eleven:fifty-five a.m. +LJ034-0082|None of the Depository employees is known to have seen Oswald again until after the shooting.|None of the Depository employees is known to have seen Oswald again until after the shooting. +LJ034-0084|became apparent on December 2, 1963, when an employee, Frankie Kaiser,|became apparent on December two, nineteen sixty-three, when an employee, Frankie Kaiser, +LJ034-0085|found a clipboard hidden by book cartons in the northwest corner of the sixth floor at the west wall a few feet from where the rifle had been found.|found a clipboard hidden by book cartons in the northwest corner of the sixth floor at the west wall a few feet from where the rifle had been found. +LJ034-0086|This clipboard had been made by Kaiser and had his name on it.|This clipboard had been made by Kaiser and had his name on it. +LJ034-0087|Kaiser identified it as the clipboard which Oswald had appropriated from him when Oswald came to work at the Depository.|Kaiser identified it as the clipboard which Oswald had appropriated from him when Oswald came to work at the Depository. +LJ034-0088|Three invoices on this clipboard, each dated November 22, were for Scott-Foresman books, located on the first and sixth floors.|Three invoices on this clipboard, each dated November twenty-two, were for Scott-Foresman books, located on the first and sixth floors. +LJ034-0089|Oswald had not filled any of the three orders.|Oswald had not filled any of the three orders. +LJ034-0090|Eyewitness Identification of Assassin|Eyewitness Identification of Assassin +LJ034-0091|Howard L. Brennan was an eyewitness to the shooting. As indicated previously the Commission considered his testimony as probative|Howard L. Brennan was an eyewitness to the shooting. As indicated previously the Commission considered his testimony as probative +LJ034-0092|in reaching the conclusion that the shots came from the sixth floor, southeast corner window of the Depository Building.|in reaching the conclusion that the shots came from the sixth floor, southeast corner window of the Depository Building. +LJ034-0093|Brennan also testified that Lee Harvey Oswald,|Brennan also testified that Lee Harvey Oswald, +LJ034-0094|whom he viewed in a police lineup on the night of the assassination, was the man he saw fire the shots from the sixth-floor window of the Depository Building.|whom he viewed in a police lineup on the night of the assassination, was the man he saw fire the shots from the sixth-floor window of the Depository Building. +LJ034-0095|When the shots were fired, Brennan was in an excellent position to observe anyone in the window. He was sitting on a concrete wall|When the shots were fired, Brennan was in an excellent position to observe anyone in the window. He was sitting on a concrete wall +LJ034-0096|on the southwest corner of Elm and Houston Streets, looking north at the Depository Building which was directly in front of him.|on the southwest corner of Elm and Houston Streets, looking north at the Depository Building which was directly in front of him. +LJ034-0097|The window was approximately 120 feet away.|The window was approximately one hundred twenty feet away. +LJ034-0098|In the 6 to 8 minute period before the motorcade arrived, Brennan saw a man leave and return to the window, quote, a couple of times, end quote,|In the six to eight minute period before the motorcade arrived, Brennan saw a man leave and return to the window, quote, a couple of times, end quote, +LJ034-0099|After hearing the first shot, which he thought was a motorcycle backfire, Brennan glanced up at the window. He testified that, quote,|After hearing the first shot, which he thought was a motorcycle backfire, Brennan glanced up at the window. He testified that, quote, +LJ034-0100|this man I saw previously was aiming for his last shot. As it appeared to me he was standing up and resting against the left window sill, end quote.|this man I saw previously was aiming for his last shot. As it appeared to me he was standing up and resting against the left window sill, end quote. +LJ034-0101|Brennan saw the man fire the last shot and disappear from the window.|Brennan saw the man fire the last shot and disappear from the window. +LJ034-0102|Within minutes of the assassination, Brennan described the man to the police.|Within minutes of the assassination, Brennan described the man to the police. +LJ034-0103|This description most probably led to the radio alert sent to police cars at approximately 12:45 p.m., which described the suspect as white,|This description most probably led to the radio alert sent to police cars at approximately twelve:forty-five p.m., which described the suspect as white, +LJ034-0104|slender, weighing about one sixty-five pounds, about 5 foot 10 inches tall, and in his early thirties.|slender, weighing about one sixty-five pounds, about five foot ten inches tall, and in his early thirties. +LJ034-0105|In his sworn statement to the police later that day,|In his sworn statement to the police later that day, +LJ034-0106|Brennan described the man in similar terms, except that he gave the weight as between 165 and 175 pounds and the height was omitted.|Brennan described the man in similar terms, except that he gave the weight as between one hundred sixty-five and one hundred seventy-five pounds and the height was omitted. +LJ034-0107|In his testimony before the Commission, Brennan described the person he saw as, quote,|In his testimony before the Commission, Brennan described the person he saw as, quote, +LJ034-0108|man in his early thirties, fair complexion, slender, but neat, neat slender, possible 5 foot 10|man in his early thirties, fair complexion, slender, but neat, neat slender, possible five foot ten +LJ034-0109|one-sixty to one-seventy pounds.|one-sixty to one-seventy pounds. +LJ034-0110|Oswald was 5 foot 9 inches, slender and 24 years old.|Oswald was five foot nine inches, slender and twenty-four years old. +LJ034-0111|When arrested, he gave his weight as 140 pounds. On other occasions he gave weights of both 140 and 150 pounds.|When arrested, he gave his weight as one hundred forty pounds. On other occasions he gave weights of both one hundred forty and one hundred fifty pounds. +LJ034-0112|The New Orleans police records of his arrest in August of 1963 show a weight of 136 pounds.|The New Orleans police records of his arrest in August of nineteen sixty-three show a weight of one hundred thirty-six pounds. +LJ034-0113|The autopsy report indicated an estimated weight of 150 pounds.|The autopsy report indicated an estimated weight of one hundred fifty pounds. +LJ034-0114|Brennan's description should also be compared with the eyewitness description broadcast over the Dallas police radio at 1:22 p.m.|Brennan's description should also be compared with the eyewitness description broadcast over the Dallas police radio at one:twenty-two p.m. +LJ034-0115|of the man who shot Patrolman J. D. Tippit.|of the man who shot Patrolman J. D. Tippit. +LJ034-0116|The suspect was described as, quote, a white male about 30, 5 foot 8, black hair, slender, end quote.|The suspect was described as, quote, a white male about thirty, five foot eight, black hair, slender, end quote. +LJ034-0117|At 1:29 p.m. the police radio reported|At one:twenty-nine p.m. the police radio reported +LJ034-0118|that the description of the suspect in the Tippit shooting was similar to the description which had been given by Brennan in connection with the assassination.|that the description of the suspect in the Tippit shooting was similar to the description which had been given by Brennan in connection with the assassination. +LJ034-0119|Approximately 7 or 8 minutes later|Approximately seven or eight minutes later +LJ034-0120|the police radio reported that, quote, an eyeball witness, end quote, described the suspect in the Tippit shooting as, quote, a white male,|the police radio reported that, quote, an eyeball witness, end quote, described the suspect in the Tippit shooting as, quote, a white male, +LJ034-0121|27, 5 foot 11, 165 pounds, black wavy hair, end quote,|twenty-seven, five foot eleven, one hundred sixty-five pounds, black wavy hair, end quote, +LJ034-0122|As will be discussed fully below, the Commission has concluded that this suspect was Lee Harvey Oswald.|As will be discussed fully below, the Commission has concluded that this suspect was Lee Harvey Oswald. +LJ034-0123|Although Brennan testified that the man in the window was standing when he fired the shots, most probably he was either sitting or kneeling.|Although Brennan testified that the man in the window was standing when he fired the shots, most probably he was either sitting or kneeling. +LJ034-0124|The half-open window, the arrangement of the boxes, and the angle of the shots virtually preclude a standing position.|The half-open window, the arrangement of the boxes, and the angle of the shots virtually preclude a standing position. +LJ034-0125|It is understandable, however, for Brennan to have believed that the man with the rifle was standing. A photograph of the building taken seconds after the assassination|It is understandable, however, for Brennan to have believed that the man with the rifle was standing. A photograph of the building taken seconds after the assassination +LJ034-0126|shows three employees looking out of the fifth-floor window directly below the window from which the shots were fired.|shows three employees looking out of the fifth-floor window directly below the window from which the shots were fired. +LJ034-0127|Brennan testified that they were standing, which is their apparent position in the photograph.|Brennan testified that they were standing, which is their apparent position in the photograph. +LJ034-0128|But the testimony of these employees, together with photographs subsequently taken of them at the scene of the assassination,|But the testimony of these employees, together with photographs subsequently taken of them at the scene of the assassination, +LJ034-0129|establishes that they were either squatting or kneeling.|establishes that they were either squatting or kneeling. +LJ034-0130|Since the window ledges in the Depository Building are lower than in most buildings,|Since the window ledges in the Depository Building are lower than in most buildings, +LJ034-0131|a person squatting or kneeling exposes more of his body than would normally be the case.|a person squatting or kneeling exposes more of his body than would normally be the case. +LJ034-0132|From the street, this creates the impression that the person is standing.|From the street, this creates the impression that the person is standing. +LJ034-0133|Brennan could have seen enough of the body of a kneeling or squatting person to estimate his height.|Brennan could have seen enough of the body of a kneeling or squatting person to estimate his height. +LJ034-0134|Shortly after the assassination Brennan noticed|Shortly after the assassination Brennan noticed +LJ034-0135|two of these employees leaving the building and immediately identified them as having been in the fifth-floor windows.|two of these employees leaving the building and immediately identified them as having been in the fifth-floor windows. +LJ034-0136|When the three employees appeared before the Commission, Brennan identified the two whom he saw leave the building.|When the three employees appeared before the Commission, Brennan identified the two whom he saw leave the building. +LJ034-0137|The two men, Harold Norman and James Jarman, Jr., each confirmed that when they came out of the building,|The two men, Harold Norman and James Jarman, Jr., each confirmed that when they came out of the building, +LJ034-0138|they saw and heard Brennan describing what he had seen.|they saw and heard Brennan describing what he had seen. +LJ034-0140|Jarman heard Brennan, quote, talking to this officer about that he had heard these shots and he had seen the barrel of the gun sticking out the window,|Jarman heard Brennan, quote, talking to this officer about that he had heard these shots and he had seen the barrel of the gun sticking out the window, +LJ034-0141|and he said that the shots came from inside the building, end quote.|and he said that the shots came from inside the building, end quote. +LJ034-0142|During the evening of November 22, Brennan identified Oswald as the person in the lineup who bore the closest resemblance to the man in the window|During the evening of November twenty-two, Brennan identified Oswald as the person in the lineup who bore the closest resemblance to the man in the window +LJ034-0143|but he said he was unable to make a positive identification.|but he said he was unable to make a positive identification. +LJ034-0144|Prior to the lineup,|Prior to the lineup, +LJ034-0145|Brennan had seen Oswald's picture on television, and he told the Commission that whether this affected his identification, quote, is something I do not know.|Brennan had seen Oswald's picture on television, and he told the Commission that whether this affected his identification, quote, is something I do not know. +LJ034-0146|In an interview with FBI agents on December 17, 1963,|In an interview with FBI agents on December seventeen, nineteen sixty-three, +LJ034-0147|Brennan stated that he was sure that the person firing the rifle was Oswald.|Brennan stated that he was sure that the person firing the rifle was Oswald. +LJ034-0148|In another interview with FBI agents on January 7, 1964,|In another interview with FBI agents on January seven, nineteen sixty-four, +LJ034-0149|Brennan appeared to revert to his earlier inability to make a positive identification,|Brennan appeared to revert to his earlier inability to make a positive identification, +LJ034-0150|but, in his testimony before the Commission, Brennan stated that his remarks of January 7 were intended by him merely as an accurate report|but, in his testimony before the Commission, Brennan stated that his remarks of January seven were intended by him merely as an accurate report +LJ034-0151|of what he said on November 22.|of what he said on November twenty-two. +LJ034-0152|Brennan told the Commission that he could have made a positive identification in the lineup on November 22|Brennan told the Commission that he could have made a positive identification in the lineup on November twenty-two +LJ034-0153|but did not do so because he felt that the assassination was, quote, a Communist activity,|but did not do so because he felt that the assassination was, quote, a Communist activity, +LJ034-0154|and I felt like there hadn't been more than one eyewitness, and if it got to be a known fact that I was an eyewitness, my family or I, either one, might not be safe.|and I felt like there hadn't been more than one eyewitness, and if it got to be a known fact that I was an eyewitness, my family or I, either one, might not be safe. +LJ034-0155|When specifically asked before the Commission|When specifically asked before the Commission +LJ034-0156|whether or not he could positively identify the man he saw in the sixth-floor window as the same man he saw in the police station,|whether or not he could positively identify the man he saw in the sixth-floor window as the same man he saw in the police station, +LJ034-0157|Brennan stated, quote, I could at that time -- I could, with all sincerity, identify him as being the same man, end quote.|Brennan stated, quote, I could at that time -- I could, with all sincerity, identify him as being the same man, end quote. +LJ034-0158|Although the record indicates that Brennan was an accurate observer, he declined to make a positive identification of Oswald when he first saw him in the police lineup.|Although the record indicates that Brennan was an accurate observer, he declined to make a positive identification of Oswald when he first saw him in the police lineup. +LJ034-0159|The Commission, therefore, does not base its conclusion concerning the identity of the assassin|The Commission, therefore, does not base its conclusion concerning the identity of the assassin +LJ034-0160|on Brennan's subsequent certain identification of Lee Harvey Oswald as the man he saw fire the rifle.|on Brennan's subsequent certain identification of Lee Harvey Oswald as the man he saw fire the rifle. +LJ034-0161|Immediately after the assassination, however,|Immediately after the assassination, however, +LJ034-0162|Brennan described to the police the man he saw in the window and then identified Oswald as the person who most nearly resembled the man he saw.|Brennan described to the police the man he saw in the window and then identified Oswald as the person who most nearly resembled the man he saw. +LJ034-0163|The Commission is satisfied that, at the least,|The Commission is satisfied that, at the least, +LJ034-0164|Brennan saw a man in the window who closely resembled Lee Harvey Oswald, and that Brennan believes the man he saw was in fact|Brennan saw a man in the window who closely resembled Lee Harvey Oswald, and that Brennan believes the man he saw was in fact +LJ034-0165|Lee Harvey Oswald.|Lee Harvey Oswald. +LJ034-0166|Two other witnesses were able to offer partial descriptions of a man they saw in the southeast corner window|Two other witnesses were able to offer partial descriptions of a man they saw in the southeast corner window +LJ034-0167|of the sixth floor approximately 1 minute before the assassination, although neither witness saw the shots being fired.|of the sixth floor approximately one minute before the assassination, although neither witness saw the shots being fired. +LJ034-0168|Ronald Fischer and Robert Edwards were standing on the curb at the southwest corner of Elm and Houston Streets,|Ronald Fischer and Robert Edwards were standing on the curb at the southwest corner of Elm and Houston Streets, +LJ034-0169|the same corner where Brennan was sitting on a concrete wall.|the same corner where Brennan was sitting on a concrete wall. +LJ034-0170|Fischer testified that about 10 or 15 seconds before the motorcade turned onto Houston Street from Main Street,|Fischer testified that about ten or fifteen seconds before the motorcade turned onto Houston Street from Main Street, +LJ034-0171|Edwards said, quote, Look at that guy there in that window, end quote,|Edwards said, quote, Look at that guy there in that window, end quote, +LJ034-0172|Fischer looked up and watched the man in the window for 10 or 15 seconds and then started watching the motorcade, which came into view on Houston Street.|Fischer looked up and watched the man in the window for ten or fifteen seconds and then started watching the motorcade, which came into view on Houston Street. +LJ034-0173|He said that the man held his attention until the motorcade came because the man, quote,|He said that the man held his attention until the motorcade came because the man, quote, +LJ034-0174|appeared uncomfortable for one, and secondly, he wasn't watching. he didn't look like he was watching for the parade.|appeared uncomfortable for one, and secondly, he wasn't watching. he didn't look like he was watching for the parade. +LJ034-0175|He looked like he was looking down toward the Trinity River and the Triple Underpass down at the end -- toward the end of Elm Street. And|He looked like he was looking down toward the Trinity River and the Triple Underpass down at the end -- toward the end of Elm Street. And +LJ034-0176|all the time I watched him, he never moved his head, he never -- he never moved anything. Just was there transfixed.|all the time I watched him, he never moved his head, he never -- he never moved anything. Just was there transfixed. +LJ034-0177|Fischer placed the man in the easternmost window on the south side of the Depository Building on either the fifth or the sixth floor.|Fischer placed the man in the easternmost window on the south side of the Depository Building on either the fifth or the sixth floor. +LJ034-0178|He said that he could see the man from the middle of his chest to the top of his head, and that as he was facing the window the man was in the lower right-hand portion of the window|He said that he could see the man from the middle of his chest to the top of his head, and that as he was facing the window the man was in the lower right-hand portion of the window +LJ034-0180|The man was dressed in a light-colored, open-neck shirt which could have been either a sports shirt or a T-shirt,|The man was dressed in a light-colored, open-neck shirt which could have been either a sports shirt or a T-shirt, +LJ034-0181|and he had brown hair, a slender face and neck with light complexion, and looked to be 22 or 24 years old.|and he had brown hair, a slender face and neck with light complexion, and looked to be twenty-two or twenty-four years old. +LJ034-0182|The person in the window was a white man and, quote, looked to me like he was looking straight at the Triple Underpass, end quote, down Elm Street.|The person in the window was a white man and, quote, looked to me like he was looking straight at the Triple Underpass, end quote, down Elm Street. +LJ034-0183|Boxes and cases were stacked behind him.|Boxes and cases were stacked behind him. +LJ034-0184|Approximately 1 week after the assassination, according to Fisher, policemen showed him a picture of Oswald.|Approximately one week after the assassination, according to Fisher, policemen showed him a picture of Oswald. +LJ034-0185|In his testimony he said, quote, I told them that that could have been the man.|In his testimony he said, quote, I told them that that could have been the man. +LJ034-0186|That that could have been the man that I saw in the window in the School Book Depository Building, but that I was not sure, end quote.|That that could have been the man that I saw in the window in the School Book Depository Building, but that I was not sure, end quote. +LJ034-0187|Fischer described the man's hair as some shade of brown, quote, it wasn't dark, and it wasn't light, end quote.|Fischer described the man's hair as some shade of brown, quote, it wasn't dark, and it wasn't light, end quote. +LJ034-0188|On November 22, Fischer had apparently described the man as, quote, light-headed, end quote.|On November twenty-two, Fischer had apparently described the man as, quote, light-headed, end quote. +LJ034-0189|Fischer explained that he did not mean by the earlier statement that the man was blond, but rather that his hair was not black.|Fischer explained that he did not mean by the earlier statement that the man was blond, but rather that his hair was not black. +LJ034-0190|Robert Edwards said that, while looking at the south side of the Depository Building shortly before the motorcade,|Robert Edwards said that, while looking at the south side of the Depository Building shortly before the motorcade, +LJ034-0191|he saw nothing of importance, quote, except maybe one individual who was up there in the corner room of the sixth floor which was crowded in among boxes, end quote.|he saw nothing of importance, quote, except maybe one individual who was up there in the corner room of the sixth floor which was crowded in among boxes, end quote. +LJ034-0192|He said that this was a white man about average in size, quote, possibly thin, end quote, and that he thought the man had light-brown hair.|He said that this was a white man about average in size, quote, possibly thin, end quote, and that he thought the man had light-brown hair. +LJ034-0193|Fischer and Edwards did not see the man clearly enough or long enough to identify him.|Fischer and Edwards did not see the man clearly enough or long enough to identify him. +LJ034-0194|Their testimony is of probative value, however, because their limited description is consistent with that of the man who has been found by the Commission,|Their testimony is of probative value, however, because their limited description is consistent with that of the man who has been found by the Commission, +LJ034-0195|based on other evidence, to have fired the shots from the window.|based on other evidence, to have fired the shots from the window. +LJ034-0196|Another person who saw the assassin as the shots were fired was Amos L. Euins, age 15,|Another person who saw the assassin as the shots were fired was Amos L. Euins, age fifteen, +LJ034-0197|who was one of the first witnesses to alert the police to the Depository as the source of the shots, as has been discussed in chapter 3.|who was one of the first witnesses to alert the police to the Depository as the source of the shots, as has been discussed in chapter three. +LJ034-0198|Euins, who was on the southwest corner of Elm and Houston Streets testified that he could not describe the man he saw in the window.|Euins, who was on the southwest corner of Elm and Houston Streets testified that he could not describe the man he saw in the window. +LJ034-0199|According to Euins, however, as the man lowered his head in order to aim the rifle down Elm Street, he appeared to have a white bald spot, on his head.|According to Euins, however, as the man lowered his head in order to aim the rifle down Elm Street, he appeared to have a white bald spot, on his head. +LJ034-0200|Shortly after the assassination, Euins signed an affidavit describing the man as "white,"|Shortly after the assassination, Euins signed an affidavit describing the man as "white," +LJ034-0201|but a radio reporter testified that Euins described the man to him as, quote, colored, end quote.|but a radio reporter testified that Euins described the man to him as, quote, colored, end quote. +LJ034-0202|In his Commission testimony|In his Commission testimony +LJ034-0203|Euins stated that he could not ascertain the man's race and that the statement in the affidavit was intended to refer only to the white spot on the man's head|Euins stated that he could not ascertain the man's race and that the statement in the affidavit was intended to refer only to the white spot on the man's head +LJ034-0204|and not to his race.|and not to his race. +LJ034-0205|A Secret Service agent who spoke to Euins approximately 20 to 30 minutes after the assassination|A Secret Service agent who spoke to Euins approximately twenty to thirty minutes after the assassination +LJ034-0206|confirmed that Euins could neither describe the man in the window nor indicate his race.|confirmed that Euins could neither describe the man in the window nor indicate his race. +LJ034-0207|Accordingly, Euins' testimony is considered probative as to the source of the shots but is inconclusive as to the identity of the man in the window.|Accordingly, Euins' testimony is considered probative as to the source of the shots but is inconclusive as to the identity of the man in the window. +LJ034-0208|In evaluating the evidence that Oswald was at the southeast corner window of the sixth floor at the time of the shooting,|In evaluating the evidence that Oswald was at the southeast corner window of the sixth floor at the time of the shooting, +LJ034-0209|the Commission has considered the allegation that Oswald was photographed standing in front of the building when the shots were fired.|the Commission has considered the allegation that Oswald was photographed standing in front of the building when the shots were fired. +LJ034-0210|The picture which gave rise to these allegations was taken by Associated Press Photographer James W. Altgens,|The picture which gave rise to these allegations was taken by Associated Press Photographer James W. Altgens, +LJ034-0211|who was standing on the south side of Elm Street between the Triple Underpass and the Depository Building.|who was standing on the south side of Elm Street between the Triple Underpass and the Depository Building. +LJ034-0212|As the motorcade started its descent down Elm Street., Altgens snapped a picture of the Presidential limousine with the entrance to the Depository Building in the background.|As the motorcade started its descent down Elm Street., Altgens snapped a picture of the Presidential limousine with the entrance to the Depository Building in the background. +LJ034-0213|Just before snapping the picture Altgens heard a noise which sounded like the popping of a firecracker.|Just before snapping the picture Altgens heard a noise which sounded like the popping of a firecracker. +LJ034-0214|Investigation has established that Altgens' picture was taken approximately 2 seconds after the firing of the shot|Investigation has established that Altgens' picture was taken approximately two seconds after the firing of the shot +LJ034-0215|which entered the back of the President's neck.|which entered the back of the President's neck. +LJ034-0216|In the background of this picture were several employees watching the parade from the steps of the Depository Building.|In the background of this picture were several employees watching the parade from the steps of the Depository Building. +LJ034-0217|One of these employees was alleged to resemble Lee Harvey Oswald.|One of these employees was alleged to resemble Lee Harvey Oswald. +LJ034-0218|The Commission has determined that the employee was in fact Billy Lovelady, who identified himself in the picture.|The Commission has determined that the employee was in fact Billy Lovelady, who identified himself in the picture. +LJ034-0219|Standing alongside him were Buell Wesley Frazier and William Shelley, who also identified Lovelady.|Standing alongside him were Buell Wesley Frazier and William Shelley, who also identified Lovelady. +LJ035-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ035-0002|Chapter 4. The Assassin: Part 4. Oswald's Actions in Building After Assassination.|Chapter four. The Assassin: Part four. Oswald's Actions in Building After Assassination. +LJ035-0003|In considering whether Oswald was at the southeast corner window at the time the shots were fired,|In considering whether Oswald was at the southeast corner window at the time the shots were fired, +LJ035-0004|the Commission has reviewed the testimony of witnesses who saw Oswald in the building within minutes after the assassination.|the Commission has reviewed the testimony of witnesses who saw Oswald in the building within minutes after the assassination. +LJ035-0005|The Commission has found that Oswald's movements, as described by these witnesses,|The Commission has found that Oswald's movements, as described by these witnesses, +LJ035-0006|are consistent with his having been at the window at 12:30 p.m.|are consistent with his having been at the window at twelve:thirty p.m. +LJ035-0007|The encounter in the lunchroom.|The encounter in the lunchroom. +LJ035-0008|The first person to see Oswald after the assassination was Patrolman M. L. Baker of the Dallas Police Department.|The first person to see Oswald after the assassination was Patrolman M. L. Baker of the Dallas Police Department. +LJ035-0009|Baker was riding a two-wheeled motorcycle behind the last press car of the motorcade.|Baker was riding a two-wheeled motorcycle behind the last press car of the motorcade. +LJ035-0010|As he turned the corner from Main onto Houston at a speed of about 5 to 10 miles per hour,|As he turned the corner from Main onto Houston at a speed of about five to ten miles per hour, +LJ035-0011|a strong wind blowing from the north almost unseated him.|a strong wind blowing from the north almost unseated him. +LJ035-0012|At about this time he heard the first shot.|At about this time he heard the first shot. +LJ035-0013|Having recently heard the sounds of rifles while on a hunting trip, Baker recognized the shots as that of a high-powered rifle; Quote,|Having recently heard the sounds of rifles while on a hunting trip, Baker recognized the shots as that of a high-powered rifle; Quote, +LJ035-0014|it sounded high and I immediately kind of looked up,|it sounded high and I immediately kind of looked up, +LJ035-0015|and I had a feeling that it came from the building, either right in front of me [the Depository Building] or of the one across to the right of it, end quote,|and I had a feeling that it came from the building, either right in front of me [the Depository Building] or of the one across to the right of it, end quote, +LJ035-0016|He saw pigeons flutter upward. He was not certain, quote, but I am pretty sure they came from the building right on the northwest corner, end quote.|He saw pigeons flutter upward. He was not certain, quote, but I am pretty sure they came from the building right on the northwest corner, end quote. +LJ035-0017|He heard two more shots spaced, quote, pretty well even to me.|He heard two more shots spaced, quote, pretty well even to me. +LJ035-0018|end quote, After the third shot, he, quote, revved that motorcycle up, end quote,|end quote, After the third shot, he, quote, revved that motorcycle up, end quote, +LJ035-0019|drove to the northwest corner of Elm and Houston, and parked approximately 10 feet from the traffic signal.|drove to the northwest corner of Elm and Houston, and parked approximately ten feet from the traffic signal. +LJ035-0020|As he was parking he noted that people were, quote, falling, and they were rolling around down there grabbing their children, end quote, and rushing about.|As he was parking he noted that people were, quote, falling, and they were rolling around down there grabbing their children, end quote, and rushing about. +LJ035-0021|A woman screamed, quote, Oh, they have shot that man, they have shot that man, end quote.|A woman screamed, quote, Oh, they have shot that man, they have shot that man, end quote. +LJ035-0022|Baker, quote, had it in mind that the shots came from the top of this building here, end quote, so he ran straight to the entrance of the Depository Building.|Baker, quote, had it in mind that the shots came from the top of this building here, end quote, so he ran straight to the entrance of the Depository Building. +LJ035-0023|Baker testified that he entered the lobby of the building and, quote, spoke out and asked where the stairs or elevator was|Baker testified that he entered the lobby of the building and, quote, spoke out and asked where the stairs or elevator was +LJ035-0024|and this man, Mr. Truly, spoke up and says, it seems to me like he says,|and this man, Mr. Truly, spoke up and says, it seems to me like he says, +LJ035-0025|I am a building manager. Follow me, officer, and I will show you, end quote.|I am a building manager. Follow me, officer, and I will show you, end quote. +LJ035-0026|Baker and building superintendent Roy Truly went through a second set of doors and stopped at a swinging door where Baker bumped into Truly's back.|Baker and building superintendent Roy Truly went through a second set of doors and stopped at a swinging door where Baker bumped into Truly's back. +LJ035-0027|They went through the swinging door and continued at, quote, a good trot, end quote,|They went through the swinging door and continued at, quote, a good trot, end quote, +LJ035-0028|to the northwest corner of the floor where Truly hoped to find one of the two freight elevators.|to the northwest corner of the floor where Truly hoped to find one of the two freight elevators. +LJ035-0029|Neither elevator was there.|Neither elevator was there. +LJ035-0030|Truly pushed the button for the west elevator which operates automatically if the gate is closed.|Truly pushed the button for the west elevator which operates automatically if the gate is closed. +LJ035-0031|He shouted twice, quote, Turn loose the elevator, end quote,|He shouted twice, quote, Turn loose the elevator, end quote, +LJ035-0032|When the elevator failed to come, Baker said, quote, let's take the stairs, end quote, and he followed Truly up the stairway, which is to the west of the elevator.|When the elevator failed to come, Baker said, quote, let's take the stairs, end quote, and he followed Truly up the stairway, which is to the west of the elevator. +LJ035-0033|The stairway is located in the northwest corner of the Depository Building.|The stairway is located in the northwest corner of the Depository Building. +LJ035-0034|The stairs from one floor to the next are "L-shaped," with both legs of the "L" approximately the same length.|The stairs from one floor to the next are "L-shaped," with both legs of the "L" approximately the same length. +LJ035-0035|Because the stairway itself is enclosed, neither Baker nor Truly could see anything on the second-floor hallway until they reached the landing at the top of the stairs.|Because the stairway itself is enclosed, neither Baker nor Truly could see anything on the second-floor hallway until they reached the landing at the top of the stairs. +LJ035-0036|On the second-floor landing there is a small open area with a door at the east end.|On the second-floor landing there is a small open area with a door at the east end. +LJ035-0037|This door leads into a small vestibule, and another door leads from the vestibule into the second-floor lunchroom.|This door leads into a small vestibule, and another door leads from the vestibule into the second-floor lunchroom. +LJ035-0038|The lunchroom door is usually open, but the first door is kept shut by a closing mechanism on the door.|The lunchroom door is usually open, but the first door is kept shut by a closing mechanism on the door. +LJ035-0039|This vestibule door is solid except for a small glass window in the upper part of the door.|This vestibule door is solid except for a small glass window in the upper part of the door. +LJ035-0040|As Baker reached the second floor, he was about 20 feet from the vestibule door. He intended to continue around to his left toward the stairway going up|As Baker reached the second floor, he was about twenty feet from the vestibule door. He intended to continue around to his left toward the stairway going up +LJ035-0041|but through the window in the door he caught a fleeting glimpse of a man walking in the vestibule toward the lunchroom.|but through the window in the door he caught a fleeting glimpse of a man walking in the vestibule toward the lunchroom. +LJ035-0042|Since the vestibule door is only a few feet from the lunchroom door, the man must have entered the vestibule only a second or two before Baker arrived at the top of the stairwell.|Since the vestibule door is only a few feet from the lunchroom door, the man must have entered the vestibule only a second or two before Baker arrived at the top of the stairwell. +LJ035-0043|Yet he must have entered the vestibule door before Truly reached the top of the stairwell, since Truly did not see him.|Yet he must have entered the vestibule door before Truly reached the top of the stairwell, since Truly did not see him. +LJ035-0044|If the man had passed from the vestibule into the lunchroom, Baker could not have seen him.|If the man had passed from the vestibule into the lunchroom, Baker could not have seen him. +LJ035-0045|Baker said, quote, He [Truly] had already started around the bend to come to the next elevator going up,|Baker said, quote, He [Truly] had already started around the bend to come to the next elevator going up, +LJ035-0046|I was coming out this one on the second floor, and I don't know, I was kind of sweeping this area as I come up, I was looking from right to left|I was coming out this one on the second floor, and I don't know, I was kind of sweeping this area as I come up, I was looking from right to left +LJ035-0047|and as I got to this door here I caught a glimpse of this man, just, you know, a sudden glimpse. And it looked to me like he was going away from me.|and as I got to this door here I caught a glimpse of this man, just, you know, a sudden glimpse. And it looked to me like he was going away from me. +LJ035-0048|I can't say whether he had gone on through that door [the lunchroom door] or not.|I can't say whether he had gone on through that door [the lunchroom door] or not. +LJ035-0049|All I did was catch a glance at him, and evidently he was -- this door might have been, you know, closing and almost shut at that time, end quote.|All I did was catch a glance at him, and evidently he was -- this door might have been, you know, closing and almost shut at that time, end quote. +LJ035-0050|With his revolver drawn, Baker opened the vestibule door and ran into the vestibule.|With his revolver drawn, Baker opened the vestibule door and ran into the vestibule. +LJ035-0051|He saw a man walking away from him in the lunchroom.|He saw a man walking away from him in the lunchroom. +LJ035-0052|Baker stopped at the door of the lunchroom and commanded, quote, Come here, end quote.|Baker stopped at the door of the lunchroom and commanded, quote, Come here, end quote. +LJ035-0053|The man turned and walked back toward Baker.|The man turned and walked back toward Baker. +LJ035-0054|He had been proceeding toward the rear of the lunchroom.|He had been proceeding toward the rear of the lunchroom. +LJ035-0055|Along a side wall of the lunchroom was a soft drink rending machine, but at that time the man had nothing in his hands.|Along a side wall of the lunchroom was a soft drink rending machine, but at that time the man had nothing in his hands. +LJ035-0056|Meanwhile, Truly had run up several steps toward the third floor.|Meanwhile, Truly had run up several steps toward the third floor. +LJ035-0057|Missing Baker, he came back to find the officer in the doorway to the lunchroom facing Lee Harvey Oswald.|Missing Baker, he came back to find the officer in the doorway to the lunchroom facing Lee Harvey Oswald. +LJ035-0058|Baker turned to Truly and said, quote, Do you know this man, does he work here? end quote.|Baker turned to Truly and said, quote, Do you know this man, does he work here? end quote. +LJ035-0059|Truly replied, "Yes."|Truly replied, "Yes." +LJ035-0060|Baker stated later that the man did not seem to be out of breath; he seemed calm. Quote, He never did say a word or nothing.|Baker stated later that the man did not seem to be out of breath; he seemed calm. Quote, He never did say a word or nothing. +LJ035-0061|In fact, he didn't change his expression one bit, end quote.|In fact, he didn't change his expression one bit, end quote. +LJ035-0062|Truly said of Oswald: quote, He didn't seem to be excited or overly afraid or anything. He might have been a bit startled, like I might have been if somebody confronted me.|Truly said of Oswald: quote, He didn't seem to be excited or overly afraid or anything. He might have been a bit startled, like I might have been if somebody confronted me. +LJ035-0063|But I cannot recall any change in expression of any kind on his face, end quote.|But I cannot recall any change in expression of any kind on his face, end quote. +LJ035-0064|Truly thought that the officer's gun at that time appeared to be almost touching the middle portion of Oswald's body.|Truly thought that the officer's gun at that time appeared to be almost touching the middle portion of Oswald's body. +LJ035-0065|Truly also noted at this time that Oswald's hands were empty.|Truly also noted at this time that Oswald's hands were empty. +LJ035-0066|In an effort to determine whether Oswald could have descended to the lunchroom|In an effort to determine whether Oswald could have descended to the lunchroom +LJ035-0067|from the sixth floor by the time Baker and Truly arrived, Commission counsel asked Baker and Truly to repeat their movements from the time of the shot|from the sixth floor by the time Baker and Truly arrived, Commission counsel asked Baker and Truly to repeat their movements from the time of the shot +LJ035-0068|until Baker came upon Oswald in the lunchroom.|until Baker came upon Oswald in the lunchroom. +LJ035-0069|Baker placed himself on a motorcycle about 200 feet from the corner of Elm and Houston Streets where he said he heard the shots.|Baker placed himself on a motorcycle about two hundred feet from the corner of Elm and Houston Streets where he said he heard the shots. +LJ035-0070|Truly stood in front of the building.|Truly stood in front of the building. +LJ035-0071|At a given signal, they reenacted the event. Baker's movements were timed with a stopwatch.|At a given signal, they reenacted the event. Baker's movements were timed with a stopwatch. +LJ035-0072|On the first test, the elapsed time between the simulated first shot and Baker's arrival on the second-floor stair landing|On the first test, the elapsed time between the simulated first shot and Baker's arrival on the second-floor stair landing +LJ035-0073|was 1 minute and 30 seconds.|was one minute and thirty seconds. +LJ035-0074|The second test run required 1 minute and 15 seconds.|The second test run required one minute and fifteen seconds. +LJ035-0075|A test was also conducted to determine the time required to walk from the southeast corner of the sixth floor to the second-floor lunchroom by stairway.|A test was also conducted to determine the time required to walk from the southeast corner of the sixth floor to the second-floor lunchroom by stairway. +LJ035-0076|Special Agent John Howlett of the Secret Service carried a rifle from the southeast corner of the sixth floor along the east aisle to the northeast corner.|Special Agent John Howlett of the Secret Service carried a rifle from the southeast corner of the sixth floor along the east aisle to the northeast corner. +LJ035-0077|He placed the rifle on the floor near the site where Oswald's rifle was actually found after the shooting.|He placed the rifle on the floor near the site where Oswald's rifle was actually found after the shooting. +LJ035-0078|Then Howlett walked down the stairway to the second-floor landing and entered the lunchroom.|Then Howlett walked down the stairway to the second-floor landing and entered the lunchroom. +LJ035-0079|The first test, run at normal walking pace, required 1 minute, 18 seconds;|The first test, run at normal walking pace, required one minute, eighteen seconds; +LJ035-0080|the second test, at a "fast walk" took 1 minute, 14 seconds.|the second test, at a "fast walk" took one minute, fourteen seconds. +LJ035-0081|The second test followed immediately after the first.|The second test followed immediately after the first. +LJ035-0082|The only interval was the time necessary to ride in the elevator from the second to the sixth floor and walk back to the southeast corner.|The only interval was the time necessary to ride in the elevator from the second to the sixth floor and walk back to the southeast corner. +LJ035-0083|Howlett was not short winded at the end of either test run.|Howlett was not short winded at the end of either test run. +LJ035-0084|The minimum time required by Baker to park his motorcycle and reach the second-floor lunchroom was within 3 seconds of the time needed to walk|The minimum time required by Baker to park his motorcycle and reach the second-floor lunchroom was within three seconds of the time needed to walk +LJ035-0085|from the southeast corner of the sixth floor down the stairway to the lunchroom.|from the southeast corner of the sixth floor down the stairway to the lunchroom. +LJ035-0086|The time actually required for Baker and Truly to reach the second floor on November 22 was probably longer than in the test runs. For example,|The time actually required for Baker and Truly to reach the second floor on November twenty-two was probably longer than in the test runs. For example, +LJ035-0087|Baker required 15 seconds after the simulated shot to ride his motorcycle 180 to 200 feet,|Baker required fifteen seconds after the simulated shot to ride his motorcycle one hundred eighty to two hundred feet, +LJ035-0088|park it, and run 45 feet to the building.|park it, and run forty-five feet to the building. +LJ035-0089|No allowance was made for the special conditions which existed on the day of the assassination|No allowance was made for the special conditions which existed on the day of the assassination +LJ035-0090|possible delayed reaction to the shot, jostling with the crowd of people on the steps and scanning the area along Elm Street and the parkway.|possible delayed reaction to the shot, jostling with the crowd of people on the steps and scanning the area along Elm Street and the parkway. +LJ035-0091|Baker said, quote, We simulated the shots and by the time we got there, we did everything that I did that day,|Baker said, quote, We simulated the shots and by the time we got there, we did everything that I did that day, +LJ035-0092|and this would be the minimum, because I am sure that I, you know, it took me a little longer, end quote,|and this would be the minimum, because I am sure that I, you know, it took me a little longer, end quote, +LJ035-0093|On the basis of this time test, therefore,|On the basis of this time test, therefore, +LJ035-0094|the Commission concluded that Oswald could have fired the shots and still have been present in the second-floor lunchroom when seen by Baker and Truly.|the Commission concluded that Oswald could have fired the shots and still have been present in the second-floor lunchroom when seen by Baker and Truly. +LJ035-0095|That Oswald descended by stairway from the sixth floor to the second-floor lunchroom|That Oswald descended by stairway from the sixth floor to the second-floor lunchroom +LJ035-0096|is consistent with the movements of the two elevators, which would have provided the other possible means of descent.|is consistent with the movements of the two elevators, which would have provided the other possible means of descent. +LJ035-0097|When Truly, accompanied by Baker, ran to the rear of the first floor,|When Truly, accompanied by Baker, ran to the rear of the first floor, +LJ035-0098|he was certain that both elevators, which occupy the same shaft, were on the fifth floor.|he was certain that both elevators, which occupy the same shaft, were on the fifth floor. +LJ035-0099|Baker, not realizing that there were two elevators, thought that only one elevator was in the shaft and that it was two or three floors above the second floor.|Baker, not realizing that there were two elevators, thought that only one elevator was in the shaft and that it was two or three floors above the second floor. +LJ035-0100|In the few seconds which elapsed while Baker and Truly ran from the first to the second floor,|In the few seconds which elapsed while Baker and Truly ran from the first to the second floor, +LJ035-0101|neither of these slow elevators could have descended from the fifth to the second floor. Furthermore, no elevator was at the second floor when they arrived there.|neither of these slow elevators could have descended from the fifth to the second floor. Furthermore, no elevator was at the second floor when they arrived there. +LJ035-0102|Truly and Baker continued up the stairs after the encounter with Oswald in the lunchroom.|Truly and Baker continued up the stairs after the encounter with Oswald in the lunchroom. +LJ035-0103|There was no elevator on the third or fourth floor.|There was no elevator on the third or fourth floor. +LJ035-0104|The east elevator was on the fifth floor when they arrived; the west elevator was not.|The east elevator was on the fifth floor when they arrived; the west elevator was not. +LJ035-0105|They took the east elevator to the seventh floor and ran up a stairway to the roof where they searched for several minutes.|They took the east elevator to the seventh floor and ran up a stairway to the roof where they searched for several minutes. +LJ035-0106|Jack Dougherty, an employee working on the fifth floor,|Jack Dougherty, an employee working on the fifth floor, +LJ035-0107|testified that he took the west elevator to the first floor after hearing a noise which sounded like a backfire.|testified that he took the west elevator to the first floor after hearing a noise which sounded like a backfire. +LJ035-0108|Eddie Piper, the janitor, told Dougherty that the President had been shot,|Eddie Piper, the janitor, told Dougherty that the President had been shot, +LJ035-0109|but in his testimony Piper did not mention either seeing or talking with Dougherty during these moments of excitement.|but in his testimony Piper did not mention either seeing or talking with Dougherty during these moments of excitement. +LJ035-0110|Both Dougherty and Piper were confused witnesses. They had no exact memory of the events of that afternoon.|Both Dougherty and Piper were confused witnesses. They had no exact memory of the events of that afternoon. +LJ035-0111|Truly was probably correct in stating that the west elevator was on the fifth floor when he looked up the elevator shaft from the first floor.|Truly was probably correct in stating that the west elevator was on the fifth floor when he looked up the elevator shaft from the first floor. +LJ035-0112|The west elevator was not on the fifth floor when Baker and Truly reached that floor,|The west elevator was not on the fifth floor when Baker and Truly reached that floor, +LJ035-0113|probably because Jack Dougherty took it to the first floor while Baker and Truly were running up the stairs or in the lunchroom with Oswald.|probably because Jack Dougherty took it to the first floor while Baker and Truly were running up the stairs or in the lunchroom with Oswald. +LJ035-0114|Neither elevator could have been used by Oswald as a means of descent.|Neither elevator could have been used by Oswald as a means of descent. +LJ035-0115|Oswald's use of the stairway is consistent with the testimony of other employees in the building.|Oswald's use of the stairway is consistent with the testimony of other employees in the building. +LJ035-0116|Three employees -- James Jarman, Jr.,|Three employees -- James Jarman, Jr., +LJ035-0117|Harold Norman, and Bonnie Ray Williams -- were watching the parade from the fifth floor, directly below the window from which the shots were fired.|Harold Norman, and Bonnie Ray Williams -- were watching the parade from the fifth floor, directly below the window from which the shots were fired. +LJ035-0118|They rushed to the west windows after the shots were fired and remained there|They rushed to the west windows after the shots were fired and remained there +LJ035-0119|until after they saw Patrolman Baker's white helmet on the fifth floor moving toward the elevator.|until after they saw Patrolman Baker's white helmet on the fifth floor moving toward the elevator. +LJ035-0120|While they were at the west windows their view of the stairwell was completely blocked by shelves and boxes.|While they were at the west windows their view of the stairwell was completely blocked by shelves and boxes. +LJ035-0121|This is the period during which Oswald would have descended the stairs. In all likelihood|This is the period during which Oswald would have descended the stairs. In all likelihood +LJ035-0122|Dougherty took the elevator down from the fifth floor after Jarman, Norman, and Williams ran to the west windows and were deciding what to do.|Dougherty took the elevator down from the fifth floor after Jarman, Norman, and Williams ran to the west windows and were deciding what to do. +LJ035-0123|None of these three men saw Dougherty, probably because of the anxiety of the moment and because of the books which may have blocked the view.|None of these three men saw Dougherty, probably because of the anxiety of the moment and because of the books which may have blocked the view. +LJ035-0124|Neither Jarman, Norman, Williams, or Dougherty saw Oswald.|Neither Jarman, Norman, Williams, or Dougherty saw Oswald. +LJ035-0126|claimed that within about 1 minute following the shots she ran from a window on the south side of the fourth floor,|claimed that within about one minute following the shots she ran from a window on the south side of the fourth floor, +LJ035-0127|down the rear stairs to the first floor, where she encountered two Depository employees -- William Shelley and Billy Lovelady.|down the rear stairs to the first floor, where she encountered two Depository employees -- William Shelley and Billy Lovelady. +LJ035-0128|If her estimate of time is correct, she reached the bottom of the stairs before Truly and Baker started up,|If her estimate of time is correct, she reached the bottom of the stairs before Truly and Baker started up, +LJ035-0129|and she must have run down the stairs ahead of Oswald and would probably have seen or heard him.|and she must have run down the stairs ahead of Oswald and would probably have seen or heard him. +LJ035-0130|Actually she noticed no one on the back stairs. If she descended from the fourth to the first floor as fast as she claimed in her testimony,|Actually she noticed no one on the back stairs. If she descended from the fourth to the first floor as fast as she claimed in her testimony, +LJ035-0131|she would have seen Baker or Truly on the first floor or on the stairs, unless they were already in the second-floor lunchroom talking to Oswald.|she would have seen Baker or Truly on the first floor or on the stairs, unless they were already in the second-floor lunchroom talking to Oswald. +LJ035-0132|When she reached the first floor, she actually saw Shelley and Lovelady slightly east of the east elevator.|When she reached the first floor, she actually saw Shelley and Lovelady slightly east of the east elevator. +LJ035-0133|Shelley and Lovelady, however, have testified|Shelley and Lovelady, however, have testified +LJ035-0134|that they were watching the parade from the top step of the building entrance when Gloria Calverly, who works in the Depository Building,|that they were watching the parade from the top step of the building entrance when Gloria Calverly, who works in the Depository Building, +LJ035-0135|ran up and said that the President had been shot.|ran up and said that the President had been shot. +LJ035-0136|Lovelady and Shelley moved out into the street.|Lovelady and Shelley moved out into the street. +LJ035-0137|About this time Shelley saw Truly and Patrolman Baker go into the building|About this time Shelley saw Truly and Patrolman Baker go into the building +LJ035-0138|Shelley and Lovelady, at a fast walk or trot, turned west into the railroad yards and then to the west side of the Depository Building.|Shelley and Lovelady, at a fast walk or trot, turned west into the railroad yards and then to the west side of the Depository Building. +LJ035-0139|They reentered the building by the rear door several minutes after Baker and Truly rushed through the front entrance.|They reentered the building by the rear door several minutes after Baker and Truly rushed through the front entrance. +LJ035-0140|On entering, Lovelady saw a girl on the first floor who he believes was Victoria Adams.|On entering, Lovelady saw a girl on the first floor who he believes was Victoria Adams. +LJ035-0141|If Miss Adams accurately recalled meeting Shelley and Lovelady when she reached the bottom of the stairs,|If Miss Adams accurately recalled meeting Shelley and Lovelady when she reached the bottom of the stairs, +LJ035-0142|then her estimate of the time when she descended from the fourth floor is incorrect,|then her estimate of the time when she descended from the fourth floor is incorrect, +LJ035-0143|and she actually came down the stairs several minutes after Oswald and after Truly and Baker as well.|and she actually came down the stairs several minutes after Oswald and after Truly and Baker as well. +LJ035-0144|Oswald's departure from building.|Oswald's departure from building. +LJ035-0145|Within a minute after Baker and Truly left Oswald in the lunchroom, Mrs. R. A. Reid, clerical supervisor for the Texas School Book Depository,|Within a minute after Baker and Truly left Oswald in the lunchroom, Mrs. R. A. Reid, clerical supervisor for the Texas School Book Depository, +LJ035-0146|saw him walk through the clerical office on the second floor toward the door leading to the front stairway.|saw him walk through the clerical office on the second floor toward the door leading to the front stairway. +LJ035-0147|Mrs. Reid had watched the parade from the sidewalk in front of the building with Truly and Mr. O. V. Campbell, vice president of the Depository.|Mrs. Reid had watched the parade from the sidewalk in front of the building with Truly and Mr. O. V. Campbell, vice president of the Depository. +LJ035-0148|She testified that she heard three shots which she thought came from the building.|She testified that she heard three shots which she thought came from the building. +LJ035-0149|She ran inside and up the front stairs into the large open office reserved for clerical employees.|She ran inside and up the front stairs into the large open office reserved for clerical employees. +LJ035-0150|As she approached her desk, she saw Oswald.|As she approached her desk, she saw Oswald. +LJ035-0151|He was walking into the office from the back hallway,|He was walking into the office from the back hallway, +LJ035-0152|carrying a full bottle of Coca-Cola in his hand, presumably purchased after the encounter with Baker and Truly.|carrying a full bottle of Coca-Cola in his hand, presumably purchased after the encounter with Baker and Truly. +LJ035-0153|As Oswald passed Mrs. Reid she said, quote, Oh, the President has been shot, but maybe they didn't hit him, end quote.|As Oswald passed Mrs. Reid she said, quote, Oh, the President has been shot, but maybe they didn't hit him, end quote. +LJ035-0154|Oswald mumbled something and walked by. She paid no more attention to him.|Oswald mumbled something and walked by. She paid no more attention to him. +LJ035-0155|The only exit from the office in the direction Oswald was moving was through the door to the front stairway.|The only exit from the office in the direction Oswald was moving was through the door to the front stairway. +LJ035-0156|Mrs. Reid testified that when she saw Oswald, he was wearing a T-shirt and no jacket.|Mrs. Reid testified that when she saw Oswald, he was wearing a T-shirt and no jacket. +LJ035-0157|When he left home that morning, Marina Oswald, who was still in bed, suggested that he wear a jacket.|When he left home that morning, Marina Oswald, who was still in bed, suggested that he wear a jacket. +LJ035-0158|A blue jacket, later identified by Marina Oswald as her husband's, was subsequently found in the building, apparently left behind by Oswald.|A blue jacket, later identified by Marina Oswald as her husband's, was subsequently found in the building, apparently left behind by Oswald. +LJ035-0159|Mrs. Reid believes that she returned to her desk from the street about 2 minutes after the shooting.|Mrs. Reid believes that she returned to her desk from the street about two minutes after the shooting. +LJ035-0160|Reconstructing her movements, Mrs. Reid ran the distance three times and was timed in 2 minutes by stopwatch.|Reconstructing her movements, Mrs. Reid ran the distance three times and was timed in two minutes by stopwatch. +LJ035-0161|The reconstruction was the minimum time. Accordingly, she probably met Oswald at about 12:32,|The reconstruction was the minimum time. Accordingly, she probably met Oswald at about twelve:thirty-two, +LJ035-0163|After leaving Mrs. Reid in the front office, Oswald could have gone down the stairs and out the front door by 12:33 p.m.|After leaving Mrs. Reid in the front office, Oswald could have gone down the stairs and out the front door by twelve:thirty-three p.m. +LJ035-0164|3 minutes after the shooting.|three minutes after the shooting. +LJ035-0165|At that time the building had not yet been sealed off by the police.|At that time the building had not yet been sealed off by the police. +LJ035-0166|While it was difficult to determine exactly when the police sealed off the building,|While it was difficult to determine exactly when the police sealed off the building, +LJ035-0167|the earliest estimates would still have permitted Oswald to leave the building by 12:33.|the earliest estimates would still have permitted Oswald to leave the building by twelve:thirty-three. +LJ035-0168|One of the police officers assigned to the corner of Elm and Houston Streets for the Presidential motorcade, W. E. Barnett,|One of the police officers assigned to the corner of Elm and Houston Streets for the Presidential motorcade, W. E. Barnett, +LJ035-0169|testified that immediately after the shots he went to the rear of the building to check the fire escape.|testified that immediately after the shots he went to the rear of the building to check the fire escape. +LJ035-0170|He then returned to the corner of Elm and Houston where he met a sergeant who instructed him to find out the name of the building.|He then returned to the corner of Elm and Houston where he met a sergeant who instructed him to find out the name of the building. +LJ035-0171|Barnett ran to the building, noted its name, and then returned to the corner.|Barnett ran to the building, noted its name, and then returned to the corner. +LJ035-0172|There he was met by a construction worker -- in all likelihood Howard Brennan, who was wearing his work helmet.|There he was met by a construction worker -- in all likelihood Howard Brennan, who was wearing his work helmet. +LJ035-0173|This worker told Barnett that the shots had been fired from a window in the Depository Building,|This worker told Barnett that the shots had been fired from a window in the Depository Building, +LJ035-0174|where upon Barnett posted himself at the front door to make certain that no one left the building.|where upon Barnett posted himself at the front door to make certain that no one left the building. +LJ035-0175|The sergeant did the same thing at the rear of the building.|The sergeant did the same thing at the rear of the building. +LJ035-0176|Barnett estimated that approximately 3 minutes elapsed between the time he heard the last of the shots and the time he started guarding the front door.|Barnett estimated that approximately three minutes elapsed between the time he heard the last of the shots and the time he started guarding the front door. +LJ035-0177|According to Barnett, quote, there were people going in and out, end quote, during this period.|According to Barnett, quote, there were people going in and out, end quote, during this period. +LJ035-0178|Sgt. D. V. Harkness of the Dallas police|Sgt. D. V. Harkness of the Dallas police +LJ035-0179|said that to his knowledge the building was not sealed off at 12:36 p.m. when he called in on police radio|said that to his knowledge the building was not sealed off at twelve:thirty-six p.m. when he called in on police radio +LJ035-0180|that a witness (Amos Euins) had seen shots fired from a window of the building.|that a witness (Amos Euins) had seen shots fired from a window of the building. +LJ035-0181|At that time, Inspector Herbert V. Sawyer's car was parked in front of the building.|At that time, Inspector Herbert V. Sawyer's car was parked in front of the building. +LJ035-0182|Harkness did not know whether or not two officers with Sawyer were guarding the doors.|Harkness did not know whether or not two officers with Sawyer were guarding the doors. +LJ035-0183|At 12:34 p.m. Sawyer heard a call over the police radio that the shots had come from the Depository Building.|At twelve:thirty-four p.m. Sawyer heard a call over the police radio that the shots had come from the Depository Building. +LJ035-0184|He then entered the building and took the front passenger elevator as far as it would go -- the fourth floor.|He then entered the building and took the front passenger elevator as far as it would go -- the fourth floor. +LJ035-0185|After inspecting this floor, Sawyer returned to the street about 3 minutes after he entered the building.|After inspecting this floor, Sawyer returned to the street about three minutes after he entered the building. +LJ035-0186|After he returned to the street he directed Sergeant Harkness to station two patrolmen at the front door and not let anyone in or out;|After he returned to the street he directed Sergeant Harkness to station two patrolmen at the front door and not let anyone in or out; +LJ035-0187|he also directed that the back door be sealed off.|he also directed that the back door be sealed off. +LJ035-0188|This was no earlier than 12:37 p.m. and may have been later.|This was no earlier than twelve:thirty-seven p.m. and may have been later. +LJ035-0190|testified that after driving to Parkland Hospital, he returned to the Depository Building about 20 minutes after the shooting,|testified that after driving to Parkland Hospital, he returned to the Depository Building about twenty minutes after the shooting, +LJ035-0191|found no police officers at the rear door and was able to enter through this door without identifying himself.|found no police officers at the rear door and was able to enter through this door without identifying himself. +LJ035-0192|Although Oswald probably left the building at about 12:33 p.m., his absence was not noticed until at least one-half hour later.|Although Oswald probably left the building at about twelve:thirty-three p.m., his absence was not noticed until at least one-half hour later. +LJ035-0193|Truly, who had returned with Patrolman Baker from the roof, saw the police questioning the warehouse employees.|Truly, who had returned with Patrolman Baker from the roof, saw the police questioning the warehouse employees. +LJ035-0194|Approximately 15 men worked in the warehouse and Truly noticed that Oswald was not among those being questioned.|Approximately fifteen men worked in the warehouse and Truly noticed that Oswald was not among those being questioned. +LJ035-0195|Satisfying himself that Oswald was missing, Truly obtained Oswald's address, phone number, and description from his employment application card.|Satisfying himself that Oswald was missing, Truly obtained Oswald's address, phone number, and description from his employment application card. +LJ035-0196|The address listed was for the Paine home in Irving.|The address listed was for the Paine home in Irving. +LJ035-0197|Truly gave this information to Captain Fritz who was on the sixth floor at the time.|Truly gave this information to Captain Fritz who was on the sixth floor at the time. +LJ035-0198|Truly estimated that he gave this information to Fritz about 15 or 20 minutes after the shots,|Truly estimated that he gave this information to Fritz about fifteen or twenty minutes after the shots, +LJ035-0199|but it was probably no earlier than 1:22 p.m., the time when the rifle was found.|but it was probably no earlier than one:twenty-two p.m., the time when the rifle was found. +LJ035-0200|Fritz believed that he learned of Oswald's absence after the rifle was found.|Fritz believed that he learned of Oswald's absence after the rifle was found. +LJ035-0201|The fact that Truly found Fritz in the northwest corner of the floor, near the point where the rifle was found, supports Fritz' recollection.|The fact that Truly found Fritz in the northwest corner of the floor, near the point where the rifle was found, supports Fritz' recollection. +LJ035-0202|Fingerprint and palmprint evidence establishes that Oswald handled two of the four cartons next to the window|Fingerprint and palmprint evidence establishes that Oswald handled two of the four cartons next to the window +LJ035-0203|and also handled a paper bag which was found near the cartons.|and also handled a paper bag which was found near the cartons. +LJ035-0204|Oswald was seen in the vicinity of the southeast corner of the sixth floor approximately 35 minutes before the assassination|Oswald was seen in the vicinity of the southeast corner of the sixth floor approximately thirty-five minutes before the assassination +LJ035-0205|and no one could be found who saw Oswald anywhere else in the building until after the shooting.|and no one could be found who saw Oswald anywhere else in the building until after the shooting. +LJ035-0206|An eyewitness to the shooting immediately provided a description of the man in the window which was similar to Oswald's actual appearance.|An eyewitness to the shooting immediately provided a description of the man in the window which was similar to Oswald's actual appearance. +LJ035-0207|This witness identified Oswald in a lineup as the man most nearly resembling the man he saw and later identified Oswald as the man he observed.|This witness identified Oswald in a lineup as the man most nearly resembling the man he saw and later identified Oswald as the man he observed. +LJ035-0209|at 12:30 p.m|at twelve:thirty p.m +LJ035-0210|On the basis of these findings the Commission has concluded that Oswald, at the time of the assassination,|On the basis of these findings the Commission has concluded that Oswald, at the time of the assassination, +LJ036-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ036-0002|Chapter 4. The Assassin: Part 5.|Chapter four. The Assassin: Part five. +LJ036-0003|The Killing of Patrolman J. D. Tippit|The Killing of Patrolman J. D. Tippit +LJ036-0004|After leaving the Depository Building at approximately 12:33 p.m., Lee Harvey Oswald proceeded to his roominghouse by bus and taxi.|After leaving the Depository Building at approximately twelve:thirty-three p.m., Lee Harvey Oswald proceeded to his roominghouse by bus and taxi. +LJ036-0005|He arrived at approximately 1 p.m. and left a few minutes later.|He arrived at approximately one p.m. and left a few minutes later. +LJ036-0006|At about 1:16 p.m., a Dallas police officer, J. D. Tippit, was shot less than 1 mile from Oswald's roominghouse.|At about one:sixteen p.m., a Dallas police officer, J. D. Tippit, was shot less than one mile from Oswald's roominghouse. +LJ036-0007|In deciding whether Oswald killed Patrolman Tippit the Commission considered the following:|In deciding whether Oswald killed Patrolman Tippit the Commission considered the following: +LJ036-0008|(1) positive identification of the killer by two eyewitnesses who saw the shooting|(one) positive identification of the killer by two eyewitnesses who saw the shooting +LJ036-0009|and seven eyewitnesses who heard the shots and saw the gunman flee the scene with the revolver in his hand,|and seven eyewitnesses who heard the shots and saw the gunman flee the scene with the revolver in his hand, +LJ036-0010|(2) testimony of firearms identification experts establishing the identity of the murder weapon,|(two) testimony of firearms identification experts establishing the identity of the murder weapon, +LJ036-0011|(3) evidence establishing the ownership of the murder weapon,|(three) evidence establishing the ownership of the murder weapon, +LJ036-0012|(4) evidence establishing the ownership of a zipper jacket|(four) evidence establishing the ownership of a zipper jacket +LJ036-0013|found along the path of flight taken by the gunman from the scene of the shooting to the place of arrest.|found along the path of flight taken by the gunman from the scene of the shooting to the place of arrest. +LJ036-0014|Oswald's Movements After Leaving Depository Building|Oswald's Movements After Leaving Depository Building +LJ036-0015|According to the reconstruction of time and events which the Commission found most credible,|According to the reconstruction of time and events which the Commission found most credible, +LJ036-0016|Lee Harvey Oswald left the building approximately 3 minutes after the assassination.|Lee Harvey Oswald left the building approximately three minutes after the assassination. +LJ036-0017|He probably walked east on Elm Street for seven blocks to the corner of Elm and Murphy|He probably walked east on Elm Street for seven blocks to the corner of Elm and Murphy +LJ036-0018|where he boarded a bus which was heading back in the direction of the Depository Building, on its way to the Oak Cliff section of Dallas.|where he boarded a bus which was heading back in the direction of the Depository Building, on its way to the Oak Cliff section of Dallas. +LJ036-0019|When Oswald was apprehended, a bus transfer marked for the Lakewood-Marsalis route was found in his shirt pocket.|When Oswald was apprehended, a bus transfer marked for the Lakewood-Marsalis route was found in his shirt pocket. +LJ036-0020|The transfer was dated "Friday November 22, '63" and was punched in two places by the bus driver.|The transfer was dated "Friday November twenty-two, 'sixty-three" and was punched in two places by the bus driver. +LJ036-0021|On the basis of this punchmark, which was distinctive to each Dallas driver,|On the basis of this punchmark, which was distinctive to each Dallas driver, +LJ036-0022|the transfer was conclusively identified as having been issued by Cecil J. McWatters, a busdriver for the Dallas Transit Co.|the transfer was conclusively identified as having been issued by Cecil J. McWatters, a busdriver for the Dallas Transit Co. +LJ036-0023|On the basis of the date and time on the transfer, McWatters was able to testify that the transfer had been issued by him|On the basis of the date and time on the transfer, McWatters was able to testify that the transfer had been issued by him +LJ036-0024|on a trip which passed a check point at St. Paul and Elm Streets at 12:36 p.m., November 22, 1963.|on a trip which passed a check point at St. Paul and Elm Streets at twelve:thirty-six p.m., November twenty-two, nineteen sixty-three. +LJ036-0025|McWatters was sure that he left the checkpoint on time|McWatters was sure that he left the checkpoint on time +LJ036-0026|and he estimated that it took him 3 to 4 minutes to drive three blocks west from the checkpoint to Field Street,|and he estimated that it took him three to four minutes to drive three blocks west from the checkpoint to Field Street, +LJ036-0027|which he reached at about 12:40 p.m.|which he reached at about twelve:forty p.m. +LJ036-0028|McWatters' recollection is that he issued this transfer to a man who entered his bus just beyond Field Street,|McWatters' recollection is that he issued this transfer to a man who entered his bus just beyond Field Street, +LJ036-0029|where a man beat on the front door of the bus, boarded it and paid his fare.|where a man beat on the front door of the bus, boarded it and paid his fare. +LJ036-0030|About two blocks later, a woman asked to get off to make a 1 o'clock train at Union Station|About two blocks later, a woman asked to get off to make a one o'clock train at Union Station +LJ036-0031|and requested a transfer which she might use if she got through the traffic.|and requested a transfer which she might use if she got through the traffic. +LJ036-0032|So I gave her a transfer and opened the door and she was going out the gentleman I had picked up about two blocks [back]|So I gave her a transfer and opened the door and she was going out the gentleman I had picked up about two blocks [back] +LJ036-0033|asked for a transfer and got off at the same place in the middle of the block where the lady did.|asked for a transfer and got off at the same place in the middle of the block where the lady did. +LJ036-0034|It was the intersection near Lamar Street, it was near Poydras and Lamar Street.|It was the intersection near Lamar Street, it was near Poydras and Lamar Street. +LJ036-0035|The man was on the bus approximately 4 minutes.|The man was on the bus approximately four minutes. +LJ036-0036|At about 6:30 p.m. on the day of the assassination, McWatters viewed four men in a police lineup.|At about six:thirty p.m. on the day of the assassination, McWatters viewed four men in a police lineup. +LJ036-0037|He picked Oswald from the lineup as the man who had boarded the bus at the, quote, lower end of town on Elm around Houston, end quote,|He picked Oswald from the lineup as the man who had boarded the bus at the, quote, lower end of town on Elm around Houston, end quote, +LJ036-0038|and who, during the ride south on Marsalis, had an argument with a woman passenger.|and who, during the ride south on Marsalis, had an argument with a woman passenger. +LJ036-0039|In his Commission testimony, McWatters said he had been in error and that a teenager named Milton Jones was the passenger he had in mind.|In his Commission testimony, McWatters said he had been in error and that a teenager named Milton Jones was the passenger he had in mind. +LJ036-0040|In a later interview, Jones confirmed that he had exchanged words with a woman passenger on the bus during the ride south on Marsalis.|In a later interview, Jones confirmed that he had exchanged words with a woman passenger on the bus during the ride south on Marsalis. +LJ036-0041|McWatters also remembered that a man received a transfer at Lamar and Elm Streets and that a man in the lineup was about the size of this man.|McWatters also remembered that a man received a transfer at Lamar and Elm Streets and that a man in the lineup was about the size of this man. +LJ036-0042|However, McWatters' recollection alone was too vague to be a basis for placing Oswald on the bus.|However, McWatters' recollection alone was too vague to be a basis for placing Oswald on the bus. +LJ036-0043|Riding on the bus was an elderly woman, Mary Bledsoe, who confirmed the mute evidence of the transfer.|Riding on the bus was an elderly woman, Mary Bledsoe, who confirmed the mute evidence of the transfer. +LJ036-0044|Oswald had rented a room from Mrs. Bledsoe about 6 weeks before, on October 7, but she had asked him to leave at the end of a week.|Oswald had rented a room from Mrs. Bledsoe about six weeks before, on October seven, but she had asked him to leave at the end of a week. +LJ036-0045|Mrs. Bledsoe told him, quote, I am not going to rent to you any more, end quote,|Mrs. Bledsoe told him, quote, I am not going to rent to you any more, end quote, +LJ036-0046|She testified, quote, I didn't like his attitude. There was just something about him I didn't like or want him. Just didn't want him around me, end quote,|She testified, quote, I didn't like his attitude. There was just something about him I didn't like or want him. Just didn't want him around me, end quote, +LJ036-0047|On November 22, Mrs. Bledsoe came downtown to watch the Presidential motorcade.|On November twenty-two, Mrs. Bledsoe came downtown to watch the Presidential motorcade. +LJ036-0048|She boarded the Marsalis bus at St. Paul and Elm Streets to return home. She testified further, quote,|She boarded the Marsalis bus at St. Paul and Elm Streets to return home. She testified further, quote, +LJ036-0049|And, after we got past Akard, at Murphy -- I figured it out. Let's see. I don't know for sure.|And, after we got past Akard, at Murphy -- I figured it out. Let's see. I don't know for sure. +LJ036-0050|Oswald got on. He looks like a maniac. His sleeve was out here. His shirt was undone.|Oswald got on. He looks like a maniac. His sleeve was out here. His shirt was undone. +LJ036-0051|Was a hole in it, hole, and he was dirty, and I didn't look at him. I didn't want to know I even seen him.|Was a hole in it, hole, and he was dirty, and I didn't look at him. I didn't want to know I even seen him. +LJ036-0052|he looked so bad in his face, and his face was so distorted. Hole in his sleeve right here.|he looked so bad in his face, and his face was so distorted. Hole in his sleeve right here. +LJ036-0053|End quote. As Mrs. Bledsoe said these words, she pointed to her right elbow.|End quote. As Mrs. Bledsoe said these words, she pointed to her right elbow. +LJ036-0054|When Oswald was arrested in the Texas Theatre, he was wearing a brown sport shirt with a hole in the right sleeve at the elbow.|When Oswald was arrested in the Texas Theatre, he was wearing a brown sport shirt with a hole in the right sleeve at the elbow. +LJ036-0055|Mrs. Bledsoe identified the shirt as the one Oswald was wearing and she stated she was certain that it was Oswald who boarded the bus.|Mrs. Bledsoe identified the shirt as the one Oswald was wearing and she stated she was certain that it was Oswald who boarded the bus. +LJ036-0056|Mrs. Bledsoe recalled that Oswald sat halfway to the rear of the bus which moved slowly and intermittently as traffic became heavy.|Mrs. Bledsoe recalled that Oswald sat halfway to the rear of the bus which moved slowly and intermittently as traffic became heavy. +LJ036-0057|She heard a passing motorist tell the driver that the President had been shot.|She heard a passing motorist tell the driver that the President had been shot. +LJ036-0058|People on the bus began talking about it. As the bus neared Lamar Street, Oswald left the bus and disappeared into the crowd.|People on the bus began talking about it. As the bus neared Lamar Street, Oswald left the bus and disappeared into the crowd. +LJ036-0059|The Marsalis bus which Oswald boarded traveled a route west on Elm,|The Marsalis bus which Oswald boarded traveled a route west on Elm, +LJ036-0060|south on Houston, and southwest across the Houston viaduct to service the Oak Cliff area along Marsalis.|south on Houston, and southwest across the Houston viaduct to service the Oak Cliff area along Marsalis. +LJ036-0061|A Beckley bus which also served the Oak Cliff area,|A Beckley bus which also served the Oak Cliff area, +LJ036-0062|followed the same route as the Marsalis bus through downtown Dallas, except that it continued west on Elm,|followed the same route as the Marsalis bus through downtown Dallas, except that it continued west on Elm, +LJ036-0063|across Houston in front of the Depository Building, past the Triple Underpass into west Dallas, and south on Beckley.|across Houston in front of the Depository Building, past the Triple Underpass into west Dallas, and south on Beckley. +LJ036-0064|Marsalis Street is seven blocks from Beckley.|Marsalis Street is seven blocks from Beckley. +LJ036-0065|Oswald lived at 1026 North Beckley.|Oswald lived at ten twenty-six North Beckley. +LJ036-0066|He could not reach his roominghouse on the Marsalis bus, but the Beckley bus stopped across the street.|He could not reach his roominghouse on the Marsalis bus, but the Beckley bus stopped across the street. +LJ036-0067|According to McWatters, the Beckley bus was behind the Marsalis bus, but he did not actually see it.|According to McWatters, the Beckley bus was behind the Marsalis bus, but he did not actually see it. +LJ036-0068|Both buses stopped within one block of the Depository Building.|Both buses stopped within one block of the Depository Building. +LJ036-0069|Instead of waiting there, Oswald apparently went as far away as he could and boarded the first Oak Cliff bus which came along|Instead of waiting there, Oswald apparently went as far away as he could and boarded the first Oak Cliff bus which came along +LJ036-0070|rather than wait for one which stopped across the street from his roominghouse.|rather than wait for one which stopped across the street from his roominghouse. +LJ036-0071|In a reconstruction of this bus trip, agents of the Secret Service and the FBI walked the seven blocks from the front entrance of the Depository Building|In a reconstruction of this bus trip, agents of the Secret Service and the FBI walked the seven blocks from the front entrance of the Depository Building +LJ036-0072|to Murphy and Elm three times, averaging 6.5 minutes for the three trips.|to Murphy and Elm three times, averaging six point five minutes for the three trips. +LJ036-0073|A bus moving through heavy traffic on Elm from Murphy to Lamar was timed at 4 minutes.|A bus moving through heavy traffic on Elm from Murphy to Lamar was timed at four minutes. +LJ036-0074|If Oswald left the Depository Building at 12:33 p.m.,|If Oswald left the Depository Building at twelve:thirty-three p.m., +LJ036-0075|walked seven blocks directly to Murphy and Elm, and boarded a bus almost immediately,|walked seven blocks directly to Murphy and Elm, and boarded a bus almost immediately, +LJ036-0076|he would have boarded the bus at approximately 12:40 p.m. and left it at approximately 12:44 p.m.|he would have boarded the bus at approximately twelve:forty p.m. and left it at approximately twelve:forty-four p.m. +LJ036-0077|Roger D. Craig, a deputy sheriff of Dallas County,|Roger D. Craig, a deputy sheriff of Dallas County, +LJ036-0078|claimed that about 15 minutes after the assassination he saw a man, whom he later identified as Oswald,|claimed that about fifteen minutes after the assassination he saw a man, whom he later identified as Oswald, +LJ036-0079|coming from the direction of the Depository Building and running down the hill north of Elm Street|coming from the direction of the Depository Building and running down the hill north of Elm Street +LJ036-0080|toward a light-colored Rambler station wagon, which was moving slowly along Elm toward the underpass:|toward a light-colored Rambler station wagon, which was moving slowly along Elm toward the underpass: +LJ036-0081|The station wagon stopped to pick up the man and then drove off.|The station wagon stopped to pick up the man and then drove off. +LJ036-0082|Craig testified that later in the afternoon he saw Oswald in the police interrogation room|Craig testified that later in the afternoon he saw Oswald in the police interrogation room +LJ036-0083|and told Captain Fritz that Oswald was the man he saw.|and told Captain Fritz that Oswald was the man he saw. +LJ036-0084|Craig also claimed that when Fritz pointed out to Oswald that Craig had identified him,|Craig also claimed that when Fritz pointed out to Oswald that Craig had identified him, +LJ036-0085|Oswald rose from his chair, looked directly at Fritz, and said, quote, Everybody will know who I am now, end quote,|Oswald rose from his chair, looked directly at Fritz, and said, quote, Everybody will know who I am now, end quote, +LJ036-0086|The Commission could not accept important elements of Craig's testimony.|The Commission could not accept important elements of Craig's testimony. +LJ036-0087|Captain Fritz stated that a deputy sheriff whom he could not identify did ask to see him that afternoon|Captain Fritz stated that a deputy sheriff whom he could not identify did ask to see him that afternoon +LJ036-0088|and told him a similar story to Craig's.|and told him a similar story to Craig's. +LJ036-0089|Fritz did not bring him into his office to identify Oswald but turned him over to Lieutenant Baker for questioning.|Fritz did not bring him into his office to identify Oswald but turned him over to Lieutenant Baker for questioning. +LJ036-0090|If Craig saw Oswald that afternoon, he saw him through the glass windows of the office.|If Craig saw Oswald that afternoon, he saw him through the glass windows of the office. +LJ036-0091|And neither Captain Fritz nor any other officer can remember that Oswald dramatically arose from his chair|And neither Captain Fritz nor any other officer can remember that Oswald dramatically arose from his chair +LJ036-0092|and said, quote, Everybody will know who I am now, end quote.|and said, quote, Everybody will know who I am now, end quote. +LJ036-0093|If Oswald had made such a statement, Captain Fritz and others present would probably have remembered it.|If Oswald had made such a statement, Captain Fritz and others present would probably have remembered it. +LJ036-0094|Craig may have seen a person enter a white Rambler station wagon 15 or 20 minutes after the shooting and travel west on Elm Street|Craig may have seen a person enter a white Rambler station wagon fifteen or twenty minutes after the shooting and travel west on Elm Street +LJ036-0095|but the Commission concluded that this man was not Lee Harvey Oswald,|but the Commission concluded that this man was not Lee Harvey Oswald, +LJ036-0096|because of the overwhelming evidence that Oswald was far away from the building by that time.|because of the overwhelming evidence that Oswald was far away from the building by that time. +LJ036-0097|The taxicab ride.|The taxicab ride. +LJ036-0098|William Whaley, a taxicab driver, told his employer on Saturday morning, November 23|William Whaley, a taxicab driver, told his employer on Saturday morning, November twenty-three +LJ036-0099|that he recognized Oswald from a newspaper photograph as a man whom he had driven to the Oak Cliff area the day before.|that he recognized Oswald from a newspaper photograph as a man whom he had driven to the Oak Cliff area the day before. +LJ036-0100|Notified of Whaley's statement, the police brought him to the police station that afternoon.|Notified of Whaley's statement, the police brought him to the police station that afternoon. +LJ036-0101|He was taken to the lineup room where, according to Whaley, five young teenagers, all handcuffed together, were displayed with Oswald.|He was taken to the lineup room where, according to Whaley, five young teenagers, all handcuffed together, were displayed with Oswald. +LJ036-0102|He testified that Oswald looked older than the other boys.|He testified that Oswald looked older than the other boys. +LJ036-0103|The police asked him whether he could pick out his passenger from the lineup.|The police asked him whether he could pick out his passenger from the lineup. +LJ036-0104|Whaley picked Oswald.|Whaley picked Oswald. +LJ036-0105|He said, quote,|He said, quote, +LJ036-0106|You could have picked him out without identifying him by just listening to him.|You could have picked him out without identifying him by just listening to him. +LJ036-0107|because he was bawling out the policeman, telling them it wasn't right to put him in line with these teenagers and all of that|because he was bawling out the policeman, telling them it wasn't right to put him in line with these teenagers and all of that +LJ036-0108|and they asked me which one and I told them. It was him all right, the same man. He showed no respect for the policemen,|and they asked me which one and I told them. It was him all right, the same man. He showed no respect for the policemen, +LJ036-0109|he told them what he thought about them. They knew what they were doing and they were trying to railroad him and he wanted his lawyer.|he told them what he thought about them. They knew what they were doing and they were trying to railroad him and he wanted his lawyer. +LJ036-0110|Whaley believes that Oswald's conduct did not aid him in his identification, quote, because I knew he was the right one as soon as I saw him.|Whaley believes that Oswald's conduct did not aid him in his identification, quote, because I knew he was the right one as soon as I saw him. +LJ036-0111|Whaley's memory of the lineup is inaccurate. There were four men altogether, not six men, in the lineup with Oswald.|Whaley's memory of the lineup is inaccurate. There were four men altogether, not six men, in the lineup with Oswald. +LJ036-0112|Whaley said that Oswald was the man under No. 2.|Whaley said that Oswald was the man under Number two. +LJ036-0113|Actually Oswald was under No. 3.|Actually Oswald was under Number three. +LJ036-0114|Only two of the men in the lineup with Oswald were teenagers: John T. Horn, aged 18, was No. 1;|Only two of the men in the lineup with Oswald were teenagers: John T. Horn, aged eighteen, was Number one; +LJ036-0115|David Knapp, aged 18, was No. 2;|David Knapp, aged eighteen, was Number two; +LJ036-0116|Lee Oswald was No. 3;|Lee Oswald was Number three; +LJ036-0117|and Daniel Lujan, aged 26, was No. 4.|and Daniel Lujan, aged twenty-six, was Number four. +LJ036-0118|When he first testified before the Commission,|When he first testified before the Commission, +LJ036-0119|Whaley displayed a trip manifest which showed a 12 o'clock trip from Travis Hotel to the Continental bus station,|Whaley displayed a trip manifest which showed a twelve o'clock trip from Travis Hotel to the Continental bus station, +LJ036-0120|unloaded at 12:15 p.m.,|unloaded at twelve:fifteen p.m., +LJ036-0121|a 12:15 p.m. pickup at Continental to Greyhound, unloaded at 12:30 p.m.,|a twelve:fifteen p.m. pickup at Continental to Greyhound, unloaded at twelve:thirty p.m., +LJ036-0122|and a pickup from Greyhound (bus station) at 12:30 p.m.,|and a pickup from Greyhound (bus station) at twelve:thirty p.m., +LJ036-0123|unloaded at 500 North Beckley at 12:45 p.m.|unloaded at five hundred North Beckley at twelve:forty-five p.m. +LJ036-0124|Whaley testified that he did not keep an accurate time record of his trips but recorded them by the quarter hour,|Whaley testified that he did not keep an accurate time record of his trips but recorded them by the quarter hour, +LJ036-0125|and that sometimes he made his entry right after a trip while at other times he waited to record three or four trips.|and that sometimes he made his entry right after a trip while at other times he waited to record three or four trips. +LJ036-0126|As he unloaded his Continental bus station passenger in front of Greyhound, he started to get out to buy a package of cigarettes.|As he unloaded his Continental bus station passenger in front of Greyhound, he started to get out to buy a package of cigarettes. +LJ036-0127|He saw a man walking south on Lamar from Commerce.|He saw a man walking south on Lamar from Commerce. +LJ036-0128|The man was dressed in faded blue color khaki work clothes, a brown shirt, and some kind of work jacket that almost matched his pants.|The man was dressed in faded blue color khaki work clothes, a brown shirt, and some kind of work jacket that almost matched his pants. +LJ036-0129|The man asked, quote, May I have the cab?, end quote, and got into the front seat.|The man asked, quote, May I have the cab?, end quote, and got into the front seat. +LJ036-0130|Whaley described the ensuing events as follows, quote,|Whaley described the ensuing events as follows, quote, +LJ036-0131|And about that time an old lady, I think she was an old lady, I don't remember nothing but her sticking her head down past him in the door and said,|And about that time an old lady, I think she was an old lady, I don't remember nothing but her sticking her head down past him in the door and said, +LJ036-0132|Driver, will you call me a cab down here? She had seen him get this cab and she wanted one, too,|Driver, will you call me a cab down here? She had seen him get this cab and she wanted one, too, +LJ036-0133|and he opened the door a little bit like he was going to get out and he said, "I will let you have this one," and she says, "No, the driver can call me one."|and he opened the door a little bit like he was going to get out and he said, "I will let you have this one," and she says, "No, the driver can call me one." +LJ036-0134|I asked him where he wanted to go. And he said, "500 North Beckley. Well, I started up,|I asked him where he wanted to go. And he said, "five hundred North Beckley. Well, I started up, +LJ036-0135|I started to that address, and the police cars, the sirens was going, running crisscrossing everywhere, just a big uproar in that end of town and I said,|I started to that address, and the police cars, the sirens was going, running crisscrossing everywhere, just a big uproar in that end of town and I said, +LJ036-0136|What the hell. I wonder what the hell is the uproar?|What the hell. I wonder what the hell is the uproar? +LJ036-0137|And he never said anything. So I figured he was one of these people that don't like to talk so I never said any more to him.|And he never said anything. So I figured he was one of these people that don't like to talk so I never said any more to him. +LJ036-0138|But when I got pretty close to 500 block at Neches and North Beckley which is the 500 block, he said, "This will do fine," and I pulled over to the curb right there.|But when I got pretty close to five hundred block at Neches and North Beckley which is the five hundred block, he said, "This will do fine," and I pulled over to the curb right there. +LJ036-0139|He gave me a dollar bill, the trip was 95 cents.|He gave me a dollar bill, the trip was ninety-five cents. +LJ036-0140|He gave me a dollar bill and didn't say anything, just got out and closed the door and walked around the front of the cab over to the other side of the street|He gave me a dollar bill and didn't say anything, just got out and closed the door and walked around the front of the cab over to the other side of the street +LJ036-0141|[east side of the street]. Of course, the traffic was moving through there and I put it in gear and moved on, that is the last I saw of him.|[east side of the street]. Of course, the traffic was moving through there and I put it in gear and moved on, that is the last I saw of him. +LJ036-0142|Whaley was somewhat imprecise as to where he unloaded his passenger.|Whaley was somewhat imprecise as to where he unloaded his passenger. +LJ036-0143|He marked what, he thought was the intersection of Neches and Beckley on a map of Dallas with a large "X."|He marked what, he thought was the intersection of Neches and Beckley on a map of Dallas with a large "X." +LJ036-0144|He said, quote, Yes, sir; that is right, because that is the 500 block of North Beckley, end quote,|He said, quote, Yes, sir; that is right, because that is the five hundred block of North Beckley, end quote, +LJ036-0145|However, Neches and Beckley do not intersect.|However, Neches and Beckley do not intersect. +LJ036-0146|Neches is within one-half block of the roominghouse at 1026 North Beckley where Oswald was living.|Neches is within one-half block of the roominghouse at ten twenty-six North Beckley where Oswald was living. +LJ036-0147|The 500 block of North Beckley is five blocks south of the roominghouse.|The five hundred block of North Beckley is five blocks south of the roominghouse. +LJ036-0148|After a review of these inconsistencies in his testimony before the Commission, Whaley was interviewed again in Dallas.|After a review of these inconsistencies in his testimony before the Commission, Whaley was interviewed again in Dallas. +LJ036-0149|The route of the taxicab was retraced under the direction of Whaley.|The route of the taxicab was retraced under the direction of Whaley. +LJ036-0150|He directed the driver of the car to a point 20 feet north of the northwest corner of the intersection of Beckley and Neely|He directed the driver of the car to a point twenty feet north of the northwest corner of the intersection of Beckley and Neely +LJ036-0151|the point at which he said his passenger alighted.|the point at which he said his passenger alighted. +LJ036-0152|This was the 700 block of North Beckley|This was the seven hundred block of North Beckley +LJ036-0153|The elapsed time of the reconstructed run from the Greyhound Bus Station to Neely and Beckley was 5 minutes and 30 seconds by stopwatch.|The elapsed time of the reconstructed run from the Greyhound Bus Station to Neely and Beckley was five minutes and thirty seconds by stopwatch. +LJ036-0154|The walk from Beckley and Neely to 1026 North Beckley was timed by Commission counsel at 5 minutes and 45 seconds.|The walk from Beckley and Neely to ten twenty-six North Beckley was timed by Commission counsel at five minutes and forty-five seconds. +LJ036-0155|Whaley testified that Oswald was wearing either the gray zippered jacket or the heavy blue jacket.|Whaley testified that Oswald was wearing either the gray zippered jacket or the heavy blue jacket. +LJ036-0156|He was in error, however.|He was in error, however. +LJ036-0157|Oswald could not possibly have been wearing the blue jacket during the trip with Whaley, since it was found in the "domino" room of the Depository late in November.|Oswald could not possibly have been wearing the blue jacket during the trip with Whaley, since it was found in the "domino" room of the Depository late in November. +LJ036-0158|Moreover, Mrs. Bledsoe saw Oswald in the bus without a jacket and wearing a shirt with a hole at the elbow.|Moreover, Mrs. Bledsoe saw Oswald in the bus without a jacket and wearing a shirt with a hole at the elbow. +LJ036-0159|On the other hand, Whaley identified Commission Exhibit No. 150 (the shirt taken from Oswald upon arrest) as the shirt his passenger was wearing.|On the other hand, Whaley identified Commission Exhibit Number one fifty (the shirt taken from Oswald upon arrest) as the shirt his passenger was wearing. +LJ036-0160|He also stated he saw a silver identification bracelet on his passenger's left wrist.|He also stated he saw a silver identification bracelet on his passenger's left wrist. +LJ036-0161|Oswald was wearing such a bracelet when he was arrested.|Oswald was wearing such a bracelet when he was arrested. +LJ036-0162|On November 22, Oswald told Captain Fritz that he rode a bus to a stop near his home and then walked to his roominghouse.|On November twenty-two, Oswald told Captain Fritz that he rode a bus to a stop near his home and then walked to his roominghouse. +LJ036-0163|When queried the following morning concerning a bus transfer found in his possession at the time of his arrest, he admitted receiving it.|When queried the following morning concerning a bus transfer found in his possession at the time of his arrest, he admitted receiving it. +LJ036-0164|And when interrogated about a cab ride, Oswald also admitted that he left the slow-moving bus and took a cab to his roominghouse.|And when interrogated about a cab ride, Oswald also admitted that he left the slow-moving bus and took a cab to his roominghouse. +LJ036-0165|The Greyhound Bus Station at Lamar and Jackson Streets, where Oswald entered Whaley's cab,|The Greyhound Bus Station at Lamar and Jackson Streets, where Oswald entered Whaley's cab, +LJ036-0166|is three to four short blocks south of Lamar and Elm. If Oswald left the bus at 12:44 p.m.|is three to four short blocks south of Lamar and Elm. If Oswald left the bus at twelve:forty-four p.m. +LJ036-0167|and walked directly to the terminal, he would have entered the cab at 12:47 or 12:48 p.m.|and walked directly to the terminal, he would have entered the cab at twelve:forty-seven or twelve:forty-eight p.m. +LJ036-0168|If the cab ride was approximately 6 minutes, as was the reconstructed ride,|If the cab ride was approximately six minutes, as was the reconstructed ride, +LJ036-0169|he would have reached his destination at approximately 12:54 p.m.|he would have reached his destination at approximately twelve:fifty-four p.m. +LJ036-0170|If he was discharged at Neely and Beckley and walked directly to his roominghouse,|If he was discharged at Neely and Beckley and walked directly to his roominghouse, +LJ036-0171|he would have arrived there about 12:59 to 1 p.m.|he would have arrived there about twelve:fifty-nine to one p.m. +LJ036-0172|From the 500 block of North Beckley, the walk would be a few minutes longer,|From the five hundred block of North Beckley, the walk would be a few minutes longer, +LJ036-0173|but in either event he would have been in the roominghouse at about 1 p.m.|but in either event he would have been in the roominghouse at about one p.m. +LJ036-0174|This is the approximate time he entered the roominghouse, according to Earlene Roberts, the housekeeper there.|This is the approximate time he entered the roominghouse, according to Earlene Roberts, the housekeeper there. +LJ036-0175|Arrival and departure from roominghouse.|Arrival and departure from roominghouse. +LJ036-0176|Earlene Roberts, housekeeper for Mrs. A. C. Johnson at 1026 North Beckley|Earlene Roberts, housekeeper for Mrs. A. C. Johnson at ten twenty-six North Beckley +LJ036-0177|knew Lee Harvey Oswald under the alias of O. H. Lee.|knew Lee Harvey Oswald under the alias of O. H. Lee. +LJ036-0178|She first saw him the day he rented a room at that address on October 14, 1963.|She first saw him the day he rented a room at that address on October fourteen, nineteen sixty-three. +LJ036-0179|He signed his name as O. H. Lee on the roominghouse register.|He signed his name as O. H. Lee on the roominghouse register. +LJ036-0180|Mrs. Roberts testified that on Thursday, November 21, Oswald did not come home.|Mrs. Roberts testified that on Thursday, November twenty-one, Oswald did not come home. +LJ036-0181|On Friday, November 22, about 1 p.m., he entered the house in unusual haste.|On Friday, November twenty-two, about one p.m., he entered the house in unusual haste. +LJ036-0182|She recalled that it was subsequent to the time the President had been shot.|She recalled that it was subsequent to the time the President had been shot. +LJ036-0183|After a friend had called and told her, "President Kennedy has been shot," she turned on the television.|After a friend had called and told her, "President Kennedy has been shot," she turned on the television. +LJ036-0184|When Oswald came in she said, quote, Oh, you are in a hurry, end quote, but Oswald did not respond.|When Oswald came in she said, quote, Oh, you are in a hurry, end quote, but Oswald did not respond. +LJ036-0185|He hurried to his room and stayed no longer than 3 or 4 minutes.|He hurried to his room and stayed no longer than three or four minutes. +LJ036-0186|Oswald had entered the house in his shirt sleeves, but when he left, he was zipping up a jacket.|Oswald had entered the house in his shirt sleeves, but when he left, he was zipping up a jacket. +LJ036-0187|Mrs. Roberts saw him a few seconds later standing near the bus stop in front of the house on the east side of Beckley.|Mrs. Roberts saw him a few seconds later standing near the bus stop in front of the house on the east side of Beckley. +LJ036-0188|Oswald was next seen about nine-tenths of a mile away|Oswald was next seen about nine-tenths of a mile away +LJ036-0189|at the southeast corner of 10th Street and Patton Avenue, moments before the Tippit shooting.|at the southeast corner of tenth Street and Patton Avenue, moments before the Tippit shooting. +LJ036-0190|If Oswald left his roominghouse shortly after 1 p.m. and walked at a brisk pace,|If Oswald left his roominghouse shortly after one p.m. and walked at a brisk pace, +LJ036-0191|he would have reached 10th and Patton shortly after 1:15 p.m.|he would have reached tenth and Patton shortly after one:fifteen p.m. +LJ036-0192|Tippit's murder was recorded on the police radio tape at about 1:16 p.m.|Tippit's murder was recorded on the police radio tape at about one:sixteen p.m. +LJ036-0193|Description of Shooting|Description of Shooting +LJ036-0194|Patrolman J. D. Tippit joined the Dallas Police Department in July 1952.|Patrolman J. D. Tippit joined the Dallas Police Department in July nineteen fifty-two. +LJ036-0195|He was described by Chief Curry as having the reputation of being "a very fine, dedicated officer."|He was described by Chief Curry as having the reputation of being "a very fine, dedicated officer." +LJ036-0196|Tippit patroled district No. 78 in the Oak Cliff area of Dallas during daylight hours.|Tippit patroled district Number seventy-eight in the Oak Cliff area of Dallas during daylight hours. +LJ036-0197|He drove a police car painted distinctive colors with No. 10 prominently displayed on each side.|He drove a police car painted distinctive colors with Number ten prominently displayed on each side. +LJ036-0198|Tippit rode alone, as only one man was normally assigned to a patrol car in residential areas during daylight shifts.|Tippit rode alone, as only one man was normally assigned to a patrol car in residential areas during daylight shifts. +LJ036-0199|At about 12:44 p.m. on November 22,|At about twelve:forty-four p.m. on November twenty-two, +LJ036-0200|the radio dispatcher on channel 1 ordered all downtown patrol squads to report to Elm and Houston, code 3 (emergency).|the radio dispatcher on channel one ordered all downtown patrol squads to report to Elm and Houston, code three (emergency). +LJ036-0201|At 12:45 p.m. the dispatcher ordered No. 78 (Tippit) to, quote, move into central Oak Cliff area, end quote.|At twelve:forty-five p.m. the dispatcher ordered Number seventy-eight (Tippit) to, quote, move into central Oak Cliff area, end quote. +LJ036-0202|At 12:54 p.m., Tippit reported that he was in the central Oak Cliff area at Lancaster and Eighth.|At twelve:fifty-four p.m., Tippit reported that he was in the central Oak Cliff area at Lancaster and Eighth. +LJ036-0203|The dispatcher ordered Tippit to be, quote, at large for any emergency that comes in, end quote.|The dispatcher ordered Tippit to be, quote, at large for any emergency that comes in, end quote. +LJ036-0204|According to Chief Curry, Tippit was free to patrol the central Oak Cliff area.|According to Chief Curry, Tippit was free to patrol the central Oak Cliff area. +LJ036-0205|Tippit must have heard the description of the suspect wanted for the President's shooting; it was broadcast over channel 1 at 12:45 p.m.,|Tippit must have heard the description of the suspect wanted for the President's shooting; it was broadcast over channel one at twelve:forty-five p.m., +LJ036-0206|again at 12:48 p.m., and again at 12:55 p.m.|again at twelve:forty-eight p.m., and again at twelve:fifty-five p.m. +LJ036-0207|The suspect was described as a, quote,|The suspect was described as a, quote, +LJ036-0208|white male, approximately 30, slender build, height 5 foot 10 inches, weight 165 pounds, end quote.|white male, approximately thirty, slender build, height five foot ten inches, weight one hundred sixty-five pounds, end quote. +LJ036-0209|A similar description was given on channel 2 at 12:45 p.m.|A similar description was given on channel two at twelve:forty-five p.m. +LJ036-0210|At approximately 1:15 p.m., Tippit, who was cruising east on 10th Street,|At approximately one:fifteen p.m., Tippit, who was cruising east on tenth Street, +LJ036-0211|passed the intersection of 10th and Patton, about eight blocks from where he had reported at 12:54 p.m.|passed the intersection of tenth and Patton, about eight blocks from where he had reported at twelve:fifty-four p.m. +LJ036-0212|About 100 feet past the intersection Tippit stopped a man walking east along the south side of Patton.|About one hundred feet past the intersection Tippit stopped a man walking east along the south side of Patton. +LJ036-0213|The man's general description was similar to the one broadcast over the police radio.|The man's general description was similar to the one broadcast over the police radio. +LJ036-0214|Tippit stopped the man and called him to his car.|Tippit stopped the man and called him to his car. +LJ036-0215|He approached the car and apparently exchanged words with Tippit through the right front or vent window.|He approached the car and apparently exchanged words with Tippit through the right front or vent window. +LJ036-0217|As Tippit reached the left front wheel the man pulled out a revolver and fired several shots.|As Tippit reached the left front wheel the man pulled out a revolver and fired several shots. +LJ036-0218|Four bullets hit Tippit and killed him instantly.|Four bullets hit Tippit and killed him instantly. +LJ037-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ037-0002|Chapter 4. The Assassin: Part 6.|Chapter four. The Assassin: Part six. +LJ037-0003|Eyewitnesses|Eyewitnesses +LJ037-0004|At least 12 persons saw the man with the revolver in the vicinity of the Tippit crime scene at or immediately after the shooting.|At least twelve persons saw the man with the revolver in the vicinity of the Tippit crime scene at or immediately after the shooting. +LJ037-0005|By the evening of November 22, five of them had identified Lee Harvey Oswald in police lineups as the man they saw.|By the evening of November twenty-two, five of them had identified Lee Harvey Oswald in police lineups as the man they saw. +LJ037-0006|A sixth did so the next day.|A sixth did so the next day. +LJ037-0007|Three others subsequently identified Oswald from a photograph.|Three others subsequently identified Oswald from a photograph. +LJ037-0008|Two witnesses testified that Oswald resembled the man they had seen.|Two witnesses testified that Oswald resembled the man they had seen. +LJ037-0009|One witness felt he was too distant from the gunman to make a positive identification.|One witness felt he was too distant from the gunman to make a positive identification. +LJ037-0010|A taxi driver, William Scoggins,|A taxi driver, William Scoggins, +LJ037-0011|was eating lunch in his cab which was parked on Patton facing the southeast corner of 10th Street and Patton Avenue a few feet to the north.|was eating lunch in his cab which was parked on Patton facing the southeast corner of tenth Street and Patton Avenue a few feet to the north. +LJ037-0012|A police car moving east on 10th at about 10 or 12 miles an hour passed in front of his cab.|A police car moving east on tenth at about ten or twelve miles an hour passed in front of his cab. +LJ037-0013|About 100 feet from the corner the police car pulled up alongside a man on the sidewalk. This man, dressed in a light-colored jacket, approached the car.|About one hundred feet from the corner the police car pulled up alongside a man on the sidewalk. This man, dressed in a light-colored jacket, approached the car. +LJ037-0014|Scoggins lost sight of him behind some shrubbery on the southeast corner lot,|Scoggins lost sight of him behind some shrubbery on the southeast corner lot, +LJ037-0015|but he saw the policeman leave the car, heard three or four shots, and then saw the policeman fall.|but he saw the policeman leave the car, heard three or four shots, and then saw the policeman fall. +LJ037-0016|Scoggins hurriedly left his seat and hid behind the cab as the man came back toward the corner with gun in hand.|Scoggins hurriedly left his seat and hid behind the cab as the man came back toward the corner with gun in hand. +LJ037-0017|The man cut across the yard through some bushes, passed within 12 feet of Scoggins, and ran south on Patton.|The man cut across the yard through some bushes, passed within twelve feet of Scoggins, and ran south on Patton. +LJ037-0018|Scoggins saw him and heard him mutter either "Poor damn cop" or "Poor dumb cop."|Scoggins saw him and heard him mutter either "Poor damn cop" or "Poor dumb cop." +LJ037-0019|The next day Scoggins viewed a lineup of four persons and identified Oswald as the man whom he had seen the day before at 10th and Patton.|The next day Scoggins viewed a lineup of four persons and identified Oswald as the man whom he had seen the day before at tenth and Patton. +LJ037-0020|In his testimony before the Commission,|In his testimony before the Commission, +LJ037-0021|Scoggins stated that he thought he had seen a picture of Oswald in the newspapers prior to the lineup identification on Saturday.|Scoggins stated that he thought he had seen a picture of Oswald in the newspapers prior to the lineup identification on Saturday. +LJ037-0022|He had not seen Oswald on television and had not been shown any photographs of Oswald by the police.|He had not seen Oswald on television and had not been shown any photographs of Oswald by the police. +LJ037-0023|Another witness, Domingo Benavides, was driving a pickup truck west on 10th Street.|Another witness, Domingo Benavides, was driving a pickup truck west on tenth Street. +LJ037-0024|As he crossed the intersection a block east of 10th and Patton, he saw a policeman standing by the left door of the police car parked along the south side of 10th.|As he crossed the intersection a block east of tenth and Patton, he saw a policeman standing by the left door of the police car parked along the south side of tenth. +LJ037-0025|Benavides saw a man standing at the right side of the parked police car. He then heard three shots and saw the policeman fall to the ground.|Benavides saw a man standing at the right side of the parked police car. He then heard three shots and saw the policeman fall to the ground. +LJ037-0026|By this time the pickup truck was across the street and about 25 feet from the police car.|By this time the pickup truck was across the street and about twenty-five feet from the police car. +LJ037-0027|Benavides stopped and waited in the truck until the gunman ran to the corner.|Benavides stopped and waited in the truck until the gunman ran to the corner. +LJ037-0028|He saw him empty the gun and throw the shells into some bushes on the southeast corner lot.|He saw him empty the gun and throw the shells into some bushes on the southeast corner lot. +LJ037-0029|It was Benavides, using Tippit's car radio, who first reported the killing of Patrolman Tippit at about 1:16 p.m.:|It was Benavides, using Tippit's car radio, who first reported the killing of Patrolman Tippit at about one:sixteen p.m.: +LJ037-0030|quote, We've had a shooting out here, end quote,|quote, We've had a shooting out here, end quote, +LJ037-0031|He found two empty shells in the bushes and gave them to Patrolman J. M. Poe who arrived on the scene shortly after the shooting.|He found two empty shells in the bushes and gave them to Patrolman J. M. Poe who arrived on the scene shortly after the shooting. +LJ037-0032|Benavides never saw Oswald after the arrest.|Benavides never saw Oswald after the arrest. +LJ037-0033|When questioned by police officers on the evening of November 22, Benavides told them that he did not think that he could identify the man who fired the shots.|When questioned by police officers on the evening of November twenty-two, Benavides told them that he did not think that he could identify the man who fired the shots. +LJ037-0034|As a result, they did not take him to the police station.|As a result, they did not take him to the police station. +LJ037-0035|He testified that the picture of Oswald which he saw later on television bore a resemblance to the man who shot Officer Tippit.|He testified that the picture of Oswald which he saw later on television bore a resemblance to the man who shot Officer Tippit. +LJ037-0036|Just prior to the shooting, Mrs. Helen Markham, a waitress in downtown Dallas, was about to cross 10th Street at Patton.|Just prior to the shooting, Mrs. Helen Markham, a waitress in downtown Dallas, was about to cross tenth Street at Patton. +LJ037-0037|As she waited on the northwest corner of the intersection for traffic to pass, she noticed a young man as he was, quote, almost ready to get up on the curb, end quote.|As she waited on the northwest corner of the intersection for traffic to pass, she noticed a young man as he was, quote, almost ready to get up on the curb, end quote. +LJ037-0038|at the southeast corner of the intersection, approximately 50 feet away.|at the southeast corner of the intersection, approximately fifty feet away. +LJ037-0039|The man continued along 10th Street. Mrs. Markham saw a police car slowly approach the man from the rear and stop alongside of him.|The man continued along tenth Street. Mrs. Markham saw a police car slowly approach the man from the rear and stop alongside of him. +LJ037-0040|She saw the man come to the right window of the police car. As he talked, he leaned on the ledge of the right window with his arms.|She saw the man come to the right window of the police car. As he talked, he leaned on the ledge of the right window with his arms. +LJ037-0041|The man appeared to step back as the policeman, quote, calmly opened the car door, end quote, and very slowly got out and walked toward the front of the car.|The man appeared to step back as the policeman, quote, calmly opened the car door, end quote, and very slowly got out and walked toward the front of the car. +LJ037-0042|The man pulled a gun. Mrs. Markham heard three shots and saw the policeman fall to the ground near the left front wheel.|The man pulled a gun. Mrs. Markham heard three shots and saw the policeman fall to the ground near the left front wheel. +LJ037-0043|She raised her hands to her eyes as the man started to walk back toward Patton.|She raised her hands to her eyes as the man started to walk back toward Patton. +LJ037-0044|She peered through her fingers, lowered her hands, and saw the man doing something with his gun. Quote,|She peered through her fingers, lowered her hands, and saw the man doing something with his gun. Quote, +LJ037-0045|He was just fooling with it. I didn't know what he was doing. I was afraid he was fixing to kill me. End quote.|He was just fooling with it. I didn't know what he was doing. I was afraid he was fixing to kill me. End quote. +LJ037-0046|The man, quote, in kind of a little trot, end quote, headed down Patton toward Jefferson Boulevard, a block away.|The man, quote, in kind of a little trot, end quote, headed down Patton toward Jefferson Boulevard, a block away. +LJ037-0047|Mrs. Markham then ran to Officer Tippit's side and saw him lying in a pool of blood.|Mrs. Markham then ran to Officer Tippit's side and saw him lying in a pool of blood. +LJ037-0048|Helen Markham was screaming as she leaned over the body.|Helen Markham was screaming as she leaned over the body. +LJ037-0049|A few minutes later she described the gunman to a policeman.|A few minutes later she described the gunman to a policeman. +LJ037-0050|Her description and that of other eyewitnesses led to the police broadcast at 1:22 p.m.|Her description and that of other eyewitnesses led to the police broadcast at one:twenty-two p.m. +LJ037-0051|describing the slayer as, quote, about 30, 5 foot 8 inches, black hair, slender, end quote.|describing the slayer as, quote, about thirty, five foot eight inches, black hair, slender, end quote. +LJ037-0052|At about 4:30 p.m., Mrs. Markham,|At about four:thirty p.m., Mrs. Markham, +LJ037-0053|who had been greatly upset by her experience, was able to view a lineup of four men handcuffed together at the police station.|who had been greatly upset by her experience, was able to view a lineup of four men handcuffed together at the police station. +LJ037-0054|She identified Lee Harvey Oswald as the man who shot the policeman.|She identified Lee Harvey Oswald as the man who shot the policeman. +LJ037-0055|Detective L. C. Graves, who had been with Mrs. Markham before the lineup|Detective L. C. Graves, who had been with Mrs. Markham before the lineup +LJ037-0056|testified that she was, quote, quite hysterical, end quote, and was, quote, crying and upset, end quote.|testified that she was, quote, quite hysterical, end quote, and was, quote, crying and upset, end quote. +LJ037-0057|He said that Mrs. Markham started crying when Oswald walked into the lineup room.|He said that Mrs. Markham started crying when Oswald walked into the lineup room. +LJ037-0058|In testimony before the Commission, Mrs. Markham confirmed her positive identification of Lee Harvey Oswald|In testimony before the Commission, Mrs. Markham confirmed her positive identification of Lee Harvey Oswald +LJ037-0059|as the man she saw kill Officer Tippit. In evaluating Mrs. Markham's identification of Oswald, the Commission considered certain allegations|as the man she saw kill Officer Tippit. In evaluating Mrs. Markham's identification of Oswald, the Commission considered certain allegations +LJ037-0060|That Mrs. Markham described the man who killed Patrolman Tippit as, quote, short, a little on the heavy side, end quote.|That Mrs. Markham described the man who killed Patrolman Tippit as, quote, short, a little on the heavy side, end quote. +LJ037-0061|and having, quote, somewhat bushy, end quote, hair.|and having, quote, somewhat bushy, end quote, hair. +LJ037-0062|The Commission reviewed the transcript of a phone conversation in which Mrs. Markham is alleged to have provided such a description.|The Commission reviewed the transcript of a phone conversation in which Mrs. Markham is alleged to have provided such a description. +LJ037-0063|A review of the complete transcript has satisfied the Commission|A review of the complete transcript has satisfied the Commission +LJ037-0064|that Mrs. Markham strongly reaffirmed her positive identification of Oswald and denied having described the killer|that Mrs. Markham strongly reaffirmed her positive identification of Oswald and denied having described the killer +LJ037-0065|as short, stocky and having bushy hair. She stated that the man weighed about 150 pounds.|as short, stocky and having bushy hair. She stated that the man weighed about one hundred fifty pounds. +LJ037-0066|Although she used the words, quote, a little bit bushy, end quote, to describe the gunman's hair,|Although she used the words, quote, a little bit bushy, end quote, to describe the gunman's hair, +LJ037-0067|the transcript establishes that she was referring to the uncombed state of his hair, a description fully supported by a photograph of Oswald|the transcript establishes that she was referring to the uncombed state of his hair, a description fully supported by a photograph of Oswald +LJ037-0068|taken at the time of his arrest.|taken at the time of his arrest. +LJ037-0069|Although in the phone conversation she described the man as, quote, short, end quote, on November 22nd,|Although in the phone conversation she described the man as, quote, short, end quote, on November twenty-second, +LJ037-0070|within minutes of the shooting and before the lineup, Mrs. Markham described the man to the police as 5 foot 8 inches tall.|within minutes of the shooting and before the lineup, Mrs. Markham described the man to the police as five foot eight inches tall. +LJ037-0071|During her testimony Mrs. Markham initially denied that she ever had the above phone conversation.|During her testimony Mrs. Markham initially denied that she ever had the above phone conversation. +LJ037-0072|She has subsequently admitted the existence of the conversation and offered an explanation for her denial.|She has subsequently admitted the existence of the conversation and offered an explanation for her denial. +LJ037-0073|Addressing itself solely to the probative value of Mrs. Markham's contemporaneous description of the gunman|Addressing itself solely to the probative value of Mrs. Markham's contemporaneous description of the gunman +LJ037-0074|and her positive identification of Oswald at a police lineup, the Commission considers her testimony reliable.|and her positive identification of Oswald at a police lineup, the Commission considers her testimony reliable. +LJ037-0075|However, even in the absence of Mrs. Markham's testimony, there is ample evidence to identify Oswald as the killer of Tippit.|However, even in the absence of Mrs. Markham's testimony, there is ample evidence to identify Oswald as the killer of Tippit. +LJ037-0076|Two young women,|Two young women, +LJ037-0077|Barbara Jeanette Davis and Virginia Davis, were in an apartment of a multiple-unit house on the southeast corner of 10th and Patton|Barbara Jeanette Davis and Virginia Davis, were in an apartment of a multiple-unit house on the southeast corner of tenth and Patton +LJ037-0078|when they heard the sound of gunfire and the screams of Helen Markham.|when they heard the sound of gunfire and the screams of Helen Markham. +LJ037-0079|They ran to the door in time to see a man with a revolver cut across their lawn and disappear around a corner of the house onto Patton.|They ran to the door in time to see a man with a revolver cut across their lawn and disappear around a corner of the house onto Patton. +LJ037-0080|Barbara Jeanette Davis assumed that he was emptying his gun as, quote, he had it open and was shaking it, end quote. She immediately called the police.|Barbara Jeanette Davis assumed that he was emptying his gun as, quote, he had it open and was shaking it, end quote. She immediately called the police. +LJ037-0081|Later in the day each woman found an empty shell on the ground near the house. These two shells were delivered to the police.|Later in the day each woman found an empty shell on the ground near the house. These two shells were delivered to the police. +LJ037-0082|On the evening of November 22,|On the evening of November twenty-two, +LJ037-0083|Barbara Jeanette and Virginia Davis viewed a group of four men in a lineup and each one picked Oswald as the man who crossed their lawn while emptying his pistol.|Barbara Jeanette and Virginia Davis viewed a group of four men in a lineup and each one picked Oswald as the man who crossed their lawn while emptying his pistol. +LJ037-0084|Barbara Jeanette Davis testified that no one had shown her a picture of Oswald before the identification and that she had not seen him on television.|Barbara Jeanette Davis testified that no one had shown her a picture of Oswald before the identification and that she had not seen him on television. +LJ037-0085|She was not sure whether she had seen his picture in a newspaper on the afternoon or evening of November 22 prior to the lineup.|She was not sure whether she had seen his picture in a newspaper on the afternoon or evening of November twenty-two prior to the lineup. +LJ037-0086|Her reaction when she saw Oswald in the lineup was that, quote,|Her reaction when she saw Oswald in the lineup was that, quote, +LJ037-0087|I was pretty sure it was the same man I saw. When they made him turn sideways, I was positive that was the one I seen, end quote.|I was pretty sure it was the same man I saw. When they made him turn sideways, I was positive that was the one I seen, end quote. +LJ037-0088|Similarly, Virginia Davis had not been shown pictures of anyone prior to the lineup and had not seen either television or the newspapers during the afternoon.|Similarly, Virginia Davis had not been shown pictures of anyone prior to the lineup and had not seen either television or the newspapers during the afternoon. +LJ037-0089|She identified Oswald, who was the No. 2 man in the lineup, as the man she saw running with the gun:|She identified Oswald, who was the Number two man in the lineup, as the man she saw running with the gun: +LJ037-0090|she testified, quote, I would say that was him for sure, end quote.|she testified, quote, I would say that was him for sure, end quote. +LJ037-0091|Barbara Jeanette Davis and Virginia Davis were sitting alongside each other when they made their positive identifications of Oswald.|Barbara Jeanette Davis and Virginia Davis were sitting alongside each other when they made their positive identifications of Oswald. +LJ037-0092|Each woman whispered Oswald's number to the detective. Each testified that she was the first to make the identification.|Each woman whispered Oswald's number to the detective. Each testified that she was the first to make the identification. +LJ037-0094|He looked west on 10th and saw a man running to the west and a policeman falling to the ground.|He looked west on tenth and saw a man running to the west and a policeman falling to the ground. +LJ037-0095|Smith failed to make himself known to the police on November 22.|Smith failed to make himself known to the police on November twenty-two. +LJ037-0096|Several days later he reported what he had seen and was questioned by FBI agents.|Several days later he reported what he had seen and was questioned by FBI agents. +LJ037-0097|Smith subsequently told a Commission staff member|Smith subsequently told a Commission staff member +LJ037-0098|that he saw Oswald on television the night of the murder and thought that Oswald was the man he had seen running away from the shooting.|that he saw Oswald on television the night of the murder and thought that Oswald was the man he had seen running away from the shooting. +LJ037-0099|On television Oswald's hair looked blond, whereas Smith remembered that the man who ran away had hair that was brown or brownish black.|On television Oswald's hair looked blond, whereas Smith remembered that the man who ran away had hair that was brown or brownish black. +LJ037-0100|Later, the FBI showed Smith a picture of Oswald. In the picture the hair was brown.|Later, the FBI showed Smith a picture of Oswald. In the picture the hair was brown. +LJ037-0101|According to his testimony, Smith told the FBI, quote, It looked more like him than it did on television, end quote,|According to his testimony, Smith told the FBI, quote, It looked more like him than it did on television, end quote, +LJ037-0102|He stated further that from, quote, What I saw of him, end quote, the man looked like the man in the picture.|He stated further that from, quote, What I saw of him, end quote, the man looked like the man in the picture. +LJ037-0103|Two other important eyewitnesses to Oswald's flight were Ted Callaway,|Two other important eyewitnesses to Oswald's flight were Ted Callaway, +LJ037-0104|manager of a used-car lot on the northeast corner of Patton Avenue and Jefferson Boulevard, and Sam Guinyard, a porter at the lot.|manager of a used-car lot on the northeast corner of Patton Avenue and Jefferson Boulevard, and Sam Guinyard, a porter at the lot. +LJ037-0105|They heard the sound of shots to the north of their lot. Callaway heard five shots, and Guinyard three.|They heard the sound of shots to the north of their lot. Callaway heard five shots, and Guinyard three. +LJ037-0106|Both ran to the sidewalk on the east side of Patton at a point about a half a block south of 10th.|Both ran to the sidewalk on the east side of Patton at a point about a half a block south of tenth. +LJ037-0107|They saw a man coming south on Patton with a revolver held high in his right hand. According to Callaway, the man crossed to the west side of Patton.|They saw a man coming south on Patton with a revolver held high in his right hand. According to Callaway, the man crossed to the west side of Patton. +LJ037-0108|From across the street Callaway yelled, quote, Hey, man, what the hell is going on? End quote.|From across the street Callaway yelled, quote, Hey, man, what the hell is going on? End quote. +LJ037-0109|He slowed down, halted, said something, and then kept on going to the corner, turned right, and continued west on Jefferson.|He slowed down, halted, said something, and then kept on going to the corner, turned right, and continued west on Jefferson. +LJ037-0110|Guinyard claimed that the man ran down the east side of Patton and passed within 10 feet of him before crossing to the other side.|Guinyard claimed that the man ran down the east side of Patton and passed within ten feet of him before crossing to the other side. +LJ037-0111|Guinyard and Callaway ran to 10th and Patton and found Tippit lying in the street beside his car.|Guinyard and Callaway ran to tenth and Patton and found Tippit lying in the street beside his car. +LJ037-0112|Apparently he had reached for his gun; it lay beneath him outside of the holster.|Apparently he had reached for his gun; it lay beneath him outside of the holster. +LJ037-0113|Callaway picked up the gun.|Callaway picked up the gun. +LJ037-0114|He and Scoggins attempted to chase down the gunman in Scoggin's taxicab, but he had disappeared.|He and Scoggins attempted to chase down the gunman in Scoggin's taxicab, but he had disappeared. +LJ037-0115|Early in the evening of November 22,|Early in the evening of November twenty-two, +LJ037-0116|Guinyard and Callaway viewed the same lineup of four men from which Mrs. Markham had earlier made her identification of Lee Harvey Oswald.|Guinyard and Callaway viewed the same lineup of four men from which Mrs. Markham had earlier made her identification of Lee Harvey Oswald. +LJ037-0117|Both men picked Oswald as the man who had run south on Patton with a gun in his hand.|Both men picked Oswald as the man who had run south on Patton with a gun in his hand. +LJ037-0118|Callaway told the Commission, quote, So they brought four men in.|Callaway told the Commission, quote, So they brought four men in. +LJ037-0119|I stepped to the back of the room, so I could kind of see him from the same distance which I had seen him before. And when he came out I knew him, end quote.|I stepped to the back of the room, so I could kind of see him from the same distance which I had seen him before. And when he came out I knew him, end quote. +LJ037-0120|Guinyard said, quote, I told them that was him right there. I pointed him out right there, end quote.|Guinyard said, quote, I told them that was him right there. I pointed him out right there, end quote. +LJ037-0121|Both Callaway and Guinyard testified that they had not been shown any pictures by the police before the lineup.|Both Callaway and Guinyard testified that they had not been shown any pictures by the police before the lineup. +LJ037-0122|The Dallas Police Department furnished the Commission with pictures of the men who appeared in the lineups with Oswald,|The Dallas Police Department furnished the Commission with pictures of the men who appeared in the lineups with Oswald, +LJ037-0123|and the Commission has inquired into general lineup procedures used by the Dallas police as well as the specific procedures in the lineups involving Oswald.|and the Commission has inquired into general lineup procedures used by the Dallas police as well as the specific procedures in the lineups involving Oswald. +LJ037-0124|The Commission is satisfied that the lineups were conducted fairly.|The Commission is satisfied that the lineups were conducted fairly. +LJ037-0125|As Oswald ran south on Patton Avenue toward Jefferson Boulevard he was moving in the direction of a used-car lot|As Oswald ran south on Patton Avenue toward Jefferson Boulevard he was moving in the direction of a used-car lot +LJ037-0126|located on the southeast corner of this intersection.|located on the southeast corner of this intersection. +LJ037-0127|Four men -- Warren Reynolds, Harold Russell, Pat Patterson, and L. J. Lewis|Four men -- Warren Reynolds, Harold Russell, Pat Patterson, and L. J. Lewis +LJ037-0128|were on the lot at the time, and they saw a white male with a revolver in his hands running south on Patton.|were on the lot at the time, and they saw a white male with a revolver in his hands running south on Patton. +LJ037-0129|When the man reached Jefferson, he turned right and headed west.|When the man reached Jefferson, he turned right and headed west. +LJ037-0130|Reynolds and Patterson decided to follow him.|Reynolds and Patterson decided to follow him. +LJ037-0131|When he reached a gasoline service station one block away he turned north and walked toward a parking area in the rear of the station.|When he reached a gasoline service station one block away he turned north and walked toward a parking area in the rear of the station. +LJ037-0132|Neither Reynolds nor Patterson saw the man after he turned off Jefferson at the service station.|Neither Reynolds nor Patterson saw the man after he turned off Jefferson at the service station. +LJ037-0133|These four witnesses were interviewed by FBI agents 2 months after the shooting.|These four witnesses were interviewed by FBI agents two months after the shooting. +LJ037-0134|Russell and Patterson were shown a picture of Oswald and they stated that Oswald was the man they saw on November 22, 1963.|Russell and Patterson were shown a picture of Oswald and they stated that Oswald was the man they saw on November twenty-two, nineteen sixty-three. +LJ037-0135|Russell confirmed this statement in a sworn affidavit for the Commission.|Russell confirmed this statement in a sworn affidavit for the Commission. +LJ037-0136|Patterson, when asked later to confirm his identification by affidavit said he did not recall having been shown the photograph.|Patterson, when asked later to confirm his identification by affidavit said he did not recall having been shown the photograph. +LJ037-0137|He was then shown two photographs of Oswald and he advised that Oswald was, quote, unquestionably, end quote, the man he saw.|He was then shown two photographs of Oswald and he advised that Oswald was, quote, unquestionably, end quote, the man he saw. +LJ037-0138|Reynolds did not make a positive identification when interviewed by the FBI, but|Reynolds did not make a positive identification when interviewed by the FBI, but +LJ037-0139|he subsequently testified before a Commission staff member and, when shown two photographs of Oswald, stated that they were photographs of the man he saw.|he subsequently testified before a Commission staff member and, when shown two photographs of Oswald, stated that they were photographs of the man he saw. +LJ037-0140|L.J. Lewis said in an interview that because of the distance from which he observed the gunman he would hesitate to state whether the man was identical with Oswald.|L.J. Lewis said in an interview that because of the distance from which he observed the gunman he would hesitate to state whether the man was identical with Oswald. +LJ037-0141|When Oswald was arrested, he had in his possession a Smith & Wesson 38 Special caliber revolver,|When Oswald was arrested, he had in his possession a Smith and Wesson thirty-eight Special caliber revolver, +LJ037-0142|serial number V510210.|serial number V five one zero two one zero. +LJ037-0143|Two of the arresting officers placed their initials on the weapon and a third inscribed his name.|Two of the arresting officers placed their initials on the weapon and a third inscribed his name. +LJ037-0144|All three identified Exhibit No. 143 as the revolver taken from Oswald when he was arrested.|All three identified Exhibit Number one forty-three as the revolver taken from Oswald when he was arrested. +LJ037-0145|Four cartridge cases were found in the shrubbery on the corner of 10th and Patton by three of the eyewitnesses -- Domingo Benavides,|Four cartridge cases were found in the shrubbery on the corner of tenth and Patton by three of the eyewitnesses -- Domingo Benavides, +LJ037-0146|Barbara Jeanette Davis, and Virginia Davis.|Barbara Jeanette Davis, and Virginia Davis. +LJ037-0147|It was the unanimous and unequivocal testimony of expert witnesses before the Commission that these used cartridge cases were fired from the revolver|It was the unanimous and unequivocal testimony of expert witnesses before the Commission that these used cartridge cases were fired from the revolver +LJ037-0148|in Oswald's possession to the exclusion of all other weapons.|in Oswald's possession to the exclusion of all other weapons. +LJ037-0149|Cortlandt Cunningham, of the Firearms Identification Unit of the FBI Laboratory, testified|Cortlandt Cunningham, of the Firearms Identification Unit of the FBI Laboratory, testified +LJ037-0150|that he compared the four empty cartridge cases found near the scene of the shooting with a test cartridge fired from the weapon in Oswald's possession when he was arrested.|that he compared the four empty cartridge cases found near the scene of the shooting with a test cartridge fired from the weapon in Oswald's possession when he was arrested. +LJ037-0151|Cunningham declared that this weapon fired the four cartridges to the exclusion of all other weapons.|Cunningham declared that this weapon fired the four cartridges to the exclusion of all other weapons. +LJ037-0152|Identification was effected through breech face marks and firing pin marks.|Identification was effected through breech face marks and firing pin marks. +LJ037-0153|Robert A. Frazier and Charles Killion, other FBI firearms experts,|Robert A. Frazier and Charles Killion, other FBI firearms experts, +LJ037-0154|independently examined the four cartridge cases and arrived at the same conclusion as Cunningham.|independently examined the four cartridge cases and arrived at the same conclusion as Cunningham. +LJ037-0155|At the request of the Commission, Joseph D. Nicol, superintendent of the Illinois Bureau of Criminal Identification Investigation,|At the request of the Commission, Joseph D. Nicol, superintendent of the Illinois Bureau of Criminal Identification Investigation, +LJ037-0156|also examined the four cartridge cases found near the site of the homicide and compared them with the test cartridge cases fired from the Smith & Wesson revolver|also examined the four cartridge cases found near the site of the homicide and compared them with the test cartridge cases fired from the Smith and Wesson revolver +LJ037-0158|He concluded that all of these cartridges were fired from the same weapon.|He concluded that all of these cartridges were fired from the same weapon. +LJ037-0159|Cunningham compared four lead bullets recovered from the body of Patrolman Tippit with test bullets fired from Oswald's revolver.|Cunningham compared four lead bullets recovered from the body of Patrolman Tippit with test bullets fired from Oswald's revolver. +LJ037-0160|He explained that the bullets were slightly smaller than the barrel of the pistol which had fired them. This caused the bullets to have an erratic passage through the barrel|He explained that the bullets were slightly smaller than the barrel of the pistol which had fired them. This caused the bullets to have an erratic passage through the barrel +LJ037-0161|and impressed upon the lead of the bullets inconsistent individual characteristics which made identification impossible.|and impressed upon the lead of the bullets inconsistent individual characteristics which made identification impossible. +LJ037-0162|Consecutive bullets fired from the revolver by the FBI experts could not be identified as having been fired from that revolver.|Consecutive bullets fired from the revolver by the FBI experts could not be identified as having been fired from that revolver. +LJ037-0163|Cunningham testified that all of the bullets were mutilated, one being useless for comparison purposes.|Cunningham testified that all of the bullets were mutilated, one being useless for comparison purposes. +LJ037-0164|All four bullets were fired from a weapon|All four bullets were fired from a weapon +LJ037-0165|with five lands and grooves and a right twist which were the rifling characteristics of the revolver taken from Oswald.|with five lands and grooves and a right twist which were the rifling characteristics of the revolver taken from Oswald. +LJ037-0166|He concluded, however, that he could not say whether the four bullets were fired from the revolver in Oswald's possession.|He concluded, however, that he could not say whether the four bullets were fired from the revolver in Oswald's possession. +LJ037-0167|Quote: The only thing I can testify is they could have on the basis of the rifling characteristics -- they could have been, end quote,|Quote: The only thing I can testify is they could have on the basis of the rifling characteristics -- they could have been, end quote, +LJ037-0168|Nicol differed with the FBI experts on one bullet taken from Tippit's body.|Nicol differed with the FBI experts on one bullet taken from Tippit's body. +LJ037-0169|He declared that this bullet was fired from the same weapon that fired the test bullets to the exclusion of all other weapons.|He declared that this bullet was fired from the same weapon that fired the test bullets to the exclusion of all other weapons. +LJ037-0170|But he agreed that because the other three bullets were mutilated, he could not determine if they had been fired from the same weapon as the test bullets.|But he agreed that because the other three bullets were mutilated, he could not determine if they had been fired from the same weapon as the test bullets. +LJ037-0171|The examination and testimony of the experts enabled the Commission to conclude that five shots may have been fired,|The examination and testimony of the experts enabled the Commission to conclude that five shots may have been fired, +LJ037-0172|even though only four bullets were recovered.|even though only four bullets were recovered. +LJ037-0173|Three of the bullets recovered from Tippit's body were manufactured by Winchester-Western, and the fourth bullet by Remington-Peters,|Three of the bullets recovered from Tippit's body were manufactured by Winchester-Western, and the fourth bullet by Remington-Peters, +LJ037-0174|but only two of the four discarded cartridge cases found on the lawn at 10th Street and Patton Avenue were of Winchester-Western manufacture.|but only two of the four discarded cartridge cases found on the lawn at tenth Street and Patton Avenue were of Winchester-Western manufacture. +LJ037-0175|Therefore, one cartridge case of this type was not recovered.|Therefore, one cartridge case of this type was not recovered. +LJ037-0176|And though only one bullet of Remington-Peters manufacture was recovered, two empty cartridge cases of that make were retrieved.|And though only one bullet of Remington-Peters manufacture was recovered, two empty cartridge cases of that make were retrieved. +LJ037-0177|Therefore, either one bullet of Remington-Peters manufacture is missing|Therefore, either one bullet of Remington-Peters manufacture is missing +LJ037-0179|was discarded along with the others as Oswald left the scene.|was discarded along with the others as Oswald left the scene. +LJ037-0180|If a bullet is missing, five were fired. This corresponds with the observation and memory of Ted Callaway, and possibly Warren Reynolds,|If a bullet is missing, five were fired. This corresponds with the observation and memory of Ted Callaway, and possibly Warren Reynolds, +LJ037-0181|but not with the other eyewitnesses who claim to have heard from two to four shots.|but not with the other eyewitnesses who claim to have heard from two to four shots. +LJ037-0182|Ownership of Revolver|Ownership of Revolver +LJ037-0183|By checking certain importers and dealers after the assassination of President Kennedy and slaying of Officer Tippit,|By checking certain importers and dealers after the assassination of President Kennedy and slaying of Officer Tippit, +LJ037-0184|agents of the FBI determined that George Rose & Co. of Los Angeles was a major distributor of this type of revolver.|agents of the FBI determined that George Rose and Co. of Los Angeles was a major distributor of this type of revolver. +LJ037-0185|Records of Seaport Traders, Incorporated, a mail-order division of George Rose & Co.,|Records of Seaport Traders, Incorporated, a mail-order division of George Rose and Co., +LJ037-0186|disclosed that on January 3, 1963,|disclosed that on January three, nineteen sixty-three, +LJ037-0187|the company received from Empire Wholesale Sporting Goods, Ltd., Montreal, a shipment of 99 guns in one case.|the company received from Empire Wholesale Sporting Goods, Ltd., Montreal, a shipment of ninety-nine guns in one case. +LJ037-0188|Among these guns was a .38 Special caliber Smith & Wesson revolver, serial No. V510210,|Among these guns was a thirty-eight Special caliber Smith and Wesson revolver, serial Number V five one zero two one zero, +LJ037-0189|the only revolver made by Smith & Wesson with this serial number. When first manufactured, it had a 5-inch barrel.|the only revolver made by Smith and Wesson with this serial number. When first manufactured, it had a five-inch barrel. +LJ037-0190|George Rose & Co. had the barrel shortened by a gunsmith to two and one quarter inches.|George Rose and Co. had the barrel shortened by a gunsmith to two and one quarter inches. +LJ037-0191|Sometime after January 27, 1963,|Sometime after January twenty-seven, nineteen sixty-three, +LJ037-0192|Seaport Traders, Incorporated, received through the mail a mail-order coupon for one, quote, point three-eight|Seaport Traders, Incorporated, received through the mail a mail-order coupon for one, quote, point three-eight +LJ037-0193|S T. W. 2 inch BBL, unquote, cost $29.95.|S T. W. two inch BBL, unquote, cost twenty-nine dollars, ninety-five cents. +LJ037-0194|Ten dollars in cash was enclosed.|Ten dollars in cash was enclosed. +LJ037-0195|The order was signed in ink by, quote,|The order was signed in ink by, quote, +LJ037-0196|A. J. Hidell, aged 28, end quote. The date of the order was January 27. No year shown.|A. J. Hidell, aged twenty-eight, end quote. The date of the order was January twenty-seven. No year shown. +LJ037-0197|and the return address was Post Office Box 2915, Dallas, Texas.|and the return address was Post Office Box two nine one five, Dallas, Texas. +LJ037-0198|Also on the order form was an order, written in ink, for one box of ammunition and one holster, but a line was drawn through these items.|Also on the order form was an order, written in ink, for one box of ammunition and one holster, but a line was drawn through these items. +LJ037-0199|The mail-order form had a line for the name of a witness to attest that the person ordering the gun was a U.S. citizen and had not been convicted of a felony.|The mail-order form had a line for the name of a witness to attest that the person ordering the gun was a U.S. citizen and had not been convicted of a felony. +LJ037-0200|The name written in this space was D. F. Drittal.|The name written in this space was D. F. Drittal. +LJ037-0201|Heinz W. Michaelis, office manager of both George Rose & Co., Incorporated and Seaport Traders, Incorporated.|Heinz W. Michaelis, office manager of both George Rose and Co., Incorporated and Seaport Traders, Incorporated. +LJ037-0202|identified records of Seaport Traders, Incorporated, which showed that a, quote, point three eight|identified records of Seaport Traders, Incorporated, which showed that a, quote, point three eight +LJ037-0203|S and W Special two-inch Commando, serial number V510210, end quote,|S and W Special two-inch Commando, serial number V five one zero two one zero, end quote, +LJ037-0204|was shipped on March 20, 1963, to A. J. Hidell, Post Office Box 2915, Dallas, Texas.|was shipped on March twenty, nineteen sixty-three, to A. J. Hidell, Post Office Box two nine one five, Dallas, Texas. +LJ037-0205|The invoice was prepared on March 13, 1963; the revolver was actually shipped on March 20 by Railway Express.|The invoice was prepared on March thirteen, nineteen sixty-three; the revolver was actually shipped on March twenty by Railway Express. +LJ037-0206|The balance due on the purchase was $19.95.|The balance due on the purchase was nineteen dollars, ninety-five cents. +LJ037-0207|Michaelis furnished the shipping copy of the invoice, and the Railway Express Agency shipping documents, showing that|Michaelis furnished the shipping copy of the invoice, and the Railway Express Agency shipping documents, showing that +LJ037-0208|$19.95, plus $1.27 shipping charge, had been collected from the consignee, Hidell.|nineteen dollars, ninety-five cents, plus one dollar, twenty-seven cents shipping charge, had been collected from the consignee, Hidell. +LJ037-0209|Handwriting experts, Alwyn Cole of the Treasury Department and James C. Cadigan of the FBI,|Handwriting experts, Alwyn Cole of the Treasury Department and James C. Cadigan of the FBI, +LJ037-0210|testified before the Commission that the writing on the coupon was Oswald's.|testified before the Commission that the writing on the coupon was Oswald's. +LJ037-0211|The signature of the witness, D. F. Drittal,|The signature of the witness, D. F. Drittal, +LJ037-0212|who attested that the fictitious Hidell was an American citizen and had not been convicted of a felony, was also in Oswald's handwriting.|who attested that the fictitious Hidell was an American citizen and had not been convicted of a felony, was also in Oswald's handwriting. +LJ037-0213|Marina Oswald gave as her opinion that the mail-order coupon was in Oswald's handwriting.|Marina Oswald gave as her opinion that the mail-order coupon was in Oswald's handwriting. +LJ037-0214|When shown the revolver, she stated that she recognized it as the one owned by her husband.|When shown the revolver, she stated that she recognized it as the one owned by her husband. +LJ037-0215|She also testified that this appeared to be the revolver seen in Oswald's belt|She also testified that this appeared to be the revolver seen in Oswald's belt +LJ037-0216|in the picture she took in late March or early April 1963 when the family was living on Neely Street in Dallas.|in the picture she took in late March or early April nineteen sixty-three when the family was living on Neely Street in Dallas. +LJ037-0217|Police found an empty revolver holster when they searched Oswald's room on Beckley Avenue after his arrest.|Police found an empty revolver holster when they searched Oswald's room on Beckley Avenue after his arrest. +LJ037-0218|Marina Oswald testified that this was the holster which contained the revolver in the photographs taken on Neely Street.|Marina Oswald testified that this was the holster which contained the revolver in the photographs taken on Neely Street. +LJ037-0220|Approximately 15 minutes before the shooting of Tippit, Oswald was seen leaving his roominghouse.|Approximately fifteen minutes before the shooting of Tippit, Oswald was seen leaving his roominghouse. +LJ037-0221|He was wearing a zipper jacket which he had not been wearing moments before when he had arrived home.|He was wearing a zipper jacket which he had not been wearing moments before when he had arrived home. +LJ037-0223|Shortly after Tippit was slain, policemen found a light-colored zipper jacket along the route taken by the killer as he attempted to escape.|Shortly after Tippit was slain, policemen found a light-colored zipper jacket along the route taken by the killer as he attempted to escape. +LJ037-0224|At 1:22 p.m. the Dallas police radio described the man wanted for the murder of Tippit as, quote, a white male about thirty,|At one:twenty-two p.m. the Dallas police radio described the man wanted for the murder of Tippit as, quote, a white male about thirty, +LJ037-0225|five foot eight inches, black hair, slender, wearing a white jacket, white shirt and dark slacks, end quote,|five foot eight inches, black hair, slender, wearing a white jacket, white shirt and dark slacks, end quote, +LJ037-0226|According to Patrolman Poe this description came from Mrs. Markham and Mrs. Barbara Jeanette Davis.|According to Patrolman Poe this description came from Mrs. Markham and Mrs. Barbara Jeanette Davis. +LJ037-0227|Mrs. Markham told Poe|Mrs. Markham told Poe +LJ037-0228|that the man was a, quote, white male, about 25, about five feet eight, brown hair, medium, end quote, and wearing a, quote,|that the man was a, quote, white male, about twenty-five, about five feet eight, brown hair, medium, end quote, and wearing a, quote, +LJ037-0229|white jacket, end quote,|white jacket, end quote, +LJ037-0230|Mrs. Davis gave Poe the same general description: a, quote, white male in his early twenties, around five foot seven inches|Mrs. Davis gave Poe the same general description: a, quote, white male in his early twenties, around five foot seven inches +LJ037-0231|or eight inches, about 145 pounds, end quote, and wearing a white jacket.|or eight inches, about one hundred forty-five pounds, end quote, and wearing a white jacket. +LJ037-0232|As has been discussed previously,|As has been discussed previously, +LJ037-0233|two witnesses, Warren Reynolds and B. M. Patterson, saw the gunman run toward the rear of a gasoline service station on Jefferson Boulevard.|two witnesses, Warren Reynolds and B. M. Patterson, saw the gunman run toward the rear of a gasoline service station on Jefferson Boulevard. +LJ037-0234|Mrs. Mary Brock, the wife of a mechanic who worked at the station, was there at the time and she saw a white male,|Mrs. Mary Brock, the wife of a mechanic who worked at the station, was there at the time and she saw a white male, +LJ037-0235|5 feet, 10 inches, wearing light clothing, a light-colored jacket" walk past her at a fast pace with his hands in his pocket.|five feet, ten inches, wearing light clothing, a light-colored jacket" walk past her at a fast pace with his hands in his pocket. +LJ037-0236|She last saw him in the parking lot directly behind the service station.|She last saw him in the parking lot directly behind the service station. +LJ037-0237|When interviewed by FBI agents on January 21, 1964,|When interviewed by FBI agents on January twenty-one, nineteen sixty-four, +LJ037-0238|she identified a picture of Oswald as being the same person she saw on November 22.|she identified a picture of Oswald as being the same person she saw on November twenty-two. +LJ037-0239|She confirmed this interview by a sworn affidavit.|She confirmed this interview by a sworn affidavit. +LJ037-0240|At 1:24 p.m., the police radio reported, quote,|At one:twenty-four p.m., the police radio reported, quote, +LJ037-0241|The suspect last seen running west on Jefferson from 400 East Jefferson.|The suspect last seen running west on Jefferson from four hundred East Jefferson. +LJ037-0242|Police Capt. W. R. Westbrook and several other officers concentrated their search along Jefferson Boulevard.|Police Capt. W. R. Westbrook and several other officers concentrated their search along Jefferson Boulevard. +LJ037-0243|Westbrook walked through the parking lot behind the service station and found a light-colored jacket lying under the rear of one of the cars.|Westbrook walked through the parking lot behind the service station and found a light-colored jacket lying under the rear of one of the cars. +LJ037-0244|Westbrook identified Commission Exhibit No. 162 as the light-colored jacket which he discovered underneath the automobile.|Westbrook identified Commission Exhibit Number one sixty-two as the light-colored jacket which he discovered underneath the automobile. +LJ037-0245|This jacket belonged to Lee Harvey Oswald. Marina Oswald stated that her husband owned only two jackets, one blue and the other gray.|This jacket belonged to Lee Harvey Oswald. Marina Oswald stated that her husband owned only two jackets, one blue and the other gray. +LJ037-0246|The blue jacket was found in the Texas School Book Depository and was identified by Marina Oswald as her husband's.|The blue jacket was found in the Texas School Book Depository and was identified by Marina Oswald as her husband's. +LJ037-0247|Marina Oswald also identified Commission Exhibit No. 162, the jacket found by Captain Westbrook, as her husband's second jacket.|Marina Oswald also identified Commission Exhibit Number one sixty-two, the jacket found by Captain Westbrook, as her husband's second jacket. +LJ037-0248|The eyewitnesses vary in their identification of the jacket.|The eyewitnesses vary in their identification of the jacket. +LJ037-0249|Mrs. Earlene Roberts, the housekeeper at Oswald's roominghouse and the last person known to have seen him before he reached 10th Street and Patton Avenue,|Mrs. Earlene Roberts, the housekeeper at Oswald's roominghouse and the last person known to have seen him before he reached tenth Street and Patton Avenue, +LJ037-0250|said that she may have seen the gray zipper jacket but she was not certain.|said that she may have seen the gray zipper jacket but she was not certain. +LJ037-0251|It seemed to her that the jacket Oswald wore was darker than Commission Exhibit No. 162.|It seemed to her that the jacket Oswald wore was darker than Commission Exhibit Number one sixty-two. +LJ037-0252|Ted Callaway, who saw the gunman moments after the shooting, testified that Commission Exhibit No. 162|Ted Callaway, who saw the gunman moments after the shooting, testified that Commission Exhibit Number one sixty-two +LJ037-0253|looked like the jacket he was wearing but, quote, I thought it had a little more tan to it, end quote.|looked like the jacket he was wearing but, quote, I thought it had a little more tan to it, end quote. +LJ037-0254|Two other witnesses, Sam Guinyard and William Arthur Smith,|Two other witnesses, Sam Guinyard and William Arthur Smith, +LJ037-0255|testified that Commission Exhibit No. 162 was the jacket worn by the man they saw on November 22.|testified that Commission Exhibit Number one sixty-two was the jacket worn by the man they saw on November twenty-two. +LJ037-0256|Mrs. Markham and Barbara Davis thought that the jacket worn by the slayer of Tippit was darker than the jacket found by Westbrook.|Mrs. Markham and Barbara Davis thought that the jacket worn by the slayer of Tippit was darker than the jacket found by Westbrook. +LJ037-0257|Scoggins thought it was lighter.|Scoggins thought it was lighter. +LJ037-0258|There is no doubt, however, that Oswald was seen leaving his roominghouse at about 1 p.m. wearing a zipper jacket,|There is no doubt, however, that Oswald was seen leaving his roominghouse at about one p.m. wearing a zipper jacket, +LJ037-0259|that the man who killed Tippit was wearing a light-colored jacket,|that the man who killed Tippit was wearing a light-colored jacket, +LJ037-0260|that he was seen running along Jefferson Boulevard, that a jacket was found under a car in a lot adjoining Jefferson Boulevard|that he was seen running along Jefferson Boulevard, that a jacket was found under a car in a lot adjoining Jefferson Boulevard +LJ037-0261|that the jacket belonged to Lee Harvey Oswald, and that when he was arrested at approximately 1:50 p.m., he was in shirt sleeves.|that the jacket belonged to Lee Harvey Oswald, and that when he was arrested at approximately one:fifty p.m., he was in shirt sleeves. +LJ037-0262|These facts warrant the finding that Lee Harvey Oswald disposed of his jacket as he fled from the scene of the Tippit killing.|These facts warrant the finding that Lee Harvey Oswald disposed of his jacket as he fled from the scene of the Tippit killing. +LJ037-0263|The foregoing evidence establishes that (1) two eyewitnesses who heard the shots and saw the shooting of Dallas Police Patrolman J. D. Tippit|The foregoing evidence establishes that (one) two eyewitnesses who heard the shots and saw the shooting of Dallas Police Patrolman J. D. Tippit +LJ037-0264|and seven eyewitnesses who saw the flight of the gunman with revolver in hand|and seven eyewitnesses who saw the flight of the gunman with revolver in hand +LJ037-0265|positively identified Lee Harvey Oswald as the man they saw fire the shots or flee from the scene,|positively identified Lee Harvey Oswald as the man they saw fire the shots or flee from the scene, +LJ037-0266|(2) the cartridge cases found near the scene of the shooting were fired from the revolver in the possession of Oswald at the time of his arrest,|(two) the cartridge cases found near the scene of the shooting were fired from the revolver in the possession of Oswald at the time of his arrest, +LJ037-0267|to the exclusion of all other weapons,|to the exclusion of all other weapons, +LJ037-0268|(3) the revolver in Oswald's possession at the time of his arrest was purchased by and belonged to Oswald, and (4)|(three) the revolver in Oswald's possession at the time of his arrest was purchased by and belonged to Oswald, and (four) +LJ037-0269|Oswald's jacket was found along the path of flight taken by the gunman as he fled from the scene of the killing.|Oswald's jacket was found along the path of flight taken by the gunman as he fled from the scene of the killing. +LJ038-0001|Report of the President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. +LJ038-0002|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter 4. The Assassin: Part 7.|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter four. The Assassin: Part seven. +LJ038-0003|Oswald's Arrest|Oswald's Arrest +LJ038-0004|The Texas Theatre is on the north side of Jefferson Boulevard, approximately eight blocks from the scene of the Tippit shooting and six blocks|The Texas Theatre is on the north side of Jefferson Boulevard, approximately eight blocks from the scene of the Tippit shooting and six blocks +LJ038-0005|from where several witnesses last saw Oswald running west on Jefferson Boulevard. Shortly after the Tippit murder,|from where several witnesses last saw Oswald running west on Jefferson Boulevard. Shortly after the Tippit murder, +LJ038-0006|police sirens sounded along Jefferson Boulevard.|police sirens sounded along Jefferson Boulevard. +LJ038-0007|One of the persons who heard the sirens was Johnny Calvin Brewer, manager of Hardy's Shoestore, a few doors east of the Texas Theatre.|One of the persons who heard the sirens was Johnny Calvin Brewer, manager of Hardy's Shoestore, a few doors east of the Texas Theatre. +LJ038-0008|Brewer knew from radio broadcasts that the President had been shot and that a patrolman had also been shot in Oak Cliff.|Brewer knew from radio broadcasts that the President had been shot and that a patrolman had also been shot in Oak Cliff. +LJ038-0009|When he heard police sirens, he, quote, looked up and saw the man enter the lobby, end quote.|When he heard police sirens, he, quote, looked up and saw the man enter the lobby, end quote. +LJ038-0010|a recessed area extending about 15 feet between the sidewalk and the front door of his store.|a recessed area extending about fifteen feet between the sidewalk and the front door of his store. +LJ038-0011|A police car made a U-turn, and as the sirens grew fainter,|A police car made a U-turn, and as the sirens grew fainter, +LJ038-0012|the man in the lobby, quote, looked over his shoulder and turned around and walked up West Jefferson towards the theatre, end quote.|the man in the lobby, quote, looked over his shoulder and turned around and walked up West Jefferson towards the theatre, end quote. +LJ038-0013|The man wore a T-shirt beneath his outer shirt and he had no jacket.|The man wore a T-shirt beneath his outer shirt and he had no jacket. +LJ038-0014|Brewer said, quote, He just looked funny to me. His hair was sort of messed up and looked like he had been running, and he looked scared, and he looked funny, end quote.|Brewer said, quote, He just looked funny to me. His hair was sort of messed up and looked like he had been running, and he looked scared, and he looked funny, end quote. +LJ038-0015|Mrs. Julia Postal, selling tickets at the box office of the Texas Theatre,|Mrs. Julia Postal, selling tickets at the box office of the Texas Theatre, +LJ038-0016|heard police sirens and then saw a man as he, quote, ducked into, end quote, the outer lobby space of the theatre near the ticket office.|heard police sirens and then saw a man as he, quote, ducked into, end quote, the outer lobby space of the theatre near the ticket office. +LJ038-0018|Shortly thereafter, Johnny Brewer, who had come from the nearby shoestore, asked Mrs. Postal whether the fellow that had ducked in had bought a ticket.|Shortly thereafter, Johnny Brewer, who had come from the nearby shoestore, asked Mrs. Postal whether the fellow that had ducked in had bought a ticket. +LJ038-0019|She said, quote, No; by golly, he didn't, end quote, and turned around, but the man was nowhere in sight.|She said, quote, No; by golly, he didn't, end quote, and turned around, but the man was nowhere in sight. +LJ038-0020|Brewer told Mrs. Postal that he had seen the man ducking into his place of business and that he had followed him to the theatre.|Brewer told Mrs. Postal that he had seen the man ducking into his place of business and that he had followed him to the theatre. +LJ038-0021|She sent Brewer into the theatre to find the man and check the exits, told him about the assassination, and said, quote,|She sent Brewer into the theatre to find the man and check the exits, told him about the assassination, and said, quote, +LJ038-0022|I don't know if this is the man they want. But he is running from them for some reason, end quote, She then called the police.|I don't know if this is the man they want. But he is running from them for some reason, end quote, She then called the police. +LJ038-0023|At 1:45 p.m., the police radio stated, quote, Have information a suspect just went in the Texas Theatre on West Jefferson, end quote.|At one:forty-five p.m., the police radio stated, quote, Have information a suspect just went in the Texas Theatre on West Jefferson, end quote. +LJ038-0024|Patrol cars bearing at least 15 officers converged on the Texas Theatre.|Patrol cars bearing at least fifteen officers converged on the Texas Theatre. +LJ038-0025|Patrolman M. N. McDonald, with Patrolmen R. Hawkins, T. A. Hutson, and C. T. Walker, entered the theatre from the rear.|Patrolman M. N. McDonald, with Patrolmen R. Hawkins, T. A. Hutson, and C. T. Walker, entered the theatre from the rear. +LJ038-0026|Other policemen entered the front door and searched the balcony.|Other policemen entered the front door and searched the balcony. +LJ038-0027|Detective Paul L. Bentley rushed to the balcony and told the projectionist to turn up the house lights.|Detective Paul L. Bentley rushed to the balcony and told the projectionist to turn up the house lights. +LJ038-0028|Brewer met McDonald and the other policemen at the alley exit door,|Brewer met McDonald and the other policemen at the alley exit door, +LJ038-0029|stepped out onto the stage with them and pointed out the man who had come into the theatre without paying.|stepped out onto the stage with them and pointed out the man who had come into the theatre without paying. +LJ038-0030|The man was Oswald. He was sitting alone in the rear of the main floor of the theatre near the right center aisle.|The man was Oswald. He was sitting alone in the rear of the main floor of the theatre near the right center aisle. +LJ038-0031|About six or seven people were seated on the theatre's main floor and an equal number in the balcony.|About six or seven people were seated on the theatre's main floor and an equal number in the balcony. +LJ038-0032|McDonald first searched two men in the center of the main floor, about 10 rows from the front.|McDonald first searched two men in the center of the main floor, about ten rows from the front. +LJ038-0033|He walked out of the row up the right center aisle.|He walked out of the row up the right center aisle. +LJ038-0034|When he reached the row where the suspect was sitting, McDonald stopped abruptly and told the man to get on his feet.|When he reached the row where the suspect was sitting, McDonald stopped abruptly and told the man to get on his feet. +LJ038-0035|Oswald rose from his seat, bringing up both hands.|Oswald rose from his seat, bringing up both hands. +LJ038-0036|As McDonald started to search Oswald's waist for a gun, he heard him say, quote, Well, it's all over now, end quote.|As McDonald started to search Oswald's waist for a gun, he heard him say, quote, Well, it's all over now, end quote. +LJ038-0037|Oswald then struck McDonald between the eyes with his left fist; with his right hand he drew a gun from his waist.|Oswald then struck McDonald between the eyes with his left fist; with his right hand he drew a gun from his waist. +LJ038-0038|McDonald struck back with his right hand and grabbed the gun with his left hand. They both fell into the seats.|McDonald struck back with his right hand and grabbed the gun with his left hand. They both fell into the seats. +LJ038-0039|Three other officers, moving toward the scuffle, grabbed Oswald from the front, rear and side.|Three other officers, moving toward the scuffle, grabbed Oswald from the front, rear and side. +LJ038-0040|As McDonald fell into the seat with his left hand on the gun, he felt something graze across his hand and heard what sounded like the snap of the hammer.|As McDonald fell into the seat with his left hand on the gun, he felt something graze across his hand and heard what sounded like the snap of the hammer. +LJ038-0041|McDonald felt the pistol scratch his cheek as he wrenched it away from Oswald.|McDonald felt the pistol scratch his cheek as he wrenched it away from Oswald. +LJ038-0042|Detective Bob K. Carroll, who was standing beside McDonald, seized the gun from him.|Detective Bob K. Carroll, who was standing beside McDonald, seized the gun from him. +LJ038-0043|The other officers who helped subdue Oswald corroborated McDonald in his testimony|The other officers who helped subdue Oswald corroborated McDonald in his testimony +LJ038-0044|except that they did not hear Oswald say, quote, It's all over now, end quote.|except that they did not hear Oswald say, quote, It's all over now, end quote. +LJ038-0045|Deputy Sheriff Eddy R. Walthers recalled such a remark but he did not reach the scene of the struggle|Deputy Sheriff Eddy R. Walthers recalled such a remark but he did not reach the scene of the struggle +LJ038-0046|until Oswald had been knocked to the floor by McDonald and the others.|until Oswald had been knocked to the floor by McDonald and the others. +LJ038-0047|Some of the officers saw Oswald strike McDonald with his fist. Most of them heard a click which they assumed to be a click of the hammer of the revolver.|Some of the officers saw Oswald strike McDonald with his fist. Most of them heard a click which they assumed to be a click of the hammer of the revolver. +LJ038-0048|Testimony of a firearms expert before the Commission established that the hammer of the revolver never touched the shell in the chamber.|Testimony of a firearms expert before the Commission established that the hammer of the revolver never touched the shell in the chamber. +LJ038-0049|Although the witnesses did not hear the sound of a misfire,|Although the witnesses did not hear the sound of a misfire, +LJ038-0050|they might have heard a snapping noise resulting from the police officer grabbing the cylinder of the revolver and pulling it away from Oswald while he was attempting to pull the trigger.|they might have heard a snapping noise resulting from the police officer grabbing the cylinder of the revolver and pulling it away from Oswald while he was attempting to pull the trigger. +LJ038-0051|Two patrons of the theatre and John Brewer|Two patrons of the theatre and John Brewer +LJ038-0053|George Jefferson Applin, Jr., confirmed that Oswald fought with four or five officers before he was handcuffed.|George Jefferson Applin, Jr., confirmed that Oswald fought with four or five officers before he was handcuffed. +LJ038-0054|He added that one officer grabbed the muzzle of a shotgun, drew back, and hit Oswald with the butt end of the gun in the back.|He added that one officer grabbed the muzzle of a shotgun, drew back, and hit Oswald with the butt end of the gun in the back. +LJ038-0055|No other theatre patron or officer has testified that Oswald was hit by a gun.|No other theatre patron or officer has testified that Oswald was hit by a gun. +LJ038-0056|Nor did Oswald ever complain that he was hit with a gun, or injured in the back.|Nor did Oswald ever complain that he was hit with a gun, or injured in the back. +LJ038-0057|Deputy Sheriff Walthers brought a shotgun into the theatre but laid it on some seats before helping subdue Oswald.|Deputy Sheriff Walthers brought a shotgun into the theatre but laid it on some seats before helping subdue Oswald. +LJ038-0058|Officer Ray Hawkins said|Officer Ray Hawkins said +LJ038-0059|that there was no one near Oswald who had a shotgun and he saw no one strike Oswald in the back with a rifle butt or the butt of a gun.|that there was no one near Oswald who had a shotgun and he saw no one strike Oswald in the back with a rifle butt or the butt of a gun. +LJ038-0060|John Gibson, another patron in the theatre, saw an officer grab Oswald, and he claims that he heard the click of a gun misfiring.|John Gibson, another patron in the theatre, saw an officer grab Oswald, and he claims that he heard the click of a gun misfiring. +LJ038-0061|He saw no shotgun in the possession of any policeman near Oswald.|He saw no shotgun in the possession of any policeman near Oswald. +LJ038-0062|Johnny Brewer testified he saw Oswald pull the revolver and the officers struggle with him to take it away|Johnny Brewer testified he saw Oswald pull the revolver and the officers struggle with him to take it away +LJ038-0063|but that once he was subdued, no officer struck him.|but that once he was subdued, no officer struck him. +LJ038-0064|He further stated that while fists were flying he heard one of the officers say, quote, Kill the President, will you, end quote.|He further stated that while fists were flying he heard one of the officers say, quote, Kill the President, will you, end quote. +LJ038-0065|It is unlikely that any of the police officers referred to Oswald as a suspect in the assassination.|It is unlikely that any of the police officers referred to Oswald as a suspect in the assassination. +LJ038-0066|While the police radio had noted the similarity in description of the two suspects, the arresting officers were pursuing Oswald for the murder of Tippit.|While the police radio had noted the similarity in description of the two suspects, the arresting officers were pursuing Oswald for the murder of Tippit. +LJ038-0067|As Oswald, handcuffed, was led from the theatre, he was, according to McDonald, quote,|As Oswald, handcuffed, was led from the theatre, he was, according to McDonald, quote, +LJ038-0068|cursing a little bit and hollering police brutality, end quote.|cursing a little bit and hollering police brutality, end quote. +LJ038-0069|At 1:51 p.m., police car 2 reported by radio that it was on the way to headquarters with the suspect.|At one:fifty-one p.m., police car two reported by radio that it was on the way to headquarters with the suspect. +LJ038-0070|Captain Fritz returned to police headquarters from the Texas School Book Depository at 2:15 after a brief stop at the sheriff's office.|Captain Fritz returned to police headquarters from the Texas School Book Depository at two:fifteen after a brief stop at the sheriff's office. +LJ038-0071|When he entered the homicide and robbery bureau office, he saw two detectives standing there with Sgt. Gerald L. Hill,|When he entered the homicide and robbery bureau office, he saw two detectives standing there with Sgt. Gerald L. Hill, +LJ038-0072|who had driven from the theatre with Oswald.|who had driven from the theatre with Oswald. +LJ038-0073|Hill testified that Fritz told the detective to get a search warrant, go to an address on Fifth Street in Irving,|Hill testified that Fritz told the detective to get a search warrant, go to an address on Fifth Street in Irving, +LJ038-0074|and pick up a man named Lee Oswald. When Hill asked why Oswald was wanted,|and pick up a man named Lee Oswald. When Hill asked why Oswald was wanted, +LJ038-0075|Fritz replied, quote, Well, he was employed down at the Book Depository and he had not been present for a roll call of the employees, end quote.|Fritz replied, quote, Well, he was employed down at the Book Depository and he had not been present for a roll call of the employees, end quote. +LJ038-0076|Hill said, quote, Captain, we will save you a trip. There he sits, end quote.|Hill said, quote, Captain, we will save you a trip. There he sits, end quote. +LJ038-0078|Oswald was questioned intermittently for approximately 12 hours between 2:30 p.m., on November 22, and 11 a.m.,|Oswald was questioned intermittently for approximately twelve hours between two:thirty p.m., on November twenty-two, and eleven a.m., +LJ038-0079|on November 24.|on November twenty-four. +LJ038-0080|Throughout this interrogation he denied that he had anything to do either with the assassination of President Kennedy or the murder of Patrolman Tippit.|Throughout this interrogation he denied that he had anything to do either with the assassination of President Kennedy or the murder of Patrolman Tippit. +LJ038-0081|Captain Fritz of the homicide and robbery bureau did most of the questioning, but he kept no notes and there were no stenographic or tape recordings.|Captain Fritz of the homicide and robbery bureau did most of the questioning, but he kept no notes and there were no stenographic or tape recordings. +LJ038-0082|Representatives of other law enforcement agencies were also present, including the FBI and the U.S. Secret Service.|Representatives of other law enforcement agencies were also present, including the FBI and the U.S. Secret Service. +LJ038-0083|They occasionally participated in the questioning.|They occasionally participated in the questioning. +LJ038-0084|The reports prepared by those present at these interviews are set forth in appendix 11.|The reports prepared by those present at these interviews are set forth in appendix eleven. +LJ038-0085|A full discussion of Oswald's detention and interrogation is presented in chapter 5 of this report.|A full discussion of Oswald's detention and interrogation is presented in chapter five of this report. +LJ038-0086|During the evening of November 22, the Dallas Police Department performed paraffin tests on Oswald's hands and right cheek|During the evening of November twenty-two, the Dallas Police Department performed paraffin tests on Oswald's hands and right cheek +LJ038-0087|in an apparent effort to determine, by means of a scientific test, whether Oswald had recently fired a weapon.|in an apparent effort to determine, by means of a scientific test, whether Oswald had recently fired a weapon. +LJ038-0088|The results were positive for the hands and negative for the right cheek.|The results were positive for the hands and negative for the right cheek. +LJ038-0089|Expert testimony before the Commission|Expert testimony before the Commission +LJ038-0090|was to the effect that the paraffin test was unreliable in determining whether or not a person has fired a rifle or revolver.|was to the effect that the paraffin test was unreliable in determining whether or not a person has fired a rifle or revolver. +LJ038-0091|The Commission has, therefore, placed no reliance on the paraffin tests administered by the Dallas police.|The Commission has, therefore, placed no reliance on the paraffin tests administered by the Dallas police. +LJ038-0092|Oswald provided little information during his questioning.|Oswald provided little information during his questioning. +LJ038-0093|Frequently, however, he was confronted with evidence which he could not explain, and he resorted to statements which are known to be lies.|Frequently, however, he was confronted with evidence which he could not explain, and he resorted to statements which are known to be lies. +LJ038-0094|While Oswald's untrue statements during interrogation were not considered items of positive proof|While Oswald's untrue statements during interrogation were not considered items of positive proof +LJ038-0095|by the Commission, they had probative value in deciding the weight to be given to his denials that he assassinated President Kennedy|by the Commission, they had probative value in deciding the weight to be given to his denials that he assassinated President Kennedy +LJ038-0096|and killed Patrolman Tippit. Since independent evidence revealed that Oswald repeatedly and blatantly lied to the police,|and killed Patrolman Tippit. Since independent evidence revealed that Oswald repeatedly and blatantly lied to the police, +LJ038-0097|the Commission gave little weight to his denials of guilt.|the Commission gave little weight to his denials of guilt. +LJ038-0098|Denial of Rifle Ownership|Denial of Rifle Ownership +LJ038-0099|From the outset, Oswald denied owning a rifle.|From the outset, Oswald denied owning a rifle. +LJ038-0100|On November 23, Fritz confronted Oswald with the evidence that he had purchased a rifle under the fictitious name of "Hidell."|On November twenty-three, Fritz confronted Oswald with the evidence that he had purchased a rifle under the fictitious name of "Hidell." +LJ038-0101|Oswald said that this was not true. Oswald denied that he had a rifle wrapped up in a blanket in the Paine garage.|Oswald said that this was not true. Oswald denied that he had a rifle wrapped up in a blanket in the Paine garage. +LJ038-0102|Oswald also denied owning a rifle and said that since leaving the Marine Corps he had fired only a small bore 22 rifle.|Oswald also denied owning a rifle and said that since leaving the Marine Corps he had fired only a small bore twenty-two rifle. +LJ038-0103|On the afternoon of November 23, Officers H. M. Moore,|On the afternoon of November twenty-three, Officers H. M. Moore, +LJ038-0104|R. S. Stovall, and G. F. Rose obtained a search warrant and examined Oswald's effects in the Paine garage.|R. S. Stovall, and G. F. Rose obtained a search warrant and examined Oswald's effects in the Paine garage. +LJ038-0105|They discovered two photographs, each showing Oswald with a rifle and a pistol.|They discovered two photographs, each showing Oswald with a rifle and a pistol. +LJ038-0106|These photographs were shown to Oswald on the evening of November 23 and again on the morning of the 24th.|These photographs were shown to Oswald on the evening of November twenty-three and again on the morning of the twenty-fourth. +LJ038-0107|According to Fritz, Oswald sneered, saying that they were fake photographs, that he had been photographed a number of times the day before by the police,|According to Fritz, Oswald sneered, saying that they were fake photographs, that he had been photographed a number of times the day before by the police, +LJ038-0108|that they had superimposed upon the photographs a rifle and a revolver. He told Fritz a number of times that the smaller photograph|that they had superimposed upon the photographs a rifle and a revolver. He told Fritz a number of times that the smaller photograph +LJ038-0109|was either made from the larger, or the larger photograph was made from the smaller and that at the proper time he would show that the pictures were fakes.|was either made from the larger, or the larger photograph was made from the smaller and that at the proper time he would show that the pictures were fakes. +LJ038-0110|Fritz told him that the two small photographs were found in the Paine garage.|Fritz told him that the two small photographs were found in the Paine garage. +LJ038-0111|At that point, Oswald refused to answer any further questions.|At that point, Oswald refused to answer any further questions. +LJ038-0112|As previously indicated, Marina Oswald testified that she took the two pictures with her husband's Imperial Reflex camera|As previously indicated, Marina Oswald testified that she took the two pictures with her husband's Imperial Reflex camera +LJ038-0113|when they lived on Neely Street.|when they lived on Neely Street. +LJ038-0114|Her testimony was fully supported by a photography expert who testified that in his opinion the pictures were not composites.|Her testimony was fully supported by a photography expert who testified that in his opinion the pictures were not composites. +LJ038-0115|At the first interrogation, Oswald claimed that his only crime was carrying a gun and resisting arrest.|At the first interrogation, Oswald claimed that his only crime was carrying a gun and resisting arrest. +LJ038-0116|When Captain Fritz asked him why he carried the revolver, he answered, quote, Well, you know about a pistol. I just carried it, end quote.|When Captain Fritz asked him why he carried the revolver, he answered, quote, Well, you know about a pistol. I just carried it, end quote. +LJ038-0117|He falsely alleged that he bought the revolver in Fort Worth, when in fact he purchased it from a mail-order house in Los Angeles.|He falsely alleged that he bought the revolver in Fort Worth, when in fact he purchased it from a mail-order house in Los Angeles. +LJ038-0118|The Aliases "Hidell" and "O. H. Lee"|The Aliases "Hidell" and "O. H. Lee" +LJ038-0119|The arresting officers found a forged selective service card with a picture of Oswald and the name "Alek J. Hidell"|The arresting officers found a forged selective service card with a picture of Oswald and the name "Alek J. Hidell" +LJ038-0120|in Oswald's billfold. On November 22 and 23, Oswald refused to tell Fritz why this card was in his possession,|in Oswald's billfold. On November twenty-two and twenty-three, Oswald refused to tell Fritz why this card was in his possession, +LJ038-0121|or to answer any questions concerning the card.|or to answer any questions concerning the card. +LJ038-0122|On Sunday morning, November 24, Oswald denied that he knew A. J. Hidell.|On Sunday morning, November twenty-four, Oswald denied that he knew A. J. Hidell. +LJ038-0123|Captain Fritz produced the selective service card bearing the name "Alek J. Hidell."|Captain Fritz produced the selective service card bearing the name "Alek J. Hidell." +LJ038-0124|Oswald became angry and said, quote,|Oswald became angry and said, quote, +LJ038-0125|Now, I've told you all I'm going to tell you about that card in my billfolds -- you have the card yourself and you know as much about it as I do, end quote.|Now, I've told you all I'm going to tell you about that card in my billfolds -- you have the card yourself and you know as much about it as I do, end quote. +LJ038-0126|At the last interrogation in November Oswald admitted to Postal Inspector Holmes that he had rented post office box 2915, Dallas,|At the last interrogation in November Oswald admitted to Postal Inspector Holmes that he had rented post office box two nine one five, Dallas, +LJ038-0127|but denied that he had received a package in this box addressed to Hidell.|but denied that he had received a package in this box addressed to Hidell. +LJ038-0128|He also denied that he had received the rifle through this box.|He also denied that he had received the rifle through this box. +LJ038-0129|Holmes reminded Oswald that A. J. Hidell was listed on post office box 30061, New Orleans,|Holmes reminded Oswald that A. J. Hidell was listed on post office box three zero zero six one, New Orleans, +LJ038-0130|as one entitled to receive mail.|as one entitled to receive mail. +LJ038-0131|Oswald replied, quote, I don't know anything about that, end quote.|Oswald replied, quote, I don't know anything about that, end quote. +LJ038-0132|When asked why he lived at his roominghouse under the name O. H. Lee,|When asked why he lived at his roominghouse under the name O. H. Lee, +LJ038-0133|Oswald responded that the landlady simply made a mistake, because he told her that his name was Lee, meaning his first name.|Oswald responded that the landlady simply made a mistake, because he told her that his name was Lee, meaning his first name. +LJ038-0134|An examination of the roominghouse register revealed that Oswald actually signed the name O. H. Lee.|An examination of the roominghouse register revealed that Oswald actually signed the name O. H. Lee. +LJ038-0135|The Curtain Rod Story|The Curtain Rod Story +LJ038-0136|In concluding that Oswald was carrying a rifle in the paper bag on the morning of November 22, 1963,|In concluding that Oswald was carrying a rifle in the paper bag on the morning of November twenty-two, nineteen sixty-three, +LJ038-0137|the Commission found that Oswald lied when he told Frazier that he was returning to Irving to obtain curtain rods.|the Commission found that Oswald lied when he told Frazier that he was returning to Irving to obtain curtain rods. +LJ038-0138|When asked about the curtain rod story, Oswald lied again.|When asked about the curtain rod story, Oswald lied again. +LJ038-0139|He denied that he had ever told Frazier that he wanted a ride to Irving to get curtain rods for an apartment.|He denied that he had ever told Frazier that he wanted a ride to Irving to get curtain rods for an apartment. +LJ038-0140|He explained that a party for the Paine children had been planned for the weekend and he preferred not to be in the Paine house at that time;|He explained that a party for the Paine children had been planned for the weekend and he preferred not to be in the Paine house at that time; +LJ038-0141|therefore, he made his weekly visit on Thursday night.|therefore, he made his weekly visit on Thursday night. +LJ038-0142|Actually, the party for one of the Paine's children was the preceding weekend, when Marina Oswald suggested that Oswald remain in Dallas.|Actually, the party for one of the Paine's children was the preceding weekend, when Marina Oswald suggested that Oswald remain in Dallas. +LJ038-0143|When told that Frazier and Mrs. Randle had seen him carrying a long heavy package, Oswald replied, quote, Well, they was mistaken.|When told that Frazier and Mrs. Randle had seen him carrying a long heavy package, Oswald replied, quote, Well, they was mistaken. +LJ038-0144|that must have been some other time he picked me up, end quote,|that must have been some other time he picked me up, end quote, +LJ038-0145|In one interview, he told Fritz that the only sack he carried to work that day|In one interview, he told Fritz that the only sack he carried to work that day +LJ038-0146|was a lunch sack which he kept on his lap during the ride from Irving to Dallas.|was a lunch sack which he kept on his lap during the ride from Irving to Dallas. +LJ038-0147|Frazier testified before the Commission that Oswald carried no lunch sack that day.|Frazier testified before the Commission that Oswald carried no lunch sack that day. +LJ038-0148|Actions During and After Shooting|Actions During and After Shooting +LJ038-0149|During the first interrogation on November 22, Fritz asked Oswald to account for himself at the time the President was shot.|During the first interrogation on November twenty-two, Fritz asked Oswald to account for himself at the time the President was shot. +LJ038-0150|Oswald told him that he ate lunch in the first-floor lunchroom and then went to the second floor for a Coke which he brought downstairs.|Oswald told him that he ate lunch in the first-floor lunchroom and then went to the second floor for a Coke which he brought downstairs. +LJ038-0151|He acknowledged the encounter with the police officer on the second floor.|He acknowledged the encounter with the police officer on the second floor. +LJ038-0152|Oswald told Fritz that after lunch he went outside, talked with Foreman Bill Shelley for 5 or 10 minutes and then left for home.|Oswald told Fritz that after lunch he went outside, talked with Foreman Bill Shelley for five or ten minutes and then left for home. +LJ038-0153|He said that he left work because Bill Shelley said that there would be no more work done that day in the building.|He said that he left work because Bill Shelley said that there would be no more work done that day in the building. +LJ038-0154|Shelley denied seeing Oswald after 12 noon or at any time after the shooting.|Shelley denied seeing Oswald after twelve noon or at any time after the shooting. +LJ038-0155|The next day, Oswald added to his story.|The next day, Oswald added to his story. +LJ038-0156|He stated that at the time the President was shot he was having lunch with "Junior" but he did not give Junior's last name.|He stated that at the time the President was shot he was having lunch with "Junior" but he did not give Junior's last name. +LJ038-0157|The only employee at the Depository Building named "Junior" was James Jarman, Jr.|The only employee at the Depository Building named "Junior" was James Jarman, Jr. +LJ038-0158|Jarman testified that he ate his lunch on the first floor around 5 minutes to 12, and that he neither ate lunch with nor saw Oswald.|Jarman testified that he ate his lunch on the first floor around five minutes to twelve, and that he neither ate lunch with nor saw Oswald. +LJ038-0159|Jarman did talk to Oswald that morning: Quote,|Jarman did talk to Oswald that morning: Quote, +LJ038-0160|he asked me what were the people gathering around on the corner for and I told him that the President was supposed to pass that morning,|he asked me what were the people gathering around on the corner for and I told him that the President was supposed to pass that morning, +LJ038-0162|Then he said, "Oh, I see," and that was all.|Then he said, "Oh, I see," and that was all. +LJ038-0163|Prior attempt to kill.|Prior attempt to kill. +LJ038-0164|The Attempt on the Life of Maj. Gen. Edwin A. Walker|The Attempt on the Life of Maj. Gen. Edwin A. Walker +LJ038-0165|At approximately 9 p.m., on April 10, 1963, in Dallas, Texas, Maj. Gen. Edwin A. Walker,|At approximately nine p.m., on April ten, nineteen sixty-three, in Dallas, Texas, Maj. Gen. Edwin A. Walker, +LJ038-0166|an active and controversial figure on the American political scene since his resignation from the U.S. Army in 1961|an active and controversial figure on the American political scene since his resignation from the U.S. Army in nineteen sixty-one +LJ038-0167|narrowly escaped death when a rifle bullet fired from outside his home passed near his head as he was seated at his desk.|narrowly escaped death when a rifle bullet fired from outside his home passed near his head as he was seated at his desk. +LJ038-0168|There were no eyewitnesses, although a 14-year-old boy in a neighboring house claimed that immediately after the shooting|There were no eyewitnesses, although a fourteen-year-old boy in a neighboring house claimed that immediately after the shooting +LJ038-0169|he saw two men, in separate cars, drive out of a church parking lot adjacent to Walker's home. A friend of Walker's testified that|he saw two men, in separate cars, drive out of a church parking lot adjacent to Walker's home. A friend of Walker's testified that +LJ038-0170|two nights before the shooting he saw, quote, two men around the house peeking in windows, end quote.|two nights before the shooting he saw, quote, two men around the house peeking in windows, end quote. +LJ038-0171|General Walker gave this information to the police before the shooting, but it did not help solve the crime.|General Walker gave this information to the police before the shooting, but it did not help solve the crime. +LJ038-0172|Although the bullet was recovered from Walker's house, in the absence of a weapon it was of little investigatory value.|Although the bullet was recovered from Walker's house, in the absence of a weapon it was of little investigatory value. +LJ038-0173|General Walker hired two investigators to determine whether a former employee might have been involved in the shooting.|General Walker hired two investigators to determine whether a former employee might have been involved in the shooting. +LJ038-0174|Their results were negative.|Their results were negative. +LJ038-0175|Until December 3, 1963, the Walker shooting remained unsolved.|Until December three, nineteen sixty-three, the Walker shooting remained unsolved. +LJ038-0176|The Commission evaluated the following evidence in considering whether Lee Harvey Oswald fired the shot which almost killed General Walker:|The Commission evaluated the following evidence in considering whether Lee Harvey Oswald fired the shot which almost killed General Walker: +LJ038-0177|A note which Oswald left for his wife on the evening of the shooting,|A note which Oswald left for his wife on the evening of the shooting, +LJ038-0178|(2) photographs found among Oswald's possessions after the assassination of President Kennedy,|(two) photographs found among Oswald's possessions after the assassination of President Kennedy, +LJ038-0179|(3) firearm identification of the bullet found in Walker's home, and (4)|(three) firearm identification of the bullet found in Walker's home, and (four) +LJ038-0180|admissions and other statements made to Marina Oswald by Oswald concerning the shooting. Note left by Oswald.|admissions and other statements made to Marina Oswald by Oswald concerning the shooting. Note left by Oswald. +LJ038-0181|On December 2, 1963, Mrs. Ruth Paine turned over to the police some of the Oswalds' belongings,|On December two, nineteen sixty-three, Mrs. Ruth Paine turned over to the police some of the Oswalds' belongings, +LJ038-0182|including a Russian volume entitled, quote, Book of Useful Advice, end quote. In this book was an undated note written in Russian.|including a Russian volume entitled, quote, Book of Useful Advice, end quote. In this book was an undated note written in Russian. +LJ038-0183|In translation, the note read as follows: 1.|In translation, the note read as follows: one. +LJ038-0184|This is the key to the mailbox which is located in the main post office in the city on Ervay Street.|This is the key to the mailbox which is located in the main post office in the city on Ervay Street. +LJ038-0185|This is the same street where the drugstore, in which you always waited is located.|This is the same street where the drugstore, in which you always waited is located. +LJ038-0186|You will find the mailbox in the post office which is located 4 blocks from the drugstore on that street.|You will find the mailbox in the post office which is located four blocks from the drugstore on that street. +LJ038-0187|I paid for the box last month so don't worry about it.|I paid for the box last month so don't worry about it. +LJ038-0188|2. Send the information as to what has happened to me to the Embassy.|two. Send the information as to what has happened to me to the Embassy. +LJ038-0189|and include newspaper clippings (should there be anything about me in the newspapers). I believe that the Embassy will come quickly to your assistance on learning everything.|and include newspaper clippings (should there be anything about me in the newspapers). I believe that the Embassy will come quickly to your assistance on learning everything. +LJ038-0190|3. I paid the house rent on the 2nd so don't worry about it. 4. Recently I also paid for water and gas.|three. I paid the house rent on the second so don't worry about it. four. Recently I also paid for water and gas. +LJ038-0191|5. The money from work will possibly be coming. The money will be sent to our post office box. Go to the bank and cash the check.|five. The money from work will possibly be coming. The money will be sent to our post office box. Go to the bank and cash the check. +LJ038-0192|6. You can either throw out or give my clothing, etc. away. Do not keep these.|six. You can either throw out or give my clothing, etc. away. Do not keep these. +LJ038-0193|However, I prefer that you hold on to my personal papers (military, civil, etc.).|However, I prefer that you hold on to my personal papers (military, civil, etc.). +LJ038-0194|7. Certain of my documents are in the small blue valise.|seven. Certain of my documents are in the small blue valise. +LJ038-0197|10. I left you as much money as I could,|ten. I left you as much money as I could, +LJ038-0198|$60 on the second of the month. You and the baby can live for another 2 months using $10 per week.|sixty dollars on the second of the month. You and the baby can live for another two months using ten dollars per week. +LJ038-0199|11. If I am alive and taken prisoner,|eleven. If I am alive and taken prisoner, +LJ038-0200|the city jail is located at the end of the bridge through which we always passed on going to the city (right in the beginning of the city after crossing the bridge).|the city jail is located at the end of the bridge through which we always passed on going to the city (right in the beginning of the city after crossing the bridge). +LJ038-0201|James C. Cadigan, FBI handwriting expert, testified that this note was written by Lee Harvey Oswald.|James C. Cadigan, FBI handwriting expert, testified that this note was written by Lee Harvey Oswald. +LJ038-0202|Prior to the Walker shooting on April 10, Oswald had been attending typing classes on Monday, Tuesday, and Thursday evenings.|Prior to the Walker shooting on April ten, Oswald had been attending typing classes on Monday, Tuesday, and Thursday evenings. +LJ038-0203|He had quit these classes at least a week before the shooting, which occurred on a Wednesday night.|He had quit these classes at least a week before the shooting, which occurred on a Wednesday night. +LJ038-0204|According to Marina Oswald's testimony, on the night of the Walker shooting, her husband left their apartment on Neely Street shortly after dinner.|According to Marina Oswald's testimony, on the night of the Walker shooting, her husband left their apartment on Neely Street shortly after dinner. +LJ038-0205|She thought he was attending a class or was on his own business.|She thought he was attending a class or was on his own business. +LJ038-0206|When he failed to return by 10 or 10:30 p.m., Marina Oswald went to his room and discovered the note. She testified: quote,|When he failed to return by ten or ten:thirty p.m., Marina Oswald went to his room and discovered the note. She testified: quote, +LJ038-0207|When he came back I asked him what had happened. He was very pale.|When he came back I asked him what had happened. He was very pale. +LJ038-0208|I don't remember the exact time, but it was very late. And he told me not to ask him any questions. He only told me he had shot at General Walker.|I don't remember the exact time, but it was very late. And he told me not to ask him any questions. He only told me he had shot at General Walker. +LJ038-0209|Oswald told his wife that he did not know whether he had hit Walker;|Oswald told his wife that he did not know whether he had hit Walker; +LJ038-0210|according to Marina Oswald when he learned on the radio and in the newspapers the next day that he had missed,|according to Marina Oswald when he learned on the radio and in the newspapers the next day that he had missed, +LJ038-0211|he said that he, quote, was very sorry that he had not hit him, end quote. Marina Oswald's testimony was fully supported by the note itself|he said that he, quote, was very sorry that he had not hit him, end quote. Marina Oswald's testimony was fully supported by the note itself +LJ038-0213|The last paragraph directed her to the jail|The last paragraph directed her to the jail +LJ038-0214|and the other paragraphs instructed her on the disposal of Oswald's personal effects and the management of her affairs if he should not return.|and the other paragraphs instructed her on the disposal of Oswald's personal effects and the management of her affairs if he should not return. +LJ038-0215|It is clear that the note was written while the Oswalds were living in Dallas before they moved to New Orleans in the spring of 1963.|It is clear that the note was written while the Oswalds were living in Dallas before they moved to New Orleans in the spring of nineteen sixty-three. +LJ038-0216|The references to house rent and payments for water and gas|The references to house rent and payments for water and gas +LJ038-0217|indicated that the note was written when they were living in a rented apartment; therefore it could not have been written while Marina Oswald was living with the Paines.|indicated that the note was written when they were living in a rented apartment; therefore it could not have been written while Marina Oswald was living with the Paines. +LJ038-0218|Moreover, the reference in paragraph 3 to paying, quote, the house rent on the 2nd, end quote,|Moreover, the reference in paragraph three to paying, quote, the house rent on the second, end quote, +LJ038-0219|would be consistent with the period when the Oswalds were living on Neely Street since the apartment was rented on March 3, 1963.|would be consistent with the period when the Oswalds were living on Neely Street since the apartment was rented on March three, nineteen sixty-three. +LJ038-0220|Oswald had paid the first month's rent in advance on March 2, 1963,|Oswald had paid the first month's rent in advance on March two, nineteen sixty-three, +LJ038-0221|and the second month's rent was paid on either April 2 or April 3.|and the second month's rent was paid on either April two or April three. +LJ038-0222|The main post office, quote, on Ervay Street, end quote, refers to the post office where Oswald rented box 2915|The main post office, quote, on Ervay Street, end quote, refers to the post office where Oswald rented box two nine one five +LJ038-0223|from October 9, 1962, to May 14, 1963.|from October nine, nineteen sixty-two, to May fourteen, nineteen sixty-three. +LJ038-0224|Another statement which limits the time when it could have been written is the reference, quote, you and the baby, end quote,|Another statement which limits the time when it could have been written is the reference, quote, you and the baby, end quote, +LJ038-0225|which would indicate that it was probably written before the birth of Oswald's second child on October 20, 1963.|which would indicate that it was probably written before the birth of Oswald's second child on October twenty, nineteen sixty-three. +LJ038-0226|Oswald had apparently mistaken the county jail for the city jail.|Oswald had apparently mistaken the county jail for the city jail. +LJ038-0227|From Neely Street the Oswalds would have traveled downtown on the Beckley bus, across the Commerce Street viaduct|From Neely Street the Oswalds would have traveled downtown on the Beckley bus, across the Commerce Street viaduct +LJ038-0228|and into downtown Dallas through the Triple Underpass.|and into downtown Dallas through the Triple Underpass. +LJ038-0229|Either the viaduct or the underpass might have been the "bridge" mentioned in the last paragraph of the note.|Either the viaduct or the underpass might have been the "bridge" mentioned in the last paragraph of the note. +LJ038-0230|The county jail is at the corner of Houston and Main Streets, quote, right in the beginning of the city, end quote, after one travels through the underpass.|The county jail is at the corner of Houston and Main Streets, quote, right in the beginning of the city, end quote, after one travels through the underpass. +LJ038-0231|In her testimony before the Commission in February 1964,|In her testimony before the Commission in February nineteen sixty-four, +LJ038-0232|Marina Oswald stated that when Oswald returned home on the night of the Walker shooting, he told her that he had been planning the attempt for 2 months.|Marina Oswald stated that when Oswald returned home on the night of the Walker shooting, he told her that he had been planning the attempt for two months. +LJ038-0233|He showed her a notebook 3 days later containing photographs of General Walker's home and a map of the area where the house was located.|He showed her a notebook three days later containing photographs of General Walker's home and a map of the area where the house was located. +LJ038-0234|Although Oswald destroyed the notebook, three photographs found among Oswald's possessions after the assassination|Although Oswald destroyed the notebook, three photographs found among Oswald's possessions after the assassination +LJ038-0235|were identified by Marina Oswald as photographs of General Walker's house.|were identified by Marina Oswald as photographs of General Walker's house. +LJ038-0236|Two of these photographs were taken from the rear of Walker's house.|Two of these photographs were taken from the rear of Walker's house. +LJ038-0237|The Commission confirmed, by comparison with other photographs, that these were, indeed, photographs of the rear of Walker's house.|The Commission confirmed, by comparison with other photographs, that these were, indeed, photographs of the rear of Walker's house. +LJ038-0238|An examination of the window at the rear of the house, the wall through which the bullet passed, and the fence behind the house|An examination of the window at the rear of the house, the wall through which the bullet passed, and the fence behind the house +LJ038-0239|indicated that the bullet was fired from a position near the point where one of the photographs was taken.|indicated that the bullet was fired from a position near the point where one of the photographs was taken. +LJ038-0240|The third photograph identified by Marina Oswald depicts the entrance to General Walker's driveway from a back alley.|The third photograph identified by Marina Oswald depicts the entrance to General Walker's driveway from a back alley. +LJ038-0241|Also seen in the picture is the fence on which Walker's assailant apparently rested the rifle.|Also seen in the picture is the fence on which Walker's assailant apparently rested the rifle. +LJ038-0242|An examination of certain construction work appearing in the background of this photograph revealed that the picture was taken between March 8|An examination of certain construction work appearing in the background of this photograph revealed that the picture was taken between March eight +LJ038-0243|and 12, 1963, and most probably on either March 9 or March 10.|and twelve, nineteen sixty-three, and most probably on either March nine or March ten. +LJ038-0244|Oswald purchased the money order for the rifle on March 12, the rifle was shipped on March 20, and the shooting occurred on April 10.|Oswald purchased the money order for the rifle on March twelve, the rifle was shipped on March twenty, and the shooting occurred on April ten. +LJ038-0245|A photography expert with the FBI|A photography expert with the FBI +LJ038-0246|was able to determine that, this picture was taken with the Imperial Reflex camera owned by Lee Harvey Oswald.|was able to determine that, this picture was taken with the Imperial Reflex camera owned by Lee Harvey Oswald. +LJ038-0247|A fourth photograph, showing a stretch of railroad tracks,|A fourth photograph, showing a stretch of railroad tracks, +LJ038-0248|was also identified by Marina Oswald as having been taken by her husband, presumably in connection with the Walker shooting.|was also identified by Marina Oswald as having been taken by her husband, presumably in connection with the Walker shooting. +LJ038-0249|Investigation determined that this photograph was taken approximately seven-tenths of a mile from Walker's house.|Investigation determined that this photograph was taken approximately seven-tenths of a mile from Walker's house. +LJ038-0250|Another photograph of railroad tracks found among Oswald's possessions was not identified by his wife,|Another photograph of railroad tracks found among Oswald's possessions was not identified by his wife, +LJ038-0251|but investigation revealed that it was taken from a point slightly less than half a mile from General Walker's house.|but investigation revealed that it was taken from a point slightly less than half a mile from General Walker's house. +LJ038-0252|Marina Oswald stated that- when she asked her husband what be had done with the rifle,|Marina Oswald stated that- when she asked her husband what be had done with the rifle, +LJ038-0253|he replied that he had buried it in the ground or hidden it in some bushes and that he also mentioned a railroad track in this connection.|he replied that he had buried it in the ground or hidden it in some bushes and that he also mentioned a railroad track in this connection. +LJ038-0254|She testified that several days later Oswald recovered his rifle and brought it back to their apartment.|She testified that several days later Oswald recovered his rifle and brought it back to their apartment. +LJ038-0255|Firearms identification.|Firearms identification. +LJ038-0256|In the room beyond the one in which General Walker was sitting on the night of the shooting the Dallas police recovered a badly mutilated bullet|In the room beyond the one in which General Walker was sitting on the night of the shooting the Dallas police recovered a badly mutilated bullet +LJ038-0257|which had come to rest on a stack of paper.|which had come to rest on a stack of paper. +LJ038-0258|The Dallas City-County Investigation Laboratory tried to determine the type of weapon which fired the bullet.|The Dallas City-County Investigation Laboratory tried to determine the type of weapon which fired the bullet. +LJ038-0259|The oral report was negative because of the battered condition of the bullet.|The oral report was negative because of the battered condition of the bullet. +LJ038-0260|On November 30, 1963, the FBI requested the bullet for ballistics examination;|On November thirty, nineteen sixty-three, the FBI requested the bullet for ballistics examination; +LJ038-0261|the Dallas Police Department forwarded it on December 2, 1963.|the Dallas Police Department forwarded it on December two, nineteen sixty-three. +LJ038-0262|Robert A. Frazier, an FBI ballistics identification expert, testified that he was, quote, unable to reach a conclusion, end quote,|Robert A. Frazier, an FBI ballistics identification expert, testified that he was, quote, unable to reach a conclusion, end quote, +LJ038-0263|as to whether or not the bullet recovered from Walker's house had been fired from the rifle found on the sixth floor of the Texas School Book Depository Building.|as to whether or not the bullet recovered from Walker's house had been fired from the rifle found on the sixth floor of the Texas School Book Depository Building. +LJ038-0264|He concluded that, quote, the general rifling characteristics of the rifle are of the same type as those found on the bullet|He concluded that, quote, the general rifling characteristics of the rifle are of the same type as those found on the bullet +LJ038-0265|and, further, on this basis the bullet could have been fired from the rifle on the basis of its land and groove impressions, end quote.|and, further, on this basis the bullet could have been fired from the rifle on the basis of its land and groove impressions, end quote. +LJ038-0266|Frazier testified further that the FBI avoids the category of "probable" identification.|Frazier testified further that the FBI avoids the category of "probable" identification. +LJ038-0267|Unless the missile or cartridge case can be identified as coming from a particular weapon to the exclusion of all others,|Unless the missile or cartridge case can be identified as coming from a particular weapon to the exclusion of all others, +LJ038-0268|the FBI refuses to draw any conclusion as to probability.|the FBI refuses to draw any conclusion as to probability. +LJ038-0269|Frazier testified, however, that he found no microscopic characteristics or other evidence|Frazier testified, however, that he found no microscopic characteristics or other evidence +LJ038-0270|which would indicate that the bullet was not fired from the Mannlicher-Carcano rifle owned by Lee Harvey Oswald.|which would indicate that the bullet was not fired from the Mannlicher-Carcano rifle owned by Lee Harvey Oswald. +LJ038-0271|It was a 6.5-millimeter bullet|It was a six point five-millimeter bullet +LJ038-0272|and, according to Frazier, "relatively few" types of rifles could produce the characteristics found on the bullet.|and, according to Frazier, "relatively few" types of rifles could produce the characteristics found on the bullet. +LJ038-0273|Joseph D. Nicol,|Joseph D. Nicol, +LJ038-0274|superintendent of the Illinois Bureau of Criminal Identification and Investigation, conducted an independent examination of this bullet|superintendent of the Illinois Bureau of Criminal Identification and Investigation, conducted an independent examination of this bullet +LJ038-0275|and concluded, quote, that there is a fair probability, end quote, that the bullet was fired from the rifle used in the assassination of President Kennedy.|and concluded, quote, that there is a fair probability, end quote, that the bullet was fired from the rifle used in the assassination of President Kennedy. +LJ038-0276|In explaining the difference between his policy and that of the FBI on the matter of probable identification, Nicol said, quote,|In explaining the difference between his policy and that of the FBI on the matter of probable identification, Nicol said, quote, +LJ038-0277|I am aware of their position. This is not, I am sure, arrived at without careful consideration.|I am aware of their position. This is not, I am sure, arrived at without careful consideration. +LJ038-0278|However, to say that because one does not find sufficient marks for identification that it is a negative,|However, to say that because one does not find sufficient marks for identification that it is a negative, +LJ038-0279|I think is going overboard in the other direction.|I think is going overboard in the other direction. +LJ038-0280|And for purposes of probative value, for whatever it might be worth,|And for purposes of probative value, for whatever it might be worth, +LJ038-0281|in the absence of very definite negative evidence, I think it is permissible to say that in an exhibit such as 573|in the absence of very definite negative evidence, I think it is permissible to say that in an exhibit such as five seven three +LJ038-0282|there is enough on it to say that it could have come, and even perhaps a little stronger, to say that it probably came from this,|there is enough on it to say that it could have come, and even perhaps a little stronger, to say that it probably came from this, +LJ038-0283|without going so far as to say to the exclusion of all other guns. This I could not do, end quote.|without going so far as to say to the exclusion of all other guns. This I could not do, end quote. +LJ038-0284|Although the Commission recognizes that neither expert was able to state|Although the Commission recognizes that neither expert was able to state +LJ038-0285|that the bullet which missed General Walker was fired from Oswald's rifle to the exclusion of all others, this testimony was considered probative|that the bullet which missed General Walker was fired from Oswald's rifle to the exclusion of all others, this testimony was considered probative +LJ038-0286|when combined with the other testimony linking Oswald to the shooting.|when combined with the other testimony linking Oswald to the shooting. +LJ038-0287|Additional corroborative evidence.|Additional corroborative evidence. +LJ038-0288|The admissions made to Marina Oswald by her husband are an important element in the evidence that Lee Harvey Oswald fired the shot at General Walker.|The admissions made to Marina Oswald by her husband are an important element in the evidence that Lee Harvey Oswald fired the shot at General Walker. +LJ038-0289|As shown above, the note and the photographs of Walker's house and of the nearby railroad tracks|As shown above, the note and the photographs of Walker's house and of the nearby railroad tracks +LJ038-0290|provide important corroboration for her account of the incident.|provide important corroboration for her account of the incident. +LJ038-0291|Other details described by Marina Oswald coincide with facts developed independently of her statements.|Other details described by Marina Oswald coincide with facts developed independently of her statements. +LJ038-0292|She testified that her husband had postponed his attempt to kill Walker|She testified that her husband had postponed his attempt to kill Walker +LJ038-0293|until that Wednesday because he had heard that there was to be a gathering at the church next door to Walker's house on that evening.|until that Wednesday because he had heard that there was to be a gathering at the church next door to Walker's house on that evening. +LJ038-0294|He indicated that he wanted more people in the vicinity at the time of the attempt so that his arrival and departure would not attract great attention.|He indicated that he wanted more people in the vicinity at the time of the attempt so that his arrival and departure would not attract great attention. +LJ038-0295|An official of this church told FBI agents that services are held every Wednesday at the church except during the month of August.|An official of this church told FBI agents that services are held every Wednesday at the church except during the month of August. +LJ038-0296|Marina Oswald also testified that her husband had used a bus to return home.|Marina Oswald also testified that her husband had used a bus to return home. +LJ038-0297|A study of the bus routes indicates that|A study of the bus routes indicates that +LJ038-0298|Oswald could have taken any one of several different buses to Walker's house or to a point near the railroad tracks where he may have concealed the rifle.|Oswald could have taken any one of several different buses to Walker's house or to a point near the railroad tracks where he may have concealed the rifle. +LJ038-0299|It would have been possible for him to take different routes in approaching and leaving the scene of the shooting.|It would have been possible for him to take different routes in approaching and leaving the scene of the shooting. +LJ038-0300|Based on (1) the contents of the note which Oswald left for his wife on April 10, 1963,|Based on (one) the contents of the note which Oswald left for his wife on April ten, nineteen sixty-three, +LJ038-0301|(2) the photographs found among Oswald's possessions,|(two) the photographs found among Oswald's possessions, +LJ038-0302|(3) the testimony of firearms identification experts, and (4) the testimony of Marina Oswald,|(three) the testimony of firearms identification experts, and (four) the testimony of Marina Oswald, +LJ038-0303|the Commission has concluded that Lee Harvey Oswald attempted to take the life of Maj. Gen. Edwin A. Walker (Resigned, U.S. Army)|the Commission has concluded that Lee Harvey Oswald attempted to take the life of Maj. Gen. Edwin A. Walker (Resigned, U.S. Army) +LJ038-0304|on April 10, 1963.|on April ten, nineteen sixty-three. +LJ038-0305|The finding that Lee Harvey Oswald attempted to murder a public figure in April 1963|The finding that Lee Harvey Oswald attempted to murder a public figure in April nineteen sixty-three +LJ038-0306|was considered of probative value in this investigation, although the Commission's conclusion concerning the identity of the assassin|was considered of probative value in this investigation, although the Commission's conclusion concerning the identity of the assassin +LJ039-0001|For more information, or to volunteer, please visit librivox dot org.|For more information, or to volunteer, please visit librivox dot org. +LJ039-0002|Report of the President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. +LJ039-0003|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ039-0004|Chapter 4. The Assassin: Part 8.|Chapter four. The Assassin: Part eight. +LJ039-0005|Richard M. Nixon Incident|Richard M. Nixon Incident +LJ039-0006|Another alleged threat by Oswald against a public figure involved former Vice President Richard M. Nixon.|Another alleged threat by Oswald against a public figure involved former Vice President Richard M. Nixon. +LJ039-0007|In January 1964, Marina Oswald and her business manager, James Martin,|In January nineteen sixty-four, Marina Oswald and her business manager, James Martin, +LJ039-0008|told Robert Oswald, Lee Harvey Oswald's brother, that Oswald had once threatened to shoot former Vice President Richard M. Nixon.|told Robert Oswald, Lee Harvey Oswald's brother, that Oswald had once threatened to shoot former Vice President Richard M. Nixon. +LJ039-0009|When Marina Oswald testified before the Commission on February 3 to 6, 1964,|When Marina Oswald testified before the Commission on February three to six, nineteen sixty-four, +LJ039-0010|she had failed to mention the incident when she was asked whether Oswald had ever expressed any hostility|she had failed to mention the incident when she was asked whether Oswald had ever expressed any hostility +LJ039-0011|toward any official of the United States.|toward any official of the United States. +LJ039-0012|The Commission first learned of this incident when Robert Oswald related it to FBI agents on February 19, 1964,|The Commission first learned of this incident when Robert Oswald related it to FBI agents on February nineteen, nineteen sixty-four, +LJ039-0013|and to the Commission on February 21.|and to the Commission on February twenty-one. +LJ039-0014|Marina Oswald appeared before the Commission again on June 11, 1964,|Marina Oswald appeared before the Commission again on June eleven, nineteen sixty-four, +LJ039-0015|and testified that a few days before her husband's departure from Dallas to New Orleans on April 24, 1963,|and testified that a few days before her husband's departure from Dallas to New Orleans on April twenty-four, nineteen sixty-three, +LJ039-0016|he finished reading a morning newspaper, quote, and put on a good suit. I saw that he took a pistol.|he finished reading a morning newspaper, quote, and put on a good suit. I saw that he took a pistol. +LJ039-0017|I asked him where he was going, and why he was getting dressed. He answered Nixon is coming. I want to go and have a look, end quote.|I asked him where he was going, and why he was getting dressed. He answered Nixon is coming. I want to go and have a look, end quote. +LJ039-0018|He also said that he would use the pistol if the opportunity arose.|He also said that he would use the pistol if the opportunity arose. +LJ039-0019|She reminded him that after the Walker shooting he had promised never to repeat such an act.|She reminded him that after the Walker shooting he had promised never to repeat such an act. +LJ039-0020|Marina Oswald related the events which followed, quote,|Marina Oswald related the events which followed, quote, +LJ039-0021|I called him into the bathroom and I closed the door and I wanted to prevent him and then I started to cry.|I called him into the bathroom and I closed the door and I wanted to prevent him and then I started to cry. +LJ039-0022|And I told him that he shouldn't do this, and that he had promised me.|And I told him that he shouldn't do this, and that he had promised me. +LJ039-0023|I remember that I held him. We actually struggled for several minutes and then he quieted down. End quote.|I remember that I held him. We actually struggled for several minutes and then he quieted down. End quote. +LJ039-0024|She stated that it was not physical force which kept him from leaving the house. Quote, I couldn't keep him from going out if he really wanted to, end quote.|She stated that it was not physical force which kept him from leaving the house. Quote, I couldn't keep him from going out if he really wanted to, end quote. +LJ039-0025|After further questioning|After further questioning +LJ039-0026|she stated that she might have been confused about shutting him in the bathroom, but that, quote, there is no doubt that he got dressed and got a gun, end quote,|she stated that she might have been confused about shutting him in the bathroom, but that, quote, there is no doubt that he got dressed and got a gun, end quote, +LJ039-0027|Oswald's revolver|Oswald's revolver +LJ039-0028|was shipped from Los Angeles on March 20, 1963, and he left for New Orleans on April 24, 1963.|was shipped from Los Angeles on March twenty, nineteen sixty-three, and he left for New Orleans on April twenty-four, nineteen sixty-three. +LJ039-0029|No edition of either Dallas newspaper during the period January 1, 1963, to May 15, 1963,|No edition of either Dallas newspaper during the period January one, nineteen sixty-three, to May fifteen, nineteen sixty-three, +LJ039-0030|mentioned any proposed visit by Mr. Nixon to Dallas.|mentioned any proposed visit by Mr. Nixon to Dallas. +LJ039-0031|Mr. Nixon advised the Commission that the only time he was in Dallas in 1963|Mr. Nixon advised the Commission that the only time he was in Dallas in nineteen sixty-three +LJ039-0032|was on November 20 to 21, 1963.|was on November twenty to twenty-one, nineteen sixty-three. +LJ039-0033|An investigation failed to reveal any invitation extended to Mr. Nixon during the period when Oswald's threat reportedly occurred.|An investigation failed to reveal any invitation extended to Mr. Nixon during the period when Oswald's threat reportedly occurred. +LJ039-0034|The Commission has concluded, therefore, that regardless of what Oswald may have said to his wife|The Commission has concluded, therefore, that regardless of what Oswald may have said to his wife +LJ039-0035|he was not actually planning to shoot Mr. Nixon at that time in Dallas.|he was not actually planning to shoot Mr. Nixon at that time in Dallas. +LJ039-0036|On April 23, 1963,|On April twenty-three, nineteen sixty-three, +LJ039-0037|Vice President Lyndon B. Johnson was in Dallas for a visit which had been publicized in the Dallas newspapers|Vice President Lyndon B. Johnson was in Dallas for a visit which had been publicized in the Dallas newspapers +LJ039-0038|throughout April. The Commission asked Marina Oswald whether she might have misunderstood the object of her husband's threat. She stated, quote,|throughout April. The Commission asked Marina Oswald whether she might have misunderstood the object of her husband's threat. She stated, quote, +LJ039-0039|there is no question that in this incident it was a question of Mr. Nixon, end quote.|there is no question that in this incident it was a question of Mr. Nixon, end quote. +LJ039-0040|When asked later whether it might have been Mr. Johnson, she said, quote, Yes, no.|When asked later whether it might have been Mr. Johnson, she said, quote, Yes, no. +LJ039-0041|I am getting a little confused with so many questions.|I am getting a little confused with so many questions. +LJ039-0042|I was absolutely convinced it was Nixon and now after all these questions I wonder if I am right in my mind? End quote.|I was absolutely convinced it was Nixon and now after all these questions I wonder if I am right in my mind? End quote. +LJ039-0043|She stated further that Oswald had only mentioned Nixon's name once during the incident. Marina Oswald might have misunderstood her husband.|She stated further that Oswald had only mentioned Nixon's name once during the incident. Marina Oswald might have misunderstood her husband. +LJ039-0044|Mr. Johnson was the then Vice President and his visit took place on April 23rd.|Mr. Johnson was the then Vice President and his visit took place on April twenty-third. +LJ039-0045|This was 1 day before Oswald left for New Orleans and Marina appeared certain that the Nixon incident, quote,|This was one day before Oswald left for New Orleans and Marina appeared certain that the Nixon incident, quote, +LJ039-0046|wasn't the day before. Perhaps 3 days before, end quote.|wasn't the day before. Perhaps three days before, end quote. +LJ039-0047|Marina Oswald speculated that the incident may have been unrelated to an actual threat.|Marina Oswald speculated that the incident may have been unrelated to an actual threat. +LJ039-0048|She said, quote, It might have been that he was just trying to test me.|She said, quote, It might have been that he was just trying to test me. +LJ039-0049|He was the kind of person who could try and wound somebody in that way. Possibly he didn't want to go out at all but was just doing this all as a sort of joke,|He was the kind of person who could try and wound somebody in that way. Possibly he didn't want to go out at all but was just doing this all as a sort of joke, +LJ039-0050|not really as a joke but rather to simply wound me, to make me feel bad, end quote.|not really as a joke but rather to simply wound me, to make me feel bad, end quote. +LJ039-0051|In the absence of other evidence that Oswald actually intended to shoot someone at this time,|In the absence of other evidence that Oswald actually intended to shoot someone at this time, +LJ039-0052|the Commission concluded that the incident, as described by Marina Oswald,|the Commission concluded that the incident, as described by Marina Oswald, +LJ039-0053|was of no probative value in the Commission's decision concerning the identity of the assassin of President Kennedy.|was of no probative value in the Commission's decision concerning the identity of the assassin of President Kennedy. +LJ039-0054|Oswald's Rifle Capability|Oswald's Rifle Capability +LJ039-0055|In deciding whether Lee Harvey Oswald fired the shots which killed President Kennedy and wounded Governor Connally,|In deciding whether Lee Harvey Oswald fired the shots which killed President Kennedy and wounded Governor Connally, +LJ039-0056|the Commission considered whether Oswald, using his own rifle,|the Commission considered whether Oswald, using his own rifle, +LJ039-0057|possessed the capability to hit his target with two out of three shots under the conditions described in chapter 3.|possessed the capability to hit his target with two out of three shots under the conditions described in chapter three. +LJ039-0058|The Commission evaluated (1) the nature of the shots, (2) Oswald's Marine training in marksmanship,|The Commission evaluated (one) the nature of the shots, (two) Oswald's Marine training in marksmanship, +LJ039-0059|(3) his experience and practice after leaving the Marine Corps, and (4) the accuracy of the weapon and the quality of the ammunition.|(three) his experience and practice after leaving the Marine Corps, and (four) the accuracy of the weapon and the quality of the ammunition. +LJ039-0060|The Nature of the Shots|The Nature of the Shots +LJ039-0061|For a rifleman situated on the sixth floor of the Texas School Book Depository Building|For a rifleman situated on the sixth floor of the Texas School Book Depository Building +LJ039-0062|the shots were at a slow-moving target proceeding on a downgrade in virtually a straight line|the shots were at a slow-moving target proceeding on a downgrade in virtually a straight line +LJ039-0063|with the alinement of the assassin's rifle, at a range of 177 to 266 feet.|with the alinement of the assassin's rifle, at a range of one hundred seventy-seven to two hundred sixty-six feet. +LJ039-0064|An aerial photograph of Dealey Plaza shows that Elm Street runs at an angle|An aerial photograph of Dealey Plaza shows that Elm Street runs at an angle +LJ039-0065|so that the President would have been moving in an almost straight line away from the assassin's rifle.|so that the President would have been moving in an almost straight line away from the assassin's rifle. +LJ039-0066|In addition, the 3 degree downward slope of Elm Street was of assistance in eliminating at least some of the adjustment|In addition, the three degree downward slope of Elm Street was of assistance in eliminating at least some of the adjustment +LJ039-0067|which is ordinarily required when a marksman must raise his rifle as a target moves farther away.|which is ordinarily required when a marksman must raise his rifle as a target moves farther away. +LJ039-0068|Four marksmanship experts testified before the Commission.|Four marksmanship experts testified before the Commission. +LJ039-0069|Maj. Eugene D. Anderson, assistant head of the Marksmanship Branch of U.S. Marine Corps|Maj. Eugene D. Anderson, assistant head of the Marksmanship Branch of U.S. Marine Corps +LJ039-0070|testified that the shots which struck the President in the neck and in the head were, quote, not particularly difficult, end quote.|testified that the shots which struck the President in the neck and in the head were, quote, not particularly difficult, end quote. +LJ039-0071|Robert A. Frazier, FBI expert in firearms identification and training, said, quote,|Robert A. Frazier, FBI expert in firearms identification and training, said, quote, +LJ039-0072|From my own experience in shooting over the years,|From my own experience in shooting over the years, +LJ039-0073|when you shoot at 175 feet or 260 feet, which is less than a hundred yards, with a telescopic sight,|when you shoot at one hundred seventy-five feet or two hundred sixty feet, which is less than a hundred yards, with a telescopic sight, +LJ039-0074|you should not have any difficulty in hitting your target. I mean it requires no training at all to shoot a weapon with a telescopic sight|you should not have any difficulty in hitting your target. I mean it requires no training at all to shoot a weapon with a telescopic sight +LJ039-0075|once you know that you must put the crosshairs on the target and that is all that is necessary.|once you know that you must put the crosshairs on the target and that is all that is necessary. +LJ039-0076|Ronald Simmons, chief of the U.S. Army Infantry Weapons Evaluation Branch of the Ballistics Research Laboratory, said, quote,|Ronald Simmons, chief of the U.S. Army Infantry Weapons Evaluation Branch of the Ballistics Research Laboratory, said, quote, +LJ039-0077|Well, in order to achieve three hits, it would not be required that a man be an exceptional shot. A proficient man with this weapon, yes, end quote.|Well, in order to achieve three hits, it would not be required that a man be an exceptional shot. A proficient man with this weapon, yes, end quote. +LJ039-0078|The effect of a four-power telescopic sight on the difficulty of these shots was considered in detail|The effect of a four-power telescopic sight on the difficulty of these shots was considered in detail +LJ039-0079|by Master Sgt. James A. Zahm, noncommissioned officer in charge of the Marksmanship Training Unit in the Weapons Training Battalion|by Master Sgt. James A. Zahm, noncommissioned officer in charge of the Marksmanship Training Unit in the Weapons Training Battalion +LJ039-0080|of the Marine Corps School at Quantico, Virginia.|of the Marine Corps School at Quantico, Virginia. +LJ039-0081|Referring to a rifle with a four-power telescope, Sergeant Zahm said, quote,|Referring to a rifle with a four-power telescope, Sergeant Zahm said, quote, +LJ039-0082|this is the ideal type of weapon for moving targets|this is the ideal type of weapon for moving targets +LJ039-0083|Using the scope, rapidly working a bolt and using the scope to relocate your target quickly and at the same time when you locate that target|Using the scope, rapidly working a bolt and using the scope to relocate your target quickly and at the same time when you locate that target +LJ039-0084|you identify it and the crosshairs are in close relationship to the point you want to shoot at,|you identify it and the crosshairs are in close relationship to the point you want to shoot at, +LJ039-0085|it just takes a minor move in aiming to bring the crosshairs to bear, and then it is a quick squeeze.|it just takes a minor move in aiming to bring the crosshairs to bear, and then it is a quick squeeze. +LJ039-0086|I consider it a real advantage, particularly at the range of 100 yards, in identifying your target.|I consider it a real advantage, particularly at the range of one hundred yards, in identifying your target. +LJ039-0087|It allows you to see your target clearly, and it is still of a minimum amount of power that it doesn't exaggerate your own body movements.|It allows you to see your target clearly, and it is still of a minimum amount of power that it doesn't exaggerate your own body movements. +LJ039-0088|It just is an aid in seeing in the fact that you only have the one element, the crosshair,|It just is an aid in seeing in the fact that you only have the one element, the crosshair, +LJ039-0089|in relation to the target as opposed to iron sights with aligning the sights and then aligning them on the target, end quote.|in relation to the target as opposed to iron sights with aligning the sights and then aligning them on the target, end quote. +LJ039-0090|Characterizing the four-power scope as, quote, a real aid, an extreme aid, end quote, in rapid fire shooting,|Characterizing the four-power scope as, quote, a real aid, an extreme aid, end quote, in rapid fire shooting, +LJ039-0091|Sergeant Zahm expressed the opinion that the shot which struck President Kennedy in the neck at 176.9|Sergeant Zahm expressed the opinion that the shot which struck President Kennedy in the neck at one hundred seventy-six point nine +LJ039-0092|to 190.8 feet was, quote, very easy, end quote, and the shot which struck the President in the head|to one hundred ninety point eight feet was, quote, very easy, end quote, and the shot which struck the President in the head +LJ039-0093|at a distance of 265.3 feet was, quote, an easy shot, end quote.|at a distance of two hundred sixty-five point three feet was, quote, an easy shot, end quote. +LJ039-0094|After viewing photographs depicting the alignment of Elm Street in relation to the Texas School Book Depository Building,|After viewing photographs depicting the alignment of Elm Street in relation to the Texas School Book Depository Building, +LJ039-0095|Zahm stated further, quote,|Zahm stated further, quote, +LJ039-0096|This is a definite advantage to the shooter, the vehicle moving directly away from him and the downgrade of the street, and he being in an elevated position|This is a definite advantage to the shooter, the vehicle moving directly away from him and the downgrade of the street, and he being in an elevated position +LJ039-0097|made an almost stationary target while he was aiming in, very little movement if any, end quote.|made an almost stationary target while he was aiming in, very little movement if any, end quote. +LJ039-0098|Oswald's Marine Training|Oswald's Marine Training +LJ039-0099|In accordance with standard Marine procedures, Oswald received extensive training in marksmanship.|In accordance with standard Marine procedures, Oswald received extensive training in marksmanship. +LJ039-0100|During the first week of an intensive 8-week training period he received instruction in sighting, aiming, and manipulation of the trigger.|During the first week of an intensive eight-week training period he received instruction in sighting, aiming, and manipulation of the trigger. +LJ039-0101|He went through a series of exercises called dry firing where he assumed all positions which would later be used in the qualification course.|He went through a series of exercises called dry firing where he assumed all positions which would later be used in the qualification course. +LJ039-0102|After familiarization with live ammunition in the .22 rifle and the .22 pistol,|After familiarization with live ammunition in the twenty-two rifle and the twenty-two pistol, +LJ039-0104|firing 50 rounds each day for five days.|firing fifty rounds each day for five days. +LJ039-0105|Following that training, Oswald was tested in December of 1956, and obtained a score of 212,|Following that training, Oswald was tested in December of nineteen fifty-six, and obtained a score of two hundred twelve, +LJ039-0106|which was 2 points above the minimum for qualifications as a "sharpshooter" in a scale of marksman, sharpshooter, expert.|which was two points above the minimum for qualifications as a "sharpshooter" in a scale of marksman, sharpshooter, expert. +LJ039-0107|In May of 1959, on another range, Oswald scored 191, which was 1 point over the minimum for ranking as a "marksman."|In May of nineteen fifty-nine, on another range, Oswald scored one hundred ninety-one, which was one point over the minimum for ranking as a "marksman." +LJ039-0108|The Marine Corps records maintained on Oswald further show that he had fired and was familiar with the Browning Automatic rifle,|The Marine Corps records maintained on Oswald further show that he had fired and was familiar with the Browning Automatic rifle, +LJ039-0109|.45 caliber pistol, and 12-gauge riot gun.|forty-five caliber pistol, and twelve-gauge riot gun. +LJ039-0110|Based on the general Marine Corps ratings, Lt. Col. A. G. Folsom, Jr.|Based on the general Marine Corps ratings, Lt. Col. A. G. Folsom, Jr. +LJ039-0111|head, Records Branch, Personnel Department, Headquarters U.S. Marine Corps,|head, Records Branch, Personnel Department, Headquarters U.S. Marine Corps, +LJ039-0112|evaluated the sharpshooter qualification as a, quote, fairly good shot, end quote, and a low marksman rating|evaluated the sharpshooter qualification as a, quote, fairly good shot, end quote, and a low marksman rating +LJ039-0113|as a, quote, rather poor shot, end quote.|as a, quote, rather poor shot, end quote. +LJ039-0114|When asked to explain the different scores achieved by Oswald on the two occasions when he fired for record,|When asked to explain the different scores achieved by Oswald on the two occasions when he fired for record, +LJ039-0115|Major Anderson said, quote, when he fired that two twelve|Major Anderson said, quote, when he fired that two twelve +LJ039-0116|he had just completed a very intensive preliminary training period.|he had just completed a very intensive preliminary training period. +LJ039-0117|He had the services of an experienced highly trained coach.|He had the services of an experienced highly trained coach. +LJ039-0118|He had high motivation. He had presumably a good to excellent rifle and good ammunition.|He had high motivation. He had presumably a good to excellent rifle and good ammunition. +LJ039-0119|We have nothing here to show under what conditions the B course was fired. It might well have been a bad day for firing the rifle|We have nothing here to show under what conditions the B course was fired. It might well have been a bad day for firing the rifle +LJ039-0120|windy, rainy, dark. There is little probability that he had good, expert coach,|windy, rainy, dark. There is little probability that he had good, expert coach, +LJ039-0121|and he probably didn't have as high a motivation because he was no longer in recruit training and under the care of the drill instructor.|and he probably didn't have as high a motivation because he was no longer in recruit training and under the care of the drill instructor. +LJ039-0122|There is some possibility that the rifle he was firing might not have been as good a rifle as the rifle that he was firing in his A course firing,|There is some possibility that the rifle he was firing might not have been as good a rifle as the rifle that he was firing in his A course firing, +LJ039-0123|because he may well have carried this rifle for quite some time, and it got banged around in normal usage.|because he may well have carried this rifle for quite some time, and it got banged around in normal usage. +LJ039-0124|End quote. Major Anderson concluded, quote, I would say that as compared to other Marines receiving the same type of training,|End quote. Major Anderson concluded, quote, I would say that as compared to other Marines receiving the same type of training, +LJ039-0125|that Oswald was a good shot, somewhat better than or equal to -- better than the average let us say.|that Oswald was a good shot, somewhat better than or equal to -- better than the average let us say. +LJ039-0126|As compared to a civilian who had not received this intensive training, he would be considered as a good to excellent shot. End quote.|As compared to a civilian who had not received this intensive training, he would be considered as a good to excellent shot. End quote. +LJ039-0127|When Sergeant Zahm was asked whether Oswald's Marine Corps training would have made it easier to operate a rifle with a four-power scope, he replied, quote,|When Sergeant Zahm was asked whether Oswald's Marine Corps training would have made it easier to operate a rifle with a four-power scope, he replied, quote, +LJ039-0128|Based on that training, his basic knowledge in sight manipulation and trigger squeeze and what not, I would say that he would be capable of sighting that rifle in well,|Based on that training, his basic knowledge in sight manipulation and trigger squeeze and what not, I would say that he would be capable of sighting that rifle in well, +LJ039-0129|firing it, with 10 rounds.|firing it, with ten rounds. +LJ039-0130|After reviewing Oswald's marksmanship scores,|After reviewing Oswald's marksmanship scores, +LJ039-0131|Sergeant Zahm concluded, quote, I would say in the Marine Corps he is a good shot, slightly above average,|Sergeant Zahm concluded, quote, I would say in the Marine Corps he is a good shot, slightly above average, +LJ039-0132|and as compared to the average male of his age throughout the civilian, throughout the United States, that he is an excellent shot, end quote.|and as compared to the average male of his age throughout the civilian, throughout the United States, that he is an excellent shot, end quote. +LJ039-0133|Oswald's Rifle Practice Outside the Marines|Oswald's Rifle Practice Outside the Marines +LJ039-0134|During one of his leaves from the Marines, Oswald hunted with his brother Robert,|During one of his leaves from the Marines, Oswald hunted with his brother Robert, +LJ039-0135|using a .22 caliber bolt-action rifle belonging either to Robert or Robert's in-laws.|using a twenty-two caliber bolt-action rifle belonging either to Robert or Robert's in-laws. +LJ039-0136|After he left the Marines and before departing for Russia, Oswald, his brother, and a third companion went hunting for squirrels and rabbits.|After he left the Marines and before departing for Russia, Oswald, his brother, and a third companion went hunting for squirrels and rabbits. +LJ039-0137|On that occasion Oswald again used a bolt-action .22 caliber rifle; and according to Robert,|On that occasion Oswald again used a bolt-action twenty-two caliber rifle; and according to Robert, +LJ039-0138|Lee Oswald exhibited an average amount of proficiency with that weapon.|Lee Oswald exhibited an average amount of proficiency with that weapon. +LJ039-0139|Oswald obtained a hunting license, joined a hunting club and went hunting about six times, as discussed more fully in chapter 6.|Oswald obtained a hunting license, joined a hunting club and went hunting about six times, as discussed more fully in chapter six. +LJ039-0140|Soon after Oswald returned from the Soviet Union|Soon after Oswald returned from the Soviet Union +LJ039-0141|he again went hunting with his brother, Robert, and used a borrowed .22 caliber bolt-action rifle.|he again went hunting with his brother, Robert, and used a borrowed twenty-two caliber bolt-action rifle. +LJ039-0142|After Oswald purchased the Mannlicher-Carcano rifle, he told his wife that he practiced with it.|After Oswald purchased the Mannlicher-Carcano rifle, he told his wife that he practiced with it. +LJ039-0143|Marina Oswald testified that on one occasion she saw him take the rifle, concealed in a raincoat, from the house on Neely Street.|Marina Oswald testified that on one occasion she saw him take the rifle, concealed in a raincoat, from the house on Neely Street. +LJ039-0144|Oswald told her he was going to practice with it.|Oswald told her he was going to practice with it. +LJ039-0145|According to George De Mohrenschildt, Oswald said that he went target shooting with that rifle.|According to George De Mohrenschildt, Oswald said that he went target shooting with that rifle. +LJ039-0146|Marina Oswald testified that in New Orleans in May of 1963, she observed Oswald sitting with the rifle on their screened porch at night,|Marina Oswald testified that in New Orleans in May of nineteen sixty-three, she observed Oswald sitting with the rifle on their screened porch at night, +LJ039-0147|sighting with the telescopic lens and operating the bolt.|sighting with the telescopic lens and operating the bolt. +LJ039-0148|Examination of the cartridge cases found on the sixth floor of the Depository Building|Examination of the cartridge cases found on the sixth floor of the Depository Building +LJ039-0150|which would indicate that Oswald practiced operating the bolt.|which would indicate that Oswald practiced operating the bolt. +LJ039-0151|Accuracy of Weapon|Accuracy of Weapon +LJ039-0152|It will be recalled from the discussion in chapter 3|It will be recalled from the discussion in chapter three +LJ039-0153|that the assassin in all probability hit two out of the three shots during the maximum time span of|that the assassin in all probability hit two out of the three shots during the maximum time span of +LJ039-0154|4.8 to 5.6 seconds if the second shot missed,|four point eight to five point six seconds if the second shot missed, +LJ039-0155|or, if either the first or third shots missed, the assassin fired the three shots during a minimum time span of 7.1|or, if either the first or third shots missed, the assassin fired the three shots during a minimum time span of seven point one +LJ039-0156|to 7.9 seconds.|to seven point nine seconds. +LJ039-0157|A series of tests were performed to determine whether the weapon and ammunition used in the assassination|A series of tests were performed to determine whether the weapon and ammunition used in the assassination +LJ039-0158|were capable of firing the shots which were fired by the assassin on November 22, 1963.|were capable of firing the shots which were fired by the assassin on November twenty-two, nineteen sixty-three. +LJ039-0159|The ammunition used by the assassin was manufactured by Western Cartridge Co. of East Alton, Illinois.|The ammunition used by the assassin was manufactured by Western Cartridge Co. of East Alton, Illinois. +LJ039-0160|In tests with the Mannlicher-Carcano C2766 rifle, over 100 rounds of this ammunition were fired by the FBI|In tests with the Mannlicher-Carano C twenty-seven sixty-six rifle, over one hundred rounds of this ammunition were fired by the FBI +LJ039-0161|and the Infantry Weapons Evaluation Branch of the U.S. Army. There were no misfires.|and the Infantry Weapons Evaluation Branch of the U.S. Army. There were no misfires. +LJ039-0162|In an effort to test the rifle under conditions which simulated those which prevailed during the assassination,|In an effort to test the rifle under conditions which simulated those which prevailed during the assassination, +LJ039-0163|the Infantry Weapons Evaluation Branch of the Ballistics Research Laboratory had expert riflemen fire the assassination weapon|the Infantry Weapons Evaluation Branch of the Ballistics Research Laboratory had expert riflemen fire the assassination weapon +LJ039-0164|from a tower at three silhouette targets at distances of 175, 240, and 265 feet.|from a tower at three silhouette targets at distances of one hundred seventy-five, two hundred forty, and two hundred sixty-five feet. +LJ039-0165|The target at 265 feet was placed to the right of the 240-foot target|The target at two hundred sixty-five feet was placed to the right of the two hundred forty-foot target +LJ039-0166|which was in turn placed to the right of the closest silhouette.|which was in turn placed to the right of the closest silhouette. +LJ039-0167|Using the assassination rifle mounted with the telescopic sight, three marksmen, rated as master by the National Rifle Association,|Using the assassination rifle mounted with the telescopic sight, three marksmen, rated as master by the National Rifle Association, +LJ039-0168|each fired two series of three shots.|each fired two series of three shots. +LJ039-0169|In the first series the firers required time spans of 4.6, 6.75,|In the first series the firers required time spans of four point six, six point seven five, +LJ039-0170|and 8.25 seconds respectively.|and eight point two five seconds respectively. +LJ039-0171|On the second series they required 5.15, 6.45, and 7 seconds.|On the second series they required five point one five, six point four five, and seven seconds. +LJ039-0172|None of the marksmen had any practice with the assassination weapon except for exercising the bolt for 2 or 3 minutes on a dry run.|None of the marksmen had any practice with the assassination weapon except for exercising the bolt for two or three minutes on a dry run. +LJ039-0173|They had not even pulled the trigger because of concern about breaking the firing pin.|They had not even pulled the trigger because of concern about breaking the firing pin. +LJ039-0174|The marksmen took as much time as they wanted for the first target and all hit the target.|The marksmen took as much time as they wanted for the first target and all hit the target. +LJ039-0175|For the first four attempts, the firers missed the second shot by several inches.|For the first four attempts, the firers missed the second shot by several inches. +LJ039-0176|The angle from the first to the second shot|The angle from the first to the second shot +LJ039-0177|was greater than from the second to the third shot and required a movement in the basic firing position of the marksmen.|was greater than from the second to the third shot and required a movement in the basic firing position of the marksmen. +LJ039-0178|This angle was used in the test because the majority of the eyewitnesses to the assassination|This angle was used in the test because the majority of the eyewitnesses to the assassination +LJ039-0179|stated that there was a shorter interval between shots two and three than between shots one and two.|stated that there was a shorter interval between shots two and three than between shots one and two. +LJ039-0180|As has been shown in chapter 3, if the three shots were fired within a period of from 4.8 to 5.6 seconds,|As has been shown in chapter three, if the three shots were fired within a period of from four point eight to five point six seconds, +LJ039-0181|the shots would have been evenly spaced and the assassin would not have incurred so sharp an angular movement.|the shots would have been evenly spaced and the assassin would not have incurred so sharp an angular movement. +LJ039-0182|Five of the six shots hit the third target where the angle of movement of the weapon was small.|Five of the six shots hit the third target where the angle of movement of the weapon was small. +LJ039-0183|On the basis of these results, Simmons testified that in his opinion the probability of hitting the targets at the relatively short range at which they were hit|On the basis of these results, Simmons testified that in his opinion the probability of hitting the targets at the relatively short range at which they were hit +LJ039-0184|Considering the various probabilities which may have prevailed during the actual assassination,|Considering the various probabilities which may have prevailed during the actual assassination, +LJ039-0185|the highest level of firing performance which would have been required of the assassin and the C2766 rifle|the highest level of firing performance which would have been required of the assassin and the C two seven six six rifle +LJ039-0186|would have been to fire three times and hit the target twice within a span of 4.8 to 5.6 seconds.|would have been to fire three times and hit the target twice within a span of four point eight to five point six seconds. +LJ039-0187|In fact, one of the firers in the rapid fire test in firing his two series of three shots,|In fact, one of the firers in the rapid fire test in firing his two series of three shots, +LJ039-0188|hit the target twice within a span of 4.6 and 5.15 seconds.|hit the target twice within a span of four point six and five point one five seconds. +LJ039-0189|The others would have been able to reduce their times if they had been given the opportunity to become familiar with the movement of the bolt and the trigger pull.|The others would have been able to reduce their times if they had been given the opportunity to become familiar with the movement of the bolt and the trigger pull. +LJ039-0190|Simmons testified that familiarity with the bolt could be achieved in dry practice and, as has been indicated above,|Simmons testified that familiarity with the bolt could be achieved in dry practice and, as has been indicated above, +LJ039-0191|Oswald engaged in such practice. If the assassin missed either the first or third shot,|Oswald engaged in such practice. If the assassin missed either the first or third shot, +LJ039-0192|he had a total of between 4.8 and 5.6 seconds between the two shots which hit|he had a total of between four point eight and five point six seconds between the two shots which hit +LJ039-0193|and a total minimum time period of from 7.1 to 7.9 seconds for all three shots.|and a total minimum time period of from seven point one to seven point nine seconds for all three shots. +LJ039-0194|All three of the firers in these tests|All three of the firers in these tests +LJ039-0195|were able to fire the rounds within the time period which would have been available to the assassin under those conditions.|were able to fire the rounds within the time period which would have been available to the assassin under those conditions. +LJ039-0196|Three FBI firearms experts tested the rifle in order to determine the speed with which it could be fired.|Three FBI firearms experts tested the rifle in order to determine the speed with which it could be fired. +LJ039-0197|The purpose of this experiment was not to test the rifle under conditions which prevailed at the time of the assassination|The purpose of this experiment was not to test the rifle under conditions which prevailed at the time of the assassination +LJ039-0198|but to determine the maximum speed at which it could be fired.|but to determine the maximum speed at which it could be fired. +LJ039-0199|The three FBI experts each fired three shots from the weapon at 15 yards in 6, 7, and 9 seconds,|The three FBI experts each fired three shots from the weapon at fifteen yards in six, seven, and nine seconds, +LJ039-0200|and one of these agents, Robert A. Frazier,|and one of these agents, Robert A. Frazier, +LJ039-0201|fired two series of three shots at 25 yards in 4.6 and 4.8 seconds.|fired two series of three shots at twenty-five yards in four point six and four point eight seconds. +LJ039-0202|At 15 yards each man's shots landed within the size of a dime.|At fifteen yards each man's shots landed within the size of a dime. +LJ039-0203|The shots fired by Frazier at the range of 25 yards landed within an area of 2 inches and 5 inches respectively.|The shots fired by Frazier at the range of twenty-five yards landed within an area of two inches and five inches respectively. +LJ039-0204|Frazier later fired four groups of three shots at a distance of 100 yards in 5.9, 6.2,|Frazier later fired four groups of three shots at a distance of one hundred yards in five point nine, six point two, +LJ039-0205|5.6, and 6.5 seconds.|five point six, and six point five seconds. +LJ039-0206|Each series of three shots landed within areas ranging in diameter from 3 to 5 inches.|Each series of three shots landed within areas ranging in diameter from three to five inches. +LJ039-0207|Although all of the shots were a few inches high and to the right of the target,|Although all of the shots were a few inches high and to the right of the target, +LJ039-0208|this was because of a defect in the scope which was recognized by the FBI agents and which they could have compensated for if they were aiming to hit a bull's-eye.|this was because of a defect in the scope which was recognized by the FBI agents and which they could have compensated for if they were aiming to hit a bull's-eye. +LJ039-0209|They were instead firing to determine how rapidly the weapon could be fired and the area within which three shots could be placed.|They were instead firing to determine how rapidly the weapon could be fired and the area within which three shots could be placed. +LJ039-0210|Frazier testified that while he could not tell when the defect occurred, but that a person familiar with the weapon could compensate for it.|Frazier testified that while he could not tell when the defect occurred, but that a person familiar with the weapon could compensate for it. +LJ039-0211|Moreover, the defect was one which would have assisted the assassin aiming at a target which was moving away.|Moreover, the defect was one which would have assisted the assassin aiming at a target which was moving away. +LJ039-0212|Frazier said, quote, The fact that the crosshairs are set high would actually compensate for any lead which had to be taken|Frazier said, quote, The fact that the crosshairs are set high would actually compensate for any lead which had to be taken +LJ039-0213|So that if you aimed with this weapon as it actually was received at the laboratory, it would not be necessary to take any lead whatsoever|So that if you aimed with this weapon as it actually was received at the laboratory, it would not be necessary to take any lead whatsoever +LJ039-0214|in order to hit the intended object. The scope would accomplish the lead for you, end quote,|in order to hit the intended object. The scope would accomplish the lead for you, end quote, +LJ039-0215|Frazier added that the scope would cause a slight miss to the right.|Frazier added that the scope would cause a slight miss to the right. +LJ039-0216|It should be noted, however, that the President's car was curving slightly to the right when the third shot was fired.|It should be noted, however, that the President's car was curving slightly to the right when the third shot was fired. +LJ039-0217|Based on these tests the experts agreed that the assassination rifle was an accurate weapon.|Based on these tests the experts agreed that the assassination rifle was an accurate weapon. +LJ039-0218|Simmons described it as, quote, quite accurate, end quote, in fact, as accurate as current military rifles.|Simmons described it as, quote, quite accurate, end quote, in fact, as accurate as current military rifles. +LJ039-0219|Frazier testified that the rifle was accurate, that it had less recoil than the average military rifle|Frazier testified that the rifle was accurate, that it had less recoil than the average military rifle +LJ039-0220|and that one would not have to be an expert marksman to have accomplished the assassination with the weapon which was used.|and that one would not have to be an expert marksman to have accomplished the assassination with the weapon which was used. +LJ039-0221|The various tests showed that the Mannlicher-Carcano was an accurate rifle and that the use of a four-power scope|The various tests showed that the Mannlicher-Carcano was an accurate rifle and that the use of a four-power scope +LJ039-0222|was a substantial aid to rapid, accurate firing.|was a substantial aid to rapid, accurate firing. +LJ039-0223|Oswald's Marine training in marksmanship, his other rifle experience and his established familiarity with this particular weapon|Oswald's Marine training in marksmanship, his other rifle experience and his established familiarity with this particular weapon +LJ039-0224|show that he possessed ample capability to commit the assassination.|show that he possessed ample capability to commit the assassination. +LJ039-0225|Based on the known facts of the assassination,|Based on the known facts of the assassination, +LJ039-0226|the Marine marksmanship experts, Major Anderson and Sergeant Zahm, concurred in the opinion that Oswald had the capability to fire three shots,|the Marine marksmanship experts, Major Anderson and Sergeant Zahm, concurred in the opinion that Oswald had the capability to fire three shots, +LJ039-0227|with two hits, within 4.8 and 5.6 seconds.|with two hits, within four point eight and five point six seconds. +LJ039-0228|Concerning the shots which struck the President in the back of the neck,|Concerning the shots which struck the President in the back of the neck, +LJ039-0229|Sergeant Zahm testified, quote,|Sergeant Zahm testified, quote, +LJ039-0230|With the equipment he [Oswald] had and with his ability I consider it a very easy shot, end quote.|With the equipment he [Oswald] had and with his ability I consider it a very easy shot, end quote. +LJ039-0231|Having fired this slot|Having fired this slot +LJ039-0232|the assassin was then required to hit the target one more time within a space of from 4.8 to 5.6 seconds.|the assassin was then required to hit the target one more time within a space of from four point eight to five point six seconds. +LJ039-0233|On the basis of Oswald's training and the accuracy of the weapon as established by the tests,|On the basis of Oswald's training and the accuracy of the weapon as established by the tests, +LJ039-0234|the Commission concluded that Oswald was capable of accomplishing this second hit even if there was an intervening shot which missed.|the Commission concluded that Oswald was capable of accomplishing this second hit even if there was an intervening shot which missed. +LJ039-0235|The probability of hitting the President a second time would have been markedly increased if, in fact, he had missed either the first or third shots|The probability of hitting the President a second time would have been markedly increased if, in fact, he had missed either the first or third shots +LJ039-0236|thereby leaving a time span of 4.8 to 5.6 seconds between the two shots which struck their mark.|thereby leaving a time span of four point eight to five point six seconds between the two shots which struck their mark. +LJ039-0237|The Commission agrees with the testimony of Marine marksmanship expert Zahm|The Commission agrees with the testimony of Marine marksmanship expert Zahm +LJ039-0238|that it was an easy shot to hit some part of the President's body, and that the range where the rifleman would be expected to hit|that it was an easy shot to hit some part of the President's body, and that the range where the rifleman would be expected to hit +LJ039-0239|would include the President's head.|would include the President's head. +LJ039-0240|On the basis of the evidence reviewed in this chapter, the Commission has found that Lee Harvey Oswald (1)|On the basis of the evidence reviewed in this chapter, the Commission has found that Lee Harvey Oswald (one) +LJ039-0241|owned and possessed the rifle used to kill President Kennedy and wound Governor Connally,|owned and possessed the rifle used to kill President Kennedy and wound Governor Connally, +LJ039-0242|(2) brought this rifle into the Depository Building on the morning of the assassination,|(two) brought this rifle into the Depository Building on the morning of the assassination, +LJ039-0243|(3) was present, at the time of the assassination, at the window from which the shots were fired|(three) was present, at the time of the assassination, at the window from which the shots were fired +LJ039-0244|killed Dallas Police Officer J. D. Tippit in an apparent attempt to escape,|killed Dallas Police Officer J. D. Tippit in an apparent attempt to escape, +LJ039-0245|(5) resisted arrest by drawing a fully loaded pistol and attempting to shoot another police officer,|(five) resisted arrest by drawing a fully loaded pistol and attempting to shoot another police officer, +LJ039-0246|(6) lied to the police after his arrest concerning important substantive matters,|(six) lied to the police after his arrest concerning important substantive matters, +LJ039-0247|(7) attempted, in April 1963, to kill Maj. Gen. Edwin A. Walker, and (8)|(seven) attempted, in April nineteen sixty-three, to kill Maj. Gen. Edwin A. Walker, and (eight) +LJ039-0248|possessed the capability with a rifle which would have enabled him to commit the assassination.|possessed the capability with a rifle which would have enabled him to commit the assassination. +LJ040-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ040-0002|Chapter 7. Lee Harvey Oswald: Background and Possible Motives, Part 1.|Chapter seven. Lee Harvey Oswald: Background and Possible Motives, Part one. +LJ040-0003|The evidence reviewed above identifies Lee Harvey Oswald as the assassin of President Kennedy and indicates that he acted alone in that event.|The evidence reviewed above identifies Lee Harvey Oswald as the assassin of President Kennedy and indicates that he acted alone in that event. +LJ040-0004|There is no evidence that he had accomplices or that he was involved in any conspiracy directed to the assassination of the President.|There is no evidence that he had accomplices or that he was involved in any conspiracy directed to the assassination of the President. +LJ040-0005|There remains the question of what impelled Oswald to conceive and to carry out the assassination of the President of the United States.|There remains the question of what impelled Oswald to conceive and to carry out the assassination of the President of the United States. +LJ040-0006|The Commission has considered many possible motives for the assassination, including those which might flow from Oswald's commitment to Marxism or communism,|The Commission has considered many possible motives for the assassination, including those which might flow from Oswald's commitment to Marxism or communism, +LJ040-0007|the existence of some personal grievance,|the existence of some personal grievance, +LJ040-0008|a desire to effect changes in the structure of society or simply to go down in history as a well publicized assassin.|a desire to effect changes in the structure of society or simply to go down in history as a well publicized assassin. +LJ040-0009|None of these possibilities satisfactorily explains Oswald's act if it is judged by the standards of reasonable men.|None of these possibilities satisfactorily explains Oswald's act if it is judged by the standards of reasonable men. +LJ040-0010|The motives of any man, however, must be analyzed in terms of the character and state of mind of the particular individual involved.|The motives of any man, however, must be analyzed in terms of the character and state of mind of the particular individual involved. +LJ040-0011|For a motive that appears incomprehensible to other men may be the moving force of a man whose view of the world has been twisted,|For a motive that appears incomprehensible to other men may be the moving force of a man whose view of the world has been twisted, +LJ040-0012|possibly by factors of which those around him were only dimly aware.|possibly by factors of which those around him were only dimly aware. +LJ040-0013|Oswald's complete state of mind and character are now outside of the power of man to know.|Oswald's complete state of mind and character are now outside of the power of man to know. +LJ040-0014|He cannot, of course, be questioned or observed by those charged with the responsibility for this report or by experts on their behalf.|He cannot, of course, be questioned or observed by those charged with the responsibility for this report or by experts on their behalf. +LJ040-0015|There is, however, a large amount of material available in his writings|There is, however, a large amount of material available in his writings +LJ040-0016|and in the history of his life which does give some insight into his character and, possibly, into the motives for his act.|and in the history of his life which does give some insight into his character and, possibly, into the motives for his act. +LJ040-0017|Since Oswald is dead,|Since Oswald is dead, +LJ040-0018|the Commission is not able to reach any definite conclusions as to whether or not he was, quote, sane, unquote, under prevailing legal standards.|the Commission is not able to reach any definite conclusions as to whether or not he was, quote, sane, unquote, under prevailing legal standards. +LJ040-0019|Under our system of justice no forum could properly make that determination unless Oswald were before it.|Under our system of justice no forum could properly make that determination unless Oswald were before it. +LJ040-0020|It certainly could not be made by this Commission which, as has been pointed out above,|It certainly could not be made by this Commission which, as has been pointed out above, +LJ040-0021|ascertained the facts surrounding the assassination but did not draw conclusions concerning Oswald's legal guilt.|ascertained the facts surrounding the assassination but did not draw conclusions concerning Oswald's legal guilt. +LJ040-0022|Indications of Oswald's motivation|Indications of Oswald's motivation +LJ040-0023|may be obtained from a study of the events, relationships and influences which appear to have been significant in shaping his character and in guiding him.|may be obtained from a study of the events, relationships and influences which appear to have been significant in shaping his character and in guiding him. +LJ040-0024|Perhaps the most outstanding conclusion of such a study is that Oswald was profoundly alienated from the world in which he lived.|Perhaps the most outstanding conclusion of such a study is that Oswald was profoundly alienated from the world in which he lived. +LJ040-0025|His life was characterized by isolation, frustration, and failure.|His life was characterized by isolation, frustration, and failure. +LJ040-0026|He had very few, if any, close relationships with other people and he appeared to have great difficulty in finding a meaningful place in the world.|He had very few, if any, close relationships with other people and he appeared to have great difficulty in finding a meaningful place in the world. +LJ040-0027|He was never satisfied with anything.|He was never satisfied with anything. +LJ040-0028|When he was in the United States he resented the capitalist system which he thought was exploiting him and others like him.|When he was in the United States he resented the capitalist system which he thought was exploiting him and others like him. +LJ040-0029|He seemed to prefer the Soviet Union and he spoke highly of Cuba.|He seemed to prefer the Soviet Union and he spoke highly of Cuba. +LJ040-0030|When he was in the Soviet Union, he apparently resented the Communist Party members,|When he was in the Soviet Union, he apparently resented the Communist Party members, +LJ040-0031|who were accorded special privileges and who he thought were betraying communism, and he spoke well of the United States.|who were accorded special privileges and who he thought were betraying communism, and he spoke well of the United States. +LJ040-0032|He accused his wife of preferring others to himself and told her to return to the Soviet Union without him but without a divorce.|He accused his wife of preferring others to himself and told her to return to the Soviet Union without him but without a divorce. +LJ040-0033|At the same time he professed his love for her and said that he could not get along without her.|At the same time he professed his love for her and said that he could not get along without her. +LJ040-0034|Marina Oswald thought that he would not be happy anywhere, quote, Only on the moon, perhaps, end quote.|Marina Oswald thought that he would not be happy anywhere, quote, Only on the moon, perhaps, end quote. +LJ040-0035|While Oswald appeared to most of those who knew him as a meek and harmless person, he sometimes imagined himself as, quote, the Commander, end quote,|While Oswald appeared to most of those who knew him as a meek and harmless person, he sometimes imagined himself as, quote, the Commander, end quote, +LJ040-0036|and, apparently seriously, as a political prophet -- a man who said that after 20 years he would be prime minister.|and, apparently seriously, as a political prophet -- a man who said that after twenty years he would be prime minister. +LJ040-0037|His wife testified that he compared himself with great readers of history.|His wife testified that he compared himself with great readers of history. +LJ040-0038|Such ideas of grandeur were apparently accompanied by notions of oppression.|Such ideas of grandeur were apparently accompanied by notions of oppression. +LJ040-0039|He had a great hostility toward his environment, whatever it happened to be,|He had a great hostility toward his environment, whatever it happened to be, +LJ040-0040|which he expressed in striking and sometimes violent acts long before the assassination.|which he expressed in striking and sometimes violent acts long before the assassination. +LJ040-0041|There was some quality about him that led him to act with an apparent disregard for possible consequences.|There was some quality about him that led him to act with an apparent disregard for possible consequences. +LJ040-0042|He defected to the Soviet Union, shot at General Walker, tried to go to Cuba and even contemplated hijacking an airplane to get there.|He defected to the Soviet Union, shot at General Walker, tried to go to Cuba and even contemplated hijacking an airplane to get there. +LJ040-0043|He assassinated the President, shot Officer Tippit, resisted arrest and tried to kill another policeman in the process.|He assassinated the President, shot Officer Tippit, resisted arrest and tried to kill another policeman in the process. +LJ040-0044|Oswald apparently started reading about communism when he was about 15.|Oswald apparently started reading about communism when he was about fifteen. +LJ040-0045|In the Marines, he evidenced a strong conviction as to the correctness of Marxist doctrine,|In the Marines, he evidenced a strong conviction as to the correctness of Marxist doctrine, +LJ040-0046|which one associate described as, quote, irrevocable, end quote, but also as, quote, theoretical, end quote,|which one associate described as, quote, irrevocable, end quote, but also as, quote, theoretical, end quote, +LJ040-0047|That associate did not think that Oswald was a Communist.|That associate did not think that Oswald was a Communist. +LJ040-0048|Oswald did not always distinguish between Marxism and communism.|Oswald did not always distinguish between Marxism and communism. +LJ040-0049|He stated several times that he was a Communist but apparently never joined any Communist Party.|He stated several times that he was a Communist but apparently never joined any Communist Party. +LJ040-0050|His attachment to Marxist and Communist doctrine was probably, in some measure, an expression of his hostility to his environment.|His attachment to Marxist and Communist doctrine was probably, in some measure, an expression of his hostility to his environment. +LJ040-0051|While there is doubt about how fully Oswald understood the doctrine which he so often espoused, it seems clear|While there is doubt about how fully Oswald understood the doctrine which he so often espoused, it seems clear +LJ040-0052|that his commitment to Marxism was an important factor influencing his conduct during his adult years.|that his commitment to Marxism was an important factor influencing his conduct during his adult years. +LJ040-0053|It was an obvious element in his decision to go to Russia and later to Cuba and it probably influenced his decision to shoot at General Walker.|It was an obvious element in his decision to go to Russia and later to Cuba and it probably influenced his decision to shoot at General Walker. +LJ040-0054|It was a factor which contributed to his character and thereby might have influenced his decision to assassinate President Kennedy.|It was a factor which contributed to his character and thereby might have influenced his decision to assassinate President Kennedy. +LJ040-0055|The discussion below will describe the events known to the Commission which most clearly reveals the formation and nature of Oswald's character.|The discussion below will describe the events known to the Commission which most clearly reveals the formation and nature of Oswald's character. +LJ040-0056|It will attempt to summarize the events of his early life, his experience in New York City and in the Marine Corps, and his interest in Marxism.|It will attempt to summarize the events of his early life, his experience in New York City and in the Marine Corps, and his interest in Marxism. +LJ040-0057|It will examine his defection to the Soviet Union in 1959, his subsequent return to the United States and his life here|It will examine his defection to the Soviet Union in nineteen fifty-nine, his subsequent return to the United States and his life here +LJ040-0058|after June of 1962.|after June of nineteen sixty-two. +LJ040-0059|The review of the latter period will evaluate his personal and employment relations, his attempt to kill General Walker, his political activities,|The review of the latter period will evaluate his personal and employment relations, his attempt to kill General Walker, his political activities, +LJ040-0060|and his unsuccessful attempt to go to Cuba in late September of 1963.|and his unsuccessful attempt to go to Cuba in late September of nineteen sixty-three. +LJ040-0061|Various possible motives will be treated in the appropriate context of the discussion outlined above.|Various possible motives will be treated in the appropriate context of the discussion outlined above. +LJ040-0062|The Early Years|The Early Years +LJ040-0063|Significant in shaping the character of Lee Harvey Oswald was the death of his father, a collector of insurance premiums.|Significant in shaping the character of Lee Harvey Oswald was the death of his father, a collector of insurance premiums. +LJ040-0064|This occurred 2 months before Lee was born in New Orleans on October 18, 1939.|This occurred two months before Lee was born in New Orleans on October eighteen, nineteen thirty-nine. +LJ040-0065|That death strained the financial fortunes of the remainder of the Oswald family.|That death strained the financial fortunes of the remainder of the Oswald family. +LJ040-0066|It had its effect on Lee's mother, Marguerite, his brother Robert, who had been born in 1934,|It had its effect on Lee's mother, Marguerite, his brother Robert, who had been born in nineteen thirty-four, +LJ040-0067|and his half-brother John Pic, who had been born in 1932 during Marguerite's previous marriage.|and his half-brother John Pic, who had been born in nineteen thirty-two during Marguerite's previous marriage. +LJ040-0068|It forced Marguerite Oswald to go to work to provide for her family.|It forced Marguerite Oswald to go to work to provide for her family. +LJ040-0069|Reminding her sons that they were orphans and that the family's financial condition was poor,|Reminding her sons that they were orphans and that the family's financial condition was poor, +LJ040-0070|she placed John Pic and Robert Oswald in an orphans' home.|she placed John Pic and Robert Oswald in an orphans' home. +LJ040-0071|From the time Marguerite Oswald returned to work until December 26, 1942, when Lee too was sent to the orphans' home,|From the time Marguerite Oswald returned to work until December twenty-six, nineteen forty-two, when Lee too was sent to the orphans' home, +LJ040-0072|he was cared for principally by his mother's sister, by babysitters and by his mother, when she had time for him.|he was cared for principally by his mother's sister, by babysitters and by his mother, when she had time for him. +LJ040-0073|Marguerite Oswald withdrew Lee from the orphans' home and took him with her to Dallas when he was a little over 4 years old.|Marguerite Oswald withdrew Lee from the orphans' home and took him with her to Dallas when he was a little over four years old. +LJ040-0074|About 6 months later she also withdrew John Pic and Robert Oswald.|About six months later she also withdrew John Pic and Robert Oswald. +LJ040-0075|Apparently that action was taken in anticipation of her marriage to Edwin A. Ekdahl, which took place in May of 1945.|Apparently that action was taken in anticipation of her marriage to Edwin A. Ekdahl, which took place in May of nineteen forty-five. +LJ040-0076|In the fall of that year John Pic and Robert Oswald went to a military academy|In the fall of that year John Pic and Robert Oswald went to a military academy +LJ040-0077|where they stayed, except for vacations, until the spring of 1948.|where they stayed, except for vacations, until the spring of nineteen forty-eight. +LJ040-0078|Lee Oswald remained with his mother and Ekdahl, to whom he became quite attached.|Lee Oswald remained with his mother and Ekdahl, to whom he became quite attached. +LJ040-0079|John Pic testified that he thought Lee found in Ekdahl the father that he never had.|John Pic testified that he thought Lee found in Ekdahl the father that he never had. +LJ040-0080|That situation, however, was short-lived,|That situation, however, was short-lived, +LJ040-0081|for the relations between Marguerite Oswald and Ekdahl were stormy and they were finally divorced, after several separations and reunions,|for the relations between Marguerite Oswald and Ekdahl were stormy and they were finally divorced, after several separations and reunions, +LJ040-0082|in the summer of 1948.|in the summer of nineteen forty-eight. +LJ040-0083|After the divorce Mrs. Oswald complained considerably about how unfairly she was treated,|After the divorce Mrs. Oswald complained considerably about how unfairly she was treated, +LJ040-0084|dwelling on the fact that she was a widow with three children.|dwelling on the fact that she was a widow with three children. +LJ040-0085|John Pic, however, did not think her position was worse than that of many other people.|John Pic, however, did not think her position was worse than that of many other people. +LJ040-0086|In the fall of 1948 she told John Pic and Robert Oswald|In the fall of nineteen forty-eight she told John Pic and Robert Oswald +LJ040-0087|that she could not afford to send them back to the military school and she asked Pic to quit school entirely to help support the family.|that she could not afford to send them back to the military school and she asked Pic to quit school entirely to help support the family. +LJ040-0088|which he did for 4 months in the fall of 1948.|which he did for four months in the fall of nineteen forty-eight. +LJ040-0089|In order to supplement their income further she falsely swore that Pic was 17 years old so that he could join the Marine Corps Reserves.|In order to supplement their income further she falsely swore that Pic was seventeen years old so that he could join the Marine Corps Reserves. +LJ040-0090|Pic did turn over part of his income to his mother,|Pic did turn over part of his income to his mother, +LJ040-0091|but he returned to high school in January of 1949, where he stayed until 3 days before he was scheduled to graduate,|but he returned to high school in January of nineteen forty-nine, where he stayed until three days before he was scheduled to graduate, +LJ040-0092|when he left school in order to get into the Coast Guard. Since his mother did not approve of his decision to continue school|when he left school in order to get into the Coast Guard. Since his mother did not approve of his decision to continue school +LJ040-0093|he accepted the responsibility for that decision himself and signed his mother's name to all his own excuses and report cards.|he accepted the responsibility for that decision himself and signed his mother's name to all his own excuses and report cards. +LJ040-0094|Pic thought that his mother overstated her financial problems and was unduly concerned about money.|Pic thought that his mother overstated her financial problems and was unduly concerned about money. +LJ040-0095|Referring to the period after the divorce from Ekdahl, which was apparently caused in part by Marguerite's desire to get more money from him,|Referring to the period after the divorce from Ekdahl, which was apparently caused in part by Marguerite's desire to get more money from him, +LJ040-0096|Pic said, quote,|Pic said, quote, +LJ040-0097|Lee was brought up in this atmosphere of constant money problems, and I am sure it had quite an effect on him, and also Robert, end quote.|Lee was brought up in this atmosphere of constant money problems, and I am sure it had quite an effect on him, and also Robert, end quote. +LJ040-0098|Marguerite Oswald worked in miscellaneous jobs after her divorce from Ekdahl.|Marguerite Oswald worked in miscellaneous jobs after her divorce from Ekdahl. +LJ040-0099|When she worked for a time as an insurance saleslady,|When she worked for a time as an insurance saleslady, +LJ040-0100|she would sometimes take Lee with her, apparently leaving him alone in the car while she transacted her business.|she would sometimes take Lee with her, apparently leaving him alone in the car while she transacted her business. +LJ040-0101|When she worked during the school year, Lee had to leave an empty house in the morning,|When she worked during the school year, Lee had to leave an empty house in the morning, +LJ040-0102|return to it for lunch and then again at night, his mother having trained him to do that rather than to play with other children.|return to it for lunch and then again at night, his mother having trained him to do that rather than to play with other children. +LJ040-0103|An indication of the nature of Lee's character at this time was provided in the spring of 1950,|An indication of the nature of Lee's character at this time was provided in the spring of nineteen fifty, +LJ040-0104|when he was sent to New Orleans to visit the family of his mother's sister, Mrs. Lillian Murret, for 2 or 3 weeks.|when he was sent to New Orleans to visit the family of his mother's sister, Mrs. Lillian Murret, for two or three weeks. +LJ040-0105|Despite their urgings, he refused to play with the other children his own age.|Despite their urgings, he refused to play with the other children his own age. +LJ040-0106|It also appears that Lee tried to tag along with his older brothers|It also appears that Lee tried to tag along with his older brothers +LJ040-0108|which became more significant as the children grew older.|which became more significant as the children grew older. +LJ040-0109|Whatever problems may have been created by Lee's home life in Louisiana and Texas,|Whatever problems may have been created by Lee's home life in Louisiana and Texas, +LJ040-0110|he apparently adjusted well enough there to have had an average, although gradually deteriorating, school record|he apparently adjusted well enough there to have had an average, although gradually deteriorating, school record +LJ040-0111|with no behavior or truancy problems.|with no behavior or truancy problems. +LJ040-0112|That was not the case, however, after he and his mother moved to New York in August of 1952, shortly before Lee's 13th birthday.|That was not the case, however, after he and his mother moved to New York in August of nineteen fifty-two, shortly before Lee's thirteenth birthday. +LJ040-0113|They moved shortly after Robert joined the Marines; they lived for a time with John Pic who was stationed there with the Coast Guard.|They moved shortly after Robert joined the Marines; they lived for a time with John Pic who was stationed there with the Coast Guard. +LJ040-0114|Relations soon became strained, however, so in late September Lee and his mother moved to their own apartment in the Bronx.|Relations soon became strained, however, so in late September Lee and his mother moved to their own apartment in the Bronx. +LJ040-0115|Pic and his wife would have been happy to have kept Lee, however,|Pic and his wife would have been happy to have kept Lee, however, +LJ040-0116|who was becoming quite a disciplinary problem for his mother, having struck her on at least one occasion.|who was becoming quite a disciplinary problem for his mother, having struck her on at least one occasion. +LJ040-0117|The short-lived stay with the Pics was terminated after an incident in which Lee allegedly pulled out a pocket knife during an argument|The short-lived stay with the Pics was terminated after an incident in which Lee allegedly pulled out a pocket knife during an argument +LJ040-0118|and threatened to use it on Mrs. Pic.|and threatened to use it on Mrs. Pic. +LJ040-0120|Lee refused to discuss the matter with Pic, whom he had previously idolized, and their relations were strained thereafter.|Lee refused to discuss the matter with Pic, whom he had previously idolized, and their relations were strained thereafter. +LJ040-0121|On September 30, 1952, Lee enrolled in P.S. 117|On September thirty, nineteen fifty-two, Lee enrolled in P.S. one seventeen +LJ040-0122|a junior high school in the Bronx, where the other children apparently teased him because of his, quote, western clothes and Texas accent.|a junior high school in the Bronx, where the other children apparently teased him because of his, quote, western clothes and Texas accent. +LJ040-0123|He began to stay away from school, preferring to read magazines and watch television at home by himself.|He began to stay away from school, preferring to read magazines and watch television at home by himself. +LJ040-0124|This continued despite the efforts of the school authorities and, to a lesser extent, of his mother to have him return to school.|This continued despite the efforts of the school authorities and, to a lesser extent, of his mother to have him return to school. +LJ040-0125|Truancy charges were brought against him alleging that he was, quote, beyond the control of his mother insofar as school attendance is concerned, end quote.|Truancy charges were brought against him alleging that he was, quote, beyond the control of his mother insofar as school attendance is concerned, end quote. +LJ040-0126|Oswald was remanded for psychiatric observation to Youth House, an institution in which children are kept for psychiatric observation|Oswald was remanded for psychiatric observation to Youth House, an institution in which children are kept for psychiatric observation +LJ040-0127|or for detention pending court appearance or commitment to a child-caring or custodial institution such as a training school.|or for detention pending court appearance or commitment to a child-caring or custodial institution such as a training school. +LJ040-0128|He was in Youth House from April 16 to May 7, 1953,|He was in Youth House from April sixteen to May seven, nineteen fifty-three, +LJ040-0129|during which time he was examined by its Chief Psychiatrist, Dr. Renatus Hartogs,|during which time he was examined by its Chief Psychiatrist, Dr. Renatus Hartogs, +LJ040-0130|and interviewed and observed by other members of the Youth House staff.|and interviewed and observed by other members of the Youth House staff. +LJ040-0131|Marguerite Oswald visited her son at Youth House, where she recalled that she waited in line, quote,|Marguerite Oswald visited her son at Youth House, where she recalled that she waited in line, quote, +LJ040-0132|with Puerto Ricans and Negroes and everything, end quote.|with Puerto Ricans and Negroes and everything, end quote. +LJ040-0133|She said that her pocketbook was searched, quote, because the children in this home were such criminals, dope fiends, and had been in criminal offenses,|She said that her pocketbook was searched, quote, because the children in this home were such criminals, dope fiends, and had been in criminal offenses, +LJ040-0134|that anybody entering this home had to be searched in case the parents were bringing cigarettes or narcotics or anything, end quote.|that anybody entering this home had to be searched in case the parents were bringing cigarettes or narcotics or anything, end quote. +LJ040-0135|She recalled that Lee cried and said, quote, Mother, I want to get out of here.|She recalled that Lee cried and said, quote, Mother, I want to get out of here. +LJ040-0136|There are children in here who have killed people, and smoke. I want to get out, end quote,|There are children in here who have killed people, and smoke. I want to get out, end quote, +LJ040-0137|Marguerite Oswald said that she had not realized until then in what kind of place her son had been confined.|Marguerite Oswald said that she had not realized until then in what kind of place her son had been confined. +LJ040-0138|On the other hand, Lee told his probation officer, John Carro, that, quote,|On the other hand, Lee told his probation officer, John Carro, that, quote, +LJ040-0139|while he liked Youth House, he missed the freedom of doing what he wanted. He indicated that he did not miss his mother, end quote.|while he liked Youth House, he missed the freedom of doing what he wanted. He indicated that he did not miss his mother, end quote. +LJ040-0140|Mrs. Evelyn D Siegel, a social worker who interviewed both Lee and his mother while Lee was confined in Youth House,|Mrs. Evelyn D Siegel, a social worker who interviewed both Lee and his mother while Lee was confined in Youth House, +LJ040-0141|reported that Lee, quote, confided that the worse thing about Youth House was the fact that he had to be with other boys all the time,|reported that Lee, quote, confided that the worse thing about Youth House was the fact that he had to be with other boys all the time, +LJ040-0142|was disturbed about disrobing in front of them, taking showers with them etc., end quote.|was disturbed about disrobing in front of them, taking showers with them etc., end quote. +LJ040-0143|Contrary to reports that appeared after the assassination, the psychiatric examination did not indicate that Lee Oswald was a potential assassin,|Contrary to reports that appeared after the assassination, the psychiatric examination did not indicate that Lee Oswald was a potential assassin, +LJ040-0144|potentially dangerous, that, quote, his outlook on life had strongly paranoid overtones, end quote, or that he should be institutionalized.|potentially dangerous, that, quote, his outlook on life had strongly paranoid overtones, end quote, or that he should be institutionalized. +LJ040-0145|Dr. Hartogs did find Oswald to be a tense, withdrawn, and evasive boy who intensely disliked talking about himself and his feelings.|Dr. Hartogs did find Oswald to be a tense, withdrawn, and evasive boy who intensely disliked talking about himself and his feelings. +LJ040-0146|He noted that Lee liked to give the impression that he did not care for other people but preferred to keep to himself,|He noted that Lee liked to give the impression that he did not care for other people but preferred to keep to himself, +LJ040-0147|so that he was not bothered and did not have to make the effort of communicating.|so that he was not bothered and did not have to make the effort of communicating. +LJ040-0148|Oswald's withdrawn tendencies and solitary habits were thought to be the result of, quote,|Oswald's withdrawn tendencies and solitary habits were thought to be the result of, quote, +LJ040-0149|intense anxiety, shyness, feelings of awkwardness and insecurity, end quote.|intense anxiety, shyness, feelings of awkwardness and insecurity, end quote. +LJ040-0150|He was reported to have said, quote, I don't want a friend and I don't like to talk to people, end quote, and, quote, I dislike everybody, end quote.|He was reported to have said, quote, I don't want a friend and I don't like to talk to people, end quote, and, quote, I dislike everybody, end quote. +LJ040-0151|He was also described as having a, quote, Vivid fantasy life,|He was also described as having a, quote, Vivid fantasy life, +LJ040-0152|turning around the topics of omnipotence and power, through which he tries to compensate for his present shortcomings and frustrations, end quote.|turning around the topics of omnipotence and power, through which he tries to compensate for his present shortcomings and frustrations, end quote. +LJ040-0153|Dr. Hartogs summarized his report by stating:|Dr. Hartogs summarized his report by stating: +LJ040-0154|Quote, this 13 year old well built boy has superior mental resources and functions only slightly below his capacity level|Quote, this thirteen year old well built boy has superior mental resources and functions only slightly below his capacity level +LJ040-0155|in spite of chronic truancy from school which brought him into Youth House.|in spite of chronic truancy from school which brought him into Youth House. +LJ040-0156|No finding of neurological impairment or psychotic mental changes could be made.|No finding of neurological impairment or psychotic mental changes could be made. +LJ040-0157|Lee has to be diagnosed as, quote, personality pattern disturbance with schizoid features and passive-aggressive tendencies, end quote.|Lee has to be diagnosed as, quote, personality pattern disturbance with schizoid features and passive-aggressive tendencies, end quote. +LJ040-0158|Lee has to be seen as an emotionally, quite disturbed youngster|Lee has to be seen as an emotionally, quite disturbed youngster +LJ040-0159|who suffers under the impact of really existing emotional isolation|who suffers under the impact of really existing emotional isolation +LJ040-0160|and deprivation, lack of affection, absence of family life and rejection by a self involved and conflicted mother.|and deprivation, lack of affection, absence of family life and rejection by a self involved and conflicted mother. +LJ040-0162|There, he suggested, Lee should be treated by a male psychiatrist who could substitute for the lack of a father figure.|There, he suggested, Lee should be treated by a male psychiatrist who could substitute for the lack of a father figure. +LJ040-0163|He also recommended that Mrs. Oswald seek, quote, psychotherapeutic guidance through contact with a family agency, end quote.|He also recommended that Mrs. Oswald seek, quote, psychotherapeutic guidance through contact with a family agency, end quote. +LJ040-0164|The possibility of commitment was to be considered only if the probation plan was not successful.|The possibility of commitment was to be considered only if the probation plan was not successful. +LJ040-0165|Lee's withdrawal was also noted by Mrs. Siegel, who described him as a, quote, seriously detached, withdrawn youngster, end quote.|Lee's withdrawal was also noted by Mrs. Siegel, who described him as a, quote, seriously detached, withdrawn youngster, end quote. +LJ040-0166|She also noted that there was, quote, a rather pleasant, appealing quality about this emotionally starved, affectionless youngster|She also noted that there was, quote, a rather pleasant, appealing quality about this emotionally starved, affectionless youngster +LJ040-0167|which grows as one speaks to him, end quote.|which grows as one speaks to him, end quote. +LJ040-0168|She thought that he had detached himself from the world around him because, quote, no one in it ever met any of his needs for love, end quote.|She thought that he had detached himself from the world around him because, quote, no one in it ever met any of his needs for love, end quote. +LJ040-0170|because he didn't make friends with the boys in the neighborhood. She thought that he, quote, withdrew into a completely solitary and detached existence|because he didn't make friends with the boys in the neighborhood. She thought that he, quote, withdrew into a completely solitary and detached existence +LJ040-0171|where he did as he wanted and he didn't have to live by any rules or come into contact with people, end quote.|where he did as he wanted and he didn't have to live by any rules or come into contact with people, end quote. +LJ040-0172|Mrs. Siegel concluded that Lee, quote, just felt that his mother never gave a damn for him.|Mrs. Siegel concluded that Lee, quote, just felt that his mother never gave a damn for him. +LJ040-0173|He always felt like a burden that she simply just had to tolerate, end quote.|He always felt like a burden that she simply just had to tolerate, end quote. +LJ040-0174|Lee confirmed some of those observations by saying that he felt almost as if there were a veil between him and other people|Lee confirmed some of those observations by saying that he felt almost as if there were a veil between him and other people +LJ040-0175|through which they could not reach him, but that he preferred the veil to remain intact.|through which they could not reach him, but that he preferred the veil to remain intact. +LJ040-0176|He admitted to fantasies about being powerful and sometimes hurting and killing people, but refused to elaborate on them.|He admitted to fantasies about being powerful and sometimes hurting and killing people, but refused to elaborate on them. +LJ040-0177|He took the position that such matters were his own business.|He took the position that such matters were his own business. +LJ040-0178|A psychological human figure-drawing test corroborated the interviewer's findings that Lee was insecure and had limited social contacts.|A psychological human figure-drawing test corroborated the interviewer's findings that Lee was insecure and had limited social contacts. +LJ040-0179|Irving Sokolow, a Youth House psychologist reported that, quote,|Irving Sokolow, a Youth House psychologist reported that, quote, +LJ040-0180|The Human Figure Drawings are empty, poor characterizations of persons approximately the same age as the subject.|The Human Figure Drawings are empty, poor characterizations of persons approximately the same age as the subject. +LJ040-0181|They reflect a considerable amount of impoverishment in the social and emotional areas.|They reflect a considerable amount of impoverishment in the social and emotional areas. +LJ040-0182|He appears to be a somewhat insecure youngster exhibiting much inclination for warm and satisfying relationships to others.|He appears to be a somewhat insecure youngster exhibiting much inclination for warm and satisfying relationships to others. +LJ040-0183|There is some indication that he may relate to men more easily than to women in view of the more mature conceptualisation.|There is some indication that he may relate to men more easily than to women in view of the more mature conceptualisation. +LJ040-0184|He appears slightly withdrawn and in view of the lack of detail within the drawings this may assume a more significant characteristic.|He appears slightly withdrawn and in view of the lack of detail within the drawings this may assume a more significant characteristic. +LJ040-0185|He exhibits some difficulty in relationship to the maternal figure suggesting more anxiety in this area than in any other.|He exhibits some difficulty in relationship to the maternal figure suggesting more anxiety in this area than in any other. +LJ040-0186|Lee scored an IQ of 118 on the Wechsler Intelligence Scale for Children.|Lee scored an I.Q. of one eighteen on the Wechsler Intelligence Scale for Children. +LJ040-0187|According to Sokolow, this indicated a, quote, present intellectual functioning in the upper range of bright normal intelligence, end quote.|According to Sokolow, this indicated a, quote, present intellectual functioning in the upper range of bright normal intelligence, end quote. +LJ040-0188|Sokolow said that although Lee was, quote, presumably disinterested in school subjects he operates on a much higher than average level, end quote.|Sokolow said that although Lee was, quote, presumably disinterested in school subjects he operates on a much higher than average level, end quote. +LJ040-0189|On the Monroe Silent Reading Test, Lee's score indicated no retardation in reading speed and comprehension;|On the Monroe Silent Reading Test, Lee's score indicated no retardation in reading speed and comprehension; +LJ040-0190|he had better than average ability in arithmetical reasoning for his age group.|he had better than average ability in arithmetical reasoning for his age group. +LJ040-0191|Lee told Carro, his probation officer, that he liked to be by himself because he had too much difficulty in making friends.|Lee told Carro, his probation officer, that he liked to be by himself because he had too much difficulty in making friends. +LJ040-0192|The reports of Carro and Mrs. Siegel also indicate an ambivalent attitude toward authority on Oswald's part.|The reports of Carro and Mrs. Siegel also indicate an ambivalent attitude toward authority on Oswald's part. +LJ040-0193|Carro reported that Lee was disruptive in class after he returned to school on a regular basis in the fall of 1953.|Carro reported that Lee was disruptive in class after he returned to school on a regular basis in the fall of nineteen fifty-three. +LJ040-0194|He had refused to salute the flag and was doing very little, if any, work.|He had refused to salute the flag and was doing very little, if any, work. +LJ040-0195|It appears that he did not want to do any of the things which the authorities suggested in their efforts to bring him out of the shell|It appears that he did not want to do any of the things which the authorities suggested in their efforts to bring him out of the shell +LJ040-0196|into which he appeared to be retreating.|into which he appeared to be retreating. +LJ040-0197|He told Mrs. Siegel that he would run away if sent to a boarding school.|He told Mrs. Siegel that he would run away if sent to a boarding school. +LJ040-0198|On the other hand he also told her that he wished his mother had been more firm with him in her attempts to get him to return to school.|On the other hand he also told her that he wished his mother had been more firm with him in her attempts to get him to return to school. +LJ040-0199|The reports of the New York authorities indicate that Lee's mother gave him very little affection and did not serve as any sort of substitute for a father.|The reports of the New York authorities indicate that Lee's mother gave him very little affection and did not serve as any sort of substitute for a father. +LJ040-0200|Furthermore she did not appear to understand her own relationship to Lee's psychological problems.|Furthermore she did not appear to understand her own relationship to Lee's psychological problems. +LJ040-0201|After her interview with Mrs. Oswald,|After her interview with Mrs. Oswald, +LJ040-0202|Mrs. Siegel described her as a smartly dressed, gray haired woman, very self-possessed and alert and superficially affable,|Mrs. Siegel described her as a smartly dressed, gray haired woman, very self-possessed and alert and superficially affable, +LJ040-0203|but essentially a, quote, defensive, rigid, self-involved person|but essentially a, quote, defensive, rigid, self-involved person +LJ040-0204|who had real difficulty in accepting and relating to people, end quote, and who had, quote, little understanding, end quote, of Lee's behavior|who had real difficulty in accepting and relating to people, end quote, and who had, quote, little understanding, end quote, of Lee's behavior +LJ040-0205|and of the, quote, protective shell he has drawn around himself, end quote. Dr. Hartogs reported that Mrs. Oswald did not understand|and of the, quote, protective shell he has drawn around himself, end quote. Dr. Hartogs reported that Mrs. Oswald did not understand +LJ040-0206|that Lee's withdrawal was a form of, quote, violent but silent protest against his neglect by her|that Lee's withdrawal was a form of, quote, violent but silent protest against his neglect by her +LJ040-0207|and represents his reaction to a complete absence of any real family life, end quote.|and represents his reaction to a complete absence of any real family life, end quote. +LJ040-0208|Carro reported that when questioned about his mother Lee said, quote,|Carro reported that when questioned about his mother Lee said, quote, +LJ040-0209|well I've got to live with her. I guess I love her, end quote.|well I've got to live with her. I guess I love her, end quote. +LJ040-0210|It may also be significant that, as reported by John Pic, quote, Lee slept with my mother until I joined the service in 1950.|It may also be significant that, as reported by John Pic, quote, Lee slept with my mother until I joined the service in nineteen fifty. +LJ040-0211|This would make him approximately 10, well, almost 11 years old. End quote.|This would make him approximately ten, well, almost eleven years old. End quote. +LJ040-0212|The factors in Lee Oswald's personality which were noted by those who had contact with him in New York indicate|The factors in Lee Oswald's personality which were noted by those who had contact with him in New York indicate +LJ040-0213|that he had great difficulty in adapting himself to conditions in that city.|that he had great difficulty in adapting himself to conditions in that city. +LJ040-0214|His usual reaction to the problems which he encountered there was simply withdrawal.|His usual reaction to the problems which he encountered there was simply withdrawal. +LJ040-0215|Those factors indicated a severe inability to enter into relationships with other people.|Those factors indicated a severe inability to enter into relationships with other people. +LJ040-0216|In view of his experiences when he visited his relatives in New Orleans in the spring of 1950, and his other solitary habits,|In view of his experiences when he visited his relatives in New Orleans in the spring of nineteen fifty, and his other solitary habits, +LJ040-0217|Lee had apparently been experiencing similar problems before going to New York,|Lee had apparently been experiencing similar problems before going to New York, +LJ040-0218|and as will be shown below, this failure to adapt to his environment was a dominant trait in his later life.|and as will be shown below, this failure to adapt to his environment was a dominant trait in his later life. +LJ040-0219|It would be incorrect, however, to believe that those aspects of Lee's personality which were observed in New York|It would be incorrect, however, to believe that those aspects of Lee's personality which were observed in New York +LJ040-0220|could have led anyone to predict the outburst of violence which finally occurred.|could have led anyone to predict the outburst of violence which finally occurred. +LJ040-0221|Carro was the only one of Oswald's three principal observers who recommended that he be placed in a boy's home or similar institution.|Carro was the only one of Oswald's three principal observers who recommended that he be placed in a boy's home or similar institution. +LJ040-0222|But Carro was quite specific that his recommendation was based primarily on the adverse factors in Lee's environment|But Carro was quite specific that his recommendation was based primarily on the adverse factors in Lee's environment +LJ040-0223|his lack of friends, the apparent unavailability of any agency assistance and the ineffectualness of his mother|his lack of friends, the apparent unavailability of any agency assistance and the ineffectualness of his mother +LJ040-0224|and not on any particular mental disturbance, in the boy himself.|and not on any particular mental disturbance, in the boy himself. +LJ040-0225|Carro testified that, quote, There was nothing that would lead me to believe when I saw him at the age of 12 that them would be seeds of destruction for somebody.|Carro testified that, quote, There was nothing that would lead me to believe when I saw him at the age of twelve that them would be seeds of destruction for somebody. +LJ040-0226|I couldn't in all honesty sincerely say such a thing.|I couldn't in all honesty sincerely say such a thing. +LJ040-0227|Mrs. Siegel concluded her report with the statement that, quote,|Mrs. Siegel concluded her report with the statement that, quote, +LJ040-0228|Despite his withdrawal, he gives the impression that he is not so difficult to reach as he appears and patient, prolonged effort|Despite his withdrawal, he gives the impression that he is not so difficult to reach as he appears and patient, prolonged effort +LJ040-0229|in a sustained relationship with one therapist might bring results.|in a sustained relationship with one therapist might bring results. +LJ040-0230|There are indications that he has suffered serious personality damage but if he can receive help quickly this might be repaired to some extent, end quote.|There are indications that he has suffered serious personality damage but if he can receive help quickly this might be repaired to some extent, end quote. +LJ040-0231|Lee Oswald never received that help.|Lee Oswald never received that help. +LJ040-0232|Few social agencies even in New York were equipped to provide the kind of intensive treatment that he needed,|Few social agencies even in New York were equipped to provide the kind of intensive treatment that he needed, +LJ040-0233|and when one of the city's clinics did find room to handle him,|and when one of the city's clinics did find room to handle him, +LJ040-0234|for some reason the record does not show, advantage was never taken of the chance afforded to Oswald.|for some reason the record does not show, advantage was never taken of the chance afforded to Oswald. +LJ040-0235|When Lee became a disciplinary problem upon his return to school in the fall of 1953,|When Lee became a disciplinary problem upon his return to school in the fall of nineteen fifty-three, +LJ040-0236|and when his mother failed to cooperate in any way with school authorities,|and when his mother failed to cooperate in any way with school authorities, +LJ040-0237|authorities were finally forced to consider placement in a home for boys.|authorities were finally forced to consider placement in a home for boys. +LJ040-0238|Such a placement was postponed, however, perhaps in part at least because Lee's behavior suddenly improved.|Such a placement was postponed, however, perhaps in part at least because Lee's behavior suddenly improved. +LJ040-0239|Before the court took any action, the Oswalds left New York in January of 1954,|Before the court took any action, the Oswalds left New York in January of nineteen fifty-four, +LJ040-0240|and returned to New Orleans where Lee finished the ninth grade before he left school to work for a year.|and returned to New Orleans where Lee finished the ninth grade before he left school to work for a year. +LJ041-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ041-0002|Chapter 7. Lee Harvey Oswald: Background and Possible Motives, Part 2.|Chapter seven. Lee Harvey Oswald: Background and Possible Motives, Part two. +LJ041-0003|Return to New Orleans and Joining the Marine Corps|Return to New Orleans and Joining the Marine Corps +LJ041-0004|After his return to New Orleans Oswald was teased at school because of the northern accent which he had acquired.|After his return to New Orleans Oswald was teased at school because of the northern accent which he had acquired. +LJ041-0005|He concluded that school had nothing to offer him.|He concluded that school had nothing to offer him. +LJ041-0006|His mother exercised little control over him and thought he could decide for himself whether to go on in school.|His mother exercised little control over him and thought he could decide for himself whether to go on in school. +LJ041-0007|Neighbors and others who knew him at that time recall an introverted boy who read a great deal.|Neighbors and others who knew him at that time recall an introverted boy who read a great deal. +LJ041-0008|He took walks and visited museums, and sometimes rode a rented bicycle in the park on Saturday mornings.|He took walks and visited museums, and sometimes rode a rented bicycle in the park on Saturday mornings. +LJ041-0009|Mrs. Murret believes that he talked at length with a girl on the telephone, but no one remembers that he had any dates.|Mrs. Murret believes that he talked at length with a girl on the telephone, but no one remembers that he had any dates. +LJ041-0010|A friend, Edward Voebel, testified that, quote, he was more bashful about girls than anything else, end quote.|A friend, Edward Voebel, testified that, quote, he was more bashful about girls than anything else, end quote. +LJ041-0011|Several witnesses testified that Lee Oswald was not aggressive. He was, however, involved in some fights.|Several witnesses testified that Lee Oswald was not aggressive. He was, however, involved in some fights. +LJ041-0012|Once a group of white boys beat him up for sitting in the Negro section of a bus, which he apparently did simply out of ignorance.|Once a group of white boys beat him up for sitting in the Negro section of a bus, which he apparently did simply out of ignorance. +LJ041-0013|Another time, he fought with two brothers who claimed that he had picked on the younger of them, 3 years Oswald's junior.|Another time, he fought with two brothers who claimed that he had picked on the younger of them, three years Oswald's junior. +LJ041-0014|Two days later, quote, some big guy, probably from a high school -- he looked like a tremendous football player, end quote,|Two days later, quote, some big guy, probably from a high school -- he looked like a tremendous football player, end quote, +LJ041-0015|accosted Oswald on the way home from school and punched him in the mouth, making his lip bleed and loosening a tooth.|accosted Oswald on the way home from school and punched him in the mouth, making his lip bleed and loosening a tooth. +LJ041-0016|Voebel took Oswald back to the school to attend to his wounds, and their, quote, mild friendship, end quote, stemmed from that incident.|Voebel took Oswald back to the school to attend to his wounds, and their, quote, mild friendship, end quote, stemmed from that incident. +LJ041-0017|Voebel also recalled that Oswald once outlined a plan to cut the glass in the window of a store on Rampart Street and steal a pistol,|Voebel also recalled that Oswald once outlined a plan to cut the glass in the window of a store on Rampart Street and steal a pistol, +LJ041-0018|but he was not sure then that Oswald meant to carry out the plan, and in fact they never did.|but he was not sure then that Oswald meant to carry out the plan, and in fact they never did. +LJ041-0019|Voebel said that Oswald, quote, wouldn't start any fights, but if you wanted to start one with him, he was going to make sure that he ended it,|Voebel said that Oswald, quote, wouldn't start any fights, but if you wanted to start one with him, he was going to make sure that he ended it, +LJ041-0020|or you were going to really have one, because he wasn't going to take anything from anybody, end quote.|or you were going to really have one, because he wasn't going to take anything from anybody, end quote. +LJ041-0021|In a space for the names of, quote, close friends, end quote, on the ninth grade personal history record,|In a space for the names of, quote, close friends, end quote, on the ninth grade personal history record, +LJ041-0022|Oswald first wrote, quote, Edward Vogel, end quote, an obvious misspelling of Voebel's name,|Oswald first wrote, quote, Edward Vogel, end quote, an obvious misspelling of Voebel's name, +LJ041-0023|and, quote, Arthor Abear, end quote, most likely Arthur Hebert, a classmate who has said that he did not know Oswald well.|and, quote, Arthor Abear, end quote, most likely Arthur Hebert, a classmate who has said that he did not know Oswald well. +LJ041-0024|Oswald erased those names, however, and indicated that he had no close friends.|Oswald erased those names, however, and indicated that he had no close friends. +LJ041-0025|It has been suggested that this misspelling of names, apparently on a phonetic basis, was caused by a reading-spelling disability|It has been suggested that this misspelling of names, apparently on a phonetic basis, was caused by a reading-spelling disability +LJ041-0026|from which Oswald appeared to suffer.|from which Oswald appeared to suffer. +LJ041-0027|Other evidence of the existence of such a disability is provided by the many other misspellings that appear in Oswald's writings, portions of which are quoted below.|Other evidence of the existence of such a disability is provided by the many other misspellings that appear in Oswald's writings, portions of which are quoted below. +LJ041-0028|Sometime during this period, and under circumstances to be discussed more fully below,|Sometime during this period, and under circumstances to be discussed more fully below, +LJ041-0029|Oswald started to read Communist literature, which he obtained from the public library.|Oswald started to read Communist literature, which he obtained from the public library. +LJ041-0030|One of his fellow employees, Palmer McBride, stated that Oswald said he would like to kill President Eisenhower because he was exploiting the working class.|One of his fellow employees, Palmer McBride, stated that Oswald said he would like to kill President Eisenhower because he was exploiting the working class. +LJ041-0031|Oswald praised Khrushchev and suggested that he and McBride join the Communist Party, quote, to take advantage of their social functions, end quote.|Oswald praised Khrushchev and suggested that he and McBride join the Communist Party, quote, to take advantage of their social functions, end quote. +LJ041-0032|Oswald also became interested in the New Orleans Amateur Astronomy Association, an organization of high school students.|Oswald also became interested in the New Orleans Amateur Astronomy Association, an organization of high school students. +LJ041-0033|The association's then president, William E. Wulf, testified that he remembered an occasion when Oswald, quote,|The association's then president, William E. Wulf, testified that he remembered an occasion when Oswald, quote, +LJ041-0034|started expounding the Communist doctrine and saying that he was highly interested in communism, that communism was the only way of life for the worker, et cetera,|started expounding the Communist doctrine and saying that he was highly interested in communism, that communism was the only way of life for the worker, et cetera, +LJ041-0035|and then came out with a statement that he was looking for a Communist cell in town to join but he couldn't find any.|and then came out with a statement that he was looking for a Communist cell in town to join but he couldn't find any. +LJ041-0036|He was a little dismayed at this, and he said that he couldn't find any that would show any interest in him as a Communist,|He was a little dismayed at this, and he said that he couldn't find any that would show any interest in him as a Communist, +LJ041-0037|and subsequently, after this conversation, my father came in and we were kind of arguing back and forth about the situation, and my father came in the room,|and subsequently, after this conversation, my father came in and we were kind of arguing back and forth about the situation, and my father came in the room, +LJ041-0038|heard what we were arguing on communism, and that this boy was loud-mouthed, boisterous, and my father asked him to leave the house and politely put him out of the house,|heard what we were arguing on communism, and that this boy was loud-mouthed, boisterous, and my father asked him to leave the house and politely put him out of the house, +LJ041-0039|and that is the last I have seen or spoken with Oswald. End quote.|and that is the last I have seen or spoken with Oswald. End quote. +LJ041-0040|Despite this apparent interest in communism, Oswald tried to join the Marines when he was 16 years old.|Despite this apparent interest in communism, Oswald tried to join the Marines when he was sixteen years old. +LJ041-0041|This was 1 year before his actual enlistment and just a little over 2.5 years after he left New York.|This was one year before his actual enlistment and just a little over two point five years after he left New York. +LJ041-0042|He wrote a note in his mother's name to school authorities in New Orleans saying that he was leaving school because he and his mother were moving to San Diego.|He wrote a note in his mother's name to school authorities in New Orleans saying that he was leaving school because he and his mother were moving to San Diego. +LJ041-0043|In fact, he had quit school in an attempt to obtain his mother's assistance to join the Marines.|In fact, he had quit school in an attempt to obtain his mother's assistance to join the Marines. +LJ041-0044|While he apparently was able to induce his mother to make a false statement about his age|While he apparently was able to induce his mother to make a false statement about his age +LJ041-0045|he was nevertheless unable to convince the proper authorities that he was really 17 years old.|he was nevertheless unable to convince the proper authorities that he was really seventeen years old. +LJ041-0046|There is evidence that Oswald was greatly influenced in his decision to join the Marines by the fact that his brother Robert had done so|There is evidence that Oswald was greatly influenced in his decision to join the Marines by the fact that his brother Robert had done so +LJ041-0047|approximately 3 years before. Robert Oswald had given his Marine Corps manual to his brother Lee,|approximately three years before. Robert Oswald had given his Marine Corps manual to his brother Lee, +LJ041-0048|who studied it during the year following his unsuccessful attempt to enlist until, quote, He knew it by heart, end quote.|who studied it during the year following his unsuccessful attempt to enlist until, quote, He knew it by heart, end quote. +LJ041-0049|According to Marguerite Oswald, quote, Lee lived for the time that he would become 17 years old to join the Marines -- that whole year, end quote.|According to Marguerite Oswald, quote, Lee lived for the time that he would become seventeen years old to join the Marines -- that whole year, end quote. +LJ041-0050|In John Pic's view,|In John Pic's view, +LJ041-0051|Oswald was motivated to join the Marines in large part by a desire, quote, to get from out and under the yoke of oppression from my mother, end quote.|Oswald was motivated to join the Marines in large part by a desire, quote, to get from out and under the yoke of oppression from my mother, end quote. +LJ041-0052|Oswald's inability or lack of desire to enter into meaningful relationships with other people|Oswald's inability or lack of desire to enter into meaningful relationships with other people +LJ041-0053|continued during this period in New Orleans (1954 to 1956).|continued during this period in New Orleans (nineteen fifty-four to nineteen fifty-six). +LJ041-0054|It probably contributed greatly to the general dissatisfaction which he exhibited with his environment,|It probably contributed greatly to the general dissatisfaction which he exhibited with his environment, +LJ041-0055|a dissatisfaction which seemed to find expression at this particular point|a dissatisfaction which seemed to find expression at this particular point +LJ041-0056|in his intense desire to join the Marines and get away from his surroundings and his mother.|in his intense desire to join the Marines and get away from his surroundings and his mother. +LJ041-0057|His study of Communist literature,|His study of Communist literature, +LJ041-0058|which might appear to be inconsistent with his desire to join the Marines, could have been another manifestation of Oswald's rejection of his environment.|which might appear to be inconsistent with his desire to join the Marines, could have been another manifestation of Oswald's rejection of his environment. +LJ041-0059|His difficulty in relating to other people and his general dissatisfaction with the world around him continued while he was in the Marine Corps.|His difficulty in relating to other people and his general dissatisfaction with the world around him continued while he was in the Marine Corps. +LJ041-0060|Kerry Thornley, a marine associate,|Kerry Thornley, a marine associate, +LJ041-0061|who, shortly after Oswald's defection, wrote an as yet unpublished novel based in considerable part on Oswald's life,|who, shortly after Oswald's defection, wrote an as yet unpublished novel based in considerable part on Oswald's life, +LJ041-0062|testified that, quote, definitely the Marine Corps was not what he had expected it to be when he joined, end quote,|testified that, quote, definitely the Marine Corps was not what he had expected it to be when he joined, end quote, +LJ041-0063|He said that Oswald, quote, seemed to guard against developing real close friendships, end quote,|He said that Oswald, quote, seemed to guard against developing real close friendships, end quote, +LJ041-0064|Daniel Powers, another marine who was stationed with Oswald for part of his marine career,|Daniel Powers, another marine who was stationed with Oswald for part of his marine career, +LJ041-0065|testified that Oswald seemed, quote, always to be striving for a relationship, but whenever he did|testified that Oswald seemed, quote, always to be striving for a relationship, but whenever he did +LJ041-0066|his general personality would alienate the group against him, end quote.|his general personality would alienate the group against him, end quote. +LJ041-0067|Other marines also testified that Oswald had few friends and kept very much to himself.|Other marines also testified that Oswald had few friends and kept very much to himself. +LJ041-0068|While there is nothing in Oswald's military records to indicate that he was mentally unstable or otherwise psychologically unfit for duty in the Marine Corps,|While there is nothing in Oswald's military records to indicate that he was mentally unstable or otherwise psychologically unfit for duty in the Marine Corps, +LJ041-0069|he did not adjust well to conditions which he found in that service.|he did not adjust well to conditions which he found in that service. +LJ041-0070|He did not rise above the rank of private first class, even though he had passed a qualifying examination for the rank of corporal.|He did not rise above the rank of private first class, even though he had passed a qualifying examination for the rank of corporal. +LJ041-0071|His Marine career was not helped by his attitude|His Marine career was not helped by his attitude +LJ041-0072|that he was a man of great ability and intelligence and that many of his superiors in the Marine Corps were not sufficiently competent to give him orders.|that he was a man of great ability and intelligence and that many of his superiors in the Marine Corps were not sufficiently competent to give him orders. +LJ041-0073|While Oswald did not seem to object to authority in the abstract, he did think that he should be the one to exercise it.|While Oswald did not seem to object to authority in the abstract, he did think that he should be the one to exercise it. +LJ041-0074|John E. Donovan, one of his former officers, testified that Oswald thought, quote,|John E. Donovan, one of his former officers, testified that Oswald thought, quote, +LJ041-0075|that authority, particularly the Marine Corps, ought to be able to recognize talent such as his own, without a given magic college degree,|that authority, particularly the Marine Corps, ought to be able to recognize talent such as his own, without a given magic college degree, +LJ041-0076|and put them in positions of prominence, end quote. Oswald manifested this feeling about authority by baiting his officers.|and put them in positions of prominence, end quote. Oswald manifested this feeling about authority by baiting his officers. +LJ041-0077|He led them into discussions of foreign affairs about which they often knew less than he did, since he had apparently devoted considerable time to a study of such matters.|He led them into discussions of foreign affairs about which they often knew less than he did, since he had apparently devoted considerable time to a study of such matters. +LJ041-0078|When the officers were unable to discuss foreign affairs satisfactorily with him, Oswald regarded them as unfit to exercise command over him.|When the officers were unable to discuss foreign affairs satisfactorily with him, Oswald regarded them as unfit to exercise command over him. +LJ041-0079|Nelson Delgado, one of Oswald's fellow Marines, testified that Oswald tried to, quote, cut up anybody that was high ranking, end quote,|Nelson Delgado, one of Oswald's fellow Marines, testified that Oswald tried to, quote, cut up anybody that was high ranking, end quote, +LJ041-0080|in those arguments, quote, and make himself come out top dog, end quote.|in those arguments, quote, and make himself come out top dog, end quote. +LJ041-0081|Oswald probably engaged his superiors in arguments on a subject that he had studied|Oswald probably engaged his superiors in arguments on a subject that he had studied +LJ041-0082|in an attempt to attract attention to himself and to support his exaggerated idea of his own abilities.|in an attempt to attract attention to himself and to support his exaggerated idea of his own abilities. +LJ041-0083|Thornley also testified that he thought that Oswald's extreme personal sloppiness in the Marine Corps, quote,|Thornley also testified that he thought that Oswald's extreme personal sloppiness in the Marine Corps, quote, +LJ041-0084|fitted into a general personality pattern of his: to do whatever was not wanted of him, a recalcitrant trend in his personality, end quote,|fitted into a general personality pattern of his: to do whatever was not wanted of him, a recalcitrant trend in his personality, end quote, +LJ041-0085|Oswald, quote, seemed to be a person who would go out of his way to get into trouble, end quote,|Oswald, quote, seemed to be a person who would go out of his way to get into trouble, end quote, +LJ041-0086|and then used the, quote, special treatment, end quote, he received as an example of the way in which he was being picked on and, quote,|and then used the, quote, special treatment, end quote, he received as an example of the way in which he was being picked on and, quote, +LJ041-0087|as a means of getting or attempting to get sympathy, end quote. In Thornley's view, Oswald labored under a persecution complex|as a means of getting or attempting to get sympathy, end quote. In Thornley's view, Oswald labored under a persecution complex +LJ041-0088|which he strove to maintain and, quote, felt the Marine Corps kept a pretty close watch on him because of his subversive activities, end quote.|which he strove to maintain and, quote, felt the Marine Corps kept a pretty close watch on him because of his subversive activities, end quote. +LJ041-0089|Thornley added, quote, I think it was kind of necessary to him to believe that he was being picked on.|Thornley added, quote, I think it was kind of necessary to him to believe that he was being picked on. +LJ041-0090|It wasn't anything extreme. I wouldn't go as far as to call it, call him a paranoid, but a definite tendency there was in that direction, I think, end quote.|It wasn't anything extreme. I wouldn't go as far as to call it, call him a paranoid, but a definite tendency there was in that direction, I think, end quote. +LJ041-0091|Powers considered Oswald to be meek and easily led|Powers considered Oswald to be meek and easily led +LJ041-0092|an, quote, individual that you would brainwash, and quite easy, but I think once he believed in something, he stood in his beliefs, end quote.|an, quote, individual that you would brainwash, and quite easy, but I think once he believed in something, he stood in his beliefs, end quote. +LJ041-0093|Powers also testified that Oswald was reserved and seemed to be, quote, somewhat the frail, little puppy in the litter, end quote.|Powers also testified that Oswald was reserved and seemed to be, quote, somewhat the frail, little puppy in the litter, end quote. +LJ041-0094|He had the nickname, quote, Ozzie Rabbit, end quote.|He had the nickname, quote, Ozzie Rabbit, end quote. +LJ041-0095|Oswald read a good deal, said Powers, but, quote, he would never be reading any of the shoot-em-up westerns or anything like that.|Oswald read a good deal, said Powers, but, quote, he would never be reading any of the shoot-em-up westerns or anything like that. +LJ041-0096|Normally, it would be a good type of literature; and the one that I recall was "Leaves of Grass," by Walt Whitman, end quote.|Normally, it would be a good type of literature; and the one that I recall was "Leaves of Grass," by Walt Whitman, end quote. +LJ041-0097|According to Powers, Oswald said, quote,|According to Powers, Oswald said, quote, +LJ041-0099|Powers believed that when Oswald arrived in Japan he acquired a girlfriend, quote,|Powers believed that when Oswald arrived in Japan he acquired a girlfriend, quote, +LJ041-0100|finally attaining a male status or image in his own eyes, end quote. That apparently caused Oswald to become more self-confident,|finally attaining a male status or image in his own eyes, end quote. That apparently caused Oswald to become more self-confident, +LJ041-0101|aggressive and even somewhat pugnacious, although Powers, quote, wouldn't say that this guy is a troublemaker, end quote.|aggressive and even somewhat pugnacious, although Powers, quote, wouldn't say that this guy is a troublemaker, end quote. +LJ041-0102|Powers said, quote, now he was Oswald the man rather than Oswald the rabbit, end quote.|Powers said, quote, now he was Oswald the man rather than Oswald the rabbit, end quote. +LJ041-0103|Oswald once told Powers that he didn't care if he returned to the United States at all.|Oswald once told Powers that he didn't care if he returned to the United States at all. +LJ041-0104|While in Japan, Oswald's new found apparent self confidence and pugnaciousness|While in Japan, Oswald's new found apparent self confidence and pugnaciousness +LJ041-0105|led to an incident in which he spilled a drink on one of his sergeants and abusively challenged him to fight.|led to an incident in which he spilled a drink on one of his sergeants and abusively challenged him to fight. +LJ041-0106|At the court-martial hearing which followed, Oswald admitted that he had been rather drunk when the incident occurred.|At the court-martial hearing which followed, Oswald admitted that he had been rather drunk when the incident occurred. +LJ041-0107|He testified that he had felt the sergeant had a grudge against him and that he had unsuccessfully sought a transfer from the sergeant's unit.|He testified that he had felt the sergeant had a grudge against him and that he had unsuccessfully sought a transfer from the sergeant's unit. +LJ041-0108|He said that he had simply wanted to discuss the question with the sergeant and the drink had been spilled accidentally.|He said that he had simply wanted to discuss the question with the sergeant and the drink had been spilled accidentally. +LJ041-0109|The hearing officer agreed with the latter claim but found Oswald guilty of wrongfully using provoking words and sentenced him to 28 days,|The hearing officer agreed with the latter claim but found Oswald guilty of wrongfully using provoking words and sentenced him to twenty-eight days, +LJ041-0110|canceling the suspension of a 20-day sentence that Oswald had received in an earlier court-martial|canceling the suspension of a twenty-day sentence that Oswald had received in an earlier court-martial +LJ041-0111|for possessing an unauthorized pistol with which he had accidentally shot himself.|for possessing an unauthorized pistol with which he had accidentally shot himself. +LJ041-0112|At his own request,|At his own request, +LJ041-0113|Oswald was transferred from active duty to the Marine Corps Reserve under honorable conditions in September of 1959,|Oswald was transferred from active duty to the Marine Corps Reserve under honorable conditions in September of nineteen fifty-nine, +LJ041-0114|3 months prior to his regularly scheduled separation date, ostensibly to care for his mother who had been injured in an accident at her work.|three months prior to his regularly scheduled separation date, ostensibly to care for his mother who had been injured in an accident at her work. +LJ041-0115|He was undesirably discharged from the Marine Corps Reserve, to which he had been assigned on inactive status following his transfer from active duty,|He was undesirably discharged from the Marine Corps Reserve, to which he had been assigned on inactive status following his transfer from active duty, +LJ041-0116|after it was learned that he had defected to the Soviet Union.|after it was learned that he had defected to the Soviet Union. +LJ041-0117|In an attempt to have this discharge reversed,|In an attempt to have this discharge reversed, +LJ041-0118|Oswald wrote to then Secretary of the Navy Connally on January 30, 1962,|Oswald wrote to then Secretary of the Navy Connally on January thirty, nineteen sixty-two, +LJ041-0119|stating that he would, quote, employ all means to right this gross mistake or injustice, end quote.|stating that he would, quote, employ all means to right this gross mistake or injustice, end quote. +LJ041-0120|Governor Connally had just resigned to run for Governor of Texas, so he advised Oswald that he had forwarded the letter to his successor.|Governor Connally had just resigned to run for Governor of Texas, so he advised Oswald that he had forwarded the letter to his successor. +LJ041-0121|It is thus clear that Oswald knew that Governor Connally was never directly concerned with his discharge|It is thus clear that Oswald knew that Governor Connally was never directly concerned with his discharge +LJ041-0122|and he must have known that President Kennedy had had nothing to do with it.|and he must have known that President Kennedy had had nothing to do with it. +LJ041-0123|In that connection, it does not appear that Oswald ever expressed any dissatisfaction of any kind with either the President or Governor Connally.|In that connection, it does not appear that Oswald ever expressed any dissatisfaction of any kind with either the President or Governor Connally. +LJ041-0124|Marina Oswald testified that she, quote, had never heard anything bad about Kennedy from Lee. And he never had anything against him, end quote.|Marina Oswald testified that she, quote, had never heard anything bad about Kennedy from Lee. And he never had anything against him, end quote. +LJ041-0125|Mrs. Oswald said that her husband did not say anything about Governor Connally after his return to the United States. She testified, quote,|Mrs. Oswald said that her husband did not say anything about Governor Connally after his return to the United States. She testified, quote, +LJ041-0126|But while we were in Russia he spoke well of him. Lee said that when he would return to the United States he would vote for him for Governor, end quote.|But while we were in Russia he spoke well of him. Lee said that when he would return to the United States he would vote for him for Governor, end quote. +LJ041-0127|Oswald must have already learned that the Governor could not help him with his discharge because he was no longer Secretary of the Navy, at the time he made that remark.|Oswald must have already learned that the Governor could not help him with his discharge because he was no longer Secretary of the Navy, at the time he made that remark. +LJ041-0128|Even though Oswald apparently did not express any hostility against the President or Governor Connally,|Even though Oswald apparently did not express any hostility against the President or Governor Connally, +LJ041-0129|he continued to be concerned about his undesirable discharge. It is clear that he thought he had been unjustly treated.|he continued to be concerned about his undesirable discharge. It is clear that he thought he had been unjustly treated. +LJ041-0130|Probably his complaint was due to the fact that his discharge was not related to anything he had done while on active duty|Probably his complaint was due to the fact that his discharge was not related to anything he had done while on active duty +LJ041-0131|and also because he had not received any notice of the original discharge proceedings, since his whereabouts were not known.|and also because he had not received any notice of the original discharge proceedings, since his whereabouts were not known. +LJ041-0132|He continued his efforts to reverse the discharge by petitioning the Navy Discharge Review Board,|He continued his efforts to reverse the discharge by petitioning the Navy Discharge Review Board, +LJ041-0133|which finally declined to modify the discharge and so advised him in a letter dated July 1963.|which finally declined to modify the discharge and so advised him in a letter dated July nineteen sixty-three. +LJ041-0134|Governor Connally's connection with the discharge, although indirect, caused the Commission to consider whether he might have been Oswald's real target.|Governor Connally's connection with the discharge, although indirect, caused the Commission to consider whether he might have been Oswald's real target. +LJ041-0135|In that connection, it should be noted that Marina Oswald testified on September 6, 1964, that she thought her husband, quote,|In that connection, it should be noted that Marina Oswald testified on September six, nineteen sixty-four, that she thought her husband, quote, +LJ041-0136|was shooting at Connally rather than President Kennedy, end quote.|was shooting at Connally rather than President Kennedy, end quote. +LJ041-0137|In support of her conclusion Mrs. Oswald noted her husband's undesirable discharge|In support of her conclusion Mrs. Oswald noted her husband's undesirable discharge +LJ041-0138|and that she could not think of any reason why Oswald would want to kill President Kennedy.|and that she could not think of any reason why Oswald would want to kill President Kennedy. +LJ041-0139|It should be noted, however, that at the time Oswald fired the shots at the Presidential limousine the Governor occupied the seat in front of the President,|It should be noted, however, that at the time Oswald fired the shots at the Presidential limousine the Governor occupied the seat in front of the President, +LJ041-0140|and it would have been almost impossible for Oswald to have hit the Governor without hitting the President first.|and it would have been almost impossible for Oswald to have hit the Governor without hitting the President first. +LJ041-0141|Oswald could have shot the Governor as the car approached the Depository or as it was making the turn onto Elm Street.|Oswald could have shot the Governor as the car approached the Depository or as it was making the turn onto Elm Street. +LJ041-0142|Once it had started down Elm Street toward the Triple Underpass, however,|Once it had started down Elm Street toward the Triple Underpass, however, +LJ041-0143|the President almost completely blocked Oswald's view of the Governor prior to the time the first shot struck the President.|the President almost completely blocked Oswald's view of the Governor prior to the time the first shot struck the President. +LJ041-0144|Oswald would have had other and more favorable opportunities to strike at the Governor than on this occasion when, as a member of the President's party,|Oswald would have had other and more favorable opportunities to strike at the Governor than on this occasion when, as a member of the President's party, +LJ041-0145|he had more protection than usual. It would appear, therefore, that to the extent Oswald's undesirable discharge affected his motivation,|he had more protection than usual. It would appear, therefore, that to the extent Oswald's undesirable discharge affected his motivation, +LJ041-0146|it was more in terms of a general hostility against the government and its representatives rather than a grudge against any particular person.|it was more in terms of a general hostility against the government and its representatives rather than a grudge against any particular person. +LJ041-0147|Interest in Marxism|Interest in Marxism +LJ041-0148|As indicated above, Oswald started to read Communist literature after he and his mother left New York and moved to New Orleans.|As indicated above, Oswald started to read Communist literature after he and his mother left New York and moved to New Orleans. +LJ041-0149|He told Aline Mosby, a reporter who interviewed him after he arrived in Moscow, quote,|He told Aline Mosby, a reporter who interviewed him after he arrived in Moscow, quote, +LJ041-0150|I'm a Marxist. I became interested about the age of 15. From an ideological viewpoint.|I'm a Marxist. I became interested about the age of fifteen. From an ideological viewpoint. +LJ041-0151|An old lady handed me a pamphlet about saving the Rosenbergs. I looked at that paper and I still remember it for some reason, I don't know why. End quote.|An old lady handed me a pamphlet about saving the Rosenbergs. I looked at that paper and I still remember it for some reason, I don't know why. End quote. +LJ041-0152|Oswald studied Marxism after he joined the Marines and his sympathies in that direction and for the Soviet Union appear to have been widely known,|Oswald studied Marxism after he joined the Marines and his sympathies in that direction and for the Soviet Union appear to have been widely known, +LJ041-0153|at least in the unit to which he was assigned after his return from the Far East.|at least in the unit to which he was assigned after his return from the Far East. +LJ041-0154|His interest in Russia led some of his associates to call him "comrade" or "Oswaldskovitch."|His interest in Russia led some of his associates to call him "comrade" or "Oswaldskovitch." +LJ041-0155|He always wanted to play the red pieces in chess because, as he said in an apparently humorous context,|He always wanted to play the red pieces in chess because, as he said in an apparently humorous context, +LJ041-0156|he preferred the, quote, Red Army, end quote.|he preferred the, quote, Red Army, end quote. +LJ041-0157|He studied the Russian language, read a Russian language newspaper and seemed interested in what was going on in the Soviet Union.|He studied the Russian language, read a Russian language newspaper and seemed interested in what was going on in the Soviet Union. +LJ041-0158|Thornley, who thought Oswald had an "irrevocable conviction" that his Marxist beliefs were correct, testified, quote,|Thornley, who thought Oswald had an "irrevocable conviction" that his Marxist beliefs were correct, testified, quote, +LJ041-0159|I think you could sit down and argue with him for a number of years, and I don't think you could have changed his mind on that unless you knew why he believed it in the first place.|I think you could sit down and argue with him for a number of years, and I don't think you could have changed his mind on that unless you knew why he believed it in the first place. +LJ041-0160|I certainly don't. I don't think with any kind of formal argument you could have shaken that conviction. And that is why I say irrevocable.|I certainly don't. I don't think with any kind of formal argument you could have shaken that conviction. And that is why I say irrevocable. +LJ041-0161|It was just -- never getting back to looking at things from any other way once he had become a Marxist, whenever that was, end quote.|It was just -- never getting back to looking at things from any other way once he had become a Marxist, whenever that was, end quote. +LJ041-0162|Thornley also testified about an incident which grew|Thornley also testified about an incident which grew +LJ041-0163|out of a combination of Oswald's known Marxist sympathies and George Orwell's book "1984," one of Oswald's favorite books|out of a combination of Oswald's known Marxist sympathies and George Orwell's book "nineteen eighty-four," one of Oswald's favorite books +LJ041-0164|which Thornley read at Oswald's suggestion.|which Thornley read at Oswald's suggestion. +LJ041-0165|Shortly after Thornley finished reading that book the Marine unit to which both men were assigned was required to take part in a Saturday morning parade|Shortly after Thornley finished reading that book the Marine unit to which both men were assigned was required to take part in a Saturday morning parade +LJ041-0166|in honor of some retiring noncommissioned officers, an event which they both approached with little enthusiasm.|in honor of some retiring noncommissioned officers, an event which they both approached with little enthusiasm. +LJ041-0167|While waiting for the parade to start they talked briefly about "1984"|While waiting for the parade to start they talked briefly about "nineteen eighty-four" +LJ041-0168|even though Oswald seemed to be lost in his own thoughts. After a brief period of silence Oswald remarked on the stupidity of the parade|even though Oswald seemed to be lost in his own thoughts. After a brief period of silence Oswald remarked on the stupidity of the parade +LJ041-0169|and on how angry it made him, to which Thornley replied, quote, Well, comes the revolution you will change all that, end quote.|and on how angry it made him, to which Thornley replied, quote, Well, comes the revolution you will change all that, end quote. +LJ041-0170|Thornley testified, quote, At which time he looked at me like a betrayed Caesar and screamed, screamed definitely, "Not you, too, Thornley!"|Thornley testified, quote, At which time he looked at me like a betrayed Caesar and screamed, screamed definitely, "Not you, too, Thornley!" +LJ041-0171|And I remember his voice cracked as he said this. He was definitely disturbed at what I had said and I didn't really think I had said that much.|And I remember his voice cracked as he said this. He was definitely disturbed at what I had said and I didn't really think I had said that much. +LJ041-0172|I never said anything to him again and he never said anything to me again, end quote.|I never said anything to him again and he never said anything to me again, end quote. +LJ041-0173|Thornley said that he had made his remark only in the context of "1984"|Thornley said that he had made his remark only in the context of "nineteen eighty-four" +LJ041-0174|and had not intended any criticism of Oswald's political views which is the way in which, Thornley thought, Oswald took his remarks.|and had not intended any criticism of Oswald's political views which is the way in which, Thornley thought, Oswald took his remarks. +LJ041-0175|Lieutenant Donovan testified that Oswald thought that, quote, there were many grave injustices concerning the affairs in the international situation, end quote.|Lieutenant Donovan testified that Oswald thought that, quote, there were many grave injustices concerning the affairs in the international situation, end quote. +LJ041-0176|He recalled that Oswald had a specific interest in Latin America, particularly Cuba,|He recalled that Oswald had a specific interest in Latin America, particularly Cuba, +LJ041-0177|and expressed opposition to the Batista regime and sympathy for Castro, an attitude which, Donovan said, was, quote,|and expressed opposition to the Batista regime and sympathy for Castro, an attitude which, Donovan said, was, quote, +LJ041-0178|not unpopular, end quote, at that time. Donovan testified|not unpopular, end quote, at that time. Donovan testified +LJ041-0179|that he never heard Oswald express a desire personally to take part in the elimination of injustices anywhere in the world and that he, quote,|that he never heard Oswald express a desire personally to take part in the elimination of injustices anywhere in the world and that he, quote, +LJ041-0180|never heard him in any way, shape or form confess that he was a Communist, or that he ever thought about being a Communist, end quote.|never heard him in any way, shape or form confess that he was a Communist, or that he ever thought about being a Communist, end quote. +LJ041-0181|Delgado testified that Oswald was, quote, a complete believer that our way of government was not quite right, end quote.|Delgado testified that Oswald was, quote, a complete believer that our way of government was not quite right, end quote. +LJ041-0182|and believed that our Government did not have, quote, too much to offer, end quote, but was not in favor of, quote, the Communist way of life, end quote.|and believed that our Government did not have, quote, too much to offer, end quote, but was not in favor of, quote, the Communist way of life, end quote. +LJ041-0183|Delgado and Oswald talked more about Cuba than Russia, and sometimes imagined themselves as leaders in the Cuban Army or Government,|Delgado and Oswald talked more about Cuba than Russia, and sometimes imagined themselves as leaders in the Cuban Army or Government, +LJ041-0184|who might, quote, lead an expedition to some of these other islands and free them too, end quote.|who might, quote, lead an expedition to some of these other islands and free them too, end quote. +LJ041-0185|Thornley also believed that Oswald's Marxist beliefs led to an extraordinary view of history under which, quote, He looked upon the eyes of future people|Thornley also believed that Oswald's Marxist beliefs led to an extraordinary view of history under which, quote, He looked upon the eyes of future people +LJ041-0186|as some kind of tribunal,|as some kind of tribunal, +LJ041-0187|and he wanted to be on the winning side so that 10,000 years from-now people would look in the history books and say, "Well, this man was ahead of his time."|and he wanted to be on the winning side so that ten thousand years from-now people would look in the history books and say, "Well, this man was ahead of his time." +LJ041-0188|The eyes of the future became the eyes of God.|The eyes of the future became the eyes of God. +LJ041-0189|He was concerned with his image in history and I do think that is why he chose the particular method of defecting he chose and did it in the way he did.|He was concerned with his image in history and I do think that is why he chose the particular method of defecting he chose and did it in the way he did. +LJ041-0190|It got him in the newspapers. It did broadcast his name out, end quote.|It got him in the newspapers. It did broadcast his name out, end quote. +LJ041-0191|Thornley thought that Oswald not only wanted a place in history but also wanted to live comfortably in the present.|Thornley thought that Oswald not only wanted a place in history but also wanted to live comfortably in the present. +LJ041-0192|He testified that if Oswald could not have that, quote, degree of physical comfort that he expected or sought,|He testified that if Oswald could not have that, quote, degree of physical comfort that he expected or sought, +LJ041-0193|I think he would then throw himself entirely on the other thing he also wanted, which was the image in history.|I think he would then throw himself entirely on the other thing he also wanted, which was the image in history. +LJ041-0194|I think he wanted both if he could have them. If he didn't, he wanted to die with the knowledge that, or with the idea that he was somebody, end quote.|I think he wanted both if he could have them. If he didn't, he wanted to die with the knowledge that, or with the idea that he was somebody, end quote. +LJ041-0195|Oswald's interest in Marxism led some people to avoid him,|Oswald's interest in Marxism led some people to avoid him, +LJ041-0196|even though as his wife suggested, that interest may have been motivated by a desire to gain attention. He used his Marxist and associated activities|even though as his wife suggested, that interest may have been motivated by a desire to gain attention. He used his Marxist and associated activities +LJ041-0197|as excuses for his difficulties in getting along in the world, which were usually caused by entirely different factors.|as excuses for his difficulties in getting along in the world, which were usually caused by entirely different factors. +LJ041-0198|His use of those excuses to present himself to the world as a person who was being unfairly treated|His use of those excuses to present himself to the world as a person who was being unfairly treated +LJ041-0199|is shown most clearly by his employment relations after his return from the Soviet Union. Of course, he made his real problems worse to the extent|is shown most clearly by his employment relations after his return from the Soviet Union. Of course, he made his real problems worse to the extent +LJ041-0200|that his use of those excuses prevented him from discovering the real reasons for and attempting to overcome his difficulties.|that his use of those excuses prevented him from discovering the real reasons for and attempting to overcome his difficulties. +LJ041-0201|Of greater importance,|Of greater importance, +LJ041-0202|Oswald's commitment to Marxism contributed to the decisions which led him to defect to the Soviet Union in 1959, and later|Oswald's commitment to Marxism contributed to the decisions which led him to defect to the Soviet Union in nineteen fifty-nine, and later +LJ041-0203|to engage in activities on behalf of the Fair Play for Cuba Committee in the summer of 1963,|to engage in activities on behalf of the Fair Play for Cuba Committee in the summer of nineteen sixty-three, +LJ042-0001|Report of the President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. +LJ042-0002|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter 7. Lee Harvey Oswald:|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter seven. Lee Harvey Oswald: +LJ042-0003|Background and Possible Motives, Part 3.|Background and Possible Motives, Part three. +LJ042-0004|Defection to the Soviet Union|Defection to the Soviet Union +LJ042-0005|After Oswald left the Marine Corps in September of 1959, ostensibly to care for his mother,|After Oswald left the Marine Corps in September of nineteen fifty-nine, ostensibly to care for his mother, +LJ042-0006|he almost immediately left for the Soviet Union where he attempted to renounce his citizenship.|he almost immediately left for the Soviet Union where he attempted to renounce his citizenship. +LJ042-0007|At the age of 19, Oswald thus committed an act which was the most striking indication he had yet given|At the age of nineteen, Oswald thus committed an act which was the most striking indication he had yet given +LJ042-0008|of his willingness to act on his beliefs in quite extraordinary ways.|of his willingness to act on his beliefs in quite extraordinary ways. +LJ042-0009|While his defection resulted in part from Oswald's commitment to Marxism,|While his defection resulted in part from Oswald's commitment to Marxism, +LJ042-0010|it appears that personal and psychological factors were also involved.|it appears that personal and psychological factors were also involved. +LJ042-0011|On August 17, 1963, Oswald told Mr. William Stuckey,|On August seventeen, nineteen sixty-three, Oswald told Mr. William Stuckey, +LJ042-0012|who had arranged a radio debate on Oswald's activities on behalf of the Fair Play for Cuba Committee,|who had arranged a radio debate on Oswald's activities on behalf of the Fair Play for Cuba Committee, +LJ042-0013|that while he had begun to read Marx and Engels at the age of 15, the conclusive thing that made him decide that Marxism was the answer|that while he had begun to read Marx and Engels at the age of fifteen, the conclusive thing that made him decide that Marxism was the answer +LJ042-0014|was his service in Japan.|was his service in Japan. +LJ042-0015|He said living conditions over there convinced him something was wrong with the system, and that possibly Marxism was the answer.|He said living conditions over there convinced him something was wrong with the system, and that possibly Marxism was the answer. +LJ042-0016|He said it was in Japan that he made up his mind to go to Russia|He said it was in Japan that he made up his mind to go to Russia +LJ042-0018|On the other hand, at least one person who knew Oswald after his return thought that his defection had a more personal and psychological basis.|On the other hand, at least one person who knew Oswald after his return thought that his defection had a more personal and psychological basis. +LJ042-0019|The validity of the latter observation is borne out by some of the things Oswald wrote in connection with his defection|The validity of the latter observation is borne out by some of the things Oswald wrote in connection with his defection +LJ042-0020|indicating that his motivation was at least in part a personal one. On November 26, 1959,|indicating that his motivation was at least in part a personal one. On November twenty-six, nineteen fifty-nine, +LJ042-0021|shortly after he arrived in the Soviet Union, and probably before Soviet authorities had given him permission to stay indefinitely,|shortly after he arrived in the Soviet Union, and probably before Soviet authorities had given him permission to stay indefinitely, +LJ042-0022|he wrote to his brother Robert that the Soviet Union was a country which, quote, I have always considered to be my own, end quote,|he wrote to his brother Robert that the Soviet Union was a country which, quote, I have always considered to be my own, end quote, +LJ042-0023|and that he went there, quote, only to find freedom. I could never have been personally happy in the U.S., end quote.|and that he went there, quote, only to find freedom. I could never have been personally happy in the U.S., end quote. +LJ042-0024|He wrote in another letter that he would, quote, never return to the United States which is a country I hate, end quote.|He wrote in another letter that he would, quote, never return to the United States which is a country I hate, end quote. +LJ042-0025|His idea that he was to find, quote, freedom, end quote, in the Soviet Union was to be rudely shattered.|His idea that he was to find, quote, freedom, end quote, in the Soviet Union was to be rudely shattered. +LJ042-0026|Whatever Oswald's reasons for going to the Soviet Union might have been, however, there can be little doubt that his desire to go was quite strong.|Whatever Oswald's reasons for going to the Soviet Union might have been, however, there can be little doubt that his desire to go was quite strong. +LJ042-0027|In addition to studying the Russian language while he was in the Marines,|In addition to studying the Russian language while he was in the Marines, +LJ042-0028|Oswald had managed to save enough money to cover the expenses of his forthcoming trip.|Oswald had managed to save enough money to cover the expenses of his forthcoming trip. +LJ042-0029|While there is no proof that he saved $1,500, as he claimed,|While there is no proof that he saved fifteen hundred dollars, as he claimed, +LJ042-0030|it would have taken considerable discipline to save whatever amount was required to finance his defection out of the salary of a low ranking enlisted man.|it would have taken considerable discipline to save whatever amount was required to finance his defection out of the salary of a low ranking enlisted man. +LJ042-0031|The extent of Oswald's desire to go to the Soviet Union|The extent of Oswald's desire to go to the Soviet Union +LJ042-0032|and of his initial commitment to that country can best be understood, however, in the context|and of his initial commitment to that country can best be understood, however, in the context +LJ042-0033|of his concomitant hatred of the United States, which was most clearly expressed in his November 26,|of his concomitant hatred of the United States, which was most clearly expressed in his November twenty-six, +LJ042-0035|and communist's would like to see the present capitalist government of the U.S. overthrown, end quote, Oswald stated|and communist's would like to see the present capitalist government of the U.S. overthrown, end quote, Oswald stated +LJ042-0036|that that government supported an economic system, quote, which exploits all its workers, end quote,|that that government supported an economic system, quote, which exploits all its workers, end quote, +LJ042-0037|and under which, quote, art, culture and the sprit of man are subjected to commercial enterprising,|and under which, quote, art, culture and the sprit of man are subjected to commercial enterprising, +LJ042-0038|and religion and education are used as a tool to suppress what would otherwise be a population questioning their government's unfair|and religion and education are used as a tool to suppress what would otherwise be a population questioning their government's unfair +LJ042-0039|economic system and plans for war, end quote.|economic system and plans for war, end quote. +LJ042-0040|He complained in his letter about segregation,|He complained in his letter about segregation, +LJ042-0041|unemployment, automation, and the use of military forces to suppress other populations.|unemployment, automation, and the use of military forces to suppress other populations. +LJ042-0042|Asking his brother why he supported the American Government and what ideals he put forward, Oswald wrote, quote,|Asking his brother why he supported the American Government and what ideals he put forward, Oswald wrote, quote, +LJ042-0043|Ask me and I will tell you I fight for communism.|Ask me and I will tell you I fight for communism. +LJ042-0044|I will not say your grandchildren will live under communism, look for yourself at history, look at a world map! America is a dicing country,|I will not say your grandchildren will live under communism, look for yourself at history, look at a world map! America is a dicing country, +LJ042-0045|I do not wish to be a part of it, nor do I ever again wish to be used as a tool in its military aggressions.|I do not wish to be a part of it, nor do I ever again wish to be used as a tool in its military aggressions. +LJ042-0046|This should answer your question, and also give you a glimpse of my way of thinking. So you speak of advantages. Do you think that is why I am here?|This should answer your question, and also give you a glimpse of my way of thinking. So you speak of advantages. Do you think that is why I am here? +LJ042-0047|For personal, material advantages? Happiness is not based on oneself, it does not consist of a small home, of taking and getting,|For personal, material advantages? Happiness is not based on oneself, it does not consist of a small home, of taking and getting, +LJ042-0048|Happiness is taking part in the struggle, where there is no borderline between one's own personal world, and the world in general.|Happiness is taking part in the struggle, where there is no borderline between one's own personal world, and the world in general. +LJ042-0049|I never believed I would find more material advantages at this stage of development in the Soviet Union than I might of had in the U.S.|I never believed I would find more material advantages at this stage of development in the Soviet Union than I might of had in the U.S. +LJ042-0050|I have been a pro-communist for years and yet I have never met a communist, instead I kept silent and observed,|I have been a pro-communist for years and yet I have never met a communist, instead I kept silent and observed, +LJ042-0051|and what I observed plus my Marxist learning brought me here to the Soviet Union.|and what I observed plus my Marxist learning brought me here to the Soviet Union. +LJ042-0052|I have always considered this country to be my own. End quote.|I have always considered this country to be my own. End quote. +LJ042-0053|Responding to Robert's statement that he had not "renounced" him,|Responding to Robert's statement that he had not "renounced" him, +LJ042-0054|Oswald told his brother, quote, on what terms I want this arrangement, end quote. He advised Robert that: 1.|Oswald told his brother, quote, on what terms I want this arrangement, end quote. He advised Robert that: one. +LJ042-0055|In the event of war I would kill any American who put a uniform on in defense of the American government -- any American.|In the event of war I would kill any American who put a uniform on in defense of the American government -- any American. +LJ042-0056|2. That in my own mind I have no attachments of any kind in the U.S.|two. That in my own mind I have no attachments of any kind in the U.S. +LJ042-0057|3. That I want to, and I shall, live a normal happy and peaceful life here in the Soviet Union for the rest of my life.|three. That I want to, and I shall, live a normal happy and peaceful life here in the Soviet Union for the rest of my life. +LJ042-0058|That my mother and you are (in spite of what the newspaper said) not objects of affection, but only examples of workers in the U.S.|That my mother and you are (in spite of what the newspaper said) not objects of affection, but only examples of workers in the U.S. +LJ042-0059|Despite this commitment to the Soviet Union|Despite this commitment to the Soviet Union +LJ042-0060|Oswald met disappointments there just as he had in the past. At the outset the Soviets told him that he could not remain.|Oswald met disappointments there just as he had in the past. At the outset the Soviets told him that he could not remain. +LJ042-0061|It seems that Oswald immediately attempted suicide -- a striking indication of how much he desired to remain in the Soviet Union.|It seems that Oswald immediately attempted suicide -- a striking indication of how much he desired to remain in the Soviet Union. +LJ042-0062|It shows how willing he was to act dramatically and decisively when he faced an emotional crisis|It shows how willing he was to act dramatically and decisively when he faced an emotional crisis +LJ042-0063|with few readily available alternatives at hand. He was shocked to find that the Soviet Union did not accept him with open arms.|with few readily available alternatives at hand. He was shocked to find that the Soviet Union did not accept him with open arms. +LJ042-0064|The entry in his self-styled, quote, Historic Diary, end quote, for October 21, 1959, reports, quote,|The entry in his self-styled, quote, Historic Diary, end quote, for October twenty-one, nineteen fifty-nine, reports, quote, +LJ042-0065|I am shocked! My dreams! I have waited for 2 years to be accepted.|I am shocked! My dreams! I have waited for two years to be accepted. +LJ042-0066|My fondest dreams are shattered because of a petty official. I decide to end it.|My fondest dreams are shattered because of a petty official. I decide to end it. +LJ042-0067|Soak fist in cold water to numb the pain. Then slash my left wrist. Than plunge wrist into bathtub of hot water.|Soak fist in cold water to numb the pain. Then slash my left wrist. Than plunge wrist into bathtub of hot water. +LJ042-0068|Somewhere, a violin plays, as I watch my life whirl away.|Somewhere, a violin plays, as I watch my life whirl away. +LJ042-0069|I think to myself "How easy to Die" and "A Sweet Death, (to violins), end quote.|I think to myself "How easy to Die" and "A Sweet Death, (to violins), end quote. +LJ042-0071|He was taken to a hospital in Moscow where he was kept until October 28, 1959.|He was taken to a hospital in Moscow where he was kept until October twenty-eight, nineteen fifty-nine. +LJ042-0072|Still intent, however, on staying in the Soviet Union,|Still intent, however, on staying in the Soviet Union, +LJ042-0073|Oswald went on October 31, to the American Embassy to renounce his U.S. citizenship.|Oswald went on October thirty-one, to the American Embassy to renounce his U.S. citizenship. +LJ042-0074|Mr. Richard E. Snyder, then Second Secretary and senior consular official at the Embassy,|Mr. Richard E. Snyder, then Second Secretary and senior consular official at the Embassy, +LJ042-0075|testified that Oswald was extremely sure of himself and seemed, quote, to know what his mission was.|testified that Oswald was extremely sure of himself and seemed, quote, to know what his mission was. +LJ042-0076|He took charge, in a sense, of the conversation right from the beginning, end quote. He presented the following signed note:|He took charge, in a sense, of the conversation right from the beginning, end quote. He presented the following signed note: +LJ042-0077|Quote, I Lee Harvey Oswald do hereby request that my present citizenship in the United States of America, be revoked.|Quote, I Lee Harvey Oswald do hereby request that my present citizenship in the United States of America, be revoked. +LJ042-0078|I have entered the Soviet Union for the express purpose of applying for citizenship in the Soviet Union, through the means of naturalization.|I have entered the Soviet Union for the express purpose of applying for citizenship in the Soviet Union, through the means of naturalization. +LJ042-0079|My request for citizenship is now pending before the Supreme Soviet of the U.S.S.R. I take these steps for political reasons.|My request for citizenship is now pending before the Supreme Soviet of the U.S.S.R. I take these steps for political reasons. +LJ042-0080|My request for the revoking of my American citizenship|My request for the revoking of my American citizenship +LJ042-0081|is made only after the longest and most serious considerations. I affirm that my allegiance is to the Union of Soviet Socialist Republics.|is made only after the longest and most serious considerations. I affirm that my allegiance is to the Union of Soviet Socialist Republics. +LJ042-0082|End quote. As his, quote, principal reason, end quote, for renouncing his citizenship|End quote. As his, quote, principal reason, end quote, for renouncing his citizenship +LJ042-0083|Oswald stated, quote, I am a Marxist, end quote. He also alluded to hardships endured by his mother as a worker,|Oswald stated, quote, I am a Marxist, end quote. He also alluded to hardships endured by his mother as a worker, +LJ042-0084|referring to them as experiences that he did not intend to have himself, even though he stated that he had never held a civilian job.|referring to them as experiences that he did not intend to have himself, even though he stated that he had never held a civilian job. +LJ042-0085|He said that his Marine service in Okinawa and elsewhere had given him, quote, a chance to observe American imperialism, end quote.|He said that his Marine service in Okinawa and elsewhere had given him, quote, a chance to observe American imperialism, end quote. +LJ042-0086|but he also displayed some sensitivity at not having reached a higher rank in the Marine Corps.|but he also displayed some sensitivity at not having reached a higher rank in the Marine Corps. +LJ042-0087|He stated that he had volunteered to give Soviet officials any information that he had concerning Marine Corps operations, and intimated|He stated that he had volunteered to give Soviet officials any information that he had concerning Marine Corps operations, and intimated +LJ042-0088|that he might know something of special interest. Oswald's "Historic Diary" describes the event in part as follows, quote,|that he might know something of special interest. Oswald's "Historic Diary" describes the event in part as follows, quote, +LJ042-0089|I leave Embassy, elated at this showdown, returning to my hotel I feel now my energies are not spent in vain.|I leave Embassy, elated at this showdown, returning to my hotel I feel now my energies are not spent in vain. +LJ042-0090|I'm sure Russians will accept me after this sign of my faith in them, end quote.|I'm sure Russians will accept me after this sign of my faith in them, end quote. +LJ042-0091|The Soviet authorities finally permitted Oswald to remain in their country.|The Soviet authorities finally permitted Oswald to remain in their country. +LJ042-0092|No evidence has been found that they used him for any particular propaganda or other political or informational purposes.|No evidence has been found that they used him for any particular propaganda or other political or informational purposes. +LJ042-0093|They sent him to Minsk to work in a radio and television factory as a metal worker.|They sent him to Minsk to work in a radio and television factory as a metal worker. +LJ042-0094|The Soviet authorities denied Oswald permission|The Soviet authorities denied Oswald permission +LJ042-0095|to attend a university in Moscow, but they gave him a monthly allowance of 700 rubles a month|to attend a university in Moscow, but they gave him a monthly allowance of seven hundred rubles a month +LJ042-0096|(old exchange rate) in addition to his factory salary of approximately equal amount|(old exchange rate) in addition to his factory salary of approximately equal amount +LJ042-0097|and considerably better living quarters than those accorded to Soviet citizens of equal age and station.|and considerably better living quarters than those accorded to Soviet citizens of equal age and station. +LJ042-0098|The subsidy, apparently similar to those sometimes given to foreigners allowed to remain in the Soviet Union, together with his salary,|The subsidy, apparently similar to those sometimes given to foreigners allowed to remain in the Soviet Union, together with his salary, +LJ042-0099|gave Oswald an income which he said approximated that of the director of the factory in which he worked.|gave Oswald an income which he said approximated that of the director of the factory in which he worked. +LJ042-0100|Even though he received more money and better living quarters than other Russians doing similar work,|Even though he received more money and better living quarters than other Russians doing similar work, +LJ042-0101|he envied his wife's uncle, a colonel in the MVD, because of the larger apartment in which he lived.|he envied his wife's uncle, a colonel in the MVD, because of the larger apartment in which he lived. +LJ042-0102|Reminiscent of his attitude toward his superiors in the Marine Corps,|Reminiscent of his attitude toward his superiors in the Marine Corps, +LJ042-0103|Oswald apparently resented the exercise of authority over him and the better treatment afforded to Communist Party officials.|Oswald apparently resented the exercise of authority over him and the better treatment afforded to Communist Party officials. +LJ042-0104|After he returned to the United States|After he returned to the United States +LJ042-0105|he took the position that the Communist Party officials in the Soviet Union were opportunists who were betraying their positions for personal gain.|he took the position that the Communist Party officials in the Soviet Union were opportunists who were betraying their positions for personal gain. +LJ042-0106|He is reported to have expressed the conclusion that they had, quote, fat stinking politicians over there just like we have over here, end quote.|He is reported to have expressed the conclusion that they had, quote, fat stinking politicians over there just like we have over here, end quote. +LJ042-0107|Oswald apparently continued to have personal difficulties while he was in Minsk.|Oswald apparently continued to have personal difficulties while he was in Minsk. +LJ042-0108|Although Marina Oswald told the Commission that her husband had good personal relationships in the Soviet Union, Katherine Ford,|Although Marina Oswald told the Commission that her husband had good personal relationships in the Soviet Union, Katherine Ford, +LJ042-0109|one of the members of the Russian community in Dallas with which the Oswalds became acquainted upon their arrival in the United States,|one of the members of the Russian community in Dallas with which the Oswalds became acquainted upon their arrival in the United States, +LJ042-0110|stated that Mrs. Oswald told her everybody in Russia, quote, hated him, end quote.|stated that Mrs. Oswald told her everybody in Russia, quote, hated him, end quote. +LJ042-0111|Jeanne De Mohrenschildt, another member of that group,|Jeanne De Mohrenschildt, another member of that group, +LJ042-0112|said that Oswald told her that he had returned because, quote, I didn't find what I was looking for, end quote.|said that Oswald told her that he had returned because, quote, I didn't find what I was looking for, end quote. +LJ042-0113|George De Mohrenschildt thought that Oswald must have become disgusted with life in the Soviet Union as the novelty|George De Mohrenschildt thought that Oswald must have become disgusted with life in the Soviet Union as the novelty +LJ042-0114|of the presence of an American wore off and he began to be less the center of attention.|of the presence of an American wore off and he began to be less the center of attention. +LJ042-0115|The best description of Oswald's state of mind, however, is set forth in his own "Historic Diary."|The best description of Oswald's state of mind, however, is set forth in his own "Historic Diary." +LJ042-0116|Under the entry for May 1, 1960,|Under the entry for May one, nineteen sixty, +LJ042-0117|he noted that one of his acquaintances, quote, relates many things I do not know about the U.S.S.R.|he noted that one of his acquaintances, quote, relates many things I do not know about the U.S.S.R. +LJ042-0118|I begin to feel uneasy inside, its true! End quote.|I begin to feel uneasy inside, its true! End quote. +LJ042-0119|Under the entry for August to September of that year he wrote, quote,|Under the entry for August to September of that year he wrote, quote, +LJ042-0120|As my Russian improves I become increasingly conscious of just what sort of a society I live in.|As my Russian improves I become increasingly conscious of just what sort of a society I live in. +LJ042-0121|Mass gymnastics, compulsory afterwork meeting, usually political information meeting.|Mass gymnastics, compulsory afterwork meeting, usually political information meeting. +LJ042-0122|Compulsory attendance at lectures and the sending of the entire shop collective (except me) to pick potatoes on a Sunday, at a state collective farm|Compulsory attendance at lectures and the sending of the entire shop collective (except me) to pick potatoes on a Sunday, at a state collective farm +LJ042-0123|A "patriotic duty" to bring in the harvest. The opinions of the workers (unvoiced) are that it's a great pain in the neck:|A "patriotic duty" to bring in the harvest. The opinions of the workers (unvoiced) are that it's a great pain in the neck: +LJ042-0124|they don't seem to be especially enthusiastic about any of the "collective" duties, a natural feeling.|they don't seem to be especially enthusiastic about any of the "collective" duties, a natural feeling. +LJ042-0125|I am increasingly aware of the presence, in all thing, of Lebizen, shop party secretary, fat, fortyish, and jovial on the outside.|I am increasingly aware of the presence, in all thing, of Lebizen, shop party secretary, fat, fortyish, and jovial on the outside. +LJ042-0126|He is a no-nonsense party regular.|He is a no-nonsense party regular. +LJ042-0127|Finally, the entry of January 4 to 31 of 1961, quote,|Finally, the entry of January four to thirty-one of nineteen sixty-one, quote, +LJ042-0128|I am stating to reconsider my desire about staying. The work is drab. The money I get has nowhere to be spent.|I am stating to reconsider my desire about staying. The work is drab. The money I get has nowhere to be spent. +LJ042-0129|No night clubs or bowling alleys, no places of recreation except the trade union dances. I have had enough.|No night clubs or bowling alleys, no places of recreation except the trade union dances. I have had enough. +LJ042-0130|Shortly thereafter, less than 18 months after his defection, about 6 weeks before he met Marina Prusakova,|Shortly thereafter, less than eighteen months after his defection, about six weeks before he met Marina Prusakova, +LJ042-0131|Oswald opened negotiations with the U.S. Embassy in Moscow looking toward his return to the United States.|Oswald opened negotiations with the U.S. Embassy in Moscow looking toward his return to the United States. +LJ042-0132|Return to the United States. In view of the intensity of his earlier commitment to the Soviet Union,|Return to the United States. In view of the intensity of his earlier commitment to the Soviet Union, +LJ042-0133|a great change must have occurred in Oswald's thinking to induce him to return to the United States.|a great change must have occurred in Oswald's thinking to induce him to return to the United States. +LJ042-0134|The psychological effects of that change must have been highly unsettling. It should be remembered|The psychological effects of that change must have been highly unsettling. It should be remembered +LJ042-0135|that he was not yet 20 years old when he went to the Soviet Union with such high hopes and not quite 23 when he returned bitterly disappointed.|that he was not yet twenty years old when he went to the Soviet Union with such high hopes and not quite twenty-three when he returned bitterly disappointed. +LJ042-0136|His attempt to renounce his citizenship had been an open expression of hostility against the United States and a profound rejection of his early life.|His attempt to renounce his citizenship had been an open expression of hostility against the United States and a profound rejection of his early life. +LJ042-0137|The dramatic break with society in America now had to be undone.|The dramatic break with society in America now had to be undone. +LJ042-0138|His return to the United States publicly testified to the utter failure of what had been the most important act of his life.|His return to the United States publicly testified to the utter failure of what had been the most important act of his life. +LJ042-0139|Marina Oswald confirmed the fact that her husband was experiencing psychological difficulties at the time of his return.|Marina Oswald confirmed the fact that her husband was experiencing psychological difficulties at the time of his return. +LJ042-0140|She said that, quote, immediately after coming to the United States, Lee changed. I did not know him as such a man in Russia, end quote.|She said that, quote, immediately after coming to the United States, Lee changed. I did not know him as such a man in Russia, end quote. +LJ042-0141|She added that while he helped her as he had done before, he became more of a recluse, that, quote,|She added that while he helped her as he had done before, he became more of a recluse, that, quote, +LJ042-0142|He was very irritable, sometimes for a trifle, end quote, and that, quote, Lee was very unrestrained and very explosive, end quote.|He was very irritable, sometimes for a trifle, end quote, and that, quote, Lee was very unrestrained and very explosive, end quote. +LJ042-0143|during the period from November 19, 1962 to March of 1963.|during the period from November nineteen, nineteen sixty-two to March of nineteen sixty-three. +LJ042-0144|After the assassination she wrote that, quote,|After the assassination she wrote that, quote, +LJ042-0145|In general, our family life began to deteriorate after we arrived in America.|In general, our family life began to deteriorate after we arrived in America. +LJ042-0146|Lee was always hot-tempered, and now this trait of character more and more prevented us from living together in harmony.|Lee was always hot-tempered, and now this trait of character more and more prevented us from living together in harmony. +LJ042-0147|Lee became very irritable, and sometimes some completely trivial thing would drive him into a rage.|Lee became very irritable, and sometimes some completely trivial thing would drive him into a rage. +LJ042-0148|I myself do not have a particularly quiet disposition,|I myself do not have a particularly quiet disposition, +LJ042-0149|but I had to change my character a great deal in order to maintain a more or less peaceful family life. End quote.|but I had to change my character a great deal in order to maintain a more or less peaceful family life. End quote. +LJ042-0150|Marina Oswald's judgment of her husband's state of mind may be substantiated by comparing material which he wrote in the Soviet Union|Marina Oswald's judgment of her husband's state of mind may be substantiated by comparing material which he wrote in the Soviet Union +LJ042-0151|with what he wrote while on the way back to the United States and after his return.|with what he wrote while on the way back to the United States and after his return. +LJ042-0152|While in the Soviet Union he wrote his longest and clearest piece of work, "The Collective."|While in the Soviet Union he wrote his longest and clearest piece of work, "The Collective." +LJ042-0153|This was a fairly coherent description of life in that country, basically centered around the radio and television factory in which he worked.|This was a fairly coherent description of life in that country, basically centered around the radio and television factory in which he worked. +LJ042-0154|While it was apparently intended for publication in the United States, and is in many respects critical of certain aspects of life in the Soviet Union,|While it was apparently intended for publication in the United States, and is in many respects critical of certain aspects of life in the Soviet Union, +LJ042-0155|it appears to be the work of a fairly well organized person.|it appears to be the work of a fairly well organized person. +LJ042-0156|Oswald prefaced his manuscript with a short autobiographical sketch which reads in part as follows, quote,|Oswald prefaced his manuscript with a short autobiographical sketch which reads in part as follows, quote, +LJ042-0157|Lee Harvey Oswald was born in October 1939 in New Orleans, Louisiana, the son of a Insurance Salesmen whose early death|Lee Harvey Oswald was born in October nineteen thirty-nine in New Orleans, Louisiana, the son of a Insurance Salesmen whose early death +LJ042-0158|left a far mean streak of independence brought on by neglect.|left a far mean streak of independence brought on by neglect. +LJ042-0159|entering the US Marine corp at 17, this streak of independence was strengthened by exotic journeys to Japan,|entering the US Marine corp at seventeen, this streak of independence was strengthened by exotic journeys to Japan, +LJ042-0160|the Philippines, and the scores of odd islands in the Pacific.|the Philippines, and the scores of odd islands in the Pacific. +LJ042-0162|Full of optimism and hope, he stood in Red Square in the Fall of 1959, vowing to see his chosen course through,|Full of optimism and hope, he stood in Red Square in the Fall of nineteen fifty-nine, vowing to see his chosen course through, +LJ042-0163|After, however, 2 years and a lot of growing up, I decided to return to the USA.|After, however, two years and a lot of growing up, I decided to return to the USA. +LJ042-0164|"The Collective" contrasts sharply with material which Oswald seems to have written after he left the Soviet Union.|"The Collective" contrasts sharply with material which Oswald seems to have written after he left the Soviet Union. +LJ042-0165|which appears to be more an expression of his own psychological condition than of a reasoned analysis.|which appears to be more an expression of his own psychological condition than of a reasoned analysis. +LJ042-0166|The latter material expresses great hostility to both communism and capitalism. He wrote, that to a person knowing both of those systems, quote,|The latter material expresses great hostility to both communism and capitalism. He wrote, that to a person knowing both of those systems, quote, +LJ042-0167|There can be no mediation between those systems as they exist today, and that person|There can be no mediation between those systems as they exist today, and that person +LJ042-0168|He must be opposed to their basic foundations and representatives, end quote.|He must be opposed to their basic foundations and representatives, end quote. +LJ042-0169|And yet, it is immature to take the sort of attitude which says, quote, a curse on both your houses! End quote.|And yet, it is immature to take the sort of attitude which says, quote, a curse on both your houses! End quote. +LJ042-0170|There are two great representatives of power in the world,|There are two great representatives of power in the world, +LJ042-0171|simply expressed, the left and right, and their offspring factions and concerns. Any practical attempt at one alternative|simply expressed, the left and right, and their offspring factions and concerns. Any practical attempt at one alternative +LJ042-0172|must have as its nucleus the traditional ideological best of both systems, and yet be utterly opposed to both systems.|must have as its nucleus the traditional ideological best of both systems, and yet be utterly opposed to both systems. +LJ042-0173|Such an alternative was to be opposed both to capitalism and communism because, quote,|Such an alternative was to be opposed both to capitalism and communism because, quote, +LJ042-0174|No man, having known, having lived, under the Russian Communist and American capitalist system,|No man, having known, having lived, under the Russian Communist and American capitalist system, +LJ042-0175|could possibly make a choice between them, there is no choice,|could possibly make a choice between them, there is no choice, +LJ042-0176|One offers oppression, the other poverty. Both offer imperialistic injustice, tinted with two brands of slavery, end quote.|One offers oppression, the other poverty. Both offer imperialistic injustice, tinted with two brands of slavery, end quote. +LJ042-0177|Oswald actually did attempt to formulate such an alternative which he planned to, quote, put forward, end quote, himself.|Oswald actually did attempt to formulate such an alternative which he planned to, quote, put forward, end quote, himself. +LJ042-0178|He thought the new alternative would have its best chance to be accepted after, quote, conflict between the two world systems leaves the world country|He thought the new alternative would have its best chance to be accepted after, quote, conflict between the two world systems leaves the world country +LJ042-0179|without defense or foundation of government, end quote,|without defense or foundation of government, end quote, +LJ042-0180|after which the survivors would, quote, seek an alternative opposed to those systems which have brought them misery, end quote.|after which the survivors would, quote, seek an alternative opposed to those systems which have brought them misery, end quote. +LJ042-0181|Oswald realized that, quote, their thinking and education will be steeped in the traditions of those systems|Oswald realized that, quote, their thinking and education will be steeped in the traditions of those systems +LJ042-0182|and they would never accept a new order complete beyond their understanding, end quote.|and they would never accept a new order complete beyond their understanding, end quote. +LJ042-0183|As a result he thought it would be, quote, necessary to oppose the old systems but at the same time support their cherished traditions, end quote.|As a result he thought it would be, quote, necessary to oppose the old systems but at the same time support their cherished traditions, end quote. +LJ042-0184|Expanding on his ideas on how his alternative to communism and capitalism might be introduced, he wrote of a, quote, readily foreseeable|Expanding on his ideas on how his alternative to communism and capitalism might be introduced, he wrote of a, quote, readily foreseeable +LJ042-0185|economic, political, or military crisis, internal or external,|economic, political, or military crisis, internal or external, +LJ042-0186|which will bring about the final destruction of the capitalist system, and indicated that|which will bring about the final destruction of the capitalist system, and indicated that +LJ042-0187|preparation in a special party could safeguard an independent course of action after the debacle, end quote.|preparation in a special party could safeguard an independent course of action after the debacle, end quote. +LJ042-0188|which would achieve the goal, which was, quote, the emplacement of a separate, democratic, pure communist society|which would achieve the goal, which was, quote, the emplacement of a separate, democratic, pure communist society +LJ042-0189|but one with union communes,|but one with union communes, +LJ042-0190|democratic socializing of production, and without regard to the twisting apart of Marxism Marxist Communism by other powers.|democratic socializing of production, and without regard to the twisting apart of Marxism Marxist Communism by other powers. +LJ042-0191|While, quote, resourcefulness and patient working towards the aforesaid goals|While, quote, resourcefulness and patient working towards the aforesaid goals +LJ042-0192|are preferred rather than loud and useless manifestations of protest, end quote, Oswald went on to note, quote,|are preferred rather than loud and useless manifestations of protest, end quote, Oswald went on to note, quote, +LJ042-0193|But these preferred tactics now may prove to be too limited in the near future,|But these preferred tactics now may prove to be too limited in the near future, +LJ042-0194|they should not be confused with slowness, indecision or fear. Only the intellectually fearless could even be remotely attracted to our doctrine,|they should not be confused with slowness, indecision or fear. Only the intellectually fearless could even be remotely attracted to our doctrine, +LJ042-0195|and yet this doctrine requires the utmost, utmost restraint, a state of being in itself majestic in power, end quote.|and yet this doctrine requires the utmost, utmost restraint, a state of being in itself majestic in power, end quote. +LJ042-0196|Oswald's decided rejection of both capitalism and communism|Oswald's decided rejection of both capitalism and communism +LJ042-0197|seemed to place him in a situation in which he could not live with satisfaction either in the United States or in the Soviet Union.|seemed to place him in a situation in which he could not live with satisfaction either in the United States or in the Soviet Union. +LJ042-0198|The discussion above has already set forth examples of his expression of hatred for the United States.|The discussion above has already set forth examples of his expression of hatred for the United States. +LJ042-0199|He also expressed hatred of the Soviet Union and of the Communist Party, U.S.A.,|He also expressed hatred of the Soviet Union and of the Communist Party, U.S.A., +LJ042-0200|even though he later referred to the latter as, quote, trusted long time fighters for progress, end quote.|even though he later referred to the latter as, quote, trusted long time fighters for progress, end quote. +LJ042-0201|He wrote, quote, The Communist Party of the United States has betrayed itself!|He wrote, quote, The Communist Party of the United States has betrayed itself! +LJ042-0202|It has turned itself into the traditional lever of a foreign power to overthrow the government of the United States;|It has turned itself into the traditional lever of a foreign power to overthrow the government of the United States; +LJ042-0203|not in the name of freedom or high ideals,|not in the name of freedom or high ideals, +LJ042-0204|but in servile conformity to the wishes of the Soviet Union and in anticipation of Soviet Russia's complete domination of the American continent.|but in servile conformity to the wishes of the Soviet Union and in anticipation of Soviet Russia's complete domination of the American continent. +LJ042-0205|There can be no sympathy for those who have turned the idea of communism into a vile curse to Western man.|There can be no sympathy for those who have turned the idea of communism into a vile curse to Western man. +LJ042-0206|The Soviets have committed crimes unsurpassed even by their early day capitalist counterparts, the imprisonment of their own peoples,|The Soviets have committed crimes unsurpassed even by their early day capitalist counterparts, the imprisonment of their own peoples, +LJ042-0207|with the mass extermination so typical of Stalin, and the individual suppression and regimentation under Khrushchev.|with the mass extermination so typical of Stalin, and the individual suppression and regimentation under Khrushchev. +LJ042-0208|The deportations, the purposeful curtailment of diet in the consumer slighted population of Russia,|The deportations, the purposeful curtailment of diet in the consumer slighted population of Russia, +LJ042-0209|the murder of history, the prostitution of art and culture, end quote.|the murder of history, the prostitution of art and culture, end quote. +LJ042-0210|A suggestion that Oswald hated more than just capitalism and communism|A suggestion that Oswald hated more than just capitalism and communism +LJ042-0211|is provided by the following, which was apparently written either on the ship coming back, or after his return from the Soviet Union, quote,|is provided by the following, which was apparently written either on the ship coming back, or after his return from the Soviet Union, quote, +LJ042-0212|I have often wondered why it is that the communist, anarchist capitalist, and even the fascist and anarchist elements in American,|I have often wondered why it is that the communist, anarchist capitalist, and even the fascist and anarchist elements in American, +LJ042-0213|always profess patriotism toward the land and the people, if not the government; although their ideals movements must surely lead|always profess patriotism toward the land and the people, if not the government; although their ideals movements must surely lead +LJ042-0214|to the bitter destruction of all and everything. I am quite sure these people must hate not only the government|to the bitter destruction of all and everything. I am quite sure these people must hate not only the government +LJ042-0215|but our, the people, culture, traditions, heritage, and very people itself, and yet|but our, the people, culture, traditions, heritage, and very people itself, and yet +LJ042-0216|they stand up and piously pronounce themselves patriots, displaying their war medals that they gained in conflicts long past between themselves.|they stand up and piously pronounce themselves patriots, displaying their war medals that they gained in conflicts long past between themselves. +LJ042-0217|I wonder what would happen it somebody was to stand up and say he was utterly opposed not only to the governments, but to the people,|I wonder what would happen it somebody was to stand up and say he was utterly opposed not only to the governments, but to the people, +LJ042-0218|to the entire land and complete foundations of his society, end quote.|to the entire land and complete foundations of his society, end quote. +LJ042-0219|Oswald demonstrated his thinking in connection with his return to the United States by preparing two sets of identical questions of the type which he might have thought|Oswald demonstrated his thinking in connection with his return to the United States by preparing two sets of identical questions of the type which he might have thought +LJ042-0220|he would be asked at a press conference when he returned.|he would be asked at a press conference when he returned. +LJ042-0221|With either great ambivalence, or cold calculation he prepared completely different answers to the same questions.|With either great ambivalence, or cold calculation he prepared completely different answers to the same questions. +LJ042-0222|Judged by his other statements and writings, however, he appears to have indicated his true feelings in the set of answers first presented|Judged by his other statements and writings, however, he appears to have indicated his true feelings in the set of answers first presented +LJ042-0223|and to have stated in the second what he thought would be least harmful to him as he resumed life in the United States.|and to have stated in the second what he thought would be least harmful to him as he resumed life in the United States. +LJ042-0224|For example, in response to his questions about his decision to go to the Soviet Union, his first draft answered, quote,|For example, in response to his questions about his decision to go to the Soviet Union, his first draft answered, quote, +LJ042-0225|as a mark of discuss and protest against American political policies in foreign countries, my personal sign of discontent|as a mark of discuss and protest against American political policies in foreign countries, my personal sign of discontent +LJ042-0226|and horror at the misguided line of reasoning of the United States Government, end quote.|and horror at the misguided line of reasoning of the United States Government, end quote. +LJ042-0227|His second answer was that he, quote, went as a citizen of the U.S. (as a tourist) residing in a foreign country which I have a perfect right to do.|His second answer was that he, quote, went as a citizen of the U.S. (as a tourist) residing in a foreign country which I have a perfect right to do. +LJ042-0228|I went there to see the land, the people and how their system works, end quote.|I went there to see the land, the people and how their system works, end quote. +LJ042-0229|To the question of, quote, Are you a Communist? End quote, he first answered "Yes,"|To the question of, quote, Are you a Communist? End quote, he first answered "Yes," +LJ042-0230|basically, although I hate the USSR and socialist system I still think marxism can work under different circumstances, end quote.|basically, although I hate the USSR and socialist system I still think marxism can work under different circumstances, end quote. +LJ042-0231|His second answer to this question was, quote, No of course not,|His second answer to this question was, quote, No of course not, +LJ042-0232|I have never even know a communist, outside of the ones in the USSR but you can't help that, end quote.|I have never even know a communist, outside of the ones in the USSR but you can't help that, end quote. +LJ042-0233|His first set of questions and answers indicated his belief that there were no outstanding differences between the Soviet Union and the United States,|His first set of questions and answers indicated his belief that there were no outstanding differences between the Soviet Union and the United States, +LJ042-0234|quote, except in the US, the living standard is a little higher.|quote, except in the US, the living standard is a little higher. +LJ042-0235|Freedoms are about the same. Medical aid and the educational system in the USSR is better than in the USA, end quote.|Freedoms are about the same. Medical aid and the educational system in the USSR is better than in the USA, end quote. +LJ042-0236|In the second simulated transcript which ended with the statement, quote, Newspapers thank you, sir. You are a real patriot! End quote.|In the second simulated transcript which ended with the statement, quote, Newspapers thank you, sir. You are a real patriot! End quote. +LJ042-0237|He apparently concluded that the United States offered, quote,|He apparently concluded that the United States offered, quote, +LJ042-0238|freedom of speech, travel outspoken opposition to unpopular policies, freedom to believe in god, end quote, while the Soviet Union did not.|freedom of speech, travel outspoken opposition to unpopular policies, freedom to believe in god, end quote, while the Soviet Union did not. +LJ042-0239|Despite the hatred that Oswald expressed toward the Soviet Union after his residence there,|Despite the hatred that Oswald expressed toward the Soviet Union after his residence there, +LJ042-0240|he continued to be interested in that country after he returned to the United States.|he continued to be interested in that country after he returned to the United States. +LJ042-0241|Soon after his arrival he wrote to the Soviet Embassy in Washington|Soon after his arrival he wrote to the Soviet Embassy in Washington +LJ042-0242|requesting information on how to subscribe to Russian newspapers and magazines and asked for, quote,|requesting information on how to subscribe to Russian newspapers and magazines and asked for, quote, +LJ042-0244|Oswald subsequently did subscribe to several Soviet journals.|Oswald subsequently did subscribe to several Soviet journals. +LJ042-0245|While Marina Oswald tried to obtain permission to return to the Soviet Union, she testified that she did so at her husband's insistence.|While Marina Oswald tried to obtain permission to return to the Soviet Union, she testified that she did so at her husband's insistence. +LJ042-0246|In July of 1963, Oswald also requested the Soviet Union to provide a visa for his return to that country.|In July of nineteen sixty-three, Oswald also requested the Soviet Union to provide a visa for his return to that country. +LJ042-0247|In August of 1963, he gave the New Orleans police as a reason for refusing to permit his family to learn English,|In August of nineteen sixty-three, he gave the New Orleans police as a reason for refusing to permit his family to learn English, +LJ042-0248|that, quote, he hated America and he did not want them to become Americanized, and that his plans were to go back to Russia, end quote.|that, quote, he hated America and he did not want them to become Americanized, and that his plans were to go back to Russia, end quote. +LJ042-0249|Even though his primary purpose probably was to get to Cuba,|Even though his primary purpose probably was to get to Cuba, +LJ042-0250|He sought an immediate grant of visa on his trip to Mexico City in late September of 1963.|He sought an immediate grant of visa on his trip to Mexico City in late September of nineteen sixty-three. +LJ042-0251|He also inquired about visas for himself and his wife|He also inquired about visas for himself and his wife +LJ043-0001|Report of the President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. +LJ043-0002|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter 7. Lee Harvey Oswald:|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter seven. Lee Harvey Oswald: +LJ043-0003|Background and Possible Motives, Part 4.|Background and Possible Motives, Part four. +LJ043-0004|Personal Relations|Personal Relations +LJ043-0005|Apart from his relatives, Oswald had no friends or close associates in Texas when he returned there in June of 1962,|Apart from his relatives, Oswald had no friends or close associates in Texas when he returned there in June of nineteen sixty-two, +LJ043-0006|and he did not establish any close friendships or associations, although it appears that he came to respect George De Mohrenschildt.|and he did not establish any close friendships or associations, although it appears that he came to respect George De Mohrenschildt. +LJ043-0007|Somewhat of a nonconformist,|Somewhat of a nonconformist, +LJ043-0008|De Mohrenschildt was a peripheral member of the so-called Russian community, with which Oswald made contact through Mr. Peter Gregory,|De Mohrenschildt was a peripheral member of the so-called Russian community, with which Oswald made contact through Mr. Peter Gregory, +LJ043-0009|a Russian-speaking petroleum engineer whom Oswald met as a result of his contact with the Texas Employment Commission office in Fort Worth.|a Russian-speaking petroleum engineer whom Oswald met as a result of his contact with the Texas Employment Commission office in Fort Worth. +LJ043-0010|Some of the members of that group saw a good deal of the Oswalds through the fall of 1963,|Some of the members of that group saw a good deal of the Oswalds through the fall of nineteen sixty-three, +LJ043-0011|and attempted to help Mrs. Oswald particularly, in various ways. In general, Oswald did not like the members of the Russian community.|and attempted to help Mrs. Oswald particularly, in various ways. In general, Oswald did not like the members of the Russian community. +LJ043-0012|In fact, his relations with some of them, particularly George Bouhe, became quite hostile.|In fact, his relations with some of them, particularly George Bouhe, became quite hostile. +LJ043-0013|Part of the problem resulted from the fact that, as Jeanne De Mohrenschildt testified,|Part of the problem resulted from the fact that, as Jeanne De Mohrenschildt testified, +LJ043-0014|Oswald was, quote, very, very disagreeable and disappointed, end quote,|Oswald was, quote, very, very disagreeable and disappointed, end quote, +LJ043-0015|He also expressed considerable resentment at the help given to his wife by her Russian-American friends.|He also expressed considerable resentment at the help given to his wife by her Russian-American friends. +LJ043-0016|Jeanne De Mohrenschildt said, quote,|Jeanne De Mohrenschildt said, quote, +LJ043-0017|Marina had a hundred dresses given to her, and he objected to that lavish help, because Marina was throwing it into his face.|Marina had a hundred dresses given to her, and he objected to that lavish help, because Marina was throwing it into his face. +LJ043-0018|He was offensive with the people. And I can understand why, because that hurt him.|He was offensive with the people. And I can understand why, because that hurt him. +LJ043-0019|He could never give her what the people were showering on her no matter how hard he worked -- and he worked very hard, end quote.|He could never give her what the people were showering on her no matter how hard he worked -- and he worked very hard, end quote. +LJ043-0020|The relations between Oswald and his wife became such that Bouhe wanted to "liberate" her from Oswald.|The relations between Oswald and his wife became such that Bouhe wanted to "liberate" her from Oswald. +LJ043-0021|While the exact sequence of events is not clear because of conflicting testimony,|While the exact sequence of events is not clear because of conflicting testimony, +LJ043-0022|it appears that De Mohrenschildt and his wife actually went to Oswald's apartment early in November of 1962|it appears that De Mohrenschildt and his wife actually went to Oswald's apartment early in November of nineteen sixty-two +LJ043-0023|and helped to move the personal effects of Marina Oswald and the baby.|and helped to move the personal effects of Marina Oswald and the baby. +LJ043-0024|Even though it appears that they may have left Oswald a few days before, it seems that he resisted the move as best he could.|Even though it appears that they may have left Oswald a few days before, it seems that he resisted the move as best he could. +LJ043-0025|He even threatened to tear up his wife's dresses and break all the baby things. According to De Mohrenschildt,|He even threatened to tear up his wife's dresses and break all the baby things. According to De Mohrenschildt, +LJ043-0026|Oswald submitted to the inevitable, presumably because he was, quote, small, you know, and he was rather a puny individual, end quote,|Oswald submitted to the inevitable, presumably because he was, quote, small, you know, and he was rather a puny individual, end quote, +LJ043-0027|De Mohrenschildt said that the whole affair made him nervous since he was, quote, interfering in other people's affairs, after all, end quote.|De Mohrenschildt said that the whole affair made him nervous since he was, quote, interfering in other people's affairs, after all, end quote. +LJ043-0028|Oswald attempted to get his wife to come back and, over Bouhe's protest, De Mohrenschildt finally told him where she was.|Oswald attempted to get his wife to come back and, over Bouhe's protest, De Mohrenschildt finally told him where she was. +LJ043-0029|De Mohrenschildt admitted that, quote,|De Mohrenschildt admitted that, quote, +LJ043-0030|If somebody did that to me, a lousy trick like that, to take my wife away, and all the furniture, I would be mad as hell, too.|If somebody did that to me, a lousy trick like that, to take my wife away, and all the furniture, I would be mad as hell, too. +LJ043-0031|I am surprised that he didn't do something worse, end quote.|I am surprised that he didn't do something worse, end quote. +LJ043-0032|After about a 2-week separation, Marina Oswald returned to her husband.|After about a two-week separation, Marina Oswald returned to her husband. +LJ043-0033|Bouhe thoroughly disapproved of this and as a result almost all communication between the Oswalds and members of the Russian community ceased.|Bouhe thoroughly disapproved of this and as a result almost all communication between the Oswalds and members of the Russian community ceased. +LJ043-0034|Contacts with De Mohrenschildt and his wife did continue and they saw the Oswalds occasionally until the spring of 1963.|Contacts with De Mohrenschildt and his wife did continue and they saw the Oswalds occasionally until the spring of nineteen sixty-three. +LJ043-0035|Shortly after his return from the Soviet Union, Oswald severed all relations with his mother;|Shortly after his return from the Soviet Union, Oswald severed all relations with his mother; +LJ043-0036|he did not see his brother Robert from Thanksgiving of 1962 until November 23, 1963.|he did not see his brother Robert from Thanksgiving of nineteen sixty-two until November twenty-three, nineteen sixty-three. +LJ043-0037|At the time of his defection, Oswald had said that neither his brother, Robert,|At the time of his defection, Oswald had said that neither his brother, Robert, +LJ043-0038|nor his mother were objects of his affection, quote, but only examples of workers in the U.S., end quote.|nor his mother were objects of his affection, quote, but only examples of workers in the U.S., end quote. +LJ043-0039|He also indicated to officials at the American Embassy in Moscow that his defection was motivated at least in part|He also indicated to officials at the American Embassy in Moscow that his defection was motivated at least in part +LJ043-0040|by so-called exploitation of his mother by the capitalist system.|by so-called exploitation of his mother by the capitalist system. +LJ043-0041|Consistent with this attitude|Consistent with this attitude +LJ043-0042|He first told his wife that he did not have a mother, but later admitted that he did but that, quote, he didn't love her very much, end quote.|He first told his wife that he did not have a mother, but later admitted that he did but that, quote, he didn't love her very much, end quote. +LJ043-0043|When they arrived from the Soviet Union, Oswald and his family lived at first with his brother Robert.|When they arrived from the Soviet Union, Oswald and his family lived at first with his brother Robert. +LJ043-0044|The latter testified that they, quote, were just together again, end quote, as if his brother, quote, had not been to Russia, end quote.|The latter testified that they, quote, were just together again, end quote, as if his brother, quote, had not been to Russia, end quote. +LJ043-0045|He also said that he and his family got along well with Marina Oswald and enjoyed showing her American things.|He also said that he and his family got along well with Marina Oswald and enjoyed showing her American things. +LJ043-0046|After about a month with his brother,|After about a month with his brother, +LJ043-0047|Oswald and his family lived for a brief period with his mother at her urging, but Oswald soon decided to move out.|Oswald and his family lived for a brief period with his mother at her urging, but Oswald soon decided to move out. +LJ043-0048|Marguerite Oswald visited her son and his family at the first apartment which he rented after his return, and tried to help them get settled there.|Marguerite Oswald visited her son and his family at the first apartment which he rented after his return, and tried to help them get settled there. +LJ043-0049|After she had bought some clothes for Marina Oswald and a highchair for the baby, Oswald emphatically told her to stop.|After she had bought some clothes for Marina Oswald and a highchair for the baby, Oswald emphatically told her to stop. +LJ043-0050|As Marguerite Oswald testified, quote,|As Marguerite Oswald testified, quote, +LJ043-0051|he strongly put me in my place about buying things for his wife that he himself could not buy, end quote.|he strongly put me in my place about buying things for his wife that he himself could not buy, end quote. +LJ043-0052|Oswald objected to his mother visiting the apartment|Oswald objected to his mother visiting the apartment +LJ043-0053|and became quite incensed with his wife when she would open the door for her in spite of his instructions to the contrary.|and became quite incensed with his wife when she would open the door for her in spite of his instructions to the contrary. +LJ043-0054|Oswald moved to Dallas on about October 8, 1962, without telling his mother where he was going.|Oswald moved to Dallas on about October eight, nineteen sixty-two, without telling his mother where he was going. +LJ043-0055|He never saw or communicated with her in any way again until she came to see him after the assassination.|He never saw or communicated with her in any way again until she came to see him after the assassination. +LJ043-0056|Even though Oswald cut off relations with his mother,|Even though Oswald cut off relations with his mother, +LJ043-0057|he attempted for the first time to learn something about his family background when he went to New Orleans in April of 1963.|he attempted for the first time to learn something about his family background when he went to New Orleans in April of nineteen sixty-three. +LJ043-0058|He visited some of his father's elderly relatives and the cemetery where his father was buried in an effort to develop the facts of his genealogy.|He visited some of his father's elderly relatives and the cemetery where his father was buried in an effort to develop the facts of his genealogy. +LJ043-0059|While it does not appear that he established any new relationships as a result of his investigation, he did obtain a large picture of his father|While it does not appear that he established any new relationships as a result of his investigation, he did obtain a large picture of his father +LJ043-0060|from one of the elderly relatives with whom he spoke. Oswald's interest in such things presents a sharp contrast with his attitude|from one of the elderly relatives with whom he spoke. Oswald's interest in such things presents a sharp contrast with his attitude +LJ043-0061|at the time of his defection, when he evidenced no interest in his father and hardly mentioned him, even when questioned.|at the time of his defection, when he evidenced no interest in his father and hardly mentioned him, even when questioned. +LJ043-0062|Oswald's defection, his interest in the Soviet Union, and his activities on behalf of the Fair Play for Cuba Committee|Oswald's defection, his interest in the Soviet Union, and his activities on behalf of the Fair Play for Cuba Committee +LJ043-0063|not only caused him difficulties in his employment relations, but they also provided him with excuses for employment failures|not only caused him difficulties in his employment relations, but they also provided him with excuses for employment failures +LJ043-0064|which were largely of his own making.|which were largely of his own making. +LJ043-0065|Oswald experienced some difficulty finding employment. Perhaps this was partially because of his lack of any specific skill or training.|Oswald experienced some difficulty finding employment. Perhaps this was partially because of his lack of any specific skill or training. +LJ043-0066|Some of his acquaintances, feeling that Oswald tried to impress people with the fact that he had lived and worked in Russia, were led to the belief|Some of his acquaintances, feeling that Oswald tried to impress people with the fact that he had lived and worked in Russia, were led to the belief +LJ043-0067|that his employment difficulties were caused by his telling prospective employers that he had last been employed in Minsk.|that his employment difficulties were caused by his telling prospective employers that he had last been employed in Minsk. +LJ043-0068|While he might have expected difficulty from such an approach, in fact|While he might have expected difficulty from such an approach, in fact +LJ043-0069|the evidence indicates that Oswald usually told his prospective employers and employment counselors that he had recently been discharged from the Marine Corps.|the evidence indicates that Oswald usually told his prospective employers and employment counselors that he had recently been discharged from the Marine Corps. +LJ043-0070|Oswald obtained a job in July of 1962 as a sheet metal worker with a company in Fort Worth.|Oswald obtained a job in July of nineteen sixty-two as a sheet metal worker with a company in Fort Worth. +LJ043-0071|His performance for that company was satisfactory.|His performance for that company was satisfactory. +LJ043-0072|Even though he told his wife that he had been fired, he voluntarily left on October 8, 1962, and moved to Dallas.|Even though he told his wife that he had been fired, he voluntarily left on October eight, nineteen sixty-two, and moved to Dallas. +LJ043-0073|On October 9, 1962 he went to the Dallas office of the Texas Employment Commission|On October nine, nineteen sixty-two he went to the Dallas office of the Texas Employment Commission +LJ043-0074|where he expressed a reluctance to work in the industrial field.|where he expressed a reluctance to work in the industrial field. +LJ043-0075|He indicated an interest in writing. An employment counselor testified, on the basis of a general aptitude test Oswald had taken,|He indicated an interest in writing. An employment counselor testified, on the basis of a general aptitude test Oswald had taken, +LJ043-0076|that he had some aptitude in that area, quote, because the verbal score is high and the clerical score is high, end quote,|that he had some aptitude in that area, quote, because the verbal score is high and the clerical score is high, end quote, +LJ043-0077|While that counselor found that he was qualified to handle many different types of jobs,|While that counselor found that he was qualified to handle many different types of jobs, +LJ043-0078|because of his need for immediate employment she attempted to obtain for him any job that was available at the time. Oswald made qualifying marks|because of his need for immediate employment she attempted to obtain for him any job that was available at the time. Oswald made qualifying marks +LJ043-0079|in 19 of 23 categories included on the general aptitude examination and scored 127 on the verbal test, as compared|in nineteen of twenty-three categories included on the general aptitude examination and scored one hundred twenty-seven on the verbal test, as compared +LJ043-0080|with 50 percent of the people taking it who score less than 100.|with fifty percent of the people taking it who score less than one hundred. +LJ043-0081|The counselor testified that there was some indication that Oswald was capable of doing college work|The counselor testified that there was some indication that Oswald was capable of doing college work +LJ043-0082|and noted that Oswald's verbal and clerical potential was, quote, outstanding, end quote.|and noted that Oswald's verbal and clerical potential was, quote, outstanding, end quote. +LJ043-0083|Employment Commission records concerning Oswald stated, quote, Well-groomed & spoken,|Employment Commission records concerning Oswald stated, quote, Well-groomed and spoken, +LJ043-0084|business suit, alert replies -- Expresses self extremely well, end quote.|business suit, alert replies -- Expresses self extremely well, end quote. +LJ043-0085|Oswald said that he hoped eventually to develop qualifications for employment as a junior executive|Oswald said that he hoped eventually to develop qualifications for employment as a junior executive +LJ043-0086|through a work-study program at a local college.|through a work-study program at a local college. +LJ043-0087|He indicated, however, that he would have to delay that program because of his immediate financial needs and responsibilities.|He indicated, however, that he would have to delay that program because of his immediate financial needs and responsibilities. +LJ043-0088|On October 11, 1962, the Employment Commission referred Oswald|On October eleven, nineteen sixty-two, the Employment Commission referred Oswald +LJ043-0089|to a commercial advertising photography firm in Dallas, where he was employed as a trainee starting October 12, 1962.|to a commercial advertising photography firm in Dallas, where he was employed as a trainee starting October twelve, nineteen sixty-two. +LJ043-0090|Even though Oswald indicated that he liked photographic work, his employer found that he was not an efficient worker.|Even though Oswald indicated that he liked photographic work, his employer found that he was not an efficient worker. +LJ043-0091|He was not able to produce photographic work|He was not able to produce photographic work +LJ043-0092|which adhered with sufficient precision to the job specifications and as a result too much of his work had to be redone.|which adhered with sufficient precision to the job specifications and as a result too much of his work had to be redone. +LJ043-0093|He also had difficulty in working with the other employees.|He also had difficulty in working with the other employees. +LJ043-0094|This was at least in part because of the close physical confines in which some of the work had to be done.|This was at least in part because of the close physical confines in which some of the work had to be done. +LJ043-0095|He did not seem to be able to make the accommodations necessary when people work under such conditions and as a result|He did not seem to be able to make the accommodations necessary when people work under such conditions and as a result +LJ043-0096|became involved in conflicts, some of which were fairly heated, with his fellow employees.|became involved in conflicts, some of which were fairly heated, with his fellow employees. +LJ043-0097|In February or March of 1963,|In February or March of nineteen sixty-three, +LJ043-0098|it began to appear that Oswald was having considerable difficulty doing accurate work and in getting along with the other employees.|it began to appear that Oswald was having considerable difficulty doing accurate work and in getting along with the other employees. +LJ043-0099|It appears that his discharge was hastened by the fact that he brought a Russian language newspaper to work.|It appears that his discharge was hastened by the fact that he brought a Russian language newspaper to work. +LJ043-0100|It is not possible to tell whether Oswald did this to provide an excuse for his eventual discharge,|It is not possible to tell whether Oswald did this to provide an excuse for his eventual discharge, +LJ043-0101|or whether he brought the Russian language newspaper with him one day after his other difficulties became clear.|or whether he brought the Russian language newspaper with him one day after his other difficulties became clear. +LJ043-0102|It is possible that his immediate supervisor noticed the newspaper at that time because his attention had otherwise been drawn more directly to Oswald.|It is possible that his immediate supervisor noticed the newspaper at that time because his attention had otherwise been drawn more directly to Oswald. +LJ043-0103|In any event, Oswald was discharged on April 6, 1963, ostensibly because of his inefficiency and difficult personality.|In any event, Oswald was discharged on April six, nineteen sixty-three, ostensibly because of his inefficiency and difficult personality. +LJ043-0104|His supervisor admitted, however,|His supervisor admitted, however, +LJ043-0105|that while he did not fire Oswald because of the newspaper incident or even weigh it heavily in his decision, quote,|that while he did not fire Oswald because of the newspaper incident or even weigh it heavily in his decision, quote, +LJ043-0106|it didn't do his case any good, end quote.|it didn't do his case any good, end quote. +LJ043-0107|Upon moving to New Orleans on April 24, 1963,|Upon moving to New Orleans on April twenty-four, nineteen sixty-three, +LJ043-0108|Oswald's employment problems became more difficult. He left his wife and child at the home of a friend, Mrs. Ruth Paine, of Irving, Texas.|Oswald's employment problems became more difficult. He left his wife and child at the home of a friend, Mrs. Ruth Paine, of Irving, Texas. +LJ043-0109|In New Orleans he obtained work as a greaser and oiler of coffee processing machines for the William B. Reily Co.,|In New Orleans he obtained work as a greaser and oiler of coffee processing machines for the William B. Reily Co., +LJ043-0110|beginning May 10, 1963.|beginning May ten, nineteen sixty-three. +LJ043-0111|After securing this job and an apartment, Oswald asked his wife to join him.|After securing this job and an apartment, Oswald asked his wife to join him. +LJ043-0112|Mrs. Paine brought Oswald's family to New Orleans.|Mrs. Paine brought Oswald's family to New Orleans. +LJ043-0113|Refusing to admit that he could only get work as a greaser,|Refusing to admit that he could only get work as a greaser, +LJ043-0114|Oswald told his wife and Mrs. Paine that he was working as a commercial photographer.|Oswald told his wife and Mrs. Paine that he was working as a commercial photographer. +LJ043-0115|He lost his job on July 19, 1963, because his work was not satisfactory|He lost his job on July nineteen, nineteen sixty-three, because his work was not satisfactory +LJ043-0116|and because he spent too much time loitering in the garage next door, where he read rifle and hunting magazines.|and because he spent too much time loitering in the garage next door, where he read rifle and hunting magazines. +LJ043-0117|Oswald apparently concluded that his Fair Play for Cuba Committee activities were not related to his discharge.|Oswald apparently concluded that his Fair Play for Cuba Committee activities were not related to his discharge. +LJ043-0118|The correctness of that conclusion is supported by the fact that he does not seem to have been publicly identified with that organization until August 9,|The correctness of that conclusion is supported by the fact that he does not seem to have been publicly identified with that organization until August nine, +LJ043-0119|1963, almost a month after he lost his job.|nineteen sixty-three, almost a month after he lost his job. +LJ043-0120|His Fair Play for Cuba Committee activities, however, made it more difficult for him to obtain other employment.|His Fair Play for Cuba Committee activities, however, made it more difficult for him to obtain other employment. +LJ043-0121|A placement interviewer of the Louisiana Department of Labor|A placement interviewer of the Louisiana Department of Labor +LJ043-0122|who had previously interviewed Oswald, saw him on television and heard a radio debate in which he engaged on August 21, 1963.|who had previously interviewed Oswald, saw him on television and heard a radio debate in which he engaged on August twenty-one, nineteen sixty-three. +LJ043-0123|He consulted with his supervisor and, quote, it was determined that we should not undertake to furnish employment references for him, end quote.|He consulted with his supervisor and, quote, it was determined that we should not undertake to furnish employment references for him, end quote. +LJ043-0124|Ironically, he failed to get a job in another photographic firm after his return to Dallas in October of 1963,|Ironically, he failed to get a job in another photographic firm after his return to Dallas in October of nineteen sixty-three, +LJ043-0125|because the president of the photographic firm for which he had previously worked|because the president of the photographic firm for which he had previously worked +LJ043-0126|told the prospective employer that Oswald was, quote, kinda peculiar sometimes and that he had some knowledge of the Russian language, end quote.|told the prospective employer that Oswald was, quote, kinda peculiar sometimes and that he had some knowledge of the Russian language, end quote. +LJ043-0127|and that he, quote, may be a damn Communist. I can't tell you. If I was you, I wouldn't hire him, end quote.|and that he, quote, may be a damn Communist. I can't tell you. If I was you, I wouldn't hire him, end quote. +LJ043-0128|The plant superintendent of the new firm testified that, one of the employees of the old firm, quote,|The plant superintendent of the new firm testified that, one of the employees of the old firm, quote, +LJ043-0129|implied that Oswald's fellow employees did not like him because he was propagandizing|implied that Oswald's fellow employees did not like him because he was propagandizing +LJ043-0130|and had been seen reading a foreign newspaper, end quote.|and had been seen reading a foreign newspaper, end quote. +LJ043-0131|As a result Oswald was not hired.|As a result Oswald was not hired. +LJ043-0132|He subsequently found a job with the Texas School Book Depository for which he performed his duties satisfactorily.|He subsequently found a job with the Texas School Book Depository for which he performed his duties satisfactorily. +LJ043-0133|Attack on General Walker|Attack on General Walker +LJ043-0134|The Commission has concluded that on April 10, 1963,|The Commission has concluded that on April ten, nineteen sixty-three, +LJ043-0135|Oswald shot at Maj. Gen. Edwin A. Walker (Resigned, U.S. Army),|Oswald shot at Maj. Gen. Edwin A. Walker (Resigned, U.S. Army), +LJ043-0136|demonstrating once again his propensity to act dramatically and, in this instance violently, in furtherance of his beliefs.|demonstrating once again his propensity to act dramatically and, in this instance violently, in furtherance of his beliefs. +LJ043-0137|The shooting occurred 2 weeks before Oswald moved to New Orleans and a few days after he had been discharged by the photographic firm.|The shooting occurred two weeks before Oswald moved to New Orleans and a few days after he had been discharged by the photographic firm. +LJ043-0138|As indicated in chapter 4, Oswald had been planning his attack on General Walker for at least 1 and perhaps as much as 2 months.|As indicated in chapter four, Oswald had been planning his attack on General Walker for at least one and perhaps as much as two months. +LJ043-0139|He outlined his plans in a notebook and studied them at considerable length before his attack.|He outlined his plans in a notebook and studied them at considerable length before his attack. +LJ043-0140|He also studied Dallas bus schedules to prepare for his later use of buses to travel to and from General Walker's house.|He also studied Dallas bus schedules to prepare for his later use of buses to travel to and from General Walker's house. +LJ043-0141|Sometime after March 27, but according to Marina Oswald, prior to April 10, 1963,|Sometime after March twenty-seven, but according to Marina Oswald, prior to April ten, nineteen sixty-three, +LJ043-0142|Oswald posed for two pictures|Oswald posed for two pictures +LJ043-0143|with his recently acquired rifle and pistol, a copy of the March 24, 1963, issue of the Worker,|with his recently acquired rifle and pistol, a copy of the March twenty-four, nineteen sixty-three, issue of the Worker, +LJ043-0144|and the March 11, 1963, issue of the Militant.|and the March eleven, nineteen sixty-three, issue of the Militant. +LJ043-0145|He told his wife that he wanted to send the pictures to the Militant and he also asked her to keep one of the pictures for his daughter, June.|He told his wife that he wanted to send the pictures to the Militant and he also asked her to keep one of the pictures for his daughter, June. +LJ043-0146|Following his unsuccessful attack on Walker, Oswald returned home.|Following his unsuccessful attack on Walker, Oswald returned home. +LJ043-0148|She testified that she was agitated because she had found the note in Oswald's room,|She testified that she was agitated because she had found the note in Oswald's room, +LJ043-0149|where she had gone, contrary to his instructions, after she became, worried about his absence.|where she had gone, contrary to his instructions, after she became, worried about his absence. +LJ043-0150|She indicated that she had no advance knowledge of Oswald's plans,|She indicated that she had no advance knowledge of Oswald's plans, +LJ043-0151|that she became quite angry when Oswald told her what he had done, and that she made him promise never to repeat such a performance.|that she became quite angry when Oswald told her what he had done, and that she made him promise never to repeat such a performance. +LJ043-0152|She said that she kept the note to use against him, quote, if something like that should be repeated again, end quote.|She said that she kept the note to use against him, quote, if something like that should be repeated again, end quote. +LJ043-0153|When asked if Oswald requested the note back she testified that, quote, He forgot about it. But apparently|When asked if Oswald requested the note back she testified that, quote, He forgot about it. But apparently +LJ043-0154|after he thought that what he had written in his book might be proof against him, and he destroyed it (the book), end quote.|after he thought that what he had written in his book might be proof against him, and he destroyed it (the book), end quote. +LJ043-0155|She later gave the following testimony. Question:|She later gave the following testimony. Question: +LJ043-0156|After he brought the rifle home, then, he showed you the book?|After he brought the rifle home, then, he showed you the book? +LJ043-0157|Answer: Yes. Question: And you said it was not a good idea to keep this book? Answer: Yes.|Answer: Yes. Question: And you said it was not a good idea to keep this book? Answer: Yes. +LJ043-0158|Question: And then he burned the book?|Question: And then he burned the book? +LJ043-0159|Answer: Yes.|Answer: Yes. +LJ043-0160|Question: Did you ask him why he had not destroyed the book before he actually went to shoot General Walker?|Question: Did you ask him why he had not destroyed the book before he actually went to shoot General Walker? +LJ043-0161|Answer: It never came to me, myself, to ask him that question.|Answer: It never came to me, myself, to ask him that question. +LJ043-0162|Marina Oswald's testimony indicates that her husband was not particularly concerned about his continued possession of the most incriminating sort of evidence.|Marina Oswald's testimony indicates that her husband was not particularly concerned about his continued possession of the most incriminating sort of evidence. +LJ043-0163|If he had been successful and had been apprehended even for routine questioning, his apartment would undoubtedly have been searched,|If he had been successful and had been apprehended even for routine questioning, his apartment would undoubtedly have been searched, +LJ043-0164|and his role would have been made clear by the evidence which he had left behind.|and his role would have been made clear by the evidence which he had left behind. +LJ043-0165|Leaving the note and picture as he did would seem to indicate that he had considered the possibility of capture.|Leaving the note and picture as he did would seem to indicate that he had considered the possibility of capture. +LJ043-0166|Possibly he might have wanted to be caught, and wanted his involvement made clear if he was in fact apprehended.|Possibly he might have wanted to be caught, and wanted his involvement made clear if he was in fact apprehended. +LJ043-0167|Even after his wife told him to destroy the notebook|Even after his wife told him to destroy the notebook +LJ043-0168|he removed at least some of the pictures which had been pasted in it and saved them among his effects, where they were found after the assassination.|he removed at least some of the pictures which had been pasted in it and saved them among his effects, where they were found after the assassination. +LJ043-0169|His behavior was entirely consistent with his wife's testimony that, quote,|His behavior was entirely consistent with his wife's testimony that, quote, +LJ043-0170|I asked him what for he was making all these entries in the book and he answered that he wanted to leave a complete record so that all the details would be in it.|I asked him what for he was making all these entries in the book and he answered that he wanted to leave a complete record so that all the details would be in it. +LJ043-0171|I am guessing that perhaps he did it to appear to be a brave man in case he were arrested, but that is my supposition, end quote.|I am guessing that perhaps he did it to appear to be a brave man in case he were arrested, but that is my supposition, end quote. +LJ043-0172|The attempt on General Walker's life deserves close attention in any consideration of Oswald's possible motive for the assassination|The attempt on General Walker's life deserves close attention in any consideration of Oswald's possible motive for the assassination +LJ043-0173|and the trail of evidence he left behind him on that occasion.|and the trail of evidence he left behind him on that occasion. +LJ043-0174|While there are differences between the two events as far as Oswald's actions and planning are concerned, there are also similarities that should be considered.|While there are differences between the two events as far as Oswald's actions and planning are concerned, there are also similarities that should be considered. +LJ043-0175|The items which Oswald left at home when he made his attack on Walker suggest a strong concern for his place in history.|The items which Oswald left at home when he made his attack on Walker suggest a strong concern for his place in history. +LJ043-0176|If the attack had succeeded and Oswald had been caught, the pictures showing him with his rifle|If the attack had succeeded and Oswald had been caught, the pictures showing him with his rifle +LJ043-0177|and his Communist and Socialist Worker's Party newspapers would probably have appeared on the front pages of newspapers or magazines all over the country,|and his Communist and Socialist Worker's Party newspapers would probably have appeared on the front pages of newspapers or magazines all over the country, +LJ043-0179|The circumstances of the attack on Walker coupled with other indications that Oswald was concerned about his place in history|The circumstances of the attack on Walker coupled with other indications that Oswald was concerned about his place in history +LJ043-0180|and with the circumstances surrounding the assassination, have led the Commission to believe|and with the circumstances surrounding the assassination, have led the Commission to believe +LJ043-0181|that such concern is an important factor to consider in assessing possible motivation for the assassination.|that such concern is an important factor to consider in assessing possible motivation for the assassination. +LJ043-0182|In any event, the Walker incident indicates that in spite of the belief among those who knew him that he was apparently not dangerous,|In any event, the Walker incident indicates that in spite of the belief among those who knew him that he was apparently not dangerous, +LJ043-0184|to carry out a carefully planned killing of another human being and was willing to consummate such a purpose if he thought there was sufficient reason to do so.|to carry out a carefully planned killing of another human being and was willing to consummate such a purpose if he thought there was sufficient reason to do so. +LJ043-0186|Marina Oswald indicated that her husband had compared General Walker to Adolph Hitler.|Marina Oswald indicated that her husband had compared General Walker to Adolph Hitler. +LJ043-0187|She testified that Oswald said that General Walker, quote, was a very bad man, that he was a fascist,|She testified that Oswald said that General Walker, quote, was a very bad man, that he was a fascist, +LJ043-0188|that he was the leader of a fascist organization, and when I said that even though all of that might be true, just the same he had no right to take his life,|that he was the leader of a fascist organization, and when I said that even though all of that might be true, just the same he had no right to take his life, +LJ044-0001|Report of the President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. +LJ044-0002|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter 7.|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. Chapter seven. +LJ044-0003|Lee Harvey Oswald: Background and Possible Motives, Part 5.|Lee Harvey Oswald: Background and Possible Motives, Part five. +LJ044-0004|Political Activities|Political Activities +LJ044-0005|Oswald's political activities after his return to the United States center around his interest in Cuba and in the Fair Play for Cuba Committee.|Oswald's political activities after his return to the United States center around his interest in Cuba and in the Fair Play for Cuba Committee. +LJ044-0006|Although, as indicated above, the Commission has been unable to find any credible evidence|Although, as indicated above, the Commission has been unable to find any credible evidence +LJ044-0007|that he was involved in any conspiracy, his political activities do provide insight into certain aspects of Oswald's character|that he was involved in any conspiracy, his political activities do provide insight into certain aspects of Oswald's character +LJ044-0008|and into his possible motivation for the assassination.|and into his possible motivation for the assassination. +LJ044-0009|While it appears that he may have distributed Fair Play for Cuba Committee materials on one uneventful occasion in Dallas sometime during the period|While it appears that he may have distributed Fair Play for Cuba Committee materials on one uneventful occasion in Dallas sometime during the period +LJ044-0010|April 6 to 24, 1963 Oswald's first public identification with that cause was in New Orleans.|April six to twenty-four, nineteen sixty-three Oswald's first public identification with that cause was in New Orleans. +LJ044-0011|There, in late May and early June of 1963, under the name Lee Osborne,|There, in late May and early June of nineteen sixty-three, under the name Lee Osborne, +LJ044-0012|he had printed a handbill headed in large letters, quote,|he had printed a handbill headed in large letters, quote, +LJ044-0013|Hands Off Cuba, end quote, an application form for, and a membership card in,|Hands Off Cuba, end quote, an application form for, and a membership card in, +LJ044-0014|the New Orleans branch of the Fair Play for Cuba Committee.|the New Orleans branch of the Fair Play for Cuba Committee. +LJ044-0015|He first distributed his handbills and other material uneventfully in the vicinity of the U.S.S. Wasp,|He first distributed his handbills and other material uneventfully in the vicinity of the U.S.S. Wasp, +LJ044-0016|which was berthed at the Dumaine Street wharf in New Orleans, on June 16, 1963.|which was berthed at the Dumaine Street wharf in New Orleans, on June sixteen, nineteen sixty-three. +LJ044-0017|He distributed literature in downtown New Orleans on August 9, 1963,|He distributed literature in downtown New Orleans on August nine, nineteen sixty-three, +LJ044-0018|and was arrested because of a dispute with three anti-Castro Cuban exiles, and again on August 16, 1963.|and was arrested because of a dispute with three anti-Castro Cuban exiles, and again on August sixteen, nineteen sixty-three. +LJ044-0019|Following his arrest, he was interviewed by the police, and at his own request, by an agent of the FBI.|Following his arrest, he was interviewed by the police, and at his own request, by an agent of the FBI. +LJ044-0020|On August 17, 1963, he appeared briefly on a radio program and on August 21, 1963,|On August seventeen, nineteen sixty-three, he appeared briefly on a radio program and on August twenty-one, nineteen sixty-three, +LJ044-0021|he debated over radio station WDSU, New Orleans, with Carlos Bringuier,|he debated over radio station WDSU, New Orleans, with Carlos Bringuier, +LJ044-0022|one of the Cuban exiles who had been arrested with him on August 9. Bringuier claimed that on August 5, 1963,|one of the Cuban exiles who had been arrested with him on August nine. Bringuier claimed that on August five, nineteen sixty-three, +LJ044-0023|Oswald had attempted to infiltrate an anti-Castro organization with which he was associated.|Oswald had attempted to infiltrate an anti-Castro organization with which he was associated. +LJ044-0024|While Oswald publicly engaged in the activities described above, his, quote, organization, end quote, was a product of his imagination.|While Oswald publicly engaged in the activities described above, his, quote, organization, end quote, was a product of his imagination. +LJ044-0025|The imaginary president of the nonexistent chapter was named A. J. Hidell, the name that Oswald used when he purchased the assassination weapon.|The imaginary president of the nonexistent chapter was named A. J. Hidell, the name that Oswald used when he purchased the assassination weapon. +LJ044-0026|Marina Oswald said she signed that name, apparently chosen because it rhymed with "Fidel,"|Marina Oswald said she signed that name, apparently chosen because it rhymed with "Fidel," +LJ044-0027|to her husband's membership card in the New Orleans chapter.|to her husband's membership card in the New Orleans chapter. +LJ044-0028|She testified that he threatened to beat her if she did not do so. The chapter had never been chartered by the national FPCC organization.|She testified that he threatened to beat her if she did not do so. The chapter had never been chartered by the national FPCC organization. +LJ044-0029|It appears to have been a solitary operation on Oswald's part in spite of his misstatements to the New Orleans police that it had 35 members,|It appears to have been a solitary operation on Oswald's part in spite of his misstatements to the New Orleans police that it had thirty-five members, +LJ044-0030|5 of which were usually present at meetings which were held once a month.|five of which were usually present at meetings which were held once a month. +LJ044-0031|Oswald's Fair Play for Cuba activities may be viewed as a very shrewd political operation|Oswald's Fair Play for Cuba activities may be viewed as a very shrewd political operation +LJ044-0032|in which one man single handedly created publicity for his cause or for himself.|in which one man single handedly created publicity for his cause or for himself. +LJ044-0033|It is also evidence of Oswald's reluctance to describe events accurately|It is also evidence of Oswald's reluctance to describe events accurately +LJ044-0034|and of his need to present himself to others as well as to himself in a light more favorable than was justified by reality.|and of his need to present himself to others as well as to himself in a light more favorable than was justified by reality. +LJ044-0035|This is suggested by his misleading and sometime untruthful statements in his letters to Mr. V. T. Lee,|This is suggested by his misleading and sometime untruthful statements in his letters to Mr. V. T. Lee, +LJ044-0036|then national director of FPCC. In one of those letters, dated August 1, 1963,|then national director of FPCC. In one of those letters, dated August one, nineteen sixty-three, +LJ044-0037|Oswald wrote that an office which he had previously claimed to have rented for FPCC activities had been, quote,|Oswald wrote that an office which he had previously claimed to have rented for FPCC activities had been, quote, +LJ044-0038|promptly closed 3 days later for some obscure reasons by the renters,|promptly closed three days later for some obscure reasons by the renters, +LJ044-0039|they said something about remodeling, etc. I'm sure you understand, end quote.|they said something about remodeling, etc. I'm sure you understand, end quote. +LJ044-0040|He wrote that, quote, thousands of circulars were distributed, end quote.|He wrote that, quote, thousands of circulars were distributed, end quote. +LJ044-0041|and that he continued to receive inquiries through his post office box which he endeavored, quote, to keep answering to the best of my ability, end quote.|and that he continued to receive inquiries through his post office box which he endeavored, quote, to keep answering to the best of my ability, end quote. +LJ044-0042|In his letter to V. T. Lee,|In his letter to V. T. Lee, +LJ044-0043|he stated that he was then alone in his efforts on behalf of FPCC, but he attributed his lack of support|he stated that he was then alone in his efforts on behalf of FPCC, but he attributed his lack of support +LJ044-0044|to an attack by Cuban exiles in a street demonstration and being, quote, officialy cautioned, end quote, by the police.|to an attack by Cuban exiles in a street demonstration and being, quote, officialy cautioned, end quote, by the police. +LJ044-0045|events which, quote, robbed me of what support I had leaving me alone, end quote.|events which, quote, robbed me of what support I had leaving me alone, end quote. +LJ044-0047|that anyone ever attacked any street demonstration in which Oswald was involved, except for the Bringuier incident mentioned above,|that anyone ever attacked any street demonstration in which Oswald was involved, except for the Bringuier incident mentioned above, +LJ044-0048|which occurred 8 days after Oswald wrote the above letter to V. T. Lee.|which occurred eight days after Oswald wrote the above letter to V. T. Lee. +LJ044-0049|Bringuier, who seemed to be familiar with many anti-Castro activities in New Orleans, was not aware of any such incident.|Bringuier, who seemed to be familiar with many anti-Castro activities in New Orleans, was not aware of any such incident. +LJ044-0050|Police reports also fail to reflect any activity on Oswald's part prior to August 9, 1963,|Police reports also fail to reflect any activity on Oswald's part prior to August nine, nineteen sixty-three, +LJ044-0051|except for the uneventful distribution of literature at the Dumaine Street wharf in June.|except for the uneventful distribution of literature at the Dumaine Street wharf in June. +LJ044-0052|Furthermore, the general tenor of Oswald's next letter to V. T. Lee,|Furthermore, the general tenor of Oswald's next letter to V. T. Lee, +LJ044-0053|in which he supported his report on the Bringuier incident with a copy of the charges made against him|in which he supported his report on the Bringuier incident with a copy of the charges made against him +LJ044-0054|and a newspaper clipping reporting the event, suggests that his previous story of an attack by Cuban exiles was at least greatly exaggerated.|and a newspaper clipping reporting the event, suggests that his previous story of an attack by Cuban exiles was at least greatly exaggerated. +LJ044-0055|While the legend, quote, FPCC, 544 Camp Street New Orleans, Louisiana, end quote,|While the legend, quote, FPCC, five four four Camp Street New Orleans, Louisiana, end quote, +LJ044-0056|was stamped on some literature that Oswald had in his possession at the time of his arrest in New Orleans,|was stamped on some literature that Oswald had in his possession at the time of his arrest in New Orleans, +LJ044-0058|that an anti-Castro organization had maintained offices there for a period ending early in 1962.|that an anti-Castro organization had maintained offices there for a period ending early in nineteen sixty-two. +LJ044-0059|The Commission has not been able to find any other indication that Oswald had rented an office in New Orleans.|The Commission has not been able to find any other indication that Oswald had rented an office in New Orleans. +LJ044-0060|In view of the limited amount of public activity on Oswald's part before August 9, 1963,|In view of the limited amount of public activity on Oswald's part before August nine, nineteen sixty-three, +LJ044-0061|there also seems to be no basis for his claim that he had distributed, quote,|there also seems to be no basis for his claim that he had distributed, quote, +LJ044-0062|thousands, end quote, of circulars, especially since he had claimed to have printed only 2,000|thousands, end quote, of circulars, especially since he had claimed to have printed only two thousand +LJ044-0063|and actually had only 1,000 printed.|and actually had only one thousand printed. +LJ044-0064|In addition, there is no evidence that he received any substantial amount of materials from the national headquarters.|In addition, there is no evidence that he received any substantial amount of materials from the national headquarters. +LJ044-0065|In another letter to V. T. Lee, dated August 17, 1963,|In another letter to V. T. Lee, dated August seventeen, nineteen sixty-three, +LJ044-0066|Oswald wrote that he had appeared on Mr. William Stuckey's 15-minute television program over WDSU-TV called, quote,|Oswald wrote that he had appeared on Mr. William Stuckey's fifteen-minute television program over WDSU-TV called, quote, +LJ044-0067|Latin American Focus, end quote,|Latin American Focus, end quote, +LJ044-0068|as a result of which he was, quote, flooded with callers and invitations to debates, etc. as well as people interested in joining the F.P.C.C.|as a result of which he was, quote, flooded with callers and invitations to debates, etc. as well as people interested in joining the F.P.C.C. +LJ044-0069|New Orleans branch, end quote.|New Orleans branch, end quote. +LJ044-0070|WDSU has no program of any kind called, quote, Latin American Focus, end quote.|WDSU has no program of any kind called, quote, Latin American Focus, end quote. +LJ044-0071|Stuckey had a radio program called, quote, Latin Listening Post, end quote,|Stuckey had a radio program called, quote, Latin Listening Post, end quote, +LJ044-0072|on which Oswald was heard for less than 5 minutes on August 17, 1963.|on which Oswald was heard for less than five minutes on August seventeen, nineteen sixty-three. +LJ044-0073|It appears that Oswald had only one caller in response to all of his FPCC activities,|It appears that Oswald had only one caller in response to all of his FPCC activities, +LJ044-0074|an agent of Bringuier's attempting to learn more about the true nature|an agent of Bringuier's attempting to learn more about the true nature +LJ044-0075|of the alleged FPCC, quote, organization, end quote, in New Orleans.|of the alleged FPCC, quote, organization, end quote, in New Orleans. +LJ044-0076|Oswald's statements suggest that he hoped to be flooded with callers and invitations to debate.|Oswald's statements suggest that he hoped to be flooded with callers and invitations to debate. +LJ044-0077|This would have made him a real center of attention as he must have been when he first arrived in the Soviet Union|This would have made him a real center of attention as he must have been when he first arrived in the Soviet Union +LJ044-0078|and as he was to some extent when he returned to the United States.|and as he was to some extent when he returned to the United States. +LJ044-0079|The limited notoriety that Oswald received as a result of the street fracas and in the subsequent radio debate|The limited notoriety that Oswald received as a result of the street fracas and in the subsequent radio debate +LJ044-0080|was apparently not enough to satisfy him. He exaggerated in his letters to V. T. Lee in an apparent attempt|was apparently not enough to satisfy him. He exaggerated in his letters to V. T. Lee in an apparent attempt +LJ044-0081|to make himself and his activities appear far more important than they really were.|to make himself and his activities appear far more important than they really were. +LJ044-0082|His attempt to express himself through his Fair Play for Cuba activities, however,|His attempt to express himself through his Fair Play for Cuba activities, however, +LJ044-0083|was greatly impeded by the fact that the radio debate over WDSU on August 21, 1963,|was greatly impeded by the fact that the radio debate over WDSU on August twenty-one, nineteen sixty-three, +LJ044-0084|brought out the history of his defection to the Soviet Union.|brought out the history of his defection to the Soviet Union. +LJ044-0085|The basic facts of the event were uncovered independently by William Stuckey, who arranged the debate, and Edward Butler,|The basic facts of the event were uncovered independently by William Stuckey, who arranged the debate, and Edward Butler, +LJ044-0086|executive director of the Information Council of the Americas, who also appeared on the program.|executive director of the Information Council of the Americas, who also appeared on the program. +LJ044-0087|Oswald was confronted with those facts at the beginning of the debate and was so thrown on the defensive by this that he was forced to state that|Oswald was confronted with those facts at the beginning of the debate and was so thrown on the defensive by this that he was forced to state that +LJ044-0088|Fair Play for Cuba was, quote, not at all Communist controlled regardless of the fact that I had the experience of living in Russia, end quote.|Fair Play for Cuba was, quote, not at all Communist controlled regardless of the fact that I had the experience of living in Russia, end quote. +LJ044-0089|Stuckey testified that uncovering Oswald's defection was very important, quote,|Stuckey testified that uncovering Oswald's defection was very important, quote, +LJ044-0090|I think that we finished him on that program because we had publicly linked the Fair Play for Cuba Committee|I think that we finished him on that program because we had publicly linked the Fair Play for Cuba Committee +LJ044-0091|with a fellow who had lived in Russia for 3 years and who was an admitted Marxist.|with a fellow who had lived in Russia for three years and who was an admitted Marxist. +LJ044-0092|The interesting thing, or rather the danger involved, was the fact that Oswald seemed like such a nice, bright boy|The interesting thing, or rather the danger involved, was the fact that Oswald seemed like such a nice, bright boy +LJ044-0093|and was extremely believable before this. We thought the fellow could probably get quite a few members if he was really indeed serious about getting members.|and was extremely believable before this. We thought the fellow could probably get quite a few members if he was really indeed serious about getting members. +LJ044-0094|We figured after this broadcast of August 21, why, that was no longer possible, end quote.|We figured after this broadcast of August twenty-one, why, that was no longer possible, end quote. +LJ044-0095|In spite of the fact that Oswald had been surprised and was on the defensive throughout the debate, according to Stuckey, quote,|In spite of the fact that Oswald had been surprised and was on the defensive throughout the debate, according to Stuckey, quote, +LJ044-0096|Mr. Oswald handled himself very well, as usual, end quote.|Mr. Oswald handled himself very well, as usual, end quote. +LJ044-0097|Stuckey thought Oswald, quote, appeared to be a very logical, intelligent fellow, end quote, and, quote, was arrested by his cleancutness, end quote.|Stuckey thought Oswald, quote, appeared to be a very logical, intelligent fellow, end quote, and, quote, was arrested by his cleancutness, end quote. +LJ044-0098|He did not think Oswald looked like the, quote, type, end quote, that he would have expected to find associating with a group such as the Fair Play for Cuba Committee.|He did not think Oswald looked like the, quote, type, end quote, that he would have expected to find associating with a group such as the Fair Play for Cuba Committee. +LJ044-0099|Stuckey thought that Oswald acted very much as would a young attorney.|Stuckey thought that Oswald acted very much as would a young attorney. +LJ044-0100|Following the disclosure of his defection, Oswald sought advice from the Communist Party, U.S.A., concerning his Fair Play for Cuba activity.|Following the disclosure of his defection, Oswald sought advice from the Communist Party, U.S.A., concerning his Fair Play for Cuba activity. +LJ044-0101|He had previously sent, apparently unsolicited, to the Party newspaper, the Worker,|He had previously sent, apparently unsolicited, to the Party newspaper, the Worker, +LJ044-0102|samples of his photographic work, offering to contribute that sort of service without charge.|samples of his photographic work, offering to contribute that sort of service without charge. +LJ044-0103|The Worker replied, quote, Your kind offer is most welcomed and from time to time we shall call on you, end quote.|The Worker replied, quote, Your kind offer is most welcomed and from time to time we shall call on you, end quote. +LJ044-0104|He later wrote to another official of the Worker, seeking employment, and mentioning the praise he had received for submitting his photographic work.|He later wrote to another official of the Worker, seeking employment, and mentioning the praise he had received for submitting his photographic work. +LJ044-0105|He presented Arnold Johnson, Gus Hall,|He presented Arnold Johnson, Gus Hall, +LJ044-0106|and Benjamin J. Davis honorary membership cards in his nonexistent New Orleans chapter of the Fair Play for Cuba Committee,|and Benjamin J. Davis honorary membership cards in his nonexistent New Orleans chapter of the Fair Play for Cuba Committee, +LJ044-0107|and advised them of some of his activities on behalf of the organization.|and advised them of some of his activities on behalf of the organization. +LJ044-0108|Arnold Johnson, director of the information and lecture bureau of the Communist Party, U.S.A., replied stating, quote,|Arnold Johnson, director of the information and lecture bureau of the Communist Party, U.S.A., replied stating, quote, +LJ044-0110|We do not have any organizational ties with the Committee, and yet there is much material that we issue from time to time that is important|We do not have any organizational ties with the Committee, and yet there is much material that we issue from time to time that is important +LJ044-0111|for anybody who is concerned about developments in Cuba, end quote,|for anybody who is concerned about developments in Cuba, end quote, +LJ044-0112|Marina Oswald said that such correspondence from people he considered important meant much to Oswald.|Marina Oswald said that such correspondence from people he considered important meant much to Oswald. +LJ044-0113|After he had begun his Cuban activity in New Orleans, quote,|After he had begun his Cuban activity in New Orleans, quote, +LJ044-0114|he received a letter from somebody in New York, some Communist -- probably from New York -- I am not sure from where -- from some Communist leader and he was very happy,|he received a letter from somebody in New York, some Communist -- probably from New York -- I am not sure from where -- from some Communist leader and he was very happy, +LJ044-0115|he felt that this was a great man that he had received the letter from, end quote.|he felt that this was a great man that he had received the letter from, end quote. +LJ044-0116|Since he seemed to feel that no one else understood his political views, the letter was of great value to him for it, quote, was proof|Since he seemed to feel that no one else understood his political views, the letter was of great value to him for it, quote, was proof +LJ044-0117|that there were people who understood his activity, end quote.|that there were people who understood his activity, end quote. +LJ044-0118|He anticipated that the full disclosure of his defection would hinder him in, quote, the struggle for progress and freedom in the United States, end quote.|He anticipated that the full disclosure of his defection would hinder him in, quote, the struggle for progress and freedom in the United States, end quote. +LJ044-0119|into which Oswald, in his own words, had, quote, thrown himself. He sought advice from the central committee of the Communist Party, U.S.A.,|into which Oswald, in his own words, had, quote, thrown himself. He sought advice from the central committee of the Communist Party, U.S.A., +LJ044-0120|in a letter dated August 28, 1963, about whether he could, quote, continue to fight,|in a letter dated August twenty-eight, nineteen sixty-three, about whether he could, quote, continue to fight, +LJ044-0121|handicapped as it were, by my past record and compete with anti-progressive forces, above-ground or whether in your opinion|handicapped as it were, by my past record and compete with anti-progressive forces, above-ground or whether in your opinion +LJ044-0122|I should always remain in the background, i.e. underground, end quote.|I should always remain in the background, i.e. underground, end quote. +LJ044-0123|Stating that he had used his "position" with what he claimed to be the local branch of the Fair Play for Cuba Committee to, quote, foster communist ideals, end quote,|Stating that he had used his "position" with what he claimed to be the local branch of the Fair Play for Cuba Committee to, quote, foster communist ideals, end quote, +LJ044-0124|Oswald wrote that he felt that he might have compromised the FPCC and expressed concern lest, quote, Our opponents could use my background|Oswald wrote that he felt that he might have compromised the FPCC and expressed concern lest, quote, Our opponents could use my background +LJ044-0125|of residence in the U.S.S.R. against any cause which I join, by association,|of residence in the U.S.S.R. against any cause which I join, by association, +LJ044-0126|they could say the organization of which I am a member, is Russian controlled, etc, end quote.|they could say the organization of which I am a member, is Russian controlled, etc, end quote. +LJ044-0127|In reply Arnold Johnson advised Oswald that, while as an American citizen he had a right to participate in such organizations as he wished, quote,|In reply Arnold Johnson advised Oswald that, while as an American citizen he had a right to participate in such organizations as he wished, quote, +LJ044-0128|there are a number of organizations, including possibly Fair Play, which are of a very broad character,|there are a number of organizations, including possibly Fair Play, which are of a very broad character, +LJ044-0129|and often it is advisable for some people to remain in the background, not underground, end quote.|and often it is advisable for some people to remain in the background, not underground, end quote. +LJ044-0130|By August of 1963, after a short 3 months in New Orleans, the city in which he had been born and had lived most of his early life,|By August of nineteen sixty-three, after a short three months in New Orleans, the city in which he had been born and had lived most of his early life, +LJ044-0131|Oswald had fallen on difficult times.|Oswald had fallen on difficult times. +LJ044-0132|He had not liked his job as a greaser of coffee processing machinery and he held it for only a little over 2 months.|He had not liked his job as a greaser of coffee processing machinery and he held it for only a little over two months. +LJ044-0133|He had not found another job. His wife was expecting their second child in October and there was concern about the cost which would be involved.|He had not found another job. His wife was expecting their second child in October and there was concern about the cost which would be involved. +LJ044-0134|His brief foray on behalf of the Fair Play for Cuba Committee had failed to win any support.|His brief foray on behalf of the Fair Play for Cuba Committee had failed to win any support. +LJ044-0135|While he had drawn some attention to himself and had actually appeared on two radio programs, he had been attacked by Cuban exiles and arrested,|While he had drawn some attention to himself and had actually appeared on two radio programs, he had been attacked by Cuban exiles and arrested, +LJ044-0136|an event which his wife thought upset him and as a result of which, quote, he became less active, he cooled off a little, end quote.|an event which his wife thought upset him and as a result of which, quote, he became less active, he cooled off a little, end quote. +LJ044-0137|More seriously, the facts of his defection had become known, leaving him open to almost unanswerable attack by those who opposed his views.|More seriously, the facts of his defection had become known, leaving him open to almost unanswerable attack by those who opposed his views. +LJ044-0138|It would not have been possible to have followed Arnold Johnson's advice to remain in the background,|It would not have been possible to have followed Arnold Johnson's advice to remain in the background, +LJ044-0139|since there was no background to the New Orleans FPCC, quote, organization, end quote, which consisted solely of Oswald.|since there was no background to the New Orleans FPCC, quote, organization, end quote, which consisted solely of Oswald. +LJ044-0140|Furthermore, he had apparently not received any letters from the national headquarters of FPCC since May 29, 1963,|Furthermore, he had apparently not received any letters from the national headquarters of FPCC since May twenty-nine, nineteen sixty-three, +LJ044-0141|even though he had written four detailed letters since that time to Mr. V. T. Lee and had also kept the national headquarters informed|even though he had written four detailed letters since that time to Mr. V. T. Lee and had also kept the national headquarters informed +LJ044-0142|of each of his changes of mailing address.|of each of his changes of mailing address. +LJ044-0143|Those events no doubt had their effects on Oswald.|Those events no doubt had their effects on Oswald. +LJ044-0144|Interest in Cuba|Interest in Cuba +LJ044-0145|By August of 1963, Oswald had for some time been considering the possibility of leaving the United States again.|By August of nineteen sixty-three, Oswald had for some time been considering the possibility of leaving the United States again. +LJ044-0146|On June 24, 1963, he applied for a new passport|On June twenty-four, nineteen sixty-three, he applied for a new passport +LJ044-0147|and in late June or early July he told his wife that he wanted to return to the Soviet Union with her. She said that he was extremely upset,|and in late June or early July he told his wife that he wanted to return to the Soviet Union with her. She said that he was extremely upset, +LJ044-0148|very unhappy, and that he actually wept when he told her that.|very unhappy, and that he actually wept when he told her that. +LJ044-0149|He said that nothing kept him in the United States, that he would not lose anything if he returned to the Soviet Union, that he wanted to be with her|He said that nothing kept him in the United States, that he would not lose anything if he returned to the Soviet Union, that he wanted to be with her +LJ044-0150|and that it would be better to have less and not have to be concerned about tomorrow.|and that it would be better to have less and not have to be concerned about tomorrow. +LJ044-0151|As a result of that conversation, Marina Oswald wrote the Soviet Embassy in Washington concerning a request she had first made|As a result of that conversation, Marina Oswald wrote the Soviet Embassy in Washington concerning a request she had first made +LJ044-0152|on February 17, 1963, for permission for herself and June to return to the Soviet Union.|on February seventeen, nineteen sixty-three, for permission for herself and June to return to the Soviet Union. +LJ044-0153|While that first request, made according to Marina Oswald at her husband's insistence, specifically stated that Oswald was to remain in the United States,|While that first request, made according to Marina Oswald at her husband's insistence, specifically stated that Oswald was to remain in the United States, +LJ044-0154|she wrote in her letter of July 1963,|she wrote in her letter of July nineteen sixty-three, +LJ044-0155|that things are improving due to the fact that my husband expresses a sincere wish to return together with me to the USSR. Unknown to his wife, however,|that things are improving due to the fact that my husband expresses a sincere wish to return together with me to the USSR. Unknown to his wife, however, +LJ044-0156|Oswald apparently enclosed a note with her letter of July in which|Oswald apparently enclosed a note with her letter of July in which +LJ044-0157|he requested the Embassy to rush his wife's entrance visa because of the impending birth of the second child but stated that, quote,|he requested the Embassy to rush his wife's entrance visa because of the impending birth of the second child but stated that, quote, +LJ044-0158|As for my return entrance visa please consider it separately. End quote.|As for my return entrance visa please consider it separately. End quote. +LJ044-0159|Thus while Oswald's real intentions, assuming that they were known to himself, are not clear,|Thus while Oswald's real intentions, assuming that they were known to himself, are not clear, +LJ044-0160|he may not have intended to go to the Soviet Union directly, if at all. It appears that he really wanted to go to Cuba.|he may not have intended to go to the Soviet Union directly, if at all. It appears that he really wanted to go to Cuba. +LJ044-0161|In his wife's words, quote,|In his wife's words, quote, +LJ044-0162|I only know that his basic desire was to get to Cuba by any means, and that all the rest of it was window dressing for that purpose. End quote.|I only know that his basic desire was to get to Cuba by any means, and that all the rest of it was window dressing for that purpose. End quote. +LJ044-0163|Marina Oswald testified that her husband engaged in Fair Play for Cuba Committee activities, quote,|Marina Oswald testified that her husband engaged in Fair Play for Cuba Committee activities, quote, +LJ044-0164|primarily for purposes of self-advertising.|primarily for purposes of self-advertising. +LJ044-0165|He wanted to be arrested. I think he wanted to get into the newspapers, so that he would be known. End quote.|He wanted to be arrested. I think he wanted to get into the newspapers, so that he would be known. End quote. +LJ044-0166|According to Marina Oswald, he thought that would help him when he got to Cuba.|According to Marina Oswald, he thought that would help him when he got to Cuba. +LJ044-0167|He asked his wife to help him to hijack an airplane to get there, but gave up that scheme when she refused.|He asked his wife to help him to hijack an airplane to get there, but gave up that scheme when she refused. +LJ044-0168|During this period Oswald may have practiced opening and closing the bolt on his rifle in a screened porch in his apartment.|During this period Oswald may have practiced opening and closing the bolt on his rifle in a screened porch in his apartment. +LJ044-0169|In September he began to review Spanish.|In September he began to review Spanish. +LJ044-0170|He approved arrangements for his family to return to Irving, Texas, to live with Mrs. Ruth Paine.|He approved arrangements for his family to return to Irving, Texas, to live with Mrs. Ruth Paine. +LJ044-0171|On September 20, 1963, Mrs. Paine and her two children arrived in New Orleans from a trip to the East Coast|On September twenty, nineteen sixty-three, Mrs. Paine and her two children arrived in New Orleans from a trip to the East Coast +LJ044-0172|and left for Irving with Marina Oswald and June and most of the Oswalds' effects 3 days later.|and left for Irving with Marina Oswald and June and most of the Oswalds' effects three days later. +LJ044-0173|While Marina Oswald knew of her husband's plan to go to Mexico and thence to Cuba if possible, Mrs. Paine was told that Oswald was going to Houston|While Marina Oswald knew of her husband's plan to go to Mexico and thence to Cuba if possible, Mrs. Paine was told that Oswald was going to Houston +LJ044-0174|and possibly to Philadelphia to look for work.|and possibly to Philadelphia to look for work. +LJ044-0175|Oswald left for Mexico City on September 25, 1963, and arrived on September 27, 1963.|Oswald left for Mexico City on September twenty-five, nineteen sixty-three, and arrived on September twenty-seven, nineteen sixty-three. +LJ044-0176|He went almost directly to the Cuban Embassy and applied for a visa to Cuba in transit to Russia.|He went almost directly to the Cuban Embassy and applied for a visa to Cuba in transit to Russia. +LJ044-0177|Representing himself as the head of the New Orleans branch of the, quote, organization called Fair Play for Cuba, end quote,|Representing himself as the head of the New Orleans branch of the, quote, organization called Fair Play for Cuba, end quote, +LJ044-0178|he stated his desire that he should be accepted as a friend of the Cuban Revolution.|he stated his desire that he should be accepted as a friend of the Cuban Revolution. +LJ044-0179|He apparently based his claim for a visa in transit to Russia|He apparently based his claim for a visa in transit to Russia +LJ044-0180|on his previous residence, his work permit for that country, and several unidentified letters in the Russian language.|on his previous residence, his work permit for that country, and several unidentified letters in the Russian language. +LJ044-0181|The Cubans would not, however, give him a visa until he had received one from the Soviets, which involved a delay of several months.|The Cubans would not, however, give him a visa until he had received one from the Soviets, which involved a delay of several months. +LJ044-0182|When faced with that situation Oswald became greatly agitated, and although he later unsuccessfully attempted|When faced with that situation Oswald became greatly agitated, and although he later unsuccessfully attempted +LJ044-0183|to obtain a Soviet visa at the Soviet Embassy in Mexico City, he insisted that he was entitled to the Cuban visa because of his background,|to obtain a Soviet visa at the Soviet Embassy in Mexico City, he insisted that he was entitled to the Cuban visa because of his background, +LJ044-0184|partisanship, and personal activities on behalf of the Cuban government.|partisanship, and personal activities on behalf of the Cuban government. +LJ044-0185|He engaged in an angry argument with the consul who finally told him that, quote, as far as he was concerned|He engaged in an angry argument with the consul who finally told him that, quote, as far as he was concerned +LJ044-0186|he would not give him a visa, end quote, and that, quote,|he would not give him a visa, end quote, and that, quote, +LJ044-0187|a person like him (Oswald) in place of aiding the Cuban Revolution, was doing it harm, end quote.|a person like him (Oswald) in place of aiding the Cuban Revolution, was doing it harm, end quote. +LJ044-0188|Oswald must have been thoroughly disillusioned when he left Mexico City on October 2, 1963.|Oswald must have been thoroughly disillusioned when he left Mexico City on October two, nineteen sixty-three. +LJ044-0189|In spite of his former residence in the Soviet Union and his Fair Play for Cuba Committee activities he had been rebuffed|In spite of his former residence in the Soviet Union and his Fair Play for Cuba Committee activities he had been rebuffed +LJ044-0190|by the officials of both Cuba and the Soviet Union in Mexico City.|by the officials of both Cuba and the Soviet Union in Mexico City. +LJ044-0191|Now there appeared to be no chance to get to Cuba, where he had thought he might find his communist ideal. The U.S. Government would not permit travel there|Now there appeared to be no chance to get to Cuba, where he had thought he might find his communist ideal. The U.S. Government would not permit travel there +LJ044-0192|and as far as the performance of the Cubans themselves was concerned, he was, quote, disappointed at not being able to get to Cuba|and as far as the performance of the Cubans themselves was concerned, he was, quote, disappointed at not being able to get to Cuba +LJ044-0193|and he didn't have any great desire to do so any more because he had run into, as he himself said -- into bureaucracy and red tape, end quote.|and he didn't have any great desire to do so any more because he had run into, as he himself said -- into bureaucracy and red tape, end quote. +LJ044-0194|Oswald's attempt to go to Cuba|Oswald's attempt to go to Cuba +LJ044-0195|was another act which expressed his hostility toward the United States and its institutions as well as a concomitant attachment|was another act which expressed his hostility toward the United States and its institutions as well as a concomitant attachment +LJ044-0196|to a country in which he must have thought were embodied the political principles to which he had been committed for so long.|to a country in which he must have thought were embodied the political principles to which he had been committed for so long. +LJ044-0197|It should be noted that his interest in Cuba seems to have increased along with the sense of frustration which must have developed|It should be noted that his interest in Cuba seems to have increased along with the sense of frustration which must have developed +LJ044-0198|as he experienced successive failures in his jobs, in his political activity, and in his personal relationships.|as he experienced successive failures in his jobs, in his political activity, and in his personal relationships. +LJ044-0199|In retrospect his attempt to go to Cuba or return to the Soviet Union may well have been Oswald's last escape hatch,|In retrospect his attempt to go to Cuba or return to the Soviet Union may well have been Oswald's last escape hatch, +LJ044-0200|his last gambit to extricate himself from the mediocrity and defeat which plagued him throughout most of his life.|his last gambit to extricate himself from the mediocrity and defeat which plagued him throughout most of his life. +LJ044-0201|Oswald's activities with regard to Cuba raise serious questions as to how much he might have been motivated in the assassination|Oswald's activities with regard to Cuba raise serious questions as to how much he might have been motivated in the assassination +LJ044-0202|by a desire to aid the Castro regime, which President Kennedy so outspokenly criticized.|by a desire to aid the Castro regime, which President Kennedy so outspokenly criticized. +LJ044-0203|For example, the Dallas Times Herald of November 19, 1963,|For example, the Dallas Times Herald of November nineteen, nineteen sixty-three, +LJ044-0204|prominently reported President Kennedy as having, quote, all but invited the Cuban people today to overthrow Fidel Castro's Communist regime|prominently reported President Kennedy as having, quote, all but invited the Cuban people today to overthrow Fidel Castro's Communist regime +LJ044-0205|and promised prompt U.S. aid if they do, end quote.|and promised prompt U.S. aid if they do, end quote. +LJ044-0206|The Castro regime severely attacked President Kennedy in connection with the Bay of Pigs affair, the Cuban missile crisis, the ban on travel to Cuba,|The Castro regime severely attacked President Kennedy in connection with the Bay of Pigs affair, the Cuban missile crisis, the ban on travel to Cuba, +LJ044-0207|the economic embargo against that country, and the general policy of the United States with regard to Cuba.|the economic embargo against that country, and the general policy of the United States with regard to Cuba. +LJ044-0208|An examination of the Militant, to which Oswald subscribed,|An examination of the Militant, to which Oswald subscribed, +LJ044-0209|for the 3-month period prior to the assassination reflects an extremely critical attitude toward President Kennedy and his administration|for the three-month period prior to the assassination reflects an extremely critical attitude toward President Kennedy and his administration +LJ044-0210|concerning Cuban policy in general as well as on the issues of automation and civil rights, issues which appeared to concern Oswald a great deal.|concerning Cuban policy in general as well as on the issues of automation and civil rights, issues which appeared to concern Oswald a great deal. +LJ044-0211|The Militant also reflected a critical attitude toward President Kennedy's attempts to reduce tensions between the United States and the Soviet Union.|The Militant also reflected a critical attitude toward President Kennedy's attempts to reduce tensions between the United States and the Soviet Union. +LJ044-0212|It also dealt with the fear of the Castro regime that such a policy might result in its abandonment by the Soviet Union.|It also dealt with the fear of the Castro regime that such a policy might result in its abandonment by the Soviet Union. +LJ044-0213|The October 7, 1963, issue of the Militant|The October seven, nineteen sixty-three, issue of the Militant +LJ044-0214|reported Castro as saying Cuba could not accept a situation where at the same time the United States was trying to ease world tensions|reported Castro as saying Cuba could not accept a situation where at the same time the United States was trying to ease world tensions +LJ044-0215|it also was increasing its efforts to tighten the noose around Cuba.|it also was increasing its efforts to tighten the noose around Cuba. +LJ044-0217|was also reported in the October 1, 1963, issue of the Worker, to which Oswald also subscribed.|was also reported in the October one, nineteen sixty-three, issue of the Worker, to which Oswald also subscribed. +LJ044-0218|In this connection it should be noted that in speaking of the Worker, Oswald told Michael Paine, apparently in all seriousness, that, quote,|In this connection it should be noted that in speaking of the Worker, Oswald told Michael Paine, apparently in all seriousness, that, quote, +LJ044-0219|you could tell what they wanted you to do by reading between the lines, reading the thing and doing a little reading between the lines, end quote.|you could tell what they wanted you to do by reading between the lines, reading the thing and doing a little reading between the lines, end quote. +LJ044-0220|The general conflict of views between the United States and Cuba was, of course, reflected in other media to such an extent|The general conflict of views between the United States and Cuba was, of course, reflected in other media to such an extent +LJ044-0221|that there can be no doubt that Oswald was aware generally of the critical attitude that Castro expressed about President Kennedy.|that there can be no doubt that Oswald was aware generally of the critical attitude that Castro expressed about President Kennedy. +LJ044-0222|Oswald was asked during the New Orleans radio debate in which he engaged on August 21, 1963,|Oswald was asked during the New Orleans radio debate in which he engaged on August twenty-one, nineteen sixty-three, +LJ044-0223|whether or not he agreed with Castro that President Kennedy was a, quote, ruffian and a thief, end quote. He replied that he, quote,|whether or not he agreed with Castro that President Kennedy was a, quote, ruffian and a thief, end quote. He replied that he, quote, +LJ044-0224|would not agree with that particular wording, end quote.|would not agree with that particular wording, end quote. +LJ044-0225|It should also be noted, however, that one witness testified that shortly before the assassination|It should also be noted, however, that one witness testified that shortly before the assassination +LJ044-0226|Oswald had expressed approval of President Kennedy's active role in the area of civil rights.|Oswald had expressed approval of President Kennedy's active role in the area of civil rights. +LJ044-0227|Although Oswald could possibly have been motivated in part by his sympathy for the Castro government,|Although Oswald could possibly have been motivated in part by his sympathy for the Castro government, +LJ044-0228|it should be remembered that his wife testified that he was disappointed with his failure to get to Cuba|it should be remembered that his wife testified that he was disappointed with his failure to get to Cuba +LJ044-0229|and had lost his desire to do so because of the bureaucracy and red tape which he had encountered.|and had lost his desire to do so because of the bureaucracy and red tape which he had encountered. +LJ044-0230|His unhappy experience with the Cuban consul seems thus to have reduced his enthusiasm for the Castro regime and his desire to go to Cuba.|His unhappy experience with the Cuban consul seems thus to have reduced his enthusiasm for the Castro regime and his desire to go to Cuba. +LJ044-0231|While some of Castro's more severe criticisms of President Kennedy might have led Oswald to believe that he would be well received in Cuba|While some of Castro's more severe criticisms of President Kennedy might have led Oswald to believe that he would be well received in Cuba +LJ044-0232|after he had assassinated the American President, it does not appear that he had any plans to go there.|after he had assassinated the American President, it does not appear that he had any plans to go there. +LJ044-0233|Oswald was carrying only $13.87 at the time of his arrest, although he had left, apparently by design,|Oswald was carrying only thirteen dollars, eighty-seven cents at the time of his arrest, although he had left, apparently by design, +LJ044-0234|$170 in a wallet in his wife's room in Irving.|one hundred seventy dollars in a wallet in his wife's room in Irving. +LJ044-0235|If there was no conspiracy which would help him escape, the possibility of which has been considered in chapter 6,|If there was no conspiracy which would help him escape, the possibility of which has been considered in chapter six, +LJ044-0236|it is unlikely that a reasoning person would plan to attempt to travel from Dallas, Texas to Cuba|it is unlikely that a reasoning person would plan to attempt to travel from Dallas, Texas to Cuba +LJ044-0237|with $13.87 when considerably greater resources were available to him.|with thirteen dollars, eighty-seven cents when considerably greater resources were available to him. +LJ044-0238|The fact that Oswald left behind the funds which might have enabled him to reach Cuba suggests the absence of any plan to try to flee there|The fact that Oswald left behind the funds which might have enabled him to reach Cuba suggests the absence of any plan to try to flee there +LJ044-0239|and raises serious questions as to whether or not he ever expected to escape.|and raises serious questions as to whether or not he ever expected to escape. +LJ045-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ045-0002|Chapter 7. Lee Harvey Oswald: Background and Possible Motives, Part 6.|Chapter seven. Lee Harvey Oswald: Background and Possible Motives, Part six. +LJ045-0003|Possible Influence of Anti-Kennedy Sentiment in Dallas|Possible Influence of Anti-Kennedy Sentiment in Dallas +LJ045-0004|It has been suggested that one of the motivating influences operating on Lee Oswald was the atmosphere in the city of Dallas,|It has been suggested that one of the motivating influences operating on Lee Oswald was the atmosphere in the city of Dallas, +LJ045-0005|especially an atmosphere of extreme opposition to President Kennedy that was present in some parts of the Dallas community|especially an atmosphere of extreme opposition to President Kennedy that was present in some parts of the Dallas community +LJ045-0006|and which received publicity there prior to the assassination.|and which received publicity there prior to the assassination. +LJ045-0007|Some of that feeling was expressed in the incident involving then vice-presidential candidate Johnson during the 1960 campaign,|Some of that feeling was expressed in the incident involving then vice-presidential candidate Johnson during the nineteen sixty campaign, +LJ045-0008|in the treatment of Ambassador Adlai Stevenson late in October of 1963|in the treatment of Ambassador Adlai Stevenson late in October of nineteen sixty-three +LJ045-0009|and in the extreme anti-Kennedy newspaper advertisement and handbills that appeared in Dallas at the time of the President's visit there.|and in the extreme anti-Kennedy newspaper advertisement and handbills that appeared in Dallas at the time of the President's visit there. +LJ045-0010|The Commission has found no evidence that the extreme views expressed toward President Kennedy|The Commission has found no evidence that the extreme views expressed toward President Kennedy +LJ045-0011|by some rightwing groups centered in Dallas or any other general atmosphere of hate|by some rightwing groups centered in Dallas or any other general atmosphere of hate +LJ045-0012|or rightwing extremism which may have existed in the city of Dallas had any connection with Oswald's actions on November 22, 1963.|or rightwing extremism which may have existed in the city of Dallas had any connection with Oswald's actions on November twenty-two, nineteen sixty-three. +LJ045-0013|There is, of course, no way to judge what the effect of the general political ferment present in that city might have been, even though Oswald was aware of it.|There is, of course, no way to judge what the effect of the general political ferment present in that city might have been, even though Oswald was aware of it. +LJ045-0014|His awareness is shown by a letter that he wrote to Arnold Johnson of the Communist Party U.S.A.,|His awareness is shown by a letter that he wrote to Arnold Johnson of the Communist Party U.S.A., +LJ045-0015|which Johnson said he did not receive until after the assassination. The letter said in part, quote,|which Johnson said he did not receive until after the assassination. The letter said in part, quote, +LJ045-0016|On October 23rd, I had attended a ultra-right meeting headed by General Edwin A. Walker, who lives in Dallas.|On October twenty-third, I had attended a ultra-right meeting headed by General Edwin A. Walker, who lives in Dallas. +LJ045-0017|This meeting preceded by one day the attack on A. E. Stevenson at the United Nations Day meeting at which he spoke|This meeting preceded by one day the attack on A. E. Stevenson at the United Nations Day meeting at which he spoke +LJ045-0018|As you can see, political friction between "left" and "right" is very great here.|As you can see, political friction between "left" and "right" is very great here. +LJ045-0019|Could you advise me as to the general view we have on the American Civil Liberties Union? End quote.|Could you advise me as to the general view we have on the American Civil Liberties Union? End quote. +LJ045-0020|In any event, the Commission has been unable to find any credible evidence that Oswald had direct contact or association with|In any event, the Commission has been unable to find any credible evidence that Oswald had direct contact or association with +LJ045-0021|any of the personalities or groups epitomizing or representing the so-called right wing,|any of the personalities or groups epitomizing or representing the so-called right wing, +LJ045-0022|even though he did, as he told Johnson, attend a meeting at which General Walker spoke to approximately 1,300 persons.|even though he did, as he told Johnson, attend a meeting at which General Walker spoke to approximately thirteen hundred persons. +LJ045-0023|Oswald's writings and his reading habits indicate that he had an extreme dislike of the right wing, an attitude most clearly reflected by his attempt|Oswald's writings and his reading habits indicate that he had an extreme dislike of the right wing, an attitude most clearly reflected by his attempt +LJ045-0024|to shoot General Walker.|to shoot General Walker. +LJ045-0025|Relationship With Wife|Relationship With Wife +LJ045-0026|The relations between Lee and Marina Oswald are of great importance in any attempt to understand Oswald's possible motivation.|The relations between Lee and Marina Oswald are of great importance in any attempt to understand Oswald's possible motivation. +LJ045-0027|During the period from Oswald's return from Mexico to the assassination,|During the period from Oswald's return from Mexico to the assassination, +LJ045-0028|he and his wife spent every weekend but one together at the Irving, Texas home of Mrs. Ruth Paine, who was then separated from her husband.|he and his wife spent every weekend but one together at the Irving, Texas home of Mrs. Ruth Paine, who was then separated from her husband. +LJ045-0029|The sole exception was the weekend of November 16 to 17, 1963, the weekend before the assassination,|The sole exception was the weekend of November sixteen to seventeen, nineteen sixty-three, the weekend before the assassination, +LJ045-0030|when his wife asked Oswald not to come to Irving.|when his wife asked Oswald not to come to Irving. +LJ045-0031|During the week, Oswald lived in a roominghouse in Dallas, but he usually called his wife on the telephone twice a day.|During the week, Oswald lived in a roominghouse in Dallas, but he usually called his wife on the telephone twice a day. +LJ045-0032|She testified that after his return from Mexico Oswald, quote, changed for the better.|She testified that after his return from Mexico Oswald, quote, changed for the better. +LJ045-0033|He began to treat me better. He helped me more -- although he always did help. But he was more attentive, end quote.|He began to treat me better. He helped me more -- although he always did help. But he was more attentive, end quote. +LJ045-0034|Marina Oswald attributed that to their living apart and to the imminent birth of their second child.|Marina Oswald attributed that to their living apart and to the imminent birth of their second child. +LJ045-0035|She testified that Oswald, quote, was very happy, end quote, about the birth of the child.|She testified that Oswald, quote, was very happy, end quote, about the birth of the child. +LJ045-0036|While those considerations no doubt had an effect on Oswald's attitude toward his family it would seem that the need for support and sympathy|While those considerations no doubt had an effect on Oswald's attitude toward his family it would seem that the need for support and sympathy +LJ045-0037|after his recent rebuffs in Mexico City|after his recent rebuffs in Mexico City +LJ045-0038|might also have been important to him. It would not have been the first time that Oswald sought closer ties with his family in time of adversity.|might also have been important to him. It would not have been the first time that Oswald sought closer ties with his family in time of adversity. +LJ045-0039|His past relationships with his wife had been stormy, however, and it did not seem that she respected him very much.|His past relationships with his wife had been stormy, however, and it did not seem that she respected him very much. +LJ045-0040|They had been married after a courtship of only about 6 weeks, a part of which Oswald spent in the hospital.|They had been married after a courtship of only about six weeks, a part of which Oswald spent in the hospital. +LJ045-0041|Oswald's diary reports that he married his wife shortly after his proposal of marriage to another girl had been rejected.|Oswald's diary reports that he married his wife shortly after his proposal of marriage to another girl had been rejected. +LJ045-0042|He stated that the other girl rejected him partly because he was an American, a fact that he said she had exploited. He stated that, quote,|He stated that the other girl rejected him partly because he was an American, a fact that he said she had exploited. He stated that, quote, +LJ045-0043|In spite of fact I married Marina to hurt Ella (the girl that had rejected him) I found myself in love with Marina, end quote.|In spite of fact I married Marina to hurt Ella (the girl that had rejected him) I found myself in love with Marina, end quote. +LJ045-0044|Many of the people with whom the Oswalds became acquainted after their arrival in the United States thought that Marina Oswald had married her husband primarily in the hope|Many of the people with whom the Oswalds became acquainted after their arrival in the United States thought that Marina Oswald had married her husband primarily in the hope +LJ045-0045|that she would be able to leave the Soviet Union. Marina Oswald has denied this.|that she would be able to leave the Soviet Union. Marina Oswald has denied this. +LJ045-0046|Marina Oswald expressed one aspect of her husband's attitude toward her when she testified that, quote, Lee wanted me to go to Russia.|Marina Oswald expressed one aspect of her husband's attitude toward her when she testified that, quote, Lee wanted me to go to Russia. +LJ045-0047|and I told him that|and I told him that +LJ045-0048|if he wanted me to go then that meant that he didn't love me, and that in that case what was the idea of coming to the United States in the first place.|if he wanted me to go then that meant that he didn't love me, and that in that case what was the idea of coming to the United States in the first place. +LJ045-0049|Lee would say that it would be better for me if I went to Russia. I did not know why. I did not know what he had in mind.|Lee would say that it would be better for me if I went to Russia. I did not know why. I did not know what he had in mind. +LJ045-0050|He said he loved me but that it would be better for me if I went to Russia, and what he had in mind I don't know. End quote.|He said he loved me but that it would be better for me if I went to Russia, and what he had in mind I don't know. End quote. +LJ045-0051|On the other hand, Oswald objected to the invitation that his wife had received to live with Mrs. Ruth Paine,|On the other hand, Oswald objected to the invitation that his wife had received to live with Mrs. Ruth Paine, +LJ045-0052|which Mrs. Paine had made in part to give her an alternative to returning to the Soviet Union. Marina Oswald wrote to Mrs. Paine that, quote,|which Mrs. Paine had made in part to give her an alternative to returning to the Soviet Union. Marina Oswald wrote to Mrs. Paine that, quote, +LJ045-0053|Many times Oswald has recalled this matter to me and said that I am just waiting for an opportunity to hurt him.|Many times Oswald has recalled this matter to me and said that I am just waiting for an opportunity to hurt him. +LJ045-0054|It has been the cause of many of our arguments. End quote. Oswald claimed that his wife preferred others to him.|It has been the cause of many of our arguments. End quote. Oswald claimed that his wife preferred others to him. +LJ045-0055|He said this about members of the Russian-speaking group in the Dallas-Ft. Worth area, whom she said he tried to forbid her from seeing,|He said this about members of the Russian-speaking group in the Dallas-Ft. Worth area, whom she said he tried to forbid her from seeing, +LJ045-0056|and also about Mrs. Paine.|and also about Mrs. Paine. +LJ045-0057|He specifically made that claim when his wife refused to come to live with him in Dallas as he asked her to do on the evening of November 21,|He specifically made that claim when his wife refused to come to live with him in Dallas as he asked her to do on the evening of November twenty-one, +LJ045-0058|1963.|nineteen sixty-three. +LJ045-0059|The instability of their relations was probably a function of the personalities of both people. Oswald was overbearing in relations with his wife.|The instability of their relations was probably a function of the personalities of both people. Oswald was overbearing in relations with his wife. +LJ045-0060|He apparently attempted to be "the Commander" by dictating many of the details of their married life.|He apparently attempted to be "the Commander" by dictating many of the details of their married life. +LJ045-0061|While Marina Oswald said that her husband wanted her to learn English,|While Marina Oswald said that her husband wanted her to learn English, +LJ045-0062|he made no attempt to help her and there are other indications that he did not want her to learn that language.|he made no attempt to help her and there are other indications that he did not want her to learn that language. +LJ045-0063|Oswald apparently wished to continue practicing his own Russian with her. Lieutenant Martello of the New Orleans police testified|Oswald apparently wished to continue practicing his own Russian with her. Lieutenant Martello of the New Orleans police testified +LJ045-0064|that Oswald stated that he did not speak English in his family because he did not want them to become Americanized.|that Oswald stated that he did not speak English in his family because he did not want them to become Americanized. +LJ045-0065|Marina Oswald's inability to speak English also made it more difficult for her to have an independent existence in this country.|Marina Oswald's inability to speak English also made it more difficult for her to have an independent existence in this country. +LJ045-0066|Oswald struck his wife on occasion,|Oswald struck his wife on occasion, +LJ045-0067|did not want her to drink, smoke or wear cosmetics and generally treated her with lack of respect in the presence of others.|did not want her to drink, smoke or wear cosmetics and generally treated her with lack of respect in the presence of others. +LJ045-0068|The difficulties which Oswald's problems would have caused him in any relationship were probably not reduced by his wife's conduct.|The difficulties which Oswald's problems would have caused him in any relationship were probably not reduced by his wife's conduct. +LJ045-0069|Katherine Ford, with whom Marina Oswald stayed during her separation from her husband in November of 1962,|Katherine Ford, with whom Marina Oswald stayed during her separation from her husband in November of nineteen sixty-two, +LJ045-0070|thought that Marina Oswald was immature in her thinking and partly responsible for the difficulties that the Oswalds were having at that time.|thought that Marina Oswald was immature in her thinking and partly responsible for the difficulties that the Oswalds were having at that time. +LJ045-0071|Mrs. Ford said that Marina Oswald admitted that she provoked Oswald on occasion.|Mrs. Ford said that Marina Oswald admitted that she provoked Oswald on occasion. +LJ045-0072|There can be little doubt that some provocation existed.|There can be little doubt that some provocation existed. +LJ045-0073|Oswald once struck his wife because of a letter which she wrote to a former boyfriend in Russia.|Oswald once struck his wife because of a letter which she wrote to a former boyfriend in Russia. +LJ045-0074|In the letter Marina Oswald stated that her husband had changed a great deal and that she was very lonely in the United States.|In the letter Marina Oswald stated that her husband had changed a great deal and that she was very lonely in the United States. +LJ045-0075|She was, quote, sorry that I had not married him (the Russian boyfriend) instead, that it would have been much easier for me, end quote.|She was, quote, sorry that I had not married him (the Russian boyfriend) instead, that it would have been much easier for me, end quote. +LJ045-0076|The letter fell into Oswald's hands when it was returned to his post office box|The letter fell into Oswald's hands when it was returned to his post office box +LJ045-0077|because of insufficient postage, which apparently resulted from an increase in postal rates of which his wife had been unaware.|because of insufficient postage, which apparently resulted from an increase in postal rates of which his wife had been unaware. +LJ045-0078|Oswald read the letter, but refused to believe that it was sincere, even though his wife insisted to him that it was.|Oswald read the letter, but refused to believe that it was sincere, even though his wife insisted to him that it was. +LJ045-0079|As a result Oswald struck her, as to which she testified, quote,|As a result Oswald struck her, as to which she testified, quote, +LJ045-0080|Generally, I think that was right, for such things that is the right thing to do. There was some grounds for it. End quote.|Generally, I think that was right, for such things that is the right thing to do. There was some grounds for it. End quote. +LJ045-0082|it appears that Marina Oswald also complained that her husband was not able to provide more material things for her.|it appears that Marina Oswald also complained that her husband was not able to provide more material things for her. +LJ045-0083|On that issue George De Mohrenschildt, who was probably as close to the Oswalds as anyone else during their first stay in Dallas,|On that issue George De Mohrenschildt, who was probably as close to the Oswalds as anyone else during their first stay in Dallas, +LJ045-0084|said that, quote, She was annoying him all the time|said that, quote, She was annoying him all the time +LJ045-0085|Why don't you make some money? Poor guy was going out of his mind. We told her she should not annoy him -- poor guy, he is doing his best, "Don't annoy him so much."|Why don't you make some money? Poor guy was going out of his mind. We told her she should not annoy him -- poor guy, he is doing his best, "Don't annoy him so much." +LJ045-0086|The De Mohrenschildts also testified that, quote, right in front, end quote, of Oswald Marina Oswald complained about Oswald's inadequacy as a husband.|The De Mohrenschildts also testified that, quote, right in front, end quote, of Oswald Marina Oswald complained about Oswald's inadequacy as a husband. +LJ045-0087|Mrs. Oswald told another of her friends that Oswald was very cold to her, that they very seldom had sexual relations|Mrs. Oswald told another of her friends that Oswald was very cold to her, that they very seldom had sexual relations +LJ045-0088|and that Oswald, quote, was not a man, end quote. She also told Mrs. Paine that she was not satisfied with her sexual relations with Oswald.|and that Oswald, quote, was not a man, end quote. She also told Mrs. Paine that she was not satisfied with her sexual relations with Oswald. +LJ045-0089|Marina Oswald also ridiculed her husband's political views, thereby tearing down his view of his own importance.|Marina Oswald also ridiculed her husband's political views, thereby tearing down his view of his own importance. +LJ045-0090|He was very much interested in autobiographical works of outstanding statesmen of the United States, to whom his wife thought he compared himself.|He was very much interested in autobiographical works of outstanding statesmen of the United States, to whom his wife thought he compared himself. +LJ045-0091|She said he was different from other people in, quote, At, least his imagination, his fantasy, which was quite unfounded,|She said he was different from other people in, quote, At, least his imagination, his fantasy, which was quite unfounded, +LJ045-0092|as to the fact that he was an outstanding man, end quote.|as to the fact that he was an outstanding man, end quote. +LJ045-0093|She said that she, quote, always tried to point out to him that he was a man like any others who were around us. But he simply could not understand that? End quote.|She said that she, quote, always tried to point out to him that he was a man like any others who were around us. But he simply could not understand that? End quote. +LJ045-0094|Jeanne De Mohrenschildt, however, thought that Marina Oswald, quote, said things that will hurt men's pride, end quote.|Jeanne De Mohrenschildt, however, thought that Marina Oswald, quote, said things that will hurt men's pride, end quote. +LJ045-0095|She said that if she ever spoke to her husband the way Marina Oswald spoke to her husband, quote, we would not last long, end quote.|She said that if she ever spoke to her husband the way Marina Oswald spoke to her husband, quote, we would not last long, end quote. +LJ045-0096|Mrs. De Mohrenschildt thought that Oswald,|Mrs. De Mohrenschildt thought that Oswald, +LJ045-0097|whom she compared to, quote, a puppy dog that everybody kicked, end quote, had a lot of good qualities, in spite of the fact that, quote,|whom she compared to, quote, a puppy dog that everybody kicked, end quote, had a lot of good qualities, in spite of the fact that, quote, +LJ045-0098|Nobody said anything good about him. End quote.|Nobody said anything good about him. End quote. +LJ045-0099|She had, quote, the impression that he was just pushed, pushed, pushed, and she Marina Oswald was probably nagging, nagging, nagging, end quote.|She had, quote, the impression that he was just pushed, pushed, pushed, and she Marina Oswald was probably nagging, nagging, nagging, end quote. +LJ045-0100|She thought that he might not have become involved in the assassination if people had been kinder to him.|She thought that he might not have become involved in the assassination if people had been kinder to him. +LJ045-0101|In spite of these difficulties, however, and in the face of the economic problems that were always with them,|In spite of these difficulties, however, and in the face of the economic problems that were always with them, +LJ045-0102|things apparently went quite smoothly from the time Oswald returned from Mexico until the weekend of November 16 to 17, 1963.|things apparently went quite smoothly from the time Oswald returned from Mexico until the weekend of November sixteen to seventeen, nineteen sixty-three. +LJ045-0103|Mrs. Paine was planning a birthday party for one of her children on that weekend and her husband, Michael, was to be at the house.|Mrs. Paine was planning a birthday party for one of her children on that weekend and her husband, Michael, was to be at the house. +LJ045-0104|Marina Oswald said that she knew her husband did not like Michael Paine and so she asked him not to come out that weekend, even though he wanted to do so.|Marina Oswald said that she knew her husband did not like Michael Paine and so she asked him not to come out that weekend, even though he wanted to do so. +LJ045-0105|She testified that she told him, quote,|She testified that she told him, quote, +LJ045-0106|that he shouldn't come every week, that perhaps it is not convenient for Ruth that the whole family be there, live there, end quote.|that he shouldn't come every week, that perhaps it is not convenient for Ruth that the whole family be there, live there, end quote. +LJ045-0107|She testified that he responded, quote, As you wish. If you don't want me to come, I won't, end quote.|She testified that he responded, quote, As you wish. If you don't want me to come, I won't, end quote. +LJ045-0108|Ruth Paine testified that she heard Marina Oswald tell Oswald about the birthday party.|Ruth Paine testified that she heard Marina Oswald tell Oswald about the birthday party. +LJ045-0109|On Sunday, November 17, 1963,|On Sunday, November seventeen, nineteen sixty-three, +LJ045-0110|Ruth Paine and Marina Oswald decided to call Oswald at the place where he was living, unbeknownst to them, under the name of O. H. Lee.|Ruth Paine and Marina Oswald decided to call Oswald at the place where he was living, unbeknownst to them, under the name of O. H. Lee. +LJ045-0111|They asked for Lee Oswald who was not called to the telephone because he was known by the other name.|They asked for Lee Oswald who was not called to the telephone because he was known by the other name. +LJ045-0112|When Oswald called the next day his wife became very angry about his use of the alias. He said that he used it because, quote,|When Oswald called the next day his wife became very angry about his use of the alias. He said that he used it because, quote, +LJ045-0113|he did not want his landlady to know his real name because she might read in the paper of the fact that he had been in Russia and that he had been questioned, end quote.|he did not want his landlady to know his real name because she might read in the paper of the fact that he had been in Russia and that he had been questioned, end quote. +LJ045-0114|Oswald also said that he did not want the FBI to know where he lived, quote, Because their visits were not very pleasant for him|Oswald also said that he did not want the FBI to know where he lived, quote, Because their visits were not very pleasant for him +LJ045-0115|and he thought that he loses jobs because the FBI visits the place of his employment, end quote.|and he thought that he loses jobs because the FBI visits the place of his employment, end quote. +LJ045-0116|While the facts of his defection had become known in New Orleans as a result of his radio debate with Bringuier,|While the facts of his defection had become known in New Orleans as a result of his radio debate with Bringuier, +LJ045-0117|it would appear to be unlikely that his landlady in Dallas|it would appear to be unlikely that his landlady in Dallas +LJ045-0118|would see anything in the newspaper about his defection, unless he engaged in activities similar to those|would see anything in the newspaper about his defection, unless he engaged in activities similar to those +LJ045-0119|which had led to the disclosure of his defection in New Orleans.|which had led to the disclosure of his defection in New Orleans. +LJ045-0120|Furthermore, even though it appears that at times Oswald was really upset by visits of the FBI,|Furthermore, even though it appears that at times Oswald was really upset by visits of the FBI, +LJ045-0121|it does not appear that he ever lost his job because of its activities, although he may well not have been aware of that fact.|it does not appear that he ever lost his job because of its activities, although he may well not have been aware of that fact. +LJ045-0122|While Oswald's concern about the FBI had some basis in fact, in that FBI agents had interviewed him in the past and had renewed their interest|While Oswald's concern about the FBI had some basis in fact, in that FBI agents had interviewed him in the past and had renewed their interest +LJ045-0123|to some extent after his Fair Play for Cuba Committee activities had become known, he exaggerated their concern for him.|to some extent after his Fair Play for Cuba Committee activities had become known, he exaggerated their concern for him. +LJ045-0124|Marina Oswald thought he did so in order to emphasize his importance.|Marina Oswald thought he did so in order to emphasize his importance. +LJ045-0125|For example, in his letter of November 9, 1963,|For example, in his letter of November nine, nineteen sixty-three, +LJ045-0126|to the Soviet Embassy in Washington, he asked about the entrance visas for which he and his wife had previously applied.|to the Soviet Embassy in Washington, he asked about the entrance visas for which he and his wife had previously applied. +LJ045-0127|He absolved the Soviet Embassy in Mexico City of any blame for his difficulties there. He advised the Washington Embassy|He absolved the Soviet Embassy in Mexico City of any blame for his difficulties there. He advised the Washington Embassy +LJ045-0128|that the FBI was, quote, not now, end quote, interested in his Fair Play for Cuba Committee activities, but noted that the FBI, quote,|that the FBI was, quote, not now, end quote, interested in his Fair Play for Cuba Committee activities, but noted that the FBI, quote, +LJ045-0129|has visited us here in Dallas, Texas, on November 1. Agent James P. Hasty|has visited us here in Dallas, Texas, on November one. Agent James P. Hasty +LJ045-0130|warned me that if I engaged in F.P.C.C. activities in Texas the F.B.I. will again take an interest in me, end quote.|warned me that if I engaged in F.P.C.C. activities in Texas the F.B.I. will again take an interest in me, end quote. +LJ045-0131|Neither Hosty nor any other agent of the FBI spoke to Oswald on any subject from August 10, 1963,|Neither Hosty nor any other agent of the FBI spoke to Oswald on any subject from August ten, nineteen sixty-three, +LJ045-0132|to the time of the assassination.|to the time of the assassination. +LJ045-0133|The claimed warning was one more of Oswald's fabrications.|The claimed warning was one more of Oswald's fabrications. +LJ045-0134|Hosty had come to the Paine residence on November 1 and 5, 1963,|Hosty had come to the Paine residence on November one and five, nineteen sixty-three, +LJ045-0135|but did not issue any such warning or suggest that Marina Oswald defect from the Soviet Union and remain in the United States under FBI protection,|but did not issue any such warning or suggest that Marina Oswald defect from the Soviet Union and remain in the United States under FBI protection, +LJ045-0136|as Oswald went on to say. In Oswald's imagination, quote,|as Oswald went on to say. In Oswald's imagination, quote, +LJ045-0137|I and my wife strongly protested these tactics by the notorious F.B.I., end quote.|I and my wife strongly protested these tactics by the notorious F.B.I., end quote. +LJ045-0138|In fact, his wife testified that she only said that she would prefer not to receive any more visits from the Bureau|In fact, his wife testified that she only said that she would prefer not to receive any more visits from the Bureau +LJ045-0139|because of the, quote, very exciting and disturbing effect, end quote, they had upon her husband, who was not even present at that time.|because of the, quote, very exciting and disturbing effect, end quote, they had upon her husband, who was not even present at that time. +LJ045-0140|The arguments he used to justify his use of the alias suggest that Oswald may have come to think that the whole world was becoming involved|The arguments he used to justify his use of the alias suggest that Oswald may have come to think that the whole world was becoming involved +LJ045-0141|in an increasingly complex conspiracy against him.|in an increasingly complex conspiracy against him. +LJ045-0142|He may have felt he could never tell when the FBI was going to appear on the scene or who else was going to find out about his defection|He may have felt he could never tell when the FBI was going to appear on the scene or who else was going to find out about his defection +LJ045-0143|and use it against him as had been done in New Orleans.|and use it against him as had been done in New Orleans. +LJ045-0144|On the other hand, the concern he expressed about the FBI may have been just another story to support the objective he sought in his letter.|On the other hand, the concern he expressed about the FBI may have been just another story to support the objective he sought in his letter. +LJ045-0145|Those arguments, however, were not persuasive to Marina Oswald,|Those arguments, however, were not persuasive to Marina Oswald, +LJ045-0146|to whom, quote, it was nothing terrible if people were to find out that he had been in Russia, end quote.|to whom, quote, it was nothing terrible if people were to find out that he had been in Russia, end quote. +LJ045-0148|After all, when will all your foolishness come to an end? All of these comedies. First one thing and then another. And now this fictitious name, end quote.|After all, when will all your foolishness come to an end? All of these comedies. First one thing and then another. And now this fictitious name, end quote. +LJ045-0149|She said, quote,|She said, quote, +LJ045-0150|On Monday, November 18, 1963, he called several times, but after I hung up on him and didn't want to talk to him he did not call again.|On Monday, November eighteen, nineteen sixty-three, he called several times, but after I hung up on him and didn't want to talk to him he did not call again. +LJ045-0151|He then arrived on Thursday, November 21, 1963, end quote.|He then arrived on Thursday, November twenty-one, nineteen sixty-three, end quote. +LJ045-0152|The events of that evening can best be appreciated through Marina Oswald's testimony|The events of that evening can best be appreciated through Marina Oswald's testimony +LJ045-0153|Question: Did your husband give any reason for coming home on Thursday?|Question: Did your husband give any reason for coming home on Thursday? +LJ045-0154|Answer: He said that he was lonely because he hadn't come the preceding weekend, and he wanted to make his peace with me.|Answer: He said that he was lonely because he hadn't come the preceding weekend, and he wanted to make his peace with me. +LJ045-0155|Question: Did you say anything to him then?|Question: Did you say anything to him then? +LJ045-0156|Answer: He tried to talk to me but I would not answer him, and he was very upset. Question: Were you upset with him?|Answer: He tried to talk to me but I would not answer him, and he was very upset. Question: Were you upset with him? +LJ045-0157|Answer: I was angry, of course. He was not angry -- he was upset. I was angry. He tried very hard to please me.|Answer: I was angry, of course. He was not angry -- he was upset. I was angry. He tried very hard to please me. +LJ045-0158|He spent quite a bit of time putting away diapers and played with the children on the street.|He spent quite a bit of time putting away diapers and played with the children on the street. +LJ045-0159|Question: How did you indicate to him that you were angry with him? Answer: By not talking to him.|Question: How did you indicate to him that you were angry with him? Answer: By not talking to him. +LJ045-0160|Question: And how did he show that he was upset?|Question: And how did he show that he was upset? +LJ045-0161|He was upset over the fact that I would not answer him.|He was upset over the fact that I would not answer him. +LJ045-0162|He tried to start a conversation with me several times, but I would not answer. And he said that he didn't want me to be angry at him because this upsets him.|He tried to start a conversation with me several times, but I would not answer. And he said that he didn't want me to be angry at him because this upsets him. +LJ045-0163|On that day, he suggested that we rent an apartment in Dallas.|On that day, he suggested that we rent an apartment in Dallas. +LJ045-0164|He said that he was tired of living alone and perhaps the reason for my being so angry was the fact that we were not living together.|He said that he was tired of living alone and perhaps the reason for my being so angry was the fact that we were not living together. +LJ045-0165|That if I want to he would rent an apartment in Dallas tomorrow -- that he didn't want me to remain with Ruth any longer, but wanted me to live with him in Dallas.|That if I want to he would rent an apartment in Dallas tomorrow -- that he didn't want me to remain with Ruth any longer, but wanted me to live with him in Dallas. +LJ045-0166|He repeated this not once but several times, but I refused. And he said that once again I was preferring my friends to him, and that I didn't need him.|He repeated this not once but several times, but I refused. And he said that once again I was preferring my friends to him, and that I didn't need him. +LJ045-0167|Question: What did you say to that?|Question: What did you say to that? +LJ045-0168|Answer: I said it would be better if I remained with Ruth until the holidays, he would come, and we would all meet together.|Answer: I said it would be better if I remained with Ruth until the holidays, he would come, and we would all meet together. +LJ045-0169|That this was better because while he was living alone and I stayed with Ruth, we were spending less money. And I told him to buy me a washing machine, because two children|That this was better because while he was living alone and I stayed with Ruth, we were spending less money. And I told him to buy me a washing machine, because two children +LJ045-0170|it became too difficult to wash by hand.|it became too difficult to wash by hand. +LJ045-0171|What did he say to that?|What did he say to that? +LJ045-0172|Answer: He said he would buy me a washing machine.|Answer: He said he would buy me a washing machine. +LJ045-0173|Question: What did you say to that? Answer:|Question: What did you say to that? Answer: +LJ045-0174|Thank you. That it would be better if he bought something for himself -- that I would manage. End quote. That night Oswald went to bed before his wife retired.|Thank you. That it would be better if he bought something for himself -- that I would manage. End quote. That night Oswald went to bed before his wife retired. +LJ045-0175|She did not speak to him when she joined him there, although she thought that he was still awake.|She did not speak to him when she joined him there, although she thought that he was still awake. +LJ045-0176|The next morning he left for work before anyone else arose.|The next morning he left for work before anyone else arose. +LJ045-0177|For the first time|For the first time +LJ045-0178|he left his wedding ring in a cup on the dresser in his room. He also left $170 in a wallet in one of the dresser drawers.|he left his wedding ring in a cup on the dresser in his room. He also left one hundred seventy dollars in a wallet in one of the dresser drawers. +LJ045-0179|He took with him $13.87 and the long brown package that Frazier and Mrs. Randle saw him carry|He took with him thirteen dollars, eighty-seven cents and the long brown package that Frazier and Mrs. Randle saw him carry +LJ045-0180|and which he was to take to the School Book Depository. The Unanswered Questions.|and which he was to take to the School Book Depository. The Unanswered Questions. +LJ045-0181|No one will ever know what passed through Oswald's mind during the week before November 22, 1963.|No one will ever know what passed through Oswald's mind during the week before November twenty-two, nineteen sixty-three. +LJ045-0182|Instead of returning to Irving on November 15 for his customary weekend visit, he remained in Dallas at his wife's suggestion because of the birthday party.|Instead of returning to Irving on November fifteen for his customary weekend visit, he remained in Dallas at his wife's suggestion because of the birthday party. +LJ045-0183|He had argued with her over the use of an alias and had not called her after that argument, although he usually telephoned once or twice a day.|He had argued with her over the use of an alias and had not called her after that argument, although he usually telephoned once or twice a day. +LJ045-0184|Then on Thursday morning, November 21,|Then on Thursday morning, November twenty-one, +LJ045-0185|he asked Frazier for a ride to Irving that night, stating falsely that he wanted to pick up some curtain rods to put in an apartment.|he asked Frazier for a ride to Irving that night, stating falsely that he wanted to pick up some curtain rods to put in an apartment. +LJ045-0186|He must have planned his attack at the very latest prior to Thursday morning when he spoke to Frazier.|He must have planned his attack at the very latest prior to Thursday morning when he spoke to Frazier. +LJ045-0187|There is, of course, no way to determine the degree to which he was committed to his plan at that time.|There is, of course, no way to determine the degree to which he was committed to his plan at that time. +LJ045-0188|While there is no way to tell when he first began to think specifically of assassinating the President|While there is no way to tell when he first began to think specifically of assassinating the President +LJ045-0189|it should be noted that mention of the Trade Mart as the expected site of the Presidential luncheon|it should be noted that mention of the Trade Mart as the expected site of the Presidential luncheon +LJ045-0190|appeared in The Dallas Times Herald on November 15, 1963.|appeared in The Dallas Times Herald on November fifteen, nineteen sixty-three. +LJ045-0191|The next day that paper announced the final approval of the Trade Mart as the luncheon site and stated that the motorcade, quote,|The next day that paper announced the final approval of the Trade Mart as the luncheon site and stated that the motorcade, quote, +LJ045-0192|apparently will loop through the downtown area, probably on Main Street,|apparently will loop through the downtown area, probably on Main Street, +LJ045-0193|en route from Dallas Love Field, end quote, on its way to the Trade Mart on Stemmons Freeway.|en route from Dallas Love Field, end quote, on its way to the Trade Mart on Stemmons Freeway. +LJ045-0194|Anyone who was familiar with that area of Dallas would have known that the motorcade would probably pass the Texas School Book Depository to get from Main Street|Anyone who was familiar with that area of Dallas would have known that the motorcade would probably pass the Texas School Book Depository to get from Main Street +LJ045-0195|onto the Stemmons Freeway.|onto the Stemmons Freeway. +LJ045-0196|That fact was made precisely clear in subsequent news stories on November 19, 20, and 22.|That fact was made precisely clear in subsequent news stories on November nineteen, twenty, and twenty-two. +LJ045-0197|On November 15, 1963, the same day that his wife told him not to come to Irving, Oswald could have assumed|On November fifteen, nineteen sixty-three, the same day that his wife told him not to come to Irving, Oswald could have assumed +LJ045-0198|that the Presidential motorcade would pass in front of his place of work.|that the Presidential motorcade would pass in front of his place of work. +LJ045-0199|Whether he thought about assassinating the President over the weekend can never be known, but it is reasonably certain|Whether he thought about assassinating the President over the weekend can never be known, but it is reasonably certain +LJ045-0200|that over the weekend he did think about his wife's request that he not come to Irving, which was prompted by the birthday party being held at the Paine home.|that over the weekend he did think about his wife's request that he not come to Irving, which was prompted by the birthday party being held at the Paine home. +LJ045-0201|Oswald had a highly exaggerated sense of his own importance, but he had failed at almost everything he had ever tried to do.|Oswald had a highly exaggerated sense of his own importance, but he had failed at almost everything he had ever tried to do. +LJ045-0202|He had great difficulty in establishing meaningful relations with other people. Except for his family he was completely alone.|He had great difficulty in establishing meaningful relations with other people. Except for his family he was completely alone. +LJ045-0203|Even though he had searched -- in the Marine Corps, in his ideal of communism, in the Soviet Union and in his attempt to get to Cuba|Even though he had searched -- in the Marine Corps, in his ideal of communism, in the Soviet Union and in his attempt to get to Cuba +LJ045-0205|After he returned from his trip to Mexico where his application to go to Cuba had been sharply rejected,|After he returned from his trip to Mexico where his application to go to Cuba had been sharply rejected, +LJ045-0206|it must have appeared to him that he was unable to command even the attention of his family.|it must have appeared to him that he was unable to command even the attention of his family. +LJ045-0207|He could not keep them with him in Dallas, where at least he could see his children whom, several witnesses testified, he seemed to love.|He could not keep them with him in Dallas, where at least he could see his children whom, several witnesses testified, he seemed to love. +LJ045-0208|His family lived with Mrs. Paine, ostensibly because Oswald could not afford to keep an apartment in Dallas,|His family lived with Mrs. Paine, ostensibly because Oswald could not afford to keep an apartment in Dallas, +LJ045-0209|but it was also, at least in part, because his wife did not want to live there with him.|but it was also, at least in part, because his wife did not want to live there with him. +LJ045-0210|Now it appeared that he was not welcome at the Paine home,|Now it appeared that he was not welcome at the Paine home, +LJ045-0211|where he had spent every previous weekend since his return from Mexico and his wife was once again calling into question his judgment,|where he had spent every previous weekend since his return from Mexico and his wife was once again calling into question his judgment, +LJ045-0212|this time concerning his use of an alias.|this time concerning his use of an alias. +LJ045-0213|The conversation on Monday, November 18, 1963,|The conversation on Monday, November eighteen, nineteen sixty-three, +LJ045-0214|ended when Marina Oswald hung up and refused to talk to him. Although he may long before have decided on the course he was to follow|ended when Marina Oswald hung up and refused to talk to him. Although he may long before have decided on the course he was to follow +LJ045-0215|and may have told his wife the things he did on the evening of November 21,|and may have told his wife the things he did on the evening of November twenty-one, +LJ045-0216|1963, merely to disarm her and to provide a justification of sorts,|nineteen sixty-three, merely to disarm her and to provide a justification of sorts, +LJ045-0217|both she and Mrs. Paine thought he had come home to make up after the fight on Monday.|both she and Mrs. Paine thought he had come home to make up after the fight on Monday. +LJ045-0218|Thoughts of his personal difficulties must have been at least partly on his mind when he went to Irving on Thursday night and told his wife that he was lonely,|Thoughts of his personal difficulties must have been at least partly on his mind when he went to Irving on Thursday night and told his wife that he was lonely, +LJ045-0219|that he wanted to make peace with her and bring his family to Dallas where they could live with him again.|that he wanted to make peace with her and bring his family to Dallas where they could live with him again. +LJ045-0220|The Commission does not believe that the relations between Oswald and his wife caused him to assassinate the President.|The Commission does not believe that the relations between Oswald and his wife caused him to assassinate the President. +LJ045-0221|It is unlikely that the motivation was that simple.|It is unlikely that the motivation was that simple. +LJ045-0222|The feelings of hostility and aggression which seem to have played such an important, part in Oswald's life|The feelings of hostility and aggression which seem to have played such an important, part in Oswald's life +LJ045-0223|were part of his character long before he met his wife|were part of his character long before he met his wife +LJ045-0224|and such a favorable opportunity to strike at a figure as great as the President would probably never have come to him again.|and such a favorable opportunity to strike at a figure as great as the President would probably never have come to him again. +LJ045-0225|Oswald's behavior after the assassination throws little light on his motives.|Oswald's behavior after the assassination throws little light on his motives. +LJ045-0226|The fact that he took so little money with him when he left Irving in the morning indicates that he did not expect to get very far from Dallas on his own|The fact that he took so little money with him when he left Irving in the morning indicates that he did not expect to get very far from Dallas on his own +LJ045-0227|and suggests the possibility, as did his note to his wife just prior to the attempt on General Walker, that he did not expect to escape at all.|and suggests the possibility, as did his note to his wife just prior to the attempt on General Walker, that he did not expect to escape at all. +LJ045-0228|On the other hand, he could have traveled some distance with the money he did have and he did return to his room where he obtained his revolver.|On the other hand, he could have traveled some distance with the money he did have and he did return to his room where he obtained his revolver. +LJ045-0229|He then killed Patrolman Tippit when that police officer apparently tried to question him after he had left his roominghouse and he vigorously resisted arrest|He then killed Patrolman Tippit when that police officer apparently tried to question him after he had left his roominghouse and he vigorously resisted arrest +LJ045-0230|when he was finally apprehended in the Texas Theatre. Although it is not fully corroborated by others who were present,|when he was finally apprehended in the Texas Theatre. Although it is not fully corroborated by others who were present, +LJ045-0231|two officers have testified that at the time of his arrest Oswald said something to the effect that, quote, it's all over now, end quote.|two officers have testified that at the time of his arrest Oswald said something to the effect that, quote, it's all over now, end quote. +LJ045-0232|Oswald was overbearing and arrogant throughout much of the time between his arrest and his own death.|Oswald was overbearing and arrogant throughout much of the time between his arrest and his own death. +LJ045-0233|He consistently refused to admit involvement in the assassination or in the killing of Patrolman Tippit.|He consistently refused to admit involvement in the assassination or in the killing of Patrolman Tippit. +LJ045-0234|While he did become enraged at at least one point in his interrogation,|While he did become enraged at at least one point in his interrogation, +LJ045-0235|the testimony of the officers present indicates that he handled himself with considerable composure during his questioning.|the testimony of the officers present indicates that he handled himself with considerable composure during his questioning. +LJ045-0236|He admitted nothing that would damage him but discussed other matters quite freely.|He admitted nothing that would damage him but discussed other matters quite freely. +LJ045-0237|His denials under questioning, which have no probative value in view of the many readily demonstrable lies he told at that time|His denials under questioning, which have no probative value in view of the many readily demonstrable lies he told at that time +LJ045-0238|and in the face of the overwhelming evidence against him|and in the face of the overwhelming evidence against him +LJ045-0239|which has been set forth above, only served to prolong the period during which he was the center of the attention of the entire world.|which has been set forth above, only served to prolong the period during which he was the center of the attention of the entire world. +LJ045-0240|Many factors were undoubtedly involved in Oswald's motivation for the assassination, and the Commission does not believe|Many factors were undoubtedly involved in Oswald's motivation for the assassination, and the Commission does not believe +LJ045-0241|that it can ascribe to him any one motive or group of motives.|that it can ascribe to him any one motive or group of motives. +LJ045-0242|It is apparent, however, that Oswald was moved by an overriding hostility to his environment.|It is apparent, however, that Oswald was moved by an overriding hostility to his environment. +LJ045-0243|He does not appear to have been able to establish meaningful relationships with other people. He was perpetually discontented with the world around him.|He does not appear to have been able to establish meaningful relationships with other people. He was perpetually discontented with the world around him. +LJ045-0244|Long before the assassination he expressed his hatred for American society and acted in protest against it.|Long before the assassination he expressed his hatred for American society and acted in protest against it. +LJ045-0245|Oswald's search for what he conceived to be the perfect society was doomed from the start.|Oswald's search for what he conceived to be the perfect society was doomed from the start. +LJ045-0246|He sought for himself a place in history -- a role as the "great man" who would be recognized as having been in advance of his times.|He sought for himself a place in history -- a role as the "great man" who would be recognized as having been in advance of his times. +LJ045-0247|His commitment to Marxism and communism appears to have been another important factor in his motivation.|His commitment to Marxism and communism appears to have been another important factor in his motivation. +LJ045-0248|He also had demonstrated a capacity to act decisively and without regard to the consequences when such action would further his aims of the moment.|He also had demonstrated a capacity to act decisively and without regard to the consequences when such action would further his aims of the moment. +LJ045-0249|Out of these and the many other factors which may have molded the character of Lee Harvey Oswald|Out of these and the many other factors which may have molded the character of Lee Harvey Oswald +LJ045-0250|there emerged a man capable of assassinating President Kennedy.|there emerged a man capable of assassinating President Kennedy. +LJ046-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By +LJ046-0002|The President's Commission on the Assassination of President Kennedy. Chapter 8. The Protection of the President. Part 1.|The President's Commission on the Assassination of President Kennedy. Chapter eight. The Protection of the President. Part one. +LJ046-0003|In the 100 years since 1865|In the one hundred years since eighteen sixty-five +LJ046-0004|four Presidents of the United States have been assassinated|four Presidents of the United States have been assassinated +LJ046-0005|Abraham Lincoln, James A. Garfield, William McKinley, and John F. Kennedy.|Abraham Lincoln, James A. Garfield, William McKinley, and John F. Kennedy. +LJ046-0006|During this same period there were three other attacks on the life of a President,|During this same period there were three other attacks on the life of a President, +LJ046-0007|a President-elect, and a candidate for the Presidency, which narrowly failed:|a President-elect, and a candidate for the Presidency, which narrowly failed: +LJ046-0008|on Theodore Roosevelt while campaigning in October of 1912; on President-elect Franklin Delano Roosevelt,|on Theodore Roosevelt while campaigning in October of nineteen twelve; on President-elect Franklin Delano Roosevelt, +LJ046-0009|when visiting Miami on February 15, 1933; and on President Harry S. Truman on November 1, 1950,|when visiting Miami on February fifteen, nineteen thirty-three; and on President Harry S. Truman on November one, nineteen fifty, +LJ046-0010|when his temporary residence, Blair House, was attacked by Puerto Rican Nationalists.|when his temporary residence, Blair House, was attacked by Puerto Rican Nationalists. +LJ046-0011|One out of every five Presidents since 1865 has been assassinated;|One out of every five Presidents since eighteen sixty-five has been assassinated; +LJ046-0012|there have been attempts on the lives of one out of every three.|there have been attempts on the lives of one out of every three. +LJ046-0013|Prompted by these dismaying statistics, the Commission has inquired into the problems and methods of Presidential protection in effect|Prompted by these dismaying statistics, the Commission has inquired into the problems and methods of Presidential protection in effect +LJ046-0014|at the time of President Kennedy's assassination.|at the time of President Kennedy's assassination. +LJ046-0015|This study has led the Commission to conclude that the public interest might be served by any contribution it can make to the improvement of protective arrangements.|This study has led the Commission to conclude that the public interest might be served by any contribution it can make to the improvement of protective arrangements. +LJ046-0016|The Commission has not undertaken a comprehensive examination of all facets of this subject;|The Commission has not undertaken a comprehensive examination of all facets of this subject; +LJ046-0017|rather, it has devoted its time and resources to those broader aspects of Presidential protection|rather, it has devoted its time and resources to those broader aspects of Presidential protection +LJ046-0018|to which the events of last November called attention.|to which the events of last November called attention. +LJ046-0019|In this part of its inquiry the Commission has had full access to a major study of all phases of protective activities|In this part of its inquiry the Commission has had full access to a major study of all phases of protective activities +LJ046-0020|prepared by the Secret Service for the Secretary of the Treasury following the assassination. As a result of this study,|prepared by the Secret Service for the Secretary of the Treasury following the assassination. As a result of this study, +LJ046-0021|the Secretary of the Treasury has prepared a planning document dated August 27, 1964,|the Secretary of the Treasury has prepared a planning document dated August twenty-seven, nineteen sixty-four, +LJ046-0022|which recommends additional personnel and facilities to enable the Secret Service to expand its protection capabilities.|which recommends additional personnel and facilities to enable the Secret Service to expand its protection capabilities. +LJ046-0023|The Secretary of the Treasury submitted this planning document on August 31, 1964,|The Secretary of the Treasury submitted this planning document on August thirty-one, nineteen sixty-four, +LJ046-0024|to the Bureau of the Budget for review and approval.|to the Bureau of the Budget for review and approval. +LJ046-0025|This planning document has been made a part of the Commission's published record; the underlying staff and consultants' reports reviewed by the Commission have not,|This planning document has been made a part of the Commission's published record; the underlying staff and consultants' reports reviewed by the Commission have not, +LJ046-0026|since a disclosure of such detailed information relating to protective measures might undermine present methods of protecting the President.|since a disclosure of such detailed information relating to protective measures might undermine present methods of protecting the President. +LJ046-0027|However, all information considered by the Commission which pertains to the protective function as it was carried out in Dallas has been published as part of this report.|However, all information considered by the Commission which pertains to the protective function as it was carried out in Dallas has been published as part of this report. +LJ046-0028|The protection of the President of the United States is an immensely difficult and complex task.|The protection of the President of the United States is an immensely difficult and complex task. +LJ046-0029|It is unlikely that measures can be devised to eliminate entirely the multitude of diverse dangers that may arise,|It is unlikely that measures can be devised to eliminate entirely the multitude of diverse dangers that may arise, +LJ046-0030|particularly when the President is traveling in this country or abroad.|particularly when the President is traveling in this country or abroad. +LJ046-0031|The protective task is further complicated by the reluctance of Presidents to take security precautions which might interfere with the performance of their duties,|The protective task is further complicated by the reluctance of Presidents to take security precautions which might interfere with the performance of their duties, +LJ046-0032|or their desire to have frequent and easy access to the people.|or their desire to have frequent and easy access to the people. +LJ046-0033|The adequacy of existing procedures can fairly be assessed only after full consideration of the difficulty of the protective assignment,|The adequacy of existing procedures can fairly be assessed only after full consideration of the difficulty of the protective assignment, +LJ046-0034|with particular attention to the diverse roles which the President is expected to fill.|with particular attention to the diverse roles which the President is expected to fill. +LJ046-0035|After reviewing this aspect of the matter this chapter will set forth the Commission's conclusions|After reviewing this aspect of the matter this chapter will set forth the Commission's conclusions +LJ046-0036|regarding certain protective measures in force at the time of the Dallas trip and propose recommendations for improvements.|regarding certain protective measures in force at the time of the Dallas trip and propose recommendations for improvements. +LJ046-0037|The nature of the protective assignment.|The nature of the protective assignment. +LJ046-0038|The President is Head of State, Chief Executive, Commander in Chief, and leader of a political party.|The President is Head of State, Chief Executive, Commander in Chief, and leader of a political party. +LJ046-0039|As the ceremonial head of the Government the President must discharge a wide range of public duties, not only in Washington but throughout the land.|As the ceremonial head of the Government the President must discharge a wide range of public duties, not only in Washington but throughout the land. +LJ046-0040|In this role he appears to the American people, in the words of William Howard Taft, as, quote,|In this role he appears to the American people, in the words of William Howard Taft, as, quote, +LJ046-0041|the personal embodiment and representative of their dignity and majesty, end quote.|the personal embodiment and representative of their dignity and majesty, end quote. +LJ046-0042|As Chief Executive, the President controls the exercise of the vast, almost incalculable powers of the executive branch of the Federal Government.|As Chief Executive, the President controls the exercise of the vast, almost incalculable powers of the executive branch of the Federal Government. +LJ046-0043|As Commander in Chief of the Armed Forces, he must maintain ultimate authority over the development and disposition of our military power.|As Commander in Chief of the Armed Forces, he must maintain ultimate authority over the development and disposition of our military power. +LJ046-0044|in accordance with George Washington's maxim that Americans have a government, quote, of accommodation as well as a government of laws, end quote,|in accordance with George Washington's maxim that Americans have a government, quote, of accommodation as well as a government of laws, end quote, +LJ046-0045|it is the President's right and duty to be the active leader of his party, as when he seeks to be reelected or to maintain his party in power.|it is the President's right and duty to be the active leader of his party, as when he seeks to be reelected or to maintain his party in power. +LJ046-0046|In all of these roles the President must go to the people.|In all of these roles the President must go to the people. +LJ046-0047|Exposure of the President to public view through travel among the people of this country is a great and historic tradition of American life.|Exposure of the President to public view through travel among the people of this country is a great and historic tradition of American life. +LJ046-0048|Desired by both the President and the public, it is an indispensable means of communication between the two.|Desired by both the President and the public, it is an indispensable means of communication between the two. +LJ046-0049|More often than not, Presidential journeys have served more than one purpose at the same time: ceremonial,|More often than not, Presidential journeys have served more than one purpose at the same time: ceremonial, +LJ046-0050|administrative, political.|administrative, political. +LJ046-0051|From George Washington to John F. Kennedy, such journeys have been a normal part of the President's activities.|From George Washington to John F. Kennedy, such journeys have been a normal part of the President's activities. +LJ046-0052|To promote nationwide acceptance of his administration Washington made grand tours that served also to excite interest in the Presidency.|To promote nationwide acceptance of his administration Washington made grand tours that served also to excite interest in the Presidency. +LJ046-0053|In recent years, Presidential journeys have been frequent and extensive,|In recent years, Presidential journeys have been frequent and extensive, +LJ046-0054|partly because of the greater speed and comfort of travel and partly because of the greater demands made on the President.|partly because of the greater speed and comfort of travel and partly because of the greater demands made on the President. +LJ046-0055|It is now possible for Presidents to travel the length and breadth of a land far larger than the United States|It is now possible for Presidents to travel the length and breadth of a land far larger than the United States +LJ046-0056|in 1789 in less time than it took George Washington to travel from New York to Mount Vernon|in seventeen eighty-nine in less time than it took George Washington to travel from New York to Mount Vernon +LJ046-0057|or Thomas Jefferson from Washington to Monticello.|or Thomas Jefferson from Washington to Monticello. +LJ046-0058|During his Presidency, Franklin D. Roosevelt made almost 400 journeys and traveled more than 350,000 miles.|During his Presidency, Franklin D. Roosevelt made almost four hundred journeys and traveled more than three hundred fifty thousand miles. +LJ046-0059|Since 1945, Roosevelt's successors have ranged the world,|Since nineteen forty-five, Roosevelt's successors have ranged the world, +LJ046-0060|and their foreign journeys have come to be accepted as normal rather than extraordinary.|and their foreign journeys have come to be accepted as normal rather than extraordinary. +LJ046-0061|John F. Kennedy's journey to Texas in November 1963 was in this tradition.|John F. Kennedy's journey to Texas in November nineteen sixty-three was in this tradition. +LJ046-0062|His friend and Special Assistant Kenneth O'Donnell, who accompanied him on his last visit to Dallas,|His friend and Special Assistant Kenneth O'Donnell, who accompanied him on his last visit to Dallas, +LJ046-0063|stated the President's views of his responsibilities with simplicity and clarity, quote,|stated the President's views of his responsibilities with simplicity and clarity, quote, +LJ046-0064|The President's views of his responsibilities as President of the United States were that he meet the people, that he go out to their homes and see them,|The President's views of his responsibilities as President of the United States were that he meet the people, that he go out to their homes and see them, +LJ046-0065|and allow them to see him, and discuss, if possible, the views of the world as he sees it, the problems of the country as he sees them.|and allow them to see him, and discuss, if possible, the views of the world as he sees it, the problems of the country as he sees them. +LJ046-0066|And he felt that leaving Washington for the President of the United States was a most necessary -- not only for the people, but for the President himself,|And he felt that leaving Washington for the President of the United States was a most necessary -- not only for the people, but for the President himself, +LJ046-0067|that he expose himself to the actual basic problems that were disturbing the American people.|that he expose himself to the actual basic problems that were disturbing the American people. +LJ046-0068|It helped him in his job here, he was able to come back here with a fresh view of many things.|It helped him in his job here, he was able to come back here with a fresh view of many things. +LJ046-0069|I think he felt very strongly that the President ought to get out of Washington, and go meet the people on a regular basis. End quote.|I think he felt very strongly that the President ought to get out of Washington, and go meet the people on a regular basis. End quote. +LJ046-0070|Whatever their purposes Presidential journeys have greatly enlarged and complicated the task of protecting the President.|Whatever their purposes Presidential journeys have greatly enlarged and complicated the task of protecting the President. +LJ046-0071|The Secret Service and the Federal, State, and local law enforcement agencies which cooperate with it,|The Secret Service and the Federal, State, and local law enforcement agencies which cooperate with it, +LJ046-0072|have been confronted in recent years with increasingly difficult problems, created by the greater exposure of the President during his travels|have been confronted in recent years with increasingly difficult problems, created by the greater exposure of the President during his travels +LJ046-0073|and the greater diversity of the audiences he must face in a world torn by conflicting ideologies.|and the greater diversity of the audiences he must face in a world torn by conflicting ideologies. +LJ046-0074|If the sole goal were to protect the life of the President, it could be accomplished with reasonable assurance despite the multiple roles he must play.|If the sole goal were to protect the life of the President, it could be accomplished with reasonable assurance despite the multiple roles he must play. +LJ046-0075|But his very position as representative of the people prevents him from effectively shielding himself from the people.|But his very position as representative of the people prevents him from effectively shielding himself from the people. +LJ046-0076|He cannot and will not take the precautions of a dictator or a sovereign.|He cannot and will not take the precautions of a dictator or a sovereign. +LJ046-0077|Under our system, measures must be sought to afford security without impeding the President's performance of his many functions.|Under our system, measures must be sought to afford security without impeding the President's performance of his many functions. +LJ046-0078|The protection of the President must be thorough but inconspicuous to avoid even the suggestion of a garrison state.|The protection of the President must be thorough but inconspicuous to avoid even the suggestion of a garrison state. +LJ046-0079|The rights of private individuals must not be infringed.|The rights of private individuals must not be infringed. +LJ046-0080|If the protective job is well done, its performance will be evident only in the unexceptional fact of its success.|If the protective job is well done, its performance will be evident only in the unexceptional fact of its success. +LJ046-0081|The men in charge of protecting the President, confronted by complex problems and limited as they are in the measures they may employ,|The men in charge of protecting the President, confronted by complex problems and limited as they are in the measures they may employ, +LJ046-0082|must depend upon the utmost cooperation and understanding from the public and the President.|must depend upon the utmost cooperation and understanding from the public and the President. +LJ046-0083|The problem and the reasonable approach to its solution were ably stated in a memorandum prepared by FBI Director J. Edgar Hoover|The problem and the reasonable approach to its solution were ably stated in a memorandum prepared by FBI Director J. Edgar Hoover +LJ046-0084|for the President soon after the assassination, quote,|for the President soon after the assassination, quote, +LJ046-0085|The degree of security that can be afforded the President of the United States|The degree of security that can be afforded the President of the United States +LJ046-0086|is dependent to a considerable extent upon the degree of contact with the general public desired by the President.|is dependent to a considerable extent upon the degree of contact with the general public desired by the President. +LJ046-0087|Absolute security is neither practical nor possible. An approach to complete security would require the President|Absolute security is neither practical nor possible. An approach to complete security would require the President +LJ046-0088|to operate in a sort of vacuum, isolated from the general public and behind impregnable barriers.|to operate in a sort of vacuum, isolated from the general public and behind impregnable barriers. +LJ046-0089|His travel would be in secret; his public appearances would be behind bulletproof glass. A more practical approach necessitates compromise.|His travel would be in secret; his public appearances would be behind bulletproof glass. A more practical approach necessitates compromise. +LJ046-0090|Any travel, any contact with the general public, involves a calculated risk on the part of the President and the men responsible for his protection.|Any travel, any contact with the general public, involves a calculated risk on the part of the President and the men responsible for his protection. +LJ046-0091|Such risks can be lessened when the President recognizes the security problem,|Such risks can be lessened when the President recognizes the security problem, +LJ046-0092|has confidence in the dedicated Secret Service men who are ready to lay down their lives for him|has confidence in the dedicated Secret Service men who are ready to lay down their lives for him +LJ046-0093|and accepts the necessary security precautions which they recommend.|and accepts the necessary security precautions which they recommend. +LJ046-0094|Many Presidents have been understandably impatient with the security precautions which many years of experience dictate|Many Presidents have been understandably impatient with the security precautions which many years of experience dictate +LJ046-0095|because these precautions reduce the President's privacy and the access to him of the people of the country.|because these precautions reduce the President's privacy and the access to him of the people of the country. +LJ046-0096|Nevertheless the procedures and advice should be accepted if the President wishes to have any security.|Nevertheless the procedures and advice should be accepted if the President wishes to have any security. +LJ046-0097|Evaluation Of Presidential Protection At The Time Of The Assassination Of President Kennedy|Evaluation Of Presidential Protection At The Time Of The Assassination Of President Kennedy +LJ046-0098|The history of Presidential protection shows growing recognition over the years that the job must be done by able, dedicated,|The history of Presidential protection shows growing recognition over the years that the job must be done by able, dedicated, +LJ046-0099|thoroughly professional personnel, using the best technical equipment that can be devised.|thoroughly professional personnel, using the best technical equipment that can be devised. +LJ046-0100|The assassination of President Kennedy demands an examination of the protective measures employed to safe guard him|The assassination of President Kennedy demands an examination of the protective measures employed to safe guard him +LJ046-0101|and an inquiry whether improvements can be made which will reduce the risk of another such tragedy.|and an inquiry whether improvements can be made which will reduce the risk of another such tragedy. +LJ046-0102|This section considers first the means used to locate potential sources of danger to the President in time to take appropriate precautions.|This section considers first the means used to locate potential sources of danger to the President in time to take appropriate precautions. +LJ046-0103|In this connection the information available to Federal agencies about Lee Harvey Oswald|In this connection the information available to Federal agencies about Lee Harvey Oswald +LJ046-0104|is set out and the reasons why this information was not furnished to the Secret Service appraised.|is set out and the reasons why this information was not furnished to the Secret Service appraised. +LJ046-0105|Second, the adequacy of other advance preparations for the security of the President, during his visit to Dallas,|Second, the adequacy of other advance preparations for the security of the President, during his visit to Dallas, +LJ046-0106|largely measures taken by the Secret Service, is considered.|largely measures taken by the Secret Service, is considered. +LJ046-0107|Finally, the performance of those charged with the immediate responsibility of protecting the President on November 22 is reviewed.|Finally, the performance of those charged with the immediate responsibility of protecting the President on November twenty-two is reviewed. +LJ046-0108|Intelligence Functions Relating to Presidential Protection at the Time of the Dallas Trip|Intelligence Functions Relating to Presidential Protection at the Time of the Dallas Trip +LJ046-0109|A basic element of Presidential protection|A basic element of Presidential protection +LJ046-0110|is the identification and elimination of possible sources of danger to the President before the danger becomes actual.|is the identification and elimination of possible sources of danger to the President before the danger becomes actual. +LJ046-0111|The Secret Service has attempted to perform this function through the activities of its Protective Research Section|The Secret Service has attempted to perform this function through the activities of its Protective Research Section +LJ046-0112|and requests to other agencies, Federal and local, for useful information.|and requests to other agencies, Federal and local, for useful information. +LJ046-0113|The Commission has concluded that at the time of the assassination|The Commission has concluded that at the time of the assassination +LJ046-0114|the arrangements relied upon by the Secret Service to perform this function were seriously deficient.|the arrangements relied upon by the Secret Service to perform this function were seriously deficient. +LJ046-0115|Adequacy of preventive intelligence operations of the Secret Service.|Adequacy of preventive intelligence operations of the Secret Service. +LJ046-0116|The main job of the Protective Research Section (PRS)|The main job of the Protective Research Section (PRS) +LJ046-0117|is to collect, process, and evaluate information about persons or groups who may be a danger to the President.|is to collect, process, and evaluate information about persons or groups who may be a danger to the President. +LJ046-0118|In addition to this function, PRS is responsible for such tasks|In addition to this function, PRS is responsible for such tasks +LJ046-0119|as obtaining clearance of some categories of White House employees and all tradesmen who service the White House,|as obtaining clearance of some categories of White House employees and all tradesmen who service the White House, +LJ046-0120|the security processing of gifts sent to the President, and technical inspections against covert listening devices.|the security processing of gifts sent to the President, and technical inspections against covert listening devices. +LJ046-0121|At the time of the assassination PRS was a very small group, comprised of 12 specialists and 3 clerks.|At the time of the assassination PRS was a very small group, comprised of twelve specialists and three clerks. +LJ046-0122|Many persons call themselves to the attention of PRS by attempting to visit, the President|Many persons call themselves to the attention of PRS by attempting to visit, the President +LJ046-0123|for bizarre reasons or by writing or in some other way attempting to communicate with him in a threatening or abusive manner|for bizarre reasons or by writing or in some other way attempting to communicate with him in a threatening or abusive manner +LJ046-0124|or with undue persistence.|or with undue persistence. +LJ046-0125|Robert I. Bouck, special agent in charge of PRS,|Robert I. Bouck, special agent in charge of PRS, +LJ046-0126|estimated that most of the material received by his office originated in this fashion|estimated that most of the material received by his office originated in this fashion +LJ046-0127|or from the occasional investigations initiated by the Secret Service,|or from the occasional investigations initiated by the Secret Service, +LJ046-0128|while the balance was furnished to PRS by other Federal agencies, with primary source being the FBI.|while the balance was furnished to PRS by other Federal agencies, with primary source being the FBI. +LJ046-0129|The total volume of information received by PRS has risen steadily.|The total volume of information received by PRS has risen steadily. +LJ046-0130|In 1943 PRS received approximately 9,000 items of information;|In nineteen forty-three PRS received approximately nine thousand items of information; +LJ046-0131|in 1953 this had increased to more than 17,000 items;|in nineteen fifty-three this had increased to more than seventeen thousand items; +LJ046-0132|in 1963 the total exceeded 32,000 items.|in nineteen sixty-three the total exceeded thirty-two thousand items. +LJ046-0133|Since many items may pertain to a single case, these figures do not show the caseload.|Since many items may pertain to a single case, these figures do not show the caseload. +LJ046-0134|In the period from November 1961 to November 1963,|In the period from November nineteen sixty-one to November nineteen sixty-three, +LJ046-0135|PRS received items in 8,709 cases.|PRS received items in eight thousand, seven hundred nine cases. +LJ046-0136|Before the assassination of President Kennedy,|Before the assassination of President Kennedy, +LJ046-0137|PRS expressed its interest in receiving information on suspects in very general terms. For example,|PRS expressed its interest in receiving information on suspects in very general terms. For example, +LJ046-0138|PRS instructed the White House mailroom, a source of much PRS data,|PRS instructed the White House mailroom, a source of much PRS data, +LJ046-0139|to refer all communications on identified existing cases and, in addition,|to refer all communications on identified existing cases and, in addition, +LJ046-0140|any communication, quote, that in any way indicates anyone may have possible intention of harming the President, end quote.|any communication, quote, that in any way indicates anyone may have possible intention of harming the President, end quote. +LJ046-0141|Slightly more specific criteria were established for PRS personnel processing White House mail referred by the White House mailroom,|Slightly more specific criteria were established for PRS personnel processing White House mail referred by the White House mailroom, +LJ046-0142|but again the standards were very general.|but again the standards were very general. +LJ046-0143|These instructions to PRS personnel appear to be the only instance where an effort was made to reduce the criteria to writing.|These instructions to PRS personnel appear to be the only instance where an effort was made to reduce the criteria to writing. +LJ046-0144|When requested to provide a specific statement of the standards employed by PRS in deciding what information to seek and retain,|When requested to provide a specific statement of the standards employed by PRS in deciding what information to seek and retain, +LJ046-0145|The Secret Service responded, quote,|The Secret Service responded, quote, +LJ046-0146|The criteria in effect prior to November 22, 1963, for determining whether to accept material for the PRS general files|The criteria in effect prior to November twenty-two, nineteen sixty-three, for determining whether to accept material for the PRS general files +LJ046-0147|were broad and flexible. All material is and was desired, accepted, and filed if it indicated or tended to indicate|were broad and flexible. All material is and was desired, accepted, and filed if it indicated or tended to indicate +LJ046-0148|that the safety of the President is or might be in danger, either at the present or in the future.|that the safety of the President is or might be in danger, either at the present or in the future. +LJ046-0149|There are many actions, situations, and incidents that may indicate such potential danger. Some are specific, such as threats;|There are many actions, situations, and incidents that may indicate such potential danger. Some are specific, such as threats; +LJ046-0150|danger may be implied from others, such as membership or activity in an organization which believes in assassination as a political weapon.|danger may be implied from others, such as membership or activity in an organization which believes in assassination as a political weapon. +LJ046-0151|All material received by PRS was separately screened|All material received by PRS was separately screened +LJ046-0152|and a determination made as to whether the information might indicate possible harm to the President.|and a determination made as to whether the information might indicate possible harm to the President. +LJ046-0153|If the material was evaluated|If the material was evaluated +LJ046-0154|as indicating some potential danger to the President -- no matter how small -- it was indexed in the general PRS files|as indicating some potential danger to the President -- no matter how small -- it was indexed in the general PRS files +LJ046-0155|under the name of the individual or group of individuals to whom that material related. End quote.|under the name of the individual or group of individuals to whom that material related. End quote. +LJ046-0156|The general files of PRS consist of folders on individuals, card indexed by name.|The general files of PRS consist of folders on individuals, card indexed by name. +LJ046-0157|The files are manually maintained, without use of any automatic data-processing techniques.|The files are manually maintained, without use of any automatic data-processing techniques. +LJ046-0158|At the time of the assassination, the active PRS general files contained approximately 50,000 cases|At the time of the assassination, the active PRS general files contained approximately fifty thousand cases +LJ046-0159|accumulated over a 20-year period, some of which included more than one individual.|accumulated over a twenty-year period, some of which included more than one individual. +LJ046-0160|A case file was established if the information available suggested that the subject might be a danger to the President.|A case file was established if the information available suggested that the subject might be a danger to the President. +LJ046-0161|Many of these cases were not investigated by PRS.|Many of these cases were not investigated by PRS. +LJ046-0162|The case file served merely as a repository for information until enough had accumulated to warrant an investigation.|The case file served merely as a repository for information until enough had accumulated to warrant an investigation. +LJ046-0163|During the period November 1961 to November 1963,|During the period November nineteen sixty-one to November nineteen sixty-three, +LJ046-0164|PRS investigated 34 newly established or reactivated cases concerning residents of Texas.|PRS investigated thirty-four newly established or reactivated cases concerning residents of Texas. +LJ046-0165|Most of these cases involved persons who used threatening language in communications to or about the President.|Most of these cases involved persons who used threatening language in communications to or about the President. +LJ046-0166|An additional 115 cases concerning Texas residents were established but not investigated.|An additional one hundred fifteen cases concerning Texas residents were established but not investigated. +LJ046-0167|When PRS learns of an individual whose conduct warrants scrutiny, it requests an investigation by the closest Secret Service field office,|When PRS learns of an individual whose conduct warrants scrutiny, it requests an investigation by the closest Secret Service field office, +LJ046-0168|of which there are 65 throughout the country.|of which there are sixty-five throughout the country. +LJ046-0169|If the field office determines that the case should be subject to continuing review, PRS establishes a file|If the field office determines that the case should be subject to continuing review, PRS establishes a file +LJ046-0170|which requires a checkup at least, every 6 months.|which requires a checkup at least, every six months. +LJ046-0171|This might involve a personal interview or interviews with members of the person's household. Wherever possible,|This might involve a personal interview or interviews with members of the person's household. Wherever possible, +LJ046-0172|the Secret Service arranges for the family and friends of the individual, and local law enforcement officials,|the Secret Service arranges for the family and friends of the individual, and local law enforcement officials, +LJ046-0173|to advise the field office if the subject displays signs of increased danger or plans to leave his home area.|to advise the field office if the subject displays signs of increased danger or plans to leave his home area. +LJ046-0174|At the time of the assassination there were approximately 400 persons throughout the country who were subject to periodic review.|At the time of the assassination there were approximately four hundred persons throughout the country who were subject to periodic review. +LJ046-0175|If PRS concludes after investigation|If PRS concludes after investigation +LJ046-0176|that an individual presents a significant danger to the life of the President, his name is placed in a "trip index file"|that an individual presents a significant danger to the life of the President, his name is placed in a "trip index file" +LJ046-0177|which is maintained on a geographical field office basis.|which is maintained on a geographical field office basis. +LJ046-0178|At the time of the assassination the names of about 100 persons were in this index, all of whom were included in the group of 400|At the time of the assassination the names of about one hundred persons were in this index, all of whom were included in the group of four hundred +LJ046-0179|being reviewed regularly.|being reviewed regularly. +LJ046-0180|PRS also maintains an album of photographs and descriptions of about 12 to 15 individuals who are regarded as clear risks to the President|PRS also maintains an album of photographs and descriptions of about twelve to fifteen individuals who are regarded as clear risks to the President +LJ046-0181|and who do not have a fixed place of residence. Members of the White House detail of the Secret Service have copies of this album.|and who do not have a fixed place of residence. Members of the White House detail of the Secret Service have copies of this album. +LJ046-0182|Individuals who are regarded as dangerous to the President|Individuals who are regarded as dangerous to the President +LJ046-0183|and who are in penal or hospital custody are listed only in the general files of PRS,|and who are in penal or hospital custody are listed only in the general files of PRS, +LJ046-0184|but there is a system for the immediate notification of the Secret Service by the confining institution when a subject is released or escapes.|but there is a system for the immediate notification of the Secret Service by the confining institution when a subject is released or escapes. +LJ046-0185|PRS attempts to eliminate serious risks by hospitalization or, where necessary,|PRS attempts to eliminate serious risks by hospitalization or, where necessary, +LJ046-0186|the prosecution of persons who have committed an offense such as threatening the President.|the prosecution of persons who have committed an offense such as threatening the President. +LJ046-0187|In June 1964 PRS had arrangements to be notified about the release or escape of approximately 1,000 persons.|In June nineteen sixty-four PRS had arrangements to be notified about the release or escape of approximately one thousand persons. +LJ046-0188|In summary, at the time of the assassination|In summary, at the time of the assassination +LJ046-0189|PRS had received, over a 20-year period, basic information on some 50,000 cases;|PRS had received, over a twenty-year period, basic information on some fifty thousand cases; +LJ046-0190|it had arrangements to be notified about release from confinement in roughly 1,000 cases;|it had arrangements to be notified about release from confinement in roughly one thousand cases; +LJ046-0191|it had established periodic regular review of the status of 400 individuals;|it had established periodic regular review of the status of four hundred individuals; +LJ046-0192|it regarded approximately 100 of these 400 cases as serious risks|it regarded approximately one hundred of these four hundred cases as serious risks +LJ046-0194|Members of the White House detail were expected to familiarize themselves with the descriptions and photographs of the highest risk cases.|Members of the White House detail were expected to familiarize themselves with the descriptions and photographs of the highest risk cases. +LJ046-0195|The cases subject to periodic review and the 100 or so cases in the higher risk category|The cases subject to periodic review and the one hundred or so cases in the higher risk category +LJ046-0196|were filed on a geographic basis, and could conveniently be reviewed by a Secret Service agent preparing for a Presidential trip|were filed on a geographic basis, and could conveniently be reviewed by a Secret Service agent preparing for a Presidential trip +LJ046-0197|to a particular part of the country. These were the files reviewed by PRS on November 8, 1963,|to a particular part of the country. These were the files reviewed by PRS on November eight, nineteen sixty-three, +LJ046-0198|at the request of Special Agent Lawson, advance agent for President Kennedy's trip to Dallas.|at the request of Special Agent Lawson, advance agent for President Kennedy's trip to Dallas. +LJ046-0199|The general files of PRS were not indexed by geographic location and were of little use in preparing for a Presidential visit to a specific locality.|The general files of PRS were not indexed by geographic location and were of little use in preparing for a Presidential visit to a specific locality. +LJ046-0200|Secret Service requests to other agencies for intelligence information|Secret Service requests to other agencies for intelligence information +LJ046-0201|were no more specific than the broad and general instructions its own agents and the White House mailroom.|were no more specific than the broad and general instructions its own agents and the White House mailroom. +LJ046-0202|The head of PRS testified that the Secret Service requested other agencies to provide, quote,|The head of PRS testified that the Secret Service requested other agencies to provide, quote, +LJ046-0203|any and all information that they may come in contact with that would indicate danger to the President, end quote.|any and all information that they may come in contact with that would indicate danger to the President, end quote. +LJ046-0204|These requests were communicated in writing by the Secret Service; rather, the Service depended on the personal liaison maintained by PRS|These requests were communicated in writing by the Secret Service; rather, the Service depended on the personal liaison maintained by PRS +LJ046-0205|with the headquarters of the Federal intelligence agencies, particularly the FBI,|with the headquarters of the Federal intelligence agencies, particularly the FBI, +LJ046-0206|and at the working level with personnel of the field offices of the various agencies. The Service frequently participated|and at the working level with personnel of the field offices of the various agencies. The Service frequently participated +LJ046-0207|in the training programs of other law enforcement agencies, and agents from other agencies attended the regular Secret Service training schools.|in the training programs of other law enforcement agencies, and agents from other agencies attended the regular Secret Service training schools. +LJ046-0208|Presidential protection was an important topic in these training programs.|Presidential protection was an important topic in these training programs. +LJ046-0209|In the absence of more specific instructions, other Federal agencies interpreted the Secret Service's informal requests|In the absence of more specific instructions, other Federal agencies interpreted the Secret Service's informal requests +LJ046-0210|to relate principally to overt threats to harm the President or other specific manifestations of hostility.|to relate principally to overt threats to harm the President or other specific manifestations of hostility. +LJ046-0211|For example, at the time of the assassination, the FBI Handbook, which is in the possession of every Bureau special agent, provided, quote,|For example, at the time of the assassination, the FBI Handbook, which is in the possession of every Bureau special agent, provided, quote, +LJ046-0212|Threats against the President of the U.S.,|Threats against the President of the U.S., +LJ046-0213|members of his immediate family, the President-elect, and the Vice-President. Investigation of threats against the President of the United States,|members of his immediate family, the President-elect, and the Vice-President. Investigation of threats against the President of the United States, +LJ046-0214|members of his immediate family, the President-Elect, and the Vice-President is within the exclusive jurisdiction of the U.S. Secret Service.|members of his immediate family, the President-Elect, and the Vice-President is within the exclusive jurisdiction of the U.S. Secret Service. +LJ046-0215|Any information indicating the possibility of an attempt against the person or safety of the President,|Any information indicating the possibility of an attempt against the person or safety of the President, +LJ046-0216|members of the immediate family of the President, the President-Elect or the Vice-President|members of the immediate family of the President, the President-Elect or the Vice-President +LJ046-0217|must be referred immediately by the most expeditious means of communication to the nearest office of the U.S. Secret Service.|must be referred immediately by the most expeditious means of communication to the nearest office of the U.S. Secret Service. +LJ046-0218|Advise the Bureau at the same time by teletype of the information so furnished to the Secret Service and the fact that it has been so disseminated.|Advise the Bureau at the same time by teletype of the information so furnished to the Secret Service and the fact that it has been so disseminated. +LJ046-0219|The above action should be taken without delay in order to attempt to verify the information and no evaluation of the information should be attempted.|The above action should be taken without delay in order to attempt to verify the information and no evaluation of the information should be attempted. +LJ046-0220|When the threat is in the form of a written communication, give a copy to local Secret Service and forward the original to the Bureau|When the threat is in the form of a written communication, give a copy to local Secret Service and forward the original to the Bureau +LJ046-0221|where it will be made available to Secret Service headquarters in Washington.|where it will be made available to Secret Service headquarters in Washington. +LJ046-0222|The referral of the copy to local Secret Service should not delay the immediate referral of the information by the fastest available means of communication|The referral of the copy to local Secret Service should not delay the immediate referral of the information by the fastest available means of communication +LJ046-0223|to Secret Service locally, end quote.|to Secret Service locally, end quote. +LJ046-0224|The State Department advised the Secret Service of all crank and threat letter mail|The State Department advised the Secret Service of all crank and threat letter mail +LJ046-0225|or crank visitors and furnished reports concerning any assassination or attempted assassination of a ruler or other major official anywhere in the world.|or crank visitors and furnished reports concerning any assassination or attempted assassination of a ruler or other major official anywhere in the world. +LJ046-0226|The several military intelligence agencies reported crank mail and similar threats involving the President.|The several military intelligence agencies reported crank mail and similar threats involving the President. +LJ046-0227|According to Special Agent in Charge Bouck,|According to Special Agent in Charge Bouck, +LJ046-0228|the Secret Service had no standard procedure for the systematic review of its requests for and receipt of information from other Federal agencies.|the Secret Service had no standard procedure for the systematic review of its requests for and receipt of information from other Federal agencies. +LJ046-0229|The Commission believes that the facilities and procedures of the Protective Research Section of the Secret Service|The Commission believes that the facilities and procedures of the Protective Research Section of the Secret Service +LJ046-0230|prior to November 22, 1963, were inadequate.|prior to November twenty-two, nineteen sixty-three, were inadequate. +LJ046-0231|Its efforts appear to have been too largely directed at the "crank" threat.|Its efforts appear to have been too largely directed at the "crank" threat. +LJ046-0232|Although the Service recognized that its advance preventive measures must encompass more than these most obvious dangers,|Although the Service recognized that its advance preventive measures must encompass more than these most obvious dangers, +LJ046-0233|it made little effort to identify factors in the activities of an individual|it made little effort to identify factors in the activities of an individual +LJ046-0234|or an organized group, other than specific threats, which suggested a source of danger against which timely precautions could be taken.|or an organized group, other than specific threats, which suggested a source of danger against which timely precautions could be taken. +LJ046-0235|Except for its special "trip index" file of 400 names,|Except for its special "trip index" file of four hundred names, +LJ046-0236|none of the cases in the PRS general files was available for systematic review on a geographic basis when the President planned a particular trip.|none of the cases in the PRS general files was available for systematic review on a geographic basis when the President planned a particular trip. +LJ046-0237|As reported in chapter 2, when the special file was reviewed on November 8,|As reported in chapter two, when the special file was reviewed on November eight, +LJ046-0238|it contained the names of no persons from the entire Dallas-Fort Worth area,|it contained the names of no persons from the entire Dallas-Fort Worth area, +LJ046-0239|notwithstanding the fact that Ambassador Stevenson had been abused by pickets in Dallas less than a month before.|notwithstanding the fact that Ambassador Stevenson had been abused by pickets in Dallas less than a month before. +LJ046-0240|Bouck explained the failure to try to identify the individuals involved in the Stevenson incident after it occurred on the ground that|Bouck explained the failure to try to identify the individuals involved in the Stevenson incident after it occurred on the ground that +LJ046-0241|PRS required a more direct indication of a threat to the President, and that there was no such indication until the President's scheduled visit to that area became known.|PRS required a more direct indication of a threat to the President, and that there was no such indication until the President's scheduled visit to that area became known. +LJ046-0242|Such an approach seriously undermines the precautionary nature of PRS work;|Such an approach seriously undermines the precautionary nature of PRS work; +LJ046-0243|if the presence in Dallas of the Stevenson pickets might have created a danger for the President on a visit to that city,|if the presence in Dallas of the Stevenson pickets might have created a danger for the President on a visit to that city, +LJ046-0245|Other agencies occasionally provided information to the Secret Service concerning potentially dangerous political groups.|Other agencies occasionally provided information to the Secret Service concerning potentially dangerous political groups. +LJ046-0246|This was done in the case of the Nationalist Party of Puerto Rico, for example, but only after members of the group had resorted to political violence.|This was done in the case of the Nationalist Party of Puerto Rico, for example, but only after members of the group had resorted to political violence. +LJ046-0247|However, the vague requests for information which the Secret Service made|However, the vague requests for information which the Secret Service made +LJ046-0248|to Federal intelligence and law enforcement agencies were not well designed to elicit information from them|to Federal intelligence and law enforcement agencies were not well designed to elicit information from them +LJ046-0249|about persons other than those who were obvious threats to the President.|about persons other than those who were obvious threats to the President. +LJ046-0250|The requests shifted the responsibility for evaluating difficult cases from the Service, the agency most responsible for performing that task,|The requests shifted the responsibility for evaluating difficult cases from the Service, the agency most responsible for performing that task, +LJ046-0251|to the other agencies. No specific guidance was provided.|to the other agencies. No specific guidance was provided. +LJ046-0252|Although the CIA had on file requests from the Treasury Department for information on the counterfeiting of U.S. currency and certain smuggling matters,|Although the CIA had on file requests from the Treasury Department for information on the counterfeiting of U.S. currency and certain smuggling matters, +LJ046-0253|it had no written specification of intelligence information collected by CIA abroad which was desired by the Secret Service|it had no written specification of intelligence information collected by CIA abroad which was desired by the Secret Service +LJ046-0254|in advance of Presidential trips outside the United States.|in advance of Presidential trips outside the United States. +LJ047-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ047-0002|Chapter 8. The Protection of the President. Part 2.|Chapter eight. The Protection of the President. Part two. +LJ047-0003|Information known about Lee Harvey Oswald prior to the assassination.|Information known about Lee Harvey Oswald prior to the assassination. +LJ047-0004|No information concerning Lee Harvey Oswald appeared in PRS files before the President's trip to Dallas.|No information concerning Lee Harvey Oswald appeared in PRS files before the President's trip to Dallas. +LJ047-0005|Oswald was known to other Federal agencies with which the Secret Service maintained intelligence liaison.|Oswald was known to other Federal agencies with which the Secret Service maintained intelligence liaison. +LJ047-0006|The FBI had been interested in him, to some degree at least, since the time of his defection in October 1959.|The FBI had been interested in him, to some degree at least, since the time of his defection in October nineteen fifty-nine. +LJ047-0007|It had interviewed him twice shortly after his return to the United States, again a year later at his request|It had interviewed him twice shortly after his return to the United States, again a year later at his request +LJ047-0008|and was investigating him at the time of the assassination. The Commission has taken the testimony of Bureau agents|and was investigating him at the time of the assassination. The Commission has taken the testimony of Bureau agents +LJ047-0009|who interviewed Oswald after his return from the Soviet Union and prior to November 22, 1963,|who interviewed Oswald after his return from the Soviet Union and prior to November twenty-two, nineteen sixty-three, +LJ047-0010|the agent who was assigned his case at the time of the assassination, the Director of the FBI,|the agent who was assigned his case at the time of the assassination, the Director of the FBI, +LJ047-0011|and the Assistant to the Director in charge of all investigative activities under the Director and Associate Director. In addition,|and the Assistant to the Director in charge of all investigative activities under the Director and Associate Director. In addition, +LJ047-0012|the Director and Deputy Director for Plans of the CIA testified concerning that Agency's limited knowledge of Oswald before the assassination.|the Director and Deputy Director for Plans of the CIA testified concerning that Agency's limited knowledge of Oswald before the assassination. +LJ047-0013|Finally, the Commission has reviewed the complete files on Oswald, as they existed at the time of the assassination, of the Department of State,|Finally, the Commission has reviewed the complete files on Oswald, as they existed at the time of the assassination, of the Department of State, +LJ047-0014|the Office of Naval Intelligence, the FBI and the CIA. The information known to the FBI is summarized below.|the Office of Naval Intelligence, the FBI and the CIA. The information known to the FBI is summarized below. +LJ047-0015|From defection to return to Fort Worth.|From defection to return to Fort Worth. +LJ047-0016|The FBI opened a file on Oswald in October 1959, when news reports appeared of his defection to the Soviet Union.|The FBI opened a file on Oswald in October nineteen fifty-nine, when news reports appeared of his defection to the Soviet Union. +LJ047-0017|The file was opened, quote,|The file was opened, quote, +LJ047-0018|for the purpose of correlating information inasmuch as he was considered a possible security risk in the event he returned to this country, end quote.|for the purpose of correlating information inasmuch as he was considered a possible security risk in the event he returned to this country, end quote. +LJ047-0019|Oswald's defection was also the occasion for the opening of files by the Department of State, CIA, and the Office of Naval Intelligence.|Oswald's defection was also the occasion for the opening of files by the Department of State, CIA, and the Office of Naval Intelligence. +LJ047-0020|Until April 1960, FBI activity consisted of placing in Oswald's file|Until April nineteen sixty, FBI activity consisted of placing in Oswald's file +LJ047-0021|information regarding his relations with the U.S. Embassy in Moscow and background data relating largely to his prior military service,|information regarding his relations with the U.S. Embassy in Moscow and background data relating largely to his prior military service, +LJ047-0022|provided by other agencies.|provided by other agencies. +LJ047-0023|In April 1960, Mrs. Marguerite Oswald and Robert Oswald were interviewed|In April nineteen sixty, Mrs. Marguerite Oswald and Robert Oswald were interviewed +LJ047-0024|in the course of a routine FBI investigation of transfers of small sums of money from Mrs. Oswald to her son in Russia.|in the course of a routine FBI investigation of transfers of small sums of money from Mrs. Oswald to her son in Russia. +LJ047-0025|During the next 2 years the FBI continued to accumulate information,|During the next two years the FBI continued to accumulate information, +LJ047-0026|and kept itself informed on Oswald's status by periodic reviews of State Department and Office of Naval Intelligence files.|and kept itself informed on Oswald's status by periodic reviews of State Department and Office of Naval Intelligence files. +LJ047-0027|In this way, it learned that when Oswald had arrived in the Soviet Union|In this way, it learned that when Oswald had arrived in the Soviet Union +LJ047-0028|he had attempted to renounce his U.S. citizenship and applied for Soviet citizenship,|he had attempted to renounce his U.S. citizenship and applied for Soviet citizenship, +LJ047-0029|had described himself as a Marxist, had said he would give the Soviet Union any useful information he had acquired|had described himself as a Marxist, had said he would give the Soviet Union any useful information he had acquired +LJ047-0030|as a marine radar technician and had displayed an arrogant and aggressive attitude at the U.S. Embassy;|as a marine radar technician and had displayed an arrogant and aggressive attitude at the U.S. Embassy; +LJ047-0031|it learned also that Oswald had been discharged from the Marine Corps Reserve as undesirable in August 1960.|it learned also that Oswald had been discharged from the Marine Corps Reserve as undesirable in August nineteen sixty. +LJ047-0032|In June 1962, the Bureau was advised by the Department of State of Oswald's plan to return to the United States.|In June nineteen sixty-two, the Bureau was advised by the Department of State of Oswald's plan to return to the United States. +LJ047-0033|The Bureau made arrangements to be advised by immigration authorities of his return,|The Bureau made arrangements to be advised by immigration authorities of his return, +LJ047-0034|and instructed the Dallas office to interview him when he got back to determine whether he had been recruited by a Soviet intelligence service.|and instructed the Dallas office to interview him when he got back to determine whether he had been recruited by a Soviet intelligence service. +LJ047-0035|Oswald's file at the Department of State Passport Office was reviewed in June 1962.|Oswald's file at the Department of State Passport Office was reviewed in June nineteen sixty-two. +LJ047-0036|It revealed his letter of January 30, 1962, to Secretary of the Navy Connally,|It revealed his letter of January thirty, nineteen sixty-two, to Secretary of the Navy Connally, +LJ047-0037|in which he protested his discharge and declared that he would use, quote, all means, end quote, to correct it.|in which he protested his discharge and declared that he would use, quote, all means, end quote, to correct it. +LJ047-0038|The file reflected the Department's determination that Oswald had not expatriated himself.|The file reflected the Department's determination that Oswald had not expatriated himself. +LJ047-0039|From return to Fort Worth to move to New Orleans.|From return to Fort Worth to move to New Orleans. +LJ047-0040|Oswald was first interviewed by FBI Agents John W. Fain and B. Tom Carter on June 26, 1962,|Oswald was first interviewed by FBI Agents John W. Fain and B. Tom Carter on June twenty-six, nineteen sixty-two, +LJ047-0041|Agent Fain reported to headquarters that Oswald was impatient and arrogant,|Agent Fain reported to headquarters that Oswald was impatient and arrogant, +LJ047-0042|and unwilling to answer questions regarding his motive for going to the Soviet Union.|and unwilling to answer questions regarding his motive for going to the Soviet Union. +LJ047-0043|Oswald, quote, denied that he had ever denounced his U.S. citizenship, and that he had ever applied for Soviet citizenship specifically, end quote.|Oswald, quote, denied that he had ever denounced his U.S. citizenship, and that he had ever applied for Soviet citizenship specifically, end quote. +LJ047-0044|Oswald was, however, willing to discuss his contacts with Soviet authorities. He denied having any involvement with Soviet intelligence agencies|Oswald was, however, willing to discuss his contacts with Soviet authorities. He denied having any involvement with Soviet intelligence agencies +LJ047-0045|and promised to advise the FBI if he heard from them.|and promised to advise the FBI if he heard from them. +LJ047-0046|Agent Fain was not satisfied by this interview and arranged to see Oswald again on August 16, 1962.|Agent Fain was not satisfied by this interview and arranged to see Oswald again on August sixteen, nineteen sixty-two. +LJ047-0047|According to Fain's contemporaneous memorandum and his present recollection,|According to Fain's contemporaneous memorandum and his present recollection, +LJ047-0048|while Oswald remained somewhat evasive at this interview, he was not antagonistic and seemed generally to be settling down.|while Oswald remained somewhat evasive at this interview, he was not antagonistic and seemed generally to be settling down. +LJ047-0049|Marina Oswald, however, recalled that her husband was upset by this interview.|Marina Oswald, however, recalled that her husband was upset by this interview. +LJ047-0050|Oswald again agreed to advise the FBI if he were approached under suspicious circumstances; however, he deprecated the possibility of this happening,|Oswald again agreed to advise the FBI if he were approached under suspicious circumstances; however, he deprecated the possibility of this happening, +LJ047-0051|particularly since his employment did not involve any sensitive information.|particularly since his employment did not involve any sensitive information. +LJ047-0052|Having concluded that Oswald was not a security risk or potentially dangerous or violent,|Having concluded that Oswald was not a security risk or potentially dangerous or violent, +LJ047-0053|Fain determined that nothing further remained to be done at that time and recommended that the case be placed in a closed status.|Fain determined that nothing further remained to be done at that time and recommended that the case be placed in a closed status. +LJ047-0054|This is an administrative classification indicating that no further work has been scheduled.|This is an administrative classification indicating that no further work has been scheduled. +LJ047-0055|It does not preclude the agent in charge of the case from reopening it if he feels that further work should be done.|It does not preclude the agent in charge of the case from reopening it if he feels that further work should be done. +LJ047-0056|From August 1962|From August nineteen sixty-two +LJ047-0057|until March 1963, the FBI continued to accumulate information regarding Oswald but engaged in no active investigation.|until March nineteen sixty-three, the FBI continued to accumulate information regarding Oswald but engaged in no active investigation. +LJ047-0058|Agent Fain retired from the FBI in October 1962, and the closed Oswald case was not reassigned.|Agent Fain retired from the FBI in October nineteen sixty-two, and the closed Oswald case was not reassigned. +LJ047-0060|Fain had been assigned to see Marina Oswald at an appropriate time.|Fain had been assigned to see Marina Oswald at an appropriate time. +LJ047-0061|This assignment was given to Agent James P. Hosty, Jr. of the Dallas office upon Fain's retirement.|This assignment was given to Agent James P. Hosty, Jr. of the Dallas office upon Fain's retirement. +LJ047-0062|In March 1963, while attempting to locate Marina Oswald,|In March nineteen sixty-three, while attempting to locate Marina Oswald, +LJ047-0063|Agent Hosty was told by Mrs. M. F. Tobias, a former landlady of the Oswalds at 602 Elsbeth Street in Dallas,|Agent Hosty was told by Mrs. M. F. Tobias, a former landlady of the Oswalds at six oh two Elsbeth Street in Dallas, +LJ047-0064|that other tenants had complained because Oswald was drinking to excess and beating his wife.|that other tenants had complained because Oswald was drinking to excess and beating his wife. +LJ047-0065|This information led Hosty to review Oswald's file, from which he learned that Oswald had become a subscriber to the Worker,|This information led Hosty to review Oswald's file, from which he learned that Oswald had become a subscriber to the Worker, +LJ047-0066|a Communist Party publication.|a Communist Party publication. +LJ047-0067|Hosty decided that the Lee Harvey Oswald case should be reopened because of the alleged personal difficulties and the contact with the Worker,|Hosty decided that the Lee Harvey Oswald case should be reopened because of the alleged personal difficulties and the contact with the Worker, +LJ047-0068|and his recommendation was accepted. He decided, however, not to interview Marina Oswald at that time, and merely determined|and his recommendation was accepted. He decided, however, not to interview Marina Oswald at that time, and merely determined +LJ047-0069|that the Oswalds were living at 214 Neely Street in Dallas.|that the Oswalds were living at two one four Neely Street in Dallas. +LJ047-0070|On April 21, 1963, the FBI field office in New York|On April twenty-one, nineteen sixty-three, the FBI field office in New York +LJ047-0071|was advised that Oswald was in contact with the Fair Play for Cuba Committee in New York, and that he had written to the committee|was advised that Oswald was in contact with the Fair Play for Cuba Committee in New York, and that he had written to the committee +LJ047-0072|stating that he had distributed its pamphlets on the streets of Dallas. This information did not reach Agent Hosty in Dallas until June.|stating that he had distributed its pamphlets on the streets of Dallas. This information did not reach Agent Hosty in Dallas until June. +LJ047-0073|Hosty considered the information to be, quote, stale, unquote, by that time, and did not attempt to verify Oswald's reported statement.|Hosty considered the information to be, quote, stale, unquote, by that time, and did not attempt to verify Oswald's reported statement. +LJ047-0074|Under a general Bureau request to be on the alert for activities of the Fair Play for Cuba Committee|Under a general Bureau request to be on the alert for activities of the Fair Play for Cuba Committee +LJ047-0075|Hosty had inquired earlier and found no evidence that it was functioning in the Dallas area.|Hosty had inquired earlier and found no evidence that it was functioning in the Dallas area. +LJ047-0076|In New Orleans. In the middle of May of 1963, Agent Hosty checked Oswald's last known residence and found that he had moved.|In New Orleans. In the middle of May of nineteen sixty-three, Agent Hosty checked Oswald's last known residence and found that he had moved. +LJ047-0077|Oswald was tentatively located in New Orleans in June,|Oswald was tentatively located in New Orleans in June, +LJ047-0078|and Hosty asked the New Orleans FBI office to determine Oswald's address and what he was doing.|and Hosty asked the New Orleans FBI office to determine Oswald's address and what he was doing. +LJ047-0079|The New Orleans office investigated and located Oswald, learning his address and former place of employment on August 5, 1963.|The New Orleans office investigated and located Oswald, learning his address and former place of employment on August five, nineteen sixty-three. +LJ047-0080|A confidential informant advised the FBI that Oswald was not known to be engaged in Communist Party activities in New Orleans.|A confidential informant advised the FBI that Oswald was not known to be engaged in Communist Party activities in New Orleans. +LJ047-0081|On June 24, Oswald applied in New Orleans for a passport, stating that he planned to depart by ship|On June twenty-four, Oswald applied in New Orleans for a passport, stating that he planned to depart by ship +LJ047-0082|for an extended tour of Western European countries, the Soviet Union, Finland, and Poland.|for an extended tour of Western European countries, the Soviet Union, Finland, and Poland. +LJ047-0083|The Passport Office of the Department of State in Washington had no listing for Oswald requiring special treatment,|The Passport Office of the Department of State in Washington had no listing for Oswald requiring special treatment, +LJ047-0084|and his application was approved on the following day.|and his application was approved on the following day. +LJ047-0085|The FBI had not asked to be informed of any effort by Oswald to obtain a passport,|The FBI had not asked to be informed of any effort by Oswald to obtain a passport, +LJ047-0086|as it might have under existing procedures, and did not know of his application.|as it might have under existing procedures, and did not know of his application. +LJ047-0087|According to the Bureau, quote,|According to the Bureau, quote, +LJ047-0088|We did not request the State Department to include Oswald on a list which would have resulted in advising us of any application for a passport|We did not request the State Department to include Oswald on a list which would have resulted in advising us of any application for a passport +LJ047-0089|inasmuch as the facts relating to Oswald's activities at that time did not warrant such action.|inasmuch as the facts relating to Oswald's activities at that time did not warrant such action. +LJ047-0090|Our investigation of Oswald had disclosed no evidence that Oswald was acting under the instructions or on behalf of|Our investigation of Oswald had disclosed no evidence that Oswald was acting under the instructions or on behalf of +LJ047-0091|any foreign government or instrumentality thereof. End quote.|any foreign government or instrumentality thereof. End quote. +LJ047-0092|On August 9, 1963,|On August nine, nineteen sixty-three, +LJ047-0093|Oswald was arrested and jailed by the New Orleans Police Department for disturbing the peace, in connection with a street fight which broke out when he was accosted|Oswald was arrested and jailed by the New Orleans Police Department for disturbing the peace, in connection with a street fight which broke out when he was accosted +LJ047-0094|by anti-Castro Cubans while distributing leaflets on behalf of the Fair Play for Cuba Committee.|by anti-Castro Cubans while distributing leaflets on behalf of the Fair Play for Cuba Committee. +LJ047-0095|On the next day, he asked the New Orleans police to arrange for him to be interviewed by the FBI.|On the next day, he asked the New Orleans police to arrange for him to be interviewed by the FBI. +LJ047-0096|The police called the local FBI office and an agent, John L. Quigley, was sent to the police station.|The police called the local FBI office and an agent, John L. Quigley, was sent to the police station. +LJ047-0097|Agent Quigley did not know of Oswald's prior FBI record when he interviewed him,|Agent Quigley did not know of Oswald's prior FBI record when he interviewed him, +LJ047-0098|inasmuch as the police had not given Oswald's name to the Bureau when they called the office.|inasmuch as the police had not given Oswald's name to the Bureau when they called the office. +LJ047-0099|Quigley recalled that Oswald was receptive when questioned about his general background|Quigley recalled that Oswald was receptive when questioned about his general background +LJ047-0100|but less than completely truthful or cooperative when interrogated about the Fair Play for Cuba Committee.|but less than completely truthful or cooperative when interrogated about the Fair Play for Cuba Committee. +LJ047-0101|Quigley testified, quote,|Quigley testified, quote, +LJ047-0102|When I began asking him specific details with respect to his activities in the Fair Play for Cuba Committee in New Orleans as to where meetings were held,|When I began asking him specific details with respect to his activities in the Fair Play for Cuba Committee in New Orleans as to where meetings were held, +LJ047-0103|who was involved, what occurred, he was reticent to furnish information,|who was involved, what occurred, he was reticent to furnish information, +LJ047-0104|reluctant and actually as far as I was concerned, was completely evasive on them. End quote.|reluctant and actually as far as I was concerned, was completely evasive on them. End quote. +LJ047-0105|In Quigley's judgment,|In Quigley's judgment, +LJ047-0106|Oswald, quote, was probably making a self-serving statement in attempting to explain to me why he was distributing this literature, and for no other reason,|Oswald, quote, was probably making a self-serving statement in attempting to explain to me why he was distributing this literature, and for no other reason, +LJ047-0107|and when I got to questioning him further then he felt that his purpose had been served and he wouldn't say anything further, end quote.|and when I got to questioning him further then he felt that his purpose had been served and he wouldn't say anything further, end quote. +LJ047-0108|During the interview Quigley obtained background information from Oswald which was inconsistent with information already in the Bureau's possession.|During the interview Quigley obtained background information from Oswald which was inconsistent with information already in the Bureau's possession. +LJ047-0109|When Quigley returned to his office, he learned|When Quigley returned to his office, he learned +LJ047-0110|that another Bureau agent, Milton R. Knack, had been conducting a background investigation of Oswald at the request of Agent Hosty in Dallas.|that another Bureau agent, Milton R. Knack, had been conducting a background investigation of Oswald at the request of Agent Hosty in Dallas. +LJ047-0111|Quigley advised Knack of his interview and gave him a detailed memorandum.|Quigley advised Knack of his interview and gave him a detailed memorandum. +LJ047-0112|Knack was aware of the facts known to the FBI and recognized Oswald's false statements.|Knack was aware of the facts known to the FBI and recognized Oswald's false statements. +LJ047-0113|For example, Oswald claimed that his wife's maiden name was Prossa|For example, Oswald claimed that his wife's maiden name was Prossa +LJ047-0114|and that they had been married in Fort Worth and lived there until coming to New Orleans. He had told the New Orleans arresting officers that he had been born in Cuba.|and that they had been married in Fort Worth and lived there until coming to New Orleans. He had told the New Orleans arresting officers that he had been born in Cuba. +LJ047-0115|Several days later, the Bureau received additional evidence that Oswald had lied to Agent Quigley.|Several days later, the Bureau received additional evidence that Oswald had lied to Agent Quigley. +LJ047-0116|On August 22, it learned that Oswald had appeared on a radio discussion program on August 21.|On August twenty-two, it learned that Oswald had appeared on a radio discussion program on August twenty-one. +LJ047-0117|William Stuckey, who had appeared on the radio program with Oswald, told the Bureau on August 30|William Stuckey, who had appeared on the radio program with Oswald, told the Bureau on August thirty +LJ047-0118|that Oswald had told him that he had worked and been married in the Soviet Union.|that Oswald had told him that he had worked and been married in the Soviet Union. +LJ047-0119|Neither these discrepancies nor the fact that Oswald had initiated the FBI interview|Neither these discrepancies nor the fact that Oswald had initiated the FBI interview +LJ047-0120|was considered sufficiently unusual to necessitate another interview. Alan H. Belmont, Assistant to the Director of the FBI,|was considered sufficiently unusual to necessitate another interview. Alan H. Belmont, Assistant to the Director of the FBI, +LJ047-0121|stated the Bureau's reasoning in this way, quote,|stated the Bureau's reasoning in this way, quote, +LJ047-0122|Our interest in this man at this point was to determine whether his activities constituted a threat to the internal security of the country.|Our interest in this man at this point was to determine whether his activities constituted a threat to the internal security of the country. +LJ047-0123|It was apparent that he had made a self-serving statement to Agent Quigley. It became a matter of record in our files as a part of the case,|It was apparent that he had made a self-serving statement to Agent Quigley. It became a matter of record in our files as a part of the case, +LJ047-0124|and if we determined that the course of the investigation required us to clarify or face him down with this information, we would do it at the appropriate time.|and if we determined that the course of the investigation required us to clarify or face him down with this information, we would do it at the appropriate time. +LJ047-0125|In other words, he committed no violation of the law by telling us something that wasn't true, and unless this required further investigation at that time,|In other words, he committed no violation of the law by telling us something that wasn't true, and unless this required further investigation at that time, +LJ047-0126|we would handle it in due course, in accord with the whole context of the investigation. End quote.|we would handle it in due course, in accord with the whole context of the investigation. End quote. +LJ047-0127|On August 21, 1963, Bureau headquarters instructed the New Orleans and Dallas field offices|On August twenty-one, nineteen sixty-three, Bureau headquarters instructed the New Orleans and Dallas field offices +LJ047-0128|to conduct an additional investigation of Oswald in view of the activities which had led to his arrest.|to conduct an additional investigation of Oswald in view of the activities which had led to his arrest. +LJ047-0129|FBI informants in the New Orleans area, familiar with pro-Castro or Communist Party activity there,|FBI informants in the New Orleans area, familiar with pro-Castro or Communist Party activity there, +LJ047-0130|advised the Bureau that Oswald was unknown in such circles.|advised the Bureau that Oswald was unknown in such circles. +LJ047-0131|In early September 1963|In early September nineteen sixty-three +LJ047-0132|the FBI transferred the principal responsibility for the Oswald case from the Dallas office to the New Orleans office.|the FBI transferred the principal responsibility for the Oswald case from the Dallas office to the New Orleans office. +LJ047-0133|Soon after, on October 1, 1963,|Soon after, on October one, nineteen sixty-three, +LJ047-0134|The FBI was advised by the rental agent for the Oswalds' apartment in New Orleans that they had moved again.|The FBI was advised by the rental agent for the Oswalds' apartment in New Orleans that they had moved again. +LJ047-0135|According to the information received by the Bureau|According to the information received by the Bureau +LJ047-0136|they had vacated their apartment, and Marina Oswald had departed with their child in a station wagon with Texas registration.|they had vacated their apartment, and Marina Oswald had departed with their child in a station wagon with Texas registration. +LJ047-0137|On October 3, Hosty reopened the case in Dallas to assist the New Orleans office.|On October three, Hosty reopened the case in Dallas to assist the New Orleans office. +LJ047-0138|He checked in Oswald's old neighborhood and throughout the Dallas-Fort Worth area but was unable to locate Oswald.|He checked in Oswald's old neighborhood and throughout the Dallas-Fort Worth area but was unable to locate Oswald. +LJ047-0139|The next word about Oswald's location was a communication from the CIA to the FBI on October 10,|The next word about Oswald's location was a communication from the CIA to the FBI on October ten, +LJ047-0140|advising that an individual tentatively identified as Oswald had been in touch with the Soviet Embassy in Mexico City|advising that an individual tentatively identified as Oswald had been in touch with the Soviet Embassy in Mexico City +LJ047-0141|in early October of 1963.|in early October of nineteen sixty-three. +LJ047-0143|The possible contact with the Soviet Embassy in Mexico intensified the FBI's interest in learning Oswald's whereabouts.|The possible contact with the Soviet Embassy in Mexico intensified the FBI's interest in learning Oswald's whereabouts. +LJ047-0144|The FBI representative in Mexico City arranged to follow up this information with the CIA and to verify Oswald's entry into Mexico.|The FBI representative in Mexico City arranged to follow up this information with the CIA and to verify Oswald's entry into Mexico. +LJ047-0145|The CIA message was sent also to the Department of State where it was reviewed by personnel of the Passport Office, who knew from Oswald's file|The CIA message was sent also to the Department of State where it was reviewed by personnel of the Passport Office, who knew from Oswald's file +LJ047-0146|that he had sought and obtained a passport on June 25, 1963.|that he had sought and obtained a passport on June twenty-five, nineteen sixty-three. +LJ047-0147|The Department of State did not advise either the CIA or the FBI of these facts.|The Department of State did not advise either the CIA or the FBI of these facts. +LJ047-0148|On October 25,|On October twenty-five, +LJ047-0149|the New Orleans office of the FBI learned that in September Oswald had given a forwarding address of 2515|the New Orleans office of the FBI learned that in September Oswald had given a forwarding address of two five one five +LJ047-0150|West Fifth Street, Irving, Texas. After receiving this information on October 29, Agent Hosty attempted to locate Oswald.|West Fifth Street, Irving, Texas. After receiving this information on October twenty-nine, Agent Hosty attempted to locate Oswald. +LJ047-0151|On the same day Hosty interviewed neighbors on Fifth Street and learned that the address was that of Mrs. Ruth Paine.|On the same day Hosty interviewed neighbors on Fifth Street and learned that the address was that of Mrs. Ruth Paine. +LJ047-0152|He conducted a limited background investigation of the Paines, intending to interview Mrs. Paine and ask her particularly about Oswald's whereabouts.|He conducted a limited background investigation of the Paines, intending to interview Mrs. Paine and ask her particularly about Oswald's whereabouts. +LJ047-0153|Having determined that Mrs. Paine was a responsible and reliable citizen, Hosty interviewed her on November 1.|Having determined that Mrs. Paine was a responsible and reliable citizen, Hosty interviewed her on November one. +LJ047-0154|The interview lasted about 20 to 25 minutes. In response to Hosty's inquiries, Mrs. Paine, quote,|The interview lasted about twenty to twenty-five minutes. In response to Hosty's inquiries, Mrs. Paine, quote, +LJ047-0155|readily admitted that Mrs. Marina Oswald and Lee Oswald's two children were staying with her.|readily admitted that Mrs. Marina Oswald and Lee Oswald's two children were staying with her. +LJ047-0156|She said that Lee Oswald was living somewhere in Dallas. She didn't know where. She said it was in the Oak Cliff area but she didn't have his address.|She said that Lee Oswald was living somewhere in Dallas. She didn't know where. She said it was in the Oak Cliff area but she didn't have his address. +LJ047-0157|I asked her if she knew where he worked.|I asked her if she knew where he worked. +LJ047-0158|After a moment's hesitation, she told me that he worked at the Texas School Book Depository near the downtown area of Dallas.|After a moment's hesitation, she told me that he worked at the Texas School Book Depository near the downtown area of Dallas. +LJ047-0159|She didn't have the exact address, and it is my recollection that we went to the phone book and looked it up,|She didn't have the exact address, and it is my recollection that we went to the phone book and looked it up, +LJ047-0160|found it to be 411 Elm Street. End quote.|found it to be four one one Elm Street. End quote. +LJ047-0161|Mrs. Paine told Hosty also|Mrs. Paine told Hosty also +LJ047-0162|that Oswald was living alone in Dallas because she did not want him staying at her house, although she was willing to let Oswald visit his wife and children.|that Oswald was living alone in Dallas because she did not want him staying at her house, although she was willing to let Oswald visit his wife and children. +LJ047-0163|According to Hosty, Mrs. Paine indicated that she thought she could find out where Oswald was living and would let him know.|According to Hosty, Mrs. Paine indicated that she thought she could find out where Oswald was living and would let him know. +LJ047-0164|At this point in the interview, Hosty gave Mrs. Paine his name and office telephone number on a piece of paper.|At this point in the interview, Hosty gave Mrs. Paine his name and office telephone number on a piece of paper. +LJ047-0165|At the end of the interview, Marina Oswald came into the room. When he observed that she seemed, quote, quite alarmed, end quote, about the visit,|At the end of the interview, Marina Oswald came into the room. When he observed that she seemed, quote, quite alarmed, end quote, about the visit, +LJ047-0166|Hosty assured her, through Mrs. Paine as interpreter, that the FBI would not harm or harass her.|Hosty assured her, through Mrs. Paine as interpreter, that the FBI would not harm or harass her. +LJ047-0167|On November 4, Hosty telephoned the Texas School Book Depository and learned that Oswald was working there|On November four, Hosty telephoned the Texas School Book Depository and learned that Oswald was working there +LJ047-0168|and that he had given as his address Mrs. Paine's residence in Irving.|and that he had given as his address Mrs. Paine's residence in Irving. +LJ047-0169|Hosty took the necessary steps to have the Dallas office of the FBI, rather than the New Orleans office, reestablished as the office with principal responsibility.|Hosty took the necessary steps to have the Dallas office of the FBI, rather than the New Orleans office, reestablished as the office with principal responsibility. +LJ047-0170|On November 5, Hosty was traveling near Mrs. Paine's home and took the occasion to stop by to ask whether she had any further information.|On November five, Hosty was traveling near Mrs. Paine's home and took the occasion to stop by to ask whether she had any further information. +LJ047-0171|Mrs. Paine had nothing to add to what she had already told him, except that during a visit that past weekend,|Mrs. Paine had nothing to add to what she had already told him, except that during a visit that past weekend, +LJ047-0172|Oswald had said that he was a, quote, Trotskyite Communist, end quote.|Oswald had said that he was a, quote, Trotskyite Communist, end quote. +LJ047-0173|and that she found this and similar statements illogical and somewhat amusing. On this occasion Hosty was at the Paine residence for only a few minutes.|and that she found this and similar statements illogical and somewhat amusing. On this occasion Hosty was at the Paine residence for only a few minutes. +LJ047-0174|During neither interview did Hosty learn Oswald's address|During neither interview did Hosty learn Oswald's address +LJ047-0175|or telephone number in Dallas. Mrs. Paine testified that she learned Oswald's telephone number at the Beckley Street roominghouse in the middle of October|or telephone number in Dallas. Mrs. Paine testified that she learned Oswald's telephone number at the Beckley Street roominghouse in the middle of October +LJ047-0176|shortly after Oswald rented the room on October 14.|shortly after Oswald rented the room on October fourteen. +LJ047-0177|As discussed in chapter 6, she failed to report this to Agent Hosty|As discussed in chapter six, she failed to report this to Agent Hosty +LJ047-0178|because she thought the FBI was in possession of a great deal of information and certainly would find it very easy to learn where Oswald was living.|because she thought the FBI was in possession of a great deal of information and certainly would find it very easy to learn where Oswald was living. +LJ047-0179|Hosty did nothing further in connection with the Oswald case until after the assassination. On November 1, 1963,|Hosty did nothing further in connection with the Oswald case until after the assassination. On November one, nineteen sixty-three, +LJ047-0180|he had received a copy of the report of the New Orleans office which contained Agent Quigley's memorandum of the interview in the New Orleans jail on August 10,|he had received a copy of the report of the New Orleans office which contained Agent Quigley's memorandum of the interview in the New Orleans jail on August ten, +LJ047-0181|and realized immediately that Oswald had given false biographic information. Hosty knew that he would eventually have to investigate this, and, quote,|and realized immediately that Oswald had given false biographic information. Hosty knew that he would eventually have to investigate this, and, quote, +LJ047-0182|was quite interested in determining the nature of his contact with the Soviet Embassy in Mexico City, end quote.|was quite interested in determining the nature of his contact with the Soviet Embassy in Mexico City, end quote. +LJ047-0183|When asked what his next step would have been, Hosty replied, quote,|When asked what his next step would have been, Hosty replied, quote, +LJ047-0184|Well, as I had previously stated, I have between 25 and 40 cases assigned to me at any one time. I had other matters to take care of.|Well, as I had previously stated, I have between twenty-five and forty cases assigned to me at any one time. I had other matters to take care of. +LJ047-0185|I had now established that Lee Oswald was not employed in a sensitive industry.|I had now established that Lee Oswald was not employed in a sensitive industry. +LJ047-0186|I can now afford to wait until New Orleans forwarded the necessary papers to me to show me I now had all the information.|I can now afford to wait until New Orleans forwarded the necessary papers to me to show me I now had all the information. +LJ047-0187|It was then my plan to interview Marina Oswald in detail concerning both herself and her husband's background. Question:|It was then my plan to interview Marina Oswald in detail concerning both herself and her husband's background. Question: +LJ047-0188|Had you planned any steps beyond that point?|Had you planned any steps beyond that point? +LJ047-0189|Answer: No. I would have to wait until I had talked to Marina to see what I could determine, and from there I could make my plans.|Answer: No. I would have to wait until I had talked to Marina to see what I could determine, and from there I could make my plans. +LJ047-0190|Question: Did you take any action on this case between November 5 and November 22? Answer: No, sir.|Question: Did you take any action on this case between November five and November twenty-two? Answer: No, sir. +LJ047-0191|The official Bureau files confirm Hosty's statement that from November 5 until the assassination, no active investigation was conducted.|The official Bureau files confirm Hosty's statement that from November five until the assassination, no active investigation was conducted. +LJ047-0192|On November 18 the FBI learned that Oswald recently had been in communication with the Soviet Embassy in Washington|On November eighteen the FBI learned that Oswald recently had been in communication with the Soviet Embassy in Washington +LJ047-0193|and so advised the Dallas office in the ordinary course of business.|and so advised the Dallas office in the ordinary course of business. +LJ047-0194|Hosty received this information on the afternoon of November 22, 1963.|Hosty received this information on the afternoon of November twenty-two, nineteen sixty-three. +LJ047-0195|Nonreferral of Oswald to the Secret Service.|Nonreferral of Oswald to the Secret Service. +LJ047-0196|The Commission has considered carefully the question whether the FBI,|The Commission has considered carefully the question whether the FBI, +LJ047-0197|in view of all the information concerning Oswald in its files, should have alerted the Secret Service to Oswald's presence in Dallas|in view of all the information concerning Oswald in its files, should have alerted the Secret Service to Oswald's presence in Dallas +LJ047-0198|prior to President Kennedy's visit.|prior to President Kennedy's visit. +LJ047-0199|The Secret Service and the FBI differ|The Secret Service and the FBI differ +LJ047-0200|as to whether Oswald fell within the category of, quote, threats against the President, end quote, which should be referred to the Service.|as to whether Oswald fell within the category of, quote, threats against the President, end quote, which should be referred to the Service. +LJ047-0201|Robert I Bouck, special agent in charge of the Protective Research Section,|Robert I Bouck, special agent in charge of the Protective Research Section, +LJ047-0202|testified that the information available to the Federal Government about Oswald before the assassination would, if known to PRS,|testified that the information available to the Federal Government about Oswald before the assassination would, if known to PRS, +LJ047-0203|have made Oswald a subject of concern to the Secret Service.|have made Oswald a subject of concern to the Secret Service. +LJ047-0204|Bouck pointed to a number of characteristics besides Oswald's defection the cumulative effect of which would have been to alert the Secret Service|Bouck pointed to a number of characteristics besides Oswald's defection the cumulative effect of which would have been to alert the Secret Service +LJ047-0205|to potential danger, quote,|to potential danger, quote, +LJ047-0206|I would think his continued association with the Russian Embassy after his return,|I would think his continued association with the Russian Embassy after his return, +LJ047-0207|his association with the Castro groups would have been of concern to us, a knowledge that he had, I believe,|his association with the Castro groups would have been of concern to us, a knowledge that he had, I believe, +LJ047-0208|been courtmartialed for illegal possession of a gun, of a handgun in the Marines,|been courtmartialed for illegal possession of a gun, of a handgun in the Marines, +LJ047-0209|that he had owned a weapon and did a good deal of hunting or use of it, perhaps in Russia, plus a number of items about his disposition and unreliability of character,|that he had owned a weapon and did a good deal of hunting or use of it, perhaps in Russia, plus a number of items about his disposition and unreliability of character, +LJ047-0210|I think all of those, if we had them all together,|I think all of those, if we had them all together, +LJ047-0211|would have added up to pointing out a pretty bad individual, and I think that, together, had we known that he had a vantage point|would have added up to pointing out a pretty bad individual, and I think that, together, had we known that he had a vantage point +LJ047-0212|would have seemed somewhat serious to us, even though I must admit, that none of these in themselves would be|would have seemed somewhat serious to us, even though I must admit, that none of these in themselves would be +LJ047-0213|would meet our specific criteria, none of them alone. But, it is when you begin adding them up to some degree that you begin to get criteria that, are meaningful.|would meet our specific criteria, none of them alone. But, it is when you begin adding them up to some degree that you begin to get criteria that, are meaningful. +LJ047-0214|End quote. Mr. Bouck pointed out, however, that he had no reason to believe that any one Federal agency had access to all this information,|End quote. Mr. Bouck pointed out, however, that he had no reason to believe that any one Federal agency had access to all this information, +LJ047-0215|including the significant fact that Oswald was employed in a building which overlooked the motorcade route.|including the significant fact that Oswald was employed in a building which overlooked the motorcade route. +LJ047-0216|Agent Hosty testified that he was fully aware of the pending Presidential visit to Dallas.|Agent Hosty testified that he was fully aware of the pending Presidential visit to Dallas. +LJ047-0217|He recalled that the special agent in charge of the Dallas office of the FBI, J. Gordon Shanklin,|He recalled that the special agent in charge of the Dallas office of the FBI, J. Gordon Shanklin, +LJ047-0218|had discussed the President's visit on several occasions, including the regular biweekly conference on the morning of November 22|had discussed the President's visit on several occasions, including the regular biweekly conference on the morning of November twenty-two +LJ047-0219|Quote, Mr. Shanklin advised us, among other things,|Quote, Mr. Shanklin advised us, among other things, +LJ047-0220|that in view of the President's visit to Dallas, that if anyone had any indication of any possibility of any acts of violence or any demonstrations against the President,|that in view of the President's visit to Dallas, that if anyone had any indication of any possibility of any acts of violence or any demonstrations against the President, +LJ047-0221|or Vice President, to immediately notify the Secret Service and confirm it in writing.|or Vice President, to immediately notify the Secret Service and confirm it in writing. +LJ047-0222|He had made the same statement about a week prior at another special conference which we had held.|He had made the same statement about a week prior at another special conference which we had held. +LJ047-0223|I don't recall the exact date. It was about a week prior. End quote.|I don't recall the exact date. It was about a week prior. End quote. +LJ047-0224|In fact, Hosty participated in transmitting to the Secret Service two pieces of information pertaining to the visit.|In fact, Hosty participated in transmitting to the Secret Service two pieces of information pertaining to the visit. +LJ047-0225|Hosty testified that he did not know until the evening of Thursday, November 21, that there was to be a motorcade, however,|Hosty testified that he did not know until the evening of Thursday, November twenty-one, that there was to be a motorcade, however, +LJ047-0226|and never realized that the motorcade would pass the Texas School Book Depository Building.|and never realized that the motorcade would pass the Texas School Book Depository Building. +LJ047-0227|He testified that he did not read the newspaper story describing the motorcade route in detail, since he was interested only in the fact|He testified that he did not read the newspaper story describing the motorcade route in detail, since he was interested only in the fact +LJ047-0228|that the motorcade was coming up Main Street, quote, where maybe I could watch it if I had a chance, end quote.|that the motorcade was coming up Main Street, quote, where maybe I could watch it if I had a chance, end quote. +LJ047-0229|Even if he had recalled that Oswald's place of employment was on the President's route,|Even if he had recalled that Oswald's place of employment was on the President's route, +LJ047-0230|Hosty testified that he would not have cited him to the Secret Service as a potential threat to the President. Hosty interpreted his instructions as requiring, quote,|Hosty testified that he would not have cited him to the Secret Service as a potential threat to the President. Hosty interpreted his instructions as requiring, quote, +LJ047-0231|some indication that the person planned to take some action against the safety of the President of the United States or the Vice President. End quote.|some indication that the person planned to take some action against the safety of the President of the United States or the Vice President. End quote. +LJ047-0232|In his opinion, none of the information in the FBI files -- Oswald's defection, his Fair Play for Cuba activities in New Orleans,|In his opinion, none of the information in the FBI files -- Oswald's defection, his Fair Play for Cuba activities in New Orleans, +LJ047-0233|his lies to Agent Quigley, his recent visit to Mexico City -- indicated that Oswald was capable of violence.|his lies to Agent Quigley, his recent visit to Mexico City -- indicated that Oswald was capable of violence. +LJ047-0234|Hosty's initial reaction on hearing that Oswald was a suspect in the assassination, was, quote, shock|Hosty's initial reaction on hearing that Oswald was a suspect in the assassination, was, quote, shock +LJ047-0235|complete surprise, end quote, because he had no reason to believe that Oswald, quote,|complete surprise, end quote, because he had no reason to believe that Oswald, quote, +LJ047-0236|was capable or potentially an assassin of the President of the United States, end quote.|was capable or potentially an assassin of the President of the United States, end quote. +LJ047-0237|Shortly after Oswald was apprehended and identified, Hosty's superior sent him to observe the interrogation of Oswald.|Shortly after Oswald was apprehended and identified, Hosty's superior sent him to observe the interrogation of Oswald. +LJ047-0238|Hosty parked his car in the basement of police headquarters and there met an acquaintance, Lt. Jack Revill of the Dallas police force.|Hosty parked his car in the basement of police headquarters and there met an acquaintance, Lt. Jack Revill of the Dallas police force. +LJ047-0239|The two men disagree about the conversation which took place between them.|The two men disagree about the conversation which took place between them. +LJ047-0240|They agree that Hosty told Revill|They agree that Hosty told Revill +LJ047-0241|that the FBI had known about Oswald and, in particular, of his presence in Dallas and his employment at the Texas School Book Depository Building.|that the FBI had known about Oswald and, in particular, of his presence in Dallas and his employment at the Texas School Book Depository Building. +LJ047-0242|Revill testified that Hosty said also that the FBI had information that Oswald was, quote, capable of committing this assassination, end quote.|Revill testified that Hosty said also that the FBI had information that Oswald was, quote, capable of committing this assassination, end quote. +LJ047-0243|According to Revill, Hosty indicated that he was going to tell this to Lieutenant Wells of the homicide and robbery bureau.|According to Revill, Hosty indicated that he was going to tell this to Lieutenant Wells of the homicide and robbery bureau. +LJ047-0244|Revill promptly made a memorandum of this conversation in which the quoted statement appears.|Revill promptly made a memorandum of this conversation in which the quoted statement appears. +LJ047-0245|His secretary testified that she prepared such a report for him that afternoon and Chief of Police Jesse E. Curry|His secretary testified that she prepared such a report for him that afternoon and Chief of Police Jesse E. Curry +LJ047-0246|and District Attorney Henry M. Wade both testified that they saw it later that day.|and District Attorney Henry M. Wade both testified that they saw it later that day. +LJ047-0247|Hosty has unequivocally denied, first by affidavit and then in his testimony before the Commission,|Hosty has unequivocally denied, first by affidavit and then in his testimony before the Commission, +LJ047-0248|that he ever said that Oswald was capable of violence, or that he had any information suggesting this.|that he ever said that Oswald was capable of violence, or that he had any information suggesting this. +LJ047-0249|The only witness to the conversation was Dallas Police Detective V. J. Brian, who was accompanying Revill.|The only witness to the conversation was Dallas Police Detective V. J. Brian, who was accompanying Revill. +LJ047-0250|Brian did not hear Hosty make any statement concerning Oswald's capacity to be an assassin|Brian did not hear Hosty make any statement concerning Oswald's capacity to be an assassin +LJ048-0001|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ048-0002|Chapter 8. The Protection of the President. Part 3.|Chapter eight. The Protection of the President. Part three. +LJ048-0003|Hosty's interpretation of the prevailing FBI instructions on referrals to the Secret Service was defended before the Commission by his superiors.|Hosty's interpretation of the prevailing FBI instructions on referrals to the Secret Service was defended before the Commission by his superiors. +LJ048-0004|After summarizing the Bureau's investigative interest in Oswald prior to the assassination, J. Edgar Hoover concluded that, quote,|After summarizing the Bureau's investigative interest in Oswald prior to the assassination, J. Edgar Hoover concluded that, quote, +LJ048-0005|There was nothing up to the time of the assassination that gave any indication that this man was a dangerous character who might do harm to the President|There was nothing up to the time of the assassination that gave any indication that this man was a dangerous character who might do harm to the President +LJ048-0006|or to the Vice President, end quote.|or to the Vice President, end quote. +LJ048-0007|Director Hoover emphasized that the first indication of Oswald's capacity for violence|Director Hoover emphasized that the first indication of Oswald's capacity for violence +LJ048-0008|was his attempt on General Walker's life, which did not become known to the FBI until after the assassination.|was his attempt on General Walker's life, which did not become known to the FBI until after the assassination. +LJ048-0009|Both Director Hoover and his assistant, Alan H. Belmont,|Both Director Hoover and his assistant, Alan H. Belmont, +LJ048-0010|stressed also the decision by the Department of State that Oswald should be permitted to return to the United States.|stressed also the decision by the Department of State that Oswald should be permitted to return to the United States. +LJ048-0011|Neither believed that the Bureau investigation of him up to November 22 revealed any information which would have justified referral to the Secret Service.|Neither believed that the Bureau investigation of him up to November twenty-two revealed any information which would have justified referral to the Secret Service. +LJ048-0012|According to Belmont, when Oswald returned from the Soviet Union, quote,|According to Belmont, when Oswald returned from the Soviet Union, quote, +LJ048-0013|he indicated that he had learned his lesson,|he indicated that he had learned his lesson, +LJ048-0014|was disenchanted with Russia, and had a renewed concept -- I am paraphrasing, a renewed concept -- of the American free society.|was disenchanted with Russia, and had a renewed concept -- I am paraphrasing, a renewed concept -- of the American free society. +LJ048-0015|We talked to him twice.|We talked to him twice. +LJ048-0016|He likewise indicated he was disenchanted with Russia.|He likewise indicated he was disenchanted with Russia. +LJ048-0017|We satisfied ourselves that we had met our requirement, namely to find out whether he had been recruited by Soviet intelligence. The case was closed.|We satisfied ourselves that we had met our requirement, namely to find out whether he had been recruited by Soviet intelligence. The case was closed. +LJ048-0018|We again exhibited interest on the basis of these contacts with The Worker, Fair Play for Cuba Committee, which are relatively inconsequential.|We again exhibited interest on the basis of these contacts with The Worker, Fair Play for Cuba Committee, which are relatively inconsequential. +LJ048-0019|His activities for the Fair Play for Cuba Committee in New Orleans, we knew, were not of real consequence as he was not connected with any organized activity there.|His activities for the Fair Play for Cuba Committee in New Orleans, we knew, were not of real consequence as he was not connected with any organized activity there. +LJ048-0020|The interview with him in jail is not significant from the standpoint of whether he had a propensity for violence.|The interview with him in jail is not significant from the standpoint of whether he had a propensity for violence. +LJ048-0021|Question: This is the Quigley interview you are talking about?|Question: This is the Quigley interview you are talking about? +LJ048-0022|Answer: Yes. It was a self-serving interview. The visits with the Soviet Embassy were evidently for the purpose of securing a visa,|Answer: Yes. It was a self-serving interview. The visits with the Soviet Embassy were evidently for the purpose of securing a visa, +LJ048-0023|and he had told us during one of the interviews that he would probably take his wife back to Soviet Russia some time in the future.|and he had told us during one of the interviews that he would probably take his wife back to Soviet Russia some time in the future. +LJ048-0024|He had come back to Dallas. Hosty had established that he had a job, he was working,|He had come back to Dallas. Hosty had established that he had a job, he was working, +LJ048-0025|and had told Mrs. Paine that when he got the money he was going to take an apartment, when the baby was old enough, he was going to take an apartment, and the family would live together.|and had told Mrs. Paine that when he got the money he was going to take an apartment, when the baby was old enough, he was going to take an apartment, and the family would live together. +LJ048-0026|He gave evidence of settling down.|He gave evidence of settling down. +LJ048-0027|Nowhere during the course of this investigation or the information that came to us from other agencies was there any indication of a potential for violence on his part.|Nowhere during the course of this investigation or the information that came to us from other agencies was there any indication of a potential for violence on his part. +LJ048-0028|Consequently, there was no basis for Hosty to go to Secret Service and advise them of Oswald's presence. End quote|Consequently, there was no basis for Hosty to go to Secret Service and advise them of Oswald's presence. End quote +LJ048-0029|As reflected in this testimony,|As reflected in this testimony, +LJ048-0030|the officials of the FBI believed that there was no data in its files which gave warning that Oswald was a source of danger to President Kennedy.|the officials of the FBI believed that there was no data in its files which gave warning that Oswald was a source of danger to President Kennedy. +LJ048-0031|While he had expressed hostility at times toward the State Department, the Marine Corps, and the FBI as agents of the Government,|While he had expressed hostility at times toward the State Department, the Marine Corps, and the FBI as agents of the Government, +LJ048-0032|so far as the FBI knew he had not shown any potential for violence.|so far as the FBI knew he had not shown any potential for violence. +LJ048-0033|Prior to November 22, 1963|Prior to November twenty-two, nineteen sixty-three +LJ048-0034|no law enforcement agency had any information to connect Oswald with the attempted shooting of General Walker.|no law enforcement agency had any information to connect Oswald with the attempted shooting of General Walker. +LJ048-0036|that agents of the FBI in Dallas did not consider Oswald's presence in the Texas School Book Depository Building|that agents of the FBI in Dallas did not consider Oswald's presence in the Texas School Book Depository Building +LJ048-0037|overlooking the motorcade route as a source of danger to the President and did not inform the Secret Service of his employment in the Depository Building.|overlooking the motorcade route as a source of danger to the President and did not inform the Secret Service of his employment in the Depository Building. +LJ048-0038|The Commission believes, however,|The Commission believes, however, +LJ048-0039|that the FBI took an unduly restrictive view of its responsibilities in preventive intelligence work, prior to the assassination.|that the FBI took an unduly restrictive view of its responsibilities in preventive intelligence work, prior to the assassination. +LJ048-0040|The Commission appreciates the large volume of cases handled by the FBI|The Commission appreciates the large volume of cases handled by the FBI +LJ048-0041|636,371 investigative matters during fiscal year 1963|six hundred thirty-six thousand, three hundred seventy-one investigative matters during fiscal year nineteen sixty-three +LJ048-0042|There were no Secret Service criteria which specifically required the referral of Oswald's case to the Secret Service;|There were no Secret Service criteria which specifically required the referral of Oswald's case to the Secret Service; +LJ048-0043|nor was there any requirement to report the names of defectors. However, there was much material in the hands of the FBI about Oswald:|nor was there any requirement to report the names of defectors. However, there was much material in the hands of the FBI about Oswald: +LJ048-0044|the knowledge of his defection, his arrogance and hostility to the United States,|the knowledge of his defection, his arrogance and hostility to the United States, +LJ048-0045|his pro-Castro tendencies, his lies when interrogated by the FBI,|his pro-Castro tendencies, his lies when interrogated by the FBI, +LJ048-0046|his trip to Mexico where he was in contact with Soviet authorities,|his trip to Mexico where he was in contact with Soviet authorities, +LJ048-0047|his presence in the School Book Depository job and its location along the route of the motorcade.|his presence in the School Book Depository job and its location along the route of the motorcade. +LJ048-0048|All this does seem to amount to enough to have induced an alert agency, such as the FBI,|All this does seem to amount to enough to have induced an alert agency, such as the FBI, +LJ048-0049|possessed of this information to list Oswald as a potential threat to the safety of the President.|possessed of this information to list Oswald as a potential threat to the safety of the President. +LJ048-0050|This conclusion may be tinged with hindsight, but|This conclusion may be tinged with hindsight, but +LJ048-0051|it stated primarily to direct the thought of those responsible for the future safety of our Presidents to the need for a more imaginative|it stated primarily to direct the thought of those responsible for the future safety of our Presidents to the need for a more imaginative +LJ048-0052|and less narrow interpretation of their responsibilities.|and less narrow interpretation of their responsibilities. +LJ048-0053|It is the conclusion of the Commission that, even in the absence of Secret Service criteria|It is the conclusion of the Commission that, even in the absence of Secret Service criteria +LJ048-0054|which specifically required the referral of such a case as Oswald's to the Secret Service,|which specifically required the referral of such a case as Oswald's to the Secret Service, +LJ048-0055|a more alert and carefully considered treatment of the Oswald case by the Bureau might have brought about such a referral.|a more alert and carefully considered treatment of the Oswald case by the Bureau might have brought about such a referral. +LJ048-0056|Had such a review been undertaken by the FBI,|Had such a review been undertaken by the FBI, +LJ048-0057|there might conceivably have been additional investigation of the Oswald case between November 5 and November 22.|there might conceivably have been additional investigation of the Oswald case between November five and November twenty-two. +LJ048-0058|Agent Hosty testified|Agent Hosty testified +LJ048-0059|that several matters brought to his attention in late October and early November, including the visit to the Soviet Embassy in Mexico City,|that several matters brought to his attention in late October and early November, including the visit to the Soviet Embassy in Mexico City, +LJ048-0060|required further attention.|required further attention. +LJ048-0061|Under proper procedures knowledge of the pending Presidential visit might have prompted Hosty|Under proper procedures knowledge of the pending Presidential visit might have prompted Hosty +LJ048-0062|to have made more vigorous efforts to locate Oswald's roominghouse address in Dallas and to interview him regarding these unresolved matters.|to have made more vigorous efforts to locate Oswald's roominghouse address in Dallas and to interview him regarding these unresolved matters. +LJ048-0064|While the Secret Service bears the principal responsibility for this failure,|While the Secret Service bears the principal responsibility for this failure, +LJ048-0065|the FBI instructions did not reflect fully the Secret Service's need for information regarding potential threats.|the FBI instructions did not reflect fully the Secret Service's need for information regarding potential threats. +LJ048-0066|The handbook referred thus to, quote, the possibility of an attempt against the person or safety of the President, end quote.|The handbook referred thus to, quote, the possibility of an attempt against the person or safety of the President, end quote. +LJ048-0067|It is clear from Hosty's testimony that this was construed, at least by him,|It is clear from Hosty's testimony that this was construed, at least by him, +LJ048-0068|as requiring evidence of a plan or conspiracy to injure the President.|as requiring evidence of a plan or conspiracy to injure the President. +LJ048-0069|Efforts made by the Bureau since the assassination, on the other hand,|Efforts made by the Bureau since the assassination, on the other hand, +LJ048-0070|reflect keen awareness of the necessity of communicating a much wider range of intelligence information to the Service.|reflect keen awareness of the necessity of communicating a much wider range of intelligence information to the Service. +LJ048-0071|Most important, notwithstanding that both agencies have professed to the Commission that the liaison between them was close and fully sufficient,|Most important, notwithstanding that both agencies have professed to the Commission that the liaison between them was close and fully sufficient, +LJ048-0072|the Commission does not believe that the liaison between the FBI and the Secret Service prior to the assassination was as effective as it should have been.|the Commission does not believe that the liaison between the FBI and the Secret Service prior to the assassination was as effective as it should have been. +LJ048-0073|The FBI Manual of Instructions provided, quote, Liaison With Other Government Agencies|The FBI Manual of Instructions provided, quote, Liaison With Other Government Agencies +LJ048-0074|To insure adequate and effective liaison arrangements,|To insure adequate and effective liaison arrangements, +LJ048-0075|each SAC should specifically designate an Agent (or Agents) to be responsible for developing|each SAC should specifically designate an Agent (or Agents) to be responsible for developing +LJ048-0076|and maintaining liaison with other Federal Agencies. This liaison should take into consideration|and maintaining liaison with other Federal Agencies. This liaison should take into consideration +LJ048-0077|FBI-agency community of interests, location of agency head quarters, and the responsiveness of agency representatives.|FBI-agency community of interests, location of agency head quarters, and the responsiveness of agency representatives. +LJ048-0078|In each instance, liaison contacts should be developed to include a close friendly relationship,|In each instance, liaison contacts should be developed to include a close friendly relationship, +LJ048-0079|mutual understanding of FBI and agency jurisdictions, and an indicated willingness by the agency representative|mutual understanding of FBI and agency jurisdictions, and an indicated willingness by the agency representative +LJ048-0080|to coordinate activities and to discuss problems of mutual interest.|to coordinate activities and to discuss problems of mutual interest. +LJ048-0081|Each field office should determine those Federal agencies which are represented locally and with which liaison should be conducted. End quote.|Each field office should determine those Federal agencies which are represented locally and with which liaison should be conducted. End quote. +LJ048-0082|The testimony reveals that liaison responsibilities in connection with the President's visit|The testimony reveals that liaison responsibilities in connection with the President's visit +LJ048-0083|were discussed twice officially by the special agent in charge of the FBI office in Dallas. As discussed in chapter 2,|were discussed twice officially by the special agent in charge of the FBI office in Dallas. As discussed in chapter two, +LJ048-0084|some limited information was made available to the Secret Service. But there was no fully adequate liaison between the two agencies. Indeed,|some limited information was made available to the Secret Service. But there was no fully adequate liaison between the two agencies. Indeed, +LJ048-0085|the Commission believes that the liaison between all Federal agencies responsible for Presidential protection should be improved.|the Commission believes that the liaison between all Federal agencies responsible for Presidential protection should be improved. +LJ048-0086|Other Protective Measures and Aspects of Secret Service Performance|Other Protective Measures and Aspects of Secret Service Performance +LJ048-0087|The President's trip to Dallas called into play many standard operating procedures of the Secret Service in addition to its preventive intelligence operations.|The President's trip to Dallas called into play many standard operating procedures of the Secret Service in addition to its preventive intelligence operations. +LJ048-0088|Examination of these procedures shows that in most respects they were well conceived and ably executed by the personnel of the Service.|Examination of these procedures shows that in most respects they were well conceived and ably executed by the personnel of the Service. +LJ048-0089|Against the background of the critical events of November 22, however,|Against the background of the critical events of November twenty-two, however, +LJ048-0090|certain shortcomings and lapses from the high standards which the Commission believes should prevail in the field of Presidential protection are evident.|certain shortcomings and lapses from the high standards which the Commission believes should prevail in the field of Presidential protection are evident. +LJ048-0091|Advance preparations.|Advance preparations. +LJ048-0092|The advance preparations in Dallas by Agent Winston G. Lawson of the White House detail have been described in chapter 2.|The advance preparations in Dallas by Agent Winston G. Lawson of the White House detail have been described in chapter two. +LJ048-0093|With the assistance of Agent in Charge Sorrels of the Dallas field office of the Secret Service,|With the assistance of Agent in Charge Sorrels of the Dallas field office of the Secret Service, +LJ048-0094|Lawson was responsible for working out a great many arrangements for the President's trip.|Lawson was responsible for working out a great many arrangements for the President's trip. +LJ048-0095|The Service prefers to have two agents perform advance preparations.|The Service prefers to have two agents perform advance preparations. +LJ048-0096|In the case of Dallas, because President Kennedy had scheduled visits to five Texas cities|In the case of Dallas, because President Kennedy had scheduled visits to five Texas cities +LJ048-0097|and had also scheduled visits to other parts of the country immediately before the Texas trip,|and had also scheduled visits to other parts of the country immediately before the Texas trip, +LJ048-0098|there were not enough men available to permit two agents to be assigned to all the advance work.|there were not enough men available to permit two agents to be assigned to all the advance work. +LJ048-0099|Agent Lawson did the advance work alone from November 13 to November 18, when he was joined by Agent David B. Grant,|Agent Lawson did the advance work alone from November thirteen to November eighteen, when he was joined by Agent David B. Grant, +LJ048-0100|who had just completed advance work on the President's trip to Tampa.|who had just completed advance work on the President's trip to Tampa. +LJ048-0101|The Commission concludes that the most significant advance arrangements for the President's trip were soundly planned.|The Commission concludes that the most significant advance arrangements for the President's trip were soundly planned. +LJ048-0102|In particular, the Commission believes that the motorcade route selected by Agent Lawson, upon the advice of Agent in Charge Sorrels|In particular, the Commission believes that the motorcade route selected by Agent Lawson, upon the advice of Agent in Charge Sorrels +LJ048-0103|and with the concurrence of the Dallas police, was entirely appropriate, in view of the known desires of the President.|and with the concurrence of the Dallas police, was entirely appropriate, in view of the known desires of the President. +LJ048-0105|but these routes would not have been in accordance with the White House staff instructions given the Secret Service for a desirable motorcade route.|but these routes would not have been in accordance with the White House staff instructions given the Secret Service for a desirable motorcade route. +LJ048-0106|Much of Lawson's time was taken with establishing adequate security over the motorcade route and at the two places where the President would stop,|Much of Lawson's time was taken with establishing adequate security over the motorcade route and at the two places where the President would stop, +LJ048-0107|Love Field and the Trade Mart.|Love Field and the Trade Mart. +LJ048-0109|by these Secret Service agents with the cooperation of the Dallas police and other local law enforcement agents, were carefully executed.|by these Secret Service agents with the cooperation of the Dallas police and other local law enforcement agents, were carefully executed. +LJ048-0110|Since the President was to be at the Trade Mart longer than at any other location in Dallas and in view of the security hazards presented by the building,|Since the President was to be at the Trade Mart longer than at any other location in Dallas and in view of the security hazards presented by the building, +LJ048-0111|the Secret Service correctly gave particular attention in the advance preparations to those arrangements.|the Secret Service correctly gave particular attention in the advance preparations to those arrangements. +LJ048-0112|The Commission also regards the security arrangements worked out by Lawson and Sorrels at Love Field as entirely adequate.|The Commission also regards the security arrangements worked out by Lawson and Sorrels at Love Field as entirely adequate. +LJ048-0113|The Commission believes, however,|The Commission believes, however, +LJ048-0114|that the Secret Service has inadequately defined the responsibilities of its advance agents, who have been given broad discretion|that the Secret Service has inadequately defined the responsibilities of its advance agents, who have been given broad discretion +LJ048-0115|to determine what matters require attention in making advance preparations and to decide what action to take.|to determine what matters require attention in making advance preparations and to decide what action to take. +LJ048-0116|Agent Lawson was not given written instructions concerning the Dallas trip or advice about any peculiar problems which it might involve;|Agent Lawson was not given written instructions concerning the Dallas trip or advice about any peculiar problems which it might involve; +LJ048-0117|all instructions from higher authority were communicated to him orally.|all instructions from higher authority were communicated to him orally. +LJ048-0118|He did not have a checklist of the tasks he was expected to accomplish, either by his own efforts or with the cooperation of local authorities.|He did not have a checklist of the tasks he was expected to accomplish, either by his own efforts or with the cooperation of local authorities. +LJ048-0119|The only systematic supervision of the activities of the advance agent|The only systematic supervision of the activities of the advance agent +LJ048-0120|has been that provided by a requirement that he file interim and final reports on each advance assignment.|has been that provided by a requirement that he file interim and final reports on each advance assignment. +LJ048-0121|The interim report must be in the hands of the agent supervising the protective group traveling with the President|The interim report must be in the hands of the agent supervising the protective group traveling with the President +LJ048-0122|long enough before his departure to apprise him of any particular problems encountered and the responsive action taken.|long enough before his departure to apprise him of any particular problems encountered and the responsive action taken. +LJ048-0123|Agent Lawson's interim report was received by Agent Kellerman on November 20, the day before departure on the Texas trip.|Agent Lawson's interim report was received by Agent Kellerman on November twenty, the day before departure on the Texas trip. +LJ048-0124|The Secret Service has advised the Commission that no unusual precautions were taken for the Dallas trip, and that, quote,|The Secret Service has advised the Commission that no unusual precautions were taken for the Dallas trip, and that, quote, +LJ048-0125|the precautions taken for the President's trip were the usual safeguards employed on trips of this kind in the United States during the previous year, end quote.|the precautions taken for the President's trip were the usual safeguards employed on trips of this kind in the United States during the previous year, end quote. +LJ048-0126|Special Agent in Charge Sorrels testified that the advance preparations followed on this occasion were, quote, pretty much the same, end quote,|Special Agent in Charge Sorrels testified that the advance preparations followed on this occasion were, quote, pretty much the same, end quote, +LJ048-0127|as those followed in 1936 during a trip to Dallas by President Roosevelt,|as those followed in nineteen thirty-six during a trip to Dallas by President Roosevelt, +LJ048-0128|which was Sorrels' first important assignment in connection with Presidential work.|which was Sorrels' first important assignment in connection with Presidential work. +LJ048-0129|In view of the constant change in the nature of threats to the President and the diversity of the dangers which may arise in the various cities within the United States,|In view of the constant change in the nature of threats to the President and the diversity of the dangers which may arise in the various cities within the United States, +LJ048-0130|the Commission believes that standard procedures in use for many years and applied in all parts of the country may not be sufficient.|the Commission believes that standard procedures in use for many years and applied in all parts of the country may not be sufficient. +LJ048-0131|There is, for example, no Secret Service arrangement for evaluating before a trip particular difficulties that might be anticipated,|There is, for example, no Secret Service arrangement for evaluating before a trip particular difficulties that might be anticipated, +LJ048-0132|which would bring to bear the judgment and experience of members of the White House detail other than the advance agent.|which would bring to bear the judgment and experience of members of the White House detail other than the advance agent. +LJ048-0133|Constant reevaluation of procedures, with attention to special problems and the development of instructions specific to particular trips|Constant reevaluation of procedures, with attention to special problems and the development of instructions specific to particular trips +LJ048-0134|would be a desirable innovation.|would be a desirable innovation. +LJ048-0135|Liaison with local law enforcement authorities.|Liaison with local law enforcement authorities. +LJ048-0136|In the description of the important aspects of the advance preparations,|In the description of the important aspects of the advance preparations, +LJ048-0137|there have been references to the numerous discussions between Secret Service representatives and the Dallas Police Department.|there have been references to the numerous discussions between Secret Service representatives and the Dallas Police Department. +LJ048-0138|The wholehearted support of these local authorities was indispensable to the Service in carrying out its duties.|The wholehearted support of these local authorities was indispensable to the Service in carrying out its duties. +LJ048-0139|The Service had 28 agents participating in the Dallas visit.|The Service had twenty-eight agents participating in the Dallas visit. +LJ048-0140|Agent Lawson's advance planning called for the deployment of almost 600 members of the Dallas Police Department,|Agent Lawson's advance planning called for the deployment of almost six hundred members of the Dallas Police Department, +LJ048-0141|Fire Department, County Sheriff's Department, and the Texas Department of Public Safety.|Fire Department, County Sheriff's Department, and the Texas Department of Public Safety. +LJ048-0142|Despite this dependence on local authorities, which would be substantially the same on a visit by the President to any large city,|Despite this dependence on local authorities, which would be substantially the same on a visit by the President to any large city, +LJ048-0143|the Secret Service did not at the time of the assassination have any established procedure governing its relationships with them.|the Secret Service did not at the time of the assassination have any established procedure governing its relationships with them. +LJ048-0144|It had no prepared checklist of matters to be covered with local police on such visits to metropolitan areas|It had no prepared checklist of matters to be covered with local police on such visits to metropolitan areas +LJ048-0145|and no written description of the role the local police were expected to perform.|and no written description of the role the local police were expected to perform. +LJ048-0146|Discussions with the Dallas authorities and requests made of them were entirely informal.|Discussions with the Dallas authorities and requests made of them were entirely informal. +LJ048-0147|The Commission believes|The Commission believes +LJ048-0148|that a more formal statement of assigned responsibilities, supplemented in each case to reflect the peculiar conditions of each Presidential trip|that a more formal statement of assigned responsibilities, supplemented in each case to reflect the peculiar conditions of each Presidential trip +LJ048-0149|is essential. This would help to eliminate varying interpretations of Secret Service instructions by different local law enforcement representatives. For example,|is essential. This would help to eliminate varying interpretations of Secret Service instructions by different local law enforcement representatives. For example, +LJ048-0150|while the Secret Service representatives in Dallas|while the Secret Service representatives in Dallas +LJ048-0151|asked the police to station guards at each overpass to keep, quote, unauthorized personnel, end quote, off, this term was not defined.|asked the police to station guards at each overpass to keep, quote, unauthorized personnel, end quote, off, this term was not defined. +LJ048-0152|At some overpasses all persons were excluded|At some overpasses all persons were excluded +LJ048-0153|while on the overpass overlooking the assassination scene railroad and yard terminal workmen were permitted to remain under police supervision,|while on the overpass overlooking the assassination scene railroad and yard terminal workmen were permitted to remain under police supervision, +LJ048-0154|as discussed in chapter 3.|as discussed in chapter three. +LJ048-0155|Assistant Chief Batchelor of the Dallas police noted the absence of any formal statement by the Secret Service of specific work assigned to the police|Assistant Chief Batchelor of the Dallas police noted the absence of any formal statement by the Secret Service of specific work assigned to the police +LJ048-0156|and suggested the desirability of such a statement.|and suggested the desirability of such a statement. +LJ048-0157|Agent Lawson agreed that such a procedure would assist him and other agents in fulfilling their responsibilities as advance agents.|Agent Lawson agreed that such a procedure would assist him and other agents in fulfilling their responsibilities as advance agents. +LJ048-0158|Check of buildings along route of motorcade.|Check of buildings along route of motorcade. +LJ048-0159|Agent Lawson did not arrange for a prior inspection of buildings along the motorcade route,|Agent Lawson did not arrange for a prior inspection of buildings along the motorcade route, +LJ048-0160|either by police or by custodians of the buildings, since it was not the usual practice of the Secret Service to do so.|either by police or by custodians of the buildings, since it was not the usual practice of the Secret Service to do so. +LJ048-0161|The Chief of the Service has provided the Commission a detailed explanation of this policy, quote,|The Chief of the Service has provided the Commission a detailed explanation of this policy, quote, +LJ048-0162|Except for inauguration or other parades involving foreign dignitaries accompanied by the President in Washington,|Except for inauguration or other parades involving foreign dignitaries accompanied by the President in Washington, +LJ048-0163|it has not been the practice of the Secret Service to make surveys or checks of buildings along the route of a Presidential motorcade.|it has not been the practice of the Secret Service to make surveys or checks of buildings along the route of a Presidential motorcade. +LJ048-0164|For the inauguration and certain other parades in Washington where the traditional route is known to the public long in advance of the event,|For the inauguration and certain other parades in Washington where the traditional route is known to the public long in advance of the event, +LJ048-0165|buildings along the route can be checked by teams of law enforcement officers, and armed guards are posted along the route as appropriate.|buildings along the route can be checked by teams of law enforcement officers, and armed guards are posted along the route as appropriate. +LJ048-0166|But on out-of-town trips where the route is decided on and made public only a few days in advance,|But on out-of-town trips where the route is decided on and made public only a few days in advance, +LJ048-0167|buildings are not checked either by Secret Service agents or by any other law enforcement officers at the request of the Secret Service.|buildings are not checked either by Secret Service agents or by any other law enforcement officers at the request of the Secret Service. +LJ048-0168|With the number of men available to the Secret Service and the time available, surveys of hundreds of buildings and thousands of windows is not practical.|With the number of men available to the Secret Service and the time available, surveys of hundreds of buildings and thousands of windows is not practical. +LJ048-0169|In Dallas the route selected necessarily involved passing through the principal downtown section between tall buildings.|In Dallas the route selected necessarily involved passing through the principal downtown section between tall buildings. +LJ048-0170|While certain streets thought to be too narrow could be avoided and other choices made,|While certain streets thought to be too narrow could be avoided and other choices made, +LJ048-0171|it was not practical to select a route where the President could not be seen from roofs or windows of buildings.|it was not practical to select a route where the President could not be seen from roofs or windows of buildings. +LJ048-0172|At the two places in Dallas where the President would remain for a period of time, Love Field and the Trade Mart,|At the two places in Dallas where the President would remain for a period of time, Love Field and the Trade Mart, +LJ048-0173|arrangements were made for building and roof security by posting police officers where appropriate.|arrangements were made for building and roof security by posting police officers where appropriate. +LJ048-0174|Similar arrangements for a motorcade of ten miles, including many blocks of tall commercial buildings is not practical.|Similar arrangements for a motorcade of ten miles, including many blocks of tall commercial buildings is not practical. +LJ048-0175|Nor is it practical to prevent people from entering such buildings, or to limit access in every building to those employed or having business there.|Nor is it practical to prevent people from entering such buildings, or to limit access in every building to those employed or having business there. +LJ048-0176|Even if it were possible with a vastly larger force of security officers to do so, many observers have felt|Even if it were possible with a vastly larger force of security officers to do so, many observers have felt +LJ048-0177|that such a procedure would not be consistent with the nature and purpose of the motorcade to let the people see their President and to welcome him to their city.|that such a procedure would not be consistent with the nature and purpose of the motorcade to let the people see their President and to welcome him to their city. +LJ048-0178|In accordance with its regular procedures, no survey or other check was made by the Secret Service, or by any other law enforcement agency at its request,|In accordance with its regular procedures, no survey or other check was made by the Secret Service, or by any other law enforcement agency at its request, +LJ048-0179|of the Texas School Book Depository Building or those employed there prior to the time the President was shot. End quote.|of the Texas School Book Depository Building or those employed there prior to the time the President was shot. End quote. +LJ048-0180|This justification of the Secret Service's standing policy is not persuasive.|This justification of the Secret Service's standing policy is not persuasive. +LJ048-0181|The danger from a concealed sniper on the Dallas trip was of concern to those who had considered the problem.|The danger from a concealed sniper on the Dallas trip was of concern to those who had considered the problem. +LJ048-0182|President Kennedy himself had mentioned it that morning, as had Agent Sorrels when he and Agent Lawson were fixing the motorcade route.|President Kennedy himself had mentioned it that morning, as had Agent Sorrels when he and Agent Lawson were fixing the motorcade route. +LJ048-0183|Admittedly, protective measures cannot ordinarily be taken with regard to all buildings along a motorcade route.|Admittedly, protective measures cannot ordinarily be taken with regard to all buildings along a motorcade route. +LJ048-0184|Levels of risk can be determined, however, as has been confirmed by building surveys made since the assassination for the Department of the Treasury.|Levels of risk can be determined, however, as has been confirmed by building surveys made since the assassination for the Department of the Treasury. +LJ048-0185|An attempt to cover only the most obvious points of possible ambush along the route in Dallas might well have included the Texas School Book Depository Building.|An attempt to cover only the most obvious points of possible ambush along the route in Dallas might well have included the Texas School Book Depository Building. +LJ048-0186|Instead of such advance precautions, the Secret Service depended in part on the efforts of local law enforcement personnel stationed along the route.|Instead of such advance precautions, the Secret Service depended in part on the efforts of local law enforcement personnel stationed along the route. +LJ048-0188|These substitute measures were of limited value. Agent Lawson was unable to state whether he had actually instructed|These substitute measures were of limited value. Agent Lawson was unable to state whether he had actually instructed +LJ048-0189|the Dallas police to scan windows of buildings lining the motorcade route, although it was his usual practice to do so.|the Dallas police to scan windows of buildings lining the motorcade route, although it was his usual practice to do so. +LJ048-0190|If such instructions were in fact given, they were not effectively carried out.|If such instructions were in fact given, they were not effectively carried out. +LJ048-0191|Television films taken of parts of the motorcade by a Dallas television station|Television films taken of parts of the motorcade by a Dallas television station +LJ048-0192|show the foot patrolmen facing the passing motorcade, and not the adjacent crowds and buildings, as the procession passed.|show the foot patrolmen facing the passing motorcade, and not the adjacent crowds and buildings, as the procession passed. +LJ048-0193|Three officers from the Dallas Police Department were assigned to the intersection of Elm and Houston|Three officers from the Dallas Police Department were assigned to the intersection of Elm and Houston +LJ048-0194|during the morning of November 22 prior to the motorcade.|during the morning of November twenty-two prior to the motorcade. +LJ048-0195|All received their instructions early in the morning from Capt. P. W. Lawrence of the traffic division.|All received their instructions early in the morning from Capt. P. W. Lawrence of the traffic division. +LJ048-0196|According to Captain Lawrence, quote,|According to Captain Lawrence, quote, +LJ048-0197|I then told the officers that their primary duty was traffic and crowd control and that they should be alert for any persons who might attempt to throw anything|I then told the officers that their primary duty was traffic and crowd control and that they should be alert for any persons who might attempt to throw anything +LJ048-0198|and although it was not a violation of the law to carry a placard, that they were not to tolerate any actions such as the Stevenson incident|and although it was not a violation of the law to carry a placard, that they were not to tolerate any actions such as the Stevenson incident +LJ048-0199|and arrest any person who might attempt to throw anything or try to get at the President and his party;|and arrest any person who might attempt to throw anything or try to get at the President and his party; +LJ048-0200|paying particular attention to the crowd for any unusual activity.|paying particular attention to the crowd for any unusual activity. +LJ048-0201|I stressed the fact that this was our President and he should be shown every respect due his position and that it was our duty to see that this was done.|I stressed the fact that this was our President and he should be shown every respect due his position and that it was our duty to see that this was done. +LJ048-0202|End quote. Captain Lawrence was not instructed to have his men watch buildings along the motorcade route and did not mention the observation of buildings to them.|End quote. Captain Lawrence was not instructed to have his men watch buildings along the motorcade route and did not mention the observation of buildings to them. +LJ048-0203|The three officers confirm that their primary concern was crowd and traffic control,|The three officers confirm that their primary concern was crowd and traffic control, +LJ048-0204|and that they had no opportunity to scan the windows of the Depository or any other building in the vicinity of Elm and Houston when the motorcade was passing.|and that they had no opportunity to scan the windows of the Depository or any other building in the vicinity of Elm and Houston when the motorcade was passing. +LJ048-0205|They had, however, occasionally observed the windows of buildings in the area before the motorcade arrived, in accordance with their own understanding of their function.|They had, however, occasionally observed the windows of buildings in the area before the motorcade arrived, in accordance with their own understanding of their function. +LJ048-0206|As the motorcade approached Elm Street|As the motorcade approached Elm Street +LJ048-0207|there were several Secret Service agents in it who shared the responsibility of scanning the windows of nearby buildings.|there were several Secret Service agents in it who shared the responsibility of scanning the windows of nearby buildings. +LJ048-0208|Agent Sorrels, riding in the lead car, did observe the Texas School Book Depository Building as he passed by,|Agent Sorrels, riding in the lead car, did observe the Texas School Book Depository Building as he passed by, +LJ048-0209|at least for a sufficient number of seconds to gain a, quote, general impression, end quote, of the lack of any unusual activity.|at least for a sufficient number of seconds to gain a, quote, general impression, end quote, of the lack of any unusual activity. +LJ048-0210|He was handicapped, however, by the fact that he was riding in a closed car whose roof at times obscured his view.|He was handicapped, however, by the fact that he was riding in a closed car whose roof at times obscured his view. +LJ048-0211|Lawson, also in the lead car, did not scan any buildings since an important part of his job was to look backward at the President's car.|Lawson, also in the lead car, did not scan any buildings since an important part of his job was to look backward at the President's car. +LJ048-0212|Lawson stated that he, quote, was looking back a good deal of the time,|Lawson stated that he, quote, was looking back a good deal of the time, +LJ048-0213|watching his car, watching the sides, watching the crowds, giving advice or asking advice from the Chief|watching his car, watching the sides, watching the crowds, giving advice or asking advice from the Chief +LJ048-0214|and also looking ahead to the known hazards like overpasses, under-passes, railroads, et cetera, end quote.|and also looking ahead to the known hazards like overpasses, under-passes, railroads, et cetera, end quote. +LJ048-0215|Agent Roy H. Kellerman, riding in the front seat of the Presidential car,|Agent Roy H. Kellerman, riding in the front seat of the Presidential car, +LJ048-0216|stated that he scanned the Depository Building, but not sufficiently to be alerted by anything in the windows or on the roof.|stated that he scanned the Depository Building, but not sufficiently to be alerted by anything in the windows or on the roof. +LJ048-0217|The agents in the follow-up car also were expected to scan adjacent buildings.|The agents in the follow-up car also were expected to scan adjacent buildings. +LJ048-0218|However, the Commission does not believe that agents stationed in a car behind the Presidential car,|However, the Commission does not believe that agents stationed in a car behind the Presidential car, +LJ048-0219|who must concentrate primarily on the possibility of threats from crowds along the route, provide a significant safeguard against dangers in nearby buildings.|who must concentrate primarily on the possibility of threats from crowds along the route, provide a significant safeguard against dangers in nearby buildings. +LJ048-0220|Conduct of Secret Service agents in Fort Worth on November 22.|Conduct of Secret Service agents in Fort Worth on November twenty-two. +LJ048-0221|In the early morning hours on November 22, 1963,|In the early morning hours on November twenty-two, nineteen sixty-three, +LJ048-0222|in Fort Worth, there occurred a breach of discipline by some members of the Secret Service who were officially traveling with the President.|in Fort Worth, there occurred a breach of discipline by some members of the Secret Service who were officially traveling with the President. +LJ048-0223|After the President had retired at his hotel,|After the President had retired at his hotel, +LJ048-0224|nine agents who were off duty went to the nearby Fort Worth Press Club at midnight or slightly thereafter, expecting to obtain food;|nine agents who were off duty went to the nearby Fort Worth Press Club at midnight or slightly thereafter, expecting to obtain food; +LJ048-0225|they had little opportunity to eat during the day. No food was available at the Press Club.|they had little opportunity to eat during the day. No food was available at the Press Club. +LJ048-0226|All of the agents stayed for a drink of beer, or in several cases, a mixed drink.|All of the agents stayed for a drink of beer, or in several cases, a mixed drink. +LJ048-0227|According to their affidavits, the drinking in no case amounted to more than three glasses of beer or one and a half mixed drinks,|According to their affidavits, the drinking in no case amounted to more than three glasses of beer or one and a half mixed drinks, +LJ048-0228|and others who were present say that no agent was inebriated or acted improperly.|and others who were present say that no agent was inebriated or acted improperly. +LJ048-0229|The statements of the agents involved are supported by statements of members of the Fort Worth press who accompanied or observed them|The statements of the agents involved are supported by statements of members of the Fort Worth press who accompanied or observed them +LJ048-0230|and by a Secret Service investigation.|and by a Secret Service investigation. +LJ048-0231|According to their statements, the agents remained at the Press Club for periods varying from 30 minutes to an hour and a half,|According to their statements, the agents remained at the Press Club for periods varying from thirty minutes to an hour and a half, +LJ048-0232|and the last agent left the Press Club by 2 a.m.|and the last agent left the Press Club by two a.m. +LJ048-0233|Two of the nine agents returned to their rooms. The seven others proceeded to an establishment called the Cellar Coffee House,|Two of the nine agents returned to their rooms. The seven others proceeded to an establishment called the Cellar Coffee House, +LJ048-0234|described by some as a beatnik place and by its manager as, quote, a unique showplace with continuous light entertainment all night|described by some as a beatnik place and by its manager as, quote, a unique showplace with continuous light entertainment all night +LJ048-0235|serving only coffee, fruit juices and no hard liquors or beer, end quote.|serving only coffee, fruit juices and no hard liquors or beer, end quote. +LJ048-0236|There is no indication that any of the agents who visited the Cellar Coffee House had any intoxicating drink at that establishment.|There is no indication that any of the agents who visited the Cellar Coffee House had any intoxicating drink at that establishment. +LJ048-0237|Most of the agents were there from about 1:30 or 1:45 a.m. to about 2:45 or 3 a.m.;|Most of the agents were there from about one:thirty or one:forty-five a.m. to about two:forty-five or three a.m.; +LJ048-0238|one agent was there from 2 until 5 a.m.|one agent was there from two until five a.m. +LJ048-0239|The lobby of the hotel and the areas adjacent to the quarters of the President were guarded during the night|The lobby of the hotel and the areas adjacent to the quarters of the President were guarded during the night +LJ048-0240|by members of the midnight to 8 a.m. shift of the White House detail.|by members of the midnight to eight a.m. shift of the White House detail. +LJ048-0241|These agents were each relieved for a half hour break during the night.|These agents were each relieved for a half hour break during the night. +LJ048-0242|Three members of this shift separately took this opportunity to visit the Cellar Coffee House.|Three members of this shift separately took this opportunity to visit the Cellar Coffee House. +LJ048-0243|Only one stayed as long as a half hour, and none had any beverage there.|Only one stayed as long as a half hour, and none had any beverage there. +LJ048-0244|Chief Rowley testified that agents on duty in such a situation usually stay within the building during their relief,|Chief Rowley testified that agents on duty in such a situation usually stay within the building during their relief, +LJ048-0245|but that their visits to the Cellar were, quote, neither consistent nor inconsistent, end quote, with their duty.|but that their visits to the Cellar were, quote, neither consistent nor inconsistent, end quote, with their duty. +LJ048-0246|Each of the agents who visited the Press Club or the Cellar Coffee House (apart from the three members of the midnight shift)|Each of the agents who visited the Press Club or the Cellar Coffee House (apart from the three members of the midnight shift) +LJ048-0247|had duty assignments beginning no later than 8 a.m. that morning.|had duty assignments beginning no later than eight a.m. that morning. +LJ048-0248|President Kennedy was scheduled to speak across the street from his hotel in Fort Worth at 8:30 a.m.|President Kennedy was scheduled to speak across the street from his hotel in Fort Worth at eight:thirty a.m. +LJ048-0249|and then at a breakfast, after which the entourage would proceed to Dallas.|and then at a breakfast, after which the entourage would proceed to Dallas. +LJ048-0250|In Dallas, one of the nine agents was assigned to assist in security measures at Love Field, and four had protective assignments at the Trade Mart.|In Dallas, one of the nine agents was assigned to assist in security measures at Love Field, and four had protective assignments at the Trade Mart. +LJ048-0251|The remaining four had key responsibilities as members of the complement of the follow-up car in the motorcade.|The remaining four had key responsibilities as members of the complement of the follow-up car in the motorcade. +LJ048-0252|Three of these agents occupied positions on the running boards of the car, and the fourth was seated in the car.|Three of these agents occupied positions on the running boards of the car, and the fourth was seated in the car. +LJ048-0253|The supervisor of each of the off-duty agents who visited the Press Club or the Cellar Coffee House|The supervisor of each of the off-duty agents who visited the Press Club or the Cellar Coffee House +LJ048-0254|advised, in the course of the Secret Service investigation of these events, that each agent reported for duty on time,|advised, in the course of the Secret Service investigation of these events, that each agent reported for duty on time, +LJ048-0255|with full possession of his mental and physical capabilities and entirely ready for the performance of his assigned duties. Chief Rowley testified that,|with full possession of his mental and physical capabilities and entirely ready for the performance of his assigned duties. Chief Rowley testified that, +LJ048-0256|as a result of the investigation he ordered, he was satisfied that each of the agents performed his duties in an entirely satisfactory manner|as a result of the investigation he ordered, he was satisfied that each of the agents performed his duties in an entirely satisfactory manner +LJ048-0257|and that their conduct the night before did not impede their actions on duty|and that their conduct the night before did not impede their actions on duty +LJ048-0258|or in the slightest way prevent them from taking any action that might have averted the tragedy.|or in the slightest way prevent them from taking any action that might have averted the tragedy. +LJ048-0259|However, Chief Rowley did not condone the action of the off-duty agents, particularly since it violated a regulation of the Secret Service,|However, Chief Rowley did not condone the action of the off-duty agents, particularly since it violated a regulation of the Secret Service, +LJ048-0260|which provides, quote, Liquor, use of|which provides, quote, Liquor, use of +LJ048-0261|a. Employees are strictly enjoined to refrain from the use of intoxicating liquor|a. Employees are strictly enjoined to refrain from the use of intoxicating liquor +LJ048-0262|during the hours they are officially employed at their post of duty, or when they may reasonably expect that they may be called upon to perform an official duty.|during the hours they are officially employed at their post of duty, or when they may reasonably expect that they may be called upon to perform an official duty. +LJ048-0263|During entire periods of travel status,|During entire periods of travel status, +LJ048-0264|the special agent is officially employed and should not use liquor, until the completion of all of his official duties for the day,|the special agent is officially employed and should not use liquor, until the completion of all of his official duties for the day, +LJ048-0265|after which time a very moderate use of liquor will not be considered a violation. However, all members of the White House Detail|after which time a very moderate use of liquor will not be considered a violation. However, all members of the White House Detail +LJ048-0266|and special agents cooperating with them on Presidential and similar protective assignments are considered to be subject to call for official duty|and special agents cooperating with them on Presidential and similar protective assignments are considered to be subject to call for official duty +LJ048-0267|at any time while in travel status.|at any time while in travel status. +LJ048-0268|Therefore, the use of intoxicating liquor of any kind, including beer and wine, by members of the White House Detail|Therefore, the use of intoxicating liquor of any kind, including beer and wine, by members of the White House Detail +LJ048-0269|and special agents cooperating with them, or by special agents on similar assignments, while they are in a travel status, is prohibited. End quote.|and special agents cooperating with them, or by special agents on similar assignments, while they are in a travel status, is prohibited. End quote. +LJ048-0270|The regulations provide further that, quote, violation or slight disregard, end quote, of these provisions, quote,|The regulations provide further that, quote, violation or slight disregard, end quote, of these provisions, quote, +LJ048-0272|Chief Rowley testified|Chief Rowley testified +LJ048-0273|that under ordinary circumstances he would have taken disciplinary action against those agents who had been drinking in clear violation of the regulation.|that under ordinary circumstances he would have taken disciplinary action against those agents who had been drinking in clear violation of the regulation. +LJ048-0274|However, he felt that any disciplinary action might have given rise|However, he felt that any disciplinary action might have given rise +LJ048-0275|to an inference that the violation of the regulation had contributed to the tragic events of November 22.|to an inference that the violation of the regulation had contributed to the tragic events of November twenty-two. +LJ048-0276|Since he was convinced that this was not the case, he believed that it would be unfair to the agents and their families to take explicit disciplinary measures.|Since he was convinced that this was not the case, he believed that it would be unfair to the agents and their families to take explicit disciplinary measures. +LJ048-0277|He felt that each agent recognized the seriousness of the infraction and that there was no danger of a repetition.|He felt that each agent recognized the seriousness of the infraction and that there was no danger of a repetition. +LJ048-0278|The Commission recognizes that the responsibilities of members of the White House detail of the Secret Service are arduous.|The Commission recognizes that the responsibilities of members of the White House detail of the Secret Service are arduous. +LJ048-0279|They work long, hard hours, under very great strain, and must travel frequently.|They work long, hard hours, under very great strain, and must travel frequently. +LJ048-0280|It might seem harsh to circumscribe their opportunities for relaxation.|It might seem harsh to circumscribe their opportunities for relaxation. +LJ048-0281|Yet their role of protecting the President is so important to the well-being of the country|Yet their role of protecting the President is so important to the well-being of the country +LJ048-0282|that it is reasonable to expect them to meet very high standards of personal conduct,|that it is reasonable to expect them to meet very high standards of personal conduct, +LJ048-0283|so that nothing can interfere with their bringing to their task the finest qualities and maximum resources of mind and body.|so that nothing can interfere with their bringing to their task the finest qualities and maximum resources of mind and body. +LJ048-0284|This is the salutary goal to which the Secret Service regulation is directed,|This is the salutary goal to which the Secret Service regulation is directed, +LJ048-0285|when it absolutely forbids drinking by any agent accompanying the President on a trip.|when it absolutely forbids drinking by any agent accompanying the President on a trip. +LJ048-0286|Nor is this goal served when agents remain out until early morning hours, and lose the opportunity to get a reasonable amount of sleep.|Nor is this goal served when agents remain out until early morning hours, and lose the opportunity to get a reasonable amount of sleep. +LJ048-0287|It is conceivable that those men who had little sleep, and who had consumed alcoholic beverages, even in limited quantities,|It is conceivable that those men who had little sleep, and who had consumed alcoholic beverages, even in limited quantities, +LJ048-0288|might have been more alert in the Dallas motorcade if they had retired promptly in Fort Worth.|might have been more alert in the Dallas motorcade if they had retired promptly in Fort Worth. +LJ048-0289|However, there is no evidence that these men failed to take any action in Dallas within their power that would have averted the tragedy.|However, there is no evidence that these men failed to take any action in Dallas within their power that would have averted the tragedy. +LJ049-0001|Report of the President's Commission on the Assassination of President Kennedy.|Report of the President's Commission on the Assassination of President Kennedy. +LJ049-0002|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy.|The Warren Commission Report. By The President's Commission on the Assassination of President Kennedy. +LJ049-0003|Chapter 8. The Protection of the President. Part 4.|Chapter eight. The Protection of the President. Part four. +LJ049-0004|The motorcade in Dallas.|The motorcade in Dallas. +LJ049-0005|Rigorous security precautions had been arranged at Love Field with the local law enforcement authorities by Agents Sorrels and Lawson.|Rigorous security precautions had been arranged at Love Field with the local law enforcement authorities by Agents Sorrels and Lawson. +LJ049-0006|These precautions included reserving a ceremonial area for the Presidential party,|These precautions included reserving a ceremonial area for the Presidential party, +LJ049-0007|stationing police on the rooftops of all buildings overlooking the reception area,|stationing police on the rooftops of all buildings overlooking the reception area, +LJ049-0008|and detailing police in civilian clothes to be scattered throughout the sizable crowd.|and detailing police in civilian clothes to be scattered throughout the sizable crowd. +LJ049-0009|When President and Mrs. Kennedy shook hands with members of the public along the fences surrounding the reception area, they were closely guarded by Secret Service agents|When President and Mrs. Kennedy shook hands with members of the public along the fences surrounding the reception area, they were closely guarded by Secret Service agents +LJ049-0010|who responded to the unplanned event with dispatch.|who responded to the unplanned event with dispatch. +LJ049-0011|As described in chapter 2, the President directed that his car stop on two occasions during the motorcade so that he could greet members of the public.|As described in chapter two, the President directed that his car stop on two occasions during the motorcade so that he could greet members of the public. +LJ049-0012|At these stops, agents from the Presidential follow-up car stood between the President and the public,|At these stops, agents from the Presidential follow-up car stood between the President and the public, +LJ049-0013|and on one occasion Agent Kellerman left the front seat of the President's car to take a similar position.|and on one occasion Agent Kellerman left the front seat of the President's car to take a similar position. +LJ049-0014|The Commission regards such impromptu stops as presenting an unnecessary danger,|The Commission regards such impromptu stops as presenting an unnecessary danger, +LJ049-0015|but finds that the Secret Service agents did all that could have been done to take protective measures.|but finds that the Secret Service agents did all that could have been done to take protective measures. +LJ049-0016|The Presidential limousine.|The Presidential limousine. +LJ049-0017|The limousine used by President Kennedy in Dallas was a convertible with a detachable, rigid plastic "bubble" top|The limousine used by President Kennedy in Dallas was a convertible with a detachable, rigid plastic "bubble" top +LJ049-0018|which was neither bulletproof nor bullet resistant.|which was neither bulletproof nor bullet resistant. +LJ049-0019|The last Presidential vehicle with any protection against small-arms fire left the White House in 1953.|The last Presidential vehicle with any protection against small-arms fire left the White House in nineteen fifty-three. +LJ049-0020|It was not then replaced because the state of the art did not permit the development of a bulletproof top of sufficiently light weight|It was not then replaced because the state of the art did not permit the development of a bulletproof top of sufficiently light weight +LJ049-0021|to permit its removal on those occasions when the President wished to ride in an open car.|to permit its removal on those occasions when the President wished to ride in an open car. +LJ049-0022|The Secret Service believed that it was very doubtful that any President would ride regularly in a vehicle with a fixed top, even though transparent.|The Secret Service believed that it was very doubtful that any President would ride regularly in a vehicle with a fixed top, even though transparent. +LJ049-0023|Since the assassination, the Secret Service, with the assistance of other Federal agencies and of private industry,|Since the assassination, the Secret Service, with the assistance of other Federal agencies and of private industry, +LJ049-0024|has developed a vehicle for the better protection of the President.|has developed a vehicle for the better protection of the President. +LJ049-0025|Access to passenger compartment of Presidential car.|Access to passenger compartment of Presidential car. +LJ049-0026|On occasion the Secret Service has been permitted to have an agent riding in the passenger compartment with the President.|On occasion the Secret Service has been permitted to have an agent riding in the passenger compartment with the President. +LJ049-0027|Presidents have made it clear, however, that they did not favor this or any other arrangement which interferes with the privacy of the President and his guests.|Presidents have made it clear, however, that they did not favor this or any other arrangement which interferes with the privacy of the President and his guests. +LJ049-0028|The Secret Service has therefore suggested this practice only on extraordinary occasions.|The Secret Service has therefore suggested this practice only on extraordinary occasions. +LJ049-0029|Without attempting to prescribe or recommend specific measures which should be employed for the future protection of Presidents,|Without attempting to prescribe or recommend specific measures which should be employed for the future protection of Presidents, +LJ049-0030|the Commission does believe that there are aspects of the protective measures employed in the motorcade at Dallas which deserve special comment.|the Commission does believe that there are aspects of the protective measures employed in the motorcade at Dallas which deserve special comment. +LJ049-0032|had no special design or equipment which would have permitted the Secret Service agent riding in the driver's compartment|had no special design or equipment which would have permitted the Secret Service agent riding in the driver's compartment +LJ049-0033|to move into the passenger section without hindrance or delay. Had the vehicle been so designed it is possible that an agent riding in the front seat|to move into the passenger section without hindrance or delay. Had the vehicle been so designed it is possible that an agent riding in the front seat +LJ049-0034|could have reached the President in time to protect him from the second and fatal shot to hit the President.|could have reached the President in time to protect him from the second and fatal shot to hit the President. +LJ049-0035|However, such access to the President was interfered with both by the metal bar some 15 inches above the back of the front seat|However, such access to the President was interfered with both by the metal bar some fifteen inches above the back of the front seat +LJ049-0036|and by the passengers in the jump seats.|and by the passengers in the jump seats. +LJ049-0037|In contrast, the Vice Presidential vehicle, although not specially designed for that purpose,|In contrast, the Vice Presidential vehicle, although not specially designed for that purpose, +LJ049-0038|had no passenger in a jump seat between Agent Youngblood and Vice President Johnson to interfere with Agent Youngblood's ability|had no passenger in a jump seat between Agent Youngblood and Vice President Johnson to interfere with Agent Youngblood's ability +LJ049-0039|to take a protective position in the passenger compartment before the third shot was fired.|to take a protective position in the passenger compartment before the third shot was fired. +LJ049-0040|The assassination suggests that it would have been of prime importance|The assassination suggests that it would have been of prime importance +LJ049-0041|in the protection of the President if the Presidential car permitted immediate access to the President by a Secret Service agent at the first sign of danger.|in the protection of the President if the Presidential car permitted immediate access to the President by a Secret Service agent at the first sign of danger. +LJ049-0042|At that time the agents on the framing boards of the follow-up car were expected to perform such a function.|At that time the agents on the framing boards of the follow-up car were expected to perform such a function. +LJ049-0043|However, these agents could not reach the President's car when it was traveling at an appreciable rate of speed.|However, these agents could not reach the President's car when it was traveling at an appreciable rate of speed. +LJ049-0044|Even if the car is traveling more slowly, the delay involved in reaching the President may be crucial.|Even if the car is traveling more slowly, the delay involved in reaching the President may be crucial. +LJ049-0045|It is clear that at the time of the shots in Dallas, Agent Clinton J. Hill leaped to the President's rescue as quickly as humanly possible.|It is clear that at the time of the shots in Dallas, Agent Clinton J. Hill leaped to the President's rescue as quickly as humanly possible. +LJ049-0046|Even so, analysis of the motion picture films taken by amateur photographer Zapruder|Even so, analysis of the motion picture films taken by amateur photographer Zapruder +LJ049-0047|reveals that Hill first placed his hand on the Presidential car at frame 343, 30 frames|reveals that Hill first placed his hand on the Presidential car at frame three forty-three, thirty frames +LJ049-0048|and therefore approximately 1.6 seconds after the President was shot in the head.|and therefore approximately one point six seconds after the President was shot in the head. +LJ049-0049|About 3.7 seconds after the President received this wound,|About three point seven seconds after the President received this wound, +LJ049-0050|Hill had both feet on the car and was climbing aboard to assist President and Mrs. Kennedy.|Hill had both feet on the car and was climbing aboard to assist President and Mrs. Kennedy. +LJ049-0051|Planning for motorcade contingencies.|Planning for motorcade contingencies. +LJ049-0052|In response to inquiry by the Commission regarding the instructions to agents in a motorcade|In response to inquiry by the Commission regarding the instructions to agents in a motorcade +LJ049-0053|of emergency procedures to be taken in a contingency such as that which actually occurred, the Secret Service responded, quote,|of emergency procedures to be taken in a contingency such as that which actually occurred, the Secret Service responded, quote, +LJ049-0054|The Secret Service has consistently followed two general principles in emergencies involving the President.|The Secret Service has consistently followed two general principles in emergencies involving the President. +LJ049-0055|All agents are so instructed.|All agents are so instructed. +LJ049-0056|The first duty of the agents in the motorcade is to attempt to cover the President as closely as possible and practicable|The first duty of the agents in the motorcade is to attempt to cover the President as closely as possible and practicable +LJ049-0057|and to shield him by attempting to place themselves between the President and any source of danger.|and to shield him by attempting to place themselves between the President and any source of danger. +LJ049-0058|Secondly, agents are instructed to remove the President as quickly as possible from known or impending danger.|Secondly, agents are instructed to remove the President as quickly as possible from known or impending danger. +LJ049-0060|but to consider any untoward circumstances as serious and to afford the President maximum protection at all times.|but to consider any untoward circumstances as serious and to afford the President maximum protection at all times. +LJ049-0061|No responsibility rests upon those agents near the President for the identification or arrest of any assassin or an attacker.|No responsibility rests upon those agents near the President for the identification or arrest of any assassin or an attacker. +LJ049-0062|Their primary responsibility is to stay with and protect the President.|Their primary responsibility is to stay with and protect the President. +LJ049-0063|Beyond these two principles the Secret Service believes a detailed contingency or emergency plan is not feasible|Beyond these two principles the Secret Service believes a detailed contingency or emergency plan is not feasible +LJ049-0064|because the variations possible preclude effective planning.|because the variations possible preclude effective planning. +LJ049-0065|A number of steps are taken, however, to permit appropriate steps to be taken in an emergency.|A number of steps are taken, however, to permit appropriate steps to be taken in an emergency. +LJ049-0066|For instance, the lead car always is manned by Secret Service agents familiar with the area and with local law enforcement officials;|For instance, the lead car always is manned by Secret Service agents familiar with the area and with local law enforcement officials; +LJ049-0067|the radio net in use in motorcades is elaborate and permits a number of different means of communication with various local points.|the radio net in use in motorcades is elaborate and permits a number of different means of communication with various local points. +LJ049-0068|A doctor is in the motorcade.|A doctor is in the motorcade. +LJ049-0069|This basic approach to the problem of planning for emergencies is sound.|This basic approach to the problem of planning for emergencies is sound. +LJ049-0070|Any effort to prepare detailed contingency plans might well have the undesirable effect of inhibiting quick and imaginative responses.|Any effort to prepare detailed contingency plans might well have the undesirable effect of inhibiting quick and imaginative responses. +LJ049-0071|If the advance preparation is thorough, and the protective devices and techniques employed are sound,|If the advance preparation is thorough, and the protective devices and techniques employed are sound, +LJ049-0072|those in command should be able to direct the response appropriate to the emergency. The Commission finds that the Secret Service agents in the motorcade|those in command should be able to direct the response appropriate to the emergency. The Commission finds that the Secret Service agents in the motorcade +LJ049-0073|who were immediately responsible for the President's safety reacted promptly at the time the shots were fired.|who were immediately responsible for the President's safety reacted promptly at the time the shots were fired. +LJ049-0074|Their actions demonstrate that the President and the Nation can expect courage and devotion to duty from the agents of the Secret Service.|Their actions demonstrate that the President and the Nation can expect courage and devotion to duty from the agents of the Secret Service. +LJ049-0075|Recommendations.|Recommendations. +LJ049-0076|The Commission's review of the provisions for Presidential protection at the time of President Kennedy's trip to Dallas demonstrates the need for substantial improvements.|The Commission's review of the provisions for Presidential protection at the time of President Kennedy's trip to Dallas demonstrates the need for substantial improvements. +LJ049-0077|Since the assassination, the Secret Service and the Department of the Treasury|Since the assassination, the Secret Service and the Department of the Treasury +LJ049-0078|have properly taken the initiative in reexamining major aspects of Presidential protection.|have properly taken the initiative in reexamining major aspects of Presidential protection. +LJ049-0079|Many changes have already been made and others are contemplated, some of them in response to the Commission's questions and informal suggestions.|Many changes have already been made and others are contemplated, some of them in response to the Commission's questions and informal suggestions. +LJ049-0080|Assassination a Federal Crime|Assassination a Federal Crime +LJ049-0081|There was no Federal criminal jurisdiction over the assassination of President Kennedy.|There was no Federal criminal jurisdiction over the assassination of President Kennedy. +LJ049-0082|Had there been reason to believe that the assassination was the result of a conspiracy, Federal jurisdiction could have been asserted;|Had there been reason to believe that the assassination was the result of a conspiracy, Federal jurisdiction could have been asserted; +LJ049-0083|it has long been a Federal crime to conspire to injure any Federal officer, on account of, or while he is engaged in, the lawful discharge of the duties of his office.|it has long been a Federal crime to conspire to injure any Federal officer, on account of, or while he is engaged in, the lawful discharge of the duties of his office. +LJ049-0084|Murder of the President has never been covered by Federal law, however, so that once it became reasonably clear that the killing was the act of a single person,|Murder of the President has never been covered by Federal law, however, so that once it became reasonably clear that the killing was the act of a single person, +LJ049-0085|the State of Texas had exclusive jurisdiction.|the State of Texas had exclusive jurisdiction. +LJ049-0086|It is anomalous that Congress has legislated in other ways touching upon the safety of the Chief Executive or other Federal officers,|It is anomalous that Congress has legislated in other ways touching upon the safety of the Chief Executive or other Federal officers, +LJ049-0087|without making an attack on the President a crime. Threatening harm to the President is a Federal offense,|without making an attack on the President a crime. Threatening harm to the President is a Federal offense, +LJ049-0088|as is advocacy of the overthrow of the Government by the assassination of any of its officers.|as is advocacy of the overthrow of the Government by the assassination of any of its officers. +LJ049-0089|The murder of Federal judges, U.S. attorneys and marshals, and a number of other specifically designated|The murder of Federal judges, U.S. attorneys and marshals, and a number of other specifically designated +LJ049-0090|Federal law enforcement officers is a Federal crime.|Federal law enforcement officers is a Federal crime. +LJ049-0091|Equally anomalous are statutory provisions which specifically authorize the Secret Service to protect the President,|Equally anomalous are statutory provisions which specifically authorize the Secret Service to protect the President, +LJ049-0092|without authorizing it to arrest anyone who harms him. The same provisions authorize the Service to arrest without warrant|without authorizing it to arrest anyone who harms him. The same provisions authorize the Service to arrest without warrant +LJ049-0093|persons committing certain offenses, including counterfeiting and certain frauds involving Federal checks or securities.|persons committing certain offenses, including counterfeiting and certain frauds involving Federal checks or securities. +LJ049-0094|The Commission agrees with the Secret Service that it should be authorized to make arrests without warrant|The Commission agrees with the Secret Service that it should be authorized to make arrests without warrant +LJ049-0095|for all offenses within its jurisdiction, as are FBI agents and Federal marshals.|for all offenses within its jurisdiction, as are FBI agents and Federal marshals. +LJ049-0096|There have been a number of efforts to make assassination a Federal crime, particularly after the assassination of President McKinley|There have been a number of efforts to make assassination a Federal crime, particularly after the assassination of President McKinley +LJ049-0097|and the attempt on the life of President-elect Franklin D. Roosevelt.|and the attempt on the life of President-elect Franklin D. Roosevelt. +LJ049-0098|In 1902 bills passed both Houses of Congress but failed of enactment when the Senate refused to accept the conference report.|In nineteen oh two bills passed both Houses of Congress but failed of enactment when the Senate refused to accept the conference report. +LJ049-0099|A number of bills were introduced immediately following the assassination of President Kennedy.|A number of bills were introduced immediately following the assassination of President Kennedy. +LJ049-0100|The Commission recommends to the Congress that it adopt legislation which would:|The Commission recommends to the Congress that it adopt legislation which would: +LJ049-0101|Punish the murder or manslaughter of, attempt or conspiracy to murder, kidnaping of and assault upon|Punish the murder or manslaughter of, attempt or conspiracy to murder, kidnaping of and assault upon +LJ049-0102|the President, Vice President, or other officer next in the order of succession to the Office of President, the President-elect and the Vice-President-elect,|the President, Vice President, or other officer next in the order of succession to the Office of President, the President-elect and the Vice-President-elect, +LJ049-0103|whether or not the act is committed while the victim is in the performance of his official duties or on account of such performance.|whether or not the act is committed while the victim is in the performance of his official duties or on account of such performance. +LJ049-0104|Such a statute would cover the President and Vice President or, in the absence of a Vice President, the person next in order of succession.|Such a statute would cover the President and Vice President or, in the absence of a Vice President, the person next in order of succession. +LJ049-0105|During the period between election and inauguration, the President-elect and Vice-President-elect would also be covered.|During the period between election and inauguration, the President-elect and Vice-President-elect would also be covered. +LJ049-0106|Restricting the coverage in this way would avoid unnecessary controversy over the inclusion or exclusion of other officials who are in the order of succession|Restricting the coverage in this way would avoid unnecessary controversy over the inclusion or exclusion of other officials who are in the order of succession +LJ049-0107|or who hold important governmental posts.|or who hold important governmental posts. +LJ049-0108|In addition, the restriction would probably eliminate a need for the requirement which has been urged as necessary for the exercise of Federal power,|In addition, the restriction would probably eliminate a need for the requirement which has been urged as necessary for the exercise of Federal power, +LJ049-0109|that the hostile act occur while the victim is engaged in or because of the performance of official duties.|that the hostile act occur while the victim is engaged in or because of the performance of official duties. +LJ049-0110|The governmental consequences of assassination of one of the specified officials give the United States ample power to act for its own protection.|The governmental consequences of assassination of one of the specified officials give the United States ample power to act for its own protection. +LJ049-0111|The activities of the victim at the time an assassination occurs and the motive for the assassination|The activities of the victim at the time an assassination occurs and the motive for the assassination +LJ049-0112|bear no relationship to the injury to the United States which follows from the act.|bear no relationship to the injury to the United States which follows from the act. +LJ049-0113|This point was ably made in the 1902 debate by Senator George F. Hoar, the sponsor of the Senate bill, quote,|This point was ably made in the nineteen oh two debate by Senator George F. Hoar, the sponsor of the Senate bill, quote, +LJ049-0114|what this bill means to punish is the crime of interruption of the Government of the United States and the destruction of its security by striking down the life|what this bill means to punish is the crime of interruption of the Government of the United States and the destruction of its security by striking down the life +LJ049-0115|of the person who is actually in the exercise of the executive power, or|of the person who is actually in the exercise of the executive power, or +LJ049-0116|of such persons as have been constitutionally and lawfully provided to succeed thereto in case of a vacancy. It is important to this country|of such persons as have been constitutionally and lawfully provided to succeed thereto in case of a vacancy. It is important to this country +LJ049-0117|that the interruption shall not take place for an hour, end quote.|that the interruption shall not take place for an hour, end quote. +LJ049-0118|Enactment of this statute would mean that the investigation of any of the acts covered and of the possibility of a further attempt|Enactment of this statute would mean that the investigation of any of the acts covered and of the possibility of a further attempt +LJ049-0119|would be conducted by Federal law enforcement officials, in particular, the FBI with the assistance of the Secret Service.|would be conducted by Federal law enforcement officials, in particular, the FBI with the assistance of the Secret Service. +LJ049-0120|At present, Federal agencies participate only upon the sufferance of the local authorities.|At present, Federal agencies participate only upon the sufferance of the local authorities. +LJ049-0121|While the police work of the Dallas authorities in the early identification and apprehension of Oswald was both efficient and prompt,|While the police work of the Dallas authorities in the early identification and apprehension of Oswald was both efficient and prompt, +LJ049-0122|FBI Director J. Edgar Hoover, who strongly supports such legislation, testified that the absence of clear Federal jurisdiction|FBI Director J. Edgar Hoover, who strongly supports such legislation, testified that the absence of clear Federal jurisdiction +LJ049-0123|over the assassination of President Kennedy led to embarrassment and confusion in the subsequent investigation by Federal and local authorities.|over the assassination of President Kennedy led to embarrassment and confusion in the subsequent investigation by Federal and local authorities. +LJ049-0124|In addition, the proposed legislation will insure|In addition, the proposed legislation will insure +LJ049-0125|that any suspects who are arrested will be Federal prisoners, subject to Federal protection from vigilante justice and other threats.|that any suspects who are arrested will be Federal prisoners, subject to Federal protection from vigilante justice and other threats. +LJ049-0126|Committee of Cabinet Officers. As our Government has become more complex,|Committee of Cabinet Officers. As our Government has become more complex, +LJ049-0127|agencies other than the Secret Service have become involved in phases of the overall problem of protecting our national leaders.|agencies other than the Secret Service have become involved in phases of the overall problem of protecting our national leaders. +LJ049-0128|The FBI is the major domestic investigating agency of the United States,|The FBI is the major domestic investigating agency of the United States, +LJ049-0129|while the CIA has the primary responsibility for collecting intelligence overseas to supplement information acquired by the Department of State.|while the CIA has the primary responsibility for collecting intelligence overseas to supplement information acquired by the Department of State. +LJ049-0130|The Secret Service must rely in large part|The Secret Service must rely in large part +LJ049-0132|The Commission believes that it is necessary to improve the cooperation among these agencies|The Commission believes that it is necessary to improve the cooperation among these agencies +LJ049-0133|and to emphasize that the task of Presidential protection is one of broad national concern.|and to emphasize that the task of Presidential protection is one of broad national concern. +LJ049-0134|The Commission suggests that consideration might be given to assigning to a Cabinet-level committee or the National Security Council|The Commission suggests that consideration might be given to assigning to a Cabinet-level committee or the National Security Council +LJ049-0135|(which is responsible for advising the President respecting the coordination|(which is responsible for advising the President respecting the coordination +LJ049-0136|of departmental policies relating to the national security) the responsibility to review and oversee the protective activities of the Secret Service|of departmental policies relating to the national security) the responsibility to review and oversee the protective activities of the Secret Service +LJ049-0137|and the other Federal agencies that assist in safeguarding the President. The Committee should include the Secretary of the Treasury and the Attorney General,|and the other Federal agencies that assist in safeguarding the President. The Committee should include the Secretary of the Treasury and the Attorney General, +LJ049-0138|and, if the Council is used, arrangements should be made for the attendance of the Secretary of the Treasury|and, if the Council is used, arrangements should be made for the attendance of the Secretary of the Treasury +LJ049-0139|and the Attorney General at any meetings which are concerned with Presidential protection.|and the Attorney General at any meetings which are concerned with Presidential protection. +LJ049-0140|The Council already includes, in addition to the President and Vice President, the Secretaries of State and Defense and has a competent staff.|The Council already includes, in addition to the President and Vice President, the Secretaries of State and Defense and has a competent staff. +LJ049-0141|The foremost assignment of the Committee would be to insure that the maximum resources of the Federal Government are fully engaged in the job of protecting the President,|The foremost assignment of the Committee would be to insure that the maximum resources of the Federal Government are fully engaged in the job of protecting the President, +LJ049-0142|by defining responsibilities clearly and overseeing their execution.|by defining responsibilities clearly and overseeing their execution. +LJ049-0143|Major needs of personnel or other resources might be met more easily on its recommendation than they have been in the past.|Major needs of personnel or other resources might be met more easily on its recommendation than they have been in the past. +LJ049-0144|The Committee would be able to provide guidance in defining the general nature of domestic and foreign dangers to Presidential security.|The Committee would be able to provide guidance in defining the general nature of domestic and foreign dangers to Presidential security. +LJ049-0145|As improvements are recommended for the advance detection of potential threats to the President, it could act as a final review board.|As improvements are recommended for the advance detection of potential threats to the President, it could act as a final review board. +LJ049-0146|The expert assistance and resources which it could draw upon would be particularly desirable in this complex and sensitive area.|The expert assistance and resources which it could draw upon would be particularly desirable in this complex and sensitive area. +LJ049-0147|This arrangement would provide a continuing high-level contact for agencies that may wish to consult respecting particular protective measures.|This arrangement would provide a continuing high-level contact for agencies that may wish to consult respecting particular protective measures. +LJ049-0148|For various reasons the Secret Service has functioned largely as an informal part of the White House staff, with the result|For various reasons the Secret Service has functioned largely as an informal part of the White House staff, with the result +LJ049-0149|that it has been unable, as a practical matter, to exercise sufficient influence over the security precautions which surround Presidential activities.|that it has been unable, as a practical matter, to exercise sufficient influence over the security precautions which surround Presidential activities. +LJ049-0150|A Cabinet-level committee which is actively concerned with these problems would be able to discuss these matters more effectively with the President.|A Cabinet-level committee which is actively concerned with these problems would be able to discuss these matters more effectively with the President. +LJ049-0151|Responsibilities for Presidential Protection|Responsibilities for Presidential Protection +LJ049-0152|The assignment of the responsibility of protecting the President to an agency of the Department of the Treasury was largely an historical accident.|The assignment of the responsibility of protecting the President to an agency of the Department of the Treasury was largely an historical accident. +LJ049-0153|The Secret Service was organized as a division of the Department of the Treasury in 1865, to deal with counterfeiting.|The Secret Service was organized as a division of the Department of the Treasury in eighteen sixty-five, to deal with counterfeiting. +LJ049-0154|In 1894,|In eighteen ninety-four, +LJ049-0155|while investigating a plot to assassinate President Cleveland, the Service assigned a small protective detail of agents to the White House.|while investigating a plot to assassinate President Cleveland, the Service assigned a small protective detail of agents to the White House. +LJ049-0156|Secret Service men accompanied the President and his family to their vacation home in Massachusetts|Secret Service men accompanied the President and his family to their vacation home in Massachusetts +LJ049-0157|and special details protected him in Washington, on trips, and at special functions.|and special details protected him in Washington, on trips, and at special functions. +LJ049-0158|These informal and part-time arrangements led to more systematic protection in 1902, after the assassination of President McKinley;|These informal and part-time arrangements led to more systematic protection in nineteen oh two, after the assassination of President McKinley; +LJ049-0159|the Secret Service, then the only Federal investigative agency, assumed full-time responsibility for the safety of the President.|the Secret Service, then the only Federal investigative agency, assumed full-time responsibility for the safety of the President. +LJ049-0160|Since that time, the Secret Service has had and exercised responsibility for the physical protection of the President|Since that time, the Secret Service has had and exercised responsibility for the physical protection of the President +LJ049-0161|and also for the preventive investigation of potential threats against the President.|and also for the preventive investigation of potential threats against the President. +LJ049-0162|Although the Secret Service has had the primary responsibility for the protection of the President,|Although the Secret Service has had the primary responsibility for the protection of the President, +LJ049-0163|the FBI, which was established within the Department of Justice in 1908, has had in recent years an increasingly important role to play.|the FBI, which was established within the Department of Justice in nineteen oh eight, has had in recent years an increasingly important role to play. +LJ049-0164|In the appropriations of the FBI there has recurred annually an item for the, quote, protection of the person of the President of the United States, end quote,|In the appropriations of the FBI there has recurred annually an item for the, quote, protection of the person of the President of the United States, end quote, +LJ049-0165|which first appeared in the appropriation of the Department of Justice in 1910 under the heading, quote, Miscellaneous Objects, end quote.|which first appeared in the appropriation of the Department of Justice in nineteen ten under the heading, quote, Miscellaneous Objects, end quote. +LJ049-0166|Although the FBI is not charged with the physical protection of the President, it does have an assignment, as do other Government agencies,|Although the FBI is not charged with the physical protection of the President, it does have an assignment, as do other Government agencies, +LJ049-0167|in the field of preventive investigation in regard to the President's security.|in the field of preventive investigation in regard to the President's security. +LJ049-0168|As discussed above, the Bureau has attempted to meet its responsibilities in this field by spelling out in its Handbook|As discussed above, the Bureau has attempted to meet its responsibilities in this field by spelling out in its Handbook +LJ049-0169|the procedures which its agents are to follow in connection with information received, quote,|the procedures which its agents are to follow in connection with information received, quote, +LJ049-0170|indicating the possibility of an attempt against the person or safety of the President, end quote, or other protected persons.|indicating the possibility of an attempt against the person or safety of the President, end quote, or other protected persons. +LJ049-0171|With two Federal agencies operating in the same general field of preventive investigation,|With two Federal agencies operating in the same general field of preventive investigation, +LJ049-0172|questions inevitably arise as to the scope of each agency's authority and responsibility.|questions inevitably arise as to the scope of each agency's authority and responsibility. +LJ049-0173|As the testimony of J. Edgar Hoover and other Bureau officials revealed, the FBI did not believe that its directive required the Bureau|As the testimony of J. Edgar Hoover and other Bureau officials revealed, the FBI did not believe that its directive required the Bureau +LJ049-0175|before the President reached Dallas.|before the President reached Dallas. +LJ049-0176|On the other hand, the Secret Service had no knowledge whatever of Oswald, his background, or his employment at the Book Depository,|On the other hand, the Secret Service had no knowledge whatever of Oswald, his background, or his employment at the Book Depository, +LJ049-0177|and Robert I. Bouck, who was in charge of the Protective Research Section of the Secret Service, believed that the accumulation of the facts known to the FBI|and Robert I. Bouck, who was in charge of the Protective Research Section of the Secret Service, believed that the accumulation of the facts known to the FBI +LJ049-0178|should have constituted a sufficient basis to warn the Secret Service of the Oswald risk.|should have constituted a sufficient basis to warn the Secret Service of the Oswald risk. +LJ049-0179|The Commission believes that both the FBI and the Secret Service have too narrowly construed their respective responsibilities.|The Commission believes that both the FBI and the Secret Service have too narrowly construed their respective responsibilities. +LJ049-0180|The Commission has the impression|The Commission has the impression +LJ049-0181|that too much emphasis is placed by both on the investigation of specific threats by individuals and not enough on dangers from other sources.|that too much emphasis is placed by both on the investigation of specific threats by individuals and not enough on dangers from other sources. +LJ049-0182|In addition, the Commission has concluded that the Secret Service particularly tends to be the passive recipient of information|In addition, the Commission has concluded that the Secret Service particularly tends to be the passive recipient of information +LJ049-0183|regarding such threats and that its Protective Research Section is not adequately staffed or equipped|regarding such threats and that its Protective Research Section is not adequately staffed or equipped +LJ049-0184|to conduct the wider investigative work that is required today for the security of the President.|to conduct the wider investigative work that is required today for the security of the President. +LJ049-0185|During the period the Commission was giving thought to this situation,|During the period the Commission was giving thought to this situation, +LJ049-0186|the Commission received a number of proposals designed to improve current arrangements for protecting the President.|the Commission received a number of proposals designed to improve current arrangements for protecting the President. +LJ049-0187|These proposals included suggestions to locate exclusive responsibility for all phases of the work|These proposals included suggestions to locate exclusive responsibility for all phases of the work +LJ049-0188|in one or another Government agency, to clarify the division of authority between the agencies involved, and to retain the existing system|in one or another Government agency, to clarify the division of authority between the agencies involved, and to retain the existing system +LJ049-0189|but expand both the scope and the operations of the existing agencies, particularly those of the Secret Service and the FBI.|but expand both the scope and the operations of the existing agencies, particularly those of the Secret Service and the FBI. +LJ049-0190|It has been pointed out that the FBI, as our chief investigative agency,|It has been pointed out that the FBI, as our chief investigative agency, +LJ049-0191|is properly manned and equipped to carry on extensive information gathering functions within the United States.|is properly manned and equipped to carry on extensive information gathering functions within the United States. +LJ049-0192|It was also suggested that it would take a substantial period of time for the Secret Service to build up the experience and skills necessary to meet the problem.|It was also suggested that it would take a substantial period of time for the Secret Service to build up the experience and skills necessary to meet the problem. +LJ049-0193|Consequently the suggestion has been made, on the one hand, that all preventive investigative functions relating to the security of the President|Consequently the suggestion has been made, on the one hand, that all preventive investigative functions relating to the security of the President +LJ049-0194|should be transferred to the FBI,|should be transferred to the FBI, +LJ049-0195|leaving with the Secret Service only the responsibility for the physical protection of the President, that is, the guarding function alone.|leaving with the Secret Service only the responsibility for the physical protection of the President, that is, the guarding function alone. +LJ049-0196|On the other hand, it is urged that all features of the protection of the President and his family should be committed to an elite and independent corps.|On the other hand, it is urged that all features of the protection of the President and his family should be committed to an elite and independent corps. +LJ049-0197|It is also contended that the agents should be intimately associated with the life of the Presidential family|It is also contended that the agents should be intimately associated with the life of the Presidential family +LJ049-0198|in all its ramifications and alert to every danger that might befall it,|in all its ramifications and alert to every danger that might befall it, +LJ049-0199|and ready at any instant to hazard great danger to themselves in the performance of their tremendous responsibility.|and ready at any instant to hazard great danger to themselves in the performance of their tremendous responsibility. +LJ049-0200|It is suggested that an organization shorn of its power to investigate all the possibilities of danger to the President|It is suggested that an organization shorn of its power to investigate all the possibilities of danger to the President +LJ049-0201|and becoming merely the recipient of information gathered by others would become limited solely to acts of physical alertness and personal courage|and becoming merely the recipient of information gathered by others would become limited solely to acts of physical alertness and personal courage +LJ049-0202|incident to its responsibilities.|incident to its responsibilities. +LJ049-0203|So circumscribed, it could not maintain the esprit de corps or the necessary alertness for this unique and challenging responsibility.|So circumscribed, it could not maintain the esprit de corps or the necessary alertness for this unique and challenging responsibility. +LJ049-0204|While in accordance with its mandate|While in accordance with its mandate +LJ049-0205|this Commission has necessarily examined into the functioning of the various Federal agencies concerned with the tragic trip of President Kennedy to Dallas|this Commission has necessarily examined into the functioning of the various Federal agencies concerned with the tragic trip of President Kennedy to Dallas +LJ049-0206|and while it has arrived at certain conclusions in respect thereto, it seems clear|and while it has arrived at certain conclusions in respect thereto, it seems clear +LJ049-0207|that it was not within the Commission's responsibility to make specific recommendations as to the long-range organization of the President's protection,|that it was not within the Commission's responsibility to make specific recommendations as to the long-range organization of the President's protection, +LJ049-0208|except as conclusions flowing directly from its examination of the President's assassination can be drawn.|except as conclusions flowing directly from its examination of the President's assassination can be drawn. +LJ049-0209|The Commission was not asked to apply itself as did the Hoover Commission in 1949,|The Commission was not asked to apply itself as did the Hoover Commission in nineteen forty-nine, +LJ049-0210|for examples to a determination of the optimum organization of the President's protection.|for examples to a determination of the optimum organization of the President's protection. +LJ049-0211|It would have been necessary for the Commission to take considerable testimony, much of it extraneous to the facts of the assassination of President Kennedy,|It would have been necessary for the Commission to take considerable testimony, much of it extraneous to the facts of the assassination of President Kennedy, +LJ049-0212|to put it in a position to reach final conclusions in this respect.|to put it in a position to reach final conclusions in this respect. +LJ049-0213|There are always dangers of divided responsibility,|There are always dangers of divided responsibility, +LJ049-0214|duplication, and confusion of authority where more than one agency is operating in the same field; but on the other hand|duplication, and confusion of authority where more than one agency is operating in the same field; but on the other hand +LJ049-0215|the protection of the President is in a real sense a Government-wide responsibility which must necessarily assumed by the Department of State,|the protection of the President is in a real sense a Government-wide responsibility which must necessarily assumed by the Department of State, +LJ049-0216|the FBI, the CIA, and the military intelligence agencies as well as the Secret Service.|the FBI, the CIA, and the military intelligence agencies as well as the Secret Service. +LJ049-0217|Moreover, a number of imponderable questions have to be weighed if any change in the intimate association now established|Moreover, a number of imponderable questions have to be weighed if any change in the intimate association now established +LJ049-0218|between the Secret Service and the President and his family is contemplated.|between the Secret Service and the President and his family is contemplated. +LJ049-0219|These considerations have induced the Commission to believe|These considerations have induced the Commission to believe +LJ049-0220|that the determination of whether or not there should be a relocation of responsibilities and functions should be left to the Executive and the Congress,|that the determination of whether or not there should be a relocation of responsibilities and functions should be left to the Executive and the Congress, +LJ049-0221|perhaps upon recommendations based on further studies by the Cabinet-level committee recommended above or the National Security Council.|perhaps upon recommendations based on further studies by the Cabinet-level committee recommended above or the National Security Council. +LJ049-0222|Pending any such determination, however, this Commission is convinced of the necessity of better coordination|Pending any such determination, however, this Commission is convinced of the necessity of better coordination +LJ049-0223|and direction of the activities of all existing agencies of Government which are in a position to and do, furnish information|and direction of the activities of all existing agencies of Government which are in a position to and do, furnish information +LJ049-0224|and services related to the security of the President.|and services related to the security of the President. +LJ049-0225|The Commission feels the Secret Service and the FBI, as well as the State Department and the CIA when the President travels abroad,|The Commission feels the Secret Service and the FBI, as well as the State Department and the CIA when the President travels abroad, +LJ049-0226|could improve their existing capacities and procedures so as to lessen the chances of assassination.|could improve their existing capacities and procedures so as to lessen the chances of assassination. +LJ049-0227|Without, therefore, coming to final conclusions respecting the long-range organization of the President's security,|Without, therefore, coming to final conclusions respecting the long-range organization of the President's security, +LJ049-0228|the Commission believes that the facts of the assassination of President Kennedy point to certain measures which,|the Commission believes that the facts of the assassination of President Kennedy point to certain measures which, +LJ049-0229|while assuming no radical relocation of responsibilities,|while assuming no radical relocation of responsibilities, +LJ049-0230|can and should be recommended by this Commission in the interest of the more efficient protection of the President.|can and should be recommended by this Commission in the interest of the more efficient protection of the President. +LJ050-0001|For more information, or to volunteer, please visit librivox dot org. Report of the President's Commission on the Assassination of President Kennedy.|For more information, or to volunteer, please visit librivox dot org. Report of the President's Commission on the Assassination of President Kennedy. +LJ050-0002|The Warren Commission Report.|The Warren Commission Report. +LJ050-0003|By The President's Commission on the Assassination of President Kennedy. Chapter 8. The Protection of the President. Part 5.|By The President's Commission on the Assassination of President Kennedy. Chapter eight. The Protection of the President. Part five. +LJ050-0004|General Supervision of the Secret Service|General Supervision of the Secret Service +LJ050-0005|The intimacy of the Secret Service's relationship to the White House|The intimacy of the Secret Service's relationship to the White House +LJ050-0006|and the dissimilarity of its protective functions to most activities of the Department of the Treasury|and the dissimilarity of its protective functions to most activities of the Department of the Treasury +LJ050-0007|have made it difficult for the Treasury to maintain close and continuing supervision.|have made it difficult for the Treasury to maintain close and continuing supervision. +LJ050-0008|The Commission believes that the recommended Cabinet-level committee will help to correct many of the major deficiencies of supervision|The Commission believes that the recommended Cabinet-level committee will help to correct many of the major deficiencies of supervision +LJ050-0009|disclosed by the Commission's investigation. Other measures should be taken as well to improve the overall operation of the Secret Service.|disclosed by the Commission's investigation. Other measures should be taken as well to improve the overall operation of the Secret Service. +LJ050-0010|Daily supervision of the operations of the Secret Service within the Department of the Treasury should be improved.|Daily supervision of the operations of the Secret Service within the Department of the Treasury should be improved. +LJ050-0011|The Chief of the Service now reports to the Secretary of the Treasury|The Chief of the Service now reports to the Secretary of the Treasury +LJ050-0012|through an Assistant Secretary whose duties also include the direct supervision of the Bureau of the Mint|through an Assistant Secretary whose duties also include the direct supervision of the Bureau of the Mint +LJ050-0013|and the Department's Employment Policy Program, and who also represents the Secretary of the Treasury on various committees and groups.|and the Department's Employment Policy Program, and who also represents the Secretary of the Treasury on various committees and groups. +LJ050-0014|The incumbent has no technical qualifications in the area of Presidential protection.|The incumbent has no technical qualifications in the area of Presidential protection. +LJ050-0015|The Commission recommends that the Secretary of the Treasury appoint a special assistant with the responsibility of supervising the Service.|The Commission recommends that the Secretary of the Treasury appoint a special assistant with the responsibility of supervising the Service. +LJ050-0016|This special assistant should be required to have sufficient stature and experience in law enforcement, intelligence, or allied fields|This special assistant should be required to have sufficient stature and experience in law enforcement, intelligence, or allied fields +LJ050-0017|to be able to provide effective continuing supervision|to be able to provide effective continuing supervision +LJ050-0018|and to keep the Secretary fully informed regarding all significant developments relating to Presidential protection.|and to keep the Secretary fully informed regarding all significant developments relating to Presidential protection. +LJ050-0019|This report has already pointed out several respects|This report has already pointed out several respects +LJ050-0020|in which the Commission believes that the Secret Service has operated with insufficient planning or control.|in which the Commission believes that the Secret Service has operated with insufficient planning or control. +LJ050-0021|Actions by the Service since the assassination indicate its awareness of the necessity for substantial improvement in its administration.|Actions by the Service since the assassination indicate its awareness of the necessity for substantial improvement in its administration. +LJ050-0022|A formal and thorough description of the responsibilities of the advance agent is now in preparation by the Service.|A formal and thorough description of the responsibilities of the advance agent is now in preparation by the Service. +LJ050-0023|Work is going forward|Work is going forward +LJ050-0024|toward the preparation of formal understandings of the respective roles of the Secret Service and other agencies with which it collaborates|toward the preparation of formal understandings of the respective roles of the Secret Service and other agencies with which it collaborates +LJ050-0025|or from which it derives assistance and support.|or from which it derives assistance and support. +LJ050-0026|The Commission urges that the Service continue this effort to overhaul and define its procedures.|The Commission urges that the Service continue this effort to overhaul and define its procedures. +LJ050-0027|While manuals and memoranda are no guarantee of effective operations,|While manuals and memoranda are no guarantee of effective operations, +LJ050-0028|no sizable organization can achieve efficiency without the careful analysis and demarcation of responsibility|no sizable organization can achieve efficiency without the careful analysis and demarcation of responsibility +LJ050-0029|that is reflected in definite and comprehensive operating procedures.|that is reflected in definite and comprehensive operating procedures. +LJ050-0030|The Commission also recommends|The Commission also recommends +LJ050-0031|that the Secret Service consciously set about the task of inculcating and maintaining the highest standard of excellence and esprit, for all of its personnel.|that the Secret Service consciously set about the task of inculcating and maintaining the highest standard of excellence and esprit, for all of its personnel. +LJ050-0032|This involves tight and unswerving discipline as well as the promotion of an outstanding degree of dedication and loyalty to duty.|This involves tight and unswerving discipline as well as the promotion of an outstanding degree of dedication and loyalty to duty. +LJ050-0033|The Commission emphasizes that it finds no causal connection between the assassination|The Commission emphasizes that it finds no causal connection between the assassination +LJ050-0034|and the breach of regulations which occurred on the night of November 21 at Fort Worth.|and the breach of regulations which occurred on the night of November twenty-one at Fort Worth. +LJ050-0035|Nevertheless, such a breach, in which so many agents participated,|Nevertheless, such a breach, in which so many agents participated, +LJ050-0036|is not consistent with the standards which the responsibilities of the Secret Service require it to meet.|is not consistent with the standards which the responsibilities of the Secret Service require it to meet. +LJ050-0037|Preventive Intelligence|Preventive Intelligence +LJ050-0038|In attempting to identify those individuals who might prove a danger to the President,|In attempting to identify those individuals who might prove a danger to the President, +LJ050-0039|the Secret Service has largely been the passive recipient of threatening communications to the President|the Secret Service has largely been the passive recipient of threatening communications to the President +LJ050-0040|and reports from other agencies which independently evaluate their information for potential sources of danger.|and reports from other agencies which independently evaluate their information for potential sources of danger. +LJ050-0041|This was the consequence of the Service's lack of an adequate investigative staff,|This was the consequence of the Service's lack of an adequate investigative staff, +LJ050-0042|its inability to process large amounts of data, and its failure to provide specific descriptions of the kind of information it sought.|its inability to process large amounts of data, and its failure to provide specific descriptions of the kind of information it sought. +LJ050-0043|The Secret Service has embarked upon a complete overhaul of its research activities. The staff of the Protective Research Section (PRS)|The Secret Service has embarked upon a complete overhaul of its research activities. The staff of the Protective Research Section (PRS) +LJ050-0044|has been augmented, and a Secret Service inspector has been put in charge of this operation. With the assistance of the President's Office of Science and Technology,|has been augmented, and a Secret Service inspector has been put in charge of this operation. With the assistance of the President's Office of Science and Technology, +LJ050-0045|and of the Advanced Research Projects Agency of the Department of Defense,|and of the Advanced Research Projects Agency of the Department of Defense, +LJ050-0046|it has obtained the services of outside consultants, such as the Rand Corporation,|it has obtained the services of outside consultants, such as the Rand Corporation, +LJ050-0047|International Business Machines Corporation, and a panel of psychiatric and psychological experts.|International Business Machines Corporation, and a panel of psychiatric and psychological experts. +LJ050-0048|It has received assistance also from data processing experts at the CIA|It has received assistance also from data processing experts at the CIA +LJ050-0050|As a result of these studies, the planning document submitted by the Secretary of the Treasury to the Bureau of the Budget on August 31,|As a result of these studies, the planning document submitted by the Secretary of the Treasury to the Bureau of the Budget on August thirty-one, +LJ050-0051|1964, makes several significant recommendations in this field.|nineteen sixty-four, makes several significant recommendations in this field. +LJ050-0052|Based on the Commission's investigation, the following minimum goals for improvements are indicated:|Based on the Commission's investigation, the following minimum goals for improvements are indicated: +LJ050-0053|Broader and more selective criteria|Broader and more selective criteria +LJ050-0054|Since the assassination, both the Secret Service and the FBI have recognized|Since the assassination, both the Secret Service and the FBI have recognized +LJ050-0055|that the PRS files can no longer be limited largely to persons communicating actual threats to the President.|that the PRS files can no longer be limited largely to persons communicating actual threats to the President. +LJ050-0056|On December 26, 1963, the FBI circulated additional instructions to all its agents,|On December twenty-six, nineteen sixty-three, the FBI circulated additional instructions to all its agents, +LJ050-0057|specifying criteria for information to be furnished to the Secret Service in addition to that covered by the former standard,|specifying criteria for information to be furnished to the Secret Service in addition to that covered by the former standard, +LJ050-0058|which was the possibility of an attempt against the person or safety of the President.|which was the possibility of an attempt against the person or safety of the President. +LJ050-0059|he new instructions require FBI agents to report immediately information concerning, quote,|he new instructions require FBI agents to report immediately information concerning, quote, +LJ050-0060|Subversives, ultrarightists, racists and fascists (a) possessing emotional instability or irrational behavior,|Subversives, ultrarightists, racists and fascists (a) possessing emotional instability or irrational behavior, +LJ050-0061|(b) who have made threats of bodily harm against officials or employees of Federal, state or local government or officials of a foreign government,|(b) who have made threats of bodily harm against officials or employees of Federal, state or local government or officials of a foreign government, +LJ050-0062|(c) who express or have expressed strong or violent anti-U.S. sentiments|(c) who express or have expressed strong or violent anti-U.S. sentiments +LJ050-0063|and who have been involved in bombing or bomb-making or whose past conduct indicates tendencies toward violence, and (d)|and who have been involved in bombing or bomb-making or whose past conduct indicates tendencies toward violence, and (d) +LJ050-0064|whose prior acts or statements depict propensity for violence and hatred against organized government, end quote.|whose prior acts or statements depict propensity for violence and hatred against organized government, end quote. +LJ050-0065|Alan H. Belmont, Assistant to the Director of the FBI, testified that this revision was initiated by the FBI itself.|Alan H. Belmont, Assistant to the Director of the FBI, testified that this revision was initiated by the FBI itself. +LJ050-0066|The volume of references to the Secret Service has increased substantially since the new instructions went into effect;|The volume of references to the Secret Service has increased substantially since the new instructions went into effect; +LJ050-0067|more than 5,000 names were referred to the Secret Service in the first 4 months of 1964.|more than five thousand names were referred to the Secret Service in the first four months of nineteen sixty-four. +LJ050-0068|According to Chief Rowley, by mid-June 1964,|According to Chief Rowley, by mid-June nineteen sixty-four, +LJ050-0069|the Secret Service had received from the FBI some 9,000 reports on members of the Communist Party.|the Secret Service had received from the FBI some nine thousand reports on members of the Communist Party. +LJ050-0070|The FBI now transmits information on all defectors, a category which would, of course, have included Oswald.|The FBI now transmits information on all defectors, a category which would, of course, have included Oswald. +LJ050-0071|Both Director Hoover and Belmont expressed to the Commission the great concern of the FBI, which is shared by the Secret Service,|Both Director Hoover and Belmont expressed to the Commission the great concern of the FBI, which is shared by the Secret Service, +LJ050-0072|that referrals to the Secret Service under the new criteria might, if not properly handled,|that referrals to the Secret Service under the new criteria might, if not properly handled, +LJ050-0073|result in some degree of interference with the personal liberty of those involved.|result in some degree of interference with the personal liberty of those involved. +LJ050-0074|They emphasized the necessity that the information now being furnished be handled with judgment and care.|They emphasized the necessity that the information now being furnished be handled with judgment and care. +LJ050-0075|The Commission shares this concern.|The Commission shares this concern. +LJ050-0076|The problem is aggravated by the necessity that the Service obtain the assistance of local law enforcement officials in evaluating the information which it receives|The problem is aggravated by the necessity that the Service obtain the assistance of local law enforcement officials in evaluating the information which it receives +LJ050-0077|and in taking preventive steps.|and in taking preventive steps. +LJ050-0078|In June 1964, the Secret Service sent to a number of Federal law enforcement and intelligence agencies|In June nineteen sixty-four, the Secret Service sent to a number of Federal law enforcement and intelligence agencies +LJ050-0079|guidelines for an experimental program to develop more detailed criteria.|guidelines for an experimental program to develop more detailed criteria. +LJ050-0080|The suggestions of Federal agencies for revision of these guidelines were solicited.|The suggestions of Federal agencies for revision of these guidelines were solicited. +LJ050-0081|The new tentative criteria are useful in making clear that|The new tentative criteria are useful in making clear that +LJ050-0082|the interest of the Secret Service goes beyond information on individuals or groups threatening to cause harm or embarrassment to the President.|the interest of the Secret Service goes beyond information on individuals or groups threatening to cause harm or embarrassment to the President. +LJ050-0083|Information is requested also concerning individuals or groups who have demonstrated an interest in the President|Information is requested also concerning individuals or groups who have demonstrated an interest in the President +LJ050-0084|or, quote, other high government officials in the nature of a complaint coupled with an expressed or implied determination to use a means,|or, quote, other high government officials in the nature of a complaint coupled with an expressed or implied determination to use a means, +LJ050-0085|other than legal or peaceful, to satisfy any grievance, real or imagined.|other than legal or peaceful, to satisfy any grievance, real or imagined. +LJ050-0086|Under these criteria, whether the case should be referred to the Secret Service depends on the existence of a previous history of mental instability,|Under these criteria, whether the case should be referred to the Secret Service depends on the existence of a previous history of mental instability, +LJ050-0087|propensity toward violent action, or some similar characteristic, coupled with some evaluation of the capability of the individual or group|propensity toward violent action, or some similar characteristic, coupled with some evaluation of the capability of the individual or group +LJ050-0088|to further the intention to satisfy a grievance by unlawful means.|to further the intention to satisfy a grievance by unlawful means. +LJ050-0089|While these tentative criteria are a step in the right direction,|While these tentative criteria are a step in the right direction, +LJ050-0090|they seem unduly restrictive in continuing to require some manifestation of animus against a Government official.|they seem unduly restrictive in continuing to require some manifestation of animus against a Government official. +LJ050-0091|It is questionable whether such criteria would have resulted in the referral of Oswald to the Secret Service.|It is questionable whether such criteria would have resulted in the referral of Oswald to the Secret Service. +LJ050-0092|Chief Rowley believed that they would,|Chief Rowley believed that they would, +LJ050-0093|because of Oswald's demonstrated hostility toward the Secretary of the Navy in his letter of January 30, 1962. Quote,|because of Oswald's demonstrated hostility toward the Secretary of the Navy in his letter of January thirty, nineteen sixty-two. Quote, +LJ050-0094|I shall employ all means to right this gross mistake or injustice to a bona fide U.S. citizen and ex-service man.|I shall employ all means to right this gross mistake or injustice to a bona fide U.S. citizen and ex-service man. +LJ050-0095|The U.S. government has no charges or complaints against me.|The U.S. government has no charges or complaints against me. +LJ050-0096|I ask you to look into this case and take the necessary steps to repair the damage done to me and my family. End quote.|I ask you to look into this case and take the necessary steps to repair the damage done to me and my family. End quote. +LJ050-0097|Even with the advantage of hindsight, this letter does not appear to express or imply Oswald's, quote,|Even with the advantage of hindsight, this letter does not appear to express or imply Oswald's, quote, +LJ050-0098|determination to use a means, other than legal or peaceful, to satisfy his grievance, end quote, within the meaning of the new criteria.|determination to use a means, other than legal or peaceful, to satisfy his grievance, end quote, within the meaning of the new criteria. +LJ050-0099|It is apparent that a good deal of further consideration and experimentation will be required before adequate criteria can be framed.|It is apparent that a good deal of further consideration and experimentation will be required before adequate criteria can be framed. +LJ050-0100|The Commission recognizes that no set of meaningful criteria will yield the names of all potential assassins. Charles J. Guiteau,|The Commission recognizes that no set of meaningful criteria will yield the names of all potential assassins. Charles J. Guiteau, +LJ050-0101|Leon F. Czolgosz,|Leon F. Czolgosz, +LJ050-0102|John Schrank, and Guiseppe Zangara -- four assassins or would-be assassins|John Schrank, and Guiseppe Zangara -- four assassins or would-be assassins +LJ050-0103|were all men who acted alone in their criminal acts against our leaders. None had a serious record of prior violence.|were all men who acted alone in their criminal acts against our leaders. None had a serious record of prior violence. +LJ050-0104|Each of them was a failure in his work and in his relations with others, a victim of delusions and fancies which led to the conviction|Each of them was a failure in his work and in his relations with others, a victim of delusions and fancies which led to the conviction +LJ050-0105|that society and its leaders had combined to thwart him. It will require every available resource of our Government|that society and its leaders had combined to thwart him. It will require every available resource of our Government +LJ050-0106|to devise a practical system which has any reasonable possibility of revealing such malcontents.|to devise a practical system which has any reasonable possibility of revealing such malcontents. +LJ050-0107|Liaison with other agencies regarding intelligence.|Liaison with other agencies regarding intelligence. +LJ050-0108|The Secret Service's liaison with the agencies that supply information to it has been too casual.|The Secret Service's liaison with the agencies that supply information to it has been too casual. +LJ050-0109|Since the assassination, the Service has recognized that these relationships must be far more formal|Since the assassination, the Service has recognized that these relationships must be far more formal +LJ050-0110|and each agency given clear understanding of the assistance which the Secret Service expects.|and each agency given clear understanding of the assistance which the Secret Service expects. +LJ050-0111|Once the Secret Service has formulated its new standards for collection of information, it should enter into written agreements with each Federal agency|Once the Secret Service has formulated its new standards for collection of information, it should enter into written agreements with each Federal agency +LJ050-0112|and the leading State and local agencies that might be a source of such information.|and the leading State and local agencies that might be a source of such information. +LJ050-0114|and the respective responsibilities for any further investigation that may be required.|and the respective responsibilities for any further investigation that may be required. +LJ050-0115|This is especially necessary with regard to the FBI and CIA,|This is especially necessary with regard to the FBI and CIA, +LJ050-0116|which carry the major responsibility for supplying information about potential threats,|which carry the major responsibility for supplying information about potential threats, +LJ050-0117|particularly those arising from organized groups, within their special jurisdiction.|particularly those arising from organized groups, within their special jurisdiction. +LJ050-0118|Since these agencies are already obliged constantly to evaluate the activities of such groups,|Since these agencies are already obliged constantly to evaluate the activities of such groups, +LJ050-0119|they should be responsible for advising the Secret Service if information develops indicating the existence of an assassination plot|they should be responsible for advising the Secret Service if information develops indicating the existence of an assassination plot +LJ050-0120|and for reporting such events as a change in leadership or dogma which indicate that the group may present a danger to the President.|and for reporting such events as a change in leadership or dogma which indicate that the group may present a danger to the President. +LJ050-0121|Detailed formal agreements embodying these arrangements should be worked out between the Secret Service and both of these agencies.|Detailed formal agreements embodying these arrangements should be worked out between the Secret Service and both of these agencies. +LJ050-0122|It should be made clear that the Secret Service will in no way seek to duplicate the intelligence|It should be made clear that the Secret Service will in no way seek to duplicate the intelligence +LJ050-0123|and investigative capabilities of the agencies now operating in this field but will continue|and investigative capabilities of the agencies now operating in this field but will continue +LJ050-0124|to use the data developed by these agencies to carry out its special duties.|to use the data developed by these agencies to carry out its special duties. +LJ050-0125|Once experience has been gained in implementing such agreements with the Federal and leading State and local agencies,|Once experience has been gained in implementing such agreements with the Federal and leading State and local agencies, +LJ050-0126|the Secret Service, through its field offices,|the Secret Service, through its field offices, +LJ050-0127|should negotiate similar arrangements with such other State and local law enforcement agencies as may provide meaningful assistance.|should negotiate similar arrangements with such other State and local law enforcement agencies as may provide meaningful assistance. +LJ050-0128|Much useful information will come to the attention of local law enforcement agencies in the regular course of their activities,|Much useful information will come to the attention of local law enforcement agencies in the regular course of their activities, +LJ050-0129|and this source should not be neglected by undue concentration on relationships with other Federal agencies.|and this source should not be neglected by undue concentration on relationships with other Federal agencies. +LJ050-0130|Finally, these agreements with Federal and local authorities will be of little value|Finally, these agreements with Federal and local authorities will be of little value +LJ050-0131|unless a system is established for the frequent formal review of activities thereunder. In this regard|unless a system is established for the frequent formal review of activities thereunder. In this regard +LJ050-0132|the Commission notes with approval several recent measures taken and proposed by the Secret Service to improve its liaison arrangements.|the Commission notes with approval several recent measures taken and proposed by the Secret Service to improve its liaison arrangements. +LJ050-0133|In his testimony Secretary of the Treasury C. Douglas Dillon informed the Commission|In his testimony Secretary of the Treasury C. Douglas Dillon informed the Commission +LJ050-0134|that an interagency committee has been established to develop more effective criteria.|that an interagency committee has been established to develop more effective criteria. +LJ050-0135|According to Secretary Dillon,|According to Secretary Dillon, +LJ050-0136|the Committee will include representatives of the President's Office of Science and Technology, Department of Defense, CIA,|the Committee will include representatives of the President's Office of Science and Technology, Department of Defense, CIA, +LJ050-0137|FBI, and the Secret Service.|FBI, and the Secret Service. +LJ050-0138|In addition, the Department of the Treasury has requested five additional agents for its Protective Research Section|In addition, the Department of the Treasury has requested five additional agents for its Protective Research Section +LJ050-0139|to serve as liaison officers with law enforcement and intelligence agencies. On the basis of the Department's review during the past several months,|to serve as liaison officers with law enforcement and intelligence agencies. On the basis of the Department's review during the past several months, +LJ050-0140|Secretary Dillon testified that the use of such liaison officers is the only effective way to insure that adequate liaison is maintained.|Secretary Dillon testified that the use of such liaison officers is the only effective way to insure that adequate liaison is maintained. +LJ050-0141|As a beginning step to improve liaison with local law enforcement officials, the Secret Service on August 26, 1964,|As a beginning step to improve liaison with local law enforcement officials, the Secret Service on August twenty-six, nineteen sixty-four, +LJ050-0142|directed its field representatives to send a form request for intelligence information to all local,|directed its field representatives to send a form request for intelligence information to all local, +LJ050-0143|county, and State law enforcement agencies in their districts.|county, and State law enforcement agencies in their districts. +LJ050-0144|Each of these efforts appears sound,|Each of these efforts appears sound, +LJ050-0145|and the Commission recommends that these and the other measures suggested by the Commission be pursued vigorously by Secret Service.|and the Commission recommends that these and the other measures suggested by the Commission be pursued vigorously by Secret Service. +LJ050-0146|Automatic data processing|Automatic data processing +LJ050-0147|Unless the Secret Service is able to deal rapidly and accurately with a growing body of data,|Unless the Secret Service is able to deal rapidly and accurately with a growing body of data, +LJ050-0148|the increased information supplied by other agencies will be wasted.|the increased information supplied by other agencies will be wasted. +LJ050-0149|PRS must develop the capacity to classify its subjects on a more sophisticated basis than the present geographic breakdown.|PRS must develop the capacity to classify its subjects on a more sophisticated basis than the present geographic breakdown. +LJ050-0151|it makes no use of the recent developments in automatic data processing which are widely used in the business world and in other Government offices.|it makes no use of the recent developments in automatic data processing which are widely used in the business world and in other Government offices. +LJ050-0152|The Secret Service and the Department of the Treasury now recognize this critical need.|The Secret Service and the Department of the Treasury now recognize this critical need. +LJ050-0153|In the planning document currently under review by the Bureau of the Budget, the Department recommends that it be permitted to hire five qualified persons, quote,|In the planning document currently under review by the Bureau of the Budget, the Department recommends that it be permitted to hire five qualified persons, quote, +LJ050-0154|to plan and develop a workable and efficient automated file and retrieval system, end quote.|to plan and develop a workable and efficient automated file and retrieval system, end quote. +LJ050-0155|Also the Department requests the sum of $100,000 to conduct a detailed feasibility study;|Also the Department requests the sum of one hundred thousand dollars to conduct a detailed feasibility study; +LJ050-0156|this money would be used to compensate consultants, to lease standard equipment or to purchase specially designed pilot equipment.|this money would be used to compensate consultants, to lease standard equipment or to purchase specially designed pilot equipment. +LJ050-0157|On the basis of such a feasibility study,|On the basis of such a feasibility study, +LJ050-0158|the Department hopes to design a practical system which will fully meet the needs of the Protective Research Section of the Secret Service.|the Department hopes to design a practical system which will fully meet the needs of the Protective Research Section of the Secret Service. +LJ050-0159|The Commission recommends that prompt and favorable consideration be given to this request.|The Commission recommends that prompt and favorable consideration be given to this request. +LJ050-0160|The Commission further recommends that the Secret Service coordinate its planning as closely as possible with all of the Federal agencies from which it receives information.|The Commission further recommends that the Secret Service coordinate its planning as closely as possible with all of the Federal agencies from which it receives information. +LJ050-0161|The Secret Service should not and does not plan to develop its own intelligence gathering facilities to duplicate the existing facilities of other Federal agencies.|The Secret Service should not and does not plan to develop its own intelligence gathering facilities to duplicate the existing facilities of other Federal agencies. +LJ050-0162|In planning its data processing techniques,|In planning its data processing techniques, +LJ050-0163|the Secret Service should attempt to develop a system compatible with those of the agencies from which most of its data will come. Note:|the Secret Service should attempt to develop a system compatible with those of the agencies from which most of its data will come. Note: +LJ050-0164|In evaluating data processing techniques of the Secret Service,|In evaluating data processing techniques of the Secret Service, +LJ050-0165|the Commission had occasion to become informed, to a limited extent, about the data processing techniques of other Federal intelligence and law enforcement agencies.|the Commission had occasion to become informed, to a limited extent, about the data processing techniques of other Federal intelligence and law enforcement agencies. +LJ050-0166|The Commission was struck by the apparent lack of effort, on an interagency basis,|The Commission was struck by the apparent lack of effort, on an interagency basis, +LJ050-0167|to develop coordinated and mutually compatible systems, even where such coordination would not seem inconsistent|to develop coordinated and mutually compatible systems, even where such coordination would not seem inconsistent +LJ050-0168|with the particular purposes of the agency involved. The Commission recognizes that this is a controversial area|with the particular purposes of the agency involved. The Commission recognizes that this is a controversial area +LJ050-0169|and that many strongly held views are advanced in resistance to any suggestion that an effort be made to impose any degree of coordination.|and that many strongly held views are advanced in resistance to any suggestion that an effort be made to impose any degree of coordination. +LJ050-0170|This matter is obviously beyond the jurisdiction of the Commission,|This matter is obviously beyond the jurisdiction of the Commission, +LJ050-0171|but it seems to warrant further study before each agency becomes irrevocably committed to separate action.|but it seems to warrant further study before each agency becomes irrevocably committed to separate action. +LJ050-0172|The Commission, therefore, recommends that the President consider ordering an inquiry into the possibility|The Commission, therefore, recommends that the President consider ordering an inquiry into the possibility +LJ050-0173|that coordination might be achieved to a greater extent than seems now to be contemplated, without interference with the primary mission of each agency involved.|that coordination might be achieved to a greater extent than seems now to be contemplated, without interference with the primary mission of each agency involved. +LJ050-0174|Protective Research participation in advance arrangements.|Protective Research participation in advance arrangements. +LJ050-0175|Since the assassination, Secret Service procedures have been changed to require that a member of PRS accompany each advance survey team|Since the assassination, Secret Service procedures have been changed to require that a member of PRS accompany each advance survey team +LJ050-0176|to establish liaison with local intelligence gathering agencies and to provide for the immediate evaluation of information received from them.|to establish liaison with local intelligence gathering agencies and to provide for the immediate evaluation of information received from them. +LJ050-0177|This PRS agent will also be responsible for establishing an informal local liaison committee|This PRS agent will also be responsible for establishing an informal local liaison committee +LJ050-0178|to make certain that all protective intelligence activities are coordinated.|to make certain that all protective intelligence activities are coordinated. +LJ050-0179|Based on its experience during this period, the Secret Service now recommends that additional personnel be made available to PRS|Based on its experience during this period, the Secret Service now recommends that additional personnel be made available to PRS +LJ050-0180|so that these arrangements can be made permanent without adversely affecting the operations of the Service's field offices.|so that these arrangements can be made permanent without adversely affecting the operations of the Service's field offices. +LJ050-0181|The Commission regards this as a most useful innovation and urges that the practice be continued.|The Commission regards this as a most useful innovation and urges that the practice be continued. +LJ050-0182|Liaison With Local Law Enforcement Agencies|Liaison With Local Law Enforcement Agencies +LJ050-0183|Advice by the Secret Service to local police in metropolitan areas relating to the assistance expected in connection with a Presidential visit|Advice by the Secret Service to local police in metropolitan areas relating to the assistance expected in connection with a Presidential visit +LJ050-0184|has hitherto been handled on an informal basis.|has hitherto been handled on an informal basis. +LJ050-0185|The Service should consider preparing formal explanations of the cooperation anticipated during a Presidential visit to a city,|The Service should consider preparing formal explanations of the cooperation anticipated during a Presidential visit to a city, +LJ050-0186|in formats that can be communicated to each level of local authorities.|in formats that can be communicated to each level of local authorities. +LJ050-0187|Thus, the local chief of police could be given a master plan, prepared for the occasion, of all protective measures to be taken during the visit;|Thus, the local chief of police could be given a master plan, prepared for the occasion, of all protective measures to be taken during the visit; +LJ050-0188|each patrolman might be given a prepared booklet of instructions explaining what is expected of him. The Secret Service has expressed concern|each patrolman might be given a prepared booklet of instructions explaining what is expected of him. The Secret Service has expressed concern +LJ050-0190|However, the instructions must be communicated to the local police in any event and can be leaked to the press whether or not they are in writing.|However, the instructions must be communicated to the local police in any event and can be leaked to the press whether or not they are in writing. +LJ050-0191|More importantly, the lack of carefully prepared and carefully transmitted instructions for typical visits to cities|More importantly, the lack of carefully prepared and carefully transmitted instructions for typical visits to cities +LJ050-0192|can lead to lapses in protection, such as the confusion in Dallas about whether members of the public were permitted on overpasses.|can lead to lapses in protection, such as the confusion in Dallas about whether members of the public were permitted on overpasses. +LJ050-0193|Such instructions will not fit all circumstances, of course,|Such instructions will not fit all circumstances, of course, +LJ050-0194|and should not be relied upon to the detriment of the imaginative application of judgment in special cases.|and should not be relied upon to the detriment of the imaginative application of judgment in special cases. +LJ050-0195|Inspection of Buildings|Inspection of Buildings +LJ050-0196|Since the assassination of President Kennedy, the Secret Service has been experimenting with new techniques in the inspection of buildings along a motorcade route.|Since the assassination of President Kennedy, the Secret Service has been experimenting with new techniques in the inspection of buildings along a motorcade route. +LJ050-0197|According to Secretary Dillon,|According to Secretary Dillon, +LJ050-0198|the studies indicate that there is some utility in attempting to designate certain buildings as involving a higher risk than others.|the studies indicate that there is some utility in attempting to designate certain buildings as involving a higher risk than others. +LJ050-0199|The Commission strongly encourages these efforts to improve protection along a motorcade route.|The Commission strongly encourages these efforts to improve protection along a motorcade route. +LJ050-0200|The Secret Service should utilize the personnel of other Federal law enforcement offices|The Secret Service should utilize the personnel of other Federal law enforcement offices +LJ050-0201|in the locality to assure adequate manpower for this task, as it is now doing.|in the locality to assure adequate manpower for this task, as it is now doing. +LJ050-0202|Lack of adequate resources is an unacceptable excuse for failing to improve advance precautions|Lack of adequate resources is an unacceptable excuse for failing to improve advance precautions +LJ050-0203|in this crucial area of Presidential protection.|in this crucial area of Presidential protection. +LJ050-0204|Secret Service Personnel and Facilities|Secret Service Personnel and Facilities +LJ050-0205|Testimony and other evidence before the Commission|Testimony and other evidence before the Commission +LJ050-0206|suggest that the Secret Service is trying to accomplish its job with too few people and without adequate modern equipment.|suggest that the Secret Service is trying to accomplish its job with too few people and without adequate modern equipment. +LJ050-0207|Although Chief Rowley does not complain about the pay scale for Secret Service agents,|Although Chief Rowley does not complain about the pay scale for Secret Service agents, +LJ050-0208|salaries are below those of the FBI and leading municipal police forces.|salaries are below those of the FBI and leading municipal police forces. +LJ050-0209|The assistant to the Director of the FBI testified that|The assistant to the Director of the FBI testified that +LJ050-0210|the caseload of each FBI agent averaged 20 to 25, and he felt that this was high.|the caseload of each FBI agent averaged twenty to twenty-five, and he felt that this was high. +LJ050-0211|Chief Rowley testified that the present workload of each Secret Service agent averages 110.1 cases.|Chief Rowley testified that the present workload of each Secret Service agent averages one hundred ten point one cases. +LJ050-0212|While these statistics relate to the activities of Secret Service agents stationed in field offices and not the White House detail,|While these statistics relate to the activities of Secret Service agents stationed in field offices and not the White House detail, +LJ050-0213|field agents supplement those on the detail, particularly when the President is traveling.|field agents supplement those on the detail, particularly when the President is traveling. +LJ050-0214|Although the Commission does not know whether the cases involved are entirely comparable,|Although the Commission does not know whether the cases involved are entirely comparable, +LJ050-0215|these figures suggest that the agents of the Secret Service are substantially overworked.|these figures suggest that the agents of the Secret Service are substantially overworked. +LJ050-0216|In its budget request for the fiscal year beginning July 1, 1964,|In its budget request for the fiscal year beginning July one, nineteen sixty-four, +LJ050-0217|the Secret Service sought funds for 25 new positions, primarily in field offices. This increase has been approved by the Congress.|the Secret Service sought funds for twenty-five new positions, primarily in field offices. This increase has been approved by the Congress. +LJ050-0218|Chief Rowley explained that this would not provide enough additional manpower to take all the measures which he considers required.|Chief Rowley explained that this would not provide enough additional manpower to take all the measures which he considers required. +LJ050-0219|However, the 1964 to 65 budget request was submitted in November 1963|However, the nineteen sixty-four to sixty-five budget request was submitted in November nineteen sixty-three +LJ050-0220|and requests for additional personnel were not made because of the studies then being conducted.|and requests for additional personnel were not made because of the studies then being conducted. +LJ050-0221|The Secret Service has now presented its recommendations to the Bureau of the Budget. The plan proposed by the Service|The Secret Service has now presented its recommendations to the Bureau of the Budget. The plan proposed by the Service +LJ050-0222|would take approximately 20 months to implement and require expenditures of approximately $3,000,000 during that period.|would take approximately twenty months to implement and require expenditures of approximately three million dollars during that period. +LJ050-0223|The plan provides for an additional 205 agents for the Secret Service. Seventeen of this number are proposed for the Protective Research Section;|The plan provides for an additional two hundred five agents for the Secret Service. Seventeen of this number are proposed for the Protective Research Section; +LJ050-0224|145 are proposed for the field offices to handle the increased volume of security investigations|one hundred forty-five are proposed for the field offices to handle the increased volume of security investigations +LJ050-0225|and be available to protect the President or Vice President when they travel;|and be available to protect the President or Vice President when they travel; +LJ050-0226|18 agents are proposed for a rotating pool which will go through an intensive training cycle and also be available to supplement the White House detail|eighteen agents are proposed for a rotating pool which will go through an intensive training cycle and also be available to supplement the White House detail +LJ050-0227|in case of unexpected need; and 25 additional agents are recommended to provide the Vice President full protection.|in case of unexpected need; and twenty-five additional agents are recommended to provide the Vice President full protection. +LJ050-0228|The Commission urges that the Bureau of the Budget review these recommendations with the Secret Service and authorize a request for the necessary supplemental appropriation,|The Commission urges that the Bureau of the Budget review these recommendations with the Secret Service and authorize a request for the necessary supplemental appropriation, +LJ050-0229|as soon as it can be justified. The Congress has often stressed that it will support any reasonable request for funds for the protection of the President.|as soon as it can be justified. The Congress has often stressed that it will support any reasonable request for funds for the protection of the President. +LJ050-0230|Manpower and Technical Assistance From Other Agencies|Manpower and Technical Assistance From Other Agencies +LJ050-0231|Before the assassination the Secret Service infrequently requested other Federal law enforcement agencies to provide personnel|Before the assassination the Secret Service infrequently requested other Federal law enforcement agencies to provide personnel +LJ050-0232|to assist in its protection functions.|to assist in its protection functions. +LJ050-0233|Since the assassination, the Service has experimented with the use of agents borrowed for short periods from such agencies.|Since the assassination, the Service has experimented with the use of agents borrowed for short periods from such agencies. +LJ050-0234|It has used other Treasury law enforcement agents on special experiments in building and route surveys in places to which the President frequently travels.|It has used other Treasury law enforcement agents on special experiments in building and route surveys in places to which the President frequently travels. +LJ050-0235|It has also used other Federal law enforcement agents during Presidential visits to cities in which such agents are stationed.|It has also used other Federal law enforcement agents during Presidential visits to cities in which such agents are stationed. +LJ050-0236|Thus, in the 4 months following the assassination,|Thus, in the four months following the assassination, +LJ050-0237|the FBI, on 16 separate occasions, supplied a total of 139 agents to assist in protection work during a Presidential visit,|the FBI, on sixteen separate occasions, supplied a total of one hundred thirty-nine agents to assist in protection work during a Presidential visit, +LJ050-0238|which represents a departure from its prior practice.|which represents a departure from its prior practice. +LJ050-0239|From February 11 through June 30, 1964,|From February eleven through June thirty, nineteen sixty-four, +LJ050-0240|the Service had the advantage of 9,500 hours of work by other enforcement agencies.|the Service had the advantage of nine thousand, five hundred hours of work by other enforcement agencies. +LJ050-0241|The FBI has indicated that it is willing to continue to make such assistance available,|The FBI has indicated that it is willing to continue to make such assistance available, +LJ050-0242|even though it agrees with the Secret Service that it is preferable for the Service to have enough agents to handle all protective demands.|even though it agrees with the Secret Service that it is preferable for the Service to have enough agents to handle all protective demands. +LJ050-0243|The Commission endorses these efforts to supplement the Service's own personnel by obtaining, for short periods of time,|The Commission endorses these efforts to supplement the Service's own personnel by obtaining, for short periods of time, +LJ050-0244|the assistance of trained Federal law enforcement officers.|the assistance of trained Federal law enforcement officers. +LJ050-0245|In view of the ever-increasing mobility of American Presidents, it seems unlikely that the Service could or should increase its own staff to a size|In view of the ever-increasing mobility of American Presidents, it seems unlikely that the Service could or should increase its own staff to a size +LJ050-0246|which would permit it to provide adequate protective manpower for all situations.|which would permit it to provide adequate protective manpower for all situations. +LJ050-0247|The Commission recommends that the agencies involved determine how much periodic assistance they can provide, and that each such agency|The Commission recommends that the agencies involved determine how much periodic assistance they can provide, and that each such agency +LJ050-0248|and the Secret Service enter into a formal agreement defining such arrangements.|and the Secret Service enter into a formal agreement defining such arrangements. +LJ050-0249|It may eventually be desirable to codify the practice in an Executive order.|It may eventually be desirable to codify the practice in an Executive order. +LJ050-0250|The Secret Service will be better able to plan its own long-range personnel requirements if it knows with reasonable certainty|The Secret Service will be better able to plan its own long-range personnel requirements if it knows with reasonable certainty +LJ050-0251|the amount of assistance that it can expect from other agencies.|the amount of assistance that it can expect from other agencies. +LJ050-0252|The occasional use of personnel from other Federal agencies to assist in protecting the President has a further advantage. It symbolizes the reality|The occasional use of personnel from other Federal agencies to assist in protecting the President has a further advantage. It symbolizes the reality +LJ050-0253|that the job of protecting the President has not been and cannot be exclusively the responsibility of the Secret Service.|that the job of protecting the President has not been and cannot be exclusively the responsibility of the Secret Service. +LJ050-0254|The Secret Service in the past has sometimes guarded its right to be acknowledged as the sole protector of the Chief Executive.|The Secret Service in the past has sometimes guarded its right to be acknowledged as the sole protector of the Chief Executive. +LJ050-0255|This no longer appears to be the case.|This no longer appears to be the case. +LJ050-0256|Protecting the President is a difficult and complex task which requires full use of the best resources of many parts of our Government.|Protecting the President is a difficult and complex task which requires full use of the best resources of many parts of our Government. +LJ050-0257|Recognition that the responsibility must be shared increases the likelihood that it will be met.|Recognition that the responsibility must be shared increases the likelihood that it will be met. +LJ050-0258|Much of the Secret Service work requires the development and use of highly sophisticated equipment,|Much of the Secret Service work requires the development and use of highly sophisticated equipment, +LJ050-0259|some of which must be specially designed to fit unique requirements. Even before the assassination, and to a far greater extent thereafter,|some of which must be specially designed to fit unique requirements. Even before the assassination, and to a far greater extent thereafter, +LJ050-0260|the Secret Service has been receiving full cooperation in scientific research and technological development|the Secret Service has been receiving full cooperation in scientific research and technological development +LJ050-0261|from many Government agencies including the Department of Defense and the President's Office of Science and Technology.|from many Government agencies including the Department of Defense and the President's Office of Science and Technology. +LJ050-0262|Even if the manpower and technological resources of the Secret Service are adequately augmented,|Even if the manpower and technological resources of the Secret Service are adequately augmented, +LJ050-0263|it will continue to rely in many respects upon the greater resources of the Office of Science and Technology and other agencies.|it will continue to rely in many respects upon the greater resources of the Office of Science and Technology and other agencies. +LJ050-0264|The Commission recommends that the present arrangements|The Commission recommends that the present arrangements +LJ050-0265|with the Office of Science and Technology and the other Federal agencies that have been so helpful to the Secret Service be placed on a permanent and formal basis.|with the Office of Science and Technology and the other Federal agencies that have been so helpful to the Secret Service be placed on a permanent and formal basis. +LJ050-0266|The exchange of letters dated August 31, 1964,|The exchange of letters dated August thirty-one, nineteen sixty-four, +LJ050-0267|between Secretary Dillon and Donald F. Hornig, Special Assistant to the President for Science and Technology, is a useful effort in the right direction.|between Secretary Dillon and Donald F. Hornig, Special Assistant to the President for Science and Technology, is a useful effort in the right direction. +LJ050-0268|The Service should negotiate a memorandum of understanding with each agency that has been assisting it and from which it can expect to need help in the future.|The Service should negotiate a memorandum of understanding with each agency that has been assisting it and from which it can expect to need help in the future. diff --git a/models/hifi-gan/LJSpeech-1.1/validation.txt b/models/hifi-gan/LJSpeech-1.1/validation.txt new file mode 100644 index 0000000000..a083c34f6b --- /dev/null +++ b/models/hifi-gan/LJSpeech-1.1/validation.txt @@ -0,0 +1,150 @@ +LJ050-0269|The essential terms of such memoranda might well be embodied in an Executive order.|The essential terms of such memoranda might well be embodied in an Executive order. +LJ050-0270|This Commission can recommend no procedures for the future protection of our Presidents which will guarantee security.|This Commission can recommend no procedures for the future protection of our Presidents which will guarantee security. +LJ050-0271|The demands on the President in the execution of His responsibilities in today's world are so varied and complex|The demands on the President in the execution of His responsibilities in today's world are so varied and complex +LJ050-0272|and the traditions of the office in a democracy such as ours are so deep-seated as to preclude absolute security.|and the traditions of the office in a democracy such as ours are so deep-seated as to preclude absolute security. +LJ050-0273|The Commission has, however, from its examination of the facts of President Kennedy's assassination|The Commission has, however, from its examination of the facts of President Kennedy's assassination +LJ050-0274|made certain recommendations which it believes would, if adopted,|made certain recommendations which it believes would, if adopted, +LJ050-0275|materially improve upon the procedures in effect at the time of President Kennedy's assassination and result in a substantial lessening of the danger.|materially improve upon the procedures in effect at the time of President Kennedy's assassination and result in a substantial lessening of the danger. +LJ050-0276|As has been pointed out, the Commission has not resolved all the proposals which could be made. The Commission nevertheless is confident that,|As has been pointed out, the Commission has not resolved all the proposals which could be made. The Commission nevertheless is confident that, +LJ050-0277|with the active cooperation of the responsible agencies and with the understanding of the people of the United States in their demands upon their President,|with the active cooperation of the responsible agencies and with the understanding of the people of the United States in their demands upon their President, +LJ050-0278|the recommendations we have here suggested would greatly advance the security of the office without any impairment of our fundamental liberties.|the recommendations we have here suggested would greatly advance the security of the office without any impairment of our fundamental liberties. +LJ001-0028|but by printers in Strasburg, Basle, Paris, Lubeck, and other cities.|but by printers in Strasburg, Basle, Paris, Lubeck, and other cities. +LJ001-0068|The characteristic Dutch type, as represented by the excellent printer Gerard Leew, is very pronounced and uncompromising Gothic.|The characteristic Dutch type, as represented by the excellent printer Gerard Leew, is very pronounced and uncompromising Gothic. +LJ002-0149|The latter indeed hung like millstones round the neck of the unhappy insolvent wretches who found themselves in limbo.|The latter indeed hung like millstones round the neck of the unhappy insolvent wretches who found themselves in limbo. +LJ002-0157|and Susannah Evans, in October the same year, for 2 shillings, with costs of 6 shillings, 8 pence.|and Susannah Evans, in October the same year, for two shillings, with costs of six shillings, eight pence. +LJ002-0167|quotes a case which came within his own knowledge of a boy sent to prison for non-payment of one penny.|quotes a case which came within his own knowledge of a boy sent to prison for non-payment of one penny. +LJ003-0042|The completion of this very necessary building was, however, much delayed for want of funds,|The completion of this very necessary building was, however, much delayed for want of funds, +LJ003-0307|but as yet no suggestion was made to provide prison uniform.|but as yet no suggestion was made to provide prison uniform. +LJ004-0169|On the dirty bedstead lay a wretched being in the throes of severe illness.|On the dirty bedstead lay a wretched being in the throes of severe illness. +LJ004-0233|Under the new rule visitors were not allowed to pass into the interior of the prison, but were detained between the grating.|Under the new rule visitors were not allowed to pass into the interior of the prison, but were detained between the grating. +LJ005-0101|whence it deduced the practice and condition of every prison that replied.|whence it deduced the practice and condition of every prison that replied. +LJ005-0108|the prisoners, without firing, bedding, or sufficient food, spent their days "in surveying their grotesque prison,|the prisoners, without firing, bedding, or sufficient food, spent their days "in surveying their grotesque prison, +LJ005-0202|An examination of this report shows how even the most insignificant township had its jail.|An examination of this report shows how even the most insignificant township had its jail. +LJ005-0234|The visits of friends was once more unreservedly allowed, and these incomers freely brought in extra provisions and beer.|The visits of friends was once more unreservedly allowed, and these incomers freely brought in extra provisions and beer. +LJ005-0248|and stated that in his opinion Newgate, as the common jail of Middlesex, was wholly inadequate to the proper confinement of its prisoners.|and stated that in his opinion Newgate, as the common jail of Middlesex, was wholly inadequate to the proper confinement of its prisoners. +LJ006-0001|The Chronicles of Newgate, Volume 2. By Arthur Griffiths. Section 9: The first report of the inspector of prisons.|The Chronicles of Newgate, Volume two. By Arthur Griffiths. Section nine: The first report of the inspector of prisons. +LJ006-0018|One was Mr. William Crawford, the other the Rev. Whitworth Russell.|One was Mr. William Crawford, the other the Rev. Whitworth Russell. +LJ006-0034|They attended early and late; they mustered the prisoners, examined into their condition,|They attended early and late; they mustered the prisoners, examined into their condition, +LJ006-0078|A new prisoner's fate, as to location, rested really with a powerful fellow-prisoner.|A new prisoner's fate, as to location, rested really with a powerful fellow-prisoner. +LJ007-0217|They go on to say|They go on to say +LJ007-0243|It was not till the erection of the new prison at Holloway in 1850, and the entire internal reconstruction of Newgate according to new ideas,|It was not till the erection of the new prison at Holloway in eighteen fifty, and the entire internal reconstruction of Newgate according to new ideas, +LJ008-0087|The change from Tyburn to the Old Bailey had worked no improvement as regards the gathering together of the crowd or its demeanor.|The change from Tyburn to the Old Bailey had worked no improvement as regards the gathering together of the crowd or its demeanor. +LJ008-0131|the other he kept between his hands.|the other he kept between his hands. +LJ008-0140|Whenever the public attention had been specially called to a particular crime, either on account of its atrocity,|Whenever the public attention had been specially called to a particular crime, either on account of its atrocity, +LJ008-0158|The pressure soon became so frightful that many would have willingly escaped from the crowd; but their attempts only increased the general confusion.|The pressure soon became so frightful that many would have willingly escaped from the crowd; but their attempts only increased the general confusion. +LJ008-0174|One cart-load of spectators having broken down, some of its occupants fell off the vehicle, and were instantly trampled to death.|One cart-load of spectators having broken down, some of its occupants fell off the vehicle, and were instantly trampled to death. +LJ010-0047|while in 1850 Her Majesty was the victim of another outrage at the hands of one Pate.|while in eighteen fifty Her Majesty was the victim of another outrage at the hands of one Pate. +LJ010-0061|That some thirty or more needy men should hope to revolutionize England is a sufficient proof of the absurdity of their attempt.|That some thirty or more needy men should hope to revolutionize England is a sufficient proof of the absurdity of their attempt. +LJ010-0105|Thistlewood was discovered next morning in a mean house in White Street, Moorfields.|Thistlewood was discovered next morning in a mean house in White Street, Moorfields. +LJ010-0233|Here again probably it was partly the love of notoriety which was the incentive,|Here again probably it was partly the love of notoriety which was the incentive, +LJ010-0234|backed possibly with the hope that, as in a much more recent case,|backed possibly with the hope that, as in a much more recent case, +LJ010-0258|As the Queen was driving from Buckingham Palace to the Chapel Royal,|As the Queen was driving from Buckingham Palace to the Chapel Royal, +LJ010-0262|charged him with the offense.|charged him with the offense. +LJ010-0270|exactly tallied with that of the deformed person "wanted" for the assault on the Queen.|exactly tallied with that of the deformed person "wanted" for the assault on the Queen. +LJ010-0293|I have already remarked that as violence was more and more eliminated from crimes against the person,|I have already remarked that as violence was more and more eliminated from crimes against the person, +LJ011-0009|Nothing more was heard of the affair, although the lady declared that she had never instructed Fauntleroy to sell.|Nothing more was heard of the affair, although the lady declared that she had never instructed Fauntleroy to sell. +LJ011-0256|By this time the neighbors were aroused, and several people came to the scene of the affray.|By this time the neighbors were aroused, and several people came to the scene of the affray. +LJ012-0044|When his trade was busiest he set up a second establishment, at the head of which, although he was married,|When his trade was busiest he set up a second establishment, at the head of which, although he was married, +LJ012-0145|Solomons was now also admitted as a witness, and his evidence, with that of Moss, secured the transportation of the principal actors in the theft.|Solomons was now also admitted as a witness, and his evidence, with that of Moss, secured the transportation of the principal actors in the theft. +LJ013-0020|he acted in a manner which excited the suspicions of the crew.|he acted in a manner which excited the suspicions of the crew. +LJ013-0077|Barber and Fletcher were both transported for life, although Fletcher declared that Barber was innocent, and had no guilty knowledge of what was being done.|Barber and Fletcher were both transported for life, although Fletcher declared that Barber was innocent, and had no guilty knowledge of what was being done. +LJ013-0228|In the pocket of the coat Mr. Cope, the governor, found a neatly-folded cloth, and asked what it was for.|In the pocket of the coat Mr. Cope, the governor, found a neatly-folded cloth, and asked what it was for. +LJ014-0020|He was soon afterwards arrested on suspicion, and a search of his lodgings brought to light several garments saturated with blood;|He was soon afterwards arrested on suspicion, and a search of his lodgings brought to light several garments saturated with blood; +LJ014-0054|a maidservant, Sarah Thomas, murdered her mistress, an aged woman, by beating out her brains with a stone.|a maidservant, Sarah Thomas, murdered her mistress, an aged woman, by beating out her brains with a stone. +LJ014-0101|he found that it was soft and new, while elsewhere it was set and hard.|he found that it was soft and new, while elsewhere it was set and hard. +LJ014-0103|beneath them was a layer of fresh mortar, beneath that a lot of loose earth, amongst which a stocking was turned up, and presently a human toe.|beneath them was a layer of fresh mortar, beneath that a lot of loose earth, amongst which a stocking was turned up, and presently a human toe. +LJ014-0263|When other pleasures palled he took a theatre, and posed as a munificent patron of the dramatic art.|When other pleasures palled he took a theatre, and posed as a munificent patron of the dramatic art. +LJ014-0272|and 1850 to embezzle and apply to his own purposes some £71,000.|and eighteen fifty to embezzle and apply to his own purposes some seventy-one thousand pounds. +LJ014-0311|His extensive business had been carried on by fraud.|His extensive business had been carried on by fraud. +LJ015-0197|which at one time spread terror throughout London. Thieves preferred now to use ingenuity rather than brute force.|which at one time spread terror throughout London. Thieves preferred now to use ingenuity rather than brute force. +LJ016-0089|He was engaged in whitewashing and cleaning; the officer who had him in charge left him on the stairs leading to the gallery.|He was engaged in whitewashing and cleaning; the officer who had him in charge left him on the stairs leading to the gallery. +LJ016-0407|who generally attended the prison services.|who generally attended the prison services. +LJ016-0443|He was promptly rescued from his perilous condition, but not before his face and hands were badly scorched.|He was promptly rescued from his perilous condition, but not before his face and hands were badly scorched. +LJ017-0033|a medical practitioner, charged with doing to death persons who relied upon his professional skill.|a medical practitioner, charged with doing to death persons who relied upon his professional skill. +LJ017-0038|That the administration of justice should never be interfered with by local prejudice or local feeling|That the administration of justice should never be interfered with by local prejudice or local feeling +LJ018-0018|he wore gold-rimmed eye-glasses and a gold watch and chain.|he wore gold-rimmed eye-glasses and a gold watch and chain. +LJ018-0119|His offer was not, however, accepted.|His offer was not, however, accepted. +LJ018-0280|The commercial experience of these clever rogues was cosmopolitan.|The commercial experience of these clever rogues was cosmopolitan. +LJ019-0178|and abandoned because of the expense. As to the entire reconstruction of Newgate, nothing had been done as yet.|and abandoned because of the expense. As to the entire reconstruction of Newgate, nothing had been done as yet. +LJ019-0240|But no structural alterations were made from the date first quoted until the time of closing the prison in 1881.|But no structural alterations were made from the date first quoted until the time of closing the prison in eighteen eighty-one. +LJ021-0049|and the curtailment of rank stock speculation through the Securities Exchange Act.|and the curtailment of rank stock speculation through the Securities Exchange Act. +LJ021-0155|both directly on the public works themselves, and indirectly in the industries supplying the materials for these public works.|both directly on the public works themselves, and indirectly in the industries supplying the materials for these public works. +LJ022-0046|It is true that while business and industry are definitely better our relief rolls are still too large.|It is true that while business and industry are definitely better our relief rolls are still too large. +LJ022-0173|for the regulation of transportation by water, for the strengthening of our Merchant Marine and Air Transport,|for the regulation of transportation by water, for the strengthening of our Merchant Marine and Air Transport, +LJ024-0087|I have thus explained to you the reasons that lie behind our efforts to secure results by legislation within the Constitution.|I have thus explained to you the reasons that lie behind our efforts to secure results by legislation within the Constitution. +LJ024-0110|And the strategy of that last stand is to suggest the time-consuming process of amendment in order to kill off by delay|And the strategy of that last stand is to suggest the time-consuming process of amendment in order to kill off by delay +LJ024-0119|When before have you found them really at your side in your fights for progress?|When before have you found them really at your side in your fights for progress? +LJ025-0091|as it was current among contemporary chemists.|as it was current among contemporary chemists. +LJ026-0029|so in the case under discussion.|so in the case under discussion. +LJ026-0039|the earliest organisms were protists and that from them animals and plants were evolved along divergent lines of descent.|the earliest organisms were protists and that from them animals and plants were evolved along divergent lines of descent. +LJ026-0064|but unlike that of the animal, it is not chiefly an income of foods, but only of the raw materials of food.|but unlike that of the animal, it is not chiefly an income of foods, but only of the raw materials of food. +LJ026-0105|This is done by diastase, an enzyme of plant cells.|This is done by diastase, an enzyme of plant cells. +LJ026-0137|and be laid down as "reserve starch" in the cells of root or stem or elsewhere.|and be laid down as "reserve starch" in the cells of root or stem or elsewhere. +LJ027-0006|In all these lines the facts are drawn together by a strong thread of unity.|In all these lines the facts are drawn together by a strong thread of unity. +LJ028-0134|He also erected what is called a pensile paradise:|He also erected what is called a pensile paradise: +LJ028-0138|perhaps the tales that travelers told him were exaggerated as travelers' tales are likely to be,|perhaps the tales that travelers told him were exaggerated as travelers' tales are likely to be, +LJ028-0189|The fall of Babylon with its lofty walls was a most important event in the history of the ancient world.|The fall of Babylon with its lofty walls was a most important event in the history of the ancient world. +LJ028-0281|Till mules foal ye shall not take our city, he thought, as he reflected on this speech, that Babylon might now be taken,|Till mules foal ye shall not take our city, he thought, as he reflected on this speech, that Babylon might now be taken, +LJ029-0188|Stevenson was jeered, jostled, and spat upon by hostile demonstrators outside the Dallas Memorial Auditorium Theater.|Stevenson was jeered, jostled, and spat upon by hostile demonstrators outside the Dallas Memorial Auditorium Theater. +LJ030-0098|The remainder of the motorcade consisted of five cars for other dignitaries, including the mayor of Dallas and Texas Congressmen,|The remainder of the motorcade consisted of five cars for other dignitaries, including the mayor of Dallas and Texas Congressmen, +LJ031-0007|Chief of Police Curry and police motorcyclists at the head of the motorcade led the way to the hospital.|Chief of Police Curry and police motorcyclists at the head of the motorcade led the way to the hospital. +LJ031-0091|You have to determine which things, which are immediately life threatening and cope with them, before attempting to evaluate the full extent of the injuries.|You have to determine which things, which are immediately life threatening and cope with them, before attempting to evaluate the full extent of the injuries. +LJ031-0227|The doctors traced the course of the bullet through the body and, as information was received from Parkland Hospital,|The doctors traced the course of the bullet through the body and, as information was received from Parkland Hospital, +LJ032-0100|Marina Oswald|Marina Oswald +LJ032-0165|to the exclusion of all others because there are not enough microscopic characteristics present in fibers.|to the exclusion of all others because there are not enough microscopic characteristics present in fibers. +LJ032-0198|During the period from March 2, 1963, to April 24, 1963,|During the period from March two, nineteen sixty-three, to April twenty-four, nineteen sixty-three, +LJ033-0046|went out to the garage to paint some children's blocks, and worked in the garage for half an hour or so.|went out to the garage to paint some children's blocks, and worked in the garage for half an hour or so. +LJ033-0072|I then stepped off of it and the officer picked it up in the middle and it bent so.|I then stepped off of it and the officer picked it up in the middle and it bent so. +LJ033-0135|Location of Bag|Location of Bag +LJ034-0083|The significance of Givens' observation that Oswald was carrying his clipboard|The significance of Givens' observation that Oswald was carrying his clipboard +LJ034-0179|and, quote, seemed to be sitting a little forward, end quote,|and, quote, seemed to be sitting a little forward, end quote, +LJ035-0125|Victoria Adams, who worked on the fourth floor of the Depository Building,|Victoria Adams, who worked on the fourth floor of the Depository Building, +LJ035-0162|approximately 30 to 45 seconds after Oswald's lunchroom encounter with Baker and Truly.|approximately thirty to forty-five seconds after Oswald's lunchroom encounter with Baker and Truly. +LJ035-0189|Special Agent Forrest V. Sorrels of the Secret Service, who had been in the motorcade,|Special Agent Forrest V. Sorrels of the Secret Service, who had been in the motorcade, +LJ035-0208|Oswald's known actions in the building immediately after the assassination are consistent with his having been at the southeast corner window of the sixth floor|Oswald's known actions in the building immediately after the assassination are consistent with his having been at the southeast corner window of the sixth floor +LJ036-0216|Tippit got out and started to walk around the front of the car|Tippit got out and started to walk around the front of the car +LJ037-0093|William Arthur Smith was about a block east of 10th and Patton when he heard shots.|William Arthur Smith was about a block east of tenth and Patton when he heard shots. +LJ037-0157|taken from Oswald.|taken from Oswald. +LJ037-0178|or one used Remington-Peters cartridge case, which may have been in the revolver before the shooting,|or one used Remington-Peters cartridge case, which may have been in the revolver before the shooting, +LJ037-0219|Oswald's Jacket|Oswald's Jacket +LJ037-0222|When Oswald was arrested, he did not have a jacket.|When Oswald was arrested, he did not have a jacket. +LJ038-0017|Attracted by the sound of the sirens, Mrs. Postal stepped out of the box office and walked to the curb.|Attracted by the sound of the sirens, Mrs. Postal stepped out of the box office and walked to the curb. +LJ038-0052|testified regarding the arrest of Oswald, as did the various police officers who participated in the fight.|testified regarding the arrest of Oswald, as did the various police officers who participated in the fight. +LJ038-0077|Statements of Oswald during Detention.|Statements of Oswald during Detention. +LJ038-0161|and he asked me did I know which way he was coming, and I told him, yes, he probably come down Main and turn on Houston and then back again on Elm.|and he asked me did I know which way he was coming, and I told him, yes, he probably come down Main and turn on Houston and then back again on Elm. +LJ038-0212|which appeared to be the work of a man expecting to be killed, or imprisoned, or to disappear.|which appeared to be the work of a man expecting to be killed, or imprisoned, or to disappear. +LJ039-0103|Oswald, like all Marine recruits, received training on the rifle range at distances up to 500 yards,|Oswald, like all Marine recruits, received training on the rifle range at distances up to five hundred yards, +LJ039-0149|established that they had been previously loaded and ejected from the assassination rifle,|established that they had been previously loaded and ejected from the assassination rifle, +LJ040-0107|but apparently was not able to spend as much time with them as he would have liked, because of the age gaps of 5 and 7 years,|but apparently was not able to spend as much time with them as he would have liked, because of the age gaps of five and seven years, +LJ040-0119|When Pic returned home, Mrs. Oswald tried to play down the event but Mrs. Pic took a different view and asked the Oswalds to leave.|When Pic returned home, Mrs. Oswald tried to play down the event but Mrs. Pic took a different view and asked the Oswalds to leave. +LJ040-0161|Dr. Hartogs recommended that Oswald be placed on probation on condition that he seek help and guidance through a child guidance clinic.|Dr. Hartogs recommended that Oswald be placed on probation on condition that he seek help and guidance through a child guidance clinic. +LJ040-0169|She observed that since Lee's mother worked all day, he made his own meals and spent all his time alone|She observed that since Lee's mother worked all day, he made his own meals and spent all his time alone +LJ041-0098|All the Marine Corps did was to teach you to kill and after you got out of the Marines you might be good gangsters, end quote.|All the Marine Corps did was to teach you to kill and after you got out of the Marines you might be good gangsters, end quote. +LJ042-0017|and see for himself how a revolutionary society operates, a Marxist society.|and see for himself how a revolutionary society operates, a Marxist society. +LJ042-0070|Oswald was discovered in time to thwart his attempt at suicide.|Oswald was discovered in time to thwart his attempt at suicide. +LJ042-0161|Immediately after serving out his 3 years in the U.S. Marine Corps, he abandoned his American life to seek a new life in the USSR.|Immediately after serving out his three years in the U.S. Marine Corps, he abandoned his American life to seek a new life in the USSR. +LJ043-0147|He had left a note for his wife telling her what to do in case he were apprehended, as well as his notebook and the pictures of himself holding the rifle.|He had left a note for his wife telling her what to do in case he were apprehended, as well as his notebook and the pictures of himself holding the rifle. +LJ043-0178|as, in fact, one of them did appear after the assassination.|as, in fact, one of them did appear after the assassination. +LJ043-0183|Oswald did not lack the determination and other traits required|Oswald did not lack the determination and other traits required +LJ043-0185|Some idea of what he thought was sufficient reason for such an act may be found in the nature of the motive that he stated for his attack on General Walker.|Some idea of what he thought was sufficient reason for such an act may be found in the nature of the motive that he stated for his attack on General Walker. +LJ044-0057|extensive investigation was not able to connect Oswald with that address, although it did develop the fact|extensive investigation was not able to connect Oswald with that address, although it did develop the fact +LJ044-0109|It is good to know that movements in support of fair play for Cuba has developed in New Orleans as well as in other cities.|It is good to know that movements in support of fair play for Cuba has developed in New Orleans as well as in other cities. +LJ045-0081|Although she denied it in some of her testimony before the Commission,|Although she denied it in some of her testimony before the Commission, +LJ045-0147|She asked Oswald, quote,|She asked Oswald, quote, +LJ045-0204|he had never found anything to which he felt he could really belong.|he had never found anything to which he felt he could really belong. +LJ046-0193|and 12 to 15 of these cases as highly dangerous risks.|and twelve to fifteen of these cases as highly dangerous risks. +LJ046-0244|PRS should have investigated and been prepared to guard against it.|PRS should have investigated and been prepared to guard against it. +LJ047-0059|However, pursuant to a regular Bureau practice of interviewing certain immigrants from Iron Curtain countries,|However, pursuant to a regular Bureau practice of interviewing certain immigrants from Iron Curtain countries, +LJ047-0142|The Bureau had no earlier information suggesting that Oswald had left the United States.|The Bureau had no earlier information suggesting that Oswald had left the United States. +LJ048-0035|It was against this background and consistent with the criteria followed by the FBI prior to November 22|It was against this background and consistent with the criteria followed by the FBI prior to November twenty-two +LJ048-0063|The formal FBI instructions to its agents outlining the information to be referred to the Secret Service were too narrow at the time of the assassination.|The formal FBI instructions to its agents outlining the information to be referred to the Secret Service were too narrow at the time of the assassination. +LJ048-0104|There were far safer routes via freeways directly to the Trade Mart,|There were far safer routes via freeways directly to the Trade Mart, +LJ048-0187|In addition, Secret Service agents riding in the motorcade were trained to scan buildings as part of their general observation of the crowd of spectators.|In addition, Secret Service agents riding in the motorcade were trained to scan buildings as part of their general observation of the crowd of spectators. +LJ048-0271|will be cause for removal from the Service, end quote.|will be cause for removal from the Service, end quote. +LJ049-0031|The Presidential vehicle in use in Dallas, described in chapter 2,|The Presidential vehicle in use in Dallas, described in chapter two, +LJ049-0059|Agents are instructed that it is not their responsibility to investigate or evaluate a present danger,|Agents are instructed that it is not their responsibility to investigate or evaluate a present danger, +LJ049-0174|to notify the Secret Service of the substantial information about Lee Harvey Oswald which the FBI had accumulated|to notify the Secret Service of the substantial information about Lee Harvey Oswald which the FBI had accumulated +LJ050-0049|and from a specialist in psychiatric prognostication at Walter Reed Hospital.|and from a specialist in psychiatric prognostication at Walter Reed Hospital. +LJ050-0113|Such agreements should describe in detail the information which is sought, the manner in which it will be provided to the Secret Service,|Such agreements should describe in detail the information which is sought, the manner in which it will be provided to the Secret Service, +LJ050-0150|Its present manual filing system is obsolete;|Its present manual filing system is obsolete; +LJ050-0189|that written instructions might come into the hands of local newspapers, to the prejudice of the precautions described.|that written instructions might come into the hands of local newspapers, to the prejudice of the precautions described. diff --git a/models/hifi-gan/README.md b/models/hifi-gan/README.md new file mode 100644 index 0000000000..4c63381f7c --- /dev/null +++ b/models/hifi-gan/README.md @@ -0,0 +1,105 @@ +# HiFi-GAN: Generative Adversarial Networks for Efficient and High Fidelity Speech Synthesis + +### Jungil Kong, Jaehyeon Kim, Jaekyoung Bae + +In our [paper](https://arxiv.org/abs/2010.05646), +we proposed HiFi-GAN: a GAN-based model capable of generating high fidelity speech efficiently.
+We provide our implementation and pretrained models as open source in this repository. + +**Abstract :** +Several recent work on speech synthesis have employed generative adversarial networks (GANs) to produce raw waveforms. +Although such methods improve the sampling efficiency and memory usage, +their sample quality has not yet reached that of autoregressive and flow-based generative models. +In this work, we propose HiFi-GAN, which achieves both efficient and high-fidelity speech synthesis. +As speech audio consists of sinusoidal signals with various periods, +we demonstrate that modeling periodic patterns of an audio is crucial for enhancing sample quality. +A subjective human evaluation (mean opinion score, MOS) of a single speaker dataset indicates that our proposed method +demonstrates similarity to human quality while generating 22.05 kHz high-fidelity audio 167.9 times faster than +real-time on a single V100 GPU. We further show the generality of HiFi-GAN to the mel-spectrogram inversion of unseen +speakers and end-to-end speech synthesis. Finally, a small footprint version of HiFi-GAN generates samples 13.4 times +faster than real-time on CPU with comparable quality to an autoregressive counterpart. + +Visit our [demo website](https://jik876.github.io/hifi-gan-demo/) for audio samples. + + +## Pre-requisites +1. Python >= 3.6 +2. Clone this repository. +3. Install python requirements. Please refer [requirements.txt](requirements.txt) +4. Download and extract the [LJ Speech dataset](https://keithito.com/LJ-Speech-Dataset/). +And move all wav files to `LJSpeech-1.1/wavs` + + +## Training +``` +python train.py --config config_v1.json +``` +To train V2 or V3 Generator, replace `config_v1.json` with `config_v2.json` or `config_v3.json`.
+Checkpoints and copy of the configuration file are saved in `cp_hifigan` directory by default.
+You can change the path by adding `--checkpoint_path` option. + +Validation loss during training with V1 generator.
+![validation loss](./validation_loss.png) + +## Pretrained Model +You can also use pretrained models we provide.
+[Download pretrained models](https://drive.google.com/drive/folders/1-eEYTB5Av9jNql0WGBlRoi-WH2J7bp5Y?usp=sharing)
+Details of each folder are as in follows: + +|Folder Name|Generator|Dataset|Fine-Tuned| +|------|---|---|---| +|LJ_V1|V1|LJSpeech|No| +|LJ_V2|V2|LJSpeech|No| +|LJ_V3|V3|LJSpeech|No| +|LJ_FT_T2_V1|V1|LJSpeech|Yes ([Tacotron2](https://github.com/NVIDIA/tacotron2))| +|LJ_FT_T2_V2|V2|LJSpeech|Yes ([Tacotron2](https://github.com/NVIDIA/tacotron2))| +|LJ_FT_T2_V3|V3|LJSpeech|Yes ([Tacotron2](https://github.com/NVIDIA/tacotron2))| +|VCTK_V1|V1|VCTK|No| +|VCTK_V2|V2|VCTK|No| +|VCTK_V3|V3|VCTK|No| +|UNIVERSAL_V1|V1|Universal|No| + +We provide the universal model with discriminator weights that can be used as a base for transfer learning to other datasets. + +## Fine-Tuning +1. Generate mel-spectrograms in numpy format using [Tacotron2](https://github.com/NVIDIA/tacotron2) with teacher-forcing.
+The file name of the generated mel-spectrogram should match the audio file and the extension should be `.npy`.
+Example: + ``` + Audio File : LJ001-0001.wav + Mel-Spectrogram File : LJ001-0001.npy + ``` +2. Create `ft_dataset` folder and copy the generated mel-spectrogram files into it.
+3. Run the following command. + ``` + python train.py --fine_tuning True --config config_v1.json + ``` + For other command line options, please refer to the training section. + + +## Inference from wav file +1. Make `test_files` directory and copy wav files into the directory. +2. Run the following command. + ``` + python inference.py --checkpoint_file [generator checkpoint file path] + ``` +Generated wav files are saved in `generated_files` by default.
+You can change the path by adding `--output_dir` option. + + +## Inference for end-to-end speech synthesis +1. Make `test_mel_files` directory and copy generated mel-spectrogram files into the directory.
+You can generate mel-spectrograms using [Tacotron2](https://github.com/NVIDIA/tacotron2), +[Glow-TTS](https://github.com/jaywalnut310/glow-tts) and so forth. +2. Run the following command. + ``` + python inference_e2e.py --checkpoint_file [generator checkpoint file path] + ``` +Generated wav files are saved in `generated_files_from_mel` by default.
+You can change the path by adding `--output_dir` option. + + +## Acknowledgements +We referred to [WaveGlow](https://github.com/NVIDIA/waveglow), [MelGAN](https://github.com/descriptinc/melgan-neurips) +and [Tacotron2](https://github.com/NVIDIA/tacotron2) to implement this. + diff --git a/models/hifi-gan/audio_processing.py b/models/hifi-gan/audio_processing.py new file mode 100644 index 0000000000..b5af7f723e --- /dev/null +++ b/models/hifi-gan/audio_processing.py @@ -0,0 +1,93 @@ +import torch +import numpy as np +from scipy.signal import get_window +import librosa.util as librosa_util + + +def window_sumsquare(window, n_frames, hop_length=200, win_length=800, + n_fft=800, dtype=np.float32, norm=None): + """ + # from librosa 0.6 + Compute the sum-square envelope of a window function at a given hop length. + + This is used to estimate modulation effects induced by windowing + observations in short-time fourier transforms. + + Parameters + ---------- + window : string, tuple, number, callable, or list-like + Window specification, as in `get_window` + + n_frames : int > 0 + The number of analysis frames + + hop_length : int > 0 + The number of samples to advance between frames + + win_length : [optional] + The length of the window function. By default, this matches `n_fft`. + + n_fft : int > 0 + The length of each analysis frame. + + dtype : np.dtype + The data type of the output + + Returns + ------- + wss : np.ndarray, shape=`(n_fft + hop_length * (n_frames - 1))` + The sum-squared envelope of the window function + """ + if win_length is None: + win_length = n_fft + + n = n_fft + hop_length * (n_frames - 1) + x = np.zeros(n, dtype=dtype) + + # Compute the squared window at the desired length + win_sq = get_window(window, win_length, fftbins=True) + win_sq = librosa_util.normalize(win_sq, norm=norm)**2 + win_sq = librosa_util.pad_center(win_sq, n_fft) + + # Fill the envelope + for i in range(n_frames): + sample = i * hop_length + x[sample:min(n, sample + n_fft)] += win_sq[:max(0, min(n_fft, n - sample))] + return x + + +def griffin_lim(magnitudes, stft_fn, n_iters=30): + """ + PARAMS + ------ + magnitudes: spectrogram magnitudes + stft_fn: STFT class with transform (STFT) and inverse (ISTFT) methods + """ + + angles = np.angle(np.exp(2j * np.pi * np.random.rand(*magnitudes.size()))) + angles = angles.astype(np.float32) + angles = torch.autograd.Variable(torch.from_numpy(angles)) + signal = stft_fn.inverse(magnitudes, angles).squeeze(1) + + for i in range(n_iters): + _, angles = stft_fn.transform(signal) + signal = stft_fn.inverse(magnitudes, angles).squeeze(1) + return signal + + +def dynamic_range_compression(x, C=1, clip_val=1e-5): + """ + PARAMS + ------ + C: compression factor + """ + return torch.log(torch.clamp(x, min=clip_val) * C) + + +def dynamic_range_decompression(x, C=1): + """ + PARAMS + ------ + C: compression factor used to compress + """ + return torch.exp(x) / C diff --git a/models/hifi-gan/config_32k.json b/models/hifi-gan/config_32k.json new file mode 100644 index 0000000000..0e27f108f3 --- /dev/null +++ b/models/hifi-gan/config_32k.json @@ -0,0 +1,37 @@ +{ + "resblock": "1", + "num_gpus": 0, + "batch_size": 16, + "learning_rate": 0.0002, + "adam_b1": 0.8, + "adam_b2": 0.99, + "lr_decay": 0.999, + "seed": 1234, + + "upsample_rates": [8,8,4,2], + "upsample_kernel_sizes": [16,16,8,4], + "upsample_initial_channel": 256, + "resblock_kernel_sizes": [3,7,11], + "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], + + "segment_size": 16384, + "num_mels": 80, + "num_freq": 1025, + "n_fft": 2048, + "hop_size": 512, + "win_size": 2048, + + "sampling_rate": 32000, + + "fmin": 0, + "fmax": 11025, + "fmax_for_loss": null, + + "num_workers": 4, + + "dist_config": { + "dist_backend": "nccl", + "dist_url": "tcp://localhost:54321", + "world_size": 1 + } +} diff --git a/models/hifi-gan/config_v1.json b/models/hifi-gan/config_v1.json new file mode 100644 index 0000000000..ce97366f7d --- /dev/null +++ b/models/hifi-gan/config_v1.json @@ -0,0 +1,37 @@ +{ + "resblock": "1", + "num_gpus": 0, + "batch_size": 16, + "learning_rate": 0.0002, + "adam_b1": 0.8, + "adam_b2": 0.99, + "lr_decay": 0.999, + "seed": 1234, + + "upsample_rates": [8,8,2,2], + "upsample_kernel_sizes": [16,16,4,4], + "upsample_initial_channel": 512, + "resblock_kernel_sizes": [3,7,11], + "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], + + "segment_size": 8192, + "num_mels": 80, + "num_freq": 1025, + "n_fft": 1024, + "hop_size": 256, + "win_size": 1024, + + "sampling_rate": 22050, + + "fmin": 0, + "fmax": 8000, + "fmax_for_loss": null, + + "num_workers": 4, + + "dist_config": { + "dist_backend": "nccl", + "dist_url": "tcp://localhost:54321", + "world_size": 1 + } +} diff --git a/models/hifi-gan/config_v1b.json b/models/hifi-gan/config_v1b.json new file mode 100644 index 0000000000..11353b6a47 --- /dev/null +++ b/models/hifi-gan/config_v1b.json @@ -0,0 +1,37 @@ +{ + "resblock": "1", + "num_gpus": 0, + "batch_size": 16, + "learning_rate": 0.00003, + "adam_b1": 0.8, + "adam_b2": 0.99, + "lr_decay": 0.97, + "seed": 1234, + + "upsample_rates": [8,8,2,2], + "upsample_kernel_sizes": [16,16,4,4], + "upsample_initial_channel": 512, + "resblock_kernel_sizes": [3,7,11], + "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], + + "segment_size": 8192, + "num_mels": 80, + "num_freq": 1025, + "n_fft": 1024, + "hop_size": 256, + "win_size": 1024, + + "sampling_rate": 22050, + + "fmin": 0, + "fmax": 8000, + "fmax_for_loss": null, + + "num_workers": 4, + + "dist_config": { + "dist_backend": "nccl", + "dist_url": "tcp://localhost:54321", + "world_size": 1 + } +} diff --git a/models/hifi-gan/config_v2.json b/models/hifi-gan/config_v2.json new file mode 100644 index 0000000000..8d2ee486da --- /dev/null +++ b/models/hifi-gan/config_v2.json @@ -0,0 +1,37 @@ +{ + "resblock": "1", + "num_gpus": 0, + "batch_size": 16, + "learning_rate": 0.0002, + "adam_b1": 0.8, + "adam_b2": 0.99, + "lr_decay": 0.999, + "seed": 1234, + + "upsample_rates": [8,8,2,2], + "upsample_kernel_sizes": [16,16,4,4], + "upsample_initial_channel": 128, + "resblock_kernel_sizes": [3,7,11], + "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], + + "segment_size": 8192, + "num_mels": 80, + "num_freq": 1025, + "n_fft": 1024, + "hop_size": 256, + "win_size": 1024, + + "sampling_rate": 22050, + + "fmin": 0, + "fmax": 8000, + "fmax_for_loss": null, + + "num_workers": 4, + + "dist_config": { + "dist_backend": "nccl", + "dist_url": "tcp://localhost:54321", + "world_size": 1 + } +} diff --git a/models/hifi-gan/config_v3.json b/models/hifi-gan/config_v3.json new file mode 100644 index 0000000000..2e79c793ad --- /dev/null +++ b/models/hifi-gan/config_v3.json @@ -0,0 +1,37 @@ +{ + "resblock": "2", + "num_gpus": 0, + "batch_size": 16, + "learning_rate": 0.0002, + "adam_b1": 0.8, + "adam_b2": 0.99, + "lr_decay": 0.999, + "seed": 1234, + + "upsample_rates": [8,8,4], + "upsample_kernel_sizes": [16,16,8], + "upsample_initial_channel": 256, + "resblock_kernel_sizes": [3,5,7], + "resblock_dilation_sizes": [[1,2], [2,6], [3,12]], + + "segment_size": 8192, + "num_mels": 80, + "num_freq": 1025, + "n_fft": 1024, + "hop_size": 256, + "win_size": 1024, + + "sampling_rate": 22050, + + "fmin": 0, + "fmax": 8000, + "fmax_for_loss": null, + + "num_workers": 4, + + "dist_config": { + "dist_backend": "nccl", + "dist_url": "tcp://localhost:54321", + "world_size": 1 + } +} diff --git a/models/hifi-gan/denoiser.py b/models/hifi-gan/denoiser.py new file mode 100644 index 0000000000..4d21d104ce --- /dev/null +++ b/models/hifi-gan/denoiser.py @@ -0,0 +1,36 @@ +import sys +import torch +from stft import STFT + + +class Denoiser(torch.nn.Module): + """ WaveGlow denoiser, adapted for HiFi-GAN """ + + def __init__( + self, hifigan, filter_length=1024, n_overlap=4, win_length=1024, mode="zeros" + ): + super(Denoiser, self).__init__() + self.stft = STFT( + filter_length=filter_length, + hop_length=int(filter_length / n_overlap), + win_length=win_length, + ).cuda() + if mode == "zeros": + mel_input = torch.zeros((1, 80, 88)).cuda() + elif mode == "normal": + mel_input = torch.randn((1, 80, 88)).cuda() + else: + raise Exception("Mode {} if not supported".format(mode)) + + with torch.no_grad(): + bias_audio = hifigan(mel_input).view(1, -1).float() + bias_spec, _ = self.stft.transform(bias_audio) + + self.register_buffer("bias_spec", bias_spec[:, :, 0][:, :, None]) + + def forward(self, audio, strength=0.1): + audio_spec, audio_angles = self.stft.transform(audio.cuda().float()) + audio_spec_denoised = audio_spec - self.bias_spec * strength + audio_spec_denoised = torch.clamp(audio_spec_denoised, 0.0) + audio_denoised = self.stft.inverse(audio_spec_denoised, audio_angles) + return audio_denoised diff --git a/models/hifi-gan/env.py b/models/hifi-gan/env.py new file mode 100644 index 0000000000..2bdbc95d4f --- /dev/null +++ b/models/hifi-gan/env.py @@ -0,0 +1,15 @@ +import os +import shutil + + +class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + +def build_env(config, config_name, path): + t_path = os.path.join(path, config_name) + if config != t_path: + os.makedirs(path, exist_ok=True) + shutil.copyfile(config, os.path.join(path, config_name)) diff --git a/models/hifi-gan/hifiutils.py b/models/hifi-gan/hifiutils.py new file mode 100644 index 0000000000..aa2a536e62 --- /dev/null +++ b/models/hifi-gan/hifiutils.py @@ -0,0 +1,58 @@ +import glob +import os +import matplotlib +import torch +from torch.nn.utils import weight_norm +matplotlib.use("Agg") +import matplotlib.pylab as plt + + +def plot_spectrogram(spectrogram): + fig, ax = plt.subplots(figsize=(10, 2)) + im = ax.imshow(spectrogram, aspect="auto", origin="lower", + interpolation='none') + plt.colorbar(im, ax=ax) + + fig.canvas.draw() + plt.close() + + return fig + + +def init_weights(m, mean=0.0, std=0.01): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + m.weight.data.normal_(mean, std) + + +def apply_weight_norm(m): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + weight_norm(m) + + +def get_padding(kernel_size, dilation=1): + return int((kernel_size*dilation - dilation)/2) + + +def load_checkpoint(filepath, device): + assert os.path.isfile(filepath) + print("Loading '{}'".format(filepath)) + checkpoint_dict = torch.load(filepath, map_location=device) + print("Complete.") + return checkpoint_dict + + +def save_checkpoint(filepath, obj): + print("Saving checkpoint to {}".format(filepath)) + torch.save(obj, filepath) + print("Complete.") + + +def scan_checkpoint(cp_dir, prefix): + pattern = os.path.join(cp_dir, prefix + '????????') + cp_list = glob.glob(pattern) + if len(cp_list) == 0: + return None + return sorted(cp_list)[-1] + diff --git a/models/hifi-gan/inference.py b/models/hifi-gan/inference.py new file mode 100644 index 0000000000..96ba10672f --- /dev/null +++ b/models/hifi-gan/inference.py @@ -0,0 +1,95 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import glob +import os +import argparse +import json +import torch +from scipy.io.wavfile import write +from env import AttrDict +from meldataset import mel_spectrogram, MAX_WAV_VALUE, load_wav +from models import Generator + +h = None +device = None + + +def load_checkpoint(filepath, device): + assert os.path.isfile(filepath) + print("Loading '{}'".format(filepath)) + checkpoint_dict = torch.load(filepath, map_location=device) + print("Complete.") + return checkpoint_dict + + +def get_mel(x): + return mel_spectrogram(x, h.n_fft, h.num_mels, h.sampling_rate, h.hop_size, h.win_size, h.fmin, h.fmax) + + +def scan_checkpoint(cp_dir, prefix): + pattern = os.path.join(cp_dir, prefix + '*') + cp_list = glob.glob(pattern) + if len(cp_list) == 0: + return '' + return sorted(cp_list)[-1] + + +def inference(a): + generator = Generator(h).to(device) + + state_dict_g = load_checkpoint(a.checkpoint_file, device) + generator.load_state_dict(state_dict_g['generator']) + + filelist = os.listdir(a.input_wavs_dir) + + os.makedirs(a.output_dir, exist_ok=True) + + generator.eval() + generator.remove_weight_norm() + with torch.no_grad(): + for i, filname in enumerate(filelist): + wav, sr = load_wav(os.path.join(a.input_wavs_dir, filname)) + wav = wav / MAX_WAV_VALUE + wav = torch.FloatTensor(wav).to(device) + x = get_mel(wav.unsqueeze(0)) + y_g_hat = generator(x) + audio = y_g_hat.squeeze() + audio = audio * MAX_WAV_VALUE + audio = audio.cpu().numpy().astype('int16') + + output_file = os.path.join(a.output_dir, os.path.splitext(filname)[0] + '_generated.wav') + write(output_file, h.sampling_rate, audio) + print(output_file) + + +def main(): + print('Initializing Inference Process..') + + parser = argparse.ArgumentParser() + parser.add_argument('--input_wavs_dir', default='test_files') + parser.add_argument('--output_dir', default='generated_files') + parser.add_argument('--checkpoint_file', required=True) + a = parser.parse_args() + + config_file = os.path.join(os.path.split(a.checkpoint_file)[0], 'config.json') + with open(config_file) as f: + data = f.read() + + global h + json_config = json.loads(data) + h = AttrDict(json_config) + + torch.manual_seed(h.seed) + global device + if torch.cuda.is_available(): + torch.cuda.manual_seed(h.seed) + device = torch.device('cuda') + else: + device = torch.device('cpu') + + inference(a) + + +if __name__ == '__main__': + main() + diff --git a/models/hifi-gan/inference_e2e.py b/models/hifi-gan/inference_e2e.py new file mode 100644 index 0000000000..8ef07d0f08 --- /dev/null +++ b/models/hifi-gan/inference_e2e.py @@ -0,0 +1,93 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import glob +import os +import numpy as np +import argparse +import json +import torch +from scipy.io.wavfile import write +from env import AttrDict +from meldataset import MAX_WAV_VALUE +from models import Generator + +h = None +device = None + + +def load_checkpoint(filepath, device): + assert os.path.isfile(filepath) + print("Loading '{}'".format(filepath)) + checkpoint_dict = torch.load(filepath, map_location=device) + print("Complete.") + return checkpoint_dict + + +def scan_checkpoint(cp_dir, prefix): + pattern = os.path.join(cp_dir, prefix + "*") + cp_list = glob.glob(pattern) + if len(cp_list) == 0: + return "" + return sorted(cp_list)[-1] + + +def inference(a): + generator = Generator(h).to(device) + + state_dict_g = load_checkpoint(a.checkpoint_file, device) + generator.load_state_dict(state_dict_g["generator"]) + + filelist = os.listdir(a.input_mels_dir) + + os.makedirs(a.output_dir, exist_ok=True) + + generator.eval() + generator.remove_weight_norm() + with torch.no_grad(): + for i, filname in enumerate(filelist): + if ".npy" not in filname: + continue + x = np.load(os.path.join(a.input_mels_dir, filname)) + x = torch.FloatTensor(x).to(device) + y_g_hat = generator(x) + audio = y_g_hat.squeeze() + audio = audio * MAX_WAV_VALUE + audio = audio.cpu().numpy().astype("int16") + + output_file = os.path.join( + a.output_dir, os.path.splitext(filname)[0] + ".wav" + ) + write(output_file, h.sampling_rate, audio) + print(output_file) + + +def main(): + print("Initializing Inference Process..") + + parser = argparse.ArgumentParser() + parser.add_argument("--input_mels_dir", default="test_mel_files") + parser.add_argument("--output_dir", default="generated_files_from_mel") + parser.add_argument("--checkpoint_file", required=True) + a = parser.parse_args() + + config_file = os.path.join(os.path.split(a.checkpoint_file)[0], "config.json") + with open(config_file) as f: + data = f.read() + + global h + json_config = json.loads(data) + h = AttrDict(json_config) + + torch.manual_seed(h.seed) + global device + if torch.cuda.is_available(): + torch.cuda.manual_seed(h.seed) + device = torch.device("cuda") + else: + device = torch.device("cpu") + + inference(a) + + +if __name__ == "__main__": + main() diff --git a/models/hifi-gan/meldataset.py b/models/hifi-gan/meldataset.py new file mode 100644 index 0000000000..44b0bf45aa --- /dev/null +++ b/models/hifi-gan/meldataset.py @@ -0,0 +1,168 @@ +import math +import os +import random +import torch +import torch.utils.data +import numpy as np +from librosa.util import normalize +from scipy.io.wavfile import read +from librosa.filters import mel as librosa_mel_fn + +MAX_WAV_VALUE = 32768.0 + + +def load_wav(full_path): + sampling_rate, data = read(full_path) + return data, sampling_rate + + +def dynamic_range_compression(x, C=1, clip_val=1e-5): + return np.log(np.clip(x, a_min=clip_val, a_max=None) * C) + + +def dynamic_range_decompression(x, C=1): + return np.exp(x) / C + + +def dynamic_range_compression_torch(x, C=1, clip_val=1e-5): + return torch.log(torch.clamp(x, min=clip_val) * C) + + +def dynamic_range_decompression_torch(x, C=1): + return torch.exp(x) / C + + +def spectral_normalize_torch(magnitudes): + output = dynamic_range_compression_torch(magnitudes) + return output + + +def spectral_de_normalize_torch(magnitudes): + output = dynamic_range_decompression_torch(magnitudes) + return output + + +mel_basis = {} +hann_window = {} + + +def mel_spectrogram(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False): + if torch.min(y) < -1.: + print('min value is ', torch.min(y)) + if torch.max(y) > 1.: + print('max value is ', torch.max(y)) + + global mel_basis, hann_window + if fmax not in mel_basis: + mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax) + mel_basis[str(fmax)+'_'+str(y.device)] = torch.from_numpy(mel).float().to(y.device) + hann_window[str(y.device)] = torch.hann_window(win_size).to(y.device) + + y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect') + y = y.squeeze(1) + + spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[str(y.device)], + center=center, pad_mode='reflect', normalized=False, onesided=True) + + spec = torch.sqrt(spec.pow(2).sum(-1)+(1e-9)) + + spec = torch.matmul(mel_basis[str(fmax)+'_'+str(y.device)], spec) + spec = spectral_normalize_torch(spec) + + return spec + + +def get_dataset_filelist(a): + with open(a.input_training_file, 'r', encoding='utf-8') as fi: + training_files = [os.path.join(a.input_wavs_dir, x.split('|')[0]) + for x in fi.read().split('\n') if len(x) > 0] + + with open(a.input_validation_file, 'r', encoding='utf-8') as fi: + validation_files = [os.path.join(a.input_wavs_dir, x.split('|')[0]) + for x in fi.read().split('\n') if len(x) > 0] + return training_files, validation_files + + +class MelDataset(torch.utils.data.Dataset): + def __init__(self, training_files, segment_size, n_fft, num_mels, + hop_size, win_size, sampling_rate, fmin, fmax, split=True, shuffle=True, n_cache_reuse=1, + device=None, fmax_loss=None, fine_tuning=False, base_mels_path=None): + self.audio_files = training_files + random.seed(1234) + if shuffle: + random.shuffle(self.audio_files) + self.segment_size = segment_size + self.sampling_rate = sampling_rate + self.split = split + self.n_fft = n_fft + self.num_mels = num_mels + self.hop_size = hop_size + self.win_size = win_size + self.fmin = fmin + self.fmax = fmax + self.fmax_loss = fmax_loss + self.cached_wav = None + self.n_cache_reuse = n_cache_reuse + self._cache_ref_count = 0 + self.device = device + self.fine_tuning = fine_tuning + self.base_mels_path = base_mels_path + + def __getitem__(self, index): + filename = self.audio_files[index] + if self._cache_ref_count == 0: + audio, sampling_rate = load_wav(filename) + audio = audio / MAX_WAV_VALUE + if not self.fine_tuning: + audio = normalize(audio) * 0.95 + self.cached_wav = audio + if sampling_rate != self.sampling_rate: + raise ValueError("{} SR doesn't match target {} SR".format( + sampling_rate, self.sampling_rate)) + self._cache_ref_count = self.n_cache_reuse + else: + audio = self.cached_wav + self._cache_ref_count -= 1 + + audio = torch.FloatTensor(audio) + audio = audio.unsqueeze(0) + + if not self.fine_tuning: + if self.split: + if audio.size(1) >= self.segment_size: + max_audio_start = audio.size(1) - self.segment_size + audio_start = random.randint(0, max_audio_start) + audio = audio[:, audio_start:audio_start+self.segment_size] + else: + audio = torch.nn.functional.pad(audio, (0, self.segment_size - audio.size(1)), 'constant') + + mel = mel_spectrogram(audio, self.n_fft, self.num_mels, + self.sampling_rate, self.hop_size, self.win_size, self.fmin, self.fmax, + center=False) + else: + mel = np.load( + os.path.join(self.base_mels_path, os.path.splitext(filename)[0] + '.npy')) + mel = torch.from_numpy(mel) + + if len(mel.shape) < 3: + mel = mel.unsqueeze(0) + + if self.split: + frames_per_seg = math.ceil(self.segment_size / self.hop_size) + + if audio.size(1) >= self.segment_size: + mel_start = random.randint(0, mel.size(2) - frames_per_seg - 1) + mel = mel[:, :, mel_start:mel_start + frames_per_seg] + audio = audio[:, mel_start * self.hop_size:(mel_start + frames_per_seg) * self.hop_size] + else: + mel = torch.nn.functional.pad(mel, (0, frames_per_seg - mel.size(2)), 'constant') + audio = torch.nn.functional.pad(audio, (0, self.segment_size - audio.size(1)), 'constant') + + mel_loss = mel_spectrogram(audio, self.n_fft, self.num_mels, + self.sampling_rate, self.hop_size, self.win_size, self.fmin, self.fmax_loss, + center=False) + + return (mel.squeeze(), audio.squeeze(0), filename, mel_loss.squeeze()) + + def __len__(self): + return len(self.audio_files) diff --git a/models/hifi-gan/models.py b/models/hifi-gan/models.py new file mode 100644 index 0000000000..cc06acfffc --- /dev/null +++ b/models/hifi-gan/models.py @@ -0,0 +1,283 @@ +import torch +import torch.nn.functional as F +import torch.nn as nn +from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d +from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm +from hifiutils import init_weights, get_padding + +LRELU_SLOPE = 0.1 + + +class ResBlock1(torch.nn.Module): + def __init__(self, h, channels, kernel_size=3, dilation=(1, 3, 5)): + super(ResBlock1, self).__init__() + self.h = h + self.convs1 = nn.ModuleList([ + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2], + padding=get_padding(kernel_size, dilation[2]))) + ]) + self.convs1.apply(init_weights) + + self.convs2 = nn.ModuleList([ + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, + padding=get_padding(kernel_size, 1))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, + padding=get_padding(kernel_size, 1))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, + padding=get_padding(kernel_size, 1))) + ]) + self.convs2.apply(init_weights) + + def forward(self, x): + for c1, c2 in zip(self.convs1, self.convs2): + xt = F.leaky_relu(x, LRELU_SLOPE) + xt = c1(xt) + xt = F.leaky_relu(xt, LRELU_SLOPE) + xt = c2(xt) + x = xt + x + return x + + def remove_weight_norm(self): + for l in self.convs1: + remove_weight_norm(l) + for l in self.convs2: + remove_weight_norm(l) + + +class ResBlock2(torch.nn.Module): + def __init__(self, h, channels, kernel_size=3, dilation=(1, 3)): + super(ResBlock2, self).__init__() + self.h = h + self.convs = nn.ModuleList([ + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]))) + ]) + self.convs.apply(init_weights) + + def forward(self, x): + for c in self.convs: + xt = F.leaky_relu(x, LRELU_SLOPE) + xt = c(xt) + x = xt + x + return x + + def remove_weight_norm(self): + for l in self.convs: + remove_weight_norm(l) + + +class Generator(torch.nn.Module): + def __init__(self, h): + super(Generator, self).__init__() + self.h = h + self.num_kernels = len(h.resblock_kernel_sizes) + self.num_upsamples = len(h.upsample_rates) + self.conv_pre = weight_norm(Conv1d(80, h.upsample_initial_channel, 7, 1, padding=3)) + resblock = ResBlock1 if h.resblock == '1' else ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(h.upsample_rates, h.upsample_kernel_sizes)): + self.ups.append(weight_norm( + ConvTranspose1d(h.upsample_initial_channel//(2**i), h.upsample_initial_channel//(2**(i+1)), + k, u, padding=(k-u)//2))) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = h.upsample_initial_channel//(2**(i+1)) + for j, (k, d) in enumerate(zip(h.resblock_kernel_sizes, h.resblock_dilation_sizes)): + self.resblocks.append(resblock(h, ch, k, d)) + + self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3)) + self.ups.apply(init_weights) + self.conv_post.apply(init_weights) + + def forward(self, x): + x = self.conv_pre(x) + for i in range(self.num_upsamples): + x = F.leaky_relu(x, LRELU_SLOPE) + x = self.ups[i](x) + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i*self.num_kernels+j](x) + else: + xs += self.resblocks[i*self.num_kernels+j](x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + + return x + + def remove_weight_norm(self): + print('Removing weight norm...') + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + remove_weight_norm(self.conv_pre) + remove_weight_norm(self.conv_post) + + +class DiscriminatorP(torch.nn.Module): + def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False): + super(DiscriminatorP, self).__init__() + self.period = period + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList([ + norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(2, 0))), + ]) + self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0))) + + def forward(self, x): + fmap = [] + + # 1d to 2d + b, c, t = x.shape + if t % self.period != 0: # pad first + n_pad = self.period - (t % self.period) + x = F.pad(x, (0, n_pad), "reflect") + t = t + n_pad + x = x.view(b, c, t // self.period, self.period) + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class MultiPeriodDiscriminator(torch.nn.Module): + def __init__(self): + super(MultiPeriodDiscriminator, self).__init__() + self.discriminators = nn.ModuleList([ + DiscriminatorP(2), + DiscriminatorP(3), + DiscriminatorP(5), + DiscriminatorP(7), + DiscriminatorP(11), + ]) + + def forward(self, y, y_hat): + y_d_rs = [] + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + y_d_rs.append(y_d_r) + fmap_rs.append(fmap_r) + y_d_gs.append(y_d_g) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class DiscriminatorS(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(DiscriminatorS, self).__init__() + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList([ + norm_f(Conv1d(1, 128, 15, 1, padding=7)), + norm_f(Conv1d(128, 128, 41, 2, groups=4, padding=20)), + norm_f(Conv1d(128, 256, 41, 2, groups=16, padding=20)), + norm_f(Conv1d(256, 512, 41, 4, groups=16, padding=20)), + norm_f(Conv1d(512, 1024, 41, 4, groups=16, padding=20)), + norm_f(Conv1d(1024, 1024, 41, 1, groups=16, padding=20)), + norm_f(Conv1d(1024, 1024, 5, 1, padding=2)), + ]) + self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1)) + + def forward(self, x): + fmap = [] + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class MultiScaleDiscriminator(torch.nn.Module): + def __init__(self): + super(MultiScaleDiscriminator, self).__init__() + self.discriminators = nn.ModuleList([ + DiscriminatorS(use_spectral_norm=True), + DiscriminatorS(), + DiscriminatorS(), + ]) + self.meanpools = nn.ModuleList([ + AvgPool1d(4, 2, padding=2), + AvgPool1d(4, 2, padding=2) + ]) + + def forward(self, y, y_hat): + y_d_rs = [] + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + if i != 0: + y = self.meanpools[i-1](y) + y_hat = self.meanpools[i-1](y_hat) + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + y_d_rs.append(y_d_r) + fmap_rs.append(fmap_r) + y_d_gs.append(y_d_g) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +def feature_loss(fmap_r, fmap_g): + loss = 0 + for dr, dg in zip(fmap_r, fmap_g): + for rl, gl in zip(dr, dg): + loss += torch.mean(torch.abs(rl - gl)) + + return loss*2 + + +def discriminator_loss(disc_real_outputs, disc_generated_outputs): + loss = 0 + r_losses = [] + g_losses = [] + for dr, dg in zip(disc_real_outputs, disc_generated_outputs): + r_loss = torch.mean((1-dr)**2) + g_loss = torch.mean(dg**2) + loss += (r_loss + g_loss) + r_losses.append(r_loss.item()) + g_losses.append(g_loss.item()) + + return loss, r_losses, g_losses + + +def generator_loss(disc_outputs): + loss = 0 + gen_losses = [] + for dg in disc_outputs: + l = torch.mean((1-dg)**2) + gen_losses.append(l) + loss += l + + return loss, gen_losses + diff --git a/models/hifi-gan/requirements.txt b/models/hifi-gan/requirements.txt new file mode 100644 index 0000000000..60c193b0a8 --- /dev/null +++ b/models/hifi-gan/requirements.txt @@ -0,0 +1,7 @@ +torch==1.4.0 +numpy==1.17.4 +librosa==0.7.2 +scipy==1.4.1 +tensorboard==2.0 +soundfile==0.10.3.post1 +matplotlib==3.1.3 \ No newline at end of file diff --git a/models/hifi-gan/stft.py b/models/hifi-gan/stft.py new file mode 100644 index 0000000000..edfc44ae8b --- /dev/null +++ b/models/hifi-gan/stft.py @@ -0,0 +1,141 @@ +""" +BSD 3-Clause License + +Copyright (c) 2017, Prem Seetharaman +All rights reserved. + +* Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +import torch +import numpy as np +import torch.nn.functional as F +from torch.autograd import Variable +from scipy.signal import get_window +from librosa.util import pad_center, tiny +from audio_processing import window_sumsquare + + +class STFT(torch.nn.Module): + """adapted from Prem Seetharaman's https://github.com/pseeth/pytorch-stft""" + def __init__(self, filter_length=800, hop_length=200, win_length=800, + window='hann'): + super(STFT, self).__init__() + self.filter_length = filter_length + self.hop_length = hop_length + self.win_length = win_length + self.window = window + self.forward_transform = None + scale = self.filter_length / self.hop_length + fourier_basis = np.fft.fft(np.eye(self.filter_length)) + + cutoff = int((self.filter_length / 2 + 1)) + fourier_basis = np.vstack([np.real(fourier_basis[:cutoff, :]), + np.imag(fourier_basis[:cutoff, :])]) + + forward_basis = torch.FloatTensor(fourier_basis[:, None, :]) + inverse_basis = torch.FloatTensor( + np.linalg.pinv(scale * fourier_basis).T[:, None, :]) + + if window is not None: + assert(filter_length >= win_length) + # get window and zero center pad it to filter_length + fft_window = get_window(window, win_length, fftbins=True) + fft_window = pad_center(fft_window, filter_length) + fft_window = torch.from_numpy(fft_window).float() + + # window the bases + forward_basis *= fft_window + inverse_basis *= fft_window + + self.register_buffer('forward_basis', forward_basis.float()) + self.register_buffer('inverse_basis', inverse_basis.float()) + + def transform(self, input_data): + num_batches = input_data.size(0) + num_samples = input_data.size(1) + + self.num_samples = num_samples + + # similar to librosa, reflect-pad the input + input_data = input_data.view(num_batches, 1, num_samples) + input_data = F.pad( + input_data.unsqueeze(1), + (int(self.filter_length / 2), int(self.filter_length / 2), 0, 0), + mode='reflect') + input_data = input_data.squeeze(1) + + forward_transform = F.conv1d( + input_data, + Variable(self.forward_basis, requires_grad=False), + stride=self.hop_length, + padding=0) + + cutoff = int((self.filter_length / 2) + 1) + real_part = forward_transform[:, :cutoff, :] + imag_part = forward_transform[:, cutoff:, :] + + magnitude = torch.sqrt(real_part**2 + imag_part**2) + phase = torch.autograd.Variable( + torch.atan2(imag_part.data, real_part.data)) + + return magnitude, phase + + def inverse(self, magnitude, phase): + recombine_magnitude_phase = torch.cat( + [magnitude*torch.cos(phase), magnitude*torch.sin(phase)], dim=1) + + inverse_transform = F.conv_transpose1d( + recombine_magnitude_phase, + Variable(self.inverse_basis, requires_grad=False), + stride=self.hop_length, + padding=0) + + if self.window is not None: + window_sum = window_sumsquare( + self.window, magnitude.size(-1), hop_length=self.hop_length, + win_length=self.win_length, n_fft=self.filter_length, + dtype=np.float32) + # remove modulation effects + approx_nonzero_indices = torch.from_numpy( + np.where(window_sum > tiny(window_sum))[0]) + window_sum = torch.autograd.Variable( + torch.from_numpy(window_sum), requires_grad=False) + window_sum = window_sum.cuda() if magnitude.is_cuda else window_sum + inverse_transform[:, :, approx_nonzero_indices] /= window_sum[approx_nonzero_indices] + + # scale by hop ratio + inverse_transform *= float(self.filter_length) / self.hop_length + + inverse_transform = inverse_transform[:, :, int(self.filter_length/2):] + inverse_transform = inverse_transform[:, :, :-int(self.filter_length/2):] + + return inverse_transform + + def forward(self, input_data): + self.magnitude, self.phase = self.transform(input_data) + reconstruction = self.inverse(self.magnitude, self.phase) + return reconstruction diff --git a/models/hifi-gan/train.py b/models/hifi-gan/train.py new file mode 100644 index 0000000000..191c9f478a --- /dev/null +++ b/models/hifi-gan/train.py @@ -0,0 +1,280 @@ +import warnings +warnings.simplefilter(action='ignore', category=FutureWarning) +import itertools +import os +import time +import argparse +import json +import torch +import torch.nn.functional as F +from torch.utils.tensorboard import SummaryWriter +from torch.utils.data import DistributedSampler, DataLoader +import torch.multiprocessing as mp +from torch.distributed import init_process_group +from torch.nn.parallel import DistributedDataParallel +from env import AttrDict, build_env +from meldataset import MelDataset, mel_spectrogram, get_dataset_filelist +from models import Generator, MultiPeriodDiscriminator, MultiScaleDiscriminator, feature_loss, generator_loss,\ + discriminator_loss +from hifiutils import plot_spectrogram, scan_checkpoint, load_checkpoint, save_checkpoint + +torch.backends.cudnn.benchmark = True + + +def train(rank, a, h, warm_start): + if h.num_gpus > 1: + init_process_group(backend=h.dist_config['dist_backend'], init_method=h.dist_config['dist_url'], + world_size=h.dist_config['world_size'] * h.num_gpus, rank=rank) + + torch.cuda.manual_seed(h.seed) + device = torch.device('cuda:{:d}'.format(rank)) + + generator = Generator(h).to(device) + mpd = MultiPeriodDiscriminator().to(device) + msd = MultiScaleDiscriminator().to(device) + + if rank == 0: + print(generator) + os.makedirs(a.checkpoint_path, exist_ok=True) + print("checkpoints directory : ", a.checkpoint_path) + + if os.path.isdir(a.checkpoint_path): + cp_g = scan_checkpoint(a.checkpoint_path, 'g_') + cp_do = scan_checkpoint(a.checkpoint_path, 'do_') + + steps = 0 + if cp_g is None or cp_do is None: + state_dict_do = None + last_epoch = -1 + else: + state_dict_g = load_checkpoint(cp_g, device) + state_dict_do = load_checkpoint(cp_do, device) + generator.load_state_dict(state_dict_g['generator']) + mpd.load_state_dict(state_dict_do['mpd']) + msd.load_state_dict(state_dict_do['msd']) + if warm_start: + steps = 1 + last_epoch = 0 + else: + steps = state_dict_do['steps'] + 1 + last_epoch = state_dict_do['epoch'] + + if h.num_gpus > 1: + generator = DistributedDataParallel(generator, device_ids=[rank]).to(device) + mpd = DistributedDataParallel(mpd, device_ids=[rank]).to(device) + msd = DistributedDataParallel(msd, device_ids=[rank]).to(device) + + optim_g = torch.optim.AdamW(generator.parameters(), h.learning_rate, betas=[h.adam_b1, h.adam_b2]) + optim_d = torch.optim.AdamW(itertools.chain(msd.parameters(), mpd.parameters()), + h.learning_rate, betas=[h.adam_b1, h.adam_b2]) + + if state_dict_do is not None or warm_start: + optim_g.load_state_dict(state_dict_do['optim_g']) + optim_d.load_state_dict(state_dict_do['optim_d']) + if warm_start: + optim_g.param_groups[0]["lr"] = h.learning_rate + optim_d.param_groups[0]["lr"] = h.learning_rate + + scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=h.lr_decay, last_epoch=last_epoch) + scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=h.lr_decay, last_epoch=last_epoch) + + training_filelist, validation_filelist = get_dataset_filelist(a) + + trainset = MelDataset(training_filelist, h.segment_size, h.n_fft, h.num_mels, + h.hop_size, h.win_size, h.sampling_rate, h.fmin, h.fmax, n_cache_reuse=0, + shuffle=False if h.num_gpus > 1 else True, fmax_loss=h.fmax_for_loss, device=device, + fine_tuning=a.fine_tuning, base_mels_path=a.input_mels_dir) + + train_sampler = DistributedSampler(trainset) if h.num_gpus > 1 else None + + train_loader = DataLoader(trainset, num_workers=h.num_workers, shuffle=False, + sampler=train_sampler, + batch_size=h.batch_size, + pin_memory=True, + drop_last=True) + + if rank == 0: + validset = MelDataset(validation_filelist, h.segment_size, h.n_fft, h.num_mels, + h.hop_size, h.win_size, h.sampling_rate, h.fmin, h.fmax, False, False, n_cache_reuse=0, + fmax_loss=h.fmax_for_loss, device=device, fine_tuning=a.fine_tuning, + base_mels_path=a.input_mels_dir) + validation_loader = DataLoader(validset, num_workers=1, shuffle=False, + sampler=None, + batch_size=1, + pin_memory=True, + drop_last=True) + + sw = SummaryWriter(os.path.join(a.checkpoint_path, 'logs')) + + generator.train() + mpd.train() + msd.train() + for epoch in range(max(0, last_epoch), a.training_epochs): + for param_group in optim_g.param_groups: + print("Current learning rate: " + str(param_group["lr"])) + if rank == 0: + start = time.time() + print("Epoch: {}".format(epoch+1)) + + if h.num_gpus > 1: + train_sampler.set_epoch(epoch) + + for i, batch in enumerate(train_loader): + if rank == 0: + start_b = time.time() + x, y, _, y_mel = batch + x = torch.autograd.Variable(x.to(device, non_blocking=True)) + y = torch.autograd.Variable(y.to(device, non_blocking=True)) + y_mel = torch.autograd.Variable(y_mel.to(device, non_blocking=True)) + y = y.unsqueeze(1) + + y_g_hat = generator(x) + y_g_hat_mel = mel_spectrogram(y_g_hat.squeeze(1), h.n_fft, h.num_mels, h.sampling_rate, h.hop_size, h.win_size, + h.fmin, h.fmax_for_loss) + + optim_d.zero_grad() + + # MPD + y_df_hat_r, y_df_hat_g, _, _ = mpd(y, y_g_hat.detach()) + loss_disc_f, losses_disc_f_r, losses_disc_f_g = discriminator_loss(y_df_hat_r, y_df_hat_g) + + # MSD + y_ds_hat_r, y_ds_hat_g, _, _ = msd(y, y_g_hat.detach()) + loss_disc_s, losses_disc_s_r, losses_disc_s_g = discriminator_loss(y_ds_hat_r, y_ds_hat_g) + + loss_disc_all = loss_disc_s + loss_disc_f + + loss_disc_all.backward() + optim_d.step() + + # Generator + optim_g.zero_grad() + + # L1 Mel-Spectrogram Loss + loss_mel = F.l1_loss(y_mel, y_g_hat_mel) * 45 + + y_df_hat_r, y_df_hat_g, fmap_f_r, fmap_f_g = mpd(y, y_g_hat) + y_ds_hat_r, y_ds_hat_g, fmap_s_r, fmap_s_g = msd(y, y_g_hat) + loss_fm_f = feature_loss(fmap_f_r, fmap_f_g) + loss_fm_s = feature_loss(fmap_s_r, fmap_s_g) + loss_gen_f, losses_gen_f = generator_loss(y_df_hat_g) + loss_gen_s, losses_gen_s = generator_loss(y_ds_hat_g) + loss_gen_all = loss_gen_s + loss_gen_f + loss_fm_s + loss_fm_f + loss_mel + + loss_gen_all.backward() + optim_g.step() + + if rank == 0: + # STDOUT logging + if steps % a.stdout_interval == 0: + with torch.no_grad(): + mel_error = F.l1_loss(y_mel, y_g_hat_mel).item() + + print('Steps : {:d}, Gen Loss Total : {:4.3f}, Mel-Spec. Error : {:4.3f}, s/b : {:4.3f}'. + format(steps, loss_gen_all, mel_error, time.time() - start_b)) + + # checkpointing + if steps % a.checkpoint_interval == 0 and steps != 0: + checkpoint_path = "{}/g_00000000".format(a.checkpoint_path) + save_checkpoint(checkpoint_path, + {'generator': (generator.module if h.num_gpus > 1 else generator).state_dict()}) + checkpoint_path = "{}/do_00000000".format(a.checkpoint_path) + save_checkpoint(checkpoint_path, + {'mpd': (mpd.module if h.num_gpus > 1 + else mpd).state_dict(), + 'msd': (msd.module if h.num_gpus > 1 + else msd).state_dict(), + 'optim_g': optim_g.state_dict(), 'optim_d': optim_d.state_dict(), 'steps': steps, + 'epoch': epoch}) + + # Tensorboard summary logging + if steps % a.summary_interval == 0: + sw.add_scalar("training/gen_loss_total", loss_gen_all, steps) + sw.add_scalar("training/mel_spec_error", mel_error, steps) + + # Validation + if steps % a.validation_interval == 0 and not a.fine_tuning: # and steps != 0: + generator.eval() + torch.cuda.empty_cache() + val_err_tot = 0 + with torch.no_grad(): + for j, batch in enumerate(validation_loader): + x, y, _, y_mel = batch + y_g_hat = generator(x.to(device)) + y_mel = torch.autograd.Variable(y_mel.to(device, non_blocking=True)) + y_g_hat_mel = mel_spectrogram(y_g_hat.squeeze(1), h.n_fft, h.num_mels, h.sampling_rate, + h.hop_size, h.win_size, + h.fmin, h.fmax_for_loss) + val_err_tot += F.l1_loss(y_mel, y_g_hat_mel).item() + + if j <= 4: + if steps == 0: + sw.add_audio('gt/y_{}'.format(j), y[0], steps, h.sampling_rate) + sw.add_figure('gt/y_spec_{}'.format(j), plot_spectrogram(x[0]), steps) + + sw.add_audio('generated/y_hat_{}'.format(j), y_g_hat[0], steps, h.sampling_rate) + y_hat_spec = mel_spectrogram(y_g_hat.squeeze(1), h.n_fft, h.num_mels, + h.sampling_rate, h.hop_size, h.win_size, + h.fmin, h.fmax) + sw.add_figure('generated/y_hat_spec_{}'.format(j), + plot_spectrogram(y_hat_spec.squeeze(0).cpu().numpy()), steps) + + val_err = val_err_tot / (j+1) + sw.add_scalar("validation/mel_spec_error", val_err, steps) + + generator.train() + + steps += 1 + + scheduler_g.step() + scheduler_d.step() + + if rank == 0: + print('Time taken for epoch {} is {} sec\n'.format(epoch + 1, int(time.time() - start))) + + +def main(): + print('Initializing Training Process..') + + parser = argparse.ArgumentParser() + + parser.add_argument('--group_name', default=None) + parser.add_argument('--input_wavs_dir', default='LJSpeech-1.1/wavs') + parser.add_argument('--input_mels_dir', default='ft_dataset') + parser.add_argument('--input_training_file', default='LJSpeech-1.1/training.txt') + parser.add_argument('--input_validation_file', default='LJSpeech-1.1/validation.txt') + parser.add_argument('--checkpoint_path', default='cp_hifigan') + parser.add_argument('--config', default='') + parser.add_argument('--training_epochs', default=3100, type=int) + parser.add_argument('--stdout_interval', default=5, type=int) + parser.add_argument('--checkpoint_interval', default=5000, type=int) + parser.add_argument('--summary_interval', default=100, type=int) + parser.add_argument('--validation_interval', default=1000, type=int) + parser.add_argument('--fine_tuning', default=False, type=bool) + parser.add_argument('--warm_start', default=False, type=bool) + + a = parser.parse_args() + + with open(a.config) as f: + data = f.read() + + json_config = json.loads(data) + h = AttrDict(json_config) + build_env(a.config, 'config.json', a.checkpoint_path) + + torch.manual_seed(h.seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(h.seed) + h.num_gpus = torch.cuda.device_count() + h.batch_size = int(h.batch_size / h.num_gpus) + print('Batch size per GPU :', h.batch_size) + else: + pass + + if h.num_gpus > 1: + mp.spawn(train, nprocs=h.num_gpus, args=(a, h, a.warm_start,)) + else: + train(0, a, h, a.warm_start) + +if __name__ == '__main__': + main() diff --git a/models/hifi-gan/validation_loss.png b/models/hifi-gan/validation_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..5d9426586f48cf606685ef41ea36d9ca4319f7e4 GIT binary patch literal 10995 zcma)iWl&sE(YO>}R`s2}eY>91@uq+N zjpnbraDS`Gbol8v0nIB5gg5AhTFZ^vNzukzUmM>AMZl`)o+=EX-Bp|~T|XBew5u?t z?d8qKvHc0HAyNYIq0!iF~fFM#Z4;}{y%tQPCnoLDkKf|7Ul4olT@9KKFDVceA07XXA zjSL>68`9B^v zV_{*jSt30V!wjr2L!G~#R`MdVva^eMon(!Y6iUg$Z{iFnR>Vl(A^MlVLR z6O?A8r+@vJNUu0rNt?oHyx8pgn^mW{^l5pEEzwWL|Eyezk~PO$KDqJF-T9WXvhuHA zzu26^`yz30$OXrjmJD_zkZ&hu3KSJ%Nb)WeYL`K+t*x?POn?u468uU{#UbIA+KHuc zCKD3;g(V^)vazuN007vWk-DU)HeraaR%t-BFRbCmyEXXue#t7_G0hg}z(+xJF+1+# zsqA@P%&pF}QZ_2wQhdy<>oge%z?JU(hw!0Nj?!vfH1N5K8ZGEW3eT31sEVucWub8w zD!jD@7ntN$qh}%%G#DT~9rWlukxaTVA++frua4%6LE}lREmBqfh=#%vDY#H5t*-8J*^p6D zi5?}-tE{Z7udg@n4uqRwzk!sPn7IKg>~&&dYilbf7d}cJ!j%yD2XW)T`>R*#TrL!n zLY6ts8rfnJMR~b4Y1pSv zpXQgADq`@?%(R;)oR3r9EScx}P&`jh&q-6S zbs_7k!u!kI=9S}eUD+VJzcC_3MQk}^L)1ah_#ma>Zzr@ynNnlD)*Xaiz2I-RC(AE0 z2UhziO`4gWw|^&$7cj_}8|rsdF^D-0>--6^!Sk}AVSrF9l)@4(o!`j?(uT6EfNwQB zq=Sbp%tsv^U+e04q@}m@6Ej55)HJFqX1YF)GTduxX?--I;C#j?E5X)&|BhY}n~U{b z;(j%~k~>;?1>SR`nQIdyrg>8XGB8jCm|?5Vs9sNzdJ!}hkTbJt*f7^E!?l^*F4s+^ zT{0?VBYtBFo57s>?y?&n>*=Pg-OXcZ3uXdc8>^;Bif?rd4>M^r*_w7=&2E(r3l4Qo84e z7bzM7#?Y-&fE^h7oy%;Gx6~%>o?&XKNl9HVF#GhBKMV30zjf1z{^)_&=Ew~CsmYa6 zL7)mV+M1*0ppsE?pqDw0>*>iM)y>dN!pZrN=P|dK&&AcBUq$hValD;uA4LX@eTu}UgUG8F*;cUTcP-m<&kghLUZ|p58|z=M*5@m$lP3&;9+m*WsK|KBX@4?vmTS>|I2Q&y%h>P!WX27yvzi zVLZ*#u&)b7U|X?Lj|oJ%5nFKO~ovwbk8j>wc3p!O?E1fBogO zuDvF*RdVfUIt8OOCN|F9dFhsV^u&U9q>N6x|j1=s!+SrXOw z^Rg#zgVH(aECw`l$JlE2Gry5htnMLM{|BxsEs;0&W(w#PVzhkdA)uz!)m2*AMpv=0 z|3>O*yhsW|+M^X-)RU2s5i}f}_-O4zd(zGSLVo`c=)m)vi_l5FG~389e5;5KBC;(; z?~qzXWoYot-yQhds>LlwR88qdX%+_rRtJ%C2~MU_H((fxh9Ktm&gOsSLB=zHqK`*vg`a1$lON)*lv5r`TU9eAeuPY&l(U%-PCG)D7mf9T!mmF&3 zUadAwcaz>vSccd2kQY*UL<^Gdo0Ppxwk7HOhB4g@@A+r7UGzKJgXv!J#3=LB)D~=( zGMi|SEtSNf7>$9~OJ_^E0BB$y%45_Or{<~Lm#4Iu?Yy?>@YYSd$XU zxou~TM|>svQxvby)u%e`V+5siOLqkY!7dxT)&ep`8x=PW4TSDrsT>&%!OUVlxH5x= zTWV`dHFJ>)>FMcZWI``5-i@aA^pg*b82U>~OS_)e)oURp5)%>Leo1>(HRCdIJ+EwL zI#9vIZ9ULg>ku9vPgPbD6eRF=6?3?Srh8fdcNZ359#1KzVC9SIyGK28=Gtifhzv*I zq?XGQnK7T0MXqa-rRt6P&mR}e1FmBqPjib`Z1(&IF7)CjGG=%^4GbPEh5BlQr%Ff_-<82DzDnE%uwF zm6iANBb+xw7#dEyyU7@Nu{5$ppL;QC_G%9J#VR?gb^VW;N;wLCEE7-Vh@T~M3T4h3 zOa4HBLT;S$UmpqDz&;#L3vQf^QT3kJNfi2RTDE(p<%M#pHX3~C^qboq_bE)ba#~(L zVKtPsi#@+9d`LhgY;m4uA(ZvgDc~wuV^S%Djc|8ndxG1kp5phNyE`q$_mnj!0;~^o zjZXL^o~NJgr(}B~ai}}3+%hQY7bEL|73Um#cGoWX1HpnkcA~98$tG_jz0j-Pykk@G zE@!nwgyIP@Jd{X2Yx-Iufs$fK`Oq1a#@t%>U)9wuiLLm)d-aZzZ`QW^(yz|1v$Tvz z@y10pljjxOEuVhK#9@IUoyv|p6c2Om@k{|I_PUC$%{75WT;b`= z)vDP6)Oc*jOmPCXU>7{-xrNVeu`3F!(Y6EcXM0!FauT7p z9FCs3T0CVlIG2jnM73j7eX8WL9_Q|SRpV@Vh)J#4n*#LGq%JaRw4ProL+ZQo?l6RO zy(1!DB*${5F<^av7Xj&%PHK&@OC^p1`zBkbQN5pWW;K8+#12)Z5I=wz$7c60SGib|cT+S9c+A zPaGD;!;_^;l{Kf4OdY?@!TMC4XpN#-eB0CO#J~|f*f?{hVCk3kw~BN5r}TvGh>oid zY4NTl^bX>APH==H!tac7up*2m$NNJ} zI>$*Svr~YSTFZAc_^0c5mfT!bK4t;$2O2`Ca49rm9VLS@+?jSShkxq(3&bZD{<2@5 z8@RMRMITzUX^Zj<&(6+vU&_9XC6%y)J?)xW?jb%GrSttAc0v4jp3mr3P@)8|tEI&^TP z5_CV?E03YiVIC1CZDKM$X_~23S|jU3l`(LLM2pAP0fPhLJS(KbZ2Mr=w7%3K6 zQI^C~m-J&aK41(_KFz5wyY}l>)q?y5@j#)22$*BjK@Q1ciTn9w4Ne@5&tV+3k`VZm zfoWAa_P9^1jCf+CK|k%JKiezSK?(#pXinh>@*WJBV1w8y{U;*SPAS-U-X`b{oqxGK z4N6Y&j1*TW*?qKZSM<8QW}lR8dfyh$aa+aGqzj0Y8Sc^yi#w@4mTcU7#4+OYT4Ka* z#Oa#cC+>kka{6Eps#(_{P^nQ7|Hrz_h#56a$*cKQ8xKABdH?EAs8WW!y|mU4ywAs& z&{%E%=L#TMa10%p?D-SSmkwN6$ABHo)vBM;aGG1BRrhBPfSugkrG_sQHW@=bZmVhtn%v-N9tBoDgZb0e2R-Zu%(p^o zbKLgYf1?O~lU|$Z*&@LsNs61HMOT|TQ?YT49B~P6mvRmi7I9bc!@3WAx?OJ~v8m$X z;&e&pwstkE>4DIDg#jHL5wewy8UDp=FC+i6j8k_UaJGnyDm|}S>K_eA>BrxnZyn*W zuZZ;(1Bs1YATgIwXjAi=S%d4A4FR*-XUd=OAhRyE%`+Uc@=zWrvO?oRy1>@V2nssi z@?m2zjCJf;EZXc_g4l)clX=@Qylym~W9PKBx{g>Q&-#E8fIeAvw6GMloDq~}It~0cKXJq(R{&GGQNE%df`FH=icK4g*DuWhIV7VfG8W`@Hl*!5el{2l!-IEulIg7*zD0yY>ltelQJk&-U z91*aSC+cJ3=v`T!F1s)-*mR!3b^D7;Zk<)-?Y4GSEsi;XxpRSwOznYIyI|hLl9}>P zkG(9Qcv!?5ojQ?^UdPR7dnCxPPoqSG0m9CbY*#$$Y0v zen$ni>6@-}KbGvzN))@-_@tUZ;4Q)rM+X*l#pD}`jBdruxbmu&?QRFt@h&WjwM>hl zA$xobuXw)WnIat(^{x1-3o7@KwWW<(h;4Z~J^yUU;=AWI=>QMv`DRL(G?8CX4F&St zmxnzoRlTxNRDkEw(L-7wRvw^tX=g>{4xEAB4d+7g+7|3_4b~SPW znQ?L8qqyPx(wSn>9YG-`~Ew(nvNkv0;lr? zYr3S2d=Yy7O)+>tdCwt!)QmDYe~wWr%A|< z3zn)d8d=?BFP_vQSX#fA;Iyu8j}W1-|8^fip*rKEYFfo+(}70hiNhpiJhN;uL#Uy| zhwT$NG(GKQg0K7#j}xW2o{!u8rj>t6{H2mbxPp=sV&TWkaYThVPo8`9WnoZe^SkXc z_zQ{KeaZ*2+eN14;e}LBXIuxEiaQ|h@`A3;>$~Bt_1@^4D0XvPk#yZ&Z-Rf)pr4g~ zs&re!Dnf!CWxEvGL$7dN^hKg6m({V?gB}?t_4tspin&k_m?P#s6bXcr2z7a?_Y92TA+IVFRQ7IOj}@ScH%!`2fMb;s*c98dK2QV zG|4DN*&+IhQVG!)81?Z9r?+iF1AhtUCsJ}%*Y*h+8Qj*KF6BIQrsHa{28ITy>{_c^ zVvu@0@TNU4a5I(>9S^bddX_Zr?h<%{uHqTIU7ipGx@XAD&qMFVw%%60i2h z+o6!SRA=1=uA-5v2)?BjWmzEp?!CTMTD9D%hWh|n zK7N=&;#%OQ75YwWKbKj_^WKPkDBD}u1DEnahI?s5!OgApDq1@{dIG36>n&%zS+#15 zt!2SmNfksR+t#s3xK+x13q1)`6BjpM6%6CJzXy?V`knKf-<;CHeoi;Xhzn)*aGsgT zFb@sEs|h2sMW@&M-)rw`xSC?t24E0)Ax^^ser5_E&S{;ZM%C&j1ApTZ#a6nAKB2Qd zB^1>&cnozvaaoUrg>Y_^kD}zz25Cq2Z5Vh|)3(JEsCzZ`8COx-{!7TOGVUOnli3l1 zg3xt#gK8dE|06T>{-Z0L;G9EzomxdShJz~%nn;%y1Vvlj`*}q+CP~}&4eS+`j0h_K zbKQBrAoTYw3mL6L7VMGb=QL;1glijToLpz@lMQi@@m!6q>p|ROi0lIAH8<4jIcadP9vk4`f9GvqgksMwx-Z~Q z{GY_^(QVU`n2^X1;D5A^`1b3Kr?K2%S(L*D4;^opppWgymHBLzU}Gok--WMsS#VrF zu#~kJz~Cq8vP&0kGsqHAcwx*XgM^=Pwf@QUidGml7T6ch^4x*vth)dFPN8+&;&Si~ z(uo-G*QY}2o_jKOZYCQh&2Ti7Q*u2;dwOP@aTqEghHiBRVn5$qbK^HtQA@!?^-_MnyOQN!z7LO6B^pdVN zye=oIQ##zFlU-}E*75&#cW*vLJ%L)&9Km!&D|B)9>A~v@#mm<%Amp;XOk^6gmZtEp zzReZePx7j~$*^{Wk!S%Vs3hux>bQvOMl2soX)S*2dj6-TUv}z?AM9(!s@pmS#|<@g z^g7H54jY@N-L#Sooa}oaJ~q!#DUrV;zvPq_5liqw<^UD5#)cVQ&ji#0(2EH@4WY*T zGxS?!xOTQ(n&>|9H{^1!w{9m2S}UNT>e8)7733;lp5G54ceZ#FM)D36N~ZwSmTfx5 z41mFwO@)@-U*3;EK4fWjQ~g5P&&mnOYNBspTRx3zf(7r`@|=cAGJd@C+nT<4U-5?A zot=QPJWtGRZSm6!mDKYk{yTk{aa6uth{9Xo;KVQ40N;h#ApWX!)Iyo{4B3 z=6$`5GLj?tWPIQq)&U#=7)V~nLOQbK0za}x5Cz;$S?n^?1yn%lG$t)03InpRsY&@D zdLxdXYxx{|daXs#4BaF-MT`%H^qE`v>&%`IPSSJ)Y zP5lar=N&ePjE*BOoeUvryR?4uA&2$lJ)7*2)2RFQM5LOG+-nI!^Vkt#XSQED?0ZlT zVo{j8nBtR65BqmdqX=IV{jV~F@7^pxxc(+U8JpaUz7dYvWc2o0S2J)!M`TOuVv&+! zqT8|_z#L@Tub?2(B&m(*8_htjoJX3MV(O)}bJXO*xI$@O{(-u@*}%;ohBu0J&R>UD znP(*nez`O*u&WH@U~L{cYOlPH^YxXz^^T0N0rDA8*$wWnRaAei9%R(<^0j3+&im9j z^1AAJkz4V%bm3*=awD~1+}n4bv8S1an(Rb4H^77Un?4MWnlH5LrU(PbNkY}X^%GQ0 zUGe#6b~d^3=MTsaeku!Cn;peQOn4B+^67EC`lZOM+A~?DQSfPsSzxhOoKeZ}x*5KuTSQ63XGR5o)~}F)UmlO zt8zs~0tdiy0`9yi%&9ZC4buEZ@FpS-HV7P$<6R#f?)8%&Oofk8qZ)BY2@>23Z^{9% zdlBrHA5d7|8O1rfxG_1ApZG3l3`5ST{HJH+y7V*|1$8D6Q;Rx%8aEx`gQ!fwaLl7l zQcy{dRscv+snP#{{)TqRz%)oveRLSQoi~6p&h@wvq$g*uvqM%#wM^RT379`H=oRWD z@ON?8n-rL#hB5jd`pvKthdcVB>;P8`jE<7uV@G;$G|xjgl8aKQaIqtuU2}@^4XRIu zHE57OQoqJ&rGr6c{gAHR`6q!giP&Ft8-8n@BNP3u+#0Hmjt!3}85M`z&|;;96#mK| zT=yLg|6+)@G=;ZPQ&=hf_d%KN(1FJ~dM`*oVC?*TY4dA-*z?n#?DOj~lP(|n1H@~G zxvBvygA>I2_m4*nY|B4QI+S!G2=MVgE#gLEQ=(&FsH%S^zU<_Tj*fPjx_@l8J@t_a z41CWfNwiG(2}xW6;CcWVSzK7yfDZLHcKylnJMt8RiX|pey2dCH;`z;&S}o$10+ErC zJu5rs{=Nw?Nlx1y={6<}T>C{@j=AiD#=ilXPo<#lW<=dQoA+XW>QX`mXxgRk%ukyQ2pF)*b zes^F*mVb6pC|O}Cw2uWwd-J2ZxVkH^qk}_X-t6M@&@E^=Z<-AL#;$r<`x0`@(+Dl| zm@y^q7|es}*ArbA>PGnUI7JhEx9|n{P<@7O3c`SHz~;#Kg1Y8Bp)V^zbPwjTz`>w9 z;r>&KAR9f4@(i}uvy#NQHH8{2{*I2;BHK*|gmt#nhL573=kpHY$dJnSB6b7HRRWIn zlH_R_OpThEr zLYUPX{vKpLY(I@TXvJPOW9_9k|K6ddZpv)2z?$p?m`=-ABquke;k z6!m1amXQ*sz5xn= zi^g_t9@qC&&L_mu{S#YUEUX)3o2n`{I*5wd-u%jrtoewfRMz;9U-#-`{c-Vj$W&ea zGC|ir$z6j&3?axLfqqH`7~~Vqv44qCZ%DsQCu5SqwH+jou7GSI5}J09WctRU#{SUv zvSld4kdalV(nK+QoOa>8j(R=t1E8B5ugdgj?(yo_KR3fZ7mQrLYkS%EUiPvyA$N?< zpvl>D=4QeA!fpg2U7={@&x#Eq@F`JhF1n9{W{8%cvX2oE9jh+t&+Yl1pq<~E&+&X( zAKN1hDM7X;Efr{i*9s9aR%Cs@*OJ@qAhwpbF=mv%dN(~Vdx5mv2dZHpl)mIH*Aiws zuHC3zsC`nB>hzvU0$TJ4Y_=KGUO-{8mtZy2+=m~F@~3ZgY@&feS~h+!d5C%*BHdUS zNtAcf9gR1TTuIV|R%YSDV5?IFSAWUtSu1KriGqmVvzb+ASuH&3MFOklG<~-$eQ6Z@ zOSInnGr)ZEcEDa%;mbeDLWc##ugC`SvRI9OGeTa+%rs^RCBW-i$ zA`+P^%9-qFIo3XxpH9PWaDK$CAIVtSN+N3SKd(57t7A=5U-fO=Kc zF$>FlK%8T5e;uxfiuGh#gY^V+6dKeh1#@esN3NMMFiG|DA5XnkJq)3 zljZK0KhU4#pV|rwubyuW#u`7(^a{BKkMf81X~i+8k2Vj|VkPqo&lagRgO>AokwrX@ z4E6MqM@bmy=?l8L%&cO9UwU84W2Da<&$IOgX$A%cN=eINq%Zh&2WjrK{2NC}!r`=m zmTswIV~`M_zM;V#j7_=8)F#o#y(aV6>q)@a-qLcl)E%fFXDLKJYE4p1Zf6dmVYKRb zz^a(V0WFcw{?Ep0;+Qn3b-!lwL$F6f4j&lUh8Z6|!L4>08Zp7m6#oA?6^dZQA%TeJ{=7W4R`H;kvUZl@vy8*2)=-B7wejJ z-8nrtk{nx%O&QLxh(G<4Vc)VLTD#fJz`|1cS_rEr=(G1<@&6)0^e-(nTe6?NR;sW( zK)jpAuJokYz}Tj35ych?X&>=w5ECxRgPZ^J!P}e~Hp_(ZaW!#sxV=$us46QebV%v{ zbf0yzS>pN|``f^Y6@iRT`tTi`%reWzh1*;Qclg@c|DHVTYJ~f^OAa){2Q}&v9 z_1)ubK$SN1I)~ep0QHuG+)9XT|W&X0r44C;cYzc6ej7(y&WK#rE>=ROY@U{gzVtxfqqrjDFJ2?fSD) z4zHjD(m=*SE`R%sE1_n*nOJ3g`2nwN&6XJd z*JqBo|0u7~kcqYGf7x$u{PU3Vr{iJjLncXnDq4GC3sQHYnxecsa->K9Z7JtjSg*K7 z`>m4%dzOX-VjiZ(9#biIC5uP3qWT%`>O_S6KuheID|4L9n&--=_D+SCI|ilY=FgjV zHA3s&HgNLDgs$uTA3KW%y|J3atW_?)OX03Ue>987b@eS-S(#5a7r$!AC_s+OJ*48F zaMc=fBF+NO!bW)4SX%}IyYhD-yut5Q!jC>uyEA6FW~n@zeD3xfWwH}>S1=Z{i-&+8 zg_;8A$Hwn{rKhc?o*Lh~%<&W~3h^;5z&9b$F8rL}dc+0RVU#+v<8HTI$Wa)>gp(j8 ziR1GJaP$MD&hk?$0PYvpMimV3I#e?#H45~z4feoU@YHA9Or}c$^*jD4bkZGUgwloi zJ{3A{qzXFGX>hd^sEGQ3u6XP9l>i45ef4g?JJHE=$mIQu@>a;4z+2E(7+76n(BU$rUz+GXFz=TdP3pLbzr`A2>J`vdaad525Ri&(tV?0s!Bi-+UW z>(n`qoN8HSiI)@|oX~OJSiiQm3@6*A4Ak+rh3f}VzXONfioZ2FPTuEk**(+w!MHk} z_J`k!6OT6xqHD)I^W?gJ(J{Cr_55bSR0+HHdqgT)KeEGe{mlISE@TnqXUDP$m&L)4 zxBOKzL_IGD-C_T%X8IVb^*EqO^G~?|JHa}M=Ze-`=V$MSEB<95*npOAId8{a^X!|G z?2mgz>Hv}nQ_}>h&|+e;kG!!$jYeTE>|@w`NyE%J2m~j6=6}*}7ho$?&aI?O^i0K)W9JGyucyP=LXqScOYK%#K(sE* zjQ04c44Yx6F|P!haG@mPyI(J9*pKn}x=|$>oo^VOG$=&-)m_p($(!Ds28*IX?`@CA zaAqUx8HI1>E0ATZ*@Bj(M%)sJPB9M|Bn@8hJkG?`)xMlo+9R^Pce0T5+Whx@FT7Gv zT3#(ZfXNliC;K8g+erq)uXw*saK|wJFXaEu4Y_KW HkD>nqG8@np literal 0 HcmV?d00001 From d0ee1962d81909f75f2c331bdbf945c256067d7a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 22 Aug 2021 19:16:33 -0400 Subject: [PATCH 0504/1042] Move hifi-gan under tacotron2 --- models/{ => tacotron2}/hifi-gan/LICENSE | 0 .../hifi-gan/LJSpeech-1.1/training.txt | 0 .../hifi-gan/LJSpeech-1.1/validation.txt | 0 models/{ => tacotron2}/hifi-gan/README.md | 0 models/{ => tacotron2}/hifi-gan/audio_processing.py | 0 models/{ => tacotron2}/hifi-gan/config_32k.json | 0 models/{ => tacotron2}/hifi-gan/config_v1.json | 0 models/{ => tacotron2}/hifi-gan/config_v1b.json | 0 models/{ => tacotron2}/hifi-gan/config_v2.json | 0 models/{ => tacotron2}/hifi-gan/config_v3.json | 0 models/{ => tacotron2}/hifi-gan/denoiser.py | 0 models/{ => tacotron2}/hifi-gan/env.py | 0 models/{ => tacotron2}/hifi-gan/hifiutils.py | 0 models/{ => tacotron2}/hifi-gan/inference.py | 0 models/{ => tacotron2}/hifi-gan/inference_e2e.py | 0 models/{ => tacotron2}/hifi-gan/meldataset.py | 0 models/{ => tacotron2}/hifi-gan/models.py | 0 models/{ => tacotron2}/hifi-gan/requirements.txt | 0 models/{ => tacotron2}/hifi-gan/stft.py | 0 models/{ => tacotron2}/hifi-gan/train.py | 0 models/{ => tacotron2}/hifi-gan/validation_loss.png | Bin 21 files changed, 0 insertions(+), 0 deletions(-) rename models/{ => tacotron2}/hifi-gan/LICENSE (100%) rename models/{ => tacotron2}/hifi-gan/LJSpeech-1.1/training.txt (100%) rename models/{ => tacotron2}/hifi-gan/LJSpeech-1.1/validation.txt (100%) rename models/{ => tacotron2}/hifi-gan/README.md (100%) rename models/{ => tacotron2}/hifi-gan/audio_processing.py (100%) rename models/{ => tacotron2}/hifi-gan/config_32k.json (100%) rename models/{ => tacotron2}/hifi-gan/config_v1.json (100%) rename models/{ => tacotron2}/hifi-gan/config_v1b.json (100%) rename models/{ => tacotron2}/hifi-gan/config_v2.json (100%) rename models/{ => tacotron2}/hifi-gan/config_v3.json (100%) rename models/{ => tacotron2}/hifi-gan/denoiser.py (100%) rename models/{ => tacotron2}/hifi-gan/env.py (100%) rename models/{ => tacotron2}/hifi-gan/hifiutils.py (100%) rename models/{ => tacotron2}/hifi-gan/inference.py (100%) rename models/{ => tacotron2}/hifi-gan/inference_e2e.py (100%) rename models/{ => tacotron2}/hifi-gan/meldataset.py (100%) rename models/{ => tacotron2}/hifi-gan/models.py (100%) rename models/{ => tacotron2}/hifi-gan/requirements.txt (100%) rename models/{ => tacotron2}/hifi-gan/stft.py (100%) rename models/{ => tacotron2}/hifi-gan/train.py (100%) rename models/{ => tacotron2}/hifi-gan/validation_loss.png (100%) diff --git a/models/hifi-gan/LICENSE b/models/tacotron2/hifi-gan/LICENSE similarity index 100% rename from models/hifi-gan/LICENSE rename to models/tacotron2/hifi-gan/LICENSE diff --git a/models/hifi-gan/LJSpeech-1.1/training.txt b/models/tacotron2/hifi-gan/LJSpeech-1.1/training.txt similarity index 100% rename from models/hifi-gan/LJSpeech-1.1/training.txt rename to models/tacotron2/hifi-gan/LJSpeech-1.1/training.txt diff --git a/models/hifi-gan/LJSpeech-1.1/validation.txt b/models/tacotron2/hifi-gan/LJSpeech-1.1/validation.txt similarity index 100% rename from models/hifi-gan/LJSpeech-1.1/validation.txt rename to models/tacotron2/hifi-gan/LJSpeech-1.1/validation.txt diff --git a/models/hifi-gan/README.md b/models/tacotron2/hifi-gan/README.md similarity index 100% rename from models/hifi-gan/README.md rename to models/tacotron2/hifi-gan/README.md diff --git a/models/hifi-gan/audio_processing.py b/models/tacotron2/hifi-gan/audio_processing.py similarity index 100% rename from models/hifi-gan/audio_processing.py rename to models/tacotron2/hifi-gan/audio_processing.py diff --git a/models/hifi-gan/config_32k.json b/models/tacotron2/hifi-gan/config_32k.json similarity index 100% rename from models/hifi-gan/config_32k.json rename to models/tacotron2/hifi-gan/config_32k.json diff --git a/models/hifi-gan/config_v1.json b/models/tacotron2/hifi-gan/config_v1.json similarity index 100% rename from models/hifi-gan/config_v1.json rename to models/tacotron2/hifi-gan/config_v1.json diff --git a/models/hifi-gan/config_v1b.json b/models/tacotron2/hifi-gan/config_v1b.json similarity index 100% rename from models/hifi-gan/config_v1b.json rename to models/tacotron2/hifi-gan/config_v1b.json diff --git a/models/hifi-gan/config_v2.json b/models/tacotron2/hifi-gan/config_v2.json similarity index 100% rename from models/hifi-gan/config_v2.json rename to models/tacotron2/hifi-gan/config_v2.json diff --git a/models/hifi-gan/config_v3.json b/models/tacotron2/hifi-gan/config_v3.json similarity index 100% rename from models/hifi-gan/config_v3.json rename to models/tacotron2/hifi-gan/config_v3.json diff --git a/models/hifi-gan/denoiser.py b/models/tacotron2/hifi-gan/denoiser.py similarity index 100% rename from models/hifi-gan/denoiser.py rename to models/tacotron2/hifi-gan/denoiser.py diff --git a/models/hifi-gan/env.py b/models/tacotron2/hifi-gan/env.py similarity index 100% rename from models/hifi-gan/env.py rename to models/tacotron2/hifi-gan/env.py diff --git a/models/hifi-gan/hifiutils.py b/models/tacotron2/hifi-gan/hifiutils.py similarity index 100% rename from models/hifi-gan/hifiutils.py rename to models/tacotron2/hifi-gan/hifiutils.py diff --git a/models/hifi-gan/inference.py b/models/tacotron2/hifi-gan/inference.py similarity index 100% rename from models/hifi-gan/inference.py rename to models/tacotron2/hifi-gan/inference.py diff --git a/models/hifi-gan/inference_e2e.py b/models/tacotron2/hifi-gan/inference_e2e.py similarity index 100% rename from models/hifi-gan/inference_e2e.py rename to models/tacotron2/hifi-gan/inference_e2e.py diff --git a/models/hifi-gan/meldataset.py b/models/tacotron2/hifi-gan/meldataset.py similarity index 100% rename from models/hifi-gan/meldataset.py rename to models/tacotron2/hifi-gan/meldataset.py diff --git a/models/hifi-gan/models.py b/models/tacotron2/hifi-gan/models.py similarity index 100% rename from models/hifi-gan/models.py rename to models/tacotron2/hifi-gan/models.py diff --git a/models/hifi-gan/requirements.txt b/models/tacotron2/hifi-gan/requirements.txt similarity index 100% rename from models/hifi-gan/requirements.txt rename to models/tacotron2/hifi-gan/requirements.txt diff --git a/models/hifi-gan/stft.py b/models/tacotron2/hifi-gan/stft.py similarity index 100% rename from models/hifi-gan/stft.py rename to models/tacotron2/hifi-gan/stft.py diff --git a/models/hifi-gan/train.py b/models/tacotron2/hifi-gan/train.py similarity index 100% rename from models/hifi-gan/train.py rename to models/tacotron2/hifi-gan/train.py diff --git a/models/hifi-gan/validation_loss.png b/models/tacotron2/hifi-gan/validation_loss.png similarity index 100% rename from models/hifi-gan/validation_loss.png rename to models/tacotron2/hifi-gan/validation_loss.png From 6516a04eda91ed8b3ef13e77740b32f356f18fa2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 22 Aug 2021 19:41:50 -0400 Subject: [PATCH 0505/1042] print devices --- models/tacotron2/vocodes_common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index 13960873e0..992e5b4d33 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -26,6 +26,12 @@ def print_gpu_info(): print('PyTorch version', torch.__version__) print('CUDA Available?', torch.cuda.is_available()) print('CUDA Device count', torch.cuda.device_count()) + try: + from tensorflow.python.client import device_lib + print('local devices', str(device_lib.list_local_devices()).replace("\n", "\n ")) + except ImportError: + print('no tensorflow - cannot list devices') + pass print('========================================', flush=True) def load_tacotron_model(checkpoint_path): From e4fe7f8ed129dbcdd274d2d0503ef28f9a6ca400 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 22 Aug 2021 19:42:07 -0400 Subject: [PATCH 0506/1042] don't print devices --- models/tacotron2/vocodes_common.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index 992e5b4d33..af3b251a62 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -26,12 +26,12 @@ def print_gpu_info(): print('PyTorch version', torch.__version__) print('CUDA Available?', torch.cuda.is_available()) print('CUDA Device count', torch.cuda.device_count()) - try: - from tensorflow.python.client import device_lib - print('local devices', str(device_lib.list_local_devices()).replace("\n", "\n ")) - except ImportError: - print('no tensorflow - cannot list devices') - pass + #try: + # from tensorflow.python.client import device_lib + # print('local devices', str(device_lib.list_local_devices()).replace("\n", "\n ")) + #except ImportError: + # print('no tensorflow - cannot list devices') + # pass print('========================================', flush=True) def load_tacotron_model(checkpoint_path): From 76fd658ae6e0549039b8f19045c1e6da7383acbb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 22 Aug 2021 20:07:43 -0400 Subject: [PATCH 0507/1042] move hifigan to make import disambiguation easier --- models/tacotron2/{hifi-gan => hifigan}/LICENSE | 0 .../{hifi-gan => hifigan}/LJSpeech-1.1/training.txt | 0 .../LJSpeech-1.1/validation.txt | 0 models/tacotron2/{hifi-gan => hifigan}/README.md | 0 models/tacotron2/hifigan/__init__.py | 0 .../{hifi-gan => hifigan}/audio_processing.py | 0 .../tacotron2/{hifi-gan => hifigan}/config_32k.json | 0 .../tacotron2/{hifi-gan => hifigan}/config_v1.json | 0 .../tacotron2/{hifi-gan => hifigan}/config_v1b.json | 0 .../tacotron2/{hifi-gan => hifigan}/config_v2.json | 0 .../tacotron2/{hifi-gan => hifigan}/config_v3.json | 0 models/tacotron2/{hifi-gan => hifigan}/denoiser.py | 0 models/tacotron2/{hifi-gan => hifigan}/env.py | 0 models/tacotron2/{hifi-gan => hifigan}/hifiutils.py | 0 models/tacotron2/{hifi-gan => hifigan}/inference.py | 0 .../{hifi-gan => hifigan}/inference_e2e.py | 0 .../tacotron2/{hifi-gan => hifigan}/meldataset.py | 0 models/tacotron2/{hifi-gan => hifigan}/models.py | 0 .../{hifi-gan => hifigan}/requirements.txt | 0 models/tacotron2/{hifi-gan => hifigan}/stft.py | 0 models/tacotron2/{hifi-gan => hifigan}/train.py | 0 .../{hifi-gan => hifigan}/validation_loss.png | Bin 22 files changed, 0 insertions(+), 0 deletions(-) rename models/tacotron2/{hifi-gan => hifigan}/LICENSE (100%) rename models/tacotron2/{hifi-gan => hifigan}/LJSpeech-1.1/training.txt (100%) rename models/tacotron2/{hifi-gan => hifigan}/LJSpeech-1.1/validation.txt (100%) rename models/tacotron2/{hifi-gan => hifigan}/README.md (100%) create mode 100644 models/tacotron2/hifigan/__init__.py rename models/tacotron2/{hifi-gan => hifigan}/audio_processing.py (100%) rename models/tacotron2/{hifi-gan => hifigan}/config_32k.json (100%) rename models/tacotron2/{hifi-gan => hifigan}/config_v1.json (100%) rename models/tacotron2/{hifi-gan => hifigan}/config_v1b.json (100%) rename models/tacotron2/{hifi-gan => hifigan}/config_v2.json (100%) rename models/tacotron2/{hifi-gan => hifigan}/config_v3.json (100%) rename models/tacotron2/{hifi-gan => hifigan}/denoiser.py (100%) rename models/tacotron2/{hifi-gan => hifigan}/env.py (100%) rename models/tacotron2/{hifi-gan => hifigan}/hifiutils.py (100%) rename models/tacotron2/{hifi-gan => hifigan}/inference.py (100%) rename models/tacotron2/{hifi-gan => hifigan}/inference_e2e.py (100%) rename models/tacotron2/{hifi-gan => hifigan}/meldataset.py (100%) rename models/tacotron2/{hifi-gan => hifigan}/models.py (100%) rename models/tacotron2/{hifi-gan => hifigan}/requirements.txt (100%) rename models/tacotron2/{hifi-gan => hifigan}/stft.py (100%) rename models/tacotron2/{hifi-gan => hifigan}/train.py (100%) rename models/tacotron2/{hifi-gan => hifigan}/validation_loss.png (100%) diff --git a/models/tacotron2/hifi-gan/LICENSE b/models/tacotron2/hifigan/LICENSE similarity index 100% rename from models/tacotron2/hifi-gan/LICENSE rename to models/tacotron2/hifigan/LICENSE diff --git a/models/tacotron2/hifi-gan/LJSpeech-1.1/training.txt b/models/tacotron2/hifigan/LJSpeech-1.1/training.txt similarity index 100% rename from models/tacotron2/hifi-gan/LJSpeech-1.1/training.txt rename to models/tacotron2/hifigan/LJSpeech-1.1/training.txt diff --git a/models/tacotron2/hifi-gan/LJSpeech-1.1/validation.txt b/models/tacotron2/hifigan/LJSpeech-1.1/validation.txt similarity index 100% rename from models/tacotron2/hifi-gan/LJSpeech-1.1/validation.txt rename to models/tacotron2/hifigan/LJSpeech-1.1/validation.txt diff --git a/models/tacotron2/hifi-gan/README.md b/models/tacotron2/hifigan/README.md similarity index 100% rename from models/tacotron2/hifi-gan/README.md rename to models/tacotron2/hifigan/README.md diff --git a/models/tacotron2/hifigan/__init__.py b/models/tacotron2/hifigan/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/models/tacotron2/hifi-gan/audio_processing.py b/models/tacotron2/hifigan/audio_processing.py similarity index 100% rename from models/tacotron2/hifi-gan/audio_processing.py rename to models/tacotron2/hifigan/audio_processing.py diff --git a/models/tacotron2/hifi-gan/config_32k.json b/models/tacotron2/hifigan/config_32k.json similarity index 100% rename from models/tacotron2/hifi-gan/config_32k.json rename to models/tacotron2/hifigan/config_32k.json diff --git a/models/tacotron2/hifi-gan/config_v1.json b/models/tacotron2/hifigan/config_v1.json similarity index 100% rename from models/tacotron2/hifi-gan/config_v1.json rename to models/tacotron2/hifigan/config_v1.json diff --git a/models/tacotron2/hifi-gan/config_v1b.json b/models/tacotron2/hifigan/config_v1b.json similarity index 100% rename from models/tacotron2/hifi-gan/config_v1b.json rename to models/tacotron2/hifigan/config_v1b.json diff --git a/models/tacotron2/hifi-gan/config_v2.json b/models/tacotron2/hifigan/config_v2.json similarity index 100% rename from models/tacotron2/hifi-gan/config_v2.json rename to models/tacotron2/hifigan/config_v2.json diff --git a/models/tacotron2/hifi-gan/config_v3.json b/models/tacotron2/hifigan/config_v3.json similarity index 100% rename from models/tacotron2/hifi-gan/config_v3.json rename to models/tacotron2/hifigan/config_v3.json diff --git a/models/tacotron2/hifi-gan/denoiser.py b/models/tacotron2/hifigan/denoiser.py similarity index 100% rename from models/tacotron2/hifi-gan/denoiser.py rename to models/tacotron2/hifigan/denoiser.py diff --git a/models/tacotron2/hifi-gan/env.py b/models/tacotron2/hifigan/env.py similarity index 100% rename from models/tacotron2/hifi-gan/env.py rename to models/tacotron2/hifigan/env.py diff --git a/models/tacotron2/hifi-gan/hifiutils.py b/models/tacotron2/hifigan/hifiutils.py similarity index 100% rename from models/tacotron2/hifi-gan/hifiutils.py rename to models/tacotron2/hifigan/hifiutils.py diff --git a/models/tacotron2/hifi-gan/inference.py b/models/tacotron2/hifigan/inference.py similarity index 100% rename from models/tacotron2/hifi-gan/inference.py rename to models/tacotron2/hifigan/inference.py diff --git a/models/tacotron2/hifi-gan/inference_e2e.py b/models/tacotron2/hifigan/inference_e2e.py similarity index 100% rename from models/tacotron2/hifi-gan/inference_e2e.py rename to models/tacotron2/hifigan/inference_e2e.py diff --git a/models/tacotron2/hifi-gan/meldataset.py b/models/tacotron2/hifigan/meldataset.py similarity index 100% rename from models/tacotron2/hifi-gan/meldataset.py rename to models/tacotron2/hifigan/meldataset.py diff --git a/models/tacotron2/hifi-gan/models.py b/models/tacotron2/hifigan/models.py similarity index 100% rename from models/tacotron2/hifi-gan/models.py rename to models/tacotron2/hifigan/models.py diff --git a/models/tacotron2/hifi-gan/requirements.txt b/models/tacotron2/hifigan/requirements.txt similarity index 100% rename from models/tacotron2/hifi-gan/requirements.txt rename to models/tacotron2/hifigan/requirements.txt diff --git a/models/tacotron2/hifi-gan/stft.py b/models/tacotron2/hifigan/stft.py similarity index 100% rename from models/tacotron2/hifi-gan/stft.py rename to models/tacotron2/hifigan/stft.py diff --git a/models/tacotron2/hifi-gan/train.py b/models/tacotron2/hifigan/train.py similarity index 100% rename from models/tacotron2/hifi-gan/train.py rename to models/tacotron2/hifigan/train.py diff --git a/models/tacotron2/hifi-gan/validation_loss.png b/models/tacotron2/hifigan/validation_loss.png similarity index 100% rename from models/tacotron2/hifi-gan/validation_loss.png rename to models/tacotron2/hifigan/validation_loss.png From a821afaebed4505194dece1eb429b11067950274 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 22 Aug 2021 20:27:02 -0400 Subject: [PATCH 0508/1042] I can now load hifigan models from TT --- models/tacotron2/hifigan/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/tacotron2/hifigan/models.py b/models/tacotron2/hifigan/models.py index cc06acfffc..8a960b3aad 100644 --- a/models/tacotron2/hifigan/models.py +++ b/models/tacotron2/hifigan/models.py @@ -3,7 +3,7 @@ import torch.nn.functional as F import torch.nn as nn from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm -from hifiutils import init_weights, get_padding +from .hifiutils import init_weights, get_padding LRELU_SLOPE = 0.1 From c52ce4de0ac2d3b5991089c5bc2cc399ce1e3a0e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 22 Aug 2021 21:08:01 -0400 Subject: [PATCH 0509/1042] WIP pipeline with hifigan --- models/tacotron2/vocodes_common.py | 133 +++++++++++++++++++++++++++++ models/tacotron2/vocodes_server.py | 4 + 2 files changed, 137 insertions(+) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index af3b251a62..8f96b373f4 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -15,6 +15,12 @@ from denoiser import Denoiser from scipy.io.wavfile import write as write_wav +# Hifi-Gan +from hifigan.env import AttrDict +from hifigan.models import Generator +from hifigan.denoiser import Denoiser as HifiDenoiser +from hifigan.meldataset import mel_spectrogram + # For metadata import subprocess import magic @@ -56,6 +62,21 @@ def load_waveglow_model(checkpoint_path): denoiser = Denoiser(waveglow) return waveglow +def load_hifigan_model(checkpoint_path, super_resolution = False): + conf_name = 'config_32k' if super_resolution else 'config_v1' + conf = os.path.join("hifigan", conf_name + ".json") + with open(conf) as f: + json_config = json.loads(f.read()) + h = AttrDict(json_config) + torch.manual_seed(h.seed) + hifigan = Generator(h).to(torch.device("cuda")) + state_dict_g = torch.load(checkpoint_path, map_location=torch.device("cuda")) + hifigan.load_state_dict(state_dict_g["generator"]) + hifigan.eval() + hifigan.remove_weight_norm() + denoiser = HifiDenoiser(hifigan, mode="normal") + return hifigan, h, denoiser + def preprocess_text(text, raw_input=True): # TODO: Handle ARPAbet sequence = None @@ -150,6 +171,39 @@ class TacotronWaveglowPipeline: self.waveglow = None self.waveglow_filename = None + self.hifigan = None + self.hifigan_denoiser = None + self.hifigan_h = None + self.hifigan_filename = None + self.hifigan_super_resolution = None + self.hifigan_super_resolution_denoiser = None + self.hifigan_super_resolution_h = None + self.hifigan_super_resolution_filename = None + + def maybe_load_hifigan(self, checkpoint_path): + if self.hifigan_filename == checkpoint_path: + print('Hifi-gan model already loaded into memory: {}'.format(checkpoint_path)) + return + print('Dumping previous Hifi-gan model from memory: {}'.format(self.hifigan_filename)) + print('Loading Hifi-gan model into memory: {}'.format(checkpoint_path), flush=True) + hifigan, h, denoiser = load_hifigan_model(checkpoint_path, False) + self.hifigan = hifigan + self.hifigan_denoiser = denoiser + self.hifigan_h = h + self.hifigan_filename = checkpoint_path + + def maybe_load_hifigan_super_resolution(self, checkpoint_path): + if self.hifigan_super_resolution_filename == checkpoint_path: + print('Hifi-gan (SuperRes) model already loaded into memory: {}'.format(checkpoint_path)) + return + print('Dumping previous Hifi-gan (SuperRes) model from memory: {}'.format(self.hifigan_super_resolution_filename)) + print('Loading Hifi-gan (SuperRes) model into memory: {}'.format(checkpoint_path), flush=True) + hifigan, h, denoiser = load_hifigan_model(checkpoint_path, True) + self.hifigan_super_resolution = hifigan + self.hifigan_super_resolution_denoiser = denoiser + self.hifigan_super_resolution_h = h + self.hifigan_super_resolution_filename = checkpoint_path + def maybe_load_waveglow_model(self, checkpoint_path): if self.waveglow_filename == checkpoint_path: print('Waveglow model already loaded into memory: {}'.format(checkpoint_path)) @@ -202,3 +256,82 @@ class TacotronWaveglowPipeline: print('Generating metadata file...') generate_metadata_file(args['output_audio_filename'], args['output_metadata_filename']) + + #sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long() + #mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence) + #if show_graphs: + # plot_data((mel_outputs_postnet.float().data.cpu().numpy()[0], + # alignments.float().data.cpu().numpy()[0].T)) + + print('Running hifigan...') + y_g_hat = self.hifigan(mel_outputs_postnet.float()) + + MAX_WAV_VALUE = 32768.0 + audio = y_g_hat.squeeze() + audio = audio * MAX_WAV_VALUE + audio_denoised = self.hifigan_denoiser(audio.view(1, -1), strength=35)[:, 0] + + # Resample to 32k + audio_denoised = audio_denoised.cpu().numpy().reshape(-1) + + normalize = (MAX_WAV_VALUE / np.max(np.abs(audio_denoised))) ** 0.9 + audio_denoised = audio_denoised * normalize + wave = resampy.resample( + audio_denoised, + self.hifigan_h.sampling_rate, + self.hifigan_super_resolution_h.sampling_rate, + filter="sinc_window", + window=scipy.signal.windows.hann, + num_zeros=8, + ) + wave_out = wave.astype(np.int16) + + # HiFi-GAN super-resolution + wave = wave / MAX_WAV_VALUE + wave = torch.FloatTensor(wave).to(torch.device("cuda")) + new_mel = mel_spectrogram( + wave.unsqueeze(0), + self.hifigan_super_resolution_h.n_fft, + self.hifigan_super_resolution_h.num_mels, + self.hifigan_super_resolution_h.sampling_rate, + self.hifigan_super_resolution_h.hop_size, + self.hifigan_super_resolution_h.win_size, + self.hifigan_super_resolution_h.fmin, + self.hifigan_super_resolution_h.fmax, + ) + y_g_hat2 = self.hifigan_super_resolution(new_mel) + audio2 = y_g_hat2.squeeze() + audio2 = audio2 * MAX_WAV_VALUE + audio2_denoised = self.hifigan_denoiser(audio2.view(1, -1), strength=35)[:, 0] + + # High-pass filter, mixing and denormalizing + audio2_denoised = audio2_denoised.cpu().numpy().reshape(-1) + b = scipy.signal.firwin( + 101, cutoff=10500, fs=h2.sampling_rate, pass_zero=False + ) + y = scipy.signal.lfilter(b, [1.0], audio2_denoised) + superres_strength = 4.0 + y *= superres_strength + y_out = y.astype(np.int16) + y_padded = np.zeros(wave_out.shape) + y_padded[: y_out.shape[0]] = y_out + sr_mix = wave_out + y_padded + sr_mix = sr_mix / normalize + + #print("") + #ipd.display(ipd.Audio(sr_mix.astype(np.int16), rate=h2.sampling_rate)) + + + + + + + + + + + + + + + diff --git a/models/tacotron2/vocodes_server.py b/models/tacotron2/vocodes_server.py index ce5c39cb30..21852fa4c1 100755 --- a/models/tacotron2/vocodes_server.py +++ b/models/tacotron2/vocodes_server.py @@ -28,6 +28,7 @@ from vocodes_common import TacotronWaveglowPipeline from vocodes_common import print_gpu_info from vocodes_common import load_tacotron_model from vocodes_common import load_waveglow_model +from vocodes_common import load_hifigan_model print("TensorFlow version: {}".format(tf.version.VERSION)) @@ -234,6 +235,9 @@ class ApiHandler(): pipeline.maybe_load_waveglow_model(vocoder_checkpoint_path) pipeline.maybe_load_tacotron_model_to_cache(synthesizer_checkpoint_path) + pipeline.maybe_load_hifigan('/home/bt/models/hifigan/universal_hifigan_g_02500000') + pipeline.maybe_load_hifigan_super_resolution('/home/bt/models/hifigan/Superres_Twilight_33000') + inference_args = { 'raw_text': inference_text, 'synthesizer_checkpoint_path': synthesizer_checkpoint_path, From 3d61120ecb0ccc4419f9c37a032ae449829f06a4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 22 Aug 2021 21:15:19 -0400 Subject: [PATCH 0510/1042] this appears to work end-to-end, but I need to output the sound --- models/tacotron2/vocodes_common.py | 105 +++++++++++++++-------------- 1 file changed, 54 insertions(+), 51 deletions(-) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index 8f96b373f4..af915e62b3 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -20,6 +20,8 @@ from hifigan.env import AttrDict from hifigan.models import Generator from hifigan.denoiser import Denoiser as HifiDenoiser from hifigan.meldataset import mel_spectrogram +import resampy +import scipy # For metadata import subprocess @@ -263,63 +265,64 @@ class TacotronWaveglowPipeline: # plot_data((mel_outputs_postnet.float().data.cpu().numpy()[0], # alignments.float().data.cpu().numpy()[0].T)) - print('Running hifigan...') - y_g_hat = self.hifigan(mel_outputs_postnet.float()) + with torch.no_grad(): + print('Running hifigan...') + y_g_hat = self.hifigan(mel_outputs_postnet.float()) - MAX_WAV_VALUE = 32768.0 - audio = y_g_hat.squeeze() - audio = audio * MAX_WAV_VALUE - audio_denoised = self.hifigan_denoiser(audio.view(1, -1), strength=35)[:, 0] + MAX_WAV_VALUE = 32768.0 + audio = y_g_hat.squeeze() + audio = audio * MAX_WAV_VALUE + audio_denoised = self.hifigan_denoiser(audio.view(1, -1), strength=35)[:, 0] - # Resample to 32k - audio_denoised = audio_denoised.cpu().numpy().reshape(-1) + # Resample to 32k + audio_denoised = audio_denoised.cpu().numpy().reshape(-1) - normalize = (MAX_WAV_VALUE / np.max(np.abs(audio_denoised))) ** 0.9 - audio_denoised = audio_denoised * normalize - wave = resampy.resample( - audio_denoised, - self.hifigan_h.sampling_rate, - self.hifigan_super_resolution_h.sampling_rate, - filter="sinc_window", - window=scipy.signal.windows.hann, - num_zeros=8, - ) - wave_out = wave.astype(np.int16) + normalize = (MAX_WAV_VALUE / np.max(np.abs(audio_denoised))) ** 0.9 + audio_denoised = audio_denoised * normalize + wave = resampy.resample( + audio_denoised, + self.hifigan_h.sampling_rate, + self.hifigan_super_resolution_h.sampling_rate, + filter="sinc_window", + window=scipy.signal.windows.hann, + num_zeros=8, + ) + wave_out = wave.astype(np.int16) - # HiFi-GAN super-resolution - wave = wave / MAX_WAV_VALUE - wave = torch.FloatTensor(wave).to(torch.device("cuda")) - new_mel = mel_spectrogram( - wave.unsqueeze(0), - self.hifigan_super_resolution_h.n_fft, - self.hifigan_super_resolution_h.num_mels, - self.hifigan_super_resolution_h.sampling_rate, - self.hifigan_super_resolution_h.hop_size, - self.hifigan_super_resolution_h.win_size, - self.hifigan_super_resolution_h.fmin, - self.hifigan_super_resolution_h.fmax, - ) - y_g_hat2 = self.hifigan_super_resolution(new_mel) - audio2 = y_g_hat2.squeeze() - audio2 = audio2 * MAX_WAV_VALUE - audio2_denoised = self.hifigan_denoiser(audio2.view(1, -1), strength=35)[:, 0] + # HiFi-GAN super-resolution + wave = wave / MAX_WAV_VALUE + wave = torch.FloatTensor(wave).to(torch.device("cuda")) + new_mel = mel_spectrogram( + wave.unsqueeze(0), + self.hifigan_super_resolution_h.n_fft, + self.hifigan_super_resolution_h.num_mels, + self.hifigan_super_resolution_h.sampling_rate, + self.hifigan_super_resolution_h.hop_size, + self.hifigan_super_resolution_h.win_size, + self.hifigan_super_resolution_h.fmin, + self.hifigan_super_resolution_h.fmax, + ) + y_g_hat2 = self.hifigan_super_resolution(new_mel) + audio2 = y_g_hat2.squeeze() + audio2 = audio2 * MAX_WAV_VALUE + audio2_denoised = self.hifigan_denoiser(audio2.view(1, -1), strength=35)[:, 0] - # High-pass filter, mixing and denormalizing - audio2_denoised = audio2_denoised.cpu().numpy().reshape(-1) - b = scipy.signal.firwin( - 101, cutoff=10500, fs=h2.sampling_rate, pass_zero=False - ) - y = scipy.signal.lfilter(b, [1.0], audio2_denoised) - superres_strength = 4.0 - y *= superres_strength - y_out = y.astype(np.int16) - y_padded = np.zeros(wave_out.shape) - y_padded[: y_out.shape[0]] = y_out - sr_mix = wave_out + y_padded - sr_mix = sr_mix / normalize + # High-pass filter, mixing and denormalizing + audio2_denoised = audio2_denoised.cpu().numpy().reshape(-1) + b = scipy.signal.firwin( + 101, cutoff=10500, fs=self.hifigan_super_resolution_h.sampling_rate, pass_zero=False + ) + y = scipy.signal.lfilter(b, [1.0], audio2_denoised) + superres_strength = 4.0 + y *= superres_strength + y_out = y.astype(np.int16) + y_padded = np.zeros(wave_out.shape) + y_padded[: y_out.shape[0]] = y_out + sr_mix = wave_out + y_padded + sr_mix = sr_mix / normalize - #print("") - #ipd.display(ipd.Audio(sr_mix.astype(np.int16), rate=h2.sampling_rate)) + #print("") + #ipd.display(ipd.Audio(sr_mix.astype(np.int16), rate=h2.sampling_rate)) From d7c6e0d68ddf44c30e7b2283141f3de982534dde Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 22 Aug 2021 21:26:11 -0400 Subject: [PATCH 0511/1042] almost working, but sounds super distorted --- models/tacotron2/vocodes_common.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index af915e62b3..f5ea50b289 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -323,7 +323,15 @@ class TacotronWaveglowPipeline: #print("") #ipd.display(ipd.Audio(sr_mix.astype(np.int16), rate=h2.sampling_rate)) - + + print('Encoding and saving audio...') + #save_wav_audio_file(sr_mix, args['output_audio_filename']) + + #output_audio = audio[0].data.cpu().numpy().astype(np.float32) + rate = self.hifigan_super_resolution_h.sampling_rate + output_audio = sr_mix.astype(np.float32) + write_wav(args['output_audio_filename'], rate, output_audio) + From 79ee6e408a9cf0a65e26d821818477d81cca9615 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 22 Aug 2021 21:28:18 -0400 Subject: [PATCH 0512/1042] I think this works and sounds good!! --- models/tacotron2/vocodes_common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index f5ea50b289..6a169caf85 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -329,7 +329,8 @@ class TacotronWaveglowPipeline: #output_audio = audio[0].data.cpu().numpy().astype(np.float32) rate = self.hifigan_super_resolution_h.sampling_rate - output_audio = sr_mix.astype(np.float32) + #output_audio = sr_mix.astype(np.float32) + output_audio = sr_mix.astype(np.int16) write_wav(args['output_audio_filename'], rate, output_audio) From c9112b77149ca4dd5b31bf36ce5eaee90692e9b2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 25 Aug 2021 00:33:24 -0400 Subject: [PATCH 0513/1042] remove unused model code --- models/tacotron2/vocodes_common.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index 6a169caf85..7f79cb5c47 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -259,11 +259,7 @@ class TacotronWaveglowPipeline: print('Generating metadata file...') generate_metadata_file(args['output_audio_filename'], args['output_metadata_filename']) - #sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long() - #mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence) - #if show_graphs: - # plot_data((mel_outputs_postnet.float().data.cpu().numpy()[0], - # alignments.float().data.cpu().numpy()[0].T)) + #return with torch.no_grad(): print('Running hifigan...') From 21cffc1f03fb7a6760f956510fe653edee3ad4fe Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 25 Aug 2021 01:11:36 -0400 Subject: [PATCH 0514/1042] intermediate plumbing for new vocoders --- .env | 6 +-- models/tacotron2/vocodes_server.py | 6 +-- src/clients/tts_inference_sidecar_client.rs | 40 ++++++++++++++--- src/database/enums/mod.rs | 3 +- src/database/enums/vocoder_type.rs | 29 ++++++++++++ src/tts_inference_job.rs | 49 ++++++++++++--------- 6 files changed, 100 insertions(+), 33 deletions(-) create mode 100644 src/database/enums/vocoder_type.rs diff --git a/.env b/.env index 995c16971e..cef4ab21e2 100644 --- a/.env +++ b/.env @@ -45,6 +45,6 @@ TTS_MODEL_CHECK_SCRIPT_NAME=vocodes_model_check.py TTS_INFERENCE_SIDECAR_HOSTNAME=localhost:8000 SIDECAR_MAX_SYNTHESIZER_MODELS=2 -#TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt # THIS SOUNDS HORRID -#TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! -TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_v4.pt +#TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt # THIS SOUNDS HORRID +#TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! +TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4.pt diff --git a/models/tacotron2/vocodes_server.py b/models/tacotron2/vocodes_server.py index 21852fa4c1..dc120d46e4 100755 --- a/models/tacotron2/vocodes_server.py +++ b/models/tacotron2/vocodes_server.py @@ -210,7 +210,7 @@ class ApiHandler(): # Request parameters synthesizer_checkpoint_path = raw_data.get('synthesizer_checkpoint_path') - vocoder_checkpoint_path = raw_data.get('vocoder_checkpoint_path') + waveglow_vocoder_checkpoint_path = raw_data.get('waveglow_vocoder_checkpoint_path') inference_text = raw_data.get('inference_text') output_audio_filename = raw_data.get('output_audio_filename') output_spectrogram_filename = raw_data.get('output_spectrogram_filename') @@ -221,7 +221,7 @@ class ApiHandler(): raw_data.get('maybe_clear_synthesizer_checkpoint_path') print('synthesizer_checkpoint_path: {}'.format(synthesizer_checkpoint_path)) - print('vocoder_checkpoint_path: {}'.format(vocoder_checkpoint_path)) + print('waveglow_vocoder_checkpoint_path: {}'.format(waveglow_vocoder_checkpoint_path)) print('inference_text: {}'.format(inference_text)) print('output_audio_filename: {}'.format(output_audio_filename)) print('output_spectrogram_filename: {}'.format(output_spectrogram_filename)) @@ -232,7 +232,7 @@ class ApiHandler(): if maybe_clear_synthesizer_checkpoint_path: pipeline.uncache_tacotron_model(maybe_clear_synthesizer_checkpoint_path) - pipeline.maybe_load_waveglow_model(vocoder_checkpoint_path) + pipeline.maybe_load_waveglow_model(waveglow_vocoder_checkpoint_path) pipeline.maybe_load_tacotron_model_to_cache(synthesizer_checkpoint_path) pipeline.maybe_load_hifigan('/home/bt/models/hifigan/universal_hifigan_g_02500000') diff --git a/src/clients/tts_inference_sidecar_client.rs b/src/clients/tts_inference_sidecar_client.rs index 2d762df964..4699c740a3 100644 --- a/src/clients/tts_inference_sidecar_client.rs +++ b/src/clients/tts_inference_sidecar_client.rs @@ -5,17 +5,27 @@ use hyper::client::Client; use hyper::{Body, Request}; use log::info; use std::path::Path; +use crate::database::enums::vocoder_type::VocoderType; pub struct TtsInferenceSidecarClient { hostname: String, - //client: Client, } #[derive(Serialize)] struct InferenceRequest { - pub vocoder_checkpoint_path : String, + // Vocoder information + pub vocoder_type: VocoderType, + pub waveglow_vocoder_checkpoint_path : String, + pub hifigan_vocoder_checkpoint_path: String, + pub hifigan_supersample_vocoder_checkpoint_path: String, + + // Synthesizer information pub synthesizer_checkpoint_path : String, + + // Text pub inference_text : String, + + // Output information pub output_audio_filename : String, pub output_spectrogram_filename : String, pub output_metadata_filename : String, @@ -39,18 +49,33 @@ impl TtsInferenceSidecarClient { &self, raw_text: &str, synthesizer_checkpoint_path: P, - vocoder_checkpoint_path: P, + vocoder_type: VocoderType, + hifigan_vocoder_checkpoint_path: P, + hifigan_supersample_vocoder_checkpoint_path: P, + waveglow_vocoder_checkpoint_path: P, output_audio_filename: P, output_spectrogram_filename: P, output_metadata_filename: P, maybe_unload_model_path: Option, ) -> AnyhowResult<()> { - let vocoder_checkpoint_path = vocoder_checkpoint_path + let waveglow_vocoder_checkpoint_path = waveglow_vocoder_checkpoint_path .as_ref() .to_str() .map(|s| s.to_string()) - .ok_or(anyhow!("bad vocoder path"))?; + .ok_or(anyhow!("bad waveglow vocoder path"))?; + + let hifigan_vocoder_checkpoint_path = hifigan_vocoder_checkpoint_path + .as_ref() + .to_str() + .map(|s| s.to_string()) + .ok_or(anyhow!("bad hifigan vocoder path"))?; + + let hifigan_supersample_vocoder_checkpoint_path = hifigan_supersample_vocoder_checkpoint_path + .as_ref() + .to_str() + .map(|s| s.to_string()) + .ok_or(anyhow!("bad hifigan supersample vocoder path"))?; let synthesizer_checkpoint_path = synthesizer_checkpoint_path .as_ref() @@ -78,7 +103,10 @@ impl TtsInferenceSidecarClient { let request = InferenceRequest { inference_text: raw_text.to_string(), - vocoder_checkpoint_path, + vocoder_type, + waveglow_vocoder_checkpoint_path, + hifigan_vocoder_checkpoint_path, + hifigan_supersample_vocoder_checkpoint_path, synthesizer_checkpoint_path, output_audio_filename, output_spectrogram_filename, diff --git a/src/database/enums/mod.rs b/src/database/enums/mod.rs index fa9a069908..7ffceb80cc 100644 --- a/src/database/enums/mod.rs +++ b/src/database/enums/mod.rs @@ -1 +1,2 @@ -pub mod record_visibility; \ No newline at end of file +pub mod record_visibility; +pub mod vocoder_type; \ No newline at end of file diff --git a/src/database/enums/vocoder_type.rs b/src/database/enums/vocoder_type.rs new file mode 100644 index 0000000000..cb1ea153ae --- /dev/null +++ b/src/database/enums/vocoder_type.rs @@ -0,0 +1,29 @@ +//! These denote the pretrained vocoders + +use anyhow::anyhow; +use crate::util::anyhow_result::AnyhowResult; + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] +#[serde(rename_all = "lowercase")] +#[sqlx(rename_all = "lowercase")] +pub enum VocoderType { + HifiGanSuperSample, + WaveGlow, +} + +impl VocoderType { + pub fn to_str(&self) -> &'static str { + match self { + VocoderType::HifiGanSuperSample => "hifigan-supersample", + VocoderType::WaveGlow => "waveglow", + } + } + + pub fn from_str(vocoder_type: &str) -> AnyhowResult { + match vocoder_type { + "hifigan-supersample" => Ok(VocoderType::HifiGanSuperSample), + "waveglow" => Ok(VocoderType::WaveGlow), + _ => Err(anyhow!("invalid value: {:?}", vocoder_type)), + } + } +} diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 6bcfd18f4d..e157d02c2e 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -61,6 +61,7 @@ use std::process::Command; use std::thread; use std::time::Duration; use tempdir::TempDir; +use crate::database::enums::vocoder_type::VocoderType; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -102,8 +103,14 @@ struct Inferencer { pub virtual_model_lfu: SyncVirtualLfuCache, pub cache_miss_strategizers: SyncMultiCacheMissStrategizer, - // Command to run - pub tts_vocoder_model_filename: String, + // Waveglow vocoder filename + pub waveglow_vocoder_model_filename: String, + + // Hifigan vocoder filename + pub hifigan_vocoder_model_filename: String, + + // Hifigan supersampling vocoder filename + pub hifigan_supersampling_vocoder_model_filename: String, // Sleep between batches pub job_batch_wait_millis: u64, @@ -209,8 +216,8 @@ async fn main() -> AnyhowResult<()> { semi_persistent_cache.create_tts_synthesizer_model_path()?; semi_persistent_cache.create_tts_vocoder_model_path()?; - let tts_vocoder_model_filename = easyenv::get_env_string_or_default( - "TTS_VOCODER_MODEL_FILENAME", "waveglow.pth"); + let waveglow_vocoder_model_filename = easyenv::get_env_string_or_default( + "TTS_WAVEGLOW_VOCODER_MODEL_FILENAME", "waveglow.pth"); let sidecar_max_synthesizer_models = easyenv::get_env_num( "SIDECAR_MAX_SYNTHESIZER_MODELS", 3)?; @@ -253,9 +260,6 @@ async fn main() -> AnyhowResult<()> { mysql_pool, public_bucket_client, private_bucket_client, - //ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, - //ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, - //imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, tts_inference_command, tts_inference_sidecar_client, virtual_model_lfu: virtual_lfu_cache, @@ -263,7 +267,9 @@ async fn main() -> AnyhowResult<()> { bucket_path_unifier: BucketPathUnifier::default_paths(), semi_persistent_cache, firehose_publisher, - tts_vocoder_model_filename, + waveglow_vocoder_model_filename, + hifigan_vocoder_model_filename: "".to_string(), // TODO + hifigan_supersampling_vocoder_model_filename: "".to_string(), // TODO job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as i32, job_batch_size: common_env.job_batch_size, @@ -507,25 +513,25 @@ async fn process_job( return Ok(()) } - // ==================== CONFIRM OR DOWNLOAD TTS VOCODER MODEL ==================== // + // ==================== CONFIRM OR DOWNLOAD WAVEGLOW VOCODER MODEL ==================== // - let tts_vocoder_model_filename = inferencer.tts_vocoder_model_filename.clone(); - let tts_vocoder_model_fs_path = inferencer.semi_persistent_cache.tts_vocoder_model_path(&tts_vocoder_model_filename); + let waveglow_vocoder_model_filename = inferencer.waveglow_vocoder_model_filename.clone(); + let waveglow_vocoder_model_fs_path = inferencer.semi_persistent_cache.tts_vocoder_model_path(&waveglow_vocoder_model_filename); - if !tts_vocoder_model_fs_path.exists() { - warn!("Vocoder model file does not exist: {:?}", &tts_vocoder_model_fs_path); + if !waveglow_vocoder_model_fs_path.exists() { + warn!("Waveglow vocoder model file does not exist: {:?}", &waveglow_vocoder_model_fs_path); - let tts_vocoder_model_object_path = inferencer.bucket_path_unifier - .tts_pretrained_vocoders_path(&tts_vocoder_model_filename); + let waveglow_vocoder_model_object_path = inferencer.bucket_path_unifier + .tts_pretrained_vocoders_path(&waveglow_vocoder_model_filename); - info!("Download vocoder from bucket path: {:?}", &tts_vocoder_model_object_path); + info!("Download waveglow vocoder from bucket path: {:?}", &waveglow_vocoder_model_object_path); inferencer.private_bucket_client.download_file_to_disk( - &tts_vocoder_model_object_path, - &tts_vocoder_model_fs_path + &waveglow_vocoder_model_object_path, + &waveglow_vocoder_model_fs_path ).await?; - info!("Downloaded tts vocoder model from bucket!"); + info!("Downloaded waveglow vocoder model from bucket!"); } // // ==================== LOOK UP TTS SYNTHESIZER RECORD (WHICH CONTAINS ITS BUCKET PATH) ==================== // @@ -611,7 +617,10 @@ async fn process_job( inferencer.tts_inference_sidecar_client.request_inference( &job.raw_inference_text, &tts_synthesizer_fs_path, - &tts_vocoder_model_fs_path, + VocoderType::WaveGlow, + &PathBuf::new(), // TODO + &PathBuf::new(), // TODO + &waveglow_vocoder_model_fs_path, &output_audio_fs_path, &output_spectrogram_fs_path, &output_metadata_fs_path, From a61e144ecbf55969841add1904ec30e0650ae82d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 25 Aug 2021 01:32:37 -0400 Subject: [PATCH 0515/1042] intermediate plumbing for new vocoders (2) --- .env | 3 +++ models/tacotron2/vocodes_common.py | 2 +- src/clients/tts_inference_sidecar_client.rs | 12 ++++++------ src/database/enums/vocoder_type.rs | 6 +++--- src/tts_inference_job.rs | 20 +++++++++++++------- src/util/semi_persistent_cache_dir.rs | 18 +++++++++--------- 6 files changed, 35 insertions(+), 26 deletions(-) diff --git a/.env b/.env index cef4ab21e2..9793be9269 100644 --- a/.env +++ b/.env @@ -48,3 +48,6 @@ SIDECAR_MAX_SYNTHESIZER_MODELS=2 #TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt # THIS SOUNDS HORRID #TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4.pt + +TTS_HIFIGAN_VOCODER_MODEL_FILENAME=todo +TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=todo diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index 7f79cb5c47..3fa33bd77e 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -259,7 +259,7 @@ class TacotronWaveglowPipeline: print('Generating metadata file...') generate_metadata_file(args['output_audio_filename'], args['output_metadata_filename']) - #return + return with torch.no_grad(): print('Running hifigan...') diff --git a/src/clients/tts_inference_sidecar_client.rs b/src/clients/tts_inference_sidecar_client.rs index 4699c740a3..354cdfd1b0 100644 --- a/src/clients/tts_inference_sidecar_client.rs +++ b/src/clients/tts_inference_sidecar_client.rs @@ -1,11 +1,11 @@ use actix_web::http::header; use anyhow::anyhow; +use crate::database::enums::vocoder_type::VocoderType; use crate::util::anyhow_result::AnyhowResult; use hyper::client::Client; use hyper::{Body, Request}; use log::info; use std::path::Path; -use crate::database::enums::vocoder_type::VocoderType; pub struct TtsInferenceSidecarClient { hostname: String, @@ -17,7 +17,7 @@ struct InferenceRequest { pub vocoder_type: VocoderType, pub waveglow_vocoder_checkpoint_path : String, pub hifigan_vocoder_checkpoint_path: String, - pub hifigan_supersample_vocoder_checkpoint_path: String, + pub hifigan_superres_vocoder_checkpoint_path: String, // Synthesizer information pub synthesizer_checkpoint_path : String, @@ -51,7 +51,7 @@ impl TtsInferenceSidecarClient { synthesizer_checkpoint_path: P, vocoder_type: VocoderType, hifigan_vocoder_checkpoint_path: P, - hifigan_supersample_vocoder_checkpoint_path: P, + hifigan_superres_vocoder_checkpoint_path: P, waveglow_vocoder_checkpoint_path: P, output_audio_filename: P, output_spectrogram_filename: P, @@ -71,11 +71,11 @@ impl TtsInferenceSidecarClient { .map(|s| s.to_string()) .ok_or(anyhow!("bad hifigan vocoder path"))?; - let hifigan_supersample_vocoder_checkpoint_path = hifigan_supersample_vocoder_checkpoint_path + let hifigan_superres_vocoder_checkpoint_path = hifigan_superres_vocoder_checkpoint_path .as_ref() .to_str() .map(|s| s.to_string()) - .ok_or(anyhow!("bad hifigan supersample vocoder path"))?; + .ok_or(anyhow!("bad hifigan super resolution vocoder path"))?; let synthesizer_checkpoint_path = synthesizer_checkpoint_path .as_ref() @@ -106,7 +106,7 @@ impl TtsInferenceSidecarClient { vocoder_type, waveglow_vocoder_checkpoint_path, hifigan_vocoder_checkpoint_path, - hifigan_supersample_vocoder_checkpoint_path, + hifigan_superres_vocoder_checkpoint_path, synthesizer_checkpoint_path, output_audio_filename, output_spectrogram_filename, diff --git a/src/database/enums/vocoder_type.rs b/src/database/enums/vocoder_type.rs index cb1ea153ae..e75d3209f8 100644 --- a/src/database/enums/vocoder_type.rs +++ b/src/database/enums/vocoder_type.rs @@ -7,21 +7,21 @@ use crate::util::anyhow_result::AnyhowResult; #[serde(rename_all = "lowercase")] #[sqlx(rename_all = "lowercase")] pub enum VocoderType { - HifiGanSuperSample, + HifiGanSuperResolution, WaveGlow, } impl VocoderType { pub fn to_str(&self) -> &'static str { match self { - VocoderType::HifiGanSuperSample => "hifigan-supersample", + VocoderType::HifiGanSuperResolution => "hifigan-superres", VocoderType::WaveGlow => "waveglow", } } pub fn from_str(vocoder_type: &str) -> AnyhowResult { match vocoder_type { - "hifigan-supersample" => Ok(VocoderType::HifiGanSuperSample), + "hifigan-superres" => Ok(VocoderType::HifiGanSuperResolution), "waveglow" => Ok(VocoderType::WaveGlow), _ => Err(anyhow!("invalid value: {:?}", vocoder_type)), } diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index e157d02c2e..9bfc2b387c 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -22,6 +22,7 @@ use anyhow::{anyhow, Error}; use chrono::{Utc, DateTime, TimeZone}; use crate::clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; use crate::common_env::CommonEnv; +use crate::database::enums::vocoder_type::VocoderType; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_inference_job_queries::get_tts_model_by_token; use crate::job_queries::tts_inference_job_queries::grab_job_lock_and_mark_pending; @@ -61,7 +62,6 @@ use std::process::Command; use std::thread; use std::time::Duration; use tempdir::TempDir; -use crate::database::enums::vocoder_type::VocoderType; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -109,8 +109,8 @@ struct Inferencer { // Hifigan vocoder filename pub hifigan_vocoder_model_filename: String, - // Hifigan supersampling vocoder filename - pub hifigan_supersampling_vocoder_model_filename: String, + // Hifigan super resolution vocoder filename + pub hifigan_superres_vocoder_model_filename: String, // Sleep between batches pub job_batch_wait_millis: u64, @@ -214,11 +214,17 @@ async fn main() -> AnyhowResult<()> { info!("Creating pod semi-persistent cache dirs..."); semi_persistent_cache.create_tts_synthesizer_model_path()?; - semi_persistent_cache.create_tts_vocoder_model_path()?; + semi_persistent_cache.create_tts_pretrained_vocoder_model_path()?; let waveglow_vocoder_model_filename = easyenv::get_env_string_or_default( "TTS_WAVEGLOW_VOCODER_MODEL_FILENAME", "waveglow.pth"); + let hifigan_vocoder_model_filename = easyenv::get_env_string_or_default( + "TTS_HIFIGAN_VOCODER_MODEL_FILENAME", "hifigan.pth"); + + let hifigan_superres_vocoder_model_filename = easyenv::get_env_string_or_default( + "TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME", "hifigan_superres.pth"); + let sidecar_max_synthesizer_models = easyenv::get_env_num( "SIDECAR_MAX_SYNTHESIZER_MODELS", 3)?; @@ -268,8 +274,8 @@ async fn main() -> AnyhowResult<()> { semi_persistent_cache, firehose_publisher, waveglow_vocoder_model_filename, - hifigan_vocoder_model_filename: "".to_string(), // TODO - hifigan_supersampling_vocoder_model_filename: "".to_string(), // TODO + hifigan_vocoder_model_filename, + hifigan_superres_vocoder_model_filename, job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as i32, job_batch_size: common_env.job_batch_size, @@ -516,7 +522,7 @@ async fn process_job( // ==================== CONFIRM OR DOWNLOAD WAVEGLOW VOCODER MODEL ==================== // let waveglow_vocoder_model_filename = inferencer.waveglow_vocoder_model_filename.clone(); - let waveglow_vocoder_model_fs_path = inferencer.semi_persistent_cache.tts_vocoder_model_path(&waveglow_vocoder_model_filename); + let waveglow_vocoder_model_fs_path = inferencer.semi_persistent_cache.tts_pretrained_vocoder_model_path(&waveglow_vocoder_model_filename); if !waveglow_vocoder_model_fs_path.exists() { warn!("Waveglow vocoder model file does not exist: {:?}", &waveglow_vocoder_model_fs_path); diff --git a/src/util/semi_persistent_cache_dir.rs b/src/util/semi_persistent_cache_dir.rs index 5cd05d51f5..93a34ad372 100644 --- a/src/util/semi_persistent_cache_dir.rs +++ b/src/util/semi_persistent_cache_dir.rs @@ -8,7 +8,7 @@ pub struct SemiPersistentCacheDir { cache_root: PathBuf, tts_synthesizer_model_root: PathBuf, - tts_vocoder_model_root: PathBuf, + tts_pretrained_vocoder_model_root: PathBuf, w2l_model_root: PathBuf, w2l_end_bump_root: PathBuf, @@ -32,7 +32,7 @@ impl SemiPersistentCacheDir { cache_root: cache_root.clone(), tts_synthesizer_model_root: cache_root.join("tts/synthesizer_models/"), - tts_vocoder_model_root: cache_root.join("tts/vocoder_models/"), + tts_pretrained_vocoder_model_root: cache_root.join("tts/vocoder_models_pretrained/"), w2l_model_root: cache_root.join("w2l/models/"), w2l_end_bump_root: cache_root.join("w2l/end_bumps/"), @@ -59,19 +59,19 @@ impl SemiPersistentCacheDir { Ok(()) } - // ==================== TTS VOCODER MODELS ==================== + // ==================== TTS PRETRAINED VOCODER MODELS (WAVEGLOW, HIFIGAN, ETC) ==================== /// We'll start with just a few vocoders, but this may grow. - pub fn tts_vocoder_model_path(&self, model_filename: &str) -> PathBuf { - self.tts_vocoder_model_root.join(model_filename) + pub fn tts_pretrained_vocoder_model_path(&self, model_filename: &str) -> PathBuf { + self.tts_pretrained_vocoder_model_root.join(model_filename) } - pub fn tts_vocoder_model_directory(&self) -> &Path { - &self.tts_vocoder_model_root + pub fn tts_pretrained_vocoder_model_directory(&self) -> &Path { + &self.tts_pretrained_vocoder_model_root } - pub fn create_tts_vocoder_model_path(&self) -> AnyhowResult<()> { - let _ = fs::create_dir_all(self.tts_vocoder_model_directory())?; + pub fn create_tts_pretrained_vocoder_model_path(&self) -> AnyhowResult<()> { + let _ = fs::create_dir_all(self.tts_pretrained_vocoder_model_directory())?; Ok(()) } From 6ddc641475fcab6c5664624cfbca910d49bd8ce4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 25 Aug 2021 01:39:29 -0400 Subject: [PATCH 0516/1042] intermediate plumbing for new vocoders (3) --- models/tacotron2/vocodes_common.py | 33 +++++++++++------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index 3fa33bd77e..2c1a7d5755 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -246,21 +246,27 @@ class TacotronWaveglowPipeline: with torch.no_grad(): mel_outputs, mel_outputs_postnet, _, alignments = tacotron.inference(sequence) - with torch.no_grad(): - sigma = 0.8 - audio = self.waveglow.infer(mel_outputs_postnet, sigma=sigma) print('Saving spectrogram as JSON...') save_spectorgram_json_file(mel_outputs_postnet, args['output_spectrogram_filename']) - print('Encoding and saving audio...') - save_wav_audio_file(audio, args['output_audio_filename']) + if False: + self.vocode_waveglow(args, mel_outputs_postnet) + else: + self.vocode_hifigan_superres(args, mel_outputs_postnet) print('Generating metadata file...') generate_metadata_file(args['output_audio_filename'], args['output_metadata_filename']) - return + def vocode_waveglow(self, args, mel_outputs_postnet): + with torch.no_grad(): + sigma = 0.8 + audio = self.waveglow.infer(mel_outputs_postnet, sigma=sigma) + print('Encoding and saving audio...') + save_wav_audio_file(audio, args['output_audio_filename']) + + def vocode_hifigan_superres(self, args, mel_outputs_postnet): with torch.no_grad(): print('Running hifigan...') y_g_hat = self.hifigan(mel_outputs_postnet.float()) @@ -328,18 +334,3 @@ class TacotronWaveglowPipeline: #output_audio = sr_mix.astype(np.float32) output_audio = sr_mix.astype(np.int16) write_wav(args['output_audio_filename'], rate, output_audio) - - - - - - - - - - - - - - - From f4c8bb0095fdc5bde0b74ab06460a053b7d37b84 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 25 Aug 2021 01:59:30 -0400 Subject: [PATCH 0517/1042] plumbing for vocoders (mostly done) --- .env | 4 +-- models/tacotron2/vocodes_common.py | 12 ++------ models/tacotron2/vocodes_server.py | 19 ++++++------ src/tts_inference_job.rs | 46 ++++++++++++++++++++++++++++-- 4 files changed, 58 insertions(+), 23 deletions(-) diff --git a/.env b/.env index 9793be9269..aeee919a5d 100644 --- a/.env +++ b/.env @@ -49,5 +49,5 @@ SIDECAR_MAX_SYNTHESIZER_MODELS=2 #TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4.pt -TTS_HIFIGAN_VOCODER_MODEL_FILENAME=todo -TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=todo +TTS_HIFIGAN_VOCODER_MODEL_FILENAME=hifigan_universal_g_02500000.pt +TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index 2c1a7d5755..3029392c17 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -246,7 +246,6 @@ class TacotronWaveglowPipeline: with torch.no_grad(): mel_outputs, mel_outputs_postnet, _, alignments = tacotron.inference(sequence) - print('Saving spectrogram as JSON...') save_spectorgram_json_file(mel_outputs_postnet, args['output_spectrogram_filename']) @@ -259,6 +258,7 @@ class TacotronWaveglowPipeline: generate_metadata_file(args['output_audio_filename'], args['output_metadata_filename']) def vocode_waveglow(self, args, mel_outputs_postnet): + print('Running melgan...') with torch.no_grad(): sigma = 0.8 audio = self.waveglow.infer(mel_outputs_postnet, sigma=sigma) @@ -267,8 +267,8 @@ class TacotronWaveglowPipeline: save_wav_audio_file(audio, args['output_audio_filename']) def vocode_hifigan_superres(self, args, mel_outputs_postnet): + print('Running hifigan...') with torch.no_grad(): - print('Running hifigan...') y_g_hat = self.hifigan(mel_outputs_postnet.float()) MAX_WAV_VALUE = 32768.0 @@ -292,6 +292,7 @@ class TacotronWaveglowPipeline: wave_out = wave.astype(np.int16) # HiFi-GAN super-resolution + print('Running hifigan super resolution...') wave = wave / MAX_WAV_VALUE wave = torch.FloatTensor(wave).to(torch.device("cuda")) new_mel = mel_spectrogram( @@ -323,14 +324,7 @@ class TacotronWaveglowPipeline: sr_mix = wave_out + y_padded sr_mix = sr_mix / normalize - #print("") - #ipd.display(ipd.Audio(sr_mix.astype(np.int16), rate=h2.sampling_rate)) - print('Encoding and saving audio...') - #save_wav_audio_file(sr_mix, args['output_audio_filename']) - - #output_audio = audio[0].data.cpu().numpy().astype(np.float32) rate = self.hifigan_super_resolution_h.sampling_rate - #output_audio = sr_mix.astype(np.float32) output_audio = sr_mix.astype(np.int16) write_wav(args['output_audio_filename'], rate, output_audio) diff --git a/models/tacotron2/vocodes_server.py b/models/tacotron2/vocodes_server.py index dc120d46e4..a15a57f983 100755 --- a/models/tacotron2/vocodes_server.py +++ b/models/tacotron2/vocodes_server.py @@ -211,6 +211,8 @@ class ApiHandler(): # Request parameters synthesizer_checkpoint_path = raw_data.get('synthesizer_checkpoint_path') waveglow_vocoder_checkpoint_path = raw_data.get('waveglow_vocoder_checkpoint_path') + hifigan_vocoder_checkpoint_path = raw_data.get('hifigan_vocoder_checkpoint_path') + hifigan_superres_vocoder_checkpoint_path = raw_data.get('hifigan_superres_vocoder_checkpoint_path') inference_text = raw_data.get('inference_text') output_audio_filename = raw_data.get('output_audio_filename') output_spectrogram_filename = raw_data.get('output_spectrogram_filename') @@ -222,6 +224,8 @@ class ApiHandler(): print('synthesizer_checkpoint_path: {}'.format(synthesizer_checkpoint_path)) print('waveglow_vocoder_checkpoint_path: {}'.format(waveglow_vocoder_checkpoint_path)) + print('hifigan_vocoder_checkpoint_path: {}'.format(hifigan_vocoder_checkpoint_path)) + print('hifigan_superres_vocoder_checkpoint_path: {}'.format(hifigan_superres_vocoder_checkpoint_path)) print('inference_text: {}'.format(inference_text)) print('output_audio_filename: {}'.format(output_audio_filename)) print('output_spectrogram_filename: {}'.format(output_spectrogram_filename)) @@ -229,15 +233,15 @@ class ApiHandler(): print('maybe_clear_synthesizer_checkpoint_path: {}'.format( maybe_clear_synthesizer_checkpoint_path)) + pipeline.maybe_load_waveglow_model(waveglow_vocoder_checkpoint_path) + pipeline.maybe_load_hifigan(hifigan_vocoder_checkpoint_path) + pipeline.maybe_load_hifigan_super_resolution(hifigan_superres_vocoder_checkpoint_path) + if maybe_clear_synthesizer_checkpoint_path: pipeline.uncache_tacotron_model(maybe_clear_synthesizer_checkpoint_path) - pipeline.maybe_load_waveglow_model(waveglow_vocoder_checkpoint_path) pipeline.maybe_load_tacotron_model_to_cache(synthesizer_checkpoint_path) - pipeline.maybe_load_hifigan('/home/bt/models/hifigan/universal_hifigan_g_02500000') - pipeline.maybe_load_hifigan_super_resolution('/home/bt/models/hifigan/Superres_Twilight_33000') - inference_args = { 'raw_text': inference_text, 'synthesizer_checkpoint_path': synthesizer_checkpoint_path, @@ -248,10 +252,6 @@ class ApiHandler(): pipeline.infer(inference_args) - #response.content_type = 'audio/ogg' - #with open(out_file, mode='rb') as f: - # response.data = f.read() - def main(): parser = argparse.ArgumentParser() parser.add_argument('--port', type=int, default=8000) @@ -260,8 +260,7 @@ def main(): api = falcon.API(middleware=[MultipartMiddleware()]) api.add_route('/', IndexHandler()) api.add_route('/infer', ApiHandler()) - #api.add_static_route('/script', os.path.abspath('./script')) - #api.add_static_route('/sound', os.path.abspath('./sound')) + print('Serving on 0.0.0.0:%d' % args.port) simple_server.make_server('0.0.0.0', args.port, api).serve_forever() diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 9bfc2b387c..b59ce4608b 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -540,6 +540,48 @@ async fn process_job( info!("Downloaded waveglow vocoder model from bucket!"); } + // ==================== CONFIRM OR DOWNLOAD HIFIGAN (NORMAL) VOCODER MODEL ==================== // + + let hifigan_vocoder_model_filename = inferencer.hifigan_vocoder_model_filename.clone(); + let hifigan_vocoder_model_fs_path = inferencer.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_vocoder_model_filename); + + if !hifigan_vocoder_model_fs_path.exists() { + warn!("Hifigan vocoder model file does not exist: {:?}", &hifigan_vocoder_model_fs_path); + + let hifigan_vocoder_model_object_path = inferencer.bucket_path_unifier + .tts_pretrained_vocoders_path(&hifigan_vocoder_model_filename); + + info!("Download hifigan vocoder from bucket path: {:?}", &hifigan_vocoder_model_object_path); + + inferencer.private_bucket_client.download_file_to_disk( + &hifigan_vocoder_model_object_path, + &hifigan_vocoder_model_fs_path + ).await?; + + info!("Downloaded hifigan vocoder model from bucket!"); + } + + // ==================== CONFIRM OR DOWNLOAD HIFIGAN (SUPERRES) VOCODER MODEL ==================== // + + let hifigan_superres_vocoder_model_filename = inferencer.hifigan_superres_vocoder_model_filename.clone(); + let hifigan_superres_vocoder_model_fs_path = inferencer.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_superres_vocoder_model_filename); + + if !hifigan_superres_vocoder_model_fs_path.exists() { + warn!("Hifigan superres vocoder model file does not exist: {:?}", &hifigan_superres_vocoder_model_fs_path); + + let hifigan_superres_vocoder_model_object_path = inferencer.bucket_path_unifier + .tts_pretrained_vocoders_path(&hifigan_superres_vocoder_model_filename); + + info!("Download hifigan superres vocoder from bucket path: {:?}", &hifigan_superres_vocoder_model_object_path); + + inferencer.private_bucket_client.download_file_to_disk( + &hifigan_superres_vocoder_model_object_path, + &hifigan_superres_vocoder_model_fs_path + ).await?; + + info!("Downloaded hifigan superres vocoder model from bucket!"); + } + // // ==================== LOOK UP TTS SYNTHESIZER RECORD (WHICH CONTAINS ITS BUCKET PATH) ==================== // // // info!("Looking up TTS model by token: {}", &job.model_token); @@ -624,8 +666,8 @@ async fn process_job( &job.raw_inference_text, &tts_synthesizer_fs_path, VocoderType::WaveGlow, - &PathBuf::new(), // TODO - &PathBuf::new(), // TODO + &hifigan_vocoder_model_fs_path, + &hifigan_superres_vocoder_model_fs_path, &waveglow_vocoder_model_fs_path, &output_audio_fs_path, &output_spectrogram_fs_path, From 90e1db42c481884152f9fb9f2d0e7e24dce24fde Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 25 Aug 2021 02:21:59 -0400 Subject: [PATCH 0518/1042] Sidecar supports both vocoders and uses the one in the HTTP request --- models/tacotron2/vocodes_common.py | 13 ++++++++----- models/tacotron2/vocodes_server.py | 3 +++ src/database/enums/vocoder_type.rs | 4 ++++ src/tts_inference_job.rs | 2 +- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index 3029392c17..423d5fe02b 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -234,6 +234,7 @@ class TacotronWaveglowPipeline: def infer(self, args): assert('synthesizer_checkpoint_path' in args) + assert('vocoder_type' in args) assert('raw_text' in args) assert('output_audio_filename' in args) assert('output_spectrogram_filename' in args) @@ -246,13 +247,15 @@ class TacotronWaveglowPipeline: with torch.no_grad(): mel_outputs, mel_outputs_postnet, _, alignments = tacotron.inference(sequence) - print('Saving spectrogram as JSON...') - save_spectorgram_json_file(mel_outputs_postnet, args['output_spectrogram_filename']) - - if False: + if args['vocoder_type'] == 'hifigan-superres': + self.vocode_hifigan_superres(args, mel_outputs_postnet) + elif args['vocoder_type'] == 'waveglow': self.vocode_waveglow(args, mel_outputs_postnet) else: - self.vocode_hifigan_superres(args, mel_outputs_postnet) + raise Exception('Wrong vocoder type: {}'.format(args['vocoder_type'])) + + print('Saving spectrogram as JSON...') + save_spectorgram_json_file(mel_outputs_postnet, args['output_spectrogram_filename']) print('Generating metadata file...') generate_metadata_file(args['output_audio_filename'], args['output_metadata_filename']) diff --git a/models/tacotron2/vocodes_server.py b/models/tacotron2/vocodes_server.py index a15a57f983..e940b9b240 100755 --- a/models/tacotron2/vocodes_server.py +++ b/models/tacotron2/vocodes_server.py @@ -210,6 +210,7 @@ class ApiHandler(): # Request parameters synthesizer_checkpoint_path = raw_data.get('synthesizer_checkpoint_path') + vocoder_type = raw_data.get('vocoder_type') waveglow_vocoder_checkpoint_path = raw_data.get('waveglow_vocoder_checkpoint_path') hifigan_vocoder_checkpoint_path = raw_data.get('hifigan_vocoder_checkpoint_path') hifigan_superres_vocoder_checkpoint_path = raw_data.get('hifigan_superres_vocoder_checkpoint_path') @@ -223,6 +224,7 @@ class ApiHandler(): raw_data.get('maybe_clear_synthesizer_checkpoint_path') print('synthesizer_checkpoint_path: {}'.format(synthesizer_checkpoint_path)) + print('vocoder_type: {}'.format(vocoder_type)) print('waveglow_vocoder_checkpoint_path: {}'.format(waveglow_vocoder_checkpoint_path)) print('hifigan_vocoder_checkpoint_path: {}'.format(hifigan_vocoder_checkpoint_path)) print('hifigan_superres_vocoder_checkpoint_path: {}'.format(hifigan_superres_vocoder_checkpoint_path)) @@ -245,6 +247,7 @@ class ApiHandler(): inference_args = { 'raw_text': inference_text, 'synthesizer_checkpoint_path': synthesizer_checkpoint_path, + 'vocoder_type': vocoder_type, 'output_audio_filename': output_audio_filename, 'output_spectrogram_filename': output_spectrogram_filename, 'output_metadata_filename': output_metadata_filename, diff --git a/src/database/enums/vocoder_type.rs b/src/database/enums/vocoder_type.rs index e75d3209f8..8ebcb54278 100644 --- a/src/database/enums/vocoder_type.rs +++ b/src/database/enums/vocoder_type.rs @@ -7,7 +7,11 @@ use crate::util::anyhow_result::AnyhowResult; #[serde(rename_all = "lowercase")] #[sqlx(rename_all = "lowercase")] pub enum VocoderType { + /// Current state of the art vocoder + #[serde(rename = "hifigan-superres")] HifiGanSuperResolution, + + /// Older WaveGlow vocoder WaveGlow, } diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index b59ce4608b..9a1291ecfa 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -665,7 +665,7 @@ async fn process_job( inferencer.tts_inference_sidecar_client.request_inference( &job.raw_inference_text, &tts_synthesizer_fs_path, - VocoderType::WaveGlow, + VocoderType::HifiGanSuperResolution, &hifigan_vocoder_model_fs_path, &hifigan_superres_vocoder_model_fs_path, &waveglow_vocoder_model_fs_path, From c396a9188132a780e0e918c5379e2dae5ad061ec Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 25 Aug 2021 04:00:57 -0400 Subject: [PATCH 0519/1042] Massive update + schema change. Add vocoder flexibility --- README.md | 8 + migrations/2021-05-15-071601_assets/up.sql | 3 + .../2021-05-15-212913_inference_jobs/up.sql | 3 + .../up.sql | 3 + .../down.sql | 12 + .../up.sql | 15 + models/tacotron2/vocodes_common.py | 2 +- sqlx-data.json | 2357 +++++++++-------- src/database/queries/query_tts_model.rs | 11 + src/database/queries/query_tts_result.rs | 16 + .../endpoints/tts/edit_tts_model.rs | 13 + src/job_queries/tts_inference_job_queries.rs | 7 + src/tts_inference_job.rs | 10 +- 13 files changed, 1312 insertions(+), 1148 deletions(-) create mode 100644 migrations/2021-09-25-010101_alter_tables_add_vocoder_type/down.sql create mode 100644 migrations/2021-09-25-010101_alter_tables_add_vocoder_type/up.sql diff --git a/README.md b/README.md index 2e47d24bc2..8e59abeffe 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,14 @@ This is the main user account monolith that we'll also bake other pieces into. Local development ----------------- +### Debugging and Fixing CUDA/PyTorch + +See this spreadsheet for previous battles with version incompatibility: +https://docs.google.com/spreadsheets/d/1BEdLmwOzo3r83-iJn9sj6co1VT92t3lIeJ-lNH25bdQ/edit#gid=0 + +Ubuntu might upgrade the driver by accident, and it might need reinstallation. Who knows. This stuff +is a nightmare. + ### Database Setup Install the following libraries, and see the notes further below about MySQL on Ubuntu 20.04. diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 1fdaecb4c9..6efd46ebfc 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -25,6 +25,9 @@ CREATE TABLE tts_models ( 'arpabet' ) NOT NULL DEFAULT 'basic', + -- NB: ADDED BY ALTER + -- maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL, + -- Optional Pointer to a newer version of the voice -- If there's a newer version, we can disable this one. -- maybe_updated_model_token VARCHAR(32) DEFAULT NULL, diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index 450595392a..0b25e2e584 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -25,6 +25,9 @@ CREATE TABLE tts_inference_jobs ( -- This also determines which architecture we're using. model_token VARCHAR(32) NOT NULL, + -- NB: ADDED BY ALTER + -- maybe_override_pretrained_vocoder VARCHAR(64) DEFAULT NULL, + -- The raw, unprocessed user input. raw_inference_text TEXT NOT NULL, diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 3b69328c24..12061ef51c 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -14,6 +14,9 @@ CREATE TABLE tts_results ( -- The TTS model that was used model_token VARCHAR(32) NOT NULL, + -- NB: ADDED BY ALTER + -- maybe_pretrained_vocoder_used VARCHAR(64) DEFAULT NULL, + -- The original raw, unprocessed user input. raw_inference_text TEXT NOT NULL, diff --git a/migrations/2021-09-25-010101_alter_tables_add_vocoder_type/down.sql b/migrations/2021-09-25-010101_alter_tables_add_vocoder_type/down.sql new file mode 100644 index 0000000000..16dcc8614f --- /dev/null +++ b/migrations/2021-09-25-010101_alter_tables_add_vocoder_type/down.sql @@ -0,0 +1,12 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_models + DROP COLUMN maybe_default_pretrained_vocoder; + +ALTER TABLE tts_inference_jobs + DROP COLUMN maybe_override_pretrained_vocoder; + +ALTER TABLE tts_results + DROP COLUMN maybe_pretrained_vocoder_used; diff --git a/migrations/2021-09-25-010101_alter_tables_add_vocoder_type/up.sql b/migrations/2021-09-25-010101_alter_tables_add_vocoder_type/up.sql new file mode 100644 index 0000000000..17b09f19b8 --- /dev/null +++ b/migrations/2021-09-25-010101_alter_tables_add_vocoder_type/up.sql @@ -0,0 +1,15 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_models +ADD COLUMN maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL +AFTER text_preprocessing_algorithm; + +ALTER TABLE tts_inference_jobs +ADD COLUMN maybe_override_pretrained_vocoder VARCHAR(64) DEFAULT NULL +AFTER model_token; + +ALTER TABLE tts_results +ADD COLUMN maybe_pretrained_vocoder_used VARCHAR(64) DEFAULT NULL +AFTER model_token; diff --git a/models/tacotron2/vocodes_common.py b/models/tacotron2/vocodes_common.py index 423d5fe02b..d5f447d123 100644 --- a/models/tacotron2/vocodes_common.py +++ b/models/tacotron2/vocodes_common.py @@ -261,7 +261,7 @@ class TacotronWaveglowPipeline: generate_metadata_file(args['output_audio_filename'], args['output_metadata_filename']) def vocode_waveglow(self, args, mel_outputs_postnet): - print('Running melgan...') + print('Running waveglow...') with torch.no_grad(): sigma = 0.8 audio = self.waveglow.infer(mel_outputs_postnet, sigma=sigma) diff --git a/sqlx-data.json b/sqlx-data.json index 8055203f77..2dbe11478f 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -593,24 +593,315 @@ ] } }, - "0ede38519f58e65cceec9204b5a28780b3f32f7ba6efc68f5d33f46a5865d987": { - "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n\n file_size_bytes = ?,\n duration_millis = ?\n ", + "0fb69a1cc22eb6d9fc189005f031e1636acbec6ba2033e980ccf5340eb337c08": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n tts_results.maybe_pretrained_vocoder_used,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.creator_set_visibility,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n\n tts_results.creator_ip_address,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n ", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 2, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "tts_model_title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "maybe_pretrained_vocoder_used", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "maybe_model_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "maybe_model_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 11, + "name": "maybe_model_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 12, + "name": "maybe_model_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 13, + "name": "public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, + "name": "public_bucket_spectrogram_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 15, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 16, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 17, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 18, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 21, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + } + ], "parameters": { - "Right": 13 + "Right": 1 }, - "nullable": [] - } - }, - "0efe809b1ce59d68c5e8fb43d42951b02ad7c17da8e7ef3d0812b75db9c73174": { - "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] } }, "0ff13b84956c2625050211a5bb2352a489ef8435571c364c8b2d1d21d936cbd4": { @@ -700,304 +991,6 @@ ] } }, - "1150dfa30863056960462f868b148241648fb6dfd94e8f3a8ae6b5da4f6f48fd": { - "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.creator_set_visibility,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n\n tts_results.creator_ip_address,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n AND tts_results.user_deleted_at IS NULL\n AND tts_results.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tts_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "raw_inference_text", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 2, - "name": "tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "tts_model_title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 5, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, - "name": "maybe_model_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 9, - "name": "maybe_model_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 10, - "name": "maybe_model_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 11, - "name": "maybe_model_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 12, - "name": "public_bucket_wav_audio_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 13, - "name": "public_bucket_spectrogram_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 14, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 15, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 16, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 17, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 18, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 20, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 21, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "1168f39a5ec892d06e941b0dba0e955a27ef976567eeafd6e550bc42ea4c4331": { "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", "describe": { @@ -1197,38 +1190,8 @@ "nullable": [] } }, - "1b1a5d54d20a834c73c7afa3d80681bdf7e0b8ca9767fe7ea0861cc25f10af54": { - "query": "\nUPDATE tts_models\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - } - }, - "1cc62e5cdc8de18969d7e8018d91c38794e5a7d23f117e5b0a4126845f300e60": { - "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE\n token = ?\n AND deleted_at IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, - "25e7858d370ac1359deb42ef5292e85ccd1fe7debf3a1ba73303e5882523a4d0": { - "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n download_url = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 8 - }, - "nullable": [] - } - }, - "27aa92743ce8b2221feeac96de6eb8b378b427912de09779fee938b757de4bdc": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\nAND tts.user_deleted_at IS NULL\nAND tts.mod_deleted_at IS NULL\n ", + "18a2b7a076618c07614a7df9039264ec824ca664e6e7eeb6a3d680c8ee464c3e": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { @@ -1257,6 +1220,30 @@ }, { "ordinal": 2, + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "maybe_default_pretrained_vocoder", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 4, "name": "creator_user_token", "type_info": { "type": "VarString", @@ -1268,7 +1255,7 @@ } }, { - "ordinal": 3, + "ordinal": 5, "name": "creator_username", "type_info": { "type": "VarString", @@ -1280,7 +1267,7 @@ } }, { - "ordinal": 4, + "ordinal": 6, "name": "creator_display_name", "type_info": { "type": "VarString", @@ -1292,7 +1279,7 @@ } }, { - "ordinal": 5, + "ordinal": 7, "name": "title", "type_info": { "type": "VarString", @@ -1304,19 +1291,67 @@ } }, { - "ordinal": 6, - "name": "private_bucket_hash", + "ordinal": 8, + "name": "description_markdown", "type_info": { - "type": "String", + "type": "Blob", "flags": { - "bits": 4233 + "bits": 4241 }, "char_set": 224, - "max_size": 256 + "max_size": 262140 } }, { - "ordinal": 7, + "ordinal": 9, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 10, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 11, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, "name": "created_at", "type_info": { "type": "Timestamp", @@ -1328,7 +1363,7 @@ } }, { - "ordinal": 8, + "ordinal": 14, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -1338,6 +1373,54 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 15, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 16, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 17, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { @@ -1347,15 +1430,45 @@ false, false, false, + true, false, false, false, false, false, - false + false, + false, + false, + false, + false, + false, + false, + false, + true, + true ] } }, + "1cc62e5cdc8de18969d7e8018d91c38794e5a7d23f117e5b0a4126845f300e60": { + "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE\n token = ?\n AND deleted_at IS NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "25e7858d370ac1359deb42ef5292e85ccd1fe7debf3a1ba73303e5882523a4d0": { + "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n download_url = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 8 + }, + "nullable": [] + } + }, "28cf5843d68d6cb0e67eb316ef9e7e5fb05a6be2433f86c081576b279167508d": { "query": "\nINSERT INTO tts_result_synthetic_ids\nSET\n user_token = ?,\n next_id = 1\nON DUPLICATE KEY UPDATE\n user_token = ?,\n next_id = next_id + 1\n ", "describe": { @@ -1615,304 +1728,6 @@ ] } }, - "32d1eb129b7fd640c4f17c08eb3da1bb1dd8b057c9860cf1cb4949800b002dcd": { - "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.creator_set_visibility,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n\n tts_results.creator_ip_address,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tts_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "raw_inference_text", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 2, - "name": "tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "tts_model_title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 5, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, - "name": "maybe_model_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 9, - "name": "maybe_model_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 10, - "name": "maybe_model_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 11, - "name": "maybe_model_creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 12, - "name": "public_bucket_wav_audio_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 13, - "name": "public_bucket_spectrogram_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 14, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 15, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 16, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 17, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 18, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 20, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 21, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 136 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "33cd0c0b8eaaf494ef97232ac89775b8eb6a22d310824b7aa7db310dd4fd41b1": { "query": "\nUPDATE w2l_results\nSET\n creator_set_visibility = ?,\n maybe_mod_user_token = ?\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -1993,6 +1808,317 @@ "nullable": [] } }, + "3f79ccaf090955739c34988ba8bac4bd609bada186d96d11e9eb2815a3e01799": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.raw_inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n tts_results.maybe_pretrained_vocoder_used,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.creator_set_visibility,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at,\n\n tts_results.creator_ip_address,\n tts_results.user_deleted_at,\n tts_results.mod_deleted_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.token = ?\n AND tts_results.user_deleted_at IS NULL\n AND tts_results.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 2, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "tts_model_title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "maybe_pretrained_vocoder_used", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "maybe_model_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "maybe_model_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 11, + "name": "maybe_model_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 12, + "name": "maybe_model_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 13, + "name": "public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, + "name": "public_bucket_spectrogram_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 15, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 16, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 17, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 18, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 21, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c": { "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -5113,6 +5239,16 @@ "nullable": [] } }, + "9c04e83ce78dafd7b84edf057f30a834d459f710d0836dcf7f211f7cd17e24e3": { + "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + } + }, "9cc5f697350814199344d3d9a0ff026e0d1e09e9097648400d4b63cfcbd31178": { "query": "\nUPDATE w2l_results\nSET\n creator_set_visibility = ?,\n creator_ip_address = ?\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -5487,6 +5623,16 @@ "nullable": [] } }, + "adc494ddafaa13aebe423984f32a0d8810da604b19f5d6e0081b5e891dfc0f52": { + "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + } + }, "adf2d5eb967fa2bc00fad1c8fcc2e1bf4384d39dc54e8240d78bb4ca6fcc3409": { "query": "\nUPDATE w2l_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -5747,6 +5893,16 @@ ] } }, + "b8746ae4ff2d4dcd54563eff64877f924bb119681ad78f1300146e00b762ca4f": { + "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n maybe_pretrained_vocoder_used = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n\n file_size_bytes = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 14 + }, + "nullable": [] + } + }, "bc9c6b57b2578b8163956d11a5f6a9a77ddab6098d2c4207e1a70502db30c9d1": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.private_bucket_hash,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\nAND w2l.user_deleted_at IS NULL\nAND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -6950,252 +7106,6 @@ ] } }, - "e79731cdcae40dd4ad37d8668a86811df45eb5e05cd27743a3758f91bf8f293c": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "text_preprocessing_algorithm", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 3, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "description_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 8, - "name": "description_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 9, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 10, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 11, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 14, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 15, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 16, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 17, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "e9ec861d6eb135370f9baf3a866282ec5686d4b299fc4ec10900ab32ea8577d8": { "query": "\nUPDATE w2l_results\nSET\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -7335,6 +7245,148 @@ ] } }, + "ec37e0acc4c897e8ef5aeeb66dce9df684ed8ea1ec29b0d9e92f44e0bd6d645d": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.maybe_default_pretrained_vocoder,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\nAND tts.user_deleted_at IS NULL\nAND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "maybe_default_pretrained_vocoder", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 3, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "private_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + true, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "ed67238afe48c159443d215cd1fe2f0264ad840768f789d068527ae542b08194": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -7461,6 +7513,265 @@ "nullable": [] } }, + "eeba4761fc241c5d4dde963f4687fc3f3dcda6de9d556ebd86a25621449b0307": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "maybe_default_pretrained_vocoder", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 4, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 9, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 10, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 11, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 15, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 16, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 17, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "ef828549a221fe6fad3105fede0b38a439a685d6ea7f83a0657df9cdbb80c589": { "query": "\nSELECT\n ip_bans.ip_address,\n\n ip_bans.maybe_target_user_token,\n banned_users.username as maybe_target_username,\n\n ip_bans.mod_user_token,\n mod_users.username as mod_username,\n mod_users.display_name as mod_display_name,\n\n ip_bans.mod_notes,\n ip_bans.created_at,\n ip_bans.updated_at\nFROM\n ip_address_bans AS ip_bans\nLEFT OUTER JOIN users as banned_users\n ON ip_bans.maybe_target_user_token = banned_users.token\nJOIN users as mod_users\n ON ip_bans.mod_user_token = mod_users.token\nWHERE\n ip_bans.ip_address = ?\nLIMIT 1\n ", "describe": { @@ -7921,252 +8232,6 @@ "nullable": [] } }, - "faafd5b65bb75e2fe993d11a6e12aefe6121702b40aea641aa7aab56799364b5": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "text_preprocessing_algorithm", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 3, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "description_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 8, - "name": "description_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 9, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 10, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 11, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 14, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 15, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 16, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 17, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "fc0045b6fe338a903fa9f35307457d8dbb32c5838a4783fddf0c9beef7fdb4a5": { "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,\n\n jobs.model_token,\n tts.tts_model_type,\n tts.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_inference_jobs as jobs\nJOIN tts_models as tts\n ON tts.token = jobs.model_token\nLEFT OUTER JOIN tts_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", "describe": { diff --git a/src/database/queries/query_tts_model.rs b/src/database/queries/query_tts_model.rs index 9358b34796..eb357d5a9f 100644 --- a/src/database/queries/query_tts_model.rs +++ b/src/database/queries/query_tts_model.rs @@ -2,6 +2,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::enums::vocoder_type::VocoderType; use crate::database::helpers::boolean_converters::i8_to_bool; use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use derive_more::{Display, Error}; @@ -17,6 +18,7 @@ use std::sync::Arc; pub struct TtsModelRecordForResponse { pub model_token: String, pub tts_model_type: String, + pub maybe_default_pretrained_vocoder: Option, pub text_preprocessing_algorithm: String, pub creator_user_token: String, @@ -52,6 +54,7 @@ pub struct TtsModelModeratorFields { pub struct TtsModelRecordRaw { pub model_token: String, pub tts_model_type: String, + pub maybe_default_pretrained_vocoder: Option, pub text_preprocessing_algorithm: String, pub creator_user_token: String, @@ -105,9 +108,15 @@ pub async fn select_tts_model_by_token( } }; + let mut maybe_vocoder : Option = None; + if let Some(vocoder) = model.maybe_default_pretrained_vocoder.as_deref() { + maybe_vocoder = Some(VocoderType::from_str(vocoder)?); + } + let model_for_response = TtsModelRecordForResponse { model_token: model.model_token.clone(), tts_model_type: model.tts_model_type.clone(), + maybe_default_pretrained_vocoder: maybe_vocoder, text_preprocessing_algorithm: model.text_preprocessing_algorithm.clone(), creator_user_token: model.creator_user_token.clone(), creator_username: model.creator_username.clone(), @@ -144,6 +153,7 @@ SELECT tts.token as model_token, tts.tts_model_type, tts.text_preprocessing_algorithm, + tts.maybe_default_pretrained_vocoder, tts.creator_user_token, users.username as creator_username, @@ -188,6 +198,7 @@ SELECT tts.token as model_token, tts.tts_model_type, tts.text_preprocessing_algorithm, + tts.maybe_default_pretrained_vocoder, tts.creator_user_token, users.username as creator_username, diff --git a/src/database/queries/query_tts_result.rs b/src/database/queries/query_tts_result.rs index 0b1b3b6bde..aad0e452c4 100644 --- a/src/database/queries/query_tts_result.rs +++ b/src/database/queries/query_tts_result.rs @@ -2,6 +2,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::enums::vocoder_type::VocoderType; use crate::database::helpers::boolean_converters::i8_to_bool; use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use derive_more::{Display, Error}; @@ -21,6 +22,8 @@ pub struct TtsResultRecordForResponse { pub tts_model_token: String, pub tts_model_title: Option, // TODO: Shouldn't be Option. + pub maybe_pretrained_vocoder_used: Option, + pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, @@ -65,6 +68,8 @@ pub struct TtsResultRecordRaw { pub tts_model_token: String, pub tts_model_title: Option, + pub maybe_pretrained_vocoder_used: Option, + pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, @@ -123,6 +128,11 @@ pub async fn select_tts_result_by_token( } }; + let mut pretrained_vocoder = None; + if let Some(vocoder) = ir.maybe_pretrained_vocoder_used.as_deref() { + pretrained_vocoder = Some(VocoderType::from_str(vocoder)?); + } + let ir_for_response = TtsResultRecordForResponse { tts_result_token: ir.tts_result_token.clone(), @@ -131,6 +141,8 @@ pub async fn select_tts_result_by_token( tts_model_token: ir.tts_model_token.clone(), tts_model_title: ir.tts_model_title.clone(), + maybe_pretrained_vocoder_used: pretrained_vocoder, + maybe_creator_user_token: ir.maybe_creator_user_token.clone(), maybe_creator_username: ir.maybe_creator_username.clone(), maybe_creator_display_name: ir.maybe_creator_display_name.clone(), @@ -181,6 +193,8 @@ SELECT tts_results.model_token as tts_model_token, tts_models.title as tts_model_title, + tts_results.maybe_pretrained_vocoder_used, + users.token as maybe_creator_user_token, users.username as maybe_creator_username, users.display_name as maybe_creator_display_name, @@ -236,6 +250,8 @@ SELECT tts_results.model_token as tts_model_token, tts_models.title as tts_model_title, + tts_results.maybe_pretrained_vocoder_used, + users.token as maybe_creator_user_token, users.username as maybe_creator_username, users.display_name as maybe_creator_display_name, diff --git a/src/http_server/endpoints/tts/edit_tts_model.rs b/src/http_server/endpoints/tts/edit_tts_model.rs index 644c4b434f..befe0efaab 100644 --- a/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/src/http_server/endpoints/tts/edit_tts_model.rs @@ -25,6 +25,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::database::enums::vocoder_type::VocoderType; /// For the URL PathInfo #[derive(Deserialize)] @@ -39,6 +40,7 @@ pub struct EditTtsModelRequest { pub title: Option, pub description_markdown: Option, pub creator_set_visibility: Option, + pub maybe_default_pretrained_vocoder: Option, //pub updatable_slug: Option, //pub tts_model_type: Option, //pub text_preprocessing_algorithm: Option, @@ -150,6 +152,9 @@ pub async fn edit_tts_model_handler( let mut description_markdown = None; let mut description_html = None; let mut creator_set_visibility = RecordVisibility::Public; + let mut maybe_default_pretrained_vocoder = + model_record.maybe_default_pretrained_vocoder + .clone(); if let Some(payload) = request.title.as_deref() { if contains_slurs(payload) { @@ -176,6 +181,10 @@ pub async fn edit_tts_model_handler( .map_err(|_| EditTtsModelError::BadInput("bad record visibility".to_string()))?; } + if let Some(vocoder) = request.maybe_default_pretrained_vocoder { + maybe_default_pretrained_vocoder = Some(vocoder); + } + let ip_address = get_request_ip(&http_request); let query_result = if is_author { @@ -184,6 +193,7 @@ pub async fn edit_tts_model_handler( r#" UPDATE tts_models SET + maybe_default_pretrained_vocoder = ?, title = ?, description_markdown = ?, description_rendered_html = ?, @@ -193,6 +203,7 @@ SET WHERE token = ? LIMIT 1 "#, + maybe_default_pretrained_vocoder.map(|v| v.to_str()), &title, &description_markdown, &description_html, @@ -208,6 +219,7 @@ LIMIT 1 r#" UPDATE tts_models SET + maybe_default_pretrained_vocoder = ?, title = ?, description_markdown = ?, description_rendered_html = ?, @@ -217,6 +229,7 @@ SET WHERE token = ? LIMIT 1 "#, + maybe_default_pretrained_vocoder.map(|v| v.to_str()), &title, &description_markdown, &description_html, diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 086539714b..63ff083d13 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -5,6 +5,7 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::enums::vocoder_type::VocoderType; use crate::database::helpers::tokens::Tokens; use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; @@ -227,6 +228,7 @@ pub async fn insert_tts_result>( pool: &MySqlPool, job: &TtsInferenceJobRecord, text_hash: &str, + pretrained_vocoder_used: VocoderType, bucket_audio_results_path: P, bucket_spectrogram_results_path: P, file_size_bytes: u64, @@ -314,6 +316,7 @@ SET token = ?, model_token = ?, + maybe_pretrained_vocoder_used = ?, raw_inference_text = ?, raw_inference_text_hash_sha2 = ?, normalized_inference_text = ?, @@ -332,6 +335,7 @@ SET "#, inference_result_token, job.model_token.clone(), + pretrained_vocoder_used.to_str(), job.raw_inference_text.clone(), text_hash, normalized_inference_text, @@ -374,6 +378,8 @@ pub struct TtsModelRecord2 { pub model_token: String, pub tts_model_type: String, + pub maybe_default_pretrained_vocoder: Option, + pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, @@ -397,6 +403,7 @@ pub async fn get_tts_model_by_token( SELECT tts.token as model_token, tts.tts_model_type, + tts.maybe_default_pretrained_vocoder, tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 9a1291ecfa..1f32f858ce 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -662,10 +662,17 @@ async fn process_job( // false, //)?; + let mut pretrained_vocoder = VocoderType::HifiGanSuperResolution; + if let Some(default_vocoder) = model_record.maybe_default_pretrained_vocoder.as_deref() { + pretrained_vocoder = VocoderType::from_str(default_vocoder)?; + } + + info!("With pretrained vocoder: {:?}", pretrained_vocoder); + inferencer.tts_inference_sidecar_client.request_inference( &job.raw_inference_text, &tts_synthesizer_fs_path, - VocoderType::HifiGanSuperResolution, + pretrained_vocoder, &hifigan_vocoder_model_fs_path, &hifigan_superres_vocoder_model_fs_path, &waveglow_vocoder_model_fs_path, @@ -724,6 +731,7 @@ async fn process_job( &inferencer.mysql_pool, job, &text_hash, + pretrained_vocoder, &audio_result_object_path, &spectrogram_result_object_path, file_metadata.file_size_bytes, From fb24e58d564871b2f93caf74d5704dfb9f3c9d03 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 26 Aug 2021 00:00:15 -0400 Subject: [PATCH 0520/1042] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8e59abeffe..a86671f4a3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ storyteller-web =============== + This is the main user account monolith that we'll also bake other pieces into. Local development From 3702332669eed335d2265e41428288cdb9c8ae6d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 26 Aug 2021 23:58:34 -0400 Subject: [PATCH 0521/1042] new redis keys --- src/util/redis_keys.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/util/redis_keys.rs b/src/util/redis_keys.rs index f93fddf321..7939ba0ad3 100644 --- a/src/util/redis_keys.rs +++ b/src/util/redis_keys.rs @@ -5,13 +5,27 @@ impl RedisKeys { /// This is a counter incremented with TTS inference. /// We can use a job to read MySql and fix counts if they get out of sync. + /// These should be long-lived keys. pub fn tts_model_usage_count(model_token: &str) -> String { format!("ttsUseCount:{}", model_token) } /// This is a counter incremented with TTS inference. /// We can use a job to read MySql and fix counts if they get out of sync. + /// These should be long-lived keys. pub fn w2l_template_usage_count(template_token: &str) -> String { format!("w2lUseCount:{}", template_token) } + + /// We write extra in-progress status information to keys. + /// These keys should have a TTL. + pub fn tts_inference_extra_status_info(inference_job_token: &str) -> String { + format!("ttsInferenceExtraStatus:{}", inference_job_token) + } + + /// We write extra in-progress status information to keys. + /// These keys should have a TTL. + pub fn w2l_inference_extra_status_info(inference_job_token: &str) -> String { + format!("w2lInferenceExtraStatus:{}", inference_job_token) + } } From 1c524ed78bc73ea1bf255e59620f6a8fb9b05ae2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 27 Aug 2021 00:47:39 -0400 Subject: [PATCH 0522/1042] query redis status in tts inference endpoint --- .../tts/get_tts_inference_job_status.rs | 32 +++++++++++++++++++ .../endpoints/tts/get_tts_model_use_count.rs | 1 - 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index c74a66dc34..a87b51fa9f 100644 --- a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -15,16 +15,22 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::redis_keys::RedisKeys; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::redis::{Commands, RedisError, RedisResult}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::borrow::BorrowMut; use std::sync::Arc; +use r2d2_redis::r2d2::{Pool, PooledConnection}; +use std::ops::Deref; /// For the URL PathInfo #[derive(Deserialize)] @@ -36,7 +42,13 @@ pub struct GetTtsInferenceStatusPathInfo { pub struct TtsInferenceJobStatusForResponse { pub job_token: String, + /// Primary status from the database. pub status: String, + + /// Extra, temporary status from Redis. + /// This can denote inference progress, and the Python code can write to it. + pub extra_status: Option, + pub attempt_count: u8, pub maybe_result_token: Option, @@ -149,9 +161,29 @@ WHERE jobs.token = ? } }; + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + warn!("redis error: {:?}", e); + GetTtsInferenceStatusError::ServerError + })?; + + let extra_status_key = RedisKeys::tts_inference_extra_status_info(&path.token); + let extra_status : Option = match redis.get(&extra_status_key) { + Ok(Some(status)) => { + Some(status) + }, + Ok(None) => None, + Err(e) => { + warn!("redis error: {:?}", e); + None // Fail open + }, + }; + let model_for_response = TtsInferenceJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), + extra_status, attempt_count: record.attempt_count as u8, maybe_result_token: record.maybe_result_token.clone(), maybe_public_bucket_wav_audio_path: record.maybe_public_bucket_wav_audio_path.clone(), diff --git a/src/http_server/endpoints/tts/get_tts_model_use_count.rs b/src/http_server/endpoints/tts/get_tts_model_use_count.rs index 00d5109827..11d18204b0 100644 --- a/src/http_server/endpoints/tts/get_tts_model_use_count.rs +++ b/src/http_server/endpoints/tts/get_tts_model_use_count.rs @@ -66,7 +66,6 @@ pub async fn get_tts_model_use_count_handler( path: Path, server_state: web::Data>) -> Result { - let mut redis = server_state.redis_pool .get() .map_err(|e| { From 2cbc86ddce7738a051d6fce1415504093aa24fce Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 27 Aug 2021 01:22:27 -0400 Subject: [PATCH 0523/1042] report detailed TTS inference status --- src/common_env.rs | 7 +++ .../tts/get_tts_inference_job_status.rs | 6 +- src/tts_inference_job.rs | 58 ++++++++++++++++++- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/src/common_env.rs b/src/common_env.rs index 706d9fdfed..961ea8ef83 100644 --- a/src/common_env.rs +++ b/src/common_env.rs @@ -1,3 +1,4 @@ +use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use crate::util::anyhow_result::AnyhowResult; // TODO: Move more shared configs here. @@ -7,6 +8,10 @@ use crate::util::anyhow_result::AnyhowResult; /// similarly across all usages. pub struct CommonEnv { + /// The Redis connection string. + /// Currently we only have a single Redis that serves all purposes. + pub redis_connection_string: String, + /// The amount of time to wait between job batches (not individual jobs). /// This prevents the outer loop of querying batches from flooding the DB. /// (In theory, there's work within jobs that prevents rapidly pegging the DB.) @@ -27,6 +32,8 @@ impl CommonEnv { pub fn read_from_env() -> AnyhowResult { Ok(Self { + redis_connection_string: easyenv::get_env_string_or_default("REDIS_URL", + DEFAULT_REDIS_CONNECTION_STRING), job_batch_wait_millis: easyenv::get_env_num("JOB_BATCH_WAIT_MILLIS", 100)?, job_max_attempts: easyenv::get_env_num("JOB_MAX_ATTEMPTS", 3)?, job_batch_size: easyenv::get_env_num("JOB_BATCH_SIZE", 10)?, diff --git a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index a87b51fa9f..28fcf99533 100644 --- a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -47,7 +47,7 @@ pub struct TtsInferenceJobStatusForResponse { /// Extra, temporary status from Redis. /// This can denote inference progress, and the Python code can write to it. - pub extra_status: Option, + pub maybe_extra_status_description: Option, pub attempt_count: u8, @@ -169,7 +169,7 @@ WHERE jobs.token = ? })?; let extra_status_key = RedisKeys::tts_inference_extra_status_info(&path.token); - let extra_status : Option = match redis.get(&extra_status_key) { + let maybe_extra_status_description : Option = match redis.get(&extra_status_key) { Ok(Some(status)) => { Some(status) }, @@ -183,7 +183,7 @@ WHERE jobs.token = ? let model_for_response = TtsInferenceJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), - extra_status, + maybe_extra_status_description, attempt_count: record.attempt_count as u8, maybe_result_token: record.maybe_result_token.clone(), maybe_public_bucket_wav_audio_path: record.maybe_public_bucket_wav_audio_path.clone(), diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 1f32f858ce..6da2fbb584 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -48,15 +48,20 @@ use crate::util::jobs::cache_miss_strategizer_multi::SyncMultiCacheMissStrategiz use crate::util::jobs::virtual_lfu_cache::SyncVirtualLfuCache; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::redis_keys::RedisKeys; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2; +use r2d2_redis::redis::Commands; use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::collections::HashMap; use std::fs::{File, metadata}; use std::io::{BufReader, Read}; +use std::ops::Deref; use std::path::{PathBuf, Path}; use std::process::Command; use std::thread; @@ -88,6 +93,8 @@ struct Inferencer { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, + pub redis_pool: r2d2::Pool, + pub private_bucket_client: BucketClient, pub public_bucket_client: BucketClient, @@ -205,6 +212,16 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; + let common_env = CommonEnv::read_from_env()?; + + info!("Connecting to redis..."); + + let redis_manager = + RedisConnectionManager::new(common_env.redis_connection_string.deref())?; + + let redis_pool = r2d2::Pool::builder() + .build(redis_manager)?; + let persistent_cache_path = easyenv::get_env_string_or_default( ENV_SEMIPERSISTENT_CACHE_DIR, "/tmp"); @@ -232,8 +249,6 @@ async fn main() -> AnyhowResult<()> { mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe }; - let common_env = CommonEnv::read_from_env()?; - let virtual_lfu_cache = SyncVirtualLfuCache::new(2)?; let cache_miss_strategizers = { @@ -264,6 +279,7 @@ async fn main() -> AnyhowResult<()> { let inferencer = Inferencer { download_temp_directory: temp_directory, mysql_pool, + redis_pool, public_bucket_client, private_bucket_client, tts_inference_command, @@ -510,6 +526,8 @@ async fn process_job( // TODO 8. Save record // TODO 9. Mark job done + let redis_status_key = RedisKeys::tts_inference_extra_status_info(&job.inference_job_token); + // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // let lock_acquired = grab_job_lock_and_mark_pending(&inferencer.mysql_pool, job).await?; @@ -527,6 +545,11 @@ async fn process_job( if !waveglow_vocoder_model_fs_path.exists() { warn!("Waveglow vocoder model file does not exist: {:?}", &waveglow_vocoder_model_fs_path); + let r : String = inferencer.redis_pool.get() + .unwrap() + .set_ex(&redis_status_key, "Downloading vocoder...", 60*60) + .unwrap(); + let waveglow_vocoder_model_object_path = inferencer.bucket_path_unifier .tts_pretrained_vocoders_path(&waveglow_vocoder_model_filename); @@ -548,6 +571,11 @@ async fn process_job( if !hifigan_vocoder_model_fs_path.exists() { warn!("Hifigan vocoder model file does not exist: {:?}", &hifigan_vocoder_model_fs_path); + let r : String = inferencer.redis_pool.get() + .unwrap() + .set_ex(&redis_status_key, "Downloading vocoder...", 60*60) + .unwrap(); + let hifigan_vocoder_model_object_path = inferencer.bucket_path_unifier .tts_pretrained_vocoders_path(&hifigan_vocoder_model_filename); @@ -569,6 +597,11 @@ async fn process_job( if !hifigan_superres_vocoder_model_fs_path.exists() { warn!("Hifigan superres vocoder model file does not exist: {:?}", &hifigan_superres_vocoder_model_fs_path); + let r : String = inferencer.redis_pool.get() + .unwrap() + .set_ex(&redis_status_key, "Downloading vocoder...", 60*60) + .unwrap(); + let hifigan_superres_vocoder_model_object_path = inferencer.bucket_path_unifier .tts_pretrained_vocoders_path(&hifigan_superres_vocoder_model_filename); @@ -609,6 +642,11 @@ async fn process_job( if !tts_synthesizer_fs_path.exists() { info!("TTS synthesizer model file does not exist: {:?}", &tts_synthesizer_fs_path); + let r : String = inferencer.redis_pool.get() + .unwrap() + .set_ex(&redis_status_key, "Downloading synthesizer model...", 60*60) + .unwrap(); + let tts_synthesizer_object_path = inferencer.bucket_path_unifier .tts_synthesizer_path(&model_record.private_bucket_hash); @@ -633,6 +671,11 @@ async fn process_job( // ==================== RUN INFERENCE ==================== // + let r : String = inferencer.redis_pool.get() + .unwrap() + .set_ex(&redis_status_key, "Running inference...", 60*60) + .unwrap(); + // TODO: Fix this. let maybe_unload_model_path = inferencer .virtual_model_lfu @@ -694,6 +737,11 @@ async fn process_job( // ==================== UPLOAD AUDIO TO BUCKET ==================== // + let r : String = inferencer.redis_pool.get() + .unwrap() + .set_ex(&redis_status_key, "Uploading result...", 60*60) + .unwrap(); + let audio_result_object_path = inferencer.bucket_path_unifier.tts_inference_wav_audio_output_path( &job.uuid_idempotency_token); // TODO: Don't use this! @@ -759,5 +807,11 @@ async fn process_job( info!("Job {} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}", job.id, id); + + let r : String = inferencer.redis_pool.get() + .unwrap() + .set_ex(&redis_status_key, "Done.", 60*60) + .unwrap(); + Ok(()) } From 75a938e0b3600083e58626608cbdaa5e8b59c6f8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 27 Aug 2021 02:05:40 -0400 Subject: [PATCH 0524/1042] in-progress extra status data for w2l inference --- .../tts/get_tts_inference_job_status.rs | 2 +- .../w2l/get_w2l_inference_job_status.rs | 28 +++++++++ src/util/redis_keys.rs | 2 + src/w2l_inference_job.rs | 62 +++++++++++++++++++ 4 files changed, 93 insertions(+), 1 deletion(-) diff --git a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index 28fcf99533..8848d69f32 100644 --- a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -42,7 +42,7 @@ pub struct GetTtsInferenceStatusPathInfo { pub struct TtsInferenceJobStatusForResponse { pub job_token: String, - /// Primary status from the database. + /// Primary status from the database (a state machine). pub status: String, /// Extra, temporary status from Redis. diff --git a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs index 3ba632c06e..bca3335d34 100644 --- a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs +++ b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs @@ -25,6 +25,8 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::util::redis_keys::RedisKeys; +use r2d2_redis::redis::Commands; /// For the URL PathInfo #[derive(Deserialize)] @@ -36,7 +38,13 @@ pub struct GetW2lInferenceStatusPathInfo { pub struct W2lInferenceJobStatusForResponse { pub job_token: String, + /// Primary status from the database (a state machine). pub status: String, + + /// Extra, temporary status from Redis. + /// This can denote inference progress, and the Python code can write to it. + pub maybe_extra_status_description: Option, + pub attempt_count: u8, pub maybe_result_token: Option, pub maybe_public_bucket_video_path: Option, @@ -147,9 +155,29 @@ WHERE jobs.token = ? } }; + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + warn!("redis error: {:?}", e); + GetW2lInferenceStatusError::ServerError + })?; + + let extra_status_key = RedisKeys::w2l_inference_extra_status_info(&path.token); + let maybe_extra_status_description : Option = match redis.get(&extra_status_key) { + Ok(Some(status)) => { + Some(status) + }, + Ok(None) => None, + Err(e) => { + warn!("redis error: {:?}", e); + None // Fail open + }, + }; + let template_for_response = W2lInferenceJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), + maybe_extra_status_description, attempt_count: record.attempt_count as u8, maybe_result_token: record.maybe_result_token.clone(), maybe_public_bucket_video_path: record.maybe_public_bucket_video_path.clone(), diff --git a/src/util/redis_keys.rs b/src/util/redis_keys.rs index 7939ba0ad3..7a5455bc5d 100644 --- a/src/util/redis_keys.rs +++ b/src/util/redis_keys.rs @@ -2,6 +2,8 @@ pub struct RedisKeys {} impl RedisKeys { + /// How long the "*_extra_status_info" keys should live. + pub const STATUS_KEY_TTL_SECONDS : usize = 60 * 60; /// This is a counter incremented with TTS inference. /// We can use a job to read MySql and fix counts if they get out of sync. diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 2b0148358e..2e9f9da0f8 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -44,19 +44,25 @@ use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::redis_keys::RedisKeys; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2; +use r2d2_redis::redis::Commands; use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; use std::io::{BufReader, Read}; +use std::ops::Deref; use std::path::{PathBuf, Path}; use std::process::Command; use std::thread; use std::time::Duration; use tempdir::TempDir; +use r2d2_redis::r2d2::PooledConnection; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -82,6 +88,8 @@ struct Inferencer { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, + pub redis_pool: r2d2::Pool, + pub private_bucket_client: BucketClient, pub public_bucket_client: BucketClient, @@ -190,6 +198,16 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; + let common_env = CommonEnv::read_from_env()?; + + info!("Connecting to redis..."); + + let redis_manager = + RedisConnectionManager::new(common_env.redis_connection_string.deref())?; + + let redis_pool = r2d2::Pool::builder() + .build(redis_manager)?; + let inference_script = "TODO".to_string(); let persistent_cache_path = easyenv::get_env_string_or_default( @@ -219,6 +237,7 @@ async fn main() -> AnyhowResult<()> { let inferencer = Inferencer { download_temp_directory: temp_directory, mysql_pool, + redis_pool, public_bucket_client, private_bucket_client, inference_script, @@ -347,6 +366,10 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // TODO 7. Save record // TODO 8. Mark job done + let redis_status_key = RedisKeys::w2l_inference_extra_status_info(&job.inference_job_token); + let mut redis = inferencer.redis_pool.get()?; + let mut redis_logger = JobExtraDetailStatusLogger::new(&mut redis, &job.inference_job_token); + // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // let lock_acquired = grab_job_lock_and_mark_pending(&inferencer.mysql_pool, job).await?; @@ -364,6 +387,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An if !model_fs_path.exists() { warn!("Model file does not exist: {:?}", &model_fs_path); + redis_logger.log_status("downloading model")?; + let model_object_path = inferencer.bucket_path_unifier .w2l_pretrained_models_path(&model_filename); @@ -385,6 +410,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An if !end_bump_fs_path.exists() { warn!("End bump file does not exist: {:?}", &end_bump_fs_path); + redis_logger.log_status("downloading assets")?; + let end_bump_object_path = inferencer.bucket_path_unifier .end_bump_video_for_w2l_path(&end_bump_filename); @@ -434,6 +461,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An if !template_media_fs_path.exists() { info!("W2L template media file does not exist: {:?}", &template_media_fs_path); + redis_logger.log_status("downloading template")?; + let template_media_object_path = inferencer.bucket_path_unifier .media_templates_for_w2l_path(&w2l_template.private_bucket_hash); @@ -461,6 +490,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An if !face_template_fs_path.exists() { info!("W2L face template file does not exist: {:?}", &face_template_fs_path); + redis_logger.log_status("downloading template metadata")?; + let face_template_object_path = inferencer.bucket_path_unifier .precomputed_faces_for_w2l_path(&w2l_template.private_bucket_hash); @@ -476,6 +507,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // ==================== DOWNLOAD USER AUDIO ==================== // + redis_logger.log_status("downloading audio")?; + let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; // NB: Exists until it goes out of scope. @@ -500,6 +533,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // ==================== RUN INFERENCE ==================== // + redis_logger.log_status("executing")?; + let output_video_fs_path = temp_dir.path().join("output.mp4"); let output_metadata_fs_path = temp_dir.path().join("metadata.json"); @@ -533,6 +568,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // ==================== UPLOAD TO BUCKETS ==================== // + redis_logger.log_status("uploading")?; + let result_object_path = inferencer.bucket_path_unifier.w2l_inference_video_output_path( &job.inference_job_token); @@ -582,8 +619,33 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An anyhow!("error publishing event") })?; + redis_logger.log_status("done")?; + info!("Job {} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}", job.id, id); Ok(()) } + +struct JobExtraDetailStatusLogger <'a> { + redis: &'a mut PooledConnection, + status_key: String, +} + +impl <'a> JobExtraDetailStatusLogger <'a> { + fn new(redis: &'a mut PooledConnection, job_token: &str) -> Self { + let status_key = RedisKeys::w2l_inference_extra_status_info(job_token); + Self { + redis, + status_key, + } + } + + fn log_status(&mut self, logging_details: &str) -> AnyhowResult<()> { + let _r : String = self.redis // NB: Compiler can't figure out the throwaway result type + .set_ex(&self.status_key, + logging_details, + RedisKeys::STATUS_KEY_TTL_SECONDS)?; + Ok(()) + } +} From 71b35183f92e0adfba49b2849f68d6d068855281 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 27 Aug 2021 02:16:33 -0400 Subject: [PATCH 0525/1042] improve safety of redis logging for tts --- src/tts_inference_job.rs | 64 +++++++++++++++++++++------------------- src/w2l_inference_job.rs | 2 +- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 6da2fbb584..e3adfd507c 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -53,6 +53,7 @@ use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2::PooledConnection; use r2d2_redis::r2d2; use r2d2_redis::redis::Commands; use ring::digest::{Context, Digest, SHA256}; @@ -527,6 +528,8 @@ async fn process_job( // TODO 9. Mark job done let redis_status_key = RedisKeys::tts_inference_extra_status_info(&job.inference_job_token); + let mut redis = inferencer.redis_pool.get()?; + let mut redis_logger = JobExtraDetailStatusLogger::new(&mut redis, &job.inference_job_token); // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // @@ -545,10 +548,7 @@ async fn process_job( if !waveglow_vocoder_model_fs_path.exists() { warn!("Waveglow vocoder model file does not exist: {:?}", &waveglow_vocoder_model_fs_path); - let r : String = inferencer.redis_pool.get() - .unwrap() - .set_ex(&redis_status_key, "Downloading vocoder...", 60*60) - .unwrap(); + redis_logger.log_status("downloading vocoder (1 of 3)")?; let waveglow_vocoder_model_object_path = inferencer.bucket_path_unifier .tts_pretrained_vocoders_path(&waveglow_vocoder_model_filename); @@ -571,10 +571,7 @@ async fn process_job( if !hifigan_vocoder_model_fs_path.exists() { warn!("Hifigan vocoder model file does not exist: {:?}", &hifigan_vocoder_model_fs_path); - let r : String = inferencer.redis_pool.get() - .unwrap() - .set_ex(&redis_status_key, "Downloading vocoder...", 60*60) - .unwrap(); + redis_logger.log_status("downloading vocoder (2 of 3)")?; let hifigan_vocoder_model_object_path = inferencer.bucket_path_unifier .tts_pretrained_vocoders_path(&hifigan_vocoder_model_filename); @@ -597,10 +594,7 @@ async fn process_job( if !hifigan_superres_vocoder_model_fs_path.exists() { warn!("Hifigan superres vocoder model file does not exist: {:?}", &hifigan_superres_vocoder_model_fs_path); - let r : String = inferencer.redis_pool.get() - .unwrap() - .set_ex(&redis_status_key, "Downloading vocoder...", 60*60) - .unwrap(); + redis_logger.log_status("downloading vocoder (3 of 3)")?; let hifigan_superres_vocoder_model_object_path = inferencer.bucket_path_unifier .tts_pretrained_vocoders_path(&hifigan_superres_vocoder_model_filename); @@ -642,10 +636,7 @@ async fn process_job( if !tts_synthesizer_fs_path.exists() { info!("TTS synthesizer model file does not exist: {:?}", &tts_synthesizer_fs_path); - let r : String = inferencer.redis_pool.get() - .unwrap() - .set_ex(&redis_status_key, "Downloading synthesizer model...", 60*60) - .unwrap(); + redis_logger.log_status("downloading synthesizer")?; let tts_synthesizer_object_path = inferencer.bucket_path_unifier .tts_synthesizer_path(&model_record.private_bucket_hash); @@ -671,10 +662,7 @@ async fn process_job( // ==================== RUN INFERENCE ==================== // - let r : String = inferencer.redis_pool.get() - .unwrap() - .set_ex(&redis_status_key, "Running inference...", 60*60) - .unwrap(); + redis_logger.log_status("running inference")?; // TODO: Fix this. let maybe_unload_model_path = inferencer @@ -737,10 +725,7 @@ async fn process_job( // ==================== UPLOAD AUDIO TO BUCKET ==================== // - let r : String = inferencer.redis_pool.get() - .unwrap() - .set_ex(&redis_status_key, "Uploading result...", 60*60) - .unwrap(); + redis_logger.log_status("uploading result")?; let audio_result_object_path = inferencer.bucket_path_unifier.tts_inference_wav_audio_output_path( &job.uuid_idempotency_token); // TODO: Don't use this! @@ -804,14 +789,33 @@ async fn process_job( anyhow!("error publishing event") })?; + redis_logger.log_status("done")?; + info!("Job {} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}", job.id, id); - - let r : String = inferencer.redis_pool.get() - .unwrap() - .set_ex(&redis_status_key, "Done.", 60*60) - .unwrap(); - Ok(()) } + +struct JobExtraDetailStatusLogger <'a> { + redis: &'a mut PooledConnection, + status_key: String, +} + +impl <'a> JobExtraDetailStatusLogger <'a> { + fn new(redis: &'a mut PooledConnection, job_token: &str) -> Self { + let status_key = RedisKeys::tts_inference_extra_status_info(job_token); + Self { + redis, + status_key, + } + } + + fn log_status(&mut self, logging_details: &str) -> AnyhowResult<()> { + let _r : String = self.redis // NB: Compiler can't figure out the throwaway result type + .set_ex(&self.status_key, + logging_details, + RedisKeys::STATUS_KEY_TTL_SECONDS)?; + Ok(()) + } +} diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 2e9f9da0f8..c88c1330c5 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -568,7 +568,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // ==================== UPLOAD TO BUCKETS ==================== // - redis_logger.log_status("uploading")?; + redis_logger.log_status("uploading result")?; let result_object_path = inferencer.bucket_path_unifier.w2l_inference_video_output_path( &job.inference_job_token); From 33c508754413f993e9b7ab4f4df137c244ee7e88 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 27 Aug 2021 02:40:16 -0400 Subject: [PATCH 0526/1042] job debugging --- src/common_env.rs | 6 ++++++ src/w2l_download_job.rs | 8 ++++---- src/w2l_inference_job.rs | 11 +++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/common_env.rs b/src/common_env.rs index 961ea8ef83..eb2b6cc1eb 100644 --- a/src/common_env.rs +++ b/src/common_env.rs @@ -26,6 +26,11 @@ pub struct CommonEnv { /// Time to wait between no-op logger log lines. pub no_op_logger_millis: u64, + + /// For debugging. + /// If nonzero, sleep before exiting the job for this amount of millis. + /// This allows introspection of temp directories before they are unlinked. + pub debug_job_end_sleep_millis: u64, } impl CommonEnv { @@ -38,6 +43,7 @@ impl CommonEnv { job_max_attempts: easyenv::get_env_num("JOB_MAX_ATTEMPTS", 3)?, job_batch_size: easyenv::get_env_num("JOB_BATCH_SIZE", 10)?, no_op_logger_millis: easyenv::get_env_num("NO_OP_LOGGER_MILLIS", 15_000)?, + debug_job_end_sleep_millis: easyenv::get_env_num("DEBUG_JOB_END_SLEEP_MILLIS", 0)?, }) } } \ No newline at end of file diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 243a9a809e..681318341f 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -223,10 +223,10 @@ async fn main() -> AnyhowResult<()> { bucket_root_w2l_template_uploads: bucket_root.to_string(), debug_download_sleep_millis: easyenv::get_env_num("DEBUG_DOWNLOAD_SLEEP_MILLIS", 0)?, debug_face_detect_sleep_millis: easyenv::get_env_num("DEBUG_FACE_DETECT_SLEEP_MILLIS", 0)?, - debug_job_end_sleep_millis: easyenv::get_env_num("DEBUG_JOB_END_SLEEP_MILLIS", 0)?, job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as i32, no_op_logger_millis: common_env.no_op_logger_millis, + debug_job_end_sleep_millis: common_env.debug_job_end_sleep_millis, }; main_loop(downloader).await; @@ -556,9 +556,6 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) Some(&model_token) ).await?; - info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}", - job.id, id); - downloader.firehose_publisher.publish_w2l_template_upload_finished(&job.creator_user_token, &model_token) .await .map_err(|e| { @@ -578,5 +575,8 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) thread::sleep(Duration::from_millis(downloader.debug_job_end_sleep_millis)); } + info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}", + job.id, id); + Ok(()) } diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index c88c1330c5..717d7e6af0 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -118,6 +118,10 @@ struct Inferencer { // Max job attempts before failure. // NB: This is an i32 so we don't need to convert to db column type. pub job_max_attempts: i32, + + // Temporary for debugging + // Arbitrary timeouts can be inserted so we can exec in and poke around. + pub debug_job_end_sleep_millis: u64, } #[tokio::main] @@ -254,6 +258,7 @@ async fn main() -> AnyhowResult<()> { job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as i32, no_op_logger_millis: common_env.no_op_logger_millis, + debug_job_end_sleep_millis: common_env.debug_job_end_sleep_millis, }; main_loop(inferencer).await; @@ -602,6 +607,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An .await?; info!("Marking job complete..."); + mark_w2l_inference_job_done( &inferencer.mysql_pool, job, @@ -621,6 +627,11 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An redis_logger.log_status("done")?; + if inferencer.debug_job_end_sleep_millis != 0 { + warn!("Debug sleep after job end: {} ms", inferencer.debug_job_end_sleep_millis); + thread::sleep(Duration::from_millis(inferencer.debug_job_end_sleep_millis)); + } + info!("Job {} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}", job.id, id); From 09038a4b347bedfc0b6af2764b87030e33584a95 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 27 Aug 2021 22:47:11 -0400 Subject: [PATCH 0527/1042] update to use youtube-dl fork --- Dockerfile | 9 ++++++++- scripts/download_internet_file.py | 5 +++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9d05deb6ad..4f5959cc10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -102,7 +102,14 @@ FROM ghcr.io/storytold/docker-base-images-nvidia-cuda-experimental:080f96bc7087 # # The downloader needs youtube-dl as a python library -RUN pip3 install --upgrade youtube-dl +# NB: No longer maintained: https://news.ycombinator.com/item?id=28319624 +#RUN pip3 install --upgrade youtube-dl + +# Used by the downloader script. +# And it looks like youtube-dl is dead... +# Per the README, it looks like pip will always pull the latest version. +# https://github.com/yt-dlp/yt-dlp#update +RUN python3 -m pip install --upgrade yt-dlp # ==================== Python Build Step 2: Wav2Lip Requirements ==================== diff --git a/scripts/download_internet_file.py b/scripts/download_internet_file.py index 054be80aa5..ccce983600 100755 --- a/scripts/download_internet_file.py +++ b/scripts/download_internet_file.py @@ -9,7 +9,8 @@ import glob import os import re import urllib -import youtube_dl +#import youtube_dl # NB: No longer maintained +import yt_dlp parser = argparse.ArgumentParser(description='Download Google Drive files, Youtube videos, or web urls') @@ -55,7 +56,7 @@ def download_youtube(youtube_url, output_filename): 'noplaylist': True, } - with youtube_dl.YoutubeDL(ydl_opts) as ydl: + with yt_dlp.YoutubeDL(ydl_opts) as ydl: ydl.download([youtube_url]) # ytdl forcefully appends the extension From a8886db089941cd2cce24056a7e48ecebe303c5b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 27 Aug 2021 22:54:22 -0400 Subject: [PATCH 0528/1042] move redis key util (making more) --- src/http_server/endpoints/tts/enqueue_infer_tts.rs | 2 +- src/http_server/endpoints/tts/get_tts_inference_job_status.rs | 2 +- src/http_server/endpoints/tts/get_tts_model_use_count.rs | 2 +- src/http_server/endpoints/w2l/enqueue_infer_w2l.rs | 2 +- .../endpoints/w2l/enqueue_infer_w2l_with_uploads.rs | 4 ++-- src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs | 4 ++-- src/http_server/endpoints/w2l/get_w2l_template_use_count.rs | 2 +- src/tts_inference_job.rs | 2 +- src/util/mod.rs | 2 +- src/util/redis/mod.rs | 1 + src/util/{ => redis}/redis_keys.rs | 0 src/w2l_inference_job.rs | 2 +- 12 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 src/util/redis/mod.rs rename src/util/{ => redis}/redis_keys.rs (100%) diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 8c9059f446..30eb0f307d 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -12,7 +12,7 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; -use crate::util::redis_keys::RedisKeys; +use crate::util::redis::redis_keys::RedisKeys; use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; diff --git a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index 8848d69f32..86b639efb1 100644 --- a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -15,7 +15,7 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; -use crate::util::redis_keys::RedisKeys; +use crate::util::redis::redis_keys::RedisKeys; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; diff --git a/src/http_server/endpoints/tts/get_tts_model_use_count.rs b/src/http_server/endpoints/tts/get_tts_model_use_count.rs index 11d18204b0..a8052b6722 100644 --- a/src/http_server/endpoints/tts/get_tts_model_use_count.rs +++ b/src/http_server/endpoints/tts/get_tts_model_use_count.rs @@ -14,7 +14,7 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; -use crate::util::redis_keys::RedisKeys; +use crate::util::redis::redis_keys::RedisKeys; use derive_more::{Display, Error}; use log::{info, warn, log}; use r2d2_redis::redis::{Commands, RedisError}; diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index 0b259e7877..f802142697 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -11,7 +11,7 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; -use crate::util::redis_keys::RedisKeys; +use crate::util::redis::redis_keys::RedisKeys; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 6dee654e52..1fb2679fea 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -7,6 +7,7 @@ use actix_web::http::header::ContentDisposition; use actix_web::web::{Data, Json, BytesMut}; use actix_web::{middleware, web, App, Error, HttpResponse, HttpServer, HttpRequest, Either, ResponseError}; use anyhow::anyhow; +use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::read_multipart_field_bytes::checked_read_multipart_bytes; @@ -21,7 +22,7 @@ use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::util::random_uuid::generate_random_uuid; -use crate::util::redis_keys::RedisKeys; +use crate::util::redis::redis_keys::RedisKeys; use derive_more::{Display, Error}; use futures::{StreamExt, TryStreamExt}; use log::{warn, info}; @@ -31,7 +32,6 @@ use sqlx::error::Error::Database; use std::io::Write; use std::sync::Arc; use uuid::Uuid; -use crate::database::enums::record_visibility::RecordVisibility; const BUCKET_AUDIO_FILE_NAME : &'static str = "input_audio_file"; const BUCKET_IMAGE_FILE_NAME: &'static str = "input_image_file"; diff --git a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs index bca3335d34..fc527af400 100644 --- a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs +++ b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs @@ -15,18 +15,18 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::redis::redis_keys::RedisKeys; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; +use r2d2_redis::redis::Commands; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::util::redis_keys::RedisKeys; -use r2d2_redis::redis::Commands; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs b/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs index 48ea0387fd..69454492ac 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs @@ -14,7 +14,7 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; -use crate::util::redis_keys::RedisKeys; +use crate::util::redis::redis_keys::RedisKeys; use derive_more::{Display, Error}; use log::{info, warn, log}; use r2d2_redis::redis::{Commands, RedisError}; diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index e3adfd507c..74e5d6d46d 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -48,7 +48,7 @@ use crate::util::jobs::cache_miss_strategizer_multi::SyncMultiCacheMissStrategiz use crate::util::jobs::virtual_lfu_cache::SyncVirtualLfuCache; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; -use crate::util::redis_keys::RedisKeys; +use crate::util::redis::redis_keys::RedisKeys; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; diff --git a/src/util/mod.rs b/src/util/mod.rs index 12d47f5db4..00377d97b0 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -11,5 +11,5 @@ pub mod noop_logger; pub mod random_crockford_token; pub mod random_prefix_crockford_token; pub mod random_uuid; -pub mod redis_keys; +pub mod redis; pub mod semi_persistent_cache_dir; \ No newline at end of file diff --git a/src/util/redis/mod.rs b/src/util/redis/mod.rs new file mode 100644 index 0000000000..f27cdd530b --- /dev/null +++ b/src/util/redis/mod.rs @@ -0,0 +1 @@ +pub mod redis_keys; \ No newline at end of file diff --git a/src/util/redis_keys.rs b/src/util/redis/redis_keys.rs similarity index 100% rename from src/util/redis_keys.rs rename to src/util/redis/redis_keys.rs diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 717d7e6af0..79257877ff 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -44,7 +44,7 @@ use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; -use crate::util::redis_keys::RedisKeys; +use crate::util::redis::redis_keys::RedisKeys; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; From f0aeb42d0651c86b9c20c846828e1264bfa14c57 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 27 Aug 2021 23:30:31 -0400 Subject: [PATCH 0529/1042] reuse as RedisJobStatusLogger --- src/tts_inference_job.rs | 30 +++---------------- src/util/redis/mod.rs | 1 + src/util/redis/redis_job_status_logger.rs | 35 +++++++++++++++++++++++ src/util/redis/redis_keys.rs | 3 -- src/w2l_inference_job.rs | 30 +++---------------- 5 files changed, 44 insertions(+), 55 deletions(-) create mode 100644 src/util/redis/redis_job_status_logger.rs diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 74e5d6d46d..a6cc2af609 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -48,7 +48,7 @@ use crate::util::jobs::cache_miss_strategizer_multi::SyncMultiCacheMissStrategiz use crate::util::jobs::virtual_lfu_cache::SyncVirtualLfuCache; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; -use crate::util::redis::redis_keys::RedisKeys; +use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; @@ -527,9 +527,10 @@ async fn process_job( // TODO 8. Save record // TODO 9. Mark job done - let redis_status_key = RedisKeys::tts_inference_extra_status_info(&job.inference_job_token); let mut redis = inferencer.redis_pool.get()?; - let mut redis_logger = JobExtraDetailStatusLogger::new(&mut redis, &job.inference_job_token); + let mut redis_logger = RedisJobStatusLogger::new_tts_inference( + &mut redis, + &job.inference_job_token); // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // @@ -796,26 +797,3 @@ async fn process_job( Ok(()) } - -struct JobExtraDetailStatusLogger <'a> { - redis: &'a mut PooledConnection, - status_key: String, -} - -impl <'a> JobExtraDetailStatusLogger <'a> { - fn new(redis: &'a mut PooledConnection, job_token: &str) -> Self { - let status_key = RedisKeys::tts_inference_extra_status_info(job_token); - Self { - redis, - status_key, - } - } - - fn log_status(&mut self, logging_details: &str) -> AnyhowResult<()> { - let _r : String = self.redis // NB: Compiler can't figure out the throwaway result type - .set_ex(&self.status_key, - logging_details, - RedisKeys::STATUS_KEY_TTL_SECONDS)?; - Ok(()) - } -} diff --git a/src/util/redis/mod.rs b/src/util/redis/mod.rs index f27cdd530b..7626fe14a6 100644 --- a/src/util/redis/mod.rs +++ b/src/util/redis/mod.rs @@ -1 +1,2 @@ +pub mod redis_job_status_logger; pub mod redis_keys; \ No newline at end of file diff --git a/src/util/redis/redis_job_status_logger.rs b/src/util/redis/redis_job_status_logger.rs new file mode 100644 index 0000000000..e03d99d8ad --- /dev/null +++ b/src/util/redis/redis_job_status_logger.rs @@ -0,0 +1,35 @@ +use crate::util::anyhow_result::AnyhowResult; +use crate::util::redis::redis_keys::RedisKeys; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2::PooledConnection; +use r2d2_redis::redis::Commands; + +pub type RedisPool = PooledConnection; + +pub struct RedisJobStatusLogger <'a> { + redis: &'a mut RedisPool, + status_key: String, +} + +impl <'a> RedisJobStatusLogger <'a> { + pub const STATUS_KEY_TTL_SECONDS : usize = 60 * 60; + + pub fn new_tts_inference(redis: &'a mut RedisPool, tts_job_token: &str) -> Self { + let status_key = RedisKeys::tts_inference_extra_status_info(tts_job_token); + Self { redis, status_key } + } + + pub fn new_w2l_inference(redis: &'a mut RedisPool, w2l_job_token: &str) -> Self { + let status_key = RedisKeys::w2l_inference_extra_status_info(w2l_job_token); + Self { redis, status_key } + } + + /// Record the status to Redis so the frontend has the latest state. + pub fn log_status(&mut self, logging_details: &str) -> AnyhowResult<()> { + let _r : String = self.redis // NB: Compiler can't figure out the throwaway result type + .set_ex(&self.status_key, + logging_details, + Self::STATUS_KEY_TTL_SECONDS)?; + Ok(()) + } +} diff --git a/src/util/redis/redis_keys.rs b/src/util/redis/redis_keys.rs index 7a5455bc5d..afadb4f7e1 100644 --- a/src/util/redis/redis_keys.rs +++ b/src/util/redis/redis_keys.rs @@ -2,9 +2,6 @@ pub struct RedisKeys {} impl RedisKeys { - /// How long the "*_extra_status_info" keys should live. - pub const STATUS_KEY_TTL_SECONDS : usize = 60 * 60; - /// This is a counter incremented with TTS inference. /// We can use a job to read MySql and fix counts if they get out of sync. /// These should be long-lived keys. diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 79257877ff..f9354900cd 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -44,7 +44,7 @@ use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; -use crate::util::redis::redis_keys::RedisKeys; +use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; @@ -371,9 +371,10 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // TODO 7. Save record // TODO 8. Mark job done - let redis_status_key = RedisKeys::w2l_inference_extra_status_info(&job.inference_job_token); let mut redis = inferencer.redis_pool.get()?; - let mut redis_logger = JobExtraDetailStatusLogger::new(&mut redis, &job.inference_job_token); + let mut redis_logger = RedisJobStatusLogger::new_w2l_inference( + &mut redis, + &job.inference_job_token); // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // @@ -637,26 +638,3 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An Ok(()) } - -struct JobExtraDetailStatusLogger <'a> { - redis: &'a mut PooledConnection, - status_key: String, -} - -impl <'a> JobExtraDetailStatusLogger <'a> { - fn new(redis: &'a mut PooledConnection, job_token: &str) -> Self { - let status_key = RedisKeys::w2l_inference_extra_status_info(job_token); - Self { - redis, - status_key, - } - } - - fn log_status(&mut self, logging_details: &str) -> AnyhowResult<()> { - let _r : String = self.redis // NB: Compiler can't figure out the throwaway result type - .set_ex(&self.status_key, - logging_details, - RedisKeys::STATUS_KEY_TTL_SECONDS)?; - Ok(()) - } -} From 0fef8a6838ee517bcb241dc84f880dbd56c960f7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 28 Aug 2021 01:33:54 -0400 Subject: [PATCH 0530/1042] report on w2l template download status --- .../w2l/get_w2l_upload_template_job_status.rs | 28 +++++++++++++++ src/util/redis/redis_job_status_logger.rs | 5 +++ src/util/redis/redis_keys.rs | 6 ++++ src/w2l_download_job.rs | 36 +++++++++++++++++-- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs index 0674d2ccdc..6424bd7cb7 100644 --- a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs +++ b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs @@ -15,10 +15,12 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::redis::redis_keys::RedisKeys; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; +use r2d2_redis::redis::Commands; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; @@ -36,7 +38,13 @@ pub struct GetW2lUploadTemplateStatusPathInfo { pub struct W2lUploadTemplateJobStatusForResponse { pub job_token: String, + /// Primary status from the database (a state machine). pub status: String, + + /// Extra, temporary status from Redis. + /// This can denote inference progress, and the Python code can write to it. + pub maybe_extra_status_description: Option, + pub attempt_count: u8, pub maybe_template_token: Option, @@ -127,9 +135,29 @@ WHERE jobs.token = ? } }; + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + warn!("redis error: {:?}", e); + GetW2lUploadTemplateStatusError::ServerError + })?; + + let extra_status_key = RedisKeys::w2l_download_extra_status_info(&path.token); + let maybe_extra_status_description : Option = match redis.get(&extra_status_key) { + Ok(Some(status)) => { + Some(status) + }, + Ok(None) => None, + Err(e) => { + warn!("redis error: {:?}", e); + None // Fail open + }, + }; + let template_for_response = W2lUploadTemplateJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), + maybe_extra_status_description, attempt_count: record.attempt_count as u8, maybe_template_token: record.maybe_template_token.clone(), created_at: record.created_at.clone(), diff --git a/src/util/redis/redis_job_status_logger.rs b/src/util/redis/redis_job_status_logger.rs index e03d99d8ad..a407e80339 100644 --- a/src/util/redis/redis_job_status_logger.rs +++ b/src/util/redis/redis_job_status_logger.rs @@ -19,6 +19,11 @@ impl <'a> RedisJobStatusLogger <'a> { Self { redis, status_key } } + pub fn new_w2l_download(redis: &'a mut RedisPool, w2l_job_token: &str) -> Self { + let status_key = RedisKeys::w2l_download_extra_status_info(w2l_job_token); + Self { redis, status_key } + } + pub fn new_w2l_inference(redis: &'a mut RedisPool, w2l_job_token: &str) -> Self { let status_key = RedisKeys::w2l_inference_extra_status_info(w2l_job_token); Self { redis, status_key } diff --git a/src/util/redis/redis_keys.rs b/src/util/redis/redis_keys.rs index afadb4f7e1..3455c60183 100644 --- a/src/util/redis/redis_keys.rs +++ b/src/util/redis/redis_keys.rs @@ -22,6 +22,12 @@ impl RedisKeys { format!("ttsInferenceExtraStatus:{}", inference_job_token) } + /// We write extra in-progress status information to keys. + /// These keys should have a TTL. + pub fn w2l_download_extra_status_info(inference_job_token: &str) -> String { + format!("w2lDownloadExtraStatus:{}", inference_job_token) + } + /// We write extra in-progress status information to keys. /// These keys should have a TTL. pub fn w2l_inference_extra_status_info(inference_job_token: &str) -> String { diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 681318341f..b9044dea7d 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -43,13 +43,17 @@ use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2; use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; use std::io::{BufReader, Read}; +use std::ops::Deref; use std::path::{PathBuf, Path}; use std::process::Command; use std::thread; @@ -80,6 +84,8 @@ struct Downloader { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, + pub redis_pool: r2d2::Pool, + pub private_bucket_client: BucketClient, pub public_bucket_client: BucketClient, @@ -195,7 +201,17 @@ async fn main() -> AnyhowResult<()> { .max_connections(5) .connect(&db_connection_string) .await?; - + + let common_env = CommonEnv::read_from_env()?; + + info!("Connecting to redis..."); + + let redis_manager = + RedisConnectionManager::new(common_env.redis_connection_string.deref())?; + + let redis_pool = r2d2::Pool::builder() + .build(redis_manager)?; + let firehose_publisher = FirehosePublisher { mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe }; @@ -205,11 +221,10 @@ async fn main() -> AnyhowResult<()> { firehose_publisher: firehose_publisher.clone(), // NB: Also safe }; - let common_env = CommonEnv::read_from_env()?; - let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, + redis_pool, public_bucket_client, private_bucket_client, download_script, @@ -335,6 +350,11 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) // TODO: 7. Save record. (DONE) // TODO: 8. Mark job done. (DONE) + let mut redis = downloader.redis_pool.get()?; + let mut redis_logger = RedisJobStatusLogger::new_w2l_download( + &mut redis, + &job.token); + // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // let lock_acquired = grab_job_lock_and_mark_pending(&downloader.mysql_pool, job).await?; @@ -351,6 +371,8 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) // ==================== DOWNLOAD FILE ==================== // + redis_logger.log_status("downloading media")?; + let download_url = job.download_url.as_ref() .map(|c| c.to_string()) .unwrap_or("".to_string()); @@ -366,6 +388,8 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) // ==================== PROCESS FACES ==================== // + redis_logger.log_status("detecting faces")?; + // This is the Python Pickle file with all the face frames. // We'll retain it forever since it's expensive to compute. let cached_faces_filename = format!("{}_detected_faces.pickle", &download_filename); @@ -421,6 +445,8 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) let mut maybe_video_preview_filename : Option = None; let mut maybe_video_preview_object_name : Option = None; + redis_logger.log_status("generating preview")?; + if file_metadata.is_video { let preview_filename = format!("{}_preview.webp", &download_filename); @@ -461,6 +487,8 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) // ==================== UPLOAD TO BUCKETS ==================== // + redis_logger.log_status("uploading results")?; + info!("Image/video destination bucket path: {}", full_object_path); // Full path to cached faces @@ -575,6 +603,8 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) thread::sleep(Duration::from_millis(downloader.debug_job_end_sleep_millis)); } + redis_logger.log_status("done")?; + info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}", job.id, id); From 1fa3b9b880251572fe316fad15e0a711b0dfe385 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 28 Aug 2021 02:35:00 -0400 Subject: [PATCH 0531/1042] When the w2l job fails, fail permanently --- models/Wav2Lip/vocodes_process_upload.py | 14 +- .../w2l/get_w2l_upload_template_job_status.rs | 8 ++ src/job_queries/w2l_download_job_queries.rs | 24 ++++ .../wav2lip_process_upload_command.rs | 127 ++++++++++-------- src/w2l_download_job.rs | 22 ++- 5 files changed, 132 insertions(+), 63 deletions(-) diff --git a/models/Wav2Lip/vocodes_process_upload.py b/models/Wav2Lip/vocodes_process_upload.py index 5261ffd0e1..924ce674d5 100644 --- a/models/Wav2Lip/vocodes_process_upload.py +++ b/models/Wav2Lip/vocodes_process_upload.py @@ -127,6 +127,11 @@ if os.path.isfile(args.image_or_video_filename) \ elif args.is_image: args.static = True + +class FacelessFrameError(Exception): + """Throw this when there are frames without faces.""" + pass + def get_smoothened_boxes(boxes, T): for i in range(len(boxes)): if i + T > len(boxes): @@ -160,7 +165,7 @@ def face_detect(images): for rect, image in zip(predictions, images): if rect is None: cv2.imwrite('temp/faulty_frame.jpg', image) # check this frame where the face was not detected. - raise ValueError('Face not detected! Ensure the video contains a face in all the frames.') + raise FacelessFrameError('Face not detected! Ensure the video contains a face in all the frames.') y1 = max(0, rect[1] - pady1) y2 = min(image.shape[0], rect[3] + pady2) @@ -334,7 +339,12 @@ def main(tempdir): print('Number of frames to detect faces in: {}'.format(len(full_frames)), flush=True) print('Detecting faces...', flush=True) - _face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) + try: + _face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) + except FacelessFrameError as e: + print('One or more frames without faces were encountered.') + print(e) + sys.exit(5) print('Done detecting faces!', flush=True) diff --git a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs index 6424bd7cb7..e7eebdd0b4 100644 --- a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs +++ b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs @@ -45,6 +45,9 @@ pub struct W2lUploadTemplateJobStatusForResponse { /// This can denote inference progress, and the Python code can write to it. pub maybe_extra_status_description: Option, + /// If the job failed or permanently died, we might have a reason. + pub maybe_failure_reason: Option, + pub attempt_count: u8, pub maybe_template_token: Option, @@ -71,6 +74,8 @@ pub struct W2lUploadTemplateJobStatusRecord { pub attempt_count: i32, pub maybe_template_token: Option, + pub maybe_failure_reason: Option, + pub created_at: DateTime, pub updated_at: DateTime, } @@ -108,6 +113,8 @@ SELECT jobs.attempt_count, jobs.on_success_result_token as maybe_template_token, + jobs.failure_reason as maybe_failure_reason, + jobs.created_at, jobs.updated_at @@ -158,6 +165,7 @@ WHERE jobs.token = ? job_token: record.job_token.clone(), status: record.status.clone(), maybe_extra_status_description, + maybe_failure_reason: record.maybe_failure_reason.clone(), attempt_count: record.attempt_count as u8, maybe_template_token: record.maybe_template_token.clone(), created_at: record.created_at.clone(), diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 6f8d8705d9..0cc6394108 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -168,6 +168,30 @@ WHERE id = ? Ok(()) } + +pub async fn mark_w2l_template_upload_job_permanently_dead( + pool: &MySqlPool, + job: &W2lTemplateUploadJobRecord, + failure_reason: &str, +) -> AnyhowResult<()> { + let query_result = sqlx::query!( + r#" +UPDATE w2l_template_upload_jobs +SET + status = "dead", + failure_reason = ?, + retry_at = NULL +WHERE id = ? + "#, + failure_reason.to_string(), + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} + pub async fn mark_w2l_template_upload_job_done( pool: &MySqlPool, job: &W2lTemplateUploadJobRecord, diff --git a/src/script_execution/wav2lip_process_upload_command.rs b/src/script_execution/wav2lip_process_upload_command.rs index 5092a1d0f1..e60047fdea 100644 --- a/src/script_execution/wav2lip_process_upload_command.rs +++ b/src/script_execution/wav2lip_process_upload_command.rs @@ -1,8 +1,14 @@ use crate::AnyhowResult; use log::{info,warn}; use std::process::{Command, Stdio}; -use subprocess::{Popen, PopenConfig, Redirection}; +use subprocess::{Popen, PopenConfig, Redirection, ExitStatus}; use std::fs::OpenOptions; +use std::error::Error; +use std::fmt; +use std::fmt::Formatter; + +/// The python script uses this exit code when face detection fails. +const FACE_DETECT_FAILURE_CODE : u32 = 5; /// This command is used to preprocess the face detection frames from user-submitted video. /// This should only ever need to run once. The frames can then be uploaded to Buckets and saved. @@ -13,6 +19,26 @@ pub struct Wav2LipPreprocessClient { checkpoint_path: String, } +/// Command failure +#[derive(Debug, Copy, Clone)] +pub enum Wav2LipPreprocessError { + /// Permanent failure + NoFacesDetected, + /// Unknown. Retry allowed. + UnknownError, +} + +impl Error for Wav2LipPreprocessError {} + +impl fmt::Display for Wav2LipPreprocessError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Wav2LipPreprocessError::NoFacesDetected => write!(f, "NoFacesDetected"), + Wav2LipPreprocessError::UnknownError => write!(f, "UnknownError"), + } + } +} + impl Wav2LipPreprocessClient { pub fn new( w2l_directory: &str, @@ -31,7 +57,7 @@ impl Wav2LipPreprocessClient { output_cached_faces_filename: &str, output_metadata_filename: &str, is_image: bool, - spawn_process: bool) -> AnyhowResult<()> + spawn_process: bool) -> Result<(), Wav2LipPreprocessError> { let mut command = String::new(); @@ -58,67 +84,52 @@ impl Wav2LipPreprocessClient { info!("Command: {:?}", command); - let command_parts = [ - "bash", - "-c", - &command - ]; + // NB: Got rid of the previous (unused) child process spawning - if spawn_process { - // NB: This forks and returns immediately. - //let _child_pid = command_builder.spawn()?; + let exit_status = match self.do_execute(&command) { + Err(e) => { + warn!("Unknown execution error: {:?}", e); + return Err(Wav2LipPreprocessError::UnknownError); + }, + Ok(exit_status) => exit_status, + }; - let stdout_file = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(true) - .open("/tmp/wav2lip_upload_stdout.txt")?; + info!("Exit status: {:?}", exit_status); - let stderr_file = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(true) - .open("/tmp/wav2lip_upload_stderr.txt")?; - - let mut p = Popen::create(&command_parts, PopenConfig { - //stdout: Redirection::Pipe, - //stderr: Redirection::Pipe, - stdout: Redirection::File(stdout_file), - stderr: Redirection::File(stderr_file), - ..Default::default() - })?; - - info!("Pid : {:?}", p.pid()); - - p.detach(); - - } else { - // NB: This is a blocking call. - /*let output = command_builder.output()?; - - info!("Output status: {}", output.status); - info!("Stdout: {:?}", String::from_utf8(output.stdout)); - error!("Stderr: {:?}", String::from_utf8(output.stderr)); - - if !output.status.success() { - bail!("Bad error code: {:?}", output.status); - }*/ - - let mut p = Popen::create(&command_parts, PopenConfig { - //stdout: Redirection::Pipe, - //stderr: Redirection::Pipe, - ..Default::default() - })?; - - info!("Pid : {:?}", p.pid()); - - let exit_status = p.wait()?; - - info!("Exit status: {:?}", exit_status); + if !exit_status.success() { + if let ExitStatus::Exited(code) = exit_status { + if code == FACE_DETECT_FAILURE_CODE { + // We want to permanently fail retries. + warn!("Failure to detect faces error code returned. This is a permanent failure."); + return Err(Wav2LipPreprocessError::NoFacesDetected); + } + } + warn!("Unknown failure reason."); + return Err(Wav2LipPreprocessError::UnknownError); } Ok(()) } + + fn do_execute(&self, command: &str) -> AnyhowResult { + let command_parts = [ + "bash", + "-c", + command + ]; + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + let exit_status = p.wait()?; + + info!("Exit status: {:?}", exit_status); + + Ok(exit_status) + } } diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index b9044dea7d..d56c94a258 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -27,12 +27,13 @@ use crate::job_queries::w2l_download_job_queries::grab_job_lock_and_mark_pending use crate::job_queries::w2l_download_job_queries::insert_w2l_template; use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_done; use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_failure; +use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_permanently_dead; use crate::job_queries::w2l_download_job_queries::query_w2l_template_upload_job_records; use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; -use crate::script_execution::wav2lip_process_upload_command::Wav2LipPreprocessClient; +use crate::script_execution::wav2lip_process_upload_command::{Wav2LipPreprocessClient, Wav2LipPreprocessError}; use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; @@ -400,12 +401,27 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) let is_image = false; // TODO: Don't always treat as video. let spawn_process = false; - downloader.w2l_processor.execute( + let face_detect_result = downloader.w2l_processor.execute( &download_filename, &cached_faces_filename, &output_metadata_filename, is_image, - spawn_process)?; + spawn_process); + + match face_detect_result { + Ok(_) => {}, // Ok to proceed. + Err(Wav2LipPreprocessError::UnknownError) => {} // Ok to proceed. + Err(Wav2LipPreprocessError::NoFacesDetected) => { + // Permanently fail. + warn!("Permanently failed due to no face detection!"); + mark_w2l_template_upload_job_permanently_dead( + &downloader.mysql_pool, + job, + "faces must be present in every frame" + ).await?; + return Ok(()); + } + } if downloader.debug_face_detect_sleep_millis != 0 { warn!("Debug sleep after face detect: {} ms", downloader.debug_face_detect_sleep_millis); From cca6d9cdfcf01fb87c21f95c761d67e6864d0924 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 28 Aug 2021 02:38:24 -0400 Subject: [PATCH 0532/1042] Return empty maybe_extra_status from TTS upload (WIP) --- .../endpoints/tts/get_tts_upload_model_job_status.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs index 877698df2b..af1c7191ba 100644 --- a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs @@ -36,7 +36,14 @@ pub struct GetTtsUploadModelStatusPathInfo { pub struct TtsUploadModelJobStatusForResponse { pub job_token: String, + /// Primary status from the database (a state machine). pub status: String, + + // TODO: Not yet used. + /// Extra, temporary status from Redis. + /// This can denote inference progress, and the Python code can write to it. + pub maybe_extra_status_description: Option, + pub attempt_count: u8, pub maybe_model_token: Option, @@ -130,6 +137,7 @@ WHERE jobs.token = ? let model_for_response = TtsUploadModelJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), + maybe_extra_status_description: None, attempt_count: record.attempt_count as u8, maybe_model_token: record.maybe_model_token.clone(), created_at: record.created_at.clone(), From 5cc2ee3bdad627547e52c4afae7ae8582b0d7355 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 28 Aug 2021 04:06:38 -0400 Subject: [PATCH 0533/1042] report status of tts downloads --- .../tts/get_tts_upload_model_job_status.rs | 23 +++++++++++- src/tts_download_job.rs | 36 +++++++++++++++++-- src/util/redis/redis_job_status_logger.rs | 5 +++ src/util/redis/redis_keys.rs | 6 ++++ 4 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs index af1c7191ba..956fe3da5b 100644 --- a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs @@ -15,10 +15,12 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::redis::redis_keys::RedisKeys; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; +use r2d2_redis::redis::Commands; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; @@ -134,10 +136,29 @@ WHERE jobs.token = ? } }; + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + warn!("redis error: {:?}", e); + GetTtsUploadModelStatusError::ServerError + })?; + + let extra_status_key = RedisKeys::tts_download_extra_status_info(&path.token); + let maybe_extra_status_description : Option = match redis.get(&extra_status_key) { + Ok(Some(status)) => { + Some(status) + }, + Ok(None) => None, + Err(e) => { + warn!("redis error: {:?}", e); + None // Fail open + }, + }; + let model_for_response = TtsUploadModelJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), - maybe_extra_status_description: None, + maybe_extra_status_description, attempt_count: record.attempt_count as u8, maybe_model_token: record.maybe_model_token.clone(), created_at: record.created_at.clone(), diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index f5c7594071..95f30b27d4 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -40,13 +40,17 @@ use crate::util::filesystem::{check_directory_exists, check_file_exists}; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2; use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::File; use std::io::{BufReader, Read}; +use std::ops::Deref; use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; @@ -68,6 +72,9 @@ const ENV_MODEL_CHECK_SCRIPT_NAME : &'static str = "TTS_MODEL_CHECK_SCRIPT_NAME" struct Downloader { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, + + pub redis_pool: r2d2::Pool, + pub bucket_client: BucketClient, pub firehose_publisher: FirehosePublisher, pub badge_granter: BadgeGranter, @@ -149,6 +156,16 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; + let common_env = CommonEnv::read_from_env()?; + + info!("Connecting to redis..."); + + let redis_manager = + RedisConnectionManager::new(common_env.redis_connection_string.deref())?; + + let redis_pool = r2d2::Pool::builder() + .build(redis_manager)?; + let firehose_publisher = FirehosePublisher { mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe }; @@ -166,11 +183,10 @@ async fn main() -> AnyhowResult<()> { &py_script_name, ); - let common_env = CommonEnv::read_from_env()?; - let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, + redis_pool, bucket_client, download_script, google_drive_downloader, @@ -276,6 +292,11 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho // TODO: 4. Save record. (DONE) // TODO: 5. Mark job done. (DONE) + let mut redis = downloader.redis_pool.get()?; + let mut redis_logger = RedisJobStatusLogger::new_tts_download( + &mut redis, + &job.token); + // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // let lock_acquired = grab_job_lock_and_mark_pending(&downloader.mysql_pool, job).await?; @@ -292,11 +313,14 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho // ==================== DOWNLOAD MODEL FILE ==================== // + info!("Calling downloader..."); + + redis_logger.log_status("downloading model")?; + let download_url = job.download_url.as_ref() .map(|c| c.to_string()) .unwrap_or("".to_string()); - info!("Calling downloader..."); let download_filename = downloader.google_drive_downloader .download_file(&download_url, &temp_dir).await?; @@ -314,6 +338,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("Checking that model is valid..."); + redis_logger.log_status("checking model")?; + let file_path = PathBuf::from(download_filename); let output_metadata_fs_path = temp_dir.path().join("metadata.json"); @@ -336,6 +362,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("Uploading model to GCS..."); + redis_logger.log_status("uploading model")?; + downloader.bucket_client.upload_filename(&synthesizer_model_bucket_path, &file_path).await?; // ==================== SAVE RECORDS ==================== // @@ -373,6 +401,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho anyhow!("error maybe awarding badge") })?; + redis_logger.log_status("done")?; + info!("Job done: {}", job.id); Ok(()) diff --git a/src/util/redis/redis_job_status_logger.rs b/src/util/redis/redis_job_status_logger.rs index a407e80339..42d2dec919 100644 --- a/src/util/redis/redis_job_status_logger.rs +++ b/src/util/redis/redis_job_status_logger.rs @@ -14,6 +14,11 @@ pub struct RedisJobStatusLogger <'a> { impl <'a> RedisJobStatusLogger <'a> { pub const STATUS_KEY_TTL_SECONDS : usize = 60 * 60; + pub fn new_tts_download(redis: &'a mut RedisPool, w2l_job_token: &str) -> Self { + let status_key = RedisKeys::tts_download_extra_status_info(w2l_job_token); + Self { redis, status_key } + } + pub fn new_tts_inference(redis: &'a mut RedisPool, tts_job_token: &str) -> Self { let status_key = RedisKeys::tts_inference_extra_status_info(tts_job_token); Self { redis, status_key } diff --git a/src/util/redis/redis_keys.rs b/src/util/redis/redis_keys.rs index 3455c60183..260c2ba8dc 100644 --- a/src/util/redis/redis_keys.rs +++ b/src/util/redis/redis_keys.rs @@ -16,6 +16,12 @@ impl RedisKeys { format!("w2lUseCount:{}", template_token) } + /// We write extra in-progress status information to keys. + /// These keys should have a TTL. + pub fn tts_download_extra_status_info(inference_job_token: &str) -> String { + format!("ttsDownloadExtraStatus:{}", inference_job_token) + } + /// We write extra in-progress status information to keys. /// These keys should have a TTL. pub fn tts_inference_extra_status_info(inference_job_token: &str) -> String { From b897245a09b013a5b4a9401ffb7e0a9da66bb23f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 28 Aug 2021 04:36:39 -0400 Subject: [PATCH 0534/1042] re-codegen db --- sqlx-data.json | 203 +++++++++++++++++++++++++++---------------------- 1 file changed, 113 insertions(+), 90 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 2dbe11478f..531d9e09e9 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -3095,96 +3095,6 @@ ] } }, - "5c13a3969006332b5910fcfd4cd58eecbcd5866059b76afff610a2ad3900d094": { - "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_template_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_template_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 2, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 3 - } - }, - { - "ordinal": 3, - "name": "maybe_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 5, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - false, - false - ] - } - }, "5e54fd8569b2d6d3a0d11acee6bd03b75400b6325952eb16c8dac68a7933b13a": { "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n is_banned,\n disable_gravatar,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", "describe": { @@ -5623,6 +5533,16 @@ "nullable": [] } }, + "ab981965f518a8986aaaba41ed821fcb438db4c6156cbb26f639921225a88d59": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = \"dead\",\n failure_reason = ?,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "adc494ddafaa13aebe423984f32a0d8810da604b19f5d6e0081b5e891dfc0f52": { "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -5643,6 +5563,109 @@ "nullable": [] } }, + "ae04054c4d32e581dd183fd7ebb8e48fc7df856d2ffd8036d9ce6006bb7a296a": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_template_token,\n\n jobs.failure_reason as maybe_failure_reason,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_template_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 3, + "name": "maybe_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + false, + false + ] + } + }, "b1ad806aa93f66cab358d7cdb405d57c643e2ff131cfec9f1ced55c2e6ca9511": { "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { From f9871d9d051f9fa9398df63b21d001606368f27b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 28 Aug 2021 23:00:32 -0400 Subject: [PATCH 0535/1042] Add NeMo from https://github.com/SortAnon/NeMo --- models/tacotron2/NeMo/.dockerignore | 19 + .../NeMo/.github/ISSUE_TEMPLATE/bug_report.md | 42 + .../.github/ISSUE_TEMPLATE/feature_request.md | 25 + models/tacotron2/NeMo/.gitignore | 176 + models/tacotron2/NeMo/.readthedocs.yml | 31 + models/tacotron2/NeMo/CONTRIBUTING.md | 68 + models/tacotron2/NeMo/Dockerfile | 104 + models/tacotron2/NeMo/Jenkinsfile | 1451 ++ models/tacotron2/NeMo/LICENSE | 201 + models/tacotron2/NeMo/README.rst | 174 + models/tacotron2/NeMo/docs/.nojekyll | 0 models/tacotron2/NeMo/docs/Makefile | 216 + .../NeMo/docs/source/_static/css/custom.css | 199 + .../NeMo/docs/source/_static/js/pk_scripts.js | 19 + models/tacotron2/NeMo/docs/source/asr/api.rst | 203 + .../NeMo/docs/source/asr/asr_all.bib | 980 + .../docs/source/asr/asr_language_modeling.rst | 292 + .../NeMo/docs/source/asr/configs.rst | 528 + .../NeMo/docs/source/asr/data/asr_results.csv | 21 + .../docs/source/asr/data/asrlm_results.csv | 2 + .../docs/source/asr/data/benchmark_ca.csv | 2 + .../docs/source/asr/data/benchmark_de.csv | 3 + .../docs/source/asr/data/benchmark_en.csv | 14 + .../docs/source/asr/data/benchmark_es.csv | 3 + .../docs/source/asr/data/benchmark_fr.csv | 3 + .../docs/source/asr/data/benchmark_it.csv | 3 + .../docs/source/asr/data/benchmark_pl.csv | 2 + .../docs/source/asr/data/benchmark_ru.csv | 3 + .../docs/source/asr/data/benchmark_zh.csv | 3 + .../NeMo/docs/source/asr/datasets.rst | 268 + .../source/asr/images/citrinet_vertical.png | Bin 0 -> 138615 bytes .../docs/source/asr/images/conformer_ctc.png | Bin 0 -> 105304 bytes .../NeMo/docs/source/asr/images/ctc_asr.png | Bin 0 -> 54705 bytes .../docs/source/asr/images/jasper_layers.png | Bin 0 -> 25461 bytes .../source/asr/images/jasper_vertical.png | Bin 0 -> 279466 bytes .../source/asr/images/quartz_vertical.png | Bin 0 -> 262776 bytes .../tacotron2/NeMo/docs/source/asr/intro.rst | 59 + .../tacotron2/NeMo/docs/source/asr/models.rst | 97 + .../NeMo/docs/source/asr/results.rst | 181 + .../source/asr/speaker_diarization/api.rst | 17 + .../asr/speaker_diarization/configs.rst | 65 + .../data/diarization_results.csv | 4 + .../asr/speaker_diarization/datasets.rst | 72 + .../images/sd_pipeline.png | Bin 0 -> 55717 bytes .../source/asr/speaker_diarization/intro.rst | 52 + .../source/asr/speaker_diarization/models.rst | 5 + .../asr/speaker_diarization/results.rst | 76 + .../asr/speaker_recognition/configs.rst | 117 + .../data/speaker_results.csv | 4 + .../asr/speaker_recognition/datasets.rst | 57 + .../images/ICASPP_SpeakerNet.png | Bin 0 -> 211990 bytes .../source/asr/speaker_recognition/intro.rst | 48 + .../source/asr/speaker_recognition/models.rst | 52 + .../asr/speaker_recognition/results.rst | 85 + .../asr/speech_classification/configs.rst | 104 + .../data/classification_results.csv | 9 + .../asr/speech_classification/datasets.rst | 131 + .../images/marblenet_vertical.png | Bin 0 -> 393467 bytes .../images/matchboxnet_vertical.png | Bin 0 -> 490256 bytes .../asr/speech_classification/intro.rst | 47 + .../asr/speech_classification/models.rst | 71 + .../asr/speech_classification/results.rst | 105 + .../NeMo/docs/source/common/intro.rst | 39 + models/tacotron2/NeMo/docs/source/conf.py | 242 + .../tacotron2/NeMo/docs/source/core/core.rst | 963 + .../NeMo/docs/source/core/whyntypes.gif | Bin 0 -> 309073 bytes models/tacotron2/NeMo/docs/source/favicon.ico | Bin 0 -> 25214 bytes models/tacotron2/NeMo/docs/source/index.rst | 67 + .../docs/source/nemo_text_processing/api.rst | 35 + .../source/nemo_text_processing/intro.rst | 17 + .../inverse_text_normalization.rst | 90 + .../text_normalization.rst | 78 + .../textprocessing_all.bib | 58 + models/tacotron2/NeMo/docs/source/nlp/api.rst | 74 + .../NeMo/docs/source/nlp/bert_pretraining.rst | 133 + .../NeMo/docs/source/nlp/glue_benchmark.rst | 10 + .../docs/source/nlp/information_retrieval.rst | 10 + .../tacotron2/NeMo/docs/source/nlp/intro.rst | 15 + .../docs/source/nlp/joint_intent_slot.rst | 244 + .../docs/source/nlp/language_modeling.rst | 6 + .../docs/source/nlp/machine_translation.rst | 466 + .../docs/source/nlp/megatron_finetuning.rst | 216 + .../tacotron2/NeMo/docs/source/nlp/models.rst | 22 + .../NeMo/docs/source/nlp/nlp_all.bib | 109 + .../NeMo/docs/source/nlp/nlp_model.rst | 43 + .../nlp/punctuation_and_capitalization.rst | 300 + .../docs/source/nlp/question_answering.rst | 300 + .../tacotron2/NeMo/docs/source/nlp/sgd_qa.rst | 17 + .../docs/source/nlp/text_classification.rst | 169 + .../docs/source/nlp/token_classification.rst | 219 + models/tacotron2/NeMo/docs/source/nv_logo.png | Bin 0 -> 109116 bytes .../docs/source/starthere/best-practices.rst | 312 + .../NeMo/docs/source/starthere/intro.rst | 157 + .../NeMo/docs/source/starthere/tutorials.rst | 117 + .../docs/source/tools/ctc_segmentation.rst | 20 + .../NeMo/docs/source/tools/intro.rst | 14 + .../source/tools/speech_data_explorer.rst | 33 + .../tools/text_processing_deployment.rst | 78 + .../NeMo/docs/source/tools/tools_all.bib | 49 + models/tacotron2/NeMo/docs/source/tts/api.rst | 47 + .../NeMo/docs/source/tts/datasets.rst | 187 + .../tacotron2/NeMo/docs/source/tts/intro.rst | 153 + .../tacotron2/NeMo/docs/update_docs_docker.sh | 3 + .../NeMo/examples/asr/asr_webapp/Dockerfile | 41 + .../NeMo/examples/asr/asr_webapp/README.rst | 69 + .../examples/asr/asr_webapp/asr_service.py | 374 + .../asr/asr_webapp/docker_container_build.sh | 19 + .../asr/asr_webapp/docker_container_run.sh | 20 + .../examples/asr/asr_webapp/gunicorn.conf.py | 36 + .../NeMo/examples/asr/asr_webapp/model_api.py | 131 + .../examples/asr/asr_webapp/requirements.txt | 4 + .../asr/asr_webapp/templates/main.html | 150 + .../asr/asr_webapp/templates/toast_msg.html | 28 + .../asr/asr_webapp/templates/transcripts.html | 43 + .../templates/updates/remove_files.html | 22 + .../updates/upload_files_failed.html | 22 + .../updates/upload_files_successful.html | 22 + .../NeMo/examples/asr/asr_webapp/wsgi.py | 18 + .../asr/conf/citrinet/citrinet_384.yaml | 428 + .../asr/conf/citrinet/citrinet_512.yaml | 428 + .../asr/conf/citrinet/config_bpe.yaml | 183 + .../NeMo/examples/asr/conf/config.yaml | 183 + .../asr/conf/conformer/conformer_ctc_bpe.yaml | 189 + .../conf/conformer/conformer_ctc_char.yaml | 165 + .../asr/conf/jasper/jasper_10x5dr.yaml | 208 + .../asr/conf/marblenet/marblenet_3x2x64.yaml | 180 + .../matchboxnet/matchboxnet_3x1x64_v1.yaml | 193 + .../matchboxnet/matchboxnet_3x1x64_v2.yaml | 193 + .../asr/conf/quartznet/quartznet_15x5.yaml | 281 + .../asr/conf/quartznet/quartznet_15x5_zh.yaml | 483 + .../conformer/conformer_rnnt_bpe.yaml | 201 + .../contextnet_rnnt/config_rnnt.yaml | 237 + .../contextnet_rnnt/config_rnnt_bpe.yaml | 238 + .../quartznet/quartznet_15x5_aug.yaml | 290 + .../sclite/speech_to_text_sclite.py | 158 + .../structured/conf/quartznet_15x5.yaml | 235 + .../structured/speech_to_text_hybrid.py | 56 + .../structured/speech_to_text_structured.py | 146 + .../speech_to_text_structured_v2.py | 90 + .../wav2vec/configs/wav2vec_pretrain.yaml | 123 + .../configs/wav2vec_pretrain_large.yaml | 123 + .../experimental/wav2vec/wav2vec_pretrain.py | 97 + .../quantization/speech_to_text_calibrate.py | 160 + .../speech_to_text_quant_infer.py | 216 + .../speech_to_text_quant_infer_trt.py | 226 + .../NeMo/examples/asr/speech_to_label.py | 146 + .../NeMo/examples/asr/speech_to_text.py | 105 + .../NeMo/examples/asr/speech_to_text_bpe.py | 101 + .../NeMo/examples/asr/speech_to_text_infer.py | 114 + .../NeMo/examples/asr/speech_to_text_rnnt.py | 104 + .../examples/asr/speech_to_text_rnnt_bpe.py | 96 + .../NeMo/examples/asr/transcribe_speech.py | 167 + .../tacotron2/NeMo/examples/asr/vad_infer.py | 171 + ...et5_image_classification_pure_lightning.py | 65 + .../conf/sgdqa_config.yaml | 115 + .../nlp/dialogue_state_tracking/sgd_qa.py | 145 + .../nlp/entity_linking/build_index.py | 201 + .../tiny_example_entity_linking_config.yaml | 90 + .../umls_medical_entity_linking_config.yaml | 95 + .../data/umls_dataset_processing.py | 189 + .../nlp/entity_linking/query_index.py | 166 + .../self_alignment_pretraining.py | 49 + .../nlp/glue_benchmark/glue_benchmark.py | 73 + .../glue_benchmark/glue_benchmark_config.yaml | 82 + .../nlp/information_retrieval/bert_dpr.py | 35 + .../information_retrieval/bert_joint_ir.py | 35 + .../conf/bert_ir_config.yaml | 99 + .../construct_random_negatives.py | 49 + .../nlp/information_retrieval/get_msmarco.sh | 22 + .../intent_slot_classification_config.yaml | 111 + .../data/assistant_utils.py | 151 + .../data/import_datasets.py | 265 + .../intent_slot_classification.py | 85 + .../nlp/language_modeling/bert_pretraining.py | 38 + ..._pretraining_from_preprocessed_config.yaml | 80 + .../bert_pretraining_from_text_config.yaml | 109 + .../conf/transformer_lm_config.yaml | 96 + .../convert_weights_to_nemo1.0.py | 61 + .../nlp/language_modeling/get_wkt2.sh | 23 + .../nlp/language_modeling/transformer_lm.py | 35 + .../machine_translation/conf/aayn_base.yaml | 156 + .../machine_translation/conf/huggingface.yaml | 132 + .../create_tarred_monolingual_dataset.py | 68 + .../create_tarred_parallel_dataset.py | 143 + .../nlp/machine_translation/enc_dec_nmt.py | 137 + .../nmt_transformer_infer.py | 122 + .../machine_translation/nmt_webapp/README.rst | 7 + .../nmt_webapp/config.json | 12 + .../machine_translation/nmt_webapp/index.html | 140 + .../nmt_webapp/nmt_service.py | 100 + .../nmt_webapp/requirements.txt | 3 + .../machine_translation/nmt_webapp/style.css | 111 + .../nlp/machine_translation/translate_ddp.py | 122 + .../conf/question_answering_squad_config.yaml | 135 + .../nlp/question_answering/get_squad.py | 68 + .../question_answering_squad.py | 134 + .../text2sparql/conf/text2sparql_config.yaml | 105 + .../nlp/text2sparql/data/import_datasets.py | 134 + .../nlp/text2sparql/evaluate_text2sparql.py | 72 + .../examples/nlp/text2sparql/text2sparql.py | 112 + .../conf/text_classification_config.yaml | 119 + .../data/import_datasets.py | 243 + ...parallel_text_classification_evaluation.py | 41 + .../text_classification_with_bert.py | 154 + .../punctuation_capitalization_config.yaml | 133 + .../conf/token_classification_config.yaml | 119 + .../data/get_tatoeba_data.py | 242 + .../data/import_from_iob_format.py | 124 + ...are_data_for_punctuation_capitalization.py | 99 + .../punctuation_capitalization_evaluate.py | 127 + .../punctuation_capitalization_train.py | 101 + .../token_classification_evaluate.py | 137 + .../token_classification_train.py | 147 + .../examples/speaker_recognition/README.md | 21 + .../conf/SpeakerNet_recognition_3x2x512.yaml | 157 + .../conf/SpeakerNet_verification_3x2x512.yaml | 145 + .../conf/speaker_diarization.yaml | 27 + .../extract_speaker_embeddings.py | 101 + .../speaker_recognition/speaker_diarize.py | 49 + .../speaker_recognition/speaker_reco.py | 73 + .../speaker_reco_finetune.py | 98 + .../speaker_recognition/speaker_reco_infer.py | 139 + .../speaker_recognition/voxceleb_eval.py | 110 + models/tacotron2/NeMo/examples/tts/aligner.py | 33 + .../NeMo/examples/tts/conf/aligner.yaml | 123 + .../NeMo/examples/tts/conf/degli.yaml | 107 + .../NeMo/examples/tts/conf/ed_mel2spec.yaml | 103 + .../NeMo/examples/tts/conf/fastpitch.yaml | 144 + .../examples/tts/conf/fastpitch_align.yaml | 185 + .../tts/conf/fastpitch_hifigan_e2e.yaml | 138 + .../NeMo/examples/tts/conf/fastspeech2.yaml | 144 + .../tts/conf/fastspeech2_hifigan_e2e.yaml | 136 + .../NeMo/examples/tts/conf/glow_tts.yaml | 143 + .../examples/tts/conf/hifigan/hifigan.yaml | 67 + .../tts/conf/hifigan/model/generator/v1.yaml | 8 + .../tts/conf/hifigan/model/generator/v2.yaml | 8 + .../tts/conf/hifigan/model/generator/v3.yaml | 8 + .../conf/hifigan/model/train_ds/train_ds.yaml | 13 + .../model/train_ds/train_ds_finetune.yaml | 11 + .../hifigan/model/validation_ds/val_ds.yaml | 13 + .../model/validation_ds/val_ds_finetune.yaml | 11 + .../NeMo/examples/tts/conf/melgan.yaml | 108 + .../NeMo/examples/tts/conf/squeezewave.yaml | 104 + .../NeMo/examples/tts/conf/tacotron2.yaml | 140 + .../NeMo/examples/tts/conf/talknet-durs.yaml | 175 + .../NeMo/examples/tts/conf/talknet-pitch.yaml | 179 + .../NeMo/examples/tts/conf/talknet-spect.yaml | 244 + .../NeMo/examples/tts/conf/uniglow.yaml | 96 + .../NeMo/examples/tts/conf/waveglow.yaml | 97 + models/tacotron2/NeMo/examples/tts/degli.py | 36 + .../NeMo/examples/tts/ed_mel2spec.py | 36 + .../tacotron2/NeMo/examples/tts/fastpitch.py | 35 + .../examples/tts/fastpitch_hifigan_e2e.py | 36 + .../NeMo/examples/tts/fastspeech2.py | 35 + .../examples/tts/fastspeech2_hifigan_e2e.py | 35 + .../tacotron2/NeMo/examples/tts/glow_tts.py | 35 + models/tacotron2/NeMo/examples/tts/hifigan.py | 31 + models/tacotron2/NeMo/examples/tts/melgan.py | 35 + .../NeMo/examples/tts/squeezewave.py | 34 + .../tacotron2/NeMo/examples/tts/tacotron2.py | 44 + .../NeMo/examples/tts/talknet_durs.py | 33 + .../NeMo/examples/tts/talknet_pitch.py | 33 + .../NeMo/examples/tts/talknet_spect.py | 33 + .../NeMo/examples/tts/test_tts_infer.py | 153 + models/tacotron2/NeMo/examples/tts/uniglow.py | 44 + .../tacotron2/NeMo/examples/tts/waveglow.py | 34 + .../NeMo/external/get_collections.py | 90 + models/tacotron2/NeMo/external/get_modules.py | 159 + models/tacotron2/NeMo/nemo/README.md | 9 + models/tacotron2/NeMo/nemo/__init__.py | 28 + .../NeMo/nemo/collections/__init__.py | 13 + .../NeMo/nemo/collections/asr/__init__.py | 25 + .../nemo/collections/asr/data/__init__.py | 13 + .../collections/asr/data/audio_to_label.py | 921 + .../asr/data/audio_to_label_dataset.py | 129 + .../collections/asr/data/audio_to_text.py | 1330 ++ .../asr/data/audio_to_text_dali.py | 444 + .../asr/data/audio_to_text_dataset.py | 247 + .../collections/asr/data/feature_to_label.py | 139 + .../asr/data/feature_to_label_dataset.py | 31 + .../NeMo/nemo/collections/asr/data/vocabs.py | 352 + .../nemo/collections/asr/losses/__init__.py | 16 + .../collections/asr/losses/angularloss.py | 68 + .../NeMo/nemo/collections/asr/losses/ctc.py | 70 + .../NeMo/nemo/collections/asr/losses/rnnt.py | 288 + .../collections/asr/losses/wav2vecloss.py | 118 + .../nemo/collections/asr/metrics/__init__.py | 13 + .../nemo/collections/asr/metrics/rnnt_wer.py | 478 + .../collections/asr/metrics/rnnt_wer_bpe.py | 242 + .../NeMo/nemo/collections/asr/metrics/wer.py | 247 + .../nemo/collections/asr/metrics/wer_bpe.py | 210 + .../nemo/collections/asr/models/__init__.py | 22 + .../nemo/collections/asr/models/asr_model.py | 91 + .../asr/models/classification_models.py | 750 + .../asr/models/clustering_diarizer.py | 439 + .../asr/models/configs/__init__.py | 46 + .../configs/classification_models_config.py | 95 + .../asr/models/configs/ctc_models_config.py | 85 + .../asr/models/configs/matchboxnet_config.py | 253 + .../asr/models/configs/quartznet_config.py | 308 + .../collections/asr/models/ctc_bpe_models.py | 304 + .../nemo/collections/asr/models/ctc_models.py | 649 + .../collections/asr/models/label_models.py | 409 + .../collections/asr/models/rnnt_bpe_models.py | 290 + .../collections/asr/models/rnnt_models.py | 835 + .../asr/models/wav2vec/__init__.py | 14 + .../asr/models/wav2vec/wav2vec_config.py | 180 + .../asr/models/wav2vec/wav2vec_model.py | 504 + .../nemo/collections/asr/modules/__init__.py | 30 + .../asr/modules/audio_preprocessing.py | 643 + .../asr/modules/beam_search_decoder.py | 103 + .../asr/modules/conformer_encoder.py | 255 + .../nemo/collections/asr/modules/conv_asr.py | 504 + .../collections/asr/modules/lstm_decoder.py | 95 + .../NeMo/nemo/collections/asr/modules/rnnt.py | 958 + .../collections/asr/modules/rnnt_abstract.py | 236 + .../asr/modules/wav2vec_modules.py | 306 + .../nemo/collections/asr/parts/__init__.py | 13 + .../nemo/collections/asr/parts/features.py | 39 + .../collections/asr/parts/mixins/__init__.py | 15 + .../collections/asr/parts/mixins/mixins.py | 193 + .../collections/asr/parts/numba/__init__.py | 15 + .../asr/parts/numba/rnnt_loss/__init__.py | 16 + .../asr/parts/numba/rnnt_loss/rnnt.py | 225 + .../asr/parts/numba/rnnt_loss/rnnt_numpy.py | 213 + .../asr/parts/numba/rnnt_loss/rnnt_pytorch.py | 186 + .../parts/numba/rnnt_loss/utils/__init__.py | 13 + .../rnnt_loss/utils/cpu_utils/__init__.py | 27 + .../rnnt_loss/utils/cpu_utils/cpu_rnnt.py | 378 + .../rnnt_loss/utils/cuda_utils/__init__.py | 27 + .../rnnt_loss/utils/cuda_utils/gpu_rnnt.py | 254 + .../utils/cuda_utils/gpu_rnnt_kernel.py | 365 + .../rnnt_loss/utils/cuda_utils/reduce.py | 362 + .../numba/rnnt_loss/utils/global_constants.py | 68 + .../numba/rnnt_loss/utils/rnnt_helper.py | 129 + .../asr/parts/numba/spec_augment/__init__.py | 18 + .../numba/spec_augment/spec_aug_numba.py | 290 + .../asr/parts/preprocessing/__init__.py | 41 + .../asr/parts/preprocessing/feature_loader.py | 69 + .../asr/parts/preprocessing/features.py | 432 + .../asr/parts/preprocessing/perturb.py | 895 + .../asr/parts/preprocessing/segment.py | 270 + .../asr/parts/submodules/__init__.py | 13 + .../asr/parts/submodules/conformer_modules.py | 190 + .../asr/parts/submodules/jasper.py | 960 + .../parts/submodules/multi_head_attention.py | 340 + .../parts/submodules/rnnt_beam_decoding.py | 791 + .../parts/submodules/rnnt_greedy_decoding.py | 731 + .../asr/parts/submodules/spectr_augment.py | 152 + .../asr/parts/submodules/subsampling.py | 138 + .../asr/parts/submodules/wav2vec.py | 222 + .../collections/asr/parts/utils/__init__.py | 13 + .../asr/parts/utils/activations.py | 24 + .../asr/parts/utils/asr_module_utils.py | 82 + .../asr/parts/utils/nmse_clustering.py | 271 + .../asr/parts/utils/numba_utils.py | 88 + .../collections/asr/parts/utils/rnnt_utils.py | 89 + .../asr/parts/utils/speaker_utils.py | 400 + .../collections/asr/parts/utils/vad_utils.py | 544 + .../NeMo/nemo/collections/common/__init__.py | 26 + .../collections/common/callbacks/__init__.py | 15 + .../collections/common/callbacks/callbacks.py | 97 + .../nemo/collections/common/data/__init__.py | 15 + .../nemo/collections/common/data/dataset.py | 178 + .../collections/common/losses/__init__.py | 20 + .../collections/common/losses/aggregator.py | 67 + .../common/losses/cross_entropy.py | 138 + .../collections/common/losses/mse_loss.py | 57 + .../common/losses/multi_similarity_loss.py | 95 + .../common/losses/smoothed_cross_entropy.py | 103 + .../common/losses/spanning_loss.py | 79 + .../collections/common/metrics/__init__.py | 17 + .../common/metrics/classification_accuracy.py | 152 + .../metrics/global_average_loss_metric.py | 73 + .../collections/common/metrics/perplexity.py | 71 + .../nemo/collections/common/parts/__init__.py | 17 + .../common/parts/multi_layer_perceptron.py | 61 + .../collections/common/parts/patch_utils.py | 132 + .../common/parts/preprocessing/__init__.py | 13 + .../common/parts/preprocessing/cleaners.py | 259 + .../common/parts/preprocessing/collections.py | 468 + .../common/parts/preprocessing/manifest.py | 120 + .../common/parts/preprocessing/parsers.py | 185 + .../NeMo/nemo/collections/common/parts/rnn.py | 553 + .../common/parts/transformer_utils.py | 79 + .../nemo/collections/common/parts/utils.py | 57 + .../collections/common/tokenizers/__init__.py | 19 + .../common/tokenizers/char_tokenizer.py | 142 + .../common/tokenizers/chinese_tokenizers.py | 63 + .../common/tokenizers/en_ja_tokenizers.py | 57 + .../common/tokenizers/fairseq_tokenizer.py | 126 + .../common/tokenizers/huggingface/__init__.py | 15 + .../tokenizers/huggingface/auto_tokenizer.py | 260 + .../common/tokenizers/moses_tokenizers.py | 47 + .../tokenizers/sentencepiece_tokenizer.py | 353 + .../common/tokenizers/tokenizer_spec.py | 55 + .../common/tokenizers/word_tokenizer.py | 72 + .../tokenizers/youtokentome_tokenizer.py | 70 + .../NeMo/nemo/collections/cv/__init__.py | 25 + .../nemo/collections/cv/datasets/__init__.py | 15 + .../collections/cv/datasets/mnist_dataset.py | 134 + .../nemo/collections/cv/losses/__init__.py | 15 + .../nemo/collections/cv/losses/nll_loss.py | 54 + .../nemo/collections/cv/models/__init__.py | 15 + .../collections/cv/models/mnist_lenet5.py | 157 + .../nemo/collections/cv/modules/__init__.py | 15 + .../nemo/collections/cv/modules/lenet5.py | 114 + .../NeMo/nemo/collections/nlp/__init__.py | 25 + .../nemo/collections/nlp/data/__init__.py | 40 + .../nlp/data/data_utils/__init__.py | 15 + .../nlp/data/data_utils/data_preprocessing.py | 479 + .../data/dialogue_state_tracking/__init__.py | 17 + .../dialogue_state_tracking/sgd/__init__.py | 13 + .../sgd/data_processor.py | 676 + .../dialogue_state_tracking/sgd/dataset.py | 85 + .../dialogue_state_tracking/sgd/evaluate.py | 294 + .../sgd/input_example.py | 399 + .../dialogue_state_tracking/sgd/metrics.py | 341 + .../sgd/prediction_utils.py | 251 + .../dialogue_state_tracking/sgd/schema.py | 222 + .../nlp/data/entity_linking/__init__.py | 15 + .../entity_linking/entity_linking_dataset.py | 135 + .../nlp/data/glue_benchmark/__init__.py | 15 + .../data/glue_benchmark/data_processors.py | 323 + .../glue_benchmark/glue_benchmark_dataset.py | 365 + .../data/information_retrieval/__init__.py | 17 + .../information_retrieval_dataset.py | 278 + .../intent_slot_classification/__init__.py | 22 + .../intent_slot_classification_dataset.py | 292 + .../intent_slot_classification_descriptor.py | 163 + .../nlp/data/language_modeling/__init__.py | 20 + .../data/language_modeling/l2r_lm_dataset.py | 240 + .../data/language_modeling/lm_bert_dataset.py | 406 + .../language_modeling/sentence_dataset.py | 273 + .../nlp/data/machine_translation/__init__.py | 18 + .../machine_translation_dataset.py | 445 + .../machine_translation/preproc_mt_data.py | 1020 + .../neural_machine_translation/__init__.py | 18 + .../neural_machine_translation_dataset.py | 146 + .../data/question_answering_squad/__init__.py | 13 + .../question_answering_squad/qa_dataset.py | 443 + .../qa_squad_processing.py | 644 + .../nlp/data/text_classification/__init__.py | 19 + .../text_classification_dataset.py | 297 + .../nlp/data/token_classification/__init__.py | 13 + .../punctuation_capitalization_dataset.py | 466 + .../token_classification_dataset.py | 338 + .../token_classification_utils.py | 118 + .../nemo/collections/nlp/losses/__init__.py | 15 + .../nemo/collections/nlp/losses/sgd_loss.py | 218 + .../nemo/collections/nlp/metrics/__init__.py | 16 + .../nlp/metrics/classification_report.py | 173 + .../nlp/metrics/sequence_perplexity.py | 78 + .../nemo/collections/nlp/models/__init__.py | 28 + .../dialogue_state_tracking/__init__.py | 15 + .../dialogue_state_tracking/sgdqa_model.py | 580 + .../nlp/models/enc_dec_nlp_model.py | 85 + .../nlp/models/entity_linking/__init__.py | 15 + .../entity_linking/entity_linking_model.py | 193 + .../nlp/models/glue_benchmark/__init__.py | 15 + .../glue_benchmark/glue_benchmark_model.py | 279 + .../models/glue_benchmark/metrics_for_glue.py | 66 + .../models/information_retrieval/__init__.py | 16 + .../information_retrieval/base_ir_model.py | 211 + .../information_retrieval/bert_dpr_model.py | 131 + .../bert_joint_ir_model.py | 84 + .../intent_slot_classification/__init__.py | 17 + .../intent_slot_classification_model.py | 487 + .../nlp/models/language_modeling/__init__.py | 16 + .../models/language_modeling/bert_lm_model.py | 272 + .../language_modeling/transformer_lm_model.py | 276 + .../models/machine_translation/__init__.py | 15 + .../machine_translation/mt_enc_dec_config.py | 134 + .../machine_translation/mt_enc_dec_model.py | 836 + .../neural_machine_translation/__init__.py | 17 + .../neural_machine_translation_model.py | 245 + .../nemo/collections/nlp/models/nlp_model.py | 463 + .../nlp/models/question_answering/__init__.py | 15 + .../nlp/models/question_answering/qa_model.py | 404 + .../models/text_classification/__init__.py | 15 + .../text_classification_model.py | 324 + .../models/token_classification/__init__.py | 19 + .../punctuation_capitalization_model.py | 490 + .../token_classification_model.py | 522 + .../nemo/collections/nlp/modules/__init__.py | 33 + .../nlp/modules/common/__init__.py | 34 + .../nlp/modules/common/bert_module.py | 93 + .../nlp/modules/common/classifier.py | 87 + .../nlp/modules/common/decoder_module.py | 58 + .../nlp/modules/common/encoder_module.py | 41 + .../modules/common/huggingface/__init__.py | 22 + .../nlp/modules/common/huggingface/albert.py | 33 + .../nlp/modules/common/huggingface/bert.py | 33 + .../modules/common/huggingface/distilbert.py | 34 + .../common/huggingface/huggingface_decoder.py | 79 + .../common/huggingface/huggingface_encoder.py | 99 + .../common/huggingface/huggingface_utils.py | 126 + .../nlp/modules/common/huggingface/roberta.py | 34 + .../nlp/modules/common/lm_utils.py | 179 + .../nlp/modules/common/megatron/__init__.py | 19 + .../modules/common/megatron/megatron_bert.py | 216 + .../modules/common/megatron/megatron_utils.py | 307 + .../nlp/modules/common/sequence_classifier.py | 72 + .../nlp/modules/common/sequence_regression.py | 69 + .../common/sequence_token_classifier.py | 80 + .../nlp/modules/common/token_classifier.py | 165 + .../nlp/modules/common/tokenizer_utils.py | 144 + .../modules/common/transformer/__init__.py | 18 + .../modules/common/transformer/transformer.py | 248 + .../transformer/transformer_decoders.py | 200 + .../transformer/transformer_encoders.py | 176 + .../transformer/transformer_generators.py | 541 + .../common/transformer/transformer_modules.py | 210 + .../common/transformer/transformer_utils.py | 118 + .../dialogue_state_tracking/__init__.py | 13 + .../dialogue_state_tracking/sgd_decoder.py | 195 + .../dialogue_state_tracking/sgd_encoder.py | 77 + .../nemo/collections/nlp/parts/__init__.py | 16 + .../collections/nlp/parts/nlp_overrides.py | 233 + .../nemo/collections/nlp/parts/utils_funcs.py | 130 + .../NeMo/nemo/collections/tts/__init__.py | 17 + .../nemo/collections/tts/data/__init__.py | 15 + .../nemo/collections/tts/data/datalayers.py | 768 + .../nemo/collections/tts/helpers/__init__.py | 15 + .../nemo/collections/tts/helpers/helpers.py | 362 + .../nemo/collections/tts/losses/__init__.py | 16 + .../collections/tts/losses/aligner_loss.py | 89 + .../collections/tts/losses/fastpitchloss.py | 150 + .../collections/tts/losses/fastspeech2loss.py | 127 + .../collections/tts/losses/glow_tts_loss.py | 81 + .../collections/tts/losses/hifigan_losses.py | 132 + .../nemo/collections/tts/losses/stftlosses.py | 248 + .../collections/tts/losses/tacotron2loss.py | 84 + .../collections/tts/losses/uniglowloss.py | 58 + .../collections/tts/losses/waveglowloss.py | 51 + .../nemo/collections/tts/models/__init__.py | 53 + .../nemo/collections/tts/models/aligner.py | 157 + .../NeMo/nemo/collections/tts/models/base.py | 305 + .../NeMo/nemo/collections/tts/models/degli.py | 330 + .../collections/tts/models/ed_mel2spec.py | 321 + .../nemo/collections/tts/models/fastpitch.py | 414 + .../tts/models/fastpitch_hifigan_e2e.py | 432 + .../collections/tts/models/fastspeech2.py | 297 + .../tts/models/fastspeech2_hifigan_e2e.py | 430 + .../nemo/collections/tts/models/glow_tts.py | 279 + .../nemo/collections/tts/models/hifigan.py | 357 + .../nemo/collections/tts/models/melgan.py | 338 + .../collections/tts/models/squeezewave.py | 227 + .../nemo/collections/tts/models/tacotron2.py | 321 + .../nemo/collections/tts/models/talknet.py | 413 + .../nemo/collections/tts/models/two_stages.py | 205 + .../nemo/collections/tts/models/uniglow.py | 264 + .../nemo/collections/tts/models/waveglow.py | 260 + .../nemo/collections/tts/modules/__init__.py | 27 + .../nemo/collections/tts/modules/aligner.py | 106 + .../nemo/collections/tts/modules/degli.py | 617 + .../collections/tts/modules/ed_mel2spec.py | 261 + .../nemo/collections/tts/modules/fastpitch.py | 309 + .../collections/tts/modules/fastspeech2.py | 314 + .../tts/modules/fastspeech2_submodules.py | 160 + .../nemo/collections/tts/modules/glow_tts.py | 441 + .../tts/modules/glow_tts_parser.py | 364 + .../tts/modules/glow_tts_submodules.py | 645 + .../tts/modules/hifigan_modules.py | 447 + .../collections/tts/modules/melgan_modules.py | 508 + .../collections/tts/modules/squeezewave.py | 212 + .../tts/modules/squeezewave_submodules.py | 151 + .../collections/tts/modules/submodules.py | 297 + .../nemo/collections/tts/modules/tacotron2.py | 442 + .../nemo/collections/tts/modules/talknet.py | 136 + .../collections/tts/modules/transformer.py | 262 + .../nemo/collections/tts/modules/uniglow.py | 177 + .../nemo/collections/tts/modules/waveglow.py | 263 + models/tacotron2/NeMo/nemo/constants.py | 18 + models/tacotron2/NeMo/nemo/core/__init__.py | 16 + .../NeMo/nemo/core/classes/__init__.py | 29 + .../NeMo/nemo/core/classes/common.py | 812 + .../NeMo/nemo/core/classes/dataset.py | 109 + .../NeMo/nemo/core/classes/exportable.py | 308 + .../tacotron2/NeMo/nemo/core/classes/loss.py | 26 + .../NeMo/nemo/core/classes/modelPT.py | 1419 ++ .../NeMo/nemo/core/classes/module.py | 77 + .../NeMo/nemo/core/config/__init__.py | 47 + .../NeMo/nemo/core/config/base_config.py | 30 + .../NeMo/nemo/core/config/hydra_runner.py | 110 + .../NeMo/nemo/core/config/modelPT.py | 181 + .../NeMo/nemo/core/config/optimizers.py | 265 + .../NeMo/nemo/core/config/pytorch.py | 45 + .../nemo/core/config/pytorch_lightning.py | 97 + .../NeMo/nemo/core/config/schedulers.py | 251 + .../NeMo/nemo/core/neural_types/__init__.py | 19 + .../NeMo/nemo/core/neural_types/axes.py | 101 + .../NeMo/nemo/core/neural_types/comparison.py | 32 + .../NeMo/nemo/core/neural_types/elements.py | 324 + .../nemo/core/neural_types/neural_type.py | 243 + .../NeMo/nemo/core/optim/__init__.py | 29 + .../NeMo/nemo/core/optim/lr_scheduler.py | 686 + .../NeMo/nemo/core/optim/novograd.py | 145 + .../NeMo/nemo/core/optim/optimizers.py | 171 + .../NeMo/nemo/core/utils/__init__.py | 15 + .../NeMo/nemo/core/utils/numba_utils.py | 142 + models/tacotron2/NeMo/nemo/package_info.py | 35 + models/tacotron2/NeMo/nemo/utils/__init__.py | 22 + models/tacotron2/NeMo/nemo/utils/app_state.py | 386 + models/tacotron2/NeMo/nemo/utils/arguments.py | 132 + models/tacotron2/NeMo/nemo/utils/cloud.py | 82 + .../tacotron2/NeMo/nemo/utils/config_utils.py | 258 + .../NeMo/nemo/utils/decorators/__init__.py | 18 + .../NeMo/nemo/utils/decorators/deprecated.py | 72 + .../nemo/utils/decorators/experimental.py | 33 + .../NeMo/nemo/utils/decorators/port_docs.py | 90 + .../NeMo/nemo/utils/env_var_parsing.py | 207 + .../tacotron2/NeMo/nemo/utils/exceptions.py | 19 + .../tacotron2/NeMo/nemo/utils/exp_manager.py | 715 + .../tacotron2/NeMo/nemo/utils/export_utils.py | 225 + .../NeMo/nemo/utils/formatters/__init__.py | 13 + .../NeMo/nemo/utils/formatters/base.py | 135 + .../NeMo/nemo/utils/formatters/colors.py | 121 + .../NeMo/nemo/utils/formatters/utils.py | 46 + models/tacotron2/NeMo/nemo/utils/get_rank.py | 48 + .../NeMo/nemo/utils/lightning_logger_patch.py | 58 + .../tacotron2/NeMo/nemo/utils/metaclasses.py | 39 + .../tacotron2/NeMo/nemo/utils/model_utils.py | 528 + .../tacotron2/NeMo/nemo/utils/nemo_logging.py | 421 + .../NeMo/nemo_text_processing/README.md | 8 + .../NeMo/nemo_text_processing/__init__.py | 13 + .../inverse_text_normalization/README.md | 12 + .../inverse_text_normalization/__init__.py | 32 + .../clean_eval_data.py | 342 + .../data/__init__.py | 13 + .../data/currency.tsv | 35 + .../data/electronic/__init__.py | 13 + .../data/electronic/domain.tsv | 1 + .../data/electronic/server_name.tsv | 2 + .../data/magnitudes.tsv | 4 + .../data/measurements.tsv | 110 + .../data/months.tsv | 12 + .../data/numbers/__init__.py | 13 + .../data/numbers/digit.tsv | 9 + .../data/numbers/hundred.tsv | 1 + .../data/numbers/teen.tsv | 10 + .../data/numbers/thousands.tsv | 22 + .../data/numbers/ties.tsv | 9 + .../data/numbers/zero.tsv | 1 + .../data/ordinals/__init__.py | 13 + .../data/ordinals/digit.tsv | 9 + .../data/ordinals/teen.tsv | 1 + .../data/time/__init__.py | 13 + .../data/time/time_suffix.tsv | 8 + .../data/time/time_to.tsv | 12 + .../data/time/time_zone.tsv | 7 + .../data/whitelist.tsv | 10 + .../inverse_normalize.py | 71 + .../run_evaluate.py | 109 + .../inverse_text_normalization/run_predict.py | 75 + .../taggers/__init__.py | 13 + .../taggers/cardinal.py | 134 + .../taggers/date.py | 163 + .../taggers/decimal.py | 95 + .../taggers/electronic.py | 66 + .../taggers/fraction.py | 26 + .../taggers/measure.py | 103 + .../taggers/money.py | 112 + .../taggers/ordinal.py | 50 + .../taggers/punctuation.py | 41 + .../taggers/telephone.py | 67 + .../taggers/time.py | 124 + .../taggers/tokenize_and_classify.py | 91 + .../taggers/whitelist.py | 40 + .../taggers/word.py | 36 + .../inverse_text_normalization/utils.py | 47 + .../verbalizers/__init__.py | 13 + .../verbalizers/cardinal.py | 55 + .../verbalizers/date.py | 87 + .../verbalizers/decimal.py | 65 + .../verbalizers/electronic.py | 52 + .../verbalizers/fraction.py | 25 + .../verbalizers/measure.py | 69 + .../verbalizers/money.py | 48 + .../verbalizers/ordinal.py | 64 + .../verbalizers/telephone.py | 39 + .../verbalizers/time.py | 87 + .../verbalizers/verbalize.py | 62 + .../verbalizers/verbalize_final.py | 50 + .../verbalizers/whitelist.py | 44 + .../verbalizers/word.py | 39 + .../NeMo/nemo_text_processing/setup.sh | 2 + .../text_normalization/README.md | 11 + .../text_normalization/__init__.py | 33 + .../text_normalization/clean_eval_data.py | 342 + .../text_normalization/data/__init__.py | 13 + .../data/currency/__init__.py | 13 + .../data/currency/currency.tsv | 35 + .../data/currency/currency_minor.tsv | 2 + .../data/currency/currency_minor_one.tsv | 2 + .../data/electronic/__init__.py | 13 + .../data/electronic/domain.tsv | 1 + .../data/electronic/server_name.tsv | 2 + .../data/electronic/symbols.tsv | 20 + .../data/letter_pronunciation.tsv | 2 + .../text_normalization/data/magnitudes.tsv | 4 + .../text_normalization/data/measurements.tsv | 115 + .../data/months/__init__.py | 13 + .../text_normalization/data/months/abbr.tsv | 12 + .../text_normalization/data/months/names.tsv | 12 + .../data/months/numbers.tsv | 24 + .../data/numbers/__init__.py | 13 + .../data/numbers/cardinal_number_name.far | Bin 0 -> 325078 bytes .../text_normalization/data/numbers/digit.tsv | 9 + .../data/numbers/hundred.tsv | 1 + .../text_normalization/data/numbers/teen.tsv | 10 + .../data/numbers/thousands.tsv | 22 + .../text_normalization/data/numbers/ties.tsv | 8 + .../text_normalization/data/numbers/zero.tsv | 1 + .../data/ordinals/__init__.py | 13 + .../data/ordinals/digit.tsv | 9 + .../text_normalization/data/ordinals/teen.tsv | 1 + .../text_normalization/data/roman/__init__.py | 13 + .../data/roman/digit_teen.tsv | 49 + .../data/roman/hundreds.tsv | 9 + .../text_normalization/data/roman/ties.tsv | 5 + .../text_normalization/data/suppletive.tsv | 37 + .../text_normalization/data/time_suffix.tsv | 6 + .../text_normalization/data/time_zone.tsv | 14 + .../text_normalization/data/whitelist.tsv | 15924 ++++++++++++++++ .../data/whitelist_alternatives.tsv | 17 + .../text_normalization/data_loader_utils.py | 269 + .../text_normalization/graph_utils.py | 216 + .../text_normalization/normalize.py | 246 + .../normalize_with_audio.py | 327 + .../text_normalization/run_evaluate.py | 115 + .../text_normalization/run_predict.py | 78 + .../text_normalization/taggers/__init__.py | 13 + .../text_normalization/taggers/cardinal.py | 115 + .../text_normalization/taggers/date.py | 201 + .../text_normalization/taggers/decimal.py | 101 + .../text_normalization/taggers/electronic.py | 63 + .../text_normalization/taggers/fraction.py | 50 + .../text_normalization/taggers/measure.py | 162 + .../text_normalization/taggers/money.py | 86 + .../text_normalization/taggers/ordinal.py | 49 + .../text_normalization/taggers/punctuation.py | 46 + .../text_normalization/taggers/roman.py | 61 + .../text_normalization/taggers/telephone.py | 91 + .../text_normalization/taggers/time.py | 115 + .../taggers/tokenize_and_classify.py | 107 + .../text_normalization/taggers/whitelist.py | 68 + .../text_normalization/taggers/word.py | 41 + .../text_normalization/token_parser.py | 192 + .../verbalizers/__init__.py | 13 + .../verbalizers/cardinal.py | 56 + .../text_normalization/verbalizers/date.py | 100 + .../text_normalization/verbalizers/decimal.py | 74 + .../verbalizers/electronic.py | 91 + .../verbalizers/fraction.py | 72 + .../text_normalization/verbalizers/measure.py | 76 + .../text_normalization/verbalizers/money.py | 117 + .../text_normalization/verbalizers/ordinal.py | 63 + .../text_normalization/verbalizers/roman.py | 46 + .../verbalizers/telephone.py | 69 + .../text_normalization/verbalizers/time.py | 74 + .../verbalizers/verbalize.py | 79 + .../verbalizers/verbalize_final.py | 54 + .../verbalizers/whitelist.py | 48 + .../text_normalization/verbalizers/word.py | 43 + models/tacotron2/NeMo/reinstall.sh | 40 + .../NeMo/requirements/requirements.txt | 19 + .../NeMo/requirements/requirements_asr.txt | 24 + .../NeMo/requirements/requirements_cv.txt | 2 + .../NeMo/requirements/requirements_docs.txt | 6 + .../NeMo/requirements/requirements_nlp.txt | 13 + .../NeMo/requirements/requirements_test.txt | 11 + .../requirements_text_processing.txt | 2 + .../NeMo/requirements/requirements_tts.txt | 7 + .../create_tarred_transformer_lm_dataset.py | 318 + .../neural_rescorer/eval_neural_rescorer.py | 288 + .../ngram_lm/eval_beamsearch_ngram.py | 345 + .../ngram_lm/install_beamsearch_decoders.sh | 6 + .../ngram_lm/kenlm_utils.py | 112 + .../ngram_lm/train_kenlm.py | 154 + .../NeMo/scripts/average_model_checkpoints.py | 171 + .../get_commonvoice_data.py | 178 + .../dataset_processing/get_hi-mia_data.py | 185 + .../get_librispeech_data.py | 171 + .../get_openslr_rir_data.py | 140 + .../dataset_processing/ljspeech/README | 14 + .../ljspeech/calculate_durs.py | 126 + .../create_manifests_and_textfiles.py | 71 + .../ljspeech/create_token2idx_dict.py | 68 + .../ljspeech/extract_ljspeech_energy_pitch.py | 88 + .../extract_ljspeech_phonemes_and_durs.sh | 150 + .../dataset_processing/process_an4_data.py | 88 + .../dataset_processing/process_degli_data.py | 70 + .../dataset_processing/process_hub5_data.py | 256 + .../process_speech_commands_data.py | 402 + .../dataset_processing/process_vad_data.py | 486 + .../NeMo/scripts/export/convasr_to_enemo.py | 76 + .../NeMo/scripts/export/convasr_to_onnx.py | 66 + .../scripts/export/convasr_to_single_onnx.py | 62 + .../download_resample_freesound.sh | 115 + .../freesound_download.py | 585 + .../freesound_requirements.txt | 6 + .../freesound_resample.py | 119 + .../nemo_legacy_import/asr_checkpoint_port.py | 72 + .../filter_langs_nmt.py | 403 + .../speaker_recognition/rttm_to_manifest.py | 40 + .../speaker_recognition/scp_to_manifest.py | 96 + .../convert_to_tarred_audio_dataset.py | 601 + .../tokenizers/process_asr_text_tokenizer.py | 282 + .../vad_overlap_posterior.py | 259 + .../vad_tune_threshold.py | 58 + .../write_long_audio_manifest.py | 173 + models/tacotron2/NeMo/setup.cfg | 36 + models/tacotron2/NeMo/setup.py | 238 + models/tacotron2/NeMo/tests/__init__.py | 13 + .../asr/numba/rnnt_loss/test_rnnt_pytorch.py | 256 + .../rnnt_loss/utils/test_gpu_rnnt_kernel.py | 169 + .../asr/numba/rnnt_loss/utils/test_reduce.py | 86 + .../numba/rnnt_loss/utils/test_rnnt_helper.py | 319 + .../numba/spec_augment/test_spec_aug_numba.py | 252 + .../asr/test_asr_classification_model.py | 208 + .../asr/test_asr_ctc_encoder_model_bpe.py | 313 + .../asr/test_asr_ctcencdec_model.py | 305 + .../collections/asr/test_asr_datasets.py | 127 + .../collections/asr/test_asr_exportables.py | 301 + .../test_asr_filterbankfeatures_seq_len.py | 99 + .../tests/collections/asr/test_asr_metrics.py | 172 + .../tests/collections/asr/test_asr_modules.py | 325 + .../asr/test_asr_regression_model.py | 95 + .../asr/test_asr_rnnt_encdec_model.py | 276 + .../asr/test_asr_rnnt_encoder_model_bpe.py | 246 + .../collections/asr/test_jasper_block.py | 295 + .../collections/asr/test_label_datasets.py | 106 + .../asr/test_speaker_label_models.py | 63 + .../NeMo/tests/collections/common/__init__.py | 13 + .../tests/collections/common/loss_inputs.py | 69 + .../collections/common/perplexity_inputs.py | 49 + .../NeMo/tests/collections/common/pl_utils.py | 557 + .../tests/collections/common/test_metrics.py | 150 + .../collections/common/test_spc_tokenizer.py | 267 + .../nlp/test_classification_report.py | 48 + .../nlp/test_entity_linking_model.py | 83 + .../tests/collections/nlp/test_huggingface.py | 86 + .../tests/collections/nlp/test_megatron.py | 76 + .../collections/nlp/test_nlp_exportables.py | 176 + .../tests/collections/nlp/test_nmt_model.py | 144 + .../nlp/test_pretrained_models_performance.py | 110 + .../collections/tts/test_tts_datasets.py | 43 + .../collections/tts/test_tts_exportables.py | 62 + .../tests/collections/tts/test_waveglow.py | 110 + models/tacotron2/NeMo/tests/conftest.py | 188 + .../NeMo/tests/core/test_config_utils.py | 125 + .../NeMo/tests/core/test_exp_manager.py | 357 + .../tacotron2/NeMo/tests/core/test_fileio.py | 299 + .../NeMo/tests/core/test_neural_types.py | 224 + .../tests/core/test_optimizers_schedulers.py | 809 + .../NeMo/tests/core/test_save_restore.py | 443 + .../NeMo/tests/core/test_serialization.py | 165 + .../NeMo/tests/core/test_typecheck.py | 1075 ++ .../NeMo/tests/core_ptl/check_for_ranks.py | 155 + .../tacotron2/NeMo/tests/hydra/config1.yaml | 1 + .../NeMo/tests/hydra/config1_invalid.yaml | 2 + .../tests/hydra/config_subdir/config2.yaml | 1 + .../hydra/config_subdir/config2_invalid.yaml | 2 + models/tacotron2/NeMo/tests/hydra/my_app.py | 44 + .../NeMo/tests/hydra/test_hydra_runner.py | 95 + .../NeMo/tests/manualtest_model_downloads.py | 92 + .../test_cases_cardinal.txt | 24 + .../test_cases_date.txt | 34 + .../test_cases_decimal.txt | 63 + .../test_cases_electronic.txt | 7 + .../test_cases_measure.txt | 112 + .../test_cases_money.txt | 48 + .../test_cases_ordinal.txt | 33 + .../test_cases_telephone.txt | 4 + .../test_cases_time.txt | 27 + .../test_cases_whitelist.txt | 8 + .../test_cases_word.txt | 50 + .../test_cases_boundary.txt | 7 + .../test_cases_cardinal.txt | 18 + .../test_cases_date.txt | 32 + .../test_cases_decimal.txt | 8 + .../test_cases_electronic.txt | 10 + .../test_cases_fraction.txt | 11 + .../test_cases_measure.txt | 15 + .../test_cases_money.txt | 38 + .../test_cases_normalize_with_audio.txt | 107 + .../test_cases_ordinal.txt | 6 + .../test_cases_telephone.txt | 9 + .../test_cases_time.txt | 14 + .../test_cases_whitelist.txt | 3 + .../test_cases_word.txt | 23 + .../nemo_text_processing/test_boundary.py | 39 + .../nemo_text_processing/test_cardinal.py | 49 + .../tests/nemo_text_processing/test_date.py | 64 + .../nemo_text_processing/test_decimal.py | 49 + .../nemo_text_processing/test_electronic.py | 51 + .../nemo_text_processing/test_fraction.py | 38 + .../nemo_text_processing/test_measure.py | 50 + .../tests/nemo_text_processing/test_money.py | 50 + .../test_normalization_with_audio.py | 33 + .../nemo_text_processing/test_ordinal.py | 50 + ..._sparrowhawk_inverse_text_normalization.sh | 64 + .../test_sparrowhawk_normalization.sh | 64 + .../nemo_text_processing/test_telephone.py | 50 + .../tests/nemo_text_processing/test_time.py | 49 + .../nemo_text_processing/test_whitelist.py | 65 + .../tests/nemo_text_processing/test_word.py | 52 + .../NeMo/tests/nemo_text_processing/utils.py | 54 + models/tacotron2/NeMo/tests/test_data_dir.py | 27 + .../NeMo/tools/ctc_segmentation/README.md | 31 + .../tools/ctc_segmentation/requirements.txt | 2 + .../NeMo/tools/ctc_segmentation/run_sample.sh | 124 + .../ctc_segmentation/scripts/cut_audio.py | 313 + .../scripts/normalization_helpers.py | 69 + .../ctc_segmentation/scripts/prepare_data.py | 412 + .../scripts/process_manifests.py | 111 + .../scripts/run_ctc_segmentation.py | 180 + .../tools/ctc_segmentation/scripts/utils.py | 173 + .../scripts/verify_segments.py | 143 + .../NeMo/tools/speech_data_explorer/README.md | 26 + .../speech_data_explorer/data_explorer.py | 717 + .../speech_data_explorer/requirements.txt | 10 + .../tools/speech_data_explorer/screenshot.png | Bin 0 -> 1135618 bytes .../text_processing_deployment/Dockerfile | 38 + .../text_processing_deployment/README.rst | 10 + .../docker/build.sh | 18 + .../docker/launch.sh | 44 + .../export_grammars.sh | 59 + .../pynini_export.py | 111 + .../NeMo/tutorials/00_NeMo_Primer.ipynb | 1281 ++ .../NeMo/tutorials/01_NeMo_Models.ipynb | 2636 +++ .../tutorials/AudioTranslationSample.ipynb | 280 + .../NeMo/tutorials/VoiceSwapSample.ipynb | 325 + .../NeMo/tutorials/asr/01_ASR_with_NeMo.ipynb | 1174 ++ .../asr/02_Online_ASR_Microphone_Demo.ipynb | 540 + .../tutorials/asr/03_Speech_Commands.ipynb | 1628 ++ ..._Online_Offline_Speech_Commands_Demo.ipynb | 747 + .../asr/05_Online_Noise_Augmentation.ipynb | 1320 ++ .../asr/06_Voice_Activiy_Detection.ipynb | 1169 ++ ...7_Online_Offline_Microphone_VAD_Demo.ipynb | 830 + .../08_ASR_with_Subword_Tokenization.ipynb | 1445 ++ .../asr/10_ASR_CTC_Language_Finetuning.ipynb | 2163 +++ .../NeMo/tutorials/asr/Offline_ASR.ipynb | 506 + .../asr/images/alignment_example.png | Bin 0 -> 23938 bytes .../asr/images/vad_post_overlap_diagram.png | Bin 0 -> 20961 bytes ...Language_Models_for_Downstream_Tasks.ipynb | 410 + .../tutorials/nlp/02_NLP_Tokenizers.ipynb | 616 + .../nlp/Entity_Linking_Medical.ipynb | 756 + .../NeMo/tutorials/nlp/GLUE_Benchmark.ipynb | 565 + ...Joint_Intent_and_Slot_Classification.ipynb | 587 + ...ural_Machine_Translation-Text2Sparql.ipynb | 2260 +++ .../nlp/Punctuation_and_Capitalization.ipynb | 735 + .../nlp/Question_Answering_Squad.ipynb | 724 + .../nlp/Relation_Extraction-BioMegatron.ipynb | 630 + ...xt_Classification_Sentiment_Analysis.ipynb | 828 + .../Token_Classification-BioMegatron.ipynb | 648 + ...ssification_Named_Entity_Recognition.ipynb | 849 + .../ASR_with_SpeakerDiarization.ipynb | 555 + .../Speaker_Diarization_Inference.ipynb | 632 + .../Speaker_Recognition_Verification.ipynb | 1211 ++ .../Inverse_Text_Normalization.ipynb | 514 + .../text_processing/Text_Normalization.ipynb | 394 + .../text_processing/images/deployment.png | Bin 0 -> 154145 bytes .../text_processing/images/pipeline.png | Bin 0 -> 225984 bytes .../tools/CTC_Segmentation_Tutorial.ipynb | 811 + ...edCrowd_x_NeMo_ASR_Training_Tutorial.ipynb | 1853 ++ .../setup-asr-preannotations.ipynb | 249 + .../NeMo/tutorials/tts/1_TTS_inference.ipynb | 389 + .../tts/2_TTS_Tacotron2_Training.ipynb | 347 + .../tts/3_TTS_TalkNet_Training.ipynb | 499 + 971 files changed, 180643 insertions(+) create mode 100644 models/tacotron2/NeMo/.dockerignore create mode 100644 models/tacotron2/NeMo/.github/ISSUE_TEMPLATE/bug_report.md create mode 100644 models/tacotron2/NeMo/.github/ISSUE_TEMPLATE/feature_request.md create mode 100644 models/tacotron2/NeMo/.gitignore create mode 100644 models/tacotron2/NeMo/.readthedocs.yml create mode 100644 models/tacotron2/NeMo/CONTRIBUTING.md create mode 100644 models/tacotron2/NeMo/Dockerfile create mode 100644 models/tacotron2/NeMo/Jenkinsfile create mode 100644 models/tacotron2/NeMo/LICENSE create mode 100644 models/tacotron2/NeMo/README.rst create mode 100644 models/tacotron2/NeMo/docs/.nojekyll create mode 100644 models/tacotron2/NeMo/docs/Makefile create mode 100644 models/tacotron2/NeMo/docs/source/_static/css/custom.css create mode 100644 models/tacotron2/NeMo/docs/source/_static/js/pk_scripts.js create mode 100644 models/tacotron2/NeMo/docs/source/asr/api.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/asr_all.bib create mode 100644 models/tacotron2/NeMo/docs/source/asr/asr_language_modeling.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/configs.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/data/asr_results.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/data/asrlm_results.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/data/benchmark_ca.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/data/benchmark_de.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/data/benchmark_en.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/data/benchmark_es.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/data/benchmark_fr.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/data/benchmark_it.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/data/benchmark_pl.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/data/benchmark_ru.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/data/benchmark_zh.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/datasets.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/images/citrinet_vertical.png create mode 100644 models/tacotron2/NeMo/docs/source/asr/images/conformer_ctc.png create mode 100644 models/tacotron2/NeMo/docs/source/asr/images/ctc_asr.png create mode 100644 models/tacotron2/NeMo/docs/source/asr/images/jasper_layers.png create mode 100644 models/tacotron2/NeMo/docs/source/asr/images/jasper_vertical.png create mode 100644 models/tacotron2/NeMo/docs/source/asr/images/quartz_vertical.png create mode 100644 models/tacotron2/NeMo/docs/source/asr/intro.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/models.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/results.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_diarization/api.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_diarization/configs.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_diarization/data/diarization_results.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_diarization/datasets.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_diarization/images/sd_pipeline.png create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_diarization/intro.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_diarization/models.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_diarization/results.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_recognition/configs.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_recognition/data/speaker_results.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_recognition/datasets.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_recognition/images/ICASPP_SpeakerNet.png create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_recognition/intro.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_recognition/models.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speaker_recognition/results.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speech_classification/configs.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speech_classification/data/classification_results.csv create mode 100644 models/tacotron2/NeMo/docs/source/asr/speech_classification/datasets.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speech_classification/images/marblenet_vertical.png create mode 100644 models/tacotron2/NeMo/docs/source/asr/speech_classification/images/matchboxnet_vertical.png create mode 100644 models/tacotron2/NeMo/docs/source/asr/speech_classification/intro.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speech_classification/models.rst create mode 100644 models/tacotron2/NeMo/docs/source/asr/speech_classification/results.rst create mode 100644 models/tacotron2/NeMo/docs/source/common/intro.rst create mode 100644 models/tacotron2/NeMo/docs/source/conf.py create mode 100644 models/tacotron2/NeMo/docs/source/core/core.rst create mode 100644 models/tacotron2/NeMo/docs/source/core/whyntypes.gif create mode 100644 models/tacotron2/NeMo/docs/source/favicon.ico create mode 100644 models/tacotron2/NeMo/docs/source/index.rst create mode 100755 models/tacotron2/NeMo/docs/source/nemo_text_processing/api.rst create mode 100644 models/tacotron2/NeMo/docs/source/nemo_text_processing/intro.rst create mode 100644 models/tacotron2/NeMo/docs/source/nemo_text_processing/inverse_text_normalization.rst create mode 100644 models/tacotron2/NeMo/docs/source/nemo_text_processing/text_normalization.rst create mode 100644 models/tacotron2/NeMo/docs/source/nemo_text_processing/textprocessing_all.bib create mode 100755 models/tacotron2/NeMo/docs/source/nlp/api.rst create mode 100755 models/tacotron2/NeMo/docs/source/nlp/bert_pretraining.rst create mode 100644 models/tacotron2/NeMo/docs/source/nlp/glue_benchmark.rst create mode 100644 models/tacotron2/NeMo/docs/source/nlp/information_retrieval.rst create mode 100755 models/tacotron2/NeMo/docs/source/nlp/intro.rst create mode 100644 models/tacotron2/NeMo/docs/source/nlp/joint_intent_slot.rst create mode 100644 models/tacotron2/NeMo/docs/source/nlp/language_modeling.rst create mode 100644 models/tacotron2/NeMo/docs/source/nlp/machine_translation.rst create mode 100644 models/tacotron2/NeMo/docs/source/nlp/megatron_finetuning.rst create mode 100755 models/tacotron2/NeMo/docs/source/nlp/models.rst create mode 100644 models/tacotron2/NeMo/docs/source/nlp/nlp_all.bib create mode 100755 models/tacotron2/NeMo/docs/source/nlp/nlp_model.rst create mode 100755 models/tacotron2/NeMo/docs/source/nlp/punctuation_and_capitalization.rst create mode 100644 models/tacotron2/NeMo/docs/source/nlp/question_answering.rst create mode 100644 models/tacotron2/NeMo/docs/source/nlp/sgd_qa.rst create mode 100644 models/tacotron2/NeMo/docs/source/nlp/text_classification.rst create mode 100755 models/tacotron2/NeMo/docs/source/nlp/token_classification.rst create mode 100644 models/tacotron2/NeMo/docs/source/nv_logo.png create mode 100644 models/tacotron2/NeMo/docs/source/starthere/best-practices.rst create mode 100644 models/tacotron2/NeMo/docs/source/starthere/intro.rst create mode 100644 models/tacotron2/NeMo/docs/source/starthere/tutorials.rst create mode 100644 models/tacotron2/NeMo/docs/source/tools/ctc_segmentation.rst create mode 100644 models/tacotron2/NeMo/docs/source/tools/intro.rst create mode 100644 models/tacotron2/NeMo/docs/source/tools/speech_data_explorer.rst create mode 100644 models/tacotron2/NeMo/docs/source/tools/text_processing_deployment.rst create mode 100644 models/tacotron2/NeMo/docs/source/tools/tools_all.bib create mode 100644 models/tacotron2/NeMo/docs/source/tts/api.rst create mode 100644 models/tacotron2/NeMo/docs/source/tts/datasets.rst create mode 100644 models/tacotron2/NeMo/docs/source/tts/intro.rst create mode 100755 models/tacotron2/NeMo/docs/update_docs_docker.sh create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/Dockerfile create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/README.rst create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/asr_service.py create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/docker_container_build.sh create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/docker_container_run.sh create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/gunicorn.conf.py create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/model_api.py create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/requirements.txt create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/templates/main.html create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/templates/toast_msg.html create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/templates/transcripts.html create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/remove_files.html create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/upload_files_failed.html create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/upload_files_successful.html create mode 100644 models/tacotron2/NeMo/examples/asr/asr_webapp/wsgi.py create mode 100644 models/tacotron2/NeMo/examples/asr/conf/citrinet/citrinet_384.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/conf/citrinet/citrinet_512.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/conf/citrinet/config_bpe.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/conf/config.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/conf/conformer/conformer_ctc_bpe.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/conf/conformer/conformer_ctc_char.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/conf/jasper/jasper_10x5dr.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/conf/marblenet/marblenet_3x2x64.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v1.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v2.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/conf/quartznet/quartznet_15x5.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/conf/quartznet/quartznet_15x5_zh.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/conformer/conformer_rnnt_bpe.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/contextnet_rnnt/config_rnnt.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/contextnet_rnnt/config_rnnt_bpe.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/quartznet/quartznet_15x5_aug.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/sclite/speech_to_text_sclite.py create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/structured/conf/quartznet_15x5.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_hybrid.py create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_structured.py create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_structured_v2.py create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/wav2vec/configs/wav2vec_pretrain.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/wav2vec/configs/wav2vec_pretrain_large.yaml create mode 100644 models/tacotron2/NeMo/examples/asr/experimental/wav2vec/wav2vec_pretrain.py create mode 100644 models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_calibrate.py create mode 100644 models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_quant_infer.py create mode 100644 models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_quant_infer_trt.py create mode 100644 models/tacotron2/NeMo/examples/asr/speech_to_label.py create mode 100644 models/tacotron2/NeMo/examples/asr/speech_to_text.py create mode 100644 models/tacotron2/NeMo/examples/asr/speech_to_text_bpe.py create mode 100644 models/tacotron2/NeMo/examples/asr/speech_to_text_infer.py create mode 100644 models/tacotron2/NeMo/examples/asr/speech_to_text_rnnt.py create mode 100644 models/tacotron2/NeMo/examples/asr/speech_to_text_rnnt_bpe.py create mode 100644 models/tacotron2/NeMo/examples/asr/transcribe_speech.py create mode 100644 models/tacotron2/NeMo/examples/asr/vad_infer.py create mode 100644 models/tacotron2/NeMo/examples/cv/mnist_lenet5_image_classification_pure_lightning.py create mode 100644 models/tacotron2/NeMo/examples/nlp/dialogue_state_tracking/conf/sgdqa_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/dialogue_state_tracking/sgd_qa.py create mode 100644 models/tacotron2/NeMo/examples/nlp/entity_linking/build_index.py create mode 100644 models/tacotron2/NeMo/examples/nlp/entity_linking/conf/tiny_example_entity_linking_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/entity_linking/conf/umls_medical_entity_linking_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/entity_linking/data/umls_dataset_processing.py create mode 100644 models/tacotron2/NeMo/examples/nlp/entity_linking/query_index.py create mode 100644 models/tacotron2/NeMo/examples/nlp/entity_linking/self_alignment_pretraining.py create mode 100644 models/tacotron2/NeMo/examples/nlp/glue_benchmark/glue_benchmark.py create mode 100644 models/tacotron2/NeMo/examples/nlp/glue_benchmark/glue_benchmark_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/information_retrieval/bert_dpr.py create mode 100644 models/tacotron2/NeMo/examples/nlp/information_retrieval/bert_joint_ir.py create mode 100644 models/tacotron2/NeMo/examples/nlp/information_retrieval/conf/bert_ir_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/information_retrieval/construct_random_negatives.py create mode 100644 models/tacotron2/NeMo/examples/nlp/information_retrieval/get_msmarco.sh create mode 100644 models/tacotron2/NeMo/examples/nlp/intent_slot_classification/conf/intent_slot_classification_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/intent_slot_classification/data/assistant_utils.py create mode 100755 models/tacotron2/NeMo/examples/nlp/intent_slot_classification/data/import_datasets.py create mode 100644 models/tacotron2/NeMo/examples/nlp/intent_slot_classification/intent_slot_classification.py create mode 100644 models/tacotron2/NeMo/examples/nlp/language_modeling/bert_pretraining.py create mode 100644 models/tacotron2/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_preprocessed_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_text_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/language_modeling/conf/transformer_lm_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/language_modeling/convert_weights_to_nemo1.0.py create mode 100644 models/tacotron2/NeMo/examples/nlp/language_modeling/get_wkt2.sh create mode 100644 models/tacotron2/NeMo/examples/nlp/language_modeling/transformer_lm.py create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/conf/aayn_base.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/conf/huggingface.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/create_tarred_monolingual_dataset.py create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/create_tarred_parallel_dataset.py create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/enc_dec_nmt.py create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_transformer_infer.py create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/README.rst create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/config.json create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/index.html create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/nmt_service.py create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/requirements.txt create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/style.css create mode 100644 models/tacotron2/NeMo/examples/nlp/machine_translation/translate_ddp.py create mode 100644 models/tacotron2/NeMo/examples/nlp/question_answering/conf/question_answering_squad_config.yaml create mode 100755 models/tacotron2/NeMo/examples/nlp/question_answering/get_squad.py create mode 100644 models/tacotron2/NeMo/examples/nlp/question_answering/question_answering_squad.py create mode 100644 models/tacotron2/NeMo/examples/nlp/text2sparql/conf/text2sparql_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/text2sparql/data/import_datasets.py create mode 100644 models/tacotron2/NeMo/examples/nlp/text2sparql/evaluate_text2sparql.py create mode 100644 models/tacotron2/NeMo/examples/nlp/text2sparql/text2sparql.py create mode 100644 models/tacotron2/NeMo/examples/nlp/text_classification/conf/text_classification_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/text_classification/data/import_datasets.py create mode 100644 models/tacotron2/NeMo/examples/nlp/text_classification/model_parallel_text_classification_evaluation.py create mode 100644 models/tacotron2/NeMo/examples/nlp/text_classification/text_classification_with_bert.py create mode 100644 models/tacotron2/NeMo/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/token_classification/conf/token_classification_config.yaml create mode 100644 models/tacotron2/NeMo/examples/nlp/token_classification/data/get_tatoeba_data.py create mode 100644 models/tacotron2/NeMo/examples/nlp/token_classification/data/import_from_iob_format.py create mode 100644 models/tacotron2/NeMo/examples/nlp/token_classification/data/prepare_data_for_punctuation_capitalization.py create mode 100644 models/tacotron2/NeMo/examples/nlp/token_classification/punctuation_capitalization_evaluate.py create mode 100644 models/tacotron2/NeMo/examples/nlp/token_classification/punctuation_capitalization_train.py create mode 100644 models/tacotron2/NeMo/examples/nlp/token_classification/token_classification_evaluate.py create mode 100644 models/tacotron2/NeMo/examples/nlp/token_classification/token_classification_train.py create mode 100644 models/tacotron2/NeMo/examples/speaker_recognition/README.md create mode 100644 models/tacotron2/NeMo/examples/speaker_recognition/conf/SpeakerNet_recognition_3x2x512.yaml create mode 100644 models/tacotron2/NeMo/examples/speaker_recognition/conf/SpeakerNet_verification_3x2x512.yaml create mode 100644 models/tacotron2/NeMo/examples/speaker_recognition/conf/speaker_diarization.yaml create mode 100644 models/tacotron2/NeMo/examples/speaker_recognition/extract_speaker_embeddings.py create mode 100644 models/tacotron2/NeMo/examples/speaker_recognition/speaker_diarize.py create mode 100644 models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco.py create mode 100644 models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco_finetune.py create mode 100644 models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco_infer.py create mode 100644 models/tacotron2/NeMo/examples/speaker_recognition/voxceleb_eval.py create mode 100644 models/tacotron2/NeMo/examples/tts/aligner.py create mode 100644 models/tacotron2/NeMo/examples/tts/conf/aligner.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/degli.yaml create mode 100755 models/tacotron2/NeMo/examples/tts/conf/ed_mel2spec.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/fastpitch.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/fastpitch_align.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/fastpitch_hifigan_e2e.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/fastspeech2.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/fastspeech2_hifigan_e2e.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/glow_tts.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/hifigan/hifigan.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v1.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v2.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v3.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/hifigan/model/train_ds/train_ds.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/hifigan/model/train_ds/train_ds_finetune.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/hifigan/model/validation_ds/val_ds.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/hifigan/model/validation_ds/val_ds_finetune.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/melgan.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/squeezewave.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/tacotron2.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/talknet-durs.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/talknet-pitch.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/talknet-spect.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/uniglow.yaml create mode 100644 models/tacotron2/NeMo/examples/tts/conf/waveglow.yaml create mode 100755 models/tacotron2/NeMo/examples/tts/degli.py create mode 100755 models/tacotron2/NeMo/examples/tts/ed_mel2spec.py create mode 100644 models/tacotron2/NeMo/examples/tts/fastpitch.py create mode 100644 models/tacotron2/NeMo/examples/tts/fastpitch_hifigan_e2e.py create mode 100644 models/tacotron2/NeMo/examples/tts/fastspeech2.py create mode 100644 models/tacotron2/NeMo/examples/tts/fastspeech2_hifigan_e2e.py create mode 100644 models/tacotron2/NeMo/examples/tts/glow_tts.py create mode 100644 models/tacotron2/NeMo/examples/tts/hifigan.py create mode 100644 models/tacotron2/NeMo/examples/tts/melgan.py create mode 100755 models/tacotron2/NeMo/examples/tts/squeezewave.py create mode 100755 models/tacotron2/NeMo/examples/tts/tacotron2.py create mode 100644 models/tacotron2/NeMo/examples/tts/talknet_durs.py create mode 100644 models/tacotron2/NeMo/examples/tts/talknet_pitch.py create mode 100644 models/tacotron2/NeMo/examples/tts/talknet_spect.py create mode 100644 models/tacotron2/NeMo/examples/tts/test_tts_infer.py create mode 100755 models/tacotron2/NeMo/examples/tts/uniglow.py create mode 100755 models/tacotron2/NeMo/examples/tts/waveglow.py create mode 100644 models/tacotron2/NeMo/external/get_collections.py create mode 100644 models/tacotron2/NeMo/external/get_modules.py create mode 100644 models/tacotron2/NeMo/nemo/README.md create mode 100644 models/tacotron2/NeMo/nemo/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/data/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_label.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_label_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text_dali.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/data/feature_to_label.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/data/feature_to_label_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/data/vocabs.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/losses/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/losses/angularloss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/losses/ctc.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/losses/rnnt.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/losses/wav2vecloss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/metrics/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/metrics/rnnt_wer.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/metrics/rnnt_wer_bpe.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/metrics/wer.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/metrics/wer_bpe.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/asr_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/classification_models.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/clustering_diarizer.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/configs/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/configs/classification_models_config.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/configs/ctc_models_config.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/configs/matchboxnet_config.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/configs/quartznet_config.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/ctc_bpe_models.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/ctc_models.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/label_models.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/rnnt_bpe_models.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/rnnt_models.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/wav2vec_config.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/wav2vec_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/modules/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/modules/audio_preprocessing.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/modules/beam_search_decoder.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/modules/conformer_encoder.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/modules/conv_asr.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/modules/lstm_decoder.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/modules/rnnt.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/modules/rnnt_abstract.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/modules/wav2vec_modules.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/features.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/mixins/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/mixins/mixins.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt_numpy.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt_pytorch.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cpu_utils/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cpu_utils/cpu_rnnt.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/gpu_rnnt.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/gpu_rnnt_kernel.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/reduce.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/global_constants.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/rnnt_helper.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/spec_augment/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/numba/spec_augment/spec_aug_numba.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/feature_loader.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/features.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/perturb.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/segment.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/conformer_modules.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/jasper.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/multi_head_attention.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/rnnt_beam_decoding.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/spectr_augment.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/subsampling.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/wav2vec.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/utils/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/utils/activations.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/utils/asr_module_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/utils/nmse_clustering.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/utils/numba_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/utils/rnnt_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/utils/speaker_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/asr/parts/utils/vad_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/callbacks/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/callbacks/callbacks.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/data/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/data/dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/losses/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/losses/aggregator.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/losses/cross_entropy.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/losses/mse_loss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/losses/multi_similarity_loss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/losses/smoothed_cross_entropy.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/losses/spanning_loss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/metrics/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/metrics/classification_accuracy.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/metrics/global_average_loss_metric.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/metrics/perplexity.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/parts/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/parts/multi_layer_perceptron.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/parts/patch_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/cleaners.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/collections.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/manifest.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/parsers.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/parts/rnn.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/parts/transformer_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/parts/utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/char_tokenizer.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/chinese_tokenizers.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/en_ja_tokenizers.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/fairseq_tokenizer.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/huggingface/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/huggingface/auto_tokenizer.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/moses_tokenizers.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/sentencepiece_tokenizer.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/tokenizer_spec.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/word_tokenizer.py create mode 100644 models/tacotron2/NeMo/nemo/collections/common/tokenizers/youtokentome_tokenizer.py create mode 100644 models/tacotron2/NeMo/nemo/collections/cv/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/cv/datasets/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/cv/datasets/mnist_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/cv/losses/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/cv/losses/nll_loss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/cv/models/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/cv/models/mnist_lenet5.py create mode 100644 models/tacotron2/NeMo/nemo/collections/cv/modules/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/cv/modules/lenet5.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/data_utils/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/data_utils/data_preprocessing.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/data_processor.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/evaluate.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/input_example.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/metrics.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/prediction_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/schema.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/entity_linking/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/entity_linking/entity_linking_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/data_processors.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/glue_benchmark_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/information_retrieval/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/information_retrieval/information_retrieval_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/intent_slot_classification_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/intent_slot_classification_descriptor.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/l2r_lm_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/lm_bert_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/sentence_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/machine_translation_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/preproc_mt_data.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/neural_machine_translation/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/neural_machine_translation/neural_machine_translation_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/qa_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/qa_squad_processing.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/text_classification/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/text_classification/text_classification_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/token_classification_dataset.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/token_classification_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/losses/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/losses/sgd_loss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/metrics/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/metrics/classification_report.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/metrics/sequence_perplexity.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/dialogue_state_tracking/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/dialogue_state_tracking/sgdqa_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/enc_dec_nlp_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/entity_linking/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/entity_linking/entity_linking_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/glue_benchmark_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/metrics_for_glue.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/base_ir_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/bert_dpr_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/bert_joint_ir_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/intent_slot_classification/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/intent_slot_classification/intent_slot_classification_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/bert_lm_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/transformer_lm_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/mt_enc_dec_config.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/mt_enc_dec_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/neural_machine_translation/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/neural_machine_translation/neural_machine_translation_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/nlp_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/question_answering/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/question_answering/qa_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/text_classification/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/text_classification/text_classification_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/punctuation_capitalization_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/token_classification_model.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/bert_module.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/classifier.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/decoder_module.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/encoder_module.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/albert.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/bert.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/distilbert.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_decoder.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_encoder.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/roberta.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/lm_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/megatron_bert.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/megatron_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_classifier.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_regression.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_token_classifier.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/token_classifier.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/tokenizer_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_decoders.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_encoders.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_generators.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_modules.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_utils.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/sgd_decoder.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/sgd_encoder.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/parts/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/parts/nlp_overrides.py create mode 100644 models/tacotron2/NeMo/nemo/collections/nlp/parts/utils_funcs.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/data/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/data/datalayers.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/helpers/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/helpers/helpers.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/losses/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/losses/aligner_loss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/losses/fastpitchloss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/losses/fastspeech2loss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/losses/glow_tts_loss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/losses/hifigan_losses.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/losses/stftlosses.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/losses/tacotron2loss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/losses/uniglowloss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/losses/waveglowloss.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/aligner.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/base.py create mode 100755 models/tacotron2/NeMo/nemo/collections/tts/models/degli.py create mode 100755 models/tacotron2/NeMo/nemo/collections/tts/models/ed_mel2spec.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/fastpitch.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/fastpitch_hifigan_e2e.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/fastspeech2.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/fastspeech2_hifigan_e2e.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/glow_tts.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/hifigan.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/melgan.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/squeezewave.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/tacotron2.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/talknet.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/two_stages.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/uniglow.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/models/waveglow.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/aligner.py create mode 100755 models/tacotron2/NeMo/nemo/collections/tts/modules/degli.py create mode 100755 models/tacotron2/NeMo/nemo/collections/tts/modules/ed_mel2spec.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/fastpitch.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/fastspeech2.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/fastspeech2_submodules.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts_parser.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts_submodules.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/hifigan_modules.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/melgan_modules.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/squeezewave.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/squeezewave_submodules.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/submodules.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/tacotron2.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/talknet.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/transformer.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/uniglow.py create mode 100644 models/tacotron2/NeMo/nemo/collections/tts/modules/waveglow.py create mode 100644 models/tacotron2/NeMo/nemo/constants.py create mode 100644 models/tacotron2/NeMo/nemo/core/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/core/classes/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/core/classes/common.py create mode 100644 models/tacotron2/NeMo/nemo/core/classes/dataset.py create mode 100644 models/tacotron2/NeMo/nemo/core/classes/exportable.py create mode 100644 models/tacotron2/NeMo/nemo/core/classes/loss.py create mode 100644 models/tacotron2/NeMo/nemo/core/classes/modelPT.py create mode 100644 models/tacotron2/NeMo/nemo/core/classes/module.py create mode 100644 models/tacotron2/NeMo/nemo/core/config/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/core/config/base_config.py create mode 100644 models/tacotron2/NeMo/nemo/core/config/hydra_runner.py create mode 100644 models/tacotron2/NeMo/nemo/core/config/modelPT.py create mode 100644 models/tacotron2/NeMo/nemo/core/config/optimizers.py create mode 100644 models/tacotron2/NeMo/nemo/core/config/pytorch.py create mode 100644 models/tacotron2/NeMo/nemo/core/config/pytorch_lightning.py create mode 100644 models/tacotron2/NeMo/nemo/core/config/schedulers.py create mode 100644 models/tacotron2/NeMo/nemo/core/neural_types/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/core/neural_types/axes.py create mode 100644 models/tacotron2/NeMo/nemo/core/neural_types/comparison.py create mode 100644 models/tacotron2/NeMo/nemo/core/neural_types/elements.py create mode 100644 models/tacotron2/NeMo/nemo/core/neural_types/neural_type.py create mode 100644 models/tacotron2/NeMo/nemo/core/optim/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/core/optim/lr_scheduler.py create mode 100644 models/tacotron2/NeMo/nemo/core/optim/novograd.py create mode 100644 models/tacotron2/NeMo/nemo/core/optim/optimizers.py create mode 100644 models/tacotron2/NeMo/nemo/core/utils/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/core/utils/numba_utils.py create mode 100644 models/tacotron2/NeMo/nemo/package_info.py create mode 100644 models/tacotron2/NeMo/nemo/utils/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/utils/app_state.py create mode 100644 models/tacotron2/NeMo/nemo/utils/arguments.py create mode 100644 models/tacotron2/NeMo/nemo/utils/cloud.py create mode 100644 models/tacotron2/NeMo/nemo/utils/config_utils.py create mode 100644 models/tacotron2/NeMo/nemo/utils/decorators/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/utils/decorators/deprecated.py create mode 100644 models/tacotron2/NeMo/nemo/utils/decorators/experimental.py create mode 100644 models/tacotron2/NeMo/nemo/utils/decorators/port_docs.py create mode 100644 models/tacotron2/NeMo/nemo/utils/env_var_parsing.py create mode 100644 models/tacotron2/NeMo/nemo/utils/exceptions.py create mode 100644 models/tacotron2/NeMo/nemo/utils/exp_manager.py create mode 100644 models/tacotron2/NeMo/nemo/utils/export_utils.py create mode 100644 models/tacotron2/NeMo/nemo/utils/formatters/__init__.py create mode 100644 models/tacotron2/NeMo/nemo/utils/formatters/base.py create mode 100644 models/tacotron2/NeMo/nemo/utils/formatters/colors.py create mode 100644 models/tacotron2/NeMo/nemo/utils/formatters/utils.py create mode 100644 models/tacotron2/NeMo/nemo/utils/get_rank.py create mode 100644 models/tacotron2/NeMo/nemo/utils/lightning_logger_patch.py create mode 100644 models/tacotron2/NeMo/nemo/utils/metaclasses.py create mode 100644 models/tacotron2/NeMo/nemo/utils/model_utils.py create mode 100644 models/tacotron2/NeMo/nemo/utils/nemo_logging.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/README.md create mode 100644 models/tacotron2/NeMo/nemo_text_processing/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/README.md create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/clean_eval_data.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/currency.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/domain.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/server_name.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/magnitudes.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/measurements.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/months.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/digit.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/hundred.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/teen.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/thousands.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/ties.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/zero.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/digit.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/teen.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_suffix.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_to.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_zone.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/whitelist.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/inverse_normalize.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/run_evaluate.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/run_predict.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/cardinal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/date.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/decimal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/electronic.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/fraction.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/measure.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/money.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/ordinal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/punctuation.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/telephone.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/time.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/tokenize_and_classify.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/whitelist.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/word.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/utils.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/cardinal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/date.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/decimal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/electronic.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/fraction.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/measure.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/money.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/ordinal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/telephone.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/time.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/verbalize.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/verbalize_final.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/whitelist.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/word.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/setup.sh create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/README.md create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/clean_eval_data.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency_minor.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency_minor_one.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/domain.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/server_name.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/symbols.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/letter_pronunciation.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/magnitudes.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/measurements.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/abbr.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/names.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/numbers.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/__init__.py create mode 100755 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/cardinal_number_name.far create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/digit.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/hundred.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/teen.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/thousands.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/ties.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/zero.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/digit.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/teen.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/digit_teen.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/hundreds.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/ties.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/suppletive.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/time_suffix.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/time_zone.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/whitelist.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/whitelist_alternatives.tsv create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/data_loader_utils.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/graph_utils.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/normalize.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/normalize_with_audio.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/run_evaluate.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/run_predict.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/cardinal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/date.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/decimal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/electronic.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/fraction.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/measure.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/money.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/ordinal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/punctuation.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/roman.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/telephone.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/time.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/tokenize_and_classify.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/whitelist.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/word.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/token_parser.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/__init__.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/cardinal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/date.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/decimal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/electronic.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/fraction.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/measure.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/money.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/ordinal.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/roman.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/telephone.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/time.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/verbalize.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/verbalize_final.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/whitelist.py create mode 100644 models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/word.py create mode 100755 models/tacotron2/NeMo/reinstall.sh create mode 100644 models/tacotron2/NeMo/requirements/requirements.txt create mode 100644 models/tacotron2/NeMo/requirements/requirements_asr.txt create mode 100644 models/tacotron2/NeMo/requirements/requirements_cv.txt create mode 100644 models/tacotron2/NeMo/requirements/requirements_docs.txt create mode 100644 models/tacotron2/NeMo/requirements/requirements_nlp.txt create mode 100644 models/tacotron2/NeMo/requirements/requirements_test.txt create mode 100644 models/tacotron2/NeMo/requirements/requirements_text_processing.txt create mode 100644 models/tacotron2/NeMo/requirements/requirements_tts.txt create mode 100644 models/tacotron2/NeMo/scripts/asr_language_modeling/neural_rescorer/create_tarred_transformer_lm_dataset.py create mode 100644 models/tacotron2/NeMo/scripts/asr_language_modeling/neural_rescorer/eval_neural_rescorer.py create mode 100644 models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py create mode 100644 models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/install_beamsearch_decoders.sh create mode 100644 models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/kenlm_utils.py create mode 100644 models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/train_kenlm.py create mode 100644 models/tacotron2/NeMo/scripts/average_model_checkpoints.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/get_commonvoice_data.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/get_hi-mia_data.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/get_librispeech_data.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/get_openslr_rir_data.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/README create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/calculate_durs.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/create_manifests_and_textfiles.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/create_token2idx_dict.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/extract_ljspeech_energy_pitch.py create mode 100755 models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/extract_ljspeech_phonemes_and_durs.sh create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/process_an4_data.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/process_degli_data.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/process_hub5_data.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/process_speech_commands_data.py create mode 100644 models/tacotron2/NeMo/scripts/dataset_processing/process_vad_data.py create mode 100644 models/tacotron2/NeMo/scripts/export/convasr_to_enemo.py create mode 100644 models/tacotron2/NeMo/scripts/export/convasr_to_onnx.py create mode 100644 models/tacotron2/NeMo/scripts/export/convasr_to_single_onnx.py create mode 100644 models/tacotron2/NeMo/scripts/freesound_download_resample/download_resample_freesound.sh create mode 100644 models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_download.py create mode 100644 models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_requirements.txt create mode 100644 models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_resample.py create mode 100644 models/tacotron2/NeMo/scripts/nemo_legacy_import/asr_checkpoint_port.py create mode 100644 models/tacotron2/NeMo/scripts/neural_machine_translation/filter_langs_nmt.py create mode 100644 models/tacotron2/NeMo/scripts/speaker_recognition/rttm_to_manifest.py create mode 100644 models/tacotron2/NeMo/scripts/speaker_recognition/scp_to_manifest.py create mode 100644 models/tacotron2/NeMo/scripts/speech_recognition/convert_to_tarred_audio_dataset.py create mode 100644 models/tacotron2/NeMo/scripts/tokenizers/process_asr_text_tokenizer.py create mode 100644 models/tacotron2/NeMo/scripts/voice_activity_detection/vad_overlap_posterior.py create mode 100644 models/tacotron2/NeMo/scripts/voice_activity_detection/vad_tune_threshold.py create mode 100644 models/tacotron2/NeMo/scripts/voice_activity_detection/write_long_audio_manifest.py create mode 100644 models/tacotron2/NeMo/setup.cfg create mode 100644 models/tacotron2/NeMo/setup.py create mode 100644 models/tacotron2/NeMo/tests/__init__.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/test_rnnt_pytorch.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_gpu_rnnt_kernel.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_reduce.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_rnnt_helper.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/numba/spec_augment/test_spec_aug_numba.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_asr_classification_model.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_asr_ctcencdec_model.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_asr_datasets.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_asr_exportables.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_asr_filterbankfeatures_seq_len.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_asr_metrics.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_asr_modules.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_asr_regression_model.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_asr_rnnt_encdec_model.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_asr_rnnt_encoder_model_bpe.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_jasper_block.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_label_datasets.py create mode 100644 models/tacotron2/NeMo/tests/collections/asr/test_speaker_label_models.py create mode 100644 models/tacotron2/NeMo/tests/collections/common/__init__.py create mode 100644 models/tacotron2/NeMo/tests/collections/common/loss_inputs.py create mode 100644 models/tacotron2/NeMo/tests/collections/common/perplexity_inputs.py create mode 100644 models/tacotron2/NeMo/tests/collections/common/pl_utils.py create mode 100644 models/tacotron2/NeMo/tests/collections/common/test_metrics.py create mode 100644 models/tacotron2/NeMo/tests/collections/common/test_spc_tokenizer.py create mode 100644 models/tacotron2/NeMo/tests/collections/nlp/test_classification_report.py create mode 100644 models/tacotron2/NeMo/tests/collections/nlp/test_entity_linking_model.py create mode 100644 models/tacotron2/NeMo/tests/collections/nlp/test_huggingface.py create mode 100644 models/tacotron2/NeMo/tests/collections/nlp/test_megatron.py create mode 100644 models/tacotron2/NeMo/tests/collections/nlp/test_nlp_exportables.py create mode 100644 models/tacotron2/NeMo/tests/collections/nlp/test_nmt_model.py create mode 100644 models/tacotron2/NeMo/tests/collections/nlp/test_pretrained_models_performance.py create mode 100644 models/tacotron2/NeMo/tests/collections/tts/test_tts_datasets.py create mode 100644 models/tacotron2/NeMo/tests/collections/tts/test_tts_exportables.py create mode 100644 models/tacotron2/NeMo/tests/collections/tts/test_waveglow.py create mode 100644 models/tacotron2/NeMo/tests/conftest.py create mode 100644 models/tacotron2/NeMo/tests/core/test_config_utils.py create mode 100644 models/tacotron2/NeMo/tests/core/test_exp_manager.py create mode 100644 models/tacotron2/NeMo/tests/core/test_fileio.py create mode 100644 models/tacotron2/NeMo/tests/core/test_neural_types.py create mode 100644 models/tacotron2/NeMo/tests/core/test_optimizers_schedulers.py create mode 100644 models/tacotron2/NeMo/tests/core/test_save_restore.py create mode 100644 models/tacotron2/NeMo/tests/core/test_serialization.py create mode 100644 models/tacotron2/NeMo/tests/core/test_typecheck.py create mode 100644 models/tacotron2/NeMo/tests/core_ptl/check_for_ranks.py create mode 100644 models/tacotron2/NeMo/tests/hydra/config1.yaml create mode 100644 models/tacotron2/NeMo/tests/hydra/config1_invalid.yaml create mode 100644 models/tacotron2/NeMo/tests/hydra/config_subdir/config2.yaml create mode 100644 models/tacotron2/NeMo/tests/hydra/config_subdir/config2_invalid.yaml create mode 100644 models/tacotron2/NeMo/tests/hydra/my_app.py create mode 100644 models/tacotron2/NeMo/tests/hydra/test_hydra_runner.py create mode 100644 models/tacotron2/NeMo/tests/manualtest_model_downloads.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_cardinal.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_date.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_decimal.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_electronic.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_measure.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_money.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_ordinal.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_telephone.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_time.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_whitelist.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_word.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_boundary.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_cardinal.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_date.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_decimal.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_electronic.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_fraction.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_measure.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_money.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_normalize_with_audio.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_ordinal.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_telephone.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_time.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_whitelist.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_word.txt create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_boundary.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_cardinal.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_date.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_decimal.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_electronic.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_fraction.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_measure.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_money.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_normalization_with_audio.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_ordinal.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_sparrowhawk_inverse_text_normalization.sh create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_sparrowhawk_normalization.sh create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_telephone.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_time.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_whitelist.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/test_word.py create mode 100644 models/tacotron2/NeMo/tests/nemo_text_processing/utils.py create mode 100644 models/tacotron2/NeMo/tests/test_data_dir.py create mode 100644 models/tacotron2/NeMo/tools/ctc_segmentation/README.md create mode 100644 models/tacotron2/NeMo/tools/ctc_segmentation/requirements.txt create mode 100644 models/tacotron2/NeMo/tools/ctc_segmentation/run_sample.sh create mode 100644 models/tacotron2/NeMo/tools/ctc_segmentation/scripts/cut_audio.py create mode 100644 models/tacotron2/NeMo/tools/ctc_segmentation/scripts/normalization_helpers.py create mode 100644 models/tacotron2/NeMo/tools/ctc_segmentation/scripts/prepare_data.py create mode 100644 models/tacotron2/NeMo/tools/ctc_segmentation/scripts/process_manifests.py create mode 100644 models/tacotron2/NeMo/tools/ctc_segmentation/scripts/run_ctc_segmentation.py create mode 100644 models/tacotron2/NeMo/tools/ctc_segmentation/scripts/utils.py create mode 100644 models/tacotron2/NeMo/tools/ctc_segmentation/scripts/verify_segments.py create mode 100644 models/tacotron2/NeMo/tools/speech_data_explorer/README.md create mode 100755 models/tacotron2/NeMo/tools/speech_data_explorer/data_explorer.py create mode 100644 models/tacotron2/NeMo/tools/speech_data_explorer/requirements.txt create mode 100755 models/tacotron2/NeMo/tools/speech_data_explorer/screenshot.png create mode 100644 models/tacotron2/NeMo/tools/text_processing_deployment/Dockerfile create mode 100644 models/tacotron2/NeMo/tools/text_processing_deployment/README.rst create mode 100644 models/tacotron2/NeMo/tools/text_processing_deployment/docker/build.sh create mode 100644 models/tacotron2/NeMo/tools/text_processing_deployment/docker/launch.sh create mode 100644 models/tacotron2/NeMo/tools/text_processing_deployment/export_grammars.sh create mode 100644 models/tacotron2/NeMo/tools/text_processing_deployment/pynini_export.py create mode 100644 models/tacotron2/NeMo/tutorials/00_NeMo_Primer.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/01_NeMo_Models.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/AudioTranslationSample.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/VoiceSwapSample.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/asr/01_ASR_with_NeMo.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/asr/02_Online_ASR_Microphone_Demo.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/asr/03_Speech_Commands.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/asr/04_Online_Offline_Speech_Commands_Demo.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/asr/05_Online_Noise_Augmentation.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/asr/06_Voice_Activiy_Detection.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/asr/07_Online_Offline_Microphone_VAD_Demo.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/asr/08_ASR_with_Subword_Tokenization.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/asr/10_ASR_CTC_Language_Finetuning.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/asr/Offline_ASR.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/asr/images/alignment_example.png create mode 100644 models/tacotron2/NeMo/tutorials/asr/images/vad_post_overlap_diagram.png create mode 100644 models/tacotron2/NeMo/tutorials/nlp/01_Pretrained_Language_Models_for_Downstream_Tasks.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/nlp/02_NLP_Tokenizers.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/nlp/Entity_Linking_Medical.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/nlp/GLUE_Benchmark.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/nlp/Joint_Intent_and_Slot_Classification.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/nlp/Neural_Machine_Translation-Text2Sparql.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/nlp/Punctuation_and_Capitalization.ipynb create mode 100755 models/tacotron2/NeMo/tutorials/nlp/Question_Answering_Squad.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/nlp/Relation_Extraction-BioMegatron.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/nlp/Text_Classification_Sentiment_Analysis.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/nlp/Token_Classification-BioMegatron.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/nlp/Token_Classification_Named_Entity_Recognition.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/speaker_recognition/ASR_with_SpeakerDiarization.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/speaker_recognition/Speaker_Diarization_Inference.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/speaker_recognition/Speaker_Recognition_Verification.ipynb create mode 100755 models/tacotron2/NeMo/tutorials/text_processing/Inverse_Text_Normalization.ipynb create mode 100755 models/tacotron2/NeMo/tutorials/text_processing/Text_Normalization.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/text_processing/images/deployment.png create mode 100644 models/tacotron2/NeMo/tutorials/text_processing/images/pipeline.png create mode 100644 models/tacotron2/NeMo/tutorials/tools/CTC_Segmentation_Tutorial.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/tools/DefinedCrowd_x_NeMo_ASR_Training_Tutorial.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/tools/label-studio/setup-asr-preannotations.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/tts/1_TTS_inference.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/tts/2_TTS_Tacotron2_Training.ipynb create mode 100644 models/tacotron2/NeMo/tutorials/tts/3_TTS_TalkNet_Training.ipynb diff --git a/models/tacotron2/NeMo/.dockerignore b/models/tacotron2/NeMo/.dockerignore new file mode 100644 index 0000000000..14f5114d01 --- /dev/null +++ b/models/tacotron2/NeMo/.dockerignore @@ -0,0 +1,19 @@ +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +env +pip-log.txt +pip-delete-this-directory.txt +.tox +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +*.log +.git +**/*.nemo +**/*.ckpt diff --git a/models/tacotron2/NeMo/.github/ISSUE_TEMPLATE/bug_report.md b/models/tacotron2/NeMo/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..5aedacf07f --- /dev/null +++ b/models/tacotron2/NeMo/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,42 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** + +A clear and concise description of what the bug is. + +**Steps/Code to reproduce bug** + +Please list *minimal* steps or code snippet for us to be able to reproduce the bug. + +A helpful guide on on how to craft a minimal bug report http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports. + + +**Expected behavior** + +A clear and concise description of what you expected to happen. + +**Environment overview (please complete the following information)** + + - Environment location: [Bare-metal, Docker, Cloud(specify cloud provider - AWS, Azure, GCP, Collab)] + - Method of NeMo install: [pip install or from source]. Please specify exact commands you used to install. + - If method of install is [Docker], provide `docker pull` & `docker run` commands used + +**Environment details** + +If NVIDIA docker image is used you don't need to specify these. +Otherwise, please provide: +- OS version +- PyTorch version +- Python version + +**Additional context** + +Add any other context about the problem here. +Example: GPU model diff --git a/models/tacotron2/NeMo/.github/ISSUE_TEMPLATE/feature_request.md b/models/tacotron2/NeMo/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..e56d0d05e0 --- /dev/null +++ b/models/tacotron2/NeMo/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,25 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: feature request +assignees: okuchaiev + +--- + +**Is your feature request related to a problem? Please describe.** + +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** + +A clear and concise description of what you want to happen. +Provide a code snippet on how new APIs/changes would be used by others. + +**Describe alternatives you've considered** + +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** + +Add any other context or screenshots about the feature request here. diff --git a/models/tacotron2/NeMo/.gitignore b/models/tacotron2/NeMo/.gitignore new file mode 100644 index 0000000000..ff04a26262 --- /dev/null +++ b/models/tacotron2/NeMo/.gitignore @@ -0,0 +1,176 @@ +# log and data files +*.model +*.pkl +#*.ipynb +output +result +*.pt +tests/data/asr +.DS_Store +bert.pt.json +work +runs +fastspeech_output +.hydra + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +**.pyc + +# C extensions +*.so + +# Distribution / packaging +.idea +.Python +wandb +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +#parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/build + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don’t work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# VSCode project settins +.vscode/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site +/docs/html +/docs/docs_zh/zh + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Emacs backup files +*~ + +cifar-10-batches-py +*.tar.gz + +# Test data. +tests/.data +tests/data + +# outputs folder +examples/*/outputs +examples/*/NeMo_experiments +examples/*/nemo_experiments +examples/*/.hydra +examples/*/wandb +examples/*/data +wandb +dump.py + +docs/sources/source/test_build/ + +# Checkpoints, config files and temporary files created in tutorials. +examples/neural_graphs/*.chkpt +examples/neural_graphs/*.yml + +.hydra/ +nemo_experiments/ + diff --git a/models/tacotron2/NeMo/.readthedocs.yml b/models/tacotron2/NeMo/.readthedocs.yml new file mode 100644 index 0000000000..9b10e64855 --- /dev/null +++ b/models/tacotron2/NeMo/.readthedocs.yml @@ -0,0 +1,31 @@ +# ============================================================================= +# Copyright (c) 2020 NVIDIA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required field. +version: 2 + +# Build documentation in the docs/ directory with Sphinx. +sphinx: + configuration: docs/source/conf.py + +# Set the version of Python and requirements required to build your docs +python: + version: 3.7 + install: + - requirements: requirements/requirements_docs.txt diff --git a/models/tacotron2/NeMo/CONTRIBUTING.md b/models/tacotron2/NeMo/CONTRIBUTING.md new file mode 100644 index 0000000000..394ed9278b --- /dev/null +++ b/models/tacotron2/NeMo/CONTRIBUTING.md @@ -0,0 +1,68 @@ +# Contributions are welcome! + +We do all of NeMo's development in the open. Contributions from NeMo community are welcome. + + +# Pull Requests (PR) Guidelines + +**Send your PRs to the `main` branch** + +1) Make sure your PR does one thing. Have a clear answer to "What does this PR do?". +2) Read General Principles and style guide below +3) Make sure you sign your commits. E.g. use ``git commit -s`` when before your commit +4) Make sure all unittests finish successfully before sending PR ``pytest`` or (if yor dev box does not have GPU) ``pytest --cpu`` from NeMo's root folder +5) Send your PR and request a review + +## Unit tests +Quick tests (locally, while developing) +``` +pytest +# If you don't have NVIDIA GPU do: +# pytest --cpu +``` +Full tests, including pre-trained model downloads +``` +pytest --with_downloads +``` + +## Whom should you ask for review: +1. For changes to NeMo's core: @ericharper, @titu1994, @blisc, or @okuchaiev +1. For changes to NeMo's ASR collection: @titu1994, @redoctopus, @jbalam-nv, or @okuchaiev +1. For changes to NeMo's NLP collection: @MaximumEntropy, @ericharper, @ekmb, @yzhang123, @VahidooX, @vladgets, or @okuchaiev +1. For changes to NeMo's TTS collection: @blisc or @stasbel, or @okuchaiev + +Note that some people may self-assign to review your PR - in which case, please wait for them to add a review. + +Your pull requests must pass all checks and peer-review before they can be merged. + +# General principles +1. **User-oriented**: make it easy for end users, even at the cost of writing more code in the background +1. **Robust**: make it hard for users to make mistakes. +1. **Well-tested**: please add simple, fast unittests. Consider adding CI tests for end-to-end functionality. +1. **Reusable**: for every piece of code, think about how it can be reused in the future and make it easy to be reused. +1. **Readable**: code should be easier to read. +1. **Legal**: if you copy even one line of code from the Internet, make sure that the code allows the license that NeMo supports. Give credit and link back to the code. +1. **Sensible**: code should make sense. If you think a piece of code might be confusing, write comments. + +## Python style +We use ``black`` as our style guide. To check whether your code will pass style check (from the NeMo's repo folder) run: +``python setup.py style`` and if it does not pass run ``python setup.py style --fix``. + +1. Include docstrings for every class and method exposed to the user. +1. Use Python 3 type hints for every class and method exposed to the user. +1. Avoid wild import: ``from X import *`` unless in ``X.py``, ``__all__`` is defined. +1. Minimize the use of ``**kwargs``. +1. ``RaiseError`` is preferred to ``assert``. Write: ```if X: raise Error``` instead of ```assert X```. +1. Classes are preferred to standalone methods. +1. Methods should be atomic. A method shouldn't be longer than 75 lines, e.g. can be fit into the computer screen without scrolling. +1. If a method has arguments that don't fit into one line, each argument should be in its own line for readability. +1. Add ``__init__.py`` for every folder. +1. F-strings are prefered to formatted strings. +1. Loggers are preferred to print. In NeMo, you can use logger from ``from nemo.utils import logging`` +1. Private functions (functions start with ``_``) shouldn't be called outside its host file. +1. If a comment lasts multiple lines, use ``'''`` instead of ``#``. + +# Collections +Collection is a logical grouping of related Neural Modules. It is a grouping of modules that share a domain area or semantics. +When contributing module to a collection, please make sure it belongs to that category. +If you would like to start a new one and contribute back to the platform, you are very welcome to do so. diff --git a/models/tacotron2/NeMo/Dockerfile b/models/tacotron2/NeMo/Dockerfile new file mode 100644 index 0000000000..9e364b4b3e --- /dev/null +++ b/models/tacotron2/NeMo/Dockerfile @@ -0,0 +1,104 @@ +# syntax=docker/dockerfile:experimental + +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:21.05-py3 + + +# build an image that includes only the nemo dependencies, ensures that dependencies +# are included first for optimal caching, and useful for building a development +# image (by specifying build target as `nemo-deps`) +FROM ${BASE_IMAGE} as nemo-deps + +# Ensure apt-get won't prompt for selecting options +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y \ + libsndfile1 sox \ + libfreetype6 \ + python-setuptools swig \ + python-dev ffmpeg && \ + rm -rf /var/lib/apt/lists/* + +# uninstall stuff from base container +RUN pip uninstall -y sacrebleu torchtext + +# build torchaudio (change latest release version to match pytorch) +WORKDIR /tmp/torchaudio_build +RUN git clone --depth 1 --branch release/0.7 https://github.com/pytorch/audio.git && \ + cd audio && \ + BUILD_SOX=1 python setup.py install && \ + cd .. && rm -r audio + +# TODO: remove when 21.04 container is released +# build torchtext +WORKDIR /tmp/torchtext_build +RUN git clone --branch v0.8.1 https://github.com/pytorch/text.git && \ + cd text && \ + git submodule update --init --recursive && \ + python setup.py clean install && \ + cd .. && rm -r text + +#install TRT tools: PT quantization support and ONNX graph optimizer +WORKDIR /tmp/trt_build +RUN git clone https://github.com/NVIDIA/TensorRT.git && \ + cd TensorRT/tools/onnx-graphsurgeon && python setup.py install && \ + cd ../pytorch-quantization && \ + python setup.py install && \ + rm -fr /tmp/trt_build + +# install nemo dependencies +WORKDIR /tmp/nemo +COPY requirements . +RUN for f in $(ls requirements*.txt); do pip install --disable-pip-version-check --no-cache-dir -r $f; done + +# install nemo_text_processing dependencies +COPY nemo_text_processing /tmp/nemo/nemo_text_processing/ +RUN /bin/bash /tmp/nemo/nemo_text_processing/setup.sh + +# copy nemo source into a scratch image +FROM scratch as nemo-src +COPY . . + +# start building the final container +FROM nemo-deps as nemo +ARG NEMO_VERSION=1.0.2 + +# Check that NEMO_VERSION is set. Build will fail without this. Expose NEMO and base container +# version information as runtime environment variable for introspection purposes +RUN /usr/bin/test -n "$NEMO_VERSION" && \ + /bin/echo "export NEMO_VERSION=${NEMO_VERSION}" >> /root/.bashrc && \ + /bin/echo "export BASE_IMAGE=${BASE_IMAGE}" >> /root/.bashrc +RUN --mount=from=nemo-src,target=/tmp/nemo cd /tmp/nemo && pip install ".[all]" && \ + python -c "import nemo.collections.asr as nemo_asr" && \ + python -c "import nemo.collections.nlp as nemo_nlp" && \ + python -c "import nemo.collections.tts as nemo_tts" && \ + python -c "import nemo_text_processing.text_normalization as text_normalization" + +# TODO: Remove once 21.04 container is base container +# install latest numba version +RUN conda update -c numba numba -y + +# copy scripts/examples/tests into container for end user +WORKDIR /workspace/nemo +COPY scripts /workspace/nemo/scripts +COPY examples /workspace/nemo/examples +COPY tests /workspace/nemo/tests +COPY tutorials /workspace/nemo/tutorials +# COPY README.rst LICENSE /workspace/nemo/ + +RUN printf "#!/bin/bash\njupyter lab --no-browser --allow-root --ip=0.0.0.0" >> start-jupyter.sh && \ + chmod +x start-jupyter.sh + diff --git a/models/tacotron2/NeMo/Jenkinsfile b/models/tacotron2/NeMo/Jenkinsfile new file mode 100644 index 0000000000..2f4c453955 --- /dev/null +++ b/models/tacotron2/NeMo/Jenkinsfile @@ -0,0 +1,1451 @@ +pipeline { + agent { + docker { + image 'nvcr.io/nvidia/pytorch:21.05-py3' + args '--device=/dev/nvidia0 --gpus all --user 0:128 -v /home/TestData:/home/TestData -v $HOME/.cache/torch:/root/.cache/torch --shm-size=8g' + } + } + options { + timeout(time: 1, unit: 'HOURS') + disableConcurrentBuilds() + } + stages { + stage('PyTorch version') { + steps { + sh 'python -c "import torch; print(torch.__version__)"' + sh 'python -c "import torchvision; print(torchvision.__version__)"' + } + } + + stage('Uninstall torchtext') { + steps { + sh 'pip uninstall -y torchtext' + } + } + + stage('Install test requirements') { + steps { + sh 'apt-get update && apt-get install -y bc && pip install -r requirements/requirements_test.txt' + } + } + + stage('Copyright Headers check') { + steps { + sh 'python /home/TestData/check_copyright_header.py --dir .' + } + } + + stage('PyTorch STFT Patch check') { + steps { + sh 'python /home/TestData/check_stft_patch.py --dir .' + } + } + + stage('Code formatting checks') { + steps { + sh 'python setup.py style' + } + } + stage('Installation') { + steps { + sh './reinstall.sh release' + } + } + + stage('PyTorch Lightning version') { + steps { + sh 'python -c "import pytorch_lightning; print(pytorch_lightning.__version__)"' + } + } + + stage('PyTorch Lightning DDP Checks') { + steps { + sh 'python "tests/core_ptl/check_for_ranks.py"' + } + } + + stage('L0: Unit Tests GPU') { + steps { + sh 'pytest -m "not pleasefixme" --with_downloads --relax_numba_compat' + } + } + + stage('L0: Unit Tests CPU') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest -m "not pleasefixme" --cpu --with_downloads --relax_numba_compat' + } + } + + stage('L0: Computer Vision Integration') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage ('MNIST image classification with LeNet-5 Integration Test - on CPU') { + steps { + sh 'cd examples/cv && \ + python mnist_lenet5_image_classification_pure_lightning.py trainer.gpus=0 \ + trainer.accelerator=null \ + trainer.fast_dev_run=true model.dataset.data_folder=/home/TestData \ + && rm -rf outputs' + } + } + } + } + + // We have no integration tests, please enable this when one is added + // stage('L0: Integration Tests GPU') { + // steps { + // sh 'pytest -s -m "integration and not skipduringci and not pleasefixme"' + // } + // } + + // stage('L0: Integration Tests CPU') { + // when { + // anyOf{ + // branch 'main' + // changeRequest target: 'main' + // } + // } + // steps { + // sh 'pytest -s -m "integration and not pleasefixme" --cpu' + // } + // } + + // We have no system tests, please enable this when one is added + // stage('L1: System Tests GPU') { + // steps { + // sh 'pytest -m "system and not skipduringci and not pleasefixme"' + // } + // } + + // stage('L1: System Tests CPU') { + // when { + // anyOf{ + // branch 'dev + // changeRequest target: 'main' + // } + // } + // steps { + // sh 'pytest -m "system and not pleasefixme" --cpu' + // } + // } + + stage('L2: NeMo text processing') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L2: TN') { + steps { + sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/TestData/nlp/text_norm/output/ --grammars=tn_grammars && ls -R /home/TestData/nlp/text_norm/output/ && echo ".far files created "|| exit 1' + sh 'cd nemo_text_processing/text_normalization/ && python run_predict.py --input=/home/TestData/nlp/text_norm/ci/test.txt --input_case="lower_cased" --output=/home/TestData/nlp/text_norm/output/test.pynini.txt --verbose' + sh 'cmp --silent /home/TestData/nlp/text_norm/output/test.pynini.txt /home/TestData/nlp/text_norm/ci/test_goal_py.txt || exit 1' + sh 'rm -rf /home/TestData/nlp/text_norm/output/*' + } + } + + stage('L2: ITN export') { + steps { + sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/TestData/nlp/text_denorm/output/ --grammars=itn_grammars && ls -R /home/TestData/nlp/text_denorm/output/ && echo ".far files created "|| exit 1' + sh 'cd nemo_text_processing/inverse_text_normalization/ && python run_predict.py --input=/home/TestData/nlp/text_denorm/ci/test.txt --output=/home/TestData/nlp/text_denorm/output/test.pynini.txt --verbose' + sh 'cmp --silent /home/TestData/nlp/text_denorm/output/test.pynini.txt /home/TestData/nlp/text_denorm/ci/test_goal_py.txt || exit 1' + sh 'rm -rf /home/TestData/nlp/text_denorm/output/*' + } + } + stage('L2: TN with Audio (audio and raw text)') { + steps { + sh 'cd nemo_text_processing/text_normalization && \ + python normalize_with_audio.py --text "The total amounts to \\$4.76." \ + --audio_data /home/TestData/nlp/text_norm/audio_based/audio.wav | tail -n2 | head -n1 > /home/TestData/nlp/text_norm/audio_based/output/out_raw.txt 2>&1 && \ + cmp --silent /home/TestData/nlp/text_norm/audio_based/output/out_raw.txt /home/TestData/nlp/text_norm/audio_based/result.txt || exit 1' + sh 'rm -rf /home/TestData/nlp/text_norm/audio_based/output/out_raw.txt' + } + } + stage('L2: TN with Audio (audio and text file)') { + steps { + sh 'cd nemo_text_processing/text_normalization && \ + python normalize_with_audio.py --text /home/TestData/nlp/text_norm/audio_based/text.txt \ + --audio_data /home/TestData/nlp/text_norm/audio_based/audio.wav | tail -n2 | head -n1 > /home/TestData/nlp/text_norm/audio_based/output/out_file.txt 2>&1 && \ + cmp --silent /home/TestData/nlp/text_norm/audio_based/output/out_file.txt /home/TestData/nlp/text_norm/audio_based/result.txt || exit 1' + sh 'rm -rf /home/TestData/nlp/text_norm/audio_based/output/out_file.txt' + } + } + stage('L2: TN with Audio (manifest)') { + steps { + sh 'cd nemo_text_processing/text_normalization && \ + python normalize_with_audio.py --audio_data /home/TestData/nlp/text_norm/audio_based/manifest.json --n_tagged=120 && \ + cmp --silent /home/TestData/nlp/text_norm/audio_based/manifest_normalized.json /home/TestData/nlp/text_norm/audio_based/manifest_result.json || exit 1' + sh 'rm -rf /home/TestData/nlp/text_norm/audio_based/manifest_normalized.json' + } + } + } + } + + stage('L2: ASR dev run') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('Speech to Text') { + steps { + sh 'python examples/asr/speech_to_text.py \ + model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ + model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=True \ + exp_manager.exp_dir=examples/asr/speech_to_text_results' + sh 'rm -rf examples/asr/speech_to_text_results' + } + } + + stage('Speech to Label') { + steps { + sh 'python examples/asr/speech_to_label.py \ + model.train_ds.manifest_filepath=/home/TestData/speech_commands/train_manifest.json \ + model.validation_ds.manifest_filepath=/home/TestData/speech_commands/test_manifest.json \ + model.test_ds.manifest_filepath=/home/TestData/speech_commands/test_manifest.json \ + trainer.gpus=[1] \ + +trainer.fast_dev_run=True \ + model.preprocessor._target_=nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor \ + ~model.preprocessor.window_size \ + ~model.preprocessor.window_stride \ + ~model.preprocessor.window \ + ~model.preprocessor.n_mels \ + ~model.preprocessor.n_mfcc \ + ~model.preprocessor.n_fft \ + exp_manager.exp_dir=examples/asr/speech_to_label_results' + sh 'rm -rf examples/asr/speech_to_label_results' + } + } + + stage('Speaker Recognition') { + steps { + sh 'python examples/speaker_recognition/speaker_reco.py \ + model.train_ds.batch_size=10 \ + model.validation_ds.batch_size=2 \ + model.train_ds.manifest_filepath=/home/TestData/an4_speaker/train.json \ + model.validation_ds.manifest_filepath=/home/TestData/an4_speaker/dev.json \ + model.test_ds.manifest_filepath=/home/TestData/an4_speaker/test.json \ + trainer.gpus=[1] \ + +trainer.fast_dev_run=True \ + exp_manager.exp_dir=examples/speaker_recognition/speaker_recognition_results' + sh 'rm -rf examples/speaker_recognition/speaker_recognition_results' + } + } + + stage('Speaker Diarization Inference') { + steps { + sh 'python examples/speaker_recognition/speaker_diarize.py \ + diarizer.oracle_num_speakers=2 \ + diarizer.paths2audio_files=/home/TestData/an4_diarizer/audio_files.scp \ + diarizer.path2groundtruth_rttm_files=/home/TestData/an4_diarizer/rttm_files.scp \ + diarizer.speaker_embeddings.model_path=/home/TestData/an4_diarizer/spkr.nemo \ + diarizer.vad.model_path=/home/TestData/an4_diarizer/MatchboxNet_VAD_3x2.nemo \ + diarizer.out_dir=examples/speaker_recognition/speaker_diarization_results' + sh 'rm -rf examples/speaker_recognition/speaker_diarization_results' + } + } + + stage('L2: Speech to Text WPE - CitriNet') { + steps { + sh 'python examples/asr/speech_to_text_bpe.py \ + --config-path="conf/citrinet/" --config-name="config_bpe" \ + model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ + model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ + model.tokenizer.dir="/home/TestData/asr_tokenizers/an4_wpe_128/" \ + model.tokenizer.type="wpe" \ + trainer.gpus=[1] \ + +trainer.fast_dev_run=True \ + exp_manager.exp_dir=examples/asr/speech_to_text_wpe_results' + sh 'rm -rf examples/asr/speech_to_text_wpe_results' + } + } + + stage('L2: Speech to Text WPE - Conformer') { + steps { + sh 'python examples/asr/speech_to_text_bpe.py \ + --config-path="conf/conformer" --config-name="conformer_ctc_bpe" \ + model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ + model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ + model.tokenizer.dir="/home/TestData/asr_tokenizers/an4_wpe_128/" \ + model.tokenizer.type="wpe" \ + model.train_ds.batch_size=4 \ + model.validation_ds.batch_size=4 \ + trainer.gpus=[1] \ + +trainer.fast_dev_run=True \ + exp_manager.exp_dir=examples/asr/speech_to_text_wpe_conformer_results' + sh 'rm -rf examples/asr/speech_to_text_wpe_conformer_results' + } + } + } + } + + stage('L2: ASR DALI dev run') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('Speech to Text - DALI AudioToMelSpectrogramPreprocessor') { + steps { + sh 'python examples/asr/speech_to_text.py \ + model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ + +model.train_ds.use_dali=True \ + model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ + +model.validation_ds.use_dali=True \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=True \ + exp_manager.exp_dir=examples/asr/speech_to_text_results' + sh 'rm -rf examples/asr/speech_to_text_results' + } + } + // TODO: This would fail due to an unnecessary torchaudio import. + // To be enabled once torchaudio is available in the container used for CI + // stage('Speech to Text - DALI AudioToMFCCPreprocessor') { + // steps { + // sh 'python examples/asr/speech_to_text.py \ + // model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ + // +model.train_ds.use_dali=True \ + // model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ + // +model.validation_ds.use_dali=True \ + // model.preprocessor._target_=nemo.collections.asr.modules.AudioToMFCCPreprocessor \ + // ~model.preprocessor.normalize \ + // ~model.preprocessor.features \ + // ~model.preprocessor.frame_splicing \ + // ~model.preprocessor.dither \ + // ~model.preprocessor.stft_conv \ + // +model.n_mels=64 \ + // +model.n_mfcc=64 \ + // trainer.gpus=[0] \ + // +trainer.fast_dev_run=True \ + // exp_manager.exp_dir=examples/asr/speech_to_text_results' + // sh 'rm -rf examples/asr/speech_to_text_results' + // } + // } + } + } + +// TODO: UNCOMMENT TESTS AFTER 21.04 release (numba 0.53 min requirement) + stage('L2: ASR RNNT dev run') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('Speech to Text - RNNT') { + steps { + sh 'STRICT_NUMBA_COMPAT_CHECK=false python examples/asr/speech_to_text_rnnt.py \ + --config-path="experimental/contextnet_rnnt/" --config-name="config_rnnt.yaml" \ + model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ + model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ + model.train_ds.batch_size=2 \ + model.validation_ds.batch_size=2 \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=True \ + exp_manager.exp_dir=examples/asr/speech_to_text_rnnt_results' + sh 'rm -rf examples/asr/speech_to_text_rnnt_results' + } + } + stage('L2: Speech to Text RNNT WPE') { + steps { + sh 'STRICT_NUMBA_COMPAT_CHECK=false python examples/asr/speech_to_text_rnnt_bpe.py \ + --config-path="experimental/contextnet_rnnt/" --config-name="config_rnnt_bpe.yaml" \ + model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ + model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ + model.train_ds.batch_size=2 \ + model.validation_ds.batch_size=2 \ + model.tokenizer.dir="/home/TestData/asr_tokenizers/an4_wpe_128/" \ + model.tokenizer.type="wpe" \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=True \ + exp_manager.exp_dir=examples/asr/speech_to_text_rnnt_wpe_results' + sh 'rm -rf examples/asr/speech_to_text_rnnt_wpe_results' + } + } + } + } + + stage('L2: ASR Multi-dataloader dev run') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('Speech to Text multi-dataloader') { + steps { + sh 'python examples/asr/speech_to_text.py \ + model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ + model.validation_ds.manifest_filepath=[/home/TestData/an4_dataset/an4_val.json,/home/TestData/an4_dataset/an4_val.json] \ + trainer.gpus=[0] \ + trainer.max_epochs=1 \ + +trainer.max_steps=1 \ + +trainer.num_sanity_val_steps=1 \ + exp_manager.exp_dir=examples/asr/speech_to_text_results' + sh 'rm -rf examples/asr/speech_to_text_results' + } + } + + stage('Speech to Label multi-dataloader') { + steps { + sh 'python examples/asr/speech_to_label.py \ + model.train_ds.manifest_filepath=/home/TestData/speech_commands/train_manifest.json \ + model.validation_ds.manifest_filepath=[/home/TestData/speech_commands/test_manifest.json,/home/TestData/speech_commands/test_manifest.json] \ + trainer.gpus=[1] \ + trainer.max_epochs=1 \ + +trainer.max_steps=1 \ + +trainer.num_sanity_val_steps=1 \ + model.preprocessor._target_=nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor \ + ~model.preprocessor.window_size \ + ~model.preprocessor.window_stride \ + ~model.preprocessor.window \ + ~model.preprocessor.n_mels \ + ~model.preprocessor.n_mfcc \ + ~model.preprocessor.n_fft \ + exp_manager.exp_dir=examples/asr/speech_to_label_results' + sh 'rm -rf examples/asr/speech_to_label_results' + } + } + } + } + + stage('L2: Speech Transcription') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('Speech to Text Transcribe') { + steps { + sh 'python examples/asr/transcribe_speech.py \ + pretrained_name="QuartzNet15x5Base-En" \ + audio_dir="/home/TestData/an4_transcribe/test_subset/" \ + output_filename="stt_test_res.json" \ + cuda=true \ + amp=true' + sh 'rm -rf stt_test_res.json' + } + } + } + } + + stage('L2: Segmentation Tool') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + stages { + stage('Install ctc_segmentation requirements') { + steps { + sh 'cd tools/ctc_segmentation && \ + pip install -r requirements.txt && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ffmpeg' + } + } + + stage('Parallel ctc_segmentation test') { + failFast true + parallel { + stage('L2: Eng QN with .wav') { + steps { + sh 'cd tools/ctc_segmentation && \ + TIME=`date +"%Y-%m-%d-%T"` && \ + /bin/bash run_sample.sh \ + --MODEL_NAME_OR_PATH=QuartzNet15x5Base-En \ + --DATA_DIR=/home/TestData/ctc_segmentation/eng \ + --OUTPUT_DIR=/home/TestData/ctc_segmentation/eng/output${TIME} \ + --LANGUAGE=eng \ + --OFFSET=0 \ + --CUT_PREFIX=0 \ + --MIN_SEGMENT_LEN=0 \ + --AUDIO_FORMAT=.wav && \ + python /home/TestData/ctc_segmentation/verify_alignment.py \ + -r /home/TestData/ctc_segmentation/eng/eng_valid_segments.txt \ + -g /home/TestData/ctc_segmentation/eng/output${TIME}/verified_segments/nv_test_segments.txt && \ + rm -rf /home/TestData/ctc_segmentation/eng/output${TIME}' + } + } + stage('L2: Ru QN with .mp3') { + steps { + sh 'cd tools/ctc_segmentation && \ + TIME=`date +"%Y-%m-%d-%T"` && \ + /bin/bash run_sample.sh \ + --MODEL_NAME_OR_PATH=/home/TestData/ctc_segmentation/QuartzNet15x5-Ru-e512-wer14.45.nemo \ + --DATA_DIR=/home/TestData/ctc_segmentation/ru \ + --OUTPUT_DIR=/home/TestData/ctc_segmentation/ru/output${TIME} \ + --LANGUAGE=ru \ + --OFFSET=0 \ + --CUT_PREFIX=0 \ + --MIN_SEGMENT_LEN=0 \ + --AUDIO_FORMAT=.mp3 \ + --ADDITIONAL_SPLIT_SYMBOLS=";" && \ + python /home/TestData/ctc_segmentation/verify_alignment.py \ + -r /home/TestData/ctc_segmentation/ru/valid_ru_segments.txt \ + -g /home/TestData/ctc_segmentation/ru/output${TIME}/verified_segments/ru_segments.txt && \ + rm -rf /home/TestData/ctc_segmentation/ru/output${TIME}' + } + } + } + } + } + } + + stage('L2: Multi-GPU Megatron finetuning') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L2: Cased Megatron finetuning on MRPC') { + steps { + sh 'cd examples/nlp/glue_benchmark && \ + python glue_benchmark.py \ + model.dataset.data_dir=/home/TestData/nlp/glue_fake/MRPC \ + trainer.gpus=[0,1] \ + +trainer.fast_dev_run=true \ + model.dataset.use_cache=false \ + model.language_model.pretrained_model_name=megatron-bert-345m-cased \ + trainer.accelerator=ddp \ + exp_manager=null' + } + } + } + } + + stage('L2: SGD-QA') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L2: SGD-QA') { + steps { + sh 'cd examples/nlp/dialogue_state_tracking && \ + python sgd_qa.py \ + model.dataset.data_dir=/home/TestData/nlp/sgd_small \ + model.dataset.dialogues_example_dir=sgd_outputs \ + model.dataset.task_name=debug_sample \ + trainer.max_steps=1 \ + trainer.max_epochs=1 \ + model.train_ds.batch_size=2 \ + model.validation_ds.batch_size=2 \ + model.test_ds.batch_size=2 \ + model.nemo_path=null \ + trainer.val_check_interval=0.0 \ + trainer.gpus=[0,1] \ + model.dataset.use_cache=false \ + model.language_model.pretrained_model_name=bert-base-cased \ + trainer.accelerator=ddp \ + exp_manager=null && \ + rm -rf sgd_outputs' + } + } + stage('GLUE STS-b with AlBERT') { + steps { + sh 'python examples/nlp/glue_benchmark/glue_benchmark.py \ + model.dataset.use_cache=false \ + model.task_name=sts-b \ + model.dataset.data_dir=/home/TestData/nlp/glue_fake/STS-B \ + trainer.gpus=[1] \ + +trainer.fast_dev_run=True \ + model.language_model.pretrained_model_name=albert-base-v1 \ + exp_manager=null' + } + } + } + } + + stage('L2: Parallel BERT SQUAD v1.1 / v2.0') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('BERT SQUAD 1.1') { + // Cannot do fast_dev_run because squad needs whole dev dataset + steps { + sh 'cd examples/nlp/question_answering && \ + python question_answering_squad.py \ + model.train_ds.file=/home/TestData/nlp/squad_mini/v1.1/train-v1.1.json \ + model.dataset.use_cache=false \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ + model.test_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ + model.train_ds.batch_size=2 \ + model.train_ds.num_samples=2 \ + model.validation_ds.batch_size=2 \ + model.validation_ds.num_samples=2 \ + model.test_ds.num_samples=2 \ + model.test_ds.batch_size=2 \ + trainer.max_epochs=1 \ + +trainer.max_steps=1 \ + model.language_model.pretrained_model_name=bert-base-uncased \ + model.dataset.version_2_with_negative=false \ + trainer.precision=16 \ + trainer.amp_level=O1 \ + trainer.gpus=[0] \ + exp_manager=null' + } + } + stage('BERT SQUAD 2.0') { + // Cannot do fast_dev_run because squad needs whole dev dataset + steps { + sh 'cd examples/nlp/question_answering && \ + python question_answering_squad.py \ + model.train_ds.file=/home/TestData/nlp/squad_mini/v2.0/train-v2.0.json \ + model.dataset.use_cache=false \ + model.train_ds.batch_size=2 \ + model.train_ds.num_samples=2 \ + model.validation_ds.batch_size=2 \ + model.validation_ds.num_samples=2 \ + trainer.max_epochs=1 \ + +trainer.max_steps=1 \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v2.0/dev-v2.0.json \ + model.language_model.pretrained_model_name=bert-base-uncased \ + model.dataset.version_2_with_negative=true \ + trainer.precision=16 \ + trainer.amp_level=O1 \ + trainer.gpus=[1] \ + exp_manager=null' + } + } + } + } + // Runs out of memory on the 12G TITAN V (GPU 0 on main CI) + stage('L2: MegaBERT Token Classification') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + steps { + sh 'cd examples/nlp/token_classification && \ + python token_classification_train.py \ + model.dataset.data_dir=/home/TestData/nlp/token_classification_punctuation/ \ + model.language_model.pretrained_model_name=megatron-bert-345m-uncased \ + model.train_ds.batch_size=10 \ + model.dataset.max_seq_length=50 \ + model.dataset.use_cache=false \ + trainer.accelerator=ddp \ + trainer.precision=16 \ + trainer.amp_level=O1 \ + trainer.gpus=[1] \ + +trainer.fast_dev_run=true \ + exp_manager=null' + } + } + stage('L2: Parallel SQUAD v1.1 & v2.0') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('SQUAD v2.0 with Megatron with ckpt & config') { + // Cannot do fast_dev_run because squad needs whole dev dataset + steps { + sh 'cd examples/nlp/question_answering && \ + python question_answering_squad.py \ + model.train_ds.file=/home/TestData/nlp/squad_mini/v2.0/train-v2.0.json \ + model.dataset.use_cache=false \ + model.train_ds.batch_size=1 \ + model.train_ds.num_samples=1 \ + model.validation_ds.batch_size=1 \ + model.validation_ds.num_samples=1 \ + trainer.accelerator=ddp \ + trainer.max_epochs=1 \ + +trainer.max_steps=1 \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v2.0/dev-v2.0.json \ + model.language_model.pretrained_model_name=megatron-bert-uncased \ + model.language_model.lm_checkpoint=/home/TestData/nlp/megatron_345m_uncased/model_optim_rng.pt \ + model.language_model.config_file=/home/TestData/nlp/megatron_345m_uncased/345m_config.json \ + model.dataset.version_2_with_negative=true \ + trainer.precision=16 \ + trainer.amp_level=O1 \ + trainer.gpus=[1] \ + exp_manager=null' + } + } + stage('RoBERTa SQUAD 1.1') { + // Cannot do fast_dev_run because squad needs whole dev dataset + steps { + sh 'cd examples/nlp/question_answering && \ + python question_answering_squad.py \ + model.train_ds.file=/home/TestData/nlp/squad_mini/v1.1/train-v1.1.json \ + model.dataset.use_cache=false \ + model.train_ds.batch_size=2 \ + model.train_ds.num_samples=2 \ + model.validation_ds.batch_size=2 \ + model.validation_ds.num_samples=2 \ + trainer.max_epochs=1 \ + +trainer.max_steps=1 \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ + model.language_model.pretrained_model_name=roberta-base \ + model.dataset.version_2_with_negative=false \ + trainer.precision=16 \ + trainer.amp_level=O1 \ + trainer.gpus=[0] \ + exp_manager=null' + } + } + stage ('Text Classification with BERT Test') { + steps { + sh 'cd examples/nlp/text_classification && \ + python text_classification_with_bert.py \ + model.dataset.num_classes=6 \ + model.train_ds.file_path=/home/TestData/nlp/retail_text_classification/train.tsv \ + model.validation_ds.file_path=/home/TestData/nlp/retail_text_classification/dev.tsv \ + model.language_model.pretrained_model_name=bert-base-uncased \ + model.train_ds.batch_size=10 \ + model.dataset.max_seq_length=50 \ + model.dataset.use_cache=false \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=true \ + exp_manager=null' + } + } + stage('L2: Intent and Slot Classification') { + steps { + sh 'cd examples/nlp/intent_slot_classification && \ + python intent_slot_classification.py \ + model.data_dir=/home/TestData/nlp/retail \ + model.validation_ds.prefix=dev \ + model.test_ds.prefix=dev \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=true \ + exp_manager.exp_dir=checkpoints' + sh 'rm -rf checkpoints' + } + } + } + } + + stage('L2: Model Parallel Size 2 Megatron Text Classification') { + when { + anyOf{ + branch 'main' + changeRequest target: 'main' + } + } + failFast true + steps{ + sh 'cd examples/nlp/text_classification && \ + python text_classification_with_bert.py \ + trainer.gpus=[0,1] \ + trainer.num_nodes=1 \ + trainer.precision=16 \ + trainer.gradient_clip_val=1.0 \ + ~trainer.amp_level \ + +trainer.fast_dev_run=true \ + model.dataset.num_classes=6 \ + model.train_ds.file_path=/home/TestData/nlp/retail_text_classification/train.tsv \ + model.train_ds.batch_size=4 \ + model.language_model.pretrained_model_name=megatron-bert-uncased \ + model.language_model.config_file=/home/TestData/nlp/mp_2_bert_toy/config.json \ + model.language_model.lm_checkpoint=/home/TestData/nlp/mp_2_bert_toy/iter_2000000 \ + model.nemo_path=null \ + ~model.infer_samples \ + exp_manager=null' + } + } + + stage('L2: Model Parallel Size 2 Megatron Autoresume') { + when { + anyOf{ + branch 'main' + changeRequest target: 'main' + } + } + failFast true + steps{ + sh 'cd examples/nlp/text_classification && \ + python text_classification_with_bert.py \ + trainer.gpus=[0,1] \ + trainer.num_nodes=1 \ + trainer.precision=16 \ + trainer.gradient_clip_val=1.0 \ + trainer.max_epochs=1 \ + ~trainer.amp_level \ + +trainer.fast_dev_run=true \ + model.dataset.num_classes=6 \ + model.train_ds.file_path=/home/TestData/nlp/retail_text_classification/train.tsv \ + model.train_ds.batch_size=4 \ + model.language_model.pretrained_model_name=megatron-bert-uncased \ + model.language_model.config_file=/home/TestData/nlp/mp_2_bert_toy/config.json \ + model.language_model.lm_checkpoint=/home/TestData/nlp/mp_2_bert_toy/iter_2000000 \ + model.nemo_path=null \ + ~model.infer_samples \ + +exp_manager.explicit_log_dir=/home/TestData/nlp/mp_autoresume \ + +exp_manager.resume_if_exists=true' + } + } + + stage('L2: Model Parallel Size 2 Megatron Evaluation from .nemo') { + when { + anyOf{ + branch 'main' + changeRequest target: 'main' + } + } + failFast true + steps{ + sh 'cd examples/nlp/text_classification && \ + python model_parallel_text_classification_evaluation.py \ + trainer.gpus=[0,1] \ + trainer.num_nodes=1 \ + model.dataset.num_classes=6 \ + model.test_ds.file_path=/home/TestData/nlp/retail_text_classification/dev.tsv \ + model.nemo_path=/home/TestData/nlp/mp_2_nemo/retail_text_class_350M.nemo \ + exp_manager=null' + } + } + + stage('L2: Model Parallel Size 2 Megatron Train from .nemo') { + when { + anyOf{ + branch 'main' + changeRequest target: 'main' + } + } + failFast true + steps{ + sh 'cd examples/nlp/token_classification && \ + python token_classification_train.py \ + pretrained_model=/home/TestData/nlp/mp_2_nemo/ner_350M.nemo \ + model.dataset.data_dir=/home/TestData/nlp/ner/ \ + model.train_ds.batch_size=2 \ + model.dataset.use_cache=false \ + trainer.gpus=[0,1] \ + +trainer.fast_dev_run=true \ + model.dataset.class_balancing="weighted_loss" \ + exp_manager=null' + } + } + + stage('L2: Parallel NLP Examples 2') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage ('NER finetuning from pretrained Test') { + steps { + sh 'cd examples/nlp/token_classification && \ + python token_classification_train.py \ + pretrained_model=ner_en_bert \ + model.dataset.data_dir=/home/TestData/nlp/ner/ \ + model.train_ds.batch_size=2 \ + model.dataset.use_cache=false \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=true \ + model.dataset.class_balancing="weighted_loss" \ + exp_manager.exp_dir=null' + } + } + stage ('Punctuation and capitalization finetuning from pretrained test') { + steps { + sh 'cd examples/nlp/token_classification && \ + python punctuation_capitalization_train.py \ + pretrained_model=punctuation_en_bert \ + model.dataset.data_dir=/home/TestData/nlp/token_classification_punctuation/ \ + trainer.gpus=[1] \ + +trainer.fast_dev_run=true \ + model.dataset.use_cache=false \ + exp_manager.exp_dir=null' + } + } + stage ('NER with TurkuNLP/bert-base-finnish-cased-v1') { + steps { + sh 'cd examples/nlp/token_classification && \ + python token_classification_train.py \ + model.dataset.data_dir=/home/TestData/nlp/token_classification_punctuation/ \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=true \ + model.dataset.use_cache=false \ + model.language_model.pretrained_model_name="TurkuNLP/bert-base-finnish-cased-v1" \ + exp_manager.exp_dir=null' + } + } + stage('Evaluation script for Token Classification') { + steps { + sh 'python examples/nlp/token_classification/token_classification_evaluate.py \ + model.dataset.data_dir=/home/TestData/nlp/ner/ \ + pretrained_model=/home/TestData/nlp/pretrained_models/NER_Model_with_BERT_base_uncased.nemo && \ + rm -rf nemo_experiments' + } + } + stage('Evaluation script for Punctuation') { + steps { + sh 'python examples/nlp/token_classification/punctuation_capitalization_evaluate.py \ + model.dataset.data_dir=/home/TestData/nlp/token_classification_punctuation/ \ + pretrained_model=/home/TestData/nlp/pretrained_models/Punctuation_Capitalization_with_DistilBERT_base_uncased.nemo && \ + rm -rf nemo_experiments' + } + } + stage('L2: Punctuation & Capitalization, 2GPUs with DistilBERT') { + steps { + sh 'cd examples/nlp/token_classification && \ + python punctuation_capitalization_train.py \ + model.dataset.data_dir=/home/TestData/nlp/token_classification_punctuation/ \ + model.language_model.pretrained_model_name=distilbert-base-uncased \ + model.dataset.use_cache=false \ + trainer.gpus=[0,1] \ + trainer.accelerator=ddp \ + trainer.max_epochs=1 \ + +exp_manager.explicit_log_dir=/home/TestData/nlp/token_classification_punctuation/output && \ + python punctuation_capitalization_evaluate.py \ + pretrained_model=/home/TestData/nlp/token_classification_punctuation/output/checkpoints/Punctuation_and_Capitalization.nemo && \ + rm -rf /home/TestData/nlp/token_classification_punctuation/output/*' + } + } + } + } + + stage('L2: Parallel Pretraining BERT pretraining from Text/Preprocessed') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L2: Pretraining BERT pretraining from Text') { + steps { + sh 'cd examples/nlp/language_modeling && \ + python bert_pretraining.py \ + --config-name=bert_pretraining_from_text_config.yaml \ + trainer.gpus=[0] \ + trainer.precision=16 \ + trainer.amp_level=O1 \ + +trainer.fast_dev_run=true \ + model.train_ds.data_file=/home/TestData/nlp/wikitext-2/train.txt \ + model.train_ds.batch_size=32 \ + model.validation_ds.data_file=/home/TestData/nlp/wikitext-2/valid.txt \ + model.validation_ds.batch_size=32 \ + model.language_model.config_file=/home/TestData/nlp/bert_configs/bert_3200.json \ + model.optim.lr=0.01 \ + model.optim.sched.warmup_ratio=0.1 \ + model.tokenizer.tokenizer_name=sentencepiece \ + model.tokenizer.tokenizer_model=/home/TestData/nlp/wikitext-2/tokenizer_bpe_v3193/tokenizer.model \ + model.mask_prob=0.15 \ + model.short_seq_prob=0.1 \ + exp_manager.exp_dir=PretrainingBERTFromText \ + ' + sh 'rm -f /home/TestData/nlp/wikitext-2/*.pkl' + sh 'rm -rf examples/nlp/language_modeling/PretrainingBERTFromText' + sh 'ls -lha examples/nlp/language_modeling' + } + } + stage('L2: Pretraining BERT from Preprocessed') { + steps { + sh 'cd examples/nlp/language_modeling && \ + python bert_pretraining.py \ + --config-name=bert_pretraining_from_preprocessed_config.yaml \ + trainer.gpus=[1] \ + trainer.precision=16 \ + trainer.amp_level=O1 \ + +trainer.fast_dev_run=true \ + model.train_ds.data_file=/home/TestData/nlp/wiki_book_mini/training \ + model.train_ds.batch_size=8 \ + model.language_model.lm_checkpoint=/home/TestData/nlp/bert_ckpts/nemo1.0/bert_base_uncased_mlm_final_1074591_nemo1.0.pt \ + model.language_model.config_file=/home/TestData/nlp/bert_configs/uncased_L-12_H-768_A-12.json \ + model.optim.lr=0.875e-4 \ + model.optim.weight_decay=0.01 \ + model.optim.sched.warmup_ratio=0.01 \ + exp_manager.exp_dir=PretrainingBERTFromPreprocessed \ + exp_manager.create_checkpoint_callback=False \ + ' + sh 'rm -rf examples/nlp/language_modeling/PretrainingBERTFromPreprocessed' + sh 'ls -lha examples/nlp/language_modeling' + } + } + stage('L2: Pretraining BERT pretraining from Text with char tokenizer') { + steps { + sh 'cd examples/nlp/language_modeling && \ + python bert_pretraining.py \ + --config-name=bert_pretraining_from_text_config.yaml \ + trainer.gpus=[0] \ + trainer.precision=16 \ + trainer.amp_level=O1 \ + +trainer.fast_dev_run=true \ + model.train_ds.data_file=/home/TestData/nlp/wikitext-2/train.txt \ + model.train_ds.batch_size=32 \ + model.validation_ds.data_file=/home/TestData/nlp/wikitext-2/valid.txt \ + model.validation_ds.batch_size=32 \ + model.language_model.config_file=/home/TestData/nlp/bert_configs/bert_3200.json \ + model.optim.lr=0.01 \ + model.optim.sched.warmup_ratio=0.1 \ + model.tokenizer.tokenizer_name=char \ + model.tokenizer.vocab_file=/home/TestData/nlp/vocabs/mini_vocab.txt \ + model.mask_prob=0.15 \ + model.short_seq_prob=0.1 \ + exp_manager.exp_dir=PretrainingBERTFromTextchartok \ + ' + sh 'rm -rf examples/nlp/language_modeling/PretrainingBERTFromTextchartok' + } + } + stage('L2: Pretraining BERT pretraining from Text with word tokenizer') { + steps { + sh 'cd examples/nlp/language_modeling && \ + python bert_pretraining.py \ + --config-name=bert_pretraining_from_text_config.yaml \ + trainer.gpus=[1] \ + trainer.precision=16 \ + trainer.amp_level=O1 \ + +trainer.fast_dev_run=true \ + model.train_ds.data_file=/home/TestData/nlp/wikitext-2/train.txt \ + model.train_ds.batch_size=32 \ + model.validation_ds.data_file=/home/TestData/nlp/wikitext-2/valid.txt \ + model.validation_ds.batch_size=32 \ + model.language_model.config_file=/home/TestData/nlp/bert_configs/bert_3200.json \ + model.optim.lr=0.01 \ + model.optim.sched.warmup_ratio=0.1 \ + model.tokenizer.tokenizer_name=word \ + model.tokenizer.vocab_file=/home/TestData/nlp/vocabs/mini_vocab.txt \ + model.mask_prob=0.15 \ + model.short_seq_prob=0.1 \ + exp_manager.exp_dir=PretrainingBERTFromTextwordtok \ + ' + sh 'rm -rf examples/nlp/language_modeling/PretrainingBERTFromTextwordtok' + } + } + } + } + + stage('L2: Entity Linking') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage ('Self Alignment Pretraining BERT') { + steps { + sh 'cd examples/nlp/entity_linking && \ + python self_alignment_pretraining.py \ + project_dir=. \ + trainer.val_check_interval=3 \ + model.raw_data=None \ + model.train_ds.data_file=/home/TestData/nlp/entity_linking/tiny_example_train_pairs.tsv \ + model.validation_ds.data_file=/home/TestData/nlp/entity_linking/tiny_example_validation_pairs.tsv \ + model.train_ds.batch_size=8 \ + model.validation_ds.batch_size=8 \ + exp_manager.exp_dir=null' + } + } + } + } + + stage('L2: NMT Attention is All You Need Training') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L2: NMT Training Post-LN') { + steps { + sh 'cd examples/nlp/machine_translation && \ + python enc_dec_nmt.py \ + --config-path=conf \ + --config-name=aayn_base \ + do_testing=true \ + model.train_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.train_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.ref \ + model.validation_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.validation_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.test_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.test_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.encoder_tokenizer.tokenizer_model=/home/TestData/nlp/nmt/toy_data/tt_tokenizer.BPE.4096.model \ + model.decoder_tokenizer.tokenizer_model=/home/TestData/nlp/nmt/toy_data/tt_tokenizer.BPE.4096.model \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=true \ + +trainer.limit_test_batches=2 \ + exp_manager=null \ + ' + } + } + + stage('L2: NMT Training Pre-LN') { + steps { + sh 'cd examples/nlp/machine_translation && \ + python enc_dec_nmt.py \ + --config-path=conf \ + --config-name=aayn_base \ + do_testing=true \ + model.train_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.train_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.ref \ + model.validation_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.validation_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.test_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.test_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.encoder_tokenizer.tokenizer_model=/home/TestData/nlp/nmt/toy_data/tt_tokenizer.BPE.4096.model \ + model.decoder_tokenizer.tokenizer_model=/home/TestData/nlp/nmt/toy_data/tt_tokenizer.BPE.4096.model \ + model.encoder.pre_ln=true \ + model.decoder.pre_ln=true \ + trainer.gpus=[1] \ + +trainer.fast_dev_run=true \ + +trainer.limit_test_batches=2 \ + exp_manager=null \ + ' + } + } + stage('L2: NMT Multi-Validation') { + steps { + sh 'cd examples/nlp/machine_translation && \ + python enc_dec_nmt.py \ + --config-path=conf \ + --config-name=aayn_base \ + do_testing=true \ + model.train_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-en-de.src \ + model.train_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-en-de.ref \ + model.validation_ds.src_file_name=[/home/TestData/nlp/nmt/toy_data/wmt13-en-de.src,/home/TestData/nlp/nmt/toy_data/wmt14-en-de.src] \ + model.validation_ds.tgt_file_name=[/home/TestData/nlp/nmt/toy_data/wmt13-en-de.ref,/home/TestData/nlp/nmt/toy_data/wmt14-en-de.ref] \ + model.test_ds.src_file_name=[/home/TestData/nlp/nmt/toy_data/wmt13-en-de.src,/home/TestData/nlp/nmt/toy_data/wmt14-en-de.src] \ + model.test_ds.tgt_file_name=[/home/TestData/nlp/nmt/toy_data/wmt13-en-de.ref,/home/TestData/nlp/nmt/toy_data/wmt14-en-de.ref] \ + model.encoder_tokenizer.tokenizer_model=/home/TestData/nlp/nmt/toy_data/tt_tokenizer.BPE.4096.model \ + model.decoder_tokenizer.tokenizer_model=/home/TestData/nlp/nmt/toy_data/tt_tokenizer.BPE.4096.model \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=true \ + +trainer.limit_test_batches=2 \ + exp_manager=null \ + ' + } + } + } + } + stage('L2: NMT Attention is All You Need Inference') { + when { + anyOf { + branch 'v1.0.0' + changeRequest target: 'v1.0.0' + } + } + failFast true + parallel { + stage('L2: NMT Inference - PostLN') { + steps { + sh 'cd examples/nlp/machine_translation && \ + python nmt_transformer_infer.py \ + --model=/home/TestData/nlp/nmt/toy_data/TransformerLargeDe-En.nemo \ + --srctext=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.test.src \ + --tgtout=/home/TestData/nlp/nmt/toy_data/out.txt \ + --target_lang en \ + --source_lang de \ + ' + } + } + stage('L2: NMT Inference - Pre-LN') { + steps { + sh 'cd examples/nlp/machine_translation && \ + python nmt_transformer_infer.py \ + --model=/home/TestData/nlp/nmt/toy_data/en_de_24x6_preln.nemo \ + --srctext=/home/TestData/nlp/nmt/toy_data/wmt14-en-de.test.src \ + --tgtout=/home/TestData/nlp/nmt/toy_data/out.txt \ + --target_lang de \ + --source_lang en \ + ' + } + } + } + } + stage('L2: NMT with HuggingFace') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L2: NMT Pretrained HF Encoder') { + steps { + sh 'cd examples/nlp/machine_translation && \ + python enc_dec_nmt.py \ + --config-path=conf \ + --config-name=huggingface \ + model.shared_tokenizer=False \ + model.encoder_tokenizer.library=huggingface \ + model.encoder.library=huggingface \ + model.encoder.model_name=bert-base-cased \ + model.encoder.pretrained=true \ + model.train_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.train_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.ref \ + model.validation_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.validation_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.test_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.test_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.decoder_tokenizer.tokenizer_model=/home/TestData/nlp/nmt/toy_data/tt_tokenizer.BPE.4096.model \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=true \ + exp_manager=null \ + ' + } + } + + stage('L2: NMT Custom HF Encoder') { + steps { + sh 'cd examples/nlp/machine_translation && \ + python enc_dec_nmt.py \ + --config-path=conf \ + --config-name=huggingface \ + model.shared_tokenizer=True \ + model.encoder_tokenizer.library=yttm \ + model.encoder_tokenizer.tokenizer_model=/home/TestData/nlp/nmt/toy_data/tt_tokenizer.BPE.4096.model \ + model.encoder.library=huggingface \ + model.encoder.model_name=null \ + model.encoder.pretrained=false \ + +model.encoder._target_=transformers.BertConfig \ + +model.encoder.hidden_size=1536 \ + model.train_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.train_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.ref \ + model.validation_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.validation_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.test_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.test_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.decoder_tokenizer.tokenizer_model=/home/TestData/nlp/nmt/toy_data/tt_tokenizer.BPE.4096.model \ + trainer.gpus=[1] \ + +trainer.fast_dev_run=true \ + exp_manager=null \ + ' + } + } + } + } + + stage('L2: NMT Tarred Dataset Creation') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L2: NMT Auto Tarred Dataset Creation') { + steps { + sh 'cd examples/nlp/machine_translation && \ + python enc_dec_nmt.py \ + --config-path=conf \ + --config-name=aayn_base \ + do_training=false \ + model.preproc_out_dir=$PWD/preproc_out_dir \ + model.train_ds.use_tarred_dataset=true \ + model.train_ds.n_preproc_jobs=2 \ + model.train_ds.lines_per_dataset_fragment=500 \ + model.train_ds.num_batches_per_tarfile=10 \ + model.train_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.train_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.ref \ + model.validation_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.validation_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.encoder_tokenizer.vocab_size=2000 \ + model.decoder_tokenizer.vocab_size=2000 \ + ~model.test_ds \ + trainer.gpus=[0] \ + +trainer.fast_dev_run=true \ + exp_manager=null \ + ' + } + } + + stage('L2: NMT Script Tarred Dataset Creation') { + steps { + sh 'cd examples/nlp/machine_translation && \ + python create_tarred_parallel_dataset.py \ + --src_fname /home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + --tgt_fname /home/TestData/nlp/nmt/toy_data/wmt14-de-en.ref \ + --out_dir $PWD/out_dir \ + --encoder_tokenizer_vocab_size=2000 \ + --decoder_tokenizer_vocab_size=2000 \ + --tokens_in_batch=1000 \ + --lines_per_dataset_fragment=500 \ + --num_batches_per_tarfile=10 \ + --n_preproc_jobs=2 \ + ' + } + } + } + } + + stage('L2: TTS Fast dev runs 1') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + parallel { + stage('Tacotron 2') { + steps { + sh 'python examples/tts/tacotron2.py \ + train_dataset=/home/TestData/an4_dataset/an4_train.json \ + validation_datasets=/home/TestData/an4_dataset/an4_val.json \ + trainer.gpus="[0]" \ + +trainer.fast_dev_run=True \ + trainer.accelerator=null \ + trainer.max_epochs=-1 \ + model.train_ds.dataloader_params.batch_size=12 \ + model.validation_ds.dataloader_params.batch_size=12 \ + ~trainer.check_val_every_n_epoch' + } + } + // stage('FastPitch') { + // steps { + // sh 'python examples/tts/fastpitch.py \ + // train_dataset=/home/TestData/an4_dataset/an4_train.json \ + // validation_datasets=/home/TestData/an4_dataset/an4_val.json \ + // trainer.gpus="[0]" \ + // +trainer.fast_dev_run=True \ + // trainer.accelerator=null \ + // trainer.max_epochs=-1 \ + // model.train_ds.batch_size=12 \ + // model.train_ds.num_workers=1 \ + // model.validation_ds.batch_size=12 \ + // model.validation_ds.num_workers=1 \ + // ~trainer.check_val_every_n_epoch' + // } + // } + stage('WaveGlow') { + steps { + sh 'python examples/tts/waveglow.py \ + train_dataset=/home/TestData/an4_dataset/an4_train.json \ + validation_datasets=/home/TestData/an4_dataset/an4_val.json \ + trainer.gpus="[1]" \ + +trainer.fast_dev_run=True \ + trainer.accelerator=null \ + trainer.max_epochs=-1 \ + model.train_ds.dataloader_params.batch_size=4 \ + model.validation_ds.dataloader_params.batch_size=4 \ + ~trainer.check_val_every_n_epoch' + } + } + } + } + + stage('L2: TTS Fast dev runs 2') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + + parallel { + stage('MelGAN') { + steps { + sh 'python examples/tts/melgan.py \ + train_dataset=/home/TestData/an4_dataset/an4_train.json \ + validation_datasets=/home/TestData/an4_dataset/an4_val.json \ + trainer.gpus="[0]" \ + +trainer.fast_dev_run=True \ + trainer.accelerator=ddp \ + trainer.max_epochs=-1 \ + model.train_ds.dataloader_params.batch_size=4 \ + model.validation_ds.dataloader_params.batch_size=4 \ + ~trainer.check_val_every_n_epoch' + } + } + stage('SqueezeWave') { + steps { + sh 'python examples/tts/squeezewave.py \ + train_dataset=/home/TestData/an4_dataset/an4_train.json \ + validation_datasets=/home/TestData/an4_dataset/an4_val.json \ + trainer.gpus="[0]" \ + +trainer.fast_dev_run=True \ + trainer.accelerator=null \ + trainer.max_epochs=-1 \ + model.train_ds.dataloader_params.batch_size=4 \ + model.validation_ds.dataloader_params.batch_size=4 \ + ~trainer.check_val_every_n_epoch' + } + } + stage('GlowTTS') { + steps { + sh 'python examples/tts/glow_tts.py \ + train_dataset=/home/TestData/an4_dataset/an4_train.json \ + validation_datasets=/home/TestData/an4_dataset/an4_val.json \ + trainer.gpus="[1]" \ + +trainer.fast_dev_run=True \ + trainer.accelerator=null \ + trainer.max_epochs=-1 \ + model.train_ds.batch_size=4 \ + model.validation_ds.batch_size=4 \ + ~trainer.check_val_every_n_epoch' + } + } + } + } + + stage('L??: Speech Checkpoints tests') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + steps { + sh 'CUDA_VISIBLE_DEVICES=0 python examples/asr/speech_to_text_infer.py --asr_model QuartzNet15x5Base-En --dataset /home/TestData/librispeech/librivox-dev-other.json --wer_tolerance 0.1012 --batch_size 64' + } + } + } + + post { + always { + sh 'chmod -R 777 .' + cleanWs() + } + } +} diff --git a/models/tacotron2/NeMo/LICENSE b/models/tacotron2/NeMo/LICENSE new file mode 100644 index 0000000000..f49a4e16e6 --- /dev/null +++ b/models/tacotron2/NeMo/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/models/tacotron2/NeMo/README.rst b/models/tacotron2/NeMo/README.rst new file mode 100644 index 0000000000..585064f17e --- /dev/null +++ b/models/tacotron2/NeMo/README.rst @@ -0,0 +1,174 @@ + +|status| |documentation| |license| |lgtm_grade| |lgtm_alerts| |black| + +.. |status| image:: http://www.repostatus.org/badges/latest/active.svg + :target: http://www.repostatus.org/#active + :alt: Project Status: Active – The project has reached a stable, usable state and is being actively developed. + +.. |documentation| image:: https://readthedocs.com/projects/nvidia-nemo/badge/?version=main + :alt: Documentation + :target: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/ + +.. |license| image:: https://img.shields.io/badge/License-Apache%202.0-brightgreen.svg + :target: https://github.com/NVIDIA/NeMo/blob/master/LICENSE + :alt: NeMo core license and license for collections in this repo + +.. |lgtm_grade| image:: https://img.shields.io/lgtm/grade/python/g/NVIDIA/NeMo.svg?logo=lgtm&logoWidth=18 + :target: https://lgtm.com/projects/g/NVIDIA/NeMo/context:python + :alt: Language grade: Python + +.. |lgtm_alerts| image:: https://img.shields.io/lgtm/alerts/g/NVIDIA/NeMo.svg?logo=lgtm&logoWidth=18 + :target: https://lgtm.com/projects/g/NVIDIA/NeMo/alerts/ + :alt: Total alerts + +.. |black| image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + :alt: Code style: black + +.. _main-readme: +**NVIDIA NeMo** +=============== + +Introduction +------------ + +NVIDIA NeMo is a conversational AI toolkit built for researchers working on automatic speech recognition (ASR), natural language processing (NLP), and text-to-speech synthesis (TTS). +The primary objective of NeMo is to help researchers from industry and academia to reuse prior work (code and pretrained models and make it easier to create new `conversational AI models `_. + + +`Introductory video. `_ + +Key Features +------------ + +* Speech processing + * `Automatic Speech recognition (ASR) `_: Jasper, QuartzNet, CitriNet, Conformer + * `Speech Classification `_: MatchboxNet (command recognition), MarbleNet (voice activity detection) + * `Speaker Recognition `_: SpeakerNet + * `Speaker Diarization `_: MarbleNet + SpeakerNet + * `NGC collection of pre-trained speech processing models. `_ +* Natural Language Processing + * `Compatible with Hugging Face Transformers and NVIDIA Megatron `_ + * `Neural Machine Translation (NMT) `_ + * `Punctuation and Capitalization `_ + * `Token classification (named entity recognition) `_ + * `Text classification `_ + * `Joint Intent and Slot Classification `_ + * `BERT pre-training `_ + * `Question answering `_ + * `GLUE benchmark `_ + * `Information retrieval `_ + * `NGC collection of pre-trained NLP models. `_ +* `Speech synthesis (TTS) `_ + * Spectrogram generation: Tacotron2, GlowTTS, FastSpeech2, FastPitch, + * Vocoders: WaveGlow, SqueezeWave, UniGlow, MelGAN, HiFiGAN + * End-to-end speech generation: FastPitch_HifiGan_E2E, FastSpeech2_HifiGan_E2E + * `NGC collection of pre-trained TTS models. `_ + + +Built for speed, NeMo can utilize NVIDIA's Tensor Cores and scale out training to multiple GPUs and multiple nodes. + +Requirements +------------ + +1) Python 3.6, 3.7 or 3.8 +2) Pytorch 1.8.1 or above +3) NVIDIA GPU for training + +Documentation +------------- + +.. |main| image:: https://readthedocs.com/projects/nvidia-nemo/badge/?version=main + :alt: Documentation Status + :scale: 100% + :target: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/ + +.. |v1.0.2| image:: https://readthedocs.com/projects/nvidia-nemo/badge/?version=v1.0.2 + :alt: Documentation Status + :scale: 100% + :target: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/v1.0.2/ + +.. |stable| image:: https://readthedocs.com/projects/nvidia-nemo/badge/?version=stable + :alt: Documentation Status + :scale: 100% + :target: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/ + ++---------+-------------+----------------------------------------------------------------------------------------------------------------------------------+ +| Version | Status | Description | ++=========+=============+==================================================================================================================================+ +| Latest | |main| | `Documentation of the latest (i.e. main) branch. `_ | ++---------+-------------+----------------------------------------------------------------------------------------------------------------------------------+ +| Next | |v1.0.2| | `Documentation of the most recent release: v1.0.2 `_ | ++---------+-------------+----------------------------------------------------------------------------------------------------------------------------------+ +| Stable | |stable| | `Documentation of the stable (i.e. stable) branch. `_ | ++---------+-------------+----------------------------------------------------------------------------------------------------------------------------------+ + +Tutorials +--------- +A great way to start with NeMo is by checking `one of our tutorials `_. + +Getting help with NeMo +---------------------- +FAQ can be found on NeMo's `Discussions board `_. You are welcome to ask questions or start discussions there. + + +Installation +------------ + +Pip +~~~ +Use this installation mode if you want the latest released version. + +.. code-block:: bash + + apt-get update && apt-get install -y libsndfile1 ffmpeg + pip install Cython + pip install nemo_toolkit['all'] + +Pip from source +~~~~~~~~~~~~~~~ +Use this installation mode if you want the a version from particular GitHub branch (e.g main). + +.. code-block:: bash + + apt-get update && apt-get install -y libsndfile1 ffmpeg + pip install Cython + python -m pip install git+https://github.com/NVIDIA/NeMo.git@{BRANCH}#egg=nemo_toolkit[all] + + +From source +~~~~~~~~~~~ +Use this installation mode if you are contributing to NeMo. + +.. code-block:: bash + + apt-get update && apt-get install -y libsndfile1 ffmpeg + git clone https://github.com/NVIDIA/NeMo + cd NeMo + ./reinstall.sh + +Docker containers: +~~~~~~~~~~~~~~~~~~ + +If you chose to work with main branch, we recommend using NVIDIA's PyTorch container version 21.05-py3 and then installing from GitHub. + +.. code-block:: bash + + docker run --gpus all -it --rm -v :/NeMo --shm-size=8g \ + -p 8888:8888 -p 6006:6006 --ulimit memlock=-1 --ulimit \ + stack=67108864 --device=/dev/snd nvcr.io/nvidia/pytorch:21.05-py3 + +Examples +-------- + +Many example can be found under `"Examples" `_ folder. + + +Contributing +------------ + +We welcome community contributions! Please refer to the `CONTRIBUTING.md `_ CONTRIBUTING.md for the process. + +License +------- +NeMo is under `Apache 2.0 license `_. diff --git a/models/tacotron2/NeMo/docs/.nojekyll b/models/tacotron2/NeMo/docs/.nojekyll new file mode 100644 index 0000000000..e69de29bb2 diff --git a/models/tacotron2/NeMo/docs/Makefile b/models/tacotron2/NeMo/docs/Makefile new file mode 100644 index 0000000000..417fe2a0b1 --- /dev/null +++ b/models/tacotron2/NeMo/docs/Makefile @@ -0,0 +1,216 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/OpenSeq2Seq.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/OpenSeq2Seq.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/OpenSeq2Seq" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/OpenSeq2Seq" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/models/tacotron2/NeMo/docs/source/_static/css/custom.css b/models/tacotron2/NeMo/docs/source/_static/css/custom.css new file mode 100644 index 0000000000..f168711c29 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/_static/css/custom.css @@ -0,0 +1,199 @@ + +/* Width of template */ + +.wy-nav-content { + max-width: 1200px !important; +} + + + +/* Standard Text Formatting */ + +h1 +{ + color: #76b900; + text-align: center; + background-color: #333333; +} + +h2 +{ + color: #ffffff; + background-color: #76b900; + Padding: 5px; +} + +h3 +{ + padding-top: 0px; + border-top: solid 3px #76b900; + border-bottom: solid 3px #76b900; +} + +p +{ + margin-bottom: 24px; +} +/* Link Colors */ +a +{ + color: #76b900; +} + +a:visited +{ + color: #218219; +} + + +/* ----------------------------------------------TABLES--------------------------------------- */ + +/* Table head Color */ +thead td +{ + background-color: #333333 !important; +} + +.row-odd p +{ + /*padding-bottom: 0px;*/ + /*margin-bottom: 0px;*/ +} +/* even rows*/ + +.row-even tr +{ + background-color: #e5f1e6 !important; +} + +/* odd rows*/ + + +.wy-table-responsive table tr { + background-color: #ffffff !important; +} + + + +.wy-table-responsive table td { + white-space: normal; +} + + +/* Removes bottom margin in tables*/ + +.rst-content .line-block { + margin-bottom: 0px; +} + +.wy-table-responsive { + overflow: visible !important; +} + +/* reduces the size of text in multiline table columns. */ + +.rst-content table.docutils td +{ + font-size: 80%; +} + +.rst-content dl:not(.docutils) dt +{ + + background-color: inherit; + color: #000000; + border-top: solid 0px #000000; + +} + +.rst-content dl:not(.docutils) dt:before { + color: #333333; +} + +.rst-content .line-block { + margin-bottom: 0px; +} + +.wy-side-nav-search, .wy-nav-top + { + background-color: #000000; + padding: 0; + } + +.wy-side-nav-search img + { + padding: 0px; + padding: 0px 0px; + margin-bottom: 0; + } + +.wy-side-nav-search input[type=text] + { + border-radius: 0px; + } + + +.wy-menu-vertical p.caption + { + color: #76b900; + } + + +.wy-side-nav-search>a img.logo, .wy-side-nav-search .wy-dropdown>a img.logo + { + margin: 0px 0px 0px 0px; + } + +.wy-nav-content + { + margin: 0; + min-height: 100%; + height: 100%; + background: #ffffff; + } + + /* List (numbered, bulleted) padding Fix */ + + +.wy-plain-list-decimal li +{ + margin-top: -6px; + margin-bottom: -6px; +} + +.rst-content .section ol.loweralpha +{ + margin-top: -6px; + margin-bottom: 12px; +} + +.wy-plain-list-disc, .rst-content .toctree-wrapper ul, article ul +{ + margin-top: 0px !important; + margin-bottom: 12px; +} + + /* Alert Boxes */ + /* Background color of Alert Box Title */ + +.rst-content .section ul +{ + margin-top: -12px; + margin-bottom: 16px; +} + +.wy-alert.wy-alert-info .wy-alert-title, .rst-content .note .wy-alert-title, .rst-content .wy-alert-info.attention .wy-alert-title, .rst-content .wy-alert-info.caution .wy-alert-title, .rst-content .wy-alert-info.danger .wy-alert-title, .rst-content .wy-alert-info.error .wy-alert-title, .rst-content .wy-alert-info.hint .wy-alert-title, .rst-content .wy-alert-info.important .wy-alert-title, .rst-content .wy-alert-info.tip .wy-alert-title, .rst-content .wy-alert-info.warning .wy-alert-title, .rst-content .seealso .wy-alert-title, .rst-content .wy-alert-info.admonition-todo .wy-alert-title, .rst-content .wy-alert-info.admonition .wy-alert-title, .wy-alert.wy-alert-info .rst-content .admonition-title, .rst-content .wy-alert.wy-alert-info .admonition-title, .rst-content .note .admonition-title, .rst-content .wy-alert-info.attention .admonition-title, .rst-content .wy-alert-info.caution .admonition-title, .rst-content .wy-alert-info.danger .admonition-title, .rst-content .wy-alert-info.error .admonition-title, .rst-content .wy-alert-info.hint .admonition-title, .rst-content .wy-alert-info.important .admonition-title, .rst-content .wy-alert-info.tip .admonition-title, .rst-content .wy-alert-info.warning .admonition-title, .rst-content .seealso .admonition-title, .rst-content .wy-alert-info.admonition-todo .admonition-title, .rst-content .wy-alert-info.admonition .admonition-title + { + background: #76b900; + } + + /* Background and Font Color of Alert Box Main Body*/ +.wy-alert.wy-alert-info, .rst-content .note, .rst-content .wy-alert-info.attention, .rst-content .wy-alert-info.caution, .rst-content .wy-alert-info.danger, .rst-content .wy-alert-info.error, .rst-content .wy-alert-info.hint, .rst-content .wy-alert-info.important, .rst-content .wy-alert-info.tip, .rst-content .wy-alert-info.warning, .rst-content .seealso, .rst-content .wy-alert-info.admonition-todo, .rst-content .wy-alert-info.admonition { + background: #333333; + color: #999999; + } + +.section +{ + margin-top: 50px; +} diff --git a/models/tacotron2/NeMo/docs/source/_static/js/pk_scripts.js b/models/tacotron2/NeMo/docs/source/_static/js/pk_scripts.js new file mode 100644 index 0000000000..23c74982d3 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/_static/js/pk_scripts.js @@ -0,0 +1,19 @@ +document.addEventListener("DOMContentLoaded", function () { + var params = window.location.search.substring(1).split("&").reduce(function (params, param) { + if (!param) { + return params; + } + + var values = param.split("="); + var name = values[0]; + var value = values[1]; + params[name] = value; + return params; + }, {}); + + var form = document.getElementById("feedback-form"); + for (var name in params) { + var input = form.querySelector("[name=" + name + "]"); + input.value = params[name]; + } +}); \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/asr/api.rst b/models/tacotron2/NeMo/docs/source/asr/api.rst new file mode 100644 index 0000000000..174e0530d9 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/api.rst @@ -0,0 +1,203 @@ +NeMo ASR collection API +======================= + + +Model Classes +------------- + +.. autoclass:: nemo.collections.asr.models.EncDecCTCModel + :show-inheritance: + :members: transcribe, change_vocabulary, setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact + + +.. autoclass:: nemo.collections.asr.models.EncDecCTCModelBPE + :show-inheritance: + :members: transcribe, change_vocabulary, setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact + + +.. autoclass:: nemo.collections.asr.models.EncDecRNNTModel + :show-inheritance: + :members: transcribe, change_vocabulary, setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact + + +.. autoclass:: nemo.collections.asr.models.EncDecRNNTBPEModel + :show-inheritance: + :members: transcribe, change_vocabulary, setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact + + +.. autoclass:: nemo.collections.asr.models.EncDecClassificationModel + :show-inheritance: + :members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact + + +.. autoclass:: nemo.collections.asr.models.EncDecSpeakerLabelModel + :show-inheritance: + :members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact + + +Modules +------- + +.. autoclass:: nemo.collections.asr.modules.ConvASREncoder + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.modules.ConvASRDecoder + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.modules.ConvASRDecoderClassification + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.modules.SpeakerDecoder + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.modules.ConformerEncoder + :show-inheritance: + :members: + + +Parts +----- + +.. autoclass:: nemo.collections.asr.parts.submodules.jasper.JasperBlock + :show-inheritance: + :members: + + +Mixins +------ + +.. autoclass:: nemo.collections.asr.parts.mixins.mixins.ASRBPEMixin + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.mixins.mixins.ASRModuleMixin + :show-inheritance: + :members: + +Datasets +-------- + +Character Encoding Datasets +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: nemo.collections.asr.data.audio_to_text.AudioToCharDataset + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.data.audio_to_text.TarredAudioToCharDataset + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset + :show-inheritance: + :members: + +Subword Encoding Datasets +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: nemo.collections.asr.data.audio_to_text.AudioToBPEDataset + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.data.audio_to_text.TarredAudioToBPEDataset + :show-inheritance: + :members: + +Audio Preprocessors +------------------- + +.. autoclass:: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.modules.AudioToMFCCPreprocessor + :show-inheritance: + :members: + +Audio Augmentors +---------------- + +.. autoclass:: nemo.collections.asr.modules.SpectrogramAugmentation + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.modules.CropOrPadSpectrogramAugmentation + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.SpeedPerturbation + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.TimeStretchPerturbation + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.GainPerturbation + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.ImpulsePerturbation + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.ShiftPerturbation + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.NoisePerturbation + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.WhiteNoisePerturbation + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.RirAndNoisePerturbation + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.TranscodePerturbation + :show-inheritance: + :members: + +Miscellaneous Classes +--------------------- + +RNNT Decoding +~~~~~~~~~~~~~ + +.. autoclass:: nemo.collections.asr.metrics.rnnt_wer.RNNTDecoding + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.metrics.rnnt_wer_bpe.RNNTBPEDecoding + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.submodules.rnnt_greedy_decoding.GreedyRNNTInfer + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.submodules.rnnt_greedy_decoding.GreedyBatchedRNNTInfer + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.parts.submodules.rnnt_beam_decoding.BeamRNNTInfer + :show-inheritance: + :members: + +Hypotheses +~~~~~~~~~~ + +.. autoclass:: nemo.collections.asr.parts.utils.rnnt_utils.Hypothesis + :show-inheritance: + :no-members: + +.. autoclass:: nemo.collections.asr.parts.utils.rnnt_utils.NBestHypotheses + :show-inheritance: + :no-members: diff --git a/models/tacotron2/NeMo/docs/source/asr/asr_all.bib b/models/tacotron2/NeMo/docs/source/asr/asr_all.bib new file mode 100644 index 0000000000..7008355c79 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/asr_all.bib @@ -0,0 +1,980 @@ +@article{matchboxnet, + title={{MatchboxNet}: 1D Time-Channel Separable Convolutional Neural Network Architecture for Speech Commands Recognition}, + author={Majumdar, Somshubra and Ginsburg, Boris}, + journal={Proc. Interspeech 2020}, + year={2020} +} + +@article{marblenet, + title={MarbleNet: Deep 1D Time-Channel Separable Convolutional Neural Network for Voice Activity Detection}, + author={Jia, Fei and Majumdar, Somshubra and Ginsburg, Boris}, + journal={arXiv preprint arXiv:2010.13886}, + year={2020} +} + +@inproceedings{panayotov2015librispeech, + title={Librispeech: an ASR corpus based on public domain audio books}, + author={Panayotov, Vassil and Chen, Guoguo and Povey, Daniel and Khudanpur, Sanjeev}, + booktitle={Acoustics, Speech and Signal Processing (ICASSP), 2015 IEEE International Conference on}, + pages={5206--5210}, + year={2015}, + organization={IEEE} +} + +@article{luong17, + author = {Minh{-}Thang Luong and Eugene Brevdo and Rui Zhao}, + title = {Neural Machine Translation (seq2seq) Tutorial}, + journal = {https://github.com/tensorflow/nmt}, + year = {2017}, +} + +@INPROCEEDINGS{LaurentSeqWiseBN, +author={C. {Laurent} and G. {Pereyra} and P. {Brakel} and Y. {Zhang} and Y. {Bengio}}, +booktitle={2016 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, +title={Batch normalized recurrent neural networks}, +year={2016}, +volume={}, +number={}, +pages={2657-2661}, +keywords={feedforward neural nets;learning (artificial intelligence);recurrent neural nets;speech recognition;batch normalized recurrent neural networks;RNN;sequential data;long-term dependency learning;convergence rate improvement;intermediate representation normalization;feedforward neural networks;speech recognition task;language modeling;training criterion;Training;Recurrent neural networks;Convergence;Speech recognition;Computer architecture;Speech;batch normalization;RNN;LSTM;optimization}, +doi={10.1109/ICASSP.2016.7472159}, +ISSN={2379-190X}, +month={March},} + +@article{graves2005, + author = {Alex Graves and Jurgen Schmidhuber}, + title = {Framewise phoneme classification with bidirectional LSTM and other neural network architectures}, + journal = {Neural Networks, vol. 18}, + pages={602–-610}, + year = {2005}, +} + +@inproceedings{graves2006, + title={Connectionist temporal classification: labelling unsegmented sequence data with recurrent neural networks}, + author={Graves, Alex and Fern{\'a}ndez, Santiago and Gomez, Faustino and Schmidhuber, J{\"u}rgen}, + booktitle={Proceedings of the 23rd international conference on Machine learning}, + pages={369--376}, + year={2006}, + organization={ACM} +} + +@article{li2019jasper, + title={Jasper: An End-to-End Convolutional Neural Acoustic Model}, + author={Li, Jason and Lavrukhin, Vitaly and Ginsburg, Boris and Leary, Ryan and Kuchaiev, Oleksii and Cohen, Jonathan M and Nguyen, Huyen and Gadde, Ravi Teja}, + journal={arXiv preprint arXiv:1904.03288}, + year={2019} +} + +@misc{ardila2019common, + title={Common Voice: A Massively-Multilingual Speech Corpus}, + author={Rosana Ardila and Megan Branson and Kelly Davis and Michael Henretty and Michael Kohler and Josh Meyer and Reuben Morais and Lindsay Saunders and Francis M. Tyers and Gregor Weber}, + year={2019}, + eprint={1912.06670}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} + +@article{graves2012, + title={Sequence Transduction with Recurrent Neural Networks}, + author={Graves, Alex}, + journal={arXiv preprint arXiv:1211.3711}, + year={2012} +} + + +@article{graves2013, + title={Generating sequences with recurrent neural networks}, + author={Graves, Alex}, + journal={arXiv preprint arXiv:1308.0850}, + year={2013} +} + +@article{sergeev2018horovod, + title={Horovod: fast and easy distributed deep learning in TensorFlow}, + author={Sergeev, Alexander and Del Balso, Mike}, + journal={arXiv preprint arXiv:1802.05799}, + year={2018} +} + +@misc{NVVolta, + title = {NVIDIA TESLA V100 GPU ARCHITECTURE}, + howpublished = {\url{http://images.nvidia.com/content/volta-architecture/pdf/volta-architecture-whitepaper.pdf}}, + note = {Accessed: 2018-10-09} +} + +@article{NVTuring, + title = {NVIDIA TURING GPU ARCHITECTURE}, + howpublished = {\url{https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/technologies/turing-architecture/NVIDIA-Turing-Architecture-Whitepaper.pdf}}, + author = {NVIDIA}, + year = {2018}, + note = {Accessed: 2018-10-09} +} + +@misc{Rygaard2015, + title = {Using Synthesized Speech to Improve Speech Recognition for Low-Resource Languages}, + author = {Luise Valentin Rygaard}, + howpublished = {\url{https://parasol.tamu.edu/dreu2015/Rygaard/report.pdf}}, + year = {2015}, +} + +@misc{OpenSeq2Seq, + title = {OpenSeq2Seq: extensible toolkit for distributed and mixed precision training of sequence-to-sequence models}, + author = {Kuchaiev, Oleksii and Ginsburg, Boris and Gitman, Igor and Lavrukhin,Vitaly and Case, Carl and Micikevicius, Paulius}, + howpublished = {\url{https://arxiv.org/abs/1805.10387}}, + year = {2018}, +} + +@misc{MPGuide, + title = {Training with Mixed Precision}, + howpublished = {\url{http://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/}}, + note = {Accessed: 2018-04-06}, +} + +@misc{Mozilla, + title = {Mozilla: A Journey to less than 10\% Word Error Rate}, + howpublished = {\url{https://hacks.mozilla.org/2017/11/a-journey-to-10-word-error-rate/}}, + note = {Accessed: 2018-04-06}, +} + +@article{Waibel1989, + title={A time-delay neural network architecture for isolated word recognition}, + author={Waibel, Alexander, and Hanazawa, Toshiyki and Hinton,Geoffrey and Shirano, Kiyohiro and Lang, Kevin }, + journal={IEEE Trans. on Acoustics, Speech and Signal Processing}, + year={1989} +} + +@article{Lang1990, + title={A time-delay neural network architecture for isolated word recognition}, + author={Lang, Kevin and Waibel, Alexander, and Hinton,Geoffrey }, + journal={Neural Networks}, + year={1990} +} + +@book{Bengio1996, + Author = {Bengio, Y.}, + Publisher = {International Thomson Computer Press}, + Title = {Neural Networks for Speech and Sequence Recognition}, + Year = {1996} +} + +@article{Bengio1992, + title={Global optimization of a neural network-hidden Markov model hybrid}, + author={Bengio, Y., and De Mori, R., and Flammia, G., and Kompe, R. }, + journal={IEEE Transactions on Neural Networks, 3(2), 252–259}, + year={1992} +} + +@article{Bourlard1994, + title={Connectionist speech recognition: a hybrid approach}, + author={Bourlard, H. A. and Morgan, N.}, + journal={volume 247 Springer }, + year={1994} +} + +@article{srivastava14a, + author = {Nitish Srivastava, and Geoffrey Hinton, and Alex Krizhevsky, and Ilya Sutskever, and Ruslan Salakhutdinov}, + title = {Dropout: A Simple Way to Prevent Neural Networks from Overfitting}, + journal = {Journal of Machine Learning Research}, + year = {2014}, + volume = {15}, + pages = {1929-1958}, + url = {http://jmlr.org/papers/v15/srivastava14a.html} +} + + +@article{Hinton2012, + title={Deep Neural Networks for Acoustic Modeling in Speech Recognition}, + author={ Hinton,Geoffrey and Deng, Li and Yu, Dong and Dahl,George and Mohamed,Abdel-rahman and Jaitly, Navdeep and Senior, Andrew and Vanhoucke, Vincent and Nguyen, Patrick and Kingsbury, Brian and Sainath, Tara}, + journal={IEEE Signal Processing Magazine}, + year={2012} +} + +@article{Graves2014, + title={Towards End-to-End Speech Recognition with Recurrent Neural Networks}, + author={Graves, Alex and Jaitly, Navdeep}, + journal={International Conference on Machine Learning}, + year={2014} +} + +@article{Chorowski2014, + title={End-to-end Continuous Speech Recognition using Attention-based Recurrent NN: First Results}, + author={ Chorowski, Jan, and Bahdanau, Dzmitry , and Cho, Kyunghyun , and Bengio, Yoshua }, + journal={Neural Information Processing Systems: Workshop Deep Learning and Representation Learning Workshop }, + year={2014} +} + +@article{Sak2014, + title={Long short-term memory recurrent neural network architectures for large scale acoustic modeling}, + author={Sak, Hasim and Senior, Andrew and Beaufays, Francoise }, + journal={Interspeech 2014}, + year={2014} +} + +@article{Ko2015, + title={Audio Augmentation for Speech Recognition}, + author={Tom, Ko and Vijayaditya, Peddinti and Daniel, Povey + and Sanjeev, Khudanpur }, + journal={Interspeech 2015}, + year={2015} +} + +@article{Tjandra2017, + title={Listening while Speaking: Speech Chain by Deep Learning}, + author={Andros, Tjandra and Sakriani, Sakti and Satoshi, Nakamura }, + journal={ASRU 2017}, + year={2017} +} + +@article{Tjandra2018, + title={Machine Speech Chain with One-shot Speaker Adaptation}, + author={Andros, Tjandra and Sakriani, Sakti and Satoshi, Nakamura }, + journal={Interspeech 2018}, + year={2018} +} + +@article{bahdanau2014neural, + title={Neural machine translation by jointly learning to align and translate}, + author={Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua}, + journal={arXiv preprint arXiv:1409.0473}, + year={2014} +} + +@article{cho2014learning, + title={Learning phrase representations using RNN encoder-decoder for statistical machine translation}, + author={Cho, Kyunghyun and Van Merri{\"e}nboer, Bart and Gulcehre, Caglar and Bahdanau, Dzmitry and Bougares, Fethi and Schwenk, Holger and Bengio, Yoshua}, + journal={arXiv preprint arXiv:1406.1078}, + year={2014} +} + +@article{rush2015neural, + title={A neural attention model for abstractive sentence summarization}, + author={Rush, Alexander M and Chopra, Sumit and Weston, Jason}, + journal={arXiv preprint arXiv:1509.00685}, + year={2015} +} + +@article{micikevicius2017mixed, + title={Mixed precision training}, + author={Micikevicius, Paulius and Narang, Sharan and Alben, Jonah and Diamos, Gregory and Elsen, Erich and Garcia, David and Ginsburg, Boris and Houston, Michael and Kuchaev, Oleksii and Venkatesh, Ganesh and others}, + journal={arXiv preprint arXiv:1710.03740}, + year={2017} +} + +@ARTICLE{Britz:2017, + author = {{Britz}, Denny and {Goldie}, Anna and {Luong}, Thang and {Le}, Quoc}, + title = "{Massive Exploration of Neural Machine Translation Architectures}", + journal = {ArXiv e-prints arXiv:1703.03906}, + archivePrefix = "arXiv", + eprinttype = {arxiv}, + eprint = {1703.03906}, + primaryClass = "cs.CL", + keywords = {Computer Science - Computation and Language}, + year = 2017, + month = mar, +} + +@inproceedings{vaswani2017attention, + title={Attention is all you need}, + author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia}, + booktitle={Advances in Neural Information Processing Systems}, + pages={6000--6010}, + year={2017} +} + +@inproceedings{abadi2016tensorflow, + title={TensorFlow: A System for Large-Scale Machine Learning.}, + author={Abadi, Mart{\'\i}n and Barham, Paul and Chen, Jianmin and Chen, Zhifeng and Davis, Andy and Dean, Jeffrey and Devin, Matthieu and Ghemawat, Sanjay and Irving, Geoffrey and Isard, Michael and others}, + booktitle={OSDI}, + volume={16}, + pages={265--283}, + year={2016} +} + +@article{tensor2tensor, + author = {Ashish Vaswani and Samy Bengio and Eugene Brevdo and Francois Chollet and Aidan N. Gomez and Stephan Gouws and Llion Jones and \L{}ukasz Kaiser and Nal Kalchbrenner and Niki Parmar and Ryan Sepassi and + Noam Shazeer and Jakob Uszkoreit}, + title = {Tensor2Tensor for Neural Machine Translation}, + journal = {CoRR}, + volume = {abs/1803.07416}, + year = {2018}, + url = {http://arxiv.org/abs/1803.07416}, +} + +@article{gehring2017convs2s, + author = {Gehring, Jonas, and Auli, Michael and Grangier, David and Yarats, Denis and Dauphin, Yann N}, + title = "{Convolutional Sequence to Sequence Learning}", + journal = {ArXiv e-prints arXiv:1705.03122}, + archivePrefix = "arXiv", + eprinttype = {arxiv}, + eprint = {1705.03122}, + primaryClass = "cs.CL", + keywords = {Computer Science - Computation and Language}, + year = 2017, + month = May, +} + +@inproceedings{chan2015, + title={Listen, attend and spell}, + author={Chan, William and Jaitly, Navdeep and Le, Quoc V and Vinyals, Oriol}, + booktitle={Acoustics, Speech and Signal Processing (ICASSP), 2016 IEEE International Conference on}, + pages={5206--5210}, + year={2016}, + organization={IEEE} +} + +@inproceedings{xu2015show, + title={Show, attend and tell: Neural image caption generation with visual attention}, + author={Xu, Kelvin and Ba, Jimmy and Kiros, Ryan and Cho, Kyunghyun and Courville, Aaron and Salakhudinov, Ruslan and Zemel, Rich and Bengio, Yoshua}, + booktitle={International Conference on Machine Learning}, + pages={2048--2057}, + year={2015} +} + +@incollection{Sutskever2014, + title = {Sequence to Sequence Learning with Neural Networks}, + author = {Sutskever, Ilya and Vinyals, Oriol and Le, Quoc V}, + booktitle = {Advances in Neural Information Processing Systems 27}, + editor = {Z. Ghahramani and M. Welling and C. Cortes and N. D. Lawrence and K. Q. Weinberger}, + pages = {3104--3112}, + year = {2014}, + publisher = {Curran Associates, Inc.}, + url = {http://papers.nips.cc/paper/5346-sequence-to-sequence-learning-with-neural-networks.pdf} +} + +@article{DeepSpeech2014, + title = {Deep Speech: Scaling up end-to-end speech recognition}, + author = {Awni Y. Hannun and Carl Case and Jared Casper and Bryan Catanzaro and Greg Diamos and Erich Elsen and Ryan Prenger and Sanjeev Satheesh and Shubho Sengupta and Adam Coates and Andrew Y. Ng}, + journal = {CoRR}, + volume = {abs/1412.5567}, + year = {2014}, + url = {http://arxiv.org/abs/1412.5567}, + archivePrefix = {arXiv}, + eprint = {1412.5567}, + timestamp = {Mon, 13 Aug 2018 16:48:07 +0200}, + biburl = {https://dblp.org/rec/bib/journals/corr/HannunCCCDEPSSCN14}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@inproceedings{DeepSpeech2, + author = {Amodei, Dario and Ananthanarayanan, Sundaram and Anubhai, Rishita and Bai, Jingliang and Battenberg, Eric and Case, Carl and Casper, Jared and Catanzaro, Bryan and Cheng, Qiang and Chen, Guoliang and Chen, Jie and Chen, Jingdong and Chen, Zhijie and Chrzanowski, Mike and Coates, Adam and Diamos, Greg and Ding, Ke and Du, Niandong and Elsen, Erich and Engel, Jesse and Fang, Weiwei and Fan, Linxi and Fougner, Christopher and Gao, Liang and Gong, Caixia and Hannun, Awni and Han, Tony and Johannes, Lappi Vaino and Jiang, Bing and Ju, Cai and Jun, Billy and LeGresley, Patrick and Lin, Libby and Liu, Junjie and Liu, Yang and Li, Weigao and Li, Xiangang and Ma, Dongpeng and Narang, Sharan and Ng, Andrew and Ozair, Sherjil and Peng, Yiping and Prenger, Ryan and Qian, Sheng and Quan, Zongfeng and Raiman, Jonathan and Rao, Vinay and Satheesh, Sanjeev and Seetapun, David and Sengupta, Shubho and Srinet, Kavya and Sriram, Anuroop and Tang, Haiyuan and Tang, Liliang and Wang, Chong and Wang, Jidong and Wang, Kaifu and Wang, Yi and Wang, Zhijian and Wang, Zhiqian and Wu, Shuang and Wei, Likai and Xiao, Bo and Xie, Wen and Xie, Yan and Yogatama, Dani and Yuan, Bin and Zhan, Jun and Zhu, Zhenyao}, + title = {Deep Speech 2: End-to-end Speech Recognition in English and Mandarin}, + booktitle = {Proceedings of the 33rd International Conference on International Conference on Machine Learning - Volume 48}, + series = {ICML'16}, + year = {2016}, + location = {New York, NY, USA}, + pages = {173--182}, + numpages = {10}, + url = {http://dl.acm.org/citation.cfm?id=3045390.3045410}, + acmid = {3045410}, + publisher = {JMLR.org}, +} + +@inproceedings{prabhavalkar2017comparison, + title={A comparison of sequence-to-sequence models for speech recognition}, + author={Prabhavalkar, Rohit and Rao, Kanishka and Sainath, Tara N and Li, Bo and Johnson, Leif and Jaitly, Navdeep}, + booktitle={Proc. Interspeech}, + pages={939--943}, + year={2017} +} + +@article{chiu2017state, + title={State-of-the-art speech recognition with sequence-to-sequence models}, + author={Chiu, Chung-Cheng and Sainath, Tara N and Wu, Yonghui and Prabhavalkar, Rohit and Nguyen, Patrick and Chen, Zhifeng and Kannan, Anjuli and Weiss, Ron J and Rao, Kanishka and Gonina, Katya and others}, + journal={arXiv preprint arXiv:1712.01769}, + year={2017} +} + +@misc{NVMixed, + title = {{NVIDA's Mixed-Precision Training - TensorFlow example}}, + howpublished = {\url{https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/#example_tensorflow}}, + author={NVIDIA}, + note = {Accessed: 2018-10-09}, + year={2018} +} + +@article{gehring2017, + title={Convolutional sequence to sequence learning}, + author={Gehring, Jonas and Auli, Michael and Grangier, David and Yarats, Denis and Dauphin, Yann N}, + journal={arXiv preprint arXiv:1705.03122}, + year={2017} +} + +@article{collobert2016, + title={Wav2letter: an end-to-end convnet-based speech recognition system}, + author={Collobert, Ronan and Puhrsch, Christian and Synnaeve, Gabriel}, + journal={arXiv preprint arXiv:1609.03193}, + year={2016} +} + +@inproceedings{Zhang2016, +author={Ying Zhang and Mohammad Pezeshki and Philémon Brakel and Saizheng Zhang and César Laurent and Yoshua Bengio and Aaron Courville}, +title={Towards End-to-End Speech Recognition with Deep Convolutional Neural Networks}, +year=2016, +booktitle={Interspeech 2016}, +doi={10.21437/Interspeech.2016-1446}, +url={http://dx.doi.org/10.21437/Interspeech.2016-1446}, +pages={410--414} +} + +@inproceedings{Zhang2017, + title={Very deep convolutional networks for end-to-end speech recognition}, + author={Zhang, Yu, and Chan, William, and Jaitly, Navdeep}, + booktitle={Acoustics, Speech and Signal Processing (ICASSP), 2017 IEEE International Conference on}, + year={2017}, + organization={IEEE} +} + + +@article{Wang2017, + title={Tacotron: Towards End-to-End Speech Synthesis}, + author={ Wang, Yuxuan, and Skerry-Ryan, RJ, and Stanton, Daisy and Wu, Yonghui and Weiss, Ron, and Jaitly, Navdeep and Yang, Zongheng and Xiao, Ying and Chen,Zhifeng and Bengio, Samy and Le, Quoc and Agiomyrgiannakis, Yannis and Clark,Rob and Saurous, Rif A.}, + journal={arXiv preprint arXiv:1703.10135}, + year={2017} +} + +@inproceedings{shen2018natural, + title={Natural TTS synthesis by conditioning wavenet on mel spectrogram predictions}, + author={Shen, Jonathan and Pang, Ruoming and Weiss, Ron J and Schuster, Mike and Jaitly, Navdeep and Yang, Zongheng and Chen, Zhifeng and Zhang, Yu and Wang, Yuxuan and Skerrv-Ryan, Rj and others}, + booktitle={2018 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, + pages={4779--4783}, + year={2018}, + organization={IEEE} +} + +@article{griffin1984signal, + title={Signal estimation from modified short-time Fourier transform}, + author={Griffin, Daniel and Lim, Jae}, + journal={IEEE Transactions on Acoustics, Speech, and Signal Processing}, + volume={32}, + number={2}, + pages={236--243}, + year={1984}, + publisher={IEEE} +} + +@misc{ito2017lj, + title={The LJ speech dataset}, + author={Ito, Keith and others}, + year={2017} +} + +@misc{mailabs, + title = {{The M-AILABS Speech Dataset}}, + howpublished = {\url{http://www.m-ailabs.bayern/en/the-mailabs-speech-dataset/}}, + author={M-AILABS}, + note = {Accessed: 2018-10-09}, + year={2018} +} + +@article{merity2016pointer, + title={Pointer sentinel mixture models}, + author={Merity, Stephen and Xiong, Caiming and Bradbury, James and Socher, Richard}, + journal={arXiv preprint arXiv:1609.07843}, + year={2016} +} + +@inproceedings{socher2013recursive, + title={Recursive deep models for semantic compositionality over a sentiment treebank}, + author={Socher, Richard and Perelygin, Alex and Wu, Jean and Chuang, Jason and Manning, Christopher D and Ng, Andrew and Potts, Christopher}, + booktitle={Proceedings of the 2013 conference on empirical methods in natural language processing}, + pages={1631--1642}, + year={2013} +} + +@InProceedings{maas-EtAl:2011:ACL-HLT2011, + author = {Maas, Andrew L. and Daly, Raymond E. and Pham, Peter T. and Huang, Dan and Ng, Andrew Y. and Potts, Christopher}, + title = {Learning Word Vectors for Sentiment Analysis}, + booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies}, + month = {June}, + year = {2011}, + address = {Portland, Oregon, USA}, + publisher = {Association for Computational Linguistics}, + pages = {142--150}, + url = {http://www.aclweb.org/anthology/P11-1015} +} + +@inproceedings{Povey2018SemiOrthogonalLM, + title={Semi-Orthogonal Low-Rank Matrix Factorization for Deep Neural Networks}, + author={Daniel Povey and Gaofeng Cheng and Yiming Wang and Ke Li and Hainan Xu and Mahsa Yarmohammadi and Sanjeev Khudanpur}, + booktitle={Interspeech}, + year={2018} +} + +@article{CAPIO2017, + author = {Kyu J. Han and Akshay Chandrashekaran and Jungsuk Kim and Ian R. Lane}, + title = {The {CAPIO} 2017 Conversational Speech Recognition System}, + journal = {CoRR}, + volume = {abs/1801.00059}, + year = {2018}, + url = {http://arxiv.org/abs/1801.00059}, + archivePrefix = {arXiv}, + eprint = {1801.00059}, + timestamp = {Mon, 13 Aug 2018 16:49:10 +0200}, + biburl = {https://dblp.org/rec/bib/journals/corr/abs-1801-00059}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@article{WaveNet, + author = {A{\"{a}}ron van den Oord and Sander Dieleman and Heiga Zen and Karen Simonyan and Oriol Vinyals and Alex Graves and Nal Kalchbrenner and Andrew W. Senior and Koray Kavukcuoglu}, + title = {WaveNet: {A} Generative Model for Raw Audio}, + journal = {CoRR}, + volume = {abs/1609.03499}, + year = {2016}, + url = {http://arxiv.org/abs/1609.03499}, + archivePrefix = {arXiv}, + eprint = {1609.03499}, + timestamp = {Mon, 13 Aug 2018 16:49:15 +0200}, + biburl = {https://dblp.org/rec/bib/journals/corr/OordDZSVGKSK16}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@article{FacebookGERENGBackTranslation, + author = {Rico Sennrich and Barry Haddow and Alexandra Birch}, + title = {Improving Neural Machine Translation Models with Monolingual Data}, + journal = {CoRR}, + volume = {abs/1511.06709}, + year = {2015}, + url = {http://arxiv.org/abs/1511.06709}, + archivePrefix = {arXiv}, + eprint = {1511.06709}, + timestamp = {Mon, 13 Aug 2018 16:47:05 +0200}, + biburl = {https://dblp.org/rec/bib/journals/corr/SennrichHB15a}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@article{GlobalStyleTokens, + author = {Yuxuan Wang and Daisy Stanton and Yu Zhang and R. J. Skerry{-}Ryan and Eric Battenberg and Joel Shor and Ying Xiao and Fei Ren and Ye Jia and Rif A. Saurous}, + title = {Style Tokens: Unsupervised Style Modeling, Control and Transfer in End-to-End Speech Synthesis}, + journal = {CoRR}, + volume = {abs/1803.09017}, + year = {2018}, + url = {http://arxiv.org/abs/1803.09017}, + archivePrefix = {arXiv}, + eprint = {1803.09017}, + timestamp = {Mon, 13 Aug 2018 16:46:53 +0200}, + biburl = {https://dblp.org/rec/bib/journals/corr/abs-1803-09017}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@article{IoffeS15BatchNorm, + author = {Sergey Ioffe and Christian Szegedy}, + title = {Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift}, + journal = {CoRR}, + volume = {abs/1502.03167}, + year = {2015}, + url = {http://arxiv.org/abs/1502.03167}, + archivePrefix = {arXiv}, + eprint = {1502.03167}, + timestamp = {Mon, 13 Aug 2018 16:47:06 +0200}, + biburl = {https://dblp.org/rec/bib/journals/corr/IoffeS15}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@article{kingma, + author = {Diederik P. Kingma and + Jimmy Ba}, + title = {Adam: {A} Method for Stochastic Optimization}, + journal = {CoRR}, + volume = {abs/1412.6980}, + year = {2014}, + url = {http://arxiv.org/abs/1412.6980}, + archivePrefix = {arXiv}, + eprint = {1412.6980}, + timestamp = {Mon, 13 Aug 2018 01:00:00 +0200}, + biburl = {https://dblp.org/rec/bib/journals/corr/KingmaB14}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@incollection{Salimans2016WeightNorm, + title = {Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks}, + author = {Salimans, Tim and Kingma, Durk P}, + booktitle = {Advances in Neural Information Processing Systems 29}, + editor = {D. D. Lee and M. Sugiyama and U. V. Luxburg and I. Guyon and R. Garnett}, + pages = {901--909}, + year = {2016}, + publisher = {Curran Associates, Inc.}, + url = {http://papers.nips.cc/paper/6114-weight-normalization-a-simple-reparameterization-to-accelerate-training-of-deep-neural-networks.pdf} +} + +@article{wu2016google, + title={Google's neural machine translation system: Bridging the gap between human and machine translation}, + author={Wu, Yonghui and Schuster, Mike and Chen, Zhifeng and Le, Quoc V and Norouzi, Mohammad and Macherey, Zolfgang and Krikun, Maxim and Cao, Yuan and Gao, Qin and Macherey, Klaus and others}, + journal={arXiv preprint arXiv:1609.08144}, + year={2016} +} + +@inproceedings{opennmt, + author = {Guillaume Klein and Yoon Kim and Yuntian Deng and Jean Senellart and Alexander M. Rush}, + title = {OpenNMT: Open-Source Toolkit for Neural Machine Translation}, + booktitle = {Proc. ACL}, + year = {2017}, + url = {https://doi.org/10.18653/v1/P17-4012}, + doi = {10.18653/v1/P17-4012} +} + +@article{paszke2017automatic, + title={Automatic differentiation in PyTorch}, + author={Paszke, Adam and Gross, Sam and Chintala, Soumith and Chanan, Gregory and Yang, Edward and DeVito, Zachary and Lin, Zeming and Desmaison, Alban and Antiga, Luca and Lerer, Adam}, + year={2017} +} + +@article{yu2014introduction, + title={An introduction to computational networks and the computational network toolkit}, + author={Yu, Dong and Eversole, Adam and Seltzer, Mike and Yao, Kaisheng and Huang, Zhiheng and Guenter, Brian and Kuchaiev, Oleksii and Zhang, Yu and Seide, Frank and Wang, Huaming and others}, + journal={Microsoft Technical Report MSR-TR-2014--112}, + year={2014} +} + +@article{nvidia2017v100, + title={V100 GPU architecture. The world’s most advanced data center GPU. Version WP-08608-001\_v1. 1}, + author={NVIDIA, Tesla}, + journal={NVIDIA. Aug}, + pages={108}, + year={2017} +} + +@article{post2018call, + title={A call for clarity in reporting bleu scores}, + author={Post, Matt}, + journal={arXiv preprint arXiv:1804.08771}, + year={2018} +} + +@article{Ba2016LayerNorm, + author = {Jimmy Lei Ba and Jamie Ryan Kiros and Geoffrey E Hinton}, + title = {Layer normalization}, + journal = {CoRR}, + volume = {abs/1607.06450}, + year = {2016}, + url = {http://arxiv.org/abs/1607.06450}, + archivePrefix = {arXiv}, +} + +@inproceedings{Dauphin2017GLU, + author = {Dauphin, Yann N. and Fan, Angela and Auli, Michael and Grangier, David}, + title = {Language Modeling with Gated Convolutional Networks}, + booktitle = {Proceedings of the 34th International Conference on Machine Learning - Volume 70}, + series = {ICML'17}, + year = {2017}, + location = {Sydney, NSW, Australia}, + pages = {933--941}, + numpages = {9}, + url = {http://dl.acm.org/citation.cfm?id=3305381.3305478}, + acmid = {3305478}, + publisher = {JMLR.org}, +} + +@incollection{Oord2016PixelCNN, +title = {Conditional Image Generation with PixelCNN Decoders}, +author = {van den Oord, Aaron and Kalchbrenner, Nal and Espeholt, Lasse and kavukcuoglu, koray and Vinyals, Oriol and Graves, Alex}, +booktitle = {Advances in Neural Information Processing Systems 29}, +editor = {D. D. Lee and M. Sugiyama and U. V. Luxburg and I. Guyon and R. Garnett}, +pages = {4790--4798}, +year = {2016}, +publisher = {Curran Associates, Inc.}, +url = {http://papers.nips.cc/paper/6527-conditional-image-generation-with-pixelcnn-decoders.pdf} +} + +@article{he2015, + title={Deep residual learning for image recognition}, + author={K. He, and X. Zhang, and S. Ren, and J. Sun}, + journal={arXiv preprint arXiv:1512.03385}, + year={2015} +} + +@article{huang2016, + title={Densely Connected Convolutional Networks}, + author={Gao Huang, and Zhuang Liu, and Laurens van der Maaten, and Kilian Q. Weinberger}, + journal={arXiv preprint arXiv:1608.06993}, + year={2016} +} + +@inproceedings{heafield2011kenlm, + title={KenLM: Faster and smaller language model queries}, + author={Heafield, Kenneth}, + booktitle={Proceedings of the sixth workshop on statistical machine translation}, + pages={187--197}, + year={2011}, + organization={Association for Computational Linguistics} +} + +@article{dai2018transformer, + title={Transformer-XL: Language Modeling with Longer-Term Dependency}, + author={Dai, Zihang and Yang, Zhilin and Yang, Yiming and Cohen, William W and Carbonell, Jaime and Le, Quoc V and Salakhutdinov, Ruslan}, + year={2018}, + journal = {CoRR}, + volume = {abs/1901.02860}, + url = {http://arxiv.org/abs/1901.02860}, + archivePrefix = {arXiv}, + eprint = {1901.02860}, + timestamp = {Fri, 01 Feb 2019 13:39:59 +0100}, + biburl = {https://dblp.org/rec/bib/journals/corr/abs-1901-02860}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@inproceedings{Saon+2016, +author={George Saon and Tom Sercu and Steven Rennie and Hong-Kwang J. Kuo}, +title={The IBM 2016 English Conversational Telephone Speech Recognition System}, +year=2016, +booktitle={Interspeech 2016}, +doi={10.21437/Interspeech.2016-1460}, +url={http://dx.doi.org/10.21437/Interspeech.2016-1460}, +pages={7--11} +} + +@INPROCEEDINGS{Sercu-2016, +author={T. {Sercu} and C. {Puhrsch} and B. {Kingsbury} and Y. {LeCun}}, +booktitle={2016 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, +title={Very deep multilingual convolutional neural networks for LVCSR}, +year={2016}, +volume={}, +number={}, +pages={4955-4959}, +keywords={natural language processing;neural nets;speech recognition;very deep multilingual convolutional neural networks;LVCSR;CNN;large vocabulary continuous speech recognition systems;word error rate;Training;Context;Hidden Markov models;Neural networks;Computer architecture;Kernel;Training data;Convolutional Networks;Multilingual;Acoustic Modeling;Speech Recognition;Neural Networks}, +doi={10.1109/ICASSP.2016.7472620}, +ISSN={2379-190X}, +month={March},} + + +@inproceedings{Sercu+2016, +author={Tom Sercu and Vaibhava Goel}, +title={Advances in Very Deep Convolutional Neural Networks for LVCSR}, +year=2016, +booktitle={Interspeech 2016}, +doi={10.21437/Interspeech.2016-1033}, +url={http://dx.doi.org/10.21437/Interspeech.2016-1033}, +pages={3429--3433} +} + +@INPROCEEDINGS{Xiong-2018, +author={W. {Xiong} and L. {Wu} and F. {Alleva} and J. {Droppo} and X. {Huang} and A. {Stolcke}}, +booktitle={2018 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, +title={The Microsoft 2017 Conversational Speech Recognition System}, +year={2018}, +volume={}, +number={}, +pages={5934-5938}, +keywords={convolution;feedforward neural nets;natural language processing;speaker recognition;speech processing;language model rescoring step;senone level;switchboard domains;character-based LSTM language models;NIST 2000 switchboard test set;frame level;word-level voting;acoustic model posteriors;dialog session aware LSTM language models;CNN-BLSTM acoustic model;Microsoft 2017 conversational speech recognition system;Acoustics;Error analysis;Training;Speech recognition;Switches;Computational modeling;Context modeling;Conversational speech recognition;CNN;LACE;BLSTM;LSTM-LM;system combination;human parity}, +doi={10.1109/ICASSP.2018.8461870}, +ISSN={2379-190X}, +month={April},} + +@inproceedings{zeyer2018improved, + author={Albert Zeyer and Kazuki Irie and Ralf Schlüter and Hermann Ney}, + title={Improved Training of End-to-end Attention Models for Speech Recognition}, + year=2018, + booktitle={Proc. Interspeech 2018}, + pages={7--11}, + doi={10.21437/Interspeech.2018-1616}, + url={http://dx.doi.org/10.21437/Interspeech.2018-1616} +} + +@article{Wav2LetterV2, + author = {Vitaliy Liptchinsky and + Gabriel Synnaeve and + Ronan Collobert}, + title = {Letter-Based Speech Recognition with Gated ConvNets}, + journal = {CoRR}, + volume = {abs/1712.09444}, + year = {2017}, + url = {http://arxiv.org/abs/1712.09444}, + archivePrefix = {arXiv}, + eprint = {1712.09444}, + timestamp = {Mon, 13 Aug 2018 16:46:33 +0200}, + biburl = {https://dblp.org/rec/bib/journals/corr/abs-1712-09444}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@article{zeghidour2018, + author = {Neil Zeghidour and + Qiantong Xu and + Vitaliy Liptchinsky and + Nicolas Usunier and + Gabriel Synnaeve and + Ronan Collobert}, + title = {Fully Convolutional Speech Recognition}, + journal = {CoRR}, + volume = {abs/1812.06864}, + year = {2018}, + url = {http://arxiv.org/abs/1812.06864}, + archivePrefix = {arXiv}, + eprint = {1812.06864}, + timestamp = {Tue, 01 Jan 2019 15:01:25 +0100}, + biburl = {https://dblp.org/rec/bib/journals/corr/abs-1812-06864}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@inproceedings{Hadian2018, + author={Hossein Hadian and Hossein Sameti and Daniel Povey and Sanjeev Khudanpur}, + title={End-to-end Speech Recognition Using Lattice-free MMI}, + year=2018, + booktitle={Proc. Interspeech 2018}, + pages={12--16}, + doi={10.21437/Interspeech.2018-1423}, + url={http://dx.doi.org/10.21437/Interspeech.2018-1423} +} + +@inproceedings{Tang2018, + author={Jian Tang and Yan Song and Lirong Dai and Ian McLoughlin}, + title={Acoustic Modeling with Densely Connected Residual Network for Multichannel Speech Recognition}, + year=2018, + booktitle={Proc. Interspeech 2018}, + pages={1783--1787}, + doi={10.21437/Interspeech.2018-1089}, + url={http://dx.doi.org/10.21437/Interspeech.2018-1089} +} + +@article{Kurata2017LanguageMW, + title={Language modeling with highway LSTM}, + author={Gakuto Kurata and Bhuvana Ramabhadran and George Saon and Abhinav Sethy}, + journal={2017 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)}, + year={2017}, + pages={244-251} +} + +@inproceedings{Saon2017, + author={George Saon and Gakuto Kurata and Tom Sercu and Kartik Audhkhasi and Samuel Thomas and Dimitrios Dimitriadis and Xiaodong Cui and Bhuvana Ramabhadran and Michael Picheny and Lynn-Li Lim and Bergul Roomi and Phil Hall}, + title={English Conversational Telephone Speech Recognition by Humans and Machines}, + year=2017, + booktitle={Proc. Interspeech 2017}, + pages={132--136}, + doi={10.21437/Interspeech.2017-405}, + url={http://dx.doi.org/10.21437/Interspeech.2017-405} +} + +@inproceedings{Povey+2016, +author={Daniel Povey and Vijayaditya Peddinti and Daniel Galvez and Pegah Ghahremani and Vimal Manohar and Xingyu Na and Yiming Wang and Sanjeev Khudanpur}, +title={Purely Sequence-Trained Neural Networks for ASR Based on Lattice-Free MMI}, +year=2016, +booktitle={Interspeech 2016}, +doi={10.21437/Interspeech.2016-595}, +url={http://dx.doi.org/10.21437/Interspeech.2016-595}, +pages={2751--2755} +} + +@article{Yang2018, + author = {Xuerui Yang and + Jiwei Li and + Xi Zhou}, + title = {A novel pyramidal-FSMN architecture with lattice-free {MMI} for speech + recognition}, + journal = {CoRR}, + volume = {abs/1810.11352}, + year = {2018}, + url = {http://arxiv.org/abs/1810.11352}, + archivePrefix = {arXiv}, + eprint = {1810.11352}, + timestamp = {Wed, 31 Oct 2018 14:24:29 +0100}, + biburl = {https://dblp.org/rec/bib/journals/corr/abs-1810-11352}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@article{liptchinsky2017based, + title={Letter-Based Speech Recognition with Gated ConvNets}, + author={Liptchinsky, Vitaliy and Synnaeve, Gabriel and Collobert, Ronan}, + journal={arXiv preprint arXiv:1712.09444}, + year={2017} +} + +@inproceedings{Weng2018, + author={Chao Weng and Jia Cui and Guangsen Wang and Jun Wang and Chengzhu Yu and Dan Su and Dong Yu}, + title={Improving Attention Based Sequence-to-Sequence Models for End-to-End English Conversational Speech Recognition}, + year=2018, + booktitle={Proc. Interspeech 2018}, + pages={761--765}, + doi={10.21437/Interspeech.2018-1030}, + url={http://dx.doi.org/10.21437/Interspeech.2018-1030} +} + +@INPROCEEDINGS{Battenberg2017, +author={E. {Battenberg} and J. {Chen} and R. {Child} and A. {Coates} and Y. G. Y. {Li} and H. {Liu} and S. {Satheesh} and A. {Sriram} and Z. {Zhu}}, +booktitle={2017 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)}, +title={Exploring neural transducers for end-to-end speech recognition}, +year={2017}, +volume={}, +number={}, +pages={206-213}, +keywords={recurrent neural nets;speech recognition;Hub500 benchmark;CTC models;speech recognition pipeline;RNN-Transducer models;language model;Seq2Seq models;end-to-end speech recognition;neural transducers;Decoding;Hidden Markov models;Transducers;Task analysis;Speech;Mathematical model;Neural networks}, +doi={10.1109/ASRU.2017.8268937}, +ISSN={}, +month={Dec}, +} + +@inproceedings{ +loshchilov2018, +title={Decoupled Weight Decay Regularization}, +author={Ilya Loshchilov and Frank Hutter}, +booktitle={International Conference on Learning Representations}, +year={2019}, +url={https://openreview.net/forum?id=Bkg6RiCqY7}, +} + +@article{zhang2017ndadam, + author = {Zijun Zhang and Lin Ma and Zongpeng Li and Chuan Wu}, + title = {Normalized Direction-preserving Adam}, + journal = {arXiv e-prints arXiv:1709.04546}, + year = {2017}, +} + +@article{park2019, + author = {{Park}, Daniel S. and {Chan}, William and {Zhang}, Yu and + {Chiu}, Chung-Cheng and {Zoph}, Barret and {Cubuk}, Ekin D. and + {Le}, Quoc V.}, + title = "{SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition}", + journal = {arXiv e-prints}, + year = "2019", + eid = {arXiv:1904.08779}, + eprint = {1904.08779}, +} + +@article{novograd2019, + author = {{Ginsburg}, Boris and {Castonguay}, Patrice and {Hrinchuk}, Oleksii and + {Kuchaiev}, Oleksii and {Lavrukhin}, Vitaly and {Leary}, Ryan and + {Li}, Jason and {Nguyen}, Huyen and {Cohen}, Jonathan M.}, + title = "{Stochastic Gradient Methods with Layer-wise Adaptive Moments for Training of Deep Networks}", + journal = {arXiv e-prints}, + year = "2019", + eid = {arXiv:1905.11286}, + eprint = {1905.11286}, +} + +@article{kriman2019quartznet, + title={Quartznet: {Deep} automatic speech recognition with 1d time-channel separable convolutions}, + author={Kriman, Samuel and Beliaev, Stanislav and Ginsburg, Boris and Huang, Jocelyn and Kuchaiev, Oleksii and Lavrukhin, Vitaly and Leary, Ryan and Li, Jason and Zhang, Yang}, + journal={arXiv preprint arXiv:1910.10261}, + year={2019} +} + +@misc{itu1988g711, + title={{ITU-T} {G.711} - {Pulse} code modulation ({PCM}) of voice frequencies}, + author={ITU-T Geneva Switzerland}, + year={1988}, +} + +@article{han2020contextnet, + title={ContextNet: Improving convolutional neural networks for automatic speech recognition with global context}, + author={Han, Wei and Zhang, Zhengdong and Zhang, Yu and Yu, Jiahui and Chiu, Chung-Cheng and Qin, James and Gulati, Anmol and Pang, Ruoming and Wu, Yonghui}, + journal={arXiv:2005.03191}, + year={2020} +} + +@inproceedings{hu2018squeeze, + title={Squeeze-and-excitation networks}, + author={Hu, Jie and Shen, Li and Sun, Gang}, + booktitle={ICVPR}, + year={2018} +} + +@article{koluguri2020speakernet, + title={SpeakerNet: 1D Depth-wise Separable Convolutional Network for Text-Independent Speaker Recognition and Verification}, + author={Koluguri, Nithin Rao and Li, Jason and Lavrukhin, Vitaly and Ginsburg, Boris}, + journal={arXiv preprint arXiv:2010.12653}, + year={2020} +} + +@article{gulati2020conformer, + title={Conformer: Convolution-augmented transformer for speech recognition}, + author={Gulati, Anmol and Qin, James and Chiu, Chung-Cheng and Parmar, Niki and Zhang, Yu and Yu, Jiahui and Han, Wei and Wang, Shibo and Zhang, Zhengdong and Wu, Yonghui and others}, + journal={arXiv preprint arXiv:2005.08100}, + year={2020} +} \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/asr/asr_language_modeling.rst b/models/tacotron2/NeMo/docs/source/asr/asr_language_modeling.rst new file mode 100644 index 0000000000..2df9994c4a --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/asr_language_modeling.rst @@ -0,0 +1,292 @@ +##################### +ASR Language Modeling +##################### + +Language models have shown to help the accuracy of ASR models. NeMo supports the following two approaches to incorporate language models into the ASR models: + +* :ref:`ngram_modeling` +* :ref:`neural_rescoring` + +It is possible to use both approaches on the same ASR model. + + +.. _ngram_modeling: + +************************ +N-gram Language Modeling +************************ + +In this approach, an N-gram LM is trained on text data, then it is used in fusion with beam search decoding to find the +best candidates. The beam search decoders in NeMo support language models trained with KenLM library ( +`https://github.com/kpu/kenlm `__). +The beam search decoders and KenLM library are not installed by default in NeMo, and you need to install them to be +able to use beam search decoding and N-gram LM. +Please refer to `scripts/ngram_lm/install_beamsearch_decoders.sh` on how to install them. + +NeMo supports both character-based and BPE-based models for N-gram LMs. An N-gram LM can be used with beam search +decoders on top of the ASR models to produce more accurate candidates. The beam search decoder would incorporate +the scores produced by the N-gram LM into its score calculations as the following: + +.. code:: + + final_score = acoustic_score + beam_alpha*lm_score + beam_beta*seq_length + +where acoustic_score is the score predicted by the acoustic encoder and lm_score is the one estimated by the LM. +Parameter 'beam_alpha' specifies amount of importance to place on the N-gram language model, and 'beam_beta' is a +penalty term to consider the sequence length in the scores. Larger alpha means more importance on the LM and less +importance on the acoustic model. Negative values for beta will give penalty to longer sequences and make the decoder +to prefer shorter predictions, while positive values would result in longer candidates. + + +Train N-gram LM +=============== + +The script to train an N-gram language model with KenLM can be found at +`scripts/asr_language_modeling/ngram_lm/train_kenlm.py `__. + +This script would train an N-gram language model with KenLM library which can be used with the beam search decoders +on top of the ASR models. This script supports both character level and BPE level encodings and models which is +detected automatically from the type of the model. + + +You may train the N-gram model as the following: + +.. code:: + + python train_kenlm.py --nemo_model_file \ + --train_file \ + --kenlm_model_file \ + --ngram_length + +The train file specified by `--train_file` can be a text file or JSON manifest. If the file's extension is anything +other than `.json`, it assumes that data format is plain text. For plain text format, each line should contain one +sample. For JSON manifest file, the file need to contain json formatted samples per each line like this: + +.. code:: + + {"audio_filepath": "/data_path/file1.wav", "text": "The transcript of the audio file."} + +It just extracts the `text` field from each line to create the training text file. After the N-gram model is trained, +it is stored at the path specified by `--kenlm_model_file`. + +The following is the list of the arguments for the training script: + ++------------------+----------+-------------+------------------------------------------------------------------------------------------------+ +| **Argument** | **Type** | **Default** | **Description** | ++------------------+----------+-------------+------------------------------------------------------------------------------------------------+ +| nemo_model_file | str | Required | The path of the `.nemo` file of the ASR model. It is needed to extract the tokenizer. | ++------------------+----------+-------------+------------------------------------------------------------------------------------------------+ +| train_file | str | Required | Path to the training file, it can be a text file or JSON manifest. | ++------------------+----------+-------------+------------------------------------------------------------------------------------------------+ +| kenlm_model_file | str | Required | The path to store the KenLM binary model file. | ++------------------+----------+-------------+------------------------------------------------------------------------------------------------+ +| kenlm_bin_path | str | Required | The path to the bin folder of KenLM. It is a folder named `bin` under where KenLM is installed.| ++------------------+----------+-------------+------------------------------------------------------------------------------------------------+ +| ngram_length** | int | Required | Specifies order of N-gram LM. | ++------------------+----------+-------------+------------------------------------------------------------------------------------------------+ +| do_lower_case | bool | ``False`` | Whether to make the training text all lower case. | ++------------------+----------+-------------+------------------------------------------------------------------------------------------------+ + +** Note: Recommend to use 6 as the order of the N-gram model for BPE-based models. Higher orders may need the re-compilation of KenLM to support it. + +Evaluate by Beam Search Decoding and N-gram LM +============================================== + +NeMo's beam search decoders are capable of using the KenLM's N-gram models to find the best candidates. +The script to evaluate an ASR model with beam search decoding and N-gram models can be found at +`scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py `__. + +You may evaluate an ASR model as the following: + +.. code:: + + python eval_beamsearch_ngram.py --nemo_model_file \ + --input_manifest \ + --acoustic_batch_size \ + --beam_width \ + --beam_alpha \ + --beam_width \ + --preds_output_folder \ + --decoding_mode beam_search_ngram + +It can evaluate a model in the three following modes by setting the argument `--decoding_mode`: + +* greedy: Just greedy decoding is done, and no beam search decoding is performed. +* beamsearch: The beam search decoding is done but without using the N-gram language model, final results would be equivalent to setting the weight of LM (beam_beta) to zero. +* beamsearch_ngram: The beam search decoding is done with N-gram LM. + +The `beamsearch` mode would evaluate by beam search decoding without any language model. +It would report the performances in terms of Word Error Rate (WER) and Character Error Rate (CER). Moreover, +the WER/CER of the model when the best candidate is selected among the candidates is also reported as the best WER/CER. +It can be an indicator of how good the predicted candidates are. + +The script would initially load the ASR model and predict the outputs of the model's encoder as log probabilities. +This part would be computed in batches on a device selected by `--device`, which can be CPU (`--device=cpu`) or a +single GPU (`--device=cuda:0`). The batch size of this part can get specified by `--acoustic_batch_size`. You may use +the largest batch size feasible to speed up the step of calculating the log probabilities. You may also use `--use_amp` +to speed up the calculation of log probabilities and make it possible to use larger sizes for `--acoustic_batch_size`. +Currently multi-GPU is not supported for calculating the log probabilities, but using `--probs_cache_file` can help. +It stores the log probabilities produced from the model's encoder into a pickle file so that next time the first step +can get skipped. + +The following is the list of the arguments for the evaluation script: + ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| **Argument** |**Type**| **Default** | **Description** | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| nemo_model_file | str | Required | The path of the `.nemo` file of the ASR model to extract the tokenizer. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| input_manifest | str | Required | Path to the training file, it can be a text file or JSON manifest. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| kenlm_model_file | str | Required | The path to store the KenLM binary model file. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| preds_output_folder | str | None | The path to an optional folder to store the predictions. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| probs_cache_file | str | None | The cache file for storing the outputs of the model. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| acoustic_batch_size | int | 16 | The batch size to calculate log probabilities. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| use_amp | bool | ``False`` | Whether to use AMP if available to calculate log probabilities. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| device | str | cuda | The device to load the model onto to calculate log probabilities. | +| | | | It can `cpu`, `cuda`, `cuda:0`, `cuda:1`, ... | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| decoding_mode | str | beamsearch_ngram | The decoding scheme to be used for evaluation. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| beam_width | float | Required | The width or list of the widths of the beam search decoding. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| beam_alpha | float | Required | The alpha parameter or list of the alphas for the beam search decoding. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| beam_beta | float | Required | The beta parameter or list of the betas for the beam search decoding. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| beam_batch_size | int | 128 | The batch size to be used for beam search decoding. | +| | | | Larger batch size can be a little faster, but uses larger memory. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ + +Width of the beam search (`--beam_width`) specifies the number of top candidates/predictions the beam search decoder +would search for. Larger beams result in more accurate but slower predictions. + +There is also a tutorial to learn more about evaluating the ASR models with N-gram LM here: +`Offline ASR Inference with Beam Search and External Language Model Rescoring `_ + +Hyperparameter Grid Search +-------------------------- + +Beam search decoding with N-gram LM has three main hyperparameters: `beam_width`, `beam_alpha`, and `beam_beta`. +The accuracy of the model is dependent to the values of these parameters, specially beam_alpha and beam_beta. +You may specify a single or list of values for each of these parameters to perform grid search. It would perform the +beam search decoding on all the combinations of the these three hyperparameters. +For instance, the following set of parameters would results in 2*1*2=4 beam search decodings: + +.. code:: + + python eval_beamsearch_ngram.py ... \ + --beam_width 64 128 \ + --beam_alpha 1.0 \ + --beam_beta 1.0 0.5 + + +.. _neural_rescoring: + +**************** +Neural Rescoring +**************** + +In this approach a neural network is used which can gives scores to a candidate. A candidate is the text transcript predicted by the decoder of the ASR model. +The top K candidates produced by the beam search decoding (beam width of K) are given to a neural language model to rank them. +Ranking can be done by a language model which gives a score to each candidate. +This score is usually combined with the scores from the beam search decoding to produce the final scores and rankings. + +Train Neural Rescorer +===================== + +An example script to train such a language model with Transformer can be found at `examples/nlp/language_modeling/transformer_lm.py `__. +It trains a TransformerLMModel which can be used as a neural rescorer for an ASR system. + + +Evaluation +========== + +Given a trained TransformerLMModel `.nemo` file, the script available at +`scripts/asr_language_modeling/neural_rescorer/eval_neural_rescorer.py `__ +can be used to re-score beams obtained with ASR model. You need the `.tsv` file containing the candidates produced +by the acoustic model and the beam search decoding to use this script. The candidates can be the result of just the beam +search decoding or the result of fusion with an N-gram LM. You may generate this file by specifying `--preds_output_folder' for +`scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py `__. + +The neural rescorer would rescore the beams/candidates by using two parameters of `rescorer_alpha` and `rescorer_beta` as the following: + +.. code:: + + final_score = beam_search_score + rescorer_alpha*neural_rescorer_score + rescorer_beta*seq_length + +Parameter `rescorer_alpha` specifies amount of importance to place on the neural rescorer model, and `rescorer_beta` is +a penalty term to consider the sequence length in the scores. They have similar effects like the parameters +`beam_alpha` and `beam_beta` of beam search decoder and N-gram LM. + +You may follow the following steps to evaluate a neural LM: + +#. Obtain `.tsv` file with beams and their corresponding scores. Scores can be from a regular beam search decoder or + in fusion with an N-gram LM scores. For a given beam size `beam_size` and a number of examples + for evaluation `num_eval_examples`, it should contain (`num_eval_examples` x `beam_size`) lines of + form `beam_candidate_text \t score`. This file can be generated by `scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py `__ + +#. Rescore the candidates by `scripts/asr_language_modeling/neural_rescorer/eval_neural_rescorer.py `__. + +.. code:: + python eval_neural_rescorer.py + --lm_model=[path to .nemo file of the LM] + --beams_file=[path to beams .tsv file] + --beam_size=[size of the beams] + --eval_manifest=[path to eval manifest .json file] + --batch_size=[batch size used for inference on the LM model] + --alpha=[the value for the parameter rescorer_alpha] + --beta=[the value for the parameter rescorer_beta] + --scores_output_file=[the optional path to store the rescored candidates] + +The candidates along with their new scores would be stored at the file specified by `--scores_output_file`. + +The following is the list of the arguments for the evaluation script: + ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| **Argument** |**Type**| **Default** | **Description** | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| lm_model | str | Required | The path of the '.nemo' file of the ASR model to extract the tokenizer. | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| eval_manifest | str | Required | Path to the evaluation manifest file (.json manifest file) | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| beams_file | str | Required | path to beams file (.tsv) containing the candidates and their scores | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| beam_size | int | Required | The width of the beams (number of candidates) generated by the decoder | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| alpha | float | None | The value for parameter rescorer_alpha | +| | | | Not passing value would enable linear search for rescorer_alpha | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| beta | float | None | The value for parameter rescorer_beta | +| | | | Not passing value would enable linear search for rescorer_beta | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| batch_size | int | 16 | The batch size used to calculate the scores | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| scores_output_file | str | None | The optional file to store the rescored beams | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| use_amp | bool | ``False`` | Whether to use AMP if available calculate the scores | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ +| device | str | cuda | The device to load LM model onto to calculate the scores | +| | | | It can be 'cpu', 'cuda', 'cuda:0', 'cuda:1', ... | ++---------------------+--------+------------------+-------------------------------------------------------------------------+ + + +Hyperparameter Linear Search +---------------------------- + +This script also supports linear search for parameters `rescorer_alpha` and `rescorer_beta`. If any of the two is not +provided, a linear search is performed to find the best value for that parameter. When linear search is used, initially +`rescorer_beta` is set to zero and the best value for `rescorer_alpha` is found, then `rescorer_alpha` is fixed with +that value and another linear search is done to find the best value for `rescorer_beta`. +If any of the of these two parameters is already specified, then search for that one is skipped. After each search for a +parameter, the plot of WER% for different values of the parameter is also shown. + +It is recommended to first use the linear search for both parameters on a validation set by not providing any values for `--alpha` and `--beta`. +Then check the WER curves and decide on the best values for each parameter. Finally, evaluate the best values on the test set. diff --git a/models/tacotron2/NeMo/docs/source/asr/configs.rst b/models/tacotron2/NeMo/docs/source/asr/configs.rst new file mode 100644 index 0000000000..4c93043fcd --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/configs.rst @@ -0,0 +1,528 @@ +NeMo ASR Configuration Files +============================ + +This section describes the NeMo configuration file setup that is specific to models in the ASR collection. For general information +about how to set up and run experiments that is common to all NeMo models (e.g. Experiment Manager and PyTorch Lightning trainer +parameters), see the :doc:`../core/core` section. + +The model section of the NeMo ASR configuration files generally requires information about the dataset(s) being used, the preprocessor +for audio files, parameters for any augmentation being performed, as well as the model architecture specification. The sections on +this page cover each of these in more detail. + +Example configuration files for all of the NeMo ASR scripts can be found in the +`config directory of the examples `_. + + +Dataset Configuration +--------------------- + +Training, validation, and test parameters are specified using the ``train_ds``, ``validation_ds``, and +``test_ds`` sections in the configuration file, respectively. Depending on the task, there may be arguments specifying the sample rate +of the audio files, the vocabulary of the dataset (for character prediction), whether or not to shuffle the dataset, and so on. You may +also decide to leave fields such as the ``manifest_filepath`` blank, to be specified via the command-line at runtime. + +Any initialization parameter that is accepted for the Dataset class used in the experiment can be set in the config file. +Refer to the `Datasets <./api.html#Datasets>`__ section of the API for a list of Datasets and their respective parameters. + +An example ASR train and validation configuration should look similar to the following: + +.. code-block:: yaml + + # Specified at the beginning of the config file + labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"] + + model: + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: *labels # Uses the labels above + batch_size: 32 + trim_silence: True + max_duration: 16.7 + shuffle: True + is_tarred: False # If set to true, uses the tarred version of the Dataset + tarred_audio_filepaths: null # Not used if is_tarred is false + tarred_shard_strategy: "scatter" # Not used if is_tarred is false + num_workers: 8 + pin_memory: true + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: *labels # Uses the labels above + batch_size: 32 + shuffle: False # No need to shuffle the validation data + num_workers: 8 + pin_memory: true + + +Preprocessor Configuration +-------------------------- + +If you are loading audio files for your experiment, you will likely want to use a preprocessor to convert from the +raw audio signal to features (e.g. mel-spectrogram or MFCC). The ``preprocessor`` section of the config specifies the audio +preprocessor to be used via the ``_target_`` field, as well as any initialization parameters for that preprocessor. + +An example of specifying a preprocessor is as follows: + +.. code-block:: yaml + + model: + ... + preprocessor: + # _target_ is the audio preprocessor module you want to use + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.02 + ... + # Other parameters for the preprocessor + +Refer to the `Audio Preprocessors <./api.html#Audio Preprocessors>`__ API section for the preprocessor options, expected arguments, +and defaults. + +Augmentation Configurations +--------------------------- + +There are a few on-the-fly spectrogram augmentation options for NeMo ASR, which can be specified by the +configuration file using a ``spec_augment`` section. + +For example, there are options for `Cutout `_ and +`SpecAugment `_ available via the ``SpectrogramAugmentation`` module. + +The following example sets up both ``Cutout`` (via the ``rect_*`` parameters) and ``SpecAugment`` (via the ``freq_*`` +and ``time_*`` parameters). + +.. code-block:: yaml + + model: + ... + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + # Cutout parameters + rect_masks: 5 # Number of rectangles to cut from any given spectrogram + rect_freq: 50 # Max cut of size 50 along the frequency dimension + rect_time: 120 # Max cut of size 120 along the time dimension + # SpecAugment parameters + freq_masks: 2 # Cut two frequency bands + freq_width: 15 # ... of width 15 at maximum + time_masks: 5 # Cut out 10 time bands + time_width: 25 # ... of width 25 at maximum + +You can use any combination of ``Cutout``, frequency/time ``SpecAugment``, or neither of them. + +With NeMo ASR, you can also add augmentation pipelines that can be used to simulate various kinds of noise +added to audio in the channel. Augmentors in a pipeline are applied on the audio data read in the data layer. Online +augmentors can be specified in the config file using an ``augmentor`` section in ``train_ds``. The following example +adds an augmentation pipeline that first adds white noise to an audio sample with a probability of 0.5 and at a level +randomly picked between -50 dB and -10 dB and then passes the resultant samples through a room impulse response randomly +picked from the manifest file provided for ``impulse`` augmentation in the config file. + +.. code-block:: yaml + + model: + ... + train_ds: + ... + augmentor: + white_noise: + prob: 0.5 + min_level: -50 + max_level: -10 + impulse: + prob: 0.3 + manifest_path: /path/to/impulse_manifest.json + +Refer to the `Audio Augmentors <./api.html#Audio Augmentors>`__ API section for more details. + +Tokenizer Configurations +------------------------ + +Some models utilize sub-word encoding via an external tokenizer instead of explicitly defining their vocabulary. + +For such models, a ``tokenizer`` section is added to the model config. ASR models currently support two types of +custom tokenizers: + +- Google Sentencepiece tokenizers (tokenizer type of ``bpe`` in the config) +- HuggingFace WordPiece tokenizers (tokenizer type of ``wpe`` in the config) + +In order to build custom tokenizers, refer to the ``ASR_with_Subword_Tokenization`` notebook available in the +ASR tutorials directory. + +The following example sets up a ``SentencePiece Tokenizer`` at a path specified by the user: + +.. code-block:: yaml + + model: + ... + tokenizer: + dir: "" + type: "bpe" # can be "bpe" or "wpe" + +For models which utilize sub-word tokenization, we share the decoder module (``ConvASRDecoder``) with character tokenization models. +All parameters are shared, but for models which utilize sub-word encoding, there are minor differences when setting up the config. For +such models, the tokenizer is utilized to fill in the missing information when the model is constructed automatically. + +For example, a decoder config corresponding to a sub-word tokenization model should look similar to the following: + +.. code-block:: yaml + + model: + ... + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: *enc_final + num_classes: -1 # filled with vocabulary size from tokenizer at runtime + vocabulary: [] # filled with vocabulary from tokenizer at runtime + + +Model Architecture Configurations +--------------------------------- + +Each configuration file should describe the model architecture being used for the experiment. Models in the NeMo ASR collection need +an ``encoder`` section and a ``decoder`` section, with the ``_target_`` field specifying the module to use for each. + +Here is the list of the parameters in the model section which are shared among most of the ASR models: + ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+ +| **Parameter** | **Datatype** | **Description** | **Supported Values** | ++=========================+==================+===============================================================================================================+=================================+ +| :code:`log_prediction` | bool | Whether a random sample should be printed in the output at each step, along with its predicted transcript. | | ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+ +| :code:`ctc_reduction` | string | Specifies the reduction type of CTC loss. Defaults to ``mean_batch`` which would take the average over the | :code:`none`, | +| | | batch after taking the average over the length of each sample. | :code:`mean_batch` | +| | | | :code:`mean`, :code:`sum` | ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+ + +The following sections go into more detail about the specific configurations of each model architecture. + +For more information about the ASR models, refer to the :doc:`Models <./models>` section. + +Jasper and QuartzNet +~~~~~~~~~~~~~~~~~~~~ + +The `Jasper <./models.html#Jasper>`__ and `QuartzNet <./models.html#QuartzNet>`__ models are very similar, and as such the components in their +configs are very similar as well. + +Both architectures use the ``ConvASREncoder`` for the ``encoder``, with parameters detailed in the table below. The encoder parameters +include details about the Jasper/QuartzNet ``[BxR]`` encoder architecture, including how many blocks to use (``B``), how many times +to repeat each sub-block (``R``), and the convolution parameters for each block. + +The number of blocks ``B`` is determined by the number of list elements under ``jasper`` minus the one prologue and two epilogue blocks. +The number of sub-blocks ``R`` is determined by setting the ``repeat`` parameter. + +To use QuartzNet (which uses more compact time-channel separable convolutions) instead of Jasper, add :code:`separable: true` to all +but the last block in the architecture. + +Change the parameter name ``jasper``. + ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+-------------------------------------+ +| **Parameter** | **Datatype** | **Description** | **Supported Values** | ++=========================+==================+===============================================================================================================+=====================================+ +| :code:`feat_in` | int | The number of input features. Should be equal to :code:`features` in the preprocessor parameters. | | ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+-------------------------------------+ +| :code:`activation` | string | Which activation function to use in the encoder. | :code:`hardtanh`, :code:`relu`, | +| | | | :code:`selu`, :code:`swish` | ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+-------------------------------------+ +| :code:`conv_mask` | bool | Whether to use masked convolutions in the encoder. Defaults to ``true``. | | ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+-------------------------------------+ +| :code:`jasper` | | A list of blocks that specifies your encoder architecture. Each entry in this list represents one block in | | +| | | the architecture and contains the parameters for that block, including convolution parameters, dropout, and | | +| | | the number of times the block is repeated. Refer to the `Jasper `_ and | | +| | | `QuartzNet `_ papers for details about specific model configurations. | | ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+-------------------------------------+ + +A QuartzNet 15x5 (fifteen blocks, each sub-block repeated five times) encoder configuration should look similar to the following example: + +.. code-block:: yaml + + # Specified at the beginning of the file for convenience + n_mels: &n_mels 64 # Used for both the preprocessor and encoder as number of input features + repeat: &repeat 5 # R=5 + dropout: &dropout 0.0 + separable: &separable true # Set to true for QN. Set to false for Jasper. + + model: + ... + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels # Should match "features" in the preprocessor. + activation: relu + conv_mask: true + + jasper: # This field name should be "jasper" for both types of models. + + # Prologue block + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: 1 # Prologue block is not repeated. + residual: false + separable: *separable + stride: [2] + + # Block 1 + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + ... # Entries for blocks 2~14 + + # Block 15 + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + # Two epilogue blocks + - dilation: [2] + dropout: *dropout + filters: 512 + kernel: [87] + repeat: 1 # Epilogue blocks are not repeated + residual: false + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: &enc_filters 1024 + kernel: [1] + repeat: 1 # Epilogue blocks are not repeated + residual: false + stride: [1] + +Both Jasper and QuartzNet use the ``ConvASRDecoder`` as the decoder. The decoder parameters are detailed in the following table. + ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+ +| **Parameter** | **Datatype** | **Description** | **Supported Values** | ++=========================+==================+===============================================================================================================+=================================+ +| :code:`feat_in` | int | The number of input features to the decoder. Should be equal to the number of filters in the last block of | | +| | | the encoder. | | ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+ +| :code:`vocabulary` | list | A list of the valid output characters for your model. For example, for an English dataset, this could be a | | +| | | list of all lowercase letters, space, and apostrophe. | | ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+ +| :code:`num_classes` | int | Number of output classes, i.e. the length of :code:`vocabulary`. | | ++-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+ + +For example, a decoder config corresponding to the encoder above should look similar to the following: + +.. code-block:: yaml + + model: + ... + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: *enc_filters + vocabulary: *labels + num_classes: 28 # Length of the vocabulary list + +Citrinet +~~~~~~~~ + +The `Citrinet <./models.html#Citrinet>`__ and `QuartzNet <./models.html#QuartzNet>`__ models are very similar, and as such the +components in their configs are very similar as well. Citrinet utilizes Squeeze and Excitation, as well as sub-word tokenization, in +contrast to QuartzNet. Depending on the dataset, we utilize different tokenizers. For Librispeech, we utilize the HuggingFace WordPiece +tokenizer, and for all other datasets we utilize the Google Sentencepiece tokenizer - usually the ``unigram`` tokenizer type. + +Both architectures use the ``ConvASREncoder`` for the ``encoder``, with parameters detailed above. The encoder parameters include +details about the Citrinet-C encoder architecture, including how many filters are used per channel (``C``). The Citrinet-C +configuration is a shortform notation for Citrinet-21x5xC, such that ``B = 21`` and ``R = 5`` are the default and should generally +not be changed. + +To use Citrinet instead of QuartzNet, refer to the ``citrinet_512.yaml`` configuration found inside the ``examples/asr/conf/citrinet`` +directory. Citrinet is primarily comprised of the same :class:`~nemo.collections.asr.parts.submodules.jasper.JasperBlock` as ``Jasper`` or +``QuartzNet`. + +While the configs for Citrinet and QuartzNet are similar, we note the additional flags used for Citrinet below. Refer to the +``JasperBlock`` documentation for the meaning of these arguments. + ++---------------------------+------------------+-----------------------------------------------------------------------------------------------------------+-----------------------------------+ +| **Parameter** | **Datatype** | **Description** | **Supported Values** | ++===========================+==================+===========================================================================================================+===================================+ +| :code:`se` | bool | Whether to apply squeeze-and-excitation mechanism or not. | :code:`true` or :code:`false` | ++---------------------------+------------------+-----------------------------------------------------------------------------------------------------------+-----------------------------------+ +| :code:`se_context_size` | int | SE context size. -1 means global context. | :code:`-1` or :code:`+ve int` | ++---------------------------+------------------+-----------------------------------------------------------------------------------------------------------+-----------------------------------+ +| :code:`stride_last` | bool | Stride on the final repeated block or all repeated blocks. | :code:`true` or :code:`false` | ++---------------------------+------------------+-----------------------------------------------------------------------------------------------------------+-----------------------------------+ +| :code:`residual_mode` | str | Type of residual branch to construct. | :code:`"add"` or | +| | | Can be pointwise residual addition or pointwise strided residual attention | :code:`"stride_add"` | ++---------------------------+------------------+-----------------------------------------------------------------------------------------------------------+-----------------------------------+ + +A Citrinet-512 config should look similar to the following: + +.. code-block:: yaml + + model: + ... + # Specify some defaults across the entire model + model_defaults: + repeat: 5 + dropout: 0.1 + separable: true + se: true + se_context_size: -1 + ... + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels # Should match "features" in the preprocessor. + activation: relu + conv_mask: true + + jasper: # This field name should be "jasper" for the JasperBlock (which constructs Citrinet). + + # Prologue block + - filters: 512 + repeat: 1 + kernel: [5] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + # Block 1 + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [11] + stride: [2] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + + ... # Entries for blocks 2~21 + + # Block 22 + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [39] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + # Epilogue block + + - filters: &enc_final 640 + repeat: 1 + kernel: [41] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + +As mentioned above, Citrinet uses the ``ConvASRDecoder`` as the decoder layer similar to QuartzNet. Only the configuration must be +changed slightly as Citrinet utilizes sub-word tokenization. + +.. note:: + The following information is relevant to any of the above models that implements its encoder as an :class:`~nemo.collections.asr.modules.conv_asr.ConvASREncoder`, and utilizes the ``SqueezeExcite`` mechanism. + +The ``SqueezeExcite`` block within a :class:`~nemo.collections.asr.modules.conv_asr.ConvASREncoder` network can be modified to utilize a different context window after the model has been instantiated (even after the model has been trained) so as to evaluate the model with limited context. This can be achieved using the :meth:`~nemo.collections.asr.parts.mixins.mixins.ASRModuleMixin.change_conv_asr_se_context_window` + +.. code-block:: python + + # Here, model can be any model that has a `ConvASREncoder` as its encoder, and utilized `SqueezeExcite` blocks + # `context_window` : It is an integer representing the number of timeframes (each corresponding to some window stride). + # `update_config` : Bool flag which determines whether the config of the model should be updated to reflect the new context window. + + # Here, we specify that 128 timeframes of 0.01s stride should be the context window + # This is equivalent to 128 * 0.01s context window for `SqueezeExcite` + model.change_conv_asr_se_context_window(context_window=128, update_config=True) + +Conformer-CTC +~~~~~~~~~~~~~ + +The config files for Conformer-CTC model contain character-based encoding and sub-word encoding at +``/examples/asr/conf/conformer/conformer_ctc_char.yaml`` and ``/examples/asr/conf/conformer/conformer_ctc_bpe.yaml`` +respectively. Some components of the configs of `Conformer-CTC <./models.html#Conformer-CTC>`__ include the following datasets: + +- ``train_ds``, ``validation_ds``, and ``test_ds`` +- opimizer (``optim``) +- augmentation (``spec_augment``) +- ``decoder`` +- ``trainer`` +``exp_manager`` + +These datasets are similar to other ASR models like `QuartzNet <./models.html#QuartzNet>`__. There should be a tokenizer section where you can +specify the tokenizer if you want to use sub-word encoding instead of character-based encoding. + +The encoder section includes the details about the Conformer-CTC encoder architecture. You may find more information in the +config files and also :doc:`nemo.collections.asr.modules.ConformerEncoder<./api.html#nemo.collections.asr.modules.ConformerEncoder>`. + + +Fine-tuning Configurations +------------------------- + +All ASR scripts support easy fine-tuning by partially/fully loading the pretrained weights from a checkpoint into the currently instantiated model. Pre-trained weights can be provided in multiple ways - + +1) Providing a path to a NeMo model (via ``init_from_nemo_model``) +2) Providing a name of a pretrained NeMo model (which will be downloaded via the cloud) (via ``init_from_pretrained_model``) +3) Providing a path to a Pytorch Lightning checkpoint file (via ``init_from_ptl_ckpt``) + +Fine-tuning via a NeMo model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: sh + + python examples/asr/script_to_.py \ + --config-path= \ + --config-name=) \ + model.train_ds.manifest_filepath="" \ + model.validation_ds.manifest_filepath="" \ + trainer.gpus=-1 \ + trainer.max_epochs=50 \ + +init_from_nemo_model="" + + +Fine-tuning via a NeMo pretrained model name +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: sh + + python examples/asr/script_to_.py \ + --config-path= \ + --config-name=) \ + model.train_ds.manifest_filepath="" \ + model.validation_ds.manifest_filepath="" \ + trainer.gpus=-1 \ + trainer.max_epochs=50 \ + +init_from_pretrained_model="" + +Fine-tuning via a Pytorch Lightning checkpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: sh + + python examples/asr/script_to_.py \ + --config-path= \ + --config-name=) \ + model.train_ds.manifest_filepath="" \ + model.validation_ds.manifest_filepath="" \ + trainer.gpus=-1 \ + trainer.max_epochs=50 \ + +init_from_ptl_ckpt="" \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/asr/data/asr_results.csv b/models/tacotron2/NeMo/docs/source/asr/data/asr_results.csv new file mode 100644 index 0000000000..3b179b5c59 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/data/asr_results.csv @@ -0,0 +1,21 @@ +Model Name,Model Base Class,Model Card +QuartzNet15x5Base-En,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemospeechmodels" +stt_en_jasper10x5dr,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_jasper10x5dr" +stt_en_citrinet_256,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_256" +stt_en_citrinet_512,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_512" +stt_en_citrinet_1024,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_1024" +stt_ca_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ca_quartznet15x5" +stt_it_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_it_quartznet15x5" +stt_fr_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_fr_quartznet15x5" +stt_es_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_es_quartznet15x5" +stt_de_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_de_quartznet15x5" +stt_pl_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_pl_quartznet15x5" +stt_ru_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ru_quartznet15x5" +stt_zh_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_quartznet15x5" +stt_zh_citrinet_512,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_citrinet_512" +stt_en_conformer_ctc_small,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_small" +stt_en_conformer_ctc_medium,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_medium" +stt_en_conformer_ctc_large,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large" +stt_en_conformer_ctc_small_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_small_ls" +stt_en_conformer_ctc_medium_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_medium_ls" +stt_en_conformer_ctc_large_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large_ls" diff --git a/models/tacotron2/NeMo/docs/source/asr/data/asrlm_results.csv b/models/tacotron2/NeMo/docs/source/asr/data/asrlm_results.csv new file mode 100644 index 0000000000..d9a395cb8b --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/data/asrlm_results.csv @@ -0,0 +1,2 @@ +Model Name,Model Base Class,Model Card +asrlm_en_transformer_large_ls,TransformerLMModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:asrlm_en_transformer_large_ls" diff --git a/models/tacotron2/NeMo/docs/source/asr/data/benchmark_ca.csv b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_ca.csv new file mode 100644 index 0000000000..ef5220fe6e --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_ca.csv @@ -0,0 +1,2 @@ +Model,Model Base Class,Model Card +stt_ca_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ca_quartznet15x5" diff --git a/models/tacotron2/NeMo/docs/source/asr/data/benchmark_de.csv b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_de.csv new file mode 100644 index 0000000000..f0f86121fe --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_de.csv @@ -0,0 +1,3 @@ +Model,Model Base Class,Model Card +stt_de_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_de_quartznet15x5" + diff --git a/models/tacotron2/NeMo/docs/source/asr/data/benchmark_en.csv b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_en.csv new file mode 100644 index 0000000000..ccdf05b806 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_en.csv @@ -0,0 +1,14 @@ +Model Name,Model Base Class,Model Card +QuartzNet15x5Base-En,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemospeechmodels" +stt_zh_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_quartznet15x5" +stt_en_jasper10x5dr,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_jasper10x5dr" +stt_en_citrinet_256,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_256" +stt_en_citrinet_512,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_512" +stt_en_citrinet_1024,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_1024" +stt_zh_citrinet_512,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_citrinet_512" +stt_en_conformer_ctc_small,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_small" +stt_en_conformer_ctc_medium,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_medium" +stt_en_conformer_ctc_large,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large" +stt_en_conformer_ctc_small_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_small_ls" +stt_en_conformer_ctc_medium_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_medium_ls" +stt_en_conformer_ctc_large_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large_ls" diff --git a/models/tacotron2/NeMo/docs/source/asr/data/benchmark_es.csv b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_es.csv new file mode 100644 index 0000000000..2997fa360c --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_es.csv @@ -0,0 +1,3 @@ +Model,Model Base Class,Model Card +stt_es_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_es_quartznet15x5" + diff --git a/models/tacotron2/NeMo/docs/source/asr/data/benchmark_fr.csv b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_fr.csv new file mode 100644 index 0000000000..994ab0d54d --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_fr.csv @@ -0,0 +1,3 @@ +Model,Model Base Class,Model Card +stt_fr_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_fr_quartznet15x5" + diff --git a/models/tacotron2/NeMo/docs/source/asr/data/benchmark_it.csv b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_it.csv new file mode 100644 index 0000000000..d605b68809 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_it.csv @@ -0,0 +1,3 @@ +Model,Model Base Class,Model Card +stt_it_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_it_quartznet15x5" + diff --git a/models/tacotron2/NeMo/docs/source/asr/data/benchmark_pl.csv b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_pl.csv new file mode 100644 index 0000000000..bf646e1073 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_pl.csv @@ -0,0 +1,2 @@ +Model,Model Base Class,Model Card +stt_pl_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_pl_quartznet15x5" diff --git a/models/tacotron2/NeMo/docs/source/asr/data/benchmark_ru.csv b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_ru.csv new file mode 100644 index 0000000000..b46d4d9ca6 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_ru.csv @@ -0,0 +1,3 @@ +Model,Model Base Class,Model Card +stt_ru_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ru_quartznet15x5" + diff --git a/models/tacotron2/NeMo/docs/source/asr/data/benchmark_zh.csv b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_zh.csv new file mode 100644 index 0000000000..1fbc49879a --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/data/benchmark_zh.csv @@ -0,0 +1,3 @@ +Model,Model Base Class,Model Card +stt_zh_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_quartznet15x5" +stt_zh_citrinet_512,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_citrinet_512" diff --git a/models/tacotron2/NeMo/docs/source/asr/datasets.rst b/models/tacotron2/NeMo/docs/source/asr/datasets.rst new file mode 100644 index 0000000000..d4b2a2062f --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/datasets.rst @@ -0,0 +1,268 @@ +Datasets +======== + +NeMo has scripts to convert several common ASR datasets into the format expected by the ``nemo_asr`` collection. You can get started +with those datasets by following the instructions to run those scripts in the section appropriate to each dataset below. + +If the user has their own data and want to preprocess it to use with NeMo ASR models, refer to the `Preparing Custom ASR Data`_ section. + +If the user already has a dataset that you want to convert to a tarred format, refer to the `Tarred Datasets`_ section. + +.. _LibriSpeech_dataset: + +LibriSpeech +----------- + +Run the following scripts to download the LibriSpeech data and convert it into the format expected by `nemo_asr`. At least 250GB free +space is required. + +.. code-block:: bash + + # install sox + sudo apt-get install sox + mkdir data + python get_librispeech_data.py --data_root=data --data_set=ALL + +After this, the ``data`` folder should contain wav files and ``.json`` manifests for NeMo ASR datalayer. + +Each line is a training example. ``audio_filepath`` contains the path to the wav file, ``duration`` is the duration in seconds, and ``text`` is the transcript: + +.. code-block:: json + + {"audio_filepath": "/1355-39947-0000.wav", "duration": 11.3, "text": "psychotherapy and the community both the physician and the patient find their place in the community the life interests of which are superior to the interests of the individual"} + {"audio_filepath": "/1355-39947-0001.wav", "duration": 15.905, "text": "it is an unavoidable question how far from the higher point of view of the social mind the psychotherapeutic efforts should be encouraged or suppressed are there any conditions which suggest suspicion of or direct opposition to such curative work"} + +Fisher English Training Speech +------------------------------ + +Run these scripts to convert the Fisher English Training Speech data into a format expected by the ``nemo_asr`` collection. + +In brief, the following scripts convert the ``.sph`` files to ``.wav``, slices those files into smaller audio samples, matches the +smaller slices with their corresponding transcripts, and splits the resulting audio segments into train, validation, and test sets +(with one manifest each). + +.. note:: + - 106 GB of space is required to run the ``.wav`` conversion + - additional 105 GB is required for the slicing and matching + - ``sph2pipe`` is required in order to run the ``.wav`` conversion + +**Instructions** + +The following scripts assume that you already have the Fisher dataset from the Linguistic Data Consortium, with a directory structure +that looks similar to the following: + +.. code-block:: bash + + FisherEnglishTrainingSpeech/ + ├── LDC2004S13-Part1 + │   ├── fe_03_p1_transcripts + │   ├── fisher_eng_tr_sp_d1 + │   ├── fisher_eng_tr_sp_d2 + │   ├── fisher_eng_tr_sp_d3 + │   └── ... + └── LDC2005S13-Part2 + ├── fe_03_p2_transcripts + ├── fe_03_p2_sph1 + ├── fe_03_p2_sph2 + ├── fe_03_p2_sph3 + └── ... + +The transcripts that will be used are located in the ``fe_03_p<1,2>_transcripts/data/trans`` directory. The audio files (``.sph``) +are located in the remaining directories in an ``audio`` subdirectory. + +#. Convert the audio files from ``.sph`` to ``.wav`` by running: + + .. code-block:: bash + + cd /scripts + python fisher_audio_to_wav.py \ + --data_root= --dest_root= + + This will place the unsliced ``.wav`` files in ``/LDC200[4,5]S13-Part[1,2]/audio-wav/``. It will take several + minutes to run. + +#. Process the transcripts and slice the audio data. + + .. code-block:: bash + + python process_fisher_data.py \ + --audio_root= --transcript_root= \ + --dest_root= \ + --remove_noises + + This script splits the full dataset into train, validation, test sets, and places the audio slices in the corresponding folders + in the destination directory. One manifest is written out per set, which includes each slice's transcript, duration, and path. + + This will likely take around 20 minutes to run. Once finished, delete the 10 minute long ``.wav`` files. + +2000 HUB5 English Evaluation Speech +----------------------------------- + +Run the following script to convert the HUB5 data into a format expected by the ``nemo_asr`` collection. + +Similarly, to the Fisher dataset processing scripts, this script converts the ``.sph`` files to ``.wav``, slices the audio files and +transcripts into utterances, and combines them into segments of some minimum length (default is 10 seconds). The resulting segments +are all written out to an audio directory and the corresponding transcripts are written to a manifest JSON file. + +.. note:: + - 5 GB of free space is required to run this script + - ``sph2pipe`` is also required to be installed + +This script assumes you already have the 2000 HUB5 dataset from the Linguistic Data Consortium. + +Run the following command to process the 2000 HUB5 English Evaluation Speech samples: + +.. code-block:: bash + + python process_hub5_data.py \ + --data_root= \ + --dest_root= + +You can optionally include ``--min_slice_duration=`` if you would like to change the minimum audio segment duration. + +AN4 Dataset +----------- + +This is a small dataset recorded and distributed by Carnegie Mellon University. It consists of recordings of people spelling out +addresses, names, etc. Information about this dataset can be found on the `official CMU site `_. + +#. `Download and extract the dataset `_ (which is labeled "NIST's Sphere audio (.sph) format (64M)". + +#. Convert the ``.sph`` files to ``.wav`` using sox, and build one training and one test manifest. + + .. code-block:: bash + + python process_an4_data.py --data_root= + +After the script finishes, the ``train_manifest.json`` and ``test_manifest.json`` can be found in the ``/an4/`` directory. + +Aishell-1 +--------- + +To download the Aishell-1 data and convert it into a format expected by ``nemo_asr``, run: + +.. code-block:: bash + + # install sox + sudo apt-get install sox + mkdir data + python get_aishell_data.py --data_root=data + +After the script finishes, the ``data`` folder should contain a ``data_aishell`` folder which contains a wav file, a transcript folder, and related ``.json`` and ``vocab.txt`` files. + +Aishell-2 +--------- + +To process the AIShell-2 dataset, in the command below, set the data folder of AIShell-2 using ``--audio_folder`` and where to push +these files using ``--dest_folder``. In order to generate files in the supported format of ``nemo_asr``, run: + +.. code-block:: bash + + python process_aishell2_data.py --audio_folder= --dest_folder= + +After the script finishes, the ``train.json``, ``dev.json``, ``test.json``, and ``vocab.txt`` files can be found in the ``dest_folder`` directory. + +Preparing Custom ASR Data +------------------------- + +The ``nemo_asr`` collection expects each dataset to consist of a set of utterances in individual audio files plus +a manifest that describes the dataset, with information about one utterance per line (``.json``). +The audio files can be of any format supported by `Pydub `_, though we recommend +WAV files as they are the default and have been most thoroughly tested. + +There should be one manifest file per dataset that will be passed in, therefore, if the user wants separate training and validation +datasets, they should also have separate manifests. Otherwise, thay will be loading validation data with their training data and vice +versa. + +Each line of the manifest should be in the following format: + +.. code:: + + {"audio_filepath": "/path/to/audio.wav", "text": "the transcription of the utterance", "duration": 23.147} + +The :code:`audio_filepath` field should provide an absolute path to the ``.wav`` file corresponding to the utterance. +The :code:`text` field should contain the full transcript for the utterance, and the :code:`duration` field should +reflect the duration of the utterance in seconds. + +Each entry in the manifest (describing one audio file) should be bordered by '{' and '}' and must +be contained on one line. The fields that describe the file should be separated by commas, and have the form :code:`"field_name": value`, +as shown above. There should be no extra lines in the manifest, i.e. there should be exactly as many lines in the manifest as +there are audio files in the dataset. + +Since the manifest specifies the path for each utterance, the audio files do not have to be located +in the same directory as the manifest, or even in any specific directory structure. + +Once there is a manifest that describes each audio file in the dataset, use the dataset by passing +in the manifest file path in the experiment config file, e.g. as ``training_ds.manifest_filepath=``. + +Tarred Datasets +--------------- + +If experiments are run on a cluster with datasets stored on a distributed file system, the user will likely +want to avoid constantly reading multiple small files and would prefer tarring their audio files. +There are tarred versions of some NeMo ASR dataset classes for this case, such as the ``TarredAudioToCharDataset`` +(corresponding to the ``AudioToCharDataset``) and the ``TarredAudioToBPEDataset`` (corresponding to the +``AudioToBPEDataset``). The tarred audio dataset classes in NeMo use `WebDataset `_. + +To use an existing tarred dataset instead of a non-tarred dataset, set ``is_tarred: true`` in +the experiment config file. Then, pass in the paths to all of the audio tarballs in ``tarred_audio_filepaths``, either as a list +of filepaths, e.g. ``['/data/shard1.tar', '/data/shard2.tar']``, or in a single brace-expandable string, e.g. +``'/data/shard_{1..64}.tar'`` or ``'/data/shard__OP_1..64_CL_'`` (recommended, see note below). + +.. note:: + For brace expansion, there may be cases where ``{x..y}`` syntax cannot be used due to shell interference. This occurs most commonly + inside SLURM scripts. Therefore, we provide a few equivalent replacements. Supported opening braces (equivalent to ``{``) are ``(``, + ``[``, ``<`` and the special tag ``_OP_``. Supported closing braces (equivalent to ``}``) are ``)``, ``]``, ``>`` and the special + tag ``_CL_``. For SLURM based tasks, we suggest the use of the special tags for ease of use. + +As with non-tarred datasets, the manifest file should be passed in ``manifest_filepath``. The dataloader assumes that the length +of the manifest after filtering is the correct size of the dataset for reporting training progress. + +The ``tarred_shard_strategy`` field of the config file can be set if you have multiple shards and are running an experiment with +multiple workers. It defaults to ``scatter``, which preallocates a set of shards per worker which do not change during runtime. + +For more information about the individual tarred datasets and the parameters available, including shuffling options, +see the corresponding class APIs in the `Datasets <./api.html#Datasets>`__ section. + +.. warning:: + If using multiple workers, the number of shards should be divisible by the world size to ensure an even + split among workers. If it is not divisible, logging will give a warning but training will proceed, but likely hang at the last epoch. + In addition, if using distributed processing, each shard must have the same number of entries after filtering is + applied such that each worker ends up with the same number of files. We currently do not check for this in any dataloader, but the user's + program may hang if the shards are uneven. + +Conversion to Tarred Datasets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can easily convert your existing NeMo-compatible ASR datasets using the +`conversion script here `_. + +.. code:: + + python convert_to_tarred_audio_dataset.py \ + --manifest_path= \ + --target_dir= \ + --num_shards= + --max_duration= \ + --min_duration= \ + --shuffle --shuffle_seed=0 + +This script shuffles the entries in the given manifest (if ``--shuffle`` is set, which we recommend), filter +audio files according to ``min_duration`` and ``max_duration``, and tar the remaining audio files to the directory +``--target_dir`` in ``n`` shards, along with separate manifest and metadata files. + +The files in the target directory should look similar to the following: + +.. code:: + + target_dir/ + ├── audio_1.tar + ├── audio_2.tar + ├── ... + ├── metadata.yaml + └── tarred_audio_manifest.json + +Note that file structures are flattened such that all audio files are at the top level in each tarball. This ensures that +filenames are unique in the tarred dataset and the filepaths do not contain "-sub" and forward slashes in each ``audio_filepath`` are +simply converted to underscores. For example, a manifest entry for ``/data/directory1/file.wav`` would be ``_data_directory1_file.wav`` +in the tarred dataset manifest, and ``/data/directory2/file.wav`` would be converted to ``_data_directory2_file.wav``. diff --git a/models/tacotron2/NeMo/docs/source/asr/images/citrinet_vertical.png b/models/tacotron2/NeMo/docs/source/asr/images/citrinet_vertical.png new file mode 100644 index 0000000000000000000000000000000000000000..a9a7f8c2e8530dd866aa46aa95f5c59fa56b442f GIT binary patch literal 138615 zcmeFZcRbboA3uC#bF$^wD?1z|vS;=d8R1xEMatfeY$YQzBSa!AEBi!*tdNnJmB`-n zeji*{*YEz`kNcnd-}`zzuE!$Yo_Ec^wD@iwOe3 zhTvTSzoFJEF9H9-c%q{y3n}WRTLgc=dLW}N1A&xA6CRmkgFoXxQZjr3feSzoebDw(*UlH7cpkqD9z z3OH+604$z&nbhzT3zjSsNdgHI&xn$etdi0v8WK33DD}cVPtW?AT5FHaJ+A=Ramr zlR(dZ3b_COKluNP=S%|spN(S8vEeY^%B-)>iHO*&M2t=*h3?&L>q)jvBW%bTS&_;b zIRpZSl0b`R{ZCzQND4pRIG7^sJSLGZhziGNGCW>Pv@`92z{Q=Noi~5y7ZzFt$Hc@; zPfwfFx@~W6c6N4J2H%m*gZ5LnC2CQWb>dNEy`XwsQu^t|3ldS1hhcT*`T6&JtqhnHjq^0v$}bh=&_^8DUi-@zfb_)zIy?^%#ABo5gM zfCfgfh$Qn*L_T}=#xLzebtYic=uY=(6#{h$oTDic0$F)V!#K~7g`Xm5OO_U~Pr?*{ z@VIP6B{)w$)40C0I_|uq_t}F~<{o}>Yg8I>q^QtC-7iunxTi{(E&(}N)}0B=%*+Fx z*$b87Neu`D879Oe6DI;I==}#Wa;5F;6mEMe^71#~QSG$Op; zRWOLP3=Ng}WJ+{@=apVU30NE;1C^l=sO-as52dA>)~1_*%sgW>FQL|mbTs561tzQJ z__-9G;$=M~@{xM%qIN@nQYc4p%?K}mHnaPdprGK@t5-!uX$Uj!WL+F5=+WkUS9Nvu zBYe|YZ)td4Kx&m;TMJf6-$YsVIHjcwmb2pQCTz%0Xuqqhsab24mX_A|ZW~9Nn!j-6 zJZCOHEU5+3RFadIr>LlSIM8wptC=vY=b6<^opP#?g*C;`QhEC2Chm#CKPH|gSWP$I z`acT`LoDHz;JvdQeswMp%TM}&l@24<#l`DtMmctnp$J&o>62Ow2bJ30+A2ypX)I^6 zGCzhMZU$_3Z%p+hjzD8(6V&_`HYP9*SaMBGO?CAxMnggZ6kK$nc%hVZBA$1br_~?t z=SrAzR6KcKox-K5t88p+19=uhvs4G!XqRk|M4i57%!xcbSYb(YD*t_Lees> zT<901*N9!~yQ^`t*J@*9<5P39oUE*mk1*wBXSgq_=_(Fn;?fAV2u{U|+yN+f+px?D zPCHG%=a*x#7Rt2Jy>9GpJ?J!=OJd53p`~bIGCqY05^$fEtkzLlV}S%z#Uv);q9CG7 z3`L>1uB>MmDjcJR1+P^?MG zDuuN!&HnJGH*}2#Kfo89PF7CtL08J>odtIh=8E`jw(nV>#W zSZhmv@5xh85 zzqXFf&u2XyP5*hMy|c5~t^DBAxGa(i1M>AtSE_LR3Xw375D^w;pD+oO3EU)>5u^TM zr`+6}oT~x?F&Qx0|2zXO?=(_o?dp0{3b*EBEj4v(YilJXB{emulrS772&2ct$#Pv=|ib6Eiw8TgA94~NNdqE6X_X8DuW#RV7x zXl93p4g4(bz-{27BvAQDklf+yW4u>ANFfjy=HC-Ba`5pzXT%6Xop0e2-qF!fQc{u{ z;4y*0w6( zG$H;N>SqD_dk&wt(b3V@a-kM0pT1liH=`v$>=ELx(hzFr??d1?f5p+D;N>OC&R2Ko z!bY35mgq@IQJPygK{(l%sFK7eGfCRgC5c92)BB4J%W20_-M6;=7j{nX=Wv#H2UM`q zHwJl7ng=5GC_MD5sQ$TdQcaEMjCc_7`69{Ht0y#qfApqtu3aVmjomyU7UmRKA`SNm zt*a9kFZdC8aruTWpno|zx$Nxh{f~aq zxJ2&pu9_$Q=XAtde_SjiMu5PDku|~2w~i=j4{aho?b5B{$#zL$P%zxMg_Lp!eF-0#5lKj;`6cSZ-eUwB&pFS~lm>~y=` z*~ywk9iEX>BR$k6@=wLw$4++NbFeMrha9*|Gr@ANiin5^3dUzZR-c}S7>F^HQF5#~ zdbjl~FgS!H2eHsE5L;GeR(~~ub8@3qn-mSQ0stc7qopeq6&1Y3m0PndH_z3} zpUNjMNp%Am9(dYvdT{F@vV!&&2{a-Fz3sf^M4cxP)Eq~X{XYv32?z+JkKG_SGg9fT z%UI1mSJ~Nr;y}qSLPLPSUmt`tC0*DQ0{YkOzCJaj&BAp5=Xp7xw3(kj_rk^g##}f# z*!FY%cy8dubwfe$#KFb&MsynejTrsX$0}%z;jj5!Xxt_WxN~bu%NluA;J*RdoaU`A z2;uV_O@|k^$Q0n_;Q`5PFZ{>f!@_Z0U0pxi@A}EnL~(KI%uB>$(V_n?^uHQD3+(^K z2rWebNvy5eq?|GT1`Bp37M3iu1wuajb7N6(VnHD9ob6Ww{ypOKpQoXrwX2hF1D^kL zhZGRx&_0)S^Y5V`F!BrEio*E!6gZI9f2_;$KPwL2xqEk$6(M>d;zzLnd=7p8zCr%y zuk6-LK?)wLA!P2Qx>)zsRTdT&QPGqPoBv#MS9V+hom((+gD&*S;oL8E`)`u;;0MO( z($*pj$GdQ~TbdvrsjFX>>cazye@+s$@#DjJ`fn|6q(V+V1CaQ}_3N=2NzYTvPF}k2 z`&(@Ir?}3BuDLGAUy|fqKeUgD2G@G}lkG|934iBl9&vP|ApeY)*LLxs)M?xhF(lxI zmOm=G921!I4PRAG+6nr@jQofgMx+N*;eiZ1V5N6Wpz^KTukXJTo~Tf=YD_|b=s&AU z(mswx(elebnB@GpLt88ifk>bQa&4HtJH#D;Z|$ff0He5$hw=&!)J++`+FymFznM?; z@9L~(D9gMj5HAlT%4sUgc{M9;$28a$l8MvBA;1Jvcp&G6MLYdjk4XC*h#fEb>L1^K z*mhrjLf*tAP0S5@9q#-_D%96kLBP&xJ;RiO#$@fN=4hyDzDx3zdNQl^Lm7Qp>dwBj z?Ge3%`T}vW7>)A7Jm-Q!9pq?rQ|J(_ z(@FUxya{{!bnc$>yLdrW70$&4-u7GmNVfi19hL9c9qrlM?G0K9CJ&XLv(3eiMPAF> z$N{Ey%Ps%s@S6_HNkg3`-P>GNAJs&B?p4(paaP+zBI&;0#R<-EWNnh*@@w;3P4E?fv z{529br3SUQ9eo=AwA|)OatPAgWuvDRK;nhNQEnVGBrfFp)%9*NAv@0F8P45@e`>N+ z1k~ydKg^8l&qqh~Bxl4_)C)Gp#!;^)Sn0T}fn386TR6Snj;&gx{m#&iSHy4b@K+&W zvD5n+{~{eBWOj!|WXS>sq%u4zYQSws1_0xapwfD_rqCwXdEi-RaRjij>i(aRpG2#h zL!0OWG>PN;{nWd2;=$uHF7mneMV*Dtw?1{Y#UCo3OfV8?D$==h2RDZP&18LP1-Fe` z(BX+(HcxEN(IZ_Ug*`*{_8 zC|_IYf=C0n_pEYrf&|JxRhywQd@H&PU6f>lqD1krU$VGpE_H#c?VN`}YGPlRb*KGM z(K@*k!nDN@atF8AO-#AI2cnWdYL2r&mEAkI}#&2dDn6|Ny@z1nEA!Arr z&0+?gk2q<6o4=;`85n=Eq`U4@Z<{K6((^~ulr&Iy%0w8^i>E2zyx#K7a`W4b%2!Hn zq~T1k6ueeJ5{aynh1=Xwv=kHQUTPF_f6`7`Mu>-l)%+gmY}j!WMVPgM z?PC;cus&I5dTFZQDhG$XM}{y~b1G9C$tEnmPoPWb#u`MjWqbDMx~GY@ROthXV23Mi z)6Zt-=EnU`j|Vi@o@C3;h|mm%lTOr%m3=w8LNq>(^@9=0eFohaTzii{SWO(DkpaDY zd5(h*6Sa2zWYT%q*o!s`>-fa$w%>9T89e2iZ5`d2 z8s8kQB)8=;s!iVWfgyo_OAz<=+oJaqiz`e#bOu^q8y7hL5MDC`km=~?=vU*>*i9yw zsq%<>`$j&rMaDFGgYZ<^Fl@LJG(vJ;g|(WDn-aJ}Q7DxAxND-m#8WY)!aXN}bi(J{ zNg2tN;meo69=&pKPRgO;Mwl_z@7bLCyPoRUO}j8jACHIcg?h4|B0L0!{EDJ_p1o>d zyCa|=KfBmFv?^uHLr8l2%U$6dMTB~MRa|=Sb9bq4mN?QAtM`9=_NRK}tLv_>HSSF)%Rv{QO$YVu=El(pV0;` z)M7G@(#95OyJm-W8<-5d=wwDwAX2z&7R=o-&W zb}3<)Yt<>QTTZ?A{Q76ZeA-B28DPe(q)~(wQf@|{u&d`@>1>#rx?`+S3y$SqL1L&y zp0_EIXe%YTHIP))`0Iiar;E4yiE|%b6U9OwKm~n(Jj=U&9tcOSg14{xD1}cwQ}3EV ze1ck9TF`*uCk+V{VbpTgFK8Mo+19@E$I`;WBgQ<*brx61COc{@#3d6Rwaq2E-Z?qYI;=^g3cR-wl z5tt5~v-2@uI8Cb0Je6rQ#u|o-iV719a!Xd=tZxA4aqy*}hiSoEe!i9Rb#;)_Ccwf( znmRLnpN%E1jhuH&rH+u7UO?zL#%!vtao>slnlVHXJCA~n@EF;U;nprpar z-}=0epO4nzT|9-w3dR;$u4N-aY4QLBg#5bw_iHQU`k>#GM9jL%$maIk%g!_XT;o)D zIM&}LHVc2_D84iY4XFp9K>vWkF&^V^0L>-D^_Bz9>=)-6 zRbJ-2$v-1n+z&}g1C^NHELyNapf;&i9>$1+HCcUGtp(Ou?VO+#z8L9e|JWk>Mr{BB zbd}`b4X3M(UFvgH#K6mS4u7oH8DS#qY?4}{(f9kR0HEW(u$<(}n53jHs+$|YiS`B| z9wHIj+Aa<1NG(sXj(hC46%WtT?G9i z#Zop@cKnY&uHws5fsy#*bA1xFw%ZRGTIWvbJ3;P%p3QyGYWe!i!Z%)qWGNeYpWe*r z{7{kDL1a6w*ahDV*s#^2_4}v5lJ`Ng>Z9r2KDE^%9>mDJUNWNBZoiNx>^S?A{sLN; z>?BeAzjq6sDb;wVkd{1P(>vt`-OGM`wVKxuamK4 zg(C)z{QU`2Og`DU&P7*w+{SH+`n$jf3$TEUUK-<~{C3Z&)lA8uBtH|d8KwgT9+Zcz z99zL~IOS>Ak|^Z9e1U5v`hMW3g~LCjRCc}6(N;?2ZRg1cnm-ao_^q?`J|9LssfRpG za3RB>Ed2@v_CRU5!gw1%Rm4n*3~d=RENrDu1>V3_{gr!XZBMnX2otgla)Bd=jY7b< zE8M4-x|zS*judeW5lS@{#W+XYB_(}#yZwRRbOiy7!Fwq?3UX7?flkPxc8*8#U}LK+ye>BNw2t1hL_xfRpfpu6ecnq3YFyw$X`2L0&c?;y zcvH$#0#o7h!~v&l!Q2P+5TBQNd0#Mr0m$##u)*^DXH)q@uW>G*SCnmLJ9D;GR`yel zQe{G9H7mf)?`w~pOs50ZVd5T-Kvzb}Hca1FE^e_VD`GH9Q};P&;f-g%L`%P-(84m$ z2;s;CYruQ9-?D+3a5#Tu%Q#N#q5r{F(BkHeet9Y1ytsq|+uu1{-xM}%Bc9Fv`NQJ$ zgwWIYuuZtkv+jM`%;(DWwz(gzA0%c*Le{^fuL}uB-CenopvA4$q|MI)Wq7t$i!O5v*}{(ZJgdI?8L*G3(Cr~Fsfe9vwI}*H?C!^B4JhS@e>kHSx%LldqY{)%B z4D1%x8ISAvZ7)*#xG24jvFPGI{kcsH3jtUXoh&S{u`p$3foV@sfX>AN331ySAgK?z zT77zQz^qzQ_vvn#l{Pv z(gJt=$%uVe)NWpX6S&~CBr220^|b>7I^WdsU8;AklkC6?v%^G5N0H#+Kr{j$+WCfk z3GzAo15{hcH+BA&J}R@W{B^9J`0}A69bIuWi_2tg!XPzBIRdiiq=fCbu|xUW@zmHs zMNELpWa~RRo77^nZXez3d;NXN(Frs2zHj`BuxF&`zdM0wM!CUvN^IYU%qEyU-^US^ zDXFJs0i0~=CfTphlx=MiC{_%*FQf~)cG#tocD*4EXt39eTvta~>mPp_N}B;sDrPTb z+qb=zO<#eaFIxY(f1TL7oP z!?AdJA*()8;-N?t`?UdX^ahy7K*_;dF#l7|khgFWXbT?DBw7{OtPGL0*M+gs!MF|~ zzCk%@88D=jNe<>&H9Z<{0|&4C6-s6PHswO%;bG-C_I2`5@Vo!q`RrYcf#PT+!b6h0>JWQOwQ&#UXi>RwXkq_&`^APu zOY@nP4fw3H4gnkk`pxLwX5N}H-nJPO7NpiUE=;{C ziF=fLmCNk<<1-baPm4zDcp=f*e!djI5F{AcVW)k|&sD!FZF)b*pUl=YqLVXS8QkyW zNMLM}f*=E1Ndj5}!Iqo&t-0knr*)^%JZ=x|$`CCN*iP3^4!vF-?7jeCI7OfX) z*%*AUS^bQ%HO$kLhX3}~#|wGAo^YHEE37mSHk4$8UCW`44mBL5E2fSA0mYIVcxXC= z0og&XABW#sH%z&UbJJ;lbFiO z?He+vOrOWY{J&&WgZd9mOs_wFGZ%lk-z>d-NWZ<&?J?3Ul_5bEr#3ODzU33n0kOo2 zuwVy&b(%{|#-y)*SzYSU^()(}xz#2)3#slM!CYJ()^0)k0w8Swdq)dPjTJU|f5khi zt-sLNE(NjtMcK3Lf$%z!9hT+%PtKa*$w~iDQMSo?pWFEK=JgZ(8yuz?8LORU8nEyb ze0#u9}66M;3m3s=>r;b1~ zI1uPT|GZ)m1{hGylSJjjjS!c6h1osa^S{i}WrIY}z2o#K(Sz4FH{Ctm7`QP$fY(~I zEY}r;{H2e@f(BD7q%~MkXT0RqQ+AcyA!c2pWk$vK$+Ls{R)p-NBLHJMH^#tHQj|dKK zES#S!mo%!u-*Y+@^T+Pp+3n>?V`dZsZ+i7_KHgD+fLvbj&2F8;jNF zi)yM4S${W0=;dDp3hsWE#VjMGsvdrDX|`jBrS7a`#`bcEQr>&}g0S^1G0R(@^^}(+&|a&Q63GZV zdZ};tXP%Hj3EfZObXNJ2M`}>mwk6c5#~oO?st||>t2u)QPfxz?$Jk55qn@{~ph~!4 z@o#=u=4*sXOh%7(Ct^l}V<9w?h`hH=I(JiWS87OAP)>V}@vj6shf!&t2_W&{T^Vtxq8XmyUCG z1Nh}-9OEKZ0q=M9D_C-Tu11CfN4DLAK=coALV_5L%}5&6{2(&q zd8yh&^wA?a{MI(Y`CjMZz~k9n#4R=$b)G7N10@{^%%kLLTPdRMhMS;o@lKl9ijvr) zqm4yNDxQ!=CxHl}G_m<~2`;Q43X8>%LAJ>ZRj!Ve8}L^u2aSR+cwv(m&Lw#HHpURuQg!3;7*07B)A z7}7aYJbFA&=DD+z)@GZSs}f(2eHmgo-m%H%4m|d;ZfX@Ci|25;e_k~%>#EO)+8fv6amb_g3lhV<;X>@6b(KRoMCrx+Bd})nJ`)I;#d!X{h2I*^^Wd-n9)K z`eVS32ew2u^Xo^&>X^WxT0Th>=(Y`CZ9OVZ%qNYC+m3?~HX6zxV|0=z2Us|d-UL>+ z(^kv6xNL!8*dB=?GP|O^s`8*PXe1CV13#F9Lji@r+q8aY-N;S&E z3gV5)>ZF?tLSbdEMq)lS5mbj0U9l2!BBeNx&-+A3nJpQ%w!1r-UIxW4q6De$}ao{=gKb1f=Nx&RIhJoZ|N{pC9HnO-P{xKW=kd zbjTBbMQACgn3mdV`aD|Fw>SWOfE0zeAw})3Sw&IQ*~x>D0wRIY8*|Mr-bfj9ORvu# zt`#mwJ8%b9tHq0LJ4WU4N5nijX zSd~Fd^MNw-BO+`8QQ{5%mCC@?1)9st5->Hm&>!_o+19L^Jj}2r%9J`#8ZiFrj$}}p z%tpa39VJeMSt6{UdU+S>meB1SPKQi>|0hDdd9jI0jN`m~tQ5LzPiNdM8Pl2OND?Iu za^5_BK^U_A(v%bpTV(ua&WaC(HbpE;|6}@z4Bm_kuObh_I_n`Ln%z}CHK#Jf?4YK_ z3+^xu&p4B7lV#6yvV0ew&%(afD$#R(Sw8mh=hj|k8GMU~zbEp;J#%iPE+JTF5Cjll z?#7I_wJ!f7Hft|bNjv7lJ-HQmIy{`<_Mn^G8ME&_DnnA-vr9e_iC8f$bLf1H<1E^W z7(vJEGfV~i722TqXLfhw37oUIQ5dN*tWlwU1?si^ap6$e z^Hulu#3mE@ynRiBIt^%ig(G|#f-^NWh75q&6;bqD;`r5er;y+V;$NpgBGlp>>ur9N zHlLKyF~dSN86ODyL?1l57l+Nd`Yr!Rm9n7HX?Y|sae&$A*MZvZ`_LVrTO~N>A>s82 z8q0O+knOm-e642)x<-SM-Tk$~h#9oNOl$lpf(EZFv3Is0r>Eg4E$A_+_df{A@;WLr z=M|rth_1&MvN`n$3TD`jTTGUfN#?7owIHSvId0rXYI}TC84HkMu0WV0wbe|SS@3Zp z=eWGT1?@aVgyH@HBLyBo&wo-q5@ACLrkMYL?Yrn!0l@CTkv&7rLKV5hSk3uoNy^T2 z?<}%|89y4oaoxN3pA^UhGr~qr-#N2&tmeOKEM``7@Z@71Lr0|y|Wo;~o$_Y9kn$}^mqR%)Fu_Br5VwJDP%5cXsZ z6F++eIn5&|0~cb@q`rQCc)V-%jgf5m!6vJdub`WpU)ck!nbg_6qMmK5$rO+3uIy~7 z^y*r{$dS+X*^{q7QFk_MVWYx~Dqr{wI65Oq(CJ&T*aHf7`IlQPc6uXt61L!o#29 z)bFzAgKq}t)8CSg+t$X?6Yyo!jQuYrU#U6etijBh5Yuich zIQE!c;9WhPDNCbeh8f}GBq$}5z2Axc^nWNS4{N4gq~AvV&?dBUX>(^&!{~QjBd_RK zMbC$=Z}yB)kK+)9Qgut|ne$t`CpA}9oQYe%FJl_8V51;J@U|0e=o!WU)x7FIKK7Oh zD&Q+%e)3f;t$aXJmqWsP=CRje4?V-A%koshPgavdBOl9+Pb0GMN(QN~Q5L@6C-{Ym zv+mAm?YCwh(&M%AtP*8+rc?K}t*cL{!C9!fe3ydyc2#7_z7_rHFi91piW(HF>P@X$ zqD#@%yG9jN)%HAh!&5*t2{C&&gGRy`;3t_oN>G{uGqG<(oLH>vj&i@q(}uB6PELxU z-Vz^~;;<}NzF`->ApN#e%9Y`(fsu?x zGeDfLZy3FgzJBR{_4}48%60$nbgviF>QccscO$~qR%FmFwp8!eM#F*c#W$&<#;r#r z4kU)YPWrb$H@Jy>9nkP-uBdAVYvxPxjQO!e z)ll-k;{b`C09C@i72eI3sPTh5D#AYTpm9&Uu{_;+>8=Z3cs>?xU(|9D>?f5kEpG0yexc}o9iXWk7W5K z9M@M9`6g1lK^*O^_U|5wBu*1)xVhAX`JN#wwq)VZmIoG}9GUeiC5($a7~xatk^aW& zw|u4iZ+csyY>>p(Z4aZ(Bz>1HKCkWaveApWe--hL#jy_~mk5PEOS-Ri)So8dMw~2cu%?}rWk>{54Vv_#d)B*)dVUwLRdf*tx{YLR#I;=9 z*}n6w(lYeeT7-rVq=a-nPpy>VTL!Um$QbX}%OF+u+eTO)WIfh&v;$TxcxAGjUuG;Q z#(QP<3_H#r*KTwtmd-d4FY7!ZR%n0u!55b(GsC(sg_39mHC*FQMTs*(pJeR$DHrh~ zwA`!urpcpam*fw~(no{Wy=<+QY21iglNtG{{Pw21P10|fJW-zb(7d`fr z^`zOaU8FVkKx~D85Oz?RdEVTIm>);3-CxT(7w!-5d4<;$CayF#ThC)HWnv4c zh%$Ki_y8JDbMVQXKG=`ZC>gaFw~~RAn1Dv`d6Q%cfU_KuCzXX|SD$}}zSEAmfsD7(!8~?n(4F*~A%F449P}clwnSrDsXc+^!{#3S3$Ls&!lU z{&iqa`B)4z#VAO1c(DyTUs9GxN- zqdVmBq|Hct>HQ|R)0pk9hOoo#@QUY-P;a(@P;cB$Z^5KOvEPY)%BPo3*=?iJ_rdrJ zA`Sq;FA(WVBL{e96pRQ%bZeDFcwiKjkpn--St;tv8jv#d@Xd%p{7K0H(Ij;A%XwZUk>vvr2_!lHEkgFxs}g_61PbwfrL`8wj~-PTxv9$>(A%hsmSD z!Wp%gjp^7#@3O=AkF2#UC3vvpgv(7Bt+CDuc_!EjNz~gngsiR6=wcB92cwm25Pjap zrBmCF=!O+CDHyS_VFjt1^{Ot5z=W_Z2T^}^m3RtM*kOcKesWHn87(9xWitziYS(ZI z*Cuq}qBPgLW(WtYK@}!LO?d`~>+Eq#C z1~R0%+J7&RLf9I!VOiX;^ZNy}oY1N&8gW5-u`b|%1=&nYCG$_^3Bp~6a=+N!sTe0M zWu?%zYBQtud>{}QhHM)1|O%se0$2V4ap2} zs)O3tM0Vnwe{V~phPCheUp8+VzK0;J2AsI9HQ{9%29EW=2Ij&$RtqSRGUMDoZol(w z>m5MSegANup~rjoc(?oHk>U3bvhXdLP0;e)ugnid0KEd0wey9AtvABe4~4#BY}EH7#@X%wpwaXh48_Ov$&9}4*dqlKg{a?1 zq`m9vyV9Lm_hh$7K-u`-N8Dyt)EChwSwjI0Tx=o)sLW-lg!TDfVp7?lG$G{I^Q)O6 zbiT2h@uQMdbAzwJXTH8YVIKn-vRvG7tYC3pWw+$PFAw7m26otMO_zv$p#*{vYrFJW z6X!qQzLZ*uO6+4Dl6h4Pg%W>okE`n7@YH;@6XOxPYx#%a)MdrtD*}e$UKpKq`uRwS zb#Lf#{rg^6>=1%;6hha0)Y?v4eS!^cV`BLWb6NCcW)pP-B`A21<9pe|{+ zbXNbs*&IjFZJYxnqQ&9aoIZ2&^>F8j>){xvl5reaIAo0pgbvKJVn;1}e<#}FB7q1} zL;*Xk))TavD7#Gpt$oM@)+I)(_`F2%sRr=oyax*kGXUOq*&k(;{2UxIRp;S0QA_C#UP86_@(XrwY4XD&eXiQK> z8I_++)9^wDK6`1vHBR@)3;Y0u1$+#Um@H_kjst@e;5j&{DDWUbx&-`RFzwT`` z?E)grik?M~#CRlE*sxYUKv$(YR7)bLTaRi&^cstJ-T~?o-%JCjeb8Y*8tzaPh)yPW zAepqhJo#!=uA#bdarRdTwF&P)y+&p4;td-7d(H3B^F#Cm1I$@0yR-N|nnI~Jw!dFK zJ9{ezsKo6257R1aPWmI~=|nC8HF~JQ`A&}Fs++}DH%d>%Q`^Ip$hZZ`b==lQ% z>Py33Gn)SJJ`zmbhi*^NFc>ffr`DnG^W)5BC_xn+3Ft0{wewQ-zcyJF(Xp?JpmmW} zN^*nEjE=G_vueyqGHa)5@-zAFr3VZd@$p39g4N3K z=cY$8!?&CA*j`Fa{H!3Uj;cK?(*N@!dKj1K%*AhUJ6^re(;dg)b;d{Gyiyg<{w@HR$mq(8=jtJRU#6o6OvButEJXaUflgz4XGq_ z+q0SzW=K`6F80WDnff+!YOsk-QNPLWZsMKzJ}u%nz#2q%ZY!J>Z(7g4rgJ)VG9lMzeb{D8{m`pm{UE zZnCUj|1SSAIxH*oFI_MID_0V_%jLP7==ifqQ5~kk+ z7^;OyO%6~j7DEj<9y=jg?*1`gZxPS(pd>~2mH?Bmsr*ac*{4eBZE?T>17J@9pbkRU zGhQX?ypZQwYG)gXM{@wX%N@(r7ciA4$pe5}T0+p^Cwf8xnQ!qr`}%518L&tqxJ5uC zQ~)ms!Gwbk!szxnqXEBE#21B!hqsnCxF+rX|Mr})FDfiNe0m6D>b^MWt7g)4N4v

hq-Px+Dm0b? zv%1uPon9x%Ph5}_fXg&QFY3Z6wNBl_1VskIuKx3IGWfWlh|14zma>il5)Jp+TSK#> zf8Gd_fq{{*5@>zARV+Y}fn}-jaC#$w#U=0>d-(}4ROa6%xsCqcq5oxzm;(Ac5u@a* z3=fibzd;`TyW7AScsS#&Y{?b_#?>G|8~?{Hpy_l*&-jHj1-kv8*9S$7&nYsQgkE6U z=2E<1L4t#lhNAm!=@-n7Ur47F#?K@{a4-#i0qkN8bkNS|A)CLd(bMt&r%SDgixx$* zlp9I@u=>wwH`VHJXplnGPA_QW4BJ<|I(u zay@e1?_TA^%sCX2e|l`f)KlysK-BUFadPkMh$7+opQAV&|iGchU3r1#sdU zV7i`um5+~)g$0?h$Bh=`BNCyMD$oo^50ZbW zS<}u>g!csD#pGD7buHLpbP6(!`Hu$el+Rk_b*Uh!1Gc@S_=(P5&66-!hUJsQgg;(o z;#6tHAS}K%ibwMk3j*R`VDOHn4UwK*loKn6<({nHkE#P1DXeChGmkeSJL`oN&(2FZ zDKZq|Kj$X@LyxRZ2Q(%n{R00Jj^YQ>*x=h5KxI8@ft*nnw9~*HEO=oQZcF!fl>v=C@lf9xqEo~X+Ge4_I+axn@=v^wF4};O#)mxGFnEFb`b z-EEmSn4Fj8x=&9m;kVH3l&YPL`}6qyNbZGOi6G#9!zf)9;tMLT!CK~;{FRoJa@HT7ka*~B`Z zmo0O5PUxklaay~3cCYA)cwcLOb5)yrV@G8~G5>4ca&b@KJIi>hgnn$?rMSm6@Hw?< zX4U7qL>bmxqAYCN_XJc&Ka!~&c&c9S&Oo$_o2j(dNQ-#hQ4xAm6BTQzEW)$b&`KS7T_*2*6}2K@O1G_(Cd>CgxLdyNermOF~d3i^p&t(NDYo*E2M>QyuzciS_RXnJ^5 zuULA?F~1`Ls$W)TH{g#ayDQn-x_Yc$x7IA*n3%hNIjKpLi+h1_c)k)V{HaG(JN_6S z-agUCTyzrb0W>J&rLe;iG5<)Njyuiev z691L=#*H^c>UUr$fWaSt!NpMyvtbgOA~+tZy-iys#6Jb6&D*f%vUN?dp2wRywY`W5(R_4|Yj08oA9mm@pN2hVt4 zqj)&m`9D;O$8KgP4OWGV{q32Cs~cL6FTjn=WYY|UT3_r zI&>*yf^{7TA|iA2Q?v{6oLqK>ByiD{BAV|l^d>DSp3lp^2e0)he{R3Wn`HkFq`FeW zZsM-M+q&Zn8M`tDHyWh)BFvmfP`@PL_pZelm3dx%iG_v3z}cNZi1RWhepb*Beo((B zZp`AkCqJ>}`60gr5OwS~9G?&>{NA_El6#LDpU{UrIpuWFjbw_rfCJfafRt_!d4*1o zY4yDX6U;1cxtin4W7?^K&l!LTGW_D>&@I)RS!9fBQTb4SU)Ry`5Xa|E?4VzVpCCin zLE}LJ7GvIbzHkV9<$(?4MJPx8TBZdBBtAf5#QNsotBw|?3xuhlSQt%H&itbpDPT2V zELi>re1!n0Fz$)gjq4|Gy^$&X+w!N0W(QuALd>w@=YEH!&;Phn<~54Tj{5iYti}lW zr*Cdsu)|cVit}PC>BW}$J7Wl2msSlE?spkUP6Ap&F}GidUdcAJm0VNm_USXD3VCwW zp3mYX?s!7bF~9NQN;kQ?y^D9{1D)^Y$bK+fF2CZar>*A zAYoGk?`XVlemX_e9y#^+LCDQbr%J>Rqc&velKlKJ>RVC7buf+NS084=3Q`$$``u(} z(TfeJVl1dz2CmD=YZERtc0>1>VeNSvj5gaOo}X4exoo3_=`#DY^v-qnIg8H{Fj8H1 zKQQ`eY&;F|6)R6c8B}|TkVDHN8!(&m<%xWYKilhj)goRU1v;5sDa(9?KJM_sk~!eo zX`jIcn=dFdN!;9}rFlE+8xgSXO3F7AR7X=Lff%%9JD^d0>g5j8YipDJ@p<`2W$|^X zeDMX!c20(+Vaav*ToR~G*oZ(RQ99pn?8n9jrxSO`c6`h#9Zhcruem#V{K9 zSbkY5&ZS`BE%^B8XU7ON=pPh`7-`QAqDhGC3Z{&=D}n(SIwK;Y?Ie@oyqkn?*pyLN zcU|~%91?X#srk+@;i{>%dHa_C_rSbU))oDs25Ltr0)A0!Ss4IbbF}Dt0YpE(_?UUz%fT$wi zn)Lx`-gQaHcl~((QzRwoy>YN)U;dPA@DW3lHluv6b>vXtmK(}rmZzZ&QXDgNdfW&; zwlw^3f1Nx?27L!WZbEY|(I{IhT0afl=)gm1ZGFcfjoLSEd|k6=U{jm+nR+9&Zsck! zhicXUUPw|)(#K^UvZ(jThbx!uChQSX&T+dP^t6k5IN@y*Gt@gPs4&&>q3z_AJHrdX zkL*#m+`{WVm1)_{G|&^E3^J;pX;5$R>S!{2;D9|8EC2D*Q%fAzvVVg92-{d1t_Fq? z2Fl9#QUb6$#(RHw5YCjX$@o?B?1RHzAKHmCa1b`&>PFMAIAN9=^H z-Df<)aM@Nz-pK|$EWI_gYHRoW%Cu{QZ@|RjCqtB5n(U2t5g>t?x_YiI9O`vntd#mH)RH$+P%|EhY4pK=i zSToWZUOlEgI_7PBX`Mx{LP1wI2+*8CO^@U}vjoB4Q|J7~1A>q(!6?%^KMJYWM$M8C zZ+(aqvXRa?eq=L%ex;k{8C!J|L!Wq57f%7@{&M7jDx)3!7#8Z*`tiqS8^Jm^7oyDy ze=&UT-*{%dcX;TeIl@W>O@+d?Se`^xV}^r?>zWSrx2?~CAJCr@_EOChM`wvPqZ8r< zVFY|u1^M0gjvP&k^#!BAf@_DH!*c)?Fkg1(b@_cVP}b75%mCfFx_WuhI|S+<)abCJ zJaOl(0-Pv*L88BLx9C$j&}VjZa8!om&H8s=4n$xCD9*hiyN4F{`myB1?R(0!-km5< zkTPnJRJlDZ_jUOFpr7&vOo$XI(|(PYPQWdy7pfBp8seSTMBH}r3P_+qh$B#%KlvZ` zF>+-AEo4j3{W{8&AcIC&P;o_&Kvl0+kRW3o=dQhAYa6-X^W;-N(HK|Fk62;lc1IfS zpE}d>|&p54s z?+iXwkVo^ueuK*GpDs6^X{kTOKYa%@#0P)p2)YrAgK?;s5BS>WJRG!GK|jD8{7oZh zm`S<@z8F#n`0xMy@0yn;Hhd&>9d+$(Yp-=tIn~N_Po6f~nt}(kf-ZM^B0@CmmU8BcTJ@iBRCr{4{LuH)@TS*G;gxdlyoVLuJ1LU+ z+Xd$D&3rKpCJzj61_+Hxg9~vtsZabPhD)zszaAVMJZfo5+F*CW3&8~Zo*KBt(Jjy$ z#e}GXLBTo0NX}h}R|&osUJCwRi+Ds%0#)6?Xn|Y^bmV^WMQ0sQtu8oFE)UYNqy&UW zpc!OQ(Nd{0ZZh(ILZIsg=F6_=#`if>%B9U1H0eMy>8pEieflv!%}sAW2QtX0xa*GE^FYxZJk6G*TQGpTSkg&Dy1_y&3{qpK+BN%OeKWq?lrc3jq{?HPL_!@o^A=zGS8|T^hBeTDr z6~-#%5zS}3*F`{#*NS|py6AnG?kM&i{CPSj)UJw21y>@=3N?rwX2w=?KL`ztDMB0gV;Rx##9kw;w8mYXmHhyZKos9MvfXH>Kesvi^b_q0Qc7zH($@2?x zuY2)qUOTqIW#)de&;Cu4Ld(%l5-sed$5$sRPv>d!sUA3vjg;*iY=p=VqJP6^S@+36 z&#|6$|3jMTn&$zNe^oW*6~w;j9odf-@F}jhYr8|2&Avf^?l1A6TA^)~k3KNhKVm?e z>N9=vx0%@Z+j+d6$#xH1$|?e1?d3roShT%1uqWX9QvpWo%)nc2u*ziC`_5JdKga{9 zaAgns-rDc#}ZZnvOWF@ZdSygVCn^a6vg_+M)fi9-78DqHcZ^&On!s}8J&(V^1A#8 zb5?!*4x7PJIvv&bnTm}BFee{~`)=rOjyaUzEk4ooO!koDGaCQ&Df=$tmkqBRRQxe* z(JB6L(!DAve>hP_!C23m@9yg0y{0dIns2nKA<~b{UIHQmcgkYLh4iAuhV!k0C-BcN zY+gG?nJnA=G&BFrD~5WRo=EZv8}sfL&AZd7R>jOkoGVCEf;6J+R?h9dyStF086v=Y z3H(5;c-4WfX-+vhmH^SRv>WGl#|8<}gtt-ef?tXUUdg>jPO?zq{_Ty2Y)?GJeN3um zr_!5pPxhdCfM>0C^?ypWZ?eVJaBE*>tQAjtF}Zk7<&cpQcdrPwQ#>Z)^M!ehrB_g` z=)C*=FW{V=!|dzwHxomu$F80$U%p9xom7wJkSxH}{#pwe{4jm%Goc_(WA?4IBF`Yf z)AAzs@8tyuN^wrbnna}bnb#g6VWD~wI=rl>nEr*RFO>VjOEu#nJE8ELiMHFC*ITJtTcLYong zVN`M1TIS~q<;u-KGnhPlS8`dJiYpch|8nn^oX5pGZFCJmf!hZaN6h$54d(TkTY2? zo>9|#S{&X063Vp4VoABF!B3n&Qh-m2Y%NlxQ>l8~iDN@QnOS~Zr zB$YBU(6@g{LqP~2@GHjJ5pTwk_SvrYRMbM-z~UeS;snpFp(IaB&lcQ0nUchxA=%hQ zo*X0jo485rjPobU0-2*l4(Yo$wzI33GN2SPc452-1Z1U3 zZ|uIN-kG{e?Ih1`$MpldDKHKoH92x6Ex zr6{u>qQKuE5itULr}3A)GX$q({B^;sDnv0w7YX%ty{Fl6w_&A$IjLM8-s@Ja(%D9@ zHs{@9Ej&vsyyt4XZ;BLe{8)S`%0jF#&bct;V)vbA3bS!@{Dslh$zOGVNCdPbkd{KS zbP_SC5tmg|#u9=k@y~VJH!r|JI?YTqdhihcMuWT5h?#1r4 z{^BN)@2AC-4J(gVge9axmJqNQNU*t$bsz*m!Q>TheLLE_x=rj16uRc}UbX~v#lNiW zZ%Co+`~E=~WJVDe*lR)w@b`1<0 zddgSiSjPwe5E-A`rVS!nJM*o^Dh<(iAlXHNlux91P%Yxwg~f#rWR<&4{D$esG0WXQI~F4Q z=;>+W(EU`S!J&e4J*$|U3_s-`>k_}xLqAjD`cZ3zOSdD8wfRvkq@_&9tS4(eQ{c~i zk?7p`a1FA!158vknE^}*ypL>zr60z+twtZ3<#jih-T3gb_DxT$RGiV7qS?g*#4P7W zgi$S}S%t+WD5)wJI0agdmeT$K7uOM+4@Sn{D&7WkAb^8ihJ-5OBpsDItzEijUb}e5 zXw-p`a69Mfm$G@Dq4_QQJI4Ll$Z@H(C0wX+m2SrSK-@HmHA+wH2`}iD+_3jOzx0(p znM6A62+F0pG zn!affVG-{f?CqR|cV-dslR|I-O$({-!AGH~0iuG4U2vmfeKvMGG$()O0U8X^-H%XZ z$Q>#EfM^APa20|X+5V==s2yC(ZjGJ1^Th{Dpqncr(vPPJ@Biyg?RRZy8+&A z^1p#B1X#!pJz@WsOoW~gtB1}?30S&h*J&)~sWWnKft%}1+OVYig)QOUg*y;?+HgR z4ZD(dBOwt4qACLt4n?w~V4Or7I1bs*4G#zNP72pOajv#!41C|>Exk5x2E2*+%naj+IhIs?@X`J375na>!!YuOyF`g$D~i<6^lI6smxgN|G!d$&Js&O< zV-B7+I>mMRty=IKSC-c)EP;<1DrTp?hX61fkbar$=P~he_ zEw3GP=i{B_bp-)pu9b_M08OVLqq-=9I>CHfuIIkG<@$W`$R))OESeVavJ#%{>$2TG zCga^u)aSBpmtS=OvVqPu4Uqyp#V)({ZtKGoeVdQJL+e&*_{rZKXw7t9+%%(!zIgaWye@woi}Kf=@l0 zV2X_eP$NSWy38d>A-l!7SKC5YdYAwrOxTelR-K==i^oNZ=*2LL?2&*{VgDyfHK~NK zskjA7s?v%2qyn?i4qgT0YK`nuBK4txoETlxA6d(K9+zmME8;#;kz!{qm-7{*E`7XF zDeis(%I@u-TzN}+3|Tj1TT8lLdy@lt7b&E!Jt!5`YVQ&ow^39UTmUcOjcx|Cfb%~f zLmHyK>F6BA6-_?TkMCBF=Q~&22P|0XemM3Rxos*AJSx5B)o%9n28eraejt5E%r$aV zY-MjDgTk%ff!zEcfI)S+2%)(6geKtr??(!(QBe&=ioGghcoy_b)`3 zB09-3mH;l%TTJD?D*ky~TQaxy^-GGhc@(b$mRQ&>aiK&fO>%Bu!n zUiggsvlL*03KzuiXxI2Fy1Fa}4|>fFYTTwVRJw>FgL+}+CmmO3$*;56xXSb}f+e>? zYT~(K_3 zcuD)E+IazK)x0wznM0U1MyRPTb{%LbswTwA9tvXqFX*T$>+v_5>G|oA^r1l#C}@>$ zj+`8}FCLXZg)SLmzBx2?rxYp0km;lazr0cjp)k^$0v7I3!QnTko)murV)F~Pho6Dv z;K;oP%tYVqxb*ylLuod1osUQ3Q!eGeqbPuq``9l<0j&2&KU=Hj>}|U*l1Q7x(%sSP45l?uF&#iErW6Wy5dipW4HK<pOfz_W;t!ngt2$uR-m~6T9zOkH)bq-TfRWkRMOweq6TwXcr`wSd91| zKppjth%Y2$f1sZQkBNPsT#TEmSWCDkD+P6tMt03kqZsb7Cqf%DgY&P1rQ{3(vL_g(tMnZeyTNhG4FmtoNqv|D4+TxqqvC?4>_?Z1cjbPjxGC zqn;evmO{y^UHfx}v3I8sD(s%liJhPj+sj7Pb*1jdNzej|XO}R$3eBi(AU1pCPHxn) zzo*?^v5uaf2NmQnE8~(p+vXZUhUdHb!H|1g{uW-#joPc&Xpk3;_rF)M6jl+D)Qf* zM^MTln}^3ae>OjjEKUzx=y0YR!bhmC=)~q~lcYUir8*SpYae{^Cb_pQzPG(_j z`Jbhr>Li(r`I*=FL;$2^wYz_7x-?tO{5YE&dP@zI^2 zo!6S`A{XVh1OB;0hBXyJ2~rq(IOSg>GM6(3J!|B-W-H1ecr&i3^XaBpTMe`t><9SI zFrD`oOZf!K@x(rQMGB^otCE%c&-m3Q2K`C+twSsnjmBR`4XCWSTT3LC8y5)NdMS5V zZ`O&6-(tk_S-4o!v!SaABXatmwFeJN+r2W3j+(v3TF)PG?cLnTV6PJSds3~bf(fXy z2eg{%>ixWf@&ts%6(YPAS;gb*l{5hDyD6dhBv=S%;b&~s0R|xbx^aU*(_>AG!$HS; zLwd#p!|(e&%XzB9`pmm5%+;9^P^zHMrLr|JwDWxh$oIk7qW^r#_%ZPc{y@zqw*Gvd zZ&@Z;=6EfyaLz_zj)zk=&vuU7&seA)VDyhMxo7Z*7RW=1<&l$T<g*?pxdhxfIElL!5sVna{{7Vj-DyUJvz$LWqC6oYBLKzZ(PlxVsi;apPNIaoJh6HX zKxw5~gqE3PD`-%c`hXzPQIX;6La6BDxyY$Tv$q&-gU*^cw`KD0eE>u(c#Mo~n-ZjZ zmQ2ae@SsCpPG7A^=~ag`B{aBHA(p@|S$lv@1}_suYb{LhA9t?rYi6NRH{11T{%`D> zJi4T2C42jA%rCbZsS6#ip|I@Grrb4eG18PMZr_Qy#A-!(l7&#{{`hoz$iVbXFH;L|10NOUJvz7{%wDq_a}N0=h;!`D2-ud{^V)9$(Ra$)TnrUaSscc# zPa4%gg}DlURso2F-(q}T%wQq*A4V+_nJeNcAKonD1R+CiV$I|D8>#z2aKi17@d_xt zqXB)35j#i#82835e?q`c2$&NHp*r(H$TH9c0)}^XH))8#+@Ly%PaTb!wb~<0`9U9J zVU%s`k4OF;Ovu!jxCgp2AbTuI+|jZJbAXhvw97mtAd&+02prJ2x%(L5wB5sqJ}hL< z$xrJ)X@kem{^&T;jMxRK)X8IR`x+P*9{HOC^w1VOR*27u{@{d{cy5up+OpI8P5MA5 z_2Pc7-YrQ)(TxGYiNc-B%UaoaH@ktyz}q!dzN*>nC&5hh zSyv)9>FJtUS{y>k2qY)SL=)BvQRzFg{B@O&leVLURS#{Z#v`aScmDGUxLN%t!Pb>! z5PtF_ARGuk_sJFM1R(%#u-bRu?tR!;q%#wUIe03Y6_OeBQG!G>VQWEH=+5*r=sKQ3 z5WKC}$1Z98DNkhuN#;lUcH^VMopl?-9sl`-GdejjBoDShoie$Ey(3`>)Zpc0GWS@R zxr4Di%BHccB*{#WleT(TbnGVN;)q<}Pn*B*kp}*h<6>Lq_!?uGNBGTr%$J%1!71%S zNEzYk&gN{JZ-TO4I@GrREF5rAXYEGRKUW!BfL|qOjQoD521a-DU(ixSpwKcw@F~uJ z_E8#tPS)hOZ2mK|8|ge?Ses(BlU*ac}MN&)?Uo64`){?Ls!RMuAcu1GP#C%Wst! z!f*LZCKj1R8zChS-1e-B805-Z9VCOf3OkU~Unk(99rgtV&hgU3o!^v7`C#^bsux+; z1w$D3f%s6Sos(%*w4+=J79#S+i@ZtkBW{QOpu!uIFj_9Usk(v7(of4D5`WhHiu`W( zp6vsvh@^Q41ib7TV)AKd26;<}{<(5osId2;CVV6thiW6Tsuj0IJD0Wv)HMhQ0>Uck zd;L~UB%rM?QYa0sFchBmp6Jch&d!?r;chLw{m4c;kBkr<9>MYwJ_rt~_yYQPedM82 z#k>M6 z+vfX(h3zILydV{TL4iPkbNH40+k-*lC6zKP)IfD>RdLaqLa^Zo@`#R(+TDqLl_K#t zxUNNv+vlBP2OVAFUk^xq=pW@nvlpf|d>tx}C=wSOqoq)|!EN)Rr2vT^mOO(FJFcv}%Q#y{I$R6`@8^jOi z$#;4AFN{ADIGOO(rU};;Xywecyu5T@M53^+Ip=4g001eFM`YL@l%cVMY%lx%k#MKVfnNW3;Nqd$M4jXT;+y=Qvm%!cX1_ur zT}G%>g4XA@sy%M!ynP4G321Zo^PZt4K-72t1tutS{m&gg31cTfyLOqpVJ~6^q4gRR z#_kEt@as{z?j{Hov6l;-i<#zp4kI3z5k0wO#`)5y&EPo(6L3+hL71~bMpZcdeSMWk zw?$@8${ZDVuitW`#+uc+J%bj8?1y~#LHhgb<2!4IEM4U8)zx1tdg!Ocq$)ErQ0EmozJ=JV=(iJ@u-kTBYqstexwZMUMfT(d^8s zu)b4;V(N*ARrALE;=}%l_RHrrQ-^T_TUFPp6CbXx>TOZR*LCM7l`Q4X7d}k;e&LNx zX=U!H{lt8(uAWKCxYdEhtRa07n>PV=4W(C{(_UUHKDv+$`q5KI3$s$jM|1WLNeW#@ zA82(}yab8#ob?bj#+z|3>=hPswC6o8IO8gn=g`E{ISaZ#N$SHyWr5S9!mu`A1Kb#3 z^Pm=_$23!~Z3}A-yu{u=%9p+}U{W-@|4?f1 zak7`2gxm<4`_=Mh946~K?koJ^aqL1>K@0U&&v^87A-m2WpBQTdpw&!iFlYNlE96*3YB!XxQA z5dtk}1TX_Q{?PPwa+@EkY1Ns{28C+q%j1tcvvg_S-U@8x2o8FBTBd~0$QAFW(pE03 zBdxhGF6J{q@d5M2P%6Eia&IbeU-sv@;Z=}2lwb}_VhV`KCo^;S>bBZ`2=qN%RMC0k z9V#X$nPFl`+FcJn7IwSc5YUP}X8Gxq$MyErZ&|{v6F{>54d(V!A|6X1A|M_S@y{C% zVB?ml*$vuF+otnxtsH@*4QndzrWpBC5dsZF(zsi$AJ5vF)+L^galTvJ{X+;lWBvaY z<1Mq0NFAQq>TGo3v*1e(STKogtmSD)X5TBIU4{;}#}&vLt3p59>*>j>=j4Cf9@`HU zVP`=5OuM)}`sXQ(&j7n4K;t?uPof%I-UA}@Pb|&JZvk#BW;RON5%fm8rzl%=EMX=4 zO7~Kw;>;K|C#skf7hvyJJ$UjWJ*eS5KV=|{y6Bj3-lPej{%fqE9CwTc3C6tqHL-Xd&cekBLdX)|-?3}a)N%IA^JYJ%{ zu`sqJqAs<*@ZNa+Bu%DJRAalio3rR4n3DG{HMO5t z2G~AGG`e#}E8vV`0D^6NN00ral|mjbu{l5X0@;L-dPJ0FS?SG@mUv8fI zFxz&bw|KN@>OU$0yLWQwfd{QAk#>_=JFoW zBNB7qKc_Y))~?b}y`h(wFJ?~Hul6vU0x>q(>Ec>>=CHX@EA*fWjKkVRD6b;k^tnV4 zvJZ&kYJEV5$j3^KAH(|zo^}Lekt;Dtx#h&Iqo}QgNi^$jS>_+YmJSCOYCLOh89xsE z1+aYv&8rmn!}p`uvi%DBg`(Fy7N<10P3xBuH=~Z*l?}f5FPX{@-EcV@}GqF*A=i(o18s-NFhw7<9jWU7o$=QrGy9Go6{LZz*{P4oC!% zDkK#OqIaM-_Q|FQ@j~(nGz6xW$ln5&?-A?CEPB{wdkQ3%dU zmx-TpERQqSAQpn*Ud~&q0so1$@iRr)EISuxjeR;N{P0 zqKZBNWn>#(9{E4usuj0o8o$#@iyD~IrYx5Aw+i~XIStU zwO6bJ$n}Z6t9+*|uMzikPz7AnvtJOdM~b*WCo!ilwZmex*q`2v=^1)lKcagMD}Yg23NvQq(O!4V*=A6NNolJW?kxI|M=r0EPfsn|}K0^80m z&w_uRVf3mSCHrIV2~^8|TDB8Ui_EI?oB{fi^!?*)ZJ(p3`$(iSTc|rg*kfqrd~Rmb z`CL%g=yr8Yvv|SCME|uL3>$yk`7wu&@3~MLCalBHEAM08?@bNP+Gzy;%Ul^__vK39 zaCvp>u|jQmmpM?sBv9jr0t@%-&FeoB5Y*o8iPrNevh_U5bR4WtX*D+U#G2OJ^TVDe zP^o(@q3QDndEPq0!*caWT8QJ-xYONN@#loTQd&16J2TX~F&mUOK6?K$ZT{<7qvYx= zVSt|Qh8%)CMTojX5GhGwcjw$C7;hwCpahJ6lDTCl3eW~RJyM<%^iLj&y_j%07xbb{ z(OSVnJ9x8RUzK$dQHOkWkx7cdzTN1p-5VOxnp?nU+5w@zS}U@n_jf8z4^?%T zD;-KtyDDv;fW6=`U^orvy%?7lY$0dFOAY7!8gvgi~2Z|XEQB=l{ zri@}NY~f+lnFBj65t16(7YUg1_^RPn@xY4rtyFFkBI@XiGNcI?pt%;MVU{sMxqvm* zxbw|-5xQ>04vXzg53Z1k%b}AByZB7_E}6HVd+VvESt}mV3kXd<@V}#sUg4C8rKgS#Y3;T9w<8FfR!6MSq@~{TehAwiPFEA|H25$6i`69fOR4 zFo$o{qR?q%HrQv~H=L1~xhZ{|)@NDa`gMN68BQlgs)+jUB0doQTz_x#LTBks1h@S! zw(LZQuN@&JLn388KSkEs-JXaufuPXd0e8R|7N?-!L2C#|HGOY1HPTPt9%5e^hY1t} z)ViE&LP5E4YjbUDy?!gK;-c95p*YWb zyh-;)h+xtLpU7Pd$hltay5(D}a3;0QUJ9qgWsNzvjQp6o`L4h~(tCob^}B#1ggs0UozhwlBh8`zvFQ+vec?w% z5w7#mjT9B)gsiMA7#FloBzlkaGy#1pYMYL@C!pSw8251_)jAzJomv=?li))*`=+e; zsFOCAot#8vcT+;$`AvB<9!m!nRpgi(x9RD{eMMGw6(m_ZhD#A&S6ZfkR+tI+k0IVTK_m63@vX}pme z?qwIg7ythD#_Dr9-?m7)jkP6Bm@Sw7?bh1j1Pr>I`RY_RJ5W24LtdL7vx{KZ>7dMb z@BaN67)GgQM=+#xT>$#uPbi-eUT-W6&h2o+i*Z%}qH=Nh8l|2ok)v+^ESO&O$s|N> zipQSQRS*T@k5H&w>#@WLqBC!JdN*aerTPX-j%K=umQmeTf zLFGl@CYWHLgb*E7=uX-VRhMHS7(LuH^^2Rb4CFXHPHvGSGAsQIiXT+*8JqIBqN234 z$wp=NM4e+Q@_zF5^2iIS-7>P|%lITBDZoPDl@zIQ}V z}Mrk>-? zJVSKW*Qaht&KacyhF=t>=zn1GLEoy1wmYGg`F8fFz1U35fcC2KUeeMdEP)BU6uwt- z$Uu4U*DWSXCaTonS&Dz374(rPywzFsaG1A=d}n_hyxQvBT==Gax=$9Jde8#vR=o$g zzF?9V0|T?UP4})gg~z8G!nc2uEKU3*a#Dcix7 zP}Fb=dvkF$e78u+nfrp~-^mKm@fa{Nx*x68$`*myJIwsgSzDeI%RN*=>KpF&F0o(F z=SA%F9gG+H`%Cq%HQ8$++}Z%u8NMId%A}3vHy8g4vmF3wxbSCoVsymqc|9N7IG||JyXND)o zzNeqqf4F~+9Doib#7tK_$zXE8RJQyKZkARdvbr>R=w=GW%X7WK8&BIb@__Q1p#5x+ z?Bkiu)Zi)dg734EvvXr@6V(GM2KO+j5;=1K;J1_SuoF?=sGSyXJE_C+l>_f)G2a1- zZ}D1TVpoYod_eFv+l;xIxbKX`X8(ugDQfGl!fv4BKNiXma~QS%nwkgoa0jT3PQ=-% zqX80&vj`76@_z^Hlm7n(?BD9?mxkG3$Z+31;|k$-2?R+(g!0k=if-Ez9+7&Y*))VhLWcl41xnHCJvl(1_WuavxY4Sqm4uO;8_Q9%u2QvJ+P*Y%WsZ z_W1bt41%Cw!us0irifI%h}}6L(`g#ZfT}+y&+U&v8l?djZ``ik0&v$#mr! zrf%0MCWgVIvkMm-Qys`|l1?%N)+c_xaD>F9!aIR3ur$yk)Ohqn|NV4YZbYBqLd^b| z5k`Rzf@qA+;%yF8>eX-E?_r>$jShVq(&iVf**a?33qlRCCl(jSw+@C&t$CU7pwb_? zKR8cXbbPoYu=v5(gR*+gwx4Wkl>r9VrXMw$iaByMQIwWqR2u!1h^x3JOXjW7n+9KA z$96fTOUFP*zyXzOP0kRAof4RPTkH96MnKkZO*BFYs9$_VAFlSQiVsK|I+t(to>LM2 zLkbuVVODMw?+T4Ev&UD});4dn?r{LnA}6BxyyIgO%z#i!&OlWk2m?1{WV(;j&sV+c zwx6-()I zpQo#?@YLbyHSj?F(GZ0P z{4ejTo~!Xrb)KHe(I3vEkbc%#FO!=h@6DJZVsFG_;39Uq)E>%>Xbx$p%x63Zx9Z2e zyp9V`Uld}zo@GR*(Na!z*qkW^8f$Y(P$&i@Z!=S$j2>Jay;19!o^+}45rxv)SGMf> zFb^fPp&_Rt)xq_q`=#G9q^%{L-CxdH=kg;<^;CYThiNCd`P+2Jb5S6ve4r9t@VD$i z2*7=>9lGDkgC`rAULG{(KY}tC6@Ri?mG!uV4L`iEy90v3ZEnEH_SX^QN>?uJ>xhX6 zX;lsn#q_JBywWp0gZYVn@`oAlCe8RTJ}LEs(={&s6BH0-ND%K3zz89zpFK=a-VO@4 ztLh3&Foap>&3Pu*hzB?v5>FrU9E@DEpJk@~5h_h#F7mt1&~)9lRR&<(IzQsJgN5-^ zE_#Cas=J@ zI}Cw;dt!*~-9j+Am{|wcLry zW^x+ZjL(w^cBW$%FU|3M zMP44uYY8JQJw00i12Agk2#e*y2n7WC)R*4106_EJtC7B)`ySbu9EgzCxd3U&26W+3jGNk)<;SO8Hf4BBkj5L^nvG ze|OwIsv6Q;LawoJ1rCyvaBrTiNZH3#9$IAZVEuY|%S_A{Oa`xe=>p!$bYF1J%B^w} z-hlm7gQzY~Sv3)0BIc=X(*xSVZ@pi;Rd0EbNn8L0Jw6$iOtbk{H|r9WZ-RhDGeR5Z zK`BORmWsPe72a_hm=F&#I+a)mgZ(C0VR`Q3eZL%m@Oc3EM36zxCShQj5#@nNmX94m zrCU~mOK*5LIL*Q%S8RFgOXq4cD`d}jQlqZx-?ixtZ~ymOfiSHYe}~ZZ!>C!(i3@FP z-5@{Ec1u9qUongF17RtK$Nh$qM%5H6cM}xv5J8`XC5%>EXb}?@AJ<>U$wI|{ z>Hg6LES|HJwCqXBHGX;Y`4;7pv!k!LS@`6EGJh$+N)R<9l+0161t|Bto4)>63 zB7BI*cfR%cvc?GW=^l~;|9&ivBj94*tsdgzCL);(wB`rd?vG3H#sN14%5VgV8)-gasELy zf_&~JM!rpk4+V}Eus`As-Tgyn*bwPG9O2O(0E?{bwp;{t{)R_siWfO3=J{D_<=oXi z0X2M@zuTE2yO5f%9R8f9CcBgyT6Xu_#!JyoLKnSxo^e*nSz;8+mK z^73=c?@6W4S+`jmU^Z%+v5muQETUm7T)M+(i>oK{l`xsl{Z8jbowOz<7yI5UtdQ{mg%uZi;%9zNvNV7 znWP(SoW@WLB};iRb?R8Rw*21V$ZG;U8Uq%srakBRbV7TS&n9B@#_lbMob^JjYCIev z7MnYqPt0@v;Ww@EE?eQpjvLp){sjMoMgfvY_u3%Ww;QHQK#*Bajzcjsh~k*JF1ojc z7#+*GbRy330vS$z*ICBBR^gHSX40Y#Kh~=H+`i&hRbO+8<(3aRG=YvUZcH}euYh9@ zetHj*WWtCs$hnLOIhXr+v|hOrk*{dkDti~x7G1X}9y4WmOYDpcr>=W+T_oAD0}VM# z7tR*F+cOp)_USe`ZO)8&>{gA>^~Kwvtv6NCqAUBMJ@?yrKrerR1Nn`3dc-&qv-iBd z9OoJ_YkG`hQ=-$O5^X<7j{<=5ZhzX+j=eopUU!Y0t zh!qA7ceEdqE{wC9jp~+r>cNmn^H8hm&g#;?mtezm7Ey6?+uhBLf*t})&}b=^s}m}I z5Z_Z~cZh9cro4;yxHK@zlcA9?^10+BQY&MyRIg1Eu*A+Rwl!WvF=dyBVlzc}b9J|5 z#^>lb?AT+)Je8^wL0Kyn(zO6BwmyV442NA+6aT!*fi=%{=nQB=6vg%_N(zXW?~5;A z6~6YSyVCe6D?Xe3GH85nog0-6)WN=}uIB^ct=|O+q%hg5h<&UV^X-GOuG&yTh~?&H zt&2VIMIM}IIUgJ}+SE5f=*k>(lqZ5gApTLuFNok#YrRB^`x({O$1alLG$7tm1haKz zjPHxwQvqDwB2ihlg}hXM35RfYP)#l0qEh!!*iqxB&)hM(S?QepeYeBeD?l&%{pN`p z&^XyH4k6eok&i$>(A0NEiX~zTqhzAC=Ad(h)Bf8k9RCh&&=2Q2J3wXMobEbx~ zB4IwL8Odw)_z)iyOrI*byzcSN#CrCxEg4^h{zA4DGLE7|NBhek$Tz`>V9HG>h7Z zd6U~TMRbIi{183WvEt`## zODp?-9Y<_q`E(c0xJD|kHJ=#eH!pZ@xeMzaOeVi-{(9^)I)szMgv~5GTBs}=pt$a= z61Jex9nja>q-H0X*P|2T{G2l#vQ($lkt~t>nDK3v%{9|eVA-f?a{8*9Ynp<#g`1!m zqfx6T=6zpmQ8dO2M(-6e^f(AV+GMxsv2QLRawUTaaYx5;Kbf%V1;rL!3XO_7<+Xg{ z_MVt~!45i-68Q8E8`p~0{d=F0cCYB(aK{PFQrf!?WtB|Lw3C0pG=!`sUU%(&R2*JM zEY;eR-*-onHvh8w{#ee82VS=CW;@ygS@9Whsajs|*7L8rph#>sKrv$F>!TP-rNMTFVv_kp zag+ROv8;HlpHQ{GE-QhvgZT1d`!&p|(i{%9xM8R9b$%Fm4_Z>UMW~*;b|XW@UL4Az zbfkheB>JZQyGks@i_!>M9&%r4*nbSjg1>Dd6dVwv4ujTP*$9GCpmos0|1MaY440$& zHeVHg!^QvN6^lkhpW)YrqCUiv6&nGq=>p4Xjy;)y&|%O~>0u#-C`K_-z}Fz3qDXSu zN^u{N0=FJ}BKgG=1jTqlJL7NxP~Co4sCL?6WEG?4bW{hhk3%dz>{cj7I&gZCC}r*m zNmnV4*DbFxqIUilRZTBz1so=jbfl5+E|GlOw*DT!`TJE6M2#Op%}GM2IXAYcIlaZ$ z-b@*$u>?^_p2eU(n+;=)wx5FFD$jt_W$J|hsSs-*fiGb~@TtwgyI^H7OuE;jFo! z6An1>u=}8)#j${}y_y90IZzQJ0Y4uMb1xpGip9>@Rg^#~1rkjq=iAgB0BB&VONK+{ zJY4r@B(KQ#WEg%1c{I!$9^U5GI1{)vrwd3^gPSP8;Sxlu{y5|vYERLfA3e9PJ&M_y ze<`6Hgu0A|_V9ME~&z#1mJq9Jdz||Yz zL?EFHMfD^6o2x9fB}N@EA;o)0YY+uWa-Pg=286~=(?ktwa>p$ z^D2*;3!aE;K%GNEDKbto+lHj*IG<(xxsF+~^4Fvh@xdBKGBW(KY-JhRjiG({XO?b{3!a$hcs~V=J;V4~GwB z$fznL1wlW7P6&Dw$K?OH{K$<7LR8tCw^iAbaZFSX!3IY_rN4$lZCM)ipq`n{R6@5L zp2H>W-b+V+Mg}UNGqM+6bRU;gyN6-SGJJib5GclcfU^kn33dJUU!)l3g57Yam=S@+ z*}p40Wj0A06RtDm#UG<9dASo2UL5ct6IgucT=q?0S3#zJGda-$9S=x(>qKs{se@V| zbiMCSBM9;J4p^Lr=ubeHfv#t*BMs7Vze29G(Btzo+EK=8@!9DJM6hK0sa~9GV7D$4qXlN>6 z`=M@vOndwD4LJpmX|cWDo$Esr8R_+Y5$EH6CShqBD{X=G@zEuM(aAPMe#|NBS>H4L zh#y}WY8bn!kE<25X&kE!b(w8(jG&MGN8SBRSX13;LAy7F+`I=gXd^C#9VZ>R&NWa| zt~aaBCGmMS)wyKwalNizNO^&$1tS!LL`iqX!2NDJ0i^Qu4z0!I#DCLT-nly00wk*r z*5I!)q6T27DM3X4p_Cp2s}TpQ%-LxBsb%Sg^ZI_GRMpmJe_7C``agl8)srv=_^(~u zLCJ%R@9y2ZMf+8uuc!DC4Z)=(kPH4=qHTaYU`US?9xia1s0p}Z3fD-76JazU!+p63DnAmMd>z@4xbeQTIHF2x*2XexMq49K~4l{!v z7LsMH!W~UY9ZkX=IpP>*cRN;8qrDPI#5#KIuPOT#m>jU{<`NRyPaLVU9LiYj*9CeUyXwBHK&drZx3L5 zC;l4>?!PTb-v@@x9y|J5ARahlGMly}gvrxvf9KPf^JeDe+;6OQ@f@Q9jOhp3w?j}m zBH(iE@ok}=_o%aB)G7iqWxxB)odf_r1go@b@D+58pXl%*KwjNl8?gLKa^X*RM7IIX z&+a4kj5-_nO(b*V$~aw*fB_Now~Na8^OmPgHO`zl1J(Renqi_`lY*(&)|bEJgZx0* zYs`3cZ!=3^@%GS>-_TeD0?uFt2+6xDRNfqTN(8U|(a8_g)yv*tIe;bf_}_n@{Wvlb z8SN2O-VphFKsM$y;oW5Lh}iz(;A`ZHzkZ> z_@S}9zweWiHmwaNWHXVa>-WQ@W~;cslmwMhaG%xIZi3lfCCk48PF(w%@uYNXV=CXi z%d9d6*#ZQF{Pz%*vhi&=>+9uS9*0o5viB`57g^gHcMIwlIPtGBGRHq9qYbdL1$f|y?*h8&&Go8<>NN3M^8UH zZlTI>I~o#O`@^U~W-6c(1s1zsN0=D?y-Y?}mfgsu?53vv*%nXws7a6f(GY0Mop{a= zb4FOs8f^IEr%%`8h#uKMzg|IJK4}_wL+huXV|#qIjMS|<3!DQk8Oe9Dy`qP*!mfg~ zSVRoIlLJMA9F_Pw1kMCYH4ICIq9?=2tgnuxkDJ^lUym)wYiVf#v;g%XDLq>z#Vomu zzJh`RP$6wZvup)*izbZ%y)D9e>g}ZzJ@OIK@T7g_@C(F*Q$>Rh`NW=6=|M+wD2Eh3 zz3&e)QL6G=if&~Pn_z3S?T;LPrGl@U2hcaD+F7d=)YYkEBSR=jxTK|@39A9waOk7T zIo|%0(X1ZYK}u75K&`@rKp2-rMr3VK`WY1fFKffeoUK)M+3YIpcj8ZUw=P6J(kq$y`PtGkR6dNEj%j~AmyQi1ob}D&bM{;PgwZH3gf5` zEUrzJ&S^iVc{_PFAx6&H^p+Da`2CQ#_nc*DxL8wL%c~xhfx+*&Z4aT{EY+Xs8UOio z$TYj5ww&t>kIHV)x-x{UW6d>BSlieb!`(BL@8;p74NGKe!^XOMt{2@96B0tV%AIf9 z{v=+K+?!)(PbL^|2zFC%X3Wm+>ea$>Bevp}&Cqj+?BNa|q zYrq7~(ZX(>*XrQUgMpcY*nH$JZm;;8BWlzCsrdD8V9$KVJ5fsLo7W03GcawWd$}U-MARO zUHN$_@DUARjaJ9`XNP zJRpco)6r4UY8e$-rugNUILQZ!#Xx%U>aq#o|JCnecWG#{IQTIjc98&E3 z8(yr};1_Ypl~o?#M*)#FPzDSHclh%sX61>%eMB{&VQWto<>{DiVE4TDXv(Bor0{?uHS}ueXNwye8h~+}{mAA%=Hlhc=VL_u z=gYcrc^@^xA|w0k9qP{rQ4t22G^?U5Emt|j7-lX>?)$9fheXX8nasYw@3d9fJA?e7 zL(&CcU_K9C00Eu{$tfuT{{CRo5=@qTX=U09oNZ1Y9y{Ee1cE^1z5js?G&FrsuRml(`Fu~)p-bccXhnv;B@zb0jM(xDG!-Lyu;)X#;nJ3=`eH4_s z^B=gY12+lOTh^m9NF~_nBY30(#m*qe=3d2J#kVX{d`O#S^Y|{yd;d%o`a7*5@Hh^^ z#dK#RkbZ{)SPzhIm zhrBl0JLCcv34kQltCXfxhoqvmx;NH;{ra`cv?~f~D}>~f?w;;(=LU>FGMuJN;S747 z6NcYxb#tId?u@ya%hSYA5vb&dtv(Y7lWefls;_AX>g*3-Vk|LHnO4TaJtA4sD+&w0 zLu+K2?;b9ClB|I6H+u!fgDzp?JMbFVn+v1gh{!XenY0=b)mSz?m*zZI1|r@0GqB?U z0@~me(hVl>j=^`@X-WUH3y`;wrC%L|^h)V2Z6xafG)xPI&=6vKrlrWeq)3>%zBDk;Mv89CNRVQTD8}B4%DGfiumXF9&p_&ehyW1 zr+L!fdn^d}?B6x9ywvR2+*!*yF1bFUQu=*pWn&>^){OMUlF#)RlN*woG-nyQ%a>o6d{5AvW!JjV6TiM1 zth4!wOLujJbXh~iTB~Zd+O+gJsgQ})Mrm0%NK+GzQqN(66Hb{nxZDLOg-4^yezsL1 z{5r2O`xvK>yvl;R^K(J-?Ooy1BtBeZxQ^${I6_Lc-vk|HL?=M(I|Kc8Xx66Z@Aq_= zWv^=~@y#!vNmol_ID$G`I5ZjN2ftLV(HpMLHZNX9=s*d0z3-2UEgHDRBKq}KWtz+` z(T7Eco~RXS(?&mHSU+e*or@-fQD1&yk4f}?l?A%U|KaPaF8L6Ghi5fJG{x?uz<3j`5qDG8PC`qp4P&pGe;eSb6Tz4lsnulv3({AQel z*tXh~z(^V;VmlWan@-tysv7at{Zbr97r>c&}g%Fivx2GNx>4BBhoU1FsuwSc%g2G7V zFG=?W(+74Syt#Wt4a0xk1;I#H)=9hQ z{KCd6QB@$pah&DN>#_cTpxDukljJ2ME@Hfd?*TVWe)P zHK579eCbtiA(9U07|5xpYHg-~T3nnTfbbr&8+OC=uJvS{!-Ma4a2^sOO=+oKGDZl& z0qrz_#=97l6IJvt{U@2cY{DN;Uxis{Vs9bFeXcOvS`g1 z_3dB;YQ9L{ER7id_|tGvY9N@rR+u8U8W>dw47t1>t$gKb{>g=ki>5o4UMh{?mD@T7 z4DdiQ)y%SM+g4YxAvMv-<}xsR5{aQCt7+1ve9!6mUDrw_0~e#)wC-Ia=p9UmVGD#!6;0kN=N)d`=a-*D~6Xvw$1fE~0r6iIi3 zX#<*uJ)eJ{Yvr_Z=Y?)B>3!EUcy&{86#t7Q75-}?p`R9=E>&EeQ^%$pvdrjcx#Q() z5z$B$1cPCmK-23dRWzlgs6ULN^)3RlOJ(uwD+6;L2r@r`F+@bK{C*)u&!pIWqA)4+ zkN}_k(C<1=VJ3O5jvJ1R)L3#6wiW3h<|Wc*OZEodjsBRiXq)6agBCdM(zNy}_vF_5 z2EUvjQLZ6O{n~hw^Q*@=ZT+%fB#~nqt^;l}uwQ7f|7d@TEPb}qu{0!uf$VvewSOeG zJlMv=Oh7M>DnjCrKPr6`4>J@{FSw4ImNb99K|+9`WJ4~-5jdV6EmOk;WxzXmH>tdZ z?Y)=rr};$Z)ZG+4r20q=3o&=;%cUP{BGlE@;R_8QowKr!-d(c}0h82r;Y>7zOV2(w zea!JQu2p*YP`0f2y=;DH00rX1qg7&xfpV}Nf?}PTj|q}!=<}owAyJQ-jDnea*AZZz z7#+EitpR7;u%;LgnV8)@kFPxf%Xn#S3O3UE=Nu7PFan$`&>4fd#&V~+=#(au1<)GX zR?G|tgy{XMRYg3o=L_*?d~^53TZ!Pv$gZq7$A23~z!2+%J2(fe0wj-X05$l@v#!pb zE$5N>EN83z_aCJ~IK(EOLkG+=L*5yr%OHrC#so_61Eh4&mHbJu04TaCDKUQh>zNCQ z*RRBiUbMo^0{PHGkfDP>)Gr}BbkoSoZ0kIn$J=)i`$tr}8JhqUj!QiFi>;en~2_X{@3 zR<0uVeY2#@9>c^&(?MBH2U&;C6g1;Lvm|p}D!P;;MM)!dN&A6&gI{Y?Ua<0hi!%H` zQJbH|lbfGY^sGN2>9)r0F^sfiwSr4vf`%)7N!wG=yhYn){Kku)zl|3R+Rd69b{QG> zyJxr``vydDn!eYMGOx@oNU8P@{HYwE$tP)j-EKfbi(h}xtkROR(}d4`Ej*f+LheS* z;&H*?`s~)!VdYw{L)wUVrP=mEs<5;vmm`;)L5G}a&Q99svx`hNDqg@acC)128!(KK zjMWZGnGV-lxeV2w*MV8{%Iz{nMu}PPuNnpjFjKEn)zBXQrKX#YWQteTo1c zZzYGc@QWHrU1PuGM(hV_gmcwTHYq-6SamdD7&1b(_0_ZUJ z)|L9JKE~&{VeLpXybjP^8X%>w^9=iPsp&K-+;j zbewF9CKj4##@Z>M*dK(teg`P@$?(&CfgzoU5Nj&t3k5oC%+PjRphMqr!kS{$QX=Fi zF}5v4N3p}&OIg-R+vTucZFxQ`zRt}^TD6RGJTD_fkYJSzXG{%v-|%7LF!4cbwZV8@ zvvS3yB^|kJO=-eywF8Us*5`tPEm26hrF#9a;<8gZDB7h1sdiEl%z#fF6k%SxFBM8AYd-T2W9wJ-(kyY@Iv$ zFuFE-jO=0rut?ldmT9D9_FFWLc1k?8UGMZXuv_i(+(Zx9&&J;$Rd2a#+qT5ECAmub z@&ng0!&0cHOs!}i3I#$A`>wSwQK+azkE#ORVz-G+6QGBDfT7wQ7s3 zhmW5wjx7P^o=$<2XSmM9oqEkzq7|_&7l6!?q#yGH8vyyR{ZL%}NzyQ!02G;;Ou@{! zyFHg7=THp`+2)rI*UzC5Y3;e0gNuN% zUzRT5^`WSj>0(%j1k~vb*3f?Mfi6y1h!o@EFp%u;1k~Rxq#h83?L9DLg2p0CPnFnV zP2Cn_m)pQV8l%?1UH_#>Ob7OF>-Vo_c_3-<2th#$76(^l7Y|c#)Cw;feUhc0;)NYv zSM-0luR*?X^UTuU2!;$Va|iG*uXuid0i4Hk-{kLrJD%WvfC@pJiDtVF5{C}NnA$+j z5!@p|X9#A=OgdkHH8JIvkN)}m1_klU89o$gz|NvV z4CtPb3GUTfPX>RbQKxCf@!wf~4^rD*hK85Ta=%8q{xkJQY6kk}d*4}>#D2w)-tEcG zWdsQ1+#a%IVHAFeG6O0K$Vy*u01k+M#(o*`Ixz8oyaN3vt)Hy{tYgUEQCuqZ5Y2RB z3Bk!Rei_VzkP-Z_MU4m1bhtBX_L}gp4&b2Z82H~sABMFh`GJYEvdyoCc4&Z`4D6hO z8+{+Y;W8Tl$}zEFA<9>T5l`o(!pW_g5!_;X{cV+ZqUP;bBf@^yV?zJEDgKX;O#z{P za)R_=l#QVRoptY&sN6PO)pWC^k}E1cmWd_|I5kja&FgYCf6Z@*tc1cKEufuh9h)({ zy=|2f3`5FX$GKz_e9a!Z>}6?0&Pb~uf)7&1=QiAzc`>M({P)(8m>vt4!DLL9f3Nnf zwC5w^yN0){K8vbsy(CfY{-@-!>5VXZDe!Z^$|u|R$gMw*wJ2`ZAf>Vu-gD*dd`?Mr z(S7gCPVdkgu$%nZpTC8oj3JqA1Xlk+RmuEcs45chA-sKb)}Qh$mYb)~sVXq+-afv! z`*C}(Oos{jT?-qeUty)VavviBSME#Poqp3kU~zwf*3g-WULS!7zS5Elo zH!p9#-bJ|FX>MIz3f1kwG(L-!vsSP5edOF}TYnZv8g$yDNAHlZ9;&(aSL19@>%*WM zIvGq<_}r1#DW*jbPipNucGEOv4-b{F!(LR1;hvBpL zVOnF}8t+WQDwegJ5c$hIxgAF@`j_FUBh~)w4J_%Rwqrk@gcQCl6|EJyWq9Z2?IC;T zr?-X{ov-8E8Bwk^nXTE1G+%Evw-~7{GJJzKZ@901ao^T1Y;5LAML}xv(p`vRG8Ld)cZW~FUnwIwL- zAO(knIrtV7WSTZj&Q6^d=MRpxM*$ry0fA+WeWdMkS8as;#Sz6X83t$^IM&)lAEp39 zO+*RfU8!uas?6wYjTtmibd*KPN{?Elsbzyecd>;GQFn-VI1J>OE}$EDb<^!RsAh ztgPjx2aPC5hT*7xmM$Hg7QB^nag-y1UsDK*>KBK7a)(}+l=!JR)+n@gB&Sn^09*W$ zGIYi2_^lCDVC71txcyJj_oQ6SfP2Au0t^6w2HBSyTkZ%ZK$6qt1!BFQ5%)F>avey) zEsMMo3(;8h)t%GhyVzkT=nLvcFHNlKcEi^AD$r`HH}$H5EXu?L z-JmizYz8BrNJvaXsQyq2CtOnLrNkEnD*~1|zg3-pdXF+-j^S%NZTr9jAM<_MjrbOh zV#yQglg`tRYiIvLiZT5cQjAB3)mwDC)%`srI&%0WDTeQv%o}if{^pw$8kyPPc|-WVxXng`yz~52 zr<)do^;}uCvBTyUvc%X+`1h-ceu0}Ir+sBb8J!m&9}_6TKr31i;deGn%WB)t>}yFx zCiD;3+r+afC+f|yfA{|?XIHnvEZYH`6A!(l4R7jO%jPG02t_1d%<0`N>A?SnjNxl` zFWl%txyBOS@p)d9CzcE9xAZyyYq!x~lY098{*$BE&vrgnwdno!)ww62j~)qw3oNy9 zX7O$}5?H7=QAjobk-!e{!CXjUq606uZi!WWaA`pa(T~CYZQ*F}0idcI$SjEt=&^hE z)<0tr(AZP2s_$;h$^oF)PWFZUACoU%z68_uj_Jb|vEUA-v^1*S$p}WXhrXB7dD}M+ zewRnHUEmM_z&6C}fB5`@ra5kAz;mXmb*1%s3wRHz)K#}YgLEW^5XgWO-^Q6MTW+V7 zpUIpY6ed-FJE`IK`yX_t`R|)Z{sWojXXy5C!qf8qCOrKIndW}+Cg)xgN9D7#-8Ub; zhyq;q7%nK^^@it-^@U1(aZ6MmS~Oq)9spUPJY!r`Yhht%6u+e;hTcto*)H-Cpj34O z7^-)I=jixr(SB#%Xn&1Kt}(~FHv-Oj;`D?r^`zBF z4}LcHAu7n%eJTL+pQ_2zw*T-s; zI#A!CRIn@3^)!K%ztpx?&vN{o%tG8TQ!u|p+s%SRjD_{sb-+5B z5>m6)wuH6H`A`n{a#FMcFIelR>rlLL*7!+!i)66$x*+- zbYkSv*adI$Uqs!XulU2&+Ijr`u;*W7t?r0~*^rlf&eoqDTa1w0aupaM_pKZLe`KwH z&~#o|poe&sX{aTY3|$bNV+k8gGed9my^Kzoe0~vds&2aGav>!V<+?7(#YG>STLDMH z{URHR!rD;4EvVrK%T{D=HXz!08vWT-{Q2MQQJpGGi(Ob6@;%%6AZHr$9#ogw*|ekF zz1MvC!8GX=oT|2M+^nllg5kv%r?|c&E~@_*uE$O*Oobcp8oNE0KG-`@1>8LG8dD#U z+*3`6fEknEL<`Fqi|d?wTU0p)0OxazuO&$c_`+g0F=i;UjugmtnY77X?mA3&)pr9)WBP8zL0g!Wk zok%}{pq1xCUnn5(KYkT(q9#brs`xtyJix^a30vLT+B&^z5zM^IR?a~^{`xIvk3*cZ z0zXt+o$fmd+ohNN6?9}S^!dUuL+;P#+J#w|7&}{B$d`qW?}h^M96esI?lz7FE+~O_ zX+0_aAj9oHFhGI;2IzTbu?^;0MmhQ#%*=GJYbKR;A z(QW$ST44EGOF*;dy|RPqCT;966Togacx%7NksuR*F>YcGIUvTZ_hibt$PcDx-ue=UdW`@CsLV9A4L zxwtE^ui|p4Etf2SACAtqfuUf)UFXB_MPTFBo8KI_dfGab8@A4%x=}S2$J`iAbuOen zQ+w5l#p__&C#S`CjQ&?W;0Uue?zD?rZ>W4>mfTF-oEX0dgLVCTq|HX#yYt<G_ zGC?u+@Ctgw>hDsO^hwptCi_L~HJ!K{&N2vZ`+5~BXeD0ncI@n?9J7>i3?mj|NEs$Z*ZHcHzhJx1qD>dAdvm?2GcCRKp0z+uXfFg@ zL`XA3w;gonuRSaPm2$WyleOUMs_M|=4^=_4HBP;j`DU}cG6Z>47b^t+X6IEfr*zHe z0ZhjJC2}S_(6P-`JPRcK%5(c1zJOvcH!^f2fY2z{8{PUo0TWN62VhnI#m>8ar;&b~ ze)N@4hpGHVZP~+P#WmX>xZtRxP0VA0CBv$tM?}*>Elsprl~6A5j2pdMEB69ioj`j4 z8^KyU=#O`h3}=G!V%juqssOSBN$gsBnqBP^o*~I7h6Epb=&2_ma<(P$rHHrXgpx!4^ip;zq5Ast8ce zC%#$ioVMKEIdt^eyJ-HlGqAik#Vp=@lD*ntyjU-c-D;MW{>Kil0uNEy4b6@>#94+GUx2nrRmZ|~*PqmlsBtDW95>r!MGA*5L zwb>L{`dSY4*)ANkM_=gHc_Xmu^ZNTIya&9{qGxF*;evUkez6megB9{Nv{ET#$nc{k zM?Y83;h6)-4E1>;&K~Uz}zTjOzoln__&ro;kOml}5 z@Jrn?bLf-!5i_!CT@%fBmr4U$b$|HFhmKuo>xn1q2x9s^Zo}Il=W#VlgcC#;JXLpv z#dGhxxx)G0b_KN7mi8!`uJjTK>M(otB2Qp?pn;Wz0cQ7lz zmQ{6db928)>vWmET7vp?NHikf5;l5O7e_ad#nvjeBD-uYS<-^*{vgX1IePNxNn=-* z6!VgVuO;ki*f(9fT65S7tYzOk_ixO8I(-S{n0C#>R!PAmGWZs~aMBnGdJ-8(A6)=n z@bK*w9g&q@#(Q~TlM;467#QwUpgDgtXwX88zwA7UYrT-?`|-pzu+>FvT)L?=@soSh zwT1I`xx-Ev<{m9}4Kv z#03M6Jzb7Kmu;L@dP)726+wZdWtkH836u!XFL2M{&K(}d{bjq-Z~LnwYSS&58zF)Y zgXH|Z(j294mhzJVFDMt2jB_ekC`6@|V5ImhE+qIn_`QAs@4>)%ZY9F^LsMI(@5B3& z=-sVNjv3d5Lj^?Lcm^42y{CdPz=~AWioRV%U~kM`r9)D(01rj6J=w$Raoay@)GRV1N7*XTB-s=Rx z2=9B-T_|fb`0g?2pW#nT6G6}5*K0dJz+OH4whyNJS0qdl2VoQEer9`ivwoh&w_f{$ z5=r;U?hOv8sn1kn>*o-!riRI^V$jW#4E=efJy_u!26}utgc~zJ+gN&f`oWJ$@%z2- zt{{}{#qx51W~CyRe#_aJUeqb&zT`s+(Oj^$2;V*J#+g7OWL4vBxAU`?V#bI$=QL{E{oPU+*191#`NM(1h&CBTiL#LQ>wXvj$gA5vTJKWxxxDz<> z?UrE%xzGfmq1dvtl6rUtbYYDu9a)*HB#m)Q`)T-tEZwS!@(t)1*1(U|M`f}oK|!cf z+@7{%(?*JmUsJ5`*!~UJIDaL*Z$Qyjy`5Y8Nr&9WyhW&!eJlr*+-Jr^{FBYrmP9hW4-)E2`U$%AG9Cn#+HMQ=hf&mRu0XtEUr}wff8Tbc>t8fQQ-Xg z;N_>p;q`j(cyqP(^Gr_df9U>b_UKK7WB>wAszlX-NrJ6f+Rs*Irh*&%wHRK^+%R%T zBMQ{Tt8umSCvWg02PYYVJy^|>2`~s{bIs0E#hmXm>$aOybC9(tU7TL})Jfwz_|i4G zORg6jd=yPF=%9g$U9fJh#lcCz7Fn6waieuC-|K`&#Mx1l)O~&L;KWyVw&OOOsWVl{ z>E2t#ZCMAHs1ViWFtNr4tFsqV7)dNucRfa|&$*n*o8o^x?*{1YVgL8jHjS{kLh#uo zszNg@N`ORqCg3%rkUm(p*+4)or2O=pK)l@S zrJ_@#^T{&pl=n2BJj_sq-~2TfWdim71h3-2S#r1c%T<*6w_#;UJ6TK0>FA5E& z%QAy+m|nlKiR}boA*8>0rCuJ(tZx_1kkS%3$a3cfHq%CjbtaF{Nv%^4l=oP)57^(~54%omLG+3A{6+CNGj{UdxTAQh?3l^eV;2 zFU&Ft{`m2k8M=jT_oXsz9h7$8>I|wrZP9fCFt~}( zS&j1_`J}=MSfCWN=dUamOA2_9{l-!Nh_5OXIr!(JJ;P#q%XkdG5(S@y&^c!pX#ezNBic9zYx|j5yexUiz2Bj z7pLnP@F35HW(TL_E2pnNJ7^sc`PUD-hpMt;p|e_%OE*>kwm#ua_%s}(DX?DKKa;7O*IJ{% zvv%rV93a5PRGUjA7$sGG>s-igFuhAL7B+|up09*7Ue%H07YeOC;--}c* zEPpku#Z2ZxjYY0KSR>5rGthw(_i?`l?19{JYbMzaNtH906h&4&Ohmh|fZ zZgRvi*Ag*7NSChyi)aZ5jQ4xL1^kh8+1;ftTdY-gjlVx>T6?DZ^6J(1vGjM|FFYZa zdf(VP!Mde=<+vx-i;k)%l2LEDIGXxs%xspcP!>jyE;cJ?(<= zCrtctBzt4nD)UMFwq$(;!&^6sV0z|m=nP+oU4I7u-YG7KArN^XaEWp92%aA$;?{~D za_i*V*ZINeG*9A3Lr^qp=kMFPjUZTr1Oa{nCOhymB3`3)5RQBjsbZnfIgG9XqUEl& z4$xFkhZp*(I}@ksTUL2)+v9+~MY2wQBfJ-n;6X7+wdd4Htr!LB^4J>z=`z1RW}?#f zGV0pIa|YFhpxHk^0=5Afbr;}w_^W=H;J);#NY{aJN{EhT&kN2x_YN^aKZVsn z9c7LIj4(b1<^uI$(Tf*~4|T<-V2t47f*!dbog29NIgH7!H*UD6I4Cb}Rw(8X^k3hM{`(gXahV;S)2`Xr;~mmtWA>Z< z67b+$PGgqOWi;xNgn&Fo`0pnHZdoBp&6fOCC75U4ukZfY)vEafFz-0-mRHook{M#Q zw zfC_gB)w?{pF2=ef0l=0&$Xj@E{IlI(9|B)8Y>7Br&UDSkg_sK2_Y=F68R`-u&jk{c zT?2aNfk7B4)JeWH2uP_PS568R1jq#E2)e7D>ZK4jwkdraE{*}tR24(&a zLt_|vDN3GNFm-NqzQgp9kXn6+I@#K(%nLAV*M(Uep2#=n^T+U6hG2)WWUi#aiH(ho zw?`0>hF$df9F@8=>-WU$`hC&=L!5v63Oi3z7`7J|pMkZtKs`hPk*gGrNkG$6A(*;^ zb%^~q<&jIZw(~ONA9E9W9VuG7@-WWuJngtZIdCQ-Y5i!BaXhctLH;m&YQ-96B2PwH z?7vDHn#7M(`^L){f9|@#wuwG_5q~d}+h=>Y9B6mecT^o#H8M(;9%3Cw>Kx=g0CTtJ z6^T$E35O#kwFTq$+$srvFo2!2Bd&`bDF~7g5f{&t+5&ic1-HO9AeSg__vlo@wdw22 zK89b~qk8;Y#JIr_xXxcDB^UGvA%$ppT@d^Dw%1V)`TWoxu-%-Wl5?){a*F0# z(?g~eExYImWJ(VnQn}x_v6vu*`2*#ibA3tg~Fqf(XR|F&z28d;E;@xx|1x zE|YA2`Dj`{&sc>82Olp1z(Y60gzSVoAdX#l}aIy zE~B5iCKD>((lmX9v2Ax2OYLa06iEmkVYMgb-th79jgga(ia0I=pZa!$v= zh$~^VFnj4>AZW!UZzwmVtOL*LPoFh7kF-M3ApF1~Aj9!Xm;3kk2`DI3enkXMYc&`U zi0to(9U29c1u2@~*}X!@%&+^Ny*zNT1Ww;Be+R^nT8jzV0#@XrKqJAw?M?kFn~v)i z)BR-efDM>SUL!onrVNlIh1*GY3qX|tBzX;q@XN0^N-*8@b?c+| zQKuLpsCWB<3W4MxbrWQ*_$nnrAXso(VO5K}Q4N1|?KPO)!QPIgRi|)^TVB4&K(xZa zYAi2)YYhUK1(E-s{{gePF7v<&8&ZJr8`get7bfE_Ns!51yMA304eiV+A9n>)haNKb z0C;y0YqW%7;8D7f2sh04R?YI_{yBpc1~cA0TarCgo*L)g2lYehjX* z+S;UlPCWtBEP%Vx59&*xr5YiB3;(wLfI?OzIFT?Gf&{qRathol@b8W>?{Can8yl;5 z3nBwH)Y}jp8M5a&q&{~jZ@E3P^Ckc15i-9Y;lJ}vJOq<~1;NS37whvA>+k;f6FhC% zko4VzGZD|Z@I9TMxBWuoi~IKhAViy1@$eA79Y${Z&&Hwfe{UcF+Wug0a4_E}8jW4B zErN|znMou}MCgISm;)IxzFLColzeFrS&T^6j|7N%){7S`tE2OffnAog7-V?-vHv@U zdAe@L@mkp`rr|$_GXB2BYyFUgO*00TnutRu>+d^SY>;mKu=#ZV>hq_OWE_T$oF@*X zG@Z$>UMc#y)0LK%+MCYN|J{GAv=T3RfXe%gN6_}B!QXZK;5Dlb3ZxO7rd@eW@MhFd zC}2cKe1J%Rl8jwD!LnW3$cRi*DJe%~!N<_hkmW-pT3Y$u-<2kX920{?&EE3r*Vj&w zD;M&?aEW051x!PXXF(hXumBr0ms@Mx86BIi@GiFE1KpM$p<#h>qu z8!!d}iDPFqBij@_E*ZpnSCe5EiOF}B;Q`e58n-g`nQ?6??*TNxc37zQG(G)x;o75q| z<(kP2im6vPypg>BOuMn6A&y2gip(ciXn6(ix$#XoFm<)P?Y{HAJB*&}O1MpmDSq6=$k485aR=s&7#$dThxP*J6W2~w@b#>O{jT_I*dis*`VzZChH zx%K(aY$F~~A$n=wA!tyL-6kXIIHu^sbBdI-G);9w)Ou-g&-PW$4hQ z;VqKl{o$FUk+%@QbG2Pw(O@voDkUcQKHFc2Sx=CM6XMu zPT*q=_cj8l$jPC0#BP{J!Vuc#qvgzvmi5Y>gXieytO`OENrynjON8_q)bLB0 z)!1uzc+`>y#SU#KGaEBy`R{!)Z8O^ez!Sw#!tkA)og-#A(WQ{bY2?TT@MehQFp+Ou zZ1~|SX33^7)tXk7dH)#YhyuEzt2(O|;aZ|Sk7$BdL6-Am z!|z)MG73U6E>jX;O3*}A5}%T_|4}8QB%~^(djD)%Abj7b8b`-v z+{N#EdNUt4pRE4XqpNb&0Ar#&r122}&@8o!s?Nc^ydu1$(IESqK=c8qDPKv5TAmgx z>F7V6>zU;kUC#64`)p_6;pSJRFnmeRm4s_3tr3eD^ho!pfx-2-rTsH-6V6sCX$&lV z38?CFm6nBK$EY}}OJ6i@t^p_elOZF!vJyM~$UI^cBLpdJ_&xuUwi-0e=21~-okER) zv2kdD*WDm2SJl!Wlm46%nOV1Pf-Ulvf&z@j$`2y*Y)k$s=wNnBuf`tTfac!M~qgvj`Hr?8h~NUK(LfN5(q?1akSO_kIY~YrjnnCe7gzd3fsUv77~js?UIkEpX8{IM<3l8{Q-B zE;4Vb{I@g6)Jeoe_Rt4JNOtm>jKP5pg@s{O-gS0fdZ7HoQqf@_ zSj#Up4tS;v5^iuXGTn<}s~1mf$&hidhzDybEG!JOj;^LN2~gcR$5}!6kj4W@_?`EH zh*v`lum5VazBr3T$)oZ&Hk=1F`FB#v-aQq)^E!$G>7rnqjjpUBc+iI~_L-lkti?w)xv*xMH5ObaY zaaU_S_jMbaTv`+HL3Gbpf|3M@4EFgfa)53k0`G!6i~tbunG*M2EMaTH0J6eWFdNNA zE9hf##lt?BE+lT?B;g@Gtys(fbNzFwsQ2fq?>4oumBK}>nOq9{W;I06um~4r78I1i zzT>uqNYX1a?S_feEDm{P16%|O{k6jF=01i8F*}l?>It{qbdEo2c8-DSwnd(&qRw;B zUH0^jVYAU9I>*4NjY zRUeRMNQweY9ozJ$$^Ge|Z~_7crq``a`eSA5aw_rwzY#lB5j-g0w94woj~_KWn}mOS z<9C36bQ*)-d>I!AWy>=KwPEEGqUTjq@QI73x7ObRlWq2Lb6mXCHod)Qc(w02`&(|8 z*LPs$Zs}VQfq)MLTPQ0u)b7aGw5RJ4T3K0H0pw%YhGP_H*mi)l-`|_hey0rv(?n+S z%!FRO#Wz7^@!}csbitI1F6FU|^`C zI>>hz7#gCsZt|zaz|a=5&CQE$F#V8JK|Uw&Kc6$z2{Xi?W1%Co3w=m?`H&~fd1f8{ zl4ni%_|nmAtZ$2b;WlGl2Q&+wSTYT$vEkbw$krk}{&;EbQRKSM@K`~7kNQa7!XlGg z{=U2sR#!Awc}!Fe^+AU8)@b=;k;C4(mHnhGhR=uoHLn|Efpn>&yP^DRtL-SPv=P7vT-69#kITxiIM-{z*OEe|Y*m|`QJi-H*61NrHf z=@i2iXg;+e z|Mh>)&d!hILB>t6LPHDif)k;~T(MBW;D-O8Nxo0*S3 zxY`c?eN+&=q@0}ili&iPSzaU1@Ic5Y>klGrm5*3xJ9Xm#^Qr_uJ^>Vr0z}e5F1i2m z+l)3nAegfq0nHtHIkI5kH(Yb;>%BnG?{6&^90$8M-?I`OajOvp4Rcdgx%1)KN)WP# zdGV{r(CBDNax!a`BkD-V46wI1Z+hOwpt|7RE^fhGc>AV8s`zqGGuq{05J*F?O1r#J zJ|`5|jqg~D{$24;b7kzT^ckkvj|h~NTs0zb5dx0P6=VWHsq?RHYZ|1arVc#pGP>Ne z9+NH(wG7k5s74NO4N8a1U4l^{)v3VVKkzJIg>wu%V<|96jH1N9IbLleom?Go;ysg0 zE$%}1`SaM=HaZ5*fc3?(=>mjgv@BVxxH+P*kUg8oU_;Z7P^LXM_mNMY-HiJizST?w zr+`3_X^a|xi*eHkqk3uoID;CeT%I~~Nfx?gQX2Y8NI=;U62y`For4&g2}_f#Be8K5 zquaeBp|nxkq&OpLa8R@IORJB7x!T)wlPce@p;SqVJ^pB`g-uQ4zMc0FToG0v>bQBc zrxvx|_u5-2zrGu~xc@B+;wk7S%Z4b8xp(fmJsZVP!a?hvkl&+3L{xtBD4v%@>lDo1 z(iet)r>#+Y%Eh^2RwPWAMnow62pEO85snyJ>^v`uM6ZDxS!8~*qLqxizJNf!vyiyf96p3~uJF^%g+OfN(PP=98);T`Z*Ep^7SOX}Ru=`u739B#00OlH_Mx zgrsv#@SavoR+M~dfwoP>#rhF{W6S}+I=kJqPdwzf0}Nev{>f56QC}D!1hTQQg<>M$ z#21a<7(j1MRCII!M4nt12NOZ%XF*x1lTDtJExn7f8uD#HXV#Xuv$BKs8WSls>Q4#w|RD7qj*}>w(HW^tbg;U(P~<|H)#^ z-SrRHE=|h$6@=jy1_nkAnHxDN{zrT+h>kv9I$u2aXW})NS}L*aD1idI%LJ&|O5JEQ=+5#ZL$vl8!yy-Q|*T#5V}0fN}TxeyzZfO+{| zaP#T*n>;nV90kT_-*(^i(6OH>KYZBXKPmx^NXr}J{qWcrB{q6KMC6f0KH8khpW49} zElixSV@7s75cX*tTxX=Y?d+T-?FS&Sa-Jf;eNqnV1g|%b%jE*@PLDVznjq#v$i!poWfxPC#jjlraN8ne37h}K_$%l= zB_6J_?gV!xes6=w0yKv}pc&E;aI)><=jR6?f>LC(J^e&pG&FdFW+SJZl{Z&@oBXZ3 zo4gxSiqZ)EjoI|z;hCB2IiZ+m&z^xj7_96vSP`JWMqjb!3Q5VOlMP0l4 z9_PL`?>Bljk2>Ew^iKtnhj^gUAtdciQLez048D7}RMn4PP}`K#SjNfA+v7DrldH-P zuBAbteik02fzJpZz;-5dph{7l6;j}4G<&i)sbNla{^~*)DktpnAbI*tH*TT?sO@0| zP=MQ#41TD8RLW5dYw9`?2w^u+RWr^2_|wrUH>+u|AV^yG(X$KZH0v#r)L2)VuT`bw z6&ChNBz;0LlvB~H*!X#JB0rpNZD{y{|B!u_lb^qLDs+;QhbK=inOuHQ-zXj2nrLci znZsa_U(tA}bF&m!?x`x+r%12j6E!Kn4T4`mI&EMPkfNZUT~X;J#<6v3eKKq+XCZXr zTLd+)bXMQKB^c?Kq5J_oB-=L3U&)UI@}871{NB5ERTzB=^RK=!tG*911 zz1CDzg>1S$-+tmV=YLlCGHXA7;egsr8Sa~NtH7Ya^?^rk~owY5)g z%;(aUz;sDBohZ66$BeVfv!AyiY!!r^+JBCjcyYkkxJk5q8M}*+*%?x+q0}>yBnkT zmX~UXzWp3F>E2tuE1O|jXAScVAliQnij4!e5E1XI$?_L)v(X|spljH57Taoe*DWmG z`8;;0!w!3G53I;a=;2|SM=zv0=f#T`D-J{u$W}GL=X}z$DFOtp0jOHp!2!!akfd-N z75CS|@BZVv>I7^n81!#mm8qmjyktjep}M;FLfy=F$PGJ<`lDt_B4*M~+jxwSJH8)$ zi`2Ie@8Z|a817>KKALdJjiMJVBCp)HN%d&1$UCfUp)1j6SaBZZk*pmP;H?tlvoCX* zLB{n!z3kq-m*nzJ@QevJ&7E)En$wIeQh)7FBp6Q= zbXHeax468liIR_b2KWIEQ=VM*iM4?+zK(A+u-AtRigT?eDY#F&oH3%MN%4~zJIl4; z>{FfzF6cxj7Of&OI(-2Ac>fIq(_!c4JL(3%^GA2)N%UY6cfLKA9mp%$bF|Yf%GZOn zJm7?U#KOT8sKqpV{BX1e!(a2)O$9Pco1nz`=%7^{)g#^5x2-?lGvTjGzER>Nv1ht* z<;b+o2=;MHCSU6B$Hh>_JmJt}^PATP({`oLLF?*uX`;w|M#Xfva;GtbF2AX%33M;u zGGVN!@mX}mUwLvVP6j}noD)x}rB>C;k3Jnm`V;TeCIzZstIq$b@Haf@A;)JqR_3aA z1of=ZFPd0{bRhQ15puw#!qMI3x@Vdm35u7cv|qi}Z!O=*IS^zVoF`oix}q`=p`eZE zQkm&;)|aA__N^wD#|80NVDhgk6EzUBHK=SKxjZGkqkEv?dT4(0gT^@pGc*!eP_%D> z6AzPN*weSwV(iHBLIqz=(FkPZaeWZn8B!wJdKi9@)(f-qT2+w z?LlrE5%40#;q`c4yzKFqg{x>)KA@E^(#si0<|vy!-dm^gsDH;FuL_`5DJdzp3D+cc z8%jRDAP$#udHrs2X-UP}IyZs~x{LYNG#PCU6QIL(f2{@=C6>%CHsI2x76_@7lEd5K zW{KCX9P|V6lA?|Fy~ETVjih&s8PAASlB%A_!{*y#8SpSwu6Bgi{Z8fx$#!V)=fpT| z__(;3HCTfYOz#=)3GZUzB6A&?P|y#A(#iVw4VfVxx8{3{;-}({XEd7|L)hXuM%#-w zGT2sYB%3!}Y~P)bZ7rim^i=haFg-S>y~fu5;{H!ADrKjLo(E4^ufsm>%H#|DktNV+ zMFg(Na<;d(t=jLaJnUoVj7|-I^oHq7$^F^Y(SD#5^!M;U=A4ZbQ$}&wE1+7uyv~w> zZx%7ban$p)NZA&!x(P) ziUnF%)IuS?m6aS%ccl=npTF<`dE2E$c;6NHD`?bxfQ0xC$SUmq&sUeFGu$a<8yjX_ zJ1r8Y8PFdT*A)8b8p?-%im0AhnnAh6;h^E{Er9*NkB^96$`0#d8)78n$t=Le=kp+L zgn=B{{BPt02FDMHx3L5xB^DPK`zyP29(R8|J@w~;KuB5s9_H`LJ&1{!L1t}Bx|L}8 z=neKCN6@1-^$O(pjzEF_-osG_HGztIy8dgd15Xb=U@HY(+0l~FdG0w7&QpRi3$g}r zdFuqKA9NZGhTX!*KtmS|sneuW^;f$!wt24lA8zW1Pb(Aj_i?mhg(!5h&Iez4TmM** znV0PuE{InDed?Eme^7;|QGP}pbuDg^ZOwKHAQ^h->{c|%pwB*@^w-(|-PF!sYE&3> zW3P0wJU|bTEexSD3BApCCS_*wUVGt^fV<_`-& zFe=DC!O>!wnV7ihw++TdWt&@bMrDMdZ9Qor!3v<|O~z-aiN z>%IEAn@@@4V&eqx^S0Y`W0q{w=c%dvY*x%M$g-UQH94oz;-j5!9*JMSpN`wb>m+Ly zSinY=qiTbZvG9&2zNXdco%F0A9%jk%;$q~BkQem-_msQYe-|^3V=N2zvMKS8rAo=V zo>AZjcX%AffQaz)p*ryLOeUP}a`seeMrO!S+^EP66e`3@t z4Ime19@PwF2s?hvyD^}bKh3WD3>u);24d+aRS_$ntE_jo0YINu6e+zvY3$>lVq($mtmGpTVYu*Vz2Ck2 z0;k4%hVMR0@SMOS=ExTzLz5=L+{4z1$JooR3t*uzLbofmdSFBF=^r_&9JbC2Oz}cJ z<)d|#d9WZ+a2OOfG#}D}c2`j}6BD}n4RWKcC z8lc0FYEQP?U@Yd!OpS=h*zG%s;th{_zxV$bd+&HG+xHJ#g{(4?ZX;QdD7PfZCM(%8 zGa@NvC8MlR_N+uAGkavq>@Lx;g+%ryD`fi}*L~l5>iIn1-|zR|^LjnG?&~_w<2=q| zyx;HR@#>l6<8+hvKh>Brhi#K>{5oax`5i%2T22mB)}cb_6d(8EFTam<6-xUJz5T|B z-aXo?t-pMNTkyF|edj3&LoubhqyL!Jcj@z>~aiAHjQ@zekKk&qyR6zwFFIdQFv1Wo!;zk-AYnh`N+;g6iW z{?3tJ)IV9@E~MT2`IJ72*@KaP;PceP=kRO_Akxgu4$O{kl~{pIt-xp|6}gO@ubyA+$BCnMwR_Bb@*vm4na2JhOJ<+v6vG)_m-AjAjF{!AesMk{yU;4)-kZ zJGe2DLI^P`!SC@{tHxi0W!IMnWiBQ1k`g=sI{hDiUq&1bp=?Ay5P4FeD8>U_Q#B(a zYi*)?qz+a>haddB_LFC!JYTEU+mZR)opF$*C{3IBcI9r#WTm>J`Nx%{YC?gQ(2SB) zaZa}Y@Flm+ovMR}{pl!{XeMP>nT&@`Q8()nX z?f<+7_p#9*M-d(l(=;NVyg#W&SHE|Fo+2YCa*=27aB#2X;`X4uSB~1=&@Gz!8dojD zb2Q2&Rl(L*s414Z8Xj}-^DIeweSt3QhdQ+9gLawIf`Xuas_?sZ=U-!vT-@d2mOUK}iuO8AR`)Cy=;ip)` z3dK7|qM6ZtjQ5I}1G!;p)9zpYIuT0yKOy~O7L?c+pxTheKEKm|^lf{4Jq{X{^4*<} zww zTuW*SVnxy6EB%Mls^6IlC_Z8})&i1+&FakdzU-BJpd$c{Kn>xUWJzQK=j7Ug4u%i5a(&rT2~^*}no z1bvK%>WBoX+?EUJA6Z8rEG3uPC%rj~PT9f4{ zRy?hFYi#9_sf{9vbC%vy(Ho4TKO{8HZ1n(iSK>x zE5@y_?hAp!0~9}+_k*3VaK8l`dhu|}Pj`!lej~3(+o4ToRkK*PwZ=xGV)U(rPS2Tw zW&h~+0%yLQA8d<$7Os1KzL=2k-L_ZWkR%5KdtPXw|Bwks$8?uLaq0Y1p%T|$>Y2BO zgZjV_;tSFJf~G3SkJ)m8rP~KkBokCzw6?ZZQVOKKuS)7M?=YaW@9CGY3e%dHqis_e zecxD+SqwF_L9u%!bmo_Rn%q}MpZ1!G82*aWDSmC)-IrXnYnI9$3r`e*Ga+#Ra30K@ z{H~HPFGW`NA?^Ly>#N<{2P?a-HWC%oWo=IIe3s$K;OXECI`3lDAX5 zzMA}PQmB(J`94sp?(w3Sme!Yp*Ww2ZD6<4g@_+zRihg$hSBFTTzVHm|s@{~~if-@n z<6H>{wigx8OKBOUzI=I9NJxi_W&f8i=AYTsd@P=op7m(`MA|NBlVvQ4ti*Ndpr#<* za;Ls*0g)4oVG`y_aCB9_vCZvn9fUUVbyr zJl?LD#DrQJ<*pA0m-%5A;2zOXLG^U1?w8VEsUy!XWqziwrcPD5OLAi#GAd}SP$90y z^jVcj*zc{2c%arC0PrB1ZbNdR-bKBjd9g!DIzJzvUoIkIeWa(1LoR!(L zA<=r$vpk&6hUJnZk8TRx{qwq5$fiq?4PMcd@Q=_|(%+4ckczuVdyj zU+5Ovp2NPxzylMtTh>;aH}2{deWbR}>rc=8<14=6>q<8`Q3|`x4=k8lPAzX5#?m`z zcYPYucE_#7*<&asCoOB85NiGTmxVh0;@IzKCVQ!fb(Z}?H2A9H=SaR^jbE*Hv|C6q&Y}pK<9$)fm z+34icnOWVakr%R+X}~jCT|IAx{j<|KSBgxaL=~rYjFy7S#(YE{Wdv(?DO%tvrB0gYyRzkvJ|O2 z_eCQ`xv#M1n?p+xU}5Q$PG?Z`!=^RFQv~}4Zz5QxcDu`A0}?)fJp!UNp7wsZI+uLx z##lsd)+UKpf)ZM;=T{x|ui#-gSF}Nu+doIbN=odK;A?_J%u30ySFS$x{7Zp@Dt#z* z<;oRBMbeJ6rM-Z6LQu;DoIS-gPw%do0y!M;T)uSXtW;tlFGy>B+~P71ynJPuorw){ z;tzg`#GQNJ!jh85N>CMfrH|$q0kkOqV;G@_xAn7oD$FprJ;|X z_m$r_VYwCjf)p6>uSH5HMuYnN6e)hOAVm~O9)vDTJjVzf#jDg$+=`2e=&!MUc(=Xf z1cP{9@7>z&ZrdsHMxliq>e9b03#qgmt95(Zc8H4NL3WE7=K8tCm-_0K(#n~IH~LRw zlLY*cH6M8c2v+q5;1VI6D2)5*dJ^feNVBw%uso|FhwkjmlX?o~#-s}UXtw1O|46pV4oCsl~M}2*)@Dt`UuoJ?Bn8{ z9U7ReYr#joi$GaRVhx1{JRKj#3!{5FDLtK=iwn+^;U~^PQet&oUB5tn(F^$Mt1Z3d z{4ujNHdAC3*`Xs~XaY)usZ#e;O5Pp*YYLw|41nhxAg7tz+C zo^o1RT8N(&ccDtJKKs^RLD%>ZxQ|(`4Wh7)=?aEu=C!nN#-DF=7+<>W8QZa;EUXPJ zV{WDr@7~oIVBiECMI1fu}a$t6c7oN)t}OcFiKPD@7r5v_1&?1jtB=C8gc#nflgd$`Zr_1M*Y zG$wHl23SM+u*CHDJ)2ZlKMQ|ZRR^wT1ru_9-{!%q9wU3vU9Zewp(u)y^7-_h>zRU-bFX7K-6GlWOSD4>1>$*FO7!Z?n7iQ z(Ezlr6%jlsVE67+k*v*yGmh(7AmRJA3kcHWj?+V&hn=r_|9<3UQo?uQ^HoN%U31Kd5}k$W4YcVpru2(>eoPA8}xLs4AVRl4WG+p~;3hNq8sdD@;SB$4~P zEz<0A>u|~B+#zEnnY}95Gx-LFCrks6=iI^){jnkwI6gz^@F*0&>3Kc9XSDbClKQ@b zOId^9Wc&o*_OR%*$dGn8_%W(RpOcuy zwZtIa!=$DnT~-tm-}7X@!(6`<_Z}O_%^@jF4x1l8=T=;&Wwp*Yc17Yv1qHmR=pE8p zF?;j8ZxFY)c6c7Zy3=Mdb3N=8vf+NddLWh3BKitMwU=k1m3t-%(c*Dc-B8G(6 z8;W*Mpp}bF`aU?PxrCqajb9A>`O3iJe9oMY2}{gAL$CP(?bl9MfmHv`D+WMf=5^%= z6@|+&Zf*bqtMiaQd^b=&Th+~r!a~=Gt{ys@TYCiYaD&$Vht+yDpc~1y^%lrV z_m8A!WW0_0dh}|%NskzNW~W_G8`HordA{|tu%*<{Y!UWbfKO|}dj&&MQ1B54x+Z{O zMyAfhCyEld%ERoZnGcOvb6C8mHY2~h@yXBdYL*F_>qWHSBGJ?qbFj-b^MhNgQ!57w zwtTz$z%Jggk^+xugz7k{%Uy?Sq_rMQ4mC%a zaqaQc%TyD0>z7`H#07`A#!MR_$`6lmw)T4?MUXs0LDKEwRqjFE63zP4X|bQ5%A|4d z!f*iZ(?daC3Qa?=lSQvdqP4o87VnS2Ic06E!~q!USATPCm>-(Z_hiUo@9SMamO@^} z+e9tsMO^#3OL*g0K%ZiRRg+vmOiJHMSd4+W`;1sYzDngWjgNChOdC{txz@O1!!Ml{ zhyIuu+!?bmOv2H3g+hiyuf4uJJZOA4T_@%00JOVb3^x9=94&&6q>)2?g%ODxFPg4I z0Qo|aMRNOsu%=ywjk+R=(ED9Xx~4j5?dwf;#-Mw5nl&}dq*h6Zc%&{iR&0E*ru}$i z9|T`>WZ08}^*g|-qQ<2yp$Ty)cfF9LM=-6)cch+;j$$<5;HX=*%e7%v@%>UaHeXGI zGu%rvd6k#R{anWjqnaA+!V@~3GMr!_uf&VGY_`f`1mnhV=FpF~C10FM$Hs+Yv>>Ra zL9jmo`}ozEW0yGU+enyNgx%`bI3grxTaK}yGOUyg@cs*83K}gEA2;w~tQf4yO11Q4 zR9qb2!;25s=(}!{M8LqWv!+n7Y@DwUF0uf{8ZWwFx4!mi5~6$PVy+=ODHmgt`v8(>nFs$#*z_` zoey>W4!MXcWMKe=kDxt8Q}?{=!dn&;(^Cp@H?J!%itd{LxbL_5{>NY9`|7{{lGbwb zXi?RHYv{jD(pVSUR6Zq%&*q0tD+q!+@zE12`_49dIe4jcO4l5qxa=LC^7|Ka2=yY7 z6_c;BhHjy|FQNBo8L~HyS~SF%Qj4coS4U5ay)aR;Rk3*;DZ9v|aPg>GdHS05;f4kG z#85gL)nDK}EU*h_d-OvYVrUWz4v_H7iR?s~#YvA|U8JM9(41S%r=h5OYJt!TLrLvM zte~$N@YcU-wLfNDvmy9!$&7Y6_0x13w)vKoh6Bk=mL-}e{k~9L!}#^K9RTyA$2)2s zDcKHl@PO;#x!urW>jQ7|X*j~qAy&NGOpE5}4KM1)2UH`T_oRb>2Q4pU3&&k9o6nVk zkAxYBnjE57_YA0Bea4kHJ>lO4r+mi^t=;&nZOJl~A9{zR7T9jr=tY;mnjK{!!)|}g zm9@o*FJanQy7LV}eF{GpIPv-0v%P{2SgSJ6H*p;s&ia|taQtM+j3P*=9Gh_!oB9$M zl!S_L^}M96%cx*TwrxA?aHGV4^Uk;6q|Zs}QDX)q_r?lU7X?ujsbz=GN*!sttsO-U zBVYxvZItT|-M``Px#pv&C)+UY&3f`nI`G~e9ib;!I|k6Z6DL5s_am@0PFOYG1RrNH zz4`dPukV3xgI>0m!sy~X&xBjP;=4qV5lhtCKd`0$qkOAib|jANAv$LF?i z$AX?glz>IZhL3fr=P6O3=95lNO$7!7lnRQ2t!hkJuTa$5-esj(w4hyLx7N~5Lhwb15-;CvjbV#KS>ItA|aFXQJAmzfDya( zRDW!Jl7Uyk$x5&k%ro^jM=!>S$38zgGBZ=%UtUq+3V>IQ52#0?o2T5;IFRrSMAu!3 zL#2R{qXW_{p}_x-UFbDufU4Ry@mLXU)tCE{*Zo1iCbM``Ia-JJ$y%9*X*Rx$^ycIr zU$B=2z5o)Oukw#ThE}LAtw;(wVke8#wS?Rv1l*G*XQ@bnshzLNAQ_D2R6VdROYLYt zFF<2T8ygD|+>$C{gIIbW;z&+y08pVhAm@fH+!dhRz@wejZ@mA8T>axDM5yj;uI$ns zlJn7~{h7ggx0#cOG8BUX0?5*D^MFU2@pyc%#bNDMsQQn+zrRo%AswlO97m9;uR=M9 zWWHk>q%=Fo2lS(RqFIew8E$+upGR$P8HsoL{o;Ph zb4(mP`0fOcaK_oa_5{5RvQx~iX>O>K#Vub4KS@xK=EgVtDM+G^bI zaQO8uI~JPY@UQYMHFmarjoxvY{g`&qstk;SQz~ZZa~I=ti%y}w?ST-;NVNoCyIk$#W_p}@f&L_|aY=<=!bHA$?OvJ~C1Q|HAiJ0-+dp&)z+X`6bWr(9o47U%`l zJKA;q{G5`rQu^~gGRI?44ee|@627!W$#_X%YRSY9(Faq0Va@}`3n($hMDLgJ685$oIPIzMIwt&8+tD_wJkm|ctzku|fUXfuBmx+6kw!<=6VFUXu$ zi@aC9o#KKqLDP~cvhGo^U3)2(W({)_aA%QHZ09UqC6w8$n+rI|ll&y5>E5eGSIn=i&Gpv_wK10V2Nw<6mbUp>=u0tbkfd0XfXD1l z@C`qXjoiY6kO8T4P1m}Mtu)L*Q}AA7)?LR{c_f?`^OAaJiLt-krP|fX9%+E+L-eS9 z_lR}+-A`(zD4`?UIijMAK-Pg;4g(Ve@=)Z4*4V0r+3UCErMHEC-4s$WZsos*tgjN3 zq>E=K89Ww0QDzX>USJhnaBJ7YmV)==J%dX&;U^o%`$86PkDhEKY5)9wa-3$_VLdrl zz{zReMKq~xolQISQ2Q*sp|O%zPSdyb_mc#yC?nsx!+*JUh1cHDYnti2aHzo-%K6hU zsYBZ(G&_)4TZ{t*g*NCDYRrjzsr>L%##S={VpA5>FNqBTq--57JXiXMU8j@yl)ddf zja}q3q_v@26|DDkm%>b_fF+rotxQ(sgFO}2$74l!NaXYhhbqnE=4@;m^RY6MK5wjE zAPa7YjTdA-W5=Fc3Cag5n!1)2l(mi$84s))i5Ab`_( zm855sK7BXT$BtVB*2^3uBFBUJ1tN-pq&z@@v*tX^Fp zNky{~Z3Awgi}{`MOM43kT@G*Zrg7cI=aE*SsQ&C8;HGIcK9iWE?DFYBq*sWs*=)A&DemMDoKe^u+ zTX^VCX?q{pB%)Y;*Z3VX6T^!6TE;sY8)8&H#l%!43k{Ohq8-y`?`erzmmD#Rmk zrxc)Fd2CJ@n@jMK5O)(DCiJ4d`2Z9SQBI(NoC2hbv@``W7BO2*>@pOR`nO@3 z_(KYPig2fA#BjO5M7J;j7_r&!MAOPIqXB^(K`Mzh4Q>Vj(4mur@7ke~wRiWfHJwlH z53q?P?i?VKUv@3BoajY1^O!uSoGIj*P$8`SUzRGr46!s4cVIA$zlOLvi<-eJ+@$BB zN$KG=`pLCfdM9b*Mk5r8XpB9GJP1Q@58safEJ-1`+?_JeF@#syOM#LMJ+-;E1Tpuc zx{G0?Fgg0ykM2mPB$<7O&)xa_^u)$>NKcnzuCMqb3JJNn5+hGEuQP&GC!|&w+hNQP zecb!u)7q68?vmiOcilHL4<8C@&YD4JRY++&uj z&A#~6c=5Y`YO9NDeoooAG29x+S-Id^LO2xst1p3T`G~EdC?;*?)g#B!Yj0GO>b`aR z_9;>-yRWW>sMC41$UvtF6RIWxxe4Bik@G1@`MdA;$Pv?D-z%C}cOVETP#K}MkM`A+ zf0g+tQ)pt_08&FYt7ti*pAi>A?7tuLilnyTeaR<-+4}f_YNrGfboCKofPHBRxLfKq z5e|j>tD<}hZ+C9~l13*_Q;(;Pttg(q@$j4f|FOD0A5N3i0&@!H&MPIwn)kA#m>jc; zk7Z! zbd7ai)2wURN~r1+!m6Lfs!gt4N4@Oc7d^D(c(J$^sMR(VWtP!#LXe!e-Bqh3s}3d+ ztLFMsRJt&V>s7nfKTsH|Z)|igm^38uDZyh&=MIO&5!B-R2I3j+GvoV-(L~<2Sp=?9 z57h=2#6-|)q6cH&UQHh}9ehnvV3M{JU*@0z(F!lB$!&2tuQD|0m+0h#-P3{92wD5@ z_5n}y&`+no+Ujj*hJI>187I)ji>kTQxuj}{c5bd*YV6ZAM_lRvH&9YcU$~GKzWI+g zm^et`bJt5`EenNRl&-EK(gF%RD#I9rtrMi zGU(p#rHhg8e3-h|owtHEtt+-xDSI{c{N(f58ZESMjIPXCt2OX=FS0TIW!ruDb*dhk zv%&e9rHaJZH07C&XriH~eex;|$DeJSFCk32HdQw3Cc$-&cxWSTBWmmLk?1s8bglC7 zmlq05zWnGiS{Q!Ndsye@W#@_a_H<2=Qir?-7_eFsPj)x&`>&Ihf@yQv2O?Q*Y3fOA z=})=!-~>c9nb%Vu{n`*jL*bylmoTZ$i{IX@;i1k!rU*B!34=Bo^4jU8z^dmz#a5jU zGeyLT-RR)CfKD4-v~L>qMSZ*SX?o)H@uu$lg-36B(#|?8-@Jg%IFo&Lew{0vtD?N@ z++B=r&#Qsr0VlPFq$cr(cduX^bhwkm#^QtN>1mx_4-p4UKYb8aTwMK-_4gN6I)>eG zmi5gy#Wn{EU>vv9riPKx3tCs-oHh#=ivRegC%1rbn5=_FeZHZxOhSB=s!(9Ndqk60 z68*%aqR0{Bbi3+Tw%NVP{1xvSulb*S`x2WR^n0Hr{OU{0yQ|{%L)k|?#<5W^V(Jh# zN3CJQs&*B${#s?Ha)hV`NWMX9}s_E zaw(QrSvlKl@o^MN&N|`GpLe&mvJrVm{WxFD~|~K?=S$o8WayUxTMIS zNwgV$(rf0+>mVOFM}#sYt1VnD2>s!Vd z`8-W~jC-S=y@cTfnT5SepMzHejApZpB)#YB40Me3oy~L2LehgS8sAYxM+r1FWOn6u z!1!z8>hkHwL#pF$b!p}0JgAoQDSSb8PD_mq!V)X)y{lt6mOZza@A&pyx>$~ESMV zy($Zel;&PR49Bxy21Y7kJgZX`tf=n?gSdKwdgA8_D+K!Vbj+Ba>m8R{>&2o}8z`<^r-5Bg_hv@*+q3GETXoDSYaB%0s^R)wh1_F%`%+DBTwQxB;WfXJ$@lX^SrzN$b9dQv zUU*2w)o!|Ma-l%_3ZP%%Ue$h`C0~jE=NP?-fD>eDG~D2#p z{!BiiOWvZ%prYBsOA3#@CUE;-Q@QMa4HF;4!HJi0UwC6m6+H4fnT_9AL0JvRLQBh! z?1)Kx^w71{bcsSt&uXIh8%~L4uA9sWqWaFIbUrV-WO-`_CJ=}+O9&*1E)da+imv)I zlh=wUNr+cD%{mL9M2RJCY0-Q?NWoxtmz4Vy8M9NS_6asa))C#p8_J(O4YNh zA|mUts^aL-qNRpl$m=Q7s%}eNsyoK0gMNBT*KM_3$GWRr)8}j2yu(7_*ac>fEG3NY z>lSMgCShPT{Kf&r<0=gz0)r^^Ood8)v9Uz=NO{Y~iNXM_NU!;p zrH-IS*~;pc{?3wjL}`6IF&Zmo-^;{&Qz9$7Y9(JOeIZ7CtaG-jh5u76oZeJ% z8Iic|F;_>5&ns(pu9JzaO?A7IV*pM#IkUUtg(Q%p#eDw5_c8o}w6>6xc${XI%*ReB z=<{&4Q_6OOgc(L*K^$Bk@p}OQ_QxnnRrj~3M~@E%UB9~KB{z5$MpfQf_j&9d*0u)| zfA)6bmy{0|if%eu&luP$4nI`BYGKi|GA>qNQj@%WKW@Qxy~B5QS)BDzhe7;M{Y+F9 zTCS_qOG;*1QXUOhCUYcWIj##tk%k;c?k{v!Nw}vh6n*&0ME`<)&rhEA+3)C; z$OT*NAd$P8=BJ?dnlci(SZ=yJ`B025NK@tW=DR5M_6j*GX-xdCm6x9a5b%lBH_ZT-cM(W3PGyLZ_RSwBRZIqk!RL~8H)m>nzsS~e>nSPCLo(S1Ew&1ogRv@7koUQ!Y=hGOY1wyrAwYjgNU9X`*^hN=lNB!k)`e%Z0Uq`J~wKB`Vf zu(VcGHe(|2pCK7;d??VP5Ed3j$g;%e{x=Z-eA^$1`|}+7pr30oan8gthTi;Rk24e) z8$R8;a_@CDkvDWdAeno#yLs3Dcxq?@reLtBgPqqKy@}3Q?;>#6Cs)HiXzYE(NOrQ( zWhRBej5Phn7F`ey;nh#hBry^AU#DwxryIEQJZ214ISQ3W&TRai|Kxaqvap;h=K5OV z$OnIoD!L%svoX+dBr^X0F!bM9zP7gsQ(oZNUl^)7KR(pytOYkTNptGh z?-K)+7PA^R4<@@lh5e9)v&;13Dq9GW4r!yqd<0~3l`o-JRC=!-I#uJyY&=`#!6A11 z*EgkOFhRm0=Gc1iBpOG?a_v)MEGv$j5TrYaQeFno7n+5uf`J3$0r!lZ=n zhYUpp2#~;vPA1U67au%rd6A|EzA~olErWEwkyx?xF%JLPZ+8dj72?kbNgdMUaPC2h z$G^KZNOB7D)fxaI)Y9T03YGaYGymf~@L(b2=|C@3#38J_>JpF^3O-8x`*o0k2kU7Uw&tpGf*Jn_4xjeRZI&b!6%XZV2z?mvDHG?2^PJC!WVuyPTN zq}QL%<|y{)Lq{B6rbfzn*rkR)4CuWz=>DUFUJ$9TK)Ga~gU;FoIukFke?88XK2>dX ziplx}sMO=SKG+4pp2uX|+yUq;{xNr;`}L3DsBYedmNym%NfFnM^M3I*`{C}LT5~9} zQQ05?2sJhN<6bQ=ECldsLpeYGnltbvtMJ8C^8sAbuE5g-*ZRW@?FYqKp0dV*hp>E;UQ$nT#aipsFQ0bk z8o!$i+Ug`e%oyE#`@UBf^bR~r0T%!G|(@2jvr6%R~}qCQJkO;DLb{%azO~(!HI&OU6F*qo~nR6~xGb<#--mEVA=ULA8AwRS8 zmK8(~_>0?_9j+s?8?1XQJZUOdCD`En+qpb?WUI`Y>BV;i@3NQn_kc^^@J z$--CidZ=ga898I8E42_r?C?<8wGMdwYn8ALKS3QG1!%UtV@1PHJ2H(ZJr($G*V=G6 zWm~UZV9;}gA0{Lv2I*H(KO|EQvQ+xZ*8e&D*AGz;FxBK*3F2Z;XDZy*{L@eUv*->f zsKh{Lu|D2YP~uK7(mS6YoT1lCHL8)pP`f)@YgdaDY%S(Fj9ncB!7BL`yflgeQRVgA zp~0EQE9$BUkOEhOCv)K30k~nG{tc)yAo0>Kg)IZCAw6#++cQT1=g`0)Tvz513Fd|L zi~lkjsAI2?9w}+}^71MxJIC!Dj5uSYFH6n$#H8o6w&hHAec6_Hy68lZb(5JSng-k3 z`t3BZA2w8c4kJkx{~Y&ciA3J)^s0XJPy|KVsG^9;>aDeRWN8@{i35r-FECZRM})gvot3w>&b2f1_X66`0&);jy(Ph*vh?`C?$nwPJ#3v$>l1Nj@lE3f+PQJ?L_}rdpN081QvnB?1^nK zVP9Id{<6fVzs9Y@T(Ne;pJt=NgP7BXk2`tz!QR?J)Q)c zJm#I)s0oWB`XeHNV~>Z%_H&&|&2Ht>>3ov6@AnDALo~3y1&8JTDt`($n4{~@!5r(6 z`T6;vCd^&R{4e(`L(Kd~zVnEh;*;5mMH}c!Um_?C&8^;wW}FOTl}Y|ojaPLjWv=5g z_ycR=%e;d%f#1E+%CXgzWl*9513``e1ZdC)6c}vN7XW?i!I_S_WZRDcweFWXiGUE@hm$k1ukTqW-mQ-ttGVt?XC}ax8{s z2&C@;_qZ{H(bsrm1xm%nMNJgK5&;1BOQbwWbm(!sP2S#r{W=bXfD9$^@B_uR5FY;f znII8AQv2+N00`mY@plYft(niovacpF~^8{FtAWCFTBR;Hekjyo-5+XxFy&QG8QF^!yt(g z^ZyyrmK{a)x&BlJe$Ms-*tO{A&qsC<#Pq{d)~cZpky0rbDKiWUvLMMRC1Ft4_Uj}z z#rOlI<%}}@2mNAAN_<6S0(6O=s2}>WTPM$B+r6DuK+oy-(bV}TOu-*bg_?gbEl|^Y zTVYBGXw{>X1SW?7&7o-_M@{ekmU%Pa@y7f)qTjCcZ?-`JW4zJRF|ZmG{C7Ku~JD*2yb( z6yGCYap3t+Bq5#PLjL9OOwe_td=$URXNWgn7d=9 zE5-D%>`%vlbDCLw-SMU}9FDiH``4t>1k*AKFAH|e-qgsu(buRPr*vz}Rfj{Lo!WZy zqUpAU!Sd|rR|bDEgKEDTEF6jnfkUC+hwvG&{R4+W(}+;I=a?RPcmOl^FD{n<>J?6;42}+xmqV`)Aq5>~9BbK@*Txk{9t~kfhIetn7l+ zgpl1nj32=>Vxe=;azO?n)o(}EBj}eWMAqHKKajboq;V_*XfbgLWdGt12I ztPFxgY_Rd&YrZ#Mb>+}Gi>}?cy2F{Ke=u~AB3`t&|7jK~#wn4dt7fUYTT)2Vh!WxjH3EF?ox!NgwQLpi1?6!ZmhLGCG9a0~! zi)J8w@4vm5A?Ds>I;gfh-R(!o%~N!A0dP>-$WV#a50bQatjOO_R%<(S5slzAAP7D3 zyBPzX(idi!8QumtW)NX&ITUix9cWDm`J(?+S`n5X052wvL8#vLktZHNmB2Ohw9_B= z6979wHhh)5?vsQhJ03A@OCR#<1a_rV(Y)}(cyNnIYY!Ggfek-y$ze&d1Z8PiSvcV? z3uNyV9-CA!!l3eJ<)ZkvMctU%0mTB#A_+@hB{1C4<8H*R947`D;W@{FEvT3!X9Ij2 zu4%%9*a#a-e2A(cC}T0+2P%=a2maYe7(u%O^MDTvI(-NEu!O6iU4#qHfB$JbF&*fz z^mF1l6R;-xF$!om4B-)X0?3W$__6p99RC+$4j(K0z?VWbLqjp!eo5|9Qp;gHG&WfV z9Tdj+L-CRM{l8%w$i)RY)EN+_9u5rI{TL-l4&_Eq$@^Oo`8VT1BoRj=K%f^O`D z=-|U`13xIH)))!$yT*5TVa!aSWcBq!TVaPil#orp0u@C51Qt!Rf--6e)831w1Sc+{ zL0?D`xr&i;)O8$^2p50*h{61xwR`lpNmpwN@)bmr9anE3deOC8RNm0gP)f>^R&wvZ zo)0-F$4-1Xy|+(Tfd?OE%-sK0frn%xKhzw8tLY}V;+I;%@dx&w!-Z*Gp6m;{V>tZw zpZ^$-4f>d~*)Ov9#!*~#q;wCS6!X`2?ra1{`Lg#cQ*y-9r%!u&E_44M|M(#d7w)B5 zx{8jtjz{7r&hm(0H@vC=42#(nZ+4x-M4{C^m%6^lKs z*c;sQYNz{LtoI!moVqpk36kgV{N=JoN8teM{1l=C1{xb=C;xvf&?`@1f!dzsB}z>I zmg;|)1GYSedqK~A_^bQ>_1~ho%7GB4ML59;Z&5X$#e3PGy!AX31Ju>SX=8l;b0ZRZ z@VmAsG<5OM+1-SDlKpo$IbG^Zap_nTIUEauZ`g0fhLmRDVjfNI=86)%o4;|CpjjCt z&l)j6c=+E1hG9M+uKwuOp|4Q=Ne%^BjStYM`|IDq>E7n`Hu(yiP{z%X`&LZ$_56u& z>x8MKO7Pm|xVfoDa2={!IJImAv z8Y5ozymhAlApnQ(UG;AZdea2?SuE&778e#OuxesS0^hy(2dgAPN^Lk0fL}N(p0Wb0 zd<#ewRiJKWCm|p4VB*jE_YOx53C#N~tSjc_S^~4OfKH%vf?&dP8s?;t?H{Oy3B;>x zyIZneM_yznNf2_iSclA3alYWGk}pOtUhu9PMx4M%bpe_^2od-!dU23u`PJ zohpdpVQUpez$x82WKbkDr)=%L`7bv$v1>+42LkCBWE`gLq7%JLAyAd8_dDMZqPU6DB~yR<)GNl=bApvLEYFjL$~e$yL6 zwJ(*!(}K@$=EpKUe`-Jdts;?Kdxi`jpyJ|FW%;a^Ukidw56dF~YDne2DUsg+YWSZ* zEOE2LUpH2B6bfg^O58c~whKJe};`{{OM=B$%xYS*g`ZUY!U%y`2w z{yAQP0b>OHhzG(@FT(E*j1c6Mc9gmiH#u}K)bQuJljX&vB8CiPjBhewlG)YGNAewA zOAQjwG;$eqt#7y?3Ly+f#`oiEaYnB$p4g2HIQWM#Xp}hKz}b#5+%)|Wn0X>e5INqD zOy*wcmt9Y_yv9*@RwrXGKG(mB%9!85t^YRq7YV=*ZZF@Lp-X$S&7& z+Mgeoe{MwzhqSCOY7ZfoyWVleaophC#yFWU=AN7b#fY zdZ3SR-Yf-ndIC5G9thG5p2Mh(S02nsB{BnzsS%YEpllMdt9w7jc_yg@0A);WHc2lS zEM5~20kEcIB-7ar#Q7vkbqKEe5$(@QTL!dP!y{j%6r#bmIcu}WQfL4_M za{Z;SSMtZtpKrpL053zen+QObSZ3GX2IiW7@x<6BGVB2vypWSHA0^IsiH70uyEdhKaG5O-!wRfl1H8`iFEqK#U|Bc&S&KfB#h# zcRfIrTu3pyyf_BCcW5x(l+0Kav8SmYlEBxp8n+oGR#$_K(qGofQr5 z)AC`bF<2@J<|IrHjvKT{V&7DB%*_h7&!S{~1U;;sQ+$~VRX^SmZoTxDoNuE}VvF@s zI;03N0iZ&x?9^Pl+{_obs_HjqFj09`Yi~2jLI7~LeO!Ycl_l z4>XG#AuKoW_h}fvt#`s)48nH@oqIM3BLR>x&g_fT^H}>@Rv-paC8a;QN}Cu!o4cgt zO5f>@wJ^TlJkL^^NHU+7y_aL4V&<_m=AqGYiyTBU_YhOWUr0eF$#su&QU#riQ-b+{ zr4=I*J3Nqu+EsR?y5R@+utd8y|uKb{?Soi(UiJZ`+dt^=a0$PE!*5^fZi9f0k>47#)YKlP^d$I%#>GpjY;?3j4tf*c*ryh=jl_^F| zJl?VVM{zgXW};_wUs`&4ct&GX)2je3)JM_C>A{re){pjHY;J*3FXpLt9vK`L(s6g} zk}oEe8&7&xw=ABcgo#?77lHrUN}EplbnR*8O+Y5US}7OiN>`+y3*vvwnRXNX=~~5k z!PGeWjvm#>M<%Xn)%8)i<4cVhL>#CiHim9D1|me=fS760C@PeD@3VfON!Hh*)$iX_ z!FM~&Q(co{ZqJ5LcNKGBtkwhti28YB&F)_P;;@PJczc*?gET7ydDUQaci|DPdOnH%p5U3 zZi@8=rPau5=qM0*4cAd?ZQ@b>@x45fi|&IE0|Q&Cf18Y1o!nOFGqv4Rn&?NaGcYR= zBM1_V!){-NOdEZZ_Z&r~?CL(@HDANn)TUzxhEved9>upXsuhs8#IV@RW;QWe_ISY_$Xxo;hHOJDdjMV3TlHj?9~18B19 zwXJSCjC14xfLzn{q*w}zpSH^$3OahnI`$2;htjdJ}JWQR3o*S;CJdrYZ!btQc2 zgu$&v%1eRI00FxFPn#annst*y2;t z;^dg&^Glach91Y1Iyq`%Ow1_F;j#%)g-RY&y@ptK3yDEr_ zco;5<-}M#`xfiz>+ibmfK1XZ4+f^Bc*5hdT&19P@fM5a-3s_W|CJCzJEurFpbY~{F;sngIu!9*4U(%Sqd#!2&UnOMky>#RPsV>^|(aMt(NB8K+KdtoJ9Br7r z+N>EKeZ5{)t4_3}dOm7b^(FVXRAYuAc4NnzL7!g4+H@5BlXSH##hK<%1w}=1?d2_w z@3)UJeUWkX?lFC^b!$-Dx90_@*L?27aRFu=Y9Ck}Gv1g^6W6IIBbzVUODha;>MgE4 z(LDNdZkP*{@Md{8S5|r}hZ;=gP8GSj)51AVo6?mI2=XFIvTSE-J*W{P`@CaO7fqLC z(7XQNKz>0%-?ME`J@lC~LhjqX67g$YcJsiSdgvM%P(o_TtR5)3!8h9pcwoUX>;_(9&cPoDiRKf%NL6rRc zcwoiqMp{&p^4|Sqwe?Gahl~#1jF#ZZ`~yB)%XRoMO4Kd9!sL7#&!n6SnEsNU^ThH# zu1%KIyV+#ag5Vlc*ezS1`u`C2 zmSItL-QO@sNhs1NB`ruJ0@8xCfCADm5`uJh2?Em6(jwi`9V1fGE#2MS@azFDulV2Z z@qSVd2j)C?ti9ISzsQyd0EBY98}v;04E24swJs7*&Ks`YAq`(F`ME#+O#rV zH}yT;=rHvg%fmPoX($QF2AS(pCh>}tnAZv*ah9qZWK98C6NNM<{Dc4o@a2t44?9%u zaKu0E(SQZ*=8Th1wiJ)0tbj>>gAM~bA?qsuEH-FZQQIunc#oXf$mof}n59;6-uCXc zclwP}ia7h#R$wQu-QCYYH(St3;Xta>e=|q!AEq3-W)EblOTJMSK&Z+?$wzGMpLy_m zG(N(v#%UAoX;uJzr_!66eQy_d=WLZi?y)kL2Jz?9sj_^c0(UQywHg|ZGC-7q_T-)O za)Sj_+&P&KJD@@ooMvfq9 z;sI8`;s%2StZh9*K{|gkz2+i`A}LI-zC$Dm$0-Td@D$i*Om2IBTtNslz2`A~=P${R z1aJqEK!D-;iSVg0VgA$6k&$2CkU>=bRbToJ|6ZD7^^rh7XV-AbYlx!kRgcu`iST$7 zAh(r8cAiJTG{)7e_+Q4^<*MV6)L{AmReukyJsw)I3u>U2gZZRV=(acKO(}Nb07RPY zHCb-k1b#$n!~4yvCG=&UJbJNdR38M3NFsw2R9IizWBR?SswxmAP6aPz^yj6jAzyHo z?2lc_fO>rd<^pgk!gUt()~c);f{2iX-SU7pzA5^hOQcO;pg0bWJA^C`d*9#RTOH~u zyo+Z6+-CuP6F4xb%7m}%RDgc{!ftZA$YcsL$@L573&R+=Ue=(ig`%o!=l;pYj4#09&84TNcd=t9@?7!e>M})*y z>~XRcy_JY}a20&p`D2-jRqNGy*j?z?DX|WAmb2$QB5o^Rnm)5jc~hwon7_ zFyJ02V)Hb-62H1Ftg0BslNMyDmbuBMpMlOOU(gp1q2q#MNmD#ny73FCu&b4zB|dBU zw!-~&gazvKS}V)+kE4ZyjCFA_-QcQ7?(>1N%yuQ%gYv;SqNK-ik+OH7awcjc&&+S# zh-oFJ!{9W@==~=9-(Z%H>lJvg^#DRCOrCYxTp8`XXC(~`pUfD4`q&J#0s$E9uU_62 z9FppViOD)()7dAfQ7|)`0ZYLH^+Jj3*(GBEx!!x>q2Jv)<*?iU^op+FxPPkeThJ}I z>+wWk(5fBt-i`WCI|)`AizBalSS%*HePc*5t7~ib*j}r?-%RQ@Ru;6pzw@mk2pvXB z>*gW0+>G}{nq6<)4O(?^22`$N$htP52^wo0PU^?KDKX7dVZk?3bL+*l1OZ&j;nD;@ zU~=4r7LN?`rSwC>;PCV4iPreT@i1i;)2h|O;?Xc>=>j1Z$;5b(e4F+9)eesW-Sd?R z+BkC<20rgCJ5ZGR!xLVHZ|kSH_qR+LfMQO=8{Z4?tMn46ptqbKCrfnxd54`)0!_5z zpwi<)sV8AJ!Q~gAZ`SQTQ{+Eb>w5>aw>XGZa2Rx0WEfE$(t&;zqvcgQ2$N#Y%G6pX zh5nwXOuv-PPyVIfrlnvD#8Szmx9O@9OiAj zz|$-(qbjKIu56gbQ|Ms@M7)f(blp1FsdWj+Mx1TU>@vS26Jp7I)3!@DH(DOk0vMz# z6!>*l+OIh>Gj+}tSP`#KECXMkN>J;nl^Tcm$a`+?zNFIZqkHM`N=?nE0?QugX7Q%^ zTBA25_fckJKA;ur*p7o#*Vs>N*El6Qm`-?X8R7@T9S zAxI+{BsT;~%W}G1JG}^`6L!6^0|*3M7ePBFF~)8oU;y3q=}}a5R|YK^Ikf!!l9>$8 zVvdRt!Gw8s|K(P{yySD)Xpj$W`8ko2SgCK$I{54|pR~3q7CAm7Fr=kzIN@sy`RcF`InY%SbpK8Gl98Ix;@?FK-#exq&FB zz#C$KE7+@W2D)TA|JvpSI*(>f#fe?^dRFQVVq;=N10Qg>T{ttb#pR@L4%&d|`P18M zf$!RsIx8s(=Hr>+-EHq50LHI=l;Uo8+2nkrV#>Rm{Mznv`i6Eu(;PqCOMVCCnh)sf zta_xy_iTIJ5+8#;!F6L;z3VfAwCkcGK1QLx|EUH!rnMjs?_PT%U|-^5@*WeeJD-wW z)?Y1F>kKSRq)ax;$nfQ}jO@JCu59r}oPdHR=NC^T(7e6Y){JcfzIeQf zt&LY5UYu@5d373GDcLb;U;aGFgJ2Ul@1Me4& zm_aEqW+Z(^C%u^AfbZU4foH|ZoOsZ~!2iMz$+PgxM_AUiTyNfp!-{#sj(b4eu|pAX zt0yni=nv}N*ROI|2~3o}n~02PXcnD#oe#LYMUFytNIHWv!9knjr(owwuw<2qbOt}$ z9jkU&joKFNh_nF}4T;_UpdvJCOFiwS{#(@X311QW$< z0xdwFRtTG;d##HbrsZOxBQ;BF(`LY#MOS?>vWMC<+SK}L@7pHh(QS`uyczI~My+GJ@}v)c^_JVi>D71HA8_A8&qtYb$PZ5)wZN*b$XfBeQxyB2x233Nr{ zH=RyiQ4>>{0|3%5^08lu+6o-74ThRlU`#kZpO_ueHNVDzUYd~9pF zF!TKtWwE;F_Q{xvLYUn}g;C58RwUtX>FMGcuk1MziC>&qNP~P2@2`9c`QP37Dakc3 zHmG)2W;`J{GtoeH!~FDMeH&3{bE0O293AN4l@Zm9x}D9E6^1RV`ZLig_p-`{(m)}u z95aXD*dvzJegw5&E6Ln3X8>m*jM=TJHoypdYdz%95n1UFH_Lque`L^cl zL;a!_+aE&hr{i0+)(yb#mHGi|*;_nM9jieqEQS&2MOZtHTO^524*d28AMp6%-hwq=}v+Hiu5Q;n!dzaY{_?pQFWBiLgv9X{ig8&UP;A--Ei{yXA zxtUZw!wwwWaEEN;iRWw{SXQbJRavx2=^ahUCn}_SWMy}7>GrW7fEe6MMOdMqxIOYK zSe#{INm+a)IK(R@=yuZ!FC{AAMfC#usCA znMW%rt!CX1z(w*5(3ZENL=`R98v?)KA))41;4NWB@D};s*@P+DPY@DRmA}6H3L-F=PhqCX2AVV-8dg71~$Q9CB;S>j3#~ z>2zOMKwBZmk9L02y{`7doix~N$z^%)RG4K<_J#~)nnC^l6$gZVZX{)tmcALsY1nbI zhj}0U90z{aIX~D&Ay@&31n(b(R-~vU-6gb&p~eHbQgkBI1ADbO-fwCbEa8a4xfC`# z_tRw@H#tDxW-`D0f|IjLxrpTjR1P$<4x`gwW;@0rC7;a>3)Jl^ks0T3po@Z`rAs`N z3H5;LOW4GbQF|0vH>X#-Dr~n@;h;CIjH{tw5=k}!)rDpnfkuy4C63z>P5hwdjj7tG z8{z|%-|E1{)%HcbaS$wi-XVC?r-X@Y!I)<^B@dvpc4E9(!~V$aIRr!jb}{jxQ~K0j z)z}H&KjZc*5MxP*=k&}!xUg#Ury|5N65vkX3=TA#xxKd(#;z%;!LneQ!*@B8Na(}+ z1Dy5p`gKhn+km@2z;`4`BLD=fwp3o1U50c8V?)2fGO;;~c`JQeI-s(^uUPiG=qqBf zsaZp=ov8Zrp3M`Z@rQ&`gZT^DHH!>qAV|9ch!l`HPWg&wNc6vt^qFWqLmlj02AWtT z$_Z0G%9pfatUx=WQ6pHCNpg8~(-IrQFM8DoTo7hgh8<#B%GTQSlNGl^w&JVZT!!TI zuwWNU{+HIlF;GbJl2c>qp4`=-^PQ~qfvBRvQ@1%EHE66dlnTiiuCbZ#><|_kbEvj> zY&t_5rLJ2W#);lEGOp(4)_BWXz$#Xf@9wtJw`>d4M@rVteEr3@0 zgJ*(+aMMKfsZYaq^dg8e1aTAzU6cwYsfP&M9zFc4x$&R&Q@h9 z7amsV|0fHve;=)i?Vcp;V+wP+2KPsCOKLK*uXo|XGu7BGOi{hF$bpJ!Uw5}4Ferkx z{Uw1{7T$pAPoO&ftF0|vI{L>8?|Gk3u9(Jj+Y_<)FnG>w?@Qq2`VXnrzY#TqnofaT!s3@4>FLwt_F!UxxpF9~z ziasrVw$Z0hHrP`nQHDQW)VMPCv_qxT{m0Un_MX{xWw|)BzO{wevu9S!`X5)8wD)aL ztEyaj$D^I9kKz-0HeIl?C;eAW0w(gQg*Nrq;dJ&t+{4Jfa#0d|9Nq`|7bzMn{na15 zz7)a7vSMe#(EoT< z<$pH92bQes_%F;> z=;w6OZ7PK8%9|R%{30*g#7S<8E?g_8Jz2egiDC%?5i8SO812&DDO$Cg@p=xKGcz)P zGjpT|4VXiqNtOnwR(jk=&_3334fu5XgU!RdZYnrS+|YQ=Ze1@l;W`K~e4!zX{t5ez zv*0WA;Z+5D{S8`IhK?T0H;I4&K1SOf61&~-^V_27cR{ivmQryADUAX+B)VHyjf*qM z8}hL=D8Sf-{@`5SZU9|7SCN=1nDFTo#U=*mB7q+`iU9EnICn`2p@5R)&|}*)2d2Tc z1A7JGXM(rA$wEp^VWtuQ#`8nXV_VZ@o!RFOci!Q4zo;)+g5$65(@j^qbKr29tPMRS z&~mt6_r&7Y`-%p#^89yg&p@F+#u`rF>_Y?mfq`;EdCp|Snd_&qKG`a>H3XZg(xwMS zGX3JjUX}scK1QxeCC82PGJEj^P4+`)VQxoI zXe{^zhv5cfy?iWlb|Fy{#7*M>wV!HjUQ%q)y6-4m zAuWcebH*;)+oq?lA54G0$aWF`fhp%x&PeZs<4)g6m($rpoj`EuBTaS9n@qs~erRP7 znAMC2OxEx7^-4&(F~h-}!UGpwKZK15idGHVlXp&IBQ=aaHBVvLrAV;6V?160S1-%d zq%H^V`TYeoJhv}m8iPPdyS+kPBm1d3HC@YjkZp`9Ps>w2#oZ&ke*19|#}8OH4+!+p z5H57S3}rwVGr$Buzdxuz(s~?^?Tw~X0u7RaPXaHA$@u66G*|;kpPk1QO^kj&YUpSK z)fEyRne&1LH6d9)GiSiJA)agkUiP9mQV!v??TM8bM33#&8S>&$ydiwiVCaH>HBeCD z+nw9oAhoRyIZw15jP9FEi~z_5OQkvI+#jp?fdS>O8cgB}fR>-{pJouL5A0$S?HVlk zmx2+4e5{cDbC!3&9uxo?Twu3SOtA!f^j*W>Kb}oztgW?fR%dDaRgEl7G1nZvVe@zk z1Xi3*Q055Jl7^@Vv+ca{t4rl<*V~5|6RRwn)4qV~iLQg}{8d@8gzX-udl=o+pes~) zV$C3r4z`sz3b;U~ycS{e>Lt2@@uF;fgj7FqgK35dtb>tZ?F=pq_|>HHS_w$yKEbwI z#m)4ZE}1-ay}Nz}9cgU6jXZxYTf-ZHJawov7`Nw1oK>-se!h>^G@>ybN9KI_#f+>- zcUSc`@=_>(qKe-!jQ)EK!+Oh0LVp~5z?F|6YSZpxaza)_P`tJv5w@axA3q3)qJNyR zkh9yVQfSf;0(;NFB+j0%PP#wO4fbayX!j0$AaO^J5GmJhU|uQLEy|k}2eV)5Mnk0Z zYtOKMs>dCKRc=Ue{3{qSFyL(>D9OmixXXYP_ zkUaB&4Az$;TZg@j_94h;fp+z6ix`;@$%_X8rl_Ly>Ee1$mk$DRwVYc`nDx&@0ZfGN z1`J1z1d2Tn!^ue=!ek?1CP*;AGwD~LYHDX#)|p|E3&{Y2K~{Dadl_T7iYb+k37=r> z(t}KI8%k$XH zpbY^fm@uR%rTzsFUd~mHXJ1);SsU1p zX=^b>!Rvj)I^3@)T593-eS6=O&zda2Y2m5mC=qL!N(`kq<_&3Li){=71PEXq+rKdC zpf7#@C$!G3Azt<1OG&Wvj;|5WM{Yf>>*9S0{%Fi9OWvGw**aJpf`2aIcMf=Y1W>)0hm?sm6)ej0=Pr9}Yy>&Noh87%D7~ zM-wM^H403-oJ>A>xHl0CL{1*@PgI_8x|PSZ`m&UAOM zQgE(xy!LPnb->2WO7{RM0FZ9HW<*>w1-39=m_4-^f>|Dw%i@>6Vd<|YHdEN(rE23K zOoQ4pVm39|=OE9@P+9;^E2(D(frA)kJ*${jjR)3!i)8OnE}dL8M4h$^X4k#)IN}I3 z)hlm5G@)njufo=aH3zYXl|e*^`XI0my&CMmUZR&dox4!00cOYg_AzxR{O#fr~ff&fa|eEI7etm>Pg{*@xG}L_>4UV+X1yWyqiXK|ioZ0e+GH3Q|VjAr&WlbkC ztkl>7%g>(5sS}cPH3ij%92K{ff6}3EY(liK2*@ZCeS{rljTOtq6d1Hf)xL{IFRpFz zE}ZhusWgt#)(?C1dl+Ol^ceFHyEJ)VOtz4TAu@I}t`_oLfnr8vD&&ry)ZuKFDGIr; z_`JYp^AOJKyp;9&AVKngrvN@@&$#lb^NovEj_wU|ov(;1(__VZ>Xv{=*Xe7CfP<0J z?Q$I`5F46r=mNKq;WU5^%yRnklyWvqxYAeG2Ebk$$wtOYSJbh8t~?NyZ*_BkRG5LV zm>iI76K*^@OmFu0ZSUYe^a*#fYpYLprH5pBKsU%^=nB^kmcu_d7Jq-}w%2ngL=H_; z_&>a!Tu{itVZQ}#f`BE&D@d$c!bcy@jm}c7(9+Rh|}h8qVLrO)|GW&A=8vT zM$x?#G`LMUW#!-F!-hA;D*vQDz}V(01Y7LhDz^a-+3pMEq1T*wb!PS-7W+*TA2zd< z*I!)?194T24(UXz{(~X#4^KY0t-F8gSRLGg5oK^ZJ z^RUWT0KJ;`#kH`t_$>x`f59(*kf8=Ilxjcbpda0TgD#o{w}pG-6Cp8WFR%2UnSe<@ z!%PF4SkQ&QM;}yFRKT3-meBrx$7Bf9hA<-5b0w~cei&ivUt|i9u)t{m<#yo7tLb^g zc+vRq?;A5E57-l4Er)uezbG#I{~$|WP5eJU<#{7qSWfhRrWK^6fmYSg1iKW}upt&3Y7vphmq3&(q zL<3BexNr4%dg{vj4d%eRxnyS-m~0fu^GW1S!GW9XVl6wenO0Q8GbW<7;#Yp|1v?8|G917_1W zn|xL-Gjs%h|8Cex35ZO(#ZO?HM(V~;X$Eu<;sZ(1k<(ANUmxhEaVcz}W9d@DaB{%k zOOb`^wQvS!Q@%I=84rH`yw>zU$2CknIo#7T(J0ee|L@uFmQ@49)vLi`{SF(fqU*fc zi|b}%eCG{UW;w#hU;+@-?>$&%c@3}0a=KZWo5Ahf)FT0o&6BRAfHQM1N+Q^p*PiiLQ^!khfQF)^@`U zA;(uv8yp74m*&`HW~zcDr9f@9mz?lyJ77MT8F>;*Gt#kr4qwT7i{+Dq!bFu{~aAMBJ1 zXIF)G{miL${)N7dkf}S?s}C^zAzQJRovi>IYkN+rj`ufH3+BU{=Nk|P8&*~~?C zefAm@xVHN#%jRp#UT;_=?YNf2R?>2EW2}BS z9;l)w#H|(xfy6X={e3I`yw2jUIAGoad(?Dr^ugrkU;UZ2pB`8^*vPe%no$6Q8<=F?bq^Bl{0xA2uSbyR0t-V=j1Le6y_fGxLPi*0zE3K(|fd9kn!{?fU4fk=Y8rT^!Fo!%n5!e z=4O&z(9qC8%9_%B|KEpFIrbl8Tet~*pgF1T4V%V)9J<#b;QDB*?gED?i54S0>3@#N ze;t6;e{AKA|I^O@ceHng)_*|A&dyGS{Mb%QV0WdYeVgjPAPit7e_Y6}^kx1&@tJRO zIT|plz{3y#7o7gy6M)|1k<+Gg=5~2;Npf~5aJjR2PSIcIo-_VsY|P z?yZ8T?dtEv7G$&7gtkl3Ac(JUd_v=?i0y28>|F|IW0229*V$*ZeO%YDh4ii1<=>G@ z)eRxPxO$dfWTqJ&Q{ait3tTQvu3s`2j)n|b{#QtFJLa}GS_!+^mt-BXXDuO7PY1|Q2aqqc&mDsihm==3B5+YCwfZ@G*W6D5 z)feW+$ueCI0gmy3Vc*3T+-{5>br!Ymnm26hL)MVS-1%-65hyg|-dxyB>{gLF>Buez zEHvwv@E-ImV&$eE5%?*HBKII)@Kisu+WrhRM!S_CH_>xP76YoIGs# z(bZgiFdW9b7PEJjCsABuaG8i-8Wde(cSh79TV1u;!+qLRQ)GAMG03`Cxms84G<Nm+bT7L-Opf5;~pfvzUj$G>N~tkY@idbZD(YQ`ZT6WqI)np{)5D3Rq* zQ?b{0!k&9Z87>@=rJQ05Co~Z^2C+N(bE?8z^-1WFC})*k2lyHBknt0^15_|z(B%SM*bvHYIybuJRUdde$=o;hs zFwt~++SGd}N)~;=gnk&DHp?(6;CEMC>!Ol*Gs)yx=UEq-vxA+=i{07cT_Y#S#8K|~ z)GnIZqhakx2=!3xzMnlI$|q(=SIi%IRn-Yx7Lj#MIGjiY z5=g%%F9Csc5Xrd%-Sbj9$#Ba97dwA@f7|wY>XI4Gc*mTAj(-~d$_^_c% z#vwJcOGhoAe%$uBZj*y3lZ{ zKY2QKS^E-!blh_PC|8NEcK_EKJ$bilbJ#z*C3vLjXgYl0Ay$fe@qSlU%;W)!;VxWOomt;LrqAi* z@$ySyPTas8vMu8VFIf}jud}=r>pS|TT5PG)Gw%Zh(hzFD&(iLhNWWi_+KkjZ%#GLY zgKCX!>zr7FFFAAUtH9&gU7AcU)jSYCGCUzA9dlkECo&_hIj$5isdZI27o?XN-PUmo z+fzq-Mnm#}zY$}W%UXJ5dg3IpCt>ehlj-td^T`LdoJ$q+-F>$S$N6HC<30Mm*iI+A zq4%3Ewh#Ec61eP!GWR4s07uwB+5GW%A?;3B2d;ALle)VMh~X*rZGkRE&> zcl6+-w2i1MJy=j{Cw!ftZ*nPw@tky`_cFR=a?j&^;4$-kvas4#O#ai&m~q;HwN3|z zW+dwjpN65an(alLQq334Ld+t}FP$zNR`G)Sgw@c z{FA@kUN`Iq8`n6Tm1>1^V0@{bt&EDzb2U@Jc7(QcT##90-@3grQohvrqG6x$@_a*|ptfL_&9BT( zsQ;`duEh%%j&hzCT|En;s+83}*A>+gB`S>D!*|Ew3Zct{*M` z^j_66UOQ~cYg<^mXst@zk3Sx!#x--ZZ4HQz>NU;ZtaR=TT1|hdDaBJ=vf9gOhu?m( zM)7RqH{%QF*4I$RGTw@F{k;halu5^3&uOjkvgI9SnHP_?41NdUh8blZ3k;GcQlWN7P z)ArFmpM)390cYg4c-6JLg|ubo0sL4l#TFs5hi$K7U4;{ag4hlFASdqmA_vu9YMC6J z_TZN>MciVV%O;PEXG;{=T^8|*_^Y^fUT##*CG#!j8* zVkK{s^fg{<*a6RlDA?@OVflb)$+J7^QnF8x`RV>+BSJ3QHb%EGYa{dB{KRpK;k3(q z$ruuI>MM;@^E=du;-t2-d!SMIaW+YcMN_#Xmgp&6@gLNAF-Y0i!&MUNoV%Lf!z*3~ zOdjmo@mFP(9=8uLriZHLj5(cTMfOFm^Lcu(*-OS~8LS9cG{gOBqccgRiYU~Td`!0) zSG#K|X+o!IKu^W_DJ8v%$<<^LPu8^VgGZ>$ukYz== zTm56Y^2Wxag0?#I{2wW$-T4*Jit>mrl`>35VXZoRryq6HRmQ>1VOB8(C%ju=$o|fI zgqqJGzqDReoc`dh+NCQRNbJ-y*dD z-6;R(i2+JAQ<~&*&QP~YZ%YfwaL-sbVUO*p1uYq)YDZQ1>2UAWrBq>_6IqjLZzP?S zAR|qKHQDGVsk3|hbXw_l+SR61TIrj9Pv1G)(#=mQP+282lg|*UnUYt1MdaJL2=wg6 zT8al8EZKCsmMqatzlrt+HP`!C0lA!)?{~k3=9UGZUdGg-o!p*-?0ws7jyWHGv6neH zc|MDIs4hrmuBt=+zG+5ptLEG#!CHN)YG->U(oIv-G)1IgoEMLabcHAuwz0v8pI)d?t zE`ag(i!LV6oFc~5@YwEd4jlU~k(BzEaraj|^||<@_OZjAP0lgv$+Ss1$@Bh$4_5qb zT4T;n?;}x%C$4eZ9Z#@Jutz>ge9^j8#FKYtz1&-2+IO9kzhK?F*Mw82M#L}sz2$z+*QAaa0f>DLM479@O< z?>FK;LC$nz$ce(uQR2wefv3No&q1P z7Ch?6jwfbvVq(7g5KZzI+3m|6*;u!8xudAC2dLHbxcf4>z^9(l=EJ8fyKvpCK3Wn{+@L??-{k#;r%S#j=c|_l>Eab6*ciPSbc9^4A$H@n zbylPY$pZNwZx+vLneM*FP6^6CRu5q6?kr3!3lB}6iOmXZCJt}Q|LTug<1>0GwE;p- zR_v}g(>g9@!AhLv;&S~Q#r%y@%qF^Q{4@E!{ees; zb(_tKKwkfLk(f%V zBKZ={W4ClsnYMB0G&u3@!m#di+$mQ-x5nh6&Ms9plrFU(Ncn9w z8#e-;@}dit#cy4Moi1v%_C24xo+BNLJdZmnKU*0eI}4WwjM7QDRyTh|LD?vBobu&v z-If$EX1C6>L!_R$sIXHiKlGNvcNZ}gs6I@#qo;v*Km(RHSnD3St>^klG`%H&G~Y3n z{xVJaf`-Mr)uaXO@}BXN&e+d_3_sMc2-~+yzyG#?&o!rs6Sg>;mqbeyc}|T%|7n&W z1r=lZ*7u+0pZWgFv)x~Kpu2); z^lAe>+sPt@wlo1FSoQwThA=kDNzu;@&$TX4@+{1hq;GwuQ3^<$C51nLBW%uLa%zZv z!VzV0jyuFf`si@y-UnhhPj5b)Rdd>&Gi~08!bioQ#ZY(c(Y~U0$_bsGS-O#0j4&fS zqQF?_ihcLsU|jNn)flgAr)N4r+>nf-(moI&G##%TIEL)SBQ*879oMWkh#Q$EysGcs z8&4a#6=2&Ie@9c$U{OQKg~0Onj~4FKEJZp4HJ+0S^Rw6b%@_mPn%!vMocwftq8dnRM8#fbE!$xKF!un zZ99ArFXiH}%f}|E|ACXNFCb#^)~ESo5&vx8L4uj}?~S0wTu|>)I#T6*-0AcZa*s?1 zspV~{(;P~K&M}&7KsxPXKTRUy77lxhyh*GL6XZAh)FeT+qeQJgFLoI15s2XMbUP*3 zKN-Mz6Ip&)c!2Oe2Mr^7_5Qa1&XAkR#j{UHm%oGJW)C~;sVv>^e0uv*F2miq9(6|@ zhgie~iV4?n_qAUnHz@)n_$Bm$1ttuZm?j*d9_(Aup!X(0*;G*rj zQ8Sr~I`2n?U6csF&yk06GsC@v`ooJ^@ZF&(UTC9z+PNj@3$G4uoL15N{DG9S3LYAv zjV92lOco8aAZ69!kF`Ch; z;vN?f9PI>-b~d_s;t&fyXmb|c{*`^stt^c_fm?*0x_0*v8Tor%dAs5Vs2r5S>;hwG ziEg7n@HY3P%o72kRddz^+ef`5L!J9aI;PDzR;Q8X0>*qaUxJkttC=+&WXlPrKIAs* z>P(^|2(NLEsWb4aEf{~&YR=q^!qDq}W&U;f-scd7pPtc8oH`!s!;88lKY5H}j2PUZ zTF$#X5n&Ka`cEG&>N!X$DAZzO))26fRa+#zx#zppyV2*d&)&)ZER_@ZNxd?!E9M-U{Ql+!JLays42x5w^@iKGNl%Ljh1ED?T=EU2mr+w-Mo-4QL$X?Y<(7 zkW-m#jE~#fHmK-JyFHq~>a346=E#6&LZkF$n;d#!sf_jTQ;Fy$4e@k?QbfqEU7O&8< z(lp+;6JEALuUcb+Xb@2Wm0d6#t-$S%W65y<1Kpj*wsoi#yCHFUsdoyEsqXj zaAGQ-3)hsjigg;%IkILqVL4z&bv?z;=#T;&(~<{bC-OgQRi&3VG49-LSRgH|j1MlU z3Nf(Zp1{X0k2L6*teH+YQmv+JvMbrc!1&>%6TV)z+fhy@V~W{U9uq%3;Y3mMef-EN z^z5_QJGic$fFC6dMCOey?7nA)-#W0e{rOVu)iO36*`44i{q?Ci8F*-1T^^m!U?Ml}2>#7)wXMA9VAgStk)WUb&y9<@mg@!Un>LoFQl=yM zyKilqq5g7DeXhl{2#<>KEH(d$Rs8zo1?$qm5mOF#Zt>&_Ipp3(&C3O|mNT3OWPQQ8 z>9{+Kkx1^*PW=pr8JT$&TsE0$)5Ml(UUp)RSa8i$F+oIkwD+mh%5SyMJ?SgVWd1G^ zaeV4JZ=W!fCU-d_E?cyvR2KpN_UW>gWnijE=jXxJZf5vW_JSa|#R&V@nh?^R@>E*G zcH+a^G1?k}4u?~k@nZ)nFMLm8-+XLT)g5;pk+%;Cq-tm$2|I43*{kO)f>lby=2G2Qq3mQ|e|9iAdcjB>YdZOf671dW z7{Q>AN~~`abCzO!c@h^%g3Ou>yFxjX_56?+-rD8n9AMI*#6Ee5R?D^_AT{WYb~l`x z{6c7qYm~>b(CF;Rp|aDvTN5J(P_xLoi#NwqBOkcdsUrrFsBqvA%D;Ia>SRe+2xr_C zfxkQbof8Gc_|Cy7L1Y5XkBR#$FLN^+CF=sdQwTrzK;u%O(qKB@p6`E9|8?*#kw_(3 zsG=h~cZERjtOL5|YpWV0t7=N!gff-q1dnCm#-0y6iV>yceGqpr-GpH2WG5a>t0((& z5g)r+H#l@gC=4$bx?enirp46N`MKSk#S@|GvGd^5mWeHy4DMHZn%dJWkz#Z6rEQLSLK27#AM-DUzNA6$B1PHsNT; zWV+UVY5&8AJ`6ZqA171veM(_8iZBi+u~SMnax^i86g$@R>$-O5--PXji>f($nB*vr z^sEfvhG%?DX8DB|q=OSyNuW^cA%lRbXj&Hh-iN+PoC3$PR)Nv8_u+fx?y0oqSX|1L zYv<}d>KPfo%_Ce$itF_pUD?)2vX%Z4BJOTvkas*W{n6xb@i$ko1XJWdM!A2b{A$g+ zQqJ6;5+3OiP@?CJX|tyiBF~G@IQFQ~5zSEV{H8H7M2%RtNULazv%eeZxXYhG{P-Nz z_5PibpPXwX!|=krU1S$ip1&;ACECP`p$svmA2<`6rrFAaDz^ssr9@*$iYpU+ZK`QD zg{2^#MhlK;3o$PW4E&BKZjB6nk877n`ifLLt?>D2l5gb6&L>+x^0LFn&G;uLr*)M} zuo-5!U_ELeFPc1ND=AGdD~qaHkGbWYw#njRRsVY{qzJ)#V_Jh??z>lrm84>Jufzm% zrVQT*%RmI5#(_Iguz%zIL_ZrQ|8yY&eVz5L<`Jz=B~|yN9^cj1o*}<#kS6BO&#W%Jztu_0nO!$xMT6oq(ig&?=C-kPU^VuJqq5uS0~zfq*7@vn&&*qm)3Fr+ zyGKX+AB@kQ3}fe>%bzQ-pGF(&JBZ5I1v^YqXZOiOsr2YfW_{VKUJV0taSj z`#w0joyE)9Neu9w_4Ost4~X@wUfRv9gT$ZBM@2m3o1NvEH*U$4``a6Y4ZK^YOUid& z^M>v#5X(FydrMw)YxDK8VoU5S$%iDLC*OCT@cTYZVM>W-+x)hfC(ca&_?-6L;^zUV zP%&zBmlfvauqRoy#&!%m_WiD>jeHvjL-wS5!DMIavSerOrK)B}K3i)~SyvYqu;FCQ zoysu>rnztVrS{0Ae~Y(&eM-qEJ-gUx+a|o|E^=2yTTrS}`wg;pPWXtPmGt_WCe75= z`7aBvT6hpvpLZqjIn75X9T~ahpodlW?7n(^xf(Z$Mi^7uu-5^vQ!g5I&bE4#QTgz% zWa?Iv6g>hVg2jg~L)5iz?SFyPX%RyyF35%#d)@NsApCEx2Oae-GIoJYNq!9nm`wpQ zJp@N{QD#n`E}w6H{|31$W0$C6i0|`D54~n5?0{xifyu{dGBfT|13%6xoQAq=F+cvx zGA!-bV{SJW$U|=;_kwgR|02x0^-*ntX*&|?2#iw}tV_yjym{@kcMnGmyfJdU9NITe z#-7ux-Wt{(*30q?sHR7oo2ssGA4`#~r!;yT!YBg&V4EOg3hvwIbBbSDv)T9G1rB-D zzU?=t8$hU%#n{!=$P}^_`(Q2d5qa&Yd~qhjh1$D>FN%)G;#5TKrJge_9Ut$wqmb3r ztI6HIjrvkz(qF@FhkW?0PRI0}9B$?35ULHY@_Hh{+q^|$x^GwM2ngZ2TKt@)A?|)_ z%y+XH4(`k+Z5d=R9M&pve;5&Z^EEtv-8s15%l7NcQ})jfKWa=*;0h@a6Bj!Y7qQ7S zKGX0^CoQeay@R1jL*y8c#F0V5AS6`HUQeY+WBENS3{j9uVT?vr(m7rGOxvtfb`jE# zqo=3Gx7nXZ>At;ocsT+|D9TxkZT0?*^2J8B2*0HMDKU$(Ad4DA0wcIV8G~qEI0U(sQ=vjmQ`~oOLH{ zafrvm6Py^F45br<^4)qy;tPZ=Xt{YkjnfH&omutw;!Y$lA=;XzqJkFiRG!I=eX%7* zdP(Vu7v$cog%+)Gw2)4Q-T7(9?LoNqTV@vwDyNErdK5>|r1xEP53xasDN`TPWf_c% zN%Fw`L)vY6@d{lTZ?zwVM93_{7ljJh(UMSBtc|;eGwu&s%X1)|*!ck#x29VHJ@3CW zzz-0Kn|oIT@9+SljqRMM&wvi{vES zWz{5=+sbJn_IjW3Du!br(wW4~tX&}sUSBgvN-pZ_@lcAYDIKK8&}-z7EEhKS?X#;| z-wk1#7$E)*wO@-KL#!7o4%~Ts>GeeZCYqo-?rB%XrOi%i(aY6ut#dCU>^&iG_;OxBzg>jeXQ8z1 zG!M27bW?I!r-1wYBIB)Oez9nLj`QT$rS5*xh?@@VJv+9A+?xoD6)jP9KLx_e^rD%| z9)0Q9{f3o+`bM2Wv0&?bWnZvdN*Z70+I;v%8>Uk&=Xsa#%g8JB2IxekRA5E7u%nwP% zS)bp|oH=9>E!;G*R4#8oG+VJU6r)eoDy`G13Yggol64hCjzoj^(6uPHKeUM$ZR_S%=jd-{W9mwx1nQAa1ml&ds2s8b-u3*sz)jdLEK)8rGVc^D<$2614HX-3RkOsdY3rrC0R&p}) zn#d$Pt2G+)N$4>`KnND=V{Yop`#Gxlsf-7|gNtOfMT}+E;0Xd*z_!L3nO{V+yeZl{ zi|^f>#8v^ajVd#xb)c|9T@x zj_l#b)iN(#T`XyB*@z};okLdW$Gs0myl;;9bfC0KGVV+hnfGo-({)pS7EUnoM4&@@ z9Q?uCS@)SwdtztO!`YUi`mFJHX&he)#@8f#)}O+$=zg%KY@xBXs-a{#`HCLxj{mg< z+5X`}D}_&y5B0q5G6dEu1W%sn4nJC{l3s9-+We(9pU=3{a*vMbJ65A6m$IXA`v_E5 z%F~!Vt|1xzfnKii;iG89nb7fVSgi?KNT}|9v7%vOs9R@e)0oDE6bqnBB83^=h86SlUhm>?71x@I}bK*OFAs_~VMZ*;z8I zSXP2H9p)p_WfPx~tpvP_%P;L1zGR!7y;FwAVo1g^acWL40YA%(NZ4h!Nm@cXSKXJj zsrt`4Pb{1Bd`{?sxlOAnB4jIT>MIhz%5A*%`uGM$F0&ozNvvL!u8(mqsFApeRaq#FO5E;g z{y(zbGOEg`Ya890?(RlXN;)K^JEXh2yGsP5L2}cLNOyO)ba!_N2nd|p=Xt+#&Un8+ zjIsZ+?m5>rub3-k3suD|gH5wG<~KmT4ray6hxcJ9LP7s=KbE<|5kcq=5ze+HFiqEe zg8zfW6CD@0c@`bqH6$j+tL5=C$e%|hjCJB_*bN75i{uKrHMasyvNZ$m2o@RhbDnT& zJiRlbn#IBADRzEGkQ}2zAZ!p=F$kVDOhHxoSEExG%|_m3l2Ya_SKFC4vPGLzufP=? zCG&5Tg3p|>FHBYjwM966bkLNXWH8JWf&@Zsx+4%uvH7hZz>Eea*n_iUVi#Mt^6q>x ze>_bZNVa{c{q%?A@|*X@wlY$XGu8Ac*^~y>vCa{8ZT>n!ati#+S%SNoR5v@*i@>Vr zi-(j?72?6KMu5%rtNVNB$w2qI*Mol027gR_=~JjuAPGF`@b9o+ooFR58f}pX8juO3 z95b8j7ix&{aae6gSsVRH>J0V2pI!^|U)Gq#O5O2C?oity-W=z zk5&+{F$dw|n>N42vJQ?|m5g0C0yCa%jCp?@%gw=Vg=L$Ru~-XicuW~fteFEFKZjAn zr>H&+O`&v0b1|!Erg)?KI^wv3fkvPow2>$Y5}yW<(pj=}6S`i1)T<2~eu{^HNZI9@ z;p)Z-vaiz|Z-yj4-0d^7>GCk$q7T6AI-I*hnn95u(cz!;6fN7y=AB$kY1<;rP8T{` zgq>@4VrU~qvG`A6Q@=%o3f@Dt#*@Ku&>6eze+B+4VCep8{+Xz?aW(j3Ysfhn<$~ct z7*Q<^Q=4RvJUr8JD_DX)fn(1xl|He=zxH6>?oxWKMUWukw)Qny{4`oBpY%R#UviY; z20c@cl2IG%0;O5W=9uzklp;mj8YI>atsgIpVziL<9X8AAEADHLTHUaGB!y0A(fW@M zN~o1yZGKpHPZE!|mum8y9fYRc1UeR-Jb|)B^7ine#m&IB$VXRXR#Mku%=lADBvCx#5GYj16$1yvlYOa+2S+@b?q(eBqxu z=O{nlNdj-nIJRHI!6(uwRjNK;;I2Q7H(CV>?q7_aS^`y$`LD_&?dyQQKcOVgp(Kwq z<8)cPjk;PBmFTjXwAJCwx^5f)!^|q)1jx9vwwzCdi#aE11-9?LN>91g-1F>IR1OZP zx-ccS?^-{<6J$$p6`;cNHiwUH*}yazTHpE96#l$2Mi`7o0wUHrr_xBx7eU+Haq|p& zV!GN>g;}^iL=RLx8e@)eZ~tzkN_4J9bXw29hD;T!I+SM8EYobj#O{EbGdeWU^UAB! zes0JlslI?m`o92jTV)HKmJ@lj&6gX9WPBTaxO<8j$mo?6QiLP|ye)xA^@p)uNi zZaioH({~Nvc>eMPT`LZJ%@O*$-Ik2K0gR2GiU=+$@?Pg+<*%NS5DcAZ=IKzR!|EB| zWvtKf^_c(57ejp~1Ag@^O?HU9jLdd! zrMtPn^GjDTp2mtKv*oEfY&4*_6f9<4AnOhlVT*wa7C@$xp}8+J0gfvjbZ@`;5A8MP z!F-PG|2P6)pI4<3p45MrdUpRL`TB>nv+u|71RC;t$6DdG*9!n#X6awmk1p9}nvz&0 zZ*1KmN==7gxeCJSp|Ea!U~4E&{G<`Jf_Kq0brNhk`0=vI$B2^$7C!C#wzg)mdnEi~ zI?(g&GQ}%AJFclK!HR}M(B%A{pc^1|@~zt9prlz%!}r3kVkossBYfE!d_gE2b>7K= ztiwGcMeOW07){bhcYHXr85Qvkk)O3MasP&#~9(I!-KxO zdz`-^z3m$OVCY-^--ZVUT6wOpZ250y5g+27Cf(xt(_PsXAbtqnpT58tjc-U#sZLLZ zmpgP54B@EYjB6i^)g7C}g5gYP6Xi zNecOAXBIzXln!QV0LG5PxJtEsM6Qc6|w$7X06Z?G*ZAS9IXFh|iu|r0R#A?lFSF4X4 zs0E<$Rr2Fbsx{vT;kXk!6x1!ow#0Dg8Jxu>K#93i_*>T9g$gtP;LYzE*PjkYkJ$u; zJaNhIjpmSy1TH(PxwJpB0c{_iwKA+M+i_iR*RSE0tn4H#lV{_f{w4_MZ|xt|8uo|K z!`{xe(Kkxd>eo_{rITn@3}ovCiO-!_7Qk@TB_`wz>{qjm!U0X619pevOyt?Fx4^Wx znIJ3=$)}}tG>qQH>N9OhN#S_8+6wMQTyi}VcR)$h!c!6?Q#h=2?+NyDE9cw5neu0y zdC3jcO-6lg3@=iGMvsx%mmPhlG+x8DxpT)htJlJ}L~dw6bm}#CBC4S`$czJX3!siA zjA4Uga;O*(R^2*PWpUwZ zo9vPxfJ-t9*6htK-b2U1Jm0qBNk=2ttup$q-Qba^EsTHL(VZbY#1F;L7dyp@t@2Gd zefZY^+HyGmZ7vPpFLoGtRbe`E-eB>z=`&Wi;Tp!ZwqL1I4qr#5j6&9|;L`^%qTC|7 zxhYU9V6b};9U%!Sd-uV%fO+#3q_;h)P@I7gn4Y1TuD9<58coH2R*$MUy`f748PHiruU*%~7`mGO zk-JY6#LOoYM!Dq^0jrfScS&t^AM}*F|AYlyOReKK#%?zM2)dpC_uU<-?7}{rFLIeX zKh|x&mEB_fJM+c$|C~9P0C;T8MaYi>_)0iPD>wcg^tBr%b=^A*^e2NMq|SUB_dfaa z16nH$UKzS0DXBa5V3bMIqzRF%>s;7_3}^wK*ZFMz;0!~{ur)iB(YU8ov07E)6wibY zf!R&n7d+kBlWt?J^3PQ3V_2n&-z$_P_%D$jR&bB~In$k)(qrw{iI1Jx7pwyd=`ySN zzqin=-A>oUho{{VwOHM~2|5>s zW}!4xs)^g^+@k?@MJCUR(mxJ$>s*E6(}xp2ogO@2z$nWG7)beg+As@T9x0Z4qMNMC zG5i#a-7dS6PUL8@TKbBT{IkRL(9OO2oaujVA|u3y&#S?_JrLMzzaX26<;nJ6Ikt|U zCsir1unK!R>X)9BQzne`pv&YvM{e4vO@cTrTBMXOHl&M5K?Q)gb|1anb*^t_cqkouEPmpvYWsHq&=m!#Ou4oD?DFG=IH!0i|NYqbs@MLqFTN^w zJ(m%o(Ui#SbY4O4jK1^OyAH+?gUg)@iXB%nCkY#jwsY>z*al5iY1#15|=Ve<*piPdqTDpvc}q zu__LI9KAT!z#s1aZ4?dm{c#Ls__~^$IF3+R(~bDM_9O2P`m{d%`lVQ9$1@=k*d+`) zK2=rIYjYmKa^a?UMBKlReT$FCM%|G!rzY;9QQf#Fumth+D%^aC2;A`_*-lk6T0?Ck zNxmZ%p5G@TF!pj~C^_E`oV7R<5XH1_o=h^|cEASF7>h7joWN2rkbbySk2O5LJk{*} zq3HuG0Aj3a-|_q5+gm2MiB~XD8^^t_(*1`6ygU2`oczLd#!N(B(6uG>V4Q4bCHH5U z{lm$&vY&D;8}3wxNk0{3!3EjJUK4}IEX%))QY_~pyW?Lhtabe8f}FWF4kUqQ;I$H} z)mES}c<;1Xu^rGnUs_pHCrE{=KTHC*%5uvbMrP)sRDi3e0>o5IEz~i-_mBe59ia@ zd2gjcd^Y7@t;+X<$_w|(hLOo zuWpVfDRPSQ**ipw6{Hu;d|>ob7i)g6_O2(G0_L7HLuhKeb+BQ#m+eUz!>Ddk-bR?< zIDj)9Y@zIY)+YZ9%r9uDxlDxq)Md;gNme>RAgh{=gzPoBC5!z^mJ$;v0=1k$Fe?Rw zZw3}jx%c4)4LV?>EbWiRfbG~{?>VT86|4)TM6&AE#CvVQZ}9@9>~s&3K28&M55vcm z)Ae~1rk5IuAk8zhAEKM_9U4lvxg18^I}l?c@V(X^sVwck$?oe@x8hef!RaT9{QU+m zTQck=I0sB(>XABH)?1?C_CQ36r5suskaUQYWPLjWw>6;R8W(ltKuLC}`uO-{>4=`h z{RRr^$*~O=70TQyMg>;KLj_%UjmaCljw*zu$8f6n~Azl$PI`TXoDAEk@K2N!~oDkbI|Aqzk8`MNIJL(>|{+ zAwByV#`n%Ig4+f(8a$#jhZPnKu)))4YQe=p2?({nKGgM%co+NPSf^N0OSmE{h<)XoXa?o_v(X{ogUEg6=N4SO8 zT8Y*CanPP^r00!qo#Np5_yY$4Y{B#!ZP)pi$&4{pG8*>Er^eSGB=A9)kDXP%7G9D~ zSQQtvszQpz7}XuC6zHT^1==Z3;Vx84;Rt!}+iI^6#ig`Ca6cs1tz|%_Elpi-a#?%& zQ=F!Q(K)ZbOa8hGEehfq`2Jq&Q&{qq0PiZnMhBMsr^@khSzl4QdADQVCgzmRcvPF_ z%f?&`-2T=0y_%MmmhRgZdf?8_@~Jm&WwZ_RopY;e=}{>gOZL+J?uIO0WSPs9;KR7Q zl}2w`E6;9MQ@$)wHIQp89P0kJwu_qf4f|D{e9OQ;{iT zJMiFc?b7lq>i`1K7Pj45(Jo=>&@f4KdV(W(q1=w*(Ha7@<|BCL*^#YHdht z{C&*c?qW)9S)Y5E2(Nluf9ZNQ4?XiSK7h!l(V7Nh2?22u2HKv;QA!SLo&8&=%rdJpP)@KB!`2ux z4dS*ZR+Jg-kBlm&~~=t74zInqfttgdM1LHs!cGa_oG@N0-d* zrBaR6K0%&8g&-IWqxn;vrSU;qRq*&;eNyE-fQPc%4Sc<77$D0EQF(K_J(vItc16X- zA9GF%Y0ux9HY4EeJAeYG0}po1`WxHcgsTEyfLgbfC*w^f%V%W9oY%3-tW-#;Z4{xX`fqF3Tn;=#Iocjc zP3u1ilK9jM@cLNl5uL+WXGVJT-t|IDRiLnRr;E{QdC#<2I5q=X7R@w^nUYzhGL=cG3q*kWZsyvY1>neMBhzR~~z&>wV-OdpD>pi{& z<`TM|3*1$88~fEha~@n?3xw(S6vXwP0k^4tYKh@ z$&R0+P$-lydzJoRauIoq$M*5BKj9S)bQP0B`N#bhAnS&$#3j&qwW0gMk>V@-EyfuJS1f4)qWJx%K^~HU~8=)|CPtbq^KP=VkuS z;6FM(-tKvseY>3@17LvCv%2J40e`$dX2>lK!Dp&9v4^I$F9R{+jLo7XPUV7u=o$N} z*A2YGfaHGD*D_BUWj$4N@FBZ%srcJ{hx*2%I0u9mDU!V~0tGy81t~ z9jh#RZ$L#7pr&az0AclMJqOCbaset(`a&<$4!jxZljqHkA9PvUB8Yty;y zb#z0l-b8(#K6H^c(EVPU=SuVRz21*PA^=7Kb|bZgx!;>dXHYV-%;hLAPllPEYU-N% z0E6(aJy_rDjWjf%<^5xV8oCTSe`W9u@tZ{2p6R@GBZsk!k00MRVKG-yXXebC`%@#& zt_7w-%zF$`=#zb| zEm~cS(fhq>qzFrKuYT@LU5aX)@!AAy&qDkPAO7i`Psu0=@3!oXAs}e_8>=4_6kta? zbYz`);-NigqjJR2equ#g+cvWS5I*uLoy}hA-TT$%x>cRkHv90#gvny%~#t) zgCEwGh3F)(0m_^z%y)<9T-XH7W3RY@kYxuTj`v5#Pmh$qOl)|>$}(uQymA%uS+Oly zyiJ5jPhd(DR0bjl6^tnfQLf7x1)Y7v zPocDl+PnV(A%h&gWS_;Sb@hH&jlf`4Q1HT`oPG)yK@)*e6Dv?b`U6Z>#kD_cntpD> z)(PM*#?J$Wv>xGQ)B;DuB&)kEw(BHoV~4_iF%3_Xf`7_C=kvEX{r%I^=bQHb)Vdyv zrjJkSZQf*A;mz%#?c#SN5B*IaO|TfC=%EorvB2hB&MSOz&*J&4e)vD-{a@Mkai|R6 z%V(Pi`Ncc!2D!$AoJMCuy1p~T${)D(6U{h4(1$i}pe`1QRf{n!C5t+nXf8S)_~%`> zJgBwv3#Y0+cI&p;X{*#7rz(5n>z*`4U)?72&h{MPzu=4eAUne8ZNG9-`CeFx9&j0a z6G^n&swpkOmhuQ8HoBaVu5i46u9RLxkwJ7*h%0NS9(0G3c7%4+G2rP1-Z%=;SiB1^ z_qQn0-|7E|y%eEk8QYTP*G$GCIJL`ET0blwl6}ZrZAtyM_lhnIKx#XAkC?Y(u~>7x zg>(QUrc(qXk?>V!71INjqv{dt&U~hyeKnBo_&PRDBPCw!vxH1RSYG#=!w&)+>*lHx z%23eKtVeUO`MH>igJvxK#%0J!qLXTVrGY=}pYYSqXF}Qh9%t80H6G!?)N|i#&9c^{ zd2}+w3GOppYS)cFs((v?+ax=Y(^XMD={=v-GX;d)=6-&hUP@O6#h78F84yYk{ZARR zUzl7wL=0XW(<_>HKT!z;Ovre8enmrRZf^{gN6%gr&W8-Qg!(`T?`tRcCVG*@_`B5E zjwfywLD9c7EwccRZq@Hm#;Um}G8Em1pRc7o?cq}u8-ZDm*XZa5>fHeUhYTJ};n#=~ zK57nfbnl9KqWN!O+sN?5WTJ0s{qyVsTJD4ss@B0+);VD?X*-VG3;py?D3t8@vAUy9b{*bZd(-SiohpIzMpY#|FW;5Z1Bm>pex$i}AA0m+BGnz$qyv zesUg|I_q~xy`xH=aoNWS>JFOE@R*1&qQ~{m_xHBJ>2WA#cP}GE)U@PjTFyndc;qBxBQO{ZuLC zdi|8t{EpQu#TTdO3W0vQlC(m>p%Q=A+=lO0=%TEr@&0G&?jQ+rUZ&7Q+uZ zN^W-ZUWP9Efwt5ly5)pn8ALOD-Z6?O?E6P2p-OYFYJ-g@Pl9BoBd=~laihaaO6KEx_LDS&$ZRA!>hDMcU;!X+BQbKcVqrZc{6(>iGXM6 z|C~IHdPcFvs8c$?*`%!iik^j!XVmJ*f#$A#bsRa3t`W=BSk-}lO?EHAL5XK3CpV6x z&##lP={3R=d+yL9B!)B2uBs|zwEhYQ)O&tF{~!OZ|H0`*YstJb58!wwE-$j;G6qVA z@d&zg6FEHXIXm!p)j%Pioo<_NVK@HnewVTA?6o&-i}=bq&bdGCe}mj=H>V)9WGjDP zswD*shh{!-_#g(>nI36Rgnj}~NWMrsf&bc@-qbffvQKDeX1zgj_v?vS?fM1ICvb9R zw#aDoNhE*}u99l1ky@YuH!sZ4fx!-b37XH+_+o|z`C^5k$JabZVtOM~`Aa6ue5 zd0!59t$DC3ya>I%`AICvO)N1Gy}6a|(pZ|bCP|~1Srx=m6WVaixvfWgSGoMv3B2r1 zT?tY7&|oqU+4Y#wF+>ngl z9o7$kpJPD=|4wC+*l{oO^Za+H21p8oKI}|~WL{oXbVna!>0e&N; zv7J)Cw*C*N$mgv57Ea3}R);U!*?J)gbe1FnjktAVF?&JKgLPj7y-d77!+f~sC<1l} z<|-_kvYXQ4_g}gTRx7iBDs~UbQ(Fa~U;3D3Gz-E0E<(kRB zJnms*s#o$DOR0?D3hv=`WDwP!;V@nBDr(35v_W*q4H1Zd2b;O{z2=udMyL>D=^&(R0kU(HNFbQh zxQ)G%puz4NiA@X(=1w$c-eNn2f_#FFekwI%f6RHi4Xhl~^?956x}6bNdVp?e9{(BH z@vV998e7^1&tleBBHw`@0+6a%NmAR;dMUDVo-;`s*jgzqRmW2=5tQ%|9bMe=e6I-S2jdNBS_aSqq& z`E^sjLAdNk5g+_;Uz6I$Z%VNG)vsPJj5YPOX-sKR%!BD;YVdV;WSS;z_&&FQ2S5ae z_t{hfP7em>n%UsZq3v@$Fr`@hMYdjWrYR_;BohdhBIjCiaJOcfhRyRm%TcL%Q zJTG8zhhf^LhF;hM3*h-w6{KZ8vY}X&X-#8v9r~Z<2WJZD6jG7x zSp=C4>yV}r21_EE*9Gvk?;m{f?@C52(96>GzBN#Z{sG(N+HhWCGD6D=5S`1St@ipa z%AyY%HHTXEiv`n_?#e$5d!C8DbkxHayqb7g*@ps%te*5_+Qf&3j{{eUQM_YXqcVC|!kjHdn07tDK-zzR{ zP2m0D_Y1@ao#WIOY^QlY0jy-^D@7^(UUWeFsiavk`nU*VzQ3>|eV`)ek$nIn_F-Qj zq@{!izRu8}0ui%qE6vP@gDsE|Xh?XSOOn=gXw_-v07H*yF}_5*f(!d_g6-imJz_U6 zxg$K{H@Vn`5*(}W?&dTdj5mOzvz4bqIfBv|zr~YxjVVfc(BFMT+VKIpfu2$@s=!{C zs*kl3)c@6Zi>8GqbA+nw+V4AFpYiD+fnkZ$Gb`GPWyk_zsx@|H!GmtqquDoYg5@zz|M&+<>03cpfW=R zKB{~^+A&s7M7llY+VCuk?n`i&{uHHd5sx+LqWh!DzWK2YHQc7y4Is99)=b&>Ix!t% z1O>?*yZC2pB`7>YAGOzCi*j_UFLRME)|eP*R<-q_OTfe>CRxpP70c8A2!f>1nxzYX z7hck4>BCEv#v}ynfg_PKEo3}?d>aR{?Gp`Nr)Ad*IDk971w*OmbCJs@OFe!z5N?!z z-+?dk)jn~EwpwN!^=e=UAw@VUfvW&4;0hFp68;-Qt_VW;21wFeD1(L|IJp?)E-Xi1 zz$n+asJ30Gtq{e^9sZ<($==gH>y`w&XLQYWM7{b;6_x?YcDeiuS9n1X1jx;e>K~*I z*Z@~R1u}{N?y0mYh{H0UG*-LUom)SMM5<6#`lC~`QPeVeKRDW`v7><-v0H0*>OB_( zW^_Yl zgfG*Y>1T6B8+{`siLL${QPltc>6}HtpyO9YL?F#M+CwLvBX+rimiGLfMgDxq&*F=p zgXDC&YgINL^W`{S=LZQLc>-{%+UdS~bEXGVHtI9Md+pd;d}muAU-jpmLG|h`G~uCy zzm@xYRn(dHN3n9#@~&Uq5eaRJg{7{pmnW(qrfCp-38dXCoDXl?VJ|i1dC3PFN=W-e zq-M8~E_!>j*`KvkVPp{cwf?S-;CZ=?v;6KIINUw2q%p%n=!ZfBCU(d^VM8|$$qXp* zn`KQPeS41#mLy3bWS@+0f(ves0#sg?E7AN0*eSCTHtL}XM6#1YZ^L>3grcWVPkUB& z&E1Vc;EEDLW~+oEE|Do`rsyrS;V(u%W(-tVoqORxaRMPXYm24UU)gkHaM0^l4$-LSd_i@-Zjf}@X_YH){FALeSDs00l3&V{B9Kv#hCwb zvWVSjrxzTXbO$bCP9TTc+Mm|m>a(lGm%Y04SF>hyz930a4VKy!#PIP z>*6M!?=vGj3XuTXdRPHO6<@cO8nr&u295h?3v!|IcVGw@zaST6f@2&mrcNNYPujj_ zeJ;mC6fRytYvUS}6=yP6g&oHmqL)sQTFkmD`^@&1L?dg0pM{nFW-xGxS6(!P!CUVw9n)zLQ#XkSi_+p!8H z2q_;mVHG2)hep40)~Co=nbzoUpAlZNBjt5y)cj}(aE&YwOg(}3Ht@t4^8!lX`4S-6 zbDc4nC~t$B3?%eiCBF^0IWJAT)0f0@B|bq_=YN4?uEy+*JrEL?9kkG1TVpZa#5 z$Flvw9kSw6{&v6lx;hp3>U!80RphJwrsj59_n2x^%l&M4Fp2u#0{@`yW4_?O@s%fT z$eU(V)bMyk`9tG>%~(%^t=_j58X76Lc`1ee>=^rX`%xW&_>VW1zLI!PuIN&&>(cf8 zoQAPYFVxFx`t!r;^M5|6tNx`A!x}@N+nQz@T9Vb{@5ILT7D=%@H}%AKB0RW}Y@bgS z*Zs>F5vMo81seREQ;rx_Fz|AVdw?q(iFj}s{H^@PZS8x5xeUK(Z50$sHP@139#3GW zC$E>${bR~81^^H;c?La|ninw{L)K8F-RQYEFs_z^Q1kG@eTnL&`;b=Iu&-9SMKXUw zU0%JbJLE(oID?$N?}nd60Kt`W4G1~_fYg#Z^l7zh{|ii#JN*o5_WKwQ7z+moDPq;A zB*x6XD98_15n}@w0Fk&h8}p_Cu%7xSoaKIB74xQ$^c$);81Zac3NS{Tzru`gFW=bJ zL%R=EsrsXRi|@vJx)FAZGq@nG{ji2juH-9A%?7!EGRK_Z<3RkHp8aiPxL(~T3rB!QSoA#w$-~Xrf(*~3QMzZNFLl)?71nU65$_||$zb9K$U3Xn zXNln8LWvDyz;3N`N^XU8F3u(6gn-e&MosK{OB5u+ZZ>w}(R~759lwsw+##yt`^5|# zU5>i4@~@xjjE5m75=o}Dz~Tdj`6!nEcJt1f17bdM8<~HUjCq_U%kST{n)JO?JKw8D zDwo)2iRyg$vVG{5cRs0O>y;ln9Yt~+5TGpCKEZ>a%jrsaXIwp-7XvtM2^@V4GN=f8s^3O@amG|*{Ww^lm^M=j=%n`CW3`0Gxk7WUk(>R7yS z4OYq?{Q-LL2QQcY1v!+Ch`6f$?l*YLE@tXB`(GwW!!ewl$FuR%CSUtG@-Gc5vj^ItlAMm;ZW=g1a(}#x{voVWAkOkg< zlX&_(UU;tXDUZLCU3@XXwVE6UCQQXM8!hyvVj>>~<)?ys%zLEYH-63=-sfs}%7>F# z&q{B{!hGavAM%0b4NTmn25$dV(e7fBlatk>_>Go6cmJ^b`0~T5xJ>xKy%FLY#MXgf z>p6{5bqIQkbR2{+WjhCpW;p#dUMBDrbLv*CtHv5?n>bi*>+Ohyg1#;{gQ7O_sLAMt zdnGp7y5E1*Be@lbi&l8Y*tsvcN>9$#m1-(qGgjO6HJa%p>|vcA)_nU_Ce621l{Y7ZJaiH7If7308Gf?;C@bI1pW}r?xJQ09vo}Ru+VZy z%A>hT>HA9O+T}*w1BR`_E}*hhptMfQJT6k1d-JgEV4wfBpGFN+RMtB>VA+k0tDfF(lqE`RL5gqdTx z@59CX4jMt@zrjhCydq=!M0io&*>f=@5{Yg2+yqwjz>I5F;qMB^$F@Oa4A!c_gna`VQT4 zr%`D%?phz1sMchPfq*S7j}~lS!An6(y@9pbR^;mY zFRh7vWXC7RV5}g6HD4fHXVcC_;W;S3E<}1RA`z4$VPRhuLrM5fLXA4e5@-jWKNfWY z(UfZzsfjdvP~NQrCdv1yI^;#k@4mP?h050;94J$HMG$oWalc4gfn@ePj}V#RU`r&* z?nuVJVv%2#uMl~Sp*I#-5h!PeyhVbr2`6uH{I-aiSmIHI?UEa+3rVfi)6-8&`2+2( zs2Nsl5gYN7de)Qq;Q-d@M zyM4eM-ec)rFvJ9H%t+LngexD$oGY|RQ^c&W9uO0Fv zpH1L@QG};u10>&ew>?S>uicaPGTAJ`G^@r-VEPxGX>*=M2*Pd%5wz8a`dgNGlJbpFvL~S&JGJF(hGj2vA<0x|qZ(oB#wEKcxxLODhvLOUX${WSpKXSq zxo+Q4{{e(zWWG?Ht|l)x;wdK)36WZmk;A%+nG-{}Mj(sCl0 zKcJr}hJmVh{KIJ(R7mQxOUanx2CQZNjOr(XctpsmKuFinT40iuGvoA)PCjZtRu#g= z1?m>j4mgoMtBMzTX{#Oir6sDPPb0136z?%?y9@tMm=e$Gc`)IuUM=w;um=g9VaOiA z!XD{9+^q5IlN?=`Fn({s=7E_16tLJzx<19eeF93E1Ou_ZVSe1W`$#qOH|)SiQE}vs zM>9qDtLru7ldUb8HTMBK*H;<#{luRZfOiDY;ElrWdhFVGLq2P-Av2F(#{$Am=&6;A z)0c6;IZ&TxoP3XC|IvHKS}>a;q0hw@iVuDqE>O+2*mBRWE@umA(`f0yrX6*nH(3B7 z15X1UPSrt1Eo?-kVC;JC^YRAFTvCUW5eGZiJ-6Q;#z-3LSH7FWHfiocHv#Tm3lNL{ z<^=5FxSR)K2-?|Sn0qb^&U=2mA%%)I2-Rw+OZ{IBHuqnHeR@*)PlIIKAREpRgLLyHko%K0OLBwL;vh1>(q|nDc{Q~VAPh%0I<^|mralKA* z5;w3$cr2!&H3P&k%-%)i53XL@=Lw^4M(5$*f9qygf6!%@9U?P!U$TpV7v55C$pw6Q zjGB+|Q*LoON5o-;yZ&{*hk@mO=2V7n_RQZ}iJhuJU=*<7|z@&jG4e z(1X$gu^~v!d&?GVWq{N{0zkYp>p`iB`{>Nid=9O)<0P{gY_oz`MDLM?Ps%JthjQI5 zz9WLSO12@Z5qMw4kDU}Dka~aamF=K$4W&Z@FA4^FsOL`clpf}4TXBv0CGXkUcZu!T z%#{PH7=Js=kRp^zk3Bf($c7x3iZ`(WH=rDa&P70Lasu_Z0%ElFtEF0p$QGq6ggF+) z8HTcjGwpS21sd=!ijwA}#t!h1EzXk$mUsnQzf7wDs*O=MAvb=D^!1A+%0H_SDL}de zlK}m~NsU|Kumu~jo^?Xd!es!43@KTMF7b}Td`7yEk-SBu*cAp2+Pz&McPv)fuugRG z8^C>*U)|pYI4}HzK^nl~ufV)O0kzGR|BZoAe*^2^@bK_HlK)#`a3@b^yD;VcpnXKj zJ*8Sr37m5OscFLX!?PKcz%n)QtRd5>_(-}aM(8-r6&|pWVL)~z1!t_2Z!J(knbwc^ zwX>6H!VPE8>VJ#9ZjOu9(_bP;MoHl-Q5kK z)aCq7gZI%J#1|0{>-ZNvuz(}1Cj=-HJGQ?e;X*d%!@UYMoSA}?8I1c)escHE*nht@ zeMr3c;_>JBF7HgD^KTLDD`3+|WTXSUwV3tYok|?UUQ_X;4`$g9YN_%+H%QL9nCIX* z?0q=k&JZnUD5T0Om3xLp4i<~B3rJC6aY5Wpy~kSc1Y0&x7FH7PsBL8R0S5#lv>4u> z8{iY{0qJ!!1GT$NZo*G`3PTB^&J34fy#O}W$-Q{jqlZEf`%z0j6HY^go2wh<;rO02 zu@_=`X|bXkD;w%qlm^n1cnzPlQq2u9iE`a{L?YHxPmQ6W?=Z049>k4h9|?LuNYNv< zD{~b&Y^0d{QQf3-W8W zfCSS2y}u2w!hK|~_@ML=$j5J-EZ1405^+k?CQe$;BU1z8E3p3QC%*oF`No%)r>Ixx zynkbAxz8?qGW0}q>KFCbo;GtI2hX+Jk9EgApCCh-H6Lb3hq^e{;1_La$q!90G>yDp zyh-dh3+^9uYkja{$)Ny|frNLSr^6&wDRhQhvDViyzJbC>(jbzyrO) zRe<-OYfYE7?QVpZhPMYpWpG4`t|t%JV`T@7MV=}OSVNrrpDKT&ZxBy@_27nmCZ0^F zfU{0f?WH@P66a4EFbl+TDM+CF_-^O{nvVTqOzP09PdRZ_AcQA^+4j4+<|Ld!Q2y0+y5_nM-P&wWL3*8PqGM&tLf z{R2Cg*B5Pl9TsiorxEtGpOw25dcc2l?AQXGm$b%#eIx^&e?z7079yv#) z!jknuHFD7V$^Q_BzH><>E_BNUc9=qafuYKZLMY1&%hr_hxYmrjp zdC{HyDaNvyfv6%{c`Jsk6tXM-RQitB=U-z*UMq-<{(#?_;^SsUAjO)BFRKB{F=sTs zdh7^0VC7Q#(E2~ozQq3wqbhaAE}?cfg0q51w~<}&=vUvCZ2ulbc4H81f&r0T>-2`gY+9Hib9T@(8 zi#Chdvk9F*Hj`)ExI)oKn60wcGWwxSor$B(GCmjC3LL$i0>pqXAYCMvh?ZpExvI=s zIf9%g-AuMFC)&+|X_(m6f|7+v&!s;}^aQhFsZ&^<-MFRxR{I5|s&7iG89Ve%$u%x( zo2&!K@}+q_x(7NI(p|gmrYH-ym4W5_H@s}Y(OwyHwX(^X0GCwQ@#JJcY24t$zk-NvU*=EuopYG<; z#b#ROT)XD8EL-PzP>#}1$e{10&%zbz$WwP(K`OnXpJz!N{;i?Z9`tt)GO&`|$eaki z*w{QGK}r63G0LV5I4+Yk+y5T$-9;!gh2;OrTl2O{@FKP$sZ7b)FC95_Z zq_@-^p%K&h1M?DGWJXgf9;_>#_bp}7ezBBEr_vJ( zMUHmuV*d}w#WZLTr;djt6pL?zsF6}YC!`jo&#jq2sf|TWn5)iGCvn`1PFoGhw}XG? zG>z$oas54S00V;mbhZ=Jydf!x8)v#Py!YDokzd8adD!q$3YsY7)+z5eS~6W0N9hZ` z5IRI)*af`9ZfFDH$0 zDI<}4k7ZzR`DPV1gsvehvf@sCdk?*<0h7Dd z6OvQk6|`Fq;SD^Z2r|~S6O{%2V0Sq-br`_sRdrlLZ9El19PQE78PFbJ?U^-VIb%K$^9H~kaO`}zhkp3>G2nV&mSZv%JvBh@WO2^y~fG@ z*E2)s5C2blZ{3wu7qxxUNOyO4cT2Z)iqefVNQbm^H%h0pfFL2=hhh#stz8-YElG95&CauupLrc{#B4jyg-&ztx{PKCuE&2uXPDh*F^ZzXYgFJR!Hb}+b-t!-3H4LN$cC$-P2;()u+i0SQ%Z#0lwaJwHP? ziHuNfc*AKHDX(qIOK$Wu#)x3h{Z1~u+}|Z7J$>8tQw~N2fz*Fy(d#LA!_UXVEXafv zfjB`&j(wD@?@zHW!_9w6z5!`ruZw-cyQkyl>YYkc!HC~Rnlsv-^L>Vi?^E1qHxEkd65sDI@1PJmh|J%vBSM{LE3P(e zF5>KbWLvxNV=;ItnKq?B*%l00f?;<2tIAPU_;Jj0AhtdhA|QZj z0f0u~X-QXY@_VLeNPb)fP*E@4z@q41NwjpR#@ zqOc0*b(g$RpUth8m#5beC){DTf0`+V27;J5*c=-rTWga}!(Z69Q>szca)Mf)1S ztilk7 z+M|c!-wnl$)5GDrD#;{hyC)LYIfJ_R#ikm=q6TFS*0yz>udk&|-VFurO^-O{P6x|( zU-_~LE1u`D^k2UA@O%9N*gmsFe{MF563|SftLsfNM)hk|%7}uYSJI5xGmO!pxpZpc zc`?I=Dxt3ebSK0CR*AwskbWAvi?S$eX%%E1oFSpAqsPggA0gK3%jFqxwxemQnuXRr za(4xffEarAlTEDo$ubA(xKxHzIm<*k!Ywn_eWlNAKf9OfTUg^qxE!FgGMA#NGhEsG z540H1vVO6>hmXwZgj38uOi!?QlBRebitNkcZkS{ZE%U|D1VJ_ULxkVnIHD^QvrUdH zIVN>BeSugp1i72ohXKmWweH#f@?6Z`D@$GJJ zAxj?P5+@W7rr?~-ep4SDzDQQrGh^h7MB_UQ?QJ%|RL98~O3gXVvrq_DNg$j8mBZg? z7roZfl+kq`@}YeYCtm(`oHZjH%}Szlm+sW z46!EYq6=mQmDjQPMyR~o!W0iZ^ZQFN``sAzl>&i7m=UV<=ZaPQP(iTtrYcNN z)=@%N@`m10Gom~Q|Ij+jx8@xu9Et8k)NYEnVSmFunx$vamsx`{){kkvPTV>b&L#W%T>D(4#4Blz$1hQYHu%hPQ_wMZz#oqGHiU$d@jC z!E-IcrF9mayRxXl2_uj4gcee=Oq?YL5EAP(GGKX8DkO{7e}VdX)9kd^{oyMlV0Q2} z5`sr5#GvvmQS?*ToJLr#6%X=UGNr9W7A|w)H75dYBgmJxCcwUe_fm&1i}87dd{zqY zrMUrHfEl8)`edUDW001JM5AoQeIf%$tMo3zp?D%j!^u4ZBO6Pd2r=t>)0=s|W#hLw zI9Xtgtz_7rhgyb@0<3PP>^jM_{9mvJK|~kCG8<=vQcyXi^}TB20OtFi!!W*;;{*em zQ~dtS61>H7Z4b5oET0vBHz6+@OtV<#@2MzgyNJzZzrfzIEV)QC+NT;@xtjaA%twE& zzvz=PP^egu&g2#C&kJZ5rfjodz1Ym6n6!MMJ<50p#lPY{CTeBSVcWoJ9Xd&BCvwJS z9AEb3*Lrqy*tH}2xiJZ$1qK>7DrZWUg5dfP-ugwTWfFoMg~S**8l@p&=VxAr-w*j{p` zo3Gv!dn^G>klp3Z-BcqP~wCIuel)-YNxcdi27O zn#+85y$oSpBHMdc?&+&0hj%}AjXsMF%J}ai6lsPJ97ioX49Ox^F3othEA0x#Bti2bLEva2>}Y_ojuQ2NjE=+*3ZcvJ85 zjWFjl&za-%ahx;7m4?wZSG^GBkfaG^070}OGwOz!erel)iMJxRK+kAz*Z5lZ7i{)$ zB4A`q%p+JUr=ZI={|NXCS?#;7ujPD*IDdlefISVV8NLjREF+dc%nl;#mJdqp{fI^L z+s{IIAR|Y=l7I1B>^zib71?<(DAr*&qC8c{MVgz>+llrhXLcyNbH-@5`^9;v75X#+ z;#yJI%Td?5??0AueVVK{+|wPZ5e1{L(uJ1mcgVRiW+(2rrVy#cn&575=F7`Y^%6)@ z!v@f^cXnuZ%^sE}hy32Kb-)%X@! z+v5zsiAm9La3`yjv7*_(_Qz!Q^BCNHhT`O&7Lc?r6Vs4RCUO3}e)F7*<<9Rj86qbF zuWRq9Xuf;bQ=4I)q_;u%KX6_h&}HY=!pRQ4?JwecN^fAwzXS*XXPNp8yROvJnVkcL zJH7NF!=gQ)cs_m8J;~ zg{89%(K43aVeMRfnc5&;LANM(n@N93h)h*RrJszyacVCyA{xE?Iab2erbTx$J3iqya33JvcIUvk-;(eEcbN9L#3>eJAbBJ?*25LE@a^EJ%Q4h_$e zRKOG`e~>v(TZvJ z%Oc`Byta#r%cBiUQs>=&7tV*;C9m~ZA|5M2iu>+Ai|38a7`57#rg0vsF2| zFJ7V`)Z{aV6+!b@a6Y_2JFa{H{d65GyL}U|^|z%v z98}wdX7(34CHbGa)|Sc_aC0LvyuAQ67eB7)YToH66wc`r|DJS2*Wd&)Oa+Csw!6Nb zA0@5*X77g*e1nm{8D@T<(&|y(@QXmH<25?VwTO<;QJok$T|YRDsKkSqg{*lbbb#d<|E0~U*f4H8lWDRMfzxEm?e$041fnG?Hza1M(DOmY zija3{wuDv}_6bp~nEG`FcyAGma8mGmKZWaV&PN|L)_cwE%S7MksDjUUL8<9;Zpu;& z`0-0$ro8@rUcp+#Ak(`j?=1U0-isNreDQa*&&gv>DZ|0GJv)TEU}&l6w|mvp2bqzZ zA^O^dKK(5Bxll``en|t=3XG?+v0(N$>6!pYOOhkq*EnM}{b6z|$Y%36JKn5zJ`&wx zpz#4&^EdA}P>gvI*IYdPew> zAPMLs05qud^ZyN4UGIm7hnbm~UB%0GKm0cZK{3_td!ebR3B(Rte)lNKkKgsRwepWk z9w?+3%0QRf8JViXlCMxuP`>Ri5PCp&nXwgG*5J}@?uMvPclgmGT!3|6YvQ`jY0g{) zBp18}E0*yamByYvueK4ZHhXci8UO+Y>+NhZ_2F8EE9t&`RNA%~SsW4b4B_5{<#|gk zT7K2pmnv^^qR24gfv3yQniyj}>jPJ|VFnRf8Do(r78DvQs_cwjLN4Cwe9WrZJC7RjVrgLpP>8A9nT; zY7Kec?FdV54;e#Lx4`czH{k-cJG{69aN!N5J2q-Df|K2ZIn6iHHq@*lTY0i#ms}TzB$u$kI8O&T5 z0iQfTSR{L8!C~lsli?AoFCfr|Q4XU13g5gJe_G_Sf*jbsq#}L~=|&5FMtHq>Pjv}elHDu3 zX04AgLq?y$Iq(>Ccp<+wQxffA^{Y7*dUlsY*&m4RI0dOs&{|Cg>MDM$ako6+%x<=B zddwbH(|9KPaV{@J>b=||K_*!dzmVfrlNZU0^88GH0})nfXF<9oi>hK z^#J7MMy$^+3X4XU@TDC*npan)suo}IWUhJ^^u4&6zXd+w9sSZptW0@yjgI?sPe;n2bK-eG zKVuA|E?WV*L)hxl_qnStDdC1XGQ#0GL8V>8J}6eMzio;Vc($AvvXif(Y_3yfNV>y& z%g`I4auOKap+_AsMyP8~+Iy^9Qr&<)Z|x=*h5xF2eii3gOa_JOt10L-GNf-!E&yRr%W^4~cn6+=E55hN$Iy{%L@DaAuu%EJ~o>G1iW2pWUfEEC7 z_t()4-*s8iX# z1ax&}>dR+GmbH09Ta4m`k;A?)%N_44cIwRwaO3Tk-cw;gqP~te_N&RR(+PDEn9WIB zr4uq43TFtyDV}2eOz4artJ0T#<%$t1!e~;a?P2iWuQYH_YS6g?nN#>B*0!Shpp_S$6) z?I=F7ZvR2#ULj(rNrzhRw@)sa98PtC8DhS{ZG-qIjuS8!x&p|00*Bgs*Nl_(yv$l; zphCKbdeJ{HaTKG!GQu!X@5Ovi0y`EQi#cPjr&FjzWc2+f5Dd57z|eLbV>1iZv3w(G zbBdHrfNJb=u0cod4%CvT&L7Unr~Vq2xhm$GDdALIYVy{e)au%idowS_z|KWTa*`|( zc^EX=Np=Xv=L~`+{WP0ajhG5kXTx|2jotMnJ)`|R(!#?8b6*8&!IgdK9)xhkUb`O! zjS`V#UL>X3&701G>E$hlfR=Gf;=(in0z0oSK{XnxFzJrx@UBYo5;m>!747Walm&9^ zR<+$F=j`c}1&^zUWfRN-_5lXFR7mX_bA!aSl<_^a!B@qS+GL?U85S&N%b_Ri4Sj;{ za565wQIN1pq`V-3@cJ^~T}ME~9d^&9vNk7kqBlILlH@oHA3%JjUNdF&3zgV3UF;ca z5>o5eifDgnjPQ9xaf!{x$XVmom&f*7k<%;D>Wbb!OeT*`ecl4i==i@Tqt z^O^czDc?$`Uq`TWw{79UBRC2K1@|=vW=NQY50i4ckxV06#K|Z&K28cbr;|1WIX{t< z5x%YPSz<9X9Kl!PB4AdM{R%y!#ZkF6Tjso9Gr01a*?p_me`g-u$EYXp>75D{;;pc; z25$@?rD3>@vH|$a`zO#VL09J)slgl%w3(V3keNTSa6;cZmjWR_DMzEI<|PZ=xjt*a zVj{1#Fyo~RUVVu(YJf*b-M~6(6z!e&Mz+H=1?EM@yDRxo=OJ;z%j0+)Huk=GO)_p# zv+0~-gCKmfEkZrjgpH`;aCRFaOnZC%^9xGR`4TNvqnnDZ6})>RLl0$}u5opWxlkt% z{Rb0$Uo&GxvujPW0R_wh{i2b31vTpJYPS4<`HrQFkce*BIg$epnx%037JMgvfkan< z*;}82kNK1a2WjT6uR=(gZEHbRGFTi-q`JCtsviA~>I9u#62h@2TeB zi=xl%D>S0$&`U8Z(~HU{`&1oC=hf2*eJo_1yDG?xe@1PR)!l(I4Y-rCO`_sTxZ*{< z>H}Z586KC}H*W(yKu3o?KXr(}f2QNzWgS~vTRXem=7;A=kMDO!UiP;xYSt=I7*_8! z$Hn$gwKw$Dm~VS#YdaMqZO|uHJOPK_9Fs&6{$G>O7B8bA;-rbQFn*#WPENX{Nnu8H z6$>2<)GY7JEGGIzOxk|Hxmx({Jw>-lvA94W{n)yo$Q!5*ISod=u|=|7c-ANgIJa$m zXGyx*Y{a$e0E2Qk?a475r~WQk_d3;!q)I3Q`o&5u7|F)*&Fv;~M5ejvBlj-c&7qVR z!fp)p(+F;dQ&v5(RLcJKq|=DiM-;8{pbl{&x!GcS>k*s1MjXNRa$~3yTe**GzLFU@ zTINWGie6@qZINtPSt+^Czfy=Y_v42^bC!t@41h?89j2*`gCD`NtsOuf#=bzy7ZS;{ zNpej2Ck|U$*73%Hk2y&~XwZvoHph{Zw!hu$8gx|Z!niV4YZM~fL3ex+S)AHu0V5{! znr%t0-=J?^VG+K^qEZ~2Q~@Pn9>ytnjPQH9AeBbt_twT?s9kU6*G`$!HIl8 zu`n+QhHD_5PLO>$3kTbhwMBg5*;fR*jz)8?-J1d`^Aw9bMU;CmsSTpif+K%J2;I3d z$|-{YwGe(P8pWRRT|R5|0{+2BAL%@8e`>b$T{*Y$YIlAZ*=h&x_)wYqU`}75yvVE= zVHaxRRM9!^{>R{883mLKXovdzOY8^raagL_Ofl}Ul%xQlZGWRF*n(+~!M}1$6$~JO zP!Lh4;MK<9bN<~EQFu<;eKpBa^W5`L-(cuk98;%M*)EGLw8WFoHqPvYGH(kH;H1%M zF^_DDSjL{+`hLjMMx-O2&nlb1$E81bHF-?bjU z+DgrK(mz_?PB=(sWwwBmu>5lMW1b2q3_o>Ep>*kQP4r!xhjB%Ubo(EEv?@eGbqvfA zCJcJzPnpfnE5W9PB5d+>6^)c_{L-(g2jayXcj8wimUpJDGHoJ;xnp|a+J&(ld)HS~ zjkX1#JNI~(H@Q|5XsjYvU=&hs&r3b|Fk)sq@FB)rw}0BX^Q?ZlLWQ;eJ9bEn*7e-L z{0XT^j9Su|__L;#WCw3!E3f9y#7=RJ_;agzhKl4RvG-L z9NgnF3kjZ;^PiEMy(nZ!gfiiM0oG0aIts+%5n@~@D6;2@GLqU=u2gyb@`f*4C8h43 zNWV$9u}t}{$>R@mNQpfT%P*?MzJ!&OXrA%}^{Z|dLNi!q_L3znkGc2ItwD>QmlUJy zs2i>#Uk+8`W!nx}L!Wlvfhvh~kOhp(q`DE4ulpe~yju~A9DOrx)!9Y$?d>DmM!TXF z*dc+Gr84IM`=5@3+Y~kIr=G?*grV>HQRdoyWTH_c)S#no(F71%V=$A(T0O;jN1K%{ zCqnVj0uh1rY{v_6;j_Vz_PJ-5sWd^c)dx)#h9v-JBDs))@j3znea2{y-f~uA=r_{E zoYPsC-EvHGhcRSS3vwt~KeR7{LK*xV&}l;W(p8yb%8Mg=thNP5{pX9tTv7;x7X01pgft5TG;w6hV3o>VnJnL9I!TuJ>9L?RkfH@Q34;o! zQl|gPu5P$-7YO-3Ak%^klH*Xf0k?UmuWcxcRWl}W!l0g8u|)ct7Q*pWuWHsVzLe2V z=?Z{r=SmrtH7#!EAw)Cq4Z7Q#=b0jcy-U|RO~66!I+5hMG*c~d^T-zMktcAUlpiBc z@kug0SBakjWCVlv{8=sup3&~GR*Z?4ASXvjGzL#Z9;cM-D)O{j+5jaQ0dXc*xslLx z>R9vuAnF@8{M7CZ18d$c8v!UiMESi_U@R6y0C=CYE7Enw1Cc5o5<%voBRNrH;ty%~ z{3uIyY4juXtcVlHd4-b*sMRAvOj3@@GnD<<;vNhOJ{S;u#q(N zvX?8Bg4sv4-{J#eCD+)DS<9Od-7Hz{c!SFqk|?8GGqC(!3?Nyj?5;eU! zWQC`rDz4Pj`--BjRUK0=ZO}b3hDJABtOi!9>j+Tm2)rXq)60*2b~JW4^zjH|9=3{R zYmJ@`vMX#cz#%$2Sa~SFd#YiwlBci*=yjDYnZJMuIt{;tnx%Poh23$MVh7!KzQGUJdda4f}y6$7wJjmd-S&pmfLp6Ob zncB!NB^{r zZ0oU-=)TWqzFX%d2Q?R+nL-FeX8u>ybjzzfL2YNT$*px6l!mJp@}OO}`4JO5iBeM~ zY)DPNW+C5l{14t*J7QUT;_wQRPZh+87znQ`9eTN(Q-K>T6UsBpXFv9oQ%h2qqP8_v zPQ>ze?uN&Z$5AMFzir{Ke z=4-HwQLlYq?)(aW;GMWLtxq->EQ~&Qtj--bf=+S~)38jC_Gza#P@KKeN>cslB%2}| zN1x{Y>{0@REHnMvF{tvH4$?-bOJs{Z*};L;26oqjxp|afpO0P7Q8?G-q~x+@Pbh4> zL!|3J3OyqepXF%S7s193tQnYxp?ybd*rm<66k?%e>>=G;Rgx-2I}j>e6!L@{PS&er zq&QoOUv!pe2e#Y0f*AaJ8QTmm^@Ok}7%_6+mG#W(Gas8SzFJ{9#`72Pih6PaOPB|A z*y!)kIT0FLu$*KULEDKS>OfsO3j}#&wq+SWx+mK!f z<#B7B-1PN^7^^aEj%0Ll(aVX4+lfElbsI=Gre$Vc?bj&Z$UkP8VK~#ge5=t>yPuj{ zC<>a~o}N3ulU>TEL{tcx>PrL>)0wuwk+uN87=o&i&e(zuQ|gdz{w{vh?$&(;KE@*j zUG)y4DyGA9z??}C4Bhu3Aj=`{_l2Io8)@fC{TbM_+rd}AZ(}1m*lv3HHi`9gh8c#Y z-_bu`>6A=K<}yf`M#aQF1|%AHwn+qQKM^L90AtX?G~0o^IsmVLlGe$i(1g$VxI$-& z^5uztBs``tMf!_6iD&$Q`jDc}6-VtBQ`tjlL5QsGv~!jbvpNY`nwtOz!eX%$<^U^Y zuP~u#LWyu^^!@z09g0o#y3QsYMT)8ZlJ$Uy;z1q3MzQ|WFGuRlg4x3LLYWan9o65o zmawrn^}QdeHhjPCcKq0l!V08Bn;Co#kHOoDfzkUMgz@<=UnkWnv5bmwA`m*S57{sb zB1?6B%#OpltibVn05%Po)tUf4jgvdOa@Tcz)#TkQy&eeK{tL-6zI}{skW_`i3u62U zBg4dfV#nS;FG-N{EYx}(CIVsJqT{k1POZsQR>wSVg^s5jzzSmBw^gC~o3hVJppd4x zAa7TG>+!*+4Svo0gdY2K>&MzFuQ!dbTyA%+v(&X(8z?VI&NnqpO-(_O;UGrC_-|8D zV6r-8W?>l_8F3T$d;R#&6;9c1fK|HblRm~?hPBrE?x`e8J_)w7>hX?X-x?pl_`!Dmhi=AiTQkw%mKiCe+XI#qy|hLp_L@+ zK1Hq2gBtku5o2fCeCR{E%V-qz*@OIkN7KE|^wU@k9TR&nb!-a++p!PfHBEyBIpGRK8U6z`TwQ<&7j3 z>GHG2CqF)4{~ACGJjuxVcCYG4_uN&a>R7SfvbL2=L2+LYHfkT$I1CVqcjOjW&qnM4fD{)Dl^`OUzQM z=VDhm9#lH6RyvZGIc|M$d`YEMncY^QHEfyXaiW&YMn23MS+c2^nRldE-#0LnhEa@2 zQ(TYCF|QJU74IQzwuNYZfH8zmlne5jF2T*Mwk@rRI#UT7m3C&YgPhTdodm`SyH?() zd+=12XcV<-=&1Hgkhi~0j+^FvLQ=aLgv*08Pxisqon~q4S%G&!tE8%GNGRb^I7X7{ zZ~E*-0BOc@dO^I8YW@_L6QRlXg~@vr-zR5Vqf$=yfNTFolx+XW>wPq9&>R#%V#QU9 z)YWyyE+l+UJ0Er@E7i#PDgtB_RNEm;mp8k_Pb*D}5XKW{#C^X%2|1`qv!#MUn1M1| zOvCf|)R8X_P;tS~2h0kOcRB`y&k`MmhvqGaqJd!?3`^ zTjzuWZlCi4?cv^~r$RpVLsrS;F1;dj3rP`tmmgAKEnUxwf+PliJt()8sJ0>^8{_lj!g0 zwO}Bz-F&K5SyqSnr|NWJ3{+{(9?26Fo!1xf#JUjP6`{QJ@mTb9v#~YfJP`P$Dgn^+ zpRU7Tg6IYHvLPSx-K6}A!c_xTUa8fk(_n6>oV*_E^u?}(!fDp>UZBVL=exF zrTR|PQB#M;inyx<)9E!gy|m+CVEin_Iw&uVXP~U{eSKpBg)j7S5$eQ8gsL+#y-!O` z)>0i4T|_s;+FyqqGxRc`VC#l!%3!s|k+3X)PyddYUqF5F>vPD==vczLB?(o2lp#Gd zF>akgmoQX{ByDzyrP73}I-6FpI}s>Fa4po&uLF?L)X`@J{NMI?fv_;&|vvQr_0P8B0CWngk zss1R$0@+CNTAx0*c?+t)GT{5l5se)c8MiTKrn7HIYp^-(VZes6Ev%t4P!EYx}L5Vgv6=}x5>{TkN zZ8OSjag)ERU*lK=C6g-qeCND>La-ExlEfE_!xt+QXHr#3?4RP`&A&u0-O2IIymW_6 z2!9Ohw6OwDD>A-k$?3!s&oLQHVO}`A_A!bdDXTy^;x+4PI>sZ?byHRaIQG|y2E8;< zHA(T5n2t^s5R;BDUi?j|1s-_)a%2gh%_AAQGU$V+&oV_*GlS!Kp z#!a+*X<6aUdS~{T3`*LIFTUt{b(j8qqRt|(Iz-fH5lr}mAE3#JIw)_rlw5+w@CDq32 z^88u4JQ=hgc{@8uk&KG0mz~9ZO9|Jq&!{_kB+RHF0YTTHuE2)Io2R0VN*k6dE{BmI z=#qxShCoymOFm}9Rc0GSjf}6!7w#~8;7{au2KuG$(oId3s!dHzZBM-W?WYR>;tr;d z3tfj2j`6d@d_Y7UNC{yb5vh^W+Vh zSLYUusILy7c?GW zz9Mayd#T3TLGI)#`0c(H(joZGJFtkxG7*w#^Go?CWnFjoi$du){cN5({?dY1%G6{^ zg`|x__^OLI&$j2I>$WyPN$I!hxm{+&ygogD-?HZo3SBfP>=T+;&hG4Ox1fuo9o{2Og4FP;<`&37NRk5^sWRq z(&+lafF8a8CO`R|n?17mGp!xc}-=^2ei3|0Y$Wo2FLRbH}Pd2|`6UX}! zW3+VVdGIYK2NxhQUeEMjT`7Yr*J?>Od>7v&Y*~y2`RFy(Hl+8Pq0{NxLjVo?s{oFx zJCwRD?c{AKJaif1G?}?Y1>R5D2`sR)05a+ao-N>9t>z6bV?)YkW~S;UNu(vVZaW5l zO|g_ZV%#l8T?$GGW9K@5&2sPZ*^iVRmB{$*?ezR-tlLqdeBnA%E@gaQe`>#+aU?-X zf4{TJ8ot|)U-MED)c4pYiJZEZ%LOgTbuJNAKRj_1FXb7`&C3_BS0+5e9&QqU%3w=k zOYG=7+)n<4drLV8+@WN(^e&^`%*tfo5ov4|)*Ur2g@08$H~qF zR5(p4O8rfg5?sZm&yis0^(F=o_ls^yyTlr2O=+vrb30B3tW7Ae`laPMHUL;{`;=>0t@tx9$f5SDB zHYB*6oj!~oJd|SnJvlYzPw`i?wsJ{7wG|RUU5d9*!VR z5`owA939vD>Gb|bR}D2CxBKZiSI5o9F5%|)_v=m)4~XnHiv-5EX-+~9@B|t58_(U^ zNR#t!M~hjqi_6tkw2;&iRZ?woSH&nmYr|F)UG3nUA#8F?fEVKFtB{TMKOAo)PbIdBDi99 ze7ifF-dQ(tm#anK+ECMIz8*9ZnPz^)C@Uh18IwygBy-#+aUX}@&~vkfg46VAF;43l zHnw#hfvL5lwOKQQabnqy^~3f9xUoP=oEu^o%4f+Fopd*aWF!N|fxj$RVsAZ8=9cG% z9OLB+=@ zLQ5BeV;}M^5m{MSe#HtbIg5jSasP9)G*A&TF)_`B{5hjf6=*>b%k2sd3HqNyp607F z^6)gUPXX<{dbu_;Bv64peMCqTw3z&7p{Qu3f=5kND%QNStjHt zCya%25R|WQ()c%cK`8zk^i~+C!p_|#kV5G^78Vt z=5+nPzqhOnNSywgsd}KuSk=JY!T0YyoEckMT7rAa;4+tMKOoip^RRT-9@(8I+Y>)l zHJ7vgeWuqO_Z-(S?oU{N%m8ev(l%W`x%YlI{-;4Rv(DXj(DN05;>HMe0<)0`!4oO2 zu*4%En5QYZ)SCj$9bf&=kE%}OiLo#<`y7R5F#kQYg>tdL z0zAG=Ap7@>*UCFr7_Q*Yr&M|D^!|A;I{ugs<_nLL{3?9vKd)>J{MqAu@!EcizK~vg zP4~}7Ug7u)+#9&kt-$bG|GM;E^}pkznaa0}>!Cg^B}K4Qd->_VXI`8Je8iVYL$?3? z{5hPJD1`vLQ%Y*;>_CNt&cDZzX))ryVZ02?J-I4Q{Cf!TN&d-+2j|Gns45eU%IkrRv)qv_iaYAo- ztH`tcKL78?)G6(gSiOCI-%sLXVQDG$;DI@_LK`?a@~HA4KZz&s?4w|lni+hKR^XZ5 zM9uf)rqiN4_h;>=4EtX)n}U#Ui1oYgUli8X9$M-@Z^AGBGo?HZG|h55)xU2D%o|{E xPXC>4{`bHB_i6m^*!X{b4r!->cg7EaE{Za~Y-fzaK#dEfDEmyNTFT_j{{io(nVt(_I+QoB+1AggGnX(z7J!v zuQQW%j4|eS)Tifpf4#_)?{ltm&ULQk^}5c4K762Z>IBOP8XB5YYO0DlG&F~o zXlRbuA3Fr@ASd#A!7o~O9hLhuh;CN!3C$53c`bPwnzCqyJ+q_W|Krb94cuvHPJcW2 zp>1(_Z%IQ#El^XGf8qsQ#n3-FZ@>{^cK_^^q#+Yu6EfPD(9p5Vzg!|{8u;xk-*Bu3C54b31e zn(g4XoOBL`_VgyNb!`Qws~AYd6BAblrn6 zW5Q`hcBveVf`1|-!Q)qn%ymyrxJBn_6zpu{u9)y1HQ{-ULdlx&G`B=^Ea}|JlJQpu z67P}Rfvj-CWM6^K5jr{`aehHTUqKuf56^(7&Z}3idJBR=VDvOE996fLTL)`gOu^&+ zcjy`^oC-EgXTpo;qG$UcCi65$w`w z0*U5BA9TUvHr*L-H41EpK3oKn**VlbITzOq2CS8K7u>!VxGx2U+Fzgn-0unURZC1(T#x zVkiox$Ed~`h^(0^3|7OuDd;GWK_6iZI(u)5k^wS!CxDmkvtRA=)TwmAW(i!Bs)W$c zSO&uuaxi`PBk0WDbr=sko?>3R-|+U3Eu4m?l@8Y}kIcgyuY|mv!#&tX8XliLK z4etg2o-mn!J+CL+^z`&##(H}ng5`ifAmDF$Ms_eAvxbxTVnAFsY1H;?TRd3vq@*M; z1*Oo2Trh>Al9atY@Nt%uH<(CQ99k5-Nx-58{w|GiqhBhR{jmxgFccXX8E{1)5a3!? zRtBy%Jy``BHzWe~?tv8tstB55Wn%-AbpQVSgI@i2TOzNB|L%V&8xjAty*&&OKRwrx z4Bo}#@!$%DLcz5sTeeW6;Ir2nrwOl;y*;l9@9|r{bq`J$85z~xr9YS>8k!eAa>&C6 zf4u+yzi~|e>C>k-`6MONoDb3TnjcKw836?a1-yPjXy|dw5t^LlUOWxWRpT@a9w>fa zUmuZxC2ntTA9``SHIwt|)vMgx-2D8JlV&eo!|2Z>jgF2+M4a@Ym0Pj1MDp%H@G7C!6>%%gPUs_sv@ZiD5 zeAi5_Wn0`0upaduOK~M7C2Ta7^7sqPf>5dFKfg>jhu4xjzP@HKF)?Y&{AV(=awuKg z1r~-u1!g;U$)2z$ZVL9vBTmD%;q~mzJOfTNUzu50)L9B#XMWtACW5Ao`MQT_+Z1y$Y@K8Fs-2FVFFfx6A($&Z5 zaIEuKZ1Nz9<7qD*{cQA;nPcw5>gwvv!!(APhtU)=k-(>q;Hq+^?0T|XrTN7mVA`mO zf9NoR(0cGAw2!2n1=9}rRH|~T;eOy2LHDt&5wcJk)7SKni&yC9_u|{H)0t|Eua6n} zwJs4o7_{x*&y}Z&FlV0JV_z{BvydzH87dxjoN$_e6^>R?bu2Q84>)@3CbJWZhwVh4 z4WG>>j-g5Jqs^W(yD>fQjf;FROwuhMWteZz-F=0z|w>w@N2bcBN`j$b3qEVUcTFVTq6l>q)hq`1~0W6)-1o-n!G!cd2Lx zd!~QVEAy#ju2F^LQ1OuCXuz)sHZew-0mheU-oJj*GT%&1yK5H&CZ44iztEj|J=1k> z#})XhUk;6aB)4Bb<=tvH&uSB_V*=e=h=P;FModF|eP-dM#1G1JV`vH{bYDr4^Bcli zXzUPVD9}pbFk8ck^K6TBS%m7D_QkLW$;e&PQwp&!ACZ=IHJAr4F8vcGmR;v(s|yyV z3CpWn{P3$>eqJ8gsE3lugY|PW8Vfp7)taODI<3kb<2nK_aoC1HC)B(@Lt8L^=Ce?un3tSp`7oY26kZc8x2|= zGcGMAmUQ7qu2T3UDo`8PrvL*#zrA|zjdu?+ceimyJ~qLAaT>@%6g=Zm%fr&n7kSzP zI)}Y+S*t>*q{b-d(9(km+7-=gemZ6*A)buh4k<@ueB3bKxK)+Hh4|(+?bAeRrepuv z%frGFiyiC^+9x4*UfdS3t>leqXsqOcKrWnH(q(gM@4f7_Uc%KP zdukgZlpB;4C7-U_{j!PC0u`f_ooPLC>uNtg#-T4hXkRNz*3BuKl3dSFHtg2V$XVVUT;_9MA{U&^{~UhuuFZP7`qubpiC)+4<5FSQ z%=6=TNzULp_l$~+u{f6c1~@J)@*T9xr&^<&T%Wcp`G)eKx3pUpD^002sLAETU?a6{ zAhi1xlvnCLoWa&B_q}BgovD75r`%`QGu3`^EmW5#$GI_(D&s!i>Af+3gNv(of`;ZO z3B)HK4JzilGsB6F?WV$L>8+OrJ^emMuXLh1wj85iRG;C`|ERX)V75C4nu_v`9`i0O zUAha^65M!>bM7^>Npja}G-Wd})Gw4H9yaLCp!ziBzq(mb7%$B{JUnvtvK~j}RGjJU z^o`mnwDd^%1M>R(OH6Lktgq0jdxT6=60mcn(c8Cf_w!4v-sdDwX_}Ts@D}j#O8vY5 zB|Y(1R8&-zTI{y#&7qwPIV7j``qeAVEUB@HiHvGGnwx`tc`9ut5qm5JaEVq%opDNd z4_Z&zr+M6IeWCkdtj7dp`g#8_T}h49bQKvsUZ$B*GkG6KzJO$AF;AVdB@rDg0!D^} zSUn^)5(Z!d-wfZ?7^YA8c&2G%@9v=C>Wk*ZnYVDHmb7EvAwXvTJ90w-YFK&8l@@_#t z*=@|@7J}pBGE(mjmsYff6T{?JN%-8ahr5=n*b$iUeSZOs!j3+C1feH0x-{!f&QSJ^ zRGad1oDFeZ*3if#zmz8v`01+)`7L9ej^Cx|&MYzA)-5p`hbr(0jCrFy6z4u_fSv8( z%R>6pnzQJF($Hxf-+o?+^zqu;DT@fkXY6jR%=AKDNJ{$c+&LazsGZ#p(vsIcz{J*u zXG(g8P0m#+SX$0LD_a>U9Vj8G7$QSlY_qni^xU5DicA0aSo~Ji^6uTraTU-xPSSK;OS zp@M4P4}|jFS)&g^Ef)nHKg{n_h^BRn6^50z^84h4>_ENu4_}Udtp%|d=;XRCM9dv948H zo|_I7-<`CQl1%L<%Yj8r4rwfli%D$$5)LMXN_hN}hxz)halt!|SF*D0iI$}$3X*X4 z&u!O@q})CCXKodYso4`N3&+PVFD@o#Kg-@4E!%sSYAJo{ zez&z*&x1_rBTV@i!k))&~Hy2k7mC*ZEK}V;*DVUy5?_;Zf!k4#j znyLzmmec2$2$KzYZ-HTIZlGIf^d~QxJb=8{Kx)KnKeOmqEhX%m8h-DNdaabpSu(O4 z+GC;}ja+vT^^cXu9c4ck{rbG@5PtKl{Lt}gLcj%GpVc;b)Y9_4(7a)iHd$T=S=b^> zeVSur{wyFOTx(c%Y$i-JOT6u>s8Qk?P4Qf zi>=BJPrg;-kGLWpg(OJayZ5~GD2cRT<5mU|qKQK~zd}y4O%G?t24;;Pqgg(LJDu~y z)z#G&UMEW3GepL3x|$4ric!s{`i4L1fUk~M!0c=6XLEO~ZrmW(x?A^~8Zh#e7UnrT zDJCWiL5LiqNUOG#(}`_~7ws!MADh={$k=e>LhX{xIoUAhwy4n0PJ~IP4xVk&ig@sJ z)r#KWNMEU#C8p1)vt#}_LGVVuph2-obXiEsNRgVaGfDdf@Y#K74TU9R{?Dk0q4UTm zR;Upn=79R)8^=Z}YRK?gl!7*Nl1`dW3Ph&YH55MxI{B} z9L*+I)QeB+#pkjC)^`py4{1>y|pYr4TAEkXd!le>*pm9K4gQRa{AAG(i|< zBA;VKrLtr$s0A`Rx>$KnZ3i=M-t?IkAr>!mL)}gUW^es;SWmyR;4o~{ZT!hSa8qDB z@6tmBV`PofcoLM3#`6t~{vmHAlCBE!!iO{Pd@RyDUCe3Gil(Fn_*cyBgRD=@A0P<{ zCvZ_P`c|7+7fv^QMdV>i;k$Qd1C&xzQhK$P4)qATUN%_DX_>ZKV%x=4ImcapU4~gp8dXFQ9li z82REGH*Ul#+l`gqXgHa(1t#fgB&-_Q`}-3fo;(a~`F-T!7k6#p*W{4FdNb$-rEsQA z;TOXS|GKN8rgkl|zuw#3=d#?&TMVrnOC|D54(8&_&!*t`Svh@;13LuLYzz`UP@wbq z^JkFZ-jkFBaKh|vq@t1%$h|w$L;(h3Ia%)wvfQ35X#f}8^Vz&uboX z2O;8feTfITFWAVxlkgNbit?dDhd`Pf5fQPdqo$t7t2ClEg~Mqjs|z`f-T^uJsVib& ztQq%Qg9NM~>5g{8A1^m_E{~R7uhB_cT(kwq240HwvOUO3lGWwe1HiML84~e8?CuN+ z(36;$7`TEV{0~s0=ID6aghvSp!+~x)ySjkb!w3@%(4pI`DgY7If4vZRefwY;PoYhY z_2R=`!g}(dxU8fks3_B@dGU4uO|K; zy&Tf!{sDyH=|Y4y__#W>C-G~Dzmx>&{LPDh8jnq_2T7Y|Y!o7i^ZNB*0AGoU0@Noc z0W9261354zL=gCQ;y|*x2a;g5Ha9)M)s%WIrZ`7qtoVAM{Q;_zY=GKPXx-|G1qfB^ zR!GcBwnbLJv2%T%{gi3OYa_BP;Dy=Ug_|=WFJStQ!2%WBM>5c&O@xHUF|DotI3MzS z=HjBgKgc5PtSNZ-Q!n2B#GefVl$MVUZK6e4YvGvHK7~6iWeXPtFu_A!oFg$NCMMS$ z3_izbgAud^-ueI_pjmAk2nsiaX&7*bcLzqJAQQ)a;lg){^3}iijsD3>WEv;#K`#Rh zz$^Z=R|!FzxEmv3k$c@&xAB-i6BVrZ`(x^bi>pghQ&X2&w?dt1A6^B|vR%!7i;?F9 z?ey=$wEbLt%!{qN(yG!tBza?<4?kr6uSX76K^~Y@Tlfn1&??3kA?#qd#QlMJT^UwBL>}rsKwIsGFG9PABtFL1TU*VcZ5b zC?oaKnG-;Z7LERGgF@dTLQb*?nRTZf*7x)3N!L17!(+Pf#elf@y+>#=>56qUC3~&& z@$6xaL#6DaHH0rmvqwUYKY|V;leMl`2wV34q(-Ke*;BWAYbI=LHf}9vZB6P{{5ZuS z>Gi93Y8Ksz*r9x0yWKXzB+yt*DKvrl?f>fRa$DI8y-sDn``vP4LaR_ZFQqj0j3;5D zyE$Hk;_~b2%+z^iXUGCYxTQyKNz^iHF)|O#r7hs<KFs#dc9d$fH9-BQTkhmR5N{d zZJKjml}k3PD-C|>UO-ct7UTiu2FFkPI)n6Tr?i??iGZ1F ztmlJYm6~EF$E72@AIdX1OX3zUva++U+69_-y+?FbRZ(ng1V)$=y`4%7YwRcAVK=3j z@jGu`d5x`Rjb?h^>&><5VCsVGxwK8K5bu8tk}Ace_eGOxtM|oQ6fW@tuf@*9w2d5% zy{@wO8$T@pyWpohvbaL3lFUu%F)j;xt{qF)kjYwp&eQ z_r1DMRkc{rx!=ObcWv{_b8c#S)gtoiwfPqD_2p&b`Hsp=@2QKJ+|0@f%xgN_X+vO< z{%E8v@zryrg0B|QV8TDv-|T*C05{YqQg$mL;O9^kDKkCZi^F$MrJhGdG`sb?JJIU& z1TB8FLvvh=aF=Xe?PgQoP*w>c?TJ0^QLad~(-J4t4$An_52QIu?8(_ConH)%My#Z^ z94+BaQ7Jdwi^mFBxLq;YG$N$UMDK2GDVelYRk3>B%V?;og^tUT4^=?N$tw&H;dFYZ zjc%W`eI5d?Qz0|yI-{$pp3z&(Ynp${9)Pfa_dUAw`ZtQSvk6)jDeFEVp@9tacS5~t zn;M>ZAoUKDw$mA;Mr3Nx!_Ef`b2yi-$|qS;K5B;c+3>`xJNE_VL3~q(J;=5C-rBNr z_I2LG#F}z84W4M8J>p*1tBsVrRc{E`l9xL2eM?2*-VCANlWhv`Xib_m^7{_BA*%&6 z*vkE58Ef=}qwgAOOEYWKe3BvIS-(!anzFt&#-VR}`A7k?i(xRVwZ7WkUX*x0j} zBg|7yth6eLpFlZn~417<>|Nc$}3Zdotf))$FQqa52MBOOxW_0SQr&EI{^xyaKZ z{Pd3NISbK&0UjPFNnB&-&W3h@?h*{W(^eb?zj-xqWobCBTblaw^GAjO{H)3s}8%X$2IRW-d!FRk;o~4=hxpwP2f2&;zHu@=1?u)7k*jBUh;nQ@isQ^?kvf8^iVi1lGpI)$<7gxG3E>rC+w>UDHkXINuixnzH7Q45k%*uvgU@fM(^z}gf#wtU-ew$iEgIKnIYO2{Z{n`H`gPl9FShNyZkIOpq zDOp0~op+%ckAeucWOIgrXi-6q>r(6}<0(hs$s@0F{JAw7s_hi^$F2S59l9TK_!0M_ zE1z$iz7Cs+Z0{V`j;*PVTvasuj9FBeUZiZnsA-q%q~pCY?M%||GJpAIk}?ClJgnMi z=Es+X$%lycaklWI3V#BuKYDQPEoSHfIUb?k>=z04|s@m_jEzH zwe}snx=fWWN+SFR+|t~)H^FLLFvRNk(5p{}ud6l+OKR4%lSY4HC>HT`RaZHj#_W1O zk7?+Ae0ta21H0YnS-B{=G#gEPs3V7r*NPV8|H*D$xx7pKUaJ9kI4Soh*)ZTmR;8)a z(JgHMXR3Y+CtZOlX!j#+et(aA7w;46chLdkzcYOuAt=*4`E)Y9WSWU0UTY&wJHPSG zf-1ZMw7`;&gi*qVn2rm-gv3T!ofqufau0!#LKHJ27sI}N? ziz;Dd*ro(gE8RX^YC3}uCVWEY5+s=oWT#$zeonli|FI_%`OJMBGF|2Nn7EPNGFjs& zV3=7|XY1QSY)fC|S$~mhXM?0adejwl_SoackAJ6a7Z3jqPOqK?vF%$-;?l->hIzF@ zey#1J_FHdlfO}LyI-CZ<<$dITLQwrT-xcQvx4$RXjGVL|`{-S`q&HpoGHCmNqZC!K zw6wg+ckf;b2&7;9jV}>LI50JPrTSFb2FW`MUusy(yUxp+@EfR&3h z0BK?2A9MUS2L=&7`}OO2hdu(2-~V6PFGxFoU1DKzL*;5}YA$kbz5JbB|4(%LC-M1D z_cS2br-@O9j8mG!nZGr&vaG2^cem=pkx0NPW(-C9l|3jD_z#sLIv~AXQ`MA z7kY2mTY6oo@;he%<@Wc=j*XRvv-JueG){;6ohec33KKF zvas--4AO@*{3}q^4Z|XqkpHyPyI4`1m`@XUl3P3e=Q67&dIH{dzlD108gc-$W?_yo zc0CE5`B(0)a22n#5J%9KEy9*C-vz?p)9E?OA}i%wz*wxz%<3U&I>c2M<(p zAQ|&+a5(CFc|}EDdZjWjpM0MV$V>4eM!>J?{Ein8m+spV;cu_isGPp;IalLh7PMjl z4KTZdx<_DLmA2Nr21)4{R5P76Uve&d3t0MQOkT(T%{K|SeX%<)e8rXruG6Ei#2H;? zy1W?D{0dvS_u;WF{DU2$hUQ5O>F%F>ae%4(UY}MhjJmUU%{_=reWRFN_OqFZE^qW& z^hplWZS88K;rAp4Gu#}8#?UhE0NYL5SXK#9;mfg;!gYpEla2*7k4mwo4js% z|166Ytqnu{vQSyXzn2Hbf4Mso!gU)I2iP4nG=@gUAwd)Z`P~rYW@k&w+58NAS85X< z?@49RjiHv6?!~E^H`MsJ6jivo)=MDCgzRFxDdO&38o9TCT~@}MAJj0wC2+vdygm+b zi>dTYD3B<9f9F!7)=sxm=YvJBV0=Q_Az&rIwc?l$YDsDVR_wMu ztLC=g=IZ(uE>;4pMzd-}6nEEbjmNdpf;*QkMsIu;I*>anO&zWk`FwS>?Da>`RiBpO zx2~?P#I2uAhwIcz{K(r*pgiR9J5BGs9U=bHG&@!xt>>CZzHJbfk)cU&Gz36h*x4Ds zhNd;IwY7DqSU(=wzfU0#+}(_hOB3@Al$Rh(kYfJ3 z9&S96u+^s@tfh-?X#?6d@c!-FwN{Q0K7(3KNEdPVgI)=ihed z%T@9?f9z!K;}}_PH~K03tyy_N@6Q@KS(CAHcfejgd-m*9*ZRgrUV6ewT-4dt9E@nh zd=+G2yvBu{ot@`F#?{EUv9%mwiyskF+r*T|d_wR2yqryO)R=S+mBIfX`Q#iyWFplBV zXp?WYYRKR)r-{$8C`|U{rk@l;g+(z_8D+>yA7LQ06*jyS|lB6+X0KArONmNu( z(XBt$_a&^w76-CCInFCruHXZ>VjCOxHx_!9bZBT5kPh!a+`(u8>YSpSb%ZM+%2z=o z@FD{koiuTmZcKFkW!V5JlT%lpfk{32KOZ|!*0*6og)iQG_+mdh1%Yr@R6GLswznYA zI+%h(m%$ikpMXj<`~$@&rBFXESS(r2;kWjWohN!_u_L7))VOEX&Nb@tP|4j zbqdr$-JLdo(XRkK(W?I5+~zrk7Vj;6kFJLtnq<0svRx&B>{1nq%kDY!|@7 zo^7?YL^_E4tI#Tj8L)5|P@MyI?~;`e6zSeSFi>h-yAIHH0PKTQ`LEmvGw?Upxp6kR zo&WbH5EA5Xe<$p>MxLvLyf2!aowY3k4D~+&rL}uKnFc)gr>ex&ON-LG+n$1faiX{_1@%o&`*` zItaKP9{XQaI|WwzhCG0>kNnlnNJnCsMazDhJ*G|+lP}txCGF7W=I)-CPIH0w-|c~Y z;Sm-u=H?bC0CB5J?y@=7ML9NN=o6WGh&M$W5g^=7=TQeUg$wwhf zNh=hTgvoPB*v@c~hU@+w<1JgfbQ!1k();W@{V2v;&(;euDAOw&NCDsM^Owbpr`KLP z&y#)jvjM}%MqzQs#4T~xg3Wc|H10Hj+Yza+(D*DfB==^Cae_mJjF-+j3XlqLvxae%##Xu|Om65ZvWKINn* zXhy-{jgS(FWbxelRz`3#w%=fbc-!EUYi?Wb4=aZFtiHZmdkei9QSIIEk&1>ek&$<% zm4=yg(%fthehT(}JX?bdX!}sYBId}Sr589<^ceA!cDn^}op;hMOLxA97Pl`_UJW5F zU399sp{SDq(Dx%?u@BgWuvjA`S?cLP?d<`vx`Om5q#H(rtRj71jPZkhjNg+@qwxZ> z00Y_l#kDGEbLB_p0u<3<=PcaW78$O-J+>A~d`pw9KkT5aZ!x)}{G@WA&uH_}c{IhQ z7dFBVEj80Utg}#eqi)qZMO}6W)Eggoq#Qs`~}k}j6d{Dv1+R- zfm|>f$Ahn%}N;4L?0A=H0N3f4$p z@&km0988EJ%bbGkSbMbQwMt4Z;&?PB|8A_Bd~d=)bP5|4nZ9RGFrV!y31hD}Z}9fm zw%Ew-c6uspH~nFV$CT_VD_dr6h7v2W*DZNepj-?)tnXaKl--HLp{$m})D$2>Bf0vu zxL#zf^tbjh3Xgso$o=hLT@a_EQ;@e#g9M@!)&C!pG6E)`iaMUcY{N zlh9uuahF`V`$4Y?+F(ry(Uf*=pK`AjoqRcN^D*VF74~e4%YIEp@?L4+Oj;GBv#0zj zd3G3Y+O2JV->~rKm&nbful;CB#Y|c7dHQW9okD0L@7A)|l{=RGp#wEsviob*Ri^hF z!*@@8&_(E01dQORj%YKAZ@Eo9|olprr?v$nf&z%dutIszrqAM%k7@t9j>zPLl6%qm0Ff7G^oc za|V4LniR$8&t9*t-bPKHJREZj?2Bu~82=vr(yrOJ0hG&4<=r8*%s#d=ZCfjyj3(U^ zJ3|2rn?B09 zb?4o6=ju5T4qFaU$WQO|J?E#6)jssC6*YKHd}ickP&!X#Dz%5k9k1ayxe__y?h74v ztG{CX)PFai@Z;L~5}x5PJ-iRg7RvZ0R(iZbTvkoPsXWRur5-)rNrdL?6!(DnjMUAieDNn7 zos$L4Z=T>J|NMC6IYL-t<<{(*UwI#a1lZckxMZ?8H>t5Rmt+kOY zWv{DJVFQ8~5A<~AgJI_2kI`A_;-8g{zbij6#OiM3w|i;?FZt#uL)d1A=;b;GVjDTR z2Z>7+|HzUapoHeAU(zt_^~H?V=Iz!{BA+Be6NeQRB3P;%^F-ax=;QgO>-1_A$JCE{ zDk|~97G-SFi{90p%nN9fmM?OqPjZZVT#FH@M&!)V%J<&4xTid}wmg-TMSV)KKAnW= zF{5jf@D3x9DAo6jUiTTEuYnG?P8)#y3Q%A@wbiP2~p8b-}k-R#V9sJ z#7twdz%KiEo#%5(e`j>zv*5t!eXmp&;`I&t^1#so7Hy-t@wvTbGSfTjZ!97P#gFp` zf;u`~cUeKnMfXe6R?39n!y;mv!{+PjC^SD#zB)AQF6$lY^sSW`zF}wj5yYZl!5xx1 zT~)Gyy1##0HUujz-R*NvpD_K{3~}%y-*w1 zV)U5O?NLIv|L!Ei%jjI0UcpmWX%0fS%eeh!x7&}SC3i7Nn9dGT*CegTO1$}ni+1&s zyyNahSR+lx_I%4Gv5F;$l}e0SaJ3O@%A98Os{xA@p(%UIa!j!Eg>1s4cUkbxkJ+iP z&xjkFd@=2vKKqPLa5hZ&^=j(9jJ>V?O6yJ{BF_jxB|&7N_j#NZZ01)kI0lZUj!(OJ zo^j97L-prqOfN)~&n$`L#hx`Vc;74l;eauW;9mt)O|E?Ve0@y#O*JxVpWstgQ>s$@ab2Rv*@#iXL69XozfiE_>~@y1oL{TvXEfo*a)y+szu?v#JbFQA zZ{-2W?Py@XK!g-mj{f18(gSx7N>ry>Wp&lQEnZ+RH(>2ukBBA(+52$4`EXJ?X68sJ z(#9w8*@?x{CHd7pb2NT)Gx0WG1EVJIH${t<*RaY$_iA4#;-6~d`^`_d2la`@$d!?d z#mp_%b3ObDsr_Kr^BOq>$cTQRt9?N6aY%Kaqr_(npXTaNR3ewO!A{&0!9!G%IK`Vj zk%jR#8sjEU>S`|H0#dI#!;2J;@`;K}`YyNlK3N0KtQqB*!l7NO7UEVPnADL(EtAax z0f;cOelKi}8Iu9K)g8++OREy^1Z)V3Z(6uE z-1d05r*)Szb83pCM0h4|PY~v*i=qv}FZMP-L5QI_fYdx3_5s|-2X+W)=a;Nn-_iw= zuLC=pva(g{q=sDDm+6im2E|$WVP|3}kgOAuvie$foVEXDJEA&AH2(+|_B# zStU)5Q#HO>U?F^`x(jzolfG3U7i${Fw6)gI((~f|_H>wJ?0#A7o@S@Yh&j5U`^0wB z51s*9tCU=aa*TjOVfjp;yQ#x^qG$2GdkW_Mc!O>7DFf_7wdH@tG4Dw`6%cqvPX?P2 z|FD1|q~0Muu2aSH#Bho;EQI{h(gmI!vXRm|O!5D8ieRJhW};fJ^Ey+Wb9!sD3v2@| zV)}70CsJT6X-#)kK zJ)jYz)_`i1dUs-jQ|M%Dz)+b!w8Pt{_7#juKygv4_)}^G&RldXpIG6mIQmEV>5J^~ z&&k#9F9WpCdmrBq-?$?-*O{q`f3hU63OZ>=lugxil+f4Od!B} z+&o)k)1GWOK2F{c5afgwthXCInpyif({9m8yMq}fqHfIjk?CmoLB*p!Fd1$BGiSEz zV`WuX<4MN4f;AGE5HY9zA|j_-EPI%kt*=~v>a}dxmQ~zS6bb(k*9{R*i(tiN74o&0 zVB9l8`1O(GSKansW-Bz0B0t*5>NnxKvkWG~B#KR`rJk#HD`O~?9=^k=k6|Y&P0qGZ z6@{~ycZFPZ9D0Mu*RQnhugGIWCqm`=-&8}c@ftd`uG8wKG8|ngk&55gQfT1RE-?{$ zZxZjORI%r$sPtj;Zbj0r%RST}9!nXnwuIu}G_dDYrw7eSqjc4gR3g@9q1H2%x%lkG zmCx62U;Lv?UELyao0~xtJi?5_tE%MJao^N>it*Ljx{{;-FCi})fSw8Il^70%4i##1 z)Ty;uRw1!fC0Z%p+IKy{5-u#i zfD7R6kPwj_!F$(n&Q&4_8#nk|thRO|%@)4x^}dR5o(LzSNMG9Spshj;qdf-PKE}q=eV&yWgXK-I zh)!a5YIrGOC_rC?+s6iSu)Z69W@=w<$ygEN4lEu(r%u-R<*BL($}CQ8-F!JRrk+M$ zcFR}bO$;9f_JdIcOM3u~T)J|_9fdn6X5_vRKZK{1G22MHqXOIF)s(rpCH)x7A!Mc< zE;wP4%2vD66(Fy|{Y!nXLVBTXzoE-Ax*`Zn{ zYHfAr=N%l>``KK3`0zy6V?QLy7?b5ar9_b|7PvE#Me*JAum5zJ^h5E6Po{gZPWAuQV;2tYTgnY{PR@6}+6E;TP4-bxl@_rP@wn_>I(&_H|?3?dUJ!ZWiM6FB;z4 zCM!+La#P3I$MxBtR?WZRC(tGmza_Gz1RlF%YnO&Q>K)LjT|?F<=yr{|QT>00s!SG- zQJF8=)pkd7%*27%m)bv58fnp!J3?t#h&kQ)*u4+$4R{JWDTNtT1;Ggd{^Hx*=L_>@ z#_jlxrr8f#f=BY-wfS4Ky?ApU$@k_98bck&uwAEKwaR<^v&U@P02=kHA*Gp13Q(U57TzTh($#H+@0Y+SAP5Y z#uhs0LHeN`a-`u0_Rqbe^bby3x?eUGHVOvK2!TaAAcN!-K zBij}`lZk6phD}iYH5gx(C0T9KovlO0>NDOl5SxMPb{rc>Q!`voCD%SB){M8+e|{Gb zU$n~b%O+KDf%^{6r6s;P3~It?GXG5Nsk$4!_`o=2^+^Y%$MfayRl|oGYe$eDK|sT9 zBOT$t<#hs=MGpd*o7^(4QAHgS4G8=c2dMK3@;;MZzNJai2ERhI*`%Yjmvr`GkHR?+}=4H+ZN)fh&pBLn@N883#Af>3}#E&kn+%|8uQt8wr&iptLuJ6S=$@7 zOZ+_O&ryp1vd6U*!0~yne&?vTe;#AB-4E7L!BOjNrFrdZHpqCwppT?)f5T!l!eOj* z-apV#di`)8rQw@%@olQ!jOS22hkFT~C-*MzdPb@syGHFo0@F(6^^ASK9p>m8s~Yv; zC&Z2Bx*F9ZPL_`Yi$7R&ZS|YJUe-8eYh|Rz0Gx3BUG<{ypsmR6N|%3%M70?^E z;k##EF`VMxqt8Y;=a~*?7t5i_;;0`~1Eox&t))wviRf78Rimb`6M7%8S$@nvO7?Ge zLB&3Z_oe!GzxWR0!6t8vvdr?R+G8yFDk$j#h_Z`|iNki2WBGxoKuPh^798Jh;tv98 zEoifz6bjS!FdB?0@z}M$V;+iNSiT}W&LC1xc0V&iMbA}U2L9s6 z7~ay)G}}nLbRhfgu*ec~@j$?ApNMSK(*C7v2j(BuMcvl9DeqOAhc~KCx(yL1G~RL+ z$$6t1a=C|L{#&~4RcssMn~Q0urC~BF3;Q0(jiqJg+qwwlLa4` zO$3#{@k*JpegWUw{WX$SjT+Y0+d5ozOvB%FYs>2EwgU7wb=&lGJGR2A z{IvI;9J^9X%Q?b%w?OF07s|g)@i<%Xqe*~H-+ij-$9bG6%HzL5bWHu|KKa@^v%Nz= zdr|Ai!*al^Trod5!~S2?(hE8Wt~D!f`o=%O_=#5ynrSZcV%W}$g2dAw&ituM;~$nYCbGFI@-KwbttFrH$%cE_ZWw~q$PvAKS>z9(3J zbG}{iga7w`&eGA*@n~n|r_;=WDq2v#WO3x6lqGUgDoR4o3OjcPdY8cx|A{bQ>B$o` zX?#nyN*jDH0ur*ay!plB5wD`+`p378PTeD^1cdMrK!jH2IJ@;+N>cFB-& zu(>L$>GQ~GrHvjb-gTx`BOMtlHfnT z(C!Mos0W8k{){Bj5$a{8ZzEy1t zfi^2DHh(n$q+`H61C>>Q4z(ZpFqH$@17-9#L9qOXUaf>Y--bTRde5@6ccnBX&>BQ2 zkdG4f3Nnn;LQ29lEg>2 zBd6=WmHv;gmB=?>X>GpU${u?sP;b27_UHkUquO5;O@V&7US8@3DY<)HCF<_si)d6J z6&%_GB?aJ&i|<_f#r{T9R6qeFiN%MsK6kPVoZ9REe;Klu74xq~>irRa+pVzZZY}N0 zIaSjV^hNi>lOpit6$9_IAb-oTLN%A)pygbI8<#Ga<^#%9}k3Dk|FAr5ecs`lZljBP+@g;JU&0eiVfD zX{oz$eUqP4W%yxS@_lUnkj1$W;~*w&_<%DLVJat9^nuPzd?xC8KKf4nz&qtf?6sZu*R9bAG#$z~it;5o?+TrX`|{;W z&{2jszp${d+;nhw>pz0AjN!$dC+@YES61c&ycaC-N(GaEv8wn>J)bq+m*3YWJIv~B z_fsF;M`BNVX*g$XWghc!FZg*Pi+r@CxHw#a@U_yP3|pta>$~mfbJ-G@#ou)TfIglq z))&OHn^Zd*)E$@W08865zEHOr!*j5+5_i+*9*T5x33zgNRlpV51p=2bL z?6SA4bF!0ANmjDSRz&tVIF+nyG7b*e^Bmi;&hdL3>Uv+F@9pHlr5_!ZIm2IR=ov|Ihni^jhzkMm)dCKIr!D z>&&=B(9vFx3ehEjYjejuCDes9Q+^X>F+NNej;9IRM0QVk5%8=P2iyO`3wDmgmeyDBk(zQv)-&S`K)E9l(pAC znH|hYTrzpC@2>M@(BtjL+g-&Pl!tbo(@We16Ff6@TOl=AYcSnyOo?&1@OcB{B@`kzy8gJfuZni4LOu-Y^zXtp=OOXulVk1pE@$kesRP@dY4tJLPIo_DMd8pX?_^$sZA zR!c9Yy4vtT)~C(@a3wAP&Aa~+qkmKx;DWm-A^pgN)2QL4nIfm|{X(EZbP_)h#Br!% z@|l0_K8CSPpUBa;u0~qyY+hNyyXPJKLX6$`@9qZo;Yv_g%dI*esc%rxxQmHeu9MBX ztT0%4iwrmdX9kx>zqB~LL?{*YFQ0HO`p$V(%%UNR!oj((Wy6aYFK4&M9GF{M5S*G` zch)(o98w!JFL%I4ec^@mfDEFV@Hla|N&dQ}R5iiz@rAYPSshD?_CH<6PIkA_Hw5*c z(=3=XJUt+He}o21HNZyQpPlO1^3vdgad!oT-}6gPN5pn;R%|Vpj{5A#*kGAIjZpsQu4iIcfEj>0}SIZ2ZZ;(TwQtqTmg52r)8MwR77hX>edrCt(c5H#Gvu(TS zb+Z0w1CP|CD%+XX%llOPch@wK9$6k1+hPHA(PGx=peUio5SFv_efT_vhCTM05>n}o z*G_C(p{Am>qyeaH!J@B;w0B|Obc6jByV;nc6lEq-V5&0lr-8(@C=zyd03^-fu(bIs~aEM@>uSm*QfD3we-rRzmN3Lpn=-+RpWGk%U#8J{0%_OWtxAq{oTe1C!X)@yRyMI-??ClAX%8r>o! z*i;%-FePT!z(H#eI_n)Yvhd;_x5$&`-V@KjNZ7i+SAYG8fIW_lfkB?v$7%P2oCB29Aur0r5%-?Q20kDnjJ zrmHCZcr|xotYtHX4yi!TiPFuAxs0V0q63UWZa)@Vy7}t`&9O-iax54Qu*(XhA~ywB z*Dq5%?RRZ8I5|p&C&d+BEv?o#a|-t-0BDdPt?~k(K3SmpHi=?ZQm=$Sj-ARLV>$Z|BVtp7O zY$R;P*It>W_^uw(v+?uQoj^|`j$o-^J3S0Ah1sR1mwZ4e7% ziItoCGxX{`ak49P_$Ymug~^-jE9S+V+!-Ms;h(PmKD|}e=!2(ayz-Q|Aygb< zMJOq`wzzF!VUhB?R0zbwZX`%~)HNSwX1>i!3rM)r+Y59%BmPVVRh7eh?a8Aq%L!u} zB%S8G7jcCNTG@@)LedYN7Pm}_JWRI)yyKbC-StfYG8E{STk=0M5vt9IiD15?<82S_ zT(Ce$bpEAau(zAZ0NCqo4X0Vbuu+W@6G|ww#t4+r?7!73izE-n zE}ZW)eDLUYH5DKRcYv;>ybgXcL{BEzm}O@*nTzgX_>3np`J5QZodF=zw7Kh~cIKqj z0rJo=+jXtUiu8GIAj{;ZPc;Q^1DGIfxetDnK9etCiyr&qdm^&c^v~aZEp!9C>0ir| z#~mpD{Ru~8Ns#~vUIKSV;L4(Ore;pF_WZTw>@gB)b1pPX%z3&M5Z<7z7|(F{u#Cu_ zi+g)njVA{x#Q)Q(H33-Aoy|6*y>Xjf**jB^Jyv3-a>S(JeQFHFuZJ5b|9Bprc>2NA zjzjAJ6%=rq=?a`>BP%ohHCZWnL`6#Z%i?me<8bcrWfj-?+i^)rg;c!a4o%gyEe%uc z0~kH7keus_6=P!C=Qk`4kPS8fV9OzbK+dnDLGCCVjU>58k8f#c_=N?d)mq>0khnN}c!#G&T^P((n69xW z3KvhBHXlqnOBAD><5!bY@tUwMdiw2obssKilD2tkzB67jMxppuU4zqY)v4UBsvcL7 zEOcvNC0ElXvxen~UFojlM$tiC`VB+&1u!~eM| zPW7@rOPBSbD%1jML2`^thJzJ>eu6}Z$<`nHSez2q7)m=G^CABR)%2njb-G>S9Htn#4dQYp1fC3dD%Y)9cB}jXvMfd^=`SfYHMA`lWvX%=6 zW^~!L@`7xAUz*iWsYp^YVeJjwSX6{__YsCqUC2V4l>VIAc+&XK9~zb z#d1)R2u=ji9b>{x<$H^$P113=4WB`T6HZyn8weDqKC7OBW^w<+7+$CAX({OuOb9~* zX8jbadse@H2GB2_P&V**Abk191GH!0C6Q5g5kTX^wg5Jx=<0(#f2QvlV6zpBf#Ntb z(l9ec;-40nJO6^0gmR5A)9+9(UNhj(VKo>fdfY?1=8o|gPQE;7sa)j6Tl-NMT9<0m zS7b~$Y>JZh+$t%IEG!h-LE(Ssgu77!cJtXxELu$YQ%1UuWzYI-P6YAWV{Mgvcl&)U z1H~9tHKLZs1{OJ#OPOpdXM2to4ILuIJ|*hmS3u|DB!~}PM+D^ zegifDE%^C$ylp+Y#Z)};0=J-<$Q&7``E*wD<3XjMGC9qO;K`yd zd-|rlR7)4UZ8zu8El3QL3)7S)tJ{6k3ZOpu!Bz;t?!)oWLpTDmw)yB;r=7Iz9mvJ) zcA*PjuOBco9q@WpDwa6bt3lMfdi6KSDHVj~hL7qsFJVli++OJAEcRq#T7uS}T2U>H ze>zL#GUDk_35AQr50tnPx?TW?BF|^p>g!9GRF$=^*#r6QRxsi-nAKEslMLSNM@q=L3R;v%sV$|fkNIx%jp)$ zxFa$G%m`f&w!2y!hqf_Nbkl#$s?F;oQ?1$@;X~~*Cf}lj)Q|TcNJx;vt z4?T8Wo|J{QEKh0d25cxbu+R(H8i;P z5hyHiYZrjj&X>GA5HfmL=`IRyURxLeF`>G1_n}Kb?*WzP$(b2Yd6sZpe#z5BWbJHzMcHnv~iR}9WzIQAxW_xw$yLHk?XC->>mvR9#QO(hLtxI3_BV;lAHzN z-Uv?E0y1Q~1&@s4>yreDCZg|GJ|Uq$VL6xwY+l_;;U%D}fa3BIa6HMnNb4Z*H0t`P zI3j{ckLwj#xQSC9#A88YXgvysu-tteX6$S?dieSu`C{Z)=#?@tY6Amz6h^(-&zZS z#Zg-eE9wUp8wXPiI)6?UbfdO%Ogr=spMzimaG(?w6sRh$khv>+zFFPaOOXNbR%7#D z8AmXZla$^N=p^5e6f#1vQTFWw?sg6wicUx)90Z}L^C;?y(|qKo$uZG*@_r_$>9S!! z499A`z;L38Qk-St@uJ6cK?YYl!)Gz^SH-Mhy!jKW!^iMCxaTb|YSzxDdZr`ew>mz^ z?gM#i-hY4#L%u#KB-#wg)@aNxMN941gN|l`>ESauYU8HK*a90mN948U+DCj*`B;0< z>Jb#2L$HX8fU@PNb@>dzpIgOX}5J=0)G7mzvMYD{Z!y-#>cb$ueb9V7N|{y z0qhfx15-g^@!~i0mQGRne5;ELF8N?3YbM+$s5s1Bf#6IXU0nS6bH3ZaD-kW&&W&mL3!lrL5cXuy z&CI1I^RT}ABfdz3C>V{pxEX~yQCBmKN;sp!(C(9z!8CtcrGT?N_Vo3ruhANVK7#x} z?C+khpG8dVO8GTR?E3CyKKKl|hW7uoNe?1nk~R&2TIYG?MxMK1$j$PANjn`Bc-_by zq!sILqL{euz0{|GEGD^SANB%~(g209xqJ6$#TD!6c6>jLAg3g9{`o$iZx@a63O!+{ zNRK)#*9eU~IQWwOohXxry1Ekr#{b+J0;MZSm&F&h3|}BfgES7QWp)ZXO?jg<>`Tch zr)VaY69*N}96K@6A|bKbqcG>7^dMOw5d7$lf1CXd2L$PUkbE|8OCl3|*`FDVY}NU_ zpd5HZ19@NTQhvu4Ts1JLjlD?gW+7+g|2IZT3@j|ME(es-fa?V+SnUk+`L@y}H+r#` zzQ5IEhA*qbKETt}x{#kv!9v7LiT4?B@5p=g&L?29wm)jgM&NqAy}g}|Y`yCnQbr91 zewpZ`qViGyWsQtY&qDygS*%x2_3txlYip1=X@$pRy!V=#ngWZ$=@O74Aw#{Ik%!H^ zpJ@s_P*`U7?)^d=0{v!(>A16G-f>GxdVoYXq-+Z?`OH_%TOmO>dAANcG^*rTbrX~( zn8Xir4VRyP-A_r`hR$>TpT<0SZ*o*9@1-N=w_EPlM(;`C)q$kSb&K z4?~WMY)8tP96fr}kDgt1d!t2FQxi$tSb>Ox)1|sSxVPUfy4}OzA=fC2YWTC%KD`SG zv4@+6JS_%$G0s52BHyIGkp1F6VG_jESK%=B^Mca55C5JVcPc9W{{cn5D7?8=acJD=`da+L+@F2&yG2!Hp8=;$&a zzrrEwC1^~TLvNFpQmPBk=v+3uT*#f`>xa1o6&r&xCyRd{r1pN|;J~Gs%|aXt)`O|r zyKmo=SgDYC>(`c+ApM&C7%!9akJbq2KCx4F}?ev-6T;fi&Y&3%lFD|dka+8))J;V{7Ig5|5^M-iq(LU6)-r zR+IJBTy0%#;8{Xga4?_y+LP==7bjFH|hTDYix#Me0pF#doPgh&3nF_Vd{_oc6OO3QA{s_ zyu>EYzW4FjM|{TecW_{7ZR=<2in{KBlyGLn@|$JBfwWO%&ynuHGxoCO;%rpO^JjwS zOP9=SeGO9F&RpyO(akiqgyt0|RmX{D74kmRGOozZ<{|QB-U{PbWOr3W8S$O41(jt8 zaHcbnQ`o(ZUhY0yd;8|fVVr}JIaQC|U4@wi115}hZeiwd*}3PTdZiAjH(njpy}ty) zl*?rM4C}QGQWdTA#`Y`I_cw<6t-X#^-(gzhVEKHZ$G$0 z2@237KK$y&)%yHR!k`zZ;leqp%xh+5W21t8tpIo$zMpLOi#F}h#0S6w-yP=MZW`YD zoa_qZ`;q%5eCqV-<0Dadf-&wLGXnTSu%}V-9 z1fgQ;$uw{i^J<63ZM;aFTwaE;Il7WA)6>;`ybkUc`~F$exr3=ARn`c~0l8U)bIhm3r6rqZHE3_U? zVYjLLo5A^`n%#k40?Ngisgm<@7bR~XzHjn6com<%llA8^e^7i`CL1-R8}x|epZ7KxO&K1jE1h{drg~*kP3c||A;e9T z@g--Msp_$Xpn`{XBt=i-B3j#kQNQc1=v8bSSMjuCCG^yw(#et-dc}-j4Ze>~J0qWg zaQE=I71A-t$}FCVZHbZKkK8X{)(-SZISWGG3)#S zDXmA3z9OW0^?Y%#au>Mne`?*i1*%Kf$5cJ-^k@oxV>`&2B9~h9j5Xg7*J43a!~uL- zP}xXFKXlN5Ssa-8Mi!(IG)D@!SRjegeFfPm>F2n*MxL3y6>6jr%s&-&-&Y4=*~sFq zh{`O&f=Q^f<0%{;G3}vTbZeLjVV#Tj0~rn;>gX^%{=+tY3~ zea1stwG?GJCmZgT) z6KT$wNNulyUhDDY^$yc$txXdxmf=$5x91-_7#sT$oK_F~N(^+wjYz*vqOQzUzY?jr zmVgx@dU*wd-^1pT*H-W8@wz-ZzsnTTb_SP=H;re5)^9DewxJEEOZU zZuMPG*R+c>;I-4k@;skWL{O!hYxlb|k}+MG8ikf|uH3HMlGw(dD(&kxzsZhUnXX#( z6(B(7Xu4q9GH<-KiD?UgMtVrDA5j^68tJ8=(I_e6T@0& z_56S&R11O{C++$Iccb%68a#;`;hr>w;k*EXF3TQF@GW5a=GWF60b8e1dt$#1!V4*G)Y z<)KZC@vA?#Zq#dfydqBKY-Rnx>3b~KHza2U+a*8jZhBnaZGFaybk^zs(!k9_aWfXj zHjGB5ys(qaxslIr>WLZn)!*%Dn>n~HQs7j3>0DoMU=)sRwj|o+!Ijcm=`Mw=rTLob zcc)VWTVCGQGen`k0FB=A@y3ea)G478WmpYK#_AU@Gsd{fh~i|NyV z&2*&^ldd(79yWh=FwI(pF@qoX&MscluOyB|Hy)E=P&W2ujPbpu9w12RhljI{mXpmDG!kOBa2efb3d4 zwN7}E8}K=HA1~cy#rFH?frAr`_niA$eWY%**$HgcW@2uz64T3hESkPCR?adGju_RM zu!gAP_GBWW28(hX(M~mnO)q%HqHYLh2;$zZ(JGZr8<4cx5Z#|8wtfjV}Y z)75I6>77F-dL#d6v0Lh6&ZNC9s+m83xV0S0j+$5s?wghH(NZty@N0e4{n;k3*e$pz z&(CjkIzTuo4EO5;pY-M>FMfLc1HxJs4W3rsUHSFJs+ENIw0KjklM}8_?lHJtXK2F@ zysw!Z$@p2rILyu}z6WV4awZ{DBcrn>2IKilT8PV*Aicd6w!0nvU|=&FAz(V*6k}$7 z^Qhb&2uj~T+ajV227ml`VSDsEWf97$jG?AOL+X2#u(Q!{eMUpP^~WEL9g@VTBq3{5 z(*?E5`*`VdWEOAd^!Ci`!Mv}aD@9eXI`X{e$MNWm7aukk3X>d3ZI`ZfWC^K2rl`?4 zpq6J^?2U5XQeAyji^ZrNAoTZkzqIBRb>c<-bo)Sjx+s50`;dr*%w1x!cX~^G5a z;d3~hj2Uevp0`HU2;MjWmx15T4S4Lam{G*LN~ENI^@^pDh`N$ z-d_04H90LOjCHep5v!7x67X^^Hq4vcFJoIke2VW#q2B%~L&V9QR;s?M}QR_%SIrkL*Ne zra`BlEdF%=G(W#RCeLIP$xOs0{sU&h8c1GMJik)#1-sJFWz}8i;YQP8U)R;pul0LR zI<;V@O{;aT-Rj2qV-tC=T}+d0eeF5xN`Ne}9vS%|C>{tTj@o5hD` zO1@lhwDEd#Ef)7dTs$!`v-kLEgS4A1*R9<OuC3c3VuaDtu>H0)ZzOpTq zc(#QlR2Q*o#)KWaJT*0WKk6rCW^J0jv4N0zWwZKX;$X=$H zqgvN5IcBju)8tgYaj^NBc!Z|fiQa74K!Ix7P@T~GZxIs)D>>U)dBvMEGMQRl0t*8s zfr4&N#MbZyWusLJdcp$f#Q8qS`C-0sYY(6C0HT8g+qs~wM(X#*Y@Z8t8BZL2^lgj9 zX}L6k-Y#{nOKP+^^%;WxkZujmn96ru{Dh!yl%8Dr!vt&y25mh+UUwOH5+RJ%PP_LQ z3ybw>ju&AxZTP=ya7i8j+t${zO)(;1>I&KRmq6Nv^h4d{+_`g$!OuM9`3q1fG(`q& zFAts7I=E?F-=Kt!+ZQP<;Dczuk4qGvad1;xl%*^ZmjtS&XL?GvkXJ5}A}nF!vv6*X z9!CW>YNX>(oe<-c>G(;~@O*1tnOJq1T_DJY>kO9iSqwEZv%R3~efu#fJsDUl_f11< zO4*_?A4*DkRt`j5F+e@;ljZ0r&f&|JJu23X*j`Cs$iY@UXXWTR9YI^H(-d2C%*kX= z48B67RL~${5WhTl_dtQcAf1#(|c$>e_%ZG z;&O?&8(WcQ*nJU)_}ZhV@{)w|^N-`4e*|I9CSwh#7s85|q_sHQr=bWSCnYH$DZ%&na%T|*_?%moi4hc~xyZ+TC2?U#wWK=1! zlpVP}mcW3SaFQYK}LZMODCKzy0cESV%y$<&MmVMnde6vS`7^V1JNu1GK(#ZN( zf8=+f8ri7=VE3*w&9KXopnYIzIStMPz^zLlN{ikmL6oPrtZevPf+2Xv5JhJo?q}@P z8{V&dh`Z{S4$eYVGAt&lcjdDvEZdc9wEx${i@{@Aq$mm&DSwXw~MNBJ9Hwv2*Six<%h82<*; zdeJ&D)|Yx7S2y)()T^oA%j$bi-S5AKW1B{&P?lcdY9pLjQjGO*C(xOAJq{C1CK)JMVX_8`qM zrXYe|{)%{awg~#&61`Xs7(AR_OTp;m-6+Cem9s#1&}Zc1Yi)0Lm}*aj(u6tMg;9O+~}3%?B5>+w=bU_+9!9H%2?$&1Al*zObj+szP+rQEWuF zC_g>navq3CC!ghNbJ;iN^!8e@4Y4#LD#|rV{1bORwkiQPrDhxVcw`m#=0;;Ze)(2Z zzYn92-{%-6rdO|5dR8wkX>1|rh0lt#f9T*6Df4|hHx|;^cYbUDPre2)_{ZOnk;dJZ zei`=MTr8AV@F;p|fPu?k*evh$Yl3czxCQ4`YaKOVtc5~tY(itKN3zBXiSu;`OV5`x ziTaJ63(45}!Y4U5Fh-jrS(k~bDjAJi_lyfRt#Tx5W(&LGjJZjO$sYp`f0pCl;@Qv9 zjXu2+faRgTpIp?RI zF%x}-_s$8Y$2`Xdl7>Gsisny@@*R6ZXfNksh+xhj^|v5(~FA?>vg_eXkg^byQJgxxY*Z&okP98nHldW-TqsruP2$UR;jeb!4w zd^XgDO???h)R-0&*BbK6cZ@FxBS2)I9JkyFxcS|VS+l_b)xE#|5auH%d+ zw0xDqGQYq4{nsbkj}HR!Douj=LBL34)!5*IdhG|}z*X$#c8=WyhslA^pEe!IE9_;j zZIJE0bZ&0%N3wCn(mJbCB0VW;wHzk2{KdD?%IH98{qGkd5cH*})I&?As-rVcI8ly$ zBe4&hek0g?Os(=+S?L$F+umO?Xn!|$YGt~^>&^sYp;G8I;f!CY$eE71LtBx`e8Ikp&Mq1yAC(XM? zn!2=njAUlsj2L_HKC2@SweZzbawg)^4Z}*87P{0ZW-y57%`?HU?oNg(TB&+LdpL+t zzwX{WDBTw*%*{1j4+3)~V_TmdIB{ln6*L5U`%odRALjC*!osZ!g*W@`h_m3YA3Auj z74HC%)62U|3bMqZ)-j8G8^ZvBWzHY`S2|pKI3^k$QA_0ltwyzV?^7|)jiwm0+mYX< zK2&YL(|n2wqj9<>l<2`wbglY^@8!a{$_A;KsL9d-af=0)z$l$811o&>l?j~=sjjvh zk*X+UtH_UPONY!n>B`%2dsC+m#k)!HwGw}P6_{Nx2Qw?G`&@)eCwe|Vv-hxs$!J3BkxE^_*XLl(5$w`ccDe-U*tTq?t5im>#0)ielBgi#7A;=>7Z zVr)H?Za6=mO;tD7S8D&L_x^Ad?TNM-%r{+mdq}(`*4h2a(vbJw35-obck)5#IP`GBZB@ zT{+l_rrGE?_=D_$LgKv5gakWp(88CB0<(tf&P=48^b=xDMdOm$ zasE%%NJ+^GG-)H?|7@c(2BSt0-_;g?#RNqGcM_6%f2lg7|7PWYD!G&P{A-QSt?&Jt ze>xkI0cbPGT)~|{OzZ&&!cIl#|DV5vI(PsINMYK~0VoMKF1~;HM}JSpI;P#&^)SK? zFb4m}xb}AkekSDA%#i@2-&OnkfIVZdo16RDD+#rljG+*5;FQUZU;c+W`nLmGpdX$d zcEz?mg{I;PEbqT2``@=h=Ydqv-ayVz{pFATZA>085^kTe3g;qC33nfQT0=Qt8Q3}k$n%36fO$L+yX5g~ODAc+^Q807U|L-3| z>9cQ-O5nsR20W5JBjhsw%l9N_piFS_pv+8aK4Ogm_1~(prmDi|JaiM`vaJ*!TLc_$f_ho$U>H*toPhRA+PI215PoU9eH~Ei5(1=bk@N^e+zq=k$G5jE zd{N@36&}l zs%>mDx+T}=_=TYwR7{b1XMJ#R>zxLMk+Q3wH7hz?W7rcIJP6C#m#(`Cf7VR5>=awk zUM9JfMTP5RttB_^9~8@fy0K;97=+nh>!L5lXL4tPO6Zg3=eCmj2^N`GoO2baa|Adhj%p2di*Xth9+t)%Veu_MA?m#jTHmh)7iC z5}J#=ZLUcgBg(~tW}k|m-%0X2M&{3S$pOA0?Z=NByF4(Z+r5%og*j2TEI0-NoCXfp z*uLuTM4hM~=nYe~nbh$WzpRgL-n!ktad^Ap95X|QNwI07W)kLvAiD7){Wa0by;??B zAGMXCGmocOU94ztO4mcsrd}S~j(!YYc=U8w)BT#BdHplBcHLjkiist)pQH0b)7fMC zmE5_cQwPn{4$VD&rhTZFb&(P?R`(WpZuYF5|Bpv}2clXv&oJZC@Mu)%z}UMf5I-I* zeoW3V{h5a}Y?TDT$#7A-IJUl;!W6XC$}TkAzX+h>yQL~>BMU}MI--K?uBQCCQuzEKoBHSIj@?WDR_gPb<>QMrpJ62_|}6 zNXwK@MOn`?u!~0!tjP9dWw)kUq*$4mjfmXx8_B+grwdjL(x(h%ZRwrk z+im^HqbC~m8VM8?4VN_J(|@sR-VCVZm=?IycIof;QWrV^p?w7G`ZkoM(^W@730Ar zm!bMTPX6yKitN+_jBkP`J~W6qY`@+-E;+C4q%^trHH_O)PveT8Gy?@b8Tp<#&(Z2nq zZ%b@JJN^Sf#AE|vVe8JlA2m2(oB%X*Lffgi(bp^T+)&cp>#F&CnhM2B&7(f2lSAKX z?@wMen5`9m*R1?({cGrG^g(SN+VkwGIQ=gK(?OQY`*5|}9~H|gAw2$}++BrS%kCcc zrIk%#e^FT5>*e#maqmwgs*H>W-XgGEt4JtjJ@@qteJ(WSyjLzYdN~Oxd|`8bXdgyW z42f6fr3F;n@#|K-lA~R{rj0{3K54WbJrhAn7KCRVK2hDtms&k>RqV`yoWFljjGLFq zHjK$e;dr=g%l#!kO7`s+SI&|>s=J+|mHo|SJvy(~!woG5pgog$H@q?e4H^0>{aa6; z1sq}qnm}N0%w6ru zl)Da7WC8)qX;tGh0Qp-G5NKB$?2})WxjpCcQtq2-pQ=sN^kY^rAII4Jk~~4IBE2eo z#?q1`$14>vmj>ZltFHt+JUIzvaSKJ=AF1n4V@yOd8XOuIhGDt04N_NfJm5BOI$Xun zAm>^K>lmg);=@-;KW%uaRQv$Hn4BoO`@4)mMfJwml+|mslJ3l_)0V40vV{YcU&Hk83_juqq#cl-JPwJJ79Dl6%DXu@}yP?MvtMUahm}E7O47J)d zFezB3nG-eBKAKj#Q z2D*@*pX=XZ#5b!;UpYte-gT(6wR7O7Hwf$R4WD(?40L7t$+$t%RLIM*e}$l4ISd)c zM%RU5O6>Wk?C45CM{U{F>-nh6@_5uDLS31p+U0|1VR2>py}y=9dy|y|nY7ZKzF&SY z8OeR$WpkAq!(Zk+qwr^KCHG18@ctP85x>A%N|5N}?IX0}*zKISKu*@bf1vY~=pMb| zdz(TJ2m%_LzS#F?!h6`}%7>%~6f*sIv81zswXC#9*fRng`Xv|DRy`s=y(t^6wSBj3 z6{;o;k?ZwOXOmZHb1FVu_L++be8lbGvN4H#?BqXDaFCHr37wwV)|peMp4 z5lEkoHavs2}Wck5RxYa#vT5X$jKywZ~qg3S6ako$Mio|$L+McnScf+`N z{^OBV9$s2mZdm4X{ylKxQa;wX;Q^OjMzUtPQh*eZdK+74F?QCvl_`sVVLnL2I+S1) zN>czq8_`A311)e3z+b0cnd``M;MZfVvb5<9?{4VA%W5)x$%r8$@x9%0%a_Ao{Urvg;~>o_673+gs5wzeMsC-pjiGx<_AJSoHg& z$30s59c!tIZt|^PJ623Tx|$mMv-@I2GvcV7B!_|NV4R66eUAIq#@hkk#VQDJ*69P| zk^av~d#VlpGdj3dcJ;5<Bd zyMM@SWBC&1g5SZDa=bnO#stNKZC@bcI_xTu9B9ioQ?peJ{WdTVpG}1uA0551>K`R& z*4_ZvTe5%V1-A6-U5mUbXN1)W5O2DGd#Kx9#`S5Q-&p8RR0upZ)GB@$6jV4itAG*m z$53s>qrm}^2fj6wDj4bpjL@sTfSZLeQBhFL!vm_sj2gzPXRXHPg!5cG^v=pPDz3KCQ28$BGhya}B|NSlSosNo%i-)l$+hWVEJy!Uy6h$zEM@HX;hThH5 zm&`NjFLMsC-75+j4u1j#`(z+E8r%WUu6q3i^foa_pYan|4&H_ku>T{n`nTkUZX(v( z4BSNHh>IF&jew|};z&Gj;J{gt@IM9Y|4LQBC0qxu7w{l%I71z2W3SUgBy!oF`(MfS z$F6=19YraWvxdX~Aj~=eUK1Rb2&!N}RY+k!Ec`#`ai@9OcosZSEzR-te?BqwGhKU}Csa@2Y;^(!9(yOJg4|qDr^!|*ZUUu+A7(z_=5c=oT-o)t zwN|X03i%`je-nyduu?cw7mAy6dUz0_E>h?;1F7Tgh5wZiLHS{z>bhzd0Px8LT51Y6 zGt`sEeHb4CE(oB!tGK?2-@oSpO8D;G@iO)dU|0Ol^ZgIsY`)=1rFaJKe}4=A4Fv># zG5s$x0MS9#cfjzNru~l}4nFx$pb)OCfXvik{xjtT7yextWVIt70dnmJ%Kux-1Gvl- zuc7>u=Ks`sJ{*L3kteI>{x4p32Z|RxOZwkxAnW9i+a#k-_A2}F+2z6C0VF^-6w!u{ z26!*I9e0kyJm=9!$Um=N!szWvV{Eq48fz|lBK*mfm0*PzB#cc?-W1uh1w5ji_ji7t z4>%u*mnduA5&CcxedcoF^$do3j?LDk@>uAyu$c#4 za2((1pHKpef`ZJ$@}Q)o45^6tcd4HW*k-w%XVHR7B?C?6$tAn?8f}SS)CTcrE5k$e@tswW>j7LCmKw!0BUbNoRNrjMD8g`^kD4lgEjdVs5F7AS4TbNqaq_)RVdI0cRTPi zG(H5s@Z)hZ_Fm|N&DaOm{0um9s~dR9Sm2A}l`8X5aUx8?dR8%2v0olam$i6vz!1VB z-|1E9uyn1KEAC8SHLH)bvt0xj5dbyy(0{t2=r-sz1#}lNW6cGZA$yVGn$8a$wb){? zE^6@V8rhWb%F0R~P^Ziy(|!%u;@h!8h-oF|10>kD7s@FU^?6Wk$8 zMqk2@(Dt1U0NRHAk)RM^s~nZ{Redjqn!w7j^EcR-U*X19$RsDieH0?&HNRC6XEu5y zCfJ#?!VHJ79WZkGxajp(*O1?_STG8-s+3!(W6{Tkj*+Hu>2}ws>B+O8@;>_g;$oG? zoLO-Wb)3|k+k6&b0ARq8S%OBy$SCHTd11r4Y=?ol-+`@;Bwk$u&T{3Jud)K%fPGJR z9~(Np`MVI4div;$ZgN z{}^PDz5ue^mLPtK+&dUwxnezo`Ksk+{)4^j)Z*HhB8wk&FdSXI4L^UrwmNQQWwra# zGisrs8oVe8Xr0_ow*e{5&{+^By%oJ=sJ9yY7KlxTo%VLwzmZzlpjT=sniLddTIp?j zYHXM^WL5b0NL@PG760@k_0f=k?!BSlnC~EMrGw7Nypo=dWHfaQ27} zeT|OVvj&$;Dp@H2q%WR9VASP&dRiRv^>4Z`5Q19^=!|#1`OSdv_8dJ`o#ue?5HmLIy}`5{6KGuw8{ht9*0W21mnt7ef0lF3vx3Yku`K+_h8CwCH0) z7PeXcvHkM&o&xa33RIhS+!<<$V4TNyBz`}ks)+tV<+C^6CsbSeO5OD8gYZy}rS%am|_w%lqmHTBx{vX9rqi_I`k@`Uky~UC9uN zXE!V?ro>y;!=6YRFUf=tH3w?6_JPKbagx4Qn#eyg_bc7khcA%<4U|3jDT5>*UDnyz zxu=q^)zbPXj^C)+m6V_ub;6zteX)tilJ9N~fX5NShl#E7A>3qz#fWg#l)nWJaeUozh9nm-w_96&aeM2A}Q%JAVqPZV-=z1 z`6Wm3QiYt?%G`YoX&qF$B)09_o%3A$zkg(2@r_|D%{+(EcCVWvIMJ$EW{l9ZF2tz8ZFiJNn-OWgYh)PRK4M>N8bPS<%cSsB%okKG;?*`8~ z&w2ju`{n)p{on_k`@VOsz4p4Ub*)%lvc#JwRI6|J4a9)}ZSJt;gw=#|=C6`-C4)q? zB6V-oWKvY_uo>U0VZtQz=-8pduO_G2qt88AI@zvBQjwN#qGar|cJ5k@enZ8^oW(a4 z0_tF|i%$}G9g)^z25mkJ&n z7sD?OHYkVSl4#4|yyu*{p7i3#nke=2ZOQ^cOx9E+sDa_;)RE%KAY{3~Txf8`+9mNM)%+&3n+-S}$wthkq zD^aJz-W)Jd@9I=@)V{v=k(H4_m&1>Fp|yu|Wkp!m&BG-8Z}3@+P zM8vjGK_GW+y~fxtigW#1obtRrCib1EQkB+shQxgEn@#|tUQYR0xe2`m*p%U=&I zN*yZ1$XtDzNQ5SrN+WT=i;Bi@qxee`DN9OL@q#iTHwryI#Pv?fr?8T z{3a+@s|Q8`L0j_~OY`Jt&%*TzhB#Z;<4M!*gk{w6<1n(`Lr6TwMuher(8Q|wiqW1{ zx(=uEu-0Jk@HIE!;$3W6G6&R~5()SaBgmnDma%?^xG zg}Rp|y^w<%_FC#PZ^&TCHzG-vu`muc>S(0`mWri=?n(&wl+oHfCvq9D6xS14Jhm+$ zGxH-$xDwUd?+o>LHC$Yx_Y-?(!(Jpl6-*~pW$FpO4~HsZxXx9Ix51Llr)CT^m})4=gNTrS0cX$NTnD8BsB2g3ZC1)g)|tkg7%p*D|_cFNV24AWl*t#txPjxcoN7 z&FVQk*Og`7>DR<S}}r46ijew#^8(Z?)pv2q5dLV?w9ZPJ0V94D)Aug-RaY^5{T5xSB+ky8f2 zTPp?=t#z&UR`ZGmH3j59?aK4VyYLCEy0D9V8rQu)m$3UdJ~sGE3kI-L={bCs;}%Pe z3=n91NLGRqO93*%8cIh^N`B{b9VXWf8>i~_q{zB@%oOU3a{yeg1gO6peFzYKT$Cyn zUjU+(l`bKZTuWV1Au7ke3JuygIf(o(ij|gMZ4F*~Libl4?$iM1TT{Kls}^~4KUAz; zgEZBjhP9fuF0bACs_d&Kz~9?fP~I3nvOCCC;+9bY?Gff$Pn;^3ejS!((qT_u<`%x1 zKyf5g#W2oUMY<(ed>D~n{8=q#43%Bmu@YgE-F#oYZHg$1s2n6aV}Scv;g6Or`Vgp~ z=x@--=EW@&`yB#Z{0sqgB9z8OE_(U1a`T}8;EX3`r)Uu+Rg8}s!`53X#*T!JO3y^x zyM~9mnOOJg2B-NH@BwEuq&A^6e}EPSV`c8oecUGDUWVTTEp;mz8Geumxiv8t#sRMs zG9ik$Zk1NLI7uhF#bpXkt{s0X=REE5Ka=L^hMjLj?1?crNdcuNP<^6x7sik0=Dc!( zCeVLWvtsM@b>Ony+{bQV^dal+dRr-nCXDT7uJK2F<|Da~%^!)sKweiiaPWj+1*1#E1hxkc>+94xH{ync)jTdpFBFrjB})hxAwx*}aa zq^vC>cw2ts3$3{-s(VxjWmLmklZOo1MFubcjRrEaQLP3E`B z%JUzS9M75^v!3UFdy#4cCmPV*7UVU9K1bXkUS5jMI!D)A0oh$`(ZU}6I`TUOy5%Xg zA)ciVRzoJ==<$Da^I{3}Y|~nI4H9XrR#X^s=u3$3H#LR{lm=uZuPKVs5w%x z(zwBj_f)4mDevaBg2k407+-k=)*D7-HEsrWTFLI~kG%8X9j4yMJ<{)wz}pczKTvDS zR9k!0augP+>?^KWkt7O?rzz|RuYQZ( zD0sJ_BkFe4DTPyK+O`pq#%xBNtss?Ncb&NV+0DnqY!c31+g_iELl>-7Q-64sug^U=ck8(fx*ou zWWosKs!qco-No%_8c2!Ua95!Ph4ro4<&GpB5{i)#gX@Nl6OE0nY-IJL#-!23pnl9{DQ} zSJ%9?cV3&*SITd0gCzF=U*Cq$51v+tg%Q$8S9>MR)~rYnWRdooPfXE(al`nOSYn_@ zw>{Ya{agK-bx$yrh`fY%-iz?i7|F7rj~%XDRHGhH@~#B{j}=a z$;pW%?MskIUF-f5rG0b)bO*e>ZDAmr7XzcH!WPZFY;dnYLr-AQDK|#G5`brVOL%1yudrj26-=)_XE?@GB_Y>+~b2y=gVYz zPZv8RtL2feNhHTa4%)OYo_BF|4X<-xR7{Y)1VH~t&X~fc?Fe4!1>0ph-Cj18PEB>lJ<*AZWng=K_kN=)2^`( zw%PDE@QwEy zYmdpVc>wN-edE_C(!lDRD@goD05c-p(=Fn*%W^O}#k9qbdV+b|{^|4K!trG2 zhjKzTOO^ei;(6JF8@e{GfE zZef$n1Qm%>s?2sKZdI-iD0}34Gl?i-_&%EUu=ZU3Aoa2893u7n9uaDe;*_d3L*VM8 zJ|NZ7cfpvo2-zLnDA?#)pkN!zI34Mw`{We#gpsZp^82i8k!&SQceT&|>c&$A-pv)u zI1q6z@_7Dy><5(xs9kwycjg{fDam8P0wt~r>8`O^VYkrY^HhP|6^W-n2?%uNwwW9? zT;D>ff5m^E+F#=M?YiI3lOw~%BhOvEvn9g^Me$9Ol(Av(Cos`oc2iiZ>P-0@<>HB4 zEvSfP2ifE(s3hT$=&4)(ZanF0RvT-fL9ZCq=YFXc%_pJN-_qI<{H0-iWGq7pInO|b zo%%!0(|$cx)v2hsY2pk*ClYlLuVz$;dlcfd)cc_7I?czyl3fdHsGN4hs278d56(25 zUb^1aTfwQ$VE6fTlhawT8(UJC62F1?DUfBa3EGV$D_f6^uA1bdYYx1Kh}Sc%Hzl_2 zl(uoA+dWv3cvr;PL_*F$*SrIOd@K5`MuV2)gaY zi4R;3i@MvEC!4I4l&+bNZA+G=Qqq*GYZ>k677<)G8A6L;njSg$2-4H);-*_UJ?X_T z!E&Ihr|-9LR?KB$EyD?M4|ONxIp)HtIrMH=AH%wo8g@NCasY zhx|si+uIFyszF1$ugzdQnUEIWtrC`!f%7J zF6%v)+%BHaKA}BA%w;>R=9eGd;dwz95GCB_m<){^*unGqT}s1!$g|vMO}HQ(sz{>v z>_{9YkAblu8MkXvm}Gq@9Xo|PWxgyh6~wYv#c_OxOV{(Himg~dSB)AiL2m^c3EifLjl@|vu!G_OaLY+Y)7!AA@zTiq#rW?{02}n7?+93$Yi)M z4FP!v=$+_#c&$R?Seo-87vK{~57^dJR3lJmQ{WQ}Z?Nb=ocvPQ|L9;7*9?uOa`>a8 z|HfvQ4lz?MFbQ&_F+Kyh4S*hCOR1}ay~^-2@~&m`AC>%W57JQ*F{v*D02mV|0!n>< zRVrwzeAq*Ha<<9Nm3Hw}nIyhWu`7NH=lUkew=ID9?$4sKvW$3)JK(DL_nWyD8{{4` zNa;*pLT+-KQR5Xybld_cW8=DK*q=%QKfM5`9|0`3+&@BU ze^)@D0HEci;@+QK^>4`WqQNwC&OimipYYl9=a(8we zzuNEzxV@kP{QrOZzsfrN=YxOfqkpB8@&!dj4QWOr5D?_GWZeHu*kvUK;KKm@!2i(} zN{l(6vj||<3nTJZcX2R4v4o4RxVeIJS;O3%^%7S4I~C-LI2CxaMC&>bBbT(k*D^R5 zY277uZnDAtBV~#7VKsvNYCr(;;*X2!8mfD$8CBo>;|I+(61+Ps%b#ywc}e)9o|GHo z`bKP`=C4F4T}RlcW-a;v`$qdrl!Z5sre;|L_c!T+#NX-3#B;>n3L5V?g z%NDkIZyL4zuUDGO8XHU7D|KeZC>AtuF$#-PQri=??KrOYdS^kC2dvcu!9959;vP*tP^Imn##xp3^8DEL2FA+%$^fvix?S2Ev&`xsi~kUwKlv z@~*K!+>&es@=MI5#iSplTtL|YsoSaB1pYONqHT`h{nclyp-_cc%rcQ0vEb+f=AJzk zxq>ePa`Qis^Ejw()oIkHxND$#<t%Utgu>zf7+Bj@ z zIVO?T`k}GL`44+78vBfrSU#Y%kB%L>v-3GEsU%nbB%-JUQ{uB%&$A7_oP<4sKf~Gg z7S{8&qB@>mFw75GK4#Qv8P(Y$vhHtT8bxVN)-?G(i^dj)@5_dQAFcBd2AYPFu>Tgn zE%`vs-4mYh#)gjli?U+dKYh^jK`dHMUW07s+Vhvw6h@S}ZE!KOFv z<=mMG;s1SY~4$L+b{R+yo4OuS;+t1(jshauyRX1 z9*@+%v&DiXqEPRK;^62`o|@kYTm6$(=(+2<_xSQOD-~;`Pi1;a9POjr3m#h&l0|de z8!pprFn3xVTpcr6T6k(QlcMRAm4*cutOK&B`+F2>1u6*Rb!T1IZtcn;reE4QAsyZ8 z&oXVs%))Q49yjmTxp>6!XOC`q=Y|?mDgfg$cO4jn7bq$RKcjoN@0eMk8_;0eFYO8T z^4Nddnx248cUY$KAjFR{#v1vn_HpU0>(`_xY%13|5BHO}zA4_yP3f`hdGe~)^p}x9 z8-umGzW&{*qAc$ol~(E8x5}xx-I{!LiAzR5X3a*D?5(_#D6k~?-a7@|Pdm|z00z(H z>I9`kBxMDp3#f?N1h@cFsd60ba=DAJ)!D+}aCnHgM$tgpe3<}yePR#t*RS$gDu#Y3 z*KdJ6S_(Y-A@iH|Y5e*H+uxh+QLb@3-AF7M6;4rRd0^*=6q?nA`&)>L2_aIvRoJ!a z&QZinS9S6lD4*M);1hNBSCopy8gKONvq!n6*kx4c zTbrPm_%(d1>L$s+i#9tjxnQo?+uf>_K#%| zkh-}FRnI}VabBWj|H6}x%N+1#UZugM#k1aD& zK)mc;$+L*MAYEqhW1k+>PL4*8S*3?5c3P*06VBh6b)7t%Xb!IF2;>CfQQcgFdo#+} z<)-G-C>p3Yg6>lhCM?qjf!Txs>uKkj(9LZwA0H_%AsL2)wk9Sq}b-QyGbe>GnnrRUbX zJcV$p8b5KKo45JB>Rdx(j}DUgsLs!p(sajygjFwJVyEl_u}Q)&!61r`kBA1Ok2yH; z{;Q4p2si0zFOjULO9W#ct5%<$hPa&n;`4JYSY0nP@~_x&Vk6Kl9B4sb@xWv&g??P_ znnnRTMJv|{vY%pUORu3qXxa%T60uPPcQ7YJ*u2(JVyOf@>`XKT@tl5KeI!dIg=Y&i#kdqaF32Y@=W1QN&KkshCa)(tC0K)zK5GO%ftBa|?<7&*ZHx91)Qr+Y9H)%Z(9v?2dn`93C zdL#J5&2^Js2&vPVC!$b|QfNW9UmG@ZEb*8NVj7DL!?}+G#upb@4+Ws>aFx5eyZgD> zcD{HE%Pj^($a#;3+p!-9+nraew;kn& zDT|LxEq6WQ!;B<-H{O{mKUbPYdC&eeK~htLyU4XcW4Nc*Ao8K;s8;g#r*ZB+({@=2IOS5Bi+IYj&)N3NClfBBEw^+*q5>NuTA zK8wTM2N5@q72+P7&mTppp{?DG)MsUhD4%S=ZEFpV=*MDemJ7nqI_vk!{5w42q5f+M z^3Zw(F9`ACI6V_)gYYkHhHyfIbr%_xo$lpcvVg#$BeyshiuROfqEs zG)rten2eztmTB{LQa7VszvXAzNB-rTRsvd?#R8`+=excytijU0(6-Bb0Hoyq%bx@1 zX^tZriZ+I^u}l-!&8(_g4H!&%?F6IqA5A{;IO}bW+C5Nl`AOl>x&Am~<2)tECO9*i zF$wXbH)8ZCmkQNQd}qUqsv-Z!+(1+w$NMxS?KH~kALDU(7m$D_^T43D_$jnL5W(_@ zx<)j{7@ygkF4VcmQ&JYGzNkhPic_~KtWZfI++0i6=|VexdctZARh3m6QJjMh&N{hx zW=iOCuO>Qz%*;|3cdM_aOO&Xy#UDtsX_WYdMYy;}%w1=-M@t^#`37G9q)B33($^PN zqZ+BQft9%1R=4~vfH)|^uq?;s&N|BFZZpzP)zyiEX>KbJe{x$Qqq~Pbsn==GU!M#$ zSSE4^Y|8zA4wcKxL@Tk%EXs_EH%=MOWLa@w8PKU_r(@L;(@(ptXWY789zL&Ux4$~9 zur!!$7A#b1Ub6JdsX0eMG_2oX$A(K!2ZA*1a_xz!HG@Kvbo%RTR>zzy)XXNT8U}|T zPiDTDlfTS_=EYi<*biSvxbg5hxKB;#mYdD*;!8uQYI;~BXM1_t54I*MzhTF+MOsKt%$wO zNQZUDJP&^jj=Ys{-Bfe4t!q+}EzOU2$@PEPBF;A!8q$P?otNhriw9b0i!xljs+_ExMa&!ZC|++#KkKr|FsczS z))kuxtIxRi+{BqJijO4zgH5k-S&zlbQuN|ku0lS++oPJ{#$q))xe~D<1dWTqS8F!? z{`fiJN802bwB?V|cZ(M^1fpqbC%JPTW^3Ayk=(Mcpwn$s?jV29F``NMy34E<0qvac zBkzU?GHJpCCc)UQyr$^g7wIaGARjV+fGd*Yzj@cYQqyrJbV)vQ%ImTTAw zA9Oui$G#H%bC_P<(eY9A+j~;4j_hW`K zKbrnt(<)JesXm;MPrD0&dvcbBziJ*RJ!NhhpmyKQ&&rticm{z%?e`wSODy$58C8zo zmmUXemc2^Z!e8kvLY*~v+VhI!XjKxV66lLk&cPU3trsW#@w3I0HXY;Hg;?NkBoPoO z6e>f*KI+nJuZk--~33R**Ba2M(I`(OK*FGl}xIeWiy}~j1 zz+yGr;Lb9F^Y>xS zUKsupyy-j)sIm0;Ze(#!LkH_G>&5*rzJTq+6udiFm2NYX&l>CpLlLj$U!Pk!+LqpQ z5;H5dkYwuP^R=$o;+%tV=_k4`&8;knNy)I@M_Fu*lk44CR?O^a^m5-^l@KL_0L~Yg3@bx!z zFc0FWUbH4rJhh{W{aBDqf*m9U^=?LlBPBar=l&5FQdfnxM~Ii2qr99}UG#+3Y7t9b zfx&*~tboeN>FbCg+0Mw#W~9r>%qdz^u2W%qS5wK?^7h@`(v0)7&A?B{nu+Rocs9Z8 zs%coNvk;w2*iPRYWo6o0KWwVX_%rvYB~wil>h$f?r}q92m?s<&hy%=k?IAnYXDee* z%NvZxT5H^6hSJ8AOXjVi{&D_jYlVFFBa8q$v}EpsD6F}a-2O)nNA1CKMaL5ab2vW- zRtt}@J?>oGZNN<<;L2a|8te%X$j0*p^{*N5J5)u%q^5^6^EDN5vg2oqpMQ+aop0%E zgiE^m;g=oymqgjYUI+4#juvZXy%l=Skt(%TG0U^{{ELO#7zvFH-`($9R%)e^h*^={ z17V3nyJJa%UzLI-;Yg7oJ0*|pE7#_Y$E%EnUK14?INIf~k z-yGM;&oLz<>mRSddN|tc(Uk%sUbSL~cX9fXk3_WH2da(3Z=BF3_DCKX3+oRl_amp0 zwB-zwn6iZSvN{dQ64=j8XoSut9@_tG*u}}K!c))`5Hl3kM5;StdecrIf1ikZDE&;X zdwKOZpl_)8`(ra7J+vxMd?W~Z{c*1^y$y@Iw&=BLvC;}`zM9n5@069cXlR++xDu`A z>zN0`@51RN@4LM$(lc!&59{j(mmbm4CvWb@1e9;|uQWa^)$R#>WgW8{JUNgu-vLeE=3KlvFvKgP9Nc zVULbx%fllX|M100y+|kV$w0M8*0B?j`7>T zm)(i?%QI@{Sm_drY*=5s`OA<0JKXjIS-jI`{)~4MCJnvpTmdFrcPKPq3*S_cGIJR1lSf+TeeO?*A0%ap&E4fgews(I2*{{0DD+v^p4$;T zJ#JgTln=X=gWT>Vo-*9=q0P1~V$vw%wkE?~*(CXmB8Nu9QsY|KUS!#FW5(V|$*A55 z+%G#?r>B*|LMcXk`C`h(2+G~CbV2?V+r*yGrOl7|xTIXBX3&Wl-7U{FWcT}fNS6?7 z1=iW3Eo_~G+<2h~#dSr%-kDa(qD7nspEV`8}u>pgvE#_Ex1Tw~P zOXk26GG2`9lGHk@IrJ$k{uT>!CPJU;8~Pq#M?S8jbeX-pF@tM*%fDlf6$nXp+H(H% zT7uU|e7F)GmV6836_yhoUEPaJ-j|xiDSnjv`Y|G8ev~ zS=$z`2cQdG=aUGlQa)0J79d~c7W}esa56nao;Iya&IMQAhr!}B^#V)ky6P8+)=k1q zy-L{lOfr_fQ>G9Bk2bvg+9H0`kR?>Eroc#qq#-R@973l`p}h>v8>qmg-RZ)4c)g zJ0L#yKdxcna<_CQG-KS!wT3b&59fK6iJy~x%4i}(l``^&Q4k2AnCFOxC<}SRRC(LZ zJJQcmT*&syGvIeY8uvXcf;tjNQp_@^Be5Rs?48XhIN zC5X&Pas`jtiAsb`XE*6wL|8`#04XSK2T&|kRi@Mn0{U=UH>cCn@Dp#duB z&Ye}u_BfreC0nykrpWN-0pBGoRr^(x9LhPO-E`}~=%zYqDXoDFir40<&PmYa44WD` z9)HoWD&?jVU1SAY)Qn;_>PkfLK0|CCNMTclSw8UJ7#|S%RJG&h6ePTK#j*);+!GqZ zY_z!S3>sVb;s{&c40e9l_lB<2oxbkv`QBhU*{O5YLSP=9WN}bjAI0fb{TTM3;|{Iv zd4#9)w+)K{_~7U2_lStp7fJq^V)ZI3;nU+*4O)YKlLbhTwkFrerFUO{-qG(9h)sl! z1VKpVcCL{y-tA!bPmTPq8#Us?6_^PF-J8YttdWJ+1BMe_8rSmDN@{AIo$Pja8jIi7 zk42zOL~b(EIU;|&8HBX&>#Pw+%=c0eG93*kJC-d?)0N*%>?#F@Y=ZyW*CygSELu8g zX^XV!_eID)8ne-j7Mt$l>en5s;cdBcO>!^LQM|NBh_Js7^J=-tE1+w6BsU=(jW~Z^ z?UZ9fs3e!`h^SFi)I5Tp*be+Mk7SEks)K8uDUqW%JsI<(#l=q(-}FX*GecX#pxeQM z6Jg9k`Y%#8h8?JjcTe+zM_U8ZQj*`AC>#69Co#|N50-ZbELCzN7RY*{p0>Y)+J!s9 zwoVvoB^H|=%tSf&KS6ipLGy~B?I5gm)`n&h$jDE*QL6J6M2`A8(dE{i1vcVin%{p9 zPjyWz0lZ-4`oF*r09`G_qpP_2$mdysunmZyF$S-S71<03H8E!{ZaG?n^4y4^pxyKI ztTrKe6CteUGc$z1UxT>lBwgWXzmsB)$3`&uS)_vjjXK*ojU`D5)9(MGMCYs|kS8yN zdfLG}+k;*l=1a-=x`TC83UNw$n7`5MOcHqGZB^-*V_q7=Jc^ow{}CajugZc&Id|2Q z_fhL8NmXC|kZ+P6Qtnp1*ojpk_XKR8Ct*k3*p~BmK17)mt6RMMfZ#393W(D<@ak+A zRxzANbWd1a{1Bbq{enpfm`m?p3b<+#VR$?4Knz5 z#|tE+L}t)kGL9VC49h4vJyACEz-|T~iLU7Ldm^?I-Why>xCcV2ZcglMCwX9M?RUl} zavli^LpzeC2Y%AwC?4szlttZ8S|_}_`-|~(ki!TYl@I@yl@5wxQ|BbM`(2L((2JMp zhzP?*9pSnx2TbYV;@hMJ<94Hkc%#0hqg$??F7{LAk>^Gam{khVDT##PtBL1+2<7tT z;gsE%FU)i1F_#q_=OB&*?Mq2(HP%l=JdWDM-#hy$66fQm-db6s)S7i397>y#Akjghso#r#B(J$4Q`82bDWa4MH51|=D|h32*L zAJ#2YC2uE9BsPld?k?reXJ^_OeO0d=otP!nFIr_5v4!nj0~p~Ikn{xP)$1bE!5N=P&wbk+hl2)t>LeZb)VSB-Qrs*~FnfMN zu=^h5imL6|QEyTL?b?2VSjEagm=$bd%}bml~#f_#LwrHF&=kbmm9%~Ty6xI68N+r-7%8LLpLQCH+7BlkzP z@^1(3?r*nq+TsjwhGVt$gJL|M4{hvhdH%{n8h=!ezd!1_Gg%tie=AW{;;&uQh){e` zA9>$1?6Q0A+nUg-nU9}+pO0J_wr_bl_aq>&s!op}BO1JthZ5d%if)qjP|o9}HHNydx1=Dk_I^Dc}1I_zVF z;+F%JJTl^I6#ou`2Mso6O2)(Wb0W!CyLecyy_D!rL5Xm4Z06^yNl;4>LQ$!PALU14XVd0T>654^XDqw5=3v0lN0HEsPqv$2KNL9=wNXol3(NY&< zwE)LTv;^Xd9_}MtN=nM3BxaDT{@)KaMa067`aP3PcWba2duZglMr0VFJ@-C$?>Cz; zR9YKW(e%>%6??uwc9dw26hmvpxQ9-sYp=2O(AM*N6E2nhEXGvN zDtJ=A_z^KF1Pv`TBlHa5T2aK(b7Pl{4IFExUx_`6%!GBDlFP&%>?;hLqJZzb#GtPTzL35oDF`h@#hQP+Vss;L@+Nh*DnnB0lrHw zHi`JG%s%Eu{0|;2arVTyHxV0Tv)KmDrQM|^<*FI^{vK1}wjVu9dqTpSf`)Eh@Zy&y zu=A~1nd&bE`gJ@P6b8yUF8lkAnLm3W824#;dFkq3YgL?91VQ2Ki{iKErid5oVk_MG zEyNZgm3dayP#|7j!gL!{M2YRM#sp(PSV!^a;R)IYyXp@f+l-GGrG#(3(hU2&>m z?~)MvHN}hc_XUv!=kU|i&8mS{OOzM{VMl!MieuiEW%8tIjfD#i!(3OY$K0GN@#~)n zmhCu~u=QvCgVSUP{)fpc)1-DkTe4Uxf4XU2{8*n!@J#UwatZfiwpq4ZXKaT1C4~X! z5NkLY5fTytu;zT+4B2ZJPu_0iphfj08_!F}7M}pRLY?f`N1UESlL!Y^?)@tcaoN`{ z0ey*D8V-(N!F9>uT2ON5{rmTz-bgg3Zrn)=Q0rj8r@9!~PX!xK>(`_|jt2vpi=if< zYgFDKM3d#?eX3}|qBc-`sCwm+1JO!mh+j1@tVq_6Qal&1+H|A@i{}T2Xp0+>$1Q}x z0q3slf{A`KSbVh5+NR033zDZKt3wSntOzMSyvx@3EJQ>{4_DY3$XR5DIl?H$rtk5Y zzyAvRY-84&EWtqt>hrw`sQ3{ha4Tsd{&~r4lP{M3n|wn+B5}F?j;2n2MZ4v|dwp*A zfxnkKD=%H5V1WwmCBZ>_$7N>$zhlhI5SjQvh0*4<9>(XM-oC!C=l(!Fz%bx(edxx9 z-qCvUDG(;;6bRjClF}w(kvm@s+iD}0`26j21nqjwYWc5w)bNB)TRL({d{9aghMIjN zMW(1fe45s}lXm2l)*jmtp(k?WeYn5K+p*u;5}(RUPo!BrMt@qIe1Yzb2i(^M8>>~Toj_Wne}pOE9tY+@ROZ1 z@+j$9sF*gOPOgmWA{8%Zu&j<*SrmFHxS{0B_EkMDo7jBv(t58BU)N znVoMt!@lb2wgRkHo{mZojM%D*UtnHccUVd1$k zCI%58+Qeo4HurZGZiR3HYj&?*69S_`nB{HrPeTYAMqi>?5<@jY3C*!@8 zR;%C8&SJr1;xG3X?cYu&^Y7)V=M9y#L?T1%mc<5U5^i-<4wiWax2vv}k(W%r@4;b# zDZ08F4Mcj*QAV+;7L4SeB~v?o*2NBnQ@u7Sd3Cme_khJlB(!^FAjNroSv~e%QC-b< zu?kbk5#!QawH&2X&s;r&7-|R4SlY43t)F)1DS{30q$Hc~qYw)5<*!or9JZ!z_fNfz z&8HYol|rrzne(9@J5ExKf2TJleu4jY-^2Nq6n()Cq64e#Wgfz7kKa76z^K>kV*&bz zaZCwyHtG24fvA4w@ox>5pNc#BBO39Us;{dkIT|d(ZJx8xO_QN`JPj2nYn^{z&^*ybw64EllPIQOENn7pFD(Li5KjFL_2TIKlxTB}3#90AT z!l#MEFC}WD*3!=0k))ZTyyO}IceW0MO7TMv1DeO`ghhousSH>biN0{G-ZgQ9EN_+O zgwz})P(h$xtEHLxf*iHh$BDlMJBnM8hfl1m)_EuwC~Fs1?%m(>Ups1OIwzqU0h*~o zjRQ&abS%3k2XSNWdF{oNtuk^R$HKpQ94)ap;LeD>Zl9mO?eyi+>0A7q@HVj+Ab_aO zU@m@yb55LU5k~7fDMR~u12~uj-t2&#ApX}mS#+_QjZpBd?d>J`C+p@BR#}4FJSey& zZ~o&ji?-{VkJ`#JqUi2vS$ig=_Fu$zg{D<>qQ||2AhMx7#F6o~d~UZz+JbbhZsLVL z%xrG=V$=?0Xx25|tc0cn)ApZ~?Puxc{$8_))ajtLhf(PxCTUIlV>!%zB94c%hYa`m z+D19Et)H1+cM-yT9$Qnqh%k|)M1Acci=?n-bP@@7bkkzssC5ohc<`+NgBkC_8m0Nm zL0iPH9x(lN48P(wY`xvj4bHPlI4*zIfVbr`M(J##fMX(C zDUcgZ3PofSioEIm;0#+=bYobEqt}mrq4a~TcPFc{^Uy}^1NI4flH#6*o=ofPstR>c z(-@ax+K$&dXS7(H)6s~)>vFXc#w4lCDV7CnZ9rnI79B01(SKoCGDOz5z+S6MssH2$ z*W)Gj7@N-dJ76neyr2>!xIDwZ-+gRLvwW9Jzt&LCS3i{qcD_?tRexe3t`SD_R)ooypty{7E}GY{M(A*17c) zg1_93(A*e5l&hCbxo&53^q%6x_0xmZmyBabr=!HkO0)e{hhw_j41(3kjMsZbQh2yo zZJT%M9MLMFMy1(!ANxz=zh~)6$D9#J+F;7#pa?K9t7WfYM zSZ&d}%(K?6?h?!S4&|0)#a4C&H?)NFZ}>4@dOmyA9f;*HCXBWVU@O zPn>#Rg*}bu_)NuOqPFMLG8;OMJT#A8GYZ7C2^?CDYs);hlmuVawxC}0jeGZlg9O7n z{GjX7?eU=pl4vwFGvfl43s17XfE=kW{@3yF4CHqRR4d$;qNBqt@><0;t)!wO0ruH5^olXZlvxA3wi)^L!c{r%ckW8q_*tb;c5&im;CUxg``!exUgcmv?X0 z)pbH;+wITpw74`5c`c*NYIE;}hpIS)S#(oh&$X%Zxux-f(wcrQ zdB$@xupXb??|FIYBOszrRaovIAzJrhJlJ!la+Q2bok^8=yL6S^p~uAuUFQf}2uNNj zxwlcw)N0t6NUo$q!z{8@)i_?%)`sMr$r2;Qy@{RFR?EgOGVHA8L0P-&4l}>6hZv}3 z&KPd7Anvf(Dz)gTX7GH+K02~G?yy2KZJ`=#_L1pP_KnLyzd!T($=A;silyd5pGr-P zjl=8PBA6n=y}dEEr$8ZUCTuRE2f(SkaqHHte)5YlrPXS@(BK{;9@rwIg!AH9EcsO0{- zaE#JbK~ynMIfBaRwF-k(t$3ZyDe2D<&LR8e!`=j= zxCI+vRjZvQg_2#regmn)KOH+@TBe65>N~3Tj#gK{vH#w*wYJ(i zI!2Z_Q)t36A7FE7<8I#rweA;o4hstl!IH;Boymj~zuOqs;X%A@>fSKw?vYcvvz!EX zLl)LLPOkMwX~$F|u9A{*X(8?1w)Iw6v9qpTtc9!(`jrq2C|vk~N2)t85_)-gi6-e6zG*7ZkAe)%=z~rtbAt4Q&sZpe$SV_+t>NJEk)d#}pF3Ztvi?ps z12Fu9Sm1@=fc4=4zb2KOkO{dj=PCu#Ukz=n)!4wyX5sziX&Hf*g{Ng?^v_HJ=|I4q zoIQpf=-mz02jz-e`u;natCCjSHYhcnQO%X9rE=oBb= zy1&1Femr%)H1*J+ImoBt9E6@edjUg911=2Am=$iK^t`;hoSfZFP4hq!dLToAO|Sa# zDI#63##K>K@j1h1H^5mv2X}mNQjh_GU4j7mkF_^iaZ~`k?s+T^NEg}VB8~$hAwkq) zXU)O)#5_-4Rbd@~PW{(Y0v5xa7;~5bWL%09H6Vr{;cas~e} z$3KvKNhQ?FTU?XakNpJW+;kx`w)5VJP4IDCxxV_QfIF=1teuOM_CL;!3XeI}-Ki>N zs)J8IhlTxgq{F~Cz`6JonDl_w&QAJv&-0?iHr42sXkBT;Vz-dCd_hJKv*9rbvYaRo zJ2?;vTUN!u*i`>NY@KyjR9*M?QB*_(1a3g-6p)gZ1}SNfZUO0%9AIEj5owU_?v@y8 z2kJkR7{AC>GX$>UJ z7~FMQpOZ4w@g^Z?{T64=%O$gyM31ETMS{(wvO#|{-6PUzbfL=ozCC5!N}LTc2#Ior6(Z;41b`c zs%X_nhJn!wRQ=xpcTNDW7*yuS(7r3oj(fqP8eziq7 zcrQ8;4|r)D0?%OnW)*aW_Ec;dX7hz2xAC z2t2fEY*rSn#+`3~4dwSG{~Fm&L@{aWKXOJvaZLM2J~E4_tSv|*MjV>XPmVzOIJa?u z8F8~be-G<vav08PK5=BxSDw7OS4;GMIue}5EcWXcw zc%b}|bP@JZ7_Xiw#Ctbx&m=ktd7l^9q15jEnVxBL^Zf3#NTjcD;!S?_<2>xAWMf98 z(erW253;}y>Bsmp=Uo$3onnFnlN?&8&@a*%Z$1j#+`~>klKkzmamddP%E`cT02cP| zH85=2-0DRXo{8GUR?rn+G@(c%F?Xz z74EG+_xNTtu0M;4G$sVKf``*qZ%PlkyLC)P2S*p2KyIGGswaZdK*q4yJC5b{&kW#) z+^lrNF2o`i!P60PNw%n$kB9>yDjIIq3!Gj!7-|N<(q#tT?K;j4OH!X$de5*Gb00}} zdA(UyZ8QtI&;YXd7~e)RfQ$*X7?Zg76|!9QxC^#O>HUxv+2WniRYXp^zW2$2nT+w?N6A*d@j7PO%2Cu9 z+-`maSua&Gv=X!34Av9gWx$b-r*IRsCbr(ARF=+|xr_0$?xyyH)=ll98u?v2h*-z9 zEgQq=QM>tdhl~~CKGpTqm6yGk`u(KTZg4M_pJ8wwgOPfyNO5mDhBNW&^XG>td*hr` zi@~Q?P8`Crlr>&W{R{>N_(s?dy-)e+F$G#MPnH0{>}n^_hdbn73XCSlb|7ms-PwXR zb3qgAA{6n(pEnR72@knMu6o9&HxE0-4@&}EYodzqn*+o1jniolp6E6{D8vO)k|JE) z-Q9od#s9Y7(IBD^>Cwg8M|ZK9*Ny#VFN(qTy7tc$hgOxOUnbl2_F(EH+P(K(!(U!7 zJ$aT%^2aOjTh@wk7E}0Zgj~5f$m$w2h|ULzL^;JrTpRhMKizHm>y-y=|NW6D;_<_8 z{vtqt0hj^)c>8*tfKxGHNq*p5q!B*&m^*0|asAlk$-53##~D-Gt;;Od5AulDNtl>l zNdMd$cc)TiSHo_Xw zcp<8DuK8rk8toJ_%oYswEPm$gM)+33Fx93Zu}$r+OW61CzuT(G`sSi*Ksy<7-ztb|J!8wd)U$6 z{gJ>Ud@JZiwU#W$W;2!t21*5IfBm0NrPkzWgDQ{wItVvJlyv7KiF@WrpTHP=6)p6! zB_Ox|<&V5MC}p(fbG(pX-=K>6(oMM0G;}xcm5a>c#&xs}GsYSA|Nd*eM}kj7l%$jL zhP|k+dn`%2$%6IpJb#WkT$^ zM^?Zbo`{R_4mh#ts0)B14?XIBG+Xqo^nI4P60q<6AwRC|IODX9>?!Y#>b(2IxK6~* z+?7?ya@=H9eII_rPqOYP;r$m?|2Y_3u|Ukp1;f3-|IKeUe{xEt<#oWPR$R3kMfPslAFvRQnO7xG9ErhHA{C>Spueo>= zClL`rcpsMx*T74ZMtY5XE4|m;H+0|g@^uGCul>2J@f}&QgHRvB$0sM&cN(c>4?X0m!`Ljlat!=DJ<4cs*40>w z@}X^zhv!&>{Qg?=*lmGpky0Z{ZIc1B3IDxcZlAMksuPXqR(uzSRx>w3vI3R zxP1@a)(X74LnLwMMTbG_?+GWI%+YJEPJvZ*A%r_swOXdHWz;MOpK#ceWW$YefK!qL z{fy5PII0pyG2xoK-x(A`zSs$OS~#+vDouJ-0_g*Y8wr@iRY0zS&u+?ou6AgCejb4k z=i&m+rSm|=?k}kY7)d0{Fimq73cm&3tRM4{?3=?I!uEsL^V zsP~v8gKpPti<-?qjvj4Ro)yjs>#!wP`=Iie)Y~Md$k*}req=+STa$Km8~pY&quV2i zwzOGODe;`i)oy*COYki$*QC7+5rR&e*$pKT#-V+`4GT!5m$?8mqimg$!Lm}rYxS$& z$&%o}6vZh6X9pV|M_RR~C;H{Y^lS5pXkX0wWooeR?XbORcAHTD`iK^F^n{61?_sYpTg{$X08cZ;#jeKx{pg zGf9*D_Y=9NQPG8Eq}O4`1BOO~+;MAp+D-zoW;>J1#&J&2GYW#h}vlESvC<-uby=t){bFSE@F> zJzT_0CB}^3Ktm5n9fv%n!fnW6{Jo5Ari@ZDF+epnYOvIWvusElstXqiU4=kcn^EXWApT+>+rD|F4k`^`0t6aFZ9Q zNn@FX_8K94N$#_YTG~*#ltvB2%Oi`xwc3O;l8_qRcT6L4s6`}(HG{$M8Jozr56-Cl zLIGiq9cHcKztCVVIm^MsMGVbe0I%f)Z*&KmlmeaBR1=?P8Nobmg2It(0Fn=M?@~}! zwm5%6`ulE7BdJjDqy)D5HU&3!s0l6kTW|5<80$8!#wrJv&%xh8h5cN1uak)j?if}q zOq48DzMX}82{ktQm?mlG)TOtq>{x^6j$0R@(yh0Sh4=EDK7`j8>t{sIzJv>iheW?n zQ+oA1A|ketbkpOTiKYfp>N*X^^vQUh&+2gS%KgFXp29J^t(BQ-9X<8Hcz0mUx^BIu zmeQ?)=ki&VIv4^O3DH^~`*AZ(4Y%zHax8$r=Wv0-cihJc`*g$*k~wTH-RUrrx0514jUM`E=RO+T=P9WADnh-IgglVE{mGB63A-OubV7R zIqoF9QP>m0ry8Z-PKVW|83$4%o4ozu?5EyGpRgKUc;JXG$6_cLk`~&|iYH;P7^kdp53FEI zalQ;yG{!HPK#I)kt1nhJCU+nXV#=0E6!R6*xO2!TXa6ziiFnv$KlhsWO5Ra!@xEPbE$ruyL;2i?8~hvW78O!@@!z#0qzO279*BfKtGU9 zmz5v0yZk{`?e!c^QsJ9_Az>rzCHkoq>N4W@Q!{2tD-NCgU*>E8Y;PZGghFl+O5}ABRAHH z+h>I2309q9dv}{Go*Yphj+sMJ`YrmkzNqwlJia5jo1OO~<(Vd#PckqGs+JNjLUJ6F zl6FquuDqTPn@fm+K-i)`mR39gNR@HR*a87D^XDcHR3a-;vAPs%z&*;L09a2WyOT17 zuA29b*NxnsHnJ9)!>iSba{jH3CzKkNf7qxIT%)TNx+C+$Y|>$EYJ@q^fwGYHgwEIF zGxy}n+qJdR^nt~1tXW((3@P;+kXK9P1W*CVnbD`O{gK(7XXNhr$*-+{nvQDN*Z@g; zYWL*Hou9(4c51-hz=-sMT|J#Q>HyXvh3S#XS@! zIrTGvO|RV^(^kUTnz?KbXjWn#GC>T)x(~f}k(GEj$&XxDY(VgI(Ft^qvE**OXv&rf_oZvDzB*;bJc~B^n>$qI|kuRry zO<{RC63DnF(m_A*I%PgKFfjOI$BYA>)Ti%oOq>`*fc|8DmAkgkhdLdU^>m}m1+86b z&~$Q0Q>{dgXpN~N)7DC+SuC`6=IM=Y9U44+y@Skhz9oE4yh!9l?b8|Ry$P+2;V60y z9%cVW053ifnyit*3W|JT`)cw#vZX`HP?C-yP*&{-X!d6B68{eMjHhf7DZzL%99h}5 zLwbB|3U{4v%So|6kybl}ETSJ&5!^Qi%b%#78I63UCxhNP0udB~FNL~+_kPl}1RfB6 zGmgN~0nO^HOF1H@9Sspt+y>QzHqG@MTuR1mR0c1rEcc#{A70s&OdyB z5~O|B7YEaDox&nqUnQk(5~q9G>7+p}y40#n(iqgtuM-%LfJ@cZF##2@`imr%?ED)2 zA$??;LMozBMN{J#NT>rF8{n^?!HL<945EgKUG*NW8On_!RXCWYGER-ZBF>Jc4cYi! zJ-h;X%~TyNOHxIrr!UmlXp9LU5v?PCeGtD1HN22Wu1JyT+RS+A(c%}IOPm==WHw?d zyzDJ+rX41H^PJO5W(k5<$sdZY0aoNk%XUJBZV8sl0nw=9{#~HPsDjI)ywcqHO(T+` zww6hG+@@|p4E#n+ATZt;&M9&KOl&cHa3_Fy3d5j zT#v`toK#=B?SY${U-p)>Fxmj5>S@MZp8CG`;<}d2QFSNhO<7zIf=7x8H$CK|&qs=ob2=aCBB{vM8;Y#di&^p-*;^4k$9n)w?6VAea{MdcG zSaU_JCJgxVk6~L0eJMBCyEqSDtZ=fa9EZvF+r}S~DGhWO5^xBSS6fI!TOX4#(w7K8!6MbdJO$>&QH3Fkq^0wa;mW!HN~UvWX_2@vSpFOD_Z*w2IIbg9%Q>eDo^T*lI$=B_-2A}E35h7zHb6Yk)mKu z!|F)2toK(ZS?1L67H_i)WSQ&*UdsGYoNsr`3)Qr})Y%qr&s2=I$4Kt_RP|B&+`xHn zr6{=v@`strX%#4FhwOzd0E)s$9xC8u>5L2JB66}!stF;A`cm6N7i9SA1y++885Zxy zni}3LHC>>+)jz)vf{a)J1gW>adj_!*nJ+*V7h`_;8E>PhU54o6dCjL~TEiZe9u?Y5 z6l;|L-R?Ix>jM`@{;P5S{XKGnyM#_HZr68#qzwt8z=VX_&B*6FBN9%UuP5QZvCu3W z*#o}NE)R5Ab^Whp3u6UK>?vc#i>=z9@2k_U$>X(3Xxc{mwpGC~p;*)3bn_)8sy7g= z;?Afay6%@}(X0>5xo+LM^?wqD8IMR#OtUI+g3Z4R#~ZNn(N?_X&p4uAaJ^^7aC^7oYz7?MV8V}=qR&V>(oBhf#0h>498Yh?FoW?{vWE9 zFvi)Jn`EX6n4r=a+&{qeOH>hd%_TXCz&G(#@@N54nMN5K4+ot`u>ONwy5Vu*NnEXf z6_S&DA9LA9)9Xn2X84-2{4!zS4GRRk-Lc7fD(7X7%3qwu(CU4iYd~uYTKlwH0J_A;{h|2>Ab=1DV zh?Do@v!Na1mBXA^C?~f^!|sy+Y~T7W_VH~Mf{R{P3$AnreyI;s*$}KtM_MQy7>5Qvgig=p`{W^{b50eJ zWaRII`G+YbbAqJu0;`Th>*p@p5*NYUe>JUMJ0L7;+)q+%{F8MimNh6FC05mYkUf

WUrNa(%5K8wV)^1+`GY5DocW|h!<`834(oOn86 zO=XnHQEbMU>wP-8l+}P>2DpXGq@{oxh6yAlf8UajVz?-Sr(dL*^k_ib6{uHC*4u6c zT_(k97sJ8M%=2rIzDW6y!LJc)zi-4OOfHATG>oXk_{}@{Nqu(7H@WS6nFm`;8F{Vx z60NLFB~U+2{50aZ@mlPUUbj5k7KD27J%?tq=3JB?Yn_)>^s@73RcGrNVE#I> za5y(VEP&am+~B-`*x(C*3e|CnzD>8FWKsn+y=WtK^7Il3Il@--JU=ua6L?tZZ$B{u zM*zL&V$+_|(7gXsEs*H~5)#n*@-IE(#X^%|d>g|Im*0byCcgUQbm^FkGDdPBO}MQ# zmnIWLsO2ZaussB8t!J_uwpHO->phC~;LO&p^BMnH-ZCSoC0y3%v(kbi#N_lH0bbyg z>ehIrd>NTm$%O5Xrs+aGtn$X^SaH3q2t_Q?$ernO$u!%zlIha9Y>kY*0TBEEKfaj6 z)b}lm%?cS#hMYQ8{#?PSBck3ioXmah_(|Q60JNbGRz*!iY|)-D%J8uoZ8}R;NfEX4)ee1JV~%eR$32NK>@|K z&`m0c(}>sUTC1Y|mCN@{`bJ(GwH$@;MrH4?Wbjh@4zc>Hpxs)T4q@I3gEH^3&y-lI zGY-3lTS-G)htBcHuQ*m(+LnLceE%CciAo;c0_Fsfy9dY}6G$}!b{bB{Qu~+ioZ@k} ztRtSJG6r=jlro5Fw)JS9UBJ5^Tsn6+6}zjGWJAfU#nXM=`TnJ}xgt4LfldJd47@Jr znoL^obV8AadCaFaxbv0Y*Mx&F$=-wA_Mh?^%c?`(ZhiSAA|%`(uU@6W|IutHLEkI5 zBnl@?8IGftzz`!aVrTO5{D%;qRdPDwJM^#ZDMPsaGDqli2ti%b$mcZ7l5K9{$2B2! z25$V|)l~5j5Ndu-E5#>zDip+Nr}NQW4HA`qJ#)?j%U-gpYjG8{O^&|PUe!E3xS`-t z42P^s_-^aFG{``> z@-C(Q?qKcUwqZmNbEHYlz}npLecw`;`ZS#nRjZmm0qcly^V*sl`m}3z;9NWJZHok) zQJ@R!fw*WT-3)!}yJpGl$TS$}dI}R3YAKwLDpcB4g|(%Equdlbk;CI0nM2b>Kb+f; z%XLWOW{9Wk#cNIfJ1?`|n-n6y$S{=~x;MBYH?mcTS&XI}8@A#nOdi&_9Y44q!cSJ) zw(>D1rwrc3egumrg*JI+z)^`%(26@Ky}TUHM7Mb8)8u-&vER=(5D5C)Y_=US8kzDj zk@druZ}jMaC>&f!+bkvt1z(*G4XYy~i<&KFb)BDO0|J?c(`W z&T<13omgtOet`iHbpx9dbE6u8<|RThSZiGr55=OSaN=!`O%(SQYN+cz(M5x81o3cn z4QyY5jTMUwKYgi5R|g|n-m9F^Hfz58ChTe(-Mt@*_d*7N|Hzcz|J?KAp%g0AAzdN$ zI;v7@V}P|@_(tn+qep4m`}?h1I1exS1b=SCRcaodO(zvA0kb}t!X(+{!S{+X5}XF8 zWx}TvU~SM!1Ho#MGI4=^blM$Z&LUMk`#!Ors+AOlSMO)G7>}&bpw9Sov#Wm7>;Ahu z3HC1tXtHWQ24UG^V5Er1OG|1p+~gC0X25knHilh5rs#mF=<#v-$kRLqqmq8MqHe5( zMD+mO>aPbcbw-(pVK9{oT$FC8xvVjT3@+MEE8UTC)_)$!VgGy`GL>;xI=F>5k;-oQvn|IbKQf?+(^c&^;SP=~hTEXrifXKafH z?NvtC)i~zGtB@9O#&TBlq%<}{JeN7+hjl?2hjc_n#uJq?%Z_E@(kZX0&zQCNO>dpl z) zm%rM6K7dp}+*TxP@neP`vTF{kdVnCo*Oc*(-3ku3y2EC3;q6cD(t=CHT`nr3Ho%mA zT5H|@I-qs_r*pvkywNZHdqGDhAUTqLGL^4KUZPq(RyX~SfYz?%G5c(+0cg6Idog*( z=|i7j27=LFl$s(W+x8T{F-Ey9zbqd0NMI9(J!u#ME$%1( zPro1l;Wu$dkp+;}>Lck7bnu=^lEb3X+JWkaM67RpknhDQmkh=w4}g0GWJ-WbX8qaL z5>jdDKG_SbtGsz_YDR{Roixs`31LqY_45?YvR-b$N|V zUit<>ofP*abI76a11Qb0n^rUU2fRzb&T@68!6q))r+*N{mKN&}WZm8Dc{mEx#Ir>jA@;q%P2)Un03 z=98MK&Rz9)^E{DQq&npM%!8!&3)g@M7Gn!ImR?f1d!RBiX~y`;{)-2lM3j(OHe@NX zbVEWl?PrVD_oF&wagj4+UU5s(FIn75@ZrdG-&UKSX?NXa4DJM!K)1wwHm=WW3bm!b zjTcQd7iT3bg9Ix2dklxvV8%WL<^+2cdN)&L*W-&bI=4)Z$f+Yjk%# z;DKnT{qGB9+p(FSQC_ogJxPthBzk@;4Gqn`4qo2G$;VGb-n|>qhK9xn%@Tu8927;l z;B&$)OB<1QsFdPM)BHi}|TDjf^V}~Dn4YVgnv73!|ofE34e&kAkm=OLWi$%Ca zs&oOF($h2a$zPIVVT!ww4SZ^E%q#xQH5~upt#3& z1KQAb7j@eRueUt5qUq+r`T-H`KZ}h~bv{ic*5o}rsT!+pXSNC`+E6l$n0VlPAvD<( zc>oe~YV_-fa7c;FsK$#nlK&j{3O_;d;?xQ8?&tof0TpDYOzAc?T-4nA&G_)pwg+;- zcj4S4u{T#Q_?TCKXQ2i>tuj6^U8Upa=Y8EhstlAHt6VI_J5D&GSCJP~0Z*Iu51Y(< z-UW`Pqr3T4sPsbT3b{uu=?AcT@3wzG^1DnY29lQ;KO_Hj5?RTD(J#GffDLLhGIIP} zYJ(0;RlvRB8GJ>ezpm(P^IJV4A{|~E18?=AJ|)%b_(#`VB0K1upQtv;SLtt$f3|gk zQ`y*bSA#;1XpaxpNjb-&W0J%Spv0w%ROgHE7`lt@>8a3KGyW7nXXqRXpax>aj{+5vIsiHi6xDontDSUX6kq!q5a zYp=U42AYCLb!-xKx-XPIhRr<@hdf_brTSyX1avtil7PEUiYWaDS!B6M>LEzJW}r63 zr8^BB%bbVxH+`J4|E31EgeTqt!4>ol;*Z9v&KKnoOx{>ZB>WjkbDtH_Bx=sic~PNf zBkuB8&}5!u2?%2|C@87QO_YreA}B~~ zTp4+IVgz$EasIiTe_-tQ9Gu(VadmLZj5w0u?%|am7H`Aa?#UAZ(w+ z{g)o8jU6;HICzX`Oe&PEKM;!#05IgFXYZzw3mXKe?uhgFMdz2|IA^g_y6++DP($5)WMrY5xOUP$a%9qEP$t;_SVk? zbynS%l-I9C|J#QAUsczO6{Hj9WzlHc|9!9a2c? zWqX#rX*NS7If}uburTfb>|ti%cOc;Gq4p4vr3XE~D3I!kZ}I0OZYUNZ=AiCyx2}z^FaC|k_>D$w^1KYW9|PAD`13jFNQ7HgF1|@p zIre*ne}k>}caK1HN(rC^vzzO3tJ69I>g;s*8vyXo=cWuaM*bJQIZRR5k;=sJYC`;T zp2q#`syp-e^wVEJBPAPIGZJ?nK7O8A7vwNk3rvK6!!910T*yhM7_#~52?5I8&|PPy zF>I4hO=d!jKgEQEgpS44KY#wL6s7+!ki~=Lqd?A{coVU@16#LOA_Ikwr0)V64fCq@ z%bj~E$2D&G8n=;uV-tR}s)Ny9w!!!}isT|dk=%L3Hi^EZq~d`SSl{`ZjbH zEYc`QyfO6;J1lhhiqu|KZ3=AzT75m^0DwhdjT^=MBLD7 z1S4U)9p(cZCgNMYuK`qsv9U-9tbX_2rl;zq@pDQ7>j(dWFsz?NC3|>Hvzw{o+h6K^ z1jOjCj+cKKFd4l{fVp-PD@xf?{rA(br~4ep<-SSn&b;Rbl{?X8vGsKff3>O*?Y@2QvkuX z>D9l2l#h-P1Bp}(SRBH;W&_H-w%jsvIPJEnkKm*I2eEz4)l&yoJIaIy!L=e@0Lp*b zZkRje?UEmgb~2ras*dWbfJIl&a6++^STYkhM&aZn<;2SNM&xD&3CADOBr&RtPYo?xV`Dx@dulkn_Xnd+0t z3)(G=HT9NKWm^hyu-_P#N*b>7kb40daA5%aXV}{QmQ6dLSMMwVgzym$y}PP7?ws{V z|B^L8Lg^_7?FUX&ooJC{I3PHBJ8ov^liS=GqxYUCdlt>u0e14VL`r$@ICbR!fJCM~ zbk?cJp5VG?azsda?&W*@?qXx{j2)IdUU|Ag^HG65B3KbZ8L5m}bPt!!Ch`u)d-fKYi*U!ln{wt($zN_lk~9%kLXfmN&yn->#^TlJ924xjg^^RSYFwI^q0F??Z~R8NdXw+T;*!S}UOSI0e~Sn0Xk;Z) zIQzYHKxc#g#i-6_9el*a;U@X`;`^uB1jylK(p{Ui=rWqu2e7J2$#|V--Ebm&JdTd_ zg9mP+%TZeFt&OmXp-~P-=qp9&gJ}oxw=?sMSlSh{b=K$eV#*~O(R)EXw%Ib#XSnzh_UjXyZTqncRjX#Jk<$-i=>zU`&5!3< zecFVNxx#6>HV6Ax%VEb4^Gj~wuFh0jsmgTr^zg{o_Eo@*bl+ccw)TE?h|o)6I2!U+ zQG6}QoNKQaj6@Rojkiy?i}YSikH4wP7DOV+n;Yz$54G;IM$AA&<-ND%wsCWwz0QzS zvwc5VQCXT;J~-j)BWIu6>k~}S zV}n`;MVs8ic~OB;3Q((Gb~Tla#iGh~wrk@P8q>Y^hq0WbRF&g#(9My)HaKTlxeU73 zV7NN|T3B94AS;5zOQRVooX<~k^5GUaa%~@k!RH2w`?GiPC4ro&sSQsAjGUAeMLw?Z zlh)3j&x$LY|BBmujZugT7(L61akRsVC0C4EONw?pi!)a5DZp_~^%^tr^9E*+>F+$y z=TF9`%U&GEuI9JIC#;R=^045uz#6n{w^!x0R^!N? zGF@nx;{tseJ{R$r4|N#;7Syc(9J90XIdlB>hGj0hLdsa-y7u64DGm^-`D;kDO&OWF zYUMq6lIISkokW=C54U{5?HEYCWfRB;ksiK1QKp~c70RKpZtqfwiY}$uMJ`+kTUMlT zWh=6{uw-gApcDCcEbV6;PD-3q>8Lz%3J0*#@2BkTlcfA5;%R^!t$2@n}knC#Dh8q48kmxOq|@-psk?T&DmAVS!fYZeThqU z5PE1_H@=isKYM{g9&V{bZlLOxYb&Z9T~{i47E<$lduc{+e0osdWjDW*!PAA$%x~+* zaT_{ivdkn;0qxCx4>!fsw)?8G-^*J2H%`mwsBf^>hx(djVmoP$@7kci$UaioSkwnU z%hfhV5WD2Lc%WOPu$7^x(><{R@WC0HlnsG55fAi+SiQ@nM4~xGXDnvT170L}T_Bi; zq>P*?&C18Kg27aOEahi|BHiBxSM)?&6VvtUx*XD4*6WPHTM_6A@;pDn``Zp*pW5a{ zpCOI4doHwT%VK2JYLH)#rgJ+{F#m(>^MK&$s!{HZ7uz)xzRiwSbu}Dq=SupFv!zt< z)S3f)Bu7hvM8RTeprEn%FX3}CCoXtX*Ln6zWHbggh$AFC`eb~z69B5_kQ%VTy{Eq%gWp|UPxbjJ zk{*qsGvaTGB~v|tya-I*R2tfNr#l(Lo#^L|`b%KEBc_LQLp}76-*olLyoWQnW+%eG z;@Ld>dbsOqwU8U$?_c25+yzM%1;4Wk?WTn+}I1 zZ0Fie%mIm&xiC@B-laT&l8^R2^q|ucy7R|;FlBMy%a`v@7z0HItoVt1!*27DWDp{r z(}*IG%{GTi2bgoDOw@`c3UdUZ-H|5ElS`4y$OB>T*F5US!c?t_?A5=9EYn~0LgzTX zq`EI}HCc{jJByO3ClIwKr}eOri97Ed*r^Eq74meh7ww;UH@P6EWPmi@PRs>y#e?s_ zMopp1-I%-~l&jWbo~_+14D+7(RoNk}!_(fMwiZd<>ULK7cr3w zgMksTZwKNOnCs-(-1((H(F-mhij@sD`qTieR1?pEAtICFVC<=<<~Qn zSKp#tFL#xNG+@kA=863#Eaj0Sl|?dmPvJ9|^3fTHONUJ`n2gWeh#l zSiZh&{|u}4VC@m&X^BrE-&Aw56B?15Czf#rKX0aDylPagoq3M%a>Ppm)&Ss&z<|D| zaw|}kx#2nqLY9%$N+KFQ!yHAj_HowUwJN^QRC%rMdp=7od!Rrqc%NTM0&GAk9wwZM zl(n%Q<&fnD1vKg(7l@eD?LM=NshG@b7}r)Yfpi#q+fOE>@3ZgJAAjC3IBc?eHFS5n zSZE=M&lu^q{ti5Pye-}!OpIcUKe`73szb2!@UO3i)_ipwg;1}}@GJO1c`ZtZQVl&+ zDBrpQ6PteHIjU>{bT!Pc75XvRx`UJ7yDqf;JCmC{k8SfS)1f%hENNGwW7oq2)!HMMKqeq9Lo?ynS+e0t4;*0H@`1wsg6n*DSs-p?5C|Op z)79d={+B;^9gO-e>Uq&`+ih4Jz38a4zZ z7>g&ob9w8{mbtZn$>AsVmMg?w*_MiLz^YZoBV3;A64-@^{`bUleFfZxrrxbj zH4e+Q>^>GvprI~}U`NQK8i}J&{F*4Tr%co3i^@YregqW)&A%`d3}7~r@34OD!pxo`sSuSt5727W)g z!JU#oFHX!e!hJ>D`5w(-v=8V&F}>E<)W4m3l*nOx&zA)znm(dZZp@$QFi@Nq zx_Q5_Y$m9A+z#=}EcSElK=NzOD13v)t!Rgacc4Dc))vU&&Sh5Gz}Sd1;Z(|71tY3o zh#5KdTGa=~Kd3*1P@R(?%qPtg-NTiKGNb zfuf7+2GFtw2)Hp;L`x+>WH&2?7Fgb9ETh}db36iA1qlzc!1SJXi2gS&9b}7pVft*t zMV;NzoEoQJXhgl}70OY`k(p5Q0lSg^vq|#P9Z|F0eibcYljEe%6dL@WpT;JFicHT=@|(pQj=*EjZ`38R+b zT_spoUG>7%{CSSo&^m@wmbe#8I^J6YFe#~`6c#TbzQ&=Jw;tQR@je&HWw|X`rOqII zC0AV^QM<)<1Te{dR84i^5I@tm`Q#Ac>Rt#$8kuVg`!#b^&UB>2gSKJbY11NcHsVAD z7keAc=olDyo}jEIN2|Oed!{;k?UmKkSG)-tzSf;@%`>cN1A@r(6zWee;qY-|H`k9} z``XoJGJ;y0o$ZPhi8rDC93NoUH#mpGM2bJW!uRFn98e0H3~}#cYXDmr)d5Gh8mG$+ z5llPy{#@(gg9bNl<1hGREHG>%k9SPV6mlqYSD$W9Y;5U2c!xp&WZ(29Ur&9T{&}*I zdbW#r0g{V|3Inju^{T*x;)pPPVVq%2ahU7dA@@@;F@u`nhs#I^zk#&t77l1e!esGP zCwlfvwB{rt0@gY=RrY+c+gn!0MgR5Z5YGdbwBZ87KE)C84jJm4Q#X?-Zcc{swJBM< zNV46NJlm@j7;KkPm3hU66y*z{i#E ztAUUGG8N`IQMAJYEKb{c?#TDQN67_V^+%ZtR zZi6q^p>j!l2&RneXq)>(UhJLXWk%e7tvuUJ zkdYa!@$CG?<`n_1^ho(xGt5tnSDF`M1fJ2es>EQKgQJN?lpS9!ba77jv@aHxO4?-@ z@Goy`c$Ch)qhEGdNFQgRUng~h4m=xpAxf{x_L#;y^AU7}vG!Q36|N6YNsb@kobkTG zBt6IuN3L}nDU7;yUGpN~tF@*Uwgn%}D4BL4vFLZq&rB$XR3Zvhl~oo2Y`aguS$5NK zd8XyJBb;e!zh%dGU9hO=^YXh;r6$Mih9zu~8~F8Uk5EXtG1(yQxz;lLG=j(|^)F;S zM@Y-b_?%~zbz#fF+wp0=z5XIKB(-$;o!KKHbYR46OsY}WwE7q@$^Eu<3+PtvehM^w^(zfwl&&VDiHzJf)9B> zIpzJHyhJ}>pYtFW_1ym@2KN>yH#9unjh3C*spFDvrx7>Q{h{neUltF^!!tUMp#A}K z%Bjlk5yn=iHBLoOWVO1Z_Uhs_6jy-Fi?i`*Nx@RResKPf8c(U?I1Z6s5Xn=*wrNK` zRkzA0^Q?LCp!xeXb=QMC^u*T8jazRTgf~5F$pgnp-@NqqW`{*k@xIgSxkv0?M318g z68ECNh-7+)@}qtF)bitFF!)dL!r$l9PA2#k!5|LTjR*S)E>Pr7UX_M}Q(Z|&*EW#1 zu7b6b zqTyN*gu*ud9(4ZEO&=MTe!sIH=8G`y9Co?D$V_-21Gf1O_mh6p}k`GNPyyvhJ%7a@O)qf(B?*LF4QXqueFtOw9 zRJ=(9%Syj`)eAs!6LIMPPVFK4ygR6?bA)AOB7NLWhmw2N5P>kr#QXA;4brz^zlYfiRi?69i@e8&#zS1p@Xf6TgDp{P)xN1V~cl3GHi8m zq{Ck!Smj*cU$(vX^zPkXO_ib!)$onpEj}iX$FPNWQPBNXpr{7Gwf!$&PGt%^?7GqJ zB9HwP$wWRcNVYJx!hjBlOMlaYN#DMG%j+@$;xfarZ~Q0j-08BHt@crmEb~4~>^Py2R=l-v(Da<2R6WLOeVq zcXIrv_Twwe<~|RzoD4I)(uqNO@;*(<{UM?FXz9)HEWxrZC7D#*JSbIwZ~_wTCC?J> z8reD$8S`Mn9A+X5SJZGzE&z&MyyG*1-C;erv^d&F@Z4UHb_|((?j}b=%0Kg0xT*L1U!NPs zM&mvbk{TU&(dk7H+Uf0|9x?RXq;9Q#l+2MMbGE8t&bINwuu@;^xVMaR$;Te8yihm8 zZn(zsHY4Y+U7?PoC^Fq~)+}-tu?5NDwenTI{$;x@YwGe$F|~(`pBj!|iFauBW|V?d z4sF=+uz~AfCD8#IQONQ{fzz<0irJ9$u=)Cqeh;&fNWx2gPNXiYvGZX1_2e&hm8QFd z7G)kx=OnL=Ntg#!^kHQ=nspk5Eu{WEllmkvO%~TXypXPc=HSASf2!uE&A>cUd#+S( zEkG*HCfMwE@hNn}KEDB~08i=V-A*}C`Mrob`x8y${uM4>+4z}!CKMqsx}qC3T3)U-_S7Gex;m^?I!a?_W5 zc&))EcALU{b*mgRwpgc}(lOxMs5KX?`$OJ@o0rwLz3%i0hhlRESz8Z%)T;t&^vCFRN@_dQNLr(ae!6Zdf|TvCDEx{DUVlr=DC=(Y@(VuQ?6#vB3E3_|NV%tro5TIw z&jA}Pr-_S@f_VJGSsYS3>NQv^X{y4!$4}G$p0%Nn&ur9;^Msgc^ea$~Z5=JDY*T-0 zB&D$hE_7yWaqcp96;gSVks7a(dmu_io9jo&hYJ3%<=tCZehCsX@8oLu3L^=1;mx1# ze}>Q!A1fOG+gNpD>dczL37W>fZR`<2wv?z&uzYy4$5N2q1XJ`J3)&ld#fGzgDh9o9 zGIf5!F9&Z#7OecdONTO28JIQYV8bxL$V@|M!-1GhHFf(a^+)_Ij6GiZ7xw1~(P`HN zIa63z?8t;Alvl4BwiD9+F)`|%v==K-iLBEj zqI}&MRV??6>gUSc7g6DMEr@2c?*FSa>I!`sd~J`1&VF!xbm>6sjST5XolN7ZVGvz3!0z}M9r{M~Gt=gYqhkL3f96wB_EqDS>~Qxt9)s6V=W z;biE9DT&}&*BMKL95K?(Ates7LuJUP8q^WTDtSZ2qi_4ZJp62gs+SnWX+CMrW$QdH zx73E4b>}&9JCK&_ez8mshSMZ!H+Zz{0j@INew16WXJGL{))C7{mzKi3G!9-3s3_F6zSQxWcGNO5j1hiPRL+P9Aux z#P68<7!?X_PxvW57f+cR@m72_P>F$Ob9OyRIGQ+amRb-wB7=)=UJn%8(+3LS;UT4^ zr5PW>@0TYR0>alMK>F>dokZ!Io8d5Tx=!?~ewO7^6l<8P`B}aJZhT8C_5IzLK`k|u z{l7it)^FHwRYz|Xc7&5kq^qR2?)4{$>s7!{-ZNTBaxUnQrioe+#tM1D;aXtpJ@-Pf&w z#c!_q%~K3(&ByPGU6=H^G9B+y1RNdOiuE!622)&sg+cU@+;J4)pkOFDjb>Gg(*7D+ z?2D6y^gVt!-dbSQ8PBqNzBf-@lPRuNag=j*B735tyUkZ_ztrbP)t>BJ6GENj);hH3 zUJaEekeu>0j1L`am>HUxn9>{NZ6rQ+yRK630D587pfAm}`ur9HRJ-mdXNiCzMG1}y zF`=(9qLugj5r9&M3EKhLN+1(MW9NWMG5t;wZ@zoekY-Gr@Lvg8($X02g#SZAmL{;f zHir8Dw}h1QenPyamqjfu6}u!&e09Q=;4?&En5u&xg(4uqbB8i{YWA zd>Xa{0hv)ddx` z-^2}=-KvInmOnAPWndibr#2IHy5{BITTAWRuCS-E0dMZA6tV+?EmysELZz(Wz<7w} zcQMNm%QVUGWE@XNMYbop@Sc~bL{%ubAQ%_j$&-a|^TWKQUr)t7X};w>fLF|NYhZpJ z`OtQV9P{RFpI(C*f&?;ip8R?&UDtXZHmqWVsV=_K+FW_SLiyoDx$!ivfKbf(+WrRD zKE_sgURPNoUi`;f?({F2ArJuUkc)wbec3&b12I*s2-GXmFyo-&%8O@tuOgUUGoQKZ zzp0FIKxOVmtwtVu+p&bg1J;7(zUosRsYcUSaBX6^939Q)-aJB*_ZM&K>ot(ds)qzueMPglQVxCH}u+i>U3+p9{ zF=O-SP8LRE??Ebr)D9J~kH6kg$R9XGB*q2b4C-GfclQ2Ke$w-D0ye+=os@ybDNh>f z^Qb|&e-oUt&9+DSm}SmN^pMv}sPTev#&PRs5!>Ji8*PzxnQUDk&?}z!B15yO(7D*x z8#{{`G>q&s-?24Iir>q@%%C^DBv2jqjDG<@1ozj2a1{dgk$^Yj=Gxaqg;N4AP3n~P zE(MTw=3k15@N9R!by9!_FD`E8?$<(H8xkRo0S16MjZ#zIET?cN6;68&#SUm$#zc+1 z4&AY-Sf&M)l~2SXijFo`Yex(co=)-gJIg;@bp>_sngw(xnS2LmxfPgsp)d8UU(JIG z=fy^UY@@e7e*6HLJHp+FBsHs{VMA?t`HYdLhZ9*EJ)Gq6rP70^8*ZrJ${tNLW#*6N z^mn;FSslq@GNxRZVY!APWi?Kx++Hi)U^L=zL)cku&K2tCcu?Xm>sfbdWH6rrur*8n8hsdHhzp%9Ib;)2oQD{XMO z!`*<^#qXK1bl(u}6sl#J@vV4-^jc$r)V^)9F16|IPDYzoR&f=bqkxL*$zgprRcVmw z`l-(1X(FX)#Q93~DCwHiQa6)!^#b9+f$FZ^xyZ4#hzY&oYzGLt>kznhj=hypqI05QqxdZl4BL2 z!}Fn4m*kat{TJk2fm{`eT(^c3T@E`zmnnp;qukd{-7vTqi1v~yRt`o?ViiBK?>gh; z7W`? zwjYl!dW?hO(-$2y(CDgrN{nMIe$dV^w2(;wUu^Xe&WYo|Mas%$jOMXU2m#8NY{+No zy;IlvlO>8qdU<$WUCWOI@BpHQv$fD7O=$BEQ6s#ks#n2$(s^?bKKKH%WLymph>Lu# zOHV6dsoyZ=s!rLY5pcDFF~1nLqrPMXwm~v@2G(j0>nBEpSqF6dN0y6;vIlmi-LndZ zcHf+BCU1(We6)eu;k#0G0KY5~`qXA)2XOP$n3n-c;IEjs|%kzde5 zr6LTFmf6fc5o1jyyDoZ(;k!mwq+~|;n96sRk^k*Qv`ew)z{VNoni9h+d<#}Frva>| z;4}$1e1Asi6VNZdP1BnhrEgvE!nk75Z0R2tEh8)@%EgT@r<~en*{3-=U4s4 zj-d`KU~2QV7rRIz8bJBwrMJcHGK3m>9UQB`5${}=QM6Za;WFtbT1}W54N4>N6#=hx z7mXkb`n9HF>!3t%BCGzQ)~ZF!Ct_5pboIo;(#hX=LdThr`}^rVrPYu4t!qDYDpx6>94LWCOvhaXC=4o*b#Cfr9@qBF`yq^3<@1CJt!`uDI68q zmK7wV>UE1vSo)(ROo~wxS+h7U-w$j1HKHtGo4WRT0%yuEJ!GF`A0+C)SHt_+Cp!;j z+gWxN^9Kn|_!2@3)h)lCz1wxnE;wRo)BT-UBvcQiE&p+7T*5bhub_;-Q&xHZ1FHdP zvwsiCZ)Dm3BCY+?+$)FAWl${#=RXdYtB5okPzV0!_b#Ddsjq>j`6n&#KK-GZL5A*t z%7RBAq4{ygpRg@4Q0RQudm1 z^fcHR03e>tg9kH~9}IdnRwaJWi@ZH`s&%d6_*Jsf`U%#ak^wZjKQZ0fx%S5!cA1Jx z;0EYD_2=`*rY}4hAfX!1*0W`83p00S`Z)sAg!Mn4tj=@`2-zNg#V{HL!+U$i?(u>1 zXk)m1s1oH_Yk$5yeM{&(5!#b$jdXW6J>y>=liFna<)kmd*YTX`P+UXu>EQ4?zpG@? z(82j_8|aTPa)H@qx3lButD5q7VI!lKleRUzr`@iCvE$NL#o~T+V%^GDpf( z7j-lqmZf`VCk$ltSqld@A|FIr9dJ)Q1DX9RZi1yh1K|Pzr`Vz+Pk`(R~izkl*>-O?uC&(8+>)&cd1pg)mWBA zJQhZx_kHhGs*p`HBI<2=fZAd4QP^+nSP5ZlI~}3N$j5(j%M+K*f;7`{P`BGGs1n0IWTm5n#N6MHIY zGSfrmP9@37Jkohq@frNp{j9|`Df3w*fT0;=QeCAgYuQ3yZGAs$Vw^4LQojO>`Dx+u z$Ph!VyV}U}G~V30;)Pl54tNr6XJ10dK-Mz7xZ5}OfwSqq7K4rS3=IvR{Ag_Sp;J_y z7QpztICowdoJD%a`&1vKE6d3;*|~K6gB`<)irfqQG-w~orKWPbJu1`F2D6eJH9rjy za6Y_GfOf9?%G0+GtgW<+7Y&ra<@Hhy-mL`M%A7A5@ zZZ+=p+-M1bIy;cu4ar*wNnw6*clP0od!70@;F--Vtn>@dmtl;RCvkOy#PX}M8Bq1{ z9~@iKtNYPZT)(hvQ6`%*(hWBS7j7cQC3xpR3Cyz#P>sO6gY4;sXZZ0*KiEA(@GL%3 zLI=FIhYuglGz0A2i%Zs9Qa&Y3^L=C2l{&vGJ!@XaL|JX&&li8yP($*l_IHh!9x8pP z7Z7ksG<aFpETj6Zhz%%r4$VztXy74_nGg3^XV05TM0l=q6rHTD^hDx<3YREQC)2+q zP1+JnX)NC)TFUEU#QtZ@m=0-iKF=RcUX>Iq4VIQ>GI)Hb=?3}wJ0CJXEu&#B;T+k- z^YR~`#(`;jCUjYBv$yRqz8bKo2QI+vJroF9_EQ)Q8A>cG)qZB{rxQeU(?X4;%td%> zMCMMUXQ`q7&zAjqhFG|Cf^-TgC0Qp?Yjusm=WLe$);L{>SIf&O;^Adj&$w0DCkzfr zsKsj3;aY)rPe)A-frW(|HjOG!@^`ywSoj;Q^s_eY}KdXcK>)lUcp1vv8>BhHB z8R|`Gs11G+A9=V=7pXelV5+p+5+cAC@>euPp0uLFP~~XxuG>d#!oSO({>V803%CTJ z18OQNU!*D&Ii(i}35WciaA1jtfXn|LD1rU>9f*Oicns&MFtEjCUK{!&-~u8E3Q?CT zFrfZ}la7v#fZ@AaeoH;*t}Gzp=N_@8Re zscL^PQP2L=S^r1ex#SNlOYMKC;1uZr(W&e0wpUSaK_F;-sV(#0rak_{LK6R9qX^!r zfq!F%)c?;%rvKsl2oe4^75@}hZ8lRg`q(wG6#-j{Mq4;fp;iP}zlthiBO=^7{2v z253?~IM1+wQz+!IgqhiDt;;TDVUphhl6CJvt-n9oF7do7_=?Y!ZqmQf#lL3^bO&Tr zEOXWH+-pHx`ya(^5QxiR2>ix#=dHg6_|h{n;zn1&VdNhLbjUo|HXgza9%tgH4?#it zI{6^TmipkI$D(PtC-b{^)ZP~KwnEfIAh(ggB9?PsPRljqRrMVBfz;upkwQ&?|D()# zQy8C|tn>Qy?Olx|5L#;n_|K~aN}6}_i7l4L|EK*|3OeXDy5AAkf&dSQ2+WXjuA-@_ zsdPj<4hacdJUA!_^S5p_hChPum)}YC1v@h_m!-u;0G=PC7WYL{mVJq}VP3Avtfl+r zC(apm+x;>*eajATLjKS1KLh{7TfyamaF_V>QeynqXE@yV(`m6K9l#xL1SII}tx~<4QkBP~vJpC?#Y;?phu9$^(9WercRTFl9wQ zgrKNU&Y|i=LN4L*z+qDX%k9K7m@4T^U7v~5rfjt1t)XIv4UIXcdTE0?se3!^M$m-o zUHi8i#;P!x)-jGiiP)7DxrrzS=gCMQaTlfY!gPRgku4nXBUFDS!i{++SqnIE{?fbG zSQxRHRaswe=Oog6eSVMNW4!aYgjp0ZWw-rn`zNLeLgTpexIVgWZ6m0cjyphS?B6jK zBDAuys;Q|F4I*?uJ7!b5IS&DPIh;$#%`m3{nnlg3xfNqjvZMU=K>C?joNdW3o=CNX z9Znscpgdg1Q7d-DTXjpzfu^TPy}Z3E6!W8hEE}#v1CXpIeu_LkZo+gXV$%96@SWSG zi5ZtB7bIiZ zh9MK>W@W1bup_T1TL;oNxE5BMj|3`U8AgtI>=q+$+cL+Eu$*!WxE-buHe*s|>=ue} zJH+jKFf7vYwe;rfN7-w&eEh{2!$nvLmWs7T6^{vf2iNK@WN+td1m~{Aue{EWo%B5x zkEU@E;1{rN?05+4Dwc>nsdt>Ha+02`xR*Clq}kB=0i6apkzLo|=Ca|CQJ!#$OKF}5 zbS?ffltxC2N4ra=wgA@2s)R9r8w_OCB9dQkjYQsOg!^8lE3DmuYWwV`?a$c&8KQ3H z+APxJq;@yS_&~Uhe_*jD+@-x5$!K-ULL#cniQGEgafaf zx=jn@oET-ih#NT)V^ySHFsGq+3b~z(TrE(m{@hxICynjF|qeklr&(( zTt&w&c7J7x(Vt*;_i%ywffRntvE`;(gc~XDmapM$>$rhzNVT*Hj5kth?ieh)48Sas z5zX1KSA>YAd5(TZ>~aB#c2aJK$9zJx_^+leF5v{frV`_ZPVm;0d`5VaBrRjsIXv)J z+Vj3)%^@{hT_-edJ7!xx^T$B8eQPRHtIg2iqdC1aV z#`TR1eJEjLPQz{dl{)s&n)EI7nZEExiMpxVvYxry`%_hL>#1>mg7P*Uj1hA7lYzn@ zJ&)K#Y2&awCHd4W(Q+biqwEuDE9SU9#pj7jN?HfqoCYRy5Rs}`X_c0i*2Nw!1x`u6 z_Q-RV?fS;_8qww(b8dQR9U+<|vME+>JK;0>s;2sczP3=s(t&4dC%E>?wLhR0FCU-u z=T-fz@Xu*l7%h`8N+_Nf`&vD_gMM=vmyEK$m)ybq%khvL%UP@*- zwMcahv^&>(x;pl+#(^vIJL*?wt4rI2Oj;%wtVvNca%6n24glpcm@T&nK)=2i1j{M> zX#oAZxVQ*dEYg*bYP8E0x`b3Fd#@eOQ!X_;n|{q1{e81JI|rGRY53P6~mn|PsFr7NMTf;!A_QtW&8Pzjg52#%A z3vah)^ukoKuI@A@l2HeT#a(1nbArMNtea@3)Wxv_OjPm5`HuKM0-WK@?n;43;o z(V~#^9Zx$EbyJHf)CBD+VmVz^u=us1mLc}bmfiTLW%_eSTTOo=YImJ6yFx=Osg4hk z+j|XULxKFU5XqPR= z`?uxDWal+C=<0WgAH)}>of@U0^!2u_H8J;x)#jcK7y?!LiF==qC1ytA);X(ja186e ztFc5ZG)%A+QSSG&boLm_2xdo-RTU)gzgSQlED&LM<-t1Br`$%fxn6gWHT01{uDE$P zSAtW58twJeW-C zFBh+~0QsWf9SmcJLDBOpx-7|?aUzV#z&8ysp|*iux-G@K9*ugR;HND zG4tJ9%boJ8pwuhJIAj*8_T61S)PY`hUORv2xqfG{w9E_#2TetKa(BU` z-fpzRDl=s5HvSW4-@=fb*u81BBQvf~#GjT3h;7u$o>L8K)C(;&Ke3MEHq|>>;>b+7 zldyplz_U(9ifKRCa%2dba$S}PhPIrX{2GmwGc91ge}}f$A4t-$;#Bcmx-wHAa5@*S z-e|Dg=ukTDnZw-AuQcIt8%oX9Ke@X(p5tU|8S!%mV`Lq6p61+xuHWd5G2F4TQx-UE z@uJJ-=ixP(d?Lc|4!1gDOiN0#v5$XMz@=W8Sp~*;bXO#`29rLok!z$sc2iW!7s+== z;zHpbwH`(`56y0K+b+eBN_0mc%s!@TG&2vl)u6HIwQ!iJf+TK7ug~xsp6*f%ES7$% zIcuIThVIwL_6AN21rsHwnsgI3ieWzz_@Z8ze!X6HSSP6ioeGhpMZ!A{PepfB5EAxu zs!w%J5f#>qjFCq&@naBON}vF2PSJ<0ot?LV5P49i|7TkJ5E8XI6HM^~6&4khFYnyT z<47s_Jqd4kK_{*(q%|zn;bTf@&Wkxr~C}xWbnYv9C6f%3?I?z{;+xAE; zz8S4lW$HE4&?zgTyr*l@=g>5Zwr7H(StBjXHAk8-Ld#~@|%X~$ugHRfcNcs!um zvn_qFo5pz`=$NKuG$x8(DjDEF0s#Z=GEI>7wp9%e=}fU46HlBq8=H>H`cGGuQemBQ ztOA!+<*!p`A9i~wQ{=p|Eu=>t36%9OhVJ|8GB|1s5o8A?$UZD)XXE>cvrM@wpS_;> zI-VzS#(BVbqZ(v0Je^Vp2pxWM^5j~xuZv?|j*BBibIno4^LO0`3sZ-_kyN|sPu~WL zFW_b$32tz|jc6y5NjGy!+law9YWv6D>4-19(#i(b6F-C`E&CeE9;W)ZwUs^B_V07- z-)j+^ePp^~eu=iP6eHZm$C+^#bopXY8E-o2`t$%+$SPa~WL;hPFmNyDkl*v%O{wcr zeE&*k0r)KvLxLKEJJJ&jF^7@dHtn9Jl|^wie;MqW`R2ghIIe{{eoWl!_5Jr55d7;` zASJHE(HLWz5k@)h^7nNJ+{b+{rH11-krGz69*aU`gv@WN z%Eqdvwq-E*O-rni6TbR{JxL~IE*mP@Jnie;Q-72(slX*u@X4iyF1YZ%C;9s_2tp3D zK$JrjBxwFk_L2kvhj*|3myd<=e}Aw9kTMCLGU?O;@L1}d|M|Z>kawa$>mMM2io8ky zxNF>c+h*7nMdh!xv|%6LFyj(Y(ixb6U6G7>QFnA$VXUk+)o%<_@-z z7~~y@>HVz+g9f3@{>*jUpyk=YA~k}oUOYyPZ(-hfu9G zg!K+)mLBw;^DN3rok~oGW2Z#L-GGFTzW4VZ#nT62h%20D(H{>PWQS2iB^`j7+5VwC&M@ceP+w|I5Ks1^6D=3Q*Zn6A z!H;Zw)yL=ROZQ<4OK#A|!hY8Q-nD;wSs**uB2yeG`_%lFZd8=KNVLw1=m<@IbWALH z==a-vuuj(Nkm1X-&fl(^dA$1U%#GTgVohgl<$Y5{l)KE;sI3hkqYbpNe`E+em$@bg zHqE~nx);{dwdzl0lenAluYok{*G7u~WbhP;RzQ}kIJGjh?Jk2=w-;{y1Fhg3OZkd< z^|#v5zrimYh=Aub%2Q(9B|pydljP@)w#AMcdDnQD56{Fq|C+dYsU7oAmo7J~eKW)Kr0D1lFwieLySCpL+*RKcl%tx>OaJ@KX8Il#nL zKS*4vMheM%ue5b(Va0!UdK1{Xrd#AxKR!P4OrW|XaJsoV#W*x$D^vop9!{(&QUjs+ zPqZ&_3}d*LT*w+Efo8npf-P!qPpe=*IWBHxV*{k7yekF9tYhuVuE?i)5zUgTUx%iN ziL0LCul;b4`39`|^KAx3kE;Bt1FlW^_jPthkq`Ls59$6h}gMU$`w7y zsAYeChWqZK5=#D;OcLx-Ym}gf%`dXTH-&Iyow2s#kW0ykhH8ElV?_}L29x~#raA@u zg2%LvpFS17cFQm${ii{0PR`@UjVH;8M7Y;!L?j+Z%q$%VpjNnQBlWyz5@EPM&3}2! z;RNW%k_0IBpHl6PnJSX<=Nfm1l1$A8x$Orn>byO8DV1_h?E1%4qarARi)elhc7iZ6EA&c#xyZgC|nyV|A_l~Z>b|$tY95IlO_ zl*P6_Xm|Ry6am3)rQ1=v6Bo&?!7#FhB{%8ptLIw?I0k*Rb#RMOe&N|pn~h*3ecH|@ zEt!`6<+3L7V13+;-(E zZa=?y0pzrrx_VqfLPA`e<@nd;v9U3rSy$VmUTzlc<@NomVJ9K8PVHacyXMi~=Um{R2u9mUa{ecBq8t)u7=U zNs@WALoHVtaUk_i6D&VvOuUd7S8P6|%1CsYS#$H(YNTo3#Kpx0-j|%*-1ZR_1*w>4 zPwEol<2Cx=0B}yXjQH^3LwI<&_w@e30n@U!vGJ1kba_b$m)*R$(58l(8vb5UM1+FS zCX;Tx)>o+K$wsC3G^v1xo6sf}F{^kUbh6BJFkLk0Gnj4ATfIg<;G`>5FEs+abaZf_ zSIMKie`h&F!G)NZcz)CSlo`>ZT5dLcakl9>-*dY4>xO%HE_xXct>WT|F?7na8tSsx z8au71?fae9aUg{&!lq%9jT{YetzMEObV|)=fg!q6m$SnZb%O~d4!yKg>mrm;E_a**l5h0O#Y{rBs&`0=h4;kUMS>|<+0$+cg3B4 z6keSwa0GVE%ME_(Z8P>f0~63}AnK0>@`D_f`*0{;gAj-7RFy3rSfii^U};A9Z2^IXckkY%rlx}B1>%Rx%O;Uuz)sAB&dtq%Pbw)Y zif`6?UL+4$IyqGkt5}$ZYp%}CvDhz4A22Vpl6$%xtUo-9Hxq{YcrPq0@S{(N(635- zqo3#be0NblP_6)VebFM0hh{nBKBS&I!NSU;or`7rSAx+guR+t;t6lx#*Xg=P&Bgr{ zl|!+wgGuYV)Q=O|IQA0bGAltc3U#-3t|FC!7J>fZ)~~%cUKmv_yGF%2b!{_JU?;b) zJ*-jnO+yH6+m*7A-8$`jJ%8JY^wJhBQcl~g{FEziCyu23o-9V;eO}(H`P|< zJ)L15bn>jg>h~_IXE3D|yd*=sgVK-Z)H@;3%c}X*^_iP0GSbpo)pl8zIP!Ymqr;z2 zf&B2PDJ_76-9Zh2i2#({fx0}54J=8aowIEjeMRLLv7VkJP}~$=ABDsDtZC`$QdH3i z^SoZn1DE^1u(lj|9K4S3W52T4*d)ji|Bg)r*?T}!TYm zECOtX0d4sw)yD#qhU&~7Y)HxC<|BUozX=A%ZipampS zw&8A`*QwG=vmx3Y)l%rDPK^{5n1$+>isbwlYGTAeU1tpTxWeoWE;hUr{fs=#982W_ zw<%_{dS$mI!UQh37v3C5tynK9oHJJ37;h&Z= z5n-?V;WJ?9=B{V@;i{M_fBXd zYtV<3p_LPU;=g|k!S&cnt1ieZYvG(etwzrYH$To)_saTye|jp14_e2*>V? zzHq7bm8J15CYd0`C=kH^(lxMDI#ZqX6YUi8;3plx6gDC_9xgpu)SR9M6%g*?A?Ul=azHJIFm zmc#J#BUH4K$AwGp%SpUfx$c{TC0|Z-DDJky4dvJ1CD+zSK~3#v2*Z!o;?Rh|3gJ|zWw`-Qwt`{q&&Eb-glG@pN&c>EC`5144 zls@N)u-}K{_oJB-QC(fqHAJaDF56IBTN@%~9WIF#Dif8r)Jq~-c~YVDWL_g8mT)i7 zJ024YiyAvDj>UMuG8`RrLqSmR9j$B*ppFhQLzJCC6Xx-bgOzulj6TS9WooVV@IUgr zA#Lfio%TXEWN1ivhb3Dsh2Me1;1yh0l+|fntwL69e{EDwPA*&bM$T9t@-f}6e$Nc7 zh_Q0#p8K9qURtO5Gty7G!~|{}aXO-h^h3X2@D4XFpWsVzq$hXFCY}_6xq+p~gm9MD zaEaCbs4N`Y>FifesoHWl7Lxer)NH9(a&x+_JuC?vQ3fo3`Z@Vy_~WA|P)C{p+HQQH z63sR6!yEceOs0m4U9rMC;?WKd+-773oIo17%uz8FIhUuAYlZU>}75UB;2NlBgUchC~oUhrbTWhQ-UqFEVb&l zEt{YI(x-}{#eMJ5@!ABM!pNdegY2$>M%-}`Do#uL9^+baetu$IgAajF8Z6CQ@jR4X zJqkCM9$$jXHRT(=sh4tc6=+&|<&o8g+yT>k>#IRKZbg5!a*-C-lP6w9w;>Sp;u%A* zvITb#Eff{ccF&HJI6OCYC}LqO#5BU5<;v9v-8xrCUR?(VR)nTUA5vZcAf<_F(ZjOx z^QF#JH8m3u28>!&;pf*N5Zd+Tggv&C;;3N$b*Bc49R$xT0SS^nt#>*4U2z0`C49bh zP+db~;P+N&2F7`rC9HL=FRjrRlZaX-UgVsImbR^}?ZbvM;kf`01j5v-n|%Qb=Q_4> zXQE$Kk5M*baMH4-WGdZz61zY`D921zuV?~2N<*Z*at1N&lY)HB$|7*pv0~c&{%ze~ zbh_H!{}cs+GSV_K<7}ly-MD8@Kr>tND<7_Bzb@jzcV;{eQMO?)NKZCIIt5Y_l?)9m zEs`eE@9qnOv_3G<-T?s;b1fm*q}(5q7|-8%KV&fqU(yp1xrsR6TLF0yBO@c=kP3$p zr3l@Ie4}GwIGLwf#3PAa-{ZI2u78XD4UR@rJiR1Wf7Ft}kxbo?L4>Z$Rt7)qLvHPN zc6RbOt}26+4MY!z`N*>!-n;1NX_H4^zkYqIufGB&$L3CeOHcB7y=%jT`b>8Jb{75N zr_A;IUvx%JY4aqJTyq3bM=$(!aNRb3&u=rE^)xSd3>y?oaz^JPW9REk(|}|^4B#Nvg4DJK}*Qt zUcl+(k^DS}b~_LWO6Yv{;Dqg;>%!f}52*=cpHA~}9yw`okGA%U@OAD5xEVtCBaY&1 z4({a|I3?W+asA{yv!Zz|4Y82TzLA`8z}hv67~YWlSx2PqKrWd3g&c)kTa&81_H!`X zJ4|tSa4-W7ncscFa^foDU5&70ZyHxG>Fhmp-l|Ya<5_PM|D2G7g|OQ>SUu}nS-(|k zbm2%dGuHHx^*AJTRtVlHXE?#vz0kYqxF*ZB?Pn#`=Kc$Asi>(f!w;+6W*FY&H@;wW zyy!L?hsqWuvY02w5qXGeRjZf0z2zbZt~&je5J9-l(^{!s4A|CeV($%e|1XYcCZBs} z7Wsu{63;dpoQ~p?Y8-b45az$o1DFTxY*Xui@$8fno#5j*8X9uFo;)JwA5;c28msWC zxt^SA7WB(r)q1RIvA4f4FRI&BU?WIjt@4Y3E2Bl;_k30h=+j+OUwN|2Exg0Kx3!fz z33=y)M%8xwN!ZL|kZ{mRpv-d@DeT%J*2FCiI#@h4$;Y#A+jpdW?7nKY<n9oU`LvzM)zFEYO1S$_s=dfJG(TD0VLzp zUjCvx}$J&RCW9vZhpZ2s`&^9E+a(G-ZGI651bm z53A(pg8Ws_?l+43Z0xm>&n~4gq$hG}WoM_PZe@jE9}p8qiY0-dEaEl-+4oy z)R7N}Bbu4ap;O_k6UkTO;wy z^qVT~P>V-LORs7Wr9~IRl2O_H4r#H<84aazsf|QQHm{)g)Mik{0pfdp4{8MTzH+X8 zKQD1?XCl+hkDi`>++T8lQh>EY82*r56OPj@kK6sL6=GTcdT^zpWIb&+k;rROjAQ7h zDiXfIB*qgokyK2FkFkc*G}xFY!9QQVcQjILK{HBOOzK*+BW>UrV2p&DBs^q%?-oLh z$=fc|?utV~=4NorLCmX{vOk^n3ySv%HPW!FG4UB!SKj!i#;91jdHC+?h?gm2I}-v2 zJ}18!SP47{JK`5Y- z9XLYMc`)=RlXUgF(%yTgU|ENT3@+~8yywY7!ykB#yy=4et(d)8)}dfZj~g9HKNlvY z<-7hoPEB?28ifQGVR82mQwLdStzf?BoT5-ByU7IT8c`^B#(Te*qf9} z*-SoZb5&ERPeo?p`O3cYTw<>*xve{9sZWWx76?4(uRRpUCME{1W-DES)WP^yAJ3%$ zkSm|$I~&TRcT4_~qW*p5r}E;$Q3#v!j~>4?2QfVhI40V9IOiy3<4Ps>b)777lz1Xn z%6UpMlR!gu>8WbP$ozJB{DsBr{lsrWZtY`E#vK}~&7yi-roEe0=ZL5&!oq(4WfF_5 zOLO+qgtR_&vFu_xs9x*w*;Mev_4=MFeFqYJe?EMN(?i>ud*I+H&zkFleD=r{jSjp! zau01$r48W~ga^~Ea(M_ArmnZMiEds<+F-sJZP=_tK;y#09+ou>XAOar@Ep4d|gmPsium-xg+ z@=AARiBUqsjVzDoyk$3Neb3=sb3@QaOH^KzoRouW`$kWj^St|*4YEpSX4GK1O>&FP z%t>>7@8jIEpa*;jjf;U}`*c>eD+p4bYPVrTdJ>xW(t0=)FJ>@7oGIE2B`xQ^V_5&i ztgxopl(kEGFIzAIa3I9a1KTgaTpsS3&bLJfi-NGm3e!7gm31K~Ay%CaB{&Tz!x8p# zh5lySN5j;TXEnp?&l`JQB^$w2I5Ojtk9VAy!=s3oHvifmm1A@`Y_3y}p|3!&xW|_F z&7+$goK12qYJcy|k}ej_9OeRV2oo8YU6jVNUB+{2FWg7m4j)c4zf@Fw1}B14&X-tM zBZ857H6rJ9JX1OO;WQ>&2^sU}iT;E>^t){JXN^f7F*Rk(rl^8%mU!R>({SXP=43RN zNM})#@nv>`k7lGGB|N*|q$pv&ywlyoz=G}J65La={6bgenyz`O%dcLaE*Y){yQb9d zQI%-@q^X=BJBdPo&IvgsQF#p_XlwB3YDwb@f+gKNd8Q=W73#(9<`F$j;=S9e-7CY3 zg0aZde<}aIKa1_>sSY0`x+$}yBaU@7u8+4Jg*5=(2Ma^pYVApG6H@$oT?OjiwXGc2iR>zhM+idkGgQN{< z659X+)GikYXdU3R1CbBM{k>w;F4wawJgWHc9HEfV3wVk8EfX7XCo8q*1rNvZ*%KCM zkMiGOo}}9w*+s2qvkm-2&nqWpo3Uv{^1abi?S9nvUMt2ao9(o~zrYaghza6AQzqG{G=KzK$B3WZ|LjcBtvkN(_+CnR*imA9q8QD^ zs9GD5vL%<~OxxRDEBi(l;rKwSwXTb%6i4(xeQRqu4@HadufA3{An&{_fbuf_(p@jX zt&`0L><7=^ouXPoNPus?7fE9_l;Z_5&vWeH0st93>V`xj!3$sE%!wxYWI6GBs`V#O zZ|kOqKzfde*bkMp5dr1DlOM3bPm3xT{P=!aZx&wPh)E7u;JsBh=0r2fIf4x9sUhAw z*<#>WXEWaHUV75P=g`Jfk7PEETMKtI*T?Q>)+e~o8J3H~KG%Gq=BCt6e&*Eo@&xuM z`?e4r7bj;p&qM3B8xZ+|y*-Bm-51`4yi`_0j=0F00NHQ`o04J(S#wsv)LUulb=Koh<6ov{{^hyIi(&-KCbM(hn96!Fe(foB18ORaH3 z4!GNdUjy&`%P00G^8s4~!V1Ed9V^nYn?v0Nq7?ZTo6_6}rNl&@1Q^Cq^l^O4^OfY? zFAm-=ROvzl35bFqT}mP~uc9CzU8EVRNK-&cD51Q7N(Yf12q-PIgkD3r z1Bid``EZ|mp7ZJW0iWzWd-lwlHEU+pZ~fNR^f5n+PojK?DW$OJcUxwRA`C=6I*}9Y zgo8MWS+*UsNlvVNW=?Lb6)4I+5;*SN5n3ud&1@)HEYtYUp-(GWl8rhi29`nvUrje8G4ynJW~bG+ftyWkBSjC_Ugxm> zTs&FAF<6~Kqylb>Kcgxp7PI}HPSp0&{x%rh+_EBWYXNPvNsvo!g?0E=+P@OF=zr|X zsj=Rg8SHM;s&C%|{Mm!i`wYWxlqO;}6}%Q8CtrPD<(uh5AQC+%>ir^bWt*&M7RbVSB1 zP{5&v{nZyQ!m=-G@rG=u^CXnl_0jEiY+KcfakaGG#uM)0#X~Q;yjqpKs|(r;^6ykc zbFGuPB|cvA+~&q!aBh~yGyT<3z)IYHKQ&Oq0XnM3Q?3gm&tWUVx|D2J_CH1PCpz^iq^- zK4x&-_%u4?Zw=7VA>Y*KEU||G4M;K8bXXt4sWm`w_9!y~qz>0$r5bmk*SZ~^FE%~y zR*^4R9%Sw`3Yi}Zof+(LO-c8&p?7TwTzc-1*~QU@}g>$Pst?d`U$-Dn-S%6v!*MsOle=8J63 z_{G?pN0SpcXu1tF@Vi?p-KNLEzM9_+*&0QZ9`BDjb|KygXWu=Io_UaZG5J?S;@rYL z$`8XOIRIdA&0ihkAE==ZRtELt3=#D8^;ee}WuHbI;AbS@BY=05(V0m*8DWZJiRDgT zeT5`$Of94x3yll+z%7;}?pQtjl{%C>mqgqi`Z%wSqI@o$zAneMH{aK1wCVV|#Mmfz^w5 z%%>&x@E`bc{Mu-i1*+4$wZPf=&_bOcA<4QCk58Hu&WsF589v_tcPEX3wfNj&ofKg_ zCFb{i5H`Jt+hgikQPeMAz9L&)@HN>fe_XN$SgXPV^G6Ey0$PO_IqXijv)IX!OeEK~ z_ZxHcY$;XLXs?sl*xfskCzh7f$?gumB^6?!^M6bq3CSz%WO?75MgZVV^XnPC!zX#{pH>ank zJE18#Wf(c0g=AF1c24>l?QYrH+`U=pa|1O{o#zQ%j8m>$^FF&2jFx@2!Y$bgn?la~ z0uTJkxhi<>H zLJf_ZoM6w8uiELKcU>{*TZrbjAZWm8Hj8zcg7DLR%EzE93&NAZE@eOb0I7!sSDn|HvRDH1q;Zxi3*1o53zWX$1|Fw;vTw9&daE*{SbCNxj58#k#GAtX4AN zUe?|}H)%RYolJ}!V(I7~E*Os7Q(kW@`PAe*HrYRV=8%ceQNXSEe!s>u}l6<1sswmx zoViuo}S{Vr$#b;|zp$OO6cJlm4|ny(i_DM9{_0 zWfSwT@O0gR!n9`m?yvg5S3Uw3)khP*yQksZ$Esn~Gw}1O`fg*3C5~yd_2K5j>AFS2 zhkHQPJcgDd>>f-}2xMM5vQcuTMvj}yfLlqBt~E9k@|wZZ6u)GuWzNuLld$xs+bi;{ z9?`kGs1Plcq*Gd7Nl*K8we_DQux&EY;WUyRngb73UeL`b-PTi4v|-m9-WQ$#QgMBr zd@`Y|wX~v;T|gjKe&o&hgM9=rG06ZlfkJvCD$jV*u*ERao&T`nwt+zv%FZ(WDbn{? zK>hCAx>)2n^7E(DBc6oA*mOVV!jaR4I!y<=DWIxbL7HdSEf+^VsTkBgtG!PptfYIq z(QvN&0?*bPL*au>AJBeroA!4JETQybY3MsqBfby{n%J}sjZ>f>`i(bl+;Vd58JQ?~ zD?jn4eXqcL1hew7y?qyyd%79v_8yx*E+WY-V=v?;SlUIoDyo@cp2f%v=GxfB;B`H2H z;X7bkTod1ijPyVLUh&_G02zu?d0S;dTzg+BpEM( zXx_-#!fE!~mQwtkK=-I=Y};o2Du78mJ8ifR#BNAUIIq|5|Yp7x(SOgL~*B;^T+cE(6?sloJ%$_ za{2)~+>tGWvTd)jl~^{l7-!}9P_uQT{!Ij?(>pj{W=2sl^3-Oui=RpCHoAC=b2|U6 z-S&A>vUr|~-W$qWS~uEMmi2p@#Pa19$8m53*Xg72H$FB9Ez()m!OFlVgMVmt~+@k$!39<-tMUFaW4N{7JFiaRIfVY8)eLRlxE;&oX4cBb4L7};aUkE35 z*uhgH0N(&!HVw@_HTNRf0-MBp>D(=3I7|(l%l2CpjUIq`*?}uYPpLkSu04_3cowC2Ik(qXDz1i4{tmWXjmNKCO^Z<=BF^&0-+885Lr zb{Axp^3E^AgcD@4og0`F5PfOdefnNJy3lj_572>LGR|>(EkE-Z?}_eI-$GZbWYRf& z4&n9TTEy}7PR;IB?b_bFd2_tUqv$l;anL7X@rw8CSF*hy77mWgyq_iOMw<)%mt)`{HfKws-7o!?~fuqXJdz+;uijybH6M|3U0B>iM7*s=VracE4PP4wgK| z6lGGQvNf<@O z%hdk+ue9JE%@dS&L?S-Ngpvre)8(HGWMbtIK%rLsxXV-S{ADy+;x3JZKCJG0G|w!g zRDu#|9XdxEJ;dXjWImWn?#9Pv+f67M?IWbM8oHo4o)Qw)RXmP)3q7DOk=}7~ld8Ob zvK42&CqxIgt!+xP%1@VH9?_9mdjHcq0G&6@X`0$;HT)z9!v0L=8w9s} zGV~F%C0tdNfb7Yhx)H4aDH?31e0B;^un_{gvOJPtpkX~ih)k9FCjO|`UV<5Qzb|#qr(H%8Z?^0W(bqP%OGWTb6+u)t7?Iy! z4P={S;)+YeIgmb#j^z9dT_3oU@56XdSbrVf$A%{EkFmIqSdZaf&y)SzQe&dgy=+_{ zd%s-6G&{8EZCc8sdR2_9Uktmi>g^<%S!S8}??qkoQSLnsrH$xoMmx6yx?(xO_Y*feN2(uT`MbpL4SFRj;H;`09*`|k8>yo?yyA6f zu*}jU7w&!DEC^iQGvGFk}-y{1g+(k(#-ZD7+|N_oeiw50fjhidKjxhg>-5{5rNb z(*+%fwCK_gG4tu+Z#jpgr=p${-y@v_DMR`9%r9Y!^@thcAEF0`yME?;X}&p9B~qOZ zxpNusH#W}JIh(~E?XWa2-TA`I1oLv+Bng(U&GeEmBS2q_wmYzaTl)9WZj?SEfk{i3 z8B09rD0O=R=6K&|wtgX8ZK3*PY+;z(VR~g|qeX(PMl_+#^(zW z2Cpt@zpV1>oK-Za>fLxar@!VBegUgaw7ZD?S^w?Hwr5Ds@}Ak}Ek)a3U3W%1_^il1 z>|ujb`eoP*fr{?+7%OyiuG~HMao>o~8()p$x-|7G!UY$#UN2yK(Ryf*@}`v94JVf1 zslH@}=-f!uy@d6g+rux6Ppz=0@#G*3w(C?xBU1x*;$xl8J%-otrN`I2vSYnHCC8hT z_%_;J3Zv(a}C8#gD3=kZ=LB%7Cds2#L6iimR>hE<{jL#?8>ch#XlizEK)0 zCLNVU`U`DytX77E7|V7U$TPZv7R0C--{(vFc$@9kdMll%Dl*6yYP5LPB0P!a zOv5tIC*6(pth!b`@`dsjYsJ!Y_O39R@_c#t(avxt)1d<%URIIK(Pbm6jim3%rM<$~ z*_Wd&xc4rm^wSFxo+l zxAX)!JkTLVI;{hf7+19V61#Vnf6420rKya3HG8>hU6Q>rtFl8J@mXfmuz~@PKPt5M zL7r{)pq#M#1>!Zz8Fht!<2o&wC@N>t`Iu;SQOO_<+1Si%NO$uc5_z=CS~W!BnFy%7 zXwFwwo%B_!ZpNqQ=^EZCU!NZdWFlXtvy7JQlazJy){H zUDSjnv@TAQk5lBHFIo%tt3+s?IF`Q^KKLLlE&Vm=Yy=IHgDVI3N?q%`=u>(=Tt7A2 zXz>Jqe$DchwIW#PQs$H==@8G$Cfs_}O^YwZW!V z1|CmSZB+qO+0>OgE_1TOySkHGu|7jpz-G9k)f$p7LyKX_pOPXM$+z%LG5O*SvB|bi zBF6~iX+?N3G~E!V*X7Id_ilychKeHSs9jxI3^x-TY0ddovLjpByz*x|hWeuh<(dlw zHfJNA3?ZwUz$fQRy=gq%I~Qx*3};>t1?Y-pKwKL*0hn4367*AHhBG3oy{03%>d`ky zlZpn)EE`nq^A!uZ)|*{QOIok=R_3UP-_qa0F31Y&&h?H2J&)UJ>0P8`!+4%{S4tXb*6Ec{eg-u@#<%-|FCq#qH$6gjzA zFWDeIB$I2fk2lV@lFr-uC@jDDqV`2F8o{osfj8eG=71G8P>R=Cd+iQWL3_R1v2xnY zozMlkp9$d?~lF|TDli;B|=HPiT&vg+r45qYh_BHkkUjw^cKVqggRzaVi| zxX0^pYdo97+ji-tlZe7lcQmdhI>zGRi_hH}GLiZpSZ_}2h)D0YCbteqRC15sP)api zX7yPQwPKl5z`B6-`6cM9%{KMNE>xOuQF>+EaIJV9V)K0zp-sE?Uva*Ct@STdP zY42bkwsxC8C(=)aPV3`?nX#w`VO7REmT!^p+S>N-_j+sKd8*u+=-H_4E~#x%-40QX{U7z*C>aYh9nr8hUf#<^jL!~a-JtTOHmGQ(^u`T(IBJi zZ#NnRQma&3=gF2Mp`lA5?`G4qye+7^2%U4;!m`g46|0?Bnb)Z*DN}$xsXa}&_&~>w zwwvH=MXTr1#e{aE-eF?*YpZ219YBk27n&eCsXq;P)=VW^)@H z_zw^6Mw4JpFxohOuc3L&m!w+4UUfQrb7Z4a0p@V`9mG|c)y!Jx41_&q(B|p9*YU3?nQ3Ks?&S*uTDhyf9i0!^r$U+fGoFdNRf*1q z53uc?i3Oq2qcgqotpaW=BY|=QNNn;{iPJwcX!Kx!vRVV{@xE2-=2%hddL34^@EuQ= zaAPrzt7Hrpuk@g~H=o0n<;Q3ojO?~`T8 zwf3%VN32`_`(%#Wbp5H9tCj=qX>PY?Z-gX%6Tc7Xb?91d&=_lYZEInX_{C@a@`)Ki zh-eKqJsH!s_Y8b+Bn1QC2?+0E5GjU?<$xd(8ecD;-0Ns;)9hQ3S?^9#lcGEo_OLzA zx_`q)W?nH_iFrG6ZqJrhyZ?5QWbp&SizAB+Y!3Kq45W%>;sbHoxYewV)7%+GHS5J~ z!>4dC#9H(B*+B%}MD30X%W>qqpRnX*)E**~YM5Z9j16zY?;*1!mlr+g*P*rqc29P% zvv?g-pIbJQ^!6xIKjTp!drX6D8!V(UgM3SMqkm^0_t}Q=5_gZ2gyrV1@89j!zV*>E zhSP{{zTA{`ji-eM6Lb1u9ea{6rkbt0hCvpmjLgu{o?cJ-qWnT{dAJt- z5@Jg8%`p!jeXg~d&!gOZw#o5=(~;shR^VWr+@cBt%CSh^n=>JHp4Yd3nhXe_dE{-zf12u@ zLi(%U(#zW@DAqCe{S-yvW_`bL-NfJIc1qFhx*t>vNbmZ#X zQbIozY-akiOan)q>VQAYX5aBdqPZ`b|39B$N`<&1W+lBbPUztje~kM?80%wrJ{=?* z^Y4AZV0y##2^rblG+n71hq1^&mR??7a|-$x48}cHb1A4?5rZ8Ii@xJ{Uy$zQ?K=kJ zk4kSDLyWsJ-$fmHy@fm;uP@jfBOh?0S;r_!~|hMuzwCE$b0=}j+7wX z?P=W8e@=nrZ`@_yZEWNOH#+bdf`-6H4Z5%VprK{a(GuJDzE8aPLh20b8Tqvc(QzX8 zt>(IN1pFAL!w@lR?}~}!7yTDr=w&Yfd1PE6{yZ_>G~qih>s!dh{8v#9{ucugbp>P? z4kISI^OGSB3WS`XmTi2ju}`CBdWY!5AMwrf5T;G$`S|nMHvKjUp4q#gUJJqN9nX3B z_q3A?BNrv6yFT7Nf-rW`=P$X63UeuQs{ujEjuC$@kzYJPL9lUVY~;%=&7wIB*8t<$ z)YSBu1u(eWW2ImDGFihOOEr<_^4J46eUN7Gn;8n=GZ&L!CMKULX3=F9re)QQ^(~|P zjVk@LUs$WRC#GN>5OEgfkC}b`T_(It49^ezHgZm@EEJ$Y?4rACpE4zUnayD2GUX$E zFMNCNdgXB!>ADH*$3X7;JALGsGC`VBywH>Qj>$8sw;uT~%Vx{b^~B2!2PlQ-J+ij> zVWaPP)Lx6k;#PR)*;0PBp!k}5_tB z`x~ODu6hNg9bq>BW~6#Pdr+IwxuXXG(_*^mFxoWY`EOVWm@hrA!r$?TAlg^~q*h$_ zVPaxBGAf`Z&Fm?qbr4e&y$5*9^ynrKts3#Z8J}l&0E7Vgioq6$6zUJW)cHZ1!P}c? zlxrNh2LsFuF)?eIB#*Ig7q)En={)!8B`SKW_AA|cstB0UWajpHDSFAMh83SHFyoFgW@X!zNYLIKxCRvqw+r^H=U0B%C^4+Y?M+?4ZLxppxMM3Yop z7&_RprQ8IdHlKiiE(B5v2(kyfqTi*JP;cH)EuFa{c%KAoEEE(Jv^Pn1=FDN-5qQm; zGXUa0b^3IaTH`w@w<%QcSu!$KNVxUm#~Q!Q8VToR022>Zc<^|>07T_ZN9qFu_Rl$h zCKq~4|9B~@d}r+hOXoi3qxTHp(*gbccb68XuxJUaF@S%-osLnP6Ek6{KrO&!% zqXlR-W@ctn=QezP1A+c^pYcwXmYw9>0U+}pP?<{bhPjKB#uLpkfG?{qy$hD{mB?cj zK*9%Vt0#%V0&XnJi(n-!tz5gF96%MD@5<}|tpcoBkeYX-%3F>3(!LB1!vyMny}Lqk z>Ngy7(DxL@Kli`{rvjr6Y6f)O4HYZE#Rb-68g%3E^1kYk>Rme+qW!T~eD+7j4(mM7 zLZwgG8bl3T9(<7Vw+u)Wg8+50cONu75%*sesGEL&BuKa~70YCa{TY2d`#<`T(tGfh z0}c9gfLM7Tr9D7jjrMun7FH(*E%7DDo37e)RH8)rq&OfoRhQ9~OnwXflGX^4M;6F01f-HK|2IP5pdG1_+fHU2h zag_jKief>^Hn9w?kRq>DyNa;-mRN!5A4a)eAjJTa&OuZji(1&#R4u6uzyiv+y1l&( zUIb!z)3u}k`wLv@&4X&i(%1l^DR@y?nPkNqpamzO&9zu8l7He76O&G?Ye8WlXc+k7 z!=%X~J4LHXg~B+%@sPF7GuP|#VZ={qpiT^?5(N1kU5 zUfGwH9v`s0DX0%3S|vP#>UMSG-Zl(2SY-G3ofQZM% zD4@;1fQo{m{Ll*UIxsh;e~d=--3JZWPi2vD6$BfDk|2!~C^OMG3IbffCg3<;Ck5L{IN1); zrjhilf^39V@qme{9_#uP@M@u^SC@<`!cJ@gNszi;0B;4MWT3}B8w<~Xz-M}Rz*|A& z8W4D8rEEo*il*i{X!CVeR)27pfRzE=8Z31Nj{z<{sDZ)}`~jt4FmY(ZFEB*l=Mu#$ z{)vdUZ&hPmuhA-jKaeQ?3B|rAPZvi|K^v~~@PzPBTtg?GU82>Fb&c>vNh>iqfPMpR zHW-%iTp+$BE1+T#nG9guz~?)F2+e`QL=fXaHM<}7rQCb*#!|sibO-?8<@Rk4_}2~M k@Am)y-+Y15XKMw4Ka0m)t0)J?iG)Psmd?#Q<@?Y74}H0eFaQ7m literal 0 HcmV?d00001 diff --git a/models/tacotron2/NeMo/docs/source/asr/images/ctc_asr.png b/models/tacotron2/NeMo/docs/source/asr/images/ctc_asr.png new file mode 100644 index 0000000000000000000000000000000000000000..56ef705602da10b6dca5e5f01afd66bc1f5230d7 GIT binary patch literal 54705 zcmcG#WmFv7)&-g%L4q`H!GjYB9-M|oLT~~EC%6;b>0qI82o_ueBzUmk1PSiaxD(tR zn&#EH-#zEvAKx3_`}4+V22@j3yQ=ouYt1$1tZ+3Id0cEt>_?9t;l5UYXgqp^rvK>C zcl)vq+(X?2R|A8v`la%%o}}dw-=Y5sG7T-`tJbXlh|UQ+uRDgSq>5f;`5q1KA96NT-yu(*SgClA_zpK* z9eV$%3)w>hgZ}5UNumMydn3_Ufq>xO8|ne{vXFo81Gxz$Civ$7vi~0rI0@(W_L7#C zHvOy`fd3{CJB8(+E9t^TpL@>Z?DYeojix}*jq=av&Wx-5&*@1cU@-Io7-kUNU!OHO z!t99N6#nL@EF*@0u3re-Sc4%!Mx=n3kRtrGgvV>@5fgYlTQ^Y{3loM`_^1|6CWetD z8_&DfP5c#7q3FL$SH+4~_4X{V{xeuUtN_MOz`!@S8BD?{8$;o|tQLqpZ;_%b4vB#M zYwRF8N6uuOAsb4_cfxF4|H^8?Qb=)Ou{p7DICzy1;rLcqog^7WabU9btil5KKWApd zDuOY<@Qk!3UTHJPAGOb@oreyk+ICuC|Mx!mFi!q3Zeg3Bhk7C(f{EklN&o&YB(*a| ztofrV{uvW4cgfY~1qJLJ805Qj6-S;)wcFIQ|Cu;-_V~RCPv=Hn?k|G+?I?jsQ3N^2zGySX zZ|P(fsm&`bc^cx)rc6vM97xDRU}})`num09RFOnsgSg=3oklP2b0png6GuR>ZcH3R z2P2d(PncT&tQ7MzuGYX;UX`GP2q!Zn$w{u3Lc3+!B1P^;59dmC#$6CBBZ!!^0o%IO zepvXg>xRLIix^IslBOhIqJfDS1tWR|7h$Xx`PPXT>AFkV&r=4vUEe)d#JIKl zn!71NVXS2hi{jku2{`dF9eqtO|HGRXpSx8g46ed5EWSg#W~ZET?Kjh`1IAwW3hASv z;WPVe1j?GPROa3Lk%@J&5>V>Dhg)E{5{2QXw4GNmOyeb8TD%3ns~}4F9KA?LQ-h{q zfTsSWN>hSY91+n|{$q+j)#TLbD2SC~1wVY1I|eaN955fD`B&%@827eyP|i7)S)yUx zx76Lfp%r0&SB|PSo53X>5!DqJLP(+QY+W|P&*0+lOm7nFqOTl|or{eIPQ=|9cA9HA z7{BPvkMPza7R%cv>p+l^RyVE0hsSVPkPNixl*X8h)S*%QCr$=Z4Y;L}N`)wZ5~^6XiF> zq5OcKyNc;EiK3Q9hnZJu(&B`H&!~;@8n;RM*Sh@Mp2%W7hBd0Qiv)DPW?zI95F4;- z>G((VrMz8%A6ZUglV|r|<#JeuzGHb;A%^o!8(qdG_mclyU0;~rY`^G)>SZlKt{ob< zi)F1hZ+=z=J})N{)%)F+E%n*KSjX?D;4a3m3Iu(%QlQO#*$I&j26XwLoTrn?WLu1k zKS#df)u4vmNJD`jwkpxm(>!@3yaKEddor~V;dC%Bc^ZlH%B}B^GgPO_?PA zvJdi+Cw*tRmtpsggukFKKZ#~2I7vp=$kR8mit)FT)Z0;c6Lx73ghJ|>?moW$`#7zn z#XLdMIUQYCy z!PqmLlOyNgoB%FY5crlYK{o)wYwJ&auS9f`mjQ9Td*AvnjX<%$< zCDE4%5U?ah@dQts1Q4*CWFG1iv@!3=xbB@5=|ptJ6owyd+Dix_qZHF&G{rn*$<(RT zM#`VX+lCe_b8}~Nm(FN$Q>!UPPlFq{u57)2*r5m>fr!hyP1{+yt&XCh_nqu`WhBWq zzByHHA1&6#7FJ+`^|*M*HwRIX8y z8QUXOFme6X(?qkAL1K?J`9J-B{*RMODjd98Bg+e4d7Asqtnfs9E-f97EYAB#nD;SS z#&AH4d1vvDSE1w#RwTXp74P8#K5i+xrB(Y87>wi2p_{BZkw$BH)uv`PLN-*O+=@fduprft4Hwa}zv zFXLLe6>+0~}o6-S1-uk1-=3BRFTLDKg}v)Hk%jmz{nHhpV z;y$M#R_~J0YR>!nyaOOaj+uK1fqZ@kE${le5bK-&oN!{8_EOI~xxz4YgbPWQlC|yO zV(4yUTxQQ+-XP=M=~VZX=s(X$Ap_yB9pG;f#&=N-wkSA{i`@R0dVi)$cokWr9 z|2&AfpZ`Dmob}-L7fQ)#_bWU8gxpB|eXm3olz-nJ-rUXllfNI1L7v|~^BB$>{r3XF zI6ycD|2;Ka>Jn;}*-@M-%8sLuRJp$cDA)t}6Ff`Y+pE~nhnvDm%UZ1}M_R-zWSys! zuOy-?mXAw$=U)D5idlzn5(q>jZ0p3TZ6!7O%&` zXy(J+*|Ox_W%ZR)TiZh;(vEj}!+0Qt0z%B1jzXc!Ce*M>xQS|U+1I-^x$#6f(dM7P zaNHi2M6O&**xiXGBW7fpXm(rbuEOLc6aT*)yD$u0+YgV&ki(WeKRyi8f8mg{AGv%* zo$~1ac%(5@^=ECTha4bjR6!i_DyyzG4`j(vV{4}%{vBdq#6pk&>;HJ*J<&5a+*`&* zS#^ledv39xXTJieo=6kiyY@(8l+(^w+E6uq;PUz3g6zL%U&wl^&2z6s4btg}M_U=t zp6^TJZ*qrvJ?fxUKebb$YAD9^l;FMk-_IX6@<=J|%B}H1$#!YBb2(em4Y9yd#O-&y z9vXa&m}*eHw;GSh63_6x>>aDlX5{$5EO0d#Uoql;deNZTCj3Bvy^$$?+TAd6EPRiV zvgLLgent!RM1uBbW=C9y3)&A-tR!U&4_7O%-1L646&-&xSl+40sFwdRQsC!(SnV`I zEBz?%#iT3)cKZXFm885EYk1)*2JI*|!aV!6tVr`JABt3YanSBDR#{nRDz-|7q=?$n@?&nhdmKGgFxMWuVMLuK*OcGn zniIkJaxzE;5ovI}+WqcGO=^!auVW@lvQ&bsb&F<$#Stw!)4WZ_7c*AZdx1R!T!j=NY0@ zq>>`$6L(>K`ax+u>)vp;u7Ld4TJP%*v1H?O^k1wrMJ2MvFXB5FX+_5`!Wj(@k>;bW z-ws$r)7nS61}&{k>q?MrR*VIMZqvrQmgfA3+Z41@3xB9*^OVGq<^RyxPEQq9V>o>|!7U-%mqF{I_9d%_ z`k$=2P$`zoyB~V~n33_YJ7)m02*L%siJ-@CvJTc;S=L>YF=Un1wF~yp>yaP8B!ybd%xno5?m5e{nR8@4{YjE~vlb(E1B* zom$lG=b6qn%vJoOed6}(Ir4RR%+tlPlOtj)_)fXX(}gm1`?88Yb|=2BkEO!82ZfD| zISou%$#%`kg(6L2SuSJ<)wMFQOp)t&M$h$S^_}~}hy9~`-rGUX`piH3{3DB&xlJU^ z^nY}9ssC8j>aM+p;8Zr~V*Z?Rp+VTi(7EBZ1xw;OZGv$w{dNCa$f z;V`%6*>N-vjgtDc#tA+OD712i5N*@3)AW{mye#6)qt2Z5wYPNQuV0uj4NVcwh?_-a z49BnjF{kS64&IBUZlDtX_j+9$AMaX^@x#DN{m~zNLO|F5>YwV^-E>w&t$GI^hKm2# zif?ngFZmf;{BVox^u&GWxr9nlf(hJ>X1$TCH#(;koUfX5I#2Pw85i}Zh5VDh7b>G( z@$O5qd4h}WF9p1~oBErCwXQ7S2)d$iQsdy%@)$M7b!zdd5472{M7Qr#i#Aa_spxHz zz4fJSOwc8NjhX$Zo}c(8Blq=ck*xT}+p)V0c{?gOAg&ZG^+by5d4tH%kSzuqy2F`| zWj#}twv%~n*u$(Y^$OAER>OSxrJ(+n9nLDxz|`Yt6;Jo9%qbdNWmor4XImN=t%uu4 zTe)Qxjj)cs2{suz`{TzC!-Kn+I_rGC(L>wln|!llBtu~07cTXmGCtqG)caFArMJ&3v;ve{IFTJ>uc(^=!SU6E-Ww0)2bGmC% zgWSU}!a#E?GcRvC`=DRW!`xH%%q==X{zf&9@8q5}+{JfZiKPgF_U`aqIj9G}HH)g< zRcFlSYUDirZE+c-?BaI>=F94&g)BsDKZGT3K()mVLpCOWM+)%d-X{jW4R-Nmb_Zc|U)2~K z*#+gUv($c4a&`;#G1zttZoI25IT|VtjA{F`e_}94x2;rz=LDl|kxP=znkHY0Z}wCz z#Ot>1{T30zd$$P>DSjnkpll@5znkF zJD=9$%cuB8qi)|FoJ^w&T+O4Hl4z(>c$FG^KjYQOomCwi`Fey=k%qevs zd>EFz!pE+=pFxtg{bgthMXJaH|Zy_&O%wr{|y!P;I-dE7J1M?bLjHct}YtA54v zrgwNGCKm4lFQM=UjSQCJU6Z9@5HMIeD`G3>t3yy^(MWo@L0dzhiHdQEftEQy62FO^^B#qbsX~PMW^qyQT`*C3Me;``Y=W~@sDcQR|5XqNDqgFAH)@JU$2mr%Afn=%~=ksh>Od>e3n)m zzf}M8riu+uqN0&(*^Hh5#O?m(8Sgi`j7G29z3S!U@1k^^TW`}He_RUy4JqR#jQ|P@ zYNobZ5Xi@*2ldLnbt|=HWTEXo0Fxf!dz9>(8ydn8L`n$&+sdI%ni-^2=nl!@oEtpy z&KR-uCr&-g=_-DgrIG9c}2>jw*H4*Yl5MBdfO6?PEX&kI9d>)8UZ3Jqsv>@f!>{tB!@psl@{}HrO*c!{;5_euNcG0LyA1*A)QOIU{Yp@;QTFf%=T7ApuXw}a|A zk4)y-*dt3S#RXDMb*8&)>DKmyprcu z5GDToQX)YFllMt{={djtiX-X{I{ruzF2o%Ng}wZkAIfp3g2Q`@Ido<^_pvXg923=r z*jd^*S4<>EorKV?E5516{QU1gKqrZxs&y@Zt+iic7Sr8pAiJOHj2Qm(SsLdu5{K^0 zc^_e^u7@?oE{VT~r|7F>JlrZGE#3-4FYJ-EB}bx?wuliyaZ9BC28*WTJ?}xf>vB$n zHZ|ee*F`>zGRl)-oxWvb#H9<8$?#B^Kl99RX$v`drZc>2N7Bn>+~kGqZwSagbmDD3 zXs3v6AF-IuR||~1Ap>n)#wJinj(W^YKRkQofvLXpG(@N?8G6vq+ws-lS4+u{crgH}Ti)7)FO$-q8mz~4L%wk#gIi;O{aK8G} zl)ZGvuL;?^z4xDAKG9;$Yqh(xjo5B2yVA*2#VkJ^iU*M97I-`9#BicHA0Kz%DZoR+^01il~NcC z(uq(`3-TtklV0&{vS#p>Nmouv&aa&KYm$%qfAWM-Yb6HD_+iLyk6gID8_;R(&vrmK(og`reFcUwtw{dt_fm#7x zO^+zr$=25Rkm8;fgWm>NMvR1qrY0Ng;1vPhw&p& zJcQP-Ud2jizF*5EAjNM*JE!6h@sWg0U}1hjWTQJo{ZazO3iw zjB&9@3s0%jsO~kE}rTsm`bgK_!m`gD<13^HV6shQW_~uM3 z{rv4NsF>M0oowPZA4ejeVJ@^XHHC@mz$}*k<7cpl2Avb=f!~TO5yIg16eoJk*jXF9 zAx6W&;i*0dXu-gv6A~JMl;YP;KKt6pqt8UBX zilmQR4&B@ipldj7z>X`L+33|i^H_Sj@2%-~5d`G?{HyWbrv=y^81VsxUoq=>jewBlw$|TBvN!7{h<^qH25sF|;r0iDBz>A~ds85s9xR@NUga_d(AxHl@^DbcAAXen_&HEa!NuJNTbjwC-2>W!@513^ivQj&mPz6{2HF#-HI?tS zV5>y3%M~ZtK-${AphUvrVDLz9&~@Np5NGPD`2*L=DVBTDtJzfAkT5y`hyWx7Z5LHY z7tkglT{pNwFt*}oVOtF~foja7^!qm$Vipbc=*ZBS_BNm{g2suh$lTt$SvpMywFeUu z{Nj^M>@Fa{!T#`Ek_lb*Ttn}HMKet5`_Puo@{5z_ye#GAptqP#+S>98)Naa{MLn8_ zK7#Hbc2n4B|EmC}DWD_IAz7t0Lno=}m{=Q?FedDl5U&Ks)n;A;S6?HB6@>6fM@&sv<+e{Uj`5s|~I(e3tO0`0LlM%=g>ZQh$2P zaq{^!AQtm~5!0sXkdg_VdcE=8oBC|p=J&?{zeyOe9ly9wu`nv;>C{T4r;tIt+tA3O zVKS;vNmJtMrZ@I&{WNCu4d{)4CH#6T2C7)(+_9F}y^St0cxz5NX*i{;b-#Qkt4;7< z*eP-cwUVnREg-mJEO6h}7gZnuwRQ<-pib;)4S(a;m#C$e&Kv~hoz10! z#i60d=NO{OXe@E>Mjx@_8=_5xa2BH|;Q2|t2*GGT_eW7jn?+51yu&nXC!WlU6EE|C z`6BR?lcwAiARFijhI8de8};*dQgx}jy{Wv&yT4LH7o>Y;%P(K6T#ll{7(xQMWE@+7 z1gMBphG5y`a-$6Z%@_#zs*l^nWN8{0u8@L-a?^&M7Fz$5-eMKQ;uJBXcOJ)BKnv!qjjfTWE9+sqo};qcy8K6~n+n7?p- zcEGCKfJlg&q@6C?d=y?`-0HFVgcX!~^3L^!%xQxy2PPL$Y@ZIW=>)#9d%W-#kK4Z@+pm738ZA)HTP^R&go!IiC zoS5LF5|2VZUSA-ScyUmr?GD$r79ytev(~=h6i1RsCgKx!P?dXt9w~rj{XVLE+20n4 zX3JCA|G{7WV}*hpW!x^S2|z7xcJ6Z{jfS>^cR@4&&I;5c|FtwBg0?e}xj*X}CzN2g zAHKw0i2IQ+*fbDRch{zBW&-<(Q^K=FtDhpVTUXpI8DnHq1f9~VcC7qBXarzK8V~;F z(vk6GHopuRUX3X3q0&A00=3eal=Xs<*0nEEqF}9IQ=139^j{(R=6Qd2+)^C*c{}A* zUvM1=>xtEm>Zea`f;0M$Hs5BZOeJS}xVn+ic3Det|GIe=ptS^$UZea?v8)Nh1SF$4 z5v9x7}GdvOeT&ykX@^Brv}h%J8(n{oFvO-aP4ID+# zpMIv3`mgyU-knZ`W;k&_lb=zn`po-x?wIlQ+i8C@a_CDcV%3#J5QhOsD-RX=Ul?wI zq<}4pb`lKrE#JX1M8g99?AV`cxVUdB-5>XPZ3koU_+f2PH0Ns9_VYm&h?r}DDtnuu zUe&!uDoJ~H_WLO%JWX@YWa*b3-1PXR^M92SfXEv-RHwMt_R+IH6<8oCuAZBBYigsx zi9`a;sJzwrZ$hY>TsE9~Um!p3aU93M%$)~i+o~Dh?BlS%bEEsb`R_+6|C0z(W z;`j(u&EAwA-i8cw{8RS z8;Cx(XhC4MFUt>iEOchHqWcg+Op~y**O!>fc#~hBYn2lNoVrWxIN`1TDs46aIYZAY zC0BRp#|ni-6Em~V6QN=jGki9|W1^7IVTN8{0ZFY!^>07zoB{aEI>{!HIe&p_ZelW1 zFA;~{NSk~5RiK+>kalj}ORWJvNcfE$cS66ia1hTZ@ycwImIp;e^d90dQ0JM0j~KDp zo^|8lO6K^6$bd8;Pu3Z8g52K77JM}@^?dy8YHMW=U`Uom5XUoD#_zJkgrS@JxuvR$ z#@-hVIu(Yb(E*+(*Q@)9R}+#7;#tV42JQpZwhT!C823dk2khSW~m7;*KU4~@asKDa5*7k?^zU|v!$I${)$-M#E zz&JZ{#4SU4eJMTHZl#IAQWZHm2M}#iMcUr!zjA~C1whvGT3J)+AWHFwW%z7x#P;q< z+o5B0J)2^lyju904agd^Bb=jA1%{>^HMAe?l>^H;v1|}&{j(iIN`H{UuKuoBlf+^L z)QKV)$3H4qj1um?%yPxuYkT%yg(-@#BN+NB0*w`MF_yEHj z_NYzyH|?X_OL#CRoX_x?sZ;HzD`Yyt`oP(lDU0GiQRa5oPjT|Bp#%MmjttRRZaHVZ+L5W~-j*B9$dv1Uzpgaie1%-jVm8z>|+%IO~n~tu!(|aA*{2l5_ER z;Y>S*2VK-HzZ`u&^0n$sn_S^l&kvftYe0cXbz6<4_&66BASKc!Y<2AVE!g9FuO-va zOjJ`dle9sf`>dwxmgYxDwt?x$`ls`baVV*;kU9(I3mC>VWE$J)EV}iRnqtCrHJr{e z2W|8gfX6^91SU9fg|yIzWNok*S- zd#Br9z~#x+B>1XDq!+=vIn`^5V*LD)k>X!`jkNU>b+_!Ahv&W6+g!0spSWxS*)EQM zo!6&h6{u+Oua?G$L_iu5%eh@|hl-VX%XFH&oUQ$EC>SYLHab1=EjU|| z3elKP6(papW?|`j{%cf>G?ZD7omVAz69l|L(?y zG?HNx;9p_JR3ikk$+WUMka1KX%Z=-m3A=i0K+xYp_FqrmSg9I|&)%fwl03{-b_U%+ zo3Gcq5w4O($j71G!g=@xw9u2_ZVDRWk7>6vR}EJ`jr(NvK?=);Vp*RPF2R3tqli*_ zMxYMR7)+1L$*&37KKRh>Tk^|`SYrTr^_`|Vsi!v!VDgS=d}8z9c|hQ&w2p{p-Y<4L zeRT2yCter?)KLE>9}7+zthR&B86gT5xGyf+t9iFDYSI8zLUVHV8mJ2~g z1*2Zhen2LeW%qJ3b8Q*_c2*AHIRPNrmV?n2BKF|ndSXMe)@Y8hv3xp1uXbzlusM(K zx$&6sYAFz#7QLTDhfQ?N?gtzZX$v1ZPnC98!tfqQ7#;_0v}CZy$pEM&^!qGDapcs^ zI-93>7VrL8r)S$B$yjMA9tagA{F3uFmx&JRdj0Xty>LG7tk(OZP3=(3z}`R^B!kZ7 z8k}T_G*Ud=b>V=H3gB_T8LyheM`*gI$;M4Xi*D*3HL2i-b~nZ57y;E)`c+H+GS*YA zmw|1x+6s7=pQa~s)#PEgLAaEkGC#YxZEitt^Ioon(C8o8H4hon{%Ve&!s9jL5>=i& zAPL|jB*xsBeyfuruvEDJLqzrLMe7qPzi1Q9%*<7y;ZPc1AghNA(qOs-1!tC}4QKyq zKMFU+*ZElRYxF&cE*V53S;UXiP~ta~bJC^xEXN@Sn)8Uq4U_G~Ro&#mJl#wVOpPFr zsX-)qaGT}uQxJ|mrfj0%%3qog&~NzQMECz&(6BcTFMR#+Eu^1~(U;=~gQD0+MjSPo zRjG*H{=az9MGb=Ru4*0Vvxf6p=wN`~#!>-aii!*Yco){Odc+vC4=#71jkioWfV9dDaHM z@`<*H_OwVI9<XOz`;QpE1>mL9KH0*0TGoBf^CP`O!R#r6%_0T4@t| zQAe=*O(a6R2=rneadBw@_sZVOJ3uzRtCh+AOJtp``4xxpZdvBW#6og$);pnsz-4wF zJ7clO;xgoO>hMuph)^Xu2rMRsomL3ixfy0^IAaO6LRsyEvA0fMKu!ra2ke9V2U;xxpl4OYJEG# z=$Q{B3OUpzHFD(~`Y4)++@i?EGN3q6_kTkKsAxq!LisaN8vK1fypE1LH|C|{he*uX z*~-b>2MHH$LjfI=NY*!QzK!fQ_MF9q#M$7mc#`;{<^=&r_ht@(ef}r zk~xixe`vnRUTymHV7Ju<$PxbV3zDMnPmYvx0PAGSSHjes_z3HKz_~5N<@)AJHcRv8 zk(ZI=V{o_b?ggn2{bf?Af^f!IN$ie5acu@w(hbEUSU3i$R=x;jsymXtH2XbJ(E z#k1|93CF``-gB?nqhTnE#Iu{--s-!VuPwA{b(X71b!2D8gst!b@#>s7QuD@r8qE)>F_lj?t zx|C|k3={z(HCL6?R}Ub#yC1yz4c{z;1Br`{u=qUu;swp z{fj1AoVx_qZgCcOZhtDOb!|GWNE1Wm#R^{m0kPp{Kmb}OK5yNIVG?TCW>)kucWiyHpHJcxfeo)S$ z=nq>9ryK9D41cU9)u>Cb$^mqjx9&`tZFUzOlvjn}*&_yV4Qqp&wB6`VK%a7%<#Hrm zx79Nsxjm2~Hy~NfeUK?kwR9YCP%kF$#&ex0_Vq}tq}?Yrw%{^>N0i>uLmx9Z;j<>h zo?C)oZlFB-Rf3;XD~R)aT>))!>on|yV!D5uo4Q%Es^BFJ?d%zBOEeZixZ#4_w6GaC zocCDnY@%%ipr4E&Uf{DJ+<7q^k)n=CY&D5dNvwP#LZi)3BiEj89jZq6eNM)A4g=jU zpaMu8kgOU6nuaJcj=t0CC=Pi@J(sK>`Ps76>;sZbkofXkpPd)ez23^RgAYrMtu6h! z$+v#VcW3wOMa%^ep2|?Q_?adcta`ectT`6G|GyQ z;*z!i$40M==t#Zpr6>Ma@~xa}ow{Yz^KFQ|%%V(#r(c>r!3mvx4e!Jhf$0%Cm*Q5F z{#hj9Y^Lm5ybnG#lEXadZ=oQHA?U=QvmuTDMnaqD3I4A$VV@w?=*Uu74JRrK;`Q1> zJaW6tt~eu$?Q@GzkXtoF(D3V_;p_H`Dv($jn80wjEiRGwXv{O4_HJi#B1JVEh3oSQ}UtY1H1^3G{ARE@R0ZWHmlW_(T#l0HZyR^0J`SNI` z+DZIwLMK4aK~BJn$WeeUFXuc!90>=xZx5@=J#1Vh1mN2>=Qs!7;65p(r5hl@{wv(8 zZ(ON1Lut*02B&FMxo|NgT6G~NpR4b-2bV_Ksr+%5aTdJknr;CdY`MQ2E4~u2jgi8i zaI_3rHHes8;J^PkGDYnQXaLC{Yz;^ZGvW7Bg=?YL8O2mddZubtcCqE7U!eC`0 zH<$Wc!uVa)#f>W7zWLiZCRm2eft($G?aI(_a$Zl+4z$|_j2Gf*X3Wnlkq;^wwZ~uBB5<{TU*L0AuFrN#1>1d7a z5N%<#>Z9cBk(xe58rU+-^{|IKGE#_pyR~%%uIu|ZVBELOZ}~+hCHU1B zTC)$Eb6*jAfSt~8n#D913a|mQ97PwZ$&uC|@;I%pTZ7}9dt|Cv4_vu-FSyGyVBpbl z9!L@Q!-vzgdaDXV?}dI1OXSK1ttK)-je|aS*6BzWy4+f^)Esc1rcnse=8aJBT+^v1 zWBEaS{BwwxEzguKM71TJAtENo+aRIp&%u7ZZ^gmE?KD$D5aPpXST0qV^i!~`$xbX3 z!-*W{b;8StY&?y(akJNK`44Hw#y~>PK*;yzKBOa{qa=UjY-~PuDRsc>>9KIPu3{|W z|7Se4L;UNCB9f#x*SBt=`nkpHjyDG3(MT-h{pL?&r1xod$xykryY^Jb7UAx2*&P6) zAbUfk0(Vo}##(Q_Cp(Lw@_qw=y6Ta?$m(mF;{>cXnkkBKFf(l$&d7t)!h;M#lIo1l z4TJGmG69K^Y5S_}eL|@6X9<FN*+LIW;yuY752n!>G?;EVmi*Ns!=jk?T=hR-vb-l z`va=dTgEXIO+v#Kvr!!?JB>XFd~rMnybYrN5}^w_uxS)6)7xl}hC2>*nafEvLQTxt z57LBE*-}%bC;YM;YA6$5Sa%<>q|b)BUesMv0-id+kmTTJIa0gy8}xjQlp2<|b+czI zZ*lI+8rbF0zu4bpO)+RF&(>sW$M{dPtaCSi3b^bYX#>? zZTU;X=9iA!MgqJFh(-*!5xMt}P;HPfc01J*B1J5h?GKKj?#;nOKu7G_D}4iW2yw0x z8HR;tyMX?Yy-J2L>{=iHUzTDAv>i)Xx07A;5uMia3?=iOiBNj=FQ4%;A4x`(95eTm z6)RRzQbllBNsg!t-64xlCA4W+?sT%r$5x(y5WA}`9g#SrIe{MCA=m18yoWeCq>7VW zzbku|4!L<6^OJ(Bi2I9{7_MV=)-xlR(9bYmj9EITKV+V_{ken1WQoj~x#L^sw(i}o zp>B*)^BrFAe94w}Ax~5#_Z3*(GJ^&H89hfX;BL-|o}Df%KztPrBB}i|XdmJCTe;B0wNCeb*G9H7>{CJ*-Mx;?po)C{pDkg1166S z#f%2qmI8qZFIGX`+=1_0Ddk3l!{DWF&1p`Ph&~QUFn0ien%0cpLW~`=d|WSPr;ZdF z%>X&#hubf8!BE1NEvHoqTcY}vQ8LizusW*w%*&}4=NGEx8e`dDym=5YfNk=O=B&tK zkv}zOzSP!rY<|*$%vJl~l5=s^iQeEn{+7hXlTa06@u}p@4iN~CL%T!S&&9-WWrrqe z+LYB}X%GgK#vzs8kBW`S>gd+%6>n-aXb7p^od$;?LyuB7#c%dSc87%bt=T zE{SD#T>w4V7j|d;#Dsj}lrL0$)MW(=OfAO)h=|;@OH25dl&L4-PsPfLnnYxx0B>zp z=LlK!@czBh+)Dh;Jsmw=be1wTKcc!#I^ch-!!q=GD`@Yfbbw)7R+haDIys$Lq}>1) zuhEXplM&EcHThouP-9luDJG!^1~mZ-RPdGx@bO-!Rg3cDJYRb*@2QZV!V46WTG`k_ z5z+jYOzM8oTvXj z;^H(Sa!VG_slh^=-mQ{Y`$?k%QwE&-oS!O!{sgcv;?uTX;rjeQ(G}(c$Skl(!sA_O z6pU*l= z8Ii19f9$7l`+R5Cf5g$vzBKXe4%vDD{iMpP32bZH>r^goHDqwgfVrIpVc-;JXOsCnG zIAYsGCF!ccKThAfx?+_$z=mS_1X1Nwc1*&HHD|&GbssTRV(ZhaDmrp7jk z%bgpqMEa{RtM8J zX{G0afMnLc)Vk}~mMCfbcn($XsfCegEZmGG6ZA;rLQcDhBugpwC(LJF&AxhX(8Ss{L|<02!^+yB|n*a-?Y^W28j;XsE7Z0_7G% z8V_$Z+J+tp!KTqjh;==@7OP*gy|fP>CMPZKUknt%aQR+wDiCZ9so*i)W@GP|2ShW& zIN2>&Z+}W3?i}WL88LzGw>qi{7`8W2paJT;ht-&0(OB>A?MR1I`&I*1qZ6rI6L7m? zP`sKyY@!szygO$ROR0pn=~x=H9U^}B9p0q+X#$nEv3y_8)kB#zwYNithbv4iK0J90 z3q%q}y4H+a0|E4@?}Md5_-(1j>FI?FlODSIunYoUIjOB)I=7!~u3N?fVGKTQBl*{e z$T1~!a5BXG-pFghb2l`kiz$4VXxKGnHkL_rQFL}?8+h|+P&EV=1cu++oMy8R7r(l^ zuVUy80yGCElfqC@q4k@E6&i~rj18kQ2~pYk&;tFNshYES10JVkwdeS!CAup+oQ+?; z{;3jvK_roqZex-;T}!iu-J3>T@(G-8|q80X+xkln?0mm^gKeYGfq!$30Y1zB!C>?>J@# z;%Clq3L6@g+SP*pkZFR;1@}Ts)vWHy)`vsb*J%_l*a|&_2I#^ zrsOF-7F?z%ZUVYyz*qZr^sSW+eOOMvkrEZAxLgFRv^jMlCz>vkSUnJ@6J3+gsdDjy z?Aw{PHWyrzZ|_V`gv2u*a%3+wU96|*;X3H|_d%Ss-rY<#~K}gMAhIq z(UP8-&KF}P_G60xGwCnH382I#@tw<#4C`ud(p95B#{~{CbQ3nO(;*Bv?N?3XwnvT! zrpUGMMPw@e7z8JI9GA2E37t5TJtA@`=cYxmr0y}Mkf$Gn~l)tnOu zuuN3WMH_7X0RMluI?Jdk+i&Y5ozfu<(kVztcZf;|(%qe>RvFRn^%{UNoyJvBf#Uhs4gT7v;QLZdtL|KpDd?5bs5)rFnNtsYL!l)w zPdk(|@=`P8kwI73vCpe3D%JRx=oog#hgdd9G%8e~j*0)bm&^;#sR!QZv?j<6Y!Xpa zjJ~Oa*^x4Cz;V?c4FO}iwR=vL(1t|=3rfbsw1 zz;KqiU9#INpgv(ax`i7=USKAQ5&zJ-I-G{<^@c=VEl=N*I<3|d`bKHOMK^}y4eVSGYky4Jj@#eve`rg;+kt|lBn)1=TumGE*z6-r z{;2@@h{26Zwd1=`I_WfYf=|rRuO+98hRYs%aa>n6I`7r#ET9H)=XZub%s{>AeSz_1 z&o~dh+ek>vJiVaTlOsB#b+1FmkxSq)iL@bc9k@wo+}q=TX|H#@_Q5C*4}9N!vJGPI zFV`Gp5tP<|7#lBkTm8X1K^^1m)?!*w=K9jHvYTs3e`ShF-!30rqF4?N4T%>)_PYv) zZe5yP#LR5LrN5iWC|01Yo_{YJ(YN2rMaZ?k4Ncaqta2qtVb=w~I*t?Zm<32`Nz!r>X24$4(DqncFOn zOISc^vc$cYMwV8|X{dbClBK0)p!|;Ezqtc7_<4!+kz+$fLS7erJA_-UumjjJSte7X z_l2|+wUlozvD%TnPXMD7S=U?ZObFJRQA*1(I0s9w6mOvY$v<2yH;d%CukTP{RiguU%(dzkUnN;;x3S07+$N8_a~V*vPxkI_{Ut8MDF0A4UmuV`IumCHEM7}@<&*_3XxS2fs_LHO zXTYT~iRBPwf6X(P6Px@*M1;c}TgHb)|4LuoC`VoP%c?@`+fPFk-8~+(_#TV1#CvcM zDLk~)a;9!MZ-EwNZY{xemZ)g9Gy_Gy?b)FvKYCf=-}VAMbmNvy#ras7?&9K|%4t0! z@b^=2UHni}3~_B`0+Y&?Wo14-X#K)CJeixwKO@vU%A4?ed!V~v2i%)UZNp(-ZD`Ay z9!j&XGBJ_iwjGMk36mctW#2&G?lZRBLs=_#jj%|vud`SkYanZC#(OK8B#g|4RXONf zpT1{@#TfL1?)PJJtuXeQ6E`;;?@*`t-c(6v5OR1(uwIWQ`>5jU=y~H@g#bqtmWGLy zuPEmTevQb|sB>U0Gb$PZ3IW!b@%oTsjMQe+{^~W|hZsBE5w6C7`Ild^v zpo>=iOW0tYD!-~M6q*cHAt5use3QLld<4^UdxMQgnn{&>vpT#ja}JVmK++1;{r_qx zkA^JI6o%Y0S*M*{lu-r7Vh1we|5W(IL8h4gA4hQCo>q)7fE}ot$VN3>j&X7 zU+3Wt_Azy42aS-sx1a6VbDU+zI;qqfXj@l~4_ZFC}U_?X^@Xwn94L+Boj3h2xP7m^>73FqE{{&FM= zY!mM8U*FJJw|$Ay6K^`4#uEg?+rshblGZi(dk7!qSw#Bwsv8A!X9iz+U)?43(@pQt=t-_+`_AwSt-W~Vwm}A zK$U>nVH}-%i3p#tQ7pXHu?0RUyA9H|ee4OH7$WynxdYANd|dth@({hq__s;*(uS+ld&@sptLllKU90IRD;tzGhs;&1O#jG?y$Z z>nkOE;w#c#he$?}HmA_jVa~pg#Mi7HhiyFkv+__D|85#?I68laa;T)PXy&uMM4h>F zgMauOH}D5L)wA+PQ#bKwi(2mYU^B~LeMuEk`IV4uxI`a}P6edf^bmn6rzhjNYXDjg?%TiREwxiW))A(>Wz43{7i{CCy@Jc}n ziHj+5G?+0$f{h&OayQQ8GZ1f#bAJyLj`&(0&kOULT@i)C?No zb|8h2dd~dbs5`pE&t`y!011>Sp2psCnf2|aWo=MsGISetlz(=1pV8N1u7BqaM6vHR zcH@D8c6MJ7W`l;Fz)I=+ol+JBZjtYJ71X@qAmMinrGPmuIs@GIkLz z%!0%pGNcTY|M=cuJl%lc)J-TyhGu<)5zEi1Y2*~NuB_TD+-}RWjtW{- zp%oy&f2`npDWAKlBpnWZmd#S(rUOQ=_u5i$X@ze?i^X(<^Yq&@6Y7DOkdu3xXd^TF zucJ$aM19TuV+Ie+0YJpaXuc*5@3L#QAyxle1FQL0De75J*>ixq`O&&UT{(;8A=O(O-Si-@FqZ#9~sR=Zc zoxUGSFsH(S!rR1Wmvi;;J6bu0ajbLDPQ1B^qe40LSgyMz$S}oCc`&ZYAoBGv(stv-Mg9e$d&(|rvy9-Fa5cynsT6%!OOAt`{~XHMEO1k-fPM|p%QQ4 zgf95-yR|*PV7^A5Kl`qmQvvXo`Ux{GCgTrA2koWo=@i4^eXqy#Z{+d5_#IqXmG0DL zcp4)eUXk^do!xG{xL7#KjJ@9B{&Tkq z7G8e+6MA6!*FKF&*=G8qq{%}2kGDbhr`|W8Cr!5(MAd4t?LmQGs4MAQnKf|PM>`2E z28u!?8q9}4F^0(T$+S9&hfmHUDfO8vngmdir_XKyFEyUTsx<$0{JJFPHfDGx%7Fkd z!*;2o(PGoOf(i@315Fc)1PNv>ah%6p7%|gVg%shG6S{!>(2D~tQS2{;qsG(YQ8-u4 zpm=~>w8>b+UqpiJ-I!CYC^?H6mm;-dt6-7q-ifJY0TRJj}J_TpvUffL0S* zVhg`vK{cppU$%}ZXGI=XQaN>mnl@158B60c3W?aA3F;#zcbEUr8!}X%$3_jQ{%+KzRZMo>_-nZ^g#3r(ZJeBj10SS z;nlojn@|7!m)DplXG{4ZB!QYcH96{WlVXtaeKF%Z|5$cm=`BeD?Wi0z4DkHhHvIhl zvl*uk?M6ZL1I2xZU5XQ-xqKqZo4=G(?_U<%6*r#i%%f%cEX%QT&P)OZki^L(eh&eY zjp6HGXl}R(@q!SWGfb~B_eD=v!q43d%^e5VQVKKGjIw<_!EDtXlx&z8=!=TZjtAZ9f9SaM3bRLHr`zRQ| zX~SqEuhdT-KRbI?-w2JTSSk5#`*m+Z?(Yak^ZQ#$hgxQ-M$t-7Qk`UpR>t&YnOh+OUA!{kVKtbH|Z-L?Q)$n*c}V{zj1Z-rSti9T8Kni z+5MTB9{a@>DYQ0hPXEBBVPf7%1JZ^j-h|ty^v^8D?YJhQ>v+c)+p~&pPEa4$q_AAa&9|zoDvwrIa*=Z-8xvHY-B0JYgG{e1Q z2M1~3c%{?2DL=#&LLMFSGq?v!C~4aE87w`cF%0a%pTHYYC|GU?l-1qga@=G`g)yoJ zdl+A!U4VEYXmvpf>9%wa&7Mlr#eWNMu*8oy3A=qondW0-p^3B3;;E+#^TT&G9ht!F zt@1rNiQr1k2W)CZm?GtQW>3d?Btn8IJdMrw1>fI;c2|2q(U0!Y@!`cm2GAS?Ud>hL zhQmUFS;KxPPD|l!4R^y~ZAQ0;5^t`@51HYH3nDIiXL{L}BZD`myKRYgLz5r0EOrq8 zOMz_)3E@s;TDE;puOZHY%JB@QgEJqWPI!*%Omr-(#^llSjNl`?ciG3l{A4t#opi{@ zs{PI5x5wi@qnS8fa$O+Q^E_!GZPc^h+%gxqr8ro(N7mi6c|_N);?7WZDX{d{%oG6& zSQg{+iK{AN!)E;BUwl)4LPJ|dawJ$L%`24%#hjQ;H1Mh}7xyFLPedyhX%b5kKz2gB zQAlG0h@ORw@@k2uKVCxbmVxO<>{pq*ncbqT8}crlg;mypzPp}c`pejZHD0fFB;5cd zPodoVQ&rkR8PHM)eqFwJKHICzNH>v`VHq!|sty*Zr0^}PNOFlo zQbI3mB=ea7=x&(P!4+RUJzNeXoL8Y$J>w9N<EptcHr`jo1gW8q(7luc;7S8S>c(`k-HHdiiAordJuw1)97awdwq@xjuiQr)Pi3jv<$4 z+X7|A-q<+g7=5cNyYLrlMcsZBs@l%fasN5`>AO$9_c4VShdg=4)Cz5akgryxe8WKW zOnfQz!%wv3w&-Al6KNW^vyJ7s$J$>Ir zs~%Eo!0{TM&mrmlPN@jlb;8&;sEoboLVujl&Z~4EMQFmpf^_-iUac`-$)J8eb$b1& z6rG3f^A*9`rGoRl<2Fk{@fc<6x(c1K?w3H1MKFcJ_b;FN2X_-3%tL6{=&7=+rYmA! z7l;=(UVp^BxlNt+Jq1=AZ4+THB5LRlRw(|;Hew^H!RHg#NDf_xe`SYfCV~gy7FP!D z!Z87N-45Ab^`98mTeW>tQvSw`8a09Er}ftQX#m>q=f`~2;moWqeLZ#2JexHPK6=x` zWJB}8!xomiQG+Kvun%vxeN@@L|Ao*!d8z$FWH$9Jvv6biX;m1S!-lGTQyZ5=_NY^J zc{@S9N3o3+I$gx4+vF2`mV2=rylDw7dbW z2G>Q!bbiP0y;wVtB-S4!##*1^fNN@Az23|Z1;f~lp>mQ~fbOcPkhfU8!G0+;1(2bP zdbhEJf@zHuDS9#t!~^I?%IYp}?tg+&kL@(MAGukawC_H^4L~in|SFG*M}6Ci&D~f4}1rluKH? z3y*j22Q0sGqmB=GrE03RZ@j{GMQ6?FAR{OEk;fit%w>=m7lE{)UYg$4w-i^NjV;~F zLSH8`e;5D0np?KAopdIDDbJ6B9{z{?ga%H!bktBOI4cPS)-N*&>QleNurEK|_ESKo zF_~O$ICuXRq(QT_QSHe@t$$}V2`q?x(L{f0410+FawBcAas8V%oQz{`Y((6aAo|7+;|3XrqE*%pZh@-z zQgRxab_hRLLf4p<4VB>6nDo&{M_YC06ac+7XPH>O@=+^{5o#cFP8VDuY4q3bPD!2y zAgv}I2jgJRAr-qFd%q_mxhwX0(`|m?%j$zrRDM+F&!W3-gCkY}Hjo=&S>Azb{s#cn zz$DKx@IGb(zEI)@at}Z|EjbYgLpH4^xZZ1h^3z!%i%VmcMoM^D0p7!41 zF%D7#N{}-Py+@GmJoE$lC}U)H|A-|z0y&0QdVKGY`y0!+sO{(DlaBa@FV5&(u;eiw zt6?MB0PC>1)imH0ZTFno1j*+u+?fK_gUD59g)ssT`Re=MK!A~Q%)U`laW1GT9OBBo zP9(nSimavWbwk-T)}Z4b#VmSS2WWD@)Twu9%kl3AS`Z;I(2I7}-s!7g{Ix9u{SKoZ z&M$jyB6n@`=fHz;GK9-VKBcVmxpJ)pE6cqowu3wh6Ky`Ia`?pwH@U(V0GL*h-^!%d>X3>{OrjgwO{*`_^R#Vsp|Py9GG1JsMh8Ww9x3DWd?WAk~w;WfxZBsd!dlg+XhP zQ0?_rlUXCcgDiQ;55NdP}Gk`S=YhT z)A8Gwekck?{m}FkCtRcP4Lo$O8MavnZMd2|`TlTMFc267nag{bzR5zqdam7A3{!e20GGc-hY{dN4d?7TA3 z>2<>OgTD-S@#n(0laX6YI=r#ES~$878W#F5aV~-Hc#FnwGbHU()9ww~E2>Sjn_iKDa}uCa?W7R@Usu`{HCLPAqf7_pAHP`x=!-!qAt`_=1zO^Hu-3!m}ZhiPTj`W!3*3%%ZfCqQY%%L6!s-ohKe zQ#c+6O$nL#ussuOv(BcTt`f|5c%lypT+R<@$KycEKs_awq+O9&Y;aYpx;wd8)N@A!xTH=SIBSihZvU2g52lF3B-a>3spOjL1rhqrF=M5<3WO z2^3>%WYyfteGZZNLwh_qAg)enkl)xC-c1K=y3)at*T(p`ay2uH@4@z!Vs;%bzv*`d)?LkAs3Gzok zoSmmCXz3isoMf)778b+fWavA!LCR{)3dg(P17@Tp%OeDz+J$*(4WS-oyh(p}Ktvd0 z$*LC8fq9&*b%;D&@gHGG=_P+Cs>}_O~f{lT7}RZ%1w9TdH-FOYUQl$cj=KW`B4Q#VqRE(a-NT!{)B$*kcTQ$@U&{^5 zhpAb9T^F2&?>f-(XRizk`ut36lj4_Ne05D)rr)ITI)~-{W`i(ag`wu=_e||L&ybQG zAK13QHugT^9ZB`l%W2d-aJa(1cduE+a*H-s+*MuN&%omVmUHhY{Wo0ot08F?dv zbwe4#(g4}t72In5WR$bnE0}qfdDhP@jAx$S;0l!K=vY3TnRO6H!rv`BPng=qf}}Xo z%Y0!c0%DtH43XMIs4K=YF9&)XTcZ?{83q;~YLgJNE$zgRi&g&gb#)%7yU%|#2}}I~ zG}UPag?dnSuKLQfUmb=`Ha}xQD_P49uUMt|Ivcg@H2ydOc(6z++!F*$i+YjNCN?Ld zmmC5oB`8ftaVG;d#3wplT+C%YR@CXs)W#V^z(Hz9*sq+Oj+bC9qa*5&RfwCW6axu! z^5jw6jikSTmMQ^6JlWS&%<+!Tz_cN!+7wuf}>2|vrC&DzjdS$X&e>i*7rRT(3O+>GCN z>OJ=h{BhFbYCR5V6?)NJ1cnhV-%J9cwbPxTc*cJli*LQe%|y7X9yw`)=m#%WeEFX# zk#mD#@dX@E#R0+fM&of9xU`!CXimE?Kx6u@DdaZhKH%M6c z_P}{W_UF&kzbda*Clo(vHo^Bhho~j}0U`7erGT=LAg3$%g8OYO<<&_T_itHrnU8?9 za|7+{9#>Cn&exYPyn&m}oP<1+^#wJzUVZtL1~(Y@&(JseMz?^Lx+Eesq2~`kgFn;~ z_a~?Su)9hx?`-#$;Vrkzm$qg!sJ3LtVhBRPesp29!HuN{s4EP4`0IF852nwolsT>s z>}6R&c<&o4Sh44JjTa$mN@DKGBIunZ&1B#N7X@0T%=K&T5@S%@O>0pwZVS&OAy93jlfL~c3}a_ zb2_Jrg)lP7Rc+s-Ic$#Oan!P{I0wl`C08qW5CkWbKG`%~PZ3qKs5WtpC*okPnp?Gh z#$7zPr;I$}Y7UWL5k!x>&szxb%iaJ(`U!_bSWrjQVgz`QFRKO1s`fTIgPtr6m9>r= zkrsVql#mNjmNI_*_XhKkOrvDA)O^Vrv-gzeEQO7hhaT-;(1DKYr{h02r@jHiB?LFt z^eBTF)brYbdBhW&AH{1k(Mx1JiXsX{Fm1AGWmsnffWUIcj8|g%-q+! zDJ`Kqk%UcaNovoK%cH6#N@iIivkD_wQn<8}mAN9(EzsnIvA9`c)_EBE995H4Mn1c& z3@UB){0h`LE(up<0_6_0vL-%$OP`Bp8`!NtSm6Bkb>ddJtgHRok8ea1R{G9gt#}1< zWj1)9EOK8+xNl#V(D14l+ziR%7b@pGVHq634D4<5{VrLX!}EecP9yZ60@>0%I5Y?h zvH>Iz^Wmot+?K<6xVZTmo;95|c;~YMJ6WG9X8G>j0)|`@$Q6vpjYwb79AcIac3e{5pDFgoc(y%xy7m)A0>!#Oc?O6LEmzHV+hS*kYj`i7 z#D`!;{fCrWvFZnirk^o1x3CtV7A05Fa-t*$@iGsrQ8w4@2bq=PwaXb<`Y0MECZmyi zh1QRgSIxSm#?#FaTCo?Ni=*!r;%a&lUyOHgl?D9_>;vlvD|#;@Rrem$?aUac=fa=| z#GZv^`wR9hq}Y*MdjF zqGl%XMepd{^;rP$X^n8n#%|+l;M{P19xEw{Ex=;s@1MkP@dRseTCEQRWpaSX8 zpk2iE8gmXZ8m2i~C_>JNkwhN3Fbyr!6B4*y;S@<_e z!)U(AU(YJ1FzI?fMyVw3%uK31l}ooU5f zL$*<>LIx6UB}Wuv(Bymmw;qjp?rQsf&*OtUSKWxU{UbIE5wDvQ+~qV0--|vQ^)EHf zTFRL%1FyZiSh5`kHr~+vcWfL$RVD6L$E_~JpMN~90IpORxF|q}8fr_q5KWB_ z2wi}2DSaJoE-q@`DaV60CCp<$;?W%dy0~s6*`p_T{js`Kyjcd6@r?*XPYu7b0+inD$m@7;O0_oZyoPT5dFtnL*lIYy%e zMq7||XfyA?)ho%-;o8HwDZ$hi!E@j4>LH2E7EFd_uT-!3a_YVZB}GMZq# z&V&)Ut2qmcuT!sd*OsLyPywK)6*OzAvImC?W3GkF8+)IF^9h%ReU?wt^8poV`{QQB z>trpy|1c_fd0UpB7X~q#U*P2{{HR1@t{5kJZa|gIN2^OK1e7_@%4EG*3*Z-?I>1^e z1y(scTDhgZvIdE!8=xe@L#G6uDI$9KZ{pKrWyu2mESqZ&wgNLc@fA(_EX5KnMi!c? zPg4CXHGC=y;M5D6Iu|i9!!S4%+*#;l(I@$sas_MfYMunAjmY82fnJB>v?eWc(hYEs z74eVrpui{EFA5FDehV5YsbEPv+YA0+s~5p6(sC{)U)v7C8j^ssNrC$t%L%{Xw@JuH zOBGoxo>0GTVTcT+bY1pSpa^YtRL%VE9OZXo>NfH-{NhnW{r>!cJOvNQbZ;n=dC7;2 z5|RMqMqiO8CS(etQA@*3w&8w<+&FJz7fnx~hEi^lXhgNTM8~PoQWWh3z6G6(r;+qf-u!T` z&CKcL4MYj86KCz-1m|~y~RtL7phKqe19A7P!EFE@Y2dMe3T|a<`XF&UMThI!e5=scXsM0JE62-<~ z6OpTM5UMZL6>^|Cu5S>3BCmN1d=r4(UnvwfP1z&A=K5)iVK{E#XAcq*{bXG)1BmhK|DKk}u)_mca|#-b1qxN9 zYxDF%nGyqkv&=<0QJPoA6WYkj5&6vgIBxj#Csk7lqDC}2xd))}Hgr}PxKB|o_d#hz;5Q|7*PkBj#wvj^ zh26_K!@asV0*yLc3pJZ$Vw8|oCY$gf*4{q-f;B2Vbahy?psC;JHg`cu!HNB&<*Pvq^uXIz)x1_*U#>{=4DO_N9`%k%pg$>KaHGEW+omqAR6| zJA*0%U*E7?KIRinZ;P{z%-!SiqjJF5r zIIj=~^+b+DN0mdSa{+YhY(IG2xji7WqxGQ?SOfly&vca)=NWJxIEAz3fUR%vP1&A^ z`8eLOcHqQX;h4Rs!};8#c8jFu|6Ga^0d6`iT$Q8jutKp%le1Q+#_zgU-xBoE08$ zgM$7M?~bcgJxG7fipf2>14uRL(_iyTl7qx}oYWU2SpXQI%4+RiTF=@{hhiacL!NDO zX4jm{jEq4oSB2)Ma5f_W4n<=MSUi2p`ryR;K zV;<@_MRRU)0-GO%syC*hAN{K6psJODimhbH);f=NLNsNyz@@dHCQ}8(syp*i+Bm^_baA_-Fp#%7};XOE((E`6Yq%E{?(|FBF8r2HVsQ!ef6wfbe5YN+Dy>S2r@E; zsEV6Qmw(BzfDSN)xONX=ZD@)o=+Qkt)Reu|%$g!l>Gy;+{sd<_RAs^Kl;&^Z2tf$x zLAvADVjA&cuNFB4uPFf1ZN77PFp`OOl1Zp(*ts5%15oAHKv;Pn4mqms8a{voM#?<7 z^5h3WzJI~~_BFl44U0M`iH1sv_9*=L1laAT0&J1nn0*vt?P#n0C_YC-k)ITOUBHg_@ElV6k1WK9?(mx5#y)@^c`FO2A^o zb`h$=GPP{P0mwtx3Y@0lLp{Tu+6I2jMZ$vGzIn&nP-p4Rx+ z^$s*~68F(H{df_9?D?E6m!{3PWXaM9Nh6r$hT#(Y($2qivcF}?AG@7>2eKFCM>XG) zCmkc2i21OmjiDGG4L0j%0)WSncu`?w_>RR)(uS?-3{)*s9_?=@2SVJeu*VT zBvlu|zJZ4=(zIdoXV`*2lr0F7ya~l-3|f*3lT5`3;+q4F0V{u$m`ph*9~mlg3^0St z?v#feJP=BF4}st1dNyxYZ9WR+a4?abV!$m&JS zR70PM50W$yb_+qILI7?wz>V&UCS;~Ia2s8Nl7aD$v%aN@3nAUV3Os|6u0HEYD8fx?4$!w`tctp)*(BdC@be(Wa{JkRxk>C^K1Qj_! z#Pu}yUfR~vD1Ivzv*SkmEBsGE?C4hj%`U3!ZCP&C2%!#o>V!g8!2X8_=gy3oFB6d( z$ykl<@}xEj_5r8L{W*l~q>xIx_}bZ)llmjcLP{Fc&by{4MTyRbHWE7X*Usm3I4S_lRro z;{aZ%sixUUT59p3!jqSF-)o`1sTZKnLU%wHLe56?tmvJ`D_a^a!!$Uvdx$pj(^6yw z(gGCb)og+$c!E`VdM9lXs^|M*jHTdyk0%(astbc?B@%c=K=5aUK)QL8-%(0aj|GoK zkfj(vw8G8HnmDWQgyuq81M79l9)olToi6R^h(bYAJ!0*}iIt(uurbSzYWB04=XA6y zf33pGsrC1drq@({7Y|W?AIzM^5ki(Y3T;*Cfwoi2erjK;2u^T_t-Cs9n=85CpWiLV z{ry)G+zJEzA>j06j|2_B$a=y5ER129G zt?jXt4{_PVt=9R>n*-o^X^0;-Ty)Nm2qXV;YDT-4*YDTo4)4<`*_>^I{RYTn6~mi9 z?tOa_8({?g(T;u_v-dl#PVdnntJyV{cz33)rYL4* zqZh8UcYom%m*#t{hrWMdSFj}Kd@jGQ=eAS6!(kHouoctR3F58tjO&%LOPZhJ@;o}? z=PgJ(hnb?6VymKF8+>`ACPusP8uv>t3G}o9G3R3i&CMhD*}`jcAFq`UXCX zJ;yDl2E*BRPaNnCS`vNne9pXgx@e*9RIt@EQVIQ=i<#L4doJ&D0Pz)v@Aee-OHke( zko6On<~QFi&K|akUR$upY43U+T6%ljtj3X^44Z7z{$9v$rm^(v=ur0DQa?25Jj%YT zKFd}1e&Kh1c06u$ia&z7lHPn6%gg%BkI8{POIStbE3WQPp}?Z~@rQ?J-jS6)Kvg!5 zQDO?f-vHy+<^JLpAXe`F4x!OSg7y#V4w$EmVOX5VYX67|4YN6x)e^}*7geJDudV+x z8-R85tq0g^P%VDLJYkR)m7EOEM=?gZmr~w%1Dn7TmqBg*s=ogl5G#d2|Dd24cHz&b z^mrnW59B#`*^7fzned*2_+HM@?d&87?nr0;V$SQ3Mf0-ymmP#m`F?18{8#%Xza`6UvqQ>w1ra1e@={w!32$}26G`Ha9+(vtdgem7uN)}lkU;sz;0EQs2fhzay1|Q3+@cgS0iT`_aH-=V>?)b83 zQdzsv-attiBAsgF^Y?mv?7Z42WAIZsd&|jMTJ-ENHN;YBcOCGi41PLY0J>VXOcuTG z&$IMBjoq2yKW|{(PU0doPWw9+W`sq|ujkW0!@5%WN!&ivw%#jU-u%Lary8m(?2Gv$ zMj4!n-&`0EO?zUb+bLOJ>&-&ejNm4crPW`D%ro7ak1n=@!`Pczj}6zh(zDo=jf8e2 zHg49k=2)2e1@CR8_aySRgv<0L?(e$nM+!-vLYQ6d4ukQDnOmBU`t92ktyjvu0`Mhw=PqonQcP01_mhTli{0?4k!~HM@{kjeAn6ei$eP7()5jq>awEh-< zRNwY-aA990v2v;&`ghvPef^_-b7SVA_eGW9v`M)3^_R9+)>ly~qM7AB_EE1#OJ@BT zU0R8YHv7(_$kM>2j+Ud@t9b>1H`Z5s!Kf0Y=d@l^8%ES0gpOhqj#r?yhs&?M*a3Nn zKo=%hJb&XyK;S3tzO00;20zCtx6epY?KVyMzjv(8VP>}yS>YMpdoNjX{0`xlNb@xQ z^f%z{pcK)$(K5}A)%K6kJuJp7LZ}K+Y z!O|8p)pt~tlR#d?nw;@#qh3|<<^ptxk*p|gXd5Ylw|$hVA@MrO)X4eb+re(l%y?vG zj7^Z6Rl`g!ZI*)@%6plvee8-b2)A3k^nn{38C{XXg>Z6Iz2LKZRWa3og&_BFq69yX zX}~!xH@6^q=_u6~57V33A-}0gcPQnmaJ<3&L;ima>CyA)MZ40kEe>uUExB3vSm@98 zKu)(O1i%HEUoAdtRC&Gw`%E`f2OH(}RgaWj{ao$@K(uz?pQ|pee(T@!knu>pcUUzin+t zcUoJmHy0iC^o;j&?`HVFxNfXYVlfY8UTun+V>B`C7bh~m6)RlMfFBGbJzq)`f~z;= zb7eHToagLO-0!@CPOl`AwgJF8^gH|4@v3xm|N0Ea&crXtiLU(jyTF2NlqvM-;`{At zQdIaOc!)tGr2W$#crli~`gUeI9uPzkHaP6v@-p*xf4WJ!O|4!wYII`l`kK7^YFNk- zTL+K%>lLGSE1_qykyq=BTZ^0Nj0sAsNUbw5v93%{q5BikW#6RLH?PDtZ;t6jc3gJH zdaFT*y!Gs9qr=YJ_d}J;PCT2@j0T5`_2k4W|Ik1>3(Dgkxe8y+*U55V<-_R?G?(8! z)EN2q3JyX3W~dz2z}zPL3^Z!s?=TThw(R6UiX_&WswHT#o?Z_Y9=_GK8rM;fPTZ(S zQ@3sGU7dcoT(bI37AuZ}@{{Q5PK(N=?)=^-u{hD6Yv7>jib$9YQN%Q%6h6-Ei*;}t z39uKT2ZQb)Ph)ama})gdKf~Z_SJ&hY!1F5(AE$!SP3u6_Fh_)FHWrS-m5$7H zjT^rDv0I^cxEl+V&sDvQ+|GgR&F7msMx!<0S)d8M18PFIG|ejV7)t+*&_Nx@JXgmq zt%^d5pC8tJZjMw{@9AKpp?@U&9>b62!b#;6U`)cC?No+~kE9;2tACGfo?jXc{bK#| z*J6rMjwTu#Vi_$Nt?LhA+9hGBXOfn*e;t!sjvis6>7tMar5mr{x_$$UHELSdtWH2a zxNviG^^o+qsu(w-a6)-EBgCk}9()JhC-9H-`=)-q2n?wkn(b|>A`XG3WSH2GT}Xq+ zK~ggM&yr**(kwrC%>qvpPT0JGWi@04(?{p<1ILglT^?u&V{-**1`2_c7%kN%rL@0h zH7RhL%p#oz7F;XdUy8-}=7IhZPSWt<`g=r=T$E}wSh9ATUxYgUqci^3ZEtz#Sj1S| z|LS*%Zk6E=S*dVvVd%N(Fgl1x`~&q@>s%V|BgPkpfg5RIw&bWE+M4fRzfUdRbbw_3 zzvtZTw6pvGn@C1YQn4b{VWG4exNp-ZeM9Pn)&p+0OJkeoc~96Z*WdN^l%_b>yoDHU zd|sHR@^sJJT7+|rletO^w;-*eWmdWYvU=zwj8`R!vsL~vQy3d{Zz+V~Fc!#dA8{wr z_f3b{{OKW?FF&yznok4ur)X=Nv(OW|W2@s?e-#U7yq$S7Vl0B_io2ny)Z2uU3Fwhl zasH>3({<91rAptpxP<9`TZf8w&AL)Gwd5ZV2zrTL3it&r1~lR++M^z<=jXHeeDS)? zZy-KC--o6YdJ^M;Qu1q6_!lq-;R)<`*8kX8L{+;aNo1VNeocD{Z0qg&sNA>imHGVZ zcnl<>=Ie_o&SyWMX5rPQLzSsi%?BBhrD6ed=n0uUrsej5_tDLI+T1YL`m4R)y&$^s z$%z5ir@^_pZAI5UF&WnDyN%B#9dSHUbabkWlU=pcUsffkYF5dN55YG1*9r9RZF+%| z-GFU?in>v^4Emha#W@{6Ea8xUWQF zgyh0`J7d&(vwC6HZFk48|DC$FMotS69U1Dzj zePKP6g2Xx^$&g8d*!Y*uvm$Tx4$&uEL5LhWlNn2WTfMctKP1|dEkdbae@WG{=uBj? za5h<)>J+z#9g&m~MikySAOCn~wWn!c(eQhmQ&#wU>Y-B105l6%|8&iwJ4dc2TD(Om z(eBv0jqOT3ST~#G--Z)A1rHqsdnx0?O0fSmm$}Z{`R_X5Up*ti*1q5bjq#LB98|joD8WcfFWJu}mkVd*okWjk2_1uHc z^BkY^zVCU@b)ED5@o{kt%*=i7z4lta6?^S<@9n(*$ZaQ)!0-m?{zslxztW+fJLLNZ z-uZwW*P2X)P-1=}75k*Au?n2W07$-!U1$(%T$A#bOi1`+Q;2I( z)NO>LdtuT$WcMM#!wZ`3slY>3Kt$4mCSAx_oa@W4I7O%M$4m++&GBBTd;-sPiav8L zI;}`50iGFq^hOVDtljs-O(rX<=?UMFeVEyrh70gZRlM{xV)ZB0%~8GFrY8|F%WnA6 z^Np*a{VS&E_|Z&w0!OYCTRTF*GqfFy2@ha59pDtOk|y+gKvhS3hG^kh`x%RGiMT{# zCLL0eWm)*B5u3#i%>X$x%0>al9c=#S1Lc`PboDESyzV^VNt|^}@r8tW zHsE0+7wSf;z^&u?CpA~wfVV>r`Tj-Hkj=d8I-Sjerg4>p#Fcd?YSwOp+xyuAZz9!7 zz-1oC+yXA3c(#Y_2W6L|#+oTK)96gro-x0u8SYO@&aC!!fh1vv(a}Sz+MFK?F)D+d zt)v0-azHfzuz$bo?^OG53(M5U|Ib0vAEbe7Hx!r+?|Gj5q&gR?+)oBF zo2?hUCWMZD-);F@M}d7AAfD|}|I;tl;aWGfkDG+&O#)|5l@}SV8e2pId#c%*;k>XB zU6_nt+*owsou=L~w)_BWa&YF)m~{@8t_3!1d$nLUbx8r#UxlWa2wlE(lLqy{H*wwC zX&M=qo6ke6<^djPECC))85pvz1s-qYtKRo=JJjx&{Fa;F*f;LtA~g&gAY$ViP3Icd z5+sK0onAU>4OH<3ysC;}0~X^g8s{9>Ry8v4ZZ&JG&CfS=noT4cVD2!OBbks>?%Y zd<=k-wZ_I>noH>lvVO3ABO%oOZWq-XliEM^}06 ztn`XMSX_G`133WiwO41^M;=j0A>bLzMc^o?fpO>DLWFKRR`w~(=9#y?%wX~d&puPs zYl7M3JsQ>04dPnDTf>Ri2tzFD=S+j?So5Ad;c~0h{UHm!+M5%?90k1Jk+G9Jua2eyMH@wVF%x_UhKoo1&ls2&SUSWzW;JZpktjoV=<+gs69N1 zkwaM!iLXM{-noOZ52bJCcRneFW&EF$#^>wOe@%3$**u)C(I0c}?WTJrmf^iiE@IMX z^pE!8uOgV8ed%49_<{T{ecl*q9dAAc7F6M9s~Kio>N4S}{Ie)7pMdlCjye1)xhhQQ z9Mteque!`9zrrh-Tf}}f-$)SR{vGz}!+bst{59_n=Zogs{C$Kr%I=L*(BHt~QV~@) z|0|FKCrc%}RCw?MwTnS+KkNQkdP`!VVX0J-BFKNBclRIgW|c%0{QF}>@dQH|JiNc&sQA%A!2h4z@Q%bsHXO|h zpMW5Y`PzpsiG|naOUJAW!>Bqg7_n2u)}!J^4&E_uAfbwL(_Xz)T}=qjcnk7M``ay@cWuX8#K=q zmN_CRR=x4-H=#)b$wa`wRq*3LAk;wG2ozQDDpG27uyHaMl4@M%vh0_X73HI6BU*L3 zQtgTSDEYY;Z{FeuVdID4tmoB*V&D`}gv)*b4hoFVh(q1Kd8(Zs3d+Dt-pOT49{Q^I z^4;kmQkL@7Js25Es;P_I!vetZHmZH6}?qs z_(C~|Fmck)d>(~Bpgy})8sK#&H*m%_W(Uo7;0>qd6h66+uTgcrIj|q7RHrV;wvf2m zR#`p;VIr^BOhNgA}j$M zVu0R?#3NXS;{4-~kS;5WuCq#{70(#AOG?)_h*4^`_sgm=XOaNP%j3~tJ@4T5ANb}1 z0gDwaT?)ENvi42mhLDN)+BA>~xEREel8Ps&cGj@7o*>)IA++y0&6#=aWLsNiRLp&G ziRt1p3$dqBFC+j!eTdBptE107$0PgI66t{6Aog==dtatnBA62xS_Y`ldOGrpt9)6V z)jQL(Tz33PSdY3;0?g-KFK+aPNkxg7r6NgIXm2bRPasQm=ZYxk`x&)Wtf}Ld&hen? zA~(K;62_D=iCUb3Nwh#(s7Dx}L2XSbQ<{WF!;NQ(jh|l_Jkc(0rJqK}&jb);8c|fj zJOTZrSSe(H*NDn^H!!Uj%+l9cC?~}_M58$n%qZGE{wrU$BWey>h0cQ!mAa_JA~tqugUuvSPKM%O+$CR7PD^#GMX zQmx&iAz*12E}xd{<;>8c?NHUu>A;vE4NfCVMNWDYh`BVpKf*T77dzO_4sofC0~nP( zR=sL{FxX8Ik?FPQx!&ubAPstek>ID25&2EDS_J5!Bu>zr*J#JE7a%^+-N;;n+fH!` zZCkOIOHOB<>$BgYcY{)>C-6Az)U=XJcETO46~;%9%ryEZQr@L*AkahkLM}hz&Skdm zJJkqe&8kcmY>DQxD*%d-fF^_YjwAs|0_NOUPV2G)O%IlpOPpTX)|_v^pL=&1Z-W%B z&UazMIV@$*GTzj3pH8GlOZdtxf~6FvO&{EPFp3kVXdo%K+}&1@IG5tKju3P?DiOm3 zh)-{c2X~$d(;ES`vWJYO{tRT?y5yOXhK@<1H zaYs$Btb%C)dj;6(Ly05D@7$e7TKc+kxQKWn*-+qf)Q8znXx%gYB9qYPfGJnG_uHVG z!>dYdW?=a8#J+Wa!&T1Wp&M_A8oPzy&*3TaF#H|tMn7AEceG#LS`SMex@E#DN3 ze2$Y1DB%KF>! z%VuiH1%-@GXjRH@pmi7Y{Mdj<@w9a|L)zRaDOu0iM3 zLw1rr%z9xsEKf-8M8k)}3W5b2iS<_y-KUQ? zsN;UqrTg*`Ur;@VnhxZ+Ur}N<)aEDsDtV=jQ54+!{7tLKcHC}? zmi`qXyOtWJ@V5C6rF<3qliXxs)lXDepkSC z#3tEpL^79YC8M2+<;N1a;Opg)n#^)u%!IC9iM$VTb?Kt)BLVima;5?HGGE_-ms~asVJfPXRTCeY_1>Bh|!PR?%dpE8+vQKaKaV8Zc)MIH*sQ$2F*t zhzG^f48|0jl%YxhqhOQpFrPqAj3?ErVS~v(472-ApdD;J$^F4xtiYPamY0}c;~~Tz zbGU#^5>3sQ@k?4M1zBXDCRPNwoJkYtiZc6k`U-tpxBT_T+*x6M^CQ~8MNC5%?Hs*< z_fkKOo}xz1X>Q^*g~w{T4rrf519qYeh$2-OF#V$k{ZBeR8h|_eWN>bGYxI|brw~nu zjFAcnM73%wP>qv&SydS-KI$%bk-fwV7VVnT2tMeXez{Q0Lur82FB8R8G&cgtszSE) zc@X(rn?V*uv)5TL6$^D-M@`HYghO1uRht=8-@^N)yHy4+8TlMevNhph#70#okE6=x zuy&8)G&)5wndKVKg{p(twYKW#Qv3D1-;a&ye9a-lK%09Im)f z!n3n`*oG~n)~TA0qDCSZZX)P>N>&@X{mRv{WnPL`dzI;S0??i+0R@xX6n!i-A`2f4 z(V%FsJb@qyiUySE;zlxTen$5dJ~pTKVYSpyxGf8Rs@++z82Whas+)o0SLoaYh#b{> zGL0N^qZ*75LdsL}O5)vWe!vLnmv{jfjl>Hy(ZF8ttTxDm^kojfX_!8Nys;qN6dz`*kyg#c%swJ!p5yL0Ap)xNM0z~4O9Nj zSZN>ZZ*fKi0J-I6KrF=Xs}c~cWHgEv$)Fauj4D9hrcRwc0(AZ?AfRZlmHRZNl7jyxvn*if#wbt;9GZ8RCbzl2>M7uWQgzGuX8h9Ek=jW;wd7$yBOj zcBZ&s#N-nez%xrZ@2)S34Xl>Gyxa|Ff_==-_J7G=`tjI=EL>CNM3R#pK>#tA<_TE> z7o;Q2Y?fPPz+U}yLoih)UBC|kPQT2sE|Tk3(08Nlv2K5sE@p@K>!j3WMaJk2Pw7$g z#b9l`SXl9tt_yKGQJE(%$#u*EG2iE@g+;adMG_$?FU)*&_4054;eL${)Tjx61Hc&- z4ht}9iM&Lw*s*i+N0=TW;%eZB5k6XaEm%gdUl@P)h_L5@2!J z-Jqz=koZ2*d~c9obK#hul^J9D!_)LYHG(}gr9$1ZhA4RbpoenOlZAZyM1o2KU^Ns3 z_+A#Ys9LdYfBN;p;3al;VdB=0Hq0udx!yZj!;svILqh{=-UwpZ_tlYf4~kcptWcMw z(j;(`r%JRviqgJmZ8+0~mQYQN8tE7q7K8AL$n6th3X?A+F8EV8u4aXHyRnp3Xm0IlKQq5@G0> zJ@#YVsBO2qpd$he@WGqFl#eX*wv!5nUiT)#vTpgM-a+=Wd|BfAcHV%UM|27TrgRBh zKul)!o{8<&LNvm2YnQ8i3>g_IVY}fjW?_-K>kv41oXYTxbC1Gi)<^P0t@y(7lF%Flp&pwD47)U4v7CmMzAVc zM7#wxFM41i+KIMu-@w?9MFkyv>8;SF^in!HHvMQyL?dcZl>6A8ziVa?r2IGwBuB6n zbcsK>FUIL6;h==PRp=rg43(Co+Tmf4Kn&pMBlAI6mo+^hFn@|LOzGy62Mj_Ixb;5*4nb109 zsPZ#P)Aos2IuN0s%czLjY?4Cj5I(p~+1=)N@!ABk*O#+9_6{H!%gs+-uc$0%7>nX@ zAZ?&F8|b0)aFrTZ^Bw;}$1%R-5o97lAHwM3QZ_(R9{h@;2Z$n95X(_Qbr8TTj^7F6 zpK2xqbRdW)@FzA_&v)5>{ziFNrp8Ao%Hc11UBP%~q8|t9lLsKRMLpAqpF_ytH-sFP?@D5%2o>qC94*Eq!7AEl ztTS(f+4`3lRidA&AoK2{QI=nByy|w@UvVe8vMFHh*yAtPC?OeRq$>p zKiG}zRpHdZYrAs`Aen&CQ9j^fq!wefCP`VCNg3htqEkAkXQN=lW`mq4IWepS)&4sv zAYweUpNicX&dK;xCMbL#&yC^-QHjvBp6yIws(jO*sv!MkKQnyFs)?mb$pz-^{|*fd zku3HGDP5zVux5TnYMUw=pR^C`RV}0EH^5yQNpycQR4KCRBS7Li`%1JA{&! zG;#ef&{A!xL-+zYk=awTBWP7mL0#s_{RVpG8*Fp!-#KS6fe8dg=L`08(GHTBM~RlK zeOlM1(esBPlHYl03)lpC4`Qdn68@q(p6AAFYF`Jb-DOCl0jpSI0v`|Ky_?%YNlA~g zf-v&N@ynpIT_lySs~a)2Q(>6fKR25^&PERkrWRq@2sgkiZ38x z6ihC(AsE8aDn_(H-qzXArt_$l2?1s+*uWrS*+aQ0oq!Fv5=z?;*)O;_Dap_G@(Z+k zj8?g15bwYaV}Uj^$*7zmx5Oe*1-td*hNcUd+&><&*FYkTJalMdIWfc}!>>jPFe}(Y zHF+WvipS$UnnxP3RVH5>DIgP^3pq)A*FjuUU?KBxLJ$!zFiGW2$9X7=FO&?!q&|Gk zNN759knx5XUv%25lHf8FY#Fn3Y87e(QpZ{JBCNNrVNhp-8uwIm7YGN^+T(^c3$Ae4 z&Se$H2s|!|AyLsN4{w|BgZ|BL+{T1X=k@Y1Za$aEYTtpTj0e6Zoe$PZ-L7kWIe+rj zO#`7>eh<=e+mWU52G$rFeRlEX)$8uuk($T@!5O3%;+R)+_kXmDNKbX^2V_1-lKGWnzEdk)JNKp-*9mbj-PonBGfVlQK2UaLt=|5whUGbKCsubEh17hA-I?XC6eRSgGx~zDG@Zmd z9$sjJuGs`uG5OAKhQ4{W7v7Mbv>)Eqtn;#RzilBlJ;99CSu0xmnT^EP>aLO)#P7fo z4%0^3c0|OMZD|tP@^gaz8V{%9ro%OhS^HqTkrw1wku~CYvt(n1Y4?W(xu-vEb&Z0# zIYhrl+RB~Du{)G-j-qgjN(moY3GtA6XCz17^e&(59*b1{#fR-r4qnc#l% zQZXT@54CmqU_oNZjP(`*_){4;D+6y#OJT+Qfi@OIyTe`yNG|>NOy8bu`+trsU`M^h z0f|rWk-IIIv%!{8>5aZWLtTAer6v6H zU6DM{7@w%#KTc`W7GKjQL}1Bh;yu9zeP9x7(N%F7Q+}}y2r{r$FfzBZbo}YtMe8ij zH76*#;O4VU^SWG&vYCk7GB-oqpxA(cdslPDfhb7io-j+;H z#|0_tnKR6DXsJXBfJN8i7X)_NJ9NL_JbA`)^m{n1Hl9l=t5^bd}d}JO%n_ zDd7^qFQLSxhY(5{t58zdk!C!Nta$`b_8wML{ClAdp0bgp?F^xA*&(ozlyW3uA+7}S z@u1|l?;kz=(o^~nohhH$4l_A?bYYJ`7B>V$0KEVUDaQ(Y84nT#b$&EwZfoAD&~Z;z zwN3AT0VMZbva*g5F(rGtxn?cw@wAyjSvHzb-N~B_lDiHVt!gctq$_kpw;HVQUQ{8b z>Y`7Hl#`+)TC)|(t6bfSLc(ezWzLBcT}35*3y;=enSauWhgO0(nd7c)OB_hIfaj9S zhDRY`mnyQII8sxlYb^iC35pCc4bOa6Efb3yq?qxN2E3{qHj&^sRD5)zsJCW%Xm33R zLZ>f(Pr|9@nu|2>FlrH&ZwBVK9jGEhDYV~K9jLZ7HAYP3C{l{4=Yxl};D1RSKa|Lz zjo7~Lfe_RQ%Tk}9vXZ_CUmjnq9fjbfCt1uHhR72=Sbt5BaJi21HV;bC=?l%Xa)0_x zT7;=|$m`sRRf_B3k-T7cu2%w!ZTozW8IvXxhYi1rZ_fRIbVQ?;;@&oqsz`x5%_dVy z-T_7Wt|2=%WOxA^-%%f8gZlNg<$1jkDoWd0*weYwhG~r-!gJFSAOGZ#N?fG8?2pw? z)Tz}Uoa%cqHg8j=l0f2w%Ng%j6o>}Oq+rkm%CZAH7iv1>!WR@0{5^&8;Y;)STGMI0 zRh!J*c^XyL-KgB9+{V_AZC`a?Fvgni^Y^@r2=V@u+c65Pr+opgpq+Ir>D$ zNACu#HUvV+&FmD3W8s@v0V)>}tUWJlOYAB*!+{vx4|&ns^kH&(j0lgb7_~P0Wi?F4 zIR$~ZjdVLSwvAGhhap9NLlxvQqOV2r@R#LRC0{OgQ@uB)&gSD)B~T#vc3;P)Pxxfu z_2v0|a2h&E=99MxakkcExK5ikp%f^uxDyCp%qoAL%Pg6dxSYUE7F-6_eQ-mb{i%2$ z)%0s=aIE-*vX|Y)+xnn} z1XL>i>B%lz>_ya4%*D1}eJ^bk0%rI*BcKXCqkHq^`%qB@$=>#GMWfI+UI1t#AHW$= z*oQZUzN$uV%LIom>c61N+Np~ZDe@hEx=#N`2<(NYe8vul?L){4uz+Bk^tLE;kWCNr zXxf{gjb*Z*~p!uX0 z^%>9j|AfDi%VWf)(vVq&87CWLednTJI#e^?JFRf7SM=>E_vxNYPU0q zy{8(gA6yD9F68EtqY%+HrF6mddl48z^s=PB=gN9b6C7d3Nq@yAzSGmTlR&VQRL(43 zt9d)VML5PRJu!*H{vS14ylSyu{NQB>pw^X4Q`~rikQ0n=j_wPQ4JzD1UF{l%`iQ3b zb{OqFpPj3BI+r^O!P4?AGVa9VhOhjPwGN_l=f8o+I=FkzD5A3YxCN zGjyHZRmU-h(5610iX+3QJ$Lm zt1tj?Tr5^a(>sjfv-+FhdR)TA^!khyp@e}e`+2vXi-mUh%14VEs zcaD&vPS?v|(+g?$8C`Mp)qbBxg)sCCPsrTQ3x`wX|0h+KpPP`YbyXu)U)pNM@oJxb zlbHz*S#CQ!xn#tbK*n;x4^Ne>V*kM$Hs*YG z-Fj{QB6-sWx!6yY+V~5R+|hZ$176QuO7Gd&Wp#`_c`BGd4q@10V!&Qc>Lz2RMIMLO z5+r#3A+h|Xk43#*EAg3}w65FgYFz!2W@pDw4dWh+GpgS-uyLx*4;bsGhwc>`lNgan z@_Ga@cYri8ig5$TDmx_}Pt;L0Wqzbz=(W_&3F%lWdp)VM(3$=6_QK@#s8z@7?S^bj zSJep>T}{IBt$3$MOwpSPT^Mm(xdUD;mg!=<0uG39*5?^ks_gx|&F*M6IKNZ=jkVan8P6zdGKY*hqgE;(b}LF_kriBq2W<*Ml2sBmu#Sca%Uw!W!UvQbCO# z6Zk(Ph+)puC0}Ejv%BLd0`0_h8R9}=7Y1=Qw$A8NXlUE?r;ms%&o5=Cb^2vT zh|2n6DJwly2UvBXn&0dv^5K3U2jdTZM?AzV>=}VQBnNg|{TV+wB`VYNI5rek{F1CF1*N(O7!u>NVuFeor3u0>Z50)9yESyjyA^i4 zv8}>L+Dy&~X)EL!0N`N>e}*@QAvMUWgpR)5S{}B(dA`cyC)ilqxqC!tMqQr*-=ljs!b!Xig82;s> zP4u~`iB(=5c{5S>p1%mkjyMw3^^j|P&(B@si?u=wz*6sO*=&1=#2WZPpIgOB+l~X1(vf0cv3UV6+4NdD$m1_ z@T3OkO$%!{z7xNve7~LgmF4C)%n<+J6q^=QfQEkBL;*z>eL$^il2{N`7pd5p$38l){5vzEkhejuo$-~+*%vS>eL<`i$?93C=h+z z54w6#6Cm9Vg9*zVKMUaz<{_!U8k)z48}EOa%u5N5tAw01?Qfqj!~p4-m$Rs4Yt@a| zM;$V5PDjCbT97KukeVeU`E?Y-i|z#kEJa^Y`Zd2D;K@{I!Ux!Vkrn2QYci8XNuGx8v{_igjD0ODwI}ir{o^gl&b5vlLVY|DkR{L41gH*- zUMv$c^28S!`RwLCQ?Gcocwh2{czHW~BMwW8<O$$9ztABcG1E2$PoNf{w|k0{?q|;Ly|}xRuG!YD4p7+sxdGx zg%!bC&7X(mfLLhCT>XXjMB(Q2o66Kwmx(IgrOOJlla%h8AbD(4uT5Tf(Uqp`RO=Z^ z?^`_Lb>cph5y_PsY~T*@1f_&(*YfuvAbchb6k-+|tj(M@cmyyD)<-eRZL)rli?eCX z4Yxn~KpYX-WlMx-TNf0Ti5@E%ZWDW5WwYC{D>41fCSt&)rmk`HqXbFoNv9r@lP zf-4(`?X8gmgq~^>(J;dg?QD)lRwf!<{NFG;yIW^YM9i5e!l{7CZbkIr&sdgy6*{yi3g=5EC(w&G+{lT~^a98##TYXl>EHsXMC?(AB zf$x3<{KV;?zf0^?UI2cj%fy&y6tnfhO0&_C%VP?aPcl(y;(5)+NjOLMhry5OIm^tB zlvc3U&p)Th>QcCbJOFX3E4&Ip%_;1x=3|Q_rbo;jH)6h#U}Z$^CrSEpW`c5;9DvgZ zxKTKbY@k~+^9wd5GAbH-7E}D^71}ctaU4UUIytEf$vmf`FcySy!uaJMs=TR|yrI7{ z48R!SAx?vK(Y*+nw6JnED@XjN1_aMtd-)n(iHkq1vOGsbVif87%)c&}&Pn~9vm5{K zVnMLy9S$I+(3f8vc|r)jWtQ=no#?a?E4F{hhr%#18L^zVk;xo*6XQT^+y_aDx@|OI zyNoCL=UWQwAd#BEo4Ni+E8QqG(;e{45pE6*9Pq@NpL+|3It6T1p9hO`o0{Z92fCsn zSo%IoFv?iH0yJd> zHPdq^ac=pQ0DAcgW{5hxY6j>ET>e|Cz2QssNlOr$GofFk94RI$-=rBLA?`FPou7g0 zoFJ6!Aj2_7H|j#fzmh3tm~ax!^Pt$W$(jC?X$uG|gps-L2gjGAEOM$!3)q3pZYW5B zm6Q)G1`(*BFveFG@@i&}dy>+!o7DuI>gsHjSWPgtWlU6^oi~b$95a3#ZxX2cn)UJ{ z64bZKNxyM{U`1DGhL|p^Ny?MVU?$lG1Jojl6QbVrn+e6AMqd1p6@j8dI!7;jfltR! zafE$qMJZWs)ZjrAugT%@Afj1ZhM1J=G*9Pm#4;T@nxFc3fNV!5d zdZuSo@YCSoNoX{v!a8bkuu~Z^h}H|sX!UEL@>9)S*0z~*Ejz_R%W14s4gpHZNMm~V zVlOJ}8H>%f39q93`_V#-!7={wY@-A+6+(;_4jD+j)N^mULNduOmJjG5;CT+{*Gbc81UiN`&z)Gp)YSBMP7+wZ);Pu6Wj@c=TbQM+ zvT@za82bQ;!~^}=XC>(I_a{zISzYXVY{Non9_@`!N?iUxu*y9m?ozLPh>hi*hE^>6 zT-%z5Fla75FIL+F+y706$7C8%3J3tf{*kjU2WxNggG|JosZjtA3frf>`5HyK=s7z4 zpC4xJefm`OB})bua$(U`+Y$(QgluavE`*1dg;U)ng9i`D5LIaZ2who*a0`uP#fiJ< zCXFE%`q2_aU6X8@Ac+Ly@JuCg!GaECHLF*>4ItQ8P3TSU?PQ7Z%k`kZVkz|5q$u4c zUEJcbRYx8CZZWuYaozrm(&tsOuc13zaN!i;Drj>ZzME`f=^*HmP9Q;w@S05109qYu zL;{KR{d5lLusMV^>}k__TY}k(YDifn03ij^q6hymxp_y@IW}kSL?fz2FBkH$$1B{y z#aO0N852qS=9Sjf(OR4}x{)<@d-O`I`c#Q&bu!Hy)} zYdl8HC^S|@ZF%YD3_eG^PmqGh!*Xw}^W%M&7eT|=OYc7jP-Q~i2J(d1vVa0dJ&!Nn zgodG~2I2%gP48mw7u~uA*(Ne}j0*Eh**?^-`A>KYObhf%^>9VCIvBWrjESY0$JzK8 zWEwGwq$nklJr4j|##?M3E6}k+>OQNYz7s@opiQ7+&>raHJhz$j{8(R0UgG#ytGIti z45$>;TybOHJg6r(_oO4)F}6W=s6-1#-|eOdU8c%0kG`MaUJzr0BD(Jpz`+jUV)t3$|=IFq6>8giJ4>%khjIjGkK*J1@ zQl_u9gTG5IHPqsDrLCOZRhomX;GxvtY2-ePDU4r7+0dE77lZq5jK6EkSAj6+pYZ^0 zBBa%7)9{K&WdIjJ*}CDb3LGr^XQFTw=@XPt5&XLh5qk7*sQ>6Pmc595PLRw91?v3d zSIQy>5@7lLC87kg2OaZFOpKKnV;6M?Lhie2FhIrgZ9`au@2 z6kk9NxzV0HX!<4F(6Y()TlKg8R03dPXvDn)-*NwAXnenq9vVJKTa!?6qVsjWxkg8z za*?@{NnaIe7_*K#lAIwXqDXS3N6bnFo+qMu)fqfV9>{Y0r^crlYrnJEzF=9B_XUdF zX;vSkq=_kxK!Kz6dfFNh$`rRQhXu7)y{~d6M0+h!G>1i&ce{Y0D_5 zSM0Dd`K2vjY}0d zCtqoyV-bHqwu=hV9LoZgS$Y7w=k@H1kXA)J)?Mnd(4oZk9mCkfUf4ND2hSwmMXWXA z3H5(U)=^-WwPDeEz;hgyes6=6=j0na_0{oMFlcWl$?kpo?yI#v!JiX6Ag{#5cYF7n z{o(ad+sYjO*$9sU*>E43Kxp*GY!)%E7~XTX=y;7V&-Eg!fX%leD7O2FJ1$L$CTO2A z3?Dr?epzI5P?{Kj*lR10VC3nGey6NCe`c52xbq)~x4*4sY*t4^4osH7$){GZw|g1H zVKRAdgc6ZahULo=cY@vPRjH$*#Ih5I-R?&Za!bfm;f^?ouFeB@=gk^P6m;+h$H;FQ*5!`6)M(tnml>4@Sa^`X>F5{ zPKi8%bb!^4JZE8!SM|l->P{fy0MeEis9AM5 zEDpG-avKSJRQVJ{+ej&IvxyJ$i(a*NGPb$<0EiR-l9PITUO4hZ# zAzvx%_%_zOAhdOLO~K&glj8c4OtWh} zo6=iWe7t-2EXqEdW;G-7${nY4xIc|e>gA!{$Rjgje>ML^WJhZu@{tJ@4|}Z~K`eJo zOUlzsgBAl3>otDvK;!m9!O+Y4%~p>*>&NWp`-C?jIA+v(!)~*M7im(`*0$+ycFHn_ zwjUbLJMG6Y1(rBzw;HtYeO*{_YPDND4DXAZ4VKkw4myL3*tw75%q~8QU(9I{mT!KN zZpest|DGfMux~6&`hF6E-(x=|F@fn1gU8Ttp1nGE1%k*H-j)~}d;ipwU9M_ZcuRcW z-)5w+f8VyT45hTE)$4KD2Pe9X40OZs;J#NRi2QvNgMYH$Sw{YCmX&>va^*`%`dR~jC zVyN1`&kpIXYvwC`8#chben2Hdw7sXE{Y_ishX+D^qaSdpM_b)zb~wf#@nErRE5!`a zgAejwDK!L1nq}oWEoi8({p4 zPtEx*nWqoRbEr-pz_kKlgZ9DFY?jaQ1+XJx#tXO1fXm!J9bBaZ@2B-SXpR~FjUk}_r*4g*pgmQvhdem8g*L!HT`d8tshoJREAi`)w^pWXvmVHQJ6cyAGmv zf3T9iow|7d=U?6kBKI^&s zQC-d+WF0;1*E);Y&UVYK_2muYRXc18=xpC#X|bLcg5u$}fvm{lc7t8|jH?R#N!K>Y zKizE@Bx7jLBC1fFXgzpV?O`QmW%R_^9U-#Zk)yV9VweH0UFer(89u9>qKeddgfe(W z=IO_Eiv{~jh&I#8KFp>H2w$#vKRaIMp0rkF=v>Vj`n%V{X}@_|>JyuLbqIzlAva~L!u_rGD%_>r(iDp2@dgnAxW^WY;Mw^U^ zSnfnmGITNReOdE-l{+}RbR{+Xx6)ES=ph6_P(FiJDPe1AR0-{vJ{Sw50Z{Kw4AKYF9Ic%;9(aoNMRehVcO3{~VTmd`Hp9IzNs(1{uihZr6$WW>v3;v*lELm z16-FYnS?bhty%#eUcA$@^O3gzHy(woE$OGXi6tl<)=G<-i0##@in^?~Z~ML<6Vo%@ zi0mq?Qg!~!xLD7MWm;af^JD2(pWl?wCw)D@Z58eDb2iG=8uwkyYxM2*#6L3L{g#_I zU2nXeYTY}9i^Cq8wNUDB zWIh{#gW*xQR*UombKoxNhnU5OR?o?2b^5Wo+=tF@uTn`JOa}Qkb8fST%*$oR%`Ixf ztQdyEmEKy`xHR{~HY!eii%^Uds|; z?j*e{A0O@J+^SDk&&kEQxp}8a9Z?hSR=CQK0R!{5#vy`ZCFaNR>2J01@lv`yeT70@ zYa<-9{@u%UD7Ird$p6~9{8fv?+jd&9*L2Ul&BQ)6>TwY5d?Hpb7oxF?F`?~KEKM^f;y>&2V?MD#gBf#^QptbETg@dYA(DFH1xlrs;H z(xA^nmwu?Mo7a+G6AJ(dyuFq!jE&6EIkn#9uFBv<({#+YaYDJ?v?$Gj{3NYoORePyk2% ze0$8hvDUN$NdwDwL>H#DnX0k3w&uG!)b+Ew+x^Gxop0t7PH-P((7tB&jbx;!a!JDE z;hLP3>r2a1Nx?qjQp2Kx&zxONL$2k57lJui)QI+ogMW(|A|kFfXY?^*Zu=+b{QM?#Vz&XSg1WH6$^YzrHZd@ZGockXj0{g+3Zv z+h13UzvW6p)0c|Li>-u5SEFD(`L<`TqboLVYN6a+UU>Ay=9+b&v5~cmWI?2!hF#@e z!#8OPiYY&_JCQ2jtj1p3U2n;&$DDpYU?a*NV!A(Z|7F_O+MN<_ePKJ}++@5-iU_xo z{DOI+*)zW91{2XVcDA}p&A$gv-KQp>mmTQj@%S_^@#$&3(x9)&LV6z!Qmiheo=|LK zK`3SxiIs#UWggd7cdVcw=fMlOh$uI4RyR+bO}Gtxt<>5)LY4A-Bt6}p z@q165Xl{MTu%cS`E8AQ7=sYJmdEzl&a{Q4uI3b-*F~Iq(X3V^!_?xnDlP=bx8;dIj-o)TP(rt2y2r2K&xLNuMxw^+zAS ztSPlS8Dg}fj}e+`z1+WxKu}hWF$*Lwy^hVT&|BZ8G=3NU`3O((So}CB?`v#&Xyen( z5~WXPhYjgxdFq9mpRP<3d0Wz;63edRPJ>%(>dK`QXU%qxpUi9jT5C;(5Am1s?Ux!k zD%=kzzVCkV-?@*>>+0t4H~X`i7Qn=|rrM}sh} zaLX#L=E|eZ<2X1w!lqc_JG$s^goH}MLGymHST^g#L7JYLuIKAct+`4it5026-J=p? zX5vEyh&I55DV`Q#ax)lEIRpVi4Q%CA4UGa(_Y%j66@tzYGTD9GRfgMXvzb8 z6(%@$0(8hPdGH{y-d`ZzpI)9WgtMG;ajT+bqRvgte??UVsoIOpk|st zWz)gu2>s_HxJl z6T}PA)BI#UBEoqya3_nhw^DsL2$OObEKcTXxx?nGR>A}VbAjB9umcm8&v$bVgeO)n z@$3(R2k?{${JK literal 0 HcmV?d00001 diff --git a/models/tacotron2/NeMo/docs/source/asr/images/jasper_layers.png b/models/tacotron2/NeMo/docs/source/asr/images/jasper_layers.png new file mode 100644 index 0000000000000000000000000000000000000000..af88379af98f95cd02141b9dfcb669ead346700c GIT binary patch literal 25461 zcmeFZbyQYe+bu4LiWuC4NC?thQcB;_-63rd(nxoRBHbY%A>BwzBi$(7D9uetH=MQc z(Ff1>ofzZ%&iUha&O64-c*geLYp=axUh|sQT-!%ZM)VdMKH8;Amu`uR2`OB)IKub}Bnwl6Aw}X}=G>7W@Zjs(y7C6Lb+Z3v^M=N#sR!e|`)D1A~eW zBH|lqz-R2Ymykm+iQ`4GDh=t8QH8`nhmhlPT?csFPOAnKcG_Rcqj1GhlDfmiQM}VB zo~6n`eVhi5ytDqgSEcSGJFS{Z#LwiqHuP3!$c>6;Dmgyjz28&96?D~u=G1id@?&<> z{BcGwt4P7{oL=!xKh1rD1^UD+?Vi$FOw^HFqg`t>8q2D#EBVtG+&YUx96!(T{1$jp zovz0ZO4axZgS(4MOCb(h^PF_1W2MqKOxi0|svT4^9~P?&kfFJ#({Q&(tk!En1455{ zKIIL)xT7IjPfw2z3XO;IR_F?_p&t^bU)o@!!k~kuMC7`~;RpE%VZDnp0x!EH9%3fn zE_iXqlTpNc{Sl0`eivu7;)>Q4h-8Q=;j8sVE4F2U=l z!7@s&@N0(#<@b_vHv<fz?LLNnsBjD;_eggSvtOXIj(sVEj`-GHwR zn)pm8#&gST3MT70_OSn^2l0)U(!NTh+lziwvrZDk@n{B$g- zYj}4$HnZ=K(;@`pYG0zkWRpkMi@Ds8rVUT^O*}3kPa>;_CJhE!ZGJ5hjpOyl+wpNe ztBF&vYk1eqLkZ!;_dXBf*@K}onMr=d<(xzjKUbCndERT}0$7cGs5TtZ1B`H<0T~Go zR3VYX#T2nP+J`6U@8T5a@y%UzB0HUhMYL^5SiIS=kqD9VtJoE_4QpFy5Z}=WX4k}( zyPYWOnczIP-0t_)Z=%9vJ7zkyVW?s7Zwm{b^GOuQge2F;#09X^$ljEOOFx|?G) zEIA``*?44UR{Vf;Q-UWzOth~v6rj7LOy_MFY8+f%T>j`<0>y-$OdQL@=fihD3MM^9 z4VV1RZC8!Zbh0ZpfNOJEmugn(^WOomsq*>qu*)Jq>W|9`_;29iS9~RW)u;Ftl-39o zC+G1=MLn;hpkc{9By;$_`|v}+-o2yYXnoRguy8E)m~!cAFKrYb&#z!0g+VvE+7(V2 zIti}6GKjg30;ASU+?a2yO?@b{dq*3lflmSFVF$>NLjo{gkjvMI6n4oYDKsh1DuG*C zu55fB;E3;CxuN-y^pCc2GhP%;elhaM8~y&SNs;>3VX|EaCL=#M1@s)gTIZ(BN;IBh z5_fkZkE{?M`6s*-3KCyIqV z=qtE1?Iy_v;;Xu91E=ovFO`}y8@)p?ct3@|1~dW-(1^SaLuT+sa*RkKzU`#E68g2w zg57L{;T%h$yc+4P#T3cUo%p^o(y3beh{e&Hn}@#A*y{)*aEw!@&sRe5}!bKwUE!DcU1Iw<*X&v)(gceYYT7NSU6ItZ6w~Ylcpe^F|0?q1g%66 zzFWOhsVf;~k5x~P+d;E!pgh(=XcpC~X4K1Fj(HHjKPs|+oRMr4Y)3u^;x+5|r&ag;SHhIElIM`K0RF9Hbt9| zbrkP>3}9Y@9L~dlLEi+0?~prBQ!@Ou`15K;C7#b(By4-9_-9#*Yxs$sAg<{~@cQZw zNoz&v25mp8g*D@hl0Aqc*lphxpR_BNWayg|QNP{Z>^OAlJ07Gpv67EaozySyB5Lyu z3YPAu!H@*&%Yz(gI&+#ho{|u|3VOCL3_K6D%K0H7cP8yJWoDaS^=8PzOxcl)xOe}i zp>d3!gELCj>JruxnOwlF&FXOSj_LrX{;Z6X;fOh~Dqz%}V58q`U4f{RcQ9DpPq`}9 z*Q%IYp^jnl5;(-`<6S z@i@sXrRSTE2<&bm>QRH#am1*@Zg4&!MSRP*-Ix}uw<$Wiyo7H1OZ5azQ@OcXV{9_P zV>!mrU`iLHi4NzI_ng^7BcSfM0km|J$pNezY_BLVVA`4o`Odb>u4)HQk0VDB(L7A{ zQ;!)n+PdSW!MbQqxt6!vQ6swBESi|5_eCKHBcp+E6M1M*UWxKS@at+Li9C+aR%)jLLJ`&MnOpr>qjgwxBXKrDGRW+B| zGCPz^079q@@z{sy>xzLg_c89Bd7hkU@K-m;d~ZK}%#^1!2dm=GjYQ z$Da#(iI_-WRayNFgu( zi~XqojTF+Xjj;DcU4`FxXfQ{hJ_N3M*u?RX7_nN=f8W6b8(iS$%3qZKH59P?T@(L_ zb-#j<90>?mXkUsnes>D$9;z2uSj>kW>VE~Sw@|=B=`s5&VBsPQQwXb~%bK|6xmh!- z3o*b@JuaQa3w-r);P9}Ip~B53`>qI950>u$%=ivZRx5WTP& zEtdSAbvF+M&E_547HOsOPA6G^8l#E<#!z^8ga%mz4LbsRAmAXhZ_!THttna`pbpwy z`#n!UCNgJ4aQ-`FLRfl025U5sy{)cED0G(oDY$bJb9irUC~jv#rh4)Mb;tCks7SzGad1^r}7WW0JhaDUB? zV}E1Zr#t9&w-luSh_bg#hk8Gu0nSI~-pz=?tod}f!e)8!PM=rB164`8DT;%cOxNu9 zyJ2PRb97|0bTg-<4PLi#6xqf|VR%S8p7l(wTfv&2dFqX}Cc?62%im$~KN7l1I(>^G zUgY>v;(qvx_ZINrPyecD$x`?AOV@)Op>51p0C(lxe<>HbxS z*Q^PkuVScGM~~yGcGt9$v{(6UTFO27O<-n)PyK|JBkb{WPt!^cL92)KG}i&)ektyY zJ>GITN)ICEq+1BV9zg>Fm%cRWH?N`d#!nskdrf`R5eaxeh zAA<3ML?Pv%%BK|U+0Q4k|5dUf>#OuaSzx>OW6|a&_rhGsOae;zNl50{SGuUVcz%AW z+UkQNOFe3Ilv@%l*^09<1X*+U_4~b#IN2X(-rvap=u^Du60ClC+AvFuc@j(3^Ba0U zGLqsIiBRllUyOOjveMFa)*c?RAl}{gY!4s4E6*&x1vU(gF0g$_NW<; zm6{gXZ#-Y9R_A*~+SBw3{vPTQ3s|wNb<6}1YWQ3#o4qYAdT7{ue6XcAl&9sYqDOen zH0c4l(m$dX05{yk{=Ut5nLB{?qSock2XqUU$nZB8f$%CC&@F}ymGJ%mCb)eYh#5;o zDdZR7idhcm7IiYo?H6G)Q5NVH2^186=@uRUaIf0pD~o`Z{vfdnxCkxc3ztAI!AO7V z%YalgFhRfmEMfuO!a!X7!X@ITi9omTh!csph>Gw8aFI_^0(##+kGTxgo|`LdS^mG9 zgBm^rn7XZ|fO+vTxYR%bdYUe;_wT(Zu&-1=0_q73{nH%y$2vF|wf{Gi|BUDVj-mAU zap)_-nEAJqBCg$9>5HmKMmlx8U`JbMZi`@GcG|S*bw0v1tXNWCzw_SD-5UrWcDUfr z5EWpXJJ;C->c~AvJi-6C=#Dq)^c?o1);scp;)m4e^$cMdzmyO)L2>Ml8gL5X9#=cG zwh*%?4IO0hSIaK@q<>i@0EFVmD~1H7i~{BJ%iRpQQaC_Y3%rm@)h`Ehfm z9BXdli0W#gt5Q}K*m0YVMP(KsRc;taD8A}piTYB~SEYKK;;eYxS|M&dho3f}WLaEM>5goP zt|o?2dL;GqywVX&5%G?kEDL)>EhUO-N)uaG#XTw%XK8fCdpP}SZT#g6p;-{oLITsG zDS#dj`QbewF^mgRBX-gGBr7^TaOs&feqs`MC*%pEj;ga;a~vq&T?28m%6ieH+_uY= zmLATNYjN$<(N{3sfCPN~m{!%rE);kb3l*C((M)8sZq@t|!8+$| z0c)4gXE2OE)^yZWT>;%W=o%qIrn~`eJXzDN?c4Plo#|!+BPaJ6KGWsnCTad*g|3RY zlug3j-U!l&1wr`rvc?_|BEAICFs%hXyO>0?@-Cp>V>2b@brs19u5z{9lJvdHo{Tb` ziFYEz&;_*mm;s%B91jVogD$QIzKV8t?Ds)T;zd>_!%F_-D3KDL2m~b0Nx}XeC6G8~ z@`_etQ0D_BDWo!MmuYOxLFyd4XF=B0HRaFn3zDcs;2H`_QWV2~aZXqU&~@cgXGfSW zA{>eUs1m>?;1n0pEKvdAa73s zUnPpC%@vu>D|BgCBHoxde-h+k?XPm(xm;+}mT^;Z>wRvGQtl#Vh>_Z-E$wKn*cTe< z+MI)XmD+(51-O7Z!DQL@DzJj8Iw*eYEMFO5YaygN&-(fi(J-3U(mT;2Z&;XnD&+Om z{SYZ6S=Z&S;_t0=-*VE^Q+y)QH_q?^-vbrQ8C~x|W1sJ*B1MpvC&KMTeM4Pcws1M+ zIuc;z3`2{8y^rO|{6=H- zDY?cW0;DDIQXTE9lXx5bxnI~oGja+Z7J=A=j~Kac)0G*i{S5aC@lDYYS1z|Htg04I zYpQJKY|*B8+Hp85zRdMrs5bTFkc#xJNf@>KZ0n)QNbht!*>p)lBR(Px&1r_QpnNCW zpf%3t&bG94R;VM!Qn)s!m4c9h$joMbWlfemh^MnOK)Jd^7VntId;>&bW_99lGGS3| z#k&iBVl!`eWP|7mnb7T0fLhZC31F5kWp+Of%#tyS)3)EVj`WaG&6*(Mb4)|ANJ}IZ zygbFl6%hKPQ+E~YRwEMOD6m#hK;M#td8CnkFo4&C-S&V5V%tcHI-{hzngzr!tMzBv zu7(MJSk_~)b4A3x{vaO`=~-AXd6Z{zTDBCqx~nyX(;X~rY=n8IVA=Mwplf+av(4Va zAr2b5S5_j~n5m1=TqAdDS>ukpu=5=E^D5&Ja%f;%1FlRFkK?%Vi4Z*XX(XIl)OUsP zJ+pX&>BrUj<0=l0);$F?VO~?|B6=O%vCmerc8An5L}T~V+?Z`8fA*&c+~Lm_AI!@u zwe2te@pA)~WpWfr-1h2V*z_<@SKz*X=rtx~)&l^--NQ;ab z{sh3Yj1y&i7&yjx+&~ObfrPDPpRnHh&A%xT`gmMVbrMge52r2}Ka;3`hDLj0Xu^MH!+Zf+7D|UYSBNBxg9fvS;foB7)2hSr!Clt-PpO*Qn0;-;TX+x>8 z=Go1SV*hJIl@L{HXM!R67Gx4um})bIh-P%?t8nx_L{$V>#mO`?Xx_-uJ0>5}5O-PE zRv9m@n3+hk-I?;=_cR5nAwrtMH_PeO&I@C})X3mGC4@fdr6~CT$>8r%>K`hQ54GFd z#C1Lk>F(|kO6eTB<3LE*E#@;+e{)Nr?3G9$50OypM^u_+*4IY5-{!dt70bxXIjP4M z*gBHbHhk|W>`9qE=K~WU4>IYeSh9p@pb~M(gBk6OMas+YagK+!4M>I#vbzyGp7aeQ z)BfSJ8#i7cIJ#RyT#PimT*?h9OF6e4@kF}x**PCZywv!%Hoo#$bLUgM#%$gT@iZ%3 z&QgjI&bOp6QULdT(Rq(Vcx;q}ywe$PQer?QR2Zc&>5HgwO#X;$Ra>4P8{fQz3)gZrUojsXM(rDFBkD(U+LY3?`Nv7}i7c3;ToDJIdSQJDvEeNoeUThu)wGjF zVaDW?HIh5y9|JTd$OMvgcMB_q6-uNLo?3BK1C`5~E5#i(bBQKFPP00D0tk7sy%uvx zGrZJN)zBlsF{>Q3OxiKxr~dBCQ*0Y-gT{@D7}J>#&R%CyBctwPq5fbO-tg)%3YCr( zR>?BfR0<9I5ok)=_J|HsQpJd>WBe^)p)l|9z~N7Rh8wn&{9{yijng!gvac0$4GpJY zPdgFpz(%@UHaKkBBGdL!l@uIW=2}B37Y^Wv1L2Olw&kP!A;q@hlb?4pWs;3wnui*` z?VrfhoIfp6dBt012!1}9>3JV@bj*;EzZbmetcI6rD zKDzN7>%c55S2}5TPQrV-Qo(ezn3M1M?5N8L0&kKd&ju|{iYuU-sDLJM?d{5x(R%lO zhPpZuBJxL9p<%#YQGBn8!Bu3`MwG&nBZ_#mjc_+=uqE6_eranrOa*=`Yg^Ki53$w# zH^wsi20@giORB-G`u@uIgXoPLBhAUU`frtHQa__(T$?{zWzas`y(n7ZILfYLun z-WwIbO^UT78!p~n7X-IAZ(B{@i5ZkH^O!6qudPQLp{(Wo1Zmh$RWh4_+RD_kGsS)XwG-#A79Mss3FOl#)bw;pZ_o%WG0}~?K395zR)XTISD|7 z6c7r~1ON0DMH}?>r)05~3uy@8S&9GDb? ziUJ4Kx0mvYC4Fy7Ur4i9JP0A@ya7Z6k1CYi>LVM2lXAs;yR>LjXky1tB$sIckmmYx zI|Lc@Oacac=%NzLDYQ;kNVcDDDwc9;vv7w81!k>c`t?5VeLmF`t1-8Z+k(XbPWDW zFKciB*y4zwJtnq1-w%G>nxK#=oOZU&z&XnL{nDU6AJx{{%dq^3ej2j;itn9}Q3Vru z-Bpe%zuGA(L-j~@;?)``9KVUBA13|g2!y`uTEkpE&?+Q75c`5cHtCf;kZh{$SVr;e z9TCa4#5QlYvXhI(b9DO0!_CiZrgv6uvI}glG$@LBC)tUSm%O)+Y5YQf0@1gk%}+2S z6lXM$HLjeTfU3X4b`zFm+IkYN-;g`1|7Xj`jJM`)dmt?ZrWj=`i}$k&1x@ikFyu2& zVgPEU*%=Vc#vX#?clQ>%AaN`5Xc+0hlL%O~^OR$DGdEoX$9B73`9e=m zo4NUEjV50E<;R}A8wl+3c`u`bcX;g6t}@bc2vJ^IkZj~S-%l9?w2Kzgf3LCc>GtIk zc%70*D7kRj^Ec&_xw*NUovb~rFM@a}ySlobnbDd;LYo9&Obj$kZ=sas4hA(Ma{Nx_ z1MV)3@6%0wy}iB9P1CgpoKDYo--TMR`*na9l-mj+Hsin)^?~1Ql2*7nYY$YI0e)Oe ze6Rq7NG+z4A{I}T`xw2xk!rjKQJH}9kk_NJs=)o5bgF}$ni6&Y27c+1N zo`Xy`y20~06i}8}a$7bt@`r%gY5tGdsT{21@_;9oAp)!B?!+29?!<|h?CT@jn(wk2 z94$4CrPr<+cF2#E{m^Rl>K^Qz7Tg3Y_9k;Dq;Uz5)-jX|_J+8|Ja9X89w{-7N>EmV zsyc9DKd{Sg-1dWB{PLAYF5Uk}^t11g$7*Qzx+_#`_)g6A z4mRgV%sIQD$*X6m6O;dZ6g7B0?k{2c`!hj$ZDfA4nP#~u z_0tMQ`GV7nxx8ysx;YM7Gq`G>Y<=lnXZdwC4b~^CH$Cqk!+(43zJOvQnu?no0d%v+ zrR!DOy^@Y&+Xb4pz&^2}i%CDgtXty3U!&-YW|92X zceBXqd`yXN99Nomg>?%i?B`AUJnZQxt8u6;s;Ss1l?XhY(>WC;vj2``zz1j&*p~nG zbkOV6X!5)B)#KGKtE7xRcV|}PKUNsRIGMkjK6hrnc^10P zQ(ULkd~W%0=L85fzkP@O+!c}$T=#*~YIe%?tn_%%?Ib~W;m~vQ^qWk%^-}0PCmVQ7 z+QT-2BRhRUk}SEB5h!DIcb5g~ae`dVIU|CZdMxMHYM*wTZGF(JvX^(=PQKsPyVR3J_QZ@9GhZ3i>R@6{>%}vBE{4f@;NPI{%6I?-l+7)F;qH`FqjMUy%hhLp~`o z6318;pf;c?jb{xRg(O5^l;tZ*uD^~$DQ5OJCic|EIPw0!kZWx*O9?-A*OW(~UmBg>@beIwA97c5TM_aKW#jDAgC~wqY9d(9&4e z+Tk#tdvu5+y;EB8q(O zd_4a71K1TsQX&VyXT78kS zlx7@Yb}fP_7jU}bnz!LHJPbm)#bZL8@t9;1P9}32hrzY+VcAEF8*y|xpE2{+sraA@ z;JtpecR|+}I~r=8bzEJxtGb`lNP5=Q^6!SP)a+cpMIryj3@HMe_h^EKT(siZEv6t) z%)U(y-f0pv|}U@0;M6dPr?N z)=(%$%rPFMyQGJO|1SyEL6H|cejXXkiqC~kThKIwWQnKt6DeqnRv%HJ^E&l^BVt8h z9h|ywr>=RS$2mxW-BDJTetD>MEB`F6$}MeB?QsCki7sEV<3+KW7-TfKA5T7gtn)^l z^+gw^5=JPMFY)uPd|(-}-#RMPou^~j%({U=0!Zy5hW%V_^rplSugtS+Zo|ZOvRRgs ztQe5$yMOc^CsLF+2Q(%hKl85ojRvuzGxg!|y*+jzqc<;?T)4%(>s{!-RbXC+O0@kn z>AO?OOhgOa0URPi+JH4=D*s~x;4|O!MpVJuTv>+xa3g0>J4bNs zmG@cLbeIBSmS~S;V;b**pXDgQ>GllY-&g$7?&Rv42HjMo?b-+d@?S zqvu&q!9DOG_&{qTbxu4c4aFj$Ly3E^9ad#xdN9P#5Ex?Bp7h@X(LoBcCNf%UQn~Qp zJX8BqX(g6GZuF+!;V@;00&dd`%{%)17x%2Yy9_|FxR~&B6AnZOp^e)NwmjaDo)Yd~ zH~>nROZ_SIf8l@(z=6+<$@Lf8OjHCokWQ(09^ZfCz$<|8%{F*<{;F2Sfs0)1G?1qG z2X6c!`k2>&d55pkrZ&gKC?Z07)sp-{EDiJ<4<0}w9bOl3A!A??^9dyvy%SB2Cd%cp z5#Zn(CW#~R>(4nHKyS^WNM@nG^<(AoYN6%Z*3DhY=*R+xiNflo#A2V;&aSS)mnzM7 z)WD0%g9-xW9>)gTh^O5*9AZB5J^l@n5if|OSLvph#`<|_tz z6NOvVW`;B-V})h|mU0|}q9)-vIdv;HJ(z*e9p)GXWh${5H(>We120VrVv52hnH6RS zZU8;{pnm`L-r-!%5En`OQ2CE3-Kk92!{9U$pe2H(4zxr<)0H-$@?YK{fw$0;3FC3S z&~z#B-gac_vUXjm$0JIXI4N8Y0k+0ZT1rtSh6>4Y1+`K`^BEzjR1Z_uY8gA&`7@7b z<|{I^M>ZO-m#+fxd@v7i_eOS6p8;N1JcS!^-#$NyZB8v#EgyGN!1HDeO_$-f4k|5Q zRfrN^i}{4|ND&o9#@$|m=hpaR@wdoiaA#sZl)0z6Db#|p+t$sPp@-${Z9b3F-iSr? zS$^7I4pusl&Lz4YGU_3bQMJaFK3|EI|0FqQ%X+IuBtr%#Cl7~noTPSqG~sc}i&6>8 z>4(q6v`6WxaU$m=Q8;?3@v(h%S5xK5SeS7) z_b=s=A|AmRN=@O9+<0wTP%h9|#Fxjh9?#~bN$`5=q(`1@VW-wG389t3TFtz_vvpg1 zuA**C(FAq$i9pf#y#E_rf>j`HcN1>(iat@Q=9<^7;QmmlIg}ya;Z$gDD80$9NMvf; zQ}L|LTJ4iT{>tzt)lyjr!*nNMCv^%-y6I${fLn!w?De`DS>#QXIgEzJ8$<(^G8!gJ z!GzPhV#WDZ4UdM&Lp2Tk9eAF(g;h9%0EU!tDw4!OTvTy|#8{ zRUj`2@gLrD?VtG3@BK!CIYuvRxud$WFD-XN3|n)d$#!jtagi&9Z23i5C64@&Z25tw zxBc2s)_ZOe1y&_e4nckO4#&B#%MSkgPsYeg%{6|ov*_i{99tK)10e-I36RK|W!kXfx>XVl z+*xehZd$NHd~QA%+hH`j@7we1y}Os=>Fm2daHKPGoBJ@dosYTa>b?X57(VWq+WsV6S#~#gjms<+9QAXukHJUx1S)l~ucGskgBue}f z!r25bIerK1fu^ zk43}&R*-%;S6ot_Ku%J7Vr}cHACNa&7UBIQSx2id6)2{%yjouwJC)aqXB)K*6e8HO z`eLn^Yh6kdlh{3y8b9au#>VPb>>$|aQX>Mo)@>@%>mND6FXKHWwNKHU_W1!S06RKC4j}%0l*o?Y`mtC zWSQ@etyfScb7ZuH>636f-Ap!a8#dT){PIIAgDy?~XW@cAzEKtHwn)*#56kxc##aH1 zdiVle;|~8uandysZuFc>8y@?xH=bc3N%YkgySh5C z$-1vg^z%P(E}4fF=CXs@iF(=6T10n$++pTkTg|@h2imCi7AKS>QlPu;0fm_M9nWf~ zp!+`Dxpl^ZW+aPT7|&4ay4;Hh|DwSosqV)6^;}jm(xZ{0+LAos?55AU@gO^KsX|(1 z@f>~98#;gGvZdWB2aoK@d>g0!M93Fvr)Hn*MIdIs@>&g4nk>L*RC;=+o83yoAm;lV zgf#7`kzCbBE*wAUbHj6Bx>0KY)6QB_MzjBL3H`3kU0zUk*--3Znd<5)kJ$ck+o5-c zBp0EcW`h3MW?lBodeijN#2mL>lp=b*wBwVUu>w%Ak(MQ&m}3S}@fo&73!o@D?|jmq z@$?!h0<-=*Yj$YaI?RJ);FwdGo5eocV39sXkLeyoQPe=&?VUsgx>rk|?mo%i$tKXP z_quHu)_?7D7I$6~(P4$~gslbz`o@@fOT0L?8?iAcB$kxRs;Hw2D3@h!<2!IMibm=> zx*IFlY}aPS(NAf(sK<}P_}_k=^t9m;(=l;X7+bm1-=LPhuybjoF;WUR{yC8g7J^~2 zF59>p{ns$F-$=Uef3M{CL9fZQ&TAEur^IL z`ou9+Y%Tb)398zqpn@@$>fQ+|oxgp093P%qbqkcD*!-&{VqX)(BJCyoQR}}65!O=z zisFy`HrLJ&{TIXd#TRBlMLYcNUAl|*Dflo@q3Q>di_fJUkm;OLgH4dWwEOw}dK@T%Z11JxU8pa3r-e)x3t>;&D8y|wxnR1lAyvT$9RY^8W|ae29Km&2a` z4oW(EjkYV>+nt+_yf6kii{-Q;`$H}t;pgUm%%7jPo0tt|mPK#>trx@6!02UkK4{c# zpbN&o4-{WSvvXUB zV2~H~x-U2!vPck z1-?(9BW6qN)-iXC&N8{X36Gz7T7+HEAXm?C?0dQMFn53o76C@!t6is@f;Gx<<245H z5YiL-?}a&z<&yD7Efz&g`=<4(&m!Fh zTN0sc(G5)R=R6c5*wz`JP;~_a%I6dy1YCR$Kz8U@ht}|QviGxM+uOL+eZO3g|hdETm+@ays+uoD(IHn^>32z zy4yR}KST`$x?xI1cn=sYMs!&ABqW4QO&+_HvxHxJeKqxHfX1@x*1#q@t5M_0P2a@+ zAV(!_P)9l76U#-~U{fZ?T~Ee`66!~%<0Zr*pV?o7nc)j3r$H0h%6zMlSW==K?=X+& zuu{;HQeHk%?G0cDBK3gDIy^0)MaIA;7I{CiBs>v1)=4RvKM0-l1)@v#k3XUd@$bWJ z@Ng7wjQL#-@sM(th06I#yYfWovJmmgE< zR`BWJ)8f`hD53)kGjLXzH^ra|$-Cn|v&|2K1Q|}__0y($?dbm?fWw55p1&EXmQm2P zJ=4pr9%91Tz4}M5`K@nzP~U)&b*{Sr)2ZuyeUrz(O{X=cV;pK#_61wo3<-}y7N#E& zgJpCH84tWX-qZIWK1E)7P=9>cL&Gw5>RpWn^FJc@`i>gp6}WJy*ZVqXpBoeC+*N$Z zzp8Zqq+f~hAoC)l6#gsoasc2pzk|pASLPJ}F5*9+LHidD0lk4Tul$~j2NxmwB`}Zn zdW*|nOz@t@152obi%7u5QXM=QTojS={mUo11t7L_nbqhopC~?%9NC^LU|)R9dteFG z)lQIq@dutk&%OflXbMJH(r?5*9|j&sbvyLOzZf&zMZm>?_t5$6fBQKPl`4h|`u|E| z*+3F&*qB0n@iEN6UEBJ9ZYTvbcej6g0sb!>)~BDF3qyUmfM%0RSY5a#ZlG>7>1+lJ}LKrA9_caZy33||cfmrTqs$f~?uc6qc2dj7o_tj5l#rh|Hex#2s*Dw;H3MDrn0hNJZ za>%7xSYlcO9e!XvJ96Wei9cDxUf{+025hAw>G_74t{&8liqGGi&Uj%alLP~yEDk$! zOD!E#ZlOaJlGcN{=52y@)wD(_Q2pj7)6db6{Ephw9~#|P^XZRboShI+aevuWQH?=- z318TM#`FWH75lz#jm6^$r7tqx{iqn%h6wLX{-V<-tf2oHf$wCsjA9t(eX1+Z_X{Y!K zMA7cNQU3kTzYgB~NA?8Ce`Lz6Tn~(9J9VKmJ2-N++QaD|vlJKi{k*`$;-5&-bX^FhOg@GywMjF?$PF^}0 zWgJpT7ZDhO{zi`?paQ{bP%FM=)Fa1)0?tbLql)m$`Tc}kvTvi_MqZ2$a3z4zNxgl= z&{rt_sBk<0qj}WcLm#jB!-VC*i9ufp**=*Izk~e*@nKW<8RdoFp$Z8DD+evFH21~$ zPzO}O)?dfp{xk}^19*AHHrpyxH~#hTR?@EwW54~av>iqsPjrV$v;t9*_~uoge?E{# zHM()AH$~<&*IB5q#@3M>m@d=We2qJ$??p%qyOovKDof5f`w=*}yE*GMSuD)eE<+cN z2i^gkcm<#ANKSo|jXQXr~Yo9n4;L=M-`O zP%$We`MUf!t$ehnDPlL0D`D&(9GFifJ)K$RoG zqPa0nC*O=qWWq6Ae6LOd4@+i5mZ-Tn@hOUQ-jf*QSXSuv4AQVL4CRww3zS?5uLfpX zB@!Rm>8NNFzm}I_-ovW_S_!iSTJ7a^M%2THbAlAc@#1RJ*wKdrbBDsz-z3sN_5TsF ztuqEmym~eG&_nkZap`tyW(j?!>@Y$^Omai|T%2N<-mdr&e#&rb@8@vi^sxNs-F>rZ zo2vmvAuT1+BQ<`HrVKUl+m37*yk#k;W1OO$-q3IGQkRMVxl}^ad@ek}S|nJ#Ps^*) zZ+KsMQ}^-sV_V|0yby67;gEY@=c$ z!dt@mIQ~|6k*&`iG?Ut?b!#8Ox5mC9q|zsxwr$lEoY!oCE(}a=_twIQLg>@0fxq|o zUO$j#)dsg-emcBnJas~$N8WPEE)>^S=@QyPxN&<=!k|4LdzYhkGk!%w9jj71>z8Dz zq3)>HvNgfNJr~Ggpuxx&He5uv!(Wt4lrWPU5=bak zT?;M6!RYz+L7=9u)2Pt3A4i-pw_q<_yHc;xW$1hOHw}Z7C5v~*mc{|ghg3rxM9E5s zDZ7??M^lwtf*ERNRwA`5{+HCEo=EN9nwzz zP6)e>?7`fIY`rhNEK{z1dG~0#y zCta6gO7+a^A?gMyVL4j4u8lZL;}+uwfiosATvR1$yER-|Hl0oUjB5{MaE4B|h3Nb2 zDx8<~>JK4DPG|yGJR+*kbx=}+>Ono5RrH>Xm<$Wn{agt;Qq$%VmdOfnq4w8miqj8+ zd?QbXpDF@}9qk@Uj@#O-q;+P=UiOWBJ)`bAA|(_?c_yQshYLzD%Tn`VjyMid{=cMU{7LCl`{B3z8h_ZvwX{RY6)$9yw0p`$;x9jhKEjtTY@q} zhkA2d>I3?j5gtEHLvvVGj@x3~F8?-<*}}xgLA3WUnQHQEY573mj75HN^&o?vev`=C z72}UACzreB=+4U{snCg)<_HChR_b79MWj6=BOz)RHVRj+I45Ctt*mu6neu|=9J`dY zhf6N`9|~l)irjL#!|NzaQx*agdn}#P`xNJhBkINEgB=@^84YB2o8~#v&V_YlSkY7yyJZ#5B-3p(jB&hPGM> zih*(Z!!2z!Jg(N=p~1S;@grZrcLS(#Pl7+4J%3tb z5!WD%b-d(zJUMVe8hP}AO38g{?2yKy z;C>svCAYlF+QkYjThoB?OtVId&yu74MLsx_n3bk>huzVTl!gs`fnYKi6UHo%s05&K z&T-Ob=tg?5X;pihLA-Io+@VaA><)4bqqzJfZZTdp`&1VuXB&c+@G)J3JWHVD->Y5S z0orV#q1#FY;m}n?{63xPy>N!LYuQ|7?J}<4jW$mXS*Kn#DOdG~4`oE|<6YTOqtAv& z2@QB#Y}h0z%o1r~&4>n_mvNUnz$Mb}$y9#t{m)s?yP9*`hO8GxMO>&PCn#Utt;|bV zH*&4aRa)JvbPF4riAnO^(=O$X#l*wrhP@LLe()(U__n9}Uwi(7R) z#Z=2}or=0sWIj&E+}T-`<@WwOnN+Rj>Phi8U~Z41iynvAylNMZ^G4t3-f-nmK#W#F zjt(>|Y8EdH*&8nm=ul{>5p77aAi|(a^&ka|z(R7@UudG@~xYEfu`vf$d?#9obfX`fYF= zxR1jM!^@t5!}eYUs)mX%lzL2vH9v|ujJqE`uoa@hP>@jcB+#WONCNfq%;h0$<)mxF zRDY-*{1b|dO$Uk1WonFQW}i3IPvR60M}x5=Cg1b^A&sag>WLbqm1dR$-)GoL;1A!m z&oR`o3p<1rApL2_{Q%jgZ*JT)x_o42lz0zK>K(9`P=`-QB>ziDH!+_)5(LH6He)Q6 z+d)92wLc7v{P!vQIJkgQyo?;F88T>MhWtqHouW$r;<=#Eo%BH{2duJxDeW`x^)584 z^%=Ne{kbWxT(SIK&a*93{pRe-3?YPhMKk+Jmhw|t@ zqOU$X@=-g#v$PjFAtv5sd|CmXIci|S@L+BM?v9O?YZO8n#2U`fXThFW&qID-sk5bL z^7*?uU5i$m-{bN-hRhDb{Bd{R^2P!uti|;V#kqdY+)T2Z->I*bF$>-u5DkpTkuq{P-S^6!wTZ^i**PNBj=*~cu3=rXZM48_Dl3F zGiUiF`Yv+WJ=Q)f3(|FR$}e{Nsb--}s#Nq*m9c}@%~i(LX=`t;U5*G9KRi#hC+?+L zMr5j+l_%STwW(Z|FfTIr;?FHPgU0OhkE>>E_rZJ z#bG|$V78!zC~SUSAAY{2&;Z`xptk6v3JC$tRbTY3jiecC?7eDr9bnc|S69#R`X7k6 z>oLHNs?nXmZ=Rd~9}Uk!;-FPCbD-po+0_`W0VQ|R{lBZ^&gE1e-M!(Y_a8n%0H(Wyu7?D=94@I2miSbuL`Kr zlZ>6slIJhPS>)qAO(#4jG%BUr5sf1zp`T`Ck}Jy?|Xm5B=ukj^bUad&_<4SWVUU@xPnFCF>Jpg~z_M~x!h zT~TL`LUZ6yL)7-}cu-l22kx1TO#b>@+FEEdBs;00NMKH}4e~7R^bISW32!(oPWiH9 zh~34F=j$-;ezokk5ZIvIvEpD;=xh*iBC?*1=8A4Bmadu!%tO#YW|jV^HqYMwQq_5d zHF>siy!H|mtO5=|5vd5W1qq<6RHlNGurbIG8M)XtA)ye4Sdpco%rZhMWkoy50{$jhqN%>-#+0)U1bLuOu@?;i6&|(w2~xGmJ1^q9-0cy2y=TJUUF+;Q~LoHl7TwBb4Im4qJbmvOK@|s z*YtvAckJ$p?Fm-Fft=r#F#V{P8OW#%P@bN-Z?|cHV*-p{{yZWbB4_u9F}FXYy24(; z8Rt1QI_AfwR($M3r3iE!+esIGX1Kf)_@5oRLijRkQ5cwQt32(ul+w27F}ZfZ*}65n zPbfs9FO`MK+3}NzWHl;^^g_>JyD-MEsC8inFU8pE*imtwXd9S}4U6CgJm8#VQzmzx+tmc2+qgKcOfWRVaXae=@LP~AjiZMYds&B z%mkhHR!raA)V?w8UP3Qi+|+K643bf{OD`@D=}rWy@GvRWqfGg^*i&nyvllo@z_6mL zkME4NDChGwj8H?%udo^w)N3%xiS9=q*&+%d>K`*ijPm_5f@ETQKHXBt+)9YWlQU?J6SF@S$N4#g4PG%P$+KiuF$o59BWl*StmY z_oW4O9XoEu>X_#b<)s?8F$AW}e(t&Fp z2j|_cS(H}go&%NK;P^H%vv~(kH4ofkjgviA8Lm)hbSH(D3Wxg3j7v%i_^mGQE~oTj z1c>@MbUCumV|9XPqkzSbeEi(X5&=D#-i$RVh!NC<17&*w}q#b<9VkzwktNu`$F;UDliBx;r(2_o*W2P>3oph8{Yd zB+|qrm1{6l-g5I%SO=}g2hAzlwB9Hp*(AiM^5XBpTgq1$NBPpG{cwtN{Y2xvzKk!T#LxoFaU&qDQ@= z-6jZYKz5M$GdR(u%9QMwr7tZiGxXx@1HP(?5z~a=g0^0Pxe_jM1w42O^1EUg+0Ri1 zGrJ!zO<4(_a=Eyq!!#_Wm9er2V!K&_+yZ-YGReawi5<1P(wLy=#oA!~{~70!u2Fn7uu zu-A>sGW=C+X0(0ZSK90F@dxl>1~Q!o72x;1f%foztE&TWvpa6bE0WsDsmJ0VB-`ld zSBplRX0>MHvs?UQ`M8-V_hFiYCZZ7!9 zZhNFO2T^LQ=`sO1=ZD{fPQ6;qquY!YSQ=c|t2erKX+qDLaatbLCBi(d&R>?U?!MNu zH-?!w0UId~{VqLv4}avX0q|&mjPQ+sibT$8s@IhQr?ef-z-gC8&Gd=6Bf{aF+NZp% z)T1?nqesT+OTYNU-7H$fWuRrx_;BG@y>W@L)BbCQ%%HtISFm`kTFr=dd`bcMG9Y|c zR?TWb-5MO~yBWTnkSzB`=r|!~1%sL;jjB0gZTx5FqlBvUj53r;F=~QEu!Y!YY6pfv+~`;VElx>&Ct(wUbOM!7BYFUVcwff@3w(eljM0-awqZW0G0rwz_pmEvrHW zfb0r|CJDoRVCBCcO2AO0_0f8Dn#}RtwvtnthWscTt*KVmPnm1EH>^7$w|h){!qzh} z;xBp1!udYAubgAdKH3hnvJ%?U0=#?oUvd7?SI!10WY(6Ev-ahBI9c2$zfF0ah$sV2 z?+-2c*S0a|3RtfH#TEZg^SY`6IJ;(3Rl5I*{_KhEw3YUT#aI3dwtwh1MF5d(-D>Ia zPb2qgGZ6GaJBa=5*#fR{24EuJWN0#kPZs=Nm7@Z`s>^Wr@gJXnBY~igLzum=Sp4_q pcM%Bs2w7Qwf4T}V#=goQ{j=SidRUYMJg4!UwI#&jmD!yK{{a+Xk-z`| literal 0 HcmV?d00001 diff --git a/models/tacotron2/NeMo/docs/source/asr/images/jasper_vertical.png b/models/tacotron2/NeMo/docs/source/asr/images/jasper_vertical.png new file mode 100644 index 0000000000000000000000000000000000000000..13ba93c2c73b0b186a26076e1581698ede4f6da5 GIT binary patch literal 279466 zcmc$_1y_{s7dASiARvNBmnhO8t#n9t=YVvFFoZOMfYL}vcXz{(g5;1xcS%Y}H=JkG z-~T=D`2vTvSUBU%zW2TJ+VOl=R+PedNcs>20^!I=zf}c+P{F|O1qM3sPCrizF7OA* zSyf65R5nDu4ZJ|L5LFNbfhr@hZj8`?*O-pdI?f;v;SS;#X+1*xF$i>@EAv)V-Q8d} z70sPs6WSAOEWxCWqnk6%;KKV*LNBKwj_@H1%MV98IR=CI1uzQc`?>HYrWdbIo%dVz z8aLCN7q45Fc5Qj&{7~_$$nM%UF9rFJ26rW29uEqqTLOpv|NRq-mOePPgL9?g=`$j> zck6##D-^I{-SRAuR;?@%39B{&g~*!i#2{zErm^HdUgyhtJB{u?%u3yE+K^rNvqu}U zVQXjHelNcdd>pUX>UuY1ux!JSyM$~^YmKiMm)_o?PO?&P*WM%iZgvCbt+G%da zy5u|976cGPwt>NHnQaXBQ6}T@O1u;4&z?=jKP~ZYT(HHN%wYWmn@vdWm^~68>zqya z)iKKvR?GIhM{r=aDzKKVKC<@KwpPahI3Ig5p2zlmdMUpR?A>H**bQsVJ3F1;*#!O{ zi+!Zyrb-oO>#ZhsAHKw;hkG|h>gZinpPBubEMalhx#p14xilbAw=hfgZlu!IqcnN0 zbM16CB?gcUyFj&pF#~_SB5j3}Zv>+ca=jZbOITQDFeY15Zm{jPzt1M%r5o8L_RS`| z>6zzAao;*LzTP!&1RU*LGb8G1-HK*y-ryRh^mLXkRuN!6*TaIW&dp{cra4 zVf_acH+u=5J<@d~Hla7(F-t~sCP<)&{)}ARTOa2dqa(-HmeG;{KI4|ubH4tP-kVnA z7G;txn<5x;Mf#jhd%fH*redIZ7qg@lA~2FyhGg81ajuc;M<}4zboj2lokPjHBAS$~ zwp1fjrpGgFdd&kHm8>N!4i&*OD6X?wTfqQZbyaTFbpz&mFmKpw5bM#oR75SC#Xa>2 zjg6tCmw&CqENZr77Z>v{*UQ|>%dxxR^g*}X1hyMSrVbmk0|(Z zTzAH0D8;aUejhs%SY@rlvk657ovr`;K7Cd#Q&AaR%XSW2RDfRg_d7#HpGiPcVcsMk zsGA~dB`7y`VJz>~Gv`GpLumD@+30Ys>je4;&u3n|tCpbrxCiTfk6TeEZ@dTV>nz=~ z_!=cnNe%GaH%s&VhXJ4w)Py=OKbT6-JplHE#r0l%tNZh*Y`~C?nlysBqd>;&Hikn&n@XO)#AVU@1Py*rpGI>G`u3jH+FzpGAz-F4tI z-4E85Uckij0}0`bZ)43ctwt6q$f>g!sk~2AMC$Dq3Y<5G&hA$pfIt>vVl0j6(xC!Y z>mA@geu;aEc}bgN)zR89>xT9BLK5bV5agOs$W<~+^e?_XU?YU+fIYMWdL`h(HpsU(T zWisx&g;yh@E!1BdCs|E@;$UL65luHThX zw5H$d`^fGA8rmbte8p6uZFnxM4LH3>j7aVRf3eBUVUC^t4-2|2z3!&?9nFknwoyp! zW??T$Z0G87u|dm{#76;>8B1P+T2O*M-?@BpO9>TUg$#nLyy*uL7)CRU;K@}#byji+{ zjHREPw);U-_j_-cGKXblm^eD90S}#dToPNq0L#5MT;@!UN z{u~utulc6nHvd43+XMv4@QS+IKS~!Q30>#9e!!&04PzE>m)V|gJZ`sNfZoD*9Pwxa zSGG5llnkT=`eGm}>k6foa+AI`gb`OrXMY#g)Z#3a zPRLrNeE2MIy;$G#f%Elot+v9T^|)`VYu&x=L>v-G`K(tx>4>TsqWLl5TpQc#*AMCQ z%%N2No|$?lJQfzPsV&j9RRt1oN2)}lYFs9@1TyJY0guuBevSI3@w5}acI&?*B)Iog zV-La#d}`UzmN&{Pm9ET5ilbiuB?+(6F9Z1{i=*S#RkPtsSD zroKLBI88T{2;YM{l(}A{CIYq!0?|__=xm4X+v%XkMU{b@7RMT$H-E3{Hcne(8eHi- zLIHss*Lrhso2Jue45u-7vzbBd@7l#mSNza)uiB}&Nh^z3CGNN%mFiQMyjv#?yXtYj z@-PvPvG?@~-#AGN_J@J!Y2gaeNdj_cufpT(-}27{Yb#?SY|#-@9eb~naNFR;obc7w zb4>8+MQz%;GN2>g;IPQLl-mbP0sF>5!XRvPwh@Ds2T(Zx)0_-e(lM+Ek3EKpXn!Z2 zT1LK4P_J*hJFXGF9-iRSX4M0U^tpdzdFN}f-sZ?bVeUexE@t4q@@WSQh3HJw+DsAq zgx6gp1hQnA2_%ru48cZI`Q3QjcE-0UU@O4U{pePgG`@OfjMgB(SMU7Ko?Q!jUwpT> z+9Jcyr-nSvO?9m!{R!GfuLqyZPd z8mYnpf!a&6vE~d7QI~m4myzu*+9gO&)3Qf50$Ht@Kd(}Po(0xVLXsr(zCu;gsklh1 z)StI%2LHLpVK2z%^i%?*z7=Ce28Ha;Uz)&J0UqnoN!ksd6t{h{#bZ4>AbU1bO#+g) zT6f72g;~!aX1)|Nu@%$L)e;rX}rvoDR^PS8nop8+x_zs$H z@wJp9m)IvVm$-^v8h}p{$RVJ zNl*m}@Cx)AjpMhsoHw4IEXIFSK#tCTzMTI~;Q9%FR4EM^s6d?>;y=PJv4t#xxBt%U zajVt@DTo&IkH94C*_rUJ7a=F!$9}Z`sAxy~Ea_>Jw3+4Ef-+wGB`9kC+R5yGC~2CI z85D;S$oiNs54eqTb8mMweD}hca4j z=bK~C@BO0L4KypcO;JFK5B{;d%#%aN1pZP~$PKc`2c!vFK9Np+$9zrK6xtGu*4`+x zg`8>fWdv>BTsqg9)-&3Ug$&f7gCHo9YXoAdCZ@DF_=*F1je`NEhREYV{pLcTyN@&( z)$B{;$uQd$0!D!HiSg-zcH*GpCtZ|wo&OPhDZN}3e9J#)2;5EcFYi6YyCzpOAVuW= zF_(!8MAg9{BziT(kwCod07Uhdp_k)EhHxSzdcD8Q4F(kaF4IlNB^vxvrJY^fb-IP}>hhwpOQKRu*yv z!0lpPW>~NH&cSjFHokYJJcD^*%=rXrj}|?EtIEn+Xy$$DCT7i;|D(Wqw6PiAN)zoG zccchFzC9WpxR|05_gwz2sb?9f`SVswGw*+%U%#+PeA28`cMy9!6$9)25LO`>8%29& zIJVV(`F@fkm^CcB;yzI|hrEOsaYo3%>)-sIDJ@EXn9}|*JNJ=)ns(Q4x2XyV6pWl1 zYQktFw@mfb>PO~qL8hl0JP7jAPMHoK_bo#FJ zHcfUSQXXzyA~rS&*ggx1o1zp`kMef1q1$E`H{5Nzl5A|dN)Lv^nd;22n_U1|#)U~u zGxNlFO9Nai{^cUbV;6()e*EuM)jA^lgkO$6slJNhsd%#R=1Sb-ovM;YREw{elJ(YT zz{cC1OjuM)Ivi!qL;HNG=epQyF|H_lNAg^0yuPpXLHBvaT~kouBqs^i*Lb?{+$OM) zswhmbY@tz~?bcoB{K?X3o;XqWb4cMkgir!c8W%h7x`kK)WBlv%Adyc2BVCoT?3!8k z>>3W=IVzqA9)hW6lh(?RA7jeL%=~W4Q@-)d(}BTU*;CC`+m~Ldq_R_~99vPTy-4qQ ztthP7uShoxds5IBm{#N>H_Rg5cRnS<*dERCnM)yc>CXAP9-A1E^5>|#iR|e#YaBm4 zRKWd!ANAK5U~)ODPgSq%u{Ed`KT)QQtGOIU+trzs@hpA`eA(KjR1?IyAzV(LEYT6y z${$qNo_kv)N^tBuR%SW+@lc;!9N~9=>wP<>tW@0%S=jfD^UAOH9S0VX*`AcrHx%aN$ z{vA-igBVc*StGJ3ljK#apVZ#v@k-nu#Fq^|9@X`T z_W)_a(b&M)*TPrb2BiKNkt3GtHk~!imAphtk(BYMA@9j)xiIg_`^K#$9f}T)@X+q; zR1@nVI;pE-6H+eOu1pV9;l8gd%btghpuJ|`o_@s>AD7nxQYZdPE1&e5Ku_*JQn^h^ zsgau}O*?~Rzk_RYpB62WO?p#&63CWL0?v!!Q>d&`J~?%>^x7u94X_~n$^>x$AmETzj*{yAqpiKH2sB07*7GqSHe_k#idek z3)_;N%}_k)t%)pL=odZtzVrQL=*WM3x`kix)SK#|`gdug->NYq5|bwPZ^AunDT{Q+ zEYn>xEjCYbA9>jy#!1{3P7*CxF7F!=ALo7qk}d$lfFKIUj0a-=Ij(F3G|%@K`Z^Du zW%mAt*kj{b$oh*#>F@)nIA5{mtLzBudU*yg7v%8Ljz)uICVZc4>8pHKAZr$cGlr1G zkdi_)vVf2~wPZ8l+{4_}@M=*&H%?+p^*ut6KU#K+Z7FKe{Q`iz3*=ZFjp^E5rQNT9 zk?H}ds7K>tLcx7%QAdKzU3RQ9O&d+Q;mN*aSFQ2qiynd`!Xw(-$A?e5ZkB3juI=Kg zoFcxqDdrpuWS<^C@DgQ;=~2BfR2y$#?V@<#kct8Jd-Cj_{fe!w5lDujKr#gUH6m}9 z)|E+54T!D#fX&+GYRKl9H9ut!t);k_G{95~bE37%r71PZDS7XMX)MdBz|Aa-FfP%L z@k`p5!zvmxAiwNCtc$RF_4Uh#!exp4jvo920Lw2*OxY?}#z{es{zj~>5kHE~I#nA` zEbqUh4We6C(M;MJGRaNn0zCrI%ztt1Pf7>TWuTA^@Tl2lECN<;{qO((r^KlY^rrBIoJkNp+y6H|``J*lfb*(+D~J2hLH%g| zGPEDcSm<7l@o)X*kw32ENI_|OJY8zbJp&R+1W?5vVCSezTGdyAi|}49gv)|r{bY8R zX0(9In&JEd5$&SegLhQdNr1fl|D@>*R4!w-inWRV>obSux7uMRxa5?7ZX(-GQF6ms zmg*R!cP968W8B|qyNr8`jET;6UqeOhvWb8#lJx1H9bd_uLc0g zde9Nj(NC57399t@HYpHMtC0UcAPq#1hNGRB`TiT?;Qx>k5X%3`SJof9fDA@yGCGSE zOYxuT9%QjzGt6n1A}#;Lo6541mG{Pa#oVBNob*c z;pPV_fRZu6d~QQEIAExo<$0zV!WNXtIgHwUcnp9zMtEYF<82?vdgPY?phB@=Ww8W` zLMoM|Mp7J|pb<26;@W90*KA!B-l1>1Nl7-^Sc3~U4I9SR#f=iJ4`5TcP*>r(QX4Hn z$o6F1>=_lW+Of2IeM4167`e&n)NhLmw;OgldCTW82VnrFKXItK)gG9{=noP-u0cW} zw7lQx#C5gFUd%T|bO1y!R!TL*62jk8Y_H9b;uvJ9v|Y}Mz|64POMgB+D%2#x70R8M;+B{PUPbLJm9Jz`5=U%&sjUf}uDw`I(Gq^-$izaW@3m2bqm zbLu{~QR=>B`l!Md!hN6W+jIQkO|oJ31>f(VcEw!h7;b~KIoTWdkb21%V^x3)gX$mL zZde_hr&`qx14|TtsS&!lrk;Z1+;fAusH_$gS3lKT}=EIET^<#<{HQtxl7% zukJ+n_g9XdxSl^#Cx+JKdkQW5SaQC5L!i~F-I+RE&eU7U(|=HV69GGDx>|@_F zzsldRzEg0k5ujO&7*PmulN{e z^XhI%&8kj|EJj5A<0#M*a$^Q+x%wE7-RctgY>eH+4_AslA-jXPU-d3d!#vtgmLJIT ze%SgM?5gE6qMlT^X!@i!i9&Xm6(iS83w7*4aX?{b(Yq^4@5kO%oFPHFvFlmrdKBh| zYMtPqXZ^z^LAn&I`>cz2o*gAVJ~&eQ?4!LcT6{$z8)D~)+o9eonr+OJ7^L7Z!#&=x zobo-v4E}EG*wj(3$*f;1uXotr(Yx&DAoN$0zk*dCbv^pl=?$^T#HlqUzDZx{Yh0oo zd+sv#$RA&WQ`gF@KRPw9j283zw?!BW=J6kDt7fLjaP@IsCMnh>sgAEf)xL1YW0z9X%VygQvV z5P3c>REV<{3RvuWpku!EHJ6*=#&;eM(KC2CCf`aZXgQvA^}>V|=UaI$wr-K2X`?L? z=xMmzkLO9A7xIHF0^-h`_vJhWl+@YdzL&)w-?kSSjBeUVWEXhR;nM+kTs=z(@t-b6 zbgOagLVjnKstDBpJ>t6RCS&hyk&lrR0hm4#0}OZ%L1~m3O99aKW=}OKElF^NOE0 zRT-`zA18>5W%^%tda)<2?#VyKQZ0E&nkJ+1a>kP&v1BJYHC5Xa!}X8Wtl z0j7b5D2*QjSToQBNHyy^fybNRDz;5R_D!d>eI;n4rV_(3wlAU57z4i&(=o$+f|tQ? z{y=E{rE|qil9N#S>TpV4uUh`aW=c=n+6>ni6M!?Mb2O{7xAqzErITdHIz8TgNUTAJ@uqCi+;+-6pn7 z!jr>Ot?k2=JFAzSeKn##_T>lnE`tw!3Jf>IqUI28CI)?Yi1KsV5$YgGT&wr=t>ON8 z{w0!74a;n@Yq!T(#J8{gsqs@3>_RoEaU&8_TD+2O7GZ{XZe170;>G*+p`OMlkTqn> zat`H2r7)rZ%eF2QW3h^f03J$N>`IU6R}AVOespKc7(uuiM@;%&=30CyHRrScW}Ww> za7D8v7Ektot5cfKDqBf67S6lyD(PnTWH>|xT{~O@-JG;rC8}#={w=9MiRD(6{gUfL z;Z)gYGnksD^CEsfJAgNz+{=I%_r{AZfBwDkZvOI$8VVT&soT1*tz)Z0m4rkss{Mu)- zDb5HXsSvpy_Pd-mp$9Zre$RDTKkk|y#VzlgZe!`_HGv!9Nc?-v_2;v<2U zHJ@cVFgIzIA~{w=bc-f-g$=RTb(=~ew?!B%_mGV~(X$MiPG;R<6-DWP1=`q|&!?;> zSQ40kXuD9N&SSr{)W&YWefhn-ZG$ka5FI7%{zoKEKDt`R)~$v=*PIfrZe8`b|W zWdk+9@yC(%{*kW96gf1=&SrK036_PWhBjfX9-b0+&!=M-a~>##%&!ELq~J?hm0Jn)R&>d`MewRjxgc@Dv^wIS zFgnf6L-h8v;py33t_GLzHFEJclzNhD)XJs0xN9e zY*>~P*tUzqaVEH5yH_SLdJOhPTdpHBq87fp=~mRRsVCClB02^Qatj#ZMExSoq`XVh zBbMiEp%sa)SVy({LV(~mGZe4+r-6_-T3g!MX>|<^S2E@Gvm`E}uR@cdHD2#= zfZQFtii81jWQrnm?U^A$;7ZZ-##!*~hD|GnW4^rFbr;lg>-V;QQIj}F2N4ELBC$}q ztTG!--0~y{{Zqi^(AYo#k!3hyeo~(lv1?WrtlI=8rcl>%<{Vsk&@3E2$#Oo7kvn{T z1=l@(Ka213kw>y38j1ma46w*BHzM)jl6k!XzcnzuKVQXD%@#`D_f2-gc5ou_I|~wM zT|uKRtrbIrgyXkVX_wz8-GYY(8`fM985doG-w)C5&T>6o5wPcS1@h+Wi(j3dP@vtj z2gQGCEiB`H+Q6(Cu-T2R6OtE}A(q&F$9vtN+eG1rYgN=J>2A#tlCOuV@t%9EI5fc0 z-Y;xwRfY_h_Mv0yBNzi@w~vrr%H3s(la2bL+MBfU!OD1l*M))m5&glnnnckFX02C1 zvRoFx2ImL@BSlwb!5fzGf?@J-$+AMF&;Wz%+Zk^la|(=iW?|PcfoXs-9biNSzVaEJ z`RJqYDBdlY%Xxu~d|B-?x$*YK9?$!X+A94BkU9aDHp%e%8L{wpmV&3k?P$gOZ&_E= zEk1s}vOmmrS?RG-iw$H6`YXZ4xXjRRFv1(B2;}PH!m(LYL3b53IjNiUBn?|TX6Ool z?;M4$2F7>J3^NAJeWNwmfV}<#_am3-d+sZzTR|>Y=O<2ccc4MOJ1)qpHlmi3snxU2 zFV07;ggdh&Eh`3BrdBvE3VnW!Mf{^)d-=ey;7vfh^(jljGtg5BaaYP~Ki%Tph9H6R z#!o%Pg&E+1aZ6p|59ER}$#KbdehaD+W=oD zBlVeFopK#dlys?B5ux$pNSBIOLQ4A3=(gdIiqQ2!pL`MRTC?=;th{(0yD{@_WAo=A z5gpDH1#>E(evEytaWy=VM*3QbtOGZ=X-4}Dm=o0qXPCVCEwvXpL^n@WmR2|BO zLec&~SHMvG!G}vVA3ILx4Ld$sHb-^IYK|!Tb#vR_*-1@rB+*z+2FF*gA73qMi4iF2 z<|7jYvLaOaT6Zf3woz%}BdBzYT(wo!hsXFz1I4iETfBCeJrG2ggiOuCnh-@|{li29 z=&*dLWPYx$!j^U`r&?3BtcD%y{kMXv)Y+)$O5M}d{_kHj%AVa?Be$;a>H(uH9#bnBX?ne0@s{{ur#>-h(6r76j> zE@C#2Uscu{Ia{q-VDV@mU{^x(?lo$Kiw?jPgWdEi&AT80CNf3}N+B06-Z6G=%l&Cb z4@Yx)Z#Px3Sd+& zE+5S^h+B+E<<2A=?-mI}yJGLk&L7^=E(>ey|5RMqv_W~ebVPMHyQXk*D(Wyx@F8q6 z>2Gvh4a<~URNCAgAKqk$yoG2zv@(&;F{g3H;k8-I+OO8X1Y9p$+O6xEttTL{hSF#q z$yST|PGC92*ntpeI0UQH&$Db)a7Y-hi=q#?aBf;r!uz}A1$iz`u%h96_)&Sl7?kM`1fv1kP_-ZR?*5eU>kE6tSPT;jb zdPOi7Qc4~spsDAKGncIpqPEr`(Do`sGx;P#0_{JY?*KhJXv+01^1V-57P{JVjfx`l zNf+Ebqe~_Pd61*pfN?06{tsWEpao7NJ)L=KD_%!(jjZ><9P}RRps-QuPTV;>w6 zHa``nS_GW*xg)~A&@w~Ef>{~eFZYlK4~OC6M2V8KgDHOYk_f5|I+RA?B%x*I7)&Ul zNFb5-&EI8-y846X`aHRL)nYdA6f_nFxNH+}?Y{7d*$n&*cZIf@a(8T3S4~X#%xg_= z<$iPM{i-f6T4w{|TrAPn|oFwIsi)*4%?w|*t}!dcKWe9 znkNG{)Vd_Dv6De^%Lp#9#1i@mp}iSU_^g@X4FlAz3c~p%YGfU#lrJ!0hNysrvxH8! z{R##7%|!=KtJgJTbS+{Ka*`l{cB6GdCoAI|cIhH}=`Q_?MW7_(qz+kDJxU`UHvc0| z3@un*KM&=)5`mg2FA&4$=f^;sr4C}W?(mr% z*5h91bW`exnOsWQ&>b|^#Dr{F%5D2svT|mRz(>A~B9&;HTbT}K`Ux0teT6nl%8h&$ zUlM>aSD2Amm^`T6uD`hCg#9KePDjd1XD({})m(VoEyqV+J-`I9_utV~Qj8~2HtQRk zDT6n=pD!B?PTW*)kjk9W1&{t*>#_2_z|*x8`=d)aGhD#2y!?mmeWLT+&9%E-KgTVb z<<>>Ms>1?qf^D(h2<#{l{L+~lF%trAp(UR8$)J}<)5ZhinAa7m>uf$R?_BeKG$)*U zOb>=q;{PWeV1Vr-N7i>8h%PPkMa~Cb={9}yofHsDuetdI^qXQhz971(6;;u-IN+Z~ z0R6IVn_XLPm1mhS!`-*F8zu4&o1~m^gwmo}A^lnEwr=^NN|Fz{N7y&T_X+ znhGXaSM1~4@BwR3{YjE!Mr>z8ZeewtNk4~DY zb`g*RLd*C#jKv^#4ErSUlml z4V-g{q+{(x=Y6%MoG3K=trXaA01N>b1S0_vRA)E&%9yr;yQF2~8Om1B7#Zxws#FI# zp`%RVACVTZgz7efoTzzZsGIDI!Zw_;4GcR|WpS7xHsGz44wb)7B!D3RpbmNx+$xSH zopsOOj-12s4~mO`9*ym=jBdGZ<39xNiUWS=(IIP(YfC9+!DtDO9H`))l+Qz;;xFc^(rgt{&HV6<-QUGZGpvwjNppv?YM%}&b% z;^HR$N%bFsmFBZg@|&hsTN@k~E*xje!;~orp8SyJGd-;MMr?K(5-#;MtCW&cK0?J~ z)?MIDd_+Eh2Mw*kcAT{h@&~d!+L6ARx5YyJFLE8AMjV$jTj^<8*uAghb4BBs04U=OC* z3&U4h*A3?pNX-wcOju%@9BQ~3gQ7tXHIL&kgDfCVh6(hI;4_>O@QmvEDMg`60qg&NytLZOUn>`9IQqa+}kA z7|%!6SC?3iTvFxSB5>&jpG)dCNoyg6{Q~rliLiDFgef#oy3DYY?3GBM<3n#kQ;Q~3 zaJ1Ci&e#UIV-^k9m25g;ypS34?am&%T#0Ad+dU&p;F25J#uvNRJ_>T>D^(TjH3RD7 z>v@&XjmnxVZLno->B^@v#;oFri6A`te(otpyTB%UQD8_{PE;-;mLRNSg}LSx#@%Q|c_XrpuPE-C@*=Ue0I)_D zmAjgO6P!j*8?ODIrDJP68m+^*??1LlK0$d%v8m?OI48>cz-1RReB+al!Uy47{jyBh zwjrzX_j01`%TMpGm?g{d?%}ORHt#NB0DhV>roRqu#Q=d2s+y8QAMqA7Y)0OE1lAu6 zt;IZ4@0yUOlu{bG3OT|B$0wT?QkxWVvaQ;m0bsowS( zadpC~=bMP0LHvJjG{b5ADm z1m`xxpEi6-xcUM#xwSCd_%;ANfke<&2@T|&dA<8^_j@ikPKtS+ML4LB<}`+?c&&YW zl>Ok|cJU^u=0O_W-glylS{AFlmpj76tNn)0v_#%rZds3NC)h}tD@9o`Ey!CmA~yi} z>3+m%0!i6GI)i2*Xi?234kt2dBdPd}oO5^<-6s0zDmA7CVzSh`*gqnHJ|>u^E21$! zm=pfhqr@uwuZ)yM8xxWLJe@Mk58j#!zn!Yk z(Ya1-iS7?}@Q>6EXKbu4b2!E)0hh2$o8Jh9=S#uCU!NDuHaNmtc@ABkitb!FMc#ZZ zukB0%YO8pr$9`2C2Cjm7dZijX^VzyU_k)*$EImpThX zn@ZQKw^l$K@(>A%i3Cw5kq*Xw60lVfF7+el-MUI>PmFkp2Weg){5`)?P7p&&?8{uY zBumXSE_vRsN*)bxEg;3hMox!AYDlB-aOfv!98qOkQ1**hz;y1~y0P!q^xxy1GqZ78YK4Stt>vmZzfcZ{IutG5hE) zIgs&||Lc%BA^W8*A#M7Y!kYx})}^g1f4T`bI8kn!2*O#CrfU8L(Me8aHLl}GXL7&P zFXUua-XOhuk+$*$tfF-5HMwKe2~@7FEp2J}oVJs>&IB>3!)=itYPen`2kpyeT5NP3 zad$rznQ|;c?T*@@d~ZNzb2aV6jjS9GF)rt^i9TKYqMgFK7mIf)o=joW`Pm= zrP$!18`tP8tf5hgqFk&c2XN;jy}ueG?&}}H0mb5F2kbcIy2NczUQ!LRhzVlXeS0cq zxXUji_mNirP0{@0?Y=mrrv7aW#=PWMgZ@F4EM6^jtOZgR)kc=tk>EPL(_9MUu>B!o z;o>Ia8!N36;k~*BIvx>TR=+bw6svS2ug81!hkoB4mk3xMDjSDPbOy;}LVgTo_Zjt@ z$bN-$N^mhnlI10BzqM|F%K?f~91~0=w3?t}k=;%%`;|axH(@vcbdTvx;`|eX-V>O9 zd#d!ECy4&Zg{UE~dSU@K=fEiU`irNdDaraj9NF0U< z!$c!@73K_i{^|xNOG`bSW?c#P$T($598)ZfET#SCH$krbf=!jFE>$fuM z8rOZ(BHIOM8&Q#;#l3CGqJIiBXV^wQEwTMRUt>OUvNwgI+#VxQS>jNW9Nvu!J?+cbX;+7sw|I-0GI$5Q45Ol1+pSTYXS zmgzHotEmB>lHtA_tzC%@fIaG4<&tc+$l{p@{E{9V}CqIzuS@EILPi9Ggl#H0gkA412 z)?e=eo7*wJIFYaHa7{K>Xbo6JMbt}-FCh%?oZWo^v>ty5rn!T6C*@c@2k~|opOJVU zLVLfl0X1|nw*_M8uSoZ2$s&$wTlc5+!_jUDH4QgTh4Z7IvIQ!le>s$s2xMK&1xk_+ zcQfh|Vno|lo90Q`R*)7^Wu%HtBv)=daH)nlBIQ8VHfJ>%`x_X8o%s>#{b(6E*yd(Mo&C6g zZ=@ub%-0FnT1AeR3G zfQIdufOhQh6Dm}aq_|{gPshNE`i({nXvTT7quJm)R4l*bXSCJ6df@kfVnKAdI1KH#Oq&d z6^IA*`qy3y#sYgF-j#gfN};Dw8b=<;T6fMJ$g1`FO~r`J9cd$Z-5jAGHF~`MA4M#` zaZPH)Xb%U9?Z4qBgM+4;5NC==4o!?`?wVALsQAe(my=-DXr~>w^}JZUigWn>2W7+7 zd-iaH?c&j8TGH@iI7;H`5K1}3$xl-4%DU zk-KWvrI2^>|A^y58_3~XLmoTlYekxk;@Na_R$NZil3$TSg^8HekrRd|t;U^$Bq@77 z%-K!KV*J^10SqYt9?Z&M1fdZWLt|OTgI-v4E5CESQ3z(uZ2U}+m@c`|Ofb<;%gp<^ zLn&9;R5WN7*dNKnDyR|YziW>-pQJ%aY}xV#_N1I&K8D5?OKz!dQVQx@!DX}03~f&C z5=dKS4YT9RE!FEqjV|EJM&UwBIl&LRQgL?{Di?3{vzwAE$14&SW29S$yW&bq5IzD@ zz67QcML@5vfm{hIzYw-DvXjl<$0S%KFHUq{Ld=);SaQwxn~cHd4YEgI`Fz!_x3=_y zhHWWAZud!0+%53{DV%(^?xv9xZn-V#2H{WU&j8OtW@KoYVZU0nLlsMKnh=6~I1&uc zOJp`__$_o8x<@K_>@Vr2J^|yh>+B(KzWo>&!$L2w*FjSAwQu8aeoL$KWUi!N1^7u3#S2C}Xfd2b#4 zXd+d>WYv`nU+Y*3&0m~?sTXF|2SyW8)U9K$r{K@_9=QwI3kdtc4(rT*ret9cF`0#} z<)96oB0qd{{z8uAjpmS1%5ZB*W^AQky@%4;YnTNK)B5%K1^?A~(dcu8>w!Sz1Oo=B zh^=zkhhTxRj!W(^NhK<&+Fz3zPjehJ*_eULyrT1eB!RA+AS5{6XfshJVemOmHJH`t zLtl;8)Jy2W_Xp!2_U)$qW?lB)8Y)7v+0}Z~rWO|pC)OfI8xHb{%cLw3PYkH@0mIYq zl0de4A`Ec91||w(yOD!>myYQ|$Th)WKQo_;)KeLkWnPfm*%WMQTXDWx&t1~^?^<4=okgpm1C9Un&LDneSD>|0a?K) zN~F!newwqM&2szyi z z@G8}VWai~kwQDvrQrLP6V=tKSn>Y6mR#{*_CXTf$J3u#feDjyLf58%r;-7 z@JUq@;*RwT>d{}>5=#8cS8?}Ylt}H%cv5NfGnqif=#7k4AS-^^TWNP1e;FUBM_g2w z5G2}>7E0YoTJu&e#OdW%N`GQ*|3pj>E8~mh1F2uUr%n?#)?8kxluiv43<7tBt{&7hu%L5+ zjM3gwk8jVuzrDXC<+j(U2`j~K^{NdPCQIPWE3`lS(yG*fzt*yAH!hYPX`zEm0Nac& zA0AZ8<_)$~4;OZR-!tKmqmuAG{^-UBew2GK-*JmVG#4mBxc?{+ua`AYU)G@H`iYn9Ob0&mF0assrs{;n_H}lM(^Nfvfn)$k|LJ*rEVkqyFP0DRE zLg6+qI;dB#y4e~%BGLUmk?VkeRqADtN;Z1wVtx2PzFMzOzonYJlhLyA6R31Me!rR0 zkHC>dzb6p-@!;mQ4tfq@FdDn$77gs#_It>{uF^{(gOBmcC)#|3lp+s4Huv$rj5U79 z5sMEEJocoB#CTC&@c7}dB9sPDT32keh(YxCpF|FMuVFk0;385OJ;?%_nY7;Per3*< zlsTD#2ou(y#fe;X3nba5Q5kPH2!mKpNq6R~7fS2*-3#j%b>AWrTH$3OCxF6(9G0uX zPpaUu5@#~MqKt}9owZ8K4m_bPK4Ey!8V`jfR&a98gmk&XWkW6S=us|}R(4&hp%}@W zzt8$p0Diu3opI+){5{(=f@9%WB0`X=vNxwCoioY7vIO6URUAVu3L zT`-lD$4cJL&^!7p@I$ncB-bZ`9OWka)U9YKJ$OA65&XQEsMtAy2NM0=S?OiaShkX2 zS%hzKr{^TXK^W}&w?b~z`N%iqTF}q4<0A!&e1rIEw-58vb1H#ICM}s#J0UlJX{%f3 zEy!&sUE-P4{lK_T_8q)Zbem_Aw+tr<;sMq zFD~w5wWM5(jsryoV+jN;rb1jdrMLK@(uH^T2dfi#sTTdzCT`NI*kKZ52BxFN#jCeV zY^AE`%p=2W2X=w1xb+hjC^5s9ytl-dN-svDmEYD0AEpVlh8;GV z0_A|)_Cl;_8!9b+QtutvL-2)7Y?LKb=3c13RdVojUscs$aqLbYPUA1k!TMiZGv9Jz zo_PDK`FwO&0oKRQlvIaGnfWsQP<2-i+DqNERCXC%q**LkyEC3i<>o-WR6_i;m3glI z)4r7>+k#3hqkDbhmUAwqg1@N`eTNZ!5`M0+L!Y^-M6T&Wo94zrB_CC z?gb?J_YZ!?z(FwT4={`%W(>9kbRGSe9jv@>sz$3&a_UQkRpr+gtXz$sX;r9BE zRWTxx*OTkN<5oVRlE9PJhdewQH-)QWE4R&`KkMsd`LyX#$3#V!iIu=EmO!_-^5E*p z)h17b8cctq&((v3)T@TDD>`?<*r?A1M@2?eC~a^P`~vS60##kpb@3`3wJ-~CGLFrnS$Xfz3@4p9s2Q`Fay!OI<&7Tl|mDYuh5#^t?u zUUZYZx0u{-_82tCFPpp_=5<+bw*RD9JT*1+||3 z0g1V0+#+@Z0Om+1WaiHC7O=nDS@Pz!_~E-sy49h4T!7UX+YO%Y7N2`M_H@+2v!vfU`n& zUwE7cxLRM=hi()+?#B4D*MpS1D(Z^Ev;rHX&V*6zy8kJ97_?}y=@rhiEckz|zqNW} zOa-mkD{`n7*3^X(mzr8nPFS{!c_-VEY$rWedm}t1%0?&Jw*~fi~xAZ8PFEGn>?x=3RvRW+#KetlM07*V|W~f+y5==R)dPBeWZ~gx-Q5Quy{K+!kNl z|D@-(55%lF!Jw0fZtNkd`@XZ1ToMt^2gwM9FW5FIH>Vg}5HPXqFthUGJ zV!5|i%J@#kl*mC>T%$-mb5co2F>kJ@8d(Q$4~;z$3Wvm zbtE!QsK@O4!tP)gmKr;-tfeJK3IwpqvS4oOSxGiG2eoj?gUQZ_7yw=A5-+&_Kcu~N zSkzq?HH?LVAc}Mm1J8ZZe zxmR(@)E$vpYs8%00Wkw>_wqT@I_9N5LOr412xyK6<1 z{3(eM6G+3+p}dcXz!Pd1Baqpk?#VV=96;ksdLv6d={Xz$8IOUq!Yu}9{@x*GK(cO`+8z1d`Jd4Z4B*hZ_Vbg@K z{WnHSq}|7hB~v1|vT{LY6D4#c5sjcdgTbx$e)9Ol@RxVKna4}ZP{QG{k#nE(;U&l1 z*QzFZ3S-oL0d~J!u)?F};IiUzd$300N^sHjfqd-Ci<@EjgLvqV=IA!&B63$`WC6>g zSSJ!+zzew;WJm0gQE>t`YMw6y1G0S3?FD`&39oSIVD5;cc6S(g&`+YsYjYT9<~Zj? z!Gz^#8x>oj*^(eoys=^$YICX|K(M)q`%?zZWPwS#<=9ITU?FTqOb zP!{R8lE!=VftG$R6hX^EAEMyV%>)Y(N|ZFx?XdNzzVyQTNa7-Sl3hlBxFZP$gU~|? zUHxafI`2bxuIjf&vguvFR1$70g{T_a7bbL3>SczW*jw^@7Br@e)3pFb0CG!X(~j(? zAhpHHZ8XjpBR4~mw4b9PyfwJ7Dlu@5;oK33f3+~K{RJDuegH7+_AyNVXpTy*do6JX z)1)Oyra1^(V+GuOou=`f#T2Ox#Tm|tCtS*398RO>;&4LS!aLdw+I_Ad#xuI{T2o; z5rDL@=8Vi~X&yZvD}yRWoL6~Hn9H?%l>T%|e9vI(`tE`8;S;pQVy3t@MH+n=+e}3D z?_Ci<8CGYdQkR6*qweJt+D$wvId?OLPeO1Nyx~%m;*#XMdQ}L?)Nh?IqMXE7dpQTc z{6Q_68$k-USj!YPxQ9=AwOOhL?{S19^1iltPCUhsmtf>HO_DmP9X|mXLew@@#tY$; zIFBDry`TCIt2pYyZVPkN9?3T!O~sk|eO5iQh&zefv&oGQ-Am_~J9;i`Pv$qk=59Tj zul|a`gc9UM!>;}<$|c!D0&Pc2%hfV`9<`S(8dCoS@j#R@&*}}SapaF7l5@~CU-GD* zx!c6*|DYXxSbn#uysmj9?-sc-CUr(Q5-fWf?LFDZT>6Jc|gz*!XW4wCXizsG)F)H7jhWC)X~quiK!nC`d&V2Q z|;&7ikiM}_*XUx$kplEkzOGc&q@=4Qao#%u9g9+K?(9&_=)JAiReU1S z)b=A2+tc0)*WR&XFlSYhpd#GYoMWx2+v4K`!Zn>p0F{dxvN7@KgT+8kd))VPT0kkv zhT@{mYyhpw8ZA|c0iwZ_?0@H^MkAVlvZvUy@JT?gTrWY=skCmh%u8cR0-~(2cO=eR zE|i8sutzJGxfdyIhn3&%aZ6a`?K|o<(?+m6miZv5%9L!|a9zn-E4+O%WKEB$mx%Jr z48BpV6VUFy5=vOm`xeKGzG;+#g=tGxAiTyNF;D8TvevYABf?W*0+}{=SfS&O2h!Cz zdkOKx^|_*O0`_c{UM$&;38>EBN=neDKRLkkf*Ra1JdiQQ*=28~*SptdR08NC*551k zuA6GrU>Y^B$7IbGx~IeIDJ@(s(jcRUE&#bgET6 zVm-T0R^w|g{FVT6rQQ&#F5as#!GZQm9EUgOEpU6o2VT8gh|OaM=q;qKIH4AL-=%%GxAoCv4qhCY*H^^K*>ecx;IQQ(`AG8UTfl<*PzF!g5rim0N=*^vbMZjMp4KkNq`m_p>!S-G+bq zq*yxSR*-a}NHJdNg_hx9Ch8j*~*?b#Mj*=qcuu_ zNCBoszVyh@M^Rmu*YtM3X2n~Y`ju}css%y5;x;YQs*CD!l?hd*nM2a$ADa(mdTb_+ zteSXrv+>DA92t|siN1z~pHLZu64>cV%GZ|_-1mkq$3%!Sw9iaJbhyS(#PGhv_;nIi#u^8DMs_RcRN)mbV?--6S-8C3#_yO~7g z+1R3OPBa&mUyQ1L^|GpdyW>R*Q&|+nxOsENBy`3o)!Cj;>Lxh`L+P#sRLZa{t9Rz? zH^T-|uM;hKs#Z_o6ENZ?V{5G4PIaAKYc*ul6&(|He&Jb6CVW~x09|%zhdN^aN|;YwMN7?!Cyfb z04_tEw5e=cTJI2fmDdC0KV)m<^QrFxqW!z^69Z6bvKYF@fJ#Dx>$nylG#&l-P7g-} z3MG@}3+A(68H6+DM3GJhLJW%@^^_DO}pV_Vj zAlYvn%wmLQ$<{|y1R!PHBqUzAE(|!fb(KoqpX?kX5k)$ugdy7b{!va`or>+|6ZgI&kLhfDpsj~1Yw_UtC}G;YcaRV9 zR)qzDhF<*E$mggbcz%Og8->M+Ue|BI^7>)hg*Q}k)XpJ&Gi_mN?|-Yb|AK$lby-k~ zun;$A-N<1}-~(7#JBp(Ul<@y4bIyf*QT$6`1~31TR5&2VoS1K2wiS(7QFe>DEm7toBqK+?)hN5K*8(*#FQ1=>v}i+QkW`A6 z7KQ(JaXNzo%YaA9Ys9mfKEmL6EdPTI$-&lh{^l=k$cX^qlvSx-rJXEt3|}(>_*TgM zTp^4Xy!ddLl7vg|zGYewl_=%Ah?KPHp-uAnJS875#FwEqtK-!b*qf+C#EPjB7dE9x z(8hub@6o9@!|8GdIJW{2w5P@t%|};WqnWR6=~jC&=NSt7$crbI-93rfT*iQL2!7;X zBLrp2$iB2K{kkS|R@~xBmr1w(oqG@k3c`UZT$SEZ=ZmjrnTtTNRx|t}@engbXxd+& zU?6vU@5&4pUb-hFDzc?5{`s|sdn&CXqa@OqW=z4R>3Eowyy7Zp*KRntF7Wm6C%nht z!>P5Zwv8SuHGhsZG9&{z(mew;NP2c>x~H0tQKGG`xRuDtLJWd-N$rNDS%FH*+H~;= zg`oFG>pifro~Njw+m-7g<;QxwOv_b5P5$H zGm7`q&X2*xGbpR~@t|N?ONBgdMYCB}SomU$_6J3y246*Gp_!nBQX@&e)>LWaO|qoH zskdiT4cG1he>3ejuGPJ{ zgIpZ}sQLlDZpq!v0z*$F9KCGi;t6u6s~S$Q#5>OnAHlbR_tXu7k7;h}iYMkwsgC;f z-E%EZT<{A!rVm7XjZ@+OfLHvUjrN{FWZrxAip792KI@4xd6U)*xpd-3z$C}SYH>N9 zxCniC?(w<}`3S(__eAiO>4&Mo4T$U1R5}(ar4bkidDSRu${#P5b)l<|q?&0vP-pxJ zV+sU>rXWZYnJ<4J)%{_{RRuapd{dR*MWdjI@Tf^7s29Zg+}{UNcz%$SsQv~jSjd2K z!SngV*H05;Cf^-2;AYqwUAM8a)C-07uGyo9dA^$&s$33ldO+Wy#=YU1`AbT%xW<88f=}i=}4M&V4$hwA)tWyhbh?2G9N0pJCS|4l zeSUivd0MLXfPL+J<5Vievh&len$s^3{vYoCRMoLZjw{B(ciN?gTVgwrA6f~?vG?=3 z`e^XKzoU~Jd9m;qHo0()88u0ilwQd6MhfY?F>xO+B}wf4Ws_Csq7Q8J4^2{x@t7Ni zeW=#;jh4Eapx?OnvrY6nf!$A0{~L?nJORE zyGMr>y#vzaqp9$8k8MfK<2ecg4ZHwI|t0zi>d!p7$T!WZIFi;(Qsio=r= z5+*Nd?BG)}oOzbw=B(DPS{K)Dpyh;qn@l=5_X$FzV_yFXw5C6Jlsw z#8R)j)K?U>=ady*8MqgjSdW&E{yu2_ zvX)+J`WDj+9NPRRzdxjV9upUvpPtSScDirOM+o-ci1TD@{hk$e8#1!imp!g5X!!f; zqPF>Le6z;$oR0rsp%r0?LkI`vqU+_Kck|J)Nr$TxZ5Ve?|$i|8_<$dO}$MZh# zHtJ0^$eK(ycnPFkZO0T7$rVHlfq}ljmH0Pic(`Y!c0A!7dz>&*0am@wclf1;@59@q`EyqgCHqX&6ZH-XRc(K z^(s~eXYw_|am=l6?-uE7CRoo*h$v(=0~;w=Qk`4$^oi(!c@6s^(Y7pBHYC7nM4XjU zbtbR2r-Xby3p&#(zUw<0&4W&!=Y#PAif!U}-o-5nDqjdr{W7&)@t?CAmG>*{(5W}IThg)H6g_pIPqgl`e zk683bA}O}!E2gm$ql{_6KdIKVu`u>1^NQc9@ivnUkAyB~s}JovPm0#t`bT-J)@zJA z(d7g)yd;rSuX{17Q?FBd7!`?eJzJokAzxTH8 z5306i1t9sZws0R_R>B^oU1ls;el5W)# zaogT|l$YLLjSg8VP}IsW@MRN5PA3Ty^QH2;Zps0Uf!d15&cb^Rw&f8<2*`70(T;S0 z-XUI>*@y;q8k?4TU9bPoA&^))x_;yO5SSu!kKxz$k6Tq1Q7ZArG316#B@xN^v|cI1 z$7h-~4pQin!h$J0E>vsn`cjLaL6FVE_wE=vG_WarLQ&~zYOBmmhT6-OlLajkIGGG& z>LcJo+CJcy>Zi2Hi`*61hy@kh9lOyFvuk}%cEcF}KPgN6f}*0{54!EO6mWvH6l3tn zlh)^k?RAIeAEkS(ClO}k(5L$ZrQ6y#kd~AjJLF_;{Vo9+Dj&vj+RJQ0LqEypKWh{w zqeQH@@Q=!Qh

TBDb{Z(M%_5ei8~%RSOQQooO>B?IL%mk8Y(Ctz<@jE|QZNue}r7 zomf$Gry+pCen>w)u^gnshm^t^X|0l*S~ZJ5ltM)^#6MY7Gc1ZlZdlXIy*D*LAHPQN>y=^q{#r0=7tlX=@9f8K_{JF%51P#k`5Z2D zdS)J-2{@?s6~XYU0FvGfl$qG)Vw4AK?NFn9ZGW8AY1_OUiNy`>$6@3Pih4!BXewQxlNvnZPIlXSRJ{l=>GMTRN z62QOMi*4_%Swa}j&&%B55GmI!@9+RuNc%VRQ|()29|1-l21ue06I%b)JHmjf57avr z>3|Z(spU{kTFBP&{J5lsXDhfk7$kmO8bj3ipJTQ1Fy}@JpR)49i^E)JmpK|%P(cb0 zC*ssi+g-Pq`QT^!Jnd`=H*e1=$y=Csu$@STS5W9}dQDUPBlzT?VO(!KZ${U{u{=3= zHLFhR^mG}+wk?J0MP&*9RZ8Q+2-}|%5!h9$pNXGlMeCn)w&cTpuG&9KH!d~uR5h_8 zLbF-)n1em9!#ujxxMztus~FN41pRj_z-AIuI^#1^9vFBd=M2xp*GvY5#7hEj48R7C zekgiNU%I$TQIES|h2DKZMIJ@<9#5aG=-`cBYW=vHWzNiGhaAo`QU>z{YRv9ZG^}&2 zI)QwO6b%_XueWnzlZHSX*Z4(sYqfY!-?&SJQ;RsW!zCo|OX>5H3DOZ1?j!|!KAVv& zykAL@VYVKLjc52lN;qWk5CLN2M_xd;7b9HAy$6pH$?+z?&s(0|bx}j5F4bw&m4_p| zk$sFq%$*to$;dA{FfTePNVgH6|Mbe*JJCjwn02YPtAUlHYiA;MwrK53%V>*@^v1Ch z&e)1nhv(FqpMJlBDJi^bOR`Q~iaxzJFSwM(bC0_je-OGf$8cG|3FrtkaeQP*Ma7-Z z%H-6Ys7o6MsA?_45dT;EeRGs_^8xysxYf6l$W#4mX za<%DGpFd09$#zcj!;)Mt3x(y!!zx-N*1aL%QXMC=Nsv^F)Vkba&kt({#I#<8g%|;m z=W@S(+IniZovu+;EG;k2`+8x-XRxw}bfzOLeUY4~y@#e@Z11U%7<;$}Z5tI|xs2k< z(|iHl1+FrT!SDy$IX%LsYIPm z-QkFKX6l#(@<}n*&50a3oCFrwo)Shuj?+`yXXH!JAx^`S*WQJi3!`0|L68@vr!c0Y zbKBrE?_1Ijatf=H4ljEs$DY&dx^0NQ^|t+8gvU?vNM@`A3SFu5BG;I)6;J#K3Y&hv z>CYynHmMBLRH)A4@Lh7ru3k06_nW;BZx7ZJ_GX`EE`c z{eumIkn%_HuHUR2?XdcJPr|w#ACde%qX7d>*dwzib7KNLs@1D!lOo{WQ{X;bpF^I0 zm*JqGI?ino79vKNMB=%cx;y4ejK|Bt8eP>Glj!kDwDt`~Fi**iT(dv5aRe$3O_vEI zDgakvpdI|=a@elt-)A@)*o?o_8OMtM5N;bO*nRFkQdOB$qt6wxpl$GKNv?Xa-pyCK zvF`oTl(8Y7RcU+TZ`FsPp3ewqQ(ERj_o0dGXGtQB5c^sSf^n#5Jm* zg7Ig_VQqU#)~1(W&W&>4h?7HvDhljzga}Yc_|Y zsaO`}oB%$shU>}{AqC%fsY(cey#8ymM!liX zw5KBuxGz^>>VthjFDP+2YwGx8g>)S86l7OFa_Pxy4&Kz$-;qcHF|~pQfM~QG0hcmH zwSzuDUV2qn6HJ@-SBr^lNd(F9tOl6G%=HGp?HHMQ{kl=Y-dtz#t2q zp(I}X%JrhW0~?HHO^d{H?>ISvy;UC%65z3~ZxPSAz!}#D2t2P?^3p}1-)j!mT-lKM zdd-=A&zcwvm(LAVtK$+GHe>AF9*f$&V2jp|KFE6Zm+{Orj zk_G?=Qx9GJSIMRe^Icb~hdOiK57piN1J={J?%{SFzQ)PU*~Nz*oWUI<#7^!Ld^ne1 zb}hf#)~pqv2XpeUsGgUTz-zopG_t~8Kc@{p^DDwZ>P3kRHj&$mzJwoxJc>2lYEl0b zi}?V|Cv|AuU5cFfj0f3<8CVV9qU>fjd{kUH;fA0Pvxc7Ai4XuBu|WotMlRdokGjI+ z$=WZO3dyk9BgN|RfPh&p)_+oLwlsFEsJ7-NC>$J�BX zwNZbS6-H>s6TY9#syeT=l5t-z2b^!+Cp_Ymn?6btiEhE{Z~>Ex&<5s?47WWWrMUAn z_!_=TYxVrW&#?J}Wyb_&(rq?(P#Xg2TR9(8*N4)8csa&C$00Bf_SP= z@ZE7nBXVPT&T=IgeMYczx(AgSaR}z`SNMu(oP4*vOh@utk-1tfyiKsvDL_xe>6Xj< zuqF1_8Ez%r>eS+0nR7;*&LvhG_Ngxe$%#}6&xwrq>DoKBE7h+(4Hocqxl@aPnBCg! zzC()&rwp#Y4sj*sXPh;kEjCURTa2dXcwG2$^;B8KKik78Rk77wqalZP&-Q>^rwS`CR; zFp}KTf;NqC1M~W?_lu!yuKkkvFg{Rc^jrd7t5BOT_$}} zq(zNQLP!ZOoo)=_@Z~cplju1dJim-Om{8MGv}< z12>u3OX`j)(;JJcW6qV!Vr6`TziayYL1}n`ks?^sz#KO#tpD;E4R5sDxtH36{Twm? zM>sR9O@wx^>j`GBq7>42mh)O|gcf%@2_d?1jyC&*mlR$(+>ShNwppL2ix_RQ=q;cu z*kfKpz@L?W+27-=XDz6Q`(z%6h!wk#qb?=KV;&^)!<8RVTRa2X_v)t&i-P zlxD~RSL^vI|6_^MqMkMDK?AOrhZal{PN%+1MBy^q|QlP=}m0`jOP;vaYqL_$%kq4x&=ye{CKGk zeZ~OZ3L1S(6qXwelUp6tDdMG*D1Y5k#fy1Kq!?1b!T2Ytx5nwjtyr(3{LI%=AuX3F z(&Fm3Y^`TVzg(Wq{nZbAd`gN!|E`C%t+M3WF3}EH($|)}Ylz~Wf$h$FApYXA@4&-v zfUOS)^6zh6jLCP$B?2R{1`Cu-@vk$+7ezJntgr^g=@hAfWKmEM(ZZ?Z2G}3^cyp_V zda=3@rJ!a<%j*g#RM&xDO0V%JBfgr|I)_as6GUA(-s8t&{^9i9-60dZj8&I0)mMNE z_6w9|-KtZ>gw_EjTI_?{)HHoXZaudu>$%nz3420&2pG8+a9pb^Y?xj?``qpwBNqB*)sz*HRYDKb~0K9pfAnDd~9aQEW%bo!>2ly^|O4 zbsn!fzVFyTpvajcW%$dZw!(=oEr2`ozVOvj$Hr`x!}m$HM*okpMTtu4oAenfGr6>K z2U@I!LK*e_2ZK7j!bi5G{A9~H_0Ek-$e5!|zAqXAHE9c`6eC6QJW z%KrKZdfWgR?`??p;LNsPxTo)I7{B{Qt{9=mwumE|M`u#$i3Ccq!r~_cEH89Xf>K#x z5YHIr$Tr38iYI-WC&q~ z=GXVISB(J79^E{6%H_H(=DGYy>u~?vVJQEZGN@wG!W_gNvd*@KqQc?@NaovQYh5^g^Fl2(pqNR!a#xjU_rMhhgu*(!doMtz0#=Y9j+6R-)JZ~0{_Dwl3~2%YGq~= zw&QW$#)@&r>NkW=_QMW+3vjx8rcG;=btAh@%{nq~WIV`w^Ic|)YSkKF2Ji^d2J4+? zq&(}}&PM@8HP$a@yJsm+!gtKfk=If30yff!OB5fkU;y`%i4r=!UAE91;q2Eex|FX- zg`u?EZ05ymXg`)553RjmsP(;I*KIbtwerVTr{cEr9fxkX=Hc0lLs4dd+xE0ZffBiO zx7oQLhxQ%50>8G8kW7slGeTroJs#lgo6;9u7plbwdz+uV+jZew?a#*HhpD^`%c^lZ zbz9Wu(9!in1ry`bO_YtxO%wQ=l-yF_PPHRb= zI7>RKBiWPdy~r;sO}JTS>qEvH8ObzT0szw z%yO1p48a#~t=NlkttjEaY94W-9n6rpIxouX2fsk!`pgP&+wF~3wIbS*SP z<4Umqs%B*KnWQsu5gEn3=P!Jn%*XP&fNaIhi!w`y{{WJ$*FqB4R1D(Tb5 z^I}SiynQ-sxNA}qbrgb`7oKrg*Y}d7<-MkLu`RFV|IzmU zgk^)G@14a-sgNOI{(}`tKb>}ArMcUI)!Mw3v z49HqHevgS8{@{{YQC~34?x9SresoqC!=LWh(A;eW>~>8SvmQ$f^=wRz!^%MBkYZyi z^`aUr7FK*2VlLie7U4HHe7U4W8Qo@@^g-1M-0wv3-M6QFU*vc8X~att zt3a6ot027`$nuZh0m;bf%NQt50)HYbv^&P{?334DU+PJH z$F8Om_X1YkNJTGqNBOk+wPCF#dz{G@b6C~o;cjB#kCR?GlMbr$P*iLt->B-%ZrnOtO^(c1plUsrvD?L%>QMj)8yGQa3Xs936|qlDxNl9`wjlA2e)6idNPi zWdPNGx${EJyg=AI@AmF>75%Tl28w(R$Us9x?GF%y%7zJpLdg5&9()|9<2?kT7p4nasZRdIwM zRy^?mYX5^|wp(r@`~4^h10Ud4Z6au&)2ByD-5L5Y7^M3CWTs?KKUXu%Zjx$~qg?*| z1M)(zD2kfiHHM_JvTLRluzoZ4NJ^_fQ9c_|L`DRuG>{xGXJpGQslfREXOO|18R{fS zAf1~-jb+6Xd;LC3?3O7!u?xpExQYPM{~uOvAuadh?f_3Xei%l#VkY-_ooa@!Pm%H; zU+X7x6hdA=`I+{)@aSs>mgWN@!4EBZT#5dt^Nc5=?;!27yk#c?*(ZwUUMN2cQQ9?m zf@;FIDPQIeA3-(#jOtSrj_4YdznRuj`Rf3v7M<0bv;Y~lkw-Z^(D{D65KqAy`{KOf z-_wf+w9pGEn%O2oQt%O>l=Qb-NK1()9v!zw4m}q1ES_!rej!c5>r;iPX8Ps#Gpohy zZwWSYq+#=fW6i2Ht-OXSSA{j7r8iDw-$6Dgm<(5huVtX3C`-FHDKBj&Q@lq`b%0wG zZ^GF~@w1sPk`=jNs7kzbnZd+=XJ(ECivg<2B8yb}wRuOb*)cNR;0wNZ;ytVNm>k9M z=3dmNlobNL8_|o&Sh>tNQ~{$X7YfuGTc%(*fxjGwF(s{*>zBeG6K93ZEMwrMJ}TIG zE4dr?R}TM06IHvxjp|xWPT^BN2+lvZc!wB%S^fdg#IEgoFXG)uWtu5jgE6e_9l!BLRxdT z{yq_0rk?`gwIYYhDmgHklC{(4np?EbXY7n}X$Q_3s3ne14%fd}qVm~{mDMyAOx)Yc zZA?nreM%HZ0ImDnqWww+xlhl$807-oHc(03!>r!1-J~<0>z_sw#W)SWG(y#!+Q+!0 z4uXHCC_iuhJe#vM5(;Qu4<_K^&En9)bl|QAuaLtq#CjYG{`4LB;+Kv~$U|wMy=ft(%@mQsT~b1<=PqglnSmIDr{o!*F)W z{>(U#DoY+qK>z=Gw`vC)z%m>elvWW1oZ!nA;4pLTo}t$}S!oVdi6$wb`1GWsb$w41 za}I8ke%ZCXPuLC!i$I%M!eGb>y~hqrf$6Q50w`O{5>V;_<~Vw`|j(1ld__S{`297B%GwV^4SOT zbiuFj13CH_E100M;X8;Me$T&q5{0S}5nL31re#V@PbbZ(cO))AuM@6_!0?!JHuLX^ z=u-%UnDSmPXr2>s3C6gz5Gs2$qkoo9hbY_2IIun;Y>Pd?zy#h9dVFC#>Ic4F00u!%C|O!$ zJQ7TMHF!m55DX5g+pL5dcZLNnxsK*jC64HWVtK|Dre7D`3tF>*tVSl3;A+^DR2@v+ z9ZNP*NhvF5<9~!&gOh;}pk&n%Peh%{tv4>aMffoP#wD~j!_QD%z>RN??Lr2OhO^0t%HO8u?9;esNFlToxRY2h61tYPG z8k&0)YiMc9za8FR3GW@4)8f1zF*5f!QS*CqkL@&RMn4V{l>J3t)}T)yA5ERPo1=`6Kco9cmFyxOE?8%yHCT6r}LUoHP9IUxoUL&YJ^dfO@Eeq*(~e5-SAQ7JTzX56}P6#m3AW0dd}E+qT?M;mHyiw z$6q2`-|2*5fna=BK^x8_3L<^G=#O1zxOr0I{D?^y`R8+;1C z+M8__b9bU_ME_CufDwVfrdHfbWst+0OHn`(0s0F5GcJgLh#?TG9HctN$MxH0fVXCb z^NJhhLTSED`71;;Y%QZ3k;Yu}nc~7>yK!E1x}l+{Js|+U;39ms@9`-=NofRzm*$-m z;HGWP8uJ214tzC$PnaTLT=W%ylXX#=GTK26aMAz9MwCss_qVH9?rM%FbZB4LPZQ#Q zv`2qHNG2l&RIWMesMn!r_8_Z86XkOeNons9A8DO>1)3u7S@uCpQwq5|U+5aLnv49u zUe6gD|E2v|u%rzk8X1n&B2}4et{DcyebfU#VHk)xKVGk#ih{dt=HDd5hkO5inil*Z z3z16m_f65zx06`_7{Y|FaL3{pSIQe_LYpzaMY6d2jL7l|e(0JjK zX%cdYne2cvduR}Y`*)%sCU;%z90TLhogHoUM;Bev9wN_22SlR!H z_2*Y{c_r`9oK5&X+^zMSX7q%ct7%jWao*<+>p|Q`t*H`Ism*fc@u<^F5xPHHta0wA z94IvtHHUw^1rzW}AE34swxMhY<|y0|B|!T@Br`#UAxj=Vp}m&!t;t^uc=W#D5FHJ*f!-~h3n)2S=a0Ku{@BvrKQ>zeYjirMXRyxCffdVu|Qz6Z~|noc5RoD3Z0a^w#;9aJZ_!*DVFe40_-p* z0-U<)7UTz1B)hrD3c8bfX|vm2PEGOS(O1fp6zgI~`d8&!k-#yj&Np3&OuiGc_Bl%nlV0@mc*MWvEvmkiV{Tkda;iVD9wq6flNt4Pg89`0h+3Iz zUC=kZYi(V?Ncg1+s_6iD(R0vjp{%r8VeZ;|?_E8}OHf1VNay>~4iM#7z{g>Vv@b1b zLE?coBK+qF`sa6v}syft(5{;PRKe>_G1jyZzY1?^EYouCt-5Z(YM> zyKbmS=^9(Xppnks<@?Wcpaz*}w!KGmoO?9+;K{3K`kBABTd5sR^jv426;k2}R$P|s zpC$ar8+E8Bs7XA2U3I?&C;sdnZ*iwng#Lp)FS2n4aK>WI39+uN zRb=I^y}zDO;2HVPj1s{t3=Sm=#L)l5w~~)sr^oG#skphS(!3GH;|OG=09Uc`61k>_ zmup0_P>HGqg!5RmTlZt!cgC+I&3qCfe%ru!sQLUXE0SzvQ!m1LbU5n#b%K<-hn|yJ zo}|J|xo4vnWmRfxxUiSmwf^iYSgL3YfbM}t79hFha+a-*+Rmj15vcCTjB$vSTZvRB zMI~pGDHo!?HMahQ7%GO8g&FlycEEW9@Y9D~@+r`}op8SZDyP+mDjT#lWaZ+XfgBBZ z;+p`@)XVyR48R?4n~qZ@@s+mdxgOCJAHcx>l>1Uk!b;UDjav%Ya;d<)6k zSne<36ww9&C>3wuAOF`ltm>PY|n&ZpI9b8|id>T$e#Fzcl_VMa^S%xS|x%K{47p{hkf<`i1_x zbxg+}X|7izy$x=oIoc;zZfaYFV%h^qPU<&c;xRnE(r$R>OKf`*tEo>{>DXEm98DZN ze<6Xg4*-32JaveloV=yy+W1u~wbL0E1fuhQpA43)$|snK(4nLw0DXVrq53O}nhCbn z0Ut4=)@Ui}`i&X0*0@XWYN`Hd-CY+Zi6YlTsWBUTdz;fxDU%0y=pZKiecWB?4Y#ML z4F>{BX0}d;a2A`t9;O&!0g%-YH)-Uo(+U0fKmehj=koNc$jMl@*?@F1AfEUm zDwLy^yy(vsLtkAYl0vP=CVIQxG_t`akc~KDXMzr18<*d{x-O7s z)dzvu1cE7W8b3!G-e!6e;_GfjvDz^`IcYvU`3%XRw45P*qF3UY*%wma>XFM3%z}%;Bk^1cT^Uee^j6B(OoXUf|M!Px)f{W-n zdvj+(Yk;xL)c!dvNtFnGcp-)37|L8CmN<|w7>UT!Kn)g}qt}wLXE^gGQtp$k5pS2X zdY-50TqmIYO}327VoJhu0>imry@CrC^xdarCvY(lfsS+gUoPQJu(q_3Po@8dt+x(~ zs_oi`4OBuvatHzG9wY@MlFOMldgaC zjTZ@tCy*T^Oj;*S6s4*}y-lkKpcRfQr+4C#r$AqQ;q((#cuwLmSY`rAVsaWN2D%aA z(zXkqzHp$JJ37=;jTKGwZ0cGV@148;ERq?$KfgEq?+0*kSC#xBJ7Hh=>OI!E-NxIeoVwLN7*c_HBN zaxv=@wln@Q*QhEn5FpfZT>k3ViwoS_Rl$bB*d=!n%j{%7^5V*YUG5j6k$t$S(* zEdLG-#1`;riL) zwfAZO8ToYaQEn+tQr51g(oja)QYA}Y3~OSS#rq@9X>g}25~r}sGMB_7$(dQ1$)zhq zzp|^d(P6&FYd?_DvRRW#@CCyfrNUl?J5x(44*kW5Axs+oAvEUUS{~Qe2+|z1sN8VI z_R$a7aLL`uXN8Q|BjVj%N(*5xvs014gjmAguxSpC1@_5A)g_jOCA@Sh2> z5Ot+W&5Ahk%%@tNEVKttmza=y8bXp&B$*0naU+QG9y-$`!zNo#?@xFHm};SMOn)cB z#(DA;2Q&9TyP>k$GOIbsyA+(ds+ZiRdqOPXI`B0>gxJAr0g^j3s5FH+;o!@@QL+N zdUWOCoIw1Lm3a+Wjd&%i{I$Ff0!^_hZ=nH6>~5ytaWC1JMZP4m6^bV zNa)0tzu%ZI-1S+r1R|pi>cF!Bbg%lb86?VDSRtxJ7D*!&@%^xuO*#W_G4Her3(_=S zXcx0o@^zsLU28VN5^7+5&5z*=gb54cjdm}Z1MS_wr{gah!zQa?O@pp>t~USp6f1yj z!u$N=Cv8^UA2UA7r4=(mrK--vwK<~uM~xg>LUm#Y<^v`z{zwP*Ugqy8BZz(%C* z;P`E=|NE#|ja0IREsXCdefyo4We`bbJ$L8eMhS_}G|{GVJ^M}w_E0!hT4T?fX>$@P ziAx)y7Yt~uxVl2$NQ{ol4Aqz@$`36*#T~=}JFS4%-?J2$ZzdQ;R&t~!#ndK-)A?s06*zmsL5iUP8dJgW>K zPQ5;PRjEl5HD{En)p&x(Tywjo%U*G*e%kdJaqv z)+*0jU4Gq13(3Et%4+cH8ljU0Vus1 zv$8lM<7j@ZmOA%K-)&c90puMUnX~cmsHYr%^4DtAz7fEC0zx}SW1SP~rBPC;cH{o! zGo={%kh^&5iS$j~iNPIF-D{rXu4j3S>^yv!3X9)K#+S-GNnV!wg9k8ZT1r@~IpcqK zx&{zR|6pn$_HtYPY0h<#ZYf$i3{&@q5%Wvd2)~M@DKz~E^HNF#33V@EiHJ&;87gG6 zF(EW`dYli4FqAR5MhlDJrfW2BVQ4J)W#8zas;Hg-^>3blfb60;Tw^}fo73PC8xcTV zzQboHl9XTBo3#VEUb-c$#Vhd&_p$7a3entV*2U-QlqmhKtu!(f0`*C04g--VCLj(* z8gRy#v(@|?7{cO*42CwLsM(oF@y1X;Mz)gtmo_cEex^ zd_==|v;FO{=c2H@#Zp(-q4kVyKnpCLa~x(!;eAWVU2^6AyXv3RM#Ow&SLoiW5U8m= zFT2*Qp$kTC=ZLsvk3;?k%9S1qhwtiL7?6|BC&-r3m(ykk8OJw47G6K8r%3?SEp})Fh-cUNOuM?TSpn~xVxzR9yMH8TV9o6W*hTk*e&t3Z*&Y`%L zF-~8JTJl;{SSGN`DC%GwP=;E#k!rpF2=+_V>E%0 z$^wa-FVkVFvVHJz4nW0!g?dWEqv2^aIW?;H))H(e&aTC5ui#!*a$%GWC_;$*<(C%; z<$9xV8LgqG{ud-n;<5YHM!EqCs882_HVi|<@(&if0@$_aMoBzWc9A2s*_73@I+24e z%~Ger1;bc#ykrd$9Ck^{h!<7l+0Cm?D~Aop(!FcIZjE#Fr>qRNn_A6f*SJU7s$eHr z9ep$pm$DreC(CrgW{dxu#|@wp1FaVig=hq=0Zu5hPFgJ)!*Kc2FBm|~(^sAKImJaigdg}E@ zFk#wfGJbwzk5c%gFXqT(hVMv!6*up+>}{p%r66XHQrY8w+xmD=Z4k)}z+z0Me~=;ETZ z)(2?W4NW>SbM-X1_h7{U41 zDF7n^(w^F5m3DmBv;$~dVCt^=X?PPbkK|)<7NC&wLzMYT)69<~U6J#+DoeYnjj^?X z>8iDe0?je^WOpY^@t=lcbH4zh1!%|0Y>EXsZB>#HPWgaBgT~mOO>!Z3v`1N88!cGm zfW{~lE``jP^=O)aH5{m0z934MH+lQG&Wl?oSzL@D70oe33_j_obF{(_a!Nu0O4h0- z*Xcot!&g_$hAmc*gDD^;i=!ja=`(k#Nc`28> z;|n;~@^`MdMtr3EX~vepz-vH@zF=?Qrb0Ym5M2SVaKXh8-my z+2UpClE%2bCoI?NY?BokgrhQ3#iD63f3ZRsn+SB^KlT<;*!HM3b)2U9q*mn(=(KnC zC*<@9-EPPOR=O!=IX$BOHvP%kOvjcvbLT~`kS=(3Ojd6Kbi43ao$d)F?fG8a}H_!4PpeBIAo;#KFIQY-b zJIz}_#wp#vOUW=dM$;u@Xx0R?FE|^t;HtU3dAa_9Yh_=gen|4;LNWIv^Ugw*95!*d zxq)Xy>koXU8!;Su{XWw}i@BR^jd~@UDxbwuZs24N00)`IPQD2m8VyLdjP~X-pNpkn z>OMoPeV538E+-GaGT>0TE$()=21VxGj_9>r9GE|e@UIlo?Gwo!XTb$1?OkgCfn;;} z3$WzPhJ_XP6OFl^+g&6BlWqMMI7_~e5ovTig`Cho8mZ*P$_M5){Ow+~LOh|6;|2jb z7-(z&9+1AQGEK1}?^s>R0H0D{-`r(D)@%}wJZ|=fh7B5%BV@4RWJSG|;c ze*-b*iS(y<-T(%0g5+l7Y2j}`Dq+bI#p82Hmhi)skA12>L}?EJjJWmf)vv(q*W*lW z(y9B5BXJgGCj|`=hf`-ttH{{;IrXf4O+wq4&JCKLDEH9f%`5R*GlGhfaXSgcE9d6j zckE?>y8SsfXMRZ!Q2ZmuIxwZ&d@dUU_qW@-oN2G<^?p?W3E@xM*?6|;O^>iYr8;sz zeR{?t>)2uE!H0xiEQ*hq|7!iiZ=9p3h#m70@A>nYXXpAiD@86@)L#*suUb0uy`RcC zjvbOLiTTf1oxgxIAllQ(zNJ%=mohS&v9>Spwo+)6&wiLSABPChele2r<;@0PZK=Df z?eqbjE~@IiA^}8i%}CVr2(|oD*W;6D<5->W*pJFz&rBP>{fQKoTLO7YcOhFff77tP zKasl3zgDjFN+90SlP^wnL%N4eR!TZ1RDT2m?de0^eHI(H**7W+MVgL+v4m3kmU!d9 zl#N%uvrQBJ5r2x`Bj;b*9rT7VJFFda$3+3XuyW~jvV_ip-y851Ml+mPtL!%Xc46E4 zM^g1Q`NLj{c{g(|!wC;(0Li%T@7G#O%|v$At1n1+sBlT&gn*7>!N`6lA;$Qolr^RB z1fF-7*t?5^1TBnGxqk(d3U-=)VZLO zmbEOy$s}8*0v@iDrS}eT1dSA}z(()2aUDi!eT>@()F4iW0p+H#q@^w#h_8h?F#USF z#43+rKc`;i9ldJPt61p9{H}2oq|@tsYm>ubv4|8<#Amg-_|;~0X$>^BP&1daxB@zl zE*#Ova8Oqi-@BZzIQ3cBxXo};(LC3Ja?vuIQ8N3d{|k)ga%d&L3(fuRup#0byZkKg zhI2YC!DG3C%s_Dc(nHJeh5J_Hx;{T|&JL;RaUfaW>ep&Q05aVv%!AHU?-u&?`Dt zU)Yq;C^Ozs^SVj-i9!=$N~&2h*EAn=UDPvgLv7UBUbO0F|LcvF9(=7L9%ut!)(qqS zZdFw~lzpDbZix!~UTJko%$v4@4j&#T{i5%6smNu@s0Mxem_BJRy{<4;rT_J=2D>B9 zV^`G;Wx#?iGRs{urvgGLqYau^PP3x|3xF&+U{ZSDIy;>Z^ zWl4E5dCZ3J5WxT6RF1Y=;Jx3ENv$u$Vy0AUqXms>Kw3-`uc1>h&m`m{x6=dx5#pfL zs1e~aDGK*FKu=)o``49pV%)-ateNIYK-o8^$8gm0%V|1lxy;8_w=RG&#h5P@gHOA! z9W13_FL%t0_DM2xS*wyXPVIYD?W&&+7i7bfo!PwV-J=pp&V9{rhs_rJmt#m3y&JrS z*6<2GmKr|%#liaS5qO>%1*PK)#*uz9jK|6FHG=UAF*!o$TI@a9!5I36KrL0{)D6UH zd=ZbRwbFG9MffKUm{d&O7uL@JdQ3{{@ItwS2mh1{~a; zZiY`!tRAMbKvgdZ#Ma{BDe5zfeYde!pBC5bKRy%QvsWKOnx5aVvwYd_4NbT4mR_3! zQ>i4vQ*zA>ynPK)Gw){xM!BffV(I}?Vh)aFxIkHIGPX1Uf9OZ{YgNXgB~Q8u`|GcS z$6Ps1(&e=yKFa5=)bpv@+~|_=e0+21*IQUxE%lmoPBUv!dbn4_Fu{8g_qv=x+skWAWW5!KtVTC?DDx@oZK z&DCbYg@J(4E6m_Da+fo2^=zkr=npb8RmO=s11>(TmQ&u zTA|mn*qr$58M1IHrM2wd+Yl%xY)_&hDEOUK>0-btH2$q;!3i5P4Fs))25+4vN~TYU z20|A}iWLJE%I)P8^=O#b7e7YTod#3c^U`M#BI!vsm3OsH8;(BFK0j@GP=EGW3>g3q zyIHp0_P_a^DlJT+ukiv>ZNRT9fjGLg3>JI^J4- z_+A4yq14?uK;84%tXdoDJ6?EZ1Za4xir8v^NV3Z6>|n<3_j7Wz-O4;MWQXf<#^){9 z$so6-Io#=j!{)w3-!k%ihHUva?~*q5xg5qdoUS(K=Qy2E&#Q`K02Q|GJ*Ch$T^u4F zd+x!1*bV%>%5oW0pJw$2rFuTS{l?_9fDoP;z4W*_UoAguf7?FWr!jGu1crM-Mp@)G zJb|e9l>+rz$~4Eg)b(TRJ)4D;;6BB-TgEc*u^B!hR!HX!*INiJQu2VrQv4%Lik1fv z#}D@?!+Hj!{7CR}f2CEiC4g&SCgCRACW;tJWD zwil#Bf=ugQwY_G2TmlXEkmG-D{Lo?c93RKvbtLTxXl^0N^N^yIY)Eaf4qEJ$i2zS-ZnktNFxQ*WXA}Hx`#W=Uc&|do8IU}Qd$D=e; z^wn_vK-nm2x*{tWiiZZ1$bw1Ga2{UC?U|#8EaMp)B0B_;1n|Y{mzGn{={a3%hk(X! z2VVNxU?@OzwtfHdrb>fHbf?ap*ok~ptC5{AX+F|gw#9w@6a4^H^>CAG)tL2$`1x!< zD=JCX)m7Cp8%sH-w~z=so<}!%Q>2Uo7{(J8YuDs!&3R*=>Bt5d?UAh9j_M>>lVWZd zIUhcfKy)Glv{*ldU`upyh^ntX0dCnJCuSvJ_FQ|xJlK4$B*Snia6@PW<*0}tQOfFl z5a?0%GW|<_2vGRD78@2(7w>zxGco#-QRJ1wAD;HvUn+|oyscEF)fOdzplyEVdqTdp zbd8;PS3LCE3aZ5G_&%)Uc=56qZpPm6$K+hM2u%&dGl+-P58JEKn5g|r#VBy}j~s^b zz_qj!7BdM1Nwqa_g;62EfuxCwbDMt=XqAZHo!R~%yEnkXW@_!#!o(i~w?luvNgE$x zt!O_(^3_OrhbAi=(B?~*zJ(@YbN{3<2}JJ0=ldf zkLew(8db3BIZ|1;a^-chusDbs4;LLq?ard3u|2VI1rGXx-?d6|Kros5qlb-i6vaN+ zKJer=LcvL4dx@5lZhz1>#tRDv2KB2*(kIjva5-4Mf*U^Jaex}3y@JR3=VZp~8Ey7; zYeBN`vLw3p*`D^vw-=3rk3AGzW$)QWB@R|Dan)-wZAi zwOh(46xJ`wRQ%cCFzB4Mkh@B_PQ(0up6&#nje*7L|FfbpbVRR-nIM((;G4K|2Z5pj zi$-33k4@~WA4Zhblh4&3m`$}5uOVxK1Ya3lNa*EkZmWyEwA-}OD1N8d{Z#-E$>@es zM2l+-!YAfEA&>=+$v1i-*_#9psB{o}T*FH3t8>loR>_-YNz5Cz$GCNr+jq@Z`$L-U9>VVY-PYVXNf^C0gt)xvSR zpNxWp`7Vc&2$`M)RkX;>;$OfeHfYDMGM5D$2|96z3@Ai`CQ4&VI}a*gFtlVkT4bOx zNa)TO3eqnsj*S^V2Iost9px752lonjGL0)^zrH}Ns#;m#r>+tO=zCrBY82{2wre&= z@ucTCSo1%zi@cGA7YtfX*j9+B0nUZu#$WJNaLR}Z;-!%9&Vg`Z)!~v9F*?Y`LRBYA zekO7{fA#mdmg_NX3VHUQLtPG{9L391hrap8Zv}8;!zcxB7@chQ*6<{=QBIx&p~HxDG{r(7mi+hEFqAa?LJka8w*qf(RE$w7csFy84$ohmPajj__) zh>7g3oPPbcOAH*F&8JJb*BOD74w0Z(aDyK{mHAmCd0W@}N~@5sN3L}9JyJc%LJ9`_ z(uzI3t7i*wtGoGH%}qCM2~E>apA4i7;j-fcXHZ;Q>N&M;USnwTM99T-&Lhbz*b8mC zzYax0t=u>r8Bq80u%`XYsN0wQtU%eN;cA0bu6ECOCW0zs_3QmfGsmY|_zxXIN|x#% z;I8u)F!-J$U#SFq*FA0Jeerkb`5sJJ(~fBKgVrSDqa~}P=4f#B5pNU4Bq;gP6V^5W z8Vi*QM4Fyln6Ay%zb3YQ{}4V}1~^~=0W>X|#(kZM(z5MY!z$e<`Dg~c9vVpJ;F+dk z?4@EVrPUbqpTv3Lm#%&2Y*Pqg+kI*)FleFDf*#`Mh+pp6^%$Dz5o-g<{Q~$T!`4If zvh7Kcqvv~&{A~I5V~vl(2rA*GU4IPjoS&k+h_67+A&e&2tq`6r-7G@#%7zQjVZ{5M zTJVle&}r=P!yxR=ncXVYLYl@~Rga_VbkQ7+1T6SVQTq3_xmYl`C6-yCp{&B7x0fQ< z;wmTlpq!YHa-Ih_IrC}BQrfZXQHJAgwHGe7zB`6u0&hoGDztsiA=7YoHJ$1m9a2)Q zvqAbJy{oKgh=A1ptbhEfch8O%C zvLVFudYD--DpU;Fqta#cGbB*3T7(Y6$RiO!tgL>Sh-LSVIxufc9k9i#6qL-I2LBDTXqF=ifyzKEq{aJh_ z-uIY`d-^L(@L%#H3?rH*`R5R|9RsaZPa|;@$?r!K&qUGaC27~4anyFL#X}_!GL+Op zj@R}!>PE5uV+xG^M2*4dBI7Y%K)}x$jPR5*dL3&fxg470Dgh4(rh^3cqA6#10L{rp zY0<1tu|R-Bpu@r`dGY7hhcT%n!=!XI?muen zc!OYSX%*Jft|p{~xVl-yvQi6#pl4ajvqD@=X?KQO{W`uuhw0=oX%5x#T9{B7vY@~K z*}i@#{Re19KqPY%Ady~M1Jg|pE-}{5TdVt?9UJ<`#(2H8>;p>HS$!9x)o9?lhDWtz zD03IMeAR7#?B`K0)5rV2LuaAXR2fjWah9J9u%%(KVNH1i;0_s(K3KQLq|$cELLDi| z_+CL46!hiABV1kMHSOa=sm5<#YDoH{c_dngS5x-#vO|G$JF3QPl$ zKm{CLj#~fSkC9d_lgExZ9)%Cc3^8La5 zXpCJ~Q2X5c7w_bHaC0E*V?E7D))M+P`?Tt#`#2a8m|YSSN|N?+pWg9rI;hstZQ5s_ zRDCAw#NJ`vu@9^2y6LS)B`g=5%#Ar<;|lL!DpTZ#N#ixd@hA(=3nCi{MLJDf+a}PP zpHhV@VEYI+h2EpkA@kMomPGI2>olO1rsZc$dqyqb8-U)Af&4XwK0;80J`vURvSF$Am#AAAPhm;5z?VZW+0rk49#RLXU-C4Mi`AgeQIe`F%Yf9{o?6n#61Fvi+LC9MGBsG@ zLXAAvQ2l2IlW4ewsMQjQPXIeX8=A`NT*I#R#~XD+WVg+&mRm=l$Ym(K{n%l4ms!yqVHfzCNmfr;lK{C+7=YBoIS$!uSG zHD}E;j8}&|ww81o-CA^;wf%azeC03mL&-<^tg?s}b7&x4kUzv#Q32g1J9F<$hIj+F z^ZpG$l7O6Io#yVWXVo^m`N+u9J4Ayc4+r}Cq$`Vr)YsMSRXGIAyc)65vz!rVtKKo( zPHyUlG0*-Xt#%2t(U;v$ahNfuDy{Osofd7Y-hE4MQE<&#f(_-hc&J$APx*K|tx1?| zlvt!lI4qE=%59!GDvryTtZ923d(+t_`vk1Nz#w7)f?1Pe)xZjDOu}Y5ji#joRLS$J zVn#(oijqfbzE2nUWLt5_PnlV5GgQHGtzRA6n4_FWYPZ>Dy<^j{O+ziEfjA3DDYF)V z+~}XvLD!M}uskB~X(6$W_ua*09;K9`eaU#=Ly79`LX+2Uy%xt)Dqj2&&KC4EKIMO& z!UrNUH$k^?WB}$+g@#FrVQ3V1T<@cl$8hGl*l2VX%)<%|n9+qwCWRQ4>*W2S^UQPA zP*K&Heuep&$DLlKkj!8M`sdFc%pby(meGHN% zu~D@w*n2Df0oN(xsgm$X=aXki1b!%`fR(l^n6wee^UNz0Ap!c}GJ1?;4(_(cu)qS( z2r>Xm<@D@XZP%l=$m+VXWh2hd1CLDAQ8I+gY*?6b6nZ4n=kjPW=&@3W zvb4IbrLg`dE=%p-v509Q$hXZ}^~MseEOK z@@Q*H9Rn0t)1{n(KKSJ&nu%pw*k3CuLl+%nf=OKnlBJ{_R6!)c_FF><>i2HUDfZk- zOD1i7$i_KSE{u+Zk{7&>d8U2#G3KvpDTueGg0VCd$UqsfG?0c0qnCC|ayPz`Gm$e( zx|LKyYHnODivR#A^>n~PAkFc#WbMq9aZAuegD5mmGRNoL^N$Z_@A8G(=gaAsisI|| z3GlG(T%7baP-^-q?zKJdq&!9a@{*sOyHnS1PTJp;%pGS$lJ?bz)?w-d9+65wZVWrJtghbJ*=lN#r5U%_Wux_P} zJx)ul15W9&f z3ki+T?yGAC8ZP|(IoyY%*7IIl&wkuCup_e%Oa(b*W{NZnQPIkMDjuc7_dBZk~Dkc z?6=X`4Zi3>u?f>KI5sQW-R(~YGAgW7^rc(0He)a%btkq+vNhJSuM0x&j}4&(IJ zA8Bc$s!7n0rg-0!>&v{QLg$ZSY}@ZMpt5y|rOToIrLRRU53r8imdNfbs(JVN9P~kpV}|@xg6q1 zL3=&CJf<~zSnb+K)DXQ!$?`%6@nx9-wf6lm4`9BmCfx9HlQ{}gIPhNOGu4X+j9=@u z&-Zf)pc=uTCwkf28T&u7&GX|{3mUH8UmFk&)EHe$Ye}%LZ-PfmF zw6$Ih2TUzdP{`@0v37*)(J%WJ?T>ipgjrZ1e^ni3?P^0_AUiG9-*g-l&ZU$@X!Ll> z713GcHTDq2f^X@F^M&O@@7%~EaTsy2j_nUP3@1g~Wjy2D%FitdOkbIJQC(qQpKty8 z&h13YMN>;4s+psN0IGobH>oZe{l3D57HpYmm(Zo?wDhA-eOa*ruR0IcI~2tz`M1T zV$=S6iGi01BLjOv$YHu|Eg>pE^7VvqUlB8j>_q8w)ApB@?uAtdmwCj>UanXamI9^Koe-Kv zsVXG7@Z3{zt6U^!cN;$`=5s#QYM!IbbaUx4CN9TfosRc=9#AuTN>G(YO$#Hv8c`%7 zgNu!W^n`j|yWX6fp)V=4+cC#@wOc%{-YUc$91F%cbwrVorbSd)jHyxV^8WVd({Fml z9oyG6o_YDqiMH`8bbUOR@qxNg87yV9Cctrjvmt&G`T-=yy{M^b@gvw16G^|i>oMtV zQR*IE+A#f;8`LijKnjIkfdAxb#%}`xW?;7hw-aPQuxMZmvP0QEwEeU1!=bJZV34tE z3f$M<`=`JJD|)vq;kue$eZJjmteQkbwff&XBSV)RLMjUn=L>S+?51kVRhIb9qB)c$ z$^Eeo`W$5Bg-MjD=c+61YIJZ==eoWkwc%e*-M?TF;5IY>+c=dP!H(7|7Qh;+!K$|>++h**+QQ9^j7PVfAnmCjkEa3V;N0~Y#j)jlTcEw{$Mp@f zr42r?SbRD^X3(~Gn&~jYV3n9He~>!iFg{fR1caVrjjW41CZK3AmFFlW1mPlnypces zHgsCOdg}EXPNE~x@J-_l!$LOA&grI1qYs58P;+uAP784$06+t{9Kz(*2~;0JDFFz` z{wCXkP0m7jtZxe3y=WjT5d9a`&+G~sdjRC;%EzbfFOr=KfJ_Phy69?G=|f;>KDr<(Paizjn!}Tm{cEd5ab3xJ()#P9zrRPA zwEtvAQYGd!Ce(k#1K6f8Qmj9Vm)_S$tt<7rO>ON4J}uu!NA^Z)IpcV2FrSNgY3!=} z>%oDDoQsoS9SKY^FA}N8-SsLcMvlatM4ssfI8|z;8-0U^&0BdqI40x!iN!JA*<)P) z0Zu4^R&cKOap`KveEG%k^Hk_H<~X&j)edno)UdPfh6tydx{UGcIS4KUhbbW%rcNlf zYDAyZ0M;>EjZ-GlzcW&8RU~t@;rTjt?zd;!PxzOrUKvP#UoH56s^+yH4D*vVgSI4r zIpbd5*7&V@fI$AUzZ1!Ce<1^4-MuZ87UPSW^xyd?reFXCH)-RgfkDLEV*7<7*)Bqt zW3(03YmVo{NYGndqCQb$d&sxd=84IG9j1O*_fhbNvsiYygQ59HZ7|oAil1AqKvqI? z0KogT)K3PHELGGLg2yh=^17G&@AH*1OWg`#Bj z$s3Vq6iwqIwcMI19yqk-ieR3~;PQ~;j?eqFqr7w8XZl3x=Hd_shIS%z@mY=ix|_cH zV~5KtW~Cnnt19f}a3|#9X6%VeaTA()ici@zK9kA8=f$72#xqW_GYKWDx0M;)zM~J) z)=%1Z`SRY3E~hxDv}AWrlTj~Xuay_9mw9df-9F3csJ4(DAow0|jFyDktm4rAc1P06 zgX~xs;_uT2=3V+ce~q91O6{fDv_#b{d8r(fmEt-O_uVCTEc=_YV~r!yQG;#V#FtV; z>4;V?9u89LQ=18Wd0%KqZAy-N{$BY72k%dMDiHm9XaK!DJp284!J#0KC(V|k{rFAje;MUt7hW++~v!9? z#7%pIT&b{4b`a@%=&Yel%m{JECcPJw^`a5#pn%hoqkgFh3fOa4C#_iD*TBgN7jDeH}(BQ`iig zH#0(&z%QHvOl9Sz;O`J59Hjk?x?m51Ov5Fqf}jD?OL>`wMD^NWww%w)b$VPkh~i%w zvv2<_km)?iy7fMMyUzqr+k1Njl7$#Iu)ucYU-yKujx78zWub^T3X=9@dcd_b$|4X%o>J0!rDOiVj_bk61zLSy;;7dRI4I z0kFGnLnuRkB2fl2#nPqdS<%B2{nt;V{ZqhmkB>AZEp@VcV=ID2H$UFE%PS(5+ZQYB zn$#u^JvmAn#`V1Bw4SCLmsWXG>{k0wy)KHR`R%@FEDLJx)7JKw2KrhQH3z@Ovl8b| zjG}s#>?2p3XFJ-MS?;l~@qSWH4ItO@^;q5enq{3n|Hyb81_w8=K}+v1TrFX2GvfDmwW6Q`$~HOaP|~BN53USfLjDVH&=dS8gL#9nSH^ayUlaA_ zACUBDWtazX*A%G;3)aFruja_@qaxO`ddYViGfQqwJB0LPv0GNqd;%b1pQViMl{|CS zAoRXd%;@=og`?t#NdtBSb3BGlf10kS%Yd~$-O|XwB)oDUR zqxn+x_4DJ#&_hm*51ki_g-ybhxTB&cZP~83G?HtQEc_hED#U{$Ss73)TU$<;`oy6M z&1LO9{$Zz1Nk$H$Vpe_~1lweT?%W|i()8wnXhR^*9ecw#wO^Lk0JvV%=iQU;ADhW^ zv+K>K#*d*Q>*+}P4Ku+`B86`=Cz&UE^4Y7Ie&syK>?KfY{t|4wRN|>}+9g^xR@)`c z{geVmEJr27Ajb%@@kjhmE@@DKhA7~&sC}F!cpp+=ReBJZ-76X*X*C%ApHpCHclZWz zwBwH_8t$i+@h^IZFB^Zr+LDSz7cCtYF9S-Eaaro1<)_-r4pmKg;}X-4$73<7;Mz{2#h&KzcKMPfrP1MeeYo2yf8H1JMwoX6QqH#Us|dy{h3{$9#@w}zUS-T@-B zC_QOo-8is>K& zeg_c$vtsZ-MT5@AiEZwH56=UTj=<}j;sN!Vw_YC`Js*IUt{nv1A5k=qGmO~GBk2F^ z>J=I^TErY1dcIzJt)Rz40MASFR6}+!(-TH$Xn2dN)#n<2% zm#D=}_JSq`-utvMrGU!lNOGbS^_?(1bPyXqbn9LNPe%`jCpLr$6vqDKWqIb^ z9mQ|rNVe-e^e&nE8@Pezs1%rQv(%r^9BK?N#!I4%l-F0X4}YrCD<+Duu2$>H#`Hp- z1xV^;LUCb%>h1DYrJn$7^{(`&BO|-_ZK662M4>WXx1RZ5$V4VZDWx15Uj-pAX;m}5 z4iFCjtZ_j; z1Tf0OQR<9&OOT5bws(jlN;yP2;ga zjbP(rshUAn@+q|X{1gsAGe&iI(kH8Wi~G>FzPb$S(9c}gBp(R0x>zjULf7%$_T+=-t`~ur^Lr(u1t9H@UKY{U(_93dQkWxX~u|& zU%Nr85FYaN12yqt_}Gqd5D*VuL1};zj1UIM+3gMHKQ#@=4jIH2 zzxSRmG`%ao!Yoe|Z2ua#`Ttlo0GUb_5?}wsH?5A%ym2`aNg`zW^aKISqV6+`>A8dCz9-`P(DoBT`QErPbR&h}Phm z2SP91&!GcFab~+58P{l;{q74=pzXmZwEzjsxkPKCZ=qU>=sRL@#5c4%v!t5j=#VKa zp7#+*Y*vE<^aK&O>VQgNcjRMhd$n-d`WnDO{gp+;7SbPb8s0#+>O2cnTDlKbW>z&@FQa$W>irl6#+?SLsQ#&|IDi6xGAx2y!^BE* z&EIddjATV@L=XYQI@1$G8A6f0*=RmV*p6-8_oS0BKB-Ese$RR{*>Ne#^af+b&nSiy zEaudaTQa0ULYR7y!985NHj1eX=5ZU#uy(j#OSpqkk8Y<4bvLnM1(AZ1b}UhQJ;8p) zY{JTwqI&0Xs&`Bao0^4G3%9_79Smy8P=#2qP&3w2)H6&2#JDJgl?^)^D>GD@h0y=U zqNanTwnhO>6sGVkk9$h3%2~DqEsf|S)Or5J12rv`jZhg%v-ZoT5Y(Q0gxE8$aCCPZ zFr8Dhq~Qw)v?H@bhDS#tHsdp!-Fil-B+C^vz_9$0@!NC@JvNad6~c9+cb0GV&=95h z!XYa_;e!mw|Kw|S?}hNp5i(nAP(QBG)i;4o|L%voq$lgCr37^7FJmbn1NVDS$-kZ< zAx}t@8Wtj^L6odvqMy|o4O42P(O`ts-diym;j6a5v(onbl`lQ>P`&3GD~a_$aZMPz5jxcZU2ou+Ua8R`()cF1{B8^H4)b zG~fKOcto4Y%Hoy?lY+d}$@|Llj7@0kS6LjJEP@0SDVUNEWoE*ZXCr|z6H~oo3WahO z0w^eL!TLu#Hpbq?vo9bDL|NI)QceLDvXXh} z&|#u9dRdTqN3J0%KUCRk*iz5+G3fo>o6EECVG+ekV)ie>6vXf7`wZ9hgS0X>7RXbW zoR^{Oqbvtioys^iCMJ0nhVU%;5XoikXmB>m|9V9QhCs_id|1TY5^bBYD z{~TWcd2#=r?~Y#xZkf($zkBtde`8pdd?@OKBXGKz-|BQRzh7Pj&6Lp?Il3@Z;X!q@ zek_b|*$xn9jHISx44-bGlYC=FQ{}&Z;jEekQ~ei#?X=1LhlQ4rlY0~mh8yLVWi9S~ z61%sB{+@-YAA)erlKT0YmP~|{6La?{j|QmRk=r{qL0h43dEL*Ig@yIqo`n!73Q^Ud z>s-h%+I5a)L5H!^loLT8p}^mbKZnL)R^%f9R?Cxx=U1(eBP=&cC14Lre4qF{$F)ov{+U zO7xN6xD{71bqa<|YSv!UCq!Le%C;{bXb%hWpKxuP(}NHM(MYd{k=f zOvU`!XF+f9&0aP#Fg%qzhb<`p=nriXrXJnOmOl#TIb)Qj{*sbeYs9tljv-n3r-1qE zB&V_f50f+obz3uH78*@zSH>q3SL?TaOQzbdA->jmRQT)KG?3dgS2i7lz3;Ef^!5D$ zezp>O02#Mz?WMLGJPol+CKjs6sRRVc=` zSJ^67`lRoa$5`@GLPzCYe$IJ~cOU3;&+_gZt! zId|J>MGnn@BdGUN*{t9(tk4#&9LJwSxT4XNL5cRQeoe=^hNXJat*2Ugt-3e`};5T+}ULX5N3wPP}9n) z^Vv$F?kV2C~WwOsLiaBd)4BW(5Y%QZcf|U$3lv4Ng*@SBpWrM}Q8&(vknDM#`^DiY#IKcSpLp(NzFL;LG-+vH zK9rqOg`XD!G*V$k#=caj0gE>O`caySE4|?lW~k6uPQ~7#RgKpX*57TL*&6ieVNbU& zC);ww2bF_;XC3T&~My63m`=p@l%wlmy8i~{j&Pvea!rL#=)eaXG%YEk?W6_0^ znLZt@_KT?*sCZySK823%E_^)|_e3B;UKff)rnlbpQ=JGDOU8Bytpj{7GuRC^WHy&R z;ys92!O;($kKXZ{JI)SFv8Pj^K=ompJF0sB8#0bLi5E>6nwX1fo1=PlcM3?*6z!+T z?5|@ZDDr?78ll7{A?UlH?tRaGxPdyqOeawI@h|=@Z;0R@CFASsg=4(soS&cHh)+6q zTs$4^3Bza$c)YG!{p-jL-ZNeC7^F4kfY*)9)I|Kp&vnKoF4S{*09yY@n#|ULet;AR z3M#KU9S0UF#CALjBs)X}%d|I_(kN^RN9p5wFd#wOXLeovI-6~03T;qsdB}bmwV}zz^t~wPw>)~K=hCw9ffB?a(NAg-x9vIC{Y$+sj)z;w z4^`phafC6|r*&?5?%Xar$=^k3FYF8|M}vYQJ{NGO610?+wR(WV%S)bx&UAuitVrw= zJxVd6qnU(34;50+hJPdpcy?e>$(s3$&_hE@tUH0j9dCb9cS)7>N(^Om+bQ!o2XaSB z7!4Ey2cdakRz;ko+=R?fw%$wQoubx4-CE}2*X7Z9sCA2SYB+3tSIcT-khm}np3$4Q z)-(-lRF4Af2AfuaUdatm!F?X1eUm|^mV3TpJjN?bW|tCMQP3G1p>kIGl+3X)aJ^V0 z1SN{3ldTuq;05RYK(rq>BqlGGjlwFHSk$xFgY|^8EikI@tdpJ5Krwe)cD-dZ2c8Cb zkfGo)KQ41ZBx16;L_q#Nb5=rxhWyo%Y^K5KcKGtnXxt@*Fxy>3l z`PV+=IgGcZ>FcfU_K2PVYUf(lGiGAj>9&G0GZ34X6)JDwqk22C3PENBw=FFHnjR3T zCbxX4F8bftsN#JlmFt+R^CfvT8y@W}N7nDTRga2xBD3o_iY#X=v%Safn1&KGmB)9; z4~t4uVP|~!z>ud`j);uw#$n_1y8l=O+G=+E#mYAOgo~|+UCa@jq9Kkrl-~1s_Fia5 za=l9}-m|8qBWtmHr2$Y*r)dc0n+#X?%hycxa}!YC(Qu3e~E z?3FX8boyhRuRC<~M>dt#l|2o_m(3N&ZJzLZ5PMY_FDmuqZMN@Q%@t$i^*eJ zT>85iE1)k*BBSHM{!yf%(az@k%un4GG*Gx1>1}SNS;pI&f%$`}OtE!WD@Jq+V{0~H%_%yC?;{J49LBpH zx7?>)&cEft;IG58oz#X~?d@Hlh;}c)Lq3YIhpMXNugwA zbh&gyB0S_+USfIfWa8q}B5zPE6m)GN|`w{e{Zgf)Hh^RKkra*$6wkk1sG zvD#H8txfg6Fl#Grf9Sa7Nkf>PgX9H^q%Ba%&^NCwU+FE!pn*;FUm|5{4na8BvsUG_ zMIbIL;;`A_(3aS;(594V4(6LjK@n{gxk#fGg^{hFgLL|djF$$A(BxveYF}d0E1tUm z4ujRcVx$WG+F@hVm1`j+64paIbh4C$<5!0)Uao03_den3LJ~s;TKO%0|Ij7JheA)H ziB82Y877FjT%e>jdAu=0dX&hnIs?)GZvosiaFB80ZW>ROg~8mzM}uZsiKlu$f3?o| zrR?%l)e4U8T7T|~aYo=7XjTw`z;VHzA-#oTZ~33d2D4nd=CYdtphMDF(mL)ii?C6% z!Ahu{+P-P&w$g5pvo62SCw^UTv^~M=5Sr1|z%yPRo8Mxe4PD$&XSrw`2f7}QX|Z=Y zzb;wo@o=dG?ee$$m0FHJ7jy4ZiaZ$_O zM|fDL#Tg^mtD*FF^+=C#9%j97`u7oj@DQJW zo5i>%It-hg{SwW&(uk~Up z77E(JdppCM>MnzkJx73l11fd#U=OcJY3SJB=K0W{@?&crOi!uCZW_XJh=!Jerr9%% za-^(B=dOKk@t-~aqJE~G4SV9CpDX_&kI=DcQB}=LyJ-cd{U4hL^CR2rt|VOh{4L9& z8^zI}HlX<*R1H*9u=UvbP#GxH#9+_@45uga|KXFGYA$M*yr91AliZ3am@VAk@!fQK z!HJ`;$*Mt_O$u~ztVA&&jj5X^Q;%P88VY%#rzrv=QVzXm{+Y!w5axY*4M_>Y90G!* zW@S;$p$pqvk(lwZyzeE0i+Du2hOOWpaKg#XmIH$;>~T-MMjL!E!V%zA$9foFAT|&M~A|*6i|9{vS6GO zV?pa~zu&o0TK_67q8meRK9XtlZOwEW(7<0eIC7D$CLC1Fkpf=V6{t^q=cP_XcOh|X zFKP_+;g->(dHwUm+KO5rB)<^$@#LRTyVo335n86jVLXXtBV>XKsbxDdJ7Iw$fGgrc zPcFjFA9~eof@X1Gob9ba{Ed(tWDV0Q`N{7z{&8!VmCR92v!vD+O3j0^)?Z=2PEKux zH4^N_1bOlm&y#?@>laPVrcXF;s?b{Hf)<<9uWAg)K$kpRd@ znPw@o3L*~TMto8#D`NI1`afN4C(_c1=4o$cr;X8HdyvrBTRYc3wX>2~_%VzxC7GV( z;Qn?~ET#+CppBWJ`lih$Rv3DCLsuws{nmn~)I9LLh{BB#uV5IX|3@KR zO`JJIt{(~LNAYYFOq>k_o=E*dh8}+?*t~pGI1hx({CfIzGbZ6FTj3RdG-BDzY{gTI zr&SRsQFmmgZ$tGRvHknN@KB*js|Bwh8F{^@iEn0>!$`P>b37Z-vR7rtB@PLXhphJp zg9Z!n1Lr0>RFi{UtgWUb1G_--I!1Uh;VBrbsVC~=&d;rG!f)8UR%xIjBf0(lC(q;( z*!*-r0rIFTZqSC)?1n`F%AseH75yUIaC0_G-a;cwJAJf=o8e)t^@Lih)9I%d;o1~z z8Y)6ge#A08XQ-65T4#2`HPY-*jY^rxDoePt(35gyuLc89{t(MgVm;v%`&_PG&(?O8 zCF(;g^6)|MPzo3ixH6;YuHkbPU2|mso1?uzaK#9Xp9kS_eYt1*Q6SHVM+IXSl zZ5kg?(&}N2ILlF8=}8B2{cBqR$6qs~m+&Al&sBk5yN!4UQ;?YuHF8WOvRdZF=K8&> z1e|Mr_$GkfTP@+gc(-2I{PLif`4aQ>8SQipOxBxg6v)LLADv0|^VL8C<&0-*3=#dm zvQU7w?Z=G-mk_aPHR-N?F%0v37*lR2&@OZ5?`m&A+KEU|2ARkx^AE%3kP1o9# ztQ%CDj0LRbq8og9`jlR3&gPs#&Uks(R;Jyz3k=|n?`%iV2HCg{0mjWmP3jZ3!7T?3 z7mvSjRqM};u~a2@WD%NI?MRqo&gmf2cOpHcCZRfqlx^!EZ==s2V0$ELzOK}t?7(}A*r>8(27)P z$}MwLlcvwykhC>cD6v6@)TH^yo|d09MF{V97#qNN~uU?MFKJwom4EIRmnv z77vMl*(ZV1lQUuXCkg|ZrcZ~p?3zvc%rY)plU=Iy{l$eWUto>}>lLTYV*PvzDnm1k zXY~%sQSU($gnDyHw$FMRP|0)0rGBCd@ zeC2Cx0!+M3PPiRd&I(OQ5v>-{*&ZibZtJbCaB|XO2?a99d!GYb%NZp-xyzZh)(O)l zA$Pkhq_}XojARZs2^1!wlZUiVU!VX`PV3C~z{TR>n2&~L6*JzTPSfo5)nju5#o!71 zR~4zsJ@8Lot&S<^VhGdwgPMcoC@9ip_~I*CZtZEHEYeMM(#F2uruqok2Hp}q`xO26 zehl9F@kO%aS(?YZbccZXlTPPR%-Yy2lq$&(fDr%hAqR*2>Ia=yIpY{o+~WHO?qA?;ykpO1^+&en8XCKfLNQHPTP4_IbEv*C+|*@=dZ7Q{##(#CF~er#*p|p8y%LZ@+X`G#J#p}L-;CGz<8|>l_y;nit6VDjH-tO7 zqUFZ>$zockFg$3Q*P#z5zxvc9EhgC1nU*Xdm3hN-6UF5D-5MolPm_2mN$LAiwNwL4a zNzb?c^?R*XLv23iaV8MFKC*}1QCEuLy-TuX2?g|ID^9<$>4%-ZmNb_g4ML7k{6IRtx1V)$A-eYqc}$C?8@MBJdY<6HF7vAe zCYuIl93oGpUn+?;$|t>#n2@UO5c> zBJoDWQt8_oZV!uUZrxh#EFmHv%ni8`q?r4Sq~>M zHZxpIy(foWDcAF=T=0BVEw0XbL@0fXej{?i^oj0q=%y^o=yUo!G=yF*@k|(h)A)3> zhnMLS#e+)T#cij})lOphB=4)!tk)%)JzO@wSX{^pKu~qr-#~3$}{T>riL$6MxT7$91Z^8p+ z5@7J`F(%$eC69Mr&Y`3}?=&_LKLj>s!)^lvfo;1zxnu*J|+=Mg3(r8E&M<1KcNKXC?DP1xM8ckVNgAr7y@LZ zHoq*73Lt&^@U97$Pltf1n@;H2cJE9>FwmkP9OkY~`h#FB8nAgl{Vdzludr~bQR<;P zIGI@R_e})*yGzIWy84cnf53XrW&!G0eRmimN;Ik8bMoos!LymgxfqeJADEUac2`7R zn;v?xHz8?2vRagTHm`SVMr^*_gZ(*t;ekya`VqB$S^op3d8wZNfpY;oG>w(4&}|%& z{#ZlXPQd$KtIwYpqGGxt7n|QSrneo zTfH(0m>AB+XRF&EjP%p8&IP>Nk)bE;0~ukBcPP3BI#F-Z&qn&dH$7#8$xW7zl>`jL zUcT$(2GRn4kKUj+KMy67YhlzvCIPswda?mOykB(2L(r%r)$H7E|DM;xiB05ZE?ma) zX%%=OBm!jZ+ycbaNvvw(amkHR1A3Jb!gjOu>L)Nbx{D$trUzck@!M3%2Rp8VnObQwtQDcpPOlw0*}bp*O;IU0A)II@^5G68j>D%Y`Q`Eyq?ag zRmi`RJyvmV0+RI(mhRc&Q2G4|5ZChq7fFZ`^j}uJ&9d$4Jkk}r_es5L+}(59HFIh7 z#S0B+KeB~YZiFr~H>=p-YG?e00OP|hUd{xOw>I$`Tul=Z#Czj16D2)Ki?*L(2eM=P z7FKbP^4#!pKL_{eQAtRg2G8N)@Ce`NM+c^|AS zI=OVSBa3h;k6pD9*Q9la?|FaqnaMDLo-$mT4NZM-l{RSIiz&;&OLLTdFGjE7V$yc1 zdnI#X%Rzf^@w>CTw(34_iKHJEhL0G%nKHX^Rvi(OgqSx#V}&S&ciL*s#|}QJ=;Y zDan!tkmOSh2Pai)gVCM}wAVoe%=}mwDz5_xKe7cQ7aGB>^u0(3(}CJ1o10SwlEMO| zAV|+kVw>K=GICHx*Q#(Bq2L8=c%pccA`ajf)DT-9ZNUPqF$UrymZD^hpp}5^%5It} z0^}9Yzz^u;Z0_znu#zL%{q6LZ-5_t{x%UpK_Me3CUU@MrkA{cRI)hktI{WcVOrF{l z1N4`L!m>RvGDOv&w_anZHT>RbXSFjq4Q8~4TR>w$L&F#_-D8Ts1IR`b!CXBxId>Su zlOP+pU38Y%1QT-?uV}}P?uX3eUk0g!?Dtpie-e!H(yHTJb-*8DM&$+ z7Rc`k!;qyJ4GWj))iEOPX;3*he~+nJhLL7TzN;a$wVz277dlcAbujMyqBjCVhMk#= zx2)U4rrT3J_S?Bimi9}3c1S3;gDKu$0wrXA#*1yokyx^OTC}rjNRZudsvv1*{L=mL zSi(o;H8?@_G#c>uGI$zAejmHAK0G4}_XBPgZS*t2jr6@21oe!Xwl{vH&nm8-&85Sz*PI`-SF>F5VjiNi0) zV!5+FGCLJ%Pu{IP*fmw_NX%n|#wtptNd>2qNisMq({3BsQCC6SHELF=rIp)gwE8Fq zTvndM-mYT_Pr6*Y;PXXZToKyg)rSh+zOkd-*;PC@T`?+A7MOmk5x=T-pI9u2%691S zyZThfxm1U8?#hT-62HQ@mSkb>Ej-j9rk^8cI>Yfg5JYd6-1X52c5(=*pUq9 zj*+z%sy{uAOFQK`mEh3O1LSM3S0OR?+mgRd5twC1Wf;eyGd#OKh=`83AC%mw&pc3` zgmj`XVMMACXqCk4qyCU62Fj5?6ZF(P`At6564x9$lI%@i%AwT-aekCs!nkyQl7@4L z;FRPDq8g5jZImSrqb6iB4zHLRA*j^;lUpgFtdE;WNZp3hPfQfG%7!jQx20p$EvMdK zUFxLB^@>Zgk=~>h*9Ie2EwFaC$m7??KxvX+_+E35n&}jpK7@*dEDytwAwDf)xc!NZ zxD~17hZ)CNju{g?$|(UW2S>b>YBLEM;RAS^pf%XT`U0hZX7Y`nXDyBYk|U?|W+%&W z;r;cMBeWq!qJOP)?RN~gv%x@kLUk0dr9Fs#lM+6qQS!Uv)})pSMbp`>jp!y*h7vD~ zxjNvY{IFAeaWDR{`npTP8VNbQ$U52OSqi@Hid)#3P@r_{_WDruc z?egzGSez~~kJTqhbIQ!Vb44Y0+)5oEb;oNBq5oy{egmTy(8Frfs9j`KIduCi;gxMG z_am7vEU88j6QE?=AK9jdr-eJj`^WpuqDvSwD0=OqHTelAwqtYqYsVKjqIUUCT6*`l zIZp&40v5JY3f;lC;s4LK;oW^3M}!cm?%?Am;!$I=%+~Rn0m}>4Xb*k02?pYIj^y*)FO%T17Ml9&?+ z!yge%_A(IvIkL@9aKKCGnb217xP6lR)o0vvy-T>Hby+p(rd;uV1c?FAsHRhX;oPbR zUf!nshDKCy#ir!C&uNT$RAg~$dv{3mnDuFeP&z_A&_t6t>bA^tNX7am+ugv1vy76L& z;EC3l%hXZ=z{w=Ze4&*fw5vo{?Y#s(TOI6`oMkbp(1-->4URVy?xqnC{?7z)gml5;eq!9jKIr+X{2`Xuhpq^y-(&+i%p$Y1hl&bIB)3QcM^n zHQ?hmw!n@bY-R`JLY1W*q=cfO0uhG!f5Y3nud=nu4K@@Tr2m^|FiPF}UC!Ep_)SPh zYzHLNu^Nf{hP0*=+0xkZL^pGlV*e-J92KLx(-)0mSfIPJjRBx`gR(28GqATF<5g8x zPi#Leff;GHe2Pkm`8La<*s!mXO}dFD6Wv)Uzs>=M&^d^|V9GLz2gv02N?YlK`; z-_&Mb{W2u^9n?_rZRCK@8M{$xh4Z$doYAna{kp8dZlL|fk5p>*dQz14MsYbYy`8`n zcHQ3Yu(xoVe|E`8QX1(u;V+>*80MJgYk{zd0z@P(Gh!os{xU|oxF2y?-38P6E4ZCU ziwg7tcdcC_01C`J@DxdeSA{Nw zZe(o^80z`T>P|tPbsL~~nm4TSK>lSn6UeK_M^JH)RXW!s8?ejW(`1__Z|d#%)}ro> zc*<1qRG~?T)pMpgh*?4Jnp$Nmj$XLQ|Q-( zJ?3t1aNV2#vUwJO_y}ezYy99=eV9I@8@hH_Zgv5+V?trkDc#3sz#m1UNy4c5HYv}b z@mm7S{b-OcYNocJv!A{FS{pbhrihSG9D1YN_OX}x?ia#*|ZtJWsVNWz~=n_|KH z?6bd5G~>t~n(XR#_J{SDfaDh!!ia@M7fuj`JWHjEizh3IB`=xYzxu3}vB(QNux+%b z-DLI8rr6{z78zR72bI$Ks{5jk$^FxXJ&Us>^F@X?-G;}4C@I?~`>V56?9(h1U$K+; zKxfA!N9u_#PqrD+u!AHc;6s+>2Ce49cfTy*KZ6W}0^hm)pQ>??bT~snR?o{CYAfMg z%jiB{(Ly3#qkcy&TN7+~n(3UEz~&77;y|~=*ggME%ekpR>fGeyR}}GcFYU@!gY&8D zyod^~IcpYZvs~o>!8S(MBvHSIxgxb@LRhV~4jCzrrf`Cbr1s!jo*s{$)T#kp}+nXp^$KKIqhMc>3t$P;^=?1!uW!&NZLGF3G8jgznmbP(g3dwp^nM>1pEebXPU>WI<=c-+q_8!)A;!8hSD)yw#MB{Cn9RX zV7{+34C-`Sp8_Ul>_w3S#;Iof%wrNE;?_uyw*5NnaswDUHh@+k=&nxTfHF~!^WF7` z2Sbxnt}*HMJ!756h^^`0yjUojEhLAG+?U%m)<%Nj2-~1Th88TygoQ21dKbj<)2jCv z!7yljdSBBy+w_D+wn$7gEqZntSXSgkAs9H>YfcyT*)koLNV_|ol*M#$;fb|@rMk!G z@CEGuQH_Iz=;@l zPo~Ic9zef%AK>DpGnL2zDA;Pw(G5XY5im|dHqci2&J8G#=&MlUDxkBMVKxD2Z@R>u zf1gozJ6^ElntkAI>RxDmpoCiNAuZ?YnFJvKfQQ1ETsNIWJbY8~eSnP zHoDRZ1f?=WU#D*NJn1RRS(z5YX4QE0<5~4`Dj~j!Tp0S$L}+7dG~g~YNIrbCn(wk7 zj;={_$7K6n4QeFBhD;p7i2FBygM~|!3WNbk;={DCmvbYU+ z_92oKLum>4`PwV0kY5K~i?Ov!SqKnMMc_4RZiEZG0m7bydsm!j5ZR7XdLB5dBp2Sd*XR_k-KN z?YCJHav+zPNGLl0_`mZ-#*w)v<5=()RzZRCmB4l@Tjf7WpU#~NX{<~nAg`$X9*s#Z zu$5zWk!S)bpQ92+kh~iIDN0P|ESF5uXeZZ-hQh6M92>w;IZGnB@@gtzFNj*v;Ei`| z43-xlhf8=K_{bzlhVBwjU^MbIsNJlV*#Yo@P;LPKESv`VzD3xI0MManZZ$&XB3{T5 z+2YM@ImkAK?$5&R*tUdZU?jr9h7v1-0p==~D@bck;FZQ}&$u7tccqeke3v6@2YQM3 z{L?Xj2VqRm0DoGDXNhmRhMg5x?yngUMql5R5pkV^TANK># zz{~=>WBD!pD$z~b;a=40|AiNv=-4DEHN_(p!osC{W^*ymR&?NV#1uu7IO*{K&CHGH z?gWD@cDIqLC-gOCIUUAHnz!mV033Tm1h0r%U_jXjK&;r+UX?XDao!z>?+I|jO$2xmW; zBt|-@7(9_I>WI^`^4s!-Q<}=8cGI8fT1>^X=RMELThtzkh}cf^xR9^Q$|U8byDX-x z1y+3em)Sz75EFYDcwCzJ-^2z3C!q8Gbr72Azzs8_+xXvj-T}U3Y~MSkaHpDQJ(>aF z>wCRS`E78JS5`^svCr*aR{;99kX&?U(>7qxWZ4R(aeVQUL|$k+QQjUn5EPjBJ{^8)7U6t(?G(?(5(|izk27G( z0Ohp)FSvWapdiNMF1Vj?y-&`wl}mr=hHjV0o(QaE02-_xFAh(~3+gf&<5>6-F6*aC zxYnQxe|erHbKCVm$M+&J_3e`WY)rBuqSaGXHQdU~)@v)}rqufYa10Prx&)>ak=ONt zX)Sv6Te1bLA39`^FmNJ@QvYbwH+pYe2UtUlq#}fhjZ!8fw0EH)i+!Uis1d>?;L5+M z&ONxpxiZ?I5*v&D-Dqfgfl+um7a9R@Z<<)d@LU6ds$l4rwKB~J_(|Zjr`RGfDr~2{ zH(92?<_gbC?MLz*JS3j)9;$v;h>FBbi<1iFa4$oU#p5vW!E*XR;4i8!3kDdm5+sM* zj6Z?xml&*kNm-)F6%MytiDjqR7p40id9w_^yi*;plU&;DyBG&CvtAUTft~xw??KCZ z_im46C0=R#Tk9f-kO&LBJ59k>Dl4gkWz0JQS~KYqZfr@W{vvFUd%yn&M*r`dHChns6+!8WL#&UYZ|(t8vMrgXbjmb< zsGg>seHR2QYE2FFAzidDFc@i*j4Jyu38@Vlw0>fgCppslh>BIKypWbrO28iHCBwg65Xry(HIy_59r1GI1%$aO`KM z-M-w>OYL_8{x1LGQ8wr<{Q##9h|ApUwbx5Hh%RrshIZ5b|Li24dGZbZo-O?JO>d-M zD)WwqYLRf!zf5Of(Iafq>gcGz-5)}GP_EYf-y;!V-`zp>?~xY9HFZqx{zgSEABq1B z8gn4*K*s0*FOMQ7XsGkhqM-^`p7Lh8#9Oz6;hnHQH$HA$N-hyl2qD%UuvkN^!)}DL zH-xjrdc=I`K`mo!GJe#GeZv=hGARQAu<}1pWXXhzLhEPCy5cl~f=#rll?+lnsvok7R(lMJJ)r8=N0F z{bBlpu@#Y*e)18=9k108Hq<;&LVX&1eJ{yW@Sz0*HQlo(uTl;$?v4mVok5p?+l>V@ z?y)k6_cp)V>%wxAfvv%ZQON+$*6y#2tQ1MhN=pt?bfrh0hU{_;7U8M4dG}0Gj3aNR zk-ld?jFyJHJ>7f1jH|xM3$BBUAf(sBJN|E2SHZE2ZpPJ-Q3QrV0ufT#LF>f7a;$99#tpdLX#%dKL zOz37b)QAmc193c$jQaBf&?sFDxWt1XzH1{S$pMX!eIk1i=e}Zu7G7_RgZydt$Mg7# zXoH9xc)PLTH=IA%)09xdx!aV0#~HvX5P?e*q{TX| z+>!+r{GErUb}smp;PU8W;&ZcZ-4c+W~a0xMze86RX=IIn$O$yMdS>XJh2szo@XJ)+zQ^917KWvb(i`3KxYKZDqz z{l<4=4Z?I3Y#Bf*^E`!IEh2oH;rKj%Vqo~WrnCP7WWcY`^~+`1b>~*&`c1xX9_nbG zZTqXOPv1QQ3!*Pm4(wlvin1TX#OHSJSJ0$$8XO*Xjt2R6<#mtQKTi1~lWROQWc}`&QOR#VlTvff z)*qtgabiVErHs=>!r#X*$_aTxHvUbcMAL>xqiJ_1^(;}BRFsN%#JRW{yT>OEGcK>V zoQ>;%`F|S%9X%KxFe-;BWGDU;`t7eYs{^`9*8*>BdD)RPk8+H7LnthQx~v&^^|=F4 z14Z6kjs>^fjNj8f`TejY-G-9mySS$y&dNiw>I^&pAEeZ=glQ9+TamMZNgHCq9DQrO_J0e{pYS-ON4Vs(HS z3jvW6*g3}60#0?h)z6I`d+PVMO?i_$|?X8iI0I5l;vqEY5NDp2CahG zD^g`CF)UDXCUku--|s}5{roxgXD6`LPpOzg7$2U=Pr6yiU(kpCOO<3j=M#h|*|Ik% zTrw(Rq>QAptRdGyeQL|GZZYJ=|}$4R76;sJ5VnXK#$5E_hE8q5jMO29Rog|Mhy95ZM*Mjd&(F z1YTH34ZkVT$+}e$4>Lzu{`m3!amO(E)q|a)MOdc4x~C9HF(mI2=_AG{O;1r#XF~XB zd{DUB7b;<>+AHg9VqtrgNb4ZWSGdLO67pJsiTU?$4?epAB5at>jmH}K9XOGh0wBb{RZDuN!JBUUzpgC z%nT_A!3tM@ovHTfk|nQSlI>{`zXsiYPCK&@D-PbYD=n#=Fo$grjVb7eX~>21PS7KP+u)a4{dm=(3TdG zE7+JHAx+3u@BHE2o`BlkY6;DO4rndQuskKe4DTlOb#!#YESVcs@$?D3{H-n;P#rL- z_eEdl$vO|*?vtohT3i8{POHphJ`>`42G2D%;0uRe@ptO}@urDx!v=^PqIVfWk&D2` zbLY-x<6qqba#6a~ypM$6leNPUTVBz?8?kDdKYaw)H?mqO10om%jP z+LzhPKza1hRox1rvWHDwA!Y2gM&DgInsJT91w$BUYp(J8j{ zA_FE#CYDk{`d!!XC*-BkA2nV&Wf7+)-a89@c}_2p%0i*j(7lq^!N>5(#}OXdKLqRv zI+2a%DeE1!_XPO~=>`}?H*(t17mo60g zYp*SeZ>A+c)4o=!gikMLHO)TrZvi=n<*@^qYfEl##WrxDcx<`77yQ`pT_^QSHQJy& z8#8o+M5%xB&-MWfywo#!e{7R%gL@F zF~1iYG`=icqHBqB=f*|<;9W-uIbg1DQc%1fZ@w_6g%Y!2*x$IIZqJUXINMb`@9QsJ z+}1n_NQGP{Xa={){-HOu9H|B--O#jr93UFmO{s>Ve+%#09g_x~Cv5uHZg|kkObwa3 zeGzQIC1lM?tZ$1_IYI2gCo4#9wmA#aUajyo8(e(^GIsMLX3Q{3+0LogBH zL!(N3ND^?h*9SJPk$k88x;b;}85Qmt6)^XU39VrOQI8k@^%pqZ{(;<(`mGhpqTF<@ zireTk$?*5e=2X|2$9@8DP?tyg3sKG4p%60==;7Yj9lCD_xvHo`H~?U6kU+g<$RE0MGY@21>NaioS){Sgx0 zs1w#YvmTlKg{z1}Bgw*dTf6+SR(Gk`J5nX`AE*rnMIf{Q;GU<0nM?Y@Lw$(G7hMSVFZLi-@?}}rYxt9GOhGon z_yEj6c{oi%gkng7$`oTrq5m-nBo;{4wniH6x#Vuoa?#d`aWkRWfvY2kQtUQy9rBk{ zwO~%TKd#sUx#zSH?i@-nHE0b5!AUwzJXWU|z$3h8bIr0zq@2q3=!B6@ z5s5QIb+oXSI6fbK^7wnlB~n7f;FtB7H$flQf+h?;fYFWmU z#YNMK`5y@w@vf-3pCIu}4p5_jJF{e;fPu{g7u4v~S5?4%vy1}XI-;aP{AFNgj(M+5 z+8X(1SfhB0vO_t(K~`t-bYK>l`)rH+zF~;rYfo&D5pY!dGwKb(C)E5o_XANO$#X%>)VeBxTbLu+2suH51=8mci>a4?Tj+voZm;qGv^kNS zYH&CmeD7I3%Ly)YR!;UpFILMbS)bfqRx(XI(8mmknq?QMh`7iVNj%~US^N6 z@*-JylZ_P00@3DR2l@WbNG6a2z$GbUjnIC|O96Xt5+$DF^^Ebx-M^OvERZiQU(7ev zTBTgZ?Nunnd!yARy+&#=-O#NJC@#ZflbT;>H`oOUOddu}0*B-GmICF^$lT-Rz?Da; z>sVG(g}8*J2Km6E<*{VbtY*Iqs+pnm_6hEBwXA4Hb->b>HPnf$Gi}wb_S?p!hyw?e zuv~nW=xWBj}>hw*Z`$lr99m`h`oevH22_)4*gHvv%*lXp6e&wF7(%@dp;aX+vDP;GjvBvxbnDR-$fo?gj{8`)PoVajb?(cSL zenA7pCOwK#v)hY!nU;)#U_#E*SLxO?dLond->4)O#yuLl7k+mBuqB;v^ifn_8~lnp z!_Dus$s(nEqZ5WBQg6O)Uh5PqlQi?~O8pt0=uFXIttl_}e)`0p4@n!sn&UN$&vC1) zV^>D*u#{L3(VjNcn_RBHY0gz=7!d&tswgWE?qUcRfk)LbSum?XRJmA^ZXAEc$sNgk zA$C!mZL}Mkr{xYEl1+~1O{(2q{>{Zl7>_dWf|kmFW{{7TYOd>dg@YIarM$t0`~NqWpg&A+oA3Vb4q)<0LhWh09h2kq2%87^O+ zh@v*+Np~{QYn&KA;k(ju*`32)2$5%oL_LX}GWV8X)q(Eip-j21J`&nUwvi()le~h* zvl$j{uiI%q&iJGK+R`Ie6`uSm$V7_AiwoW3IM*0hizhGT0jvBsU2^9tBUW-vz@?bz zXY76cz0FLIseNW~AXk(;KdAxwQ4UC03wgS9&l?fpwSwa8$_j9|5IXyikmm)Bf*}{S zXf%bND-h5}^Vy$}{JEafU?-+9=Y7r(FJNxpcjxcU<-18l z_^}j$xpW8UN1$3(9_AUw0pocaj=x#ZrHa|tg@?Ch0wk(7JM82OMW$nU34P6 zzZ<}D!$UcXiMUzpBJQ4ZBhW&%JPh5I5ey>nx!QDACZqH0E55CCSxob!Vh@ zeR9>4sms<0?6ZZ1Fh*dt2M>&3RcibVPiQ7rYBuy!f1Ak+T(RVL1Y9_556DUd8}PXk z*}I3)O@?dSxlcaLZud~n)(?MT6WDZO?IwlujaI<61!_y3OX_vIM8>5dG2oIcOqrMl z75#nKS9*I!{X@rINe@^I`Y)oG;L#oe=w_{Uv%V_+A1BTfH&G&IEo-rr2w=O)$o6PH zQ0pyPi-|ZqiP+D^XV-eu>c+s}(Bb9bUoUVtkD4tWxM>fnjIQ36uAv<1zXsg`BfYd()_OW)jc~FdTaG#woPDt29Hz&=<;m^$q z8*Bb>_xo;K^Cf{wJ`VOoUBHy3MK%}LYw(sf8ejkYE|n8Vs@NZ%edT=A{#>L_pSGa@ zi?W}MzjvSW=*J%_!QIWyQM2Y0c_Ox#?2IVJ-^!QA&yMC|W{PB8D1VKuI(?%28RD6( zepWu=9l&_~aQ%2+47t!-4TtcpVaZQE+k;|FOA?tRKa-FWNEob#5u1M6YmKUSveeI< zRd2C0BS&B_kGW%pm%?dWWS@L4=JL8dBBrroNEaVZ7t~%zb-8g{MNWYvtniD_P>eVUC|s^?R~U z7fOlE)nkf#?Fp9uA6su77G?Wx4L?dqihwYbAPtH`OGrp}cMOfvA%iq1os!bsAtfL% zbc-NeN;gAy$9M7k_I}@Yf5-lde>motx$o;bW1Z_<>!iljkP((`mv6c=XIotG7(3|_ zm#lD(`Dl_wqQ4XeknM0;Hw7!G^83?1f2OT=vrqTAbxU>pE^(b^XTsvf|N+-Ak3j!({X;2>a zU1eL{EZ5z?UGshT6!8GVYgI~+(6X*Y|BaxEh-*l+d?y*c1PL}yJk%R?9-;()j?>Yn zlg@B{eMex50ju-i3M0JL^=xJt2?%<6U?t_5E`f)~$RSTo6SHu{5&)M|@lo7A@w|Rw zTqn+rXa9@-Je9M}u*1{Da)h4wNvu}?@*cf`TPfd1n8$f4w9({#8P2u9OV4@!{Gz3e zoqJ3gfoggZowj)$cCy*dKc(>s9~J}u^WpTlm4IOtu+^J`CeSt>bT`-+rIUmG&IR3X z!UlE|>c9IwbPl7Z3#=5rMc|~dkwqjoek;I))t)Ue)9V9T{=ue%zR`Erq-OOigKC=_ ze(0k*YC6`UU^r+Ume?5awR{DJE@x{G^v8=6tE@tFt&;b+>2-UF4Mh3bxXFMnzs9<5wI^n%B z9FDheXTug1u?p9q)5fMpc*-52plv)xW5i=Wl+W{x3G3y;#rwPV@CaaSOgs;)&-RpG zvN!FU*rtA{4+pjgu|V>tuj1NlTfj>j7q3{Y7ARSk7N<9WsD}oW{MUi67i_a4gI*p8unzNZ#=$Xj%5QEWQN|C9fvD3a70;LeD-OgruyeLwy~{+0aYW(j zhM%}hr>x9={=igxJ(oBd3RdacktGm~NHTB5f`3?}p=5kud4FRCwt!d(Uc0My z*##IqNJD6m-;u3lq9f+rzV$#(*Xh5DI}T9X#Xn7hU&N9~Ig}HNxf8G`qZn=aoEXE}UM-3Oa^hNq30Gg$?wnvwVo|oz^N;-aat(aaKB` zYVN4XeV)Sy6J;}td5u~KD|oEyc!2@Zfznwu2E~WQO!qJ_=XaTIiA?6UHd#Js+8+po zsyf8KJE2hptcH#L!ti{8#9KWKLZLPOnU7Vo~gv(Q#IN{)f* zi+>o5@2&Dt>v8jrFS*^dI>|n7&`IQSO6DJE(x)jLT6s8J5v7hsOB8}ws{A%We-lYQ z_0cs0(JWCC?|^$Ewt2G^5ZbfdvS)N8RLZDUGEq*;#7@SnjsdpS)-L=P=5s)S=jNZ+ zC+IuPa!<&KhDwf@_k@T6t-ln?f`Rl0^Kqab{ZF!?VO9WnhJ`uM3DNiq)OkR;)_~{g z4kLYG!h!o9(3we*WOU@fR6F$VfQ3KAZ2l6&>*!iTH}_trLo6b^<0bUKZ5KHbjF+gu zB9CCZw_|p>A{!DW@liVH7|d~C=)}D@+M!TFGgFO3Sty^7yHp1Tb`Abx*rhQ*sKC7Gb9jk2p@#z4pjygS^Vj~>sFWIJ~R^Xy= z*`8vxSH(u3Ug6TKK5a47x{_bop~#)->v3BzvDhDx+oUuImyb+6d)0|HdRh_geRQ_NwRtQ4j;C)6khQ2`0i~zyqSkaqy)>X7+&! zSaMAB^q#J#E!1pEM))kokMB#bW5UCTQ}d#s0`T{}6E)gZ_T-w|f5S1W@|R9(=f^Up zZM|?l|2bgLV&#Z~$}ZXG7|*S3Hy-P(R5!~nHW^k%0EON!ngZw>&EkgS(wB&Tlnf zkm}O8UH@;XyTWwHxteWDKn9sj{-a|l?-__cIj8VH3j2HpACuPu*ooqq8pF8_T>=Yb z9TIG2Q5&yyUGs9F;y%9XDT~W@BFD7-FM&BUi9w}B#u1w5TuF)U>!i7m>|aiU$Xd76 zG_Ya2C?CaXQ`Fih+qIN#{r+Srph)8ijawk7cHZLX^$%?-&DhQRM&G<;`0t~OJ*gXQ zafA(jlVO_mST*LrIselbsA4bmF^TUFs@jf{Qr!iUDDoLKT6yABdYJ^GH#|tg683Av zbhR~r#Lk#1mr@i17>Gh&3n^}TMv!qka!-{6!x!5*1ccE5R<1Qzxjs_~pv4y&)#WYA zoo%8Zk4QnET;Yvt%2-WYYjF270jWh#!uIhQD2v}Yh4AJI?r7F%fdlTD)b*0D=>^)KWU9QjtR4R5G7hj(x>j2w^f9E?YxqY029;kN%U!^I9 z|C0s&XVok4Jy2LK(JMU{{zq<~dlFMjMpYMSB4CfritJyvBk20A+TZczJ4Uk|3KbKE z(n{Y;$PHuLI?m$P2nPH$9{bM$vXJziuE0<^>lbg|+Z%+~33~>e@mEVtD+#spptzfM zR%2@Wto>8QfZqOkLd($q@HXimMtNNh@}|IJsfI!R%$`hd65%r*iZeuUAQ=)%8TuMx zXBb9#JcIqC^fT?5H#3t^nwqFX=O1*hAiVa9UQtBo&ksC2beA0RV`{iu*zXH>6A)@RG8 zQ?W!lOkx-ZYw!*`qlRhsZz6>khPpPXM^b?zh_&IPNaWt-!n$RP z(fE%D@b=UKs3DO8S!EF^g6NN_e5oXH=r^Kkori9p^AAw8cv6BHiTVLp3ke%2eRO8D zu@Sv-Xsb2ek++A3vcHv@RB;Bd`!DeJt~#Kkc{tWFlG;@*et7<0b=JL9Fi^zF6}T_A z?g0h2dq9D?>@3(3Cl9;az)V)NQPLsuX>(wL$(je8Sq5=T?@CEx@Ok5nkkOO$v4nNl zBA*Gv{sY3r-cR*B*sL1ArxE)VIn157G(ih8&bA>*sI1(!>6J#n7iI(P1@8_pxuzZ= zXS}9x;AmAL%9~UbHInMeDx!(QmM^Gg1;{NVp+%wE+A2xd+8`<4eAok$&0LUbY zfltV%g%rT?Ac^K*p#h-^g3sOh%CMEbe!R9eu?<8^SyJV}&j336pb=+;2}cZ8m;akT#sA>QM~G;@%ibJsmlNOe zm!m$UwAsS`<(sXlC2!SbD)OKgB>Uh-ReVS&eo`-nLYS0|L7*;2^12^$(fw0Z?>Gvq z$h8CNt!o6kHN73y7m=vy&ozh2{Fc?y|7=e zy`<32wnu>Y8YYT4%Q1fK&Wi7Y-8c`y-4%(Qty zSGCkf2ZINg%AnCJed}{*v(T3>_C4Sy3Q;GCR~O$5?V>?jR(L^zjn&2LJw_CT)>G%O zBFY{TD!hBsj+Uj#J5ovTZ-(VIeA0)}mr_w<5;QG#o718C-vaHKf@a7M=Oamgh~#1) zp23AbClhT&0tlaW_kL*iW1(ztUbFV8vd^np{zw5?Y8Pq#=(lvq&qK7O91Q5-Bdfv~ zS|$(LOp9H!|FZcGvR})MYX6F(gK1Ayunh~=3Jxa?1$Z=EziC^)r~xsX#vConL($;U z?Vl`cCDdNAm|Lnv@zm$On9b=P%lk+A4DzKm$2MvrxqWD`|3Nj_%#~YguPTJQDZ_;j zK^!ZUI471qn7}yx!(K*yZm%qqKY=c!Cqbk1Q}P`HLYBW?a+Q%eyb{unZ2ICa9gy=t5mRNXOt6)YU3W3YOd3bunXX59nz*C~ zhF`ERJ&dokGmX-yO7L{9h@W85)Zbn=rd>(td%G}93hN;j#(-kaupDG~d0J>=^vL6Q zdk(Vfo&K<8h0bbpq$o{J4RuRK%YP%&TJ$@|6Dx6&*w`XQQ7OB4nvujt8>@LN!nA~( z=yLoVMlJl3WrXC+Nh3|?{7+W#_6Tw^9&Q5rInUP8l(D(l7hp$unyi*v^>{`lceAcz zChv7Q8#2^m?AFrk=~6R|T94qGiy3M3@0mNW-+d+VuOgKb7~Nu;bG4}jBzkGbR1`Ad zHo8CQQAX^~yzHK2XK}16z&cB1SG1-{S=c%+FX~2$t>7cpbwx=6gPv7ljHL1ScSUD8 zEbbdxJ0JCtw2LK@%nGqG+cLSPd;M3XJ+g`n^l7oD*Zv~c50GuRZE`j^mmK}LQlf*# zTCESaACFZk1L-rGuDQ%5?Qk#oitbw)mSLy2gQ^ixdAt^_)C_St`Rf%X_35;M{(=jI zU-pSYO#`%bf|t=!gW1f+QE0Rl%~q@g3*^6}7vS;|)@#ykK6oEiq3EbM=pNtN;Qo^{tOdzr%!om77M^OZM3PBN2l{$1nzC*RIJ_ zj>ANB@uJDY^|7Y?-s=1xNAY0(r*ls;ZV%hh79X@Rptj(y+@#*;M#207Tm;gYgr9ic zQaFa6M=Ve>cX^?ZJ-JR8tLfD8Y!!E{p-tya$QT?m`%+FQjGuyO?JG>TPXQH(E-Tu9 zLYvO|4*kHX?KL*>I|0X_g|iMpgLA6E`;7A+;+qoB9${m9%Hoh>h@1bNLR}iiNe|Z1 z0B01m00?I=T5tmjyK(w-)%&tfeCYkoS$kn>HII0;+Vn4LmwZ^WZ2G-L`yGC?uYJNR zRJewL0-Xh*wQyYqW+(v$<2Lnew7h(H-afT&QKhI;oAksuO_?T zBZ@ZeX%|Po#0jq5-RggT?PM~+X!9A5pgpH3uQn5)>aPOyj{kg&K-0d1Tpu}thW#GH zLJboGzi{Rr|0k6-vrHjkWr01!BqnNnT2&8*z3TxDW=C7E9vg6^Xc=;p2gBV=K$jBw zQSpa9vfX!8B^SF7hBqdVCiD;kcb1kyl0Ln zyyEJN{emYUyPbA4=?jwsg8CE~1|KhN+piDfa{GEQv{EBZTS?pF;CzJ0FGuVKzzQeA zNhO__73kao#{Ezzi5+WjOd5b6^kIgQHkBjv&p98r@r7XgQUr>_zUSJS zo<(`s*u=yy_zyrS*`7_$!AgYhmh5`J`tD1AmKffYygNiMJ!_T$_Cd|*k^>e|3|SJw>-e2qSDR=sb0 zwNxoU<(vShm852yrr1UrH=R&#J2e#2h0)7Gh@oXg##0|opP)yOhg_s+bqKy%0_vw8 zwfbTE-S}5WylZgG-4J7Nj9qk}&Jc(Kwox`J7(xO!+yK*D;V;wEt$#Zx=X&uk4WJ3v z`U}K_>dy@*p|FX)>Ce$%ovJUo38PN|3bIt$Q%2YiiaEaw1|TX|C57)yJG-4Lhjpe~ zpMv*B={<#z4_Q#DQd{>Q$~NXo+2VaSQ)GP3cE32+MH=N? zxuVf%BXCOR_j3=vn{n0NV{5+qboU4M?7!I1zo?Z&5Dt6s$P}$iqe`K~@9(G$>~67=Yus_SzTqu-Rwn=z&9h@^d2Kcr?#rv}tI7Req27h?1KF^4 zOo$$r+X)euPZRxP!uJ2 z$qrgZ9FzZQfzk5fti_3@^)dNt63T-tj9;=$DhP+c_-4P$l1bt70{q~*^x_0)G|xz! z{-UiI==T(lJA$bdxr_Gv)QY}1SG)noB zryS)BZrlc%cKsDJdP(CIUPMrHxkja+d$0Sn z%-=0*T+hDnilIL}@Kfr!=mXxmITUY2@!e{*PTj)6Bb6SATa?mEVGh9E-V=;Y;;vBB?{T)4rcT1K( z5El;}kQWIW6w+7+_Qa8_ExYpYHaT4KV$*p%vQLTI>KfM}aAx*#pnv@H-b(#nsvkys z>^)fWZ=^y>)LEpg5<tZ zYha{ftg{)XAC)9N?kg5VRT?7~sUv(J5{>E;<<=-bkL0i%_dKCK`KtNqctj#3flbTv zZ&MimWWE^iAFE~ks0g9g&Tbkv1$%?5yu$il&a`5H998>=RV&fAAjkkGyX?Hi+%OMW z)W*>qwiPf`_sVeDdN}yhuU;QWct0h~?8MBtb6#XYHR~xcmIU#&YRR$t_v7(m4K+dD zQy6d2nzM0|3DufkeMv&G=>uf`I)LX{fr}xc2SBua3cm04ndU5~C7;bZEMKd_AE<)f%9kkzE3j za9dK%7Pc|*a-?ZhIe}NoVRd$Cw*a~Tl=`1j%dBG(483^f5D{L;QmEPk#X9MA9Rx33 zP%D@e%RoH%_u|L`v5hG!B2HDNCpYGna>Ky%UVp=28be7x=V+HS+9AgjuyFb0(Z-g}Zf95Hz!GL3>Ah>6UVqr;CP z4tu>br=_~y8lWb9m!KEik`M-@Dj4$QmTvTc&yfh5S6Wm!YUg~f3dgDrE~{*FY}ol$ z(ta?8=;a>I`1Pwsca?$pGl!8=0k8RzqB)j{(}b@Mz4;G!s#t}!_SO?fN28=B55WI@ z7>@9{LJ6Dvro4Xnc%uMI0*VFrSmXwhEhRsg*_Ry}e%_U975*!)B~ror-+@SpW)T&l z$Ah_xJ|L)BM!;fv8>>vkhM|;=W}KRgdPcKoU50;x0Rx0mAJo1ScF#l7fK(!7F5BY$8WD#g(TMZ+!ul{b1HU#y8{F21|a`X>sEWv zqAQX4eU)-ql8?fclu!8-`Nl$0ALfhMsGr{k~|j!)~xLy2~Q0F zBen5AP6$78Npjz?AyF6A=X_889Dftwvr1#n!#kRc>a;m;9ro$&#}9^tVc9`>^ap%x zM>&Ze=iE?6)nGf%*5@pB@TJxH;Ij~^K2U~Bp*(Y7`H4^EeUij~sM9hwzR|&M&tr4RkzcBVI?XFQC4M!OBaFb?EEsS`P7-E#O-r2o zG7?cKImOm)d>sE-~}=} z_7z^PT$wJ1`|e@a=1J4hTtl$5k~lh%)1oD&Yt>uCw$Mimx4Sy~=Pagtpdeo zoo0E3zRI~_y$)Axf-sgWm#j`zka3@|0G&w!C$Kw-xhmoNOtijB8Ig#kv|uGwh({)_ zbK4;(gN%{d-&&*=BYJgB9dIXW^I~QzV4eq$ldSC#CG+50#6H1D$9sA8tBB<0@ZSka zg;|pCX=VVYW!-^~a5wNVFxD-*%2bo;pRFW*Qq8H_#G%S9d*3^99TA^cYs>v{STO9&uKj||WlG$2vCm+3jN4OKE?OhVWg+tj?Hb`p zc~6^7OF@_jrK*X2g*#^Vcnjld^ysymJtnq+Mfg3Qn6mXCpe;~Pf0!@(b-luM9i4k@ z!!b9^UK0n5F>-fTWt??>oWJi`F1$hG-Y(n;+F|n;Cvd(!Dob0;APoW&SC;uok&`it z1NWEu`yWLe6lrFbd@Cm}QJcv{=U*b4*FWtYkI=KQqO}Z68km+s43rO7H;AlLohv)W z(46*{!dQ!)SYWPQG%ayVsnspPJagWJ_ukNQJmmpi6^u4ssf+l8{uu5;0J#HRTSRY^ zC^|KKcv$AYKW60~VaT{~o|C7W=>^sDqQR?0oscO{2Oi>5*u^L3mz7h&qgqfJV3Pv0 zAU4wk#LelJ)6S~vt-2M`gTD#eA+?~Fg{F_3Q`dcaFOJH{3_4ftmWATW4RZ#@J)0S3 z{Jh2qMnF6WoDQYBH`Tgd8hNfw0czDX%6qnYB*$WM%giA#4Oo@!m z^UuCn`E*DVs?woV;)pcFVBF+sWzs@Amp~Hh+3`X;Xa23|uh`RFhTE?Tld zN*Iq@Bq=kRV~5hNg4u|6n>sk39vPa~Gh*T;Br8)hq3Is8&2SLfQonBnHbilbhkMl(M&?2z;&KEeu-K_3{YMNii8n zL7eh8zVB)$*A@qoa$tb`&t4-hcWUsu)4@PQhyO)Ok%w-r8~K1t)_&%4C1W@1G^E_=HV>q78e-sg&j4YpxL>z&FPMC z+$}C`E(p_$D)-*NKYP671w9Arc9>_smNEa31J}J3ng4mLeu?{9-+qv@%*75vC&u!e zokztRU)<-A-9(Xou#R!hb2_H}j?u9`bRdk1jkBDW?>18p6y>f(9qQf;31f6P<+}&& zi#Il!5VlBmSSR^V6B9UN!iotI>B1~Qp&o()^%X_Eb~HSl|BuNn~MtljFeWrP81^P{e-49g}Q?^KS*SNgWVO7q#3rO&To zw2ERAV9izA?I`R)Zvc?uSo>ym!mUFGn%S7)#?a_2c+z*9vSePZTHk0nchvktr;+-| z=4+fE(FP8E;i4JF$umg4bC1+xL?|s3f=S*b6Zulx!>nL)N3~W&@NWy&;I|E_1ZWYy zRfAl)ea&Q@1Mx#A!UOLhym94puh)$DaS8tA9?n|p33htr?$wPaa}Bb}zhPB4Sr@(g za*>~f@f~S#M~Q!r8~A`9(U|nPaT>|v*KdWqtCUD+65)Ik4&2TjXs)?se-1+-3o z)S(HTTHuk|P$YG_mU4_%03U zpwHn(#zfpqFplZ70iXFt29DIh^(x?}E99mCzrEhGLrMo>Jm#~blUxFsMvmd%BZJ6x z!p*~j2Vym6e|B^5@M9?a!moJZz=Co>hIJGkVCe-H4_Z25vfDQuJvOi_zG-klp5l(| zt(0T~q$%2gRcx8Bh;EI1SL4@Rx%mlAfHXwJH3q;;i5{R`VH%W2T+45``&GmZ1u!NV zBU^j=Ucl-e{btF=Rf)iR>fewH_8$0jI2sj6q!`^@c~xZ=sU>PLn}u{w@4KaMPrBL>H)pI8wB+L~>1A^*X1fw) zT_I3{r3OLqTK>Z>M6e7}(X;aq?7G7Ywrn>6FA?_`lni9SEXkj zdpRFB*f%`jIg^K_>CGEL6|C+f2O`Uot^BQM=}D1uqX7rML%SBO*ER643~t@PSb4KP zP>CXkVPMrfyFAc63-}CLOe|?!^D2_-9;*~6J&7VM@t1Q1tD21X1~T{?=jM#gHUpdV z!HBf1H8*x1u&xIa9a&~~IuO7i6(kVr2_NC?|3sZ5Wz-XWBi&T4kgrFEj5+9eW%H(5 z>%^f*r;6c=K5xV~VK->P3;>tDF3hp^@;EJ05k9c?c%y&d&_=|=$mNz-fYR9avqmbTHSSOxW*9b`+u-v;DyfZQ=eCT4~p(bQ8k}}6`?48 zRImWD3`Vr{0YQ{EkJ<)H$Op_wV3CnQr3Rfx!@%*Z@%jYj8+jxEWf22qLmYgv;?p8{ zueX(-I72Imx3R3gJK8Ror9~ z!qk-wLyy~g46RC74zMJeCkYo~h#%L&p9$241?su4n3fxKy5>o=(+iPge7yB-W*y%b z-T7;!m;?~b=WK5t%oVBZD@1nJ-dq-ztCWj)o$xv@+Mo_!05MIrv@@m1uJw7iW)H() z+LxaxmhYHBMnl%AGsw zhNaa6^|kAseuWCm1P75hwk3#3ke>yT^6PqJ*f$>$KU|`TIs#ZdAyRQa=uFCR#Ne>{ ze}mi>>tPUpxy1sySlW)UqA^ zx@kf@4iZC+%ZXfkdzz})hGig1zKq-4kI%{$k60ou4?g$c_`q)G4xJ8B+XaJr`qbi~ z<<&M{Xdg^E@V6^Ca*Uxp!`5VSFY(f&z=NxUL($N9<@m^L7EV*J)R7ghTnjOTVJtiF%^__DO1t-=h3rhn(aa z{J@dbuwBEBH*bE(-?x1aAMjx&vUw!CKVGZh(QV@vA`!Awkwu`6TzqYKU6M}bnV#S! z^x;In*NG5J3lMxT_)9Y9omm*HCw41d5>ahAsrhW))rir={=MMAqQo$K<|PBF-Qokh zV2n>5;tMDn+72IB2<*&-)e*Tfy&6laJ<%OwTPyTgO*DSv*HFOV8S}-2>ma_({vg#r zdhLt%=$-9|g1duFbnjvPG6cRjui+HDH=f?93MO6uLZXR;VU6D^rldT$oDC`j{x6!7 z2VV9>O1bH0jzqktD~@@6SE%Km%mk37)PWsIa5pfyajM;ChjKL>0-A+roEFE@R(5bV zs!!pN41^WTUKYWhN;dgBb&D3w*m>j+!1RZR6lJFXS>&QUM<&UpeUmAGcO*B=gJkn z(I4rkgbzw`hwL2HSQ%f;{oel>8ptWg6Gl8)jWH!LY>nlZccV;iAi$Z!W}Wuq+tJou zV@Myqft5rK$VnOD7Tle|EeyhgpH##aGM*LqV-rRZ1z`()u}~QzFNF0(E|MUFT756{ zLDgb>QKa4Eft_A*nX0?%+EXMJ+?*CrF=KY$p2vHO)$HXMAXez8nFNWcyt5VT1iTLP zJ7;?(S5COZ=cT30+FnR1nJ6Zl>*-j<=nJ}h7^8sl$Aon1{iQ>IWfg&M7H5@Wg(^9L zmvg%SeC4^lVKuvPBmd)rtjnPb1NDQ!?~S1(!pqg_pm4IX(6<`Tf%){~+DBI*OYw~f ziSO%5m+$5U^e4Up;oB!+jinWx3Hw#;swn4)iux?aY+mK{90zqo-hXWa3m|;miEYf?ugx!n&Wf8)PXjqRTLc=8n5dAT4SMf=z5AcTJ|SAsJSsB}=Hkh|*eK{}h)9JJhIl#ERmn1NY5SCsuhbr2^{Dc0ldpfa znqUub*4mCwC}Kqy#4C*ccwTeI@Rm+egOclysNI5 zgzpq3W-tFn(Wk*Ndu5eBtoE?eR(lv&hT>j*T=>>U^zBa#+G(4CY~#Iz-2OGx{a^LT z{B+jD595Z@CC)yCYP@!X=5>Jo}kY=_O0@a-4xBNq@9c z!BAo4kN%GZtin>#sx03N>*OYhLTO+)9VNsEsTu@*a__TQ#0-7|IiKSSij9^rEGjSi zfZCR7(XfwX+>zU8c zDU}vkrR(LX)=N*GmX$e9&)rFTfQ*lp^QLN(Smysd<2|HPB}0zHFXUxk z@GrZ`Q)&S6PZ_?aA;STYapI}J-Pa68YM3|8`BV#`;SmT9S>o?8o1x)GZS6kzp1J_M z5g$6KHh#dJ%9L?GH!-e1ZEI1fVh8}5TtDQGhyU?toRcKP0r;I?6?|Lt8_!3BH-3X2 zmNA_qyG5;Hk8Zza+eq&UUmo;VVDc+r#_)GqYQBU`j63fqnihUFtDB5b3dgCI196%2Sk4d8|9 zfAU+T$q&<`=B?jq(5EXF3PCQJnfSlfKQD`u>%hiAOipG->j_vlwuY0`^kwouHj5a; zA${=rQvm;N_w6NRL*GkvTy~gv z4a@`)tZiX4Y&hsH`IQ|;%r+r4?ToQLngnH|G(t=*x4vcj@FWA3UE>LBw?=T72`=`| z0G#ia(JWStdh;(%Sh`u%seL{O^2{i)m=_K{`lbl=TI9&rwy3eGNnLdIcLF(NY_NFx z-5g}enp$M-<>)7Nu?>$NMe{oCd~EHno}cn>_K8kBuQc2(r>3i!;Mk=cQXIOJ4GEIJ zlSFO;n`M+-cEp?JN#BYIQAmyYoF{8`@2bJVCKv*kOwF$+s|p9-ln<{2WQ?vI|5VNI z>mFm%9lVhA;*S7Ligd;|U1>ee?9}|eQ?PkD;=|>|8fngO;rx_G$?b_mL>}+iKV47rF@BX9^itq_3tc zMY?Q1BOelcSs95B_By_;`{!`y&)3VA8~%tyw!qf`J1VbW&H9K>bCg2<&9EyJ`e(nL zlA+8!SQdI_PnsZXsojIAqAG68as6A?zu+Q}+{78KHDD{N?u(O8I6_bL4h#J@`CaCWA zdBEOGd0Gfv@CG5G#2(pXF=WoO;r!1mjhfIfk8M{P6Z;guid+cD&-;7 zV{Lr%Tg!Z-tNBvLLgk3BgDzE9#H_#I7ZaK95;hytP`cm;Pr-?_YUqUHR~FkIXE#|D zeKU$TdP=#%*KIUL)$oD?HspoTjWvPxyo~|XPl8`}NWd2rl9hxYTJK6jnmEhx&-PUb zjOKCs5xYQRW_BDQR*f@{|q{(;sSE4De1Zr>~nImo)nhDvZYaH^B zPt{Fs2i0$Q6iL|{zEf0}KaW1;)O-%>gSR0B`6{2PM~QnlEm^p{lmBXhXoXi|3@GF- zG}3X`q(J|=%48`U1ol28>>h`=NlQcrSX!ymR(}X#Suh`kKX|MtWle%Q()D3xks?$$ zrlkL?S+)8i^TRFOd^wzTxdnj`DZVB)SdF(xEiDO4@RhH#jMU-S$KV5#Bq(NAGn}n~Ke1<(;3Mt8#fuVStGJtCeHfPHw(u z!K5f+s8_bbe>3q@8hhXm&RiytWmiO_d6!Q{CDf-{t5r=7}cSD)e>5XnuFG6JRD>CoWMqu9;JiE^v!~uI9kaxULagD zEcsA99r>^dMFb`K7TeFL<|B2yESKNO6y@Z@v{K|qc-#J{_8sSFiX+j$jvBgK>-&Cg zoN&k+9`S-cF zUs5vEVlg`$!jms&k^Wk@-BmXoN$?;?qQ>F#qUv9`NFU@0EH0~`XdOecRo;mkW};y9 zy%nyaM$V@FZ8&^8ANx~%Od(F9lPSVzC3`!zOEEQvs0crl1^xuqfb```e95NA(g|nQ za#gz8)QL{OoTu=7&*q!ya{rpj^0AkqdG#DnkteglLtDcH`IB_EZ>P>4RCQu;V2gD> z?6Zq_I=2^R;kSD>dewAH=67A|=$$Ay^@HN5Yt}KfZ;h{k6BhMl_wB^%iH{oWZXnyi z>wUA9+*w~(Bed6*|2Ta=G!iTuu7B0fuHRsJTWG$rysCO3KgMq3wzF00rc`ZK{21Kw z@sA-$8WW8aR}p z#`->`DbhwEVVbn$1n;dr`Q!xj8HtVcst8YsTKLIXC1-dM!fFJr(%ok%pWuHJ3O?d0 zYIrpn+Dl!)15@qj)g6CYfIChKT}^`ia^Qg`{m(} zYYT;=v48x^>U8#Md%MsD4}C?HnzL)SxGOtJDn2E#zl#(p_cT^6ICGBrtCzvGf&7f# z#5J9{`0k9(<~@lc20#mqTL&hW#0ir5aK2IL2Cv7^Qm^5Y2; zFFVa_MO>&n z+jrfUO`BwG&HQ-n((2XY)hgs@zqereoaqpo6NjuukIM8M{f5Uq>h897(V+R=koA%5 z-HkQj|B3wqc8FY#O~0jwIJh{mI4BQa8U7-ahm=VCB%`a_&_rMJuQBw%A?~yl`J`)| zh9;kGfOA`4niT_EZq{$j%nO}Ot=Q<%77ZhN`F!uCv;)5G=|Usy?7?I9Q!f!V*bg>s zJineaq_3UWgpj`8_soX@S5~Di5z&5GntfL(btvyN#z729go3YwT;db#>L073VSNe` z*gNOR;_8ob|6ZWJA#Q}|7pz7r?zS1zk0^*G2EV}4$*qc4{c$cnY#Z@Ca$i|x9~>?J zMx*b>J`H+XwR@=B>fg$~{fdJP^%yqTFo^eeChPa()P`Aj{RMlI@Z|Nvt|B_xE)$=k z#FPT06JB{5vVi!d&9R61#V{Rd@~mJ>Tkv~~Gv!CN4^MX^%{`5xVPElG8;tjj)T(B4&H(nQsBup=zXzRO1zxMmKJjw8Xt{m>WeW8;?cQ$}% z%$cykS5V~5y=(k0J~v-m;Hu6sy)7l7c!d7hdPtXISpj|R@q~-%w{v<7-K`#@w1#(E zB(sl7MU9&K%_TWEipK&g{SA$bf8dx5J?lv$|rZA+b^~M_pvX|-lV1zxn+Up&}@md z#{+{}WWT(=)dLpvGUP8Jn{pe7V4g{3g*n)vHKiP|_)X^xJ3o9s%1yf=S}*6-77>&t z!%lw?cl)G8O5&iVIEXd7rrYdzbY~K)sCgzo&l_j!$6F+9>~rUue)bUKDx9tG>}m5LYXS*-0<8Qj@){I~DzO7^1~WA$H(MTs zmwoBLaZkMy0vwjbbJKGl?9U;Uk&+Q7HS_Zl^IPxd*ft1`C?As8;F6KS*N-kKhAtoV zwDMmI?)xYwU8%+fBWa3v2u3A3MK7)#1z!@R6@gbfDj^SHhY8T4qD!%Ta6zyB`37|q z4Q6@Z=l5^a|2T@4Y2wpMgwAcJ8T0WX0=-FSFjGdwyq0Q^=*Vu_*#%PKjDz#ZQ9#0l zSp8b1UW$hpbx>m(Er}h*zjdR>4l}YTtfSF||@U|OQ`+5QE78RLkYu_}$Va~!? z5{LYd^*YZXLPK1vtgeI0@iwjXu3y1fe{g_l(7Mog*X`_-40 zAV2g}r4F3UG}}`&^jS1fmYNO4p!a;AXy_P+liGGmh?G#~)-+1iR<~W$bHXZa2wAtd zs{e6cbxJTjbnB?D+{M&QYqe?n_;Ivrb1Cok@sA@=0@!)AhwGT$mg)oeVnU#~Mo2qF zX>ulKa#-8Gx zp?cLE7B6eI!`te!rm+t*(J=ZSX;rlmnM0>QMx>nTfAmT78LM~hWXFu_LF{Plrh_$N zzkZ={xD*RnD_DBeTwn1G>3ii=>ZZ6c$7Q93bb9!6^nJ3Ot-t4u@c6nF4{UeKW+?aP z$n*o}+1F}lZO$+S$TiH~reu=V0eWgXPpO8w)^+)28fgs|>~RW7gl7MRf_F?4{hlsA zY%&?jrqhn4`nB!fU%sv#+#A5FtyN4PU6>ETIic35`G%D(L*{o}Bh>J}MJOXQc>Vq& zeehahn@DEKNA9;x1b$u5Tm-cc1U~$uUBBLw3T-@I8Ax*0N(sT5nc&oRmyH0kORNCba|iupwc@1K1}nsn*8 z9gdH@3kg8jr651JJhJ_+-P1Z|2LOcdo8(eY#nG8WaWkyzZpGPSI*PA*P7C}&7s=m-<5oel;uU-fjvd1n`nA}J!1 zfxu1lKi5IsnqXAfacxnyIm1T=8>hPa)1Oau3>LV4SVAKYUfk@=a~EDcLcED0>R2f| z)!>9VJPvmA&8+!}awHMKrQUZR2b(PJ(hY*jv`P3d933lgM$JD|i|m!M6`EV8ohtLp zJ)ZtMfVuE@PN&$k^Z7g{N?$URWJ2ful8V=;-J=pDe589S(Hku4QVNi2n?Eu55-+n| zM$hZ1Mu#kyH*WBHMhle+hO?WjlA5X)`m(v!(Qb>D~!vAqko=_-F1hi5$18^DscqPY2oEB|= z&AE2^lq`M~Pz_J^wd8EfJe3tRQ0{i<<#mhlRWq>|7(wIO3DY?q9|@@L!_Sr{Nqh;) zz$8XI=84@|N$WG4oNcu(6U10!H=B^VCLRBy`})pY9x}eODmL4Maq;b+(w}ED)BL!f z^)ftq`Uk(dsc#I6c5k|?X@GK(z_Z+rU2jU?D^i5Ewr<4{<edOZa+U=}L&FZRb{@|2#?y#D~rbsM?O^d!3);)ZUHFP2TA5+@37Z?PIon(t@l zpRRc`w6QBS_&_d~KiT+B+}<$!F_8fra(wJ2Yz{yzG9A!umI}j>`k>}R39p?gS*yPB z=tTGYN$cj10;_Df84j&UWZYIiNB#S>bf&+<6BsrfDT^qWnarW@vY3j$x4Dw>@7UQ) z)xLJ{C*OAuQXh{E?Q<;?;IUx0QNwR|b!6KsR zW$~J^XrO4gmVJFx2jv0#oIsQ9mOLoBTWE-|j&obbKl4~jj1m6sELI}^XzTACF3vhS z$3mp7Nc;l%2i=YAAbk1xSE|Q7LI#V^QVW{!VbU-EV_|Xctt`6cfFd#Bz-yi;Bb*l! z(V$7o5kj4&V%co(2uj2S3)mZp&4j%KV3YM$2(Q`QO+}>?upYGE;GrYVEPE#v5N!v-e5u&6qR3Ybi1mu%6r;I_gSI=$1xhNJHPo_US5o$0c1h>K z^H=jsu-9-o%HVgsQyUYaMdJu4c}>a6&6QeB%GGoOpJC0keanWs;OF0}X!8AKf!x4F zVbGoY$QdX)m7M}x$L&)$_;O&n(TrwMyBOV@2FtG*Al;Z@<>GB$p>p9tA-{eCdC<5P z8sI;8=;|FwOWsQxfl@(trgY`69pY0#%7#iQoE%_J%A`|+oM}kcp7CCUScp--m8_hk zB5^C}1D9U^3~`;+qWRSuAGo}*A~9BElp8xih}}=S3tip!o8pma)#eJLl&))Oz1s%E z7(XE|FzG%w?6lb)gZ+R@6^Z={FDJF;i%Va-(r&& zI7ct`!9ey$=Zn0`92{6VIHL>_NwnM6d12qdY53yO`eaZewxa$S*oTpcKJD0FEYA!& zdSglhcKn7qvwJ+kK>P9&rlRyhseqFzDLY%|W%|NF047`RP~Cgg+nNET9%(4?Y_8~g z$~ho0Ay!NgOFOa|$^d*I;N&*jHw~vmbE6~F4B?c}{#^9d6p|+xrQ~100SM- z7&SMONru2r%t-bJ?26n_3R4j*Odhk?AL*zefwSe0i{;Oq1hKFBN`VfjfvuXyMiyFQX2qC5t(F6A`)|74o6$yk zy}WWE(LO4Dz&i*3jaJD9uVb?To22PhRiq%#ATk9P(CYF$EE}-0^LcJJ7L^}uE$9eZ zZ>G*Yn6zGCum;$3V~e#CMK8M!s@wP%3_iaRdXDVtbN!LFnVZ}FYC$KpbiKLD^$DYL z(Z^UiUVw}j1s&Pb&VKb~&$$YG&XFU}Kbul&S}i}D%9DM+kV0DV2p;n5 zb=l7rfO18kmMF6pMJP8v>l!W+FZorr&RWHpU|&9pTGok3nz$$jANG3F^P$p>S+XEs z{fF@FdZ7C+yJDkh$^D$dW;yJMIm2Fw*}^7<7C3^8_k&!8hePow6;9cK47+Z8>@Rii zPGu%B{jb!jAjOoWoY?75W1swK(}Soi-f(9>(fm=8Cxr$D(3s08>*{2w6(FLibV%>M zQX)>JvPuhhDineh(qa-l{P+u&2HMtB{Zzod34C;8x1iiD3;g{jhI{9%#0^Tgh_W2G zqFWk`@`8Br8Q6C6wa)HobOz~(oN-__q(}9unwRo4#8e*Zy7(2fLm(R3@^SOC>8sHU zpK6Lz0p7N*T%C?dneG;Du;VzQhO2ZeOR&ZQs>|vuh&8oce&&+FTWr!uJsvrmzrAo( z|D#Be7&w2?|Bb*Zk!k(+eokYS24qM%f?h^0XBS&O>zrrKpe>C)Z8eOvlXw^G7E)lRT|4vc)R8KAv%<8^Uo*e zW*ohIf0;n{vxR2;K&{75y{ap48upgLy|8`Kv_lYuzAZ7Z|@L)<>Y{0$<|0iRX~ zYA|7Ow);IsGmVITc(OdlGlNX>bDfYphqdE~yb-xNFVebdirY|~6R?IKpAI8tTWGRJ zmMk!}m{StXQ3=v1Wyjd;%I1`=oQW*%TvLf^jPsH+xb5WC1Q>1VX?fzOrXGQ)38z1L ztWM5fE-)va?{N@1@^Q1qRCmH1m=-UX)mhDZUCW%@%w@Q?%U;JdVHS`0lOo8& z0nCt|9c3~;7$M)nZD&7;e{-rjHSEaI6ioARjx20@9g2;GLh=%$Yq6KxUUP}qAcd@c zeTQc#H&Q=&xx>z(`mjI?OXrW;k-zyTaXv94-W=4?e4P$%t49bbx=6yV^X!<-R1nw6 zpO9#5FDGs5$K4dV{u@1B85QTpbX=1nT5>C$=9suu;6HNn3hhH+V-cD zxp^Z=+W>b}0N}G$N=cQ}OZ2JztgcMNsQ#@iPo-=?I(vMp!fOs1P<@u`)2V#otDfL@-LKCo(njfr32^-gH9LygB^7E$ecTp=(Ic>bo&8sipEn*1 zCs>NaKVfyuzU^n;3q<4+#vlW^`Q^gLXy-mHRDG1=Ws|PqjN|HCqkEDoc`2)2?Lkcn zI}>fk<8Kz8Lo@JVe`#Ambqk31V`6j9Iw)o!UC+H5&Ap))@sVYxvvyYB(y0>j0`dx` z+A+=vM~zS0IkhH(F+MqOL~z27-n+h8k5WVZvv>mBG>f+`N;@9DFfLzt_FT3UJ|M_2;_AjdTK2EUc zxf#p?>>NhcFxX_QqDAU(wUkgV=qF~!G648yV%gg|x{9`{n$Bb*#G;(7`MJ)R>ri-+ z@8uKj>z*T<40@r)f*aJg_4M@Bn!+i^iYYfONA=&n=IA#$<^;e2J-^ zZg3B*u|zP*m-OOao_zZaw^<%*T9psm!_!OEu7>0JbEj-o8%k7FH)p1E+o^+yd0N9d zz)ex=$^^iH{=^Xol7GD7tC1C&#TWW4|CLcJ5$ldTfkA2>7HMTGVh!C2=<^~-nApxp zlQI|0l-uT;c^E?F{pT%+op9>UjZo>2zOl@!x{w)rn;5?!c&c>*V2mC3z+Nn6v&9PZ zTaWLdH*)}N!7vK_*BN#w#{7+O9>!>~gUL_?t=12+3^7i%VbbKxS%2Jh2obUiV$u~5 zgH#rPYp|qC#^Dk$99b>6GHh1b&ajSF~9;ZtN+P47sHw>+t5az*EHm@>2zb6ffF zlv%N?_THuGXxS3l4(y zrdhOd(GN0!p)N-P8Cd=?aC)lhLD}bsQnT~?I&Sb}yA{>M%dIDkRDv{zQTovDp;6k! z3Qk0d7AH3CHUJ7wG1T2;F2@wlVJ^&dZ}1zsmu zj^*ZeX6FCE+$y)g!W4~+)wsBNJ1nTDc-fhqn_Fl#3q{{srQk?>%0EV~l+x1)msmG< zi60`teGJml}jgy{EZL zU6Y~q|M=*|$#8_P;rA1J*yeKhr}A!J>5K}igSn*o*DuQfPRJW**y;MYnM2a~iY!(B zGQ_E#$x9_V1B6j{)wLlg`nD;>ICqlE3hvxktnhn{MZeMf<>kqa#Du$72TIPf*N4Y0 z9Te8HE3Ap0+%<+U)E>zm!5^kl4Felp`z`_&Bt}jl`FMmiet4 zN5D~lTPn_kVlKA05ThV{aKF+#xJ$p$M*7>(v*DY~)ClxI?)s=7q=brkB@v7_y12x} z&(yym=y&?&jn(#q&VS1=h9Z+%*6oT%rdCzd*W=7xw=1PUB>kLfY5l1?3j{)&sFtbO zgtIg>pAYWL`~S&N1Ae;wn{)tT`}Ht7!fPBOipKmMmdLd<12Mas_$;{+ag-*v+tCXf zXEhDNZS!9P304Nl1yOJ_Y|`Z|sIBJa5~9y&Ye(99=OFYz!}>rm<*o6=Su@<3S`3Ii zJ=N@De>N@RNiy=Y4OPMM;&TjHixrmQUg&@ELtXZp0K4u-&sQKPmvpQBh`<{7q4VWK~ z;*m(T1w3fVf?>g(Obw2oEb3g}j&y~)T`}7AdHZ@fGb00se331zNUk9WVt&|ali^O0 zQxeq8Vo~Y|pp6kV=33L@HoTzZC^dGY7a39FZ=)}uHi2ZEB_u3TzKgK~TP=}-d z_&7=pki!2bulB~;B?9tltEEp{A^tO_Jz_D{sW0AAQM_4$2`QzxVFkcNr*Eyb;2n42 zjrD@=RCe-`>|mJd+Nn@21-F&N$6*)Kg#MGw{-XMqBjUDZXH$VHBIVPeij-CQow~&% z9RVom>>flsR+G7aFZWsCZy2LEUzNN<;#a(bq7c8D3K*-<)FJ!5QS~v8z$md;kqD@3 zt3{t#^s76@*zd2y52P^~$>T{?zLEMUlO7Q~;>6DkI)_RP8K&9{5Xme?6*${d`;G$9ybW4#NM}wfiLjQ{ z{IVPGrdDTJZ?kPVd?S7t%SooHhU+lmvm=`Ei{}lL^eu7#X8m+4FC$%N*hW#zelos zXWJ=Z{rK0|fTwuY#(u*4Bhg}2@LDnY>;Mv1)^1Ud^;Bkv5}9NkWN27d++=w$kd)hD z8RsNw0$Fc0ViUojSdp9xiU~|(vW(=iAIW2(PcCzYrh;~if~F!VJ%7=-*`GK0hDpFS z81~=c7G*K0za$jdEz&1ec9=B~4Mw2-L8PG4;RdQae@`F!a2Q>a$iP{_Mk&V>y0h=yue7xqNpR z$o;1$oZDs>1$v%`u1N;9_U*W^wWtNsIVd6OYgrg(O-JD|^~VDMXJy=&p#f-k?||G^ z`@^anfHul6SJ<w zc2Nss|MKSJatETVc$%a4B>sjx30^Ci#>BzByD>CVuLn2JYn^IL#8`OjQv3+$_^@tc zS!oQ({ykYeBg#U?Ya`cg>>WNAa*+S!nqU^V5QSPqU86?l&pqq$gJB-l6IbMq7ZjQW z#h4*c?uQ++iZ3KtmAHn-ld5X+zjDzD7&Pf>xN-qcH5$5KZgNA;u#{n}1^@}?I4@%= z^Yto|#*o_`j@TK49rkjGe&@R`$0Ix#Dr!%Hf9AS87?+0}UsC45qaB$$v#6Mkw3>>> zou`(#fz6>} z07IXc2iwf#i*T%+^yNG3EaM_zi#=$t-wvJmJDHk*-TuO9*w*LDXsLX=NpJ@QaeHM) z@1k6Hv_ZFO31js2jXM^WpMhB2ChmW`K5dOHCLpqhNW3igLsGDQn;-IdOMVrJl(e$% zeFn1f%7NF4>c#2?(G>8#mzpji2lCQoHqVd^)0#{!tRYWImXFQIjc*}?T=wc1-%88{Frq+#{wp{DiN{#UkR*Yr zp7=Guy7wEmv|Ki2i(DE=n99E2TRbY|OWV5=_#=~U{%01Y{H&?yJl0U^Tu@ZX%m!-2 zyHt9`g>VTYP+{-O*Orl}iX`=oGJ#@y4t(7BUi`RS^gC8jc6wf1vj| z$(O>c4{|bxet#>FkDUVKF(0+r<~~ooAqO=q0-QM_64rrR{-JrtL>c>`VxkRmH_Q|c zLy{My6m}P~^}IQw3;BC#DrrRjf9nA-iSoC%QwWkzPMu^sHoZw*NLhiMvX(q?=f(Tf|Mosd9jo zWQ3*p@I(3xf^YSVJgL0Q5>}DY`WSXfm|7nm!>OYhGk!lvTZ;hmy1$p^|6v|S@z&e! z_ee~>uYg+p>RWEqO?`d>^f%do0Z6mVpkIAV5=ho{hS67sxGK1)L{A8;UzP*iSgxk= z-D>aR6gh~{6By4-A`phTKQy2{tbp*(xf|F)4o{wjHO08CS2Bq z;U<-?=WR7DY~2%NK{3HBeHMPL{wtFqH!~r9DIxfECR1UXhP3iAAv)uotkS0h$q0$V zgyNl&b@dmTFSaBWO$C4b3QI6M#N?L`1@j;8anQ@M`iUtI7+!9mQ1A(y5^u#OB)T2* z?~j9u%e-CSA@J+W=&-FvCRmJhFfp%4+~+P0m)vu7ZkBBhn&fI)L8+jQQKf%s7}TB= z|3!zs0;f`|AVl^L=_igKOw~a)r`?1S{^($&DT)X$ed2;Ig}h&+#7@yB`i``;BdHN5 z1OkzJ!u*Ee)WgDpp?y7>-D~!3bHYQSif%ZMv=pO}F<}`AhIYyXIq!-g{$_}k*GN*s z)JY>gdV6~{H-bOh>9~4`n*OVWr$f8+3+%=^nw6pF33&(KG4N8Q=)zaFFV7*jH%jTI zgz-{6BcFf23+l5ZZhQqBrd}N-z%E^FaeBm}o;Fa%VLqzt^Qd~1 z*1M_i6nfm%!tMbd$c#f?&{OID9?pAG{d+tw9P7Bffj7a(D0zHBnKKK6z$TZx<)lSjPV*!$3mo_!^-vMm&>%t^c+$af z?+iVUoC@YsMEiGk(3iIG$8lrOThu*s)~MVPI9`<(UA~)j&vP4ju|)|aIU|6mfUu;L zC_0x%!I9o5$e$>X@R1R6ZABRGalbcC<=Rws@p% z$C_Y}P?m&kJ6-LRVCn9->(7*7i${|(1I^DD9A-`8buSTwKqcSyo(=8x7-j%KCA4v- zK11KJi5-Jl0)Z(=rErln>_{}c!kg930$_vCX5V^#_t#_LScNrj2uL>HRh!kD8M#8D z@@njSG(x_fL`!n54)H6E^8cCZe?-5L#e(vcieFFjStRQ^UE-S1bK~y#i*Wl?sa1nl ziupwrsg)NrQ)cpv$!RPeoDEpa@!ZT=iY{xC`WA85G*!xJ%3{5Wg~53RGuJkSuaN{u zI@`=q#}R$vxlc1`7H?sdY(26T-#?#yr32R;aet7{(Fz$O&1t6BsC(_=&e&+_x0}r} z#!H`*L}yU9983{0F|vl)ryTiW48-?)^fF%JsgCM575e+_k_S}JY-k5~B7U$;C~YIE z0{-YZ3Lz_zRDSs`s*Ks=aGTEYPw-l9dz`x1?S7+{}bLjG>0XLRI4K<643> zVLxO)_6r;KrOuhb>eTGCmG#JJ8mY1OoNVq3ry5xbwB!z}2KnLte8ISgo%&8#mZ*g!%ESBHR%PuUg0q|m^zc{)b)+a%e*R&m}Ir%h3T?}?*>** zMVhR8-6JZnK1-++g<{ETgLyoiOKxm@etMUPx;dh+Xbdf@HT+=qX7%^EBh;CgX#n#RGzokD>{!hhIl zr{J!vTsd)Sk-I4-(Cfgz{L>n4uqI1NlrG^&Ur9o2UHnWRx3S;%Mxz1u|IkyVl0E~Z zjkv)C32}1ju>9$nG7WO!>%*St2U}zB3c#)I*l{6cX((CmH}~gjRxd5nKS7wYh^u^c zUe_#c%$~ITh#iP*fG>?ZOy-K;A;*Of=4jv8-m^1e9o6<}8~HtHH6nQ+3J+k#R)4D;5(?;!v;I{KFl(Ztb?JW>?z~fUC zPfPE5x;e#|><03Orjs2ct3Qz~ zVsPlp&?#jG`BSB(_UqC^VBcf>PDthICHDFWM&S{6!nxCv>1Ib*9P)auKsI*O*VmW6 zEQ&?h_!}GZhdYhXXB76FF#Aor*q3Z7DWy>?*>}$&54Ry`)|uO8E4+)sD-Y|sBYd&y z$D=1qsi&!xF*%T1>(I~rSGEK2&h!CEk0-B{uP6uB>x~87e|s`dJH}-CjkaxBFujc2 zrq|!@zs!-T=a>)b77s`HLqj8x==vlfK2fqiYi7tF-eYt{Oz|+f`nx$&cBPEm9wX zH<{yQ<1NgHb6KP(iHlKq)1n0>;(1BM9j0igEM)aN+q9S#sb1z;gPVO68E*L9Ru$y- z3uKI(|1Bv-CfjpJ+&BbEi|sGD;t`+5_v{=O?z6PghFEnH&W?FQ)(dG&8xZP->(E3a zg$I#hY1_QA7Vn%EUi|9w^Yq7|t@1#u=s0FeZXGo`#&at1PdM%^IesYorn=@5DM?nN zvUR!x4R+b-FLl{CE%=P)v>=0AiV?fJM(Si(pL3Rs?p^#{*@wjzkpS*%szYa71d`o> zYl73&@gP$)8+LWvvJ_VDlIxIa002zcWv@4Um!J0m=leBVnuIf+^3=o9y?J^*#2Bl_$)c$sEogX&ta?Wc(4X-6 zirjL;w9Z6hI)B=%lA$@a>&*EaESax77#<-1*0)lw@eh5KT;o8+9#~xor7@yD$B@VpA;BK$*NVf_&p#1lG?> z-lszDy4+}@WB1Lo$Yb(2Yxxn{i!F@ikGBUQx8_F3XGw8=$hs;(Qdw~!DeNV8AGXG} zkY5y2JStr_F5~0^`BiFLZj1$h+3;71#5k<*X*wre{F#5rHl6`PNRr)AIQ+D`vpDdm z86I4}BptLZK3{)j0d?J(6fZjJaK1nkGN}rSJ>L%5IJ)hp#$MlFA9cA(Kr)#JPV8Cl zEnNr`11BTf*dmJ@u|ZRKBw*A7zvD(ov(q!eg11X_xBDS~?)5}j%re`7qF`<^ArSs)2PL{rYGSnpm5Eg4VK#%7s8eAu;t|&%MbMJMrLOUOr_F3_i zM}|&?HV7=3!fVJfxryO)2D{bS1v9!|p(=UHgttVT}+gqXI-(w<63}FRWJOc;Oj~bPjKLV{hjvu+nDv9`mV@7z$*}6$PS?0o$CCr7GIa_KsO-a-ia} znxwYSJ~~@=wBKs?q_BQI!NwI!49k;5pM4sN6zM8V^M{24Ta6t3G9DcpP7|zUi#LvM zYlM8HtX_<{?LF^qPw_dVlV zsGGeqJ_>R$^2Tw+`*Z(53)Z|v_M%57(2D2)7*V*N9G6lG#a7F{@Ac7u7W0eo`M8wX zmZNLxLh;PTL180W^E54wL!Ssp?9{SCI8uonX?b~ZvE)$P@Jm>&O^IegDD-5je(2i& zifRR2^f!3)7n-nt81g23L+?C4J?IMYK^&lA2Lq&`d4Qzs$BAmu57z~WHtK@xe|oS0 zJ;9$|gmT_n9YAYT4P^jMYS7BVut>&o#QN}$zH153KAYul8m`g;R<~w3krst6!TYB5 zjz5X}BZX`9vA>WVp4qX~N&lMdUiGI4T+747Ldsh57U+Knvne?ge-Gj!HYHf;p4iVA z)%DY^ktGvr@<3j9n)DuW0}A7mz3ABzi=JbgYC^>|OoA8QCIfJsZDFtT8KQ}*jmHrK z<5t1p@^N(m@4g`vl2*#hWyv_5NrI8?_nws&PwUlvm=hHF99Gm(VLM;lam^{WmC|*W z7aziXZ8m{SfGm6Yaa`y+aY?zUTeTeVbH(b7M_ICGw*|ucb1R;ibx}_qEJqDu&ej-< z)ysKD|CqpflYW``l6Oi>z>Z1`QSLUOx68YwAk(mfH!HN}i#N`1&<%+OgxLfD=Pf~p zf2I<|az)wN-rN}Zz5RyhR}WFhA^tNNnYnV_=XWyvQO6p1{kL%r$VKnxo0J5YYalbF zU2Z^UdvN_|oGqBUq9L->2xFuFPFiLj8dLAtBOd_}P#(JWATAWnUv-ImGp^eDi7t}fq;Ar<7$xiNc>T!={w-lSrnY#BAwk>ZyTM#|tJ>?9guPJ=4>eT_1aW#Nd{HH^mIBa`*Y9fS+DR=DSxFh%a``11H?e!DBkHGW2 zXzR_^Bzmy7XqmK@FFxUHSsRc#4ORsI?7uUcl>L(WtnhIJmP0Q7ZmDU8vaHs6QzBFvLsCUGJ^P!lE0qiKTS^%fMXFRb^7M|vwv9V z(r>2c<2aX_TTCcY?aGHXlTX(iqLy91@(<>2VwyF551xvc9Nu*8xzwI>{U+ekpF0~_ z`Yd9Ve4a~o%wy31BI`K5WmrotMJ-Zl1;|_e9Y>1t{-|g>&`AS%WZ^|#M=aGzVl}w+ zHn_wgeTamNWUO;pYc3dyI+`*SHOteRB5oTDd(soK7#amqu{u3u-1r?rTW%m>`Uw-m z`0&J7c)1sW=K=mcS@cU`%N;;DAO-)91$_9sCS3$aX-fERIy+gofGyBB?e6zIhmTlI zAi~6ud0S$cE(d6JM-imx&b?BTUh4bc@zw=he>%4{p3KislP~!E2{D-$$9OIjN9_8G zZ|YG7NtESlJaMbH5a^t1gUY{~1bq0b8#ggX(G;Y`FZ@a7 zcr&EXSL29WZ1+ z^M60tJ>Cj%C;!!l!Jd0<$AcABhg0n?UYLR;u2HKAZgw{|kY8Gc@c^PXiFfdIXTZGK zoe=afhw&CiET^hiaZPrWDoKo2L$*j2yX(ngFKx~FK12%N#qzO=tsLAN(cbi$-oMAN zhBIup^*&F#e8UP!_`kB8ZZEBV0uH}0efBMI4CkWy8HHdyr;eWUd*}tn4jrv0ap|vQ zV!3l!OPUQq9|v?xVT~!rDOHQda9&Gxv#~LN(YQTZrcvU3v8+BAjIn+`x}aU9pC`TS zx+Re$uDN=TBcdTl|F8PvRtN=pmghqYvqiice};Oq5EC5mCj17*ZEg&~g@hgg*RXMt zfd{rYW7!W-@a<6_*A>!g0c_1C+9)H}IDNcX%%iI!d}6YGhwon=eoAyOS!%bHI=$~A z`qz?36$9%H-~D_c-cbu2(WdlB!lC>7B1CW@De3H`OFtbhnCpN$;03ZofhKt^>2jb^ zO*G;q5a@RRj|m%jI6suohlWw7f;{P!)D%_#NA|D${?U6i3fxBq&Urr-dL7q7y_|t) zvYK6|B_S2$CTj(DPSxwuv^b%z|BybRIU`uaS&Vr*(eZu>n}($veLbKj>v~+T8zz?a z(6YAF4--uMo@;0Lz9$^8`u`c|mnmQs@n&(4dVJ8uy6%4esH0zResO+u8 zdOv-etR|sds0RAeTp9l)eq*0f|3;+OGowS7{8>)3_1`vZz$5%~I3%zXXIi9Jm{M6p zyiS_G8&*aE8}TE!8D!N^8v&nK>VHGBY!#iY#cIGVh;6uFG5SVEzd3BU6xhV7X2*v; z%9{Sc)uH<&Sk?&eJr0ZGbY%_kNu{HS(tZ^rq*oTES}9*2rdjEZCvE#C-qKFpm2vKM z-z4v13-mD}VOgdE0To<&Q z#uy=@=dNyDBkj(#KcehXYaCaNG5mEf_c3pP3nw8>3y~hdu7sywBW?HF7~0w{+LU}n zvuA|U6*cYBS%QwJLnT5Yca=D-Kfc7_#SkPEFs#0fN|5*q97_EK4$-q-=@L+>zk)h9 zr``jHKDsVa3G2VYXlK_;>GpS~I18zSv?HR-2Y3c}^afNC9oJDTa`^QkGVx2e>3Ld| z{WO2+Wj+#9>%c^f$SM0Nt7OXPr$(T6jpA1n2zv2I>`n&V{}xa{wW|L_K`SgSRy(ZM z?1&AxkZ0+b79jv8#tAKtM47YKPO(_hBnd5j)a_J$CV~tV!;R6H&NCz@fE0_T1o*yw2@NIHq$Gtun^0_6vOx~7BO+;h98f@VF zY2AY>^;`u|-8b_*0Q3ZOMp=j%{UecD7SYr#X_bPmcC_df4|@_VoF}fqh)$vYgC^W} z_Iq?WCaPTmX1(@-ma;1a`uDKHKQJU&H*iZq)jE1Jv5xzT>%-fFy*;6BMbRu^mRZW| zM(U|KXxC(d{u&1QxYeY0d#+Qy-udemkc3MS4$sdw zxdGBO-a4Ot;m5Z8fewHp*}c^G(7xui0qx!YgIpRn7}1+X5 zw^HG}Bn58syVVC5{*V~la3%ucSE5_pef~%G>zCPg7CV)@kKn25_Td z+a8BHhFJ66JP=lUkDmOUAf&qnc4zYYj+`hXbvP>Kb@{;h`Lu~26|N$2c8&=h5IE5Q z^`&onf9p#Y>7c7osu4mhk_Xb7eB7zxzVg59j*-aLNU8;$R#!M@4As`iKP!(D0uP#m zSzX-)XWg-jM#*VbU-+Ip=*%cSJi!K5Zu-Ce^Dpe(nT`P{D9h_6)?whl|KqdRPWDN| z*X2}Rp}-iLvB1Iyvlh58(rxcL9Z`cZQHBVBeK-bcu?eXqf5ZQK8qaP8oYTH$%Ec^^ z!*0|suaV2<9KyAF^{2eF$!o`aM$ylNRwl`(R2JzMJyy1o$th_(PM?DIF*tImd5N-|-Sd5kGvf5tsy5{~+rA{w2^_8hw0pWW8Etz_$x4!3od; zaVId}!}jiZ0Vmi<6*O^I4(V&6t2^}R+ngZ>MF*Kz&dh%F6uZlQn*9O2_|3OVpY|_eVw*4E!WW z4$Pulj{T^S>+B8(SSwH`az%BC%Rar&ME^401E@8L0Sv$ez&H%|wzv6$NQZM3nMA!xJJkZXm48UO{|Ey$Fxe*`Xf!PVQU-VU zO1seM&eY>?O!OH;NwWeR;0r^xt>N$BWvJWUc+%HAeur$(HXO~eT~+93=Try~Xi}_+ zr6KPM-1g=Ku00I)K%I``T`qM&Y5o3n;-2(hsu7D#Y-R_U8b<<~KW}=sk##83gpeda zYJZK5`15j1oODpww+OGK!3>m*d>~#3!j)+5=2=#(uCy6-{cGPH`zl0oHl(sc*zdYZ z6%O9KR^mVZ>ELv=-=u-u8iUu4fC?(Ap%IjxKOb5gwv4c4k{jkwO}4mfcOG&z29|sh z_Wf;$`gL=IcmtThAs--7Bj7TIe+ifrVg0N=oBQw9<*stO8 z1tsEYD{vQKwWuO@&MKlhNm}^|r{;=S&3DO+q@TFUxMa&_zH;fe7CTn7%vInYmY`J}Uhgp)>FtTk(`eJaToqx5b%|}GW+y*eh zl6mf3Nhj*fBQG-zE-qW5b8=)a^>)W5CcWE)Q%a^rjagmr4>=XnHihp#Uxr0U&9bKY z+*#0-{p~Am$e`wH9@%E}yxZt{C}O-nfIp7$k2(X}B!0nz8@yP48xN4IG0omO_Bqh$ zpxh0VA07#hCi7R5JX?!cHP4)z83h4^Ll(akea?xaI^OJ|hHS~Y&5@((p6Ny!P99~<%YzdNBEBdPviPP*v#d^p(rch7R8)~~?l?QFkS zfo5g|>-NUJr6(yJR!neNt6gxqEI=msM`s*3?xyQ#YvKT*;F6S`L&qJalG3znDfd?KIwrMi;wR0B&)W8J=XFOarc3{*HtYIE z_4-CXr;shwHlbAe@3kdNWBj1Wm<7AZf*Y(|&f#_^PSM_6**_$%r(iU&ORV@<4QJ_5 z9~#&zT6})RcCx1(+shQnD`yhcY;SBGaB{S&|8CNpZTH6b`)Dp^`p!m!mEEL>DB5Y3 zwdzl04kH`Pe@?vTq-)FP7R|pF=~Fq+C)XCU(+4`W`IR8-wrf74Y_ymCqqKP8<8eS> zckpu?S0>U+^dTE!eY0YWg_0Oyj5@tphB9511JKF+N2UHU8jEDaO^H~wDG!Q9XeT=I zu!-h<*ww4Ob2ZcYSm(vwO@3^7Z`s!M=G*1PCHYKc($$TDp?f=U>48;X|8hrAIuL&T zN654en&$`0Iorc9j%><}h%q^?mfGV~csC)`;Nn70+oD<_1w;>7STW%~x6Y?!~;v<3<=r|RtBc#;$2GZ4^KWv+<9wR6*ewhSvPWSKsw zI#x;{r<{gNuK$A1V|i^UEt72)H$qV(8e)rC-6~RWJnyQn$}n6Rk4f!S1CghGpf1FF;tSS;PqJVXyDF)t&S}$=d+7kS^z3mYWn0H98 ztMz>KvV~#UQyEO+>xgmWkdG){r3OG-S>LWVPXU3SnnEA=S?V0mY&a!fkm#%}Yi3nh zOXb@D4sSMwF!iiR1q*tdtiJhKJ`KcLGk5idEDA4`YDfFzLScbHJUfz&Zzt;Uji0G%g2KreaRYnls;t-DoKq!)aNEMW4;)!(tEu@Jr}XqpIO)6gKG0{!r^%bs z1YC4LjWHVx1haKhdKn-;KR;sUV4FNG$cDT&CEKb^HBq=!gC=0p{f0lpq1BzNUbCoh z=m4x2pu>@nei3v_184m|`OfnjeRp~af1+;$Z>I?Z327(>}>$~haMags5%1rnp zm&aGF`;6_&kZm{-8~)3sZJlB{2pw9NMi~%SNQa%L-J8K(sxvg5*LAZ z+aG&=^MY*{LxvV{FZQQc&w3@SXLOt7AvPbyZ!8n**@4S4P^qaMy4(Y+p<3Im%}L@8 zER)5-B00CwHiO+kfl~>iH7tXz7c?CvkB9AxHPfR^4N+{GR77 z{|F65mTwhviXE~;G z3N99s0o?~?>zt$P3CtB-ykON&KoEUJM#$6aympUv0=FVbcdH&;AAA{jS8ns>z!|mf zewDY3K#7jv5&U)(<)6->*=M7Vd!;{duxCSVY$t$j1t%|M7RE*4)AW@&o*atG&JDdR zs*or8?YW=#K@O0vNzcNX6?Fa4rS12VA8W9@-j|6l_mjK!jd_7#L}=~2IvGG$jES(` z_S0(USgvxH1YRBqh8dYvy*sqyN%6u%M-v(Ja zzo!go@L{7Jk55C?Sd7r@gW(29q%!+oUEt(rV&hT*so6~tM2fzQzZ)jrtT%cf;Im>X(>$#Z(%+3K)OIdicfmb2^gUHh4~t&Z z_TIv5lI{bbqt^J=d=b5}S1(k0)3{nXPRh-(K9ilaYLCbM{!t;i%OmddM&e1zJ1lK! zt4qC;rLVa|fWS{et%1BF!*4&9t`OO>ks)}lOWGdPwF*hMR6EDK&LXB1oDDQro?8@# zh|eh3&0lZdsOldyTllQaXy1z3?XpWB@a>QA%#)XFHx*rN6rT-f@271Hhr9)2mCFn+ z0I#J&X%5V^9|I*JeXrHOGnoxesasO&)@@u)j}K^{y6RCrRJH>BF++qZwqhcLolp0? z^jXY1<9br4Ndlw~0#-zQkF2b!qr~=gcrw2AoU$9}#+4j-$C_+8I3_?yEs*UelLjR^ zHo9l5;j!CKqP^2C-)Rk0x@p^lSOR=U*phDyq%miGY2xeyDFHJ3)GwlPR?l=W%@|Qg z`F9S*7m7cX7J6TZk(fNI>Os<0iw%t37Ok?QYM~AmTfRmA_eOooTy6G) z_$YWrx#X7I z!11*mLkonO;0g{VrLqW(tEtyJ?NA&!B>R$ly4ImN0~g={9mkHv8~2~{H{4#J_W=6S ze;;@`xN9`n@Y{1hEuuDoE=q{>;vVX9Nh_E&O1VOb%Ey-#$V9-!NQ9g4pa4IEO*R(?8r;A7Zt3-_LYa4^c zA6)w#RF7i}y3~CGlS|h!8s>2(o+-K7D2^`F9ts2Rt`!8ooDnMCrh`aR$c{weTA*7k9?_-+}Ev4X>~0e4WkIyc*){33sP^@lWUs7a|x4a zdLh#WUVQ|txP#hvFEj+7ZjEOle7-!aa{p`l-K_vBX1nR`XXQf4<4t<^G5qCbI@t=QP{LC{xQqQ zJ41_@g;m%yzcq6rQ4OPg_x8VS#GS=~Udt-0DxbB0m*d60!;Q1hoKQ1mnQpI|=`&A_ zEO28(E`fGe!#nWim{Pc*`r^1BvZb-ao7Bg*ApE(7Ee5ajDh|=8U3`!VMkAUn`mJpJ zu?JH&cBAJ$@0TnJey#K@p^RM&4Y|B+UC*74C@TOA= z`L@Xkb=_}l5iZ@0e06_<6IztMG<2dQ7rXh(FGrSs)ev;wDeNS31T@Sh!LOIesu5Gd zd6y@=Fnmz0F;C{k40 z%vNItR1^$n4l9#gXiIPhRKJ_{Jkm?}A#0`qd0_mxe=!jmp&5L-!ZAnJ)pZoj@L>qW z-OW`{@T~xAa*J>GLIrJ=e?Y!F-izzz#NL7TbkIYwm*l@Py0Bi0eM_yo*V$Z)dQo0vZY;y(%!Zg7<|_BG!}M$;DE%tK{G?KTWgx z@!ej2Y8LU>XfFJfQW|wbd6xL+17ktRJC~_N{!_=entr@Y+Ey$8i2RoOYw)|nFScD{ z5SNEwD1=t|ii9C>|E!963k!zNoro&L-cx*XXZjN4AI%roxOvf8#1?Pv{WVKIXuq^h zPM2{g>303Q9GsFRxjD?S>gZZ($C(u>KO~EeUdTb(#v#KIJ`ka^6_juIViS2*1#bLA zRuTVwEW%U7;pJ~a;bV7e67^^JlRG!CKkV|_(!G54nY0gY4b^=6p;n4g0`XU{#ge?8 zq4qXm^ld${`#>HYLKjz^g^pC0#ImZ8!B5`_ zR+>cNd`-4fS;oSpQ#RF+QiR0_1k&{8si8N2lq*H03A4VaQ~tjHP_p!`j3+!&NwSEp z=w6$$f9P-D&yuR?3<&ntubJ!-l#&T#lHci0X=T*gK2W|(|NP`Z8vp3q3_2k22ZCL9 z{!F45(wxazT9ke7vp!Jv}Novm%zd@k;8EfVU)`ve>*Wv_DMeG3tMeos3;Xit{;vshYqcr_eFU(S<{;R z(FTkKbRK0d>cTiu_lkgc(&gj2S;X%xP%MYqGvxGSYixCcdrz(fHlKa^ri zh!kouSTDNX?Yq^~?fveaQ78mW=_|Rx!#OxMept}beB2%@AAqs&zE<*SafLx!oYq&5 zHe+316{&2wyh7`&4E)@5XrepYzu~@XYKfS?l(HT=(Ef(W-87Mg;&5Dj>M?5kFNSyx zStDroi;R`-sdgN|b?mcsy?O{Q$;3P-9o{A$Rz0zP)Mzf87CAmLWL$b9-+*w7_ISLaK$uf9V$@#`xoAS+0A?Myxj5QNJ z=<5((R|#%64YBP#SWbwe9(w@6_34>wo{J;Zsz>FTv*GNMqRx6e)(czr2F2*~Pdz~% z+E>3`ozECoH$Mn7P(Xp(Z{PQNl~r%qs-mqtzDAB*fns=ZMeyZ=R$$-0CFlBB@su&> zsi+wv+d;RVUZdl0XHiu84^M%NBi!4(cqv+qYgEI0gV)@`!hHTF7&{s5WmD$n)4UMx zU8R>MB!94T3%tA0hdBdOD)bTD*5QkokMK|J4b&=xJX0AAXo{7#HHu)5B?=$2J{+`& z1VedWx;yzg+4B4Eq07HqqTI9mqv$UZ zG80BJ(G`Znh-2uW%5CTcblpttVv-E}C=ZomaQ-j`^Du_I=woxZ`96eJ zYXS#e5+FJ5yZAKce4G$=UPB0543C6upAou;cAo!ActaX4IZhw6YtPMT5zWC~RCe(# zh5&Kzz@c%#y~%5HqRi-t!zlUJGl$GSlPl_6Z&h!P$sDTwb%ft%roC5EL!z@PEBU-h zdkf2T`77*a?OK^7hM&}`I zXZwq6ThhdED6=xF9(bN~cn(!Pm=c)?9Z$}!{|lqaWKqh4B^7TNuDI+{C1XxsMu{-G zr69l^36ve->K_sFx5UwQJ(MojE9C=wv?b@FSAo@Ca8~-AD=pBrwSwyrWhyDNYPn3<@4G{(6m+`BrSM&Fa2Y2yLP#g zLhUL<7-r4+rPh=uZM}!|sd9yE`?_J~c|kGF$6AfIi8SM@K~?ya&0s`+@wz$u*hO#p zU8kq-(-8fRfu{V>XIE~hNBC!)>lrcE{pN)ODWF zEWh3ILd^lF;oP`h2-!~l@<{m%Xd411cna*hE(QaB>ZTXt14&#xNYeV>^^YM!_s{jH z`n*KRST%z_OPGByAZPmAq`IAvdvbZ`E$}twMGS|J$1i^K-hluf3(d%oDwLt3Y)=jF znuUI`qw14QoP}JUu1H^QZUHPNJC}Z`rVBe=Bu_X*rCYAVzg7Q^UqGSTazyxUR(jx2 zWW=FwTqAwMcWdnoEzBJKMTfxrwQb6n%goQI#YTf00V=udA2;j+@xpfFHPX5nDcd zw$V)`Rm4Nz_do3o4(RN%(RXu1gEd3!7}&u(Uy2O+{1^`)V15qY=uAu(U}sz7OvJOQuYv+SwrTj8~ib5#>a+znpYW(EeS-M5_}aW^Yma zPU3wqgauyLxVPN$rzC5vMR^34!GlFo94iZ6p9zq0P2>{GP{1hEU8f=vFFsHWT=Di{ zg1i+%fi@kupzjzfmQBGrGpKd{Lcf3B#beE_zjK41M5}1Nf%VOAYs5XFHFoAlWal?k zg0@a971`LQd~xCC!dELkS3n26UVx>7RlLO>enz#3Y@saq4QmxC4K?OoXgoMIys>Vp zYS|Yr1O#Lyh2eTra~hzX8OUDi6v6kEMG_zEIe4(_A02I( zf*Ye6FSJ(tG{G3Le|?YZ4+JnvQX(nh_&BWQEgCqO4c=1sMkwQ6b7v+W}d)gw*Ery6HM)C;CZM< z)rr3LRS#L#)a!D{*^d(al z-PRvJnP+{i8qxd7i6vCUSqVF!O^D#TK(B#pr_v0%SHE{u;k)*`sqrMG#UU%j(erPq zBH6vqvy;>LXFes|>$`?Gx0yV6T9F@SV)XrKz1dPoPfFNRiMMb6mI&e_E?f0mnu?u+ z1sVL@$G8A^T>t(2jkm+kx>lifSD~vIyDieR@jpKR&lkCZ=XpwSwUM6UO0z!sWjA;t z98f#gwK&4(cu{P2T=k&jrw4>~S ziQ$luLHTW>gN~L`GM_vi`*kj+`w2MFknYY2Cm*}Q@cjM+>`c#uPonk@yvgZV>ZQE9 zv8B^3fqB7Gem}kM!Zrx_q!$6HI@hC~m5j0>qEah9QqGzf=B)p&eeswg(!4(}o~aNe zdHI(l2kuIeX-ys!9*Av6uc~Oj12( zzVhp$?t{iS=ps4!nYIF0`7^P0N(LM$@;8>aEG2|+^ti;5+r2RdV__DvY~R{v>~sdf zhT3TP)QIYj+LWY$ImU?ECyiJGxQk(Fvb2ZOqdrKN+ZEw(lFw|KIU&R{q-!R{HMrka zO>E~GFSm}$_I?o;)qtp+U`JWm-&$oLdUknua?oV(7m)y&pf1)adffn(iWOn>Hg;?@ zu_S^;w%z=^8sUW6HV+zh&Rn5&f_@*`OE14OV^NH1u`RkH=@1e)uBq4U!tJPY7J$Qf zcWc@UbWURIh6jOhxp35)d~6vg9-)sXT^pBmioMN zqhNJx!KTLZ)&K)U zUQ1gN=^2(eyMq45I?G~T_Za1X7l!grQ;iX{*2Ge}4l@ryJ}huv?3U9YqXmi+{|B6- z|9P%+X^#->85f_26crw%q#`{e@BdpHCGbJR{yKeM#HY!WVooceXg9057>W5FBp-{~ z!1DPr=>Mt+VkhN+JX`e>!4I@I6!iFTv^G{TK`p5Mgh3YlDmH4vL)~?NC^{q#7{QLvmarXeiL!at22%Z zM+zS^;d{6JWy45A!%)Ce-@+c6LiTdnD=g+xcOhr;pW?y(D&QqvzCr(__v zU8a=*tiDCaFfZ*Y+Oa8avYxccO(Qp*WjGPChBjz{VG=`3zgn`^?F2SZv$;WL!{EBh zZ24|yyBeGx-k`+Xxx4D&<;WhB7{O3^D2e6u{4-wv_p2KrK|Ws_d0-*TlLByN&~zA~=gN;$%sNd&cG9e0tWb&7&b`4D)Iak9%4o!sj@=rnt~%9G3jc)gyx zJa~^m)modAUWjMm*r?Ea@*sK6v)L9deYJVQo~fHs9PZ<2h^WOplL8ALML>bX^HJ*< zNQ}_~O?du^oc?Kx?Qv`5D~f!OxK@ZsUL5)c#sjo*K2r5Dc?6Ce#gX0B71md$&?g0eCS=_uQR z&0j{YJF(_}MMJTZp&+m*v8Fl&q_uaYtxQg|ROgk3Pro-zYLR)}5WBU@m#4h+-Rk9= zwSFfP(q|QtSj@-J&X{eQj&9@ejTfg}?vW*`OP`aaqbq|^_~&{`SpV>WDctg1hNA(T z9IZqY%OTF}@|+$Q_fYmU@L!4=)lM5EhT-~-M(b<=;| zF3@bh>-v?`W5D(MZ0`9t>qbE38RL@&Kn_|}M$q%N)_+86jg8vE+mOahYS(_VS?`D& zYpp{rtwGa+Rr)$4wuXcphRj5qpy?qJsu2n`zOCz0R6!IYQ7i{*-~x~N;fgHCW51r* z?xX%=Fl|gZdO{BC69!&^W4)Hur=LAi#A-yc2YP}NeT@w2tY`q|xR7~%(j9AL-1}aW z4FqO`&~a<-_bX8W$SzlfF;-=L?8XH{vx!#61u}j{H(65eE1rbZ2W<#3OEODMt2o+P zQ^jLGv*p$AZdug}7gzP`4gut95gWr+@ctmOS7rjs7Kr1xv~j)P;|H^c!&g5~$6RqO zZz$&WTt6ao29AO0*AL}8(TU-A6pc@VI{D21@A*GJ5xruvS*%3rh7HXL{AeGDSC9 z9pZ8Sae4-vu}n2=t${R2+%QYghjRBRF=y;%bv_Q>6CsgO`^kA~#D~T1U*U*1(a}VM z6DRZA&!|p&58D@0`z)uyLu)?4PM3^g@}$Xt{0#jJ3tDA6<99KeP7FuTH1wy?%u|k@ z?2ACh{|JB;ECdGM9FL`k~7MhXNN4rOdk(aMZhL>nCuz`0xEO4wAk67MX3- z8P<1oy5si3P$B+p#^rP12=P%w2}masHx93_?^iwp-<4S2&PdjV{c(+ojxN7KpXe82 zWOi^`xgPbd6pp>*3Qr%J+^Ey($tvRTG~ilOZ}#-m-TDZz%6Al9L!)2(gDIRvt?S3( z;YFP`X3e~a^=<3Nfqw&E1;M&W&WS#f`9Rd?r~X(K?B>ja|DJt0|9ge*4dC1xEEP`X$uEpJLr1pHWu^Y($Y{2_+ zg@}qaTgKJI)x=-@M=JTBKaw3KG~JlTIIAe+5%GoHy%Y$655o~tShm-n;P6c8H8kY4 z%w6!SRzc>-E=DYkq_x5R7&Hl6=(z6yQI$B)5>RnD;6NGp%7NL- z|E^B`C#+s|cU*nV6@5z!jnFXvc%B^CyyU+?qloWGEq+9I{$%w|@?gOQ>IBNG8(=~~ zlMTSQ{`Nl>XzC{a^DLlV@we#k%L#$&E{85+hdLb7iHeGnH~!c7%1&EGNc!%7K08#n zke>N?{t8M@%dxfm4E8RR0@UfFng23hAwp4h{JMy?d!9AfxHe zKjDBj$OAP=a|DY_rVpqLmq4Qar)s@BDDUs?80S<6ffv@kg>0FmqoV1uf1Fk&fe;KE zUOTq${E^J(0SpPZz2yFk=~V%uj{l^bd?IA)JAQ;u3?y^$Hx|tO)BpMHfSzUj6UgKL z$)5lId5IHU_Q`!{d=Y3j8D#)P=!hr$-LSnlk^EGBWHV6bV~JbkFv!O)c_n4UUmaa& zZ_?7UW3WPJdFH5jAWcqp?j<6TRS&gxU>lU8*wu*> z=l)JGXH7+6pL2jwKb4vt`aYakX)HYG#g;0Vg2xs`1~Kie8ATl-Bkbi#w^?q-303#E z?7CN7z^TsQ1gg1z*(DK`a*o_cMK!JW;FUAdd9`%t?x}XNz*9wfT|~@kMh4tNuM5`z>j=pD=D)H*+izpeO83p2m6jxq%T&Hz zZ-ydCZ}whINp_Xk)Qg0(MAwY!}^}1jT)P|IdNuHYmA^@dO%SNq1Vu zqx65WYeTR%O%XzNUVr>lrm>$!uZ zC#8meyY=T_zMTPp_)N9oQIseCc=v`O>VyGt_7%7&@7Y?Zks0rhUIf^-$e5 zwF9G0k15p^ou}L(#xm!WVNSs?N7;j&z!F`iGQU zc6)T@!`^VPj&WWF^qVOEi6&ivBkY&+TZ-$$ENmKQ3*sd*bM|LcBV~$wuERCLGr&)2 zq}H4e`x-`J)%E3A3u!#rpg4NMQ-5ygdXYSSXTPm9?%BA$`pQOSV=dX48ZVVUFyo;K zzkKY8LH)+{(UbSU!96?LjaLQU_xY~H%Uv37GxoCh`^IqXTf(ZDXseP8=~U0})=f9+ zhyr;_!RCbv=9}+8-yLMNrFNDH6iTb^&97-PPt>A0p;Gb}uUEYGhpK^9Y@zd~j2!vI z_eUhGW_|svprj_)V<`aln%r|}1VBSK5_O}*_qu`iK(zk(TWaJS+?yY9vt(N{#|7$5 z*`-taNs`$CaV@gU4@4EeyA0b6+nvj@KP)c4l22N_s(qfVNpU938ajThU;ffFpTXdT z*Q-|i=g%zEqKY(*=b6|t3^_Rdd= zb_PaE^NLVb^DQE6BMqleq$UU+v5F2(0XW3qO>6BoNHQkg;kQhz3x%?#=@g_ zf}8U;Yh@NZP=#x|VBQ?+j2J?K*QCEBrGM~TTJf9C@s~UTtuCxlIV)bMzET?}W;{KE zbi5El+(od9{|m)hCRO*7~I35qo1 z^$9XGSDr58ZnBgme>dN0Q9wZxr>;`Qof^~6L1 z_0q>gNvR^k`cNuzyGr^B|K#|CVCV;@$LHFn-;-PB5E6`6L~ny{1W3BBi&++4D+JT= zkJCkq>tOZCs`v}M)$m`_-goZL@NR~-yHwilFuUG_)|l(Pd+9Y^&Q##tI)Y|S;eNlO z%Rm{;{(MR!LDQCf{B2Tux{zc8OZ&z34vV~@ozH41L)Y7Mx<89w7*?>nOrbKuuBAlg z()i|K=ELE{v9=9y&3-|$5iZwXR{a@zx|AiW+uz#4n}f2u)F{{XG8ldkF%n&ni26oe zd%To?dcHyAP~OWog3Rg3zZH;FIa`k4pi6*rbX^OFy3V;pL^ABS_nNzuU0LO@Ai#wkq4eli1;@N-_We z7wZiF+EBYa8#&>#^nB2#RprfVDiuxMh=qaMOQHcrrh3(MqNLs$o!gmJc2{b$u~dIJ zjOAik*-bw`A!B>xBCLh@B!`=GqEkEDzjd~3|8#^KTAkyM#DILXQNVoxFwQwRxUp3E z5tRYsHi}r8ZfktD0bD$^mc*3Cz#Ref@N!8+CW6+pjw;vn;(277i3 zT(ZSSSsRC=<&sgKiO~CA5=ZeZ%s&12u)#Duw~MRFs*$taqnwZnE3Rgn zu^M@=xpz%=msjfjeEp}%zxevYh}oMyW;M-o&I!UtG-k~_67P9+77QaQGDuPo(YcKQ zxsq>L2B05uEjL^)$5O7%iQL8i#AcE-8`?=n#ktZ?B?X?E6_#);%$Z0cc`lJ(ZPmNrchlg?)VP~MyY z)$#VfI2;Y6Sr44DR=?7&w4~bU`|3apl~yG!yew8 z{U9iO$Pr0cR3d5Kv0rN0%hw@5pn>&2_az*JPG-si#|Hq2scr~FBW-NC4i!szDr2kc z0j$FFvxgDzw& z?tDSK#uD~o$HzH6>X4V7Wd<*1+|Q^sXANk^r#}l<)o#D9nr<-0p*UUmJQf)&;y!H3!#V$0Bd|%n6;mZC zp#T2G*W+Sf$u3NJ85H$9PXtQr+=uC zphN{5fo6-jTpJlCed`n=Ypg?Jo3SE_@eqetHuYbUMsa3wZ1Dh-?fqwXC1=Vyd;VB+ zbQm4_Rh;W_o)M{D)x*lVQwzlEE77Gd-ATSsfp<-Z(}W_Nqz3! zf8N*inacGx$&d>fvn#HbthCe*{@4Q@+W)40IHo))E_tYHVCP3ZCj}l|6A|9JX zSSNkMCnDtXc*BoC+$63mS`C1@c#S28UJ>3Osj8`4(7Lp^dN&zI;t5JjdQ$fB4w^JK zY7ZH!t9h01p`XBa)=KA5F??G72d|YNv z^Fj5=wyTt858@f8y=03Sq{vS6#$(kui8qYl%2#zaP4#BUMl?G;qs7pNM9+pSe>ukH zt<_%@sIy3{fu%-)bBRbC@?RSN%mlcy410vqMAT3-a^=ncNq3&IW&*NDdKp3{j( zt~LNeew(Bxjoqc78la#U2T5g%AJ~W|>bjbYe3Y)$q%(BMT(#TSLJ(!2p+wB2e9!%4 zhEMHs2gC~5nj%0<$T%8aug7Ciqtz|ix?SBWu#J0>(*qmn=OTb3z5LE{1yU>t;80Jv>P! zLS}o$L!czN5h%##EeBpw`f{R2R(Abu#Nz+1ZvEy~VHJl9vje2@qon%qze3?%9V2G&nIDMctF&#~n ziMp}NCYRL`&Y17G{=PCGyn>I-CkDeR9n^mEp^ye|la~o^S|z(G+4xd;+qQGHcbinm zIk`vxdnzEQAH6S42qHo9Ohra4n68pN;NkKnYR(MIt}{DUPpT2KK|*wUcDXtm7;QVs z+Nz@pU-lAOxGXQ15jk_$0RNqU4t~h8&pCY^HW7tvA_Sn(-KZPFQ+AGpgpmsZh@;v1 z4fm!Rna(d#t@&k9zo&9in6jQH0u!n}2p!t>3{iQ|q30_F^4wZ|=gmhex1LGB1>!YX zN8MXw4p9t&;s;SG5@dh?KxBNp`?5I_NgHdb{MO|HngtA_Jknws)aa>f$bIZA*3DoX zT6tOs)qQzFWOjcXLpCu(Q`~hvMLUPoC;lkQhp;3Kp*g7Mq0~(9ma(c?<6b|M# zn4kgzVQ_CvTw;%2GHU0q7diZ<|0jo%x21u&{6{|auT1F!6LdI1QRi>rGxid}IQggX}^+74i0wY7d0a*E_nVYmM1;1 zKN&IaaLS;bhNaYRUTc_A`n%2z8^3WyJ8_kL^7KsPl$|wXQgTbd5RhGh#3!TKk#V2dmo-W-&Xc=BZ>hds30yG*jw3nKvLgLXV*T*tW>Au9R_DlO1VNq`K9c#X(#pmk@w{n%0X+(u2mtxC!}vxSUUG1mQ& zd}d+&-Mif!0=1#@BlL z4IwYg^piw_b=&dx%^mIk6B*Jrd>^rpsdgHbV-DZMw6g!cCvP#v&Mv_24s-d%t^wNh z-Ot!H!wJ4gagnh;km{|+wZ9zH6E_~^D>>*DUd@d7+%{Bt zOXU|cH zySm#S%E|#fT--oL-&qrb#v{v*w18LRgTNxjxWT=`MS$ni#J2>!$?szlyXo^8M6ONJ zrd6pe;S4Ia2H%-&r6{}76ol*ZV6Rfy^kfKtF-Iq?ED*k0iA97_*YZU?NAX9A#Z83> z6%!MeLUuilh_!B!kh18Ns;i)@1>kZq%J!;fFu(u-0uW8&lq1k`PIZd8rr!c=X zLf}53kqO}|e!q&lIMU)D&+Ud`{1+xe%tWw-e(a*w*`AL(SEM6e*Gx@}47lr^K2+tC zBmF&BMmEO_+_W5_?%vG+oeSPH`i#|3cl&r}F8Jtu#@LX}*An5Uj5)H&ywH6-c>z~A z^|{d7MojpSofnssKnl;?qczq4+dhglMnqv=x?~RP>P8-cp_MZ3SUUS!30pReUkuxC zBb%{nX1|>Nu`lm-4NEKaw~J*#;b9e@N}g40@@OENxm+&x@I0{2SD?7o7sgfr|IiBK z0+L98uuF3r`u#`c!HmCtn0vWX8AW5BA@X^Wy2MV-RqAJnnTqlI7|8xT(?Q{ys+i;<)tV!$Rq8F9Cu58fZa&)N2O;si5Cr0!Y#ynI; z)D4AmLk7paxmef6)5j+i`;Sk^CsJcDt?oe?i7M6ceAiZ6HbY78aJ zQ!Vo+?To?7eAk?Ik#Dk{y~@I|Dd8?Xu&<)u-^f-MrHi;z-(WPoV{78@z67q_9*vuB z?}Q5K$^RH3+<$FsfX{U#GT^x5%1?C?xPB`pik}tSt6$e8>I=|7>uN{jrh0|tcyNR; z^d~3~iVn$@bF3i4*Ix%`N+C##XUjgwk@A7%v&8+e%XQzu%epP` zZ74QO$n5;p->=>3iTMP_@fLu9EDPm@(w!&PxsNk!&SrjMA;oEP+PE?lolKYFEPuiV zP5}?F38Kk0)0KFd9Ig&A`hNa|qN*z{VWosxdFg+HyWa-tFVq?Dm_*vJ2iRA)A4{!C zRic5LSBF_~p&8t=%|a~24;R~$wo~rSbxa2!Px3+M%`#VTQKtz{J(f%@%S23`R1$p0j3!t&Ut|U%?ulb#fU=OtY%niEDd=`G1DzY`+I$h+1 zsO`&*BG+8kyS-BR^s=EB4Mr|M8-F)#o9+$ss_VyAS4456Ma2uPh68{J-R!v7T`E5i{nrx~)qiP- z04uK~a&NpkACG2xvq0I{<_6}T-#|eg-;IYD0B7v)kuvn1p$w@CyV`Y{X~eyi10w2k z5*$!&$39pqA9rkX>X~P-O>XeYp`ww0W`QV7J+W4t>27AsHLH)H1@+*7rtbx)vz4$a zdI{qmmH6a=G4`XPF$?4OLqAMuIDg+)L7^uyfMqr(n5%Ix0%1k5(Ia8#@FudBeHCIy*{cp}?4JBZPpIUp?g#ASvABBvEF zkStKPY}(=Oa(!VMy#UrXaIjOz;>{e}Ac&1HZ8cnCG6^%`3jHYy>~6&(Ri7gOSW8gq zr0U5Q%M2niNBpAoBuOgw)Js~&h>!(g&X@1_TR$@kHy$S^ z7KD;R9tq-nG3)xNPo2{<-b13|hq0y{62oe~5+<*D-1*iN|hsJOg!$#i)4Ir++emh3ln3B z1(^DRvoys{g>KD-GtCzj?E~Ynf{vRKR4IF}JuXkP`y#HaiUIvDUlw3E>Utlrr5&L@ z>UoZmNh0~g&q=e4W}vQPmrGqCgwIU8c3#cD?k%AOYh-5Bg>NZ?DN9sP;iPPi_#b&v z=S)&Db`I6{0P?SBiQr63a;d*0u8&_Tc#3;SE7F~eTYZ%quO7WH1Aajvza5m}N3XZ! zP-nDqWL+{z(=7>5w`moh`8~~_DKH&KVXgH&Mw|cXBkU;;cuqJK5aMDeNj`z^Ire%O z=3aKV2CFKocJ*3E!P6|#?GMsBZs*EpC^u%pr0)r2&MK+%WwPGJcX>yKLbj+4EQfh6Z8AEa0 zOLBlP75WzQy&+Qxd*mYayDw`i=fLi9k(!nTv1Va!}W`cr`L`4 zN1&i@F9cFT`v)r)a-82=wSk8LEHc zANI#>*0W|SK&!J>KV?_tKz~4eU-EBi$_X53(RA81%QV}dALcVSgXQwbmh3EF5F8Bb zbf(V?m?}SNbRB1dWb4FlqV|qPjOKfeKaDHE`a%(?VFuC7)3s66w&}KR)qbvxLwebI z!{q`e9&0=lCmTV>-B){g1Yf?*p8jDpx0?1OmLLs~Ww~{jaR(ATcD}*Mj#1d3NJ3dz zKi)!X(Jys}8;3S8m_#?=@}y`03^J39XdDSb9PdbmAG+x3Cn(fR+r6@aPvkCE12btJ z#OohGgti}|%T4;|EVpA&7=QN1q3S5CuC$C72E68EL}C;z&#W>FCo9#r<|sJ)zke1E z3`*fR$Bd7WLN_jcboRT<**#_!qmIQKEH859~R` zMV>*YsheEGEZJAh44^W>z4UYS?^FdlVec^P>OFq6S%)UceEIF zoM0b0T5|x{+~$k#?zTqgb3coF%s=2^qJ^t6S_`>r1L0G<3}&o~J*y?u2urvGNN&_x z-;v>e@+S^4(oD63tBvHZD6jzYxCt@KnEELZ85-79Maw>j6q9X9sOanaQn>HwasC|f zHLlYgZcO>9OYCM?>EuZ)5C-&!xc?I;|Aa>acru>l_6=c1l|2qz;FL$AGvFC^*K$FX zO5v}*gGst>afw_1VSpN-2k_*hqz+84WXqF&v(NCBl21;Pdi%2}c5mV}m=}Q(#quwu zaVhuwQF+PzUvhe`A6Pao7IBEaQxqa7T45I!8{#$NJJ`O5t2eH4#3aZtlK1mj2e4%G z+CS#gwPlXXPG`iW+QDxL5wS*VOZTLIoE%-7G@5+D)~q5G2V_O`@MZYre_psrhlPi443!fxr;Ibz zeVVX&Mcgruum1LxiFfrO9Ip*th8su|*WJCT>%X%-6Q5$}Y5Gls{DGPhkm0gCV%Iu} zni>wEnRuwfaS60FR~HxPpw9=mWBtB)yt=$55`(@TPt5i!?M9pjU$@Wa8<|ZyJD5ox zQ&0Xxmp~h?!cI?A%3H1!NQ{ZNE$TY%-gh=Xhd($zD8DF}uM5Qq??+`K$ z4zp|AcBi;nWtNfH1tI6G=8X8mP5v?o70cS7zq{J2F~^q1T41T1&di>%{{?k%w$?V` z^v}zL*3N{3==R*~IDs>+NdgydPDSd)pGN7-OsjwGqr|@(tykEr4X?i-G?SUfN;p1% zY1M9GMl+tRWm;o{fT6kQ{2pyqv0*?Cke1j2$gIyj72eA_O5X7wq@+5ZWFAnqD-G=V zohO2gStu8DRnxOm5HT8qN zS!gIcW(}U#Mtu=YG#Pk||3mfBJsg|;%-WLZ^8e@)DrmLdlR;A{-f5R-n7b(y8Cq8RRezKy@Xf6kXL=1MMH`<2b5Gz8plLx7ZG$%Tzq-3Xp>9sP40n{(oe>WmweR`Ua}f-CYt&4@e^=jg){O zAsvczhr~#CiAZ-N(j_4sinMeL9nxI`oHf3C|Mz>&`NVaJtlx^K<9_;Xg^EoZiJc^s zaqtE_=gDZhT`Ie-pZX)vBtQ%o6q8`-B}Hp1xjy%+6pQ?j)XG?7xAJ6! mGRy7%6 zNOrkY`jBQsylugb0ix?`!?#Gy(OT<}WKQ zvg;2Q*+yhx8rK)K%e)3^u9Gsb^-^`_-99D@L>gFAluFNis!#5xP@0OdAT2f7AwepY zPNXZnj+4YUP~w&=jf(-%Afq0uv&gsrS~F7an>b^j2wa}|t85O&|1hS}o8U8JW9)Zn zkDau9ALZP0;~CuEQq`z$Qj<@X>@*H0^3(zE5rzX0DGdewmgl9RjC|}Uf`q07sU`Yb zdKM^YVP>t7h`WDc_5RFn@RJK+^Kobm{~AFV_}EdBRklER&jm9&SMm>_7N zNW#HxkmaJCh%Wlgd!oeKa`@}3`HhuZ;-5bP0lNweAI|FZ9%~=B)%8s zYE*jsz)~z=PYNg5Daa@WRFTdi*8p_Tj%GVh;8*?dZ*l+GgS;e220$`;=$L(J$5vmP zQKb6CE){ZrI~9PUMEr5%9+h9v;Sa*t&G;iA^C7F zyysglx=bI|OQc-XZ_)4OBm8tt1#ykjWO#>o*S0WFd%`V-?xY-`hBiL|Cgp7z=ab$ z4tSbe_|oI*)8F#BWscCl4QVa+KpPRWGTFjp6FeiSEAnB^&zc9sKu?IsF{oRYs812vC!lSUs> zDBahkY$WR0N6`^|FPGirwbvy)LK-^OzTO?VG$d<*VTb~UNHZj;3IJJWgt?-m^>O&t z|94Zq@~bNb?bmRF2V3EPY97$cV?%rQpY}uqWTet{z{GFel6$~T_FBCb%o`&!#lZl# zi{aldBs5^(s}RAD<^dPsvuli7x~DRD-albICfxcSl=wvLeMLMBG3fK9-#CbOW`1@* z*lHF}?^iB+qAKh9LYsTjmGKA!2p3wYuoFR*^^!^Yo4+O2LRdM7@>x@^XJDfkV?pPx z_y|}s3E-={8ZuIIQRSN6+9CHG99c^%=lhK<2x7KLd=3ClXHqRoWL9!>cde zeEN^Ebx1mf0viM+uvr7|LN-4qZ!%ccIUKs@QFJoc@nzb?ZI*Vg6)*hC{QP4X_fr#) zcX_o+^JE=gvIaSi9-o33nky($zDDv@8&N^jy#IgccRB@aGML0h`sNY?&|8$f3#&;6INBRH9KE-`5 zthZ}IP`sFtB%0~p=E~TSq*Buf{T1{*2i#rh#9^Y3A=)h;=yTNTs(xyF#ON4L>xEn; zWMCO@xV*n;|3qh4B7wqXZbHI)3%UyOFKZY-m500IG)8`M}2vFzUxt;{M`+CC>v-O7d%MGxHM{GpnQ~0u5P)mU3OZK zJBD7H1H~4ab3&1?B|0eet`DP|*+b8n1?Co?e5&35pEc;m)+#7gulEtb%M#yI$Z%}Y zXSf0UCn(0BB_9}??<2S3e-dt}@Aje9uj%qS&;x{{|MtGQCPPB!oh~L~m%FTjNX{so|F_G?%C3vUH-#x%C!f7V*o!COAcLB5z^{@T*@w zSPvfrGoT0n@|}L%x2%_is@nH_KYXDSYzMVe&{a}mwjsK;tQXS8rGU%y?s1kMfb_f; z<|os}wBoHC%#4ffc?q^2Z!4uA@da4+u+gER{Ub2WLqX#5iWRhNm_9DbzbacA5d=_A zTsY9)%+HrjhFJl7$uz3_wnA+%@gy#!Yz#xBE355+-0K}bv3Yuac|Eg>1zYy1h7?@M$z-#<;GpsV9R;VEap}RW>Y1QiH;S``nne7j8y`<_ zN_12~Z!6|TxEZ&Pw{a`a>sQRXYxWa{x52w(On-`#AfU5bIa#dw(-x1(y5=oVXF09O zYTcLl%HR1j0;)SNSt&ps?uev8wX-ucNbir2IL-AJQTNeZ5TNkRy z=?va|b|ea)9>Ml0MH#TG9_)jnT(Gy?}&%?y@5p z>(0NYXSnoIIkAM81vv602z%=#pJ4NcL9E4&B3jyG_WvBUxm>w9)8jqBM+=p)89Kk0tM78-36bFZ(qF=NVR#6Gm;l{;Eox{R_Et}-veVCz>Ou?GY6A4k_lAZUYpCZ zolQ-pl61pL3C}F#+0Qnn91rUInIQfI49*6c8WcxLCP_jyUn;dqC4ngjz= ziJfb|G{=34x&w9QgV4X~9l()IYbrL4Jf6bHDQ@nO*>|6ijZ->)Qvg!~IVoEDsJijK zf}S5=%b}Z1+1o?+A6)2mE3n@xe~|80YI>tbEOsG_ z?5ayGXJ3YqFS)Hq{$2P}&pvjg-x{I#6veu^KYLS$We$V>6O0{H0>0_N48nq`q(J}Fc_Iy4K<$+s|dCz7YKbfxIk~7n>n(|9N^0qO1rIxlFk3@QCkwlZ0b@4lB zD2?gqB?CuL@ZiNKIJ{qmzNA0XxQ5a&KQPBPEC3Cc_tlY6#<-YJn& zd+7o|kP&!*UM-zK4%;q=oRa!FFyAkm(yZOe1JD7*CQ)E~J2_v19r0$C`^4@V-sY=Y zg$M*DOA=(`r;G4qFY8R4u5uz{C?rK6O@{8J^mJn!vkIB7d~@%03IIIa7>YnIVFQ@0 z<^9_f6d!BZC~4dh2=G^WNqS>0yizQYd*Cm%ct$W$<*?=Guoe~96?`+~7s-6fT=X1d zPdc6~$eIzkZ_k;ml|P42Q^~vI!UG#!CqVl!UaM$uit){q&f@~Yhvg!3t28fh_l_Ee zyHMOcRYSJLty1JRQAIdXBF|M0TBBZZT!6u*q$K!c>q@`w4$pVAeZ|knZ=aYY7Ycn1 zyJQ0QpnL^BhAtilO)w)OjbnwzrOs z5Lv!%fLe-wwEzumMNqD$|fUJhS3 z65fUNOqufllNhH5bG)MgyU}8CBVRf3`Z{LAl~q}9tJyW|$p$+0>{aF~&hONBi+7i> zqz8--d)oVFBh{my%Vp&T{K*mIFU`K}3YDof{O``S#0vTw^48%dE=B%}q2%w+wvER} zEgV5G3SmiT09A?&av{kT?1_|FtaX$H0)Hx=JU)AM{Em|m80)meU_KWG2KmNw6a=hk zOmjqQwYdKMU0Yq9M%UdO6|xNRd23^-fVMt*r~NNP-|PR@ zQ^n0+KwD^>Bd||7j@-a??w>*s#t}eV+9xPuqFF=}NZJC(_pipy8lT;c3?1|yx{t%- z*}aB*h?I>2ceD0a>P7$v_g_s2!v4sEoEKZjQLtnsvaDjadns>v;h<;mu;lVnV*;SF zg-HhA7s!V$yZ>#b0)XF7bM3^3UevHD{-N30sP52zL>MNM07ko=8V)2`BO1)Q@T>*d zV$M@2&=iYgMXm=p;u3ELR7Yu+31<#FS3eIl;8mv^M^sUc9;Ehl!NZm&jv{tHyfe z=m0dHmA*!uKr!_h8+z`5KwVzDIY!}Efx3Kop=rA~g6ASOJHJA6(vwpR!Y#Y^IPT?k zOL{4vz3CO2E)3P$8M^OCY`*e=M{k)`NSo+m>LL@LV4T!3pGi2AtcSS5D;XW)Jhi+i zOEniUpO}01?oOV|NNOhi>B?4k*`-7L&NO0FiNKw^^LcZ3wz90htRv~8##qg?ufnCz zU#+rea72I5SFDY5V~QD#{v7MeX9SdiV`TD8TxV8<(Ix}%`g*bJ)CSz-{|8DI{!mxVIbYz+xX(4Ff;Te_m0|1(rfM#W;BdJ6A zRw9Xp)+6cbFQ8x35;VD&7>-%6@pC`OSrB+Y-TFoR8>88#zxs&Fb??$|s4Tw9Y<;RY z#)52S$Zif-rxGu-Y5g)gvzKMrkLJ*p@N!8eU~wh=ikT}9n|T3a^C!kiqs|H%A#Jbe zGp}R_Bo*P1+7gLKQq#*p&)t{gKrhS}@C_Z~sIC+h@A!ck5vZW|&P+>+_^aaY(pNI< z6HNEzStlO%3h9EUD%I{9L8YRKUm;1=;7@%?lxFi>#idFsE^q4ds}8q_6Z4rRMY-_! z^l;5?NQCmh-me;soga$sRg(-N(v?FD%<*SU_@CnQe~eSBnX~(I9zDYinnEAw&mV>p zc5q;F9Hj@6eSR#Zo)yX=o6Uep?8gx9)~@cN=6vw5TSz!Bv{nci4BV6xWf9+DYF|De zP;^=A8_{6meG(F_VcAZQLiG7XI%%Fp#36bGg)Qp53|3+hUR1HJ$2vQ4>q~(6(gi@X z_QSnF?lWJFe&B6z$s;gr2t=s>9CzS9h9wyP&=@KKufOjbIv*{k;a4*0Qpq=Z$f13; z{F}R$d;%}G(D*{BD+%2}?HEJdrJMMT(^^9n&%x~+(eCFjB^|$ij?8dIS3m4|o~r0{ zT2uX~PO9vXk1iv_n#7U6$CZT8NPKK45Se*WyDM^ZuB~WxD(&2IH?+fbN{A)+#XA&$9g4ag2K1G6CIi9r@Yr8eb#^-d7t zRR-cWv*ZhsUhBiDJ@Pj-57FPEWH9olaRiUJz$1jMkY!#xk&|sZD5^m~-+(bO1^_x9 zjCdzncJH$RH&P2vYWa)Buc)pBIvWbX==02_Eq+9Oj&ni8BWx%I{&CS02B!zw=-LPH zSDg}i_<_|ZLmJOcD9y}&%Zu9DV-HAWm9g~$5!}j1yLcnkU9%?p4ot)gYkmc%#i{|G z+x&Eyfe1y34w%jU91cm|GHonfH)mxWr^9)pwn6F{58@o&4W=jBz8-fyc!xR&PVfXQ zLO5=9=+zbf#DEY{AR!4PuKgpF=j!SBt^L6O63(}rh8(oTGdkBQfQ1#|xD)|AqzjJ< zwyF~P6$+!iF_-YZ4*{;>Y!{v_UZno-!WVSJ_Yl?Oh@ok5MV~}%8mal^UtNPB{3)0! zr2Y$ATZsY8wCWd?-Yvj-5rCi!87-1l5xH3vG6vH8z zOri#MEIm8$H(3e(KLWSK0(IXSTPnUUO5FZ91o_$@@Czsz1*WIghfUImDb!@iQV3jG zgbX1}0AM#{>Nh|GCM$qP;+4}u__@Vt< zr}Y1wqO(Bhb#tmWAlF4%i~{z*DSin-Oz&WhhB9~8xMy`Qup%+;(Dn^ zdO@W2jw~9_(;2$w(;4!0UMm)>xrhCppLg6_J2C>pb2je2^YZ>MuYNA@!>%gza$&N&K5&Fz;vTW*L=9&Q&zm*G<>Rhfv$KEbyGznr!(=J|@cK&5! z!%T-3q2I5WI(k_v%0yUY{`ha3Y=;h8(DoZISb@bb{iwqqG%d8EH(_YfAvc&sv(3idTMX>OQ#(D=-!F_5tC^(~k8@E}H9$D-Z@8+qem zqkvI%1b$8qiwlMrrhx_l@x7Efo9R(~*KsY;cbNyIz2Q=1^d%t#2VNIk7ZNYTBX1uF zw%Hq->8y0Jtvn?h8n$38f8lXskgdeB1UufZC`>0}2x>o7LoE6evFQ1;AV@K)>ddh^ zI(baIo2cp(RZDke&aiCg9~0Eo%Ts1}(?ot)=^V!|($TvwO|fkm`O4~NR9 zV;L`pu}nz4$!>|9#JqU9j;WRT4zUgNYOQKtuk=Knx81s(lZH8D56k|hbP&_rWC$$c z0e_87_jd{Suti!r`wyg8euD-TK}ZqnGDTzPaOZFkbS!2{m|lZiACIIFoxUUz>_{WF zCyI(LAij=Db%NHzLOz{D5}Z80bI?|}bavdW%-f@L2PHRM+IN$ueK29wyn)0i^@VYi zYbn*xFv?_ps;5tf_$-Cl$AW5Obz9JRiMujb=PZ@S-9XFcLLj9{p)XC`h#qIP7&x$w z(T|VJY$JP@S_JNH)MW7h$@~~S`l%*qQ@i`ap=@z|d`Zvje)U+IKTfHV6|VxpyJMC= zJFI>>N+OlV^X~-%Q=cACyKCfO3+blv*)Pm_Fb0aQ|3g8aaPE5UP(3sFHvCz*9XzS7 zvVSgde>{>Ri`GA(S``>_l2q~ARuGj=bZr)2Iu%5at|scFG~y>)?(n0NIUL;uvSMV_ znNEj}1u4gl+wRxY>{^%x2YMqT9|IL+%R;baZt>(MJWMvmeR0T8+0fc7Kan=>pug~Y zNk3W5Xy8`y+H8>;%4?X7bv8~t`B>mdBWkskAyc4&u7g-4wgD*%;$Dam_Yz1Y(DWwR zC@>T=1k%7=vli3=QcX?uI-xVFN*7|QYD2zJiDJgw4#Mc4gXtP={JFfoBTqxItVbPa zLgLFtR9!Fz9dVDT6NhC3X-~HptZ-P4&1C{1*1}Iy>-Es-O{KDxHq|#jhM(-sdwJa! zcE{iGm!p&58jOyOm3PQ4!wff%V!_wA+m_Oyf!NQ>{L7^?c-K%MU3dnH&ip&~*q`or z&9!^p@MGQgxIx;u9$8pc90AkO(mic8Oc-h`PZ-p#+oVS=2(tT0^8ULB_t7hQOK#2AtFZ2*X=)FnIpj@$^zt?ScKY#FA}A&yot=#;M0X{Pw$gu3X6bFKj4hg5;A0K@JQ^ruLQSc<}UU z#y=Tus-eWFla>7e*;s~x?EhTsUt#6-u>X!B;o1GHcM+tVHu8-^CUobdy^2B7WZpf~ z4MH|x7dPzcO>&`{!Mo>$&+fdpu+7@>40@)8$xtc#=@Ys9(K;#fO*(^J)zDcEtg1K_ zm6T>-ueDemqI8C~iOx=;e-s2ifv2oR)$auqPG5puMuTb4F%Wh@iXcpud%r{29Y@Q$ zFucm^g`Rx~h3lj)G|*6(&UnZr)Y-uZ&up0b(FjGl2Qs z`}bG$HH0*cT0$=0F4t4tySwQ#$uRUed;{b?zXTQyAR{Fn9^fb8wgym-7jbe#j7pKY z@jbhF z08$2D3w06~a7|z&AYNp$U^36jtOm6T-iP#7yiL*)4kng_*!V&p3FP^%$T-+NSUFwj z-%3rSEM5I@kkDbwx_7{bbx)}NuF&~)7Y1&a;bb~7gKPV#`oB+v#LRIQF-KWGnU|lR zzQY-vpcn*cP}5&GVmbJEH;SeH+T`9c8U+42MnG}oGKq_(LES`U!dR$zR?N_^S~+k6fAm z$`un{HS`OlTQX)_E_=sPO*R)3dZf(S&p~+2c3mhd9`*_NwHTxYXp$LZMx9kwS##*R zD?#l_HR6Upu?)`qcGU*QQ?07Zlx-F_X?;Cj7=h1_ zAnB>}e+PavXL*vNiHvejFglDl z_$J|j65A?OH%pdN=lCFUBCm*VN^gp4Mbkn5S9ARbR(F{y*NQIPY6 z`@Xj!$_kG+|JfMrwKqZxCYu(PFz8o2*Oln|9uuhW&Oxk6h`cciermft@fD$t*pSQU z0S84%CE{yr4WrnEU<_&xJT$^87euC-nO^F4iFBc);e%8c)xv495lGaIH?w&El_Cft^!V_X2&)BuKjZ@z$He1-R6#*dRJ{89 zItd7o66!`^zuLQd@1b#odIZHK2o;z$k4uFzp=|s?Y1pz~_uQU`(;dF*#?hymIJ^p+ zg#C`QV=>w67|dpWT9twSJOp8Zh_=;A1EMQN#k}`U$OLLTQ94~N^sdP(!Yu!1HkL^? zQl?lK8LpuWw-{2!_E%CVCJLb!!lE@aXY)-<&|d_R&Et+pL7~&l;)m5hgAoQpwoUZq zItgBJ7iNvw!sohyTo8vC957DC6YB9>GQ50!xLx^q`WWU~68%^y5ah;Sd>5JQ;IJW|o=tjvXdTRdUgbNy= zt3MSY6aG2X>2VV#=((T01Utx2wQGEbXI~xo!Fu!^uq2#^x=O7?(djiv^+o>LRzMxZ zW%qLjL9%mbR(b0c9y#O)^n_ir!$n@00>17NbD@u-5?ny)Jw=8KVzK;gYFJ&)86_Ii zy2YlK*+RTCbDH(3Bd&PFv0WvU_Wi~(o{pQnYvkiVvn0QvYpQ6o+AE;%fL-MKA45xI zPwv|$v^n}9#eeINh09TfiU4=-12BnkM2GoS`ed~YTuENcFgJHsevBRNMkD(e;h@~< zpnl05umAG^}cPxJ(7EMT_}~P)?sY_sCn`qGriT&}&XB353C(miKl!@TNw$`c39?p6wox^MSz-@rH^669+!SyTwjA zF77xd;fd-Woo|}$5`NJ8(oI-yn`<fm&BRDCI23#_`fi_K8d-a}UL zJI-H^o8Lq!JiQRP!vjTs3~TNDCqnu!bX6;}vC6FP+N(NS#--2+h#K7HFaxst&A~%0 zLyCqUp-e~LNuJ$C=#PjKl{|8xA4uV!?ccVn2jMkn@ST6%85SLGWz@Tc=5NY`uWKw# zfGstUpJ-SDQz7h!;SAAb$O~oa7@TG{B;S1q@f%6{PL<)mjWl#>6{3>lVOtVipv=I! zw14A1HTL&%o_b-I725o?l_tB$l#8NtG#5ZPTwpUBPaA%u6PskuNF^$aj*hDKpcJAs z)8U2SEuV7gy9iZaKg(*WyccA^)^O@G8GUPKH0xIe+taKW$t|EdnbJ7XsE! zg}&B^I$&I3ez4YXH_Ti#>miRaBo-%cJh(cnKsZSMFgptos|x|cc-!;YKo#sMV&t)y z_Yr2Y*%2%eU$XApE757OgMjxKKXCwrQEM^dyjn+=?Zeetnb1$Dc(T3AOPj4WgU$)t$C*_Gl_#-%JMbo}eDhdjU7{HW>#i9(Bh&S&H;A9JEf0ug&+^df6A%qc9j7=B5 zXm+3*el+jmF}*fYaW#%c_SRxYepCxjA0ew;x_xFCv z0JW`xGzCj1W<^q+m|u$f`nrCBnUOI-MVifDii}|OJvsa}OaG}_t$|E3pY?fPz};T=LHJd%0*nWMpz+Z-#O*sp7w-hEA^t4A62Kg!}r> zmI)Y8OjO9y8WI-+ZSiLHJ(hx+JinL%-<2ie0eIvF3QPhb>Pqy<0+FR!Kud-1BV-$l zle{EEhXygg67=#w2eK-F0!F-^Fp*jyd_>WMrlmEh&AQgM6rGCpp>d|6!;X3#R8bLK3#3MMP^S5;M& z4Gj+NWqke|{5I_|LNOywg$05Lql+mn-6iNR16H?-i1 zfQ`)C(>AZCxNW3b?|pC6cb=3jP#GHY-4(Sd)K#xfe9kyjZfBO>zQC$Y0tm2-kY4Ja zwWUH!+Qbd){NGM}`@OE5g!;PU^#emBD#@}9F)b$A=-(bFtNQ`Ddni!4fyK20Ar=|S zHtD4yJQZi<@*h0YyTT(+=6gM(U6o@Fqdko^PHZiX`i{TM>by;L|EYx%1YsR?v$6W! z;J9Mee2L(#pWw4a#LcAn`T4sslQhk;3~9d$Mk1~{p>22Vi>`6EbaUwP?bs}Y{qzp( ztNV;c?uX296sMY&$)zyvg@gU~@_lORp?UZ4_GjO#I?yfWe>zOu(5|l>e82zc3ntJi z$&rzfWW2n*g60+$U+qWpo*URZdj9ueF!f{pth_(5c5`#1EqLu2A$zgdxrK=QDA@*g zES|>V!J#r7x%xfgktfd#KX%fWqiNlE2@&67-t4DID%IQzzD3^d^k@@5rVVCXn?iQB`kAopLbKE$U*lviw6q`3RygeGTZF9 zmuZ0(7rsbe2v8SP>cM?Pbp)uE-i-s1_|G#m(9VZhkLWj-^$iY+x4-7-khjG)xWyV; z_D9G4KquXLIU>Unz6kR-T1Xm1(RxVUs2pVThbVA>a>(xseSNlj`_ECzqw~Fe_7t1Z zyaJ1yo;GO^>jD+~)Y4>eKl$wDJn%$UHO(6=flT^)1@Hb$mbaR@xe17nkt_w_&0y2h zRyh}|aJzg9#maFYhLf$j?YzvxC6@I@yPW4{DaLFcenhs1lRHA-M0LeesvOUCM)y(= zCsAQ~RiV{copCW6MC(3bXaZ7}Y z|3j4{yT2@!k;*~jdj?K!yWSJQw)2P5NWu8_Qm+KX$>joIjMk&xm{e_MCiX(OgF$p{ zXeJyqy?c1&LH3>RV)0LaPE7f z4Y&Iq0@uB2f3Ya<;7-n784TfkSf9NE4a%j}BEHSYam~IqGnj_rv(r*x1-5T}st|`2}BBS2uCxs+zdGeDT!9z7oIIPe+O@m;xFyXU8e} zKyH*A^%@%X=_X(fn^FFDS*`C@ApW7e66W`zL=!ye=+OtL!H=8-HZhu0oJLOWDq zX?C5OSh=gp2~e3Oq{}u~91Rt8t$Xm!8m|K(BvxSM5pNK4Vw9 z)!~I^U3bB6F!d{x-&vsE)}Q?CvM@`{5g_)7ShPuBzMhnLr(vbT_+BDVuI z;1T~srtBS-1-P=PRn^r)LqC6B(Lg$o@V`uaMsP0TfWQ96()8VFJNP$6^u0(ysQKs_ zA{yXlyxFlXCXS629Y`(tmS|M~Tdc|$9~I>_{UmRmC1!w6BsbGysDf`R^PF}?W}WdS z4=pWH0Jn$ny<=$XtW9t`ftOBvV6A<_9-0R~_5^&fQXk`nk`MbWg9!FuFrCzy!3vt9 zH_Rkj1f6GdQ81pfP6 zL3f{4Z-B!9v|$ss&2uHCh*w{Q2?51%0FwhVJ|V&Newn1V2eCiis?9`u$f;&8p21Wr zTIz^H20v?0M-aEi*Ht7a_wd6VXgBnZUtbNgdGP9bs|SU)eK<(+*}G{2!)wG6xb6);0Vxm*I+y;DP)s_Hfyxh z;k>85n;M^Yz61^EFd&hg2`>xY>0;DnEVxNJ)dBV%N=*?EVTGF1F_Pz|!RQ6duWI>9gu1_SHH#b!&ONjx?D7>ygMG z3x`2m2a3=Ol}sI~6?UDO4ZMKXXLkL1j3ZBrEe0N){pe7&p;)dsHuU{`$Lr*2-rEp{ zX>f7uK+JS2e!+V~edb`1BS)My;_3y?CP{&3vTSlbFNR6ioQc?IQRHw=A$yXeRxSnWjGK8B%PD2-!32?RrS#=7aULux7YI-!zfsd%|K$r-pe>QQVA()bI7&r7qhtTU0V zYb&^jg2jDfH3%rKxIdhm!1&>`IuTv4;A;z_3}YhdV776RM()MKkdk%xaWW_2&dNf1 z2|?P5I{Z74AI~Dv$$81mrn)GVC)?Ar@cB{ycZ`a(HfV6@%mb0}AgV<)V8@5QI?Y5O zzLT)h_*@9M-M_T7w3>&9=S|x*aMWLId}BbT^UytByf2bCgDjnh_3h!a3ev8Z`uO*) zW3T81o_ z=KHk~_U-B%2QQ&rdz#o1 zL#R;L17HN)Q374Osf!mTFFdoc0zQ`DQ#8s5_?cR+(y_;|>b#Z@)x-GDrIIv;CRxrm zo(=-(KVMy4t-VrPS)b%UjC3m)*{OUaVWLr$B9uyA>=_5h{*lNc^*@b1d(9 zJl_RX`6F;T*h&IpqQmcm7~`6dC-hcGYeA45?LdxPT{w$;wo?l#;M8Yc#xudHgYR?5d(v+xvu2>DG=;u4Jtsm^;SM`61z?bqnTf*@9a3|Rlxr!zIz;aU|{p}7O@sCw*yS5|93_Y@qTr=-{-oi+ve#?Onb>E}dTDfmD)>s*sK2xik znX%hb_pPzWaVBW7&eoQRunjWS{LzK@P7H?WT0&@+otYTvCO#ZqJD3p_S)0^QD!^?7 zLNF&}rirdpLMEi{G6lp-2ToW{tgkB6C~dpC=wr7p#cG!P6$zC#rO2N?F<*A0P7mv4 zmHCQ{7ldh$r<~=8Z7?IbkSJq|g7;kh&6_t1Tjz(PrMlHEfaB~$Pk+B0XL=?RApPMe zXYtGMqdY#_nUjb5>{ZRWs9f?EZ{Na#EWC%0myq7psLBKt;dBu;#%gCn2jJwW`{gl` zP?`X=32~ED$X{M1`$Qvs#SVZBXn)4I2$P ztn(%zU0$ZshqT>NnG@yoshUnxRNWckvip$OAbT1uLU-Vvi)zg@k`(V zFu#Q`XObJ_R^M7!+#pS2iKn65A<+iMrjC_@vFmP|*uk;;}_lwny z>dP0a=qjZnq;Omh4&MRNE1cCLh1<$XPpj2~={b3c%~oa>5r#L+ZIx{)0=L}r)KNd^ zc%D*DKD*=9(Ge%bRNwxea#Lpo;jW=}>EC!DEM`W>)!H7>q9t_!j6R!`sp9AHkAo+} zfBpKEOi=sU|A0P*F)i)wMxcXqqJ%LQkeV)jHaSNqqJCIEKd6N0T(2ng%^1b;3kjv* zTX$9TpkE7{Y&23UUO7+QuL_o90!BslPeupi6WGwYguNs#o&hb?Zw}46mm>h%6Z2tA zGQXiB6=pKcz4O3QgQ`)l8|0%VK%pi?%($EbH?mklI%A(R+oL_FHQzzZ3+qMQ+=QsH zu$^_-Lt61BVp*+S7=pV>pLg0$m#aG4C*#**{_EY2KS?E43`A;@{zxK&st-J~=S%+cmo}#Hc@Rg;jm8emFOA=c7sLYqWD5 z%>Uld4=$#E&rA%Jp739aZVtWGnOW9S0g=GmU&#k?U9{zFE5}!_CSZWsGM2f=+!zT! z*eNh_eIS+2@3@uDx^7YLAOp4p{Npp~q{+rb;EIJSK1a#BW}s zfW1TPiI2TN#A@wS`9ssQHZxG+U!JFw<_gcRMUyY7u*iFOI=OJPGEz>x--iJd1Z?1I z_tIv)cr=$>-n>~4b7+~|ImnE?>;8(4CMA$s^b0x<2J9dLK0BLQnm9oa{@x$41I7O5 zoeG|<<^weJR>=Q-Enyb|6I49I5DRnj=0LQsk37EabG{)z#?hYv@_G=oDM7xx^D8o7 zaVhTRR_^v3IQAhj-z8Q8BFFs~SKDcpNGhpk1bfH!fs4CFAoVSC$j#>p-k0k9#ErkP zn^B{K+r3QQImr}PT_qAQnEXp^hyWWj1m=A_Q1?a&BLf9VQ+?8Nvl`%)Wz~7afxV~L1|&gk~r1>%R&a%{Y(T;75LphhCF;m zUN_UwjvAi(Cm9{YejW_+uv{6gaz1FQBPgdUsKiO+e5E9t-{2c;9R8*@|JDVnGr{b7;M>bE5w z2i=m-LLabCrb5PnRsUw{P1L!kZ(+!+*>uQ4UG-v^e&0~{$pc95)kZ~M-Ers1%0lJW z zNV>l9zTaqNe_{6Q{{Ga&$|)wKvyv|S^7+EKKy0fZr1!gBN3)oKsxyV}*?fgftUGr6 z^iWv6t@JOhbMgKA5XL0m`B^uD0$b+ahJ8=^uOSP~+ht8QFU4+^7b|XLZ(A7IoTj#^O-0!dV=oN^=8(gTHzcK5q`D*cD-_%H%>)@=( zKN~&K?22-4OeC5a9PE8y{f_;lp%*7o_T0@m47TXw2Ii7WNVv2Qod-j0F2rp&ClIO4 z!v&m${da7DkSbMhp_y~Evra4G-U6p@gARaxzJ0>C! zOzSrFQdzL~=*Ax|O!2i+>?HO+o)=cAS5!EX4_!v2r1O!WjKfL;|XUS-C#(F z$$LP{yp?!VY|B3G*S*2-L^{Y9TCwVW97o&o`bMvOqwY-#TDv>+y}LUyq)>Cz#J^RZ zbTZNh1-wOCSPSmo@7&4d$+=JcDqWh{1Mz?SYU0*w&`6;cFbcIoL)ddxQ?x6GPs>)1 zWcr5wsHy}+xGeGbX80JtiNa8hS@k{qt{w*p;Ha;0z%Onac5Rxu zbjB>Q-E9#68kybItsi^?=AOCiH(em(9eX(m7wC?4OVojHH{ZxjxE#iO1)G6%{`m$3 z&&y=6qg_T(0h`x31rhExMhC~=sK@G$8XClEBq;+pIPPXd>M(TdI@zero_?-=gO^A) z;nKd*0v`W?jT2>}30+-5#JRH!*vYjp^;H5O!g}x>GxpLg|5HIpFPpS=9@Gy>w?9RK z!L_hAudsk$fD%E@*VtIrjR)>KJ`1l2z_wvg7sCGh29G_U_C_WxV4lv__LCOBS+f?L zQFsbup;9FPakyRYeb=fx=jFGHumQ${_tc4E%!Jr?WjmMbQh2G=(+4j?iDmc=WEB7U zmHaHyv1umJTRc2hR~|oy{=lQGU4ukFXwEUit$hC7DW{w76z>6$c0`UsGxT}A|qh%x+X z9YQ)6TPpajrCwWWhyFoD5L}6_z`n_*V=r2?Fr4rF>G0K$E)@Ef$0F;uTr|!{BztcS zJPXZe(BTv3&IQ{Ys`8MgdTj#87{i|2R zr81NM$CLWzm84Ep&c;2yhS%9jt{{+@|Acy36_Y*?9Fw0ms2xp)KpgQ&ALSd2fXVT2 zEBOz{#`2rDPHu*rglN}+^_j!c6M&nIBc1%v_TO4t-jT4k8ab=m&*2ccE*(b zwPf@+`w!PYvw}0n9V^UK%ZzI78s&xGH0{p63xbS)K@_0mfOvMv19_8W=!acPeTGj5 z?l2NV^9>nYWb3ar%l_m`n~sUnVi#BhrihqqKIINbE=`JU7=FQfy6|LDbS3E*BxrJv z5CkKLf;oGQHj|`PU4C6^ueAHP94-c_gv>PWfroHDPtI5oFj#(D*_t$e?)Yl~P5AMb zFJ@)XktehJi@u+w68~s{6d;2-a9*_5T#I}S0$DQOWe)`4p*c%h9DS`#OZz`+4O1Ze^bgH7TOG8?+)a6ft zRipH=y|DIpiOFI$boKjxH=bh7Nird|bUrW~AT79}{r_fPf&OH$g-t zG^wFTKtL%9SU|col@6gvla5Li6r>6$pdcW=uCx^P8L& z_Xq}tpniqlV)=VuIwFtgv2OF>1=NdZ*!T^8b@n@gC|?SY)tcFN+4c6JpmP_S+qa5W z)7=Rfdw0aXGVOEPpO=Ta9Pdad$v;|uBOUgkHPCO@7{VWi2_ha%-^2EmQ02BhvlE%? zWAx{ZTfSWIz#aLO`mg(V_9YCS#82<=dBwPWVl6#y&9A&(-Y$}Rf-k%$iS4U;)6o5F z?H+Kj!ut#lxWU2tz|_FA+5EwuS`LJ%Obj7?n9$X9YR;9xaB^b$Lts(Iqopnm1{u&J zUN-jVQXKUVnOScmkKIlu;&OxU)KdS{kiXnH=gkF?{J!7>*GE3FsdFwnoO7>P3DAKGk7>9y&*cA4M zy4TEVXjf##$eQiQGP^B)N*U!9=zv)0vuEhddWP2R0OuJ(j@l9dw&-1?-< zgUAravqE7}OvmBc$CH=O*4;jg9nOMio}bySCoZR$Op4SSCzOCyk_MPlDZW|lxK8d| zD%B&*&)agn1`s6>uW8JTOv1;K&()PSvuqEYjj_~a$2mR^>%k1qMUEZN?F1_y+Lu2z zRDdP53SgZ^ROlRP#ZPdoh|ZE+B0rvrJF>wpRQsP35IN3cB!iy*Z0-76+~rf*Yuoj! zt0`433Jbv<1J%$^{U7X(>!lnS8Nk%RFqGH%=cQ^uKY$e=Qm+U~qVQ4b1D3)=hosy; zBglh8OSah3Jt)3bk>0AV=G`r)wMkiw~ZG3Mf>^y*s8v!EQ)tV)a+0>(tnL zd;KW%K(wP%vOx$8jP3Z203eegjiprTmW=dSWRl=(BuwV?W@Lt4kZ53Hb|?DX%mZw8 zUhS<;{@i6uYQ$96k?K)R=c}F=y=VWaQW~6YJ6l(pI8LIf>xruAX%&{s!`xqU$2oo+ z8aHbibdG(D>!N=YLfknv|6_+25R%ai0pE2C54g6TZ2%QSstbe9dKGk~UN$}Cj>gfr z*beh<2ak(!pK*A~l1q9Qr1)i+?mxk}Ha)ige53V%^t+#l-_&4ZrjMend!2t3PuY5V zbE+6w^}Hc9qq?*6YnFG$rzEPIW|gN&r1i?|F74}Cp7R;t>K+8SeU3&gaCf5#)Ftl1 z;fzf04G7502vfa!chz)D!wS!M(!BW3yPvNc-kl6P>|=k~_&?nB465g*Gf2e+fI#dB zr}Po!2xd#5Pya*9q^2G?dAba{|9Ak2m3+_&+JQ=A;?%| z07inu6&CQ*(_wghD|8I|mA$=xo&ie=2fQaH?^HGV1Q>kSaZcK3G9E4P2bm1PTksFL zojK1yZ^1F3A@m6n=OO&ERqyE#a?P7Tkqsxs#SFLMet*7nF zJ8lieA@9HD}B<8w+^U zvmNT~pA}~|hmFrigx{c)pwnY(DeX(Pl;X#kKRylN(-hek{yl~1n^_->u`FxJ>N~2v znbwLw7yj>;*`WEb8beaUF}d=+nNgN zWK>^2)N(94dvzCEStHQ&LA}IO#=CHm>DSFoYNfvhDva=mB>X{_~?sXkh)m zmN43fp6L>rg7<<-$Ook}FGeYNYl5E9IfQF@>IFM~mp;`SqGknU`>bZk|LNR+Qi;d6 z;Mc?nJR6&+BTeV1*pqtrC0bteNc7A@gfRLrYi2IW!~co?|M{1G z{0Q@C)`~Opj~H&L-K%E>;uO%(j3)r6Fy_$yCv$k5S(z^_0ho{a{EE1@WHf7;dSa!S z99=2nIsBI?l}@&Y)tix>4z4v(osUbE$;fTHj7A@~Alt0K9S*ZD4f(mTBi_~v}gJ8SW|f(32Y ze_GL>iM@K3y*TMZ1eWD>@OickqV5mp;l*#J*i#Vzww(FPOsT~3%yMnDsLRm|&gQzD zFT4-@G$hjIfJs9+iqed3BnEJLTpxs&%WrkDsvydy z9t9_>@?YwtzYU$Ky-kxDe86)A4WyY0alu%v`w@cripT<&+E2P(_}$#K zX$pypMBYg{kM~=9U+_&FWhM0$oOE#NkS%Ifoe^;VcSMEGzu||J)Nz1YteG*NGRUE@ z&{@yr(H=SMrPHy4(_mMg*w3{7#WSCxwC$ne1@+f7Vs#q(gl$ePdoti^md_vic(_-B z-ZHY)*2Q8BppEZ;TQH5R2s1-;meeh$E`*-*V(d7ity68j(5*84Ts*pzhPF902-C@M zZaEU3hG#croMyweyj*4vlu%@RJu8nTO0bezGqM?9;{$x65F$@R08QodOmY3%=y65n zQ|*;k42mpyHU#uD{5F)!cYE)+FY< zL5EO8CfoCxBHt^HjEp4P+S)3u_0T3UxQmx8*8HNZdmB3?k}KT0 z_daujWPp^(z(t)nrlLy27^P=r5Rn?@7_ny%fV4~rb_u+Mu5>#BXDTP@V1Ew5D zJ%8rLjy%5K?pWPuovQqs1E#ti(CUd9!PF$uW!k- zm0O`VMnpz+K z?uzL@3fLg7K#WF4Zvvi9K{rF$&%H`Ufu#;CRt|Cb@-k63O-9T88uxE>TuP8Ua3|IJ+h3ukL$MFHMmC)Yl_T1V_R@rj(c7SEcHh7=rKoy3#6GGcL_UGJ__S{m5A@6a;5yOmPxdL2i;S|>i zDkKkaQk?Yw)IVt*uM(J}e% zY!1ubyTGXv?N@__onleHUdPt1uvQ!EkV;+<2Lsn&f1`suNyjo&&tPoru10&hd$={+ zX*ng$+>>{o{ldDmeF`U|na^kIvifsJ{ua(%`Vk!C^E~a9eYIKkLsq?h(*r=_PZNqR=VYZg1`-uV-CO#Q``aWw^`sJL_I8@9x<8 z8j|v!`$4zU_=3iP`EVbk8Md8;A%=84CXaD>Km+X)f~*ybV=WMQKNHuyeV&~5CM`g% zNPVSZfk%@Y8*Gm44Lfpq0|CRZxW~~ZFC$1QACzNv6xu7=h@sNyIpKEFB0<}7;qakr z_lg+L0dA`}b{%nl_K^}DFWZ}gQ?-E{(R31QKU}#T($yh0%Aux5_pu9d%7Fy*wLC<$ zKcih@7B645ReU+*cP-5!MDm@bW;5a%@0u``Bo^;y%z1$t!gkPyvN6oT^_P6a^j|#7 zgBX%X|LDm74Ql4IQluosF1fzZ93&H2%R|9sAL?cPQVZ`RA#v-4nU;gX6MHP;qL?YRth*t}H|%9ac`DKAc9 z5U1H>URz;7RFD$bBatr)-&3YMOLuaY=cv)%re=A&jWoRBVsq`Qwbi>;C$HfF2~ofL zF30JRDeZf(_7&s&-NoHZ)LZMdUnVS%03pS--n!Rwh&#Y*Kw^JibwHV_cFYu&LE?d{ ztcB$C#h}_VxIHEyvA%MBN{7>RW;>|vm|b|Nifq1%-;ji6y1u{36dCFr)aT_#^s^*X z60_Hcna%au&MyuppNeis(TVMq|9r1_2sXo8^)Nl>lsIZ5GTKfYO%=pYTTe%sW?OpU zrj3Qj3M{-*>X4>n2K09~7*p3mBN&di<3vcV;Pu(-SSPmb(jYc8Ut|YIvXr7o%q_4x zJ+MPr1y)M6Ch&ytWrm=8#URR8cD@wKTUXl08XNMJ7!&Yy9Wm`L6JWLeAx{kXo}uV@ z`?S#y^WiWQh<>AzY)F9B05$3g57us(0^Jc$Q>$Ybr2!|-@B(Tnc=k7M)!ur~17WCM zOq0{&&EPEQJ93|xb}&>2+Adx?8usm`t})iz%Kc_d2wb^5y&9w`f0fgx?=W`6839!1 zQ<%j(9W2&Wv#g?nOp7+qC&KVImh;C0Z|^o?_5BZ0FBFMjc6c^mAU}YIFWSpAP0?Si zSt~Uh=YTI|aCvw!x37fsExLO>=}MLNzjY=1C2uB|pL2(02+5LAHf6OY1Dl#|ifpBF z7)>{ACiv83Z56wB%lAw?xZoWSo#phx+S#u!LnSY>p1Zj)&D&@rFG}Qj+Qq$ zb60P#(ca6=0sBL~)2uhEWIlIo8!UHrDLG)R;w2}anOi=|*Rz4FADccGWG>0}H*U<* zzw^wJa8?(0Xj#s^9mhnIHsP9fH(pwGn0@UdQ|5^gSx7aBc3=7)2#+GQV=2=@cYC4_ zB00R`JAETqS6~s9ZUQP>f>;iP@z~tq(#JQh)`%j|HbB5}&OF7f4+QmwYtj4Ali4U8W}lCqc0g@rK{zBIyuShf#v-P9ztYs7 z9c^Q07Z{~`XYBe9IdG_A`YE7?6Kyp;w_S(rl9hlNZ+%mX-kj^}$c}}|k4#WmjGz`z zu7u*x?0n--CZ7oJxi@?#9J&tHVV=aZBIAVjbF2Y%oF)SntplOK0H|7YaW0p~oH58a ztPmu>oAu6GMSU$r2U&P|?*xo5aua8_wmp)wP_@PgE|mgxtlfX;p^mT(pL$p zEJCr$-20phcK~l@O1s>A?aSVjO0%Yl`0Nh%LG6)UJ zx(wAr;VOSES{_aece5|k6eh$g9&E;nmh1yw#+7&<);XCXw>uB$`S{h{TP;)hiwgqQ z&09DC&s4n=ZTW3ku{y<^dP~WC**A?=-x?r2Ata2`ayobdO21tMQi4;Symh%wyt>X^ z9HK>@Wf(p1>UOz}F1zxs#mdh-t3kF1HmKef5+-)$M*Y;PdewS>oJ<5wWY$`-eayMF z_9 zWRs?4!0;U3ZQ1D)(01x{1|Tyu@7?)8Ya9*=nMR;!;H2@*vuMO%hSc8Ivi^Ko1j7xV z?OPlQk*^k-YzH-WlU?PNqY90+rxA8^^&!g+27!@ov}STz3_%+DCjlD1Y&z}6V#-jP z-s3Bvd1}VBH*m3R#UxxeiuJVsV$^m^P&wt!`#cCK&Pqzyxjj!Mj5Mv7#w8m3Apouh z30x3o8$bHlPrRC}IxrNkvg6=NG3SoEdILo`>x6xT#uwDms|oQGG79X~@{|P>76gEz zhKMd;|MhlscOPilh{7zIU0@8ttBu#AI8H*tKQh-%tL__#T`diG-$0BwIZzdMVW%}u z-d`o{oF|*9q>rUs|F*98#V5|m3A5XG0ni`z|6pV^tu$U90U0XqfX2PfYA0`b9E?gi zz5v?oP&tN;C!VFu7J#WKkJhsuS)@(o4N9mkiJAojv<_&sm7#D1J6*^tB~2Qon^BVwr9^bZL6Rs4vQ`bsk|)PrV53QeO&j7eY{aRRwVpU4d!$2O?|qG6shESQ4HF8 zrarp}|t03XP6L^@9iE8f=l}7BFJ3N8BSBq*f0pRiVx&24k5RyM$p=vem zfajsm8{kbK4z;d`s$^!cPIN&d8`Y_2aLPVn(|=NSp!^0e@^sV8Q(nLRWB3?&*2Yoz zp-OdF0TT%76}XpX_){^E+0x773GmYt%5x-gh&XCJ?F5BwO-rD;oB=~MqW&YPzi(6w z5NH>x-G?+I;B=h3fnF!-$=l9Bf{r{M_p0iZG?8T|PgdFcv{;prQ+Z(K%w+qyFCIRR zx1BJn!(mJOT#KSNc`c$wOhKEw+H9x2=<8%cNx|QR{#K$*+?10O|EZ-3G`H;a} zNn@Vu>(cLfDDa`cqWLfuH1A}|0<9Xa6wc38FPj@c%ORavU>_D+)9+XRH$x7<#Ft70 z&zE`6%bLAkS~b3g`dzK%CWXoGbL})nW-UdG~NSob+nC(6xS#BuA zE1d`F5oOHbuNwToA2AjRwga=_RCr|l*rxpzTu`)dTt+Sl{Ey+-DeX$YY|vn(ZJMh7 z(N95I2P6wn7!cNXsG}3v-@9xu2pVpqGyu~Zk{(FUG|Im3%BJfT+cnQ=SrI!dXLD6k z<^`QR2u)7joMY@b=yt@$y4TW2y4rbMC}Z|#c9$>opZX;VLG$MrUsG=BtzB<^y^@_p zOrzZMpu)#LU!jUvHZBz2RKamEbr2sVlRM8mFFI$)0gisg-o7~FOi<_E#6CAkFSvCU zh7^=-$opskD#m4{0ml?Oz!aBnmjyi)r`1?hTw5Kgr}v|y>0Tq$V}c{mNi=3Y>0A?S zygVtDiq1!TPh-w?G6q9Qt?RQoC0-pTHDQ11&-z2FUAF4R@*#-v7+k*gbv0P0wax8T zvbze`g|}lod$5u=O-mIzZX*oH&8An7$?9CfDp~DqPQ&?K^yXRcB7J6U5VL4P%zVZJ zr}ggZxy0WJvL@~5$p_S&lC(xx4s+4Ke5j;MI|C*ZlB0YEkC#ESqhoEsAGC~f(~a?%UYI4I}~sx4av~cy(*_u(HvVcE<;U< z*x2cMaMAH*KWiB;8wUhn72>kt?kRuvKuWUSOOaa<-&1*M_+K&O%>B-m8Rsz~1-XHV zUIu%s`vP=7r5!8AxPfCVnGLb^(LeH%Wo*Tdr83H^P26r#Zc%F*rJ5LseH==42sb~0 zrrK&VZ|3zUG;fc*f)YW%o$SgPq2Oe(gO0uw0axNnll_ut=8Un^@VAv?Dw3EvqFcP` zF+}u;AHzGA5;?q-$2@KHKXX%R?f5V7ARLxT*_o4NYY;O5yL!F9pQNsM7t~+zi5}nX zg>HB}RkMac%+{YSYF;Nb9pGl#x#_|?Y0Zj#DZ4i6t#`ln6SAX-2}Gyi-T<*`YkD5s zFMYzpTINlAmUGVl+|*it?a08Akj#W3wB;68toa#xnN+@$-Wcbp(^CgHW z_R81N5XqXypMkz!PsKOZ zeI7%Dpsg)ni)|PN8T!(9F5Uo-Y$qeG|iRk!Z&Vd8I1W7aRR zt?~oODYmPnYU69`+SnaQwpuEUKn+31H;EL+CSj`m_v4Qq+a>1CT2)Xv!W~W;*W1Y1 z^rH~JBfr{F)f16CM_)iqI{C|+E=DZAU@BN%oDKP!GE}%}XX#lH`8#J-gyW3h4Aqg{ zh)ztib!@5h>Fsc*>CD?vK9~nCuU8+L2nsjYuk%8L|B9o9!v)?R-}mA0BCLp`bvtP z+Dqk^S$L?KF1z@bS@R%n#L^#!vUPiQw8VX-2Up#hF)`B{8y=%U^C`IvR@I?vrRAK_u6}{Cx=-u6yhpku zWE%mi>vka3P~&t;XslQ*Z)g9D^HLk6vSx>sMACNTfSuewgIe$|1loC5Tl+s$@aKY> zmE%HdVBiV`&{p3Pdc$!O`IQy{TP%2zkm2|8M>yz2v$>?hEE0!=C3J&;yp ze{w45=tMh1S2trAIZu(%HfyE(4fnS)9A%o^NIzdGy{(+84QhH=nJh>eU0DPw>{?D2}Nvx6M z+NAwhAJ;FO_(Q`_oNlH)@MSgE4QL_r+@UljDpb}3_IS|rEq{A*CBv(ftDS(Mar@vm91aZWd-q}3Te=d z*INg*oln>8dgvqg4*gdB0DW1*xr@9zreShx&-YL{=kv}V71`0cN26p7M6YC^4A1!~ z%mvkf*mjkBqzt#L@vI$>y{vOZs>~IxQ-^+EL>ILkXGO(U<;djScKw^ zi|q!3oAt4V1oWZiUHDC=;gX1>=|Vdm0rOIqY&UU9OOc*Ead#irW4pJkpUhyXJOR)% z?s#faATh86T7qq4W5FEwhkfCZ*>=oOrQa`nT6^I`t$;Lr^$6;I=Mw+Sx7VexXqjKB zT1p4XW=5CwqLP4c#__cOwiJKr7fN5DlviLn4U3gLudXP^d&sO2aoKi+jh0C)_8HfF zOosf+ACW-e6qCCG^woeWqlk8vX_3O%n^2-g(%BS}DMAfg9inJ_?OPY%4f^M2-Y5o}+2h+wmz-Cl;|Igz5 zS6{%LYR)66Ky_H9ElYe8FT*OVniUa`x#6*wYs#k{%|$PZ#x|E|9WnfTUaWPdzNN{P z7a?rGPFdThI++M$zyMRx<}arcl)3?v80H_9L6fY(#-(ppSbfC+7rTMqoig3N+>ikY zD>$LcCwA#@%Jh9uX!SMzv)@j~pJ#vmqm8#QsuCXl|Nj(PRcY4(YjN8I9u+)jdAg=N zgE{A6>Blj9@!>CPF0L$dj|7o|uil8a1Jx2bwd5(QTL@uw+T4G3{=bv^&Tt8J!vG!@ zNO0$b37)ujTI}T}uKne&eJ*`v;4<`Wc6Yp)4j|*qvH`Ip`Qx2Jrb z&*uw540tJ^iam=rW-n-VUpyI4?0(y;c}UP5ufM8&SzNO-;ICd=Nbpkqs~ zvi&X!OCaV_h?l8dXYH(4U6Xb9+rEMEL`JP|$lWol`Vld-W5Gw1M!vuL&)x7}*#bNt zXduO?m1EGVpk=Ut;ZNlRrvRFQ4+Hck=$lu}~lYx45p$-YQc zK{{BD#KSIzBjQ5q9MrG<>D>bp|~RKboq<0|NTeRKn^?U&I(t2cRm$-Hf-3@2Wkt1+0Sy zGJ*MprtNups=dL5d_%WFSH!|fN9B{1d;VEKpVUT_6EryBaS zt8OSE0PMNgBgtKWrT`_qGFZO^;Pt;}0P3o1!T#IlZfFT0ld=6@?d9h7nc`PBLeEU8 zQ0`~bIaE4EnD#~r@`xEWyjyv;a|xTLULh?w{w5eQh2YFn&r&Uh zg4As5jQ2PK8XHnX!>qfOT=@tdsyh5TN;k0v+R5NMGWof}DleR|GE0q*6gAoh>=$%< zImjqK`u~6QcvZsuabQ)-BH{P1F%Y}wcT~Mz)tj)d(VkC(Cj{`NarcCe!^n)AO2A0J zFwgZ5ERM=2JXu%hdk~$FQ3XkG2g$Gwy8qkw?fQUOO^x8f_vCoMy)eyl(crG9{G z#ixi;W}9Y&t|j+Hbx(H@o2?}5a)MSMNE3V{s2z)y{`+D$)|wEM_{dmyH6+l!<{&!P zl2N(E!$v^)8ihRdX7C{jabF#VZ8}lY3;`wd%Un%<48Mc&lgQia4>3>N9Jl$^JxgV6 zq%cY==~_ZSl_l8eSo!e(y*{+%ED1AIjMW~h^5Uqy%X8a6l#UNO&&7X{V`c8m@(89R&&2r$yx$p4+5Rm1~I@bsP6z4Gw72n)p5{W$O4MEjUt{}Y&*M7#mP^@t^Nx*xwLr9wEQ6Mq{!O6#vB{|2JSmh)bXlKX>_xV_po zzV#3xpmq*tXI0MJloe2{Uc6&9C=ht)u`Z3xLvp<)sI$c|^941*_^TI5&d=!F)c>^J zF6KRE=y+TihX4of!wu>rEXb%|Ks)Mxw>#Lxc{t$Q@T+UdpMr-a;BP)IyJ2Kwp@T5k25P2Au&0QAP=(Bgk%zo#?pk;EwQzO*m8Y~TI4-DzMn@u!V3dY}Ns zy?1%V`Tsu+$8@xf8O2>F8+%cbkml#JISP*`ewJuI0SOec%0EJt|M)Ak1yAZiih1IP zr04OAu-owq)5bpT5hBG74VR=?p{$P{S8g67tmbK&hyl$xi-|UQ^6_t$9{uP5j#41Hbc!PO z$vJ>J?39mLU<_U%D>^%_;I2RUA4EmXbnE*sS>n?TrfRqKV^v1}&;Ywq`D+#NLf^e^ zNBw#ayOUUTJcyEW|EN^|mBdWQ%nW2I77A*aoD}%`Z*sq?Q(j33JZZdjE$Y|Y+1(U` zh{3bfY36m>|J>iyOk%ENwmCCR8R;iKv?{@p352asBG5!Dn_+aUmqjq?8vm!XMtpv} z^hH3;8U?5b=vF>{{1`U7zYzqgM3jeO=g+HEs$wXzfcOSLPG~p1NGKX=X|c=F0Elpg z;*P#&Kb^McSFe2a`faTM8_@RkuZ<{$A@4-(|5+xcCM0Zx2Z>s(hwN1XQ94E3nEH=E zIh)@{|Imu%YiTBz^k|tRpWd1rSSquG=3B4b&}(11FQ2jHf~uRV_Hf_KN_viJON>da z+W&aNE043z#x`|!RC(w1pzYf1Ky|X)7sEAXkQo`n)Ay2VljY4e8Bh;`YRd=53aWCo zZb(W=i3JDQ52GDuY)^#Ru9WMAosNJzD)9lfn4oBp)3-pgI|RMJ;e2iDf}`oXZ}beI zZlkvv*`V*d5%RSb)BLE2#KfKZ`G%83AF`PF(bqz^y6*Vr70Zr>&DS{x1*InF%_KSA zh7wmC(w#kWUBqUEkDB(pCq;Iq^cD}M#2ed|(q%bH`VjJiCNtx5m|BO1RK=G`d_V%^ zokiET`? zw5kUy5Bz8el@&?VZ|rj-<2hCTslJ1ok9aL@ulKyw7eL7qwxKZ~Hcxys*0npDFhfk< zbjey&O&)1ml8MajmJ!ak3`tBp3K_I|p!PF_spICc@MlfnK^N2{Eiy1r~+inUyO<$LiNbvD6st%QjcNEY!+#B&D{Cz7p zrJ)}IkLu(%zbq}R!17Zs5O2v01xSv7n$4L(Pr{i&TkM&wbu7(7rO51Q_a^L6zRuN> zTb==!t)7pGrun*(F2=$pQ+b0+EL*XQ^-Z;g+PmiF2?73F`NmVwUDdSK)>D#FeuU%9+cAk<}+ykC5Afk54EbU#(1%e0-fu2Z4cTi(p6 zO;fJ5u1&}23nj@SrupT$T|bl9=E8Fedo*3TLsr_|f3mKKFE&fIb8UUqNvEQ)i-h~O z)v63BsnqpK=*gyEmft&CM0KVSQJutjRBB-ADYk?wKS1MgJJ9DinQ&ZNxS&y3+T%89 zqP{xM^H;t>hboXi_zBMc7Qi1#0yP20SZS9}LYo^F0m)$D(sC^Kc>h#H;SzY*+MX zQ5)>p6T2nxnWQJF)Wq&$o~hXDFX<4yq~aaD_mF{`6E8QI)$7SE?(Hu@)tg~S+3P|6 z?$O`yym(7EwWKJqD0JzR!ifK>{8w*0%X0rS46Cjvs)ji880T^-)<|(8+Q?uID^!cq z*A894TTOZx^YF}VLAV`6OJhye5T~KU>v~4Yg9>ip#4A!F=#IO->c&--Y59i%m7)@~Xg}IJ!FSMSjp@LB-oJl`p0r1t?-2M+&N8?7vmQuVqdFU1u zR?@{+;+lryB;ey7Q|QHm8?f6ii6 z==ImIuMbrA^OtB&Q94O-TP4a{_NounGWT3t7FWF8h4#W@%UF#x#r|YiwVmOrN9B2rOHLWv z`G+yT>8sMaf{OM0PCHd=SGHIo4e8cp%GS;jQ<2_VPEPq)xByf5+)sg9YARepCURQw zn3Wgw`%hKA{7jKhZ1T`fUCv{|VCYYAIjxyp#%g`5XK%#3P*2+f@a>74LvMwc;FZO)+SRr2%L|mt?tQOiqDE|F4l6o} zNLI{&d4KFy$|@H|HoAJQm-I^bZCmz;~*xiPT2k<8IDYyy8)kp8}VIW0dV+uK%^;J6FrFE-A!-8JsM9#y^ANksX-1(KR z3g@~}b-nJS$4f?*m^V_HSx?s3#3vhMUgJA2ZY?<~jQ&oO9`#pbF1Z&D2hjj`-Ib#^ z&Q~u_->liMkFi^YI-kG{wL)+y^;B>fVMX~tnJy3a;e#kL|2%nUNsb(l-&JAQD5W3u zB)HO)t-!Uwm*ew!vmmprg&y{XhUvz;fQ?H%Os^N_|0pN1(H1FqTFFw)+W@?=*m#lq z$!~%0R^v2!?c>n&UWy}E%p1jWwM9h_+?JtEdrA9S?=6kAKD#(c-aand{oZFjw!EWZ==$bVr7){nVx9SMYZ~Q2 z7M|@f8Pt3mD5aSiXBT~y7fA^=j|GrLpWh;sb$awb(|X#ns#9(h;$|C69(k>$5@ir& zCm$VZ2;1+|PmWwtuYBg9H3Xh`c?6aKKKVqS1QJWgDCBT*SF;tM9or76@;^yc!Vj+7 z0EAXHV5^~|_+y?_Y2wx0A(h;q+|eMOVAV-2>nP%S8X)B3Zwp>P8UG>MEk|TkrFg11 zOJsGMI_den;lA%zY3{yx!b&Ti9fc=Yy%Zbz z(hLKW%qtBAsDUx)caG+YsSzP*sI#NvsMTu}fCR3ixjS3aT9m@)yQ!oOT8=*kCi|1S zZnyZz?WOS^nRSamn751L*}I;x;?c;( z8KrxI*Oh7#GnhL-G3hvW1*24$pO{DW9`(qWw1k>ACSORqpBF72VDE1|*~e=&DMn{{ zyr!jV~}+mUc?_YyDgF&o}th55G{M4hX|g&1+(?%lTEUxNRxB(2#i z2~>8`hNSZzr3db%#|sUaUnwTjsMM!X-VqQc*}r|!H6u{a@&6}t2aSCNU@&LR@#=vyTB`=D2JbFN zJ)WoZKpU>S6d78b6%r0A9rUQ{lou!l5isbPvaw)euPa#Re?^b0dPvB5aKD4bp?`Hj z4~JB^g&64QFtI<9n&h7{{ryXD3s-?(AhTm2|Fixe0|Bubat~8n?a5Dm=8H|s zzh&s+!=oU7;qn=)Gz_+XUi_lE}!D~WlH zUr+*2>DaOXj~sVc`qS~yw{XvY?SWxFjAYj-tG5`U0uXk!k9{NKUGO#np_Z{}<)5bY z1clOT-A&+0B4q`=pdU`)R~eR+dK4vM@g1&P=9$Ol3xsy1QD0iSEUMooU-*0#F;{9m z!R}~R@pCD~FTr~OqAvt?IMa_W>th%r8mPa%r-ij>@;Nk8C-^AL+afX`hxoZ}@p@E` z^lCGYlX8kgH~oLC^wsz5uKld4RB-yGhyIw%{UM^W-*4H;?dVMo)}Nx*2YKVLOvJn6 zesX@Q7Ht1@rM;InHs09mTo4Ysw+1p^XqE9QuIkU8<`brZT29EJZ!|sLKlIrOY!%-s z=B>FSg0d=*?C``1ZgmuOgal0gC zsPFBP4K1R;l@RLwfbS57p|Kxgm5C^@JcEWTJ*(yniJW%x9m79zgE?b@k=+VM zcDL?H+0S0^Sb%U1U?JuTi^>Euim(EZ4n~6V7?2)-M}ibn$L%PaId-g9nSie)VN+!{ zKBG_+^))n^+u#kkMn}d)$GXMfPyYvEv_#&m*8GLnk287-Q3|UD1P5F(ZpcF$&=1Z} zrwOe%Pj-6*6!!BJ?u+Jv$vnpB!l(3L?dkJ^ZJ4@XHDk*!GEWDDu`@K)G z{&&!kp9Abk$_CwWN~Ck6^SNE%QkhHYN~If*@~q-FI15zY?^v7-yiP0 zw=5%t8c--V**P_fs_Kajbgk=A9I8J{tuOM|nn7NOAIYF8T*n)DVk+Fe|?5vjjzu!{gqr@Td&l9@O)6-wd=Y&zY#7Pc6o1yoK6}yMBC1*xcx*S@<)q zsk=B3yQGk}uzJ>DkE~hVuHODToldy6#H{`Ll%B$c{D>9t?-G5w2uu+f-(C|2U&iNU zRlhZ9HhP%8UE79pUQQx?@YtF&YrGu7cWLWDpgkEyQTDTs_~kbC@kWHQ$7YL^_S-Xj|Q zw)QWm|9$3wvGTNe&5o;ZS2+I0{oS)R_U{J>(B-;Bp+|Jah=q#))AYFWHgsmNfMVKB zTE4>Xc`~1y$6J@=0vJi|BgNL4kAHfniB_I{TA6CG9KoR@w3k$U*J~BR_HmU7|(E|d~Ii!dJ^UED*ay%|g20<^^ zQo8pSkfq^OcIE!=m+I?qq~% zckgSm-e9w9U74|JX6=6NmxWM&SyxbJvfWI<_J;@5caK$!a3V*yS2tbq4G)DT&w)o6 zj^9dn&?0}F8+ND4&MA@P9nRxb zOYR5a`Q>k^`TZH~&)qOaG|&O~n}kV>30a8@hvqk0T{-&j91tIeH!t1Eugl3>I(2vX zlJB(i^P#jK9|nIQjw1$71=bX8i!{qg`ZLFtp~J(>UFUf4IPGm^V7WXY;ceT}9nfT$ zI4GJN%0SH|1^itfcYL3%fK57dxUd^xb$BqSnpfvjya`a`V$In~(0$oZ*uQwy0 zD4WwZ9lsD)pO_P2#YU2{v7h*OgLIec!v5uLe5o}EQ;w$f7eYh8`$z9}kl@2bkt<{K zpW;|Y@*?!~4zoaZu-f-NWyHoMFCgTE`Oar}vct!lgNww{GyVSD!Nt+}~5 zQ8xuW<q@zGvf|+3vVLA1ccs>`dKD??>v1`gJwEBbK z33{}V!VA{tm}Yn>tLd+S%4sp5l>Ls5w~zJwoXhY9*Ih1ymjxbwF7CG-n`)9iy-+Mg zQoKDaBk?PVRJrfT^!GK&btX|1{*i`2 z#@GAZXW;JyuU8WAM1?nRD05h|`#*T082}+D+^Xva&;zGJL-AH##VEjZ+*N*t^Hx)Wt8y|vqQbVJ>1ucrx@y6w6w{a zPo|6Ayr^D#{4@!dUFLc03&!J*m8nF32sQoA)pWY6ZdeLs1l0Uk1W6`F|G zj@LB}E0D|Bob;^glR|j){vW#DJD%#lj~j2H93zoK8A(Oq*n53LsZg}+6^>*($6gf& zMI_a+XQ=G$9GeOq$Id>EeaJXuJI3#Qs_VM1`?{~+ANA-_Ip_0v&)0YkNl@tnZM^Rn zDiqYw&IcQLn7NX#OPyUwZU^^mvFE9xm&uCffURj5DE{7XZsGILsv46-%6~}`Zon){ zdQ}6f8*ht&Uo6(IV0s7>+9;&Jy)YPL10;9T2yl=E&ph()FrRG>pjM3ypb5!kssq zV<;K9sdscM1FGymy8FTj6RME(izF{!(stsGj<~3{^+ymtf443NFM_Z0vsX!}X_u4t z{xKuB96sWVIoRE{LHI@Hg>%7S{&k z*nT-1X58v!_;ePFY*!p?Z6$o`oKA!a#LH$eNs~dDp{nLm)8%|145Hm$II*fMAQ%>1 zWK2lI_5Mh}d*seNQb*gR`06wNU3(=P2q01e@*oYng`)jU%20|e#f=# zSu?h$IhhoTb4Ik~ku4~$PFtrORf|9D=5xTW8S4+{wRhU|h!W#53FWbihCP|XTpM!| z-G4{@x9h5gp$W}F@ILX|frY)G#+9za zIRJt1Ras|o-!(xXsT=7%xgsk>-61UJsSa|P4Kp0xVp|ngA*u0C!tSSwFVJ!x3j2xT zzabqYFod~(9<-VLe5$Y~3n@Uf*~lu+psi=oQJC zDFh2LQ;$9HTe{Sc+Q#Im-Qm>0e>_TsG(Oj}U`9iH0l@MRA9#`g+v@F)|D>~6-<92K+l?*T5itH12!177_`6bJ@a(zHy$ zS3IYl4hyQ11mKeeaAsk&nwR70fnn3t?1q}Olb;$BmaPGKk3)C%jcx`CR+Hs3-;8OX zw|5c)1vod)1a1$cc29Dow^*n7%{vRtWCt{5%6CC2D}#n2rv4}(N-nTr*kn+yRth?iZ&OAW>pdS>xB9|<|K|ZlRU3SbTBSu zl!KDeha+HDf>Mfo*>M)IVx{s?3w0N_JXzk(=Z6Aa5y!DS5Fd#HI(nMKF+mor_GJt= zy389dUbQss)Xd~z7!-8%$kDi-v+^^s2(FmE27*dXxK*SgQJli_GdAdsOBTf)6W#&E=y~^%O!?=9SsEgREwC}ha=jtN#J-QR zFeWQ2eX~y%F$!!FdUmoUMpahRePYMd!!rtvbMnH>`gbYu4Il1KVCYG4a4j+3BC z{^smhAaJQV9}n2cVAmH4_#NeOAZfVk30)VxJ9{e7ipg|Ly=R0aqLRC2X*<2srU1KC zth@nu&5FMzcrY9C&DNkv{%)Ei)@C74&2YO?gcuV&87otuIP*=P3jv4g)rlv<<;Fu{ z&F1&lFb>5rgI~pEtAqR7m&6gsi}{na!~Q=5*on=DLY{@!5o2Jk+JHLBSD3KwrcJOK zna*u!nzYg3tVQ>jfRe!0XXx8wwUGUhb+Svvprf-aa$IIJa4z-|;5@e6TvFJN$nA)+SCVJZiuJ>dE7#$zpzsdhopZ;N#NI#=tB`@FlOfuK)n zas}98rac6rYbS99l_j&IJrm<^(jdy4R~%H^O`!`FD2fH^d8Z~Lfqs=B|J&|CEJEeV z2WLF@ZdYNXAMF|m0hry^r%1blOy7uo->vx3F30apj<9+ahnAp@nyM5l4`RP*JJVA{ z2($*B;!kRJy2qcdIw#SLmWIWmP5_Ru;Bo@=8>&pU#@+MV9*Dxp~~UHkoHCY5wWb|v~bf<|DEPW zBY7YL1@I-esj&(Fxs(hIYHafTkt%Q`OQopu-HnzUV|DRV6!?6UT7OG70^)9#U%two zCWIQPno1xo5hAp;?G`Ghvt)tTQ-YBZW?KNCYWv@ z+tx+j=DIf;i4v9z1B}!?6*t1&4*rqlxGahsuN~mbCssf5m#*t}V{-$gjX$c>)wjqUry2;ZZHtdSwiHrX@V@-e+B?j^*ae1`a)^p5&%!MV)Z+ za_Kc(^5D7}rsKuo0Yn^1iZeoC3?i_qAA}J9+>zb#vsrAzk@#1Igt!5^iX6lTtEGLm z6c(#lw{_OYL070}6?+s_G%OPELp}FPuSd?w(38WDn^yAfb5$*jJ2mfHuwn0mYlyl- z&TDK@Cy8om=J-L~YPeU> zO%WKp+(vd!FL`Jt&+(j%Dyjw?$hPPRjyEofkzrqSNnv%BtIfR#ZIbh?+NfsYhEDTB zX-<`?QK(=V&Ys@%%I7~LazdHZkWIMB!cLtb@FY*pa;4?)T9-iiZfT}zso8LVe7iCJ z5`#jd!Z0CZFU#89HHeP(B6vQBUslArAjMz_fUK8ckf9>$)$e#zSk*V~^%y+CCJuKK z;1x9^Oy0chLiDNz_F0p!-;gIycHw#5C~@N+48~q2fWQ1QLy~jNsW6E%u{7X!N%% zI5(h8YRgThFx#ghe;tVczBVE3iz@EgN&GKWnqjt#r$h}bUpGWLV3MV^fzAXi$4EP% zj?fnsyl>eg3__b-o>nnd*yOR-_LDV3##^NqwxGBzo?0};f>&ap5IwwXYjKaCM>Rou z{E7{Fct}(!vYxV4lJaw1`QCYC17*IdtlJwiBg>Kyzrgm(K6n7 zZNjoe`iz+~>Fx;>?1VJfrjG0$uYEh9PWZP%WVg2QY$(B(L%UEul6V599t>=~zMqj~ z9KsAQmWErMA&^3N0t~v?Q|3b@{22SRSJcr*Bi+G8&bAn{S30s^JM0%WZ)lZi_R^SAbJ?H6CLd_2i3ns7#3>KDmA2gEE_KyLq5TM)@^brzz(8T%i1( zv7*R&(NYr|rA~X~d{H4Mw1As=5OW_$EUK*^rBy{1Igw%)J&(8f^7xlqIR)ehx)hNM z{sCwEqvLLgvKOR|p73b%HM6f6JlO<+5&d~RFPsmE%;S8%xswJVoE}a>a;oIKu!>af zSmgh<n!oBd0Dy2X8$$WA4;+|7wYhf=)D&Sq^iF}*S_Nu9Ukn@n@o1N~G$Uj!{dao+%OFCc1X{?6yli7*ZlR=hW zZ=w}|hHaIp*q29V87{N$q`_$U{M zHRDT+SJKP=%M$WmbkbtG%A=))+T&=vOqY^0>z1$cxF+vGas5pcmwT%9iqcICK~G8J zaUQoX03R}X(_8QGrX06S4$_6xqY}(q?E?y5F!xJ3Tow_uVCAb#AzRUv2k737A7QQl zy2FZXh)D`TmsG9-lnl+P*>8X)%=CoZL;(si;&a?wFf-X-83Kj@D9em(#5|t1y8H$RkN<#EY;vk4TZCqE6WTjRb?6`I{f%+bMpC%fFZ!% zp$%`*av&JFr$j5g8-U5JCJX-feJH%eflK=Ae*QeQa&-Z4vc4XL#%#ENanXqo6%}*2 z3==CDFJm+lwHv8iF$9(=ifjeGUd~XpFE;?DVRQY-nOyT;O<+2kf2h>|GHFWLPh=_@fiYpOasd*9H#=RM<%J&~h*s zEl~JXax`Y$Fwi498>1xD)gnLG=2ypU!wrA0GP2M-Dcpbw^AIvVIsFneiE9Qh>;6FgYu-Ey(q7JBa=F&Dl@^x!+yzCrb^8LP%U zZ5kt27x|JV>Re&9!yolB{HqG#8)dID0eu8@fLt(pJvB`lkr zG|*_D%CNTeCU)SmUB9cp>;@dVHKc=93u)RLUR1ZPR!`~h$ZmBR1f*Bo9IzOlk*v%C zRT!{7(jKO1J^7te2}V(EhpoRzBOAVE$iukK?$*<+&bzSK;l~|nbN~SUSOVYpZZ#`u zy5JcCd7`~*wOXf7HvX4A-xkYJU4H5BXfeEP9#9y+6h?Q44Rb9DTy4 z@#Yy~O=Dy^`xm?e!A>HBKQ{ICci&u==GeH6Vr7K9QN+ev23gYyd4_fMp6@)7Mwg=g zbuNz_1|aK*E1@ty-KJ|+)x4E1XK))E4&$Wu(ud0~jg zr`dfPq`)Kt5zK?<)Os?AcHUkFPHBJM@QE3l+GjR!^0^+~eZ_3=wRZyg8dv>5EsX83 z?9RN)(tfjIl>Va~1Lp@Em9(Riw_hHs|0DRGd5T(wmZwJPhipVpu<@*miDYoX=+1Sb zpmZh|c2=+At1Oh}^Ij$~^+X}DY$oH#_9`CP9GTm<8+7h|{~SwXC*9BLvGZw5GU%uJ zrYGY9UX_8H#(BFon#ui($cD1>^C_c&&H8|szkA&I z!QfhQQAJPDm=PO1R4!??Us2vW@awCe%+>q)w+yc-PVc|yCxU)WUTpHqIi^^ZHbu%U zR9UYJ&q15oA*cO1;B*>mslhS8v=Kx%D9_>yq(f?dmw%}J3%5UKs4?i@ALo2WZ|bwT zRHF<#X)fmSEYJd5Qt{cr?U%7a55}RvCTHjf* zruP3B;(ve`^?1hJGzg51GOo5#qmQ$(0o;1N(a*CZ(zXIh$5kefaVu5+$G@nLd&qmk z59qQJaHFx0jn=tshXQ2!kQ_>9B2R?bm`6!-L%3}fy7;cmY%y^=E6^=SAk|J(u`6^T zGn)JaP7nDHSoipES@-Z;bf{$;@GkGcKhK?>3;Q(FaGdvyJbuR|ttWi@IQKjI#YIJh z$I0HMoqnqZ_09gxl`7&{^*bNjL*@eSc*jjbnXknqHbouRkazWVw9oKPbQj-Tqc(`rlT zdC$HoLY%vbHY9G~GjS_p52mUzv5h-Cc*J5%Me&F2oLsWz*457EKfDZter0gxtVrfI z5{VLs{`#+=E(%XbQXrkpR?k0MkNoQ0f0>NP?&-bT@uRe~G{#lspy~wxYLq|u@kG}7 zfg?$2s~V8>fTQ*0d$*}s!atl^Xf;DZ0Xe3=ghX1x@+s~n%R4wO%reT z5IwN%GPg-b_WKE32fb{FO|(^;{XN3-IXapqy=5-=PCHkK8fjULg5L!&$n{s-A39lP z?kV^NZ?`FQ`zb8`5 z4~iB48Woe<Z!$H!&%s$^eQrh_y*(8Q+SmYV1&Yto!+%)(XrdnPDW4IkD1(%6%|72rQz_8h$8t~<+*}0 z-FZR~({lHB^D_O@!w;(hs)lm6JaokM1FHJg75g9bx%d63CHAbtfhl?)qhKW_UkI7u zuDsXk-gGD7B_?{mdJx;Xi5BU@Up>J7a5z!& zp=o)tST{BXJwKwjYC&EQ4(YGJT_`xT%u0V0>YMXK(S6*PnrI)RHafq5`diHF5P7{v zcyUti6S&DjqBttOpbzU{W@FYOZMZ*ltBOqz!gg7lfwr|)Ws zrUqDZ+w6cDe09V=dJi616d(~ESr%Y4E4Rv-w79dqu!0H@_kF(osQp+^UFQqsa^>S*Gt>3wTC7=ztJxuE_nnXP0;KLIh{7}ax!hd{Mq+V#L~CZsP^H?Zugk^ zU7Z{g*#ZfOOaW}B5pQo&Q3)&lH*EDV+NeJvbJzr^w>MxD1Lm<4!>_)Y+|;}A!3K1u zgoJ%)zOJh}N+$#khS&_Z8TaTX1o2!?dtGLR#lst1p7w}%ijo;bXq|QmhKH7q{1}L5 z%u?BnQg`OK7x?h}1wqgin+9CXKl$117F#El^H;dARM@z=3D?ZZ`Dfu@nE?eOcQWtF zO;R;dK!6Mw3Y>5f%zI`=^jm7DJR4270p!ykyyGYB*$F6{q1ebqzw?T0X#>~(to9_b zZ8hPH@EA3qW^C7f1wi9uA|h^ z;Bc69bv0N-!K}1uZmcC&3(@D@8w|y5$a`O+?70fQW_spi2|NO*#l;uQrn_qAgV3H? z1v0)xpY(1YrVquaDsV$0H`@Eu;-&1|F0{{I1A*k^wlX(qb9D6cP`gbbbfgPhaN|*C zYp+4C;oq0d)4yG(`*O$znu#MOT(#@4Qz_KkmCpHrE4W8dN9GR(&|g@hbfgPz+c2Wj z8$mgN8i#)tMGlUgK3T9>(c%dIqVis0`aMTl7$2m4eom=Qyyg_oSl;Kw1v+AIX8#E5 zJHXf(7TGMN7H$7^*8LuUXQ1?b-R23_dTe4jLBD7EqO*0j9YTe(~4*zQQ`)u5*8Y0I$(URjEl zCqQ|)krD^Kjy3OJ@dvb_e}sCja$(0BU5>&|1r~1~&Z_bX4@a0UrB!YVThA7o7tGBJ zo@mnJSciQ6JL4}pQDvrm4MN2Ds75}}uiKlLn(E(dA5wD(2wXya4xCu$aCYT*d=!An z`~Y;)OZ6T&3)25q1VDfHyObyv1*I?O2u-N|`W55+{aWSrE&v_cwflOfdn3-!fBgGJ zU7aY@7b+rSji;-}kYnIFLG%>3mbgqodVc}; zHn5?qPNu}pMbK%|ib^{|?G!-8T@8~f#Iv;LZ77d8WgQwF>FqSc2_m_>AjAn<){E=R z)WM`_QS3OuqlBY=ta&zP9Qw%)%Ami|pMYn%3)c7I$?kUlo>x8oF!~o5`B6jT8)52D zChk3$QYTnTh1}J&$xlEV^2~A5opuhZ9X3C>SyCUNT82=enN~pO$KPLJ8_XOlYnME4 zd~xBy>@Z4YHvS;hTVSlJ^r_1ISznZj!y+s4dNi5R?)PQqs<PfkIIvZ!y>nWge0lZju5DZMb`9Q_)i-v3H5t}y zn|su6W%TfBiuBP8OlpfL#J^oZup8BZUKS!=%MXJ1^L;iK^(p!g9mq1y66&yt@+mvs5c?9=jYD<<Dg3#9E3h z*?&!{vP}AoDa^0Ck&qL=yE)2`;>)fEpG_SvhOX~>8T z9(}Kr@XCI?(4HC8yKa^29=%!hW>aZ1Kp#oOiR{&q2$Yw^&0C|C>%N<{>An^#6k=ul zUcs`8!2*v#yZ^~e%!(E~-dCYamUC`yripJ)X9;@^wW&sYIl_vo68+}pj3h@@igpWb zHXAHOM{l-Dkl$sFiq`nTizqe@n{CD91eVSAMvCo5oO z6sdBK(~KW<5iqSyd$;~vH|gi80}qaAV5$y?2kbJ`GvWj%Gd0h<&+^LbKJ~o2i|o+K z1IMJvDCHOH*-|l=@t{*pXo{Ocd@GJLCq;Hon-+UM%x}3pxRC{R%V3KyJ0|}}IeCf~ZL7UtK z!Tj&{E&KDnH-L)5n8532&pw`^H>D@r(wn|q@&=lQ+Qv<_CMp%onsJ6sB2Z#!sT9K8 zwxArct3;M#DEq$JK0ZE0^(Z*pI`)#6h0j4wht8-^h4b(G612TdRH+@3KeO`pDZ|v! z3FDx@`|$_>mNW7e{_bjahQjXk!=(IcciPM#B@*Fuu-e?k#^{JkA_-PxU+RM#*W;=5 zpf))GumZFi|9)5t=wOV8>*_>&ZjW|x_LM+@EB0rn&9CnuLZ9GtGKbkct2Idvv(K7S zM5eYHS5&HPe)+W&4jM!wiV$+jS8`JH<{hxr;u_q6?R=Ob7wp#|FF~aXW5B!#@m^EC zMUUAj6qqqE7V%nV%PO&Z6vy5G4BmUG3c6quW@=DZta%uih>6Z=i><`XD~(`kbg3^R zEJkM78lb)6U=&Q3{^%+M-_}x!I}6x`LuGZ$n>diz3%vI<2R}JIYUp(!96*k!4g|Sy zD?>gnwVrrT1l`o*-lmhH&tEqoV-CYmtm6*~OApPQZaHTmp4-|;UU}WzX65}3a_S3H zA)t(sP63=T3U-kmOxv#72Q~D`P+0ExFOF(J8eInc$5J39fYJW4;}o?fk(Y1hm|rvQ zBUBLXcO#v_D&L9l-NbWpeI|asKw7rk(Mp9|UQT!THyML3J?Qsxi+SA@&XFY?I>A$mj{A&R|@y@k=(O8E)BOP4RLHJ&h5Myg;HbN_|*sdtUD_aj%v`s;TK*B9&$3XoN+L;F_%}h^50cM zUlUvw87gmQKZApq8t9t5Y$SA(Wn)o(=9g(&Y>3VhLV&hZ0UUL5{Ht`cN zmQU~`)0K>KXM}@kA?dBBun~qP7PYZ*jeZJFAGaNL#rj>F+MZs{aA-j6U{NR<9aZG` ze82Z3{)dTl72UP)Gb!KEmPpzO$^2c@b!``Zs>HUz>h89|%IfcFIAOM1now==rmR&88|-IDxu)<6-Z<64w&B8V zcJ_?Z=LFjDvKgycv4%P5*o(qq?~T0f&%SxIaC~s21iLfv`Bh;YJ6VW%_;+nh4PEaK zJjUiWwR}5DuOL_c1tg1E?wqy)J}xxHwXf$%jV%GjSI%Gy@4Ve3w9=O-3Ofd)fSFuT zq(?*jG-HcDpkmhf^w67rznpsM3X=1w8a$9q0g)Won}`@hWwkfZ#V2_Vel>#jWcMld zjC8N7NLuuH8P8w*p@x1Tc!McX;d`Qc17_yqWTbc~um1=6wU#D4pC*@6q{_G);%0`B zHd#MKI!1Td`fxY9jGh5k#m9Qi~3e4v1| z1ul2MQ3Gg@vy~`x#i<&*v79yNs5uYyw#)=Ou9}*=Jh_0h z#IwsNPWZ@3Pi$NTSj{cX?;e{Oe2Q~>oUma3F`GhR4zX%F$QbjorgOYHs+f~a{ls0M zXRkJ$UV7Pg;6yTA#npfiHMAT3#}~&<4YqbKIG46o=4TGNHghj-77g=M zC8$TKG;(>gMaw)LFUCon9c+AjD+DG)ito6@AW7*P?ARBk{Kx~KpSfd*winF_SUMgH z^y1T$y*{SJHa0e??A&-;b#xJA6+Z|~w)Gfv*I)hKOioM1NIjW-x=!wo{n1kDXmq@`e!W3uo3gOWG+h+85` zKr!j6gvC!9+SZ^yX&O4+H(3o`OudbsZ{ovzZO8XYwrV7Q?StKafeNH%7V>eCB2@yJ zVU>yJ$GgXg zVIgE|NZmjAltb5PH8UH|$nT0?X<~OfzsuyN`e)srM{-tL6l3=Hhaq+0v?9fAle{&S zD)AB{eoFqq-U#bJL`Q})z9gl_o;G95!02kPDDt5ck~8u-9LVW08v$*}Z^G(VW7yf{ zw3tm*QshQ@m0K8jc>?Dz##99qi9Fs&k!mchoxWbO$26Qk@E{V_xW!wh@{R9vU`<@M zFK#FcBzwFBnr-4@x777!yVv4#7G(MfqvfFG8I+5p)PB<`X?s04+4gjW7`s`!`=vB6 zHAjA{P<;^_b6fRmgt>vmjDwM0uI?OEaPU~`nM^S|bu{O_!UFY$+R;T~!APgg*sNuF z2x|qeB*F+dgucj5cBt?kPwEGjtSSSi$ zS+OMAYUiI^$W+(z;dX1Yz|-{ElXwfZeS)aN zE~i(>g?W_4S$?5HgAXZ3|Jv$dhYbJwnc*X|vF}RS1goP^cWy_Ssgx9swVwPY>KCce zAYn8s;WHSK{#@WFhw)g12-16fmDypov2hx(w($ekXl9-UF&EGC_%C8Ura>ILv6+Nq znpU7;E<5U;6IbW>YO&d7wC@MJYjeJhwLsO`W280Ddu@BR0ADSV^YZi+!*eFT_Gl%c zWXor6E1vWCQ!URk%t5v?0W1U?7Y?U{RLh3FZ1NQ(CoL{eF;V|>#Basy#V^;r=n@n{ z+A-{@=yFtQG{#>jIoBWcoa;?oBI92CWv&O7<_0`KJaI!ch8g24+Xo&Z(x?w7*sKSAh6F~QmBKU(S4M@S>Rw|i9J zJJ@SxR}uMfPaT<5dKoW)arTbE4GFWPK@vC|>-ILXntN&-pGgAI>h%hu1Kioj5}sP$ z(m0j-3fC6){3g7kW}|rCe0KHvIiw+c#@W>BTU-1S9-pNXh$e2?gdFGUzLEaC81K2g z3WsJ4gE0uzKXqo_MFulWUW=uMtJLvE*Vo(!fi4sr>F19Fb$Y?~KS~W1QT&`mGvUN) z+l$q-*TH%HOJ+(z=IEnWF^@|_M?z92Lr(-Oa?r{^U9K`5Q-|*N$CTY4&}1ZeFF+eV zgD^M*I07b8_NUTj{vkiBZpHV<7qH)1rT<_P3JYjhU7}M=@Sk7c-Q~%1XI5!J2INm@<4E@B=Fn=?w5ZVwnsKWDt_LKn&~-M zHgR3Z2V)d@mo1oPfPWnQdVa7%i=CH(5_hu#7Pz@V*lq?iS z3GSN&ytF6rk=dE8XWmOCo;s@u1(J#jl|gx&zmRnm4r6M(ev|S_ZDfZ!XPIftMUvU- z-GfcMy0en~?Htc6(||q{DekIdb3#!LShlKUt0>2)nhmKG?KAKM-l8#i__pxs_{Xa< z90Le)jdtCNx&s~!Vkk#)9ZqTE8)PP0KUrG84Ab5#zkCSs4P%>7QMX{5Rbcq|k`)D@ z`CT1|^VH$HH+XM&+@*+2 zdF&o$ei~lw9cjci9|#Ld!%1u<$m*4&>eyIptTi#{w6?7_@x0oE!=nY4BTdXK0Afp=>-kPx z6j5rjudQsaD#BEGxLRrKXbCRAYT(>>P4rO*UQzRoq(`?m8)GRvb57s#lC2=gllpjR zSx*OBw5x}LSiBU~a_UNp|8u>oU^U9~| z*#PK=I*^_g`MI{szd<6pl9Jl0CTZgi3qF1i#rnN!oZu(dS`#Ze_9<}z_v#vo?NBtG zEI3G8lXIN~@5rWVYNWU0ax=UQxTYfqohXa=%eG0DJESOaV(LG0>`JDF!@(54xo{Qu z8a*4AK3+!tj>=>E@kH0r!Sa*ev#yzUBwI(yZ+lf8kSH})m-6FmDF{vp`#0X6Ew(c? zY32qLhP@GnM3o^C6EGM0M8BH3-L0Ukdt^H8DjxZnT^y`9sgEaMO>d^`4pFwJpWXcf zaI#~rS$-Q=<@jpbZ`TF!rAX6~lGMiPTx|_^Hsq8>#$o;Pw;k3_uFkbI%=GA)C)`<9 z@cJ41F-=SAj>00_>Z24{M@>wLfyGE^BSVr*U;eOFhjLrJD>2p;;f9|zyq9MM*O9yj z?T}eNTZPYe;$T4^w=c!jtJUUSMU62h94aL zIdpEamD#+c%xJfBdP?$+Y>LiVs<_p!Evp5UlsQTWqv=56Y(|keNE(zzHXIux5!p^= zk9_L3pED}`H1(_HlyUx?H!flM!(?@hQZT$|pcuSV+q^XM9^?wW?vPu-u!7BlmbQW7 zkqv@GlC*K*{^xQhM6%Uwx%P-eMZApUO9oSTVR>uT^vtIU^KzXOCglm2)%CTatZByn z0mog?#rS6F&ARMOH}o`E=Yt2)ius;uUYdw7;9J_7yX3cef5^sU8w(&(WnXsJ5ESFC z5%Wr(K_G7zdZxdsI1jihlB-MP_orC#Cj?Tg%?Wd?_?sYF-59cUuF^@Ua*_-$&y&uu zy-n>3Wfu1abpW;>FoN2awT>D2pVQ=dRYmd3JHT%vj61YA_BD(Z;=p9(=8rJDnI2}K z1&%|}O9$uFqj7vWF9K<{Yf-Q9Z>d+%+W3(u<#A-o^u6ls?|*Lgjk#a#l~>Ejr@3lc zy~n#^caWB|-5+%9Daorp%_E*aUQwGp$##jzyZMe$s}g^-K-uwIL4v;1tXuDi=E221 zFf<*mIWw_2&NW@Ix>`b>2V2HkPQvhZe#eFk>%)#RnvNAAQY>@_uxV6xZXjX>vWM3f ztg~`OII+hxMbi_PWPJ(;?MO-Z4Mhj%}$!hdyxNcrwch#V5a(kWAGqBCUO|HP5%kWMUHVwU&HApHT z5B^GxY720m_mVa>i~5wMRoZE@_KR~Sk1u5?UPP12x72$PS6hCw2UIcOwaL<55r#5N z>`IQ7tTb&mbB_39b73`ifc`3q26y=0^?0lqYj$MRnwX@FXcQQer*RblMceS13y2}Z zzy|E1g9y75YuO#>%grv%Sa^d3uwT>^ohbLtVP_AR`KoW?Q-}IYiE9ozn)z*l>$nU$ z%Iu|z?QMTy@V5i`C%}dBfz<5_NZn}bk17v(FTGo4YzZ)_3=pygR=TokA)F=$D%NXrKIqByIqeO`3h zoTQItcxZ6kX104(+H=d+<0a)|>z>mbn=eBy!O03f)-*;a0+&jOz^0B}{vv%SBRvJ1* zf+n*UxdMr$5k%JhIGAj#M*;ntVtO>#^jsidf`F}j!}Xr`k7!y>gKDOg zt&XJcVhx%w)01W%|4tQ{qJK4E9;^rt;|_r-sq&QnuiDZ1Q-{@|fp61_O->YqM03%T zt_CXhd07toq-dQZ;@lUiY_4z}<{We3v~Lg_IMHXsm)#`aNfZrzH?zyoa@%chbLTKU zuaW^{7%ro1-gfm;ofR>yP(px79ZlxjT%m{FKT2;}t#(;c%*P7UEn@ZU$KN9ML1aAA z&tUq{qSi?6hhG^7>Z{Fm`Q*qvxr*Q?Zk4-uW3#5nJWhd*_p$n03eUV0BbRu9eM}UC!O)0*m8oaTcxYTf z_NyX%c$P?l@j!0&SNA-JG}psNITaJ7=mU-WV_U=F@$!O?uR}~#xJHP%6v`%ikU7C5 zaK!rIZsj0$t%|X{Q!a4w3Zl<@ePJWFRnjit8JMSsB@4c669FTS*qw|V^X(>S-4_Dk zjfqNUzkrjYYY|18CE$k8@ znhgi_DHi@5Qx{rpyJGH84jmixdwkkjQ(Cd%F&C405QD<*%=Nb^P5WkYL(#+wFEV)W zp}X?Qz>(0!8bT|>;IhTr^|CR=ihs)XzJBN{debAQyBm|;efBU7X62pZh#%aJ+WlxA zS?*--<2|y2S!yQzQn8mvn9(fq1-3TLHCY{}Cytt2j}8R6 zzCQj^d}I~hy?ZMZ=J2Rj)j7hSw;vHZt5^P1z65CI=xmNS@pw-gu8xJ@8Ptn_RZmD% zC6>?kwQ66xDn2xOEjmpLbo?iwcrmECH-_+gs$v}AQXWqE@;?VM7I(7xOG~dZB(T)% zbXApSSkNjD>dVTKnRj~qVr0A4c$IrcJPrslVw{S;SmS>pj+;st%SbM|Ey;S8*7cVK z%{y)qKrN6WZERq@^=s{{=rHj#aTSedmx-#`%Jru7)RrH-FTE5OvN!5tN?A&RKtNIO zi3pcrRM|Octwq5&YxGk)>5BI~fyl9*r zn3r*ch$QACI-kvqYGUIo1vQEfeeaxp)}LH7AE`L4u{zl63?i#gF7IH|gWk}B(;)L$ zPAv3D#+3EcY?LJL&FX+-ZR8P62Z&RsRb5~5eB=Tvo!5?A&5!EF>A$ zf)OvX62%L^Qczm#DyqDaZ3|@`DlHEh)rg`i#AT!$%m4ji*6;mRj;PmctG{=NOFk7_bX2n|Hq`*K>~q@ zqXDRv>j~QM!hlqbcn6~wv9y)~zeoNRj*o+;*utFOz@27Yvgg)k5as7~emuX7qS0el zsxy=O(FezDBpc(uzA6yYm%1ri;Zf63Eiub4>1S7KysUBf?Kp1UJn&KcXz$R-XiW{L zoK+%cq|G%{$K1?JqOW$(S)(|W4BGhUTy9oDFSo5(X3apme0$GuPMN@Xk;j06829w~ zRKMh%>CqthrqS zQH1;OyiQ6(NlT;e z)=sZ#_iOsb^*K6=F)MEdU#_RXGg2#EhK(d2Hy-b)wT1SSx@%Ee7YW~VBy>ojzVU~x zLIxZ8Ag}$BioM=rst~@2Jx9lJ{Kk67qfrO*6t8XXim|XG7a-wUb;QRzQD>ekEp~TF z*}K2PeB95>x`~^a$&cKOs|}(g*bp71aJt8Z>y|%uX0q4wkN{K(P@ez02&UGN^cR2d zLh$VlP4`tuwvlQpucJ!qS=96-{`EzSSO>cjq4Dv`bHS!}mRhS0M3JBw?PBgZrlyT9 z`m_Mc9+9W)cu8atN~gL&W1JOty|D1bkr`RR)L+uYV}@q46ZR0a;QhOGF0E;4tFq~b zFz|q)#!F35BZESdIm26rV`y4iaWiiPE#5zXcVMJ3%f9BG?W-z?Pb570o5>^N1@tVo<9catH!e$d_5w5-$ap@ z^9l{)Kv05jqz5)#G~XZ7Xq_!iEi`zX`nxzk62?8(e(i8kX`{7`I(Tsd4L+AvnTk}7 zyhETs*|!R_PO(V$8*q5fe^SNzHwGVE4Vza{u+bVSAOxys18@jH>krUWwd8+W9Rh>$ z2DQh9)Q%>*=u>uP(_12bv}ypUXFIKiK7ku)go8+Ftuc!i%N8O!9SqAnErz>d@?O8t z8bBEZ`62J7PYYUIxp7@Y1N`hGk$*r8vpxbDq>B~g4T06Ed&H&~{cX+u0Xbd!0s-`h zP<$_-6);fSV&6p^sQ+^QrR&9o#oI64KrP}gDCFMkDV)Dvn%3L@x1ZCIsjKrt9TZFd z4Ju*zDtAVOPX2C^;8_$5wJC{$UE5Sb!HP!yXUiMF*r-KwEey4x4hp#924X(!$xb8< zL!eLOKh5%g8p{9i)W2m1{?7y7dVMqjB##W`L5_TyVz)%7x5|kTI>%WzgmFyrd&^>p z5}gd%SEsRTRp!6n5QMk@8V|q`pjr0^5)yXGg!Z1Z1`mrO0CxE)A*#J4Hw}9bBl}rU zZgY3u+_&zZ7u4Ums_puDsS&mmit_D<1^Jjw|E;U0h&w6RtmZ=OO!Nyn4dUo(^!Jy0 zeIIHjXP%u{?GjW`8vkc?CPHe2IS`OF->_53h5vXxQ@q^x{9naAHXt#ny+A{0;0CB3 z|K1z35)IYSl*&1YZ{mb+-#?tHi{Y6JA0`;{xKnOmlC&$6H05W$&XL9fwlV-BvXok) zUHmI-^7kls1e7-7&=xuN_naA>zy0#AMG9;;vc2rTp`KvX0yg}gAGi#5RIL2z+QXc}VY83d(*GN$ z`7g8)^?y@evTXs95~K;}5{zI$HYNWi5B(?HS{c;WT5dSZK5u+7AJrVs{3kki!e*jb;NVWK!f;}-Y=js1Kj6P_Wf~5%9kYII2r^ds!(W&)2 zB|Uaon6>p2PJGfGq93uS2Y-g)K4SK{+C({k3np{cb<1s%II|Il?XiWXcH)3Ee0*d6 zm+(~C{kMSJ#aoE}zcIT|zDCVQYUp-v0l+@7rz&AZN8OPY)qz`rb_R8INONKx3Q=Di zXR_`<0T-dR;pj9_aP$_bE-43-aQYG5mUyl`Nkb|!lv4GZH}r2%@?UUcKramFN%1E- zm8@8xow1C(&(r{mhx*`8055e+!gdml=;W7t)k8C6f$U=$HKZIYB+DCPAZk^P4eoDjb`afdlUcS+AgSWJai9Kue&JrLc9@SI!X~%5V_4~GC zEcfge2|w-;dSWcppf$B5@09$)9EE>87kzB$?ipb3EA+aSGae$2_d=&y-4rb zgMn*P%0*e(g-A;dLPIwyBeW1%L;=5=?iU~plRJ$J?JDzgU(nik%OQu(FxZpNGDHXma<9%73&#*2la)g{#--YEfXz9P#ToVt_(hAPR`P-=ciZ+>g z{QuGQ)nQR?-P?k6_fXOuLzf`kCEZ<8l0zvW-6-NQX?WF9Rf;=bbJpU z&w0=Le%J3$u4~xO-fP{f?zQ&w$Toncd^WKEujfGn9sG$nYuQ(8e~k7)H=?xsFSB{B zmiMDbH9`HPocZ7wPU__T4P_BxR>p-DyL9n9ewhp%rVF$@LjM=xstK%lpFF^WY95lA z2V`cvEB8UMqVHj%mc&_WSIb@>sFz; zs;U^RF0C>pQCJcv*2}L-2ku$OKP1FkqzptbqI6OabtNc=b^(w~VaNQCAWWD}abLzb z%}S<(i6Pc|`Pqd(VrSS4khMY=UsUM517CKV->vMA=tj(Ca#m7Ty=#hRnvIfgL~ zq28aX9F3BZo7SI8;lQ*oBA6P;spe&x<|S_G#Y&~Kzn-1y%WCLlm^S*NIED?_>!&e^BE_$LH1Y40e5v5r7H0)+RZh;wh4-hyHYBQL5sS`wHcygmvWxt8A-RR2JB_s^{n97kFCO}DD z#Pp=NNF$H2hOY*wNh1ZgLsOuSn|UFkUA1yu(K(7KaagdAk$V*9u}pKf8gu?Ik~^+s znAySml(U>+;xocmGl#78N2o{~Ur@uOxy2$0>{NUS{YyaIpGpFa0v_RB1k>L4F(hB_ zZex06NQsrMZ%v%AG@^4SXawPx$Srs0_3yZ9piD51uI)eWVJww(gJH-*M*tC7&v8I# zZ+{lNE>#lP6F-eb#Jl3JOOBT~6D9K_5lUK`NLLjote@1ZShefY@Se`mG_jF0%LBVA zgKsl+jAKPuIwJPO{!ro9fg>gZrwa3UodTi<+2?scc2bXbhIIRq^7Hwm3`wobD3vMM zB_8L6dP-VDO0x{+n+eLUMEfYfFrP^2IJa<%H{{@QH484m>oPSh+sIgLVkv=xjjbVo;j*dZux`6?!us|^O zD6U%8}+2n7ounL=u)^0Nn-p6iZApI zFZ{)y>1g?5^JZqWd{XSw&yU{5oPFsLe@6$3`1zE0>gSli6QXU&GQs%iKErHn!&5#+ z@L35a#!O3HtM9(3iV-Y!6Ldnm}kqS1zIT@jUO>sx$0A01;|R4tPS4q{3D~ zuB{UA_ln1dUc^SIY=c~NYCN@hG%S8huG+|fR_nE=cf-Sk*k`{;9Tzo2I~jpzf1PE3 zVXdP!Nxo`rI}2DeOZ*yEouv=mygAN8_W3blTR>rW!FSxbXr>!@qhh&osLz&2R7Efp?+f& zIOX8AZa@xNWUE#64R0X<%Yoq?V9>54 zllhIjaE#ESNI-(N?B-G9T2{I-BqWW=>%%7as29nZmp??(*GeYuwa<(z*+iVA;y{ty zov6uOY%73(2*m8)p!F?uQZ6uCpH=(M`jf5T(ecjCLKS^h)bvg=PXYw!!DaWZ_YlgPWs8q;?FVq~S94VLCHd^RnC%W0S8lEHHC+A`fYlfz8~O}4ZWKY+ z@wltJyQ(nJJS}09nqj>WuOeCnB$7#X$??lZqP$QmZ^uk$b1-y#(D80mkC zWm55L>=iqe4RPr&Hvl&a2dGZ(4CLA(bm-ap>uPbfgKcB>$ACeA|CoC_{_?4#*|R^wzY8DsG%wGjKAD*Oqbc|lVzZ`h(spCn} zt&DK&zsw-TE7A^aOmupZO%#rw%5B_l1oo%KOo1g~K_+7;C_lmkbDGnRgTi6))qVfY z5bxmX%=4H*Q#m`8oasTV{Cw)duvUbL#^-azS zkY<7{8z2>T9_r5Zw@)xc1TTsC-gyztT^PO>Xe#)Y>)d^v=mSNJPN}QmKnR+ZbnB3~dbzY!=qV!_*Z1aXL|qPu78;ey7rkcgKoY z1Q&}i22;ZCQJyH6iZ`L46?{0GIeATpuR*Ub`Cfbk+lPzou zMeN$(zv9a(XEJT8{^;kA8_AoCH-PWJaC0X8yb02xCU0(~EfgvrahX>N5~pN%O@ZYQ zkvF$)gnt}>#I|fQlg33-I^YL&tPo_zQA~p!!~@~$zTY`b2OXdAGNjh8!jTK9MY3oz z5gQ9*Sb?OBLoe&l@|7R~z1|&1gfzB0!qd-N1IIOtqL@aVLmwBK|MX>efJ@{5M#Ef$E`|yY>{65?N?d%t-ZwqiM^pcl&MMc?7i~)9z_o zT_j^ASTeUGHSJb)U+E644(Pxoe&r#DFJ^?VnXG+k!>T$=;ooeC$C9{cJoiqEf%2It z2D6y9PP@>Z3OUzwrvsPG-=76%*Z!^ORab{-TM2}|oD>ri zmqQLG$IdlimD7Hh>Sz3fjFumxejtH{iOTyTUsQ(eqdm~T=Ns^pbL)~A|l4To}iND>slQ$qZ+`6XSj=gC6rd(D^$yS zgFN}8L84&Lk-nmS&OGTy{XxRcGx-UX+VCfwaH+rd2INGacK`A)KSm)U0K5ej!+y#Fbq{loQdq)}E9Tbt7P^Ke1$QYfgYH8(6Y* z(CY6b_!-0WuN+&Yibe>9xbF_iRq7c}F{ExN7XxpIA7ZO=AEq-(5;o{XAFE@LMOO{^ zqCl14ln_DRcnv;6*0Jr`Z^70_J~*VscCqD*S=-imc^1a+337%6E)A3wQ>vg*_F^Ua?|vUi zox=2f=$vFdfMriKw zBaGeR+!;!zDA8xfTS|w|A3Sw^PbTiUD!-cmeDH2mX*jUW(QrF@(ofd%e;>K01cwaL zMnl$;<$J%ncP&h}#`o2=RLLpFOd5eRKgvo)X`v=`>ar5JEWHFiOA+ays52BxU!l9( zuw;e$8b8e?FaHDZ_D}sB`r*$QV-Edwzy6yqnxrjuEY(eV07D?`&kk089S+It1|K|z zs|ehkjQRK-;9ASow8?bl8YlBpF(Nt*qf;GXgf~<`BajFagw`v$M(9HU-<{dhw+sh7 zYp0j@+=}+-^C~fYK3yO|zto5})BL+`^rbC$H;kw2v@P318>MkgXks|-84;d=43JAH zTD&X4QViihRbC1N(0|b(0{GBpkA{-GED@+7OlS40HLd7L2wGqT`6<#dIgF@=toy9v zmf$`u|Gs>BalZ+b3M@_`;$12HD7A3C$UFjG_J7iA^H~t+GJZekSkC(D&Dj@r-O|Z6 zpv=P)aGowV@bAGF@TDk;yEoR@S5crfn(XU!2*p>)l1jz>9{}5&8ovOFp@#Vu2!()y zy8Bh#FyCUyHFlQeovKA2Vq5G;_3gi%v!9eUPfTbR)h`qpHflSYysM2sQ5FCA{6Yha zroeBr+IyyOt-+)R?PMW>HxsjS%r8vaJwz`?x2B3|2lm}7wO`-Ff^Oaqn&*9g&n6$T z@N|y{sN!N>Nq(m+|3v~ENdrpaom<9NEIoAC4Y^zN%bonT=uBJOs^gbdY>yw5VoO;Z zFGS(fY6`G;R`Li2Pk{Tqunwd7n9So+?EmMeceYBS4A(k{UNyg3ISUz_0pIr9KhJp; z-Z$l~-0IrBUo~g4OVj(2!Z5_~-o}24q|NRMek>?wK>7bOKpG=B4y|N|%?;odilJ`uXYZ~>)=p0>xzt?*f26bkNM!kMB+4y(s0C`Ack7|Bcs`#4 z^fl3ij;Uk}@RFhf`KhNUHPs041`EpN)1SH>v$xR~`5QH%LIs|~&hjDL(mTxm?WhYG zAQNMC#d;YK=$C}K)#NDJg?@@#ubaZ-Vt)IP^k;(%)Ocl?0)Urs@P{r%FO7vDPN)6r zq=W*F)>#G_@^Vy^)-iM}t2|wKnpn&`qdr;O)osJh;p3KLDWcDepc9jGzNM>^w};=Y zBBPO|qVVjtd+W9V*j#?N<2H;d0ZQP2WsmR*NlZhEN%8Qg>r8GnOo~Y^K|z#SlRLtJ ztZ#o!^hz?yv9s`2-gkvRk*~Mnd`Irt`+;a-!2&gwdSxK`G%=V5uDq}Yxt;`3HlOLI zyD`t20M{pf!t~*WIkh8<(rWl`0?knaZHu0$Adf!DMLn9O$$9#YW1wXQqY~%csQq(& zBCd0A>*I{0cls9w2`Pv#G}EA@0rk-NrC@tt$6u@#;$iVY0O9hWR27CD>9%(;w5E1N zf6fE1WAwOwyngSd`&fi*%|ld(jtq>oCfE`9ZIkL9}xz zkaJM4uA-qdJTKu`x?>qqQp+>~8kaxM^&mjO7#}Q0!5V*s5%yEQYF*Y9cx9!n+_FvH zdRi){Cp7I$%Y!5WGcN|E_nu**=o_Xe>>8)m0ut+91!ggw<;oszF|2u=IM2tiWb?|O zi0;#HIPopNB6I0gBVawyG>k>3#qHe2MDD`LkUwalqZ3deaAYthNQb2%_i%h+C&-0rAI~Ecu!<;qz!ULNjrbC@LmE+tvj7<%zo_;QDHjG*Y z$}}!tLuUa~PSZ*^4C~KD1!!!p^!YTVB@T{nzJw+7olc6C%VZ^ssU@#kxs?su)aFBR zu=!E_=zkV4jV3$~bIq`fv7{RcGh6*^Nxx~xRf~0(SyKUJTKfbt(B#O6jr_*>(3;#8 zA-k6QQNqy%o?BOh6u-csQtAE28lx+EK?m(KkmIc>^TX%O1YO=1{?@m0*LVm8n%JKG z6=hpb&D{+OyJWLKM>)jCW4my`;W}%;{kz#S*ziRkY&fkZbgL~BRIhT2y7Z!t6Y{0~ z>;2Owxq)p;Fbi?aR|NGFq!kU&6TQ` zE2H4B=osa%N~7nZK)qYTY5AK11j11w1anL+tXj$qy=r38)txu!8;%Fwuh6&ic6WIw zPyuj|R=updb!HTCB@VuAfxWPHE{r8+MYcd$Zrd#2 zq#q{#+Uq}c!+K;`LV&5zITX@mcqZ%i@l|*r7l?LbGJQ5_`0}qngBV~lD>t3`MBtt+ z;S<-&f=EPcX&0~jFkiw>1q5*yFg5MNx`39VD7Cm+qAKXAViNV2B$Muh2jadY{751G z-hXH7`J!4X7~6IzfmB4L|IBkOGQu++Cms0i@LjgN z-CJRGAo9Cze^Ma?q~{vtL^8KLzOh083Nq-$d{LZOXGEKUN{q`qq1r4j?2+{dx~@jG znVxcQVAD@=j8V=#oBq2)MeAAsAt1Ea?vG zZ3XVoz$~EoK}){c%<;gi+JoNFH4EQCJ=z7@$#VExK)_lDrvNP#^h8qj)PPzXpfNR_ zpxYg|F(6Q$?h8@KK}xLH5lE-jCM&^uZ! z)cWlcm?y+V0y4j=@V9%GfOhLiegMXTq33Tx9JV31PJ6d(7vnVnaAuzX9I?7A6;W!08a{Eb*XM9-N#7g9&enn*J;u@Ym7+z2Qp^Jj%r>td()@ z7Q`WVgA(0W0iWDIz*l};cj6IQGFmu{_7~^wvQ3ot@?Mfm0q@g6^dhQMgGQsGW?w#% zlHZzHNvFD6u-dj+IwyfX=lESl51A-3+=e^eT9#jt2H>uEi0Wr43=6iMY5hi(xoTE- zmFVoj7@7a|rxjl3b+%A@P%&Te(DF$t?`opR@`B2K?ZxIGKV$4u`H;Ys@3=Z$9n?ZrUy%zyx@Ig*%l} zBmf!Hc`z0AJ1;Uka|v!h+Um9d&kAUH#Rworfi4c9QdbD~2Dj2aCGRJ>a7yjbte4{;`fqN+5uj4c z59l9(7!J^JbwKF=G@5{uD^XF74q#5*gs>&P!2pDvGp%&O_^5xb@8!c`e)*8k0K4eR zW9~SzCCy0R7R&2B&LyxdTW!{htuvrE#qrxCh;H0L_&0Yo_3eTDWe8MakP*n1oa>PT z48dN`0JrwXUI1X@-iZxrFc$QO87#(4EZvQnT-F4o1OorcNC`DIh@mA+6TDl1pZp6L zB-R9M_MasxqV7^AF6ph#xZjPSsi@!sD)NfxJlV!uk+Z0jzKTvkMgJ6MQ=mun_rI;^ z{q7!pqlZ`y)B$*3NzWh#V2jq|2B6?m7Eg-d`l=iuSpQnp5pXw3}AMT z3lut)%Hx;%2-3iUsMmkN_|Bkz<4;rQZj9jxq3o&tUp8xehk|L>IXWiu0uq-CV9|e$ z;~{t_#O?S%7LeEpK&#}{-|4jC+*OVy)AK*F{c*NPA-+{VygJ>!ii8NH>? z{j0Tk=Tm<}`qN+CM_@Z;RljL$N%Pd};XZ!HUq6+Vemjc)+U*~evvUI0kT>_9Vx6#U zxFX{>djEZ-GLfUVjgH^+c!8hU-vLEe+&HO&5l}0z^Z!xn!H$A2xeEd&)w`nAd8g3d zdwyw+q4HSs>2ERp^)DWkzh_E*62NO$XSJ(TK!)omeRL{!It8qz%mIPVa)Fzu z=8X}c`+w<_e*4Hz{<-ZKFU*tB>4gWNQ(XPdau@}!?3>NW9fjksbqfF2ze1_Dy6EPp zT%y=<{Yvg-i|xG*JIBH|I+1u0D>3Yrwt@6eHGWo(s-i^nu$OM=WlHO(Th9X<_1G_U)||% z6iJN{QrcD&8?8;7{UZ?oVw88I6+hYU@D40_=aWJwv)SIH*4ZTLdjA770Nl=ZO$~e> zswaS~JmfDJS_rElo(jK_k;^{J-&Wq9BB-iJ1|%@B%#zUxWp`0xQnpw;l;|xw;wPv* zVcvWk^kF|)WLJ<0Iorso90p-V_s!y7nhCFxgXm+E@3G31EOVqv%ZJ`gzQj(2{h%~U zvr%uYWE#Sn7i?u63Vp81C!({8N>gD%)UC_Rjnsi4((uPlLy_F(c)$Fnhv$*Pav?Mg z3yBQH9)=mFQGzX7ug_06t?VBx9C$W0#QSVLm)GrsIPYo`pT|`kqdbW81-;0qd~2-h z?|~TXR5E$O^8l72eXJu?zDyD_=_*vu$jCGXm5x{jnXnNJmDRCaW@Wg~Xu`T_3v&xI zd&@qF_ase3_Rcsj!k!&SfG2D-wt zUEns zk|GMjXv3wV^5utz-g2%J1FzBGt3`Ppqm^;)u!C!^6;`O%OV%yf(@S*a8oQ#~bE6~r zny9lJ1zqjn)v}g6yL>QOG$SZi)Fy-3V>$sAW~&YJmT-QhuPOhAOT?4I7d7x`fH%9@ zl}_FAkXqgNiYsvQm_9Go5|!1~AY0VF$4S5-a~X%wB~c$4U2Rp>OW3W6X)#SG%$syM zu-8NYoNi^#7yDzmNPi}`RgY4oAFb64&+x<9e4brwF=-9cQ6K%z+c<0tVJT z5qM`2eS}UiRkDo;W@;!O3lLGqB0J;JQ#EZgEdR0JCOUIXrSqQ7w>;JvWymfam4%;t z2RYOZylZe=P<=G2nrCl$>vYPvybZ0CltZTG0QWQmOcMc^rayXPwhow|m^t^IH#+&r zd;~X})(6^uI`Av%lSBfB9~~6AeM87D_Sq(p^0PH>K>(UvAKs10`4Cjn0?b9yu=;Pq zy@ZBBKEGEmkK`Z)ypi#5JO2;8EDBgoF>olnAz=iiSoAdD@i4E2q8%RPvi$E*31DJV zccTQhR8TU;_7n@i$gTk+`xgfbZ20M&Q8mB(GHeET@)N+5|7-d!@CC_;0ZyjkPrv^lSqx1rB;;D@3dE5KPRxOEdV@Qkypz!7 zjM-NoRvVd%p&6NxeurB>Qejpei5Lp{q8K`it{=2qzn{f;!+5Lvs`Y1~YV_i3$j<$d;&;;9r!%)4YFD><29dXA zpKP<|tYIrOX^AMSr(2x%##?muoReeV&6QCbyEwCHg{|9_zjM?7l3N_$1odn)6@3j1 zqN;{+`^5d%ZbC+K$h8a@Ig6!@lR(@y#`?tCzqo?3F-5{g)q17Rxs2pu9J(}(ZAQsC zCio@X#BNNd{9-tQ-W=XusrGfH>l2OxqnR#)?lhY#@4bipytF~T9j3K24gmAt^N>$) zRvAEId6nZiliEhHiH0+2c0JhBULD3})-RfKWudqDp}eavq|_x=&>O}tyIohqYORt; zzgZj*2dbM1ZoZ7aNmIMweN3PgcFuyV!KKwkje(6SGhl`p_P7=$%PKW>0%ItJh|3p8 zs7`+1eI88H`t5P0*r+-=$|&X&hWY+DBKdAr#y5I374p5#+@FiptN7|?WYIyj%<2&d6FeI0PA4{*dEh=FEhbKWr8ZEl^`5{XYDkgzBT`XTJ zN*zZe$9sY@`T{mpzSP%V0;EJu^Qz%Y*lF%Cxud5laj`ful_U8$%Iz&bTI)T8748gsF|?2sNPJJ?&}<>jN7Bi4ClZ084}96&ph^t6Q%I zw%hfu%-F9SA?>(+T<5_2YxMjLr#_1NnoDQm{w>*!C3}R12>=}Sy z8~Krx6Kcdap!9S@Pfx2()}+YGxhlIL;g}$gp$Ow~Rh8?!98_Az3jJZYdLQ7lh*JreS}S|R_-6ZsGD-+qYkHX3NzBsZ zdOiU!pN5r_qwL97BDs}B(h%R4fg9W6rOt%uRmab)BAm*qOuwfLjK=1svar2qY)4ITE1Hx?^Gs$7 zQ^&4<>s4M<&6TOI{XtH-gTxBikJoZvmbQbUaw{8P@urk>tCQIX8oF-*({FV|#t3X>$s++Wph$pq?6|whlK7 znSl}ZCJ5h~*EW%j%_XB75z-!-LN9b}c+KK|MRT_9r4b^1*6E9|n`(ey4#(Z^8xtg6 z5&6ih;6#vZU165{J16^qSBFtuF|z0KeSbZ^YK~l z0(A^ny_RS@&0idyKcownqHBpH4JNUjcB4kPezi+uckS7(x1Hu&d++G>27CQ_7o^-} zXvB$7Fd#}KRj_3~{6_1ya$>m;d?pI?@aez^T`b6$UgC|b&r|0sU$5g_nqiBQP@^?H zU<%;hohTE!$D;~(n=%b^leC;wU2JcPUyozmPY_?lj;`~1COEH(`pKX~t%}x61uH7N zAj|s8i3acX8U?Szh`hh|;3co$uFfoE8C?{#f|`WMWO2-De$}wr-E&FFwb-aw?|)h-YS}}$`I8_mmnkBS!`$)9BVqF9og2IkqTUE~U}#LXp2%vUnfvWk{4Wpg4s<{g>uBApi2XHpFnv~`J+|FAx-+WPw;8?;G57y1l61Co4Y7g9> z;Xam6qj>z>qRZ!}`C)3`7Xn*%Hn4tlEh392?|_IDNx_v6m0K7AGBM*y{C;PsyYKFc zgJTqW3s)Jq7HX#>@qmuNB=7+ z0J`m>`so``yb+TQ3~ub`O~3QL3Hr}<3?_io&`Pj<*x@9!_c;B6G4tCWDOg4j2|CW+ z`}Wj`-lSto8;-KU(hGXt*>JHK)4WR?szR%Azg9k+gvZ#k$Y&JQ5>#&50_HMlksT&U z<8nYmnsnO@$`ubL_6MBgKSE25+}=>lin-FmP)-CNKGdTWbi8+sE_wg!>3-O3?yI6t zqu0t=>(9^M{@e=o9M7yak@aw{O5ud5WF;)cS$)}bkw2Hbe<3HUO;$4hUgRa%xN9k@ z;qwE9SKqizU#O*s*xfw)=ui@GqSA5w`MG~_)_v}o=0j^Cwb0 zy;60#|DA5xoW6>le1x(oDc7K?26iG-3cInnmC)`UlXqNvbIh*y5bh9hXThPkekgHxto!!*15#@aNtNmL z(MBh2|E0@Jhu8;;dII5_16-~zjbw_C1Boex(h%!?k+gD-7WFD{GrSI@UsTi4iiuQx zoee|uMZ9B`5T}LaMA6{e$d?&u;KW=(!l8G(t{5HSwj2r1o>O;RUA&nrH8-t$=lOE* z?Pb4AOjmx;62g8WR&Leo0nHB#dM8gn;-5?c5>%VhsYu9n zn7Ujk<8fA}{V<80DB?lI7*(uD3SB?Nwj$LIzbO=K;g84pW3^vhVW0YE4^~+K^QV~- z>?V&do4zzAl&~HFIkcXdH)Sg(CZeKcQ7gqrH7wD+!7bPxVuKUkv%!WhWtUA<$y-Y( zyHROXYK&fj8z*o_%Mk;KOsa|~{&w=cxPg*D@yXg2_Vv+6Mzsv^;w$atm!$U>DW(8f zJrI9{x5%DKkWL&e zLf{7M0Uie6M^CXvvW>loY<@UZtAl{`ttE$=y1peB&2r%3Cn@di;Vc{IoY|f0W%pC! zds_p8&@bjmXn(LH$&9~zqXpl26og{q<}gALkA^+{KQgM>vHP0ObWuPQ=nkk zCUQyVph=bS7?{ANzMk(9;2>|Y#{`-S3I=%Csb53du?xjuq4i8SoY7N)u;akM5bB^! zl%oi<#xFp}lK2bV@6rnYQ6W0pTXu5V>+1a{$NyXvX~2M9;^dAi#k5k$2Fqb3YQZ!Y zG*Rf18H6aDL**u}IghxWz8$T?peU`1Bd9h*+r#M|DXS`6^}?RJdbNc-XV$t_ zc1y6>@zxRuuiY}@S8LZqH+7ct`>)H*pj6nkQ3i-HS#- zpePQeW`Z)W%-6l;w-i?M4&IlA?i*%l;0h}l#huDwMnX=FG2K4+G~s2LRaS_6noY;& zlFDIDDd>u!aro}hLWB5pwIN72@RU@Z6lRzUp(onk=xLGKC%U8wDH0Gxe=Z}@G)U$zfv{4rF7vVJY{86J&O8C^ z31VD`hJ&<47)|liK)-TWCV|^m`KiN})|>6kVqUKWnRkH#Y}KTSO5Da@t$XQHIM>71j@vHex-7-6k!?UPHH{D(gM-R-p8mKdMY64zRoqI>&|KgxgUoRGTD z{k1VF>9GVJ4^9XL%@2gJD{I?fc%RI{NL<((KBR*kweW!xe93(5&Nfe$?bX}|=;Gtl zVNr>cvh8@u-E~x1J@_s9dT@ygBc)dA9i%-^`en)oDA9T2vY!hUBJO=TUTa|4CGlzb>t=jV9@Uz0GgC=vr5M>pdZDWKcxj z&b;TqG9QCO6261zRGMo6*>-n`y0SGoe$sLMajG8DO19424@~p7+fZ(2_nJ;#wv%gR zemoW6GB>7FX;}6+&g~h-_nLC(Lnyo}#XM#=7e-^_b%P-%pK8B;%If{_m;URCA6V*h zMct~OzY}6m9pq8n_iV!|!Jsd}Ud;0gFqi?MC#3$s=P*}(jgwo^?LxcO=JBraXR52Q zzhdZQfaEWp@kI ztTD6-=T_TTV@;9VYWowF(IJ^QcPtl)-QK43V(rTU3>w%-53&zGxezM10tr#Gx79cF_5?`ObHf|+VK>NozS zY`1$#AJuy!d1EtiVuiL5Z>{CFZm${+cH+Zde|{0H1u6`e{~&_yE9Y-biDEyHl>+8A zX+}w&)iUY|;+(F~1);6h(Re}$3JOh1A==E?8iX3@EauI-mxO^tILm|&aLUYUyVE8N zbJ0$(SBKnTDLp?GS3K92f|Oio`P>Bkt(rRx9!K=ZS(021s{5-oLR74`88~7@ z>cT`J?hsC*r||Uss>6Z;2f5`w=)X&$tQzq4KYOaXjYtFoA=o>}(^e6`Q1I zJ7KQ7*1Hm?_kEB|0#!oUzmgbjM18H4?#M0~6*%(2CpQ-YX3ZyG-X|?zBB&j;9RBQ9 zP5rdcM>@2Wd*f%aDZP}fi~XySKSP;7#4_3D4;40gZx%6fg$+gW`y>&bAbSxjq~H{n zpqR<1MmXpGN_aRP1J_>wn<6INpr^VV0-X_=ez^H`U0}`Du%x5(CO@@q7%Rd8{}K9F zy8L~tLdC77nP7tmb0{C<_iFxbgU2FO5p!=Lu^Hi8Nri`<Lx`1cIymj$-B=kYhh0uqSlK)7^bFuCv>kH03Zx!}T@LH)RO7aI~J`eDF z^8GyW*;c>yS*3+Yvjb`x3T>MYS>S~Nq6YLJAAgavZ~dW$4uX03ok;lw-x*4sZnj5M z4)oi}R`_I_?FQra6eHXf)iJkq%G@6Cg%kR?6tO4g3cIk#PrlBw3^Omo!~9~S0dJK@ zkp`wA68v4NL?@!Lzo~AgWmxIMF?!On7*N#YWpW%T7}w)pK~ft)kB%IlZ(Z4eWZF-X zW1b%0IB@5n6Z)&`SyM7Cj7Bc7;>iFX&V`=72qws;Mtv1BPlkwGS3OzU@BKx`<4L_j zOodj_MOF{#!CI}oJ;6?pNTGYeXJUOuYu^MdZMX8O6o!nwMsl_!a%s+mVwiUw11&=q zWef=^8LTk0MEf_HucX*{Z9x{J2JDkRNjplXQ!k&ttL&ysMhB6jypOFLC3U=vK;DJ3 zy`3hUFh%8dQb)kzOyMbzfD6=s@w@v7TUI;HcG^jcpLfEs1l>7DlZGZzU}~Sb|6C8+ zq9;uSGZJw8?811DDddLnR4TgTg*^FZ{Za;J<%HMRP~3a>ZjV*uW%NEM?$OWj%hMEb zzbR!vY@|RG=(o0-nG4X4WCdFz)o%)rLKGXw2rHlVYW5-ua63QSuck?T13f>^cu{-Y=if7dvM}w(!0@Jnz-+K1Ps9c*;5Y=mwe7SYibg(@+bn})oFDFnK9cP) zQq5vpPan0-*|4- zcgnyDLk}i;LcLj0&?oXEMz6!4B28yJ<(@vjR02b8I5o|}Wl?r3H~Q)x zQP9$L*zuIc(CU>gGfa(rLc0dSzR`L#sR}CEuhQ6^NMNVCPR+81rVq)%XH0SCUxZ0cNp)$&#D_AQPq*~9Gf`O5+(*F=gDjdm46js zm>w%XY2e@CXo3Kh37;>|g=0oXY={Yj2aaq=bbmA*tQjlg!E%fyA+DFya&)m3sAYt`LxzjPFDC0awTVYg-hZZ+)SN9bw8;-5H>TzQ-j2)fP>tHb?El;RFe zC|AaUMrW#*%~IYt?jH5NksAnTzPY-8w$YEZO=OnMg*F4*HSv8&9lG36LWEEdf3A(O zhduT<3iBgt{B?L#Cx*zX937VwECPZ#sup$7t9RzLkLNh~I+Wc1w5XP$itInN?ndaXU_A12W%-i{U zBT%#p9Q2$Qiy|uqk{p}ll3O)v<53W)@n!VaJR-)x_(On^01nh8;_20mlseG1?$v>F zJ$$kl>8P7Rot}B}xw^KyKT${8aRQ1*sW!?aic$AqqS&p)PC#;Bz;+|R=_=daFmb|k zFtJOtP#eXmFuFi4h{gpG*&O@1iYEFN4FcZ1*>=3&xEMIlm?WYN!**)gMJ4mfBz7a9M=Il z4#?dlTwhxA4;Fd_knNZhX(&IFRDt4j`5m>8A`yTK{r2U!(Cn-I7Owo$SlMcwhjcy! zSoya^vX{Y1jkx06FT!*qk}F`?KFo7I zCm37{^>_w>O@!yv20;y&;AsCasXb0LIEvk%tiLvg>*9HJ%7dq0rI0rRYvK>hfo{4p z60t=JvhL=Hr+ri+IUcu3Bf%(OJ$v$5SMw2~#<#l$MqSrxMlDaV^WJ%q7CCiALe9^3 zJeALw@rhDDnTp81yAT{#M;BfD&W1L_yiJrwgWb27ve4{k213u`^w=3O7X9XL(D+PS zEZ`S%US}|7FaIk2Mpl?}DtOZ@|F!4Zpe(!hR z_xSl21=K||#tDtu>aS`#*<97~KPamq&y68@ke1JzoV;g`zL>|O zNzpXPH;+IkD)*_d5_&?)U}s{8<<8@W+e_u*!88yW4yD46IaEhNm9;$v|EXhG@vd*a zLK=ZFHCb&r{s;%2A_LA3La@KQU#;epnXSh&r=lu`V;|YKRKn}+eLSN*GGSMJ= zTGC0HP<JLaei?jn016759@M8hOYjgzPh~aF(8h6gdva zBA8ZufvM1OzD-SV|I&xb;~fQn+Y77p4dNL&rI@6}*~U!qt?gEauV4KFQ?~~iBy9im z-_%?tCB&h)3vvK7w{ng;%o^J!X)#XdZ2RwC6EF|~gBW|MxG@I29!cAiTMs6p;H(A3 zV>z?)oxVj*_T}tUG(wcV)Sf1vP_TL7c2b?8zkO{|m-jl8YWl)v&<%ibi$#Isltt*P z=diaxsH42ss@s*e5zG4a!_k|8{W2~<-%OY)j3%Y0U^Mc<&&nIXRmo!sCfTSUrpUyL zhmDiuBKskUo5i2y_a8{rDm)=A|17;15?q8l(diV7^A3%O9dg5(a37;fLU&vO#$AT+ zc}rbe_Uce5M_Va0tCT`2WbCp9^*>%vCF@pB*)4^f-f@SN@~iwiHWa!{5t2&Cds!MN zt^op7dt=zI5t=@E#;F$tXRoNX0`?@HA z0zqRT8gA*KjkDClmtYj!yvv#BdQ*)bzE4-8M~7CNRoTpv@Y4~@iG9x0py$1mi}V0mB7zA@fg}^+j^g}AQA^g`KrXjAR6`L}* zK0Ja>uyZstzU(#3v(iD9(_CGEoC5umKD!z`#VWQvAD16q_6I(N{TndC0)UpxxDjq{^+Xm?8r?;P{1hPI38~XS# z6(SDezxBe#%NJ*sbfGjmJBa+40=zGVWm{I3c{5p@*QJMbH7^MLI9e}_yEPw}mLZe1 zP2f(C7Y6y9Fi;*C0(U8FfZb6F8{EBbK~l1^J5SD^`x3bp#G@*6;4fwShP2p>P8mn* z2XPv~*jIL5$D(wceOEoJgLW)IH}C{!Nw+;afpQ5~SJcE$;QQ>2UFVEP1mZQ1%k?u-MO@0|GanpaJ%D}+GLwy(UADd!>C27{5KN^ zt39C=O|d7BGNKINlX`RUJY?8-ls}{3r<2ctNRyHcbt-cbe-tudxUtC#l5%mjJ{o@M zMf;wB@WmtI6{TdjBolCZNv~{;y`0_8m!QOLa z9uG~-@WL9g7*7L7*VKvTn)|08*gQ0_K~-OWB@(bP;Wp=AQOtiFD@&+J079w)!1KKI z=)>~lIvh`6U(PCS_NwTBY|n0c9%#nCcpIk23s9bkXIf{FYb?G2q1{`@DOP|?#)r14 z*sEq~MZgKUO`>bZjAW{@wi!&CN$k550(4w`_Kr}@uzwLYfC2{oA6U|qzVYBd z=eqW+z1F$TKu9rKZPD28kfqV!_qMF!|A7lqz4h|AXBwy245a!rFV~Hdgw8v|!(1F+ zvq4oqUUpJ5Jrpynwm(RWeegtKFDP*7V{8!#|IgsL@YQUQ{lQ&&%Y5@0$zarrq@zoE zF2G!NwU>Zh&NDaIdN8BlD~aX7iX&iqcH8KW0r`5!Pwdq@-!kUaH*NB4L<@KMXv2!X z=R4_UAY=lq@CkoY%C6K$t?bhow>_U?oKXfaaHF(-LAr!mr41XZ=*+JK^JFP4GJL~pH^=&-}O(~?0hZb z`|pBzAwzwgFypBUII>2zBt}N>-q0?<%Y3~I$F<*so9?+*w@&3iWKS@wZUPb`ow6nz z7njBh@Ap_S{!>v?l9^0flR8Qou?QR1LidDfG@1E)+qXtl1~=@f&aZ8sna=EmkMJtg zFw!eSmfhiZc&{u8VRn;g5*oH6Uzj~n3lvz5=Z%{T69d9#z2x)AHnWZK!l(T;X_~Ju zP;_PbpBeOMXumtS#FK{o9xL*bxmm~F3(+xQGU+~K(_eTiXzG|J>|msXT6QJ*E#!EI zK5tVOjsy_>t5^#&%K$2xt>-PN=sqvHbd8XX*LIOUL0Rl5l-ZMf|2(tM?z~OjTJm=J zCbV2!ESw>w2ZLrnH;g`Eqzqv;b5CJndO~UDcVg`pU|oVw=%doTBjPo!3KqS682Vw1 zo_jxZw)4(gpGR>kQhYKX%9X_BGwKUYWeD_dUwt!fG1;cExHe@@9=3JLcus)9CLiN0 z&t~e)D0#n68=sm?{q6qsn7>}ZiMna-tG%De$*@0g}JU_*Kh8I2?cxV`r zZ=HfZ^AMwn;lw}%1s7y1e<;tV@%i8_p}S;nq@iU0)Q8==h!W2NH;v8f#!5|R)?NVz z$>wIwt+DU~eGpNglJ48R_QK{5olI%(!{ayn;(T>>pr5ZSG)&zToMdp|GPPBu_xffh z83diEi8>t@h90{-6J?NO{e2<(_oh1dyRD(wWy6ohfbsW_DFpGO?^L`HC!rYu(}iNf zbVa%1ezBflW?g5P(|z_?fSv3`8IAI(AvMofvz($%wNx_3a?kMm`>Sle_t z`O=pu3fDGw%Nbjm7IpzDJ;z@J^B1RNhv9uXb!qCR-@Z4V^mp7iC^5Sdf|Z-EP`r3)vl*M8yylA90P)wlnxZ&ALLDX`t{hiMPwmw^7XBPV(_{03QseDQmo~k2P z*}m6?ed|mcf!W~Kv=!)yPK2+6L@q>uOCz2`T_9RBmlnJh!=^L3n@2i#WIfq?-{774 zA{nVB9Eo_&34d-ulXK!4QET9QR^ATaE10*+CDBg!{hz_Rh&MnBkx{e*fY_~Cr%ne6 z*BiEuU!?*j?%-s7j`J~XWIXxxUhSLr$lg!1F9G95ID=7;l^V^ng1lk6u=T<0*t3~x zL!AScXX@n4bGEz?nsIiN9<=yI8%Of3H~~Kqt$hkV_r#N}qCbpb8lGiTQke+Ar&Z^( zq1K*>3#y?8{S^7)zl65iD}S>qw~Lb*O>YG4G_Me8;=D4fn( zwuW7Ep1&&zd(9KiQ|N6-{b2HnqnTwa+$@`$yR1Lndpm7Rb1jtHls7LkpOQ-ozTwJDQaxtPDYA<*dJ~{D*W2F~B_3WoO zYr+%skm8Yl^-7{SKY23$EDj{{EB1Pd9C4Uzeo;f`HC}i=K3Uy4l63SsUE35=>igAlLKX6+DHMLjd^)XCRIo+(39ruUYkn+YcNd-zE9v zqo`C+WHE>}wsxhd{zXa(9E*ltur=QoH(amow-yMBHMhi&r7V7<1Yw~Y-dfzn0P+!x z%mJ-=GxK$LXGqHf2l>lX>_(L5WAe)*Z35wc+$ z@7n-L^BtuigCRe}30NE0@(R-2H@j^*0emw;*j!72sYH~*9H`LqaN6PmNmUah)S@P%fqkkjd0mM}5KM(yeN@Thnq9rYPKJ7_ld;2A zH2W3SBxn$NikaT-Z;SgO1u?I=l<2>P+i70qvCidR{gnS1k6n+GerG@jlUWdbS}J4?VDyDO#!nx; z`98L-RK4TU(_bx3?#0-fGG{xSKpzKE>+L8uV=wZs*0e{g6Fre19nLlwZ5irGvXkOL??=7&Pd zLSds9U?5L=rc|HGYSh#*KhSXktBhwVhg;PZU#0MuMsWOue=@d}GWB9I4XrI%zL_v* zw`-xKJThi3{;S8PSla68vi7W}pu#WG7E30=JVI;hS zL&{$vqY_ya^$!7lzT{7@9Lt`?0=2KV$Nl+9STdQ6UtV>Pz8 z*i);)993BBqXlM*39ce`_L5lTFH(4!W5p`A3Cf)%DhNuRF?McAH5_U<9+*4HiVEK} zDa!GfMx7V(j{XjRpayZWzCA%Hj)gvbw)TWF0PC7oEUS`V$r(Kf(V+m}v_eoNG4Kbu zi`9+AEeilREkz%Ou;KRe4{J<*7uaDhgqcglbf&L|OiIeAw^|06kGW9nU|^ zc@M-L=}QJ zeWhb3d{|{%Q-_+S%=?CGHVuk_OP+;UE<hj%TdI>I~gHj<%H))LZ+ zIOS`$8LW>^l1p&Z1V25=#EZw2L0{7rc{H)iey38;Ooh^tb{Uk*|A?q+l7*7u-II^L z{|D=CW^(CLawhQ%Ygx{^t3mamrhqzOoi5GV)<3Mv{jx{y(8N<6srBwnVA%w8=aDIaq!!67>;jzQ_}f9sh~NsGF&V8GH=` znt=CRojbECI%>IDh_c7&ILLOyLiF8*#O9NzsVABl1wRWnkBV0Q&%1v7&ja|(%BE4l z#6KFLj6f2O5-%Ic@;p|?S6fmIn=q4Cv>zFO!(KsaYPoh|wTKI*6u(liGfgOWJ^aih zmc+yc{-&Ci8exDxm{5)7CUwfhlWc+WNbQ25pw$z_=nT=m{D+o)`)t&m5XhojxlPgM zJOH<`BDn}dG~eQ7dPtZfOE}*A_P7CG2e#yL{FR5hr1U49j=}o8yC3H6`5fCdG;Q#> zzg@+iFVpF#AY+Je<6ua6Ty5?&2!CkNR=RvN<~iYMMn*!k;`|j&gFCkU{L&Z+&u9!r)>5|sLmlBMhza# z!x|s)Oo9H@7qX$zC9yz)@vnOvSnw>R5j?FzF@gd4hxdQWm~@PpNhLL*GqQ@v5#bjw z%BKj>DwvJ08-B(g&cSaxN%c&mf_M5Q z7QXphBH|CPlb&lCuaEozNCsh+t_iZNE!1c)je~9Tdi1jJ3~t^1Iqh@wX>^8|`K`&; zoI;*pyn{?$i|AXJv}t9AdHc~y0~l04r=B)x4{gh7&|p#zKDJgvidzSm!X8vULIHBn zg19wI1u_%}-1p{zptz-oj#kASVIx!6_rba>``&fWxMU8iQ`xQc&A*B=Fii0Yp!o0C zk|CY790vZiO0f(@e8=DGg-T!$X7H5p8)v}mIbM(y{*28KgQ$>)cHq`17C6HR_YJO| ztHB&tyMqip&0>;8y`J;nkrDqsV0&t_o2Yv8qMRVEjz~^fxhTkwqwht$naXvY7@Brl$*7-P{^^KJp(vy+< z>?~*y=~*BhWGC6n{f{6n9jG$D5ff&@zmj=4-UqY@SUjx8)|U}1iJ;1`Qo?u*8HnYK zhK~BwKT6<#9+8f5LF-holw(OO>A%zFJK$qvk1Wp%R16WJvAtfgs`}9-8ZbDqEX!k3 z-R-jf^w}hu$cND~cD#W7upGOn1kn92hnKzya-h%Xu9{gcxSB z^mSdZaD)BNfk3nRg%I-}-ARY1zs}wrH}jdXHmzfJok{5!7>gzPBzlSm4XrhGXZ^R; zV*ygjcctImUKlo{f>wB~(jCnFTNsM*Xuedy(s8A~Z>+?qI99Si`IiD8l+7m7?L4eg-LgXz?ULnTw2I|fwrtdSNjQ(^{)Lk;75gDXb3)q4~Wr4BHW z!c)G}EO#}=Wd~4jtMGB+kg?6s`iV+LV>$6a_;fQ)weX3w5)$f?1WT(tp>v-l^zE#F zoaxT^ZayjBt7O?rynmKMi4y|fsH;j`5Zoc(`xbXrP}r>JP$;Y7`}sinYzmm1|Hp6l z6dwF3fS{n&qbZqf^)Xz>aQtCS5Ou_av?J>yHq)UFzxK{msrn>3z|qKwKMCO}Z&k#} zNjw{`@nr6GA6>+iH1~QX7&O(&s)m;dn&)6W5I#{NWoz-Cgm+D>_Cfe0+eZ(Ral@X(yNt+zvy0Ajy(-{Hx zWR9ks%>+c!!y7;1hrwL$lROLho-O?fqDTZEfS3wN84$0TscVb_TvbW`X#W4m%w{!s ziY3R$?WAcPp^uorr}1f@9KU?F9d`!s{=}vFR<&wJt&;u*1M}$>Z_0%?f`nZ`4{)2k zr-?UK0=&TZm>&C555{|K{{-f=K-Z<7i_qLW~#lgmhRxvKRwnObjV3)yb@|D2*tvF6W zgZ7s+cGEKJer(2I3_z4Zh{QsqJhdW}kedAv`$2dpXuc+PWxqlO!ze!)wWSb#V7gIH z7*v6f@p081E@sb*eV61Qs0wN_EcNVdk1Y` zzR_SUFdu#Q%RosRs*v8@ zd#Rib1fMB*UoDo?&X%G%ze%|Uq)h_>@X;$&q$Y%Q)()+_`~Ru}Ri&#|SvG>k?i6HG z72*PF>qL_0o4;#DZh|+{IvU`^;N#vr2=2GP<>^&J<>8dxLxC831Am)LF+kfqU5qCMugW2;nA2bY4(=ljz8{xEjFD#7mu=Mzf~}lfciY@k`_rN1rA_=pf=;^%aHQ<&JE>$!g2ZXNv*?Q9AUq2 zhUwrz-IKog6@-=-^~}dAE-h@{|Mb;J2XLv#d9N7Qfo8`wDSQTor2jb<-g%ffKmI!K zsU<7*sCQ6^#Ba-$@%tDfE+X__Zc2&|-h~mCZP^Dn-O3-9xzg z;E;3-tGr>N(6eAiQZz5LmcID@>%*TW%oZOBi~X60h@qTVAy)1WZZ#(TxFLEa{S@lJaHZi$6E%W}EjtTEVd3%9b17!+iV_4E^g}zt5lVi-^)FqG)^}sbNcqK)@Mc&c5f%+SZJtpLZ*WI(yqK%f zpjvWvi+OaI^ZsWDQDyq+m#4zTS9*6U>#AhWSRKaL51gSLD?xs$$ZTx4dPn@R@X16| zQT(QhQ_|1r`I9n0h;dK+tfF1A-KuLicXRXuC+H=g+OR%^Lnr-aSUm%$^#89;XPsI1 z@8@*D|H(HPSBt{TRkX8uJtFm$js88$??|bnJMUE^n0{9t*~t)wb601QoHX!7E0`(R zbJPHoV{vZ9qT<1MyoKfEtAem4-E%^L=GYR?hBT@>-7+Q?+3nKjSKo=I{Q~&68@#cg zoEWVo=XX+X3wTF2Zct+YUBVxcFc0PXbf=AX&FL0E`1aME%L*|Zl>0mBWA-GU=XQ(D`ZkJQ^D{`W$UQUIHWPch`tZ1XGR$@8{y!btwN5i$_<5QXOt*ye^Ro}yCz%&JY_j6)AZ9s~28h-OiOtjA9FH2}coVkxL{ zoVP#7HL7vOlrg8~Sdgp(`dBw$gpwRvLvvx(8_cxz|5)yK?&<(fh(~Zq-DP$?-|f?0f{>ba zcN97fa=rs|`FXd0DpFpJMkaD7vAtMB0$~+0CA&rWTU+*%acPh!WJYolZN0)A%RKLj znxT86XroN0j)-@zk*0zp{QEgEhjiwyM!Q9#$^+jUUae+}q6h$!qi@c9!2Wo(i|?xP z2oy?MYSJ%zzo~r=PAo>=J2}|CvZj6P%$5-7FvBLi^O8{fVIi>HJGYSy(^tk2KWg(> zo@URUg1b5w+Je3{u1nod!IAU#L^=95`UELgvqGxjPN7FMekYlHLX<8hgrwniuA{#w za`CicdC>t6rY7j=q`0+_JR*0o)%I7Kr7%?_n>JNiKrKfAb#JYaK-O*2Yohp zfxS~Dxxp2z&DrLA!@=gebFM#f!XJ}kEGbRw4@C-;x@7^;>7W;t!L7ib6Ty>Pv9EY} zYN-kV%M$dB()poCP-4F4XPRMfx?D*H-M}Wg|Ax)`c>)iJ@;>npI$8T~WC~MG$WmvL z|Ht(3+weP_`Ck)067rkTe?vY>+h#p~G|{<1@%)Q{;4lLsAyyU;7oL*P__&Hc#4C4v zSvr#fWn_Yd!=ky!4D8FyCDk5X^CQf32Csg6r7Z4`-lyH)>696%Qgax!f3~~PzXZ3u zBP{FxMkzBE7wyaD7U4*}9l4uhODnp?z(n-^a+H37+?JlD@n1+@q)x$ZVP~wB<%77l z7Zr0iZ5|_kP@Dxon6clNUG!|kK}K$il4ixar)Q~9b)!NfMvr}%&>y!;HTRCj*)DUf z4o~WQrp7Ct8XsV*%YZhkzTIBs)OoWod>SV7t_lBh%Oe^F)U>o?k(I_D(J%$S)j$-xt$-wpv8cVk}Z;sk=xLKe{j-8c!<}VKbQBnqR=* z>TgiG_v*&dpLH&%Ywcmjub~RihwcO&EG<$~6PRf+r)-k!PlnF6jV&|(Q8#MsHoIUcVR8el_BN4Au3{$?AI-QKIthT^=IrMty&qhd<_I7yTmYXKb+};C*jc1zOa_ zqF3Afjj}a-;Bur~ImpF+4>#nq`Y!oKp@*vZL*k0k2wc>|pbLLX+Jp7DUduVk%wyMj zk~MVw)X*i%G=2hFR_qTuV@*7c?ML+`3a=FOU^%$%;Xp0Ida8DAOXIU#yVYCZ+Pa84 ze{q~PT(fEA+U0Sv+QC%mI(eE4lKyjwiV^I$k399^9#9~1&UqdLTgB?+KL zBlyMelV?cCzRl-+sRQkHReZcXMJrBDNa8Cfqe`lFiA~_A9`_5r(x9ndxcV#hSto?H zqgjA_f!7%oNXmbn<3_#uf0FkV{uSnga%h)HehbbCfal_S4b8*Cn_N;3DI z=82)%1>cL-dr&DPs1k=7L-le#JL{DpiM-Sf=0Ut#y-#(RwXOVk{Vhm%aF7P?7fBRe z+S$5U*DXD@JHLoxnEy0Zb2Zz57XZ)62d@#C45KHI%>Hchg*@D2iV9-}(V zeZql+0F@wOTH`Ce9A!zYQ>)_fCsl(+X*_DM^}BI7-6K0h*szFhtq};+eN_Y4zZ7y$ zJ&z?pt3>6wG`CMTZT`!@&CB&q-VtSr_>bmv;v)=z#?|(e#pvc{(d=OFhs&ZMNpd!C zhqCWjXVRNH9+KI_tiR~+i+(YA~5>FOe1m&VzE3t^!#`Xk)l zN8NTqlJj{N2$1N`f7O+HBEh^jip zAZ?E%spzT7jOI?c1q#jw%Yr%JecI>9>@qHJ3j5`QsUBg~V8F<1GSszbRSY?%8qwco zeB#Bz`?Hx;;@bUfH2du1whj^Z#pwi7(RQ>%mBkzyn-u704LW|Tu^;NxPF$vxjwZMj zFkxqS(kqOCDi;aJWv;IsiAl^c*4Z%*)y<@BWxP?($Ll2?gJvi4j}NLYzVSVHdGf+C z=S?Z7xIr(l_lIL!T)>*iy0Z=Y=tb-b=g{lBW?ac51>-B=E}#`&K@n)xeSP+O?v3T& zwwnIdpNl-j($`ogodog`S7|^UzCBO(dcXB~<<2LrR3=!a%>!kqN8e_ja^@c%%4{|2 zx1RhOariQyGW?EFn9aCY0$;TZ!M7SRn%`j8LIGdjtBd)wLJ1W4v$BtYK0le{);Uj< z=sIMX6PRy&c$>7(yma{f|8ac=@5_Z5Ho`D(l3dRuqBUv0Ue-?#O+!ZFIpBmSRW8-z z{FdAhMUw(0#YsPsxd{b#$bp7Q@&}wJQ%oI%1$u(I8Cf#MMB*xILrlvYBwVs|$$?Q9 zbQpwYx2%b{*XDrNMcfcEvQ^S8NiuVz-;w26`0#gw(y4A~O}`Khy~5r*7Wh`H4zRY# zpCpQ{b3vqkYbGNrH4D4gM@?-lT9U6{>|#UQbHOdy*nAab|* zhH)M<@?zJrmyh9@V>eOyO~Co&8TQ7gi5krAv|y(}VOFCryHBHAgD;5!#>$z%kT9zl?3#T*fDWDWnF1ENH+wH$=mp&2wjzkmiv+2D&*7Z|>Ho@>C{X zenXtw<)Du=-Z_UtTYiDN>RapVDe3LEj@dxptr?n9RA9_6^4-l)HBS(~>Tf+bdqroO z`bNL~`pX^vcQKm|EOVAWqpe9<#~G}aGL4&{&HF`yU5stX_xny`fuh}42%rfKCddN% zR39Fq53sMuc(D+tqkldsF^CFzsT4jm0$`G#t+51rrPrWeb5)X*Tj6@2??@bO)unk> z8nZ5j7*f!5D9lmMq8y(^borI_(>f3ah zR_i(N>nhOR9Y3h2+H+Y>VUkki3Vg$0*Jn$kScg~Azoy72khD9JT=leP;?LLboI*r% z`;EIkIc)F&=VD8zKHJ~Uy40xUR0cw6U@>`f9lcD4(VLknMAxY*54=35mT*haA zHXJNE<@vT^S_8ZJ^CGLw@eBbGg@NYz+lus@zW;lkUogPWglcF~@Ha?m`U%~zK>{+b z4p`HV6*vC$2Mb{~K3lQ)#E0pR4Wetfico*TNM^io0f9uVi4VvQTbOirJyxZUXy9Wp zLkja{?}H~<9zEhUW!Rg+Oo(iV$$-8q5?=~{-x)CaW-!}n95VbyZ?{5kL6h^;yCgF)S=P>dk^+y>a{-vV`d|r& z@3Yc)f~Fu-0lwyhx&E+Cnq&V1-^#z?LJ_Uy`?RhCt&Gi2f^o9y9GPodT3jh}@H`-& zjG-(uH?5^{mQ!oX=_@Hs4R(f(57;nJ|T3J ztYHM|d0>>=TjiEW_oC#{!oZ&RON@XjOCEO9D?zKJ_pSPIj4~UBr=6msNU&{^{)xLt zNsBQ|{f)9oB8cF8NdyEu0i>Ty%f{=6-g_=EcKOAjw!dYVE-x!(jfsj z=^Gp}1B0sLu<@c_)Wi`vY;&CNViSM~&SMn(Uwi9XTg;*qwxs_&@SkCds}3(tKJ=D- zg#`W^k@mmfNyh9H=shwH7|BOb!XtIWjeTip)mS*zEFV9>+QM+0hNYChX#v6s7VAsW z8E`(d_hcyGeQ70W`^h81+>KD_BGi74)G{N&K#zdcft`o>d)d<}DB<$5WW+`qtW zoTG5yF0VhAfM!&t#^Z{6=+k4TLx75^QWU}ZS^8T-VvJMKLa;GTrivRH92uAWSjjQ? zY=yc+IkUirh+EL5{g7-7EMbg0ns4~UIg4Xel&R)=q7Z!8!S}vmk2WW0c#@eWlx=Cg zdxwm23Nxd})3-uj#KNWE4emSR7PrnhZSd{Sc;JA7@At-mgt~)AF`4=biLWjQX`(9R zW-_2?^~0KJczGWU2)wp_kgqh!&ZX(dfva$M@-x|ItBKha1ZXFWS@6d#Iw2sG-uIFy zu?_z-NcyyrhtDd^NY5jy!UrdQ3JwE?U&WpT~$6k@S-91o!lT($2XCgfh&dGJ^L;y=gM# z#7I6r27V_e@7@_E*v|URXbu**Sx;0LE-4ObP#B?)bute@G6Gr*t7kT^uLNCT9i6IR zqDomL-W&LH$Ee1M@Y;+DM*;oqQou)klciL)HnfY#rP;-?qTc;=*xu37UbI4XybI*GnQnW{gsi`4 zI5U)TWg$u^vfRwOq?qMje6E#!s;#iKa(#{~6TTds2LYQKLPmbW`w7Oic6=QU#VX75 zCP2kT#IQGqdrNG_QB3$4h&r6*t6$&4y@jurH~n5gR>4y0%b52?ALbthGlshnX+B_a z@EgYz(hRpRf_ML8lmM=GRWI?RK40*+TLZVubKk2EMSQK@vZCs{E7sg{=#u-FNCOU_c-mKsjlgfNSd2^zF?3@uR#Eh3 z@#%_eExGg3H5RYq1f!Dg9`ga2;cF~2TnpFHVZtFl(e@wD_;H5fu=6m`g)x0Y^-W$U z@xhV6OSz18^hA?Ucw>nmGhqHZ`v4?#%(By4bk`_~_?%NN2CDGhGn#AQ)t3PbSHbhJ z7rGB**WP8}-sWA~y~Jm#e1Lz~l9h=GdN(DeD*PFj8Zya5iMuEC`?M>LQe4Eb7^WpH z0HG+}VZ!J29IX7{8PMO4)POA6;yti||0m?Fn{phZz*&fm?207izWoGORyn(>-6xIr z%E-pB5${l>h+#i|6kwd?DA>{xrp+cko>NfdPJj9FV#dz#HGOq~o;LXy`xv^qoNvs} z7&^D-8s+qhjt+wAx2;m38?JxgIe9td_XTjED@9bZ?33oz2=K<3!P{&V zKa=+Z7s{qG#y&liI1CpjI-YpCCeiFl7;41 zCt?3M@IGotrXk8jC0jmb-5c0cjt9MwTvX6Ec`PD#4mSi93Thby>!ydp~oq-(*Py)zSF-;0r;5%`S zlD!mE*@jp(y;g)Qha+&6(x`a#XUD=7Ta51*8oVje?x0zv(Ob8=9 z7jQCeQ&^TJ^vc4!XjVmJ#rlb^Hvstx_Ty`f@~k%ay4{@FhNXjG7-p38yq7Jb*RMBG z2DL^7)HD;Vt~CIo^7;dGz2BGy5ol9r85(1;J_y{|9awQqC068X*438 z(w=>Ss(b;gNJ*FFdLUfS0M{Bjqz>=~Em80?K?o@XBZ$2?mTAcRc36nL^&$L{2IYp= z8ze)aN(GCSSq+LqW^CcHUZ_ItpIk*xZz+i22m&FmbiOjz7`w1BOSp;>iY3`JLDx3P zB!!O~H2d%_+2))kMHSuVoZcR7Q11&ZS;yGbG?|Q=6qmGXJx4+5(TVqk6gb$ZMItwv zLq;AcnU`at+_}f_&&(sPEoRvxc3i*jzl{zH-D6)`r+Zk5PUIj=b`ypv*_MpctHs)_ z%My%O34Kes-8RK3QclJ$Qo`<67d_4`;E(ZQVA>%fEct7tQBbG{w_Aq<`39yWKC}bk zF&cLrs=`6^-5S4wjjlSzs<4mQRa;3ELq9d~`=9-J*MXO`CTgMt`sB&QtTw{Nu4ZVDE+$zyYaG1m#>F=6h6r_>3p9?oQS(G8@(3*j8ysapHtAqJaP3} zHWC64PQE0s?~i^2~PW<2RdHot;pQb--(>n!kY(qQQerkNyFH|dFoEi7zy zWPFO!WhN(J4*3s%ur`CJhYQ&6YT8R9dmLy@+plClWR8yp`S9UCaML5XFAOOnV-t2^ zq55pC_;JYXlekHT>%|2NNe&hI4vV9}{I2XUd@BA|CxSze& zF`jeahpHCignrf4X6{gQ^`=2Up0@^JV;fX^vSXBv*YFXxRgE81QiTB*(RyWQGA=2l zESaw%N1^Va2gYc|z18JM!#UHT;{U|sH2)iqqpJzxal&uoaZ+yMagc@sJJ~+;jn*I@ z=W+2uI*7+%1@SlvbDa+PB5y+dlm5hld>nI(pWH2EEz2h(IpWK8znCCr#%2m&mU9G& zd)pGGq=$DNF_1ytCw;-Xj(QSi(621JGgh>QjnHO{mbRTGozYa+W7f?Wvitcqa_A9l zfn8C}a0z@&)-*TKR)$YaLT+O6{$&2H&g2b^qP9F=YNV%Zb%7yQ^#&Jyip~#^J@_AE z_GzQCPFz!T2P+duVCiIW?7@spV{`e`W8XlY=C0l?W3H)s?Fx@WBhJqrol~Ncu@0t~ z6))2p*VDb{bqcm}|I}!h{q3k3{(bU!+abS^NIu<{e!>j@S}iT;2|=5~TMcD4UV|ze z5Y&aPP$3*uQ9a{OxaL*4S~GfuG+3yq>^Mojze8i#x>a1`U|l}dk7p*x6EF`m6&w7V zoiU|;wDscPk*#N>Y-Pt9Y0QZL^EC~TrF9QjKV9}>Ju1=lgn_`x2*oig!h zgACp+!otSY#dPm?%vVa0=#hrLewiHmB^^()h>t4YIUu=7iUa?>MSjik-Sxq0O zN~Sw^8bz6^UKv?0FqV(Kwn5WQ-eQUbHK-(Z-H$3O69XG8lPj&d+g=M;Esk5qwh94| zYk%fHHuUP&fMB^RNL)^VYV&;!0(}4f{MvOdL}@}r?+7DS*(Kk#w9IYx*jT3zU-1^> z;O1FxVK^MQNJzH8+k~kA6^ja4zBQ@b2)ctA}0n8#E$ z{D4D`tWw(m^cU1TQMbe}Ej?E9HOqLAO!3cRWz_8Kn8_5r@8`2QX2)z$-4;$_^^Mn* z;HG?(By{S@KhL3sflmER>n~!E6hoh-<^CEtq3;S2S9%u2Y@AFym|MqwCo{K1;LiT| zj>&j?D>Y9_?S0b#!9wN-LZqSYpmnD8t-^qlcE&5mb2$|zH}ZWyyc~) zNNJn`9W~HWt;D0&gIKa&nF)dYbvEU+?Ce0byj zK^2t}la4$#kL?hBN(KrBL&^x#-Ve!qKQ|Db0D+_TYyq;M0dn!th!8uTgrP8FS3JEj z9I2_#igXk8Jb+Q*J^$|~9NNMNo(tU&`Z{wuR9&gnRbmXA(%Fn~xap7#Ap8&%I(lUD zopIpbhV5b!x@90x+_xU440->|&jvV{$xomwWq!}`sb`?S<51@kGS!uUxEYqjZ^}bq zheZ;Gxb3OE+EEC)yO*YTFx8l!gYo+7wvO#|8S^c538!DBmiiy|BQv$zerMwB;)wuj z!QNN&eBj^K7UgF3Y~_@nnVbJ<4^_?KYfm9!b>X*>w>uM@)%eOHUMj+In_wKb@Hd53 zwuT`C=QTJL7xS-hQ-6htY+;7Q=HF79cS}@o z1`PlQI*oT_Fg!L!G&MQcsJj02pWu_FG|1Hs5Z)}gGCUwI@N`qf3(c#@3x5$oPD;O??yjDzJX$IA!uR3}s0N|uWRyLr z)PeD?D=LQRDT7WhAHwDI=Glx(JZCO29l(DmYprD*qIz9ILX`UiZAVO;yYGbYoVPyw zTv{|Y70~;bapSihYCx{(Njr8P1f0=S9wEygx6UeY*3kYwXV#x4%Tw?zWJCsvv6dA78Vl zr)~|IzHAt~cH%8x18ydYbbqbj?Y?5<>4Wo{g2EaV>k`2Eq{}(<4CVahMe+FqSSdU> zNlkoqw7c_VXP;rZ+Cfb9jTpHNzu}!AlmLm8HajPp4fH}bEjW16GdmeN(bvhfh;^%- zoRJtKnsl7UA#zS|9X%5LcwvcN?KqET?m=%*X~zdFFG$nnQlPgSjRlpng(ntgmFQnK zI+&+ba>Nn+*h`!2*jfb>dQ9WNJfY+X8wK;MiBVSAgubK!mTgQsQ-vw%3Sg4Cw!s5v zg}X9HANZC5J?@SQ1pLft0V{x>bPL4`LhP*ZofsdvvjrTY?#533b9uT7Pu^BQ8Ple- zTPNk9qa-EB29PG#TnM_6F93qY`R{2nS0&=ettb^Ez*v)SBmKxA;b%Yql@Vj zm<$G$yc$R!TAtT?s=?+tE6E9oZ3(HDa+nXRnK0ppm4*7Rxi8-Pvq#2O$m#a-GMgTz zcEa9}%GoI*lOELkXu5y5e+QU~p5}_&f~;4!p7ki- zOPBfN=}Y?8G-YoMzW{#7M#%WX+6nXFi5Z%YEC$$bYs9W#5iwxoCbuRXWM)tD+%j)3 z)c6t5(}|Nv9pLIZ4V+~ph0m962+dtW64wVJM2;lDAIXrH9I7NM^&|0QPt@~7(ASqw zf94HPtTg8wxlUGl-(GD0<~e!i$>GeqxC#>|MFJmnxudeZr<)TeN^YzY)~$~95$V!1 z6`v3(20vFkX?_dYGW7vp&6#xs#bi@`;N=_iMUrg}?ri(DRPL+|_?$&#JM*;v8k?|{ zA!IJ7=j5(xmY`|QyTAD%UyiP{rrp+=t`OE0otRl2Ds~j=E!)E9sAXJ0PUD*Rpig}3 zH3Y}kk?Yhp9=#_>P1*%^c1YDXf;zf%8c5TKPyQ{DgJLb=I?f3s5fswzWhbjWqucr{ z=RWa<)IL;lw;n3=U4BID6O_McDQoo&$U^bkBchyo&%P{2rZ*LTYbb036|6XLPc@e< z!EU^Q&*bFf6<7*qnE8SVHtI41v?)-_cq4i$%1Ct92HT=wH=6FjL!XhFe#kB zTb5k3c>V*QB#=kPguh*}cCkjo-}E`k((QjbDS*>N-u?y&5Hhr}=*qxEob2h=74z#i zV++C$pjAg1e9ykMGuLO385NQ#rw{TN+=L66kqYg|$Twm!7>>FwEaKjJd>U0dV`iW2 z-Km3xl^>##NpX#dckWrW9{cW63zNIj=d9ed0Q{ekFIuc_3^alfxtU1plg(e)3I3v? zFKs;EjCBg}bWQZF9YEhM`l&LV+fd>|oaA)$9;+y43Xt!!JlJ`1Fz2;3NijQ9<3fC- z2YvjMtsiVEY*{MZA=U^R#u#Yd1L^q)ocB_CWpF#KH^nhlZExwQ1e~jV0$nt6F-8p% zxMM+N_LU0%tD-zHI${6231$-bLM(UXFa_6~o6GMieKI357v@7g7;QPZ-M<(9y+W`0 z>-Ud=iFPF~P>0B*iT21puQvFwpy`nH;Cbyo3*G4^088YD$b(=*i}f5so3XXc^sgjl zN%j{qh&8X|+kuCu`}WxX+i!u#+mK61^mU4tRA|ri^7vKzLV_y=Puyg2|D%t>>ZUOL zw4D=2$@*L}yp1;T{9zCDj@pxNEoegGc}mFV@@-vt&FGDnAw#Blp{Wx3a%&#f6)ubH zMU5!Hs|jfI6BsaRCdmkDV1P#Yhj4l+a2K_qQmF~SeWOPwxzx^OiN)IAXX86o;@spn zQst#W^~geez`O<{B9WfGmi*UMBR+vaFR(H}tZ{^BSYq6&#RWlq?V87>i=x$kB;0Bk zaI6tcmfq#glSY{i+7OXNDpz!dtvMwEd)2R0C0E)^+=)i0v3ezP+vjNp80mp8`urE^ z>A8*SG_3{Dc+yfZZxxgDXw*&nEQdYdcQ57Iz}cqjBbI2!mqgk*2k1l_pfr z0kkdrRZ9Pm^#4^N6*lC8o=+Df%uENavwHq&T@sgqjak>_mf-7&+lgo5el3m8J_zA& zxkF?jB+EW-#GcmWWLR%kbAPr(L0_1Bo4d;!C8UU#MCf0eo&?Q44dh4O@j~6ea`xTG ztiGMEceckkWp@a)cQq2BVJao5a9XM(E7FU|gwOth)rJ8nl`>72Y!-Ud{oQ!(hiW`TjOOS@!XNwc59BENn{Wmf;Pu6Tc19txB%4M0&IsQmlVe z&SbnY_2MywT3DxY=d&&Z>dXbSz+P^;?;FJr&9n zL~_q3WH_dyqloDL7?s%YANEE?c%W5uQ3u|eJ{%{)sZRt%)+A_jYQCe0gVySx=k&`P z3{CvkoJa{=GhD2n!=^e76MDR?U&#DgrzmdiJ9RdO5X2HUJw3dFHUrt{`gfgsoaNV^ z-hY&?L{qd@mB>@=G^O5XM#c-Xi8b;qmiD&uE;=jK2>*b_P!$S-6AF*U7Q-4Cqpixe zCD=ATEjLOpzMy^6f+Q}L8vK}ZK3pdZTk&0hO)|4 zfcbTAgUB(Ea2C`vS-OKf$R?m7U<$XIFCX4_k!+a5qw|9BWV@^Iq!Yd%8ll{v@w4VN zB@aVf{g-^p=igX9e;z?16~26aCR!9AjCua}pgZS&7^vs?r;yjE$r+o=W`@2@_*;I? z&?bci*sNXC%8#jqR2^B}ajM`nrd1CDYO4~$Nw8cGQ#d-y<f-fiB8X3A0v3{*d8U7m-}_MLHb+;Uyd2D?SH ze1{i`p?S8$5|c$4EqR}D2zVF0peTUNOBYcjr`IAT!a_{HMZV5D2g5?9KuJ$Fn3OF! zI!n+IXA5@c=ybdAy!z345NC~uh~6@?`dpsD&1;tav!y8$3iQjgEVr)UjA^YT8XAg zjK5Et=U1SOBRD(3T3t|!H9m<*zJ%!mYn&E-zD3K%j`EJ)9r(r1n?;ePjZGGmkpc&O zhnx;`C*%5a`8z9LVZ#I({&o9sj*%&#r2-1|xuzNlm9_2pn%G;#Jw)^)>?G2nGY4aZ z|7R@_y!~79bJ9o^xU-i4N~4FOCqSpgU*`gGC@jNL*3|)0vdVN-f8trvZxcXT=mpNk z@5>Ds9a;?Q?GYDM9vv!4FmlM<^~A>Y0%YsS zBZ!rUq}9trtO+LblT}WeikzWm@ggGy!$U`D{hX=*ts6Cu437>|zo<8l_Um$8f#QPU zKWL`A7j^;8(N?CENF<*efSUVncpqCYFHU#dGy?VOeltc#v!S@z$54 z-}EmYX6ZStK@p%2?QUCBWYu4#N4P&_enP<}@4`h52tMpw0dg!w4;mpm3Zcm+Ui!YE zizHC+q1bUaMs8-l5}jmX@An-|aeZ?*OZQ8dIrx;pSCtY+*-$_UsZUQ^B#KanBG`DSEG?F{yRAj>8<5&bkoK9rY@HPO#BoIT-%Oi`aXn= zBDvL-)2pD2HPU@-5ND9n)H41`CM44uz1DZw8*O073^3*4AxnLt;d7RRv&PzW?;p6r zliDf=Z!BskI}YJqR+t2CM(-dEQjYiBP>kier1<`WlFS0ERbfCI!fBaw_=v*ii{mkK zZ=2Vs3d95~TKaEb{glk^F_DHkZZf~S&PT?il0p(vwpctr^?P#pR2Q8#<=Y~r4*h#Vy2V+5sF#wmIF}qG+9`l zD<*KXUAEW>PXBOb*uA^w-k@}B<0W3Is!?P&87gs_Vv}9b%ei-&9Gt+5(KWT$EL6_+ z8YV8hYB*Z#-Wb=71PE#c(NEWVV)`Kp+(kg#7)D9e=ac_E;DZJr6vRdUl1vupuTGSS zS!t)eG3kf}vGP~)+Xp-|y@*%+$Xou={~TSwd#`L)R+>ou12$ng0b*qg88bvPGC9-F z$3l?V>xbpY`S`5(jHv^OF10(oF-EnyJiGDA>r}KMjno$g^2tZn9RidX%4F>nHmoxx zqKvMO(Pcz``$3*!IK39@iXz~WtqlOQPl}i6EJKN`SAY!BT>>v>Hek{wtH?da;v{9B z`)VyeHl8z+KDBVks(51>)Z4PYLzl1!YP%dLt#W|!m4y!}7BwL3r@jw70%(LTdQF1g zvm2~&y65Z!idOk9%Ff+i;CQxtRQ0s!D|LFk_IgPklY~e$<`PX{{Rj!@_e57t<-_NH zJfb;3+kDxHSrI`L3ql(;mnItJ@V_-dO9uoVB4(9hmQMu-+r{8aQ{gCDX%JL-ppSRYw85!kgsh6`2 z$B$2WCVSHXC@GYG;45)YC`;7wdQg@1A1-ICdQy7la~_atswi>=vR8!50>-jrg#5>sZVISc`#7bvj2o0_AUeZPbrD(8C5;0C0~+Bph#US zX1Zl=2F2O?;()DOvgqL0B;{&ei>e?Qg~k6Yrq$*2!2Eh4Y0;)v8mPg~ASVo+$NT7n z=0V7=zckGt9eSw|f@_qsSxvwHq0Cn!befx+pyK7{uFzMiO?dn800St?QhPYSsmJH{ z1TDX$7XEfxA9@!Y-z_KHG8h_;Vf9LfzCQ8INdj8+_)9@a9iFdKp=e!PK(KwPf<GwtA3+)K<1%3-dd0&;1gI7Kw$UP8~>W1_7i zv(lXguzOIfHPaxz4a=g?0a=!kv@4KEYP}|`u8BxgpwmaQ0N0jb{PjP05to-D+48u*N>>qiHkNZuI4qJtYZ3-*pes zD-Z#lSw9kGZGo9d0Od|uQ~L7HNEdc#DYPlP`>stdBZJQxW*%$&?Uf#L{#6Z2td{<> zM9H$`tG={OCUeKxGL2RXs=tr)YC$aU^m^&5_NPgYdr zrv6v6h|QSA-yT3@{ILP@FfY9)j0;d^ObS!h)RDfYbG*emolC+?@$065Ah()>0f(2a z#rY@3jk%t{AmS<0RK&X{nW&=KUn^ZuyD`;YQ&X(N!(I?+`FvWV1qriE7gVa*qE*MG zpQHE=nI)E}M0g>^jrslo2M{+iX1d?R@hl+e>f4M^3n-AlNdBXKP3j>rPrms3x_ry2 z&L)sjjvC)qn*tyZ4e^Nli*^7q??$-#EeZakWvA}bZbv+w4hAMdPuT`QM->px`x8&} z)*?YWPH<>})Qhlb39h{pGuXz;*DS9Um9ht8f6rRC%#7Ie^@6g~K!}h)pZwUs5?(rk zIl8KVoJM)3=xld5mOgh9;)!%*)pseh%pakK@y@gwiaw7C6!$*L< zNt7?6RYX8m$4e+6!0slOtg(0#@_rDb$3<(SiolwYABQ#I>;*j%y53p5bgFy^Q=)Go zWB3f7)HBAkaBb!{K?3YD#d?$6FLRivSiQeV6OsipCN7fcRS)wpT&X0H*?g?2=6b2n znWj0?PFdCkPW}08OJbYj8yn$Gu-%5P!dNBp`z;KN>T{iNBK|~r!ka$R@H9PV?xItu z;fpU?k4@P8>^H3S@8gyv92nG*`OjW(puW#g{hUNW|3f=6fxVRe5w$bGC!zi=d(^5d z&LEe2tou)q{}#DqWVCL1r*qAr5@KM!M1oL{JU zo*$U)cCeG8Jd&bS(x3&d-<9KAZ@9CghSPfXVlb2uU@_n*$|liDzZzo637+hua~(+N znd_ZFtL=ibh#@sg|9ni_8c;*1qp&9mw$!nk;8vR|BqoX-V|LOqIH^@6stc3Zeq>?-KWt z1Q>g<-jRbkZ(W747RWG{8)T&N>esW`IERSmTME?l$y(I(ok9$_oCGYvRMG+5SlU+q zrm6va@n6O5X;K>wKp1vkfy&bI8mnvs1rYjzY0dQiBGcAcLspU}-Ml7jKxqg0ymzK0 zIK}fN{u~x#DXX&v4J-Uq=-^vQe|{>|V0^mIZ$NA@gSA0JgQp6JD4j$#ms*?_BC{<*p32yMc6w93ddn5)=rwTXjI&+>NFJ&sA7HrOK z+3v_Xfa2wEzz;xpZwge}?ZC~ymdlggeu$gX8JhZ;G&FSjVrZtE$C??ya&o#IO1+H$ zE=EhnDeeD4(f~iB35c?rKj{UpJ1K^wc0Dx4-P8s3t_vsRI2$Jo_p?O4_L(I4r;7QX z#4g~Q|B)yM0L?Iqq#1d7Nii&Gf+-Zh$Z$=q)fx%qwHs1$T^}Os{%KbICpZC&PzxYM zobw|FwCn-JXXcWKKQ(Yqe`5t&jTVD+<~1(@`IfAN!gJn;U20{rjGqX6KOlA7u4vOwOlH#s@>;n6k3_*j)7!`PD z(s%;o&Ke46#72MC(n-xXQWZ6RthJyd=f84N01fcY)6!0Ayb)+BMVlvh3vi#w!~Y^y zc;;zysuD>YYZuoSSc&XURt-Gv+Nuv-Ymy*^3|B%{(ly>}JAM9@$B3S%>wVg*4@e5h zu6XLP4g_rPzdaXHz$>I@RN^9KLwo6>`9}K3RsjY*`!CfnCd}LmZbsy9q=est^%5l& zjNa;?Z$#FN$}-n5c z9Lj9QIshh6%)7S{y_o(yZmh|Fw2!&ly?c*|h@_j0uwqSIT4_wS^}91@focqpoTX^S zbCn|SK?Gq4BPE9m^bmq5nsW(G07>+ggJnH&a0xxxbbog>D1f zs2R?$yhr2(e8B=CTm9nVI!zJcg!sX<-1KyF!FcpSC5(cT!D>n_7PKP>QVwN|sss-7 zDtI~HI3h-3I3%=!1SgTvzey^*d27xo6M<)#@J=?JmTbd8aYH#DjX=u*zc|)f;u|gY zHwUF}qw*yAKbGjYX#WKBY&g8gUmbm_qk5{Q156L6e?bqFbUFumvN{yBD{g=FvUyja zswqAC{8LtMN2vSCSF=y5%2RXOH*F1P_E^Ilbs+^fHI#qo+yk+k8EDeV2rW6yl z#dyjrqp}%QE3N2pz}G&LLwEq>QF25(Ih&^a$AvG-8R-fXr-D z4PNn(v+?D3gLu$t7dZ#=j2CyqfHzHFBk4lJOm5Wkx~yVB9WRzDR=ZaKbGS0lJh~ia z#j)WeFjNExBPM<8Ct#5hBY#?XjIRgh&ovS?+gVy0a#cW$^Uxgr@dtT zcetx>q;+T+&8QiV@W}HGpGtE6Kjh~jX~#ctLR!e@P5<9*PbQKc|Iwd!{b4(yzJ*@> z-toV`#2l(0G9f|)JTLso1P_IV<356OGXPupuL=39H{ZJs)D8g)aDpY#kr)Xj`oO*! zHuCI$y?E=t<{DV(Xul$47YrMg85WvvAXu0k@&DgP!}5RWgrNAfW=)(0&{O=MmFFh8 z2?IiS0m>)PQ_ch1fR=<6;1vQat8n07^rdrti!-JOs%myz`OwGiv>0f z>mq4d&_6tPe@_hir?saX3j6ueWl{wwu$s9RhC6HQh{=9P3yw8I<_nXz_F{0z^E)Wj zi<+>_xtQMcHoaN81f={*ADdH9fwG8yDD%FD_x+Rs*8cAk-(^bT{E`tuS4uW``zZ^7 z=19{F0)HU+DSxmmizuZ|JLs(R8ELEJ`Su6o6_`PgAS#hwB>xHpq9V1S{%$khHS6ga zKJto5Q5S11k>}eQggnvsaehFNz*hL3pzRh1lD8eWvoPB*z;*4w!K~J35@8!ZH@zUsX1Md6HseLw*fm5v4&-8oZOwx$rbaruI^FIC-r{-MbuRhX_1L1Src| z2FOptdO3lui4&Iho~K3zq$2=0f*@beUzJi`1auq%VvA(fi8AH0`a-t7^WPUWZ-(V8 zKjy~73=I}95_wv5*d66&?#!Q^-xDR$Gg{047JiUC6ZCE?>xx=#dT4PL69v7l>{_%X zx^}|t7`wAm?$rV^J^x}A|M=uE-~XNL-)?gL(y3qSzl8qh`daMo`z*yrFUUVn(Z~wm z5ngcn#ueIt4`9OYx|xwPJsYl#9LvkENjVtROtN%+?J9Kv#Xy!;(FkI0MyJ!bA)tSp zV(5d+C(At%H9>FVOL8S!_q-0=@-FqPUN^R+YL6(e^Ye)W)}tF5;(s|N4;pfLenpTA zZ~#3`ie~!2Pmla8y;8lW_^}#e`M%OEZUafo-g-qP$ZAz9`z>L!39zr4D1Rp92RC3v zV30>OzSYz}2E>HeK7vdIVw(V^vdrXYTR;yWn{M5_+*g0Xwg}#Dhg#ii;|20Sxt7@i zv?eqvmTzG;xH8LRwT=qI*SkS4dqj*mhN{yb=-1*qvMX-7^_u%s1a1$O({5Z0Het=7 zyIz;lWpH-qboElv&Jc|95)e3gvSHw!>m;!7ci%6l@#i#dnTJYFh5GZeEV36vfI3Qp z9157FE|K+$PS8p(k=|~0*OC+lj{{vLFHlxxoiV0r(E_mG3Tk%7a}AWGBlA4x!R}HT z#6uh)M-Bzw30EaGHf*Ft8Fb zhw_zZcS%)|md^5&S6z@m@oVb3^QSXk9g~+CcgyU0g@XPw_Rmso;;+W9-u_z#9|Mq@gOw!Upd6|TG5^sZ*SOj zc9+IgW5EYA$+@2!^jQtb83p?Sy9UT&jK=_28vDO^FKxCH>FqQ1V^TpED4R3-2>=$OGz}8i!EYr=M2)27L zo#)TLW&UdiKSe8Y0KN+Ro7V7CA-_Y{VxV0**z%ko<98{?FwS=e zVNGt#Lv zYforkv##>y?Z2I>O?chY1t9F{I9*Q|@y{?-NL=-H&f6AT5p8c80B7 zBKiGjD!a7`N3O?>wteI&^iN}C9PjRV!)?!JPy+pwjnQD^{Ut{j&cfqj;=$E3F8xD@ zZV0HtRZ$Jq{ zsdmn_Oh0ve>v*ZtGqi}J!WpD(O(}3o*g||FtM!l#s$jC@PI;JB^&B~Gh|x1oHs{3geH z^!|vw_&;m*&!n^i>t{1lHXpn#O)n?sm<;6kgFrB9e=UE5^fHO$#y=!4+~9?9eAccs zeI73>_tl3xv4r7>41H@>5WlDm-Ip*6?w9&r%`CixR@P3hIbvR7yWRA8)iYHbc)Y$I zl_IhCze@6i_#B+j+`5#*UxWo{3|8mU5Q=pEXIAbsnqVgUwt)2*d;*Q8^4@ttpzB%1 zjsZEmyHnZ2m0OsEEkAZ#x|p3uY3KC(dox|GE7h`)-QdMu(;_TTx&$Dzh@E&Jjlw9d z&nmjfSxW65Z+R~&aAyOMJ&b#)I35b8BYTWjiAJwl$w4cnoO!fW33)e*h8pdFT||E3 z-L3h(IIwOt(+E7??Ewb~5AB zcptXFvFd}Y%VhYh;p(oXwe4|)AJaHWm4;}_9sA>(ONp*=+GhsjZU36XDPZrvAKh;J zXaBFxGq0!pX@r#*!)iWW-(5^^yu)s3D}PKAPhN%fX4dQzi+~>tAqk26<*_-Ro*R9M zdCiJhqU;~{i6n5KkjAS$V>N=WfV>!5a}k?abM`1=*?eWf@ws&j^vMi$DSTGt=-g*g z?C3|aWN6K1cO}WfLv3fo3Ch8Un~ib@OgRU0b_~8~#e$NF0j z^t!)3icHj9LiJ{OwG?Jp3tojmC%}HTun`z&)RkERW?PV1vrJ;kX^at%ym)O{@reE- z2Q)2y1CcD8yXVD#HDb_nj&tzAO9FB9@);ESGT1{(lm3vvx8^ z#he{fZ@L~uz>c~VEQw+Py5-Mn2RwL>cC!~*dGt?BDx;lE92+s-Kfb>y`3djR;iMcY zw|v$0bEl%j;+&1whM2O|`2=vt0}*l;Fuu-%VHK&37Lm;hjiwl2Tl|lsvZwp`ot=qa{!jA4c2m)(RhRJX z-Rt_9K-iC!Vb7uWQ_gOko%0?^W{wjD^BZQ^Eo@i+xPcYm_ooq&%&V+-?h5RBc&h`C zO6#O& zF190EIx{A~{WKjq1MQbCH=lat_;#lbXMuMH4hr7C$0*8O5t z9;;(eau{RF2|L|IX2yiNd{<9{;UlKJ$sYJ&g9j{@! z{sxb{_W`)Kok>}J2dW#5G`P2_L!F5*@=KZ`SeDalQ)LVEs+R9jwGHa?Q5m{?V6ymQ z_QBJHqT;0DCi&3tJJfHM2I`doyD?t1^{MoED2FH;4(!+a$hIGUwRf}knEh?lQC)uQ z2aUSD*@J6hR2wTOg$W0pp_3>rmp40 zhpy0c?*RL8$F?!&*_heYk68zIvpXfNmEOjPC$_Mi6Q834ia|$g6WAgD!_myuRmUi- z<8h@utF!(1;r<~L<~RHKK^v-fWCcWIMR&*UCz*mUlgil{KA3xl*WRu6j@cvM>a|aM zRe;ldr~nYtmsn zKmM_nnkOqP2=Ff#e*MsYZisGd!gRJ8X^R3om@J#;0G#7fsG`Oe+@gvz@DV&H_7DJu zB!iffFEcbMGTjxM9Uu|l0l~rKR?~X}uUjyk<&!!(rIoZvSgyk_*)cCgJZjTGx zn)T~MuFb{F>%Ks0*xt26-yz-4w|S>OZ`v{KZnOz@0FFz>VO4GOmG?RXOAWi%sxW;V zeK7F%7ru|s{hajH8TLcBlb{kc5!FPvW=Rm5(gd>wcO*xH#s@ye$ShhZ3pJ*TY_?fn zP|RmJSNi2dVbfeJkPPq6c4$$1@13@tR5Y88@0<^Adf|4uNkf8GMRzE`k0dcHAIEOI zoAaSjF6t-xV8iwztGk#sUp*rq&t|VNXK+bSyv5;Q;$q&{Gz*;L@Vlt>J>Aa+WaBt;M|VdAwW$b>B49Hm%mS@)snzY z2<2vZZOqpW-&*0b@t4VA0iNF1HsHMgLC#xG&yipF^H#3cm!+uvx4sOE^(y@1qM!hN zm};ksQ?ui-IyilAC#e(_c(AOGuCytLLO|rs#r_)nP?U6nm7aRneN7Y5?9Fuy&U||m zbv2nXjJnwCc;W zW_Ic5dHD6te3bf&pf8_)qiR`leSP6V%j@kK@7*2bv$n``+f}l&$8;Cx)$eCf7OnxL zmAnvd-KAm22rEMcd)|=9*-eBl4l|+2o$vSCJ>-(f4HXJnuaIJ}s(qTnGiG1^6LSh|fBI2uvOD&pscovu1$1fZx@3BJa;pDU;ln zH#G~)s)@B(c7L7GGT@BP*M2t>wNT#rK6%`>ZkRWZEd=vlcZz3eG%fc#s{UFXg5XFn z_xOnh4q2nCpzl2n*p{$?l|*GZd+ATBBpKi7tmSprYQoFARpY{u#|y3xGp32Cx5>;< zXlLcptBVKll+8hMZIPXkN=BQPpx;m-xQ;n3k?@71zDj0|jG*KGXYl!+ZSt^Hneo{7 z^Pnb=<@0izgT%-}d$kV2)nzW%M9+b9aLf17MDC4_%@+>FDv=(*PY2Pg3722H%UX(Y zuA`6D-N(xctxWRk#dGQFO~exxTv-}n*htf3u;+}|G~bsUJ0nn9hf4FSOy`BI(IJKS zMy8^WEdDD*@X~JYDP_Br;JZT=@VDXvtAMeH)eceTMVE;bxBbVFZ82AS| z4NDePyY9^$354A=yn}pDUz2}K)Q{m&ZLWthv%Fe zx+=}^}qJFT&au`T6XQ`;TSd;i-l`0ox>CY=^D4u> zE>=_Wc+9$NUE_keJM16K+h#5{NOuw^2fkLLR>WMjdJm9;zmVC2)2T!qiq65DtyV&E z1O8n6VX}s%{SE2lL=S=Hz3t+=+?k)zTxFo8vJ4?&-gn2Ky;ZpUoP+bafy2t$bj6 zSf38tU5u#K+m##Q?8ALX+g~RtbH~uIx_cQu635%Ksb-W*u`v z4)S712HRh5^G+7&y*${}ZLC5Dsm))2LpyyOv5rb;CC$Mbq)ew z77Ha_6x!1q*{N4#Hrq6_+~0nuVbUF2Vmk)<(pyU%GtRrZ$ibgGz7E-FdVI+a7-8;{$MM~eMJ@eY4CM0a%c-rEgNDQ5gE&%m znk=62T|owS;US*HCF(_LZC=(u@v`IK$K=s#Gow~_2(<54>p<;CP|=%}=5!(`>EhMo ziv0sc!httGX!g>9pc2>}n-BHe-5J)d+uPuD(xN+HpPCZ-HL+w?;vWA&9T$3cA7(*$ zw)CrYu+}4L&9^b7w0&sy#W4^Pbp{A}9?H?j!aO^~{HKo3Zt?)L5LL&CIvJ1ZFcR=f zbJ?z=s(~es%$4}s4kUr6PEYzSMvHDZS9g>bLBaR-=RSUuCu!dNU%cj-23llpX6o22 zu=zN2w|(u&U8^hs;+jd==0k^Jd15FjxW+( z3mvjt{gn<8U44v4el=Q%c8b2Obotpdr%DKo;De61Q@uV8bmYk14Y)(9!j^4W^+a35}eu-$pwhr~43jop4Y8cA+&!O3gy-So<$dHCg`HbTL_ zx_xurWZD@uPicUA;WZM{UTZq@u%Q=l;Ur_^^QnH6cO`#mx>Lo=WrhV@tj{WEIx~K@ zP-L3bqHF6=w5Me zgD-y>CWLkPc;5s7QNPcD-N|Xr)e(XDD)ohrh}7rR1=s7j^DsFqjzM9kfM&aQBHceE zlD9`mlmmovhFM|f27;Hzdj&gd_ytzh*L1z2!q=uT0FnTJ7y!PI-1BG1Zsu@wK6nD$ zXCuX+Muc@@7#(ePNpT^OY-LrR(q-9q2p+DQXHBx|%?cM!!2H%pD6w;ce~Kxq8g6z@ z3cuGUGE|H|lF54{g)Rp7MTtSY-S>8T{#XWo3&RaQ5?IivRWY@()bkt{yCqalYgD+i zj1DX^=sKEbBP$fAOg}SXC+2(Y8!rcl3%KK&jdb?WeR4pVn}0~xuf^__xEMbI_@c;O zw<5jT25BuVnY-vXw)>V7-^rBt0Y4;wwL*wv0FIJu7sqV zzE1AgzPF=KdsL&~PFpz#sE8QE2pF?)=2fg~K&#sJ?l_SH*%g4ZuCrMKMW&A=Kz z?bf&5m=k#7POo;Z0cZEi{`3QGyqvEXaV$)NHs>Jp;K1;_mq&?JsC{r>x=^6T&&KPb zXZwW%(YE)$NHo1Ot_OcNO{>y9EQ;xC_Zc*p;lrhd#(CXg&Iz(EAQd0t9$02Ne(^#n zJ1lLzon_{~bV%5|hg`C|fE>fG{y-~L!yyCy0?^2YAeQmz9jT3nWZ3{=Xp~n6+)@Oe zV>s`$oULVB7Ug4!6Td#3J6sePpLGFJNcjbvp5QL(F&tw1PXJga3GQ^Lv;yOd?t8%JrNW9CwoO zbPH;nnnBl{l_{D0W+5e)zU73;TA^9E#smdX`3*F`b0 z{`;4ZOdv+=ArU+T9$ksa;{gI+PmhrmI;Y7cr0gq3k7Q&wpD#*Mjv*pe10?1$jfgu z2lnM`ilwjZ?xJha`6T4MIF1keWf08pa?n7R^zB%q0`HgRKQu$%7w=|RX9ulZ4+Xn$ z#8DxE&l9+xz1)tYA-{n1PtwFZ+^wJg;iMV0N+x<{L9G)JE7KJ`)#e2B;Q?RO=xf=!VsnAoSG=S52RYeoC8jD z2J7^uXSc-sHQV-HNQ7ZB;Pl=DhO?r0S9&_#yPBM3(zuGD+jNr$Kq$4JWO0rJ-)SLf z=MBBA3~2IgEg<1_?2x}(L^j^Ohx7C*w{x0FnR3M|7(NX+61*-Jqq*mht9{=&@pR-r zMah_h#33NlEML8w!5y*GZrB8xK>;+76c@WJ6aMAy`EFMUxcl+X@O15_tM5lsm;o6Z zCImU}@hbC!_$zi#R3kW23HDR^2~?4>2wP^9mcXpn2u$!q`In4c;e7dl@|yT59wo>V zmRn`uVb|JZ_86=-u}?1W<&Y0f*lt?SKd&N9YFLOzx)%B|!AC;6qv$Mfeg@K@xnw+} z3*3rAODUH1eOVqlRPYHC!Nm;D=ybV&yJX>034ED=%@7DZ2b9{N&7z64kbr~WE`C5` zdaV_;6X-IPZ;S$unmEt_9_BnTLt1nH?YSRoJUjt7z2L8_92MoK821z7)AX@L=K>BJ z$ATsn$u10YMGLQJZUb&cMr%35(W;OMj7S}*M~XyskVkaTTSkN?5ZuTjtqcsSy5`Dh z)Ldjnl#24tmciVtOJ^CjL5!P%!nY14b=a|Lqm<+pxr*@8Bg@5SZ(L2_DPJ|pl*o>7 z$L0lOT8v4MT=>{U-JM#q-U)g&9+vu{yqoJ3{}W^(ssU_Y*NWy+(7BN4?YnL4WqO9F=mhsM`QHfDlcNi@a0~&7vAldk zWq?E3s*2KUN_{bj^#S=iE~JuARbO(?#T1sRzqxU9eJ_76dn8UoKk-aYQwEMNc+EGX zO{tLl`DHj95u9^*9Rr>e3y#c}gm1M)_B5U79|}m0bX1~zko#{L!--EdGUTCaF>+!| zDqW$Vqr$>*IA+6lYreng*a=rW3*k)nzrs;0!J&YnSA!5@dt9$BMs|xJNI!$!*PatG z;K$xY*MyuSGa-oA9d(rK&vPa+YNR2}o59mgF>{JSdtMUC%5%DifKk_SJunL}d84u^ z-Qv7AgzyHqRj_BrLo_u%GVS#HxN^)vu`bGKDuf*O8_=c1X%w*(p9Kz6X7TCMG%5;s zsE=gu!r>CQ$+XrJ|2h_=p;F_1X*SgFoaIMHg=LnF*4+H@lZ)Z%t>8}P-fLb%CJ!c& zI;(p*)g>X#q9024I1O4|vXfexwLiX&eXsFrX1le07^y!g{+v&q2{7zoOAg$#!Xb^< zYRjT14p&Tz;Ns5Kl@wQ-JZ_$wJt&1(nz&=dk@kT^;`)2=2wLVJL_yBCzbeQMG5Bh; z9ojGwkc+Z0(GYa0CVmAC^gN-qatXU!m*ep|R1Lt03KK(G>Nit6ip0z-=p8h*Eb$!)ndgf1Y18ja^Me z!i4~*NyY`H8rnh-`f0vTCge{%d`($s_50;yw96o*p(G@kb_(#rHfh-z77OK3ApQ1Y zN_VhPZ_}91+MAL$W9ygfHSAX(Mk)=M_m`Gt8dVTp4L!46_g9RqRGt$uWtRHk9ZlJ( zD#${4`0DY{RvYd`VnJ_F zQfj2?^rs`48p{~eECAu(4Wu$b?wL+jNxSH%Y2SHMD5O&Rm`sm#@IBRUme=p6GOuo` ztq!eRGh!X_;5=T9wFMgvpp49^c0k{#{U}V=-aS9H-QfXn@$v@CUJmqRjooHKuTS+9{-r-5;s_747ARDpbJ9*iN*fN41a0YrdpG;11TepAh4JoQ2-Q7@ zEO0FS{+qAH?S-}cV<(M%HNw(Sq0!u@|2XvzBE;JQfQOGMSp z(E>9HFZG1)x`X~ircz`pOD1c{*d2M^w#EIixBt`kWv1U5)a)`Au zL3x})*ZZ@u9QfR2Tq5roq2tVp2`4a%g%{Q+ozTCJ(+ZPt9UBsmQEWKshaj%QlbD4XT4qiHc(tO9loqcp#ru;QyY< z_Pgk&ZAF<5Hh`0wP_&mMI=@ozNtB`&AhKRv>2%1)sNUjaw{ z29BtULV^<&S0$^>qA-V&`|Ws$qFHc!oJu%E=o*fAx7JyGyy>J*q4gbL%$T{zXK-Ph zRv$D4>X9bah3a)j;0sVgkv)-w)t>o5LRFJEVs7-t8oLbfjhSVn=u`M%QUlm5d;=$G z57QNVr8Tl&)RFc45kmmkM<^10AdyV;%`OH)8VR$|j_NXBBH3CXLghB2ghZYF(f}^O zciV|B?tRa|aP}IV==!1NXe+hcKz#l_XaXJJGZ8^1WJkd-sjm(Q!)u#;xpB*lWYIfxv}h)xW64K<@>5XX%hNXB&}!f zT9089)fXpaffBs-3zfJ zLF;$=(2=5{{@!EIDv;U(GEq~jIJ@WPxTdTzDp`h)*Qe?QJ5`g<|2V+}?#EGJkx%)N z`v0*OiCveegdF@I$C--oLm2SPvCg)u-r$Ok`XLL%(xkh)Sti^NF+m@&Rtwy#FKZ^G z5+8J@k?hu2N`Fb7t+mr?XOIT9*-8*eR1lgYHB!;WO}Tg@RRR_hKiJ~f1&7$c$##MHE1*BRud5p+2DHi#zo>iD&pv>*Nh ziC%!;RHl9a;xo07S*Un0r@~8k;uinXi*>Ix6yAt6Dz%9UhTsAdbrCm3`g$WuTx=Zq zPrtlq-te9)q9anPwX0D9uTTZ2u{cP1l<)vob*V%%N=6#zZc;eg)6f+ zgcFF|B|fV{eL>|aCBxvS;ImSClZy=n8(=rU(+8^ZLuTseOK6Iqdb+ME>+q@Yf@ZM^ zlnzEyt%t}D4Sdho(b;nCyK;|X*oXd1T6JxudPZCMWbTY*4WLBNTWsTvAaHkZZCl)| zxEA=hn^kClM0N85o0_nOt%#IgQZm->-hBJe1b{u* z{}gFp_5)X*A{C(~EUHGi61@DTA35~aZS^19wVH%EZNb9b{XV?f`NV>vEv41!wt>GT z>wY669Dzh+)b>A?1Xheinyt3~#*aoI*84=99ZZ{U-evcJ1Nw7a|)!D#*Ts-v-Cp<0#sbN+rhl^^e*Mm1B)p{b1_fD1;H#aKk4XKx3! zm-I}D-A#9#`??~N8c(-6OOpF&8pLU~W=1`rbm=V~^Sn5fF?_~9Gb>zRZtG-9Dy2NV zz&u;{2{bj_LF`IfkLHOOsR9%?d!7Gr1&Tx<9&yUz7(2n>Tz&S*rYK6uY$4{JEKhT)bvrt$g^ z@`^BQ+hN*sc|bHO8^-6H>JggC?^+jb|RqUekR+L(pj%^XIt%%@0%NI8Jp7e+vK(FpaoXh>++-K2qVZuniUv;)`hXhUQcyMdVeR<{yyZ^tB3*ieKZ7G&YG<99`$ zo@E|+{eB6gd3YnIN{BS?7%i>Skf@&tAoxmj{^oEkqgMQXn0m{oDE}^8cxLDZ=|)f~ zX_00?M3C+-L0TH5VE_@Nm6RO1q@+PgQflau?(Q65IJf`joOiw7XD#@!@BQ2Rs$H=6 znz02rBn+aoc)_zNG~QOA*8a&lXuT4=^6U_6gl6V@<54> zz7#u&ME`1sP(Mzx;(odj7U{3nU8OILWjxS?Z~WF9{7-GR(Ru@3Z0QDqCG6OU(b`Xu zo(O-@#Me_j0iX{*3ls*EB|KJ_aSW#x{Mh66!ySm=c z-h!n+XS@WdGbZkEfUE1LHL8#eLipl6X zN>OK6 z3%3WOPg_DmTdcfIIM}+QCKX7B__m7v0tU&*TYUsJs9JqqJ7uWlxELWxl-|!w9GI#ZWgF&5A=(7zj$Mrz|h??6_T$PYhh`Z z=MQmIoZAp_k;Wp;qwV^=p+F3ow9T1wIT|XDZ$5(Lby`JhH?8rm`2=pc#(N(a1dQ#8 z=~5X5-w$g4&hugjjl7ovqoIOW0|YQH-gC#GU_5X{-pvwmRR9gf7n} zEe8Z0C3D2y->5XSMVfA6K}R%)sp<|5A;wk@@B&Kz>)kK;TkR`PC!J)nNW}pKtvw)W z`ln(2r>}Eb1fCOX#I+sz*Jdkc9=3c1;unk4ZC1adeUT-o{8bjaB#ac7cug!oql}$-S;*0UlIXa54*Y@%#usRf+}w#6 zQTPkSkRpv!XqJNAjA2RrMtTRj|45dkjdjs%(@QS#D`sT_Q)H^TtZFMHy2yXww}P#A zu{RH+xxwR)F35|<0{t<6L`i2wMdEfF`II;Dn@BUP)fW!ha_E?s4bCR_`n^~oA2C0S zNojBr;l8ZW{G3;48^7Blh_v>DGZ9;e+uf>_q*$m8eO1_^f25;PVVd| zl<`rk*WZzqtkMhGH%YY!FMz5eZg{S=?K*iDo22+A-89D6@80!}6uQTn=cIQ3$8<>b zj(X{s68RPnsp*&N9dPQjUHGd5s;1(NOuj8{8xc~09(*@lP<^0YT2Dq&XFvypxC(s_ z<7qc;v$m{JZ(3)?p`~Z<+8VPY5JG{uL=Thq=_`hJ1_6#+EEMNND20H&0ek|d;lD}+ zq`DtgW`J$_2Q`-La?@Y4krvNal+vur%+YcLlyHn~kf~pw?!(*$ZUj!yc7jiT97vD% z@JR-m9AjL^FEdK7xSy}TgSxOYi+RusGg(Fwt(a}Lf;C5&$KNnFXd7KN*l4aShFMRN zCgh1PEY_R6B9*TG-Mni_2X(2(|Ao;#21W0K1;ZLS3q;@VhiN*1l+>iWbSMg=TQ&YR z$VqW|{JEY-!**d;N)W{{LthAy{fIF&hlNX*b)fx_Kj-H$C`bCryP&Rv0t8Z{p;;>2 zwY-zO(dQ-9qdJoY5Mz)rT=l86tTg>te)p4vaLMi5rD7%SlqLtrbF+a=I#O;0w% z(Vl28Xh!1eZbY~^q`6t%pWX0zBIPX6AL)^<)o|m~W4;SH@fdmAH1d7iE_?Rzs3-3q zFEXvEz%`*Mbe^>KP)VUn~+kO z#VWB|>hSjpQ(gX~buJ@_xw+}FU*{cuif?n;bIf)&3l&K;$x?D~-PNJkZ~?i?>#8cb z7^=1_GhCe?ZhlUt+oghP)_r$-X(Hq0E$ErvK<5kPo)E!GSYvfC&h**+Qk{#5WgQzKjn8ftNVBrGuG}SeLeG7-&TgMYb6+V3EHI{sKfCW> zKf9edRol|+O$sZFN!yCei&YJDH`pTWw>Z058-JD$`l$-hpm?J{&4-4jj3J+F6+R(pZvd^e5e-(th;Wf{kz1ozDxc*3s^LK!?lb!32OC5 z`E6C-j+^R@Y~d%2;$6>(RF@#iBpovmDvV1vk&~>ygtx5yRu$crmg7x)s*nSdtZjE= zd#HXz0Ab*1>nJQ}()@ShV86!iDmsyLbMkHYY?LV6Ai9_bwUhXg`fD$b5h0y*yS zu0andB87*mHcAJ4NOCBSalo~~&3SaZA|i;z4!n@f5<`7gAp>AoN&hHiLq#6DI_D3- z-nlqf@tRn^?57iEP6E0X8@;>w>*QSwn?@&)pMFcxNlsiDNZX8yN*gafQ&cNN)miEo zpAftMUPb>glSe8cNQ%VyU{E=thD2?30=pq74u3zNk^r+f#z&`9)`0XHtqIt%xn3^^ zzA@q~!eGg(hl#;s1Jsw)G`Evs#<21t&f|FJC&5+~898X5z)y>$+i5srxTCSD;C&#R zlGWT`sx0y(k7zy!?IGBhO*ba9K^ODd3={jcgS1Bny8!wMJZL;u3Jn+A`4tWB#gK?* z9pa|M3~PIqkDLPO;Mh)LF=={#r~y0AopM*dEhsSqPR@m*;iicBzcdAfzF@;8j*C+G zG}d!8r{UWzY;UKnEO=^|Y?bG4zie_V-}1;G>Jf2XBH{KND^`LjQE}^u`|Qs}Mh8n# z^Mwjhh zNjcL67;c9atDK^@HfJ8-c!`lLWAquB%g=f&%0YDf&!%b9+JeyEoID3%snwub*F1O} zlCMJar*6ZYcVw4RDjmF=$j2NmFXu#7?5FD`1Uh+sI8nOMik{+0gtZ=9*d<3J{C^sq zVlMxzgb##llniGDRvO*k$(-G#x~pxj0AE!G^jpA!cwmhYLQc2s-MNL+F|8cb#F0+( zzCgOodysUpS5n>aZ6N=>j${l}+mZJCuNdQlJCgqFCWAEd`MhCT@4n?$68l}?-r8?} zp4#cy#m!nLunKui>1%qG>@)Bw8M(MjI(|FQm0YTtx`nnAFBTYz;LxNvF5amgn9wa% z^$kJ1&04&%_jCMS1w*Va`MYHDC~cpP5t(wuB2o-dz3RCNh{uTHjZ-@S@zbHcJ-rpC=&9k^c@eD7E)=|2B3X2y&!*cnkx+s?D;Wt)zq+!}bJ8F1RN{SB zqc8Tt7?_Cz#qpjBW@sP~o?U@+$)l@g!gbK=8lpGy+M`JUlbVrsC7K0d-POecKjZ8; zqzLi6ZH%@yM2h7&g&8_I2CwS7HFoBT!-1e zQ_&*!WOF|<7n5y+cV$7FjDNV>D>WJ8~TIuXR3_$_LZGdF$BbzEP zls5C2Uz+yr~C~? zk=Z}5T5l8RJ;NWkvwU6w)m#M_GeAv)DCu7T50j^(_Rl6Qa)=P|k;6216|aDfTnGFm zzyT3zj`s=>&d^f@ElZN@2fomz2);gg9Y^S@_Uk=Y^5R9m2gxr}0gL!kvxnWCU^8&* zUh$Jhym@nGZTJfG-#N_+_|;iuoH@M;>eOyusZW=e<9W%DGpl(>6v3lAbGcL|LEm5! zvG_l94v$Q$=SS-u3+QY$)eabl!E~dP;B);GKlbOy9U8kNgT9Xy_^B9O8d6e}?M#}{ zzcFUW=_Ye?N^^24V#_#^%LKx8vp#D;21k+_o!gd38b?eiN-$?t0?-rLtiz12k@#C?-LS-?YY)SC;-=xRJX#=c2{49YPi&jrEi zgS54AE%rnqVeOx9cw-2Q)N@m&D`IZ+27N=l=O7poM9c9sSeOe`ZqLfNV-*?oYUjy) zhcYWOaS-MYdKP=ilNL)`bUki)K_VD9ajvOJEh}A|&I2vTEj@cwH(yq`$ZYH6KRoEK z!bacn1SraEz>e|ve-6A? zRe;k&M4*+XW2cT27Z&=@Hk{9ON>9JO;NHtSf+_pA$woPCxmx`$I;f_%BTyjN!(N8r zj=8lH6w2t!xS8#5PWye7d+v6?8UBGg@00q0jSNl9&7gZ0POVAQiAH^1D*GmDtV_%u zzq0i=i{`!Jw@etsTLw&BqTk(aOG4NW!3IT!Y&NZr&SW>tB_&MXneDto&a+z#R(N~7 zBJ)tf`gm?AZ9+U?t8n=XDLuB&_VYaG(e_LVOR?G1oaE2x{O9xxXR-^H(SxBL1&2Q= zdYtdp7b49k%cr*xv8dN2sp%%?>zrHvwU`)}6J;$5RrLyGa@r zh=p`vJDt1td$Y;YiDg@9qQc*K1()-G+jKAsm_E0gE-y+`vOm#aoxscv2eF9iNlTAo zpx}`c%tz7~woi>mb=f(PuXauVxhHkU#i4g_lonC@M)`KytESyj>_zD$t>901KFy*` zX41MuT}K)g=<5;L&t9Y&s0#8E?qY!@K<|{`k#!}3ysBt9hc37yp(HR(e(ay@27{pi zS(1%YRoPZ294T1Nuy5QAR}#dd#rkYcQKUv-G|98yh5S^MeYX;kvpv^Qp^wv1p z4rbaCN)!3l>S+9l`O=!kX{*0URK)MJNE>P2PGfIF1dtZ^V<~R;HJ3;*+5JM1YI3%P zWV+e$7yZ(tch$O4_jNwQX%f%0znE{PkzV$RY{z8RnGStNl^(cx3}VG?Gn`o+v@!S> zF!NVv^tujps;1s`obH!e|F3wK2%*e=y>~TvvmKpvfXCqUVUHSVbi*h|#AzD1jB@RE z0}NYVM=UoRPL-~wRYh9&Yt~|@pX49?RGjr}^B-7XVW_Zu1N_EiB(#5HBQT&Azr}uc z&sJ6@Ok(tGqY&kO+=dQlB;Nc7PER3;{_m9Y++F0CpY31(j=~+DB#hbNH8LirQ5FBk zw}&~Rk6ww6Pg=G3t4ed%_BOscl#E!~3zUSpgEghEM@h6E*3PmxmeLyx8(W<2m2SMA&+nCuLZ`DNFlKJdARwuA zLy~hlQX~2N0j4K?!Nucs${>)Bk)RC8B4F-x#(oZ>lL{7Kk;nYPCVZ|9B|pXB2R%WQ z70YY5*QqkaAKTcKeaR^+k(_cAVp{Ylz{bbtjNJwSKjA^iDv00kI6Q|#Ue|Yg=_)~G`R-m4ziaTiLe~prpx|Y zP}}%3&IWO>7oOsO8>I7|*U;V{;NGYDmeBPlh4kticZqoA4n;fJ!=Wtc_Njy$vZ0JxSP39{Y_OmxhFrnMYq-MB zGWlTkX;IJ#AvHFdw6<-v)hi0=+K(WN9k~h8xv{|eJ&o^D*c?>LSjX_BIA*kTDm2~Y zDLwRuU^|f968gT-RasV{&_cTNyK&}dtRlD>@3RZ535D#Q5sG8x1=%J-uMzfO^!s2R zuxBVSqw?~%9oZ9HOto?~d-*?Lv-7uU6E97Q1z?XZRHd-ivJJqLQqKrs^Yiu6!#J`d zz^K0A88A)=TJbsf9^&JpE}{Jv85!V9^gCU^qd8qJE?=y>&ud5ozD7OP1!8C)ylbR< z0nw_lCm_eieBCO(xmGNkllgUcW3H2RUpMAEC{LK%FYLzxWI3fdvS89bx2?xkvgZ4R zR!-npXshTZ3r=MNQaFqDzSvbMYGsw$<;W~cy6U+Vt-aGCa9sRgtexyE5n|;~CbZ4} zahQLjadu#%^=uzKh;L~*!XMuR41nEc7WF$aKr@7Y>qbPDdZ*qaOWwSjv5l;(X`VjRmwfTZ04#NUL>+wF+rjW^{GYl^VE$&Dmdo@CExb?zLyZghqlPKMOK9 zS+J*|7^eQJ^vr!zs>|3w^T`W^;p-x1go7Z2E( zPnFVKS-LQGCZKeV+=B6F@3>06;tn+qLc_(+xNT!=t9Ibj=DaV)W$+AI=FCEs?3;PP^GDShC(7 z$)y&HjsVqL)9lz-TOrxWN%eFCm+Tuq2HSR)frwotRje+;8T}=(?s&N!0bQZ=EY@y= zy9I{nkMH-n2oUkM`Tp)&5k{^s5Bb~RQkUCg8KCqCqpZagXq~H}`xt72sZZCx@M$oE zdFL+E%cUx7e>FOF^7~yzSk9+gu09-d$G5Au(5Q8v4dI1bmjyx9q78a+444Ks#_Rhe zAtIL!hZWbx^pp`lD|(Ot+>2;xz5h`q{oWhB|Hu;GX!pyUV}j0UJk!(VfjH}z|ELl{ z@T?ONy}IPbgWQe&EkK8*0UsFmUmI{xg=$^SdNN9nBBUq@d;HF7NC&F@nG^w78RGu^ zEJU||UDuS`FdbTcPAB{;FXsSGB6LM}ZH5<-#zhwS@DUin&r=_qUT?U%1=s-|3qbt$ zG&U;pWD?upiY<(&KpeY{C#_MogoPoq*1-EcZ^qWFKIgMfe{%EIggWY&OpX;N9}ngJ z4n|{cpBI3PFYs%JJ%R;GAMdw-beZ6aYFKJKFtDzC$QP*@Yj7!?;e)p* z@LJ}h_zOXB)q%_#SA{*&-!?=fd?ULN* zIlCB)m$mE_YOD{VJT#3hR2-ubQeVw7N_y^>gJ28QY#ixR?>_{)fJL8%>YjmLQxO$9 zzFq#tK{=Ao@sf~14#P!?gJm91X?wa`MD_2}kv}X|wL;miut%`ogEA%zt0ZGkhbnOX zrHQ9wrmX@k1u{WeT3qejU)QeQ>X{RzM8+5Fe~bH_G8lS0bXLpPtHprhj-39pmcTB3 zw`p^rHTc?aLR4z%Eug}>h&h?D2veDXf=BZOpnAjUbuuD6Q+kiheSiCy@H*_x>mo7T zVO)!Dvi?Ctus%Np9M}2Uk8QljwPJjTAc#Q?=R+0G{+AN>*+x16+X}j)QPQio?Yf%X z91}T?Kf|JI|9M{A+>!>GxZ$~5hG=mrG}M)lIj}iA)#H@m_@}3|+(EFh#-&~MQtUH2 zNM2r?w}kCj(nhiFuq~ka5_=aGYD$Xgbe8o<78kSWf}YbM=VN^cskKqdl`9*H^%6$q zeutUBOpU%hCe{i-RBMZR1D0Sci_2N3CD-ZW-u~mDx4;v(IfCK*{boV8Kdd5C53rn@ z_z?PMpW{Ut5!@K%i00x+z_(Vgxm~lJZNSZe8vp`X7aAWm@zI;9m2_~*ht}}Ta)V+p8dxlasSlm19b7_Q0#m)64*!ioa-AjS-~35@H+ehx^3me!TMZ8> z8dhO26~qhFITNjTPX>Pj&tk>s_GQ9x2frCCPGIk+uZ2e8D1G~wYgReO00@`gGZYK@ z9nqXIk5oumpP|dQqj}(lV|)Wc#|-!}IFWeY^s>0`d+%W2r5F^FV0jsaX(pAyoblyR zCs;JKPIlPCVbzf~JA z%9RX~u9`v)SL`f2nPi5g*iUSq+$I9r9ZKga#9*=eV|$!kKkpBrRQi7{=TY7VlcN;c zG2>zT>}z17Jn|sM2+2Kx?uZ-384CXf)+ywdphUn=pc5l;)VJ-yPhVD2vHL}9%JkvD zrE=InkJCX7y^+vM0D)xpzz_eqx!EWNguHk^a=&?dpY~5p_}9kp6O;m#wZeI0{zagW znt(pQ3XzSlUI=?|nW*GHFAwY2<^j5nuuQTwk~JZXp{-*7;VaRD5?3;Yu>&P5b#w{< zC#~WeTiuTjSPny5Gyt|Q;nWZonUIss^3Z7AS4l*GsulX@t4B4D0?3nnAPK7FSo2g9 z2xRD?&}uT`T~j16J4y#GkuL2T$n!iZQdIYLHj75*LJXR)XX9NnU0{QWhR(8JjUX|MFH9z|Ir_6fj5%*}HS`KtG}9zIAH?;(vC1<9 zos0ejykUg3GW3xw%&6mckGH`i8>#$}VgvNe7-W6|*tO{erL9=Gi$rbsWgfXmDzazM z$LInOvu)0l;#xFoi8e4U^S+T|$Rb^`U;CC$SxA16+K+w4Atl7RX~6L{q1@Y+Z|!p6 zoV!*MM0t$y15PytJ?!s;w4h{Ty7-s@>dgxlJQEXGiA;ZAJdPs}h`z5*N z?>Kg`WX-ei{S3{okZ;arfK=HKm+3F<=F#c4Y1u&ayI|�lM8bM0;{AmIMe7HsUi` zQ?`(^bh~M!|F`0^;=oTrvKK-_^SoDKAGxDWG*8sd>0Vss7Q2IcLG<>H!-E+KKMFB* zxI3-N)H1)8Bjb7sTi(U{U#xhTA8u^f3M70t5GT9^WUs3Wp$p2_O zQc=nunEm%IvJQ@pqGY!oLY;OZpN@Twxy#KA2{ZlcmVzW=888iS&=Xme9LbYT?$+ON zy{@!FpF-TvQMEPLQapiQgwWz|%ZFhAsX zJd0XutLK_hvHX#z`RJVffNTW8M&ul~Sr}YG8;8UOmmgymqDlB`Lerye0H(u>mxmoN zY4w%ke59RbHU?y<@U`X}I+{Q!4O)$}cx8J=6XedNB0&BuTOn@sl|SU${3C}(#95J1 zJElL`HSX_P7XPgj00I>AnI3AzZVG~!vAlaLZhGu*Us03e@kr9vG*FuLxZ5jQy@di7 zTT1?cod$i5OkymW&7IDEzwu(W_ENrVXH|G#H4*XIlU2?GLnkPdTl+Id&lgFRTRuXj zh}73`rxvnF!HS8u`iQ5H0CoW>3Mr;qye`OV5ZM`;59~<~r^j1~M`nAKE`lxccKFHM zYd*&~@A-)6-Am=dE57eoU!to)-W0FMe787Ydj-P=^$Ng9c)!g35uatT%LGXuUOfj5 zsIOjvh}UL88TV+eRLvI<BpjPCdXJ170#PTGV+buxNl&Hx<;U+`|D@YO9S3!vK_}9VyUl78r z{`TpH>wXtKzj1fl6y7Gr`V}jccVIB(!zq!ZJSW5`GqqNOf>s+XB(Wv zVqP8WSo@}kj7_l|0r|1_zW;O6^SM)Mx>1U!_)JXNvd4IKZ2gKZ;HG(jN}Wa_qsOLT zXy23Lza)sN6@kc|t$ZUcG%sjCIU)^R3x$Fon zw3s&wS&Al6-2euRo{dt2JJYocY2`;A5UVJP{h?*AP4WSWcqW4Jf{v}XeKgD9rlUztO zi?yl5SY)pf+4z;&`$X@>fL%_zTsO8I#w z=Cl0Rbg^2$T}BiDSB<=RAjStfT?`PCQ0w;my<^KQE)GqDb8i_b+D38Mxj(Kh%Y7viiKg(H{eX|ub zLz_8~|ApMAG5hrGHIu>ya|7jdsT;HNkxB^3C#z^G-aMy!tvJ|IGERswE>7j^IF5t$ zFO6Fh8)JjX)9&Uk{uR!4AY`OLjnyib`6M$&57(TaZ~2cD(VyZ zxSd@*h}yY{dk*}?F?KoY6oTzl8F=vtZ?nr^`+KzqhmsV7bR!vOe|$+mk(?UiOUH-Q zCYfjwX>Tq)rODG3bSaa3@?x2s@(2S)DP>)wWQE^%Mxwa|^6FMZ4}1A&2xRuaI3XEe(x^`57q`S@Oq;7WH^m2|SoS){Z>G&L% z#9QP8RkMESQ{U|V%L(%>{1ETl%TgYnmE=x)I@s*{S<=@Dv68C@cm+3N2^mc!U56|- z{<0i$>Pk)n!xXwP9rgVL9QqfU?RPpJc!jy`I$JiK__exlU41kIpjo`6l2ON$4>yVY zerArdD`O`o^`c(S&mXe{&by6-hg9qQo2`J*$N%wkXEMnT$Um|DllKqA9m^k!yWqs| zxCR!8aU6IC?wI`{Tfv(tSak^y;c`I0ZVVY|K|oMLKefi|mntx3Gb~>2sAXf&e!YsU zMj`?26ZE+L@fKV^jBYKe!(;`GM7-N4Y-$bKOYM~14i(CYuL)+Hj*NwF_wJ)dD@t>Q z@R%F0Ai|cA3xe7|#ql3GuCB_yf;!yDUgwv%7#FJWzO?1SJ@vtyf&6~oKc*yop@O5! zo!Y9*!9Q<-LVtkagoZJtHDSI7;}%KeQCRX#Q)%kfhyMPAi|)r~;*Ur%C@YRmRs8c!#L$I?5oCB3T~8I$vfDirJ$D@ zfi)JxgCSK=hP?{-X$BuweU}#_Rgs`$yYP-IJbAB!*zETl`>F4n8iH+g&xxwTv^6;V z>*qDe+YX;OQmq$K1N1F|WqZSVs<`Cvkp56+MFJYBwWvR0Nx2pQWW{B24x0FY>G}@I zuCHQ(8{4w#z1cMrW&H4?-b09JtOAtfdpo5Nino0_zi&Cvd0%RY`^{I3capv67gocr ztu|vnq6)&X_^3PMaeX8;*Iq0Y@fn63TF~$-b28_)+Iz8HWV%NT4~prCFuA&QlBTtK(Z1Uj=!492ql?dzsLaP!UKZEFVFg?g1?jBB?kCiu49l)Hy* zUP?Mu$1hFScFqyYz8SRs(KY+Iq@{LjX(3_Im^*O+=L=sOKOXI#qkj%tKMXqYqu*4D zC7uP7SDXEA(W4{?5Ee$HH+uTrWky4R*ha4(`|A%;?GZ0UywoPymj}5;j+n^Q;MhpK7?KQZ+w}#LSlWSP7_h+RvOGka%*g#x);p|?e3*` zx+OC`%Pf4Dt5v!DEb*=^pC&`pGV*kyl&hs^R?Mo6G~na)ShoAl{@E4E1rfJWxZaj7 zHoLw2b;G|GYDn0E^`a39X~q0!HXPYF_{4p`6j(fX-!Wj><7vtfwj@w{H~SUvrAh;d zd+&fo_Ba>0dgRgD(Y6SPO8zsb))lV^O#<1DEpH9gc3d4EmJ;I&KL{Mdd{?>ehsn|usc4`46045cDs1n3XMd^%p zSghZ~+zmqu#Q;fXtoX9O!QHhI9lb7@QN!11B5{_Eek|MF^u#8-t}Nax%e>1^T|?gC z4~^JpTd6rcUG;uFUJe2T^1$j2t95Ci5dy(3i}{-%)u6YbPqlAytzvI5><6JEz~xDg z5|%x6t(b*zD8aihPq}D#q{uu@XB{Vp(EWcA27FPPm|pe((dnEs;a$h zRtDoCB%e+6o-GY5-rFjgK-bOoqKQOf_!YHImFm>;?@QU5Mj#$WBp?Atd9 z+%ZuBo0HY6PF4SN3D8#}J_Y8q8b|EUSOhk2Ew3={Q1h=QmJbgwGD!Z~zb<3#pH-Dm zEB6Nb0f?0}Fz+ndZLhlL=s%omGmi>&`c11v+Ch&cZ0hmtz@QFE@+h)cT4oTryYAD@XT?3v^jL=I8xusqA49D% zCdR}jNWhAR^m+nldc?2WeJ#+N=UfqFD-`GCY?a`rtuAY@MQ&ba3pMB7N^UKHQU+~6TmO}sMM*hk92*LonmQ0y9;$v=J=)PmogbezNXG7Kl zTTb3JMlT=vdBcrv{QmcP{p8J7H1&G8GI`E}I^H-{dPqNQG<=o}75<>M0I36Ze>n*( z0_an1TcF)U|6_jb_d*dkp6yb=7__(<#{F;;|L`10P2`1)(R@;LBN0}_@Ub$>KYjfw zIk8`0MU~)(QRerSSFcil2(c__|NCi_11b7M;6=sq>q>T5ejdok@yHM=9n-EY#l-~- zrfg(v82H15*x~I4W*o+Hb8%8Ad(z8c_eav47a>Sy4K?(Jh0$KK&f1ksE-y^+qDSb{ z)HZUwsCiJ?xKi12ag?Y597MNqp14K9!Bt87VwxQ^V$6sxBF{t_6K>p_)BBT`Sxry z=&JXDB*648Yt_;Xq2A_wqdEZp%RHoG}-;w)~4~n#fhRAA1ui2bj{|%Rr)_z~c z?n>q%X*ot=xn5fM F0!oxU#I6x`Dfaaj+UEru&U$_$hUZ316SyS3fUjTYbdM^+ z@|!IE5YmXlOuHK34YT?OBh|KawlTKq|Hrh<0eb%^TP)eilEXLJqQVZF!>t~q<>Dgt z`TQR)PjO7pvsH&p8pXwYyMcjj#ZB<(fVCq-#~#HE%DnL4^|B(hwDIGH zq2I+o7?swyRESn~csvaBC`hmycLUo4XXLRwfclqvSB!tK<6H3HbpNufb9`Hd?N}@G zj!6v>@3$(XKm>Mum32A_0f#K!v1UKkr`Tj)HDvy1MIKGH&Il|= z-ym)~gZQs}1g&a=jN}@>$qC41u|>dG&*UW_zns-N7pt?*sny&L>T@gXs%Em<)BKmk z%y6an0?n2jst^nS(g_$$6Wwj?k!IWz0WMh|VIsJc7A{>)@!tkHM_!G-SBZYAEqG$* zm!P^S4J|t}(YW7eXDc3gZaw=zeT|CT=qN_B?8}HBSPEEcEcxI< zcfMi)gjp#3Mj)VU@?Gxp$4L{`A7!3lB%AR)f{hr{4C`f-ONWtdsEXE#=)D5bK&jUs% ztafSL_moDB&x4JntCo%9_I>5FePb?*b=he1+SFJ;bTFk*9NP6WX>@S477LTqh@m#I zVgBi>rFMG6bc_q`3}XZwIc1|rEA>j5uUH|{9t*EU3G_m>k^uefm+W=* zzp|$1%FBdo4axmzR3IKv*s6^!EHtOvrGLzq%@EK<02$elWZ{ApQpDG2%qgt37E16?q3~EO+$= zS>-q6ozBp?uNmBTCV0r`r+$_DDwFiTYURUQ7_>mI5gxYdotZbgx-rjA?~D?|C1aC2=5g2E<0$q7JNm|D+6QW(t!}7nHDVDZI1p zRXU2>Q`yPE9NN)*$=PWl@ec|-sxc;)e0=A0Ha_;DSMs760E-UB1>Dyw#S3&=k{@3r z*RJs>`!jobeX*On@;l3I_3lm!QV9BI}MM!O7h zZ;1=$?jsLQHd=avxDvQft!x9I9}E#A>bh5S!shoKZNGsU;`^^bk^inQ2JE+n*mX?q zB5ueak{^fhLGaw~i1gF!JKftAApb>_I|etv>M~*{7%6An0u9jsbF?!_CGW24nO6jR zj&$!xXxO0FL(~=)@G6R>_b3Awr@cEJj8)ryD2byxQRb2p zO&WQAGNMORtPKMTLxkVc=nSIqVuE07nG56%7^SD!Tc}Aauu=xgJ--t`W-v`39BvzG zla{4fxaVyD5>Pu`ja3<_YUO0okswi6wUVf>oeYfZo5uovHJJ;Vz;vAz3XK=BnPzon z=9%vU+`6w1&F3>=U;(h^Gj6qYG5**b2tREOi(wncy8bYG@RN05q?v)UP!c8IXg<*i z7>GbeC8=6xD7PuIshpQ~X zKihUbEyVyLNl`$6@j>XtB4kt(k4-QtnLujCAH+Ka?5pMv`1>fH;5C7x@h7YUn8492 zkzmYz9I2I{?Ftxbsv|c)V;v2-y(Gl4{BSxGyHV8>D|@@^?7kaj?<@4y2815;G}xts zKQZ{67Lms^3iTJl0%3Vct21Ly2VU(YCy-(2*E(3#bCS ztC%q_BM;JZdmZERX^dq&#Ogy*C0Y%bwVld8!SO;0swNeEj&e10!r-v`INGhUn~sFxPAGZQ>b?P$CZ7lNvF{VPpr!G;5f`c>(ld zw)H=<%$p|jcu_LLn}}Bk(5XY-5~wDM7#+0jloT}f_j1bfLMR6JZ+vQX)Naa(Wk_`v`DXnV!hm7rB8+1XRP3N}>&F;7CI%S%)05VDt@H;=ep`9cb9CCqz1|Wfn=IzF?c* z#P4_>*9e={I!s|<6?p3;xDtHlJ_^~tXA%*SKhARK)cQD^TMs0lyhqzF-F5o z`g4-%0E#BRt-8JbGq=KomHV}=n{aQ*$NulP(EQA)Yhln{Yvt4}gr}shD-sV6na*x` z4w1APp4fFOjm_8^AM5o%nO~gns60?-9gjr`U3Vp0=>#?+^O6DMj;_zZx-MYc(Hy%S zNB}%KX@GGD_^+Z790C-_-vQ&!;1Tz>+nC%?s zpRXrZCzC6J%7`UJ)Pfh(RpD-djxkV`mH?Fqgwyu{=YD`2DTZu7zi)Kjq+~H$DCQt3 z*iQ#x{PKD?{Bz?*WWc(qJC`I1!rN(GM*Og~;%_;zzwt;4yA@2Zj4=WKsQ%+}5L-nW zEX^ed4hR2$v3(bP1a5_kkmHm5z%%Fo;o>9W#$Ja)Qm|d6DP_$t%9f*j*|CZq*PJ*% z+-|We*Q`ij%Q!&rx3MNaH?ha;Vwk1}pR1uy3cWUzBFqma^{&an#SiidL?0iO6by$w zprkQy^ki^W#4a=kDM4#wY3Q^rSxk&@_&GScRT2pL;h7*J1M;*NCB_}-viF5;cd>S! zwd)pSMcq0V^rtLo*$&RsaG0!CC$JuGq?*{O)s}~!^IXW0d0^-dvJ+2qpaHi?u)tBM zy=J};nlby~NRO{Q@0F!-Fl^Y}e-)mBo&!cPyz8;D@49f@_mDR*Vi-ivh3nYtaB@8R z#ZN>OMv*HI))&Xr#&?Z=J`(vwXrfcp%RqAQl#hs`??A&vub(hRglnJrB=CWVZ9UQg ziylS=d}r)ULbI1rOXkR*ioCj1r?8Fjw=g-}iJMa5(MJX8dqs;$@PV@_Q-_y+* zPBTKG&$JfYwc6vkbt*@2JSHnupQ9VQO^}Q!9@HL)Krc>4{L^EIIpFGK61Un1*j-jX zJJ=o}83+w*1;RO40ciKc6)^Ka4@J!V(BpGCVIq}Nu$%kN&&g0RJFlBki_eK|-x0}B z7roi#ed__F^}XNH@|MqhCn6x%Nf;6Kf4F+fxTpfQ+k0l1A%>O)2?;4_=^7eAU?}Mj zL=dD)dMHI21wnG?E@>nNB@~fv5Jr%0>5{X#-{+j?{=HxLz}G$3-dC)({zHdxQu1pw zmw2n^Q8{~S9lH!S8gd@lu791*iq^d4HjPex250BoD9)UJ&EfgHqnG=mN;;_IWzdnK z-63J$ho3_syPd~;3)rC>y&J4d$nUjFSm%6G=30J;cjtaw$HdTCn$=H%vW{bcqguO+ zn?Zr9ju}7O{!e}KsV@|-LV}iC{uZmnKj;i78GG~EXjoWzb2!bk+^*y(H>nQDqi?HS z!#@3d6u2U!Rww4vsBQ{ugbJ^9&-QLvMSSc)o`m_r=|_*h$?5)Vg47ZaYTLC3z$PNd z6-WY&CQwiv^0V>_9`vvq&sAPgF@ytRDVzedZ6V6Bczu~!Fs82^_df5tprLw`?l>;EPAUwo|2Nji|yFBlxqUXsB}yDMg6a%nYR=3A|9 zx`YG5zO1Ah<9P>cGmAPMO=LPUdyU9%I2ty5nrM?WDEMHb8F%;cds-Zo*MW&ug+f_v z(fC-(wh58|4aTkPT$@%(=#eRNRz6>8@q;{>_f)OOIY+Y#clN#$eT`^L)12a7fBBQk ze}Fj3q4LP=rZZ{7?()N7v5WcZ&QV8z0>u6}dZg>O*6L09w8DR=9n*`acZD}Ifndb! zT|h=@C1R%-7p+b3wNUS2h|_iAh+TQRyrBP^3bh}pzzvmf0m9{LP^Kd)TEUHI9RCxQB}~04>mW3* z<9ovaW3gDX{Kb(qw3!l)#p7Z*GrA0qUe0?}#TMBq`846LvAO01A}OyZ${odJJ;+Gl z>Ud492EpN>WDVN`&PD_NBIBf`aNVJ2HG87?At8$RJVZfq&YRjm+{)#j4WzyHuY321 zRtPa1E2^z=`K=SK2MfL+A~%)^brkt1j*<_BcnO#kZ>rq`<(n1cmPBc?E|0IE+ zW`kz)c`eB}AE~N*p;0KKoTiaS>${~6Ofn^N8D~(6uq_jc#uIVx8B<`e;BJTw@B?jb zT#|-=keT3Iy^8|H3}!c4l=*AN`ANw3hM3DJJ+-?QvbT;7>IxO}4&2V!b#TLkMHe$2 zv|F}VJ8A`NCTn9}rweHdY-)!JrvhMNQ25EG%L18?nhj4n=xjyTdWmKW{CQSKnUYmi z>~MG_Q#6@#4cLGFx!OJAjSz< z$eZ}=8l4umO}yEpk)PC5X7{hY?BiMlTSz;OS{>az+oqR&6F*&M|80Nhgox4I1Dbhy zM8tUVvMq;k&nkW|qth_>o2P(l-4y3Pr&9}`s?bq>>15xV<))Bn&vioin~PP#zS8#B zvFm#PFk@t&A>cL9rRq>u{>hHyQ44WE{N2WvLyPjqZ^A?F0k1K!+vTJ^FwHrgCEf11 zBkh3pnfKz~y)ftg*T3}nSca(s<)}jW?P5&%(Lb6a`1c<5{3tPKt@zL-;Q$sKUyc#~ zu{YjLwjaj5>4ASz`mjU9^ANb{=HokvN1>W=C&p>4@j&^s4;*xLCT7GE1fPUQYA8rw z+~>l5gkY+kEr&iUur&0;{4PLxOUC5~IO+5?Sgs#12R^|8|GbGJMOum#n*hf| zKf;c=hCjoxy}#rBrO0x>$MJna85qvd3XL$XSA0qqTreP z)~2N+lOtpC3rSu4?Yvk?pCu*&y2=*6@x}&d9Vos0`S?Q~5ck&KZjnM7u*~KGO0KEb z+|fR5R!2ZHyKr7pWIL3btwa_eiE%SKmlR$7d7T!8O(o=sDPtK}l_44y+*0lSzwD5#pa)kC`+T?JN5UVso9W~C%qdP6H~e9kN_{QX zmC!Gta213l6|PhA8~e&9s;Xq9Ut+VaJAi)w-dSn@_!H*ils_$eW|D=Oo!nUe9sWDN z%DP&YqCot4C+FmbJF{Ngyny*j>7(0dK^ZQ2iCzVg7TE|0c+%xGKK%Sa9K^tZ0MQ;r zdMu^mzv5PgcEP`BghLR}VbY`O52#+_!Y&YzV<5W>JaraQgP+{R7W9FMgF_Wn`TFe| z@*Js*3`*l&+zLM+SW*BVYA`1C;rEG1q&+i7C=^lm@TR=p8)pZTWWM|pe25k-zYxT} zcMn&%fSQN0>cv#h3EI3IjYBbmJ6qN%<)`{yRgxoZ{(;%8;R7HGRMfWG5uFTJkxLRm z1*gZrhLeJAGlNev&HHGjn_)cQ2hq6Fvn^$>P6m7meN8AlcaJwM`$VUb`44{;^6NID z98Oe=#lUmgAM5ocB~Kz+U}^(zr1-UQ-o5i4p2qom#0k&(d@-pZN{xh>e+;oTz??Ow z*`vSHg842%SGD$FDl9SA_oMO`bF1xe9%d<*&Io?T$?2Gj=U4L!N&Zj?T{aG{wic2RCL)ytwq}|6?IMEa&}qkPb@tsS6fO2Cg}^q@reIQt(ww z!51!tmcRq41qpo^?>%5yWxPxN?ZPOyl%u25v@-Eu`lokYC0H-A4-RS!(Jgu$NDUAx zIuv9Y5av^DvR0_WEcV@+asRskI#e{jomkJpbHv)3PbGo}vfrHFH0hq(_Wes4IhTIm zv!L`BX2t%vpiOvrQNJD8{a-U{-?l{WNPrPdM^&kbup@9@tY&*k#o`@XkRO6kTZMIc zjnNuRP{IUV8t2!Rx|2xCc>23SU-6!&?vW-!Cp6E(cF4ZxtWM{8>sgnTr*A+Q%5=X; z6DBO5s*OXF=>uh&ST2aGr9Q2Ue0>}a#|gwyK~Zt5oP^SIj;0p$&sUu}$BxuhW=IO( zB=d1<&vH=RTYEg$k;&*X7>D!9{efR9{mO(yYr|N45a<>1o?V&IK+B%RkWNOMZKEsc z1~aYU)4_j!f5gs~D7hlDe%a?32V4$b%`I+18JPq6%{K-1Aw&@87I9mvE&UUYlg3sl zfIjqma~m0H_jK7U1cxBUhUu`N+?cnYCo$oU+T1K8w%%@Fb#g-mH2wmh1toNN`CRc$ zc7DH|mB0D~o|XT0`U7ltsVV+P#Pt8}vdmifH-ZWffP2+(!yn zbOUjkx@_)rQkYWLQhk;|gYtgmYeHBrkh7TL2xn2vx|4|OmCK@AFmZi>pA3SDpOl7v zaxW7G(Ri}%^w4vk%!RTXdX9`cf9m}Bg~F4_Yks=vsUjz;yY23sLtAEJee1`+&5QB4 z9e95h^Y)i(c|Og=eppM>yq0#4)~~pmQ2EMTt9tW<<~Uwd$TwLhnPW6$yU-O{BO`%wcF|< zF!vtu*1Mt4!j&&!;6mo}n#|lBOWZ$67$HW|)nB&Zhv`%&Uciew+T~t?JEbWIW$tII z7u}D{Qr=j}Gy%KbnCMDM%P*aYIl%D!7$7~~LUxI6SFnK9yHg@C<3CkhpA*S%#kH7L zuf+2}YF#miMT-NhzSv5K)K^35;Wu`XgUe9U4d_Ua0n6<5RtNTYn|sb4OK9q#=EfgY z_^G+6b8^eATPPA-lNkM`xmN@PWkGa~gLAtS6i3YpFv!4l zah5{nDp(ZsmjPS!>;>_@5RCyXBq&tk%+_)+hrY`Xx(M+rRa4WN{(P6b>uN;kCjJBN zYc31SeI7c@YyYq%MXfv0gfQ2icA?ks`@OCP|Ss8Qbt^nLg^A20B&27S)F zta7U~;6eOZO4~auQlB|TLOy64vfA|r4B>c<@74BRIR5J*6;!*y+;J*g7)q?gV;_)w zHV+o^#bf`L=dy9aE&Ac)r)DN0Jh5QvNfQ}yVmmf)s|^f zPf*pAk@dG@n!d$|#V+Oj{zGt~=Ca88-^h_1E-R+X+O%q|AMXXS0D5JeN>4E7)xtjc z@6Np#pp5bUtGqV{ZuVYGzWhCRHLRjVqo&F%?f&L5UeYcrF%+>es^gMoLWqJdIQ@$A z7N(_Awq)pfb}H}>9fp2_AjAbxg#}GbB%^j)=lRNl$k zBi7ld5*#QIT?k_GZSHJoQ2B<= z{3GS@hvGeHag1(YOW~h9vqpUpkad7Q!C#p*%t0mhJDL0U-{{0B2 za`Lc}nUa}bYNhsV`uJ*{L23Qk-8Lszn%zIAyuYt*S02sQ zPqXD~JJ8+&1Xbw{*6tmRT<8++pa6`fQtD8csrgnPl+MSSbKThT|5yD7Bt? z@X=_)4ZcdBBRUu~*7RFO<$Z^GOnyD2i-dv3NyO(yCh~{n?&?@W(~)$)x$cJ69c4YE z8KVot_pjWx@mb9Q=U3}G&!omx#>5_cwYs$&^XZ+G?&m~Qf4HrGqW`V`TgEkE&n@FC z-+b=0^lu!2v>gWCT9SIt+WU_SZX0<-Sqtmg5z{ak`gwmhOFh{uJxXLQzBMd+w@{D_ zbzyEOh<$9>jnp+~oPs~;qUiK{&0<+QsfPFDM=-um))CI;Gsd`At3Syp3BSO4{r-3@ zN|n|l|2&b3{Rdy-c89K_G;z1cGns{--aHP_D^{jYOlHR}HPU7@hTdC-vQZWX`AkNC z)MGPzGn$XPw8c>)NYl%eNd2n#3wxE}9>-k#IeJnE7agGpL!s|~caN0TdXz=xsY9$- zc+XIkky7FOqrM}LZZQT(;fWq0nQ=XI(C*@mbfJ=+tX8`);w9x|FN0I@uzCJw8mivMF<$355J)x&^Cp3vmx%1;VIK5C=8AlI{Rsgl3X>HN}iRQ6%M`}%A96ijc4tr9ayv* z6C0}4*OMi$V%#vE{2`mHdIGhBqo*0Kx7k?_ZFOdq7BcEYY$`W5^t5GXMHS%d&+ncz zQ9iiFY1-y?TMJ85!fGU~RWwmtlCzhII-LBFlWt!xP0biBoME-Q;SJ2+m;Vm5?0mTY z17Usr|4tn|7FFAqUeKEO?q5PTuz#7{b85r*nI#jomS2b+@f6w-7JOumD55j_m+4*X z6K_z5(PvkcF91aZ;w?)FE;qC-%oxlg`ly&IH|mifg>xv}n!vmFDCYOfN*Le@}l5GRoGCA@%hNiE>z*gJK+TWF(vevpqwJqStFylJ z?@r=W9Mew}h@8WXR>wsOjEQ3;M_ir5*)b%7C0P|scLJ!Dy`)khBY<$nV5%zNi&Ose z=i_P5Qfow-eqn(+SR9k@_daYzDyY=^h1es9Le<1h7_ztH;S)G5c87uDgbOd>T3yn8 z>B-9XRBZr+eA}ZhMEuWVehOU5s8%C-hu(y6-R1y10X()UTB(HgfkKcSXA!P{z1^~P zt1yRqgdDN>y(!rk>{J4#vu5)9NETFgt$hjt;qFwwV1rF=4 zdzAOIPrr`^F3I)3US4h)C;v!KX`3W>G*QcZo4||Wg=0tnTv}dQl^-RR`CX)!@1WQ~ zlDn?IIhpNWxlqk&eP1~w6P>YaF8C~$d7Qx)SSmSv0N_|!;!M}L_oSv=ZHGG~%MuCM zXJm5-y=W5xN>x!D!vqmRoYwEf?kWW7)CbL2OKV5z4Rz1cpK|N_mIb^V&2DL=L{3r! z3Wa2kb=8?#w`4oQ=w&H}j_yt&Pmlxa42i{2?!M5&;WU9m)dQOZjYxm`i0BU8pVPK2 z*#}Gyg+fqKezcMS=eHiB(pe$GdZ|2JtvvQxD zd>kuDZlsSeD=!S|b3F<3JNuiKkox|9a80cLN&(QeivqlBt^c{G(B7_Qo;tn0(LbB| zk2f(A5NZ57BC|)QRnGej%bP*QfHY>J(vhfvRJz*C8WMIJ*ILGUH zO01$YS%Wlm*Is9iKYW=tPxgg6#4sl`@kjBLDa|bGDWwNhB0MUqV>zIUn+uG)f%8hq zg2mw!2=t*DkI=1CLKC-wRO0sx>kSU3D{}5E=LsjxD%g2ARj1!$u(!Ii&@48q6B%Et znt>T>C-<5PYcqnQY{nqKcJAsv>%jf0oR00iliCXWz?3TE;- zx$3bvkZ=wF6~V{jc?Xw;tbGBPuzr~G=4f3-l4Ko|#lgbk%a1B?bv8Vflx>Dih2<@I z^5YuuO+{g)j#OB?l}ti7XAQgT9`%^Q<&} z(D3HpEVJWGiK}f+kaP!*Oze06k0Lu?YE4AOex1-wv{8p=c5?c)*%(%QXp4J*cQ;*Z^6}Y+X z6|QYjoJ8-*rQLiY{dQ&)2spjaNOOrtZ^D$2n*r6JH(s=%Rah~7-#m-cvf_K~;pymb ze(detnHoPN02t1~BO!mj;g?b6rhNEw@mN243qy|L`29$xoJXNiAo@Iv)9`@Ra8;=^ zxwKUFbE-bzTOUR#{*iX?^eMQf`wMwtF0~UrgICO;`xMxup)eoeVY3`#o)?qa>MS)y zj^xvXCxvY17S<%xy0e*56T1Bk=xzJ#8-w%Bw1j6?TucL>kJ56NPpdb0DO$r?FIU^t zb^GagM|8L)H^0)x)ZfS8CBACDxnB9#A*$yNd57W$F8EI^rO0X$Z3*xmqVB(5mfDly2)g`vtpblAA)emyc5f$86QP|Xo1A)xl;!1 zIFh;t9J~XDnCK7oX8JrSY9-!Fev^MMI{ZD}CAVnWRQt!fdvQK4or56sf=lUeygu66 z3O*0`#pIiHj)NNbEW5JTdlf8l@+D<#l7HD!y6QjAe}`F4=BoO(xMOPm>t%m%zrn${ z`8|Ee91Wl!<+xSS*!LQ-8W&=su=0?gjRzn9QGi7nzHyVw5cYz}_P^*hhv_O*J3%)n ziGpz~x$ge;0#4u0_^8UDEPcLJx!GtLN#5XMVX%tcwQ$gu7bm^$48Cb8YkV{dV=~z{ z-)K^B!y2JWmYh-&5CZ(C?re9Q@rQeOKZ6FvP<~r(<1n?uJ&tY(slph!XPvm0>*A%- z<3wRM+J;LJto?Vm1x+6K?zf2A^^*U{_#?VICSHd7LbSMBCT-5OJ1`XbrYAGZge77K zFAj(RC1s&7x>2T2?m+%FFeW>cvn}5)4mL6zz97)q-L#M%bHGq51g|AC!LbXE?1T<+ zW^}4In?Xe50v93Lv6ltwIUO2{xfu69P5!bUr9^7{6^R^)o&N-OyPCsE&OpymN~>cF zg2j_fnie7pY`uH>{OM(w3MoaQ#acBPt4*EkmsCL_dn~anq6rtqF>}Ezxr8;iA3_~h zAr3;mC)MM5KuS#N)U+Bp4br-*vj~#E99Zy?SBd#Du-bjski&@YU3p@S&8b$gJp zOpHd)Kb%mEgr`mXK)8Xl6Rz>hh|8iVr0_rF3LX-|8&ha+GD$LX*jIi-@ zZe)0AKt+hr)p=mb)~+vUE}PGU^Pt8z%isf)l9IxJSSVpteV7C{>#iq+D9si2bi(aG z?rF>v!DUMRQb0c$I_8tgFET;Aaj@9{j76bRKZUov_=`$3(LSq+Y8F*LTsssT&+hg% zc!BQp)?lj0&XZ7XlWG0>pXBaGNHNoSn_)a&raRx6yIBZd)q`XBGmxv^!Dv%(nE{3x zRiRkxAKNUZAge*S3;|G>-sSuG+57p%m1EhPx=R`C5w%mTiUTZkWO+LAgN?rPsT(Se zm9dNBUD{<9HcUGrTfc*=S9zTXd#+daW50Xr1oF-bc8-hsl1z4igIWU=s5aC=3$-J@ zWW!9O9qo?0t@T_^2KsZ2{q44{=NVwT%wCmy9K>}+7ab6*80>$3kYYhYqy_%e4>{o-VD_G-AWe8b~UgoQ}E9ex7>nhXl9CnXD$;z(iD z_ffC8$3;zcNfUTu8e$I33$ql}Ktxd7-%0P{#weIzr2yeC{(3197jtGuK1&5X&qv{h zv~7A-_$B&6tDvXh0v)G|4CgKh**O}D8kdlp$l{hleFUtt5YBAEFH=DmxebVKn z6L&xx3q?V{%ujZ@KnNBWsYwvJd1qN+J?RJjb+ zbAcn!##wZ5iapq1m{~tfSL>Q)&>fa=Mr-04kLl*?zkXrc^wX}{)-d)Gn3eA%L!3eY zR8t9ZL)tg14L@egqvfl)SF@*=gd%29)*vzlb&vatO0$21lya#fAT=;+aQQ$Mq-KF| z`>-iZKS#c$mMMpx8Z% z9moyqKC$*~r8s*^w5GEtc4=@r>QhTop7%q}P5I%p9+h^+nmNuRAW8tv&m$N@+NU+o zmr#Hk&7WqP@VDdH?2{R*02+ybHq}IprFTm3t~A?2YKvL+1unvD!TjqGeh_(> zrixeBdxJh@GTv_mw32)fO|aN8?l){VQ?y4Qf>?o6+DV7- zAg$Ij$D&$A)KXO48k$LvrEC&3{uQ@ICVc^_Qs{i}ErJDyN-5Pxal)pe_bJJ|rvh|+ zvtONJ0~K$tA@pu=nUXp5x0O&I2&5GMNLj(4zJpe$j1G5Fz4%|E2kZ3d`~{F~5agLR z>{LXpLKb2GDZS3OYM`w+hjiZJ8t8=oeU-!oDZr^PPZy43J}u=2M|w`Un25%#@T|gX zfHR*UrC_}IBxfm-0QO~%;j!eVo-gXIQ{slsfy_1ZX^CR&&@?4djLDOvP2C7$swXto z^rVQvgwQkRIM8OmKyC8^c?z;eGcETOKko)9MZj0W0+}MX4*M)9LyU9JC`3c!}4_5mF)5HT-fzH!5WK!`fV*b&4U%tik z#$dXW{_j9hnTchCrtl%g!m|IIWAgy<0T!#Eu(33VZLO2>X z{vl7GZ&)I&smMU47`#xaiO^YYy=YKGZoDE>Q^@2F$hAbZEG)0=X_@2m&lSuNcHrP* zIE?ckw}iY|gCMtpOu}hv9lxy3*cG}DF%1Ruc)I|3e3asZ2)9npSZ3<*Sjmmwff}oy zNSi~MTQ+!xC=#Q~-vTjB!M}up!4haT6z+pI4F&K+3@Dmt;~_f2rqnnE339}%#BtIu*H`*If|gwR$j-k4Lh@{L*BNk@b!_RzO1mXp}o}lNd@{EsD3jlj#6*Gi&iO%pL(Eo*M(R?>eNg;fL(_uy{`2kzg7biZ?3PG)`*=Af<4G6GZ^@O;HNdshUR%26u4D(( zN=7LIIX+yyi?|=M!L_DF_F%Z+3E7z*RQ;D5BW>R6*9aN}$O77=4PSyd!%21nLd66j zauR%MDB{clJ&v$b{w!rT_$L%+RbdJF{PRs!^S$#h+r5U7QfsdJB~PFikZp=WLY2f{ z4NAnhL!)X{3GU|1U00$aV%(0C>&!-;s>9|JaKl<_CaHDEB#Iv!y#<{K^D0%-K1(Xg z6SxReJob^R4je6To_|h-?A=`$@X3zMSff3ccqEeSy&LfT~$%|J&1aKWO zy6>)rzQ~iIyfBEF)I?kWLS+Xhbo0zuhx6skTB=4&%+|ImNayC0HnLgP*40g=M0PKN z`}6AJ_@nnt4}1sP(f_%W0eGmfMpy+!4i-IJR zGE}^WppdX3bSZ!BaXGgzqoD4hGSY8zoE}XmRca0W46&-l5{jyYCE&Zk77;+^j}d}K zoorYS;?<8!rh?_EDSk~>->Sn$IYnq$Di^3juJktl$(tk2DP21JZ_*R+6_P>%(Fly? zBJ>~j4%ThQ+VmYJ^Dp9NFxF?hD%bpXbT)k9Cms?6=dQY#t}qn#exrx`0Qp{dGYMm{ z*o6Zp;bymv2JSTjscUI8*9Qz3)*kii^!zW^s+Ibe`fKJHAiU!U$zSGfBkB^C{TVmpyHz-c5RL!wJjw06Q=Q)U|K4@%!SpN~5)*~L>FR34y zzez&x4|F!v-Y?c#;Ql=?RC}`EL`s>f2u_lKy{%_F4w9M5L}{K-7RJi1C9@{(092M| zvJCJH-eoyYhwB|0kZP_?e{R*a66d%+x`HOEcyIED2$gGNXIH$7JjiPmMpm1xS3K}h zC`@2nVHM=1i=|bD5`Io*+es_t_tp$KA{B&q9gQ)4&ubH!ipIZ`hO}&Ny{Bk3p9bz> zJzbR9X7>9HfR?WRzxF5+m+kvoU)EmV6bht^2ABa%QP%eLOgbxohhC$vNYhmU(%a{9 zfJ#BXIyz*ETe#0V>x1v1D>vvra~P9d!N?ZjZ!a{HAI5vg-9NdTXvbd?Kq=PY zi+QBaP^8xzJSuyiC@JQmjsXhX0<=H`Y4W6yU|Ji zq{&24Wq}yUE_|35+u9z|K7S)RQL(!FFP^7iEDW@X^o4xI3u!osk@S7dLupSE zlH?rQD56hvtXZ^FHVJzYhejK?l%Z=EwtOhcAY8u^R~hCKNF;{0)DQtqW4D7p7?B^2 zWK}h#tu7|1Ochnd;MmqTCM#xhUKg5)Y^VV~eu-%R>20dR2zDMiGF5kDOd+pNIqB*z za}64Z0hd&Pal56NeR12rvIQ*)ZQ2ZbP6>tx^HIN}jcD3(rz}TU7&1T2YPht_E|aI> z94q4UQ;s<`z$L+8Xv>$pCHXWO*!|QS8}C8UyOJ@Om%p9A7)b zA6g4ilnFyIO^a%D${xjl%3`V4II)JI!GH*IU3`Vn!}4$Irx8o_3QWXhkGcz%TwkIOZAzvni?`2r4f8~* zZ$@LRn4#FsQV?6J%1wz-Eb}(llafNtwhUP2hLC&dFWwbzDWVJIyLk4m-DoRvvqvV~ zYWb&P#q}0v1lhP*PR6pd(@3def!PMm-UUqWPkYc-3WH5BV>mI{3p{0r*g*O5_wkCf zP8SP-%vSF&BryNb7{HjP6FHP$_k=fr7}#ISf%Ga;L19a)(j%ORDEj^@U&PAz*-IuA zuBV5S`PO-|e#_LCLQU@MK$X4c1pXKnPm+3Mv;;kaM6K_*RV_aP_gg_$y5bWZ^>^v8 znj?9gf@pi06mHvcPJ>qQ`Rq@<7dV^7Y;b;Wzk1gkuTn#F zPD2y!N#VF?z+hd)dr@GOIv$MNpVHkd)|zRP5j@@gn-4T*d`ykHXE!v#PXI&oOY++P zYSsD+s2OQH)NYN^JM}=bwl;KTBk5zq_tGqd#4-J)Xvy@nl)j+ejBwB#5i6_`P@5EC z<;U2IBtEq8SRiONm^Q=DEXXb@@i(c2VZwT~rA+jBpdW8_V(4Z)x4{jC0R?sPQ5=nTL3<$F<2PM{A;nvr{dU@fb)p%PmA^Dq@P$5l7ceUgfjw#1}Mw^*oYj?0mY@^&F^XU^ry>2|OA)?j? zLoAX1FGjMp&Nr)-beTzm3EN}(oH5OqV(SHCmx5rC^vRX8NU`^!fm-QS1HmNg9Hl0-3dF3p>8ao^&U+Vr>%YG}0qP%r#%-xjY z`!}iMBR0@-k!UkyRTNhWp|wIBOEaaq9>X(#6=+rP5460c0v(upA~cz;k2FWjua79Y zi>Qi5S;=x#CrARVw`$~yi>WFxzGw4Py4m=2p^3H?XVRv0NE z)<0?QEp*P zKqfp!c?rksds8Vgq^FgJuluCtlo~*8d=*ae^9!F{ARLbG`)n+7GTP10KCQeBHPmy5 z5_uz$Qtw(mXg?oBJUD!1-qsnox;n#T{cLBO&rRnBL}uH@Z@EVsVKM~_c)TxEPLC@; z))eVKoGCw{cvWbIJKds8|}xmId2K=Ygy!%VBDR@qS6^kPeito@+r*$zEOwKL?aH zU%ghSc`@yX;+gO%>h<|U&e?apuvTMEK(w^@x^z=;@uH`aO@#y&dk(`NfAz??AkMh6 zu4orYDLUzTs2K;nkf9%96bm?qvjVp~jG}_D$0U5EG}KU7^o=4EKYzPMF1zx^zIztT zYL(G!A5ZG#cfZPLpN0hz47#)BYNgHrF;`W+v<2RcQ{P)>l~9D=>;}Q|D4j|w!3g-u zI>BZ4rL4Y!ZLCSR&!l7NbW%3-Pbe$;4ok6?pib>C)4Q~pCL)YAx({g0lf+|Sq; zLU2xsy^UykX`4^V?!3qf{MSBGbYP%PcPW2TeSI%TGkS(&OpI%&El@)m$dX!m9wK!% zTQS25=qCQ({xzbK@?X>M@fM&L5V=LS9O&HS&&vkf;5UnFrGz7pxpQ0o3h(^Qt5__F z$plp~T;HH~x%k`mqH|hb!@=z2^Mr!1{4lL!0Z@j48P~p)=h(~TxA=bwi;zf}oujm~ z2S^w#^m>*eEsQ@i4@^5kYW*9m-~rvxAu;fdJz~ZdW(U{n9zCtwRx2LO?RAF`tHk{K z`~<1B!-|?TQV>!3);`in4hUq>A?5&*EtR3-#U~p%o!OB%Kp*c#j@4uo;y*M@kS_W* z?T>`j*(uUgkw$NCDj5IJ^wKuRtYbXSw@=KAm8p$>B^>EAHo%v?r*RCKu~S@BCK);+%&d z2!!Xhbg-TPq=Q3@^DL0@j*;PtMm_W(-CnMpHyVH97moG5+dRhhhadEJi8t63P_~mD z$_5V#3!VN*Ca?P-K&pIsLD-7CF*Yk^0^dWW)U!sun7FS;+82o8y$G_}rYK|F*XUYp z74z0%_?*t`Cv0e5jM!%s5?`GZ0nxx|FY(5O)hIPjvSbr49iIGBL&qH5^MtdWzs%m@ zbNg|cEIUbHEX^@_+E3?{@>?Kh9Ei#UO8n&he|7$~P4a{x_pdtk<&S`4%wANx?#ADI z58E5gA13;nx~&Ez+F=I*2ax9)ARcN#)kHXbvo?q)u|ma6d=Pu5hK!QXMgf27Rbhpj zK+0PL6KbEahLxqzF&F+#iv4$(Hz2ZjJ$}F8p+sa4OoTps?b}P`);U#iyZ&ZfLqx02wmD9>YZz!aEbqP>@FQ7r zZ*WStE|u@=bHn{5rfaAKe|GJ0n{`HXw~YB6ThZOBZ_bK2rE~X*1d9z+Kp-;34c4?` zmt0B+ai*o$Xn}=r$*GJZDV%{@7XgU^{RQ!Wh*n;Y#T=>0w!%$%Pief)c0hZtN zAeePsW3kT6y)M#;JFiw})t*;=IDFBDiW%pDlP%&1Q{FQ?hk6CEfMbHwa2zRXiZ!Ki zeh$Fw2sl!_Q>EA=*Ut^eA6b~@#%5i12nYKNTxXs>mr6YnvK6cHEOw=Fc1bE0k-S!) zYY-c>^n(}}r5W=iZfz$8;7)2IegBO+>jl}#Pc47mIc!~MIyRE<7{ZuJx=!v^yoe85 zcsOZY**2JN!i(PiXu>-bt};ZbC`l&p)NUIV@b%#LNqF#_(JZ782=@&NV?`X40gs>j zb}8Fdd7U!?%mPqdg}E>pJIgF`nhPFF>35$qDzn zBRW&g?~PxBB8E_*4r5BjJ$wpp!yoQ~It%O)>PAW>cO<;J*@OjGym8#|0!N=%5>q=_QJg$-<0p3i|@;5x*8wSO!12G zuHy!ms6&S+iL1bbAXq;k8^ zQ*0I{-umhIb!85j-W%)Pjadt~If;~Y$V80#jWzxBE5z4hhB=%+ADH>QeEM_jJt*Fw zs@rs?eD^2YP_c6kR(TO|XRE|{zh@q0Yyy(1@(rVsu#4YzUUsw5^&KU5z*99`87h;+ zv%beMl3dT?WI|=m6 zPU7DLpB>+J1HPldOkU@+P*I_i0j#{vzqi;s>txMvN7!#s6}6G!pTb#yqyDy^kivqZ!bmF3r-7 z8DEuAwkk9;G#tS#MYj5=@9g8J<=!ow(u%hZCcd zFCX+)X%#;O(~Y7lI0|F(ey)D7iPDS8>yfKZkSWFqpi#c5coY0FLCTkbGtIrS{iU1) z2aUD2*1VInGJjq@sgPL9_POp3zgbrsd7m; z8MiKtO!H+jI3I5jd~*_Zi4t>Ub#DJ>F>qb)A1H=aSxHty_63^DIDh>+u30Qr2rPeR zo)CnHUVYeKynVo~O@K%e*UB}@dLwE-KSSJg;mYWUZK;!y^kNU+b=a=X_u`#vm=+m#yA90v;$WV7lH{iRm85T{3B#C``1g$Vs@Z6~9 zqrk;~r9kY4a1%&84%|~_A8){=xP1wQZI7P>18<)hifybGnq*5v|jZ z)D^sEU9BK&l0Rv_HQ$3))!k=eY~nsAvmBB*nlQ6KjGF~k4Q3x} ziI%Cja=TZs^&~a?kcB+#nVx8s@{hN@Mb3VAd_8hM}3B8dm4?h*}5 zJ?22=N5WRIF~HyQs@zW{7`tl^979!V0D6M6iR&2km>?QN1xj2RFQtICn9D;qHp5Er z^d`8{3|Ot_F|7wnFKyD4EU#*7=^%4yb>Hdq|}ntxtq0NovZnOVTzu-@G@o(AW1%r)Bc zE8g6Q{tGc%#T=gXj5QAW32QGmd+vX8UjAI_8XibGn`Q<=$XAs5;l>I7pQf%n9_sb` z+d@<*p(v%>wG%>iDOXX}ikJ`~>&PVAEL2qXB4gjOCQI3}&Dbf+*vCHP+A}k@nK5H# zejnZL{e9=J&mYh8`aI`+&Uxl}&Uv5r@jj9q34eA7++-~Kq2E_AoY#@`xYrw*di@kz z^iv)IV$1f%89JBnmdMs)BadWT)yX6$7j0F;j|P|LOdP_p<)EUTU~s4+3*TLV&>xe* z`{c31YQ5!~<5?*vLCm2lipC}g!Ztfmn8Famu(vno0ik1nq@r$UUksBnx;Bngb<%p)x}h*Yx7nmesR%bzrJJ;IdRHLO%eFCuD?Dl zyR8B@T5q3xfVi1*2ON5-FHId$shj*U5d4dohF0x;b!969g#h?AC%%l$yv}|Xsc(Bk z|Ej!iePpG^nuh?sU&{mb!&z;N~mx{7vf6Ph%2%-NhGGFNM ze;$MlV}A~60gv5Vow>puboJ8Rh`GJ0m9%raRmED1PkPwg3bE#2%zMpiqSI(7?Tw}I z34+PR7~AuvGXmUJ2I(il>Xh7HIpvs0v+{?&v^|k8#5hUOsAwS-74vc^YesMOackK)^Nf}DARf`mCz%@C(W>`d+`-^G0 zlfA@Qa&5n{wJROt`C!XlgcpeKnB5d}zdg2MTkY-y?eyNqE2(#W=yU!P|NdOh*_?>C zXWjT%1LxwGlbLcJVjuBg z0ZT%I4A;ei6w%IlaE3E}jrvDb^tx=-or?PBwWh0n)Wp7?3yK+M5K+6}F9(@`&j2w? z)(Ak%($pjrHVB|7q5rt|2zeS}W{gzfVyX;#4OZ&$;UpT*9Gw9k0e{a2?gA`TXIF0l zJeBtX+GxY9JkkQgHD;Gso^hKW%59Dklvouv!&tt!=DYrP)@aWq_v@9`#^JA@SFvEqvtOm=>5yWl2|{CFfh_$pTzAw^(CQ|U&${J4(fBnO-$Tzldl%kf<<#I*otnWDRq0yUHN+XVQiskgjy8^fqttWAJ zZkrRh=VFS_VnI~lRZP!VvzqLxs_fp-6nLK~(b@^Y=|jamIYBI< zc&_N)7eRMamfDjQF)XSqG7Gg_St-?)Wp5t3vb8JMW|Bc3)kDZ@Jdk5|e0;Sx=qLl_`-L;jeMWJ#PS4Y{No8K5Ym;NPB z<}t_I{+HF}9eA_g!)k}Ni9=wveNRRqa}Rafb6* zC>h$QZiLwrL))q7ABrytb&K*FJW_nu4yjUrPhQ}BRFn-W)MM#t`phuBtX!DM+Bu=_ zTrT1E7r91U7UbdXn&Y`sIr}Z0Y&h5Cb+`Qbz^&D-ku;~F^}j+- zw`>=2kH6{&n6>)G!iMb|zw+Fn>!vkPqWaZ$14p?)t{Y-qyzk64%wZq_bkOX}x}L;U z2u_rN0;tpdTz*V{yulj6(YY4>_}UzyUDWun`xQ1%mbv+r^=ov3+2c)frpIBBD~f_w zzBrjK=FP4#-0IdI0E6NiPHaWTx@ljk2>z(ea$y0XK-KXW`*K(py8QQfo?{+7@mEiu z^gYxXc*tQ#nz!9_>imlk;-Np-Y@{wr>FG;DSYDfB?{uGV7I`Vg2JO1Sm)7KDz{CIO zQ8bPkDl$}>2Y=%0cxNZUV{ty2U`NBsV##5q;@wpu?%GQeVkt)W@t;bY9w_lBja=V% zwt3^140-%5)eG@lYMS}4n%J~mj;7kQ-IC`m{PW2lJ1gT_2~Kdg&PGz-p@Agn6U%bq z5C)3F!|`->;kin;`m7h)B<}1Sq1e53)2OZ9WA$H=62@nQD!5h#^&niS(UHrdS){D! zrqxF!pB|iWRw{QCJXhR#^zq+>=N?g!QtKx>#I#jdd|6)$YOD>9IDGX9|JaUQFunaSB6Ibi zBwOPk=QP7y_3Ipw-+qh(vx?0Ry)IG@^|ecVI8qiIh#WYQv70~a@mJ0MrOVOC1@2ct)Og|x)r|e3J(s9EK+#slTAdQy~cX^ z^aR?+fTQ@L<%8yn)i?h%GS`FQzX*)HxGej!T*Uj6U<>1m%U?SJUM#0M@CnO}g|}{x zZvo|f4aS7&fDSt0=Aj5Hn3S>LRDQ3>359T;m24vGgB|)-%G{u9@f}vs6@7E7uOGkM zkcrw6QGG0Rg<}`bC8Up3cz@G7o`<8`yTwMhvp}o z6e}1{;a^B6{qS=cntq>)mhGChzA|$1DyO8+&%5HT8Mh|bkfBkVO4ud#UJJn#E16A! z5vk4l!Tj+ZS97$s-$y>00jaFso;zaLt@Bv{({VZ`s}j_^7vkj5D62iDHx`b53~mzK z7NchgE>u_xs&gvtVADaj+R9suY=E0RUABagjn{YwWIT0QQ zw&ZFOTguOHqCn;m>j%bk^T3!?hyjcH#nnq3-3D*wZ>-$;GgP68Cu!`$)BGWG1#=#A zZf;8lxckJ`WRGm3wSzsXamSLrzhy#@h1f?oYh1lWG3iM>rO4yK`Ysi(B>nv7O~|ZO zXTK1?qC=*LjfR!AYdfJSXhZo_k#S7WrA$Lon^O%vy|?Tl zv2`cEm_Ka96JiOo4Tj*X3Fl%Zw1rCB1Tq!Qry}K(>YE;idVf%U`AtyUq z?&BKLmdo(L8EGjt=(b!;v5cZx@0|-CF-njmlfaHcbVXoW6XD9<#RFTRvdiC}-dkE+ z4vK}>an1#sMZ2YJHHj?qIoQ`g!M=U2?Vu+94GWJWJtH|MPlyB^y5F3pq;!@6Bs+U5 zl>*|9nAb~HG}uzsluSP&$-@3HpFyn1viIkE*!0lq3%M~j1!kW>?kO;LwnB!{ z%&i7z`Cb*4FQ62WSz-F22hGV7N@|3z&eZ1$j(=Gb!na;Yd@Qgn&ZxYGf8gj1-9Ze2 zoQsN`FrDx~2^6{zk#T4J#@p1B-U#(free9AD4iITqO7`9(*Nyy@yjpE;1%{-F^8sm zV^Nz z@+X^uc38p#z1mqq?dGa%JFGghtB5=J(DTS)ET2p6d{ijydjQ$jtM*dChNHO==U=7{DeN-jf*E*`6#O-@D#kN04=P?z4aG zFQ#J`_R#pf%`nBsDb{~)t5^TFXb6-gJ_uoOx>#4=q>#bl-%FQO43p1p59ygkvQ)Y_ z-Ri8Fo)g1XXUhzc-+rDYT@N?tL2|j3CLQuGuGq>e^8F}pm>iR@ILVSLh0M!gy)@L- z**SaqOvpfBZ=dH2BG;$1a-@G-o4^6B=oHNdKzk-9PlWkv=;ZMMXQt;^N|uQJTvSSR?93Dmh8`gHD&8 zRWo;m=fmM$m6+>;)>muvZCYlnf9xrC?JA;n%9PdFp6W{Oclh#O4?aA1{zf4*`%-pt zkc=)SmL<8qbr%nbvo>#SV(F>+8Zl{j6j@foo4xSpLQ7rt>561A9B+Mo+O24UBsX5A z+u=|4&71R71D&loU{{t1lW|1u9jdQSm9%`+tU##b2ZOkzq@K$X0Z<(TFQC3EFP9|nr@};;uF2MNW?aF(mOB9qY6{M1x4>&zDH)c(W#;f z1-f`wHWj*Aef)I)?9Tv$Ax*=LOaN1pdTyak1n11|naK5nW*0I_zb{!<8vZ)t+4W7f zP}WOJ;kJ)mcRX#Y_*hs^Ei7s7dMr{vS2?i&Poh_Dr$zJLUf|i$?!KmVNiPdb$E>RL z$o&hhrRm;QW*2XyR6|`;>V+FKmx#Fw1tfTc(a}9oMLNs&wDGb-( zCufJBOhVIsnQ!sa&xqDAd86bK&qv@Q??>K#piK~MX*OAXn-T0tqj+zKc4^PQ59qj?{n{BSbRbxyBP=5RrQb$Ig# zcjl-CKm%#qSa0;u%J{{xcjmRBx>QbTQHqi#ejR3^6SHD8ZFGKcw{7{3J-2hzSUhRuUCK#W&c3grn#ml^An2Ca*V+QsvzHm99yf?BmZDu(+K4SdRvSv z`+08{Lq-{wgcFr|42dY?F-@gErOkpg2Y31^2Ndq|@>^Ne*BI;P2z^&=ie0hS8~&8n zmE6YIqk0_a{8ZCj#f{(V4J1?&cDcqY)Kq?`lXooJjyyj`FE2LkNxH9?(&OT%Z9lW# z6qI?&`*sHJ*&@WXd3Ia#E2Z%~qV#O&DV>n_%1`h-PrRG#Qxh~*)-pucLmx8S!uej3 z4QUx>aqD&ddEF_Z_>=PvzK|8_?S!rA@c%=V+W*+H~DsM27X_YaG~BK4|U zPP z=L}V#M8NaCsB4*3S!&%?j#AO~5@#Q5R45{2XNo!|Rr?AzXCcR0XT3PAWU& zO$@v@&1{$Rbqe>h0);q318##BY|F8R=qgA6(_<0E4KH7RfCh=8S8-a9np4qPUC}Sruj`=Y= z$mC82N+S~-D;|eh449`nVv@}O>gP0PRNrXK(w|Gjc?MeE%AOYdU%A{N64yM!;((pf z_aD55s@;#Dacvkr>qK-$#5l0EsOuxM7;bfLr1JtC_!D7EaDh-n9%z{MnVYC*i_)PJ zdh&<$&a^Bt7V9a{zTP+^^9Dqnraky4{RT%?2(qCSOu!X_G`15Kb61(kN60U1mcM`Bh zw1B^+`1&VHfl5?|y7N?j!~JIuwHKnL9pr>fX!CJE#hzm5}8Z@Ana(*%WTUJld*HSRf&r_RRV81}Q zLG5dz8C_y-`c1d9e?Vu&ZW2;pjWT%jvn3tV-{8`1 zB`)<2F^~|fUWV$oJ(r%G2JQf52|%6zN_nsK<1D7(3pqi9soqX&_<$a}hJeI6o5IRV z3D@nAnEgAk!=S}03#Yy9l|J<62Q^8cYGXS+Vzif&eoiqK-<`@*z(v-LAk%&XKvpz5 zp;li%6H7h-yxSM?==0PvCaOU$6Tzs$Bvux}%Rv0)!lukyb&ossk`g50`C^o5i^ZSP zSr}Yd+TL4e#Il3t2B9bDvn%z{?VVw;2B`&QuTEi5Z4j`}Y60!q+YOu~Y4VYu?;!y6HmaQ zz%}dO7ARJ=riLpqYC&@w2L2f}6-fk~I8(GY99 z<(<1T>rqVd`~5yu5YB;GYrS<>YjOnjEXW&TIy%y52-h@m&pFrqxbe8nP@SXv1)dnujUtBYY4?sYGBovoh#Mu{FzZy~*{?k-pD_ z7#w-#N&~4BE6cP;V@(5YtKWnDq+;j=^5{Nvr~iH#0`_UCO?B6nUPOP&G-0Z1)9zrM ze0o=V5p+hQKOA;cdwX%~$6TOa>@xix`8K}JH)H_1S4Nvvo!MWc8fj%Boue_;S^i>^ zYWCRrE^^uWByj>fv%Ri8rUrEb`TN*sk%LRCpLuUUYK7|Xd(q=1FPt;g$uBS!W7XxE z>LvR)CfV!-;_botRmLlUs4+j>iv4w!9{UCN1i~?$aJ^}!I3tc|wwHUqvZEh}1Rr!M zs5;neOx~0V=t}Q*5akcc$ge8wGPHV|PzhZEPR4+kisiCoWvS44WH5Z}%> zy&$Bi(qjJUxP1`xnUHWkxLMV#0L1WpNy;H+6oq+o0uV^19sNDX?vr;Dk^GC*oAuYh zI;!j+aFQz;_4^^JbVH0`OSyRfPr29v-(L`CwmCwvKor2R7fv0{(!E3}1h9-lk9#7- zPnxn0f0`)wI(RGadM7u;aYi-30jDQ|XR-qed1S-kFZ&9iwyy%Dz#Q_Xaf_rEh>w63 z$JAT}3S%TA!1F*j0e4mfrnoJI-x0WIJ@G7R;K)ZJW=l` zcG8+D<(MAyFG)?(O`cbx7?sUq^x0Au;Rkcl4;O;xCt0EsGu3&->ctjr((cldXP z^8ShW*a)BhlvX1$8PZmML{DK_n!7K_>9=sR7E|Z;!E00hFPh1_Z=!@F(7j4(&BV~7 z-|N+KS`SBPfn)|P0QmYr{O6ah|4#i;AMjG|RM`q%{H3}b&+os<5WAyZ7GmVAHlRy` z{Jzfr2$$se;prZTQaOKdKa&V+V_s>TIR3A#Ot~^@$C#*oZauaAng5+5LEHAdN**UN z!9-Z)#eXHN?JfF?S%J!MBZ%(&&*DGR;S?XBNxNBj8uJ513;4JfTYl>#+=sx&gnmt& zv1_v%r2hXL^hh4iY=?23BHVVybHYoh_MhXWSI)$-`Exy=h)f!ty?y3Ot>EAdqH9#~Z-3$KGv&!TUUmg9Gryg6eEeT+0X9S_Hk)|P-FWdcE1U(9{vjFbsO)8xCd5wQbvmpd} z8;m@I)ci&8ob<x8L1#GE48x{OI-&JfSE={1BBPBEtDGm2HWKNAm?Psu^y z+lVUR#wC?`*;g3wdbncXjAR>AqEV&GQ>)LZWqpL*rUEQzz-wP?NZBXhfjNVs+1tCxYWe$<@5X9Nbk^rXder>`-f0L%!7 zx2)@m=^`pt=(_WO05HnFD;lZ^1^On+WqLE|%#92^lQ9di4%p#-0imX_ft*|cP`gt6>E}rWLR`YaNDg-?t_vQJ$m8c%wofUD9mD)BG|is1IA4Ut(YLj6 z3%j!0Ub+qHW1#!MKg zuD{;-QFo0_C~433JHpyO!y_vU`dzKc=x19ZjCtI?C4|C_zqd)3SX3c5b_@<*e9PCOV!q;o+$0V_ZPD?CjCteG@oM zj8D4DB;@Z?d6~@;Sz6o?M_$@r3xPiep1caTKfwAwB(^w({Cz0z$du5YW;9XmBk78y z$`$2XPae4Abn}k%{!AlzU;Q4MqvKSOFd0S)EPK7)8ShNso(y1-Zqr(}qa6{o~Av4$pS6e+8?t0Kc1trq|019H0Lm`KS;^ literal 0 HcmV?d00001 diff --git a/models/tacotron2/NeMo/docs/source/asr/images/quartz_vertical.png b/models/tacotron2/NeMo/docs/source/asr/images/quartz_vertical.png new file mode 100644 index 0000000000000000000000000000000000000000..4cbede907736f631cb9774b207289a35dd07c757 GIT binary patch literal 262776 zcmdS>c|4T+|38jja-!9=m!&}zl2k%sm{QitQju(xt+K^f$6Bf>r9ouh%1E*lS+i7% zv1_vL%P_{y%-CmqpJS8Rm`}cRQ+c}4B*K>L7_s4SeSo4DN<_%mMAPCy5s&Ymf zg4hrc#LmBt4g86+sqR@@XiwnuiR2`N!ha`BM<|9=mbj`daY&`a3FmCoN6__v{&7aJB!x_~+%*{EV?C z38AU>qxRaJ$pPEv+1iHfis+BIF%ED2sNv$!zm z=0f~h2+L$WdwQLSQJZ6>eodQWm2nRQ&DzICylRW=AsI+ZDoplDZEqV!QFbdl+d6wC zqRggpda%29m|Rv$CkiQW+VwmZ8eD{p&)+C5ZJw}ma*y20YXN@Dq}Zj~1|TXAOS9VS znhEPRzt0_^*mh>*F$7gbI_h@XfeYyGJ$YR4zW=r}4bpm~h5az7)=k@}No`~BL0;dc zjx_DZSlRCT+u7RiasFS%o-FUXdPL(J?DL|T>jDoRp|Lv-Xb0j+ficRy+h#WkLeP)2 z1GK$NmKadH{kNP}3(rj-omJ<9_+;lUb02$D;eQy@prx395J5!Dbvgu{GTKIyhWC*I zifF=%rnTbyiw9w)Ft3k0A>yuA9=lwFKI=x6%bY7FF_G;es{DRT}j;akpdaq3j0v**G z#9Q%JCy4EUTTOSbC@Y@Gt4kDhu1(uX}OypR+V8FgJjF*pw zi(WDU!GG)I&vo$9ca?J z2i=U?)S1+!scW6sdKlk%HNFXXqvSH&V1xxiT3#YwR|@7Eu7QGT!nV(kX6%KG5MxiO zUrtxQhr_p>R?!j|3QxApa@1&0^t$yAbieF+M&9M%uSyTib2HZzOSB{y_r-mwheObZ za}0am1+yKqSCc|`+ZY>=xn~T!k2wY*BN>W!ZqsdF2mipG6(UxIJ)hGV!k~3)=3mkvpw&sc^k6iaR)e}1`5*6>7j+#& zYsi|}!jL^blawXSj^d4z1K8tPk<(8j>zva)V}O}yA!7kTy#Z-Q4>w*^irwwnhR#Y22<78iy; zDk^;a=-3?4P%IB9Mw@!nHLB|=&~ZDUNu_cQ*K6@@Cz0)^k*R~whF4W!j}s~^5MR;E z$<476N*4wraJ)TLp7cvY*LnHxUqYnI4-VlX^cdP3AlK_J_6D*@ZweBQbXui(gDNBa znL)}XD1}qzy)9DVtNucVZ8F9KXdMkdmzWlk?m4F}0uev2lOB{}3EC?xrk~3{VGKg$ zH|b(Ock7UXl(r2UAp&zX*7H#|a3B_lVS)SpNy-h%9tA%U4=Lg)kfNRTObAmz-6(y=FEFS1KU>5=-a33lGtjfx+{mp+ohi+dc+@SfC{2N zoGFG{vB6V)ZG{_x%|zt{85Ix|8)>>Inw=j=YGOgkQJ@+pyfP3v992aHR~TT+bqvnS ze!s9iQE+v8NYnAJO;zn3F$N{#md{K-`N_3y{q=Iaz`R+sCHu8dS8oeh;&1nL@HHq* zOC6ng!VT|uRGzUDzHQMAf;18K^SaHVs3B44??cwx%ZhS#n5G20l_l~&L-wt*C2?2o z7saXa^xgjF?y$xVRq@&gUGtQn^nVdndO7WiK_5VCwnR{a%!F z(XISL>lHcuy;H{hXT(_n3`jirr(|xPt@6twy)(Gy^@tlfzj_DAJ4pgIZiR8bzvcAc z(Z+LLB5S=r#E<(+p4k`DLHT6Q$G=L{BLT1PDpQ?MdF7v1Ao!D<@pJsTu92b&5TDG? z1sN57!`wNUhcwOq6u*CvT(Qp=)|)=7_x-9hZ}+(E#x5l1U%cS5$CZlEY}cq%tTH29 zrrZym6>Tuf5^MDz>_u)fpZF{><$dhv9|S^>;f)nthE!F*$3;)PuUVA$K6d2K_qbC8(lrmfPgl%Mh~uPg=Y$6e)~vk`96+ zAFgisZ*RTa4S(LQ=>l=CsvZPgwt#mvbIh*ezh7sP8piZcJzJ!KcI$EHd|@5fvdVJ& zy#=!a&8-y*EIa)Ew4X$FzVbZ2#rSdPL$a>qO**<-9poFYBE=po@axVUX~;KUEN_-JkVEv8QMg1o`h5k*`C| z3(I@I{EO|siAPWNtDjt};uN<8ksTsN{&BSTV7HVG61%F)NFkm+C132t{4o~1^q*(f zYoi2Mv2w+%$dFYx@JsjO%XYp!Z1*foj?g?d86)elJGu_vXuBAUYS~`o2=}yh=q)G; zb2jQi+!BbnxR^_;>~MZCP4OVlNvz?eNb5!o(&vbcxjmg}S)PaZwU*6DbFa2TqIxTC z0jf1yr5gI_Y*hD|J=8nWw9o9(ThxT?Y!?#bZ~o^n4|Z=O@!Bvb)|2IN=y|I>~7 zn+JSREJ{}_6w|$^GRqerdN@ATzmeEF*n7ZwUa=QpUPlUSG77=hbIqkG+AkE7{*l-5 z-7_mwUy4stPo45_%pv&1`z=KQ@7q_plo)NWobE9AZ;X_-tI+4eu4uVkKo3$}F8gruZf`AlDnq3jO$rr`g#y2`Pe-$A!lujuP4l1H6iFPMzGk98UTY z6iRz{%(6fQCaV_Pz7KV8_>`c*1PgIR6$KBXzw_zp2p0GXs8ZJyPvvnIxPP~eJFD_y zMG`j3xd9fg<2*1v_Eo!!ji{@K8*JtHwTJfM#FwYk*jwGM`pxBUYaK!povzN!XB;wK z_kG#qV!2$NvHP>G%CgpavQnY>30{GPcz0PZ1rH8(69bcNg>WHzvKndlnv{-+KC!HI zM^{W1%KK?}dFL7SK?atw}mUsi9GON1TUZ$0lvT}1oad|M6(ki+9uuS)fmW7V{=`(|x-_vaC3cW#c#!3E8= zF-fEn1cq9zw!U1BCCJ=EcZwA&?nb7si0hZ%dF*?Ti_xESF(oZ~J1uuVpl$$H3T)R;%Ey+QOnpN6RF3>ryF@*V~%8=W-vVF@yrTmo5 zt^Ho9eN#G+efhs^RI1>9;guDBZo@{JE0Y3X#Aka8<_#Gri*H57)*ih4BPB~LL{9kK z5zuv2c@_xh&{(CzfA=w6yib)`15|r{IiC3^4(MJnHIz!M5q)1^qf|6_U*90$!m=2U z6JP95rErC+Y6#AZz?u>i5)issfo9Cyyl?&ihoDz{7bA$4Vmm~AG-6BV+>8E^Q`eA$u z2{xEu;wDOVXOK>eB9Qe zHYH-u-L44SufvL($Lm$TJ#;CKFFPy{pWRnlH)Jn*u$A=vm6v?$P?pO3m!17;`#Kdy z9(SUPIA&kZqAwji{YY$?04P>{)d2ZycjW{W_%3WmO0NW-z%#zEf~H5l27Up=EbY0{ zCL^q`yA9nJn}|=l>{*pnb4umXGC=zcLm@lxVh(0$`??41W4Tuxma4#mV&F|T1#{Rt zK4@z>NT$dM&Xu+r+3uRnC6el5WcHoKEVK|CL*rfezdrp74dVrJqf&d1$ZMAF(<13USij)jxK@S`=FQ({DjZ)qemdIBt&od(^ll&X|QzWwVxp9SkgP;6jFKK>h;{!Ghi-RzsQ7s7~z*~qfrUgMT| zGq#-$*$yr4v1I1=E7b%C%43k04itoQ%TNuvJYe6P`?@-5G+1;~V^MBHV-Y#7kP-J_ zbh>e>p3T^yh@KY#YqMRHmR^jRx5iu&0!nzpOl&d>sXSzAQ zr8|{B{RPM|W3MV*7;b&V+mk|{?$m?LgDSva08PtS+>FjF1r!LGVS?Z!bf@C`r!6j~ z#@6GLg)OxiHbzcGbOmnqi2Hk(cHQc>eqx`cZ*~@*t2b3Ly`Q;nKG>JL7B;b~Xqx-? z+Dvi`$b4d_iLP*MG*6EQY*q?AUJLg8=8^fi*@Ldg`+!4u{bMHlY3fi{IVV5az^C`b z^!yr#FNP>pXn$GO`}luVePXlgC~{7&or>}_TZV!0V(m_wtXr68_v4?f?C6VcA*O1L zUR?_zJrj%C_DjIL?h%SF<#)RDc|kY*3(yxJ%C-VsNs zdt#H7vcXhG;B7(W3#-PAk}wm~RMXNHWdGXqYFD(yiJD=ab;M@FK$@SNh50m_MPa~N zBPk9N&ae85l9a2SX@B1-i}wNh{Wi&{8@(aTBBf;2Y5950di`eV0u`*TpY<3m8>=@6 zh#I$^jBFgz>+Acbq%-7d5)mS))KK`|5&?-30p(q#0jhToFDd8FxQqy$Ju zT6@8(r;r_1rloFE!|t{kC&tF?oLpT+gWA#ZF0B>n4a1H`FWmQ3i}DtlRK4XTyJiP8 zQr!0?88GTHr0@UrvI>R}VrMPq6IBD6r3E^(U+6NQ+LkM3Pv zK)=2MTUmskJokZ^R?2a5@eYa2g81ArYA7Vd9^UeS;Cf?dgSbXgzOWwIEMcn}Obxak zr<|z(z}MJ!d{$uvmPXeWh`qI>XIFtalIwA#3nY z1#lR-;w?xn0ih|6L|&p;n{yWJ(;u~1e_r;v#WQJCTL#u6Gj9G6J0{M3M1wn0!aNz3 zs-IWx>OM576g{pjE+>pwe$s^zmpZ7E(IL6A9Zn^OTI(<~4@qNt%!;ds>vzD<2kq-`Cl z-EX8wxL$i8(fh#6=7lU)ef7XLhYadXVrK_RYQ&>47hPsq9k4xHuko^XJUgT0qI8+H;q8cBdK`=S z?u-(@YS#>1S**3*;JFaySt=c|)L7~srspB1P+cl&M&5SZ;rK{DI}}dBNqu;vZ^Qzf zREZwq*L8~=@_->vg2Hf>By2j`b4a(r2gz(-fFDEd(rRlAyvqKmb;IO@5&K!k+y?R) zpWi04d)g4opFJVfS8(iPDlB-g$AaJooniUS#B!Lk-#-w9EL^y)-MMwS;yk|8t*LBB z<rsRcnboiK zqhUMXK`W1$D57IdBM=tZiJ3({-u&Yala$s*%k_G|qK=vm#LVATbQ5EDt8NrZbv|&&cy#B_6E2G&I-D|y z!Ol^=@Fvcyy9@B)B9=lRvP_zQe(2Hu>oNB&+yN1wKO>i)GxW0P%o;q_(I%eS)sYpY z&CO#eTRT7W-luvkN=F83lNz#j_3D>J5tk06c>HKy92ARlBs@uu1DoQp%>Jx}?94}W z$L^@JV^JoEL!jsRQmH{wuhVKMIvK+ttm8AD$_*g&?ih2!>ntYR%2LcvYb)%?wXRP` zcb_6JKU)vVUYgQ*N%c=|&!~|xFMn2xCR>CTC9MIKN7&2M8UJ66Kn$qk|7-+)*$GER z(=`BZ@T^qk42^|nk3Wi1Z+8t-l-R2>JjLu)*z~v zZAwp1vGkPh=7b|tqXv{Vz;aty^~jwIUQ`+^RT4Hl#VtKxFiBM_8fR*AEVWjV%tWJZ zqPpl|x(4sbX=|5?S*m1Kt6^EoVCs8toePHKYU8;z>EmxzJ(NLDOSrwO#6@beciJss;eB0HC9(?S6Gw*`V~ajB(+} zw@+6GEkd#z)N*2!+pl#g(LlMa+Mj$(1r@TEShdcKx z_eh{(PD@uz6>FY&XV&Lj%Pv=o+p&LAqYtR5;XNtNm_?BEo%N$B{FY+h*r8a-=j|$F zEegm+^nn#cxikH%@`d5~TDjwbf)xuCoJaL^b!L&0$^pB^qqYcCqomhkR(~($+~&)R z2U_~u2?yODlxjU^i^M#9;qrhTxB6gqz_+<|^M42AMJ!I;rM$v8t!>az7!9ugs1|w? zH3aJkPq|9eXtRMFc90`=9Piu52`@dTM>@CnYMb!Sm?De-X-(5c*ke6(V&u_Y+$S!5 z?8m7!kbkv^P0lxQXh&q*E6rV~$A^X?Xq~G!@AUhs)7Are6uwj~7p8Y}iCa>Ko7q$s zp`;mq$vu!ak;6g2!qjV`kR1{h%r(#mhPRdc1M%f#Ue=kzB!scwu~&bqd}+y6x*rgH zz$5U4pg3tVl#(iSo}vp{L4p2X6sybPAr@%IQ=Z#zz9>Fmw%9@bzcKUk`~J_Zot=3d zu6KUiP=gf>FRNoL6`x0KqS*4DTr~B{nZ4rW$Ogr7x~5_bXcx#V(Awk0etKy9$TP#HZ#`{gkO9gtqQ{ObO54=t z-ugyRT5$^3?-K;JsKJ1+RkhJ7IztDdrr7E~KGj|O7pB)VdV#eEvLu&~G zyy4s@Oi`%s`8)6ab-}<7RR6Z3iIh)Qhb>Zzt}^>Y!T~ueE!HGeU)$M}S||Zy*ny9y zwCO&&VeD24t}cF*@!EZAFojiL*nSWp7$Qf%eeHZ|so`7Tz@Z?uww1ng{VN;m{lW-B z-i?8ElxrjJS{vPdQ(t9UHLUYug>v=Li-%xRA(Gr+EmDN1r(fKrGO&18ZsvL@_GDd3 z%Ke3lg}V~RUWK*P?xQ0|nAl4?%{`jQ>19h$&Q{0X&F(;PVgMxcm^;NnhhpnF6$SlEMIpcDuZ!{_ur-rBbxP`JR{uX&(iTG^#j@k2|( z;}ZUGR67B;%3nWCH1QSn%2Z3?r>o#X%zaGR0ws>-e%lH0m0ry)SA0;^w-M@l-Py81 z6(lU-uHH^x1l|s49kJDLcZ3|0N z8gx{Wyl#ej-kI{E+r6UrA6&8Zr%TH7Y8qGiak0;Y^hCNSaVOj;VfO7w2s+K^c^8S9 zNY;n&?^nCz!s5)WUV2fyj&oE+Qb@_GmbUHzLBJSSiWutLklaqou>|WK+4BN2jxe?z zfrz0zreXV)THe*NB3;WmuLa)YxkZJwzvT_xD|g*Tcgbfk{YGdU6|YO?pDiD0{qUvu zy!C**mm@pmO<;vKoZe+6p{46}XM8PG&ks99YS6jG%;LJ**o!R-m7rDa|d4uSF(}%DRR>jf) z#F9pBA|0Sw!Sq_0YQKXE#}4rgHs}mmt~n>YX+a!hqKk$K!m^ad!ON?RKs*9Bw;Edz zryi!D$=xoNa^PI=FE5ww`td>VTsU+QB$l4Ysa}lEMZhRc^zqZ=>l<;(p6Xcj{({>< z@KG)>wgJVy6GlL>EiZw2);0b!a0qs!sDG9uvXm-t#Zlh@Ar;|DDQOEb?-Ck&Q)yrc zwBa$}Z<6R}gU)G>^WLdm0~_eqhW#U?Pr#%A7+!hYgb|baXq7_wTOV~?Tynn{(^#N- z$u4$C^GFaJ;P~h5XWCoT64Tb#(qPsN@|hGepB*5AT<>JRo7w&(c^2|MXaZm3 zeQ${^x1B6Sw@&+0ZYOOvOy+&G*R$07mmYzI9|1QP3!^?06dT+jWbhk_V?h3zXfX73 z-bpboK*#z{mM7oTlTsZmCM%b~b^-iiwBts6wAk?_bLH26MzCGZf0oE`G%_< zp9mvBy5~oHPjw*|W1j-1&M%qM24GH?lw0)m-=wY2u^%6!D!$V&&L{(+`wlZIuHz>^ zFD00CdeL=Bi*vmjKldmWGRILN;Z00^;mFMlHXsKC&4qh2K5wV7zAGwnMBv!@?|%r( ze*y2zwNM6rIgNP`w84j|*aB4`(lFnylB7e!O>T{vXi%4pD6=FVoh;M+0Q%LQ%s`7| zf7)zs1&UbDK&U?C+$-7T>4u#j(YP0J^t5(=msjnRXWa{vp7ZnTRk!IynEFjN9CWS< z%v5hU0u=~mXU#o{h|kX2HaN6qP8t!PhI*}GtjeUt`al}4G*i8E)2MZyC18~=QGA0s zWSW#kCKeu5giY78DQvRITURC=1doph$gI@=(U!RxnT;#R4AwN>U90dx)41kvSoTwb z&z93I=Zw^`Zz87Irr%9Ug3*|>m5RuOId&{6qowKQ)<6$W&yPQ(Z#bOTEobSaz#n$D z;e>MvYpvKVTkHKzL{YkrB~8+bx&Q*u{VfFZ$*btf7Dj581#K$DlAgu}yyA8Enqx&iOIGs;pq!I8n;HoW<;xXFgjzUp9G@IM985Ec zmN@1${Sdj!rTI#YI(O6qKdZ@fm6{5l>Kg*;`74DmB`G((q{%^Zi?9$ezeBdfwHC9M z;wMC(Cc9f(BGMW&$}652EfvBtO7JbxSuj%7NOhi)lMAP2vp_RBIoq^2+K`qn8R_RY zY-9SP)!fLQt!mh#jI(7vT{5B|zsJhc;)MPh*ZPW$fzCe7bEB4@m+N4d^z(f+w#U;E z{?Y}*{`tErtEZ>#Qy6fIkdW*(%_$_G?1KCq@7tH<}gN$jVVJRVsTWy`~FhT8wn;=5o2qola@6IjH>7> z!JZwJy6orPo9b9SFk3OKn6ALX&Q2V^!y~R^TX`OTY2O#?A1b9qmCEC^IG#7 zTmv$-2`IsYtrm+5x6D(gi+@x&}yYMZUONkh#+#^Ydat z+GXbyeU+bvRl)0NI;_zS%-#DS@LgB8**z#FOX%$94y1RVII*l|p-br9htOMfI0SyM zH^xig1H-l37sjW)oH{IO7{4Cv%<=wtZvb73*rpSHGU&GbBHBZsuRkr7%i*vqdUiav z%}Qko)7$0E1Ao@bX0j3@vC?jr(D-+=<6TA;MRqSp9^+48e4@-VO1NlU2h?Dv*WHqK z6+HqF*>Mm$|- z%}wQx*;zgIHs7wqEIDa92#f&VpIz*->t`GMz$rT7W1qp_%=^rT)!yaAGw!F$2tpXq zQF)5Y|6ZZD*ieGygW3kHlg&!hNrY@$KoTx}^ncQ3=GjI5&VR|G)9 zwVp4<3Ai?4#10d~mrU)RVuor+lg;EIyg~XxXO5-e!vA?bQ&g^my7tff7aJ}>w=&%< zML}2`ungp05){aSAn!=-aaKMZ>v2VoUWZv!#cISCMm)Wa>X+)h4bzili4XXelJXA@ z^D<4<<`*mNpSiBPdAst33GMt3&t9pu56xC_lp0LZ%5#|Kgs1wQ+91gtNNqjfTHlrF zQfc_jKY56(J_=(wU(YsHD}BB*I{(FT{>cPW7#XlSQi_i2iWRXG_Kr`_$qWwdVxL13 zA1>)Hj0sgq!U*q#+lm>~^FLfOLY7TxuTlUa_}D;$rt4ZlqZ(OyDfzZT2Lc5LPdPk*>ZW#3zU{ZqRQuuP%_eVF=V|BS$T zGf65(x$}7YJFBBh&=0elNjR!$p-Jx}iN1Vk!E}JW9sc#oF#{0^9~*O7kFDTFK(HBD zOLtxrK5?_0YeOujjdi`qzbt)v{<+MdRcED%PP|Srd3ZinvaKcQ;#1nPOIl`)(hSR% zvZmu&EBKO$F(J;1xdxr>3!knm5fi^`3_GIU-+H-J#aPa8pksB9_4UhQWEsQjO%T89 zn?p$gP}`pmE50_@CcQ9oCk}HXS@}0R^{xK%+;VX4J%ZxV1_3o#pyP$YgqqnN>7m)m z5N@xjH8SI|b3>h+&7{bAe>%nCq(TngjOEFp!}Ifj@$EG@uVnYbt!^rX%R2wp9aMyv zDZt%*OVpe0&45(4i6HX~JvGt(hyJf6Rp%R+5C*xm1bv2lD*b{e4&!UE|Hgb|6fnp# zP(EHUiNHTOLjasv?-;voi7$GkhaloE9ql!Vk#;CLB1_+xZW}X*I_%lTaLwTOxzR

r#Kg%K(*;;0~-i)^uk9+_H& z|1wFS@|zq$j&|c7@1CKp;FOcMHbSn3UPW@mwyDBoML^G{CEKVoVnqs?`~U?0d{|6l zFJ@=mt7ghyrQsG1I+8Z^u0UvLl~;I~%tq)ht+5W&ze@kMD0H&y=en65X&+t_H*9o8 z%o_!&f&2yWbHr8*J+?;1^Ag|$_A&^(P#fl9r$FZmUWv4S z-g{I#J=uU5@jOg7=|eSlC{FGLF#uWZ2R!Frz^NI{iP*Nm-^k}cbxFN6ocwI z^?6#!u+MIy_wzo*4j~0|x3n|mFgktGn!?Eo_))PGq5d+;mLpvWKR{m1?t5g}h$`3c z-HH!=bnqB)*X21k+FRL|H;g!M7>xjFFDU?o{LUExg7L2>_ADlcT#}OV(^#T()Ue{7 z67KsS!s!KCH3RR?7PS;FoB$?jwiIvwvSYD9u>Af>&HJldqxx&VY`)B`%{Rbn^3hP| zQv5kBUZKtg<5w%>uC@a@iMa&=G*?Vx7szinf1guY1s>Vg}*Fo zv8px{IwmvB-aVZdIu>k!!eeoMvqS#8KNQ4q?6-uLe89h%Ztc=7ewl9k+D~`c_q!Rt z=(``C9|Z3Y77v1d`@bzP-XSQ0NyfQV^$@zn!l#g0q@lK@)jE~X%Lg9ghXdOZs zL7*z_`+`R-RoI*<4w=_eT+qHhWXgxL>1f$-a#_=a&WG`3XA(LsWyht{?mzmtQzzV1 z4;}jLY`Af+vl5%a)XV{xlhEvcT&}L_mtDq-Y$t**PyBNk3>5E%A`STI)S}l>_fGGx zASf~r)S6%_MK#WZF<*gg?oeN`hBpOl-YS1Y36;3kUp_}w{{WA|!5LtyG7Iooo;kO8 z;dj9T=y*B*8f&-nfo5d>0QTge7!6N9yO@if^vSU#uctw>S<`_Fm}PHL^tU(B?rP8} zFkN*f2SRo2o60v(q%^mQa8=fbcZN(Si3-2;^rSS3D7?;>zEvrD%nE$LM(jKvJrO5= zq0oAl;K3?ip1{tf(0*Y=8ai`K>b#j4!qc+@Ujp6eS2vzbM^UlLCw4#O9cjxLC`)|- zZ|rc!pz4k;5*HGjZsF9iPtth$+=rT8MajZ*voGOFllCfBi%nm!_gu)>9!Xd5Kz1#D zOmiPuM|z5?cCaa!zF<+6D#pwOkXpcz868`;>8m0L;ydHTJt8oxY**id8+*1n((ca= zo#UQ$p`g^_uDjU=n$o8_>R5pyN>%NVZNUOqMuLRd_sVA>aOWzw zL*tZZxxI60FW_|PXRrnNN~4UN9k@~=8TP>US)zL%UVs@^y!UBmXANep_q?{sLbOVy zsLV?^o9BY?{LY*Q$uIIDpso&$yd!=7^%TPnRn5{#o>G8mk_var@dlV=NrB1kHg!T{welR zyi2ITqPgxH+V+)@{1%K5Y+Cr_bC1`pr0V-v@MLkm*oz?kNs~#-n>vCHNmR`MGF$YtpxV48iP55u0>uEx#+7>G3;U700yYV z=>oWNJL&<`?ShidJQEr~!Y56BT%v)_5w5tX%_U8)bE%bdi#!oE=9N9+HoOzAzEVQh zJY4_)u%@Nh?+9@vK49ys@mjkjJJbP42>3ufk~TS0oO6_h31?zVs1Fo;xU?3{8{cH^ zS`y4U{*h%vJoZF&$xedcW$IXN-iRI^uaX+pnl%9=ZywZw-Gz}U1*I48B2Ew0UOL(e zhe3}-zkcoZpj1GeBqOvixPv4SvuMyOIXfjTzOdV(p#;}JP!yN#d8vkdE7V>$^Iju& zQQi($x?PmyHsFGd6zggFT#ZoYVo0aqfSkm;`7w94oCO%?$<^qZlspRSkq*ir4&ulKHu`y^SBJC%Q=rmiddmL#YW=s9f2`Ix~$? z6|HHx>31pckwP)v3CtK-dYiPH6jSeFdx>n9G!>nw!g=i0=&P_GxB|UiqtSMdUGMWED zX`C_-9GwqV|Kb(U?q=;XG1`SDq)qO9ixR^b>U zkl7|mvi+}?fV$m9W;if$>?-%;jj^*&q|QumD%j>9o+Wf+PFi3Ko1Pf-#T6aU^^|*O z>J&@OF`M%+nz;Ug*A9mX+LH8W25XF{oV6ZtHez&#UW=Q+j!svB()CyFD9W7I7*D+6 zUVB2C2CABy;zn{84{-&ZWLLw2X$y;Jb1_78bAHlH#+vAk27p{oUVby_Va0?@(3epAcT*muI@IE<&C^3*tohX#>a=VZST6mD!^^gc7 zfDcs2`|kd3?!?HG&Pxg11zaLGu z8?lyV^Bg7DR6RR{5XA7#N|I*d97`y29ZsIKhz#zrfP0GK2%-$!USE_pm$uk9j0dhk zQ^%$mNoEPsNVBzG8};v;iKEWUx(cx3N=Gty?G6_?x_51Wm4$FDrt7*4;4lWcaYhPm zb>9MD$BMklhCh+=%;pNLL*UbL2!Y8{{q{5ry3@=nKMB|t+%(mg^QDa)E(Ff*QRo@` zpgaUSqn1bw&AolS*D?27@W`qDTWZ*O3k_H>TnlzgVc4=b`|F0C@T&m7fO7vdgQBW% z9`s8lsn7keA*NdBr$yE|m4Tvi)Gx*E{H-8bSl zts1SDDX57)ByFQFIWzc7lRQydN%m_jlI@w7!}AEgHUdr3Ww^Ie2ExlYPOV|4AW5gm z0fO2#ZAa$=kmRGI9j$B96!L6NffHay8f_gNTh>}+8v;e$dKr!PsjQ!`9$7RzfGZ8# z9nw~((~wK$YbF6)T3?OpUTw$ z7QuDVhg#ta1=WgVelS;7BRx1M%K8AKhpseyFFa03^CF+MB->TO3Q0TRS%h93YzhsN z&>i!*!DvGQHi^pB4K*&3Uucr!6h684f3W~a#D0*Lk`gPy1Vf;PxLlgLXd%9gda-lG z6|WD4Hw()F5806TgnMju?t7wBNUQzuAYNlQ6%Ou{YVZ+ohUTK1QLBO8ehQsvU<1lBuigxTx0V9Wb=rl0dc7u^$ zGGt%>$OxTWoBB^cWBigi1HZo?$3D4~bbuz&Qpv^sLR8UGR-XhK!P)NvRlZL=p(vk} zmOI%TmJ+-dP3Av~L3_~~bS8jNPID-nnU@%?(fdF!VOI2w8H0Cj@n`1*e(9YErf@49 z=qpWXG^Unw&`dN^4Z47^Qc~pSANXK4VuQdPQ^z&Ov}CA{ilc^vI-h|D3P1BqoB- zMW&c~8gyJuelPjR2|%cXDte3V3vGaflQ*+a=E^d_rUTP}pQ^(RFU6zw~Q$sL#Bg9Qg;=D&QdrhcQW z!Idg_h%Xc#cXhg^H%L${(k8Zb8n@91y#u^FtuBiLmZ^hR$VVsciug^}@!m8NN7S)a z7QB^f8@q^xaDuQSHO2i`%XhE;J?^fepJ6a9Xh5Ez)|Qu);+gmJ2hP2(;qqIf&cR@K zpe=kps$@R8HNE6xo}x#x0qwgDP+gKMzPPeVTQR!zsK>+)5cY#IJp1I1knBjOBv8E5 zsRQr!k4S*giMeJCZUxe3;~Do`X;lBjd%fNoUM+61vpy}Xm3_WU+#59MU@4l}|8E7k zM~4wq3pBt|j}mao-($%901%hvlt+;4PGfU#%KVPz_IOMyr> z?K~I;KS(c0N%kC6_++5wAE)cgM1VoBfkTsA_j_DlNzIiVmfQ?ij7bsel2KT8B!88biB*(u{zfBKzrNFiC`ajuXh;)QMrd8JbTYtwDU@Lx{VlSI{|*6|&5J90@HBpRMiL`6*d(~QFssaG;A)f1_;X>EG|G<%OLEx^J$jn1VraJyx% zqOH)5%DoeTz)ZpT!!AAS&N3!KjlI>AEM_L_#k!#VfQYZUtbeu3lG4zUDJPg}}FXMVD$^ zkbke`0mxiUqUYs=yo)C8I@%2i~}Hm zM)8|g1AmW$ADF#i6^_V9oG?g1EY+`mx_4&hVX1!h2>2sof{p(E0QSfj#M}+>QXVgZ zE=dsu^JL?oCxqVbeKl+w*3V`cOxQ6(D1bu*c;`0pJ^Dh8acs?wujh z9d1Jc(;p1cJm|08evN&t+72p}m@s4XWic?13_iL5We4vQ_4K16NI`9f0p0s}#mZJO zgwDvG$<*;$uMR<(v2^t{IE-oe)9y~^#|9JpeWi((sSbx{ig6tP`Y9Wuf-{Eb2q1eS zM@swFPTTy&vb7q(+w=tkYgwN;^*;u^)FUZpmc)+(TqC6mKHH(qzT8(s&G9O;DjFZ~ zAWtlaMnt{eZo9$#f`o*QoF$mMV)tO6I^5vKG74E#7yG_7V;)Rc|7!eZCeM4{wmBaQv-d_U<#}`-NW=>2`7wfM5b~be?$D%|au2sLDlA!B8 zpzD67ucg(-!3q69d0Gw2ZqcyV>OmVaMir$o$4-%9#1SwclXCfg1$@2ZiK&o*cV=}s@2Xa^Gs zTNt^gQhc_14kx>(7`dDYhHuMFsn-jMY;+>W`NBXZ4%3j6*YtLu+Q|MOM7aM+zR-?@1qU)uq1 z{SH)Es^^L;>0gR<&??Nl&Mh>TG0gE;ZrC7o=1Tgl^K>U&!@~%{Nj{1y>;g!q%SCO0 z*Kl2sq&@rtTYg+Z-NBhu7qxWj%K`uZ(3*HnQ#WAEL39n?n?*4Byws9E69PxSaCv+f z@ggbS#Xr|q*F{hmp<`*LoIx;YkC4sZl(+|nfdl9qi0m#%OZDl~Z(YDs#th0CgRWSLVM$?_X z*#@0S(lZ6O|G`nvpH*^XUmrkCL2%#SZ@A+7e|16bmRHP=ykd4!|FgRlER>yfGM-Uo z@RF+eJ1u_~)-Sb)*i5=F^;MUu+F(S7*&s$%Z~nUn7fYD*TzOL~S{*Am|FIik?n;1`~>T&h$Ri1gJ)xN#f@N#!KxP zHSED)_>-(;)CC0nIEfL4ot;0UhD|IG#9>-2wLv#k?Jj5`FKOhNO`jQE+;DGZ5B{F2 z#5-+tsL#!|Eox*Ihf`Ytj`qDHsph=IVVrWX8g^*W;Ry=-m2T*7${sN) z^Gwc*0Hr?9ER|y)@i+0h7qcD#y*K3}x7c(*k9Lb6_h}J?_u<%c!dy%hh@~c~k>gS) z{RxWublsR{0X~ue_xfom|Brhur=yzaOE~Ht+3q~0?zq$`W-C>bS*8p6X&;aF+@pV% zmV=muwSS(QU&B$4uz%0t45A4pJabz`#a7-()DUoxkC53hNRZ-#wh+WqHv&MN?soH}l^4$Oa^v`Xi; zvA%qK$+XE@4+zEi<%(uR84JwRV=cFDiS&Iq<%Q^DvxV9{qXHs`4+TQZ&Y{Gr?f-{} zi@`YiR!(OB5vSa&jui@#w-l4R1ov>#x%OXomiPi&?hpLO);_*q;1Qmax*}+>`}f|| zcQ5uHBpErJ`mfy~2Q1SM{Rf|8%WQlv?Zf|UZG+Ydiu`@}AS^hnqF=gQRB2?l2%={? zsBAiTn=`K}Ic@|Y!W=xwD`iIS|4;+X?>5$F!Fb3rL&CtQs*<6(m`r8pvOaAZlU&*- z>GS_%9~H3A(wxVVS_*^7JzDeY!_2@tIKDI7U z0?@W~2dX})k*{;Sv{MuYE}4I0+m4ahJuALY6CIRERY3=1$_MOpwV7WntQ-CR@k*cW z+AswIW=2-1<5b6f0ORO~Jg@J?VTyjUxlOCy?mrlX+`2NzvP_@}IbaHK-)?*AE9&@k zU=aLvnkwkx39s$vj}`fUP{?vNKQs8+^R!hxQN!w_O=B>Ow56rmu zxPc=#SM0io-oyG%`u{vL)2#!xG_~^WQXS~zy-rgGvpT|v@=NIZQ;!((6IidjZ&z*F zBx#-&^`l}PVky*Q&s*v@)AnQ<$5S75UOdpLtJdZLTyxemQ_W04Blc#J*G**S@%nyo z;IP=P+5W57uXPWCqnz@&m&R>;LCu4dOlNkPI~->l zUL8>oNhsdqUGVRYo1%qo%V3Nw4cKVP0i1DYX`o1%P+;`cQ{&~x^gXx2xx$(@O4xj1 zcL6UBlPg`B#PrK2K{0Bvvh?&yKnx%Vcxr^X8;jUY2L!=eI+)`1=LEs)O-t9_xdd|S z^G5Ce=Jk))MLf&&V)XTc9~8oKm%jyCH@f}Rl&{wOSvBm;-_m4`<=px?FQxV0M}RTM z^~>VaeyFdndR_6B!1DMM)9wCOaDW_if1Boddgia-FycWzifWZz8VZYbDZgVWdYGg` zl`I3JS^w|78$;tTG`}CxqgR#5*9Fmkx}Te4>e%NAqZ}Mz=}v3D2|Q3vVN`($f6bNc zm3OK!J$fwljw1Pbr-je5Han^69X9;|0P){c7BdscPQeuqOGw8Id1>oceYi>nj{Qw z;s{MQU-RuE;P!eo*#38iwa|k9j>)vpp>B#yW5yTY0N4YT3VB=6U~mhY*b5Qw3G%|y zzeY37eeBva&yl}dH31XCTxo?YxefM%SbhH?;SB37S2gTJP0`jX+(UMa=l(z5-aD?w z_x}Sw30ajDQBEjQB#HK^2x&+}NfatAZKY1zC?X9_X-`^|mc}X3UfM%T^R&mQw12Pr zoKxb>+vofF{C>a3{do9C=f1A{8n5d$p3mn+nhV6(B1pr(O~p~}R=`yz2BmiG?{$mt za-dpOJ;`ou_*5<62J*R6Vp;184 z5-t3H(rAv*%Fv@E^fc@A{$3(l^d;+OfUsHpV<}8ZIaBdu4D|ErNg2F#KYnqsG4Kw* z5W$`9=*tH%hj*CxBtmwbkXkOxrTu4XWWMHX4A1xV#?1RZi-Vlb!zBplUq03pGSp@d zL98&^Rg_DjZO>=%9SHuyi3EH5_YDi?*Yg>gaZ7`AD!)}OJCQfm)>gxK$Co-049DAE z+SeBzu!o5kqZ;fga^^SC^2YP5_zZQ^hVJkU#ONymF~Q)`{>}2J0}cKp13RxnQ)csP z4mj_>AoQ^1UaO!ARw!{`SGPBt;K@G!?TTCMzOh#l13~B)vKU+cb@;%TOMRkaa?}5s zSh@pa<72f1?)9b4H5B6t6HmD51#bK;UCL-(0}pUhx9pVMLLK^<4dY+;p_X5(SFi7C z_oIVCvX<%yvN0D62rI;`Z2$W7@LNbZMJw%WU_CQH7c%kG9X+xbPJ|N5C;XAqxES1O zU51R2tz$sXMHsb~-To#BH(5Y++o5}USKn3n#J9nAC~uVlN9LE&eHl!so6DXleZxDE zs*|PMJ!qHwN{-2*fZrg!Gj$hx{6xXdHCq{yli0acoBnNH!S{iK?O6q7{-?+w=4ov) zV=;z9z%tiSD|wMfdq5cc<=eRp$cHUH;E!Z%vPP)|+YIyZMu{lIxAOMi-Kk z+-yA8dKn+g#hxJzq$>F4N>Y$)ths<)xel39gzNPFHXIl;+7Ip#szJ}#HyTJ9qxWF`%cv`R_;!e5~}yoB5(cHNP9ZbXSX8y*em5#W40%i$!hG-B6?okLB9 z0r}2#h!W+-)`t*KKA_r%pDuw!5oU@k)<`%P=t5gv7;9dOabOU^-vak>5)I4n`{E+1 zsyv_IiHOkk3F?yFDY7=ZVrke8uQk3vdw$*=O*0ff$*$9b=9QlKOE+Evn-E-uE{ z|EqBJLH^vh=9a5h?(V&(FHmZJYv%GI{}vYU1`{4&P}zo(xug#vea6m3ns-N8tWZMV z&2GCA=CrGKPjwDvx4(K6CxpP!gB~HCkwxz{;B28JU?LFN0$;s~k3qnEIUY)oQ|7rMA(I!`tmw{!&v zMm1lN!MAM(jRO*00L4<(OQ!LT6A`Q$^{>a7$b+UV>718*J>aQ^>3d(+WM?>-fo8}l z#6sM?sE3TDi4ZLz!q}oF;A>P((~>8yHY_h*p;q7p&wSR^-CK{_G<-j&(He6?r)>|A z4(ZTt8H=skYP%xJyZfUkqvdlzH0n((J5rYza3HPRFGUrq&9M%z`fAZ?95jypItoKW zhp_4bVGdh&VTFkT0|>Qi_w#ERK{SbkSAAzGZ1WvH#v*C47|^QHpXpD)S#Gh};Mo|w|%_-!lI zO9;BS5Q6NKZG{*Dq6(Dqc0dIN!phs7%8ryJmKyp11(fEyZi(~Tn4-mp!LaP%`4^nW*eIQ&G*2chG>VJh8oKDzAG*qG9+daVLljdH()-B4aVx-LlRk>Y!_h>9wyQ4j2VI^CWj#-P_l9JZob}w%l}Qsy z>uuk3xo$6o`MAv)=0MEE<^EtRrm`b|N%Cw~hEY9oz!=|aI*H#@2p~0L$@$q23Z*fS zQ3}TOgWbq2$iN{t=#PJ@5x20PbG;?I43dCAPff ziBP)ksqlAmjDC)jH;kdL7HpVVBvFMzZG;)v7*|N5-I8|Y-C*#itb)gFadPI93(Y-h zr-i)+x_i|8EzH-2Ie0Dh7*q{ll{{AHB$5=#%PAg`*y-uvePtsDpcb8+dNn~KOrA_# zMw*Lc>O#!;*MyCz>FMW2a@8baD|xnS!tx#{9e$7QzjyRB)O05Gryec;F_h?R`7UQ7 ztiLT}>kJ^(LuOjmwQ-{^-eGyM*VA6v+1Z_{Pc@EMxL}e2?EHhuFw96MnrIa$VTqoK z8eP7YnPE45vma8T9TVJ@09d}s%!r{?s?0;xVm)vdnu)J5hXs^(Yrn$zuquts_c?87z`dqLe&y8)8Os(MWH%ifkrNDpEwyFt;qkK5lV>Um`LoZt!)a z<-J84&Qf9**Rn64o#Swe zT1bUC-?|anTv_lTKQ~Ec-R`yI&1fp=TrY@zi>|vTs17wEfO;TbMKs`|$AO2Jbwp2> zE;Mt07P)5;1T%x8M>RRqdJ)lKd2>oo_kpw-5z2~+gho;HfNsAJY;y7}4Y;Gt&HNog zqoIr694kW5AXA9^1laNB*_Q4e%1OxY~>swh?^ z%3<@nE}-F(poWI`-!i~~W7dDV%U z1~yE*i#q;YNA(VXEuD|M`fb*l(CLmo^CbwuWWsp~@KBWKuH>Y3a!Y&b{Z*{A&4@tX z9;*gf?5FP6yV|cF#$2jLu0p|rh1`w?04_?j04f!ztq3r@ssNk+(0I6 zVCjUpljej?@NlO9GSGI=@v}>_3IssMABRhIMf>16R^(^7hcKptU-5LC26E?cm1cllcF~I?c3}R#T zbRXUR@fzQnxr}iaVci3mZ^i??bAX);Rd)c>m@(`^%>K&^D3LhvfxRh-bT7F{+Ym5+ zu`0$cSx;&WBrM=nae=({d@*gNQ9oJ@w5(^>&N$qGVTn{kXOSaYN~~ z5()IUX}KD)>GwedP4s1~tSR7mYng3cuQ@=Os85;$+c&W9HJ>8}0hCx~@Bp*WU!@Mj zZ^B;IR|YF=g4U3mfRB<@hrLvQm;!oup@`pMjQ^D*W&C}-aDzn$h^*f|w*IRL`T*rr zk>Ttre$;{8Fx!`5nSUFZ30eNDs(~2)D`%&G$9)}`OQ37Uw)ls@T}Rd_xJO^CKU{{= zh&9qjGhsV3fEi2{;c_rId9gE2Sbf*Y8Eag{y!A=9Q#Jb)L(&rquP3Mt zxEaNQ0`7&I3#dUqR$$_U(DogzGmyr!JZA^fe<+r&L-Ee zJb4j`B8aF!90#WKjT3BU7a3rv>d8CsTm^lsm-#!pV@*WiauuZhdP>gW#gUpLh>?Wd zVBuh`OWF-}i8@|m8ADca11Ss1td0Y!mAAV~{pRB`4IC zz%=jnxgaJEL1mXc&lPB=CjTX?4paGc*vUIgvev2&AO<~O3Gcd!ZSU8G#x19teG?B! zIg2OF?>EH1Gf}~u$9CrN2jvJZ?>gNP$VnBQ(HGh`xMSCh6T{t1<1&Hn`}_y}xYm0L z^2S{0m98I7R~MzB%_~{mo)VNIypfjnbC>AXJM_<)9&(V<^sOc}6nsM8X=rg>tqg($ z9tN|-J-mGV4Xc|fj$|W{9+a=Nw*Wc z6ke4KZO{O%N9;T5m5-x+(!rLUwlJBSG`cX0 zTPTTR=!H|R!?@o1`LTj{!j&|Q%Z*`(RkjLXBX145zzJd0#V08r+X*|O+^7rY4Hy|+ za&in4vibB5gcCG*`VlH`=q{P2J@3DTITU{HZdns!qYPI%`F%biT)|xfmJQiCwDnZY z?+1rssK%R~Z>9cVDoHzaEh!*mJ8?_Sh)X&Hrwnp`hh(vVE2^5QL~kW#t9UcK#(mH( zPwl{W<}Bxp>+#U%QGW((jQw@wUTs!0L3VM}O8tmbSWcSnu9Lltf< zi(2#rtLJ?D(-UOvN7YA=SY}J+qrD>vr5ig3jTZcGiPoC`*_-Lt_x)r^>Bnn}L zC;cNa-UMJF+B;amYnJ&t*>v`~#B9=o^QxkfoLybsNVmD{ZyTwy?|SJQIZcPoUCtt3 zo)d&z{D46(O^`b}$ZpIvJHzhym#5JlqDj!wRn_S8OJ7qP(&YY*=wLRFf$mc8+5rwg zyux{^7)TIZ6`4V6i^zCz^U6k)T+w23_x;!DbFat|FlY!EX^Zs{H6^tK_#@)Dxux;? zsbSs^-6px2H~Z&Z!9Rnsr`^5Y9UX~Em9t$bDmV8aYP_T}C^^m<{nw_Ljar3n7nZi# zvgz!1agsx1Y3`;RJ+n4i3K%Yv*iAE-CrK^q2d8>xgpJEI250Oitv?^aj2b+RI_ilA zIxcwh4<|e!@jzNLC{(YuuM7|G`H*g?o0V_I{95*ZpB$DKJutMg<}4>vNe0?cA{fNv zUMKs|E=t#&ROZR`@>sH{myL=TS==8M_ zTWp`_z-jjHV*7atQK%}AajD{!@e%={mYlPMU(p}s&0c8kuB{#_2Cr#)wJ+Hoq@6LL+WV zrLh>_eyG-jMbPjCk*l2$DyebS6;ZwPdQFG~Y`NJ#TM~^OO%ZE3W@RyYRgzW}HSv1{ zMTt1xuz}!p|3P?kkZ2zS^g2}(tmFcmIK+$GY5wog-|jK1vtxP{U~e63TE2hj0oiy_ zSJN|7OF)0Z=r7HbO(#DfHlqX0^er~?hpMK^JJ3t~MBS2%?zktQ&W3?hjJ)PQK9YEF zoa$xT3EB#YQ&Mc0XB2e|zlES2V?tuzq19C96vganq)ic7rJVgLYh!_*ef0pQffHnq zj-%H^lDeZ%v#X$_%Mw}%<=n3!`Y-jaBzgaP zIXAj#%$g_>p-G*br0wT^dh3~!D$b-Oqli6=9l1jbGQ8q(lFn@!<|`*btP*e~%`~9+k8g-l{;R3O(v(8FNdl*Ri3UGBug=#^O&a08A$z(`dutVA+*2D%^ z)32!2BzW;xlL-MJg0t&-D~)fX{VibWrE|rDn_G7HwAmk|D;Fj~j6k2!1=ChMuqDP} z#!vE#l(Uau?&-l_H<;)uu$6xSF3Iks`p{M*pHBoT%3>kW11}DB5_yXrzPA^mPFJlv zBH|HylM@{q|JZqV^dIpMfycnq#uL%~KQdQyFCRox&4r=?uxX#nD~CPy120t0ff!sU zoc{%{At`3h0TI|8GF&4<5)R&xx#b7?5~^OL5(VcwdlBb%=lamplJ>E$Hm!2&_^1{iY8ih_ZpB zy}GyM6hlYZE}MzP*TMuMxzF)d+#BiiwH;0i^+```XXqm`VNlM`=vEf(dVPmw+FEN=WqTj;=XTcl5V#4*>bBJ5K~! zK>oyP3FmV+ir_NWIshTY)s>g_L3rI0J-H4pA>h*fKMzA2;63TzFYm4tt;FTH@1Q>! zI{dhS_sqbr%NI0huve&V7X5LqgkY{fH{mlv!9Re3Z1UN`M$2$WC+l?VK)?C@OGQ`Z z8}BujXFBYB} zfJSiuZ%)5X%ipdrq|dP)GWSh3aT+QgSV#Wr_rIWsl3&<=ba(RFcKMq`gzEte*w*|^ zgZd~Pzs6ddoU4rFQbVzSGaa-ojFW@M80qA^c^QlC1!`8N~$AK9fVkc#jl8xC_e0Bx&a3sW_QHEK@o8JeoJ z*B6jdvH0*#Bb0D9EzUh<=By;G?hv58?ttqgcT5l@dw*lFmY2Xl)xli*CQv zlmQQ+1hK;BP-$zZ<2*C4gAG$LxE$mOWeK}=^q`$Z)N%PU+HW1m(wO9~6d^+PC7t(z zF!n)ZrEW<4t?TZx<5ZG*B6HgDxp=a2?;$H)WlbD_WpNSGnZM4$-|S)=*u{xdID6_} zh}@g4CsFnB7eaxIJYj!Og<*J|iy*JlaBS9F54xB$!Yxg{&yD@-tUbnNqsDGc{st0& zj2{r$G0_j+s{i)P1~lveJ{W8|^djH)$<<`QZTNnFET!(NUVF`lcOe6F?OT@pnNPl{x5+jwKk zLabMrRoj@I&32EWlsG@ri8#Ns2oZ1X8@dq$@A>}zthhaTqE0(ym;yvthxy%T!l(0u zQd{wl?Z7AT_ymW-b@C#asA%`JV6CTlmtGY&0l+9-i_3=KPcG ztMb`^0J>lIhUI~b#G6MUS~8f0L;9r8dlB$bdu}fN^g1W4_1&L5(WAm_5^%0XmU*$L zOf@L@MR=B;KlTIVZ+8;%Q$2@C79N1v?`|m&pnjj+WT9~*K(p~6uw|>ZunXOqa5tPO zH248iC60VxsTxs_;__k4b7AX;SC4H506hf1P+vMlho}jh%Dn!tLM(nEf|mZU@Bz$8 zI2N85!aRGC=Q$O$2PBd9E;<6V`Fg)H;F?1GW!p9-L_BL$uy0=i$depill?^A+8Zd* zgmBBquebNY@qbS!?M%MjC#`raL3m}_gTePE>tC&PK4;i5A+I7ZTb?^Cg$L%3x7a|{ zssuk!jG;3tGh{`#fQGqHlBR!UACTGd&>J?yaC-@Nf9b$Dca0gym`A4BjTF^-b5ENU99W%Ng^3%y_jk6^L^g zNL>^eJPCl04EjKgACXYQUjP_x^f6~C6om>E@bAVan*{sbg#)5sJF|^XV})wx^Xx2(PM!^Lp06g+BvSKYVm^q+ zoFJcaz;OgdfZf0_`_buYIJDi-ISBb9a2?_}dvF3RQ%KM3ZoJjv!2;E^Jvq_ynW50e z`O3i}&>H&EGUH&VdH{0>zGp4+Jr{L05~)5FAymXSSxab?O4Bz(u8!Y)s_IWdYqJ0!bM5zekSp;|t7uOhKe%GZPiQVE-)?*~DsD}B?KU>i@M1GSu?M4k`r6_ap}Xnb_cmx?g@?D2qb*3+(Nrb zIUqsHTn#_sG-RDNl8Fe}e5q(p5)bEN2KD8_^*gj*xT6^+`bT5G%Zix3o>4Vt$5;~* z0(hEP6+%u?M@Ix0-)=Q7^gw&5ScQ3Um+ruO?3(VIlpMLaA<`YqW`AC{hY>B*ZZ9KA z8~PFU!ucP2!2=zm+!LOjUY=wX-zVV?3j6jJ~I=|Z~IJ18-no==J1CkkzszF zw)k>sv0t@PVAU(MxdM|xL)Oo*Bio>c#{kT_edrj=qqIH zZp1IzxUv|Pwy#3!ZeyTU#5*iJR3qs}TtmXZ(##nag+(7S`K+e=>*U z!0+;F0=@avBW>4Vx6}ANW&N@N=VSZn(J&q=J50mAm)i^~X?$g($BtL1eosZW*(vyU z2f=h%LniF<7X3Tv^OGP@1KjuN;1-@{V_+t`QKxve@#x}5>Ny$vJ@hJr_Y-q02ho{mZvKAlqO~;Bm93Cvu@+J3sT_3Soc{A zYPVWxw^!uOu-pdygDBprzDdlCnr8mx0PT!bs!eCq!%dpPr5KN~?QpAt6zRF{W^PNhXv2GlGgX;#(Aj{SntiKX&ujvhs zV6hB^buPEV@`9bB7>2x*uHpYNyNE|Z@Fdw}S?q?PEdY)`fFY8ZMCpi?D6+pL3DO0- z$6o6MJVldoO|)KU6_IHrr?N0T-Qz;v?Lm5BC@o|Jg!bkW?1Ffq1Y8=f4qSNc=c&G) zy!O}TD7r@PAc#5bNROKn-wukeY8JE@{|3@^uM(wpCO>tvDdgBFa2?x2dsHP=^@u?l zySa<`*Q__VWRITzrQ3NOkIqM){#aDUmVAbsJyqhw==oTyw074zG#`?p-C=NkcKo%| z8fQkx*9FvE1@+RA6ChL&*D95rkh9%{cS&e&WHn$I=h@6wxD z^;U8-Q_0U+X&Dc<{LXMZ{#wz?IWJ<@Om-qbDej(3MvJLS=e30hrIRzK)M9l+?h&n zqVfjKmXBh3`jXD=GD4ZKr>3^oDg`8di6I@(>Qb#YyBf0Ih*XN{m6X%No<6QE+jMis zCBX~WJC^1amDzWvy2kCCg9ru>-XQ5449U3%*wx^cYGvSHrF&gkONZvNC`_?Z=lj8K zU7q;3lvtrU`jM5ADg~VABG4cbujSC(fYU~0b~QIL1ZaTx2=D3^1N_K62c`zI1^jmZ zE%aFEfVA0gZ0-!r^lbOe7EcGCNjQtXe{9II;7m2TBn3X;JU{n}0I7-?SV@DPeQ=?i zHh#7|3)`%o^SKV+{i=SfLKR>BA*;KGNhAq>qrqk%B`fa~Z75`%j{|%~i7cah00Uvk zSKcwM#uJ|0dLK=Ktb#KYLJ4pG-!;4H*E+>D4$cEbNC zfZUgxo+sK3q$Z=BCJT~F>`ZFnSJ{;vZ=AE{grZUR!o+%~GQ+G0kpZCuZHbN>)BSI{ zDMh-^+gM*L7KI4oB19M&VKsY5S$c?{3<5L<>DVwINwai&rtrqkffpnp+Gw(zqs>dF zd>@EJ(a5IfPjcc0u4)tp`#4p!pEnK{&~W;|Cwu4X_d541RCLQX$>w`w1dF-sduHD= zbedJ2l%yo!0-@JI$#P+N>b1_)=?k=j$k%mCI!DNJ7{#XbC1vTxusEO%%~ZX0p;iv! zJJor2(};YbEQ^IBdeWLVs^cd3S}UpeCZ%oa1R`Ub5s*i zO?W!aWC2$vR8B7))}PWd+CtULif9{oBVlkDPb<_B$DdzyAnITec}d~If<<5&o}urZ z*8DjFEt&Ah{yDxvl4nd0^}(fV4eFBE8mc9-Dj&9EJsc( zc6*+^eWl7=*P%X(_M=@5hUZpal%#d9*#jtn%lfYqNy0whA8h){+A!e=_HZ2Thg{_Q zA6N-+CsddN&zd5!VrQXg-`?H(c#w0Z^b@~aMq4eTZ6{IBq%jZgYSfXvFCj)uA1XTq zah%gJlgd*X1eeAP5Y_tb9lnm%#v76mF4EqX4wGCNAdN9VZ$wjrelw3afXCiSj_fHT zl#A$JdOkn@ZWrK&w=2&J_RW;!d+grA^vdnVZt(fFAVh1H)7eZ4(EyW>a{gJvK_C$J z1`*09eYq@TzO%Y!P) zItUUE^IkXu;gBW)saP5y4^I!JWO99Pux|#aw=;)Ljm41aL4T(mMuHHm#6*L3jNEO^ zPP8UB)R+zCKq#!@1?;=3f=Z@!mc=0W8c!2d>2F@h-~Ng_;~Rl>E_7D49?rS{)1-X~ z6OZ4M3}T_wNhS6T`F8RLF~#7g74NCx$!b`w@I==vkvjH5n0SRuH1rG!@uyXQ`N(H1u%->UvMj-UvnX{A~hb0bjGKt_oB7y>%|C+D0TSu<;{#(Jc z`?(QR%w0Hto~_?<;QrHsK@UF^3}#9C88GAjLhJiE?f6G22+=a|EIY_KO-K`6Nm2J3 z*j=>4ytF>Z>Q@k8Nv&N2p3e``9p5 zh^PVL<{~EEWMk5wT00z~v5_D-Soi zor6N;x8P;fPRQadK2XuvN(3?hc$)`GA&yU%S1XVplk@;SL3Sf?!L8X-57dYQhnH8D zNrVOD-$*{VoeVlO|BqgdZYO+~!ipaH`R0iWhA`a=9XQvOzq}ll9>Pv_K=|;;btW$= zLiTo0bj`;2SS~9v{^-na{(c3VmRyVbcmc}Fh%odr-4A4O0CyRX_{iNhi$Wj4hmhu% zU90nBxgZDboEYVF_-h`}R3HxW(j7HE3GGcp35`PFSJ16j=sbW3b-;~8N0F-FH8P^- zHQWzpRe86$A4n$1ql~PPVz_ap+kaf23PFoqSj`w5;=OpK&yTi=GAY@Dpjnd48W#qg z4kQdC*YVw?@nsT>IB&y`Tyo=h6j8FPMvK1t=f@*8_1ruB@-4>(c=br41V3)r;WwD| zB=+9%xsp;)%5R(Bo0v@t8Wo*o906?{2z}pX&R zV5Tg6tu$t3%|1jMw*Z_NWR;*$_J7ozAl*v@h8n$GV8WdODDuE8{(Qxkc%c!w`v^Of zy5u%rCveJtyo$INU;irNfb^)7 z1JhqNY{0C;T=;D(FS#eGe!tjgfCwq-_GFgZ7na;ij=QJ5-gD_H%?q9t?XPu*_+X+o zU7bwNfBE<*;3GcgBI0MD$?(g5PC5lzl2f_Y`(Aw+^Q!;tCqc~oB*wxI^iz^zO_WnU zhCG9Des|$28&Paw**76@pQJAg=ed~lV&NLL2c`E06}zt;(Fo(doo&<|PABEZ#^4`8 zUitag@$Y6HO!{5c?>DZs%_wc6)aQ>7o^Ug6H7ha}yhOZ5PaT|CS#<7)pN9^bae49^ zmPp~_sFm>7h)}}lE2eAZ^yL|K&0=Sj7Y{bkfr#HJCb;i_qs^RNhdxYP-)Qu>Rq1w| zqVtT(=a&lua^tnmOx4}jW>eX<+c?x!R##|nJxlhL{`evkC0Rp@)%(()$l3`!SOVUatKc$+h zWmE90YpyKl2!m@{yNV_zPeEB3n0KTT_{c|~xn-%TerzdT?E*l?%QQk>r-__=?3S@3 zDzTw1VuqMe)ucw1M5CxM`H+R#%McoC#U{Bf*iYvbPU;RZcOo$_SZv^m4~3@6Q-*)_ zT#dR4?sK7-)pepw!WUm}2fQKGw~IzSL+30$pAZD~ulZRwZoMR2?S?S%aq^U*Q2B!O z?vWHPawJg$a4K~MWgaF{CwtCSxI-;SfB^Z;$|n$#*;qdjYGE#T(8-2*14jzt^#J+M zA5Rh%u$5=*flP{r1DOmM!Ycu>v#!UopL?+2w!5RN0r*Akc^QFpXSe7XN>v;o z!En_`r`K#+{^jkx$EUt3iVS_4of;1~ubg1@OcKS)jm{jx zwe%iO#h#tZ?8C{!gvAxbiD&G0Re3s_d@Rn?)j45~F=^8C_@c$NO^?6c8T1h-{S|H$ zi1d@gG?qxICw{0mRYB6^*>^WQ*IjUZKuFP$s}tVOZiqAG@0Xc|s|du$YSre23veOk zQJmbP*aV~9TEzh>)>hM#?ek5YiN}Vj7Z}9D>TP(H@Hd6GoB`7Iv{i*o%xH_0b@8Ew z;{3d#ZqKrt+n8@xSyaa*L@%u);{C))Fo@q98&DO$-|CT=R-VdmTVN%q=+`gpd8>1);hu5w-RZm8U_A@>>QEq@ zOXucLMf@HrywMF^^a0+P#?d!hWr0FK+tHa-KWjN^J=b2+qo5Ue5asqs-Yh@Q7q@?E z(sT2P=7YFx38HW5_(zqe3{8qyrs$`vxNL;ko)XHcC4rSZYdKxL|MrCtHxQC;VV%Ql zp3;x-q8!!D(7f=@N_ea`YTm}B#bTVlvo>yFUZPkf>eH;1Cn$WEv}n*xS(V!~jkYwT zv$OV2z5xgTX-fuXqTv#5mO)VMpgBUpvQGzUjqKZ}h6-%tZX(QR!v3zdy9xPiiHv(2 zg}u<`&ep?so+?^?%D_PxVpmdBj{;rYUaFBK6^E%A7Kfhcn;GaSv$>3tb$%>dHrC_q zqsDlLg{1rYB%lDW!Z6I!{Pg32ma>V3%Ly$OPo{J{Y?31vIEz~}BravMEL<;f`!tam zzvzhDPG8;d<2HEP+laj>LDg~J_oNK*3eY4} z&e&+8e#Y&S2TfAG$KmOjM$8TUiLpL~67N+H6j{7+p0`)ob97za)@36&R_jaHrcz=Z zbGu^J#&yDYJO83PTvdAuB}x&B+!h~&L49(~wr|`WlulY(pG^5U9I~+;`i(_FN#;?J zDN3|>S7)yWCK(T1oIM_3=ZtfAT}{4-jhu{_@{SDUz;sNWv%y)mOk{>?O^itu1T82R z4kcq_>f=7o4h7@3ZJ#rKiNkfp@oJs$oS5V&S=g>6a?cg0>%^E%)QSU^x#&x>$=V86 zi>Ux#=%Bk7qFaGj+sYD}`z4b_C(;J2Hm5dZTRuN|?Hf%@C8M5HF{R#=Nn^9uu@Xrt zZbhcItW#4Z0dh?{-_*yQw&5rkig8r-^j=^Jg^K_sW@bFQ7M{KjNX!4RHn<j0O+z zI#IBzr!-L7B7x!*Y2=@n^Z9k`uCqj?4rF(AalCM287vnlX-oPf)Eavdfx56hdhxX) zYN?(1mQ%qv_&TY514BwL7dR`d2loPvoHWB<*-o!O<)`k@TOoGIF<;D)B8x=~+G1y& z8Y;9=`mqn(9^4o+r|TMHET%dm6}r;ZnLFTkkP64fdzyY-21>Wk21U4t0V|0Y6B|YP zC?}4I;N~s%;cz9^9Md`y*7}zT0ph_p(79?P9ljuVG8*Vd8dh6y?LB<2&XC)R0S&XZ zch+@>6t?xm9k&=1uMq1y=Fqr#toBM6Tt~*xlT26nbRX=WFDg=_7E!?>VhJ};$TcP0Rt8nwOYJsn-aZfg1RG}s6 zS`hXlowoa6T^G%CvgqcAdBY?Y%zK$>Va&A08l%n!);clo+2F#EC)ilWDIKmBRt3kN zxW-%aBD-k3a?ya}{c4b;92ivP-o8bH>QNykqUp&fQaFFr%_!JI1AF$) z5SQwFnh({OmK$lrk!%^$GIU8E)@!`dafgMbkJ5fAYG9 z^Pd}1K{3eT5wUc56zAYm9^BwRm3$o&sIucmCfda0(SY&iT~~dRO9ZBN&dd<<%YsB2 zPX=#4mXx38O?%sG>pOzW*&RVPuRcxae2H^({e*G&Jga|mX6VO61E_fzG%j}kt~o9K za?hL_n2TG_p18Sf&1SHjV?GMh`5voBReMd?=tAEfp{?IHIyGJvCT_i}%c65E^5SX7 z?NgcNS2#t&ZC;(C$?MD;k3S8u!$b1y)}>BH)`CA3mDZk z=W(18RytA98^=2^#1Q6y`PtAwY;{(u#=#0Hlcc_mqasS9fKz55a4vMsJ1d#F-5(T- zC(l^PNVt_nZ9l}G^esfOxZ8DWD%=^Oe!40y%%g=hf$`#x6$A9gxX&o!&GRnXKzhWU zEKWsSCcW(I8@pfRS8q?!km&gmS0$^U5P53X8P=)v(IM5DD{NwS6uj#8;?QGbp_or) z)3zDan@^lxuzEg|m~XQMCY58JTH)8)+*S|{FzFY8d2R4 z*T0395kFnHI&~d$Z2u8ZNs(H|FYWa}E49h(1Bb^cJ??!0N*Ea z)i`nEEzed(j&XruN5=aBMDtjpLc{X7scMQI6=(}CW@t79C>Ww{$dAw&D%+NzOeG3C z#0@IUk^70tK5(@%*)5K=1?e`_s#AChqCdg`Wn1LHT$Y0)uX_Z~sC|LFcbBCEKnM+p z`?f4qF)>#d-iTRYNYZ0JA~Ev*bc20Uz36oP=JkNv>HYKT;V8d+d~unQlj|y%k;1# zxK-l%cniN?6qWk10FcN777y~DvcH`@pV@PICU3*KH}az1JOxqQKhB`EQJB}YRk?I3 z)!}6G_s?nE5Asw&|JVOzzmWxD9OKgvNvE2--X9dAZ^pH3??1Qa6QOb77Z~(u4{-Me4EIX}uNeYbgSu}-Nx_e`(3VM|}l9i5K<4F?LM>lQ~t{PT(&PK0p}t|C!Dgk-fs5Wm#Bs+N*0 zX>^&LC~Q5nQ3o)Br1T_0n-6RtMewkXz>2+HG*d)Lds&F;^GWz>BU)FM+&78GMH}_+5{K$vcNc&uA&mLyE_@hP26Liojl^ z8};-FopqU@!fcfoq&S$XlQlPio@@+w$fNc!w19%@wOGuJ!)7%gEA#Sn%|69|bd$pC zacWI&9O%Ns3*NE{h7H@$oEqdvJ^(*E7#fOYLj<`AQCw7B0MojjlW1K?Lvbk!NnM zMqdTdFyx`Q1QK@$DKWozQJvHmQ72VZ%Xnczn{hO|1a2|5+c>o#wt^(j0z8RL_dFaN z?e=9%o_0Gl# zhZYt}gEwup^jNV6`Uyb_40m;N^u|uq@^pNljP<=ush7~kWAs+{XjP#YJJm%0-5Q4a z)-VrT#&*o7{kxQzscIjJ^*!|rs^*DaO;z1?^Y|B+om_$Y8Du8062}lkLA(8z$P>rv zH@?JvCs36(F{KYhV^srYx^3c&C$p4FUpr@XF!EVHHMu@^(Pni%=d8!ZCYWJcmS-l2 zyl6-?TxkWdf?nEW>9z^S5kzl$k(>v$28z3j<+aA1Y_1?DkANlBTUi;t-MkFX+gGcC z&Po=rxERzxCYd&(lM{3QLHBUh)Tf$^Qpolz)$()3(**m%LU3_^r%YT^AXTJZ$kN#6 zaU2j_$m}n14rD1{DJW@+O+MnetVfe(l_96}_K@|T+N$QjRFzLvCEa@ks?%#?%$3zJ zr;=f>?q*YsK_nm!fHHm;L@aZ}z2V%Ju^)0}tH^a3^-T(9uY^@E$DiPDv;BI9?>WOW zHcUSyh#l&>gb_4_UqPh%%x2!HyaB{|g+H7IL}>8*In=vnCPeX-XJO(<> z-ih~c5NTF`cDTBC@+~iqoQ+9!pm9NEk%L3$?FU95$;(6zTSuiVN!z$v3jf8D4}>kK zen^}=(6GGw5)d391_uQBnghgLR@Fa7Y}9Mz)YJQ->>>8gk1$<5 z{3!4ASnnfZrAoNBxhgGg`G$WT#>YGP2Fku0)Jmzr#qoPU zpoUnTA?o%^zr@;r1dgQ;RPcV_Qc^>(SQMA zfom#|B9e2R6&ggVm5&63iEG;Q3~k~B4Q@b$_@ol}k-Gs(-+1*Z&VCf&jB3({S)X)s zE4My+`xnK01kHiUdW#gjsH=!YulJ!lY~`qNx#bKyEq$po8Ul3~OnqrTxT7WPYxA>@ zx0S<9E3DcZ&+$AAM=t^Tb11usQYhk_@p^lukO@WVN*{>BJzkOXm34p1!Q3IZn&Thr zSPB$94^pigL#lP3X^To$cRr*vN@(NLI}%QKOPdaYIVU_TLZ%EP@tXt^G6%)Rl!^^ypum=KjZJV zPy{$-U|FR=31fVVq6?L`F|3Qvh$1^PFX>DJ^vY<)tyyb8=ngEu9A*)6wc8cBB_Wr5 zOM=*2-Ze8h`fe0_)x$a)GJ>*`>&v64KP25uo(h!0yQ-NvF<#q31KAL4@aivWVw$~6 zga+;p1U}tqsq7PSHRs=hbH0Zo_2}6ZRv4rd2oGQq|2XwTXx8S})Fl zn$n(%c5+Eudq)|UpFU`n?ff4raJ zq#SKPLpi^YSuV97@X5%kEaXnHrvN2Y6TtwSO;&i;0@nhCHZ8xeV-X9+auJbUc}139 z`JXz_fEhvx{UTpzkWmpM>#`7Gc?FX{AG$=5_a%VaDFph!5Rq1!Dt0tX-0Z(xsTz0j zBZcBjt4Lk&FG2mEy70{D1^*E_?Dq$6MP~Q^7j`K`iq^iQ7>s7%E)oXoZ!6Me$4ma5 zpku&?=u~{1ANEy4$sC{EDWMH3yflb^pP^2HGnB6URy78d&F*1l)C>^JnZw(p)r#$d$W0W007!NE;??^8C##O#)=r>^3s z8sXcmYxn-Z3rE=ql#i?1g^h>KP*6mPc3rT)6}~Q#eKiFI9mVOQprDN8mNXNUmU}}s z8OhVJ--lGXn$1mTjq==gxcY1wnTj62?!kIXTun`_igRLUvLu7qMujT__ZWZSy$ajW zxwF!=aYIk=6i3*PU;8Q;`9C%_PeCysj+=Qk6xVT#J@T(ROu!o?d*6{`yl{+~=(SJYp0gff z$$I3b;z5|$m#HWb=hb8H*3^G$dnNPv(U;ySUD8*%FqZrF1y8w^e=Labx?9TXuO-16 zF8*kEy_N0J3nObPDJw&$!}B79$MCXy4t|*Q5ucuZGBx9fpF1t>96TrKiWk3yHM2S| zO>29>4o|^>s-)b#{Z0ErysOKa#tz=|E3O@wx-6lu^3B(SfPJEOpuR-rE*EB$CGN=+ zozhUc${dA@dtY*{<#=U@mFTwo=Hq$y7WUF(Y2C!rllxE`oc@0t;t>hv@T3(7w$M2r z!647KIVe+qCd}^*7yn`%mQP^j!lY-ihErulW)W<++2Sc&)>j*TB%D3_PVJ&XE#HY{n^G_x{_gAH31#Znd929oMLE#J3$}+f?H9aGQ*9MyRpyh_sXd7l!@* z>ljqwna|A9wD35aiE#bP%02Xch3%zFd21_~_iRck(Dz@NB0C!8)|~l?_aJ_JbCYbm zuV2qCY=?yMmMyifBo2l<#D$(Zrn31e-xR^Ebm|i~2mTWwa!q*B{rD_V<4CiIcsqDD z4Y76&9Son`&V_NDICwgo=?T7V9K(e{ommwQ`oniG>|&L6et4>GKhAt}>&b#RN$kw# zmgvx0sk>qYdZxio@vf}q=cKP0?~n0DXS!|vVkx27)=?=X(7mPXedw0TkQLqy>k3cJ zw~W}FmnE(x%1Y+u^TjO+(nFlLuv-Td9Hm7zSLXCt+`^)st&;`_bWiaAxN=tw7g-+I zB27!)ow`Rb&qd}sdLFCinY{1F&%u5qWDyKL5>T4F-hkFAKm9)%gLOBw50aaEaR2e}d=QEv>ss{QI3Rl-+HVi163xhlTBhw`WZxju zLYx{&9xlvkwbV^=N+*17VEHZyeI$sk32J+Ur;yc0Oj-0h@`9j*J&HWc?sslsH>`ob zum9cOFPT2l@;-t&S+GAm4|ODw?h=KcC2_dc>^YWB?)rPw=pNf^k{onc!`D5aq_z(X0l-s)S-76wDBYSq^NlmkxSX32*G_58DG?FR{i}$CYzhHaL z2^M6a_CLn*1{Q@J3%Tmp^qB`X?emp1F~=&~gJ<-g(Fk;U_lOA&(PwDCP{>QNn5j`Z zBRG($ba(i5-We|;1LI)4A--2Bc+OdIXNGf*8S>#?N5jq--j1Uf)9-c9GjubpXTN-2eyDAR zb!>a#A2WI*hz=DBTk;$ahDKzjLpX$4;+E}0Y3ItgDdoJcYjoRn8V7`3w6Kg8&B@cx zp0mtV^C;aUe#yg91vgI)5lkh)tVDHNPs|~f^;77tjXx1j3t6;^w*9gEH4(ckLi7V8) zXo+pfxOmQ)(iw9s<+__`i@v_J^K*M{%qdJeb2ee4)i5(t>?3>|ONj6h%=9BX*EX2@ zH@Tlz{&wRXJ6tKonzwgS$XWI9e9}RKyO#()XevL;yLT*h47~f^Iu~3Sn|WYD+Ih?2 z>*xWCVz<(>&yDrwP6XiJjSU7o9?*yrqj`Mz`dm(_vCv_qv7DYBB2F(UZ&aX!UVwpY=0i>nCF5BE^(Tg%-Yd85OH5s>7?l;Uom z?)O30{y&Vp2|SeR|2{raDals$jO%UDK9){LdB*-}FZCHpoN zLS$dEX9<%nV;GF#cMqL&s&hWy|Nk|w*Xb1VJooZ`U+?R_?(2S@{Mu}r+fB1a`_(P% zlc7(!FKvJPqC=DX{SyM!=;FK-*=FwRUQ8H`v{@45mhY1zoJWPyqZ+TqyvWx)m3hHT z8}dhD?BT19IP6lz_YYr6t?xV;6qSuoXc`_v%5`&Nl%}^=#={bDu_^uLNKArx%Lw%};Z-5Vd;oP$wIfB%t$8NaCRv%VBC?>JVE)BAv9J_6vu zXwrJ$e@|u#W;vav23?jWE;n^|T1^LOmzZ`;_MmT)ZyaWv4LXSX;!W5H&4?U#PY{z6 zfxQ%kshZZaqsGyT)KRVm4*;+7 zLN!t-6a^HTTXd`7LJAj`SfKHC@SA0m{-(YUF?39*#wHT?J0QdtL?R}?!e!98E0CCo z@8{1ocM=UHeqQ5%Crm#fYqusC|aZmU5-h}y?=Oz~fMy+KRD z>la$6#-Y&?JGay{`IY7LanUd^6}UTuzNPsEU8r{;$WBHv*ml=YV5hx(c4}*pm^RYG z_3D7{mx>{uxV0}kM_!;pXLH>$nAx`;a@!0!hHqr)AZ^;@D#|RVVgTxXBDg+WSPDhv z7YyUhZ#|pzbFr=0=nHObz?Cn*uY#+GZwVrFQ!sBfOvk#MtCsp}pE$sups{fd@%^=> z{gPT2k+6N;bv%hlQ1bUZS42+#I}04hrc_AfA%`y+x-Drb^F0Q%fV&-buC(3Q1)Jgk zQn|?mzCHxKGoST532Hy~T1=8v#iD(F`GrHqbIXOOA78YSId^8+ILNx|+=RJdYcO19 zG%j!rHt#IjJ^4{%F_OJefGH!*0p8R^-M@G&(x=c9p2tvT$*}+s9X{7kT^_jl(hNni z)vDjYB?RE=j@bbW{Vrv*6tFN-c2EcDmWGz=Tm0-6aM9kjaadHtbNexxxLQQDSLJQr zk$Pq;LB}1j|Bn=RAS+Pc2YHbjefhFB@~!JNmZ9nOrL=R!7|uaoyl(xP;P)hsivX$s zMuF(MH&hty;(^7tvkl_Pv01X#+;ZDasrgUyI4pYc+Q{JA<<&2K8SWb);b79(hpGil z-0?aA?YQ;z8ToE!?t`I$s8DU^Z+%OpcktkH58aY8DCb@hT+jC6G~=Uw%(6hGb!a+s z@SBaJ*x+b08H2r=B@VlyuyWON@l#~>2tGZdQitcVDY4wXicjB1;S(Jt)piZ%OdFVVw5+Ne=|0>|+tcUmNlqjrz>PMLXK&b^pYgcsV_>$#R zD>4Sd$rn0S3Myt^Bgg1aMiMR74Ec;AxqWLg2kx>nUB{u#j&UEU^t(qRhQ3a$sR7*9 zwL~2$l|C~*7&!B3sX-e_i1J=h4{kjxTiK4gdLFg|S@9^S^t45V3bQP~ z`5qp6I~DpgvLMxx|6tRIZhq>aZWZ-Ao>a)O+AsESGatN5jW}i3rj5iS)>UdsIlFBtu1)u$m(Qj)Y9j@g(hxxvujW{x z?%k8ac@s0u{bLKS%~6;vFg=aOXw;a@-imEbrTf(nx~!(VP$omMXK0cKKjkM16k=1L z+wlA|-E|1Piay7gfE;J<^8(#A&7CXVDQ5>=P%DZn65T3O3S*NG{M;IGVZ8spy4FD> zrr^J!^%&gg1dVTcqfo{}g{8?L8=8e==%&(iV>&)R3(%!27PLbd%+PD7M%n6EPK|VT zYhF~SLyJ#4@}M-~(?#&W4=@{(Z8BawC=KIwZ;%cfE{SOYa86K&3h*~Us?kUn4&uPw z62ZKuF>}@EL&>^TqNw}5X)9*nR`7D$W8K_jU;)d_^hfdQ*vDu>rg-}0WDigGcvQ_2Ijqjc6NFXcB!{p=AZ5ZasddamCjQh|a;i=9% zA<|YtBM$S1RA@@rRDI-&4t8@VfHeva3B8;WuHF5ki0&>pNubOz=v#YrlPT3rlD>PA zM5NgB+~EFqI>+z{kz`#xu~07iTh%w>b!2EH6uwQo^&4sqL-KUnfH1X9hFVHn7xZ62 zm7jKkhh4%&QQHTde@xV-ae;?XOT)Lx)}A^q)t>9deX&`BWiv>bAImSV$a@@6S<1~a zGF~qZ^CSUH5D$&DKUk6ey9|QY&;w|V2hey)UZ!+JDd)uH3XJ;%zCFAq#|?gl*e7K@ zH?>rhj&W~0Dk!c)gAOoi$K4uS`Vs865DlAqqFOY673H;ny4e(X9t?D}r~#C~z? ze_>&2nJohN4jMmFBZ-xB87mv{rD3O;;rD-1L7jySJykWyC=V! zx<%D2^_Gnq%KaivoPu07eSiLzDA^z24uNDw0$%kQ*O3hyT`M>%`%wc_|L(;qHMcQ% zi+uNFTyL3bi;ygTFD_>0PC34-9e3#AT8t#q=}DDT=w(0uBAZ(d-IGT=EHMi+B%1va zGp^7+nSEJSEiZ6W^Es+9pk1NUH$BvO&huMMS@dVz)zU58)gi>@r$F*uM2u9ZHv?T$ zDF$R?*qOh5qdZ`QC<^}G>})yY>@n=fA3cIwA7+y07;3-9l%D7Sms{Hsf1?k@UfZPC z(?+^;hm(W8<+{87&%V)aw6D>EC~lxwVoLK~RRc?ddDFr$YWdZqS>ftfWuxNR_X#gs;2~irVsqQ}y!+)EV?YHa`2Tzxq z9GrdamewIEZYC=#?GN}>=;G|nxjx;cy*^ zxf@@?Y@GzPk>NlRvob3$-(~PU7b(W9HdE}qc8=z2n6tkw@=I5qqkXCTmxAz)RS=>H z1e_EHxY?btqiJ-|ZF%f$yyc+y4S3i9ofA`+P)6DUGZfU2Z^31&EIhu&gh0PWZJJIg zp|Fh9&!bx?hZ{&;HBd#JiW|3gdNzhEchufi+uVtIhbteOCCc9HcSf{5PQ2m@pL!g% zc@^LF7K9X%xM|;6RW0yV-4Wj8A=hTUQDb4+={xNk=v5A=4%oYRaC_eN-=MR zCh;djeR(fmb)1}RJ<@)pR3~1pRW2#W8Xk2Fq^|q8H92Xe^yVadh`^+p>dJ@?U5{He zU2xcR3_TOJRv}W2)OWzgGpZu6-xo!j(FR{=N}Q{_hkdrI{OZ6X?sOiR(O?I@!M*y! z-dxc>+9Tg)=5tXg^#JaEt_eMvt7vm9(BwVJ*p>I9Oc#Z|m!P3&`TaQpVD8>Hd_ zCc2+b2Rn~sTdb)8gm9*C3niZa7}^~BOjvj$MN95m)^rl|7|;dYm)~06O_{JU1*|3| z##-_In$h!T4htNi$6u=~yo`$yq61%7fLbUBVB2GJG2HS~ez8lch(!sVO`oe8f6GAy z9FRRqtb-I((;XfH*`CC)G_pv16A+R0W<=}9j&;MM@rc(_L7E$GO)1T~$C(OsxA z_8eU>ZlD6?rA`ZKyPZsw=WwYajy0_Dt2a_dZp+KecULM2q(Nz($q`c-wfV=w3EGdT$BqG^N(dWq3nKr0yg5X`q}kRb0#khpD*AgDUR^4 zXt@f)s|5EXD9{VgHs0kf@>TeBkkqr$UNou%_N~VbbKs4d{a1<~ZulW<@DXk3t*)d2 z{pXnF>dtHhgp7&z#}{dmgOGviHP0WEc$ssZ0k8mef3O=AhyUk&| zsZbbkYk77wx(IyZr8cr4JzN0%IBA^I0Q>?s`)4+&uQt;6s2jX~RawIkrJ69a!xB;$L06GAV)v4^h%F(SyfE4bBiHklm+2~a!9wD*x1PkrQjv7<3j6EU4+ zmkfUNKM3v)Wemk%3$fkgk{VRmX8+~^+tTNY2lU2g8YRf~4KMD*%L2Wfzv%6`#mX43 zqAk;(ZvWxY-&*FoRR+jO29m=6?&s$ai4j79ydiR!l(M3hXBeL|;2Jy+C36@UQkS^J zeYb@dD_NU3)TyA%Tf|>JKiWTc{{CNbZMi)8?h0jZ%ZT*fEI`@nNE z_$P{49a>akG7nRr-M|TpqCO-#g?7Hm-SN%6YXn4 zHPe+i8whgQ4A1h>h=GrYj>^K~rORzqPj)(dV~O7aAeHkPgLA|tpOa%%C)^yWl%8w& zHu;4P9?iR6U>duVw(j!Hzsdeb&J6ve-dZw_*aGjr^2o<968vO`|0V5Toq63sNeazB z)ZXhr(M;O^lVvVjzdWv`;!^B+SGGr;vZm5JE{?K47_2r1u2AAD9r7Pbo-@b|h^k6r$l$Da>J1Hkw-aR`muMGwcf`00kI_!(kgP3=hG7CqreuG#w`ayJX)VbXlEoO? znU!(v3^zGTx4j~M@}Um0SNQKT!^%Bfq7ry zvKQwN13JDnv}5LiY>tUzyK}5pmP^;8oU0jpu!xVz@^&v?&0U#7@MCMf)1B6e!}qI4 zZ!K^bL%DSO8#05)o$C;AcPqnVAZz61E zuPwI4SLe)Iq6?WB;Rv-TiCA5GCYiR>u%nzdiaZTBMkISKO)W{aOVe+t%{;T5 zZh3=g&+tCs%KEC<`%j>PJ|xwNUR`)%g4)x2pou}PePgz6b#o`8ktEUIcoYhHPU z#v6-q1uoYXmxOUOnrsLJr%k;ULPKlG6!kxMg$Cf=Hf>?5WPD`CTYCuI(D~{fPH0q< zoFAR}GIHIZT1PM}(o^4Tl)I-O7ZYnwnJM@2{;3wtLjvu>D=IVMFY=) zt~IQl*_b)gju)gSfA7N0T6|~;A*p{2rNIF7WQ(_JIqmwnHEx?t%imWo1y)t}&%ORl zCjD!Bh_pd;V@`*L6r%4L-tC6y#zgKZoU^Au=|!b}VQ*QCW$Mg1k>ag;9&sCbJ-DXJ z^S#^@$}XyEc)_@p*(F&|p)HT^8RQ$UD)Zf*K4YkU8*PNRvC-z4%7dA?J2i5JO;T;N zJ8@?~)h-!oWLD-|U~Gv^4k=!YhT`8`#$_^-<`X~Vb7ZK z5DyD9q%_|QS!VEjS96pDKGMTkjDNeFPzwgcW z4;7kgiT1b)MwrEF`+xJQ#pI$^t%}WE_;Tp5-cMs za~4O$77?LrD8^79YB!8eW*a}l}GI!uQvThqOO@Nijk%Y{|3bsBi^zXL`{ygEq{04j0Q+QckoKzbvrXvex zM0(vlZ}=lW8^byCX$&o>9C>|8*k`6k*nz;U;vdX2*H7?88bvo`q&FwtWZFz82GC@K z(oYOHDx;H-ux+;29^b;VIn}68GudtYbb&xDn#S?6El=BDq1CZ(7)JjukRCAURzVdH zI_w+>(o!Zt;0D)=CWfl#cfLRJdC8&A&@lN-5?<4UK=`sdO$xeQubd4G*l6##rT$>2 z79PQc&Rl2+PGgb}R>jPvj+W+I4EM!-AKRtb_rNP8~FFTj<&Ar9PTX zVuHY!2>oNx1|x>uTQ;GErep^j`~tOS$!3gx?!74e-a9SIp{%vq2e2y}@>_HqwW8Ue7&Klk zoq`c>OX*=fe3=8X;$Y;ZI`&ZtG#KbBZ1r#2;b0AcBSq`UIu*$mx?mV%D685-LO zbgXbwesjZi))G#_%FxW&8}@1B2=s03S{Ba6!kh{Ey2ddJo`b<->TRJEf{#DL81 za2mL)iSD72)#*<*$ERYCqrJ$aFB+VnS!i*50z{25pE!SVr50f)`+zI_h|QP34!$QO{mOVh z_a#G}e~#br-SL|~f_e%D&_at$8rMp8RzPeiPV7US<1p}qDI>0Q4x}Ng54cDRcgYuO zaeb1Z*xrpMCcIWx56?K>x#*DBc`V>sh%PFqJAasD)LKp^IS7&1yf;lO5cf{*1s?GpttyN&y0 zU$tYO?UjRc|610(HQMD~JpVSi1TiYg@89!H{j#dc7ln~xm|bRr`d=F%dpmW%QTqYb z-LUEUnT5Z#2f*fS(X#Eat>|u_E2%szf168GcvB0L+P81Zbv{}jkR!Q3gw~gv05|`~ zfwO1oimdPw-ek`?wpe}i?=NQy5BT%Kzc!3D9McKU_-t)Ta*E|136vB2!;?eukyv9k z&jmI*E>0W$#YU5$0nO8bo#%9kGU*N%wr(`wlmI zQPMJ8o=CKx3`fM=bbVT)4p|R`23K)R48-%?C-ZP#)u+6?sooq zawm52Vor$rk6?uQmRrK@Nlh*Hvcw# zdpBQ1$?9EnwFueuP+bqfbh>s=zU1#cMH5^9c^d3H=<$odPMj9{ZR{WrT6;_-VT-y) zR@aYqch>7DyR%17jcYSo=hLk&RNrJ2D`B8f4z3KsjE6Pwl8j8JRsLQ$O`I{ui;jeE z;v)P$G~d4R^oh9ArHj#f-4}uJy~mxOj-Ih`e}6|zPYwSm?MEJbO^(f?KjWmaT$_evywmL6c z{^L;EHTRx6kX&a08y@b@i#B5ln4RzKtnD8@#tv^W?+o`TjA-+nrPij=-NS;~k2Y|* z6~dSrULO5^YX>2)*_L9dno&7F;LvTeuYKWGTzP-B{hf7C3=~K7-2oZr zLg4R#2wC$gqE*ffkGsR~UO3Q1-kJ;`tu6P_R|9%bK4b#lOcM^oAVo;_jI2yT7>Gc z#GUU8x92M;p#Z@rQ+LZXG{XLmD?F;pLN30xavdpSY|Tf1@~rMl>K8`A$oFR{55oCO zUN_$u_d3L&k7r>QPU+{nlzfNR{o;hNESRud@gyiU!_04PKJb*6B+pWj+YR{vwqy{h zqagoB-s5iNP5_AfWoE&z#YbzSCnHSR>`1@Xt`by9sz41u()-0q2!4%^h2A8%K^QGN6CEtpl0!D?k|@GzoW_UvCNTghJ}Timqe-FpSJ_`Hz_a=?2le zEP1XgTkgqp6vRpdjBZL$6-UtNz~7LjV_#D!1mbkT;Zf+ZY#8KV=!^W!!&alx|GMDU z?>697h}^c1LXY1tr=Rz~6)nfyiGH3`=p{kxT@BXXBGJdwN7wi~QPWdbLqV_7 z09RilYTnx&z;1pP-qla+Ja?f#A{~AbHD_(YZoV@aIPzoioao4U@$CC_*Dm;U*6fYv zrZVt57VYvUh_7=>*hgNj#(QPWX#S6KBq`W+_n(*EUC=NkujN^Q|m-#Q#=fO{T(&C z*_$U*s7B+3^Tc%sPr%D_?5#}*7xV39O>$NZjOQ#jwe*mlxbE3l%Z614ZG}0EXlG$b zdL*=eu%j0#xh0p&4~NUcC!|*3Vkjm!{CnAT;(D3z+MJKDx~>l{QXV!rheyxP4rE$H z1afW81QKx>2!@_^i?!9c^z@b02nQ7n1>KGnu-TA@2%igy4rJ@+2xO~azTE!TW%_GRwyXPF@z2>rJ=6Z>_7u?pnSPbDoCA-z)|5_&XvHnK&px3 zfR}c-lB~StmCWF!?)P_M6E%ZfOJJBL1(l@k_#Oo!B(rB8C}c6HBwQ(1y%zj z?=ont4ymypSh<<1SHdnGQIsbINce7ApfuQkS)y7WdP!`Mv~H{e0@5_n2`x0B~7$Ajd>n~hJl170vwf~^HG5O_U#wqgLUG-TuAyr z20=jtU)SE3%Bpp}Pb)B1$i#Ju(zG@SMR8IAAcIc*2+b}sh`>^u;8QH$#y#s&)*be0 z7EO>h>CA&-x=YK73)%gFfdL)fAG$g^pOY2QX4@rsGG|PR9kQ;WJ4dKm=e{7I*OvO; zq_1+D2!{+_x<2Ps=x6ShR#w}1ABv#q>$k7D#_#^RQ2b_saJ;v=S1Mlwn454CO(3C7 zclHag1_2WaFaBA=6{|!l*iaM_>{w0*e(PId^BL^VtP&aLTh6c+rLXelGuadzYZ8TB zRhU?PgUM_yav`pARXKHSJR_MTk7dO#?INQ#k}n0SrtW)wBYSs1NJA3R3;;M26GNlq zceU7|lacSWaJ+R|r+PMnw-E(9?s=Md?{z9QwLZVs^5sylg|QiK$`P4iEtdMNEa{sV zt8JXR5ozEV3S2^ePIf2ml-Q@bW{%ykB2Ai*w#5SsnGOh(!QjKAg!T8s`@xe$JZMZG zT{Lq`Xb5b+*{2Kdj9rBuHAf@qP@#8Mv(4R(&rzkKZI*i-D$IM|&+Dds67BYlB9+Q+ zeG~6ySvPq~_~D+R3$$ej?+(-hhuF=AH`n;ksutIK zl2b>87PE5d_c4pZx>XzvC_i;5@=zbN&Pn>vqBC!1vpr(P(|>~|{KbuA4g`Oe_-C+Z zZ6)eO_T2wl=uL3PbS@v~_+t6m0gxg?A|{ro86 z`X)z^L273VFk>{6*`;1n`9{~P$N`Mb1rurNCZn@#e*Lr#$$*p9DCUF(DzN(uK zk`c2h?whrIc!hPr6-SmW#iAvWi1_AqO(+}oEUh?k0rK%dy?uBMpSq9wy;*$^Z{Naf z7}p9AJ|Kg;^~?$`(Troh@hCJt>hI)b+ByviFgOY`zb2Ve&8{S1<3+%52$tI4^B9BaIxB~8t`P4;44 zOTZq#gA^Yj`T7km=T;U#^o|-Fk5_crnLpVe!Sw2Low82OG<00^ z_l}meB8~x$clB1*UT%X*dHSBkiEN@EELl))9(y497Z;{b93q{+NK&k?zojO&;x?v9 z{nnr3@YglnAQ|rNw?zBw7j2f9DNIL8pe^E!;~3=Q!}^=q=jgk2sE|4erx3;+8#kN; z9b|6nW_ED@>(a4Xf}R;70JR~^OnistC?-C=JpxI25Y{2 zSJ8F0(^+g=kyn1u6a)ghm<*g_*&Cg|w&@_n?BJM8&|y_=YK2o5Iimz#S+#05X0}>B zYUC;>UXY7mpNGUb!Wk3cBuX#qPg|(@)wChTv!C^|XTm zP4*RS7zdJ>eH2EJJ$TQM_(@D*Dr@pu`;U&}q(*`bJ3dCXNt?7EoI0)p2>_msmJ{gz z)nU?DGK+gV#n)O+2>}2+ax5xQ`~x}Peq*!StRJ_ML^rSD?KlsD;UqK>$h6PU&$&?e zs~i^102e%4PN4gb3kEU1{zz$sjhRT;Z+5sE>ug!YY(?+hz?-ksiY!t0^Ri@A9xXhD zmB$7`jy=?NWJu||puK!gJ!~rjFn0&?Mfv@E75J}nDP$eqq`DhJ^WN54<90pts%!CZ z!NDX6GZ7sFixri@EoZ#l^{Ie^N~6dZftlW6-W1ei7I;; z#BI2zb}sH-nfhLJMzye_OH5mNzvxmDw9CCLFFt0+=lET+!JpLSw+I3*;q8MW$yX;; z+6RQx1bvpd4DB6PCbhC@JS`d3*Q5?%1F^M`k`KddS2P>ta>fDkg}A7Z81g?K$RPDD z@$n&NR^TfalKLbrTy?x)xFJ&S7_i(XJ{+}`I8>CO6oZe)AIN1tu*e3=X~&iS?&GRt z(DKu1`>{r$OCQY351mtUj$(Xim(8fiErO46A;;5Us=?4Jvh4x=&*}?Qt5VXjqaYKe zD2`)>WsfmhhM79z^1~sKdL!RjsVTT&kOx+sPt}f$5LGRP8h~F;ouu42aQ|mCXu{qV zmXSq4kKIh%Q0ilR)rAsVj{JD2SATH+)(>zmvRJ=xak=4|dU+aBX@1z$#HTKG_+OXO zE&%lgB0{NcHPU$C+TmJnclyBPFHG;s51Au5d6U_rhoZcfYR^Q?BWS0HQvqa752NO9 z;pvtFxzsE%?Q-qyLtbbeZF z`Ork5`aroGVK$50SgVz`Q#2JBUt*!REy2>tfo*Y{{9mmlk*Cui7@r<&;KiCiCb>Le z5!e)VYJt}6-4q~_QsAD5#KG>nme*zG953_iD<<>d*!Lgt4}maT-}kDuW*hfZE%Km1 zX{D;g5i7G<=Qp_n<8S*M3Kv_HsjvveQH@b>%r7ORisSBoA{!-bcHRCjqjfhPCJY(* zx`$UYzAL}g|5fBhsLjV&dl2l*Q#(P`$Z8Hno7mWp+Zm$PIEv!Ko9fj6(WH)j&*b^{ zF|*UF8tIMb-KrE!YyuBLjudO9&fs8W?+QpxeY!(oyMvX7JXBp2^ftWSXe+hTHGWr8 zX{;%0kKgVQ+Mkrx1hPRJ#Yl)8I(j;H(Z+bD!MxlAqB9O7RUN~bLjU6%3MUDVi7f<( z1igpbz?Bha)u^?gQpNw{Y7}f%QdH|Lrat3NWD?=mvUhG4d(0Hx#BOuM5Ss(-b|4Sc z7exd4N4IJ|6zp@5X?gXNcl?!xKaAe4YY)O|0R6JMM-!$gkd$Rn9@sUhSD|7z2O=E6f$@h!u93t@e*e#Ab!JT8Q&Dh84 zU6;TmAy%}<(qlOwXO;gGoWQuS^4!|I+q3`6b!Y)b$n0!T=}B%BwOHlnS*`4yZq03) z>YFFZVvQgVcfsN_(Y~T$=cdGt>+49eOw8VjTVwbwB+J0_iwoj}VeVPw;GeFJ z-365%yV&!`)X!8dd3Uqg{bD58v*$*u-G% z>fAd!@8#7YnXfOKos}9)ph#9ZlIb?}uLfI*krGmICV9{uJIP+*&EayS%*_|N91HzDy3Z@2> zxx@s^d5DfWDFr?yCed3{C%sL}ThBi-?aLE_Io*ov@O!S+n#7mFK{59FXOaM+StL78yEFbm(5o$o!*dLS#HPv`cfkr{dq zfgbg|4-bmkAdnrLma3AW0PqT+w`XFUrF*D<@32YdT+(SWi8L-CrhQ_)exQSSYQsRT zg*=m&6lWT~T@OMXXuDM?7sTMo39`+(Dbq0Z&W81y%y@@4nt z;;T9$@>`g+LEs3UGKrtl<{9(JJ^KCUnQM{{P5@H!wEz7TfUOx4l;(tM*2O>%C_YB2 zCZ{wErl?uyeIv#ytR02BxCylMTcyV?@bk^WKZq{%{^zai%3bI-2(6cY&^&Z~K@bO0N5{zzxZLQD4fR7= zsOAUNr>{92(3AHr@I)jHmwNZUu9q1|horP-f}sQ2=2uSj>Ui`09X!AVM=G{dLZU2N z`zkcqAizDs1)Io@axB;)emlD!**xyXf7vo*uvaY!>OlKzH=v5JH{#C?s!(xnJb}xKYS&jcubc^wk>;E+P$1MVh|eXN5#+kY^0ui* z;2XvY80D-cDT!< zu^QN#Cdo}{(c9YCZM!HdRvJW}h_hZ@v-#ZhF)iVl*S zAi;^r8laM~M*PGUDj^R^**LBeL92_geP@|46L&O06kKuW!yfta^g4&!!jhmHo%yqz zp}0xPl4BT;M?3=ea?%vgdcq+-0#67-p*#z5+V}McJebL@u`IzULQ8S|$wTasXI6%q zSDz29$2qr`d=@)YtUsN8;BpV{5X_kGS+PxGJBE$V5xaxbdHF}9fpOY@puvuI?l^p( z!)Y;Mv^LEQ&vb}pdN183TEu=ogQ-5f1w7hB6iu3mOKomteBW&RQGZmuRisa#4fVNXk7+5>|6Z&kAQ~lwA}P z@XWFpY_8}g#M@QA6Ix0$RK(*`@h4`N-fPK6BId>RsUlrt=s34)RX4}iv)Y%Y*v??3 zyQ9buIIS*wk2$q0XZ?(eJ5O5tT714o3)s%yr=pS8y-jV-HqbVP9BUiv=Q;dmqp3)Z z2J8+(`{8Fj`t4fA=5lsu>)hJx%ye>T5CN;$1M390ugD)3%es$ z;Ht{AS8bC`Gj3Ri*`~?YK3kbzl3J4P7|S}o8ei)t&1l}*KOMBSP^ioLK6}jgkwm4k zrJ&ZrmgMFM%;W`e&bYND1dZd2*742Ju0iDGBxo<4%-A8$9*Kb%V_vM88J%K&nAd(7 zS@F?iNg*5}s}(Z^&?2y$K(-5n6|5x~PHfp(Cqsv$Z-vLs&~D6aaUl|v#(55SArOPi zr<^X$-m5vGk!#*bjKKLMt#)|2e7`?PXxM&ca}*}UL-Gv9;n>$*a3!a*CAL=8G%RM@ z7t#gdP+aOQ=RqN3&sup%tE>}$)YoWY@j2uhH#l-_$CP-t>Q)J4@E}Pyy?+Z&P9h1& z+u(e?aG4pR_8A?-r1D)s8C(664)qHTgSkp;c?vdt_yP^{`?Ve}w6iw7H-Z%G3}=Y} z$9N^4C&W8*Q?e)S$YW(ZspF?IhIl;qc*ceWibhWTM*}X82V0Mj2&yCqv4d9L-rkoj zy+sw{V+auZKBe92*PE+D<3crcwu|2}DPW`FR;+13yB0||-S*zBrn0`|z|Cp=^NabWQ2ACRIxQJVGrfT^3dk7p^I~PsuGs6 zUvskX!f?RxOn(s=jzSyfHlYP@E5 zl!_Y|P4^FRKQfk-*Lq^Y?cP$Iu+Hm#^G*FZkF;J^ymZ>UHN=5yub_Immx9w1ZmDZW zw&k$+$7NrarrZQWZj}v7(LIQj7q&|}mgQmoF>u9yDUk4NNzUbd@YX_KxNr8uqZ>md zt2w1ubZX3201ta&VA})DH9?YP4wyk$)w765V5$5y(iY_@_2d~C9|P)tj!s)w*G-^7 zDZ6*`#7RToSnVk1S=N~t283$Vv7llwO95$41<_I69qY%(!$GD9=q}6Za>{FMN*`4# zJqmDU^d9<@OMSQ-7Zo>g;FAcy@=RV7ae8BvxTZB`+I8G`J9s=mygsA0g+ptcjptf> z{jCL6tw!E#8QYyeQ_G@D$gPeU2j&ez+VNlO5GbdFQ=pud%_c)HU%s50+I}alv%50s zfu7OkmqvNUfFHpe^NykH-m?Nd(5%xu?z$5h^-65?{7@pxFT?sw^Jd#^T~N|}DVMdd z8vDYEpqXjBBwN%rmrL#X0iQ8d+U6NHRrr1-mg)QR#$^IFXx~~NbinNSCtHqn!@jG? zP6Ttgn|!mkwVZ>>={eTrAryO}X6-TJljHoNCQr1uyJj@yg9)yYrzCg>cnnz=MawR(^NWj%^Iy`{-Ki3K%6VTe)pqi0I6ad~*psWRZ(MP9Oyr8kNP-9Q zHm}+>j|9I@m$%w3htX(J4hAZjJ({VYbleRLVQl0NF7)ey{*9rd+z7NU@?3j`(9c#6 z`YfhWTJx-TUzN70b6es-Gwo|YgrPwk`K0&u;640BmW3*7<1+o zuncp427*FZbTe3J>i{m-5zoG|G!ACKFBze*wv0sh3$DPARS80Z!j&iD>jlD%C0*HF zC%TO)?419 z241HA-fD2zwI|{#MOU}x>jUp*JhOB$Z6GUy8PA#&hm24`GU<}m*RF)S(a%&TF?D&+ z3oqP&h?v-#bc*FL<_qtop1K`8%ss}?4644pjb+?gRmt}f9TNT!xrIr3(0OwDKKkLQ zT?{#`Lle7QkQPV9GdBC2Tl~)f$dp&zZQc%iD;6G&MGDsvkp=NZX@4U}eKb`npK0 zcOL>1oZ(Bkz1?$>OTczcx%xz*WpQFZ`~phJu;)p~j5OxS27w`D&;b)LLrS=7V-?V| z+;KA%_Xi_zVT}6eFY;BlC4+%A4#BbQlf8qWs@%~!4jz1QzfDT{&^aB+DYBkp=Icd~ zEt&o9hA$t1krTw((JO4>hi=&0ORa)v zl?ac;J#^Y-nvVp{d(IBwVoC>p&>rJ#7hYd!8gomDJ^ArLU|8iT$Kr5#VJu5g6wSsf zN1Bnf&*5jf`CSHi&$@_Au_$Gu>*U6&#^m$Xg&UX3n4Z^!N~Wm#YJ6EAx*vTpTCGjL z3!l>-qVW96Yz;mvrjKt82k5sgAm*R!)lNGRAMNB;%A_lQ@2nXayGVSk2(ZGvz{ zO3J3k&~Zp+ucq9ia^{Fc^*NsOL6nUeN2)nh2hZsRwqD9z?r&UKo{_VjA6akEnrK&R zi-zyNcRBs3zr_aOaV(j&%z7hdhzG2)PGFAZDCb*s`9!xbns-e{yK6#=wN4kzt#Be* zN(23*r#qG}m?A7|ZQ0QICxV~Ylv#@UPr2=suS#OeOcO!8q(7}yIA@ry&$j8ouM_h8 zo7*u^d#dPE>J(^hZ?EECJro3o5``%Pe2Oe9Mu%tGJfc4eXd}NLN^=8Nj+C2CC>iqiONp| zzp;nlrcA9X7l2n~%~CZB-<&O%edt+c{*ejaM~3s4#O@Y7vVWwN9zbZ=0-?BF^0;x9 zKEAG`o%Qf!iwmiIoB0H$O7JFMyuFpR=6FH;-M{7oRWsuoHa{U2=5+}6ID&ngPE4*3 z8+02I6;EvE=&cU>~GTEvbkWf52Q-7XL@?>XG# z_%;H&qA$1Uj|OPT!mMq(?*GPaiat_iUa6M}NxxtR7HMfhQ7rkzd1R226!cO?Cm1Ge zYRqAbtpqm`B1?o{0He28|BM(=p~owp2c5a0)EhF^K_J9U(sF;uSahV-S~inMN-{nx zKN|vPk&d%N0d8LHO~55aPQh>z+P=JFIr8corqS*_+%Q$_sP9n3usg#>0ro_qzjp<- z@>s^1_LYsaDU-OB7Z3CF%~$MxBd(x~*zA>&3pO%8I&QzcSz>+^yj~?{oh9tWvDI51 zK@ardW4Wcfvdqi8Vho*Aa7XU%U7py~0rEc3FQ1FRU^UuIGf~XrK(*dauVG}yFqz%@ zf`-frx`A%k5#Hrhdgj2uz^C|uS5ErNqg^D+tfcHF*NOWbug%bhGyOfUgM1|i8tBo7(rq(6?&wyP7Ba}E_wvJA zb`16zZ}T)h&d<-Iz>4B0Kf3$i|7SDO^3LDkIuKt*Y%sh0Fl3)jF#PQ1{dSl znmmUPE(4K~smfFP@(zI_yw2O}+xpVkA|Mp*IWJ!30nYUMI^$*iYAWQDeWiLkE-5)# zx>GnPd6KvUnuTyBH7i`x|MB^`M)T}r3*SHKFUj^0$sv3vNEoE(1-xr1JioIb@EkxO z%}}Ge6oInfA`gQRmtG$7enJWWhLi4n$SV3!Ug%KoW9w1pY++J5H6e9AZN1kD*xfoY zG`uF6_siKeyDCbssh~QMAA!~#GJ=SlP+5J*8bYdNdtTjt^b1l& zG!M3i1E0ifTHt$~;_gebHGKsKAwiU}ck*mrS6JS3G$w!dm1xU=?EJt4Dsw$_lzT0*AI)GkdLZidBd{Y5yI`5o%Q<&%vhmQup!3CEl`?gzYRZBPRoGn%F17dZ`i{jnTm6n(UnE)OVHak#30E^Q)OS z!>}};(T&wjW+oEvF^4fAmP0KUhlraUX%oEAX_SR|j)tsVl@cA-~7yV1~ z1eMSkq$+suJKH&WQi0eLXGxk?nG#?PPo-{3n@rl^<0rWFm0e!;@FYWTL#eN_xF+)K zd=Z5Wp7l6t2p$0sf*t-L10yOf{`M+EEwm&r!sE+kzwp({#raxlM!+*H_|~+jPV5Uu zpN0yT=ABlmF&-ySC>~|bb?ebIZ2imjNm_r79K=}HYG?4Y_%}E0yS7;7;i5u#rEHLk zACv7tUBImHfY~?VC2m{GgbeaFk1+z*u2{cfZ@Kf)%qzHi&kQ(Gy(@C9V`FdfBHq{% zCrE}y4a*ry&ijL0;dd`Wp6oXnMhSOl@=lBMjq-@O;=+q6?$#_JZXv&TeuvdetHOah{Xi zmuS2_ z)D)tjRI^!38H?YUR!D9>_^z8T+si~4#+Hx3@GHT(I(4%(0?!p{*lb?YK!yjh_fMh& zg$j3Ap>+Zs|*y<|K>RS%FTs$<1Xq%lZB#i>UO)S@1m4pKd<+n&U)OTsv_S zqp)+jAQ8bpe3fH~$2}=Dmh{A}1f?UsNtqKbnh9^UfGgCw>2zUwbQB439dd_r6Yo?` ztvuhs6>V4`?6;9bnaZ!@mg*VTmT%(h8|7G|eA&;6in>F1wqM@yN0}c0ICc8cuMp)L z3{JFj2^13+G>o>u1A)-!cx?iMQ{gwa;GYRMTyjuP5UynKg0{Q6+nAtM`r^fln)#Rh z!1PKoI2R8brqQsh++3Vj9pGzP=BZHl)faA&+kw-(AeX4H9y!aNBiG5aBNO-KDs4DA zP%dHpCcrJK&{-v1MOr!p{y5vWjU z)VaXe&$Bk(Mv_?P8zaeLgErr4R!GkDo5!Z^#0_Vh%;9bPYhzP53Rpt4W~3JVvxKR= zd9uG!o?PxnHqvJ-oQ4GL8OtA<7@Ul=K%bWu3|Memdxp^=+#t3A_wK^4p1LbOC{QbogTG7^q3c|~K`AmU}Gl@NU2OCEl9?itTtm`ce=c^e!YNi**)#Leh zO>m`p;55`{xJf)%x%($^xb{3(8jfL?sXWbV2gZB%9D|{4t(bG_?3oK}&V2s3PV3=t zwexeGZ|ir4x|I63=GHkIXIP)V^yPSS{_U+gvM)Rg1|AFuUU;Pdar<%%))ichm~RHv zdvlkj=M($Q?(Dsl&598?`{m?cdS`XMp!b1r3?|gdXd>BTf-~!s+2w6KOY;+Fda^GX z&@Q?7P#iG#3+UWSvQFFwRn<$(Pq)8-8FJJaunrc~dd6JNdwSqXIsiqhkp;?#!5<2i z)0d?~Kj3Tz2WubUS+}com-_E`Fu&M8&ZQSHHyZk-k%Ls1!oGE|x0x?MNFrh>cd788 zd9`6kt{T2$LsYHtQ93|Le)b{h)Irrd0_F1fJiCtXg^bfVUp58Fzj^X_U)wr~I35!J z-@^!Wj3&slBSi=5f1$vg^Jz#Usq)YJUz^_ zo|S+9Y<`b7(+35zq7?%*_Xx(P$1aUS(3-f9*5_4WtdMwvq6VE$t$Vkz(I@gOnek~9 zO`%(wrYdKdqVLFjWO?29{$M9j#hC7SX^HBe#I}!%%DDJ;n+3P6T#HL6K!Qq>RWf?G z2c^C~3|Ga9IkD|N)Eih4A#3Dl)NhRS^me6pF89X8hG7rhAbPbu3x9&e#=nUbJCTC@ z@ik9#V2Y-R#VOEB2z5D*f6(LH%u*6AJuldCcYfrfOX&Y~lp~>9X7}OZT}xNhk(%S( zk<#muU4=Y^g>|~pLee6ki}rGmOE zrj)~M=YI0=-N)I?Ni}I5NdtI}wOsqXEj#v1GEMYLd$#&mEEjUbUmolWamSK6Cyh$0 zLoHs26UjU`C zl`T!;S9vC2RQ#?$n4o91kecG?_N#!8OV#{3wvs{R@~pYRLX%jx_?@_o!ZLQpU0JjF zKHF_l^Mw?!Dc#7~aUo?doY9^wOOkE}&u&mwZMCYW-`IkW!K>O%=6#a1QOcyljlS+u zyLd@f3R&m9ev|fteflO+*B4b50~^zWF6>c{Tpe0EEOX(*+)1M4eByw*#+J_sIvN{1 z4pQ7gxt%bymGB~k-9OsEd$<0ngJDSl%kGosHancU{&9Z#wQ=OcGYQ&J!_C-J@|#EX zSQ;h)P$qU6)W_%Ae^H%(?mZ#NS&z@Yv#>yHM zXwMHf_Sr<(clsK!xVM?&!U|y{Jm$Hmb&KPa!wZDCkkgX7^pxc@mxA?)oINa=! zASfSMMO=O}?&D`s!{WHByxQ-#X~z9o?-zk}>QAP&>g%4`5<1H)a_IzJn zi_%qpf~+Xm@-f;}=7Xy{`M?uAiiAA|@2= zQFo(+8VRMk+&*|NP=r3jCf*{ZS>7v{E6E%|q3+WoESFP`^(}LWYo2;HiG6jTp@K)| zm8z<{?TgK~HwD0C7**Q`@;hyQ&v0$gT0z(u^za%=4+h=}c=5b3HC(ShFZSM0ici!w5 zK5}0{*hy7+&ImC#Ll-b%9s(HTFBA%Ygj!-IiyNjM`}G-SFS%4*vo7{ls9?Joxx*)s z^HWD=N=19{+Q*o)3F%YI2z*OP>xK z&q!%K>vTClX|h!TY@bMsWVb7?^Yyav-s;ryx-FY4`Z#y>yR7l2jXM3d=DE_ay7W%C zRVP!PouSRyrC<7-os=jZ zjSm~0=-w}habaV{ZZ!5-b|%@w{i(9);)8Mgt}^1H2D=|FqsnO#XX$3MtIt(puI*B6 zUQ)}0E{AX9DH$U1C0E-eHtdU6hURzsCxUm5#T_=Mh#%d8d`~?i7VTqnDQrJS0yIcMrA_SH+3*uf9}ac@qlj|(+*iYo*7s!hvRE^R$(Y6Xnx zod|K|**5R?TQVQHuVE#{9=3W<=1%pqka@Q~Z0&32P;$jBzRt=UBww3qkC5HDEZu*! za@vaBI(!S|0(>lVf3lEU=FV^bcy7d?_lvOntHJdfpDHj3I6n^z)B30G`Y2CpnHVX} z1CIF~^t?l|2ff4wRTr)ZKWN>ObdZLd=Sr_l^0O3ybCKVbK1Z<&uoB!J`tOk8;`wy_ zc-SFIk`?u9pG`5{vwwfS?`yx0dg)|kWsD<+{f3n)bcUWfV znlmErr}GcVZdkN`jXgMTn5jf3>aJFZ!ujCFMOH_MdkD`nE#5x4KE7wGnaSWC!Xy3b5u?+h0z8!k`9i~& zaIF?0i-gg>+uq$hxQG#z@PgVjO=h>aJ{zo?>guw>T(z)mbyY1{|qntNXJ#OOM)IwyX=(~c9g534YP-)a* zZde~(+!$+7-DL?)^WwYua&Q?|rm3p$tK{g%cJY|q%D*&!M#pXSV+m)%3>g|jYiL8n zo+{qj@IXwq!MFY_S=0Bsnd=})Ev#4@TIfQ^3s(TdKTLW)$b8PFE^XdbnLkMg-Dcs`H(;w67=GjSKcW_@(Vx&3uzrR3eZqcFRC-e?B*s@=lgdlJ5P zRe=|K5OgDTOIhAU1`kJHJ5NOpL_G$F-b8x4JkR@{(rjmY!|nCOosftL#?#lJ^jBv;ps4T9G|2&R>BPH7TDFC_$>0{ zfnCizLI_?g*r6I zTJJt6EIc`)UGwlrn5llMm*~~LI|{?PB}bp9;unP$#5$KBxyXP1U8ziGXmMP#FMc!K znXwPeu;5-l#}yL@vvNHjh&O3}_QhZzGKJxx+r#dC_$2NdMc|(YXqcI)P8g>;Hq;IT z<1Q7v)E1R;RV%u0?@b+S;Q$Yg03AWErW^VY-vfkL5qrW={Gf(!(R81%D zzLTn={9iL_CeBQkAC0d^%r6t>+2^wc-wltIquf25sANbOr~UsS87KN%ZfR^(xg;4D zuh0OQdZL^YUh2pAyNHw?OUO@q@V&7cec*m@zJ@gZ{`W(((v7@Ha{U+`(cdk3FHUrt zGmX>e2vDE8_WaA0r*eKEdeb$Q@jeN&LA{PM^HopS{Tz;w+$m%7r{MS;EbVzVvGlB+ z=Bo0aW>!kwX)>NW1^lho6(~s)`opCWl#j-WD8bsAs<%m(v<{s!4$p|0Ft3-=asn)k z6*00)lAr%ejL?V)=(%W5Bge9Y0(t}f^An*nkMW(nTe9-4*+LxAy#88{ksk`cpm-JT zbPN0PnSX6El>P+{niQNcO5gq1CVSAA3vS@2Lr;Ao zAe+W5!ew>S_E|j2^H;wpnBiMK#ivo5*5xF!JA#l!K5pC;?_|P8D#>Qh4DuVp2}=$> zGEbr(EjA~<&@^Ie8yh`uvupG%`9}TwXP7+~C*OUBtq;`;^2_bIl=vNg&AIKs|g&7(433di0I3gx+5-XsUR|EdFQ*ENmYpMTZO zH^sSFL}QZZGD}>qr{XzP*f4FD;o*B1?KOhrdkE+V{V`zpFuk!D;4VF1RdsnO#l^ARe0||5VDH zQTDMQWafVN4Z$dv!FR+>gqS%O59Gw0aFi*?-3jQt*)BqrL{${Urlr><_PLVdJzv(g+7ZuxH5Ajl~ymx_d2%rio0av*rQZuNj;gT1Xo;U+s`TN zNJCik*S9uT8>Jg#eD#-YO6en;=tpMOyDYw|DK!aPGEs;>+$2=fD6TQe{zA5g{&WY^ z#aq1wM(p)7j3u&3`;KhA+Mu7N)|j|6mf`60uEF_0=c`S4yDs76<$XiboZ>YkPNl>v zr=;|rzPNHK)>-b!#N)d7nyzO|QL+{%aUv;mrI*evi?rj3kEH{ye${`mzwx9g{s-3$WTxK^6mfqpEFnYzVE5pXS{n=G9pxkm|+)}_s$glD#V zJiali^<=Kxk8g?wlTz`>r%m+McdCe|pS?UL7d|V9TO!w+K8xSJI6Z@HnFB7vHUMR4 z`Dx}e!lo_F6l+^OSATl^JD)?TArRoa0OAt>*RSczx!PzB0Ln!%X{{Fa)1pC25mD=w z)s3YNxAx`4ly{QJ;zq#nqtm7l^gD^zZMJrkm-K>>9A=TQ1HyghhXDByrWwSmg4sEm zydDz;h?rs8PLvcZ(@K>1T^0JJTAQpFzl&J7{1-M zSMejN>JMprUaL7QUU^mi>n82ZeY6gG=Rfn~I@aB(xSywF^G$kbBv27I9niPJ@ptEg zAJ8q;(C;egZguPTF3^CqZh3W=s-{u8m`q;*rj*b<+*)K4u#3HDMC?9gE%Y$rV7B&H zcn0=+keV^hdZ~b-wl$B)0JKQavqCCjgBCDC-EekLEbiJbj7#nnnnZtV4LbH|K3o~+5-|3~mX=^@Z^ zB8@MLxOia$YX3ZqFF<|WbYa^gU5FB7KK9<;I36LoYa7o8zVFkd>Cd0O5P07yY0x{p zD`>;#wqX;(7E<}Be3_MuMjG_7RUZepwFqVcVA}I6J|G(KiX@K*5$XYT0I`z3Jxu@0 zyy0tCG&TbrDOYLK2pT?}>D!^Fw1U3z{7NcbrIu{_AMypN9v#`jWb2hOmG0my^%H}% zU2Sb`Vhl?Ln(LKFU6HGw%5JvwTxj&Wy2lg%>pvmA>YU*M>MURtF`&=%Ep=XKNydF& zx}Qb51IXh_E+fm@^-mo;>eTFD(Jf2&VloDK3J#2w?r)vG9Umy>G8N-I+&A4rGvjmP z4A#c@UQruL>O)#$lYJERO>l8QxMYU?n`u!ba1wBE#oM{25+<6t!)J`J-RWc( zpJCU@bK;G?=YI+X?HtX$Mj=FKyBtF*6o2 z$bj44dCjG@;wNwcyv$#mH`){R=4vw|{iSG3=(~A(c{!lhn&Hjt6_Wwf_TnsldKvAL zr!sNm+%L|+Nu^gbx?5(Uk^g=*Ok5cP2sv7gGP!rAaEVq<(Cu=COeBWQ6opmDzcJ00 z|K{XV^0_3O3t zlQrnEF{oL)YjG@iQ;Y?vy)DCEQ=7N-5=r|+M8=Zo)wLrU^if|vT9dk!uwClzdW>EE zDWs7-rz^K?c9O})KTC28ch3?1P{#h8lcAUUGckCyW2rL>SjXJxWzv#yi@&^_txt$V zEAr>Rsd;7W`h$g$vF&z7@!OWM0Gsc{a?cOd6NCF^e}x75$&@d&MLg+JH?!{2N!L-NLIlJcR8wes6t zX;59qu+vrZWe0e8gX(`tk=v#ri!dlPwu#g`N$fxIe(6HKwilEPbaMz3WhxkN+36^4 z{Fv>zpUXeqoMG2Rs|mtgwJ#%pXGt5&=tsC1b;j6Sq2f5T4K+RV!D$$z)B8r3wq0_~ zYrnFzKz%)+dz4tP{wN|tp=X(=t9|#2cL0CsnmenidrNuiV^e3~nwjg#XJt0KQ^Q|8 zVc|HyF4ugGEY7}U@`6zBmZ5U2y5m-i<8&+An~}6L_h5ErPnWcubu)0QbwqsYoE%)B z={C-;=BuY?cS#U>{tCbaZr23&bJ`cujppw0nWGf>MB6o*J3Bl7qj~Ojub{T1mm(NBsuKG%(?(E(jFyU6;lK>wi6i|dDhyEkQ{Hb6M1OKT(j zU1q%UlleGyw6zE#j<-C97>lviQI~HmX0#bRIrt||w`>+a5Ms#;ZBIW9u$#EQ>-Jg~ z*7`yNu$(5cCTE7}fW}28M@&rVE3;#J6Dka4Yv=1rU-N4AVD9oXvzKJ`q+SzYXo@h? zyLeF$__;!XJ}!f=113Dz2=aMc9t$a1_y&D58C(B-UY-IR`6I8!Im1d77c|(aA&CwH zkLrXczIRd6XW}To{@xRmVMS>~Cm)T>3a4yXye6bhDQgN-c__u815 zdvDiXl$K;l{v*(Wk;q^By9{V;uy2O*#x-n@dJwY46OD&+U_Ih8w~WW5IFi`!&>xBL zoC)ZpsmUxH|5}=Iy6>ykqf5@77z_9JM-LEO8GBfmfSa2j+>YpK>c%=4{l;8#CCvnuL0YPyrdel(Zx+*vk{sO-E=z9_p>RAymg6!vmgEh)6Csa%|s zAYPXhsl}T4NQ{jX zL1m&ewR~p7_TRTeUZ$Sy8ZCAXCjO7TkM(2g)?Rtj3`Y5CVhmupiSXaSFQI_=!G3d~~In_DO=BTQ1Dq=4i z*27IV*7p7PtUjj3zIcA>ye(gZvRJE76PdZKzJdmEiJuckJ3nAP|Z{i6%L}UbuCSH7V!2mhseYF=h z&fdCQwZy@}fsE5LgjOKCeS%pcgi&OHNyYE$qBGF|{#CV_S8}yMx-mj7`#S%18JHns zK;u7o^A;7h3RRVdWjgI!87JHa8m51?4jQ@qWmSMxwLe*^`HvukIuoxxBG}edPgR_% z2lllvifb%3H>!QF7v;aw?jq*@m`OJuw&=qrpxDtZ65=oQsrn;xC<>$XDYH1GMQclN z!KJCL2O77J?gzBbysqt_5$)9=nnz8=wc#V1fvV?eCfm1b+_ek&ZljBCYA=>Gg2Zvt zha?#E!@N^@o?VK;Y{cBs;n+2*eS&2?+tCIP_pVb4(;q6FzI8nIA#CMbJqN`Cob308e*7c5 z|9CIiCrhFs4hRjou)gU)SvoW*C08sBrn#C%9M5PPvoOQ|eon<%wP#q{mREA7*ybsJ zsaW?SuuKzqOR)U`?XyD@#)ha>xj54yYCi4)Rz={b3{O_aWXJeWj$5*uN5R4{Lz^vl z{*X{|wpNm~oo;y=g$aG@e)zC;bDBm_bFK7FYcL1r4j)Z66%7gSW$EDlK zX8z@eq%Qs=jLlnl@AqwiE0qjA^> z#$lc|)&X*Eg5gAW)3F03i^biaZBC&Qn~wiEQU5%4p5bDimVP+FcI50cAXSwdd#Q)N z?EJELasHc~K1x|!THgGM5qkZ=z}z_4cwQp5XwoH%6pphGEwsw}vb-GFH=VB;Bq}7G zmj?=2$km89dE$iKWrw^o10DD0Fezv?zZYAGIwVZV(ij!V(0|9u?hvaULkZ{33R=&|UA|&hd9-Vo=UtMG zv5OjoM$-DgJh9Zpse0UX?aTO4PRL{CTDq(DxKt=Qp1{Xqm5D8F%NH{Pif6()4`}WY zR&1Xq$jCBZoc{Hqbj9uC9@4Nrx)u@w-&?MW*le!4CYJ>W<+i8=FTuSrjz2yzs$M$| z_jLMfs@J~S`!(s%0Ji_}4ot=?joUc`-|?CXlJO~>rDJ!$snQC|V&)->qwr7q z$b2f(*7^kOiZ-1+uSyoZEk36EWnQQkmBEnbMC*aHkEMj{HXe|dtq1hw2-0V9V|16K z5nJ9>n7~L?IRp+r%5t3Fw$RonIxMC9ur$;6#+ed<25N|=!ln^qP;2d7F%-!2kmj8Z z3j5d%!w~$V@rdU8W;KK|^SL=O3_+!S;b=WE|5xcANp2uR@2dw^^PjF<*F!sG=p{J5 z38vuM=VN1aF;l2057K7F4G0946N3S$!IQLy#Z~I7va)N<3KMYv&?(0!eXi+Rniq=h zPdw3LqnAp^LhX7h`^{O-$1xbE>E|+}O?<8R=>G=vdgHA6A{%?z;Jg!rDi(xOl!RT6 z*g;4-lEdP*oSIH%Xac6tuuoROVwM+fIvkcb1pLAsx0Wx~pKk(Iyy{IsAP_zYxUvYM zdih3&k>usk+tVH2-6V8h%dw0#QmYUaq&?szHC*=6JFgj(eHzE=SI?$4vcV%USQj&f z5&mpNZ?!YS2ogf>8_%ycE<;90e`ES{C<1x+J7LV+iyezd_?i96c!#2c{s4z#!gW2Y z3pmsph2=U+r=`r>K2-%?lqLHNqB+zE1h@y29@BU3beqdBOR~;eE{e3}%)DfQj|h@$ zmo1m(B;=@NXk6p{Z;W-ZCmfrAqBgIh$T5WxWic?bQ;k70KyQI5*o+dEj)DaWeYs@P ztQl)v30$VR?&C!+ZB;a16lsSPf$dLn>@A~|-kE}2F2Gynmea!h7#Zx2Cu&>whc=36 z(nt7zJaW1*n|)kVzH_mZr0j=#?P%BhO8hc^VkAa*)vJ<^i8}9+&YrzQ^tP{8Beg?Q z8`+twt?M?mTst_fXXfmA)SHksz=8`9lla(YX5?>lIB)6no{}{tn`%nBcRB_|UVgjr z2%Vu6fcdZ-i$z`QmGn{9dy#1UMn(QgpaPo_rwpj;XmUl1OMHI(h>t&|y4va-moFj23+gF=_gnTAcOL)hl@?&mE0U&97#Rz7 zb`8(W*>MHBFF=d=3|y62<9fUf+`Vy&5Q z!2!nfH6~UyZi(V6$W9W#!iv_l-i;!Oi$>lpam)qAqVjy@4^V;e}=@evW>bYp36h5+3ZcCN~kONw3lw^V`rF>tWSI~vGOH&;}l}QuHNlI6kCNvT5U=4Jg%%KC0={|hs z@?w=e3fn$3{)u-rYJtBVap#7-o!jEcbZ7x2$t=eV=BSWl@V!s9{NerzlRQgdKltQO z?4Wm%)VR$AT#?z>LqKZeNsUwvqn6}?Qs`+&C;O-JCP8hp&JYjG zyHQ@raBK4D(MZhyJmH0etf>duc(aUVUNHfhcfpY^?{nungHx_3yU3TEH3yg@Nt4!! zO8^Omc3H|cPkbP%s(M;aw#BbQ-(7CNbESeaiXEALqx3+EPVrA#MBI zpkid%^Q}=bqgvmPB?En9B;>WKL%b@+Zfiw!-Z3y{7k(Ze*<@HwR2|nI7}xXFV1Lt} zil@WQx-~pH%{LJJjShLLdy|4qpOB#%r=3CH%<%{FWpm>Ff{S~V?LVyKtRToh6JWN? zYUiLET_if(+swJo6m5Li+Wo^GTL6QPHg?xt@XzU?f#xhwap+if4NX+*|Wk_tz&p$p5>8CL_cSYhCJuH*o)6?4Kcc5rg?X&?fl>u!o zjhm*~cTHsgfOTw6q@gUl@_#HGLG({rzDQrPbS)kN6$0cPVXgnU>-T^XTM_4N3ExSLoUyogmZektnQa)n+Q8k+e*_VG5QU0z^=AE0~h0~)a~8be>4&)K^4 zu`96km1D#Qw@WH}kLi*Lt|lFXdf&Hr0jHP;hoXJ~{@WJ(AeiG3VWu7j}Bftp2V zl4zd6rFxSYycu<x=G>86ZG&Y%SG~h9B!wBAfaEJa`%+aflo(zi<66e3uI=DF z_6+zEPfPPHiJhVtf35ssK@IxMTJf&&GGg7L0xG(4(9N+LLBJP2JZR3ckS`<%g4Q$R zNBp;c{I=_GPM{-{kzwtG*jvTkF+N&+&G9HBb7Roz6ywM2a z4rgO2s+EXczW-@{-dk<4Jf)N?RmB-Af{j0AF(173s7$G|;)iUh!ople(KRM70q+di z;=NK2MQiq++eRF0S(|l1^Z(x}^mp5!%U)=Vnl7ESoM!1dvxn_whuiZ-Nar1B9v>bi zn>u@vy>f$~rFK3T9RHJJmfI|yT`fP;)$in&!T8_37{yA9Fs=c#sCRwcsO}zocZ0=C zH}>fNQf$H7N-4AJ(D!#Q^w{;qA{H2_%n{mlu4FC-Mkma+n zZN4J6xw1HL;A>~APXKaQ7+tys=C3thDf(fGZrfh98yxSOtF2F)K~bIerI@bf$&@bkA%Px*(i@ohMHaGi$_G_dXdf_57IS^L?-rAxRehoO=@Gv_p( z{4hvLhH3k^gecdlv$S*9g=V`Tz1lsk1sCW%tB{eVlmN*#GIbod%DAqP)`7*Y2v)k z)S1XMZk$RIR)SLCwfOlAh!depX=r)81?^(zF(!=#QbA`c@)Ev8*&S z!U&EXPLY(DjjCd`oBaPp#XtgU1cr?J_mFYAEV zt0BlhI36>8UGQ3eJ7^>a9D&iS{3vKg%NA@Mdh3-O6uWHlc6O3NzZ4W$xVU8X{US-_ zSr^vW?#q2rwENGJRC;jtyEc{+tA_abrkRCh8pvUD0KWWkLLjx0H_I=J7epP{i|ajr<{D3O#}d zW$(J>Y86-G>_nQPi%)7ArZZ>RN+{%OUZ-yTWgL=?Qt{u>c|9ieKNFOQM|+jf(})bY znn7Bjhg|w+&X4CDbcny*Y8K!Ber8kuqjEmRWY*?2b?a)}Q2L1)`5ZG3+g@}^2-Si7 z*%qU`?$2zp88Lr4k9AV4zGPm2>zgF@zxrx{>6yfZiH)_i6>wTZy&9k7tl+I?_Mk~Uc!=l4k?_D#H79ENwqWc zD`#cA#xsD(C$C5m=5nt-u(6w^niEO&me^NP-o`})E$WSj_U@Tz%mMaggM(rYm&`6} zZSbu|f>&m65Buyb9TU=g5_|SFF{EIZ!0NY*K@j;*lvN`h$LoF@bh975=ij*7zI<)TQ;B~cyMrB-EBZ4kB4gpH z65q{vZt|Deo};&gVK&v{A0Y{HgyuFonlJLur<@MJv9?2fqjzW+$2v4i!Hl?DYZdeu zJe@n#17ej+m{lf-Ei7X+uPqcaKJ7TL{4Fz?4g;92QOvc@EjynT|z9;(!ZqzfL2E<+W}uRq560GA;p!kwdiyP+#~2k zc+9im<5b|ogtw7T?*H!NgJIxJNc7#vJ@cgkq5ZkE0Pwr_w(&cn2On6!obuJ1g7Jrp zjQjfrd&`Y}ccBH&D(V}*@J-c|0oBN1$`ep82C*<51E;diqRJ~F9MJKot1rvyt9Lwf zrf?<{jxw&nm=#Lim4lDht82Uri!@|-TyEhp!}UPZHwHPh9;HTlbNDYs#2I_l7_D%v zKCIvN+k6n*M3>5YPls*J9|pTTHU7!yJ@i2hB}G_oDf&D}gePB5_xNZo0w+^FxLv)= zP>J`a1#pI$|D|j$P>%o_!svEStU=_DEAI-kTtRC?kRxEn5Txn@Riaz;4{WGsI!msV(GPT2yp& zbR$VJeO21psI;knHL1?^kLAL%H^6tPz)&ZGByU&|&pNQNA`}fHD)E+bnSt%f+yp-7 z@8*oi$_)umQ6Eo!=Rh3gT$;`*du37-EMQQva`4eS`bfPVXdn>$pn#v^5wfm&gu)2o zzs5{N{yFv1qmMw&M!J9={Y#y!x~>1?iCgYbC5K*zKa%@Br3KJ7>yXbdZ%o;GSld7J zNYaO1AP>9eFTxju!W69B=RY4TA4bi8P{fUX%s<5aE|U86ckOL6^(4u9$f|E{TZbU$ z0t28nrm=k+H3KFWnq>d#)cZK< zkPo#EwGKGa3WCt|zq|lN?3$Mo&I!NE2jG9`Sb)se8pLGF_An6T$M@|jz?_J1Ft55y zs}Xe(;D{pT9-wa#ooxfMz7*AkdVjmpPHzdOW0ZG7FI(TefBphxXQ&pgfU1WnF5++b z)vn+>A&Aww|9mLhvDMZ=0clwT*;lpCd=m9l5R)c=%D|rL8h&{rOA|15`_T`fKxb6q6|kyjgO>62>?Y3ydqCXb$+k~jIi8VQbOZ$X;8iG{kHBPZD>iBb|dT~^GNq=ka8W>g1xlj3B4Ek9GfUJ%nDQ08LsER}Zse=A_FOG!O z?y9~mT5iS4lo!O~8;!XaH*|7FSL!dtbcB|L9!YqxD#f~+plKzk#u3%jX}@NX1gENO zkqa+F_NQzRmWAeNvZ{zE063%s;I^i!Y*>-*8jvnU!L5ZDY}?#+eXoBiw8>Tw<-9oujXS3Q3>=7i+H}JJb@W zXY`2A@qnP>hh92O#9%jqrE{M|s*m}9XibX#>mA}=Ex>|fwgFL1PMgpz|Fs>&u(YIf zrIs$r3gn=<5sNmT!K-%srP(wqq5E~EjtHL379?27s5d#So71YXmx=o`*}rd&!};8S zT@w=y$>$U_z*O$r_0wCIX7s9jST~EEq9HdmHSswxgVsD-iRXCem7p2K73%e> zh2anS5Eu)p!bdM|h8C2i_&l`cMP-bLG-SNo(!{F~zz34{4%s0hzTv2}=W&go2!9Kr zign2BFy1_w1<-@w3#g~fbc_}~4S>wLavBEj#p`jVG-d`)n4=%D!!@i7n|4RHOmQDH zUmY>bQR#ufoOHpDf6&^fpj>MaC=jD66bAGV*6oz*T&E=WOd%4%=1_2MT)rD1h|)v= zeLbvdyK``Xe@}2fms6z|Zs}ZdEB6Yr(TkJ6x&Rx922L6`oV1ikjCLCzg4ErjobnTt z@CR)@+H_mP@`br{d&XL1-@V8^X0o#rs>-fgMEdosh;>zVr``8_PyMR8U@$SOQS^eL z$)#Fi9Z(#~QtoH2^>m3A$^?XLL$&=H;Wc(i@dN-O(vf=|gTt8vz*Cs{U>X(kb}(85>+&+*FE?fmqR&Yec#jndL5C|ZxFQ}q&Bm-LHo>=@QGv7>P%97* zVMI0uaWsBzR{9a!Ls&-@U{for878yLTjo*16D-JBb| zZ0;O`iwot$4|kQ_+lm6NdX-;f_A*4l0*|feH|QRv=r_i%o@2~@5TIJcAWgd_In%Z= z%Ja`hVeUy}e7h~c<1Re3+%?=&>$NjhAN?18_6nZk)y?6zwPej$2FzNRb zQU4Lh4ekGUN9Oky!sm}UwqAMG(%Ot5vPxU&Rd&GbB5;yH8!mO-(5K_+9AdpL`F;Ft z`hkBBUo#?UI|khUEZz1fgU2K_?FUzGtEb)4V($?w_B$M-n_I5lIW`Gjs!4ymb++xo zHT1zRW~D;S{FmdnsF&U|F5*(od$Bc$M>w!sZN)dMK-c?fNPcuXUtN4k;It$Elbavd zzr)9V@DK4j3Lorc_7qSHsY=VkeMxuuiRfbCa-)rO-4r+m|2 z_gaj4uNOpRJOR7x^XG}hqRm7+`q*v40&uVQzHtlf=G&$3w%grr>ed=Inrk7x;yDh3_t&kgJOMvD^jZLrcK1hOjb2t@WP9wSF5H z-})0=69(wh0<=J^_+kTod_8x$|zFE@o#h#(KC9v1kT54l@}TSS@ygS$C*Z)W@zF=!%p^yoA_oMq&Z zN4WvpHQpuIX`O8%udJf<{MO0`P+z%X=IYK#HkJ|8I@ z&ae>|zl)`9U-$K+bV2Se1X&k+Z2!cm3hn**3x4H>ZPyQ~*5fI|mjVP>ybN{mH8N;s zWXsB+)Z;H;m*gV%9Iu+abN1`^%Lwv=b3(T##W#kn?EW{}!wdo-w|9MTE5J!dT{LiQ z&jHELtCPXTJTdQZT3wq?4De;S0uXYowI1SentMX0$4sI4g51~+$mS}fG9S8qRTpk6 zMUAcpL;mFCwTil18px|@QJEtHqC}bY3l`zaj#J~d)cci+Ag>fCK=Sm8wzsa+9|G@& zN*F;6CFH|)sl8%7=lk2UxK*QR8ntvG?*ByhW)QcY@;UB%98K#W9?E|O97m!;ntc-= z{MJg!A3z#L15?t^VusM{RoLsRIwgP1af^#}Rgz;n^G)7;ozxt9LEMwuoG3lcMFaeH?~V}J9>_-@Gejo$j-i}SC$%it$347 z!9cAiCgAq@*1C;ISj7p*ZI}=}$GW)Ct9CB1x*xs#6i-@nFU9_S)-=yz+DPSd$Q2J~ z`HWj0xFN7<>x-6>G5dPHnClD|yy_C(OsMrV?e;l8kW((?WA4YB@qIfy{!{r3JpM?& zOJO)gUMjar>fBIO!9l|)zh@cf1ckX{cQ`KE9Dwix*#dX#oK{*eF9miNqL07tGhKr9 zOW|8B%Z+cq^NIRB!#ucW&D*`PeiEp%{wR9D-~bh`Z@ms^VR<*c+BhD<2!ePV2X7XHagz|NaEZ_1{g6`5wA%dQrCSBt!%{ zTN5coMWjmJab%Rj`a+A;ow9Ha4Ia4)r&5BC{=;9%7Q~tpdNxoRuw9XX3#iTq3J(g5 z2_wh?vDywK@O9TO1Z+t-J-gn>7_sKCrt+nu{|d2W)fTw9dkonDYt`Mhg++c3%E$#J zTk1a}n?yiCe6DQ9BOme9&l7ZlURz-KS^0(eNGN<CmBJue&_@IdIY&|vi8!r6H0Dz8*(s%A2Jx_ z^>)FSCdw4gRA_F4`m2uAMGlrx|Ag@&r1y~Mk3c(4<$FrrrfKPZpHNd&@4aV40Hxmn zRSEyOMM?u~6Jqk=7hPc~5?yt8rGcYP@WKM9Gz1wmM-wk5J-_@4{@1@BMk@t6as8(E z25vDPF^t51n$IaPu9=48qOiR21U#snvWFZYyC7~5o1`(@AwL&sQqWHSgjcqvUFg^R zS}VrHWg0F-^?iF#F0)?e5^bG+K0o>i@?3mEE5`&XQvD*##|*z2P&wShcFfChto2~3 zSxgg9#F&Ksh(rGX@rk?ut3u6(zSp{w%+fYl=?^KKjMiL0IJgekUsY0t6R&^n_3%v% zMbruOu`CiH?8g-ols7aBOGyivJ3DT4`1m7!QvA)UX48);+xv#;! zME^T?w;E`q5~_8$@1N^ox#PCxtxw(ZnjJ_Av=V*NRB;a&ZZd~k3#l(2+j1E?F3%TV zOS@XR=G)(6+n(JQbh-FiM&Z)OheZO3${Q#dM`Yz2CA#kppnrGxisxoeNILR-ziG(Z zR7J@x(j{nmF423#zjDMD$;g;86~_BoEIJ0rS`yg5R)qxy_f41(_xnvTn_18H%gfC9# zE;`c5vu6w$3h@8+OOt;8vF`@=lfpLsd9cV(9gZam2PXj0^mI{>TyoRn(PNs^+v#Js z$5@1SC$}sgjj+_h{#0l@teXB|EM?|T5G%Ftefh#Ev)({?kN*N`iCy`YzYFV{cb*Ia z+hJ@y%%Y4{-lWO9b1(#heyH2>V5QG#2<#g>MZj4Q+m6^mDC>G z+Eve~G*)+ekA;LkyS0LUD{(tuT11+eRaH6TN>nqR320_>2M}Ez+Qrr{UtNbVD*qN8 zGF_>;p9}Zq(3O)rI!ql`#lDFnwa5lVJolcHNvyu9t%}Fl_`ccPe=6ZCt}_B>x}fob z(0w~01Q(D7Fw;MG6G$f;la;aPvrqhac0ygO{U2$W6{4*3pKEHa15Sv0E4XU{?Tif* z<1tCn4{pshmv#nPy7sIb^@O#Py5Ly$xndE7=Ku+CfljvGEnjkr3=DelwA|RPhl8p65cCSbHJb&-#3yT2e zcZ0$Em4HYg{F9xbgRZU}e+TXFgS;YGn3}T@m#k2g>pGrV6>7Yo{$8S16hoXDTfFol zCB=r5EXwGd~ zUEJ~FK?gGn@9&9ubZQ#Ne*b5Q0b7N)6SI{SLdBeF)_FxT&Xw8m_UIIRby#SaxXDL4)a*_}S`b&|aaAYf%iebrRZROrBBj)}+-kqI zMeS!^V5*nD*+kxfsVEHgdf&K<&Zq?(4f)?k!=Ptg97#5XEcpXNk0GYSkr!MPllA}6 z_0?fjb)4sFZXHNSC4@Al)G--6+y1b(Hjwhy2z- zA0OY}8B#F@LKiKJl~i zKM?P1s^p-L+rwd49uDyCFTC51? z4FagSk=;@qe%w7PfX;%8G^+c2**}CJK-0{mH%X|R-ts0hO*D03GMnvTyi495+p;8~ zy^N)AQkyp5=>`g$tMfMQXi|XVL-=H{WI(7IcfSUlP}Gb@L4%=gy9lbZzc;L>x3^bP zo~r0uD?1EwgnQrZyc2`oz&(a#7aJk1$a%{SgXzs14V!C`lF!7k{E*b1@{~3hOdH!O zUe}P`6cw*xDw~}EyK3`oL;vL2JaN22rngo~t~j-o_{b9udbB@{2PKs=jt`o+0WSE& zfzI!mEkz(8NCAG5qOaYqPF0!x(w9>^a7;)0+a4uI^<{#*=#CrW7@0`&(r4T$z-KQ- zvFE{8=wbH^bl9JUN6Ks@={NKL*D`)E1hFGoidB8B$gTd=PH-iQ%-(=FsVwU76|BQS zn@Azo>&DSI6sRi-1E)oW7krQ0ah?o}u4ZBv4e0EXqWI&bQo)Dv$@WC6AWh-aT4!3% zFyG+vR3JtdB3#G)w!fzqdpRMzHJeU4{qCsz_|kRo0co>?zb7TQYA3B(b5dN{V(LXm znttR1rm;9xei2SYhMm~%7$xv@B>#FkmspNs0Ky_5=$EoTnf`RSU*_Rw5}7S@8Ao!qSs-88K1*v*&~G7BdCANBUphACZk}eR=ss$mLD83ty~UBO@Ni3^2sm zR#uqND_f8P;wt*0@;&_v{uZvaL;eDYXj#|jAzHBO*}r!kd+`1O+7cGNPThPI@pE!% zJCdy_nv%dls{LfrJZd=g=IKm^le#d|AP!=g00v75j`c`~)R$DdA;BHtRhy7r5;J+^xBl@8Z)lYOJoX^r)TvZ9fxz5BJuJR!zgRl zDK2?SB?wKYs2UCfGrGUG$!oZoqF0dq60u&kQ7WPBh*5rI0mZZHYA}hlXAw=CLoY&o zg2k6^&>kHx@$4T?h7J$11|Au+pvcoErshT)Bh)O^eg_3$7R_K5F}6xC>S4kg5}}`t z>Z3tUyHx`fV-0%rHkt+1Yk2J8+xy%XnkYIX&2kGBqdgeHaUI(0=ibT!e10AK(@aA07umY3kv!_?{_Ai zxA-6#`U$5EgFXU(?iLi;A-UHjDlYQe>h|Z4QK^x;t4)|tX!b_BXYZ4S5u>MQS%jqg zl6lFQ!K*40`4SqkRAh#^mll#`J4hKVqdG%lxS8Y$&hrb#h3$KJy^Q(t6^EvTeT`x} z!91tPMS=jhFXtoFL~8%sj!gVJWielMJ2&}yK)i$es^TQ8F8spEI`s3e0#O6FZU%krfB?~LYvLU-1V236|^b~3C{8`Ey zY!StmFP}fa^{0~{h=03L2XELDN3AhIs82WUN34R9+)2#0Sbb-1PszKT`<;b(L$tQR z!OsK@x66deS%oH;^76p7LntPfLiVHzQJV~G<>1dP1Me>z8aN&;<9&d+JtE;CVZuCZj$CMw3VUT8#` zDU)rZNv`a)!iAklACZRrFm2d`ilyvwq`0-(CrJcf1h5?yDl;o;GDs-mH!y%}mntDv zDh@AXFlhLZ=0z?49NFOxMvv%cT~B1wkbXTc8sr81IXJ{=5X%#JjgVuZhaICa{d`M4 zoM#1*ObX8QF^Re@FV+dvMTx#A+)sqA+QB}cbGFPsCF516A{IaE_T89yBAS=rTEz0$b^JNu5r21ih1$Q(wz(YE})U)UaVqrsZSzzkiAqaEbW0 zn}G62JZ3t0IAB#+Hh6wRZRFBxr7%uYi@wO@w+#G#oi!&BLm_p%Xtyx-0<9l^LzCMN z8+s$pOVzc5FpbO*ICwz<)5K1`^E-yZ2oM5S?nw5|f^Xfju=eF(rck9!hhuKez*^_m zYw{hg0H4mS3^@WFb;f{-X|TsRN72GGZ$uCm31+dw`t+)rIvz0+(NRJcC)Yd zm(3Q`&zKxt>}!lA_siRsujjs6R|(K-k;qN~)HP`;<*~o~$&S{IOp*UaXV#48LQHK0 z{T)U>gc`1^k|6gk*4I2ac0*kBq6)xmLD`Zr?t9s%@vMHA$>-m}ZSC(RyB&VMvT}3R z_4?eahgf2D3tjvr$2N@Ysi&4nU`9BL#B4{qr7QmCDQywkg9S1r#;Hf6R+}9 z;j-RhS}Zz#uN<&77dp54%m=PKLGjy4;uZ_0R8Qbim(W@;e`}s`V$BJm^j6BZOB8Qk z>9i9M1zo9sEt?gyT%8WjVO~jXq#b#(Tp%&uxW! z=9Gm87v1%Z-ASe`sxG`5;oMj>vVwIjF{{1TKEKgeT#1{%t_FAlztlP|@EX!Ue~5LIKaS<8|2}#UA&>lNEzp7?M-2+8c|HkLPstshifHf1BRhJPQ|oQX|FI_kZ*W`w=aoa+pyVBn?f@>>|EMMS`CL{ll%sA%qNQ!>e4MIZe` z^#JvO+M#CYulUgLcsu-5@Bo(H3_ZsMV`;uX6|D&A|Cv0BpRfb%l$0p9EG=Re=?+fv zvSFSg`wsd`AgKLMHJJtViJxi`( zNFHOC_t+V-!tVOlYnjM^J8bpgk2XQ47Rw(qKbHJGbIB9B+|yj3=6CJ`SCs&cPQg2U ze6->~6-R;H&rD3#F3KVkwkPR{GLtX6M9qXWD1xe# zhUs&)!Ht9eL;i>Zl?J+6FcgG}a4akgMr6ZI$|4oX58@RCl-qB_3|)NSZyszJ*$_iQ zE+{{2BJUr#%hNNgIPpR1;&^pK(gU`RB0_xI!pW`^%i2|ysIk-g&NBdOf`2<=5MiL9 z==EFgtK@lsV*_9O7;h%!9A&|MSx5vZ+c9w%f@N$*Z%p72@pHKz=2dx(hOX+XK4rF) zng}HkY*&cB6x%G@c{hG9X&wtoruYMKJfmLXfRVG@8t!kSLbS-o9I%x(=44vXhanr) z<-n^fz)wBFj!TJ;pe6+O5!*G(bnb6vc4;k%VL=pPzjXpNJ!!0?7MoMx5ahpvHS!@< zAAgHomN?$c;oim}Lf+1OS%_DuBqJ5lE~>k&7|pGN4c*E4t$jq`d5CjPMP47Et`MI7 zYEaip@HP7Chy$H~(6GFnW%k~NhS>$L7ae@d7oaHs)K|_T3$pT)JSf+4#_8vC|RuZBN;_So%K$3VjxE z;R^V3tLtSXxW^FFhM+hLHy-vvK`ic7VO)%blAzRcM3zfRKj${GtYQ?vODIqjmX&}+ z{2Yk^yb{^1uj1KCC`SG3ff#C_Hwyz1C;syOqe%QL%E7J@n7y#XcH4)=9R&&k zif@g=zH48By~#Wuc_uV{7%jKk82Da^1w@uH+~VQxzEHR9&L}&!8S@bkGl%zH>A3?*!;AN<+TGR zRKq@aX2|pK^BxU6*H@Edi)VM~FZ|$JeI`!1rR)BAC?LwK__$46EZluhhvh&#;^4Ta z{gnF2y=y;x`a7*2YIzNTzC_R%HiV|Gy^!?k(UH5l0D_@nfRnl7>x(a!77xRY2E@O; z=(sX4!hS(OynT~I@bcOv>w6VLJ!yNdoyJhFleR;eMu0O{!HsuNfYz`z=-rCtcwvbt zZ!#<&+%SET8Y~HO15rAw{K+!K%~PA76fH`Ga3y)6z==>xkaQv$zugzMssmEK5L0 z$>OZMT*3z&R8zdZBMyhdFw>H2UDR1~&{IN!kXmCHF- zy;fJ@3k3KGoD`iG1Gms#v33HJlN2nzn3KE5%2Q#`p$j ztT5zW-Byf39Uf9gMf)y8K6@ZZpB%KeprW{_TT6Ui%3N`W!$Am5Dvs^%?n-g!PBwHP z;b!ExrE=Yb{7q$EC^e@D?4DPs>-9-f^0N3{x6c_#K#h0~HEp$_4JuPQI z>puj^w||s31vz?w@a2?8UDzW`QQ{@w0zBUcMnl%zD`go-hEPg`7?W*@LTu;|!-!?q zL5!`o{-;aGOu>?NdauTx(czZ@Dd%Me#pAqYDyP(dVJH2=t8U*h^+(Y{T*BrhWg}B@3WLx zr4(TVd|kckY6T+2H%_>OP(_I7r7oe!#>evNYC!OqfIK}De7A)c{9XNwJYH2c$D;-P zgA>WTJ8$rNwb^&~7FBk?Z15IF##4tUV-&}vZO@$iutlzcyNm8>a?l>`$hILsI#AU3iFDcvgq1(?!jZnKwO%w-nk(5C&B27Duu>44IG*$zp8`B317dDnv71 zXe{s7(Q}eeb2fT#JC@>3w)?FUPudqa8_m`1qt?W_PSf{B!0l5L)Og_h`yZzRGu|!V zw5B`{>Ikc6DhZo*-e8=IcpJ%or5L02^E;6^d0FR{&*&Bw@vT{uEsSl+1gy*4XpEcz znv9B<&da0U=trk~MEKcmW39Ncdat>0vhidL{QKc+yVr+neir(BTRxHl3c@2hyOR#d z-(MfRt{a*0E;r#~(5b)KF=QLe1g@vXpw;=Lag#E!1Cn-MIM`4Kk%JtDmO;RBdE=(~ zO(J#{OfjX}OhlV9#LicY`|d#@QEu^ye{PRuy(;`U+1Ra=&!Q{Vgnak=lqT{v864oT zH#=UYYxof(}EgMepqWLufP z-{%`ze6{cM(~URBCMlCf6C1HB0(W;Mjax&HZ(R!FnbxFHcsW!;4ncVRPrDD4IA5}{ zqtR12zk(s@N+HSINq?Z6n*?K+X4SeOLkR+1b{Yi}g;ZrTF0E+=!S5mhdx!2)V|XNtXD^f~(#JH=ZD`;b!z6RC&%|;F>NN7<~yj>Ia8G)Ed7d_q;Fo{IEBd z@+4GA)McO$nenvv>gC1CX>>Rpj7r^}0Pt$oL41N52{>M~4Nn4fb6P3+86$hwJx z?R$XQJ`JwlQ)Ok>50*pCG+_P9bNbb)?|k2_aq))lyCXhl^tvCzPPpTdhToH|)00J3 z7Sb!K9k_W)`qI*>*sA+1;`ZJ*jD>u*ag&|dlTO%_jT#}6TDZG~?6o)Cn^s@j8&2DF zklT_VGj=n2;p=dX(kpr6iH|(u+LrgQjj(35ZpWzYwjDO;n$?lgs$_>O=Xn!Oi>GRK z&$^nVlN{BE-EX^5xZO_g?00q`N?JNKA~$v``?-aYgKbR-iJlR~UACSBy$8jKjJ8X@ zFx5qawSjj87klj+lMSi?FS_!OyHDK*_3lt>X%+_12I4e5*O2tu;=9m6rEII1il_gA zm({U~U5y^cBuwkUt3c-;hWRH%NAbfuG{jc*KBjq+3D|mGkSm?7M)QSVoBHI}rlvlO zoBDK;xsNC>%y`9h-tl9PKuTtQl&vUG*|ZoDH820w3z!q!@#CJe7`Kt)y|`|IQ>D>` zGTQ z<8XV^YT=My`6$0V>vm_E>Fq;O-JB-wSQT8d)dTZq7jVGjl9nWI9D944H;o@x=d3h# zxukFzel!a~Pv4($zq;-C#*2h%dMls&^*nmPkSd&^I}AGM@L!Hi6kfth%L$~-DS{Y5 zMtnVgWvee2WsO@bi!P{1xlW>b=raD%tSSrANfW}Ds;jL;7{ez?bz&r$u4zBDtBTiR zw3j8Z_aj-3cy(dMCm)_1-?lC0@&@d(lDskcjb_gx#6;iPu9Fom^}@BmUhijlzG9rd z)Jn57S{bG#q)v^zYbU755*u@cP9P<#Rosjq;nl%;Y~>WpvF-t9C8n|b63bz#tKl?? zodath{D>NJ+HW>tBfQHqit+u*BZp6nwm1)=0G}h1T2BMaO}yz{;86qa*0AE#zquMw zrxd^7S;5vPmy#yT=4565DdYUb5!UBF5#_m?`9v|>^#E0$m9?^iEUi#8jn|^j(;h6v zshH>PwM>;z#~)u^ed_AH7(iWpjO=D9wQOFpiz_=)r_*>gz!dFH%+l@DWf!-cxRJ_k zb=XM0Hc&V{J6SZ#^F}<{aMovrd99=KN8R_GTWLKSIEF3b)2416pFP>FxqPDsD)MY3dn#29& z;2zYozOX_}Wk^5BQ&p|s&9@gy$*hdB{q)p!EU>1hBJtc3rd#gV>qI_=o+dSLoNw_* zG^K7kcL@i<%#vg)MyWz)qb0k(+=ddwrB&amE$_+5L0O?@rZpe`$yZlJ0xo&Vz^=2l z_Z=K*oc3#149dnNN?9MPQMU21N7l8fT&9Dif4myWrq1- zLK^>oKshFsVIr=ji;Rd;ilzGPFWJ9lnep@+H`&)n*7TjjSU2ZeGvu!_g_%jfESGkL z;vb{EDe~&9K8C*t#3fPA@qv_*L z=mQk~Q4+()j~lx4ii$3mvc)adS`L41VAg)Vgy}u(Wc|wG`9t$ttUW9Nj*Ss8tz(^6etPy+>HUkX>01G%vPwak+$_~x|~l0RrGUp4**Fx3@)PsNtAWVhR_!6 zLYw$wm<){ODG|@>)!%t^lJx?u!M4_j5q<8MwAtE(9jNk(>5BsCl9MtH!=uMYk0fo4 z#}V?1im_JyIn*7`x|x~zIuTxSp8a{HFFUhCd}X)fA6?k*5KZXdTIQUMNB|~Ng(mG? zqbEju5v}2R<|$g%V}n9*gEcf%>>+cbU4T;fVS3$@zF2`Z_B@df;^zfWu+TF#zOIXYh6w|wxxb==PK$JiZT^fsD z-_ynrj1BvD>>C45Wm>z)ygCD`C0=D#d~YQZUvLt*{va4j``PUHVM52m=Z;r{|K`bHZ>Ld+SJv^NdNmiFESr(}j4ZDKP}gSB;wm+cC~&v}@EiJiBB1sjE_h z;oIINaEXE6f`Go#HsiKUo{wat2^09>ex84E>>VTET%4TVAO_1OFj`>1rG$oB3EW%1 z`ZR+s=&?lztzn`j1MW3yKK*OVW+en~^`(+KI2Pp#;%_{eHC?WH)+*^=VV6^d5?ePO@yr(~mLT5Vak73-rz?tJQV!DV)- z3^}f8$PS6Q*wm?v1->RwK2K&yINUpbe|1^oR;_mXpe$E*w9x_R>wo@IqgMRq&=o(W zb)N2DRm@qI%br^R(tVJtH7j@if~$Pntt)HxN}WGa4-+g#lA9gm zJc+10VQZSXvBk$YyLO7N$M9@7;_-2%S2yI!xl91HF;x?i;2kIz@J{eNE_8vRhZDwA zC;^2<{-X#qEy;szA53_L*c}VJ;AXQ45`{`;!N-^4zE{dj93}+%rj*DhiGwfRE!8zm zs4t@7MN95sNMm)@cFXqMg9SYP`kem84eod?7wkd4ZQ|9d0$rI#NC|}(TIrl%WDYE9Li(e)+gCOF&-isq_cKbf7l?788uAD6( z7)e0_Hg}mWF*R5&K&H);E{jp<*W}UvoqP-jm^`2Cy5Sd@^}Rcv?8elJQ*k9$Kb~KD zn=HEU>?6e+^AaB$*N8w`AX5RAT2li|cA>OAG)@`8UlU~gV}i1;btsVYR@5fL#TDKXh4$5xtfeIAHq*)El(J)0=s;&v&Hrp=WN zjGF-$f1-yb-@hi*<;Z+KttdO>jz*LU&eyu$#Kr-4)zM{uRTh;SVrC8lWdlO1$HHEH zyJ5+B2fKdd#qT<01qWi?7C|IsX<@gx0%SXgl5Ik=p57TaX^Q_lJnlS$MkkpUjzF#b zCn*(M3yqu}3F%93@&-ls)ZdBs%Zv6sTP|~8U$5(Zc04h%-kKe2A*3H+vwGoj@tz#h zMzpVwQg%pVP7Y1d*Rn1uug^(@WRq@`f=~eG7T4m-cZ||zL(6_p0e!KfY=j7gbgPpZ zF(c!%u7cVX=78PTNL1 z1d7j0HH&8+_1`pU_EFzu>38)7RBif7q5{y2cLFH;XuseQI%NHSKqW+`^)1_I`JGdZ z)4W7-C9)%1R<*3jjuc%nkujDUp<-PI!vy*kdB5e?E80;#J|-OX@0b{R{HWgeDKQN= zO^SH=qe1l74%wGciBiDVFL>hZHiFQ3>OaxtM)sT#O6=qIl9z4sA00{B70`FY3H@-O zz5D67D=d_o3{iH8$*zELAl)6wW=QhIEA5qd5_OWVDFM=#)Npr_yDu0tL16;*T${(} z_6OewLD2iQzbz*mwVcCp#xH@$E4UcFDq<;}?KOdPyDnW83l9Z6GtRr)^mZs4W~f}Q zw%pKpM}hRsJ+6)E@;6L3H+`qH%h8-P$6y!AxYOq-zag8^#@u67E`)f_+{wRJI4kSf z*LRNCP!{HCx`13PXI^C&oz|gc91sW6B3Y zc-Y|JfXkZSn=6??u;3bO%Bk^lP4E7;rbx63EE!lpKSFHrUjE57ySz~pL*xBT|1$oW zApY`@<+5Txl{6e}(2!Q=d-L0(ZVM=@@u(7U%5nN)mrVKxMHprP-q-)TSJxm1`fUhK z&BnPit8b#Ai7xGJ_Nmayh!TVBFW5fez@gC4iVhl- zRM&Ifpl0W+Tf_eUzkuM0@^SUYqB3x08QnF`OoDJINVfVI(!6f;D1^fPT)^Y`-?%$R z2tX;_ELi*0Ubqk>PZK>-Tv8MOse>LIg5ahG9mYRF0AY3f<6ekbsBk2pt8F8{-@xmV zaEJoO?8!6JZ26CAwgGNN^Un5yZXsANG6fL~5Q^OfpkD;nCUI^br0MuS2m(Q{JNeks z{NsAaTL34||9z#X_znj!YBevhrJcsJjx51{B>*hYU)DILv~2`FNIai>J^w*zs_rft zw6@H;_{9tWX4c++7FZk)ujLEKD2sRQ0gMh&mDcxj^=Fdz^D9WBL8ahc(f^(I(+qr! zPzHv;6Xyz@@sg*QG%JZ--p!=YR9cqMX$Hb0hZM*xB;k|^6(elZ>d!IAr=n%efxw-F zq{nh}%8gWl?i_LM?d%Sr{Mutb7SNrMJbaYel_P!*y4s`DO>-6%Is8XKKqTg34)NDr z@S@HUs-As?8DJ^-(&&xQ>4V{fIs^p-=O*Vc)WU+GpqBq?K1zuxHguJxW0!Q5U7U5% z3G^kv>?3EQjh4QbBk&=cPAuiEje-18w1D-}dS@jdW6ZzH7+X_uRYgas01cX+r|O`sGjUPx%}jcb6yJ=2Tm7 za^4`WbqFo4*ZcN*66d&KscGA#$2-o--`Rnz7CTIQDKlZ0zYAo70j55>9BWskNd|rK zQBLRoYBmNtb0dxfGc)mOYQdy(%qB*DAig`FdEttP%!9Ja@rK2|pOqcI)Y;b4AHKyP zOL8{y!~h>l!+HE~Hc4@#Z=aveg#Kc}_38rDxG||Ffwbni#BD-$#<&#B5sETZDO>_& z>}XGEd^!gV_n2A%*|`6{v77t)Rq2;Q$GCr+AId3a*-kiVz^)B;l6r-bQYROi2inM`)@zZtoTPNw8!Kc~FjnPO%H$$NuZm%p%Qs!_T?;fQ zYhP8@g#03!SJ=88=rj0>BASU~s*My5%A$wn=~4n-qnQ2r6a2B)_~h1C7GU4#TGhr~Hf1Sc`X-DH!hB=};$tb!pbp@V{sJ~AThtGj+UDH0aUCZXbd7PoQYC=p|v=NB*c8n)gepxbNIb;qR2!FtdEe%G-Wl0r?6zP)Wn2f+hp++b zZmBc3wm$!VokqQS&hl+EXIXfxG1Yjr(Qdb{zTYeyCB3(vddS7IX&V1!n(k4UooXLt zVB<**OARmH%*5C$eNo8hu?2^Ny{)rT1_y23mmNK(xc@F30YZUQmgaAk8%DEjg zHp#INr*D_IwDlC>4Qc?<5{WR78)RgGGV)(w7g%dItyia7N1weu$F)fD5M){z9%y?z zvS3;~)jE+|V+&YEwG;h+AH`Y9e|e(hH-yzYPhNc!G`aZ~X{rNXW~{IW&T{#SYs!kb z$p8Xn3pQS-eA|vgypMBK%C&b}j22U4L2}$B0&K5YGlT3RT3yt-!rzpbMw;Q344fgU zChh{Qb$SI=JU$}1!turf&5YektBJH_Qcx;>!qQ$l3J_=bO^`M~hF1@aK>N#ILO;tZ zr1YY?I1-q(DK&-)3CG(gRDjbHpZPrbqToCl!4>2Ko){vZ}o(*^9-JKS=Ui&jCQjmokzB>L0LRE>HC zw9y>Kvx!2B{LmoPy3RWATXkWQ*neBIF&oNob@Bl-X1^5KH8 zBFihSz}45Z%;PaBR5y@?_ztX9_y23HNWOrnn>^ctiWE*ShnvPi#FM5<+Ag>!cUl!$ zz7H9FxF%cWL-7 z>IPD~qk~ZKU;keX*0>L8)4uLPVsn0n>il2DeYk3#JgZgf{Hz!Cq3Vkk@9)O1$* zCJ-eA=bU8!{hu2Ykj;tydWX1KTYruY4p?MWdtUK=p)3B>Fr8^P+7Cnt(4elm=69u{ zhKvHtfEj_DBWs|6SL%*ItZ5!XGXWA0NIDY5k9HxiKaPg$gC`*4W&Zq(&hFPBQt=v~ z>@EjsTE|&YlivMvcBBYkv6BnMw>=+!q}62oSp`K!D5LR)kv}68iCN9=S!QX`GftEK z1EyZ@*=u~yhz(5v3K$yX0pgJ905%RcDC^1*`QoYei9JA%{+t0UfF8X{+#{$bZc=xg z(GywRe6;A$=6R&){Qt0aKMV?v9)RVrpMf5Ayn!%?Jt0k})ib_E`YDLa!5=tj2WJTY z83;;m6^>;AQ5ux>F0bf}jOvC^Q3@oTwC}wb4)i2TF3Nak$mvPpK?h+mErneNn+tStU0?@$r*lDh8C#)OI#X%fqv1399R~@5TV}qcpmXzOVUIOK0o+ zIlXDg2aXGG_{kDNIj*vn+^hS}_+nr04PjiB;si6anIV71O9YUD6bH;>P>$$kKT0U) zo)k(6fd?rTZ&?hJ@K+b(qTbIRGA!~+M7dHInxhf>$qGo8y8t#5tN^X{!R{o`YzqQ8 znISRzeclg*MKcelT3#bREtt#Bx_o_oXT?Pi)3o{Pmpk2kJ&#|xBw=<8n359kn2@Gb zFJ#XRhtLre(|GcM`+Tq;r$lp%Nm6W8)xkbC82e1QDmjKMrhg?$(Zi}CrspGQiFZXrVbbx6A^u`*rKM zW9_=GNU#|%Y>yH=1Xbew_Mm`E>;_IDqu0Ky681ud1)3+Nm zr!`5K9c}M1^lhS!7t79+Zs$ib@0jmb!fK7Cw-1V%O~}P{w-Wg!?GIA*?Ih=2eK(8t zmkct!xAfIB{d=}6>v)r!cpl(nc-uYk^7p&IeY@JkcfeHFKXc`A>Zb;3qo$+$I};7d zPa5xv)A)SL>3Gt7ZIR?IRs8D;!(n$Ns&kM@jN4DT5NxR(bK{8-0A84f5)uhOoy4Hu z*folJ#uX?Ca{ipciCeE_PFr(E73cmDR?icTYHmVxC_6=LU<}R+T7w+Kw<=)B)xsKF7Jp4^)MT*z0^^>qj~&Q9kEdi?5CmnUW5krDNMx ztzGzZ1lHnpHS2xCZ7oj*fBtf5vO{(kpF4~;Y(3R~lDpw4x&PftxSAEGqh^tk>oMI_ zgHVmf{>hyKrtpODCzOoY*k`|xveaya>Dfk-BBRq=jz>r2j|bA4KaIN^nnF=)2j9#&AqraXr>Hgiuw&54qBvtkr*g~5!G#t--#c&G=+a-LedrkE$d~$Zu6_3 zR5O62>VHZF*men~pyFn)kV%%6L{K#GaB8kIn4j!XEZr41tIf9CNtmJ%Wu+J0ZEz)U zBvf%FD)i>A!?j@Ibu?Z@htnl1L_B`{`<;~0S0`&a2`j6JAuk!KpQ~wNIK7T?)Ue;L zy33{U&#TBgt&Gp<^}DE^e4JV(=X#tub)xAw|2(Kw4JEjceNymuM*;Bg+_&IAlo!Nc zjpy(C_FMFelq|J$ieL7`(xhh9$T;E|^OW%%$naiA(k2bbqWL`fK@0`Ftog+>`nYQq zTr8eXTJPT94NmH0TT)*I+dsM=mb%VIkl(&c}L97#w5 zLCib%G))pFV1)}_LS zXcW@PcB=N?w6|a2Jz%;GvbjpO;+TltP=s4Z!yYm}cpMZ28HbmnQ>JHDIwmT4fDE^+c;S@ylFRj$=? zry;b%A}Umb5KnGDF0df?WI7CJlR^;AhXtMPmvo;N`aA@cK{(+PriUL`#}lxp-TePi zyi(@vP$l>mBJp-m0rGesrf+WVw2-c42L{Xf&tOuXnzafBm;uFtHQ$$aF(>ovo?(tf z^fGOSGM{NN{We?=&OqzDxAj&;>}KUhXpr=m2$B2{6wUvHNC(=CCrcm)_6+Cji16vo zFqk|@3;9!D6&z%u$4)l&O6T*Vs8Ur@2_d!ru1acL9jS_%hyb6!K$_;^iftjvOr_yw zsKDdQNv;iA>?evoj*P0_*m&z~GU|mPO;+H7o{gPAv(y2Q>I1=5RkRk<6RlcX)IfZt z_hvs^K6gHy+$h05Gxeq*+Cw_m038&RzR>`}`BLWgR;2Dl#WXZ{{uqea50EJkr14qC zGY}z}5BN)~`cVw%wB|N47lckivay4qPv}m2K=e=5`?NwMQ;OQC{GQMz5E1{{Bd5?% zlh12|vPT~hHQvj@Z0qw0u>q+?c4p+);rhilPp*r})+P-DhIUvh)=S%z78^QoOpa?k zR}m&jC(U$DFT+7C2aAo@ND=-hG})Y2CR&d7erV-Kfbsv16b-mElf^A%Z`^81*W`P7 zP2%dbyoYE3U&|{Ik*HbjA|WG!;*$!AMvt1!S*d7ahabePj&EtZZ8UM^o(eP0&g6n{d&5Tkic zBc6oGVcTS@Lcd2Zj`%$jy2|fROst;_su2rLg3&$8b$Cyz)>9)-_5QaqxXdooVf-<= z^xN20Z|%>xF<3Iu@?eI^!nPNWwxbSlC5TA`KlcpI$Ay?at(?w&t%e=Y`_KGhE>Mzc zsumoNCKr+2E|VJP<&uQL`pz#(0HEZ+k+{Tv#W_XHtm@k!;ue6RN1>lTn*`H-Wg9TG z!=&wg6x);gi3SoF)PSQc-n{}o-JA$j)q6m4(S>O&FHuD+|IKD;m*88L!^C5kwc7z- z5on9efa-cdzpZHDxD#R?Z`FNl9b5=3m6(KTa%FJ-h-Jmx-lLBxL^Sf2CuAcn1(c8O zgDeEmNQ)Nt1=2Z>{Aft3?3#?}Ia&0CvTDSOr$BSXLa@5uB@CwJR5GiLlIfm#6B*cz z_K4l9+n=+jT^wMxlW6p0XX=NFTwI~)ntSEJbjRJxZRVGs+jcH`E+C<-g7 z?Q=IR@||Z(lkvwNnk@VrQjxt&SP^ZaujRXMpHkVW`*1fqa9+|kdhd|{5RMhA;Sa4*27Q*T(vaJP|jY>2W}le&`YMt1JfE@E_@J z_2HJ`>Py`~Y4Ql@-V2%9X}I83@W;%Wf6mMfoJ%v0V^vN=Fs?7ytJ%GUF3k_BWuSpW zy64J+JAUsTV+GQNhKlhh!{5%;j#=1y72I1vqfZ3N#xCVGL)gMKliOf`Owjg!O5&^c zE}EsgF`))&&R|kkb|14O-gI5MeozMJpBf=M!G2QB_=7ql>p3jm6uKw zJ^FdajW?1I^O>e73{<0e1Yy!1^n@F*(&>q~)N+7!SRW;%2f!8dvRp(L@I^d=SIR!6 zQ5>#sW244OEmh3!=)yM@z<@c~kABCHM*jy;Hp2*&y~`-xs*2Qosf`qWL23DI?>5MT z@VccqL_vg-LK%kHH>#Ui%KHAX%d;s3loAKZ;YrYrZ>Abdn&?dUK#F;pAPUKH?96|& zifex3z8b`S`DZzdd2RQ;|qLb5r)+Hrx$MP_w*kB1p((f#3^ zAX=A9qj;kLj(1JyHv1_tK#ju(=R74nO(1p=shj;oVDAQ(X^25QoJwXkn|=a# zBSv%li~QSHJiqkl!Luj)AjdN^1o4Ik5dVI_@v$oYHB~*%J;LzXnB~rJ@n~^6=zOMH z!YQCH1+oXE=m{dk?bHL+QboO_fS0<_eguZASsou-8=N19b3@R@UqJEbA+c8g48CFY zsZ@5P#7B~QNhX7f7a@-Df#$F^I3D7JLxe!F%tGAOu5)t?KXc4q!khrKzZ5V?ZxK@pnNPZ75Ucxivayaj23J8ba$#1WrW% znS8y6WJbD$ROmy%UTXcLkaJ7U{Pm` zik5I)N5~Ojr&?5eOh=EO!56>`zvLVM*>lX()>!X8+)~-~TPR388>{$s;|y?~&UCU6 zHY!V6kgV^Uu_HfA4L`z1q}Pjg`5<4fhrp<+2~1H|2_iogkMtxTsDY(G<6qPeYg_+HTyN!^N-@3E45E) zigda+KztK_Fx3FdGU|N&W)IY4Z)|&_ozs)BRu53CcR>3s!M)WL`MKqt4j}C*+y$K=s=-D*Aid5;@}E%*5D2OtgR=eLQVbw{ z(ByI*j>Z=@HTwDFwqkuXHnbl-mAhP5iW3tm$<{L0>ClT?T27k=^w!>OkL&{$9nu_* zT@|*|^}T8N$(P-=ml628%@d88>eo&J_W9Rbvg=JQ-kwfE8jL3u5?!1G-IjJHhmn*X zq0L&ef>0@n_eF^UVpwVvzrW_~4M}^(9BvhX&3OvnLL(?TAP0N>&T5a;>(1>;i~w$B zXn1WC1W%K;K&W?TeR>GQ7Z7@PYZyC&vHYH|8QSjd`}4lO|NrH>uj{_<>z?Q9xtw#J z=XuWKJkA#|t*o`L1031jY*Fc#?GuYEp80STnfJn4ZP5w)*Q0RWYmJ%wP7vnp<_ z%JOjQEN2*3RQ5v#*^>5s#vA}<-aC_NUsZLin_yGmkLXYhWJQ-|=5={4Ajx4nq33&X z78w!s@qKfK!ZMe7LBJwz)R9a(+z|Z4c8&t(a8`guvVMi>Vd2;tUGS9`V1vAf(FFVn zqE$#!eY82L@P-I*<-=6(?5}#fx_`dXIaVANiZk;p`dE~1P;wCb2jx*9@6dv78j?f! zk;JjBV1%2SVv#=3^ivrhg~3+}8Yp0kmT;|jXO+=gxCjh+r5*Ucw~dxlLO>YV$d4d@ z1_1{h9PZ67fxGtJ=7eKuV5r%5OD1|w=R)Fk_s;7f%qb6D>P4>>8hyJ6HpL_#+6i#m zt=@t39Hg)z&~$bxG+hyd8Dw zuiK!!_5SCa(p)zjGHmnO1`}@^+ylV2He;R5vT-e(H?8_RjV&Y#tRTfN_vG zToUZR&Efk?yoZQ^xJ6`b)v<4KcUr>xipK6Uc||Wgv+f&!#GRisGCp$#YK6z`_Rs$fCMHqY>5sCJ*=l4 zMl>p$yBC4&hOX3N3P$4fdwIAAgbLGrTCUZOJA`3ry&$48bYI8?R~z~ zu-1XDx_7`i82e$8m?t9HGqc~8(?$R!Oqn+*WEujSP=L_iXRU6(ZUX z;g>K#HaSACvu&5q*86$Nb}&~zt)yCi8s^>?s;rkmeB2{%1;1@!3Rh>WVen#o?efh_ za@NWVyB){U0;6kFJs&z-YGaRRJX%!2=I}RXzMR^(3oaqEZa0(wIY11X`*qJ4 zIxAb?Q`W+rMJ$<;T;h_X1q9{aHLV@U-n15M=noB*eREYyCV0sMMVSDOYMR4C8`k$x z^!N@Y$=&GdLmdY{a93MS$>W!c9nNd}-%lne8J+iHgnTKXA9gV0tv@21SM5pKYWx6?afxDU$(c3x z49T2{^!2qB+Y;QH{KIy;a64d$7tRoPa1nW07fYP1TY_I*EXEcUHw!5f_TqLFH$v%b zZT}sA|H6|1L=#2-&bE71&P^bq9TI)qqM48C^PM5+&XtGqq?wXdauH^CWe;mxA49hSPQdHt-^ z-u!?}=h;W8hi^Wq&??`{lJs_Tosp;)ckjMZft&#*Tb>79;A)3*^;4th(VX$j65E%o z>tX)PZ{LA#%iepS!Hzd0luJe}g(K5bkU?qk=5D$l-Sus5Zcr01no~7xKkU$hfJ&)i zR`mSBg6HKED-}+0w7m~)E`qi!BB6^Hv1ikJB2?$?ETl@_5>w&oyY^)ty}tV&qoB+8 zR)onVw!J_P3k}W-Z#KAsGe5Fq^iHWZtWv=j4kOCl&&>r6%JNm1-RPCk zB-T}pMH*R&0{}A%+ZXEm#Zu|nF(i}CozGlJ5B0w7fc?uSnb4_syz0+?YMpv?8Xyk1 zsG7YbvFf(Ay*h(t!w#1neJo9W6}~ft#vzSQMOnf7X*<%8wx5mzj$~r>)^r8!E`9Qn zHA|FV6zA%_lm_Vt^}R&3XUuWhQfT5U=T7h)1lVSRB^Zt%+?yT}b7fZM~! z1pG9KE(Un+>;Kd?4oGTHt+vyIm6h^+HlU@47#A+Br%jo_W^-rpwc_$ZHuY`=d`Kna zNdECt8j_;67H)jDoDdEMz!F)uu*FtOJ8xdxjl6x>TAjLMZ}+Ry_Q-7eb;*iv=^Ula z9(;nZ5a?4%^6=Ke1ltqy8*T0Fc$cQ(w&-Gq*}|$;tyKE?vbMmlSh?oMQ<8;wsV1F>c{$P(mPw_1=1TUASMND#5mfhvn z?2Uldjs$ia+u$mtH=H1o-Bnnn;m!W#yZkA^+i9NZ`HpF2R@*Gh8=SSU#ug#T4H(3_ z9sugFwR`y6+@D`*?g{TZ1?a0`TBWLjw7J< zOaS*J&s!qYSre<9yB4{45*&z0&MZJ*9QL2&ywz&(tM!uIzBia+`i`nULQN-VHL2;j z2W>Dr>bc+TyV^(KAy;b7opHCyuTVQ3o9KD&vH^l6Ngh&R+%D+NHV(NGLH=yRC+qiy zFrsC3jHD@go7ylY%_(OSve}mPojVV1n95RW>We&g+)d1G4`gYxPIy%-fE50o?G<{( zTqvW@Coib7@dSP`5u!J{8*-;t55NAn;BJToC$(*U9Dk-~_N0vyXPO&f;JJ#gl;D)d zRg0dIt3o>`y^c<1@CKNC_EPpJoMive@;()`*DlfO|^Z z)al{1q_=9nr$!0a70Oyx&t8qZsMX>rIb5f8KH+6io%7nUst+sXwS~pQj#XoZLB)rL z=bSj*HW41e6d|o1qQfMJ2i()+fpELWx1b!<^84b#O^8RdTI~98>*K};_MQ#qeeC0o zcI$^Hbt`r}$s%0o*>kS1_rl!i7gbU?yEQVgVQ0$tfwsLAL#APe?S+u*k;Gx z`PC!`YzuVyrP_AB~qs*IKgQup;^J1M;Cfxue9SA4vNZD8P}^ zrAD0sRfg=jl5O;D5xZ{y%i3-k5CsWmchV||6s7Lf>>`FwiXrdpz}Gtxl*is*1ussP zc@I$^@w_AR(IUog2@{lFKty#<#j*_)2GkPN^cmS*`h1rXi<4R7boE*we960swvavkBni!CL zhwdmW3p(|^w6hF+VeWL(^Y^q5&WTH1ayXlAf&aJzUi+8NjAIlcc5|8tIUU5AlBcZY zV@)Q<li!CPZQv_dJ6zyVt=s)SWcvW7{pfUoD>evL?&52c((7Qf_Bs z5B$Cq;&mdWh3>W$n}jYS#n|S3dy5(-PT9B(SAYMDEP%JnzMEcO-|~L*J8vO=Idrkz zI^>yWu$Ua7L)Xm^cr0Axnv}16J6V2N4rKT5D@iGY7^mObKS916h_^x6 zL&ohO_T!H@I$AmtwlGumJTI3248iJ6Z8J&#HdseV zo5^d%*=kx-?X3i7__eVv34)8xitE*u+E3W_Pd$Ciuhx5`QHGDa60Xk^9W=Vd}bC*kh*wVIUjr=w%S&lmSO&a+jQ zo0RsYIxTS4E^=3|n_S|&#Qr7Z+;W{X|` zy5Lg*1!IUew0WDB6^bfqhGsUPNmacG>F?4kJ^a1R=! z9{CmdkoSJ-@*yp^c$1Wxh-r6!rUOKcYNclcf}(zKK2!s-5|^gs4uPTogbbEOw0jmC z5>!4~3}_zOhhHc^_#-B72`2?u$*PPtxrQo_hDlEI%unBv;S&e=kb||B0h{nA0L1fD zZA~Ylq4I#;ZY9R%@;2hgD+Q>LOq3rj$gj!6@s&~F#z{Ole_i^_!TB>Bu1x5q%APt- z5OR5FlCJ>iE&$oviUN!nfD?jU2zZ1cPs9=;`6O9X8gzS#Atn($Qe~OE@pHg6^M>xk{wA>8jw4jq%>9;j;Y@BoNtOI z@Pa(#eE=ikhe@taQoG|ANWDxI6+WkL_)ASF68YCJCiq^kI~B&VKr0%uKm zdHJnW(9i4UKKT%TIR`Art5qYX0f34Q2p=o_5)JYa)f0b2`Lx#%CorcBz?_oKkkv(%m2@rg-2E!Gb-PK z{CT2IMu^UT0Fh&>%tZ)Wwijmtj$NJ12?4>54h-nMFYe#%kD&1z%mZ1(0QY{C(XXr+ zCiHPc3KKeoUYZ8!@r|QyLw)@`j+c)a*m+HGX#hOv8_dA)A)rwUmLSwCAJT*ZhD>(% zg%KFYtxG{~?f=%#_e?{!V3x#J2Wgf^<{>bfX4dbsq~8Qb{kJU&u!~D66%mxb2a0Mp zPxta@Q{7?stBl`hY+cOC%gK4y~?^XP4a$;$1EeHhdWOZ~O#3D25&ul-3% zz3VBdD}#RNzcbvF@8|BLMekbh1Pw|{6%vUmQ7nRVku)K#=h80LVT_a*f;@T^U4 zJPXQ^j}Vb{b;19OIr+dy%<^x}8XU9hhKM_-Xj#IU(Xav;{ao(H5K2G%C z%~hp+lOTd}9Z{URkF@<{^Ja&0rxm{cL$2D6OhZX=6jv~1)i=6NlQi^__99CY^oIA5 z>ua&Lmk2#oyc`_e$EAh4xU}h<)ylt>CfZFQ{ovMb&RTnOc1$Q#sfL#`vOCx1RORoS zx(W9RBFnPEbh%cJdymcxWliq2UVCYCbt?X)b>CDrxnD`t5%y!zC#bTt`rI5}t=|^c zi;JK^!PQ!m!rzpqZ1MwVrO44=)bPXY(IOGfw#i!lTlqeii_Lq}_T@V2q+Cz;DE*q>Dl~a1icR8Zn^xq&U#4^GnHbPT{L$U=4aFqS zPNc^%vm{ZBzJdKBlr5ehpXbsKDi;9SHi8ZUYJzz1UdUKpbM(mw&YQf~uFpUhtVH!8 znS9{J4Qu-QAt&ejI|fYiYa@F{{B0r_%_ONar!>CxEq}kRW(z%i#rv)+(zi``5EbaHKsy1xTWVlH)Ldk z8G7K)SBZ|qvv_#b3b(HVkB1$tXJ0$t;j@@&y3>+7Afp%Kl+w*N&+sDN&9a)z`h`M> z4QlRXhmTX|mkETGNNNhP5>?qvM>qT61cZ`|X{NOk`8O3!<#C#8^!# zO@0qM&LB25Vhs0vA>&~!i$T#`K?g z+3rH$Ef9?E`GQfH91lBgR5rKnmeX(YZF0@>g7PRpb(7uajg-1`dYb(&5;UoBA9D0y zrV8=Q(1UIVCFIX^vU8;$cooY#zcjySm?5J|h4?II??*O+35t3jEN5%>eywJCK8-feG3cv<3s@5K{2y zNo2oyK-+j7-Deu)Hzu^tHzM9eTIrvgSV#!{7G0^pGoQL|+zQMSX0QpT1=Vf<0s_43 z&HZFWcce>u-59*}(eE22`Bka;Gi=l^=zJU8eLZYzg-;-0;;gHzr2yq52~y@U;t{UV0+aTC6lD2Cl9|c@m!CUrONT9E{t@Mf!FL5=NM{# zQa}luKbS@uIRB4x%AcPB5%=vs0L}n#^mFe{%y$g%ULFf^8!74At(SYvc{Z`I%4ccC z6kikkb?MV*Rd#A#I$yQY)$R@y2u0&|AZ>MXL{6?oBv5%*3^?bEEG@m!rE=$-Pwv0- zF87iGTz{iZqM(Jc_{&rR^TeXx6a{DZgu;RiXSekiqq-^rRjwj}{n&?|HLAW>?SyZt zUE;JzN^kqCW$RJS*d&3Z05`*seRET)bjU%);Vw^{z(Rs>>CE8~SX%*Rpj$wg{t&L* zD!b`DTRUei&K$)tQPx*p^`OY*IsFBwphbGfE@DG%IT)+ff2{ZjjN~nF&ELhtoj8G0 z+!L^?mI48;;s$NQ%iv<0-u45ZO>#kg9F)gkxNa+F?k5))70zAatPJULQ;)hlo|elL zbuOFY8|rtXJw|*u+c<^s3m=ZdY>zK#kaw$?H&COD{F%>AdSXI2$a`HuRs&zbtasy6 zzKLy~?cpvnX%TU;4`s`@T={N(^1~m3-)Qx54sQ^D)s7s&ev(9>+1=msFG#-rrT{l< zs7mKR*L%6zBl*HW$v!~L^>x`D?+A~5!h0Lx(&uexQ<52btq)N`De%O-TSE3A@cyRv z>THvi5cIA6%Y6Yf$YdKl43lh@k;M;uMAutTnSpF)lj8v>Lkc{2F{%$^X8^q{vc48F z#m!VCb$9PJHLqRdk4nShQ~i#?eTN)0S8@zv=7Zk&BckdbGw$R|>8`*R_^naFvtCV$ zXj$Iv6ZeBFjdF*y6S^$B)F`RgJx?_Vdzcc?92YemJN)#P<9ksf zzIWV8qd*MT(eqE0^`WXN?|reXQ-(Xrba^d)`Al{gdI#3VJzX^gULM(Bl;anx@AEo& zL=np5{!~;Zw4Q6@gANnbqRF@Oa`wq&_Q$E8WVa)7*6!y|n~YUe1y$pp)BBn_1$fna zU4~mo%tse_$XVa6t$ouYS?KxhZAYROWwBBmYXwE~vbbUVUO>Kp{$L4rPjr7P-1wli zWcW!;j@;zCV_E5TUDxK$GgB41pX*FFm>S&Kh>$9@;hsNpS<|(JJmk8F6}^EzsGo}W ztT5bIW2~inPniukJFhw2K^m3WFWOY8c{`uU4zn*3;)l!3rLso6r%D6=qAfTbs-k+bzz|-%%FhT)& z`e<7U$Tb5`@nM(_heeMdfK{*$n_zdN?^4B|+*`t3q#3|@*3T%{R!TmkT5VrEdsL$Q zM(rmsk*%D?y)8)twX_c6o!0ZOHO%$hKSq;H zBCQ^^6;!w<2l={l>PR}@=(>eb&~){1Pbl+CpNwW?tAFg*o`Nx%VHWU%+m!2gb{*L{ z+(HFy#cRG>0U0Yd=+xm$okdqBY6DX3OEB?ezFTKysJI z{z41a-e|dOofn|)Fk=Ek=<=Qx8A;TFI}0ZuC6k1EiyZ8)j$ULNJ*-czUldl9>C!JM z!*=Wga>lo*l}k$PiHMnx)XC#$@;Sc{WF4{)hfxL(nI-K)XSZPxbRN!ZvSIS+-l$2# z!rrte)jsF!m8O+uqD&he3;OOjd4>*oswgiq)7$Z;i6fGWsP%E~5tjjF;K-lHIa#T| z&g{xFe-MD5CW*Je1=_zaQ(R@#zRgCf7V%{gQ_W7Th8!8h1~oK{9P@)Csl~`2I{PDr z=i=SxG+rhJTyM+L|CRH+rS&xjunDhpHl7>Z!}~8+&um(Dkbac35wySrc8NH+$G>G-s{7 zoGIM|y_OLJK9=!KMA*diu(+riTWas|kzlkwjZ8DuOXJZTe8}P}nc)b>#=CSRp&sEI zS^%W4j(>6S-miqxc%B80jtkN1^HGxSLi3R`5FZl=1jLeGb4qcFEec!C*^>~4;{z-qJ9%sPX9=XHc-Bfe0xb)bf_{}qxRePG?B`bq`-=znMY|&m z6LlO!Merw117Gec(f#%8oE>0#y>se}hH_)0Iw_OlDcm&B;D zbmT)i;li0coTtVL)-IKYLT<@>!sgFT@-9ZPnknfHP<_~5* zKX%}RL#tgTUDt+`a)B03jyLNZW4(g_T(&)56x?U( z$Hy0VVEgv6t_cc(TUFj4!K5AS*A6PZX!Tq=^Ro=v^VopQeM#WBlyv~V8eQS=G5S`T zn_5T%s?^O-&eByaB#+FpL?Oh=T(y8QWwL5?Sl6R)eXl(HdKf4F6n$Vn_dgvf6Kl>( zD! zPe10JFWk7NVWR1F7clESLav?3{vIHn)gD{uz7c(Eth!NJvrkA!J3ntEsn$%hCY_`sa1twEqu_b^~y`{8}Yck;ajwGdqz`xWWj;E$$Ov zw7{*VP!KnGEXnZ5-Dz~>_0)H6Y&x$?G>|GbqcH;yoR3_=KLXDpl$l!h|!6MwyBGE zY7N)FxO-=R|zI$x8<>r68dUc*ftFsJpjXZhK(?J;?T%&J-Qc1fyK zp#L1T-uYg_aCaXpi2U3-@oAiZQC*&a`?)_7%!JExqOX}w2hKGHa}@AJz1ctSBX;B7 ztpC66ru{i60@3nd0kYgM;Z48mM}(E+c9ka8w!$0IfOm}#duO+HGocBJF(`L@O&f6l zVUBYg6kSn8bx#59a*Tjuzx!BZvUn?tOkrn$?|NAU$ApZlYc^g^CT-2ycH`cPr$=R1 zCuShgBqXC$w)X1%xu~ z+Lf0?Z+y#pOoedM#y#OgcLA3&j$HlN{D)*DS6Tnqr}%~Dh}ak@Dln7K1Y`spJmzV8 z4*eq>^lwiQz(dq4_wyw(OoyzoLOk9*F%!<8v&O@pjtnlD&sg%&rb9Pqa9ON_XMGH1 z)$4E7dN0FLT9#)jlP0UOSo@Wy!%StA;v^eH`*GHzqAm>&(QEE)f;hL)nWP5sonCHh zyBDf5BwTI8<_kf(Z)FV$s1NS)#;*mUJuliC*}^ZqNbl;c`+Q69Tb%^tjyT*)=g>e(!NN4S`cOuF{j^Uuv~e8ema7$aiQknunN68H+@IfxQIMXjzibsUetMMKb}mM(mGgl3Nf)|H zQQ&oPV^+evMacNs_H$i|fZLKh(H^|{#)sJ2~UBtgF2I>pIW$zg>UasVJU(J>BC$>ku_t{R zPP&2^S0UxA7|L`S@>XlVmiOXEvtlib9*gmvXAL!DdoYafXZ34rBD;?5562{L`(Sy? z5tLs9{NicJiA#h&zzC8kBwu1liM)|jJJXA`NvNpmFGa%~A zzvbSNAFQU&PJyR^`Tsp(Vy3Wp-H{%a^`E)1i;Bbi6`U+&($%CqE;aJozh*d~9Bc9+z7^^q^|&cFAW~V`+tpd6I1Yi%U_e~b^ar%RrkR>a{TRnSKl9LGGgHT|AztV z>>@aU15WlMMMey@{d+=}SPAtX>2N-Q0zNo!s{a2t;89rdF+pFdA7}vLyZ&Fq6P)({ zmAFmL@X%g{gJY2eaLoVhtRX}p@$vkxiCWWPiJ_>s`*%UOmtRyq7ti>GaL*s%MhJg8 zCc}i@>!}LfF}C1gzT1abA^9H}{E(xLF#y!>?+f9l^A%3hBcCHh=yQ%B9QzCIX*(uHbdXCskfW+F+mT zLEM%t*c4Q!bcWC<0W0;z{PB$(%)#FCE)Y=qLWlb6>;Z2)R2LTf z1A7NTK{vy;*Oh?-i%2>{^aXzdyMl-5UY;%V7f|*+5{hmb;3S?BPvQ^ws^6cxGV3;^ zV$8VX{-%!p4y-o_KvalJT^KnMqXga;q(4zJ%(Cva=NR=^+Ok2Lp+{hl zyOW%9r2xPIcGnLC;Gse_P=<$QLxZYW8|OsTIGKbW8*EaaIEOQ8+G$twbC_$-XDfje zoC2b59T4%y%#+&%kSsAAsXdwQp?{1r{bbJoxvGWC(<3zvjhz$U8I8)+nkt&t$;PI5s~?GLb- z@9#tPD=38gvO&YAhx!6e^X&zO%SdsAQ@m*jSQ`0FztB5#nA4Z0MVY!bO=Zl;!(NWOMxbuwN#Ir4a^dsK@+qb5(gQ5GSRLBf1BVtYNS{B@PM=V z*xHLs>N}GOGR~Op>y>f#iE&~-%*&|LXpVIc8wo?Y$4c2f=GG(Zwb6s-{EeXAYI}`k z`0x5hEF$1j9OVW`Y+IL9o~4tMD4m%t>Ax5cMMO`M!!3A^H9LUCe~YaP1BrIo4Ci~l z*)3ARP8>!76tv&!dHGO9|05E$g{T9xwR+hTkvUO$OjL?O53bam%Xo*()ZrU~weiy2_By)jbHO`ffe#=xnR>J7SG|6Hu|sQtj>g{nP59#_wGXAF&GPtkpksj~upGg^Xl&J|xltBl)$zJ@~j z44fk7?0hfi9zv-d=udJVTDtga`q z1q7ZNx~bVfG3QUDA+QMd_o1nS-95PWYu4A}r?V~GV`+N^0|BAN{#_K%l@$N07(s5v zJ36F)Le}yU;PLuyB6$ULBGJSmEjF$s9P; z^&0qP*L%9mm4Rs0J^QcntK)QE#b3thvyEN{-|nX$QUlf%z)QEK<9_|N&kmx+@bX6- zx~!#&5HDQW(fakuaa*H7i-oBvlX#I+HQxw?)U_+>R*fn8R>$i=TfXZ`-iE-|{ zkI#&VG*syN3p~Ho@>rIhhKQIpEBaxwG)d=$wrdGMm|r!s}L*mV2S>iLs7mPnX3sP-N8ki zblF8urlN=&*EZ672xECvjeGZ4%b4nk%z#2qRm*ZJz+ZEUCPA<33&`d^f~;HedtKmd zJKwAzWo2ExaGx@mqORk^j~XBI9-U;qB{^&z8G2;K;$;Pz~Wsv=9w4 z4cH=|zSJ2Oh&Umhm|X_!AJ8K2+^gGd%!vN5c{I=}oxU8Z88TpEx*xqz;uZp*b4y2{ z@6PHJjurj3He#H3Ym^H0bgNQj{7I&|_1&zRU_hN#f$jJ}4C8wSX%H)k5N zB6pw6ECw0C)`w7iS&J(uRVD`iV6X@V?BJ!DJbo8G7;Izy>?t{QABVYqBy!_ja5g{z3?S6UE!0oKIKLV88lyBo8&Fzd=W|GIcFbq6IZP+&*t{|Rc!#CN0d__%k$gC zkN7%~anSw9$u$4=^2)V}S?p}L^9Uy zf%{Sk^2njOdg7t5oxl6(+G`z|?@ti3?o#S!E**M}hsX+?0p#GTjjHbperCerU}2H| zq2Yx=kHtTl$^zF(v-^V!3Y z4XG;7cVe0Tp3eB=+ACszofL=n1zQ#%KYvatR!;GXzT-n=;Nr$u`x}AzP=;y^ne*;f zySpEqz%X_><+?Z~Fj0Q6q&-$6>POxs2=I_Msx-$+P~uG)ytR+s*g0xE0FQq{+(Pth z%Xz8Eqrd1QAXtLZh@{i}UZ}D>0OSFBRzOu=M0pqo0`}3{J@QoYS(BKWN9ftTIub^P z^k;ox?!H%iX>x;K>4@9-_O*d#SJ8trW~tn@RtC;z4YG>hP2B;SW3fqhCN68rMmr5I z*RLHS;vlic<4FGb(*tad2P3Tg1rue4`4gd3qp9WlN)P^WyG)x^4G&vYLn{r0bTO} zjw@8<+Qin%8>Z6!EJ5#ac$biA>=W8OY`4jWMX!ObIu~T09N^!JInxTDk3xflBK`W=-azNkFlWKpebG+u zQbw}1=nmdKVH)~F+3JE(oJ0Z@lBjVcTv(SBl!t@x(+``Jdz7q)ul_?ahceH0`lmVo zM`C>E@YP3W@jMM{AI5fTalA74{xIK*rC}{5%Mi75&PlodDoWLQ7vQyjr#Kb9wxZW) zVY??cGb398UUs$mS*dfbZ8E---b&A@j#$!8vO^r6LPXLOm=BYDte&!ghLeBH*jdU5#LQn) z>=nYb3OX~G*;?h@uWg}60$PWL;viI1DPa*1p2I3O?~fHdQ2GdrH1Yn3(0d_$_}4E# z%{+W=SjLD`eG?tZEJ6^x^W?(%x4v zmFENjx(GgxBbS^gU^A$BK)%n0oNgxQ3_+_@TES-9= z6OMHwDO!dp9D!^5Oz8q9vTN26(Os`g4X+SfNjva_cdFbo@>d}svFMP5;NXjR=% z^a^5+MeS88G93l@OLNp`)bl(W3vBH`nu%qb&AhZ59+&>hoj6|vDv`F-+CnFN)e^94 zlR>5RN_OfAr+PBN=bRIb`x>iyg_viKlG&-IellS_9~W!(>ct7(mD5^~Wve5s>h!fA z&qYv~DL2Z*GJ+bi2jYM1(~2XX&slOVBOs-QIRN`^d!UK%qN z|917|s(y|QAAKruQ7B5{XhBSrj)O(pdD6afCa0?+8a#(Q*E6Q99=km;+;&J`Ya8#J z!QsW64msI2O!azPQK^Rg5A`RZ0$~@Nco<4K$|B=o)qVP0BW}74>)!qn}BLNYG$m!VWgiY60kWdF~?f&7Es5zfWS( zzgXg{&ALI~<^AV#y-26-eo+2VZAy%bVHrJ=XHax%Dk(o?sDIxKE}`#8Ir1z)%E(T< zV90IiS?02~7RP0v?IRyc>$2+!E?bsf$46bqDha{EC*ddGn4 zSGvie(%UMr=wdWJV=^Aym5a#Yb84t99s@ieY)G|UPXa*pYp1zwS=8 zWgDMnQm`ETDjeQ%Wm)G?APysU^VN>*sRb_P(c!h@*P8Vf&DG+@i-*h4;b^QR5Xa&| zN3wa#hB_t&9*snr9OoT-_T=2^c%pnSm7~nlA~?ZKj|CLyv_En_;P}d|^r@Hk!bp1G z0fM8-N`1YgTX6e&oIcTx{%J)%-`Lc!i(zcHRk*mLqAgj_dWUNDZ{1M=ZTXmv=GaahYxG*+DO*0NW%T8Y zdwxLCkxx8cpmOgb>V;x$b3K7)0f*PMYWw5}^3F8>5HjepB{6!zPM@BPO6uY%%1}4p znZ^)8V}n063eDa@`04=CJ9L%x5FDr<*&3eyhx*xqCaEv<4;}t~2>bl?itO0O`#arZ ztrD(2T6-*TGViFUaMV|aRu}#%AC6HVTj##*0e$-x{JpU`9|+eFE{|VdIm8b%w%9FL z+)l*!KP+BiY6rgY1G5KkhiD{UON)WVa{_AhEB*+~&fp8ZyI7KTZZcp%zCqIk8rEw^ z^$rE)cB?2v(AdAUGq`fLjw!k$U4kbyv~OIty4x(BoYt}8DSN_dFTKDy)7W!uUCe~x zaTRo_PDRroV9E-J(07`R#xLvF*5dH`N!QTK=tEtR5QmvKzEAEhZ5kZN8N^-#qiHR3yvW^pV+PkWShk zFa5W_R_brw%jk6?%M+}U>NN#t0|mmD1KojN4^0?)sQLP7 zHNEr>&_NLzLY&VVko6Y(6S+eE_%=H=@!LMaDcd<4D4`*)Y1lqA(7qlnP*q+h0j^jT zqLtYyk>7fR`VpC(U`zfPh1u0Xn%m$>F_`3R0qNs;A0zwy>bgg)d>;Kq-#~o3)P06n zPM~H7<;DT*ufQff!6=Ne%pwqC;CElo4h0)CN^adSzkQPQPN`JV6W?WbXqrI}KOegL z;1z<~V9`#vje}{Steb&y`$ z6&KfKhNLWIw%C$so0|CYTLmIgv~bC^*zr0+%jEJ#dT~RpkwSH3ITe^ zql#!Xxx;?c<4Wi}k7!lHNHk*uERBCJAhrV=&%w~FiKkcd@t<>~v*2EAk3Kux(AmI8)3Y}U{H z^-i}@_w&yF0{}iPAk`+&P@Fk1D$%@t1((x3!CLo8D?-ix^e96RHel=zUYE^c&Ciht zrP!d7mF1_`nPIX59k&Af+0Gtk_5f}@1&st~bnClo@~9Rx;7n7~AwJ=ar>nByWI~^L zrcy!%>9=XVSLo-!>3tU^RfU6&q6gLblRmO}fS#}vMH!Jg8-k`upqkk-pqzQv;RGlB);tlltJg)aFJ8s;~_Ko5Xj+mQWMx zL7(7ZpaeYtq1zd>DpWG@+8#7WOuT!X4SgmNJXbU$tzS;;YwpF`WRG3<-j4-NN%3qp zU_0ji2YCs)7g!}ME1Ehj$GgsYu(dHjF&6ZL@cTh@!8-9G4)pGl{LLN_H>&)TM?#f= zBqKV`_{4k#3YYg~T@Ye;;gH4ZSlBfzYGDViA5-@`AGrazZW{i1ZXoiAML^^oL~kH0fzO-OLx{4z}e;P zORFu3*hpH<%1=nt--;}3nMvq@_Ac2ZDIk$`c~6Go#*x-hqS*Sdgf6OMHAu`ct)>tV zz+Y@D#`7~9v`JL*SH2nz%@!OWju~@!JX;DDz7=BnHOEFwmn1_U-{4jnkRcNDoae%& zwq^oA$DOAQ7i`L(dk6~3$8H6@e$9vx$%$M?xl@@dk2nuxc@+>(T_SYxOVjgSCL2nI z^^{eBl(LcCc3S@QZw8)7J_b>MLNyJEs@o0(hr<9tHOzqv_q;Xue)`h}*kiZ4DZpAM zcJ=quhWbeW2MCjoF=^L#>A4h7?KOUqF+-hqo2@ONBl0ED5xvNmUV4?61sIMRd}gsg z-WC5ivQ450KxRY@s1^FoQKSAKM9SEX$cqoHG>^yvCCmRdiB`}RY$Gg#bwgwEqpUjb z{XoFmQ%ETR^O46YSnOfOeqz*gl_mE|UIhfm*O~ue_K18K`f%j&u@sTv;o+w%YZWiO z)kyl2wgYy7pTy|}*T?aQ9|;ib&XDE-)c7=3v+1Dux-+`|T@~$2(?z>XH}4+ZypVhF zmeN*-=Y@+^(FLuZsl&K$o76D760}+8GNV8FmL#x`|H2K^wZ#o{wZ&YM0^g>uT@pLY-jyHZFppIPqsr%( zQiTJA(g~*;jK4U23$LoK)7|r#X42qIwkAtdyT@Veb>;5eP6@$9vl?CImYJ{hIVE4O z>%2&}LhZ>|*XTYWnvg}fyYv{hvnu%wn>=UN^wTf-PKG<$%28tjqK27-8aCSm50f5k z?=m6Rv|#}g5A3{nR-4EB?<@T}GYvT?{BTwrYU9y&DOY9< z2$XV-j*oX~#nT#OtI{4uO_l<*n%eMnlaoq!w= z1%}@;5TJ0M26-(q0L44rN=#;h4lX{Ho2gp_`g+URt7)??nasC`ST&~hFHGfJG@4xi zR2w!{8ohkj-7r~jWuC?ITVy?T9`8uy&`rLX@96YQA9U};?v^!pa@mVtP=4n>J|_O_ z$CBS$LFQ?DvIs~OdaW+GfQ|Y-=0C%^_;DL!j^Q~tGXooImG6PRQC(>CBD`P>Ge26h|clonYnLmf3|0jO|ai}|pLt{KM zG)~(jt`yZfG09oK=&~u$pS~^ZQB_2jb6J_dlJBT{`aGv>Nr(3GRGm!L8b!m_4Z?tq zK+YGMezhJc(v1TQ1f)zN*#)PTlYmOIj`-}`g_;+Mfv9m>?wQ;_N1x^`d|{&(5wF+w z(t9ICP4I5R$~wM6rsH~YP+S*7>t)n$QS4~qqmRvsPG33>GCobVLq>??dtHLu$qPVBA`arv7#Yj z+V~UfhCil$1Oj%9x*a99Zl>PseFQT#zU7bw%zm@Pqvt=R9mYp=)LQ)%dm}&o99l*}W z?j*i@{1K+K$XP(a`k2I20pf6JP%pm(hj_LKhkZ6aM-xXrY!dYXwgu2s4wgt;`fL@n zth~;&c}&(xwGHRN5Mw0u1DeoEm`GB+h86e2kQFnoc?eRwZ{kYF@Yuetx6wk?*(u)Ad3u8D(aD&M82>-H@RiJF{49b zBl*@e`R6g>M>)mx@%~m!>BF;5YKN<1qhd+KH!$@rBovv(!A7u6-h06DXz_*61%{G) zB8UmTtv+KA?0Jm%egS8kx_`E;v4luQ44aObIt)4qe7J?8+MMcE&K~F*Bt5i50uCzm zFf%}@u-pZMB`n+3Z3qj>)6Ywg7y-A?MZf^2K^`Xw^+cr}1TptgMZjR?udghFCLa5m za59_6zlHySX#@cN5QxYNbb?A{1+2UbLv|*9Ot4C=&ky`3J`^bsQv!di0(cpqZ#0oO zU7>tV_teH?<IOvsG3fkrj5*=HTQj-*0@>f;y2gf5Dx#T;08<2&QlcaoWBE-W3tjI;8!?h5< z3``#UlW4iBT69|5pK{Y@-#R!*y1>zqq)V0>5Ob&pbw$e11!7>Hs{bw(J`7OgkClqw z?tlD0IWL#*W=SGjiGz{s2#{scpKL41YX@uR#`EcW@9ev@4Hgj@6-oP^HMx;yQ?|F7 zlfl$Eh&M8rHiJ9}VrUFn4jF0t@0yn&Bf5ouikEC=@O~mSG-+vJQ~V#IAiaFFxi-xRK0yHWUPPN4B-9-(^j%IgZ9UFw0GNGE?#GW zj(cMD_Us0qIOs(ldWU3HwoZIgRYxU&p%49Nu=ehBO%8B%WOuvD7MW{PoFRJm|0Z}` z&jG0^Or!tg&@*F3uI+ynyiFi1!xs=w_c=UmIs`g;LBtM!nFbKMd)*b)PexDITz~{> zM}-#1*;dK_{d|DQy;gh)KFlZz7{p-ew@V^q=mU+C{#8gEavc0FX?k*Lo=3(EImpt$ zE^MdQ=w<(p@nIKE@pNFAuYi0%?85Fhmq4n~_cn_6Z^jO^`ANJB8A<=rwAGv16y*OV zCIBl6+C+g{14dn>DRDy3_txJBOxy+@VX$dPv3x=~`KIUsjS~O1G$+*#ql5!BfQi_K zQG;xn1M0nlfW(Sn7B!UccK_bLZS__)8sgvP(?_LYW{ZGn+zyG3SmA< zDYp@;jRru|NBGqMrxnXTl=tFh{cJD@5=etP6=oEH8Un;U|JT6K&OP{%*73Rb58?R7 zjPK5cI^eUZjsB?wq|$a`A6D0zf(W(<5Vj*qw8TZXi`o>o<943pq;4D33R4KNfIm>M z4y^6QN2%&NF~)+1W1=GFmqc*ANWbvm4X&kk-`4yb5z@yCuXi?8&7Y{Co%A#_%U*4L zR+Vd44)hm8PV6KDOHBN5?Gfk}cg}%Kc6X=2G7x|*Ie~CkkdyS}wNXMpR>Pbk@JWLH zPC1~Il4?+!7U1pfg3kElhQe}Ewzk`r=>kCr!0>JC-bPMEQaFO}hAC9oc||Btn|fQmxx+CWiIKv7B&6^0H; z3F!f)5fA|>B^0Df8itZkK#&3HmJ*c`X(>@;Xc)Q&$)RiLy6=FVFMhx8-uwS&v6gEc z@yxun>)Crh`}cmZ7g8sz>!z-qBlcTMgOXg(C2yrC$ZJsS<%azzyek~k&(6|}ccjsr z*@C)X2QAbIXcq9c0mOU!24^Cg`u=rgrAes)?`@w}FS%UzebL_fab#r}%uc$1oRM$= zCg6537;|-d=PFi6+Oa}H*_qLk2#zuV)zZHZBG5LK>bXHPO0eX$_GPT! z-5Gv{CX1t=#dR#x9VE%^f6Yz8ijC<)e)sFLSx2C{HFn;vrGOge-`Rr^eNf(Gi{HLK zJuvq1a#o!$vE5P^A6PZ-uW41s|3CIJ%G@N&jkNune)A#Qa7z)ZE=|}Sop4iZWAp zC{<6#YFE%CUBLPJ7G#*3;K^9MaD4pF3bXBXu-|&mxY=#gfcrOL&hgfM|z%Tz{$BQC@x!OO+7j^l0x zBPiB%HyvuT3KeYN&lHm^x`Yh-*_9eeg5n@$k$RWF9h8HqTI!R%BGXsZ?x< z<3-C3-TiLJ1V#X5Lm99I(<5Na(ebRFN>kH%UpT|kD3mf5>T}5cT;^CXCrzI9^hFU5 z_*qHU*|RPtSj^zOdOh4FwX~?JfjM{k)8NfxsF2eFz4~P>-E@?aN;ta);-QoTT~$$^ zSz2pE=KzCtJOZwvMU^4t8aR+FXql`uM`q`!LIwVPrRp>fdJau8N#fT%&c|ibHsN4Q z-n_)Kxsh5G+r8?m9yvj6fJOaSq@F8*skVHJ5b9+y8;wr~OwGQyORls+yU9OlXScYC zxb&)NyKsc9pK;uSmkY(xiC=>`jONi9?fFGBcc=VNRW&puv|pjY>8< zjw^3ZzID$aaPRARcSw%}frf5Y2m>A~Q@aP;p7V)cMR^^q`>a}ZH zvU;jpv$5<3w>;lXvkh&u<1x&qP*2Rox1Y)OzBP9|J9_fg&=p@fYr#qqG0pWKv6l1I zmvTIzm{p!?-Sr^B`rXOv^W%pwy_E;8T86)7NAuNh+)D9u$Z4zWtS_NH<> z-mq5QktE0G2+ut+cv0gjM0jZ80Du2ke?2+T5)seIS$T)M>h$=CZ-Zrp}2q!8F zJC_XS8n06ia{BQg=KK||^Y7a9CLye*RJ2(aLVp+Zf+-7i_{r(GZI2*&Gz+kit~Teq zcSU|UKaBG1Q$A@66F~Vj-A{^m#Q~*LR(k~pkkce+kU0DEwBxT(1B|i53lGdo#bF) z4JHf;{`9@d?sMYIGNuI2n~oI^b~K=U#HVi)62bTDUvnq>*P2Ctt@#DSo(%|-odR?_7CM{DFoUsvC@P2ZAo6%fT}S{x2b2Hh#QcL<`BHA|m|MoQ)dQ4dD^sVe({AIB=yBgx9)l)ylou+!96sUDH_Gw$6 zP;&25z^I(IxDr%=*p%Squo_+SEcQ{`A)VrZngsfrkM5bwV5@#g|{ zJZbWH@?HhQ<-CeHF-P@8~L#}#I-ExnC%O&8jVRcR<4KQC|RORlMk)Yi+4F0g+l

YrYgQ z(i2YHqkkHcOnl==?)YPsPd*l;{D@lOiiQcqZGycn+`sYHY7L+DNCAdf@)@y2vHy9{ zSpH&{N=)N6j{2bDbHkM)`@K&DQ4GCY$2v(WMGDSfsfMS3;3`$T=g3ccG`oYV|nXxB(2fNr5P&NFP4mW$0UsOe| zP3K=Jkozl*bBhP129=+Wu5(R!vmRM4W?9_4L}-%mt8oM+Mf_wgF(`&I8%iNvbM`$R za1U^B?oXg^iN;Z9xj;&J1s?xiB_KZbY!9*=$5l3$I0b9}-o&O?m@qYbtq^)bcWeVF zJ=16IjouA_6ptD8`Y(c9Vdjb2upr72>01!8c3D;XD{=+I!mXu#%flEYkGD4#S#B>2 zE4%4y=imV)bo`0iQ{jd6XZCa{59bF%Q}NpgP%M1{b@#Mig4TstzSUmX3Xe-?D>DDo z-4l1Sn-a@3Ep#xs`QtZ|;tw%8n_!C{Berkx>pONn+FSqAcZ8e=L#FI%-Nm(!LPJzb zY@uGDaQwYp?V(o17V^TDh>`9}sLKlk<55#xb+&#Fwf9%F>6TCKuv61}51Ps1<}D3x zm7=wSXjJp2zE?oA72wXO74{s0=WWJr(;&od{!BtLKUeEIc582)m3!G5+Vyq##lJ1j zMQruq$?|x%Bi1~h6kyPe5`m?1{*b#Hd6R0)olMW4EoMhFWfZ$n@%k)&?7Ly?6oe;{ zf@-a*VvS!8*WSELvsT_90={+y8R(e%xz7gLJ|($ z8eWI0UdF+BkB6Jx3vRfJV&tFOu|DlMe{vkjpE$naSG83jpY@|Non`N<6&&o(!eHkurDv|h_SRJ#+RND!Cqjw^Dt=zG zB7@!jxl?Qp42=aC*kl0*{kP-@9cIkpSo*mwIa&;B^rp-G)5My#nBLDXAnG&sL>>YS zGow_=1a8*+WG2!DoUm6tc38UOlb>f7K&|-OmE-iABomgV9lkwj7ftA?vx+UwXdVGn z*H0OAdx%MBjsmQUOnVVP`W~l+=l!d9D3Ka+>=E6cuSug!CE44Vwi4wNK9#VPDeG-G z3DLD@db6%KOdk~ z-o7WWLywDrFs4VOr76AnGS@#yKAb1Qi?r-hsdnSuKwS(3dAq!Zb`MDlvr}w&Y)`SQ4+OFX| zEv802LVDO{a3BWEYS!Y)zA=lUo7SM=eNV)h7qy3;T$O@JgP5c`MfddqP9H-(=%WGU_E&7@M*$4kGV-)j!Kul zOuAHx^FZEmu9+9}TKORRgEx^grG8X917dj`cj6DItz3kjK5FVPYO9-=OGOL^X#xAf z%5)6w3X|ff6^MXICBJ|2Y5^UwYi(xJ#5ej#2)6exP$k5K5V{cLvd6<)ZV_-nO^!LI z)l*5#Vgb)#@JP0lWmC^-(yc`#jVf5FWL^Q4F(UwP?;W;{-fAG?m39-?f+8;xcWO#E zcsWA*Cd>NK+KY`YI|I6Uw99^@a%%EVXbQdHtP}wsmwNV-h#b(gOb1WtFx0?N{q2@L!+k+8>=Hz*KM>;9|ah59d z6?3uVhiPJ^?35(Np9k{hKeEK{wXQ*erZXHmk6L1k3Jtsm?avB>y4jjzAw>nR!RhI0zTY4Kz6)Nu9G%vRvl#5^aV6@bJImB$S zopsi2$Q-<-bW@e#FJKSJn1{difP+J)i3s|u_4NC#eo!#Y-7ml69qFyV_aPD8$o)%^ zn57fW2G!ri%@t0yP#JHYv>x;?GgEoh>`*p0bpiTW_cW{GO*(;dWV$9hDLZ*vd>`_v zVA9Mbiz0^UM%Mi0qw?#}=Cz$yWVC2RVC%ha3Hi!aT4-d@a_)hrb&ayO^ zbv@=GW68e6h2FuVAu-G6Pb}{VvAiI51L##m)n?NvEoz=+FBvzQR?mvj^Xt%CC|tK( zXFWb?g@hFw$DB>e==PSphzw7OupD?XZh~05xka@=fnKT{OLeJEb^g1xL=(?G%R-d5 z){3$B@|$l3&X4Llq+&%(JF_={Ba~sCZh#vJaybAQ{llyO&K)`9|3fW}*riiz#R)ypZKbE1^}Km>{>pYZ&dO|n#9z|;(GwA+7*KL8K)+gRv3i+-35c%iEwt^|3 z(P&82*jx%e5CQ2R>#ut9kdLKvJWt>Wv`10jWl+zXpl zWe(CKA2pBoYCagzmPfeF5z+9PdEdJ5p3=-nzH{0}%~orlv3{F8O5$5$hk%w;KRZ(= z^Hd>K_JNn>lu)NHo&zdZyQt&)XnuBRbJ4OjfWX`}ULBMgdIe1GncB=}7+-(_uR1Xu zs6Pi5PZprEXR+|vpVp@ZCIh>FH=ON=czmL<$unL@MxEnrOemp$JJ4w4?%RLsCpJV` zjw<(t5_0;4@3m)A0%EFZrF~Vb!3Qt&@?#Kf z{z=pQ@R-{A1{O;GrBk&>`?RGpxMg3Tep@| zQ~ik6-4+!u#gPy0dfxPAsg2M<3A7h9Wt33`v+swY22y>?ZB-#dA?CF@@tbS`vdc1> z$E#M;58e@4cek|VqnS~ISoXRcTrg|B%xpqn^3rG#X8ZXe81yu^5TrHNVf5@C>L=D; z1M%bo9M4d^=MQp1$$zJ?Kn=hr&;hA$SG4x=leP$Y(h_T3(ho=8$GJ$NO27#_eAaYS zeq|abEs!G7Y}qKnljKtKF0Yw1E{66qOKGdoo2?&}eF6QfH_fKCnfAjbioO%?40wn; zpwi0WHmnFk&^VyMtz*6El<8?JdKK2c?1vYjkR9*^`$&b!+xawMR98S;<%s1W>G%&S z_i~DQH5G*-;n`NKYgJ_MvK}&irAfZa5%=Yb2PcbIrox6VUJO< z#QC`XzM+HzkD(7&DT5Zv>NJ3OClLWoGk3Zar1ZogPu}%JmqF}3GZXbl5o?438!U2c zg8}c)9~`8W(9%d6Fel(H_LPhq(;l>l0wbJn(ZEZ$cI z9MV7)^%zC9f7e}oa+Ys%CSl5TFjp+gjmSR7z9sLA&SKqGNmnWu8$0S()FU-=+&cDO zzy47a{tFpaCamAeBcjSCz^7mnf9vbbVwM17HBm*VR#GMTVXHsHeB(_Kw<=SUba8eb zYYhA6`CuJQ+DT7hUFIr%Q4({xu`n$+aZvW3Q`P{ZA~Wkxy^TGXi%{u3Pm;o%vvd9+ zWIP!poWg$U5AN-Amy?nKHAAc$wMy7!!qTX`sBv!d>8B6WXET)3K=aHeZtqBArsug= zDFWAq^I#7}T^nGHX&lQw3+y*;hi{?8G~LNYyxKqGUF7xQPARs0#P>y#Q8={dzI>=E zO~t%>@6xh$?{aiWT!&OB(j%n5#ifuMq()B0qk883@g0w@l1>3Gok+41e}*UwDd-_%qVw?o7||FbNY)U;%$4(pIp?hiu(7x z017dCY?_gT>O6|~!(`(2OIX6L--wvV^E7t{V^626BlvOg9yzF0AT4|1W4N2HQS!rx z7w$7C?-1#=v)2^*!8w}tte3MdM}TVbrg{4p@{9YU&I}k5r}vB{dk3uy*dCo<iko`R7B?+M(I_{N-!1eDus4g^3*U zV$d^Hv_>hq54sp`z01XrFflf)Llos{Bj#UkTET*?^YHr2 zy$20%f3z7j!)GJo250Q}1eel}sBEi5Am_G99;F7bA|hER=YPw?AN><Ne!sSdMy7|U)$i{V>hRPUj49msDFek zv3iKMYkf0NOKO@#X(jQ*W2No&=w%LVYX;tO7AZ@vU;!4jkG4y*g+EA7gzrH9zr2X21L+9dykG; z_J_ZCoB~YX1*wRbnlDD5HnK{Bs$AZE_N@7SkqFdu-n@k=+igp*+~BVK+0>GXD0V@S zx#v0iLnNT%VUGjy7cW@>RcmOH|7pBKZ59ujirV$v1rZQ(M{=lnv2eiPHsr_7+#imH%>iwLvKMmv#AedpOQ*HciBboK%Dg zT%t)wun|i<$dify?B_(IaXcB@ZKw9$y@UiLIu@nR{Gw>)Z zw^qySwnxVRi6kMjmo5C^&9!DxTX_(OwL@=}(C91{puVQo%DA8?E3yy+UHJqTp;P^1 zxCx%DWzIj7z$Ut+;OtAW5b#^QW{Pgfi#k=Q^YZ1YXu5l%>)l`;ul)!1^<>Cat_N$G zRef=-{+@CWG(TsX(rX%YxhaCJQr2kLkW!>x`j1-$$c*WTv(rKWnQNlndC@gg!|*VC zxQk&{c6$0i476AJR@j6r%AF>rc7DdNz8L`7m7Ufa@F?Q!?gWZ@E* zr@D=H7@_MGjS#egd@ARlEnmN1go>9=pj+FSR?0fgi2>FX@;?nqz=QlaZvL5WZL29o zf!zJ^&hgnyU&^qa`H#;@3AB(kP2kHxw9ktTlEVCbrWM+x}|)DHOKCtk>wpm z75Pir`7yUJB;Fp|cVx!^{ftq-YPwUV5}Fi@Gp4PVi?Rit%D;R`7YbEV(`?3f`zolj zIkq{V&TwDn8_ z^4rTLIal&w`kxjm&u?55n{EJt*LFZYzd}RaR-0TupD5_+bEn;&>u|DCi zt{6~M(Db~=j!hlC8m?la;oZW88p(yY@{_|FpwN=xY4VMO#@06O3R@GvUz}b)fn{}m z9J;Fq7}~*rg7+f}y33-CRm8h?7HB%iQFE6cg9?;uQ|J+)1bBCK=i72@V{jP3yv&`B z%KpP8&!;*FDY#L)P^P||rU(C*@N$1hA&tH+@!d&-opQ5de)3BFt>-D!@su!Ko{8PZ z>8}Oz`jqWgGR*xU7SBKcz%m=R)$gl}; z55RY3SLKSSU{o#x0|lPd4(VVb01)iyj_)1|bS%^((H#?SbodOjdH#9o!1mkAFFjC1 z;#63F7kK^8Fr>E@52&8Cv>T=(7J7((gihpGaw`x)pBe|`l_DamL||^&v+K(jH$@k| z?m824WA(i{nojU$%w@<13XGk2aSn=|-%~JC4aStTVMkAJh4u_Bd{5}pbS6p^%H7Q0 zx3Y;rfoVBd;O73h>a0W>a9O2A@x=FJ(_W02}z`Zc+zv zXmYwzLIn#X-yCEpF({z$Qi2L$^;TWSMAJ?w0^%^O98*dKYtgNyE%`P9)I0TXwc*cL zcNmr3T)R=b=V7ZB%S*^OqV(YD&4WW>qovt`A>wL4D1IuyQ-v5bm5o;KMOKkX!IcGQ zsxI{cN_I;y6YU+CrhpH*Dn-01y607MxW^EBeE}ryyBID8y?`_)dolh}X|MIaC`T$j zg6fn25$7--5wurQknQa)`u>c1$*`!Tg!7;tq!3ArFvyU+IF2 zND&Xbo4=5_vfh0?PLVckesi&aJrxm)#TO%A%0-H!w(tGZl@4|oEoCfsV}r{X08fH_ zvEIOfVCi<)r&IG}09vg0sQepoxeG%1`#K=jZD_h!RRacNWRG-+-N2N2Y@+?EU1yqp zfCqKnU?7jewM>$@+@~CHzV=m_O*?^2(M>JYTZH+flGpn~L{%VFK0BGEBcK=P`vXl6 z4&4edz|*Sl(1#WvyuIXYr~%4+bbhLy7aP?UWX6=uAMVXD*_B++O(0&oT|?7g4v#*>-kiNk zkIOCU7UpK@kN_}B*Q17#9d=}%^rsd}!auNt==WN6E*%tKkI#Am2&cW_I>c=(n84#S z2EWq%JfdLzv>`mKKab6aa$16NHp|WuAc}1nT3Ath?v2ff1J9F9@tAtQKe!#Oji%dL z4cl{5+SlLrqQOsbbcA9)**^8^oSG@Q1WV+bXzD*?Trs-@^BC5?YCpV_<-&*|Dc$Wx zTZ92_c>`jw$k-H*p96Y?P|1`JH)2Y7=P&Bn>x1XLKbcjU+ZUdY=pussL@stwi~ylD zmgyfb%2R)_;hWn7C@}iu(be2PtgufXR5?ujmkroWrd8K;0&efY71vn+4gO;Xzz$R^ zOK5wyKqTTiksVSM6LtB0De8b-DHVYq(nE`ajs<`gEO4q{4#7a{Wo@+G_!-mQ`B*`I z6yq14^8Jb;h^r$vCT!2ua#-E53S?W-@?z(J{Q^?39$qxC;P2X!G4U6np~+CYd8{fJy8;6> zZCH>C*k>@>O+;t+P+6iss$&AUYeCf6S79E;oq}^UF*q4HfFPdwStXbWVj;*cF+Zp{ z2amW+qmU|>y@dhSKqX-PR$q(DE=mwXq9W*fk@~sW5Kw%g0QD7lf;qc6p0w3&NeKf3 zg2Zbv!Y+7SGcVgHCz5V~p)Jwwu>Rwm)MdmC?hCESTY!g4a5zyta-=83MWy*jF=3SnP(VlM3;o z4+5f-@6!3g^bDB!ai3k0z`mV9@jNL^{xMPsJhioc>EPc67DzYzuzBY$rf)J$_jP1JL8N zeS88OoE75V?lc%exXF|^T8^yn4PL4eRtL^cXSwnV2;kjcP}Hrm1cp_Mvy1`!{ruV) zBS|>?D!brCiK&?nfKOiIO85LLmpt8JBoLW+rVUtvbhdaM^xcBSc`+CWtmiK3xd11v zA9z$pBkmYEs zi_$UlqFC2i;Bnu->Xkn%#^D?D+hL9dI|7YqFf@<*xbMF`oAc0x9th3Oq-OCL11OC% z&L8pPEFG{=I(F`SQ?GL?gV|ByklpeHcUYOGyL?i1?ig@scQ+3;ImbQ$_IqL~O;kI+ zz}|fe^pnx{2}{okyz?oKMvxs7sWH91OEanS7N+7t`TxwTpKuXSkuSYM)#)y10-k!dY8lnC)7)H|0?K3oV0WN|@MMXIFs+WbI)+(QV+E zKm2)b3jUqmHRxAdZW9MyPX)7=(s2g$kZ1Q6CNwGX(MuUL@7ib|A|Ba(pQxqXIDY?h z{aX~x$t&ceQaLP{7vGfXiwOHp z1U2C@xbL1bD(T1W+gEmNpp0w%(QVF*1y`p!0u;|=BVxEF!>y=cwK(J&>o9vLCq@X-j0`?sZ1CeSr)h_lP1T*JjF~PsliBmuay-x zyuPaxyb{>VY9x2q+UO@uTo;-)@hWGkj6rFIl?)2;*xR8yt#DOzlRfJK8T3>0^0kG8 zn%#c6&*li4C{OeHfq_+1kJ`h~nxoG{w!R2i66ol&lA&t%4I1yg^euQfTD3QDH+HDP zsxA!y--|XKl2tUZRzmuTK~U~1qlZQXqQj^Hv`=_DgZNn-#C}t~MmX=8v5lm?#2YkSMB`J2Qj;stL6Jw0 zi)nI0IlT_+Bj0e&wo1u1s*Ok2Of1u(U)XP}v@JPP?Bz$>d7KM^ybdliV1yC)-aLJB z4v$gZo+g_++$sF;6Yt6vAzqXk3~{^SAsIqXe=x}^NgCjGy%YpVDqpc{ne~g={qR)U zUDtW$f>d-V?|i)FH#w?R)y8VCoin)L49K8)2esDD3z{h-z2_kb+L-NzX_{u6%H?~_ z%RCR$5DbN)d+Y92R!Ce+VULGLs~ip+o1b^U+1tB)D$)>o7azW7G;PTp@&R42wk**9 zS<{?Rj!86o@w&~pf|^4R#P{Y|?RhKw8etofCE|ZQ0+SXCVTM4+A~mPO8aRS)4)taa z?+%k$U-xc*lP+mZQXDSp6v?xnB$*~RLBDhbd=_m8PNScC;VTH-@VtJZ2PBY2bfI|Y zEiWnmrbXOSMcwcQF$ytXG--6%;1G4pIa+BSDyQ8W1JvAN0`e>G8DV$~;X6DA!ncOZ zm)kF}5vLlVdi_4HSG&u|(mUAxNW(7|gSfktF!7;`mwkq*`=f`fjYw&>YQ*^5c__pL z-k^OBHl}G>xypA2m6>6*j#M6I+3=+X@_6_=w2z+a>-L_{H>BZLM%};Hg{~w_+!J*R zMEg_;ZM812&vvE!DdR6qx(kB?YqlD>;s$u zm(EiIan`7Y(e0Hg-IXvj9f6TQjgaMf!o=C1`gDw23_%dyI3)plOJ}6f?gaipj2H5| zKin@$-S8%DO|}l3Hi^uAM_nr!@r%fF96uhI6EZ)+iNYhsOD9kIarh3tVk?x44!cBk zCFVOy(unq-&XwU6WK_QmQYVfP&_yz67fVm~NE7Egq<6@HAk+C&^z}Kz-L8UPWJ*tu zzLJ4I=H7Z_j7&#qdc{_mXgt2PrklMLs>|fCNxPgz(-urP8`)pnc)ebm#ISkV( z`T9?KF(RdyBN}CGTDf4)H`{uzoRx-ahQHdNV``qNDZo#YVRPkprYXzuk@+X0s zC?AYJMBX~Kpn46c`CY`bw!gwWBG@I(m7po}ni47eL)y4mL&NGY?2 zx1ASe1QBPxG0Mv?$&o>y^2S+B3Xn>}*Y%UuW)_n;kJt%ltP3A*HPw(=NTng_W#(+@ z#bu2Y1b%<=Wj_z4rbb4BTj63tU{Sx84Nq%)CrzAczN%-cwcTpxbCZzl&7LYlFvK`{ z@oe()excscy514rXm)1u>baElCb9V^JB9iKrqy3k_C8IP+cp#$#LY->zz(x9RN6~A zpE}ncA1cL)5mP|Pqusc2cCEChcW%1LdOUSD-NZa9%ob;f_SVcftkj;3BV~u#r45k-Og?p6Vzs^7k9Y_7plrBTe1l6QWbG-I&{V`D7rpEIe z6+Yx!N%vDbjqz1z?u%!mWjW7mJZ@^1Kj|z9r65e+G+MD^O0gULjAY`dcmZEk_PE%T z0iC3}p{}`2y+Qc3JyJR8+ZMr{NJSbAsbQ0~w|JL|bpD&P-eqsX8L%{?Q(5cxIlY1klxo-^B7yVnw_>1r6BL z-@5-5+r$HIQ*RsZTjF^w%Exb?5vFtr>BkB;?|K*a!VCx=8Prz+!wN46jlQz?0v;MY zv_2x~WOWXS%dW_3WDMt&8tnI(^jC6^3pn9WbhDM*;>zIIhv1fMy% zhdGH1>Q5NdmqfiJ_`=TbSn;GGq)DNHYXoU>fty&lWQdi9ubCy>nhFE==Tvk!kI7=O zP4;GbAbXdJI3QBFnUy}W%jjmHVcQjX7xA5nH#dFcNywW632q4MDaf;ui3QRJ&NuA@ zot8i2XMsGs@x0{6)Mt)vv($zfN++7U_S-9GrdEcuangn#gpF_C4TPVUjg+RMlZ=xY zmkD2aP|`{jBbzhkM}LVBk{X(N(PPa;HBM$D1Rjo74H=`kL-LUxtM*i%&?l;RH7h-z zkKbpG7^NdeJ&YJNOMApWkPAgdp;b-R9@>pRwC`j?GL+mUN7_epd|^pJW;i?^*Rk#d zDlImwtbZWaOPx~i_od$teZ}upaJ+&uUYDM^yUYTC! zaIN4!$Q#J{Cx0ROE-3M7SGrFr= z#`GMtbrYx%*&?}XSMN_|{iNTprGl#Zce50S=D+OLimJcwLFB1M#KGTapJ?#{II{wJA^SZKQFmpvQ%NH_e zlafCBJzd3n51C+fn&$KmZO9&OP4dM%UQ`mxRu*A`5s*_HjV&;MOS|&x()cUTt1Ank zbiQ(*;wTn*wuKA!3k^4BWP%_Y;#!7F!fI)9n1z8dpr_}pJUPh3M;l$v|JvszYH|h& zg8sah4~-GV8jrkg$WYbZZ(nA6V)P19+{8S{Anuj%)I(R3=?;VVyOy1Cg5)%w`?d*P z1ZhPV_o=-NE)lBXNf%kv1lL^ukdkouZ&bQZc)AA6|8Yn$6#4Ql8kA6yoojNKG^ipEy2ga(uy%o-{GvR*(72JMBSJDVMV)xbI@# zZyg(7GH9A7~GVT?{g$M&n$ zS%0(Gc?)^4eut?Jp*8Ok?ei*~%R(pf@D&WGDf69^C;*urCj++t0>?4P0Q>W;RijS<1!FLO1uF8;i|1grF=57l5LZ6PCdXu-|YuGI6pqYzTqDKz9K{cLm9GN@l)5pmzF^I?=c9ko!Uy)6D|ZF zyIaVv3E+Qlf zeMhILK%bqAm9-*1Q=Z(sudrK_^I3J-LfU#(AUx~!jrRr~L@x;>C-H^voEFlmoYVlf z(tJb2;MGH;$3;U_-UxfW)aCi&OLpZlsPn$GknY);MNa~?~(^W+z zwXXf^Rk6al$)`v;LXWMt(U^{Unw$6lb{0rqEp`U^R4;6KL<_$<_Qa0VMP4QeixE#|Lm?lS+%Ny_epxEJ@FKxHaWUWG) z+>4-R;yavtr}8Y<5|m&8N*g)C{}Cz2C2B73he28~z7xSCM{0hj#e z)yn`ag&)3x>;q~FvZFF?2O_PW(mmnb zK_m?ATx%+-AR6Ztbjnaor{9(%@&70fNo0gQh$WbbK=G1~ePOO?%y4O_EA7n?j7L)_ zJ6ozaGQlF_Dm4?!q7#emVghqJrHh6ci?JrgW|fdD+7W-9V@ zil5Z!ngayqq#x8Fx*ux3LWM*LNz$*BaZ#q7a^q*C>kcpqPLrD3;0}bV48_YrACMXM zr$8Qkr~!E8c1bTK<)VP!cB1yxa9?zig=dWHHI20?Y1tJxErG^yRQUcTX&Xz8G4od^ zWv>W@<**>zTm@TxBxn3WZ!#K4C!brckR7M6c@&!TQD%@?xq{h9M7vq1@Xgg6f!Rnj zos#R#0*_`}0mgs0>iiLCo;To$NxNg?S)P@g8%QG|v*lQO*GF>mG--kG98Z&ZFrAo% zUTxe}NZR}ATDiXO7wkrr24p00r#?$Cr|NVPkcpD?d|#Ejnjy^R3!M#F?>ZZ@YiGqa zm?3^+!tc`Rs=27TH(k>wZk!Z4s!UaON+(W952f^1kiaV+^355nU%$9eGH_*1`$5g~ z2baFZ7|)4>lX(1nUJ=B5I;5aAtE1t!SM-NXm&G1BHs>uSeX zbN)0D;$5BB!+`AySqdnQcxbyptC>sg=2$xYPpOqK3B1%xe}}%ttMy5Q2<|ASJ_$cj zdEgEH*+~^Xl!0TRAA}*ep2Cfu{B{x7oY4ha>5MwN!eGd$unrEUC^BSXFhNKGg{ayy zalF?Lp1$?!{-8M;_;Z_9y!v6yM0T0JUDCMai(TYP+VKo0f*Qm%Jq*FymNMnq75qKF zG4-?C0S@(xWMY%VcUWKi9a{a)v;Lt0!+*LvGU!=}Ulb_PPYEIKmOo|80^w|h|^1mt2#-o$VDwOrGT>s0{1@KAjpB>z3a_H^7 zoZg8c(MR57z;09L9_1GXx0id)rA!A90#DDE)(Q)c{~c#!D^!896}VKkq}Q^RmOm?TG#RcSyjQ`M(bQTi#MbK~kX;@S$mu z48Kej9nxs!&k&Ovke1mbPIZuHptur9Q)4!7R{nAELvQZ7)S2T&37j6BIpFeo_GOAw zP$3%4t3?{kt4MpcE@z?Hn6CU@II!k+mUfBld5dIfrVQ1nsF zxK+>NpA4m;y?tQC)i)ibg`$0_avl4+(mhR%<+d}yCO!OxmMu+UD`z$3IRaVpZ~O?Q|} z-ciA>^$v+FDc^K!5KZEsX=^k(T*=q^|8}mfhN`HsAm%NVJ7zYCb2;B_1&9FlLR1Za z>p&$e+_;$|B~pf63;kVaunMf<&;hh;!--YNA|1#-kNXmW2L-MV+u zx_vfr7wMoMze}U5dKAE|3eNi*E<{q-pSD|iuM#nipT#Y4_*okMl+23!r^nhOlhe9= zlY()Wi*eh%@fcb1k+epe#h^TCIB!JXEAqZK5t>1e0*&~<=mh^>=uQZ85-xeWMMPUu z8p7IkizwO5{^*kbbiG{SK|0EcQXWGPxGdKO+&H&3fG17q+)tCs{Wo+%*iHtHQ+J*$ zq135};~G6I#HYA5(cak00tl9g-?WtP{d0>B`EZ)V&9V&u@HaB{|6kz0C>eOVcahbK zbi`g3Ex$yxkYz?D3nq+)1O-n#f#v^$!~usoq{Q^yLU%9HUIqI?K&0fYd?pxq6Pvi= z=i#Rz^H&qv^7EX)&yuKFf&LvUt=aFz_hX44mN=8TV;5X#yVLp-hw)_~(f^0V2GB#X zj}_!e@lyE*XAU{MTt%|hnSvleJ^#B9# zyN16CZTx=`^JFw~lxHIw>k6I?x*lHaR8&0xF10bF?ZfLL|92hzu0M2W-3K5SLS*86 zZ}t)xX_ENIn|Uvs7u$*HhLOfc#8WO6x-*|OEat3r)}uSi?!-27sv5|1bK>h6a+ zUfb`!W+m#)Tlo~;rZPO5Bbb=YTNr zO;!R6xYhuw)r>H!(2K4XZ;m)G_Bk%)W9DB-B?S2}{Ls$K8Or*Fm~kDHE~r6LlTdB>@oo< zgT{%=C+%RGrC6GanVXnc8j6__Jh0b0j z>Ibo!_ik(Xz*QAG4|xo$=>HDkNO?B8p|7$zbH(eXt6}3o`0>WX=5t6#VMGz4bk!2V z5yv@C#Mn&TTKlV>lq&#E+tUpypOFuOAPY^`gCL7Z>Ly@MHY&+|Jc|=Mbg%s#327U~ z?mp+}?bLit7zFXMdbxX+oln2f*blgbN{@@$q4b`ylG`h!G)yf2<-F_2@6n6BB!EzE z$=!)9GjrI)ttnR^WpvTUlF#AU|3@{pkxGVQ0SmLj3Fi8J=`7IyjKacwHnT3shY6|!xUQ&&IMl~(=H zRB0jx$BVrFHY@pgV~SHp^@!-pZk?9Z3qn`$o=B!j*5Q}Tx7r$87wWSmv&JOS_Qg|a zB~U72Sd&)(A`{;LaC)e4Qdt^8ie}R!V6>uWMcniq0r8!{)whL(`_s%)v0n)q5AOa~ z4L|V+^AB{Q)~G%wdUY*YxX!`V`KuwlGvo8H*9t%H*7HQk>ybo!4J5tcWAKy;pHQE$ zo0DmrT_^YRZ4a@h;&{$%8FeW9XS%ALi+3rGN(esWv-QxGJe=U?idf|RYGB;<+U^iZ z!oteJ=~CpQecxpG=DVv#Q6zhjysKKd_N%7hDdVW9D(49vIX$`WN3UNBmxF2u`cE%C z-wcbDNYp&DDVedhHO`V)wQyeI<2Kc*;uWnpNCILUne_kt>FoGB%&Hs-xLfzYTn@HO zN?UXis5ibF4;Xl5)7r}2Tyk&vNk?#_8oh9%ro54anMcwd^huCr~|GDGZ6xs;4y!CLyoepnPh>RFF2xYf1WjwSa)Uhu}!)3!Udgb%+MJ z_{;h1yZ$0zFlhzFiZB4M^NoAj3@ARdTM}w)7z!DEQNxp2=t`51y{kFtT`%RKbgSUZ z*v$6!Dt%jTaECbXvO<2TQkDR*r;Sf}b@;Ari7vVO(q{Az59A+AGy6f-aNoGcoXEpT zWeHC_$S}JNft)#NBbb_+^T<W*pd6 z^xspk&QYt$s7~Xuqu$Q2p0(4yiVRw`09k5Q!A)+btfU)AgVq{jk{B6kS@I%wE}Hb} z+1PvO%AXvqeCUrRi1*2M&q*L%K1ism`b|tMTkh~wiL<}XTn#~F}v@tO~iT{^`p+?7CltlKuCbOE}Je8 zr{}ib#9DA2`F(i4+S>JOnT|e?^hZtjSxmfppWhT(c zuk2oH%&|*{V)`%57C*{GO_W@U>fs=;zJ&Lr8+kbTGa*TIc`u>3{a_#rpF7Tqf3Zxo zu=Sqjr)^R2A0d0OBkjYU$|U~0Wg`PGywv+M2i7okc`-BhHxyH18N`6-R_#d&`HS%D{X1j(jCch?!71o})%fwU(hn`s$qcfl8V&o{lqCg-52%c2V zcZo_99G{q@`!rz&$GF((HN6@5PHONfjBGS&4I~~P1QT2(Y$BQZzN(@0-pCvk&B-U8 z2T8qgPf0@8hix(Ivo*Kh&ydUqy7KELksIc><11?j?Bb&8<%Xkq4VztM&9VUObMqcg zqfS}ea@|ENR1h&C@&EAk-GNm8?f;25LRJ~sBN7tI9@$Eglr7^BvaiG*Nu!BSZV!)beAU)Quor$85#KO2ADVJQ*e`ZA5I(l>s26)2%m7rY@fJ%; z??&OE@c20eme*@mB_Qy8v4~ZHRdnYoyiBi*9A|QDi8L53UuG<} zd8evqQ;c7}OVn!KL^{<#rB@OBV|uu3H=~_LNfGid3L4;~xfB3l>=b~|!RD~*H{0b7 z^|=aW7+-9cQ`7`M`;*EfqOw$W;u@%5=Nlgt;*!L-+RJJ8)e&EW>{J-YOq2Osb)s0= zZ*pB>R1L15sg%1=f%wE=<#&~GESh9L|n0dGEU;Z_%~h(0Oi1+@t-lMJ(}J>cVvr zfmp?>qzfVgN0&Px9TrJDw&Z>~A4(XiCFO2nL^T=Y4O{VWXsEWDc9PmH6<_>@5}*87 zKSgISU70^6c_XE%HNT{MxEs%fl!5R|$nAq=%gxpM;)LhKAz)jSWd7`zoXsfYUig zky(BR$|?^gyX79{)2T+e;Wv_-r~FB(M1Ke<&d{l)XXy#&?f~;^&V9X7L^2>zhu#PM zrv-zz+s$;S);^>qU!>?zshMF;jz8ksH)=V461Sr8}32eu~NT~I;F>5-lyqgF9kP0dD5IC{=<$# z4su&1E~&TdS}VS)F5k6Ouz%g?IWs(d`Sqkew9mi9-t>vmRRx;3_xFpg*^Hm9m8vmB zU@A#6^$af-`8Sa$iB`rTuKAoeQ}+LJ%Cv>fSWe1uY|Bl&Hy)ekPgTlJ+6Xv56;jB& zSwr-biR+|{jxW2l!t^Ibq~~2xYHw`n^^b0TEv+}Wo^)C1M!Bg^>wKrokGTEf0zZ?p zvMGIp*}m#x;H0(wRTFI|`EhC8uSKb5)hc-O@G1?y)#4Pbi99;98(gTafsconRs1&LsmAQelP6 ziwt}cdPlVZ_kDV(Nyq#1pVdq;Jz7q?%qMIT6MvI)H!A+(W!o{;=)4nw)zjNwbH6g= zHXF8TwK?@46$^Y7Gav7!f2%p~sj?r@L$2%apm523fHkkqtpV>6XB4(qZ_uPB6p+@Rny=Kv{ zDD^SU$FkS_g`vAi0$Kx_FiI$?MKtV#k)ID35{-N?qmV!k))G`?;C~$w#mKRn)uEAB}K6LSXbuieN?3;G0lV zD}P<$>YAdh$nHDt*MB)l(H%<5zLM(k|LoEDgh97BXR_?V@Rd{p;-5}0NacrK`S@MK z7aA<^e1gAbNN4JS4n@q^fKEw3q=)0Zc|8}(9eG~IL}ne}P7Q2SdipIpDk3<22GhoW zH9&w6w4v$}{Kn}=M(SQvCGiJi1mY_fv)EW`uyEu%2=xJ!8-N1VWY2 z0Gz)p5#;^@y{a)vl?|q{@|E%z*CVq(KXyL%j`JR4$B2sm9phVKYb@W|w#~IoUVSUO zzwl^A%YLH$CD0&|^Acx+^^|$bYKpfE84UZ%e;NDsf-c_7;}$^hdJ*xn)75J!0`GOm z!xQpLDJh_=g369!{On3=90U9b?R;KU%_a4B)035x>K&~HkTKX7?u*7X{lT|U^?7wT zole9!mrtO`d2MLBI+jmX50hP+eC0|1SY0CY5x37dbm|oriP)8>QQ~9MIEhtpe_GX<>1lUSB^l~n)o_R{mcj5g9#Vn;o6`!&nuOFTMD3*B4g~^6Q z3;+j0vjHcD$-P=ZDIte8`t+r?f=c@Fm+o7{-+X@169xIKQpf$Qm#OWToKJi4qi){Q ze&YNwb8Jt4@X0*J=cvKpQF%Gub7+Rn|J!D;47dIi4~vOBv(aZwA`t+)bmYxZ*U5Dt z1~@!)H2?PYS+G785yy-u{&C`N^?$9;yp6Tcp{990C*JG)-bh6#YY0tR-6hnJzxf{f zs1jAE5A`M^-aQb4xdB$ja+uhmsro@r3b*-oq1ydB%Bz9)Ay#^ zw?>gaBI<8H-~v!0cF;cO@6ny1FKM+j@pfHi(ADD;;e4q1YfB%}Ebx@!&i zd7q7pKNCcXUfUYKM`d1I_lHR?e>o)~I18V6m>RPDcHg2NVw+a#cPvaBdyGasD=mN~ ziTsv)D}O540?&uDvIs$YDfKN+Z_-kEa8+;^4dQhAP43H*bPD z4MihZ_nd41%@n44r%WP&O6R$LCPtq3e)LQP-*xtE1=e;qWT~IfhkGRuyo? ziUj`O7}PZzZ((4jhW|d(x7W^Q$}{ZgQHWsHSvd^}#2+=`MmjRqN&N_b z_lmoI!?j-5EmovW<~l&j zGrBqG32@NxGYbBcaLXo-E*-zIX_H;>`!LCZKf43o6SyMt2~csW4P~uQ@(kJ}ATrxS zeYj1i2wkei<(_$Z$}*Aw*VerE_4;(hjG=K}-h~l7K4#H(x*1gi=vla!UT6yXxYf2^ zRgOc)@`a#bTtpTR#QJ{7#hIdgG8_oUS^Tbw$F=Vt(-RiCdX3hudpXkmv5EfR|ibz zRx4Gu&hL|Z()Zi~7)KGgAZ#6g6z?HFI%Q|a>s<5<`)QUnnL{RwU})tK=_Ha&xp)qI&Bx5Pv&Tlj^-8K<%C_1fG@$3Jo=Q&) zJ%FJy#oj3fFyJcpCU$&2hF<)=SmE25oWMRsEA45C(b57VUZppZz)HjTq9xDH`?esR z#N;bn+-oy1n9t|a*wY)8(YFg#jdcy$*)+Ox3`bc(s@0;$EWxA;n(4+gDhoud4_tFX zMeN4;m`%ICLNu|r{6?(b5Qfe0R<}F`GE}-ID!i`krogbJHnGfUDp2MXVpx zgY?8{?diaKxsJjR@4PLFXi;o2pn@24fK-_|4HtQDy0>EEFNio}Kgfrkg7LfU3_Xc> ze75*taB@}Falx$u^Z*p0uII+o^ashK>br6-&&g3rvmupd9?l6dV(LL@~E=$9c~>eILt0yyib zm@?J7l*rpwC5iDm5k=)z$(^tA!2AY+qg*kDMV4dl)r+8A=lbu%;sl5JbTsYTzwXcs zo5ZbX>sGhl*k?J3>q-@QrFGQ+V`Df9wA+v!=c%c!Dg7-{&L5GY{}QQ3%6IhNMKZ&S zbW+Qjo7b!QSQlp`a4dI4hJBZ#3;Z<){SH_szQ=jl$ZZotR_x>8ELT0wFG!bxBleAO zv+8SQ($ws>ZiLhv&IbB#m|;c7Z5Bac9=f!|3oEbhJCGs3Dq!J#t9935&74`u==I>OJ%?H?u6<@{t8|%=5%r0 z6!&NE^X;fU4-XvE(KaLtTpNsnF5f5aV?M2Ymdja_kd&NwxZ@dr4E#)t_a5Hn%q_aP zPKMB<6~5J~v8LBI;@n{6ACKmE2YKIGc#|(ZCz24)TIUPql@-$EYvg3%PmYa>yXT+u zy%+!sKj7-wJ|2?tuhzZjak_7}pdLqtRf!u;*;;P8kjuD}I!rw4J3M z9Xxya9uKWj8;ZTZQ;JW9laKSlr-wbs6h;4d9EzyC@XKd*#m}z?{uJUm_AiI?258YE z!+t|QjXb9=cy00ua|x?0zy??6^;dtxc5Bv8<@q8QBsmxVH!BEMW5Ru7UlwKqTv9d}g1!Biscy zhDome?HbOO@cpTx~Vlz`N8oeX>v?g75A zj5pgIc`FdE)YSU410L`Rrb_>}4KI7YxUL_?`qQbnrOmW^p|YRmq-_DA7#egkt0y1z-dGdn z8;3??nk(MRU5&Vs4Evx7IR0pWV|f;Pi z31IG&fiuZv=yEVD?MgS)$FFkjRE)2Njq=)NCz*i+72dTyW*dF=rEXt;G|@7~3?L$y z&j3)#m-V#Q|EWMo!Y1i8bVmNg;{d97d;*|4!ETk6iY+BSGeWuaW@|m6ZC@Pu__Z7Yj zXwcg~!QzVHeOw9$EL?-Q5OqW2i(b`yQtEg4YYX43;%_YZ6-~V2diai-FDA2BM!z}f z3f)Ax&4-U(_XL!~y=;pm&KD+zfh&B%NI9h+bKJzF2~%&C*9?C50%FPFTbn@ps*Oje z@R=`E>ZIhYI1715w4|_!2koeQS1?1Wl*-vBiJ6_N=m5c>Mj3*FB zfRF$#ANpL4;Txbj?Gnnkmo#Vl0(n_b6%Lr-)id{-Z^xvx_ao-;i48QB`f9{kxPyG3 zdJDRq1TAb8wjJ?k==waA~jeV#*~@o{TO*Afk5;0o0Q;et%e zCP^I5@-gmFvp~*IZ=xNdk7KFNyx zBXwuA;P(ILRFNS(zXN@b+A4opqBQs4mgs(3>K%2I$kGH@1;q!p$^)@i>nJuyKjm!V zkGNY?@*wG6dL>vS01Rc_nY_W}#^Dl*v{PyCq=ST98IN+NXVV$F5uq^#Tfw~0)jVhS z|6RV9!Br^WaDDvO;R?hp$$%Bk(Px#v+4&v-2c3_WqF*X3vP?4lkX3O@I|XWeuS^7j zOfx*MDFCu3ALW8F?8l#)lUDlV1$`WklIo=99gjZwpsVHgq*fqSgG=@5(7#k$m`@xZ zvN8X04Bhl6Y}T-YjQsP^s|k00hxTPLVJifE6hYDYx4w5ejUIyYSwYl0E?OPJy{=QQ zFOtPiQ|&9qj?w5?jD~DHdfv>2Wd14gz$)hC!CEh3%o(ia1N7dmSLhe%$@)JncvhX| z{Aes-UFh41S@bnr`b9k^yTQ1We|DGVIP=!3R@;Cgi;4?0`L%AFSgXcA-j$HL+rGU)GcfFZ!f9UY5c7XIUb z;$!rHH`{Ul&)378(Xh;8U#y+wA77y)_6$(ek_3Q|_>WGGwp>zr#wix`$J6$97X$T$ zJBfcLtyl$q2sF^)t<0#>#WNGDWxT$w3zOvhlgV)~l>maZpuxq5814lSH7Zpr3$6eyXqy**$EZN|nalXkW|>|~T6)Oo zB9|&&fzB=#Faq*DJfNnMz-rq2S3LGlW4A7^69Bmw!*4JHf7O8|Kaladq4BwP+@DD9 zpPsCBM$`W1llY_rAg$3U$GJ)y6#)6){}Pv}1>e@z4n6a#XIB<{ug+7mw$q}LowBZ7 zJvGlw|L1u!9Y{*|0Q3Ae*hVyxRc}!8ht|;rO6#)(cekAAuW=vr<1rhC@asP6X9&q{qIPNjVuf3P+I$2E&hJ8v}&mx;GO9X;PbbCd{SJhY0&9A z#h>`&_ev!J`PD$wS(cUhYc}{x$$v$e-SP#Jz61aDV00dy;qMgk8yv!)rhk}d0M5ud zzzuDU*7N^c%W5k&7Bj>L&@#4F>#v(Ad~`*z zaX+(kJSl_m104a$)NRo-jroI@s8I5y0aoA(v)4CxE~M(_-%2*EkACkP85aA(1Jl)I z)n^&=e2C{P!4(G(RsfX43;WMUVkk%>b9P;P?eC4Lj4_AOJMN7;fUc=5{dtpT^&lyr zL5jej1sR4H!aT`tTDTX+sZ~T#)|Z25Y|31KC>5dL}+f0&lrV28-7|-z{Q?v?$G4m_GP6_v_T&9sV6oLY`x!|{U{hwc?P*|mdD|jS+c82Xs z$mbV?PG4+`GLEv~QGev@8;w1ibcB%4Ab{fHH$nQJ%vP@VHih*8K$Ug;ZE^G&HJ&z5 zPdFKt-TyxLT>!WuNT<}7LfN|-SAKSv2!3M!L!LJ4%zdDh=Ip#8Vn?NIAY! z9i~bdj2!7f^LkDT*rh-7GmaM0s=Q1NKY9!YtZO3+q z)2G_ARz`$sY4qq(s}U8?`G1^1!>2iTEkvh??96tF(WwTzh)*3k2;|97X@6a+iX>ME zI=Y-31l+GP4W$C{I5U3Gey%-CO%#Qx*H`{vX`6RYK@)BXwi{yi$&_A;FD1^CF|4Xh z*FpEfXhI{-0e7}kf${y-|K=WU(?eF)?O_HNfueBr$T33r&>htTIJtZ$Wu^Mj?@r3b zms!M(HNt{Lx}75XBULDsFXu@wAfAeZ#@$}v(wvE9TiA*sSGdkxe(iI2`JIm|qX%os zdp}MCFXF3FEvDNmXAmlg_5$EE|HHj1y3RkbW`R^)#=A@G45E{0a>z-Re^g0qHAVIQ zWfc|jiR=6~nYh|HoWy{*~BZRJfzU?eKa^-VLdnwYNu9uev8jvi?kb`r-Z36&5x! zuJKg{HZ=g;NQj}ddoqfQ8JCZ^KxzM;$tmRJL$!e*h3(JJ-D)2U;kTaX3_WN+S9u<% z%`>fxP7OFr4Wk1XO)#sr8V))c{ka#d zfGGIZxrbCTKr0H{fG1hIGH~A&KEgOB88;D!usjtt$~fj&8#qw`?j+(VDRo5`uLmr8 zb`$(1QMjaUhtqfEw0b<3nvvGbq3XHDUVKbrLxNauWGrKBmxdAbHEp*C30>|S2z8uE zOP<c7RjWH?JK1i;p2Vzg9iC7+ zSqEUlJ=g1sjSY&k_D>w2)oyyaRf%qX?eOL zlYSQgL-zagVh`DtEdx-)$DW(7{rzFSBmMct!xLDn_a^4x2xfXRvRg%Y>TFw!$)N`% zJwTR>_#f!#$*2?KQW`i)V;ZQeGtbg{EPAWh44 zTh#63c~~o=HQ({$&mr%PuFBw5hNBu5Z8jsUa$aqRgZkYYhvSUR9f& z>=DMiYTRo;ze1FHBFycKnOY8|TyMrwxEGSIArXEnYd`#2W*GNBAGGM6G?(<$fZrD* zejx-tZGnyEAp;EaUPqqkOwt(72)ng~07H9Z?*z5!k(fc#uf{TGSVzgojB_ppgMaT< zkrj$syW!!fPt(72d1Vy$=0{5nQ^dyh1wnJ9?@rtSvw@C6_<^?hjXaC zweauOt+8aKH&2`p;mlFx-;GQ$wdVGs*$iCf>qxz+s`jH0yIZvly7F>|8HjImWlaIN z)QLgJ#wL@zyL*8n<`wols@qA!Bx`bnffUS|LjrScT`D_W_Y`|}tPg_Z9?$4uI_r#` zW+sY6k#BsktuK)WIdsq2P73(f4krS7w_*r;ow5K_<u(;cgV1(iF+DaL*R%9H}J_wS9QM{-(m>i$;Z87#z!+bd0WtvfTv%f~H#Y-i|RwINq-B5?w zb|WmC#&r$o?OBIS3|m=t(ri8M(>fCwEKSqY&CsJ3i|Y;+vIIW2mnb_w_n3|BsLC*5ax`{MJ6z?dJ67_XQ5t5`k~D2y^gGV96NRmbQ)jXBg^$M+dw4P4> z5b)aHs!gxbl|ER}vkzUrI&tYJ+e73btLKK_ z*O*c1EW`+IdjS5EEj!y}Efzdz@)P1Y_w{v^>#SFyU}}Tdv*jt*`LDE9ViU1ZsjN=^ zNBK2#r}A!c{+%&PyvFh}%l%#9WADF>|8$-xq%507kRt*{YAzs5KP`5y4~Cfz4;%Qs z*`3}I+qBt@DU+XUHT!uF?!Ae_(0AlaGzs55HHG1}*RjQyJiz{1OmRYG)9`Mo08x)E z6aRSWgH1=~%`E?s7{YFOhXOgFdf~70tKAYvL!Zxw2@nTUmydP`(oN=YxK}p`s(#12 zV6EIq;3%&Nt>uo~Cqo}h%%rt^J`8SjxyfX`*&Mw=GK)Sifr)(IIC=?bti^&RQeK-# z$5xym-_+afUlu+3h21=HwmuPu0o{mqoSfr z{68S~XMtgx!R}GxVIK19M@Ry_?}BBgLT;@m9uuQwCgF|3&LznTr?bdmO>Dz6Ng?;9 zcLN7u`w9FxKHs*gPo0*f+1yMLTd+-64qKeh9hT})g^m(B@h2Pj8Uq)#iOx0g@_ddx z94*F&jE>Hi33d5|y;|r}4dPZ3zVn_qmMF*V)$30}M+~&2Eo1C}=*U>a*ImF%!u6Cy z;z5AVuVn`oLD&lZUK&#{3}`d~vd!NBq-OJ>FL%C6YmK(wnTqVZw-&N$dJm6lELF$#8$e=#B8Fbsh1x*CXCk-vH9lt zMsC?w*{P+N=P_@kWZ9+@+9ACKId#xnjBP2Kt7|x#n_u-Qm)<%s@!xb?G9P;_vgP95 z3eQ3{@fS#B*rRaS(%C+)Gc_iq@CzETE^@Lro4kG|z#aH64MG+n;aZz62t>7?i^pMQ zN2QZg0C;RodH}ef&_ufIXZ8z-&tT2|&ukWx3{PPg?n_PbF1R|wV;c_J=ts$=o6o!;MkUD;Z*4qXp^PqnXR zMSoNJtK5hyOwx0w0zI3&$xu6D?Dg0Ko%g!ZNkEDN`~B;PbzON{(rm+O;JTFFa9z`- zp)wDLX7j_#Yh0_|O`@x6zAL7Fnh3-^L}`|h-m`29ap(qHZ++@(#I)U)KMYx!avSOQ zA52Z^VL)QlCN@)KD?JX@wYzh$Llna!dO{g*OlrtsK24qDr4FOc%Lf{>5|pomwmZg6 z!YvOPeSQcXt;rqOg8L5BrChRkeLXzD}orc!Cn4#mj}PcTvurLonaRMn){=+TG}joox{C@{@2Y;yhnv)k8tD;EB(McM$H z^Ab4^&mMU`vgf!}>*>BRJ5&~RNn95QmT;^&b4&frUuxDeyLc^8Lb6jwn8qcY**l|e zOjcLh?l>RezExJyWv?F%nQ)bD4LctYthytAg=B3VRG?iczF5yt`XL^Oz+5+k9fe#E zqKg{~;fOtYe;>q~784fqog8>eAjB;PvwrA4Z6#DbE#>Pln*uN1d@bfoVY0UBQM#5N zmAO@bI^gQ&bjZE5xAkD+;Z(2OCY!O)*u3F^cREwl7*f}>eBSUjTfK1T?CPr4>F)8B z%_(W89pUek)%z9`BnZQWO|@dJ5jvc?GsKhmE@Ta$_3yoGsA9rBe`=hbaSM5jPCGY! z;OhijhZ58HzCk>;r4du_BY3lFR0W7PZbhVi;{3(6&L)WH6)Oc2N9V5{vk5WH_83{b^1_Ja(wE_6>a-FLMMl^I^kSAJf_0GRl3q;*}N07=y`9FBhE>i zNK13Lqn6ToLxC0LV8>YB%CM+Ri?^&z0dR7vsqlx1*H-(Gl#9r_l*Vnt1pmAqk#o<|%% zl2xxe?)!SO?0A(1;QFnr=qLPn*4Ae^nw*B@;QM9FT9dSVL7-8KPSJN#mMG0Kic|qv z&L*?IQMhMgp`q_H_YH+@%+AqJ6{WEeWZga+i@l&HW+{%sw!N79g%}{Zu&R7K`KtcC zaVRxTNm_!0Xt$*pEtWXZNrHko-ISGT%3HftOa?-e1a~hWXY_-Zd4D0*u|wji%o+X;^L=nQnO5a7 z!UYy&srvb?>=+4kSZl#?G^yN91U+r0(~MUT18nv%8QHw8jNpo9_gOdhCz(H9oH99@ z4$}X;WyAC;j@3QqKw+#&M1t}JRLIn18>^iXJA#;pElYp4>4KQkOwZtRh^e@9McBJ1_m zO&1i*2CmJ+JWsY7=VU{Px|N8C3A$vhRquSa$$Ec1O?IpMr)$f!wdlm=dRKdmuf8lM zTLdFJ!8XBy%7@)_JMM8G@*;P)b={EjKcO$#qGFtyaTc9H)fvO)79deJpYK2^uhX`P zfF9Sqx%K@;Fr5`=^=#NBp7$xvg#%}&4Gtljk#zt2P7sTDYyEC~6rDGZ@r4uI=Bx`Hp{_>Oi&35|(j5)f5GD9vaU&lqQi1$yOs z-KSk;EfeldzJragE6V~5(hw{p@d&943JkLAYxcQ_%lFN@XKaNDYL{x9&XVN!_Zrps ztyPSl%jT&Gh}f4bm42F}5ff|aVO=(FuZ(N-W<69HlO`h+*}AC*<_KH~HcfNzi5&-tWO zA9>Z9b@Wso%%C&Y zGur%oWWDvy35D-4S;JumF2-4Xq>yVgM11;i$TNk_!gBvg*)SEFYF+sOJNrzvUt~fq zeS@n7lw&LgF=7ByLAL>R(pN6g{TR^lgNXo6-!f}ou1_2dr8iZTw-RLGRmBmMV-eQ_ z*_XexQn4ub9tyfE63uV;cb*67c7mX0zSx=Llg+2tbhGn)zERj_+a*i>kRyd*2%?6* zo$+gi$8a_D{M;&Hc6=b}7FAymw;K147@Tib`E}aSVAsb1oOjv39Fi-NDxN!OczKVt zW}cjYh>cTSo4(f`=kuXTBjxZqC{!5a674_~YS&m`g%Xkz!>$JFREpAl8XV_?H$=r% zK*HQk<~|ZSWT>+VxvhgY^0PQ-0(%Mky1kv8dpLC;1#rqz`EC;ipWi5;WHqVsMb}TO zjrAY6{=$+B*XksBcf%4FLcA~ z=#e=UYRAD}K2LV1LdBsD(Jfy*$0CFe((DNRt+XY|U^WB%JW`Oap{jYEHab+-t`oA+ zT7?EGoPK>+#@2n#pC(Uj(-WDs`|{!D)0wKWm&XqnqxrhoS{Em`%IAQa9owy`WT}1* zSMT^3d-zYB!1LGQMs#s?7Y=9tgtlEyS;y7@qTPAuS_dWl1gpmY*Gp7t*WI?hAa+&5 z`02oqcHeNiWIgK#eG#Dg|7wytR)q+@92f+PqRAZSE(HA8PL2@PncHh{;GsWeOoO%! zX)I26K>?pL%i{rbe9k;#HehtT+O@uSL8po=+=_<3#Lz}-wYMw4 zC$HkH*h+Xs7GA@B*W|78dZyRvdY^4g-|+p$BI-M99bFRU7dEzBYNzfsw-;^$vtS%0 zTntW2@t;VJyA|YhPS|#UrgUzAlW5Z&))H_>!(_vn`$i{E=XETH0!JE$B8MqQarr9e zLH84`RE~6Nc}%QU3PbRZV|1nMW?fUp%FaUs9X!rNn#z&t<+F`xskN2a(N)KVz=X9X z#)^%5PC~&%mC3a`-{nK)i1^#jE8nqDfiC_KgoMUzm8My0vs3j6^bsYHHC&R^V3BPL z2&As3YJ@Th9D)aU;fMbI!J{5?s{!#=*~SKA9RaLnR!7i_rmT4&#m~oOCrMEid$jJM zxNBd7R>(WA`$4w@*ZN*Vt?U&nS<_Q zDM560Eub!jcMbv-VJ4eX?)T_}3HRlMvE6yJf+8Uvt`O7IkO!aHKej3Jpu~@28pGs7 zguDGKGii~L&cUB8762a^^MLow;(c)*j&16Q$+#b%>px2f4fIhnDq&Ydj}o1`CHyL_ z>6yrQF!ivU%~{Ee@g^m)VrvrGZ3gi@X@`QFaSc+p42L=C&8)Ox~>jk(PAjh!qIKq8UJyC#IG`9>*qe`kM=Ob5k zjv7k^PQ!Tl<&#?y%6m@BrXh7(I%wHE5~o})kXPlDP(+AVFvcuTnOlq2D?N= z>s!B^e4t^B*}PdC(?$DRfjW3_! zhAI`QnlXE-sF7*D>16Ne8v&O(8PqNORG+jB@m~^2OtUJUN@aFRpac=d&(_*&ALJ&i zINJkh6zvc$@ovx6dza=ZMq|`fjUpV)m^*6(~44Ti}#_W`0}MH;czOd zJOQX&@F&VqhUyHTFFA{(6+TTnSNfIK+r!5$?7XsSppt%I|MIx=5Hn=OQ^X7&rBxn% zc_}5(Vg`umIn>R;dSlI!1_{3VvNmY4Gd@G@+VSA>ukkD18dY zZo-3bgY;@rQ_eII-Db)L9$cl{9_c9Eo>G0mGsC)4!PM}U%`(}P5=CQ<3DADNDHZt+ zJL2mG`Px2poxcx#3GNs&-kJhe~!OfQTn$TgtAD4=zr3bnTd- zXTn3XpLJE5&*;NaDUmtYys^snxB$2%X}8gWb3GXKyBLvBKOIx6pgSU>=bL@|$>pMJ zW|s}#jBq`?9vf4$qJ4Qkc(gSSX@&fDReR5?z)tDZ{{;5z#0{O?6L-w^yA$+tjFJKs zG0(@Pa_D1O( zTNz>|1&J zro#+MnDJi<^$-QWZGWXFF>%dp2SMbh*B*!#mjEWNXC=S-x+0j()<q#7h83o56U@gwYD;@A!!E}`$No`fIl5IXe%D*jLrgf-A7Y>QBU>H#@QH~% zqd>|PrL?zkitUcd0#gASE^L|#H($dJx5?;s%sFGQ84#5{ZnzvW;pm52l!OavQKmZU z`jtFsoQR{`Rj7ya?5H*1&!fg(!g9-^jmoN0Cmig~uTJ~t^gnYxNbd3VsA|nOR=)42 zeELW&R7RPyjITDLnlwSq*a_;lCwbf>yLMh2y)B9yCVDLH$RJc#|LZ5c-_Cv*?{ia* z8-268+p_*{KOc+JO=P_(#nE3Ap%0A--N@RC6$X>%rv_*U z&6Y_3cZ48#AMBZZxY?Om=<6xMc_lHWvPitMi5gZWM}^4D>7|F-q-+IohbtEdQq@8) zH8YM>s)IoBQ$Y6g+8G6xi!9qtklCjJ5;Sa#`Rma z(nPl{J+D)`q;Iw8&xetRa<6tAU}%56K?=vJSq47gL3QXEBVJDjYpH`nNsm^4Il?Zd z$L&eYVE@9ERSh zj;gquWlQMZp$>{O3*K6>3uS^RWw2K=($DDj*UzWagMp3t{=Cx4JL@R3$J{t|z?O;<+*ub_PQT60ZH3imR1DjGQ3+>27$bkDYJHt1#8 zZxg~ zBCdmo`+@N)d!UFU%}#^UpFq_Ot$VZc;}Ynl)noe8+Qztr72cP6TUMh)%G{5v)!l1% z=wXE?&83@tmY_vG9(IKpolA-jZRCjr1HN6FQqN!y@-EJ)8?zO9CvLTC)a#(`0CHMw zYyASjsWeox{E|_Q`$SG_rEu;WF3Ar-FujOC(lL*uvugU);45Yx zPJ&8?M1}>1V0a$!OhvP7KnhmQPc49x`w`nwrud$)klA5T3Y7vukmfWdwvtK}L(v-K zR>yq3_&`+YT$|b=cP<;Ilub$bp4Vu|`znF(NUXH-*drb*9o||F7{@Wkgk&n|qp}eI zZaF?@d*ZJr-yM25wNL6FVxvssokGvG&`xwqFTgJ{Fyxdv^nuLF)~=ps3>nKRaqrdM zfE3)w-$AIji_O;tqE=hv$vJygz%cY;L1L^uH9GrCPg%>rdz+%t=~N^@gZ637_f?G$XH1oMa4 z8!A<-ZU>fU&|BKWu(AoPa`!Mhbdt`f{^J>}=3OP8KYlWDv-Ta0dmuZiE=@&OLSFHR zh9F2gTaa|QzY5)O1f&-s5+P3PE7aNBT#KUI{_2k)%fULD+Y;705#M<@83l5}cZH8h zC5N0b>Bi*B=({4OFd(_PpzeDV)A}weXs}SIh|*Gy^fO_^hDscrbc&d0HqPZnisr%2TZZR z+6EZ3%$&{xWyKwF4Wm)hYHIDJJmW7baeFUY^Bm_8Y8fLIJ0VA-kf{gKj_x_p7jUm& zMO^!epYzxREqeB4!4#*|Dmz{yWjr??Y4taK*}s4?CqMFGcAVgxjz5L67;h~eO9Xrv zLAjnh&Uu^d!pM72GBYyuFxV|+#KM>(^O3cXZl&K$MG7hmk)ZX;=j)}_2N}H_9T=T-o|_lYnLhUL~*VKvf*rid;+4?<1p)P z&gp6I5zk@kT>~h&Vsx+S?}mmm`QM2_O0-Ui=tCK-`jza9ctq?4M!7(D#tK(_;h0)g zTG=w+<9)2>tE|KUL3n!g0 zN$~3}_9za+dwIC2#!{fdDsQ)8#-@R}%RoZVAruqxX+FYdj``q_$PYq_9wlBSX}eev zrSMD1h4vbOZ4_n1rNDQ~F&?YbbIpar%93W)AzOms;rwwf!5~E=NnSZGuOU|5X905I zPoGgKWad7k`WAmOr(e(q<6v{i-4+kMy(r(gzhokpAWluZIQCW+VE}<1uCIs5O^`|6 zVE^eYo!v`0<V+9SNE1x^2{ z0Pw?Sv>y~^53`qwi-i1xcHM!Tx z_cSF)OF#)oNkVrh5mAajXhC2Q5SOI2^MI-f=&ZPEqM4Pe~y{K==X>*;fI6CQip!Pm;gloDOC z*Tf9%8KYJ!9NlF5B7<%rm;Tymbj4W~;ziz{xV!mvTh4RxsB;~h1ZIzIpV5AM_P-KG zV7eN%uz6Bi&_l6Q??g&f_Lp{t0uMrfKt+o30wSPbZK!jcBKd;ZD-sk3FkH%Uu!i#E zy+b+Ki!*&c>mAM0DqjiqwayQLx_iNk3h;pB0n7_X8~A%Tey$eoOH&@_u6P)|5u-7q zVfg0RihqHB>E4VAOTE;y<1LGP7DrV?svww0*aE*DNO{FGHsu+qaQr8h zU+<49jP9QNLAuH9sGF~IRZNJ#bB$tzg~uMqgE*2Kz%?lICWxO&)H$-tupS-p@KNbL zv*>J<{xxvMW|<3o>ZsuE00I+|{g;@oKlV56Hqwa1`dcs96qyMn$+vl z3uSqA@qg7Z^`sz#n)Bq$N~&XiIwu;Ami73eTa_xqfMI)EGeG}DqgvFwmSf)yqW;=5 ztYYnh0ozN5z_Vc4o4Ct;tt30{>~1OGTd%091iy7>T8(@!an|@0Y{$5GF7WB)X3G;- zR&Q9P0;M_k}?St*uR-CM@sFW?g3BT@o11|=@s_B}2ooXO?fCF89In6CuG zQK|W`^k^-;6%d`tkh6}Rt(Q9UN{b~Nd~!(Ot&GnUvSU$iQ((DFoy;to;UTa($fcYG zNoR~S|J1CCb|m31h8x7Tz3!ngMJDXqx1a1u)1KaN=3lM8Fr ze3ielF7?4$5K;6_&`dgccxH1$G&8BTc|&uSw%)lo*?v3I^t@>{t!sWqt6_McO>{T` z6iYh(Zq1w`0ul!A_NRHZs%|Lsv2uTu@WA)7hnJntFe0F2&2RX!s=C6rAR4qB45S1; z*lN*EPHIu%U;?#(2L$L~eV|g;#E3)`fY=U|a(fR)+<+l<MV zUr2-wKBUHL;*hU$LDCt2HMaoE50O@u-ydM4-m&d+3Pck1-@eMrdgf?;8lPF^u)l}9 zWVuZ*Ia*T*JNk7A)G|1shE|r!JuI)GBO2~4o!Pp`ba9GT z=Xq35Qpu1U_Kh=^SINc-kWJ#&o#ogRAUq%ubO?B}IrUnD7e#OiOa;dMog8)C2O!8g zHEO>|Ey_1wJ6|FLep?}xOLtf&_r6HvHJhC><&7~ZV5udKmX1=G`*OilD9tjg<`AKS zl&k?f(Tj{JM#9t;U}IM7!Op8eUe$Kz8SYd~aJ8A;e!TI>;rP!22lZTKw3{avah`K$ zBLn3o-oo5B-6EmyM76C09L^hbOT=Wo+Z2jaPR==c+u+(z!{k6n z@LT5q!BDv9mw6Q83PYbHUi-$m`ePa^70Yx4Rzn(@eeqb{exsQp>@2UMs^&7;Daogu zsLNFu3f!EE(z;{kubMws)qWjrFzsuQq?DP8u86WciS>Phyvg9TyWufPZ{T#L8zgn! zt+_18@8AFIbrVy$@jc1Mb-^Hws?67KL_ASVnzS5+2;_!T=(J$hocc;(UV`G@BrEPu z8-b-J1sVwUVsNG_@WC}B{A&JKO}o@Y`-Y4tOAC|HH9IR9e~bPCCh9yTI=Z}*b|b7% zWJiI(TvV@jw>IP@7rJ7@5G&xErRO&@-%&tIeO*FJbeV+sTOaz+52LO_KDJ%cAF^V` z>$+lO5A-z8OcjXF?I;F!U-`SAGlik=S@ms^d9kU8O%p6Q0iUcbt=RnadG|FV>XiPH z(+L{v8;pO5_T_+J@FJu7Iwr#2%ze~c-oB?Ngv3nb_C9O-S)zmMbZA>KoB{D6dt_Q>mJteA<)VScJC+8wAE2*>$ zy$|sLwY2?>SHO%%gy?)7`p;w%@EgR~<>>4z9L$vb>!a<+@_fQ@fIDQ1QqQonq|1{?HT2g;N%g2w|ZbXF^WB*~}5O-?Im<|5mj0^M+OOj#T6jBJ}S1Ri22Td*IQ z3vQ4T$v8jtgWPMC44T~%1iAe>h=tES3nWG^L~}$JMWfd}-2%USn&ced+e!z2+Pb;U zmBGuXbK!e-1>!LRi^x_@f&wy@>XiK!qyWn%SF=mo$vK##_bWB(0AF5iUJdObuZ9H^ z8&eMEt&GRW52q*|)FWkA2st@9*w?*JTk5WHzrS2r37sY zpUoEO5?xemyWXM0w7sWh|0UrIpwmIB{;FQYwv|?Y>Byc4FYgDp#6mSpx+wRxM&K)k zk5>(GEbPOce`mT3c#EpyZm)fV+PK0$x5V4h7z^EZ6xyz0CM5SQ4|7oM57_~}fp;#c zaj+~lWjNs)Iq;AZAqKLoN--ro-jL~*>hnt=Gon>+OkO6c=;eSuX1h+L-IoB+d=o!! zQpwm@JAz3U|6#%c&0`SKeg<*@ElZOa!=(n}9J}LG3sLwpG`{kb>p#qCal|4y;xS6< zj2rip;?Pb7o($FP&PLV@jlx9QrG#M0GbBF-t(j6E?IUK7@ZMlMNGdnB-4a|1uf1^q zLeZXLBcQ$w3)*Uz21+dV1&@ra1(q}3qDYDOmo3hUV^212t|$6&BOb2^JJ0-V?nvkW zueMsuD}w$?xHCL)27wHMJ6%rPP#x(H$BE&-ll*h2d;9wcb6Z}G+)~H_Se;@O@O1XW zWQhOf_ly;~i)n8VP$IOnx1Y|g*ybI_9be-QA|qQ80_XLLEcW8wkEi@wVzib$yA-r@ zvgA>j9lzc!A96AXJkonjtjZ2*)E#ZvfCuxl{=wSdKmG+n{C4a{T>(ZXx?l}TRw!QX zj_y^KBB{==fy}uEEM#)=7J5#W@xnMIUgd3}ZE5Y017S4-uXzrzxyVMNHpl>CVabY($*!Z8tJ^2r))RXFpyN8ZX;As%? zw^Fk@6}oo?wO14M;FE3vcnq35W05UIB;82(nYy2w*i!iCG?kp-#g^Ub!l1&(nlrEPQ!5saB85yBKb( zU`oTAeoqVw>s$F z(j)OVWSu{mhUUE$!^Cfkx?kptL$wUpyxahmp<^=?eFxTg@t`=8fDDZ<)Uqvh`x@#> zfvTo%7d>QiN;srYUUu_L;CiOp_S8)SgI=YMTOPz>5&qRWV*Ryx(WI7%Tq~nj>jGjc zQZ0-rX`^x=5%ISj;9Bri>C8QQrE2S;s@{EJrq@uvn@64_zo;d+!%H((ukt}UgM}`B z%6*Y!F7HK_^8Pge_k`P$m~*5_F~aKwMbb?uQN4euH=uoBhjSp2WU}wGD;m_9q!#ra zo^DND_4)8k3jc@XRGmV?U^sW}zV+C(OW4}|D%Zx08lEWQaR0ndv>xr>ki_<{y*iZk%jn+Rf{J2BL-^b1wovcYk|Flx>TVnB8sq#9jFFg}^L!sqDIkW;8KYEVecSWgzVh>1FfH;6!Ws-iQ|LPfu*A+nKwvn!S*1J^9rjm;X4Ufr9#q`T z^0ycK-H8|+Bv31dU$N>Ejo+*#ZC=)NM(iQDyv6c_1~O}FJ12IuuXx&LGVEf6L-5tQ zwZm3*eG9o&Vlb_BI>JL87TZLLiYYC6Fc|tat+Y4^HEEI2cTOi~H&RfrDoFvD6;M|y z%kz^q!L!X$A!3JN*Wtdm%!mIRQH8F;Q#%SgLhD5YT!?NcRdr0P0c7d4@o(1q={?F& zn43{@ic7S#2%4|$d}8r&X1&i3bC3v*;u+lT?P`0t5kEx51-wwpcbH?f91%#v+2=-j zJv<1oqFPT^Lq;DKlI{kl1|G&p^Ka#PHj3Wg-2X7Xx;h@N^=Ci7s)ohZQ|je*0q)6h zyWqL5*^84NIQ_9rPSk`lEqD3(mac<&tvlrGChu-Ba>%0(9(Xk^>A`hMb?F)kj{UHo z?)uxxICx%bu$JO+?)5j+esq>lj(6|!zvc1=6G53iOb@lR*U7*4t9#(`C_;yIZ#9#Y z-uUgfNa9Kd1@|~m+)rg%1PRg+_+O@n%;}^dIw?e6Ewob{i-9p^NW{ZG#wCQgM)~UP z1cZz^yBS^t8F8dcWCOoT+Ng-^lj!n8ES?o;9_0#0?WLmXvnhUH=Ek#DQ zTQ8lpPo#9$El*>4Sf|1Z-=A#RO!g$t;@mh8(J-AbD{Qmj`^~hEvbbS zYIkpTtyz9D^-EK$Ng1J7*n_6{H=_=9e|eYGn;+u1!XG%jlgHZ-V_k`W&P0rec&-YY z=VI&KyhP+AaHqigb7m%K`O2q|5O{{Fls&IU-L%Vi6lw54n2GS{$y-H;AVs6smh6ZsWjTrV;|K&?AoBj>R1rT?Cc!dh< z?51$=cck}+jp=ZdKEo?cyeZLIPAPv=A_mmX|7C%hpjJD+!3*JhC4|0Ci6(bsRVcvF z-0hj~nWr8YonPP@Smh>e8&QK5ADILDosPNA$O#s_x4TW*eXr~Ilb_WbY3ynZ@or3^!jVK$9*k_2OaHRfJqW!>pOsM@a+Fq4&!&S_Lt`? zw)tQW(GS3aaIFKuEdlQo7Ai=vQbF|kFE@;XL?FuIkEOb!RW~%mz*zM1n$@RJzrXhp z_eKcQuG!)QCAjb<-mUYu`3ZSGIz(LqOmcZHOqA~m|MxU%lmSD=OVM}Nna*?{X$6T2 z@xL3J6%$tZl}1W4zRA|6(EP)>E-t*|ghfuIIjys`l+W^H}qPhmfNQHk@sY9-{W}jy@&s^6Z)7 zM?+`2aYfuu^x*^;SOI?)p2^y#8q8+Km11HORZ7wPLTrWJm_Vw1XHh)7ASyJKPqeR- z2F?H5;P(tklprGvbOj+o51CISNUTrvXJ{$Q9^QXyu9A=PFxUF*Jvj#fz@Q zdKR1N7BRK-`OYH^_R0#%r-K(W@3jm78sR~F=wUkfOgp=gPPuwIv=+YBC_U2=+ndcR z8U?P7xE$Me_NXVY<+g*BCH}%I8!KpT0gBU1W9ORRs}X(8+tlQjn*9ad(vi~gyr|q$ zi|bqpXGn0ZZ~o%3^C&IL1u@Z2fC=!+AsAz*ioUZZR;7@z?P7J$CS9fdkoS~Y=Kb{9 zg{}@wO{-6t^~!s<;3H+r1yf)i9;n|?uVs0NTGKAL33x{lziXX^;AUWq(sW{@JEU3$>G7! z=(vC7z+Gap#-{}v|DwXt~!QV;xL;MQ0 z733>B!!{x3vBWmiDJ3q2zB+TYXnwYvOYwWQxtvCV`d(%DaJLV>}M^K7nR!$ZJm~D+&{CN&GWe@|>P+3buO4 z?NK<*I-U^g>O3TZV`?^W;GC&}GUB4xZdl?EUvgC`fYiXB#ZUI^kCF!cF zoAtmIyf~Sp`SujIBQ~obUz@>yN=x!lARb&Tm_wnHCq5URZa|KohBe>5TO|O43-$St zL)Ve(l@s-Fo@la2m%Y%&*lVLs!Kp%jz%)BbF%8xc;g0tO9cRbGK}`JxA$nEy?bbCr z5+Q}gcyT**&*Px&?mU$U{JX#FZc*CEK9}K^|VGoFBCY!I^ zMSi3wtV*$Yfpzdq+Mt8Ui=>tQ-Rguyse6#y1|>JM!IU3Rk%Arz-&{zFI5W2y2>dSj zAITu!S;Knstvy3*tk{iT^vV|K93+9zf}B9i)HzF#2NnQc}BDhDSFgVFx1;7 zvC;qMxMY7o>L{`?@^Sa9RSD|P zDJ`)V3dH>uCh?8)^QZ5sKk#+f)~%GSsJA_pKp|Ppl<~0SW<0cCZAZ9B^|)maei(zf z3A#7ei9x59vG9qbP_ac^5JfysRa~&_ov5ZS)l#c*q8lC?z&6pB!R=))! z)O+?p%VDceD#YsG;l-$=kuiU~IW5b_OMRLzG-_r81hO)!614k5@0N=0=+y=znokBlfe~STYz>gvsFhGU4~b^sv^_; zM@?BxQVEP(%em{r%h4jp;b6>U+%H@6IYP%F^*7SYIO^Yg3!O{n#jk3v_L(cyEkIVn zwfDUD;Cqu{8V-AZk~D-|QG5RkiN$F=l?!l7*7)U7nxl~@m$3S5C9W;MjjvIaoPW{5 z!_gEe@>mhtE8rk(R8~X(sQBsixCOw+{VANj=<2msU8(Qn?_^54Hd0;g|24y5{e!UU`rgQQ?L8aQ$K7A&-uzm2 zv+*#0)#&jnuaV?l7JQkUu|JYjPVB$y0DDtCK611x!|tX{`q;(i%3l^8>H35ucW78H zchdRd)n1KDg@lt!?BMsokUxubO{+hA3#yb~an}7vcV0f{cF>DiUaq-Nek@!)?&^5P za@Sw5;n=?HqY}OI0i9mp;Bqz_s(!<0r*`E_S|#jj*p#yYg7vef#E{RX>WHDVOmsKK zKLfYi;vR}sI|{^t4_(?Fv!aDJ?Yg;N3)THx$?3&z#@yYgLd+D>0eT z5N8_rc8=ejvyjVpSw9o!s_4U1eU?qO)6{LoawbI6R=lqmjstI#5DRyjP(%6&ITFr? zJP`DL;g{dIF_R!s3a1Dd(O4%LAd$p~sABh(Vq?%5yGJs^P!{2nH{I zNki-sC7|V@;=t51ncymB%oSSZ=X)*g{&HnK>jIHSi6PFeqAH>WSrc>ZmsZmn1A(aQsH6YU6^}+h@v4Z5oWWw^$Pu%(?^$Xp2h9cfoi0#d6a^ zTKEMMptwKB-YH{#A%D(oXI-u~95;dC>X)0Wc5ayfy$X7;Z|EXvD9O)RT-8}e=KQPR zHDQH&dwnugU>w?4xQI@wd=sic|1MArcp(=Q+LQpP=L`Yn09D7L{W{sD?lWyI+o;Ki zX&*4dMi)v3^eg>?#(mwa6{J%FWdr`=)u?WC*B1B73PzIZfu4rzUwTy(BOp3%Mtbbx zei3PKvFod=JB}zMGg_`q;Xob#RBQC#;lKF#P%)kct*VaCt2ec@RI*n0Dsh}F;^Ona zy^J2USf-y+DKSk^AC63vUG{aqV+kTxpzLk6DY@O22^>!Mgxa{odd=ytLjvdp%AJVL z15TD};$@I;;MX!Z(Z_fvd_cxxo$HXD;wE6=d%hjF1>A0G< z!xp-B*;+bH8SlB8K}s|l+)OnPj~+3jWgQwJRb#P(BilFL%Byza&9`#DiJ0D{ljGqw zx2O}58!^{e$OyH%YOmo()_H4XuJ4&@wUL3(|t5Z@tOuDAo?*-HwUgZ_aoH z8~AqI7-DCW)vd(?OhYStTL;E6)M~eP@Bc;xF@#5MYGD_BjEJkl3)`<~ubUof_UZL6 zPpRLIRxq)Ie0eCvf|^mmILu) zlv2A7p3fl0;iY&2?XV@ymA-eOsukI*2m9fVaAv2wd!h(9UXa@dTEhj9KFFaz`}(E; z3#?t*tq@nzEGXc-_PNj3)$7JrQ&$Kf#Esg}+>;$~`=wA8Te+)38#;fhY=V7F(3&g3d}v{I#qzZ z3W%iuZh)DCUffpzQ0$JyuD-<9DEpfN0tAu&03Oc*&MQ@%=(a#S_Z3m_R+4ffLII(M zcmV-{uUSx@lDP;P0cttoZHib!`;&XTYGA!GXKXcKk_6dBvA1#Yp1n&e`Uy@+{*jNq zQ3fyma#aoHaIHwfp|y@%=(Zq%4_*b~qy?+6I&S!B zipxi7#8ECKDMh!f;Dv2=27hHNO?~uPI{WIJf@vd@_WGAB-2{u++~RRp>Jo*yylZ4O zc`D!H1h|TA`UVz)R(XgdNVqf-&2=mED9nO!$TTNVS7b_12Z||x+8y0C0-u6V>Fk$8 zwy*&a*$r^|3y_vtgyxzTeXzHj##RBF%AAcOU(zlR^8-&ZtGYrU6pvdw$-*m0P(K`31{yDi<^iR0Ab^Oy$X)py4{`#JL#|#DbXJ(VnEG| zn~b6`61dN<+q&)wf)@N2d$;teV@K?4eT0fO3)@M|W#_u9_$OPL-);bF5Tp9bZGTXV z>hOQizR?HbZ_;?}!WS-f6(ec+fQUZ7cE6K$Nzit(6&?gWMeM@?_qCpdCZOSE5!^Xn=GD1e8*u#P?RqkNCmUh86Qf@&I??D@?WYRpSrhN7z zdA|9(H`=miX{$Zci&O_{cO+gxgox=^Bjn0@%|G^uo~r=9;; z-@3JSzJZo9jlcgYUy+Z#=TlmEQgx6jai^jKYuXbc3ZO}qBt-oDVyIVTz1pwUrX5S1 z$Fw)l?Z2;RTDk*Amq+7;?hd>QkkBB8|BZlqFnw=_hJMiIbpFHTzZ?cA4`{g} zz0On%l`rF?PR~xu())JDf=*Vdv|G8B9Z|+D-Xq|3tjw`UxgiuD{*)B zAYLL|R-di9lbuW*bah^2Jz$CrWfo!@2T|5^6KHxYW#qW$tb%G4~VDT)S_C(dKaJZ|-EzVNe{mSHgsrARMEbU$X|e z1HL-Q`p!fE{0Uc)n<5uJn|M&N%qnUiJwW=oo|ad+!W;6Y(PjmNuG_vR!l_#qo-0)f zLZjvj*FZw({cos*)$UwvD?o9;jY!y>h~~~XyVY$zX-Z%oL}4clop4YdyKS{$1GEl# zR*M?*Iu^dH>%i5nvdPqzawBNk7GPOXoC7bha2=-4jB9#h80?nx-v{sDlO7``4@Bmm zWY5DPvY)>@MGq^d1K(Fhqg3zT%<^7U^J%_x11I*>0#=LRmzrz+JMBXOO|JVYLiX~F zdv7>MF|Ugv7W4#YsY1}1bK3BhOuS>pOY9}HYvZojWnVh(S9k&T9e&STHqMf9D?v1W z%K7|bnwYGYlheuel+X!>8Is0Anl|UfT3H%an-U*inzr8@I&iH#P z^EIzfe1A_-^huu9@PG+rUX4)|=T7CN?m=CKEu+5>Q9_R~nJ3dh5)~Mee52Uek6?{j zu0=H#G_HBwBJn9{aT->;zM$DArfH*DVEt*7{T&{)bDXB3X4%FX++oi5T=xIU5%y>ml~PAP!OhEVX zw+R}6`RvC*)Pi2oga`RA&|^_C_#Hq#u&y9hm!~Za$k;pk46V2(-Pn4txFXgf$tSNl zK&A;r-|^=`9N@f$PALQlH29gJHC(rW*wg~Q) zSS_*J84rOez_xcEyaH|$>drZ=`Y9EM>e}K;{?Up*oux>vDW_x1#3?JQ!RDyhY|j+l zRq#)c7B1E_GNqMK2YRa2grx@B3qYcZsNkk_f8$IjgY50*U{ei81Ex&rb1ew-4~ly!HlDZr z+K=wICTLNn@8DSNdJibRRIj{pZd=|bT37)w|IgkF9`UK37&e6TOkI;E`spoh7H-s| z0==%wbE7I+t}jIZeW{VX0#QXur~8w;(B=FOW8H~6My_(4_8F#+O+vRi0oKn9KZ{U1 zJ0{&FJ474&iZda)Ojcy|H4Yb@q>{0Hxull6tYDcdB}HNji~?)e`0qb+#`=-h?$s4Y z40BH>2Bo=bq7OYaCXKqpgO)FvpX|!uh&k{WZ?23D`%Cien$ye>Je}=?<2X1Y&X-8- zwVBbY_mF$CAD#ar{Ri-R%UQH9WrqB?+#xXVGIjr_4jrpp#4`N4()r)oyF@QZu@Fch zK?9*3$qNuy^`|10<{x0brApHx@}L&a-;rtn)Ef8;&2FTK%**EH=8=Kjs%`w^9Pfw}vvw!E(BHd{I8W_RE_z0+Kex-^>O{wVoL8;Q%UY~Gs1WBb= z&*&`SEPOM#zl&IXmFGCJR{Ip9+FH920cshY|3HQBUV?^4hm=*%VYpLsw*iQWbQwqH z?3m~7+~|=!{vw(`(5p3Gq$;vpR}<{G4KUA)g)kgCWr{ZU4wrJ`lPE-D7KBP&X*3}-Z*gkChD-#9d2>o$t z=X%SW1<7(@$^LkytBqUMJf=*$2+*St=azOjqZpiNkwBc-gjknTLbgb(l>auQ<%Q!@cpHKYv#iGg0?EDV4;SJ{fx08?LY9_WTgAWD>^;WQ#3GOQ zg*DXpFL5vi8xLF+JBFyhf2|sC%ai>`4cy|F9Te*n-0wB>n9-enuM4ql1cM{X;pASl z;R6tlwnLz6kNb_St-Gh!x{C$gb!{f+)yQfNc18Uq={>O$EmiNAmu$T_47!wXInr}G z*JZ$UpadSIm7jWt@w>p*g$NP5NI+(ps%i^{O2-}!=j^%*Vg#hf9^~Luak?@&iM)2d zuHdH{Ugbm%SHK%3#4CGxMxciZ{+hgt7XKzmG0Kr0cKp=qxXJ8q0(_JAh1d}=gTSsE zI}bWxVz^BDY!dvH+;;@jN{4=wsuyxe!oY#UJ9jpcpLfR$-lR)#!B4|I%i1nby*Q#& z9aQYO*$9h1c;_XaBYVer!c#Um_gw8?$QB@2OJQ03Yn^iOVN~d1S_xnfBlAY|y zk8Po%MsJPCimj+&S{PjrCgum{(5Br%9X}kk9rx6ll+dk z+nboNKh^WZ@;;M2nkh9?3~o#LIqzkrSUgU2({;Kq zvOijRZaT}Ve8Jx)yKIFwjMu!5Q_0g%%X+wdvy{Fv^fQV*ii@BJH+AcTo0l}I5`%2T zLJ>~ECsB>YJ1-l>?PDJ54L*2BFvj(D_bP?8sGfXOy zgLuuXEWvlBd0Y4=!>C)r^GD-?wLQU9?-v@k#e>@0?zixR=jE1>xhLP-?%8Pw~hMq zq(SkXUQg_9!q26gGF^e`&>A2}HPZ^Y$;47LTb^lE$(g zBwUdP|MBpS#rqBcSU(Y1Ps9@R!EjipmP0&Mg zWem5pl+QFBRJ-s7|JlzxFM5TEg!Ssy7rZPlaEcR6^bA3Z96k>M{%z;92Y;Lm;m+>0 zC=T6-Kj;v4F*2=lYZq$|vAq8E;Bcu#_vX&ak4IdL3!?M>QK0yCulY9~Lr$<1k)vm| zO#x_MZ3A|Qa+R<`w4VYOn2Wp|eoJSKXX(!ZItc0ABII^(#U#r=$nS)J`4BwAtfLYP z&@_Y7W9!#M?$^g!t0g$X*wBPvf8$ygyq8bMQ^NzE`#VrVG?Wa;fXaB?J0s~mJ?>B1uMN6~vVH11 z8TV1h-UDo{JtZN#dcZv}+J5XNU|D;Uo6pm;D&>8_@6WSpRfC|XbeCsYz8FV{iH#}g@3cR13V|%n=G_(`l zVL1d6^4N9DMwaUy*C=8Hkd)AP8Q!V$gq&)vVfrW!HYdz3F4cN$&@_<$i5Qgi$t34? zcc!qD18PRDL;I<2m$$5=_T5-K185HZrxCBXHaT>qgpWT-s`$*RGfam=T)9sL zTu-~$B2?fhpI9jTiW|O0c;uG7zY68SJ@XTB$pD8}0U+8R}$4^E`?(!Zv)*?;Wzy(8_-(mmJ3#O5ar`=Yf zi5xAg!8e6fm8Y4TBfSM8K@nEP<+mK&EP`_7JoK;v<_T_z&7=gjHib5+qgIWn z%Hq_k{seq8?mPT?H0mW1?9VTV^R`)Imwlq=%QbaopJUAwucIti;7XJ_jIS2^M`3M) zb)UPWINq&67}F<1d+rV1{5l%46ymneICN*(BppcQ$`QF~a|+}cCn=7*i{@I!eeEUy zjvS?dMj%B~Ax`9(T+r{#cc(s)6yV9%CFmE%i(sk5hu<29ZgzH`u`2=^)~kvEoAH}p z)N1?GxO3G-ls{m` z;R}p$hBIulI@fhWwk4IBB5Z@Y=B;bJ^x`1};I`zq;P&LD1}_w*ulzFJ^{I)~)K_vPLkZu!5&{IK`ExpIz_sr=7~rlh)@^jiJw% zPk)Y6XY?kw7eA?4!YZmRQWb72N{M1NV5l3Qm+CB++hOe%x-60X-1hol&4-E+fMvko zbK={}LrX1Lb_tNzx_S7US{q>x^#cOG3v_!Z+>nmkhBJ7Ih~BrbVM9b7gW_rD`dW|X zIJWW@;O(FHc_YG2xhR88%Kgt>;-t@ss)6gl24EDwe-)r^hP_5d1R&MFCFKsZhtoBV zLw~j0S!sCXQ73YMZquPncmSx=sc3_C;;93Sc=Sdl*0H%Ez%v1AG+L*E97tljKj%|m zI(yy!gl?q?5mpLqopYZe2JXo-Eq(s@K@&m+Up-RU5k0>Ti@LFN89sdA_>aXuiNr4+ z+#iR=4qNBZn?^Q$P_%9NSH9IrBH!VQ5N+B-+Iks)lh4u^_W9?xGP%9U%9-_dY|Pgz zdQZBA%khaIkA`U4LSHRG|95((;yg#<9&nud3ISlIVu1~6A%CT3>^)G*T>7_^0gR`& zkpHor?)_#}UpWP1>2zQQrK)y~-mdejR`aWY33(3+Ab4ChNZ)Io>G^JC94l(5Is9u7 z4IcXyIMd-Q+tJ~p0Z1hdS#|h$@@l>!WFe{+=jM%IA$=5W5k`=K!JlYuUesM4>&Z;r zh?RYaP@3%L2-3SGQ=8SaTukz)>=4VlZp%6G_tjg#D4r~vc^+?qw;I1IwfF>1F?@J3 z`gt{>>5V!7C}N;&m#HN>rhzr#tpEvdy3>bgh_lbc8jOHXBii0cxlMcwqI3!_&8URo zW5Kb>5>yr=!%ayPF`PWhx%2)rx8f^?3O=UXJ>6=}Dayc5eI%Qac+z}N06N9;ZUEE8 z+d_~t-uzoRL5LtYkANX@G+s2*H+Yg092B9jHShJV4Fu7!E4XJ{EeB^;Y3I{t|-2$Q+P(r3t-nc+>V($5q zH)D%h@7Ng_PInB^+J7bOuQIiR@z(%nH3H(iME$*89#kV61-hy2Qk&G4z1b>sfJ*Czm9%XwuwBhE2o2I0wWq ztbNU0{!d(#n|A_u#cE>&<`W(zVgz;j9|)1k-HnvXsp^(hCm458Qs5q7=$B1RaQwoL z#GkiWIcV4w9^fSwuYwReP|Vo^&(Tlr)L{37%kmrSq(leG=l02NW{ZFtRb zt<&GY(`r2_2iZ?7DP3C{s5sTm*qSO(92B4hbn7(!+$>_^=^@WrF=rgVqN)itdm z-pS_cr+mi~OAq2cWFAC~f|tRCV1h6}Nvs_|b6Fo!}qhN*;7d z7))zT=wxkMUuNV9O1M{j?32Cop;k9llqd^&=C%p^;m!~EX+fA#UYkee$C zKJ`T@w8w30oXqdz{{lUrUEra{d+5*hvB`$ciK!1JdC3-`i1WF8b~A@lDDyqt5)oSP zoA=D(8U$#lWl^jkZ^9*@8H_$Pm@sAXXjh+a+vo2yp>>Md=N8aJ!WZ{0psxW^wk7S* zTh6#vi&w6PPq!fF>y#GEkD|dXD1r$7Y7484?+$^7i6uAzHu#1sq34^q4S6eq0Z@fM zXXBz2Xgln*jOWi;w&&Qsp&WdFegUA@JYOfNLrCv+@z)U%v5w^0an#^Ooa<&xl9Q1V zt}UZ3Xnf2RAF|bw0K=yDQvFMJ2BtrjxK+W-cuAfbJ{-GMWJzZlXEP`qPiNkBYM9g= z{(8l7G?d~qomq~{5FE+sZB$gHoAp8~pOBU*|AQuQRTr9B2_{|QV28>d#rF))@3xDN zkm^0%{eH?+N(8|J9@Gs2ufG>MDTtsR@#Vh@))xSc@*mD2TPt<(B>mE{Lg0JA`CG^@ zfDX8v|D1XY+$9Q-IrtVg-vf65TIC>q5L0w~y=a0b1Dow+G`n$Et!Uo=9G1+?7l}@S`DLj`fFeKeS|z|tmn28>XlMk z0NlUT+ZdHno3s@+dgHIKgxMY+&?5sIa;UeYRbOy`pCPnc(ztRq6l@A79tchew7^o+ z-zLltxXll5JbHgBSYnJn2Oa|n;d(8P@SNyR8_M$?fYUv_8zZau92P&}wg33^jYgDW z8`4cz7A~-T{C=-XHpu`Vc|IV(HxXLH#hTm4XMN2Kck(vwc>i@5MWa-Dc#DUc4grF6QADazESJ)|G=WG!nxN7l zbdVleLVvgS{ml10Gw;hxW|(1y$scR4z0P$W$M49GKRFU84koz7ySBsa0B|UImQAJR zlh7>d+cO%Gpwu6v9oR9szuq>K$Puh@T6zopCp3tSlE?+-g5Z~|2)^-N64(y1m zA3E*lT;EthG&{W`H(My?vxFxIQ-Uj?-_YV^>)d<1GQ^kjBD-noDvv(|U2N*}XwPq> z(;Dw-iv|2YABE>jvY~Bc2eVMemN$W= z=cx`rnixPu05+X0-RJv6uL3HAztrQfC0XjjKF6Q@x7V!>qH1aokNwTE6)O=rCO1I+ zZMsigY#}1J&tn%e>rYZg>H+5Q{uQWIy@A3f-^0~?ptnu3=$U}L^+?s)r1R5yGqwaz zf^OIB5ARy-7wUPQ!gh=QbiLn|F~Dy!c(BEa5Ef$|Zde=4tJ@*r@F2gkIXDXam~mb+ zS_`j<*P{It*Rz<+2p#{uacw9~d8OvcbhJ;y`Y2<6&(=fm3YJ&h4&?@R4TnKDMt%cz zZWT0lrnAzwGl*f050#Je_6EFmVk$UU;&LaS{T5A-TA#Bf&5gf4%dh98=e~Sn4A=eY z{mj}^Cax`Pas|V7?3`L=k57nr4J#6~7zF^tHE*)uPEtv&X*ADVF!@(P&F(Gsjwrh2 zu&*utks8r4n6L`~Q^PHJ1U!rQ7d3ZAqeF8^eeO{#JZ{fDT8{y7`vo%t`-wv*3iZ{( z!k-Q1DU4v{p4_@!I}5pK%*qZZW57tZCEd}lcOUGq;5KJ@j>4zZbGVuuo0Ad|Lvf*9 zx#jSF$QW2jDd{))wRU=~7c)IU7F2dynIJN^_NZHy58O7K#C4vBU=6Y|=NH&j7uZZW zgL%(ffIhMLEWZc7{q!n9#rGJv)_Hm6y@R8)YUcrGK1{|NvmLV86?&%lGUP2#Pmp{ef|YCMBBO>k(X;W_jTD*kO&V3wzt+P0{!nh`HlK79X*GLTpmvzb zxMh>d0!y3~^ClC-m-ahE4_4~kU@6MxGN6k`Y1>=qR>Tmq?J!#jBgCOvG;^$4Y{We9 zJddbcC}8ILYzTM--WpvQ8<7R-USV|6GMU@W%<|C3hYEOLL<6p-Em3}40lI&b(|-gB zg4#RA+f=nKjIV*ljOThM5V|G)pTj^qHKiA09SSURg#I%E`SJJL_O}30H8CSX%gF`F zZNTSKa)aT{strCAhzb$+OAZpC$M5cwpXwAaYtj{{8gNk-uJxa9pm69LxN(6VJXg|E zLiwquYu?%8S$7l4%bFXY9E6W^^cqN9GU>@AA3R{=5#+0TU>bdGrePed29s`~W z&li9Fo-b&6Q(Se8XqXi#ELz#7t4ZdXj> zqd6}>fWXi+k5_6r;Cb6AlQPGXV>Req8+$o1QfD#0Fw>d?!V0)spZGh zga4p~mi6qta#W?%H}9~RrsjK`>T@Q(M?puTI6V6mdIVW!g*Pq3UARS$lFF9HYBfqrh zHo!6N7^E=x;gVU9aQsS^xs;We-?Nm{wRNR~lqT(+DPa%`4I1ht4Xi{J);Y+_HvW=o z7~|^z2un6;cqXc=EhAkqc)>h~HJ*o0e~6x0I79 zbo!np;>wr*r9L1)H;$M0UD{_N4`Q7rhoe`UeD)E0?biTz3J6`BAr|eU;1PzOFDV)u zog9s>#qRidO#CNqMUW*POt&vcWa}{KFzS*hi^or3tQCmwAZrc5>w5?We9a!h9kGNc zMm&Xi*!M1v^v7Rd#wV$-2KL)#q`Uza&-vEMNJQ_W7sHb0$CJx{ZIo4sBFX4L?(ml1 z=RN3WUTkY^A@71aHFj8CHGi?#Fye}A(}voz8Ii+ji>4lLe!uime}@VHKc)*e^ruB` zLEwH`u;?n*D*$hAasODPt zdq367UDyBtH~|2X!k1t-j%5y$o2`zhb_=ylA5~;altE%|ui5G=QIsJ4-K^$>D+=8E z_@h8a1LXw))QiXR!JL#^r|{NNCV?^VI)N^}3~Zr!l?ymLh8;}QJi*C$)?waGcx<`v zq4Y`wD1sJ2FH&%sT5p76`yA9a(fyL$U=6F2}10q)PQgm6*tJq{M&(S8Rud zcH)xgC`ho)8Hfs^nHi$^>N0w~7{SCfs*UGxeScIxzP~BaD$r%Re+0)K2;+vR)SC^I zUCoAa0^37ABUAHwsYnvq_I9q-15OIB8hJLP%^UxyrRrwh)1Y>m!|LQzuRaX%!K5I0 zYyqm77eZ9Sa<+qA^M%{#4<%Z)a^4}T<)(s>~smqVX!T7*ZILMa%|Kf)= z)%k@QKK0$l_v#E>RwEMp_5AXLp%qS1YiqY$^Eiw%bM4>y*$S1RlD7orF4)kuGIt45 z21CIL9p|U@_tHX8WQKhurOO0*Oi7#ij3>xAO(Y_oGcQI=v1ay2PObV^(wGk2lrNa{ z;!Ity*)agx_!n`Pjv?Fp%-rKJ?4n)U7^0C>h;_XHsFpfcZ6DU>c9~oS3~3=dJ$!@ zdT%&)U?kvcaLOpHI_$RV0y1_Ik%ti4@ieiyvnr}uBS7o(PZ{v8K!KvG{kR8aSr@G= zbKJ)LaMkgn#By8*;3;Qnv}#&@{4nN@La6+D?9E3(aD3xxJ0_1gMSDlgwZTpM)A`x+ zccHSqGArI6v_(qsSz_1wf@7AnPI;>6%M7#kg<5{3{o^&yEMMlfwm5D~v3+@Bw|`Bh zgQf=$(m^XR_y0o(p8kR?2U&mB&7)-@K^hzpM@+_<>QxCu<*i%`zKg zFcj5bAR_cqSM_Uwfwg=m!hM7U`%r&U)<&J;_;jK+-M7$Rk>qK^WxoZ@T;fp>wzv=x z(5YMfgZY2%n{28r0bkcu?vvgT!)yNO*a=U zDIObL8!OAbppb`_2t;JXK5JQ|Wr`Ijo%H|#R4%M)ft=>6jnt2O3#y`5`Xxo{s_o;_ zpsu*KT-noKtCxBSH{slL5kmAWTPUT9!V<8?c-h>+8)3p@g2(8MObC3lv9YMRySWJ1 zL+!Ima*c2(C&D84opOQ?x9DF(i#^DH?Ik=gF+ z>5?!T`@pp?1#bw(lZ?)%2uLlSNx2<575ZIcF0duRw_R0JSAbcH+|@=?ultXW28>wS3}&G;>z=ILY*Ka>nxayZ?+qQhXvFK zHt}TLk&UA^W4n0wTLDI`y0EgFyhuKKd4*NnLxooIx8gN^neE{CB`uL9M}r5=OQJlQ zV045#q#aIWIGRPt9PQpDM+sl~4@?u>l5p~_i6Ht5ruAsI>)=rO5+E9=lg5dGIPj#bBA<3|rC5??+r^ph+%utJ6=yz`?anS+Acv&i_FgRX)9K0kmuZ!rRCC{rb4=Es z|9^Ga`Tyla7rn|PNra1vGf9goGhK=}i&Zf&)!5Ob*;jM-H@`?CZ5r`N{20MeBA8$98!qA=59%_?6}E9*K7Df%b1TD5;pfI zx$WpO2x=WGG5KtfBe=s1;Nf_u9Zm~01%}Vp0)nWtII(AzrquIhaGq z2MLVjlKHYZh_O55bO0Ib(4XiU+vvlWm!j~IQW6B|lKJY7F#A$-?z|fWrm|U24<;ZZ zA>BT7$&2ZIS2k{xPb3#mdhi}%H^25;`q}Kwa~XQ=oe4C94k!W4 zQs8r}r}WioG~`*vYjh3Gj4pQ`eAF34N!bu?Rc4E|1>@RL%pzXZJH!4{fID;u4KEY0IYIAu8`1RX*@n9^M zkK?tgX?h=zZ@~+t^6qY)QRql=Yq-KGb-qyrSGyp6gLs^nb;0~qe7$oTZ04KDka$p{ zI;nOykL2NYGuNgQnAV_w&|)h9{u-2k|CwFn+3v09#${cg zhYyO{iTaFR`bNvN&mn|7Qo|mZiR%T8f&#EyhEF$eop_Tc(MR0iS2drNItWh;i7+R? zpW8U(i=tb_6eh2=6+i`u_*ccMGI4kXUeo`6)U~Q6fMctz}?exX+ zph{d_*EBqtBY2S`&5edvW30X`zVrsPYVN(UZ`6s;k{Brds_B@mXBAm3LoCoa9<@j?=Hr?t^xAj~7^n6C!Cs# ziG#2uRzQ4nSE}b58CY9%dAx{p(hy~2QnQW3&c}0b|9dztGiIg+|BCPbQhmi9o#zO%)bG@%nqRmin7+ z{d()QC$iI{SR!ObY)H1rQSwSNG0kdJ5(MA5blg=Pt6>q7uGdm0P?5DV0+o4+SzlJS zJ9GTlNP18L^AKdqF&$BM6AiXA4(N)@4K!8;C@-!Sow9v_+XhO5qpC& z$cQGs>$gQ}`TN1_M{@o&=m;zlPu>JRb~6z51GVaeW4Yd%czmi*?rpOByIA4;&tiqP33IT2 zB{e~kk)7|BfMB}y{-^-_SL{V#Hc>M$Ez9ESGW9R_HRD0YCtKD0v$ycYP)MJhzO($n zbkTMx{v1|3RIPHb8MsXPjv=@Z3{W`_*PII!adm1@yPFOwEZ>3?&-8xF+-;yF>e1=D z4FMn;6drPNoq9{|{pMY~SuPE%Sei>l zv~HU9a=m6txjjceA3$jrGFZ&2+T=ymgw|#RomCm#fP}eOUF#%5*(3OXT?gtvRsIb^M~#vTX0wMFP*UZ z0PuPBW*ezvY-6hcz$VpE;q6*uFZr8{9hme+zz7= zD*=PZJ-hgpJrS@0hRR;a3KvAPq>gg?DR?&K zLsm>y-$zsgACfnM%eA7g=u6N*$W{>8|Ivw}8q8J`k_T}`K4i8%4~fBuwzf)CQB%aF ze09*YbzSG$Xk*rd!lI6vvi#$ODPY4r?3pc;Nf5|N`f^7RuXd1yDJ&Vr*tSTX_cv2Y zo0|inuzm0k?i|rxNIYDp>_|Acr4D>SfJ!jHb^!v4`Nkt}JvAg2BBrf8t_xnnjF;AB zeh%%9*W*Gy0dqp)R`<_C0w-%?3?GqJc13$2o1{=&<=la8uLrXu=1ow_;-RSR-!FoGd#MO}_m9fv;Tz@mWyS9k%>4X3I~U5|h-L zfs3VupCvZ{F+6aq^cCpsEKuVOJTH3P&(U>{pFi}<5_CBs@KahPH$H*sFOeaFze%7= zchm~d(P#lGd75qYCQPOL(-(|mffVkS)#R>!VB~?yM3GJtk{!@T2B@D!1vLDwi7M(c zv}4}qXmBqrs&I|}95WsxI=|)yD9P@mJ+jG^YXhiHc@4q6A=QYBh_@RiRVDL0Q$?W! zr|%JdQB(5?fgxO-ogP?4^?>a(jNPSvI&cSwJ9!P9JNKbfEHh2mHPlPh9x_k=x_jUr zm;{~#ydTom{*nYMEVJJH>s}gXS-9v@GHpm%ZKCq%5o#S_hG;T*7VE^ECk}U0<|Fr? z1+>4i0-{CI9#!9ESa_LWGCO|OQC=o?H@NyGFt3>Glny&4TKpON<78aUBYdGpXElOa z25ieW&#JYaO*xx=-QDz&r*7PjS0(}Om(NkQGP@Ot%DPI*-aic zK*4{q4y*rRew|MDI)P3*o3NlEs)^UwnGOU!We21OXRwdl*^i@W^pM1_-;`S5n}~Ad zw1R~}UuQMPrny?731xwwqF>)h5tG!TQ61*Zt-9h@_JCl&$Pc1<7nd%j6(RGYKOl){ zWG2+7$zD=2ttP-0;DF{5P{kZujc|taCF@1yA9Vb4bSgX|A!LX;X6HHPlL zkx~Q^9yRl3NKxB>9mrDH5X&^gr$z=m|TKiRAN zCUkAb2qv|m&1B|$DloMHwvQavXc7!^pWSM#SUgDVRXFm|55fIU0`$+R5XX#$#B)Fj z6ua2!g8jYF&3FHuVYpM5j70q&HS*lvp)KE=&|1%Q7{7N$V>tbp$2rH?zk%D9m@4!Tu!I|Wmd)eOC5$1Mj-qLnY1e~%w9??B z;AVjoRy>x7IB@MmI)V@{y2fbWJnD7Z*o$cjauGrEqKAn!gJ=k*GlU0Z^*s#U4mIw{ zT+FQ?Z$$^e_5ACzghAknN;XY$nzJb&G*7DSdr2Ot&p*GbZ5MVm=Ba~;RN5v>idc1b zKcJ$SYG;;m)ec_wU8Lgb4dqY~^fu?Vgtt!03uDHV5zhWy+4FUqFxMD6iN#3mXeBlz zh)E7t6yuZ%4=PP4CGL-iKZ~wLE~^(*1o3n!m@v2uDPXynP-fpdf72VaE>6N9#z{== zbIN5cai+;ff^@(dJCE4S8(xSG>HZB17tfqN(ox1BiE#*Q0L_U$7c7Vxo$WsJepqNn zSl5m*kB@=QHmv%mMCHq-YNqwxnB>fa67XcY! zeM?~|LHx$~!+5q1>evICJ@G>3HVT=s)Q|rCyXsm!0A_*3ST#;Pt=x@DsIRIw71{hb z>+IuYAMiHmdcQ(inGL&pPy&DpEIAneYiJf1ve+uZG+Zp+c+j+zKMy)FL( zc1mPn&Ig`g81ZbeK`{zTf|t9CoPJG^E$$k7SXMczAs$d#Af$Q5Hut3ATXvk+K;N$} zru-@u6MQ$b5N2I`CDT#O^3=m@;n@dSCe;I23zKzZ!HT&M$bVJ$s5LjS~#X(}v;HAF`cn;Eav+0|$-6U4| zg$YG`tylG$H_nmlzb5wIQGHIlKVNV2Yk)-j5@jJ{cVx1Le{SPS;6~F?@kM~+3Sa}@ znn2pi0p}_Z-Qjs8Yj%)i=s4uPkzw%@{Tt)~dWyo-zq98mnr!O`F&qvZB zdC)<3J)SG$>~Y_0=d!1Id?(XmR`GOI1?}jZt76|72A=nk2G`x}JUh(j{&WfL&ix9T zQ*8AGj2`QnnlVl+-=pS4BN!k|3<_B>4XiJMH=Y>2Ik4;f8UK|8@AQv@4y0e;>v#fa zx}|3lp6X4I#ufZ|It*t}W*Ar3$T{(Y$XT!i`~kvSc8j@`oeRHd@o7)~^o-`=+uynJ zHo^et4|TJYi5rN4D_&1N1W5#GUgfz4PV3)lCEYf8zs-U>GUJU=IqP};IX|~RegZZd ztRKk*M3gtl_y)6@u#d2LxDZR_u^gU>#z;zWThEb4Ogar9cI!~#H$b;s~ zeU!&c(#T8AjF9Vuv~1m)OH~gFbARV5zN(H@sds(Jueqd)XJ-N*b!-_*m9=}Ua*f2Th&0@2Q&n%Hq@rdDix2?=yb zfE|W^r5c^Xge%V=LcP%L7T}4N0q3mDl|>d`x8+;MT3E(*;6j#7*^KZ;Bdp+9=RjuL z;I%csjLLUHD3R+R^N{Z3t8Y-WxvbR2e7Z~DRsHn1Ytp<_G3N}vzO?=!3yXa)(0`()_z%d4kfJ~#oFVB=aqE9d>&%27U^?Mjw5 zeopOtygxM$N0Hj zaE>z|QRxEF0u+vt%KTMVZ`HLxjYKQVZx~h|Nv<3IoFh`<7#M8HrWXpb)N|_QImO!f z-A&(u%L4`|^;m|ftOgd&PBwqeeXiFAa<(9&w2pw@muDWMD|~f%&Jp&)l%li5a4oa@ zW_#eSxuZucP>^Guf>Wl1uLLzW(@Uc%;H_gQDoc40xc}^dS{^H-zC(-%zu>;i&D7=| zwTHE2m1a1*(X037+29PnPq~K6&%mD48%zHuo)@n0MbNR)&iW!hgonu`tjDucqQR97$UuOp3NIi!ilvC zmQ_{<{ML!hc5SR!6??W<15462Xb2O?3y1)pK zw^l)?`%a?mK#WuK05k2na3pNAP+EI_52e}f2$*C@n+R$`l#oJSnj7syMN;iy`};Mh zdZ;|-1$lydmQ2$FV`zeGq94JiSUaRHFkY=)7h=TREi#U& zD`Xs+X)#t6`nRMjCjY~cI!k2k<=n|^O`jh3XT}X;N@@r{#-^idDlS=um;f8a|5P^X z=bDe&N&H_9%NaLJRp*+V+R>?jfcYY&R)nz+YKn`%BWCagk!5xn#L5m#D9lzBbwC)iUN^mkG71Oq;Mhr@a1;1)fWa6@UQ-i$DLL6f`8ck1rw)L zvCgGZ+@e<&-!<}Cmp^&l_>C*P`?;Fbiq?JhJZ7x)8Z`xpV8(x{`80&SMD zVv{H3tA=GZ8|>WAAi&jb2i<3m{?USw>DZqi+`P2w zsQP35lf~B^bW0I+mDSOu=Ug5JsnVe61}0!AHmvHC7TA05ojJ%P~2-xztU3Z z$ShO>k`L|%e&F<^QY2W(c!vlNB6ZJG~p1L{1mKW2v;_23eKff;xt^;^5 zj8w*h??AM+eVvNA*@t`5{c= z5gs&Vab3}Jxb+oghBOf~?q|(@FFICzC$3(nmj6m`?9ue(^w{dF!I{x5O_qqMkU`z) zFZspshD4RT=YY;)cG*6)JkguvxKi2h5`Gm~?CtZC%-hcUA2Fjtcc3Sz?A6#?YLn!D z*DU6%fC>9^&13DP3daW>6Vq$@^^U9&rINrF#!07A#a5X7SWwiNwAu3I2wWC&9Q-gEl6KFfPLOb?!&&cf zhd$^Zf~nr^$)*L@Lm>Xgp-Zc*FU`-izCI<(w-{MNH`7*}#;)98(MLtp==dGNW^Z{R z83}L%LzojGE=~9`&Rc6dw5I+W!Ij_)=z*^USWoyId&eBTYr^KKj#PXSam0VjcHW40 zV(-;GMv#Gb%CdeAdy7E7QeN5ZlA|5 zKX*RWcpz^6#q|K+@;wxAkBR3ekV_l*cMnc$sJR-s7XM@{=@YwymbiFiWg&`!$<#>% z@WY~rl17A%x>^n4FzQ=ths%TvW zws4!c)O}DC?DOtfz^7+<>UQr2^M@&li&?>YmVw#LTfsSvgGjp4O9R5^aor2yOgOcQYoEx3k@KPf$Jkf@T?~Z24i`BdR?qR)9x>PmUZYTHhXmo@s3; z(Dn>n26qc+4Sm-Plbpt+IF!WJXFO8CpuZB&{t8}foQfSUkh5!RhNMzNH%!g zrPs32yFWSX17f}BCG7`Rr0-!I4Gh_OXzkk?a|pirIaj$XwQ2u`5UE90_slNU(7L;# zCFcme2A-YITzl~rWePitB?X;YXy}hFf18NF1 zIEA5WSzOKsA+4QjtTv=SvXIlH7-AFY_)e*ty^mTYFVd?WE%9{srEUz<5nMKX7CJX!DQ?9&PpbB@I8tcnWAF zfc~*FVPYuH(F-@05Cx;%x=ayRu0JWJlcA!GtzE^s?@)OYT>Y65x8g0DCnZ>5UfOg9 z2)h{Na=yA^Kc)WJ1F5M7v%TL@S)z|6664E;dqO)li+&FXB1XCNTOyny?R2+HnTK9q zVla{Sy@jXKoNRzJ66~pzC*2D_UFR(aTW|X@p=Ie(5`O74gd#CkSJrzajLZJvId7VA zfypzc`REr%Yq)bPV#Ay^m{q~6Z<#W`I-;F4XgU{rlDv4}_~~+hvDP*70`M#|`ulq} z*g|eS%~s^I#xq08WCJaF_P|f)QN>gwZ4lzHK;t5H(mP>EK7Nqw0>Of1#NsB z(z?-aotdO|^$7RyDkUGA#Ek|@wBObMjK%~GEn8FjFxNJ4mepR+Cdp7CHsIexaHCnx zE>^?L-rk`NJ({C%_vr>sZG>EH@S66}|Bj3)9M=Ga*o_FKDv8<=HfvSxNw2A@UK!kg zhrqX35Y)b^FVLjkwJYPw6}sHn3Ff$d{(%#CAfzlAAlw+za!`{x=8H{AOI!M_pM}y-I3wvyQfq}G5=z}Zmh(nVs^niJ zu<7~Csd^3zT-%igm$eVG+L9V|@)Ls3ywG0 z2|p0AIHJuxQ+!u+{k0)TzUlTPwerrwU=(AnHp}K~uUCLMKO(4FVLi&J>B=DKlk)i( z%8nEL_8Ksp9|p@fPIjW|plpF97YC;a|wxJ#lM?$Dj%a=wbA-Hv9K=*oDh|F^I1iHpG z+e{%?Q62|82aP@0KY2V-hD(9f>rK9W&z#yb9NV zKV9HujY<&kZ@}v%F`urdO^X=96oc&iKY2VNsmTdR#=GBFs(W9F^ix%>32yrLn#Yh- zzwIktG&FGoc=j%DZ=>F)4Cr-V7K8^Cs%YYoghILR6FaG{q^501^JvXOSXxx0b~M}| z<(K9G#s$e*Hsn|XHl z$va|Ldm)3FVbt#$wI&ciix&RfUrZ(g+LMvs1oVnO5k1`IcaA15_t>ZUaI&@=@;G6* zPh{Ph_ao-w+Wg6?L?a|TlJM0MFI2^ieFs(@wvzXqt0;50uV+4ljf_kM={yjELBi-B zNv)3HV{PS%&UqT+Z(ec;Y2Tct-Gpll$MGsX1m5RJCeyjE^@Y06>J%Z%Reu<$NdITpgohNaV{e*v>JTOlEe2Y7`=TAn}v$#u}Q0Ylj{eB9jyXcT`Qt8l^@VE?2 zUATltq*s5~qU~W>~d+#;s z(ZF!>!KzeIn&u4a<4ZnnA-;X<8D5o5NQIq=qxBEGV>_SJPCkTEZ3?|j=O%mJ~X`LR$2 z8PgmdrD=T+3sk@6+dIRA6cW&gp|pCl=+muVLeXCQ&B~o>b9*sqIcWv?+1sFY=5{^K zh?oNrEGH5)&1e+Y5ajMdntrQ9>UH%$Or^`Y5AayUG*4_3!}^MpFl?`d z3}vb(I(F0eK0qp%5>R92vrV8NB6=V*+2&!W;q-5oDbZ6~H`X$;pmV|H*bD>`)6J$4 z&nq5OxvwWw9a<^Q^jcTw(=VI%c0@pC-Y^{V75QC@RlMp>^7c_A2pIsRxRUvrylEwd z&FF8Oq%4#+H~oC;KSZ9D5$xq{jky|7c=e>kk{q+GyhJ@!TXp<;qi%8G1>x<-wu&{u z_om7Q^8{D7YTO15@yWY(FfxM3a&h6tlzu;aFjg-LL>&n%alI+ca7MY8b2WEG*!G&` zo?^%f2@~{9m|(x|yFVH^d!(SYc;xU?EmXaM$E=d;a`4a1l*0GyLq!uZIfWLUN9{d^ zpBZK2lcEVE!1{h@s4T{i&y| z`bITpmb52n+W}{c&FL?%E23XF%K!qwVJxM`DMwe?jV)9v>u?w&qeAV+9i#4UDvomI z208hy%u_k}CqsdWx4HlC&;XRSTFw$>b7yrk-0T6z;YrWW?GvA4ox=x(73mt_cHT+R@Q?k5s$yG!_p{Uk}ir>^! zbN(}>4?~8KVx|;aD_BF5vL`Ng&8RJ~K&j=cYOMPt&+Flwu;jdZ!{JU2M?lKzU`BQ~ zN%h)^-DR|9|p zlD(!>s1)|f3rU-W9>VCg7u2AUS~Y%*etP-4Q?bE~(9q@3o>4yhC5A9PuIgEp_IzlJ zprYhzd=G6n>dj*p3(@C^2_+VV^InyPtj2h$l_g(I!3NS|wGcFO3^VE@nGwJ_GDK6& z{e$YS`Sq|UL^seU%eSWz^d^lV;R5lbeM-6WYQQRl94fKqrL1O&ZR@GP2|AoM4Vax{&AgEKY{CBsw*PUH^sO#w?Fbkz8fl84R{t@~qMA~c z_n+W4|C-5v@^o3DVgWCfC_+WpB#t#MM1Xmf(-T^k&;w3g^9H9$@`uo0z5a7u`$7+a zfrI&fRXL=OE9q!;VcLQ%$)J_z@$ry>p@F85a@>8a5=O2@OTkFk=LwGr2*H}J5^~s` zqyL3RJ>=?7g5%sJ2 zD6dDVYKL)a7GSvRe&FLCoVH!s%DnJ1>TDGzL%>6Ga7_KjGFi4C_4>NwzrU$J%yCeH zcPsOAevdWPAw*^Z$4KIR?d8Mb)RF}~C1k(_)?lsi#c6@Qq`ZyWttdis-Mc~^O`T7^ za95r@Wt4!<-Pac2Usk5xPbbNjsW%<3RpGe1uylE(?27uriCvtfg_Jv#q5i$#6R+!V z(hn@i8+IEy**tek`pjJ&-pbv5V^*juwV79cGH@QXVG`spsOdM9SL1+ZX0Y(^YKpw? z0ISygiaNa~mG0E~gWvWz_9Bw|^Ld6$x|cROpChwR2iJ=Q)DU0IKL_$$BZ%A*g-}{# zhPzZV-OF*T4_w}=9a{>nXoXdb<@ymxG&;|$&BzOiR@KSk39et%5Hm$DANB~=^L?D0 z%w!HlU(n_nx}Nj_E2T9bIAq_F%)_s)JA89PLH;inP&sIB(4Zc0_cE zQ;uw_#-z+b;K-%H7tKBegrr&ZU;jT@^hV1xl}P5 zxPH~OQ>SKoO6CKg*d>5sAAbHdmx{vyN5c(#3Q34^w7JyhCfIB7bZ7Ie{d>oY!lIwA z)8(3n%xq?N4vBgcKOm*{ujhsb+^E^GA_96;#@cHE5mcuk0tZ=;=N1-;oI^xyl5NMCao~kYoLs; zVH)pNazNgktCn!5<*4a}im|F&aLq7l=x&RY#XC6+H4GVk^D8>#de3>o`kOgqdou1O zTwUK93)UbC#d)BPaT%j3D6}ah=)|GnSugdfeqhb^%+7#YJ&pU!YnDh~Hp%hr&m55`^o;VF^NW5Z9*R&3L`c^jR$PZ1Y&8yA zXr73fNS%87TEe<><96K9R`+@)k|p$MqsP89eD0$b%Cqy(u*7d-v+Yj}m-A`JK5eWP zGHd7vY$s-I^{@8X{oWn^wb_4oS^~`FWorH-%1PsLT(4<;)u$vYHM!N3k(lWG=B=-r z)m`UqaS97og5tGN!yD26hob;^z_@dH;8!{>M_0997MDF&?9TLojG6Ag);_5dSPHi zXl>B?r=PZQy#Mjeg`*YOEkBk&MNlBwQak{zZYY8&n<*HlB(XpAGZiJ>^BUtc7<5IC z%U7&KPmCei)^jKP{p@^VjCe;Bd~qtd0v;oCE&4miSCDo>$edin!vGj{X_q%3m>~t) zt9=`pT=>ffo1q@TuB||I0ukN$C6lK!K=csk-99Gzq?<)(Hhs7LcTQ1DAcLtk@G{YMhSo^rrqV$s89DT0ziLT~Tw1_m{2Q^z5K_;KRs@V?c|Y5p2)b$ zN*(cfocc85;4kWcKg4{?B;95MJ(ks$mZRL{QXa*sBuaDIx20oN z49Ji=J&K-=G~hIR-p?5` zI%=i*{Ce)l<75YB#FO{+PYv}QbVBiQH@&N#d|s6GxEZ%@Jn;kuf0J_L=<@!?s`th39yCI&q0xblSF+Ci9XLHoH=imweIUVEW&+vH!wsf z1AQKEU80jVR)rL3Cp|M*w(`zQ!xihvUH!@Q?UOVAk{g$#t;c>w!)_|{YlXIdu_gsH z)I?5X9rzs>KYx_Cpqj#=D52kQ=uYjvEcq1vtWdDa^djY78KSby;exSzzf!yZ_OHpJ zT!Ht>r6pC?JgtnGmDhPBBoMg9&#_LzA5$>O2T{tgcfq^;KApL*AuyOp$diz?%OB3E zsCm)+vYdOyNg3q{qr6ODsy7#tavA)ma#7jB>XVRL=Fhi2#KDM61tF|v9-PQ*ng`jO zM=5zs@a}|TL9@5@%H<~zV~`8sshu)xQzVo|e5UA-5_qCl09Ky9UM*UVIIU2??)+SC ztc;*!<9$nV+9kD1l^!l%){Hy(H3UBkh7oQO#qwg>i~kQ>U;Wqg-+zsWGEiWnCC5g7 zlwjK|(r2V01cQGy{W12B=7PgTe@v8r?`p_vpOm^}Vm}<8fX0{SWMyJs;2a zd7X2f=a6s*)}k=)KgH}I+`G=oL3{1^kHKxx?$p*N2~I_BNtgPjgqLk@ZV{iJ(siGm zTR&4y_{YQfKcAN_6l)fi=1V_RPx5CQBJ__uhGGF<8Ffz*uTG24W5z*T)(UAWji1Z# z&Av2#J&|+!8%_(-*0ocgi!)ctG9~2-N1qI%gJ53eWZ7b_i(p%Elg(5QsE_IQ4~uLt z$}z9jk)lH1eZuVV|GpLuT7*)gAfnqpu1-1aHK zTd`v5SD)`eYtv?idWp|OeZtkrtvqYO3|pn^?hY)=)*4p_f4YA2)4Fz}f>ikT|^;#syS> z`J+fwfY2^xqX0)1e`_^=4fn$=E;}2s?k_s`-z%bDK=$% z(wN!-<;b_(vzg>&Euv#ow8C21_~)YcPf2W(!vYm^esh~E1m2VG0ym1K@O~ zD<6?C!s?}u+7=X!t26>We(4t7vDW}@#*R8b%AJpm+du4LR27O+0n-hC2SW6ag{$$qGW zi-;A?H)I@$9%>_f)aNdCqmWLd6`@VjW!L-btnRZHC(9x6u36K(;@N?7V`(=vqM=We zWznPf?NeCYQEUw$pLAuR9(5HLrF#|#CN{d?k6tPHyt%-|rePxV_-2~D7>WbGVz;QY}J^&^K;rUQ9BW=}^ zR3XRul(43sZCFUl)w`Z^eqv9bvh?wE?cz)a;jkpK>*#NQ7o`Ai94+Fi_NhL(lw?~Y zx)XK!oU!4~O_WC-_rhq(W{vunng)vVyB~B~Y;PKEF~RB#Y|6~Gx?f^jB6@iIz=>pB zh^0n`w(+_KUMpwvxEL&$5p2r5HcR+j0LpbqWdF*0l~|Y?tMt0J zVxN+k#ORkj$u?ACH)JA5&+ z0Ct6Q^M4?E5M*K&WUXCGx3|h>J4XQkX&_@$M+U|-T2S?$QK9^uzJR=G1$q2X_P(#4 z?_JRMTDBnB5C+U#QwQ68+D7UgqhrOJG;>;B2(9Q_q&8@mJO%e$GRiKyr}1jTYimPA zZ#?w^etGc~>1!bg`H<6kFq z_O2xFvL8#Qd7Oy1ZlR@@hq`h!LqGM!i*vWw{4Cf7jiENnJC_MX&3BylPv3RrTU7Ca zI(p!9O%gkCn;*XR+@G=^Ma%0uqgFOXcLOmu6%U+^B@xg>P;Fl1Ab>j{bk(xYWxJHE z#1#LP))Lk(#(`Ve-PqM3PVo1MfT04oE;^Wie2pjn_(fUN1;;o~a+Pve)ln(pd(0Vw zw21VcL>9B+r@wluve>Y1IBGAK#sjDDE8Ko8)YdZADFqsT!ADnm?#%QoeGQ;#1)XV* z*oN{nZ%%jh`A*bR1#i7vQSh@4XgPklBDepvDI0a<+;n|zf89!fwXOaD`@idal73q5 zJm}^H{fSQZT*OIS_Z)fay&%~INvxY@&SRf~99Tlz`!MtV#fX2Y?~<`{vxESk&q3uc zPIfC)lP@j}w+X#`^KZ4?JlB>uZr-?Qsr%co$~ApD@Va~Y$f#ZQ;r*=wuPNI!rfPXu zC>VuqL8)W!ht1trp>arC^Bt`+Cee6CA_AeoeSP0n%RNG*2X4q$q|OZa!lQN)3sB3L zl^Y_taS26j-BwR5#Q$#InHefM=qI>eUZ0O|>MC}k^|-Bj0o8y_dY(S;JJ5Q4q2hMv zTlFAjo^rUVM58+4k6)40AJ7Az?>2o{jZ6@b`4wXundEFH>eyd#jo04(N z3HUMZ+sg^Y#+f5TJG3iGjUN(r;y$?>noknX?S~@PyK8*IqZCF30>l2{tWuemU;jRy zwtF~9t2q8m*oz)`sNqr^^N_!Hr0*UVv;yTeBRu0ALc*hg@*LpV(`MJWruCIbTs+SU zL@yv&9FnWF%jsKQ1l_|?clF>+=IxLquqNBy!&Kz?v_Ly+Wp6wzo_F`qeC0_`Krc7{%JjL-`~)x}@^_W*y~=q^P%ydtXlrP{7i4s8YcSByXC#R*ajC5$(GCsfaw^-UL4E4PxE7oai29RO71AS$BUAKm@K~R85Qxp zSUI@idRP43qo4+aGIa&13-}CF%~X+GhIR;l&iVlyuXJcpFVy~txh|DoNeGN`P^+M6bB5{_UpA_xa? zM_)em&3So(f1^5$3T4mgEs{oiM*Mmh>hz%^asaL!tXN_vd-RgorWDW0V-C_x?7$GA z_zB+)!HV?X;ZpDa;U!yr7l%}HE?na4A%iL78_pltyyE^15pm^FobHs&-oYM% zS9|ZC6}Q~7c-0^CoIO+Gavh6q4=*2e?u_p(iSw~DA;0i2ZJr$! zCh8Q_L#w{>VYGtmE#U;x_gm|9W!_- z9e~v|beFCGrh(oIM$vruRWBvrl7@i|!QO-}ejz)rn%718J_`*ZSq@&=+}1R%?T&bE zpb8^u8y62G4WPL_DG%zLg*HUV8ds`TW=GgdzE@zo{L&S|So2)og>ee}6aHm&N>0$! z*SW~6(CNjB;(}2F7)_Mg#;Xwg`c$*sR}1dl#Fr~;Y?d(U_fv>33Fha+6_P?je70bZ zB|&LhZP;FD%v0WXBich`ACO->l zsoIa5q7T45QhkzBDaMXs_}{XGT_%#f5<5w*XvKr)Y%BY2A)s&{PG#+55xbVrVQ1$- z@jV3iG7u1Fb|54OxrzH$IUMjgU;2C@#p3OsB0_IPKF&*v)?LxIw`ukZRJ}dR{){<#ap+J}HX} z5{N$sH3P(VOb}{4*G2RA`{uIKV)##%4UT_@_ND?uF+!P++(hcXRx8P_b@0@KAO9NK zuKyX@lbDSN-3N+3C(~aSMJI1+CUmPgLasms-PTwEkvers@GyE1QD5k?J3L*Jy8KY* zHnM;#`R1R z&{ejR>QNGn^188u*X9Er4ImU)&$QR>9mnf-`FVgka2)-}v(E8reu~OL1JB@SU}6c% zwiHW>7qUe&S){)0T=l~3%xi8pFn~f{{g$(a4>IMR2dbLINwEuWHSd*qt2xFN6b|Pp z$raAL=;Qy1h8{0JsLSuH9glj{VK;4DJTaq*6Of@M-$^CrC-m9m7-?f+a3AgCREoZ03s9 z`+1w`yvPf+fa3-(%nQG&N2Mbb;Lfk$Qx`{1tRKy*#F_Z=0&zzqBBsyMlWJ zSJff)L@HF3ljeIayrjwilP$cK{wmD$gMO?Vjn%L&N2ei!b@RiRLz^+${RIUQdYpvO zra$Hm%jeJINf@nh{H50H+qnYa1*vrEB)>e;wk)Eg1(P1l;AKbCi^#-}Aez&hYOiTg z&GG$!GU)u`!ZY=bh75Ge_@GwxTSx4j;~kJpa5Zv#Z<*kl3z*tvh?z?|dR1a|=KNrC z!HGs4TV2}Qzzn;s#~K8MMONDY@2MW95Y)|Dn8RdrhR@wu=F0B0^y(qVO!6^cVI1WM9vp4QjNZ?b2R1SoL z(fp7PEoJ*<^J#09k>ai9GFK9=*pne6g3!Xs!tSD1b%}pMV2~lg1ST#n^3Gp}2tjoJ z3l5kG95OVEJ?r8$Y`!T!GM_Qzcg;<67>PbD8l|y?mC4CPHx}2qLqATHI5p0XoGA@J zok!}x=Howfnv`A#WhV-6pVZu+ibrtf%dn&LN$i|3{KG$_9)yRKcKQNzx)QSy&d^YF zZ@2ae#1M35JcrvdZrnhBmj6dtKKu?(e4jAbeMT{AkOgn~2C|OQ3j`a*r3-~g!P6!! z6H-1r|2jq^Y!Kcs@hgKilwnH={CI#xGfv5s50M7WtCQF_x?u+ic`J)3PA|ldJOy@q zZ`oF7T=h_M-sqb2M285yAsI!%x`S5)Q*4vf`eMX5*f;_~yX!{aCGmn1F=AA;`U@KV zO_Uo5WOsK(KshnHI_IPGo)T7tM!qlB+RipF#F$&-O{MW5=O8r2-Wu1^Zn|+k2 z*5bNNmG~xS(4<0qWn=vuQ&1OG6gwXgR($kos?Z|6h7%|?cIcdUD8o3qqg7}OBf6F_ zx4!r-qQ~Z@EsN6TXp+v>i84VQatH8!EctlGHW94Ghh&}Umzo%3jM52sx)lG^cvJ1E zrNzdE0tpl|z|&5}YNw?q(fQ@kow+g6V9q7aIVY8fZa_yj)sK3u9jvW^HtSoDKMb%W zTFQ2+yy4f^{SOos!eg{+={CO8kE*-SZyasI-Po=mOgFG+f%X#xYh76JK;OAs9u(WM zwLk#YEuy6-^c4rD!ws_>wzExU!jKYK-;Hgb;UR`01ayI&4!-N?H|L!w~3f?lWS$I5dy;eg=v zt@d>bLH1x~Nvn&szqV$|zUb${1N8sTS|ul#nV@QHf;peM78P6FeXbXvU1=2Lso5gi zNJVk)$51ckwUR_{Em0&ujR?8~T0^8l^8yp58NQ?Qw$fB@FGReR2Lz!h5VlYWbZH+n zOf2byLh}o?BCrbKD4T^`jh0-JK6B#4Uuiyq`fe?&bwh&5Kzu)Y#^YLpqZwgTO9f_! z%lRh~udfS})wypx^c3pUcA=vv_nWYR4uEP<`pCl5y4ur+mGD*N7HT-l4zl7)L{=wC zV>c)NGuP>Ku0=+|xLR1u?L=w~v%T59_(Hr<5``V0YP)D$Q+F{lo-7}V^cIYF7m&Wc5|zi3 zak#;N>_ypaK5$%`P0hs5V(AxdHqtnqkd{ZR&ZjuRw}H{98;&_;?6S6bnoV}4`1je9 zgHGXQbVMuJ7;S(wM4>3hK-@0z$ZT~$?k4OeNPhGE6;t+1V+@4<*_2b#fo^uRKYq_# zeaYi~Ji9`}=8~zR72#gHca~0vY>;mNs%r*~kX) zr*opvnSn<}hQ^Nkc@osD=!7M2*Wlin?ZO0Yt40PBSG~JkR191tX1jJ_H|TSB)!wrTN#Sv5GMrJ0mZe;{FLZ3;P>2oxQgS{ zVUQ0wN4H&9++4MVaUh|taEAP7)Q=omP}q=y#@)5Xl5zZO0H4Y#Z_8~UT+{-py2Axb z%PAUdmC>i4(?)Tl=sYcM+vY}9?>Qyqu+<+)+K zocrH*2cwyccsOsN-e3I;fIlviU6>At^2gjmQ9&{SbU9xs)`)V4J;$HRM>MvrOe=0o z_H4>b%Z)hZ0o7e&@l8B!Mi$7%VRMrB1VKf{5JEQX-pzKNI~51m8}J%Ni~4e8g73@yGsV{NsZ z^SWRB2!0gG9ivGSDomW0P$~DeSZG{#vWm@#`slG?zO5!rFG>@swa6o3)EbAQi=_B# zhEed~8^_8`M9-a32&*59^$FbF6AiN2M(RPL?cNBz>^l>3TilgfYxfTt4Kj(>mOZ0b zUpY5(RCzoNUDjMZb-Obs(ImM{U>ZrExUq5vS36&LO{4ZcS3c%-Hm(y&Tfq6RA_4m!1Z^DH378iewNc*Tp6JunIahrh=Mb>3CWOq>W z&v`KD+-{#dISh?MG|~k}tG@KenP5S(gGQidQHGp+dyDCXChKXO-_!?bCnoiEn*Z7P zbG2?Z9-0uV_7Hm9TK14BpPmdu)pn32G51ff;SVVWC!~3@PXX-5# zT_3Di^p*YyoQyyqlaCO`jlK&+E0Tubs4MdBZ z?Sm4wQdq6T`>1v}~7e_rx3I)@BIRNBwFfN~u36k*G8_#jb1t?XFqpY(1ovqem{HE`vs1n)>H zYc+A4xhiS95Q%?sU>M~1iCRRRq}888D+ zt0b~g8AU^L@|Bz*;T}C70N}4sLZJab*62=Kh#dy5n3>_E(E~AUww|Agpn4!J52w6Z zP=HZ6VQaFik1Db-=?y#7h5@>-lCxsKIVmkwFE&fr7e~j(q|(byzXmK%#9DZQdZqe| zd=NeJs3Ie)Y3yC6p2$WeZk2B1^JWguGOtP5SsGV4{~+A}2_E7YcLZ+n`f&I<&7q`4 zWN&@_h2Xtl)_)^bHoxdX#b1QyzGX~XKv<*%n*d>-s{EA)mScxtf|sP+xYO+pH#UX31078{h&8B62W$_Ce5>4Tpa<;s&4a2uC%J89 z{1i*~1f{1(tD{{fG&ISQDSLJpegSWQdLF)vY_bQz#PySflu(H(U0!it)e)BsOy1?sio9ph4OeF$}{}I9;LX^fSfVX1$q$mDCC&IdsO=-`6tnu^WC8 zXfv9aW$uF4g8RwtYS`;>ZpOV*Ym1@mt~PXU`$)Kn_`&p0kLwoo9As6);pHX?<%YZe z2?MhsgG&FFdyhBvpRilkkmsrF-@DVw?_}2@(wfbats#DgVs@B2z~3;Uyt*@L=Ou-- z>N*Ew^73YVDgB-%y3ILDAKuAJ583?I{9TX)La^Zd_*;3p0Xu7j2Ma%%hqXX|ELkr0 zRV~#EGEg2LB1_Gi*1v>~NA}bPVX1mhy_=Q&|5>ngv_(k!1uDZAHHe8-yetJ9apG^9>I7)Z0Zbr6JD_ZHE3-$5tv=7^9 znwSAh%u5Uf-FAzMMJw@6M)2pmoPU2xBY4_k_bUU%CY{At2oMeGn#=3iH;q|$r;x2$ zsTwhNZ#HPX6-1L(217b*ta)RF5e*94E=}>}WESs?V}Dy_w$z+`o9|2&B~Y!j5`rf; zzl-{*2L1Ie(EjJPm9RTuj%W>fph!m^9xSWI=7z<$FIbq12H2h!M)a5XsH$-Vh%o6h zf_PnYbPe9T;PHEaiuke_rcQG^O(RiUe)I?xq4qN-L+Q8cbgIq+Dc%lZJ_leHYEbQQ z&0Cm;nQRAWBsDvDAH^95`ba{#0(pK&*w@>WmfH=ocHaFnCxN6_bWW5^Emi9&w6eff z^ZX>0#I$r_PZc`l=wb1$^iNC<@p9_OIguf9RHGY>W*)r^@5)0ZrAcZ9m>>8MF8k#} z$hd@nwaQh6L3S|IovNF4%{inFK0d`ESTP@(nwezNs#p^ELSQxleG8lMJU2J4H2GKO z2-INn6AxS6Xmh>;Bu~TRrTlpE7g*kDcSUkCiU)=|P*R||1yd%LyL{CDb=Sm_wo`& zw*774k2hOFJ>9aswQfWTc`gnsFn1F`(cHqDZeIyYEr&L-tAh^1bo0jB<+EL#{c*hw zyDw`v7pe*M@lV98R2_QrB>g>I&*y)5O;PF=PjOoWN|-MITKmQHg}tpndpB{ef$-d_ zGZFr@Xd=i%H4+LmalNk4(}Xv5ofdiAcx|AKn3HpFls%J5b}KbegIkT&G5^mW#?)VX zKJ3E8e)}&MeQ8ByQVGcNcUhZ`9_0R-l`^efe#UoTc*ldgshn?)!^e}Idmx3u#Bn** z%4B@us_rPXEM}E+!`8HOWio5|r(M6$JaXVWo6&vl&Lzoiye0r&MD27!KKE$Pb^YT- z4}wY2zT>CKPySXqLL-&k_L{Wh=rW$)@kapJ+u+Lm>h@>_r`}p>-=9 zozU$EgNlGGQIAIeH&HM&8z8`n^!2?^P1-ug-W!@FpHRYF)Es@O)Z*oC)b$$LYp)Y~ zwxc#-@mhD@%8oCZB0r%($9;esXrMKT>h?ms0nC9%zE#AM^gKuN{}Du*rXePGAox)azeux4uZqBWbZ#n|rwN%ipKK?F!{ zd_dlB-tADnhtwe!Mz8#hyW7f(C02H52b&?I88fUlU0bEuxE!#ZIU^3e3oAp3B$`J7 zem4n7t&PbgF2;GzIuM!+X>kcpRW&Z0J{d>f$GkDy!T1Be!4e46+AjlzkKEx08W5tM z@UonNOJ5v-)zJrtU?Jp#6wTEh^vGqi0+cZtMdI$tXM)*ey77f?!cy#R9xbVk>g4!Y zQRRFe_qeF!&u7@rShZ%W)!xcPO6)gJxMWwANbHXrLxfUivcI~dwXw+)Ip0J zvAfUo$aLh$8v8dhte#L`=kD%*o9=73_L!p5?RAlr&{L+CZlh}EO zC%DH!g5!z&uBeUio=K+<#qyj9(ko#B^d@;A(2%H8Q`ViUBgsA4Z*0hfVe zbxEppybCVfLSA3|Tuk}`r=Nre;fKwQ9!q=tDV63VGvxH3kIs5NaQ;lwoLV;VSRT?Y zK6nI%_M0Gnz8Lc@^?Jb|2Qki$z6*V6c08g9uQtKGlwCIAghL6DciVzk|GC0@m`)Tg zEy*jl&J2g-1(wDcP?K-}v-PMd)~jxPn??iKgI z%fAZ6ZMEQ$%Cvq$3rcz%Ck;v8T2QZgc;mfw`lM`>;aW!CPWe{OJSq{ecH7MtU#{kayQa1dx+>nV{20bon>b8`2Pi=aS#Y3Nq#P!`cNJPU8OR zUbejyt(yEyvG$%YwY#UqmK)rzTbUC5cHaA_i{@RE&c3W?U6quNrlr8*9`x6*D`?cWNec@b$=CuaDo>s{l1@L;nx*C-V24l9&}7s zyJDm7$uGEizEEN~@Cx)8*dcT!%NUKZL!OQzyemTVd<{|tz07v3QgMy(6@ znTiH}+no`Q1_**cL8qkxS$i+3$$k57lCo-FPgFh~n_+a$S44D#)p@!k*g6}-u zJSu~{;9O_-?rBjU1#|f|v^-*cqbz%`+S7)=IMePc{1IzPzo>F7An$#w-Mf!RA7c7g zTAJ3q_#Ag>tRY{fK?<)4cDH+Exhbs*CrK-bGncNY68E@O=386x!53NVC%xdKLsY(E z@R@EzKfhu?o&R>8<q0@uA=Qnxz=J=2 zvP;RZY#H8LR2`Xt+_P1fskkbh04W1<$U@vIamUA~dwuHPARkR0vJ@00;)-vFS|he9 z$->_4&NGn|N<%8)4>=21@z0JIn({x}Hj}~X{WaH?WG+3?9ZmYBqT*KQOlFw#jh>|_2tCvnJ;#{P8R#6?)t^EV`W zrJxo7Geml}S;O*t=*h0V)i|H#+xzwEq;)!<{aDN! z*?8cfaYCS#KVHt>%S=Smi}EMIv5~xQVlPsh4z;UDR}dO#j_|}Xia`q*F^NX&8h9IO zUuhCL<(54we@oWs?Lr?pNSX(!n+3FXAkF|Ja{kEGJ?42~50Ojrp}w@Kg5SSA2W#=6 zkUe%{F(^9UuMRl=LMxX?YJJD4M|bd8-9JVm(dtJf#~)^n(A5>`-%|!f(n8@0^qIK9 zpNGbb7|xShIVhrJ9C%2Fae=*rBg+PK{jBSA*(Y5mC3~7w)h~Y66`~ctTL{zpgqG?C zs$0`^@Z_U!h4A%K;~7Vg54V^Kxgt*haKcm_xwbYkxP#H{5Ai*l>EVz?y zl%?|W-5P1FJR6{Wbi@Z%Sb0{mvP*9 zv2?bCg~Mb^)BZ^NO^>d>-dw^?ZQ~C6FK7TF|9~HQ=<e4m-s z*hF?SEpq^1jFk($p_hv3TyKw{5|n>=ayN!ZQP4=dy09uMg&qY! zEl)7ah$PB7^Ed1Q65>xnaC?`Sv0ZXJ0ad3iJ96Ttq>Mo?yAszi4_=Q2i3NhR0Xz^j z+T{0bEArOlgp-;ZaN`FBMXxDw7BtGn?jAen#&}Rr-l)na>K7;`3-c}fhFaJms%{0x z*90DEZdkFi3L@bQpGP;s$l)XC7DVS~A|dMO<(Jd5(1c7oZGOkxtpoKlM8)~C!y(I& zh6vl~R5%bs<2)8lwLqpnv5n~3$KJqr|LrMfp?qi0fT! zu=c*ucP9|p^3b5ExO=;60PX>5(L5v#y#Jkxkij3AxW&*V=O4}Ev1Bs({p?<$uck?U z_nEvfJIPh=6Bv+K?LKz4BR+DaX72GobMhLN6@u(0wIZDo)B8Ar-TircR9!5ZiIUw6 z@%qy%)e5@<-PjLq^^asYk5t}5N6^ci7>hX&TS>IA9r9u9R6Q1$4JQFfO>3QfA(&il zFS})uDP-NOvDtBbVg<}eWUo^rp@dij2Yjzw8L=bWBl}5*p-=IG$9DbPO8aY2{e!Yj zi*#ov9PESSQiHML#&Uh_r!1*dq*KJS<68t!GqrJ7WTe>6?Wey$nqq4 z2SrOFz0rMzaf^Ls{Lf+yeOB?Px?Gnfb1QSu9aPR{J0W3i`<+9Rn*SWg+BcRRd$y6i z`ay!qVqpkNiRR3LBDxPu-GgFIChmtbcmTILu8>W-5sKyweyzKul6NaxgZ%ziQTRKh zF{uBZQnQ1{V+13?ziMcav zZOnP1RkORD96Xr{+&1|akRdPZEqh!!u3UKln?4Chc6`)#bUY1|Advbrqy>IU zENA5l=Rwp{WJ%#CW6$Wl7#OJjqtfZjy_e-4YECJ+tmiPwUT-Y(T3q5441iNaU8$WN z(^?eIYSWj+m+uPV>6$T#K84?3w=uf7>nEM*b8u|Zk*O2hK(W1NYuOeYJEhsK1;qfi zZA%tR7_91xQJbBJK3_oIsq=|@&x{8gxybC?+rn<@H@Q0!1#M%C{6q!P0{C2{g^Xb% zxZ6_iE4RssujRu+fSnSsY;2)>5Z88>L2vrjsmu+L@Nb4sKx`|WOwj}72tH$9wS|$B z!zrjvkB*Ggl|#_76+fBW+42|@M?2&AoR@qX>N1uuJiVH9rLv$T4u5((r)%d;CwIWQ z^{?5K{)cLH8s!YuM!?IcSM#J;K94_hLEd?S=YwB!7bl#({KsR>iV}FyQ&uclH;*?! zlI+L+v>w-;*wA))4;~6px;4UaL*H&>DQ1~K$=><)-<0hCu22F>g(^1k!Ksiu6;Ghl z<53B}cN{*y9=k%Ct#?gIOdaWXR)^fe7G z%4e7qq)ZFgudk+gBB#U%6$91w;Sy-|CgFQ62^7lHng%4r<%CnS4HwCC2h9U&Q&Xr) zZZb||poXS(BnzC8LW;Yo=d|LZ7^$RMH1YV{W`tS4>_y>pFAPEz!rHlj~8OgRGt&Eq}Qzd=($3*C+&iDg|vtI9_~ zoANYr7**H!&y*-4V;YF@B$tO1aim5(eFp~<*a`7K!O7>CaL^N6e!{Pr*6OyFs`(iD zHJ|HjoDf6R3%{2ls50#uNa9I;V_6P|71bNA zZlpumX-Sw1vDU-}1@>=mzl zd-|%g^eXc7q0>lUR7i(g93C@LOi59$yf{Pxvto9ClMPZRy7|3L=H7>v6XeQ|X}QYI zAIBFGyP{|#zO3-eGtpwfOQ1~A{YGYsXa`|#YdzlV-BsT+KRHrJRuRKI6so$wr03u~ z2v11B68;7+jadAFWp62cLTmiLUD(X(a6IX@>0e$oUH(6L)qks4ICw&E{S&7%^=vLWRh9}Jr2(&rfq+|p4v7uTS&`=wBBukK z_HfTts(I&Wl*y8i$UTR`QnrG`YB_H64}m~pm~@KjCup=SCI7Rhr3^rcyUZYYV^~`# z-^T;1Z6rlN&y4K6;MP5V|6FZCw$B@?j zJ|M3C;7i1#Xc`274o2JY8}16c0i#E;=qE&cQ_Uz>8;$&C%RgRthl_-Sq@FvRt|D_6 zH)QmuF4y9Cv!JVKMT^vjQJ-4Ylsw}E7}UUWLUGL^H2JOTK$h$`SM~9q;D;X}xjr z?XpU+L`Li^=vUeL2dyrPt~kbEnZH?kRAG^JNB#SMU4KRY`4alK>o1BTyoZ4;9VRO5 zm0j}6eAm=zbl#lU-j{g9!QE*UyX-SD$xUb!K~JJ=2;B&+$vg2=Q3>OHPTW%h{x)(d zP5F+5$OGcUrA_tS)x1()`{ld|uWK9~Pb~^rmj2~k?E5HbqgvM|3Q(($bP|fPQJx(G z6SF?RIT#s!{(kqe+A{llyznOzYv)>BnwKz-&G#L@vI&81PZ&WGuBnsvyL30fvEZ`& z76{6;5z%T|n}JWOiC^|vtkR$=AZ)hp%K>kMMliCE9(|81`y4f3>5WmFiHR5}zdS(+~i_hA>1Y1fYq2-}^ypf)ivHwS%Y!b5FD}oj*{~YSm9L zqr>wc>T0F!M@to*7`i`kI>HUe41Nur00L(ToF z(#};!x{>c%O%<+W{%tx91j@=)(rppG%r{a9*Rq}M|8kuEXTA;9<==dpn0QSQ=C!DA zqfNRvRT9;tGk)^+l5P5T1A`(Q%IFLP!a~9KRnj&IIVKVi-x?nTyKVC@5AOD(A7vri z>yIR@cnee}N|gCJcCiX#-&Xf2*J`~Z1bB)3V$2L6#9pa9O-p~&1W}D+E#Tu~NE!>+ z^kk^t50{FlgR?=44lN%nCwh@!K!&jt^Jn>bDCKFb#@rAY_urMm%UQSYe&4LC7biYN zayB;#!D5w8pk|f2_!y@sDB_yN09eE4hrLhSyk+m^@U_9`wU?UaK?9xtA8(W`amZey zE{$B`L2)N>7~9?h7qnNU8cDRaxY{PF+?d(gK~JhUM4DJ;1M$E`;@~t;tLiz6`Oo`n zCB~Tvk~{?ozt!JDTc+gG!OJbqZh?7h`c0;jUmIva!TGqfvm> z0_8G}Lo^a6Sn-Vd<}xB1C9aYF9FypE$?!k`wX>bh$>!;W$nvkbDE(RJl@i96Q6msb9sYKx|P~yFK~#>YGu*2UY}2BaamYY&L+yd_vaA99in?8pSeN zp}yxMaDEm;KEDBuj(vG8h(;Lf!Enm_AiYrgOaXi`id#4va^9~Tm8hYWy ziRCw0E||1cupSS@_Tc4A?`6}U&mB7h^X*ihW@U=b{#5MMx@`K{=ly_Y`Ynq%&Vcg0ri^}wNZ#%!{K6#(h`R_C++d5_Go5km9joIRO$Mji@{A?%N z(XOF<5{oP z9Fj=F`aUuI(^a&uh2)H*$u3#TAk>4_*$oyPbMXUnX6b)f7IUcj34cezTDDDJkWqK~ zw;%~sDK@&Op4wvxiX`G|foU5CODvae_m+@jodS@>C^HgOVj@gSlwEPnowjT@JnLNf z{XOJKC%YI(K<9Bm|I@@F~f*|ksFF*Q;H!XDw{4$>t6nO|~%V3ThKkYNL2&3PN zFLfGybnToJSj0VAZ7ehBZAKeplZDAXsr2`y?~S}P``j7R%{9ktcdSkbrCzd!kemZe z^7}$2^t3u z?{=)tZF^fqn$4O=K_RLE5)pTvPtMOEe$iM?KVk1&7KIq}1s_*0HYJbNHwGSVB#l4P zvzsW;l#^7>8DZVa;tiD}@Ri#6w;_H$@IL5)$Jwf#Wv?K&iU?5^x23WmDklT}@3Y=5kP;IL0yGm^z;_uL(knwJAs9Jh1x>!O0Y@1BXmP&IR}7yKW%9M`fkF1qN#A z@{Zd|o~OaM0~`Ub{tsJk;TC1vcJT_1GGH(RNXIa=2uPze3@}IuC?H+ZDbgh{$P67y zhqOv}gVHKE4$>(hAvxsG`}TSF_kG9n?)^91*KytFwa&GED-WDei>6ZQ$Vh$VwsJK< zJ`mEW5|y5ytX42MTehV#P>9j2_)^svC!@qIk^?r2se8M-2?)3@KBMWs+`I@C=@c6K ztJ)emU!Jq;?%NyZJuH0|{F-q0^QUw(s3W!)L?_ePfpJ^mljP`0spuA#PyKbdqB5tW zQuqtHzIqYZhB542diGPC#R^63+7*e-c^AoD082)8bon;_-a*phuIQ zC={%~F6atJ5ie(RXn>-9DOsfVvU$%dNsJmB1dn62=y>P9xvy_Z1UJ|Od=s`*2fe30 z_!(Jz8gqQ|=V}DYVD`h(*#F$3zQ4k8%m*q;xs`NQGM^P0$Z7tXp#CylJU(gT zn$3166moYWI)OZtU0$Cg4RZ>}t0?)IpP{mC?L%@d_6hF5coD%IGhgGPOnCyES36kvvV>Qmro3c_rw_d(5ngjdZ!j}cN zOn5~2PtWlmx0lR}4~toFd9c#LPzfKOfu%OS=)uh)<>+0O@<7k%^?3GIgKB2IeamnY z$Vi^XG(&ENeY9nHZ&6x9`gN~wa}sq&?=*>W<;ZyZ{HY=Z**-zrbb8m1`e;1rm5kKB&^SZMfC4K!` zGD$&Jm%#vCs;8X`6ui(YHI6Kq@sK&f)-U|-|8v@Vl&1h@LEVDhLtXForI~=NI%M8M zZ&#R14QZ*WJ)cDp*$<{06f(!WPakuIb(Aq=P~MDA0*d1cA}#eDKQ;V(4QE+s@6~#_ zW?#PBoJwQ%JEEJkV&m7_Mg(a8o;6BU%IYs%?*wBT-T1q8SwC`-B}YScdCXx}Bq5=4 zBr1zC8GW;tV-mH!x7k70BZ4(AYKH&lIr)>cI-k~1zB!VfP3;)qb#YswgOqg0w(hv4 zc;BFRE6o|MQsT|#Ov}=IzqhwSRN)3RG;3gxzLEnU7tcx-_PD3-;PWM^W3T=6#;<_& zuQsCtv}n80FXK{s*=k;uLD}Ssn;kdG8^rQ3h=83tM{sT^<7timmu`8wwW{X7TXsGo zmZ;^4Hj%%Mr9q1ST2@|lLH7!r$LMfVUBv%gv&Zej{H)3Hdw~72*^t+EiRi0n6)>eZ zq$FlQ+NDd=zMyGu^M^~p9_zE03^-?T$;4?v=uNs~NY31YqC?+HittD+johniJ{KR8 z$&em03uj@1v8|VYT-EONwZV`?)kU+NlT^J7I#{8WM-8216W`c5lCHX@sAlRgwZ-L9k{EmI%vF;;2?dLBTM0>6NbF2nQw)(Vinf%&3tSKuU8h>EX4CDPJ_9e z0FOyO+oF+mzdPLre29-;GXd8(ou`6+rLq5u!NC<`O1QytUmrVfyD-ULDVlgmFs@NA zyJIDI-7@Swxudt~xC)9QME=oT4;x(ti{LF%%VlM5VeufBcAdJWZ569&n$m#4Ni?>K zJRz~4(CN7_msh+0ne#-n-X(?VJkQhOu}EUjz|Vsq$>|K~@sK&&&xffU4~%qW6gCzG zPIjkD>exENcHXpCCa&(+TPTeHF1vaV;VK~$Uk4cSi~ju*8XsoeZ3Ps-1K?Et08S+c zr^xD&7cqT^dF=~VRHeS?mjH2)@0Fn>{OpKcAy`8UKJ+@x8C%_ z&KwIWFN%<}3mL&{N5&p!C@PNM6)B$u6E~4^@)eTr9vV3s)?!}jnI_buXH9A3XNUuB z{$y7%zrKL>JB4N{Q%BuvdTz5#b$gh|g26FtUrC6~A7g|n=3EfbnYJBt-(z!ubxr0BO()jJaq)mY3od((m1*lSFd`dNNk)yC6!Q z05S24rKe9KPB^^&lzfw8v=`N&a+8H&t#r4QJ|Ve^F!$XA#A;&UlBG{i9uNEsASDXO zl-*%jS&!>yRZwA9i97t|IGUTa_lN3g0Z&cQa%cM}n)|csabTpkg{+Kw&~!wb@N-&O z7Pfm*S5b5xm15;bf9TkLNc5dvOa}%nG|mGMMd+Pw*O<6?^NoN9Y|@=tN_tJ4O5c3E z1YnQ4r)SJw-sxD(M`=3Rs+$B@QDVtkcK8A*UV^Xn9wbO;J!%N3Uy$66nK>UnNW^fW z(oJSS>JZ;FQC)-z8$Md=3F>S2zn}n^N2Gn7Y)KQxg`9<4AzAZ{Rgc- zI6nPT!mvP|QmbH7drFbWt@D)~S$Bnuu|Rd$s=PtGPJc7(>s0JzvgMa^(Htn&G=>Gh zoVm3=%(f03_)w8s_eXNqqpZ_5v*ekF*mGH{tpwhFU;F$M!)II`@h4yDKlUjs^@sjS zs`#?(A;1hEyhU-D!5#zav1v1}*Nf9r^97)EWT|T5JFI*rp)EvRHj*i1XxlhTMG3fuL+oOqiCh7=drsR6eNQF>f4iugFu_YQu`J zEEI(3jW6)dW6x#`eKtj{gkqkKzgJCN#>wFWYM#QYeX_d^!iH=*TOHEld=2H*v_nfZwtzp&~|3d6JD08u?x6f(-#ZM(>xh zRxW9Dk)(X28}dqyqW>GpvNpN8K-Ae+r1&p!0pi4;aM6{~7!H1c?y$|(q6>kCG zvgS^lYXZ+P&LJO8IbV*bF1{iAvlD#Z3sil~`UJyBQ}&Sah6zK8JLF-(9xh^L*Xiwt zPdWq%L1!ck27);!Od{g8OJxA0Y93_B?By%}HQt`^-y8d{*{f=TzFMo( z@gAU4vo9ClA)X6t^!8f0g=0pAvu7_Fvp*bq26YLISy*k-mF>kKU5Pb-K5;W9>b^3X z(BrOe{u*(~FTmRTF>ghl_xs&ao&DnBfuo^dw2#X?bYMc+8tt0A+bk@rcs*c^Q~RqxK27V`N`jaY;Ch ze*0iecl5I#ZT_U-#^*R{ga_cn5MCThC(N6|r7sl!LXLqGScx(JuM^lCjw4u&(A}dl zz4td9H?rSoFJX(rsT#6%)-kJAv6hWk^QJ~4Ys2W>Bk%7w1$sM0GML34^#`y!(%j38 zYDHATLxdg7pYa+*i>SOj$;|8gN~!zl2iI;YZ7%CK=m5QrAv@?rP2rTE2}QHU>SKY$ zclRt~mJsh4$SIz5pdzv54}%)5tCe2A)DGznsT>v1N!zhpX?);lW_r*(zfj7X$9=F~ z<2f`oE=woo*fN)4Zr#_iefi?cM3eu;I?(1ck?|pUAscD5-eSYe8*{9_P<%|O-mkVo zt!lB+A9c&4piEs4=2)W=^|6a@b?> z_L3qXr%_{STBbpXxQ%38iHsJ4q4phi3K>xmXBY+}nz!Tkhv=ZZ)w5qFufAZK zXi&Lk&4=C*zt%WGjN78ct3SNA_@Zr;taLk>7>%U(P=|(%er7C<;%3;!Km%g1!I2$3 z>oh8YA7~lbq`35{KmgXZ;KO?R7xMARJ@1Q0_MI0m4GyDzFIFrn zN6TjYqmWaXBK{rKbo%-j8}@4+m>>V=znqXWBa+#&x+;H8Y2mGq{)23!xiQj=ecxHQg&YX@2u;6j`F5n1``Hupy4^q1+){J7*bMY>uN7#F z3sjPZJr-xwrGalLQCVCC)n9D5(0m9vdkQ-bhf=NF3)*x0o+nNYT%UKp*d4tj2TA3n zY8Xc>J=Z;l7bG;J;+#OCjSVm5(3agbzLV0yt^<4RcHQLctybkt5~DrR(^{K1EnoS< zzQwfM&^FCjbytpLK${|czWjQM` zKl+M~F`hQO3CsZ~=jiaA{K}@q?;7&3Wt&?8hi4R!fvHJ`pInwsPWaKl zU=v@0+wo)r`8Tz{FBcJl6#<(!^N;Fi6ZUW451{n?(=bzuj05OWd=JG)9X~vuZCd5S z`{^a-yjb8Zno%N08{&e_PLOc-UC}=Qt_zuSXK3@ znV6kXa)kreg@C>L71=^x&mi@5qQC$;$EsbYOwku;Io%`~@SHc)skq%2?I!0^Y!KFH z;;Zw$xcCN~SahMC6p6MCgapKYhYJ`xx@>I+^`ckBHS^-Ib1_HdO0;Z=q}RpOH&IuGsYDRoBDh5> zL7TmAwMh5U8?fa+jDa6$;vmk*; zRrZ@Y**Rhz_M-Lywu6CvvWB2Q|(q5PW_! zh@n8$F&sb6@uTTQ26ruXA1xx(_-6M9Fp=o(_|sU{r<=W8e$(PDx}!F7RB9Ax?T7 zp-^czzogB_#yo@ z>Q{*t#ksVBE_mwSxIEL+Yq-L(7D!0q-oU<6Ou=@`JVtjk0g~tIntd)kU#b#QnUv0h zh~P)S+M%PPYPAu5x4TK_(gc7{XL7G8V3W0~sJEkPG{P4uyI-gUNP{U02KzIJM1As< zo`bWn)z4fLV<$cu0O&g~iu<8h&lPQo2950m`z~fu23f|<-2K8Vqi)6VS%2tFagC}| z`L4HuvU}$Kjxo_=Iezy8VqMcHgZCy3jXZ+#j)UY&%RHBhziU+8J=v9dp8LWJeVodL z|Fuilm<0!Vs)MQ0(br?PtH(Gj$bsyzQXBdTewwye6po|_kcv1T8CTMW_y$pJR#kpx ztC1`!JC3*g+$beLQz7#(m?iq#8Rka78?EYmJN5)2@9tEzYPQCKo>bS69{$$iKI8td z_nJOXy5jyQXH=>7rlfDO(tB|YsonP?vtGKh4ANpdD8x}mz=u)p`DWh6%6wovVq2lV`yhYKGUx} zK?F1G7c>*8ggREDT-@4|upxO!oUj^VK%Mr=S~;46AM*A@2h1pVf&cBC;vYp}*He-3 z(1%LcGRrcdqo&rE**w=LBjA>E3B*e^gb4s~U zmg=J@a)>e_0M;dn<8zQhS{+>2p@98(X`v@aCg!FE`H`Z7H@@Eo!9^Qoat*IzD*r5H7Y;{1$v!};ySj|$EoXYbTk2aB$Yzu*m4kZwkxyLj7)7* z>ieCt_;p$_txLMi!TPxG&N-bbl+pzblMk-Nk=S~x3!?5Tyi!sXu~Oga z(?Wv~g{Fil;ho&y#lO89Ue85-jQ==z9DvI-E{+$U<4R4V@h5BqA)-rjpo?Dw2RGkf zudx_PFCT`!<&Ixbn6mYD}|zshSN8iBVDM~hd&a(ww? z5-NY6@)q5`Daq~Et|e1ERyrda&brv*;dc~$VqQZP-I=4*M|%So1x^Vyj1Cj1d!y@$ zl*Qk(vDPe2n#<>`6!|>RD#c5GYD|qyr@!mu8VyQ4lL=Tef(La{2b_heX)$~^d2zUt zU1+-uB!VCRw6g!(oR}QRkmsA-25i9xul!p_|NCfy0^3KJpVaZ*dPJOnqy)ySKs)>BWa?FS^49XxE)DPmUHC^Mcl!fs05)iE&>pmIVR;VFr16oZ zZQ2DY(Hb3T^t>lZL2x_hE{Hyq9eF2EktS&y-fw^Sj@R=l(;UNSyL2T)%M~IpAVn6#M-FTis^zk%<6L=w zQMasxItiF2=Li*>sp(10!*dd)(wLbxzjSWv>h^KU)qdkr)VA|U$z|n=+xEcsw8_ZE0%@^^Kr&|Ui1TfXGL09$i(jW#$nn*)GfqgPi>{8ME7Uwu6=%MIhz={AuE&Ck4mM}v@+(Jhzb%L zZ}rRPYQ&g2&%n`T1DwMkWGq^ZzBZ%SPy@PfC&N@|0>OHyj*r&bIrYI-7%o8)M9#Rf zSEBEyN7{+26=t|}%33Nh#ZdQd%rn3TKuDkam9 z1q09gzYOP|j#e9q;A0}yW;RLap^jdl#X@7W@#Bbvj~`-iq+k`)dGC|Q&f*W}6>3RB zKuw#|>cPVDpX>@`I@9c4Tzp)tS(aDB@Ac}@SWWQIv4&WS-)^bOih4IDJAc(4Lr_j- zvmW7-D`l&5HE7v(>)U7UngJeIML?)y$$ZyZM#jqg{;~ztaJFa07o0ULO0LJ4kBvvOKtO0Wu#WM^zd3OgX@mXhMHX4snfM(76Vc9QF97#e+xhmuOIwM@}6iT)(%8T72}%FnQ-12DMue%O}mf=NL}9bINtQ z8T6}O#T}R&AG%jX>griq-z-S=N4 zz<%9PjTS+`;9ZhLn3tT4u=RwDeobwtYGs9s_AR*L#!^Gqi7>CxE%F0UL5o6qFyPv1 zT4ct@mAbA)Y$VcDz3Nxa>$t1>+uz2EZY#rHl-eb`YW@6GtE)0Xy}^%|hh9C#{_SYH zUzh=Y;%OgOuImOJ>wSPbDz^G3)t0OEK%fwtg`QY7x*RKaR(jWQXN*2)-@vr+7*4yI zl4@R#e7bN2PGb@EZgehbeKOWJFXqjY6!rSf9Vb-u)6z_s;tw2{ov+%wSiPav+JtR9 z$DqOFPz71p#f&@6GR{#xqZHQpfxakxYsa|m7(h=f)00_eGT;E^GJ`$pp@(K8q6^;L zaqQ$8*IEkUhK7D`ikizzAVNXbd7dVNnLE>vG0JF`X*h+=QNJ(xJM@-0U&i>5j)w2H zDF{}tI6$d5E}8$#GKQ2pKqH2RSe1-Pklea)z2*L>($#iVZhR!c)@mWqe@Ap|t*faQ z8NCLlzijfaFSHukimIBEtZ`JPl%}gUqgsD2&EyEgVPbKHWZYw{3&#oVVE(f5M^Qn* z1fV~PHNYXrwQnQhigotgG@_jF*|JeP^CTc*ZKUb3u3BB4E}`>~ryF8}u-O@jUIurZ z=@8g0G)1nR8tOYL)z)M`>y{oZ#V{8a|ANtfJLZw-IVb!n%a;SJj>^%hg0`6qEJNdo z=R(rP(P@?PJ9{6Yn(63j|L>m#S0Wqj;n54nWqLPdAK?VPvugUrPRGghu#8X8&dkv> z=(8|PQNE6r?~b42>);JQF^wA!zQrmpHkf5gNQ_?YTO0_aME@_1obReyo^N@{dtX3l z{V#`zuKb*_C{2vnU#H#m|IkIgnwdNZ@U+C1+RG8-rh{#I@TxCLNGuv2 z6d7%*)(Eo?fPQ+0EjeptMdYNXKvw*)-VhfCCMUsG3vGvAjn6cTJVnYLbcuS|G`!JN z-HMg$CFMpwVJ~_jLynzvk8?`av&&Eyv>t1$Rn!5Ty`P5VRU6sNh0g~k6%#j<-oyPQ zRkfVtfDPO+hxWrZ+gX|>-lsqEJdKL`fBZZ}I=)=rXLgSTN9oCBNDN1{vf-7Uz@8?2 zSVj|6=o|V&n1q$$`(sQ`U)d=JSOkwzq{L50%E^jMkvZ7nHwaD_Lv*4-L4EVtt19WR z1fX-dLX$bogB6n65tML>eC}woDMofHqIX<8k_aAvHW~e9s}1$~wMWX6-fAoE zr0tP|Va<~gk=&o)VB@jeLi#LPOFE)Uy*46rouhjZ@!D3QGeRbY0Y_P1k`s}Owlm~^ z2kV@KalX61(NsvkNnx1+Q#Ab7psmsV4Ou&${8+62i6+a5!dJ?H{lSa40Sb}&Z|Te@ z>sF6-m%r1AKKEicR|+`}TwsJsX{5?`Qs%c}^DcRQF{O#!8T{nZw$su%b=2F&5_6h5 z=cy}QU07&j#kLek^I$yTm$w-qY{mSzS>NS7enIM@`w}QZH;*U+Ki2@qZMhB(?>rvs zlmLiG`+y}c=dA{Ntj4*amkSOO+6b~_&=NIbGh***++lk z5)>>b`~LNS>nysmR!U%;D-mf1?p(_M+_?u8LH7eZEXU-DerM}EiXlQ(&fvT{*~Fny z{TfT`mbwUJMY!BAxjZ&o;ZDouLe$rpxx8Bobx9TAReq82Cce=8F$Ou5cb;Cy*4hP6 zwfbH20K2UvJ&(`)?(70Y@*(!{D>~GCl3d5!w90ipGo&=1Vi6# zX_EoP+{uoB1l@XCSOLo^tC%?%f{M-;DGkC3JDQg3czpKlqO--mO0~pQHu>VAl4rt^ z>?e+v5mW{`YHbg%HB)bT1SFm#gJr(cm|(Ai&yB2N{830XKF;TpmOl4(nxps>J3#ygVxE2W-OqG@dG1s3tjSu-*Bb2762nR~jf6n(5;Sq}gW>qiN zN(;t^&@T|=Yh}iZbo-?Sie4L@wuEa_hAL1!&|U32pwwBa50|UeNgjj8Vi~g%im2)0 zvS+0vMifQhG(%1hnY8r(9Z3^rs>uY$@1Y{6HH7iEHOw71yI z%^y-pQ^)-8z54%`ZRe3=A`kNZ2jvtj^`D{kcSoy3qg#)RDwxeuRPx;>QspN6ZN@k* zPcm~wLWpq!!`<*L!+b$PhY~;jMH$3SJk8q)wU2x^Y`ZOT zxX{6i%4Ac=vv6WS$DR>t^B#>ikcjxf>C^fG7y<8eeYA5>k!shh`m-q>zkv3_k;1R; z|I~?NdVqE^*S8PZa1mFrqV)Ws>QLb|W@P21B2;qlNz9CvP3GZ;&4)RIDu&g}eD7Wz zlcZ!Nua(cXEJaV}R>=1JCcVn{(4sH;G4izMEaqs^pyL}Cxu#I|xYZW?1=_khUHM(q zCwm-xW4Jquy2o6GzOzNeF*TU0&5YZ)k;W)>y>}AIRrIOv_A=l}OQ5TB@7r+kjp@II zYU3sW`MaV-wI6O|!&Nh_26Z-WbO3-x9jZgFKeUpviu-%6mZ2*oxNMv#Kza@x_HL`w z$xs&gx#%%f7@xb-F#NRtBBCqXA?n_j5 zldXpuxGrH|-{>d-yg+>i=ZUSPm#+894c@qK+>D(OtF#J9NUp|APS;HzZ_i{58X7>Yp)^b&e;rA0dk)O2qWqE+OjeL9$SmRo=a15?m zH*pCu%+{S%p_&yoUiXvfGujRz^T@WK4vDItVs!sgcPH7U=PQOCY`HxHaFz3}$s@F# zLO+)Y`*S={cLF+YsO;G<))VikG*hsnuZ(F#71^yN4QOo=qXMa`P0uG8wX}YQfe-v< z3?;uFeP#OHKXM1?tWuB8y9y2W_&h<9Mh8}Vs*M$?or-0d0{iT{#BI3rdE#a zhd0w!)Vja*qbTMH@fFq~q7eR=(at94*WpcJ9T*nmmT9SB0ST*oI`1N)oq;5@;SRT~ zlxpz8m4y^(p+WtC6!>Rr-anaZD9+e#_{i^aVrsIl(XIuD_~ThW$hG(bIOuy+lPYE( z!0Z-D7xGg6#%HHi=oi!vtM&3rQw=p^#|->Ws;3WJ+xqY?Uu65>dboj%8M!v|>56E; z-5UWBS_cmf4ixJhbr>57itzF(y~#p!RaHjDc_b z>8X8<>WDS#Yiquuykr;pX?^eQ456T(WpJ-c08NG(@HKP`}ddJN&*8gt1s$5!D=vTj;q_fPe;w(<~q7UruVdD>~Wt$YN>)7_}A3j z$=m0pu&t-NMxx#WSvh9S%6{!hcZY!NI$#IT0R8s?-S-Q5+S>Yfoa#ORUsze}=nD?& z+8I+LS4-4x&T5b9*j#e0=q9!nfAKgwaLJl2En$c+CSG}|a(z+|bYB|<{=vZ7b=!X} z9P$bIG!SN;kdB-h_`))1+g;7H{D%sbJ+fWdW#q^5ArCTzoK@and4t#{Ul6XQB${+& zpxrLk&y3e5C6mZdnfiU%JRMTurONd)VKJR)l}w)((zcwYe^VIS8eq_NaVPB&GV9hN zWiBl8+5=a9X%k9fG1cF{xwt+%HMCO6_Ityl@M-zJ$EG3(t&{>U^vo$Lwg(qK*?kq{ zO-3Tf*{M2uH+5EW#r|RgZN6P6*{TBkhWg@1cwR!1^cqC;hbXgH`11Yp0)<8=PS6Xi zuT-yb^gK-*E{Xk;hh;JK`U*5W6|4B`eviYB^9hJ1=1p&#dG+o9)z-I;up5cQxEZ|P z8211KQ_7{2UbcB6F1EOhV>m;aP*|NW|LID4Z)?;Cr1z_iQ;sJ-Z%g2!f}xTK9J;;H z-e%iaSK-OwoxV1&p`0JRx9y8nKO~!Gz$1|%>J2>+mNPEtlU(cC=nIQWPQ*rh=NtVT z;hvEJ7hl&Sk+^ST@bSJ{x8vwmF=y;{E$3NXKV^`QKA{FoPRt@Z@9#7YD$mmS#401z z>=Hrs01VSHtg}L3VtP82-d9%HCehlu$lg4~-%to|qNFw){!V)Q2vt=d+_Y3We_quN zQn8;Ho~W{Mn#-#)WOe(n3AWN`wYyM0V^cnyySAW4*?9QH_SaBjw$`qarJjtiJHJ_@ z({8-F^K(Wzd+L}rgo?CKgOze5y4>RG0#%hNBFYw75PcjV+gnl|VcTiA%j>ZX47@^` zR$SwO<`Zi?`w-3!$)-!UqYX^gq#57;S*eWpKensnR+9ePO*J1EbDpp2_%v7x$RHB~ zStbCRyRdTOH95bpGevfA+kP!O5r(n9PZJHl=D+2SvM;Kq4<|5WyzCQ zlMP3>^z@rD8~jHAYx^5ZFd>E4jfn6SBIAmIN#c zwokULA9S`Tc|+7DMUmR@uvLR|&`&=yV;u94SSdFd;TCvDeYml}0APbIFY71DLoZ)IOL%T@Q3aYdG<;^9L!>4IN6rRfrdu)1R- zE-Oo6l*k;bI^o_yeCK)|s`SIyeoEzs#f=dZNL0u$*YAQ;RG;#PUx7-L6ArA4YNV1< zt~Oh5FMSQhbxRnl4G3w9{gwrR6gr*pen7n|&wB$d;%m7JqaUOrN=CU4E5yEirptQO z%4KJV1~%8zg~{+{-y)eB1?M-5x=%!6TZ%^BPjKkwHdTO`9nTC1p>}B`%ZAQ$30xXZ z_U(eJ#EdyN*oB;9pV8HcqB7#mBakztL&wE1f0j%q)?j;$9wx5Ny)Qpzxn2 z_T?Q`Z17$d=PcTqb;F=*Tj`sl)l7_v#UX!sV~3|Wvtv4x*1i3p#r(ONbT2<#W<0#D zCjLyF_r2!M&m3E2mH?E%o;UV>NN?(KD>k64m|ECH?1zBaa>?LzSH||XlUKkKE9q|A z)7+h*u+C)<9F@2LZ~`I!|I1e(1rLzm;pq4St-gdUqz0vyq6Vhl$U)Pb ztH+pO4iw~RwT(h%6jCWtQh)BGMpD+h6}vHlh@)jm2t)tW^*VT?47zd)pT1U?C(9m4 zDP@RrTiOf5(|sx9_##zf^qr*2;pQK5*sD8noMf2&RTPk%1GB|E*&AYX)7yfWv!-YBax=i;T!jG#MW zqkvv5z$siYqd2O&YML||893fE=@p{*G2ffd@7qp!k1NwG0A>V=aI>>D3L^A*$}e8v zpLI#nw288TWZI|eTp9TdeP2fE9`#PDVXSZItYWJ6-&3NbaxBEP*}LS|7A6ZVDoxls{MAYnVXm~tnHHEms!3T9Xg@BGVr-Z?uWT#XYN6Y(12ijF6X02i*K6DGS#Ic zLp!Zir%dxxh`5p%)|2P5o!S+AKaB*YgZ-m50_ob}w^BTgX*1GD@FBG9^kJ~i;_VOJ ztz}B`gdyiF^lVI0l)VlcI&W1yd$=Zs7!STNXWuwX$@?mtM69)6C=+q4nqhNm0`cZf zMpJu5imixJDs9SMscj@hRWDqv`ToQ;#r5a02${~PC(k!J$M%|Tuvnf}bW7~Bc6QOu zB&~LlAhLI4?SGz`LhL&lPg3VS*48&=obU&jtBXg!-&pMGNBv)7KAAL6HpUm z__v%)%o2c$BH~nnynX^Jo&2O<$zG4hxVzoBI-v11rB6x?Le}+c6~8jJEMjia!*vC% z4#eYsAbWX7YhUUj4L*1w{9#PcH{Tm-G7REDJ|cjOxvQeOyd}6bK5N_f+y*N@Z>JC@ zqBvGDBTK4_;)4G?=-2M94iXw--(t(chOWl_dbLsa&Gai{w*y5*$xKJ#F9*|{;3ZOAx9|=5^9<}d?M$0)-a*Tg5Xd5f_{x0j3m#j+jqclHs6eVz> z$nDO5xuy_Nt8^A&5aM+C!Z>O``5onh0vRhwZTjyk-rsG>1C1!HoXb z%cY>DOYf zQ|pk`@hwRB@46f5XY*6}0tAg;VR`1&$3Q9hfo>0jKY=sYC z7*&C(nXE#Jy{{I_DA)8g4S)|E&8x)A0}q>)8n4 zKS9p3brLs~1SzQDfBvE2K6F-TUyol-Yv)voR;FU>XYz3E=397%e_<^%bi6<-IYcd3 zN=Fkq)flKOB9J;O@Kfc(9P8U3U4r!y(7BqHn@7F0E?JY3TQhl5{B+JgQ194&YMNL# z0pawC9N%hEA%k=t$>oIhIfFn%i`jVpG$I*!Luu7VWcfR_R?jp`+%^l-_N83ysg0LL z#>7OB#=zKjZ)>6Xt7&mhyehJsH&4 zyfA&l3uW3iJzJh{Q^Tq!wiYTmXx~bIqQAl??MaxmrmTu4Vg*`a@4rs!*)RY%3oI&LX_AA6KBOoj>eq+%vGQ-<}v(X|k$ zcViA_7lV?I9~2o_S$2o+6gpb*AN$%=)Jh6vv+$$0m4YF*Xu;lJLEW^S3grn;sD{=D9M zPlHCXxqNK0O6)ra@dJI9jjf~Kp|rY@%($~neBan7$+>J`?xGT^jIQ6 z#8qf8rUCfM6OM9-g?}J@5MTP2%(5dIuGlHa+kBJ8$WI=6f1L(uG)F3Xt^Pn=X~va* zPY9+?Ppo}%D|xJz@XAuMQpZu(0Pr32!>xzGi|>5u7DfY)8k)(GN`T!J$D2suepl$q zgGi|@;K?$MGy)Z_-b59Guk#cKH7Q6srr)9a{^zUO(M}w&X^}SlNDA<>fO_rgUrK&(Q%U#rJ0X#8 zl)BBt+k&_5(U|1wh2s5#PY1G_^39eW<#2`4tnrt<;U+tWzuA1PQWW{#jZr%etujDC z9IZA?q>yf6L{{1+(csN7iy>@D@B@9ebTrI^|{z819elVP8f`Ex6 zwkVPBD7LGDJosM#yA$<=Vt~r_kTZ~>DfWtM`;>#lZdIosh<{+&Ro_KfGesUZuod6! zL){Q7_^7Ij-^`Hj7b-jf@ODEHvS8IV(JOvAq{PE#^|+XytXJUWm9- zRdYEy0Te0OdaJ(3IDa_5{gef1Y9;X=F+>J(5lZ9p-NiByeE z@4W+qiVFFc+Kb)p3HRYBv=O}|+d)RohWPtCp#~}elyT@Apqrr~+%ZPBhLN>WTxS?^ zwkak5m-C9CZ+-f;p`+d4_U>TFMmM>r{rI#$FlduFE}oLudOflzz_3Z$P$ljWWLV5e z2K^Uz)4d{_46k-3^phPOoS`@F>xGYCL<~=C$JOy}p~)xr;LHWD)|J|ne0Hu9>vNq= zgxzQ5qs{40=N7DR38P-~P%mGJ5YBp9;*n^-xiG;Mu)~IJwaYLnO8?PWe^Q+cVt0IN z6@#awF6FBl6y^wheWja((D0sNk6=2m2y~Rbvn(CO(oRI$aqfnL6L2(w%6ptI2^x)X z-K@4|={@|!-hu>c#EwnY$TbI?ol5K+OHPo>clgigh@6(V<0z*G0%xZ{M;c?${eyao z5rLHdTMO{9{BvSaI|{jrbLTC=4Y@tJO%5X;pO)x3JDtzD2si<4WIe;?puCSVxC4wIx>7mUHP%UQg=;uGzjN* z`jRxBlr;;=Ea>WmpY_R*ko)vd1cNsP8$}B3wTLZ$dOahMPQH<`r?{MfRj#t`kT1AR zM9Q;enn&vASHd}oa;=aOTj|0o{h<};_!HW9)kD&T51=o$JJtygn@vcryV~Vps*z_; zN(APgnUl8{nPEwGXQW%?5KkcTY+ql=R<|6~C@<@_Jd_SldUS8GemNb@ zGDgv`kB!ad@H9bS40t_&r~dHC;S=wZb*S>maNOysIwa|0Kv-FX)kj1-+|JAH&^Fd- zV?dPzXNf*?dG0@9^-Av6_;p@b4b3{fIAv=j)0cl7@Mf4Ohw z&10C!44mw}&))0ov-Vow`o2P(zuP1vD^+f$)}r0$M}pI~QdWxLc-YC;UQ@=Bsa)Qx z`SH0QA)3`y1np3ODb`;FdgZJ^H`X}Q6q}iL-BvNjvV;IYnVda~7h`_T*NNDc;qKkk z5SLDnf8T$wrTZ}7@dnuEsOwNE=iwV3PpuJW8a(@p&)OWkFd0Wio$bHDM^p%HHaNg_ zpp>_RJ-lPj&M(&sa#WMkJSk&>?K?m!TkeT+=#zBGi9JMLxbbnyVI{Z*EY*H~xmgP= zl_F6CNIQocNR;w5RY0&6SFcH9yG|&CSbbBqlc_g2tlJ9R*xi&=$*hhY~OUR zIGOWQjHz~~B%8BF%h;R8E>0{;uxNx)UC4!zU5sQ)-Bz3Ul^p_Sl^LA&k?%?)C$s<9kdj~CA2}G&v0-dHeV|QK zNXk+u#y6><@DQNsB)KoiUUw6~Vage@f#(kN>m%7XMvup8jKsM}fj9k<>^z3X#?|ut z_Q|gpkw_7oW3Rc_3H#{ke$hj1XZ@G&FRs5v%|UJeY9sHKKQ8u+0v~3^@#25>&&A+B zdndA9@@<6`iz+<(B;3T2Ut7JC4rCRxbIcHPzW}(@gxMWP^Ck$}{fq6a zq1c7Z-pMPk`96&1-&8kMJZ-faS^OK+8XNCcd&1V(p)s;J>HYns+$vp}jquThvW6iR zuZBSm$Yg+7{Qe-j;qi9~TM^+wOh#nWf>AZw#}Rl)r#n9w-fO22$g`|u$=%^F{AB&A z+0nG_;(gOE<{_#jH`^=|IIPe>ukJ5d6QCVn#48|&-m(2A+8KM+=-$G ztw;}t;Ztf!+i#l(S|ZXskhUO$NwpS-B-niV8eE5~<6H8zASNXxdOqW~AfQ|TUY+Ig^tV1%dvHC{`l3ma`L9c4`KIJk$ zN}Wc#6t&tie8oKS2h8eq7lnr?#gTEf!ffo`j-$hT(t53Ur!dL5mcLki{`HINg3B?yp|)pqbow&uOn5UU5?6;C|RCF6z=S@tuxE(4S( zHub^?*KMgJuCG*yo9l(jUcH*(w;E4lE{8{6?DMx~w7qmTmJMq4wox}r6m@Z+OWa#A zT9ddE;wL_1RhaT|Y12IlaWB)zCThce>^+BElsQz=G)@+fy4-WZtDmp|ge0}Fw9hQ8 z`15i$H6RsRBtV;{W~$oN(wpvz_1DKC~2Q&1Vkn3_vi>t4kU1RvAAn81;E!(tvcX^Qi?2jzHp>{NwM$JCspbYGFUh zHkfWfUxuyc1gqCgm@-90*$1RfV8%U#B_c&ueC%|jhOUI^5_xOl{JYS+W9GjUv7+gn zn`yR1)`K>udG*56*w!YV1#HbcJudY6Y$cHJ?^9DjYf}3Zt1Q852`)eOg+lGG z4j!$%a@60_)Ub1mmomkFU<~VKc;tz3`#fOBO{=`!ot4To9(`k*bTvZ}1z$7pV5T3# z8AgE97oxvmjSumhU-&XQ*3+2iN;JYc&uge}y3KoK51!-|?z(8}G<_nECff+Wp8`L)yTp8dMObsL#M{_dFa3AulgXy^#pOH z(xZm8f^x48zRz#|;xvdkcVQuYUGvD^uVji|p4`4_{%=~Ai<#5=`Ghax;_U(H(oX5* zR4RQy;swHBdOOtRjQkmaXCVO=EBc=T#0IHagqA-MydEyF$M@- zZI>)T-k+nN(qqiKX7W)QnJ+D6dN=AK;~*~YSLIAaNuBE=&vq$Sbm(cKt{2yr!sB@@ z!-`jl+5NhE!IZcf=@9d_k&D5G8D!$1MsKhy{Gn@doLqf{ zl7qS=Zv(-Y8#)yGC?v@E)=k1l<&xL6hhZk0Q7_#*TjEUZUl7)QXT#n~z~i1zL$(l~ zOw1dgw!G{3YmDX7kTaMgWr-C$pHGUts^pb39*+Aa6?z)p%_Zf&J{ry8`b=k4n82@M z@-q(pg+0|B(W8A>KKq3;q`M~PtfL&C`Sq8DX{MhFb)6>!-gR>60d!QWC8q~R(EbcAHpnr@^uB+ zaUb{9IyhH%l5Lw^wEX}!zH}*ttk^MAS?MNFHvaC-8lY0O``WHN_})fNb1>m=XV_>u+?o`>?1CM#-$hu(`@9 z_;Hx0Z&T9K`<(9{n?J4=R87%PdQV4BW7%z@>SEqvlW=7WFKnx4h^ud`i^sa;UjzIB zI7F&~aRxpXC_l8|Jt&OmHIZ=hpDp?Y?JthCkcIa;ZJ9gf#0uzx9Rt4KC%kb_7VD`s z;_KxeSbSNOuyE7tgT?1A+wRtss{(7PWzJDVZ@^0S+Bc1NuFOO1I*cGLFa zvRz0<+>>8_9W)o?dwR(F$L9v0=_%aZC%&Yh?8*Xd-r5vaPX?Wx9a$PCHw$jF;e2^a z@cspcg#~BeWt?i;bmzJlS(4V2Zn%Eijzz4Omqg;LMt~#N6o`kgQY?MeNpNr6ZKII0 ze%437y6RUS3;fpma7`*nqB=9B+HB%Gpti?pb?1v5M!4w*-7@7oOKa~Zd&$XomGB9- z`G)%oxCpzFu)9aNme2@Bn)#9Wmyx{mZ+W#JZCmTI^~Lx_+-HX79pdbl#b7T(OAp7F zTiPJyF**D+s|yIk?C6`)6~_{)8YVk}_7|phfHqm>(51+`Am}0l^=7$pAxd!u2ljWw zh9$K2l25h4QBkUZ(4xzRisOUy%YapTlKQaL`%4 zxi{q9FaLFnnV-AAIX>&}AjBfl>o>zeND-jY2gGzwvwqf~07E`?u}38-=a7_bhNMR@ zPp&<`2A7zzqQpIbTK)R_RzsW3$phRT?n@bXA0FP7V04QsqYF_d9=S4Ei$WuD_ZZ^= z4n4sRoZzsaV#d;6sPbt>d3M;NT8J4F1di-l2Cp*hxWExK0{1*}gO0jfJsTdeaYn>P zc?xZ6kpsiqUYx^GEzQp-NXNhyCfvQUJ*M}%IzHS#$gi5{SS#)L@G*WSC>xhwhBv4p zSA=eB`fn>vi{su)Y0t~Rlm_d~2(9-He(7+cl# z>+{(WcawaIiQ{gy@&3O0m-YE0z*SC14AjuxNPryx%H=fF9G8-E>ZeC>v8Uw!`OP0 zd)0BrOF52V9n|-@7^V7hdTbT_l)D$f4HO_9u`U|Hk0}j>R+h8JC})Qth9ULJ*wYi& z0(3dKJJTugmEaRz)gj8>mJcPgJDW%T;u&$d(wSsZX%H3n(?2P~c+z~k+8Yl?HCFtr zLVqN2RmSLYRbgy<8ubNhd6irpPrO?g@R(-Xo5MS^bm6#X^nB)bnFs;ipBPp9Cy%F3 zXqpx#>Mc)nDdxOzdz>Qsz*Oe&g(OSCTXJ*mt)uo4NzY<+-vbbU zNVwnT>(UN0`-j|;Ru`CpM{s5V-uNm@p4XS#lOVi@Gr22Y3qULdq$U>McrEGFmPwro zl}Y<0htZ4TY1>=260V;$ZKC)F9lUgufHC9Gq+OD{_%NFM#oslyH%~+1x~~Wnry3qw zupEdkY}D30N;d<|DDoL?gD|#Iy6o1*ph)Rc3BN9wS00mciWF@VhKaOEgCl})A+Kta zsj^XBe&>(Gx$G?8eHvSNlT-lhn`6JoG0K)M=DM|T`pE33tt{zC$Ce=aO%f?45Vo_< z!2L%`Eo63j&^==G_F)h}a}*_%AgCyPx6d z#na59dfx{TiAR^S}^L}nwWy2loUK;mUTPt+PgaG?yGI~Z10 z)AoCZ+fj^p?FKO@xp2qKf%#OHI>M3p(5&}%QQ7d$8Lp=>)t9gNdpo@`I-Oy^vzo-i zTba`JmK?`EXIp`WD_o6>>6K5{IP`Oq)`9N5uW>1_x>0`XSqSEUSgOdNL&Tl&&$raN z0)>lspjREUzhef4#l9_^edPZ9N6?aVSG+}cy>d5YJ79&?vvMlbMYffnS0?#Iqb#Ct z@p*v2XSMASrQmXZ2X(6!BH)OV>e(tj>^_tZv;4+W0?ur^Dl`9( z$Nr`i5y5zdZuKW@zsQrj%>(Aa_W~sV`@Y37TS*||VzcQS=#|Fdl!734>6+9y_I0y+ z!6%LlET-@47^LbRwa8I&t|L{rJ2fLHaZ(mo#m;8v^A{O(E8GTX@w2^h?b%*;a!554Q2{K24fHjC|e z#{MFGuIIui+UFGfcpmy;B1Lf zBrhZAr}W)XIY62q(j-AZ_%rWfwCkbgZ3Q)k(^8F2v6D4SNTp9_Z391Np3&T$das2~ za^ZI#8uu-(=rj@P(C&goFyp4U^C?q(6Om#4UTaj2VtBti!os#qnH(igmYz}gIU;BK z=H+aeV^3|4d&*c;s$W+FL1g<`)?urJ`}{TRx--xuv_5l{DCYuf5lety@ue`GiebO% zRTb2w>Az&_@2g82(wmygsBX=;y?!}7JDy%l3dd+C=KNIN85cXKkj-vHkI52;Q+<45 zlO!;)aj#jIE_>A6n8+xPt8ox-EiUbIM^3rwBP-v}L8`0oAO>Qr-q#P>Wa#VF1=Mbd z_GH|ya7c)ndjvp0Gby9SU@&Y+5QR2vM=54*J*+3xu zdF>XSZ%r^+%IQ_ZlunaFSrvIq%_rip1DlnGKBJV{s5L;MRJmBN{?bfFOPMo#0L__r z#z*aphBYrTOTGKMri`l;4$XJ0MtwM3%Ze4YQ zaz&YW&6^Jx0CljX@e`h|yeTPJ^J`cWu>%eL&_`pD3jW@Ja;N7h*PDF`GjelKf6*7m zyl%W1jmov-vLFd!tj1_& zK;y}`fPvuruYBXK`jkYSVnyRPoBigeD(?WxVE>bjlup7!3e;B3?ANS0wcrF|0V)TyhUa zY4d^-Lgu3!d|cu|A?oo$?N^BUGe-tMmDpKKV)q@D+A)DjoP%vlD#kzz6fLcqO_?JI z91{tquZG_kMZsx7lN*B@-dT%w&==xl3arHE#UgAg1V@CKu5E&+(`jH*{fr*B)sH1K@ zSa7k*Q+Lx0x6rHxu@~P=e|I(kf}8%)z%%U9LoRNb=%=$Tawb(Ko5#_)0dWldSlxShjuXN4 zcRkx~61HP-(=F?^oo7=`l@7do=n}>me3>FfF0%B$-{h9SC9miEKgob-8;OJ*!NNK8DT`%0(6 zv>@gp+m|X#HF@G{CCt zDa&S1*IEgp4=2Pxh>a{GdCBzE=8=nLP@O= zSa_^O(+rL-av-3Mui;1yArZ=Osy(RRciXwN0jm#?UT?44(NvGgH0V@U8)M($K0i_k z_uK0An9a>NZ^-lLA)c!NHN_R)^u4q$YX^I))5>%w3eOJ736B%aF<>{B0ymS@67DKq z>33uI90f>A1)8Q##=Flx@2eWUWUZB7{&NyIRDA8*yosP52`b)}${*qo0mVJ)sn6GW zu(;`t?hanqxJ-ja_7{vr;f+Ch^aX6Gz7Q=btO;vhty;Y)(b7=POk`R6L0wzq`3hL*?wGUVKP`6Ch zNVjHF>-b&S?jKcd2^G?%X|mBTI>=Izc`x`DWmaR;4F#+vATrTz?e@3Yl=slc%$S?S zax+2j)(&S30t-7~zok88@%64!oL064I>B7P z8WbEWme=sy)lwbq&(9|&q|;q+SXO`Jl@6G+T~%&QIQcOy#9j; z({4cTpu`&z@mJ=}X7XOIMAlv2|MEbQ+lca2)a^QLKRR*hHU6wvve>Vjqs<8iL+%F^ zlo)*v0LRE+hs#nT&4_MIjZ}ZDaO@Hs-D~6Vke#>pj&2if>s)*}EvDmhsZ#nxL4J?F zlcRLU_x1%hG?CbVf1t&LV#NFL3r*ei3o-=#Yrl>sW*qM1%;9!0A29;x#(8*#;k^@F z{;l)kaoysZhRQ0Aa!M(UCy#hq%nsS;DphAjX1%;3l9?g#73-I&VclGZgM4{!MmpCt z#!FV3KA#uG3ci>_pVc<=qKsS|e}vL2N%ZPRiX|ac#ttBJ?8e{_o<>d6NYOX70gef> zsr$_zrW&%z-f5kW^N{igw=I5>K5ZZ+*oVJ)Y$k-!?W?1#(A|d~1HBm!2vKiM8&DW> zuDsZX(omXySHs+J8rxY(Po#bfvbl;EA%J;C!Q!Kd`vA5(qrMxLKHyPTbYokB2G&aV zGS^ib%-Rs>R+zSTKE$JxQ8Oo7cMY596e>!7 zm@!-VQYWTm7T}~e1px%BLEbCw>uBdvU${ENfjj3q`+4CFUP^wm%@#rJ(@x$L-?rpP z+PXta3V0(o5dvaxNAiX(|<4;{cTiq@fZRy}HMOZCLp9{?Rrz$iOppX{QFe7Ee^u7hd(?*9jI zbPxUq-SKK?HGx=M93-p^S^GfzQp4mNy|Spkb$^kCy8Gf#cktY~vC?u@bOtC2vAxRp z%Zn~nmKY#(c~Q7LjB^y=WbCJo{$xhP7L(o-k1nZ&t8B8cch-3$oK>fczyKfdr{!&T z`1}yP6s)02`#Mj3WFpF>%>e{;CFdS8u%5Gjaf!FKZfc=G_w7M}p%>LuC-CTAiNH9CjU8nZ5Q7ouz99FNo=}!p*xMO{@#x+`TD&-o-Xso%Y z458Q_vUYIBw+dXv?NBsvv!b1IsHumkKPb+TgUp;>Q}TC2+I5e$g2xxAxxeTGud`OS z6rrLEq0r_X%(^m)iW$}LhqD~R)?Uu%5*DgTMVesXCBLi^xy-7&ehW}%Z2LT*o+)G{ zDh^GXN$S1Z9M%=On8}(B3mR&xbmHH3HrTY8IMqG69i@`q6xK9#vwHqPbJI+{Se*g( zZGW;QEiN*7By^>QHq{6VUVXb&;*-7~1UHyOPkekrtvgZlf10$s(iekJF$9o9gRZLleH zis*uQzn1MFvJu;E@i#}C9^W@ijt2=#Wo`%x*N8oZG zK_9PRbtBZVlM}b3X|1QQU3%pSVap(NbWN*g!Zx>PtB!)kRB-@HtB1Pjz(XmI4=4$y z3lRd5BjkL4C5Mc!h@1`QwaHM;q>IRUgbdZo@m=+@Maizp2*#MjocM2&i#B^a)c=sz_dGf&c^wS_h#sKezt^#u)? zer1c-!-e`=#)s0|s^|JqJ!v~V)lHO160_2FX5qwWWZ-m?fl5gTVo(~OmIWcMM)z^Lq$01e=af2$I?fv>NHk+Wx-BxXW< z*4s5_+^;|~v~_!4vFd8a)3{-3*tL%}l`*;f4-wmM3MsEd9M1x>lN|Vy>@(IYp3JtaJmO9>!y; zixyfoEFyUI$D2B+_ia({?;0>h3KaM&PVwm4_N+1&PZ!cYOctyS)Kxz*H zW!fUkq#gPMu^j|a)uRo0#rD8LEo;ppL-EVstja;+a9XcbnaI-43MWboy)EzTCz{Q` zdBvEK7|*~BF7@CH;Po6tu<{iZ-~3EPy?}*LE0+DrMYG|wVS=-Uo;VrIL&j4ll~66< zR*w+vfk)E?cH{_g5u6bi(5_bMr7tqQ*2KG)zgr3Yi12lUk(W{9hhjvs8v^Qoj^koQ zNK`D@S8QoxrC;aRs4nG6h{lpx&xW2L=_VuCuCu;;z~8{zz<(+W+&CF`qq#4^)ubxGalk~J@kw)VG2ssc~m%-GI6}jBCO&W zD+^||?&XbO{g_A}5<>}A(K+b`!O*eUiG|pR@xB`Gop0`kyr$1>Pj2-sqBQ20;x!sh zwn6>q{?3rt>bv3eJCnGTE*KrfNv4KXkG3A`pIL|q?n8);pB~lio*TO-JV&Nj{#Qa!4 zYM37M{IFcadcm?8Yx-8h*X<6i4LLBAwZ4t|d5xP392#SV&oH20b=)b8bluv#gUCd- zNRN()j*pBbzuv|-Z3sK8RR-y8Yjl6}NmbfhG8pon@OHTpKCG{9fG&5MH<&>!l?8}1 z>tpmoLEe3}OY`;2{1%cycr9V5YUyT`LuP^3X22lKPCb$=+E0zh!iC-(Dvs@O74=>= zPkI5G60RMg4|VgmDJ=P1(dt@jKSaeW3~mhsV#nUV1iW&%N4064Rxs_$tmzpdj0KC- zWS#wwSn81-zO9w&ycVDiX2d5 zQ}OcO-)uYZMs#dSgzuV?8+EAYR-gw5?f<*0QA3xtP3hvR46)s+hXO2xty|SXN5p>%yf^=*Vfa^v zl&1kXBcNNrYxVK&yFUZkVm7e>ladeQ5)^;?qkd0T3ab8O*g6vbHBrlkJX)c7HMC0i zXyEdw|NrHML?YpEj$U3JpzQs7EPveHM7E}HuZ;OaA%Kk2;eV|jp>q~f>*vSu-%Ejs zacv(^>nsg|Rew=)l<(Wkss6tl;O|B)**Yj;PfW@U08Kri-GqN3vpG3B|LYqdrixrN^IN+bZvG#f&U-kRR zqRepQ0`Tkq+!N5QiB)hu!}Z-&_T6`+BF3m&byVNsDuk*1t@ihOu)vqbwszZ-_XZE% z-=kU_FR9@@vi6@#et-0fvc?nzoFDAo!E3`Gwi`W+wHK#SrvKYL(BQteGd<7U1-l_3 z7hQI#bd0TH=0mPOH2Y_^ye~+Etfxda;pY&o9TvN3X8vgn=sQy(YAu*(wAOHSEN8co zc?V^zP$#KL}uTw(fV{y{3FoU@mgb2I*=G z!q6qLbJrSFY;<&IapWpoUD{b>`8ZJL|CwRpNiKylV;Jx>$o#8)ZcR$_)rDpIo%g_Q z@ZTYIfH`35wv}rg_Yk{G-DShQ)66o{SXGbl7banc~9> z%YIjO+LVcZQ>pwt?=T)7@C-03*H}MJ3EUbx{VxMKvFvA|P~13%+3COKVY_=W4;@%R zf%2j|?(IHA5)m9Gy@$ov;L1T`?j*vSn$Pe5G3z6+>tpf3M(PTOf#+uT`geeYTHvA> z5^E(OUhKb9`P*1!&M*JcU9aZI9dEId-@@!3S+h2S>={rfWM}w~i4|w63GCUk&+(%1 zh3i1wmkt9;i>?8OzWDgo##N9dzuTBu_Zj`?#7=tuvvh_oRnY==Qu@y6$PFZp2=sSh zcYj-~^TR3Gb6OnFy;znvc|YwQrqp)?gjZs%9n{mwO9nRP;N7#{26jBLs1*(WF!blp z{U2LT6WHS2>Hx#HT`sc~x&DeKaO^fNf_N0ijw!Q1N^`4epzWt|sCGdVeAiRb{6pgB zU*M$eCtHRqxplBphIv;RrT-c<(e{n7;jJjKJ)N(1}5joLzgy@0ZPY#Jo^~1gkfj zo3)32sE#y+3%sdebYzEpw^f0={YzZM=-%L`Z%UTZuX_Cnu zF)EdsSKxLZajC5$dOC{#ind#;rnvI&*>wqsg`u{t?Z8)BcF5ybU324L;BocAT+NxB z{zIcWWG$`q;kLA0#tS4d+Cg~zQ=OpYw%6kGNIrd*PNlKYp1wo$Ov={OT;qm=lNPLa ze#S2lzg1e6*)|fmR@dmuvK{Rwu>MP9^`GH($g;Wif(0cFk5+o6n&)glAx6g-B$$Pq z*8j5^T9YIBgthQmd!UqtHy>{)k#K>Pb={;J@*rgVD{Cj9eo)%$Ly+Sh;OC;r72`6) H+fV-w)^vp{ literal 0 HcmV?d00001 diff --git a/models/tacotron2/NeMo/docs/source/asr/intro.rst b/models/tacotron2/NeMo/docs/source/asr/intro.rst new file mode 100644 index 0000000000..193e544dd7 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/intro.rst @@ -0,0 +1,59 @@ +Automatic Speech Recognition (ASR) +================================== + +ASR, or Automatic Speech Recognition, refers to the problem of getting a program to automatically transcribe spoken language +(speech-to-text). Our goal is usually to have a model that minimizes the Word Error Rate (WER) metric when transcribing speech input. +In other words, given some audio file (e.g. a WAV file) containing speech, how do we transform this into the corresponding text with +as few errors as possible? + +Traditional speech recognition takes a generative approach, modeling the full pipeline of how speech sounds are produced in order to +evaluate a speech sample. We would start from a language model that encapsulates the most likely orderings of words that are generated +(e.g. an n-gram model), to a pronunciation model for each word in that ordering (e.g. a pronunciation table), to an acoustic model that +translates those pronunciations to audio waveforms (e.g. a Gaussian Mixture Model). + +Then, if we receive some spoken input, our goal would be to find the most likely sequence of text that would result in the given audio +according to our generative pipeline of models. Overall, with traditional speech recognition, we try to model ``Pr(audio|transcript)*Pr(transcript)``, +and take the argmax of this over possible transcripts. + +Over time, neural nets advanced to the point where each component of the traditional speech recognition model could be replaced by a +neural model that had better performance and that had a greater potential for generalization. For example, we could replace an n-gram +model with a neural language model, and replace a pronunciation table with a neural pronunciation model, and so on. However, each of +these neural models need to be trained individually on different tasks, and errors in any model in the pipeline could throw off the +whole prediction. + +Thus, we can see the appeal of end-to-end ASR architectures: discriminative models that simply take an audio input and give a textual +output, and in which all components of the architecture are trained together towards the same goal. The model's encoder would be +akin to an acoustic model for extracting speech features, which can then be directly piped to a decoder which outputs text. If desired, +we could integrate a language model that would improve our predictions, as well. + +And the entire end-to-end ASR model can be trained at once--a much easier pipeline to handle! + +The full documentation tree is as follows: + +.. toctree:: + :maxdepth: 8 + + models + datasets + asr_language_modeling + results + configs + api + +Resources and Documentation +--------------------------- + +Hands-on speech recognition tutorial notebooks can be found under `the ASR tutorials folder `_. +If you are a beginner to NeMo, consider trying out the `ASR with NeMo `_ tutorial. +This and most other tutorials can be run on Google Colab by specifying the link to the notebooks' GitHub pages on Colab. + +If you are looking for information about a particular ASR model, or would like to find out more about the model +architectures available in the `nemo_asr` collection, refer to the :doc:`Models <./models>` section. + +NeMo includes preprocessing scripts for several common ASR datasets. The :doc:`Datasets <./datasets>` section contains instructions on +running those scripts. It also includes guidance for creating your own NeMo-compatible dataset, if you have your own data. + +Information about how to load model checkpoints (either local files or pretrained ones from NGC), as well as a list of the checkpoints +available on NGC are located on the :doc:`Checkpoints <./results>` section. + +Documentation regarding the configuration files specific to the ``nemo_asr`` models can be found on the :doc:`Configuration Files <./configs>` section. diff --git a/models/tacotron2/NeMo/docs/source/asr/models.rst b/models/tacotron2/NeMo/docs/source/asr/models.rst new file mode 100644 index 0000000000..0b9f071268 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/models.rst @@ -0,0 +1,97 @@ +Models +====== + +This section gives a brief overview of the models that NeMo's ASR collection currently supports. + +Each of these models can be used with the example ASR scripts (in the ``/examples/asr`` directory) by +specifying the model architecture in the config file used. Examples of config files for each model can be found in +the ``/examples/asr/conf`` directory. + +For more information about the config files and how they should be structured, refer to the :doc:`./configs` section. + +Pretrained checkpoints for all of these models, as well as instructions on how to load them, can be found in the :doc:`./results` +section. You can use the available checkpoints for immediate inference, or fine-tune them on your own datasets. The checkpoints section +also contains benchmark results for the available ASR models. + +.. _Jasper_model: + +Jasper +------ + +Jasper ("Just Another Speech Recognizer") :cite:`asr-models-li2019jasper` is a deep time delay neural network (TDNN) comprising of +blocks of 1D-convolutional layers. The Jasper family of models are denoted as ``Jasper_[BxR]`` where ``B`` is the number of blocks +and ``R`` is the number of convolutional sub-blocks within a block. Each sub-block contains a 1-D convolution, batch normalization, +ReLU, and dropout: + + .. image:: images/jasper_vertical.png + :align: center + :alt: japer model + :scale: 50% + +Jasper models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecCTCModel` class. + +QuartzNet +--------- + +QuartzNet :cite:`asr-models-kriman2019quartznet` is a version of Jasper :cite:`asr-models-li2019jasper` model with separable +convolutions and larger filters. It can achieve performance similar to Jasper but with an order of magnitude fewer parameters. +Similarly to Jasper, the QuartzNet family of models are denoted as ``QuartzNet_[BxR]`` where ``B`` is the number of blocks and ``R`` +is the number of convolutional sub-blocks within a block. Each sub-block contains a 1-D *separable* convolution, batch normalization, +ReLU, and dropout: + + .. image:: images/quartz_vertical.png + :align: center + :alt: quartznet model + :scale: 40% + +QuartzNet models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecCTCModel` class. + +Citrinet +-------- + +Citrinet is a version of QuartzNet :cite:`asr-models-kriman2019quartznet` that extends ContextNet :cite:`asr-models-han2020contextnet`, +utilizing subword encoding (via Word Piece tokenization) and Squeeze-and-Excitation mechanism :cite:`asr-models-hu2018squeeze` to +obtain highly accurate audio transcripts while utilizing a non-autoregressive CTC based decoding scheme for efficient inference. + + .. image:: images/citrinet_vertical.png + :align: center + :alt: citrinet model + :scale: 50% + +Citrinet models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecCTCModelBPE` class. + +.. _Conformer-CTC_model: + +Conformer-CTC +------------- + +Conformer-CTC is a CTC-based variant of the Conformer model introduced in :cite:`asr-models-gulati2020conformer`. Conformer-CTC has a +similar encoder as the original Conformer but uses CTC loss and decoding instead of RNNT loss, which makes it a non-autoregressive model. +We also drop the LSTM decoder and instead use a linear decoder on the top of the encoder. This model uses the combination of +self-attention and convolution modules to achieve the best of the two approaches, the self-attention layers can learn the global +interaction while the convolutions efficiently capture the local correlations. The self-attention modules support both regular +self-attention with absolute positional encoding, and also Transformer-XL's self-attention with relative positional encodings. + +Here is the overall architecture of the encoder of Conformer-CTC: + + .. image:: images/conformer_ctc.png + :align: center + :alt: Conformer-CTC Model + :scale: 50% + +This model supports both the sub-word level and character level encodings. You can find more details on the config files for the +Conformer-CTC models at `Conformer-CTC <./configs.html#conformer-ctc>`. The variant with sub-word encoding is a BPE-based model +which can be instantiated using the :class:`~nemo.collections.asr.models.EncDecCTCModelBPE` class, while the +character-based variant is based on :class:`~nemo.collections.asr.models.EncDecCTCModel`. + +You may find the example config files of Conformer-CTC model with character-based encoding at +``/examples/asr/conf/conformer/conformer_ctc_char.yaml`` and +with sub-word encoding at ``/examples/asr/conf/conformer/conformer_ctc_bpe.yaml``. + +References +---------- + +.. bibliography:: asr_all.bib + :style: plain + :labelprefix: ASR-MODELS + :keyprefix: asr-models- diff --git a/models/tacotron2/NeMo/docs/source/asr/results.rst b/models/tacotron2/NeMo/docs/source/asr/results.rst new file mode 100644 index 0000000000..df9fa83748 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/results.rst @@ -0,0 +1,181 @@ +Checkpoints +=========== + +There are two main ways to load pretrained checkpoints in NeMo: + +* Using the :code:`restore_from()` method to load a local checkpoint file (``.nemo``), or +* Using the :code:`from_pretrained()` method to download and set up a checkpoint from NGC. + +Refer to the following sections for instructions and examples for each. + +Note that these instructions are for loading fully trained checkpoints for evaluation or fine-tuning. For resuming an unfinished +training experiment, use the Experiment Manager to do so by setting the ``resume_if_exists`` flag to ``True``. + +Loading Local Checkpoints +------------------------- + +NeMo automatically saves checkpoints of a model that is trained in a ``.nemo`` format. Alternatively, to manually save the model at any +point, issue :code:`model.save_to(.nemo)`. + +If there is a local ``.nemo`` checkpoint that you'd like to load, use the :code:`restore_from()` method: + +.. code-block:: python + + import nemo.collections.asr as nemo_asr + model = nemo_asr.models..restore_from(restore_path="") + +Where the model base class is the ASR model class of the original checkpoint, or the general ``ASRModel`` class. + +NGC Pretrained Checkpoints +-------------------------- + +The ASR collection has checkpoints of several models trained on various datasets for a variety of tasks. These checkpoints are +obtainable via NGC `NeMo Automatic Speech Recognition collection `_. +The model cards on NGC contain more information about each of the checkpoints available. + +The tables below list the ASR models available from NGC. The models can be accessed via the :code:`from_pretrained()` method inside +the ASR Model class. In general, you can load any of these models with code in the following format: + +.. code-block:: python + + import nemo.collections.asr as nemo_asr + model = nemo_asr.models.ASRModel.from_pretrained(model_name="") + +Where the model name is the value under "Model Name" entry in the tables below. + +For example, to load the base English QuartzNet model for speech recognition, run: + +.. code-block:: python + + model = nemo_asr.models.ASRModel.from_pretrained(model_name="QuartzNet15x5Base-En") + +You can also call :code:`from_pretrained()` from the specific model class (such as :code:`EncDecCTCModel` +for QuartzNet) if you need to access a specific model functionality. + +If you would like to programatically list the models available for a particular base class, you can use the +:code:`list_available_models()` method. + +.. code-block:: python + + nemo_asr.models..list_available_models() + +Transcribing/Inference +^^^^^^^^^^^^^^^^^^^^^^ + +To perform inference and transcribe a sample of speech after loading the model, use the ``transcribe()`` method: + +.. code-block:: python + + model.transcribe(paths2audio_files=[list of audio files], batch_size=BATCH_SIZE, logprobs=False) + +Setting the argument ``logprobs`` to ``True`` returns the log probabilities instead of transcriptions. For more information, see :doc:`./api.html#modules`. +The audio files should be 16KHz monochannel wav files. + +Fine-tuning on Different Datasets +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are multiple ASR tutorials provided in the :ref:`Tutorials ` section. Most of these tutorials explain how to instantiate a pre-trained model, prepare the model for fine-tuning on some dataset (in the same language) as a demonstration. + + +Automatic Speech Recognition Models +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. csv-table:: + :file: data/asr_results.csv + :align: left + :widths: 30, 30, 40 + :header-rows: 1 + +Language Models for ASR +^^^^^^^^^^^^^^^^^^^^^^^ + +.. csv-table:: + :file: data/asrlm_results.csv + :align: left + :widths: 30, 30, 40 + :header-rows: 1 + + +Speech Recognition (Languages) +------------------------------ + +English +^^^^^^^ +.. csv-table:: + :file: data/benchmark_en.csv + :align: left + :widths: 40, 10, 50 + :header-rows: 1 + +----------------------------- + +Mandarin +^^^^^^^^ +.. csv-table:: + :file: data/benchmark_zh.csv + :align: left + :widths: 40, 10, 50 + :header-rows: 1 + +----------------------------- + +German +^^^^^^ +.. csv-table:: + :file: data/benchmark_de.csv + :align: left + :widths: 40, 10, 50 + :header-rows: 1 + +----------------------------- + +Polish +^^^^^^ +.. csv-table:: + :file: data/benchmark_pl.csv + :align: left + :widths: 40, 10, 50 + :header-rows: 1 + +----------------------------- + +Italian +^^^^^^^ +.. csv-table:: + :file: data/benchmark_it.csv + :align: left + :widths: 40, 10, 50 + :header-rows: 1 + +----------------------------- + +Russian +^^^^^^^ +.. csv-table:: + :file: data/benchmark_ru.csv + :align: left + :widths: 40, 10, 50 + :header-rows: 1 + +----------------------------- + +Spanish +^^^^^^^ +.. csv-table:: + :file: data/benchmark_es.csv + :align: left + :widths: 40, 10, 50 + :header-rows: 1 + + +----------------------------- + +Catalan +^^^^^^^ +.. csv-table:: + :file: data/benchmark_ca.csv + :align: left + :widths: 40, 10, 50 + :header-rows: 1 + + diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/api.rst b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/api.rst new file mode 100644 index 0000000000..a820e1dc7b --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/api.rst @@ -0,0 +1,17 @@ +NeMo Speaker Diarization API +============================= + + +Model Classes +------------- +.. autoclass:: nemo.collections.asr.models.ClusteringDiarizer + :show-inheritance: + :members: init_speaker_model, set_vad_model, init_vad_model, setup_vad_test_data, setup_spkr_test_data, run_vad, extract_embeddings, path2audio_files_to_manifest, diarize, make_nemo_file_from_folder, save_to, unpack_nemo_file, restore_from + + +Mixins +------ + +.. autoclass:: nemo.collections.asr.parts.mixins.mixins.DiarizationMixin + :show-inheritance: + :members: diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/configs.rst b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/configs.rst new file mode 100644 index 0000000000..f7910be8a3 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/configs.rst @@ -0,0 +1,65 @@ +NeMo Speaker Diarization Configuration Files +============================================ + +Since speaker diarization model here is not a fully-trainble End-to-End model but an inference pipeline, we use **diarizer** instead of **model** which is used in other tasks. + +The diarizer section will generally require information about the dataset(s) being +used, models used in this pipline, as well as inference related parameters such as postprocessing of each models. +The sections on this page cover each of these in more detail. + + +Example configuration files for speaker diarization can be found in the +``/examples/speaker_recognition/conf/speaker_diarization.yaml>`` + +.. note:: + For model details and deep understanding about configs, finetuning, tuning threshold, and evaluation, + please refer to ``/tutorials/speaker_recognition/Speaker_Diarization_Inference.ipynb``; + for other applications such as possible integration with ASR, have a look at ``/tutorials/speaker_recognition/ASR_with_SpeakerDiarization.ipynb``. + + +Dataset Configuration +----------------------- + +In contrast to other ASR related tasks or models in NeMo, speaker diarization supported in NeMo is a modular inference pipeline. +Datasets here denotes the data you would like to perform speaker diarization on. + +An example Speaker Diarization dataset configuration could look like: + +.. code-block:: yaml + + diarizer: + oracle_num_speakers: ??? # if number of speakers is known input it or pass null if not known. Accepts int or path to file containing uniq-id and num of speakers of that session + max_num_speakers: 8 # max number of speakers for each recording. If oracle num speakers is passed this value is ignored + out_dir: ??? + paths2audio_files: ??? # file containing paths to audio files for diarzation + path2groundtruth_rttm_files: null # (Optional) file containing paths to rttm files (this can be passed if we need to calculate DER rate based on ground truth rttm files). + ... + +.. note:: + We expect audio and the corresponding RTTM to have the same base name and the name should be unique. + + +Diarizer Architecture Configurations +------------------------------------- + +.. code-block:: yaml + + diarizer: + ... + vad: + model_path: null #.nemo local model path or pretrained model name or none + window_length_in_sec: 0.15 + shift_length_in_sec: 0.01 + threshold: 0.5 # tune threshold on dev set. Check /scripts/voice_activity_detection/vad_tune_threshold.py + vad_decision_smoothing: True + smoothing_params: + method: "median" + overlap: 0.875 + + speaker_embeddings: + oracle_vad_manifest: null # leave it null if to perform diarization with above VAD model else path to manifest file genrerated as shown in Datasets section + model_path: ??? #.nemo local model path or pretrained model name + window_length_in_sec: 1.5 # for better performace try with 3 sec and 1.5 sec + shift_length_in_sec: 0.75 # for better performace try with 1.5 sec and 0.75 sec + + diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/data/diarization_results.csv b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/data/diarization_results.csv new file mode 100644 index 0000000000..fba1a2eda3 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/data/diarization_results.csv @@ -0,0 +1,4 @@ +Model Name,Model Base Class,Model Card +vad_marblenet,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:vad_marblenet" +vad_telephony_marblenet,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:vad_telephony_marblenet" +speakerdiarization_speakernet,EncDecSpeakerLabelModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerdiarization_speakernet" diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/datasets.rst b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/datasets.rst new file mode 100644 index 0000000000..f178ad44d3 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/datasets.rst @@ -0,0 +1,72 @@ +Datasets +======== +Check out page :doc:`Speech Classification Datasets <../speech_classification/datasets>` and :doc:`Speaker Recogniton Datasets <../speaker_recognition/datasets>` +for preparing datasets for training and validating VAD and speaker embedding models respectively. + +For Speaker Diarization inference, ``diarizer`` expects following arguments : + + - paths2audio_files + - oracle_num_speakers + - path2groundtruth_rttm_files [Optional] + - oracle_vad_manifest (if vad/rttm files are known)[Optional but suggested for good performace. This avoids FPR and TPR] + +paths2audio_files: + - a file containing absolute paths to audio files or list of paths to audio files. + For generating a file that contains paths to audio files, you can simply use ``find`` bash command as shown below: +.. code-block:: bash + + find $PWD/{data_dir} -iname '*.wav' > path_to_audiofiles.txt + +.. note:: + We expect audio and the corresponding RTTM files to have the same base name and the name should be unique (uniq-id). + +oracle_num_speakers: + - if number of speakers is known input it or pass null if not known. Accepts int or path to file containing uniq-id with num of speakers of that session + + Sample line (uniq-id `iaaa` session has 2 unique speakers ):: + + iaaa 2 + +path2groundtruth_rttm_files [Optional]: + - To evaluate the diarizer system with known rttm files, One needs to provide a txt file (path2groundtruth_rttm_files) for groundtruth label files. + use above mentioned ``find`` command to get all reference rttm files (use '*.rttm' as search pattern) + +Each groundtruth label file should be in NIST Rich Transcription Time Marked (RTTM) format. Take one line from a RTTM file for example: + +.. code-block:: bash + + SPEAKER TS3012d.Mix-Headset 1 331.573 0.671 MTD046ID + + +oracle_vad_manifest [Optional but suggested for good performace. This reduces FPR and TPR]: + - To perform just oracle diarization, that is taking speech activity time stamps from groundtruth RTTMs or another VAD from wild instead of NeMo VAD output, ``diarizer`` expects an oracle manifest json file that contains paths to audio files with offset for start time and duration of segment. + +To prepare an oracle manifest file, use the script from ``scripts`` folder as shown below: + +.. code-block:: bash + + python $NeMo/scripts/ --paths2rttm_files= --paths2audio_files= --manifest_file= + +Here ``paths2audio_files`` and ``path2groundtruth_rttm_files`` are files containing paths to audio files as shown above. + +AMI Meeting Corpus +------------------ + +The following are the suggested parameters for getting Speaker Error Rate (SER) of 4.1% on AMI Lapel test set corpus: + - diarizer.oracle_num_speakers = 4 (since there are exactly 4 speakers per each Lapel test set session) + - diarizer.speaker_embeddings.model_path = ``speakerverification_speakernet`` (This model is trained on voxceleb dataset. ``Use this model for simialr non-telephonic speech datasets``) + - diarizer.speaker_embeddings.window_length_in_sec = 3 + - diarizer.speaker_embeddings.shift_length_in_sec = 1.5 + +Input paths2audio_files, paths2rttm_files and oracle_vad_manifest by following steps as shown above + +CallHome LDC97S42 (CH109) +------------------------- + +The following are the suggested parameters for getting Speaker Error Rate (SER) of 5.4% on CH109 set: + - diarizer.oracle_num_speakers = 2 (since there are exactly 2 speakers per each ch109 session) + - diarizer.speaker_embeddings.model_path = ``speakerdiarization_speakernet`` (This model is trained on voxceleb and telephonic speech Fisher and SWBD. ``Use this model for similar telephonic speech datasets``) + - diarizer.speaker_embeddings.window_length_in_sec = 1.5 + - diarizer.speaker_embeddings.shift_length_in_sec = 0.75 + +Input paths2audio_files, paths2rttm_files and oracle_vad_manifest by following steps as shown above \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/images/sd_pipeline.png b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/images/sd_pipeline.png new file mode 100644 index 0000000000000000000000000000000000000000..258781a2aeac68d98690b46840e7f51b627a2064 GIT binary patch literal 55717 zcmeFZbzD?!*ES5pfQ%q5CEXxW(k)1Lcb7DRbdE?UjdZ7U3y5?#NOws|$4G;Gd%Uj8 z`+o1|{+{>$@AvsTaca-r=XtDkt~l1(^Ilm|>d`~uhX@D=k7T4@sv;mDZy_Kcnxdir z?|igH4Fnz#T~wvS5XwhLc7PWi7TPkF3JM5Jz;jdtFd{JmGW-zWPZ;sZpXZW@3Ce5`qB9?lA(~LmAe^b z30H-J-SWfo-Cvp84eDRl7FP}$)YJK0%Ta^C(gFYLps)w~SlhmT6oUYX4hVyU`+9q1 zPfBNh%JILiQ(XD#;(BwN2$rT023hFV1x(t%j%KyMm6Ydm3U|T7nN*f+@lfX9u8Bh@+Ue8rLpvHQ-4+JsTvrnP5O)|vg;;x5REAghk`fU| zS_tY~2Fa{Ye>?DsJX)1_Uk!6f;n`$nV0OA=Vd|Sa7Mkcy@T4FKGj3+9d}fGs<>)E4 z)y1E6>B|KKd(L1ydQ6RoF3P8Ul5OR<&r-jf=+Nv=Eg{CI$NL z2EAhAWitCvX@Wxm)duro_EL52(TIZn9;AN{W7Jkfy zh!7fS(dSqSUyNm6zunYf4g`PZn~h@q_;&qyAxbJ8u#z;XDqmIP<_u}RNJhUv*Z zhoR-ieYe=8S>*=Zru5rWi~0V6OHcuTO9_)ao*ueLD;nr?7%AkP0Rz9va&K44a=^fd zb2`8vKu6J_fd0hIf(qXa%~3(30p$ROC_YY>i7?uOcmpb^9Ua^G@LD&MzQX%mYN3r9 z¿oKS@TlEMU49G18!oYfk!?H`9j9CH9^A&i2cJ*n7(@C7_sZ#>Nwy~9GH8)P5A*h`Q=Qz=c zY-AbRa`X=#X|$r9O*Mi#PXJg0!RI0JNr%SqaM&ppAeTO@5Q0!w^Ejfq9_;YacVOoT zg!R$w`OuoyKNM=iIxSKtq=>Vylq!0vI}(mhyXEvhUhg)&-eLftP&1I z81RM@*_PwKZ4Of`;L9i!hH%|sMeK=HVs)&O1nJ0UrB!VoM6sClFIe^90GGlBz8D!A zOz>yc(xO6DI3PG0UxEW9ANm zNru2?BU%Wam7fA+Wvan0&Apps9~LiLCf`*N;ljqCEKN}h*5}4t7Y=8YQ4`oG{j($5 zHo-nDGf}&M1vaOIh7-N>X-PB&=O8km`K#~8(nDTJ<^g+}W=sK^$YCw#zcFwNm$?62vW~`ZHHu`WoTTf!1#mhNVKp#55pFMdQXmQXZWxwFu zAP;sQJxu}BD)}yl`Ncorh=P^@bb(T#Asvs7g@Dg(H@aWkwOcguN{xJNOf#E^X+^v7Y~ z-h$6MXYs@D&^{HqRV(chqG;uPvEn1ZXXqBx5rTXP4&V;SqwZ3`AMGqBl4tsJ;#DR< zLmF*(3(+Cw^w7zW%9EwR%kH_W1(OPCQF>)7eGGZLlSfmde67PH4ewkkVXb)6(x3+E ztdTJ)<=F3@mp0{Sabc}2=gEI>1?P_lk6;aahx=TRya-X7M`D2SSP-qI#BoA|a(awf zrO&wAY#aCU8?sy`t0Ch!aX~*1b?T!}PqiX4N&U0jRT99BrJMDq02`GVtolStnAfZ$ zC=*X7WrGrsE(X>tIhk$FuFZ>wUsJhGw$z?ThPyaD^z$x_1q+H()WX8dhXwbO5%Y84 z`yd(xBtbjzn^huHBV@jpT~$Nk8Q_6g;MUUAIfc1pA`H`?I6WiO<#wXOuXl);L)H z^K;wCKJB^?^($hq+vjB1&Zk(MGqlRkf_qeE-!)*?0fT^zf`v!l$C0S9t_pZ|bgRjd zmP9=URNf;}R91Y!TqbQjZn!-*ozh?MLP|79hN2U^yR~Lqq)Lsf2;UyO zVp(}KKQbG9Els{1?UfHz+mjYe=EoP#pijLxDdR7?n>;-Ul*NeE-Jue;K2UXiUXh?w z9EYfa(>Ar}@4qN2(Z zBqq8waXF!Z!$R{y^rngLF&gJMMO;^5B?H8q915ZS6vxOKhjZ5dLk{DjT=Z6Z^;>Gr za69LKx65%M7sMkxfE)AXuA4DA=Dv?PtJ}L0ifpd~ukDXAU7YNx?@t`K z29Rvouy{YuWj`RVp#EfHNZ$?w0Rb*xRg-TyD$5NgCJHps5@1!Uw2Mb&di0%vmAntU zoxT-vKs*vf2ewSk-j3D4D&0dqrU2eIT67#7yu{tC6~W5J_Ct+@laE5G?yAc7K5+YI z|Aj^f;hMX8y*0Pz)uwO#q5TBK12*1?ZrvTGiOtWeZ>;Sa#U4u)x*fHc#cwX0)g6l3 zA6M!DdVqHQ!^^@R6O0BxXdV~f6~KgD(7b6KH~J%~`ZbP|05py$h(`PSd5hjz$#c`I zVBJxA*C9S$MjvK*P=jb&QV14m%`_c%w6=;%J4X?^{N4+He2}_uMZ@W-% z(1Br~$8tox^Qdb%g)Ug3>~#U7MOZJaPCxVu`PLQ-&lLuz=l6mc7x$? zGB>X#&@<4xM+ZW*&W`je%COs?EfYwxak?+pMZem@9JMnq2s}K(0?0bB~Mdd z`~kR7#^F~5v%G*jtx%J~#YT2W(DC`SbZl&kVF`(#ZC%L7D#<>H!t_f3rz{7u^T&J( zGsaalXI0$4Jq1)9t8jL{B@uiVp{i29uxl67T4GY68TPfeWG1J~m;bQbCYjvPZ)*aT zghqU$&_m}EG|s%-^rfc8V zPl`z$gb$|9gkBk%WK?iFhWu607^`}b zXr)dce(9P=IDLqqo94}Q*ktJq9{+>BsxaW>inxWa)AOZ0{bt3MH*XZ5x+p|?4?ghe z-nwp3f*X7{RQWGX120vKgy$J?g(`Ac$#QwROa7Ww9VG7X*I{(PC_(jQ(SfX>YL>xz z#4v`9{_AK1*FiVDc^0$!h|<9*TC=jzmgUxU5Y7-S8R`JCiq0R>~Q)3o0V}3 zNaRAL+jNu^dN?CdQ{X}GcRFQ)@g5@_cW4q(I;T&s-H>MRT@O=8i?*r@o1g(Be67(m z&^tdWiNHcQUuaz`vH+F*X!)z?81#=; zM_#FGg{iqzdcAMwo9&017y!#9Mx>v4!7sM9}J$kw{6QiPR ztn|FJoeXw!Sv2SIVzUE_uCf2DZWMdA^+yS{*VBq(94V&XQl+WPZ2P7BP)h(rL&)r$ zNA}?99J%aJGI6@k`?YY%QXulDKttWn3^{Vmd&&Z206PRvfy5oK498$7-IX`t2RdBp z$9VpAJ;??q`h7fOcQ0*e)pH=d;}9;mk@iXp4cgW4dNUKcF?{7BF=c! zD8R4U;T%7z)uHfl+a{asm>I1BP~nqnQ*l(72~t@lNvjdDb4vpYz7$OjN!NvSy22_$ zP~H~D{tk|5B--c^=T$yZCB0lD?w8B?Y#Vm^U~@h@+mvS24>S@~m0j@ZMk6 zdv-c1VS-13W)?Vm`ZM9;5;JZ0W55Gc-p4G160ph#g=!MNtr4(53{ZS*{K=Jd(r9ER zz4r@HGUe(=F)3mUUVO_AY*%dv;DFAt3!e)O%-5tv@8fci>+G;0#_RU39Uj$ly3JB3cP14AAQf=1=Ae<>>;8){$rv8H+i* zB=urUF69^*YVY9Vor4;C0%~yy;pN1f`C?8M*}6U?^G|Rn3OWO_XCq9A65z4A#EtFJ z3sheSdggAQD1`f^Yr(UsHcF77=fU=>c`y_n z7TMwSee#^OoJszZ2?c<6v9nBC$Z#?M%?b?*5I@=6Bm4HTJ^3%Z;3UJvC4#CiP9Gtnl%S+9dUkXbqIoAxbS^mU zVQ%!!Wly`P$vE{7SaaMP($e2w+{ful@SJsMP0suWc6jLgDybd;9(;=GLgMmkxws_( zLjwbopyfuNARGGz$zWP4%YG~md<;Mn9iRZ3dycuV&szhJHU1g_K|nN*Paz06zR*t214F+v%nt#hMhToDL#zmce7rj^%#i^&Gw7nPJZC2^3x7wd@Us95{Kl4f z#h!&iv&siDqsL@*7PIb2bbm{=2jEnx&<0U@O)u6ztJzeJ@DC^fVW_+~R?8B1*-U?= zL}Q!@%;h|}3Jgru=tFD>R$7b?X(<%GTn>pgt)!xa?ie=yw7}=wes)O!;6$LvQumE; zCiwpU2U{FUAg!dD)CMd&g(pbyU97Yr+BDqyhJn){x$!;>$z5{)=6bu%I8tGmDS{|tgov%!8Z}XSQBy4^=wSP_f-_uM{ zOVrD}Z_waDlEOfZ-{9nc2XLPiCO)}2KNj;9DbpKix9%S90p^=VUwMygJr1jcrBq4& zeRW_CaBgAiH@EOdM*lqFeMw=)Dor|`IrX>j5Fi`ez|k(=w{zbA>~R%Pdv%_G;dm$BsrM!6pG@`tZH}?jfQ~rdYX2L#^Is3O19tl|K8q#XgZ(c9x&c_LOcIfV z|I2ZR|37^H4afiQz(=v7>F~F*AV9^`#-pn8qeZ!~8Pw8Wyp20h0@%0!JqR9C_fIfs zl@cY?jz;+te-M~)A+u7660shKLH)xHgx@q%3^XU1CGUy~QLPg(0bT)?1{<)jFL9_?p4;o?8X{nFFvJN;q#e>stY*r1L`@^qD-iGC9-nr-xo_M zl%?C`17J1DC>PH~fko@&;A~qVL&ZS+h$F-N5C`BYIoN$sZZR|PolOyfidWR4s4nAO z_$}gumX>xG4{#X>M!~C_csO9pCv4*@-bc;6B=MP<{<;bZ!<@B7;kbA>PNAYP#a218 z@P$5vyrXWr;v;#SA%_H|n(BEO&nxm26%VHpXLZqzw!;iqp%eIaP4Scj?pZeop{i3W zJ4{6kT*#;2!&q%H077DWwjyAuRssR@$>kHBD$}5FVlLS@m6qIMR?GSk=lQ=9`3wzo(K&D~PsneFzEy<50FW@L}6nGvK0Maz9HQ^FB-CQ8avh zbnvk*&UfP3qq1s<#Offj^Q_2n0#)9OT>eqQ!8k)=8(W+LO9c0gEQrRKPB{gf( zeQ3G@yqK1TK*n+G4raZ+TeI&eng4KJ(Dl@^U!PhzkAJ9vZ$>ixPJt0vHDh^uuqF$to8<6#KTW- z_E1r5Qjg~9uC@03Q}``AzbpBhz?>>$VJY5Q9vntsuFSPCF3zVYDlHUY()L+%w?|b z?5hbG(wuDmGLu36_!84n_BbsQX*te!@ouhJCh~4FXb+ROR{V&{PWjyqaV=@#csA2! z5m5Wp!alz8VSPc_6}JtlC?_Un`j`*?`5^pm=}CSatH@TBeI#+uTw(pa5wZl!Xj8by25rtd$G86v`7h``eH24ie0M@q=vZr6(jdQ!W~x z3D~nSc>4S6l@Du18uAQbsvyDK<6BfADD^k$UfTKhg+H5Rm5SaJ%U&_5q)`VXl3f)3 zERsDc`q}pDXVd(zOxe{a0@~NDzUrQgG|@Jo{C49?Gy*RLF3V{pP&W8DAiFnhbaeDS z;;Sx6N}v>)|F4=S7X88?9o=^5~cfu1ssRo06B_V;nF-bAcVw$?l^a%8oN z-xzOhAa$jdEDfrAy~r=YOxrSy1uH6V8mT~x=-%A~+GLGjYV5GbALrzi$>QPkaY9Y= z5&ROv88HEaD*YU>Yq%=~gNuucRAYkH+96}OArWvZ5iT{}y&)-^2ms7{Y<=%ttF#}l z-=+}?Bv3)~AWja7#3o%2ZO#vDwDoEm(+5_=k{C~sjTBrJV9hth*e~9ier)@$p<4KX zro#aAqW}K(!H!9djFS09vB@uV2?CT0r(l!k9_Suc@fj-CNmrl?Tq@;4S)tt+l;q?^4Y7S`^>wT`Qo2yjPV~<@G9BKrmOQRCbKFbr4qTBFx3tZwF0okmV zm?lW0NDqN5%o26AM&GyHqIJbz!Sb_l2@6$Q!<cO~R%vBq&0QWhx=pb5=_M0Es!_ zj{VF60EzQSr+i(Vk8%W(*hEtT zpR#Okp^sVB&|pD<6{8wE@BMuFCK2Z1@QB}g&jGlm8iz2dH{gY|u#Qtq$k3SaisOEe zV>fNpI6pR;s!*4sGGhF}zm_kpbQhq9BTWsZ`Yhpi#ff93MuQcIAUxQOBqAnXR4Tz< z4RdzG`PP%};^KzE;=O4AO3Q2(aKo@MeD{@CL_j)DBSth9kTyLex?2V)pEg-0g|zgS z`k^DS<8%u{Kb?oYi<9u0B1y-A>=1uqPn37ZK8Tg?F zCLfR09a~MHA#K~$TnY7d=yFgo{>W%#mN_j@HU$tKIW@PZS*3%+NI!^?asJ+GfqtQG zX7ItsAEM=VBeA00IT$z$k_PL&QSOzpni4>@6^wXip<`&cT%N#rgSp#p0bDPM*;%4!JDCVHbyyeTqs~oY0p*h(W`__qv6! zdE+(>*KAnPD;#0lGU^fG6fNQt;Ot5DV?vg$5X2Vw*Zra!NyN8!JiRDsOK13qezALH zC(g=xBf}US<{C874*|&4UQFS-I{Tdiyi>nEEig?jsf@c3Hz4=4xVnZ4<< z&GQAvM9blnx!s;7Pnh|5o=mmlim>%mar~m^cKUdwiMXC?SL#)|`*IAsSx;XIm+eNh zI7W<4jit=?Nc#G-5g!&R@Tj&rGLL%ROJ9X8Cf5+_CG z*~NfrcJX5@H)7qx?eOa!%hhTiW_T&jvPZ5ZCyyfU7ue|_tF5G`dc*5=NycZ-;-6MD zjkn`#YkY+iA{)chORwJW44!j{XJHLHJB*gH=w&B0KAXId-ro<6Ie0# z2cMH}f@-$|-F4pS5g_s}2x=dLMUe2Rt|;`M&$RxkeX~ES;v<|%;`?(9bjP?%Yy81N zZ)3?&*#(}!_6Y;ZSK}x@M-99>f`@stgw9nQy~@=&=5QvA`9%Uy7j>eBiZSGSKmMZf zqW(u*W+rQ*zVrKRMGCCw{*UxedaK-qJWZIDYEAsQLb0p8&TQsttfGdJpG)z(?&8Tw z7{bqh2^A~sXqXd15>tR@F~%ZLF*`nyqk{HFf3tUa?*U@O8Z>MP|G2FTyc*GN{Il;i$fsrS70>MRD`FS zhxA!bCI<%xxhz_LUEAt3IOo6F8K*5ioo3_KMDow;2t{l4I5AD;am?7CtrGV-+oc6` zEAO-I42vrEx(^#+Ty-_aT(fsQ$(wI4#fB+h-59B%vCSDoa) zlpSNVgkX6xA2$GnpTRJ}GpezS`0Ti1*h*3@A(G*DZQZb);cY&c@QiZCDM0(I_5SY2 zVkn6nd*KGqpN+Z&hmW*!38=fzy|Fx|3-2E?5b=)xu43( zagI1xtV{<(W55OwS{k+SnVbJX?M#lQ!y_{@gO}6#U+U^K4;zn!+f5}_{BFwLJs+UH z?}>P_IU;a0d0M7j)uA6$<;W7|ajBu9(ZdBB`Rcl7K3i$xJfXl7#5F+88h-78?X(%M zl@exi^XkFOi13ZwaN~+ioYW_SMz{WwiWVj$6!dT3mu5=d(Zki|e+~)|qSufVLgz#z zPAIG9yIh584q!-XoI%Ty@{AxUE`8b@)^CWRi$A5%x`&Gu7r>AF+r+`4NR8)fX#HS#< z7ndp2k-TD>WxDxixRkCGjxAS~(!wO+Pk2~fT6y%YZ;H42-I;d=Khj+SL^3e|#l_DL z(((n~*qiFc{ZB3VtuzqaKHWwtp)K5B3YA%pKtGe}5yNz3DAd5EJ!h}N+EO}aIjzTE zqV3=)I(EhSpU+uE$gzDEIqAYpArQciyaQC#V&ap+>9n5Xd(RhIUC#bjn~4$5Zf?Ri z`_DZumfRz#JRoc?Tk^v&($^>3L&?-4cRxSs19n9>Lo$Mx$g|39s=@tenZtIPS)n&J z7H8O4#Q&EM5tmJz3dYG*rso#XdXm6@op~e{F!LOOM=%`$!pK|oC7t&K{#J)HGG)>f zL14*962`4=Won|WDy-dJa(}y5pd&YrxDecTpT>kHUf*>S7~y$*(?cfPKJMT_Gwqam zs9`o@wIn6)Q6oCj9{3*pBE|U~U;!dXBqSu9 zhuC`wd#BUUSeuj`nocCwIZX#sjj-CDwn#;h^|-VB5S8vkU}>ZjY_(K$;|GsT995 zZ8uj<3wU5??QkcC0JCOH^xpJoXjD`*|H(ts(f_hgeK`suq$*p1O}B0z;eOq9FOGfd zB&A@oL3qg$XuS;}_~MWxby100>X5_}QUx9J{_!mv8$w~&sPlIBu_rToNX!dEnCa)L zf$!HDLf&H4-4mZ0MD6X%u9WBN9a%{Q+|c<5c2s-Yuh(aq zBiBUOHY9-*D8*AiyHLXk*~M0`40LpgGytL|5L1`|a8-#0ZpoK-G1NIG>^e22My<F9MjR(>I?qbyyIyI7B7l)+IEb}kw9hSscL|{YDodG-bE^Qt0e>lqk zD@fNlAm45o7vkMsz3csH#2{I{N=Lbrxf2Lazv;yECW*yWii6}<30gFDNvz>uH*M?~llC8XFV_220t6{B;Mv|t- zzqMBXt#%ZoU^lI}+PSL3Ub4og12;4KOF9fVKW}cY;vc1a)*riFJ#dp+y{EX) zy{B(V)Iss=481b0-6uoz$qk;7g~@7zk3k`B zj#m>eke_jL*~Mz58@rP{-IC~RDr2Co|YRCYFueHXSy)UAhKGG?~ z0v3pUg9P(`ZYWhHV}KEhny<6`RZ)E5aP+j@IP_7C*HI+(g`L4&iM_4gWx2vOB01UO z!8_UkV{++cO}qACu$%qdf?fM0*Y?4z_SN8gil|;q`OW#{&qaeO$K$Q)^pXWP2kx@b ze$oOShcOCHl!^vX;&bX=3Tg3mySKmHU|Tvk7?YgEe*QCiTp;wO6H|mUQYwzkt2#M#c3gk_{qlNPlCMadYzcS(LyOt&FP zb+4@4yF6LnoK#cNxbSd2^3kiaW9%hy36HVg8cN0qmCO zB;OAzjJ6Yg<4wTEt;sGE_AR!H=F)eYX_HPo$Q~UaLqy z-=0YYKuT2}q36OX)3?s>DXIysV`wsLL_kSV@Z{yyZb`*wFJrvcgSv$cO#jQkDu$ zLtuqBZR9@VXif&ZZ+yIWs;0V~qmfEg?%wi1^Fd4ISK(P+hDKe^_*x$ZP&@yen7&Go|mr&7JIcQYYp4$a4_9r&pS39eNB{OB+V*|8E0e%btUrnO-_bl9H8 zTZwh#c*|mdc1=exTy1kbH;;d)G5!dqH|5s=u$`PT`J{hI<|P7(4WaaOsSfVAnBem1 z7U-z?A_IT>uB?8UL$MR9)wmtBuDxPCp7(wW>TsShQ_*;}{aQy_Ln!*35NXlA*zL0CSMUd!|Kz9o=?!wlShp#AbXf@k>~{x`!qyX13I zOp~vGt0)T^@5q_;+_h#V=)P8+_DTzrpSAkiLHD_WG#cx<=NirKD7q2P#r0Gh%nVdE z>$r}%t(VwSni-x+9uKG1*BRB?+_5*hYv40bH@np&wz5(TwE6ys6dLzEz~_6t?5yKu z$9nZ+zGmZz9;w3XEhQ_t_K=mnxjiQHXx)e9F%1m{@v2`O!qV98t#q!IOq0E)*sgA~ zG*i-N(w;5dJgO_5pQ@s|H_T{)WC-d~Pu}?j82o!+5`J9KkG; z1_tFF-uMq1UIzJNOkB$E#Xy(#KE>mi8ceE1%Cv;Lm})!H@gu4LVM|wReuJqP59i;y zWDv4nIeb|Qj$RBEaGYWD|MucRf2v~(BWufL4vFie%3C%rO}_ogPB5hvz=u4>8ShD{ zkO;%?VI*9A!sB2Em`CH|Yk)1O4bsCD!c&tj+}Pf>MX6E}zIsU~pM>S5LYke)+Im~< zbKL%b!?-OV^dReQLQz1$xbGg8$!-tdqS9HB2rVP+Ga@16eqZ3J-s!0SFSSn!sdOv8 z7wPAxSSB?i{CmvpTD)5Xi5q`<&w(aE`7E2!gslNRvN2pcIX0|UO}GlcF4EXG3MuZGpbDL@*XY8KKbq3=lZz6BEs`{P3o_T$W6v?DdryGqfnF-_RpYSqFgvB{jzh|~WK}EUdztWMQSrH4at7|8re{jF(GerSd*F7Lx>h^?2(zR${Z-5uP0RqMqBb^ zI#2OWfdQyUsv{d71Ni18Ojj7OirgLtPIHIfpbAU?DP*X{85&y(CGsRGL~I=%v7j{u zeqE!CiWX)&{T&Oo$s=T}5Ey!7%V!EN`S9-xZ;8Dd{JQ!Ut;29z=3miiA1%n+-624s zuLW9*5v%kJ-`k1Kb4q=N^U~zk_~7JobN!y(U0rKLqzi%P7E@QPahV)NH_N`U7l(ckyTS{8yPhg@Z%{ zw^ZK6VAh|OOqB1w(ZweeZq)G}9Kd6fr^yn$q3kG4(o$TsCjfpKM7yBT4)*NCLjR zcz^_HMXwGlFP2KFEY^7YqD7t-?CqyQOSrd0As&a*DlZyi zRKihF{8IyDq4QT4M8ZX=ejwqL~MW7O<<*1 za333FCGD?s5K8r4{rlL80uwsyct}_PJGdAfz2;J%DmT+%f96iM_O4O(N7Px_AmuJe z(~q&y0`;#tfGQY<3l`z%%a(oZe17K?uc)@81yllY3^~nJk2$t!;J1+8s2V7G($iQ-S6@ZDZ6R9EO? zXfE$MJsb%&Av3Y#c1-+L_gD)Vi}fCM<*YCH2=<=g@GVCX+fcpMU2*HUp-+0ZUUII+ zK%NO1zkc3$tNMqtEA&&U@Q0@n8jNO{sfBfvY#nB=C#BSFN>n+=VBfnto-t7hDt!V6 zdp7-$etB#mt`tvPw1Izg|MI$>;}5zzq6m9BN%=rLy4CsRqN=XD+F~&8>sQ@}4?B=N8%pE#0r*sNe%{@A*(jAa^4s-;e)pEUz6uy47GnN}d@l&zs zt;e;?)=*5Q|1TYG`}sVwp|?PWYGwZn^BqE`E>%C!oe-}nGR@GgOHa!GCREoV=kD#H zHWKoI$>L8oho_pQ(c|N)FZ!}s48Ha;7Nk*ab@$~mb;5hc{~8X|(YM1g@5lg@IWfR* zA#=KGMJ>MdpH=HMg=?Gi=39xi>oj%S+sLwQ*_EE=`fVpACuNgC-}y10*+7I4JAHEA zibRI9jAzZ0XN^B)>gE_IVGh>MKDR9T*(94B?z*g6D`J@M(TTcNv?mtdqPg=ro`d$1 zv`#VSb?L9rmz_R1SUPe4h!yA?JeXfw??O7A(lR3Dt*qj)9;a#e{&3Os1NswgdEsA| zW&p?KxRIdeFjlu!jKyNL+6Fxxmt*b6mg+ip1(NIE4B-U+=V11EW)DQCUsv0Wqy20Q$*xO6&qCHE3el{8$SipT_?+y$~pz>3g31e{dnMz>~i5)wXnGd-_D1 zY9<8{wr}E;q*Ym&;|`sY@@HwVS_R~9AJgAZ%hvharc94Q$WZ0XdKsd0aUx2hsudr8 zFC(|D5o7*<=;+UiMi-SwhsEli@%2x`=$8;tfK5dXb~b?f+56jVk+aS(^~dwm$Tv}f zr=jR%f?qOJ$UIkNFmmr${jcPbfehHNblDB~5}bVHyVYeM*D1b!J7B?2eSwgF~FNS}*+)PC`c$_Vc|dM85^NL#ete&HLYc=|5E z?iljTdQZf}G^kAr(UN`pdFD27=kZ@$PyiA*R=exDVy;J_)>|Z|nuij?!3K&~n$g}# zP_XQ{X&iw(!%VmK%rA2Umb(%ZX00 zO;JwHs%D=x%~KxtwF0_Qn~128`W>nF(i|ugId3If*{u2e#PJ|PsPM4$ASt-cuMd1C zhLhZhldaI?^x_pIOb*wD=L>iDJk@rHB0Qwu2dhK(FvR-uR9jy) zVQJ%`ZPPIo(CbEThG{ATKMe5KZvp(v)CGVsumVmRU$6J{KnM%y<&!AbjH#Ii&JPwM zY~9=)0LpflFiM$eb93{pGT&D8^MyM53Bl#EvkviJb5@yLiWi4VdaGl<07eyJNGQ5} z1=}7hRLKn&EWA&)%3zN?BNbW}5^PjEF(mEqQ-aRaued7cW6S5K#LHK55Mxqq&F#nip z3JI`J3yomOI#ViRqrry&&s<$s0r^9d-~s#KD|t+u@P%xa7baN}e2=dU?Bl{2zu;ZR znYKdT5=z7-5t{uvp;dE7ul4-5#?o=QEUgN|(}AZIp;O`{h8zCZ6PcfpUz(st;iJ%{ z^`O2FOggyE$?FTeJYWk>-7G4|;2^iTabJ8Lf`5wg;1Mxr^u@Awg`wazokGg1U=L=e zVl=TI+Q_U#t13Vyd^8ADZeBH)5{;7k-=0Lnv_-kd+H>kd%N!%u$od(guG99lc<=mR zsVjXhqe2Djy@^N#8D7Qb5--0r3I1E43s_e-)Gxzqj#3xu8{J`0b3+W{v+w2u^b-Z< z3kq97FPWRoh?YN232ELm{jU7{Wsa?BoyH^x`hgwz?C$2GVul6%6h7NGd(#Tfqh|)O zwz}>vR#;o_4jeo6vHA8&Yx1l{GkR)~(eV3#STfwK>tzVG0f9&N#SbWI_!)A~_E@ge z6M|K)R__akbwYvA6O5%?NXI&-ElOr0|NCCHC|0y%iMEaYk7uU3%r{OMr+uq$TEitT zjA%{-DV{>$Dakyj`2N>L>z-#pZYRLS#K1FpGLKbg(p~FraY@eB3n5D#Af0}F z2^$qb6C86T;xG?wc>+*d^SM4$ZzRf3AGb%dii*fg%)(WxHVCC9>x1TH2T(dx-`ZtN=Q?n9;n4d+s^2_M8HeR zl-olOv{9#-wdL_yi8>&HFcH8#J&P1Pk70WHw6@g3!eZ^iOTmi{G*-eHEEI;=f~E|{ z_&c#DqynZ_3_#n+Uk3B{|9QayK-Hm$a!tRiD*W3>?M8ZvDbaHOkbs$t!XNx`C{{0S zEC9RZeAuvU^Lnv2lJu$e23}5xlg<|>6A;(T<0_U9Hs{Jlb#^+^%|H{6>3Jb$1K}1z z)ppt2L^Qx+e;gkl$8z+sZFw1Wu|JplOg=0~i&}>1`f<^-|6qIzAYaDV$Ol88?0uaG z-=BVP_v%%h)T5zA;qr5@uEl?$h!reqXqbWDNeV>RAuT}tY_Rs#7}Z>hZ{ug1DJJ=% z*#dxneG&^{1-%(t_!hNL??_(H0~8L)S?JT>#2TtEPz4J$-C%#{WkS6hZ}N0GcmtS_ zGv~>5pmL>$C1;cfFVzPiB5e?~UIzIwneV-66=T}|Kgr>|q<|-nV2p>(10@A2%fhA{ zHKXCQe~hP~{3^UixE?8>*mAx5w2a_5C8YNj*yBmc6?G0v#H!_4S#7*^ER)wm&qw?C zw$se?>uZqAE&-NUEd2EHAwpPPM`&b_iR;c(Ng}ghleS2BLy-Gzu||pc_pbN4hD~;D zgJD1gSq^AH*o$|sHacV425uAtNQ{4mG&|61pZ-8T%5yup`rmFh=;H_7U7tNihy6&K z8(ea*g7a|ERAcBcHa~hMm+g_M?|pll*cCtslR1O=ah{4y1m8c1@)wYCf7|T$A=3(fbI>-MvvHhP>f#o`On7VDC#7uKH?J`9nOLyHIIQ5v6~ zbC;B1Y}~-^Y8CXj7AL+)1p1!Dmk$CoHL32&=5Ub=g)q$!(B$$@U`b<4TWQ{pvyK-& z;(Ps*J+e#y?v8IaS^cXrrX$yMegT0T@eDEND+<*3Tsgo+of+5*z{9Pon(MiIYOHt9|%o9b?b>;1ZyJQ}HtmTSJdoLoF)Mc7w9@5|f`> z4NMlPor&w|*)n_ErnxtByc>kqgZ*CFqT!DUUFb_Ei`-w0QZ_k_0jO~H0H{FIC<7$E zv|fHkOXjfnpxr7sK!ggkhVY6S1Dx)mp4M>lCpgO=Z_mq0h|XDDzPMbGbK7WJUzWTw z2Bk|YrTU30NN=mCW`L$Cz2M*UOlu^Gex_`JihiGGAlKE%HRAJPwfMDft2)d5_6^7R zR1uC}RR-ryF;8v|_oswLR$n8z{qotFpS|O?)b-^{zU3SP>DW^U=Ftk|Mi0s8MFwFGF=9WwRt7!>KB1$Kx_r{pnbNU;5>3= z)_#a|B}MK_BjC2rh9}MpxO2d$_eEbFe0W;UcKUK2iumj6UitIyjYrGPdH}se0kmUb zGMb(-_jUtcl_?uT6Bos#wFXv$E(y-ltK*-5;HzYfW5Kv&+u! zxJ|ah?V51Ab*8%Wgpqu7fxgoDSHg02>A1XD`)1j5o0RCeW%#)h*w<<}g*IKl-A#Yx zP3`)Z@1^jcV5$WJJyD-_P{cVp27gdmV(|Ze%Ct%gFxq$&XLa-++&)1 z|NEnR8K7!vF;{H?a#X4@R?+GzuYZWHL@gN}UBBX|*9abUzcq4#v+4g2Q(qlbW!A<^ zgLFu@bR$ToA}t^xogytI-Q6W3oq`C`NOyNhBi(iAZtgxh-@SJY%Rgqh9Ou06+57qB zL+yx)hF0~+vAprH79&Q|b$fx~wMq2C{Bw1y6*%k@w&IjTVLZ|pF7=v^jV7*Dl-h2Hb8rXe0rNw$`kNMUTV2 zyX#XEaHeWA76Bnr6Zm;(q1O!bHXqBD^g@<`XoyW26FQAn6dZZaRY;i0AE$UVpx(Q! zC`9%weul^us<|uV z9=-$V3w0dVyW<{+n<^3d9aGP1~I1Jdm32tQUTNlh{~#d~g{IG5{>``($CaUsH$u{r%W$b(m|=`%y#RO;W6= zrt4X;E0fhQJiO4WIjW#IfG|SRY!>RP8n0F(K7kjAc&o6m5E03}qE$2INPyDw$nUzc z_#QkMORw%}#rCTG>&nkw1UNs#X0Bza@KU8Kwlgv%u$7q;y zb+U=8?Ad0|9U&woO|Ivv_BEAap_daEm&|$vVFTJCxg5T9{VCw8 z4QLa=n7386=zj33`UZlz*sTHK@7mKVHtkP{&@KTQmBTJ*BO##H722HXt<8ADA82#Tk(@!$AkE~IYGz~lYW zqeXe(?d2f{%IRm+up07)o4i*VQy|L>%!^Q6;_m*?XL>9gpXCFp~#!!cP@u8+yQ?DLihO(g${2V){YRY z>D3vU(xt;hKvilLjz!1qYc$>jTBk99gL+I3<}zOM<6J$J69DWRS8Fke6e8~C1_tc9 zwQs?R+nTcg$fyl~h_Kx3lhnJ&%?GV6bB`Bog?xkGeHNd>(`?W(9FddxI_=qRf@k?T zX!ThZDLs~Fgmw$jd-CrCz;I$+A60qokv5;KDYb1=aKoU6YW5Xt`ebN#oj0C7UyYbO z0C~7=n_S`j^9Xg`SSyB~uhO4A-TOIAnLFm!P=iA3mV(eI-bz-|TEJ0~JHTmcEj0Cf zm5OG?Xkd?yxG-%20*;&#{Xvk*{Vam8yh1TNATmmEqvor~`gS|8wYRv1QKnLq=%jeX zX3!~!yEnD8gDU2!{E<(aQ3yE!ORBR-vUgXZl5RLB9`Lcqp?sXHRzNc_=Iy5$oB|HP zN-F)Hm@OlgcODWAF0e87%uIhMnVKX#ADzM+r*B6s0O z6fSK{kYoNT@wkb}<(B-RoIg5xouVX&^4zJQJO&Mef1K}&sQlJ=K^Ns z>2lvswDxD)5#-NG^4M=L_VHM^EeTu&k~;*c6dBuDvdCV&aNPdg){A+V4<)71qI~U; zHWM+V=m#*{X|qHFt;Gk;I>m!uTQT6=7Zwz}SXd4VN*Wz)lySoUrsL*T_e1C9%a;>Z zzuk##WT^rW_!-QwMWGdmRKSS->r=Coc^N}SEAlq-Iw1Q#19RePYZ!_Ljdb)=+6MpI z9+QEDCkLmib(gxpH+r!Ov(k1nMwPOhLG;ZTTLoQ_r`ClG%odJ&p}p0}w6V8i@VSt~ zePER>BS3j|zpPJsdHfsSfJ_o^UWmq%SkrNLLe>ro>*Wk@QL=xTHHwvUKT}?4vyF7+ zt#wne%=xOIWBPOBV1ktG{G8}9{i4^XUO%SI7PD;vIooe&ukHA*0SB-OPEXPP+d-rIftv>U*6L>Cnp_ncz($-raIj5ry>HY4@?V zH5oyO0&T~lHN~lWX6-qWAK%xM!gucCxf_db|B!0~w`41k2pCG)Ov`MNLK1rMN{uM^ zsx)iQ0aNiD{a`a=6?XNq<*})HnVP_@9ze*AMGuIQ4(Df`sGU%eFxTYzW!3cojuu^^ zP7H1eJ4c#JsP9ntGd0qQKO)+D(a1JcRJ6dCM*?gFGJ$`e+2d7@SU=!9eoSH@oosHzEsQ7)Ts{S9$0!`7z-a1m)n+*Ue0Lh-USir zhwl9L_wYU=9HBf$c{S1o8;bi1=9~9p@mK&D8*nkspb5;hXV~{n`KrZb6zSqhUJQQL zF$n#xKV`P{{1RW}mHedlEp-lm_iFx;@ZG9E9qKDqKeIkyXgB-N00<=>TW8i_lW31zM6Sj41gYN`Sw1_fzVYLu`fiVU>C@9G2 z6f||!hQe%pAjCj=_t!+hh;TB=Gyj!MifwX-nl*MYH?XT(QD z?EQu`oL0f8>~0HWR_ zgswZ+KDtEnZ%8Whw1vPh*{cZWR-mJ0oFy$otcUH7cxk1N=ZSBLdCKp3W#>mtf^-nU zUf+AvEnO?2hg-)nIq)70{surIn>2qH;pS)LAUxdaYmHe4Z|kjF(folE^>E2AAgoR? zu=3+?n8QOl(?9p`0vECtg$^6XYeS@F27xa0j)LpA^gc4j7hfwVDLqo79_vG5Lt5CILhnqMQIirjIjHhS z%C|d1dwpw%XR9yl%{Qu#ky?XpLtE!wp20WZd$b$kpSlJJ2+Kv@qy7+SjQ7Uv zj2IRZ`LWfKZ#-N_X?CXVV4sVhpWa-QG#rfecb;OkFH@~YuwOmp`=Wt$()e%#aaU(k z7kRJWO%t{qgZ#R&=r`gqE0mbHFI$}Ym?WL1D8#|I`>q?0_A@bDTAXSp`LQ?7C&L?% z0DiSNltSIf@>f{ErC)Y~7imle6P+c@(Q0Ba)aV9SI8rZhqNEtn8yW0hh~4I{{%a37 zJx3Q)bg{Q8O{~J}K(f=ie+!I6Y!#+MV`&a@a&nb#<561pEhiXhu%^V^fxHvBrYSV7 z?@>&kL8*VK*&&CuPgJN|1gTkcFYJbYzFFQqx%6;R^O>hO!+n?Oc}weWh(wEHTfLIu z1JnY8jf;2RUJ*b6PkDXh34>JL+&KwZSJ(ZRbvas+R4De_P}e}dHX4)S2T0XfAVF zY8jZSp;x3o1C@|Bao{-cdjD$w2!%a%(*rv?B0qk3M@pLDsE)BNG`LKoFCQf=l^F$G ztH@m(+i{unKh3crs$J2u#Amfc>}X;)^sD{)^QrK}wgE;KdFOCQV$8cEU&D)=b~HRl zBH|R6mS^O37GwJTT(ymo5d^c3Sa4EUolN5J>tLTnMnG|gin&PnW0J!s1WsX4@XGM{ z*R_DcsU7PbQ*xd$Ps$I-!n4X8((RR>ey+Zk(!E>t~|-<2y2S1s-nu>K&M=rJkQHk=fu9f~ZVxSH?8Pl_8we zW~px!>U>??_Iz4z+Oz;F+=94|MpiZRQl`db*Xyr$jlfB6NbMT}So&H&@n?@OX$$z! zPe)lD>|8{aPg2ZTv~+Ez6Pmsh8F@oy7rHR1QO(dlc^kz2#AB9;lIl$KxVtM5rky=;yLXfe#XThp?zZRrntris>^%lj7?% zVNpOIZGdnZj);w@aAw(Zi9jdDEVswN8W9KSC(ZEjg-bpBTUxD-#+06;w_ly@|B!v$ z_o)>~=1?g9>iPz5k@{^K^SdTtth{j96bKLLHVyKwVui|R3b)V;Dg$7>&Ngw`;BmE0 zMMN_4q>g9yi@gD&5??_GeApyosX=EGbGydw2+C1PPm^;cNA=A)B#(`taC&8eH%(c; z^ftw+2g3Upi1|LRo!wBb!niC+qHY;d1I zQ*DuwT8|Zi=i;4q`ytOT=;8$f(lcyvep(U9-Y=wiL*FcJE)Pq+N-XyUgnMw1RR|r3 z@5!pCkf~~jVvc=b)*9J?oN@x9blV}_;FvfZ&r|AEXo&4wA$OfkGJsi*v+~UTNYI(i zN4k0ZHlQ{EM4O+%-FSK$X7^2kb-}%pJ>-A*{TeGfk`>q4bZ~NoDG}GWw@>Ojn|5 zUxYQJv)tySx-3hnh$*ZwtuDrPgYZr4CzOj$UR$9VUl+(>MW~XfQ6q1}dE82gUPB6pnIk2~6Kr}ZLRL7BdHbH75N%dj zR{h+FKbj?(@{Wh+%?cvvxhbN8~B15N1IeG(S z2VDi+X>sJ_%IZ>ZCG?!i7KvZ31v(s(3%?0{C$nLH7F6*PuQp^#;tv(f^*8<<({hpz zE2H~7Q0i_o)*EPa#06^^I7T1rmM~+^{os*FroSwdFAj9fk(KY!gz?bQwa>nGpxBuy zH~wCr;=D-Wt@`M%Zi@{8Od(w}YVa~-(hkC}kjAgv5bbkxE(zwQC}XU>H)?`EAXr&( zmXKMZH|kfXr0vZ!0#eN{bfZ~@YdJ8&zkh#XQhQo-zy$jRF@6;*FvR&7!;dJ-=SUjJk`hpkhAFcwPISK0O)Q)GrTx{P1 zgxV8<0Lds(Fv5OzxPc{L=0WS%UenSR*~Ajs<8+9PJTQjUbm%2=?Gd}#&XG!Uu8PSZ*9r`SFj6ad09^Jyl7g(@Xt(ZA@ze6)!dgGoNwc7=hU; zbcp_U!482xqc|oew9A%oOsSdN!ijYx^%DIU>gR#ngQ*xN^5m8*Y?GQzbxr0;$c(I4 zsPI=U59b^Kq>%&Q*lSEMd@V)v+I5dYNPotGd)kj z60r2|HDe6tYwfks-O9RG>ov{fp=l4OBb}=K zA{GAQ5TQPYI!SSnFUYDhYid*Juc{9^LAhp>SuU?HqzeP{K9EuF>m~HsL*gppIk% zc>KzW_?XUYIL(5!(h?K&>3zA9qe@rN)nzsAiJna9WT>=lI!{<8TYc^e@vz4~DMO(&cUjz!I8h^n-D(*CP(8H_er^7!YVi@X=&iVk@F zdz91DxEAUx&JK$=l&sD^NdZ77@OT2k_ZhHFYRu|PTOCu3NYp35reP658I)FO@2m9A zY%mE&rM*6%z~;jOtvZM60v`GC=8(Qxa%dD5#}yI=2~q7?nGnFW!Y=^R-%5}Jx0i1@iDR9eqfSuvQUJ0*-#=%=+(*-l|(A{*GXAxcpI$0#z2 zo#I6+mGa--#kUBY24+Do7ll(m?vj(z(!e7)O{UF{BK&RXEJ=u-SmyRcFHOwEt7)pe z$T}aL8k^!R1=|{oLL+a$7djUn>3VzG`#RN7p)6d3L1F-K=sase%_Y9iy+P_z8)YQZ z`$qQYoSBXuw&LZ3TfC5dK%;*7obaYGv71vcFOMwp^`85@M?a)zK-514O-kDFfHD38O>qW>x1ogtb0F>HhwK|;oEZRyHLOT8 zN1sJM0$;er_9t+yp-sVNefAB3em=hqmES=%L3&P=b5Y2_H>j1&N)hv>i3PpvXpcew zcOi<+j?3>fF#Wa^lUy)Rti39ypc`|m zxzgkEfI6sc7pS!80*Thh&9joqb!_vK?e7Eeu4Bb%xvoEE|0({QGnA7#n(=;N! zgz!vXhNzQmePJK69nZMH>i1P!S~K=%7{*@;ajoVZ;?Zu?gRq|dB${wpbdv@gs)?G{ zPHrN}?Wl8^D>H%g7^0K|t6BjX`YBjxGKPgn{rpCGQ7cw=VVE7bH8-F% zc_*DskqmDF@8A=yiGFvHSR}kjN)ANq8kIz$gIbB-K6+d@@1<>1LH`6`SBc$l*IVC1 z2Zv>6I4uE#%iT%&lDf?l>N8F4jUIH_jQGpIMtlXgnLv^$l87C z2T_dCA|}cv-CwUkFAS^|N{TDS>Dx)mTHfiP+Al5(P8u>X*c8YaOE*z$6tC?9V#>IH zD3@6cQ`pRysQ57fRcmP7l6hgc|^?i7m4j*i|@??P3hH-H_6YId>)(9vr7gds>DGaNeHTd zXqU2Fn@Q$D9LPDD!C0Mz)0+5YD20dHe0dQ!!lRvfwQegTPQYQa_h6yHWM?dQmhgKm zhv;f3O=Oq&5XF`how74@Gz@w@S^*_c{|0o>MpnsQ%`JNirnTvFxE+J4yx%eB2gJ@NRH?T>35?Y=l-;5aK*mY{ph;V7|L6Wr zUZ%}<#G}qpJYqMt!SB^YvL_=bS6r|oPk{N#HgqjgIY~$nH*0^CI>mB-yD3aP{xGrs z^?-Yp?g`%);@$6a5BxA=6`X5C5`33zODb33)q4Q?%$R)U1&40vc8Bew;Ow>p_l6g2NJnv)@bnD6#i7M`DG-U!RW;A zzxE6R$jJwiaK96^Mg1w% z3sEKkV}!}ckPbq(Tz1Ri_H)_u#}|l;g;y1s`bumL1p+S(YYJqOm4@KVO(++ApTuX4P>#nw85PjPRl+`}OJ z4(P?41#L}tfE4@Q^pSt(dmMrR8PFCrFF9|Pps^C*1MY*Sl)3b0iA6nYFIH*44BN4{ zYqY!Cp8}IRivF1)`ikpoSz-{J7YeMlM$?Kje>arUJNx(Pob^q2k!MM^b_GaGLLEU= z6mX$hHsJVPV~e@LF;?|4Z+mW|j#Em3#f$AawDxifB!^>xm(G0a2&-`wxD2hyAVMf~ z#=K{1!ossSTwCKxVv`kpK$eN}s*H`y8Udy%v=!f*G*tY!$Ol(t|D;D#XT{wFbiM>S zKV4cF33vkIYU3m@YI1(AeYs3*NdPIoPD#ByFD-E-H)52NTK$P-wA39*oYOSesv!?0 zCD4ugDhGanb0?V^rU!T4xGnz<7=U_m5zzOJ99ux@W3}IQIPi8bM(jcJwqIH-Ced^A zR){~K)#?G57YSL{NfeiJqi!uxHrci*6dysb|H)>N9VV2<6Meip^-sh%UE_Ww|Ev;2 zh2PJtg%ELsag+aa8(tqKz{SKQH=(SIeadGHkwbnt5+URpo+AHCv7urIg!#>Gv?mTWf0sLD$ZXv@6s-6fuT zPLVP>hQ~4uV!n4$%n<&=siuW-z$xPe;;PWd>Z8xaF#MZv=DD>Rk6`L8MvGv>_ZWrQ zR)H@$g)6c}D>~}Co$#l5kdgGJD;Z1v=s9t+m}a8VBT4Ex=}BTg&r-nsjQ+*wCf#R9 zO;t~IL2nZJ&$-x=%Z~z^<`yA`yxV!5Ez6L^6oaO&VcWecKP;y;NKSWwT^TE;0hc$m zxTmaaSi*GDlq(fs=HabQE41lrbw$mM*F^+W5+GCK>Cgazk01#-}MGF#%p+E3hdVD|FLI|P+4|_4l0tG0V6UaDQ7zORZ;>?JCeZSOH&W_Z-A%sBV#>3j zW3d+1ip4v%`>n(cOUMci@6Bw{oBLnWm<12`E~Go~Nwg!pT{61NZ+1eJ^)qJcPZDA2 zF;0nL>d4XuWNAosZO|e>lAIA2?kFO3TDjZ`*MNkJxV*SNs-x`4lFj*{wb$lh%wKp0 zF$SypEi2NSR9=O-Z_!v|GdMQh=`*@mtH(jhfSH_XMsH>G=Op?)gsj(wh~a!}c!}C9 zV;G`J--c}EiSvrWQ`xs%bR5$8?^)_Zg*Z|V7pLECx?h1cIG;0MnjM9B`g2~_Zf7H8 z!VvgJz4x3OBpL?YzJANtzAdgF?|fe!xe;t(7)`%GU$z^9>u_VXKheU@EMMuMXjolA zq9cf~$TTWC{ZuGb7l?jq$|TOyr?wncE7u@p%%N)f?7__D3!p zm=<9I-hU%G_W_sktP4sdn$UWAj#zr3BtzpF?_8fTLzF&6GhypK{X=HXf>byE)j2fT z6IZ}JB3u!xOc8tK4SOkSw*pW23ThZUSyo_{b4JL{HIY|MW|EgnW$P_b+x+#+ZZURu zMX0+_v86;6U;TKKGnt-m@3&4<(au8_KG8{*lF5*q`aolo@?xv?2aTB5 zG?3I6ELwPBvksGs`7l_skng?1rFQ%lDJ&Ka$6H_J^)*Ni?Wx`FX-;Z6SThh#g7fTTM|83?G- zFb`hbOwP^+aF5?$6Ew(-!r{?2&})m zE7J15bbh>{v`hWMfqO=HIO-5~shF9=(53^^?n}xtY#1kIMYnIBLT-ECN9R!28YjES zS(6OoqS>LUye(U)=G*7&fiw5)AG9=k;dPw%PTNxG#4sz`e>YDbtQEJT5fPZ*j{Blk z-Sv7+=9GHp59?RxpAUB0-)8JaRbXk@R$2V~^oKR=XG(idPkES#*sr0Bebc*>@q%0@ zNX?7cmowij`bN_Gb$a%yJ8gsA8ul?Q>#=fIbi9c9Azdx$bY_vj5GT4@bWI9e*fzDLujQ;3mj1xtLL3J$-! z*tvD-@a~lKl3I&Fz4pD7XTjG{%jP>897MG9gkSWQ*S{Cu4ecNoKK@`1Spk3Se9igl zVuPdSUC#)?BVEZV-7h}Tf=(}m7bIG9Ykd1XE<4kkgbj4#oVJNyg?1LdzUSoci{9r- zjG|j|?=edrN8*uORIm0&4c!MqFA6>E6*dnAm&HaYcan&4A=I=+m4sma6>GD$d=ffx ze-4B3yz3aEpLck*y!olkA92?0nO$ZWANsN>ijC(o(kU3&6rYBXUwKNf`*j>EXhk@a z4qOUvtaOQa9(CQkL#v5+ot>)VxP9-m<&FHtK{x4=IK2I#^48h1e7m^R`+IfjLk>yV zs>Y;~e7faW<4^5B4G}4H58HVcyV+B&G2?_huR9qu^_K!?Y#wb4sty-V#%2iRFXn1( z9729=<_@^;kF6{%R9iQkV~dlTJAw@zj~hFyAJtYYzcjuY{l776%N+2HYNF$&ZQP^QwYggjku8qTH16W7e4I~ zB>>lOy=+X>4M&E$(BktLxn;b4Jv6;Do>vG0y(*{WjRWir<{NE~C8}CgmhtMtz@Jbo z=)9NjeLJ^gx$h5wXMm&Z<~PI^^(c?92kDn_&uc-ZjwIgc6HZgk?jlgw0h#8V`)JIn z-=M@-vdran@?jrFi^$b3>w7#)kW>&aLe?G_=JTOd45wz1`))(K=LkjS~xO zf}yb(aUzsfo*{$B8x1GW1;$W+3yjx{=Jai5&O{eXfAAQsMve4h^rD0FP#Ecy5Ta)_ zypV@foxF>GQ1J@yr4f1w5Hf;JQJ)tf-xe_3r)Y`9Ac3elU>>SwtUIwCPK5=W*jvY3yC!4W` zA!t^LR8F`oZF#7lQY1q%|0sr=g9UA=aE5WEr);#t50L1~1YH+F0auZ5Ee`<9CDN>g z=xh-8pa&m;5D%XqM^};*r3TU3^AaLsvdR)PbnzZ?mo)eFSm_s{?zNNWpp$4~=DksC z^fG5VrO&#wicnF~-^DAD2zJ<<1Oc{n8ws7?O1fQ7MF_gUJEXl&rRhMH3BHhU5>!jh zlGCl*Anw=_U`sHUD)%6hVapTbc1@)D8HrJ+C5RYKp~iTfIQSwqd*cWHJx)Wl!P6$a zGkKBU9fgT|+}5SPItGVId{~1-=-h*TzO97!K{H5dzy7=4Kntg}&EUu=io8n1{p@$7 zofdxQ8}JaCX@Kp1DpAgezZL}YC0WR&#B@-KL}=C2t@}AV$Ka6Z^B-;ZKy@er z$e(1uJ}8!g)pR)RXF44a=vf2~8&4;I&(s`0o(sJN`7huCyAe@z0U5OmoWt+Hm*B^R z4>sB`NMXLcY5^`tjr~T3#?fjp9OeVp7(;_xQUoyGDj}R$Yd97kg136^xguUy+xL#w z#|rSGL&s0JGBZ(Oxym?m_pym;xzCM)zSkK%72S%#6OnV29yQGti))oFZ;CRkRvR(3pvpdx`qo}DYb&FgmBR2QhPjK zG~#@A&95DV4}eLcyPJP@K>2tTVoG%dT%qURXBC}mBmOE560$h30f)eoUjlr(>yaT} zAAq|~>FZ>Pp3qS55h&0o04R~JosdoYdoU(>f$2y_F(_qAMwzx{uUp4;dc40f*XEcZ zhn5vE)popG1cVNQJwF9zWjAaA=O@tAv7?2n!II+%6y1qvp@V!t4NQGB~E!yo`+e*N$jS*Ov$kNlW% z3k(rTkwRQmpk2-(DiCnbf3pdH@LF{gTy^l}z@f_M4)NhH}=jCQ9?}O8qI)s@Lneuy#%?n&U&sB?302oJL0pouXEr_!*M~ zW*^>_qszy@ZSU;4&;u4hMaOu?{}+KnKmdL|h9N@Xc}1%W67R=FOA}L5 z^!r~!V$Po6(VHWK>W)PZ?dBE`$#6f3)TkL4&@@Ux5Hzss%I>kQSxB&x5Bvz_inJS? zt#fc&-xBWS_gsk`wZhMmq8eU-qs1HyKZU1pAnhG|6FBnkNq}E7@O*JR0G-O@bhO--0wHL?r~{cx9v5ED zdI`HM7U$QgSQmkZIz;-9MDi{DAI4`ET zZ@5c1DN+&HKiZ+NA?$ktmr5J~3N|LZP-HKEz8c}0qL&DIw`ES&VF=kLnP(_yh=!o_ zg^#ZNzt?N53kb^1-8C5OfJJcvbQ8m`6$O~u%IB!ME6YbZdaWeLO(Z5Ei^;b~+9Z*GDj#9Ci_aPf$PfIjV?R zIIcImd^}p|IPdum0hCJmDF30en=FRi5loIS=6=mI-7W)DY!ub=Dk2^sjSom z5QW8<^nEzFLLS$^rM|7)_-{qhzvS8ndh8jD7SQ-gp_|79cRR3Lne*oVThH>69wu{T zMk_;ZMIFIRawe90n4iYvm-qcL-7kH^Wl_NNob`OSqQxEc9EbR!5wQNYRn-Rpp|h{{ zp`4VzEYCz9fi;GnoCFPXr_)?qN^pG|@xWoZKsr%XQ9!b#s__TcEF^4!U5R6Qhi3t{ z(d+g?fq7V1-<7+aQS)ONE)8Gj+U1wO8yx<7*9=ozr?9?NS6tB;o#aXSR5EQBqwH}= z{BEnb|G#e|?Q$SYWQTh5So&8Z25X>{@6nErwPMw=+tj}B8*$BP(Na#@9rL2WChdk#7gXY1AW)#CmM8DFau5}Bpd(=}2AlqKgmYkW+~PNU|} z$7?-Swvy&p!IU7M7sgPF)=u60@8b9`EjEV$*C3k`>^5gS82JDGT2K}`4JNX}xoa?+|)YT2}+ z-K(;8vTI5ebqdy*K3WABAUwo2jKtIZ|8E07Z5hZ3d?&L@%!z1@)A0#@-UO-xZ;WSI zoC8RsdN92f~hU@lTX!Mdcir}^}iY+ zDnBU%VeZv_B$VE_baO|n@prAeIEnMMKrucT{TObpA%DYMSZ zXZe6{l8^37vTd+G2Gj}g_lJ=l4)`P$b`=FXgfVr8)ZH)klE0pV5QLn&^UboenOm3Vk8b(*LNJ1MD4zTGN<6^ZQY0VgNvO?`P_%SfkKym z|D7X$U$cum0=AT5cBU|!-XEjCjJ*_ksx9lt6xOwF$EphTtg4g=kT&z=()aKV5}ykXxfpeiMh9?1qvl3561Wf1YyqrP|cX>e+YUCy+x6UnT*$ zj%yEKzD9R|`CO$%4iY9=QfMYfzPSU!5pP7%#_(tfNC3E6WyTNZ(*{byq_!XkkPFSN z7{AT&s2Q+Qdr1KoiM$W^N-d+JR+Fx9d`s{pax;f5{P+I?e(h!OTdH)!OXF(5Y?j`Nr!S`n9E@#P5uc!AAm*wKF{NB-`kSJ6X{W|wR1BqAvdql*W{I@zOAo+vy_7n>`F77OFy$5h~C&$U~u~-liF7j z>Nm21<{^yB7AcY#(n9~pTa?(Ni^+1uQ&-z4y0IIwONgbDx)VclN|_WAWjso|j0^|W z8{NT3!y45xvk)qAbY$W^XI-N(yT2V$P3SGn)#To-{g+( zkO-F3qu~32x;gci<6)lVDhz3Mueu#~`g^74K@VY3hUbawkjM}i)3SI!%6LbR1RLB@ zK8KDQ6XiV=g^ttQ{7GjqK6g3wpw)W zFN}Z&q}fn}x{cBfet!$_x4i}UPrI!N6X9_VxR4&3=Jro2!%wf)eoJ6Y@@tdBM3@>* z3(vd)>0pn240zUG!e9d11r3iQ#`0SD|9u6nLd3NqF8Uazg)Fsc^~b(r5I7G=G_*Oc4ZwCC78 z>@L<1qJG7H8H-@uN-QZSqLv}JoI79%G{IC9?#N#PnEV_F_6+UIaL6UrS?XiOIyF*l zY}%D?L0NLMa~9KG9>o7wB)gMgevdzj?$I6a&rinj)Yt3j=b#E>wyuVbjtCcYyhlbq z0p_Fer;oQ$EitZhD@7H0~D#m^ss$6{_g@>7K0nocG1HLODGASwzgSobUK!a zxMA9fQtVtp>M{cm50>8P@KaRysF^$CeO>{hDJt1_J{pY?w;d0 zS2P0LOBjRS`$QX~#GbK_dle&UPEJ+Tr^f+UjrvK7Qdgh@RD=YVnS8gBw&Iae)U0|GZx>cfkyCfLY*p`W8*mb+`{#@ zGv{=pz3yZuvz1}mQzh{!0I&T!=+Gt@knC#eK{HTefJZce>M7`X{tRA7&<|M+dWr9%p0;)po!=2ana6a9( zp4cC90=>ylr_b#D0Ln z$5Z^pl37rsfp;8*DvM(`V-&)~=xo>X{`rtEx(hcWme9HpEe9L0Cwv2=WQrA8qqXnj zJ4Up}xiX#kT1Ni>(_>!T#Rs{buHuuu$mH}Ff})YvW4ZEGkK=}ce6Wswq?yQ+^Yzyl z5wnE4c77D;wW}`3C4q)XJ*%kb7ua_(x&P(|KuA7$VKHlI3}K)1skGn!r3b10Xk(W6 zl2-dywZkL4<;U+M5%*7owi+Dw`CC-CF3=JHXB6EN;?n9I;g^ z@?hAQ`TQ}*r-b&84~9kx5JjR<<6o|L1ur$&mz5^+A(oe z>Wv2Hv})6hZrkbN+1gYl=6&M`XOqm!g^B?_3p_WX|u8sWsvZ^?G7R|*?5a-};qS9_YYV}&xM`&If!_|jD(A4Vf%L6-3 zY4$jrJ8}F@LbaFXVt#-q zx^FoG>Lqr!UhIGRNf8qErgtQKX^_Y#9}ope5*s9zdzfb z(-_z7f0gPyV}z_w-0v3)x@Qj5l@p9?zM7d00yTyd1*kFE=$_riqf&`09U+jK1dYaT zCXPyGi(a4L~^gdLIrXlD3gkqfvx4S_sg0tc| z-I{a!i5F~Ev%efSOF}Lv2-(=?{o3?vekU{Bfk*I3EczAek3Npu#J&QBm;`#@mb_)bv&;(WDAoch?Ls5^d1* z$@LmJzn!59yxq(qFw?}fUoBI>l58v(_kH>6M?gmyv?X|x@`A~_=9rfRbAJ+X3;FMU zZKxztgdk@!?U~tBm|;8l^YAp8_;q#V!Ng;oegnDKa|dlU#66j>hB#&C2@=;fp3GHuhu#Bx52)A-yd~Z9an1=zy|!h8 z+EdZs66~tk{(u)e*)CzjxaSe9vB6;ASoCRnS(%xyRkjRs$>W3H1tD=JprTDSgUh*L z$j%EDbm|B|484>TC`Uz*<3~AzOe+ETTG|q{a!lWyWlzs2_y9%P(%Xkyxs^H?I-6~7 z5tP8&*->F8cVNo0_>sal7GME`A%uhovnl`$E~j=?wJl`8oMCH){D%;R%AgEKH`F_46lUmbOUO_6_*j9(dE{?!68$Nh=dehtf} zm)p!{l$A#}q&n0EWiKn{qaP&Y%h->Ix$~Lx$9fx|Weqv!*^uPR@7bMF*Q_^$oblA2_?*12G?@GWN=clFDu;xMh4@%%g(x^c zP0UkKEldIh+w@ojB-nM!(evQ4SBvef_gJIIuOBrW^UXPw8$bC6A)nbd=Tjnr9@&>z zaV)y0t>Rb_B+gf2)HiYY1_&h|4Rw2-wUuC!R*8>^ZE<47#fdLJS^ixQt(2NwadS7L z@H$i8|J(uuRT0LWaEW{NrZbfxGtS5sskUz({P|eOBJ1~kD5wDlq84+9sPzZebUJvHKs_Nvmg>6U^Bj%I#kCd{26Q`|eTkpDJ#LZ5Y(TB0?JnIr2G z`b%8(rEfBLBV8n><1*Z`rucs#Gzj~E78V#s^M%w(>^{w=ebJFx=Y#n3~jF4 z=`i`NyIF^|*`Z<+Evm&vr#O7Qo9g88A6BO~>3G*RNs?arM4Lt*yj0KdM_00^bU5kS zsd`VpY>QzSdDnGC@AZ&Qa9DpDi{_(!K5@t;LE_r7O`iP=&Qjj+YslQU?kRAcB!?pS zD;R>>e#_p^)N(!|6i8Nk6d=uDMAZ!RqzOHj6+}v_HfkrR!sPs$>2AiDp_(1Q!3kS# z{UM?4SnBDUsQC9aE5v4L-=IYw#&%;Okz1VJP}c^S&@xo&DwoW4dyGP7x@_n;6QEt! zyTPP3GF)k?CcuaKnG%jr&oS0n^MI{p{tN#EkKPcltK`*#5|pX{!|`0>zY96RB^wweE~^!x zfFX6IIde1oV5*d*>QqUY~lw{ylsw|n(N$)1TVdG@;3b4De|@6FC% zB9ni2{$oN21zJPB#kW=IJXQ(LGdsW)n`kEspC|x?u9O6FDI21Er{|Qj8enySG+jyE zR3@X0ja_T;A4aT8o;ll=Cet%8G1dE%68D0&7+HD_)-CEX!IaRpxi3UC0=kerfd5jI zk%Sgjeu)$A74`79O=38|`+>_^Dem+#8LCgT^BcjwBK+Rgj<$%I^>5g?3TZ_o)Xf?O zH9hB`0I2}%rZBfH2Fqn$f%1D<4+wh1&fB||830G`#_g*ASMnO3D1HK2>zY%#V1+?w zvgAFrP>);P*xS$Vgj(Y$WQq^rCPrUy&V%$lp>);h z^X940`f-C}+IC{L4msPv-;XHEcBxr*Avku1(atF6kSfH$o#>a*TRu6kZiq+16WF)# z+I&E96W(Rre65z)-5RM0=mC?Ja_cnu2KBP4LL(Xe5MNIe-SO>6WlB%@)Tk2_F#v7x zY$=wB;l!?g&aX}6b_&Dm=(anv|T>HmM)dkeQHxUXFpaOfVoq)QqFqz7S; zP#Q!!1f)Bqkq&8*5F9~i3F(#)rIGHEk_IV>vqzuD-+SKkp6?&{j@MjX!^|}^d#}Cs zT5GR$uX{m2S6;aVP48x-Be$X3a7U6$qht{M9)#t9RaNBH`Cydg6&Bz!i;j;rAaVF$ z6o3Qd8u#B77eAO5`Vek9evvM8mrv}ObLABXrWW$GbodxFwOSAm$UkI72GX497pht* zufWBLt*{M&>9PJk5!}sK>u#k1W|7q~25q0`?Htme;G~BI*{~@I5O$ocK95bh1A!f2Gi16n|Rg>*lUV@6*hlRIqo_wa}xmc&^RB71UOnmX^ z%gT9|uGH@T_&ZS{Fe3cJjjR*$&$VR~zb!xl%1kPfBujB4MtrsJ>N99%!RQX)2W)vK z2W@@l!12!-839m*B4D@$IE#r0LEb@e6iINhUD3msImnL~v*!uW*JqG^0Xp*EF!X<(1R+>p?Y`;+T;gCKs%=5usb-ft0MLt^$Rfda zr1Akp7LfqNdd|AzjO?kxE@^d06zgzVQ||Kp>qmI)pD-7_iccPE`C29%a@jKW(uu3EHBiytZ zaDhhN0_pjrWWP4Xnz!A&ZQNbwJ$O zp?>3C$L;@VMRF*yyL*=Q&9)wu`r#}seyBTsb%xxNt>cic7xW);JXXru2{eHd;rZ2Y zJOl$@1E4sHeE|07;}RhC|8NCZGYXMSMy5An+m*`?Z_57!b7m==`5pf4qDi(&2KSGL zlAyG=5Zrq<0l}r_eOl+|K7cnDsyI`84Wf&~LBK!Y1^|1K_y&7H4r9}-IJ1=fF0)lVEg3<8QE zhujbE>kMu1z6k|#VlXgfn*ISX0weHOR3bz8_h zZ?W>GWY9^FV+dn!j|8fG$a67hgdl{A?3M?6oF>Vzc%< zAf*mBpKS^BMG!u^J2C6PibXGMmln);3F4$xN{CQN0IZM2yNqaUs{~ag4#@+3=ir`l z|0h5x7UJ^O!rZr~s?-hh^75uWwILI*U2$~N95+!_rU;MgO)@pge@v9W^#M1e5rFzD z^qBLuLKhLT+?Z=>SoL26RF0~N$~3NG4em_zn*!C?{+A&vl~w~|88qJboYd7Xz5CjqO>J$h zs~mK%#YE%Jlx-rdZ3-G29kn)YT=2mVuPYCs5nrj9b`VEz_h7RgFU=a$sU2)n`30`= za7H|g4Nw!7+qaDfbeIV(@6?@$E1VdIM_2 zH==azKBtH}UWk9RwemT*2hc;L56{Ne!Pwmql7Q|@+iCd*@5X7y56+{?#h!pqV#e#j z<^F$6HOn&hn$^`kqz4hGinUi+S#%?zr%Uq@4fNfob@Mr&kGJ(riVp>nA(=!AW>F-1 z4}M?-QQ`>rdO5%CMR4A5a9y}!OTx{3K<6-BT`~ncYe809(p|6>DRFTMW_IorPLP_^ zT=I9zW?_-vEo-1pkG)$!(slL#7i02c+YgDbmVvZ!_j2EPqQR~y{^MqAN*dr;m8AqO zI^L-kO7?`Sz-VmSCn&r5PtfIkA)F94@PSAQnxxoQx@-t7S__I z+x)}Q51l!%e09cv|DrezvITmM+r7|C$cs`8xGPODnNuO&2?q_yonzUb$)bq?l3I5t zGWSnP6GZKn|4zPPAr*-hV@5+V^yq88o!_IzL7Vb&bs*@}txQ4%ZIMNDZoGU}X{!bS zO|E4sAke*F&{LQ%zv2OuLjrsqt~7=npy~}2f!L@Qn_2~lRF!FgCH)L3EGqgbv3x{h zBrBUkF8O_iHhtgn{tbE1qqr5LcXo9Qakqb9=bnW%%K`@2y$)(ms8|+M0TIkESA&UL zKy}SvcjRIPlchClJNU(d~k1aB477J6R>7jvut?*ou<$OonmFhkJh~&Bp=>Z z0N$|`Xbb(z4mOcljMcFmhU6kw9sKs|Ed^qtRtntqqS`9> zAgf24Xb`AwpV^|Nb&U^v3AE=kMG=0B@tnNO>>Ia@?4Dv_IXMutF;qssVFRv2et32k zdk_LBqnmLwA}WhF@PS8~vj@TZuoRa%)`2t6RGotYgu%YHiWYB`!V(__7W$6UClc-G zQWt-Mfxyc5SL~p5c#^ z#)j$$x(y$vI<_i-PUdB(=C08%!CDh`2~)@3M+`ljOyzE9{((qPFswdn&;0OF&Z3WQ zMhQ{({eKASeJ@{9oOa9%UG7z#I&RP%xD^%k;!@}|HJfMBhW3fY z@DfXFUEST!DNQ&GK$)S^-f_(6J)6{T=FWoH0gGjao3t#)n%5^BUFwus&M2#>_~o3< zB9ErBkABs+sXjm3bt3@&NQm#{g?}F?Nix14whj8{IVHhLNQE6h#cE+;;nzNw$K+r% z_-J1Eo&Z_u^Yzwc*!UnxJ{jnqEPHJWLkVSRJ*$g9@-y@-`51Rt`5`n*ixEXzMiMG?6Pk;KHslpE3{KH0YDks6zOEDStPwj?Sm%32s&yX zwO~NhIKU>xg-a3r?8;36J5E5$?nyZi!q$F-JYs*$u;#isSF!R;#WqS#dCYD0-8A0& zv$W>dFpy-(+DgCHeN8)wwJ7y88+EfTwJ-BKF3w2~)#~DBPKs=~3|8snrA!#_hs}kP zn3=5GRs^2d3sp0194>$%0OtdS2JrtH*-dn@GPtWoy^NV{u_(a?LZ#ssuJHm*<#}J1 zALX-WW=o>=m?<*Qfwwl+Ye~l6{UC_ebKg9v{x+=VzICYgo;Qp9Zr|slpOB8SX79by zO}{!ZJ<`c{Z_iI-LPps;zY07J^QQbx@tvQth;_G+_8?r;miI%9iA?7Zy-}J>@pH@-i&?snKzoJ4oDJ#=W$*_%v5jX zktltnU0kZL;Y9smwd&8gQS#LoR*mh!-Rm(0 zX@QUf2sJY_-Vt}k32 z_c^6D$??E8Y1Mu;dOD+k&WA8YQ6X`E!uw&bDT~uc^JYWPbb-F{ncB9%3B~C3_|HOf z!>n8O8iiw3_66kj^U8kA*3Ax+5=F6-U4^~@^m6SNZg0+?uk#RWFsYWbOFU>!X%(yU z)B33K@!m*jBRlqPWk%Wt((2mel}aJ1mTcbAimxpWyx6fnxQkX?n2)ufjDF=$?eXQU z<6C%AzC zkcdc${*c|bbL~DgSL<&&G+*`H0_kFwh=af_c9P)O0;M_*)Ti->hIwSb2j2w;*3WM9Mkv6)=&viKT#* zO~Y`{M&rN5KoG`o|z}C&Q}7 zjQ@eGgtP>D!ng|aiXn&zq~Rq<&Gu+MNQzvf@r>&Y*$kxt z$ay1+8guA)lune3136am9(3V?cCP%ydcZcL38B_O#w3o2grrRy|Fl^IdzfxMM%s6XB-O+jcQLKZhMf^Bik^T-G2S=4eN zdzfn9as=u$6d1B0W*UzlhUj!x{VS2@uYqGTc#T6!3VUGl!Fmv<4k=+5e=s675^(L@ zT(nwKWVjtm&lu=_S=UftK9RQ&e*uW#zD7suBgzgc7 zU6l7u=KH5QDk6?Ij6_MV?=eeCEwb5x$V8|_q-)4)>pmh+RMzvfsb{ZI6Z z`b=drR2CvUH2b_)Jn->oUR?R4Nu~cyYjL=s^^rACQ)GcixSQ-(0uTv5jcGL+eyw@? zLj4TcZ3;o+x_O_}Pfe-2Lt1JNUO2i>p18C;K* zM#&$2-G9soVkjU40&-wMYEVkHCKanRN51`r@WcnsZuOyV%ML?t(T20;hE=!^GA9O& z{B^LpTgzWnW8XoJH6`|tmpf5gcs+aN+*rNw)g8s;#B`La_jFcKU1;V4RI7DF zw$vt4{}@0}=WAEYHBB1ke_rvTXcWV^n(MRt#)tehUA)Ra;QbYtN!iLns|#ADeb7Jv zWhnvr{uMVI&VAOoDroJv@0Vhd^GqkPjjUv>(DXfeo@s&De5UhATu>Z1bb5Z>6Yx z4rE@rBh4Pn2aTVgE%x3ms`OjIi$P2%HT?Wjr`pG$Lqii4{P~@i^=t(`UFCF> z;iLEU)u*pYohr^A>b|$MO5(xtE^f|*yqUH8?O&17pw?Dcvz`|fm^{q zQsu~Xr~qLLZzAQNd`Rk_TTVhwtG0Vby=GJXdWQs^xrTN6*fTyXr^Qd>4py}6v~Y)g zvh9#Rhtg001_dGN>7X#K5qp=!uTV}j*Zo686(TLfFX)aAP)~j~p}n zxk>b_2R7=oSXQjdEPDYDh&{jDHV|)Q#9{uRzLO9>A1!5$_p$b~!&pnS-kItAFp+18 z!L0wKqcmrwJSi@!PB5YBZgd<`Y~<>#kleuA@U|iRFIAULbxt1@6)tUal-XwO;}t^NG@C3n=mB5U1oQzhxvDAVn!n7TyVat4_{$T(0; zo5FK)@x47oQ+N!{WYIY{+4-ZP_;G!6`*2)e>Z_KMciTSjjI=9netR)_YOD@zgW?<1s5Ly0ad8;LWL>g%WG-ao%w01r#k5&xnASn*watAHLKogujqPix+me(|z`_oT`j`4_3x>-tPX-=UWoPPM90_aI z?h^+TtwofQg}%`pH(r_J*3FgZBaIMUW*MrKb0n)ehla!XJ}&yGbIVyK8c?kCem758 z6jT=>ckHux?}HKkLawG#0R;myy4i|&O=rIR?z!~4gIV1t!sK5wX1DQ)tJ!%+edwfFPbsx_#*gF8BgdkFxG)WGTbYIFZrFC8 zZ*=}jj#ri)Q@h9siegXw`f8w%D9mkA3u(6$RCC6de9}1a=xxfF2f3?O1MdtpD9_*` zJYo;J5WESPt{E}=UBAq~ziAR^b&@vTVW)ZBS7QN<@8@uQK)5_KjhH<8tV^&%Pfo|0 z-p}#Et=`0not*m$fw|(e&F$^@g;BbKyy8++IBQZdhXEq>P@OC0`$RdxvglQR-vMae z%{yy+L^QN4r%KVJHNvffYlsyx^vf8bBk!FY#OLm%xj{GNfGB_*YFBZrj$9!MI%;!H zTMD2;&*|C5PvAp*;)kZjis5y81VeUtbx(Y?N4|2ab>fwM7ZCeZkfAb zcC!33=8UID$CMePQz|I$0NXu%c4hzRlnBo@dD6I>EyCT_I&FZ<1rXr^dxDLY1ohN+ z^k=S-59u8&QIBkR19zfl7xM-?V7B!;s5aC5Bs;=v^1k+Z2@FM&au^_$3` zd75@YAfSxA*af*k-YMm#&?K3LJ?MHSS*z1h?%dfgO-a=&_y}zQ2eS*48}gC3$!J6g zLl4~&D}=E2y_`DX%9nHHuIv_tC}#Mg9-?YJ2;nMM0@)V&g;A|!MTQ|eEiG4q=fJ1v zrK-M_zRe`2GQu6DFZZx-E{PcE!mJ&SRrlR`d5BzX4e55>c?FfzhQDKhQ(hMyfj&}~w^B}J5CJ5Y25xUSV!A&^ zGV#(2!fuI9BbfNeSISTlb~~rV^HlaeCJsUGjxl|T1_DhI#=;eGx7HltbJ1mO^KxOV zCVr&=f6MS5H8edut2$`|d2421fw0BVePp1XKhjpHFS6wR)fb$t^Ya(ysFGaeuscoM zt$_XM#KhQw#OLTKm;PO&IAZ30S{0GwPPI*0Kg1vC>VY2&exZzCz3JD$55I6^y-?*`(m-#R^{?%xWIp$b z!kM>=xS#$^GuMd!rBD)mf;6gPA{GBwhYIiLQ_lQcA=LQ}3> zc3#OO$H6|r>6r`x{Di#kWNAA)Po+0o$K@qNM_)wT0i;YRrB7Z9t0!r{VbrRz2w1St-eUnCx#p?R5RdpbzcFprC7xgt$O&3){w< z`h}1$T<5Z3KO~_aF3kn=LYyxgG}EpkuAVSAyzh3a3US)nq)%jO(%$<4Jw*UBI?FI6 z3D4AaVw`duB?u#ng3S!?J<1=0l+4u)FDxRwWIq?!F;nfGoWEK>Cb+#Eyy$K(9I49Lk94AN1S?xce$;%^#;I`yE zXEk%r^Zna@2bm_=7UI@HZ28IK>^$YRhZ^4vWR-BVuHWF$p0EVYIqvOu1w+q zZBiz6{CGFTkmbf)*~S(FGx_RVwSu(%Ck?8i@=7BLaOw~0y!6@V+9deqlc>KaLtRpV z`%v5gPX6fI@b4AfF@m*3MM=HB_$LyXuLyq!Enzv>00XrXed6fZnvvjT;31&zfQIXK z-5Wb!TW5E0ka>LVZXi88NUUH_BG%h6Y z(doU})R;sG8K108JAb&D`X?x>ri(T@y7W`Ofd0)d zg9G)$)q`0P=;Uw(?@jCBT)AU;OkF1ZrG1`#9`~VP%O5qx`c0cz&%$^$6BI5CH#FTZ z!|ENE9J|-vuewz0zp^P1?iqmY8&nhwSv8|*It?EAjLYYgsZbV2X?^AOe(_|hyo%%5 zf#?UiNkr%BLqjcJ*F!Nfp=lC{LO9`8U|AnbnWe_zHD-5I7)almKw}y{R zMBc!~in3OG1S##vBhzP_j>bstHtJDTt3UsC9a(^9u#tYFM#3uUYK&<}5<1q)G()S^ zeVj23?Op3DWLoIIFcS?~qb+7O;1yY>CJkG@lnaj_Ae&XbGmGifJL)GF0PSO`Z=1d~ zYlGb}t!DTz;Szs9C*UNlaz;;0PI}0SkHQguR@whL;bvWhKs2zi>GGytYZn`t;D6>0qXVG58}>w+goqN zR#_=4wz~(Ypl$KCFax)GOxa2W^gj#aT8&h*sxCZ4N{y07(mZ2l8PZzVxH zh28&sQ?i06Y4;8nuNNn~)Fxqld|_#I`7p&tqYz8;bo{{19hSrSc-TC^+s8;tWQ&sd zWp_Kxo4Jb-6ckE?qMVEtB0nIC15b4sWV?iL@VjQM7pGn13ZrC6GbM$0#tu|j@|;^4;R(M)HFtj&?e^5TH27NG9L~jq zb5OBMK58v9$t#%A(|+CI3YR7qx~8u_r+?CK9MPOHy)b)w&2)F7;>o|kx1d%c4*Uno zlWO3Fd7?l8zZKl@-c*c~wGocmq`fm+xaiLTqgCrfyMm{m@xIvJ?$C$Ra|cxDUjz#nhnE2~6!CP>0bM^qRQ?q!&pn{50^^CAT3v3JSqd z>$ny@G!vqbpuvKbhD&||vX`~XU937B9Ho72p_?$%%_gJ5&?53mT3*ywad$OWBA?__6+|IQ4&i$e$cR;pa3EA&~=|| zrr9lV4MfUC7A9U)iKY-=^zw@}^5HPsi@i$b6@<^okS|VPM}!;~6>fA+y@vx{Q29n$*iVOIi=yKK9PK3we=8&HSOwN0XMKU>Y`86g4esM#4e zXkG&vR2%29^?R5BC;YiItmEjNzPxjY*}b(F*UGx_nG80mEy}Iek)z=AzH7% z5U-P%BsBMaa6)tZxrART-;MF)FO}dt&s(QfZ*-53Zlyd`AxHNbz0$(qoMq zn!rb^x2>d%v%Qw(TK3J8qIN z7t)V&+kCaHiQ^c!Ii%QplGZs9PVMHe=S^5dqMN;JtM-8K$|a@^8#}KfWa00lR7k4? zdtWM_8q4)9!RPzj*9Y{EXed5_D5o&oXY}L~Z^u~PMaxbMu>ltnqVw+^w-say3HKJt zH;{A*&W}9mZKdLHrw9YIj56L(ePg+~=AGOM9LrqUWORaasz9mgscpXS>{{1#>v(50>2a&14(`jcOO8S{#c+ zDGpJlY{+|4Jf0-62v&T$7Yeqz685g?t-J?_J@IeH`n z_+POoqpk8`J|=Is1Ka2g^TL?JkdLSEn|7?r@t@ewDu& z|3F#uNsPj&17cRQgC@ijc90~&9DBy`8AFomEW4)yZ>zD1x2I8O(jldFx0q@8x%bdj zs0KZ`t)Sw-1MCZGJ_ox(1M3X&hEZ1=rAt~`RT0MtE@=bTy;wAK$HiPIW!?#?B%p5} zxJy^C-u{Ag*s}??V89>qYq2bN;%d9!qH7d zc<^j{=vI|q(u#AGCk;meoe(hfRL%td&}^-^VyLB8ab~LIb=|uo&Y8X2AGFd|O)h3l zq}rLZN|B@_b>xXm*G15)RCwkzu1lyutY7|m3u+fbk8k1KSehs)~(=^0D)E1r@^?{A}};!^2e@;N3*xusy@KBdYs8 z&)5{NYhGZkv2A8a%RdG0WmKF?8gXgP)(B?E9gi*jNFWw}-Nq(OF|lz(oLfHD2s+9(bsw)llR})j(NCOTX zqplQ5FT2ckhY<2I>B5Nf>%{lpuUO2{>OKd{+kyy#SNe=ZsWfG2+O;RY#E%+>pO<8u zdmK%JVL#03JOn}?OX?^tyd@_lg9=ZBYeMqo$%>4k*K5)8#mSQ9A8?&7eUb>CpMhaq zc5$C6l=Le)ck z&5rTGEl<_V=}PBBwo*Oc_D)`9gwL*%_x!~GVKJi~+I}>B(u#z4t=wyEYOncP#b;ml zG1MDrfLaq_zR4D4%dOo{KR<5lUs$5$zByRba<8!UUpbZKGUSX7MJDwEiSj*vEodJ>6_{J`(W^HiYm-_ZGZWI|>DFAPqpLS59D zc7p~K7n!e;4Dw-iW$^^rOBSnN<9*S#u4P_2FUFr=QG}y_Z#7_E*pZ;k-kSHG5PHJ> zSJ5)aA>EE4lgc8N1uE3OlwjqCRqZEbry->W#Utfa$13b5<+bdnbI&aHF=dC zZn()A{U z92ughm_12$dxEY~??>d!9VKb2!|y(wTpkb+-9a5jIDq4r(R8!5}2b?m2( z(<4B2RlYS3SHxjbk+F2HZ(-Fa%POpz|7ggDvwXAEQIP^|dZ(C8n#CHrEzx zKo1<`_bf6jY@XeObEhll(U zZ%YN`yI3Q?ZhLg5C?meeO!9mXnFtZpjf#s|pe-g@O#w5dLnT}_-APr&%yj-c*dVr6 zx_?{_Uj^87IkxoYQpGwA7s@T%aGg3GCR{LG4l%Twlp!5lUQZMjV0F@%xM=-tLBAwT zLKH#WghYCH5zZi)j^e+whoN~hg75Th>s3OcKqdBgZ~|vU=}V27k5eIbBzql{xl2i( z2?C0SJ(zuy2XUo;4NLa7Nl}{%he9quwMp5${PXSqYz{QkoixHv9ObbrMEq#r4y;zR zvY_)aqc7`__lAWtiy9uJQ2+z`RsB~sQATwlTh`sT<1y79vtUC~0BQ52H8Vi^!WqoC$ViYo4Mq-W z6n$RyhOvS2HxMPaJvO}9n5*}>J(^rxqhP$L}y{gkjAX=kRM9i z&k((T#YrTOHX}garEZn2$e;@8BFI)-2JD#UW1p9;;(&TjJMQFHy)+d9(+p+L*&d7Q zV)q_PtXD{}vRJD#u_y4D&wD7##}p!px#kUfOox5?ytSKxutfcJe|wud>uM6C+3kPa za=?nO)poab@RS!e)>5zHlN@0pQFC69XYEcHI~r=Vd)Ff7M^zfU%#5#x=QA^VCwflk8ax#79YX^9yP zI0~6WgC1X=S;yZuUw=3FC7z$>ubL)C!YQ#Z5)2uF7>M@9#>;~OeG4@8weazeFxVPd zg94%5(~XSbxp8z-oG$oU)b|%jldH|5;Rb`s}Od zUA~s4;^zKA2!4E zjGfI0bnlP(L5MY_OKKT3 zau*{rB=(M%Dw;P#{2jug`a-k-0?k_;hfX)cGPM6_XykSY1jZ4Sl0_IO^MpWEJXy<+ zl@@Cb58#M$IXMISCcw~+#3k?qMNrN&?$@8&no=x&e8+N`Bhby)b(U+IfZ5+>$pmmD zcGylg`%`HT|Em9Z*c{d%f40a7@_N$xp!I#1x9e>re~KEY<4abJVOv}HgVk99Z7dgk zhb;~NDoiP$I2vDIhCgHu;s^VU1Mh9GR3fN=9He|AN`!uwXsm(`4`7zDP*MBa87ZH5 zB>Z)dc)^lyCz^=?U7-pPy(^qXs3xqLMl;lK0j z47b4`UQr34|J@>#*mCU?*2k~Fth0YBledTjDYMM%tpV~1Z7|0+CL*dny7c4wc1Q=> zf}nJT)-Ib$6C8*q;GH(_yClqAe~@bzU zQoDOsxUD-$f!Wx1Kp{n{@Nh|RHf9tEQX3QJ^gwFD-p*A$1cx!9%^~HB3s7M9U<^gryZN_!ZhW|oDEDFwq6uWOPSlb1me^icEO05p%cXvbXY%q*&$j8}Nqv_(Q44&;) z_Ks;GTowfWBUTU1UBTD--Us$*H|#k+n8+U=b597YqMxJ$GAS9Tk~4Q#>rs1Fh8Qt2 zzEKSg)g@Du~AdRmv$6g)kG(kF=LiY9TetLZYvs)IdRiE$zZ zSRe{28e|8Tqj$b$A0HJh1xL5-@kR%BMk|Mj_Ucph**V=@`hMaIS@#=6{HSP*YDQ73 zW*1bL;6qr*Pqzx9;T-nVo@=8)gqirsBZ_n9yOXg(T9aF!*`y2mz3chWqc83rZ0?TJ z-R9n(kx$5}W)$&GN~f!*J=_-sFCF>oSVX$UFjId&S5Fq$LN-Jxh-1D%b)eiYCTl%o zy<{ElMr__UpEy7&/tutorials/speaker_recognition/``. + +There are tutorials for peformming inference using :ref:`MarbleNet_model` and :ref:`SpeakerNet_model`, +and how one can get ASR transcriptions combined with Speaker labels along with voice activity time stamps with NeMo asr collections. + +Most of the tutorials can be run on Google Colab by specifying the link to the notebooks' GitHub pages on Colab. + +If you are looking for information about a particular model used for speaker diarization inference, or would like to find out more about the model +architectures available in the `nemo_asr` collection, check out the :doc:`Models <./models>` page. + +Documentation on dataset preprocessing can be found on the :doc:`Datasets <./datasets>` page. +NeMo includes preprocessing scripts for several common ASR datasets, and this page contains instructions on running +those scripts. +It also includes guidance for creating your own NeMo-compatible dataset, if you have your own data. + +Information about how to load model checkpoints (either local files or pretrained ones from NGC), perform inference, as well as a list +of the checkpoints available on NGC are located on the :doc:`Checkpoints <./results>` page. + +Documentation for configuration files specific to the ``nemo_asr`` models can be found on the +:doc:`Configuration Files <./configs>` page. diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/models.rst b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/models.rst new file mode 100644 index 0000000000..7a543e92f4 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/models.rst @@ -0,0 +1,5 @@ +Models +====== + + +Currenlty NeMo's Speaker Diarization pipeline uses `MarbleNet <../speech_classification/models.html#marblenet-vad>`__ model for Voice Activity Detection (VAD) and `SpeakerNet <../speaker_recognition/models.html#speakernet>`__ model for Speaker Embedding Extraction. \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/results.rst b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/results.rst new file mode 100644 index 0000000000..c20be139f7 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_diarization/results.rst @@ -0,0 +1,76 @@ +Checkpoints +=========== + +There are two main ways to load pretrained checkpoints in NeMo as introduced in `loading ASR checkpoints <../results.html#checkpoints>`__. +In speaker diarization, the diarizer loads checkpoints that are passed through the config file. For example: + +Loading Local Checkpoints +--------------------------- + +Load VAD model + +.. code-block:: bash + + pretrained_vad_model='/path/to/vad_marblenet.nemo' # local .nemo or pretrained vad model name + ... + # pass with hydra config + config.diarizer.vad.model_path=pretrained_vad_model + + +Load Speaker Embedding model + +.. code-block:: bash + + pretrained_speaker_model='/path/to/speakerdiarization_speakernet.nemo' # local .nemo or pretrained speakernet model name + ... + # pass with hydra config + config.diarizer.speaker_embeddings.model_path=pretrained_speaker_model + + +NeMo will automatically save checkpoints of a model you are training in a `.nemo` format. +You can also manually save your models at any point using :code:`model.save_to(.nemo)`. + + +Inference +---------- + +.. note:: + For details and deep understanding, please refer to ``/tutorials/speaker_recognition/Speaker_Diarization_Inference.ipynb``. + +Check out :doc:`Datasets <./datasets>` for preparing audio files and optional label files. + +Run and evaluate speaker diarizer with below command: + +.. code-block:: bash + + # Have a look at the instruction inside the script and pass the arguments you might need. + python /examples/speaker_recognition/speaker_diarize.py + + +NGC Pretrained Checkpoints +---------------------------- + +The ASR collection has checkpoints of several models trained on various datasets for a variety of tasks. +These checkpoints are obtainable via NGC `NeMo Automatic Speech Recognition collection `_. +The model cards on NGC contain more information about each of the checkpoints available. + +In general, you can load models with model name in the following format, + +.. code-block:: python + + pretrained_vad_model='vad_telephony_marblenet' + pretrained_speaker_model='speakerdiarization_speakernet' + ... + config.diarizer.vad.model_path=retrained_vad_model \ + config.diarizer.speaker_embeddings.model_path=pretrained_speaker_model + +where the model name is the value under "Model Name" entry in the tables below. + +Models for Speaker Diarization +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. csv-table:: + :file: data/diarization_results.csv + :align: left + :widths: 30, 30, 40 + :header-rows: 1 diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/configs.rst b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/configs.rst new file mode 100644 index 0000000000..7b84f474e2 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/configs.rst @@ -0,0 +1,117 @@ +NeMo ASR Configuration Files +============================ + +This page covers NeMo configuration file setup that is specific to speaker recognition models. +For general information about how to set up and run experiments that is common to all NeMo models (e.g. +experiment manager and PyTorch Lightning trainer parameters), see the :doc:`../../core/core` page. + +The model section of NeMo ASR configuration files will generally require information about the dataset(s) being +used, the preprocessor for audio files, parameters for any augmentation being performed, as well as the +model architecture specification. +The sections on this page cover each of these in more detail. + +Example configuration files for all of the Speaker related scripts can be found in the +config directory of the examples ``{NEMO_ROOT/examples/speaker_recognition/conf}``. + + +Dataset Configuration +--------------------- + +Training, validation, and test parameters are specified using the ``train_ds``, ``validation_ds``, and +``test_ds`` sections of your configuration file, respectively. +Depending on the task, you may have arguments specifying the sample rate of your audio files, max time length to consider for each audio file , whether or not to shuffle the dataset, and so on. +You may also decide to leave fields such as the ``manifest_filepath`` blank, to be specified via the command line +at runtime. + +Any initialization parameters that are accepted for the Dataset class used in your experiment +can be set in the config file. + +An example SpeakerNet train and validation configuration could look like: + +.. code-block:: yaml + + model: + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: None # finds labels based on manifest file + batch_size: 32 + trim_silence: False + time_length: 8 + shuffle: True + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: None # Keep None, to match with labels extracted during training + batch_size: 32 + shuffle: False # No need to shuffle the validation data + + +If you would like to use tarred dataset, have a look at `Datasets Configuration <../configs.html#dataset-configuration>`__. + + +Preprocessor Configuration +-------------------------- +Preprocessor helps to compute MFCC or mel spectrogram features that are given as inputs to model. +For details on how to write this section, refer to `Preprocessor Configuration <../configs.html#preprocessor-configuration>`__ + + +Augmentation Configurations +--------------------------- + +For SpeakerNet training we use on-the fly augmentations with MUSAN and RIR impulses using ``noise`` augmentor section + +The following example sets up musan augmentation with audio files taken from manifest path and +minimum and maximum SNR specified with min_snr and max_snr respectively. This section can be added to +``train_ds`` part in model + +.. code-block:: yaml + + model: + ... + train_ds: + ... + augmentor: + noise: + manifest_path: /path/to/musan/manifest_file + prob: 0.2 # probability to augment the incoming batch audio with augmentor data + min_snr_db: 5 + max_snr_db: 15 + + +See the :class:`nemo.collections.asr.parts.preprocessing.perturb.AudioAugmentor` API section for more details. + + +Model Architecture Configurations +--------------------------------- + +Each configuration file should describe the model architecture being used for the experiment. +Models in the NeMo ASR collection need a ``encoder`` section and a ``decoder`` section, with the ``_target_`` field +specifying the module to use for each. + +The following sections go into more detail about the specific configurations of each model architecture. + +For more information about the SpeakerNet Encoder models, see the :doc:`Models <./models>` page and at `Jasper and QuartzNet <../configs.html#jasper-and-quartznet>`__ + +Decoder Configurations +------------------------ + +After features have been computed from speakernet encoder, we pass these features to the decoder to compute embeddings and then to compute log probabilities +for training models. + +.. code-block:: yaml + + model: + ... + decoder: + _target_: nemo.collections.asr.modules.SpeakerDecoder + feat_in: *enc_final + num_classes: 7205 # Total number of classes in training manifest file + pool_mode: xvector # xvector for variance and mean bases statistics pooling + emb_sizes: 256 # number of inermediate emb layers. can be comma separated for additional layers like 512,512 + angular: true # if true then loss will be changed to angular softmax loss and consider scale and margin from loss section else train with cross-entrophy loss + + loss: + scale: 30 + margin 0.2 \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/data/speaker_results.csv b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/data/speaker_results.csv new file mode 100644 index 0000000000..450c0a1da2 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/data/speaker_results.csv @@ -0,0 +1,4 @@ +Model Name,Model Base Class,Model Card +speakerverification_speakernet,EncDecSpeakerLabelModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerverification_speakernet" +speakerrecognition_speakernet,EncDecSpeakerLabelModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerrecognition_speakernet" +speakerdiarization_speakernet,EncDecSpeakerLabelModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerdiarization_speakernet" diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/datasets.rst b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/datasets.rst new file mode 100644 index 0000000000..7439d7263c --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/datasets.rst @@ -0,0 +1,57 @@ +Datasets +======== + +.. _HI-MIA: + +HI-MIA +-------- + +Run the script to download and process ``hi-mia`` dataset in order to generate files in the supported format of ``nemo_asr``. You should set the data folder of +hi-mia using ``--data_root``. These scripts are present in ``/scripts`` + +.. code-block:: bash + + python get_hi-mia_data.py --data_root= + +After download and conversion, your `data` folder should contain directories with following set of files as: + +* `data//train.json` +* `data//dev.json` +* `data//{set}_all.json` +* `data//utt2spk` + + +All-other Datasets +------------------ + +These methods can be applied to any dataset to get similar training manifest files. + +First we prepare scp file(s) containing absolute paths to all the wav files required for each of the train, dev, and test set. This can be easily prepared by using ``find`` bash command as follows: + +.. code-block:: bash + + !find {data_dir}/{train_dir} -iname "*.wav" > data/train_all.scp + !head -n 3 data/train_all.scp + + +Since we created the scp file for the train, we use `scp_to_manifest.py` to convert this scp file to a manifest file and then optionally split the files to train \& dev for evaluating the models while training by using the --split flag. +We wont be needing the --split option for the test folder. Accordingly please mention the id number, which is the field num separated by / to be considered as the speaker label. +After the download and conversion, your data folder should contain directories with manifest files as: + +* `data//train.json` +* `data//dev.json` +* `data//train_all.json` + +Each line in the manifest file describes a training sample - audio_filepath contains the path to the wav file, duration it's duration in seconds, and label is the speaker class label: + +.. code-block:: json + + {"audio_filepath": "/audio_file.wav", "duration": 3.9, "label": "speaker_id"} + + +Tarred Datasets +--------------- + +Similarly to ASR, you can tar your audio files and use ASR Dataset class ``TarredAudioToSpeechLabelDataset`` (corresponding to the ``AudioToSpeechLabelDataset``) for this case. + +If you want to use tarred dataset, have a look at `ASR Tarred Datasets <../datasets.html#tarred-datasets>`__. \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/images/ICASPP_SpeakerNet.png b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/images/ICASPP_SpeakerNet.png new file mode 100644 index 0000000000000000000000000000000000000000..ecdccccb69ba94f2af56a627113a11ff0768a43e GIT binary patch literal 211990 zcmZ_01wd45*ES5}AR*En(%qnxbcY}y-3`*+El76@NJ%$Hr*tVD(kb2D{~q;tJkS4r zxMB9}xpVcp);$Kx$%;Qkz(at7f_f?`A)){U1r`N<%i$gaPY6X$(SR4QiJ*)i6jVjz zlUqF);61T{gn|qdlp7@!l#f3Y)HU$PXA=s_kr@hVTNet7I|&L3+a|R^o(K5A-$+%` zSVjhl7PyClf`LYZdIa1-1OK3)@t`210e4VR(D;Au6`-kqw*f&x1-^v>|8AoWyh47m zfL}|$wQWzX%xOZu|~H*gPm%tT83 zvx$Q_FR7}G9I>#qoe}X1#^;R8qxVV^@S(#W_8GsfH_O4bAdM*rB_GG^X`DYvvBYOk8w>A!Mt*wY5K zJ38=^l0qi>^Yd$+4sVVBoyp4n_qKoyGC@9JVqs)v`oFpvxxD>IIpbTiDvd z+U|{wo`Df$lK*}5zpdrox)@ogioCTnva*M)i}Hvi#xwq|MMxpdp-mnrayAThj3mgc?$(403|6RsOSQ{n+EHl*oD>I8U-Tp z9Tq}veuse>O)Dx2Ruq6KrXG5aKzlBB!Yd~ajn7`5kr6~Hin>J%M?LBe-wXp6xZs#UT_B5QO-A@AsQ=?K1u2Fc ziv8a`B|s$5^=6FROYr1*+MW9b<~FyxjO00;WG8lFZEPe^Xot?AEy1Vr*cFHBivRz39@?mdxebGgFIc4s6`yxA)bYrSHKK#6w`e z(14ZxhUSsl`JH^+dD5Y~>8O9Y1LL%{!huId3cW3u_~9I>MyN0Bm-pea>Uq=%D08HVd;5m=on%=ypW-D1ah22XHqBC@I~8|MC{-@)Vkn=K8C$9m&57 z{j*{L&=m&QG52O}{T#Tk1T^empU#{PE3~f=k6XCc=;gPxM5Br_B6*BVG9!C~u`}CYHthhvjeD7z=ik9qhf^5AVEHML|1IHaV zSh=wO!;iXffK67yUxPuxZLRKThif6)q3UO$CR$Ar;H>D$Ri zx4x@=DCU9HFBvyi2~{}4M0$S6Q)kl)D2%aISd6(A&v|H{z3zxVBOA%JrTJWp{gy!? zV#@j}XED7bsa%;l{B1cs5^2@t0?zacJcjKL-O=n+LpfWv-CoXcBwqwgOSi2K6a^DK z*&jGNiMpdSiVDv<^*Z*!MhtHdS0`*SkF-d8guGOp@EVbIMwOqhw61={r&ae*yKUad zQkK>HPONZa^4*nF@+98z^Z>171OH-PkG$xt#X|+LeLP8w+n(epx(T6ZPNkm9ZB3zA zg%@_NxwE!NYXs&vpV1i$9{kj$KVt9OjE=lFhX)Jb&z1VMvS*k%{q}c7s~bwTQ63?c z+^FA>&+`q*dYW3l=?3E+=qo%nBrJHFc(fYA>2dtZ)AQ5zSwV%{(8v1h1{U7VvYyQ{ zM~K_MVmN(sQ*O2clfwx&ihP!GktUD4Xfod6iODlwh48Hq_Wq`Ozr9Nux1Yw0 zrFb#kM<>px1AAzqN9n1Q-tpXO(($Zh5L0Z76g){kraZI9xy2)M7P}@aiXqLMV$|24 z>`tkKaT|h~yl*{`*u>ez(rA~>2Ej(Sv6($;Rd-p+q(P${5(69`dnTvJE#5H6`vj4S zV89-?h+VPc-j8_QG3(T8QUh9EAw*;QqnPVm!O#lzc}#aOYxw zjcUYT*DXdtuGH|z%~i0m8w5S!{rVztuvsXm1JTIuCzV3t z6)o#^RsB3#Pd_KD4`Aa_oBPAj&*LDQOJf#jR$PHf-OXaHsQ5*enY}I8q_r748oS2Nsl5q;ZqWn&qqC~tdC*ywM<&*xJD z8xz;Q{M>3sPLNL2n2sW#T0tbMT4H;Gy>Ugjm`4P4q*B}82x(P_p!mLoUcFR7nb325 z8qmmh)T$qeP-4R~FA0*5)XAO~%A@VXt!xELx>pE%y~(Onk0xWyLV(vt1|HGkls zQS~lplrhi!vvCfZ`ZL_w5znMV_vfv46}zKnQ!)5-Banw%|ijhcT22w#8$tMiUL$lkA0_tRqZ(=-q!}8saj* zA@$MG)R6h|pan4+*rn2lYaD+7;)(trV?y+paTY(Hcs-O1QoRi|7#CmbRzt48ot-da5@kApc*Z-p1L`hc^H>Xgqko$u2$BA!? z^Z+Zna8Dr_Wz5X;N-YCLk)WC-3F+2i&&+9u%KqnN=9Pz>cHByI?44P{aj15uVlk>t z65EcJu6I^Ae2JjXPq# zrd7<(=i0#co^D;dZ{a~0}69ck3!+8YwN^^l0kX440c3X4Kl%IbTP zwkUMwiSN-;6c?RVFY8vfR5EgXiV(K^o^Dz(sa901(J5#y+xKAz(^1OIn`t&4eH?i_ z8Vi2vRtGapB8IcM+6f_LEa}^}$Q4P8NoS>KvB?_Fq*)jEY<5?gfgN62 zgLMa@ABu_CT+@H2Rz6?^;S8Th_JHIVPsz4cc$|A45kV3xSQtsWRAJI43GNaP1yw5B zINCnBgkIsZW)9`{U5*d=`D{67(N8;Gzx#|n5F`0|vg@XQBS@9PBPVNqttUjZyZ5zJ z<;&9OH;aX|QX%B-6djpw`Z#kUyn=*;J-j%jD~!!QB$>wQ3r9TbW5KK-KIuWtLziqy0aji!vH5yMD zgD}2Qz{9*Z`l+p~`gq@A0I4#js|>@o$hIj>s3HcgIVD<&n8KJkg(W?Wh!dG)oqDh) zsbmBc?iPEOPK>d=N)=#WqE^owi|^(%vX}q zaYQi#+v#)oXk4bF=#DH(T&*SQ3x*_Q2#=47NTUOtp7mEUf6#M^GasbzQY8;&LL(#O zAR4Gw4{&5p-R!r;BNM6Avo-w8)PDgnFFHTXizp3U&wrz9aj@o1o`^VGWf=H2)koE; z$-^1(MBCS|mQq8B+kO@J@8WXsiRoEvB92Qv6zYSJG5wJ#6T$ryVg zj?33Bsr6A2li{Q!EhLlsbw1(?^F`T15{l@-jAUDGRS!>sSAh0b4Y|eZz8dscoqAhh zC-18L1DkJJUsmozEVFNZ~(nQFTQ%-r+^t-r;wqX(v(;7m{!yH`FhNzK-%j z>ZL8OGm91tydOP%*khW0H0l$q}jlYfHnY|#Hc7(h=Z<(iM`O) zsNeA*O5tc7$&*Olk3QIx_1C>7bdDw;5lW@79li=bhCRL>f0txX^D2s(xweL`EdHt3 z1JZVWCx8Fz0BZ8Wd@_!^M#VPX9}?`xD1_-z5eHc5lj0_wfdL}*(dCWZb7)r1DO^G! z)%3?vr5jOf^iF+2Rt8#gB}!_8Zym69oV3SVWIky;Fpj9x=noprFVrQJSFfl#UdOz- zR1LsOAQyk9U7z>95{ky?`Pf0`_T><=R3xmFu|$|IN}Cw)S^Ianaw=)g6Z8=KT#i$9m@Ck@}q4TG;2q1RA>}# zmw(>DOrhQ8jGBYZV!jbp>kVR+zWpH4Nm#*%CQYoCO~8P6ldcZQqC;_5_k(WCRyuSd z-kKAA*Ek|Oo@svZIaW))1x$Ct|Gg3WpfCcPKfEjPiR^ZH(v5BhL4@Gld&> z=v|$)gN*zL`hp@5s$E@%A`#JQUdW18p^65R1^ClQNf%O#2H#EVSEI^jsj|cPdNyYD zhEj;XS77Y5{ctOy99>2VKSYOsFCU{ybu>g8?fCwzsIVPcWjxbKw0>z&aw{~Qu+7hL z<8%bOBLAqH909Rn7iYW~?V|z$Dw&pq)D)_JSoRCC`n$$Ut!Vbg2K7Va?cspUSq~@I zqn)IQpx58+p|jwt1^ny~#Lt$NSFYzg6iy5rZERyE?7e(WiEM!jke%IM|Wo} z6$LMCFTyEAZ)zf{MkJllo(%QL@Nrh|Di_;Ci`2mXtQ3?61Tnr|ur_z(OOL?h2w|S& zIPMkI=%Y90GuZB;+7Y*>%1CzZ?g7s+{sKRLVim-?$K57tmw#i7f1za1LI4T|_1%;) z_CNFejxO60fN5)ZlKzt@|C2Psp#X>rxL3?R`9I$DpTTPYr&_W8h2THo|8MCK@Bt`A zF2Md_fc&4){s4l&O*sVszA+&G_)m=c4+aB*#BpcC(VxO2)wOH8%_wjo7!1l6Vn5G?DSMuMJ>Lqy8Vbb6aV-bt6za=x^v+U6EZ) ztgp()fuDZq_k;>s1C)gleE`0)Vvg}JIkmiKh+cpB_oma)d)@R;PXg>*eZ@139iC`T z?Lwyn9L%=D>UiJ4M2=3&>AeSoB7F#wqYpq=vgNXADZFr3`?`NyJ2*s;V4@O2SdLx1 zuLSXnJ2bJOP5(Qp`)@V3`T4B*sBh4tQFBlTEb!YoGz=gWgi9XZurNfzV5!tX^&1ox zEakl82^Y^c1pfekd{K}8z@PRIjPL5Py{o{libt(*7u*#@rhHIP0(z@Fe`LK9F#;+) z|LP?iF+@m*jbV=!y7boux?byFO%~L7aIr?ol^hW6#1%B7vx9HF6;VdBQIKPM8-Hv_3q<>NI0)XUZJ$p-JWQbSu3(XxL%QyQ_EFZ>(}p zzi3*Ge9C{2JM_xNhPiWF^>$3DOq=j*XQ~HNMblTS(TVGEo)jA^&e9he0%RqB7gKH7 zpY;bT3gXuBR36vS<XLm_<%%`y z(dd+mhs(yt6?f-#;QOq+4~O_4G8}g%>pZS1za{~_$E46I_IE~eo>;Zs zIie9g%Yt$myZ+z=& zay==xT@_0yXz%E#wkBtpGX7;)4tYWP6}65#$`;=%MO#HO?=BX-h2s`t8l3h*x#%Wg)p#LK<-_;sA&-G@uYl33xo@5-}&jQ>b*!TYKMwng{SWCb{ zvwJqMM@2sueKJle2w`1Fd}}Ocg)qq8J$~8_Y^e;BTAMm|C|D#XXvh*529Wn&I^WNq zpYOrkE`53Dy5~1MEJsL~2?(z1e#d)=R7I(P&>V@`*|7jNI8KYU`=6MG2|uTY_*7pX z%BZNQjO<`SP8$LHs36HK&P1{mG|N|{%gmTzP{PMpVfgNV=v_kjmbKpFUC9Ua#)w3d zF4HGN)G~t_zL%*AZCNUg1W?tgvCIM#+mQE|kU3F?!E2oBz`iOB`_Sgx51LVk-T=xf z+Uj+8N(8;=e!#0)XG4;Ki|KIm8H0wJ>m0Bohi0Iytu52(^33}ovR1#4TASegrCp}_ zE5gV#A6UbZm;*33v{bARg^ zhQlHh_=pJ5O90sWrm!)XoE=}P7*yx+gcBOVj>{L&@KucBCQb(nB@fooaxH*lob9}5 zG_(DTVT-920qFP=3&_s6RWg?Hnm^bNfn)r?(d0g>Iq#~`~Vz8uegSFJvNtiP*J*cJewc**E(DYAJur8c?P_-d>(8)UKpW0tp-D{~zE@hK<{u}if)<5x zaPs48hYZKOKQiB?kJrhr?wCR+5?mEPx)zzG9wKjgw16Nv6qwMYCN>C11iat@d&|?R zHU05-l%*+@p;D$ztx;3#)K2TogR$s}W*gln;*&tygbv8lZAjZ%Uu=c>6k||)U=#WK zMq38QZd~shQDh=M0!&)P-brzf&>(B5V2O^O0nuY$nG|-tAA`w=%a%~fzakVL6?9I| z>*pWc_qhlw#K4v#SuibEKkzkeyD+sGYG3+iD|8aZalrzLtAc`(S0L8Q>h9YKD|vT) zwilWVal^hA;03Q6%ckq?A_*Cpi66bZ?&pJ(?_nT94O~m%VC7;}-!JcA4GtEX;hsF{ zNMbQKygeCm&(61s4el|Sd;b9)*uBdKAJuS9=|m|X2up^+NJM3w=WT);gMSbo5piKZE*ciiLLDdJ(eqTf_tS!LcRAzOEBI5IIQ790nH`0 zN+102>mY%RnC?iHC=RC$@yg1IT)myqt7KLfUbj=5$<+0KR$C1wB_%zCxW`Ei>n^|% z+Ya$wxRey{qYPf5`@J;YAo#f{F%TfUWEQfbATJT!-RX+Wh2}=(pgE6|uhW%APcTWn zKDTNjK-LPTg}!Rt8(?Zh9ADR%l*tK#fq{7Q+cdy~-+Bht^Py3@fnM`nk!yQh8TFzz zqLPW}-rru$(AbX*b)x&Ae$fzZ>*y73Jyg#RTbEW?XaCNp*)NH`6eBp)_J_$@u z2ZL`vwgK*QxYgSmk&qk1e5&;9=FHCbicGfzu0lC#%^ITya45vki)1JS1Yv!#R0fxa z%YYtq?bR-p5=P!ENZBPMNkyk#l_vo8?MPtQ&PeuHX%b&R%5Dx?cXW=|5}QrZJ+Ulj zKGS4HbA|Dv*n9$4D92oO;MG5ds6lUd+vU;f(IDH9X_o}x4a!YNvgR5t6?x-H`Kq-r zfMGM3!KO3{IBvVrpnGxMtK;>$y_#8Kr1ket@!!`jdd)sy^jQ_k%>xXQtFxf zGzXHiG5Synr`+Rfw%`VD?8iX?1#N0kXOtl z$m;^1`JpWxCHut~@S`0vfqEi|#3P^aq7V@wvpl-r%#OS@_tlh%M(LG2ijHmnBtf}B z3=!m|9I#XZ{Z>v4rY1eDD8jc0UVz~qVSw&-;GQmXZmfY)stozzB_h*S=zOH2=gmd`&Z>*sRcfJR0}l30yGN)uO~AA6UVkpl{B zlt&FjfdZNEfC3N0aRKoMqVMUc10vpYxD`zA!=*3RWh#7RJTQ(geY51uFvJR0&SC6@ zAQZHL_QkhuLXG4-Pr7ftXt~)JR#yjviRPLFV3}JkK{UW5X~+S(9DLyli8dH;cLZTk zKvWcIX9abUpx*uO{fL!B84&5fT53Co=H7Vj0x&xT2YwM>ue@ zuOMP9Dj;w_bO#uha@P~%5&{}O5-mP(JSn62AUwd1rT~DQO-5AIu<&&WZRL6$^gYJh z&ilJFeVh$L=+si8V)C@A9?$5p0r5a72X9Z6MIIj7q^o%c4M|7|LO7lg1+gU(KD)9Y zJ^_JeHrVv4K9xoTLJ`Bv(E?TwZ47SQZy3<4I%v81rdN-_y(2t8p4Y1?2w9xABw#Xr z2Tn@}5Eluf6d?&(5GLm0!hOwq7Kddzn8cDVm+5x_Cob8P&}LaX|3k4RHICGI(X)2U ztN5qNpdAuIMB$~os3fo=1Ypc)A&{Xn+MykYLMjeo3`g=_6G5^9ZAON!KM6Gu!6}Bg zbTG<6tY$6B(h+X0?hk3Ge5=fAY18`=M*53|mKVBpUimWLLqvBQ+J3_%@;D+f!0Jao~w)wp( zZ|rJgDE;7b#Lt{T0Hg*6Le_Uqdo!6p`uY=?h^I?b+JaHJl^K7+?n??r@~wtt%37ZG zKH5!Hr4<30%+@Qwf<;KE(;r>&^lte5^wqx?+{O?8*>V;ONalM2o?svDHy*gnlL8*Q zfgw}iUTUdF#>8emaMtEfGo+Oyz;=9D#ix|J{BF7L z=$5Z-z{3`){TW;f7SYdn3H`Wl#dL=TGDvb|o241i-XSDPN z!JIM02*)p5l~l%5e|$lE!)Ci=!L8h*EmRqgOn9Z)yFr#4c8O18wBNyj){E~Oj83Hw z1oSRjIf-&*+RaUl`g9Pp@$;%ht2>`DAE`E1p;xN}>v$Z&KpboBxAZZ?Q<@lb!pUd> zRN(Lv9~Y^vB?&2VxYmx2&``G|HLkJUjscI$Hmuay+J`gUw=K#rVg&`^a3*m{WrJy9 z3S*DTquCg`okIuh$c}8!Ke) z!qiU?Y0a&27i4f7u~_RYQ;JKs`!(|#@yL>Uo(~vmnNb)k>kPO|GUUv})iPG^r&Gi~ z>yHu8itp#LtTz!kgr{dV#${H~&SsFIFS5j%#-V6@QXm6r3fN!b#! z1cnGLrs}LnROp^K6HvAJl$eDDz1Q8D%IW3^+0QfC$3i}QqQ^qPmJsOz9LbNbDS<$; zE_{Em00dJD-1V85kwbORta$*0x1uLInRP1zm-R6D*ZT{a&Sv zDbU$ExJDsSz*~0h1i*G(DTqJrivBusycUl)`#YWWF@ZbXo{)Y>XZWVNK>;1iYLM0`wDtinjN9O=|CF!fQjKx6%i#y=l&)ECZ426D66lwpgVu)@ zM%R^Ks$H`rir>sdI}`%}@r7tItC7u${~lWml~ImDXY46g?IJp0D*7nus|Z z%NmKFG@*Dud$VvS*UCCNW=}wRKd9z$JtXZ?FgU6hkg0evK8{!9 zQTV<<#-w|+(iIdI)|nyr&d&Us+aJMVB0d}zMqb_=$#&VT=xGwbi-iNM+*f5e6mRnK zb6vl$m3X3gB&p)Y7D+hCp;9!g=G(Q`@zK$G7%x2@Ruiw1&eolio^Q(pp3$EYg6!gQ zwf1cAefRh|*y9JIe3IJ$7$UoDzit&tMD)8l0V>+>{3^=F`DdgIiN2KD0e`PFE2wuH z-WD8Mz%E^=NNe&04mirpNip1D@N(RG299IVwrGsK7_i)q0|7H|<`4B11%~gzgWz15 zl-x4FlJK$qlNbY`AP8MoVgZzGAdc3XCJ>@DE&tZifBtJTmByM#hVos*rS1sIp|=x4>grsywqAaCXzd~ZQt^f=|($8q3bWn*JGxJfT}?qQ6hQ9&Xs zJqVjsaW&sldMP$NH- z${qefTA5gxhGa!HRS1YqtnA6pQEIHF;cVBghuPFnX%p_wVY^#6h3J^VzUW`<_W&pO zDlEQ`HXSX$Y;ajD<;A(rC?U&77mr*%nDEv+S;S}FUxgQQ-kom3q{C+DDnO^IFyC>K zzACx9Q7d%?oVuI~2neeKngEm)fgeZwAA5%eX9}wEW`Ammp~(XTj!Y;nH#>bUhg&^q zA#S&a@|xZ|Uy~|!RXnO}*sZpmL#=McKI2Mo&`F8BJ({G;t`=HrA$j;>{cs=LH9KW+ zdSgx2WLT#hHki((aZxl|&c{7J{WKDhCj z5x!*LdC#`kQ|y|B=49NJ~^^kN?m{)pVK zn;Nj&=_bU_ndS5XD2)Z(d=dcvgqL$+0`xM0li+cXl&Z}f`*KH_PM&qi>9$#3Fua{d ztjO-&f2;%8K?>k#p(!y1B@0oFR2y>3Km!rLx}N^(;V?Y-E=j zc1nd^TsYY+rlBhE3KR=sPP!S3GZi`mY`C5SlLA(V5HmnJSklGl7XP|8tU4!1a)me> zZjApyflj1zW-Y#UL15?HQn53c`4!FyrLospi8EuA!f%fNBsGw{CJ{Y~U(;!86sGn5 znhuLW-M{gmWtI$WEbkYL0%U1u`C6@|AH6zscI+_?*0YQnzAj}(W|kmy;Ok@haZG)) zfQZ*8u(sP4#hh&q;Uxf+hdO8?0Y!pe#%Bo^q|ZRQ1wh)TVNhm()8Tpl5gn4Fg|t1T zR5?8-G$|g!nY@Y>7PPN z;`iqBx;+vnr=n_fB8~V{WM0Ao?afq=ow%@qo&w&}sE7+5$TGa(PMaHwI(}T_AJ#h~ zJe^rB?ZD)|5@ZofFZ^430FO)c{a$)e4*75KenJIteR6m%tH3XT%2lm_%S@ zjA{`St&AdfluwW54_Y(yswg?#4w)yDO2k4i?HS)02|%z|jG(sbvz_Py0{mLj5zzaS zHPF*e**{XQEi7>Vy7jo9PQJy%b$8Yz1hi$f>^J@{kn7pI#=IXM-__lN?HAJ}j1B-eJgXj9r$>nl3wMHgS zryqGysHJx({b=8p+ipYGE$f(p@zK!*R zYI4w(yA4+Q?cI#8ShCy^#R{>};Z@Ob0NB1WuQ&EJUoZf_tEn7)0Bee6m<)ZC?MYbG zeD#ZR0aP|eE241#Jb3sB(+2!3orajx2~vI_0>u|9kAzkzAx@W;#j&o(CdU5|=3Ro# zR1e=Sq|f_Uv&sx)_HxW1jasV-LX8>=wQ}g`&_A}B4&Ft9gjAq%ccK{$Ip8nbpxYBc zh$kFNDHHVWpT@wUk}v?Z5q%%?=#OsKg|vN3dwDh-j?dvVfkBY?Pg`OrAWKK{N%=$u&Azc6N8K?0&EE zjquy~Q*Nf#2a-$-7CEG!040Pxa;IRRHVp6jyWR-M;3bi7v-`zLy7#>=L9FFMQ@sXu z?<$*B*Vn$^dDcJq=dZWL@E&IqDmcu#PyqI6skNJ64xlrB$*~a2_5Zs#Ng=qkmd8pb zy!Zqo!|D-k;>!9G`O-WXX@4Owyl7JYV3H7kA4R`jJIXc6dwy=teBvkU*R0`!Em7-z zp*?tSMTxY>@1x%pl4%ieyy@bJ?a-7-CnQ|i!iYWeK=`uOqJuY*Cy3Q(piUsh5BX}A z=|ql|6qFV}7%@7hR#jor!|2ewCHA=is2RyX>fFEV-_O5R6mkU$6ugUoFPrBsdzEme z(K4I&!yjn!k-XgQp6GHn{30s)qZO*b&dk$mx8*@D9QN}Uiuo9p1p^17ZtC3kYSm)A z8Hpk%>?#|z?!L1FP3DPZLL!Fb>RPkhL#wwe7b;<%_l|~!hSR542H(n(DomdhPKAxE z10@=qo6()RI0yKVc)F~I40nqeckZg)$S&3M8&Z$fCG7E16^;~6X)O9qqSpHeK+oi9 zV#1N0{LK*_!M0c_)Mx2Ooe}o3Xot+c6Rk{LYth!`0li9TGb_Oy+U;zs;>(kQk5KWj zLgrIymd{`VNoCX`F`mnK8Sr%T-XWatO=}I4AdFkL_g3H>t=Ugkd76B@p`svPNt;?2 zMu;w&ss56Z-V%x2lHv#Z(F1JBT`m5iN=W1}#*>&9XJmnFx)0k8Jg--}-}m+Pxmfiw zf6Dz-XR8d98A{2BqfL19WOo`cZ4Liq2j-ylfy61b*CHRXbArxf4?loGJ_2HHkG*a$ zrJF~;)z0|`YI{Bw?DA9OgDZJiE51`o$b7wQV)LVuhW^6~WZuW;3@GsqbWh?yPvzQV zVUnF^bHC0UlTlN{F@@s?glxacZV(BbjEGD`bY0#8No}-PHO4D!rg1EEN*ic9FMdi5 z25X9ByvdX0^tfb3BINc@!ik+o2+dt7|05UAh~+8`C3(#i6k=vg@_u^+mxQ(%(E;1- zP1|)ga$ z@3Lt)>r9BQ5gVR_tFw1dJ-8!lOg$9AYO!u`>|Q#Dw(MUl&7000BUtjX8XRLN7Ah+d zjuZq9+yBfE{I zo8e+5`&W2(l@dRdQ7%LZ+X9= z<8g77DCKFW&wd)eA;*x*-IC7z6#rg*qd3LtZZ}^=z96K}cl^s;Mbh~e_H?~tV`aCb za7jb;$k~tmx~+td>;?pUE^rU-u3#ZyVV0GWhfJZaTSAi$d}rpLm3SKGY?{B;0ZCgK zssxJ$FD8@r+?2=#oolQDTgA;O#csHC_tlz%=BvKWt~EqM&6Zu2oxvS_z@2fH$#@oO zm0ve!Cx7gP9&WxrQg8_kinN=%hPttH#=Ctp$n z-P9yU#A}ZHkP?Zf+mTC(;I$r>imu9Av7upsU+%ZvC@`Qpyf~gfnzdHx58T6NUu{US z9zL5@ydNwWyS_ebQSFshNYtJ`YaB1pYLL}Pqm+Z?3ha;aAwj!3ktP_CN>aWCpI`&|a+#d17k%SH6Y#peFYG+e53F48-?FnE z<=+0D7@ZYShx7K~W3y`I5;DDf3*W*Ao<8dC$i-JZ-0O*=qde=xj>87XYTI^=b^F() zsx7AQ+V0mdx1TpvjPCU9wBt4SCu4m$AYaR7YqZih`v<3(a>v^0bj`nW*U_5}_HsOhPO$gOLX^BORr z9jbw-A_=A{#Z_LYF>q3U|BU|6{{^5f@%ej};I=GHj)>P7V8A*Zd9BCi_r0&>Z<}mk zjEhtn@wsOam3W%K+5``ey{_G9jEk?IUOe>_x!Fab=sjM%FMpeoIA~&TymYPcscGmx$YrFB2G4!M25FY1|GwVc!8q#_H7n zspq3R!W-kEHpPJ*_a+{vxP82=IJ)cw!n`;}JxZM#+^x7JOQL$x^@4pOA3er1^Pk80mtS2WMk-^=M_-q_Tv9$)Nlx zyvq}y*fLj(ev~+R)b}BF^Y9*PoZtlcS!xCL*gaSHJ1IdlQv}ot`HTBx!-Mr_QSO5K z@bw%suV=CjFzZc{ASOMd91I+FVbN`aI^EMK&%OyrPM3;qA2!fII}`*R*_u?3=k&)p z(RwP@7c*Cx$aZsCZWOnlOGI#6WZicfY8Y>;Wy>;rZj}#sc#0%usQC6&rAM=rL&I?} zZH;6Udc&^cx%YZ{Z{D#I|)K-*S$UymdhI7=?d4PoMg_-_*FjA zGRGJLh833ymx)3h6~d}3DGt}o0;=H*Ndr$_0GN)-Piik!6lLXqXpy-Ce>^@L_-Awn zoNqY3IRon4M-wW!iYv$H58g-@qOe%a*+8o8z zoSTeKaQ!qq*I8qvEURN=!rah&RahBXWOlpuOjZkq>dPlPihy7R+*PwJZ~RPZHcoPoeuasX%NtDv&_=h;A8?e z5ze~;`lrmNp@y1{NxPf*@P>+*n_rks(T=<90(GY)ez4b^!{gUq`FyfB2p^xY(c?Bu# zr5W`wuP$=Fsw-i)vRkQZd`TIES)?*?VyV?AcvE9z7}27) zuG1i6YgD*Dh+9r+I-N};PAey!k`7*0mkj0fkwmWBGmYpbu0o~a6y zcNbk#XJ=z&%|T`_EQA9SB0_{>>@;7xwQEFWv&5V-Ww5uB@2rsC*-kyW&|8l&y0rg} zK)~--EYb5r7hraaxIk7f51%=Ii>9^RE6Y~D%H`xuO(slB^IQ_{cEe}7)Gu~tS&4AK zLhs%U7pfHZi#SQdwy}8}!J%Vakhn?_6G6yhLBqz# zNWq1iMj>y09LrwDMnhaU@tg*OoS=j}_?1CABL#Q*g$OFoEj}`|IDx9;^Z@CPyXVS>tq(bjZN^l?j7oy6csWM@jX9{hur09GSHaeTWx`qWS`f)Ft!<>O<0S{`Wl zg4yfluSl4r>(a(6Ek&`+r5iA(Rxx1+#sLz`ChKtbVj1d+aPL3Ug?B|Nc5m(*$(J~s zw^oIW%Iv?W-}q_d+fpFR{Msn|9a4<#R|yg8`ABKh^VyloU>*%~f6_zyVKDd&&dH%o zYUs;s%>R$6uZ*fP+PW4*1w?5O0qI7NknZm8kOt|NF6nNNmhSFGx}>{7y1V1sNAJDg zJBEK84$nFJdG=mwuC?Zzo6PM)(*PlFSApXBMfpqzOPxzvOPo8m^uya%-xBY2RNfZ` zSD;H6R-&Xb6D^h--DY(PWMvyBm|Lno(YoJUzT%Wnw*9f#Z#WP~1p32IG~*ofM^|Yc z#cu8@!xYmSgJ%|Ut4=AuO_zmFm)$N&9v>ORRm(xib8nzC}ZEZOi>lBVL)!34NwYYw4F7lvkHO>fWQfd^_ZOI;>qu{qkV7 z-z%G0$8)rX@NMF{{Kf8o6!D5NwEtV&?@d48g|#2jVbJOHscO=yE-275+_Cp(S+Mk+ zHQMa$oPmE-R$)1$O>egNWCKTDu(H z{js#IFSBQLkgUHtg?+`JVi~l;-`zUw1kO>Sgs51$GPyJue7Ot0uG~8A@8;$v@Ai76-!T@tftL>-yVOf6sAbXmJ54Ov4NEav#4O(g2 zkDm}mWxr-9GA3pEhihM5rqnf~1*J9v7lwfEUs}NjV4&vo9Zl9oBBzIkINaoh2jZA#?cS7ypvJ#G$;pi4%Y>bM)-r$BBr_AgjGWpJOjqE zt>_+`lmdM6GAZqom^yoh$$pxqr7~5BFCNzl6XsZa)HgKL-Rdr$(^_~K&hx)3RG+Jg zl^8EnTYJpKlgA+Y6`(97&s_YVj^`yRDpg(;YgI+j*LN+hS{T5Wb@@BigDa8$e8aCV$fnw6?Nzs7!aPZ zh#Th{*EKv;H{5>LC4he?`djh*N4TMcV27ck-O@FIX@!*ha#|5Or5x!LMXq#e|0R8L z_v4kf!}0cUdH}T|ygIW*w_G+pYu(Ya&i5OaE-zIOgTJM)^M!qci#KJfs?4{IgQGE| zN^)z&6lxAiv^SZ@1p4_SvnkqP!hnaZ?#y;;8)Ozf5_?z;Prtd>M&S|WESL88wYqf!|ltV%*Bt-w}_xx zN$?FbC6^bzs%m{bHn3N^Ll7%K^21m@-Q@1adR(=i>fURt7gC(5iPQ;2Ig5Qz$@wKQ z#PM71ojbjR@yA}y)@-G^gdD1JREt=aFde$&1Nl9 zDcMIMsSlj#VX-ljQtAf#_5)9y*wQVYxBUq_?8|N?MO;#Z^7S>P8qJBu2$RDq3C3oc zo@Zk9gkfUU&*kMK4Yi|WXhOfpoRa+UVY49|HS4D&i7*f-EHA$6vzO!hu1bL7Q_#;Q zNG&8mVXC1TtteID{N=&Wg!+Y=wVr8q~=aeh=n)>BUsOQM7nn^$y8E zZQzZmKntfGgZxr&-*m`H{qN#04MuMMSVa044oa3Z@yxLmbB!)soQ)fe7WWLL#!fg7 zjymlgZ`2y*wDYDd54DT|V>P+QRpd-3m97|KRon~Q8;a5L zvTQe@#X}Xj#pgNB8G`jg3VPzZ-swrGIDPTm;k2Y=ekwvj4&wG_;`iRR`sc?#jKp^vdYBoLBC8>Ef3@dW-2OLUBhBQjwvhWLu1VjfY^>Qxgm| z4oxylkf*25VzooTT;&PL#`LeOJlgLREHH5EA}d}~*(U-n>fD4j(*4bSe_13;kRh(+ zEL{I;ElldR4A$oXjM^gUw{WiYEoklwI9^$d%s+@!3^!lEIlh8or9bP=sB|CmV^T?kY46HWIQ7EF#r5`_UseC`c4~C=2na;CYVe5Ed`_HI44Uoe0by= zR~dq}19rh1SKqeYWyE=MfeB%Rr}|6i_87u1oYlAe^yImC$gqfS+V9*bGp=IvLETMXEd=12OYnA`s}&YNg5~RZ+wY$iTjJ3I3SES>9MT^ zC}EyQl$O10rQ1?M6a()-80c8fg{RYL-W!e)K4O_8=#qJKm~L85L%m3q68)FXj1B8S z`7ESx>q%`9mj*H)DFE37J&xh@j5p_8+C^bdn5I>Mh=qL$7g7WaB`2f?J+3CQ92IK zA1&~oDI*26#)L^v&8P*Qe~p1T(drU_UgG)bbL*ik(n#KPsvZ}uSZ}1&Sl$4?JkWz; zIYZPBY21G@oB0$jF>Hw!uDz8-tW7wQB`3;-({N)L5FXy+^C%6yHLN!trh2{6r2;4< zey{B+MUulHqm>|^wkzyme7esA=?)3U5*q6A?MGwbB9YHF@df!-E;l0`JrtX6y=DA9 zNFS=!JK!0s=Zkdzqy0u`fglsDrE3P5yDQ@`OF;aRjmTO$jSr8~_g5+5$&GZa=F3_w zg}bNmoki7v5XzDe={=55V@XvgqoK!+wj`^+0OlYT@b^oF4JgaHUk8(lgy`MkJQ3KVrCo^(gv1(T6 zL)1c}4dTBqCxggd;dYaOv={ITI-R1Mp?#(ZKg_~Qi2;K74^~@JCX0o<`9&b&iNK7u zg$};@LA?@!Vc(yd-egk&!p)M%&jy4yQYp8SIT5NH69QvX`C~luLrZ=Z1M+ zl4QgXhM^P-;>hY?!GVrBp!$2VA2rFpKDn6I;{PHs1V>#iAH&BP-v6CClM7gKbueq#GgI*b74i0$?5v-wBC{UaUA==f8}XQlE0PA&nh0~zSeNhY6xYPv3k zM@os%?^6{9od^(sb4Y74JQKW>pP*hccqpiT?|bKRKXS z*~S5JZ20vW5z0Q78(ciC5u7>!p|`A2h1GGtI2;&p|DaFD*jfJ&=aTUEa4txK2kL;j zrW-JI^UMvUuwqIY_K~?M&e>R0}M>;Kq{e}`E@`9vm}wp)rgZ5 z(64HZZ&tME2TGt7El69Y^kLluZ0>#7^)9i_Z9Y?EZ!Hp$CwM)TUNAQ-@(sB z38ELgg&c&scu#9PQexuHX4JcDvxKQHN|S2e&C(d$TKKy`yAp zg;GY)mdU`x^5cJBW`zoAQo|&UL(Yc3r$88TmxyduhC&&dF9(h{%zbQ|*Yj79J!ZE( z@=Ghe*ZzM~Yex#G28XM$&~z2B&=*89ITBbvLf5Xd1q7h1)bV^fsQOzkwE7J=AX>+1 z5cU}ymYm4GdwXlLBMbj;5RDJGia-utL8S-C8Q_&ic@ky&m&>@LzlR*}L$Z+y1M#M( z$Gh#B^30At@VYYw@LZ31{V%*UuG{&#m+dzm@gA#V$5-_%9;c0QsQ^9yH};LA9Lx88 zNg#m4121dcA7dWy?5`QYN?o1r$eYqoW@;_dxR`A3udTT?(@oF?)jgm6`+p*B3^4AuJ7gbH zP7s`j#QrD42lc%Zvq&(-}=vR5XtM(D!P=A1Fapnw>wJOyqf~ zRhy_@24Ft&Y6EsSL?Gco?N|6e>K>Rg0aecal&qB&L7869TZvSTS)m0Whn^=FB)R}B7M+boLCLkcx|*le+Va^jncZPXh(b=m);hU;dANevvgIacskFCO1nh2Js@F+O z<|dW?gV!ItKC0KOwEMoU+e|cdm39>S-?Lspfi~!mvF@v@e!C#jFJYqo}G;tv@5s|jc>M>lxkf65Egeu=;_Lz8kN4MWlwM3Ppx-w% zT?X1xGXrWMeMYh0OtNe_;rvBWE)KV^24^PYJbbQzPxPi&flP{M;5oWUf%3J94 zeZ+ws^}me=B?9sFSg@n1--hkgssJLUP(b!jdzr%J#s<=#UZ1eAaHhl2Qb0fePq{AX zA8Mdt@P$XWn$~nsz3qK5ufIx8$pz?9?WvqD6)%;Pryd2eGV(UpSbSU4GBYwD$2K5Z zlGS81PnPJRVwNWs*@N@+M)&j$B(_bYH5QgM^fSZc#8?nnt)f{u1|Qg2zw(-SlT)E6 z7XEEqo+o9h#vGc0tUsgK1FGx0*X;qc@rDM~Uo;TLGhz^{!OS5{3iU)Z8dJe_AarA-&5KNb)P|~6%oUU*FL~n%>MRfD~-)j%kC48*iFIaoL%MQQoI%x=U3Ox#*;y9aZlaH#h1(P;2kxl46xMp~KR? zN#=&@#C;~EvQyc}flzh0wPx>$fhy-0d zJ$uJ#pyyqvMzuZXLk6Y@&%tD-DvFBx`N4edBjXU-2+aGcSA6ZBcv_!ta8ys}MjT?` z0&7SEUwogmtgfEOFUoRUSrJ;*dUd{m_iG-cH7NQGHi3}%UKoh`WWl$(!ov%L?p;86_bp5p%^@Qi=}S~ zmz}b&DUjh5OyP;io(b1NAMrM(j^j5(mbw# z3YLa)q@9niv(XfT0`ax4D96jOWvS1dp?+`*O&TOx#g;-bw)RbYJGg4=FRj=`_!%4X zJ@YD6_MRR$#%=P9$W2K6U#)m9PbW7O!3t{8>~ zH2+HMt>@^I)3ZOrCn6IbR1dPdzFbU_(TW-rLLo}hX`cNx1h}U6nyoZ4J+W*&V5+6* z2n?GqrnjO;Ii`RHuycI#mWzbZmjsHN2w&JF)#K5rv$JzMb}~av$ zE5BsPzmIigP*DAmT>Ig--N!hr{QiPW-dE#=RjP1=N*|WF-egg*hA51%hCH%13zNr- zR^&vi>SNtudT>W`~0u~>DZ?&MhB$ll+V`z4VLmz?o$kuO5Rvm93 z`&$7d!_PEeQ#Trd>9+(z)Ln zW3OqV_aqML&t!SRY0fuYNt?|G_zU))n9^yCx($PISsV)F;OGQO+8$LUkJlQdI0T_10@+#8i>LSS6Icz|ESNTrc9rGTq;r9gri zp7X-1&YXZRLTzO+R-{T>{uOA1V?trm_w2*^1cXWi8RysBE?ZFV{ycP_SRy6C(CHoF ze=|3kD5_EVEhiWnmm#jMh-NPPkU0rV;uqBf4Y$Na#FRGgK=WD5W5l?`*?6Q847xU$wP=_RXnRn;1lGA)snQOL)Smke0dL*Z?$- z^(J18*E9J?LB`cqdK@~k*jw9rGO3t|2wMgEe7bl-#n`B=aCEZb&5O+&Mq{W`OFq6K z8BsG%e|k!8dML=Zd2+&*$#s5+ae7;eZ}o6%&*4hG(27kg9!I^Rw?g^G-4J?+7a-$T zKViv0<{m1?|Kdwr=JDi?)zHNEVl{;x4h0Pjra|S$Gr>ezL67g={k@}fdpqQTYlB64#u5%$<2jdTk8!Kt~>h#FI#@Ev#g{M zO+S?P=AOz*(|DM#{f9Zv1nW=T=WFAq(A34RQ)53h8w4)bP7YWUt2&&Ia zx9Z$RS={B&h}Vx}eZeCFFUyP{{K%Ts9|b~zVPP)?k?|?CS~w~Iv9{ncZQ-{NN9eCk zbLLM}$4nMnNZc;vEHAu9i_psV+{jmAKQx=H|1t`7jpWSyG$Nf7!J70Xm1FcFq_A|Q zY7q{2?O@t@+&?@Va4}17bHNOmy{f@>O)^3wa4>ZcyZxs_dy=?js&`-bF`YZ+ zNLkeE~8xaRnK(y2B=yGj&rynXoKE2Qmz$gHvs$CC@Eofg_V@*1&LI zrrn1Un^2G%9#NJ5j8N``H^cyWu^PTS0Pf;cD1c)R(`Zh*SZ7T{NJON?_O7}-TW&oY zwZZr~8v!c1!8IE3<@U}F1@0@-Z;j(u3+?^`G~UJfDB{#F=03kqg!Vp-=p*2PZ=do= z{Z)0@HQFjQ7F&x>rNjs~kfh+B+Bf_9s#?NR!f2{Q)3iQHJb{M;hUo#NmM{r6e14xO z9X1cSmL?sA-r9KP)yy>m5@pQWR(n+0uRd~Xqnw#GnO{Hu)c8}E$>H>j`d)x8lHi43 z0YZA5;?k$oV{n>+wB?&Q%Pf(3Hrd7f;97~-&;s1=>k6SLTT96K^A>|f`N|2UY9cD9 z6H{Vje#PBdVL^66n<8wH3IXHHTO_dfXAQ-`4Q%xKixbgB2C2a=Tm_Zn)A zI+?u@poO&CBjr*h;oJ5JAn_G39L78zxWnO6)m=@9bKjD6KT!H`@!W;{a|68I78FI1|2g^PLkgqPVok6*c~$%FT)uUzv_K8pHd$P^4T_+RGFA|Rp@Zk#edD8 zP=oPit%fe{o%|dly^fs44=J~(O0}=rYWjn!bEOTGdQ=bhQBiCueU(#7?LXC0_H_nT zzZOUunq{M7IVVl@(mRic8So8Y-G2Rj%y|3adxu`x-7cyC$LG{_d_jU{uAueah;>w= zbFSV^ETO5PsJUhr=BDQ6?ELJ?qQL3ZCYZtHczW1re*LkiLi`nZ30YYEvXws?2V0Rm z5`uE1BPvNA=wu&Dupfd)jAKkJp{m(}ujt2XY2heCud|W`ar7AwUue{_%cSR1h?{QZ zjiU#k_rGE5;xpvP+U`{qAs3&6cJRp`j{K}EY3M?!g!`4AhmIuUG7E{o$x+#9Q}|yB zDocGv1>A}^?Ks~vTL24+E>$UvTw)$BfnI$f4?cnGB!NAf@ME?7ToLJhq>fi~7kf>E zLDELu!$y8pkS(F?xIeu&%o_>dkass~xM&DrBGuoRO zpvU#(NapE%9d3|L?WX7VsIK16oiGWnkeHSlF#Xu6zq^aTfey!svr>aD!st z#Qu+3cC_BO_jib>b+BBRf|JK)N8LOLwH#6SLY zW=}^#im#5EaTla~-fZ*?&D;+s+pv!!V?LE)hPOLclpV_qF@IKbHDvh7~~;n6N^*; z66X6GTeG<;I8fC=<1(~5Ur^RqECvCEgI8F>G(e3E<*fF*XqWr5-TD9Z3=L)1F6vTbO2&F2z&|`pTlgR&Ki`b$xy9`S?b( zWTcgf`Y(?7w|Pb2AU_DM;M)fV4i1gf`w{6RQpxvdG&on&Q^1$&{9qpQ%a<>5qBi?8 zbQ%qIaKJQV>|QTIek$>^sqTD@IjFPS0Jx|7I)Wb@b}B-^N5;R+e*`MT4Rka#Vz=ax zQg65$w$ajQ^s&hTMgGk}Aw5Gj!83YRtA}*anMH|qf01Z8=`T~i4R(n-@U%CQyKI^l zUtaRg*`2gz`z$wyquXM%mD&hSI2|a)&+5rxI6ob74yH0aA5P`y0$>#x7d_N$wldwS zJmc%5lIit2vQdhCh0cZAO|9ZnwNQhcp3T9jVx83?&il`%L1L2y@2D2=CmVAxA7`F; z&gl4LjkoR3#B^3A+TaNU&E(u3Rp{O@SAX~XC}~upg(-y(&>AH2LVy=(1T@Ur)5~b+ z=#vNve(l4B`kwfwx@+DjkFTB7w z+B+AOr!(vqU^F>NRBE*ySK3FKzW+QRQ>AZRtRZPOUx%eWZ7u+%rBy2*6hR@`tea4F z8FEr>BhX|~q!^S}bUo_?U>L%KdPh5qM&z>_6er^3#Lb|)yC^CP|LMWMkQ)w~-wz>j zud23Hf+1~+=2sAAf&}!R?$B{GLdFSCsP@o$Uf^0nfKC`g@!D`j^~2hrEmG38o6aCAdXxx!>oja;<^A+Z*d4naOr z;$aW*zTA$>FXWvc{l-Jq?A*3=Yv*`w`*@mEYT&)HH&^$Wkx(U*X0ao>ZQ|s(G80JEH)86aE3A#>-7$JB=55^Xv%!%2H~|HgrqQn^xqqh=f&wQ|K0P?m zOzK)4gc|imK&e)$8F{45;JP@&$8Rgavvy{F1~PVuTdMMk#&M?burMKon3MHa<<^+v zKV$s3{@nG!HfFOVfBd8M*0Ob(U&?9coR#{~jcLyA=sPB@ZrS0HM|MRUyV2VTi?L_s zE_Pn^D74qR_S7k#@h}CK^YXUH6^h7rnJvEy6(=T`{aHD5o2fA5EJ<#Kr8heHr~&&K zdqS|$fh};vx4U(()o8e<{JHHVbNnbI5DFeLc$>!HYT zZ6257bgx+Ie3LJ9*JKRnX%(@VUMYE?D2RKs^vy6SX9;WTkEJrur|&8DG&20pF$9$u z&l;%2h^>?BV$mV3SjG(~L12}VuEjMQsW*1G8Ti~E*>1-qdC1|&N9U<#vF_U{d^?}z zYyYgEv7OCo{A4a0oXAz=ZPw1OOVy?w7!tR;>gi36IidCkYv}t+Rc{oSZTcNm-Bro? zS{ywhru_XB(|^L_9Fp&FX_8tnDmoi`0e4=R48s9ab) zwd&YO@GAP>(&b;oSs{UuI3WKr_Mykm6Fe6eZvC6Igi(L2XneW0cgm&|BU(5l8a}e*!9~Ylg{g@1gC}xy zt8n*VLYA)nLxp06*2x%&Y^gEPITjm|N3oVy{a$?Lz#XT-;^FUktIccCgemR_zN|kk zN7{b^-L7pn2&;B(Yg*X)b{y1d1m|ymkCgD*nq51*No*d4*UNn$OMmO7m9LJy*vW<$ z5Fkh7=bF5`9SNe1Td9#7_eZG;HRdlDm48YxrkNiJL{=1%O!1^~b7`*?AwZ*6dw1kPkL2`v9jK)4okrf@ zxiP6Ey2sKeW2oAVZL2#>S&KZXby3$;-Hiq(gc$@E^rb6y4gsFMFNQKZ!Z|7*JB z^@{;{jygP=JxhNdlY4fz}qXzcB?=3HPF(}G3tvy0*g3P zCkK&{d`x)xqM)?2w8Zr-H2eqxc_CIx3TBb-G+LiIv^sw8{h}}Sz!wyg@;;NhO?uk& zmbUt&on{s#$U>N7)wW(qOn48atA}<*C%{kKe*rlisjDEBlYQf_wJ+ zmu@!{eu%d)jZBucd1&b^Bx3RH!dmWrc*3C_@fqd>)?>aTI?%5%sI}9$;fN~4&BcBU z%^0lGaP(ap(tdL0M-mDYk5zIHADKHUKi;K&!tDCc{;N`sN{&8VD2ZX6dP1I|x>^M5 zrHpt?QR4VC_HF&X>+J>hm9Qt_;i7-M6%0*n&WX(2fN)3ZM{95|1^`%+A(`2N!s2J( zdVqqG;{Dv1ui?5}vvKe~y~*i3g`*eabEFw*I#j!PHm++~p}e2{gv1h7WldMS@*CUf zoTlBfOC51$0qzJ4{@d)(mIe^6oXi}1I6J%wX?f&499h%Ym*P9+=EJ#@8@oyA@ZP+v z=ELoUY=!{bNIbG4K@^ejTQE6-;s#kbNU$@#9+v{+!=73GU`SH_#^H4TuA@7g zk5h*})4h!0=BL5eyQ@JbIRd_1Nxz3r?o~L4Dq3S6D~%(yoxlw z28*6V(e>@Em@TyUK_6DZt!a}Jr*A!DDV5C6XP^Rj&PK6%aPZMBAeqwzgVlC3#*lGS z&ngve<~vWMT$i%8+N@n|FWkTJ!5(y@eK@BbZ z?MT?ik#?!o9`k*3`gOV3l;=Xie-?xEj>FL^ncfAhEK|`%=36Bz(Ea%X=V@_YSo4nn zT;(a`qfiMf>~pcte|ukRK@eYcd4xXG>Q>u#`gVWLdoLcXl~5r)acQB|McLGW>Y5bV z;breV(dtt-xS6PpKqBclI(e>DjAXqxmf@Gd^?F|@GHN%;j2~5kXC)H;cmf#Cu@2Zc z;!3m`!t|b>j)ZaAG!Zpkc@2`O{tIAjZm`Fq$XE68P0?tiNVvG6q=n11-t-!5%)X5nvT7`zO&AEX^$W0`O5&DRcQC z`w3$%)M5C17LU_As!5@5vh3Pb{G`8*d3{)k%;K?&aCrpF^Jh9~fUP()6vf!ikdE2= z`$k@Z57CUA!N5IB2)RG(H~KbBV+AyZa(i^Sb!IDRr8x4Y*Uc`Zql%UHJrk z1An8t6M10c-nE+lLi)@vXnL+CEN5*@yxO_C?3TrCM!^N!^y%6%a7|)T#iy#>u28)7 zYTbUzkX%<+FJhRvm4!fiH_1B&fnUfi@QV9`PoqP-e%-=3{eJ1I9NHBB2yEsEA|o2s z)8jxrUS(ak@7r;xmKEBOd^0YhvU9N0_=%m&J+)8Rhz-4_tt5TK86O^QhF!Cy)0X5u z$J0Uiql)b0F`%ub_`P-7!uW8IPea z7@qx_Dc6%XqXQPE%0Z#OpEGz}!!`wk3a(70U=&J!VEL|A+8mTzkN%>uOGu;lU#FXf z*Aut6q0@X#@W1)ZN(vg-i}L`FPg!8oyrqF(8U#li**2R=26A+2*HSs3PKC>&VX*yw zKSGoUVi+)DV7VGicu6{RMiTh!Ci|z>H7*6D95&SH{t53(YzW{`8!IfeN|~qIBg?i@ zR@F|3cTLwgJ=Fek6A>@GQCKq1LC%{<_Ewhq-v4yr`$(;T@2!4#WDL}GmWW&)NyadM z${+x}t}>fN3CH7>5-~!;!e%rIIRW}?Et0?CUJ(3Cb@1umW`Ng( z#LCUt)-`b7TEEjZ*=%!h;D*MOO6Hv@o`L5_IU$Hslb8>#gt-Qa`&sS z{Eb)P7R{Q{NT>i}cdj5xXb|?_7n&Re)?wX0-yS?yVGzuia5oNA`2-k1UfBuEb~C=@`^?l z2{2Ot;lG!c7mMv?$o-f!-tOLHp+B%jOe(SgG2nE-DPJyIroqlQnysJ1`5bn>)-rwr z@J&ALc$}__;&3?1B&MU$e$lwygRoZ23O#13j8S{{{ik@4UjT0a;vQm>HeeG2Z^QN; z1qJvf^k$I=hvAILTY*`!WXq3{9{`3%9enY4u#cgT7vh#|+Q|?M?jI-EeDl>p5h&s( zvoupdq0%0PbHJBKrUN~NdVF>DN|XAMh{rHYu~2DJ>NtCSu+s2%R7a&8F$u-li~#=i z>1QWZalI{%q~oELmZ@)lF8{^f6&NtnSc73+1zr^(WWS1|)p`qml7o3F$k*B)*&(aw zhlodmjtwq7fMF&291q;&UL7@GrBZV_940FSEi}7e0yY)Yc3x$D1pmyk``w$Y0GVuH zlm&=8G4AZW{yo4Nf08o7W(8cbV8fx-_@#HjcTmmM)m5L4f@YhafkLJCegjZ$j!#d= z9Lgyr;V~3(kV`~U=4aC~VUF4)8?AQMv0$_s;M4A%Wb5p#6-v}mTe{%c?@rBITDJ4P z7O7Ss^dNrtwK%VH+`sCZK@yZ{pL^M>HM0GB^>NWf<-7;yTDdsn)E5p3|cGz9sAsj)VC-f`( zT|?76u3#``@Oa$oAD*etl7_a8t>!dgRb0db79B=53hSt)LeOZ{#XhbX{c99hA%nbT z{T(}2))kpx@~9JgcFfy#Gpr>j!JC@p98##=+k0e*<0#2$TfaD$U<1X zbRz_vzFni`wRY3oTBsU6?HR=dT)wO>s=w z{g>(jQ}$|q>Kz>h9^UTn$RoJtBGM$YnU0ZZ54>GyK@VC*?J?Qdi!betQavj>S$;~E zqTnOd8^E9}H8=m&&`D))qN)(2(Q1S4#%AB6cXG?197LVSfcVERd?FNjf35-dwc${g zbnDZ1!_r#%{67e&GH@K|rBt`Idi~J8CddEv0*Ik7LFQ(FiYMc77br<>j9GWvzqcf! z5{!}vXa5^iaD>6+F-))Y_x|dt7iffvXT#hK;G?3;MF0ylH6iW5z`$R?rz(W|@tXf~ zXDmDTypR_@8lY2%KksBW{Oi}xDab?`9F$vF!f@G@fK@|6@4i~$M#)d=;e*R#Ci`!z$t;jg8fp+a~ zVX@e_F*^K%2p^fw>E>@!#ixl87E-pC%U4f0%%vK_ar>&H1unv9$YcWTd=`4c4IkxL z>5cA(f40_>9;lPS_2H%?-IUoYI~BC0ibakj5p2&E;nM5(e~p#%k#djVBhK8Oe(=7M zE&e9CfCk@VKcJL;viDo^-Gpm3O-=E9K;oO-A2ICen~iB6e1I4y5x1e7-Xxkz4>7&T z8*b@bWY_6Ej9tu1T~aoDV&adpcfZ1|GC6zMZn6BUNXuwtz5LI+lfF-k#-kcdc-fzK zday`&Km@*yRsO=8%Hi}57S#$HIxv(4b=i-a8ICA{o(IE;UAcT(k z6v!8a#S&8ZO*MFl>hQ!NQo1FqcR2cBvpJYwT0tq<{{zsyBrf|R2%b-%QY*cQ8th&d7W9Y` zvVWE|sAE2tBIOO9r+nPt^=WM&IdO6e+=D+7BBN{N3wJzDWEW<$jv(%>O!qxQa92$8 z+pIGfXGRN}BRSql2oq{Mt%3nwwHG25QM&=e37 zSSw%&RH~>l#d)DsrVVv>A9!Uk-|}7RzyI`56Jg8750Cpwi}{-PcJDxfeMW%7T-?e5zwM5=n}@ZeI>q%Q1D4fQOLbc zD2o7&+bUeqX-GCBVlCVXXXx*>E-}ZIKB{t}bBohAQ zc54Uj=-{QKIG2*SIG{+u@rAueV={jaXpX|7`B?;8rI&HQOZhvEQQT+S*gLIKd-;5% z(bCVw*Ilh=Ke=ERnow7Du6$U3V=y|4%XU6I9)vr-L-um;d&|R>5+1sTN%QAu*^o$i ziDUcoK2b8K4fNAJm-1V2ckyn-2elmA*J&ip+)pAoPl+73%q3rhQ>MRk{Yr7M-dK2{ zj~P^zVIY?A;IbHz%fn@^Dlgxh{Y@^F_HEm^mRZr!3 zPeK*%VxLtAk@!WztJCYv4u37I*l%;s8U$|7gVxBT-TyQ-n0#ctna9gweS;oTr~tKi zclC?g7IUQ+ln~e-3BI6wc&XUgL)7ZS#zYaKM17C#QghI0ucMVJT-0h;U!hyHblexM zz9$vv)H>YW5LOPD~1Zrm~fMDl`=Y^AE=}^ z#^!RwxAL=Df59j~uZssITR+|ImHsy{J~6@5UiB^Yq(sOhv)Q4c7br?ys{GvsAoY>%5sD~)q{Op(!IX^0kdnAy zCuE6k2XqQ}m0hDgn@+JP-kf$l5Q?^b8Xq-gv*#PN5uxB$272sIdffUq1Nw>n+!4k}EXjQMb@e zeVh21{L|?wh%JhcBzR!U=L6+*QCOzet8&*R-3imVcU@~cw)#&{8qJQXZ%OPv$bn)x zcu~Cu+r~OAD3%h=vI8zJcP)6Wr}^c|wikZ|K}tb`H7Bfa=YnZmac35{hhVI7L0^MN zy63I^ntlY_K#cXD6Q!PSKc^MJl`LdhYsX&Rr=0D>tsH*t7xgpBaK$d0)T$%(TkP-7 zZ^&3&P8FKOkc+)(thJBa6V6b`WP{MC3JO<=QDtBwjLB~N#_qy(nIzg`;lRxdTRul0 z)%$3y)v`3u$3Lysyie-+Pk(nFRXaQ2AVH0ja8@c3#TzGnOw=b|K3A+K_dGjDLK(vL z4%P!O85@|jvGz)f)Lo;9${KCICQ6uwIMi1lhNi>&(pRCmD;L*hp~>zIrcv|{0u$qM zvsrq;o|R*h0rHW2j^DhiF6PEYk6^MUPF7s?k}ZL+pD?Zt#$p+E6yA1EO7E+$pMXV2 zAl&I2-Qo$9T<)Alc0|19NzGfdgjS>CcYKNjKf;tWuQ%QuU*sP5Z-i2{6Fph#k-`n% zZ=KvZaO><=)GJXy?J}E&B$&o$I267e%kAxdJ|7ePJ}#F@Rm6K!vg~@86}Hs@&uwWp zgqgisc-oz_i{h=k@AAYAL%HD&{NS~9!Bp!J8D2VnIE`NuJr;#ncQE;<9ggwUB++Gp z=3g+z-B0e3@R99ZLAvYG%N@h5-_J+sbUJJ0%IPo9R#lpG&^dhD8Il}Bqw?@p zvSbrd!ShW8vAI*sBMi${IPm(H2gDVRJIX}$j(@3Z^6I>X6!>D*1?r!hZ<@wjyG|!a z|6m~Js&o(EzuxgXN4QIfX^k1~nfGJa^N;)cV~8~j2D{H#0r!~EyJy@y*L-b1N7;9sjD(ZFX% zr2T?gPL2(>7V?px!IOTc$@=5a>rxA}N1G_68qcv9d5S(`*O93kGSa(g~^Q7HO)TWIDBzrX#7#NyRWVfvN+t;0Yn-xEgJ zC$AVnnH-xJG3a8*C>v?$C$u3Fj@s2STs^sH6tX3Z{qVNp#qpPxQ6}0H{_C&)Q!Lm( z51oY$_3>t}pZ0S2_(%nAqZ29!;qi#}wg%LX7>58c#6UD*rf_&Sn4=n?PTKj=qeU?X zvxrHrT2TZ^Yz?LGYHgZR=WF! zmvq8Y5IilG%bH5z%_LQsh4E`5YLD)>TCMfc71K}zn)2zK?@5a}PP+50{z+jFmh-b# z!SnF&80m_tfC2T5Z-f2b_oe6q3jHtr?JzjJZ5D5u_jEatK@T*U7wn?%g5~k=Pb_TVD3oZ-dlDxF^EP^HyFl zH?l&8BjS$&eHsB-0Wr85AdbMwK!_s(+K2DJ{d`n-Y=<(1F#Wh?B22;@8X7vQ^)3Bu zzMA66+0RSK?F7PdaH*As-^(1t9<-czG(zH*79szHbAQRZ%my4*o8D;RTUZ*n-@NJW zlbA1Aqd2YbOFG2mZez%G&a)#%91p+5H+s?uoGgy8pHK=OF20Q>?`17$Ch-qH6%4`| zomC$TlIQ$AUnPW!Y;V|byv55U#8mpl|JO3Y^OO+Bga}JyHiIY%TT+BH7Ny{8frpa+ z$JAGbMfrVQ3yw-im!x!ubcb}ebcd8QN|$tZHxi;!5+dE*AxL+3H}4ts_db8V&2)~hG28?%TcTs{r_nO8VQz1+KQ>Mo`l(2KR^#2Mf?k%vH|+jMn`7=0z1 z;^4LUyHr6rXxYm~-bc=C_V($?`7){`S-<%*hK!&_$7EvPUykJB8f5c^G^AlcUz6te z*gb9++lhxv0GHhovzs&6QdmTyQ6_IP9q@^K)NSAhHojd5_?R^EIjz;O<^&VcHfO7 zWnz-ID@`e+@0p*BqX;7wu9QH5DHTNeYq&I=2_rODWJfjN*wTcsDm=5t!yFXb*SV@` z=f^!Xx7sGnCuT!-E?s(?8ChQxGZFANScBp*p1@xbD$UGXa|?6XP~@1iMWVALT`h*? zOw~9b5g}|;Qay=EEv3BW#HEyt@VbokaC-C->4BgzK~Z_OL!(Tg5z`z6$$J?Tsc@Bm}gRoF%$RL)&d*UGV*Mn7pfO?A+-1H^MC0%pb)x zcKrSCJG(puU{UsiKH03O9|Zxag^usqBxQ>O5R4bZ#9Vx z_7NF9YasfpL2*RM{~bdx4-b}3y+jnil-5Ls^b?O(BWlTE92)VmhC{Tm{{>;-ciA*n zcRx>1>qjCI*ljF$8bF(4)gFxIkbya$=wS;cZ=&a6l88OND$6Dl1!!Um@Y{uQ+QZyQ zRbHU8c+;qtq<3(f2DpOZIu4Q1(Wr|4FL*+PBb9ojzW!toNGTq!1&J}9Sv@eFm0G}a z8eLuP2;*`&%w}8Qc0Z>FVl~sz%zn>q@?geP)^bA$fE{NNb0*MSDVHZj_b3A8LuJ@8 z6UGo*0oCJNW7!j?9b0S9J{v;+hfuZ}_Z<#OZytoGb}6{n9fj>F}XfiN6%d>m6vira#Lf4Sg+DA9D+UcU^k$oF(T& z>qTbCDN+?Nl+e48g5?0)h;e+QGu894riYMN%2+aWd4|%!jR^#FNr>*xKyH;n5&nE( zD-AB=YE=itAIDEm;w}RiAsnW&z7MrLqZ)K zLZ`)oCx3hQ-LoHjELwO7PGrY37_{x>4;gC~ZfRFqv6D>SR9u@a8q;XBEARhh8GUGh zQx$b}@_DGK)z}`ExHMRr?J`VZ;OkzjH{Ka=3OkvMkTT^<0rb2@AP=8p?5&ted_;4= z`+nPG=Cvyu<>D%2H2IFVM4ypP?34`$PdZi;QcJRuH{_2^AqEosTT>p^4G{Uy2oEt7JfjVPbJ zXNE__OU%tf0M#4bv~$Z(;>fTfvRi)&$B^Vlz9!Oudakf$<-tOm$#yLRUf;Q|_6I|Q zof1IoQ{ob>S`}q921NctX)N5=Z;z9V%WRGi?5|hR=C?<1kpjMpC0Kl-GPp^NIK(8e zV#?xyr=t zV#P|LWOp3k3O^s5?~rv>tS{-K1%PXn|4pIFl~Jr&V3fWOLxudhG&*#w`RS9_D7kXt z%-(a!st%nb5J{ybpx^Az;GHBnImwf4R)WxHB)^$F7Eb3e`Bwpfb{v(M&MDD7sE0(V^k0K6Sh6OR250JiV1bjrx6k z+}xz;6hyJc5CyMH12<9$e&2lW$kNi{-YoH^=6$+?b>%mRsf{-8*m0uboM7`jz*yv* zxrUanbd*m#jN%V1-9f!mO}ZFEd4HiOKx0JL%ZChYyfl|X_}YW?)f{=xiO4O^Ddkcn zgPg}xNXq~zkwTtpfgGtn7uu@fY7LRVC7;Q&K;b z1Xg}ll!>?-M;FPkp+A0!j_36Fr+-nnI4)d6gH>L{P3w zispwEED9nFc3XFaPf)r#_)dHnfRB%93`0y|#pR&FLddsv?sO+KxNHtnmO z!ZUQa%F5~9Z&?IlxV=+q=VK9can~`=S15;v6e%W)^4IVxTdJNkh&i zmU3s_hnmKvd~~Dr!4_Q?S=~H09sMd!6h4Rfja}Q35)ok|T>PqDTr5P*Vg-^BOua5a zMR)pAX*r8LV=AtcOEX5JN5!YVZ24uHtTjRseWDPeDW#F-=b9t(j=uFm6^vtJ^;?y+ z=dVknJX*a&MwwbD(oSV81J8BqoqCc!Pz3~62|s9HQg2%2j+cFSZf4PRa*Pp&(pJgn z)<7;stM<(toic^gwR|d2$p01L%uh$EDTOd=mt*P@G2)1$H1G6)uL3p8s&+8{T9cT* zN#36vi}vy|JCv!luy4?byM9PrQCwGuHzqeyVH!r|u72!+TunY=s>G%e*P}bb+!{MJ zWVo4^${ZMhbep_YGw@dpZg0m^Uy19$ICYZ>jjk|v3>uNb4Z3NkC^PCn7~X2%ZQTW2 zraTT<_-o9iwqHeiL%tV%faZySad^&=`A%V}NH)+cKNuc)5a{19x-*#>?zgqjiytr} zWst5>-1OelB64I6Jr}e$YNwK+=gW5x<-SI}c;vst^`)dWnN7w*lOo6Ty*BIJQs3$4 zuhX*pjRoslzqt|(-x>S<+P%lfx9eJCtBID&OL$;7{Z-rTZ!afK5wmretAig_OI=l9 z{h2HCj=G{-V$43C^gUm1N_deeeU#NM1D>Vzgz^cg@#7V*2u?d@8Ac$HGq zfQvu;qByN^ERIxfgD7VT2~&wpjac@c zY~cRk2b~iGl8L+GI(M=U@oZ@22Kp!YY8@*V7SK-!S`PQUW;#=F|SRKdqT`AAky{FH#<&>-Y%)LAj_&eE+o3 z-R*6qlM%*g$HC~~m*kqgz{4WD8FE@Vqprmdvdx5m6^rZeMOr}gogquxh?(+w=|@>J zz`)O7@jLk}+x!I1B`@TmeFYJ18}Lj1kWF>6z+rSAhyaU9G#fFG=V zSH1Y9EoN1|`3{5p5U}^bkYX|McX%u&!PwGVE%G`R7_gbG@|4bM3aUo~AJq}?MX+W} z?1Ad;V47|A7%9ChaTWSv9BI+n;7KDWP$lJ{3cl2Rj)RFUxM3WTdk6D{nd9GP zSxZ-u3#)nDT9{AQpv0?S8J~EPfizBuM0(9-wdtQwJNOToF@?sk2teg}s1N3LfnDnk zzTKO0`0iqm*96eQ6VTw`;ah-Qyi!*ol?`4YOJL$S9r`s%pj`vg-_7am48FJ=EoXxl zY${Dic%7BNvvc0rZ^IaTXAb_G&QElw-`ap*(GYUt!6-w_!-J=!q%?ia&(9yjNZ91h zM$8y}2?mHAuJ&M9=jY_)fHC)iuYla6I2@xyDjU=k!wI(OMf_d`LfzF8={*D+mUUxii`;*bkVlXIWg@DJ;)DKAlmXbGr=}|!DIe(y4 z{mx=pXPSfTgjyERgfx^x6*W^UEf$wro|WkC*c@B_Icg%5X3T=aXbNi&mmHKA4*tdh z5z~{J|IEX_I^jZmj*Q$1=oUu+M(!$Ffu)Srf^x$k01uEYo^dYnM48!y34Rc?HLTtJ z8YOp!rw*v?dP%F=+kw*SRuEi?~)HcWX!mOtRbD8 z6$hp59=zZ&OU^+wIx=bx1P|sBV1c@ZveO zt=~j4GSJ{-TM_Fg@d#KmYyu-gB+8kL`Irhy~i{qP*vs3ch(ou7+?w$CfjsV>w=qhtJ-I|v3l zj~6BC)z}&SBy#7~bEJiU#1baU({dFQ#Wb-Pxpz>z!-!W0KA zMLwp>@W{Z8CKV^`L_h#yS!fEZ=P~%O^4M4X#bANkNax5WiIH!btYK&Wk+S|Ea?WJU z>#X}9BwPi-M{~#~1crtOoemIf9zURGXlMBXyWn7 zkl@Ogf}j{+tm#@E$$!@h{rsjxhkMvvJMcMfy2+bf7Ct`Feod0w7_p@s?|2f|vnT0q zVQqwbM2>4)@7VBOl~Rgw8_I zWt!evMR3RDLSo&dMRmax;X0vEEbI*u=p1qIfEPiS6IWD3h0fFjUF8;6?9F95k~mxh z{kc3T44egcrmVkv%AXDK7usTU{ja0tU$R1E(RAvJvLF-#+&r-t^#nEXv?H~#=IZK7 zY7J^dbGZGr6snx70oXW4WK|l_5FzOBL69i&lDS)T$5GbQxa2BhG ze+?^deAFO$-JqyR6=2xtaV6WBEeHr;U?#XqP<(+UCWCozJPnk})XN>i1`GZs&){5?>Pw(@g6pays$T3o`V3m;fGp8n-*&31{ zimtR(%t_6r_OI^F31%!AxZDnTV~iJI=9qjPhhDNHs*4i}oB*yf!^Ovh<4;yTp$xun z{e0LgR;p#*^}IP6;-mp{>V*|97O{o)q>OR{ltrad`9?J?(~dn9`}~p2{$@9%ATpej z4xH35_<~sI%hXUZuP6!n$=INBo>%S62%-`#nNYr#SDsNAMNM5CSjs03jZ8Y zUfci>IkNlaTdBl$<QDeF?A%XB32iAO z;qcg=r=BAP_f8$1ouoWkxAk2{&ajDkye7Hp2M0rFZ{K-vb~_bV&r};oRB=4*YvT@Y zrrYzG?Fb(mlM2so^`PK*O57nyVQ=LBFF#6;dXyp-2Q{QB2{XMEQF^v)9ETx48CF$?A>z1us;#Z4)qc)`izCoyq|=mNFQPLWoVdSON)EdwiVm)Fu;Vf1$Fz zk5}DSd&3tFO>~gA=}<%&(ycw}pbd8IKkMy+bn!?y*xfqB`45#eSSB0HUZ#xXIb#4_ zgjnT&-9Cfxw`N6d~+r;BrT5wh5cj5O!a|$D=c5N?WW&Ky3EGHq03i zihYQq6@ynRE#j8mM{Tjd;82#i92r&GE@WbnGJ2r6ZZT2WUGSh<1}_#1-l5$QXk3OU zkqF;Q^fo=cOjPNtcR0Z34>h z2gEbpJFks_G^)v|26yK|eLAM$BS;Am3^m@sm_+lQOz6v)7ihUjc^`1rQiv`XA(Do| zqwDzDPP!CwQH4KXtVHd&Ly0RC(&>v9#x%PAb6a78ZgZtKJ#_%yA|?VKLY_UxDb#{p zMNMU#K#5}Y+Z++`M@MUwMl)-a8LX(bZtBvNdCQcoBxsM2SFbD~9eejcZNU;ZLMQsU zQip5}(i6ZhGz_0l_@Bj!J2ZuHpI4v90@wH!q|HW8{VR$x!D@jDR^ACDZiAjR@}nyx zS2Q~LP(D@C!QrQMh)pDVt`XW4`b6b{i$+mxzEy&FJLBVX#1#1gU)OH`s8$VN_o4mj z{r7^doM0)nixB4w*WTv8A~M(6{RQcEdOMmYWz!%rud_`)eB|CSvR%Z`?2Uo>^d;rxno4O_aw?2sOQko`!?Ik|f%1^Mggo0MH0nG$50gMne{}Jg93q;Z6n2P_Ap_ z!`pMxVbyfx6b6rwXq1>U6`#x|Sc?vTkC1j^<)K~q$CGv@{>o=@$DdiKgV{W>9Pm^s zk#@tU)*mmw{3A_S423*JA=-^Lb(GW#GT4v;iE|mJFZe$oWcz!h=2!}u;c{aM2`0Y{ zGgumV;zB0HuN5ofM9c2F@6<#b~FhrJ=jSUkCkInAO`y-=dA!)SF zG^gI&0qFiFQE+WCg{+&El74#9np1BU7np_Gttk}bwP=?6VGY$BcRnh=y?;Rjt0Q1= zRqLQ=Ay=qWe?n&wYIJhIwNxkS59fsr*kH^ljf&9+a$rz4AQ#BOj;we>djLMjW_ zoMF1`pA6C%s<`R0+t7sGbXI8Qg3G0_Q3;{m4*yCcR_lC#4Ag}}9uPjKEb}vqD$khS z){$*1>HTZw#czs=(G?TO%l?up_u}3!okIO?qf7>##Cu|ykE;{59MWIct2$XDTs=OJ ze;leBG86Vo%4++LDq1A@>i>@JQjfZvT;!_ur;@mX1AYOEfJBW=^&AmVq(o;K;}s%? zog;7?&XNGtgvlx{=Yr*z-}7Y=wYG~zZzpLgMuIMUf|j#F2%Z2>o_B&0r;fz81+TI~ zAm3Lp3)ru~CTuMj#T(Ami0jR7A9U00ZUA(IJ_BYtYw|}t;F=||VFYMm={;tD%fSI> ziL_bCdGS!xZdyz%r~LS6?g)jEc)X0RKl?z9fNyU?N4WqAu>Y&tc?*Js`Ya#RAtjO& zJTi%x4cCki5HW5+k8!Y)To3VYxcqaU$$gEx{5?NC7B%zT^7-v5{pV^kpzBaR+ZC^n z!tQ)m6Euk*fr;uLrrD2M|7_3yQXN}Z59Rjxc2@yPVmO_;hvzpXV$17(8^IpfZL+&U zi<+uV^%b{eX9dCN(IoS3C1r%Z?{G(Rebn8?X=9PZ99}uj#CTpwhG8=}a6kLXw1D*k zun7uMp9dkMMZ}zQKje zn;Z{(ztS7C9O4eU;W4@yXK%Yg$4JYCxsOTJevVf@t9wv*q1QVe>rl40TVbxt_zb&V z1@AZOv8N>NC^Y>?#l;Q5W4Fv3$E4x0Ihb!)yJR&RlpS$(vHQQe^-dW1=&+t6@ z*yfZPw;o-;F3NR)4B;X#<((Cqe~eL<>GSEnVIz+8*8<&s3*;N=@SAj0X7%7aw5$V} zE76>6XEqkMMxUTd1*XpQ{CIPkumN`(GV}bvl1{YGdOa!eJ>kD(a7C>wHjPJHmZ@KP zm@%LJc!d&SF)dlmj%+O%LS>v{=(SC!jZd^(sWhEZsT8`r%O#I*X-FNeZN^a&@YKGC z#)@t3OZg|}{K~kfz)tKP(Z_MzP7&c!#?pinjVEK@3ap~J<47q^HeKllXT2hutrXBc z-FV5AqVdR8)kbC2KJlt1@db_qM!@`ogCHiz=>Hahk2@-K_nVcN4~Ro2eo6X}lF}dF zjTNvz@sdL<}sIs&+q4|ws$(= z_?}d7IQ+}tg?3TC#EDm~@3G*7qH^ecSgkVU_VWZw{QN8}nK@CdD}CHH@=`0Rr^*hs zxK&<(JSpnhJxii5i(_({luJuwI8zM?f1PvUwv_3&5%i*qKCa*^mgbDGH?vf!IR9{< zfH+w!O)rrYR~PHrMs)i1&BgmQJvIS-M~TZx*8=6}b<@*S!d*7S88y3 zpBCQb_c!GIkxDmbz}ZRaIoGDu;}nv;daDxMlPXEVlGpNhl_CHR5ks;A61sNU-fDBD z)L{@**;07n2^SplL2f`aWbpZ*sTyR&�%Whe zy(HPUwdzr#wQM|TFvW=&vczX-l3BkVV`$$H7w_wnx(y6l1$BG?sXzfJYpe02CI07k zg%D7on#|4bKQ;wIr`;_xqu^^eeH+xt-M=AY&{UcFPnxV%86Pu%Xc%AfmppP#4zPn=sk9C0HxSaSlxw7( zX4-Dm_YbZWAym;64d=`{A3`w=ziGY`F#tfwJ04^Fcs^-bs>9V0WBO}T%FM9gU;pr@ zMLo!+RjT`(!aNc*nm4b(mrWAR%(nVHL+SM;Fif*bBYYu|!tGeV+(tvW_z3ofV^664 z&JGZIAF<>8a%@I$`4j5FHbVolRwxPB~Ye5%SvhP>2uy!?}|BwKmujmG2 zxatj#^)_Z?W$ts>?QlUI_a`Ur^dnkz*DP#1s3@e5ku9aCL_67{03FAXzHf4Lteq2ik;U z!ZmSQ;WrBm^Es>-Q&$U=^)80nThzP1tabQYJ5!PdtN`2#-E01<65{_mWja(S?#kn| zxriXM#?As$NP+5cg~ymBU}o=m^JDkB#$dqTO={5%D%oGt@rCOl&x zU;XhL#X3(k1l>UpFpV6(dLF$6$Zr6Kncaz{;zz{UCRZwih6Vl>4xOHVc6hX=i* zTmod9Z6KL~o`FgSC@l%BII`<}HO46Wf%Bab#z9tVd3@aU?o2FZJ+KdWZ>o;-$c1hF z4K4zUH;4OFEu!Pg#|2_gNXu?ZJ1l>#au+HvJ|{fG+UB%2QO-n90XQLU6*0%JaY}v; z4s7GVh~~Jis2E;-J*TY+GKzu?W)%{cP@W3FL;Ep(1qKbzSqj$UZcrWy*9#u#)m`O> z)lee8|~0T;z;W z-yHtp5}qRrgCH`N&0;iL9-x#)!cQeMhcvf2G`bb1E*59%n$*_l zz4ehj@C{gblxSCH};a|%(-q$|!-TVy1;AIRWY1|a8XB>dJ z^1HqPeJ<#)95mU-6!x?&lFT3Ix3UKvSzxFr7E|U`O6b)a`QS?qqYkM2yrjr}S6lKA z67Z9Z?al9meovt7K8@k`o%;C!qeQnRTmmhoo5{mf^{YN&ah;P8kHT8cNS`FDFDrBP9Ik{qrMQ_>1Ej3V{AmGGWe{+@0 z65yZToqbI!hJ1j;wsn08qGB>Leqkhm$Y-t=WsOVaBIODmo^y|9TcatBy2>+Gt!D5v z>NWxX*^);_pS4b~oZL#?)2k`tg8R$;f|zXsk4*Zz7CdM?Gt&885xixuu;#AogQlil z&})}LHsmwV?iC$!lujl^T@~`=toRls3OHE{?;@Lh8i-%gPFPL|#Tbblo%m-Bm&izj zUmAvAji)<1f)@-xcYRTJb=*(qi7+HteB0{IS8`r8$$pDAdT2g>1UmAx<+?S5f8%Yt z$Rd#Y9F6C_E8i--jn4>^0KTTpJ@E0RdbousL~Yk%bcz*9{96VSqp?K;4KuwX%h`SE zuW_W>ehsXhR3RoE?)fN~YIzc~rWoI#ET1l|Tv{GGf9K-cC5qhVr5;-~HeTRZd*Kih z+c9A?Rq23q&uNt2AJ+r#db3bxU3fZn^r3WLi`NQ$D3d>Kpu*K`vNiy+&|cW^lo;3@ zX(n=8Vb9hO`8Uis8+HW_B=b2xWijoc-x2<3YhXTf4_E6pFm zCwLms+A5nUIvgg86&=oRL6}@s_~T;$zU6`2ovuO@5D)F%hn?pMvDe!C!p+#)?yXn+z)=w6g`KWL?5@W6Qe zvIu+ci?#DLX0?l`>*XI4VTUtc0?Ze_ zrhS|F@t$fhi;H3GeHwe~ORFTn@0P_aUw3)*D;=)6ly1F`#Z80HI8v6=_Ts4;!Bl9# z%!A0z;GR-p2usqp%H^e_HVF+5=bT-oZzk}?8%zsw@dz>KrZOuB)>h?9l8^3n$`vn( zd;Cr=^}6F_C^H7#Lw{yeQwRI|eu$In9uialRPsZ+NzVbU_h7zWw_xEV4=UhF zbvvaXtPOYBg|1{ZVTvJC%~eU2a~|9r5wCJqGSIc&o@01Q-Dg^z#so*F==Z5`&@rd zQ)=Z!?^1+U7Ls_tmjsimH-7L4G&HnL$^&`U5!dfc1b{^lQ7{IZIwLg#;qPkk_i@$0 zsf^u1=7e0V`yfP5k|Z9GN~3=d=%R6V{jf8X%ZZ)qGlv!JC1+a7rd6eu7q$)D3|g#A z!&LRbxXcb4>$)ky?mR9cunG{-oH5n4Kz!m z2~2$U7Pdr&*t)jTrRqH^THp_~b-8=O9h}nU{rB3#?o$TQr4dO|BdWhU8ipb90i*fizWP>rjvL=rub+Siw+dCIj#4#Qq%PCjY6(a7b9YWF3+2p>TR; zH-o$$VEu&Q3A^T8T-o!It0Jr9$`*0m^zEC5QwtTcNiGQqzva;E*Vvp>OBqG>6 z$d3QIk!z5Umqy^QVOG5(lsha>nA{zi={eC2J8c*oBc}cO5WGscTZ^FD0>gM z(LYT4JjAgOQ=cY2Vn{#OH7jpjyKw?fW`n78TW9 z>djAL(Axdo;Qj-#gwI}vQLo-*8P9=^G8zI~H)#_SdP45RzU82wzRFAIp_ovtGm@9i z`Zp<#=7JKur=Pn?*~x2T13WadV~y7Z`1zz9JO}!*r7cMgSbrddfV;%jfu^F%M=3 z^owb=ja>JB%39RQ^)B|#FKjyI0`G9&`IR#hJbheki2sTs&_<*)5P8N9e}8{J9=0Q2 zOv_xR?;-Q%STB~tLPdsb!$Wf3{UQ_0WgcB!O?SNUj%YG_pon1QK3RN8I22`C9 z$7myJk~x94XT~N?BpPfx3h4rFK%BJExn(2VGZ*aj2XO9S>ks-R8d4y3NhiW8gITfqPU%I1=KH#@P-*< z5ZlrqWNCy^L~N(QF@ii9vWYw43po!!1yMsK*Abx zM1CAr;4f3b2MJ^G5Es9Zn~$B+z#8^<012^Y!E=`C>lSP8t&rE^Ve$=ipN1~lA_`RM zg>jt-rx4@g*$;m^rt`X?U)=2zi@s1wb+(8S-_s^xBggEzAZquut`p0h_?Z0D1|9*E z>Eko!gC+k#D}Nrs!#mDyrr7jY9RW`wuSr|!Qlwf&#c#o~Ueczk?80?rmFRcgq}7)Z z^5bO*n@-PnEIGeK^_;vVO>!pE;qt_u`DGo;+o1aS8V38_>DwGsFR+aI~R z_cCxc0*=gfY9n0ES8-|8zJ`mXsln?q6D^YioF zNlaPBjM3^Hkl5%)u93-c-3JTnr!~+{B}l8@WeKsA$N7Fw@ZB3qFzbfwhBhY`J2R{C zsamMfzZeL8X3<33vk>DXNLkb)n80!ts zlIxSMRwln!XcB3Y1kaLMy5{cG@^bE!!US4^ep`H|h-mPDF&f@V3Ju1n+3EN%398qm zz+MZ;tA{c$N=WMiI}3lLOI+;tiRKk4yPG$CQ(C|@!eIPR&_mmqmth)@7E2<$%APX^ z)6LHd({K5$gz1=WwOZ1(=;x&Y5`AM5QKt8(cYJx@bD6Yh z6)KTdFfweUm&9@h`VC&-t>|jJAPTj1sMzRdbofS1!j>8e8(;7hV?@~7kgb5kU9Kbh z4v2e)^Tl|750&rWEN1rdtL-Jgx3dXYNI86|T}q&jtEz>keKuF&HJTeiI(Gid;QC%@ z4(nB|l1(@oN@6nn>F#d~kf5~z_Jy%pwQ)q9bHc^wY%5(D1ue|AwurZ*s)ZIk+(5?y zfm_U$p6j%G=yh+z@rBUE$MH!O+wBCdv3WBm{$9T#H{CkwyRGRSsn8Ef+t#Lhcej&+ zI6dEncH*r2igq%g#<8vWVOAcdbaSb)_9|{zLwFL!q70*B*ms*fqpss0q;cQ8*zElv z9db_J8CR=HEULGKP>JXFA+_Ym-nhH@tH<_V&c1{fswiJi@b5fpd44pn;Qd&2>B2h! z@-oH85nb`&m&S~!;k}vEMm@alLUEDlNPxiY#j&{pBO|rfRa_E=B z8j3>Jdm+lxdQthv&(lI2hOe>_R-#l0=9 z;?Oi+Sur9X@KV2A5f6_g;l2NHKrHNX9UY#+riT}o4$RvboB`R1TBD2@__OS&?yPsP zz*wPWw->FqQz&14;^zPVLqMpS#nEP(B=|(2aIz34Y-X_S9y~gpP4@N8QQ;tt=|bZUeMRm9R*R4-m#TitUqSy524uS4$a_G@ZB71CNs1~ zf3$Bjv%<(PkQfCQQ@AE(K+FP17J}YsLdzAB!;KDe{k<9>QsKC)oiT|WR)(+~Vm{UD z>{Rh4>HkP1=!rt$4q;S_Ai5iW3J%EDh=9KH245l68`8kmPj20!e1iL=ep$i*5SXZy z^WeCS%x}3Itbos;uk*;B0NJtVKj=`X6&*OB%e8(mH?*y%DE;rJpj5|V)Wu<3?VlKY zC=!&YL^c8*f&5|wSA!?hAO*GKfPIH)U(GwXQ-$YSY13I!MY1lat!XXKhl)`q2HTR; z$=fF3wKuh#9_Ju$Xym>DY^Jhxc};iL^3oBb1|S3}(5IMgJ`rQfUkW{WDl%}lsp^*M zb#~CYV+LJ#RUdE&ir?~I^TqO`)tpj~o7cwZ^NJ^U_yQqiF37E*OsFw%D~ZwI5BFI^ z;F9Y8m={l>!rUjm72)(NT-JbmaVuok8m^+EIl8Pu{GQL8RWtAzMjsrEc=3Yoxx{`w zvzEo`WTX;>elxC!8J9D{Lze;q?8v^rO7F{@xaG{l^H$BAs1iR}x4H^4Q~IPd3MXiH8>|m&TC; z7(yfF+*p$&zkXEw!Wy3>c9k+Xo`GA0A!hNZ3qN91!cf;&o`?3C}z6N8E zsebFRA3;=R2M}?N{)p<6QuDr=t8uKc3-?^BfXQ9vT%{!pGjbs!Qhh|fZuo~E;(ZS# zuWdeAHNK)OGws8UXVk?5E+3pFi*#TWFN+a#?K0Xl=ny|Aok6|VbJeGvqClDtq5>|O z_1+t~f;f|Anbv`Qj?Sxx=HVYR=a!aIsq|JH|20$|fgxDb%(7`517EnNsEyvt@+DWG znXAx}rDFH_3?x0S%wIqaj47XC$(BrfwR-IF><4DN2}{a1^-T6Rnv8wVX-^Ad1}kOO zMC;f#8B>%DS>m*7C|T7)*_d<}#P6sdi*CtnZdO*|CYwLKv%5A)>vYT2*$Jx+o&8$k zJ`!Wkn?EZ#^ZBJNub-OuUKQfl-L`hx?s&HP=r|9nM(MO)qh6`$xM(HEmk~V6e79hw z9g~oP6v>q_;%nwyy)BuKZKQTr_SBKHURJ9l!gvFrW{(=)YV>2+bq?d6u+;>@w`Voi z1y9xxUGm=gnd3`7{=sg)mU#5O@^SRh>i$mEaygOfShYyG?Bm5$KDo;O@6^|iku2a@ z`_1H2u&cOE8)P@O`Flx#(#qrs*8{Oenqa`Fj@K^jOD&SD7pTMdlFt#|7miGbO{}Be z5G&HUThiZsF7uS-2ziT(j{Xtw(v6Q-d!*PN6D7iPVf*_0D70I;lJxf%4B2QUwY!;U za>1;}S*y4YmMx;~tPr#r{96V8B5&pv`zDpvxT&sWAGzND4+aU~*sF@?*K{ z^9vW_JmlDXEnS{P;S9$I(TK4-hLOU2Mt3?Vq=N<4cA@bJh2|S$YX++|E-k=&NetVX zuKKz{AF3Wp4%u(i^2KNJQaCMV63lv3W!SU7t$p815Vs}4ygd*)sbEh{ zqnl7W;dSfr8eEQG$9QR)<}>ChAMJR(e5{}_bM+H_TC=K>G%2zg5hfJ`gcd ztj)zU!& z_f{L)e5J|oR8$AP9<`g* z0km1qD#o794c|Y_wR2$~ujcSuvI6$?L9mctr*uIByA-@A4y{OMEbyZQs_o}eJQ76d z@X2J&Ijrm5cH9`jDMBvo_?M08gcZ5I;)70WNF}*~4Hw6%rwFyveM8=2gKS8^>ZOg; zQ|(WenTyO9j7KH!`TmQHBfK($WIO zqQKc3vpOz^yn=Bjwc|8@B21R4#kYppGdJASD;0LuG`AG#j7Gw|-Gt=*F^wrPTe5S= zc--^}G+zJ&BSskHta3(MBIx!;he7;FLe|E`gbf}JuI#wqg^LoE)fW)gJ^?~yH;_m4 z_kM%sFKE8O7deOCcTq~3%6gXwpCoING5&FDrm$haBKmc=_#3=M(ajKgW|!zCf5$NL zS1$XEui+FoWx+rLj69}&e|Ro@Ux^lNRBS`urw7r88>BzF#m(xF>h+)}<}-i+GLeA= z(oABIiP%$Q2X<Wo+eO?XUM;n;suqH$wLH{(51QlN-Xq(}<4MmWA zz{FLR5;f#ez~J~d&A7Nnh>U=u6Dk_Slp=HrnXj)4O3K;zr2ajd15nbiXjO;VfXD}t zlaq6gr8zLeh`{|(BSFP^{0vu_QX%tQ!qpM6EHMvksa^wgv?-SpC6A;| z7bKSitZMpHtsS+Z&SR`BW$kL?C)Wx-zl9!4F84on-&{0T-~4BEl# z{H9s)6tq|*lNd7`8Te8s&NoLMpMvm_hzf^Lhg+<5Y{DoRGdr+LHr*LRTa;yDNaW&H z`C>esgn$wQ9$o)WG<=s1D%gc!>ObRX1qNbW^G7NGT|-=2S1GQy;G+I^7$XNZVmLU&-+1fDDNsD+N*J+vmW4PP6sz zsqGFRAtAAIcAgAH`mu_HfY1tzOkQ*QZ2rm3XDEaO*~Y)TLCo^HE*i0#{$RG7&Z_dD zrg3w??;gsv@xZCaj$tPW3-#>%_(Bv8**%PF)f+8MU)8{X7@3F@pkTQDq@HnujFuJ`wlHx`pnh}S$!+VF{z|6-0 z9;T53aL0K}CJ{OAHNv)Q2&nVH!LJ&SNUQAa?MKGn(6PIQfn!*K-$((Ze!qtU%{ZQ@ zHFJnC87#V~R7pvRM!8YItx-oNNyx`!UQ^f*`94ng7n<6 z)TIDC`gS3MW*?Zh>|THiXdXd&YM2DPhS_1J-v{cr|54oWf|zD&Ji*1sM*zrdtiJX;WDwoLWh?TI=g>0 z@wI9NTW-L&FB#eeM}8^LpTl~q?+f1^Cv@rF5p8I}2}qN%+)t^UC08Y`Pq$1tp_?0Y zL^-fYu6T+m(`C#L?oMptBV1w4cX2`hffqutLUa$+NaOl{Y`tYvRoxdaET{-bgA&r+ z9n#(1NJ=*n(k0T}DcxPtEe#4tBi-HIyzA)mALG4u+z)=xfrowe-fOKn=dU9F=G1^C z)$zh-vP3;czbhbH0c(u?zt%~lW1)ve?f|Vg+5-3H{w}0n z^M*@%eugnd!K#q}ya~A91c2P@|M5nBf#mCq0&!KYRQzCOoILpTK{Qav)6k>r3r>cl zPV=~yN))2@9AOu(`qjfKE899dC-j}?Z#z(rmWV9lZH=h60!<5APxfW_`~r`~KPlrj zhW&S3U8iwU?VSDRw+}`)l%Gw&Qj{CqB$XckFmqs2pXu8QqQr8&Kg&hPLKi{=fdp)CJOjFSYiQNWpy{c%I??@fm64l>Jt zMkorg`#v1ji!jLV!I9};h8~D*{4rie3oR6(HPtbw`GFTW^|K5T8iW{*w7@bo1-#xq`xq%V0dm>WDl78Ork zROgqQTrjx`Ko%!>l=5Jw+Wcetm(d0}^+ZUHk5`0K`MvQ0^E3hv?uH$d#la4qIuITS)O0Ys>aE*f$^M=MovRSNoCUjdh z=pvFi;1DVKEW^t$)Ad z0I_`&r*1%c9;tKh*grcA1kIa& zd-wK)^WR>Dg#rs3baz)P$Wg_&ah$9{vIYab@C*r*T#ZGf^N`(^OcqO-X04Eix1*D` zPzy6TiZ^(0c_QVkc&kf_&=*nso|p|moc|-CS%ZQi$a(=e{lA<29tsLRZRht7r?yf~ zD0|)PShW6wQBVv6a9N%x zQ6KS9ACNYLt`qDabERIFpMI?l-S>et3sl}ocDY|gn{q{N6C|p_9-@d{Eu>XG{P!N0 z%77=Y1~c6CpAffB;t`{0^tm9z`+^dL)Q@Wp$N-oXyn_y3;1tqn9T7y8|u_t~Z_KLiqhMNexe6EECF5~rzkpWFU7JqhA~y2J?P5IO{lBBLWu{0Ep1U_JU|!NMbo zci&xK4e(;2!-m4_Hds5a?4}PkKE!OmD-F9_ zD*QHhhX_%M4eIX+&pKso-?kIV&A1Xs{NRQry*xOP+6#%;V9t@h*r$x^;jIe%KJYjo zoeL&j6476u?q}B#S0y=FePj+*72iJyXuC7@Q18(uo5NP;mFve>xu8HU>kA1==CMJ{ z)lu2qYbI}M2frs?JB_YD0dy_%vjW0T_qWN-ZA+7FLgW{mb0kX@(qyUHO+voMV z*=yOB4Nt@qE7OCoji5JS{X!M;+avfoz15*Bv$gam`V)s!i-qIwZf<<=9d{?}3L$cw zFRiHk-|D{2z>y=C!`y$pf|dWhLm>hKl!?GvBe|EEy+yRw+UF{<#A-?jI;k9 zm2;%XRxQ2FHi>puZ4P06{y{&Njy@ZyfMGPlysIBQ$9}@?}NAH@2 zV(cm(OQo-`);i4;iF0s&MrZU(=YoV$TTsic8=uZ5ebGVbkI$#$QCjcv-y%Updn6Uy z+&_$Y^WTt+T*7-E?%!^w5eUkO(yl)y;#mp_=HvNfgZEs{hqUd_a%GZP8*xDTLbQsZ zIE)lNaXtcZGH+*3;*XDUKT%=oF}MtauBVEh+={p{Us@EEj`J3|Rm(YN?TC(LaTdWK zi>fZww_WvfRmYn&1kKkgNsmO&1*67!OQm^C?MZl+$#5#(sm&I#OqkJ_M@4;Acqhc z=V`umpg}TBhpJ2F|D;QvkgeTwp@KA-4jZ%X(-M7OImo}5=Uh2n>2kTdq?1aZ4Gwuj zgF`0cd9wNv3HN>G(B$|yIw+;5%jq~zuCDrkSkmrfRrHTDu;W*3^KflwY7#S-JDaPu z#2n7xXYls1{+$U0|Av~B_oKyhc{otPh?yG>K@SoM5wS$`ir`>)`A(7fE)%o&M%??4 zrJSu*C4o^}u9GbJV|xLrm;kcOEP7C*xUN~(n5C~S$yPv|;|wlITU%QLu!OgmEY2(Y zCQnNdMIn!0WOdMOnJ^N?b`bx}e-z{AdxXWErsb)hFI})&5Lb?tG;44ZjSl-)MzLP< zHg|QgMMg8tWg}?>|2w>`zCw!9RrZ-nQhYv-AYi8PQ}QjnHWnC|u%y_6j-JVUj8tI} z1&vv*@b2nR7v_y+_8Bj*o9(TO|7(dxmd z?oC8Z?-uL5c(Iz{8hSFCm_ekvTR3;j=Y5MnBf>lBIeYZFHu;aCnFOVYT*}ww%#YUh ztmfl{-_+Sv&WgfgjHSO>zf%_dDDdp<@S`%$&@BRdD`F{D*w6A}9)yeU} zxB4VbEvtew%LR#;`zDX{vO2L@oN#G z{J|DD{C-SlqxL;i=+8;k=%G{g2uUH?56@E}1u1p;s zAQWB9;xQlHMx<(!^0GeTr>LTN?!j2cv=q-YGoV=A6 zpU343z_XUYClzWpG8ZoWN@iID**=iq!TXDNuDp4&ou0&2P^9*AD*gc$7qJtBLUV^> z^eh2mnYN4;(SqGD>r?T2Nz9)S(s^0iR(QIFJnixFi&Z&R(ZavazA5_ z?TRngK;j?y2FIa;g^BHxaL+_wbdPnpmPBa{o$>19pN!Yb$xObYVyLsaw;3;`v(JTJ zo_T%qZIZ?LxF46Xyt!$jMA0O~yiA+&kf^f5R@nX_2X?1OEPB!Rl?YQv6vyQK2k(Bb z$1C$`Qnrx7y#pn_GT0nAmQ$<`w)ac{H|wAK58S9Sf^HY{nU^6AL!EdBStECg(zLi! zvEh zUJrH!91&($W6^>(UdGk>3`UnRteWj9D7J4?m@ue!QeGdL?L6J(*{n4oG=L zK<;R`_oF7R>&^A23%C9G+^2RlQ6i+5ytm%LE9kwMTODnbn$->a`d8j_RR&%YCu_f3 zyV!1;kX*yIt-oZ*1}A?=ASIL?}3DUpLwx{Of;wN;Mv6k^5VU z#h`%plxcw$W9{eKsZRq6{TEBlpGQAjEchK%1HKT8omTm8Wt zD4_&6h3-4&{3zr?{Pr$ey)&*XzjtGYDRuIo`HqW%>7Ta=WE=&S6gxSb55_<58H|h5 zr}1pV{#xu9e~*YMm&Vi>FhJ6`_Cu$q(nN_U!C!W+PNXD)AltoNH>uu5^H0Q3}rYcuz$$VWk03XuDiX@$*?XYq5(< zD9QGj&A%3xUl|+tUR^pDwb-A;_Fb{HF_dFfy}I@+adoya)P-8e#(sS$Jm$^MY}Z8O zXraUxSNX>RH>SfvH5iNJM_nY9jvH>*A&hC~*u0881#dMS4=7LLNc%t-UZwtyRPLEf zWEOuo6Hbg*{5QsHEpyVl)jgknWyTD});!E2UM1;C8IvhYTns;;7+B+2(Glux@WaqF z+(?t>K(aX!L+8QK@ z$X@De$gP3d&MS!jiK{fFwz#8gK@<*rD-o4$OT$20siMoHVC5*IKXV{a-i*`qTM@RP z^S$y{N2}!z$qBdyC_2n3l|@rAyoNQ#Pad5C)QmbM#F32$ukOhJF5|> z`Noydg2?pcv|&car%N3OL8}5_St0atvRa=8Fp)Jc7uT0>u=YUngD_I z4$9;k`~yQ7YHN5If_Ga{6Bh^8e7olpLTkQ-QRA<^rn7F9ZvGYJ0p_S<0@Z*3lKj)} zmM{`i>e)AyRRSyv)*AI$+nl}J78e(BIBakyqI$+Pu74hl5ysRjK>eDnK?p^*>Xnz? zgk^}OZSZ<{ncUe}r7j=BXFMDbdI9W z?#_VQxAJa_$!_X%)tan@Io5}IgS`AB1d5l;jW~2Hr&kiKpB7K)*CX zJ+Ls8GMmOmV|D^vGC>@Q6VRgo3!fP7?cY<)ahVtW(IjJc{--gd8x`LLvwFo)oYm)! z(TD;f{eFD)Lo3xJer&6^^B55=gxh#LqEbnuVXx+x+rurYWi!s zKaTD-<+S*S{`2Vho|m7NT2S#xBlBz=(q8qo(Q*4l3&7D48xDd1!`x}ft~$D~+pwKg zPmvef!nD|2lpg+d+YheY8mV0Nzl_F9|Fj-Y&K%_u+06ewLZNg5$x7$<$0V%OdU$E? z>bg^@+6SLX@c0hDzQvohScs`2mAK!P(7L$9nI~AU63XJfNMW52GIe?#iE0{CbJ*)0 zLfr1OpZwB)_%jF)N>BWV?l6+Z&9VKX6JVdqEwF5eZa0Hw63n{{VGREbJlf!F zufsf-_`fAR0FV6ap&ac-``N}u^i%RIg>N9A3A;#PY|3#DNCA^s8bElN;`VFToI*IY zf#Ynq+$j9u(Cg|05bl@C>G1Uyo82mjhXbc$5YvDJktAB{8atkJ!M`~$Ceq&^j}Fy4 z`spiaqqIFng@RamiTp;f7oogfsNS${@PUo0-K6Vd;q+mVGLa{Y{0rDmAswb8(K=XC zlmu|t@-})`Jw&L?!s%pl(^fIEyY&{uJ=U?PD?g{6l$&Gv+Ll5@iur282UDCa0@L*_ zV!u(gd0pUynhy62zLaKMJ%iHl{)}mc6x~zsfdqH4nE>!PJ0rWq*Ulc|W9R56;ien!HB4J}QeZ5BB`5rdJ_t#A7k?1kM7}$xWxb6p8Lo?-k+temB*@^1Uw@XdjJ5n3?D*$o$9v=%~~lO z*6ozKz6W4>yfM}AS$2h1wOA!<+$_&VXCrhSU{c1bDw98R_1e8M?@394QQUalT;6Y7 z_((-w39$%VDo1+lhA+TU$K|N{3<1c>Yx2n) z=s)Yq#bep&j0v9TH3LA`k@GmgLxN%OUD|58gr_cejUCm=K|0{K(XhdXRjr(eo6b#G zfCJ!heXeYDJ@Er?Z%}ea54!0_1WP{az(;%AJsV}!|7EG15JGI#ATVq?@f5j@;hHr- zOwJi8;wr}!9`DcOe`W9!fbuGiyQ9z%D5bl=IXC&!ED}U*LV125G!z3Y{S#pIW(Z_C z{i&Sad5#vEIporLsC4qEL^gl;zJf((?eSZo``2Y1Hp-bbY*n z#bJ{=u|>h_W7R3`yWUwf&g`STixqQ_FIm9hvF)xn?7={@i1;~P&gCnQ`#ov3RL%4{ z9<%qZO!8;Pxn0DuN~Jb8`_zqsJ74=M z(x1q9cpV#yN9qC=HLTl)+lcbUYj%X$NbHhmQ!@K1>37CJ2v$0!ibQ;7=CrMYvpT~D z)AofWV3}1S`2ac*CK&I3AO6S#4;lnGu`fbQks&WuM+f}Z3KVO-tc`xfP;s%9|C1~E zXrMsoQWNRy)|#}MJ$kB7l3b6TcF0a59mmpfjppN+Qf^rA*i?mKb=_oHX_l0ceSp8F znC)lBEbWNEEQcipNjTIOo#c=yz3su&`v^!6)eJa+PPs(Bu-H2N4OPJ7%vug4ct9fL z1e(VbR#gFTw^O#dwnnlgO2h%cjlLhgQuGL2obT^kYsDOm`%#T2mMVY07c3s!tQEwq zRX@Y*Q0fZQ&$j`0OYg%E%@yb50YxNyJ#2Fj?7_Ec0!&a zkKV>J1#WU;-?`hoPRGa^Po9KT#d#Az??!O3Tdpg{Io4p_O)tGm(saUNG#J#@)gI${ zPr@TlpT=8pp`KY8J`$-}$W8#zLt833na4t%^*g7-d8)IRLa}70~ zb^3zpLv`*SFSF)_B((aN6|>jb?Csiou|t(i7#tdj3$azh&c&idCz5Iz^va;BP?iDho6~HoNvZb^*Y8uFj=J8zQ)j2u^6zTCB9S3w@;u#+p1(~bpH$kn zS1uC!74VjMCOnk~idWMy>`Zea;QYz`x?%KMg4C?l>jpkyL@zAT%_}5;F2$fPzij5a z<0}~NDXl}H*dCg9hYO9NmOwINFgcjiYLzt`?o_Ha@%~4d=BT9tzXuV1R~v!<;Q`mr z7K*f#8lgpxMZjV4Y3`&H>6K?rMIhjovhM8@AqOf3z%Y?oS8Q+x_Yp1S*@J9uyJZ4w zYyII;%X8}{8vWLUn*ZJ7N?z!FDg*Z`KT{(jAK?3F@Hguteb zmzNqwaL!0}BVsTc$*0bf^_X<~ja4IR(Qhd~b6Wkum=Y2>CZKh!v=|$+F}U0n$z`sC zeHt=c5{d%}9&}Kr6&sD|$wT#Xs%&)FtK+FqDT7b2(eNiG3o-XRlUzm3v=x#<(E+bs z_q*-^Hz2TMg4B6v-#SoVw~9=-YV&$a&TZ<_t^LJ)0nUf%W3=k)lp8L5U){=}0l#E|I z=A5`X_;1DF2PAT;_RpB z<5bSH9WHm_+4p<#W09D3oc)6Y^3g&c_sS?lk=kGer`!XN@rE;`4nHNFcDi02Bdo%l zd)3|fD1?WEDoUC(MK(<;17j-Y(msAPLOuuQm((_nHkC3O*?FW0=XScIaR6QZ0dmLr zgHK4c3;ti0L|`kt|E;22PB40WC7HWUAl;Cb=MjX1bOirUmetN#%u4sxZ*%oi#G`Sc zx4fcW3AOUYMN!+<_FfUkg;eSe#su0$tB9xM-k#&Z=QnPij-(D3lKkmMd6pV?iF_h) zCPu*+l)5ht_@8E$J+*Xd)8Ac?z?tFJqLrmhNl{vCnaomoDiXjnRi`Os^M;<}(!YU? zR&RnX>&;Bdw-n@FP@2#+eyYJLLoObrqB3qWl_kP93A*ySeU};1+Wu6sw`@3u4(Nz5 zbv6qY_QYTmn)r<)sdT-Qlq|E&6vAQ(93kDYu8;gd6d13#RQLSpk+7 zY7b$|_4gAreTi9$)fYGsF+bkYDf0NZt=Skze-SbTy&(RaNhvUf5kNzK-{koz)x`E} zOm+UMIW+agYU`6m>f6rnzkQ3G6gcqY+|~@gTMhmZ#3!z!j!TNi|7&jJlS^(37!5mQ zN?;b5n+_id`8ZnG`NqxASI+l_qn#R#OPki)^96x+M0)=L#yWt?5Tr9to=*A?KbmIWFXTmvYaj)ar!3rU;CY4L9D=m7GM?+CKU6rVn38is43f!=n-D%EJ*MlToEFKp| zfJy{nGU$qM`?q=CWqUo`5+;691%~He0TA+mjkITiXX55K`Y*)-`2}Sk>c%>iR$VEt zOyU;AL^Gme^xzK7&%eEbz@tK2KB}z&{ogTHi4U+wjY!Aru+NZ@`9d>h4evoRi>d8O z=S!=(Iz(`^iALh{Lc&15-w04L?SO_Q0tg;fL2)4!ccWOqVKWc00!P8Yv9L$59O_zM z{%030K?o4_SOLPucjO;Mp^=Z~yR1=fV-%|8T_8-gzxT=$703ebn)F786Y3JPa|^*t z>b>_Pk6h||C=e3|jGi$7Zz5rlBmfVCN=YQ3PrKFa4VclhmP%Egflf6G{HbmS=S0#x zVlU2_MCD66@xMJ53xqUtFk5S4(U=5oh!*Mb{Rw(psWyJib4}(G3Hp}SiL(+@`eoHj z^T;{-NqzyCD#Ld>Kbmcgsg7Z=qP=fR)%4a;I6%E0Fm{p%IOC&SrP0sMEf7E1Vmph;4;i3wZD9apdzMMr;sfs38!i)kro*Wg)29+y!WeQYC2DjoE-n(D@HB1@w`@{zltiuiGT7df z9;SdpW;Rud-EY|wz20BP+u(Tt6QWA#%REVqO?Lnap9ZH(Vx?RepUYOhNj?-Dsy{P5 zz%S(3_2;kzXb>NH8`x(5r187L1UerH-y)=Mpa1Do{2Y169dzt`q_9vR7uV$d_C9<| zK0Vu@cCb(-C#7_$(Uqn=wLPd@?>x#=%-l$U(PCL)|Mn9%Oe9W(6M-XhA(QuPtF@^% zpEH#5V`{L4qeZg1Oh=0G>$fC&KcEYhzPmwR?xyP4ZOg1KUWtajB|CgJ&yCI~D8Lq% zRM>eJ1gqThzh2Wx0#bXsj2p0QKmx)^et>mY3380gyEFMuA>r7zs zoVoP#-V|c)JJg-7XV~17B7r>0DqwDc1uFF%-E84Nq}dx_i7*OEcC0PIEZ&;d4bT(S z)_oS@UJHB2fdqbFvhA0AGV@Dd6yA4zvX&?47xNnj9$SgsYUa!SpYN7E>!Qj4blst2 z0TASTvokVqPM_XAcvzok>muxeU2qW2=_h%*yMAaUKs~*r+n#v$=WAXrxOti=6sL}9o9RUBX(?i$7k-?L zj6L4^^qq0_P7#n&p5@o9{Q{BqXqgm;fKCpj1vts9o86j6lbgL;|0H!JXbQ2D%H=Ya zAvezMmcw8Bq(D$Iryv^0IOV0!Mrf+Ww;!^?jMhpz<>G5AyVp9}{IN-(fZ!_DkOVqj_ zT!RkiD*`ci{-Qq<=<@0jRAZ*$$voIFQYuoOB?x);-}Q|8A1>{W#^ws<+U}s4Ey{i> zwa=|hq*T{*`y9nF4m$MkgvzV*0eo2X zJ7cI|-YT%6X?IraVtd3lJ7E#TZ~BWFXJi(WLB!)e=+QS%uzS{!d)Gxg0ykRMc0IZm zw0>XKP=cnE{YuaOb9IZ#e6byP+|M6CvHi@13#bLL86#Bo4-U%cwg4~jG_{4A{nKJ; zyq3KX0TLuE_{QaU*=PPDAf~A}_+?jD*9nL#u+WLN>$I)d0bR8LaJfGf)Pju_;fMrT z(Q5$W2Mtdw_wH|os_!O4G0po?ufk^2TeuPY+d77mn1pfOQ)+JmJEE~K!dwvX%_4Jg z8*pR?AU_=g)vrRpPO*sZ!smEuMVlVh@zbJq_TY3%lA+uR3e9_ZrRuq%?aBK;F8~bR zWhPW=LrUO*GWtk>J73jT2_Ja1pE#}Rchbd{SE4q3w6Tlp#+OIwa}m;z?SI zJJX6;w#(ibpqh%nN_6xxaF~=-_AnG~1_??k(?SkVyKu4pX+CENAc~E4_qTi4C9XV$ zL=P7D1Vmj>yee4?>3|0p)A+t0Aec+Cu&$T;*Z`6wZ@vXd9c*1M-$eDnW19lr6tP3z zZ%l|_#k?!#l?+JIsM$jxw#o(;ZkZ^=@KD*>)L^m{dqGc6&)mHQU{Ij^$eHR@ls{Z- z@Q?qFIk@4;T8UY`F&2ncWZC9@N6$`s`^rnbJoGq50TjNz)L`b~W=Np=5=uUOB@#|K zPk$r!M{AYIBeV8F?AM3$v_rQjI>*F}|C`xAod$?k@!dK_p({3K!X6 zg4j})Zxm;V-EfBg_br3*-HUBfA=g>QD%wv2K>vQCvS0JNRF+1Y&l{?Z!v-gO2-A*4 z07lD=-vH1EJE~ww(VGX7Vz0f@nWq^N{}EEZT)+W(>+P>HK4Sl@!t8*u!f88BCHx4G z!+{N`WHDkWw;xeZPzaluacSs1ORRwq~ZkUmEo*5yDfg(3n|{=Cj4Bf)Op zZ@{44Lx4dYRA(~%McRU%C2riK_La~2ThY(y2yv(Vo>C@$5)~_o!jSeOYutre0&%Cq zo+!zspQ{|i7<@AYblJE05=e1^Yev76m&LV}@YS!N^ z7(^uMB532Q#JK_X0jXRbGJ63CZ!2`$gl0dR8xv1{F&z=?t+sG{!w_p4nX&rim{u`Q z{AqEQ3Jjnk_4UqQ`L9JOG&@kpSzheIVM|63@DMW$y#M~`j^0%_(^gPblsf|Vy4FdK zf}=vGFWU1}rODOX+%Yn0uZvYOuLpDZes0TCWc1M{Wp~axMPYPI(GW~@n7D{SwGwhZ z^XcGlVsfmWx}e2ZRLF`at38HuaTlOeuCxPm(&Q_Q;Gj5`!QRBq@&87IX0S=#^^y6g z>VGhy;O&Tr3Tqxt^JNW{ZNbHWk7+WULUu8v#?D0(~CR|)BB$B1CGe7*Cp!P z!=<1+@p}E=BZkZLLe>$7^N~81;lkxXWZ1ambPvzH31-5W9P%ZOP;1WuiL)#hHp3Y| z%~|~VQv-dWMH7$|F0BqPJiuLadA?#OLm(@uC;WRSRKDtT2FdFEcY-CEWWruqG6a(5 zD#r_$+MgPD1BQ7@h-4~z{kmj+n}zSkX4w2OrR-Mx1>qDU_f>^VkX{QTNfyCxM6r8v zx$5TV;+67N;-z=~>`@RB7GiQ+W;913`V>K?HD$0AuYB!zqG#`nE4<()`%ZuI@W}|Z zY^U%@^hTr3I4vuMXC#ReJ%8ls`K$8OUOzpGE|T!TU+&fR1NO3sn`)W1M6No;+Ss88 z#JTgzSf7h0i?c4eC{8E@j)DUubJvzgu0DncfPx+(SpFyrJ?a*+&3C2Yz{dj-Wq|6L z6bZsTL{k5@b5S;<Xe{sYr&+{?Jnf{ly$!i2olW3KcOqM zIgf9SeTe-4ASa474lE<`4nrDWZ#0OCj7^t&x=Ve6O~jy5$kEq9YDf+utS4X!bzOcz1(pL2L!Vl=(S9|b zaVF|_gL2!0U_p7e-BRNYO~r#~TXcg|JQrJTmCzyU*>ct|?3hlrm&H?MRADTiPUf!l zY(GB<@?bP~X}8)+A}Mzj$c1&KO>QWd|0zm4-wdkq@rRF2_5@S4N3CWkdC|h8tSDXV zmC??a&o3^ImKdB@v+=4R9l0$JbU^QZf61CiNp$cr8{`G;@PgZJNBwhzMwM ztw(bMM`;Rk0Am;o=!a6Z)5KSuK!T}Q7`2*T5l|DA7?dUG7pGbg-mi-nfKTxFxT@|L zeLjJf#Y81Yd(jL8SjoR8D;FTP$SBd3F@C3^-Qo}uAkg*XfHVK}wVcOo^+sL%?v%jY zV#Y_2E!_s^J)}BXt?;)x+-A!8%rSSJ&rXgHdCOAe`b7kZSksfkVn_4kV#Qp1jM&D) zcfQ6%$FxjSExzHDmN;B&3cqu{cg(we?*C0eLQ>m5j4_Woq3e$!1=HOY0OI`XCNEGN z3dH{mzOlRQxBGQ#o`U<8htGa}^xgfh@Vx?-ne~ChkyNR#ah76eZ#2H152lg6fRZ}> zrArJa7;^`xT#{d$GpdH}I=}zb$C9!_Rx|4;>3*b>jm=^URcz*%Q%&*xaToI2 zcccjQ$7I5TAQ#CPA9a!hLIi1WIU$C0b?Cg0Akhm_8m}wVJXjabREe+!6vDhIz6bw& z0;t*jjOn0}%>q#`yXV=sRHP=$9X?5n`q2Hno)CuHxpO z%mslY&P+#8P%s7=+XlgTud0SXYx?Y0Dr!aq)|b!NV>ykm&L1e8LO<|2GZk~33ieth zZ=8<@iR66pUCKOa_@wZ`?-ykoj*4g&{vdo$vVF{#5%aNvH_fFl47Wv4*~c==YdQ+o zD070hT#&R{b*;@Y{yf4QzL% zepe=rh}m>J7y|h0(#EH-Sy~LpK0RFN#ZjwDvFDLGuZs$C>&DT3Km&EQ7*JC~0=z!R zNyV8t0#krlh4miAGEFvD**Ue%pNaB^6AGZCOO6y#X0;OvK%O1->D?PIh-B4V|COw4 zGa}u*5Owl3jA=Jf8C|DaLu9}JupNaw)A{fmx1zAPtDgH?7;rDtq)V;-R$F)qq~&Zu zAr=yUE1y<2#OM6O;EH2e zz=g;mMysdBLI_8`mxJq8Jhf@hAzKydV1W?#eB}%W6mx)!tE+kX=gU@{bY8G1^I=It zO|Gz!GMGfee*PV6osbCdr-)y5x!${RENMQ;cC;W(&oDw=NN012!pjpKA)ICc@D*ro z1vmv{g7vj#rBvR?%gdXq5D5 zEfE+XMu2#*{zFRqL>>*wi~-QL80&rNP84#kqJiCj&r-m3ILoxAk0juyk8Mz*L~(Ug zP*P|9rBbeq?R|gp3UEI2<+P;z%x5w*%{5FK)CdR&I0JztT_%9x3YA3v6p2`GQHpn6 z6kx2-5Fj7;>cNC}D^|Tc@;8)KJoSYy9L?pEBX}QzSHhlcRg0`prKW0qV~T_?yt)4; zio3mDEkS|8@f$~PuxH@OQoE&MmB||qLi;a;_NyKGz+4T_^)t-3un){6dHs6m)ItSF zhcmvjvN(msqxj!L=gDq*F6-M2_Li9q$}p62q_{@TnVG1`J`9oGGN?7>YQ8zBYLcgm zavq3~)lw+1^SlniGm$V;PZ1|}mNa5Qfa3DnhQ+Bf!}&m_Hm%(E6gC9>oxL-hgs<9K z4JH%f*W|wlo34&Txl3R{ch$kEbDbRR_4Yp8ynLs3rn(R3havO)Q-#XcBX?41dIPK- zN_>Y(`9){_)T6g+^=T_76`q7z${y&tCP z_QSVBJv|LdeJ*lP0Cb^ageO1+sW6PFs#$Ln{+3SjOBigNc6{BIYP&6iMx{VNbJS>; z0$33wU9OZ&yXeuhF{jZANs-i}L%w13Mz&e6uoe3$g#s|xc*GBUBS5DgkKDz4kve5) zQ8t0>56YPS4m2Glwps@9_X#D6*l+e!`ht01RV{BVG1}c|t5~XFiu*;cUmreCV%_G( zKx_VOgwn`^zu^95ZrUc>D9;{?u5Mn|tcWg*=~dlf=SrG8Q{B;Yrj8l2@eu84%6#C_ z#5d{K^5r{uR zz)N@}Fq?0BsYa(8L9@lM+UzB);4+obpFqN0uG17IesxiS?O3X^`effz9B;RJtmrZA z4g2_^;RWz?ke^Op7~SuhDozpYQ)kdM7)|BRJ0uwQ?(5ULoCy2+*0DqCgj}H64ez}z z;o^hsU%@s#RJ}}vAgJvAP8)5lpmF)uX0A9EOGtcQZXhvXoYrJA5B6l?h)lT6UQXL? zVb!KaU@Y(A-swPwtOEx|+?=3u&o<#rK*d3vkv0 z?>+=Po$@wa-r<54X?eFX^gU@9OU;^$qCTPdR8?8@t*P)W2=X>;tUWSB$Hx{Mgu&cp zZ)T(VBuScOi*qVaL=>?}dYTgSBdo_e+iq`jOv3qK>`miBg?sTtzZPNp{ga3m&bz^S z{)aD}Jt2eH4ixfh`HZ*o=)FQQ{Y}*h3F5Un1+p1(AQ!$s?RkcoYsrHMUrG~lwBz+G zs}C5$`Uu_pYMoBD>B;*$LH>6G!Eq`*FGJKr6pZ?#GrCz_(= zbY~NzT(Uo(r;HDmLW!KBv)m>($ahGi7#YB z8FBgq#CONh%cK;?rmi$x$VFR11?o}_wDn!0TFpEM-d1n-oIOJ=MkDXEtfpGSvQ^%a zd=uc%;P?;#o1^HY9MeU^5+w!kBB!s~lYv(^ey8uK+l2bY@lVEBj{cqV%8~8Suuk$7 zd8VFRr6a$heqOfkfqLn?kdTed#giYEfFj^Rmc4?*L^Exd; zU?PJs4GD??GQwAWK^YiCJX^1!Yyqd6z}BoD$1sx*bMi}Kj=Rss=0-#p)gh){piSd4 z|0p=-VRPt{>m4p}ADraANdCz9+rN(|4@IqT5RN-5K{NI0A zgn$xiCTs7SR)1il4CX>cd` zJA};4KD|gs{?|m!&_XRb_L_V`l>e*h{ks~0F9$L3wXo~#{fBJHbo_yBL3jN)Rjxzh zylG!od*=2(eJ!}8<-!*;%H4`bv=yQMt`IHm*#Va@oSHcW?SISrZ@b0_)d#VQ=xc~{ zstN(h=4=3&J29#M!?y^Tp#V}R<5%(j`{0jH5aHwFQI<~W$Bn`G>!qj1z;*jj)#fB- z`54Hr%7g$|09nyL-yYS(&d`W_=DHm<-{Ly%=6foEBuz++>td_=LDyvEdSi2?QhLMx zKWObGM3ezVsum2a0fJybQTP(@MqXJP?`&?qy&7&HgWMUyU|6fAHaq~72!fQ*STmSm z-qMPS3Ww*FE0cN)$f??|{aW8dqI^L>qF)_1c4HuP*U$8zAynm>)9lzY?T8T^Oy$FU zwO)bIepB`UMX7{Nsn+Mg-~99}{Tk|q8ajS9WqJ+g-`cI@gWy^jjpJb;#8(u2m**h# z$T5YDdr+uLNZ8J04quz!8_wBgUT6wew_mH_8TCYAi1tk^Uq)x+T7m81jHEW|*=sL6 zzx_u0Y^9;%LXB4VF;j(0wxj7AWzg4PubImJc*RztwJ}wC>&^+7gpnbW)bGN@dw*nz zhsY0+4#fGcID8w%Qz%t|>ms(AyC#S@?6O-C4M!{EBUFl4&d})ZO`^3gj`Q z@_7m2;7P~P`y(wh5ekbLOMjaAp(a}H|4IgIyJv6B^6ufGv?9cA$3vRf1E+iuwSfgLYZY}PKK6i=Fki9I~5rd@v zz6p{LBtsH*X*AFVz%@2YZHVgX>R_6Vjf~G@drrpPeX9%8(!rn?-d+10v-U!p*8%E| z3XR5qd zG#@K?ZzJzDr~Li>ea;66^%0ZV%(>ov)I`~Pq*=K~?x$8THd~G%FoRh)BAY0ORBR>0$p`7?L z$$7ZnPkEr}Gn{{a63Yy+)-i!&y@8iV!*@AEp6AQK5*1o{w%7MOgMo4GS9VCx^DliI z{)wX8b1Wghq5Gvx9$sYbeO{qjvlxx}pCpVJ_4r)HYd%aRbsHk5{oFmw{mlO-z`hVc z&hU?H(NIS7nCqgTo_sjJ_Sbzg>Ato{Q`RoI(xGYu&KUT_{>SG(g!M80M9_4;XTIty z8k@^fbYDTod*$l!MC8V9j$E&DSj8S9bMJ<6`PvX(a=`>qvC#=J7ZV>$83k(6WB5(dvaG8{JzqdLJeRBO?2-_jB)L|MHD(* zao6vI$5RAzw|+mGJ(**2a_ojY2C|ti&8ir|t5T%k{&9bX9uUcVIuT@EImlK*)LO90 zU)IWupi4sY_K~i6o8JI-_OT*|hXKTH(5LevFYR$u9xkbFbAM*MJeUc;BmPWYqRuXp ze)f&A#wF7r9yyBI(E?$Q1%;d4`Ul6F7zuE;WReZuGb98yRZP!TEyzCxJlhx{%z=g5 zIALTB@oPt4Ax-ot@X9`Hgrz~{M>1g`?zHOeTBbw&q!-F4xY8hjPeBTGCEBT@qhd2S zA#kW9)wsff{V{zW&2vfZ^qUb~;rYR?Nnsv8GFrwDsuw}$yx*U@4nmu=?CcpjI2j;D z)weJrcz$%Hfk$-v{M>~SpU9v!xi#K4rzSc4&gDd6MU+#kh3450pAc~**Uc!KV7E@` zt#n2KxYm{pj0oqRCVXvFSDmrvA9^xA!y9I!9nJWDp%^iGFWbsK zl-v0!$GdYck>O+~stuUQB9%oiU3}BHozv^zkAI^VhUF(s{LA>y{!KL>Eh)wt3_{}8 z2L~hJiWT2lr+m3h`EO*Qq489~$N>6^Q=6!sX3*7-fVIxzAq8@0!KyK`lRue!{c%0l z;5nR{5UWF^!KJu1@N3&91~#=9ESag*R}=g6^%+ z+p|z#(CW}eHoKll9pCZ^na5KhD>PNbbZvC9XG)psZ*vl`E&YFVeRWur-}5%Ci=;FN z(n~I-fJk?Dcej+3h%}2dNQWS)G)RLW9ny`YAfPlzNh^XDJ z%-nO&J=lyX92zupHx!3wDV-5rie~TG)IUT^kmvl>n8*wP?XHD_(#dHGhAYIWR#-i*gdlwUnintu5X z9S0SSx#`w1<{8t-c^#x}dSh=S$$6-qd_FlQKiXE6{~C8|r#fPN8B!_tASEepqX*Hg zUuX0BuvDIL*R<6yv`H%Ko^IPl|a|z)n`Tx7iN+wj5#*;KjsuS75c8HX|YFp)b622JrwcB zf@Li|g*oTrSyUF+3*u zhRYC^O`JSsrD1kNU2;X9bUnGxH#sH!QaT+CoJDzw*Fa)kw&rjQANDP zs}^R>K3GfZr$?;D3kAa#Qw+J?aXzo^ev_j%z=RtvsvQL@aBk>UMI|P!j~due6Gusf zWgF&@eJscdUCQE2th#uocH`H-Pafaja}+!>yqk}oqM~M zqlF|F?+oP9wbqvDg<=--M^4XIMgjXrGqH=&^4$J)47S-V|6Loll)k*BKaafEUJ84k zkWr*oN{8=%Cn}Gxg9S!;n@8j9%MGSXGXBraEOyc*{IDY!sO@|B?$MRzNmV*J3DZ%v?(=-IjzmY0 zu<7Q4tm0@3@s=BPldh^Kq}wc-KtqHRYZhhj$(_N-hsOpjB%;KUwFYE_{cj|x2`aN$ zW#g2aNX8PHa-z<~j_gX!brQ68eibX@aZFH1_w!3o<2jJwl9N8m!tYHn)c9&caJF&w zyjT*qW@VO6sM9A6fnDCc^DbE@P*2A@z{@%)7A__|z0x!}ifv^uLG!K|OXs z4gYTJWYm7My7`E{%_y@jrs~M+d*b;ioY8aBoA))c73|;tAu2K6%6=R1N24^soqz>L zv?m@%%^E-NL&Pr~^_>bD&EEHHmSeI6q1a-e%DEy8?fD8rMlz8SwBGp#gG>PoiZK>H z7VTEFuJ3OFnca)InEbKgE2p!)BO;y$6Zgw#G!>#-9(<8|{((ATk-XG;G*v~Uev99A z7(=^4hibn>vlH~hX#dg>B{6IFn=Zc2ObN8wh(fzquS}O+$8?i$9R8Vwe*0&SXpi3m z16f;At1TSjcrGO=dDy4D+Q-houkVL-|2t>mxGd?kx>TfzLh!5M-9q^U$(Xljc!3lJScXfr8IM z==kvof8d0S5@8URby0NJXUaf=5=FSez~Dk|#Md1K(DZfkJ)K2$QFmCzauuf9y!)7h znwM$jPY$QFTeV{C42R7q4bNIWV+_Z!Mumme>Fo0-9Af3pzJg(D@e03epU(fLsqoI1 zntfYvzU0*(;?g%z=t)s0+}%a5BNEdbmG=ldzk!IaY3rvt z`3cCGZ(kzGcsFRGk)JA5DUf_Z;)Y6Po%~$4xxr_lFhm&9oLqqZZcMzY$Lf?gA#^#y z$&3LaMpb28g~d)hQuaaSpsg+Uy`fX@!mDl87M~*2?fGB00E~Ad7 z6rgy3=aWh|k2|DSVV#}B?>;R2LWJK<`y)Qw4t`+RJG)6zrpxtK*so*6EZp|l40?6- zZ%?;+)&5U~2TG6Cl{{&~@u(-y7X8PK6x7kLBl&*X#Y^;mZUe+x3b*KA$LsbK8R-4;ucG`pLik z-Nt~r(iCpgbNa2?8g1|J>hc#vDFN{1?v|^)kcK44qYT;89@@AqR6n-rMgR0eE#J(y zxnHY2LioC@pSFHo*S)mA-?_cwRH--+<`ST&wj^lg73RQUr6c)LbJEHHAF7Ctg&B{6 z86R|up$`8p*5D*Ci20>bU7wY7ie?S*-O`8Jz)N0FFa-91{ird|?!NQWUrcI5EaJvC zqb^f*1-yp_6NMgfJYUuIDHrUj(RG_PHMtusv_HA-w`7WX5Au&ZkE{0Fo7Q3DxeU)? zoKR`4i?(+kt#)FV>Xl@fI&{<=u)x&%6_t?@N5UR$&~E5WQP-b%Ie25N@Rs6awxYhW z{C(C#qdMuY6g2u#9=ex=aMO2dyz*t9gP6fM4C~_E z*SoqGnC4pM#4umNg0>!pbyUhvV%BHcR%cI2w&Ls^Mo_zGQ4dEop^8 zwdTg?o05Vot*G+zu=WBfuhxR19#RYwM#{VCFScv^>c%U`JX-DQ2B532lbM2yy5*+#IK2D=umHo zI=U%)=Ki$zjc=_E?6M8?MdL>MoJ}#%3FIg*8B70uGD#mE(#w+do~20gNc8dZsu+K} zd?&*uUFw}VIUtF^+80NulW^y`5GWk)44B;dbQma5)$hJY(hb-BZP6vq;GG9Mu;3k}n&o9lNm zH)9wA#DU_}yGv$+x-1W?3wS{5Vs5y)gd=n^06Z;AY47L)n4obkl~IC-4&AQyq8EwWXk{I60W>R6Nuin3H%Rz3I5@ z$GV5JpKgwwtOUd+)BKJHm^-Pl{&?ZfwFkDp&;}J#ZcGbEgt4Y}XoN`RB8+iQht8f^ zkiPLtvxE#k{2gpc`-Q6cSHFY|6`2(^sOS^?uo?Wav3vO`x{4u0f0FM^kvIsHa>q;c zBonW{g5I{k*g0UM>||xfe#+nnX~pkrD7Sxgh$O1P300|;;!WN**hj@l z-1-l&Sbn`-K1|D>(fNHG_hI+RU|W`(dk;2xY1W^!#Pjw)#uZNbYP)Q!TU;u1km!ir zXHLvkfjPNws&NC+-C$>>1KFoWN{Jmsa#Ok-b@{tG{ia?vK0L2!XM11&Ug-2l z*SR=+<+DNFcbH!N)Re_NuHP>bDw9nwq7ksZ7rR)5YtJ$ZQWrA646W8G;9c`yzpD@? zFCoUtS)#A8EzJ)2=?t5$n1~~gSsy}|mW8K@)9awTQmBlgj3YA0OMcuLzvE#1VZVRQ zqfAJ~==YsKHjiXn(jRE;M`Q4!I)zV+n8`vBxKjVQat)&{;sB`>^>s%?Hm`MWgO>YN zT^lhZ^2aPGiT>-8XK&ckg_Y1>n*;uPLCM0(`yZ}X67|}JEyNFXG||d*DytST*n{p0 zU%#oL^;VCi;%Bf)=9Qu&k-c9a^lef^>I0TT)5uoj^5DcPxwS-}*Ltb~ny+7WU{9W2mIsn`aSy~;Qap;4 zFJ3O{%iUIpl=2lRZgXEu-(TO>GmU>AjyohnqT3jy>C;op)tt1u@G+$%+IA>?s%ZVB zM7qSZwy@b!vsj(Z+vCfhW3z&mg?6tT(sWt8-M4Ny&nPe?VX4C{d}11H#b;igSRFaNc^^vmPklZZa!3n`H>Y`oCR;B;MVDSJEW zQ!10zaOxwgBBxNx(I-FRp(HRJAF#plqdY46RQN--1{PZ8&+Wccwc0neCcAU$gzwe` z2X}<}iVT+bOdeJi7%9CY0Ex$g$n9PTr3DaJ1%1-+CB#JDIqELc~Ma zYy*OZpi}qB6fTCjmtb&KBkeUF*3#BfiLW=s`|Ln=OF&C~5?}7?D3o4G5BH7Tj(Cb&@)NBk z?b?`uOZ52GOhvx0l3EoX1T#Le$PD9}D1BYvQWWs@piU>Lzhm@CW2E{ct4z7UncyhT zAKpg}TbHAntV{0SFG$HkM_LVfRNmWW>n?b26c*2?*Cwa06*~`2OdeVkI-qZ)zmIM6 z6W#8znfL$xj`YRs1DoEbP^^T=oo^Uup%}d(1w25eAWyQVQG78ehz66Gb2UcT>gQci z*}>_$LjJfsXo-5C_x^UVtHN}{f}}!}H5ax1kUFqJdD;w2WCFx8Riw$lGB+l(z!R z___c3Fr*vnXL{s)yq2pxUQ^1=a$T!M7{Lm8Nzu02#B{7)hBcTLaXsRrjaadMpMlc$ zrxD%tVMtUc2B&;t)FO}9zA@&EQB?7brt2{>N&Dvqp8-+_`b=KFplF2-5=SOf{x+d| z_@W$^bK{}URIU8YB37%nrH{%Vqu1eppr?E zM2;})DD^77R}3u(2<@U1Vdl)j+#c%gOaO_+KR-cc$n@fVxl=Nstm?LJVg}ky=hFg-eSDArX<3Z!V=t2L>1iqVueA^)6gPGPFB4gc5Oa@v>9)A4h_W zBv8$Way5HqrRS+J)5uX#D4ej*ZogRsgl zl1xNOL(-ApmF&sSRVOb%G6!GLKP{Vw3ArBs{JchyIzEm@=KdtmMT1B(A{uaQ2>;ou z)hyM9(?|Ycp?hRANWA@}B#D^A@UfyTW%kt7YB)_D0*~wwtuXUSpGMr4pF!Bz;=}GQ zw({pb1CzC;QS#J315+~Pj*G1#1Sij9p?&FZ{T%Ie#@JoHoF^a1<9)p;|H1dPHSZS>QcS>+e}i0ed#LEmg;ks`tk<|zBa_Ug zDGmpM?AaX65>iTl$-t~6GtoejZidmJXnoqnX!y14K6^oZgc(b%26Fo_s6kE|+y^F# zqhLaVIsn11^+mp-V*-kCJpN9whMnY=r7H^qeiu}iL*J5BGM6^5ULM1Y9rrb`|(MsE#Q$F^>OZ?PQ2d7J*#p&O1DeHJDf6XiwHV>cNd0 zU9HuUYr%2oCqXSsiI~9uAcI4=2);qQxHUeodF~2&I|&HGN>~STj^FqCln4zG<|yL# zoad`jK}vRAn}%-rNT6!JTso$@)-*>ikX4B159tN3kDuo}ybeF@-o ztR+!&VFD(1?}QHiIcU#O*d4~QL(ZruE5wIu7pwED(U2h` z8~cOIl}}0VBtLkcy?HGxQg*WBn|V0ck1e9ZUd5g_X#d@zeav#RcGLg0niV}rTNNSK zT@17#--A}WOm`YoD48b{gcDXI4hdjDVeHG0QL3Hp9=yCA|0v%OQT+&MFCnCeH{%a* z<)o0eR}(!7c95Q#zm6laMc&qmKVbL6M+Od#5#?IyTvcV|n8q2nVO?=OSzPy+Z07lc4eQQ8rq*DzFqa$b2;+SFeSBblvXEfc$r`|41($1AGsm?EPpdfa~>VbmU6B z5+?Nl5UvQTjH&lJ31`WnJSR6dlS&3dtIug71Px2NH-sM>oU0td@ek6e>%}Vi(Z;?% zPBp9W(>B&IXxWf{O&lN7^-}qNE?Gq5ncwBH%`8K<92qKbfXmpw$EENvpI~8LBz0t< zDXmN5O6^JJK|W*r<5_60kgpXm8$3<5&JJMpraS3T)N0mERCphcW)jak_B__ODXl4mR$`cQd1k5gvvDgAq3T?{nzD zb1sb8Vw3;<9H3xK`}LzsnrapqISlj`PuP<=n_xvA?bc5k%NAnGUWICD5{TXW^j`(j($Q(6oEr%w)OL;XH}M8mq|R zOuzidxD@cxe>V2reUGp}xyay9taw6M z2kp2xN_Js*H2brnbNpbZE(o8_>(-@n2SgH}B+ z1Pp6JBB}2FM=gGo-2W4>WTD(Do(NgKK;A2?P@G>+y?!+LUDh;P7lsNTFo~G@ja*tu zHfmQ6fm)Li#(y5cuyP-w9d&`mMuCK*!4f?|AQjXB_?r*x1PSAeF(S}u{2rxoR0;n3 zapJ{9^2*9z!t`sb2ZC@@6cd8J+uHr}kq8(HdYS*9vlL*ZXr)SLe^1=Mo)#|!DNszA zbhx_^{8m)P{bi><(lcCPfc3mEsa=dAQ%n##XyM^gsp90jD7j znUJUD)rMk{qJ`HrU1g29wl{l*?(3{6vR*NU!6a?Z7W|GbN|b@{svpY zlE5wIn%foq^S^HxItqAh@`pe}#^0Ss9%y%r@d<}DW2Mm$@CR%N2&3{cg}hWv1Ft;= zz^|QIO#uzgF8s{jPp|F0HubmuG`&1O#y>z~ z3V3E7Q-|}RUPSzV<4k#?m*B#2{y_nR&YTwN22y*I4w+C2&p&ZuH}Q;0smKKIB|%V1Kj=xJHhs16^uKEx1g&G-Km2q5zwSX?9RsMp znSqwxjp~Cusc@fRE7=Qt^DpU#O}do8Is6mggPR-zVi3tyR#zObaRJPSqe2JP3w?m$ zWet?0(^D;MpAvOU7Wec>1Nm8%r+CTl!Lk2$@JWb44#63q;7$v^6A6Wsl$0cbUltsc z)D1^fw(`@ZIt9RRdrKPZ$ZwAfz>NaZS*;&H#ljP=o*OhE3WqCzkw@iO-5=$_v>V=a zjLNjw{+#7$5#KX;pnaQf`8i4yGZKd_L1@3RyQM++KaWNJ22l>yJCTHLBT1YI%4yod z0!pq);BvvD6#l;Y$*M2*c&~PB!9ijbfi5luSoc45;a%K$j(>bw|ERO<)j``IRox2X61Xg@UIn|Jsds_!uVJ3Vhi0OV zT$OQo#Nlf9<@Oc-1Qi>MntT=YsSB9Sa@tU}zOt_+mH;0ssl={BG}N z=`YXQZ#e+z`t*}9_C{vVs{HQp`jGqKiiCl8(LwrmH-Vk&c8)HJRdS^8Yp-+u1bg2Mu(e@cP|N7K_U29P`ulc~FJpDjL5>;ati>sLCHOz&y-=dR>1e znH|Y7bmvyd%!mB=&N>RxG-)A!kHK3RFg@aqAO))oB4HeRba`Vk2Y zdy@CB5k8e({)o7^u@$WzUGc1r9rONpxak2;8+6KNE*M3aq?3cOSQl?H$F(p1?-wXFQ0P2Epu*r?>TA!(`z6xBJsK>`-(aa~n3&*gwW zKQndhL}>6|-dt|-%8>61Va%lImS}#S;Z9xp3;F!l$rOQ@eE=>n6U`4fGz&T`*F~7& zznP;mW=YlbJM2WCEH~hBB%uBiDgG>(UO)C|LXKwL#~L}=juJGI%>JxGWD*69Q7?K` zww`4ydd`}VKM1BFU{C0}R*E!+9R3Ue3Fd$NV;hrr4y=u7@T@CQ+}*wAYE$fX;+=6v zIO>&=Iz+1D5T2;xGJhm+3^X14f5eGp>m-@3i=B*19Z&0-dF!NxLX^zEwRmTAL0c^^ zAG9gy<}e_r?)m;=Mwox#I@`?jn|=Z?lPHF-n2eBw4Xu!Kdlvs@e$qgK!_WQ{<$(wG z@b8r_XkYJ9K;UrrG3YmRB#^^LyE918=84-nwE1BjqwC zP+A(8T8BlsuPGUg!XD15TV5Qh|3vo1@tVR&91h4&V1F~ne>){>nV=!N8d%R&1QQDL z>m2{H^x-#L-I>hq%z*}x6qrimPPxiZROF}5c}dRz*-(N z=KvMHrj+e-j!wm>gTq6wml^$wf5lW)be66@e!Xz0}UxMVlOrb2UDEXJT@>k zW0MbI<~p$*?ND}EF#d1NjnDut1N<60n0%ZDLOyS&SLB_{uKSAuBb!4(sCPmvyk7;1 zT=7C!WrUN~yHo9+3}c!>+Fmwd70mwL&cFN-M3K8joiDm$JAivA0OR|( zofjT&gV{lIuXFCqV4Bb4zWj%k@k23&@>o{Yz^y+v`zVp7 zfiwI5r4kxp*^*|yL{SV*!1c%e%UYQIeHqS@oSha2jhE>TFdN4RZRh-kyX3ns-zNGg zS}*0m!aqh+YCbFAWbu-bQH=bejk@Jfa?Ye@%uHxB{&hy5vl-fTxjMLHOn)ir$!At5 z!;ElkbJ^Bn5Nxm25OKU9=E$&4hB3hffV-@LR&Ykr zC|gA%OwYO6{SZ?>g|7D)(^&5!z2E6lS(jeUKR~SJ9kI{l(b3WIH{SyVW8=&xjmYV_ z)ZS|e;}k4%(jZYy=!m?AP02=3RA@Un-^%dR?AOt!v>gTzU#7-5fnrqc(MMKP;w&!NAlT(+QUQ? z>NhLmBr~*Q!oOmWj`sKY!9yUaa$bW*icJF9Pp0dg^EuhdGua%b{P4g=wt!}O1+NZ4(R2P1zQ)#_`U#wBLcc_)OxhT@y<*-U2*db9;FINPtta&#OV$NT z&K_s~gAIr9AzD^N`~Xq|U}2Znuh+TeKpP|N6b3`Zhh_isB|~p%j@1o{yFeuE6WfOJ zSRRf<7RZ7eR+I1T;yW_3dZ&zhZ#y=cVA$;wDTD>An|>9cQPC$p3i*@MJ`}q}zIL9$H$-PUp1D`9IuT^_ zQIWi{d%i5@FRtio)|d%SFwGdYIiHo7BWiq`JcI`#VM1xef?`ol7fK(WS2skp{EXH% zGWt+5YQ?@SA^PM(FYlkQ*g6M=L#AddhPNxZ1Biy%fZ+iR6Jh{qCZyP1t)wtg$cDZZ zw@sz1uK3%ek>sLRiV^ z#30TXXGT$aO}0+a^mQz>VFi_Z@=$Uw?)ID1gd?RJKY>I!_%iUwbb9;RdSkDJK->+(#o#O0S;`kNEMNdz1)l+ph{_vA2B?o3uP#)PK$!ot$PxO)qn1_4d#s^7zH#&uf zS~Z#*HXoYQ8_F+D)Jooet6h9|a@c#E(dwA;T`G${^I>Y=;>kltl_iP2pDz3)kAz60 zl{J39WRY#S`oI=m;Z$Z6DGsl3OT|K49Gvg1uz{&ph~o?qUdQAx;2S^*|ylgkEC&Dqci@em_s{y-eBdqIXHC596%&DJ zKK+bLvt&vT$52t}D(kxecE$MBnrmFhBrxP5DmxpZn1nNZ(EfDXM*hgXS7XMPy$KPD zQ7)N#@`xB#Y*1SwpSAluhYn+Xx0JYPRia1|x}JvuHQ=gw41AB3LDJ{!9)jg|J?EVd*;O8h!R>n}EV z0d@bR%v6>j)p@kvd@u-f2f!p!R&KuwSNm@*O&cx{-%NU{OBT6Md0pvlom$7RXomGB-sN?1RIJbAi(?`my;YaWe5bENIfUBLvyH4( zi-~dzzkqYd)vz2L}Ph1{A8175d6o(wB(s2@x$@z8N?<@4`wRYO?jOR(! zbAS~30j~oC=-f}Pu^IYZirqjI3xQv_zo~V3O>WhYua4vr35#`0! zrm(;yw#r$WHFGxzW4~;D_$Bs+r_9i>p*1x?#>br_YJyqvlIhW<&pr?#EE!HS`wv{A zgnxh7^f6ecb(maR^(jgH$QE66WiXB5cN0bqtIvowp5{FCuk9O0;x^;WOq2<6nB`yw zz_-9#|EA4s-?huloHU>s@w*ZINg4%2Jq(B9$#5kW!R%2d>L!38*FSvx=05V>JA^aKxt*ckt%Riz_!sosfN4|iTDa5>a-J5l1}U;$t^t9ckGX+AM6O| zt5$io6=|NGB&M8tbfZUz=&dC?48;?s#Jt}-DC2eieJ}m2%_m+suSv}^sbhQ) zGo;>--y>0#Kh7<=BGu*RGE2=Fp2*cnMhS@MY_u_8Kl@efmKsxF&1t(viSA^3tc`ja z=ux8V%bl(01u@JBddp-vmYtOL9a7HhLE98u=KjEzAFYI6p5hJ7uDrD#P=oCjfynR& zDve&}F~u-d)P1YPp}>j0d$N7!JD27`Pc>afTb{S{3(CDMzSGIy=~lAivh+A1q}7mH z%ul`}Z~4q3+9?Vu8PhSEm1<_o+u&Hy#MC3(zY0z40H6R{IPP{}LAF?+=`Z*(fX(i@ zW9+d7L3d^=U3Q(p{PqTqY~*@TBYxS*6t>Y>3V*3@k&?+9u$;$YsoSrV^Y0|gKr@T( zP>XPsrsOX)KC5N*{kX4P@8_IgqLgwfo9UxIzo~7UDq|!lS+%m}kNnJ01||JCe~AX9)~yxOq(PE3tlg&Sl5y2faSY)Rbg# z z@iHjK_3%;7D@G7Dx~>LaqQuZ`3?#`p%<2Yqs=rGV2ejo44sIiwdouAcpAKTQnDoBYZSfCH?0VTBT#0}2_)u1n?Xyhx+f%+7 zj30lxr(i#6h)XKI6|AgcD(M}P-uD&GUA#)BF1h1UzAEpz;0yB)FR|g@S;cn=_<-lz z_VT6uxI$b6qK&pU@d47&#rs2;lVO$O8U30Z|!{r)wH$5w7-?OX^?;Mg%B)P_kTYkCLsn0uZ(o*~w+#! zU_uKLpH!xN3dLxA|7^#*mF9xT@T#?fztpeZluuJsE>`B90fG?I66q!jGIS<{ zzheq!>wD*Rns|IzGqt~6RP~p~orb4y zU7uEc8rYG*#N)+pIjUIJ5E~t_bD}e`e`SNs`_`t;azN#4IYkyU!0*YA77I7jzBb*? zJAO&juqs_vJ*_bWxA`Ycs3cdlRkdaY!xmQO%=s2>&@E+N+U`*cMq9J4#Pfpo+oB@z z&I!9WzeU*?U(rRxzIxP1PFc61k@q3Tl@&@bVc+6#E8Su^^UL=v8McMJT9K!Yu+on? zM=dxxB#tHXFW2xF->IAVU=@SEsDH+-VGv-Rj@^ac9JVY=b*0+ja1UaWSoSdY*^c1~ zA2!5p%uIBJ-Rlp%gDMAcd#G5x{O}}@_3V@7r zMp7Fn00;-Q)|RQ$BVq1FD#Bz&pxNv3wom$>3KuQDI-Xcqqo^?1_IJ&@3HmEUc``H! z0XwMZc;qwc&9<$|xQAzCwm+$8rk0`2XBrJ*K0&@F53L@*3jZ9ilXKs>3GsmFDjJw; z_@@`=k&AiMPmlR~9kQZN=bjFk2+-sGDQrFA30tk#Ty;~$G{4ka_xUs(dK4tKF82GF zN1u@__<-@BhMj7Lk(UvDP1Aw6qdY_`XSw1)9Lh;Ag7%&+{frHZ{Hu<{E(f|$-@TnP zAL`T+GNp|Z;senMv3HVr`!RUJvj%^Pv=mSa!aK2f&8f6D1zR~`(oKq&Y{@)dzh;$G zoQrPQ*kDFK=x$eFq;(?=Pz0%Hvd>bj~P6qj-fP8Wat zng+A?k^NFTBYfq`_}_@}4EC?ynA@w^{_3Jw9frD0p$EV;^HgmM-pE0|5i}|p_>e_O z`FZK75g1&&roItqsH)JfYk;zap7JC^)^eYbYgR?@%x1L{*i~F_U6@`|iuzWu_ZYct zkve~_-b2GRq^lA?BBnNam~cciW4aX1q7QAYov$4vcVAo}jq{KNEEItsq#E_H z)FP|^N4@v5SdraGynSvoQ-DZASs;C!K3?Zy3G@l|fBzgU<+4JlQGuB$>Tme{`*&+# zWEnE?t55l6@6{qp7ZJ=s=t+ut(JLyI%%Kq^5O$T^D%T?X>eQhTF6zNW5$W|pl^I4< zKd;JWC5|i&&^J@{vT^0m?9MuEYKby+#X~-XCN!$c`t;Bxf&l)Rf6U2+g-lUrv^?3% z`*6Y{qG)_hwyj8{oErx_xxM|lAWuUopX9lZ<|q9FZ|U$FEWd&a%^3BSHYc8iuThaN zP!gWpf@+11?CS@N0|Nu@0IF(wdGyX)#UCidLi6A`K09!@R$+|ug4oT)dmM6ZX@C~9 z+Kpw`uV4Ox;xU2zZ{kzO9eXe@*8dw78EqE^Ww|7vArw1{9z@&=dyD{4++ZT(&Z#UZ z#E~GmK$;W*K1d3LsV=HJ0BlKVK4|+YyeY9oME1yBdV>yHn0P}}bp z;q78cu(NT&2F=Ok%Y;1b-l=EA>?FNX5GtqPED@EIS@}gDkVP%yaHB^qXTt&$hMSt( zTVW6@18n4PxxJEIuGwW-)S&LEMwq)%gA)Ne&Be+qJZn(No-9wVIf{Z@B6Z%1ass&R ze8xo1<$)B-hzGSaI-8V9S zFNcw@47yD2A$}>0zF5(NwmoTJW5k5s_y`KWqxr!jYY=VCm^R6s+QI!uKbnk`o?Hnk zik51QK^}v_9hPLh%c96yMjEmdHb(1Zd0L|RF0q+V8QTV8EJ`$}$DC&M2apTa#>8&F z5YvhHoQ#($Atd}qh`RTf|~!VTC#Zb zRwjMG88kOp{Hck)I4#bp z+Ss(avSLd*o%E;34-D9UY^$b22 zPMuN~&ti1Mcbgx*uWOKya-{;YE6`>r70Lg`$( zd#lmpZZq(f&sFdez(q2p(|X-E&MphHz)JZ6l-s0xd?8CI?ptsBWJZmQIbe$S zu40FO>#uF8CU9Nr5NBWw&X;Slk`FmMJh@-8Ji!&P-xUQwSPRc4cfL6wYUgmTBhLXa znOmMYfiQlu(ySfFuo*PQ4FMV`9Wd;c?SA6dmXWxVH=sId^Rnup$)~y)Y7s%Lc53y) zI_IytU=w#y*0#7?{fRV$unJC(PSQhQ^DDYR0`x-kb)$-3? zG;$1Tvj9FEvgizq>TJCjGBKq65{#z zuJ@ZTlx(8Xv`DFSq#t;B$mViw;Go_8HZ9R)5t7B3M+~8&CPCcB94STy#kYhvlzy7s zok)1*tpFGKHaRtp%g&lBj!?r5`ToHptKUB-a#&X}?3JQ7 zIg9sVi(2()A4}1!`>lPl*KDfB9mO0@wPs9X2~asUPIvp`+Au%4qW0WhGU!gE|Hv4`7K=408saoZr`>Pz z%5N)&4#rw4Zfi=z@ya3LWnQgpkQ~aeDII+?TH*$sPong}BU!Z4r`SA8>y2u^Uffj0 zl3zsD_-SAfmVJ2&J08Ey9L$3ih=i|D3(p(!{BX1VwZp3oeJiy6>r=CK;d&S_-ln|! z-#shaO-me1pce2|#~vec!RN)cml{BUcEn))9$zt9nBrl_e32T>B`CZ}}8RA=lAa{H^kE5@y;3{>D&CLrch!0BX zkirwm&jLqdCS&+-KL#xVg|LH=(5V7~9q-O-qIK%QB3EU@J}VXx2IU;TEcK?bqqz~?%QqY|@6HbI5e2fl3=U`XTG0s?$|VZH30_=)|nh4xV3JGO-U1}H)gIPGNn?hkyaOfZN?|F4G>hY)~z zUP45B&u=IgKF+R>RqHU#I`ZHL2%M7QM1QkrEyG^Aa`hx7N#_5^yXzA8(fO+_ahjm# z&YQk(lw>Zb8la5363Av)|+72>ZK=;!a*mo@pC7TY!Zy&b|Eqk}v zmNV3I21zOR(te)G=X0EVP$D8H;%rw}*qgO};jxh8={I_VayUGorTOkTJS$!4){vuB z$#bjRE=J3R#Nf=$U^l&9o8XteBZrm92NJnJTZ0eTGceI%H^C@@$zjP+(P4&KM(0d} zX%A(Y-2wJ8!V#-AYi_d`*OPbBQ3R-|(UVt1} zI^vQrL9|b`D5bBu!4^&M-g*BonWh=4Ic64rloONkC&yP4i{!--Bz>tDY$q}l=-@yA z&AR_b)?Y_O{e5BoFflYrqeypyfPge24H83lmo!KT2!kR>cXxNEw4gKyND0y*UDEYC zqu31Tb@mo>*ag75|zyyE2)IpbEdO?!BxCqlIoTPR<;^8F`qWK znMQCY&D#Cl!TrUQ5JDnG2%}xxAvMCHOciFABl-Uw$#?SKx_sInc-v4C@u=Su;b)C zxXvB6g24ZUXY@B$DzAkbRNSN4%O!coRL4AWC*a@&VQ)K_!gTCD%^^s1Pl0P#d7X( ztQC8}@CnrmdB6UA#%W*q1tG;W6HgT`P(;OxwO9bw7#GB;aw(%4bDw!$Pk`t`^z~F( zaBu!U?dfr#Aonfv*N;o{Dp`QRS4 z@zK=}gKR`8x(F5&927BAWU3L~58q6%`h%hT>7 z%j4_igo()7|xzNA1RlqtP;xyt9kxS(~=LrIyCe=gH=O#C697cvkHR93G;a8~W@ zK++G_rk1(_v1@;Bq-Y6IV@h931SuvVlM7XqsDS%!k~`S|WmN%}1io2XFgLQoBj9!t zKf_pB=n^qU-)EmUi~CLY)O}a`ufKpmyW>rM9v%JM=eDG`stLn7#tN%{jTdm4b5`!; zVVk>yOvF0Jx0pwbD8h7x-h1^a$E(1*@iT_hmP^~Avj##+Rw2RV0*)%%cXDOxVP;Iv z!BHV#-<@jkqgcB-2cVsG$_8+>)3k}6O8c7K1Jf{8gA*>3FGv9`NLjw@e}T~f!2#7Z z#OYv(qDD~fn}|DT)mvP*%gf5*+I@^;*VC+0{St(5qogWJ^&K*|;&WGV&ss=jbh6OJV$3>$+Nt4PZ=!KCdkFBrJj^Ns^v_F$pMVc{I0+qx= zzL!hqkkm>GU|lMh(5VX-@ZK3ZTG~?A!ONucR?0}s!`gD$s;QLeJ(Hzsf*fz)G3!!g z{$Mds(bePx0P&u;yi_07?|`ad*n$T=Jj|QDMPb}>B+bT^*caT=(Ix!F)1f5S4vgRbJm`;l6}oCFlg_pa*MY|E5AjUkb8&t^Rb4Fe6Ns_IufDJg?SQQq2cH zkMUELZ4`yk6IPcEpNPF{)j>Mfb&sd9w4ea$YWzO5;an*WwPuIx=-QXhy^OdVRSV+z z*$n8*h3XSmBE(jB*oPDwA>#(^WtYb@9fJ<6fhH@#BdyHchc(W9s*+T>$RSKA!iDlk z?Nt6u0;zoVHM}T(tQl*WB3Aj!Ew<}pM(%3VD;W3_v?&OLu#3A@(azhbB2c#a`GAm7!E$)C4x5LEF)Hn~7BL~K^*x7` zVQMpcrItv+x9Q>|t==b7Bq!n2wB5Qhe^*`GBs12O%0$Z8h)nyK^xkxZw`G^Y3@~dN z|7(PEUEOrO`hW_NVs1&s!yiKccjTE(G?Chm=)LD zeMhNof7AozUTycR((o#8-H`#cv%151X~F$w@ElJHb@rDR3*jfP062D%JCX6xU6JUq zX7AI4To)?nFiy5md@l4f%M&LU+tl9ZL#Jr&htgO@_M1hPR>M9#j4bsdEraG{-L}{b zul{Y^UxXs~dEH1Q?0tm5z;P6^T!n^QqSsh4+6$Nvn$G8)f%Hhqia+hy3q)KSioFcq zPANx2138ObZ&jh!>dF-(XbQd6@=u|s+}0zq&SU~CB*jw2U;E+?_y$#q)fV=$>R*=JW?A9*I@n_H);JjyFp|4c#PWS%WskVkrRY58%Q zs>Gb_=^sLw2F+37sn05V&qIp{ty-?q(vp8h=T`Af>MlY_U+L&|+a%oT?dqIWU>GVt|mo|cjK+cGf2>rIBEmcZ~Njm+W7aagIj@3ppd6I zWRO(84Ty=ceEc3!}LK%HuyuK0N+x)%PLgy3BruufG^_J^1jPKD}5akL}L3VxId@$%-bjhR#X#cK<;6 z4U1G(U9U&71X4SL6L69hgL%P~sTSUNlKyA$cyG9Fd5I*QY!=(wo;fb)gDz?%Gs%-@ z)>J@Z9|vXzD7Y{Fys}N}dd<(P!OX`77f%l+TdP`@iAo-h++-(M<#7?x5aI_zk@$^t zw51nx4IAT}pLY5N6e(7$yj5;qaIIMPQbHP&&T_IdEH__M!li zFx;O3EP^u$ygTbXV__Xe(xT@i7!P?Ww_n1&l+%R52)Zq1FdO4+%2M6uB&U9V zYT{8ouFIa>T!2qRqsbr~p@gIoHa|PluK(@yowE!>srN{|5)iS2$)3aWPD<49s9xdj zJQvqMSqSYx?Fo$~-{BkV&+TnvO0GtKmq@IXnacH^nF51t*2+Wh;EMcAJDP+p>H!j{ zn6!`yxhn!?a--c3(=lT_CQxzt&uVP4u*kpYL+f{US@>6ToGXK4QIRS$(F(k()+u4| z?lv$!4?oz1#l0_7vp4ZDp&!p}=YhVW>iyCV+#?v+ueHW4*#n%~njC$Yd8|g_;8Q+H zN0{M`Q^6^AlhpBq+-lGYH-BY3n}(wEXdthr;WTw^hWa`*N-;#nck0XY>tnO&1E(Lz zH2(Sooi1~res8@eF5q(YX2Ybx^%pg1A1@3v#V{JgvEpf93=|u;r~AM3)IP$k!znI# zfEW2QZ0UwWR8(}$GALgRNcQ>p)6k#^2eql&lPao~@Kd1aWu}1$Ag~;1&NN%O7?y>Y2bs=@eNJaE(5yeV_^dOCrPhq zotECDlr1>=sjgpY6`7+0Sqb9#5_(zc>BKu#+_cfN?jP3KkjX0Or-0D#5e1 z4KAEFNZ!f@RoO+lG{f#A1mLb8{dRp%K3arv&yOh<59DG~kdj0eM&{w+0rmT!!rQo4at-L_!NMO8o(4jEq?Wz}Vv!^ab^a82tcl*9 zLNIsq4;;nEqMHHS8+^)=LH;KiA?f1dfp7FAs*&pOs}YwmFqn`mLS_~%TNd~~z?Gup zFxYfLBauF#mTahmnZlYJDw4CFdtWtj5KrFbGn27DvjXK$h)!|cHVS#&6?CT$9Mt`n z=x*hWzlD*IBn*nHnhLjE)VuOgxI-St@b7ZyIH6&ql{&qvwVUNm=Cw^=Gi=oc9Tq0D zZ(-4Rp)BT+k`x>U$A@&OzmGM2{Y~>Z8Y8Gk$AOx5GB2w}i8;lI zKZNF!cn5=pdEXC_n6whsC4$lQc#CUH`8i$p(yy{9wVWFhKq+Q&pntw|b7PLX$z#J!s$fi@{>xP-v5NA^Q76zjE4T7F4H)z@6dyaRKy% zI1hR8L&j??6qq$i;sokIq)+LETYrKdBB=zZ;NdS!KY3Tmq(i`@!z4{%YoIEszog&J zWu|9hqamVUYNlWJM~D9rFZ~mPX??W-<1e2hh(qvh?G75!DdK6ub@^g3HL}dzesB}D zmWY<9apOMEbk`BpGxc!)bimf$LWqAzg!o)t%7$5n;oNjm(al~re{Yo@60T7=L(u4Dl-G*32S?o39C=|vIx5t>ee z96Y;$(W(HKOs5<1hKR^?c8$-~C|+|^fnI2u`iQ2L9V=ZUpCZOdGK$qj)zu;0>;s(r z;&f~I``t`QB9oylqjGUY!!C~m5aVeO@?|s?%(I>~-=59o`*>Z)yfg}j?X9Kxzi}zD0l9R90ewC@_5DPz$5p!U4>4B6d%$% zpk~kd^COS9jETUTZ~HC1n*NL#7=Zhb{EE-jok2hDo?XnW^we*U1+%>tzCbVm*A44Y zrsVp>f`rCfFu5z4$iy%TeaS-Tzi9H}mWV@N`{P6MlJddsx_00m;MvuHyoyx=w>FUX zs4O}Bp8(A8v`m4w82aTh3nMqWGJz=Z$D2`2oaiB1e%JSrc*+tAc*^G71(_|LI(Lr8 zT(DS;mxtuOZLYklao+eKhW+{Thpk*POesrPz-2AidB8UgZf+dtKxj{T2imUnmRWA$ zp`j;Iv7)xfT>S)pqb^*9`0_kzp00U?mp~&{&iqG6iq>f-hd<0M-xDe^%MYL@_0KRqo#?3+0 zcekfnTZ2(V(ncbZrk7)QPd*;qliDn*KgjbhpbU-_62_8_R19r zz-dpth=9s=YC*ErE`T7UW|Az92@Qv0NRym6LJ=hBk{fOhX3C&Ys3K4O%#60pOqEIJ zm8A^$``~Bn1>FByrCKAiG%@#D;TF@KzOSx4(0qR+VAhr?L_v)?rX}=#c!1})U+m19 z=R#?vMUEQL2p`VCAQM~*Yh8aem&jod120CV=CBbg13S&9L#*AMEowHJsYQDrY@7Jj z;J9e$zB`lcHc@FL#tcfjl`ZDp|6X5JDGRib1HxlOkOIw42G-CjV``+1=UDD)+Jgb| ztuNsl!8wFmEu_z)&=30@5UbqQawd*9($x6SFF00&J{>T?gD}FtgAcRApTHH~U;aGJ zKeS2^*bGe`{aBo^8OsTrbWon)j`aGw)B`So z=2x_*dEHj2bOIVU6QKgib5L0&hRt~I)IwRdUxJ=DPs|Gln|(gX<9RGCsh^%EGg=%o zT*oBS$m0bZ(6q;b%}x&kaF0PS>(%uK9T@|>=zTn{z`0%|$|{8lK6IK@!7Ddn&N$J+8zfReeJ9)3SHkoS@SPFcEi?U%n-N+OX8$bf z;GvY<6#5qLUo-vcm}kN0A7P>jmOojX{hB|hv6(M(d-BDY%6(fsWX8)=Pzc1wZ)^gz zM&-|Kv06fpwYib8@K_{;$IRY3ptBi#RtJ;K5>pxKK7*EM?6jdbJKRGxkM|o4klTOK zXJXGH<$@kDDN7ZDZl}{`*if-R)xz%>>)Ou@ZI`FFnZMGt@iD++He~s8+e4QjOfTkp zSFf)^ov1cyg!>_gL$@_ayoJ^QV1{bj{|^79b} zMiEauT_3k2G{01RVf1=vj?XdLRLJEzyo{((p`A30Sc^{0NFx?nZ|r=R?f!E=%r^JRU3a zf#>HB8`>!@lUuL4QsukG#_UY`=TU03?eDMmWQx6{q}BSRYVQ+JYB%Hq2jW|yA!P>b z@(lOCMX~39H!dWC@j4fghDnd$FQYBwRBF)uCb9#U1ZgDC=LVNUyvJ*m&c}NLW_%Dv=<&dLDdr$$D%L=_ z8#Et;VrWJ11kR^&r?|AW{?y)8MPR5ZcR75Ma??&P_0BMnO=k47(c8ePmSFCeg(YKG z>__*lT4mbQ=t#?~uzI*X+-H((4uck6^&9)TqEf58)LKK-{V^^pzqxdT%_`<+N)=|ZpKzTKFl!e@*)29nSt_S92sPTpdVFl> zV4th6Ht^Ptzu8RAKEBMOq-zjI%p5u?K|Nh-K=fd?O?9oYy?zLjk{_R~J^s10X>lBo z3~iFd%ly+VT_a-*M*cNKd7sa_M*cZi6!Q+RdWIp1izuA;&een5f(R;q={G}*rd;^b z(qlU4{oP#IF=@T#Az6JbGe7tGGE_S*9Vuym>VdMd<*OZEwh0j>PHOUwx)PV6$54O zyCUTyy${s+P(LqG#z(ZZq;5*5JI(OFUXWCCF`iXewN33e^S!oJ2&f_}xZ{0buQt=> zTNp;d1)hKu=V7Nd8;R=pNOBmHhf5uPZ{eYqo_jaPgp~WqMT%1{-!w7LJbMr-eoLgo zbPsOnm^T{&D15f=&ITV>hNdHTPwS0T^^o7T+p=5XS^3_o{?fs>`0;ocBI7qw@-zDS zXdUzpMMSPbhkm|O#bgk;t$ogz?tJ%Ka%WZm*A=qyJq%SnfM=Mgpm1iKoN?dVbuqKN zeJoVIu7I{kNq*a6J?^|FO|}^nEDQP(El6S-D# z`P2{H-*V2j_$Kxcvr+MGPbE-zEHdy`mY6jh?FD5r-#^)0?|PIGW7K}3#K)RR4=Q;I z!>pB2cgmtP8nznK1MSyec4FRal+BkAap@J%pp6U_hz96*9h;G1r=L>Sz=m zn<;j!%Zd{y``N|%v-hC8&U4=gVS!z#=Iq{K-Gzd9TZ{84@*1yUK>^&xQM>!41y7xm zQI|`XE6xF!tsK0f&30wWVYp(U5^{6A?&0FJ_-^ZaTdj_m5Sq+GiWo<+JN?q}n%h%o z_3L%)Nj+AXQiXfNBKFG5jYy+BeH3NCE6bv7jW5pi*ch02vQq1rJy9o{BmEQSVhpzN zt7_Iq6*kkNxwET!ipGi!OX=|?3)J&IJK1Xx@v|y`Wg&n zLez__ezXc_8K)kN)Qfi(N3A~6*S)N3R!bd-4WRmk%F;o7n-Zs*@X{ymkkY0hjR5Il zZXi~2Vw_(YJIR5u)99%XPDZr2I_~qgOI45WseV*=UaN=>KMy4dIFOeV&2C2#2F z)T4QI88*IJ(a!Dr#ayS+E}xD`A;g8q4eJvaDPoHmji0X|ckRGWGVE3Fw>H&6zh|04 zCh|s`bunA!k=MRhWmEZFtD6GAWqww?uuN+`nrA&8^Dwf;#B2;~lU0GxDuEm*zMK<(~i8NIlY<_jH}5Qk&O&1be0A)}8X z=P`;YCxvT~>!kbyx*QJ$-FKNzws3CG z-$wEXUEH>0@=9UGG5tE&FD)M(N#vB?Cq3rW5%BA*r!V`D1yD^U^hT1nD44g8XYEKU z$a+MtTDN(4n17(6d~SWZdUdUn;_Xg$J^~^zBCoSWPC4X!Ttr3fVr<`4H+z9$nD0@I z>zw;Kvede&%PRzVTowtJ@aPAU)QG}=M8DUxHke#77blwcIzU*c#RfRb(#@h$&N=4bYV}kdFTJg%b8O z!^LvYD(umF6XCXC@~a}%+`48}X8h{JBmykdzbSRUY`byv7faKW;~MfR zhbIV0`*_Uf3yu2A`ajn|Y~o8aqt81?&CFA;D+_OP1at#eo5M?|`}H_mS^RF#f}n!x zQE0y|7+~Y;^*9SLGS~C8i$&{v!@5Sc4HhC7Z~r__=CcV&R&Rc*ak(br%N!l8t?gV6 zZA&?(bWvBzl#g4XSLnl;@Qal|!6K0|FN9)~+&>sJFm51!T9h9 zQ8(nn^eN_(~r_rzeI%@V<=ie z#V@I0kL3}`jHa{G<5Z_uf>v<)66#5xQPMsdNl$Fx_lO%`AIJ%2V3w0}xj9sI6^eKI z(s$e>LzmK<;io%jl;acR8Kdh~8P#j_m~Croi~A-yajgt9cZWr;&S}Z zMt@_$5kASA-_ibw#EQ!neI#A-+2M?W)OKkMUpb@Xj z)SmvT@ZFm1_;}l8U!oSJa!pO0GouQNR4FLkp!2;PNML83f!FltqKFXN?MFgzPvoec zVH_?z7~a7z$;|=AK|_*=K}N_(rq)izzY>RUzrvhD`ZGeFZ}$vfRdKXnwc{tun?q|M zlYf4*JIgm46UEJacF0Xouhu29O_=UAaA=V-jdL+ifQHyDR_Sc>l_hr3peQ(p5VF2@ zK*16MIoQ5!&+s&iCw4Mn3?B#RrQLm>mo_@8`8#N(6S-__62*n+)qRn?}gpfwbz8}&)uS4x(ry6 zXbc~Ex|D9XXcR_pTTN*l9}|qqx|9+_tDIhV?dDrqh)-u6$1aylp! zAtjQ8M93`q>Xb}X6tjuk{i;=FmUJOuwj*tH;NVnFH)fJ#DS6R#y4FEp7|WMlkubqr zxOOdo6(^Bbg(qX)iTpC}pz6iO5^{5~`6%86Yhi*(zv?Ls?Jvi3j+nt<5BvyoOjm# z3BKNE147b1c(Ubh-OR@~)$-rPO0y^o$6R7r6lV|hIX11mX>M*&FN2ay#%a$<`|J9J zR$Q;lRSuTFzoSLB_P+RTa=8=9D^CqR8Z@3aYq!ULem{DnQSpy8CPCGSR*FkijfpW6 zg+P?T*UH34fzK{w=i~hR8GbMd?8)itLABV0xQRlg!3y@dNo2R=<3~m6EjBPQ5!(J# z&dmAC8T97W=xg)Q2!nuI(|mbtqz`Ym+YXn7zI}9VP)1HP{H4UvQsCQZBXagFRC*{1 z2{95)(BJck>-3G?pCH^K3a-X+oR<`DkC^KruQ@aED%Zjz89$b#HScsS2S}aU+c$dl z&ZA>C?@>IG)m2b#yUG#1-B#GR%s?p1ShcAa z4#GPzpDDs~8GU~~xSuYx?jL>CWWEK3PFV}hL?YKZCpy4Y=&vVBEJir`mtMg2M=%w1 z$n}1mB@`>4eG2?TFge&PX1#(jB5lBQ7H!9|)&RWvBt)!&(QrNxA!`D$Opf#eSB%GgGP5Te#91{P?$^AKKTpR4`h_J_ki!xinkyPsyn6hmn#ycJKd+ z3jc#Dk;(hxXg+LnB6I&jkR`#hEQVx)n3XT9-K3rT_xCz&AfO1ys9hJwrUR||hv23~ z@~IjK#*K{i7r{`;wi(?qomN$4Hl1eec9#0k*&oY@5}E*@Jeb6A)zk`Z5|34cVutW+ zniD8x=4(|NZiw>lD#KkRXOwg@zvYp-D9Qj-cXL3%6-u{+illD+d70&#!$b9uo(Nv= z5M&r=ES;DzJ)EDVhd-QTOef%T(ae&~(ZHfGAl_A`S(8y*H8dDKY2sDhiI~GlMi1va z`jTiGIwZ-c{w~MAMG+T9KkQ__5mYr z$ld||5r{v9Ki5Trz+(XlS{M+5N$|Aaq)!ROgDF?0WRBX9nAq5S?P}8v0ibn$+Sm-# zlTnW$S&(cRI<5*{D>+m{XBfSOL2E7 znMZ~}Im0dk(3F+`9^|7B;E#?j2{tjTuPP0)`^q2lmA4UPOL&B`wR;Nz$u=p~8y-l`i{H;yL&;UKe3yKbOl1kJ=pZ;~#7SiUh77c)KC@ zyi4e^R6LNP??c%2Jt9F3H=n%s1Wq2~GHKB@KI1Uz+{MTyXxEKxaHLp5#h_z?O+TU} zw3Wr1=WKHPo6#Agfzx>AFMbhM6yzeyJhj~<0Ovr+R!z(zW%?@ZmYxhIq#Jjdaxy;b zWidG-Y-ll04U?pMdVa9VZ83>gS$w^~X^7VRmkY|p(AhZ5g*vh?OC-=AX!V zf!u{}n|-YaGMf5)jY{ zA!4;`(8`s5;3}j^+U#l``$^=r>|CSfVDXRn+`_LDz(RTJgYw>k6A^ zR+9O$&$epznh5;y=_o>Q7e6(YL1h;{SC`D6ZK0mvQlNM6>)X70nlSKnRQ58*Do-Wj zB<3#pX=+iga!GHC{vo4fX;Nul(+1$CVW9g~~gHg$!(^(|`O- z5Zowej^-<<>oq^v2zRMvPurM#{aQ`><-(@dK-+8Y`q;M_9cQgiju$ybG$+RyvPl1s z1m#yN{-G@LsGWRzb*`o{{KZaTv=fT!aG#*2VQV#NrDo%C|I>BAiCp0b`tkSvLVI<} zlIULuk$*b+#kF5iug5-_k+n6keysm5bPz`ySxpa)RG3W+7-7M<=mmrVei`?nG)T$efO?M(Y4Irj9){!;#NS`|G{{K&;i3-YaLL1D%V+!` z*3B{u&uZ8|*NqL9>n$Zp$}n!4C3~uXth`!>cWpU!OG6YZ72}Oo@d!p6OpP>MW33kq z$MjcOGsE>U+0TQ-fnYwJM?2z&2E=8p56?*yY@^_hXtGrMkN}+!m28spK>{ZP(C<|5 z1!}7&RZ7nu0zB4(61xe$zhD&Yy0zg{t*RciLxiTclC)lavlO%yeGWR+x7xx%-R%T{ zoduMVXO7%15@KxQN6hyjfFfx51T%V%RZ@xkk?RhB6xcz&(>2Lu=D2_(m`rFYL^Y8T$C?q=vS}aoB zO;3e)?T~;ax^cZ5mjZG>Q5L4_P9{h%<|y*IX*4Y^m)UBatlT;MHoJD`cjadV;>1VE z_kNtRXO)!Eq%v*v^}6m)WxTJ}FfM74IrLt!}9|aRDQY67ZA?_!QZB{#zoPMp$%Pm&msQ~lY5AC*kfuq&l2u*cKE!- za0y(saT4Z0@D6QsFo4g4D8@aCjK~M?7(*6Z)CmrATn8$sK!!q;$YmJ|vWOv-T3YyD zt3rdp1!=%eZSnhi_BgOq@fHD$SwBb;^hx|d>eUc%X(J~uKc1=&-dmKo3X;zp86gf% zjK4k5DwxhGm<_@}f-yMw<%LuTJlp#1FJfH_dePCZQ}&|#b1|?VKNW`~4`*AwYEyd2 zJ?5$N6@r8|FyvUiJd06n&d0tY$ z7~hEmW_-?wT>g32RK{2kT%-_aQmZ)s7NDvRPM7;uUM~F}%XJy%on{J@P3E>(%k;a$ z(~gW|)lveo^e34ws<&jE;{U>-3>8lPLwPh<2YkkqQ1|r1pEbHHoe>%o@-)|#Yz3ISRhKRO65+5fd;|gql~#Zl)?~yBhz$mqoysi3TvaT~Cc&AqEzyU2psUO_hP6 z<66%(l>aEv7_fakBwoElwdcL;@ z5xzAN{$_SP30`^LbHF!~m-?RsgM$HVrAJ2dOp4+1(M;kNv1il>^yoy?XU!C3e?150 zAI0HRUDqe0|4bCJDj<|RKlw5W>M##lT{lO#(t(12>+A5y|IKs%OL%kS{P%gr@<`hn zfEF+3iHwWCHfB6c^}qQX;MYx+jDhkYKuQ7loKUd-5f0%SFdbseyfU)$q+AD7ZP{z= zriuUGXf6p2u*spF#hC-2$ssQKc}D90?gKs3|G!z>U)hoe9{QOc^^_+1KF!X^EV#9g z>dHcgQQwV(AIZt{?yS$Hkkl0Nix$JlB zxMh%uAj4z&11EuRj-(UZR5Ag)0@w-)F6#s9J%hOCT(S1^oQP=RND#~s)B5Si&neh^Vm?&kv$Xggu_HP;3z#<*OgT`HR)ekXng50>86a%0= zBO{!jJ~Ruqb@zP4k8dcC^r=uV38R42rkDYTT#zo6-!Wkzo>c}w$6g{~C?K5U?=!Q* zO#p$kf77*xTvOo;e__u6yUKjBIhqLCit^y=nlqbj|5Ms7t5~gY031Q<-$)(fUc5(@ z^0E6gz_&s*Yb*<)xM0weJ=JCwg*>eIiWMdOPROWv$INRf743$?%)|~ucu};W*aLYu zf=dc1+N7BkT^s`*bCVRF#nF71oG#a!4xI+KjMg)FUK}NW)DGTUy|JSo_cYch4eU4W z@A3T5B_3g*v1;OU`rwOz3f&nSP>zo1>?Xej!pouSc zn2g^o-rs4pUv7{V8+EkT%j-8jV$-H-N~{B6TE?j|koDi0fLj#brOW%E!t517NZ&oc z9kbx>0C(iAJ5JydoEZN#geV|M5I!_5l;gAP&pI1g6f9yzj}h=!fGhC}7N>fbe#g}| zD_qMB_ZxvnE!-zv%vzd~{<4RI=m;vm-a5@L`8TK2MgjgG(W&NuO@A`8-rlZT7xiT9 z7pXh8F>MW22y?7ckRoId75x+f6h99!NAfa|R^{!~XLWepy7J(RQk zY3er|QQ=N5-=a?+NcpG`SffRKuh46r!x6;ocUQLgo)=)dz@^ePpej@}PrIx{P9pT@ zWvu9%xsMm<~99=FBf>HRV)vVGdJQ_%uBQmG>YethOwlZ9}zCJG>WM7+j(`lcT7 z>&26TDE~*`o8Ic=)N2m`6EG-hgQ#zcQboKQE>|%u8{o!0fO+Y$*A}2eBe0&S$`(GG zG26h-B5MX*`ndQ$z&v4xJJz|+`B{5xkk23PQ?tvOgm{NZASp2(cHG}8=Wl%#21bJ= z-t>CgX?Y-+h2SjU_H+f%xPm7L7-H3aS&fKtQ$KCEG6be>4CIW476~P+$X~+l_#qj~0$_-p+tzS!{nkq zO#%gA6@JY}Or!3T%nNCDy(in}k~hUK0tO6Y=M+E_7k58cC?{@85x)I1G2Y_g==y|K zI_znq$vOfSHW-R-B+-l@MIW~ZQaa59Vy9pJ^8gP}fd9p=bEZhMJjKNZ)-nLB<%MxX zIAS+dIw%fQKewrCZ|kkcB$$~peU9bUkv<@lR6R=t8s*~oPT{fk1UP%(tJ>H#T!rLP z2vZmaT6PrqQ73w`hUJFvt0c;}ZzTP|jU1y&L;=1I!N{;y`p7<%#LDQr!MnITh4B1S zBPg*#X+5-VYUFuXu))t{kO;W93zDU-qA%Iw4W*VNP5gvQL;trDrJNmZLKN?N+E#2M zBD_N!O;??HcPBRCXJzPwIWXhpCYup0E0xd_gTGVPhx z4dj<&pBuk|X&oi~@Cq_lI^DlK%=FcN6%}`^`(10So6;lH!(^)rF17PzY8-z4o2qCx zDxNNyToH$c%*PPiScGI|W4r+VX3y190}b3hu-pY)Xt1o3VCtOSY&8EQ^rl27Wecgp zEJ-O@P4wgWA$tW%p6N(B&j_=|?VK~Qmv^+MeDQek?GC%G>W{{3wlp!n_5p4)-|nyo09T1JomU{l zcVa;l)_ysQ{sLFMXIxw{5|HOvv`9)@QYJ;1;?ygb=Sc%GiMa8$z1eGTUr@V%K$=40 zaM*$E$_C8~#+(s0`-WXxY3=Z#n~!I;>0%}HB{YQUK~2ZA{^YILO_18Bk~lgZ^7vR4Qd+!xt@z>$|!@hl1VU6jQ2B^WhOOqn1% zmpaM69D(|^t2Yh->Q7_L?6Y)q_5@BK9sqt{GrouB+2T~PNec(RTKEk*oZ?}Dvd?jR zx1N|FV$6hf0%D-Wc%IzAFNlr(A{eL)(+efmcBRt8hHMp;hhQ5p*4maLN#1w#+xe;N zc49&>$szCSQ6r?b7qywC%GkI~ZOciw&w!zs1+Ef|i;K%D1dv4FfyX6By57gf!$kBj6H@=(?a5-DUbytCGx?Lvt>R~sBbGtP_fuTT=yHJ z$ghk2=p`Y>G11Rtf?AM>**%@Q;r&W{e~YD=YP0ni1r9*b#P!4MQUl}X;IN-2N)UM4 z-u(VfZQ8BZI#VU2Ok^*ToP2lan{5`|>HD<=cWf`HCgP3pc-YKGZiYp{U_7K90j3Hh zi|l5=7}qEYzK@h-#O+Rp0n_t`x2po0W9ZXytdSmV zBBYWLd3V@FF4cu?DsyKJpS&HwjLl%ZO18~>^fX~FXqEMgJRfp)qeLn1@J1JO3IQ;0 zk*MQE;mf+S&>+b?+aB;V*0|q8fq^ukAQ_0>?lo~`jCIAD&?6C4V=NzY4zlsPQM3?K`7bo zey|Q;xoUIhQ@;vck-{tzHL`{z2Npo!*9(2`@RAxmR7w^U3p840Mil)t*TrE-DBr^p zwbkD0?c~I`4f)+11xo6M#fAK-D7_Kqb5X9!Hc3R3F?dQ#ZW8 zjffiexmDAkzKY&jiMQlkk}X6hed9i_+gqwYY+GQeu2eP^W^;P?i{p5#jixZX7gi!Y zQ&&NZvzr7(g?*E?Ydj#uNnODHCT*Dtr}{OfGe zNG^JpOwXdMU_wsAc}41Qv_k8`?|*v9aD1B>ajl8YW%Gz=?yk&}Ptd6jt(m%H(x39= z^-b#N;QqbisnEa+%2<6{@q>YFJun(ro?Ex z4FK#>*EAN4KsQZRFV%dN4t7!2+O&hnV;obY7ng_@tYr*$*NgWlxe~_CrJa~%)?+y( z=p1~?!bcyd2u8r^yvaWiaXPQkh+O^P)W`4C*x(#ciw6NSKK^$xuP4b?_C#Fbqu=EMnt*yGZ)8X=@Ej3>DtL{i_;XdjUj_7V+4f!tj2(5ia%JF|t& z%p-$0(OVUPo~YGJ0i(jAU>Gf~9@;r%{{bcj9TFlF5$*>)tQC{rirh@o#RevBW1#*A zXZyKS?oO4bh6o_h%Ttv~F!AHjqRS_O{G9cixT*XF?(;vugs(MsrQCE&9RZ){Ql`|K zRN1tGxqhruZ7Q|SqMIX$adHhGbfsDR+RaG2+X6k{fFum%g*<1=Zq1ujXgv23ICq`>wArx6TLX;E&@H*HmMFz(M3 z>wrcrd7Fb06X2kUSuBJ50}JensIy?_mPqILiuEs6iDx>@@DM$C|JipZllMAGV1jo?lID|~O!JG%l z6N}t|4G|)C>TCUD;`8N3YTM6=b$y}t2tF6Aq#KJDsu4>KPvxdJ=0}oVY{=-exZA@7 zgeV^0G)Ggv>jF_^Qcp$r$I@Q89>j6rbk1c61Lu=qQ@-BD?=!F8jVGVZVo|vKmKqQ( zEUU!(Qbxge6L{7q-~UhwnGkJVFoT0lu7G3!8RSD(CSk%3xY7U=K0yR1rj_740)Qr9W4-sY`w{ZDz zsn4WxImYbxvLLK{zhoN~hHcJym(t+6Vg)E#RTJul^u+L8hX^1-BG@#`l4dI-K06XS z(PpDl5#SQi;4;ZZJ-KwIbaiP&8gtY*{ppbb$}tUZu^uSkwSEGJR=p!p@4gzw%8Wwd zA#xCDScDkAVi9y;&g$$ddJHGnIE#v=6$qrw2#c7RS&-^->ugD{Rt_qt5^ zBNXDyj^Cx*lR_jhy7*bzE;=rb&=#-ppXLH2in5)mkR@*9K=ie=<$N?0x#RE*-#dDk zIHZUlOY-A7h|Lu=#`XAP1QSyzx&|xzMuE^&u=Z(v-DYVX3jbHz3;uR zeeLT4|KNXOlmb!Y7=R+O!KkUvvminclq7ysKS$iH*r*jegf+S%`@^}DdfDo%R^>sfGlQ-F#j-RRxWp$mXU$Lf##50`y#HzIzH3lY052YbKm5Ue3d=LNOgmgw+;I z#XQTTOGvuC^d2fIAqx2=S4ase2bQ7<#xnb8FtO5ZBtL`@JEz0_)yLlI+Z+hL;2P=S zeoS9%fr@KuV*z%Gm3QJ6?Vi_N9aHI5k@Wi%Vzk{XMQKrzK$P62{oGYaGI&o`zl^tA zgE|{RW2<*$u0by}rxLz2TsT>*+Zq(HG|2KefyZ5>#|8~DnEckle@MBNDWrXDjwKyD z?jJKDHMo0AB}I0!U`XYG={MAUrLA}H6Wj;>-sN_kom?O6!J zks2sr&CpNu@r14vaQJ+@ALqMl_Wwq6ITFPeNW@{%Z!ewBI|@NG=gS=tY?-r zZMf7Tr+q3;hwMCm#&MqplUxGwM{#E=?ArqSW0=l+<+nuszPf1UdQyfekWT~UpF*mI zC=J$Fd6;p>jj66e-W*C|FNZ-VsjbNLI$p4~S=JeA{_RWT^T9D9Tk96S!a{q$P=N&K zDp5lct~!O_(04vh3SZ1bIk9omMWC3+%ZH|yaa41n##_U)BwGMKlvPLqm0i%P&a zfo!54PjnmaDy_gZ6o7?Uj*)u;FX)`pZJ}#KzFLPm0`sFHklxgc&x>@eD1uBLNZzc_ zwk~u=d`H)1L#~SutLC(3pRCr*llP8eWE;}x$P}~z92f}`4O!y z$`|D%rwAVQ#V+wv9iM4$I9-RaTCzzY@e$BIkS_;GD7=!&?0FQ`HP?itl}{G}J@~0D zP{S=dmRhsd+4;vsPuM1EZb>%;AXlgEOU{bt6gqmIRh!gp?ynfjtsdv&*+0qe)+>9g z{INkOA%@cFFu-WPh$wAA1<_e?KIpZq86+4M#QL`x#4NE;x|AYQrcTBCoXCU?gkda) z#vh4f6!!!{p7Xb8*^6&GDY+@ zqt^WWHQMG9#Y&+O)@9+^bt`JnF=8w07ic0x?yo!y{`~@a$HnE;x^b~Y5N-Kq6_A$6 zdPV>~N1!%fUfZVmVh#fNtV4n}DEH#$MI&yRI?ZbjyC-?OG;`b4puxNtcnYYc{rP9w z5t!1p(d+XxX}|(M^-ZL;PzElv=Vcis&x=>vhW#)#x-Vx)ZNMtQg(&gIab6Yv_@NbW zWe4!FJP8`Aw8A!3W0217fezyDCp+AiN%wpI+Bo8rM*vGS$Gb+5KXYmiSryzJ+Rg%P z$co5ouUtE+dPV^=lY|aL7W5=-5Qj1ds^$aWg2yOfuQ0CRavixdu+Zg(;V}Bbdfjju z4D0#BR*i8|lEHfLol@T+33qr9pCTw`$$OQ$k#Uqix3kyoi`w7AH@RqgN{}DBfOPVv zwFv^WoSsF+?KrL90J%bX@lgDw_l@NzdJ~z5)9PGbRuFygBJ)K?)F&BA^*X?46Fk6F zO%L{mK_9qG_qJehLoFsJ3g3%8Nxn*71&t=$Yjc1CUAwO!R!Y!=xr+v(OGgpP>PC+W3QrOQPgxB6WebNu$!eOlwyhLjm zNw-akRs(z;QK4TNde7rEY@uWIP1z|m8C{PhSQqkVda+}rZB}V{l0j_+Bo9pzleQ4bIt@(pF%k)LWyrfJs2Y}S7^PRTT^o%va)Ycc~ zrlDeWu<

tDC+@|><>d*l@jDMOb9vcm#8)O$$Omd_ZBOkg%}Qf3v*oZ zhpFJ*n%oTGD2Ww>eB-ZKk~Sv?ys6L#k30A`uQZzDM>Jk=d8D%9#T6?_Oj{nMK2%zr z^TsOmWYyw`u$QX)_UlID9|<3+9D4d(v|k}H%on*u;!-1S4F%wcuGgeQ%jbwcE>pV8Mgq+fL-)tL=$u zpUn25SEK3bJL?2*K==UN+F-lr&wWci!Ncd*I~Q&z$!9nhgNOZgnn59*3#08OxRcQc zdYRb9Ys!M{iSVdIcNc#QjA{h?oJk=gW2K}7MqIP)i?%C8YS+TrBcoXf(iO?T5ia7>m~0Qk zz3ODxr%%t4^%uhQIwZ`sm3z&aKz&_1A--hVO-lYdSy4-NaZj}#fo=E1qUOjvF zqPr-H!$VAD>*)VvyZ=n&lJN>V!~$!PrZh{wYQ@q7xwEtgTi&Vrf+1*>)#;eC7Cj6!N5a zf;V?|HFmMS^VsH;*|i?KM;*$!m+2M0vfVn~+wHI8gKSIUbm+nG0O7Icat0MdYqc<#i!A+)BQYQ5ewW$K8StbrQxS^=1)I_EQ&y@fW!)htx#VP@V&pSO@|y&Z+e zwIRXx5KMr__@&BgOO#1qs2N7O=7Hyz^Za$r#IVC_GAVglY=~wI7kz4;U5Sm9&V?z4 zT=d2R!}kE+0Du=*`&eD-7=~IPn>q=Pn&b=T!6udZvXA@suU`9~ibXMpyHS8gVX=WH zo$?M>=l1$N9-E~}@YYU^UY;=?YOqqkUeAZCWuVKrE<4yFB8QPSZoS`-{MUm(!h!np zJZbUNqDd`;*GxiS5~M~+)(1Zif+HIaC;g^2p{Q4Ga0TAUTKO*4@P>4XyK??mU&9Jc znHo2z>GFFzr;Kzh%KgkbH>Y=h@$faoD0Xd&=o4nKL(#xz&i6%Q@V?Zu!z}vf*qqo*aOq*hQj{ z>%?T&((jxSN3p&M>ACF3`Em00)MD;g1@CbvJG&e6D$iuG&P*8Vk6L@g#j`(mfn1J6 zhi#|rpC*Kxl~zAGtKErc+66J`ZLH=wogt8kn_mT^8jV*sQe52`>KOZq>rD+@nh&U! z&eq^Ql&X52Yw`RV_Y1+f_N)*@QgP0kq>WoKE-GKaFA!b6B!sav&pK-Mv;F?RB6FLR zL``a=@8E-6)E;HJ(3-@bu}2$TP*9MIX_#rGqJ-|wJ_tBLyAguT2~G&^x)(2+EHm}A zLVop16jY~*Xcr-Mh}NRl5Zmf!Cpv75wx@2)JB4f>N6dMprgGTwH?JXQK6y=zhZA@5 z#cpcu3`}AN@)s*exc<1FvRzsc{J(QogC1G~DfiWP2zbmbet&IUaD=QfdXkttykWpD zVs4+bTvm0pA4)lAJn)$O&F?zSy5w<^){3$h$<`y$n37^@A|)Ap9g0n>sfvq<2G(%+ zUG^DTZoi?h8tZdNayq!L`x#a`%t@B2XybXhJZ}C}Pe#8fX?ebx1Ds4XAYOeLHC!$x(-oA-_x- z-hJk}Hrr$RK=uO!Q+E{|X6wC-sokevruZ`^y=Hha3}wtJkUV?K|8d5q@%2!YnM=%YuX4uP8rR5Z8y}2O)ko5;(e#NPmjb0E zSc?W+^{Em>Dy_rsAKkIaQMaAMssqK8B0JPA!gpl`^i^1qQ_K`*@0_1Zez3MVG70+h%$%-?LRUg2PJ zthcnW>qrZEIa^x1;lg(BDgE$zVLIMf@qWy{Y9PkAN|F&D2cvKvNe<2Ua%~}F?C^Tw zkJrA_b`})_;ikxF^Ucrtq?mUOW9^Bx5TXO8-+ZaO4g|i$_<;?B=li(%EvXwkf36D- zcTX&J+#?uvW1j0{9)|G|HqAwxFI74nc!rytLc87-$=`5e$ZPoKe#K_+S$RnM>Jqm1 z+NU}0i#X&)JgH1!zwOM^H_kB7+U=qryU&n?NHR65MS*-eT-W+;nX~R_o*&_dMi>7I z$H=^_K|_*ETnW~-H=N^sC?gB?X>Bs#Ro2nv1J zJi6I6ja1%fmzR^j(F=cY} z29eE897SOyeuLClkirN7o@^tq$Y7A4zI!9d9#5kG6^$`!>I6GS;f$81yfKscofGX% z-CD)aQ>VO)F!FIqg>nwv?kCS*`v7U1e`^t%*E-87~$v&m_t|6Z9^6xCBGw ztG-~)Lc4PH76S8ThjY%>g~P)TOvD80Mtx4})5Mt6QqN6pYk3Y{UOlugU;I^KVlwyh zZJ@TzNKQ zk?;wg01K^;1{O5>6La0{7K?$YHetg>P+QFhDiUo{Q*4F_M70_#++;@+(`Q&04^!69 z{y8*GG5SR^V1nk3`p6<0hS6*%WH^jphX7(cXITTt0O2N+5f41VcAFSn1u~EN2$F|M zdKx#+V&ZKiZeBfv-QD#`Q>O|&7$#)7&6d{5b4kdbn)aV?br!}ds zcatapwZeSw3XqJXt=$jC^FpSnIM4!w70?Pk4K*fj#p>Cbrj)ks%ZLVVBzeky2%V?e zS4Trfvsc$DV$`VevEa-1$&f78>a7;hcaDnYCvT521fSewX6c|o$&fmk#hKf|Vm=tp z-ZV$K(?xb^-IlZm;p7D}3-b~uLgQlJ*8ZRSr%mk83k^Qk3XjWp=vlW&9FYnnE-_%& za7pR@TLF=fa9(gjsgpbJ`umbb^&V~@{SJtKnlg`0WEMm*D5HV-Q&WIoP`^v?>nW_Se7Y zfn_RzoWP5u&BPv9TEZQmkB0yLZL~KJm3qz<*FMm%6W)FEohhqWxh4IxK7THP6J`1>}rx!56 zR>NMdERi;72VVrClWI7V>xPFiTFZ5z_pp`aICY3~>u?FPLj$IOf&Js?6zdZ%SbU4yVDhN=m zt&NLiX{EOeun;4+Z9<`$XlKMptfaZ7lrDlI(CD0xJ8x|~{DOqKm&k@*r^>0&II}41 zu(7Vj@6ES8&1gIE#*CrT?#8{oEW4<*ydWy86G@2;^T)+aaTh0HfYU<^+pX)bJw;Et zzq{EbD?5%QgRw+KA?w3SNeC=bY!Q(r3sUR-XQxzP6QPf#h+s}j;evnk0N1)j8Xbkq zyXz&>nqRKS7b7F1ctckj^;+t6aXA{0Jh(1i7WgJlHjTR|sA|bupx;LS-RD07x5wYG zIkT$ATIMg9;WfGsA1lm;@21m@WOD-1RhIRLy@7vmr$H&&qyn6K za~$u)NWG;|P#95XeiqMvthS<=*Z=cSEG-?N^(ZrusGz@FljrUs9twXpp-9d{ZHt8V zM;?~Ya^IiB;}%0{cXD!0dnAkqn&%YgG(1Ym3IDWP@_%UTd$FgW zp^5dtf*mcA?-4?IX}32@t7>9w8kgQ;PwpC4pwKMu{;5pqDF>bJ*R^Ke(zr zmwE4gjM1iYB$7K>u7jO(PZR(zgTIRrA+^cabu1!fC!Mi;vFO)F8z&+Srf~zFCzp9zyW@uRzySm~M3NWM{qTV|oRBx0xR8P;%y{;l z{Vvz61%y%4$E=D5TTKx6-SO^5lFcR-o?<m&pS?yQ~ z4B4uG&TJBbY)K9&}0%)2~`<((I{jE*5T=@m3s9WIa5RjtxAw zf0r*QTL!A3o!_U=zuz7_I3n0G)L9u(fyj`WR0{y%XnC^xpmFfi7YSqTS4K;#`ly(!kRk?G!DfPhh z%m8yg#F2T#M)#sm`3CGMEtc)P*mzHu^iIt!#p~*XtXT$Qkq=bHh$G=})nNja^A4D5 zImHu`F2Oq6fH#~IprTQ5&MwZcI2tkkgL;5 zeO|VxXF)`Eu@CKvGdU6T5FU3le|s%NQ%J+=)&Ob}8n`W4)!3q=L;*aEZm~v?3|A#J;fFuNcWm4^&NT`9p>MxcCUxWdiY_P zWhC)mS?M-jK(zV?NR+^|sf-kGKzY|99A98e-Ge28PWkfDs_P|(4*p5O3X$x8;{Y3M z7~6%=W{iV$LDtQj$kbWSGqzx)OPBC5x&H}UvfsOik6HJWA{BWhn@LLir?%VR3%Uz$ZI zI;ZP7{W|ElbHp9cx19dhiT}jH^LXI~7I^&z6^A+3j7Ltfj)eWs9{>L5&Eo*%?jn$d z+#3nK!HEyv)BP-j+e1_V5*?Ec6tc_RU7bH$3H)Cc=EG#o+uTmZ=!C{{zGz5a>1q~F z-0qhg9<5R7Kt|8g`t{J!(f;_FG}~-QR@dD%U#vDq_^;hWw08h?$^Pcn)8JRimyBPx zbGH5KT6C}W<^mjM*glLZr)j3}E;DL0;FJ~5^m;8E9#O~4!K)3<%I6%7D?YV;Fn3&i zNOx0Sp`ym3Cy!i$>!N1<^8M=`QT$qzmj-Oo5gHR_xq2mrb;8<^M*6=O&{u)TjG>qb zkUHJ>1coX2_TqW%H?<>+Y!%iTMuO{p2sD z;>K+n)8<{*m?h9(IX%0KTtOD9P%O{;Q~Vtc^RMh_Je<&Ba3Xl#^P<_~P0zf&he%&q z{Xx-G_Rm$&ANW}waA+It*!sAxk*rIKHuX!j3#DegU4&Y-kVL)Kd}B6uX1zQj<|CzL z8^Jk^$JM4Q6$8sdy7y7eL)T$U@0-%kf(7h=5TJ}^1mdQBe0(YmhVfWUr|IbevMtcW z;{*&^grzs~>%{|7EZ>#<{lNnm$Wz!;j3Zx;AVqQ*OZ3?WGboVOmwQ*-Q2m0xy1Kr| z_%TywD)v4i4l#i;WpuUiVY?~s1dH5MqwRB5x#xZUao$Q6fGR>fU5cdViGHOtJOyplS+?>$k#QWEfym*J<3(eq0uZ)#D>Cpny zjC7+PO^u^N!wcxf6L~Sp&U;oE*{=wyCqt-u0VnzabjuL(uRn~ZXW7gaZj9F$GZGr zUox$y{Qf3Jpa~(@9&s4#q`m>#h2Aq5OK{_!pgj%25mo(1{ksi#Qa^e*>e`(Y?WOV} z^m<(*^*v@7GwO!*N@j2Lc5E4`EtPENH3gmZ-p?->j9M?_DoVl^h+4$75-QgV9+-$0 zCeLHbTU`V^5HxQOH3^nopFr`adTkHe{8#vHu&k@Be?J_v;9PWyn<-gKy-nqb7+@>; z)mA_(A{-|!m7v+z@DQHjGZhm52B=lB^cpXxOs2Zho)QTJDNv;8O}HrLyQzJua8Co9 zFw3NFXN@d|RrYt3tRF>TY4Q`q{rZ9nHCyr6%d{%$=0YdUkB=80 zv_D++z1Y=$tJggIeeUG3WSz7ee=${2@@w)f2XXt=+v#%Eyxju1#LzHP@?Fs<*YA&B z;1%mRi|ymSAKv8Pb=aX^_mK;!yUG=i_5EBI$6^Kr|?JPO?N=eoap4|8%|GME** zK2|$y>>#m>^d$nG8Rer(p6oF~9-uFhIWEjH!9H8@oFB+R=Jt++%`$S=HH3p}D1OyZ zt#+bCS9asD)U5s(z4uz_`rjYQxZLM~Sehn9uxmBG!iD}GO#R__9VlQt$tq3mA@nI+ z#4sj2edw0$K9L`{bQZfBnD*wr%~pBTXealj1#QG1=@H6v{5-6d=W)#VIemFHAvW25z zI8+WE{H(#Hs+GbjDH@j9a_K#20SxR<(_P2+@{zfSTkqNP!%}j0VJ9so#bVz`@XBir za_c*BI^TXRj~6Jn5pQg~D7!XmGh3*AHv0Nq9aMbj*VE@!F553t`8{7h(B|{I!tP zN%HLjVF-?}%d#icy(SYP@xv8*)6(|dnAI;$q~+U9-B2wq>sD6qi2f;X9A_5d zdeAg28Q}O6==Tn#`lPrlWtXHopNW(%){78ojBVuO6Lz24Gh*K8_8dRFbN)uJxsuad zeq(*fA?vMy&u*H%+wC=(DbcCNtxz2Kq|)8>RJ5Rn!CKI~U}x@zeQ|beGq$NR46Zs< zK)wEl-W*)vf0c?j$zvr#F4umi>84HkXZN{|K=qh7SA(rwbza7Md%TD6q{7{s)=P}y zsj|F#6D+S)lidNYZ%@z8jylDxtrAczJ3o;KsuSZ}avleK%`_>)qgHK*oXZ`RYq=*) zt~D)cQTXs<(_hx4Iqc{UgN9yiAXU#Kf6vZtuQqOA^4ZSsD#^lqlF`CYCg$fs2-PdY zsrCh*w2O<%==JGhXGAgkg>8*iJP4K@pJ37|31@g*WCk89qaKDobv=hH9ZvL_keWZ_J=R7l>*7{CHCWBE80vs z+>4j{U{ZS-qD_gZM#bN`U#Sn?{;?Gb7bktO6WT2Qw)OBi~w zT737%`zdG1G>NDm&kgQ^jix@c4dW3e0XUEbXbpNJh!o*-6$@4Si9#sGvBbavGTW3KZ0fFEjY=A&I z+0QEp%F(Q!J`yC3=#c>z3GHsLfknZG;$f&e@fUw~)$z|4bb3w>1Xa!!fzN%rh^!d| zcLddm@)7-cTz!Obt8z^mr7Vlu!?f1uI@I~~b!f?29YZZ@+;=l!Dipq4{&|@G`X6drvYwjOy zDo#{F-r6L6#ZEY%B@=Ldx!3yT@@#?lPcLJ8)`Mb%>1!-Se?zpsx3Cb5?XjHak5H(3 zAtG))r+5+3`7j@gXKMf9!a#YuO)iil&h0Iq3U?dVu^(|Eqz`S)JG6~}EUjXq$c0Jp z--qAmyn_wEqJ`iP(taxY8qs@i_MYImDk8h@4>e5fMrs10@-y%qhgwoTyvaI1YTeP( z#%DWE%60tdB7?upz(f^J;7gDbd)l~E{HgxSDwulT^$!gPj@5$?^o(hi86qq%`iY57=5ndHb8Di4~LV-SRklXc`NB z%Bv#q)4HLo3un_EJ>-&Y0X{J21mswWF(Ed(^%E^ zW@L5l1cjLRa58>8ED55b$8EQ}Nm>3dgrvCiF_nUW>(r zs^@J)7x)88q2ic!Ck|!y{;58GFSE5{L}ia>ib6RcKLfd)UI2m9%H+h3Y{4)c_m))K z9YO-!Lm|yg=K^V>XZGjGGN#kb`;_nr{2qAT?#<^v%=Y9P(xkyAc(m&`P@5HbWk zRG>jCio3(Bl&U_0)b+=xKS}+|${_z)0_SqZ%)3zI>T%9bXn4^_01}%>1`*(>*8&O@ z8kOxAowmxFoKF+@Ne_a3W*tND&4JcLP6O-f1 zNK#}R^IEAao|R}9VEISfVS=?h!RobA?OF`eg!0KXhxRr0IKCMo&8np^EvrNHJE*r1 zuf#as1X-;Bbu!ev$hR<9z7J|A52IpAqZ^vLVz1H_^Br2--p0arx1kX@w)sIibikqp zBW2n`z%;H^G#J;hYj37OgoT&q^1-ey{r5xxD8y_CKGFEBt*(C3vbi(hcQiNWsSD6M zM_~B)gkb{ZsYo5d-3q?1{9RLWpz?M?z2#gnGHLYO=J4y4?ixniBF|wafHq4d>0n^x z5=_(mH_OS(5+xA?T!N+^Ez>97nK4b!VKf~yz7gm++`I3V6#~>O%H0v(oAye=$!l=R zYf#JEGj`Kd7PJY_;Zxo=e0xeG&~o=<}`|`E)t}a)?w5#%q!SYjjMV>Bbt3c zd=VX)AWd9We&*PwV7-=#%~JLcg5l{Uyp?6SFjLbOo2%fL{w^4KpLR4SY`^VU&O9Th zATRov*()#lOePLE-rhd9m^O)p9YZaHOZ*73v&^1(fUqBBv%M-J&W>x!J7q8; z#j5&#(yudZ+Uo5!eXi$R;PH!TK~;10QrUC91YO-X)*`_P^g$4cV4mO5v#Mc<3#$p8 zWPRCzPH3=u;jPQ>2Noh)=%L>;=i`G7dXgWSG)4|Ud_!= z@eA;K+}j)5X=oGPw{?!Hy7JpDjZ0K?tr|_K-j!-g{%AV?G5Ujon?Mi53ir4utr@Eo zeH2-lqb@Q63VyPk`Om`q*kGsV9$u0p`I2i{*V{u5*)x8awt^78Udx7HWwHK?el zm%!ju4N=`0dIs0vPKn(TqzMaJR&SIIwKv~X%2sr%At)T&-8o=DSbVIZ87xw_6dMsL zE;A${7Brs6I{@qpoQlSt-3=h4D(Yay_#+3$vrqA1CMLVm#UXYRGgZY#Rgmy{<2Eea z7^hG@hg;Uy;(}=6WRH}z9Pubp)iyUS^?Hl69&L~KlpcfCsz9fpi`~Mx;+En0X))tfW``@2fynt&g()~NaZ=kUH zJVR8f(SimJ1EKG6PM?a^kd;iQgj4W&-cVC-d8V3aw!XqwDeE2?)&k*#YA3eE? zjke(pV+J-md$Rw%S3kwzZ0>1#B0gIelSd{X(5Lbt^0%GPprNBPAf+gmPtCCO%6tpy zTow*u&MT`s)h4GThn!{WKZ;*?b!PG92NEW-{BF0;@^AU3i&o<8@tF*FkQN;d({L{H zzZgYY3*Uo_PPTfv3AI)6{*-{ES}i5sB8)WPH9=R*#o0^`fl5waV~yd_@1@tD9}Zfl zSvgA8LOctkJG;hok9IG|`iP@fOxB(uti!{ynNAA>96%M=gEpJ5m0&Cayp6^$#+pz6 z`-yqIew2&@mXT!r;N0Rm#*vjCAUGv~7nk!soP&OrOjq7qO5h#!`PUKOeMg1JeGEg%f8i{$(NF)FG_czG+=M8CC<5qL{d={5ojJahg6nDwjahQxvbUMIfpXHC!_ z+wli3d5WU<`eja?$5WS|f0!ozIXP5Lb7s2U%{W@_P^oh7G|)O*a?SW06a1W1{~*@$ z&xmcs&oo1ws(E|gfCSXe_9+G^{&a8J>y<-6TIVyr6Q;}ioL^4IBOXo-TWL7?q3B}; zvQ+2UA9bH zUH{B&|7os3W-U65OYr4Hb9ZC3z$=v58v6hus9;GxwL(kc0qm9O0f;Bg~}eXYpv;H7Tu1iHSV|B4TIS-9fkF1IG)VdW+7l{ClDlQhl8w zKI%7=-SU!mJ;?9HtNuec|0|99KE~18>SY7J2I?Nc|3}wbM@8AS?ZW~iCBh6K9YeR2 zbV_%3N=tVm9Yc4ggwlvgx1^*r(v5_4=kG%A=YF2|TkBhkKUlEly7uh7&%Mt$&f`b| z-G;S?7Z-SaE*eNZ;octG7|^k$Q8Q0EO{t=}iy0PZ+P1@JJAC!QUMFHZGc|u9I$KrE zbx$eD%%w!R$}Qq^W%Q~U>t1gai*u(+^U=>6WCkrT0O$lzGV-6qs9GgA2qsV8>2HK5 zkOaY8blB|SUIMrTOEboVU0;C%;fW5>lzX;_ZCU%47?F@?+OC!@Y24;S!#O;W@cp00 zeoy6C{+8hR_h^6oy%=tCnJ?vYyM(D5mdfGp@-pEvH=hycxV=(2cKba_Jn*0! z)i8eFy>A<85mmDGF}$D#SMoRF*-$C=2-^j$KmP4f4Lga~HR}uR8Wx=4oUr#JZ4p4@ z!SYAu)(+iV{MXj1YDpq%A!*Mpw`EP{ezYKvh&j%2W?(md0vPI{shA!OO#(HS0W*4y zeMEX19k3J<$=ueq_+Cy<4{ERq*yHDnYgx$4vT03oz5$FTR%^B{p4e}H2o710f6#tY z1pEfF0O8b2LhYodoX<3$BnxyO8pB$CUDf@T-630GToM0`i23DZ+3;uiUPcrY6l8RT zo7k_*x5IQcnYDH$(c~635>Qophx<1Z<8!KGZ;F(&fexRrqZ9LD22le+`0S=7|1*Z^ zH}id?GmRZmOjbK1eV&&$ozN3T?M7@8pWFDT;=>X_Y0d19Yu7D~b{l_Wo`Ir0{W}fw zr8h)>c=)BDlE^@g@v&*yGemerY3mXzb@dKN;xz_V+frX&qQu(kew`&TPEmQWr)xRb zSk9XoS^TC%ki_hQpqhT%k*o)UZu)bdle(mBTf7q^!d#?_bT?}1VjIqXQpLbd(_7&!za~W2RAIWH>KYX~)l+&Hx z7)UE(8IU{rT zq6fhBdnU9OD;Gfj0zm&}m^?B8KWLF!<{jm-aGuPOk{XlIMR|ScYG?(oX`4Iv?B@Wc znaq0=MGDvbzLD1Me$U8qChCu|xm|IRh%0*~Z$VF=z8~?wsc=oIwr%2ZS_Y%<&6e$& zS@(AtN}4R4>!)OSDwmt0ylXHU@}i`o&X_F8Iq=RG3RGi^m&JW)yxfBD$#0sZEY|d$ zr)9gU$5zO6dE=>p58s~isg<+9{pEe?EBCwGYzzPvZYArRin_?V!E7=h8|hY2aeb;M zb-mE$d!z5}dEC9FR2jI^kw`+^O3UXuAkN)`9e)t%WB*Sr;_q2_j1wRcP4MVd-fs=$ zE(CLv0M1i;i@eP00*`V^N;==SAB|2#B0;w_pS_ko-mJ9`3(Te+ZOe*648Fn^C7gHC zAQob1uYuQ{B=VPgn8$x-d0xnbfG$q}!r9 zIs|SGm(N<1fc#EZmOyM3&4=4*Bi2?lA~rn3<}z9H*G&v;zsIw zWCCwjg7EEl1A>Hn=987J%03OKqS@wzCHE)yHq|d43*5vJ)osaK z<~bePveHvuH@wN0aB=?6Jp^Pi0WgZy!}iz`QNP%{GpuW7lM3~E%ks)3&q*gF!jE52xJ2lA3lYAk=$$AGQzCC1 z^W+~bDUBr&%-ow^eXn{p4hwa>noeB$ZTSBbik! zVhzDt#$IC-vs>wFu+g%Vw8E@unJnObtZ(y*QUR6AVlVyY3)5$C0-~tuG~;QBIrrbB zSA7eUmAzbUzLeXfdp@?GY5>^h0oS5u!mTv_#||jtvO)Ye))rfg+&C!fPC$n}lJTnO z^n0n|ld0(BFp)M=L3c1xRgjOO zmQ*`&001QvMgO1ubjS%fo*tP%?Cg&O5A>%ST^H1hK6ZeUTErIQnMB#p20Icyd#{61 zGnp@y;Kkh>>Z8DknK=?9nteC6(cj4NvRpUJlgFF6{^BFc!ZcAM54Zh{OOtZWpB8Z4 z(s>u>!{SfgL)-%;Ub3mhPLo!uRXAJoBgO6sZeu39TR#R$+FaKRs#@xFi_mq%$5b{y z8K?%eEWF&95I=LXs^-JQRD=1%oth1Hv?Cl;6DmYd?KppgWW;I67MjY66;7N9hIn zo(o3o`Lb)rV4R-*^Y3`INb$#W2)*fONn~YgN%*Qe!jSt?@Q=XnUjtDe_G8os6Ap~{ zB5>lIXZ0-4Zh9Rnvl!E^Ti-uoq3|9`*li>_nA!^ zI?-Ios?2Ytqso_XPb1-yGoPquz)kAFU8M`~S8&t)Ni7J<107hozTVso^HRbjPo`UD zp4gAXe8Y{5S&_+qFnj*}#lgUpNx%Giy-KZ!HYSnJgJ^q73ccW!6nsDL3vNTqJ`#9M z#C_5d@}@?`_)Gz;@45wQOpplg%%llR_kpmj5#7iwD*#?ICI`J{VKGVTU*34L zELGxMQYt*e>F<#uwL~$8hLOPZB_L=wAgoPOXy;i`VaKAI0|Lv6D^%JuR|>+M+!reR z9<#lOy}ebv-&zdcxQj3Zpl1Hth|OuEBoyqN2Q*_L{NSNPNdv8S9A;;dpH)A|5pfcQ zcXxczba>rOByC)4gOxZv^33-E?HO~HaBo`rFPFJz2GfcmhylVc>I*+b;H#8qArq)0 zizct0a%@=y0e#-=0cG^_Q>91kV?cm|GBS3a(uXKKhqQN*YU1Cp^kfeA39bU%Ge5Y- z=U?l*8yi0mac#Y`Ak+IsTZwg`*ZU7&_yPFsjFR^I|Bknw0*Vpk z^VUVOr^yunKzANCNxm&2{1DroR2B9J=~^TXp8e}HjLE;*<$n`@w1(f^k6;n|K{sSy z2})FN#^SJFAA#~2VdfI!$Dw&%d3?n}feQY^DO`be6loF{>V_B=~=$ zQun%?HLtj1)b;9OmyjuvaPfU5b(s9P0%MZxE4?n)yefZUz1#T55YG`12@r7Q8MyoL zUi8k9F6&REyG{?kS^~j_|Js1@b{^|;^?Z=2s-89cj8NuZA=7F_-sBXRr%@4E0ksed!>1ZGZJH;4Bn-z{f$s1uEly%uM=*FLovxM@N8Da)< zWHiQ(_xgJt24qnwy+?<_$>}a+->wKzu#f+=nBc^t8x4zcmZ`GHkJ9+gqN>#mmbOl> zGERYo0jEub<|{N_j@e?+20)O4a(jK_EL?6BS*O;IlY8{QeywI(*(^#PO?`x#{(t~> z9;>%^DsxQnpuU;bS~v^w$cDK`CN)lf6tIgJ4f=-`oH_t{@>g@K?>C63MQpHUO!tpe z(dEkkL*J`zcbeXG&uD*3yu*}l?QcwEBTVCY3gAC|Q0sc@xRFn${E9gId)Lh8eft8# zv#IuqDmr`Ix$_K!-Q*R~1zFxDvjX*~gpXMk&^haUk7+yqpEaPgALO^4(!nMg#%6xh%j9corkT1chY*i{k11W?@U`Q31)hi7VvaQ{~s??JFUR33( zF(qb);y2K!jTm>x7OFOgvc`BMMX+e9$4BtG(0p93pf!OY?56`7jt`&?c4!aiF?cC> zIE>S-F69yq=K$etG-%#4JWXHT9&(cNHyjI?f4v-dj2W-;wcPRibSLezrpnESnlutx z%qq%emt%wE2@N8;qiU;X@&s;H8Ho7d0>&w!0%!#TlVWe?d% zAf~#P&(o?l{y*4CQ@((tX0gHVK(p09GE(d4@CiSujv}g4$V>3VyF^8s1q!96GQOsq zaK^@nkQm2d(*swuIkoCUyuG+)#Kd+ zfRymK&H3cb?)9;oJBRoHVU0N& zSTp^LOsaN7A%%b0&t87ba{E|9wlyr_W(8CI=Z)!q9|k13lYrn3pQ_gvm`>ln*jSPO z!Mmmk&p&;$O4#20*5o59q- zwi0jr$C09~Pt=>`YXJm`N`$(S1e)#-%>wGXs%YD7PsNhz!igplWV*L{C;v?8+T)aR z^PL0e&O=A|Z2$n8Dq;^w1dUKG^ML?b5mcT>2MLZ<0%j4`HJ5h7beP!>@9YpnMprGO zGrv-)dEB_#@1T5uSYR-Ac`v9`G?#6T$JfO!s0mtfr$-U^B8PZ#96!k;0bq%D#I zd=eagAewzf@HZC+BP;k09){Eo4lQhf#ho=1n!2*QLgqH?jL?#nqFW`6u;0NzfxvL2;74U_+z|PxfaY=9b)a+0vwpR@ly{t}fS)w?1Ka{$( z$@^o*Z*R+M|Mia)A3?o;5x&0iPznIR9N3-421p@bGwDo}uL3w`C=vN@1{tq&dZM=8$&Ig=&Pit(8T<>HZ|>kT;u=z;fY1QnxZaP zJw5D~-2+&uAX?vqBAHFIwilKvb4s>BY&;-^_2K@RWhuY3&YAk0lZ~^hUgTO$$xwFI zPwK6nI@*I5X4pmq29os6e?ZHFKoXwP`F17~*u0V`rd`11Mg6;Zfzl^3DyoQxNT6U~ zP|!OQl`QcT%(zQJ<~g=<4}!NpOcj~ZtL)wQppv&{X1}%vi`_nmnKp4m0sK{DIq5Wyd`F!TaUtDFz2G-%2A{Y-_^ z{?8TCrw?LVqcTo-WAkTU#hrlUM5TP1Q775~NVt@6ZF!Qa0Yy=nCntrH%0 zdu7bsn6rLu=)wu5z#Z+Ma(Ao3GC<44?JV(IXmAriWpg5IIRJ*Etq5fY)?`oapQA1y ziy}^_?P$yM2*>DO!6Yc$BZi_Y^Hm9;N&xLe@#qAm2K0AogoiO{)r#YC+C!evPt*U| zcK_sF;sNtC2$og{-om5UP6)h({vY#nN)e7Hr|%A`N0mYQ?k>c<<2hv}?3}^tqByj5RU!*v9P{uW(0E~Z% zR8$3hfTtkfY62+u38Q1-u^NLY5#+lg=+6-Uf5r$56Zl!AQTk#of51@m-wIqV*(b9B zoJTq@fZIhRnt*;tgA=C*`@6&pQN(|FBbr}B4Tb%=CX|r=De7E;0^07cp8>>xxPXvK z;_s99%9L~Q&cgo_HV_4TdmoUl+P?Vr-IU9sAao+`JPTETVBRM81%SGb$wK(IY8WTH ziHwcDQst0N(f7|MTE!lz6}-5(asT^N;2aVc!N(cyT_RTi&`;lG*$8GdsO8Gy%6$7lIP!#k z0tT_Ad}2NWaV;`~w|S8>Cq9^@p_$VBb4j(SViEl1)y$2z)kPt5Q?GMY?z=nqAGMTg zHb*S$XJvG6DrSd=5@g=zas-Z<=$FOlRLBYyD17DZ+X2X?&URSrWi{O0mPV!u35YG; zB)V=}WG1YkC0sbC*X0Dg1G`)aq9P4bO4YpFHMD_Ayg<#Ud%R87U&8~WkzMf=l3+kk z0|tZH&Vs3?xdwF(roYO}w{ zNx1tJ(h!+xy1K+i5hihdf5D=7VDyqZDEiDmJ-h1P$b`>0~~ z${!%USVu;_$L{}AvUpsg$5j&eLnGF0iCwK~ZOAe2TTtnilds%`f^6>-(|#P}sNn2$ zhsMWo3neNSQxv{c;>~F5 zfP(7(e*V{N4Fdpz4y(1D+w9B0f6F2H$ywCRQ&kO2;j4>*p}_cVkslwNjI6`y%aJvRKlcJPb&uEZg2He5 zQzGL=inZ6jFytEjJqj%W)|&NGzV@U4uCZ7ipfRgs2T%=KCX752xywj()7e~o_cSe} zCmfi$0DseL{4e`~<7#?BO$yPEqB_t3n@21j=eJFzf9y^e9-js{Esz#GNAtz;H4_HB z78i;^sJx~J-0nQ28boMAW@YP6x7qEN^0QB^k@o&z|CWZv&wGi;d;uP7e!^(BQ%+A9;bU3IzyEwmI}5CEK0AK=8}) zPqRNKp&_(ASEBXsr33oEV=KX=1-dJNbxJAKGr+oR8{Rk?8e6MR@K9&Ii$blvK8S&@ zuQ0GN5bZd-h6dQD=5#_{m`*a1y}p9xYX6zHIBuh-sMvjuU?A;zROgTNpW*^A6;W8U zxH+UFU!Bi!(!VgENt(d3S0)Kaw?6~9`t?e}H&w zlNJr2#KIOr`cJhNc-kNq-fRUqJ$ALw+lcYL5r%J&5!+YkVY1Kh$sKrB2y(K0RBSgs zZ)L@g{E8!FFqb0b1Kh@SL3lvBwFl^82LLAV1(5~Hl#)lz4*!`>pl^mhK!hv+Z?P;Q zk^Bf7|4U_@DzW{T!t!nU7faj^rfl%`gAt(~5!^UeQT-Ole}ec&_TL~R3@gQWK-zJW z7X1GmDck|LzQ72De?1}Jy}JW^+rlx(hpWLvPW4<}v zHqqFaNtcKx5*-QncfYgAyg6L%pjG{VhQn--rTn7dv!T|#PulzgZxSQk-q%(juV%b6 zl@)t<@QRMK1;QL(6PdF6vqHZHC(;N_^k}ziFP60uj?IvJhU5Lsn@r^`m6j#E)ST@?-p@$Jc5<9(V1O zhX7UA&OjK6S6_MmD0?(HPeg9ezeHcA!R>&ECm0O&2Phzx0r&^La(vm%t60Y15`+Xo ziIg%tii4r&7L5iN2dxFEFuF&tJNV`cNcA`SF*minV+XQjK38MgVOdO`0OT%T_46-_es zz&5#6tasRZ*w&-c)87p`?tMPT*rppvuPsxF#(V=k1LAd$JoORP&&_4}ZpX_7}KV|w^NSTJ>RP8+X) zPc{?id-#q_Zis$0)ZsxL;DQzR$~T!_{To0b!eTr7YTjvpX?rRjmsg5EY;uIFeH`mz zrEZcYJB5fi6Qf`%*l2c2$)%Jeb}AWDsWk5jYeM?f^itth`JO1Iu}bf0y^)S__}DDP zuBAESqv=ptZb!236={VF$&Bo1r8_tn41n?ZJjR-W*F6q84T%=%}am~^~42T zK}eME-?VyQKWl`+!=Z2|sv#!_jgXz|2vW|*q?J8&xU%xqlR_2mXlJN*V-glwqJk8jP)lm0V>jqKane#10y6zQ~`-&NIcFxXf?MOZfg# zj-aE~HDps(Z5Lz8BovXFT*@+5nKuC&=?k)Adg4T{;qwXSQUWg-%kLkjNF(xmq;)RT zCnYzN^QORgo4Qb_L0Me&OMN+&V>oUcLu?~2yDMdG7KIUZr6JkPPirdLYUq&?-zK3P z7q?#WpZ6Jd1luw!P`Xtj2E|Lpb}`g+imKGQN?iMWhG}qq9GJmz;HHN)OMZ4+d&llO z%9zBg5bdY^PN9;_U`SId@;(toTn!*&>XwgSNxE1!6)`%yFB=Kg+5H*JqNjcksW&hW zKk04Ac9~#I`0>}%l96a*FD9{fDS6s`IuXSqu~Ez?V3S^sNg;j~9P#54o??RZ&P4{Z zSF%@VIjYvQsfPIJ6THug-4$oPTgWWRraSwW(&6F7Dov&ozRHqAmQ1_8&5=bL&(%)X z6lo=|XUZq^u@#-` zXP){cRedUB=zUs#%nlWna+_ZM@k?oujn$op82)9=tb|n9_qRiVW^Wg(nM9tgN1yhr z!hPNlgV!Kx3@o~-dZV9oP23s0;#8?bDHM898pubG;$XSL^yq9|3I_OFmtNk#;t|CJ z@;dhdf{-sU(c>5k$5PA~;p!p^?|pt~W2^W0mqwg_hRaY^Bgf6HM1hMIXH%R?l^Bo0!kzWj}m@tDjpwn zEF$KB4Ehx4U~UjhPGJNuwhRjYmHx&7lh335FwoJV?}-9YkS+h2l744Jsdw4F&(g(B zm3JdyQ=`BSj``A>l?fh;gPQ55J@x2SY#8bGac)irkU?m*}Z(7GY~-E9ik; z2;;G#X0hUX=_uNy&#ruLHKn-0u3*d6aPFa47DqwQ^*CQIMZVng-oR&&juu=BMw>7- zTu;R)K8Q$+yaE18^ZB8{pqbO6FCR&<$YA1KFH9haaW0NFMYiFp>DzKW?0y&aX8a-( z;U%)JBV8T7t6I@5{1?&D(=%eF;^gW&(94fwDQ}I;d!}PK(uOS@F^vb;ROe`deyWA* zIzyhrs8E2r(iHTU?}sGI*re_M+ZkF85%BTFB)xp>;c2`qdtWkW~js)vz@v-yKkq=O>+P*0HrJ!w)KKp73Q} zP?uJ(oSu)du(k^M{M_pd8L5WR93IZs`?24BbK-dC6}qf|My8VeQ@hyn+s5T!|6#V_ z{rHF3*qC9IhWq0KUYpv;a?tr*iaJu?1D-|u$@F<}-Qvj#a zB4#mTVxt$qbFa16j)Bsdmxe@$%W7))HsB=8uu^c=h%}dVk+k0Lm5hSm(XH?O7iZ6# zH}zTiro%RS7^Hl4d_MHUaV@&ZK zmlGcaIhxOFs#!+ehab3Fwo9k4zsCr>N1R&ZQ8*AoXkSx^xw-MTt%kiKk19lO zJEh2daKkAG|y2$kv6^a1mR$Mz-cd66vTSr?4LeQ`GcRMIyVoiUN_Ylj?uL8hR?+ z{Cc)qZZIu((|?AUO|f1(Hs}r2g~Rhb3AN={PzeWlk1hY#Q{8A3tZ7XRm}Y}RdOSqP zAcKu|?lh9x^W$l)yS2(Yifo7K)EQ)v;JA5hcz8)v{;%W;Uytg^dyW}s59IY&uADjj z6nf*~m5s3_Q5~Oj)G1&bl!MxEgpQEDw|Q@hR1g!@E5OK!ScCn6eOPvm8`NP%1gL1G zr{U7nS)os@LDRryU8Yml0U<|kB#_Cx@TX_DtufHA_IPyo(wL*(h)A&^-_lU%?R6eN z*$3RUs?1*|dV6>Fysvk%vv7nCryZCdt~*tSvQN`rLPYS9k&&HCSA2dJGqbY< zId^*&4=cP)^7|47ewT!#c&)os7idDaIY?n#Ee9(`CO12KxZ1w!V;(Jbz8<&Fy=&IK z57*Tu-spD93LW3ya&0=3;tF22uHbUE@ha5Kbut~B!e+ELYew3vJr-mS@b?Y&E#nkJ zRfU!yzmSQZgBod_UQLxqdPYM1_qg2Q5`=ioGF=X$zP8b9o16Tq{C@!+_ z=g7u-MI^Y^P-#~vG@id>#7K>VVbaCD?Rr?bGwHOEYB+s*5tD_YLP&j*ExUTWfBK0B zKY%hfx-Hppn9|jp9DcyAD>0`GyMHoxKUX!TIF*(zIl4@28lFq*oY`gAl%&Zc#83U zoguYZ^w+O*sgl-tUk5vMh<_$T`)u5aCQZK7Z6!Zw!6t&U_1y&oNt5=@7`|KuROq^f zjE6&vx}F`?Qwvz|$B?0e6)^80??Go`yhe=GxqQyNQD-)KmtVI$$hSpg zsm8z)oX7JV&HPk%La*W8lPV&>FP#W1oxD1Ax^WE{yceqSeRCP0IiLD1-6AYkx+UK5 z;ciZ`{FY*i9JXigYk=_tZX>|?!Siw!PcF8K*;IMt{FjbXnT{qX;SSx+kFKo$b zcW!GOQm=u0J7s@2br^u*^TuPy+JIe7{+%G^X-&lX&fR(2P-Tp)&>W=R92pve%<0`( zj2L1#36Wb@qaVyBOP8X9FAeQ7R*}5D-sjUqXb6t`lygmtr3{xMvRG5qNI|xT%;Q8~ z0eYWJM}uPcy4Xem%u+9_$G-Wt921ER{g{`8EI=zzcx1Zjmgy{Hz{F&b=v*|WH#CwD z^85o0oB8qhI8WF&r{Khzu3Bygmz0#CMYzY{J1S?#%gR`8SJ@xjq__c%w3{Jm_7UvE zA>m*fO9>UWdQ9Y}nj+KT1mo!lL_Eygbb}VIJ2FMd<&tP=QBipsh#yh)cx+pZR}%60 zRz-U#G^9%7Q^@7JXEsyc;ju~3m#^Ox%?HhoNZ2?;_pxGyMR^pK%5_O2&@h)s6G}r$ z>)BDeim(>alkgZ`ehUOBOU{se)iN=L3>vwAN^N+GYpFuST&ZZLr;;%8i3~}-D6Omf z!08vr?{d5Mi|F#|<;pp${eFhmk~34FmKa`*u3-Gv_t7oLj)#0RC5`lRkfrNUz@+Cr zKyh=^f)B%9?^bW84l9>hSep5&3~zpQuV7hQsV`jORjGrhoU$HX zCcldh2t|?PTnh=O{&r@d{+c+HW13gJT?X+=T`Cz}8uxJIPbJ=AkOz5@_q7^sLTp|l z=KCKI5#Kbac3w@8I95>oh}zJ+*U$`x`p;DUX0S=GB^#BZSMg&NGb9ha$+D)zy+W@Z zK9hxDV-NA;eA9~w5j_fs6eW&qfWc#hYZ67fd~~ovdFrTDIJ9pUnc63u=cqsoMh~w! z_2w_3-QPHc{)l_=!^;GM10#odqI!s_rAL>UeUN&^x-v20RU=W^Z5%Vk8Fzg4`{&Mp zPxx~ya|iV_Qb(}`G&_7Po?iBREvA5unN-o)a1ef;7(ILd?#-pSj?>3tTX^=}Aezf^ zu5VLb#x(}PD0DB4ZxYfaOApwa2B^xxb+r_DM7gbcteBm^1td1h5F;Z68F#7F@4GJ9 z_}0+*ZVjsAi6f&T1$H=^Ps3+ivD43BFcTWVaaY1xoF{EzD4WjWp6U?t8Cv2m7vpil z{CG_VNDJ{G^PVxTH&&Ou+0#PpF>hV1B|-h6!&TOnhE4f0NA9Iva(yoSIrf(f$&>0FNZDG4zOouIKx`oM>NPcb=$G+fN9u*S2 zyRUnWalgN{g^53u5_jxJ{IJy7GIA3-T0-$4xY8!dUZS5oK~MfOH7ARBPiMkCp66J0 zj^+oXeg*k9?BS492F&AodwDqhaBbMOa-_%P|1v`Fd9rUMg+nwr4)rr{3&f(+g@#gdF~07py4&FcL%7l7#v8dz8qFoK6q357o(LKaGe4l;y&pIs+Z@M&h_hAZ zqDOCV8;FQA==H;qMSmDm8apsb1kg=jS&FN<&SX{&#)Qg5L7cT1I%Vn!Umn5dM``Te z+6cBfWyFGuzz(WxnwDa21@GUyjhyG|pfwixs@GHd|m@h`E%C1^Q)iMcIfCRVfTX{U)b34jTWDg3p!JxfmMB|iS|C`Ag`G6lyIWr z^2Vyd#*xTFTr1BZQXrW^{~O;a`ow`64M_R5yY`EK`}iN0)bxF{`~o^$ZJ!NnxOQ*RGQ_~sfsD^dw(aC!vS zs53_maWwg+8QPuHK{|?r*YVHUivkK@%^D!W5xL$6A}x0NQf-I7I5{~(r*GH2TMr4H zdqb8yM-C~L&QildLN<49@_DzpTtU3>!~t_p;9T2?04?(+AtFFM`$Jas#I>&r@ULkU zxI$cOYI`Et3?$5$!r2Ji4%g9P)*)AxM75~B%>gg>hT=t>arp<0opyw;3cg_ zh9;%CLLjWSAlU^CE;pCNtD(jeSzvBKJaVfNj5B9Osutsu{LV^Ny^{sqTy<>GPc$Bn z=)i=JGoLQ(uQzNfn8n)6e@+80BLC@-Tm(TXe<$nPuVt!jN|6+i%Qh z!ST$S4!Jisk7xel{42(^$39%fVzk}OwF&S8sCupnS+UbQ!$QT@&^sB)WlPB?WMW?- zo`n#6aZ?nEzLJU5AU#PL$->K=?Hck7sZZH_{o3*8F%s*EIdSkAbZ7VpLqnt2q6+cL z+1z4o6uz(6maKw*b-ISZ7xd8u&b@W#J5VH#WYtU65H%guX$FP%-l#9Lv&+oSN)7#t z`BhC)7s5-kpK3lf_I6Vi8L9yt7cV%=-0caDJSF2xhT)UU(vr|qi4(*)mbJYJb&>U6 z@gZYtPs;0P_(9v~fgNg#juMCB`T*8~hXHD#(3OXNnO+bHeRDxBDHm20t`3QFs`CT~)N;!o4|xj#_5dG1dQ(kG z*m^tA*6fs8CH~MdQV{z@d9A|!-7cvW`Tm2kc(Tw$``zZB>(v1Se$3B?9o#J*R7gF1 z7rLV9JUtz1QdP$lk7-Ak6m50|0~{(Mtvo0;#0*b^BZX(5vTfHnx9u^=Z5Lxl-5SRV zbC$sIC_pB%UJFC)Yzv2=Gr@>_dG*vah(A+gFctKe1<1Y;;fM!A=DH~F;HzuTb~mt5 znG$j!C@-j@)(Mn5X2j<)dZ|UyfW&0{Rs;{Lqd0V|)`;mdNgyL(HLrTLI) zWL6Q3sIR3Og=CX$C$+xo&PeV!xW1)c0=p!{y*t@ETzwT935M5^k(IT;L{NsW-KX*G zP-&8h4~OOFHbj~I&P)_}J&o-pFNhX}^12WUp5Q3pM^}1>EdrkjA2Pu#%!KdGFoq%2 zA4IqcLW2qY_8Q2UAZ>yUi z)?1vz>O9_q@aw8-i|oP~Iohfa)&c})AF%@Jnm3p_!-Q^E==aWD&gX6#UBQ=~^wZ4>W)G2TH2r17yjbL@Xu}!g(=)j_ zl+8~hk~0NMHxNlN_GZ>@CRGZGE_}L8h|-UQ0=RgI42Tn@uNW-eJQ!d3r3w)nb4byS zTQHQONyov%0XHcUIz8%f0p0GoXpyoJdn$J2BleUWb~%yzW2%Gry?|3$3;f43XK%_>msd5Vw3ua)W-`OK3cERNlFtm#{xd{>m{m+e!Wt=D)B&U;Fo zCOd5hFUK;= zyWhslbD@qnjkO1}IB{-^k+wNCkCoGZD94BodNm5ly<2(DFhC{K@vS8|XLIK}(N4Ne ziI9Vex2d2rheji9$M9{vPkH<+#^1|QrcQQW2jW|s3;mfs5<_}qN_YyEE7eG=Ap^@g zAzIFV){qcRA^zvjpDQ2zu0Mvkx7{o#x?i-kQN*;5uVCW8{;@z#L3!;BIJiF`9z+2b ztrsn4OK%>S?ng=QMthm(3v+=0Iq{_&#_p7u|J#5#^2R=v*Bur}335+_e=rk4f4N~w z>&Wf|UE1(zGbXDA)^j2W--wfp8qB4aRah2j&!DF&vB)Iwkh0nJraRG_qP5La97s98 zAR6K}MY+HP#a&Hu)4|ZGL?|^*HBciRQ80|fKbJ{J_IaWZ8nr(oYnjxPYFe$#LW%hq zXPhsA!nci^9{SWM1V#be?q&I)EGPW0rx0>@R`YjUt_)*1=vg!>uVCWFJasKfcnzBG zp4-OlH&HVu^&Q4XtG-Z_#gFe7jrz_D8u^JGr>^E&^y#Idc!PAxm+>eM**KZQF^Og2 zKC)1$t$+|uZ3O2te4T45R4`UyDVPsNB32Raas2CCg-nsi%A#W#)o$zOB3eB9hiRyCIEKp#&frGCl_ImB{p zC_Nc)>uC^D#TRU8;>}vH=Ofy%HiAxJx|6qih8pdl9A`xgpnAR@;Qe`zw-_<;O_A|% zAhCP^xCe9GKsjgS7SF5v?sm(T+Rmr%XB{?V=1^}fNsHY*C?OuadgKM+?*C(jw4g(vA+aC$Wm_=1Sw>-4&nK-eEMSLdPs6FviUX!6ccL zsy{4ibn816DIAPOCg7!&6o%-`o$aH`($$x|-Kn+_;FHmGAX%ZI29x)oqp+yZR|I|v z_@N?2r34+Td0`JiDxBf`a!aU(qxX`CEYSpYoNl!QQ6lRp`$|_;gg#3+92WGUi*)2O|uI4<;SmN>OT8e>lW__CAnX zm{v#XI+`1qcC#UKBDI>9U`kac(e+0T36J9J^`3uhk*q6?^gpLOOiFj5gPF z#lVLvtM;t63q^|&X?!85dpX`2l0Wq&r23wQje1l<*>H6_D3hV{)<-3FtheYZYgl#G z)+JPHuY-PRFi14K_;m0)yXUYQA$9mbnu=w;;2XPgr-}9SbNU?p>B8q&qCHj>D0tcYJ!8@RGh!PAFrJJ1T`-IJ%an)*I`%7_B$V8 zQ4m9b)DuZ#5a!F@h2aQ%RNM|9_hMr)WG(9wZDYFS@cbDd!h@9IGDzWX$;^+O0`z3~ zJS>?JF<*QX4Ohh9G7V0t5qKF{lBM`Ou`mE{&J{kokt~?$0M^zTBN<1Y&_3j7=k2H! zE=6W165LIDH#`05& zAmLl1tYgD3rA)QK5{?=yWm)3S=wfLa{9$yzhHA z?RR(RYIxx~{r2nPZ)K5X#fNR7yTTR0WZGlghgCzLv776VKTgvR2zMiL)V;osi^UI$ z17spH5Vv5Qsu*7S`jh1kPy+{F=;u`Q&$K*Dw1A>yVY12A2)TJjwQ4U%Vby_ z_BIfiU_?A21Jl#Ksk6}EYBnN^C7(1Oq2E_>RkYMywD?81YYTA&I*fa$<#%BIOPEWn zMp-bzYbH$hiK%Y7of5oq-xUpUU3PQ})KQ6y2p4FWzCH}0hw4RmcqV);2rnjXs9tg6 zN=dPcAL0X_qMVb|fZ@f~NOWs&PcuQuHk?uK2JbB}V|>YQ0PS zP{l4Xtx84JPq6y8AUA`7pCalz&z<&CeH9Sxkk=YszCcXWEXPugCQ=TBLz0T0>heD% zP%i}q>JJLh^ih~>V~o;p!xGb2Fp;}X@TwIt)PpE05lsz9jjJ-~q+A3kei~253dUL4 zA`?i6{}5j#P_ZADI2;RwjqOKE#iziyxlr8^tA$$<;$_9KZ=ZEV3}_?S;UHlO5#xs5 zOIphHkWGNA2YdO5VvEj^8ExHd1Sap6XTe2zUkQ!2=GTsTl)rQ~P$U;Om=+>9>dizy z9R}fxh?o6lGyE1Z((^*LiJ83by%M>>G)(`~ga%)|<(H=EO~n;^pr)#^R5pxD zU7@s8CCui*X&G(yG#yfvoSK@l=X6mpM2p08J;#OiPZfu+ZBJ~nHe1v2UKZ|0*ED2Z zC9l!GJ1d}YsDQ-ro{Xiox@Z|1lHT_|T#`R&s=72dH;JqUCq{z-EUTRFA!J#lp* z%nzB?J!lsu2=}_GkM+F>8q)ThAt)GWz1B@m`a!DJ<;Bqy^Mj{QiZ8ZjMI$Y~_j-Pu z6$Oq7z`$fesArOigcSG+?OSmEhn_D8Q8#_$>Y0=$?hYjm1NC8pO1gESZ?OcU0@};` zI^L}`A}EI?rx9;e?xw~HuNNIM zQ%7upED_GYHwmknVwlb>D6oCu!@m!C7fzT$r6wjknj$JH6bu~fH52ZUD@x1ZGf=AH z7c{o{q2G!(K+^oi9ERkgE6J>PJW-hFPiSAbNM)X5{;;)f`Rad3t?BcF_@lzs0zpO&-#FIK$hfEVbjQkU- zRfLc_cRwAJs?(6($|~*Sf)z4%q+=2zbx2@60I@)4KC=B_TY`CorA)lv_3q9cwW-)abhzLyEb zlP#3m)!?8w6B(=fZPP8K2VMMSO(rtfhqBt5VfYgKi?*$VwRB%~ zC=tUfEc;TFzwhm&r1Q#GULuu}uk7eY0po?MBWcE=nY&qJ*MYTaq{BpX(!sbTnp4^J z9W6%0L!nyc^NRWg@(UTsJ8j1OHS{T&-U1jDn&10I*N6fJGO)fNbI5bg4J3~0k!^lW zdqrjfiJl)&9|KCxXTr;^H~WP5zew*2rcH_kCH^%eI|k7GUE9M&P@_Jx?@?q}c6Py) zeG?D?YQ);)?nO&Hc`rfG=@O|&zUP~5pCGOxDq<8rkbcPzUJRc=P`)sE*_ECcgeRhP zwEOV(M%D6}K^Ff0B%w0=_o*s13}X}{;cN(bkPfzP346zA!0E0xRfs_};joS!qU|Pm zAU86Z9)-GSf%1L^lJP^mzmmLcJ)Nd#>&vo&c+TtZNwWM<8?axTZ6)Y)DED)IET^Fd zvW0F1=0oy6ut!E7q}gdfZ+`nQHLW!qs|yaaARY)>`A#&X>!=o1D|O`ripTVU@q0oL zzG@x>!IT3fW}Ql3g~@bE>!)oNzlBF0?c|3&VWBC1TlFD4=sBpko#8mT1E|d6r{MHDPb|3%;Enr_1^JRK>z2Q2xX75bx}q(H||A7WRI+h%rcXGt*nww_TEA^zuWus`Mkfse?1=e5BJ>j8qakk zBrjk3h-FgWESp|@bwUwr zGd?@Z-n7r$?D1gf;vSm?opjWvQ9FgtKuAOPcH!{bc=wP3PZN}EeWuaf=FY_?lz&2R zy!sJu9U|VXp;az3%Ym+l!(HmT1(-&LVLm*ZcwbF7LEo0FtL4?!Qkb>u_e$_k&}v@? z?+<4O+J>lcxLT$D5xZKEs5vv|y#jV|O}Ys2*sMobsBOt}ic9qpkMh$r>ILm-VW@8G zc7Yc<21E*STh&?vK1!ZJSpU@B%Dn##P!_E$qF+YJkzr_?2uE7n$!DS2q;ZVjpX0o< zNDBn{T%sjYQ%*H~^dA`WFRnM^?i8&jNW}*m&6w5-PVdPA)oC2R|_s~%41 zVuh|qC_eSKzcix=9;rDATrF@^)io=oR1S}~*2OSI4%2V?1o`AHNGOIe_-RwXVyk!u zsar!^l&bdJXEj7Sd=na z`7nzczemv*Sw(Z-{EJ$f`gr)XD(V3!m!wF&=*Umn_U7vMAycp0N!T~L zwu{L5&yWkWZKmeNt)Di?Ik3ZGYsk2rB;&qN?8X z1UFT^-@5^B&oY8=)Mr7~E7~|JTXBd9b3N=}_I;FHu3{)P9!fy3gV`3I}scbwY{G7E&(+ub!T2`nQD zFmpQK!?w<{#*DLIZ}{KCbGHIWNJ?sfPWK`eg>fl^$gv1sZqoj-AIFDJj|l%M!J3^CghKeGaYwOHG_@ z{(!*hBm#9}iy`vrTfvWTf9S4^DZ0ApLlU5NBoXhM_uc#~lAV&5RrO+>{nqk1iW%|h zC3FAC&kb|*>lU`BvbaET)dvjWF7$N$^2j?k5&bq?b}g~-H~bfkUTU8`Z^7&lJ$A6$h3KyKiezA|ucvM%$0pocvJ_0K#zLY!3f9`c`YFs2{wsC$ zIW_1%A(4}B2cM-#z*KGER7LIBl|rL_<)Me{|IP7eBhcQg_VxC~h}GqYa(`KE(@|;r z#l?j_*>$mpxBgOb#bF|As7p8zB8D<{h*OgvSR~)*UxKINsMO!zhozd613$~KaKDGq zEiw4X_OL|9htzr5L*AL+>^>=?`@vBRVk_v8^E%VUO6Z3=dJ0dA29{yhjk|^Z$TQ@X z#6^l;tv7aP=78a_Uhxn*S_F%Mije<8_E7&D}9C>CUxwfbFp z8lObR+Rcl6Kvl?r`3sL*qN4LQDmet5MBd(_@lditQiK7`=Un&~aDG;rk7I|qz}O7m z){(G_oXol;aVIk-TAdQUpJBo(7^!SV*J`=suOhq)NFwbM_L2A{a7heS}!d-|wUO}tJd z5qVEp*l8!ChLs-?6O%!$99jWUSTc>m`F^jAKmDX`D~Mqjpm4f_Cvq!1DcKfS^6GwR z>v&x2f((!LPy_{{L>Rufe1BVDD+Sa+qg#?y+9*zaMo*e(;W0M;tlg8p)yd-@3o>|J z&1bze&(Y-_G@$)3q&uCT&J0uxC<))MGmj6ViY>0C*(corWNH@nm{&i`p!U)c3+=u8 zI+MAAYoYI{g*-cwTGh*~olmokgaxY!0IuFioJ4*LDRfy;6%$GJqp>MqOOqw;dug;K zy?YAFM)g$gy#3>O;>x5#iJ{o!vvldOFc6HNn@($<{EkHXDeE?uD*T=Jfz5R|lTDSUGW{b}W&1 zV8A!gM2Yap-^j$_q;8+Kn>O#UL1k_4Sm5;LAk@qcN+Ql>DJbOA4uRbzB-RLA?*r;tAY^5f`M zQzIsES?q0ec>U&NdUL&J&YDUl09Y?tA?{Eu&AR|_qe6Q&p9626gH`0Hod-}NpQ$FX zQ(w`MjzV{AsrIogwgFx>5(XtKXOSMiGd$$^2AFanNL0f6Di)6q)!as6q;b2|*=aY; z$sRUki0)%j{~&8gE48)J#Uh@KD`^?lNFc0As$jit?CZbX8xHopI>=EZut?8}oKLS0 zbFO{~#!Pp2tl9be&%w3m#I^kS^Jj17xXGvDwoun{sb^Ji?hNmj^#ekiT>_#(Vm5j7 zODs47dlv3k0UE+R*nM~x0goNJU%-zRN2yD*f$<_tu;T4k1vQ|X1wSw^Bl*=)C^Os6 zLeQ1wqwb@lfR#Q7kuS6m=sK15o=gCfp|%@2oe`D_t?d?r?PMPmy=^UpdYhGeZJvhs zt_hM@JZ8|5GnY%g9i}dzKGY2h1c3LkafWKzkyT&u;IY^zwR9Y_l}VEj zJdt49-@Bhq%vWFP%OYoElY&b9un(a>mw|M)m9)2&?cq7BLy6XfWw#{B=tI21LR4M% z9)V37YFV`Q4QBhFrHkoHRl{lkQKA!%mOWRLYV>nnO<|%t(?LAe_%4K)h+Z8?F@Wn` z3wQrU+5^%ENF{h|K<%u8Rft%%d724%>xHG16uv6a;loD0J)1exsAe;qkd28VMwwsA>v7#sb?pNerDtsa&*^;JS~#XmWmkQG^)H$4NbYDt|wCyAF%x^3DZ+ zLX>nSPYx57z1M1U56fSU?0G33d&QB#gAblv^)s^?us8pAT$L0c;nvM$wEXiXWx8rV zP$cw){!+AZ>Fh7?2@I?R0BaOjZFGs*KnfBMLziIhv2v0T3!i|`34@^3Q9a8}x_R_k z#iLjv9)^d8uvN)lRK$T|PV8r@S0iNe92H0W_hT|?Er7Rxbs~e@51;UHFf%dGe*TtG zNk)UJGo^q0kQNg%ml8yR3ulD9CY`tJbCLD{(;S+$X`MkveWlUPZRDZKnGS~SNO`I$ zVi#&=UDAf6fR`?uUe4lzSpJ>x+e1QGNJ^9h6v0XvX=>qB!dV^BixI|)&!M(0udCSAp z&I_$bjQfb}`_Z)X{hKs*RYre+I_%ZvLU?0fSS)9Mlt183Hkh&K^rtzGjQf#25WbL> zGhOEAI5-QnJvgNNC~F&A#-#G{w(<>24274ktJZQ%A!GTl07()Z=VphdPCF&xXrkVY&pwQIak6+1dMN#gi~mQPFnA>(%Rkli1yp zSY~#kxc>*Ou!e4iDKV&)QS=4I$1151=_4-^jO`8HgjBM9I&wN-5mGoak%MS-_h1{{HG`BqS z@QGYp_N1wFJ4u4=xV#r$LRD^IMT{&w8zy}qe$PNr5^{rH9sL5uGBQj6oCM2Q&HX+T zdlLVPbBQ2u;)=Jv%z)GAR?J7=BgxUj5Ow_)O&i?Gf4dX*T=Clr{>s{IOKG_|rK`@9sORL(r~xH{}q-7EY%E z5)Tp|g%^Xo`y$trQawwZu?4KS<$xMZt^rq;zGUpaBMS#Ul|qApw-Ofkn7fGI+c8Zj z32UFY9$Z6aa#|vA(I1EUwa3gSqRD>_9QUX6Gulr0^bs^^+j;$LsMr7Wtc zuSvw|hzd*)#Jk}etMuLT*VDVS_t@l{S%jz#B&fQmC3{)@{`PgGx5U(um<$%JzN+}M zHo71zt(@*;hA>elsMuO`LRusP+9fHQPokTp`1$!Mgw}gyVU@mZq~uRTY{2mDUHn6#kHxf|$&ypNoexfA)H| zX;r0uqA*iJ`#{+?e({whdDG8hRkr;!nCscfz-1 zoc;;JygZ*}w@g*3((jErUQw{&2t%FKe2J}$A?A*aSo}4?$u&zxZ%FGgY41uxTHBGf z-BFdrX#9Tr`)8|I+AyO1k3;tN72B{ugIoOcO*LI_|Ky@S$uMX|nvun+qADMj=Z~-_ zc+z-qcmyteXRS7isgp99Lq1@_fdH&ya{|>7H@Y*exdh!58xyw^4jy?JlML(^Y%Hmi zsz>lBzcRy|*vIJEyKls`cpF7xR0_@2TKm7@bz>&d9AEj9Dd3r1U?aGbdA6P0SL=0w z2B?+iNG2x}@_J8K5TY(nj$x%*G5aozxRwR_h~j1lnKDMoqumEX?4cO|h^N`|R6LG4 zOi!p=WgG0ZWy4lizS2~j)S4VwMyXE={o5?Hf#Z3{`Rtc#=q^^DX>FS$)BmyuXbUjm zwZE;s{@IKEw_FF|iYMV?FSeLPOnsuJfy3viwE`sOHw{G98cI>?N*_N}ICb;2?Men@ zlVMmisJI8qFDU$0Bj(IgbwoxB&EF7gu#x$0A;HcKppoS{{)3O_N65 zPMbVfk=`bn7jyUK-po5f`O$YF3nZdKs&^n|?_yvgG|vm#`bhnyKu&o3XBl@UY&MKWRnL;S50M5Xhi-xo?@Q1^uwYHL zNlPB(kL2S|p8}W$^TqYGYX}QS#_do+?6fQ2(Bk@k)M|vXi))EgILw{*8}y&q`_z2) zsa@}KSnu}gU`}&(`)qvWSHq=OY7!C+mY z9|q`&)dCs^K2UWTY-skeaEL?g&0kZ-Qt(It&zw=kR_lf1s6f+b$FMrs7*E_|4ki!z zJFEDjOD#^CkH1>QE140l>NI&bY?K0~?->|k$!y<{&}wEmnq5*m@ivg2x4$Zgfc)A? zWIN~{Wo`fc%LO{K87Z#nsc;FO#l$t%X~%CD);-zL{62_sRQ@hW!>1f(o!5UV?@_xQ zNX_Fjpf1L%o&M{+H0@)kab?@jJD=Uy4yxOuG7ihj$oct}xegw(hi4$R>57nO4jB*% zsA%Ag_uzLad`h0MbX9Ny62pScnKqh1+U5<6V9YyEIcfm~Sy$fx&Ua{8A&C*+12AN$ zq`Eg^tYg-n{*dOn-XCBb!}3{TL>5xc3wu-ZrRKQ{pSU;E2RxkZ(3P6*$uKrO+|3Cb$<+kcvO7Lk)q&>E%ULLn9+2%y)ksF@BR(EhGIN7!%uK2OO`4JMlo1&7r9K3plytAD}lq^DU?I z=+__hk%~)W<-1eREe3-R2x7y6BroL8p}1!`{I)Ts=i=GS%B(;bP?E@@RnxL`28Hq{ zV{$=i)0?X<)N2zw1}fdsYFJ4C8H?5RGz|(3KbaVk+$Ot~N%2wAAoe4C*IP(=MVbK% zBM;GB)ejF#RN4Tw-t6&q=ggz_)OiOAuiN+Y5~p4Ww-a#3Y8CYo!@ExDOz6uI@10+R z!cbgiuE*+F)G#2b;XP~|NpPrq8&*yz190b8v4%+WR@%wsB_`2TqIwe4?oTRrkA5pl z6D3|2`N`>4oUC59W#8x=#H7qVeW!D7T3C-3qV(}vd2dg<1)=4llD^m2*sUCuOT*aB z<6%DhB@gXvA6peEu7BfBCiiF><7&_Ww1Ck}8$eGIbF^pfyYZeDuKZMbbI!af<>lyO z|FJjUk`(&eO%1+|{*lw9FBS+p_$@vd;V^O+^t*~Sd^N}GrZMIXK}ZuzPO=+(eI6OS zM}5_*PgdLY|L_1IAhFYZZ+kfQB2DqY!Fls{!*0_##m3@Ci5=a9{I8z!J8+zmrc6?U z(wm6KARxCV_QX>iGBxBN7-0`PXW`s-l>KTnqpk}X>-N|NiY8mFsW|UuNSm1T6jM|h zVS?|$H0hLd6V}!X_)mhKl(-3fl86+eK1}{0k`c6vvlfO`!#~HNv=$+-t%!^B4)&fG ztn$WQZJExb>0cX0zB`(YL?lHDd)-eR?2#e`xe8M3;-NFN9A8=PEN%o=8QmGf0ftP= zmNKO>RPxC*NYngrGQ`nl0F6j)^M1o4`mmP2mQ#6mbYp*xZ)BtTMX1|irE%d0o!=&R zcm>`%$=Qa~o5(S#`@3a@=N^J>BL_!6us|LehA~`_=_t^q9d28?{EkYG^{&z1t8u&T z?Xiss56_%RY|C?WqSfi7t+RMu!0>wcKv$SKN9JP^Y!IpSYW{u?IiU7sC)U|%c16R; zNu(prU?FC|n9ovP${oc$Z#ho77e?1tCtum(ADsq6#j=NP~kz zUBZkYK?}ER!oyV`$7Ok>W9a}|@0?Y&$iPS%C+TVd&n8+k=l&V&Dz(d+pTTs^o(&YwkaMa}mv zD@N3v)jY zh7AJzdJiHjD=T{u9lS48O!O&OjG(3%WSu!G^I?7U!$0P~v#zfu!18j#Bn6W9el++Z z?s~t578&wOHQ_@5@~nd(Zj2_PiL2l}TYpt?&e`1U2COoo#k1*KA^Hff2ifv4cXNpp znBRuB0$)xcY67{Or9KP*|9mH&v$W5^Dg`JF)dPW)fE&-5!0Hb~+&PSuwHIPs&gy-I zO;iaAgloZCX(+maTd>1~J=ITaU4&V6iA=Fy z9(2mFi0THD8tDC~a1(g5ZE;uEOQhZt3CbhxOAN;s)_O8^k zGQbTXd=|)hpfe%Quq^8vZJ8#fr=upL&k~=H)h&q6Y(Ry4Uwu4QLZ^Xymn&o&)n3zS z5Z~9k7Rxnv!ZEXwy0=p-R&>)vx{2*u!LI9*6lfdbY;gLol>N{fKD-n`S(@l{S#O-W z+>{_?-^J+VUg$a!lU6>-Zd*G;(5OlV0PW5q3LW0e zkB>|78m@W#zYF;Vbn;J%K`j)xuC>|hArySttGIc#jn0dP)wc5R6zg6w%|?)I2;q!)aF6xLe+W%Fc?FUY}7Stu21BB>Ry4D=TLJ zDFC66Rf*gXrfOXdICG-9qiVx!<2^G@B^Zrfng6*}683J&rpvgAg)&oMk_H{`yw6PM z#PF&8#Ws!vE2G~YO{;F!At&^N|E66>#8$hivMDmVp}+tU1G>P;@93f0gN9yBEwTQz((v?_2@QX(!KR-H_c`wV5DHz4k)ZJ%-pX%E4uExv2Nd1yO z7*y04my#Kr{-gR6l$G4YBfF_De^B2}OiXNNo;~TDmmc&R*ds{QR=fLhNhkjcpmW3a z78sg+eUBSv<);OYRI;Uxf36Bf#l)}e5~ke{A`+oXC0-RWTIjY5bYK%&juP>Vu7Fh? zcrZYAh8W|G2-fVhrP!KkcXI%Ax_>0W>T1!{G`hmtfvH0qcPr?1?XT#sRI1a_hqS_v ziH&T-4oLy#S-`Mxj~Hd^eRoNw=0935<_NQ(g#pMWFn?~yI1id+-6A0D$qLd2IW0Z% z-v`NVDdauaN^>5jx}%d>hlC9N8cM1s_YVd)UJ z`kSguz=qT*LU*sPcG(pOmRCxvqlZ@F0b6$Ld}H1J`D>}hq2qsog0JVoKvMhwk z+www5iTB0=R-2Yg@RhISgcbFCj0?OVxxr4Y|261g4?DAyx-WzZO%J+Z+$q8P0#Foa ze{7mymPew0Q+jz0atkG-oG3(de>$VG-Gq-G;KxNYo`v^#R&%wS3{m@dQ{PDxI(eq~ zN3&+yt>GJF5LjmVdL6*+ngA1C{XQ_H;1nEraoZDQiWM2H14QEA4d8<6o&!0WXgz^c3R6L2=N}o% z^)(-emiPTuu8Pm*j8FK~Dc)g224gNFQ^_>+$JpWEkssS%KhI@R;Q})ljv_6p}W@ct@QSoZAz5H^r{I=rx7e#+Qy`$)C zGX~Wr$F;N9tf4|I#P}4$Tn!p$caGbzf|Qof5WdeEMB{z#JRb~H!p!EMW2o0j0{}>> z=DwWYk4pY#-b8VM?IbR@mMwUrYVhG5@o0{_#A@+o-${?v3=D_v5KAbewBwa`WL$mZ z1i@oK%BaXm@a%1q*I%v@)c@WPt^8g1$2?-)L-q;yMihu=2?L@^Fo3DtPFM zyyuZq_3qnSL*jG{&?EN{LmqbPez5J!#+~KIl_RAK1>KDJt0!N5#L*Jin4BS?ZD}R03r?p+AgtKwbmpbGnDv=MTrIKXYN&+f+Ij%$;2(4t0ni za$9=nld5r`Y%-F@!sY?CRg9l4WqaHil6MPi|W17 zZ5{>tri1qyr1Lb#mC{Gz_LwSJ>S6I}s|6!fbgz@ul&z&`6O?$v51ds>md66Bg(@EK zhz5S)T1fPQJR=ffLlq~jB^Tso-axw<7tDwOUC=-l!4z|&S*(<3$TG7ztDlhw6%6| z%p8lJV~}a81x;x!nmR9q6vj9H0f|qS*a}#GGBjM(nd9#XZok&+o6rp4G-8cH5RJS) zn$F}{`I6x6+|2mva^V#ZF*#KxGyE9iVJddG`6OVS1HkB5~+uXz+d-;M=f$|GnB(a_HR!(AI98Gxpj>bP* zDf$z9&+2+JxN+pe`0CjI_C`r2l4gS5O61$0x8*FBN5v*@A%997_vTEoq_9P;8P4<(c8P(O9Vp*xdrB zSQTZi;}|@(d&;8sfX3|lb5q^#xJg00zyOVHvYFWyPKbxwdm%I@%SUa2GFM%8KS?;@ z6q}H*a*-)~?SEMYlrYFh@4XQV;29)l`MoL)?M#|W_ynMIGQkVwQ7pSaW2y4uhzoa8 z!*`2A#mrs7ZW%VJ&wtBKj&w=jCla_f?I;(er&absHdowP^oV;;25421xebY=PEx=4 zYi4RS554G9wqV}$a+g*0rTw%W`j2UT3$`{i_F39&+S!cw4_8?#H=ywa|n`g)I=`uZUFYH&WIX+6U|4ESsSbT;^0a`o%sM|XVymdGZ4 zxk9Ns@vVMyl_abmOk)u1xI&M6hXgnG>AgaoJ;_G3B;;Q9U~d};40OupJ=pY~CSKzi zikbF|jMocLHvrpCeTDU33j?9wnQGCw_2T3usHuPw7y{I z#n5i%U0NiaWCJNvkoDg_^h<|&3}um|In{=*QAU7Tu_O$ZN{VPo@UOpJ<)!O0q*LGt z`Gq=u8{@F!ouph-&DX2dDcKdJa15s23a}$(%)|NlnFwLNbOZtz%VoI)sqA=++Ny=w z__zKp1lhJc=5=b(qglH(ihoj2ZXLoVD3tpg7H79KV*$&KrW`A+Guh7_0{2;AwTJXI$ zw%uG#^6ZZ`Hax@Lko9ehtM~WZNhYjWWO&b-u$T}P`wMgl?nIu?IofFB;d4eaYLoA} zoy#<7-7oA03WZ2!-Omm#z{!Uq@|`-r2;*#NbjV}4mX=og)a=#YmlJ>X8k)}=HrY#R zcS_Ft{!d1r26;zx@=q;t_obg-ea&k^DAMM$+g)+&&ER>KB9R8eC6s^(_1}+hh}#oE z4g>NVFOY7LYA0rmWYz4TJu!+SHy+4J41Px1&!f)e4+6E_K_frl;QyP{$2eMUs;Fat z!&`%?kPj^kX*8z*T*(891AL{LF}7%_CZyYcHC zTdp9;nkX%F99nNX$Dd$s;~wYcpCQ^UrhjX~Y9LdZe^{gF-0OA5O$|kbCQFZ2B8h1h z?k}>05amq4(m*fiR1^4&vnguTNnCumCnV*JYr_#OG2C}wx&ztqG+P+r+^2d5;TW!I zdWHM#Q>T>{p`}3xd6sZo_DW zD=jk>uL>`MEUi^nctyCjlW8*;ae`kW2dECjee}3Y;^-q7ps#>!oOW&n7T^puMlC2T z)z5uekMU8-OAelI8A+dd>;EMPnK6YDSj}C=EzN_ovF0GtM~{k139&W*f`q6Q|G*XR zo7bV|jkAMt%>EanHCG2Tt++Hj^_}Rolo*Hh2NMAJ2+8NlO9%MoXjl&SgT_bdbL&dh zqV80PcG|34cp{Etb7U;+)CtUMWp4=8B*Dqhcdx3~>fLS*JiVGvoZ>Eiz`Y+L)zSk! zdMePoV7DIkRO=xi!!eJl)0qW^5HiCuTI+uS$1lBb(%Zx?NT3lMo`%x6)1q@*;EpS0zKyX-Uqu=SB~+uI%zy8sw$8&z(U9mX!JG3yc|YN z^iA6yD6H&!`+Q{(K(L?{k$ggreGy+m!akcm^zgW|z8eeltFaqR=HPs>g7D_O?@gF^ zYh_4+a?e}YFIwjp*{^-KjSl)86ueG;6U0IDvINUq8mk;33 zXa*q#r)Rq_#~LEo&vM!C8k?BpROvMDO#lyb))kl&gWki@{h)Z)O13cbBoThYgkOA% zMSGw(^^9#-XaikAJ#B{hL|0YN9^Iqv5K$;CVk}OEfK)JYSPb@hW7(wtA&S^Ecg9I%>H=#KrS<>ff8*j zf>}*wBD$*D_0m|7kDkT7+!jW_4R_kBu+z$;x?6ocf}9gz&I5gl_6s2Ukho^5hAP!! z{n&-He!SOC%VO)z#EfnlgYs!$EF3BA~?tE-P z{Ut88>OY$=A_|OQ0ze#ovCX{40&@_xxOPz`OVi-oF_~ zOQ{4FmwxF4($s=GaOCuY-7iW*0HBW2neY`q!IT9FCi*DOnQ?ufF*2jjdYn+&x5J3| zz6>*!|3`@V52m=-4K88fsHM0i`AcpvOV$K2!a!pvlz#r|-N=+TpiGe@PT_3ri2YFJ z^@-J0Z!lBP&T5eRiRAUlEG0qGX(j(pu~e@MpJi(&NO%95t}66q0Epk*@SOU4$H^=+ z`3q#4?NN~9`S@ONpFAcnnRY3Fg%J}S4K4!}7|(43>48qCHi3O>mU{+T@sHbX z<-V6j)ZH5L+EI6RfZs!xLK?fP7*`PK()u*MIAL>UGg$A%D*RSFrT^MaE1sgl&u*y-GEiugX8wry*!3S$JDIlyPI9Va6}1FQb!rhrx`guwfyS@&iz06EjRu#3Gk#x;o3V8L}Fo1OAHHY=pSDKZkSU{_DVi8KBWimGOH}63%aQ zVc3AlCpV$jrUv#{iXqjVs71ur9>V*3B+?>{U_@HKw@OR>ojPK+@8Zovv>+@8P?}_e zg=EXZnRG)YM}ds^Av>HS6BRDTj_J0HfxrrH6mbJUOv5>lFx0OK&Walo0;>qdMlq^O zXn&`OJ3X6@tOpHS<`+1b2mmMHu=MiR#ewQHp2YXwHtffLPkHx1Zf_&!S1sJ08I;&fH5E%UI7l>!Dt8Ek4~M%i#BDxSgkpdq(GNst^Z62b^5@q?d`XYS35q< z{&k+q`xa+A1jEz(;MIhpY{0)y?Qvf(<$Tkh|NVK9p!H6+m{h3DpeN3|F;seo^o?JD zGdwq9XLXuYV_WG@Tx5u5@4XMI*!P=wq7Gy|y|;>Bf(gdTMu&%{xBK}S!IMOksjkVq z&xOGoCiq9)aWKydA<422IV~_@R1rw5#)7-WJ`aot7YDX#C|}gkNCGP1L)thp4Amly zR%haO1adFyK9c(^V@z8qTds) zd~C}I*!Q&8%CZ@tIdQUVb+!D`Nm^cB{$gM8sw;9;e%J3Ia?m^a^5g<^63TuOI#_P= zY~(+?c^Q;sb5lchpt*8O_itJk9_dV?a5C4z1JhX{s2(4>ZO*)kA4G1fU(OJSC5FKn zGxa$hFmPAkKBMK`H;$hSbzlF$hZpc6rA*0*kD}%0?Z@Hhs^|2u_EV6FIEwB=(*4C0 zA3!?fGkAWkZ6-ik+uPuotz7U=g@E|s6p&XL4G*IMlhh4R&_GDI#nC;ksH$uo0ot8c zc3=*=0JkIUp)y#T(xR}Yf{KBZ!V~Aojau=V!$aei-k#lzyX4e#25qs{=}ef~WPJbYBJSynWo38-gX2AO<8@7a%*9 z)E$Z^C3UmWW%*c&;oC;KC5)EqgckO-XiS?!D@>!j5Z3@ATRQ$yGN?)^m0icR{sOjs zT&F9LgS!$;)YtJcox&s1+Fk(2H{^ilFI4+hz{be`^!Ma}O&OATL3iq|-~Dyn7lNx} zpZ|&vYSh{5&}68`WT<5-c%^_+tq0r|EPac54USCyWXTm81N+pelGgj;0PvtXbRxF7 zBC0#r+F6dTwKPlA(u_0^`G$OlGlFI7VLw>h^80dPt6b!Nig#IT%k1@eaPZ#e(sF`l z?e10Qvy0BXR_dS%1BI*e;OpIX#dC-2tzyaG4a1zf1YLOTZEn5nfUPIT%v6coO=T|+ z1OhzmrJG-U82>Oz_+7}6kpPxAUwm}?(|k2{&C4(ekCVIa8~7(Nrvx_DeC*$BGJ8F* zcoErt958rnE?$sxg3DptA?O3hHjaY+tgF9^hUqE3Z|sU~F@%K(pWU>Y49)k> z!OiXMm(o7OzZqAatK)92`c&)jM2_QrH6(ugHQX>zAT7Epl05a!oe}b38r!yn& zYQn#R84>o9h>Ai_$T$nhG6+W-n%~m9!wX*!6chw`z-#M=R^;JzK{emcW+YKuf zTHf?u96B(zdk>u1`WtQCH+<0gj{eSn43JR;X$7poWDb;)H%!RWS~L5;+zSK-{%t9$ z-%R|}Ha_}sNS}re7aGzkE&_gt!9G}0&;?3O{59T=qXpZ)hk5V6@8hbGIEK7&CzE+~ zX+tL*+0*HckM9A>KVtrAF)&H<=?;O$XnbNp7h49FnUmCH50l7!Ql&N;C;a616qPjc zwd)HOllbZfq8$VBqF~F3xI==JE!F_;W@D+ zeejJ2)7LLOCar@1mLmSAIdplrmf-Js2j3e8f+Vgka;^`KhK7V<*iTwWBjSP%^Yzm8 zcFz8pKonne>3NI@C7EdbgnpO1J{=5>l8v0#yTB=meA^A&ZE(&AK8(Bixz{**-e}S& zA%o+9P?*SG*q+7ud|Px*VUs=nTiYDyyUb^GI)2O@q@=xdi=d?@Y zX0KBbxY+6iyRKXw2T?1lX|Eu@rT0E|aWD_V4({=U@_RXxAZW?d+HSS3+hu%!Ra7k$ zdzVAm2)1n1?#Cumb<(i3k+XGLUBK%eoRnPH4N@A97%BS{h>J_095;Zi($@rzZ*-mj ziCT*%9KjZ_i5VKo)-HMhVchz$D7ZxgnuA_l`5~+-uh=Xy4DUN+g4j^8 zCy0>DoVdwLCvD$ahxEZ(gDTm{m<)_ShBKif!`#XdIOYxC@u;_ZwxmpB2``Rw`Ukko zZzI7|JDQ2mdmmNiE%9uA60wZnAk2jHQe7tav{o!E_MW1{RU-5{w^LgE;^cBWafF*B z=7-lSDLo@$QW4k)5=`}4-P;6F^`%!ALs~5Xr1|jW;I>FO5dIc93Hsw^xnsg|q^YoJ zk`CHexi)Ln#Tz(){B^0<1!E_JB#&v_5BpUS&*!_L!XFIZi6BCaLZ7TYeC-fQ4umrmK%)9KrLv`BN{{$;qHCW%tX#`B+k0R}%7~IqMU} zRW|;g#5J*T&$1b_#r?kmfIZ@P)`dI!olwC z$BWu1sxOxjp@ULQg6-mOj)(#ks^OY4TrI907RmsKZYn91b0(g~N>H5xTbYk1_hKL& zBen^RZXMtGO~Lr^6Kq!6L<#{4j;#5TMo5>8en;zVJq}1TPAuGr;NVf2_=-pJ@fRs~5i1NY|)+|LqC!xW^;KoO{p9 zQP1YZum3ymo5aoIS0l}5COOYb{<+qIhg_cdwlilRK5qU$C8;_CAt{FCclBlhyD}i& zsKDTHsMTbGmTk9r-Cm(9j&CHykVN`#bmJkYfWe8CLfRsRsI5mCK*+F{Bs|pYQ-4Hv^lSe7F&KIItu#wPy0pc{^(&17Gs%;R8tBw0xkAZ)aZ`tiu3R zpaO!@xsOD8;O?i?IA|55Qn23P{;c&;q)DneQKq%Df~L)l%DX^$T3wL*nDfH(PTwn6 zT0^#J3jtfVv=LD`4GaW3jf<%mFK*gI?8 zj9a>`lvE&>+`&RTsl#QTLoMSKu6UQm11Bt+ZN(sYFdPK{tU z#S!|nJI7hz))Wj!$%pF2P14VZ+wy?PKb zyrZVQzv=IFs~}qqD=uP7s-b$}vltKL;c~twrb$%C2<{wN@r~zA49k#?i&qr2MQJrx zZ1y9LI0>O%qzV6DU0)s!W&8FYjSLkt_T6l@$QlM&XRKL5Nl1)6OJgZJVKBzNlc8uN zB@r^RB*s!%3RyxJ`!2FmKfUkseV^z1{k`w~-+f%yaUIujU-#$yoacF+OVn1gF@lc< zxM7$&$)Tca0RBapQHG+z53rt!St)UqiT0|r=Y{}jl4_9=O1^I(z28!iqyWfj;@I8u zoG>?#&g*F7b~j^&CI+AL(Dp{g$1R-YeeQS;K3+3tJ0BTi#w=cyef*W*?TS(p42IdP zf@8&JU#?4KZa@m2m#~~O7|xsq^#W5&vPeE&6lzf_S#{Ib zI3>ORjbnNd+~NiAl*qI%Q=mu@2qjGAvr_BiXsROYbzws4oRA+l0I1Cg!%EB@eG3~@ozSM!jZB!2;?-U1 z2AMJ)wAV|ODh(PD61)wJKYqL2DYv7?r$>P;75o$>2kcN@G8z3BQ71KbaF2cb)sdM9$f|{|HgXjM-WGoNI0FT?a2d$L@)ypw zKnPe7iP@phMIJp_AMr%Tk4dt1E<$1~JXnn|wY*X0O6EE*FT5X-VPq}4 zEPXVFH7?opbr#!sEr9*0eaR`vOqhiAk8h+~ zdA{*N4}#RGgd-Qb{w0gH~Z9EAZ>DW`?^mDH%$L#AuD8 z`-B{G>Nb|1*y>l90G*BA(pk@hWo*kTT7o`zX%aGB-j~@4C9z!Cbd&5k-#dcFh-8$m z+IvJ=7d!~E2-*30?Y3L3a#QM_IZ;6zYh{$zz*>zrzLSkj0}WH?&hB0rjeOZl_{y?4 z`NYJFrRF**AnqcsVfJ3e)L#{Vy?6j>Lu*JyqsOWziERG`$AE93g`&>>GfwNz;2S5^ zo5MO*^d1B+jS6~^^HB|L3(h%r_Jfqi_`_OTzxLyI&Tr=M7ov0n83op zH^wc*Q+zj!gPhR{neNqQdyt5!0x;A+=R>21N8o_AHn3uR4qMi&#?^#vq}fD@m`j@n zfS;Z-hWz}q2ZhB=9Vc+?lMXsNxmY~Df}jy|(<+UHy^IAu7)UnrCVS1jw(XJE6tX^@ z-8mo)K})njHOnn~lT;ot^@u^Ok6E3|(E)BrNpE0P;5TdemRAz9+pPRC+2UwP2P<>T z4N?jVcUNhT^ZPg%xFVB89D$|O4O}G^PA#(G|!hi*z7gO zSa>LF{7L|t51j5o0M6ZnH?Oq61&CwfT3(!W@a}MIR5ho(5bCu@EB5Nxl{PWauigFs z<>;T(L@(e<=;u-P2w}z<#a%L=WUL|lsSz134uPFqvb?V2VOh-%lo#;$a<~W9ggXvJ zIKrR6e>^}vfMb-49*jGuT%V~frZ~D!*jSI|2CvvQF#_JQ&m`nQaoMcV%dK8gBCqAn z9lp24zvO}FNp7xXo(*)$PAeTK8ito6R^eLBrTCK;#`CeUuk*Srm;6ACDGNSAxX-N2 zcqG%}#p}%QMK+8avoXWVSEQIW<2#rx+|d<=sc*wVmN6Q)ivBb6kCk zI7WDi!cJNZzoZrdPPL)pxX#aBJH|h7AUU!VIQ|33IOP6Szm4~eDO~r)KiF4>&$>6iP?S2))N0zsMV1J-l0hoOA zE+z+Qr<%174j(8KE0lN=bVEwJWNs!Gq%!rve|fb|j^49vGBA=q!hn&Lo^UZdPJ8X9 ziS$7#2i$9K#6nr*1vBhs{Ov{_x_et5GY|LT1Pk(CR#qEaU@LT%En+6_`k0yy{Z)|r zAKd4KpuNRbm%6Z9c~=Fnl)VG8bL8jN4gz08kn5Ot;)y+mF!HZ?{cD{&HPCNz&>RYW zzhQpx`Emq=(}}q>(?H8nqX+ywdgVdO@g(2-(``+Bl{Al3IXNR=+Jom3qekr0S`j!) ziJgpG=EBT1&1b0|sNo#^Uc?TI)SD{bv$TBUPpEU%8Vgb-y*^hE&r9*5b7)5;fg&Sf zj$SYWFCa2i9QU}An=>ZI2&_W^^_-^SR_`W8hxnEx%wG0M#-ozrwGS9MKg^P0cKU3Z zAIGB>S|3V37Tmt-%+jx;z^HI0VS9eE%MJDc&46x#ui2=eSpbh9FbgjRYJYU&Kp?|g zX&4SvL=Jt}>>xg&@M@LklW4b+{c4@)obS39PSCh8AR0DS%_A%#TfQxGrJS)Bv_VE)X1q=(;?px*aa=V7JN++T9 z|5Q=_cq4RRpMPry!N06KOxy=T_Bgkm|4dIYX`xh-g!Q%{&SfQUaH}T0ZPCp&Q&*=^ zb*6Nh<0Z_rxLcS(^OIs1T6eUR*tM>S--Eg#I!mo!xYU~CeHI|-wzlrq5H2Ob5ZxUH zpbcO&u-mDFy-mRdj@&D|yQ@&5xnzK*aD`5M9M0@X-ydGO{!ULOLgOQyB3&&&hDK8) z;qme>Hu|CW(R(ap#&3;8S>Efrv?Gcx(+^{*ui-a3udQiU)`->WwddodEAtf?i3NP` zu2BJ+5%|_0g|@Mk_%oPmiqYAa8K8duo!AXlzmOX@C|v%AHWD=L4S0caeHs`e)`l;B z?=>hDxAEX-eMb!CI|CanZvbDKopMok>d9%+~(v$-@N~I$p&#D*dd_sJN!*YW4LNK+1xY8D>=UO&Hz`!1|J46!#C90CEtlS^p-4eP0{y4U-v~z@|qUQ{Pm~R-g4mI2i2$=7AtB#yi z`Aa@a{x;7px|UpU0?;TryN0LQt4v~<$tCE>pAnW5xynM6GzYK^8Qx{;_9ZbT(1Bo; zdet%S(~~+2n5?7mn=&B?M-dW} z2V5xP0{0vNukp#&=bAq!UK=|&n)&{tW!(TRu&k=6{rQ8HlfVJAC~g^pQ;)V%x{Vk8 z>n8CZGO_zH3t`W?AIH#ZxxTP`mjPe5PuAx03|CPb!JX;>k<17!<6tg4JtL+d z;cCRu(%DKKsy~_ACq^%la+^_Kl~b2?Gu{JUFEY&nDHvvV@~u^pe4}L{0tY~bz8M== zSLdI^nm>Q#yLcEvQf+z3+d*VjOPx#*1ku>zbC#oZnt}T>Ygfq zDv<)fQbma6VVNEpzL|^f#KM(@z?#Y*EUrap_R0OK1%k@Ua%5~{9g_tn2>cRwL28ZJUBp6F=kmT>lHj&2=}XtvWujAPfspr|ZocYv z;zSzD>`4Qb7fL-%;8Y15_NQ?0rceu`H%jv>ltY`1$GuZ67nMMr^6}1}mYt@JxtiI3 z8t%T0DOgsY&xIR-XS-J4zu7GbERO+Z1(>7YU)ZPE=lxlR01F;Bslpht5`@9nOc4apTf~qlN|(N!qC!0R=uy?qfWVQcs8*m zKLM>f%v>JQIm6|B5f!7PZ!>-Lf-JXTwG01@W1x+fgbOP4#<~_7w-0rMRx3GB^1)wl z?v%F)PK~$WY&jELW?`fltRdC~@9Wb{(F7|dQlBa2PFyCkf7>BUiC!d3;IO{fqo7db z!|HnVIk|#+O-Q~Z&WY)ek+Z^w4mOLb(sF0q^b_7q@Fzz;rX)3Xec0CoUbVSS@S5L=HBi+n_$)-x|$Prcl)=BvKYGkz7RlRck+^TJ=)1%Vi2VC$@dgx-O=v13(eO-8}LKA)^SwDX=GKYw$txDFv zceUC4+=3LeS(R$~m9;cWa!P>qEEq*NUrVDH!6X8dG)09n&DgH8?)o}|?2{Cx4S~B~ zi?&e!+8HHP6Xz0rMyRu%2pBG#UzP@C!J1&L&&?b3Nt4}9R0uva&Q~&TGKvD9v)(rX zPCZYFlqD5V4u%?g;bB8L*+4tul);A(psGD%62=0*ExXJv<7+=J@!c{T4Mw`hVjy`oK2Jwe$&AlG~W{7m$zz?w zJg@f)y@R@n@@J=_$JlMhjSXYDS!smrKq#QlZK{EEQ?Lk¬->N1#u@Sv=o|5L`nA}ocC`E7ujL@?%S*ZqPFo}!Kg{&e^1LH>(&tRoqAvR_v#!qA7J zIP0a)poe9no3Isp_uj^EQYRYCC8J{l-2e&ceAMr(AL@9YdP)QG<1{o_YvnU+#EL`= zNfxy~28b1z20fdpB0HN6Imm6_=-O6uQy9F|cSANT&GvPewnvH93mlJ}961pd5D`WL*QnEKAz-+e+PRcsO@Bg%4Kp2grn zia{j$`PP&gg`@JCpjION#2epJ9=NU-az%}Iytv)?;n!h+AKK@8}W6Y_`KD={gsR$%Gu>U#23CO*;vw0EItH*0l%?GUDmYGphJevSKkKue| znd&|BOQ^eJ*-4sn9g9D^uwL}G$6PNM&QcULs&WWlD>GUu*I(F$!vOL0& z)^#p?oC%qny9vVhOwcynoX!t5p_m!opD%4o+d|V&jbeE6ebX#2?Kp{JVj5rf@^Rys znX(SYO1NJqDIWARn@z15;$}-A-i`~<;s_C{ZODXCCr&xUY2^xv2Vi( z=Mt@0x`~r(uw$s9hq2mJxlm-O=WV8@Zwfah9Dh)<1ii-V3_d+o{!OwyJ!U2lq3qqV zVwC)>Gx%Su^%tHpAy-Wug)02}?rEQz8kj}vg6O1gNAoe6B1mSt$y1WR#cSB>FTST@ zz!>pa&FA{$IoH(qYM;BJKTEL}HsdJFw0vnn{JI@jnwiVgql@yQNd;_P`S`_<)aASB zJ1lp^>9m#Oy=2BX=S$bY-3~hrdTQK`US@7c?*}%s$}J4Eo@x$vV?(T41)s=$YK1f{ zd4}{ZtjE^V*$++$p_P_(%fZ27TKdAdp1P0gMmllcve+ER_8i3T8M9UcBm^==V$h0d zc_q~oC6?dLs*NSMg1{e;K%~s+7>bY<{SAt(4TvJ}14tvR;~$XUzt2Cf3rK!hLfUd{ z@(1T5g)m*YP7KpmKW8a`8iOSPj+MG8API4pAX0MGAMQN%Y1=h`{Vy7feFz9=qP@Q6u;D#=eD}p0m4nY`-8s` ztM~iug=hK{H|U{FcTujowcI2XPUa%MO6sLY>vPAjC|lLgvkXbD8jPJTdY6BH$UbVc z3?Z}v-WSY`o{ej~WjHYl-_R>Z#ym?~Q#=H-s6~UspNlzBg=YfAcMSre!XQ7OEb$Dm zf=YZxVv6+x*C3GEw(7-M6s)~oP30Y}DA^(#dK|vYK~CNR_6CIYtA?H{O)W}_*)!Kf zY&~Efv+*CzBOL*~y&i^ztdbCa63|Hn$ zNWp0g4k zjMrpZ<8p^T)eD?@Rv`%^D6!uW3TjN+OCgBVK?T^$)4J?m?Z??2t#L;|t!9GSZpVVs z<QtfbebXxd z4twph_mVXA*b9$Rg4jgh@{SI~!xUvf^%9FVW2=F;9IDKMVPToDhf#B|VQ!WAKyl>7 zHz;K4=yApOj)C&wc&-gK5>dlDcUP}+;Ow}Tv{gBCB(5druJ1S^e2-GQ|4felqXb4D zzu{m3s{OD%eXqw@{$iLuF&4~{6*n#8t1khkgJ+M(dM#IjpOu`931qEp zc`;d}F~2)itykES@+&}Fr_uy-3X0+Wbmj*BCu1HR0=T+#wK+rPhft_^WW@W9^hL%AA6VG zT{iVZa8=&t&=!dS=?7?#8c&Mp$OJ>*0+(Lxnya2`lU=74U7dK(#djLM4c#@B2s3() z^Cq*A+2>9Ql$pu*XgyXhu20Y}xm2*HR8Zd6sJ(pqo^Xoq%LoQI-+A*3(O(S)u@h{? z!%v6k>vn{9#ToLb$S(xi`rC zYCTT#0^YlL$a1;aEy2#n?G@2iUC8MJ5B+xSU{~n2pG^Ju!(T(6x>Ni2BUrs*+X_Bf zh33?r>RPiLGnCLgvs}4qj8j1o^b(dWw70P!(v(XPsWl0ML*#{DR5naXK<9-T>8Fl39`WLGAHMVc* zx^goWUltbuUm=O`&u=wO8nV^JJt#I5E&bBU`Pyq5*>gj z6@5kf$MtPBAe2dtzOb?)AJ3!vepbTMwK$iU_~zY22M_4}qVyeX3XNY--DG-H!!jy6YOc!vw$+6S%IXzv^#Em_(J`0ALl=1qdD!VS*OEw6m- z_Wh9fqKihg?QZeYhH>&kO!rX{;xS)$&6zhHW3uVK?MHlYgwzoDTF=%rxbB?o1~evM zC#0OC%e$}O=?{`ro!5np<(prq`=W7D>W)%mV~;Lt>b3idBU;{<^;5sRMS3Q0g|S^X zqRZ^=@GJXJpm*!wAsbIo{=EGUF7KDR|3K&;Qy%)qlm~{D*s23SJ5zf={qM$%1S_Ua zQ%lPgghLw$SQiF^c#>oR-HM~J{sR_w^AAv zpCnt`S{K=%qU)L?vf6oB3fArE%!NmsE52a@3(>ooYRvKyGXFpks%(%ux9WuEa=3*k zp9MX&{oZs z>Tu<|_^9s2c!~ZFJDdwl>KOU`y&?a2YfTU(ZJO}4L24_08%Adu)$Ho9+^~jxHSGst z*o|a^`+`;R41|i4n5)yG1&ogZ!=7@q9>NO*&7vIOq}z=Ipu{%&L5Bb zW2@FnrwSijG>whd%4hk3&34sy*IAxk!!~OlXO;%IwSTxLQreh$N=L?QTZ4=QvH_TP z3F)9>RI>cdszbcInRPjp<+b9SD%UR!@A5AZfl^c`{Mes*l(V~3d~W`vTjLKFgq_1N zq4=$C2JWo!IbW^ST0h!{^KyH8`x$;^)0ahldk0H@+iw|8+_@&f)SHpM!+w%r-^(&~ zJa^|HMN;uc#0+;}eho?GDu&!OY;pG`;_>ulDKnzv+U??JOXHTc0aK;gRiWc&(UjEV z{fyU_Xcun(howjdyvBBuW+C4v6dcgFH#BlM9?nbQg=!l!oUz&{pxm5P)e20xlv0v% zwe3c+tZ0`3*YLSg1VOI`3pzy#!iu=IE{z^(hp9`1=d4OJ33)=>WZ+B-5|QMCr{T}v zR}%9j`xF!q$JgiT30EulWgF6AuRf8UO2BAWN+R2g326rIP(wVYjJg5X5~o(JQWcGZ z_t!c9^2-S5S4KJ>t$%vtau}Z+bt&0P0`&mOOO73PQpYr7-qaw^ax+i$S|vUaVDRzk z_K&+lh@OTd(G5cec}Bp8`dKgnWK3SD~LzwZ>sq1#lBj?-Xc~-9SAor zExCi}8{Q;kcy$ZXx%sD^Ve=L_bYV8`PIp(ESaV_ak6diSzFk2G?Ab^ro3En{&Z`rO zO43ieI$X)am{}0IoTY%MBqspT$myl03$n6xbd> z=tQSnL5|$sW%09-VNmK}9zg_G^J^wFPjQb5P|qZsNc1a%I;XyMZ&hZ_OFH?RODn8K z$mQh3BD1LtZ^iRH9ytwnRjKaBwnL6z{D<%DRK!uR+V7(E{{Z&iT9Z7=?%*x9n`3;I zsD|1{X3!kYx>6T*~HI1jCIMzr%oPyZDVw~mR;ok1I+)3 AasU7T literal 0 HcmV?d00001 diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/intro.rst b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/intro.rst new file mode 100644 index 0000000000..f1709a99e4 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/intro.rst @@ -0,0 +1,48 @@ +Speaker Recognition (SR) +======================== + +Speaker Recognition (SR) is a broad research area which solves two major tasks: speaker identification (who is speaking?) and speaker verification (is the speaker who they claim to be?). +We focus on far-field, text-independent speaker recognition when the identity of the speaker is based on how the speech is spoken, not necessarily in what is being said. +Typically such SR systems operate on unconstrained speech utterances, which are converted into vectors of fixed length, called speaker embeddings. +Speaker embeddings can also be used in automatic speech recognition (ASR) and speech synthesis. + +As the goal of most speaker recognition systems is to get good speaker level embeddings that could help distinguish from other speakers, we shall first train these embeddings in end-to-end manner optimizing the QuatzNet based encoder model on cross-entropy loss. +We modify the decoder to get these fixed size embeddings irrespective of the length of the input audio and employ a mean and variance based statistics pooling method to grab these embeddings. + +In Speaker Identification we typically train on a larger training set with cross-entrophy loss and finetune later on your preferred set of labels where one would want to classify only known set of speakers. +And in Speaker verification we train with Angular Softmax loss and compare embedings extracted from one audio file coming from a single speaker with +embeddings extracted from another file of same or another speaker by employing backend scoring techniques like cosine similarity. + + +The full documentation tree is as follows: + +.. toctree:: + :maxdepth: 8 + + models + configs + datasets + results + +Resource and Documentation Guide +-------------------------------- + +Hands-on speaker recognition tutorial notebooks can be found under +`the speaker recognition tutorials folder `_. This and most other tutorials can be run on Google Colab by specifying the link to the notebooks' GitHub pages on Colab. + +If you are looking for information about a particular SpeakerNet model, or would like to find out more about the model +architectures available in the ``nemo_asr`` collection, check out the :doc:`Models <./models>` page. + +Documentation on dataset preprocessing can be found on the :doc:`Datasets <./datasets>` page. +NeMo includes preprocessing and other scripts for speaker_recognition in folder, and this page contains instructions on running +those scripts. It also includes guidance for creating your own NeMo-compatible dataset, if you have your own data. + +Information about how to load model checkpoints (either local files or pretrained ones from NGC), perform inference, as well as a list +of the checkpoints available on NGC are located on the :doc:`Checkpoints <./results>` page. + +Documentation for configuration files specific to the ``nemo_asr`` models can be found on the +:doc:`Configuration Files <./configs>` page. + + +For a clear step-by-step tutorial we advice you to refer tutorials found in `folder `_. + diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/models.rst b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/models.rst new file mode 100644 index 0000000000..ebfc5ba42c --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/models.rst @@ -0,0 +1,52 @@ +Models +====== + +Examples of config files for the following quartznet model can be found in the ``/examples/speaker_recogniton/conf`` directory. + +For more information about the config files and how they should be structured, see the :doc:`./configs` page. + +Pretrained checkpoints for all of these models, as well as instructions on how to load them, can be found on the :doc:`./results` page. +You can use the available checkpoints for immediate inference, or fine-tune them on your own datasets. +The Checkpoints page also contains benchmark results for the available speaker recognition models. + +.. _SpeakerNet_model: + +SpeakerNet +----------- + +The model is based on the QuartzNet ASR architecture :cite:`sr-models-koluguri2020speakernet` +comprising of an encoder and decoder structure. We use the encoder of the QuartzNet model as a top-level feature extractor, and feed the output to the statistics pooling layer, where +we compute the mean and variance across channel dimensions to capture the time-independent utterance-level speaker features. + +The QuartzNet encoder used for speaker embeddings shown in figure below has the following structure: a QuartzNet BxR +model has B blocks, each with R sub-blocks. Each sub-block applies the following operations: a 1D convolution, batch norm, ReLU, and dropout. All sub-blocks in a block have the same number of output channels. These blocks are connected with residual connections. We use QuartzNet with 3 blocks, 2 sub-blocks, and 512 channels, as the Encoder for Speaker Embeddings. All conv layers have stride 1 and dilation 1. + + + .. image:: images/ICASPP_SpeakerNet.png + :align: center + :alt: speakernet model + :scale: 40% + +Top level acoustic Features, obtained from the output of +encoder are used to compute intermediate features that are +then passed to the decoder for getting utterance level speaker +embeddings. The intermediate time-independent features are +computed using a statistics pooling layer, where we compute the mean and standard deviation of features across +time-channels, to get a time-independent feature representation S of size Batch_size × 3000. +The intermediate features, S are passed through the Decoder consisting of two layers each of output size 512 for a +linear transformation from S to the final number of classes +N for the larger (L) model, and a single linear layer of output size 256 to the final number of classes N for the medium +(M) model. We extract q-vectors after the final linear layer +of fixed size 512, 256 for SpeakerNet-L and SpeakerNet-M +models respectively. + +SpeakerNet models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecSpeakerLabelModel` class. + + +References +----------- + +.. bibliography:: ../asr_all.bib + :style: plain + :labelprefix: SR-MODELS + :keyprefix: sr-models- diff --git a/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/results.rst b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/results.rst new file mode 100644 index 0000000000..eb2472e6d9 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speaker_recognition/results.rst @@ -0,0 +1,85 @@ +Checkpoints +=========== + +There are two main ways to load pretrained checkpoints in NeMo: + +* Using the :code:`restore_from()` method to load a local checkpoint file (`.nemo`), or +* Using the :code:`from_pretrained()` method to download and set up a checkpoint from NGC. + +See the following sections for instructions and examples for each. + +Note that these instructions are for loading fully trained checkpoints for evaluation or fine-tuning. +For resuming an unfinished training experiment, please use the experiment manager to do so by setting the +``resume_if_exists`` flag to True. + +Loading Local Checkpoints +------------------------- + +NeMo will automatically save checkpoints of a model you are training in a `.nemo` format. +You can also manually save your models at any point using :code:`model.save_to(.nemo)`. + +If you have a local ``.nemo`` checkpoint that you'd like to load, simply use the :code:`restore_from()` method: + +.. code-block:: python + + import nemo.collections.asr as nemo_asr + model = nemo_asr.models..restore_from(restore_path="") + +Where the model base class is the ASR model class of the original checkpoint, or the general `ASRModel` class. + + +Inference +---------- + +The audio files should be 16KHz monochannel wav files. + +Write audio files to a ``manifest.json`` file with lines as in format: + +.. code-block:: json + + {"audio_filepath": "/audio_file.wav", "duration": "duration of file in sec", "label": "speaker_id"} + +This python call will download best pretrained model from NGC and writes embeddings pickle file to current working directory + +.. code-block:: bash + + python examples/speaker_recognition/extract_speaker_embeddings.py --manifest=manifest.json + + +NGC Pretrained Checkpoints +-------------------------- + +The SpeakerNet-ASR collection has checkpoints of several models trained on various datasets for a variety of tasks. +`Speaker_Recognition `_ and `Speaker_Verification `_ model cards on NGC contain more information about each of the checkpoints available. + +The tables below list the SpeakerNet models available from NGC, and the models can be accessed via the +:code:`from_pretrained()` method inside the EncDecSpeakerLabelModel Model class. + +.. note:: While loading, remember to use EncDecSpeakerLabelModel for recognition tasks and ExtractSpeakerEmbeddingsModel while extracting embeddings. + +In general, you can load any of these models with code in the following format. + +.. code-block:: python + + import nemo.collections.asr as nemo_asr + model = nemo_asr.models..from_pretrained(model_name="") + +Where the model name is the value under "Model Name" entry in the tables below. + +If you would like to programatically list the models available for a particular base class, you can use the +:code:`list_available_models()` method. + +.. code-block:: python + + nemo_asr.models..list_available_models() + + +Speaker Recognition Models +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. csv-table:: + :file: data/speaker_results.csv + :align: left + :widths: 30, 30, 40 + :header-rows: 1 + diff --git a/models/tacotron2/NeMo/docs/source/asr/speech_classification/configs.rst b/models/tacotron2/NeMo/docs/source/asr/speech_classification/configs.rst new file mode 100644 index 0000000000..f0b17c5651 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speech_classification/configs.rst @@ -0,0 +1,104 @@ +NeMo Speech Classification Configuration Files +================================================ + +This page covers NeMo configuration file setup that is specific to models in the Speech Classification collection. +For general information about how to set up and run experiments that is common to all NeMo models (e.g. +experiment manager and PyTorch Lightning trainer parameters), see the :doc:`../../core/core` page. + +The model section of NeMo Speech Classification configuration files will generally require information about the dataset(s) being +used, the preprocessor for audio files, parameters for any augmentation being performed, as well as the +model architecture specification. +The sections on this page cover each of these in more detail. + +Example configuration files for all of the NeMo ASR scripts can be found in the +``/examples/asr/conf>``. + + +Dataset Configuration +--------------------- + +Training, validation, and test parameters are specified using the ``train_ds``, ``validation_ds``, and +``test_ds`` sections of your configuration file, respectively. +Depending on the task, you may have arguments specifying the sample rate of your audio files, labels, whether or not to shuffle the dataset, and so on. +You may also decide to leave fields such as the ``manifest_filepath`` blank, to be specified via the command line +at runtime. + +Any initialization parameters that are accepted for the Dataset class used in your experiment +can be set in the config file. +See the `Datasets <../api.html#Datasets>`__ section of the API for a list of Datasets and their respective parameters. + +An example Speech Classification train and validation configuration could look like: + +.. code-block:: yaml + + model: + sample_rate: 16000 + repeat: 2 # number of convolutional sub-blocks within a block, R in _[BxRxC] + dropout: 0.0 + kernel_size_factor: 1.0 + labels: ['bed', 'bird', 'cat', 'dog', 'down', 'eight', 'five', 'four', 'go', 'happy', 'house', 'left', 'marvin', + 'nine', 'no', 'off', 'on', 'one', 'right', 'seven', 'sheila', 'six', 'stop', 'three', 'tree', 'two', 'up', + 'wow', 'yes', 'zero'] + + train_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + labels: ${model.labels} # Uses the labels above + batch_size: 128 + shuffle: True + + validation_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + labels: ${model.labels} # Uses the labels above + batch_size: 128 + shuffle: False # No need to shuffle the validation data + +If you would like to use tarred dataset, have a look at `Datasets Configuration <../configs.html#dataset-configuration>`__. + + +Preprocessor Configuration +-------------------------- +Preprocessor helps to compute MFCC or mel spectrogram features that are given as inputs to model. +For details on how to write this section, refer to `Preprocessor Configuration <../configs.html#preprocessor-configuration>`__ + +Check config yaml files in ``/examples/asr/conf`` to find the processors been used by speech classification models. + + +Augmentation Configurations +--------------------------- + +There are a few on-the-fly spectrogram augmentation options for NeMo ASR, which can be specified by the +configuration file using the ``augmentor`` and ``spec_augment`` section. +For details on how to write this section, refer to `Augmentation Configuration <../configs.html#augmentation-configurations>`__ + +Check config yaml files in ``/tutorials/asr/conf`` to find the processors been used by speech classification models. + + +Model Architecture Configurations +--------------------------------- + +Each configuration file should describe the model architecture being used for the experiment. +Models in the NeMo ASR collection need a ``encoder`` section and a ``decoder`` section, with the ``_target_`` field +specifying the module to use for each. + +The following sections go into more detail about the specific configurations of each model architecture. + +The `MatchboxNet <./models.html#matchboxnet-speech-commands>`__ and `MarbleNet <./models.html#marblenet-vad>`__ models are very similar, and they are based on `QuartzNet <../models.html#quartznet>`__ and as such the components in their +configs are very similar as well. + +Decoder Configurations +------------------------ + +After features have been computed from ConvASREncoder, we pass the features to decoder to compute embeddings and then to compute log_probs +for training models. + +.. code-block:: yaml + + model: + ... + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoderClassification + feat_in: *enc_final_filters + return_logits: true # return logits if true, else return softmax output + pooling_type: 'avg' # AdaptiveAvgPool1d 'avg' or AdaptiveMaxPool1d 'max' \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/asr/speech_classification/data/classification_results.csv b/models/tacotron2/NeMo/docs/source/asr/speech_classification/data/classification_results.csv new file mode 100644 index 0000000000..25e3c844a4 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speech_classification/data/classification_results.csv @@ -0,0 +1,9 @@ +Model Name,Model Base Class,Model Card +vad_marblenet,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:vad_marblenet" +vad_telephony_marblenet,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:vad_telephony_marblenet" +commandrecognition_en_matchboxnet3x1x64_v1,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x1x64_v1" +commandrecognition_en_matchboxnet3x2x64_v1,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x2x64_v1" +commandrecognition_en_matchboxnet3x1x64_v2,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x1x64_v2" +commandrecognition_en_matchboxnet3x2x64_v2,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x2x64_v2" +commandrecognition_en_matchboxnet3x1x64_v2_subset_task,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x1x64_v2_subset_task" +commandrecognition_en_matchboxnet3x2x64_v2_subset_task,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x2x64_v2_subset_task" \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/asr/speech_classification/datasets.rst b/models/tacotron2/NeMo/docs/source/asr/speech_classification/datasets.rst new file mode 100644 index 0000000000..3198954a66 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speech_classification/datasets.rst @@ -0,0 +1,131 @@ +Datasets +======== + +NeMo has scripts to convert several common ASR datasets into the format expected by the `nemo_asr` collection. +You can get started with those datasets by following the instructions to run those scripts in the section appropriate to each dataset below. + +If you have your own data and want to preprocess it to use with NeMo ASR models, check out the `Preparing Custom Speech Classification Data`_ section at the bottom of the page. + +.. _Freesound-dataset: + +Freesound +----------- + +`Freesound `_ is a website that aims to create a huge open collaborative database of audio snippets, samples, recordings, bleeps. +Most audio samples are released under Creative Commons licenses that allow their reuse. +Researchers and developers can access Freesound content using the Freesound API to retrieve meaningful sound information such as metadata, analysis files, and the sounds themselves. + +**Instructions** + +Go to ``/scripts/freesound_download_resample`` and follow the below steps to download and convert freedsound data into a format expected by the `nemo_asr` collection. + +1. We will need some required libraries including freesound, requests, requests_oauthlib, joblib, librosa and sox. If they are not installed, please run `pip install -r freesound_requirements.txt` +2. Create an API key for freesound.org at https://freesound.org/help/developers/ +3. Create a python file called `freesound_private_apikey.py` and add lined `api_key = and client_id = ` +4. Authorize by run `python freesound_download.py --authorize` and visit the website and paste response code +5. Feel free to change any arguments in `download_resample_freesound.sh` such as max_samples and max_filesize +6. Run `bash download_resample_freesound.sh ` . For example: + +.. code-block:: bash + + bash download_resample_freesound.sh 4000 ./freesound ./freesound_resampled_background + +Note that downloading this dataset may take hours. Change categories in download_resample_freesound.sh to include other (speech) categories audio files. +Then, you should have 16khz mono wav files in ``. + + +.. _Google-Speech-Commands-Dataset: + +Google Speech Commands Dataset +------------------------------ + +Google released two versions of the dataset with the first version containing 65k samples over 30 classes and the second containing 110k samples over 35 classes. +We refer to these datasets as `v1` and `v2` respectively. + +Run the script `process_speech_commands_data.py` to process Google Speech Commands dataset in order to generate files in the supported format of `nemo_asr`, +which can be found in ``/scripts/dataset_processing/``. +You should set the data folder of Speech Commands using :code:`--data_root` and the version of the dataset using :code:`--data_version` as an int. + +You can further rebalance the train set by randomly oversampling files inside the manifest by passing the `--rebalance` flag. + +.. code-block:: bash + + python process_speech_commands_data.py --data_root= --data_version=<1 or 2> {--rebalance} + + +Then, you should have `train_manifest.json`, `validation_manifest.json` and `test_manifest.json` +in the directory `{data_root}/google_speech_recognition_v{1/2}`. + +.. note:: + You should have at least 4GB or 6GB of disk space available if you use v1 or v2 respectively. + Also, it will take some time to download and process, so go grab a coffee. + +Each line is a training example. + +.. code-block:: bash + + {"audio_filepath": "/two/8aa35b0c_nohash_0.wav", "duration": 1.0, "label": "two"} + {"audio_filepath": "/two/ec5ab5d5_nohash_2.wav", "duration": 1.0, "label": "two"} + + + +Speech Command & Freesound for VAD +------------------------------------ +Speech Command & Freesound (SCF) dataset is used to train MarbleNet in the `paper `_. Here we show how to download and process it. +This script assumes that you already have the Freesound dataset, if not, have a look at :ref:`Freesound-dataset`. +We will use the open-source :ref:`Google-Speech-Commands-Dataset` (we will use V2 of the dataset for SCF dataset, but require very minor changes to support V1 dataset) as our speech data. + +These scripts below will download the Google Speech Commands v2 dataset and convert speech and background data to a format suitable for use with nemo_asr. + +.. note:: + You may additionally pass :code:`--test_size` or :code:`--val_size` flag for splitting train val and test data. + + You may additionally pass :code:`--seg_len` flag for indicating the segment length. Default is 0.63s. + + You may additionally pass a :code:`-rebalance_method='fixed|over|under'` at the end of the script to rebalance the class samples in the manifest. + + + +* `'fixed'`: Fixed number of sample for each class. Train 5000, val 1000, and test 1000. (Change number in script if you want) +* `'over'`: Oversampling rebalance method +* `'under'`: Undersampling rebalance method + + +.. code-block:: bash + + mkdir './google_dataset_v2' + python process_vad_data.py --out_dir='./manifest/' --speech_data_root='./google_dataset_v2'--background_data_root= --log --rebalance_method='fixed' + + +After download and conversion, your `manifest` folder should contain a few json manifest files: + +* `(balanced_)background_testing_manifest.json` +* `(balanced_)background_training_manifest.json` +* `(balanced_)background_validation_manifest.json` +* `(balanced_)speech_testing_manifest.json` +* `(balanced_)speech_training_manifest.json` +* `(balanced_)speech_validation_manifest.json` + +Each line is a training example. `audio_filepath` contains path to the wav file, `duration` is duration in seconds, `offset` is offset in seconds, and `label` is label (class): + +.. code-block:: bash + + {"audio_filepath": "/two/8aa35b0c_nohash_0.wav", "duration": 0.63, "label": "speech", "offset": 0.0} + {"audio_filepath": "/Emergency_vehicle/id_58368 simambulance.wav", "duration": 0.63, "label": "background", "offset": 4.0} + + + +Preparing Custom Speech Classification Data +-------------------------------------------- + +Preparing Custom Speech Classification Data is almost identical to `Preparing Custom ASR Data <../datasets.html#preparing-custom-asr-data>`__. + +Instead of :code:`text` entry in manifest, you need :code:`label` to determine class of this sample + + +Tarred Datasets +--------------- + +Similarly to ASR, you can tar your audio files and use ASR Dataset class ``TarredAudioToClassificationLabelDataset`` (corresponding to the ``AudioToClassificationLabelDataset``) for this case. + +If you would like to use tarred dataset, have a look at `ASR Tarred Datasets <../datasets.html#tarred-datasets>`__. \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/asr/speech_classification/images/marblenet_vertical.png b/models/tacotron2/NeMo/docs/source/asr/speech_classification/images/marblenet_vertical.png new file mode 100644 index 0000000000000000000000000000000000000000..3174d1ef5ea56ca36159d7525f2f6acf6c2367b0 GIT binary patch literal 393467 zcmeFY^C=Z6^2B6pgf& zjdeE83{*^X*3b0ytW32pjI_=TXAjiR?W|@GG!~4st{!YH?5vM%E{&|u^(;-)P4$hf zFLW-AE$pq&?5uV!jE!wB=!GNIP4x_|&lQX`S4?!*&kXdf%ych}#mz~yE{vr1)U51n zh0nZB?5;@dt6M$V$?U7c$H(`YP)V5+`8=t(y1M$rtFLIJJ!?U*aET{xs4;!vB_=ld z%+`;Bq2{u&_PUwAiOrRsm5J{Q0*!M6E@K)xIy&$bmc%)+(k1S_y}i=XQch0J>SeaP zMLv2)hNvFlZ!=OTC@A(LItJgTf~VwIL)wCagJWW1npPO2XC*6^IBsumgQw-)+}zq1 z#=L!eEG#VG_4SBVx{HfT<9Jj)VIl8;Pr^~10+FL%rrxHdrM0xQ24}8LQZ&kdOEW9Y-ICSU~-hS6*6tW&{L|XZ9iV9>1st6W;_=}daI>sQM_d7V5@Io z0-I6<`yj1vK8>1@l$Lw#3P*aMfub3RWLt<7)$P_k)*sZyPxXQ_&C1Hr z_0HwW$k=G=kW9pk;??bG@BDbl*g;H3rLr=(`lmOn9rA<>iN;Akxi?ZqS)u1=Oa6+M zY!c*sm09M%($KBX!oot$STBlGXK3fks>!96xvr7@=AzN%+~F|S11UDF)AasIG= zqPK3kYwn<~W@dk2d3s@QYkG5Kd2g#^q&2OtTb7xLk~T@;yi z3U`F2np!i_PbPArpY+;vo`J;Zj2&u41X^z>@^Va1kwXe!UQu)fhl0#BM3Cn+q}L-IdbOp;oze2BR!O8J&t`-JZ(O2G#=hGqfZ zm^QUGB9qGqEwh}W0^t}WeRaIuC*u|dDYgFZ0X#J;t$<5xh35qc!ordQ8n zILi*I3BF9OC3wFG$+!FrS;~HqiIO60FhgWioX~+RnQ{uiiewb@NEjX4y`hGg;3d(dl z|9Si5?bC#o%kzvzQ?GUgRvd|9?$Hj_dXmiJMR)r?l{%#JGkj`N*f{({Uu1ez*aMEh zp(eJNXQ54-a^hm+Yu*=-tR#-#{^GmO#23fX#m*LM0&3NKL64f_3?-`Ojv3FKDfd2l!7H3F<6Q3~D zH|r10?i6vCaZxZTJo)1ZFPw#U`d5Y+tM==R2WAXVasZhd9}g4x!@4*2gUYL$RsQfb zrDgLpi_4B}Opim6{HxpMD|?g;U3DC`Mr8O~2%+-?(#~MPe3XRKZq@zLXqqV2c#d)U zgSkE`Wd!H+c*D}e_fMpf52_2Py1yf|2<~EILcG;NubN*EuqK~CM<5=T8x>U&`(P?U zt$t1B#eh&;!KfDHGE5=KN+aw$e>c^Wr5Ko41#ygw}SFatOyR~7d+ zL@c@JPqyu??B}b-T zwed&^jqqY&aVi2L^Y9R7;KLAuJ5emgpEM3P?VmmpV(^fE=9E9klaKuq-f-)ZxD1(Z z+JtUP1%<&)Ht0P)ozuC!uG`XmqgsUZc78<#TiFxw;8ubBq>ZWuK)wdf?|U-sn7A=(7b7G3gnZHjefq+H@D zAUC(?pFAYLJNDihmx+td zb82fdaQgL(r&)9LiN-fvEBp1M%ayLmo-d=%dii~IeAjZK8ogez$AEv_hLOk-9De#K zTNN_YJt(97%Pm3x=*eH`_~B0mX{-|(Wc>;-DYXR74*w={hh{_ z2UyrnDB~ym(t`Kto>C&buO&|BBe`TikouWlR@$P})eyE&EbZusw=;RJMa#CmzEZ*y zdm{Pv^RqW@l#m8atRL2cj-gAVgv8NrUx% zlS8{9AV9Rk$r;oQKKLo#h;A*zd0JZaFgO&uq2TY#D+fD?`jnbM2>1fNve|_BR`=(7 zO_!sLrP|JzKUa=6mHg#*Zm%an{f5(MIQqSkN4y_Mr@GZ8Pp7-#Z1PiUDhkCVG%aJ+My!7<{2 zJDtS0e>ZM_Fi{S)6^^u()v85<6gHYvmrTdJNx;A&+$sDO-*XG?~=4Y ztb4wS!Ij`41?{6TJi+uQ(9XwfTb$8EvTG2tqgNUMF8PGGUf5=a(A~s>p!8&77dwdc3FofXvW)m6DD`8{(SmHS>M) zlZyhQg!h@|3wnXytOcSmfcw=c`acJVe(#T-BR z`W)PSMW?=B5#8W5O0B6)uIW(t7i?E)ibEHpPRVvNzClmuXz7n#M{#UO`ZZW_~RdCW!>d_q)`*Oj148 zuvMo%_(*QVg)OuG%OP^X7wv#$3?T%h)U7X0({>RxRT5oOUqU3F-^5;h%J(aQPzcTL zfkb>tI2v$?j=(Bd29*s+lRs5OkSMmNDt8xmqB&nu+0oZLf^+${ahvtyk~;A{!dy_& zNeI7xo}Oy6!%Dpk6MjkR>yghugqZI-`iuavII7#qd~eY}Z{eq9)|p4mrolgwG3S%B zdNkA>g95)6%&4BuH?|+=ho60`lCum7P|XDsi2K8Dd{@~ti<|6wK@RN;EwH58i|MV* zr@RpI38Gj8Z}S(kYx&9=<#vpxH8`f&lO+0kk*o#N#h8EK&uyGbnGzms-Ie31gRA5$ z?tAhM=24qH14fp+A_c%Uyl|(!nO;zKQMkv@n2Qc)q|ORzapsJUP_D;}(D>i>@wGKmO1CxbuZ`_YcV7C-;&pxvuy3;Y!u~#D>rv9Ztf{H1tonO27WV2*pM_uA zNu17Pn4RfuF!um5*7(AiR-~kA>w%_}9OHrCM{)?&Ga8=nYRweMg9F7-4*Y)P4V4=Z z&sI~!=N`iiIRf;mInB%^T%MRovXe4C_54E03lBWT4`{?nrRW1t8x?-|{A;56vHn?h z=50Y&u`~N|4t8SXD~ilOGMC-Z=~oijcWBb9e~_!DiCW_tf;3#ae$b$y$_Og;{xB-B z@Es_x;F67Xy@!tm-GE*t#}J66V(U}B`IcCM@Yt}CRuTWG3z}RH*g#6Ie`6`XporQP zLDxL}tvSMqVgfZa7D8nT_)%3aXmomup~GI@hOav@69G=G5yF@N(k37m4Ax*FD9sWL z6IWpgC1iRna`ScWB!_;^-9trrG zSY?lA#$}M3<9g-_IA$|(qz!60%m?O~86OW&6Vzf&31Z_W16b8z1o z|K#-T$fc>?^$#n7KjUaQ=N@f=bR?h`X^b${sXge&dzg%>%=RBji15mir8(~E9b-N} ze^IiYH7KH)svg=BAAT%3L6Y&;1aSLIO_jYbtD-OokxTFW14u8Z?cI89&@Oa@dClfk z0v7rJpk8N8NTJ9me_|0wRXqA&Lic19&KBIT!4vUGSo@-BCH_%%A}evPGXq`+D0A4F z`XpDu_HZKw4G1o%zwG?hB9;MA<8sy9d{@TcVME$v>DPf7UTXCow}8mPXN&txF>c zuL#bk_8m8FUjPWyX;x)-`e}|5g4RrSO$cf5p*IWo&WhK5k(ZP2O2vKusyP!Jb z*jDkrz~jY+S?i7f1FSo8d}S~xQE6&tdXO#sauD#EE+j`}{xpeBF_H}C?8);CCSlh3 z11Q&%!@!_mKwH=R1=Db#rksYi|F#aj?l?Bb?zaT|qmMCy(6K&Q+inr^pkK+4IQwo8Bu*)Fa6)!G9z0j{T^0Gx%tMBO8q z?^T59*w_4~(1qV@4lc^ztp!M|otZ~lSgIk3ruWtC&y=L}iVdbYgjPnYU?Y>|PPms= zMl8+S4nUTGz%>+UA+@L19Ok&6+nsFNeEb1({}IAj+ahOj#djyiLRlcI8k5Ajw=~+* z&`P_^8-K1hYWT5m>3wkWCw|@%d~@9EPb$n7*QfL)!CuV?Rj_)9Fs~I5ExM?kMj(2w zI){Z{Fb`9H?2VI-FNm6*A0~~?5{b=5UHnchj z8Nkh|Q2S(^$TlF5@aWiD8u;mKpr)Ad0M)I-{Y4h@@FovQTJcVrdA^W0X~2yMY3!x> z6UFjEN$;+98jG%eva-W)U>;QgZ*eRFJyHwOcDhhnK|iqLtR78&>g_^HftqlCQ^AjN z3jlb7RP0*-7RqzSE4gVmuU;MXqot2NM%q}_g@}{1z8J@P5rg;k6VEpLbtmB2 z^=gL?#evuSt~XH_g&6B?v=C?w_3(yKx0lvDV2-PmUFA%gNGbbjoMVw&^f77 zyLJNlKA>}42fvRqX$z2|7uo=-U zsF^7t@jHb~<*P+=`{-8Qb^c;FCO&l9L(i##zB376-TDNRu}ckt*s$j~*g&7RQOlsd&k19tK>d!kH#)4PLMMEaCCti!1-Xg38Tra0d{=Qedw37b#e+G&{ ziOTHjXj>pecOBL7z|}~S=QliWBn4PE94cc9{8cm<-AhE7DF)t%e2j;Nb4NcNHuDDD zX*+K5kqKP9dP&Q9M6+A6j|5!J;k7-{Gzs>wYX-GM`(8*v6qpSn3+h)EnbQ6freD zH+0~UDTX9d$vOxiZp4>;2jYgTrI)exS%GG;WcWV|NIdV){<&!TBY^@Ua58@$-*W;4 zMhD}Cc^PFF+k?yrt@6bn#bl$GmKs`HYdTvcA9&^ti~Q;YMLLrPb(c)a7c_s{+PMVq z##g{Q4QmR{XJ!YS`MR*zKTZWim_8=?`!v157hle}qUQDAw>|F#CDYQs>U??uM7{dh z7nQJ#B_0{EZT9}S$Vud>`-MjnoFsjbL4ofs3~LmcVmgo^D@l2%YQ|VHpDG(~;~e=S4)`HoU{n@@O|e z)XYL-Ix&0l)onMqwS2c! za;shP1$X>a0@#!&>0jyvUCII`olH9I@*t3$pVC6s3sQd7A=27H4cq8cOv~W|4kdkb zUYX~@?l9&*P7ak}!%slOzM@APJ(4&k&&PXcv(cLa*$UuZNMhwzS}BVdgZJxJ#4RD_ zcZXIUtTQOH9dzef*nnEJSfX7#l&+k$d|(jUfJXyVVX%0a^mnCOsAl7W(L7wZ6=CtT z4D*i)DGr1^YTMF&J+g9}QzkYwK*q(l?V=X&e+B6PW zaO8V?qMGjwdghlM*+0zIW&I$JFk;=kQZ5-tm#x~>zGw6!x5js8_hYYoA7GMqhXl~U z?9IhyJ}w~-3;~l9jc4D^f+g?Pznx}tGrZ}N*7@^7Dm{clF`)+`e_VKej@6>wu(jQx3Rgpy6R~9*ZNN{f@t7M$CFB1UUx1Qz2ZxNm%~ChPl+) z#{FcrDk(o}{?!u4tjGTcb5U4tab_Ax->!{+okKGZ@1^2@wW%jjSy2#h{b=i|WN@ru zLxE0o4HS~ny4Q?l_Zkz|&Y@jcPVY){9T4C{5kn-%{O^4y+glO@2;b3kuA_mKdKCLO zk;`gl@yvuO{?L?t_z_Q|u%@b#dNPfJU!|DXM}Ko3-PC@i!Q%Fj>{k~hq=7oGQtqN9 ze@7;4SK>3w#KVaI#`%{5j3Y#Uek%5?B+HNi-XU}`!J1PkcVpd9p=1zTUTWA7TBy@g z<=5EeTB#BYgat9?m`UCE`W#vO6{oK}aO;NWnFq-JXn3s`29h~yE8Zo!#&~LEvH>plE~=9anXeCKMHPcbx%ze9>9SX1W-sbTYG`_nK!+cQ>~Cpl;ro#_9jK*mX6 zA*)Hg@zOzDS87CWk3z6{mgd+0f#aU|nq1Fz%_fBv0_Nq+ItD?r6ylXkN#?rpW7VNE z>{OI}_8ILvDVF%#hF|D;O6RZuWb)5Bmr89>Ppq|PC0iUvrPAUtAyRwozXw)_JlfSP zo7Y0vu++Y*qASX$4XU1YoNPYl>HlJkDBL6F$f1i<5G=sv1?xOO@jUx%JHx_U9^=%1 zLOTP1ax`}qlmja7O=1Md_Rl0qrec(2+GTkPns&fqv0H0AWo2ih(MY4{EwezjQlWn} z1s{&Z2ufHG$X~KBKV>;PB6>CI14b1(^5R^@#Lxp-E#Qt}HE9Co#0=b?-5E)ZoX_DAqG)%&weBYQn(@T_0`8tsG zN^iQ!)nc$BBwPI1iV4w_q>D=ccZ9Vg$7aOIC`&)#;CMPYg6)FncZj_&DF{p zi3hB9F+9CK^#33C+bvQVd*JICw#t7?x6LUP>Vau=#+P#K3{XYf0@%7 z{+5Wa$6t=a8$&@E&7hNwPx=&F4gCupLnUKgR=}DRwu{sRH=?u9BSQ1}J2`B7LxeVs zXxFL~o+(}$3X=izL0XJ*IRG+n?2E#J77HFN z_+EV0!allBeQZwf`UCfg9#H1{s@_z=_!G5LvzYq{9i{hfEPi1zl$>7wQLBwZ_C*lf z4SNhkX02Hbwgs|O5CnP)-K+;K5Hk-Q=37A9;dp%9{YIu@nDoyG%2$wY#vL zkD*BcLAyo0er9i~YEF@O>rt&3`zxoPP2GR+xdveX7BuE6P({9hk;Y%Np+#zlJWvS< zGYXx#B1p8)Ir{vs6Hh!R08?(Yy6C(nTAMZ4`#`dzcDXsEG66fh>8z0#hO*jtSc*NH%$d)FIzq+0 zNVy%QNhIUG=Sct9z!iXjalrA;4YB|yzu_BwVmzBRkj3@ zo=0)vnc5SjEeH6Th5gDyXO`r6uA~n;V}2m#>tnE!M@r?f!_!0BaURh$kwc*LdLZOX z8TeUN2-i5L)fM|8e#l3~ggaXH@g%0bhQ!PbKYzV?jyIF`qj+i@yNW);V4Y$1(PV5j zJGP!zglJ88M{pMJ3Y1S;OH_VW6IMS@1Hi?Kxj8PsXL3>!rn6)kNF|KrJJV!%EsGfm zdE6J0KPTFhW^B{=(%7#xJFra_xV41r752M!OT*FtL9p7Ehox|Pveh!>sC2%f)gbKT z7NwmkW@CrzG(}AI`o**3|IUB%So<|D##N}epM||8O9yz-FqKWVud}Gl&{+=F-N$}| z1{K7MS%^PA*=8gWRC>7ZqSS<4J<3GJpF`^A3|6ejU;mL|fTYlG9-}fuyp_d1+o1X< zC!Sgs_&_T++u5DPl?@pO?ZD3ZwOCYEB&vIvwFz@nd1N*`iIRnIUOmrx=${%?S(&ll zeH4a0H-r162kd*Z2i&u=i{vnYL>#nD`hBdxxHs zaq|LOva9yPg{(b&KvoDHGM6I!_3Lls4C&Bc-bvWlCb*(5YK^GHEY#CTLUfu`f3#52 zLPX;IKjI=NjjhdKr@^h3n)werOULZ1ZMI!;vAk9PsIN4QC1CPFed+J^_n9J0A+lRZ zE@x{XEA0e&;ilCNgY&nAd?BK`ak+dB(rFQJ}&0r!#af8WML#A~XD;`HgSwn9N1zsPm}2fqVt9@*nZT7)3oA7|#Pc zim)myxz|hs3Ie=PzFkqm+2Un(cfCwr(9x^Dwf~9l9!XvAqa@3IYx#eayJad8mEKkg zoLT@PX$Y*8vv%Rv{4U05Heq7xq$*^=oa2^HZSteu8@EvUbQ9OfOtA-zRmxTDMo3$s zOJ#?A&6zr&3;Wz!CLvaFkeiM*&IL2RQ*?j2P(k=<-uxA3rT?`Sns@-4o6`z$0y(*r zF;_lDK)Ov|m6{Xn^>n)fynUOB@X%9Wy-IIK*IBA81kGS3_I{7TfE$B$dWhwzJ zFj49%MbsXwY=kgbj?{Exp#PJB;;?+wk3{p)Z-i`^n5w&O_qCLpLe^hagf{#))^Op! zsBb!W)F-tWfqb~7s3xmrxc6Vb=%=Hax8ydQXjWFI1OcJK*>+oa_76F^XozaeeE)Qg zPU1T-FMc9ZBI+T7g`>)-U{ z@VD&g=X%gzVY^(~`Z}G;l_|{?;Q)mHzp)AvEwGe{X`NA%j1{#QD8C;C{CQmNXEo&SylA~J(vjqVYR&PTl*p50kF$+iJ^8t>k%c(WS+KJ>NXT9z zpz5>;D@o=}Uo7@Cs@5;LO8CMQx;xtWVMDVW8>C7KD1ndf4dn)}(wj>Pd$bDyBy;+; zLt=ck7jQpZ0eB@7&q*$M(u~_WU@YHoykkr>U|PxRBXf;^gz1Sn>`xZX)bc2r*B5}hR zmCAXG&$y1Pr;2FJWLFi@;W#XA0nHHt6luUop!j76goykxKqL{udH5-MhiOLQp+>DQ zUif5&HN~gTYKgd#+S4zCy^QCInKa4|#ZKn_%x1`-H30PC4=M~uT-u5oliWoH|C_-yC+O)X zEs5xNaJMX<&`HK>LsAm!>4j)zbS;%i!aQR?>3+|kKrz&>UZZl1MP)3s%dQ;>wxnVI zc+W6RAu?Ps_HqQ7769keBs!P2PoPHDXy>W3ODk%Aw zB+lGP^am+V9@)%tfvrtGxF7omZpSSk`+q?E@@xIr3d;d_THxb0o)%bgO{re;!U!Ao zL^VtsdAZoD61&x^TG=!}$LYIlaCLxROL6`pU{!uT+k1PP9^!zc@?# z4K+Sc@@t~(NBOiQlmAF~>92gq`tziN%6fJro# zjY+J=QMnn)(fa*GQ>0>c+2TZFG?C^cqR`2jd@7BBUHXkX!+DgnCS}vYSlbQCuzNq@ zcQUp;Zf1>p?|m%WfBAzQyr65s7>(1+p_f)AS1IXL(dcVpm%V!#3^W<2J->1)s#qmq zKQu)w>&8BFaHnC!xIT2rZeS~f^V(sCg)>>i4DeQs^O_$9?QfK;Oh!*0^_z8mEx-YV zIPi%EKZdo&4bm)5nxY0`U7kmJ{sO%(r5%%o1qA=|Ec_9U(^Sdn{pjLHPEe_4l44hM zx0Xy}frP&W@S>k)kXa{#E1D5A1yP;EhXz4qbF9(O?*Z6&5?Mwhdu2`JhF{LMsKtccY|QvwBA&%kGllL0@?kdaUWmge71ZppZf>lO zTUoko#j~;~``IX99xndv6MhBtr^uU-N5Z{z%>?%ZL7uci&ny}LyxumE4buZtSY&WOvnQ_XD-eIB5n;wMDk@V|-y8}~voNPx`Cf>cE&cAD| zmM2h6Sil?Y$8O*;24pSQLfNoWLCtd4?jYy5hCGl4Yzr8Hyb#rr&09P%Jccw5tYvI0PLwh+y<;MJ^W`MVDVq3+V~A1n1o` zYDW%hH?Lz@uhyEF_zJbf|rT4NeQjg?agkSU9%6v)Y4IUM3_G~{gZJ7^cgX@J{8}eG} z!xN2w{cbKTx(9Fy=gm9^ll^~nx*SdOIjnTT6X~c<>o-+rrkrIxrlD00Mm>>BluSlF z(?Z1-`px!Lu#HMQLTVSc=yuVAO>Q54@l0Js;HoWwt;F4O3z z$^L1e+I=~HnRM#ybfA5_n)(8`ux@u+O?oC`x!ACdY@ky@RD^v%_G8sJ@&(spsEf?3 zZgmQqkh?n$V;PvI&oLJk(f!v|b3lNed*e!Ysw2Q2d`8SbBO(OcNVVMW6P>C?weDh3 z-Q8m&sl{A-z;5<(6M)D>1N5l;FL39-25;z4vlrteGei`9j{g^P%nSULx&u_##5_<9 z2>}_QuVOBve)s^`e{;t+CnC*ayJqdZ!k$ik%KA?sQA`|@d=`_>m`ep7TI{y$RJv3p za3h&s8}ot^TOT{}T!h-*czOAbykcUlOBqWo@v>_w8S9zi{UfHj74aBEg>{9_!x~K< zvqi{0rHbPQ&3q;Q`>{m8r52w;0(WXUKAH5T6CK%$9VbM*%~s4vVH!Ztr-XpZi!BQ^ ztY4n%sn0|A59HlsYe^Q}pAzg$z52xaXysL!uQ*91wlB?ecLH(%>7S-YwbR@b>&=t2 z*gm>%hs0M-YeigD^HFhT;>skh;ujvK>=R)=iVclyavzS&7oM0r8|rt_ab~lyD2fqH zr!QvG7N}Dmb}*X8f>SQXe&~{i6UHCeDb7}v^y`3Y93uW{UU{M8-Qb3Y_5i@+DS8lC z30V>NubEi_LX?xgq1*&h77n>n1uHR~8MZ?qSP z2G}4_lvdghG$1}Pq+sQtv+1^)Kg~;8=mGu9L(6gLCXFv0^ts3%A!*KSkZTGYlZKUa z6pbsgN-j&xx-Xpbr&b7ma>fXXShW;FevbEd21Beul@zvJUl@kn4WFNl8hcvA?Ru@r z=GD8q7!OX8`hAFcvZDrgSVTlDd_VMYM(6$+e_lnq+B{(m2RZu$aFU`MzLlUIFuKWh z@vNFVvI@3BMISjYvlluZ!MMX~CHiYL-ob3a#{-#67h*jcmeM=aS8A(6dI0liRTi<} z=~TGTd6KAEH$AHr!`k4?uguH7&26di?~#V%FvEIO%|Tmcc|f;WuIR^li{%m29M<*X zs*V?4>H8WX_~`I<9-8XRnv8{IY!u63hQ^e!mS*cBbE8B@qO5UQ8Z!xl7%s#5=!0nZ zEH0-@Z02$nL0nlca-7+U!iLHgpTJoM-#kMv(VIz=(AT&BhrP`6pRtnCgy_eG$L|B= zT%J$xuar7W8g{fi5#U}g%Bti{X2X8)l4OP!EsQIMI?%~S&P0Z6)GOc}_gJffqB)a- zSR`txW($_0%6AV8l%s33KfD2 z>njbnus3xNitwp~rF8t%ISOJD65MJW(394k>?sZD#{*#>$JPnF_W^~HOYdRvWDhW_eoQ<1VUlM49mioM z*Vii83?*OKyvzWkBapd3K%!w8usz!P%hm z!i!TRFAc5v`L&dT^FfP}k0Di1I=I%A5fOl-4coZf(Pzm_x5&(@F5%4xAf z(U!3pk|g3w;l*w98n$%omF6kXjmHMXe~w7#??F|=?-TMVA7>m$;T@D5!r|yU;&$8J zg>1#}wl1<5|Qf1oHt>np6Tc93tdh0WPajQL?6ZJUZL;iAyi)jK z=4hUb*Wm<{z;b6-_~8FunRx*$Z*xFwdi=Ar>x+!`K*hYW#ZFO*`m3@+hblwFG%v3) zb$=c1ytDeULT8{<&X0-c4yYTO{6!=eu}anTQA0W1i)inp&PO}?e`hNtLI_%Zbw^~z z1y8(-%tE%&*j;gINaSv>a2Ed3aafft^e>2G9?&YL1JzTjkN&-)M%auf{F^1(%$+~Tbslt0 z-hF)4V&!F8Z05g2Ter=5n9NS^z#P7ymRDGovb|+k6pMPhuC#$n5H!md_f+kM!|Qnda&+hj(vyUR1tNOabzpL>-xhAEUc^NrI6m z-?31c#`J9Y;C~&lpu=JRO+6wU@i;K)>=Qa9;QL5UznKTd#qKN^`&k^O+a*TSm)VYg zqLbt(B@9hq&Jo#*G$p)p&?1WFa<%zl>>^3nov+ojXrPev0aKL->=Ii>TIz^XgjZL` zrb+xSnMb>X8D;hE<|P4)AmWgPYc^1#5&{q#_Urf+UjZ3SM5Z>5sLb#iB_Enl`#NZg z%=;G*u@8h5D~{?UMibRlz;k4r#%vIZEV_JY-)`FkN)*_ia@knKho%5DEdYl72DfFZ z>ivpD<9B`UR#*Lwl3&t%x*lL|rlQBgOEk@Tu2O$Hz`XkL=Bw2^-(Qk6t40OSx4VUu z2@a1YUOm#3-xcRKj0eG1TQwtrAgSZ8wEIqWqA4bm1a|z*IkR6*3sR4S!3xcHs}+CGhr3tGpgjJ*3-?cV zX()K+&HPnJ#~P$%uDi$X_H;SsAO*Lrl=#i9dRh*lz!gx#<^wKdFARf-7I}?bdZ*-u z@hC*3ZwC!#=yP&(+*Wp)pT0=45+qB8S}0l?e3gykP)~mhj)|%w znk<&eajw^rZC5-h{X(@AQfd=lS68z2*xWQGGQiaNgc(~7?bToEE7=7ZS42zr@}sbPd$5t}~vG+U2<3K61;*D|d$6VDd*i>}CZs_zJ=`CX@uSh*f2B(J5>(SIG zzSoT1IxkW~Btl-s5iec}x)O;L#J2y`&GFSue7m5-BCbK-oS^?r(wsA3%^nZAO^WTj z?Utt4)QjpH*-b$D+Uio}$wOLHbs9nIli!Y#hqt|2Cz4EUXX8}ITr z@im!LldElK;a>nm)*f6uHHE-Re zP4)5zQf3~NV*@h$u zfy(Wn((;gEcijGY^nMQ9@_7RhI=3w0yz%T$#~eaLr@?vjtLt_|;0e{6A6}bj(Y2Uj z4VcCkUwNsSr0l!C&Z!b++Ek`{++WVAPMJ;!+$1^8(cf`2;kKQfls5;3wwroB>8>>E zsU(|w_^s;r<5oybe*DA*&HCAok6-KF*j}LtbV5C@S1#-$t6RppEw-VD>9#Y7L?N>IYRSZ)c z41Z8bK2m-X-$Nwjuk|nM1?HB+m1jM}yyli0R!kkZL`ddHlSole7SzAJUtw8Jy$uIK zkk!-yR`pNUt$*um?ujY$4ncv1aB%F%%mZj#p z=DR2-+a9=;)y9Ufm6ahrpIn*5#oi#g-G|#k7wW)lz0>VRU7U3Qz9jn)lfE+?1*}W-VB*@{!D7XeG+So5(b&&o77!=|-h8P#(}a=&H@4 z@8tcf>?f)%5c&~a{%dCQIi}t6omTVpf!q$XLWejv%b#wK{5c6t)bx05CUeP#c18f9 zj;1?*hA_0dwzuOOjQO0Usg2Kbp6ZVq&fZ@I zOM-iNbj@(3K(RaTj$Uy2H#eBS`iSTJd;*3+PS{9-l8xIz-eE-NC{2aV>11ab?u=r{ zvLRy%!i*wPR<02?X)*VL#?PlqEk*uYupd8ys zwG`iWWz53QHs5T}R)I)PeFJX5{$)tySbSLe9QiyS7HVyV5|o=gC{1u=$%f>=8ek~V zg=0a+$)DqPfY_EJ*H9|avv9LG%abr^v3|)Sg;bgKWFCJcjz<4$4orh`KiyC$eo#2p zM0q6bI(Kh<({fb)mPy+ROijb{P#_%fEwEfplMjdu&M3J5X9O7*r!3MW%B$IwkU)?>2a*x^(D1eNT39H_ zNd`qHWEqPBbsQbXAcU?8w#~@|*2AN{NstcJ$Fi@oUto!#uy-a`1EH9*SVQArcmXw1 zUc7r@3jUl-BdCAOCUduj5-V=zd^tN8XMK~K`_Es;CA%)IJnVfErR)V|@GeAXP7h*^ z{46LC1p1Ciqlgd{9im3<=KPR&GvcaH#V@vRKgfNoDwuLZ%=xg?T$HbfBdVM#1L_LC zi*AUuaF6JACV~om&!neM^&1y}SHR*HXy(hbUSlBXFJJ#*Jot>nlbzl3K9`HofMsz* zNQ%PX2Z{~u0EEIC+5)R%GPB0jBa=>6A*@RhnSBGftI6RCfzM-&qoB}*Q>}dl4H}K- zgxtxz&khI#9oZop%(wI~ADR9@>4bHGq>Qb9$Z&+sx1f)Mo7ev zPIHps#!Ls7iRxbuAbb$fFFuI%GsrIF;~hTf_bc=dXTGZ;E(Y?xSts0+DLqJZ;lZEy z-7nPiu=U&DnF9`p0%2J+a|wY*or8F-aakRCT@Qz6kT?e?L3HB^=OE)3wH<)=4M_HQ z)@05=NSsLTtbc5Lzw)SJ2&hHJGL$_BZGJfiHPFz{$)X_8&p;?faoe0x4TTBybZ*e$ zp02GBE`z`J(c~+R=Pu^*ncwj;K)`DkS>i#GpP>y{U}MWSz2JZ}k?%q}((J`bx07y` zbODTC^1k>&yV|Iy{EG|WZIg9xWBabtgSbar74;&*aBRZkOBpws^Y~M)#P|-C+asD+ z+j#z9B9Xl==pu*WqEMi5ohlKL7jhBPapkti;ccLJx_Hnpbo?SO2Uqw^R1~2WM;q+k*6tyidByy0m^du;u1r`L7QI)L(rPNoVi)`hDtaBa{3_zk7m_s3W zLG_heoVL9Xu)!;v=RtcziN@hWI;dz|2AQV&^s!?aS1SJxO}Mh=RSt@aX!gvzjp0*>|g&!9H=tRGUsKJ!QjO^WtcQH z-P|C-@6pTJw|ur@rpy7_dJ$F;v?jy+%=<#u9(r!uC5%=Nm~v6M%#Srk?ML6l-h926 zFcCt>tDIZyZs;gWQ#d6#5IOvZ!Ezj4r!|1)w{%fOY2#UM(H%BMv+^T;6hLV-1TQt5 zJ)N34*MaYbpEYD*!4qd8#yWOdOg~=02WQIe;^juvolf*oGyCmqZnAd?G%qVfSiz%a)ALM7CSGc!}J^ zx{o=mtf8yz_1eIYg$qTPfWp(^{@4}AkB7$woSBVT==l*EF9SvNLIcAQT*2-eQv%YH zrgu0b5+;1%p~B!JxOLsb{A=XQHB4#&$2|7?jzZMjksznX55Je3zvl$VK{JU8D62%@ zp@b-o<9=DjJ@lzWmwd-R{XuZ-txgS{*eBY%goTblAg-?L=;l(m_qhH?tG=g*3uv6b zjwcRdIUZ3nkJAw0NH*^RoSZ`k-uiqO_dFvVj>ak0Ui_yQ6xD^-rNhm2o*Vre2{4hn z16yeqXw+Z*F!}s((IF+zIve(^#C7gZn|1LQLyvuhLUryR*gK40Pvfs~|2;45Mlr}Q zvAZtZRa5-XzipOO9^M&)jQhKMUdofFVw(5(vWl6$gsw~F+CQGNIXNeXj_k)csGfYJ z`g&0ND7|4euAiD1i$Do#j1@9^pVcKHy&%kO<@<&4+SXDoQyJ7`IM0b1FCyj7vOz08 z5mrfAY`t}3Q;ujO_QQMHZy}-~Nj;YE0c~<2%7WW+ds3aOZ15@A74P#Lc`fa2k;uDBv3pSJGTEQ95tneu7AHTdUB8)^@C1>keq?P1firZ zrt%4b7UMC|5{c&Zupl8te9wmLtf@b&M_Tidsy|#{6?*L*IB8tq1@B3(#0@5I?B9~- zVRKdBnX0!ij4Igfx%S{*te(=J)6cfgT}!!s*2uspwK2x7c`~W#V$})~_l@&@dXI}Y1uKoK{Y#9@geegjeGbw#**0F~6 zpF?*$@uM`{JB%FvP(w@#6`bhp^+i#b-CovsBFuMnB}{+mVDg>-jmeM;X?HTK61OgY ziep~cetIV(e=sKCZ8gWM;xM0ev@zNBrNnq&?X_Cj<+Lge?BG>qL52WPUr~kPlmQ@O zc$@ivG08=INu`-XVp~D!swAHkcB0EfxlYVSLPp`)`Zui&t|9GE-uIsCZ~en%S!-d5 zCfRkoASsOXXfZ9Jy(4Ylk zJRjcRo^_|{TDK|dsWXMx3gi5$cTpah_oVLJ+iy2;ElC|LH3KFY(zh#p=FVijk@7xs z<+jI?G}ori?6Wl4Yu+t;>zgt9uW3%-RHh*=H%%*yPO1X;E*iClK5EleO+bkd>=((w zTmzhUT}a4gJnZdO?_&8e_x>HRC`wvx%2C2*ik7_qa1)6d9r=s@SewU~ygKmjyMo(I z1z&qCYgg%->r2jx2Zo6*8e&({T5c24^%N_MkHmBI&s}>!&YI~|e8f%PVJp3A|61^G z+KJmt_lnorDS7+L`fRUF7YO1(>JSNhXU}g|QtspqeaY>BB4PO3#B$pcKSY*b)=dez zi2v=^qTF~K{-P^Jla6U=h+>ApdBo(?ns(IySsgcp6?z;ST$OUMSpM_4|eL zd{Y@T3Hem_&~P73e(tKbxWly)AD(8gzuX}rsw(_J+oADMZ-|-CN#)NIM{Kzbx!qn$ zVDfQpr{k@x&+!jyhf`tmEpVPUDO8Om%cwt+4rCvGRa8aJ5Fb!hn&Yx{=Mtx$*<`<7^REDqAN; zZVeUueDawYQR{#F43}Gdx>!NG#)Wt*_9_^?uyzeygE_~U9Am;&VoZnAUQ&{-Yd#t+ z{IY@NBAj)=c0Ie$aJt*&tZzSa66sTK3KtH1&9A>7KYNA@lA9r>2To?VCszNo`FW-< z=($G##3vMub{Si3K=O#WnJ|%%gPO-jsl0Ka#?alKi)&~NJnxKUx;pBsoMZ={kw}?O zro!mc?|vWrky~s^Z;tu*u`}RDiPkE6m(PcKrW5<`E%`nUxrEzd#C9ZpQ_eY4+av`6@x@7hgW`+M&puKiC!>mK@snIWHpWhUz zl>`S0X;JibY{>&?|72?N9c8A|1p5f^`26y7`gE;+^kivbq_CXCkBZwLQ*z~ksX7JZk3%amw&(}lIDasN{6^b`$QDPE1Uf4ZP>E#|9#9lZ7O z*5sXm%PbFXYp7#f9&$$35HzRJ;Sc~wVFLMxCQ6)J{qy;Hva)oLYQ?L6T#l5{)`5|c zzhMWze%o%%oe4Yk?~Lin99IB`jH{e!UFnA&w881Bh|)3gk65gL!VCWR{k4p+f41e{ zY(AHq-F!bFj|d7FS~vZ8=}S}7Lm2d$=Cl4N8gd1a+XQe#5U*qGbUPc`$C7h(Kd+%h zt#oSkzK2~c-jY|3Q3^@qpRrf zQ99VI+?L9Zk|cdLBo3Q21{aH;omo5^jz&nxT@AQ?%17w*ub1DPA`_Ua&_{&eZr*cq zON!oyhroi1!Nop{s(XzN7gzq%j1L|;2d}53t0R$5%4Dyii^=+;vKhPjJOO?`ja>M-yc-2NaPp#SB` zjyx=wIAGvdhNtVk2Tx`Z90ZvYh>rZDN|*7ly9{|pTAk$GF!|9cC^a`JDel2{k}}5! zSM_xfW=r`GP8tclipoZ*7k_9#Tt^q5_PL=?@w48vdMza0Kr{kfn-#ySHzYkUF2H%@ z2AfX~WtVqPm>|a$`=9is(Q}R3E)nw=k68)RieJz`+O|`7>pQ>(Z$d66SVu~Q&|r^5 z6(OR-0gDW$%ni(`1fltUY7e+?@ZlA|#;4ckmW$O$tphuv=P$D_3uItViN?j0*cQuY z<76=Lr<3c96EJT;xu=h*BvD?P)=kA)VweLkA7-iT`1}aueJI<)h|vYPR_6wD4ihRA--M=nI_V}y8E{&V#@)>4HyhG%v+ zF(OeEm+=6(H={*VQzRZzHKyTW0DxZm(1_0~X};eU=aPJCboj=2%b_K}>4MLLHHdiM zbKWV&FV5dq5aOY`x_!Ma<*xtlr{2Dw-K0nI3|C)a!y39sg2UEGOqSrCbT5i!vl?r> z3AVmPZM9D`KEDI;I-nBcy2EyT=A?aCLB*3w+)Sc5hjZXjf9&mx>DZ`oov6*G6N)*A z^6)DI>i^vhJImVuVE~JWe97q{O^>U)Xkkyyml#O2Nlh|Z((Nu8 z+JBdxBp-i@2n(%|ujR!uoxe47{OTBd3dgg%?Mg#k`xUt`70%Um z3{~V+R)Wcu+)|&R{;bAPYM`B7kq5q@LVWqgH-j?V(pU98ua9%FwZ(BElrIX-y^IzN zW$muB(9LpmZfBEU`il8^!`Npd_M>&{rO((kQvV#?%JUzRg}U@Op4*o6Z391?BZYdNZz^|W+s(m z@18=W-l=Q00jcQxP8{E_2bc5sA#%+VX|%dwJx@nXjC@BHH;FRDe*Gb}!rvHj(d-9L zV|cC{LyPI-)+AGIp7%*v`#dQ1DQYUm2<-Gu7jds#5fBw*tJ-kzZ)5QWG_yAGl-iJ_ z%c-?j#)G_j`0Chrur+mvvAx%wa^G*W-uNHG9%FXnSF%0mVQ_uu&hO>rzVo1-Gtv&S zGhkfCvtB){IOUJZxvQpJUPA+uPx)?P?^Q2RHDqwlN8pH{wF;&mXLpsVp;O3y@OEzB zD;`j*6x?59zT35_{-C4q&2FoZT;(uU&epjN8a^(l>bJg7AahMn#eaWrW9dbwym{Qq zatS+cflvYWvre6Ye-;&5?T}=7Ku^7DLu=77(z|G@BO6s9h;syDKFJUm;5^XhXwy$& zMIkdjyZ8HnG)ejc`urhEfuX%eNAh=ikKshJ#6R9?<`D7;2PuLqEWa3srh zCbO}*M8^rtnLyCLl&#}lH)gWRsw?h~8iK{yyZ6JQ(z>O88HZykoTf2ZbdR!|u5ZL5 zuVREfyr=G1a8x=edytSEb$xD_#V)+CM@k~d-fNeH2_{-bMOkx@75Z8jM?T_Zg5oem z_)|5Q*1GfiPKifS4I+Xyy{=D|g;Oy8X;cenoXp#rvpNfD0ldWLC*o&(No9C`Adh*B+9Y4{_$F@jqI1;y;{z%- zAJBI<+=s_P>?q7HW>Y(IL(bh^#%>1>{9<@24!?j)U|P~P!sLtjR5Mu(8qwj3Ca0&N zw7PaPO%lN4j6~#Dw$j=sLwkW3D5JLKa#ZWL9jk-@t_$BZ4t1gtN8P+$jE%(R)f#Z% zS%s|KtlxPD+2x)8Q2)kQ_s=={vyb!I1>xCzB&1^Qqil7QvDT1EF=%JZG~q1pejSs6#kZ_q=AP-v8;mgji zbfz(2|60%sqjQ-9EvU=*Sv~snbvwC^OuZ%b(svYU{V!0LkOU4Cx`7nCLwn@wNaKxX zMHDDkWaMl4Q?VDPyDf!C$n~hJHye+%;aEw?JRwtaBSu+ge}4Kj?G~q{sKcivBStV+ z;<3%ShL-(QlKc8HflBz=Gm0TxXp%d8_*&n_uhTC7ZZA4eB*|0_gei~Jpg!H2?-yrUm0$!H1r~M%Ig9zcQXW9Ag`m}BzPwT)(ie025O~?#tMYJ-cnsV=JL(B(s zB-O=r7r4L467Rssl*Z`a7wNq=px)^CmP`ebwT`L$uNmV3UN7hd-1z*lrlzJPW~fS( zz%Y%Dsv(HXL9#BWK2Bi4@LnSIh$_m0!?I%XAI_zJC$^#SV(JJeLsME)%x~cg=T^qH zQwx|jxleuGPnM}&_%QU>GkI3bOH-l|nz7tSp=REP^Jp1}Jy%Cf%I*P%asMWq2Tfq0 zU{V4x2Fpg#L<)k9vu=y9b96@6@KE%6g6Y;O8qD!s+#-G)PSq`S!!$e+}&D9p)_<7F_!30FMVHJpIvtN4KUy~7}pL>Xs&^H*#4 z?Y-=EMH2EHgX+o@8;_Vr%5f6Xm{U)eK3!!A2H+2{#zt+abAxUeNkIKR|*u+xZQIxCf5gzm2Czk`(hRmaQ z28E`=5;iMCnYJ{a_{6b}{kLP}oEXuYcos8$EY$eF@niPSnRVzb=y^tOL#Xp^aG7n) zbk2&nHeYLw8-wL6-spC7BtGVOe0}T1^xpryWA6zwS>s8il#?gatgGIO4V?o|LN-CJpWoOc6B6#X+cUT6ByKUtL)<4RV6zezvQ z@%)0qyi$s0fW7F2zL7W7^Dq7-$+zdHGh6g<^hILjT58%=CLtZ|KDA+ys#7hdeDl{~ z!*C)pUsmc`mV17?Oh&Ena7!#&!D9@b5Bg%J4ik|-eR{M~dIXrQ#-lpmY9Gw9hB-A~ z=8%RsHQ(hR(Xv~dP{Llx`M+>@CK-V7)cRT5fSj6DHuS z)-LMB09!YK7Fx$bKvAylpRZJlib=U5XjIX;H0h?OV$WzHN=JIi*R)}nWFR*6z$01leLYp-66mi;o%2<4NeJ&JyqAFAtNxIlPHi$nA>_jAm{-w}bs8iRkAVAMxldH@)#51^>?7|k-<(E^-$m-vrx9nJGqNnJa}6FU20QIv?#dIgLvjbx+T#&eK!75$ z_D{Klywc2NQv0-im}mk+9)C*bCYZeR|LeY~W!PJKM<3(SlErL66}IfD13xt_?-zeT z^rR^N)U~dhnF+ZvkZ>ztPQ)+IbPa9gH=PuK7Fd+cyB!Y#ZL3S%>&n9iu(fh2^o9J1 zd2DZjl_}7XNnl0HYe^+29s_Pzju#9!Ck)MNaKNh#K~2?93DCZ9>i7Jiqw}mscNVqd zYdn5L;1-8o?>+RG^T|54yOe}kh~IhaQ+w9UW_%6vl4Z!S&42yYhweMHQ+hn%^2j%TL*(waNUXBxT_m24;Ze9ZV?8_3k{ZwIJqMwg1DwjFQX5C~ z7ZUjJ(JDoY0idSdXTNf5GTlKJJi%RL5POShOlyG*nBNh3=e|7Va`J63_W|0`GP1r0 zjXZ`xH-tEcAN5-d# z@{cQk^Hyfm*xw|AAN8u;*>L2?`#VezePDRS^+!$ZHTNcVlUUC<(Az!mECO1%8?ET_ zsmU5ltyH;0%NqI%oNM@tsR+S)>D78+u!P@>RnJ4aFrC9ON0oBh(q^IG$`>L^@2O+m-v2DOs-cCMZ^C@5CDV z$Z^u~dyO*}H5Ly3TNBON<-jLQiKna+DRJJWzus-bOXJuZ*sU-b)^e?n*oWgg) zdD3i0P8COjQ#Ty9`l{ta_1b=ey}OC$Uck)nGt3u{v9XT-?fXh-%=O%2&yEY**=0XZ zXA|peWIKJlH<)zoBRDY zL*E}Q;SQ!3Rc#6Gg>i0%%-{cw zlZ9BSoqJ;!etxMCd4u`?S^!7k$O(n1DEo~cLyl7lJ<<1PK6sp@ay9%1wYNQEKkrGi zpE-Wc`d;htjH&NcZ!%s1d!xUO3qQAJ+mmdAm)du2gI6DtIVI^yVZM`W)10e8Cqn1D z>rwwE^>u{h;9?q0N3oEw<;n$tqB*6-zojZ*QSS*iNuNoqonvjjiGP@u=kll4yFl#fg9H_{!jRFrROtH>c+*3%hhwJWmx-cFLW_z8rR`ev@%uve(aRo zKxAfuy2yGfDv6Q%Awkq7qE1VT!uRryorTh!4~b`2lww<+IEr;7cHUD8oILjB5>VVxNRdCw`4%C}W>u0TXQ^`R$j((Y#I5^4Y)nD*GgyR)=DRSi+va z31Q^C+y<=rU~6EoYq`KIL2& zihv}$w@5ysfO3oz-OAK}6Q((T@|dicv*rRQdj9~C8na`rupDCztVJuI zg3_?QmZzr0xbuy1(t)9wf}g-PBbmP?W$^drd|&?Moq-)#;^2s3Fn0n*MK{XgD%4W~ zMPoXX2t=g7m>b=pE%LDxbMnMMAqh!zdmcdW?mFIgIQGOkFwCWU>2yT z;^iZ0?H`uHtHarjipkv(A&4}$mz|h~;@=6=3kB~Y^{ioUVaA&F$0I08BkbOlx2HMU z_`*-GRL|;!qgkwlYc#wV7#dj#nmOQ|!fGvIN8wT7Xt?$GTsRVkANM38>m@K1I7Whq z3k3yj-Eq5HzY)k~z!cp7=^Rf=Kn+1ptZF4+VPy*CX?)BEg!s8XfoNA#FcJ>~fNMo4(~gyPwpd&13i8!V$C&Qr`MT))~8D5~#2&q_jW7==6`5$1HVA zkeYZGy2C-RrZw|sOD{h0iSl_lf@TL$VzA>i`e-o`XhApb3gKxjP2@58VGRva%|3^Z z9Fb_>^hDc2B61dp)!1wM`n0Ohsnn(oc2;pB<(-QrBYP~+#<15EM^l@WQ29V2pdl*J z4hD=V0QV%nW7qUX*F8Z`La!gu5V8rf_@Y?|Y%Cn!`mEkZpY&M>^x#od8rC&6*BUdD z?M^@De^8glHsq(rSOE}?hE-oScruHFeTuNkSE{@(@us60_(@yf#)VUDdtwT;D@hw+ zlQRruh)%Bro~Oa5UFj{S38;C9*}5sC1+7EJNE~tvjzihHx+ZO+1p6i&qjy9qnq6ge zf`#)K-o=Z77QD|*U|zGYp;POCGR1@fvtf)Zgd(eQ9<87q%%YnR z8WeqUJ#RQ_m+c>MUd=zu-WS@{=)eO{Bnk#5iveiBi%g=p7qGVCO~FSSIY7V9tKCF{ zS}7*LbSL^7qvFBSP#;un-x``q+`zJWn{X2VI4-(NsiMU|1O{5W8K{I3Z{pZ(;)OxO ziqn$P1W&=eh()7li2g&!r^}EJKLC2p79L@LdxSb6VhyPC#A@sGUC7unvM>qtkh=9? z9m`7acI#p}r!>AJ_W0i>y}#A1M10^m5ow?K(9~qkmrFu!v;kM!hh1?morj19ZM$6a zGQhGYpe{z>KhNYhP3DFBl$Wuz_G`)1Ou-V#NR?th;aAbXF45xQw`LC!iAP$o7vNCs z_{macO{5l@`7GCK=1!mCI4Ud{5If2BTikm$V<+^>Nlc2r$#IustMiR&FR$sUs2*8E ze-i_(ypyY>t+Q54=lY91a1nD*q(b1mO;&keqPE%?2YSmEa2_2au;)JElV9sf1L8$fkda_gLPIR( z7nUJ@Boy`FtrO1i5T>(r11?Z}13ee7B+$iE27mg}~6=DcrV1F_P zRYi#u_7nl-sBy!0iP<7+kTQ@HUdWLbb(rbB$?Lz{h=G`>RH|+c6^1Y^h+=t(VW795 z=4(h4m^UDrP4qDvlTe>r{wKVwi-@G&E5eSB;_thyM#LatRriRK9HZ2H$zoOyke*C# z1bzE_oM4g#gj~VD_dk%84<|4inm;_-jfh|)cfArKoW1!GzeS`xtCJ`Me~TkoG|R~2 z+COmsS&sG$+)>_NL+hm9$fOD%Hw`i<;vgYGUyX+I0L2s(7HjW#Mz29&$9VOKtxsS> zCl^6PH39Y^c@O~@VL^FQFZj?Jx-9BP$hc2ho#V(^pwL}C2*`fAV9xWN)3FoJtY=sd zl~7yV)DXi`xq zT1piKkgnI=uGS+LeSVs^)xQuT_i^YYQ8oH`DhLd2B0M$ExfA3*!UoV*bS*rUS0!x& zE0ek9W(?rTqmTITiu=B7K3chf`7cI+ZM(u`Zxf@E{EEu{DBNm9k^o#{QJYX7p6@77 z^L6_~ni;-|vKnrPspyh!hk&b-^@Rtwl`mXsu zi;linF7gKr6r&ZNN{C2(rVX7zj+A@Kk1jhK8!@iLFzYZqnncL+@Z+&R$fhnCf8HhM zGI!#`Gm6wjYU~7fenBBEjbf>41kFNpouY#}E+c9*)e6t3it0Mc=1Ad(#j>0*K>!p8 zS3KSZoc**jUvkxMFAI74)eMdq5U@L+xIwe3GG+?$)c%DzPY?-yg!0v-wYv^-uCAeL z)lY9nD+26-@J|G;nw4U5w1=~vH-&{n$+t12P+=}6JqHWUTU|W!Y$69U~y1WFJ6ZrGpA6Yrpom zwXmDWI+pRR@ECmjQGsO-jBs|-mVOb0qr)w{*Cl--(y?OKt~jBrIO_+b5-LI{V}&uD4{jm|3~Iwu`yXkz1yuJ} z;1%#Y{vSg5eZ)vuop2Lwg(qX$=f(hFn<-x&?Q)`20Orgbve!ngM3vTHy>Mdr5(Cmi zq-YoD?5r*vRcfH&lBpVxCqSsev)37?q8%Pb(_-QPPBq%P>9$V^u<*^{eIu5NsSUpp zKMRRzNm>--BPNmQ)MEG4@*8*mF9Q(zPO5tbFCJP4&;l3|@HHo_+0|<&ze*BP5<-wG zk`6$^EAJe9s6`D$DWKSf#iEA-PmbiAK0x>BxoGb8xH(?ys)-_^`Rz9<=L#AAywPsJWu_eS|!M-LmH&MS1xspOZ=F^}sP|p48*@91zoKMH$)?U33JXmB_eko+ zoehXiJmX$|puaJbQ-*>V;Vi>lWW}xrqES5sr!|go3d3BbPY=i0Uv+b`@0knDb=V0XYw{hgRyolkr zg%GJ*U=KOo#|-1H%m~H!9c*kL&7r3&c|$l9<0*3RhuWF>s4XN3lwRSY$ zGx=k!y^$UnoaWf|Y~5h>VuoGx#yo^s$3_=2J5riL-oKZ0)e|$8?H8PdI?S`yBmteY z(Z)P<>KzMRYm^E&eXI~*E`cpWKzF#A5HGQ#n9O)HL5aX{K5X)7YanxrU%FzqVcv=W5omKxHh}jvw!$#rPWK#anZD zCrG6CePf9{lOM%m8q>6(St4FbFj;oD{TdicwIfm5a_?I!`nivssO?&OP%{t74Q5c| ze;;&bFD{PwF#{!@D6Q%u%M|llnSQ`q9AKvx$jQsHd)a>NMI za)sAa%i@Z(&0vT}4H>aw;qH)ta3G?RmgjQdeEWreSEqSoNbP{S) z=-7l@|8+Yp$Ad8)6c9okEVN8mH``igQ@!ROAmXt)z<$`TR?LfW!;IxI!;v#a(Ph@* zh4G-&y{GI!nc&BVKW858!KkJr?2(Descq&wBYC-Sysax6ujjdV;43y zbs^L{(-!YEs$CdH&tTQx?Z5verKU;AU~O;T<0Ak>jf+ItT7Lr?xiTP_!WkPCX@y|| zx*HvVJb*n^#gR4iHuES|J`mHnKo$2wtxICQkzIDtpcA@i)aE1UKtys;!|j&xKdeb! zzez6y?R!)h2993|f{6NgJBoF-4^=^FunvjM(ov#!=-O~{lxRq;jGRD+9;shF&x&O zM(eGjfsUi(5Bk)*oVNc?tPmu2bLo9I;>1(VgU`%|_ai*vMkaS=A;X40J>T>Gav@{1 zD!)I;$RnzCafS%?d|E+orp_{3JY=+Y-G>zpg|IZ8WC><#B}_z}>~#du!Rv}(=vJhx z^W9x^<+9I-&b$s+#V*Twb<_ao*){Ynx|Uo*zBRqN9Vty_V9=Q=dFY?nTpiQ_Vk2)5 za8J4%LmCHufDhhIL%OM)2szp&G|?+zN~ z+jpHey3Md1Fh6l^Q`N>tPfYb)4y;e0IWIm25^E#=sAo0YD8Fox_wXDI)L5fz-2XZL zpFg`S9lmHC^vvSxTw)g<=5ZFG#=WEY1Y)%|on;!@lmuH#gkE@LJm1ycP%#(hhuGG< zKMl>Ntw>I}3I!$1@@%c?LYn7*> zsPb(wPM5nD$Uv!o1Y)+6A|anf@wc*-UNI5__X}l(AO|#Gb;~)Yb-LS z+0BRzFRanXhJfUlcg}2aIqA?NE7~t5sjarXOj6rPvpT7qHScHR{k6B@NQ!G42vi7v{zXLSv0}iiD}poR$8@&0BbdLs?G*x$J7KX+_W1aQFFebDZka7LP zFt;ZnC){=#=*FW1F9ODO6Gq{=m`Nx)J06C%1jiq^XAI_rq=~7hhv8lfsz;6%8V4L! zVdF32ZWJD#WGbO8JASWHNtZ-PAR?EIupjZdswph{WL8M(~kKUjy>AGF;V zM%MGJ_cTY6O-~iA9lR0E`-7Yj()UNO-5&2>&8F(&fji0y8f(Gskz!AxT!CP#GCi}n zOYNOy`ZZt@g2-rT)OIu^4y_bH~S?g{>(Y)E_bj<7A4q+m)+{VyJ+3lnz z5QxZhS~Hs%PB9HFkfrE_+B=*K^uql;rZ~1RuU~l#mPKuN(_+cgOm!6T4lO1e%?I=h+wtas?ADh>Qg4w?z~z!@Yk#iThHm$pE6ox&XE~D&ru!ugkMbDh z{x@yB(=3_4!YQd*tG2&Uk?qN!6nkuNy}DQRuW)Yf8F_IF-5mS36K3ELWl7%3SXU)% z>w&xZ1l@9;G_jTD`+Ap*9<-aaKZa}FY9?Xi_MO^Wg3%fE2?^IKs{UW(Q zl14vUrhE_6gfVtQqrCS(KTi8Oh{w<)oVx zqx)T#@lE9lA#6TkNr$#umwv3+F{`4iI99(>nSYR3xliC^aVe$iSTMXc$e9JrLOp1f z?`a)?4@$?>8q?aZD|$xI+K+z^LfT%f&N)$IaoW_FEm)dO)}1)z!* zKLgP|20+<;BC?`vQgHaPM7kx19w)x(hF{+5$A=(ML@1?*v~hAV-WQhGHcz`(n)44a zU~H6PCeXEUxnN48(jsjyu8ARKgB`!u(!z2SPb++YsXY&??0kWK^?(ZV>)ldIDi>JF z=EGY&u`gV)YszK@Yxh5_l&p?2)gG#?etu0;-6>%FtoF8yc9@ewO?lN>DbEO%V%|GZ zi?WIYZ`NC1KRpy8KObdLiHhh+%6AnZGX>34jXtc(~EgF=}#6d;Tm`THU2bLkLr{UOM*%^G`_2 z5kwGjMIlK0>FM03Oa*12fH!Ho$z^$()%V`1<(5qr)u4vUYjdZhZe30Qj8zw37h7o@ zwYbEl=XxMWCdFUhj;HE=brcFK>y&tB7HW*})ig@AHJL@HTEk<0{j@`3nXli7u4q4F z_pijl2#hm`zp+Ukv|lHN4=O~ zf=NHWDN1No3vu1Aa9Sk-p!hTu03VWksIhbJ=((sk*;wko--eVemT)(%D~6&m6omke%U2~-9|H3u0^G|ldW_lFMe1vM%S2L_?(7d@p<49%T6c_8tu0n3n2>4h8mBe7ObKT^ zX$*nk`{cT)=!$nahoyL%31^mowo!D^#jyVs;+ZA4mHXwtk5<>TX(l&|5=5MoW7Jaw zdK%*f7Te{B5x7NiO+eZ5!hadxuDEDYgb~603Gt98157kl9c503M=N`%7yGYEwlrqV zs@gOH9f9W}6E}~xmqD8dILho*IqPChpf0bB;R+LW?{^fa%tvtIe?}z7V}-y)h|%$^ zGB#DWz9^*}oq;BgcW%+2SooxaOepBJH+P9yHAQn3U2D_B31bX=AQ*GOW@_Q0ycfeJ zq^7dhJoPld`bu^&v)qGFuk|Qj0eijKldgk}0l&C{-AG9B&Ju3i@Tm{AJ$}hU1caQ| zC6iV3_rIsYb`}+adXe++S3W(9<6-OLy7HYQyWgu*R~`ro{=09%t^Bu{N$BJde{sbr zA1;m)?EZksVI`pc%hTni#TXgQ-kgWJ_xH^BS}mX7UIXJ8VgwPv0})5fA2+jW`$alzqlSKV!!g^q-Wvl|Sf~<1!saKkMaVE7}ovC#L7+@+BWgxm2#o(uM;c zM5cD=G@(_42%|(64v>ag(4Bracda;BRm010?;_C)%w}18>5)(SJ_KE9e$lZd4wL)v z&$n{ack*$$o(;k$Fo~nKF7!-W&vs17Jhh9!!5mV#O8&UkPB;iGz|Lzn%4kYZS zW5?REb!$S9;)F^$UD%x?_@*baJRyI39M-@oI zuLL4`3p}fG%(gXSChqL9zU9>KxqfKz^qPdMG?fkYDv5Ex&Aq0)l)Qd9W8UijYXP!C zC@_Ne%6jIP>SOqv$Iu4W>*|@4^!SD8w@W=!zn@h9Wu{Me30=75pELy-Bg~`=WV}|D z($(Lf?;@+Ew1lJLDUQr+rlX8uyC-+BqLI|B|kh z;h2Yv#|v|Eb-C{;hX4#yjHmhvW?*4>m^VYHu|0%+O=kC>dPwEXR@y}u`&zNTJlS@M z`34W%sH}|!OF26$80^mYaB2n3c-a{K)K4{7ZdqrsLb4sDP$gNAl%g-#9mnOM=64$z`i_0Yu7DXk6^DWMi5Y-#qs+M1Hd(z;M%2#D)aXt|2?R3 zDwPo?Q64*vQZ1N_u0NlJ=n?pODXa|q;pLtUNfD#z6UrNGB$hk@`|Ypo=sfLPmF}u- z=&K`)kB^_+)fB<)~tlSU*XAe+Mo+nB2xOhTrS9D?(D9EXfw z0mH;NWL{$DIyQA4DpCh6%obXE-O!6w4}ZV$0q+-@>xD$%dJr7FTowdhL%ZUzf6#x0 zBC^}d_i61THpwHq8yw+ixAdB)b~+;zDS{>qSFjWXuy)4<(eLgtiMKTPICVmOSqdKy zW&^Y6Up0V113V!NN?%k;d8c!nGtd6qxSX~_sIm77FMiTM&{>Z$_VfFH4|NxHl(6fS zuhX%ZpZMFx^vGgzVc+ppM~Wn~Zt)o_E!5TGP`Z{Qr6VvNy80Tpl%=#8eDoNs0Zr7p zZx9J7I&Db7TyL=fmsNqoxo;J$-x;uey4eTV zqd;>Ud6eh4C_&RgQzrD{XbV*$G6_4Ft;?o%GhFNHNb8GNv)Q`)B*0a7Oq{+gZ@#xk z1~V>of>I<3vgk2e>Gs?l6di&LGQcj9iTmXI#&gVSswmJ`=fX8T(T%6_A}J#3r=mxO zRx9f~y?>}!qZr&Hx4a8(q!7ML$BLXHr!5xEA1%=rF8OSciWnW1b_NZk1 ziI>%Ull(V5oCWMal_G(`Rl(#Fr$uzFyPIX=j^V8vISZYwzgkuZb3CNxE1?*9AHL31 zOllEBq2(C$He!%hWNU0aCqj5$Ne*<<5Z?wa&A5==4U(wg-Ws+}(S2lE#i^%Vnx1-* zetn!D(`oIe>)Hxv;TK08#P5u8wXzRf<=(6uS3s76 z`O-bW9HhOTU~{!rP5H4;P8I?h#FK)~+nma*<u3{L)Gjq zxmkT9BDb1<3J&LpZ~jW~ZeFxU@*>0unuEC3Q7Vz|muufH`_*SBYKaJiBPzwc4lFH9 z%stu6eDheHLcjM%Cn1@K)bQpc@4wzD^M}_e(JN!PcYpb;p~ZjV0e9MtXYI+0&!F;B zO}=Db416>V!OVeaQXoeDKYe;(>lCD7SVwDD`hP^7c|4SR{PvMy7~5Ee$~yKKX-~+| z2r0WPLuDz78e6jO%9s`;N%n{~TNui|rAU+{G??s5_9a`-XU^~UJkOu!bzbL~x#zyW z_xF2!uIqiZ!qOgYCfY2FC`tBPOCX(WI5rkyEeN}DIJ2FO5Ljz)@RB{asC(R+`JA2d zN%mrM5DFoMW$}q2x!t%dW^q$b>wIrswrn|k0hxdBJc02chy5z0y^kvSl$P?uqp11q zT%D$bYPZ2Fv4oHCkIzg}>|Q1nSYe2Qkro`xPi%uMCyk>j8~MN3oh%U{`hD1qKi2BC zBl$RXlvm|LO*Eu^3&|F*gV zktB7=o+p&@?31E&kXyBvVQyU2%&$@BkVriqhNSC)h^o4H+S*DnH670)ERFsx`jlTFbc`}#se z)1jP1s65ozti18sx%$iS-}r8Ju@R2fzGe~Z*wgM;xt?60+*^pew%_3X`hARSS@`Z{ zj6U-(gbLpm$tBE`LHjtA!rmGfmKW~bl9NDAaMbTK=gtyYKXb_C$RE5iCIVy1;%ErI z4OA5C4cl7`?CN<7bs1j;I4ZR*W3gf%79Jfr3I=dL%AvTW^d`&*yAZ9oxjJ8d@rdXv zrYgj~*gE!*Ni0dby;&{h8JW_d(tAN-Ir3e5N!#vI#xlEUEHpPJt|t$o`%gdu0<@xlfb5_2TQc8FtB#BvF<#TAg3v?X&8q@p8djds1+J%qdRMO@t+L{6=I=eVWT zmg$0ri@>spgG{SDixO5!Gk&!!aN*HPIPMe2YXF@17-_KVFM4y7)P>;4RLCB|g;DW& z*kf#dl=wOAz|F!8k3DVoElQi|dloPoE&|)m?<_W~CGgG@Vm)@UA);;uGP>@}yy(8s zDuPPv;izEBI1;6BIL!Ms1sAp3=*_te&VqPX9Qcm7IQ@i{I2$;vJ_d}`6P|wHFZjt_ zF4>qNKdV@K#q)oC91aEI$7*^#K&D2nA!i#$EP85mIs7TCTV&)W1}I$aso%oSOZEYz zQ-8oQ3C0*bHTM+qvT;4-(qB5gQxge&XbZfgW_$-&gowvoSkViSOAi=e1U0Zjl9CRzc z@h*AEs*+CaQc+KU8?KvPl2)5_e}xjW78ot z`tceXlU(ZFJP4&xVWKVKlZ8-umwEnQUr}}|0h|sD?~2rG+f4qKodR$**5k{onp=%C z>(h<852zh-@RT8$o(jdzuv^l)7GT%xEBbRJn3XL}!9C=nqnc7%Q83DHUmZ^xr!mJ{ zo5P`jMa^;YMVA&phTH!8susT_gy96AUDbzY#u_i4vxRB%Cu6+Nv z%lQmtnlEa^&g<1D_tk-ssp&D5$^V*P=)rQ4n+T##y1m@S1Ej?175b2Khmmw-yIs2^dD5Ov(Az=JQ#Om(${|q$N1)fyfEV`niJSFpV@=0%89BJjs_5U&Vwdr# z?TdG|$Y)vadNOb%RI+%F5osthv73ZVrqz-jR?$3PwoDi0GWlZRD-B*CJun!u7g zOI_UG-f}Va>RZ0E{P9$9Ep_YYdPqq~U;TwS>vFHg>#*9k^bYB;-$=cQgv7LmKpTiX zn5)bW%Y}(trmrv-lS>b;cp(HvrO;kEcB6=BssUeL*6;+nHs~t27dBxzlJrD*>|>ET zq$;Z02bM((*FmgWDE4>&&Ed1TYXyMZN7XfVH*+3*r-h7xcO|#G^t6Zmg#w;+RPWN50TmmJ$r}% zV+emBSxD~w%kg??m=W(gTKnrj7`ycCDoAWvOc=AC3u#k5eJ4x{4a|!jLZ^y+aykAt zMnTKwI2YPtRw0B*rRO8>zHC%~254R-<<-A;wyi5Y*zB`n=ez371*FQae7@lNW?W zXSN?Y&vwo(Y(@Dak;$9+LeDf<1dGC+^6?aT1$B*w>`Vz1rgIr+Ns_`tOfmvj$nqgI z2lc~SPgkud%>8_%6-p<2P^LP+6g0^hpR^jbc{%m{2F$_9RfsCBC+c_oRJ{`0LD6vc zQ;nf!S~~jSY-9iS)qofyBNmF4`Ba%ABhjHqb6nM1Um9z@`Qu)HjJR&BeW6wXdO$d#qjHpsaOG&p*3okEQ+?RQmpKIVq~ZuD*^ zfRniHy;rsDnm=j`#~+xur(6nvt(^w6?7I@(`kLsk+DClVUaiTq-2VhpLrI%~%BPJv zram~An~Y@7)2v}Up<XPo zD^cGQb@`WX8k(A4SfyLC6fEF~@)3-Ys;NTskNV^#+C7Kq>K!pxRJTASUj{=-5fII4 z=@IDiRr|=jPl0lm`{$_WG0bIUN(J`djR7+|aYf7nCO3HJU$M^x2S_k#KdFo6&-6(D zK&K5hcD=9D8Y2a2r)uicDJZyYM-f^(O>QOT#PGhSzCjt$Cn@-D2`4~W3{f5z*e zM1Rlf`C^(OHVnt+u8H9Rd&I8HqbMTiF z1*V8@mF195BR8yDkMhs{Xb!9>s{@K}lxvy{+#3NoQD%fi4z4ss|IG-_gnLDfiy7v( zT;BgKP-a^QJF41X|iv+e%R~3X|?fd zvc6g=(w{PP(DN{fxl}g@CiOIyb1u<-C@-_qO7LoL`9%(pX!ed$%3dL~)-~H&@2-@< z=t>6pItyRsJB%(xrxBx!= zn^qg6YU`ike%8814PqydK`F-3Hdmg~2!n6bvm!T1k>ft$fKYj^Sm!%;4Bk zT|gEEKf*lpS9|5JfYj44|8X2`UlU9*&@ixx3NeYVP{5HKP$~z?71X}}(D*iw zf!0wY!Bn%7|4!ABhRvf}nQWmqI5g2j_LXaK6RAN*er^`TQ29>egOqqSr151d1YvS* zjLdW%F&y>D)G&xX?PV>&-@*>PY(J1Do24u7nt9<{2sf(*2G$3D;@Q*+XZkPi76BJ< zY2dj`gtEendbCIAr|<2HdXAjDlmA~`&)J*$YCRI0nTn1mznDL}oU{BGCN4Eq{h=^$ zM^=vWIBA7{w)GgS31-k*i=gWQKL>`?IDp@Q9kk!iP5Jq%ygYFRBrg0bvhe|iVO+OCM*_9#rt&qH0Xsd zG(`r#|6Ld6{7l}j!kk1T&9SNl!{X#>VH!)#&(cDdavt(UF0|T!MksYJn=}E zX}6%=CS1vQCZzrgjz%Jw;WVms*h1gfH$zJN#<7O|1@-F!2JO64?tD}x=L5vQ{PmXT zGAtN+i?9(p1dyZ75k*3=vn)i(61=;VP8?Oz;{dxFii=gHJwgArAH_8k+j^qXDdkz> znNq@65ThR}Z3rm34Py0~7E|=kNM?Py@(QLzGv4>-yq9YhF7E5!T3tnb{qNpmIGD4T z7BNTPf!R*d0kzKJ(^XJ}IO7HXUvdf+4>lnO3J<(M+{=|lW11pWa zU~SOSq3IkS)dqHh&ZANWfKQHTuP9NYf?wy+H$S;oyJ=Wh;#xHv=4qtVsOa zNwUrrjmdak48p}*6eGA;Uz3M+MSVe^JS~`FOLJ6(?>4>Xzo}gitM@k7*odY1)IqRZ z2C|#pQlnuj&Wc_ibc7?VJ4k`QaY@;A(Fs)RGtb|GxwqZxT>#cCHyC9r*kyG_rJ>k-1}dj%HS!F!O{bV^ zGq#m{)VTCdH<*CH+F&N2Eeke{i6dyZ*v&X)O*8^mT7md0&{c{c6CpVbfT4kCIOi|O zM|3)N<>P_BXU#-xfA1)M>8}67F^6P=gPPhQraU(3x@YwO5hO5@Pj&Agwu^9p0ZtFG zi*bT=WJ;G>md`#}pOaB@1ORprT7*ih#d-pK|N$0%eef{$m ztYTeN(LYq=r?PbL+ut%eL?R`2U)d*S!AP`Ui4R6ewO=*#I-?8vF!r1s(gU5uaE3W( zH|a32tXI^^c%~A_wzP(?+K;ZN9VzehRr_9o#B!nUD$9#>#C{~oz#i`Q7xmYF^#`oh zDy$dpO;TC2vSXc_%eYSR_sE{j`&c-0gaOLJo{=e2$FVNGg#Gx zU?VukUUrxyHRA=AzKwlPg@$l~*RyV#)Q=tGggX~inkKtK*mIYP_v}yw*Z-f(-AcCn zR}H%|TJh9Q0VzLlnMXbup0@e{b~KQBT$&d#v=_2kjGNerJ9F;}VLn&nBeAImU=i6~ zTKPjPk6wFahf~a|;I`tstyaHyEF7qUQQY2Gw=8#;EPl}JqK=^r?DRc6sd6gHzisX{ z5J!UEQ0In><}~f63CotJS8*Qay5v7yRSgjMtNlg-sXy5YE9tJvF&?-7H{*;b)P;zy zsBL0ZbEm~JfhEL5)^qp=0|qf{Olb3B1M3@2=PH@?Go&lfs%=m1R>94|GL0>Rh)mgq z4>BEQ9ZUYv+Hz$=3%Luqbd*Ys|>6Thb5;sPb^iE1d2p9tIDU3MCDWN;Zo`8`nijk zt^h3T67J&m5&1pmwsXv6NZNG~jQ51TUtghtH{(4K!uBxW_MU1n*c~FVq^|8w-o zjpp3S0kGt}&@s(95sHl*!6}EMM$eAG8PaEx^qisWHZpt`KQfRnGTEAOb-Q13 zMs^BR!X=_afeSP;B{>*XYu`a!Du~>`4U@JNVpb8O{>K!)v@`q=*esLikaEL6#e(9NwdnAM;eMr+HP1NK80wagWhmg zIHf#He636_<)G=*c?*tl6@%F`zIQp}+Ns;r{IfKc+gkF*q8G0?Gqz{%<#-_SZOmFm zpnv-Bc;A4rT*Q3M&=(|ifr}0(Hd3w*(6X)Kk{f`%IgG=M*s+%tx)lbh|B%hyYhjX` z9s?YnpISwm+@;NrrX>gX4}0_v)qFBb8=vm-djuyK?I+ayHt)x|kuc@ZlRi|XYoG}r zyy(l1zai~1JyBd~>-!^%8R4GN>{d~f9g?2Chlagh1;Z1V3f3?`-h9R&*(kOM>+{Y3W+ryX2rFDVyRj zd>v)KIpctIx~GM{KA2r9VR7}bKpZ12RH1l|ORp>{7hf6}RXgPs;6$_-z(ry-KQd%a($1)0Xu_K=S?y28 zhkJY1eqf{KYHR+y%T2069JhIV4f`0uZ6{*Wu^C=i*SR|rCzn3>cBMk~@Ahbq07R9q zkKp+1^bJsbu0j@x)LwICL1UDAyih3iZ<1+Y5ST69nO`68r{>FY>5p3f&#!)ZFu2g` z3?nZetxkiXRzzEq`5mL`Q6G|abov_4Ge*Ge`(K8KbTDI9_zgB?!hb_F4Aq25_GJ>w z;ta)pO@jk*7OP64hI|C2OyAa(cN)ksZrD6jvy-c)TSvGKa}m&MV*ntsgb5syktvTu zJ_`uNK3Zhln2?lnFoL;Zd`a%SC0t1H@;Xnx9YU175v&cD7mX{sAl6*freV3xdem*f zJkSUlV%d+0Gr}k~Yln^2_6edc@*rH{M!&p8-ii@#O9Y3J6U!>@O5w?k^lN7B#vot! z(gRD|A#$=9d2I<3rj&RJK;>~YpCYPsu_-b6D=>K;J>39J*8aOq% zGSr_}>c8*2@GUB zIUo<*s~w&-37T81gqUjueEHqp>uu>0njt1+YO-43%&^M)&pN|HGm@m?HTi=|JS30= z@dMelz2ELvH8-7$PWLX7%O+F$7BG(hg$0HcqqfPwH-&Cb*Fsd{laG>^>*x^@w&0F$ zP?dZ%1~Aiwd5`mC#zHbHy7f^P^p$Gl{+8q5AaW~qnd?I7@k;#L&x%}y?M&i$)ogkH z^O&%m?{gJSY=5mU{y?F`sJcrSz4x)577N|9XA)`Z0ULg#h4m@xD%fDOPcd1dIWeU0 z28jOMt3tR^L=^;Am9keO<)BO_;ky9=UnW**w=4 zdtK3y!dy451E?>=7V*E&F3OtIeMzVaF-`W9e^e#QP~?w)#f;8+*#=-AOPhCjFDDV7 zn^go%U1b%v7w&oZS{F=hha4yeOcybF0b%8Sb02<&DYu;QInZG%z|lsu6Pk~__LFJ? z=P6X3pvZ|`Rn2Ir8^u@d2~zdpfqW`q5q6vAaDBDu*CQ}_#^B<%-t%fVBcOR==MbM* zeY2~B25QE|Cc)5MT}d|1o;nJ{(vu|SJT-dq{3`Aiw%E9^(o zkzS$m%rWTOaHcXH^||=%+V^?P-=6P_9SUk^p}bklCpnePR$@bYXC&e=wyW}MW~RUV zeen|vPhNg-pVD#EM!Lod=In{%LW(F(Fax7Z^5=iA9VIzzk=sMwA%`6I2iOP?u~_N0 zaL^-s;9;uxJ#g`ZB6l~4d?UDqq3p)R$d`4|XsY$ICWZ$sS`D{}apOr1^{n!dg=JxA z=2QUIFtbkn8-eNH6lLhG`%ikI#CpJ!MVW|xW1blzRx@^>py3$%g{_tad9O_h`vp6s z!$w22H0Hkb$>0qnx@Pl_x|?YZqA?eTlFa-}!VCZRdz}m1SKA5f1f}5DvIg}p4OPh(t{+aWiJN#dQVoT_UC5`w7x(|eZ z&u2swk&2fmzS)~RpKg1mZx<%f9j6tqT53JLE>Jn9Pa^Htzv)xYrc>OxYtW@EMZZpE z{MelgxxWtz_8U_luufwNf+^lE_@hDv`D^WZ-=2eb_;b?SPf9b;SCq-Z^G803SMpD`x&+}oIdE6e#pMNX!wHz!WKJO50+VI&x_1;u;^ryU(-`r#MI)Y(6U#wJl($Ux* z11^a9`+LG}bPGtiGLu@os+w_4*7SkJp% zLp|t^6T|o&);G=(WCY7-2ZicS&6yN^=BR+xYJ=bA0X;1+hFUioU;QweXX>ST&u_s1 zu!5x>}&aKXSIHN8l7OwTg+KB zsr}J(K(w}hMZRRD;zUAk`l+!7TLIAd{GQ0oJi+(U1=LsCOrPD3KYdU6e9(~Z5Uxhk z*eB*?-&-p@<#Jv2#C&ce(b^EZe&xMIO>ZmjwUhpoYe|GEYABZ6$NQ2FFM{lp%4*t8 zI2_lpt2j{z;?G*HL;v-ynW6f8g0ZuYaC`X02$}JK&rk?;oSCdCPNVCks(yLHLZi>1 zWi&YT?n~UMZA@Uqz~62<+?cgLN5}9JJ5yjgU-$IGi??}S=v)^S(xn?yr9euMY3G1y z3>AFJLqCdJLFE8O-4o6;O*7iR$2Jb1TMq0syLq{|)7TBm{;sG! zT};A#(4W((nb+08d4CJ9cz?*cNZrH6lm4jfx8#r`F!n^uKUK__EBmD>w6|lCw#6Y! zpMgXgs}3(?M{Y{$=9^ppZ(5^0;BaywSclj6(BZe2+c1!2<&ilUu&_da@_4<4VFnz-itDnZp2itjC$5#2KfiOYX8LwHJ*Q6K^HX>vCn{BfG8zwb;j9?H6Fu4{k$Rc zdCVZl$90{fX6!c&4;TWN>(_Cv%*dC~Nv@t~^zC?Z4ga$7k6(@=_&CZHv2H)_kT+*6 zKM*Ag+DG5zs6QT2i-vvd#{4BP^%Peg7{R&L5V$SKI3}ndIDY@zLbOD^A78_h{4_)Yn~13?)t;gXnKuSvmi2ycOWw$W=QZzU z-LClq%+Ie?%0-Of?xrA<1vO8gi<=#epDX<|**ChiCqGta?#JL=i3dA`nkVzz+F$|# zK0L=}?FG4@M>zF6KX&TNm{fRlW|Z+{Q^{)!922p8mBk{N%|+06w>jb)H#)$4POODN4`7()X98;p6+8$fs(; zble}cdjdg{%Mtr6J`+LBI46HxU@|$TsTpl;BjpUI!(n4*j=njCki#Jw)?HFVf{Jk%*?-4Ux&v_tNrm&lL%K& zM32VNt7?v@PJFc1=ID7HXzuRTN$;&*TPF>}CCs>XPk3hY9zF)| zXIoqy)*LO?h9AKkYDX z&qkC~?dRlA8*vJz30Vb2KHkR0c&CS5|91?9CPJ!M+}xP;a<^24Q{Rl@=G!tEIOfg^ z#)A(~WnQ4WoU;?tVsaNRGA-3QsvD3r?;jz5#yY!Ep7Ym|!aoh5CNookN6Rz=Pl!y0y@YYexHLSKk7 zCsQZXV)yF|;R05Hh?78N3dOeg_M8piZrCJ^iHzW4FR3Y|P}ju=LxBZsA3HJX6GII< zHmJEa@D{T(y!SU0RGvTMMK?6AUh4}tINnblqK9Fq07u*Xph|y1p#%f;m8j-*8;v(k z|AxQyf05So#lc20niAcwjc8VaF!u1DV=i)t`l57^w1BI$0M% zxC%+OXQ`mb zt>lx+Xss$79-b=$ABX<()*egvy^}mM=ml{yh!-z~$9-s*axLI(c5p;#R~@ z5rj%-t?E{Il+S!U|I{x|d%I&dM6oEQKUVeZwfF>%*$?_2Qp-i1KZ|%+Rl+bcu8)W4 z8N5Dwep4@PVyd{mnO||DyqD*+oKrJwK^Y$GJ1eGnt*7yJkMo3dF+$T_;JIS9i6iQ; zVr0)0COk;MD>?-yiXedHW!$LuQCBa^y`InSb)F{0_eSWSLd;zMb~6TNdpfiAp{nYs z6ZwzNEZuCV(G}0A-zWa?c>%*nf8jGRaXIqk7I^?Wf~%{F-YAZ)EqN|}r?S|Cxft?n z&1+_~h7ZJc8OorG;Url!?3m;N#&E4JYxM?U6C-m~s%ANG?x5eon`qwZRaZ;axab{& zXgVfiC;?d)mi8k?!UBqvLC;VWR0k^Pj^gNV+=YSgPG*)*F7CSi6jOaI!C1YHP^D1K zw8qyubCZ_`V}FQVZA|4KX1(z)b6vV^m@3etyy|{JHJhKt;@o`zo}&@PJ1_0{f7&YK zd^_~(O`Tnwz0m^SBN>%AmP87+RMF7xtfH}0YODy+5Xt<_OJ14f6FZi`ApOTwI@w>$i;SzgGo9!|^%vB+; zq${`qB%Gt=Cf!g#Cgq-0fx2qj@`2dfc>&snFs{=#3HA_C_77tF<{!7G!V<*Kq54SJ zgRYySLcMInMa%4saKA#@WbkMuAVE-$!}57D=|2anw5{l_6m?h-6f_Ejq)n5 z2bb|oKU5xfUt$wXdYdz=Jc66Q_5JHBzUrA~2`d(2W(qP`rYeqfa>FK?q=B4&cINhu z4*0FIvv92D>f%kf*ln5X4%>;0o7~NFo2GKL4P{(iArwl$ZQhpDDfXkM2G!2aHMG;~ z#Ri8ce|#hM=nM%y)ZMiipJdO*c2ae?Nl)LFsR22U0cx;5k?X$;{#}J}jJ+fR>3YAI z9DkJFIo1_|vOg*ew%N!6_)E-Hh?8eUo-6dRI49$m_MIj2xdByT#X{;bvjh`VEw-?& z6mk_YZLq{QpB`eJTsne9@g<&r@I#YHc_{MYz8Lw})w{?j!rWSD_0@I%X(_`LDxdN& z>liL&tN6xD8>~3@CEAUkNzl3At=Y_m!0v6e_N2g}%dQI;E8}0g$n2Ar)`0s>8Wl8b zT%Mi^OucD**XG_8wdxkfU%>$xR1qy zMw>ra`|lwpIhwv(cJd~|w}CM`v|#dlUK~lh)Txy#N~1p=R@lefQf-`yKbT!~6U{H& zhLMn7H&NgTHO-_vxO7t5r&z2ALADw?IZ>LwzoJBm$xgY-k-dl2=fIXgVO8{)GaO=_ z)s^m4={_>VPHt!ePytmTN7vBpTEv*&;lUXY^eLPT!MMNi2=doxzFdRLfyAj%*hxu4c2) z9Cz~z#}=9pqn5CpUNIL0$n`*Ch*mfCIOD^svDhjNZ`D8KMxv``d=_;eVrV$COQ2ph z{#BlM)K$~|QDrHTDc2_=5PT06e-B9K*w3WX1z4#Nt~+sE*oOoJnX*tOqXcv(!!Ct9gWt%ZHw2{r=|}@sl>8> zd8QTbFnE81X+JkhFLj-js($|uZwQ~Q%XSm_U7hwMauHY-Ck|^-P*Fff+JItgB z3<9ShDa#`OTHO#T4T~5aPt)@>$C+{1ByS-Ge(y*J07`3LgKZJzw!~z{*C}3xc14h!AdToUr&2b8yn##-WN2$Eyd0qoJ zY8k<(49IY5Bc|v`qxX8({~UE@1ja=?!nIyqnQh((?|kLlt3JHKUi)H|-K)97re8VF zMnG{nHp8q7{;y@JyB{P>e@nW4r=>g_Wr4mhpevH~PgnM)r;n1@_3#M7z zg}jG+3Pk-*mg44y=#18sjDe9psz?|1ap0~h#QMIr!NBXG4sQeZelQ64j4^+`c(9=+ zT5wcZ&&~oKiYBlg7IXFx6j;fqKvU=T_PA_)$P!%9W@Wk!aH?nDcbM^%;SXHpaZS3Q zmnB_E^3{BuUb*p!$W-)tD}8v+xo(E(F_J?G>=6;}H^BLr^|h`UjUt8-)TMTrX$t{e z4ROTv@wa$Jg0Mr5HarPYYsEGLubIbbe>m^FDG#P}bhx#MZ4bHf`f+UzDX>~wsc}8s z1$#8s_A$vblatDCuk0dNitf6BLG;va`*s_#3Tb9=sUx}A zr^v|jm_(DZqIV{dPvz~B#3+UGEYBXR(;WS7g1k4mgke>RE02?MPdWtd6d4ueBk?7m zsSU9uTrJ$=%~yQs5v})x98W{MD^n_Pme8_6x%l1C@x_XilX?^V96mEy2rl;hMcf=pX2TA zvx-6%FXUJ#QaSS3=}OD{MZwMB?f}v!SoO!i&*xJrA?tb|$V&aBt#4~|6=GBSI_=oG zEhSiEXO)WO94(d%A;Byb&^GZ~byMj6UYm|rc?t9No(b+5ltcdxTpx_^IR(IpWqGsX zF5m%yLW`f>Q{su#&RDF2CiJisqYc8V6+WvF)S`E)Ph(<>5zbi{-BRSHjq1_xmuoOf z$j(1H^$fxljspoQuK&QBukhy){c?=fyPVH+SAX1wy?xom)7FkGVCvG{H0_-z@K4;z zE>07ooMSx-6YnZrY8!?5z##15QJ~sQg!`9xh@ACyWHRk|t2ysoHQiuA$939lD1IiU)@P5f-`u%d z>$p`qdXBL7ulCK#oA$}ve?AG8irl*KC5^@21g5+u3EnTpaEZfssSgFARv#}~w)j98 zhF->AF}_8JG14F^i=H+`aotkt)A>0%z-sl)o9Bq`;p^ePYF!g!3)6@Syt}8#RKM}l z+p8^udxfkwbcxo{#)6*A4O0`Rf`1eVznAo1S2bYh2Kx3$`RTsHL+mC+VJfiJ zoYZU}{9czcCe|}{PBbLD&&}`wm1z~Zo%zWnCJf=|gTO&nUSipjP3qFG259{I**nVG zW&;!6)62t#$sH_y;jZ?qs;NSmIDJ}#K5AX(=fsoDkmxAMMWKV45v^JelaN8pFha`M zFHP#Y|LVTjIBTiM$*p$Vg~S(gIiolr?Uk)*{xEzGMkl{OkvK#jwF1At1)Sr+tP8nz zPi|aFGY?Uo9DvfMH2kd8k(Q|T3<|LT*RHWF94C@J`ATX;oq8A^J3Qu)%l+Jv=KcR5 zo#lCl7jr*`D&YWtAaj@)Gvx3Dk-)f$FbPYuzv&o4-$F2zEj4cCLS$8Qf@XAjp*>AU z1S{?@er3(^pa1p8OUukAu}@X`ozKLwUzBk#UT12eu)~JxXUT(iL+=8ZZIx|(XPA`9 zkUYe#>U_y3eF;Bjb5QcFXTy2ZQ;@A*M(abI%(#MHPo_@9BF5rSc`z#Z z>teZ_**4=IMcj>mu*Mhkmd(uI=q0(+4$H?^aRuqGW{6w*38{TG&HKRpk&p^O9b;x| z>*>1*Tv3lsm)u|;%#m~aNyX#P))><|M>~Ipv$)#*{GHRnEY;M=aHs@0abMB>z*as% z$Y2R$(<31DM>TYvrv7l`I0I!k@9E1Dik&)UWkX}BYYseqYBh-C6V`lxYOYw3pZ4qK zk&mv#SDk@^M|5w5?BU@4Ol-%Wc~&G$rbv7-I5rJNoUxhWJk|Fe^SzZnoj&aN?OBVx z1E@{y_o z_!)`xFY2s6j}{(i!oxf1yHN5{@BGl$YLH#g5)O~k@O{`3_9JJEoh3=V*&DL)hrvg* zYeS%N=<+`msw-*)2L?+om^4Eln6Lv4J{jwGLX>h|bx+Iy;&s$SSozyCZN#B~*4I3U zK>#wQl#84UGG{+&CkJg;re}9XHa{_76-C3E*}9Y5}*WrQU(OPL!G^{$kj5M+o=$QGUJ+ z6;wcstwgRXNi=!}{QK)ixwq!JOIT_&5G37KsR~U-Sr6-)`+g5`ieU&y=hW{lG4~vd z>WOzHYNJ%L7((Wg{k#n}J~t&v;nTZakNy@_{-+|H*%dmoJn2;JYPgFtjZM*rE?S=c zGs=o*$7l{)P9K^-)L8teA)R#R*UoIX%3VykxLVDm1#jssoq5a$N!#9CM?P+~r~$5p z_A=2gdrCHb;wbx5>7Yj(?=ehYuX;8C7|_2`P6bG9AV6Nl7Y0+nrsoGGdJQ{}e|BJf z0%KS;f7I3FZ>ECUjT?2una~Zxg?sB|82sFS=tuQZX_rli1)wdEeb9S@hF|*4Z)41jK2;qw^xyC;LS7=!5taMr9tu$MiX+qH;Wb#(qz%7fjBD@$q|4D!9d?{o#hQV{e?l zWC@#hz~fdF`BRoKr6M8D7)_T&B`t8r{N{PP#@h4uE-ZoU_$e}sc8KegwhM~B+ho4d zu-rJq9BzXY{eNvX%L58x$BfbgwV5B#A0I3Hzyg-y8aCl*ab6O$*6UT=_1w z7Z%8q?;L;dL|~T2o}@$k0NKcWwHqxDL4v|AKWAN4QdwhWc zkC#K-R=5Knm>A9)F zx6{u^^`8)bowinEh1b$2G<$1Tn(pHoI|DF#Dy%E!2a3((kVhV8JzZH3!218jl~l-l zWjopP9wwZt<`yJ>)K`J7cfXtm86JhW(d9N`>c8$W-0jefBjKpOkH9;is2R`L$k*K( ze3hC|_@4`jI)d|vKb}AZkW1!V%M0`Rn4azzj|($S_)mSGKVa)^4gyT{-=+HY=6#)+ z(yq*Vyq7*?jtwMdU|qts1KkOlXtTzP+zqp@KG6Kfr#y@HWas`Y>_>(-eFnBm%e}Qr zopel&Y_=7l^mgib0d425`=s}qg_RjHYVt;Q-uDnVogG44d6=pfPaQ|6_V(D`C`_I0 zP)Tc4{153+sga&X#M9lHX-8*mRj-9#sV$8DY=7%2)u7RY`)2QK<82Q4>56?3!?6ZR z1r@=~u|m-2n8{&r%Ko^R8tnGzf3*?L@rC}^Sn>bY3$WHV=E%lK=hvx1rYuKg-Zc9% z`bCtju|FzT!K^x%0vs>Dx*l5XF(%w?Uafn)wmTy@uX2@1BW&u|mfnVHj`^})u&N2G zC@PY|jrh1o2P|wmQc#k=uW^9~Vz_xraVhV$Z8VLy^RA>Lr$vjNHS@nc_FxJg;o4J4 z#C1CT}KRvYTUB$KEEqjkfR7887KfOZ$yWY|CTo|d?m;P4gg;z|Gctrgu&Lr6! zk78`KMp8`qFpI?PyZklWt*Cuj&`?`e1i0Qax9&C(ksjK&VedyR-M~RsSntw-U(@W5 zRtOV4+nLuuu1L$Ui-l5OSE(@F^CLL?0F?dke5#4QNX!0>kApYa*0)`(gOjF)wzTZT zbp7gW@`#)L!AkZ#vaKWd;^kC+Mh2rc-XI27&v-^vA&~aH_p`z)nReL%>N5!H+tx2J z1qUTwrIszY+7V6Qyae}3)Re+T>7^0tE7gr0(fugJmlU^1w;=jkca<>+5HoQL^rVbr}hI` zT1d*588=j!Q2rKKN;E#Nyomv`L0R4@#;YfAQt=e)eu+ce!*l7!^$gGL{U{gIH>AC! z>Fzaax)8MiDtMQMhF7Z)U7Fo@)gJ$V5_ga<{YR#F8f$i0@ZSl8z|Bi`Elh#^KEIGf z_E3_#V|#<2`OOb|PKh9(faUBMUp(5u@LTZ^UcpBy)KwfRQ09by%&akMu9}0Xb9M3G zdv!WCn_10#EP!U$KNJBrcT~i@O9UJ}=kQpSZgsxP9j`4NScNYo3;2UjiKjJ~3BHJ- zB5|y*tkUNJ$A)DiWrdvH?XEqcQ#D?yYjkxz{tUM0x2)(Rj0RL&e$C z0Fv0yySk?WG8=F|@IekSsRe`lsZ_=&Azru%nJIdqC2(p98a%c@&*|=(I<) zkASi^TO^iKwmkW^#|3s7mF`n{<(xK1HTm%UO_A~vwUPyT?pbl$NK~IR5bd?Hb+f?9 zBO+5CmNk9^eGBSmIKi3M#10*OC3rgI<6~oteihcTgIMb#n@X#)^MSD1Zbk%FMjzHMM3A7VmN>*P*%fQ|=6? zANT!KP|B)(4{9j|w3Cg(3E9((hEHz0FrI4(ix@NZvg+SOr=a_;W>=Ev)zJg+U&{<7 zs?k_tE`sW{&3DvI8#LKg>=fH3<}x_=XgYtbblhQX#9Cgb8xQJF7lxB483^8!A%yF{ z8qOFyNygiIaS<~ygU_P)E*NqS1`0S>LaI88+nWt!DB+}2=mp}w`BP_h?sG0-nB#Ab zzEV!1#wd}p)ND3B${=m*1ekq>?_d4)JatbugD4@DmX5AM6ez`hKzcHMp&>HoCtv{~ z=_}g()b)Ue>_jfDLh&ODiIYdrs$s8A%^MhoF|teue4Ec7r5k08dcJ7&^R7v6|3mcv zk1)Iyf-6HW0Y$(x&JM3E9JzP8fi9$9iokIy(Hs>Ghzh##5G%c`?)K(znl4s&D)>(S zJnz2kmbW9g5v&Jzps&qZ%?ptM9av=2Rt+=-Km)lI1gY|T^*9ev=$7G{r?MyYx^iFt z2OhfQyJS~yo_nNd{^v^|Z$A6IOQLjr*_sZ&O53NRDg>YKpTI}vi{C*?8dwkqS*nwT zWnXN}5p)~L$%dvAcoP5OkW@#(WwiyNm{2U=b&&G}M+xwP<6}nqLRp)R1I6_S^c;)W z7)d!Dz3^B25sUCBBk&cSD^Eu&nNzSponAhF8w4z6muLV&I*EaYZTrd|pl!v%3kRbI zCmn(1Pv%yP=H>9H%p|8X&>XG2pRw=I<7~Y%Q4cooscs+uYIGCKKv?{g zk^sHFL&)|={r<;ffI>DA!|-`uwafgW*cmN&jkk~2qWcZAFg(AOXs$C!4r!OsfJjQVN?(+Z$Y;sLotYxnJy1lqr$7^A+`(zHRAt~JY-kG&fgM58 z==4`-U<(2&%-?oebLBbka$s#I@_9@76ifkqY?-j^eK`{4S1#x(VFmQha9V$L zDZ*s;`=(D$$>Q!tnrMf|hm5~mLVO{H#Fl&}%~=bO7LMYCQ0QC4hH6Jm^de{Q-j7~; zpz*QIU=;@{DUe-zJG}f*^c&xm_+EQm^QRPgo@;{8!X^fK9Q4j)hMZBtwp1bJ%lNkF z7?AtUxZ7WAif=lRjmkJ-ca0&#{sM%;wyqF zje%?F;I5h%_fJ=TYPY?F=cJyUDcej+& zbJiSR24IW*P+3YYZ%}&hF5jaAv`|`uJi|A80&SlkhapKN&O!~PE`t*CnkXWDvY@J2 z$a54I6o5t1!ZO<47zf-b+bzIe$slIec{0I{Zj9gu^6lrH@9>07kX`*LBQwN00XAM} zZusDdi(=WAr2{3KI_PMqIlueG*Xg1FC{*j3cR}>BmcC6QHrV%%E>DufDoI>2A8pKx zbD;C%vYM=?)F^J^*PX93J=v;XF_{suNpAr-r;NLy{S{N7g&x8&na6bgG%C~xSeulv zFiZN@i7^C|II?mjtg5-}0vzvWiNJk@`xJQeb>1kLMP; zK#{qFfl$CGPIMD9tyDxpe=Ld>A6QN|Xju%2;ZYP_}Gvs>LoRb3^Wyv77FrhUIy9 z9FzNTW1g{{7OMk909TM7GkHLP)*wyFau1PjAtE0R7O9BB2M6pU+VmP07Wda|hVIO* zIu5AJYh8Zl^^?qZSHSJvxlwqYDsN+?MM5i*vSz$aGcGHt79`Bj)6nwy93)*;&o4ue zD@&w`K+aFWLuw%NYeGzED(`Ye!vr;KHVEl@{9;tt-=K3WXYT!xFgUp9^FVeQaruk1 zV4*v&rewq6mfpL}yX8jhlUwl-Guqoq&a49Q)Cs1mSDjVL4Q)EkO(~wDV_m8cP87MI zK?@H9_$|^_l)^A?c9H%`CfoTP>r-l;N~`EtkY|wj=kU73R^50J2*tAtf#a!f3CF5L zhqViiODl&yO=QBc_tDSMyAZt^#;G8zkrQ+)SR87axSMg2Rr$CFIV1O2uZKKLM|y<9 z)7H6Rb<~xrxe}v?a{q7%|NX}LhkN`k{HsoN#3yD;N$ruprj2d9RWL;ZP3c*xbM2MTEEGY{(ubvRdB+O&# zC9IZrPzx9^k+Uun=JOJffSZMMga0PR(~pt}oxwDOb5~#AIEbKz14ny-M^|3@j`MoR zZ|NLARNvPIqiwAcL2l6FaG;V zoaO$PfAr#1`nf07au6`!w%F)^J(~{P>cd1`E>2*#I#G;|ltY4qv2oXglkDtANU;VV zcJp?;i`&}X5^lh_YAV58{+WFC2rl6_?#$0ou5|%)Dz2#9`G}JHAQZR&PrO3%Yr(}p zc&(y}lsqZ27?SB5;6lMx^PWrdvOK>P4P@R$u9|M!e{_lO&kcW>h|DDD2iQYqBF-j4 z7j26v?yYT`Rw6O8)v{k5WRXodA$m?(uuSr?rXG-I>M`@BvJjB7GiDXnDQ|m1d;6Ua zH`h;n0PgwIQ@56{h=vJ2%k--LZ-xggHw*IoFXXeg$?G#-Y~b5YiHWDG`l=}bI)PN} zvrT3V(a~MkmbK~l#MqwEIlK9HyKbks=s!K67!DOB?{fFrw{S2x->g4t$BIZ`h|tKxu;V}=#6R1>vq-4R%5;fBC+GfDM?S{XM4hj zPdIo=luIjryD44gD~2?G;A%QHt#6L1!I{nK?7=X^fz@mkNvGqv)WenbG|w=PoaP148cz@5P& zTH#Bvc&nESKtw%)0yWXuHgZ*S@Bb(zpN<)RM1QQz6Z8Z32t_$V1w}!Owstg5|68+l z5d6zL;@nc=>ELk<2Kh$pZxP`G{P2Y*pM4{kYRf#{+($6_)2_yNR%PFtut_6fJ^}+g zMJL4qL*bZADp3G#!?kyLdJ0W2AwwWeA8OnIdJ*Vp*p>I|TsOZm9Wm7Yle2v_JeIt( z4Qm-U1IF~dPI?T?3D&w=m%rp?MSm5^&&EYpVZhKD#ZLluUZioC=d^YtgI%M~#Ukux z=2GJhgC<~R*=-TWfu#eJE?`fDH{c(!q5VrZB_{a)x7wKbe>9=g^b8lI+kuer z`HS~#Pn_0&D$mvrVrJlmfcJm$TBzhqyHgA`d8~R*dDikQK^}0vfwb;}wXV)Du=u)Y z3&E$^42#9K+U?gf)Iu2eL(dg2Q6gH*bM7;+zT~=JeyH_u%z6oT3E(CcS;HF!U@mBe z0eT-REzS9yE;MxxU{Nf+Y#?0Q_kc#?2HgrZXb?07r_yn#r+s2Ofz2NcP0XYA(m#OP zBD$@(8%;gyu?i$QHVK4iBB(68)$7{oJ8%=Y?I9ZgsR|%;-pqiGu2BD#<}&RZ=A0Ke ziuD)=14(+D%jO|qDU^88uOpwYZjxp}XpM<5NlA>b7TC>mh^7;U zhQZklsS9s#z8EV~wuJ6F2QD}SH$)^x^<_2yGpSMF#8Pl`E!&KQ8KBh?UW{aP5tGgW z_@G27VE^fM!g>#SK5fPl&bHzgneTFwu+&>HVnt{aP9+H5VG=Fse-^`S9QL#INzS3g zqa5{HD7mUGhx5(pF9VPaSF1l01#6>cAugIMdI z4HoQH&EGL(;h8T#c8PJ^7-KM99cDEKDb=@rpK6_yG%*sNB z8^aS9CyC(X`rQ3l(v)}v;%no>-hP==Rv_&Et2WKAbHDeyIkW}}Afi#j0 zooTFRnze{{+Q`g>+83agDqp}N#isu$x1LxLg^gDOLr^$>3zz2x zP;^kQ+GYk^Cf472U^4b)`M25!YcWI>1HhClio1pa7!kE+rdjkQgDCeb=guTfaOJX; z^9-CDBd%ex?)W$GVDh7!pa@2s_eCI{q{!l z+#}R~ty&P9+-(#hTKD=MC3ZVpyo}ZMvepR|GwShYe#IolKWGXc{&<9vVGEoCShX=8 z?;^p#dpGCdrTVL#Fuu#hdZ$+}t<5*lwl|>6fijcpK_9v+!g}el(Pzj{Kp)rZTa5{B zKsf@um>(FqGXh1Kvl!vS@hH4I16Ee-;(i+pyqaz_@TCGb4M35Y(Nlts z+0QQYY;H6H19cB$dx-;L!SQ6vFvh>2KG9V*K8+HGP>>!CD&gNqzvfE<7ahG(9nm+q zy15-{nxrG$uo93dW&%sbkV?mS7CD6dJwm#&fLH_a$|VH%%fK)0Pg4jE((a_E%{vA6Yk+2UsaD{tR;m|m%Hnw=hdfL$Cs& z+4vj3MRDH%30^q2bKo@>RV}z-8pBA*OOXS_?2f1kkuHZElT1snus~w)Q0V!K#Ut9$ zQuU1|Ol*VsVz(tjB!j8Q_y^`m?BT550#tXgFTPy{?H#BgLy}y9!=zezqQL|lF%91V z!xKM>=ReNf=p6=G-+bIskRQz+oddkGnZl(jylS3wxy6w~&BEk%wP;=Q$(O5?Zn5&! zUdEa7xP(Uz2=hy;(DK}Z0^5Jmoe#`lN84opoi@9Z9>NW#Nw%&4Ix zoZtrmLrWhlG4R{@8YK#edh86vFrNRkTj)msEnEGoHnzNy3i3(>)@Ne1rH}_B41;Z%UFItKk+Q0< z(6Y-sj=JxA|4MzoZd4cHs9W*wbKI(4(sS8(v3#+PU7#J9bFx;5` z!(nAye)wp=FScLsDeWJbbl~gw)Wk#J1g3(fK#WkJpg;qlB~JV~s5F`5*SBCl#u-?xoS_GfPbb~u5E;)Mu8ICBvU#U}DkJemvSOO_hb6u0G|6cUcVUeR z7LGKXsESz9BTLm^7={%@6y`*@p06A-@XV9^reY$gig0?x;0TD-&|HEGJFtW0OETrq zGyqPG!8!rlh}n&rMMU*waJsnE`~XKDhL~ra7Mg823DRBoU(UXVC-ziyk@g!6hjH_J zDx@I5tY!fyRW;D9DUKdUR$r`3Loy8EBcbY_fH+U6r44j5z{Lbc%6 zC;RAq;X|Mr$SmQ#h*0aST;s)9hhW{vL`o$iWud0QI%h1~Fy5zwf!aV30gOfEfC3jN zNx^Iw;6Y~Xc0LU)N7=rxWU>!JT4Apl-Cw$4x*BqB8}kk<`eAfqJsDg^=6RQFs zk=ygI+v6AK<1}26K)9Wq<$S1hwD&ov-u1W*dT2^eYqA2hW_=oTbKE_J0zoGd(GNVH zsSXBXe!ZX44(kYs_EI~WWAu^ zs{g0x3aD6}uxDHmY}qaO^k$5=Mh{#3YwUcj?gmNxwGsmLP(XrE1;A)FV9i$xg0eJV z%(#0Hugr04m`UM;RRwU$gf7vZ1z$TI1d80Pb;(qmI^#@(xm41mlplO zs2kQluJTzb#&MFw*<8%cdQba3(rZ`1K}HA|0|6&BPgq7R#j&c3JK;ZvslM&A7j!0W z{fY()xMlWM&rZk6YfTV`sDD6LPOZ9rfs-_##Mcw*DyNe65*VQ?FkBNVz0 zKz5GWfp;eec4(@^A&WxFXc|li7QW}Ja#-_vmADi3nkAAgs{{QQMb?UsWaG@BO;xAm zts^`Z&=s3lDh8+6(NPlmDw6i}Ip-i@3K&*)m_Mm#i*0pcpwkJSk>^B3bdh zQcA>1Q&xUCL0_%VW$XDjFybl=T`!qyz2qjiYs3#|a~~N z5mg15Uc=OKRAro9!}`R66;>^m+OMH<)-Y@}B#cJFYSs4bX^gqb3pl4k>pr3|`=66( zB{1_W7LbUJYG5stsCI}nhzp<*aWN3|6nBHxUr6Y?T5c_nm7w#yRYP$qiqK<#4P}|6 zK1V*0J<1Od?nZ-IbmEtQz>|0ADy25eF70d;1dkWf@6NnN@vqj}Uc$vgNa}ZpMa+|! zJnx=l9QRl*{R!-zsUELOnZEVOD0t@}p5`ivbj&D6wNsB;>3L!y!rUL$2dsWB9ZRi; zG}(PY8Y*DKj;WpC&NI1fF$!UL<)6X-kT-yFL<1_e|H1p4y%V_xcb=53{8;&bpu)cm zb4gzxFTT9bq*VH&{F2w}&fH6Y#*nkV+B`)u8|agAPbdhyp?dLo$5t?uMQvjjLLS9{H>-S~YFN#HZC z#c*4PthV6dXEZfbo!*DOoveg=?pcGCKKTumrPdBoYG zj6wTg-}pWjGG2eVnz2dqobkz5G%y|qcY?cYG}`=n8Zxppj0sxF4>nFG-JK85WigKw zlNPgwJDrlsQ48@ua{9gNt1zt-HUk!>!yGNQ1Yr^4}#_AaybXS(h@j8k2;u z6VJ1$&%{mpiWWze2Vi%zN+KOzm!ob|->VSlc7k{B+sB*zG}N{R|JQ@LsYe9(A%pEI z-k?ADoA4`-KxPAPyjvu0_SkulW#Q$jDR^oQUSoyjI@dvY#QR?jzgh!%Np#|s8-NOS zGvEG$>xXTa)ed_iIN3k`cSgxS=dgV<1I{RotOe;$0&Pxkjr}K+E_=^hv`*+^s$Voj zlnBtHr$hgu?a1$U?G%9Fx?lhg4#;Gy0Orp$vd&>Lzg^VQ0AK3TVAPk-8L?^Vv9E}A zt3lzhQ(w&}9}Myg3xQT|QxBc#M{f_eVVXh0)3HgXSI~$-P<6}at8enwe$uDCIgzaX z1^~n~;ecmU=%dJxdZU^!kS_d<)z{yn_S*y;(_UJJvMrfeae1jWUSHn|z|cT+NB%AZ z5T;%SQ;udL@!%>UxPiaBD-%>7huD^02EWR+gWW{~0Q>$C1JB#P==+kN>^~$dBIbVr z=V;L(82D|(mXh6B61G^o6Umbj@#?N~7Fqim#}_Ft)zxC3%SI4%4s65Opc>mn^-+HX zAcYxxh6wCrYy-{l!~dRHyySMhpWzUJT?xtny`Zh2Ge}dP3cy@Hai9tNIp${HI^PFL zaHbi4eO&$H?a=ZqTkQ8MNgg(6D)!7OBt0l=gc2_NNOGq6j-uVL9)L>rQDVm>(XdN{ zeK<8&^zXZu7{uC8{{&nEw`1`Wa4}c%;1I{-UPiE#*SF(ft9T~G-6}<0p3~r#{ zD>vd4cVN%_E$wCpoL@@4SY^0T}}mZ=-J-{w#ItWoNL)MiJ2|LrTqi1n=dZ!L6}5tq(JZapWwN59iT zxf70UDWAf^N%K!v6w8{^hmzIn2uU~b*}X&Vj+~sg#V;P{j*ZiB&{!8xz1|5sZs0v; zY!FoVA~6YA%_ zpjbL)A$#02xNZmf?B%fAQIB8X1`^wl^T80V@aPU$zwJ-GQ7+9Id+xKGP8BtBR*m=Bg`6qK)o?fYgd;t=lMq*ra72d&;`O#{>r_!p2${-LgI znET{q>uCEZh5Bs{`sQe2(i`oti6{KP))C0MFs;6Dn*B75Lr+#T@6@~afhoZDj2F{h zzDmn<4^8litD;gaP)i<;fB_YdK~-6Y=qj+bv+(mZ|%*&6ss9io!J{7goNAsIZ@bEBtNVmJNku zyF0p`f2IJ43Uz)T2srBc7={)B*~DR+c;8}L#9va4<=?l(sy}-zn|EHuL6ye7WKa-M zhr%?|cHDCirAUK5^q5q5R&+0#+J}`Z`&F1qTcxpVbr&yY>qi?f^q93j0#HXg0@Ts$ zPK)I?9e9;W4Wkl-dx}bF%rs5X?{gAcHWYZIdwIjI;Is1picw6djRTU9c3ubKNBE0|J`BeZ$4iACaQdlbbXr#cTRew2qj`B5>2~Y zj*m{;xKkzQ88Sga3IzJ9 zy$sxZAV3nekB}%7pr1uhpiS191!h`TK2f||8C;=RETBN!nE-QNf&()+1i&?L$MN-9 z%*`HkzGqqB0$J;xtQt-PXb4bW4&FyZ*EmQ_&l}jC?`CxE=xCkN7XlAzd)zWm@2#ymi=u-!-dnxAqVaZg|QgLj|8B>Zb~K?Md!_s z(oArIjPh!g>f3Gp(+IMVa(qr*`GZt;4-K+1a^boZa6At&@^C;84F^cR6Kdx1{Ftq) ziYi04KX$T*+jmD81c0X>#M)%t=rr{JKrWbQr+T8UXZLoD!BN2nYp+qK7V94SHlNOcDAya6-jrDO-rd<6 z5;yj?OkV&Pt~zbj++5;eLEl^|OREw5!e8+T$=cVWN$dR2hqw3Mk0#r@a=nmg_)rsm zDYBZPR<*Y#t4m1vqJg3TWiWsFodi6FDm~uvj_&RGgxE6F^F3tV|FRTS$9Y1QvNh`~ zs$+wN+d?o0cEaLI0E+6LdR@7oiZ`^n*~JGdBx>A&3hP2M`eq&D-vg&x36|53Oi9N0 z&&S5DOp6?(h)4>~1*V=7uo52V<$;zo2&l`MH8^ym%26D0jipgnMaGeRBYVP!vs=F0 zgr8qVKB=dsTwr5>jCPYQYalO3AM&`?l${8XAs}p>WE#&a)FLP}# z4c=c;VAfL=3#Spq@O%uY=sK#U8zoDa=e42H1MJ~0P!WO&*ujU#7&nxk{tOiYZP)w4fo&U5 z|9EEq{DbS8fS=gDJY>yE7QNjGi}7-qr}@~(n+(jyRL+t5CyZsNuy5$DrP$QnO$9-e z@I!o~Wy2cTTv>626f{o-93245DO5=~+hdv`Y{jabuv8k8nGMd1(dJvn_Oy+hSo}4b z0G{UK(S_#qBIjMU&z?Pzl{vS;g_-fKDPmthE$vn+3Y&5$NSRDx66ab#jkdZ+-xAJV zxN(jt+6?RQg~uOMXdK3|qB}MQzFO8FXxy}$+I+_~W5b=i|FGRs;~?drY2W)4>fDh0 z=|5PT*&FbKGxoDLR72+|Rn6(JJSDDmMBMafI^IEuP%bsdy*x~0i40%}q)CeO53Ai=aFFW1L za#UR4>4BFUNmvw0=P@ar;|Q%enB)z`IdnimuntUN;@r56bHBI{=NkBL@Z84vR*Q;b z{-D7G8_(@!hignghFkAwBq3rL^P+je|rFEXE ze(MtW>gr*T@%$UHD4MB*fuEn7&#j-$-Qo%?I1eNhKWovs1;_^A3IV#x7T2 zR6R~Bv# z9?iKEql@abzKxCW&+;3mIPuK`j91?|?#IQ+%;vZ7i=*bbh4AO31jQKP7HqQ_>~L0y0&-Uk~7SJ_qXhp{!wHRCcRc$Eqbrpu);pLa=Sd zr)*vaE&6{0P4=!mZ4nFq#zjvzNzE>xo#a6^r`zA|V#@+okN4JBY|<6Hwij!0I*}7D z9s)DcKHDR;DW4;6sz-2~*1L?pG&;O&eO|a>eC9cNAQ>pPd?Mc*MdUi)C;wF?STuH8 z)^9mo-o8JJ@}knXU6~xoVta8Q!z7*XlVUEC{b|igVbRkX`vpZ~)jyGbV&>0(JUVRN zukahIw!9P+etpFF?q#01{XX4b7B~g1_>qQ(^NOe==^$W43o{-=bls z=&cOHx!-{wYpEapahwJ7Z2$PbU{q6@#N1WA=T+%n3hpZG3CdjM8PqNLqdF0)AGU{D zGaS<{J`M7yPvrfJo}d)p%?UK#dWrwg{(XF zRwh!9N8hR#$F=3+qYIr+9Ljg=*6GZhr)qX@us*G^ zhLO#R&eE1eP;qcL5nFo>T#kVpws_gENc|nBNSn_f>JYn+KddKyKvbGYYyu%+4VQ$y zxT+BpjJWeql!dyMr_|=P)r~ZZ`Zz0cZU~gDX<*i7;_qQd8HrLb;%W0)tPpPtpr{|Y zS5e6X%(Vfd>awu%=64rn7qh)x9`Xw5po0W6PIa_+5irr`9V!^t zM6a|iNR795EVy!!{OL85Zs)Tg5nj=V2T^3qM06B9L~;7Nw{KRnNZ5TKVP<>I*lVL% zEtg{hI$ONCoq_u7n{QM85Ekqoy;d`#O1}_oSlCAAH&KDR6x;C>1!@iSA+>Y^BrW8D zNFoO}pzaEW5sA=QSk>Gn3x*X@m~($*xz?0e#UgpqCvTtdTK0`nQAh;~q=P&cTVC{) zq%^T>UXF}q+}b2oRRwFRVoBjDzAh2W@~T(Oc~F%rM?_?JLTw24cii)#M{mZy(-={Y zK6PiZ)Fuziw)>ZtC~GsBD)}5_A_4i2#2SAqw2sIwWRoNjzjs(dcS$%}IIN6CX}s+@ zV=(qnXf?O%yJ##XDgZ0QO(slXKGg)o#Y{V;>7ICAFz(^XlNdq7Jj`V8HcSx;Lj)s; z%&W{VE+uObiDWpM+M)Gm(vVj;G`p>1Oc+8gMSe|}{ukVr~`27E?E8PG9-*UrK^@lc`Zd;pcR@H!av=#P^pLQ~A!e2)_^qBHZSts(J-v@m(~ z@mh&9q#+;llR1d{eIa7DB{p;L6n}bk0H$It+(@Mye1p#NJKO=_`H?UZGfyJ7L7r&H ztVm*IOVBK0^>d|%%?c7O!mpcIL)Mm2zr~&ufn<&bw&mPIZ6wS@0(W!|A8(PhFu}Fc zFTErWcVsLjcPOW@?kso|TNR8bi6(Nfwgu|ZQZef+B4*G?0uot>od|Ue_cTcpS>0j( zIGt(Bg=L1lshytYgA$U16Cfuk%bvK><6bTOApnwb(%prZr@vI}uW10wZuTEi2}_9O!BQ-G7&(${U(>= z&V&*7T+=s)nktXZ0~HaVWLAq|#JNZ|VYZukWw4fqL}frS^BNACky;%Q9WY`$V9cRn zrMgP;8hhn*gJ_z>nw$rSDicdpTvZz)xJ8J}qs8pIud0AKsP6!;LB_0(=JqJ}UL-pd z&LW!gowCxlp$1&kV`OM z90}BlY!^k(IoB-GL?EPORzRItK_M6+GLTF$IY7S-e>GrBc6BzDjwtWNLobN`TqSSY z@w?{!@5O8v$cLs!YYch~zT34ubZwPFmI1qiuebu{?f9SREsZLpKXKi!_(s`Mamh=R z3qDMkJ62Yxj|jIqh*BOvL+xbv+^n}?YY`ONnVQq!&3de8I)f4D;Ceb>U4o~sc`MZY*A97Z9$8a6>Z5MRvj%!LTAj(4u);ziPX2q3b(?jGMQ{47ahs3 zMoMh})Mvv=Iiaim+3a7g=<1yD+!r`~ZIK50UI3I{j+;e{aVa()?9RQ91Qrdzo`tM(0@*n+=M67z$V#0XBRQJSm!&us!nZ=hRJHTMh*m~FuZ zi$3jOg3gKn(#LfJ^5Pia-TPkHS(TnSEh}2f9Q(-_8(otlc!5hep)xTcQX`ws#V>xHJq-F#3$&-pRjc)8wJ zUA{E`;CFkal{FdZChYF?Ptg#t4krhx4~c69;ocC!oa6kz#>O)X!QuzPFxo7+eP}X_ zji6W?J6eo2M9U=JSUwLr^Q+_WR(Ulm z?=fJ|+!11h*nGc;*eCLotLPeH;F-2j&T7X>W6HToE(O5Tu`<4JY0#)9d2qX`0sjGk zn-(lCkS2HAy0G5}vVAYJ<7q8J*Mf-bV6pwfLjRqMU5t*JwV#J2k(KYAvI_4Av4EOC zg?fVKVB2LR(b7a^=nA>K+?dI}wX=P`N}cf$IPGq>J%&XogjA>SC_>5 ze|Y&Q71Q@8^xg32lFbZX{OM=WeA2Y8=k=_<{@3U~_wirPy`DWY0)KuP8Cx{3d!30e zUen@LcW%7`-rwa|oW21nnYj+3+K6%;KR{Wy_xWPS!*S_a9#7)$y!r}@E<PgLA7C!&YJpaEZJZ6A2WG^YH>8zQVIuHpsZk5$3&nDw18z;Ze)m@!+u+^_P> z@NGWWNBvEg?$RfgZ+!gm$(rpf!YEES$*MeJ2u-0NaK`)_0b_zMo-G-0jEKFJ(R^%i z&3Ud_X!>9X1_0~N<2Mc2uzOy=d?rRIZJSE(RrOP4;8ufVTru7`LOgLAT(0Eol=1&a zXlnAkM8wXbFyGF`2mqE$9bH z71$OIg|jocZPA|lNom`P^2erZ(3@Ymp)yUsh3h&gY`FP&q$x9L8m_(a0&h7^fj-$yc(#SbX-y#vm;tm6s!+jv8sPVx{N$@KJDr^Pji$~gGjv9v8xg6asdodaG zInvQNNexcChM?&G);VM2m_spRnWV>Yzg7xz&kH;DWfsld20MSVt2Ze7;Ct^R4;0rH z5ZFlm0XEcd4hyilVKe6Kfha)?T(MU7%*;}NS~ikD%5-=b1etVK>-n^eyh3Wu&1>@$ zP0?V|oVDA*VPan{b!9=vIdnGH2!KHB8CuA5$aj#zU=z13#t1gzrTj-~k1100jX!76 z;k#~LM)m)mM`S8M16>NIZAffWydep0){qaiU7~tG9q>EB8(=2d6!oY}L4?{3erbqT-Z(Y^r>!sCi-D6?5 z9^l|2-g__b|B`C-o4}~9y1JcZyntS~FQn)mt3#`rQ-XF2+ffCpho z-cq&Ir4v4P?S<+@$y)U0_toC-+1WVtZJdhFGnfgN{mnUR7sO%g;}KPqEEKV*{TY|j zO#U8zr|0()A}^~;6^~Z1(I(R$g5*GW=x$ayJlVjrMQ7aGft8zN6JdS4vDU>$ zFWr{S8|%2Ve$!8q5Rtwv7~<9H=w@i;C4Hlv6}{CrBn}N~j7Z_H5C_)Gs?b->7v&Q7wNDV;m2cUI%5<;F29Yuf8~~(fxBqzDMF1@+IMBiN{Lv0Z28*g$OAHr4z8hF&SebMLUGgcC2Xkz6&9!9--mvjIaN07ee_(e0l(1tE+pz8@*{hQxVbOnTPl#EVs+&B z=MT{}d$1oP4BdagZrPf2x{&)Cz`XBXfU_MBp#4r4Pqyb}cxs);Q0s5Qd->U?Q3gsH z{jLDiPGMM&E=z8hl-Xz-Cggnjyz7}FziM#m%PCu zCh5ee5xE?Fil7=7>T%M5R;&M-p|83~F`af!s_n|ax&CnHKXD(svT)CYj%+8^t)=EKI%Y;#&LfR5$W?R zVeh#PWkMNeTgp9eNL*aXK(yC>598M7)qM-R^O)`%T*16{03ru56$8%&-dWaFncwfl z^hdwE_Z!jaJHJ~v@d|{y1igC+&crM4vL%e&unHe;R&=gQ6I2((yK2M06Tg6IWm{iF z_-`Z2d;!Mtc{6{hy|T`)d~3E!GBr4pu+QhUm~`F6-zadq92A|xd*AtB%})_6$9K&7 z#NN7vnu6OV@2PPsOc|K-HIf&m%BIa|>>;1HS>x(d&^6wW6ahoiAMsknd*758nH)KJ zDPo=tZ=ttfX-s}H6Jujyt&r!=(F3_Ur9)xdB1p_&)Yh3%!ju|cjC07eT2e$0 zqiK+*RKllX$}og*2{@Bbz$>l<*OiLPsgJ1c%k@gz$4xZ;vB(^Kkf}}hz8+spOyR^& z8G&)Y>wYQ>apJ!UT>LH!OUAf;IC;93a|gHyz7^udCX}Jp5^^U=`b-r#fQmhRiOHvC zEE^}2uAdONdw%pAk_ef)olgtNLa_eZg}Z^0+)KPM+$*7lut{6MlxMqRk9J|XOW%8E+NAqQs=4ceoALjCjr4?bpgkXIDe|{VcdjBl1!8w$ zKIWrZwlIs!Qhq_$f-6rZTQod%)f>_Z%ki{GrIR>vXCpjvjRrioNJF_B zV7gTJP-OF6vbnX79fkHEUYjwnEJ&8QG>?1#jQeIIBgvNRJ?z@M>t9B@8_h2}`J~JsU&yzM!`Sdh!_PuX()?|(PkxzI$5VO$DUq!e zJQu^5e}L=C!>wL(6Z=<$n=tQ|KaWxYwtk0JW?e7a(f*hxsV<#|6WpVjMlP8j>ccuS ze+mv>)&Ds2w2?fsvbs2bx^1*Lsrvxw$|DZlgq4tyZJz7SBc3>X|2?9b8X=>#mW@;( zb0OQgLRn>X?D%X?ke_vic3G_~;@gbcI^wCDPJ<8Jyx^O>&@FI{CGiFUc~*G+C0p^^ zaAuqGk%eYDTirfYw-C94I$Ck=`3DG;FLuD{mBKDeCg_fMlG<*rtLQcGtxjk;ZhJ#A zfiE0Y8+v!EPHy=dS|Q1s5)TU%uD-E{cN|lFJ7=CxQ*EN9GKkyP3Wex%Ja{#P_O$sE z4h$84%vM8#(=RW*tEL1u@%`%k zzu#(2UuM=vBL?W0pvB@Al(+j>r^kbu#}3`;QJxH@_l*J~AoWuBB7EUtEI%Tsp^?bW zd{1_|uNw%Rk*Cx-`mAQQ3s%b8NI$Avtq#Qn}L#a5UZrhKXGWuUgqIqZ;r^w_WP z1x2+pTnj2UT(#kYT=>bpuz6iL2nlgW^Bhu8&CH8XZ;=5fSanIxLOu(O|$b$JD z*gti*ON|lC<{&<GPgIvIaIV;`8`i={&?a8mibXgJ}-dS_}NVr+Apv_ z%gFD6k3zGBD%3cui@u=Mmj4D_ntM@l@e062Vm7y-+?F;$)7T==IrBsD<12vqxXpIoh>&~rm z1z=6yYlv7yq=(i06DN+>P_dxlw&fG?{yD-FhM+GboTJcbxRhQ;2~O}Trj`Bi%rCZt zr0lOxVQwfw!k$2>%1B&${0kwi*|D za&Q9;5g0`%zX5Lok087$Q)GJnSxk(#H$5-zywkaRP^nxy9!8A8ZHchIE$E+bec`*kW(VGU zA7g1AgtePA9h*gt%eh80#(EnGN@%hLEzGHI;S$oM2ouuHUNzgo=)$l^XoUZ8XcpAIFYB{qvoxsgbc4p_l;;%T*oon2pSG)_=TIH^kMXjBzN;x1T+u1Z#Q zA*6;+O{d*U1i{4Kt@K%=T4^JUf$8-ulNqO0@?F?`g!oAZA%mWCkhQ&oF))&fBY(a_ z$)wLz5$rW)t)V9MOh088_IEg4PWlcOVgu=BAa0PEl>5FVjWM{!H)i_y*C5)#vTyX) z;{Ch~^)JQC@4xa>?%~&Tvg8UlgMjg4c02yGyjUl3YjRhK$AYT2t%}!as z*%&eCUX49Cfs>kTo44S~#j4WVLQd?%RmF?tqJxIzCo&L+@*r=)d&nC2&&97C1#7@x31fUM$N> z<#(LxW*boHwvAkPrENvt+qaUx^yDfe;XBDA4$l<;Nx7QB&0NCsZ3&~X49M60BC5!J zARfa;pv^O3^Pq;<$w792oKyJYbdbNK+0xcSb6fcrZ=q{ybnPQ!CDNEAU9dq=Y;ftP zWhlAnHV0XOJL1oT#VMWVwm?1T!DM8i&FyIru=wWB-0HeI(ga3!MwOcnZKH>V|;ZfSKKtcreD??gTr z*g->?cb7ctf7StZkzO7h9e3Fy5bZv_zE_ArX}fr-6n^@kaeW?aer;x|14Fp;T>V9G z9hAZ6(StrLWDi$h)Pa9@Z;zg-b@Xg*^KgKp*8asWKsB#@?cKJ#ZkI|I4!Y8W|Gw3c7<@- zFMX^17rlx|xGD?x#A~WxpY)N=FRpd`s!s_}*ZbSNVFeKyTz`7DdHS$&na!Bzar{Hp zbxy-aBgK@rs=8D$bJ+shW?lVGudr9?w-iqy5{5glN;b0`p;~hr^Jn_+6_Q$1RIOo| zAa$i@CYO!NjR=1R$y5$f+FsKBkicnY@Xv*wTQQUr`x@@`vvNHckiVq4wq_vEV^uGYsUL@8>CTl(vu@8}N`=PUba>o`6 z!svaMlqm_t`nX#10TtPEn z!#SP%q42{(!XusFFD3HttlxocWNTe0F5%p6;PYq%9wYBc$iP0(1-HweNVSms$I`@nDGd+9Y4cOlY| zX1Z&LuCu`@X%39rA0yZq`l+A&sOW_#vBkCb_+=dV_RdnAH#YRyhXD~mkU7)?lUR){ z-0W-U${nvf@4)H*!_}KdL;1e(<5>n7*&AEPBovROlwD}1RI)3KEtK6L>!55SV<;+1 zg{)b#lsw43lqFjsk+Cn?_g(#N@6Y+3^ZS1Ps?#5xdFGk>zOL)FT_IVR%*8=sP=sSP zgXG+6r>{n3qJ@RkK6K)5*x2vUl>1I;7gbJfu`E00JJS?P`=L_^=t5;S&hva)QQl&d z@#0?f;q)JveljE~oP2&I2!UJu`()z9;$q|{%|2Z3DV53^;Qs^{_G`NvDG?Yz`;OxKxCx)G+L6IA;@M19 z%l_SDn}3-lu!A^dGY>t;LLS$)gA9#q_=&nb&pg$~rY!2QhV+-NFtPTU4JKUVsz$g? zMRsr1Q+tP8urvhkkmXoj64FSAwkGKtJDWX~o8DceAr)C^;g_rk~3gFPjdo^U1_MeY#Z7JO)KgHf(iLS6ZpC4abBI zJ|&2P9Us{)sjHl_;hX-(SCe~Y_*o~EyC>gB0kuV*CjqgJeWu5aG>JyUr|SD(~K zvcUWj-$L*{oGePcjW0diH@R)Q{8!G4`TR*Pry7%wj~^_|b?a_#n3r^27)X}}m5PEc z-PV#zxEZgButv{f!1Bcx?`iLrr!^yaO&RcGTL({l$*H;;7VRBH%;&uy&G}v8HLy5B z(sTFUXq~g_NeANA;=G?I@B?g)6@;oG{ti2 zV;NJU^?A&)bC7_H$cOuH#-Pz8FmmBupR!kfLjJOKtyk^7(^TV+ryuGEH0p@SA|G_t zFtnyCq)`nnbwt7G568-mRr*%!H6ONc{xWnqfSAgw{Ef6Q-OX}N6tnd&Kk$#% zMe0GS)8Gq(q*|`ob{>lVyYVL&*T;YT=2k5uWZIO-8c09kdFW|yneTL>BSS=aF*dLX zH%X+Tescr#@RSkscN}Lr4HSGI)HyFv6yRo!5LjAVD$`t=BJsSleZ4Z9p4n;1bW<7$ zg)A2r`>W2j^QMisV^%Sh#c!<0E^4T2?SEz=XB%yzWQ0T=PWCcbL`Tg{EeYtoI-(UB zb{-D_2&#k)05Iw9vYIDuq3I1ZC$V6o1I+4N5?(}ytIAFKSULkEag%Mh%vO$IDN)x~ z8f{-KmRMR^&Tq2H*`y}Ogc1WU5s?2Ba~J?auQ5de&yfi;FrA%*M>X+!11Wl{x5!{h zu91|$3xyy%a4}`}s=eQ8r#tfQymI4WG&*J&Q+m2A=9FQgS;0w4L!up<`M?PfN|=Ty z0R#y0){(^H2ZEo^`;gk>SL*tT@%jtS@oMc zJ<%US8c_?FLd6_m$xL-ep^hG$n(LT|B$|T$?+7W(Lug9F(EIi+b8vQiR9qAu{>0f9 zh*}o|>IB2p#Kmh0@I7BUVAM&Ea0KD6Y*hF(4RixoP?dgD9r*-y`^h1MufOu7>{!`K z24t**X@w5#kyE=+Dd?wlO$TJyGT`neEF*sP3=UrsUP^pQ0x+_$h(4GXV!jZHyxs|r zsh%BY1_=Gij)%xqWCko>cFGS8kW|hK#w#8G^X9c!k{LeQ3yXvqvMJ*aBu<-WEvpw^qx*_0<~Ss+ zmEGQiHH+XQWSo%+Ra0CwmufxpSZiZV?8h6Pl45FGfZ8}U z;tnoz0zzAMMmi-o%fSq+I1GVIv>do~@AzxD<-fDHV`0?hYzt;ipYk!t)A7;00ELfZ z5H|dPkuuo=#=ffn*~ZTY|nwD1|;nR*qp5kZ`K>&<3xH^^&7V#5gU2h z1`SB3IiJJzw_QKvga3Xd&o94S{j$EpyT&}F2xZa=3-`~6bEL2q8dD3sTtw=-4p4aAgR1KW#p5J-w;kZ6+pp$ay`IVX+ z@9A$G$}0zoDQ}a05#F)?J7L%VUGwex6VE=m6sPTfSZx2~YuGw5n&1}cu)8-eRr|Mc zs^{l3=10+(g2o}~fR&XDmz|+ChU7V?gMU=T=6~P3?*FF_f4Opyrg!D(U#e%<4hz|? z?l>vuy?r{K?0IQ(qrV4Ud6FG?eY-kqs3s8qe&Te5#^3z#r(d6pGg_7z)Ovk~TfYr% zoN4{0U->6HcBw;4!U11JCHg9RWZ>L0xtd?QU-*>dYDyl?@cHjNxvsyzDzGVE!dC{@ zoeblW@+(2q9p>E^urnK&xXEw0hI%LtT3>&cb8ljgzU{tQUeQ;g8lRU#`Z!vXVYMUe zig&tZ90!|2TKtdqu$5uc*1=XgheNi1yzA5WAI&&sS`7V{LTI#o+cD0V{Ns6;bLr&A%;nZi|_br9GBbX#? zeb!qYwCUnKq1I&-Fmh?x5xCry1l-;YrHiSD@p^f{LEh>d*ZKSlS&KQ0azwc@NAxJ znKm(Q8nzcpv2+ycho%&bw}?0j^^97Pl;|N;(<`bD!ES39vzjFN!(Ti(^X4*ad?vp3 zWL{P-<#nD4;%%Jpt_QX4&aTJ1Rt~*w-MMRRo$AE7kQw>y6LE^4rpDSG9)2h!phs#Yp&+N}=b`gQ=<@a%ctG}N;`iuci3c2xnnOMtP>|2TgjX^FWIAaY$g5eL+A^v8Iuw-_ z)q%4&6!J}REiY?1S26VKQMnp7pWKq6OV<|_g6`GG6H<$Q#|xU**ZQBq##jrQ+l15v zI1#y37aeage$*$`C`BY>2OL08+go*mkx5Kvb(#uApP#@}RS+{^R$r>|hQP;DNmMLz z>u=jQWIB`fCch4y_bX?TtTUCZwsS2y)rJ3l-nk6(f3yH75#jXuW>vt5KDG5rAG-~+2Sq-+9}Nc&EvoYH%BQYHkRfWTPi>T5Oa6w-?BEcRWD*ke zuh`C8F5&Kv&AY=xc&1&}2B?ZtN#K8Gb*5U=@FfopE+cY>gxLE3{vaUlzSK)aP@l>B zGbuj7OXCYR-Nk$UT0^D7ZbXdXw#SSw@J|s`E)yOv_usT4%bZ4PAjc#IMrch31{yX# zE=G~d&ozKWd#5&0>+2|6rjss0V{9HM=UouWU4%W=&wST$o*w|=Dh*1OvkBL1O`s>BVP-{8s3_MsZ61zcK4vhg*Z2f}3%!Ij3r90fhbSNs}(fdx-Ausuigv$ z-$AxwG6b$wH?t#2hTIf1R4RQaR*}N6fhsJ_bXP3B<$Axg16LlQ#)S`#h!7P+zmwTr zM}>2GTBEOZ*ltTBAdHjH&SABK202bV`^KGe)7&*wBxsLWZSUET4}8hGKhfINycnCc zVd^8vgGVN`*2>}Tq=2wYJh+Ceo~ioHxghLAg$9mdk2?z9$q>JN?M)cldrrOgKfGDMM$70D#cGQYD%FIf1N zroV7e0J6F2Yr`F~?qkiUOUNn6qlKe21U~JPR!?NX$G)jWXs02RUZx|6p2(xw>}zai zhp0|doQU;0D%JgA9vEJya;Aj~)M)T&ok4TLvsnn%GiG{VCK_$iIo1cM(Jk?71Vlc zfFrnycHLsZ_NxQ9dfSm5UoRW`GKK*ZC7f+8?6!_-B_P*=iQTyJ3%a>eW+M!c=!+6o zZ|6CKZtJNwoFv0Db*ZuzOT6B!a;ydjvL8W*z0Q{2hV{+1f#6{YMAf%&#PUfd@o;@j zcoE`F8%C1+b{Llc>&mdb5Y-s(&kcS3^3hu+$>#wIPC=5&!B$`#no6ZM(e3)F$V=iARl=0 z@wx#896%qN!1z?R;Qf_-yrCqECV(q>LmlmNgHx}%{di=~-?;IBTZ18f6crje9Fw|p zJ?aq!Emw_L>eL9_xh8HI1Sl19Z?xth+K19$iI3&Lj}?kG1XNreCu>y^cRb2GnY3=- z-VBDNW!N}ayng{0+ISzgvx=(9x;X{)yI5`)JLS!z0t0DH<-Jw=@|V56%x4|F>JXzu zM{0T6vAC^ncgBy42e!-D~4qO=KjorriQycL$jJJOupdIwVV$zJd`KAdBj@ zdVfMt=LpEv&tB@tF@%docQ6Vt5-*4dvsf*rn_6K>hyXklWcv9;sd0tJwR^}zGcihf zw;viY4NaGCU;Y0-6$6;|W2=10PYjX`8V6VCDPSH#QgithJL%FRG^S(V#;Sovp^GkQ= zD74nW@k@p{zj)OOnJjuC3hpn@m@56Tdb=U%>j)Z%f!qPehI1PeQ#k384@ka)@2+cQ zH`{YLR=!GKzD(tomWs96_x$#{w}qNO{-U~@L?HK5F?|u-T}2J;ulC5=lh0_F1FH9W z(!WQF6!djV?WBRzXNPva9Ci-ypS|;waN$PinNqhGAM~Hd+)iO2k$VG82W>i`JDS3p zmYr1~R)@+fjezt%$AynAfEm7wP*Y5LO*J(qBWa*JsPVX$Si-AlKj4hh=78+*Fjpz zXv8yMJYoBs+S0IrIxQKp4hLEAx{fUMLrqw)g@{f5zMhV4y&u}uzV@c?RsAQAHZk6} z_Ke>y(&+1b8lfSJ+yC4hc>$^8}ubI0yKcz-F3hjiT=%=8?=l!g^lFK#2+^iF;~ z>KSOj^5K?Hcwl$KtH8k7>RuHh_TGPwLQXi#e!8sgQR)n=Y~Ph@T0=O}bxrr-CgVjl zG<07lhdOR(0X;1^&C9R(Obbr=PNF3ln{i>Mb%dP*i!*gth0H2|&hY3qlru}h;Ddjl zyP_)AQI!qfCBu2GZ8p8UE|(?3DG{COpXN!LM}Kavq0sm5I$q2IYPGcY5aw%54d00C zW+~sGbqK!Ju!VXy1aA|19`VUewqGgvCQh2ZuNk)y^n&zhF1@SU!-L8vtyFEj2Uk9q z`XCLCNo+9=AqNznu5LcHZSlW{Lb2yEFyrbTX-|>9Mb7)^OSJ(Yzp)#2b^;%p3G1q} z1(9(qDg)N{1xv6ye&mHiutS8cOx5_iXasAa+3(2fpib37QE)L6UT@^9)9O_N4pVFN zE0Fc1nDc4_2>D$FHJGA(ZK0`QwW1sArDHvgNsX^f)zVf*-LkX z-C2l0o)h@S_PGdzxq~RF2-0uM)~T-r1skZpe(1b-^?~B%eNr7EP7dtuI@f+f|@K0nZPz54??+z9|gbfdKfD zl9-Ye$rBy_3}3C;CPh)+NRLgDX`nF2MGKu=)f`f zQ)#KMe&xrxs=s9Q_f$S^2v&TJWbZv!_fk4 zUMP0!dYManw50O{{=}Gz^S!-a<3H7*;#$OZV9TDBj3Uk?@cI$XCHMJSjhC+aN#pZ6 z7=1N*2FPmfy&rjbKV0zXZyo`q={1;pVSe6!>c-T3VB?uHG!tL>E^lw1_%84CAi!#J z%uUdCQNXhEVql|BaYT*iGqLj7RH>OV*{Wugp0SpFH6n;^NcIAjmCa#t&csrq#n02) z&26QBbTsoeuJ~Jh#?si6o%qjBDc0lhw$)bOaWI{riuVReWw=qCzEYImrqkB4(;2XV zZo`!#*Xy3!=EIr^q8GR>IG02S#e&^03y5jbq?wK|(5|os6LUqMr^Bqv1ts*C-xbq~ zgp5OxuSD7ZKmkQ3ds<}w8;SX@HDfIFHq+b*p4iBl{7}ak(n8`CQDN!)@sfccIxPdT3n&k#wxuP$v>fvzE3Ba!4o_;}fs_?|Q4*1qkSau7bAl>3S24w*0p|hD0t5lq zW+G~AsQ&j2JgWj`9ks_I&wg*BzHzWQOB;7(1^}#K=QP)>aXRiExuXfy?`vrmEl{Z3 zIMBQzek~yAw{8$JPE7xkw}udFbwo)XC#AuNA+C=(T;}Wt-9LC4(){C7`29a_J%a;H zW2=G6BVP9yI=-MTT?(ys`Wf+{3&_)f>G^R6|jM2`Bx%*x-6nOTw1c{S@tim9{Jt}i|0)4D)$e> zUxenN?rnNIPRbPeA_0*uIyyAP{F<(WJDuw(hJA16oWRhsQc9h^{_fhyA zhj06G_R|+Gy!xGe!q>GP`XaS?m!sP0x$ewZ>P#-Zj}AidMYIUmP8`wE4ZKkUerXLT zL5bJyDCWQGa;`hx!H8#JGut=jATt>@nR3le4wIJzG}SDmhN~NCSln7{K;1pzt_Qlu zjdhGh_%c^O)pexiAoZ&8%h1=QH+JHEYLMS?z00qs`n+$_W5VDV?i<#=Wo3na-eSSz zvc9|LW~~_$><&%{`d53n<*Y@mpb~e-tfu-CsGd?;u6P9E3Cv*8bPwf^L%$*XEI z-MPVUc!6b@U%KnVaGUYzHdP929ov_gvF+8yiVyA&N;Jr{f(j-N!puP0eE9 z@)ucBc9noNSw+DG;CEnXXtt&B7bz3jmzOAtCfM&a`TnX+I)t2YozJ)6@^Na;rTUbW z8Io-gtXpB4*04>KAJ4e&TLy@3PJ2MdUvTa{Tu09;U1gE`dIiN477?-T^#M_9C=+|WA@nclvQe>n$nWNtfLNn6 z;B>+YgWZ-tEU}svAOcaI=*}sR_o!KGy9m>Jg2zL z(a?dwiR7#Ui2X-tCt-K$T45)Gt4!KUcYE7i(&wY%JvHY+zDhP@up6l@@<`Ilz$N%O zQHbqf6W8CkPi?|F7@u^+F0NQ!r~Uxqwc71C5Wjxr{h6nQ8aaj+_>~#Xam0OpHtXB| zAm;NJ!r%ZZ(^BJ73CzVWB+;R0X6OXmbf;`i0wKv7*rEM{@v#>67E{lE$D$MMB292dFNaU<40z8a0Tn@Fsr?~uT00;GRE?iIY@GC+wZktnJ(xopTjM8@an!}cXezf z4FeYFS8vtO-)p&ko)RL%=EvYZb|P}}9W0hqOYG!O=Qo5kc~f9X=d^%7%+w~V7CB|j zR9tOW!UxRd(tUb7G`fB`Y2@0au&@_jvlDOnnMocvBAo#Pqg2v1~QClQ0A4{QlH+vyZAzFq_VP3%xef!yXqJ4U4AwM*=1S@yXB0{!BK6g@i zi;4Hv%iL)o1glGr`HR9m9EW7f+|-d+GX~$gqrgc#%E)PgcfLePN?@S$JiWP5;s%!C z&X6MHBsKFx_;2=vXjCGO2A2s?*IYkW#f~$1>FUuL^{HR_@KqISmg+j;ZsBL=!>^${ zDJ#&?;rV7ZeKh?!f!xz}lSGnL!@EaujW6@@s+q}_UUI%)a4$+dR+Q7BYW%ku|0MdF z()WFwvqJ;vS)#RlwN;e%Z^NF|bfkK$UKMk1i^DUWoES%IbQ0Pd8HV5sP(a`^x2I%3 zwdLjKmd_5|as%U^fllULO}@&bfPbqKUmdK`i0`MncLCXIeh|KkiE+G({)gFzQUIrh>(Nfu9 z7V%0bMUco`H}b_~vVrvnPc>6;SiDDfw`BUZU)-2&HNpn%M*r3xUDIz%98Zo)`tW~VO``VbB(RFN~ix%X`gK>jp!xpN0l=qkW0kb z-A3;tM*rENMl`_7z5#Orng&1i3D%rwcNoG+$m)P3Kakrf83Thmru2`TFYNLs>QH zmZ0CW{Pk{yqc^$8wbE1EbhQpwFjP2~>Z+3~s$X5X$M!DjoU{6Hr4>>T;9^L6obr3= z@L-e<-;tJR4=16Emx0lSyqj2Nk{OVq{Rh9ifpvx07!(iv$Tmd+Zvg&f(Z)b1KclA2{1M(FSFFYUI0ZVX^eAByqe#pjLQWFw9^Fjcc4*@3~NQ z)ErV4>wQ2N2R_Zq{)Uek;29%G5ftj5i+h9o&+#&qSA&sjbfjEuWwXaupMaLmrhWBt zT*#oVN$w^}7|L7;25Nh({70RV6Ip#Yu|ex<-v`4y!dA!3k-0p!Y^3Yx@nU?AG66|< zJ6zT!wuE_Z?sF5$$B5rB4Ewx<*n`tz()^rZv70IyYWuoACWs_1I2FiQfn#GViE{+LD6g-p{}EzZRnFNvr3Zjtwykh*Q3sJ z+^T0BivQu;oA!uhK)sQ*kmfoA1AcJMD<3$1srh*Nj!mr^L#x+kay>UuDPP7oq8LkrdK7c$Jp7!xu^O{=ozC3LR1*$CaXCjYe6c z!%&JM8jz?TGM|Iy+QTvw$-{_WJ0-$E2hhNb=F&s5k63hoTH49k+8%K;+Bp5XwqFF< zcMbED8fV%1Aj&W)!b+mkL9SLgHe?mmL>z17s9^I~ z+T0NzPaG{q6iOBtg%_jck@`V@tFUDRauUEiCLq)_e#}!!>3joIulJ6JBV{}`8j^h^ zRSI_Dtz%}R+wU}G$nWJn|NS2-4=wNj5lBG|Au;f0mx$Y?!>Rljv)=wSN6L_yXb^Rx zhFlWO7(Q0bGzUxwkIBmhen`pmkyf?$`U*S1d-0LgEa#M=YSGAdVG?@&Mu64ry`TBC z3Ut_VYxAWaDc^A|Cr%b#E};s%_^I=SX!-#Z<q4r3fqg|%HNiF^+ApM zo^}~<;#*iOjG~bhA@7G)LkZKRO)ba_SYMk-NiRaA!D1r+f%k-a(qD~Adt;n~Z*fUs z0WM@{ao|{wY^?##uiT+eRwOb*L!_V5qJ0Q6Nz5M0Xu$lB?vg;H#D2) zuWsg<0&ASs;wwB%+m8>w`u*?6GEVM3|Lh8?6xJQk$Qt4MsaA*>tbXTRxhFffCe&Zv zfRf+s1ICGSAGFKv-~0hk_FtVV$LC3j%CW+f@tUKfPF!sFt5&MmJXtq_c1dAyAXfmG z@wE-53?+Bu*151iXGHHcR)x6v@8nBZEQZn$#&0B}mK-uyj}jZO@wv$SGL)jRiV_-U zzO~-Vy7ehCKp}AZ^l|`K15O#k-)mm}BhOA*NJfvvd6kB#`HK!eyA_pP=Z`mX$Wq`~ zP|R^TFTd~|CoRq7Gydr&UazlgOFomKWR;l)(g&B28H4}5fAp8gWe1z!g&4A7vm}PE9zX*qC9SpleiqXfau`=rz^kd|R z6IS-Zk>!fG9Bz_IX&@Na()gHu`n1W;%zbj4nzerJp8yE$_&D0(;7jf}@rXbkILB5B ztGk0W6e7PoPR;#xv;m+rKUnG-ITqnQNM9MK&|z+*^y-Q6vWW(gol7MI(lx^H0wftP z1ROY8N!=zdcHs@Hs`pdMA)Vo6gFPNMvqQh!fc1+&**+gh;w<7y7?;BW z!F!QJBAkP84_10VIQXRM>D5k?yX_I-mw=MI98CR9Nw3ROrLr4w14lS~DrBf-D;}+Y z$YmNBNOy!5X>vg_!l{h>Py!MjZD8eU+>yOphZ7%zUPoZ8fAS7zt8f8#g>8G3>iqkm zr99COPix=d#D8AG#{viQ%g~#^9Nl<);s0m>{(_k3I&}0AWeiyUkd=m_6bA3N%dSv$ zEMCeZUK3kfMYLC7V|#TS&vu7G9VtQsB#6}CPXERW+Pkf`cz&<~xOgAX=ot`(A=3=B z>wl(^1>!pDv$xRkPh~an#J=~I7sT{$_1E+62v{SP9Yx<&9&VzZ)*?8jN84L>E006{ zW$x16Md}&kSGh7BI8UbaP?LtWkF z)Ze#VieAyJ>G`2sA{)(J3u;?`UyVJ|X{!>D{l;=-Ns`M2o_~QGRxGF%;bURk=_FSG zsxB=FDi&mn6VUADou@Q3m)ez$K9p-tS)=-fwF=E42nn^ z*gE3Dnna~v-yCSalHW>lQY7%Q-8~VmX40#li;Jcg37+(#Ebn3!w-F|88({DAH6&QWVM}bAjY(R1kh#m z$y_MslG*>syF%BdUZ!f&RbzXAebEy7o$!N}Uxk1?|C$%i<1Ew(Gg{LAcbl^P-*6gT z>b~%ND&q$=*BbGQ|G1Gtvun|l9*eydQ#m5k&Ooj@pZ43XZ>b^FlX$-ac+_*GI-O+J zuJ{Fi8MsG0f`wOL)=jE)PNO!>1oaAIVlDB?^#2C?pP&P#W=TI?qQoz;w+z0k<)qz+}T!B)L&v(~yc zWO~LsqZM^v6q-M}!tBc%t(gDWA6aNr=-j>%PxUU_@4R5av(ez|GZi$*rfHm&j`P~d zjw$!EbU)wjK;I3!y^90N{_9}Rv4W9SrhW);)Y%D3SjYEV>gfSMx=VH+wiaK#h_8Qh zo`dCD+(%WMYw*x{jLP0dtM=Q?m%!)ojlgLbTl8nH8Biw2Bt`r?bY|bm>0tIfH1O&b z#P?HL833*ddi?@;HRqrswtaU?ONyYGX?@==8_#9DqB+%mZuki||Ktb@ZTdX>e8rwe z2(5&|VBX9ER_-l1ABnTiib3m|3yV!QrDjg2vL1!P`FuKd>q4@;Db49O}UK=yJ90*l`y8m~PoKcGC6ol5r?bx6D$6-S<)v?ZURC6sc4F|={jo0EKrNCy&2?=2TGR<=uY2y3MvSd8pRp`+#`3BJilJj zQHkZ3?zVo-5Yx;Yn2pd4)`~Td?#z21cpnJntMFy*I+o^V`S<1FV||8=LH1k{j;)&-9Q7a*8HnSMy6loVSfT_&&y|X?kr=6DG#&j zDL$pqa8z+u-k3?(3Q8+_{*?gVn-pb@?E667>75_VnCY#^ayNdJ|hw!yt2yD#bHVMPH&`8t!LOr;b zKwYrCBGsa4M1L}%xj^V2u%Cz}9vPc8-+n=Dqrs)c{LWrzZ0Kml8MccW5VzF+P@!<7 zQ5{vc7n%hE*KX5Mp@H;@C00G`@VNn;b3fp995cS4d}U7Z-7xD0OIT8aAzBkLUp+*s zp3$bJza5DZ!h`0wQ&Rl>B?&~yyNTW_O zx~X%~*L5%ak>OaTlAyKBiSj>rY|9FHH$@XgK(?wA>zluv;5%75kXzhLN0c}Q{mzsq z22lqgb=^L2{9wy1vP;s*Nji65;0X-pGSAC$n}-$%=%?ciN2w@mp-_0}ICN80hc;Qd zB}3fQyA>boKeL2Q8OilO4z@K+31_^)r7%n;X|cPngA|}Aj$jFi;ymlk$WtMHs8m3i zn@y#fu$bOM=Kd7((xn2bYSuf&H(#0O7UV@Pq3AzRLPp0|el>nkA}4 zhSmb;$6W1$*P<#*QX+y;;V`Ranj6CDV&$9I_u`a>9gh!R6gj!hqn`JVXmBGpOAXk# zz>;5Z*N-;Dte`T)4OptrUi^{;cPEQMc)bl}wltbhAQN@u|54Q(6Agv*;1Zd$UunDx z{J41k*?MAIj_b^aL3o~}HZA)8tl5*tEOzC=nPcl~b#=FxUCM8+hI`Xz`X|fkKUpb4 z>Ij-$IDF6fSQF4{+l-ZT>lojUaToEyF%b+3m+(Ra?I5y2HZAT{UjEhk3j{kVQ!!_~ z8-2v9IA1>`qogl8r+wNAu^e}5@?r^yprR_(?Es$g_$KGJ1di_XlPo7BfqgD-=OgDM z@AFq)z|t1Mk+AeAIu0IK1`AMnaovd=RmAOs`0yh|7m#b-%n)F9UM>4HYl6;w@~Hkx z(&XlShdtcRG)OH%6f4DP4b$NU7;2){FzJdphru%$t#pll7m9M2{)yh85BM`o+U;^ad> zI>ExNF{mG83i)b+ec)-MHc}>nnF54c517Dqvjl^dTCfzoCr|;I3A!@*n1qqmVEfXdu`;`uiOzK&#_^KykJq|Ga@4 z7|@tR*@;bOPQf9lZk&86VlwF+aI%dRNG0$=doP0WQo+$11O0KOFp8-E8zFEAL#g9~pB)_rBXc=r-M z&FMr&ejLzht=q}(S{)*OfI@%Jurg_j?lD%^R6#dG5g!jbY-#4}Td1ag%xQfF2UG<^J9pF(Ta zLDrSj!5nGyTRa%mT)&Z;^qEP=aH$;i3G4KOA(H2Y9MA25)u}R-%z3_Zw+wB~Y{>b% zb?avrKuw;7eE%|C6_luW(eT|g?FeGh@nix#-b8zAOvH73tT_JOJW%VYxdQ!*vmY{J zO1NOFQ{Q}QE{+$W!w*~ z92}m;@-AgBLk97`kvegfr`uzCroELTJ&DB}_#lql?L>eCgN-2(Nb_5*D=7J8g^r|c z-@#i{T(2uwt3f&E3`L%RJbvK5Ew{g-$KPx3#ilRq;6|S+c)gAPo%kIXH)eOEhfh}# zbsxV9P#~v>>YuJA-bRY7qN1a(^5UW{z1BX`M(q>OpGTStS+M`lw>t(g#)^KBrcm!_ z0FH9Jp{BxdwS$9gxgb+9N8~n!OG&c}H!yADHa%%GtSX0B?+^KlGFd|bvAN04kr8NT zhNPRJ$`$GVj-v#ZOK-9I70~>}f-$1hTP(wxQ)WVv)HC>D-Gu9m>zS-vk?;yTS7deS zlfT<}R~^R3J^3#biA6tnIN4(;{R=IU*i&&|YvXetz&L~?qaH`GxQ|)W8L6{r>};!9 z%39p2M*%g|&59IF-#x3<&p4jI33|bRBcaz}hrJL;>DU40Zg0%1Z@zcvs!X7SFjopM}KiY0E>8mxou!Y9~8|fBmLK zm6Bx?hD)Q=RpkWUyf|fJwq`Hv>vUPy4bffc;x_ij*vROe)465t+00-GQ(^89(QJR2 zaZS_iUY`Uldx(jaP3v|Ux$H2n;^@_Q053{uBRWiNO3S0Qww6n1dE zL!ubU{a4CtGpghnbynb|83XzqhDC{8&cKxo7+y?1#k16S51hRC}&q{$E0Yb9? z&*`#bYC7{^r)ijx%DokoI-t1*LmqjW;?;v?H&N2xvtwF^iuzxtAnq!jeG&e)CIePO z#s4vv?axkcEt&GPNA`VPa&UHU-6*-x&OucWWsPoLYCF3Xc+_YJy@rpy5!YkGBfk30 zkp?`FZ(#YUny!!ytpmuu{ZB7P3I44^uQrVVLtEl3*6wjbm^Zk`j1Ko-sRe zSYK~{l>%F7(*F8gsdVlwF2w7$nY>Dy4G&F!!*P{rt*Tdo5+yA^CKZAx5>r=f{*6*FT zC?PWqyI=(=-YX@73NLVt`etL((nt(JXg&3V_#maZ@^ zlI8AUvfzK>zLuBT5aZpXT7Ig{H<6N%onGv)jc?CJpS5LRbu+g2_p4y7WJRq zYUjA<#m5pUBVBVI!EFf-C7B3w*4t$sdo@Q1dM-h(_6oHeCwUv1!Ho55l4RR~V>Rct zAKQdm>ToSCaU2}6FH(H%c4r~CFZn&9)xCFS zf+aUl+XSYAJdIq@@eH^n9*%ivHK-h`RfWi?U-B)cWfYQCwR7t+ zr}uK-K%wEFK{T$KD5mhKwmE=+{DO%=qg4uwu9D+8=!{<`Q_!@5-P^r_(GtD?L00jY zuRslxzK&8x7J4~!P6I0$n}|&s&M}2SJD`rU@>jOX6S%n56qv%_hbk6P8rpFKSROKt zK5oBoa(6)g_>5xD8}6f8qIYKH(sj;nEs3hM?$KwKLoGrZ%kP7 zIrnV8xiUNX!)gO{ZPYMIG3VDlE=MsZtBZdxWj`YpG5BBeA5*T_Goo=$VRgK6;yIt> zjbz-?rvsBE-{X0<0m?%T4=Dun(Y9B0)){sC&LD{kdme+URw?^YztJaLdYIepTmnHf zdK7&>aIZq}Kus48kg1Q!&3U31u3w;lM!mZQd6u31I6G(udPW+AaW~w0c5o7v6pb~^ zg?Y%78C^fLWC_5Q%DFdmZR2up|L!I0r0nf>y(P@euyB_9uGs0QY_=|6IoOelX2a`g zBukmbS^pO}>3W~yL84I8aW#YZty6{|`w6}a0w&`^^Ew|@6|Ma?3I6PsycabZqVN5q zs*3nng-u5@tQ5f{8Rc__*-MvftV5d;^l-qx=JO}dYme?FOU1@jJW@@)>`iJej6wdEAvdScD+?pr(&q<-+0&Gg|0`E6RK2o$7*t@!nVCXzUFn&r%&mg4-((n zbG^uus}LnQ;}R|=L5GhGREY%^Sah<#-dzxFRr5yB$LzVNhFQH*(Sk2h%KhE}B#f7& zr!-7}NZg%QzGP-Vn)8EfSK`CFXS>s@-K_}(!nM?+jzM+mefw*8WffkTfxgdye7>n~ zFGcm&s+ya99c)QE0*6GN<45r}CSpKvcT*iN^cPh(*P1viZ~qFz1|)(e_aGYo*MXl8f4(42^w zOglca@~ossRn4~B=0B5ro4RKW{}Fqfe&nAu2qU@eZ!ZtmKJIv*R)|dITJjvhGCGdU zrS46UR#J|GgqirxmgnF%58r4xg84Pb?4Y~-)Ygw4E%$oAc*AZD!&I(ojy#tWBKdt6 z$Kd;KZ^BY+A*OjH-aNqQ!@`Bmg(nQsZmahYunrb}n-K%=?)W z|J?4aI6n5W4(;c{-av(0qQc|;J6yJtH(0qqS+!47-y5HY>dZkPcn2Levw)0n2>zVdi(`)%-N=>asT*S+(kVh+@#bR>$sA95RSBITrL&UMnvq9_`} z9*9}I``lAAugw?thZS7T3r0{h<}QuP+G__h<5x4=d>`%qfkYGmeTrN%PzRxWjX*Lc z>6Qjcq?_I$L!G7#d2X&gY#;Vq94cDJZ0RUXK>7KmY+UCC($&0luh!#UYQ<~LrCAos zT*KF=D}R6HDQ9X}O=OM?VsWN%0WLDk)&6v+w5e%)&&IFiD`}n=;VrTbpNm_GE+st? zm~G}E!xS)oGWI7lRHh!a`wERXvCDGu)!48_zf*SBF{RkuxD8pt|-I`h3~x-q$@ zq<%(ZFT9>A)yoCTo_9V+n8;!ekn(ng)^q;4dL%51XZ)!TIy{`Qia8t_r;UvlCciZe z=w@G%D8(9NB*4-h$m66*J)KN~>FIBq2}Zh;0zV&ZpkAVUK2MxV>91-Q3xB`5zyDMD z&V+xB(8GIjHYyS_Y@4Os0I(F8KSZN+U9ZgnbHI@w;h5(&u3M=`NZMc+>$mh20OB?a z{<~#SuF(5?spzs=)vboCs zd+00gt&Wa3uZbpFk~p1%#Qa*B&79#Rf~7y?c0{uxeHVJu;%~Mcru|g` z>{goOc?NYZdm7r}$-8Wj(lT-HCgv1nyoZ?BSN&(|wv;Pjb?SIwJwr(|<+hJP!2OQD z)lU5j)y-r18fKYRM-Batb<}`gR>xQGuTt3p4{y0d0W-P1o;86i+eA!*Jga1?Dby*j z;7sO6O`?$CU#d}XDm3;-UsJ1Q-O;gq+TxMR-_x_)>h+&Dhn!}k)@~wZscldmi|C&R zkuJ5O3=u5)wYjE(_1J@x?Yakf@9Yc zi~442U*tb!wxz1?szq`7nv)07e7RpA3w;rzP`5P1;$DW;B4m`-i|WiLi{PExTNOgN z#TwDOlNYzjSXWWtoHW_IYg!EyVqUf9SwAXxnXak8uj4SED!@CFBSgZ1zHb{Vi06%1>^@mdmH2FTyDJHHz>p=2k2kw-bGi{1$ zWO?^!1CWh~>rAd@_%j-V_#-kqx6Bg?RJS2>rwE>$f(K zKDzJaCskv~_%Y!4lO-VAL=>IgTq-^7Rq`r>ASZ4MZXXbXv$=5o#Q#UsTgNr|hVB1= z43MexKv80#|EGv)J@F|J%*xgPB73GJFj;rX*K62$XpKUs%P>sRZo-V5VsZ#X&VaPE(Aysy_Ye38UN3?`@cNcw*)J5} z9(+xRE4#`B>di5AR5$2u_aC_GX*t6osb;L#|V-3qv=nRU|bblRoWdGMKwRIcy zlgZ-J^237pZ#OcOi=fZA@erpkI)Cn{8GZhQCti_P&}XR{?&0sP1R|ZOpQxAPsxs3s zM>W9y+h$l!?}vY>I7TE2WVU05e~bxg@d|L{WlVhyr-k2~4qynr6p0>u576o}pv`qi%|M z4A)>9Xd_v7I_8Hez~zSoAM#P4r?N1)iZR;jI3SbU+Q99L_q04^&2qlzKP=cu?9b=Z zo*uRMk$f@!UnFkX6rSMc;6fS}PFRU1S?BZVP6 zeda*Q-o~X5p@Q~c5z;>5wwB>W$MAw0gW0TGWb;bRj{kXQZh>6WZ()d=stJ6Xp60&S zmT#O#XOOVh;AVI{MqRKoQ)h6yaTti;*!uO>f2IdDyG$xL$wY*oJGfM4l_>^;OEnKR znYd$wrnyN3bmno6E!3rl@gKa7fb3`u!-Nm7PwxeA6}5fD@oz{oj3<3?g^cz8P~2{1Q-ZEo@`n~j8?YCA#(CUyaVp8-0F6| z{5@d+{3a$;Cs6C0`Jaz&;TRn|BYVaqQA(ur2uav`Eq;ZW3)GM zfgZ5QFjbzyP&IcoMTeyp=^I~D+p6f!njS$QSN4gVI90UJ0VqCJ5Gn+^!_*y6dHl~z zDH%G8Sbl8WsZYJbVwxC!4^4%hYEcvnm)HeXQ!=B7`x{2#IN*lU7&M8$*K-R`jK7_H z_AYHyVG9=R*}GF)7k(U`oe8fZKxnX-jMsn^G6irArT*MLusI$9uy? zY1tyLCkEQ@^hrRwRAP zf$kRogSR6}CZuY#;&5)KBvQh(ILSTpSwbdD$_%R;t-x4L%%F*f(f!{o{00By|JrDh zM?2weoTxA&+3eNs%~zBnYrNsq(5&h0hFFc$L~69wp*L3hU&wVAxWTdz1Idis2|&f@ z#|z8aG?w}WJVa4xo*czwmN6T}RQlDe49RQO+jG(~Z9YdE^S@+r8dVZXmr1G0PP&6OJ{`H_DM04&AXVGS;-x9xNdahdm`03E(Yr>CLCRv9I z-D;^=uUJOpUC>za!saJ?^cL}z1;{?n?`KTVr1>eUk(CYrS`%YNj6B7=@o_GCfj1dm z!sju=cJS-zIDp5xQ{6T;F}*V3BPX9R?hEfkn&QJgJI6MST$$$oKP^CW#fZ(bAUWxm zuJJ8g*#qCW>fSC*lt_)MH~2VWYTZKYMOR147rea8C$ke$1N=WYP52)n8tNRywq~B! ztj51KYVnLy^uX#_mi{+1$N4tkA$}6qpYRTK9SO6d`?xUh5TLS3goXg(Op%e$K6!m| z#j*Pqj7xNgZj*8RZM1UUIE5FSA15hi)fcJ66RTSoF3CR~s(4%y_U<*9&iqtCPqj1k zN#sUN8&LPcraOLYjVki}ige7hFAA7?A z)-@3Mz#D6xPJEgc_|3qZ#Ox_GFsV_jzfWQ;O!I1QZT(={D60kq()@!>*t5!B*Rsov z$os_&EwFk#lHCiCQ+Jdtu#O_*6s}l1!@O#goI`2$<8N+~^~podwV8-3sM?!LahZt8 zl;_8Y8*$OiJJL?+VR#M~q9Wz2+Lg;WDSQfsVRyd;yzgcj5hdLDvW{G%%u43xi={&4 zTE#$qG9CQ9`Y%D173FKH^4(B;AGB0P=>nR5FE^b6@Urz8Y+xdI5>8h~42Q(KNa9;X zPXn{ovLc%=$kj+C@hAVis*@6F&j`3r9_e_1a$2v`Q{)-=pOQO~(Co$V?EpU#JLvoC zS^~4PRU(_#^jy!R#eMxZi6rci^K|p{0iqAI4lAwL7-zZwhsb3<8Q7A6PnE?B%WI}C zphp$TT?iP@;vl)=`eTXf8ndZWh~FOxeUC}%xf|3wi&}Uk2g)w5wa_OW#Phgd9~f=a*Wn^WgkM}~$fWm%28>Tv{yJr@bs|n2{2S6{^UAIOi1%NQ z9tDpGZy(?Di%7|R31+QBjh{VLoxXMS>^eKWcS zsaQ-?%r3J(aKT{ik5+_T+tZB;(|+kR{9``U_wx2wxqXiy-=xrnsnn=Y*K*y$rs_cUVj|T9arn9SSx`5{T6(e`hCAl_uXGif9n50yHJcR)n ze*~9hX%}d#(6gT6!!i+7DJ&@q^j=V$ho0*-on^#05206M98iLIe=Wc;Jp}F@hh-5ujZ_pl zjfE4pwqUU#HO=z~Fc#zwz!p7Bm5#Z6(WLrtSKH|(XrvW|ERPyBO0RI`VuqV9T@+T+ zY*~jHR1Y$N*|DTQ&y7qBR1T~3{zoBSAqYh2z?$qGx&j`ZCx0(MRJ6L_2~~StbuF+L ziUNXmxu+7jpnUT6#M+!)DRCx72K@IO$_BwZ(UCEFIs&uE8#8EHk?4(;_Qdjq-9B{Z zq5ZXPqZA>MTp$18|7u{B2XxZapC1^N)j?htnP7Y+RtqiYj;W1PU&_aTv}L<(SWc4N zUT@N0bRRb6bCu`<+u+{<$k@Ln#;tcuO$`&CBxllFgwv$v6dQ&@qoloG0T!f19;-Mt1(BXl2j{ zRT0$yQ$Y7TW*^|!#(Eg@b9GHhok^H<&?fk9Qp+4W-{Z>hp#&SX}snN8&bcz`arbNeLa)&}JcW_ujC&twjz7 zOguL}%Ax@PgfEiKAt-xsXlYc{qOA~@IK^xVk}krM#*V(u>KLgFV9c%AhG z%tGl+j}rmO?O4OKU+;L(CLz8hH>$w(JeV|grdXv$?BbNc*7gzC)|YUbJ!g&Zcm)*; z5hFJ}@ZpJUU0kR^n0p|d3VXS zgq6^J(r;AJnNh=*Lgj*OC^3yJ#Lt9*eH7OdO2b?gC-UFZVoVPT$)^y>iVPS`t~VUW zSNc%75fNWY{RyI}gWjUE(R@w(m{%D_(Zr}kzOvVZ#U}q|l(Hu8R0oQmW#M}Y`+Wx; zZS|Q6o?V$Z*BT9{URMeOTU&XD4dGLdwMWXKIJN6SB&4rf38G8uH7@bFbZD6^&VES( z$N@QCuFt(p>P46pfwr}ows=a)<3Fycv6o)azV*@%Uso=j+itlmG}S(m?kKT@j;RI6 z&g=R8IRWpTWKFv-p?o)8F5!`17ceEPU;c3P?YoYUx?>{)0Z~OkwxyDyRGGNp(a~d7 zWA~YkF)z587NRezoVz>lg3}uuh8UG|DR0O%)jzAN-1rp8{4d=?@Z1EVw5Z8UKQgSw zG~!L%(*u;)9`h>VVOJ4q&-18;r(5b4+Yi$GGmM$1+?x4CM(nDP|il%FDU70c+)|s+KwdcA#$i_QWOa}UjZzzHU`Z-QatlDS+WLv+vT z4Om%N-D&e8G2jG2{W6sXgqg@Nr0`ECgtW7Xaono{lEHt{43BSJ`(2#u3rZVaYx6zF z@nA2pl$X`(zg+)yrsCGSm=0L~RsR1mu_y6(Raa+cBYY0 zUtx8&UVXbGRNE_Y|AHW+?n2@C2bL>{*Ah!_`ccxrnHy)*zJ*%A^YKA+qyxk;rBFhhOw=mD)zqXj?yQb zXNv$;%g2F0ENDecoL*D67n;u$W*|KL9WWYZ(0LXT_D=T{#(05Tjx}`Z9}?S+vPE@Q zy1(540XnGie#-|ROwk)@bzHnBQnQi62%-P>!++7Ud0s1+bWV|Nn#g&&Bhnr}!GRaX zndIujn9)Ir%H@g9-^oSRa|;~e+h>QW1&0j;J|J)8T8EDZpN>juSULVosAKa_e2o+L z)uWHxDN$;IgS^zbQXhaP=LIl>;cP?>0>KRCN^5l_LF7PiFy_gBAbhdZx)h zK-3n~(xb@cv^pg6+X(%AsM$7@@5#!xdBX{29kv$b`TP3FdBg9Dng+AEuVEHFOpQfR zn~G&CfvR44q_LTafpL0B=gEwRYsXyuj37O_F}U|Be%;7}YUQ4O_Zy9k8uk_Y7txlz zD57+xTW}{)-y_ivw4Qb84vK2e;fe3hgIKXe#3i(Zw4mzhwamp#8;4N0T?S+2YdT3d z+n+ZXHgJKBVFLWIEvV{-u-9zrn!GaEpt=4XaIrWwSA!r$?FF0Ea1WWT+lGrtn3)IhJntPx6=9yM|jZ4L{=)b~H%> zsxjWz#yCvDr=CpGX8TtQ-$l-=PZzxi0__uTXdB|*Q|FCCp*Xip!@BdskDb%E#ILW? zU>pjqG${>cy5(3OSPsz~BU&aiL*_;h`CHu$w$tk2LK;@=+pu24h*iBe1KZzO_%Ro~ zNR6W9W)x7WWP~vod!{2}$D3BsuY6dJ78aWSB2;@#8LM!>N*FhX4+&1P$cXt9@-ZNc(eNsY< zPDX)O>(yyXQ_yb*no;BZRQ=B}b7c@fjI8iETtm8$|A0Ise%pY7NTB|GR|;e4kE<=Z z!#^LbysEmY9@uL4FuSO)^t6$Zs&YIt?>XDrq(hk2(@r@e;m&-zcFPElF~ml%YuS`g zFY^&+3sNh1GX(3xf?#*yRe_@lx)LR8*<#T{6PZTS*XD(Ot~|5Oxb%IDhiRb6D60_$ z1s|5I6yo$j<21aY_GPK;BB5;fMfBkb2rKz_abqr77sfhnT9FID7t0tcRTKz;qwvD! zJg7P{((}ezf)VWs0#L~Z{q_8x0YjgE+Szqg z(lbLSJ+{`_PU7hfOvJ2vnf2)toaIe(Y~9hGhGcH@@~5nC5TnR(%b*>__!Er}{x#F0 z18aTA(^+c)Ym&BF5uITPzdnzSsS{n8ny^V1l;&g!#Ca}%)wlvN219>5e+%)8eMC|g z=5yzbFV?Hi6W&|%*%TT!7ogoGbN-A-=G_1L4%b&2e{=j6ySd2!jpOqZt@BB_EFGbsW<`$boVkYf2fA{uQ3-HB);`Vl^JI<&Q z<@-lc9AS-QBmbr9n8j*yH?QK^$qwPhAKw@=lhKG`3jTCxnNrNAELEiKge$ z*61#QXEj2U52ETD8jG{_nr@`~IKFRzHQs?~NJL@_Jh7HQRyB_f3QX+jgS{0@BMHJT z936T%pU$6m{l^IbYCrDR!L7sb%^LJ{wby#SUJ~4aRZZ;8uiYn@8e#}ktY~uQzTVy; zstC|g7-Xr< z1l}8a)=mT4{i$)r-F(SpG)iPI-ds5gF76O?qLt|dFW`eyvASQf2L$qxAj2CNvX}ej zP>^z6umdw_7u}7irjbE6c2`%AAr5zuyd zsM5_4Zrk6f22xB+x@0C~Mr5c0lRuAsTdvB3w3vPPrd-;m6YT zBWVWV@1Dk1L=vY~ZDDP(oJbmIwpH((oqG{K6?@lX6GxwQz6vxD*ZWYt34lv(a4dI> zV6JiA2A0;vFKfk#*DYqTS!u!z>i96}o5mri**#Z)zn5(AuVhgYUiW$DMk@dFu=*IQ0ct7Wg7d8Ym`?g_{ z)VedY)P}==Z2>vEQ?EH3+;jI6t^|WE*d!^$-Pm<`Nzt6TwE@S)vQiS%20;Xt0R5%{ zBymo=6JT_J79o?SuDtW}%WW8-&LIqiB>bMiO925RSFp@zHu-uh3i!$80y=%|-##$; zkDj^pcQRm~rN~g(L1%rs_*AxhWS!Jre$oLNmAzX@7hNofCgQzKuiUb2p$X%b*n}$Q z4ZyzxFrXL^lS#>9tMmiqIS0$L{AgfoMT)>_QdDe14VS%YglTJpwh*smAa(`ooA^bv zJ)zmtoi(X69T@yUalpyHi6C9=HcMh{c7A1pmiz791L@Y^nZCY9TY-AeHZYH%M8#>R zg12YfXj9`~;CphkDf=vJB?!Vvm3Qq#yuq$-am9Xr2PSSO(!O!k&+aR!?(eMSJ4NaQ zOKau1Z}J*vFU(q`ioZU49I))SsjOyyzdI-CQ>Pct!|sLd=#SjqcVp<2zn6HZ#}{17 zk!(@9&3kKYDs7Ll%p)C_sM~`4F^A#Z3c7cVKPx?yovz9^8Cq5S#_>ZIiO%G?y!TUE zm-m0pdEdZT8|el2@<_?P`79JMKM_S>TBev(=nV^En`iOp+RQNvkaWeqB>x?bs`LqCN=zaVGvfZ(jQG)3DW)9w9B&giY?dihNZ4rZ;KJs<*FQ>$90Ffn<8a z4gby&0`%2C9M-({=SiV1w!Nz6XCC*Wz-L&dPzG~0+ih^W1tJl!xjJ8muX*66VN18= zwuO$mGc%~PyO@P@OYAx0{+z;hhM~O;TL~4trC~ogTYJWhHK}fUZ@gkW%fi3=>vJKlC6a9(yJ%2$D}rK*|-zoh%?|MK^AND!y$-poN%T!EeJj8tug&% z2;V(q@o#8R%$>Vnb8Rq0m#`iVTRYx6n>pKG^YlzRC5*o2K;&fmN&F0bJaA@q+A%*Y z`&vq4wi4(nrfQ$^C$L#yHUCxFWAI4A4n0U9leh=ssQta2%?CZ_@zdjEsKotv(YZ zbnn%A!(EYTBPR>?c3XrEi0FeMfTJ-1WP|8eNbeeT3;N*PV9rwx$!_$PE1%!WEDlYLGx^sE9 z5@1<+zO%?d%@BL2elYhQ{D70qpkg>d%E}m}R{+wXt3GJ)# z@psz(a)Fdc(vHqaYqb-Fi$vlkRK-z`*PA5E5T7CE1q}EQ|OX^HvIFLI?aCDDo*m{!Q-TO$!hy2iI8RgyA|;`vizQ^@{LE1|d){fg9i? zN@Cq7HtFI|S^S=!cM78Bp@TbU1s`1>)~y-E+%{JXtC_OaB8DL3X9=h3bN)PSopH~%Fr7s1MW73)FR?oab>htvy5;DTu2GdEk&I#Kz!k*$a)2J z^_=6zn+!xMc}@VP2Js0-%qqk){_YTciOqywk8W0>7isuBXKMO*LJ`b zWo__=bigGJFJq5W6i_Xen22ir{&(VLO-Yh+4)S%7Ian-zu>RQJbntLO!>fUnc8WG~ zxP@rP>LNehM8(>|CPIqGAR2pOpy5@T@-Z(3ay9CB?iuRi1-K>B-*`QPb7nH;0nhnW z?Kp6=qd0_|Gg4J`(ek9pI`h)DcBUE74KR+~lrM_!g)z7=%-+})IWk^){hvm;x&egj ztrY0k#(W;ntSa$_asq33QBhhhPB{u;wMW%P%AS}j9qeS`*U@JFK#cR)NWx|8b@j74yoG>tQOW42wiTM&90Bry)8>jp4%$ZD2MI zi4x6uND8-1sUtpJYXfsG^i8zd>{pL>nw{Ug$6|Qin_2;GR^i93+qS+oHdtD~c5Xs7P`N?#n5 z{5Ry)Pi`GE0pQ^Ak=QS@bUSQBb=#}vIjNHagY!yhAezIR7={z$jsY=&Z1e&JI5*L=onuXXYI5G6v%y4uav5Th>8-r^wo-GqM2_oYErEveSH3I zQ*T)<(*@=_U9EgjfAT)WCU3()$M;r`>thK&5X=23Lbd(lEC_5&7T8e)dSzWs>SmO| zR;!{vc|TkMW)VvF@z;U7OSgFT1_Io=waHsdXKwF4E_pGViHN?- z>IF8S%5U32jNOdq*MF7kfh(C=rP)0JAkbWnTH9Fyp2F7TIxc5mFR(5}`|IcpQ`ijApeX&l1TsX|`$)g#-hC~_zd7?nDh7^afIa}?P1$%0|- z3gg#aZ;;gI12nW@VBQzag4QBAM59Cw?-U#6{G7Y}gNR+*^868q@INgr>o%%8bfK+T zw5uuRDD-qBK0F)i8RnSKk2;FlXeYcZJanl%WjgCQiQ3a&p*9^|g`Uh~PbU7FHRquB z_qy(zt)CUo509U5c{@XW*H7@Ep?_#%Q-*Xr(v030a{r3C(FEOgQ)jS#5dr@e@$<04bUf)-oPB&N_M{0u+{b#$b zM^gB>4&k zn120%GAd*VUq1erK*SF>T~U#Je#2qovNME*6@ai|jm;*bEWBDLa50Dl@szc1m%txw z$@Kg5#AmM5`s~w5Ac%OXk*Rwfbywrh+gydpx9s=pxdYhb{(NH!@VVz8Hf;GbRVIIb zy#h@zsdV@mbx?eT9^yPFd7oqq)qtSoOf-{@-Q#Z6ZH z`eX_5YH!#+ip(Us-XMKEGjKVPpCm-G~m7U)YX$# zd!%t7PHrn5Iw;SQsqj*7aY5H%R9fIFGXz@l zDZjbc0NbA^pSqq-d(?ALjM7*b_1qKtyvEK!USlZtQl4KZpijtrk){F%^UdL&{SRO5 z!45^^xiBHPBOC+d3cGVz(dG1~Fk~^GYst)>oygf*maJ`T2~qP<;I;+y`@=hEdFRra z-%}!Y&{~Bd+*yd>Hp7x^5Gct!1eZa}*HF-zL!Ds`fw2AyUh=*h{i;2GMv1ryk#lxI z;qwR@L@M!N3Gv0+Ee>$9^jS!_j&|9a@1j8shc_^CuIG^MbK98dGEmgKD3}>y@dm&v z?gj&cRGL8;1UMUHL632A`Wg^P$T-B7AOU26%0a#Z%>dZf3K>`>K^sbZ*uXvA_vPkK z+nRt$Q{NlP#60zaYFwcj8VseY@;pC0?d2sVxY@sTuX_L9d1Ek;k%I|kCPt913{qkV zF|8-v1l#2gABB>%`-cvj-_V8w{uCCmyMj{fT&Y-s~wl zM1`5=j0&|cTAkq=3#(6Wh^IGKqr?@>BDgA}b08qJ=~LU+-!;E$p4z^uNpXsAowu&& z%G0GV)v5V&x#BfdI_-nL54VAqkaM6#anLsS++KiSRfHX4K`M3WRv_((A19~QE`Y?9 zvb#Oxd{w*+967TjLt(@sMGV75k(bI=U~g^h(9vlL$3Ra(coyzfHXAKMXiDXCoem+dOBvs|aH}@~#+FFL9=}P!O(d3l=}@lRIdsPyfJaQW|wxj1uwWdPd9NmeHJgg!wP zRF;MXq_#EE1*CG2W%*8ns*4s>8@C+4Go(4>#J}%q1_mT9o0c&EGXJ)M$X)D=Az6{9 z5w}uqv_C~NV7Fl(FGXHI_}B=0sJi(J026B%d}`zLTG1xKGYIoeL$Ps7*7o#yFdCCV zbGu@NL1BC*!9%5d*h>wJ#L6IPP)d#^_IfNBCV6N{;)!Kwl4^O60d28|1C`!=wA{A5WTx%NYxgPGqLz!$d+PMbR6r$SK4FmhPrBoO3zs=T9IT zn%sfWM6$gEV~fDDNLS62_Zq-~T!p+-5l8xk4^2Rqy9;vG<5)9=WhkvOt03V-|XFQYH z8%_R;dm2Goee*N&!#WK33#JhieY9di!NfG34)Id*<*L3O$;P`eq2xqJnA)NgzcMhl}jU`LZY zc(8ZyS)y!|f``XYy_%U%%I?Tjh1Q-8I7!SnRr+yJhAo7>rfMP(A5mE ztBgrbBmO>NxG21Y0GODJ>nB32XRUQ6AV7vxeH0BVNpRRObaOUyK`=s#Gma+2oSX=`Ym~tKQ^M_kBlbW&S zd+vteih}Aza*TO{@<{Ql7EeliG9yi;{ze*m zVm020K)@43G=6;b_4QIlV2Wa_fkmf|16 zBQTRgx<3Brj6y$vk^7O?d8kYfNWiEi5c%FJph6#zdYACsI>w`AkpEhGjj>Q25NXY^ z%4mjOs(9jSUS8L^Q8_VG5FSzr(F!|86kJ-K1|DJsV=$Ni)Yb=j+zb#g*msn0Dgm;k zSaK91QPfmD2!8x~9jwspAL(}iWX#+7HgBjgZIBDhU(I=kl!$Eyx0pj1aaG&?h_9vx zR!mP&a_-O4UC~tpiOr~6Sv)-B@GL12qt&aVXNCbCGxNH`ZnN)$#FKghCh7=~uY8QD zMui%Jx_%mHBHij9dP&&Li^l!3uQvQg|LBPW zL2omx=91tvcbw~MZ8XSeL8+qRzrrDth?lW6n#e>i>@Rl$yPbrLO={UJ;!*7*9z@Q9 z7=Ot9PqsswsGc~yuhgvAI~@PRbt>7wxCN!)~{nq zSiFL3+|p$SW-dbOoAuh$1>CEM)@g7K-jGlFmB;SDKvdlP?K{%N91Hm5-q7qhN;?Lp zamzoyIBg&z{m-xt&)@{-bkau}D@FqR0%`k9rgSalqY5()4HwEKoZAb+XnByd$?%v4V5`|fh-Tu(XzhdS9j zobg<&3UN%3Ud<~kJUL}fwSS?J@iXIE(@cd^mu=FBX~II+hnl-O_jFR_Uqn|`S_laf z{#2D!Ru;ah6x5fBcx0yTRdDj;UJH>gP5Q%p%=N1M;V|K*9u!yt-Iu|xmh8xI5iy4V zW{SqOu_Efm044c)Zp}KPF$N4ke=(6e_LVvQ`D|u$P6Q_2wYFF27K8=FDJ*y)4S(Tk zU5GUR;7bF>+M9|6h+vGCEny>|;=@VZj*uuigJr+9I z&3!kOxS+M_hQnS^WeFY!3y-P)_cr+N@$?ZjBwg;yOdq8LK!hT5#^20PUkNvWIQmny z8fEzX?&fRIM6NdM7l;a^h-1rVpaxPFfr_M<=b_W3(jd%LV}LEcpPX9k2lYyW)}m zys=!3L_Pv}+2rREJkfF)kv{vTBy6CzUHkw4QSJ~BI0{cc@#Fqo#=^oE6HE(&>2+bQ z0YC)mFtYoKs}qt=tnoXFzk((gK+Dw$g<{`&V(I@?*nZd=21Re%v(Iy49jtoHjO105 zbwZ_K5eS8U=!{@E4fgm!hnUjtd#xAbR;B18AVLAvObA@!<;Uv}>>DR%ma`AF!1riC=YYbDYfo;fFc)2v&12b5qaBU+Tv z>|Oxb3vycqGIXh@H+F2zbwUXr=I4z88P#7RsN#XDW4I}@1j$u^^A zz^=EWcn4j|yo><}1D?=|31&goo%cQbm6~mqc1xt7SGO5G=s|c46=p@JKEx zDex7zNPSv3E`2<>3~KjVdgU8yuWjlM*ri%i<=sBo{NyawxY_f&^jT|>No8f_#0=*u zvyq9(*iy|y&;G*8j^+8fdG24zGRihD7@hm0=T+hOVQ8po=x3hs>SSXGf?CRG(YcY& z(?%ZMnBEqT2q%cB8H1}+;Ps-G3wyl{oFAaSeq@8wrKd0lh`N#2goS{u-SvLB5UEg+ zy8_=-e94#>%yMU8@M2YR)lEKM2}i3^5cVi(rXD6}zp}5{OR<<)RCI8MGf4yOPKxT5 z9xarAh%HoPt(bE#sAZu8NbaHOpxtbFRzw3#jTZ0s7HE+S@f8#YP&it81~ee4Bq z1#Ad>7>L2G=0Z3S4r%SZL$)8@>NCA)X+Qk zX(O%rTzXeKzHTbAF^!Zx_dDW*shAv}MjhGPkLMZY;UVJr${luTrr`el?|uR9+RYd) zceiTb3fhE$7LuzvmE!1*VyasojF=B>)!cYw(mH^}f`FO~Fc+-`{~T|Oc!6a zgrqnr(HNhZh_$LCLVHG9kbOHaGPVntl95RT9o9=_x$tV`9Nci(oNzxfS}eBs%hvlS zc8Gzn2{@r4F)(;QDo?h2IROmloGjb z5O7cQ<;IWJCr9*jT8q@Mm(3+yWTCLaJL7_^oD zNpJ&9f?(nFD$t&oF0d5|6dO958ZhSJA?#N#<Eaq!Z)6eEBkQEEygbLnr$}rlzM6 zrmBMmFh7HO>R%mARkPrScqB*S6Q(qprdY>R*Cg_X{ zBAY{CrXM;-eMzrurV|iD;L9vOM|x z2kMRc+eh`ukExqt173Jy7h$!NwTJe{u8j$o`yqK(RxBOliAXzuKqY{lcT4>3k%WOC z3fW%q9SOI8OaNXPzk~d8=GPM$>6DY1oHYOA6*}4D^*v7ieKY@)jn=f&c{-ZFt1`W` z?U|KdD!)X4=kx%jiDsBfuQ_xC59<06kHeHCO}k|;X>n+;kBxa1PGT>tP-Nf$15^>T zs_0>c$I=Wg-3pfkTR`nT^#KJC9hz;Jvl}tS{4dE3GEh8P-w*f3eTp|0gTCu23tcJ+ zDvO>clD_5eiP3Dz+-fEGn&wPEP3v=2_ymj}XzV!yIBv}pq*t9w$R|MrEh$;Er#nVU zDho3T>G5F}h)^6?m8}EX0r1Nms4Egb+B!Hm4n|WnMgs^2C=J`1rE;DpWRFvzfcsG5 z+6`56%Mj8NlHoD`fuGkBtqssJVwA&~6`t72K$wFUsl6K<#1JBK;YA;%1NlpGXP3Dv za<4-WciQ+tS=-K3TC0F;L5cr?UomR`pXWq-vF>V^Vlu*;Q6RTSIwp1iRdOlsEy5Mp zw#|WrN7F^+HhGLY()i9j^&(@h6&cnl{s-WE>kZ{SFVU*-jSlIJb(N)PpWA!kz%Ykl z_6`qVt~FdlQq|*M@bEPHh68AiQEHnWxTlgrz=OOEQ*oXG>2JlhgqqDp=l4Q4w5{)pO2quO{GBf?~`|~_`Mg_eOs)?=R?a$P12I>M*1M zJSU(z4}G!?(=pyIP-K^0Zc&Wv0c)DHTN&p>Dx;@|WPQc+mvopV?Q0}7bgzg$E>DVz&kR#rCIRHu)$q3r{<}tE> zXNSFW3VX{d^65LJQDAlN%dive2f(jYQExCz@Z09``x-}o6uZZ=3d~YtwFU z4o!q06i%L1%c`F)pZUt3_zTVRSzN$S0?mG1*s#q95K?F132in48Sems{hOjC>>9Jc zeNa^}Gr4rjkgBtiFgG$Qw!^WWt2GKHZ(OvR<;2ruhT&NRd9>dML`ePgug7I~)6xfr zZL}83V7MKm$Z+TyZ=M<7V8Tjlx+*Td-XYt3{^XVV|<%BH^JNvDC zF#VO(dx44-2#_FOyC_0hU|+_A`Qky_g)fLy z+ku_3zuyDl^bxg71fMruGGiBIzq7#N-R1`+K=N2@3&_ap%CL2Ab%Y`mgC` zOk7oRN&-)S_8s2uuvSpn|5xQXLbpI?$i*3jmFBqi=xZ6m^Xh{)3UF;z#Z*nrmOCGU zBYAb&5Ldy~B=U&5Xx+m@4Dy3HaxzQ}z}ocQgH&!3K6fhwwp>jOIOLN(aO_C9a9Y<-U63k9!bMDyo_u3MRVfV#0wf=rif7i&Bz|K|G zP{dsYIzZ?hcKJhDbX!SN;Vr!=e-2aA+eE;rkxSIKJ?8 zIjLsq-+B*vyM60kt^7ju=vbC8p7VUJm0kR&O0Tlg&*UeS*>v8!d-$r4NBQsa;@`b{ z*X(=sL1#v@%}r-m)Q(M0s!3Fid+L!`g}>j?;rE@ltE*4mDzUG9_VeF6UD#W@e~-hH zZu7Q|EHGFEdY3pn>mnZv({uFca?dIBe=)Ypqx7y|{@G5)ar48}DNWHh(bwADjFns=mE?~bo$Kzki8QsM2H*&OCXHE+H z`WfS?8Y+zZcm{p>RBy`bXW5u20#)_5>+Gw$_bOy>`99bP3cb2f?_5>zrDec83692yIGAhwY#Jhe8ANDC zDEy%?7z@nyOJ7MQ&gyYS%5B_?9z&^Ity@9{n>B90o z`w>9D{i`(pbx=Hl^gP>nZ`jK+eYDl-wtlSjMYdQZU`O})ft&9E=+$iI7Ec7p+L6JI z&%GkAO?w*ot>hc_VYuKNxfQtqH(dkmj%{E^kKU=BFS$d;bmNQs^}NT}ctL447Fa}( zXNkQeS10XT-*g6}JLE>|sgWRcjIZt&t*r$Q^wrRNcEJ#Y{qD!AcasVw#F|$PFi(o- zQx*1&Fj{@Q^2>}XrI6qLwLxvesuUql8p7Tpn2YFbOE=!8NZtQ8O7k)!5qr_r=N_5K z-;r|aEFuS3B|9^^U=GY`TYT-{gSCjBFh})%8|`!DtV}q+#yiHMkQ&W#cp(AO=W|p6 zH8fvuOmNp|mIIPbaXTs>r%MCD5^eV>P9hY9!T$ zPo08W<<01g$qd`hC5J_Wm+6H!>p^-Rhj+xjXFNWH*dzg~soXAUd{~}4nQ`CXrJ>Byn1*Q?{kM;jx9kTE zd&pVSh1U&mV;4neM2AL(DXM|)z6ItmiHHUKloOx5;>PAM1f?+mjPgVKVUq}*KP6h0 z@CMfY2;tOL+25+7GH6dMn!sXP3bcg*;89Gd+cgW2uyT7+V{S=vV;g8Z?Zt0 zo@pis#lpmI(!SUjb!$z1sHTWw5e?TbH3yUdCui=q;7J7rFp|F`@EW-Rs%A1a+a$Wn zK9=dw2NA+F2wr{#56e4?BP<>n-%u1limej_4vp(`h;B(dBvS4@PA=nI5OA2$2<`WF z5m8s~4mb6}$XM^8T|(R7z;_ykNF5ptV=$*F66*yMr#Mt}t>)X@X)kGI@h+$Fu6Ni```|>?+XRRXi-QpQ(K)(65w(pFl|~p) zMPJdGJEx~(DW>>SruQZT7t|bj-_fFfHWViZe7}#?oxakDP@)1$qoZjWAQ{NdD{u7A zO$Xb^oI}`$R**TCYIs7gF~#K4Sh3b*Z^DxBcw!ViV;>&rszw>ZFd#h~+zI%{fN;xN zwXdfDS1_ij*}t3)IaL=?uWIXZw#HOP?NQJP)rD!hf<}@SuqS)h_<#2JYWf9(-Th+q z8^k|5>9Gia6K>)ujA9GwD;M@(Ke^!)G-E!6>}u$zcXgfYO3B0Mzo~w>ZZ=La3oytp z)vSIMnJ_YI{ro_G*+NaR1pJ+*sa?4kYw?TmCN3x)+JbT8?fe^W=tJYH1+G9({RnLl z0>uHoDYf9-%m32?K>0;L@S~&3G{Uq0Ha8eT?g9X3Kg?=VKDA6Oih zTPb+a5aO7AN1A1iq%^*JrvU%)=hq@ar8mJX0}tw_^|)zt(7(Hh%nTXRz<3-7MU#I+ zX3bU9DuPqp{tiP`2_1P)gIUMhU$jB6V&(HnjW9b;XgV~b3_c8J zKVy9nu|7SI_bcLy*Ee2pJeTx5@j315C&j?0fr$I0ID{j)!5X^ZV zXvKR&j}J*R@<5+SP94QrK68YQBCpiA0^er}!{};r-Nm&ZMxG$MCxnFm4tb^GswL2g z?8Za5UaO+Hhk_9-zUb6SKg{Wm5(_$i@4wd8iBxs4m2Nj_&)<~ zB$ie;K8z9>AsjbtxmWgF@bb0VSmwI+Pw#QW{|x8VQkB5u|efhi>Ucg`q=2y2Izp=lgs9 z)wNvfI_KW=j=f*|{!~f}`O_3Fqo{0ZoBM~yIuyhsXNxA@sY>1PX}pV8wM4OsM{ktA znuIw0N=!M(dERSo3;h&Z?e~lAb$xzCMVk8OPubOg?hwLSHQH^RE~>u4=8&lHyngLB za_HjPmNx18yv2&140it-pCh|2>Gn^hoti1huLxF71pb9fjz0#_ggYv>;l1f0bWwvc z_WtGI*I?5!hcsP}!Ob5ZzYo2=l9Z^ZOrgeTo*Qn7zPa22`qlwl2Y%o!g$&dScVxzy zf81pk70-+3AhH&m{5kV+_NcuOH(v<4Me#=ysC*yNAu_U2+k^QI4(j(k!Hm(5}mvWo~zpmgtPKvi^C#H+i|NhJRj}tv)NRm zYm`_akCqVZuYMA71j9L5uzuIqnE(hq8k5!K+NfWZ`8==K z`#)4L8f#Oo!6EA|41+C){dMy1-A(q>t}*~STGnD)W`l!FPFKZ)x?_fK;tbOLUbPP^ z^6)*VJzmX&0aIFnpm*xcf(3Iw+M zkDvVDcW5&T>UZq63gAxYWbFEvYLVVk&R_E!Gu@9Th`a!0zU##9GN4Vu`Q)oY2zxyxP|8zV7Ne8?q@PW-pxr z!o{CF?@bmK@Q2Xqd5l1>T)9G(RdeH)3^;AcbD`ZhAkRa8#Q;G_rS@cF-w!N``NR?^T$n&`@?|q4Y^YK)*7e3E+)s<0fMcniOI$pVM zvU;G(c;Jm@VlB@vGoL zOG@~fl+`?PX!1epxbRO4zWdX1e~XGp$;0^>>WNLQz%WO*1uRg3F7Z8Sb?k37V>(GV zn)dX0|6obAm(m{8X|YXYjiVy98q6v|Ivh!Ok@=_Ksy_Pz3(e`iz7mcC;+$snsygi{@n}7ibYJR>3G^3*9 zr|FJHk=Z>vgWwqfi&(l)J2yc+IStn`FcVabxw@tM!1>-dbh!J+3#4G@$ZNqmW3h%$ zya)J!1eHBNomd9K`@5}_n5fwM5MAuHe_8&d5I=ti0oefvOOP_Sc1m-6f!dLfLCeZ9 zgi(q#W)$L4Z%Zd|5qZY$4-3d_z!6QxNB>+aB&Zz3wtp^PvQLX#I8y+bJ7p9fF?r7G z_+;XJP*xT8v+nnMW9z(g7+#=MAkJ|pOsOFrEW1IU1U|Iq+a$?Lz)Q0NZ!lWwqs_*d zA61lG@Tl3gAJ)_`LmcosUc&dJ_GMZJ&{YdCW+LJZhsS$DDCTxvQhK8&)GgBCj1w2; zg;wBZJ~4x2GxANnu+=w34|omuyx3IHS42f1_&&YP$~)Zv!Ka4UEc|DkacgCOrQ}Lp zp5~-Xq4>SUkmVMeZYY;uCfvsvEpPp<)*wLjx-8AN<*NIg6Shc%**u%0$ugK+B>uJ zUnL9Z#W`L0!8b>GTrZQgTJM)PdQ>v*S>m$89#r((|4jx37}{%W!xkQqkMi==%LAPg zXBgIc_V>i!>CrG*E+GCJfQ(KdH+_nUmrw!}iI-6K1#I+_t2P=Q*Th}zWAS%92_2or zWDqsAV!!$~a(slsJ?e+Z=DhIVNwkCWg)PvI*qS>$vA{qkMXqAW7r<*9O}WrYR}Uf= zYTw03Xd5tU{yp2Nwll`H6k*i80MxsEgcyhxsxDv47}8aJ>smSVVPn;pz5R)Ix8wj%*cUz#v?8)W0 zdH(j}6_!Se%9I#D3gh%p>>Y>En|w=j(8--q&FK2mF!zYncKZp%|a)z%abfeE9QB3@hgdV8)r3&E1AUE`iIW8jTAg z9^aAbf6E~C5}ena~kTUXTD=3m2SsBR#var0)q5mDEW+2J6KF5`5(B$+=HCu4B-xc6B- zF^xQdM4{)?S9j7fG3wqB8OS+{zENL*!q++Y-Ws=lxeB58|VHD@!?~^KwKq%1yi9yPQ z%OE({=6paPRbceLpD>VU7pfor;N|#o+t_;zV&-NcD_4%FUoan>jQDFCgsL7Hw~6W+ z+*WOjV(;{~$7(E?Ci4ZwBJ7LPOtYFRancTVYYw96U-&+i>){gR8Fzn1ezjYW?eSD8x4XR=7B7`9bJ_S}dJ^4baAVh8PFH+hE0;nRQjDDM2 z%cXmll})5_x$U76_%&#gA84kW{N-*sjtv9P9mz6&@gp%g78X%s%1%A@zp*&cqomTL}+GKn@nX3mzkWA#Go`SS=w zh$xq}SF8DpWK6Gr)=Ls(1tn@(T>M6++me7Pxw-ux-C1E~U35ei=>P%~d>KKI+*mGr z_ue-nwUFM>s|-f4BzSO-mgGIiOifA3ZU zqpRWMr}?&ev8M_z$X8LLpXuo{OWGbkD*0LlnK+?Xv1|}~$DE?Hol2xz>?-p4z}PwX z#)M=;@Jk3`Zkia^k5<=CtA12ZjF%cpl}Rhp8^+1qQqFADSLKJ5w_*xZus0R&f7cmy z{OFOI(%h9@=w$T8!pGm=RAV}Kq|Z@lge>NzFZu&!;Re?f>mC7!?9yUqR`@nFIN|Z* zgkahF6k91hNE0L|`tr*9N&Ui^({h2hH$`y%B`Dm7^1;d+-uA`vrM)-{^~<(SA5m@H znZ!v&bASc|823H|a0GwI#ONKwlOn@GY+`8G#;(lb2tpK)jMh z93aJ<`&zgZbIb$Q7GT-mMigR)t%I*y_5Nvx2A;a`{ck^X@erIC(PzujweP+Txm>TV z3FnEn8JDjTxP!2!yhnV^Fyf>BcIr0oC5#{QH#ZzDR{Y)3{+<$P^w`R8)_z`2idI^Lec%ArutwiS zItspTzfJ`h@32-+KJPW{;{?z*zMne24&Jz+3Z?Z8JJLcaS|HOvkAJudqQ~!Yx&YJ2 zIFnU+qM_R2?d1%K98Yuxso`iqJR;J()c@s}<?LF~MzYev!_LkEf3x48!V(`}XRX<`6_f;$#$f!u`Rn&sK7ocor)cvR&~?nl@mBE@jEWuXkkQ*U@~ zie-`(=OrrcMP`(l_w(J4ugA48(FQJH5J+c(n;|@RNvq*I>Mct=O%VHAIXwwP`&hWD_IU7%m6Dm7 zFfPHO!Z3Z?TwS-w+S<$jOzhq~=)X`@1rMn0&Tl-I;n;&FQjET6d!^!ej8bQO+Xm{= z6waJT@!#)uSnIAL+)SYbCOc3&>J7zSDZ9AOm@J1Zbtc0)Tb+{fb4P7KY8{EYwXR{( zab}dqyU_LHxB~r)1Q|qO2X;JG3e9kWwV|Vp3a%M6=(ohdEHZ#}n>Zw`^!#v~mashQ z8Iw;*cp1)BL$M9RV}mo-LHrK7VrbwEapOxC{QA7D+zTz39&Atz{-7ClC=CKVL!RL^ zVVjwx)i-{aEyj;zl|E1@m2VWi``y=gFy!rP4-^9@6Yb6}G*dI`#`l~>hGeU{Nd?;v zEc-5e7ef@3osQqs=5M-LK!|lmg>Vc3Lf^)ikuwYSWE0-02M2G7E7h@~wM#BMQ?BC( z#SF!C!=+B!dyi@7TM{|SYj@3vHrf(62pGO~NSF;deVK>z3edm9WIb(vfAt+zMso~? zMB1>@{!Xi!4<|J>!r@te635don3RkY+uZ_GnNRt>s(CO#E-HHb-Yjx#RU!|V)j)>K z!z@RSCWF%H&b2mgI){7C)bV#&!HAWwijtcl_95Q;niDQjezYL5yy9~=aYr%#ki!c` zW5AmaqKH?)^x4w_)-jVa)B&|Bz%-FnMp9jV&NK!SFZ*E0(onoZj8Aqcv$SH^rr5Rnv?;WoY;JGv%_eXgO>cY4z7TzNOGieoQ zV3jQleV6k<-nhv~DAz@bldrgVLFFYgA@mxGAXBPP(Alo#!7jhiJB%VZ+k~{r7@}rZ zPOhi6$r}bDLf0OToaj{ti1d|JYbo&0Nm36XEO$W~S-Z%-fZCp|_{=x64GmrW7S|sl?;Y;Hk}cFOHL0F@AsU;TUmHU-v0Xgf@P8H0VoHHnSg*fHw7uh zN&5CnTr!(IwFi8bRW)`{>2{04luqUknf`UIrz=_)s;8?B`$v6yc=raV5hOY;ydb!LJnUcR;qP`D5-N%D zAk3VI<2dE#bmm5J(m5f1B2P1|ydrL* z77_?oXxV!~n=Zq(1VnKY%$1s*c1I>@Zdn}w`A&$v(g<*&Rc zd9$T*5a(sA_IWZRV2dLsHr#~jTb^Ev7i{ko@!-&=z8&0g`}oz?9N=FN{QamWC?(s;I=aAW{k%MA@Y96_3GeuDI_Ca~WYc8QVl^Ru!h*5k#z4_<;zm z;a#=G-djU&I({o>goCvmGP^7k`)6%+wP~qAmpu!1U#M5U3#bzKwj8=f55CP7P;-D;R_Uh=kVA9;$NT*ugc6S+`QGnu43a!~3frZjvuDTFYy69(P0?#({8Sd6YfEd%KaJZgMtYw9 z+GkDmdyx_T5u>Q9w)p3cUdXpHW5s&;oxdMQ_T$nI)wvI-_}a{y3+I)lYKMPA37E`P zxs#KWGrF*sP?cmr*d5qs+cUV2C61K`?~Onw@?J8K-|8b(vSH=zPxx|pIQkY$X(-QQ?pEM{548zgG914|Mpvh9|z9CZ4DH5q9=O}mqMD9f4zC6cJ)AR z_2j}TO>E%_3cXg4^2r3+;rwEEWR6IbXQ!8pv9vOua$wQK-^)T< z5mSqjPx{a%qt8zyuebpgt%JYQz*619^HzjqA;(M7=Jxyr*z!i9N^$V@dU-1jov}rf zI8{IlzBpnlTjymqT)DhIC~zY%drY~S%z2ge_hOyz=mq+O1RsSmaLcU zXi_4F>^&h)RAu=S4)w%RSG>p*If5V!LoUUe!3jEF`)TvPNp$^8DVDfq?W~uFhTk^$ zJO{tc;@oZY;F7rwXmX>g$FC`Ob%Da*+9#L-pQ*o|Z~%17qxN+N^P^tkmmqF<0dD!u zPK}kJpRF~@MMU&fC&Cg?U2tBTh7;$cnl@^IxLdu~0eP1OS`os~y)lEIQEVUybwGBs z>wRcT7i-#q8Eb1Up}*R z!l<>JopVbuGd0*&PgdntDx;SC?edpd7_Gam>So4al5mPGc1?&{d@y(?m5jcD(J`2< zvZOZSQWo#bA3mbo&NZ|E^M5yGIYL5_FH(eC?*0L;-Jdx;iG|GM!FU4BfHRt zVq;BFh;FJ*Pg(fAK4Cu9UfK`zPI2@|UBL3!sEpFqs={|S zLX}RuB2knY%9CAA$$gWe2xAROn}ToQRD-#Sh}hb$UPR?Xk@~NSq`T0O6b&;7SFk}V zztDl4AvW|HUkE0JX59Rsj#YSy#c18LpVeTE&YrLnDGBn>#(i5Y?%`}R7aKk-CQ|}V z@hAQj42V0`%1|uxL{ll3r9J4>xb_U#w+7Kkr8rnZBj6S@i%rS|lk->Wm0v=!4|$*- zH-l86;9Irz8IVMb4WH83{`}*KUQcPu2JIbn^vjS6IuaE_*nx9J0z)-*eXcY4*&g%> zWOUE+pr!a8Hk2mEstEttl-rfFwUd{nVR;_`!!!6z540Pi%?GQdo_S=XX?VeAMz|^p zUjC2*P_&)?VQ&a4CeKz}Ec9m+qt+jxK`Dm&Y%R9W_m|j0z#1`1_&j5hynu^+2U_IO z4x29KfSE&Znm@fELoV?cy}rUYy`!EQbo>{p?~Y&a{}JSSX9@*TsDZc5=_9JjY`XRG zwvXc`BJwdcR#s#>6%uPus~mv&@Qyon399B4#N~h~7O;?T7KkLn_PZ9P6N_Bq|xAFgYX9#l%jHWcMjek;f*SMuxl|5^Y=mi40FlR4jHt;2Ce z295wdm$$gs`U&ItfO%!2b^q*fXR_u#RE)6Up>&&*LRrv{m6B4Xc7<-0j7-wS5f@MkVr&*-sCe}iCiy1jdGcNiwX89H6z$E%W7 z5fb|rBtE!>Jn#{koFL5x4p$(Tr9*>>z)N`RiPh54 zw9uuh6?+-+dX_;oPNp&~8ke&+J1!>LrJiUCiNVQ~FWTeGEloYGJx#gqecWnc%|(6Q zhAOl_;t^M~Om9W3N>Yaeut1E;;%)mgj0-&q{e74JPIo`q_;Pv1S1+G4F?~8?*`^zB z6{{)fSLU-CR2U;NCj~*B=JwHcKSyfx`s7m4=<7E2>=5wc775KEY}rX)8;QM>o1syn zd7M1;#5QQ4V5k@SU;PA@1;xi|@;7Gzr;oO2*dPM<-mQgbqL)16+ue;T7^j?Mi_I8c zQ;0WAGv;fc3nN<*+i!Y9w9Y4l7xcjGR)SG1!#M4O0hxv_V!+(kUW(i{&-I3P|BpU3 z<2d`FF}zd3rKngENfCYC08w9%!S%lRltEg~tM1OxpI?U=2ATiXSA|7)=s+ zLn=Qh+$rl*R+jsMlCoUhU7};>uz(fBs7u+ZTrspyoqaOcNG$u3cx*9iZOY_;22E>F z_>;ekPc{GJ*w?JkAJT9|aN|!4PY>4)-irA`^G`s`UXV24lH;XW`aNnlcHQbRS4E5v zvs+0E2T9+}cHX0x0rSp0={CR~)U-N-7D|0E%jG#W{-234T!`EKK})2)L8IK$y5~B7 zq{D1y|LG`-v(MIyL6u(oE0-VEhUXD>-ELJE+}R1Qc?)5_4?$l*rqQAAEI>B&e z#Tl2!iDbxTJ~7>S1wU$Ce*8(C1cx(T*!FoUh`1$ESH2f-haJxFC~6Wn_TKp3viHe_ zUDb`!P-y3dYR@CzR)_6+DF(2(t!cSc>}>gjBFpD5+govBL(45>oCY$`)y&l))12pc zeX#IR506R7qStnJ@B4E(kxMfsRj|KYgy&(@ar(Wqs9JLvDOG>oRjtIJ9}l#Sehw_d zM`cc?X8IVZgCH(cAS-}-l7kj(0lC?7z_j{q(6c|@YKF@l>h4q{&o;)#WY>8&k!CT8 zfg#&ayKnTaW>3}|zhsA6IK?Glw(~j45%ThKUy32@^n&Bxhh?jlT(jVpl7MK7tGf;o z`=oaVS})fj6-i)rQbbG3ph==Pk~#z~;8T6F;TY z0aRTSKZj%Ol!u|sIb8A~B0ku51qJ-`Mh>p7n>Er5I$P<@%gYn_OHKI!~&LjT{+H;+4f z&*2T)E)X3vUmcF7`a?c7JNeqzr%mO}$kYYStDj0TJ!DUbinoEtQ6i?1xH1g`^jlxh z<+05|b8t_hsM65sk1)KYYLB9~6T!xK)k&6*_2E9vG&Zy;tS0;3Gx2Hpy>^zar5}To zSIQ=zzec^fjg8m~g%du`E_RLp5sZ^QBGGe_7z%}X1F1UcT1nz2#Ax>TY4-9@$2^vf zPC{O89;NyCnj4pIa*@8+??_#%kEMW8P+1jNDNre|{qfVn;tkK=sGZjE@XssoUk>^D zK98^!w?!PJmve@mY<=iGyJ9gy`aVT-_wNUuoP8(_W%hf`Jt&bttex6?PM!D2^q;c7 z4R;=o@jn8Z9|wNo%%heqpMM{HW-2u|8}%#vm*>Z_v;W#?L7m6EMY#@bywHXFQ2D}O zZ`N~t*dGZg!VaIctfZSr>YB!Ox$FNqK0FWP^1NF9@;6CW%0U6qx*Qh=mg8}EaJk1p z*Vo{ML%(hHO%$zLZ;Nz+G46%FJpY{BSHGiLfpvzl5RST<8wr#xatX!!e2xFw4_-{@ zL>FVsLd`0ESi~fHx8sL~Zsg2(#Vs9E7kx zs3FUkd50lzHOm6}6GjJ-E|5h+hYN}&q8KD{rnXp-H3b_=C|!QB-_L~}GW(rRDrdxo z8z#YF-?X2t5uJRUk-XS9&WdBB=P#(cW39>N*&~W> zOj55mx-6W3aV66)B_aFPm2y@ zNnOasUi<1au6YMKm?1Ku;dWoe8l*+vu;~9m0;SW4nk1r#o&nlHVyD0`8<-etIVeZi z0@2S55Gk*kLSHl&>qC|ptoNYRh`T6AC(jMOz!tDtA&#t2Gy~-|5U|X)G()d4RKs7Z zVnbtau(27S9(Db2=ot6}aa|uq{EN;HtzKIOHOuS8t*H6}5 zLR|qlVLt-2d}(6NFscFM5MtYQ(^0}qwlp;I;m<3@O(8&81^rk}(XL$~&j2X0dkWy%^!Ik%{2TVSI4pmXy=pHo?IrN6tdqaTdss;qaUey82 zc5St`Xw=3GWSr=RMM*SDeTd8hvT(B5Cg-vUA3jFVftZZzC@~JT%zHron{1sCz6w9T zfT!Z@i2RTkFEm^A4{B#g4|b}Ki1%*da&qQFazuI~44vS^`ZZfO zd`N)Y3KSk%q}ofpghoxG1(d5fUKP(%T>s#+z?a~+f88E;G_qUE;vR2##_$;)d;hB^ zWSrJHei1~enWz%;?Hg%-|5#=1*6*F?n#IlZc@WB1A4+nQ(xEv(@fog#%W;W|0_HhhWmIdV=-1#4QBNlnS#>fq)uS$l?2?Q9FNY}F&2KjC9wyfK zCWr*IGJ)tdA=S1(%g!H1MfysiL9RFv`T>@>5=sOj zt7XFE8UK^naw5iQ=TzwLgrrr^!V>Xv7ixb9F;6A@D>&G+z);SC0AZn=cqna=#T_mtz}!u83frtrwue;Z!2QGWkyGBzvJ zo{IQqnsx07ATOs((P~f6HY#TF{ISOu7bF&Mng5Y393z=`bz;ucj@eEEslC%~|Jdiv zyoeLt{)1dxLW-R~^j*0;`#5txST(yED{>MR~_a2AwN_Lo=#HiQU$&ze{}#o z2N@AcE^wcwlc=G_{m*dG$7eU=-k?m<8W*3jV8Bw!bHO`zvfb*-h*SALK7Y6M`7e7C zia}c%KUDWZw`Q>J)Gr>Ww?$^Gh?wA7W@cn2r0Gjo+7V5AAAv{QSwl(W>&W}0gb2v- zYpZ#aU{gHM1X7 zGAARJd%Rl9McVfmRhCZ5h7i||l0i26K<4R*Cay8F!tDK?=QjbBw;H)tZ+AyEj~D)N z9LhHS6Ms4@K(f&}Al}vJ|4%9y-FPggGvnp`+%NJJZc7&Go0J$o)}C7PZ$tZiobOk^ z1oF=&!I@7F?%8I&$_uWu#cbB`r>n1JL1P|~D@_2l0r8;<$h66QeBIuR8o_3?Alw(f z@On%5s?hrd{Cce8{|E3#PW&mDCfHq|Fpw$mp(kp*1-8D_3=G zS?ISR7ahe3z+B77^F~4n$$S>@*hU;j} zzYy`0m)Dz_xcWhOVh^GQpgdOKT60%sB;&v>yb3OrA2ThDQ-N)ALVF4)lq)y zfVy26ZxS(d%m8hblu?a~`q33cY?WTBH~I_6N~c8hf@Qz=DhO}j1y}ayKp@{@+*OH| zBT+RR(d$9bgH620yUN>TNQh?Pd?w*WuI9f@!e{qfOA5{J*=-2(BWriNX}QFZfwAEL z{WXBFZU(fhYfDg3=ebryN=mpx&xarQRPA4{@*e>6;d5Glw5b<_<-ak5`5*W*xBjYu zA=+;m&1PtN(7l`PTORToJ&yYl2)B1uJZ_R3zP(a_0l=UMn=CMWlAS1~Ge*u81+V>T z^P)7)CLMnqXJ>qA%}H4$Z~tIG-b(zA)I%5bFP^L~#Fl~IqKHF?A8Dp*529cJKuscQ z*ZvIO(sd@gqO;*_Z1X^tddtPQKC~I1Kc>)=e?7Zc(_Nk3ArAQIw?nV+5%7Zp@m3rx z&>zGRoguQeBsyuq^0v^p4}Gcm}~bZ=hN$dar8K!k}nHb4(RP9fXFsy>H}!yIf=8` z|EO@&4QbKM092_*s{PQyX|5D{E$XJz+#s;nk zPyEJ2pfj{$^aWX%l5591Cr*%DP+vN)wrbZrXbybl00jzX6aqvg9i$+GFAKRElIKOe zJZEX>LP@@Ixp}&H?)zom8#xxh`=FCf@45`Xo{O2}zW=m~R;dT!W^JOn91bM5YbXLk zAS59*usmE+UVoM9zSt*cORNu`86qr zpGKES7>(6&zWhJ{&sw^Q%w|R>amZLV%YhJT2*~wm2&Akj-Gl@78c>6;W&JvkQ#4v^ zMby|Oe(Aw0T|h#?=W$vtRmY3pWdTADSDuPOB;`TQqxWE;0E1O7h7%r88c_8i4AV7> zCw*)b8zcdv+BS4bHq#^KemHO8an~1E?2xHzHzEbWcf_Qtu_rg1F7JaTKfr%0Y9Ss> zqgptkL`u49V5=oV)c~1;a@#5#+Jvxo`2%MJ87|<_cA>W};6y7H{P0@~g@D(*45zCE zk4kJf%iuhIz7M_YcV#R1F!B37DGZl$_iXjBFUHYNE0&!dzPS54)8Fm;wVEK4g37e! zG1{SqG+ym@I^F)?o0CRBmhlE$`IeRu@xcrDidNDLUwwO$17hO=HML=i3+V?Nhz{+} zIIS0&UHuElkxhi|&VSB&E*_`s1q19x!-M!o_la78$cz>t)m^Ont~CO4?KH#2&1!h) z)-Mbg%Ycvz#QALEPeo)|_^fH5$zv+O@CuA>Bz^;sVri~2|Iy-Vsi@soIuMJ-YHbtA z3GF|wA~+GMPjM~pxknDX{6nb1cCK5SvYgb{Z4WroGhBndFq4>Xc(`vY@Ph*LlLJ1jwVWk=aEQ&>{8R4In zdSfto<0K68$MyL@#2bUZykXluxi89^*q^qq1x*CK{4%`f15hp8l_~EXs@g8#&#y7@ z+G~ShsV|6aD2sjzAU)ry6ug0s?EM9S#Iw15XzeIohEb@UWQ@GmkoF9Duj&VIF$e&q zw#@bHS4%zl4&oDBCaUzbA3)+5HgqBx&mxUJkZYTnM`&|)FM(&5j4NiRr6DUW;2A*L zCb6;6{DDK?M41%U?k)UZGhO>%BV_e7AQ})c$mkY&^+3orQ1kLqOSpo;J&5v}9SzNo z`{n=-kGP>NKsL4wm4L;6##qbgwb|q>(H+in?nAo|ja$LGJ)&Gk>QDLZ%eZ9 zBi#H>uiH!%s`E{1_ztG#so-0S<;x5Tex>W&v~%U-h^uc0++bx|<#I$De7by?0(^}{)m@ch;&;YOkHSvF zdhIWOx5 zC*Peh(?g^RMtUGMV4dTyzAGm=3!>Lj4j5hIjfDd7-|aw!Jjv zuMiVVwI0eW1F-?GPmRrPA8oT$p9cnNYe`4bZT>>07f@O(PSe{i<+1XpfG{r|3uXYG z#s!@{nS7GcPOT@A@cutpLJKs%^RCY`M=w|qJju>U<9Fwa1?x@9U8pq}K7-g38hJT; zX@3ZdNq=~e(uw8o@05+fFQKGu2^&dQ72?fd zzir-K$H}aq_Q|V?AIVsj{Z*8}TW5+73Svo(m=_T&mk19jiTV@;-HnUJE_(P2f){cr~}3uxOvL^Dv}bW8}Mkr+I)jTWsOQ0 zsLWXNj-RIB9j2b>@B9o|8R1f;dK1U(WoA3>F*DOwmd#r`>*GVjr#1B z^7*T9K79>E%get&8wYAdxe%p1Kvnp47uNR+P(y(XJ3<@nc^OoWN zN5q7Mw9-QPIXofj+2e@O>z|&!Q`U-t?}DcOmz*OAg4B{m27dHT2)^zFV0j1i$BC_! zNWlp0JJs3pk7kUsaw_P((ZzaOm4nc3162SE`-(F+qDo@06T#bEL9{Tycz-OPg)`Q- zKb(m)t4$Y1Vs}~};_{LlMA_OxeNFra`Y8sUHk#c&<{D40$H|-^t`=Zcy<^Aj=`PdS z;95fDfM7Zm?SYEvPr%Gh>woYdm>7rwm0gD-vt*1g*3l)9S*h;Ho)E*Gm#*EFN|_w| z{F^>ULBp!=&qy)+1o|l^H}0gzUZfB*O<}u&@Oi&{iyL|nWFl_ zl276TynK%FUE%tjH|1JCGaNTC#zU@ucR(B-?cEXe1Yhy8Gnc1w~$q9@j9w-#dDo!_PPxEO+1*il9$O_X!;bi-FrpA+k~k3Sb_tmn)R_EC`>}+#s%-2 zpDg3MR7_*Laiz#G_%0Qzaa$5P3HBm?4oKup`i(P<%Fx@FQKz)lC8l!3OCzbbACI)N zC$hWTgZ4}@>|Ts+LxcK1NyeR&%zqJ=_rnr@pEYby=pX(Fb_zlpnheP17X*kY6_srr z;qiRv-I0ueQh~Ei110SxKaryWKGI4oOaBhvMvxAkDtu=vrurbW+`=X|Ee?cNIOBDA z$n;jhj5g$TKk*v;^L5)g$v8?ENf49`kC{I%ntqMq-Trq&Gcf;#$ZH>qGjZ&u{V+C^ zh@ld*y_nUKq4@_+shANy56{97I1|>(Gh^E?MMSIh62Nds5%R=n$reZKJk%O!qouFQ zKJFzql^_>KGlbwR0&<^8qV#Z(eoU{gZ2k`rv4`7ey6r?jLH!Ib`U0{H-9knDRAH2G z4#sC#>-lH6o#z{j>pcVC`L=kmUXsqhYY-*IH;R(%oSGfu7G;Zz{e85k zvU}v64CDnQaxP!NxruGdYy_$LjW_ zBjdZsq3CB$BD^kWl(bF4ZPRPc&ig%o=MsU*=C9*F_S}!pL`L0dF(GR=vF!>}yK zw%fY8M{YCUg`KR1*|JZ^X@8ddK-E~l)M$-n%~DAi9);EDc^cX`;mQk{gW(xp4=d3& zc?G5!2j_qz5e#3!WbtWTw~gR`9ebqAVP3p|^O*!{X7C@1DbC}d0ijMN#j*!ao&p{W zzU5pA2Dh$inpj+7k|vOodn4-o(NMBHkG~PTQw?thKELa#bbQ&0Yr_Q|c?l}jc6^5} zru!0Z;rs*c_*Qt+%Nm}9Kr>GH6Z0Oa;moTUUTpbRH9%cJ1IWm-EM`MR`jIOA8NJpl zn(G=_$3rUa=j_I3r%YFgxbJj~e%6`qf3=KCk;n@+n#Ao+%+zZq)5b~0`6Y17gH`%= zZm4uz?g7Y}{unY(WCNr1eH0#`Xt6+SjVPl$G6=2CyH~wh`$?M%$u<4HE+NEuOp? z^wl@`T1+7uQaEK;$E;6ae=4Sl=A|b(fHJzEEM|kfk!7Xs_0^~1d(dc{c?g81aHa}m zdo7{J4-4#spO(^0gOM+eJoK4)n$2hX*2KgnC~C=Nub*0{&`vlXaD0YMW0n<+Ot_q! zQ+$JpFoWZ1FG&x`8=VXaD^f`WZg%e0X^_%Kpq;Us^ebCeYQB|HfQd$*=~Q99H$6^9 ztwUSpGjU+t_p}$g55)qsS2af4(PR1EXf0ez0fxXUyR7|Tz24~aJh-1HnTgf32>7_6 z;O0k~UXVfPu_m#6){?IdgQwr75m76_)(5_{ikQz|1Ole-^bUxT?eS`pYO^EcK|1b> zH`ed=7U2OrR~VTGwEw>rU?aGhv8)+h1sI6-WuZ>geD)g7vc*qyFW!VH`@HpIB?o>j zB#QVZ$Igv<#v!!vk4kmJG?zjS^If^CZWOJu5Ikx8ZN1CP%?2@kZd6Uugz$J2MKAnVIWdd9E_Pd_4lO*EEHl3qI zX(8SDmX}#NP~6-aWk^`hdew|jcG7r@Brqg5Z>qbLZAFAbt4OKiHr7$IIY~BKe5)1w zh3xb+WaS+a-f@Q*Q%k{NLQo;z!Yk-f%1M z0ToiZ`=@Xul^n9n@oZ?EncTob@U$p(Q&G0Q2Uvwkg$qoK-xeeUTX1v6FZOgx#(@L2 zgBc70GaBDu?Xvl?MYy9&+%!;~dBq@0WP3jpfpJ$A!GIluu);_@<>z64pL+s0A|L7h zsGe`jD1SOJV?%dS$V#b_{7Rl1w1#68f8G6y;lj$dlR9VY@w?>ktLBB}a=!#>N~AH? z1-ZHw^_%j8^GO>6rZ&LU);Lrn9{O2lcGtbvcWK^T?R;sIUzHQt7Q?sG`rG#9*-7w= zqvE;`*Uzrp@@#@^_+sUaz0nJcL^Q!SfCZYkbDwJbuY@+Ko8UF*J<%~`k(Fhx+V>t0 zA2wckySfUO5>MDxHpIV6bFlCop+KmPN;9>Y+Sf??Hvazg?xeURcr}PH@sgX`P`w@H ze~c(q{F$WG|3nrSlyFqE)p922=~C@li&6lowc&bQ1UOq9XcZZ1ut$V4S)1z}cUIxE zX;qFsR3xBN`GQeYejyk)rm`r;hb=sS0syq~>g|48gRehiLQ`n_6vf?@E#qk^HqEZD zjX3j`16DH)4@!kV?-+FI>T-WmGmNV5Lf}HyMz&M^tbD4;cKJxP_$F095;9IwyhK4o}zs)va(V$*L{&ZxuoQM z0cVL}Jwp^?_jg*e&8PFfi^R@TC?6r=TxCh8Y~m0RF?5|n$Hisn$F#{xTI@$vu>#YM z^$Ey1F0%Jc23G#@Nsykojywo%E8JQLm)`0BR#4!Ogt^O?8VDELBrqcmXn!>aCwR1q z98%Z+`Od_Y!hZUPL|)o<^j!#7nQCcHI9i$LjowC5I#nG9NL!-Q`hF-3?g zMMrY~A|hC5BDg=|jn+Xuj^wf=RBraA0V3{YAWd!CL!7v%qPaL=Y~;OdspQT(Ikw@b zjTN;6(Pro+z^C1%Y2>d?yHlgX#+MR&+FDQ^xjyLP?SO=<=OqW3svD ziSEDAYmh3DX9-WSr~daQ@z$bz)!9?a@*hv}du$zpP_KHpj zxKwTx|1`!6{(!>$QC05l72)Ug-esZ2H=rI|f9uYw(&!1Q!>5aDIzQS?JLd5xueb^o zPqey;vH0J0J&||au=%ggk46Ji%UqU*fy=iPMY#Wk(_06Cug%wVckk=l&1eHS6&n74 zDuQx?N}x0#7|h9@BTqq_^-hdc(^J91+m3Se^tBVQ$QwAK%$xOK#mqS$WcGfjVtqoh zecS2cl(|sP7lN(u-f1{{OeY&U3rh2jJGdZE`+cdtC!u@*sqo%?T*63A&g*qH<=PWo z*qe2$M2`7+_#xyVXdo>q5OX7BSY@23Na0oO>4U8H%X=bcM!8wJ$1Z;Bs6%d?xksr% z|Cj%cBSD9zHzXbpj{K9#l!O6iF@yOgAhh#@d)ysHcw)j(1s z@eFu_h)3g9ED)NU6vzcC3e@z3MZ)8$JW`sNXBpAKF-+f;X$_KQax zji_u<%(EX}9SkZukfOLn)Q51Qv{Mftwt&6ENaf=P`q*Dd)3MlF71T=NW(o4x6g@(VZ{&E0BK zj&chaz<)yp;wm$|_MpuN(wVexP%#mnX!1v0VVJxd!B-SvGb~I@iM>PsK4jxy@BCrW z+kF35A`g`(p@tb<@BgFe%fq4k{`Zlsh8W8r5|b?%$xil+RQ8=AOGLH_WhZNlkt_*i z&rY(BWeCmKvn!#o@05MX((k-Kzw3Kl{&vlC&hwn}dfoT^y6@-vC1g<$)BKC;@%Zc* zECrMNt)S;8A0HZBAe5E*U+qTmsXVo;_dP$&hl-lCOZ+vAMA$v8APN=2ax^Ed#oiH3 z7EE-zYBY;Dzu3$F5_+--dEj38nC$}n#Z>#BZtPju8H(|CwY8_a;A4(PY+-D$4;b0p z(zjFeP?BZ*HXQ8f+#8$(W^19#`7@w@HF9(L`8}6e?}Z{X1iN292OPu{pHLIbzGPxK zozM$N)Yy%WZNS!qCv{c1#0+8wF(*4i)S=nPh1Rnls6v}Am&C*vC1E)39^Ci}Y;(ej z;RWek!GAq3RWRm9hOuirQ$+~AxK?@5*Hd^dtX*eJmivnnP;8+V$o(o0@jItfTw1FSo^0zYm<@uji zU?5Pg?$NJMW}5En2{XEDM&P52lL@co@yPLoxPa&)cIF(huWM{1c$}Q2c6r@*_Q&wO zQ}18;$3faP zG_R(u%D3lGQ!-kfyc;Ana*Eh`1^c}ZtgK8#G1A>IM0v5_hT~zFhTqkzJHdSIu`!q0 zU~C&y*0VpXYYP=r%lR@GLi^rRCT+;=ld0fV5PUkZhV%FrdSD1SCGN-hOv=&P(S}Dh z_G)eISN4?Qf|YMzN#~QlMPrPZtei+{kFnG?STn~9r-&;CPdaH`7{5Fjw|z`u*d zM^5hEDb57g8n>6vKFJisKl=Q}2fX%^+#XtW4@SMI>-P)@zUw;s8Anz{ONSC^h}Yan zu&zHWxh$LyWBD(9B+td$jMNXvo-q{V4;qM1G|cV(fA3qt!90SJy39!8VsUo&7$2#< zdGv6;UUbc~ICn%09|ib1)vWnkyM{95Tl3|2p8R@u|01Kvy|`m4QisMyq!B)7zpYl% z=rd1Ny+o-*l8cZvDQv8VJvjQc0a_nr{gUP@rvHZ0=(VU{}|EPunSS3r*%Pek4nwhp-zXfB!Sy>gLAYVQ4&Q(k0KqNM2{7m4}{+Em3f8Oc~m{(|PO**Uh*BMSyI3SIy&^6&=*Cu*;E!1yDKP?~}L5b}fSUQxEDc-@S##O!f_B)Z(D-E29~ zloo`m_kKg7hweFT$J*zf?-+y1ley9#b0bt_h?jI(&G z2xUX(RO|hjW4@V;(<}*Z6riuuJiz&WymIpM!Jd$-?SHn3oCh1WJFaC(LgC-Oq4Ey@ z`>NLP-~DcVvt@m)^ONE-I!9iA0j=-f%8Qli$@zlcbuYJOcBbLaSi1kr=}1EXaHQ&xi?)+^mti4ejc z*grY*{372Ihe%x4r~jl@8rDI{Git0{oaLW>>7#D4nH>fFfSOfET`O^OFPCUZpHC)Z zyYCOCIATc@OXm1JQnA7PqA&zq_0cIs~tnHhAui4~WlI&IOr^}x^Z;w}#404x6k%9ZLbU2|975AgjP zc=>k{`~^Ot8&dfHje(cG7Gbpo^s21`10Qhl>wdV25Ya}}<9oio*j@ZHxy^$2QR8qV zdRAoRdhkZG?#7(Bx^O});`esaaV61#l+lTYyoe)n6uL!f&z%U{G=7vaA08csm+$?< zNmE5u*E-Fsl$RcTe}&4B;ZNUx=#0a;UodpTvbk z>ClynWfMzQ+O!zGXCAU~i>)&w1&Ax0is!IC)mnd+`tju?!5vM9>@|P*IUZmhU?ljg zkNM}9mq{fxzemkzYJA@_jFD2wC#vOt@IZ|hV1U^7#nMECF|=97*K7d-`4pjPDSJ{C za@X{uNWunAz!#f@wV>IbKHF42OFH^E!}GWB%bE4L)$Fa6f6#l5@LuA1)0eZC{}lX& z|7|LtZ@xU8e3_+CS3_FvL{Uclsa7U?=+pxOY%3&AeIX4xE{YL@^tdq!dkeNY+1t5S z;|H1Z8!C`myW9t+cOdcu>oGEP)rlYZauZ__k}ZnG!-H1pyRjnlG}VZdt9=3-B0_FDel>X#PH3F0S;Y~M zz_si#<2Am#blU6iBEh-;CZmf|3H4#HB14VX_7XSr zXIb_ET)1WYzhqJAWt=3}j)>fOiL|*!bC773PmD}ROR~fvgB#Cmp(-IaDJa=l3GI4y@nQwL#$FGSFKue; z52|cxAX>iUo7BDSKz&aM8hX%xTAQf-;wojWZ|w7L7e6o7neI})`~G0(y_ZP9CqJb) zxeke}L%->ilLqSjf}JaJRIb~hm-;U5z;B_4MDWP!Ar#Mw4`lZKvYP`7bgi9MG#X(5 za6!FCpkFv*Iqo@ce<$U2uL3Y5Blb_1Zd4P$3GP(M#^n9EIY)6(7l|g<7VHssFAjHx zc(Ntx=82(0yUc~XVMA0DR_w0OLDa^a+*IHz@|rL_QQ}bcme(KF90a)1m71yIZD(GY zIP86R8=hKPMZ7XjTVW-6Zx*P8gj$H^CfBUH*QUIlHwGa+FY?<$j26#Z&%sZVdV}bj zUpC|GeDiWIdfOoux!|Ue%CSqSq{SvA_gzMva8<+5{5r|G9ktNV+KxP`BtBAN7nrTN zCnI^IIZ{gNn%kmh#vK-Oo?f|eI0KsZnZ^gpy@YOnN4}{7=1wD_jpZeBy7d$!wN_%d z&c;fL@@)cgjYT|$wr-^*!0pu>LRvh`F_=WLjbKdkDzD(N`o$+|!;}`VUM`Il0fLFC zUq$o4TR`tdeX4he7kv$gFtMoTdjwZun-iE@VonoPxaV@nqRQ6$`!>s~1y!ngn+?wEzuEOoZ zR}j?@s@so3wZ%5>F<}@{GmHBY;0EZ?CD=iV@0C3)&TS>0V^^`{jPZ|a)Lj#<;Af@> zS6X3#M+@6td7}ha^sNh|XZZQz$WFZqD$l<_f=xMXx^)ycQ-UGE2Z#)_hzRxiydO}~ zT+x*Thi{yj2D0fAPiT3sTe=Q?l(7lM({DOFo{5rJmMsqa(4`|S&S5^iA=-IMUeHO7 z-|z9oVE6+x`JCM=KJ{;V{k|R@0TdGA8H$1=zrqt(+1-pkH%Pf0&())WuH(OqoeP$l z8c?nypC9gM{L9?Jn@J7%7_7EBLveYg`wGI0F~zVKQ4oqt^@d2G^<7nEpf=bK?0@v~ zh!4(>b}{(5Kg{GL zYN!SB37H(v&%Xh;`hCyU1*s+S$Qchgt(&{T9<))->pa>!t%-5@b zRiVgwb7|C^ zWSx?SNt$oRDRiNJ5RFAHOno{i9t?+kg5&oa#hmIbr|c-09xNgZij$O{>C@nmMI<$1 zDrSzOY>$IGXa8|A&Plr#2Arjtk2E6`1F4fosNKge$8j0^=)9A{Q>ljJUa@}yLqiEL z(yKTxZR^Hi@8@x8IzM&f#?{Q;xWSf+!s-7B?;>>2Oe zwoM3xZ!w30;pI6ZhqCRxO!=)jEUoL1cDikEs5=GrHTFEL7|de!RJ#A?QmAL)eo7x5U|Q(y7|t$p_8d8(EI4GiXI%3)G&^4~edr{0HB+;Q% zReliG)M{b}E_;Tek9fz?!f&UZhsOOj2`AjU{KC53gsv{znD957!g4Rqf2n>ZS7zj~ z6GR1AT4j+aX68P*{8ed^ht-A3fKbUBAXiiD>v(Oh=1dlEI}hd-uG$TZwF%a6CHv zYf^4ye;9J_f;6-%Za*(rzVxK0`qG6{A;cL`>zcmE*0;;zt;UhnzUYD z;0D*>HX4R`{ht1tbO8Yq#e1#yPeit|@FCRuj(2*aPrkimQG-M*;=TS%)CTwz3|LDC zoeLg;ZtR@5H_mD@XjA==!H^W_)$=uqK@k6B7bAvie4B5W@YIU6qhBp&7!HJDk8j7S zbR4I8LT|^x6LlAouO5;Mhafz+VMd7#yRbiM_h8yh-p$<2>hIw-m@Qt4y7)rPmW)Ct zN?)u}6`iNby-lI5Ibs z%X*^2)&;9l1}f5!{$Hy&wY9JMeyeknTz)}K7FwNy7MomDAcRYP^2`D$aQ9?z8D9Dq z`=ea)8eNx0%Ov)+XOi?m=HE=%HN(uP+Y#JFGa_3Eu;Y1 z%3{7SzH~1XW3r(W#GWZFVr9BZu&RS?pL51;UjW9uJvoc8VI^-F90wMrgI?uh(jegr zLTSioqT;UF78!zFz4bX=>5Q3jz(V>O?PHF4&;BgkKZf+e1zNt$6=6seQlx4otAODt8SVZ2T45{jdR8J3xr+P zIL3@pqg4~0^lXpT5<$*uT2BrfKJz&&)tEd+{V(*mKwQHpl&c=sl zZ<1me$*#k$xSy2CNI+0~`&kNs`&-??DN|1MeUE3~_Bw8!-zvHxzPe@Gbh>!cY|j3r zNxUdJg`Z&&i=LIu>0x&A#R@&1kL#g941s#dO^IZu=w*{*x$`V^kpA8nHRXj=t&ftf z&wnne|9sj&`~EN-!~cukYA&#}Ss~73kclB@ynRC5rlJixA?0W0N#Q-PrsDN-KDm@n z843zX5t0O+xO=NzBsV*f3_`hRbgT|8@u>c)@%B!T|EFo$54$mHk*x)rov4La^ftR^S-_y$E}QVeAw(S}wATETFpql9+U>VkZY(7ysOCx@s6$f2zSFQk z^GZW?(L*nkSSkTWUX7dBNvU>{$t1eT-7z9 zEv&faUIt|aF%ml-^LFnkI)_@7|BOCJ@L2R!yZQL8@hR84`+92Di55)^w&NvenuBBbQlvb89!y-`dM5wNU z@fDm>!M1!adrwY~CgpD~s*@}-0gUNfFh-F2`!@P`qAy@=VC`psu?Y7kjOnyzq2@(i0 z-Y3GS$iE2sj_2~A+nlD~<3gs|kFpzsfy+$b_#74H*lo`9Qcc%a6kQ={&R{2jP+S-rYkxdQ@*DhESBBqb158onZRio<84%i2fQ> zrw#RQc}-~ZFFsGX{OI44^>UqB_l`W?U%7gRt#MZsX~fsZNP(P*2}VE4!%;r&CT|kf zgNN6+rn%iG>>xp@Pi#ee9&ApW^~)AqaeOGQ>BYMTV|aV%6MHjN29i|m4+MvKW`eD$ z+cIKoVIoR;s$cj;1qUN5F=iw+0&sCu`b#SfC8BD0mT-#*0gNhP2>70a-S}?HJoE%= z+S@RwYguMhzqK1F+PDQ}6Gevxg7z#MsjdZYM~V5HMB=HyyoxU2D@*_?T#vQ$#d001 z165d;U8l%ao~-uH$4v9PDVh_+q7O)bwbk(my;BaSI3K;(9G z74@SiOgyxC?|2VQ^He|l?6S1YF@%n6p_kfRiV<16A%00eKeMYH)TYD2>#|C`=1~Jw zJ2k~Tdqw{x=qWD0QGX?%RHQrNrxUo`QSN2A?8ID&Cu&6^LPXU;hr0k|pI_QgjHW7m zPb(sI(d16RB5DrtU8pUx6D51BE^v!2DLSdNuU8PRR5U{8X6pxvk}g^OBexttT>?@2 zFvWDcMZ`8NPMa?Rx3$cSNyl}uPE!nul~S9^=jdp|lOMgCS`PrL{9a;dhqXO(#B9N zE^6alD{C_?4 zFS4&ZwJ*=y{Nicc?41*&ZM{%bW@%P)#WU76>CzS~XB$C!{XjRPsc!5bT^*gUd=6L! zV^r#0@5Vf;@64}eIjciI8kw{E=p&W7BKG16&7Q&kA<_(Yh8rbu29_YMGEc zZ*||9RQI>{yZc5`>gu28b#^qn|C@Wk1VDq~KLN1y9>+Lm*Fl$|OF0z@R;eaki4-;k zyoZX8K>h|V2hxUlJBA%8iOyMoDP1#)ma{?#2C4`8L+C|ovZBX4nXFZMZy-VMHy0+5 z0Ol(av(>y3zUt=DHY1Dw^SHnFM!ow6T3c6$_h*ySkF6e~7p@+P^TtYs-;$;SO>Hhj zV-tR$rep(m`1n$Xx2Ly6U_qufKlJ!I98{U5e{?$3Fr;By9+{}UP$OjDf72^CQaTzq z_LV^mLZ0s!?%K(U{VSbY8JQ}rwSlHV(dHz`m-+Cje6_!wBt=|nqZhs(MXG*+Vu9Vy zFxfqL$n4S$oz{P=WWbGvc1xsw28)Z2z;$`O?8Clntlb>W4AyVJD)tf2{6p&bk>> zK(WH3DoU<^O}ouv+KTFnA2+YTj9WTH`e8xO6oCuL>binK=U_ktbaDM4aS>6qn{Cin zya;>u-&w%N26{O|g06ZoLLIGt2|fE4!^2z>F>kM~!@+nh0c6iWw_u7^b<+G7kXGb7 zFzKwWXd=TaOZoEUzXlq3 zwS5>U{WfovyeHV@n5`kae~+6RS@sXRv>u{-EGRwpX5zw8Z{E9cDOQbK5ZwI1@O#|P z{%(1UTfH_gJMxNFzV?E18lGwd z!>*hFh{R2Sxp_pY5hXC73K4y)&&Pl!{}^4L?MzY|O;^2LxGqSs1`Pe%ce6Fha4MHI z_MVR6_PzHbOy1ht_1~OXRAs#(xPiS6ou+6m6HVh3)L!OXu{s(5^`YfR{M8wr=rBwE z_rgt&w~B!sYS>r1Q%Tg6<_|etbMn#z9a^G*wqp!iI#yW3lrLsHuJ*}R`ewj^6MN{D z%v+m041As4?-48~C;@cnJ+U^}!Ov)5_*wl1Xdu_+uh5L&N#>r0gVXIEbTQmQ2_4#M z4mQlDO`+)67a}NHIG_y-9w6-U4|70CL?)&HB8S6E&eV6dVG|JJaO}=bw#kxU;++;` z?PTqGoUfy#$qH2VNj@&yT;Kko>e#T@Dq_5MEwS6IS|j^0YrE=WN;?S1GUN)&kqUIF zdZ>HHP<>=JwkaHjl6;01KEOpGL_I%?R>JQXd()dsY(`xQRmAR=(0G<7)^Ei}1+eoo zfW7rxLA)YEl$_DUsa!4pE(+TFR^r17q&NLslKzAL8kNgvUW0zNZ8ujq89QHeOU)uIU3uAwU!t$>RuC6eZ6LEk z3a+8|Tk0Q)(!gD)E40Z_TpbhKp~Hg9^F!^&Sp;kal~X?HV4PWllhlAYvK{;ZtF z4y;db=cC;&=4c+}q@1wh3(3lgrd-;qNe})HMijU@4P18~S?X-TX4|S2lA_7kLhIA# zZq6NJe3gnGpl=g*RgH7xu=9vZoj68>W9=M@&KQ?jBPfuNc&liencOyXnRX#^deS-7 z7qY)Lpi^u87p*%Sa-XfAyT$+S-dNJcd&84tJvbJ(GOxv^>Z|xXI||Ks(55Rk9;xU$ zRB?&3k5Rvv7ACtP2Cj}|46pQd&X6|O44)ccuJ}4J|(WC}u zFY7;CL?>9~AKSlur%UKW(bfs2;GUhSyY`_Sn6}N;)%+lwDI-xLvD%c-FTWIuVk!ow z<$NKWJE*Asom2Im1%&4sFbdN-XZea+h(-elCEc+0c>HeK`p@J%r2;Nx?q|m(-=A`j z!e9wkqR~Txg2Fv-59OzIoo14Q$7&t7Vem*=Q~kFX#M^t_<}@g?nw+@M>GtaAwD56w zc>XV@@#>H(0|EwCuFy>2@Of4fn<#O!h3Z$-YT zeEwLw%uH}>_Zi1@if1FhDgMQ?;nE?OpWgK)gw~eGy>5Uif?r1~yyWW%EVIXtz(-;N z+y~HYD=>aAD9SwwpF_RvLM?npwf=J}YW$~POsPMAuo;=}??*Lp;>KO{v7wv)Em^OsV!XX3lU+ z$JTX7a2ziDK{E}HOwMZ4KU?6(6N_4cj1_%{VT(tY6@?;s5l2>E$Vc3uH1Ma@IuDOS z^X63Z&EYN|Z4wtFCMZmxoHHP0ZQ{QAHmrj$cR2g+4ki!;gFCVCI@rdf=;mc-X)buk zfq;!ISeFCYL~i4FZk#q8x@6XA#U5qppbp#Fu6}vF6&3wT^;lQPDM6{*=S&Idv@nOD zB%@roeC5$w;05~FWBeFwjWyfcCZRv$D$tc4kJ;Ua;p_^Z=KqOsdd zl;7jZQ52W_b0J`-Ybxojn}33l_0wCuo_nUAs>|1_%V(;TpaPmyAe;gy*dFukoN5!7 zIkjUr%>(*vYUI5QcEh>%cQFWFSvF(X zf?a!iHN~r(uoyvVpGRov>74$Z^Q7L*POKC9fUEGtI<(ZMFNp4dI^l&{g1G^F8PUE2 zVscyFRX3LgYs2+-VRJkNH}R1?mrCG~m!9n;BuaM?QDyB4w>(lmzZA{FIn|4GeXDx* zrPP;9<6^EiZ>Mu@x-=Dr^wMS^VL~1JW4V2plUnuUSiKGgz-m&-AiDdKHTdiX!kvAw zI{xK&t-60D-FF9ev2M#b zL`I!tcVQkC(dewu{;|5_@%e<^D<>w^tF0Y1dD*ooKRKP+vM3O)M1tSrHrf=-4R@rP2SE^_ z12~*dX1&h|%7MkatxCPcl~57FLkT7t`z4giCVco_N=UJ0G1IcAZDXPt{{5P&CX#39D`GX!hK{9mJ??V1!7UFsv6eP2{GX9VWk4lG%4ZB^kG;>w8x2=d= zVBQ{qjTGLLfi9>zbfGuH|32{!#1<>pIP|B+g53KI{eZWB%H3~ebv2uKi4~8^J!8wk zBgf`YTT8}CZ$7ysX-43wwj1_Uulh9;OH{x2#2iEm(c(FK!u>Mx z?lZ5lLu>u&n_po)8Fwe`>_2~p>{rD)m*>&Cs%jJzoT^9PEIp=++i4M)MQ9w;m(9)Y zcC*7sbLbRF3Y(@*&l$9IR&*(jb_bTEV_7W?&4suz-9A=tymR(J@goLnm>Ou<9t`e$ zrfI-zZdFj3Ls(?8x`5favx+Sku3a9zvGS)+abD1KAX2yZp&ANsEXF;8VMj2Lu~NFt zPfew`%85yX=Xfec?s-&f8EJ0LK`pxF%ryFfp3bFQ8|(Dp-^D%IHupFq2<%4O=?@%o zg5)o|C=$*~Z)7CbsDYfj{1`@ka+y8ej4y4SK6((L7y?YkU_apvI&`V}{eaD;qICXM z{al>TrvKRPFkP>CC`m0)4?Bx^q%6qJ6ExE#k z_xfgOJkJ=J^?ai6u0h&}pzb4lPS7m^lEo!mNrbtw#ExB<01INPN4vT5a(CHNGgz-G z`vGV8FYtSBzm{g8BlB@Q`D%W~ef4bVKlppIgv6Y|mmhN~Gu5Be{2 zf2-wxm|ZQ~ZsLWiC*{(S;36$lyO?aF&vw6DOG$NeHT;@#w#*z)o2k|_X@H&DY+eXN zVqUCiZPYeLGTxh@)e+P9(At5FHs!HCb+QVB_4_PZeTq-Fub|lfs=HmIadVX<+F-w0 zx~QM3gFaQC&~bI_jZQ9(eGK=xghn?hhzgJN@Y8cd2et~7mIe%+^r+eoU5qQ#rLAN1 z1(?yz`qaLP^o@qEevl>Apn0Yzj&*0N^i+uN;t}rv+Vl#Pv8!}HP{2xKC!2Z?W_=Md zyA3n6Zlba3Gxbm%Uz)Bz_orW<*qC8 zO42mG#yQ;)Ps7^KQ&c_Un70QjLpSDqmXFJ`lh%_^Q4=!Hr)KKOQvW7lvw)}N>3b*s z)WM>d9mV+C$(0<krWI!4x|p+cb1GJB;r=qHXh7*KKWtN za;!0c>T8MU5FVlLp?FDP`;#>j(F8`%*66R*a|qhD&K+Qw`% z-w4OiV$rI~(7c(xmO=-eV(^4%ruorh&zK3i_V)X*vRAF^;WDC+wn^(la)jGk1tU-FvWB*ZhSl8zkNgSiB8s754NWg^ll6&NlRIUuFa2AdgGBN2x?Spc2=~< zg~LT=43D%sl(qa78>mMp4zIVtekOaG`C^ycYv5@Tb_{8sQn5d$D;-wrg5kgj%MJoU zS$DGC?FzFRUMm#YK@&cMr`CY-itMEhXv}5WHR)~2Ige5!KVYMm|SSScp z9!Nh4_?Mhy+CKrv-+|2MPOptkbutwuUp_+JAC(sUEFZw+awND!3iR;mzjmOSx$Eem zch+*B=q6Z|-_s}9{=wQv3PV%_wC^)VX4eF(W(0csx_<7l;x43fk#2!GE#Z2W<}LYO zdv_|L`#SC=Pp$+QPD_pO>=ZH!>O3E?x*jTL%hV8*dqUXkX0$wAC0&$HNxw4`^!Y^n zaBWejLsxyyGx0<-gUQKuD6$;}c}=pLZ6--L`s<$Z{4NB?o53C|x~up>jzW=7ebR-) zfmzgJ2pKk?+Cn5lsx%4`coR=$+a(u?5b)kTjZ0_~t06vqg=4%nx<8^*lt>f*OkFYwwgP~jf+n^OB5N4Z4D$Od^!vU8)!b+*kh zT9$ffb>xRtJ67jb!oT^5KiJ}}jQs8??)zQj8QX+_c!=DmQB;{(6-{_rK#zbY^H(c% z*w67?LVY@I&r@wUIy@mrjo&vI*P~Q~GQ3w^dQUt*`ewEt#9VS}sDM`tjl zn7Rr9=(<4SO@_6M(N+cqL9Lkr1ro^pNuqh-EVvX1UW2HGW*Hyk2har7SMW8 z2~-%%ttV@%#UG2azt)cd0HFhGHQ%JREGhZh@#R>c`^@Xo z3Q}%rAe@tJzpB#;*al(|-#zTm6749)yU#zV)}vXcbJ;%HF{kYfUD;N3gMArYR3)iJ zT>h1Z;Cc=c2Pt92<#CI*z65H)HY~`VwSkRoM`@+$KFhg|>UvX;QAn3h9#pG{tiufI z5(Z%p9W%bVTNE;yZQZq`%zhs4E}V9~CfXL{U3H$g8B2c z^heSA_dqo^wi&Z$6DqYdnV5_bxtVod8Xxg~i~B zqfTv2*;CjD+M4uSTxy;#qPq~tN`g*Y>j z%IcDX{u)1EI#ot3&3(0wokHawU6)P{I61&?_ zW&6v_erZ9$rt!ECV|$Kst(-)lLBMr`ZYcK`sr7;99-=v=4ANJ11pD7#I>tv)mWLt zV|9%ZnexwP@48VlBteilO9ym?k7C9dPQ%hkgBLXV6P2^tn=OD9VNiaxFXW)N@ICJ= ziwQXhg{}v;*CQ=9Wgu$cJyr+PlC|ugfZp^7@XHjaFHd)~y&vn_L55Ifq;|Gc!($9P zqR^8CYM;|W3#&R+um3rELj~e2ms3|CG4%a0GspDYm4Wj8n{>vJINA}E!i@rwwNd`1 z&Q?Aq1e!M(J|Vn zM4Fge_NNEKn^znSaTokSR{7Xtz-PS-#D@7}{jy@Rx~wUJ;j{i&pph$v7`E?X7|p?c zx8&P=+`ai4VIpE%gdJ5(rg<@^7>1R|0j29O>XZ{z>+x@i^sOpNcdR)@(=cK@EEk}l z&scKa0(yT`DUS9|ySOjbxw8Xta{OsXGn4iIv;b+W%%t2W?!@S8T?0dkufsn^cMWz~ z)n^|2`C`*bSdw(@=i%L^{@VG>mBH{m1#PSF`|5V~=FLRDDBPgjcfn%GqTSdm+6CC| z5`J5QCg4v%!yFJ!5FdaU)(JHV%lK-Q5-5%?UCwJ#5LY2u)XKJi2n&YGEYb;3poE%< z92;Jo!x&BOQB3-WbLm)Ik%3ZzHKRrBiJw%79+wszY6nqLi$?Qi5p?;=rn%`y0HX^q zHL`bOzSQfFIWeSMrQeUG`g5<6W0B8t9yM4-24n`pB@*-r7uar5*y4}o=&rs4SU7Qm zgJJwzrp3Y`Gw)`+fBz0_j-Ns~K}shO1mLKrLppWsZPxkXfSWbi_H&(|*f)!q(9m4mkuDF` zV?Nj-`4ZQKTL}dP8BN~Ci*^of>WixUw!9nHx|FY-q^uxz_E%~pK9ivnRuEOf!CevR zXsDj1F~!hVd_{(!W99Ui#4*G2d}!Dz&z`&F+ganwMW3f7KlhE^KES zp|>76T z>NfNCl={1G{}vp4Gj+0*{w1v%rqb*)zx2gsliHUBJ@34pM+X#DL=L02V&#>jr4ypK zkW+tLl@qG=VAy1~5->Ax%^uMaWX_+|^>MTu8r&*4OBKbfFZ)4WB-| z_4=j5Y4t$dkHw+uM=#$LGSn|qi89REz2jqWo@uzh2MdKTtq*<9Px`w-p6A5cfzzu& z){*Rna({57NsP@B7V*idKLS!XzAsBNU$>VQz6V`FT(;>W;K*hZ8fqpAaXjcyk@XqG zQkThqEO(I$dK&tyl1CBv=NhTY5Ex?rx+zHeXDaHu2=NWYe(VT=1>YbPT0?-1}|tX8#zdL z`~u@rZ{H`eU&r*CUoguVy$jHEo<_P2UV&D+ER~2SPn|Yr%7LH4>O@jq=M3;(eCEMSGOsMw^ zqhj^7Klr=+wWb>b-nYh1*n$oM-<%QyKSMw;oWL3;W_fT6z$z^-uXXc#ts{gQo_+k? zd)LViq7cn*n$~E=3wCPi8`vq+Rb>kQuVOnAEnaf(<`jTZNP>X8?WZThgmMksdU#N$ z({NJwM#=d&dfe`tt18PSfxO1}JHG&MOH>QHc(mR>hgj-6h7dSf5?6U^Xa@wD+EKC| za*$ovcin~BUr(|)P~7mg#+5svk$Y2GGKzZD`>@QZxiq^ySjE0YQuHuNA$MM^=i1jl zju^u?j^6L;HLB|WTrz8>O2dMN>wchKcZ=5!Y|z4&!R8$)j3M{4Ji#Bd!*)R@)w+*X zDWZb0KSZ+}gOdT~j5rq$_{BfMyDKwtV&}DcPlb(=4S^fuRCb&SS{7&lSjEo5Bb~Ry zl~0)msq)i>y26DM5a)xX%OzQL%vqm}LDsi#FG^&k!xCKj`B}JB_h<;se^*OcR`dej z?xsvK9DRG6)(M7^f1`EkB?L|?ZllS%?c~+Xtgt6)Sd)x#m^RI0J0`im-(-Fof4O4V zg&GI?MzsF@e@cn#OW#q^Z%2ycSiM$O5L;p zoVGs}n#ar{o+Q?HmEy)_QNGxp(ViadiFBU|dFCa@!^+`*^lLCXy>T!>Q9%k%okt@> z!d?ceC@qtMib%qu{a~4#mr`?(#;y1su9ej5UG1N7W);JP{+{nBSv*h$@L6MiQNxg9C)KaVC?w}>lIm6dWB?d?m#qUC! zTbV82y-IMn3+HgJ52*JjIy8AVve&v1+FrsbTI3>dolV#vTjzbB^4kVMAc*(gJecaX z2UB6hbh(j&v%z0OlanJJcEdHbBU;>}wJtw&P|@S&5Lns57f~&t#3bCa*2v2_iD83n`vm-uklpde~=czwz82JaOZYO=?8G9sLZ3xQr%WvJ>sZh%oL=vZ%tM-6~gK zYqLin383AHiM|i)IHrh&BL&;nBB`M)D%e^|m^$nxqK2n^a2BA7jJbJzT_)3kIEDofW&jC|(pm@wj|KL=j{z?4v_ znokpt#j)SYsrDq|*ssvYBQ7MV=AQW6H zRT5xVtQd{5Y2AQIjdhA*TEo4qK3)1b7;^w5g;s4j#1}&MjiR2X<^OT@-tkob|Np;{ z$dQ?2L?qeAs%#lKsBFnNvPDM6&K5F{k!;zU>~+ww;@C4p$R62yWd0sr@6Y$2-#`B8 z63_EI&*$UuxZm!#+x5;|3z~ufHPT`fvEP*DLivGyFGZ1sJ|Pnc?hK#UDXrT!f9Keb zZDXZ;Sks)9wV(czLw0_^_=-|0>V?O}oNm*{X*?DcCVNnCpnSStKLXW7*MBYL(Ig`X zf}n!%G7pbNj;6_7L5bxd=!e>i(j6mAQO6d-{#9zT^A9dLz6OhA3f2aEx)7ShTLd1V zlU9lRBsBkPiX1S?MAjH03nAcTuE(h%cT)d|)_JYu2}%$!kDNLoI+=i22SOE3bpS?F zfeWAVWVu=wzNs5Fxg%A%ZRDm|lf5wikTOO2#vkx;O(}6Pc9>!ggkTN!zYtHij@IRS z*ZN*FKDdD|_?0&AP`TL%xp1mwb;^i^V2LrU};ZOf~0!4ely~&Kc?D=Npx27d$&1_F5Ys6ZtVsP1voI0Y1s1z{NAP)Y&GD z6^;}`?`~w*$$sOA(cj}zpQH7e`EAsUgS6Kby%fVa>{5jK!8zC71Jnim49B_Pl@)QG z`x4sl{5}45NR{cyDYjAexZ>nrxj=~AJF1pqfpHd4c|Pe?+t>w#V*4XaT@K{#4Tso_ zIweoFhGo!%1o;vYQSI5L#zlWYQg|bdU~ihzio{IN1D+Gob9*azKq!2L$$u+t`Y{b{ zi{PtRugPLNX?dInuI7gdsy~$a+sEUzy1tblJv^0IK;1k=_ZJqX&N_s+Gnq0r;_&YP zN?Ok!!J>!sB?KrE%v10$u46284yDc8O>tX#sWdL8@H?wpBM#3t&UJQR(M#Il0iKRs zdK}M0h%U7a0$=?3^ya>YG-;R0udzFmS9|MRUx2QZJrCY#km3}2=*{vZfT~+GK_3Ip zMB9B2#%l$e$}Aq%^{8=1&BZt3_GXT!BgdEm6cl5j6+BJm{8z%l->%FEt`ZX4Y-98T zN9a$wE*E%|v>XEu@EVtDKb4XM*mS70exZGhB~7?h=Utb6xf1!ik$#8t=Cid{y5OOM zC1hc=Y}>D$lLR>sqjGw7XZnXFFL)On#EvNp*qM;NPAsTun!-r5tMCwFJXh`}WkbM~ zJF96U_YPUp38T39>Zuv?*w|;DHsqV}+ZCO{O(UN+kdn{%3cSr8=EgW_k4?@(F_C)G zjph$|vf1s4%oQ(s2FM#n(~>#`5;LH!a&h+`1>#Byu|f)wlIndGGggee_oF@Wtt1aY zauM6h-BjYao-$TUz|{@?1`ymi`sy4BWZP z6R4{ml@vwks^YZ|o%NZq_N?LM;d+|)?sqeH1D@cd@EXmt#J+zH3seOw!%uiz4QG z)i{ccP_(%p!2%>FY{Pk{4YL%|7z@nMASA4_HyYvE2L$om`kC^llWz(b%b#AIv-@o6 zU;gWwVk!9uER8%YVD~|Ukx1lQme#qV&fS5~_YD5(O9IcA9jHLvP}e&lM%6sANcOS& z9n`_m&sjSj=KnCmB0g403#PL2TEH*P6ibpL(Va`|vP>hN&~~ZUK3iVK+L*Q_CoUPK zo8q6pR6Sr@(K~&&6&4w8(k@hu3<*OT;{sbQbcMEal2s#Hp)zYaoi zXcX2DZg7hgAFiI<=>HKgK^5CtJP*TxUB4V>hk82Kl7S{N>LgvPSPi_cyz+uNP1;7^ ze4z!41d@xq+e#hX;==W{$(c(dN1x|`ig3a7Fy`t~>axS<_$lp(kI%*S)kV|(%-K)P z9n;Ip52TmdM^`=_cAbRLem)G5wdYOaPcN6Yudu4qWN-HdE=~E1&XIyqGp9^a%iCA& zpN~reFOfx!CnX z&s6hNM6VLdkTFDOO6zIVmvg!Hh>I?zf0&bKnv}f})JcAYV)q$p+Q(-|^P~g$Ix(E{ z1nG9gw~el<@?^cT)?2PoN)rFjH}yPvS>?2KDlYCxG<)&8#Dgnky+^B8Uomg{lB5y_ znAon2>rQzLFN3;h<=pUnkUL2Fo4Lke=F?>R%MAhw@$ZJ0mA(quX2cNiKzwvUwqxHX zHaeVEsGcM?*@W^^O(*ZDT{5YpFsBuqc#D|-_<8H3W2W^MRT++ayx7uc&gni^kH&t- zebR5Y7~~T>n>q}2-&ZUkl&b@=1Xco7hX&SgL=g_DbxX(SjY&iJA z&DZewGRI<tE!SN`gyd(YQ~*UD;dg`pU+aCZX#aq}+Vx9WaN3Q|RzjO6@@mHQ#hVCp#N49FRl;p%uGr~r z%^{0dd9$WKk0)mEre0PC62Nh%h&dIfkX+vv>3w;=5)yeq$GlM9aYF^iFdg%;iDu{6 zFA&5k{df%Us_b68!(i3G+5U(l4g=S=(9T?{(qe|dV&#o-5lTxee@P1f}b#t&AZH9g(#blYm@8nOb`)2Qw)PQ zQ}>)f znIR@9G62)H+$DiD6Evo}J*S`nxi-4R*mDeAr2%e2Wv`b4)-kgU8F-V>>5^vM zUy;PlGOigd*jLbx5Uie3O8pO@BUS6{a?wJVA)eL~vV#FV2sPIXTb6rBnV<8C?0~Ai zN8ouw10(wlj4EQObv4JR_9Pl2L2;{+MFo77ez0VE@}QRO;^<(bFeP9fIC*~-k&q~t zSrr0|32vb~-}5E%yV){bAr$%?{eCliU(hF$gg@R`TH>1Ris1x$HY0d6Fa%*NCeWc; z5N-WjLo^4wVM6pSBN&B)yx;j_51RG`13uHIYQVd!A_#_%A!7Owq2$`Ls<(JaFm`1iEXPns-~`q9$orA+eB4jy+W)N5fK{9KUwb0D3n~Th zINfYBvV1v#q9kZ((O3IaSsMp)9(94Y8dpt^u<7Ydx^<1eE~Mpmgsrjr6-U-Z|v z8X{7jXTMu-37&Ve8BccwPDoA}q<~8is5_@k-l{YNM51-7>|J)jEHV@@#6JgY;0B;x zCDo+Q%f?f>LDU^WXMf;8C~5F`?^N;6%w6N|U-HzPoR})>bI@7P)^Ici};?lfw&&a%!hbFHK9&&*GMDRBgEJ24r9I0x8MafHDuD^?jT+2$j)Eq3 z51O+Ii^I`v7MV9f*<>j;vw;>QhdNT=G3j8%nwHhbdwyU;-7J1dimBzc<0k<}PLEK8 z_|2ELlMMdR47r+4d+B)EgWB1w(?23%sqrV!e?X*E&L&67@J0PSmxIz)!{5(mf)t_% z(pU#{DIvrl+025fB{G84r^mG*KCQt%o2F!s@@rM8mwg5M0x6@|^--I9#h!ZCts_

yVCz8Ra^l%qQri)-}avQ3WbZ*(Z=fU>SyoqkaBJhBiN$$N?uV|yI}~T z;GDZKmN5i#BUrp&aAlfU6l?=umCpJgG_E#325q;1pe-iw0dp_?Fad`$)vu#){_o+y zskknQ?q7@D!$9mV!S6WU+W0+{A7lIdq<`qaBbV;=Ad>e>Mp;IvXM!AFdxyBBhAWio z=7m3?f`CDo*h;KMw!zElRb@SDVgFl1URRtccE@zOe6I49pst*t8PS0E;^m2&=9H4iB!+L9&I!4Z^ab- zUtMB=zp3!So6mwrWwl$e+!TlLRix6Nu9l+X#r1^o{F!6(G7^A3=8kB4>R^Og&)SsG zx5*$i_#2w#aL?Qb)yB#~-T1=rZhL`5J~Pj~7!dwTUn(lO#$B52!p^JUBK87!?kF%g zjKLk$K|7Lgpu5Zxwn)csj<zvHantgr)-(hE7RB^WphKU7uxfq3O`D8F9c9Xj+NXj@~0QQT$Ym2B@barF_z zjxfU%CoXPdd<#q;*+BAwsRn)%EAf-pzc5I=bM9G1#<<#6Uj(1-^GUPGuO6@?~v+U#(sQPJ$T2Ead zE-y=`KYcdrybUgK0Nn2^6=oHNHoEsp78MK#KEt5d zMr$x_vIxq@HHjh(Tpm5CVON z%9%DNnL_;+uRNpV8T~ZmPR)D5n|cVuO2*!VjqbXSXzYg&U}r6T2naeXx#O}0LQm|~cLIKC zE6ZOfp0&Izcr{ihNvQYXc2$m`t+<@E;F>D;`HdsnKUp_YI|$QTIw9Hf;M=A|kk}e| zLXzaHcz7+q$Jjc79+M>RTnRF+A-MAv_T%nYQzbSkb)TDS#wjNbuw*pmDYgp8s%+dui+dX~373v{sG3 z)daO8kzX0iDc42U%dTPL36ZLGC8(X({TJr*?wotUYuuz4@c;b*a!==RjdV%H^%XUN z*SswTJ!s6h=lsL~o1d4FHK`$dIC;iv0*P4=*ApizY&pGn-HmF0RyVM7*L|_QjKqIB zbITQ#?2Zy=EaZ0zk9p(%;{qHN;J{mGR1UYd7d3nc{Oan0_EC#eAv<%bt|%hD_kA|* zX`v!RRghaxLl2np6HQCcrAc$qoQJqV4>18Y4n51gePY_l91q5~B((W+*`h4nQ1|zs zPyVt~CITCi3>y{0^M=aTjPyEE(icpEC5AtA{rL0|%SL^j`O-PNWuMl8A8;w+@#zdKC2T={`K zzH;ka4j`MLygIc;n|=ZJYWN2;<)$-VA3)`;S`3?}OeDbI6XwiCkf7P+bttMFv3Q`@ zuM3=ZqT04j#jPD%^}2%JS+^U*Sq~PU2ahj(4yt$uBrm5H)}7ouHx?ovDI~@unp!jl32Mf# zXeI8RhtU`&x9qiq<-L_S#O>Csx0$N7e9*%Z^Skiz!249m@4q>jGODVWZJl0}E-e8sKe||7?NcPMcJ%!iaaFQ2kFk&=BRer^4XiXAtVnXi0P7 z!uVZ(t1r8T@zx5{1lo3gfXE1y%eb0#=suIOKk*~|U3j<*!=W-;ssT%XY{3!pXJ91! zg8M{h7cim0kadvYrLtme40sD$>?r-OpI*}$3ivu{MvyZ_>=$%x-M_cVCf?hv%1{4p zn`~Z`2KhF0-ud#o=E(-hhwR4Pv{|YcvCKq#Qx^36k`+0-4b|gAczq8 zQaZLPBIn+TZu?jVLzdVz4Bb)b3B%!lC!iS1rNL-(?S7a-yBb0M{CQvJmQPw}RibIh zUIVlMd73jZ|Li71q_6T;IQ&ykF@8F%LNT_F0(;$GNchB>Z=K2jufI=%b z7vCO?tSCXg?}Xc_#0?UHyszG54brQ!4Fnf*64Ub$)gfX1{~dB85+1$0 zE_?e;X|4}aQl#T1kdIk6Y5EyUzss~DK7S`lN8#hQRQ9z)5szjFX$m`bw{KVNVBUXl z{w2K35He|b0QwYd>}jCW6uS#$Tl}!O4r|smOO$gGqgbZ8U&aukr~UOz$rUw$;nNZ+ zTyQ{|;_&hyDdcEoEGm*s!BLun_ow(byK;pdwuuj06(3myLS9{!=r#nrc5K%G7StO2!0Y)gsV%?Xc>o>I6tDLj*dIV_?-5W&* zxPR!I18`q-(HO9n6;J;NPpKwPkOgTiP{WS2R06*fPMqkv{>EF7=&o;A+9XLDHJzD? ztZhIxka0WNbN+oezsyy6?1g0GIH{^!v&AVQKsnH^t`(q%xi39I=f$}PvDn29C?hQr z>l*Ta#-{#Sr(ZykgN5@~2AcMTE996W>><+r{A5?eQuJ#C$wv9$ysLG?!mX=+6mOge z{NeYzo#PQ_<|kiYkv!&HD;oPot0%kO_M4T`yhQMTFRZ7li5}G>`az>7yY8e&_tEp% zuIL^69;-Ozo)wrvqA+s$?n#=k#We8TshiW!5j5qlR;a}j?}eN+2jI~mCm7bLoD!|N ze-@ng>J|3zykTuQrSw$Q7qGw8-6rkU8%JjACQogq>$6OkqOwSkLRhUYMWOZ@y!+OM z|D!*);~^e1?l~B>ht`lMl-pXrHBu-`UMErcT`n#^#2HD-Y@6bz;ZdO;H^wt^rnY!Z zYgGAozo6Txq{3NMDu|_^+q;4^u{&AcTu~=b$^25^oKMg=agpn6l90OXRbZ5a4uuBe^+@}=X;%6=`j)ow6duHxpcp5^}PaoGnX{=9LjP@8i4TE8?V zBG@vy35N}M3-9m3et4>_t0O7w?%3~zkO<6rG)ReI(+t5dsrnA+=5L-W(kp8(hW2{F z;^LIcre986)*9A4nHH;Tb7~9s0)!_QY$!tj9p{H%lqiMlH-AV$&4wyjKne2>3ECO_I1`(d)Jt_gRMi4&7V6YXOSt>|%{zlN)aaBctA7h#C z10&}DZI!)%IO_?t0%m}+vhDZE_wNAMG*vXITIWJFxH4WbdmImQV)0o|B7F)kjeZ^$ zsCw;JeBzZsnV}-QL{%uVE3=CdRS1}7(e^}P!I5~z2AsgJ4)cQqmf5UrFm59sc42`IPt6;Nj|KN!qstOi0!0@=Jt zsA>-#Nw280-z9J*UmjYE8m16Xvv*S}itcV&iZXr8m#V8uR&!|F>5U$cD8MGa==n!n zovhKm;Bo$coeqM%kY{|Oq7@7)Ik;bKCvX>lFHD@oj4#rWr^ z^z(Np1qdF8jf#zkbM(E>yC&EO#4Mn7 zfL-gU&HuJ8I?MQUT^VdZJCtshv>D+g{&WpFa&>Cro%fLkjCkLi`YJ|`DfnLL+g9_n zC1B0dlL6{!MmL>FDoOUn##j`RdfDg{m+a^~6pB(WMk|_F5TqsmFKaEcvp_v^g|i8sT>yA_BiQFj z(L12p@$AIgN7m30xAG(xqsK?-_1k$6=%$g=OxG6GwIijVi#DObMc(qt`D`?~Fhz%I zTY}u{vG*(=5R}NlA^*2@F*$%dors%&nT*hs@_@ zmUn(vKXpmqIyZUc%;g4R&6WyLNqbZw;G~e%i3i5TAi;=aPnO_o^X*g+=MbV+J1Zh+ zJr!;Fh*;&A&0qrhR~r1ntzXQo3nhkmtq7rQ^%l|fgLC<;&)%l#^Y*xrggxW7fUfUO^khhh!t7NQzK!sU;S8g?{PN^Jhvy zS7KPxjWe9Z&<-jw{Lt+*IJclzr0GEw6;kNKr$9sK$vc zCm8S>*;Ye1^PCbAE3+DKbr#k<=PGYn=s?nX& zs^f`mv1m)H)5c=Q(gqe%Wcj6ZkuJpT8_|tGbbP@yVP;rQh`zb~FhbE2qS~Wtrr&RM z7DA%=G>QR({T@PQl|HKIifSgH`;)clfeUS2gtEE3F0^(^GZIy3GH0&*twsYyY)nSI zM^*na`Ia!i;_k(_B6NF247~PB?37Xs0eAPs0h=;M#;Ye=CS_|`hA|dHz@q=@U&v?R zr|P%TpFDeU0Il}&c_CH5-RgFE0(#7!6zYF^M0DqjjGOdCl=PN(&UR^q}?*=fmh8 zT^`{!zU@81S>lWt+XcC+g?9K|3#9%h!LLhR-cJVJ|H{d!%lY>p2FmG(SY~nhm2}-! zRC4Gs(~MI!fSLc>b2~rZX~($jFV9|u3ok1s25!G|%oQ~U{Z&X*AxWZI1_j1}~EHzbZ)WKa?NaPNpAwWOaOT1pg_Ir)DkEgcw+37QJ-WI8XJBK3B5CS`9XdCr)2>T>(4$^?I#Bnih z(=q4It~#jrBvAvdo(8)sI!uo)hfV|*S{DE*2V%HD2f8+5!}QuVAgX&>C@mpBLAJRn zST~MPW$X zedu%Ab9w%N&BBYn<@h-A?HznK2q`Yg!GCW!J#z3*r5K6mFCWaHNP4$hgz39;LaDd) z=ubYurpDMmNhy&bzEU%_Z&Ws`N`3Ek>6*PfwYz(HsT;T$^MyM8QP8wWd0Rioc46MK zUdu0BHRCK1?hwQzT7RbgY%1#6bC%oA@cDP7QI3)1h_K;*a+cJbz=|z?;BkJYFiO`m zC?)hbUrI7imsUZl3wgobq)+H`@&(y&E3CjIPU~g4PxUh^w`#x3_DD|sH^^5PCqkCX zh2{r5hyM6Q1_f*Fp^`jCD9`J541pr|Z-vvg`oEZ52EyIe))dD@OQ1qW~*zmB!(~ASgNU7sm zFmC)%WF%RyJMk{!)%bI^Z}H9q|I4=k9j1yCNFO8Y$a1m?%+%0l%4xr9Z%3}^(BjT! zw~dq*9lhYpR*+_O#gV`@Q?E)>^n-Q#;Y1a`VguDCa9>Q+{AY5Y-Nc^>R`Rfp7zP@* ze!}+fXGhY-T3LZ)%=T%&WVrofN_Y@>F*&3tGGLU`EEQW)KW|?(bi&%yf#shyZyK*t zFtATCLQ~}cD$3jY_p7H`8N39&_X*w1_y^xIs;f=REDli`NlKF5?<({1kK3UnW@W0* zQsr}9w;Xbpc@D<1d6D&}b{W?G%tbxv`YJIiT-RL~f$kuevb;_@2&l4?l4`Oa-Jqdx z=FTcT<=~!(pglvqx=1b1HwXBg%rwQS6!VJ1FD1a9$pWsZ`Tx2aKoFQf#5>P!&LIC# z+>4nv!@SIpIDPlAB5u8Hs>M~+wqh+zAD4DwXi6-gsH_cGznj}TfR+UVo$i<) z2s+6#-{L$T&d0+b@1{6={iga+`Z_)E3fit}X zR}nOg_oDv>!j%jTJT;OYE)VTR8^p-t@8g$hB7r7?A=QnB{QtqEV&H(Yb5Uq2r`4bAX zV5W#`Q`3=i2V%qe6X9Ajg@>s>e)%z#=-)wT!7CM?q*>>e?a|z7KLCghcuIB-N3(2! z5m;CBS4U)5wCbl8yj(E6f!A3(@LaP46meBDs4C7EVsYcz`tR>90iyN+diqBYuP26J9#IxEBXkZ1UiybdwhvNY=}37h8Bb^4D~D%rF(Lu< z7+R&6b9|@XOd!EW-xYNdz!^@C00S-(Q6NWSfAv`QZW#s+PQKfkA{o5D+6c-)Lg#Z$ zezR(JzbnLXeT`7}@najlarQP^L>uUYaI#6?j!`OcIv`kAz2GTB0$Ys75lFF&Zsmvc zbfqWn0dUfAM_BGmvO7d9{i!?RF8sU-`yKR24|CD9%EFjVQ)Lj>i4**V!4BaReIDvv zE?bXLLbXx{r=rQtWWEO{$brDzpW%Uw;OC3%>qZpMZCKzntE~+Qpbc~YB}t*qMY{qX zBQVNM9lxtRMmHEOBtgDU?T}pnY%B z*Q`(7ivWlr&+f`ksAl^N%HI)@5J^z6&`4~#t6i$!IV4L{sT85LucVTvp}pTrn&K=X zSu5q#UZz@wflDK0s<8>X#ux%GHC%2gsVL(n?bf>1O#(!N`y$LmGd4bGDY7!HbL&@s zNllIIq>_BpFR-L-PBfTyM6iljv{>Xkr3ofSqra;%m$Nw3*Y zn${t>Ku^rHcf2p`)V@sE>e;_|Re;4dv3GF=>YKnTKu!cpZ|U1sj6C>w5%Xp6u>EwQ zCv=|G#ZSe17mGq0@f4auI`0K`>V_eWl~^96qyG}BQux6JkrqE)>L*z}##xR9xAy7+ zj}0;NyTS5u$>42Vv`CtxQU{KM;w z-Uag}szTb5&_J8~HP$?1vGri`N!hTJ03G)gae_qt1ef3x1{*LeNB-0%N|aC8Vn+91 zsz50?6?Z&yRxkiDvp-(XJb> z|85pur9XrHJp$aq40B@PrO{?+z_FnI%R68+b~cqR2B2VGl);Z0=t}mLk2I&KIeHQ4 z1O;eMmd%+Bc-OP{9lurE4rmnyLo?0v`aX?cfju|&+s9uBavc(nrxb9#!4qR=>U-?I zHT!$aZ|jnXMpL1cN^^1O=`!rg!k9v|;<+5LcC0$4?Z_kv<=zS<82J5{cHz9gq#Vu# zzO#C;FV|6Oo&MqPUzB;0gqrJNuV5N9F1xbBLFlM}vAH%J_H$tC=s74B^E_O`wtmBC zuU%GJ91H6|MV(Lwo$-BrKY#u^di-d#(n3I6SyL!#pPkR3!4=C>v(`;LKy~Dob*<}!PVm&pUSAB=;i*R|j z-vMvqA4+x)EDRp5D~zqEywon^b%lTn{s1c!eEh|mPs^1uTSTK7V^qe=8< zIiZ`wdKQD4GNa$y(Pwpa)+^JYnwbnUL8@9tFVqhUwrDE6S@e2ELGb(xxl=4fR%2Nk zOxWzQm8*Pg3i*)Oh@$(`VcIa9qk--e!Lt+H6pp%C1 z_$+NapJRH=_1XKXy{l52mlTiG4BYjKHGbos)mEN0{-3k=1q+7ROOqE(-kGn9gv&5C zLwqZKx_^Gvg$*3e6V?7q2A~72+9mpZI;?lxIW7%0WVOoSiS8&ssPxfXv$63I*p&UV z3q3nxlxpNt70xP$54owK8GbJ~cRLF@-P#q+8R`0$KTJDr&vE}=sM~Uic}rO`PM@?8F)Ke7jqCCFO(P8tREt)2Y8oh0{V1CUPJl zatrsSd(S6k|4hzJ@o?Sl8v9>}RI0qPYH_@ zW8F{|hV)lIPqf>m&Nf`%+>?~GL+S5=T}5#oakBlqAsicx&Ij|4vgUv4X}>A^Ac#M0 zltS`n${iDWVM-!Y4T}w3=79lsJGN{OT6lk;1EDr) zBaUyqDFdee&kJBwnDO*9nEK#I_FRsHujO`uGatShht0G7VEbT#rTZo2ZO~>uFJ@*u zB94xXam~-ZJ?cBq=bn%&c*MW)BNM&w`UM!E#=&=?3J=M(Qx3lGiiGzm?Ag7>N|5hf zpO*wk1)4Z(((kSTj!th6VnZR)fE4+tq8vdR^Z~FxRIH1p7I5vhP34C(2 z>_oxJN=$@L%Y?#N#-9gT1_a}uX+x>@pm-9mjhUUZ44IgMNy35{Lv?XD6sb#s=F4Gk zLo|FKZRkM^cCro1gVOzS0?jd48bW*i?>HE{7h^7k6l1)63u1U|7*oRa}!v{fO%>r)aD6N~h@pf!# zmw>Ym*}mdUmBKttI@drVaopL@>Z}C5p&PP(p5_|DB?kqqz0QDU$x}1f@x8m3<7{jX ziravF@CAATh>56~H|LrOt?jpC4KpaGGdIqj=ySj4bN#SR0T{T!4y?xo+k*pdw71~= z7NuUdr#<sog2tVBBs7}lSF3Qsdn$XF!Y z{`!K)hY4TQC@%`esWleBO(5oab3e|W8)Wkd>$eDS&Ke&;MQXGjqdpd?aoiuvNQfSv z8Ni$Den+?SBOr<3#bw`JFkXQremb*4-< z(_#zM3lM4g40EPq;wVTA?|axkVjuw&m_O*U=U65gK{{6_i_Rk!7h$R7 zU_hby^MN$v9PlvA4AL`9Zhbd4esBn^4Jmoz#YEF4X>M1xQ;8yswPWw^kocHpMv*XZ zEm;D(NX$2_wu-QS%GV?9Mv~JMr8xYYe)W_yA;!UXT2V!3x#czf1HnO7^ZD-`d3;=* zC=hk^wjmZ(d{3T|{Hh_i6=CG}f|xHR5KU8UbRsMg!{ecP@+BZ|c^EOsQK3TzkC~8N zhjI1$CoNKOdYa0+hJe6|H9iPocZ82*k|-OE--~|N>P9dTY1v&M5NXV_{&50rquXTu z^1R3pcwG7Ge2&0>9zlF2w!7ea^=z{P>O$u<`n#OJERYN_*zwF(fT>0#B!{F1M>Gmo z>;BVR5`=F9_sr{%l%<@zEvc-f+;hq$&(u4RjSuEvcA;{wabfNso}s~gef6is-*iS^3{ zIu9eM8uyg6*)*HrsKq-|;vX%}nl)NKAXnB`v860{C`I8wV%$)Qf=!us3Vm|=#cOG# zAYVcvZV21{A)2isI+kg>i9^cN+C!_2Sm3c!hVpJ5FrG3c3#oOJEjQ*Pw5-(fB@CV2DM23Ih_9GPyLaN-(l_ece^C=3!kvl zlyyJexc9!+j-sJ~rgf6%oCO(W<8b+w~CxY4plkQyM3ThLU zayY3}l*?hacp1BQ?(xvL|B}Sp&1VvJk5?$W!W2tC0(wOrQIi5-uxPb4iFqbT-+z*y zX?l7Wj5QrJS2Qu>CC;t&6ts0p`NYBArFC#fW}-F>CDFy5_e+1q$*pfAGtu%F_G(1g zk%45yd4eJ0kgD#Bn6lHz2MWj0_pM~uBAtsF&t2Hw+I5<0Z6=ICj`{;#`}`;Fj@98iuJnSR#_=3_hx2Z|CVw+*V@s_toTp006%~kr z>3+L}DGG%ltlCaP>^_Wt*?5kW0!O`)Z&!fc$#12SHk&zLn+)hXr@|H82p&)* zY!-;ysH^MuBeS1bDe0gnHew5_)*IGdAKhW^4F5)E70c^#^p*#yE^aUSd!c9e*TkRGOI16uAIP$;^GLW#I=pku?QelbMEHq%u={; zxNvJRfepr0uFiFd+AzX7m#b%i9RvflnrN!4S~h>HdLG5pw?a)uIHJ)ajk;plCWbDR zrEdjuTuMIXrjtUQ9$F zgI<>Op)1?yUB2dPUo=RgKO-ow1pDCVcV7xyP9S`r{bJ-0cNF%UV52NV51ODp(GPh2 zlOJNJ`-+dRfD&E2fwthCUs>zNSUlAiG<6nI?JJrT--+Dv=1^o)1V@CF(7kCk%}RRi3))S|)0) z>iXW)!hmF>N_Ok?8$Xltj#}oV7w$Dp9-RK2(CKVw27qEW zQuy)}ZP~;*n3*-p_Ju&cgIcGw%tSz)_XdJwG2^>@2VJ~p%%n!amoZK&0zNXUlpUz9 z@;Q+Ss>n3NMz+o!1sFGG=xjD23@CKcdEHQ+?5SVTYBx{&C>lDK#BEzo<{#rDZ0~Y+SCu`bZ=z- zLT(zI8nO#Ou(pCrNm6;-pzM#R(_au1OZb*;BD&R`H_DGj99IYcw8 znBF>#%dW`epom(qf5#9vz6T~4q5cbo5Nvt>(|Qtuq$sj}B5aU=w(P6=#aJ_5x3Zin zDOtzu!V9+5Jt<{RIS0dfbrFSsp{#B25(EunP4ido3psh-G{uokd!@2~9cWGmI-dw3 z=e7Q2Q`?=OX2nWvNfJ)cmE})i-S&?jv_S!VkO)QifNCc5 z=W^xXseeJ(tt;T88xn8&n5GCErwf4&Beg7SVD5KPDL;G%T5!4m>kjWJpmb!ff>R7! zTI$IK@NjZuKCepFYAg`rUBTCrThjzAof95`kh+n+ST0gL;FSDL!j z{ZN$H2qMhyUi#iMo@PWCv;lq)^NcAO;ze`jgHk7o% z7%HCcuz(mzNFy5VaDNzmD;veRrMHMEWVNvuiWX&4nun$9zwlsTHZ^1a~!aT@qM@d?a1Tbm17LqXp$-` zRu6=M1E8zcV^5A`G(bghn&@J1VZPhk#)0J`&<2W;s^ciR8-fw z5-mDFDJ&0d7G$~KVg(2@A66 zLdeux%6Gsd?t{+@rVQdTtd#b}gr3dZQYDO7f1@d(QD>wjCJc6y?*uL0U4{jFlpN5a z0#wA3(c@fi|E=HWzB&x^Cq--DiiceN%w3K)2-xzLS}iw9|3 z3&mB~y^*}M;3j1Lus}Q*Ev7?QA#J2GOB`3^$zmzkm!)-zJX`<+S882--E;r~SZSIi zmP<+T;E(j$R4_>3RW-GT3EF_BhVdx)>wk@|sD9#JqzWu2%SxwuTCEbh3R|>}WlRP5 zg)6kK`3V<$@x7dXxcDZDIH(z6lmAU3|FVRc*E9ks_9&pp&dy+=yvP=zigV&S6v&Lk z(8j=h|Bq1k$G{#eti4`M5DC=4gV7A|P1Mj}zJulmZ`6(c3ETFKZKj4)>trM14-#}G z0o?qH3~k$&Ez_b%>E1Wt3Yc46gZ%oqIoI-+Cu8T|yIlha9lIzjVm5=P=?gp?y;H<| zfpfc{#qR?c=t516jOsoKjs|~(cpv&E<&85_)*he8HB zHfarl-$d2Zc$4<7H&tx_tu51R!|nK=W_&411^v`J2THo7kytbSuZ`{t^+74Rc&@^B+PTHFXLF2dmt`tmKsY$zcx>>vp z46|SEgA6I~0|@0@8v=67Z(lC=ziAcJrhd}@{E8_TWUxQ%C3T%gSG?OZyM)a=cBAUK zyC;bp2F`%#gyU6=s%_%Y>f^X-$k+?rGM7!fnE25Zb53N#`LCW~!^`s}RsO)MRX(5p zkE!>Lr}}^6|INse5spH}NfeHmnXD*p*_&e%LXN$%IiVvXdu3&0uN-^tO-RPEXR`OG z@6+e|``&K9|NA57oY(VpJ+JF}-0ugs|K|De1K!&Y$#aW7Z_`y1<+C5hSLz@1K+gZ+*#7Yjk;IQ&Mh zaCwUHu61n)Vot&o6 zW38J<{4kJ1MdG~gwF=1)MQ|oX_nFS#{eL8XyD4lJ=vROJyZT;bRbwtWc-L^7wc2_h zIko;Ht20M4UNQ!3pMj@-%5@7b+%a1!r1wL;szo-0uzJx|tkZ4~F)it$=OyU8A$%oJ z$T#V(fB(P-F{j8gON$eL|hPmBycM5O*Tg*Ej$^XlQciL z%jpkd-B+WpTA=Eo9D9IL0#kS`GCnxmd(wLK4!A9Z ze`!hpxXTQU(Qf(_CP(KT9P8qTz2!^;?VE`_O`9-u!VYdE5n|%sKTOEccs(cL zb9{8iB#W~)fy;fUb3EgSox8e5JoIo^2O*wB;w8B@rvVI0__TO?`&tjMFsDY4&lLxy z%Uhaqs(@!idknC!g|)egGo-u^H+DYE?P4il2^(+Y|ggj$KYC%IFr zaf~1BLk|;_96iy(dHywtOF-$+s=u%ttEV?*)=T26YC^Z~0W`*B$jk-8OhOTeuYuiu zUkp!@_;3c7HItegHf11%suo>JGyHUC-utxt+h)`r^cB9wb@nCQaI2bio}U@N_Fo#H zL=Qd4Q1}_9T(x^eV~RI+K7H`vLrl>5@{{ye8V25+2G+Ks2)ihf4G$NjW9`4H&2Fa> z)r0WAnlEvo2Fxi6e>gsT|1W>nsf#a;8tV<9)Fzz|CEvB;fV};^dAuCUYgvae*$|

`~fskYC?;*YmQEC2CF7jU=yVt;NB+e^m7D3V<95<~lgfBeA>keFl!7*RU8 z-6Wg7QZ=hRm`EUJY@x$5qb)U5pIlu60r<0W)Ce??(=r%jML-`(W=2k*w3^uww<+t* zU2F7z@zpqve4YnP;wi+|kU>NPt^=PHmcjfk2*W!7|1+3BcA&cTi;i?5wp8G=2jDgU zWrl>py*s4<3}L{@6Rl&2Z8Jef(%3FmhIvfMsL}F>KUt8GheTF;+-~UPTu#4Z0<3B{ zhF3EmSZZx?k(6=jtjxK$#pxYCAN32dBQ^PEsl+2gdx`z-cdX3RL%H1r*&LnqPos~rRa*%KN__A9aW&$j!YXP( z2oYzNtC=CxG8vtncnnO{E${;xa>jhYT7f6mBHj_H5@`ozavbmpMIQCD-m=rW*Z|&+ zm>#P-JF;$4XX0-|A^6;+$o0vd zmavXDw9yL&ae14Y&SiG^W(z*$6>seGw>A`?^Bv`8%$+L(Ij`1b8vkySv>II3Kq1-u z{vYmVyq9MKcdoLmWj&0N82=}~dH?wX(VuOWx#usDsp9WXzGm@=KYF*POCz&k{(bQ1 zcajco)?t`~7^Z6Lilgy~^GY9^%+ktH++R-N6)C}y9)ssIeyas?_mbKf;_lE)7Tt<0 z4c-6KIXeeNK_qEMWUj5(4JZpL$2n?0_!Qnq;4MlN_e*~e3}xv6tQ5uy<_6uJ4Y+TH z5^E3mPi$>6^`$3`2~salQFMxh{ZUoW-oZMPnr6eMvc6XUd@3N*kk zDdB6Hk?GkO9A`D(=O5*fj`$M&s`_LfRoiN>b<~4mm5v0D4A&kUc%=0+)NU9hNR&VS zT58?3dC*(cG+PkDq-o#j6H=R1NaKoG{Ki(p_X6oLW!B6PH0tv>Bl9$?j4(>I*-n+? z!}Q@DrE2TKfje4OCy=`z+vuNs0h4}?;&QpJyw%tdJ+kVv^zF=y?>JTW?6ER$MZeFQ zeTH#|$5TNmbp|#!;8lqlwxW_+PJnzUudh1rUrJBgFinNS%_m+o0Mv2|GH4OVCY*WA z<$Ee;#_GT%!kZz=iGzuc-Z$k=7#N_kDII>4rX9a{ElnUzAOHx*74#7$4>1HNpd=audpC)^T#a za^)%Iq5iJlN;P({NKCHHG*0F@BP&{jy}Yi$=joVcmD$TVf^JvY*SxO}y{=jI3;|F8 z#y6-{(k-}6BxZr;r8Ky=*AF#Z&^|Zd$Zz2#11XdItdh?IZqMHhBs=|ciHwlzz2|Wf zu|U)H?k>&?D@$h{u4>g}hH-U60)s?)HrqO`*ov^P+h0(0hZYPK-fv>Rh?4!=Jqa9a z&B0FqtDHK(*b?I2sA+*tA3fBhdqpRgwFH+P1@7X(P*>pOtOCG}7!$}SkQ(3n31ZTK z4$y#l%uNhGI15`@gU5%d$(lkw@d6-WK?kBnU0Ez^3;0PYUn2KIOO;w_^E!tM_iy zB?jZ-djvhNG#M^RIPFY1s_oQHw&zrE>#HL|UnsJ5*Lo3YUZ|R4VO@O9FFZiR^U47B z&(0%5c837^c;Y7_G`r_g6EULGLEWt)n*kDm;c6YnRFsXVeJU2Dw96u}U+9JQ=;;rH z5rdN)piks_ePJWcV)_7c0#GOo4X6{UQ6WlmYJkJ%5$(PAJ*2T4Aukc+{x2x}N6np* zn~z-j@nQ)T6$qum`R(|CoQgLLC;b%|*qOwXUW~VIlifnjE>Iu#_5Sx#CTz2+Adjfl zpD&&hn?yxjE%hv24et8_AA3NvoY?@m5Ui$P1136BGBd%$0!}DW5w@{s#H2MdXnyDl zsrxVC2>Pw{$Eb&^GC<8 zJusb?ca)>EwT#W$B?nK8XX(EqvRusHHW?Y4AhV4vkW-nrJ_`n5(bqAX{DMen1w;5cv(p`b1SGBNS%Nzp_WON+jUkvxCJAMe#ohx)N#|jcgNji}{Z4oooJ)6E zX1n`SMyKI_gO25>qs_l^4SdqX*9-|g4`D>BjWJ5(PA%qAs`U<#!RN7p*9&a^a9zFB ziB%VIy7p%uNJ2oL&~7fELcqNf+lr1{mw1ijGx0+TYYPg!)-*7IZ(24$KrD| zWiE5cfed$mFgrOSUp!pp@zT;yWkUHflrtY*5a8%F)P78tKLQbR?9XNX)U;i?0T?a~& z3Qx#?oCnf3n?py?sxaObfRqBNV#S~voLSat(H(5yTe&N~<{$t*5sLoof$rQ@$anks z7-&H?$}9aaa=nNtZ&)-KFXgC?w%BF7BX85zK5g%~zlCD_Q`JhP>2LE|fdWx>Tc_Xo z{`HoEvaFX!rJx(WoB5bocG_v6NBK+{QTwl;#D%7)f|p;oqud_p3sT>grq0|Wcnl@w zytio0<>2M(Wbmg2=XNJTHPnndaQZ=$X!(4^DpOB4SX)I(4;9?C%M=^CdZj&PfK@%U zB&Lnphk2ha4~ga&zq{gRH#~AB<6>7f4_OQ11py+Z2?esZ-_!LiJ`582FSxe|7~bdL=tM zrI!QNX#_*CGI!%wL(^8#{cqb4E8_@|!$;HplZjd!ZuEWH3U=a&$>c)eKu@eL%-xo= zx5bQ5nK=Gf)8bNcXZ!Y3p8@F!QGA51LeigRdsHV|7QO9NR92LPtQXZ>NDz%`{OwA? z(d%Dxrh;~zQLx!b6CAIy^x+-BUr8RThr`8$}0!Es+qW~D)fj->7?Y!_vkXmR?AJ7MvpD?N)9l|+U) zm!L=cTn!oAK$Wu|QJAyxY@{86RHLrms1NuAJHfOnE1)CNL6!4DJ! zW6q$t2RfTNotO+YwJ({EkDzuaSGDIzCa+z3qaaCZ5uEZ!kEyn!T|0Ndr+{^EOP(J! z`tr3>*rT71-eiaY>V3uc_Nf+=X3UEd@`!gcfs~B|LKJ~;4{B`6&hNErE$Z=)XRyjZm{5Q)FuG;L zXwI*p#_YhUk}fgW|K@gfKZ9FI?9uj)b^{s;B<0=EJ7(2&quq5curj{amlQs{mYF@2 zNBtS+B-=@@W4MAvFYVKw+G<^-fT_cu6*8lDCiPq?p?Mg8qOFk+cX}S0rF1$SLun46 z96EX+#`(Em0F)TUAYoI5#T&tEKv|1QBY2ZSC!Fx~bnP;EU&?!<)sLYXXwbNMt4et` zLk*;h*Rod<`8;tQh=xnP9c;}V<*o{##H}$yr`3B$@3b=}9^@vSK>=n!jhw$p>E94p zdjmRae}z7C%jVZ_4tk;svU-AP!xwi5$~!r}Q6GWI>E2`u+n~#Plw<1eQP}eOa_5`hV{Rx z*1}JpQE>4_l)R|qWg#;}t%erZ)IC*`iWPG>%q60)8c_df-|7U1=hvOc{E<`TI@hoXQShS4`>o9Yv*e#WSclI!DN4N?#f%i+r{m z3(6xe)Df$S-G0U61!t?ZnCR2bZT*uIJ=)uJK~zo%C)=ni?=VTnA>ACv(a_;go49yA z_PC>WtRct+&kkX?FX}YgVi7VqLeyu}D==_Ky;2;P7P$I|!LAic$gU{xU$c7we@6qy zeF3F1LKH(t`Bx?s1>!6(F|NYza32afyaQUdT-1EE&|>MtDlx(@N;2*}QqSYrVBib? zyJ!Z%$UJ$ElK$L;*21l(_?H`rI!Sr!cb|XZgk#NaL%nQ8;#-mF&fV>+cD^PuJ z!m&O&ZC0WYFgbHDf-msIk))* zcIYr$))&veXw!hD#Yp#Xj# zm*we#{S0HMA{W4>*lS$DUwUD`itpsvZm-1~GcJ8R24@|wAmdFSJg1mo6D$9FJ} zC+H@5VUZs&iPJtpFOP2|a^{flRnw&h^p0v_%;bi}5Oat&d*0_vy4B*C9no0+hDH;7 zf5N4YaIl6<{|r6Z#zKEXR#Z5o3Q)4>3I{)`JV67>kxK78zBlk9o-9q!=YqXob?wkt0k}><7rAf| zI8!gf&X%2Qcqq#%xOw`4c?&oqX#Yi=mde_x4P<}j(rBaEyOQNP*8v|q{zI2aVsyx< zey_09+}q&a@TpHg-L)D2#@yqdc|2tRiS^u%d(je10*%F+b51X2sr&|IyyAC5eChSl z#I%^c^y1ZPiIKRGRMXPC@KI&7zO5mvd7d+kZqQCK3QNmtlDa$4+A0pXp|+P1;}I*X zaB+4`CZt*|dBLvol=(LksSAtKRPyBlbhqgugQd}HpPsL>zKNW$8DVfH_>T{63ZGgExqpT-c@F4@i# zm2)bZ_?3)?1G)su3GaWkZ|DMWFqUd}1;28nJ`^K-QE=u7DeuXStXzLvjg_J`cy_-lzMwEyzM{En)veYj~-ndvH{!r0Q$B41h zt%#&nJ5;n)|MF~lF&+0|K)e6$UOE6|u!Z&(*Ba2_mUfG*|&UI(GL>Z5efaHW`+LDk?? zm34sKBhEZ31I3@6%kW%H$#i*BtW5XnczLA)y8k~c-lG&u5B+>wF=?*@O^N?9J1ky~ zj^;tXnc$UdDx);!@)eP=aV_1=ehWxg@q}tAUP#jgH5k0zY1edDJ4{mK9Y)?yUz7qoq8B z%^^B#d?6AGbw<%?NAH*K-Sw6JWoD6A4XkpN#p5itw@$dmKT5ZH)#oaoo#_6Ozsa+8 z1q5^r)*@bgf`fbAPZbMme4riKA_E1_oICW1rX@5~6FvV?Fw1d?a`k4%k zBAae2ejCeQg%jw>W@5@pSB*G(l{Gv5jp%04_au_t+x0!amSttcG~c$gMP4w}WqN6% z=B5ki3lbI!SMM+q5*0yu|KjMj@;Ul)_^2QM4!_m=CF~ldECb7gj+>gCG9vk`DMC`W zXo^e@KlD%^Pqef>y5QWdpn}2{1m2r!jI(aj#z`-`>(aF&?!3^*SSjM-7?j6HZ;t>_WJl z%XvF9l5z-?LisPMG5V%ZzfSECqs3E@LJO8M<*J#EKGUf%!kx6oM&}jXDf35;qyGFT6Q2c67@4}4#ja1US~|@;RyLi!J*n^ zuTAdo+cKRSnTf^lY~?H{{nnKaoFNWhUxov{l5Tk7{Nq*0*OZ-@g}7Jx2#up-ia3&; z<$~+$5Tax8G<0=9o!+c!tzqA}S)9J4^TpnOvob?kv<>QZqYk)&8emzQh(9Q#G_}!5 z>jAR4ma`)hDp3#?@H{*BW;55`&!AjR2)TF`w(yI$f$-5zLF_X7-37Q;WAZ1usJ9Ob zhykZIabCdzH%BfC!`p+`3|3M5gE&S2yg8lDuJz)>XKeAZlW)ZR8K}NP04`qW_%|G6 z=8}`d3RkHG|GZC5h8z@$>%v0WtiymaSg!}V2~C|H_#&%=RJb=oJ5sM|#nJ%iKoG#U zT{y=**V8vp2LGAAaru}jOwy{!`Jctdh5}+Y`EQ9(fJ^{n1a#VAaL%7gx^uhzWys$SRZG4>K)+EwH`r>`!7u;qT&5EKP^W3oB z-GJAUCii2`XSxwJuRhU~Q4P1X!Z!l#$3B?v?L)U~`PU2_`L@xudIIZ6iAEIrs&|SI zCBXJFwkUY3{u5najD7av!otW5@tmuX)v&fP@>~0MVexnzF}d81Rr9tt%?_$GKV{&1 z--&eGokI$pZ-?fOoIYAEiInnIGrPx~g~^W7xsx)J+pIQpid=*}miQS0=h&k@~cnA6GA^9$Ug+araAU%Rsmz5u6OwYAzN_m z6O+FCXm}AeFaqW)8(dKTF0hBf*30`8t0uKFDrt!HU zy~;5f+tsev!pU2^@BRd`dM%V_Zgc6C&)X3Zgw56bV)Z?JJ^zR^hucW?RsJ^*Ma9DG z4krn$*@!8Ech@@ z;Tc{<&#+k}o~pO&A2kuCbQSDF1)5?=e*Jo%bt!!shFBllbSLHnHS&=({3AveFbWxw zxQ&DYnf93T+NwYjn?HfA5M=pyr5NPlz6M=kL+#w zi_SyNI=}+VcTnx@~}{6E3@}&{mB~&bCIs{LYR{y!N`Z{xivBJ2eyuGjDP{{)mZq@35oRPondE zv12Bmly6b%Woz7r7Q(Ie<1u!0{sX!<1lr{E@c(7a^N+;LCqtt!IC%FO97k)1#5a9E zy&~c{xPXWW#(kA0GJ3Sl`!g^7CD)UrA8ZMVG-ZVO0zq(9Q=(&Tdu!K-X-{c8wP(Jt z<9i8}*Jn-3-KR-5$=CHS=Bv2fwOdqz|CnicK!nqK;J+Mq2kr|%8~ljQgRkj7tX>wX`Xs&?ppIS#Dvue7FoMuPZS+M}fIY;%#9T1aZi@cFqN za;FL)Z@{k0VOU12v0p(c#zEzYoyWhp7@Z;g;F^Ei|B-8-6xTT&to-_WXrETsu(kbP z(E_Ep=?s_Ww69eubKcNW;>Ghay@NHFVJ0t9EfT2CZaNL=V2q2nk{#gGIroB`Wo(^6 z{oarC8P%G1X%O2e7yxiKJqBYPKpnxe+p1WIIRojF7FPaD+KfgeRE>v-Y2OpB#A&4^ z7#dUNC|aI3u%?556Y$)3ve&EJ)Xo{<|e74wp=C)i|Q9s<&oTwdPBRqmgQs zDRAZ8_=i+tuLE+AMp`E}73AU`Q~~LzQ$`qyej*w5x9013Y9Q~^8=c>@TJRK%7Rdso zTxJh^huVbWMZnm2twtS1M@!$#$R7NLyH#$(&pO6u@J3D1EN1AQBdFF}=HJMN3llF0 zzBU~OOE=96v+T-)3SBvW@m$tn7v$Cp(p6Y@McD*i|KE@o6)1h8hKeI@+?Oiwt(U^d z=Q$x);I<8XNGkFMCj`+yw#LIkl~leuTKV+j_{8KwirylE68f zQAEL{$=-yTM92;zVu4J!D7V8@-(?Cfa^%Ct5i|$$VPDr=i;VW}`xD+2U3P} zjdF{i=F7qK_yqQ*R+O}W9dq+G#K!B`oc|dm5{O*8!t=aF=WN0!Cvpurdj+b<1rFSh%4(FL@z$b~9hh zjr@TmM+1;Cd0m&fHJx4#AaLj{521wVA4Iq*?pL3wFCP-=58SRd6TFt~v!Pfn8#DDA z`cb68{Zz6{25bg8%M3>J*P6aKkNmOs8;)m{77ne$1Iv3&ztErAGv5HgC{QH7!!CWh zFCaEL=QgC<;SGBl=5a3Dq4M=yn{(}T@7MRl-1p{}vIR_t*RV~6R$uwqW93%vvdtbJ zK~pvw9nn*5pZZ0uhVpDx90s2au(vV%-rj#Nj?Mbb)-@be4f-8>9*|K)zBjCN-=PN) zItwKCuj7usAs=AdI)-45d}>Qhbdr=yB))A|vM%0`bZpkRcz$Sqn7Wxa`hdP|v=PKw z@J>xODv3zR@PFOCmMhp|SI_^0-n}X;`E57(W-8sO(YV6WN~tFgLrk#dvW!?*(+PxE zkUdONHPGHnVEcz;`tyor0^8)@Q$iua-(}6ZbrQWAa;4ThV zXRn%Px3a41WP?R5RxiTi-OZIR$f9qq?AgQ*ot+}OptiJMTDa@jzPZ3sN00F_-hbX? zWA@H7ynJE6A~4X2DAxTQen6EZogb%~c)-_S9_n}n^MVDQt!SCox^`94cKX&!{zeII;D;Bar zfIs}%a5H5DbV;}0Ha7a;c382DI9qaRV}tP^Cc=>W6h-|G=9V+zP7`a9r-mgIy%$2SRV*ybF9%R>&%_#7anz9rVc`6SlhL3K%H|NfyoD; z+iK#vxa~Qu!SP`Tv3DMOKmpo)sG*>(ebAj-%EU>bQb3NPpD)*xrq#GOU4R#ORVbh@->Gk2!#{g-PE&RP;IQ0s{KbLb99@`ur}TZ&aJ%Ti5u142 z<@k3cR*-G#yi6ss6H`8gjctLW38qfXIV_H534 zr=;D%+ygH$O&k9fr@>N{ zxc5gX;%MNmLx6KedQV}iH%vEKUHM5_n0P!Iv)8@))E>q|^M{>ZlfQZjQ%ICvoO_G4 zInF46=Zs>MO~(1g0WlIDxlwQqK=b?XU`#%!puz|FvVOvT)w&acsBclg_!R6}6WH8Q z)5798LAq8Hhrw!FJ_aW%BQW#uuQ3js;@}d=`iLxa$?T3N`snsWM%L9g@CWACoVCwO ziF{Rl{3=>lK8!(^y9hW95{YOUK-VA(lb?&*UG)7t2!IGg#+Z<07CtfBeEnD zus066xMOMzRLj4{GiU~KZHPth@Y?(fys}+W+gWeHpp5s;LvS&e|&_- zBr2VbYj(q=z&fx6K0hX9-3GffV6i~5FZsyPk!*+MF~Ng%5ZO~|A^JvDcz^%d{|wT) zfjPe?Qwbp*95Uv&%RAGww|`^%rDz*PLV2-fA3yfro*J2$C~)RICbs1Z*ykrxESp-I zQ4UWpn}Q{z%kwp$BKn`OMYE?DsT&M|O9cp6+Hol${LAl$!5`lMMg`|}R>WFBnKa}L z%ldxq`u4cfP;3;jQujbmpph3DK(Fr(kd~)h_FBi}>nm`B#?B7~2hh+1C7Y~IJ)jh^ zHX28J7URJsU=4}V>Pygr!eK-F6#0wsm;yoioMI9HVr+6% z9v@4IHxhQFO2G5VK9~YP=q`W4KuvnQu{+=%jL$sX?nm;h?SS0jW0x^HL@*%E)}(5! z!$)@AP>0Y#R>me<-}IWmm(h_H3{|KxL~63#SjOiO>l7}VA*FB+M_Z~^!rMe*8G#sG zpSD=@beLY!r~X=~QT~sJxh(apQ3Nwn59|w&zo0URy)J zUxsufT8kdw2jVYdoV>5x_w6XZ5PjEeL^;V6b0ecMW&nAE?bo;%m@o6Aw95n(_nJqcRKW5RYm=Oyz2VoHPqb8NZZL*EuKPjxk zEK;F_-whRv30j441#u-R2g@nmu}mWIn;HvpfB;TfrGCni0=; zxtn?Z=s4{zLNSrqQ){ro%(1mcs2#D=!|rrAVFpYOlF|G}&=)Kc`JTs6&o5CH+Q3rG zG}0JOe=%YFWaVpe40+Tk_4EIambs$a{d2je!pJP*m_sP0H}}9G0L;?WoG+G~{<3Fw z*7(8mCi7iE2>Hu1b^6MiBiIW(u0Y%qOQM8r;Hoys&Y$Thx2AH92~AJhU0gOr(}wYc z!lQ_2d|{2-!S(G&P(QtHJ$5eSzM8=E7l`|b8dtJjn~!AR>vSe)#e^j|8*-~tMpy@# zOe(|big+K!&|bzdN9tg@?8MK--iIgVW}%F$pMM4&apGrVSCg9|Z#>iXg)x}*GWe&5 zf(KT;wok~pP9%((%Zle!;Oes=_yYR@hX+9T!Lpr-Xg~IE{&b4}LPVp4xO@1l=)g*8 z#(BqJvZ_GwjCM9CGjm=71c_|21s)$hR``h!=UWqQLtHgn*#Ma18*K3+0z|ysF~EG1 z{ZkJFYwa65?JMt^y1OnRtd~F?`IPyqJv>W`71WF>sjf=3Vg8u2jfZY0X|`NQKKC6~ zLhc?!RWYSxbZ^XVkiB(U)rHELXxZDC&rZr)f~ykZs?Qry9an61BK~l@_rHb_MFur@ zG={fv^T08-b9{2LSDI%o5elK22jco*Q5V9^3k5#f72)X}*ZKDJi@o&-*|inlwrg2? z?k52l)$+aEkA`d0dP=lxCp&xc^mAV`=t30F;n1 zDeyvCW)K#^!mzM{4-##E;(^p4s4VqGT=1%$0`Su8TDuB0&e_U1kxzi+$kdYN4i$F< zhPc7WGk$68W~}P*(Z_wOl?e2tx0MHKzxSMlJ;hMq&KhQegGT*G`q%?ZViU!aoCcg@ z%72w8r6{g&`AJX~ViI#d!5}=QX+y*K4V`_MV9i!8`uU|86@RYWpzj&+;Flk(WYPD+ zp2Zo_B1a?=ev~q)V)2CK14)0E#@?2WqbznrTkST!uY*47q72K>wgb1~qI z^?VHewu2Kf0(hP`c*TglyKq2)MP@wi{N?-eOYin&j(g!yBu;_L_i>g9?tfozK#G{! zm21FfsuQcTkRGYou$^S`zsr`u>g9Z{zJ)g29El z-10N{3cs{Kr>n4L2P=ZIoQbiu-@R*FK?o*8I{2tFuY>7HKY~lQA3lHQ07_w4_VbnF zk`wUOu7J(E&~*8fDcM}bnimnvJN$v@OM}(wPAKu!9S;^KhzUB%8}@qR-`Hx$>eU{n zVvox`>wT6?)Dk>!Bx}4T)q1v~Tn>9;H%}Jd=$Y`XI|*FQOGjHt?_6tSVIJ4cmE_QX z&cycXP|R*e$D^W*1bI+zv41&Btm^zgVxS7ZIPo@8XXSuwd9QlXRw8<^R;5uMYc@N~ z>`43aXX)xdFJfglNNo9mTi-V-F#lKz`pBA}m}6!|srS^~xz(0~n|&=6EP#WPSMd6& zzVw5uH(xq|V@Adt@+YXyAu}Uj*B=gS6C`5)q_#R1;kkY?_t~RRtMF+y3Ii!8mV0k$ z%R66`xb&HUeO%D{s6bze9PFDbIO4_-Cq<4&Gf4`8?dKhi0MVPw51hY|kU{Q4Y<5o& zP;7hPKPSoMzy^+(0dEt22P%l0VQDGM4?mwl_^7)f67e?)ph4DbU;dB#1SW*2cQCMZ zYzKM}{=-@&=T)Kin8uy1WLps4duBZEC$(!n-os{{D-tau2EJVTJD2FS5^6+N4FO#2 z+W4$PW9LCLj>kgdKWc0A>S6pPfLtGd(+j+?3BR)Q|E%zUeC41Dd`Nzy$piM^!~^)| z=gDUTw2iZ~KE>50ZTuk^f2#f>J+N>Y83y@17XtK1G;d>%xbWdl`_M;g(uM{`-Vhu9 zTW)K}!z~AUm#L*waI$XWzDbqq#&}%#U18|B{z*8!xorM->9)R1>U7YI>~=4a6tsH( zX7yi2#D$&|IsvRBEqr{nO#8bf72p+`I_uH*qg`ifNM>chGq9E^*M1WC&h)t%JB~X1Zdjod^IJyQf#x62fNw#S`xqz)cuGpzp{2lbE1L{{BeBbbQVj*b*DwN5q z5nR5JiyPpTVO9*wJDX0gYy58?Nm>@pD2b+?Tm#Q|v@F6P=?`-wb{})YVa%`5-uHP+`$a()M}07Afes zPjm5I)8;VJM~D($OH3yhnIzKyS;JaM+ zr=*T6zmsowCAnVaAc`AH_YB^qEYqth497?&Gc1O7-s4x_2=1(aqJ|TmuoBMkU&En? zQ2~tOZ%8z76>ufb*!Zpp{%-QRdQW}TWhlL6IjH}U{GWEu{9OsVv!(ft$b^N5#%KEp z-4edrNs+7BOcOed6qr>%2-9>a;5SyuP=^X29O0da7U_|PXENr^Nz5SLpP2^ynigw%`+cjHp~*RL0rfxXFW zI@SIJJJ+47l;XT@grp64xyG3CoV|%qUjkeQXe;J0hcnA|e=n6XWSE)gD?ieSukKlc$CBHGfuf&-+(xLH zxRBH>*#vxpWk0h?@F|=F|8ai8s4kkN^129UG1RT6r{gMSmJu{dO0nenheb=wwfv?R zzN=51uUdvT9LGVNP2O#91`$`k8h97*dxG`BPJFbv%P4j(hMgyYw7pO#hVtpjOMjA> zqF0$DXijj^uaIJt`bq~iC_fmwj)Uxtf^`rJ;-Y5t3{(i5IYFCg|M-L5G|*8O2K}be zR*Z`jlp^`h`fXc|6m3C~hgR9u$1sBXkNd8ykAE;po=l}PycHYxAVQA$8LXeb726w@zZ;l?Q;yqflhXUak?fIht`_ZO57D)#)4vOimM6C zITS*)TZYA>bhR3v=dDagI0ufxN1$y)TAT2%$L2q`51=vRCT(F#b=UYc#Nb5*`qLEI zmP7m*3Ycz?Rwv@RWR?-W z7=8YxjBuaOgs{(!7D3#>I~CXr&Jv}5R_5?pHte6T1`UML^(GKW2llrCcl!{$gJl-m zn^SV(ri{FYsf1;K(me2LiIE6F?7C;>prhP97T+a*W|;H zj@Af-j;+3{YH@0pSTN7oPqU~Z|;x02u>AU!z-kZk|ED#Am4Q*F*4*KRm z)LFMmp8B)p%e7z}h;S27GIU|&T}5aA4khu&?99)9AyF#kY zyfcH~Mo_HTg*StYJr*ox0h{98u&>NZse9AjT{TjpQ7jXt*Rta-muZxCe2!cGX)yd^ z_K~4lA<75Xr6f?MXi8EZy#??~jm5*iz;odnQ7`pN{fQTvp@A@P)&?8Mm5~k#HKCW1 z$Wd{G1f2m0cS%7z51}8ZxFRsR^?T0JP}@jBAuLZAhQitu+`Upy>|h_OTQJr zJt!LwL@`EnS92>z0W{qVv41>X2vkGz=Z+4coC$^`tMFV!$7RF_$H7|g>)91b&$znv z2ku*A(Lbve0)sy`f0fiG_#;?8u;IBLaeY4QYsd130h?Zd{W4w{ld>+#gbe6{&#-;g zL7Eo68n{)>7#>l0{uxoD6*U%gU#Xso^tQ^VAWbq zInh+*+w>X~HHc289Taigve)^|vwPykd7u4G`ye|-`y~M&+SGXXmm|oK ze4b;)L(N4eH-i=+s&DM9awVOErz0MmMtG`M4(}eF9L@3v!+~^Lr^&mBGaQr1@2B9d zqM||sZ(+5`^UrS#y8ZlT2g%iR`%cPCo&F42s@Q(JZRp68;<+MQgBBh8x68ZcoI7(3 zGs*wu)N}u-6NnyZYmf*?%h^oeQB~3ILy)3O(I#^8U7my-TLcHjGB1bCf+0)J|A(ft zj%)gR-#;nH2&sXRP6R|oNuzX0x4@7Rq!}fQlo~ZqkWxZIN)!P>Hj!>o8YRTh-6ajb z!~665{^K9YL)~7lbIyJ4`?{XjDHCJ5VGN|RuEZ7gYaYI3S4LPZ^fJ{1(IjFU_W{fM z5-TZAQ2y>uAwcifwP$J|^wF3TXX zeem60Lp1S~|8O6w=&CDw`Z3FDR5@-IzV}Iqy5)~sQe~CFV?5uOzg}4VhaLp)_83{Y zlIK76^E)mQ-S-S|O-yvMt7j@b-ePV8A_upeNJbp-iV`6;R2B^ZJ3I3l9;!xR7r7Iz z)782oI))sSWsh+tW8pUBA=fFRnxV?j9o7BVTh`cZcoAm@=?&Rm<9G0i%$O0&2;;~@ z#q?+&gA;aG$!PWR>gTN|kTarriP)4RGO=c;`#O7` zO@Udb_a34xR)5Y)I0V+(jHuR%%fcp3HE{~mscA(qxPj2byT1^x7keC~Aggc+IwrGt z8~1ur5?u`O3m7KnkSb9{&Rv#}q( zYxJ;pAYMnpOnwNY04V{VM{~s_yUhm_BiZVNL0$^-cTmpfQ9OVCn(-i&PsM8c+rb5gS; z>OsJ+2vX7&c0}DG^NU(`B(-BcJ{}v|mo@mdhZ+!S1!DN8gp22(!=H&QK-K`>Bo~}k z{a593vDe_mjFmuN6*Sj{_{dIe5y4tUep+DMj>=ejRBux=&!O<1mRpF8+Hg4W=iebSIXx6iu=AWMY%)(--`K zfSZC`Qt09t&kz+-p;K~%Gu-bEjRkLcU=O07D}NtqZt;)L>Rk!^#|+$Jyd^LK_NiY` zPxfg|_5BiX&TX18Pv^-E{*&GNL&jh+oCq>mV0`uMMBDU7#8yaC_q)o?*F?~OPmRKq zuEHrlVngwPEBD{fN9tq<@~2_DcIf?QT<|+ojQ*FI8P4oxq6~Rh`9FrNHCu*Pw6(be zM5KqoJ)3VJG8M#yRG>FG>r8yeI99+-540UJl|kUVUV(@lt>^bgApTUFWFVx?UT0z( zWC*%2=#PdElYUt<&>~RSf~I{5-O`ExswAEgEIBcgxu?Q$i)fGua$1?v2dMd7^H4ye zpfL^J($`mX&c>hiHYhq(lF~*^>MLg{FWFjs%pu>fs>z9Yq@ox#b0huUoPiONE&;a3 z(5%Gu!sIRX*HDVG-2W#d;&x=)4}{vVF=<#++|-Rs&a?;kTii;|%0XC!C5$)q4Vay% ztVdv$P?#dkHt6IB6UPFOT%s^~rSdGZp4Y!?9R@H#u(&4;CH)1Dm8%Y``5YOZB}+ox z^4;ie&%Au0`E%4Pv1S$$QYFajHM`Qd! zi#e(vhCtHxBGfH(I>|r2BVCw-x1GwY)wSC)rtd-!w=!zfWpb?CSVwSKc;h%*^}x7c zR%F;AbYKZy;$Z%M7;3Og=?OOVOTGSQMWP{u{N^H@^08wqWIfWnOBc+tPF4U1BIvZGX!gJhp0)(u zOoVb5P+Sideoz$$A#6q4blNnIUacnY-en>b{DsJ$Ei@@mah{Ia($r+vGK9m4@@Xn# zUq@8@5tK|!#sbZ~`W$s8zGabzs5ORK_ah4PFa?1!*D+PTZ+DZMHXWP0(4rw_tY@1u z6~1sS_J%sXWx#7;7M>;roH?^_B6nEf5omM}i2v+>`n#xF_+#7#+D|+7-P42Ym1vxm zk$G9=<_jVM4x)9WZ_CkgsQ&!Q?U)wmZIF-<04a3cVQo;F^-ALaTJ6;Lrbyuhvpl%C zkt5y18Bk?cgIr?I?7xjkJmlF^4QFevzsBz3j5=u5MaA64emWTt2pD9O6*eE(V6xk8 zKW2YF{p>0szQykv-T)mHHw|y&Hn6qmt&BOG9aUy7L`7+orx+shh%F9@8BY8mxR~7Q zAK8&M@vytG0UK(2Ef_vhgs`oPXXf*+7$1H>`TTueLnlSgA53D4s}k z3Xj2zg8IR2V0854b8wfC-eLfR1lzREelVnwg9cL&a71>YFu{&VB5qM;KY-C!dlz#j>MuH*lCWI59Zs3m#6m=448f{09x;leuTwj)P&LF-=m->>r4t!W| zAOHi>1cYcx1smEMrwd(6Y;QXlf-Yu>zbkNf@=8CtekbB*&0iVcjiuI{r|$o!1;9Pr zxLMVsh~dtocU}94Br#ueegr?#fuV<={fffAyGhLBsVA}1=~u&UaPPjWrv-HEI(}hM zh8&r|*ET@wG$pE~*ApgdUFveJ`SI*>=1#3BiQhljwTYC>NP8GFZN#$@V}_~(8}Gn`k*uF`Xn zqH>#_rzb>C*Xe3Y6j|?rnsyql-0k!#Hf4pUW$y}q%nvh@AjtYvhZcnL^~sirW89f) z0z9LCl9Gz*thDG<`Ox&M2sd8=TS9d#Yt_S+L1BSx4&1l*QD-#FM3ZmH){c5@?;|yMTrB)a%R5hNAU`})_SaB% zM`}o-4fpwxm6@7ZKa8d9)hrYlClyc#dRNbG78+{CJODTcshkb23e%RrhX6xC809rp zdS4;uBb;Zv2DRIWna;(9fXU=knZ!a5=o-vAxblHbk|!SpnWL|qnz8Sg7Y&o5ssWnM zL(`Miu}8e=SRsEZAf2PUZ{Z`G^mzsWavMB7kQL*@%c!|gBKHy_i~A*D{U2HOWJAWm z4+^mHZa>kwG5mdgL7{LQrjFp9xqL;&K^9Xd)>@n+1a#Kpb#CuucITu3frr0qW&PA+ zfds5jcI~P8kZSTv>}cy7fe$xLXtx>16vkoILlUEkO-h+I_ddU{^ac5jGi;GqkE0{y z_*uMAiHddwN@bEg)UsZLeuePCE zmZXXPs&~wDxzN%gle7_{;&>lh3zE9ZC~YLIb`V4!xJ&XJF;xB>_sZY&N~HXE#u;T| zasGQJaKvwpzqKq>baC4kKxM9Ih@&o8t+Z2(hb`h>rCA=jq;)`Hzm_aEa~9*jzwly> z#F~nMG-x1)=^)BUnuBkJ8xsM9%661UTxg{phnxe1#z=u_3Lf_hq5hVb^sWXNBjoP| z!Me9j?E1;R7~zQy%Ciok2~v@=mh+|G^18qoz_HEmCf^}K+E1nU{6>+=6NGXbvJa)jK}tnXVjlwLBk{+>%uQxKhPhtjU`dtMJ5tQI3jFHv%AX$ZdWFyY~0WbA*6Ll+g*?@HLTz+fd7bxccn2s}Gz{ z6nR{YTM?4MnaIYn+`EGWRqF+C^WdtV;8aC$eOtMCPx$1KZ^|rZT_FvdveZlhS#7q>0G5M`ySz$zCT+h$j&sr)6TA zpu6#)tJ$wR*SrP%;=5qvY8HoYnTJnBmNGH{L>NdZtKFu54@FL|z!?-+O6P~)>s1Zi zh$&a!K2p|rfuwzehxJ7md@+mh6b5~M2MUYfP&|=v<1LVcH3;qMc4cG=@%0F}S>GNh z|Gp9sv8TCV;Q*B)$2dOYXRL=16+5%Ka4>K_r8u=BMfwbk!ql&e>Z>L>r?P?PprC`F zc1acYremfSNW-sZ+JnBJ>(yd$oMS_TP$u7L=6@B*jPP|Z+3T6%wT*A(&goLy%vxH) z{jRSh6)1U;8Plg5X`n-&o-7Mby7GVguFAr@q7Swa*Tk{{9_$ zq|t>`LNMfz?f?06P*E>KUnJ5>*ll3^w6ZQk37L1xg4rV}6I&r$0(v#w7~}x;?SVA7 zlzkiegYjqO-n(1g;I->ER(=UOfp)Z1$ZaP{H^R%!P!Tu-?S@hp=wrDW&_Iu^o2!-* zr{uNpQTcnkzAtH92h=PjjEh2(ZT^T4uQ<)Td|uXwwn#8krt;X3Q9o~?k+G9)psX3a zGl5QXfBgoJY>3!O;}bPvMG|JJi=iOgW`YcpuqK3_uWuuko(Y6d)%-n9enOvg2>orW z^D1@h5K8gb5nhbKpuE6TGvI_l;>V!>LdG`zSp3Q0i9aVl$i6<@iJg#=Xm;5u`c&sj zFs)-c_{W`%824H%HHk>~ui9^}J9K|xcNNMrcj+|N%$ zLesWu0s^?slwm(2j$4BtU%8U9G1Dgecq+Qxz$ySzn+I~*t785oL~o2R8oiRX+Rwz* zV<#qps&KC-%~BMS593&E(TV=juI-k-79p82xacPCR|%A^ZiAz?4nt`sA3qU?{rT$4 z3GRAm3Tlw15)^>pJBlSR(;GWaiOpuDOTPzHWKJ6jen`z#_%)`P=Y4__9lAI1EgWDt zRJpk(E}ZC|{Ci_QJ@za>cx{0XvIttLOYZ9#BGcO1!#WT0660h?e_jP7$QrnJUn^A8 zgg9ekMGejiREHr;<2U` z9=GKnCK)NPgGi3kIyc^}?%qA1rsB9U>l(xh$GSt*XbhLdTlq zE6{qBBFOR?_@}Cq9T%xTqXz`lj~6!=`iBfg!?QXRJ~i_AbL zV97LAs*ig`N%e;@mxFI^pz-*Ff1GL>Hbx7nP$oJ*m{H?chn0U|_b(8Dx`2nx^kam> zS?9G}z`HGvE>ldXWqZsq$|^CF|NgzCYm)YLb=}l<1gis&cT@eY&x0Nx+c;ct6lRbV zv6l8lz>`?dAu=_HY#MIGRnz6;|3Or2KyPIQo{~e}dS(Qj%qPdgmUwm=03n0eq1S@0 z+=5wE92j%bhT;byRe!fw;y?#DzCpCq^bzS*RnTClGz{2S;W5Q1;nruW}9 zbtql^bLR#^>qw_2-C4$O<)@!p6HmIbhk~o{GeqhB>t?7~9jVx=KDbIRf38FaHYQI` z*Ar=(MLAG-^@3v>X#SJDOynBSQxxEQ^7Cv7jRTu9{m*x*Z#_zbxS54{erby;?07!W z*Zsp!k>%=HMXX9U36zOXWYbQI1&k%@zPK=~l@)t(hP0`8jsgMXp=AK{wX|q9C*>8@ z@zLl0HyghMcHI%+;cC^VCWTb0e8}Ma^A~y^_XhNFv`uac@Zdk^Vy)-+&)*w_VoOZe zhuMJ<`mM85RPlt{a`U}nb<^CrX?U--m@N?ZfX?BB)BIC}%y~uPK!aRkKc=Y zUeOgrH&};$8`D$^7sDOAZh@UE2u)F0sd|+9D$n_G*mM2*K8Gua`1{D|A@xVXIWZ`I zq<8=Z$<(Fvv_WGC@mj6N6DmB~a(aa>Pb7?{`IN3;>!zV2hg1*vBkD#Ig|obnxj8i% z$NDMdw7eG?_CO!;UsCN$c0yBmWL}-#b8QpniKv`MA8LjteLuZZY^b|a0j?QwYiiZ#!`Zb z(#`;}9>M$GKX8w0%~rb;^vr#B8+Tj*$PTEadv|RxKx#gI+f~f_fExeNNFh?k>0jI; zd|^`4-h476-?FG^0_y7T#)mI9<*ti5;FXReuR&3f@7}N~^f6QuD<0`jEyeNz_wdZl zETT&$Q{pAs3eLUHZTUK zLRHu1a|e(it+buIUMaIq!eUo%C(+*1m=b#Idh>e^GtI4UbKt7A%vP5$T}y3N0s^et z5tOBkf@uGF8{^Izg+pXh8nFMQoB$snV#qE#z@ZPS;=| zmf&T>W8oj)Bg`|v7CqV04{cvmP*F>Bqrb}2>eC+U5`?jd$u9h~r8{<}=e3&eG9z1_IK0R{!fRz3n6{rj?b=*)h(TSoe5BihS|Ls;_8H@~|JR|y%eS3# z{BNQ*;fchPzjW1xNLn@|VI-IhDw)2XB#@Den?=Mk;lrol=iaT?Hr-}!&Fm{Q-yDP{ zBoxmSUA|Eyyb($;67{l;wHiA3bYJK46E|V}AVNwQaeomq3l9berX> z#ny($eQ-L>|G{p^f2aRj#Vki#6MidMcTT+T+~}UUd`Lk|pchP8ACM`H^2&PaxXTT1 zJlZnW*oIc6u`97Hh3a#~V|i7PdETklool${l6GbgZH$Piqw1GB1;mZ%KJTl)r$a(tYS}R;MzUY-Wtc>~71wg|GK$@W~Xv?~!gF zrd#^ASH|*QM!2X|k;9`pg5Mm*%msyv`_Y>JZ_Sb0S1jU2a|SLziht)>F z&Dn}zh(m&wU4ZAt8$a{Jb0Zs!n9BlwabNOYdfDXUM)uOi74N+6os)xF-3h!X@@Plq z??IPe)@nnvF1AZ5^<510k2@2cJzpTd^Y^P%>NT&|w|56VaOmB8d$>X$#w4CYZjifj zzr^q6i0M-lB)(vFKgLWm%)a*8eNEh$2xz(cV+MDDJsGq^8*b3C0?g=E zoKYYvZM6rY!wr=ig8?UikEsp~!pQj2lJ=WB-yah&h;a1TE-1iUm)K}5eQdSj7&t;| z))tSG3^_%4UbaE@0?M+T{`v5~VfQ}p^W;w+E2@ZE`S#71ZKBq@%G|WG!O@vX!idDY zw%Voh1$&)vq0hf-ay8e;ZMR~^$e!=AM}D)$+W;)sQEcv2);oBjt-Ib+w+5kh$XMZp z5FY}jQezavHd34zC~T$C)aR7B?cB}Ob#M8zZE3dW73QHL3CG9h-Td+~)qlpB!#Oo|UDe=mix7Z($4xEMyt?Vfx2Yg` zAaar00evsvCroR#E^c@jPKv**ns~t(#pkcr&gF%{g)fCGR3(0-Tp@jZ3rL)lP5~T$ zzMlDk5=i2Pn?JxS7SkVvF>lhrLV%U8eWW3X!f$p#ia%GAhWNR|F4SV}w^9#c&tvS5 zMk9L*LOlj!^lp{MAOlQKJztnV?i%YpRo+Q5(_Wvf_$VR9Dc;x`1J5uDE%cP5f3T>& z15Ikj>d}n0kV@=hO7|7SnBSS!liF$jzV!hmK}(08c887Y&M9HwLEN;fcXkxTZ5$_n zS*^{J{w5B!_wP@mwc(o>_)ZN|R80(%r@m-Se@fW$bA#nf4VS!6%LC$XJTYj3Vm7g4>XPKee@z|XWaXpSoQds?iqx9hzYMiv0#G= z8AxwMLGE}khUEt*a7Md?LcOKtwxKQoo^OCHeV)g5H`d4d1KZC>HS$*lWTPjDI2aCN z2}G|27JTFFmX}gk00oXAS5te+ot$c0s{?X)kv!6q;8wQdZp%JYGyN0p<$6IQ{4p|* zvXK!~R_hwov2*43_GSDeD^*UI)}o`pd1z3F9=#qmnySqujkmQg?|{ls7V#!wIpw-| zdLo<3nh&9S@@$RF{X~7^IKk&ujk@aX4hsX&N|*q+NJWWQrw@E_>r;3)G0YTVMRsy~ zHLKD-orvW<(16r!x%cC}*Sbsx6V!Y-;|Y)hdk>hLyx-Y%9g~8Yxf(vt76qveYe4`+ z0-JszLkv=%5ucyVw17s&lrvTxbhPs;nLB4%Bx}Fb zD;g^w4`GPCiKiO0ExU!)GuL69Bnta2v!Hb`N7JI_6NVE4;9CYDeI`$R;=0q1*M2zb zo$`1y0X4XRCd!qNwAYL5Z;$9`R9|2#aa z-qcLSlpnE>*0t-&w23d@2!w8R9CkUxH-)pCvDYN3^;}eJH?s^J0zu332g-KBDNB+) zvkd=M|Lv;&KD7g>YUcyItokzY_Ze>?}@}7~BFic(4w@fJq%1zwQuXXLPX!t78AoB4gvJ(2EP{jMU0Wz;G3vk|#lnoZo zw?uYh`LE&U-gY8*$8FD3?3LNBCwb}k?@rS&Kit3eXzLy39oAIZMTG}C$vOr1;mUa_ zrr>gDrrGx@<5$$Ayke5UPHr00B+M`P{ialF96TeWfsiK6R^Xbn)dMv-Gcp>ZPBKQ_ z6yl*&8b0=->wS&t97dl*Q`B5IhDUM&&fcrA9E~j<=Eg|})NCubIk`JMC6b4X-U9zy z>z51gPuETL`H+Mda>A%~sQLI@;OJ*?8*>PPl)~ShQRR|mbW>gEP4=my9k7~@t{h33 zPIsvGtr)jK#k>bqI!>tv5I(QHIiVbC^=Kj{#x3Z~tG93|3H*mP!@Qk+oMN_z%N)5%ble;niv zbX78`OYjYydtbX>pB1K$+RDoE@5qYTQ+(unFW|J@KlQhnJOVMdLm#GZ$5P{+4w6Ok zBj>9Xh*B$l%0qOGpY}BL*<}W64kJ=n|JUcH7OZhRCJTk!x3CrfA|LC$XU~`>TSk@n zYlN8Jh2j;jp~D$l5RNr1=gh3g5aMbT_j)wRSdJsd*DM>UDvr8|0F0Uy@uMyG^G6fx z8LxT|Y9Vl2hS_}fN)^T#ukLS*pp8}X8cR^hTX2cZ3mFb#%J$8)hR|AQ<>PHPZYa}# zw;-w}UxIV*+a6!vI?G3m#)mVI?vGaO#>4*|RXiu)84DV`jvwL6mYEP6u~*NcD6Ka*L4{6m2`dyP)4P-T^l^&__&0Nh8z+bFyqZdW69mO2#Q2LeN+ndm^l2DyvAY7j!YnLkE+xrS?Lo_42Rt1t5Cg*Z zWq1n;0tUJoy-qHb#?x@48e-u%<{8*)-2#?xE8R12_PCT?nKICRdvfHD$sE-wl8MLW zmIn{}>vi7VhR3;PAGkj=GEn(sIA;3_$yPt;K%=UT%tK*#XAwb`@A3KnW)b&~T$(6S zsH`5Du&jiv6S&cGJh}vGsrkM$T~_w}3x9#VCP&)^pDPcNO9D*Kpuf6+)ResNid$0b zx}X`)=q_R$4CN~)=HYDuu&r=}8zUarw7<&lLPw8#(zV%*65#hQuQD+w$akjUHuO?a z>*y7@7Hqh%D&T9E&N{EbXMDZ^o+!am9GUb;h7~k1PmW@OD4sY{O>;~s-W5<)L@KZu zD%Z7#lfKHJus{LQ!hL&!O#|?^jjCva{G>Yq}Bp+ zkaBfq%q<-G7Id=qk{AYxfaJjYpYCN~UvDvX7}NvZfBD8VY{ZA3!(7Fceb{yg8y-<$+V6#UbE89Rys(lC$_BO6C^gc9v z`hf>wqP}*$N*FGyaDo2lr6A*H^=c@ZJ&AuDCf=qJU#Sw4)8%fcAM;(amq)Qrs*^0$ z()(VUn}DAzZFbGTF8xsn@dK}g$T(y(E(IU=A8t36!uKjzoS|B#rqZN$Tnc0$2_o@HU|VAi~33INr2c}K}}J5kO{mO?XdP}Ge+h>eGA&O29nYVfUw(?!w3Tx z?dSri#4RX$P08{hv{8S`!?Cqd)lhE5;lpHT9YtCgszB*+V@R;4aO3Kt`{DxNp^_pR zR^}AoikJ0!&vXAKfj3TiX8p%kkpn+sSFP#psDi8jQu^Gulw|HC^pJW`L7)>uWfy4W zaWw$dt{21iW`x$wIdv*#0F22tAA)#yVoi^o1=G5+O5*S{sXUsqaQxXTwh<=FR5*8z z`iW)QWCp6iD#(S$l2M*Btw=uYX0;r{BxPR`BAZ zY7zy{HKu3LaO$*m9@}iq!?hUv zOjsy}0#FUu7${i&PYb{u{yOVa%|Gt83=kAz37W3Wwh|KlXVBVs{EFmn@^Vz)5`A`9 z5KJcsV+edK+t5%bD$n( zfq556``l9hQ8CJ~L1+H)jn-V!m}7NT=7#Ohws|gnkMDEz0|5|VTx)yPZ#$C;_*f8G ztF4{&C$eBB(+g}nK(pwtr&514Kb(VGD!1lda>M2vz5jaGaG~RNTB*n?9n>+K5fCi^ zqr32aNmJDA75L}XhxdzIA4u8?>R%Y%B~o~N-=jih?npU50olysIxTRPW)4hraopr$ zHM~GO|5gHFQBRs91-*;iB3gH;Cym8~2ViKP=RFAOxXssVn-M*`igj(COk1s$7X6ey z07rZt^ng}<8Nh0Rm|+!L3`jz6NAP}7Z>wu}O~*OX&cOasja^TGC7XXr67KTpE7Z2z zaed*#?0R#9qeC~cxmJc4s13}gidp7J=zfW`)+#a`$Ozs+R16}lP4VyJbJP>Duf3>0 z_?h8tTS#6MOA`wC92Z=zU;d;1!;$V@?|9T0Aj6w#{6>I!P{mun4iAgJgiq$PX*Z?q zZ|iw{OJLYYmXZG0+USr-GA$r}>YS~8ECJkAuY^?q6SQUMI^GsL<#(&52`K_d=+U70EInHAr%Pd_FEj+GHzlItv!Om51x@80D9`}+JWhzYW5vaQ=%ePr^-PywaOZ7vn4_ya__x7*+ zdem$)m&1>X5ukD2+lFtU2)~v8EtF2)^kp&C2eQ5ai0(e7rx&jgd#?PP3^p? z;MOw!&-1Ur>k}5Vk3&CIgk#WhLc({t|KbsB4;BYmA5PMG7<3X)7_OyrHXNsh*4 z@mI%LACx(`KU>={j;4!8Ochu4)}Vh>+hC0n<6qj~N= zd~Z2p=w>LtZn&wz9PO zalB{B$^k&vS0WP_EnD11RWxyyOs#X@f6$eaVvGJ%(_UJ^8Jxy=79trt_aek9TK1Q~yt>edSwH$1TgWL#V_fAmB`)SxHGpRF=EH#G)kUBU>K4{)ip-{ka3ZyWve^ zMSEGa9m7|ir0TRaC>>R2r2r4~YVlLe9eOhhlOO=pn`bq{6TQq;dra`yaf3*I>D0WJ z>CQrmypNHf)Zo%cUV>$HQceO3Q=a4YtI40_;y`L7=Gj~#tUs@?zY7g&Gu60E8U>ha zz3O)UyT~0a7;4VQ(-9&9PLhgn$VoqAA@-Wx%_iFkTsi$<%eqwD^%u86`9D8w95I^j ziG9x#6c-h8@@SfW%a{;C1{5^TY@6D(C&kjNa(}e5V z>m(#QUVZlol1}*9E}H#>jpWLN?&99K*jJ-J^?ilEfbLE(Di~1Jx`nA}1+?B0-*5mu zr1r>YEF+>hAjgaH zTqW8z9L|xJ^z4b+0}wo;E^sLZf}GSjx!N?H(%tvsq$tx?{&|yx8Hylj&jLsD1-G9% z3X>=JDi>l~wnZ-I>f?)gJ zO&LS1Q0;tkJcIn3=3M+nM@#P(RCuVFS^NDL#IxtCCy3;puNns_AUe6M*oXZyx&3@f z-DQJU4X^a>v>>jT^IJ%+pT9TxBG2bNHC|0hVXjpiadpP4;kL?2Y>>#~nWwbYj`$fH z38+#(a-9eEG|7IXD_?ziJSg502Ate9FD}~nhlW$LblduVW)R4BbcWQWsJ-lX=`Z^9 z^iL^BW?i@%1QoPA-FPiHKd0WrHk8PMzM?0QR1g73O)f4#{Wu; z_#l>gJecJxmtYSMT^R4WkRLcqq(5ifsMTjW0cx9CTmo8G5%A2G;mko!tquf?6c&_% z$Y#pbo}SfD`+IL!mY3YG1b^$8hI8v?oa*~^mc7`}b`jUcS3B*jXqzL2k?j5L^$0dL z9(P@+BOwYfgeWZdv&4(Py(`8R&$iLTu$2&dH^dXA07TMI@$(Nsn-4CT#UdE{is6`d zF~32oT+_q3(R###KO3y?gHKN*vIu+h9Xz%&`I`+H-vD|;bA8{KlEBp^;x(rvwithEw(5OaX6GcnBGc!j- zC9ogOTD3}mA?WIoEtP(I(f1#Zs`zs{U-D(j{@v?fAx5&I+@0 z^y6+UKZ9rGVB~EmUXg77`vTAfEE6brb^>9{MGE<>7!iaMa#Z9M{QFTw(q3pWX%P@t zb}!r==D*EUbO>162$DHy;WGT3!|7_{Kfb5Jqv4v1Vi~B+4we$9t#%C%P2gf%JRJL>bKc&b~TRkxxhdK+yL{!3y*;!KGo9$*$=fN&z zrQ_fJ1s~Y^ZT=|b4z&8?2$>D(O~?#9<%!N!0Rq#hzS11n=fpZv52(QaWi#4~xnvokj-UBV7KSq= zK|@%Oc@qe73mZO*P~^^z;3mv9Y*T6q?(y%DE#$pvQ7&3+_YXUMX8B+VeMt7fN_7BR z(c3#>u2_MldfYVJwa}1Lze^R)B>9Eh$wB7Kt7B=B$W?E8;MkEZl(ZnD1-g}@uaWQ^ z^9>(J%M6icOtF5k?dIz=u$LL;k10|~{}@kvE0HMv>a8v4a~c|k00@I2%-40mHyBc! z9aA{?0!NgDj=)~S!`Y$zpsG0tQEP(O!AYv+nV7h}6-J4a*sj$OScEMpf&#Op2a)(3 z@*YW81tVbYx>sDGpppRb`Ee)MVGM=&#-pjaF3W+3^*555#S-}0R4KS=P1?TfzdvM& z*7wKQ?a%fOa|?syN7BQmEo3z3)5{T_>xDFUsSy)={;hNP)>Xs0_WXdh8`bp7Mb9xG zJ_VfCGkKc4zCqMrb0iHe0QX<~L{!7-N#?U#I4D=o8Fv=pD||!9uEIXegd;4jNhIr? zmHC#T3(uRLqfYt(?S%qvo)YtG$gpw+%FizhVbq{MCg2J@nUE>Cojk1&!$2;27&}Vk zsi*LZawy`kT#r5;V2=;LE3fk0ExytiJxP$ek=goOF2BI!L5hOpNg5ijp^RiMEP zEf_-PX_HBxG=ZkKAZG$;Bwm*23|#FhzdcXBd3t&VNk>{*b(wql`Pp4bvHUAy^31va z_L^lHL-?-XiEO^*6K4nIMy~e_`P}jf_np#NV#^UMmL1yo%TVEfjfFce+~a2dQgDLB ztyz%{8BbEvvETNNw`H+ zq`Gw5aV*xfYsNBzKIs_xyt@88vH`unLO&X=wMlELM*##oTr#v|NYAmpABA~5q7Gr< zIKBRsK`bta?$j&En3Vv6Z6IASpHjQcQnS^MCD;rFDR^8mNyRF;s@i|UD^(;oI*j`2M)3{D?oVCKd%-bd< z&@D(NCGWZe?DG$2V8sbDt#`aP^N!UNyQD@R5lUEXT<_x1+IGKD&vY+h4vit zr*j=ko_g96a^YAMWls=S1liYNDg<_MDabRJUy_g^&^RqN=tX(Ir{3lzC`i9}(9^%In#! zEM8KS9g~UmDDi4>6CxHki*x!VF_cgqRj~M$2xZKP%P&rG-L2h zLQVfZynwcJc;4q2%q@V9^#)UcHM9lM|1m}$`qIjGK;>JjKjy?YA{wFyeR~ck^2b1D zb3+a$@BB;M*kY+wvU5#pl`OH*)qM2wVqtSKi0PN0-7Df8*7LP@H`f_>rm)gv-YscI z+8?}L-dF2wG+lHZ99v;YFQx2uT+fBX8q8tudwLt%LcSyPpogC?V%J14#?@vGXSknr zsjvKBjEa^ENH`&~hyyEF_nYBbICz8Soqut`Qv?6BI~dp3|HZ?{@AkE&L1VGJ4be2L zqj@2Wa>!}QkIKHVCiAI}SK(w0!?YI4;iDC9ID6 zBMC+u@>A!@o!kcrx4+yOX-B;eZY8|W{Vyij@$p*Xx9#{cpSNFzsoiEQ{YSlncbUWi zh}SoZSKR)A@)!=|i^!~*^>|&g&FUHV^*!~N@2JEU!%v5W!e=ht-jmVgbw~d;FPf-7 z!}h*k(fwYSP2Lsb4_V{=Cs~41dmZQ6Kre%w-n-m;glund75;R3boulg`D~A|6<@J> zzA)YVNqKc?q{19&(DFH4RE9Q-F3B4d;sjxyNlpM*mn7j$!-aqR>G8?pwF2BLX=;I#pjpty6&b(}&J&%w} zN&M>ZGq!qChbt6|EA+%G(lX7-|Jabx%$SsDL%jCXYp*7c;=5Og{?mxApx3|rdR@=q zF*x#S`pTo;xNd|6j~RxBjn7fPLgiHe@5t?SI_A{T{ND*IVR*swxpp>PC5h6MT^A8# zQZn__2V~w@Y%cG)<+Yeh!R)G|Q+KV&{P$p0?`H17tu?qNwA$i$yzcElZFM3~k3$8D z_m0Y3^-@_BPW97CBV9aJR9hp~wvQtRF@-^Q#o zs?uB3D~(<6kAihTA0B%C(l`9n>-dq#Gqu_{K%6c6`DnCSA-nIiY}{SQ+Z%RoD3N+} zW1wcb4>9bvnAA<`!Gyr(?;_Pl^qk|dO`NOeAyRj0j}xfzipQtF_u)UKXqTrHug4`u zK-BKjW@9U|9PW4!4p2K9v`wee*Lg`VD)$ql5=4pdgww;ccc3H!k#Et*)N}BxDcFSI z;;bac=c_^O0)ZdTGq|Z9XW81$jd0g20ZsQgyr6-U&%CQ36isPdbcC%yVR91vAl|CV8L_!) zw)ft~xJSnQL?>Xk;vy(|8(46&c?cZTtxgj}tHKWt|xkEFm?bu~p*CWPidb0lD?ePiQN7A6F zYv*4Q@qxRB-*P*jE|V2 zzPr17H@xztNU8C&e}60sNUQC*@I-;yHp9*bSs#UKV&3I}eL<=8JvD__O+R#d-B}qAGJeYxXC)v|C^Ny9-;_-AD?+LoN>Qhnx*#-14j${ zvHwhrbiCSdj``r2vl7EQozH12T&AUl*isT1&Sva+Gz;I={zybhrcMFX8HC0yPN~|K zkyHHY2=DbS`kQj#10*=aRh4C<^wtbrDhk+g9k*-BPdGZDrFXxj$HON?oPud~p=CdW zlx_>vbm=|0@>j#?0&0H!*Ym(#CAWgkFzpvXk`=Ku{9807R}WimpUp}XX&-kYX9OHs z8c1F$bZ)VFvg#UqBub4UYa^o>5D=CgdXrfVKF%v>rfo!Tn6pd!3J9QZ5xa%bu(`!VhDYuCM0vajP;9V8eAu_I5xSGQ>zzQnV=?bUS4-r z`rD%SSJx;kdRv(pAsMJCQ$rR*diP^hgNjVSn`04XL_x#VcYj0WkT$zNmig8sIG&#( zpZDEyqC0HuEI-ylzlJkEoXmgWH1RVzoe`{s>gqNqy8V9(0jv!$0Owx)LxLQbOO-^3 za9u-V)m61+tRUbgyR!WMw_L9y3Tl>be89;e^^qSrWo2%I*KK-Iugb;coa{(FnNF{+ z-8uP43Q@$>Gzz=b^HrSMspH~lkUs<|LwgZ^sIGDO`?sl5Q=Ls1YpjDYA4Q!s51L2w zZ*Sk`@m8KR8#qO9E!N<;wKX~88dQeR-l}P#nnV-`P{q?}v-^QZqy9l+eX^xh&_PxC z)$Si_JZzCIcNW_>nm-HE-5~|rNv?tjldc`fdL#9bpV)8gR=T$MOXX%L{TG!(N;|{< z$JSd0(iyjp<6~?%I*#5n!_mDlbui5^9Npt^^z?M>U^=F|r;qNQ?%H(6LQLE5_B_w` z|LXt#j_Y%Eey&G8?`V@AnIrK*uNITd;ju@5BMe#!JwbB~;hdYU85G#C82P~4t8L|X zN`(ce&F&6pz&T`U%B*@0;NS9%V}tc_#H6TR(Waut8RUdYjfobLg3@OPmc{`1YtlU! zUSI%MySGo=pQhYa9@?>XUxV+ZmClKP?XW+5_RD;56r45%HQhe0S>r@^0@0yO^$Y~{9`zKx}PA4IigRdm%YYQ4l1qXgIBO5_iNFf3(L zfxHNggObsfBld`gCdAA^|N1T9!0S7o+2CF745NUkiBiHW@_Dgg7`G+>AsO*#uY`10 zyN$K}_i3^r7!S(ED3tQs5Wq}PLw=(CQ!(}}QEaV!y;y~RP=@>v%m)oqK#I@^|F6C! zx-NV1M?e_z&Y6|M$}Wlr;Ar z@VW~_Ibxl$e1}Wc9g1VJUD|_DiBk;R!ElP|M|dHL~&}2u1RTT)*@)z{jgT_3i%q zGXLN{OzP;75cIzIhM-7@^VCK;;KhUrA^3?=dp!d7JL_aglBdMF<}10L>z3TC=E$S^ zl3m)X6Yx>;^jo9-RZ2`){y(lO{5VFm9Z1|X7`qtT9El+y%~}c^m^J_Vk|4{qx3@Q| zK1UCrprP6AA#OsI)foDT{zEXwEIaK4y}>LcIIkGr7txONyIp!r3gVj+FaU)Au7{tq zO^yLOWsG7$&UvhVXruvb2f~2-qy|S~D6iz&Up7K!H>8^x0qY4Obla(d0+P#|JpN0% z?WOqV%F;L4&mEy})Q0VrCU5; zy=RH#TURVlL9e~Mc|vju2w=0wg8<@}xaW+7yD%Ij*MjUx{N8kfUFu`^Cu5hq^fW0T zp;zJEu$W)vG}qhdSy==KQi(h%Y=FNd)t6r` z?`D-c(Pv_J{wWfIb9CAw_{PP7K48(!2V2T)C9p-d8c@sBZQ0F2_%Hmw|DA0xc^Qjw z_J|tA1}-bZ;Ie|1>H#7rvl8g**DzD>!^aOWS&Gk3D|l?YyY)Zt8wZIWc;GQHU;gsz5_TH+z4QKhkUVvL!e?Or^STGh(d(Y{xKTZ0YSpjZ% zkH7@51;@>GdND~f_BNK7NI$L`ia{?3^e!Pop-J1gfLfetK1$|q!rMpVim9*4hE9e+ zlGdu+TT{KwSGtXD|Eo%f`oW{aTtWZQHyVs)fVSV0^H$@0teG`>_1+z5JLvf6E-_V#QL5xm5!0kAW%#MNO{5!Kl}5~f z)FBhi@#qJ;Q=aO4C+2xd0@x9-8`U@R*9M(jo9`9 znVYldbfHbB0`!_Lr0;vc;hdpXG}Z#`FZ`fSp1v&@cn$b5RLLy6Sjm92J5p}%;R2|# zc5wx|i$F)oB+K5o)lhMH$m%SNYz6i(uHA? zY0w&wlYVg25XYu92XOzy=w+0c19#;@_s?V(BKha@lDmXt2$T}~tXm*{e{PxN1_Ug3 zSYNH(`>fypFtYzc_#Qq-0A<ZO)&CR}@ zDz8u)qMx3Lp|Q9Mc|DWhuO;TGhGyQvB<-vSVC6kQw7-|{z zGss!Dye)KcA^7BHdQGK@FHhbPtf%?0U&@tnq&WXQ3Q~BYRAAEedy<981fQi=#bSq3 zFJ989MFy}iZ~a7(0{1hub0s96ue0q;V}w=fz`SRjmgWkuz=a`9UdtN+O1Nvi>NrTqZ1U_`fxOKb=YCRyiV z%j>qj{y`ac%}r7RjO!*xf1n7X!O6wgsB5dj`S@^wQF^C^0#?UjF;<~OR~Zz+_1H+5 zIcE50)vKm-LAVAjMF8pV$1~I1)*5Us&Bx%8`!0^-SpO#wx{;D+jgj9zm>~5ucX8Kg zq}k=FoO~HBIs3s~C+m4p3X*s%hqX~#8!r6p0S4}2<=40P8_!>zLq6^PX|Sh%6lYPl z{NWmfrLSw|9sueF@1!eK>&kEBeqWUqWSyJOv9zNHA{Hzn!siZ>=(apBMucJ_)J97& zoEK?pK0yM-N`w&_$1?Cqaty^loNe<-rZsUv?APn5r)gsN& zGK3uC52T9(PCz`R^CS&7m)jII~7x ztc(IirN2xwD~LPOB1)$9J;&Pz5m~Zbk@j_ogdE)ircBR=wmeLi9Ytq5rq4-i$9o*p zd&`s(#aXOJu<@X+&8i8i`_#jmgY%T_-|9wi+Io9v1C9eoh2j{dy%6}ZtWK?PAM(Z& zdA@kIBGV%{&Wi$)VA?pf&#rG!rbI4Jl+fFFy4HiieGGv}Z9&TIbdmtQIN(a}W^F}t zyY3g(j{fKUiX$_Nkr1=@sv(e+hHz->3l3mRs~i4}eMQj?(0FnKIs^USyMm1hfLT5j zlQ7;>UMUO{@6n#{U8Ou1$O#Fr{<;tAWR@p_C0M1kc>EJ<-yx`{-_UcJSM%cP2Upj{ zbc07=y$mY}WVU8Vu=siY0Sfr;mrtzd6+&u1Yv)ddw^xRq9g5fIMBU~ifa=i;9o9+0 z*=S*gdA$j+JK5*KR2Rn$n~J2MmZM7u5Qwxw2cTJ!@aWYx1~yW+ zxZH{8o%Nwylk~Eg;d__Mcl4Fb>G#Czj1a)Y01lQup_x?;9}r@_jWZsTxfk9%UPT}K z#VU_gzI4ppd`3&3P1$E3;c=U~s$fjQTGJm1xMZA$rKILaF6{On$@X6w*i@&fO=r^@ z5FZ4aTJaSmFHzx=Fw*@$<2iuI6p((#UCq@61)-)rNnvuo^d2Y^xsp9^kB5tEF}nNV zk0h;T=SV0wLL;59kl}W7wyZ4QDNB8e(_{a$7$VFQR3z)sf{7i&D$z+E`Cn*(;&RI9 z2n5<3`;gOa`CCg8;Ip|g|3ob=bcly`piIzqTtSxf6ufUUoiL+mhIH!(zj$AzTUPR* zNz0Sp7_58tfmQ#3XL!!tXritcbeNoe)TQEJzbl*cso+CQMh1(1{)|D2=z9Iu2P2{E zM`7CBqQMV$Ucj{MC&%lzWgV%5%)e+@w0p@U{nV*ZpDb)F-Ap;L_?THxsQSGPh;&+^ z8(#{k$r$cQzf0`mKInfBfC+PM-xXI&SPMVvLK(S)h!l5I2wTSJTF)kkq(R|hVw!UUoOdnt`yle5*c)zE+5Sd$H%k#{%-IK2ESY7zVB=@I=lCTdv_<4v}Y+ks&6`45ti4lLLgK( zZSfAV78(UwyW%=eZBl9b@ z$3G)^Typ#Rzufr7$p4@MNPcE7$O`N^2LCCnP#265xG+*&{_*L0hXRJN5K)YJW&G9` zeS~Cwen76!dT#6@juGSbph)}vXDPxdcnB`uXbsxi_HMwN?x4I(c*!M67)4 z(la8f5>mnF2aeTwu$^ngbaCw=q+OBMcR6BDxw?qZKeEcA&yfDD)DWUky?C4+vQ&2Q zy`AuXe#T9nWwX;i$upzhg9RQa&=$O>^pCH`^c69mR99V3|shAo)?6>g&l~e zo7wO44Gr4ZPkUjK)E4-#x}V9@i~MCAc}bJu4^Jmy%-KC|8w<5Fd0BZ6p&}&8qB2BR zT+OBbXFrMD<)3CoJ1-p}L7+hrMB$STz|E!`_kSjrnhzNEjqwZqf$LuCW{6eI<=z81 z1=qF;5~JlWM;X)Dm8|b78Hv?@vF|{{7=A8rNneA*ZjlJ()ovZte^>RXe>n5?EyX|$ zXcs)Se9O!DhCcmZROT3U#Ty_husa#nb$GB+gStIp4n;&SiR9k`XD zU>r|Zs}cENr7ZHq?|uU8&n{wavC+Nqx->84gghns&UxuZDh*I|`o$05%df}Gn|*oFia7;l&g-QjpS)H= zjhg{a-SGYQsl8?la!Y}ypkt9f#e0Taf-??p>6*$dte1T5f zC*fW+e8HTp589r_Bh%@2i)_|FZ9~eSTAKO}*wPX~K|B#C$OUYEQ29;>D9m8+3#FHw zd5nhPYKpBG0PN#rC^kzX*dsKuXb3~El!{NXeQ97Eij7j)K!CXeZfOROt@yju@+KF< z8CbGG4wH>$jq7b+lFld%MEQ#nv2Y3Ss-BsC%hG>s4yqve&8|cT{*QX|MLL74G1N?aH1ti!k9a+!S-a<^EhQ=Y9r%MWxsxF$OxB%Rh5L#$y35?y4SRm9w}u< z5u2>!Ms+JYYBc>tZ0c zv;%?HVUm#Rkf#-!Q#H76Sw^WapH5BrWEsZ)=dsCpmc7A{(-D}64TyzHY-;n8;Q^@3 z2cWKO?tmO5{!(BB(#O@aUFiV~B=I04f|s8^9!M{>*>P-cKE9`YG2V))OCv^XXK;Bz zD$z}Bf?b87pA3B7bE@G{$wM*9+#tp6-+c@ayF+1*Cf14>?hQjeS7GR)a|R><@y~=vwh+9b(4*%rQ>^q5`?zeTQQ~8s_Uc9|S`~YlyIYS1;yy0uXy$Z9+sc!+#+I z392xbm;eK-yPB(leyquRy2;UYzXa|xa{zMTOSxJ&;fHetEJxjz^4P%2me>a$%2lY{ zVvOlO*!=zd1YGJ#)?Zvk+sX+pnnN?#nxIRr#=P)LlKlQ3ygsDu49BYc;$SP%*m9=G zFH2p+W={5A7Ea<}#HR4Bwk&4C1`JZvV0=5YNei&hY>?g`IdS^!-Qe?lohsOQ>DY(E zZ$DbN%#lBbGAq-`|HrI=ZD70rN(>!Srzc!UzbwG&4u2ZA01y@M>Huaz0@L!gF0^Y8 z1)>A`AAa*`-l`(+k{*)!N)@TG-dF$dtvPe}$P51+{`>YV>h7O(i#`irXX)YnkNt$l zuffC*nf!UEb6v;d1v9huhK7dMZ7d@1D?eU)o&MV!((OjWo?a>cK0o=7XsL`Z0#bUo z5ZTVidJ1TZ#%GaaA76j{>cU^or~~Ka6RlJKDp_ZEqUtXmB=9-`Bgq8;n3 zOJG;Is+;d9c^2!h{13vxSk?U5c*ynaovqqZvgkJCQ};k_RyW{45W{9jeuLjAko99n z?Yu$4u-kbA@^zLwj^f2txUdW(uT1a%8cVeH#zcJ4IWRspgo$S5d*da`nT=f{0fG$c>R9nel*4@d_dR z8qyQc8pKw}H#jS4gJ00po2A2&w$UbnLwhQpY4m+mCB_hbUnIjx{i1s z#{O;msm=Pptcc!|6ZJ)9h2avF+7QF-xdYLX-)FvYF&<`cqvEDccoa1nkG^{`HWOws zpw#<)8mlxg7}z6u_}WloB5+36j-HNP|7+Ph&*&gZYv2B5> zujXkr9FLyMx!KoMI!eqlgm3P=hZ_kgeN_z_#bUDi>{)!u;YRZ>&jeJ4B079s`l4#; zE_9^T^?oJZ584l|z`v{BeI7j(cJc|bzzwj+^{K}B^xdr5#(5~yS|_+~12?K(t~1h7 zX1l@pgjA=17fw`hzX^UX^#REJF;-{^gn9B?2T|Rqul@lApMob4{7A^o)7FpSHSHlj z+p{j0a~rUq`4|QZxeM}-5yV=Adx8^esBrav5tvsV?~$J_(u(6%VoU9K5tB~k6ML9p zd9KB9Gvomd#^~SsCP0QyDh67i3A;h#?~TwM;JlK)h`K9=<)qqZ!S@uf;CbUa*H{8d z2yt%vlF(wGA>pC3kl4(J+gqAbu=!S1XS}yI!9VHmY6_ae*1jO`;Sq0lk{i+1*+`7< z3uZ|kAlA}~vB){{!@S4%yOy)q%8u(1e`59REtYQPJJhT+Ru}h6`j!UwKFMSde`5AU z%D9F^U=NtORcahojtvw%wMV1M_}=GA8B)RjfAIG#5q<`UmBJE+nfc}42y8G*YfRWD zVI{85MGlm=lBWYk%z3K)xf!c*rC(oaTeygX!XRUnNlIkD%P_xoJ+=Qy7sJH!1bHWz zgRajGnW5Rs(>)u-O-_+wB`Ma$hH^(8&TD}=9MKddZbPVH9<#>WjclFq?C&6Wvr`O`$fJAK0R)mR;+npXxyu0n$u`&>lH0%2i_OseE!~^78 z<7^h$oL5tkGA+0M*yChvB>kwe(Xkl%#DZ%Km%)grTAAmtCR8bXp?e^XQSpv;Dy@*Z z_i&MT(<3Y$UD#7Ke^WPS5`_O$DEn24%1W*X=Hvbx4Q}>0Pteue<^`=ddf>*F=#z4b z-|6_~$oEIy*Oiw>f-=_leaIaGbWH~~yD^5}GN`duIwOJPoN8?<=Ad_X8HUn# z&b*cEIzF99eqlsB2H>u`7XP}u0>t3<)<>?o51xZq0_l+=5B329`^5)f>VU z7rOO6)^gc~XvtP6mDantr6&^sIyv{hrwbD@55039yA(aKUXgD9ozl>`YyOu|HEW;K zckgX{0f`tAGFN;^ri?Zgv}(V`%kp&3)R3A)a?{1rN78jhH1~O#9kBKerlH)GisW$o zD{X&!dn@~|vw86SSc6s>)Ra{-ai~1jzE6;KqrBr7zC=Ox&e<=`V<;o&qLHZ+=qtlN zr+xi8y;nc9|RzGql_bB%Ukp{-d=a^unS9Mv>CFV&#jH@ZT}$#L~hpIEku<~ zsT-JiEn~S8p>H)$U_K^FZ_?#VC>mWQM=!)J1$8;&a3AOOl&HFx0r;McnA6;#Z&o|D z`z97NFU<^8zy_+N^T=uN?R$^RLGe!nExQ0-+T}&i-^l*kZ7P6#%IE=;Z2C%9PhI1^ zjuxL(e}#eSSH!%Sf%dG4*yBdH{SklSt;js7PoEkP`(?I6h| zt-%XMd}>1Pz;$VV$9lcpu&cPN|4XsCoXe4kOiA)>$tZ*0CzVO0L3O-X4U3v$lDEC7@(!VPgmmv z4w1u^0dt!HS#E_oVP^L&c!rUtGyp|TAk;zIdlq9wOB*b_%#4PPKCHFmiB66hLi@7n z=sC`WOE-wiy%Ja`B>9w4fZxC^S1x4eXEX>87P(f<)*6Di^DVIWK;FD|X}_5kMZgBM zLQgJ8u7=c}j=}tiZkjDOzKU@H=(U8sBOcm!4z#HVTw8Tv5As03dce}JbMrSB1s4g# z!MQ4}9w5!yl*i)gpADRoZ71}KE>x&dbU_Jw3*CA5_cdkFV^Rrp_VWt~gBR!{Rlfpu(f@nL;7Ht;YBrtm=^pEMzv>cy_Q5#K3cIjU_m`$ZSd$bpc?IA^iya{o4gSQk z5mtBh+S^g?igbOgeUtnJn&u0+e`u=xL!N-}b|Kj^TaRG4<(1@CBkki0(&(klFXCvv zVwn#b{djARUqE;YS+0hDqbAuSB>CDI5r)8Yd#qv8mJAfmFpK>6Sud6w4G_*Q*G&Ud_UMG4qB0 z;fSD8V1z@-G<4q*(e$1wK5Om6SuAJZ!X0XjCTNJx z%(w<`t#)^dW|Co*bXY%IS!~9bDn&RQapJweGeP#d2;iWbiG37N$Y4ib{$1c$Ma=a}wcTT(H*^K9eMAQT|Q7b4_xYRQJpEnr`-5MX?g?P$)GsP(SBHvmk zJZ~&+g>H&cg+8jbq==rHTHFP$(w+=dq4rJKvHi0hKg={)AH^Fz^|r=kY)MQS6qpu- zc6ztiWcu&rJ|`-LP;KQ?Il&7kg_4M6~!A0%U(&kkYoo*Ir6FqRzmVmo-FC?k3D8Udu-&mOW_~psxsJt^X(MNxThA9|6;N&xB zYZ)G~LRNcUN$pGqJK2#{tj7Zf;6E}O0m8x&vb^-CaQbZ?jrhkn4)0uU0@i~yA$8ss z7r|sX-}E@6scRS8hOs`(nZh+*$r>UQ=ipH*eav~NP@BAF%`6T|a06X2W**`R!5G0A z_J?$kxOcdXH96G`n59E}(#Z!SrLr$wo3FO}!O_=g)+dTDnPY&f z?=hs1@vI7|)4kvUo>UmmR8}`FtIgoW7!f0t4Pp{dU~|5H%VIumIOUJ!Sl~A&jX70N z@i5`n#%W|9DxMTae6zvtGx_0MR1HN?hMj$)+I))9=r1l41adYcR8)sCZLi%6&%3`~Y+d2P z&R1LAz&NX@_4f8UTG*g-ZxSw-x?@W%&IB)X^M|=PbkB#%EQ`HcFe7DM1evp_QL$%4 zWE19?z zlcv+3@x-G~_L-xO`p!X(Hbc3#4#o%GT%0h;&?W2@_BNse1lfR{eY_-&CsNb$^^?%m zP6s?}y?{Z&pHj+q`}0dLt{$W`PFwDeq3(oeor1iv5T`$-2ZL@o3X4}MFbgEMH5 zZmbFIc6O`~<0h4=A28OWcGdsEKBIry4IaCOS)b~+FvUptB1sT5c~0|VurnGN1p}gZ zI&pFD>Y|Lq!5-EyK`&Nsdy%>d7r^e(p>#*Z?KQL(7(TT2QJD<~G8SL!4@nq=0FIDY z4ggD|9BFSl^mzph`=%24^!I~Cp}&j>Q{2DvWX(KD1UB-}mzP#gAUI_E-*s7DzrPDM zJ0pIA@!fwo;m~OGAO!OL{Nmftjee};HL%57L0@voza7{WC@wDc)(MAziW^Dded|2f z^TwD5Vft&WGo0M&eezvV*ni!8YtW|`hTARoD&;nQ=1$lM+QNJZpakz7#*`JC$7 zESCbE?==_)N5lh}`E066L^_QAlU|91YRu)1`qQ{J8vJ8$(*{u7yuPv zj=TYjm)S9r*Jlt?QlJ}&CEtUx<#7P6+gIIYmxa%)u681Il{=9QF(-r)SvMz&hCopZ zxYO3}THRD5`Z&YHr0d_epY=b-C>lG2>J$N~yyBMx7Ovhqgs6s&54$0Kkt-fc z2#$hQs2h=|BL^h)`q_I#jOy%bnQMUftX0DBbg6Pr^&V1N!CK(5BHQcILZpJOA^h&$V1x$X9Ss#yenyP9GFFVL-E(hGwjnTpe-w}eZ;k6M^3B+v1~BMk=P3TxF`jC)Xbyfyy&O0 z>WTsYC7}-HNwfwDQZppEMgnuJH;U-|I5xZ;(LH0j53+p-`3-C}e_USVHib>>uTjA7?c z6<1D)X;+JVwjR{}UQwa>Ub{lzmv)=UD;c9`_XdwBDaxyoXPbiA-=yKR&2riXU&X{R zRoJPVXsHE=nFWn-`kyiViu6O9P(I2ih;7hy8V{!%BYbU4%&{M=&1rrOzQLS&#c>z7 zTn*$iu@8NYOUG^>;X!<&57mjOmdTD*t{q%+{`^@mOKEWYO)$KAb zmFP)QTqMM+h(4l-TX2*Q(oZzdP@VFHT0t6G+Xp++&REZ<)ef(~d-d%a{ASPHzOv`2 z*uy(Qm!y|RUAWOs&V1h`R(8)T32h?xlB)ed*6M)OsD3}m}QFs;WPprIereHiUsv#JJp zCV#*s?F1@1Q-vNlP>=9J^XsBKM{w@`3AJ@bb;v+Vc8Hdgv+kPpctqf2!X=|R)_qp8mq*^8kjKiVgj}^Vd1xj%A|TY35>wXy~iTx3Os>%_n=ef=OYa;3}E+ijWkq}a&| z+VFAA9=8Cse#&2yo=xHus%+SJA+6ut)_iw_$HdH$ozat*U4mDxi{QX9cxzSq&R}GF z-wKXv&^PQM?U)1Tb}+30P|}`O%^X_RS;);CYO(uS2tL87qksv_o8G!aEvjSMD|g?r z7wr)b;vl4UioR^WC@~N*u(3BG09tIlcGvhxIDwMp&wf2x*I&bZ&w%Qt!_EQII`*+$ zz~+$<%U1awEPc{SHl~7*o!p&VA5czn!k0%(N%G>6ZQA#Ky(~l6X`Hk>xLp5Xj7U_t6hik`DrGG+~0V@wEbwfM$s&E%VH%J#u;3&W^&$ z*~v(L7B^ld<;j;`=QGBfZLQG4Kf!)3N2?6yYI-!n0371c-P8Iz8`AoAnHyg7A-r|e z+Tr5Fu(MAUt}h^G(OiGY`sL=bMjALDeLVH$lD^%_w{PCgB6$>X8-S4-z74nguenr+snVD9kxzC-!cIqR;1nL zru+|LADhH{qUIR3`GWJ_A@+Co)$FGQu8Nk?i1CZWs80d+;!#~1Ek=7(P-vKW|Mvrt z9uV`61DD<^fG{(Wx{u}DK&gU=y&y6p+h~3Swm{{M5-?vL8<2CZ3IS%^#O&T1)DNIZ z62|xCZc{&&L^FvZ8vWR4icS>!-YgQIUCq$NYtW@EU2}*HGEzGHyT<#!v|vbY0P?9n zy#1TqEzg_30-@DVt0UKDIqOrHKsPMm2|Pyu*SvpS;=3%v!mQU*veKAdx%VSmtTe#t z8WQ%QqpH*jvj1{Hm^(1mki+n81^b~5%M|cZ`t{WCQCK?=b{4XL)({Q3I-Uj~k#8f# z`7{|jyR*HfP{-++0m8Htx8xL02KreqkCEBjA?_1j(ve|cLo)+VmnQxUbaU)|i;q1+ zMiYb!q@8_oZ}e-5tj(l@2VixEe0bMCFo5lrvVeXL;lbM=)p4`Zb0SU@+laV|#rer2 zH!?QO{kwnbQHfr8#i!HPC-PjvNe z7LN2GYWPl}7sR*As-}ov4Z!b3x&oOPJ1s9TZB9tTHmmjY8OgHccBVy0RbSvDO@3)f zKxm@qh&nj_IR;G9TP8n|K055H#pEdS=>ro3!!xI7>vlyd=$-)NG;k??a;_Ehn+d{P z%%=!bY2*gR`s2qXPh)MvX>mh{`*~g7>(d5rKV;&a)DrRaEke3EJR;xDKh(t^L?S06 z$P=kf^ar~sR`S_@2;~7>gL?Hl+;BCwE<4AcHyGzohntLdk!+kdLO#(eL?q%&ILxu_ z(^I9n+X!u}N;eJkkq5a2lA(ROLEIT)*pbO&MPJ7%bQ;}@D`y#&5>{c!B%r6an_G}4 z^+;kZ-C1|_H*qDpGFG{{@7=qBZ7N0);do+hdc(_KXRjWgSI;ihnW-&l!zI3%{tuy< zgAWM34F0ZQvLctUyMrQe&bwdeu0WWIREOl4BQ1mRopXcy!{lryMv0T9;yKMdO2ry{ z15+UW@mwb*yv3LCf5|fM;!+wEUI8kM;pQ{_swp*dr_D- z*$Z;cHP?d)+s%Z%#P%84&@ZW5&}jhALN!V+jcn}}cf>6`L(U&~DOBd>`6`Bkfv&dmir%T#}C1t*#P6k!_V!^vx2H zh35`5P5}q)_5|6u^n>x-eI&$Nh}o*fcfSgL1*ZPjSG$K`m;12tiOXoO#IF`&y(xz8 z=(5Y7^4e(u=3FBzpvF_yC9DrYHe(A_n7x+}hVq;)NCNeJeEOBPc!i zcQpwSXJCYq&&wWoR{D@lqG<{nW0GTm-YUl_S_6hyF~TX>6;ajZzksgBW^IJ(A787V zPC2#sEefHHx}MJ}c|=0l;Q#d4uEg{s9C{j;4c2dGOYGdV2lGSv+(1}nrO6Kd=o#(H!{YdacvT1I( zoX`jCYibr^i81}!G67gM*J8}h;{bNK7(x2xPU^G72KT#l`Ldml(g}=j@zv>t37|v6 zy9(H@Um8SO{;ll?-HfLvMrlX%8ni~Towx?0-LNGsW1zx)0W&&l$6l z7yHV`rB?FiknHNe!#Nkd_c($-sAEGnlGQR_J=^Ymt7}EFkjr|2nX8riV;46J<+jMi zkHShM&Wd@Cjk_J$rurWv5qJ)HWcZPkdsngFEU0hm z=1lbzi~~D6qhb8n9WKN{?9}Q&1(W6M;3x_?1s|QhTqWRLZF`H3B=ufy5^h=k|7c;n zlFJqz%$&x5ajN4TZ5OME{i7}%5+Z{~-Wa1W(9BhdsgON=4F;q3MCC?8Pwntq}CBKk6o{>X{%^w%gui_;3M4=b3j4ev>RvhVE!(Gav?b{=;;lg3>@gm8)yI z5;CeOgEBvbU7mu~2jVO{)=LoUcC4D5UnjXIH4EhYbAhBQf8-nHQ?fiQ_IYbl7r%rt zeooxnvG;J%&_p+uQ?IDR#QEBm0z`m`DK{#^;oj*?q}cQ4YmeImeCFsOCSUsu5qD)8 zztpkWLapd$FaPo-dB|Bjf=rro|MI=EG=!ba4FAnt|2-@o4eG2{qgniTvUNW_aAv!9 z9WmYbFWX&Zm(mGN5cXGP2OcTLEwn9pYOrpJ>#5*w)jrq@ZL#@#EXK&hA*~5tO7@oSGld za^aVw@PU4-f>5v3x0cHqwwamT4kus%)<7l<7R~mFFDH?~7{LTp?*-vbQ|>kNsUT!S zr&$aD@+%W30!Jw~J|kjrD|@U*W`=7i6=_tq_}klfB2YD!%dR*xl@Zw1e!!NuJraZE zgv?re^N*}nvaYC>&abtmS*KB({13spM6Ybh8>|m(~R{clW3#&T66UueO&W} zPOcPh7D9+Jx}Mn?H${*+q;bqaS2jqa=FQ}SXv~xT4zGXl5AEZj>9fyhc;uV8Den69 zH=_d|Nzw1#_&J4-INo6;lTPR+L8d9C#Q9NDdi9`i%D7Ff-pOe)(N*bt38j2Y^ki`; zSGQ7V3aXYGgMVKN6&t* zBL(-z!D|`e#ifdo8l8`J=v&|E0uH{NDfqgkiz2POWad4PBxjqegpUOlqaE0Ogl6Ui zvhwU>(JZ;GbZ`)dzGS4Z(6o#fx!R(CdcMpy^kVHo_!y5He`mTR<>d0G_>&{%V;Qg>rNc6@?Xw&fq zB)Ga%l29RG?U%S7r%hxW1e*cv7`q0van5><~1S*Xl zk}xd5c&m4EYqtxd6k;pV7lB`P65Aj##3~TA8kTCWv z*-bQ>DMa3Eg&~YUc^=`}o5#Arg^F_#7}bs5C?`l#jBeRgNs>%vo_N2qG4f~nDM6-0 z&3qvwB9leg4xcxB^AWezp5cy2|VMBG%w~-bl|md zu;4B=wfWg`4kA<(aZ}F)%``i{qSAl9ukzc4 zhabx=G05Gs2^F{Y|JeG@aJYl+Z5B~hCyNj*N`ys_AVl;Q(fjJX_uisMZxNj!h#-2b z=$#N<)aae)y~R87JkRz2@_yfI=QlHF&YU^t-1l8Hu=j5^M*%C6cuhI5d9t>~a0l`vKVL-UP@kztIewM&eZ1wR~T!p`9; zGL1jKoFDn0p8}tqAl^l?$sD9lnkm=!;};gW@YxjC;&dDZ&I{GkOrac&U!xT~uot@T zJM=lE909=uLlf^>Sz!1AT)x$N6QZXyVh%QJY!|BUo zJjKsVA;H)@_my9;XK0bT-NFv&u-$)vVzFoiM$crHm9BRalNlO^Z%ZId)wq2_PYsP# zn+-rI=xMk2<8N(@5V#L1k|<5CBD3VbULl+$)*^F zR{1({r2)%k9F5!MAX`mB&9Z{cr9%Ek!*TG_w4V{*n$b__&o_ZbM=+^_XIG)dK&TyZTFgSY>f#l!_`hC&(xqD0EI(9h4pb^}TAr7=B1GGX z@+JAH$duw}IkAo9z$j+Ttu7t0`+5!e1~wuMotm#C2Qof>8Q!q{S#xgE`N4OpwZb5| zTQzn&a1-v%!s`)PA>rk1X=a^;3>Q zYGJ=?3d9?Z;k3o!@R;FkAA4s1;O=K&^7=sFhP$BsC)YDkUQ&EVSSwZ+3=o>-^i5^Wal2;jB&VgqCa1x2zIih1(%0Gz_?wo z#DiL6f2L0Ce#%ZFuczDE`O_FN+Ph6>?Vj|M@}J1>n-W7iX}EQb`AYA%o27X|Nf4yP z#SAA?F=zWr=z4tQEP#gKu5H5T$M=B=o{{@kGBKrPA`H5GQw-(m+7-6u?m-n2ZZprn zxhLWJMrI|4ze{Srsi-=)n=k(@CB6Mju{(4;&Lp}x zQ_kF|R-rs+FQZBv>khw>>@3p7b!byxaj;mIWwST55bQ*+MD3E^=&!y(?||VQ06?wk zFKL^?;-|GV8e9MmA=S+?@zTxAal)csv`ed@xpa2)Vyaa4#R-`_mOUMP`~%7>1)5hL zlIXH8?eNMUd|ilO4onQ(JgR>ON%#=Oi%%`+InHC4pYo_KlsVBHc{Rk2l13fs=yuzB zW3;wXQW(-Al;2C~VDKvbk(a0Xn`O84fq|J8$JvhlGE%U)w!a%rJ%VTjSw0oBGD4KZd<@T+*ze&~MNu&eX^(ucl7(W7an=HSfVeqoA)wU)H^SsXV3tn>KjuR}Bzi&{LQ_d_ z-_+73r50s4K$11s5NbF|r&n9>w`*8d=8ynlovjOc0R|+EFY6aC>Z*} zYg~{%GRi*QY)&+tglIC?z_s;2hVke3wC6l91N-l_ftW;$7Z10GkS+Z2%7MO0czms$qRRZR;&1hpw zzEG-Eg+KGzXRr9$1q-JfTH^0{K(WYkWxr_VXH*Q+8oV<)8#AqMiRT?l_cm@GD}lWi+S%}!unCV3%=`IZcHxsy7=X{6vP7gZ)GxCo2P;6y(SaG z`+d9ug#({gha%!0JP5b=fr$VO2={vo73e{TlZVAV(-`6eq;q-S+enUUZ^FLkA_#N{>orYN5Pi+idJK= zl=~xTBfCjrtU|pVU8ySg5DRZ417&Ys+(Q}r($NxQdhF+<-La7Fj`l_b)T(IyXiy35uA?0HexP{^YqJv zv^6u7F(WlUR^W01@!@D4!ad;;^hGMbOF~2l)smb>GVTY5nu<+Zxuzt- zMGt+I?SkPjv=HeAjK=|O9~94=_x;trwfOjOZW5CxzGU7!v8nK@vbGr*%Eg+UVg!>5 zLGZ+%0(34VlZ8er%NksEwax0Q9wM`3C`C6xg_y+IpUg!nf%H)9MX{)j()i^rj!p8D zyszXb3Vq6XBxEaDg$)>bQUyPT#jOz=$lmP$;=(U_gdXZ!1Z_NeTYGelyqtk)87Z2} zxnF8zz^2xBt&}8kQ{@|y^dbsdwKW5ue8WC=F^t+qhNPu*q3k#sMbbY=BoCH_9Bp>q zzoLz8d9-i>IkMi$t|95XcK~xp`_TK*JEN;!nK>Y}qbtL@_%SHas%5CA$lNVohC?dI zrJ~Dl+Z*DI8+hiB(neT3HM)H%vqt5*)y8fkB1r{WvX7=NZ?SYl_{XDq5hVksr)Osa z_+yI_crxjg75DooEfZSAB~Yh%e$_430qQVKlN^c`WXK(oQ?_pz{XfatQ;X+UrJI|Z z06#-`*;i>^sO~Qvr=7VaNeIQV1M#8PA|XWv`ueAYOH3=@ko>0M2WwhHAyTXG1q}LM zY+W(|E=s?mlUb64h3-N=3vaR=U5y?{Qc3?vu+QBTYOvPf3EM~BhShqM2#JMkPidX@ zH;OGoB)i1D$?P^t9p_xmL&%_`R&2DiSL}}!4jGg@q1)Xut~F67@soMspChUmBq5Jh z1vgf~6ujS!THFBzW$OUq(ZdFG0ha&hgc$_kl%29reIb+Mj-mK{EG?Qp9}SO_;1tGc zy@+VUsDEq!5FrlO)ypN~akUZ`Wed+Qf>&g-(_;CyK|nK$;r(nJU>*zd`tGW55^0{1 zB3p@ST*A7l50CvZOIvi?`f4ZWA8S)Vys;Na^>{JI-=bj-YnbBb*^~TARjkum&N8E&L)EJz|oUG8j8) z&uL>r;U(mjHP&D7yakDVzt#z76VB}HHO>j@|$?tMBCy_3mOeO) z;gMPJ`LQO$_(m;OPooPqYO@9;m#UH@HG zK-b<7LZOhSsS{c;EgXyzE*)@WG<&ks1|z5Hpbcs8_wmO_biz#9Sb>Y$2=xHQ_1DhS zrC*HiYrs9FtQg9RdaB08Qdp-=vH7CJ?+*Ftzk2B1!k%b<{JLNP&s82*q--06A_00b zT$EOyvlEl9mJlGz+eHf7Qkt@5<>!7&*1%I(YfVrlS&W4bjk(MU0u-GA63Xj_e zThF$1t0ATdG$;=w(?xPz4m%%uE#1|TNan?{i?f++TL<_lZ5jVtHHlpPu^N1NomG9g z+(C#t!KL}4Q}yF$^(QlooB;DjHA8~0;}*Z*WN&@6#|jI@AbCYdVCV(O6wf9sOx|38 zoAspCpyaivB}Yby}P`+E^K8~1>Oq&mmP@qT1aN+K-S?;SD; z^8lda_*{(K9*E_NB&frSE*s>E(Vd9RUw-X>U+Z->1Pg(@`N>pT7ygy-UlDS#0pqwd z*a#X8&>0IyBZQ0LQ(U-nl3#QRV`JEe%M1Woy9Q<9sTO7C14>f82Kd5adThaK*}hR$ zYqS%7Hq?V3WQx?_3to7{@F?Al&o@B@sX-{eFyoc|IZoj zYzP@^Rn<-G@-R+%?!EdGEoLL`UjI{?3W@xvD--RSe&h2sN~LwN)A*$aj`yFG3Q(kk zEoZeyEx}cZKB&z+Q@im@mB(f)`&>1rvt0@(C?%|y0G9u*IQ$r%AvY(ABx$YBo{E*> zS9_|GeL&N7^j(caE3PR-X;(LcWZB2mvc>aC=puqUh`z6l{K2P^?5)C7ci~|M6am46 zABe=L-nNBF9!j_u7E5D&C5^eS4a;QDi$>dt&NOf;-<&%h!I)2}`JGnI`##q_3#6H! zHMTsE07q9-qH08yf)>qmf9x409nqLpLuIF9UVa zqw zi ztl~c7yGbRJ%r|y7hT2RtsW=q8MlP+sXa2P6c7LMgAyD`j@2pBfIIqz8WB8fR5$AqW z|LJx2m0wQ?27d#FjKmdULM= zP{(hexM9_Us>WB>gQ!)Z1>yTvqlc^U6_j?}J0>sh`o`RjiKQr_;8YMx2W}VuYVgqc z6bQ(L_!fda1dRv=x0bh?C4CfZqplcCRDHt>HvF$%@FiKI-tD~Szn$0gfab!Zj8B#3 zzPlcux89ha(g@mpFpo*)#HR|T$7C(}SIR`f; zb&;dW$kCB*!(uBIPfp?UUut4^G{CxKouGLPZ&{$1b6!Lzj|nr$7#6|eOY}hNI($TV zxG~je#3vLzV}6{I>hj|GSnZxh)GVlNeBes@a2&--xDO z7ExC>(w!lgv#W}dz2%j$mtI@g4)H|hI8y~f@q06xR<$-z2gPl2&`7rnu^Bofi2Mt6 zLtUAh?j~HK_c}7;0GP)5&-LAJFZrjP6Lx6UzSgjIYuT#>Ui1Mf;cw4l#paiInhqeD zTDuc-;k8y@%ob>UNBlH7$*L75@O{zrD(_=I|3h(n$ad_%?2Te?SQgJ~zxXzl_qu47 zxQH06MpC{(BfscPV(K=l72s?;2StQrlp|Qu)IR27LX}^iMsl|JY?}D zvy;oMpE!P(mZqpEq4V({N^@R^k4ta}`!d2-sWqtyoZ-~e)IaCfc3)qaze3X3w@MZU z*uGL*vkVeOJrMZ}*eLlJM<(5rg zd-mdta(}js5qF^=S5 z9m|~LHg{MqO&kwYSop+-S`UVsTpiv}#|2scdkq&+%S}O>N}|{aPX1LE*XlM9yb<#c z@b2N8bK?7XU5?iGBj^0z#IuwO^42J7w29gP3N^kPguyI>J&^~+t<|!r_G@sh^MKPO z>FuXISSFTyEW2#km@_aYaOF5I@Ti+gHv~A}=!fr`Fd8q$!~?x`7!eAiEof`h&*RoU ztds024vGE@evDOtYMwfEx+D>!vKX*BH6-TWgpo+vhCtWrYM$P`vUeGDNW61e60%x+ z<}mLj0XXL>&F{L@*eYtDE1zn^JBl1llN#Ko`yh|mNwzS%#6UgqjVrvCa0R_TT(x@G z9c|!)RgTNFqbDvT@d2>~deV9qtd2fx#{%naDjhJPU*mQK-_-yC%gg6ylp8ufGbNok zk_l0x9#F(o2?=o$I$inH{@*wo2*R4jTEByw#b;Z{KKY%KAHp_%K@yzf#%mN7EQu*H zMBPc1^NrEmu9D2f2;c{>faKw#O;5_|svl*Bh@cA_K*8}S<00V(X7A(&0d2qi5*t?_40RTy_wu2lDY$h@%8q|)4YU$cjY&VQXt#pVHEkPMIljk zNs-{|$6UG3hGjg$c=Wm@AIIc$2% z;1@Zu-fJxAWEFeC<030n^wX-_^Op)lE2e5-vaRX|V&=zW?z4-|9Q%p@4k3Nt$VLy< zk0~3Q{E-&ooD?x4+8)CRlYuq1_`|U-AvpF)B}FEn9j#sJ^lQypBtW(^ia&5{~MIP;f7KOq(|H7t4za! zkSziFTUuUGpYgp31kq*E^N+cYXxtv$?W-@RL3bV-sJNc>SSj&>V-RAAp^ozcK&#$tNDU446pU?1^Ebb+& zO{!Bw=6S8;@aVaGVp{s)0dwy#nd|zmqkYbLI4qDN~3Z#j#lT>P-Ho7q1DN#o+^*}bTuPW!=K0nc0 zKr_PC`pfM{cFx209NSbgo1EF-Y;I`oh5J(#YHQceo~9PxSX);et-;EIWoX;)mNNL6 zA4xX|W$1X|ufhCMRmtlxp}uUhk&nXUsj+C%C2QF~yX3o4G3(;Kzreb)&9rYM*iCX7 zlk@;wK2C;>AgzAb&`WIb3BBueYBaFD+~^ze`<;`bGP&PPfkzZwTq=oe!;pWOX!E>q zg;4B}kS>U4AS*-UEjbKM{Lku`tZk&e&=OFp8Xgk+q{dloPZlknEDk00W7$`PzHIMD z1kVY5Y#|P#54ad{NgN&V0K^XD+R^_a!XV^E&IMYeR(WiJ)LQ2dh7_U66|MSrNamV zn}u2N&$N2)ugeH}h*KKKd{7y148|M5lH;^A#Y~c#Y-=EwWPf`9S@yk^rlr~voMGnP zOYz-LB*96R3#M&F@JMSbU|($%i*U0F!<`0+J>Gs1I!tOSXoP zFb?hMArxVS!M|0XJBA_#Sc21Wtm{pBf((Or@J`)W6I%}dJW0{4yZ5>$vUW8b%^%Y$ zVhfvtc{%%@9#hG*S88AFtR%$SqLI0nLn{vdAW*iWh1HnfbCb+WMtmHDo7U z^1mxxRdL#dKKfX^X_nUL>rfY9X`uwO8SFt!C8Bg|4Vu+=xoF&dl?~e^jGhfk5xu+L z@fbahL^e6#w?kT8%IB~M#qPFyn<_jJnjR<|gsO9W=;WlPl{Cn)d@p&Mg)oy3604eO zUAfQZ==Zq{e|1%+y=l*ZXE4sSQ6i+*P|GDu-Tah~QElpbe8Z?7vWy=svEPm4UWMc{>f)5R42VB7N=|6mAhq_r+A+xiqiOR47g_~sSJ&TB zDfy3IhKm2zKJXU9w0ttj@KU_S|IiDI)88jl=ieaa z!QOo))k<`xzz&Pb>8-zMn)vT(_O0fvtbF~fd6A)C4ZC54-BFxvC46`l7%nWvt_D6{ zYNGARUmCE|`36xjKYTC*32(~nPab9z>aRmHo@#YRD;s)U3*$x9=*kyE%gmG@pYG@9 zzVEDG_O>bmK}R*yV2TG6g^cnTf1)XhJMfGDq8k4BR<_|2M4eH|qc=HA*%6Ku3jzJT z=x3M7pP7X6q(IHPN58adYE<*mQHkA$B%x&(+Qm^pb|&o$<=cHrQKCYaAVY3L(?Dnxnlv>X8+5ZlPh$XFRCz@p0h= z)(gYs+GlH_1S19)hE{SZ7e0MxZxNI(M3&1%Vp;E{c3?%%_h~)wDeZ@qzx5YAZ{VWJ z52=cTOO{`lh2X{^AHONGjM59le!um31BIH3l-Y<$Y_v(U!?@TZ3MDGHtN5la=5tT) zJVlal(sGG;?Npt!H8>{ky@C26ZVt!Cr1m2qw7v(v(zOXqI(%NmyA)QY&jSK@F1lsGkFS|jo|?dB_*x2=2H`%%^EnyVpH`@ zEMdr$`;8fE;$ziP6&_gJn)eFCYGqoCjqu0sg{Koz6bF?Dxv(~Hj7*Dld>^F`X!=n- zmbg9TdKYkg#av)V4seO*&ooeVQhJ`Pnx(%`9FrH<9F8DmO9^b$- zKVR$0aIW_qdT1em$*vwMr}=JHouAlU+y+^LPe-6FJ&y8(s(hhOL2|-nvGTIN#U>Fd z`YR|%`_|$_JOtg<)5VWId?%~gq)e}nz-^5G_Z=GZ^f5Wl-9cwn%|4(if_Xsq_~Sv} zbNQOMoXyimLqXf=xDpX+sO-4LTp&c<*5(-* zb$)%CMdCOeZsAOyXP?s_PMe~MwoUkSXE6|#&vH>Bm%N!`$cA{%{Pm?|1`v};;Vn>6 z9t|x5At0Tjgg|PA#5ziY*8h3|c=#_eJO7il^|LkK9mJ}^T7a_&5a}krKZXNc3M(LS zMymXR$G7F;>D$VI32a?*$xbslBb-DHF4h2!)uOm&Z0-*2%xM@9XHTzDG>-JOi`TQXN(R?d!VgdglMy^#LZi&0C%UaEZ3d=V?j} z5$-aEA9Dp87EM&eulP28JA)z_aRZp{c^zCPUbk4`92K&>hB-a6LeMRnq4k$!C1gC+j{c; zX|-zKg2KUMnsF&LVpzxFj%G0T+xVT<-W_TMHh^Uh-oV7U_KT*>km(FU`-cr0Q9YZ1 zf4%ZcG7z}Jd6inko3a@1){iN2vV@5mf7Pa&aqf$o2JWB>j@ z+HbmC)n(~BBsmTHpIf5%A7Uv4NU!i`Nh?*6poV{CD|FOg#}m;xp6On#iAu7*x&W9n8as~s!I!<{O_h+63LpUY38{9cz%yS zwtqk?1zs^CdTAzJBvx~!e3omGI-VNaah;oii;0V{ zrF)IfYz(a2s-1Ma%SFmMek~GjA6+>uz0BCv97H;W|6L|osCfV~%;}>~jIDfkAdE4K zG<=$OjWxC15N$NIKELoQ#k}X;i17bHiSF`zAL@%*v1hhSYIeB?>M4Fw;Zw4GSR&OF z>NB$$)~*L)SpQINFpwFKNaS*mFT+!`pezw5{O(#s2B5hTD_yaEfDU=1^63tr6>(N<&@$mIK6v#>WIyX^Qa^E8ToYhomVvdpUvq}6+gYhq1j|C2ryS>L|<)p z=*p3Q%B+>cENh*lzh`)t&;E~V0cBB&gNs>hg@M<0_Gd7qwx%@+(Z#dxiQn#Ohi#ON zi`)Pe%EpDb_n#6$&9jt=d&W?XFD5QD{3o2|q9ymi4e3a>CY#Sye{Gld;fkx^0=UBX@mLQ)9~u_eJ7|ga-C3!7lKxdAauKg{l3quW;>D*3x|u(g3}VQMaKt9u z6dDx`hr5c}T-4n&j`aEFem;v&isSi+9ndDaN3vx;9X6MO-4BIDKj#1mJ5{3#fsYju zf2RfaW0)^-LK=twGd!eT{2qb)pyfDy5Cyl_hPZb$p4-wBpJ<-Dnn@ahBeV|9HeO%} z$PwW_WSNqXk7^+m$QP}dyulC8TrP2!9%5<2HI5|Z)x%Yjnx=Q}kg&Ch_x0-@N=?u; z**iZzK9-wviJTr|x?wzhHqNBl+Q_yM zvxp)*wb`@Fdi%eB39dhV>L&=!hC4~=fXK)B)zVTWd%(2l_#P+SnTGGl4w=3?!2qWo z^Nqc;Od!;sLZ+-3(SkT@ygDuJ9aIs$6fG*Ivy8kZA_I)eb@NWE0`LF|mm7=)6IBDBtMVnpcAlVX5tWQE7Ke50eQCB>S;+Q@eQ9JOh^j2X<_wzVO8ypsqLVh6_*!-6GtCFJGO1GwwdRUB7sEyYBgN)@xfK>s>5d7Zst=vi{?vpeKfp z-(g;=+sp6P6Q7l5b21U zZ;H3JbEH<*>EN>=+wL!27RB6WT^Du9+K8KWHfPBEC~tqdOf27Cb2MMg@akVvNb5^K z?BEz}LQeVReMnA~M65*Dg1U%o_hQ4v`xaRiG4^ia5V)SNl3a+*QhBI;I*0H6z`WmW z`h?!qo9kN`U8MQJ)o`Z9EANv5sWR9(!~nwwMdi0gcQh}3T}!Mexj!2r;`i~VbS&)! zvKAb1*zI~nH+$pi$ZHM^3r7}%sMi>^!afzo-R%TuJ+&tnh^MV*N3U|e?fv#S>L*r1 zsT?9+fL_mp^cYQ)eq^;d*d!s4wK(V?QTY9K_2#Vo462~xxq71!AzluN?=Im2Ew|?!zR{wp|Gn-P{Oj&_ZBIxpxv5nMy_GCzx?3K$?pG62UeN-g2 zp-zcNl>PE0!Qv;vt{vptt0H^-TaLX(X*2%xA(TnbJymP);6m5#hZ@Kn#M4-88WAGsRIcS@KH=Wn(`mkkCdj zXVnt7bV7sSdbNi&WV_$jvt|}(;xG(wL_q^7ZEfvFZuc7-ky?m<=PK9io4##SU7g#W zlYPeQ&J%){F7vu)i}Rglx;|w7kx(LpFvJG+?Tcz*or4`3qq^NXd$2m&w6lJ*t{S=+ z(#rwRw`Zq=e@Ikl{tAid=t_OmqqL1@UJAT%a;yG$(8)y?z}BgyfKyZW_vYP$%E{kn zRr+slr(dI}qm-lQ1n3KtNmv}v_;aFY|M7t(ppa?k zP~Z#t{!o9|KI*inKQw^dE`A|!0Yexn9R|;OOCFE$7q#+y;45!?GK7rS0Hu2J&27E= zUL8cJ{-d}>1!5WQhOKkZ5u_mK!RX`PN`^bMJU@5d2p>9}9*XrTH! zUhN>~_m}4bP|GFt8U{9UQl2z?7FEIh~E5&3OqD(CHqAtkn{MwjyZV0~R0)7=&a%TAsKTK$gy3 z$P92^qS^vLFj-g?n22oxl5hnhH?kW(2XVk)B0)T!6Agr7!>Ngf+Lqwx)&WBRznU@x z@qh-f0edaVcKM|i_Vn*+=R}1}p%#z`zu7~0m>s755<#ql0bdb6DD+BicW^x#xmr4y z+%=;tPmCjjUgPn>Oss>m=3=C*`Sut0gb*4I3?1$UsXT{`z#deWTR??j;s})Tp>=T- zjg6tDf*oRQl)9GG_}8e~Eg3z=`#B~bNl;DbETI|1s3S|UVQhAxv3YPm(Ia{uxKR+S z2SHmsF#w!i1yTWQhcH!2S^swHCXuAjUwpK8iV-{)&z;q>XtK)?O` zDa4F|xL%Vux0SPW-fkio(DlYvh7HF3WwLa9QI4_cRK!B0`ki(f5(a%e--ZhJ|2|{^ z9YdnL8F=lOL&{FtCfplHJPhMx6y1O@i$6|Pp$|mXdLm_XQZ8Zgb4z=qb+o6VLjbK7 z)`|FyR)X=wxkwz8iyJyuCom6TOtt>lL}^4%*Tr=vA$;JS&QOao2N^+2p?*u!6ZvLJ zbg!~sTC7Zf%r>6pF4_=#$;iqan=E^ui9OA}|5xkU5fhvFSw*LJ;WI2_GF*eJxr=GW z`DPSmG5=a&7gU-IlK4$TVJ{0QJCKE5fVYK3(&RLS$m(5Glo8W2r-5^!1jDxe_;4OX0Jub!Jx;LBi67`q1QlbtJN-YBDSYHfxFDL2{D7_ z$2>cchz$QH2I}aZP#|615*XL{ie;`1z&u-lnN0dtDb~=sLOFb)l_YdsqJ9@!2%c`l z<7v@Nrx)O5Z$o#V|JigKKl9QLgw9DzW}KbJLXo}o#*)}`>FDdNeGnRnfzT-I_Zu!3 z`XN|;LVW6C#S#2{4&U}{C+rt8gn{!>%z^bA=rZ1GnB!m0Nv!1_ylc3dFFq3G1+!*nu=x%RRMoXuLPx`Zgr@?X^W zf=(SQLYR{jyB;;rFA|VmlaIR-$$(e?32t>9M-bAOns0Aebi4VRvPypdp=6Qzs1wb?64#&jri4Mc^M~qek zeR9PhecLe)8>Ujwga!e+e6r7gt$xd7{%JL!>(o+WiBt ziClWa>oSP7A7hdM@hcpiFw=+=xbX5)040j<_0c;lEV>$%ig=H4ST{f;Rz!6-Mad7cPuq%3gHDs94w5L&0?_=p(sc6`h!XpiAeH%;vYYL zR{^3Z=G=PZeig~VDqMO7_+xy_`y_`-!g#=Stygj`f1a{Ypo>FohTSiU+#($gc_|TJ5k{Prjl^2pS%Lwm!4aPd)O?bLA}b}hsbx;Ssd z+bM#vBzk`Bwt*0jVoor)g^A8D4G7S9EZjpUBM{uL_nmZ$n~N%bYf`M!1)H)`Xri>o z{^@`61^7dczQ=8IR-O!PT0Rk<(T9pVnQ-5vUk&J@^(prI1G90}YMHNIbvxsXK z9|aPJvP%_U@##g+>J7bc9GC6cb(hJd+Ol zuouPSs^c6LZQbaT>M*p!dpzhx!TEhNpDQy*AJ`<3oheNT7!*99E9?&H&{5VY4_qT( zCLq81M2}sNg1CGX28MSjHenp_>aGYef!EkI9sQb6oPfLsw_tp4@Ps{ug80Sty$Ki= z2hVh_k}tG{)PyNeWcMCoDDGq5*^}5zW%Z zf<&Opgg(+1QPaqr66w3gRg2{v4rCeM6VfO&qf=E&`~Xrm{m-{VyVr@+zFHbI$s0of z_0q+_M3czSQiAj)Y@RC|>P7xp>F@FC}<=BAKC zd%AUT!5c&&fBZd=%iBhv01G^R^%@8#DFw~bfv&yPb}4RJ)H{71iK_mzc)sn-PFOP; zj(>}`k1HUBw#RaXs;*U&kZqjIz)Ui$+1b{$sBy^LmICD@T-MrPm}<4F!JOM`8DaG7 zC6+`ZwJj7x{$%u1kO;+cE)ku_gBkRi6dTS~moO`Narg{S%#;xzS_R<*yh(h?XmI%o zz0RY4>fK|S3v$HDr_3XtWV>mIoJ8-!N|p)JA|W_A9)bH&~s$wb+ERY2w;V})1xnu>To_YU`@J`xG- zUz1aAk~fN+UntYz6v7x^vKB{x(6?@z4G6@|^PIpP)H}?Wm&TF%pWr4`s*7`NIrfHM zoFJYeB_Z7SMQh2UAg5ef#Yy;N1MvsqpR?`s#D>l@^3Mi>))95AvspS5imwCPR|JeF zg!nYQf#`z&#F}a>yQA5HWp`<7Qv6Y7)AupEKI>OmKv&>4bs1TI;V#pVZ~XUbES}&O z#WscT(^QIWAOxe#=nI-K638Nuie~g1DI}iIkyTO^`W}SG&+uyfyZeIy_#*Izu0ydV zvKakZc3Uvj{TTCZ;j`n9d?Wkr$pv`i$vIqQ+E3ebXzkI~#O?CEqP0L>SR#JPOrK%S zC_LZxMn{lJWM0$~(c*&TaK|QIDRQ&4=Qj06JkNN5Rn2QLYi9TRyT79(3gTOcg*+_3 zImy~uS5|?I3tU(DyE_FS$_XTT>JMuXXqR zBUot8$DGoT0=r>a`+%4#p1CC}4gn%V#$1Y=7t7f=>#({f%q%*Z9TGggP@EvI5LSFE z%4??r;dw~E;$oF~EN9t?KAeQ z8deQ`0Q@?k(tHOvpTYvCbDKi?nnQw+wU+hok+s-4A-hTMuMrm2A(_Z*5BjvGPKDsi zu+8~EQg#DTbYJs;CJ9Q9$dxPPx$+?lQDbv)u$|{r5u5q;N-!}B?t|W5U&Vw9nC-Oc zEAZzDK-O1yVI|htdAqFn>{}VO4e&U%;gKmfh-@&m&?1EhzK1#~s1iMmAvi%+T_WFr zc)==zsXP}T`UD30;*WUcXZs5CnWR0?G{W(G=%JIVl~6_JM$hA`>jAY_@6v&GtY;() zA;by-JW{d5KQxgP@dPN^gw{Dz;XGZ4YO(n?P;kl--~Ysj4Hx|*-l8WiE+S!bz;KBq zqyc1UL2mR!$8gAwxKS+gVSc#?2J6_KpL<^tGQ4*3^6K$Rx_9dIf<9ny-loH^ zRhQuzZuDA*_7x#!5~n_7t)2DhVGqwL=!({tTPE97*c!OKfqL}aWM%+e-l!3_=MQ{t zdcMnys|O4}5Eg#?nK)z{$)k%$^_h-}_JJ(%3w(vb1}PmF_q2F*onJiE!CDF&3W*4c z_!-jqUmr`lfxzM>%7%ki_f>4jo_*H+1D|XPghxb(yO{zIl^&~dw|?04DXK#&R@bU* zWT@%_-w28UhlE4-&v*7OqERJG1c@Tl3o^`g@ze`PKw#u9Odp%k2LEgpEbFNF)@Ag~ zN-{}PqVYhtV!tR;29aZj2!M)@ow7) zml636B88RL<-#FvSKuIqmH!!WO?O(F*ahXRkv1-N%uJ61DszA=lTHwkgxjI@@#9D2 ztTG#A8Hzb7H$4EUoFmJ-H3?4pra&jv>Rem32%{}ju})E@Q{>l;lV~2-YO>07I7+o* zPau=TkUV;U_;f2md!$n0)^X&e(kPtG*TVTS?j|}bq;uZqpmYA3igzaAFLPqx?fyyZ5)JJ3y!m$c^7do--&eQsO)C+TiSxJ8 z*zzs5&ByMy=dt_mZr`5WF3m%4_h#LTRyOBF4`K7OPRaW>H+p|ESwhHw189?6e!}|* z9LLMc^dJ46Y}m(1_e!E&N?lP9an_Oj{RBX%`aQk+om*LYbpqI`Lf)F^`s!g)F)y*K zS(ZDEojPW)@ow_`WW41h_&eMz(5G}Xq$Kz9m|%TG>QAL9@#i5_&jf|m8lC1q_&M*b zL-cc7^HjBXZ^xNkv7he$>jn5=@wsHOtfQ+ehWh5i-_46(W&5!SwRvo-&4LchlA^Pr zA2ID}1E~Dk$)>^=s>2rGP4%mr-~<}D%EwLsk!XUP{nDQ@*)LSeq?2oFmLt{E z_-;UO_XB*l7^~yrBBM`QbH0tsbyCEbG&Ms`z%7TwAyHcs^}Ck{7UbilZ|xov{dhN*X45hRbq@+V3BB#`tnC| z?w65x>N8;Hy&CYko-_)~?pzb#qchra5;9E-=R1Z2d%-`@oxz@@$FZ-#Z2S8D1&En; z$)o^{^&NefgL`Py?w6PWwJ!A{&J4w(cH4ni{3M*J^0qG&fX;A)+We)T4PDP+a?{yM zaadepPS}0S`}8@YiLR|6PqOhKYQJIsEvf}w0{HX~{2`e=?g7ZStqhxYTVC-iq=-av zL~Wg@y8BOg^)3e-aU-bMWVmYgyC<&ddv>M6>#s-qP37cDCo4MjKizDuF}=8g$rwN* zYY9;$#!9e^p@Vcy)&h?XVgB$_p9Yzq7y?61LI8x3LAX5W6y$*Mfa_!JG z`vxU`)2{yo^im7d#8r1{guxw@JM!S*=8LQW^pvGJSbIBQRG=KP>qvsr?@gPpvmwh!+;rC?jmJ~t<;zq9^ zQaK8wfE(VOm+nOOf~&h{IZvzVq>R$T&&lzLaik89vz@(DN8!wFqs`2phuQ{fZA>4- zDDOUuw}y5e28t^Z4H$1CfOE`-qLan(tZNA}k31t^^fZh_VvQqFY!0d!*ym!-|E#R> zbiuw`Zk5b$guU}y^nYsFyi+;LYQEP-+dW(P`tgYRH!kAMhK?*!-kNM$)3ln9=ghg; z7eq$y!Z(k!2R-72VIl4E!U3V6e{A5**OX zp=jGoRjunn$f&Gm|2A;fl=lT8m03|8w3W8fzp1_T?D-)Jm?a&vxeWT+DB~G8wH;-j zEB`;9zQd8K|NoyG88>l_Ls7V7#5GDq_6VB?vbokxNJddcR!dg*3SE0e zcDjhmwfElQ_tN|G`~Cx*d(LY-pO5t<)TMH|DTDYBIAi(4%bq| z5Y=JHHE24aZf(KFIJ8jnTz}A`<QC3{O~_IiCx|5wF(BDoyy6R@;kGyBDl;mUX{1 zFZ1?{{V7{~s)iABwA70N*|f-xD=udwhmaZ+SR@{05KtP)vj5QevUe6Q5RY8b^}V@r zCXft^Z#Fj$?=}DG2|^}@Nj@xghhjWITU&U;9Y(2aHlEdJ|(n6 z)L9wvm5bnCT+E$+)dV`CWX%1_RDyqOP_!)?I$6de^T3=qK7oqzyOLk*eKn76N92k@ z^CAXI==P5+7{Fm}25g0?ur#aVGEpx{^(FW`P$)clVr~*J3W29I2>J z{DSRUIkVN?|EFwy^pB^{7hXHZkuzLFNob}iGOV-S*S5!x`!pDhVA<(+dXXtk?MwCL z0)Hes!jPhF3y&`~>;O&5uVbP$7xR|UxUlO)Z@kzJgxMID+PS? z4(kq95JlcjWXZjDRxY0bdxMX+)#GEMhx-+4S+3N5D_0$x*$IUpR4CI2GbgxS9#3|pgg9}REm2Xob&C6(w@nwE(_c=QN`576`R05NH(Koa z23k=n?{Bjsp2(@yUG`^UunPKFjGW4nm%!C`74_XR;;_$ciOBx)utJ>3Gu`@X!TUC4q9v0P6VcNfoIMFQscS9be0FgO5lhY)jq1PFP$e!%kX=h7E$7CB zCr0F83nwX?v?b&nOOwm)Xz3qhWClz3#zPkGV-*$GwX2w;`#lToh*iy7{@abxB|L@n zIJJb^NO@_r5mL$zo{l0{Y6`vcvuk&HNmQ$(B=29|&rf+R0jd){qpzl&N2DHy`U>F3 zbsqUUW*o0xyf~xfxGc+Zj$d0(zuGLl7pI%3UPc6u> z@95X!Ji-27VDDstV{+5}=GbC&VnAT_C!uA`(7~lB0r<(9G!VH;UfNs;Tyw(NmLtdRcwIJpoJ*s0~X-<6+4N zLdQCaWO*XW{zEG0kHtIKmM=qC#0`p7`{|pd-e=lGrEG1_6SVJyTrf9x*SE2^WHg!} zNnHE^qb4RGcZk{-zEL~^?3JxEI-Z;uJS(W>7_^Xf31_75*5Z(hy04|78yQQ&o#U`{ zM7Xvt_BgPtLI&?@>L+?XSB1E-gQ1dfNpC@rxWdOv6^^w~Lg9(pE73K0qN#rIO)GAU zd)A8~ZU^sJBa3^Rb73d4!1Rrtw{=6Qe@ry`ay(j?p-2MDLh6jvSZd!YoUqJse z%-iSr+7h-i?VnGVqEx0-!w z?km9I>CgZ%J&GlNh!=7cT$h{~XG)S&_Qu}`dFT9XjMJgfOu5V9nv%GRBkhOOl;I`N z6uHw$17szws0Zco8{9bT99s4SQ?MLAVV4&JCFQ|}DL8+)VWxlmxeqrADW+WM>-dQ= z=DvM?a6xv+ENb*qV>l{nQql&cmSAM>QS;h1c z4`P1JrEUr&s{iOMC7z7Fo2VgV5vjf#&?M4{`UyYKVD>^$-|IXNsRG<{Nig=MNaA)b zFic8IUli1B3uYgdpIz`b%odMJd$ap`4I5Mdi=V!HeHx?mI7kb#l&W1UAdqwTj88Wz zA}i{9DW$s_`kOL@rt7tnG#GLmz1*CAy~9~*9rP=ko_JI1UYSPAra}qX4RJ>zuvO!t z>K5|@jT(FS%~CdJ1po(zB3`3T76!2jV$;OGwIFKkE)Bn*j;1JTjc> z*W)OPPc@v0tC6UqM~}KqB^k+2w0;VJCpptA^{j@#JsIvEAF_&?H^BI;v+y{}|JLz2 z(F7OCOTk6ca)*6X&bI`00_8E|f*#NPi9ik~-ejjln!VcwP!t-$4Hl-Nnf(+D>}yc< zPyY@o?BB_JcV)~gzEJaC|Gj$kpC<@nx4EOwT#&wb{_NvlL%*gKc(XG$R*PtvT*RQ< zf_ru1aekEDAq9!!A0CLUL_f;EQFnNj%aOgd#=r;+WemYLqK;m%)G&t58h$47lk{p7 z7$Wpa(Ap77Ay(czmxT$5JQ9*EE6o!9$tz@Ih@$I2_)hQnp``EYS8u6x=I7G8B(ynv zI?-s02hQ8O9+^uzx)xPPH$;pH;J*L(^VpM!oh3)+uuFBiHw zDCbn4>3-AXZc%Le)(juuXx?ft&JY zhkXW<7n96hm};l~ay{1u?WAuh-sS3tAGK?Db~NtrXw6Gi_F2-0O?2#6o!q5~= zy_T9pJmHzbJ9bm#hvsBbn|<|`AOkF-?Be{BlA9L#01{bS^VoMZWO)+)awVt5K_&C= zn#5;0`-S0z-ZKTayYzf9V7|(kx(M*#2Pf{NGgE6+Q?*UzHRK!xZ#3{WvPZD%fB*AF z=T3#@ksU}H-Hja&M9!RO6D5cx{QygNh3f^zF{mF(K59&vwB;2}=UpdclvZ`6@EqqAhd@q80{D*NhIHX#D$SXQlbawS#xAryYX93B88z*m#C&y_#Hz z6OVfKWvr(X*Z1KCK}^=wGAiqogkQ_NSoWwbc+9G!>u1~?1*Ecc3g}^LI*{Le5q} zy+~_Rx|HZPu?ov9n%S!HRwv3rpNiT@3EAoVn-%@l^~ay?zg%@tc0QP9I)jlR7-IPw zARYEQ7886J(?i12X?|wO4%UunHcAW?deFg%sS4Incv&@LL zxetWU{W}GoB|!dtipz^zI`Po~RKI_YblKL53M_Iik_lm#WUb(-9D?@+hjGGEoF&>i ze8$`$JUY@KJWk*$ck?jnWH*tQ8qDxww4w9Ii7MVUgn>crj~pjfH!ks^~qoyxTiNvcfERx;E=0 z&(o<5D~&!|IDSM@@0m{fP%&Ep1H6!*j&nGP_4?B*SD03clCO~R3*@!!zQ$iJ?0*&j zw+#6~Ml07mL%kF(BDG$?-r&ZHt)}>ym3%38Pkn5jxEo5M_a;{PAl5Q)OE6I_nKPot zKQIsk-lhxN)5$ia-Rynuw#Th~MOpK;^U=2^3idU$wKBp7f-Zqrl{&~y?$dos_-nQs zCPhcs=J&|AQ-F?!K<;^ZCI9yc%vkpJ>Kvu!A8p>IZ-KG5b8wKpzSV{{R=>BtwNnX})1eHpe)~iV28tL2y zB>``>Pyv)CBPKOy{xIs;@Ak7f2G>~WH_D}(7L+E%H!Gg;jprS29#mm%%Ll^x>(*rY$o*6?5SMU^1VhpkX@ z*K6>-VrpI%7(gHi>nCz`(C>^J$0OwgGZEaw!$>AqLkG@Fg+V+e%{MicSUkucNAy>P zrQ^@m`@gRfr>VWC(9c02%XeMTXy_CQm3sMH%TVP&0U?&0qp9)uGQ-)i1RxHeF>9{C zZJQpNcjJ~5_+~Ms4sa2ve)2%~NxSdS$bzVwAaBepokZ_HO{*zMh3&bi&|4aob6F3< zF#Pp=py{dx=)SZp74#(l&U%=u)K4C~`eSaYhcmZ;BkmJ@Fg#1rqaW?w8d?2Aifvl;vBy{_9?Xzp zkz#brSUs6F{mt^B=^Edl+UYC7taT9AJhVZT9gz@{u@ECnku2J#b|3>IKH~;x=FWc` z#n#^kF5RDLQ!s8*{$x)FTuH>)eW!74MFb7kWGvAIsHB8IFXP!y4X(Qnf4{Fr&dgLS zOlRUSpAU+S(BRj1RR6$x=CY!U#Z;EGKH`ZJIT@j`8fRjvqN0ULIswB z+O07lZT$V#m(B@^XDHOyZyI+}SFk~$I2!IIGqsyxfFQs@Hp&QcuvJvVt7A1#*<*WF zJ$KXY)5^+?u_@kH*iL{Hv+ZjpNEr)eZ~WAj zHPqNMT@9Av7CEvi!7%AOfPNa(Z50vo4&Yre33Y$`wvN5Q%klnMWw8Lu4<~}Ny-5K7 z=Pao<(D=aCwl$Ox8Li?yC{8C!e8_8nWtAwPBJUiho(la44-y66fj6$x9pV?Owkh?7 zuOtMWbucQS;6NTFKZZ{sNAlo}WG`+U!(B0Z1* zc@k@M?UaOwier$`0+02e5e25?E@DcdN-FV(8NM%IS&{GMZ?wDmd!EjVCju2ZiM>Q1 z2U=^{gA)X+EfxeHur+GJ(B#{g>#ws-a$qD|Q3U~DL#-4UlQR>Oco+WoN%eF+5N8MX z1vf(#CN(57_T@BY*fd{K)iFfr`A+U~@+nNJx}%_y83-SvzGR;YJ~|1nhX#opk@?u_ z3f$l5cmu~97k_|{a!PO)b4UyXe0)Qy{naDh&=7$V#;RXLzV9LmJgi|O2o8ZUqnXFf zcXCF_pdVY~Sx#ZLCayRll?n(6 zek(l4uUuF?EC9*+b^ER701hTpj`@TQ`XZ;jK1XmRE=c;h_FC5giBakiN7v^$8pH6P z8gjhG<|D8Y1&&e0S+B>UejuTK{oDa?S0#X1{pgJizvnQl2hZ;1YUp>4~=O(3({ z8EO9-_RRoI-&a}229Jr4&ECk4jaE>VfSdHN+IF9%pf5-s6MdOXG-P+6xK((N{BVym zv%uqfPYCs1Pvw@25%((B&qW^{>c+Pob#2Uh>t9c}CGP ztCgd#YRyB*>MJO_-PIeAa?*gZ^{e%8acwqYyFurTx0%B5`T2yrW$A9}0Yi%gzF6AF zBkSC?KeGQ$W$c37j-HWAGiK1Na|t$9c`mUOh~bjG`p)z5!#-gw?W(1tXy$3j|4OEF zgsRO`!V@T~shO5pSD**)b#pk@4~|OJPV)2T7m`fxpm|Plb`<>M zqVpEECg$?Iu7oNuRD-4k8-(#zFGq3=GC9>aP(Ad4^eK~pv90buH=imNm4HKQ2eKC} z`j#-M(PQ26Z*^R6fYxfPhmeWS+gy9IO}Vg$oWrlz8tHL|LY>c zx~Fxrnk>y$C;Ou~|70>M8(n!SJ-S%E2DC3B^}ixa*?0R>bo)r>hS3MG0JFa|U588&!~U)-g-RIaR;0wG$o|H;N2tEaAh-8~lI1 z#75Q88QxV>LEs0;@=T0&N0{pZJ{rqvTBY{IEM2YU`EYN};$NQ#v-(slmAmm0oIkCt ztpyhPpSkHi>$g5Ph*7KfLMq8UMFrwSIYLmW2q^F@zC%dLso={TLNm(yc6CT%YmTRnVYY7*`G zlZwim#oSjyYzgN`c$OnU4*MYNO__h~4zxP@wRM;|l*9H(K!Q$RH3Sp>{JhQc11r6m zUCB9wi=1lsMx=Zmqh!bUCEaIJ`zYP>h!XZ`63ba7gqH1SbbPLpHchzWq=0nzGO%us z8F_4^XP1fl*}X}3V3Ayn7TQs7noWv1eofcyX!7yjwlR6LC*CwbUPCl#PFp_1VZ4Oq zwQ-;)8CV%lbi$A%x*%XpZKe9mxEvdcu8L?&EyvrBuXKliy zw=Y`f0+)8tZ~dSqw9;y4;k&0Xd`eT2H_UgY?n$m}6&`)`zUWdp*hE@us955CaEF=6 z5)5B7?Gy>1eFG-q7KKKU= z=8L|}7;k>Pk$b&F`@hTJ7%e;2eFJ@jY}Ko9qU+La{*jl$OuD~r|DJD}?C|gsJ=4Wy z=T~4HHCd=>uD{k2>5gPSoF)b2ujquw_b8d)NBxc#*0Xs1SPS&iw`sYU4BaS3%8LeH zYl|!(;r0z>|9ekOWGjCS{*v2Q8`{0RI)1plD{;gA+E17+J1Wz=`{1s4{;vuK9fzpm z(?W!*L)kZ-lRAgch5|zQ_`^*{!xI=foj@P_Z3;s@uqXD~CO>D-+#x1U7pH4F3FwV# zUhdEM{`FIS^uzgEi&pO05^YZaxM`379D#}`6WFQwZKxq6wQn@6owOBlVghAK$?Usy z`=9#39<=(|!CnO{AGGeS>4`S_p^FC|zx z5e?n`;7JYkqiL@vP#5nc8GS$cf4DH`0W{gzBlGq0X_GqoU+dm|7n#=XYAL6AeeQ#} z`#n{cxjBUhG|})3tn2Y;Lrax96G{#9tp+mEm7J;I2N(EEU91rQ2D8S$lrwJm^(+%x z=%$UvlY;VDZlEakQt+t@-|%;d$Fv%vIsaXBY*_U=dLX|jMn=fkz=o*2fbUZ*9af05 z2?G7>-GFFdzcvq>7848(z|(5pedh7VS0`MI<4#M$t-dGL<~PfdzhsI1tKQa8p+&eF zE^RY0d)MYm!y#nY3$71fny`&JJmMA@hYSvGPKL)|j@rz6(|@3X77VC%k9L_o8$t?I#TcJ7I4osK-J|G%E zEoD`ZxzMj}!A82@ZnoxfBM-fj@$cQ9h;)H4t{VNiM%}TK(F9;UdEvN4^!AY!eq^J6 z@ETy$F`~g0^jUP30+%rt8lX1`UYq$)7^)|<$>YhXX_)z%Svuv@9GFI@HL->TXeF2}+=?-c0d>Q5 zsP7(l`EA$XajzWVf~#Ja1m`q+bIV+F`Ru-%(!qQD07x3f(Hu$A+^G&7J^p$a^>!%4 z)v&J;w>@F$Xc>hZm0$?o!{}CYY)qE3K?A5@C1;PTQ_o$=1lsImw1Dwmcd4j_D8MvTqg{;J2*?=_jF ztM;{pQ(=S zoV>K>Q7@mTZ9aUJoxAUO$ZMPA4&y<|p-fpnxFONugX#lW4r{z5;lpp`?PUl(VD;hnittw}lGckZohfL%m~6^y0#f7gf|zbx_5)iC8` zWqj)UYeTR%DZ*chz20j9tbTQM^;@y}`YKlNnMMVl+wFjZGKExRUt^l^wJOKg+ySG@ z3wg|}cTd?Nx=wMiceEo%-g1}@I;2|jpzzbhtE_Q~nfQ3H&B}u#dzXNum0LO|2 zbIpyAey2<7`6X>EpIcYF;AC=q$mP@hZoR=F=i{h~)QtPI%L{gJ%+fF)&JqE4E+4}$ zzVGqhyXdgh%Nl$U05HBH@M>_a8nQa(cD|1TOC83k_PqL{{g$q2_Z?1Z<8R8x0sYwb zO&R(zYYvfz7g8WYLlFmJRd6D;>B4U_0IvYK(z{4A7urUSYfx{2H@;yFHT=p$Im%}c zIMAh0iy^2`iBSJsPJinc5REi2=APH#-B93@F!J_aawPH0AIa@}WhGweig=yA=cU?n%W}vw zDB=cE>hkBGcRpv;CMtk;$l7}S^{gZ~XrYG@w^IN;^Gp=x)J{f6gBZ_4P;PAlE108U zb~%6_&)=YScW!t1RO zPkL2d&05S`5os&_jda&wKec{)_>VcHA{3MphibpiIyKzHig6A z3#`Q7E1k>4-*@fQOzOP!Nv$kpf049{+$rbf_@n}Z?}e|$?$ln`3Xe&U@}HuHOZiSw z;mcPdA!6+eQMzJj0}ia{0~C_EBnj0uPswQqHpu;|c~(=0fg>1#)6@F{1k}Oakc5cW zaz2My>S9}#wxu{1>rH@OoIXWGb!dl9XhoD8(rV_9$mexxZ7#e8R0AMy=Z^d&XWnVf zMx|ZC3eGVd<-4?6c(Ul;lSw``W^ZZ#QQoErDt{K7bny4O)V2zJIFwF3bR)NZrBrOB zY=uXv(bE87{nGkeF^$MuzqQVfi{8goFCjBa5nZPSu%R)D{&eZUrYu4$NpshX*`YBz zQ1bME>-#|Nt-mrklZ$@)+fT9I*Kb$xTpHc{d5NMcHu@m+M672wchIvzn3X_ldZ<8% zOc;~gz0zwuIB66ZKGAW}}J1%QC46-oRBs z5#s)gOtEys;rum?x?VFbuzOh6KQ6Tw<4EOBEzs7$${-5!c7GEUQ(4MNBamalH;his z$iIB|;>OR;sPL7mb0$t(Fo24KsGeELB6Up{%Ne&Ki?)T#Fnbg13u548?7T)#?SBQs zb?gBYtg!f%viET-VtDe6d)Cx*GE9X=M#@J=I9Uubs&6-2YW=#zz;?u?{_Ah#2LbyG z9{k6&Y}T7xKpB{-eQAi6U*|G#Vl2Po|I1iSFI>Y-G%;jBY2@{~e1^r!gKE*?)!~pjD>J6)x1tNQ&=Hvi z+sz7>S17@!v>ntU{JzE#tSWRz3F$(wcFgMR0%@PLyFhBsIxCe}8eD}FkfgCO+p-IW zXwEX;p()uKAtFromNsT-FlO`ttY(_i`sZo&G71ir=tmrB(-!@?N9q$BDVjF4)8A?`Ul6&=6ZW_BFBjgLB+6>GGk5#$8 z1w6U(Ea(z?L+%&Et?KA1%v2rt=)|f}#w92QL1L?;m*G|nBo(kPLz_cU{6Uh!Sl88` zG!jckx`oPtYL_Js#$Lc3iekGv*H0gT&XM4+=z?)G%H&rtblYzJap!#jy#$s`drPn;)*KEzjY`pMxJU=!9OBsRb_E!MIiF+F+`{DofbfnE!3= zb&3WK{vhgf=}emr#^<`k8nl&9i;9UN5Oo|R0 zR6?%~SL3R}m>p+Qqu;B&RK=_lffArCD7~JJ$1j#$FMwVUs@@|)x#fA0`)qf8d0prQ z@Uv{D0u^k{S{Ty`oPg8@4S|MAoi)I2ZwAEOeG^cyKH}rpk8b>IxGUoHReP#oE71LP zfgV?O4q&76+?j{kFN)W=8Xmd2Zo>!K#KSwOzAxwb%Y7?gA+BV3Hy9*}6-;&OKV|p# zInTz=v}*W`pAd)BGN-+nZ%B@Yl=qKtB8~mB-rf-xS~68ttq(?fbQ*wZf`|QP`U%jJ zIWhw~ri;GD62vI!9Jd3cjkCxvM^~x@JlOZU*IW=nN(RtFWlk za&S5dIJXJl0~A3qs#3WcQVjO43THj<`ELs?34-}m^f$SaPoo`ahhZc2<{|{cVE|S$ zo9``!hf3U6q3Z-MGYGZk4Ob8i?toR-C+4R4?*uSSP#>3nIO2L!>cXJ?_w4QBoa;kx z(J9ihw-j#i89(u;FjfG0J13?DOCi9b$OzTABPT|GkZU+K8%&f6aZAp!A1OQ7&`8F; zwM-^!b@K=Xgb{@b>zd;Hk?N*Db^=({p*%BRyf^;MNCi$o)UA7C{vOaPm&$bFk~si` z>il&UBj7}P0K*74WK3wpmKH1n3%O|>eJ4Z%UK#kcdXY-c&kN?c2p+CH=DTKJBH%U9 z@k8wZyIF)dLP0Rt9T|#Rk~oV62))Zqy2AQI)KkLV`Qr{4BI*4l8&_r@yvjiSamTfj zva+yKc~5zz)D2jbmDKKZ2-&Q*0F<@b`pM1bcPnrz{NLtVq$~dvu~aE51#PWGHnOIv z>D~{CGO*;meB)vi5Clbg9;*aX-ejX|4XQWXH=TIiRb|&JtUr+3Hj5{2?Hd^ceU5P3 zeK(8IK96X|0Nwv8RHG}*j&*IxmN^q#hcg~uqoN#lAR7^@_ZS-2pbprihT&+9F_Ccu zl~o38!}esV>uWyRA+#_PDuN&osD56kKNV<@0|_)ZKr0IqwhEEJ`Vd;8aN`$@!xBH? zga7iby6GwTA!f7hPXNAQULG$qW=LULe0}G#HASSsl)=pPcpN3)EaYX;%mVSUQ8X0=;L%2ar&8Y05CujpaLTTu#!47#;Cr>gj+$8++7TrB zJDJEwL6IUF9#rH9QvJmO<-rG|C=$u89Zw@gaQ^$fH?iitU1PN2WVFL)b{`B^1^s10 zI_n3InI8UGjCaiVwSh;=P&7EtOYIa9tC4p9b=PlqppOwK4-R9G6nYhWbMfIDMR5OI zfo>+n&uKV+yC!CqCyoGKvQ<{h4s_{V2BSjT4ikdgMl`q|%EOMdfRqR@klh;uQkSxk zYhs8lvxh_9;TX9O*pXpyB+_UW1>__?HJPnnbNn(89avIZ_*uVSq|d5c_Z<9o1r9`^ z;^5V0+P=q+PlbNv@{Xk4F;Mq-ek)PZqf_{>XY4lu+8hq8924cgn6=M}1@%wYoU_=F zbr|vxUIYvLI8`qhXE5?T0MWqZ2EdJyy5D0nc8T7{1~Fs@cn5Jy^t~%W4P&P6nRl@t zlq@bS;-!aY=K?A}RCTbj9**#VpMwx&WEn72Myau9;Tk}=w!I#OI{Wmk|4xFQ8uzYT zJLNAUAN)-twEk^ORr^S{Y@9^aMzALm{VM(~|DgZ^*k+nx2$ipe{~*aFCL6FL*3SQf zB)j5?Q?;FTI^~Z?F2Qp&=-z*n_y*<(dn(4k!6cdEQo2J_Rqav0Sb6Jjk5_c|jC;&r z7>CfM#Hv7KSqN~%y*xKr%c(!M9o~t`U2Bb8*2t>Gjwi=I&;T-mSYTVkOu1%eN9fQ= z>84S}RPlcf{Y_Qnn#DU1_oL%dbAVAN7P{uyNb{$V>+(`f>ls;9(jrsJbLX#^z_h(wCjjYz)6|m1ek)#UXo06+cUGPR; ztV}(LRSz<3*4H>|22eXzxC^bUpNRx z%s2SWMX;8|96|DVEYleBXrw(bKl%Z?nua1`S`p5j_y$-z)xV_jFsv&Qyphg+c=NHL znyVC;*s&B+eFC_7Gnb&U#ZR6qP+r@6#iJTu0u8#25Ae6GbsRGs?wdARne)E)*niD3 zWC&2;w`rfVut9mSpBi=NIWw`#zXEo`16=n09+?kX%;}=+g)}mFUP#(kvR@E5y3Q`tf%QR<9|VA^&|JPznYrj|R?@MWdzh&9XcvQ|R@65exSJ@QHsT^kLidhO_N*oB# zxVPpyo5(*H9FX@TG$3YB!{|9P0V6fd_UYZp_%3twNwyqruUfZFF3LBnO|Pa8n7Wz1 zzeY=^R*mfXzwatq{f_UClL*-9mf#PqWcX6SzAtm_vxuphO%=DmSb^@3x|Fvv-aq!~ zO47F~S#RZZ3eT$S$+PQU*gtWw@0iVBGs7C-y*t0eFx@t=SXZeS;4S;AP7l!-^jQz1hZyri0IP9q? z!#;Ec4eedEh{wl7sUYZq&lq8#QL(c0ITw~O1km4%blj7J&hqE?y!QPSMyaSOpTwV) zfZm?lX0v-5ABc)_U(c$%4LF33qnKuC(Pb#G%l2j+P|ZX{1h8$iFNkPeDt+NRB{oOV zhwEt@FaIeH6?^-XGq&F1?QRQ5o!a8Ag6?~*CslS(JhfHPjzKPOs0U0QL2sR>t!tR0 zj|oSpzt*KT-+U=B?8#-r=zZE{!nex5}1bnaG$$CmqD8vZS#F}i7f1nwExoi*KS zFCterjIM;b!5Crb^S+x#N!j6?lt2|cHMnBY`INO19q-aXM?H5f|Cy?jGeR;9r7yS7 z`?*|ZHw@p3*b1akrtwr-0rs$0$Is$8;=u7$ z38W_p*Vwq3FVQLH(9P{b>4Ift+{$C6RrUT zO0bz5f{^g~dg`K;XY$LBp`K2%@W~mVk32k!&-Bj(+xk z$lXYRI!*yp<%}L9@34pjV!<|%yxpHM-X0bvk~H*W{={nz*BxT4J8}sbnj}!x1`{$) zvCv;d<^3%~+6P%gH70~1!zpPy_RF7%w^_fhcRsR6r~iPIIv(I({A3=1T^{C03)t_U zG!dhM%q<#o-{pQ|Kg19DwO(qEXt|TL63B_3pitx6Sq%c7Txu|Fq?z4ha&cXjW;Nh{ z>rW@DDbOjgki>ADU*0#(yK{OURd;>iY#b2XrrQ!lD!waGB`6ff|@Vw>&}-8fuT*XE>GT) zQU}FzLBOit&Z&HB;>sDea@!!yC%UZfUX;``23uT?9KcyzPM7SM{5-u5`F3Ym_<@}h zx!&Cl$F+NL*b1ILH?j>mfQUAUz|ztcopH^hKJY;zDfL95VTHo(mZ-+cQkGyzV7=dO z3QO87EygT>UlL(<|GwHzu{S(&JbOb zKx;a-;&v=f>svX-h2L&?`m6!ts5OkLnA527#!tT8w%0%FTa~$juU!)H*dfYTgvvkRS>FVaL4|TxY<;|`yr@}?1(qG`v z^+X0d`DjxmduLH+#6=A2N)b*SyeivEYbG{CMK2MZc`7x`g7_Ijc4}))rzj5OOpxQM z987ev#!GoDjs5eyumRy2eU-cERL*S~=}b^Q?P0uEp&f^{v7qP%;1sctPGtfN-VOxsa?VM9+lVDRy;-s{J?bw0q$L%fv+8OPwif0r%Bm zNvY};xQXE!aCiD_X8nDV=Et0hIwQYya{qOWMvJ*i&a(J^YK_e(YUpMun)3mt|9e;N zt?SJ3(L^;z*D0yg9j=OcKW^)1>jp9h(ieuP?cJtD&rwTyzYUcTxo<$7t)@dq5Ic?% z*^7)o0J}{1KC12g9nIfa+JN4^X?9MCIlw4p*De^21h|joPG)=^Zd=2NJBE{;JDCQo z;ZkGkBZwkQ3H?&=NufNmM)|cY+fS!$q&Nv`yjb;Pr?pOVM!-4DLHV1)nGya;eF)uM z2Ew_iKDDC9CpDHLIi?$7tw&NtZGxU<038&V_o&Rqlm1Zs8haS1WI>7nIZ_6jaoX=c z8PAS2ocN|m!N!BuT!YB5Ohv`k;pooSDOyO3PCd;2_9hTj;RA1x7VV4F{#<<28=m2~ z{OYmnUEZB9M%9j(o39YZRSnk??(3Ky|o8XNE$F8-*ADSbS-n~KuK zoriHZVI=*Qr?U^%rbp_}-P?=Bp>*Om@kwc5oS({fy18oiZj^fl7mLBxU4 zM;AZ%sqFYJ5Xg384K>Hiok+rgfxo%35B0MEJD2KgT-F7wqtR0zWWFKj(&W(<0uML1$ zbo1NOD)Cz9d92OJ=*^aG*}auBiiqXfmK35(e`Kt+{<03I^p%~jXS4xmN2&}A*$q%R zrehbAInj4&G3|;yq6#DChsVre)o~!MY0A~`f+8Zr#kJlbaP6Xh&?3p);NG%0EaZnO zlZ$%%8Gb=R;O1$ddmTZkH6HVUTTd0W?9(l4?61|c4}~s^-^Z&BWg6do4C>nEf0$d1 zDt|-G8c4HQLwUe=Q5czHXZwDu{=F=B4?UqVAt8%{UEez|<_%9a(tv}Es0jV$67}_G zArEIT5(>FCE(ZK$>9nbbo-yg%%87jRdP-z@&a16oahr=4{5KW0x&N%1Jmw z#qypFDAeN4S==&Ij@4px)z)QH-5#6t9{mDNW{)i$-(7$W(nuLntRi3u8Ao>oC9|FT z8>fNn`g)WU{K>4RA6Rsj-9#CR8|Zvaj-blAzA%`SyY)T1@~Gqg=K?HP7}uFzRTG@l z6m|s#sR5sH%VB)q++NNx@n`3eORO8nyoW*4$3g1W{ZsrRx&z9`G$Tr6)=@Ha>C^MK zf69H>s!x%uMDH7+?YZC#TpzJp`Er^lM9bOl;&7YyMYvs zCmfrUEsed=v4%o!A1*ih z3VMfh@dGI8oBXp5tA#)BOgKuaOftMrT=s!Ggq-!a5wA=o{mVr2bn800E@~qRlUC;$ zi?*24qEed&4fg3KZDvjqWT5?Pq#bazcG)GFZ^+9}gtqf9t0ba*;G?ksegN>%4v~h6 zJh21S%Yj^u@#d?Ivw7GIxx_h7B&gH@v=V5cZ(JJ45b0YlKYxg1GD}FNm$7`h?g!6s z0+Ty(SUy}Cq29|x4Og>6rawkMgGekpd=78+qsw&MFpRuCoYys|bvOYvjnSAyh3=QA zh$8I-KPwgk`Aet_)yrZvQi{eXE7`_fazsuwr-p6(U=378W};h0T~LVB%eA z98WabC!-G_#k9YuW6jy`L3;6%#$P}Pp;}be5^Ik?*cywsYdUGM--M3#l;aE{3h?#k z4+YI^-jZ=r_m8qXY!QcFxL?x$;VXPhGxkp48fN+3jE9h>BK3rQ)Rf+25y!XK*ta0NQu^Ce1#Vgi4z|^` z%G9L??gNn?S16hCMz6m0qHg+s4?8sAu(;E1zQkDsW>BF%=sF=Ta?S2JDhN){ayb@2?j(XJefET(`Kr8DFYd-sTYv z;|^EheI0s6+^p)$$8gnM$Eo7qK)~K}K18GK$d|21u!A3^Nu)D3>_XU}6eDJ{Ezp-A z3cv)t@s0czVIL%uXQ1-gc=?|;z85##15a4(Z#vgO;T{LjreG;3#GOEL=6%5{34MRv zoS#_X^F5}StV<+*&+lk%(z=3*#c(_;5&_a#Fa1k<8b(DNhHR~5;Np=tDK*#37+a-8 zPjrqGVcfyr7`usvv|;=rLue3+bH%TGRT6#|bWt*#g(YB;XJ8D70QPuJWc*EZ=}RZP zAp-f&D#GSJtdm5}Os_XR;ITWVse}-g0u)NPg}!1*N5a7J<1NH7Vs?yYa9^s&hEOa$ ziGf&qAlfo)!fX(<);H2f=jz&0S<|~4<7a#$T;u@#1Csl`@;VvPq695oX-GO98(M)^@)|y``MxMyUcFSn-6X6Rt zUe89LcH|DAsUd1U{tTgDP6MzCYyOX(5w$BoF(JMUBsD(67yRJMH1!{_L3NsJ*bLlM z+&(IE3Nsvk2DuKP*K~|NhdCHOV&HXM2oVb5Y94JeI>v!0Rl3bj*o?QJ2sreC?O4Iz z5Wy)owF$MJyc#J=eFnsApj`D_s1rwwK1CP4srZ|hB)9|(5cx%T*=4no&W_iPp_g(n zq(J3(BezRZiJ>ZrrJrRIv+!w{ML+)&7b!4X&+q_#@9m5L{1rAh*C2v&nK9Q-*dA{# zzhm&7RWWRsn9!|H=f`e@tV|uFz~(gQu&#XxwoN;Sy1%}&pFY~~TnPXKU6dXL5Vf=B zo=DU-af!fjEs>-tOMm#=so4Q|_ZiwK6AJ7~R9<7Gw;J8txga->_#J3$kUE6%TSLj& zTLJ{w9v)rVltKc=>j`ThGdg6pz!_`brayK!nX}W;Fs_tMYa5_n!Cr>mZW1r zMDM8ddOQB_8~s;X83@ZLK7C=o+TVAl&xyz5DHK%!SS`mc(jA)I2Q;kxHds-PN z)WB^cmg~`nW7_935eOCF?HM%iN6)g~Wps2^r8#YV%llgah@-4XuImS%_LF)5B#=^{ z8k;Rex++MxX3X%r?0VWYkQVtTT|lyjD944U&{SDNk)?D0##uR9CSCO;`uZH=X9Z7? z$3LV>ZBS8Wp2+7svr$C0i`xbxTL%~q;fOe1+4o6;D^QEPOhC!s#Q7K}%5e_9Fj=M! zYGR6is5?@IcDdO)*0bWMz>x^JoZ8@Vp>NA^GPRYP+nQuk_C?j~)_mhW%Ox??94so`F|n_j(nxA-g-c#pSlWuK%~Jy`ccCPU{o%8Y zFCltY+U#;4JR3;*4SQ5oi6fFUUr-~5jg0d}x9>zDFkrq`sv#XUMg+Ps68y|?ARlWf zBo$Z!VsP;f)=>+7Wd9|m}Kdi?nR1{C6s?hnp$;I#7v^AC^BUhI}jDFqp{ZO`8wgdq!pOtI8*uvTB z+Jf>u@%>NCqafduniRg9m)9Qom24jN2mhnGejICgr&5>-lYX??a%~5aQj7y1p6#A5 zcf1V84Nqj?u|ZFy`OI&>7V|Qftd`>q&4&{3jQr3C$Qnc8(+7|%AaFpIj#Se(oI^;| zK8mjd*Y$6y)3j*+m1Y$Mh~C=9#zfoEUrC2ZI|yzYc3-wLF&;vn=h)5xVjhu$nK$wg zNh)YN)stAkfct}mbd6#8B96;EhNV{uwlQ6$-puaGN3!U7T9h| zgB724HOVv$Q6l4mS?*@`HwW?43C}40$SVn*sZF~wyep8}+iS)XJ@XpId_4mTwC52; z96{%Pz~STU_aGT|b^QOM>CNMz`rrS5L}X+M#}Z-ea%`on#h59TH53lnQWT9f`@V#c zC2M7mtS=!MG}dfI_9SWSd)Z~z@9F*h{QmBC>*jFI^Lbs5>v7#5OF`1&+7foo+cN=| zX@93$A8r7@v(S9FkJlz!!Us;2CZZHMM{tAxoSD&C7kSnQo7Z*1>fbQgHw2$41 zGgJ|NPBb>&Yn5g^d|DALFIKmeWhv#fnOXbqh}J`J&zPZ%mC`B5H!?X|Nu5BDG7UV; zM6PXn#5Ub{2KXDJ(hs%vpbQeuV-t?5%L~HRE&NW3WJ1(92mgccl>8bZI5S@j6PK%7 ziO-KtoHsNVHzkBTqB(!TQIE#3z5T{(o0bLUi*@J%sZ8?OsKS&6!9!qCz>?98L_8$c^c9|W*Tme%i?1|9dkN`F5ywfHdi(m3;ET6SGKTUEVTE-=nO&gU54NX?HP zo+LLIHj209?d*Nr8q&R&uppyz>JuNsIcD8Zk-NP!Hxkw~?h zU1^#X|ASOejQ#LLiB{T)F-myO3}Q+Ue?@)2Q&wT-zW@tgh}E<)Di{D6sd)N7K(tAg z%BmJIiY#0dMhm9%EjUpx0*GFtDC_DBJmZ}SK+QO6QA5$$xjA#u7$)^s&VQcVZ|6*u z8P*J+?b&LF+>3kff@F?a_{c_4e$_2m`oyDqn5%{u}2=sl>RjQ%Mw|{!Z z#g0avg4vjO0TN+p+aaqJ$#=KsM7LO7cthzGDyLOw_ixSCkWtsE5$?9!wqqq%hW7wE z=l91S%Ml3QT)BG68@ikG^>Z_yydUA}q)8$pYZX5^hjelGhaT78;v4Wzv9ocv`G=DJZ|l3 zs+XvwG{Ek|;E8NHE$1lX5_1G}-v9X4XQz5D?dr50W!op04#lcZ-c8Gt#&N2TE)FWvuKWa93qCd7Fz{9aKpagOh5uK&Mo6=_5k*n3NjTwdQ8nIV&Td_bj0j0D1Gicvix`^G2%0_)52Q5 z23>7D#Uf3{c+PARMA7^ocdm5+Q|*0m%n`i0EBDKVzJUl;${Zpc6a`NM)wwy+r{UUn zc*FFECk~Ash1SV4VIa)Y9pZ|1ta(S4lOrZzdRrjs_cA<$w(9|WNvxy6zh(Q3t zr8Mh1H+r`qGCZKlk`7G$UW@%@5VQWu62SWIF#Kz#rBy*o65RCEm7dZ9OrH z0C`f=-`@Gj`j2~j?NV!3ih_%ZWm@D3@HAXq7Dfx^l8QTZVba$QuisTuVcFP*USc)h%W!`J%OTMA!r0N?-{EIvZx5^n5Mb}pu$Hk&!&-tG#q_+?B zUi$A{o$x>+X#Cco>8+2OH6Zxm9|TNxB-%}(XnMwjw*0%8T5B)}fEG3uRJQ{=@#~%9 zlUd|wnfq2mOQ}0ne)BT(4|s;IWJ)-&gc0p-j5TKxhc}y{2LXRJZ@vJ>*0Bj(-5{z1 zQOL%*l4$e;|64=?rRLz&-_CllB_QRV!3Q-4zW~DBXUzVS)24LO%Y}|XS54!Nue3gQ zKM;p;N%=5U_^&r$6~7LzKE-)6_YJjKaWRiuiM;I&RpP_5zm0xmbb6`O**gemxDHEQ zCxv-ww4%W*dPP#m_0a2hA!SwPfaYjA;7`o;ya3d5;&2RgV*r?}HjEdp_^d%r{-5Ns9TVXrO?lxkOnjdUkI>FqY1-`=bE!qF>6GM`kkw;Jctr<(G=9 z+S!=%%m=oEpH&V&rzXHM0$JO#cAFEAna{0~SYBa~hqFc!r?oa@J98AwH<2 z)uzD7${GbK-qd=UM>mxZ5MR?`KP*5kq(WBL>+TfmcbxO=F7*vQGwE?tVr113J-X^h zjA!E7xK*Gv@IIV!NI<;KKH?NzW=xxU-9`|gKU%%kvhY9Ec+}~L@GRSg27rC1-#us> z)JK&xq(wV=OYoh_RfA8N--K=MWDMGqCsm#YYulhol!Y!=ah5+B0Wf~MiBCxNAFLKR z_-!OFiDtL#fM%z={{ZPt>)`$A{5D6@6JP%sQ5%5Lcr|}&^d&5VLNoN zTv5z+-+{@`88}xnMh}H@&AF>6G=o;D^0B~l#LThIa)K^^Y5qdrvvU_v2wil%7}@$Y z2VTHLV+Ojr3UTUEzu3C;b(hiags*6)taX&TJdFAj5|Hj_0C#bMm`cQsEYdq z41%C>O)=!8PuFP|z`EGrFa@?wE!=*)pE+6(P)h2|ut(~=Jr_OHe{&4AESuOeCH zc)Xv%E-}JdbTSJrQ7D*~IeF@M{d=Q)#96wg+nfAAluGvwwQ{`qM-RpFFwYjHFNmA@ z7**V{3=w#&l84Pjr4z*J@2^A8)>?3$w2z*mCuyNI=;oMp zJL6`}=|8LQln2x3x}rK1SqhIef4Xw1GkvTY3c}U{#K6mZjrgl3Y=AHX72uur7EgT4U1AmqEuIH>XtV2P2kn+=m^TF&gYV1#UfV4=>(Q+&| zP;q=@6zroIdizwF#dpPgcRb%#VY=t|UsBn;cz*sgzSdfC6+Kq5>e^~H>bH~fsy>VZ zzs8-$w1ELhAw2xL<AY!k7*f5eaL$)$a-NbqR&^E&47{`0@db*8w2Z5@TK~JNVMd7`S->%U@ZZ+Fkh#zD zf1f-?*G3HtIz2g7#)1TfR$`Zm4$*DqFWz6%(w+t0k>74TB)zDVCqAbhFp@O%GY_)eri;@g?5^6u@kYJttn zhlK}Mb`R_K2Q*~VF@4xjsk!OF{XJUw&gHYs40#%vqaG8j0T_v&+cTmIvD`g$6a+iBW`H%7_NU2=W#u`gk-s*fJddT_Qy1}eO0{5 z;!h*RU!>M$+9V9?5^_Jw;IaqmI27XAXlP-l_$)Tjo|pIi0g5iWTV*EOrYouGo;t{$)ZaAAyjxk7Ak2_l>2H{wJVk9wc)ItKlRWU+93WHegaIrJP zlXzkK7I-;xnTFJoZ#^SRSia%Rw0nwn)AZv45U_xw94%!A8!N^UzVD*aDT~lmA=_>+ zQ1tzc<c;7C9y0lOFnxGQZ!>`%{?vBIOY}dcj5C+R04Lvrz#(bM$qF_+6D$d>2>R5~i zci`dT`XF(22F@+EzExx1#bR-oXeq^fH=9~bbUjA73iu|bo{tHVu*wsU*B|wMXzbyr zrghX)-Ga}NDQn1M;OL9;wmhuDoHx|$K<4eYtoR*eCnyJWLKS{qrK(>evI1-Mxdz~v zRbY(&T&K4yQ{I`#S0!fTuLU{Ne(`psz#RR`6n5;>QH}?RqJ2@y^s(QHFVuvSTCYTj zVn+G8pMxCp`OS|6e?=DazyMU90DRygabNwGZbKTqSyndP5#oxznR!YCr#996iuFer z6ZBSP0Vzw5IE;J%blPXGOB6A5A~#bxL#>=OPcOS^O~H@;_I;Wk7`9t@-K=DjRGMo; z;0lFiOdg(g`WSGdj9(__QY<|E!F;$G_#eS*HVt>kSX|EZX}gB_8vKY1eDblENb$Z# z`AS8i{;f8Ak3dD^W=exf$Wbx2$k$D<99rFaS3Re=b!Ie()Sj5vd$bx;*gu*F!U<3K z4oH{Zho97AWQ21@Z}0Yso-tAQjkS~7$@mmL3Eus1=y7N>y55LozqI`ID z<2t4G80MYYW#Vt0hMiQwtAPcq9kb0?r zuW3lJQ)haDH{c~AG}63@x7zJKLuyTQBA*`PtNZtvz4<7^6Q!~m>9!QoP@?wiFr&4C z7EgJt;28%8M}BWP(@12NMy9S+`{aIX;|LC<#O@*?C|8Yy2G2jSDOcAcz(v8QTbB1~ zbK3R0`PFyQaATsLR{?(0zjpoYowa90vUV5(`rP}*Ms9V2sS!!*`dVtCh=s78UPiL?$Zp?ybahD> zJiCy{{7_@lz@$&4iQR)YZv>M*I@}CWuN0?U4maUZQW#f#yqh9Bp0c_b;LU1=cjtLM z4%&&O^;7yg%$1G16Mrs0`PXUT6l~nhc#BMBk2vX;%wnIa{5xRecl~N%%vrRyJ9om1 zka;lu`FyG83pEB6%o|9>yQHHRshac5S7Z&m!V%;!(7Y;y>SAKaP9P z8mV5KMk>1yAy9ahS1ZjgKKndOsf(zvv(%=IDL?CpMaFWJo%LET3i?hZYHoU>mlXe(U!>%iqS88#`G@xdV{MJs!#TAvbFnXf)ZI1hCpMzgTxe{-Rq*E9Qsch8|vad~RJrSO=7x!(fwhH2_3cS>)P;gNg zGmaJqm7R*$B4QlJn;|7HPNkddU3pP;p#Wrd(BB1D$9HefZRs>!hw}PLazp@1R&0tW ztN5tzFX~0udNvYst3Ty^h2A) zod#pgpLDP+9F!!QKvbDpYi=1Z5K&kBFOgRy!grNZB3;~EhabPDD@+L=c%hb3*sTfY z^$s}&qNba^!l-cRakHP7sJ1(2>84jcNOm`FBQhptu~xX}Cd zN#ZB)qEw$uUmPHTIeNGMr(5ey1iL8)fhM0tv~oVYK5P|;891{&i18nA%AAZfo%2Mk zhAxH8+A{F%YlwbJv%9ihv=mzGH{uceVes_Nv|c_wPmeHbaou(Ze;VwU zG^;InV11Rdg8g#?NQa^zV-A)d34*~f%{cu5b^;bFdC`Uh)|+w9ynAZF`2cXBh7#F7OvZEo&>(c#{o!3yAxnF+o8^Vyw-h1wo` z!Oj2oR}ECgkSEQ+f*iG@Kt48(!La;GMI)x79_7nke1m7BNrpn7198QKD)O>f0Lu-cKX30(G_I@EDAE$TS(rShW6t|0Qq?QH=o;x@S3|Yejn!m@VP|9q&{NndW$XzBYuGAvF~b#4^y-6K zoK>9C=|u0l!9zg*rC5z1G7m(ZnuKK-7qw*u?5-GfP>+b5#c>YK-H1P>{}Dtk70OKv zum=%`Pe(zC>$782#>LcI>&th-fbe?|`MRtO=Um00&}5vg<)*a`ov-%!b_d$Us2N={ zY!zd7XdgUjh2nyRTCP#QZ)rw6LA)xrXcy$Ocsx{JQ7EVl0ITqi6Nw-$_KzqOgxY() zRkq_r^At@d6;UHIm34$28CUi~9hlH8G+fb)58_pkjGZFvmk#dBt{y$AIIj_>jBxY5?QgMD*|`}H?jxJhK|F>iMMRKi|_CKd%x{bZj@#}B12DYEc~j4 z(@Bf&)NH$s^$X^XIOa5dyP1c)YPoju0{SiLYU>4Tr+N6PBC7If9Lo9>9Di6j;DU8t zEkq`4ZYp1~?Dq`xYJbI4NBf57ewY6bX;H+!=lq`^eVs$?TD7iK@Am!u7Yj{49NqQ$ zR^**w`_Aa`e?_KS+lJ=uzrOqZlAIAzpBspC+53>T!SOHan@-Vwt`RY#Lf%ffXDv>=fYZ@|?r?n7@rLDi%P6_%IwI0kCOkh_^sEi=5CWi`x6lR=LcA-0hr3aq%|K&(-R+gt3{qIVEdm-Ss+* z@=jqhiz8!h2VZN{PY9;3DGo-8Wn-;-5w z&f)45<4Y`5r^hVEFFJGkkE$ma7z|I+xpLh8C{R!DQ+fk0Aggr+?>xF+7?68Ga53wJ zwCL3@o5Z-vFe%Y9llX}Nqysc_H4hhN2M~UQG2W^biRdW1#=*mJhlIPwJILIh?02Gb z0v<^uqnAmo)^Fj#=LyrW(|5KDb-39g27d2$W6AKC=R6S)6#q(v$_wryYF}@dM_7eX z!soE6@w6UP{B(>T#A8bNoR8mj8m{?dpKGJ6SE|9cHq zJ97n_!V3BYx1fxeLL)k{+O2v{x8^>6CzZ;49!*#2O&ZcDM~navr7X={043u+S# z>pw-psVz@~oA5GWw)#>nJ!t%sf=}$id*jPWei|$<( zu2Oj6b(L_d;Ud-dvDwquW<*lXGIo=w*PCNj7v`ZO6ar{I<9CX~Ral%8@>s7PUjz-Zd~AX7U33u_@0USAKdZ_hhO<&Bz-%xdi%qr%LVID5|U z_I}g4b&{ z>UW?8We7x#>ci$cC6hTXL?UdSDLqhuSQ=)EjkYsj?`Y%cKO3;%rDN0s=t zMfe_Z3w!YEhc)WoU%L9vcA987)k?;TP~1Yl^C3-3F3z^q2qfa*6R4}Q-mlodYKk21 zp(StN?-mlpz)vg7lE(e-Nw{v5?|1gaPW#zNNtvF=-|4M8!8#FtXttoOakz>V-|2A| z`?EB`^y!s-ubWTvh%$pvh6j|ngOxN6!@de7QjmtRE(8iZB}lSIF5bb%*Ql~6L?j$0 zt2F#bVJ!&$ci@eZ2r057{2GT<9P~jlq~8r1PVqOd&*8jKg)?maZfpAu!!Js)i@boY>DT# zV3E>h@Gk{X?}xaAPI)gBm*8K!QTk0tFMY;@Zm%b3U*7=xEYz>QMTpqTV%3D`(b#3r zT#QyEy5yu?)t~-#=}xNSJ*hJ|g;8_^i(Q6-njAY?`K5xB5bp; zuS4l)q(H{`k2lIz!0uYOyCF6-Ov4LROO%saWyc2@j8%b`+d?s3WCE_dZJrxBuM*E` zP1qSSVOoll4-oN0g|ktgH+E8|Fu$RAw($6vy}1BWjGQ7qt_%Ac10XTHT_-pyd}h`l zZX~y7ta%+OV#;GhEdO-2K;Yr7s)wrwQ(QlVRL@N(tEdA9t9qvMtBBPS&NnMM9V~v* zC#yP~Zp2SEEog%Ed%V7TM#RgIqAdu|K8TcM{3;L)RxV*oitgFMf+u=_*RBgQe7~_9 z`yQ)_f7=~E4Kt8283&bc{!3BoMfkqf^e?3)IGJ0q0KD8udoHioPY1$osv#D^RT1fA zbOP6MP5aLfU93^bn5mK?!>i{^5K}tC#4%A>ec~9V?~g*mc*t zH5aMbgTKOBG0R|5000r_PgYyl`3(B!O}LaOy2oTpI5zd8Ot-INf673odLEW#`WIfL z6K_GTOL4b|ko*fBYf*)vDqgxAQ@MG8IL5j*!e2I*?xq>dW zEQ^~!AnFI3Gi6ijl$>shD1z30sF_2jl-QSIsD;gQdmrh~uiw(iwxB1=GLaig#tt{n z8!73xpa`0s$PKfy7dL2e)vjSdsE#}qHd$*lrA4>f4w7|;tK|`d6!_bghmD+>ct=T| zWO$f0TR#=gb>+Sh%fcx$qPe!RnI&gdHT|qi_&e{b&g+$&m&$Ro471~A>({>UMTHVq z7vT9WI(SZZ@MT~1f+SCG3{;xX;{68u|u}fhgRs(tO|dKYvNe1$Fh}kN_BoL`E*gj;=am*kHz8q64BdPj74l26O6Z3p_~j&VN#e58)fb4 zW5wO7n?WcUjx$K_Eo7I}>hfpz7uYoF*V6!polOBT;$iw0isL54`_~&MtZ@^so`%!p zh#E>6z&pjyb&AVtef!SJ;*F9<-LgUrE(xd_L?mATXM^_dPDSK%--~%?7`(?mFNqWn zIv8hL^0+?PJbGcX)8YP0Blthd%T5n>h8IGvGe*ScZPVm(rG+on)H`ASkE_*Zk$-Khpd7-0rHkM%0#rTQ?SnO8rnE=#!&ISe@xD<4eg|pu| zU(P29(lju`j=0d1gVSvyYU2D*N6V-@BzG&f%=UT!|M~sIYRhyyjw+p%e;|9$YPSieNahoGYlYXQ?O;G z`^sK#Z|^*;;k5Ql!|(F`@~*4ibCvikG6+Owh}O}Ky|?0>e3Lh;t?B(JHYkLnik~x> zw&-kJ>ibYKh6Ceg{7rtfh`Zp`NdljbL%E$;aFk^H>>9D-ep|R7t+ON<8v*x*bfg$a zB(F-CGHgO!G_wmZ>7H#M!P=YPOU6t`dXi1lExIe`Va+F2Ht3=d#38#1yHh+3Uli&v zXEjC1{`C?$bN{Wqh^tU>yGF@$G(Kibn{!OS%teR+xo;=u6@ra@t->;ughx+L?VivmKY^?5LD%~T|UrLQ%&UQKP{AEtMjlN(0z}o@v zYZ{>nuB3!TBi&rJ3_4#$=Lh?B9=#;puz84g^NL+>?~U)jFPt`wXzaNL(kO$1q(yfT zCo>^Bn7fcz5KWAGg!+mERxp>~WN1HJc$cA>uWnWuR5O0%?N+~@{AUrz`;`w~EuG$% zV5EJ_NXW6J>Uh)sSwsop*@W(N7W$YfmAAgU@bF8Qx)F({6jL*yA*yqpCTuh?fA5)d z9ndSxDq>ERO(;CeUB%$-5;f`)-w?P=pY$o-xB?*OT^<*~2lpUHqz?D;2+Ex~VAXjp zXKMC>NSB19j{FTI2R}aV%m~7``NB|C+k4urd%cgxw$xsA@I?@D3D!>+IzP{;NsBa_ zqGr;C%UW|Os7PyT7i*Gq{Z4Eikst6?d8X)k8)o)d(7OLL&$mc1ws4*)&3bEh(zsWn zNN%c9c@0kg(}*x1=c%icCm$AllhqYU9-a(V-{LyV&JJO*MllDB!<(OU4}}fzVChWA z`?~b41ak0VR*w8;b<99)J}Ywnm&429oXz^Z<#Cp&q1Aa0T3x;?esUzUy}}D~j35nU zRXKZGQgx{|CYv)=1!OYLEvG3Qa^je6%RcnbJ1^qp1XV+6wLTW~)=YfCsaRiG#( z^>jnaTm9F!v)jW@6WfmXy-;5;>>hW|;n&C!Y3bf9`St&u@LUPsfX;@&`J_y5+ug&D z-Q!AM5i>qR*E8<~!DjK(-Le6X%*HTlxGPmt(VHNOn@a82V7hv^X>(xgFm!9rN_Gi0 z{`@&(%y}DpDk9ewy@kHpJOB?p3+Kie(50l-_R7Z7EoGg#!^CP(hRMbOx2XjS=OXKgJN7CE z@nZn0%=n3SAxbAJrs{#r>q<(isV4{$_aj(pO1GeaQXkMCzNn&89uN;9a04XD_)GQ| zqQP03{#v;LnnTLDGJ|{#N=xAPa6d|qaWUHZh#KtP;d1NKj#T2~lnU-=;3Px};r zxfb+oNS*x~#c4Rd4GJcffeJ7{^Yd8(e7oX})dhajD^By61BWcjPK|x9dJ$|}!QMNa`GJ25c0q>8j-@~Fd$a-T! z)utEu@Yyy^t6$M`B{Oe5hR%N%C>|~$y+2uT@l(^0R2jSUf5y%>znMQPhP~MM@j6D? zG>|A}tQ)$Lp5eQBgY;!TKZn%H)P+bwEn#z=Tu1BIjqhc6m~rwk)g)u*LrBo7cGvoiB;;lUwUTMQ1#%eZD* zdK%cHs?4Ttr9;olHw|8$lrchDxTCzT@Lx3cpO^9&=coUs_%u?=%6qy53@#Np0Kds z%N5dy5&6j%V%&!WTOgEQeOkn`)E9Z-L+bE2Kzz{Lvt^WUmyD9Xoh|oCH?t+ptmOKZ zRC-8Mci8gGOx8(X#h+BB-5y*`7or}TBn-KH83~LMV8iIlr@2|2(1=Kr{Pr_3**S!4i<`eC>@@^ecE1;qnSsUlCJ$bgI^Jo(7 z#K`7NNLe}MV6_UNX)R8Mu59%?P{N5GP$Mp%g5$&vW&xy;lq+_nY$V-?-gzc0N9+C~ z3%A0KVH;I5wIfgdfX5z~=Qm*LaCz=}UnAbNBMul~U6&?JDN1>vUvdta#4;n}>%plV z(OIyu$HW56ZAfdh(`RqdP2rrqZW7+Dnp+SoRK5C2W|plL zJMKZajppt2{?ZFTcwoSemWmsP|FcmcW2ii0iCHe_Yg*VGk0j9-jCJ6raL9;Q><6Vd z9KZR!!V^_UWcEPVcU)qByT47$gPhwgd}Jrg;qIVugx-pm~KHJabomat}( zF)@tV0q>T}s-(~Ia;M)r=~Vv2Ezy_;ID&WX6~QFx19L1g{p9!UEo?`!LB>uZIp@!{ ze;+;y8W0QEii$$$Ym?79SDwu;XExBR6@B}ZCiq;8&*>6SOqDr$AVg@1Exd7(!CuHVY!zZF=M~;*8E`1@%FtqG4)m*&6eoJFu3SD!d4Jc zM6w}~h0AHyAnIRrshT#P1kN4g_XjU@t?U1N*4;@{_=#_<`X1&F%8WOTPvt94>Rtt{ z8(7RHWKik*24D2z`Lw|BC&fYoEi<+Cajn0aPUVM={dj*}kGq>SPYix9hx|NmPV1gn ze^$NjL~NDbFlG^c5`mU2b+r%d4O=J5RHmiK5RmN@6g)>wuHHV*Pi_)m`HGa(1h4F? zkefLUlenBE{uVceE#R%Dvs?!6w)k>fW<(6RBu~PVPG;?;-QLq%SOwkCKZ<^X z7OwMffem{<;L0Pvs538BBW;e82b4gZ3v_44<2@?Zr&h=@4pUT?YO2J|K zomKy~4$GOsDB^rpT*tWSSA^N?=RiX1GY=D|reQEg+9tQng~FG1%5YDjO5m=9u~O&g zl(T>6i$lJU!Y*jLMcRG3d=YOo8_6*KbM%42$zWo2!}DvIYULFGsQ&{#&wRldZy$it z=roCG2oo1SKaZE?qhv8D0zkie|7IM_wxE}rgC=*kuxC1uhov~`9!fV$n;F)PG#D+nMl3WoF2ky54Eo~4yuD`IxF5( z99PIWA5LVRK-8Qjrre~=|lD?L;C!+ zTc`~DNY+!-^_)1Fp4=S5chABAM8n|vgEiipz`2jy zm-FHI^zl-?UoPXVllPmv&;YWZNN!#-mz{#GvxpS6?>?y~Mw{dUk!H-qL7jG$vgm<( zS~xbroA6J5gLTwn=9im&9S$bpz{Q@%DknSx^;R0xcU0KgRO=~Ul_2L>EWoDU2j;pE zu>chOK$MIbGdbPsIaO9E(~mvyCX6z3u>J6jrT;WK3e=Q&^!M)br~ZF0zyliw5{+mS zPowejgBLEq&)Mk$(f=#e;qX}ri>q|zBo;FSLg2x+q@Z_ z6p6=Bh@f&bJ{7x?{CuyE>P=IK&Z!i)wxoV3{V(jVTj_|L#%;ef?v6YgdM*k<%N{n^3AfeLACF zh=;|bA;XR;UhK9ezFd6|*->e5*Gy%(byWk!#d>NT8ki(~uX=e#EzQS^)OronPPvn? zF+0^r4w#5jRovuxXv)fwk)T_*388g`Vu=y&8*nO1C3go1H+6t+&d`0MqNTdILmem2 zLJ8lqQLm(g3+6K+sT+2EN7Uye2_lwFJXxX8Z!E1POJ&Z+I#$erdgfnm-u--xCiUmL zcJ*l0Y~dl>^mD!)8qS#G(S2S+=AdkE z6q9+`*q^_L4m@wxu^r^*O`?pkdKg^o<7c;rQ2QzlOygyd_K?DwhqmQ7*Oi9#lzM)= z)gnCJsfg;qivgV};sKk+@L`0tD{2Gs{b2P_bLJpQ}n81Vx#dm!wwJs)N7BcezX?!E+6A209fPvwe>MZ@$&s zQL_p_HAv@=!!0l4tIg2vdHI$FR&a&{0PHX?u1=K(TqoNRp6rQp*E4tVu%Ujol_B9aYQ0)-DM zUKFzuuBe?GNVe`|=jLPN0Fo6;0Lm7hZBMw)cgOv7)Ldj%5v2t8L8X~RwVZ(Hgr z-f9an(`{+P4xVbaMtu-|abybN)fEf6kS?|fczpvKC>4!mX+;izFc!8y49+j~7+8SwAy*3RnyCdW-b-c_^jb5DlRet7 zC|V8s?_-8^KJYipPmJmvK~tMC^hvv5ZYMqWYX4$e*$_}oyYtxC)D5cAj(7}jV&McL ze|hVqHurH=@?O9Ty;l9Uy`Sr( zoF96qD~&7X@48m$RSK8ERNB5p#CY$e=R9dbJ4c!So|cyc)7wq7LuRz{y@w>Fg5h2A zkBc)J5RyAX0Y0jo4eSGTGqq0f?&r3kIrAbsW#+J*i>y>z9v`Ky(rdEP9Pt_2ct#C_ z;&KfNC!^mnN{IhpfBf@<3n!^npdaxSyOqQaHiV>7iOk`pl^|nWdH7qd)m#x56dWS0M!`31T55Zq3kfD}H_}}vSGfhi5^OLQ1gAN1OkT;IhG-VEL@1H2{f8I5*Uj@@Z+sn- z7@1ti|Ky4Kv2VD&;fd;SoOJSRx8~}Pge!E0tWiowbO`l2AWEo<;%b*9F6as+5(}_k z#iF_5tAA1y(~9K0qWHfHmvO_G7A#wp=dxpMi>qopXzITtImGpi;qtyv*Mg|N4$!D$ zgrg|1)ra>oBEG1nXJyvrZ9?1cJuj0;mKhz~Qg>!~ZkNzl3xWrdoMm&@>ns#q|72YT z0d<{p+%*W+D11Ba0fD2>)1TGR$}Ri)@l=*LHiiaZVNMe}9pk23=E8W&5$j9P_bmm< z&{oKK80A{dV7mtl>s?2@o+$SL+mi%&JkZYzJyCx|jJ>-@hSKd*siJBxqoZb617x(vr^ z6FHvaLkjzb82Y$Z4Cjo%-yH_h-#k8NTAw^X7uuN`5em_G5)M>Y_#D?lyIK1bcV)!h zibzI%ev|5|=eLD5H5Dmysu8%U)*1pr<(c}{@1gAkT=d+vge>sR8hiV6AHwbymJ?tW5eZOM32>=mPCKdvbHH3+Br%h3fr{ zxh#5ePKgRjNEQF>I}3w0fAB$MIZu^q<|0ZCLDg&vFtbt13GGnaea6M9jHWPMe0ifa zVfqhRol4QXWQTJH;pV8#oyd*Y$2wF!#xPFyEz_-@5LJx>_B$h*^cK z>eJK{tR1&LY36vavvU0I+*J)t9jB`{G+ClP*>0p7|L4u-^Ab^)5W3jDq%X>Ga;vK3 z9KzhIkEM^3g6cToE@ZP|X${=6fv||I&vwKty!^guBj=^4%M^VC%BnNc)VBHdl*tPu zE`^E9%*mV1Hexlf8OlCx8(I%x&Bp=;dPa?R!T0Ee!mnI&b`sJN&L z^4^5D=Ahamkkf4VmZ-+$;2=%tdxto%yh&$S$YBEzwYt`;{pK94Sn5i#yZ8yFNHKEo zdu;Pk*6apUP<&B6aANsq3!e%o!ab^ z@Kf#Hf*(st+PNF1I)Q}123)oZOB;P{M#l@`@AidhkGGAqrhl9bNnz&IAD;YSzo+!Y z8%+V8JlG5Udh!oAyV9OYVZ8(IUfh^3Yn7CUir_K4Bbj#tum8Q+$L8T38oc@uR4w~{ z7~_s5z=O$}drRuIv)ME4SfHSxX^r}!31A+{(B$zKD>TiE4m4N-2d5@)6tLrTS(V7jaK6)f?q2Eqs!P%BRtH?K8nd#0njE~DufwsJNk7s&;F1gy zv1A|KAY(#MH_xaWkZ`_-u-?O8&-K%{=F4c7%|4G&9g3EqOzLnpvKloF%WKdby#Al0 zK5N1wqS<@xyAWRBafzUrRtwE50T=1~jEL2*>rt1JEoj+kvcIp}{%{s;%Wson>=))1 z8Kp9fW$=gmxK(a}8rseb+*uxDCDMwG$0&dZxHLv6mtPcoMBA_(8*H0vdm35~w3 zvU4a8_f={Wn32pR_R`=a)@Io3(Uf(q4khKS#_v3`gr7z%W}Q5Mwf;Y%zC0ev?)^U^ zGG)!J#X2dHn-E3zW}?Mb_91)3Fd=2%6{bRqlwGoAml!|pF})@jnTLJSvU zS(!Ky)>(>@i;3RDW%u<{I}FQRk4YLGU${SfAZ6TjK_f7s_St)! zanc`~?`;v|)bCDz<9De-CtT}W$KQPTA_7xmpm`?Y0>hTmENAyb4zrr$_byC$LG8;3 zFwA~(LrvLrbPI$1Ze3;?%2oo)1tNXc&@4V+d2 z<18lae9{VPV{4aklAU04`}XOR)ni*vOHbSB9q#-h&5@*cIx)CA>T1V-+x@E8lg^-p z8I&YS`+e-3GJKr2p9u!oP9}&KO%D{#)wOqTZg!8mL^(YX&kxtIZ;WJP3c)n5EN?m2*PpsatA8&kf&?=e#B+DP3k?TQDLFTBJ@?N^jsYG||m=p?(Ppcmi;6#|KBsVL&A{{@C2$}@E73(Ws{&n`P+K|+#y2XQVe2ry65CJxC0-uOS4#y@ zz54Zza4N!~mGdU8jR5F5*V~jH&?uc|*GCx9*};4*eR-qTKu%6>PRYD-)niZm0@3E& zPG)bf!u)VWg^P#XMLzs*USP_lS};FoS}fZodE9asoq|zZ-Py4iqIZ$Y#ZL4ILVt1A z@c{d49VZRi-((2**D#Ob@;u$nRC7M{Dy9957cSNaL5hy@j8r$V6;I0iym&ik2zi&J zd_tN1AyP(zAI#L=On91n&c$X?`4Y!CuIei~&NAJUt2@dV(^YbaQ@^6ikI-gWR z`1fWDm(^BKKW%H>Kl~ge1VHtN(uJg?B$+h&;7bJ4r-*8HMd%1lX)-4N!lTh;Xio4F z^&_Qa=5=NzmK6Aj+XRZvf5l?)j!FYffb>}V zU9Y0Xug@x7cqII8?&KKcI%q4puFvCtJ(0_g| zY*K#Qu>Hrm#$kUXvYRG_S04}upRy&2>=9<52{pHoo}0l@Eee~(SuH^d%pyjceWcd6 z{YWW*r>DF282lklpZ*bVYvqQYJj4rDyPRa+MK|pSh`iVz{}BIh;w3L> ziK~*WPSP}veO9E$4}I_;!0Uk~Y8iUhL*a+tvnid@UTMpCK{Pc2Q-=iU3com!%vYUK z;D*)c0hhiq0}#1mRqdEc%sdJX><)}}yuiwK%7G|LzK72oS6$v1ERKiVZuj*kt`2*$ zwNL{Io&&E*>nV_mO80`^xDjmddG{vFG(|e|wre@!5SV9C!T77s5a2qrQY-WcAeCqp0Hn&f&+!0k6-y+i=LCtJ@ril@>mK>mNcSjt;A6LdHmDYMFJ5X&@t1AEM z=~+({31lb>;w!Sb=a1l<1b3$bZn<%#g(G&}w+=eNezXgNHMlZ=XUHTVWfcY1H}|P- zOL+|5N2T3-RKcy@aW}VitG}C4{U3^XGesPS=|4MnNvGX?WCq|~M$MB5?Q=M#?!-Q- zd-9e5;9~RzViWNLI`G%?CK={dpo}hcN6Ie>BSkaP0%pfx%`ezf=Eyiw(g|E%RayXp z=jbYtS&PM7SoOz4MFe3g`EmqvS?vv=8EUR=J(_A_0Wj~{ugR!3%t)AYTc& z^x!oq`Sz1!Zr}d?-b6yEGC)cMJwUYJF;ev>IM5ApMSPy%FQZjPh7wI-Yk{vZ-{|!p z4W`&2c=3nqJ2tXg*_KNS!o3*uS|`T1DX`VCmzV6o0dohCE&}8~z*2`ca6>8x#~plU zN;)Rw=B*G|pQ5%J;%HdZ9uDY_#2xozM;O;?P5EqClD$3?W!OcLWRj@-p%^Ux$z{i0#s8nML~frGTwl{#{UG z2ta=pwhI?EP`-Gf*#!A!B-qunF2Apufg!@)&=jZw0MFGD$g7 zKPMOr7bIRFDxxE@8Qc&RB>&KmD3U=CNRKw6#{x3)1!wouNY;|d>yl`r@LzvE70$ri z1E{D+qt(M7o%k zNHAsF9e)cfZJ^<6V#6wNbJ(A-sW7R;dA$ss^0e8gy6!kw8G3R=CyMX2Bf5w zInT2xORo5pbtp#yXSRyc&LRAJ=f`1a3&?{+qKY@^WTYdrh?+IDQwhC(ME#sq00^)& zu<@1AhK6c>@}-{$iE>AT5llT0BZxOef+=9J{@jq2QmMkArg6*dpDOBquT+=a5M$wN z6&r`e1v;rx?x=%Pqknkq@{ z{Up4yf)Z-SJbn{-FIf8UI_geYP%E3*OKCaDRA%SX>!{v`kCZ+DMnAEVcD_KClq6w- z>rf^6J8Rly1Z*#f`U?As>^BkxfI)<@#KW_xKkW)oQOq-B2Q+@AZ9x3gP72NnFs%LT z-72u`iugiIBZL7p`tL|P!VH5sBEEvQcQ!svq;`IOepm-MuqSuB1P0=5d_u7WlrQ!5 zu`0I82@z#|)Q6%YK7`|Q)(ol$QfKREH|IDea=Apa-F#3gLl?-VbV`Fpx46LZzWaRt zO#mmEVF&nIP#6*78e$Fy33=y&y|L{yi}+Lvx!>dWOvtT3`UpZT%~71=9xHJ6&xEJm3@o+12MhILG^Haz(GNy{ zJlCUhX<2;evsDJ$ysy*3MMyt}AAOaav=8j3-T~&eq`fG@h^)g+jJ`>$kjUEcsG2cY zs0))IAT{axYBa|X%kn*z^#}4@3n+7aH1vctIef&}W^oZF8Z^_@g1RYdkOH(pq;FvZ z0Wpe*_$u0okhJiICuU8>CetvzajvKU?e+GCk!m(0RahW7W?)2l4orFzti)hXd~Vd1 zizpcndHdQLG|NhlUAORmY7nuDX;ZQD+| z-g{wA3}Hk;6CB?PN*I^u{_1}nA?1}D{axvBw8#|P!xI(YOrA?vxSx9_6~)e0Rm>t5 zafy_~6{MM11a!S!?+fU009;fe`M;2s%x4B2#}XtzpbSW8M%$xhL8%j#Wwfpcb>(;M zN$wPU4hF?KC&^rlUC4rLzJIM^kM`ajML&<#1hi~k?S((8Ngk&|bsXd9Eu6@(<(s9U z6-W!|zuoff?*aDskGcRMYC2jqR@Az9A0oGJ`movMg;&xlpFLQ5>fFCiTM-1N9f_W7 z^jM-h@Sn;UJg9uTIi;|*rm*yiyW+F8w^-aho&4VK1I@ORn>@Q3JRK%4nc_G#e8POT zlmmPYFw1}bxc>0j-FRA2zn>TzSig9n^Yoz{^KAgI-Nnvk&K=ijqawpaarE`qSMFwI zl9Igp4g{w_dHZe#Zs(YDyDYh03P8==rn}E!h83s^SVUwG#zbe5&;pGzQsu{zc0y7} z9dp79e>TIsUk?Z?)X&}&W{s=$bx-_b#v~T^=Xegj`-bcHO;gccP9{Si*;RauRBqxW zQQVU?u9j!<`!bnN+=MNVA2vaqGG3GAijU7F1HZ`ygM4?q;Ig9Jlrb01uiOq8LbLvA zSVs@j=G7*RCED$4R8RSfnyj4W*!IUqZzb5pZhm^-4sA<&1^Y-|^j3+l{nC+dC{#YHMS1s$V125HrdpeM&M`G*LiF`xs_hj zeX_uswZk_m`!YIs%O&GnqvS{1X0EMQs;sP<>`T4)TIujYfY4wW=J8yA<(V+9gI-Yc z&v51|VuJd^!kCPE|JrlJ4u(^-Xx3-F&tpa&+q=USuzBaJfs0@C1U_zC-)q(f*BVr( z!dq$Q+Eh?^(W_o`Gx$iTf(6a8%9vKP4%_m#(D(;s@!A2KG!pC*k?|dPSK;wRW$znz zkOIkzeu9=`R3Bqf*;gatJ5JUma6VT^%y~SPzWiHs|7?_d#-9hWK<;pu&HX6bRbLBt zQ4jAXe(k%;Z({D*N@tOh7B2C=KILAD-f~A^=z50ZVZstFY`pwp14R}v{_N_ukN3{C z682AZZEL7L$>qv3?q5D`N69DpxI;{0H9YA8>+eeN@$4oI9PZaG-nyqOx?SeR+yMi5 zu5F`UD#gFIDOm9+T}g=Hj9oE=!EH?UF}hiq!vLv3;l$*X53Bx93vfHw zr!{P%Xg>h}@xLV2?`MNHsO&`WOtjD1jf1&}dBqrl;+;ISFeuo9;Y!hv({q+vTn%h_ zE}Ctv!kB;GDC3pWhD!r<0J*WX-(KanXy&8mP#nN3!=r z(WQ0BvMJq$G8_7^G1d()D!((Y7w^UL3oV=C4EkhH;Q*b-@YAjk_!-^vtO{B#dO9eP zy^(F1=fZ|@fxTbR3Ix^urAsp1YEofb@gRIt1n@bXy3X4XQFk6j{M%?>+qEj$@!*26BBJ7p{uCjkm8uK30^K-nKx zAdKZ7f-p5!8mKZ($yqq*zU;Mw5}aR=IsPg*XBaWTGHJ=<*e4`qWrneH$HZxr6i)_@ zI98XNVwYVWL1L~rH~fHx3WVt{KOSwt*t3bbT>tUs=VQavjy)L5SI3`=E^&IF26d?s z73c&9sFhfLK4!En=vLZmKE!UDd+ZU*5k61>?Y|7O@IB{`ANh;7+7rM2qbu0QLBXH& z^NH1mrZX54IoYGUStl6#6+pEKv_MBeLV^^?_vd;?aefSQV!ci)E(%XKK8Y%9PtVs& zhlt1@zaO|t(hG!xiH}~_V0;Th=cDWir!d$KtVia+1PpkC(gA~k8S^q)b3Cj*ar*w> zs*w!Jx}U?yG@=dCM7<6wTDX4P(Tz00QSo34{g6}h6d~b5UG!hr%<|NnbG4C3OfBSG zTzF;+1GFAVHyZ@bJ9J=Xx)he6Kd1nt4Mww=ylh^+DX3=SGQD;4XnN6^cm!tKwv7#9}&@$0Rxh$x> z$OqL=zPR^qW&y}YWqw4#s5vHEol4i#_;HMiBOpe-t z@asqc^L3~${c>zSx%q>{%<9BjDGvjSMU08MvjxBX)pCkJ@{e{WfnXG0_IUA@JHj7Y zQ_ijSg<9^>NorWTMYi$^onQG`A1JKttk5x%B#vb{kc|Luv$h5eoJjX%;>$aAtl#gq z05xno?ABjB8KO_ zzOIN~;0|?WX=Zx}u)C_c{hX-H6YNd$Z0+Q+F!{Z;rkuc>-uL-Js_6ctqo8@|z%-jr zVMxj@SN}V9zw}`Ey_o3wL|gmGhR0?z>4YuT@+yAelhu{kfad7J>rAGg@-$ z1ZAhzxF5ef^(tfVZv83D<&wYJf~VQl!~R_w(Z)DG&U=tu=u{ZZ%O#ew9qnWIDba_? z8mH{FpTRcj%ad?p9|kAmHzXb3?eu01@C-r%hGRRM%!bfAc=T)=OH) zTD9LU2CVIy)~7FrhCtuQ%#j~QPmBI&giG`bdc*dWCVT7s%$=2YEUZ=5YZ1BS;HbE) z6sKyn52-3SjGJ+N83x61gUru0Glbj6hI()fN$48qJ0RHXNaVMJ?SC6TfJHu)_a7i% zRA!8m>usC(z`u#CiFs#%x10z8I-uA};s4GWup4JQt`>QNpZ6i;*^$KeXTjV zz8S@=@7tfIjEhJH3|Dbo3E+FX-u~C8R&HbXJ?W;h?q`BU`?AnedM@r86S$`le_ltj zRr+%)0qIw-l2=4xO{I|r?V1yK4eTxl`qn3tKK>eI(bYM%FgwbgTCIDbgy zwr(epNV8J5>FJ=oe2vjmr8x*0fhQe#Ys{K(tfnR2K8VBSK#70sCiPW+6%qP-FN#~& zP2cT9mG2^nn#MK%iKpDvGrp}lYwo-$pKU4hUc7i`w2b?U?QwDzv4Ov!wFICb8OK63 zp9M#p?@V?_7!8>1WF@m072b(F-}G&oqXxoZapG9cc#Fq^d>ViuKsmE* z^bfPnEk(bXA7RFhYSW6Cw_*}n-QUbd+=4)A4|3~78_ML72TJN(5zvhh-wU#Kae{YB zx$m)-c`PC{=3MugV29}@ZzZgww8|T=WR;pq>)npjNfohE*}$FRhV=6`&qJFA@NZDA|6l|yta(I-LmADbZE(az z=}ulydVwV>;*%vamBjsHqhg4zR06t0|StNppB{cPFgFddK$h?fPu3~?36cx;Zh zqsulhQJ0}-6jruNzCV18^>_FX^$U2rd-AG;(o>ICzV&kdz9I{}v$_RPY35#N;&I7I z3y`@k7U5ZjF6dE`bj7>9Uu^elcw9Sm@{#kATbu6$!*7`;018Je)mcU3$%hB`q7FUe z&5yd0bDBh6XVO%&?45DjPl^5PHZL3|!O8!Ah9IZ;jR*AXbELZt=mV(n^cHab&No7~l}hExUCpg!kwJuMm#YFD zp1`kgHLWf8I$MSSqG}Y0x^F-1B$KA<*ow7yWmSA7=HA6?cr(0PyjVS7Y876fJeWN{ zEs_}Zxs2z__q+&%MIQPrao=eF2}NP+TzvAvL6$_3a}ynhUO{%WPW6pC=*AuX@BQXl z6K6ygK;;u;^c{3RQ?ckj**?o}^sG@I>HU%U=_nR@k-(X!Y&}Xn9NtpJn}(%c%NA&Z zp{?C}ayMMhNwkSKMKk!0vuP#NL|>J<-ZKptjb>Zvmy{#8u1`sGHeNV@Vesl$I5+}E_^)pQVrNu(J;mqthL^*n@|LHdShm#a$zHq0mUz3cW z6FCHPs$54TYb4BYY76h2D^XZ07xl0==VAM?b_ECr1N)QGNJ$}{(mhs$8au1Xq?%^) z)xvtr+eKVn$U4Qlsjdn50}c@*Rbxg#D0GVr_xB;#6d3}ZwU5~L#vf7c2g3ZVSLsti zLXovR7$X@^?&EoIS=9l786(D*z*|O}8WJ28JcEk>zzK`q{j@Uk|eVIeUxefWsj%qr)%WnY)4;M026 zaxz#Ycljm^%4a=ngCk=S&J<0RS1|bh7PX@$tNmG|3xa}1eAAXTqa3BVB;>d0fxNN$ z?DdI+6&k`-;})6ARD`Q!sWi$0`U4AD-GxHnRt$#dP-4Iz1lkCo?Xl^3+=EgFTM+rw zuHhDlD3#LSx3oj=bflwS7n|BR4jov$fP*s=&S!>XFguyYsrkaK*Q|FhD(Ec&LZ_&l z8ON7ZkV!&@Ui4X8{aT9tEeCWp14J|D=BjN>jg11=xkqB2`?idtRkz0Eu}Ygs#Kt+? zE?Bo`f7h)&>q}^eei?dAkoThrpq>GYzES!x4_%1K{+R+m68*B;p~c2eD!70*C!49I zX%d6lF(ZhdM>M#zoVGst5*sdOP-;KrZl%If^yR+Y;*WMcl=Yr20PFn^01x_*9KQ6D zIlkwbd-T2Av3aFvbCUUH!?J}y_Oc+|YMtQfVLv$I*?Ft|TlSwM|9&1oseyUg$;=I3 zTur^c&-I_8f)8iUg;WBlQ$#|n!DAf&_1fXvLGnY%C~|v6&-P{^CseX=CuE@hmSL=m)|VGI@u$clX2aVXNwoXPqTQue(LtZt@@`eG+!SLc|KC= zuYN}|u%^cp>3zYx#Mw-VY^RijOxb26_ZaAv(yp>wwrvTeCfvg2?XcuRN!q7e0lAag zkG$V*e!sAkwp+@%$JzYg8o(vIIeH#aUC*mEE7=ZQT5;J{yB4jC8M(t~7-~W1%Xy@Y zu1-sgWq4unxM`wCR=J+>9AGvZ^FQCnv@L349Y2aH9A6Q=VUh#(u58yu3zQKCzTkV|5 z-KcyUiJUr_2q-v&zBmYkualC7nmn%8ul@yYB8f|_aS+CJ5c%_ZVri}0ButLI6<+$Q zx?S0FDmNrD12_e`!Nt#32`zlKm7*4)s=c4);h9?!tRac9WK7X2MHf*dl$TG4y*(Jy zS%RYPgFsWVIU{5KIArCzhfoY0U6!M@`mUv$(G#7GJmNXsJ=OeW1>}?Hv=0 zpWk86snHoI#GAL>??4@xlr9F%|Y}hXwQGA`g=in}TDX`mO@6^nJ z%#HunXdn7;)|2X1q_zTSM@tcLmhn+&68DiJ`WlpYs{2tRqc5&mAFcXCGK5C%El)0K zFz1O12*f{|fuCT?(CZw#(lMQXcZ)up+Sm54zNvP@O9a9WD&tyF^T!Q;4X}KDB-iWz zKzXE^t9SF28$$H_>mM;RghxVvlW@W}^4)a#qVru_#p!s%y-L&xaFkD(lV70T#~Khaq{O(;}ErnJJ%I@So{<WlMcc|k<@wcH7AwjYl)$2a^Q6dgHd zaS-@iO2qXVuLAg7&Gwp^)w&C17ib<}{lityI}d#DSZoK#VYUeH^$Lmvro6d_*3J{x z(X8oJvOnPCC8^;VHCzPeD0 zs(D5lYb}es3Sd_KILhRdJZdM!b$!i2#iK@srriBcU@c?RUM?JjIk66OrE$=`di&7wBh*U zMP>d(ah%xqNrSr%W|3@e%4G%q?iVO6TL^j3O?Papan&bcp0tgm*B9W$q!|< zP)k9MEvB=&Us+J(V>Lq}K`lHoJ&d92^?5CfY0es#YI8PKKC>HAcoJn(Gu8AE49hW7 zGWJdcfi9lX_Hn@-5%7Ovi#A}gYnQ=mvz%eS*|@I&+RNXIAJ6}p+BCf@<#nT5>D(P9 z62$*)^T*kki%2#!7Vx`9JD=i0>Ot**F7)d9pUO9i*kHtbVtYBI7?K<$#1Br$`9$ZT z@3~@vVduKN2M$-X9#=%_(e(g|a?Dq7(&QY@FT4T!xL)^7i!^{C`$O_hY&$WKkt16z z^OE>P7=0tw6OKve_71OQCq5npGg+ z0pNUaFxp99a_kOxlX6o#pTi9UEbuW*Lt1dw^MMz}S96;=>QJb0&yiLA3gfSWy?DK< z>I-E->%`#AS)SkXU#4IH=#ti3AG89T1B2 zx6Zms2xm0Jf1LNeGP;vV?s-EMfv*1uG!7Sk2FuK`<9<1v*L25!i02hD2mGSvkNAS0 zAPMZ8J3`eS+!kd3k@)Mhr^Z;K$mFEK)Hl`Y0-iq4vPj%~V@8jYX71=Rtp#OiJdQ{I zZu)UoT@lfQX+Fi7Hpc``r)I;lpxMHIz9-~9WZt8JSKLdoP%_D^TY##XGfw*OOKQje~O_&Fq0h&oYHJlKlzn zwDUg;1iYlv9DolFce}6|KUibIb6eS)AtFXKDeYX}! zuoQfjcjNCyiw>T*&GEVPjk))svpZ+~R{dXB4mh@z`!Rt#n1<_X8O@Nv;}O%YyjjNu zs}9jY?hicqX$IE~1ttl9I|VmHtI|0VwIfUZ?hVVpfw9f&pKe*@{Jx*NqjXtBcVlFs zp2z3q{!#+LcVBnxOAkV~X`_ZsBe4*#a;;D&-{on}>;kDHYhd6?U-?geg$@}*Lm5Mv z_THzNeOam2r-;2)yEf|s8@~tt;D#&w2AtPKkC>2h9k}+jF~ts_ZH@DPE6=SB5EKW0 zWT_FJ4oWokKA;ILosHQ_rd~ zFff%kpZ*a7TwlyWGeAuj>C4Vwc*t1R23X-J5R*=6ruW#(E0^E8&K^w&*$^0@x^VL5 zQWwUCT5G<)qb$dxzI?&U?&!Tt_AD!y`~KlH*PY+Qqk5U|8>kb^<+xc5Red`Kw&`d^ z;tmt~%lD)mj$PWEnRtIL<51d?DQy*ivt*vSJUMFVICf)-(5ikMe&R4#oMn%R=D~a| zv?mGjL|<{T2uzH>znlvs*K7Kq3C6ZHo=>R#D2aKoactnO!(v0T3XU3Hm9+RwDyqWpHj6qsjiVTZcDeq82^>@e@O7hFF!5u7T zZRofPV)G6kvWUl?#*|*5dPc)xDr|TNJ_$6x9UWa~i}1={Yn`(Kg*R|?R#jo|{e&y^ zu;Kk=IVQ8shxpcPV`Em#yo~oT%BQkgZsrAr5tqym;n3PrF6}CZFGs%IH-BzBUGR(% zl~4k_q<^b9s17=Gl4VdmCOTGCES>0o=ZvY9eq%I_ZhFbX zM`(9PFg({(OpcLBLm4sKv(x}F96xRuV&P!GWEvJ)uwO;EpO^q{#^g{z-4XkOKRBgc~!nazmzA~c#sGPj-V}*zqWUdA{$(7V=2FLL=sFrh(-Mr?P-Gq@s9Xyhau z=`un^2GQa0baxGHEeZfZs*fDUq!@uyk9``RZNXsq$jV|6nca`&*DfUogTY7@Yqml; z_ysj+!WzPUJ=mfGosyMCPf3i`DztW8FD(6bq?dgt{AlEHLF(~%r?77KV7*IT08eql zpL+F7yAq>$R~2Z2LFRclTCEh#QU(IXWDZ612Y z{_P8~)^{7kOL}X&ql0RWT7`%Z_z`&|=)4+}7-R8mX9MfK-774oYSP`2{Nu?*!Oe>A zq|@F$+fHau`YhW`D@6Bh0y;;-kxXn?Vwr~Dj+nZ$(qcl1DkHN9p!q5<^f+SP~Gvi~PT#)`!ZRa$yS+(lt)%z7! z;rLO?ueW~^Oo=j4M`NPgkl&7b;D0RmSVuASRkmJXeD;{h5E~J>XCj8_|y)Df1oE_bp1}z-c9lS|476KqYxC;$J06?SoHt2(b|ClX@TW>gKOk zELnZrQ>G~AxbHt1%4h=^syPZ*A&>|ms8)jr8J_`Xz|xkJ?hnl4thv{rfcY?Mhwu&; zYtfh8@#bzw@D>8vBqzrJJ8F-&WHKO;_hQD10wRloe>=+81C#Af#*aj1K9&Uh={4xx zGUVruq!E81%c6ZTpp2GP({3DAUWd{I^P3W2=oce1$AwW!>g$cabv#V4sVmcM5!ai> zX|-t7c%DvzIsZIrelK2&e@bqT$=Z#us88C;DiaK7bVJ@T%q>H!xgptS;9Dz@+$5$b2EKa2CGffe0X8>VX2`-B7&yGPK;rmM6yILE;%F8}Kgr;=%4pg5%7;FFJWjgOrYp z7}eMn)LXGsp!l{yd=O$i4hCUpBs=ayHJF5!9zM>UVerunG6s0q+y~q1LZxwGfV!r3 z$0fEP*5DVR^5!4UUz0kXHvsfuWHQpUHLLMpHi?*qz$slox^~~hnXz(EV|Z`#t72$^ zLECXcfs<`3s}>3w%Hq5D&dCS=UD>{B9F4j_&3ERPw{{0~^=};P!00ZJA^@$InYecyYr90ULc~c&7Pz&3KPy1S7@p=78+?c#m zlMxO+__IgsN3UA6f)$OKh$}jDQSf;few*0-chzB@^r2eM$s`MlRD{#wBgiWItJk5x|r@5ft3^E(E1NH zzT(>L?QS-)#)OP0)syzXZM4+p--Rx7#cxi90Dw*Y%s=Mh?r+>^D$F^Bs($*UAe+>I zJs1)OTLlP=RhxgkJSYjQ~kpWw$1W@gEGYZ z^-#^BTzWXkjtVBGbq|C8FZ<>6+{M1c ziR<`Qk1I!}#U2?ehobxFP~TFUrSE;mDB82NIQ9fO7Xn4lQ$rnm3`jYb0d(1vq0V5t z`iXT25IkHy|g~?!aKIH`SsQTFR_I&Ux~XLDTEhM+0`wfN6W3h0!Ac zL*c|Kd{5?syufTGDv{DCz@W?m_MK!uC_K2*B0Mov zWJEaEw6)yGYJ$nL48n@FvGIW@iPzsVPFa5h3NyKP^4L)Z{Bx`XcZwuj&=|fvhL89- z`B31M4I;)lda(b+8Z_QyAmD>&Jdv&^#}X01Pz5|7>D|LgMkI1CF|;VSZkTdNfViix zi!d&+_1Hg{V{I0$wn2=%z`ZW7k9OLd1a|`epvrhibE!~ekn2&k6H0Ln3#BL)%fxMG zO*t|Mb2`V0R&`)GUld*!^VYAiE6!Wag_ z{1RKkM(xgofB0W(J=UQ$LPwF1lFeiG2O${A?} zdciepe0T*x*JVx%#0Y~KZdp8Cy3bz!k9Z=(WiqVm&Jyv?uqlK;BAcltDwvw1V+mUB z#%Ric%EYw{pfxxQzlUI9X>9~s_baYBwzO(4dC$$YnXax>jVnZ(OX(_eL)``yk4BX* zp8+&RVAPv_3Uy@u=WH&Kxz_;DB2)w93-+<>4QzhSy5$wM(cS~ptp=ULNdSN?Kpi8I zrBmPyoOL6};jfgwy@g4WWjSnYQ%3s(E=lUc)W~UACQiVm4nX%pfw12iViTXtIsVYU zGjiCU9WqRV*}n9>X*EiwQ(0)*0-XI@Sb#Ej;^fy_hb4&HQ7ei_--W4se{aBv)!uBt zY1f$cW%nh-sZJrlYEcseC#vSVjrueAE1&5DYyzRAj_-k@gG}7eKmKwJ#MjtPwoeIf~ zlu#yB2SDdsDWxVxy=@h<3~JdWJ>9n+HawP@2o`B?@k*cTSbU;kgboKjyfu6#qMVy~3i$RL; z=`wU47-z9X={^nuU#nvsC!0U4O(Znl{sPJhZqwmfEP}JORq3k`&{4S}Y^42QTQHLa zPt>H?#=;HN4*};>q6PAGk?L_uFcj%0e{4T;?g60W?#|)(!k&Pg@k_q-y56rZ9isOE zgHl7ggQ2%jPZB0qV;;R+mn{pL**1o7X8fTI?nosP`B`XC@-4~=`c$Z0AUwJo3=8A< z2)-{vHZ`amJ)U0nxVnu7G!K6yF}ytH$k+DL+H7f%*%(LB0q()ti-rFwMpAl*m4EAl zf*EF6?HhdigUt~H0|A;mrXZ>~m<6XLKq;p{JfO^Z)|Vt?`974Zj^AvskNXLZVA-7v z!Z|gsxZrB2tk7QNnoBokL{((_Z&N-A>)S`5$|!CpTV&(4izbvG{}9Y_K}gOnbX;B~ z_9s1jqVp1An-`RpNEv|yJhBC?3|lOf3w zMAulmOiWqO!nc3LWjF9*M40PR{yMo0h?*&~?x0SLCaZ0XE52j~WVi)Y#08IT{YtMy zgKF1v9hCm$SfhUg++8r-7YkVTex9kRmS@s2RUd7Vesh_myp4$p#TRc(yzq#l ze^TCqNn8D=5?VJJ*aLGOfI`vSC;5tVtP2RJb4Tn2YXMK(76$dZ$>0TLga)-TssCwC zv7sU!p0vGvW&5cxCHVKH{E?=;z3T#vToO;lfSn+xT;}^U(#;+5rf_KmHQ6{!HaIWR zo9U`ckQ}co`b%;$)1|Z5yd6QzeqKSRL$%RHL__r;e$P;L4|Ue02aY}_gfz)Y$lUO zE9Eeoi=K5uu(Rd$o4;y>dk&g!J{6hG)T{7Brwm;~JXUf;O3;X~)?!Yi3z$BhA~BYn zYhbJV?WM4MWd*`GmV?7y=KT9jC(8k8G%gzvI18ILe&Ar6nFrcmne#}xSsgmbR$wK+ zj-$Ge3bZh<0MSRG;?&A2(6d(**|6>ZZu*6(+?WB1TQ>7WX<_&Rjn4w&ooP1Iur%$+rL z>hu4-miHZiy+vTgLxyUO!!34Q&2GjVW+6VN}7Phu`GM{E!BA=X3ax3l)wp0u4em0P$1{W(0(v z1w5ntH!v=O5+?n&BIu2pT|s}68{_g>hp&H6Nd(7&v7qg{ypIy2QCM=F{+buccDw^PM^`STW77Nl0LsOCcwL z8!$HDBN%u!*K}$+@2`m(^ywwNDGM&hcDnh&lMa5te>n=~6Mog`jFF)S`3y`s_LzFpJpTuJe+obJ65IFptjHXs!1$`8m;qGWxIY@Lz+31Yawb zp#jR#nyn84Chr2jKWJODtVRD5rO!QuFoN_h@-sW&0TJt^Pfi#D0yMA!oOirJuhV2B zkn5<-3J|jktOX^5;C{V=?xb_TJmRaW-AZ1Z<7;$^I(=U4 zJSlK8MAZjz&rwTI17#*(M@gjrGgbN*K@^s!_Tbt?L=jjTc^;~)CvYIn`(^@0& zqT_gfp3S7lY>1|CU=k9H1D~A)&=LOKN8-{BBbdc(HM0!1ia{ zNH0kF)F>np4>qAV-NK3>0)su>{+}PcFA0$Y(YvJtG=Z_S{AF)P-hKHeK6%w^MOYXT(m;>_})`*96MJxU-wdG@NEza9WY zO$C0UGwY#TN^|MBFxkPOC7EZ_rwA(fX14S<6=)D)mgXfr2BI0aF=>n-g>Zm%Lvj=Y z(!C}Js_LWjTT8~_2ZG4gV73}~iXaLYD0Z{Up&y-;Sw^z1mt4wYA~@w+E>i+lAL6XU zBfx&MbY{k}^vOnmI|IN6AQ>rZBF4j{71RrwarEM7r1bwI?XCZs-v9r9KqfJXfecW( z1vWyIZctRDl^87wDmYTQLtvyxNuvlNQVIj5WwbPiba!|2dCc>EegA>)59i|La?VlN z_T1xfkK65fD|qkl2|VvZx?n~>GiB5+>ic95US?Y(nQ?m?pKH@&YHULDo|oecp^gh~ zqfHhVw+yB>{{MH73uu$jxhw;c_WEP|eP==Pwi?n2?rmdfx1OANBU97iYG-%tJr&CY zULD?QT$_tSCoK>kI4s@z=f@hF3w47U*uu6h7w)V^v$U$BtgAcQ1)k-CIiJ$QcS6>X zJr!s5kDcImbke#r-x{?0D*8d!(AL8CigeN}0;sj(*2ef&3NkL{Q*6rbq=HTAM=W0q zZ|}g;0BzV%;y2JKfHwi`9#_E6=Qh$n2^Opdz#8?_b<@r20aK0;F9-mtX!6K*1DXOW zhkMnKP)tkI-)AW$U}0b0yt@6k5&~Qnj=yf%mR|CXJ#z+S6SlB4U2W7$a^=(ijtx1@ zk0l<^6y9kUFjA?qvv@;O4SttR`IzT$CG`LYSct}ihF)7C+nxKqrxLymFZg$BG=mR2 zGcV+~;vQvy^Odz87xS*%C&4K`h=YpQe~>;fo#hUV@rIk4AFKhA#CFRJmE1X-r_aoi zVujCx_Kk}@`HPW@e%}iUWe7_xz(G<>Z0;0^1YMZ*d3xN~ZJs!+J-^kI)9jmgF&=X` zTRU@IW>dub!}wIZ+$`2IDgC!^S^uts?)MAcgtzAd&=B0j#a>_TQ`BjdJc-NsG`OM7 zs+Mv`=i5WbS5j`1`9fFPKX*$1??{y*?%n>;JfC@cq;OppHb4VjWd$WaA?Ci9r0-eR z&iqoXd;ezV$!vy*TD0WL#08W6vjjI+L+?=iqc_oux?2~)wuw8}BH1p4bKEwmy3HXE z?KsO6l}xvKrPQH`9b#as-)XrNqy@96JU-)=&*!|E`lfbw_=C7hzo2-Dsl@OgFj6gAahSR$1B zn|7;H?*Dt5R$E&$L&3vRnjuu7{qJMr1H)|9wm}pZ1l55c3SIwmzAvJRpOAx6VPh2Z zB3Q(#RyNje4)Gp+a0kqxlx(#hrGr1CXDaB$yfM&Oc%z?iEQ>w)-?1_2oihiNqMlFRCu#RyJq((X*E&syk~pe>Og{^M}%) zDy;M0k81S$A?%wXTi#16;p<5WzrbsWS@*{q25Ch!Xy7P@>%S;@+@XB7Q>p6PMm#Ua zqUI^Z|B`1>zPNP`3Q`?ThP<6w7JOGRun)?`{4vMZKX^unz)AC$@&nG~dtNL2S_=@W z+~CrQtyN7cihTVVm;=GyaH{<|!0G(LUr)izYWw&jlk8w)YP4tG=!G^(d=R{0CR^Q1 zqaD>B0$G*?5@s2b*l}P#a~r#rHuMhw>gYkDC=~*}P7Pt z`Y^8#K@)bNrwa-abkm;U6^Q#_9-z#IC}JtMDMd|vU(M{^ivY8L#f^bu_AQVdMH){# z)uJZ32PVKhd{<%TKmU7GuYn@)f3NE9Bgz+6kmC2YfnvDnJ9xhK-k&N6Co~-Mu`TiD z>HRUU3@}}%E{>Qje$kX{WZM(r`5WnU3#DLzaT-v!&3)bhmsrDpuy5%Ax?i7={~4ey zVDUQa>0tSPvLIXm80wCJF1`B3szL+SuMS=aJ~P_2MN&|jl+x|5gS%!cfb0?CA>gio z!v4Rsu>D(0Cb+`eplP?fF{m*$NXY5yfdGZ7k5EbtlGs zPnF^VX2Zy4vH?LwplX@A4_dc4K!1DjVCYXV$flB)JLGmr;u6cjCZ*Y(mmSTKTqL}; zO$=O5FTV7xsSB8tBz};h?B9mYA@VDWo6WENGc^+$JTD1ePyX+g*ec)(0D3{a3(%A5!m`EmWfgRL(ZIe z%L~wBl5i!#Qh92$u(u$h9o)})4i4x|uK#gu`g!ftRb1NfV)qo#zcLd+OfU8>>6uLP zAnU~>Mj9`^pZ*?y_8hW!Z{sbM2HlW%>ygA;sg)iJOz<~P5IGtEzLYk?ck=cZzRVyh z%y@Da1}b=N;NXBi!A+U!`y^0tU9ciyZR~`8??r8MM8rkyNAW}2qTL8zpZXXNI%3qw**EX}t?)KN##T*KqoI)W#3RKD^iibd0kMkfP zoSRjD4&9Z_0@a=G;ZH^>dAak$z#`eh$(Pv(H@jQ1dN0j>N9PV_yts=Rs0l|M@XP4E zT$;bz9+i-gPisKKI*r77R7(c!)y1-W!x*;6hT zKG#OO?0&laE%yD@E`)a5m^&D(zH7&Q_I0)#gIYk!NNv@YIjszKcG_jHjaE9n92xwI zn#j4heWkn_b9dzD^$FZc)7*8DVuz#c$m`-YrCu}J`nPLJRb&!c_IuwZR+Z1^+}dqk zgM7DEK^-^rS|`+O*7fE*U^O`Qd*QoZcM1HH@p*qXL-y3#cmFg_Q|@#Y1SJWLJjsjN zJ>8kKT#&D2u#+=g_+V#L_5A$PACb+BqSG@H0*2kp{Xb;%F6m&)ePIp?e)C#Dhf~8# zDcL0QwsL`#GVj6Qg}Gg?Bsc^CH6CkcZ<3Scr{-5;!Q_6ko`iMcp__t@F21qB7ZNYXKR0i*$>X_ zHc8cO`SW1`0U#gsOWsLRB}$IzDx;wJElp-G52REhpHm%8baeB4972;Qyj#kg$Q)6G5_YXS=`Q~_(< zwH^9P#sn6p8&p<}uuB7XD82F9ejPf*E38@Bm&D+M!4u{r3P87=l)?Nd3Qa zpdWttX9BYV=ffm3))0Em0^_)^F$ZRhfH49=Y9wfIcKKQ{j&|2=IV|@vRz9m6$G=Gr zxIaTgo0b_i3Uon;@zqy^RWU;WhZssWs5w-|4`H>kWMQw^qGX9#BNib*l%st?NuLKw zdO$92fNy+7-C!XW+^m}@EG*PGg_SR^j{czN&UPAktOQ393J-5?`eVv}r-*L{9+DFx zNzDmN1@*g?IIc8Z-Q5*r5lE+dC`ymgdomsn3t_wLM9<=#RgCLm8CbIjMzDO^7sEEE zn?}T4@1Y;{6Ks5PzOplM^s?0NAa5K@Dj8=V&O95hx&z8__>vA^`RHVmH%m%+8 zN39x^6<23O@H*04ci>d=R5n`-EH(;2(Ia@iM0Ev8&3LqQ`mpFU z>h;y(4cgzvUNgpDJ>TGQ;447C{=<0kx&-Y%zfH-~rKV#S`Nt_PA)u64Nype$Mu+$J z{ry{;!hYv~H%PD>NQH*iy~Z1Y;F_}ajlof *dFV_}t5qIwsa3BeNpoOemrK{{!p z2_GKf2Eu;&;NfL#&S1(O08cXs4OO!u?9JA-z}OFiDA^Q>VUQ}@o>-+5iY2fsX=+AQ z72oNBqY`w~RV<-o$ZboG_hoLoJ;LMDOE}F-hBBXuaa8iuc3ZzrGqPj1r#JpudQ3H# zaTz(q%9b>h8=Lo>9d13;o%jVmS99{xg-tfX;EQU7Z7Y~dXv*6m+2c#NbN-~e4&5( zt9{dEKOEGloc%%a3pe$sY~qF6cJv3AHP`-WJ583!Lx=IYaMHK`OMpL(~ z7lBQK`MulO-Yrq?J$V|hQw?$LG?zyt1IvhO5tJ`Z+(C8dBL_CQ`*ulrlBZt+;9icq z#Q>qQC<6;c@B>v*Gs>CT4#ZHim*G>~h^ip%h?pDM1MsQF;QLpf>o%$aMH2sbKs|Un z1+l<94m4#uO=0fPMpdwbpR_#pJqPK-p0vg_tI`KneY+y%!9bqc1|#$aEVmaw+r*!} zhz9v@m=`CefHepb5rKmJkezeZ|JzM2C*EEn&Fpx5h={?evu;~5$~k+YR0PE9o>77GT4@7*!}t*xIwmuzjtZk3$(P&68;&0Cru z|A-1#aPVj9NA>k7WA#3TMq`KEUgljK4jq=k-nZ{{o&XSWp=PwkgkJ)Hoh5K+1{LGt zd-^`)W&nj%FrqN^x~%w*;4KJF892_>Dj_7Q^XTE-A6K2vS)Js0ULH1PeKKU#M6)*n z_e5TMnNOiL2m|4_I>mj=9apK!TJ#Uv6xZggh8XoRAp8|!JL;WxBRvza0tTe; zM4FvmC)^13v<~gsn{c3;L3A+bMPb0uHDbR_;Ja7yst;|G;iuZ*4n^)js2W1zp;a<@ z*x?z-e}%bO*RPKBIdYKEMQ*Dc4zHbAtyhqS+pZ8VR{Z6zgUD};x)Xhf1WYp^`x>`2 zo+20Fb?u>3qB}Z=OM#Cr_~v#8G#nda)VU&dMi+;bxSQcNl+A~>7?%hGq|y-;S9PkX zPfN@&x}#cJ&&HaK^p^=Px*pI+qyrNewy=fTuS@kWX)ETiiJemKB6-`(PT#KZYsv_e= zovFmg{Rco?c7pJd^cVOjv=DQEl9|;Yt5@aqkD?`d1Qt}&2&Q_JZ3qG+ZVZ8-Co-?6 zw-2-uvys4^bjzFJY#7SoP!C|e4EK_%aK7$U!HUN3;(3+U;i>-ljDO#-fZi{;Lalf@ zd);MFlSPK%;T!#P_|3sYr%pmeW?wuXut~+a7C*=&myUtmO0b|zhS+bT+&{hmJ4K*d29%oPn@B2G4v4>z_r0MxSVWD=JA`0~q0ysm^$fkaX%!N~WX^ zI+dez7bM&Mu9=WJXKkx&56Oe`a_g9N4w&^CK2iX2GPPFJe8r1~N>HT+Nh3;C&%b6O z^pd%6>J)fMw$CAk6^%35_$@GyUllP@wT!_p2cy>4_xIOZ89yQXkTSkdAam#rh?I<9 zz)mFgfynVG@qw#7lL5)lENq41yrF=ri^kNSF$W%`oE3Z1Btaecbidm%H)BP4wvn{- zSLuO|oBObB&7|g3jtRLw@gr!9=^%s)DF$*pd5LUei?Bl+PdGrLMHV&>+$Mt!UeeSrs zi*I|SriS|?& zln78-%>A7jmb9^pjEsWT2M;=4UJesR z1AV7ex|$riJ_AU9!DrKfTFeXug!SRwD~g!l+Po7&SR!M3+d$<%=h16oncA;H3#sOQ zc|L&es6MQ<11BHXrTW+es^rh*l6l9ev0&vpy9on!|7b$lO-*Oh)H7(3H`E>m^p)>< zOY^$#9ZtTF0RX8^MHP!Tpqy| z`m#PIdGrSyyQ+1Gea)!VWfK0D;ns)pnYVlvenZ>lbv{A)hiG5x4aR#=0u-UASBV10#pVD;a(;T9UU zjMwUUB+^_f9crkR)j5lrg_DML|AxzVNEeFk!l(9BOn(y^jjAYEv3BF+RFsp-LBT8(z#kRWqU^pmYFs%S5gJ%Fz}DTv+TZ{AL8ALmCMpen(|s z_i6nT;@-XM)H|V6g4_7PiHy~%as6z55OHzw!BW#hl(R80lc8@~EW~?(a*z7QkevTi z%86_sv~4^|lmZF1dY}T!(qeoX9#QfOcl+4#@LD=@-D*BN@(d9^3+P?bGl-yS+*`W1 zz@#vn^_FL_tA@V-sJRvhIy(QL+}t@H_0#$%=?)R;d`vt=kQ&EW_chEu%VPPx|EJ6s zL|b=<)uGh9c`+m*Z^TU;BUzSrb z(+yet>30jQt$_AN+JHdYjQ@s^em8iRx&hU>V5n+_KM+V|QT7R>+bY91xTb4<1iE2> z1;r)OWeUADU$I~PabK!_6%IgU&43r9Yt-StDfF_UE zauW`~Cd4)(rzD>c5+p8em*!VBuAPHR5UlGYTJR;syx_R$Rgh30!24s)V?(5ZdZ+jIu$O`a6VW_i86fFNs%)EU$X(OJ>Ra#_ z$McI%j)ig6NR!%R?9gl3H)eF>6LP*#U*`VEJL#mm@GG_<#AuNyvrlvB=`zB=&_`v* z^BbI5@yK$U%D}Q@Fc;W8>E^TDO#J>kpaEtV71W z%^RaZl9hAnyvZok*twQ3?g3yxO&-WooSHhBW17?9UqP+0H@ACW^v{$(u)%=RLcsiy zUa^)q{Oc`6i>Hkt3ytWSP|mblllwKpy#cea;(W~wj8Dh2-?%}W)0P(%J$Gm>I|WoE zH8cmdgXZ!ze4~)o4kQ{puEApL{iH=i9oH<);2kXw$v`NUrQ68u_gz_q<#u*4USqWaC#HF6Fl&+~nfACC(RwtK&g6i!9nyweB_zpjnn=B^H97kXf{07w7R zk+H57+L3BvBlX8W(iGSlb3m6L@SKQ~ct9f&e(p~NpvI@CJEIGP)LN5vIYZN(*5{13 zNXp%gMP%T8Gyl*mDXb$Hqo-8tND#SP%jo>@`fWA1FG1Jhy%HExwB3-V*6*ie%Lr`wsj@i1s2JX&8THwCM z2z($}OiX)D0LBGBN*S0u@8x7g&6wx>rwdB3o@WSYeQ{FON5&WX|56$=IOY|=qmi)zU9jbFbQ7;EP$o)nZDD95JC`3ts; zO^+xVQR7e&Kr0`A>BG(+Vcq~YV(Dr(CvfNU^mx6AZ_lDf8=4!gdZG@E#P=z9v=A($q^%xUGUAQ+5Rbg0A^78%Dhk28>6RPCe4$= z1N=~Y^Z~AM48ut;M5b!9;i70lol00fR34e|(mvB}L4$}~bz5cg>kFj1WLUX@Xapqf zaRFo%pSOqz8B(H>g&ZgK$7rH@( z$hU7WSG#)s*)J9kLZtRmf4pxybsQQuUk9Sugr6(_Mg<+?m2I^@fof%f<+IP5ZWiUC>#`Lf_(*RR{&6P!Oa{-2UL z#w{--D9DB_f9iV-8Q>%@F z&gTQ`Kp~r?_kFd6g29)zqJmIs=GDF%hHm#IEpWAJe-#}dKqep&?|F4@;4h8(D%EVK zl?m7P+4ya}pJ3?O%Xj0z#k8~DJ%#FB4k%V-+`UsaG#yn*@EhP#0IyriBwvVqz6}^p zjHT^=A3MH(4i^RIffJQoK8Nh|{5>;BErg?8XU^!D~y!ki0 zQxPZ5*|zgDzHsap2T$*P|F{&rC*IODVvfG=g0+gKbj=}u)@F>JDxn<$^TSpH>T!s+ zIf=%`;CV#(SYXw!G>^nHgbL-d)O#{)&xxLux8z)ymC)YU>hI~kRf2YjdX)O~%xF+g z=(Wnv#lgCic;vP{<#JrK+|eS?Y264fkndP?&+EVUa9`}ZReDlK5% z?KSt<=8MJ?y%oe_$~Q9)vmkT&J5pjBnpHvogY{IfSKnIxxSvaW-dnL&c~UxmOR;%K0EpOyjyZ9U zw&7@5Mf&jvlZe+~Oo&}fy$OC{cJApU-w^CYsUa3bSg2nR(idgHz&7y1}_%L7F*2?5}@als_Lom}ZSNq-*Neprw6`1wDU-}O3 zeT5Yqw2V5sxyYLL34z(nS z8foi+vZI?@$x@$cZzEgeFT!QydYBeggys&3KrOmRd2s2^SrzFIb6Z8ud6{J{KZM)~ z`8M~fb^UR*9k({k1(S2Nb8_!sv-20@RqfVu#OwOI?eZ5%&3^ffLX5Wtv$qwy68k<< zU{(-XB=JVO%?5F&0XX=>(hu{NBJETEHx|lGxADzByRm=5Gd}(GH7VCwUk0W;P2u{R}0agq4G@W-c5A< zIlPksJXx>Th6mmp?SX{ZW)<_J>J@6aYKbUb7_Fh?Z6pitD=jrE&V{g1ui*T!g6oY2 zzf1xuDnh2lmzK0AWpk%@Paa%s_|)6rh?&X zI}2trX)hG$bu5p9kJYL?ZhXk_o;#WEO!SI?bj%vSLf*QF3v`R z>qRw1;4h~DnAIqPkrOzKg7;KnG{V^~gZp6-VKmq)36dwJjQF&zm^M7gPzAYRULH^^ zJ$M0}4VHN>;pvPcC2ms3C#l_N(27% z-atd3MqJu9k0Z!3*Y$+m^!x9wf^7$@hGE zKG429Cx$N)wYvc-_CTH11#n02SwB7p#!a>FSIsZxQ?Z8uY<0{oh6`V$m|IILiLQEm z8pr7WMxEXlrX9z!QFJq)4r|DNJaJF7_owEY&L-@XF%74mld!I?+=&SxvAhpK1Xc1@ zpBz4UBNYu64gJ)$9^H+op!^W9FsG>woifAj`!alW)9XM*~ zA1>o_4f(C!0Lr!EVAG{2uwPI4L3vIXV>g66kyH;cqkOt6y`W(CV3#*A4KHD$|i&Z&^ z*~ZrX3b+DhIgbgW^n%P9AW9YLq;+R{QI?iaLW-L6MVwhRpLGsAEthM2b_T%gwqKTH z(mby)^WI)cG0Z3(N{kD|yfag&8o&-U zjKDv$@&hRVpXTh>{ed91QSI4u8V8HnuD!7Ts=H_tn1Ta8_c z2ESrBm>y%@kDh%5X;AO%K`&U;<+BS5N^mvc%>uw%t;?=zxA2r~q0U~#&o8~b6wz`A zCYC}@tLz6kT(UKXe7u#H9fE6jZumMRYlUeFE~kXm!FSwD!^ecdme8I3-KH`D#gTvv zcT9J#j6C}ib?s@H6^gjPp$0y2mKxh_sM>Rwki9Mb9&2Fxx^~(Fx{X!XVWxcU1HQoT z4}8qW=#eTEcMl!k#?WGoYorVBgQOYe{zsaTGyOCBWx6WF7oTb&-f<=ZVHiq_KFKBE z%3#u4L>Q;>_74BB2GY zJqKj(UTolTao)Q|ZVE5SJid)8w{y+P9+No1X9T{O#)+HEzgZ}miU(>VDN%J(K*b#i zKIJJ@iZ^F}$ggki?QJjIua!iz1QNmK6htpat~J9?y*urzb>g<^Oig=4^ewMc`Q4oq z?IfMjwQ|#FH#+;nD=@y|7)5L63BgS^h_ zq6QD3%IHoL1=yN~VCZd7mSGE0}HLIG#1Vx_c=j;woEKC z0VNUlj#tSHgoe%|h7Arg_dssD(tsk}twHfXq~exdKQFE6B$mNAW*lVtmAtF^mVnGL zkG}a(nS)1Ebt76fvtCtuykQy9Pw3&PL@{vc!6!N%uxdEz)&pQWh zmM=%Ix^?LFZ~P;t3|eANXH*(h_|!VGrnSbim8x;Tfmrq=ypy{#XY4I@NJMNgq9AyD zI$D}%^Jo*ngOpN{jCBAIty6?no_arR7D4DX*2ZpZLMq^ZT3+U0G-N?SF#Bb}x5$1fu~?1eJ^sub(@)84DNbEa1K>YwDCxSI^s z(Fk(BIR{;w?km75%0;KxM3lfRmnk6omX~fXot(8-!+Xk^YRMMxyd4MmM*91`LCEF$ zSRk1%^0bUM-$}~rk)CcF8p%`5ZVI`z{4$ORfAeJypXj7EDF)?ll~_kXiuM|8iFyUU>hola3b5Cqv8qW$d}fmf~bFTtRV_tMcL3K z`H&@wJYFLDf-ZBhy;j2;Wmg`tZw4EiV#rX3?$94m(AVs47L>+QkQby;Toax{(-`R_ z)-Q^(wu-FDHKsd0s@CDr?$6!Z2+GM$MU`o(P@BhYVt4$x`8Q@jh+4e|heR+>ZY>Z` z0asrjL{$4k_%C%^yxkDnFAh;2p#0U=44Qv< zVCUvC+G@fz*D-@Aul8h4x`Q9-0qMUK=vYABp4Zb>0sYG&AL<3(@jLp>lgV#hn!(uA zkF8umszkb+-j5vvqVn zEGE-X9EXgp$cE~Bc7I4DLgHZWCAx`0$=caC5@8He>3KrzldP|B^LlDT_TXJjojHZF zDu_nbo|H<850IQUa|Z?;QS& z>!R1gEe>`Pfb33Od7AV@nj>ZlzB%`i1m&p14hxkaL?%RpC1VC0M$Z9ur?pyn(sHS? zfawvkzY%-!2#Jmb+i}OIM(sy2SN1U9%2((;s9s40HeKRB z6Z~;HJb#DmHa*cZ_NvX)R7dbWU~hPI)n7p72}Wt4`-&}l_kMzhOl11oJ!b)Vcv@fmB&7Z>3SiWnB@VVnpbq zt7(r0FGIdeC_s0x1P+Qdsf_?$Ga##)3@fd4wf=z`xAnhi{lN7ghLpZr z-}?hX1^8mJV`^1juI!)5{_4E?83CZg=@2v^eyGU8_B=a(`}7)R+U;0!pn}JVCM|cs z**1K6MG$8!?BGD;R6%C0XsJO&FAyr^(5aog$K5yXC{4T2kwhB#A!QPgP_xq!ugX$k zPjbQ<)7>Cekr#<&Gf&#}{N$uj{aCN_xC&tQX?TMsNJ~{)7 zbW+0hhQye6_)%4*bZYD6ua)e8rKjTiUmx=HGsrT_w$QMfX>4z@=t$;${j>Gz{$Iy(xi{o1x`Do< z3RMuT;Ml++N^<%?fqo%#li3(amTp6AY^ApOQL}+PztiE18p?L;`)F&A8{SD#^W_?C5aeI{Aq6=Ouj9!$d^hGn}9lBvDOgHf38c{P@pfpP7RV0Y1rV(s2 zALR*=MubRpH^oUy=@?+@zY5=?QPoQ)-PR&AVi{G7Q=mfiCc3)em6AI~1S9tBwLv2@)OAOzPCjJ z-Xu3DtcaTPzIv8velvV$C^lg0(%CBm^`L~Z)g;od3VQ!!9DDGmyP(5v8&Z=YkE6Js zTeIMnAm#7*dGxdQ3@!cTSKvt0rGA}z65XS{2_=s%kCzam9 ziG}k(J|#~v&Vva4LnnC>_`IoR5MC`rlS&GhLKD{ecmyXI7>R-;D|fKM0B_QX<9*wD z)j=S}xAMQL=c5w=@rM?rlLkWX(4o@*mcGjjrVOL5)2c~Tpa*XOAzD$X)80&j?N{Ry zOx&x{l>)HbP-$7sEqM+8ae6=|eB;Q(X5BBw}~FWBxxDg36BPo&N&umRxl(@lwfuOHFq z2K)T}7M_(5SEvCpq!Mll>8yf@{Q$7XY50Lun-Fo{YjGe3L%2Am3PK)EI<3SGg@Ct( z03qY7jYLoRL*uZEV(!p;K)m-)UZWHU1wn>}P|8S$NjV~MJ=8z$f!K6E4^udwiH3x8~&PTZjC-+3R% z&bU>^rOe>sv30nx9hGN8d%USwJ`h||s^XkZY6C@)@nsG9NRtWGmwJ ziUIH-LAD>=k(sJMFzTXp1M+AYA;AaER}C5yEzWxI-vPC(NVp*{9y{p=^!y#BS_7Mn z{`ov9Kf}1tg!U|PGO(;UD>a8VHqZ^7^k)2bIqIXDH3EI@T0R6t>Tp>jH3*?H9eP)$ zg)ht?LbMpSkfd3)t+KdoyB<}RbfE+Ebm_95{QIYr_#gujroS4G6k+u!y_uMj1-Ez2 zv{E50v=;u-olzkTvNYpLmVj){H=g~3(I2W~qIQc{5lUs!4>UT^aJA_G5GgyU^bMZl zhPm|~J4AFNHzLel=A#t8pNe==Qwz%1$iIVVd!fn70$v*MlOx^k0}2tj*ZLH39CmU; zZoei)e8Xw_X=F+D#)}5H(hj=|3o#{A->-li2Bk5>%jFUe;yk;pC_&e4@Ydwup)cmD z{hOx;@y?b|+w|}f)fj9`Y8K}SHF^o^Su?W3JRl|EvN=>n+uZd$Y@sLaO|*&h>%)(= zlu8U}m>f^n*;!w6QJTc-Yq9gt>P|Akdy6D{MdpIl#4-@sHA(N&#;}>Y{P&)+OJUyw z5|%HX>h?@%d_IX(gdUUtA`V%_KydiJ+%1YmRee05Kea*n0f3#noC;R?A%)q2rzhN0 zN7rAbkKY0Kg5pUq-wO(S%)AT7ZL8=|uT(GVIV!Al3cFqk$J*G+!j#*O1Uhd$jZpwc z0MB9X>F35PyJ_lrbQ$)`{C!ZCd}`%eAh994;75`OqjR!4`SsWLFkQVv98vC|wch;k z9tR&U=aU0MOs~6R+KYqS1`-|&XGwDL0pDc*l)5P+#2v?zJmt{ZHtOoTzd0NWKP59s zRg!j*Fc@2jIFBSfr7ldhO5u%|y~1)+BvjAo_rE;cb;cJ8;^Nh9+slgm^dr zTm#7w$dMYvzYD+r3KU_tPj=x}_E{y69(rKF4e)>}w;KX;H?VFR$q=p>TFcg60`RUJ z5E*SbvmE9&G>wyXrm>f-P_&>dTIMsWy7jUyBq`&)aDZfx^wpm+YkfjE1-l_7H6*vq zlo_22Wd01&Y{8|ka}7oM0=JAe5)kbfT#?)@rAqIIWN#n!`pN^zTdAA)uE+M z{JS7%vjx;3H7UaZ+g_5tJvdT%bX3q_nN4cSMo6~^-kz%tB_=%Kp)#~sg2g=&C@RfmVeE5W!S2whG+YJX#Fbh;v`G)m!1Co`^H8#|%OTva?TKnk z^=k2ClSsGOFt5(o05G9{Yfo^^tOn;@6U1x=p1BJ0;7wH0GacG4j$BUH$CsIX&f^8X zyd4ZhAa_K`IvTh_JmG}bj}1fMu3QFumr$FccY!nCouh1@3jeMI0J?dzN@H?VeM3F0 zCptb&d_|=Ht{Yv@r8DCgET7D?`6Rxw9P7CF=SPNKU@pmOEGb{mOn>Qe;f?Yi6*3O# zp>I)od04G|xF)GD^g}wSq?2R18Y74xR7r3V1XM;TuHLqq)bv`&A4SA(RGy&TqU1^! zif=mLrq-XBZ$QELnF@bU^MN-&93oXJHDD<3cI;e|u65=*i|#p;fjOU~@hHuK4O8FLwg-4&`A8xmJR!>f*c4 zc|<*^R^i&|bJUt>76v(g=MbWn^i-en2;5EobtS5b)sWXdd(9dRpS}-ek21IEqjxyQ zG^^LHX?7OW6INAwAI~m@nVzo-d#^D`dF`DJaB?U(7@xhPBK3REC&SYMXNCva#zH>~NM`&4_4A6?4f;l$$S;4eiuZVAB zugd9D6$}&Y(&7*vt+j@|O8eYP7Ph{-tMdB`&wYKzjc0R$CiyP1RL7JQxVOeG_k3d& z{QjyaaN7qL0kM}K1r3TcntVfi=uO~Vfk5mbQy-33b9V2KuC`F|DZnDAR*vwupi-v= zDoDkJ4HQ0;It)MY{v{sK5M3g{;j~M7kp5*G3x1$j5uq!+&W=2}?kw9K?{O(3P1(?j zhT7NRu^mBzueZ-{y%*%AV{5v~9;kw{mxWQ%(QnTc;3l#G>ZP65FvK9#R!_6B85JY6 zAk}laZm5IKlCAajfOgZ{wJ7Jnlg9&SRD%-wmJv!(*nGKsO}st_A_Y5fEWN9)P$s7D zqV>6}ectCO*W3F9ie{CpCh*cW%{jUZLjsZ5NnXDDpm3D@GJ03tu^;$afIL@Z;{k2m zs5Vqr4h;jfZJv;|qA$<_xTOu_!97=5r@vt9QK^%!c%9z{nA+35&SZS3<;&Ffjq6pz zqby58;(k}vCYgWXb zboN%Mj_tyMIKcjwv3WDk(#$b%2vVc`PF}jn!lZO3qozCmWcV)Ou8=BwwUtn)Ny+DS znSFGXBH4|T7K6RN>)dhj&g93@uW0$$skq@19K9=kd~s7qjI3@Fw%h2%5euQcT^phLpdHIA>G4-JtimJR!{)Q& z*2j7IxdfiY;^=wGLHN3E;Nfyi11G}?U+0G?Pepb8-DEt%$3n!K9gpnbO5&r@(fnvx zty(wH(4s^Ekh2c3Hk1~fo;ms7I{mysV1IpR%!kqdq%aU5oy4n?{LjZPQw>qC6+e4- z?dczRG2e5Z@84&`6bkeb|0(ZwIY624UF@JLxI&cP=lm#<)%&QmYv4fsVwN*{u;O$Y z{EOYz+}gK$cN!(kst#N4CQa+p803+?^gOHF`jXUFBv-HGy?j@(>`dXS?!{W4syjaX z+z?;GCIJs(YRXpzRQJG0^}d|R_gZN;1jf}(=n_@OH7i-J*w)m{qKB0Xs;b8KJP+M4 zmQQHZsSOLOev!Ut$>^}u(qTA_2Tdu}(G{F#pA~uAqAaVZUzDOdNAX+%paFuMG=5$k z4o%`4c@2qiAoevaSO)eyZEgoUDV3+E!BtC{&uVDn3*`!*JwdHnYDRnr&w5=RR>7wy`e!&2aVG)DAJf;;-qye6cLpY+k=`XB2}r=`#qp`S__;kaZ67 z0zBjQYd;X(khrfo`PC_%>W^RukbFN#YN-RCviJK{3C+i+VK#s)&+)1f@Po=05kSm) z#TNsgLj$P{`3{`M_+4ROemzL&#u-gMDgnNG>R_#Sd@4$ZR9+)WaN*iYZ_2j{yn2yg z+z_${clpZuHhjm(O$)9jD(Cn?gsaiR2L=8@^nu8Qb@shmwa>O< z#RPA=gO7_Pr;@lNm^9Y>)szS31F^SyzdtK)sAB_GcnVzyI!ZN>@$6NZ*nDrvzY{>S zY;o*Z*jJP4vmi)i6m;Sk0s>8xy(Awj;GF^BZ2>imHc@;AtIyk!U)fVlVH;_SyZeB( zqx!3r!r9hiC9~H7_1{C=ifYl=rOkNI@v1Zo_Rp#i%JXp1$0Qn>opyJu+;1Au4T)U8 zTg*{JA{SJ_r3{@7(}K!lKWzdUMcdet`t3=MuOp4F9y*ty;f@nsq~0^W6-qvzO9L9K-~?pSiDZFNLmp9DCL1Ptr2(PjtuCkC(j96Q5|9d*u~Jd zKpcdztO~w8-%-S?6M^S~fc+EGAnm<-%z9F$nr+BQ`PTwo^dN&$NGUyZRb{IW)xWl# zvdDO&T9wa%usSa2pwUq4iEYUZ=CaWC_2>9e=Vx#wYz(synG;NT2&;&Y(okE@!6f!a3Xv@0fM&1Ave= z$*sxurMWPz%lWOScZIwGF`)|d?|8G7Ibd7xmmkNoLFNvSZ*sVE+^DqvF{!@7S-$MQ zSd0_2fF}F4nY21%K@d|M8_gy>da6Vs2rC?_{P|OPm8y;W_aP4FZ$VxyRXG2^+-N2_ z@68u%D-Fa@nWg8JJd`K%4Po5+N1Rz8FH5yYB)ZR5Ol8Wf67lRkj@~gwp8A2Gp=e88 zV*N9olEZ{{w?K1tfKlCqm(7M<9YiHnL3FfQ6LdX&oMd57kH@4ucILAle7&BRNm@@X zc8CHjmtt4AI?~IX;|h|v!6tAPT=fnIS`%3-(yFqs_&`u6>mP3p+IS=n+s^_lONC&4vYS!V4;?9idHd!f_5} z@8Z;N07^Ay%YBVE$h0rXa{{P(v##b0GV4ENwz$)j8@JWu)1AoG&` z#8G4FL~b_w)kwLEIfuB+i`g&+$-L?zkG_=P+F*4>Zcofy`^LlviV<#sc!VMQ^|vZ7 zKJ#ub4o&(!Lw+BhE-6`!e{;`HQCyR55uuaKDUGcB8N*qz8m(dcLvO!=E>bOhViG}` zq5HX1A0(gO>kjf2fo{RMT(f;2JLU`8$6?j(u?psD8rW)gKF)SXvh8LH@Z~3j6T|8! zQa?gteXaKMsYPIMSU$OV27k;btnHO---BPv5lJ|EJ~T{J{-PA_4p1qhPFt0+OQUxy z++DPO%fzM_MEn$mxa`qtM$CM@Ibiyw7AVB5DDyM03>>yuZkR+DS(p+e@o}l=V|oir zymx0P8xl;X?P?xyJhUPX{Obvv7(fs{8c?E|GBFX*tCCQ~Q38>`R7vIej5mOt52|yz zCiSdPdUN+fv|Ly-UOQHY`$eTG)#*n7ciyq$A27@-?6PxiekbJ%rMXEKYC5a-YE=JS zFdqnGboH;zz-P@q{dr@Y4(y-RRI;GU{`~;SSs}_jY9Yzo0mggfG?Bj!>#68>KdQ^+ zebhM?Q^~rgkGR{YqduN}m~_|BVan=+C*(7wpLAZY(YK_(q37O=W~7N2e{eFVE6~2V~_PO zv1>PDA0!&Z(Y>#a7aq@MGVAR2;GiNfssO#-PV?3SeSrnUv%a?%ulYRR*qenS?7eZ$vR}DsBp|2N+d@( zkv+;dB#}Ljy>jeiM$)l&*<18|oY(vE`CTr*Kj8J!Ip;YZ_xtU3yWT)$E%cUKcn!nH zP|8{sHqnm+;H$9^{fhdX^JH&|0S`v1$GNY>!Xfi%^Rw_hgQ4Qz!Zk!Mo;4#~$6seY z1N%DC&LO$T#om2-mXHJ8xNGmwheK1J~uI<$tcR zl>{{a+M;53Ik4`1KTvKuPok^zyukbX9H`q0{|J`KxwDc{Ot}%av5X zFT~TA>v7t$bF*b8pF1LT?M7WR+GKur)t+-GU|T9uP}_d%|GWTgiCi7de?3#n-eBWi z0p4?Z=bQS~Eo6jB12#q2lT9@TgX1r|O5|Z-qb$P8OOqy`hz zTxf{hXPh;uP^R6e3#a5IQ%-rZ$~#-MblOK-co9LQ4V5kQ9ra0Z^^h0Om?6>tP-o|A z9BzEmB4Qx)5n;hML2#GduQz_g2rM?q$EPzi^C0)O&$a5_py`+~Z zEotpJF+D5%Z73wvR4%S2B;5LK+F0Dj8Y!=`*F{-1vhziW3YUHzn$4j%&tM??YgACL zQ%B||rN40_fBpm$a{DJWM%nC1!JdYY5jL)mlj>?ATZr72q&Q=LTN-rff zj#Dw2XQu|hF4aN~RWBbqoxp=RHVbXr$7v%#tw;8uK<|F^C;0@#70MkW81x;QO;@pl&9GZn1@T*1OcKP4X)3vy{} zZr+@^QuJO)eB{#l(fmnXUFRzXOhF=u9vRQoAeJI;Fyh{zqr)&6d>{p2wP3$2La!|P-r*EaK}!i z98>@qE(jZ&x1FN53NP6o1SbqQThVRrAGe+tZXvRLpc^0#^G5WANR>9Q;2_82B^ z-2l^vFTu1E{DWq;6qo$lJ(bBSE8Os5# zK1y?7eSo>2dUN)vZv|=zUc3K`W>vNFY5WeXlgl-5J8Sly%W^XNhuRw46BYN;Mkn?= zZ*7|eznrXcct}T{0}rIIh!RQRFFcN!tY~!sqdIlb+&aZ#lLGsAXI!7AwKK7%>gz20 zjHw7r3E6Y)*WOxtY2PYt7u0l4B}&1S1MqA|VB;nm*q_ z$D246JvU}Y@I^~6_*xB$H$EXx;NI0qB}iZoQYjKO6vHTU9#SD}W6)0yZUY1&HpIDW zA)LJ~xMRN#_)owxPCxV7Mdhw{!(_G530sd8^0Kymf3TvRBp@KeFMhp}2{|n-ndg@~ zI>Dd!V=RuS<()Ez7SbQWb(_}Q_FaDH)gvI`K8s7%ruBnm6DX>HH3FhodzAMnNE`%< z;-{l5=x>ZR(cjjus7!|4Kziyvo`u7kp*6G@+cTN0KcV59fP`Jss2QkhyBi?cj0C5$ zqE|=PU2%`1!s>4Xw}>x{8uUAukFt{(J@sM=t3Mm^>|<~P;(=KiIY#Gw6biX+ynCb? zi!7QNNlS!8O;1y{LJu2LK`gs@a%N^q&JhZVGqw&WEme-w0dXB z#H*Sg9jkYziI2@`z;YrEnxVf5rNR%D1B4ZXpI*X}qn`Fjy4oeE>P_Pvfc~Ff`weCw z-34tk>w{&h>7AqDghgV&aoFF@;QQ&9Vd#N(C|7X2=sewYGrNbE)O=1`S4W z?)XpZRbaP`2=D(ufo1`eQ3KU$QT=pergP{Su04i5#)yx}RMRi+6yZCJPJVWn%dc=JG`g+8llxuIu6{F#QsR5zm)>%d z4pY=U$tdbP7s-%O(B$dwml-F&1;4iUu=)8P-Q=+3%rKhmjt@N0j(41$?CD3*}|A(2hbHmKE#l%fot9+dc+AG5Rvq*gL-9roi zlR=^-1vzs&vS*`GCI%^FB8uK@r<(Xx=R1jU4^5mCil$Wu?P)JkTd?C_BfqH)D#+fN ztkpySBL4Ku$-!FU-ax4;xb=+QPUl@lDb{{+PANa8v!bvone?0N|EW?4hh3Lx8EgGS z=m&1`XSHruO4%9&=YDe&`-zmj@&tzd8BK1_Pz$$#a}BhvT11pfBavriP2oZy)0%dzGCj;`e>IS*%ujb25U&bkSv+Rk{xL=?9u4$lbQlnZG~gfrl&NlI-$; za^4;D=}|JYD-JBo*h!&VJem0nZ=Rk`Er0oB5E?4jsDx>I!_CQM@~>Ii`c1sfnVZV~ zyAz;?Mnw}o$(pL@wI=mqyvO9cxMe45h5Ol^BUH#-I5Rw&2RjCEGCN%x&c`O`LaN#e zSN7%~0_w(-E*5I_(6zN2sM@-^m++;NwiRfYypq4rvYx6ttqg+hlKd>Mi^sr_tp0lF z9<|NyjCSoWgGp>ZK6NLtP0gIqdx3kS@MtG9ApoO#ZnD5eZgN5;3%&tDSu5dyF_qX^ zb$v;a?jS4{_WpSDj6bAESSCn4;nnBo3SL7ngKV69l&0izE|hrD&X#F_TLChVd!b#~f*&p{~XX-e%0p#do-;3N5NX@!4Y2gT=>Hm5PeQp)k)E}_3;cOb>dA} zSahP-E^~lXH_0;JtkQQIB_u%upam66Zp&_#J@vG9T4W*^FC4BiMLhu&-RP-lm;glL z)onX$I0Q&{`5^hfc}Z`GiR8e$5NAV(g{O$hu*_cI>{ZH~xnf%kdT1}nimtr+=$}r6eylPqX#7TH zJuG(rYQ2WHk3%5Na~0pHF5dQQK#9gr?PkRXcCik`__=d$irut)02y}u-iRIp$llOz zKo#&&9d(n|cbbDsXdAg6J+gNn7?6%yKJwS@)u}+PY2Br(tW22nWhRHY-}uAWIWncX zX)a0>-JhwuYR))CC|jkFIJ4Q9F%O=}&5Ops!@Q2;1n~4s|@=?CejkT4n8_u1XS{d_uqIfS#UkA+;FK{X`BltB(vPG0UOiu zbh>S5pA8kGbB#_!*i_R=-Y@U@wB-eVSTkq-c#BIlTM!iBX-c2|v7+L9LW_>!G3w94 zUWk5A7hYhi8YQ~szl-p5JOkK!_dPl011K=oBc>Uo;>q2fQyyIgGE-z z6Zc}m)*u_~-|W)qqKPh#cc3YCPAB}QF3G% zj24wx?xT|d&AxMgk`fN1+{)6>&&wkGB`&R13!+Cb;%WzKI$=tS-^&|54II^a|`jhT5oFYIFt#0?wLIcuy z{iEck7SMVyGzVYH2OH{M-B=jgfm z_bzq#j+(uAyLczsrU20*<6B)03H!opQh{8oDhwsfu-X^G$q)+=N~Q?T2L)wHp`+A7uWpfju0iXpkLEqVh4ks}iI2WC+E)9Dj)fD|a^1 zJK3CN_}`3~Dc~x_8WPLP4f|8*0&4 zqv8}Nj9rf%s;5;C@xa2zucL)#?$^40_>S`SPys;$>y{h(!X@BLtWB;vYkn8U1yM&i zyBK+-YqJF6G=Og7`uSn;jYYZFUe^YEK{wM5ef#8kZlO}Z&%!EH2lB~9@dYVYHteIeUgfnR`m*>Mp(-xH4BYy5}aGR@xTf2lPH2_7Z$TuWYetr3G(=)D6;hbv7 zusUrSbGl_8+8uCzI(t8_na42-a05ekSY33MTEm(BCvOz;1Ed1>gQ6 zV2AFP_G+TiT8bs2#O@MWTC-0TkjKDoZmRc*`&>QZj)D#&=N|*3{!C7;T#@P`l&{h# zd!}+s9#jKFw+3DzVh9{)X%PJAuu4Z6FpXSYwfsI_eu+z9N<3Hk)EW8G$JCNjZ&>}o zOt9g3Cme+Zh4n;_;+mdS*^@dzvXB&JkYFGMUtd3{qqhc#G(53AlA30*N{8f711z0q zD*=wCSUKj?z@;y<%bz%ToCuc>eqXl@eRe(&r+1mcGW#5=t+KBcoB%W*!CLNtFW|`L zR3Oa+z>~7E#a8$f;ox7FTf%D;oF0`TC-#5dIAN>4ID4w}Y&lGL^^IfR6?6+_%+@Xx zNF2nK(TC2WI(_Hlq@#$urnx}XuJ!%B6dLa}X$5lBomBpiR?fK|koypdsMk@Z-h`^G zN)smHlZ3N^klJAW5H4f<@rg#+Ssj4OR`O+E1@C`gwd4~4bPvYqIu=?m(F$G?z>|H3 zT_{7{kT^qNy%;$_VRjLn3EN$-(RM8KpjEGzjo&UgXWL45wP4jB7ScBm4muM=&xAc* zxP+#3K{IoKuzF0ag*FDtZ4>Q!7s%HL z`r8O1yMZ)&)?i;wX{W4 zJmEmyM~#JD+7IXL@tUZl1Cr5IIwvYkIP}uPgG58$D?Ww^lEBO+#YuCv>;qj${dLXJ z>Abre1n0>X3hJfc5RhXA$}L#G^DeX0BXOSyOPM9z?yooc9PaD0F!QgAA3Z$w3Jc3k zrejaL$n3}TjRrc^gk)h+!zD;M9iAWo)&a&k1+Ow<{H%lz!#yX>T*`+h=KI6OC$lKf zr4PT}dGfu!SdbRG1Ul3&(G;}=On(n zS_lsz45<=C*_=L`*x`yKKJ#j%b23O=AlD6zuqkLwM!B+2QE}_HYLx*{0mtacDfj^H z+K%C^Q0z7$*X)Ne6Us2HOezpC-m}LgzAKsii8s>Ms;50fg<-TyI4wyIZg};tO+bCb zsXAGZPPi=!UvAd%e!=1x`CM0YfEg4#K8h##$9%cZglyu=C)gQYcQA?6m=O_SjWLIv1;^u?L{#-587c%2JR1bQA+(T0s1}Epw2ukBMEWZJ z$?4YsZLlcb7*YQyIGhl$1AmRs?)gy}JDkD>yMsU=<-k#O1usM29VOx9UF!F)Kq|+` zl-C~@Y0B?YeWx3K&z@T#m0-b7%ehg77y1jTDJfhx-x#9U_XxQLDlt3DaGA5e`nOtH zb|XEV>U{Z=A=8Inr+{ITo0jSEkxd*yDm%4v$U%DXZ-z`Q9sz%x8cYw~@lO4oZ{yqE zI+Y@4d_4}YYFClFIZ1pj>Px{Rkl8;5l}#;DUwAL_`0tJk8NY z!$9w=kI4V%F7d*~XV5IYfIfodlQ1*kh@R->F{o6P22_%A+-vHeXywSbanNR#Sx?Kx z$x4nu09^2@XrO7u#RkWHSMaN(eYv<%Xcq{Pe*x-F+YsO+W?Ofq1YDywA25L*_|TN#!qKvr=MaM1y7;8kewoQ>)X;gF<5I3J6XRrM91g~6hb=T#C zGjk0EPnsCC{B(#12(sNs=q%inQ4Cb%HUZDx#tmY!MxQ`F2KW8X-12!B`-N~i zX<=uakBI40lR?m8`TNLytfuD92J~_SJ`4zSu0Q(sM%dK2 z^>RJ?2KWwNjxa)mU@i0Du_+|9Pu^#nf5Kh==<1$D5EFl!2Z{n&v2kUc`* zU(NGnmH$J}$B!R-=A`{?#I_K}D=E0(p-sr1_wX#1s%GYNyF)jD44*d4j(mr3*pe27vwLU!Q8*kgkvI_sS z@665ZMEc%vyOmN_6&XvgXneli{<3Q}PV>XQVZO`Z$?<;I2I|N&))mUl!Pb)DTrelM z1IrGSWraVT8BMyvilp*0gs zM-q}6Q8JD6L&44fG!1xpxs1l63aKlB>_2Sslg)xARbctUn^J7g9=l=s9DhoHPg0uI|sd$I7_ES7I{xI11bfTZwKoG8Z)_joSPa~wi-@Hf<4&R zP5KNNEo7$*6U>8C(L0Cr@f_$ZQ<^8kKwAa*tqvLaWm>^uuBHH)6|DmuS z#1kMOz;wF}=5!1>iOF71+X4{~qH-?n`G%6J9uwsYorB9$R~EX?E=1BuUWd5zmk_tE zP&tT@Cmiq!kiy`6<>gpA*^K%%SUJlW#RLE{3<>&}L1Z##m9HrCzX%}KFoj^}XuldB z@4@?aRs?(!-ngA(PZ=;NWE9=BaC8NuU75$=9*2Z}r#gJT=h0j^g_XEeDUmI=RG$_p z52JE#*$A#L&jo`NDR|I;3l&sBQ47lDH8=}bx9>9l|^|GGQQ=F2si6*?>`oL#`X%r{#?CiE42} zspA3H8xsfof=PxAk?#&tF=${gD17F(8SMIOe9eQ<7 z^*+PfwFRS)H}lkL_x2Qj{hSM8(74%Sr0psy5czkq%%`HX=cIA3RxFh zW9yNxZ$f7NegbuZqWwzLTgzyr1Pjhjs07rtUT$t|_ zM)A*`GXifv{K?tlmQv(?MXyYm{a%Xqk@;}Y5UugQP56rLA9_^u>zi5^_6tXJ@o!`v zyQl}**g@L3mRokQqBSUc9K}O)Kvae9&waEO6;w4ue_V5If*`+?JB3w);%Vy?uT&j; zp{Pw>b5tUD%o*`vY*;sJgy^{1-cM_J%Y|+>Qq0ls+)!rE394PTd@B+^rlFHM_HgpI zx1)%_C2*6|3h3N|*QK{XgXNyZ4_yl>yQ15EIe?AS$c{-ml z2RB#nhx|Kdl)-cbt^P?fAOq=Zu>)Mz{gH{yP(0k7dxz}7r&?6INy%DfNH8P)Fqkk~ zMqTwWe*m_G+m|@l&jo`iSwzj6?%F5CpE2>WRwRc~d(x~b!KlIdXLihPGysY)%vCz3*L&o@csf|>-tH>{>#hwb`+bm# z6C+V|W}PiS{-~#YPijrpx5w?%h&p>qi6*XvSRHBJg8n>pKi&%(=vw)Tl5(BC@{kxo zDDi{6y!w_1eR)21fgpcXgFF!{Kd0Z{P-eM|23&dK%%CeYgMka>j!FIt2RNJgaJq4g z2!-<`4Gjno&?;!1Q7x#0)o`iSeoNS`cVU7O%lXT6 zy=#nq$+|k+4d#pcFyuBu*SIQ-MAaxkZC7T-)BaPp1ykXA_v;2gi*}jKAc)m z#TMlflOAB>spsJVBL%xveFlyI0B6=9N4&xqdXnjo2)=poR+RRP*$UjzqD_kYY-i!D zt93W{h<&KW*|?qFg>q>ZHCTZ&y!>&s7ll#C1-`d8in+iz8#>iG_uwjMU`MMF07Xme?^V;)GidwJ7dob)k8?jC z4zB>CwQ>V-=5VX3s~?p;ki^Ej)0SDiYYdt){b0cr&G~On^7#$2pOUj?Bg=Wz=z))a z@3YI&{g&7P_il2v;h09_cN994Mj!Z5DW*IAgt?4ufo8E1y~|CPZ*}A<7$7#GfA}Om z&%i^*8JyH4C3KlJS3!0H%ltkstVEl2!1wO+l~*2#&{Aqk5qJIo1wx8dAAk~SSQkD@ zzZ*f=Xl;@a&|eI{Ko+BsAM26Q;QQ!%vYB75FCh>w3^OsDZ37L2t-sAVwxAz(p+Keg z7|1I@e#StCcT)8DeJwT=6aNPkUhPQ<)HVDj@&|1{%yw>*0CuTa@Mxi|txW}k>=F{J z6UF?DC;h6^7PP;F+D6S)NCYg*`{N5ipgXJq#QZFbu%y;+D893IfyH!~Ef1t4yrvjP z!0ef=|KYUpnZ!RSw=U%ug!&zt2SaIE`y)kKV*OyXfwWr)MWvJ~K5~1+ z?mO-V+UN(GOil797_bQ03j&C^*ZjXc69!SH(Za^O=$7z<_^aj;pdlCZptnUP2$TOY z%%uW+#_@!)iBJughW*10D3HOTljF7!-FdI9ZjkdR&pTY0y!v zZQI8z3c39CNYDcS)=;+k^bF?Bs0(e_M3jj0fDYhd<%BVYPfaW?!z=QbC~@v!nU~^e zkO!-!v|*ovcW=x)u8Zdra;wUDd5WHPhvfi>ItP1r3<-48Bh|$9e)H`>o7ZEjC_FL` z=nt(zs-ihwCsfilT<6n6B=Rwq&LciH6W_uU^|Bussl;UT{S)P8c|8H{wM2rUN)Kom zzViD?&P5W;!k}$qe(s`aA;GQ&nQ{spSC39B2kcJu@vZV4SIZto()XSDosZ!dnWwyL z($U~`{fh(GtbsYwnkwMd%j$%!LzGCIS`==0axx2K!MY8!C;k3pKf8I4*L(|_HEs*C zny_IMb6I}{tsp0LZ4&mtlm11tRQ+fATu(z$;hv-oJ(WnM)SxZNkrcv#Y}I^-^rwoy z5`TI`YbPCU0SCS^%|FF(xI0y)G|R8c@T&gzt&Bkg5dapbPuKQnK-!bCq8d|40SQ!` z^WWJvF<|6Ax)-63G1`QJIKEXN4yLkgT2!mJ$|+$jA~VX}I1a%jS08x-p-#Y{P2dcN ztad_ZUDcy;n5~CVNOdkj(rNeV1cF79$-pJ`zW_f!7 zEG**3gN>2w9!B8#Go--QMb>^@jXn(ncT zBW1bOgX-T~)Ctd1ODDPsSXlRo@08T_k!%k<9q zT|#aW3HFAdHavib2S<=K0ZTRpcb|MvMV9Lxn`fDt!4aSmbEvB3q3#JWa0ea8V~2Z+ zFao%(_6)&I$tetata*hsh&Q#=#im5MZ>(MKxp73V88-!-kk~HN6k~p_g9}|--I93p!8b+M32rIpBft@q*0Gd%BB1> z00nnwQn1(%qH2>jVQYSlefY03OCl=%8|p%1s!tc2s0sRIT2lM7oDV-v|5DfN2!nDi z%im|V2zUg`iq_3N$~5!>rxOO%tt&ETVA)a9D`WF&Sm+0?gq8%gx_}(YFkGw*Sm7 zcyTwTqR!|KEjsS$J&2n+E_}E3w`e%29IypgI1i8IHZx2CkwVc$J&mrO_~tjlBUd=f z52x0&b$>5*7XivL@Abz_z)z)G6(Q^4;oy-Je5l^O@xJE6qY*i$a7@7YgzntB=}eS` z0*eDZG6PkMEp+z@_u2Zf_nFhU`p2`&7kOn}WZ{|(VD26AeDR`?C*zAD55`^UdmtW?AYB^eHn}7FJPKEpuC2J95>@?4 zgK^&YG7_M|UxVNj3NVm(_YH7Y33wAwI0i*L;UF}W; zSl?y-k3WPQ@Q3~SbzkA7_<_QWD-i{tFtT2x9o4+3alrZ{5JHawfbaRpItmWuI}{8z zMz8weSe_%sf}^o#CFmS0Tp`864(S_-b3o=?+pXoiR(7zLRVw!7LpsSpaW%jryc*}f z0TrCv5$$TTVSCIq^)9h}d)!z5+J6wM>PCQ)ewrn}0hQ4UspTlxQEB=~?+39w%aPCx zl$<64HcKL`nr;zJauuME41;7)7nOkc2}B2lX~Sx>8->cq2x%nBpwg#1j2WCwxYpc} z;ox9kht51NLF2)XV^(eNMMhUkg=-2)0N=7t%a6guAMtO1vUvNcw?e*7u9B6lei!Pm zJU??#aVDL!mg(fMzg ztiq(+x$22(#j_L8F~Bjo(g2m4(g&5a;~@rpG-*q3ctntWiF@;M25v44+8e>>_lPs8 zQfr44qZQsH+4BG--PiKxvM{(Cpbw|u50j5@38Y{P^4G;}2`DXduC$^%Pi}f-9=2F* zBO;dI%uwkPT1x-NSNtH$=?`d-m+p@P;bU5Je<>6^fvyD@kDonByVF=s&g5iofAPcs z_eVJU5bAez+XcQ+-~E|3Dch7^nyL}WzzN!q3IPrqscsA~zRTr1JtLr*w|P0nvH&mr zGIu>T_tnZQoF0HB>ana|MHU`u3vdk1$=oGnj$?`EeJZ*9}$ zZ>Ubi+)1ZLRX&#f<-t!Oi~au^jNv8Tkc=+!@W37Ljsd$)#c92 z{UgW|_i?s?7>|uI=aUM#xNp(@GuDi=7e6PG-(YHnjwaCrj7r14?&8G$)%8N4P}ln0 zz4m?fBO#X)fR9R)NwAPAsjmt+K(u*Z3q~~LU-&{OL5duq1Cf;9f4iC5L-fJrrQz#x zE&nbf+6aMevC>ZvhPfzcfgDN5WZf;?fsqrswi|PNJJ?&oa z(E%3{HX1cI!sh2kWn1^UG0;*CXLEHDCHx3y6G6a-X$xmH4_Bv$y{)3Sf2TV5eI@~e z{f1FAn4>Rcb_ST7%|SVD-fzaJ^7)&kQVqSRf|TYdngdG*$4^qft^9h~x>!$zJxlta`jcJ4_h%QDzPEY1o-0Hf7LIWJ0Hl5Lv8>jkVcHDF7chlRT z!eqYb@?`6f*or>7P~+#zvwycYH--nEj8=ucUNY-!Si!4(Q2RZRuV~s7%bTwO{7Zqj zqGv6FEG^ay*-?aM@buBJ@&Di2?)JBJV;@5Zs7YP?lI3GKI(T=Km-fNHFW3~3*~TB8 zNajPJr5kl$LGX}RkE}ugxbQYjL!jWII4`rAG)wbK4^ z&iyIU{l;V$A%tY;@PIaC6H^IpE`Esu%}>5tf4Eljt)5nntB*aniG8g>oqlLlRhGKx zHrfc_>O>ddG9w@xTtxOx2v#vW^vNHhhIP=>yOJDiU3XYn#s`fk&HuvqzFtl>ve@LmHEx^nw}&1Bb11klsl1d&cu z#haO|)q8W8&T%Ln2dsDNgIE0G*Gd3asWBCV)J{$nb2sKEfY!Kw)a}DnV2x^vY|G^^MC~@%LnRo(4=9=?1qIvr1+u5x5 z?dU0 zZeO|0VxRYXk`Lxj%61H3zq7`0u!d)^68VanTPDMl!wAg}^wzuO&Zl}e;=Zj!aIcD2 zB$z|O71|pEPY9;;lV#qam@TIM^ADvSGBH`ij|cY)yH`E{ak>AgGVsHQ17o# zwi@e`nheG?oE7X-v8oBIgoj^UpT|9DAj%3$E$Nq&vq=n0Y&TX*%jr?WZwbw$JBxmW zmf$$d7*&w$VVOD$I;mT2uG@l&4xgBR* zA|q7kp72btv0~uR(1)#!j?Fy>CXm8mSW2X=0^9Q*y6d|2`y|CS#D72Na|Ml{ATUS2 zx!$cduN(vP=bN)0%m<-f$4vW4;+d9Q^;k|eOX)E58vVnk6prC>Dq5wJ&uDNtr#M!m z_nf&sZm=G!(=8+X1hmAzUPJi!e5dxx`}VV8a8g4|f&*X@Bk^?%^%2FPQq_? zjmkQ{K+}E(NmNBBqcyf(x2)1fAC`wFdie3VEynTC|dK&^!S(b!N-mWovv$%r`D`7kf zaW2JSAW20#HPiODIBWP7Co^?`n@# zz;+rn08=ozn&_5k*SB$U1Te!uh5Vq;x|U?(e@`j3uvO)1)~SRa#LTlDOMV=)^2rRpUha zx-&k8toEmyfSyeX&Um3#jn@PFcpz^;Uc!NdxXQ4vO7K~wcq>04`Uv8o`SFBGl=d2s zt}s9sGn;5)a^VMbFShCEFvf@i{{9#n2@DSYjaj0#J2mIp({DxyL9lB5(~|;D*Z};J zBPSGFvtk{omvp58yGQ6;MMf^dMaC_>RAMOnwba!>OiS72<>`Ye|5dmtvrIAL3iu^~ z;fbA9N3@Y?IS7ghafz>Z5t1&=CY>rCdX=o$JMjY%VZQT_SE8PtzZ z$6Kij@WcsWJ4$bsDL6CDLKUqiLkAZd@w}Q%0oRAg`N2mD)t<(PY&Bg(n1t@6z2#s~ z!pf~uL10d^+E!rX*+mS3AN8He#rG0`zBjXorkv^IHh^_X*JMxDjYXOuSD!HqosSV zB~&x6$dhEylxv{Fp*d;$5)qM`PWPack+n=}07PYjtlf&(s#4R^9F?V`tXcFq2dwm2 z;4S90=UvSn?E|sfQN0cO8cu`Gbo(GA<+>MQ*%Xp;c8#Cp2z zc^Vnr<(OyBvIaR+pW0;)>U8sfO=WgfGU%R8UM6pHFDz;7mGGfGR)6YaCEA{(flaEv zm1CyN-LOwJGjI?j1efV{-D|KR`zctNAwmkywbq`{1l#S!@UUQ*m{8e$4j3#f!rRy6 zwl+GN;LJa62dEyss7|GV^mMcDDA1Pbxe!n8!!#^?Q#A910(w|al=zP7M{t`Wox7a zpCfhZNwIN_Rd~cU)W#HBxqtIMko$%Xcso?26F!bk$el%Q-G}*%pnmHuOI!Q>$8)Jx z9aiV4hK=F4^MR#$nZGy6S%Ap%;J#|Wpo|eWl#{DnwuZ!gzLC(P%(4Kl)u>2~Z2+a9 za!-LhV6b`=Qcykz59@_@qC)1wll!wUDlwHxB0@8aXE5LufPkmsc;PCs1jEypZK!I_ zLx$X4==0ML45l|vjI?QlRo|^jsY6WA{3Y1M9?>kA_t(opj`sb~?1c#z3zoQ9LN>hf zs#FMpSs@oxje^+}}HH*Ro)_NNszOS*9IGojp0 zXbuO;G4w5md^RNDB7ar#FP}2WW zh=2Ey=bVK1y+qOZVLw>H7)rVO3@4N7QMSviaGwh=;;t?_$imK{^uyA;wT;I-KS$hr z4@#h(RHlCZ_K^*xyQ60j2=SrA6hBj6icO4js}1(GzJ!i3H<*j0$3UDFJbW{HX5req z@`HnH{iq#aK1wHPa9pLX#lIJm19u3q6FLWXPXT!*quPbp&_F?bjy;4@3l)JmWGIPsk-hh4N*H# zp=!K@mfp{9WqiPcF^gOL$IR|V)#=)XdWAr!zm0~^^q1=?mLk7*S-HdnD*wH--{}4A zEdK%0%TDy(T`=%syBY@n)oYi4~BI070n+JN&^*2k3f9r!?TcVr%=*)7ci=i|=HU z_B7zQMrqwD*He$*dJ zkdJKpK;%QO8yI`H^XO(*fB)F2tGM)qW0VXb!ZQJY;kzqToF21q*nbkUMCL?Ekt_xhm`)e;GHxBsKZV^OKSxDQdzw@?8Sq$96!zumKU#-|Cy+6{=mkd{jWUzG*Z&=--dAGY*w>2D3)s!mnd)Y8?rT+yHTCYWXv_Z8~^I_2^=_o zXH}@IMQrsy62=U?cwcb!w<0)$%O0$Yza;>=L>{4EJ%%|DhaL%F-v+)dx>M#Ie9JIDZF$i~QKXs7rbU~5be>9RN9a`)8HKf>0NzKP4pe?P zDnD4x;tjs~;&U^t>|UQY?t_ROSYiKmM)!_^itmSY9b4xAI0}g3ZnYwmm0({KD{Rby zrW`_BKm5fXN3i>Vb_(gnM_e?8xrs6bTK34_+FJ})*0RzEDhy_GOOe&9HtwclrUopF zQ7je_9TX9F?$e=it12~if+ z_>n2cN7qqw;6lf03mJo=ypth5Z+&1;s$8((kf~prFhah%q|o3mNYB8(d!=jtIfdrT zXe`4!NigxV*JA3iQ97Z^z)+Q0S4V{Un~c5eUH8~ms`CxSK#m}b@U3lfQL%8uF{*hg zxag0*E#VQ`BhG>OgJVyZ)cxrDu-`p8#^+4X9s?iXfX#X6S@fZ%_Dj$6$NC-%nkq4= znS?PwqSDs@kO&vj!I~M(i=X4y8-Io>$ISCgUUOAQdd6=D*ld2Vt3|VC?OgdwKn>ST z%_qbEq6SeDp9mW#68QP&tXVRSt>>iASBgphpBLbhZpf^lJZnBsteU$b+>1XEF!7IY z!?zV=Qy^kwJX0&!%|hJ?pf z10nZphq#5)UWuIn=+~i;SW#dJRAlT5$;1rOuLM7WH5TocjPCazLsj?29DYRS(ArUw z9h_zF;uQIA#43ASIw&AQYcKz=kFoP_VhQp0WO%tlH31`^(y_QCR#nDgGTp7=afU!2EouW*L{Dw z=AIl3R#!A>=gtE9m29IGa*cuVsh(g8U+(b+suPK7aQFD{&(XWLF8=fk&UF_iC(W!x z%%QQu4%w}D(}cJBS1V=*K~uP+5TogE;M?qe*m+385@z`0Fet0e=X1XMvw20 zif4F1O@gOrVkY#M9^@7`lt;{mAFSeoFU^?&i*y%Bdw(H4he1L_7{WwtT~bHO+VtRR zE&)Sn&9Bn0>#iQ^^S6IV<4JDb>`BUP_RU3nx5Lf1Ez?wg4j0e6^Oqqswh@5IAvp5j zl1SHO5Y3xgpWPbWCZxr9U|$0b#U=i*-$#0!zQQ+d_tM^owUA$PhBOn8qMrH> z87KhEmAi5H47}Zo`GK~rej%CeH3k13$$D+-P07@o_532Hd#pw*O(d`l<@Yq~>VFOV z#&x0@6R?8xPWF?n$|SJQ0dPj=K48#5CEiy?@dAg?#Mq>USDAOuPXF_6wnLS3!Xw);8kEsY@R`bv?G|*Q#6Mzv&J>p? z#yv@_5y(7+`b@#eP15Hq#s4Dx-~MD+DF|f4jdVl;W;+657-C$+I*T?EG!V2nTWOLN zp;3=*-UykY^~h}H*F{~J5mHBC+QE(rEZ@Zo-w+Hduw{vU#%%($z$`+X0sTXuLNzjc z$<4VGOJaY@7k9{{26gX$xh40@b2Z} zeXm?~P(_4N%C&o~6onR&Jc+s3^q`CSJE{Oh_#K^mcm2Th-@z64i+oyc($%2u)I~tx zYD9SSGC`k>h4tyTq;e|IhR%PQr~ctrhp2u5s6`(Tn6P!mhpYx9+0ai(~T>SaM^ek~6x#{-rA zQ6Vxla7#zHN*=R;-<|JOy&#kF!8_qy+R0gU>ttW+?nxM+Nfh5TGey6&$_;}4I)3;< z(;W5T*Wbo@*KqiEnJhlqGviVt`sRMH!i)SZXAj&!qm4Xm0swZr8@$Zl7p*VygHK{?fuU;5nt9j!+r;)Z+`Rc5&f-poQbJd}h`jLl0LPYmeGI z@$yrfK9iIqhdJv+LLR);(mnO&3E^7(lVi=|@R&2@&9rKi3^9PvHdU6KVV33;aqFVp zi!v(w;(AwG%kE&?kXGBeUu7@7Ox#!~&wu@)>B?kV3`<{cDJ7u0iyPYxT?zOu;(uqy zjM07~N*BasO`S77JbZ+&U-X_-ORTHDhDi9$B=7n=h%xZn$$KB~Tms0S5U5tjl%Uqa z-HQ?i+TkfCyzrti)zr&|-p!*=$)i66Kfbs!Afib7OQ=vKTYe;4S|v0k=hfckmzOU4 z4IR}ag9NP7>~)I?=-5b*_YZQ9a0l2o9)Nyc=vEgY)fA6a$k`QFk4_5YiueP{#lCf> zhB!scT7UY)wXw>FH*9>MW2z4%p6k>!B8C)!=7amU4;meya}!&#aW3Moz&X|!c#MX*mygcP5K?VF(iQ_&f5rO?j(1E#lQtVD72U_B z<@Q<;Jk}}T+y_EUx1tzr3~EjRf90^JKRrG0nrR1^$o0$18$+oP`FY$Go%^KD#0PB_ z&Y&jgc0zn!mwTgm{VylxPf-K(3-%-#D-@pJic^b7!e-_63M3A$HEyScwD?r0axn)x zEVi}Wte*EyVir_OG^DH?Uy*aaJqad#-R^v3v`?;tgANcRM6ck48_uUcZIB z@7l3iSNw2Y6=36WQ;e*(ok9v@mvYH;+*(V-?pX~aP~2%NfZ?fYuuDbwPM%F(Rx_yu zUEyl9k~e<^Y$5hq30wM%sg|TG_Bv`QHE`k4(Z065#(-S{W0A_(v+<`VD~|OcKwG$q zoyQ2Du0qP%_h>mbuMVfSD{NmOT6%3W)6m1bLyWOkv#2zYA6Wc+vBWq-;X$@~=I)&7 zX(f3j*tyv`Q+h9B&un**utjHdI&(3S=f2^NBsURP;A?d#V|x4rtbx+4W@Vw}^dXnT zb%o8JaV|oy!?GfAr1Tw#$@D#=cG_hy&3Vw!#Ox-)QJ+Azlq}JBr5(XN6RCWg0NY{D z6#3vMl!$oR07)!udPuB_8}Gf(SXB)mTMoDgwV|;$2(W56e%vw7zl_YqxMy?#ddy+@ zW+^&uGf4yE^!szw8yekzjMqPiS6*lc4;$e=x-u%LLfRX%A>8Yi+9j~MU|z$#RAm`+ zj(~09EXVe#4wnb@gWTQ=Y%pFSHaCCuG=;Tm3wHv>)xn(6s?=oiD5x^XP*}Kg z>5Hw|nay%f^;bPgdkYH3#IAr&d_^U*Ug(x4L;)?E|w zL7B;Dt!;VrEJ1lIyAltxbQ~ZLOO>>rJ-2XtXI=Sf@l>?dlS_z>%i=y2RhZ$_9cu0{ z^g(P7W4?KX<){%4k#aLC>Qjx!Z+v~M4_fI~%vHzdVksV#Sh=5tnhsi1O+&_4!Trq8 zkl6N|07^BF)rGJFm{HDb^YYKJk<@2L2Tw_M9D8Jp2#YR_w2eoq~8GPU2DYg1~Jq=i7_Yc5T*knepGDKOZXQq+(%Sm zXlWi7#cXh~h2h-yi|W7Q6D}c^Kj13HXWm1O8aRwXcNlGd5=mg3Q{yOI9_hUV-LanuNZCH ztDX6bwJm$tr`nyg*7Y5lt5^~2BDC{R0FzqhbBz1O6i!sBF(sB#6Ao?u+>|%L?#srh zNa4DtlYHbe(i{zPwqLU-_hpELw=7DA@+=j7uZ7Qt4FkJV6v|*q-n3!HiwfkFtwR=A zv$W%jz0oJ((b=$jyv!veZLK=ShJV(oS<-OoU-*U~S#8GWK3rAX_%}$OMuRbNl4}r! zD~DK@Ac0Vi@3_}-kE0$dEhnwNQu-N`y!C9eFJfg*b2giL0Xw`ZI-4?j0wtl%doN%Q zjqqKL2<+H&8~a$My!A0@O{^$YstxEf690DOJs`svqn10}f)f3CypMY=h1(_ecqo}( zb^%#6-A9V5MoQQKXo(=Hku6nBi4i~6%T>FS&XQ7BpQ5RQ4H9sXDbMBN0dUR}r=clO zpK2I}GCD~=ASQ|E*pslmRZy?4xPmzs)L+ReB|dntWLQq9Y@JXbthEAd5nmP`1*L$j62&l-RCRFW{n2l5O^TS#r0~nwl44i; z2FL`oeQF(E0;?OIn+w@H&>fsp3un(3(UpIxK+Qik@bwa^{%ACq^H9ZaiDY81tU?@1VPRzyDTI!q+QM-}!F>c* z+hd6xUoYBjbY-j|x+Fx&pfsngVlbdvz>;n72{VGrhnQrWC6O zGrVOELX>+}H@iBqn-dXZ_zzG9DrzCV*MLSC>W*3(NW3)J>L2mJ zX(nzN>TxvAbV=A>wllp(eHTVOs10@~4C^pufazbwbn4_QKx7-7U~3t7?xob1a@gxLg9M`zZ9#fTyK)3qGO8 zfR~Ec7C1-am&Bf?X>FX$qdxR~yo2ZzoJP}4jkb;2EGox5mk4r{d{d_uVTWU zA2inFRiBdQDTR&xZ6q&sd#CQF;+&E%uh{n9oTCT1zwz*joNb;)j0S_2F_)trX+jwKH z`*QnL51j+PQ~)kXmZ)Z$oP-T7((cZfY8neTW4lvO+7$w<4i()r3O&2(y47dnpyPi* z5-VyWL~u$7BwTsIOUloGzTDc8R?H~yk24Bo#m?IMwKu$aH^v#EwA^o~Y2P2v#X|1v zq^zW@dr>Jx?)hGV<*@;yC-%|;5_?^d=eqpIYJ9dPF71vvzMPwQlZQ7%FrtwSXZHWz zy1>LCU`&|Hlk$IYao3;q-@|TfzKzujIHIKg8APxkT#4eI8HM&oHXh)M_d;V|oSCu$ z9eQXrsvgN$uL-w{+QKmg0gG-wvXeSpb)Wlvd{{Sbh`1jpR8bU+XLd{@ycyz6E`ZS; z#wW;p8}Y*B?SD3-up{F-N;?pKIxNu5DwHZC-*|mUQ<=^9z<+%kOiodd4E+ z9hQwMQMU)S3m3l@N$fqQX53z(D!R`t3?v3A)Ioex^$is;Ce8anh{iaMB#u}vTrj65 zZ}n*icX80AE@#~tmV;R=Q4-9VA#WM@O4ygG&>J#i52+03OxB{mWkBRXoomRCksT!>$s7YXp_ z`gu@{b^^YH;S)KTRMHetQ!s zD6;ZJ5f9Vvp$d@SoPJ{1(6RB;aqOxjmee3TVe-z3#VP9vIZ<#UWUO#L;8e17cfjsm z&T&2+TNx%JV;C6P4Z=Ttt>*+U^ln>ADF%1IfSriAfWQ>MHm%P-c33W@Y#I|=49SPS zt!@QAxWvt?vg<pU`){RluwdZkIP(_#d$%Sh}kjD1hH!VrWc!1Vfl}J;rCuXic4}EF8y6!ga2qgtN7(JjR3Ppxw%?c-ZD~FkLVKwIe$(dEQayb4wWd!d?O* z9yxWYQq~(C%6+9SL7y7|q#-6vh>lB>^Qe8R;PF1Lkk#1uMn!#!Z07DsV}~VFW~pdC z3`D&F+c&Y(NfCJYI>{47uxG~X%OZc$E4G>RH@1$EM^WS4rLv={9#;tUO5{hz!Vb#S zNur{^=-x7PJ%B@%Ot&H|-k|5UCvhb*vneP21A-A%SyO9#aASpJn85EO;@x)0b>7b< zq%OP4Ux@*eL;AFxJLhhiwKz}VPVxPMY=go4gUr0>+-KI+ANTS<7R^eV?o0D8?4{e}(UiORe8RCdaL!n*PWnfFEi=xp!YZ zn~G2H^I}U7O4m;lA;5A)I=3nJ8z2>{7|QV(_Ccu-IpN}SS8t{z7sRd0At*id3m`b{ zdnLUJcZh`-t-8J&Oik8$4#F6%VkVfbGA3eN`@YouL9@)**^0S2ZVnaxW;>x{c@@k~_?PiCB z=HZN{v*HR8Hb#`KiPK{&#@Bsu;H>^Sk`O`?T87Alb8qspzDrTq8c$X5Z8iJD(BZi^F|nRU z{rU7oc&HNM{^Mkg+@O9cTSHk#9R&Bxsg-0|#eX9qtK1I#$e4Y~=A8bk#u3MELyC&S z1q4^H>JwuJeZ&nO(&z}4oeTROJFRXk?A#$Ck)g=de=W9#8+87--rr1a9-TGR3)eAq4la+mD(b+?M-;*i=C zM(&URzNfmLaWH>+3DV0O5lvx9*;^```!Jk(=aKLfZh9AS$qG&0Y@)t%VUcRz-?9eI z(vt6pHb3{X`Z!P0AzPQDEfjm#g(GRRp2`WH9^C?$ z{hUbW`cMlg)&Azn=zC(7T7RnUCSAoRee;(S=@|eW>L4e0Cnv52_^+wgURl{mt=MSb zrEE;54sE2JMfmEz1i$en+iR&Rh_Ps71FZw!e%Gbog7#r&;1)iy!8u(p>yQe@_H&00 zIl;}L6U3^^|JJ{8B`&TgM|KVw3gDj`VJwT)i#!W)~`aZ(pjRvKgwP7uMFfAR*J+$ zkqatFERy1Cn2W0_`_xbo5T6_Pk5uwv*OEA3Ku1x$E57T@H9cKp1O<&**SX2T1C(ev}%GrsG< z1N?!Ro|)xqB3WiCk#o^VhMx(7=3Ouhbxmk`%F63;D@7S!4;xiIG4Q+swiyA=>b?sa zm(;dI(>XU^O}1>?PK@!!zvZl4P2?Sz=)?*dsk>m-AS_|e$(HJucYpSIi;?eTG@0)& zAlfY9Km0&Nk?9gv`u2G5X9=o`8*lfFtK@vEe2fqJy-VaX{=Zo)r{`<#Vn?CQsI@M* zYJ$cjRMsr(e|&-u%L4~M7k}ORR8i7qT14CeIim663M5z`+)WbmLjuEu2On%GlNZo| z24hKT;A=TSFBP~HcfK+_9O;KZkg)h@VmEi0Pqd)^hy9N1%94aE6K`H&C9rhh3j##* z=WIa3*x78t-nz-K5Dc|8vA-Gbs1vqe%|m;Y=Yq;cG*55gE@8oFjrM@Qlw99+q23Zt z(i@RpW3A`Cj=TP#LmE_yn^#)Z-)py(xr`JY_mPt!J$j} z%q>@DE#d=D!ofb16#H6xS7~jIroDwPI17UD!3TwwKaQz-{3cP|?!ZC9eZyuF>Qni& z0s-E5V$_b4txZZDB;y_J*lKW$#O9iRk_JdClD^nJOA9y$^UUX zD`3N5lhAiNL6a*^S-#G5=s`J$gN~S4)cz#{AM8EOP;@Ujg|J9yI$u%m(*rLec=5rD zGK3`9>Fm*@(Z9}SFDhkihAZ0~6gYwD`9MCTeel<5z~xvs5(;hVPa(Tn;qHwo zd)m)O+Tf|{)6mSPvBe1{X650~vdb4GI9C5X^>wr7mDpQMydd}(_aq<@-mIZjH`2SOoXJ^F%9tT> zub~2pWh}lXg{!*!`Ea>iOv2Sx2B35YHlkwn@q<@EyB~SF& z|Iz|jmMBfRztaB+`soJsC__P&OF4OTWSi;UnmIb;gP;Eaz{JdbvpE4bc1T9YZ-h)w zkXhxnO~A%2O@f&xf=XkMQ;#C+PEt_+Sj0MxxciL`#A&t%<7(T?7J~zSes`!)e zgfoHmt#KRGrBP;>`=VZGkq@zMgfZ1aO6KF+7HB3g)zkAcqsGV60v)(*JlM{Po$g=I z&4=}l=zA6XTb1}1iqO=AL%@O>H{5v>oK zgUj$iPtsv-pZxj;KLPy2VxwTd_##|3bbOwPbI^OZgSE-bh2E#$LKN^hkk~jl#wlOb zQ)E916)2-)8o>z}T!?KHm{YA$98BLe--Vw0O{kG^d)-B5G*u`FB($nrX{+4Mb0m*$ zIrb`pOy)DrHelf<)>aE1EvjV> z_X?jtK0NP?2r=Q}K_R83gTD-VFFLP1m7#Qa(D)woJ@xV1@a-j6#Cw3cM0k}kes2pT z1;FvANMruM-7BB+!QCf?B#y8DJK5T1)|DleEaIj%ep=sWA2gr_y)_5(Phq_duMSBq zrdJQWhL+L1moOea;SVPE4yheDbL{zr)1p0U?qSsOg8JPltKfi>&%HjSZ=<^uy((8U zWDaS1Z&Vi{!WX~Nj!qI|%IRkxc>DUK(>hrP=+kqk)Pzy>b;pkM$Z8`Ocs*4h^>K~WR3r2?NkLZ**+pUDG)Fp zhztrSQCb)DU<_7p4kGE^BX^CEEklb)#t=4l=qjDEO{!Rwiu2{C7a9mJ3F-707aaL> z;M8CM0nP+3ao$*L6u6kBe?M-G|AE8{FL(U{SSMT_|zikn`fX4*Y*}^6R0e7d}Yao}D~SN-g9R zwOTa(#o$b|lEH9Vq8mAmoEPAE#}i!_sAxoN-h6K-FZx8IGJ;ih8y8p?$TT@7|1h9T zO+JLDVMLl*8^tS-u%}`{$;C?GgHOSHthCw4qjo^_J90Lrp#I&_--wV`|M_!^-M7fR zIX^eSS(A=kCu%;kR&I~-ZirGD!$s&f(J2@Ve@~D5>KJp!98FV+&mPf{emn|lLbC2rPr_asNTUo^{;WWTazEZ`KB!`6cJG?J zJT6`l;{;r*NgeFL77#@bQNHiBd_zGMH-BqQp8mUcr*0|NFOCu4p?REoX;N(NOQ6ya zsMfio(wkde;&o|E^}~;fBUoFp)BW$fC9xXYnO~=mC{fjZ@EIsObSOe(zmYOHv9?qe zFVP*rB(c*+)jyNMm^f28j(`m)9FJxA@2=V>~ z;=CAu9N_E#i;xoo2T&s-P{$j(?|Rk7xe3%=Pj-Cgzr>&#J|B>vakH4W8p6AdY7%%$ z8TXe)eMpgZ9)^g9u*$1&VoTzRuUIua1GOvfrP+rBM`+SY`Z8t-Q=p=Z-7O>0JIw8x zut*rymg(IGsnI9i^99IF^#FXn!L)R9Yur+qb;6M^Xn%aL4_nGa71HUi6o6}ACT}#E zB2ZAxh%1r`UL#ft`{b}1{ibgrY>4#Ci9YaW^@{m2=P z_5+v;)tZ^r8p2>UL&E6mCk@6E#-CN(+qIj~OUKFvpcC&N`VW1;9WY)SgJuUk81Tx9 znBN9*9qpT7F`ER4Ep^J5bSlU>fScmSR#l!5agAAloEce?bw4`{A8px@pz!UTkAW4t`qn*EeZQLR{hMLiq(Nbnfpabgle<8*x3Ld8n6G-JI+}STH5aIaGg~Q}z?` zfu%{%Zb;btN`jlCA20K|s7in<`>lM;STJZX2)d-gV~KKHfnd&~CDrb+bv<)Wtl$^X zO<31y6msRe6;194)b-$k^8iq28Nz?V2B06i>HYZ$|C?2Fd1)YV-K_T=9|^n!R>~&v zxE!@};3gq>c=YNR00-FhOzPfvz$e7#BE?b*qcl;99SI7g8ml%n8*_B&VoAs1P}v%U zH5FoX|7mDY)2?B-njuWtBJH-1vC!M+RoY==VR1vCjuE;A>w+B7TC<`sFhOD>KCwReodp_Mhie z2qN1~4jmO*eAheetC5!4f~h-KTpz_(MmY7GpU378tepQHiS=K-iz5pqK6y0K`TA>G zVlyHC>=2WvzFFrSVCd)a#7^-l;9-&6GrR8qC%2D1v=i?Fn{j;M)Fj)$btW7|RvOv! z&ct&r2Pnb04AEI(IIlIXRxh~_&$#*pb2Qm+%^=$4ta-?Xn(0HwqsVOn%!bP;=Tzci zzuDtnv!s(ptE;##9mx-plIe`(_P%vU_k?>azW()Bcay_t&SbgX#|djrgCpfRYY;eS zpD$t4yT2{}uS22GOY3gdN0OCL=Fj*F&O#C+Ebj0#{Bzmq7g zPoo`3&N!}gp-AM!RRMWqQvK?s$tCuvBjP8+rW+s%iWsI3T(EQulZ`Uxm3Q4t-rR~9 zDqJOUhQ@<|%q&=r2zBjh1MSxxOit_!jKGbf?G!^V@I zia1K|7cVwHp=CE99x7_bx&>!gd^@@b@){b(S#)5trkNEhsxKgZAjI!O(INNhHbcts z28gI65t}Fy1^c4OseupPycX>go%5)b?~;B?%!Kx6royHkIK^DNCeuK@btr`7PjS&- z*aY&bmr7sc^M?}m!i}Y?w7sTFf{Bs#*{o7a`0L%q4c~ss0##Y7YcwP_Eiu7 z#h_kI^0|q)F)rxdJK<<(Q(;dSg5T2I7vZ>AufXTO_&w%V@S0-cOsFVql@z`=TFu0n z=r=sC!dmvMoF0UeU#fzdIGa~#FV2%EVs(8w$|ZM|oF8(!e%+2xnNfIthqAr`AjpM~ zXaijH7W^2HxHwwyKk@gVnwB*nn@OW*7zb9DaB}-*ws?qX#lLrteelkK`<1H~1^eRB zmg}Ojo2^7CY08n%1g4$({i)kmheXRIvnFU zdc_7M?t=nK8jyYx4F1yuh4_Nl&gErOo}`&+FS~DcC*hwnT5ku<=yxDU$7uD#aQ4ig z2k%2QF`A!31BHG(0W;N;{Xmr33ou*P%PVv5+koBpZ-?m<$A1=9dXiFsGrPj&pL;h! zqXANWoyn1Gsi%NhLIE%t3JB1BU~3yBbiR{J)V3R$iSg#QV)viomiQgBba1Q6727?9 zXdG}JJZuGZ^_eodb$k;#d~Ksw#dvKJ6yn3Sx1~V( z+0&f!e zH3oRkMep@*J{^Jyb8s6!&H`qOQ2kpHQ>O8&t8Q)!b*GSixney50Qm}aIKq7&K=bWz zfbH!iU;aOm?fnFsoV7hZLSN6-paK%H34Xp<^n=ZDvxKHoh4gLj(HzkD+OY#P5!i(K z5CE3GMpXB|Q?w7;>8T)tH*2>3R85h|HWSAI5c3A-mHOV5<15)x>G94fx!y~2V8=W5 z%7E$q4^#x70@Et#Q}~fte;nc;F5QsQ441!=<|3s3hfJM9u>qls4A%$OpX1{=u_@da zG|$^FtMD~nI>4JgNAA4v`VViSpInV`ObX0a)X83shkC)U^U_IV(GkCt!Z-d2EUlNY*B zy3rvRasKZV@4`Qac87PNL1f5l?4#~BZ@CS`O$!{8bh0Ir3uu{91S7l=nuXfN`OdKq z1oG{F=a)R2s((4<>jm$1XJXYydNI+*Q0uuDAgA6;RBn~O#!bUnZ$IV!4PMBq8LR;^ z@HrjdcH+-}XY;V&B!{>~z+86REA5uM;K^OrnNW%s=O>_LvnltCmZ;GCh>m!&(5Nbh zH92+1>3;*%>VI4y5PiKXB$t3T6RAHxQ~n3OR`9p}r#clsF`vk3!p;0-0AK&^Ft4c0 zSVmTg##daZY-VYaMsJ)wa2?x9OP@i=Gber;RsRnO6jod$!Z>~wlKZT9<8=A|Ds;i+ z-4}O8N*_!tZMsDbZnv9@nL3ps7k3gTRwf9h&k&r79s!s72TPHHl{*CvizCEx6MB28 z+mz)6kCI?dI@=2(`79N;n=uY$`Kw&9H@RqOZQ>#X0H96rh|AdFElKS3;NfotPxg^a zm(tCvEqte+*i;NwX5|6d&cRIN36T};Uu`qg(n8*X7@I!oAdJ?xH`y|~5W70j6sR^u ze_bvOM>%|bq%Gn&{VnO%<(c`xF*(^O2%cFSQ|-g{A%1>#!5!)uTToxTGc!>@HUBSP z;92? z3;zyByQxKGex;-{d?L}Wm-@RIjL}iwgx*S=TCAj#%LSF%bmRCI?2`mvWJCs~w2AM7 zY1jp5i7moUQ#;$Is(`4lSCz;)ETK38C36G=AFy2?~i_HlX*CGzNG0ws0oY;*yv&3*h%?LXgg8z1*6FPqA~_M?#4+V0ugV%j$XRfUu% z#T9NB?C8{E=w`9j91bpq3Jz1lto+lZZ|;(a2j$39M4GDm;}c%r(8YnWeIIIO6h!dM zN5!Q;AM0nsTzm$sr%FLy+f3E|-*5?lXE!NLKaQhUH#?r~1=Q*JCsk+Wt~`0p>CI*- zbUn#!;03==gpChQiFA)rXB=NVeJ8TTgw@OY+T*wS(QA+`3*xI6QyrzvgESOc3SnVM zu1NorL^gxn_ty$0;_SX6WBw}E%d*WI z&BRKyQXHVIj5G>(?<}he*b@Sl00Orp(YIE}nd$C3O;%r|hMD+H2O?FNN1RTNs0$*q zTY_D&Oe~AfYrhRu@Jl=7+4nZL(=nrq{e_nI0C-vof(iQ%uOn}cjj z0rY;xrlKC)X=ThbB|wN~Xai z`7zagv*LZab;tuAsq9E$y7=p#l?-oUC71r8^gY>kFd%z99Qx^tn-*Bk)R9V2huPYw zbGfHa)#xGZJP!(FYgn8QL2di4VT26E7HeKm9Djy4C)iId8OVDj)$yeV`mH%P4!n7) z=vhTpr$6B3jfI6Rqm5n7_)^6`l$W zd?h=aa$Gsvv+(*DbfAAFf{gS$tNlmOi0uE=ijE@cf=bV6Olr>zhU z_w^UJ5UT%tLF3e9*!hnks!UUh{=+v8J+fN>byfpO!WR-uC7lQ{je;c;f$9dLBX6#c zI0P}g_##9j&yLj{r!4LAu=nUO^1Thc10Pq?h|lC!VWqC^bFp|BtH~M<56VZV zPo#5i@BeoDuVqjkFH0mO&4woVNX9ma@fNF2 zrDjy4j&b-L4RU)OXj;StQP9NnP_CTRF!A=9Sd=c@NO`a0~+I1keLQCVAkHvWch;=GXF zj2&?Ymph?ChR7dp55*SIuOL4jm|k}|a^y*n+G=}eXJgo^1N8R!dd136-saskec`*$ z^&M-{Dnc^EdlaiIajGAa@T0CzflTwOgS-vh$z*K0++Yc!hBBsbUp{0LsGlbKwqFH4 zxN~a-zBEGZjBsD@{0LkegKtud)br2ZU};3T;~ngS1wZyVPIM5E?j{KR;%yW0gB&eN zt=Qa@_+HURYeD#TMihyp(zGWxN?;nzrtS<>Et>LkZe;b;B&E|oM*)_e6B z&8i0nb#x=0u^#T~ys73$*-PQ7@Npg5Wsa-NX~s2!Vvpg=_W;^yvq;(;CO*w#bq>m z;1|Ub-5K_>l8v`U%|XX0651}nJ&6g98Mv>e2A+Kk6-M%_;HZRlz(WKc~92fug?81FfeGab$5H@$Z zhF$YFXo8YEAHp+n2MiZ`65U(r;C|d@a80t2$p7vK&7%!@yo=p8ZRPiFX@b@TAd4WW zRMhWARiBV|ei(8L5E#!z>mk2idP;|IFtuA%-mzrsW}o6bZGXtxMf~;eO{uYtnT5MQ zKdN00ll~?$?>JTV$Yc5((IY3c-e92g@HU601sEACdgK~&r09Fi3>BHE(iO)V$lW6% z=|>uXXzHn46@(e#vP%TBU_o9;u=(HEZlP63?XC>HqHJK2_Jdf4hhbKcd20~IDK$H- z+kdQcANRbnKs%9B{XRViIjD%yRv!FuhWqd9Rz*gNAMe&X4#dHLrPED8kiRGq`~-3F z+11OIZ#@fTq*1Q;2Ie>Y3fP;c#1F~+Z>=ju{kfb`>k$;kF2%Ku3}c3%t+2~OiY=)Hc&jpSWzMo#se zt*CF9=co%ezcuz%sZS%@U#FVgyj?_7HCe-A=gta*wMqB3=N&oohKsBlhU~6axEIzY zy#;)S<#!Kj%)CORDZB4>)8dr#61yu{3))6HMF31ULqn+=wT3)OeX16ZEibEgS(}C) zVmH)Id)T1^SREa>1UnIwjiBSpUHv`RAZFv=%ou+cDSExOQGlD-$af`D?bjkJ#x3GV zV0+`_gL{_(lfCJC?0e#WTB_-a7k=F({;+@EyADy{9Q)Fwek1iwV5)^4o1S>ri9h!Y zW;%DiaJqq6(=#p{&L`C`6qh6uACNj{po!BuwKvX%*QpxnXP;B;wLe^xv|z-AcF{(r zdx;SHn0+|_Z_RWSdfKNq7^40#r|8Ow4&4hs-zU$+x`n-Hwzo_6s=5~sw7#sjQ8Id; znpmpqc}KE2K$_>@@&6&?@&1xa#8ME}Y%h59%@s+^g@kkKxO`1=!vT5VDD)nm&IdNu z8njLoXO0uRrSF|^{~+j8czHlP{95D*quIx?uRXA&zcV=oBOW^8b#>3w&MsQ%Ijf)h z+hH_&0d*q~DvPh*bjo9a6O*(4FpKE0pRbT_E^su=B&C*gM!0Gcb@HVZLg;>aK)LB} zl@NL`(^h7Ce8P9G)%n>RE46GMZfhV5yr_ZVZ(n}i&;PjnQ6cWz(eX+$IN*trj5F4M z)}}hI2OtRXW~RuYNW@lA4uU`YcM_dph*Zxr-N8n`NRuU zJ{<9!xvC)4>WY=06v~4yL$2JgNkI!#)VC%dGCH?2nl$*V*D?c$xzNe|0#Tch2zIM& z!cHEFahTD1!gP=y>8sAJOpl+-rv8Tc895Z}slG=0jX_@@ahqbZ7>5pyrMJe~XAFLVB9uCia#2W>Xz8p!r?vo(b3whymQdRZ9jPXy;8nR~+ zXO1;5OIQl#@SFLcL(25bkEW?97GVY7zLnwt-R--;q#qHgRzNYP&#pic;u@SJHoJO3Vzq>h)%2YzOow^4N$+ z&Gkps(@e!z^f)9T-n`V!?O)X02D1iCz4aPRm;D1VZ141~kd4Z&{tz$NLY8HAz~?RZ zzc&tej+b>!oRY&%Rde;_gt%Y?y1F`Pw4a}>kYV4X^)kG%uJ~+gz3`YwhJOQOmpYzX zESTeqpBpNtU7ex_a2aI>D3>xTZKvTvea;!os|$2Ac_s~v@M9ynQP)nmg-{6$-vQe> z0JWo7CpA2EupVH{sQkOw>%VIY9x2h|ui_W{u0+W%<%hTzZ(ouT@>ts$iyCaZV9ob8 z;(mAz*=DXu!uQ6ZG%hP-B^| z{ym1)0-e&5P1@-~_z2q^cbR1QzA<2U6-2#R*XVjLBye zdlgw39(^kd23-RyJwpa33v1wR2naqW9^lZz&|a6s$->;yj>n0g?Bxy~;Q0A8BN@rd zE%s*oWGb=>B%)Tfh9v9^Yz$0f0qKv9NB*~xf9rW~Xs2&$Vr_3?Wl8c}udbfe zS9^Xkvgd~W>+f$r4V_H>yCqAze;x~XK*r}QjLZy7jQ>?PP?YcaERTYTlcBlFdlL&_ z%z!oom|0j^_+AS9uS@^ki-n}pBMkL;7gmT|D#P-PPYHq z=Kr|#PfK-V1+rf;>or)xnsW?t3C_iGqL>k&j&Z7)hm~5De|^lNG2Z zQQ>u~uv}GVEWFg^Q_vp(joIWAob^`eGfK{f0N{4eZj1ZXLJ}`;lax+O+Tu^6(YFIX zTej0(kB&zy9Ufh!6>_=SVbOI!GG6dZkW}qZi4gz!V+svU53)p!A+`VixcF}|3P`N6 zYw`cLYWhgqL31`M)>!{p|G!l{w+tC}{C{`i?@iVOAi7LKSR`ZS|ChUfju8F-(vdcb z$aotF&tr|SP<_~_C}%j;2Hdx_w1tN1KECdT^g{%r3+SHdvHV&xHN#ar;X-H1>Pt#9 zcm6gC3gq45ec2`&2o!7<-*OxlqZab0JG;6{z)*MpyJs`W5D4JfBO)X@RG{{*jxN~p zi+0=M`CEFV?E{e@l%F&-GzuS8gKSTmreRHtbNE%hI<1YyR4VnzdlZM0kKL=zVjtc9 zAzjbRB!S{IHMkFf3m1E0ARo^@#7Uj}&Z(!Tw-A!rDYkx?*X@1VlF$KGEy$1PWDm-AMf2h%0$ z0rQqkh)s^i91}YR*C(sj$DU6i92RppyT3K;5L+K_&DN9UIy*bd!Q*794s#~rw*kZ+ zfp^Dk#DHTF$4iq*Vu6m}`x8hf`upzcxDSzY!3z}C(c#cEhyQjb z+v#RAATZ69_^&)!Ll<3L($dB~ zn)^RPjB*acDV%*z56e%hv5Ioc6T8W44D|GICg((QeEISj;f0F%dS_e1nTdt+mMs@b z=F6>%Z>6cf!60_pjq-0#3f3*yA_&}Upi_HpTpiXe_u?9-lK$H5i>8!5Df4`gzRZq# zD@P@p?Au4=`X(wWisj~HwX6Q|cE7^mP^}7Aj%Sx*pu~7IUHrCKwd$np>0z{9iCSLZ zCg*_s>%)Fo+gdp9QRG9;#LiX0$7~^iKukh=PuX-nT*1fd>}DmwwY9aA+x@bk%ObW$ zMq-y8QVdMYLW-NaycEmMZ^#1R9MrB9t5#gkr-!;Iakty6P7b${2R23TBNTo&1jbzgY&lWuf*5)KZfykGmlM;B8?%J_!6 zzFOL4t+zC{=eznjZ6}=wB$lPHHGXqBNPpP%d*9s5eK9#txlPu}FHx&ui4NLyTy{6s z_$BrEGscqJnd}=vo=Hhhz(P*iUm=q)GLA}S$LyCj$*~~8!$<9&dkA0c=E2GaM?%8qIsb=T1u@ZGo^0cFOYg;aScL!B}E{Z00zlt8Ko-bJn^oA3L7^geK z3@5Y444fx_zbek~e4I6>I>Tnr2!wnEgR^_y=z6^xu(=FN{xNmTmA@sUKz}HnKJzLI z=o>JZt}E@{f4)wC7lkEH8}v9Im*}nG`F{0OHNqpUxmHxx8hu=IP9%K{c@E87R#VD~2lDu21WaXnNCW z6(QiAALYHe%gH6e`#sNfJ|Lgxr@Q<(Z5Kw)K8FxPE(^owj7P>hkCZ{?ju3pTkJ%B` zgM?Q-7ZO2!yH~&BgOCG}RjT;u)5bUMDKmja`&}}zC}6SR44iv4*{q8LzM;Saji+CW zkOHgu>GAH%do;A!-UwpGp~%DWY>7=#Hh7b*?63RN^|yOPY44%YQZ&S%X+-s2!GGQ( z>G120EiR@5Ci-gsH!?c499wH9$uvd(;Y>x>KzQOWQAbYa;bJU;%tcs z7f=~Cq4&>71Q#fRUZD_thG3zB|WRyj3 z9FvOrd-)QhG3{o;KHZ#cf4lwVRt+8p{nm0Vi9z|`1LEIkB%*kqEC#J7irh!>HVGL` zOR`XfpRS<#wFTA?l>k>2!_82R#US(t$EMl0=M7Q_OHr5G{YM#>{xK!tKVEPP1_m z@Fdup_Ne0F1ceN_jA+PlPWdd7PL%Ny6=D`{){T}E~eWG=wjAs({Zc1m^IJM zM$G1Z=zA0%$#dt;IOTG?Z8_%>kH~V72YoDX>(is_I~v*w?b+yS3oMbV!+C0L8;R4X@A7b7_NI zwyx;VHh1`Ax-HIZM!2_i7V2%p(SjV#wKiycqlILTRpH^`{psQBOS4!EC+<8Wi1^>| zx@ePUOcJe0*5eH zEr)cOH?GJWW7oUGCWm>)r#Hz@B=oGZBXn;N;Kf6_oz9PDp$DV}{-i*BI^PW|V6YNz zMI&0+B`eam$sgG>e*q|8`X=X1T8(6gnRZq~1at^6myezrju=nFjFPY8OW%j>U$0=1i8RzX&4Ee)*aHT!Stvhbx5cr!Ox??T z7=j-w;JC&A@H81a=J|BDET_2s3NMS8BORJe@SxhCh@}Pc;k*xx?rz`2z`(5QcpXpT zyZkZ3&QuY#9=)1z&>nN!fb-zgvs1lbv$S(AT8CrvG>UKKDYNQj>vk_obUFKdD?3xg zDa*l|^N|V0u>l7=yk1e}*jgt04<>qP0D!bd;m0O1lX=9=!p6@f&@`jFKMN)!Wj{Jh zBxs%w^U4~Nk@_Rwp=WyJ))Y<7$xSA(UF?kT9>?xK&a^=jkUm4`c2Br5Y?<@A*vIi| zjpR}}^6Fv5Zj)q|*>O1b8ko$dYNiu1LrD6Q+l5Pdfy7$uvv>`^uB5p}D)?bNo4IoD zoww5$<$q*zTm4||Bvc4PW*sd?N}l25lz+{aS_fU3Y(nEtNKM1&)Ns;~|3(-mNRVpCu}iKdgn~5!gzl(fDcNJ+(b5c2o>tDHpyP9I zOf6NRDj%}D@=&4MaZ#1)3$}w9Mh8YT>5OVHQ2;_;7^-5&ngOAb2e>ZeL3Sl-Cu64;XD$9GjpaPpWJVl1c81+26=lT z_XJMEzTk|<8puz>Goi=6Ws?8JV#1TB#?cY7T5iUOmdVHij&hzmRU|xLl7+R16$&v; z;nY^(xKH^TGc}8P<#|m|!B!W-S=@`@M~)d`RyVw|P|;m->gGi)|2>DM16fkrJ+ov6 z1*VyaKGyDZOwkgRQY~~k3+&`3myaPIE%?*~2h9$7LLneOIE^8%Iq`QY{2KBN4|=)U zNQjq9Re~($Yc#9osO>u~)|b)9xa2p_tIHbr(95HR<9r9L;CHEM_ul*S4I7^(A{cTg(s(B$W;-ip5HIS<949Xd#jD~6)aLWaPvXzDE8kd za77(7J3A{WFQ2djp@OpnF0U(qD}zBWd%NWYL4`(^EZ-8_=w=4Hu)J#U+~YDX3<6*f_Y=Be2Y zl``gwxU`7jH=#)|1Xi3VsHdmLR;~Zf=FoVXr|11T@l!hhshxrY12i7KU;88C3-Pv% zFTXj{eZ&_bM?J1zN8(QVW!|;@x8DkFP+*G*lV5qO0D|u*%uf%Vw`wEo@<)s@Q~9xp zy~o+vUYG`5-M(w4UKcCS&VRb4k`sssg~$H7(^+34)3wf0jyXkegWGWOv{UY}I6OC8 z67QUCb}w{EGGOR+@O%W0q~7m z)^dp#PQa~aj`3aX_%?mH+C)*q^Wowd{=~sQ8U1B9QcyB4q)gna4Ts`a{jjwn$p?tt zPJ~ZN(^~gQu*Tf_lFfLKysbQ(5ssh!_*U=CX1CyYbJPac><2W{z|_c;znBc=>#l`o zOV5Cu2%vT_YS!OIa=V$5f6KriR+i&Y?{MJO-P?N`iKN^E1&V-hr+pZp&RCgJ*2XMr z(OUq57F$O#tYOfxRwmZ3kraw__os)esc@FEBP%!y3Tq(t9!y&S#o*N?KaabPC&ZuEeEHxpVm?YOg29y6jLGpx!+fq99td)%@??^HR=Wc5o6KOaP%Y_ENg%@<)N+T(CWZCza(@YXilU*R zG3j-~#f1S-q@lw1=KIh74fMZmzMLNN_K*4IU|g@H-s7L2x21ZJ!!CIT6-ZE9&CUoM zgleSH;k-}w_WN@+!Psh7N_3;Tv%1utN`0S{C123f-r!=4t%H%wK3z-A4!OZL7MTDK z%#}v%n_={1*9i@VA*e(lL@4kEEghYuwF z=PP#WHU&;q=C_6k;bM0VhebN&XOj4@r(>@9-l;>!!ot6FlTS!{I}7M$*PyogHBx02 z0t&$-9I|Eus02-Ys-Z+95}wI>9sPPeLU1fw0y%$|ppUVm!CTS`s@nbTav<;5G78oh z=#*iFgB6l$OacC2whDGs@QLp^VC6p32SVfegUTVN2gMMcRA{ykp-TKwxP`j|XD+o0 zX`UoL69TomAGeVf#^1d_SI_&0i}+GQ^-H8@oZQ+_$-AzM-AtMH8dDM{LgOaQ$)fHO z5vSA-o=pdoo11%m)O4J4GT!Pw#h8j`{j0$Q3k5*Epuhv47Yoz}iHV7+w7PL?x$II> zP*O?>)6>!Y>_xqk2Lr%RUYmVKQdOv^xRO=L-M)UY2dAm8&s`}FAhfC9^{53VCi{%HvpOcR!vzr;Q`AqD> zRoYrmbh&m3>Oz*RUR7vUM?$k6)HR){g5tZg=(%=!mN>^NW2=77FU0XwpZK2$DTbTf-Qx6cQ8%N3Kkcr4t z3hF@IJr$2L4@_D+6dnxl2sr#MY@ZYrZRRcNboV9;`Or$@HHScI^xhQ>j&S8(y`W@X z$N0q_GRHZcny|$rU=@FjsA?0UP*75$%C{imb5b}e72`EMoeU9bMlUvDMF3A*8!*0D ztS3b|KriuKuY6N!aj97-wCfGU1ur(TdzM;Pv8oE$p<|>tC;amu0ss3!K7%N-pkZOp zU>Ag1AnD9G5x`iT2Ffor3sOWzMR_})Z4G#`RX!vcrwczlK0K6Dwjw7%vn{i3;}rqK zEgk}RY60s@^Z^6b7x)=icn9N7P%^S5s5GkswSRQ958!G1qsS#V>T7Mj?PUP{xLAfI zfP4lQnXT{u#kjKQxZDHm_8WoKfHV2+DCzN05CQBlE{iDG@0!6+jJwHeC`v8g2vMS5 z^J4-$$J9MIEb;;@eze=McDy*;ltg;=D&F-Q1T0e4BB9!wig(M|U-YJa*X9b%;G&=C zgElueRoXoGkJ_F*07Nl(xP61KD|n=k;x7kM1H1sL=GD>SCWbD#xBN5J43Z%sfg9UK zx}9L);ZX~>MGh2qosq;Ef-uA87cZT zK+tV6q*SkO0k|YIgrN~`b#)c7VPK)wg#;+L2g##6@EmxYpo7kFDd_1D2L=W_&?>CD z3?PT=nj32IUW`EvvCdzy5-^8+9d~a(s1UH2QIFMvm+m`|oJ=ap4;WGqdEVQc?@bLMk09rPcwdOPp`jQvgNZzGQc}{#MW&D}W5{8$$b(+if9fVQ z_|Li{o_*2^E`ad(*86f_YD7$fuwW>+T;v&BMnD_i8c7QQXtbj5obx4@K>x6)IR0<| zx2p)*Z4*^BscgvM7uQzsqGOE}AK6^~Mu8h!Yc5hQ>-BfAi7U zrr>#bc)L(opR5Qs-)yBMvKYQrt1)c@&ubEn~voRR9YEf{T|0@Y^V*6z)JsMMxiVI8E8> zKYfItCza4I5+c_F=KJNR2V#Iv^~Ys3%9M(sdR(9gUql8BV=Ex7m-NLInTxdV86&5# z4IMH8kByIs@A5na&mPWG^}*kwM)1Y`0I+M0_^+EgQw9OtywgUMM7i~<$j?-6^s6OT z#$(gXvb5c8n<}rZZNmQ5m#X)I+SB=6tpT=X{YO!0XAib^=*r59OK{HP4@1b3#~Hg} zs+YMlY9ut}t5I91n9~mPc4h9`<4`$_fM#QK5NNuj1<@)Mx93R3SPxJptN`dJcE0QG z%PX!K7*Q|iYM5Vf3n(w6A`gGC*o5u4U<1oxwG;|CUZiaCu-9CuVbE+DOST%SvcL|a z4_T7u+LdZqQ?FkmXjk0IQ`d;7bO(%~vLooXz9Xm_<+yIyBgAfmI{SMD;}q2nOwvYzx3I9Vt>egd(EyhUj=KgU$YBq6v`Ey77mv-^I<*!)^|)a2 zZHhywuEW&xubKZJrsyrF18jAPe9kO?&dyE&W-U`bLjdxdFLbrnuBj_$?B5+sv;@ns z_;UNj5dZ1z9+9!<zmc1L9Ef2M+E*(1QC?8&t76^o!8Q&3V52Oj8f1tW zSAg>LS)-#v7!JY2cSRhF;})v=^CC5q9c{{7wDxBYk@I{B3k}^d37=X)1go ze56-=bhIADS3u6GG9Kk9z484ULL{~a!NtbK=>oe8c(#%_PkYU}>$u!Xrmxzontb6(tOlzh=cA(#8j{EOX5WZwL-TxmQI(b=CaN#yxT z1Ku9yTyi4M%;T8JMdk`mhChL;PM)&Or z$K?RTENJhynTInO-&7H5SNcgwV=pX%PmSv&=CMA4C*j`GmUTzzi`fS29?KmCNh>YD zDFsu15Xvr0<&@d;dDKk#jISqS0M{w9h_7oLehgMZC31EqMv3|wQ~>se zydrKU*4j==z{M#T5I>6OEVsJj2zoT*@Gsz384Zh68Kt)!-$>7M%RK@3Ci&?cQ)2LC z&&CF^-I|~8j7!SMbU#Bsxip>|>7AP380c!XRXu$xR5&(B3rAC;$BUmCZ)r=q!;~F5 z3OY*U3MPmL-*zVm62BMzh|9p2O&@NEtv}V9-9OAImVgvyvt&|&O2a{zsed@FuR@*2 zK{WI(3gNuY0yjvpk+Vn2LO-q1dbESDcrMcWl?n7jsb>`VXtW!==*xp zR8{UAsVZ%yyV(YG>@A5yS~Zh1zlh|-F#Zmidkyxq@~8r51wfgn4aqh|2uB zOr>}HEQc#~uBs3NcXvK?xT&gqBj9s|q$A9SU|Ob=$*oQ~vpc~N(~yxvJpGKsrH1Q$ z4BTz!E_MCIS{d`DxUlHx*Kdw~y~)2YBmFHzOUIDljPGbbDpCFh`|xOn0p=AH<8O`; zk9{jPD?D$3wDk(DYiV>cB4&BCW|%8Erh})s^q^zq{2)}nC@w1}{Pztq9gEep2+Q{l zIP@uEIunT$%JwylwQ47o&)ZSWo*c)e9S9m4I@3zW^>se*eL5)2J7xwg-z6K#bh#u{~`J-NIh5^^(} zi_tnr+3Y#sHbo+XgW-GPu)lT(#;B9LKR9R*h7)IOYkRdDfPjjI=Svc$^zML%g1xsS z!BIGb3iVn8&LP9C2D!)2a6%j^=rwbTefQXV4gS(~o8Lx?AUxUCR9h+PcRYnDq$)oZ z!$RO~)ankz|)A$ej;^(4#7<~hVwr8aTbF@h#O2iowxcV zFcWm3-zk1Df`^H2ev(4!lujew6cu=Ja?nPx?qHEsA$3AF|9ZUFjM^bS%g{)U8OyB} z-?J1@*~qjgNKkpzB90nE&H{3L!Fe{WgV9F<9>JeDy zIgSm*f@tNp>>CkpxQtw%Rg|TNuJM`>qYqb8tOhr zv{eHEhv#k~43TjrJorpt!Mm)Do8-4ZlfEq{^8?dQ6fe^~Q_9ToWRtKIU)}z9^i0Lt zpg+F#aSK#1!*~Hw9s4b#mvF08IN5Sm-vrXueA^X+GM+y4$+r<tw$qo^v$Rw_`V_176x5zq~v2f2# zl=P1FnwE^^1*EA^yg9W{B{L{I-1@aEHuL3Vzur?_rw=lHvKZt;L@90X``vI2(oa#L z@WwTwfdV{ChbrG!8sP&zot+-I=V$8UZ-4ZLLE)4y_E4#NUX7Lx6e%yW=Jp=of{DGi zOkaWnNXeC%3VmA)ys*Lik6<9#!b$-HfK>{MyKQ0cpCJ#=QA?N(Cr04V*9L>I)INjo z!f^1Jn=vix>(RlG9$&}oJM*#4J4*>0>YX%Tx!DzDcYo5qjq{mJp4#PdwC9x}m?ElL zzQ^Bx;_7uLn|_Xms@@8Us=xA2mo&vBUyNF*-WMX;YOcYAz1r4Y>mg)5 zO+~u?0v9IVR*$OL*thzK?;@4=c_rau_xhMNWjxRQO?pEWFp`&+{4KmRVJBEx{n zhGkTh8dFS%!#O`-J8JH5AKG(F>ws)}rswOJDp3ETmdU7gK*x&$NH8LRgs2}C2e^sT zycNA^wEC;rv#5`)iacq2yP#^zsUIJp*&gIFq?9Sk7~n5<<->&V`Rbu9o0@5}Iz{DX zI4Cm>Lr|6Lw0{wHbq6w?k^cUYQiEiuH@J>p6~G;2W>9zV`2csPF1B-!Qhc5dy(D)Ns!n7=2V0%}1FaIXKF)L$7f= z2nlVB2pfb3&%&uai12YHmX)>3Lh#RCXAo zly$gA;(cw1!#*vjK1iLl@*E1x6=!VB`g()cY}Y=u?wlN6?JH7vL3^hWuv2V>Ic z6VuQLZO1o8i+1l8XD5Zw)P34KpNZ3rCyNeZPlAq3bRNCV{!2ZVg4H}Z>I-M$CiIhg z=#a>yFC?61@oF50=}sOR3|cqkRB91OD{1b)J1ToR5p@i9`s5 zw^+5ev((a*YMWhp^N$h(>~ISFK*0KAyR%S$3I$9)rlsA8km1patl8AJUixlvoKIlk z5Eb0ekn40(E3lKkp`skp7y?z>lyR*X8X4T)3=4~k|GJ=5qlWpFd`mfMkO-5@PQ1XU zw|C$kkNazNJX;oNl;3P`x?pPCRVo4nO9sSYgKR_EK6I#6h^sW9RHDs`>xZgDKCv

OorO@t5nCGh|mFbUWWjQcwhq&P5Ow4wcF*$BTEw%WEhK}Mg?++;OsZ8mXx*C z!WQA|nO_wvvZkRqks+l71Exzm{A6qz- zsy&kcJprWOro$l+XY|ItQ@pQ$C$NJ+TR{aw&g?E2{KG-}0>~j=$ACH=4FGP3Da**~ zxqhD6QX*WBU$bXQN;Hc{p{i>M^-kJ{TOw!J&~Ee|*pdgY*E4pnErwAsf-l-!f(U7> z&=m2e;EMG1zCW~ehsP{G*x;x?jfjvl8|=Md_Ot)fFkq|vGx@glI~=&>maPN5?lNY$ zILu2N`f<+T&69fl7t8h&V&>NtLq`NZ-5;#Zj~3ssmhPC!>=y@p=iOC8(0Ykl)3glm zJt?er#z@}~@!>pYq%Abq;Va=2nyc^6m$xD1BUPp9VuPy^=ft4?-n&UFqE&gi$YEy3n zU%k6DRV?$P4!XowCOxtgJ?0XdDS!LsYftr_>QQk$;iuLhM;`6lf zi19v|dJ=8&Y@oyaWhC7W-uZEHnSs=IdEi>PK(^p*?6@_dN#-Zk9 zsA{wdh6P&U!)`}fWW9o!@Ai3?8RSp{+*VdSyxQ49Y#~2iU<;(XC#?dl+KDveCFn3zbcY4DT0m}?jHrmkh z67=1Yos3Hm4{S0JIjun6@KSVFU1aPR5&qD^8pD8p)GUJn$qJFroC(ebF?J`blIqcy z?^A!p;{C+osFvtCbVNt0Vz|=DI!7ujbIKRx6`%f&168FSnzz7s?OpIa#uzx2R|SyT<5D<0vyWh=_wT8i&bqlk#l$@RGh*bwdD%{ou23cZsm zKTH65GOTQ#qQ$ub+%mv#I1qBTBHOJrJ8d4iIbCHX& zVFT@8z7F}g?MVQB zMDM!&OOm+awS0l@vRb~!KfvJSk1kOB1;?glw?Cwfw3Zg(Nc!3QgLQA@`}0G*0KN)b z2-)$4^3}J=>=ErU6%ZbM)6CaMzb}3uiKR^xV>JVA5~P;a7>95J z=?&z;4W{KftwRXz&>@Wn0S-})>-t6CX@_c-SphN*JAvf5KzMX2ujz=3?QZ}KKWc%& za3y&SruaP8Uw+lP6fos!*niPCIMnSBz6OWNt1B)RpfffwY_rRKxasr2*7L^+gcnzY zGA}$~nL^AUEvzM1sBx`*ph*K&BMx37L*vw%gKv=!hCMWwS{onLd*!{BW&2qP%$w>D zuUqUmYE*yQ84ibuus8~Yk_U@1wE2F)his@6-7PcqOsU*Cl)V{?S_kL2AEuOC-$K$0 z*2x?(sr^Fw5u%MlD&7E{T?@iHou`_%a{VGxBQu*uJ1=)ygN0AX=VWI^Aj8<*RrMqy zn*ygW2Cav&lx668dUkB`wx3o^tYL(d1t(43G<2Uq0RB*29AA-MN+_F$%vV8~4vKG1 z#rs zTM~6&e-jQhz&DR7tSf4g=JkV&h_s7R6pj=X)-Cog28p0?L;7T}Sz^*d!5I+IpVlS% zOFn|fpSq1u_3Xc-G9N}BRN97A2+Qo|j9aoL(`v|2S|s53#WbO@OW zO#5f5HIY&AOiU-fXHC{ll>opfYqDd z^>QWS>7E+MEsX$Za&F_>aZEsPnp8#Po)p3!E5@SVb173Hln@0_gqm5!ZacpoUh!P+ z`TOFaj=tr?zLev`&Y6!EHYGTV`O|s4A5|Uy9F8PHvpKp%S=@UInV1x&I*b81Afb5d zgTx|RI7vb^!1i2(g+ToeshpGcdCE0k8^Y{>nV= z@UMPq0k#g<`g^>sb8z4uJ>wBN<_{Ljt8UG=k4PAp@#`aIO}t^HMWWbW-hEn@`2n>N z4Nz4GD6Qw}AZ^bF_B^??V6)N&7-atQ$mQB^uZ#im{gnUTcQ`UFpL>MQ$zaI*j{*|^ z!>Xcy12m^SAHJH#s93IT&LU(s9|qt!s?^1fiQSa$;?|_pf@KvUUR2VND_D@TvvWP5 z1)M(TFaU1|2!Hw%E$BPU%;IedC=N>WSaJq`Gb^a9MS!N$P*Cx&ecD?_(<4Ivq*ea- zmyZ5SYt%V#(ecAzlDXrbycKYU7tByS4^@rDaW}4DR@h9=$}_gjbN>95A*>qHGiQ~( zV#svfftgFaSe3Kb%C2HpBrwWqO3Gs1I?JWxlGC$;%)kh<-}Ju#cg9<)5o6GWtPb!x z01mGM9Yfo_Egz7_YI`+&8y>W84Ejnb^dbX(Z2r$=0Kf@Kk#KS#SAAyn1aH=9ugKV}x1XF1W&I>-G(yUn8& z$$MWP6#ooD`{4gP z(u?8+WSQP)dHdhmr}ve$oO4rarOYzK#0Wjl3(MBmZxe+qo+m(E?{9*m4oC?tGnW}` z;0cIzTCRue)s~BMAFi}6dO*p_re7so{x;h(tz4uI%qM z2vF&81NK5EF^Ke+WX_C-!lGC6{I{0SQm0s*R670U=)SQlsj5H=)Z+vJAkJ zKV-m~!l}n*1*FoyTF(pHJNc}yfA#7Wd*Kdb+CPE%!UyFnxsas9P? z-cwCbg>*qp>i?m;w30=}#C%KRb<9#O(}F`lIQvqgfsB5_0}MfPcp~jTsD%@bfZqiJ zc#C3Z?E5P+GO~rYAD;ldbIOz08tR#voPI6(wWY8aOa9*!xT8>jiv)-;b9XeSiz#Jh z;EkH*i)nSPi%G?OR*cdVc$H^2AZ=60`Zr7gUd^G)gjO(~Zz%jLtUj3h=KzdiL+yNJ zcR2_9`upp3_&@^$^|@G$1}ypsJkVJ;pCnq{zxoA?|i z#l+t!U06XOZi9zzf&d`aYz4=dxHK;?%_8P1ajY&NKSzKLB0Fy(?kvn zJfo4+caN7U*L{8XkX7!ij%q)okWW_dXZ@FEMb z!0itC7z-Ydxo#QzgBXyf_X2o-1Gz0PjweAqYfc()JwG_WjMQ^!@DW01e-PV`URGxf z1(0xH3LiTBcdfSXLp{H`B$LL2$=G}XqWXU*d&{V*zpZUpV5@{mcS?6|LRzG|8QF!g zlKtGGC3{wK952cKUrX~Goaq4ZHO%6!`7A!`0F_ekrz~m$(Ob3u%fQ@xJCTCK7Zr*dnJq_!wp3$h3!&ViQvWnYM8`LbB_9-4A827+4cSZ=3Pfq zKr7$=XxZmXW4&9>5{Lh4=y1uEC{R@ObXUg?M8k8b6cDAsVl`H56oV-q#e-e-zIjN z{#oA)AeC;9w$6I$dbq8S=5Ch7(KXq38k0JR7Ef@N5FQ>u#IXORSz+0jn2@J+e$d+7 zL?fmpMMKm)eT$+&uD;;z*b6EpuCUz_*K}HTaQ@}>+8W3e8e42c53vaeBrG(qe^ymR zC1gZ|20I^es(Z&?K<64@lF+n+{Cc+f7-)rbU6U?rdoCEWSm6v+xC|UqRoO zB+SHU!@8fwvu!w{lSQ%=s*%8Eu_1+pqQDTN_jQ9R==E!roy5t1()dr{);edhE-Q0| zW@xpen8Cl6H%w>00$&`CR|jXM`9c40A~_OQjMj4|+wT;R-ZBD|+2Bg{KZvQMjU9fcU2zs_U1&Q5OEP29-q@fe=acev1|`G2ov8yQ%WM!R9U$|L%wZPpK-iex7TN zE=xcPRGFDK=P$mcg)Ai5xVmz#JnbTE?qHHoF=5w-epPv7Dz(#VBtktw0kYXfyoMzJ zGTwXDn#~deLOTgoR_=Pg9x~M`Qwrt#w4jH^_k9DXIRo(Ga&kHtUDCPfaP-qbZ>IAa zL&jpNj%Va@(u$6hf35}bwbCLOvpS~6ufc8XYkpmqiM2q(Bb@=ke~jtXF~-yiW%`z^ zcj2ke3mHoB1m`s{OyWrc#T+RG43 zF$Sd13(6!73Z=sJrv

#AlIG_U(%@Lc=uIBf(EWmTH0*Zto{rJ>5+42SkCtH) zG`ZBuMe5^JM)-=a5O2&j8Bv77nl2T)QcOKX*(ZkB^4-K8tBUf&-r=ehU2G}}*Die+ zpRV3~kC5WAo2qEU(L7eRDWl+T)?DOjQ1mE`yPeG=l!xBOP{yC0*r_Qi?^W7qi#I|q zVroeBF4*J7kGNG-(hj4Xt`f)l{QY`JI9yLf`Y>a@6Sr$1RYnk9@@X%{NbGemu#J_q?Wh}WP!q4czq%f$r-!AAdv znR25d-sOAq@#BuXQYP{hX(^Wf2(#aIZ=Zc)KMTeftdFpWQ8#pQ6U-%-bbEU!%|rKV z=;Y!Fn?Lfwjvfgix7EJZjvNJVzE1Qkc9n&MlAq(XgG-cdi9C8p}Vt+dE*AiJSOKEzyx2BO7&UQd&s)lX%ZMpIub4cAzd3GdX3U zJnbEWZ>4ZtpqXY60scT#s;n2s=?8Xk?x_GPb~Vj^c>x3?8{2b$pkrc5WYT&VO(b-a zZX!MY2qi}yY6{!w>2nTkVj~#)e$2^XL9se1BN>BrO3+^B!SdSvB||uo`c-tAftu;^ zdy(&xq@=ulYVTZzo>&!Hf9*-iddDS-cx;f?XP@61o}sMtIQ| zR+E?UUCwPCa(O@2UX;gkV}2ji(dmc41k4n>0kU@vpgXge2K`km;t#R6w*|?l@c2ZZ zF@t3Sdm8mdg$FU{-cSOEGwRn`(6gLfYy3Hxme4rK49rK_sHsu4J*`oc-kAj$Cto|> z=PKCTBvp4?>W0r&Q&?x8eqrvpl0PIEW1u%j9-mIK+c3q>WQQ&!D54|=?MAX_;^>=%Q*lv} zhGgCG^DfYMUga2SVXd={p;3k9ZHV(#H+l-Qqc6j3t!vLEDTih_ZmuD|&o{uMiTQ(D zOU%g$GIE6Pj3bF2=V^3^Be#WykeMALpl|{n>H-^p@<^Z=^<;fYvQ-z0aP+Z0?oCnMIG&xinIZ*>R>S7u-Zi#6S+ zom{8bY?HA(R58V$#ab!!_Aqp0qDwFfW<3>*(*oqY zq<`NVO3=5dC6LUD<=hFnVTR>P5i=gu7;7@-()^~Ug*t=G+TDA-XtjzC6oxU;)jML1 zjt7w41|Mk3-p)>Fx07X(Ym;-Ms%hl4EG!m7j%Y5@eteGNhfD<#-hCJ;{16qY?I1px z0cE&uAJFryK#*B(C)doF&oMC>@hc!hS!Z+wJfE&Cg=#arn-IBFCX%)YDAPRa;aVQ9 z78`&+5WYpz?X+j5Y^3gSkG18X1DRGeE~tDK4;dHK_cFdaU>2UVp1up|eR?fYh7P_* zfEs^H+8*&2lZAlmqK8o+^>3J=g~ehSD2H8u71VY#={`wJeheWs!sVI$P+1&DMX!YK zklZqQ$!&z$^qHB>04qDsmiR0-@0QCVF=Zb-q2IWGV7$#pEx6RoB7S37EnTnos;;}%CplAZF`hZ>Z;ndjUk2p3%Iub!10kUBT*A(j)fM6bTY1(>CS;DS zO@iIo`trQh^ddyCGHcKCTs^h>nVtil*Y@Z@oAqph7FGcPcVHafy|<6So7Pjbi8*D?++zb{elPJ0nCBuLlK$(8v;uI9QIm$ zaI%)~>rN8~T8AUze^(5RxB8NkW#`?*8y;56!I!vo?`fL(*~mD#QAoZn`;T`~Xi&R; z?F<@Z_DY5Ynrq?EI%=3OL9==06JaA@NMJq0;qJ6^wy3W!_a<4_)ld|e!DHyE7y*h0 zEh@(a@T5X82*EfyIHVzG9GS27NsV8jK3uF&-YGIlh$7$3rHJ{GdzSefE`E)ub=`QG z=8|%4vFWcw;-Vd;gIVMkX+ceC-xZ$}!hy4)nh2bYA?=|cip0*MX*vWmKKs?0@eZ$2 zdD=%FR1foh)ZaFpQP=G*F+@w8P52{xjji|Q7S#O(&-KG1a^LVU`fsJII(2yem*B-^ za=b|5$SM!^w(AGJxeBMaYql0mci6DtH=hcmEh1BwpoV=TF|g5tD#jS83R!I*ean}o zt~I`87L8}dW2avOjRIX4cUj{z60^~HDW6}CrNFbL%ajW=j%b*e2$i(BA`_F9Hj|E( z!XL}b(^Sdf=u&FC)p!avhTeN|p^r#t4O_PIXJ%%;_|$F4%Hw*`0gM<@@#VT?;Vd@M z($oLwTC{#Z>tVu3Ue@glvzcAM(BPc1yQqP>gv07=# z^sz?g18{yTHCI?c6LVbFS-5^z3J23f9ADQ^p1Vm2F^X_J zYV71@;+EzG1r<_t-OwZX6U(R-w@>;v7R7{tm+M>DD3Xr}b6n9g8@eHRQ0z$!DG05Z&#Ia>@$QXbb zwEw`0YzTPf+qt)|k=@tbAZ!L-j!zkd1BeO}3Y{>l@tOcXLfmS$(%t*Mo{Ss$Z<$O5 zF1E*zjOL0`ZKn2oYRkC4*&Ipq8W@Q4(|j~E#RX!tnX*>d*y!rkYB{8|Kzo&pz)c$!;)ueMVjZc+rfzC`> zsNMAIfeg8YJa_6Zf~28fu4Q$!d{tF`mI|D28=}ZYoD@-1F%GUxK~%%BI*cmeg6UO? z9tCghrjt5nL?_LnXSW3>A97MS)>%wtoLM#ng=yB#;~AYoz}g#hys)k5snNIywrRaFCdUbAf+ ztbP_8sJx^^A-o+l)?k!DbUkMG=q!|h4C~F-sp9{%xmOBAv=8$ZdOeko7`{<1EmgPh zInDc|B68ZI(2sAwDX@YO9A@RU#zU^hUmt=r%Tadw8U-<>zbDCa8nl~r1xz0O=ncbV zayiql)EUMqDXIA0zSIKUa@OVE@~0YUGtDN*>}_Lwc(O>#S^50>Z}ey%;Aa#@8xTr#6p->P~*~+!yrc=2d{tRa&K=SLuXr_ z@1^DQ#)#;sK!qJPU+jlVl%+`)2$!C9ZUvw~hv}1ni$K^R$Vrl{Ws+}MCN6Bx7Gh=k zrP-!y#&t+&9rOyAi?>FC%F;`$%1sjpVyY32#2>*1SVL5doWO3DT0w<%7L~r~{E;fp z-Fm_J4v$_~$c2oU*vDpX3?tE1n5%Nd8k}?rctilZh7Ro7Ibc4dTJZ%6ViOWF_s_N+ zBV`up9Xvh0#I64>pnM~&zeOmZW7Br6;;S2_V&eA8>1WD4^yrpP@z*WWstMkynb@!Q z_A&zmt9D1W-@|jrJ^XZw$41(j$FoH6aM%&-xcZXs8;?4A#!NlvkH6WPIN+L|iA=J5 z5)knHf)U<#LfI|M{!u`anUf7Wd}TQx%VkoT*Mvc(69Bg$vtgo=Ek?_6O(5nKM_A25*hTRq`G{4BP|Z)a2@`dQ1mEok z*-!7584r%L3s)^cG>kDkUo@N}o(0m`z&8PL`+Ph?_Y)%*W^!{%KxKZg&>R4?i6&m$ zY8l!I7{MfL(=R6Rb5o2V9l6t)yc8}zTb2CYz&|a|jwkY1Amw}MVcH*Ne1>0I=kgS2 zA1L#!**aN`>bsgtCtDCd*zpD|}5SnktqRA&IPJze?wI}2bN>b?MzaCA*xK818% zp*bRvj-===H;wC`-@%8#^CKjmuMvWjAFfD%2?YbquMec5oI5T1d>ywU3ee=@uDkH4 z>uUD;Z*=uG#z=Xgg6Mi~sdV28@Fsflp#XD!tk3%YM5ZpbN$8g0sP1T)nZvFa!rv_U z7LKN$sb}bDKWgSWch3mD3E0j|+v|6ArcQVXh7;_N6Wmp&AeY(Igwqb^UN>)pByd9` z@={aNmV#`Ofp{LdksKoLp{8_O}pQ?9kg=OA6oqA4y)l4Lkw-GC^g2 zb>^XDWNPR_@HX!&(+#VjzJZVnHSf;KG_C!rA*QRp`MVxlLARhr{y^h)i-8z5Ik_JV zcixT-&a%;l1ANb*DcgI1^&VG z-Pv+N_xn?k_kG>3-o3D-0(yZ+4=Ufo~KLj~DdnH`X-T9Z6vo_q|_&T~-@lRXLeH@gvtKzn`GAP43MqLz8RG zC7JI|b~AP=Ix|XaoRU%YPb2{{t%wfB*c)3EV?gpSwY`+h%zZnrW4@|EDYQaSE%YVljj*<&RI_KE1{Ru3mu?uss_Fd9I#Na6Fmb;H6fcjp&<&Bor&j_=$i|Zb;E#jCm2UvE(oDmc`IN zZwiv*Fh^|1N2k$bGuDE;g6yz|!=j2pgo>f7J4BRBpYvVLmFa$hOLnfDpUrxqMBVIK z_X)K$F z;Z0)%?e)y2urJ%AVlZf%OE%)OnJ1>st^i{%9o|2_10%$lE3jh$m0Z7DQ0X-4V$JSa zfmZGkBZPsP9zAEYEU6tC7+DC}ITf#*7x~bet{KX9iVoQ1JQRHTYR#U;B_9_~q!8;# zl)f&t?sMMNh=vMQxg}|n^U;oQ{GPJwUV_X5+`*OO(s4va6#!G)K=zg#CC<@cqS!v3 zURmia9&}v#j(q3Za_pKFG!Coxao+H21g5(XFdmPxID<**0t@i5=W&tOTDX1OyG|IM zG39@3DqaGoe1({Q)zSTSuK?iIl9!}}rcqH*eMAUcK!9p(#{>o*9%5k7Q36W$cyys! z_W&pYKVm}JGwQbeF*AD=g0TTuh(A6h*OP`_cY@DKsh5+iuRfEMM4@%@FD}ZkhIP0;WY)MbN7U!1ZWs4 zlxJTQ0B{%>h;CjU-({KA187r-u^_Z51-Hs8w_#xD(F%6#;uNsMBY3*s;j@|5a>~-E zw}e>!eN$+RFv}Wth%Z|RS6jssl9n#Ij?PATXhPcVi{y6|{=%A>*P7w+5B@U{vGYU)W2SW zb!D}+gV^1NJ9gTQg3>leWpC1qybj-MLSf(jbw)k z$hg`QfxdyCyL(Hqh?$!k4@~SkD^TL?j|#b%6FVUD(7?dEYOpbYq(e!OcwEgJ(UtWO z9Lpo->EvUnRkt#OrLKN7zY5Y;C;CTV+};HtNc z;uw>(-Ov3VDSB9pO|kUFYIG7zmE<%BHH=Yjaf4xw4@Qu*!37Owz0&t$9ML%W6$Dr9 zT6Zc~1>rc@unN3#maZkmf+*of-%lznov7$V_CSRZj^DR5ufepFxZIb|S>2y+}Zf%+Hoft9&eHk9+8%n`Jj^w5yp5he3{My$}NBzsE_hp~C zQ1r$2I=~MKuBdEr=mL?bX2oLI{0tkE#cBN+w}>L?+m*qddEGEwV`}+hOIS%wsr+q} z!|&gjQYEK-Y91@6Qs66|Vwe<46Grb1%At?i6IF`#6&-)9Z}k+GI-DrsqHk1Q46Ac} zg_B9`LJp<;Nr-j`jD{F>;JK@twU~I;pAdtI*;_^5CH&e#vw4&ALpJx0FSP|R>NGtM zc>;=lmJ5?XLtV<78uL2@<0>BFxmFR0lewQoqN7M8ianQcM&O@mJAZ!38CU(hfmp6y zek39xfvI0{r7sF2Ot8RV;{YrHNQA(MOAA%I`K=}mkF^$*l!cL}VY?chUigD=@qGxc=`hFa z*FFZrCA(&Zz@m9aqfVKi_kX+!dZYaC1xhN4@m7i`KGnX(UPrey374v~PjDH@-x*UH z#%%g)conoFe@?!~$qrLXBe#zt6ujI{5{hQH}Ifoy8(Q3+$ma zz#Ro>|H4Y-hauRs;NPFGL5oxw(mR#PNJ<0%Wn-Oz96m zYe4VxD>N{W-_^AV?{cBRW}~U8#-Q=uTVUyK1>?SfDZo%K%f#&id75%ypsY<64FhQv z@W1_vd{8|yCjB9H?=pe7A*h4)?oFfkd6ql8LY1;NCRxcuGM_`K);3BNjyU5z zcAnP$toR;K4IL}RD|I3v7~{O_AHZ%vrIrfm;^63(Ew&pS0+B?;r>1c-j%LwD0M?GobFl^0Ox`LITO+Jv-=GhMBgC^)TWjbK^i){UhV?1-OiUNP#qsa3n7=Glj}5AeG3o zU(jSru>5Zb{Zl;*!fi%R44uja8y5IN7dT%~J_nFk4*W*9TmnF#CAS0%iUVW&o+TNS z!o7LDqflywiAnOWIkadJ2INQWpJA9_6v!>9U(o;cS$`dNP)s>Xx{(1IKzN)Wo%jV$F&~i z`I8Aal3K={isay(S{!vrAb#md2ETu=l&x@w61uMy+Q;~S{Mmz}fCghMwlC`vj9}~? zAhu-F`c$$h_Woy$L{3A!rXT4i1m+voi3|A6TB?lbG;_i9$=XJ|t=*&%wuQJ54VBU9 zdKEZD*EQ3JCXm|y`0^jIO{%1tFzvE|>3DdOf(q!-Z+2KE^_xKsZ@uJv)Yr#t@R87a za>n%8Pa$xwn{Z}J3mziheZ`VFJr~!a(Ir7SF~(T9NWeMdSOvb2n$P`k0KQkohX^d< zY1N?Bhf=~Vmz(-ogNzkn@d2X+VBmx+J16H~A*=fF3qXO{!axi_-g9uM@=i^>W^o-g z5&fCLg{ZU8nMO%V`AJa!<;dRWz!AFIKN>fhWqhfVIZa#8I99}80?F+K3jHwj0>PZF zII?8#9P&nL%wp=+&2v&{JcLc;Iyflm*BJpeBECRlw=>cOo?`x-99r;)J*dG`*y$}` z`?MPELy4lR?kvc-(o%%m;XUU&tYQgrN11YRx)fzkzF$K|yI)T8o z2zs%e438a&$*9&G#;*e}R-zPOenR>v*e?);i}jA(WD@^3z@q0nZ=qe9BvA$2ScQjr zhz+0z>)o9JG)=lo4K{NXIxx@ri5EvQqn+KDY(cf71KmEBWc2t^0c=v;s10hu;~7b6 z{FD3~q@a)eYC>bb;iR0$=<#NHlPL94Q^Z09#WB?b)>P(ArVD8bR^)C2F&$#K5_ir?A8K#e@KnRS9%l{&8Va9aZOVn56a6kvb=kL!QW=3HasF$$m{= z90u(}^o5t8^b!MTD9JB?m6U3w?n{UfhL6A}v@r~~N*so=AxsiCFATW626NH(0$XpC zj)Io%t)&s~_kW~k8oQ8liHWz*xbGcK`Qg0Gi6fWdzxj(0;oEx4CDAkPT2jrE()KBm z5nxHD8o(!+Q|l#)jc#8TI?~PR07XY}xqAB$D%sngy+I_7K+V8fD^DulgwoO%S2cNG z1h70>T>H!kNNi1wXONERGfm!3FofY5hE6WR3`Db;o;OZdkRg-Za~Z)P&(43d)9i`I zHOHlm=}MN*DMhdU{%`JUEo>^PX<+?S-Pi;Ot@YK8c$F)SkMWfABlYoOGZZLi2yRKy zx#Z!IF=k*V{r%P#5d4jWZl23-B7Q)Wgpr9qI&27i35*5S+!BvoGJAE9@|!U|0`}Z7 zIYU5cpvP})46SbHNMK0266By!jYKM(@%Fbyph9%FDEKC{l!r1!y2p+K!PuUKgVYzN z8;#fi3!5U`mUuF%_&x~jnQx#Off^4=7=NAbMl!58Gc+d}Pr+$yb8McY)!6XeRB5_{ zsZ(OKWQJkd0&;EEaPvUNR7m=D>l9CJdNYF)w!2-?#sK;GX@8SsOD!=zx;{30vvKB#7{^WI2yy^$%i2Thl+}v zX)(G3IKGsWHNk5&*)t8{CVO5DW~5j@B{TvQY-r-HZ7O>Ozx1ji;^~yN%iqN10uGKh zs$&yR%=i-yD`v&n#bU69OSZlmP9T}ru}BS)+|Vpnn_)nq(h9?x%pAMWUR(9Fs z&2eX<>D{8eCzkAgNscWvizuou=Z*O>s=vuKC#6sk07i&Nrkt% z_6HQ(RsSE(&o9~C!1TEI*_3us*IF0-i+j9-o3uiJ7zY`rgCPG*=<0$K9a zzb)h4AYGBo0jAhp1_@f*-5UTDdVM+j*#_vfa%}XT(xYS3 z(tN)1EeCv_F(M+(h3;}iUyId0;!&PNl;m^x=mj_#p+1ZhI)YKe1UOl$OE*Mq<3rz+ zultU;fRdqRogc6d6)_$h`QwYE*#Mj@H!$Y90K`j%!fE?9T>4(W&5 zS^ehP*@ApU(?-yeoUulRWvH=~1<(l()t-HTe9{gxh+Gaw-Ej!`Vth!>yIBDzn=vxk zH8qOEV2BBy8l zM;#t!s(PT~aE+Dea75>srUX*N3O95Z^N&heR#12Q!;P~T>$(^$2trp zeX(Fn10!wHII4|}jXF65F$2yp#JWWL)euCtTEPzG8u?wC|6nZ3rxj*r7gD@%L7UB% zK|I1v0KF{RE|p08Bj49qC{>}#+IBP|B-kvcToTwov>*O&-LiQHo+|!$LlYAbdE4iP z5W(yI)RvVC_9OQ8mIon6u6z{}C2D;#GIVq_vSO>~U)g+>XttEbW4GW|-FWDpRCNUF z+SVKs;Yf{|w#Ox@%2gwS9u273eZt}%ti z+0^2v^YK4`2Ng|R^Pp)^XLX{%@-mx5!%LCV&s$0-=D*^oV76~ zVL^A_HRhLYzm>hsjYn3V$s2l9-KkhskyoiOr(<0i>&A;binU1SVX5Nm#J2O)% zT;lT&j?0MhET*@q>(8}98PIGHHSvr_8TFZ#AT+u7`Y?2NXH88V*6h(P@~OsDR;gi3 z1gLfRNg_vKLkaoPW!CfumC*bn(mU0#TtDt)A~*OVjggGeXN}BAO~Zn)yZ}F?b-t$? zM1Y&X{3UtDeqRC@(&T*3agG#rh#s$0E55m+(e?%wUHM;&E{uic=pSb6*CGo?eI~5C z7cu$5KY1!1?$m7K%QSZSk=5yDWNG4%A{h1+jPzP>Y0Z?g*lCkbi=vY=}n&CorS{UE%}INdHgq^PPMmZ?0x$ps4DY8Dn^vMeL)YBqPC zI|V!*>%@RB-;~ScVDA4sTHY^U!)P1SebO9{LX~m_;dFI%YkKE=v3k(Pid%p63sZQs z4}Sb}v23GM@VgEfz^{5(hToJO_^TZskOPB* z1pv<^CbJn@-+%z4$=1Xf{;#^*zpu?At_RKU;)9HpeP;MrR>G!LbhO#F9^a4Y#$XcB zZJ_;}eYB+~V(&EQT9y0a5wZgcBL(HUvw6&e{PP)3iSVyeEw<|r;Ibmdex_^P=F`!s zr4hBLip3eWufy+f^kH?b-q=N#DN4=r{fxj9v6Y8c(Ru8Yi098MM$Z!qH=4#&kYDT# z(~k~*{TVlJK>I>&NLX~f7Qft06E@9Upt-}HOtRGR!?b!9ezKWTGTGhlB&w0DUk8&# z?P`4S4D(2DX?Xiu=c>SXbNc=mCDDI8VXaK&H{ctU5bhg3U|7*}{}^7F$(NEyZ~mVr zkCdF8n3tCTohJQhwa87{uXz@v{xP#u&lm)TCWlVuXUQ6S30Yx64#fTM8eU<3l`Al<9$_kb zi%TBGj1dy1%!%=bGnz%$IZTNAWKa`7)jUn#GR?Kp0k*$DA9YR>4UxvL%N&f&y`?!I@4dby)FTLv! zKYoq+qZjFSheRl-EJ$+HX;wr?%D&0wC^Zf}YNBYdsk;-(!X> z+_lY@dtrUk+ZVThpU}migS{}~BK@=M!q@vw^%uOVDf92knHRuCsDa1hOsvpAknr;( zkAc+r*Jgj%`Ml7me}6TkCYTZaqi&bzzHx^(JXN~;bh1A%Oy;AjtEJO+CI!-se^iQ~ z?bPM^Xoy8P+*VfSs*C;~r_G2_8{5kr@Fhvij!MmvLE)+cI5^IcrJSd^of|CAbXw2X zUMBeKcfE^tT z8XXWJS-Y>&x^Iv~MI}2YYM*rbvlz{JkVHt;nKiy`xf$-1Ik{4QsPAbsYrQ_#{d!z# zt8(z08}DAoO3QbWD<0L09ClO}IZpO6`KK)DQjg+`*B*^#nGtpE4>t`AkrE#D?^PbF zkUn|@St{~C9~F0JO}-tLZBp!qHmQ4;jm&`w4beFjD}N8RT<#w6mTtmC@VXjl;X9cr z2H%p%W6Zrv#V->TGKQ^HMKyjGa>um{cSAEctx_DK2niyG=tfbW5@#URNRV6~EfX+3 zr0XP=fgP7uCpUa>g&&;xnQc7&h1<>nqv>fi>|aNfbl2(eSSTU>3-*^>F*z8ERup`-Jo=KQl9_jnog0$XvJ2=EIl zvvPW*FFA39fSUm1$)o9FrNQxa0PpJsrmcg4N$X7EFq{wj=vToepXq*59DD@fSR43| zF4?=_7*`aMkL_k)-HmvA?e%lJ=WQpJLcX(f6`s$XqI2b?jZz`~!!kX?W2$*lDG-61 z7m_X2_h=a9sGx-yI+n5X;}is?_C0+tG>%sYDiE$(sSm{(afq;_q@=pz`5TG%UC5+l z`61uyeSBoOX=7$w`UTSl_6qT${Q6MTD6wCicJjMWqACpq7&&m+pL->rvTwhnQ72hA zG??L-E{UKo*}l{jVRr8)8+p{{sb=~_jd*KnE-}lVzn@|u=5mn(E^bCS|4ORNvweJU%_28T-HRxMLGc?*8&2}W0~(uV_H`1v|g5o)aGG5iM3=Hp7nq{0`PpKd3p z+m9Nl3(7*>iTAo?ofs=re^o4Gm<{^?L(fO;MU>k+uE>Po zDE~tj*__l@oDdd$x&&3{$3B@FX;syo?D}Bt{6PrXWLKoQj{>(L@j;%`-nQGV>9!&F zcEn>0;pCqWuwcUL-v~AP^>q(@(o7biGGb^cHLoV~s56TvsRrYi z`BuA8Idl&!*j$>tR@sW@4x~M;Mi%mC(m&J}igg(jukK}HH}xwbqBTEaRzMP$kK@y5 zJ~66iUSm4Hyrd{MI4;$?y7#_DxEqP-S8b#rXn(>DNK`#-#=*hyEu#z0xf`yPYm|RE z=tpk?I>Uy#!e??m`C<>qg@N~#hIfqpx*j6c&QB+g<@<)~xc_%}iDVied0ovX8xN@wNRbSA zHRJZ^?Z!C0%>HUG3EbGfP(JZT@HcmJYO=HcfvrG>`c1zQ{KtbUY?D+sc~7cMQva2u zd8&8vRv-t0K`fL<8hcuapr1IJD#JYn{+EF{AKoXHR%^XKYbE8b1`M#utvXT^T-Z6$ zgvD(JMMx<3d0l}&WQ6C1I62k~agX?{&r4D*@V$FXU5TapeFGdm>0gK1w_=ZF{^p^q) z2{G{*ll+>^^4)a#>EhZMcb0}a`Gn%W{pp?AeQ=+ z`5y}_rVMlna|r|WHIMTFA<`!VH&(62pe`|JVYys& zeSXcfegM;#nQ->&m*iAQQn2ION!jFl(}f_)4LVQV#VuOw4TB%RF=s>RFf6Emq5ekH z?a!B!R-A@ZdAqdvvlLCho4`L;rodTKqhF>@rq7SzpiexQ&&Htdx<9fo5o-&j@w;b7Xn>$+Rv(#ilW*4OVxahHP)(gr8#2=TJe-Xoh1ue+kjw>${dd z?cp>@^MUGuV-jQi-T8FIS>3FthX6H;kx?^W(%W*T*in?Vhxz15RRP3#lcI||{x}{k zJvA&Ghh6JDQ1DDGn?LM;G_5qK{b&F_NOhn56TNmzFFG2xi_=*CeUqGd<-g=skaT zggnTxTU5lTJM81#nN(p>(c)Ma`TUv~u?mHcI53bjd?{;RiS#|G!1CGm{|;ofWd03G z8-3E1=#wf_QtN>SA>u5C@4E}M3ud`|)#^rSC^lwoy2!2X0=o4{JBFuu} ze>PaYm1B#3Voo7&_(<|g1_XEqdjYT9;X~-+{pk`X?O6m2T7F=;DoiHH!4?Y_Ew_G3 zA5o1*cs2yNIn1`pr&e8@|27LDf&bMkB=aM5r)51?oH$Uf0TZC)wzu#I(1SCUy%mHb zM7;EAj%R}%-&M%h&Zv`mCAhw(CFqCtsgivmz3Dbp?7Fw9 zER4Y^P8%Y3x(SjTJ^{Wnqv6kK{@EAesK)DUmEnhF)yH}+9yo>bunc$jw7<;;V`}0a zH)~JGE{5*n5a`~1{!lV+ONtz`L)f*K2)_OHuBRgSBc0Z?wb!WTUK?mQ$&gN-~?KPJ<@|1i(lzht=4Gh-3TzkAcH1^Eec5+p#3 zb`zhQch8k9uvfm7Pp5ZBg_YBn$6~!$X^j0T(Viofe49|Bu&eQt*f&Y`|V5gC0U58lV=Px8f(!W&A@3cmU3&i zHnRb5j7mi`Wg?$YKtmpMBy^c+v@=q%WcA921FNtHi;z)3cZW#)lakMYa`%7Ub&{yY zUJWeIw;BMqna@=~0f3wk@O&CHGOvat(3@lSKcq5`voJZ9Wrw* zx-dvVZ4SQ3TyNyoG;F_fMHq$CADYOkYocFmpFDROAd`YR)0y?~1?hTx5mga~IP zV+E-{ABhdHLfn5>2sx37=c~seYbmFZhp7jAgrf&D;_-C8`Q>3*Oh4Au7BI%Ek7Au!_#5wa{WOsPS+>?CW^)T z|2*wdE=zet26slTd7qB;PFWn1=FTvkMw~!M->g^jTG(YRXAZOrcD;A zRJiSmR)5Ha1ACP@=sTtPMF?trF`{URCHOUsF?EHrH+n1~SYzH@8<_K$j_q3IJp$9t z8F=(Hra}LLhjqO6$gf$od4g7pRv(sDy9EnkbsvaKqG?nIJSfNwylZ*=c=ubb-Tm9F zqo0&qx!3j*iFCJ5wc(@qLt4Od&%sV;mZjB4EFP;4=)+^e08C;H{9^ZDWTfVP8 zTr*E~=qY{-Te&|?Op7L?I3(phB>$dcS4VVv9WBC)=7ve*=0iqXawFu7bP@8z1#$1H z-}f=vvvHrs$i>%gs*3t{G|AtKA^K3K~M5wD7~D{?uX|U&xXCq5#doOanUAao`D1 z&2N{}Wvx~1`hflhRVo?$@pQdMYZ0AINGkKgpFq^pU!_Uy@<6}%w^yel{x9&t1ey-X z$v?aZf}21?a1&?;{OU0h#@m?H)1y^v?$Lu}vP>y_YFvh`nS)kRJwdyvS$i?wN*sxd zf$fl1HCg>q?Km0(?9`hLT{#L?MXsbU@-Yy~q48zdAW}e!H~~#ruF_Fc5os_*=j(6ArvXU`I@l%6hJ{ z|L-zbsQW)l24>)cuaAsDz!ool2t37~S*C~K3QAAEf>9YP6kpS_n2*SYV#Nd@DuORV7_gW5~7xg`2wG~WJqnlPOU+|srD_<&x zsw0%??U;Yvkg#BY%49W%pD&^{27CRiWmp5=blsh=rU*vMN;9Ibj36IN%G_}z#F>6)*ve@c2b-msVY>gGE)eA-P0YwUTK@NBDf1YT$I9+jr z>COrT)Y!J9wD+?h9GIT)R??PI9A#ZX0adR#%N|B_^$#~JC3(pad?r6h=3|d|0Q-1! z9>b#z?SX)_A62Kvj-ueMeQ)7kc(4`2Fc3lwh}aDPKjaJ8?$O8t_P@>H|KI)t=E?24 zKuMa&VWUgbX%td4`>(u5Ptw-A(s0IYU+-vBC>PHpPK;`!EAkxhGxTs6%6fYW^FVbT zAL<8KQi`j+GJ0%Kof3P?n*Ps~f3Hg4>!-k7W@1EkTE|T%{u~(WPQJ{o?0ne%9Sk>ylJb zcBLv)QOTBk0vOO~aoDVb=8h!+5fL7J^I?^yi{3%TH-LI|+f>gJ$2>l+Xw^>gi4&nY zsjIh&^CX|1oVO~up`mss1V|5r0sBoSYL;!?jf8pGs0lpxKRti$DjQ~_QjrHYLKsNp zhAyK#Zmv~7^t&nAwp{3@Y_=L(n8I2(<@DQB3pT|s+y-tWg~(R#jS+e%ri zt9;-VEOr2jJf~nAzQ7bOz7kYNM?t@I*C#AW1dWuc<#=RcnIw?<8)ZqLnBrF_eDLZ3 z9?U(Nl;P7EN@N1wK^}=4e2IXO`F|krWSYg!+)@!BGy6~W;TsL#$KC#Wxsd;>v6qba zkLndByIrv2Z_y9Ix0#3~E0EUpW8<`=E(P?NFo1%crw9}%ZkO1$cfb!hiiOxGReIbg z>ex&QE_v4^X7sGZwD>f>NdaWcy6_}72?{B6O7msI9G$(eMy{K)D6rM7!nHD7iBlK4 z998$&tnvJuOe6X<8`m>fw#`(vo%9v;A^U>0t}TqhkW!?D%>D=POpF85|Hs)^Mpe0Q zdn+O=x}gB9+8_>a3^`@&EHjRMax$S_x-?GIL-#F3jOBkZe4PqtQ88{ z(_o7uWPHBAio`P#U+j9Pqj9)h(gAr=;WHJc*&;#cb(a7|25igQT-*1!GsZ~FquUX0 z_WPA@phv#;d>bage248@KOnt)`-7Du5nGAEiTo{AeB-uA{`Ky<{QzbrrM+O*b&X{PNwJAS~MazRO`P@<`1bi5HxVBypcfqgL7>b}5lLmCq1(Cp&o{dgIGoXK)o5!q3Nqdrgp1l&3 z9$H@fwp)m-B3dn&zfei0fX?AN#uA%Jyg5{uKABDQw&Zi6UyIhJ^e!2zeRfpYG6!hs z3d@Kp;7Snm>OPWB3XG8t%b+^^iU7TbvH(VyI0N&X&(=R;fPsQ@ENsn?W&a0jPgYoa zi2v)Aa!P$_O_qdcvmnzkG!Gu6le~&+O}iXg=(XIt!c5};niTaj+qK)#AO&0Vec+>< ztqzrxXf^#lr;~9xWlZ6W4ce=ZYbRXTt79FgNW+my@qVU?`i+tV?1VJ?{{YHp5I}~q zQZ=R94(v;K(K7&8r>+;I%7kMH;Xya@%^*=BYon;UJ}{Wbzg47?TWyeU?udSHC9RY| z6CFD?O`HMIjj{8_rbY-z%gX*@M8L^3h?N3+SZdjxDWM0Och;;%u+)WT_RvK|NBul0 zMN)5Zr4J|K3;z9ECWe6!avRSsa5hPAtC7k@tyCEpiE)R}a%>tR_GmF>LV(r-qB=b< zFtp}0nM?GS=dKMTRNs8Q`l;dW>!NHGSlnp_@Pw^&?W8~Z>3p)rdA|Dd^K1%x_D*vE z>PW`1=NswvWC{LkrCD^Izd4jF2H2(E);*L0qFN_MO+>f=otY_;5h*8Sz%?kA_v!3N zQT|`cGlnQ6{*``;$^F{qF-;wJHVdw@Z5V+I77P`}i;CX?;-8U3cMR!q zwQcxw0y{)R*Tyu6KlR=dA1@F~2RhUAeFTo9aLs=?j)6A*RMiN5rTp|=bEVCY05={8 z!_@-)Zf{vE+J{(RBaW^saTyKwA5D5yFPE5o_-T38A|WsRp#@Xp{@XXym?8z_Ar!(; zM?x#MIpEmA)U-|tB~Y9$qBZFg6ySVUVxu)7k4^p4kX{tZf_1L@O7M8027Kjl?t@?4 z(XIo!l|aZV*evb3#z?fgtD|lJ+B#YGLD`%M(8G@vH+98$X}a8qDm+M(niC~ZG?P!A z=5kSJA!XM4X8PwjN1Xc?Xh~a*{E6?lkEMFTF-4j@$v^NR@s59*^Yvf$&wUxN;VzcU zA$aY;_p~}9DaiwhZ0&7;J3kOQ5=`}K1`1UcKj@WxpXVJZ6h)Essu)j9w?m|JxuS-@ zHLDJxRpQ6%J&=})Ihwk)8$$PDC>V3 zG}`BeqqOWD#z>cMmv||JGIH3|i|qK}YH1DVGX4~)xY?EuJ5NHV&-K_M4d_dNC?7GT z>r^6V1k69Q7~&8DCS<(zZd@zat-T#A1R#d!_#d!LWXs1#y_-_rMyFfkB{J9e&g zJVO6a&Q-$2)6+_0SE;5ww5JI_3>O}cq%blu%9K!lPqk4v{x0ZvRMg{f`Q`o+dWg^5 znnI%yHC_#Ugl|OwNBIZj6H|Z}n}o8v(2z0Eh=yI)vN&y5s)G;jy&HDy0MQ~G{{ zz;ldsNi=%%8XEZ1I|`6j8aB$rDs{a`6h&9;j678YVRyH6bHEY~{Tg|lx?c;xhpijk zUk~v7zB&Go!sF@PJxB||CL|1lEVvm?RTEaK$|%Z3RW;@!k@d%A7BFT?WU8AQ7}!!? zB7UAK?KdtzND?p{o;yO4bk<;PbgNCH9Hzj&b!Q^|mXs9yJax@>t?N14f-yIDqktYb z#r^}&eZ@JYgK$RK(W=g#{8~|O!wq$1*OYJe+*dJbRWe^#ubZ^0$`Dg*O?&Dvb8fY6 z#`MGP6&=aIkim5k_5LT3Ec0}K?f9^s*=kDoA}@|$Wmsi2;^f|`PYB4brD93ZOQaO% zFP$yN!MOV7)2k7et2La8EMY7l*&mEPy!b;Tn<6ku(Gv932%XZeGi5pzD#3biY73e( zzhxc1Y#K4gDOkbQbd3S~h3sLP8A=00izZ?#Y`ZalHFC@UQDGd6Q?XEfirI|Y5`X`C zB~8$QFt4<{;D?^vbnq)J<&phdko~Skx&-fXfRm%)L)3@Q@YP?&9CK`zmb=&z`5Ep# zj-Eln=V#(fT;7}*s3cnbz5@LdgRkx=CpHIEia-d4m(IZ?12Svh=osV|k%m1a1(|wO zxw}rn+iybsR)x)7;$3MxGq(4`yXoua4$7*t>buGW)@Cs!`wd*Wi6O#@ta3&hX>l+>W5ez7!?X^2->X7%k|elBip4(Y|fV zpuDWUU~OdsN%_oCb~W<4h3L$o?_z!*8Kp*JvBXA>3s}q%>s;u)X|IZoP5S*~{F+^j zoCte$t}C>d@Dt0L#BPDc3{SMiB4K#+I97|*0N(T}(XB3?S=6x1KuK*Zn*j`c>fPEK zO~nL+dNvb@0qW+n)zz%>or^w#VjfLMO@T)dG)qC+pr_V=p5n<}K0%nCLN0lWav~Fs z=^X>pJ|h0hlt$F0^zKz%@h@V<`Kx*)SqXdD&&6Pg;d5*IZSlCF<5qhDzx-XwK!p~b z3&j|?gx%HpyP|3$wFVn*(^74eSa494lwPwbVVF{X)`-dqhs*mnk|&i!;2TuFBw9{V z45!O{S@N+y??_YgK#seUCi46K7Lgs2QG|``2Bm0bH2hddj4$0`i`~&Ic(}Znv(NHG z{O;#zGWE#TxxAH8Cu$5emdwR3_YN(08OoBmI8kmLZ^wGNkV}*ap)#BLPsE?we;t^< z_v#`_;rJFloP`e4SFa3Ts<4nM6)O+9j^}>5XF38^m-2?Sa|ae=%X6Ul8q>bv9C27EBD3A| zsZx*i;(u&*dMYF-< z6^{z>!%mtSY=>^2l`z2SVL+UgJaDWVPhi+AXJgX!weKd)GwS@MXg8^#05ZeP87+ba zXd|$Ojhj$|Zo3b+{X;A5wU-_#V;C@k-Rv$l%Q1>!YYVIXFc{yT##4+}_voV;q)6%Lvg(fkic?t8%SUQbSDODO~+H85_ zoE7hnHjyrt+Bf;pRH8n?g|WpeLz4{m1=VyZSp7f`Ao;|@+I>%l)Q@vjY=Xq=HzuSI zfA90hMhT?)egYc48PD!77%S&HW7Flv2%6OvLC43(^9k5AIzZh~$DJZ}$YJgpT~7$< z%KE+E@KW3<=j5vozgf7_M({~F0i?_ElnxEeH~Fx{ziUcGU}TjvkTXrh2pC}|xuN6L zy}b9XAyU9dZDTgG_GA$cEYP8eE8A~0M*K$OVy>q9x~Gc9A!PXwf~2^^Y-`<*2L6?) z0rs?`n0>6lqfue}C0W0nEjcZ&%%e8IE2S*-LE*bPYS_0TS@H>JreuLJo|e>VLheaQ z^a5;ObFPmh#IfLdQ@HOrM$6NlNhUXsx_sVrfjxB$LlYvjctKr!E`$)(#u$>mb3E{F z%M^iv~RbW~KzbhLM5bKk>6@s{%R7NjX&^7-QPw)@r7 z@Xp~6cMPa6{@1k?30#io*!1^sV~c4)OY!vGr6KX7)LbWO*XS#6hnG+NogF{%vrr)> z4jf6x>w}+g6dOh0WK#HE)6Z3!B%Tms(=n00^tnT9E!=2XYZfOKTEe*_r=ttqB1O)! zR@d01ryanXnJV{A_Yik3`hnj(W^Sh>uao+vOLDc&8y{up|Q`c87@F8s?0%5Fp2tcFAC$5 z$)>OGcblVT!2X%7L5IzQ>f7Uc+!u{ z*t`hi46C#Ed{t3sZhvhq+|+JRWu(sJ3D1um?eJNh`1rGnZ3)5_*YyhaWyXkfBo{|y@U+n`Pm zt<~mKvxZi)p$YIEjZI8elOy=M@uPFIn8PsfuWm857uVr){IFrzNX-c$QAttW5lf!L zM?=oZ@@h?KV!c(lRbvV6H&MpZjkXxmFwl%)gh5Y^-}Bir6(l_V7E6Lj@`EXnUWR0<4M!pW63GZ+9qv;>?h+=) zOC%ptNl^kbJ#j|Yphw099`}&&@aDW*QUR{Qxu3Q(8f1H4w@gOC)`lv{pOfRFB%znA ziXTW}B4fSH@`cbUd&UEt z%|`oKu%aS0*Y)lLZSJWJHe&@0lVAPzJR?j(z9s!UUKmtx)YK~;_U@$&qRJcEit49} zEj&c8M^U*1-s28(=i0k9YOxNY{xd|Ry@yYm+}D>Cr8XL=nh6|+c&X-dAQraHil<>g z@a(;m+z+M}X)kTxUqcspd#Jx3Z|$^gjg>c%&suJfv%!V$l4oyD<{1^^0$Ejm%^qK# zCthDK%xnvo*Mz0%A1oc8AfP){< zXIJ!K3yK8>m|mdG%5|x;Dm}RalhzrHj16FAHwgH_9D`|`AzDX`NPDV3a;tu=p8Wp0 zSX(ldO0DEu!Y%aJ88PZYW}*vMX@oa(ad^(ow3R&sK#OG#L%QkbG@G7mumhTG-GM=d ziWAy3p*eWI176l0xOi`{MaMp*6Q4qHEzJ#xW$<3FXK$aazVQ8tgn-yZl?3X9eR5;Z z0xO=HSq}*`AlLjfSv$^bcg{4`+n=sSD*f$_bin->u=Z=x27lCv7~BTKU;!6FC5VwozJ3)@05xFs%hJr5Q;?dfO0&j= z{SBDNW+Wc(c$-2`k00r``#;BDfESWvOD-=*ppz(v8y5uIX6Yf=PFz^lmv;fyWC}kt zL1~FuR%D+J+Ik)55RnyIL`72w&9hHV@)sHpCt+z;a_<=XnI2}^+4QgcDdfhHo(i#+ zu*L>Jm550Ub@xTD)Gue?QM7>j5MtI>xE=a+8NA6w|L5fiCzIj2+|{MFR?6=o`O$gr zt+uI)bC6y>9zhyT+IsZk;DFNB$xxIC)@$3g}7K+7?X-loSnd%hV^k|5|8ksxfS44RScfA~Dy>JK)+c zrsthd{nPpNcTrX{xNoQel&#FQ{jU#DzzTPorhE7U=>|j_{UJKZO&k+{mU^ClbdnxY zZRvp7?Bc)}WfuQCmpu-Oyf3)Sp<|nz9SuB^$rA7Oc6gEa6y@?UIuun?%wihnwRlF! z!w$8l>(P5lIucN&ITrvaUzB3Nb*)?R$yEbK&?jYb}SV`5;?tYAuL`4?;8Eib2rFG$CSqGuvSBfmpjO@>h^(jY8t{9yM`!ftk~d_pZAlEaoHHW|@8vSqYqg%Y*KOU+rafd&oRvM;bim<7;v7g1-WD z&(B%-KJTMw&Ob5ClIXV{|2iK|>SRajp@(#(JUvV;;qq?3r5LLmfU8Ut2r1H9NrWZt z=Er~R>VY$~6_0;9CD>m_!zl1DjJ0GfFRg=hIewcj(qKeyN84B+4wU{K;y4Vh(3=Cv z0f5gV3&@OTV=8L5w4Ig_s=pj`%oCJZ7f`_s>|M(rlG+mEptPS_zSKUzZFQc{h(V2|=|)UP zCsM1iLN}S#rne=ERfm`yb*7(XDK=szx6Qn8ND^HQGo4zyZnpnFCfm z1owdn>aTW!?ufh+x-l(JEwf|LA!5o@dX!L>cX5`^_126$r{~eCZv}sWDq~`d5O*g~ zioCS^7_J=J&ng!+#9Up|;8@t;El0QG&CX|`g0DENzUw^IFFk?qSeu{>b7cg=k_^sM zMl4nNUO!UJUCvV1IU%{zL`uqinjdrFG3uT^{cwTj-QqJphiAwb65H-<|~|h@(5P zCE0s3SUJMJjDeS63%+FqrB#Kvz=j4`1$_KiZ0K3v zCYD*=z)8!Za5g8^k7FAf)6u=xUsl-&X4CqdZuZA3_|TlaDNGw|kdG?k_H@Py=ni)k z`IDPkUjGdy7&AA)B(=Ed+>HTe4xScx>u8I4v(MnE|9bUK1nAQWW!wB*yZ^jzNFD!nSmaEh9C+u$90+)%x~i;#@CJv!&x6!p(*d5!4s#&apqT}I zPqk!IWIThZUE%4}-(+r_tz5r8bRPi@jx_6tjuO%9O+OC3Xg7nCKIouQZe28iFyu&8-0B9odHu3{8L2um-nfJ?U^1#;BP` z*}}Ekjs{uPG^mtI8MI#J{#wgwI0A2%x?*sMLpXF(z2uFMoKBDd+XF;15i{` zLxkDVz!bE}i)p-+n?#7bXifQZxxUnEdScQGZ6QKaArfhf$i{z9Mq9yfR=U88oL@z=%p90mo<<`k1{loG7903QVOh%a^BQKFF zR5H8OD%0nmRdXgykb0!)L(^hbN}R6#Rg(3rL7Fr3g12{?x@Q3GY+r1 z?DGT---Z&}9vR^LV%eH&U7`rI!_+hc7|8RZR|?IpAR>avK#B5CatR#-@?@vmh>toB zNn&AWXCmpY^dW19kMY??Z`M!*{FT|E@st|A%!cr`A7)K3skv9|TbX=YB1>=s68Dpokh(LfEPsh_F1w1r&F8m)(|s9ej!f4# zJ`aI>G>*6)U=iMCpymkUj10BTv+8ahct@>6ip##Bg)}hxQ3P%F4a=qoI@_Zw`kp73 z-!*PowJaM{SgyY(&*u-0B{<1s^EdVmR>WfD<5|d5WW~x9PdoeEZ&7j8yk}KY|HzV9jHxF&?X#mt%@C*U#9aW zQ38r8P=3W70DKtIYc_Fa#dLWxq~$fveV_WvSu|JK;zw=zv23pE44V#kErr-sy-^AhfYA5UirPrutM>@ z@$2+v7vnwb?eP*?;z(?n>;dHcnE*eL@{*eed2P2bBwP-I_he+`W!6(xh=&h6;e?$2 z+zpOh7(l;XFq(5s%~}oDx*lm*W__ir?eV~h%YT3sLr18O4(9i)c@c5x%-k&teYrm$ z_Rd<4ogED|481$OSuZfM5*v^Fgvf1E@|iv}?Zloi@1az$7~+^+#dt{`BJpN+YrZWAKeNi^7L8U3vkZp+vf>$@2Ee*`}B zvo%rz(YrgV4ei%&RlV9>W@7H_Fd*s>q$j9eav8wA#awgP1kjd(uI)$YV+cNA0gYbf zU-pb6{5zQ3^sET)f;4*iKxE~h+D6%Tm0q>j6qz~x%jCrpH8&*|7!a871{-FN>cMG# z5f1tUhi{DI#H!4oPH82XB+W#*lhr_9gY1rJhV)doDbW(j$jz%WfF;u3U{}0flBuo} z=1>XIY$d%xzSAZRn@t^- z)t`|Pwfce6Obq}t4;nRyYIqd-EreA~u+iMa%g!sZB{GpPgJVHLK_E>AZ|#bdnOyi- z23n!IJn~Bs%A`tKQS#wkw0G2EE@RnP)x}(XM|w*`Cq{ckWz&dTDznAQU2cV|yWt-> z(DDs-3-fUrZcL?UbCloq-zdQ#^oHfthEW%u8S+BHt=Cr z;G{0cJ57XjT`w+RzSemb?Ey9lpH?7!?5E3_$-LbV?+pe?yzT`hR2{TT(H6=xBu`G- z;S3`FK7oa~=kxnd47c zwVE4Jj#|Yl)}}lDY`iOjz4;NXikVCEqcL2-&}zDJq$am{P_q1chsX#;x?yJ@mRY?C z)vV%~B3WPHaP|-*KSNt+@0d-BM9VI#dB)+!#m?KeSol+x88|COEZ}ie!#vqG<0^AI zRVuzii?k_$=pEo(5(wWSbqrc^A9s3N_c+mKOD*erZ5w?Q8kFXXH3{1{Z;$4PT3x6d1 z=a9nb`!st)C%#l5h|KWs0dJr2%<^S$GRTc1+2Lni!$7@%MM}7yyU#+BVOu?iF5Rr&wHU0j4*NL%Z#n z2diS4NiAMmO4}Q;59~$b@BBaHjn>l7#gI^QghU`0DUgSjDyp`KYS{U_nt;PUsk$%n z;bhlJ8ZcA?xHDaB zoZ#Vl#4wYhK)(vdmDb69^9&|8wkG?vQmUV>zZ69XC8@k9zI%B~jd5989_EXTNfnW+ zB%=3QG`jzb$aIBHiYJ<^{tEy-0v>bvqiHi~elp*CZ{ zb3ePeWz;i0uH<+4yxpg~9xOp(#W`rb*g_O~aG&kq_>8QNa@9M!bF(aBSMzOMpf|qr zc@1xKlOabCO9)`zNJtMX553Nz#a<)7#hu*Mw5e`&sXe}GTKijiK#xLLB>NYPNen#H zNyGx<42|T#!XH87;rBYQZ6Y3p0<>AOa@2ITg_sznL6-cP&byF+cmaTi2I<@Jg`$ve zO^^4fkLt@$SeOplkd1m!nK*^mVxkr%V*Q9<(crXn4X5R?qOMRBuehGR1%_%szpmkcG zo=zorN}=esYj-+G>iIadW?~kw+lL0S7>9fW$UI7$?oa!lHBbP@=0bEZ{}J#|r7Lq7 za$L(X-dVYIKIa^CrfX?=?YfxxC=f5gPj;ObnQ6fhuReWl_>zU zL5NPlgM!bM4<>*M&*n_;`TZOKkU$AC!5-Z8&`~uJK`gyX| zZ?V^kAC`@~?8%cQPhX~nF0$Eg`? zG`t&;FRjL7D4sM)Dz0fc>z8j($`gRuG!`(Mme)l0hSmBjvjq!ul_k#s{7*vI3JI*2 zNtFsYZ@)hr;2^5|_zEB&JLGWyVjf6%KDpUS&z4W;%Lh`&bhX#P*?%O6&UV*jZO=Xt zzS2h)$R^Q6qd|%?^dCDm;GyBnEZKN&>pT5h(|Fx}8g?Vk+jbNBWtjmV0&^iF8rV`o z-Zqzl3aB(_q>|{iHR`P1jXfHAC5l1Fu_;U*W6Fut(ZH{6CD)F@|HfEDI^0jhgsK$N zOC={kNn=-n3L7S_lqX1!k!bgiuH^>c-7Ri*u zz>ha{_l`0vlg_|X3SGjSOM8`+mvJJJ|nbbgQKY1B{3jpAW=e#nGe89*wZr}g4$XUNHM zJIZqwICwlia_IplMi^+RKaBp%lOnGB0BD0e__wguQpo<1ob(REKKw|0v=$XMPU%Jj^<&A*@(5giwdoS`! zK=ar9H))R<9Btv(DbTrvr0k`2Oc#MV0=L0mgIA?^Pv7o^@djATZA$hCH*8G}xO;=+=>N9@5`ugeNHyKFA2)JSzP z#!Z}Ri$)RbJjX(!38uXOoEeZhKHX5zJ@DzL8(luYpb)t;u?<;U0*1a1G5__`r}88S zG?-Fmf=@n;Ti`+rRx%q`TrRppo2EAzcNZJ){%w|PyeO-ld|na)VBjjh25Q-dMVD4O{y-n1UAFtb=cjpqv^soKyJa8S$&sIH2$L=gWsXWgGO zQ}up-96HC^#I&b`b=V?y#yyCmt;HI%O3s?B)uwVyqy?M6T5ldO3Axcd-vH!d84l@n z@lNW0Y5i##_k6^bwLw33lAq_QWC;2ww0J()73iKi{9XLp5d2g8bB|n$Vp45@`yT3G zG>f-kfhd4(2HWQZ3x^{>u8+v>`Y#u1vLh4Wf4WbMCdl_z>bcf9xw=X#1^TXGXDB~Z zs>iyAGskoWg*QKiSHLHu)F=F1`Lg|8`8q#0E5Lmt;7Ru0^WK@yFPax0``@8Vc{Zc% z-zb7QghQPz6p3|>nrd~5lg@UE(#ByaVq2L#^xgwbMbAs$ylTIdcJr_zPs%j^&H^0an-xI>^iA1bMB)UHu!0f1<29E@=>-#J^=xI2|O;$z&L~^ zyWJfrzzuKLjPOT~@qcxX0v7RqOt*kHYRT(~ck_2*yOS}Bi2D*?r8^T9(SwD4$?F#l zhLyYq`3GG2h$^aIWcd{1=~MooFUTpK%ohoeitL%j&J+QbzK$62U!Kau)W*J2H1H^Y zXLc4Z7tV;_HgysYrzbrmBTu2?pm;KmP@AFzQ9O=cG3rNc(l{%FMYE*IEMN#Jd5`7$ zN~j@@$w7yZ(}Rq>}FW^<+`|{y76u_WA3fJIQH%B8h*c0&sVcX9kl0B{^SJ%I>q$ zpDuudN8|6iUmt(9vye~G!WM+~916e;3%J{-K$~O8l&np-lyUrjUEPTm9UhtbyjM9@P`F#n`ZL*eM`c;1-B+A~{`4eOyt7#|( zuX~=l+PKeSx;&GMJWG+P?by^D=TSw=#H_RYkDcEjFx%QQ&VM7sP4zkHOa~EZ`P=9*>=>NK}c8s|MMZ; zD>5=J{8(uvf3B8vc77gtV75}gPYMf_09^)kMvsOu8RX}v4#FU zatifze!I4!gr|*~ZFetro{<_HsTk@(?URU|20&nF$9IcJ-1NYg+r*B6R0<9S><7i_ zjfjQ;p{+4By5b>hB`#C>>J>1y=LC{(-BPPIlUpGUJ$-s{h7v6}3Z?qIp=mrsRlDQZ zL}B3{J(<%CNl@|R;%Rs6c2 zwQ*swu+MgwCgAJ7e>wg-9)yVd{>Ag8aLtyl?s(W;iAk;+rKn*5&+tXe@s$pKoV+2UgEOWIR1xAcWrz z`_Ac5^kSA(G1r|lT8XjJ7fGZ!w4WOmOY_t=g zH9t$iFD6{m#4&34m&VfxoS;8Xt@W2*e0Z_se)^X0=8-n0R3%8*I?p0>@e4)lK42Cz zbP9i1jPp$i!&qZapd5Dh8al6fLgVnbWl8h6)M&mladxhKdPPn2$MC-4*i(~*@pAQT z!rxBj8<+pgkow;`na?1Ykia8+;L>dl+}`_2q5wGGlA&-NRw?)rAK3s6tfIH>u;!98 zy=GMdb`BXis>dqkS`g$2PmZByG*f+G1 z_~3hd>T%Q+%r^-{`Rf04UJsiNX!H z#(VgTDFHjN$^Y4F1(272+hnP!p~Auzlde|c6S(i}VrU2&NrjkCPJO4hj$DO9Bm~nv%>xmQnK@}g;e2X%|1tM8Msa>nL54|YY^ZDv~BlyCFvbL>{28Nx&r{y;5D zx~Cd$zFVhtXIaz1e5dI<$+QQVtDhIW2~e_-jD7>h926!1-{vQhQC2k=_tc1K7!cl3 z>13MaVlIXbHOFQtI24f90jOeZ_6Te2-k8@@4wql?i;D(w$~EIf37j7MzK042+JmjZ|8Ivle6YaW&o4;29=0srV9n)ir`)VZEF@VAij@xN z&~Gyh9ynr`xB^C+@I-q{2!uz69eEfG*ZW%cV(W6r$=0o2#Mk`3Sf(K-A6$h{!7WdX zp@B6=3VDJa^Gj#8g(&SqOTW8Uk33sPd;;nsfqaaE7Lt~#rOFJwug-XZvh&kL z4d9`|P6^`mQD#}FHaX~5s8nbA@eMcqtD$%VAYqg%P^n^ey*%qXAPmM-g^Cxv#3Nyv=5;c?>$3`T2&1w;&842h!wjhU)Y5N5)w)?Pj z>_-Xbhkg(3nN=EXyY?zDFlZ!@PTBvE6@rjnFSCA9-YMPtgr1VIQ-(#Onn9qMy+F^N zshBOsk|dAX3;22!c7vYd{c9?B`Tj;)zbk=N|=yEb7aEXgL#4qPc zDuk!FPu3Bd%mg1%-FY4e>8|hbE04aw5*^$j1_ykA@}pS*Q<7bT2S~QWRaelrof{eQMnBi&#P`mkYYB>Dv3xtLpY{%ep`AHmK6qE+k7=fyAZ81$~j9 zX%tXQq2@II?F@TZOBY`{+hyXk7E7vn%eM5WUMD%y8583K} zc;^3WtPfC%0ztcNlCDmlfdLACat+`rExv&TNl<}j&2sds{&c15E6v4W?4ibUo+S=} zYBhAZeNA*`!*n;a!Z-rx$U(p*MD+zweF57fJh9mYf2D$LPG!EFm@W?1Wt0_UJ(8*yzagn54v^%f>XB{~Y*d^X_hLJlw}F z{h?b0Vl1oCrJvMTMRw4mtB)!w|aTT^X<9hEvNHgeMJ2C zxcHBn5$1C`0Rco_F`!lnf_bkO{EXKxJ@N^W@bm^9=4vbh*2E)5K;?VK_D5+cBs})Z zBsaQYbBFtZdsdBUYdC}sc-b&0oulbEYw%>~#IC2nJtoiHIr{2CH3_QwbXtsjf$Q-~ zj%=js)ES}qj>zX0arwHL#g!rnt*$??-JxiT?A<#!`Kw)sL|=#pAXn5M4uBH-1dLj` z-p-CVzCRnX^KKw%E7h(;f4IBq#?fgQ(hQxRj@6M={6mn+M<_9LMkQrDeD_d&6hx0zPJ;O= z6g3^EDs!Noxky$h64VKG^QxE3csSnBo*&&AO(uK!PZofq8u_r)7&T*D`jjlRr&cAR zvVU_Z`rCGOqwu$tV@sL`R z{Uq(9kR}!$DQq>={1^omA2QJIbY5b7_ax>)?xNt8|sadQiyzh))EL^vf_#ZoZ zezlwV&S`|Ev0&FYG!mx&ened}yHaAKw?Bw$aXAFu7I^x^T8gak+@LH% zvYev&lVca~|IDwHROKxv`cd>DE!*%ew*Qiw?&WtXcqOAD-cu*t2V;j@!kW9+n?E8t z_eH5{+yA73ShE=l0%?SQbcue?y2O%4hRPE9d^B)-!&UoY3AY_DR0&Z@?Byx zpt^ey%_sqgPq~b;1%WfEjDe?9E&Jvcd`p4h5DCm`$K>md(ed8i_O4HBvlq_AYEJXw z*-6vV-wsxKFUj8OHp&f4Fp0JO%wztj)?_fIJ7eFy%Hj~YhZ|fN{#%%DJPe;1>JFT= zI&BuMn3#*rY_NT>WBsYuTh#fgC_*t!%rlYrbZbw#1Fr3;+xw_JGn&_9GkgT^8AY0x{tKZ%?daOF`DvBWW?t7ZJWJ`opKkE4^jy(e7#?2x+##)x zao-{DMm&wO)D=z?CMXLfdW9pT-c;1fdzHK9uAF}zF#ldQzGiC6;mL;d?+_HI2nIY5 zkPeA2&wGKH#BQzoDS=M?XDh`p5dW?D$@YOz0>ECyVwx8{nk!N+PPP7>=+yb&6PLJ@!AG0!CxyT5$vu5rs+u&P=Xdx<=@#*fe$cS)i55DFt0{!6+`%cZbt zcD;yZLp^wE2xXp4c|r6mT_UNm_twjG2ui~5acLq*=g7_1@xWVT!_zE-gDCT1TQDe} zND{sWzvu<9xzVD0ed~8ZWIz>tFP)cV4HVIc+^6lil9vx&mp@*;FElVu|0a;1d8PXu z78(u9BA*)m6jO1Z&*OFuKR7tp8(vEB-7OvUB@`QK+BB*nw5X`4BIEJIKI&>ARMp_hh$xjo~Kw67eQtDejj?4T{f&`Bavt8n|GR z17W`)hyM|JZ?f3$ECA6T_9S*7CB?$obszfLrVDlz@9>K~4#z|Bhs>uI7=eddBwr6d z5Y@1X*YP`7(D&TjHr}iGSHNh-^4Sgwu4YaMcJCW-gJ|9eikpWU{b$D8o!b$iD*SM( zd_JMb5PO1^>9etmY`;;4C^&I;72>y~I6IDhhC}l4V4w~TqV@4$h6)vsBzaCRXs=oF zL}>R%qJ_H>+$k4<;W z)xVzn&mYWe7C#CvN8$g#F73l|K&q4&5YW2Fn-?KreT(IPzXe}rXUaJ*tMaHGxr34l zohi0+36{%;tF(i8Lh$Ef(Ox&wcH@%Fei%87LeXb7Jy?3eMSOs-+MV>)I;un9a^lyd zoQWA}{f;wmpTuE8@Z%(oYSK7+3dwxDm_bHq)Mn_AH_(fRVP5YQjE*Z4V%@1y9XHjV zjqoXBO^grzsD+b|%dl-b_G$^cnnhm42D8vM>bDQoFm>GpOIPi3|Bb;};3Xlqz*Smz zM%b61`%_EH;@YLfF8$b)5V2~rXh_SG=F3XSRzLvK>A2r^F;Ljr>mdheASOVrI>A!A zUEf^)wcGOKkFuEhGJSP0sj}|X|6=VeqvBk*En%c^_u%fX0fM_*un=5A5v217W8{7d!OCsp6_<|xMTE>{K%*Rs@`X<`OGz!*=sm0ez5LRZ7wp% zXHbQFtD{_7T{^+#T}mp`yQW(UNet-yO=Jx78NI`ZEkW%gj!*PscJ@7bg~?~k^IvE= zHuXi2Bh|UyfWRF%djkndviE2z+RsrkM5sn6@+v`->_`FDIYQXZ6GVaUjlC4`AxmWck=lG9&{d%y148oLF^Y6o`+R6Gp@48Eg+LjbMwpOY-5i#tsfyGz zq%`MnF>eF0VR-)1;5on5(sU|;nLxlZ(}hIWsEyX%)@mc$@g+Lt>)Xa3ytL~~BpHzE zw;Y=ONUk1z(raG_^}?c18V%W**GYM6Yqm;ks{&zE8R@85`9l&!ayUAAgjB!uEWAbf z(zDmqD=Qa(!z2a*#yO(p%f$v=TwH9pXo0>0HV>!IX5fP?!y4GMF(F2YXt-I`8 z+p%NFPRU&^$M=e)U<#qdB>4rp6^d_>u5htQ24m|6h98|^DDEc`@z-dsX3U0k-b<7E z6l!V^utKo!b{xO-vSm_V?$n7x-VZlI2&;N^JAHz>{Ml#qIG)hd#5T1{f`*HYi!(ut z=5Q>qCMIKtec>RZ zo8`{`#P@BD&_wUSZLgZkdVA13AJ%}?ehp!#`!t*~gx_{HC%(<;-S;EN+^HF4gPHMT z2+?oNaVl?$eP|@xa^@7-TXHxM05W z9l%&aW789CR&v>w#XV#6Q?k9f$oY;JYDtYr#EJcGx4prUq{!n3S*ox3iKGcc;g5d$ z75DPLTS3<;*Q2$+H^H2Jx;~aa-LB`9WL(}m6_`d-$)Vp+$u^4@M<;KiKXAe~5xu>8t_S8`cTH0IhkXoRMSQ;O#cL#dwMZ@dD>2y0Xl5?3>Z#s~ zf7$QhTMe%Dpiv0#Erq{xrm2m0D^jBf?9l*YfZnJ?Yx!5+akd~sR5{bV$-+}${9oX~<4yZQ4AA&p73`&iu5{$TC7OmoveZ7jo3Z**J z7Dv3OYRI>1W5Abgo~Z?Ahu+-W~FI%y5Qgun2|#D(bnJ_r9*j?_uk80L#K17|9M4bL_hj zu1{JIy=o6OW3RD7J9j==AXenkjf2-(ZST~cIsG!KahM?~2~$iZJ#H~NEPK^rz;DJ% zv{}4Dw4?HbT~}YXR(DX1W9y>nlc<(6(Qk#Q_v|AxT5HuFC089sT-4SKm2Cj;*lbtt zJFIYu!RwHv|k07Uv8)4NC2AvumK>S@PLdkX4n+W%vK!2(snu(3<~n+0Qt> zUoy*u;M;z`96g@<63GK*?z!^utd2b*l~f4Z;q^l^;p-z|x6=%4g|%(iy&9K6j#_w| zLM7*45f!88&%C~ppZ;ddmikdY7PM2WN&t{I2GccZeC(VPxJqxixu1pajajOGF*S>B z7((DwS9eM*I_Mwn6OF2}@jPDZQoiESnf$E6E~8y=P{yopGSr|=4zKl7e8lZpn}OQm z6u-QDUykp@LZYUu{TKx9UYWhWKbfsmnB^tu=S_pGsVqM=Hg4>A|B}|VsOVhP;;W#Y z_`n$E2*yJvxNpHNAxbdSwR4KBGiXD+^u^)itZCm+_+lBeiDBh(MrpcG8hDm2mrsMK zu`59&{Wbn$6dqV}lCMv{aMtR=9JRUzsST2`zsWo5@89=-a$>G9?$iqL9ue|gk6LVzmg@oIi%u~S_@H~E%@=D%T^xA5T=$3erzxdt3lL**=db2%1i}dfP$_r@%n;?&H&pfXmPKT1! zB~HW`(L`n|ACam$*La!_y1myk(H`lOWV5nak&}u!3|IpT=r$^@#R7J+GH7lK-eo7R z#nj(P@~U{@I?`Tk?L2)%W6Yc;~(ptd3R&e{B;}d20hul^rM#3N)ri}UEc#lfMZ-;j6viI zbhz4xHpg>-#tyLviMc>eKbvhk-C7>Yxzw~bos51WI1}+Xfv>Sxau@6X>Ja1H@D*!b zo%kZxi`^h!8uTbTb^^Un-|FmNH+7GHbPUe+zrhi9KL!z*E!zXbRrp77`xl+WIez1$ zu|FUTsQI8;FV7I6w3+xDqi#}qlO;KA28sEqhrd;vF})l~49}DDQ(rzWBZ7 zd!LCgE|^ozk8peJ0e3*`4xi6)MX6yg|J`cv^{NLF1<4(9gI;~B2)s_n$Vu{WQV0X;UG>^YTH&kVGuGX=TdaE@;ziirTu$UYC1<6>II4P+ zTL+~EI}B35$E!UoWatpE(O=};rkJmILD*p;pjsB z^_c|kS1@f9(n%{;+QWIdg29aB7kv0{O3Dff0pZym=!lumvFn_C8~hlr0zW;&vgaBv zaiXo&*w*ziN||M9dsxjqtwuvMOG@b~&n}FBdsD8J?th%;S#y8TX-|6avX}d$?3qJ;Lg|NxObL$onz*K`~5rZbf-Y zwH%aZP`lCHwC&?;KZQf56eD|E+IX9{b`{~$jl%c4y&&!qdJzXGsNnP5h~zAs*01`g z#{?tuddX?`SxbG9gooHfa=ek_J%bbXA~WC3l;G6>#Yr0;3IS%c0Y}>fP&8S&wYHPa zU|<6l35Ca6Ak6_`I9zq0E?G(4HyT4?H zi9!qul_4bqOa>m^hfcV(hf}rp-%3<+r@?`$n_H`20_^>^pZriNM`KiO z9*RxOCWuU=_rqe-Q%VD?wJ5MU#)Jv{?$)i-?^fBsGsjHLH@E)MIy;=L=JEsLx?T4| z*-j(p5fw5cM@0O zR$c}Y+bFa!S6*Ho$jboAwU5=EVI6`NXIbGCFiJ|3R35#HZl}Ye3(fi4&uMICD6F48 z#gbBtt7Y5jRR?}H_JvP=`=P|;i$cvrRjG;ZBZAKtNaDO;0Iu;$+!1VX_6(7*+n$|Y zMZ$5OBT-MDV`nh=(&tD$r6o5o7Ch1?te>=E)W&9PufGskpRgG;W8076edNH_w?EBz zQM6MF!FAw}IOyoNfJ+dwL7J4oJt56xzV6NI51MW8uJ;p}gxnpsVUcWH7dH4gkch(* zIuwatz+_I)VhBGUPQ8}7EKFnE0q?pyxl9sR?sO{h$+~3~rpfKO-x>C_agpTgXpnUZ za~uy3#B*8N69Y5N99j`r=_{qAVe^RV^yCxB$!Qtzmv4qbei}IG%ylwJeRVC(vrk&P zqym>%lnQGX4Ef(I*SI%ky5`%)#YJ>G9qG^J+IY*o=n}X27@>{6C{6KLyS+5F(~E^h zDCL)B6!T$mrNRyEX0DJ(3io>+$01$_7UE`Zq`rz}PfY0^V#G~5MMz99W7Py6!4yq= zUP^cVKcUjTh}9SCfjGRBToln6wfDP*;z}WPSu76ByHN+x zGtyH{i6zONJSwbPAt7vvpA8lB_{WVW+8|A5}rTM^nywHm3*d%BIQQGNRSlU9Esvv)!0bdYJFWf zQfgjs_AP>RQN@FKY*UeTtU z_Qbc_cVqRgqJcSA@}->BtAYG`qL(NQM>m5_9~{-f$(xhwAJcbOswOCWS-?F&}0#}DM;!WikFD;6)HHm zu}$WuI!j5V6}+yZJ7a*u874m@UOLV}C#WPkFJp^LW;tL*#%QZ+D)R3VgklPqmJnV!k)Ax8;>Oc;BC9x# zr6=@lg|oFvtH=vvox=ckY@Pj0O=2y^ueBh1nNOPi2+Lp%Nfu9?S_0#s&qhNojSVGg z9*Hki^Lb_ry{oF$NX$6pmw4^WSm(12zFpGZlaRxx+{mP;GM7>*4sypM#UMIcwKV?gCD#yM&xYj#;}3cD!&HlX?Hu8 zlzGGUmf<6}2dqz2uKegPD+EULi_jIwR=i771?Z=_ty11wiH!JPq;vo6Wte-f>uTl9 zz^zTzeQ@_^ugGyw6@@$ttmrx)f4WSl)5%k=SZi1PD1kL}`s&uq7P@Q~20ms)rsRIJ z1|l*0*?|h`Hm&A%zf;+K3l7SuXqMTz)ufrW*$2jIRW)~Ni168yMhX+|{ZQIF_*(dy zL#q}DOCvEUp&Z#XrjM!fQ!(`u#iWm7U2lf;a9i3kY|@K)-li2v zVip6=pE|E_8FgmQa?Jp^_d)dh-<}L9(JL2EcWjo2@vbSotdc0R-*rFDStA7pkH-1{B#HQp-N(52}od8m#hGrm>ag z)05ib$9Ims^_FAn`jRk%lbcY-;%?z*bB|wZ2|v6W?^mVu$vr@<@W&oR#G!ffx=aOI z%zU&9g|FwkHvw``x3R_H74hEq;JCKxv{9bR!!v|;6@*}EBc>{pTzMQC&A~jIbGvHJ z_s6w?1Zp`p4L73z08Pl40)0C*_DcdlrVywfBHWM01BXiW#(;r%hmWLvC^t%^v~CI| zgHzF6--XF@F3Fi%r8!vCVu?H@H}6u4Wh6h@q3LL>HX>!6J2XYZBa;qII);@gf}(d6RDP-02=S!xr6{yd$Viw^CSmGFWVo)z!< zzpTyzi}~DSlx{qY!vHSZ_q5g~mBRy%CE%b=Ak2l682u6wT+kIy%nC+i9OuKGUpdwZ zu$+0)9)CuPvLnpPXQuedDH5PW&s+il7CpiMHzsz+CUQCb@^no6sSgAL426w!SfdgQ z43lR2q>Tg5DCFflsFA^0P53qOZRpoFGV9T8V!+8WCD6#=%7ybvsPq^Q5ko8JOE~N3 zpBcgnwN?`tWi3<3fxW1rJ@uJMvkGzovkkTu;7((qH(*Ops>}+#Gn^__FA`K_ZbOct5G9d|c}6XO8-ai?7)XfL(Z$_` zkS`XX<86yZ$02AV&R4BUi--~94LP1}0t>kgk)_VKZ-#?<^~hxB&R09n zcvMy0LP}ZH9iFgc{oyY9IkeA*(}awe7zaM#Qbc4N`5HYN{4C*EjzCr=ucl7fG;UtTSPTZwpF zU}zu?z~FN#$6^0bjrorF+aDrmqC~;ooR?nM=HPLy%TG9fZACJUqn^IFKG>jKFBi&= zB(H>D0t4WI=@ptRrpTYPmUK`RWN`WOxGE?jT*3(O7t4~i72x^eKFM^OR`egqKFcYr zXf6US77fO%7rJ;94Q6EuR&yagM-KD>PcxW8iHt$68${H-AyA7Ygh~ih8PFPx3~+t5 zR&ZCN`nN!OW*yn~hxjF|BNAu0*e(bX=W1a2){ta;#cP(>T*C~O7jVa>>R-JOt(XoX zK7xH7s3%ez`5)lroclFV42UO=HcU5a=&+2i?nPR`V=v_~_T+`Nj$>rsTQ8N*!yZwU z)#jnU@{Y1VS&R_QuiA%K<;sh4*q4I&Uzd1ow#4g3cC=$3&re{p=nR|Fvxr9$OtJTp z!0dwscsxcme8uwPl{Vo2G(RlWau!I7KNn<+BL6#{f_I>5HVvv;(|$uiJ6{_r$!oF}1dpbq7UfdWK7WDqczg6t;Z++f&}c$k zWk_bHyaH#dH=<{@+hWl2mSA_egLS&>hK#30e{KZ4$CR z<14f;J*Cv^ynmvc@L(WD#zf+u723e@IGsT_J>wy>+SvJ7Wiy^{l||v14vRb{;SxU7 zbcorX>(d8Mz+2P`YRz?pKr-W2ASeVCbwii%cvA;_OI@W5AyKiT(o;z5Q#J{h*N=0VWu40{aOmWxP=@s{{s^+kC1;6HK{gsH$VNZ+W~E0~dH6|`e!gXlV_!T= z4AFItBgPp*5+r{>>Mi*BCrk`gOd3ZBC(N-9)fCoPZQP$p4vRENxlust06+f@qd3|A z7}9`~*6La?24zlFD2}8$Q~+^OjA9`#&8QZqYXwLXccP(_c95>z2O@i?O3>70*I0fu zOP25gwa(1EL^Q`3rc`QmfIBj85Pjxu5@Fh<1QI6hv72I#-1Kt+yHK124v`}sY6WWc zGCX67JE;}hv2v?5ZcT*6DL3NChU3^VC20C_E#Jhw_iMHT-ka|Sn#2?>4aNM!g*@S1 zz(OI+g@pykQR7@%OB0b^Fc3bU2(BjiJu3XXe3)LXE@+s^y~}$&CAz(;%>E-qV=zTi zHtme*X~JS)e#Ut+wn|92z;SqhL^f}4F$)Lnl+)5phKX(~rNL|9>GnbWbl96$QJYR% zFpwdq21%4qD~GfqG$gpB7a*xj5hCn1gB&BF$12Q45w2RgWNjCEJY#N(BIMa;YZ>Fl z)TF?4=TFd63hV;fEl-1_7kC9ct{*Hh_&noN?c z`L8#^%YW|OTtJWQO2kbYheO*U^~Gl2)`Db-IT8>^Xi>(GBnZEXqX*OO$FQ&<0w~m3 zYsMNAFn!v9=9zRvu~$~_*S>G;ZOlG-CRbR#m{n!VeD2^D@s=KLFAe^*E1Y!wbLw8t zsP7Jckg~dit*UdW3bn)!6Q#@*pz&|K$$r=bse%0#C^UfU@VX2t)7ip>E3B0U^t8WD za>#h*`la`mq)FYuRppMkK<|_Ke{(WQfIy@ZdSDs%5vB|INV~YwS7lI*B^3q8CF#jI zb@Z8*!hGqeqd;TMnhU8@4#Mp9CQ!&qYp@_PBfqT3V^<8$JI4KVmOIXn6{n7{7G=9c zk)cKRS(s4LnPd%!`$~`gur0;u8)&#UAsgWC_@nI|C!ftEU27cL*#Js|RDhYYE6&j9 zXoy(8e@H`vNJRWtI5lIjWH8Ca%T?c)JFKwKm|s8NQQoE5g~O)FSv-4xoDz&kP^D9y zg{(D1L@&0$v$;x4rI;~iVoJ)-lEh)ZK$~h8Q$@cSf0HFp*3=fvL&scJQ*7IL_U8hL zjr~lYhj>9afXFFSp03U`Pv~6!d5EO`w zu^q_`UmYm7yL+HaLyx>|P-8*V-$<03G+{6j)iK&9z3O6wJ4RlvsJ94X)R%Xdk)YBL zD?=V#WFyN+NB4VP0v9j_1D`jbXMuH4<#ojQWKe(U#OIDP{d!6_rVK{7VI+uG*h>00 z=$iXj{8`bP=3unzT1O;M5{uOylQ@Ma>mf0+Vx8^Gf%>bSm;Z98ObI-$<_Buz{Tyh# zO!o~IeB@~}3Uw4GIyaqEwSPQ?iCuZzi8V|B$P2}!(xL2laix)=u(cXw zNZY(QlAkJQoy#g$_VI4<7Fx`jOB;u{T7HVVsBMx_?fdXi5602&3%uB^1JTd*q_CC6 zxgcu}8(;@~>gorvyjmYYyFZl%!^6Xn??O9dO|c3r-?Kzt+r6o=Tj0UbqqCQvB4ivt zTp4+N?$5AvmwLS_?^|?IxccaFQfQ1Ng^6LnvX~&FnZvU!9Am`JXfe(&kF;g)V>$F6 zDVJ8I_KI5mmD_A9`qD%r06 zVj3U(f3OG3ZyyK8To{%ZGGl!|%z~}l-}(rvFoCPRzwVgzShPk=*BmUV{>qa+;%AQ3 z!proQx+EE&r*7dhBTq(L(r74+B?3!hWzZRs?XAh!`?e;w!PC8~vEFb&vMK>vG|u{N zO6RfK+LGWgqSAw{WOo0Pd#y|H>$dO`8fB4(u((USh+w%qD?u+!-qH^e$opQBXFd+1 z-6th&>Ooe(P;X0MTsTgw5 zU&oKeLO0O+ZKo-OHQo^)k2l`#V>3QG2*@*FV2RyO0m`w+o1!%!XTag0HWzp!SI$}g z>xl5Z!rD7SzDGpPHlMw6)HMVs z$d>HV}dFfk*Om3Zn6$Y_1L?Ox}%Nn(vJ!>Z+^Zmq`GHl3{2V^L)N#$wZNlBYw+;2RC}Gvc8x zO}bHxpSZ;eD6~RNw9cuNoAcgr5kAb zypXEd*-wH!`1*C?s==2y66&T#zbe=j)yGD{t+(xiPyBu$A^iTGu-r-#L%xig!B>*6 zb7MGwt=|B~@wuGQ<8s~AC^2ktR%!72o+p|BZm zQRG-^pvLn1a#q3>W;{V{CBeZH+(bSfmU>o)q^t%yw&MpoO{Mr2Zy-z`ORB zE95JxVy~>5jpTynn8))#ITl6!vT$zrZvzTgD4R=td1$)d)9BQv-QXLWg`<Z8J zXf5)LLpXJ$3T3z-Ik5!rKVn zqnSc@Z${H$9lt!dJGG)L2r!OIg+(Fc2X^KC(CrSUJP9Cgs}E^&XXjn7_0v*<;J4qsuLS{Y4&UYB@O#|$+%f_w%4s{ zcGhxUMPJ-9FJ(KnwuU*9U{7z^9(pU?3)sP)Sn?lI8dfn9A1RFoxBEvY_0AIBR}h~} zbhc)LDtY{IBL5jE1&Uem{%9wZC;}wQN6cob6gqc(2$+dll^}UI8{;Uz7FLTnZpj5v_4I!riTD%dTKeZLN0#6QWO55?v)H(0s17CeDT((B zcTJU*4^rx^mv283^95u1inF)~xRY`QJ2I=@LUM~oe6reT8{)hu)@B6$yd4pZq@GbO& z`+F1{S~j;mg=|3aOKf9kx7Q9#4jG=3sm{8b>osoY?Jdy3Lci_yZ_j^eTORI>+{Tu1awVBAVFs)fRL>5N z=4D~aUc8KArumdYFx8^qTt!K(ATkpV+rc`2E9Ioh6lS^ypmNqklDwoYoMPtxzg8y| z8JIEAl^(CxlbLuqm=aGn6XVOWG$~-fm0%WLb8b?LobmRV)zguwu|y`u@%8{TmC<>L zGkmY49h!Zvi_5|g&>xqbU;u&<9CbkU{C-K>0Jtl6&O3zHSagPXQPldvKr^^?!%xkG z)PG~RP=U7rocFC$i5(aZeiSgAT^Tf%wj<%*lv(wv(3yg4g#m4GdMtcVXBVl}x1lW( z^$*jnstE1Rw+_9;mP1ymvrR=LnaygDbNh=)mG7aCe1c8N{vtu@HBPs;p@76+fd{M% zj9tc9B0v0a1C1dU^(){P*b+ z`W(@J#r_N;M^aE9ZQZY+{Ntn1uCD2SbQuHmj@RTgBXeJx4!?$G&yokUd^rwI(f+;9 z@B3W+tqH~P;S#!mfTKAZB-f5cqS`Ty(}bd`b-%0*MS}Z}KI5L6tJJaR4#mxUqgqgJ&VFAn`%gH*%*WAZ0}PQR%q)jM@pdbN}q8qDv0rwmfK~_ z4loQ!X|1xRmhNMY)~EGU2)5fZL>GG`PlHK-_)hIv4uWK9-3;@1#qbyVTn67tDMNmC zKYwGMYd<|x%Vg$BQ*|&U<1nAcohjjRx<&(qSqy+4$zr^9*IvY#Z?S#e{@}~f-fG&G zJ|P7kb`t`pP?m;;VY?$4E$pe$_Et>099g4ODwCHt)__AE&RxP(9x+W09ubHlQ#)(u zhLVcu4Mih)o$iqiCc|c@6H&^@YXsrxh?C|3hm0qnHfV1D>Xe}@&CJ6p_J=Pm=mzzr z&^u{amB#(QPAa<2P>X_~LV%{EXN3WfQ0&);dy`RUCgn}^T|jvy8SRUmLO8)k9m8?H%ukQ79qcwFLTX&TWVUkyF+=xVJh_`BkQ>>1|)!r+^>dr`4gsqc-fW; z6))?EE?n|zyeSp;yh}g-SoW@;3r>&@PTOBDPE>GhwJARkO_HNQ317eW1d=01%Fi1cu<&7u_;4MVd(r1_a)|Pffj9O$^iLC zP0&Z9R6kH%U7hLb$?LT(1X9z7D>z^($WinGlz#kz=JBfE(r8_Fql{&deqcjdKyxRqz(?%Q z_*}>}iZYWU5MpzxakY!v9eV|N_YH2S9|QivE9>mcX}sB8#BjAu?BZk=xlE@L_YGZz zA03s%K}^Tx?CaC%m+I@j!p?KPbTzsB&##YH&>G*)^z5LBX(;W50}^NJIkbdjljN^j z(yPN&_$ObxpyrLeER#+t1i`zY!0|%s>tH|i;LhHjtY5eFr^P;{C&{HkKm4U#tjNR? zMR|sPO&f_W4Z#qX;+P7I*ZF%BsChy5E-McaLB#mfQ8tPXhWzfCIq0ele`S1Lc6?)m z28ZRy6-I4@y4A)QfWi=qg6x48n7)gMxuJo4INw3E`tj+-sB$SB^U@?0t*G8;yHLui z99pklG77gQY7IJhjjX!JbHF4*3=q)v3S_G8@S zjphYQ8ZzFIk!Rxt##XyCVkjKqMTC46g9o2vD^DzzaUQ@v_h*Ca=@xN;_1}eAxATnZ zEKy=e_)(!VL%_f$^U;2yw-UfLv499I6%+VekkEZXN8f#Qynr?)CTwhI4!oup`};d{ z=EbuKR%+XgT`oY^8sGe@z(<}BSa0|uukCv1Em;R@gBU%*tm7iwm*Xn9FGZH!alQ*R zwkCuH#XIXN&r~7^wqEwhbW^K_TX2%CNF(Zk0u=@|BRcMv?dGoz@No>504<<@wTQQJo<*TiM{oI&u_0&Ta6(h&1%m{X8>frcFQODbzq)Rk% zeOwnujg=6f6N!QuvpnFAn>R7Rz$S#zei830yJvzBDz?{6=t;@R795=#@Ef3vZ{{A# znG!&r=*JJmBOR$9wK=ccWcqe;a>_rL!;%2&O5O{h165M`H&CC5>B5Nz5AwE5C)p_9?T_L`=S`Yu{(dQpee)2%1zE7tlMq_gy8KwZBdw){^CmYnj)5Xk za}o>nHrx1Jz7ZN8E^gnGyro5prWtJwu=ofXY%RMl*=rOk9d=BMY#{3lg6f5grV z|2#Gu7VY`hOT@9=M8r=Ye{RHk-&K6I(y%3PRG7uUx-PByNZ3c|t6;HJ5lZVV(-CX{4x`ty4>5@2-!6 znezvT#Zj7v2N#0BhjI*(zeGl4At7#_;EiNZ#i>mn0?ka58A-qN$~!)dJWRxKPpxrD z;_=Y#0rm(xb8IrWio3H09FSh#-kbYN9F$n_1>PXWL%7>o1k8|? zY7eB{^6csMA7`QQrHmYC!2+JOJu*NpdJG6Du~k8j_rKKtXpwo*98pC}Qoc@bl^Wpm zZoszs1G|6`L^6&RrXjXP}#lDsOe;^c8Y@m0~|-^-ZAjpUw}rp6H8)%oxcz zKBHNHSRjrtIoXeak#61_L`6e+MXJYe@4}r$xTs7XL$33-=fnDZviDyaC<>J7yZ%M> z-6oYTgN{^~<@WdUE1pTxxxlFg_x_R3@{z~ov>S?!fl$Q)4mj$5ndcB}p!`K5z`cW1 z%tuh=1CMNbkaDG(k`nIoeo}{K$#z10frK@{9OMPySamA`fbF5tCtelD5Fy7^k>>3T z+qGX~;^FQ`i>Zs4Kxkvv)ctw`L}CS@V?ZTV%UQo(Hm1KQ#8O(s)m$&APlGynrQpB$ zxI`b$Gf^+Tm}lHlY%BOqo8e{&c;IkYl5(j3uT%eN`}^|r;Hcl?TsSBJ%t2fOYQ=+q z?K0lGhGxJiO-6R{j)}m#?N0|rb9M^8A+2%QQL7D<7|Rm1aE@)#2n4|9FVsQf57e#I zq8F@u`Y_&c?evI3@5YTt|Iq_6<97xilSp*`ReVbm1z-*hDkG@{^tT5|$jKo>p&LxX zVGpzJ2M%;HKx5ZkwSQ-^$perwe2l>5Wgt^mq?SjZ`mr!NNV!G{D4;2G3ZK^Y5Fpk9oFVR5SaXRu^~fx+NdR+D#0 zee~I_zX0i3$oZK@{u}<14DE(*Vus!W`fKQ%-^MYQBX$y`$i(bjl!d+Il|EQ3UYq*nTCUb@MTW)-5Gmpa`@tI;|g?lcs^@q-nV7 zaGybT2yagOpI(C%fGxG~ z2Ge($F5j88!-%2pSVA5@{mPNDUQh_ROquR0?>)B z3iE0hnlX|K5Xk*HG7<#L$&y`atQH4;Lr;$ka9R=ga5gqJkb}z4Je>}Aw2-kf+!atQdx6h@wHNs&*T`c%^z^WPEQP$-n#_a z^Fn~DqEE#z6rl&DL2xW+!@L{Pjx;sMK~c4zwg%bacz+Qk2J9<0{Uy3<8P55OE8_3& zU=rQ`upuCWmeedf&j`3+iW>+x)Y)nn2XR`B7-Y*gZ+a}^MH&3G4;HNX5T1*Qu6PtS zj)9Ame0ewBRjl6!o4Iq%^5J3IFW2IE;f$-5fN0ujWpbbLkjx=5$in>XZO7Zu>a)yoKJ$twtSc$^vlt^6_Cfv~n=) z1i1gmhA|Sv&?O~SCG{<(R)~S{NSIyxbB6FhOcz|HQ=S&(Uaem%W#=G8O2G#n&(ZEX zB%*VqSqn$)G>WKhDWzYzN63?j{im-SRrP;wIWnZsh?>;tY7lTa?oa^t&C6Z@{04wW zpWj>WYrqpU9=o~WH0JTKA#uQ~_qCJ%Gkk$cEwtYsp8<$l1VuO%HzcH9m^axBX9=Az!5rE zrq^xG2npCFg1kgQ%0vQ$4ZjAJ8Hw*(Ci3Nk;~p}qLBjQ`a(RX0HfiiYHe$ z*sh<$$!5TSB7i30{xJ#+J`?@)KSj);5~gsKz#wfdivjdqU{{Y*i)9<|NdO=8dAOXb z@i;SY{`LJuu6ch{|1NV~(KVosmg`lA0pqtN3YAi6Y(=7G1p~LSfJG%YF8T8R>&T+i z{!d312>iwFIPw>Layk=NM?3L&V80SBs0&141a^T8iq$@u0ujhKFhIxr*Kioroo=vm zD*J749?>}4Yi=^}gSQaS2)lqiZK#w!r?BG=U4@suz}=BG^*=e=I;cI2+!AyhS%dOo zxsYwO{ZVS|+ZWV_+deStE+ElksQF%lr0 zo69*9v&9A*iG!CZO<0WM8#yrkk%RsHkyUu|pY2MSHm|Cx6pZR>9ZqPaB%W&C6-u+; z0fy02OH}4Sx7A}4Z1VSzhTp$|K@Ir_)||TFe{<(r=Y9wo^xV^+CjM&Q-hq`C8y;$n zh{A!=rXZS-uKaj0RN#Y&TpTSummzf+ z=Vtq(qv>k9C)eBQ5^&qqC2-yUV}~#u`&-Z9D*=JMe=F$^FmY2!ptY`gY{s4flaVE) zq@=h#n2eKMwnw?EA(JZeleaB4_vre~Emg=*e<#!W*8MNy#m;|{VTdN7G{_g$!)_C_ME4)Jcb=Oq&$GqZ4h8=idM(L%1^xgJy{?na7YEuu z$Q61V(@I-)nA%_{AKQ_Pf7_LpV1YaE29|mB&;JyN#7J<{NPHo+feSN!EPiGbY6+Bz zQFr%ALH(K;DMq-$T>)o+oj-FsR7Nv1rAX(A%N#`BramTf>2jFnI!0x+n zo5G(n9vFO2V&nC$|2H=${%?>N7%Ae{V03p2u;zcT9{{HQP_`v2P+qu%g9FES7}ZMK zyKsPY^LoY6?lOytk_$MlvI&R-f`Al=U49N9K+3b(|A{w&p%vc$4N{U3!jTHt1rQl~ zz*co!8Gsk+w#*hx3x}wHB4vLvx1{cDQqi~nP)%*uo(YT@dBq{wRwF2HnBfCL*}j=> z=$%Tn2sOyXhR0%vyQ#2?((Klsb-H8$t#+40hR>&Y<> zzXypf)lvDH#G%wFU7l-HdlG;rT7>y*O~`RsQqc4-Rw5`G)`-vm5F83}T>Sy_dah71 zUqEMln)+L=v)1q6MVdE@S!6fc!5B4rCl4BJPV~FpvdSd}^`~>Wdh9C*Xhn_zeIsX( zefn>?HcJK407Zut&+Li(L!C1Kf(7vkb%Ww(1Jx-|(9;7fOIUAOa)^#h@Wg0<%4$J4 zOfQf@4C7uGyRarcX9&|Jn)!zTUF=pMC2qI|v`HO3BLtv5GRW(4KXox}91%GMx!{YP zfX!sVbW<>ZbwT7+bclUNg!7Xn-4e9d8%tX;(Kp9jwx*-HG5Izvjuu)OV; zYWP%2eA=DNYT9eE0RBv4C|CinU9RInqq0=kw}gbJ*Rx+pJYM4F+3X`DUt(OH_Xg>c z|EJFctKQD!+GT`DL@QH>ixe+e6&MJS{RBht{$V5Q7pMmSmw};$cmO7HbPBOx2TVjz z1uzl!^sZVGT5MWmHsA(*^DP6xVlXZ?F#?k_W})8tPiO!%v?OF^`gq8hO8^ST1+29x`ztw4$CtJqx`dn$j0Q zh@tUXDq3tjWN1&}?ez&3Ajg@j^tm)k0zZU_diq|F-g1{V&S60d-k4|_g|t*zVD`vY zM&W?+Do?8~7#gW**RSt}?bG77Xlz38z@OC1P_0d-NUYzc68WYie?V$$vg~=s`*Llp z-$o>QK}h7=FR*Q${2{(@IfO7H4rc)4Mh6fAp|gRF$NLs- zg8%cykn|exVi?fT*|YNQhy5Fl)LC7iZX3#U-QDUoH9Dmfa}r}eQ@qCX=MbhET{$!7 znYPyw8gdvb|HaQ&g%2<==%@`4t%f^x^Ip+MAL|^QYU7T-J&qc# z!Ltk4iW%tsU34>sSIG>mo!b9=n32ol)HG)W*siqy(<&`*QFYV0tLK|D(I&4~f#eQD z%Z)GtgMxOPOjc%ot~^iRRf!CGGWVimBK2PvpWxpYpE((`|4FYZNVmIcT4Z;Tarhk~ z3B6j;>H{gmU|q=0lF2T*8_5!guCNd)afkH7VCiI49QlGef&OT zGKayv)_;0|S^!5Mg_*TJ*UH&^=p*tWuyt=0_&0l>fr4Sc5YiCYg`{2`-pA4qDRGd# zA_F6JSg+A)G6W5$^d?&QiP7y3jDJ3iAasOP=J&P1`CYHfHLY-#YrX9G{?Jp_8!mnl zs3*3>!t{5Q$*iFm@1H6YJLDK*UPPc3!fe%;ML?dKWE*4?gphn8U#e7djZnj-5`sU2Bo_}x?v%Wgmg%E z2rRlA1f;tJ&V%pw?X&ke#a2Co^^v-|cr_Ev)>Ckzlw)n~4fI{aXji#9Xc!;8d?o z<5=e){(hl}{g)v)#i;GVvE&DST{0OIsQq3dXn1TaHvEGl94mdZe<1MY1q(Q5%mUsc ztU}c|+TV>~pF43-28w#I^7??aK9fBrP@4sm{(%4?B%v9geJ5+YOiyvr;!Jfzgl2a= z`ceKx?A5XPAemZQY0w$57H1Its2#@X_AqMNA=BQ;QEI#KaI=2#DmAQPQ!!KRZYqL| zE3=rroA7}oDY}Tfe}U0Kp8^VuVrwHsrkjj2V~TRjEC{`kOzf}*rcUhJf9!HJJ`Uhk zTSIxqdQ!@kPHMnUklp8c5+iWLn=0ZftX@iRCpI8)Jw)={2fcv6E7O&mhwc5-FnSuT*3gAUi$OdD~DI z;MP9_>}VL(+$$YF-Q4MCfM-ntym$%De5TDD5U8Hp;f`;WZD={bBuc(E!e~9}`9%~U ze0_dSF{EhZEEh}m9B$2kw~Q_F=LC1MxLmKH3%HZ9z3(pk^Yimn7vBJvv3u4(pbe7? zi^qSaOdJs$$HFr+U&T&7%PrUM`V$S}xk>_kT9{F;dacOeEWH0X2mev{jOVCP2j((^ z0R;3ZuSS2(k`40C)$ze|1O7NKnEN|BGW5iD@b>{SRx~W)|M@q7e0zj|EWxQtBN2Mt z-hXcFuf@P|aIo6j`;wIf&&PKm@+OM8Ug32qU2-fp91R5t>g1|V_d)(d-p(mt+4N%^ zEAW9F39h!rNKC9n`>E9H7NIz*av7L)oxWwMlC=`^dui9YU-4rtg1G`mz#+MC4*TW) zb1Rpk?}YlSyoXkq^#_LcaRTV`ars}&o_I7+^j!s7`7iF3nB|@5Knp8m9@pU_CP2fY zjO$+6;&G$mpnjd6ja2r>k>MxNHMV)-3y@eaZ)Vmn6M+{C266Qcz^}mp9c`;etQD!i z0&I4uuYmKf6CIfH@6si}d-T)dxgCJ;KBFy1$PjlVBJ3S*gZ<>o8_z@f8#5!W8~6tB zY84y4+wqwH#f857z<{X9!006kt$DCGO32?iV2}o_TmI%rRfEZ+`CP*vHGIurByz~% z8iT?E1d-AcIZ1?oW8w;HyOQ@N&c0jpAc>g}If=PnJi7D+#7&i&Y#M^+Qr--|P-hKS zp1~b-w!IxN+dD2_ujBN00t?Xd$V$Suv{$~{@#wT&*+5DMs9Fw+($DV7`xW1boU>XBIETTDFnhl0XnE*wTYTpK>x4fkQMMm?3!f~HNFy4k7?%|0UkF~*{^rZ|NKlZ(sUXFv^v*NqqnOf^KiR+rA`9jS^ z{+^f2bA6YBHlMnNnmjbN_2TdwmFFrqzOGnrsdAkop_&M|vwuZ&8@o5EwpIL=8OsNl z57T-gDx+FTPkqzD{X8QVN_t0$>1s1L8xxUNVq0U5lM1i^WqJ)*_5jTt7K?4DRbu`X zZIu}S=O-FmUh$>V6TSG;tmt9e&NDwC%|2JM|H9|Z(haeGZ=Q=YV(|-IOsNLtywFZS zlNd+`J#)bKPgB#}z)JV*rH8XLoW}Y5@D0_T=+(VxcZB!JT{rV9p!dw=EylWNt)?4& zkM0vX$@cQA1hv0pWsKpSF>-NW2HuVLnbVm`rzffLN`@Yv|1V@^s@kr*sI~;4tRQgG z|5HYzfKIfWsMBI{o#sni9BR&E8>V4)4qS%2d1O}c!0Ylul{PQf$JB1}l9S4^?G0d8 z_g{3Cmw8Bwu0x~h0_6}&7y}+V*S9I#gl=VnNh9^HKBJa>@|{C)iqsa7#>XO2F%i9q z{u4nIFQwl@NG9_ZvCb=;!iI{~g?c2&l%?aUvy>_WUG(G?Q7Gh)(jxZ(^-E|C%I*`H zek=nsGv(Rz&#{{u-eNTl*f;yrD38;vSW{(&pJx-#kPB>8Io`c9KHD0E=1F1mK3q)3 zGU7bH)cgQ(Jz8p-sz3Y}-An!WuLjBEzWZZ)Gwc8``~8RPy&>Nsc9Qmc04V%W&U}Qw zkdx7gsxR*BTG->`pVxCq>Bt86wEhC{(Mz_EmLgs^7q92)g|V%93Nc|8Ks8u5$mj=S z7v6j)f3n&;kIHYM-vQ8Ktbi(Tv=UI|SPNv(5*r$_1*c`A5Mu4G&96`H zA`>{hJ6rQrd-Bf_S*3|LeW?5xlR-Asop0;+N|UK?Z}TyLtNnvmK|XKeMy?yQ);mg= z;g18Fz3Oh_b18VO+dvLp1Ht<9y}Y%jt?_nu&UFWK zj@=8Ne{s@e{75X%1Bs{(6Y7(rAIDwK?~ee2OGq9)P6O!h%}>p3aNisES1h0OeO33c zS$EUl$B;r#Weihd^h@VDgNTj@f;9ywt+hr%OB)f?==9HNL?5cA{}FE^X_H_Q={Kvn zh0v(>JijATpl^GS6CUruCK|lBQ{kk_H84GQVexqk5@OK&mrC#lz1OCAlm%Ha-IEoM zhh}n*OcEC%rd$N=T)LT(o-**HjI@eKJEFP>ZWdl7_RYfiRCJCXQId-2G${4T`U8Y+ z(%Qgbq2ypV;T?!U4(eT+7LxZTqZtRZ`u7!ng6*}m7Wstu_x~!;zYWtOp&xN08wj3i zZTe2lh5tC03d^L&!Ja0W?q`$V+nKg(-Dy|Er4nqSGcq8?Mzf*kTd)xbuaoOC`INma$1{-tKXgFs#U&rSg|{V zIK_LpKAz9cbOh`z1_w^Q`G(DaX-!g#&oE-X*v>4st6t1#nV{+w3T(CHQSGz#k(1kR z8LtFtV02w%gPbcqVn#2jpL24O;@gS%Rv`+1VnRG~hBXpr#eO25eDQiM9;WIBwQfu- zTIlasu8#vMb_kpNckCJUZxG44m>BBi9E zPYyp-0J_yjc&a!GeVc#Za~A<_uqiyCgVujdAPqJ)`^6RQ8`}9g*+7$i6yg_#uGP{w zaJJf?8q9x&!PcKqDVDM*gy+o`nQh|$!t70@_Xi8g7Gy~>BHAfWODO7OF7aowkuWL* zO!hW0LgfXCWl>YPc@5E>E`UxX5#cR~r>vgU5~l+7j>3RUcGCCRfRwyL9T}_WyQSYa z3XeA{)og@5i>XMItn=_Ci`Bsng7h^?;0%=RJFej5k#sP;*PCDzz}tl91egKB9JZx> zSzErXB;fAHKLaT3-UWL=(91SGC<^4SO_mpaO~43I77JG1dhWWg%K-~hytpLVU z_Vn9+?im2ki&{IeRggH<+6uXV94b*HkVlhdI6F@GNl*J-PIv_QKpk@NjTpWXpb$eHqtp5?Zr(ciq_&Vng>6Lxa?44s~i11#WPqpY~V++|S9;|w~4%X+3Z}3GJ_AfaFM(d0www3Q$ zrg*#mHEu<(L)MM~u!AheGol-5?XMyffNSDy_Oy5uIXSS@Q$J&&Y;G71Fw@-8;eDmMPrVmq%l#qNfbHJ#~~RX+lUQ6?-kHkUyrCx!crj(0lX~4^>8dW&zSs^SmIgWr#-L zR>E)5hhG{9CYU5JR7+ zo&3eLjf&3CFqL1z-Y}2V;ZB1jQZ|z`I0ebm(W0gpoJA2;1#=ZPNSA z;a>4>`slSre9umHz*4d&qEVULr}lRn;B4L9U)!i_FdFjw{63fa`op&tU6bF<_T6yz2a7J*4<}yt)vM~( zuIYJfZfhal&ubNKFgvYW8oO{Fx1GN9y6jDX=9)>}j72{oPuonrb=)X0`$Kf!MQtsB z@S=PT+IiMOW;dpy!Efl<`~W8V;>-CZ;h0K;0{ z?seUam~LPUm-}oT(vYqWN zwn*QAhMWu}(%G`^X8j)uo{`<|N;cCa@C~so*v%IQA?6HJDzf-s&;D{#2}IRI!d1wyzYK z(ot-3ZDv>rb42;@_JaS?eB9Fh>5Gt(!7I5?Ufx^-t*SNJy&sfk_@Uvoc?;x-;*>4e zF~Wrz`m5o)6|d58UDw~IBnE9T1}9>YuJt6+&KHgny!Kq)m3+mdbRHJnB3@pHYR^o0Rikt zM^!%s>EAIqJ}fYv)W$>ksdIxY_|BcseB($piFvzIXPE-+|0ljk+?E0AWAKMY zkmncETsX&<(%CXtgIED1>*?}9Ur|NB3wUH>xgmOe($}O%pG;||iZ$RDH)(0#+L*+@ zDr~in=sL4lox2F>?Kd)%EzQ%QIWEkoJN`Nfa>fTM=y8S$KaApV*VNRQ;L|7i+_=T1 z-nw4h?sSo3FVR?f=IUjE^jeImGTl#&8$ZpFD>uo$2=vEdSwdJiwfhl49MtdK5UK+sqQs~yJ6x6V;rMZTQ8|}w@VciP zjW6fsOTIL7p$}<=g=Q$0B0LHI5TMs+tvRTDnPZ0T|IDQW!+01PshA+(qs^hc5gTVQ zY#{>d3Sm6};YBXM6W~^pcyl~2g#Q0zA%^&$tL45H2&3+4v>~OtC1bdDZo8tQCc0^^*~8|4Q0D?o?k8wQF$( z=v#!y=*QL_#a;ImKZwq&I9bT@CDhyDG$|!=?j6TFV{H2IUp|7!hM(+TE3Ym&T^~9O zYNTv3zimuy{7vp1?<)8@WScpL8gL({mi)5y>qUp2$bla=;ZcB~S3;5Jz3$uXcunPU zT@BWO9U@6%0KINhdh;wfbmKPM7lwEP7A|ic)#pv9 zOS~BWPbCoW;r>sFS6JM0Vw_mcxQG~kMH((;4t7dbI+APruEo2o21w3U6(2k+ygR11 z-NyOb<4vVp?jvrGWub;{_SVm0e3pgLY3mjXGj=DnC6pQG&`CESR6R$?!7)GfR{UDg zq?%$|ws;?AyhnKz3RKBwDWP5Hvpzw2i~2tle6^A0|L=)2r0lP4nGa%*^gZE25*&}b z0qfV2>_l{}YnlEuy?4A^Ui}f|yZHT&^8M9=@6*=Xlyrt<^6rehVeH_<9H%1)dw3&ue6wJd`$Fsw!m=-m8 zrC5E?*0nbZnyd#R0q`q+oS?A>ZWmmy25}!QpfN5JgY5)r1AQM2qhCg^f!TGKabw?6grXa(xj)c6^#xZ4VP1qWrK*I z3C5C>d6Y2iu;8zzleu9*E^Oajr1Okl#YYsQd<_VoMngfPXOwkkZCZ05z3{%*)O6Qb zsXq5{IR06}=X`Xq^y!J`lsSe%DBu9$d(vwIG{Cv_{fZZ72Jl9c_%n<`P#_Inm`SXA zz3HJ6d%*2PYw-B)OKbQ?{Bhu&l{3TFyu9%$Gbw^M`n+Gg6{4IeK<6;JrVC19u3U3* z%D2vgbbW@r%MP{9w~*WL@o8@A)9k5B*{D{B=Ja2%wbHgiD-4vdl>YfK9qy~%p@3dK z&vCO)#_mpJ{v2k(7cE0e_BV>Whc8#3_viPi<;uT#Syu&F{a(^D&V{JPAwy2tjxjz& z|7QQ9m@XTuINqw9^dU#4ychB7 zd_#N?5~KOUQd3C^zdldzkqmBl4u;l{viX|zZNAAb%tjO{xP+1p zmSrW@aq=W&MG6O&@|hus9KWb}_6(di>xv~wSI&K`zJcK%UBAsht+Fkl)x(jEn~$Hn zDv+R>EIRa8ddv8?%a1n zFl}aVL07!l;AsLtA4+gV#G?B+s2a5^6!yW`qQB!buP0s>tK~9D!a~v1|J@-RSPM`4 z`u(|O@JKf@w%Ilz^U3tYHlw@7r`q3hFSnUB5?@MBBSM=o55zepWzCeYedgpCHd>)| zH1;K)-%{(6)nU?qAd|2yFV%SR{2EJ-Bt|&0$mhKvnZmC7JpqfN&Q~#i&6!uJv~zT0 zHhEku&)~A6TEzW^Srx(8{uX{%_>n-w?j}wVi#4cC{Hw#|Qz`0~PI&Ofc%&P(Ug*!t zB`<;pOMCn@n~0p=srtG)NV^mm0Z3!?1Mt5x%X$>xf8A3K82@3k^pnC{Sqvr0;AIeY zg{{J~dQEW+;ACfbP32$FuwQlUh2+t3@`sNrfQbpk-jwVoJ?{Z7U}F$|b-j7#jo^h4 zF93&;eYra>PKJ@PWD&|d1`l5Ux?wd);cpq*8vC(fu)x#Z$P>YWZQJ0ofb^_yb^sn~ zYV~RBTX&!7Y0EXL*HE~X+^@7bIt0+Y6Y$9?dE-9_X7!ql$%`sHCM0zap;A_j)$j{iTv2H?!cxPn3mncD* zqrJ!wrreJ5HM(HvMet0qjGy)|mX@+4VHZ?H=R+Wm<*MW&zYmXPq>vtGN~j!G9a z%O6@#I7@(4*>}j!GW8)_9PzWz^VunA!+c0SZL4f{;DYnikX+CMN95rGA5cFBmFd*b zudE?fuff)y_|Lu)_pm=xNmz(|KDDGV$-dsD;%qT@I zo{0g1d1IVka2Zvt4Od2wN#tKZ`9{R6pK3~zqIy`0jJ?v|Z zb`o;rOEIkK_2hnzzL0COSIL%cQNMwEaRmbt;(fVC=+Ug8#_5r)l#+OL`+2v6O3?7y zFulp`<#>Wro@!lbfbW;W*80!8GGt%pO_oC?*ZXvNYBJ{Jj8gTuy+XLBMZ$B{3?-W) zw9szbnN}FUzIAhxPBJAS64mCzV#5O42V6kHN=>jFf?t18%vq>qo#XAn%baEpyXXvP znVk*nZ*~QEu+b#Tk2fl<4zmxJj_E_h&^m?@$UQ=xRZjg;;9bBq<^E*Vx9_t_TlU2| zfg9q(F(?BYLTim=I+vWB=q#A5o+Gk8#RWyQ?gnYHT{#XZfWYD_-~s>jGSV(Cp+oYN zDy!~WTW=(5b{_HPv~f!Zf{ONAB-6M_GX1o!p>3nD(`pKii2h#P3^ASCTKe`dwdKC{ z5-Gw$Cnxx^s7Ar@&5Zcwa)m2te59>Lzuz?4_%4QB)Y{HG?50Xx9t09swuak$Y{Uwb ziUmCeAl3?8B+g~lS{|U&-7hiCU~H&ATP~2b8}NB8NpDLIwDA%PwaMJobiD&9gXhft18_D zh@jT-mdFSdWtbj-^;_uN%(l&Nmh*^Z1a(Hh9dyfLnrUbndwuk5PpI zPtSeTqY^_o7^A?VC{YWzom5Is`j?f_*^F1T_p2kbdB??_U9&oBs~uaoo67eUizi~S zqXWr88DA_r?7(A>ro@E~@&?Tk*AiKu-6JxQpu)LAGfUzKinv4#&i7be^pMbm)XQ`wBm=W?;lYA7Web;2c@L{j{sJq<;G70=WUP(0 z5bTMADn(Nz4znPazF4B@l1+xZo9&cM6aFR+Hdg=b8@`c1&_LksYI%e92Y$Ozy<+Wx zcv$G;pG_r~{aNNt7+obel-nBnbEPK~h(gXCe??J2GZQ9gQ3ggA9}%)W=m9e<=trOJ zPHX7FZ4Kzwq16_0iELvWS`Q3#?>ojo=ue3~$IN#HM256F#WIId&zBgclpNR?FE0c7 z?oE)s>MZkf9kWi)@pq%r4P~2@>X3TXM6+fIp>yx?{=`HL$MgcUsiR3b@=1whb4v&s z(!x%W0!2Fib8G2`y)iZV$mZ4?Bhs+m`?#!fuDB2wdD-J<6my4tDB{Qd5W2Qsz}#~- zO8D9JaSVoJj8X4tMEAY>Vf18?qDfC}_<+(duaWhY;9**QR!5CuW3DD9dqPyJ5{6&l zzR%qSQw!HNmpr*Z_y>jUey9vv-upDwQ#de&1M8D5Q^=W*Ehp)5{b% ze(nj`1ML;Dol}4&9mD_J1G2;F+;y z+Vkp)R7Z?s-*B(#4 zxJBR6s@b1d`#I0Cb{0!1LIK9Q1>YK99XPc&-+X}Rpad&273h3`|8a@S(dp6jYouW& ziV(D!=3RucvhaI9B_0J9zdkwMTM+zpH`nHN>BBUhq6qv*bc_;%D+0hv1TMRDNY;E9=~-1% zU5w@QY=5{S{Vsu@XVDuOj0nAf`eRtF(UzzgJ}Ml!&uE|0tVJLM zE&e&LB$}a-{i$lRRPo~I)0Y2xSg>t|(h}J`14%HN5p{CbW33N-_WlB_>v6%zvXwf0 zN>M!I&rYpkND1c`+Q}U_=#4{AL2Z=tTYnb}q`Ko}#D-<>ei1mZe&2X^=&5Q3Vb#_C zM?U6cm1dZ9#N9BaKZ%bhCNg+2zv(jH`1L_{A_{WMjAMms5iVfTg;&v(IDK>jvBP+tRxq%s9ZolqCPVj18>TA4smvrJ;F)VFO_>R{g9>H5C)`km^z0Va5dlNNyi%8d ztM~Gp46n#M_)j>X@&h#douFI5)X4$JzRQ)2Cc!dgOl~$o?z?QsB||@`qo~YIbj1$Zim9uTp2-e*7s`!aV$S z80zEX>;Ws8kIPXn?6U(ePxg+4_}vH4R1hwsvqsZ$QtNu=@N^hk1PvqKA#V8X zpStjY1RkY$crZJ=6#HxIx58fo@y^B%>P5{$^Slbsb`YQ*Gt&*s?YRY+DmRgety|s$ z@o1pU6Nk|bb}~9 z2sj*@VdI`8hbGf!(`6aGjaOU~{O&7a-%|dPU*eK>aSsm@Qq@k`S_9K5@DWn7L;dGdz9nxq@~BtJ z%;+|bugU8|TW-$-uayL(J(nb|qr19pk0TqWiVe`Og)U%Zp?lfLVGXA9J09LcoMgf- zVM9L%5>`?pR4sj@;a?~`>j|$5STS)*rfm{*ncBMvb zPKbi5=X3=029G^txjF{kd8Z$h*3AtbT0K@*9(GUUxoDK zTJ=eBenk-)%$pMY>or7dq51p_2p}`Le$yC}sg6##Ah{u8Fh5S@ORumtHdn&?R@Z}h zOiU6kg49_oI8G^S;utn?)4mLU+DU{eopMm}c6PW>U=Ft}K~cnt%9H&?Ow8BIn>2|6 zfL-ilL~s_wXZt+$R$q?6`knkKDeEM;goSs#hm5lFHgAYlB8xxw;o`=qCpfJfI-wN; zx^0r}5I3wN7J*lg>RNr`gKH_f4JI_~03J_A18++oYfdD3E@9G$_Lmb4VD#*5(V#@FTbcgPBdvc`U4a<&HKA z>I~m^Cx(l$Tp9rfM@GNCkjFbSbR?JkXohs*KvL@&oJT~rd1_C8!9A4YfpG^*)pO?uEUZ+^nbONn|Mmiq8*`eEANzjufc-u00~!UxTQ+=u z3$5`LFu>XO+PSYVU9oKH`2V@Bgh`(<0Nv-7>p#+QL-8i?YbiDEFo=WL8m3Bu7hFGz z_r6a05(h4ItBy%Lhar)I`r_gHv+C7Dv?>i#1wXwz93QY_arF}X=h*E1kB&`PLh=t6p6oXyo$v`EX#|} z`!|7Nwx9%m)j=wBP##2@^M*-Q`SH{?Zj-$~SPa0O*{XK(eF|SGrsn(QdsN`l?oUqyPaCBYP z`FqHsbNMqRx%s!>snJ)!pBf+|N*KnBPc28)MkSbazAJSZB)ElD9Z*od&7zoE+kqLBw!SXSRC;g+kh zBE5*{Q8hB*k7|4fIS>oNxbAIUePKSG%Yvwgu2-al+i~sIZzb3)o z+V|`Kr}n+pou-wp>HJQ4;D%mVS9;Y(CXY=}gc+KI3}0SDBBwxm7mgvQk* z`;~!+1K_I8M}lER%oi5d{nGv9>#gdd(9SBq_FAl|MhiHUsI>F;1_>5)JJeinfM#LTfb*Eib*MNiGLxvJTl|dnmCAH@loJR3tosl=&1~KR^jc-G%fUu=Bde+;q zOHh6R=IraZaK`6la6YIas`=MC(8=i6`BH#39R^?)P@Hc9If?lrzQ2sf>*ZH{E&S|N zSFHc78^{8ZZ6E{wPkKMxl(qIE;C-MOC9A9bOj+8zCHn^1CDwx(f}r0^VL=jelraTi zYt{R>e?|^?Bw0PX^tDmwR4M;yYJ@({@0Iqv*9OU9eZznPvy_tZjAc2I5pZ_{gI?*s z5(}g~^Vf|*VB1J>=X(QPhtuxkjvg{kuiX+UdSH~>D4a-_3!#(nEOL$!GUY2xoO!np z7VTfa=sDzw4RI7SUvjy)h+-d(V>fGBJI9jMHuUgZXsdr|t#nUSh7=U2Gy@Oj4v0(U z|8zzR48T;rX|g^iz{UMq`E9v1bzvf?YbL5aFXedQsV>rdU%(=L z0Zd9l0lSG;O4))>ve%|KkAP80&(9x#qc+U*prA_Q48Sr{MWP`mOIvyLb(GJ{z;jDI zF7w~DVhc0?$uPA7N1gvj0jhzSZ-QaJ9nk+0!X&CJp$zvbdPA=chS$uGf+Xn^jDn=L zk%GC#fh`K)JewFnm)aQO1UJ@~oIj9`<01JcOd~v?QtlLhUyWWHMv{I?kfmyiNSN`* zg5MHUZGdqZ@apB?;7iRmiy9ULzcr}suhUhUFyXBMLB{XtP(q-7w9grQ$VEU~q9iQi z(){v)g*A2JnBx}gYE_?;G@%)Kpi_7uWhBE`QCsbaMQ(tcJYx}Dj9~ji@URA|F;pQW zGpotRAR-y79dDB)L5?VdISQ3oUuC&ROR8)o(W$YZvdDXd!Ne-F^NkLG*+!{k)U>oZ zJY)tFJzO2!Twee`OWGuf&_Wk@`Y!A5ieZTZNLGd9Pv|+~hfeGh)+Ekn1x&qFHsG(C zLlJWMDPFJKe&%&>uqvdxxCD4K8%HMs#$H&|H7?7X3M)+*SsOoo2F2eHjA~snWzt>U zsqrpU7|XY7jqEt8D<3Q%hoIUqcys>eoi5BJMq9Y_^*?@?KjahBEdLQ zT_tC(*#v1Y*%f=54L^Vx))AGez%O6R-4HpxkE-C zkKGJJ?BoLxJ5<||pTV)x(nC5Hr@}8TIOp#AU$UgK6VA>w!fLI0WEI!=tVYNW&|xV> z5l`qcSdfF6Qe3g43Xv}^}ojcP2 zxCi-lACH~~rs;pr#-9tC;v?W}z|->RR=jmM{=wZvU9?tS5fp5@(J{gX@%eZ z5LUq~lq3{=zXA4{8TQ3_!H(!_gu3X_hi_Wi{`2xKvU{0F1s zA_UGZlr$O|7oj=_l!`v#_2*B-PjhnGPieN^6-*@T%(~3K zY0qBsx-3$qOa?_vR)D%XYX0J_0vN1=&KQ^cXGd4P+RM4ziJRA@mJfiLkdnx#^eDmv zdBYPXYf-T^;`tris8*zwQ46+WI*K$#4bR-jnJX7V)N$4+5oX+rMPct50Q5u~(-!o_ zQpN&-Wt-ztl+m;e@zaFZkWVntq?w0jT+P|} zi@d+wLeOKPWUW0z@KxJb3>^39)m~Tn(B5bGe4fA9WaGTIrHCOHijYs^$UE|O&ojL& zeL^Yj506ELy3nlioxkzR3T^d1guIhx&>sa1rIqaadK{i3qR0DCpmcD_cy z6`Ovu2Y{LMpia$KqyhHm{C)4n1QXC|$?Z7<%;v8?f>L?}-OK1BC-lu0xv0|2Gr|@K z6(%EM+4O%JI6^{#kw=81OwG(XFU@Hs;>prL7u=F<*zmq1m zdRo2d7ar)0ItlSs$xDHrA>@YDZ_3T zp}vl5qDIXwwK{J!#PkTg;;@xzb9Nb&$OyLq2h^VCKe+MyI3I9cYm=FqW$Y8kRY@n% zv-VOaU{$T#1aKFy?Bczj>CwoT6_aI}+E><_A+^#R`k}>M&uu5ZxarZ`myJ6HC1AqS z6;s!!>RPOcpAmxsX9no?Q2!zuqQQjISIH8qm6&(;h!GvYJK0#E^~o1V2c%k)4r{rW??sYrKR6lJZ7d8H*YL+fu+}=fc_@tDar>0!t)O7rOQff6 zV^Kv2{YB!7l-*iNI%<+6ub0gT28_$ycR#(V6*d+KOH9z6^zo`y#bv{p!{7OFYfw@v zJLQPH??8=JdS;9>+fgux-hzIDHjh65j6VmIdY#FG5{HQcs_9#WXZBfw#xv%?qwOPM zV!2lmN#uZNUt5;v+xjOQTkTdXjf!s}$p(v5A4MNGy8f(zCdlQR0u?PAzxMN$$yyD@ z@g%t|g#40=HVd(+3!o>YHv zmBs-JqO0G$`kMJ+6O-&Rfmw_0kR9}_6Z!qZ#`WZklD0Rz49KpB>9t)v%$gg&`0H?l z+=9M4o^-&9#Wx58z8V436&)spCz03tS$Bgi@r)O>-&{ zK?BY0qFrjY(<DLd+a?*G0oTn~7P*ABY9z1Ly~RV&hO;^)ZJpM2oNeWiPu0$1Rt~qVlBnw^~F}L^qV?<=uhcL zf^5<RNC?x8kI3I{8K5-TglF zGdjJc$&TXy#Rn2GSy%*ed%VgQ`H!Bm=;6K+X5pjALjXNCH{KfwC8F?so29a)=S`qt z%<8>Ggdh^Ax! zqH~@p^Jv-TXbkefK)(*h2N;jEw|iIHIO?QT96}2FRhUEdG`Yoq(JVFprZ}A`tgu=D zlSY&o?+y_4Gas|Ob12LAhWOAAbLfUH^IiPo@8_qW8#o?^CSnjxI?w={o@vu=+dlUUM!b4UNZA{$trpONq1SZ1rIM`pj5 z`1pZtW3@eOq&__~W5O|-Q37T+LPk@#myqYeYU2Ml+ zemdXj{vMvEQXjCCu?Ou84Ew!+{=@A+ETv6IxlI+3gor;b*ZkGk>4$Fhc?TF0wvYO} zzOaQ+#Ie;{2H~A-!)f+igcIY*Y9xMc>B$*BCr6${d>*N(hgx+D(-a_l>}jJVl!MiF z$}yp9GGOW%WIJEoRHmNtt8ERi)f}f1tx6V+S13qjQ*JZ=ga$cXvG^#T8Zg_}9$>0P(EsiH zgWt8I`FNs!W1@k`}Gpw;EBT{AfJWrT!{veKoazl>KT865$|OR5->v^@kqE zS+_(yPMnI9ES!%XFbcw|zL=70BRs1;D!@ayuM&~3XkPpROacRUcflCM?+}1Vrd$=U zAJdQOmfc?gN&x!R*ri&Fs5bp-g5j5MP-|Tu45xd%g3%z4d6*75j`uM$f)8(IKA~>$ zF0w1LU$MR}_&Mo_^eUtfBfO>C(h1b}iHG@xW5PQZjV@Sn2@G~p>-i`){U1mW)ghC^ z>VR&Pa-MVAML^-vGh^um2o1duR6PKCD7lMrt#`wR79Ly#4(ZAq{Am1mG0!NrraWB1 z_SLq4SzK(aJO3om^`#4PmRpPKDGWI1$>cVE>@7r&ZJ|s=NK%nJ_MKKT54+9}YcBo< z`%p6}GhQ^E#nxMfW!bcA!*q9pbV_$O64Ko%c~R2c zEvYmp-AK1|w{(MacXxd!-uLsYwZ85B;a@jk=A1cZPm4v4?kUFaFuu>T=Tnf+9U|&T zDz<7183Pj?+($s96q3jipx$h%J^W(V_5$N^xh>(!Cwe@W>30rBzzy{ZFYX_30EjJe z6*Gn28??Cl1J?l}4g;F3_FRSG*9dI-f_XG|u#ubBPPIF?YG^f}P7}VZ#5Tx&Wd_CA z`0DM-h^vx;SC}#-V}}`coPYACV!NDP_q@kBH z<(u#Yq;{eGR{zeO04BdgopU3RWS8Gn$3IME`S&!k`sXxi`uj|MexfM2iABgtEY~77 z^lec?;v380jg&0=k!9Z6G&(6yB=!<=&R~o}(yMsLZM01@4C38X4AlPVjjvWPa(C~y z0&)LsHcDmEXrcw${i}cDq-LVT=FQkAKtFruJ%MqwT$g-9N*jgp+`M^Lg^4#~K?<(2 zefF`Y5TJRi&(EJw&*x5cdn}U@OVmY|A5S~6^JQVUUbZ{2nUxnm$dY)xR@rHIhaCHT z{xEyApp+wLIos;$%g4|njEHt?TtLX|u zKrReJ&v27kC*pg^PrK&{L(Tb*-%jikT1tBV_~XyI9Slm-o*D# zF6y47FI=)%KOmuIokN~wia@y|Jl&mdvbrA4^VqVJrpN&Ncn!PBU2J3qmBMP?!#fXE z#t#;H{xUa)55tXn%?T+-czf%?$Kyg%42JB$G(8#hiBFJvO7GRi&+1XT*rHCl zk4;_wp8Aj)9vF5$_UDDvPJbew;#|uw@>ca}or$A$sWVN&y3pufWKa8c5mGu!L6)?6 zKD-+#)T_zC1^>Kfu``LNr}C}uo#KRVg@>NwAAoln>7kiN_)=;-22W(-i?gv=%l;uA z>3&Nu2XabfHo*G5C8ANI?L9!F-< z7XyXZ_*_{b`J*6@OIFJ(o*p!ocsjhI{$lq6S>%f~_at#aYA>>Im< z!g^-Qe5~DGGHbVTkVQOh#7Q!T?O<$H-tR_Dqtgt*JJWNF>E=@)?GJ?rEqZ^-;1=xr zrMP!oyYp)_JSO#*`o*GHfisG_{$5DXO{8E0^h8b1{tq=j3cH_!dr^o=gD=An(3H77 zwYAh=TAH*nIEWT40Blq8qX`qiH+Di}pkqNSLX2DCIFBR`XA0Hz(e;#yo^MyHET^cai&beB zhMo!o1^jOmC(c_;BSk17K;A<;7hi=sCsBoQ9PrK#@K{x&*t3H^!n7115l1r%-xE>| zC5p955oEa+(+3mV%{JTLG~aE9kvLoc}p8FIY@bEO+}_H5;CTueg)acnA4BwEx%pJ2tjR^G=q#T{ z3|5dKUiK0AmP z-#i`h=HLK5xaYg=$GCcn+7Cm?LhqTDd3wW0j`K-sM#oV#EKJ7yZ%6f5Ov=Hh?)23U zhN`b#TEtrvjrWxZeB1SPFR8b~T0+{UD8Frj*NXgh=}|knhX;+z2x337NY9Q3X8;i4 z-CvxlLrO%-2#6TP6L4p#w_Pfb#=SZLuSF#AHT)GLk_QfIsjZ+3(5DC8QwK5mVTvf9 zPQadP9rhXD7w~`CN;M!E^~PoIYmX4CJ*xKDt$+W4y?8QTp5|x&lcC=o{3a|4F62Bl z(P9>$Jqx!0uO)1B+}N*YRC+vVs@?tQ*=b@%tOYFEEu%-Zyy~C(4#8ScKXdb`qq35D0x$H;O7yePL^-eAW~ZhfZT8M!tS z=ag?V_$TUhX&_!-x@5~1(Y5N|FKI|(*v`mamgXl5nwhiT$b`T zTN*vKls4x&mk6=O$8^G~D_J3hDJKvyp?ErcZ08mCwv|q3aU-<%8Q{$e7SePi*v$Ew(19YtiGS&-vA*j0!_(Uf9A4 zKbS25FVs6l#aQ$^ASLLH)33FjL-7zxGRvREdkY{3N)YjYkouU5ZI7|Og!^-^tOZ}T z6i0BO6JhmL%V4z?kE3j#ded0*FDO%t(z>&g5wZ{rx!}wa*kY3zRat^g?&Q+AQj%14 zg56{$)Me91A$W0xlpj+M2Y|Gj{tPjlApR$cKb+hKG*~HpnScu+qPeXn{A=YdI7iIa z@J9Hg=+>Aj4~2Tszb7DE94o9pc0p=kMlPc$u8f-*fRQ%-i;;ehk+l*32Fdk)(;{mh zbl?0};Uh^(M~5KjdBs8>8CP>;Z{`8UsHY~txIlqs2}q-THv~tjcL!oN%w*RK>x9Qk ztMouKxsY_vr&g?BGopOIPQj>A2+5K@uK-41_i9@w@W95PV)m=Z(-fhgvwNw`jT_j5 zY|H-L-klat7#-=nBrRU+8 z`}*tCudV8?`KMvKy@e*oo8yH@QK+Blt>sPjb3~-J`Kgv4fGoIGzon>~*5<6)bQetI zEC;gXj6?tJEzB_+ue)N&F9mFEFW%6h3ZD3L%ifT_g|xrqNqpmOXz2qDK7e?)-Bu$p zd1{nPR}I_58T_zEf697;Eq1x#juTrfzc=?dA3EahZL`o(6c3~GYC}O^0|&L2x3Zib zFSxHwgiTl*8R~N^AJ@P?S66*aiYCt0HrDBpscs_t<4A|fST`ctQLq%MCG`*JdOQIM zA{2au#|>y$@kEpYVV(m~gkdIyGK);#*PEIPxIbXxz#DUlR#=OJ&v5z8GkBiIa&O6@^=eq@~4o=I9?oIlL zTa=YDe)UgH__>}aU4}niI{;PcS3vF6RAV1(mW0c3IJDDBFCY{bU;Px+<7=)Xug0DVR{Lht z46kUvNjHBI zumCb$LOUU7i`>$%~VM_Hr*1cV5GoZ)n}Xf(z9@5z3S@rkpL zXrTytB3(qKQ2V8YwF$`SxPpT4V&+TS|2(`MM`_t8lfD5onVk9%9c2^*dIQbqH(ubyERjb0~q#z;+eJ2`J{BOS3c#rE zZz(-GnEaglF0F7_d9m}PxcUf}PD~K=Xz4ophkLaTGMXSFp7)*<3McJ(Q5O$WFbxH} zC~P~E5%V#^CFUT!`LtbAm%|Na{G_R^eqc=NE{0DWZ12yGgm>E(`*e2eSd+vxP8c1C zY8~;uhkg8RDq9m*VET(9bRD3*HywXxy&^|r6NBmO;Bdft8Qs7Vd}F?;%45NhV(}eq zX{VEAOvKA=R!^r7qTKssN6_T~V4`w90`xxr3cIYQ3**V& zB7r1<4wZ8Nc$~txLK;-cVJ7FL_z&BGrCq3lM28i=-99Ulna{pqr}`XF9ra>7HWWvb zDkHACdB5bgI#}(uc8qe@O3>h~8b~L#W6PUHLlT%CeitG#9+g$e5Tb2B2YpH2PdF!> zlqhL~PG?u!ols#Da=@Z(i?e>xxAkbBj;zeJ3-4ffzrL=si&16oRCn8g{9EVnZ52G>ozko8C%` z!2ixheSwlW2%gH~Qp-kJ91QsnQNEq^YLg(`WH3~i1)qwk8HR2HZ5s5y*bW4Ze=&&b zu>?|{3yfI{^H|&J5+CKRi$1-E-<7W07^B6JuJGyq3d|%|Y5oC&Mm3>eknl*8>@%_# zrU0@-mWSdm*yYJxtv`%GE}6^5BSIPX`h!-LEiBw5?7I@I z+zMl~J4*zbZi4?g75e<-0HAisfdFwxtKST(qcSG)xUs3o>VM0fjXn{qT|tJKK2PiI z2=j4K_@%mgSh!|_Hi1B7Neq?*$;<4>4`MuKf_drL;2@k?_)*VWd$*~5x}D}TMONhI z&}Ma}-~1QcAN8HAd=nQOwil%cqug{Z-b@R80kF+WU5qpR_S4U!Z0H z{-LF{yGQgwiXTSpZtJX>;7?=;h|Bv9uTuR+ej!#HdJ90Y$+<>rFxDN6D@5rr1!-bJ z3lhXvsQ8YhmN>x-&-M}OKZ}&Z_o6WzM-~iv-MQjcisHA!eve6Ne*~B;N(-o+(ey}s zc5@>@vOcJ7)ABY=tc<%$I!cmGf^n=o9^JR;E7f19TQS7J9BP#JX)Rhu-3oYZG#322 z4G)v}a&x*8d9K`D^?AnX&+8V2d+AI~;0a3cvS=aEw=`j}V{RSdh5TMb9?v^C$#za? zvx+8_2UEntPst;o*Ek4~$b8kdopJc2&3%7}dR;)x>r6BcYqkd^t4ywsa25#iXx!cIF*e?L<)MMwzCe)Wk+Fi_3}bKrDT-a^iN{JCPE}!c>Jc zp_#q&?pfv3Vuf`EH}!jWi>z~E&?}5cy#G2ayUL82G_1{ncC{Jw*lZ0}9O)+j&;b-O zBpHe!ksUx5{9CW+r>%k5_dr%PPpq2kQkxHd+ua7@ABTRTf$b|2T1HX;&Jna^*#@s2 zzYT>k08oa|Vv4@Vf>D1NjR)}PTYE;DJ^`+XY(GTLQlmYHZ-8>RpW>`73F>lmHY+h4 zkPygWUcmoz-8f0C0Zk;0?1o6cGScrN?L=a)K(z67+%@sX~NJc$jLztSe%Dh~~ zQxTjU-#zp3ypCw6NVmxDQ4U)yH53>+(3AP_3CXMaDoiD73E{_chbbm2pmL3o_`B|P zh|1v!U<~8z)pYjM&Sv)A%NHT{mK2r0d{j6dyU_JxfD+Ss%WeyQ2nxL5%4}q(@Tyqi*rrn-F%wPGFbtRoiBJ?G0s~TPRUjn#gTUQd&>f)LbeK`s%%A@Vz+0|qoe8kxclqee3s=j-gdCqyq z<#<~V-jW~OY^hdA4pn6C>~cfybfTFhj%?>Oxg~6f` zwO&UEq7L3g187t(LOP-6hcbfq5|19|7LS>_=OYmn-(t;qIen@)9d{CH8e#|qqOw~b zOnM75bD@a7oDI|{<~+#HQQ!vHn2*1QadmXI=yN~m_(pjU;t8htdP=Z0oN4Hv zXx*kCJx9|Yy|i>RvS811-0BVR{-Ycw8r*97MK zJ7!E&g)eXN8p5^5ewny`+Cj&JB39a==OBG3#;`+CJ_mD%m)THWo3uScB=)P zJ~^5kll7d*|#1_&oL!>$mnHi7iAp@=_h9 zRjBebO7~2z7+bf$Bq{7g^%+3avRRCX6&ei%5ui*WaT=^>$BL8?cRx%`O7l|PFxZX` z(74Co^XT3E9wzrf+7GiXf6XkSSV|O5JNQJVx`zo6M)~;W@siQKG$c%2dvnyA)j1cY zD%trkX#%+wQq9SgnJ7ef_Xq9~7AKXEx4;QlV=-nlHP&wIe8uAZI77$k6@BYCoFT~X z{*yVA$uq6s@qz|Rgx~Rw1uzT;dDxAie^`HlM*O1f&g---KKIi4=HUZ)!B{8eY_Lqv zZ(tm;>y!c&;3LIL>P5l>KTWzU0j_IH^|G3@V9_D9+IsGaZ6uN zyC@6F%#^9erfB=@`Wj5=1=GU|#DFpwU%O>QM4?!{p zpXamAG2a_JAQ0#|q)m%NskA)VgOb~LWNMP{=5!3I2cOzsg(Bmj5Nxv+%mGm;hOXag zSH!PUcV#{8I<}t%hS5n#J5*Uxi^l0=LRwM1rwknc50WjhiyQ0>^6=Vs9+U2t#3aRu z49oN(+@?QQp|FKCc3nEMoEqF^04=`oZmE*uqdI#S{r0N&k$Cq|q1`CsZ;a1oDt;cP zxz@MXkad-~-4RBRk{?ew#>Zc8+n?jM+Fv335+y00DcW0dKim)z0BFU}QnWie8F#t7 z8;I$XD?~QA7=ME;DN}`W7~jk*L)4sPFPI$JrFHVA#sD6y;~&B_G6(E7XiT);#p7lW z{ZB|E)!Gje7z~eFsz-}ehyX*=$64E;EzhTC1U3@5sk%-uA=jkSuu5%HiCRl8=5L*e zsr75G71ch)dmw`yw!GXI`J~7YHCY8eli>*uuv}vn`rd>SXO$?LIW_v3EcR#?{T@?F zGM=wvCYm=AV&h6fF8jJZ5XNAG&sm;`{!J|=_FhS4=TAv#BvaVZY7}>Y$$1#bL;#Z6 z_}6ER%&a7L(@(sF@6H)&a~i`QXc*b}_uM+H9S{nETM{jv^Bm|p3E(z3o-W381TN)L zv%=u6H@gkT=fZlLcpt+*^@o4eA9u^!?*}>bw}m|owSW%wo7BH*!0{KgAG6Ltb8T3{ z*hL=&hKfc$zRJ$aGvF*UTB*ek>^rRitaLdm6sS3P(a2Yi;&f0I4(?q5*vBJ7y~Ag! zaKzgeaQo9YMNEC;+w_0llIBzv)T^j6>hTjg%4=yR%a~O*9WLC-k-Mhi%h^tB6?sUKG0sf~G+c zvLGl9AhRP0-bd!qPF?>=pfPn>kMTRnikjEx(m_2WL4VT^v&QN zxPoVhdvUEI8CRyf)uifaRXZ0)q-s?k4m#CLB=Dht*OwA5Qqo-$@z$c+Y2Y?Wfe}E9Ha1k{%thYv|mG6}Q4LJIe z@&W29*B(~PF^xis-BS+J0*y9Qnt&lrBao}MfM1ISUfvbv>_I_HmivpZ$Z)y(Sri zsX|*M^Y8uvlwQ|StW0WCJR`hw871_;Bzb_$6C~aF2&>yG3#|(1f-(bq73W{*=5MRiK$*52(VyZgpBeFM)t1>Y zBwivvg4xbgV{2Wkh$##DRv5_8d|6%E`?S@bXC%O?6kFLD4+UKPJmZ}62vDMRh(HDa z)Zo8w*_RK1u;GRza7Uf#1@L^2p#0gsQ$y*847%v(nEA&JCI2f(@G>6|!2~diJtbRg z6pEdQ8$5k#Vomfb{?A6q&j(xUrM$(jqEK0B1=L+^J1+zT0m+8G_<*jmryEt6aOmPb zk^$m>KM`IgK)rGKTJ`qfcd`IpXv^nAMh@~vxt z*box*N(M66-uUVdHXbv8N&b%z)?>7(w1L>4Ma{^c*tjt$ku?1}A*nk1GBDRtFL+@* za}s49z4my1lhQ{|TOxa@{ajJ2${+qQ2iB@fc)Oj0sr);$bQ4A?jqO57-1v>S#zKwN z`eq?)G@bR%tTZ;Y$S4+T#INUJ2tM?BuljxIkp!kRQLmu+7 zZ`JP4j~Vo~MPxqlY-sMggD;V&w(GY)_nPH+`IKU8W09`x#omwd1Eqy7v6IzYcxMZq zC2!lP_swJC@}!KjnZUf4x1UfUZnls{R`AJ518y9U2oB(bfTV*HoW_>-Cw?2~TX|DF z-TfkJbWCGW_>_iyUD2U2u;;dDkmtto+3RP>(U?#0;Z>AYOLXH51^63GPGshrGQHSY zxFuHn5xBEV$?GhEWg-^|u%-%&>>CQHp?x<4Sp&`D&)lSUVm%Lqg!a8IX^|k!$f#RP>Z4P_j~)rXvYVttIKme`Z#XC=q98j)$v#Na9t=uKv>i9C*df z$F*A#Czy|pXcVxVuAmPl$sK!r|FdyTbWsUBl&D{M%T)hsnB3ANb47x zg%eIQFwhH@!VFm4em#(Y{Fad_=yf9w{DASr(+Qe#a&kxj6&54K@Bh_o?C&}4}yAvT9GL{>eiM?T+$f~>o|bY!2IC8h?onJz)qpdzcOmR<{q1%14@Ddfcn{uXzmq}!8ochGNMKs00oa>2Rv3J_eaivY-GX6a7g7rAJcpB z^C}^O=s>KftjmKXIa|6xNnd$VywF}+E`_)K|3FXLOYY)gDx-8xL?6IwH}cCnAQgoI zZFn8=4E33v`4qf+Qr-^7lpjV5^Gj8c#zQcqY&*O^(}0a$_>2ebRw^m3sfS7l!D0I* z@)vdFAWDwv1=N;+@=bEY@GZzr#qCX9)|BbjGra-{;s8_0#r25)$CG-Itv&kY!~ukq zNR@+GRmUhpA19K!^6B0zgp>GOXy591pP~C=lcnP{;cbIt?TL^+%t5yMGo`|BEub9I z>1OBu0+I*9Mb3K#kO@j6chLyAli-Ai|BB@OzWBp^W;3`sUJ9GYmD&Uvaz%ilEWTCjZG(WK$GEEpv{6|6mJH&W3zY$Wu<(ed8V;)g`tS}EI5ji0{PCNzw7JhfvPGYx83z%MF5miWo zM@_STE?&4s*62r@QjSYG1=`e>+PsX`TJrz=a)3)HNznnSKp~wjX;0wss78o%_sw4m zOs4WZ`(~Sm=sB|g$pQfXx@T({EZ22_KBZD8_$FTEg|t1fwp-u5e%EUV7sQ`mK^GR5 z;X`N%%zHF27heaXuuYTZAmdtqR{$a4YJvNyKi|$irC<{TxLQnl;o^vp$(_iaK4tK0 zs8X7TE6$%oJDKv&8f9Q4aql#l5sV@0CL98z&iR@2TSSm)WCsxGa5kf9m<`x51~qns zvA4i${UTy^sdRy_p)UxR*Yi#IpWzP(|B3d~WLN68P`>@?%=o{bz@jkQ(a>UA2Kd?l zy(oQd^gPflBWL+~)xn%4djsL2f-X@s^m8Lh`t4+iAW}gsknw4v`*2{+o0Jt@#9#ai zoSPu+n~bRXPl<16rMTyMjEp~l60z*>eU_t*H+uS=oLzrCM_n3vUY{*L^XBN~ZUVnU zHqWW_%4;d`rvZw!5oHqTTvmo<_W_sf2yio7(7x~%-+;?MHMK8b+scQ!pP;A&BFTiyZv+? zpPlz8!~!CPslAn?qZyEVL%s>w;^fK4hi^@W^8sW$ z9=khO^ao-a=s=^zz~^Qo>4&|rL21!(T(jOc@cbj|7*UDxEcRSIBuD;0uazOx;Doat z8_pXQ56*$l>398hVXVL*5ewh}DLB2rkrLmq<}L%o@d^Z-Z*4`3Sb+lqu9E;kuPW;$ zg`NS`Zl(kbUn`ShTg!XBJh%(&fv_ulwr_a}`nShcM~iLoNnr_+gOmB+nDqH<`v!h7 zj@+$dG@5SzBEGYH6qCEd?^Dgk+$!2|A(it{4-mJwXSWzNnqqDzW4za`R)c>~U%A#r z5xG5H{<`Y>60%W=`lJWoOEIzth7lO~GUTc|zb85MW4IJ=Q*v4Fe5TB8a`7^RTrjk& z%_XWfdhMZYjWc2w10Tb66?cXR_B|~-h8=66CuhYRYIzDYMrUS#S3u-BCL)?Gb;4&} z;B~bw8`|^cP&(i2Hvt;m2bFY&m^UuRg!_Sie{D<7`;)ZwV!o&xOPf%t#fT!0KSXNg zYozRtt_;8MUh~JK-62cib3_V$UgT=M5PNV4=cwK=;nPj^=bhHqK)S zLd=7ck4k<${L=vO8v{(*7{8YmWAxS5(vk~oYhnNs5I6@mhu`JfdX}NTC35J-pWT-I zH<1Ip7+OU4{9J{hU~|*$^HRVv6vKJgEg`MFrDy`3A&kpOBcghFb7+y{)mBV4#}~>3 ztjsA#LY(%NVTV4O0N@k01*4jbdo^`%He0jUmy)e??+lxN+0`H<1&ijdM#5(&WrhUx zMr;H~B0H!a>wb#?r+K*gyo#U6^n|BaXBAQSgB!=_hpXVIQT$dkiP(|>$1;kTSy^!K z+08uYUY!>Knp3P4w<^UP28`HbvFOYbgraZHy4>-E74~T^d8Dbg`;}M1q zyq{Y)xddL0R0m?&3mrp;9a{?DG=>isA13*PVmMS?;DTGTa1Qs31A!Wi$~vttg2%`; zWsI>-t$m{aH){%Tvs8+GkXl@OzOgsSd0#sK2V?K=n#Yr7=KW~HU>sBup{S7`+_`#< zjsvA4rwLl}zy-KMmm(twwm3MiKaX335vE{631&;8z>}4_RPc)IzjI(FmyEu^wnHIO z28CeWj=#X3_zf6+Qdyu^=H&AG-Y?~*?^`YwUQTjwe3I672nV(t>&+YA5-^PqV8QvL zyC>E@F>r8T#~*#kX0rOwu||d|V2Ua2ype_R%xRHHzi=2{RrVr@$#wZI?2=D5In$IO zTFCybuVGZgqS@NKJ(WAc_kLIrYvKGv2joG&F`8ub&%4i(#)_FZYtw_9G~Wa*Tx*Xm z$jeJSFemdBPuLa zrwWy2mbYNlr1JTGk#;kTx~RE87i6j$?cGc%=Td9Q@{=?*G$0o@Szy3@=o9x|ml#vYUaEtGp*jbeHFb5Jb z&75)jZwV}%V2f{0L9p+ZSkfsxKy@ac>(Lj{AQ&<~--TrAY!5rS!2go;?pptO3x*v~ zZd|7_$6l%lIgKa$Up?u`Uwx85S4?O=$rw|U0mO1>kb-Hor9-&* zZJ@J`f2Nq?hmRVj-w#Pt^U@f_wzc(qu9_uxx6?V~(WyTTlNaRk)lrg9?bnGtsE7>@ zmI8b%rQPtxBoTgM?TxX+#czWnp$FO<;%Ah?(F`nMD`kgszZ6S(nMV6scZNKwT|Bwr zVdy$`JJR*f{OoU@GHSyAu?$qp?_)qo8FW4TW&q5Io`m`T_{#d6 zx^WWMsnXt3VS4^0?PL7^+7khTV&;Dgir8ZRV^Cx|zy}G)T3_SRvs#|LC*-q-oBQKj z237;;fkH%z80MS3m}eOb!%;29`cK2h_Fzu^YA@TwFxhjt~tN1z#C(sUTSALz8k?vJE?}`2hT#DSJLdS2;HNC%vLi)+JG%W0ZNDp`mByc%Qn3#?Ts$r#S zZ_C=R2cpA7k>i(D{ynKv7L@_!;nDltcwVkYT_X*1l9vj$mZ9XBqw0PjrE(^qXbA^@!2SF_Zh1 ztB|+1jtwl9oPHD-!Evsq4QD;k0~%}L?UX3^>u2El&yxxA&qUa_}`Gm9OXR;4u@uIX$ z1}aSD_@D~lQv>or@Ig1{j|p!Kq(>CksrKU_O=cEgQ`)Uc2>x-yK8`_5&+I`=Vb2N) zTJ@`e@yR?Dt7$NvuvD_s^E(`|>X3vK6I>z33KsowKV@NZFmOJQK!un?9VOBju`Mxu zd$LH8lk{V;WVe$GWddV32o{B0x84To^8^wL`313=@B~};%W-q$^DiAY zAM|9LMmf+n<-7ZfZFa?Je<#oJB1zwt7H9k0w!5vM-uIp!4$jdhLDiaxG>?wvX|9>{ z#*?P%yLw>9RTh~&tcz077FOrx{n~re%m!TpULE@v*ULNw7~jj2i9wHn^3bpsIo@p_ zzv{JX8Ej#HB}6xC=ib$iA{LM%bhMxIxivb8(uQD(LKQLLABtm=R02ezVT&`{r8V`; zbyAxeBhcy0kt?rgZ~59u7B}HBU5qYP6oMo_MKa&wwTp;-_Oe3a@L1Q)ba3dnIZ&r% zr{nX3t^v(oKc$P`&4efpg=AC*#aDqq1Z=vz6Qifd#n-#duPJ;kbOzq%y{|QHL|`gd z+K*KX9)p&H45V3s;OA;6Xro6SDLhlfMmH=Lg1tzRNQLbgo(P^WB8rNg{^3)QR4F3j zScz%f)x6BFUWp}zZmG*eNeiiS#%?pR*9RpT8Q<+nn#lwOTM;tO&4!@9w+=n#i<6JL zZXiNbe^MP~(8_$KN;qLCo0a}z=&|C40}UT9@lNltVC`0=zt(rWWLbNhhm&Z@USsUg z=)nv3&^0YjWtvD(TJYVphs%C}X_2zbs^l!hr=ZwgLXnUfSWfP7qmVu-Ybo?tvI8|3 zV_&IDJd+bjGt(RwvrUBF6kZ6rgC{w}F6|RFoBj{9tXv4!0y6mTq-dxTr0tktG~A)T z2Sg$QE!;|xaO2B*SN+CKn>55+)Z-hUuz13P+d?j*aScK>Gcy=6-rY?9U=GhEbmUD% zH{%pt3}(jt1)5G3b3VV;>?o?)x}+j*G$HrqGr@@>_QHAEp1g$K&r^K&{i@~RHF5PE#NPyey6XyH zkJZYx2%j*&K;vwlauhtj)afp0;p-*SM`1eS^sD>62OdD1&(L-JT-8`(OltrH0e*=$1l*L8&*+Dq^u$ zH^xRG@Ep-NB>3QxK&_e)1Y@d{aLhx-0R1>|>oxJ+YS?Q?=Lgh~LzMQGW_b zr(a?J4(*v?Z?T5|>L{ITATKldh0-9$ew1pryH``6T7}hPuV-3$__{~2KUsle#z|LG zwS7FiqGAea(XRa|l$i$vroW#ghva#c9i|y6;2NnyOw(Ckw$vNZP=9yy>+$T~0+!G@t0Tm_#rKnOb@E9DJVzJp z0(X;7e1@0nDDN&MKbHD|Fed0`mJLEdPn*2_$AKpWDuSWD13X=ePWF7WBt5(Bv3_Up zcym|y{DV#f0R}D)8M~-A34Y)E8P2#C>75-Z49mi%A{%Fotst|K1PEG}yO?kFtHQ6~kp!Ol>WKa5N0g6TA^AHSUV%2VLT^kGDLR54NZOlBY$BMLta4>}v z){8djkBOukr)(XEhl5}=g`5+F^h+uCc!k}<$D%HoR%c-2aGHM+OOH&l)TE)p3h5C+ z#{>~D3b6r2glrTYP&#a1eF?9DYqd7=)fvZT`4@aBi5HZf>(cc&+-k>r?thM)oCfn1 zmYa&R!>_~Z{@pO$t*_kDMI|y>`E{TeyASkXhJHFIni(!+&L^|beZp+$O%_*Z=PD`( zk?K}#sA+Wjsqy@@-Gh@}ufFdxQqC%W*wMXUcHzV-FxA{EongjXG_XjB{N0H6A zCotT_4lP6hrxGTgkBjn3f}z&S4i_UBO1~bi_r*ERxtX}a>w#9fQsFa~$L!W?Q)XM# z^z@K+)7^*#p3b_*0afF7Sh*ZHxh&QI$--u@*2z`Qm;R2{8*OV<1U=?_!rKeOI(QDx zX;Je|&5s$+un00v@-kO0+lG>hOt1JslkhDP&-tT7Ypaho)q0@BoGNt=k)oC`@C|s`&1+nweT~8&C0KZqGj*eN}wCm zzAwe;Y#V;r*-3}$aB?s;n6BhSP7EF(aw z!BWWmR$QyjnChzEVsN7MNfyV48LpPSKLgStnpYPsf=@d-w}|312A>DD-Z&N6|fJO1O;=&A!u?PJ&Zx358-A}r5S?X}e^j4SwvLt?} zZCX^XG9G_U9gR*$!#`w2^F!MW>eZAFK`nN6qXh92w;LjLXONG-Tpk>r`P~Xb9P`zN zeMbYC^yx#HFgJpcWUY?>9;hdUcU@BF?zk`$?l$ArL(V$lvRp^Ap_9E(t_iT zX@^$ez2vCsQUu;~$&-jgpPM0*d|x4v@P&}df4$J1lZUviJfX^Y-K)Q6q6?g`NJrT$ zZ^O`8mn8;W`m%lTl__5C7Y?YmF(K0v_Q1|ZYF*sv_WLYokl=y)4vJbK4GvJ@krKN7 zaj1e%5m!zTF@cCyaJyaVZZ%)}Dt3$sy6?3%lGwo-U|KYuQpz9tSOT}{Px_l%4#&Qv zvWebMC!CAr*|;U$$?t)j+rdUULVS55e!50^b%ffV?&G^Z^d@6c@Q}c$Ir7vM8BK(_ z8|a7JhL;=kZdu9;lm8=0c zsJR`y9fB3V1LoS3>=FssSWr=p_06fNJt4%@0{9zUXz1J0z|;-QF^myT2uHu}%7X%L z=rRcC8w6>|ecXLfR~>P^&(C+q{n~xjV`zxcdSY%^<$uDfpkuJ?lK2s9twm6Ke(IQ~ z-_|9(#R|9q3t6+L9g4uf;)}LES+i2Gtaov++ts~5Rs){pCF@% zd~j^^+QB7>eJBMkzS8`*O=%LoLTaQT@kM&M9viGY^@UuM_7>7vx;YmA?9AU=>L%>3 zThSaYuW+ZCRxh;D|eE~j_X`?j_>?S_tP0dsSaFG$BfAQbia^m0e~3CJrQHYI=U zW_YcQ@}C9+H02zN&P8Jd5YKKt@*TP`!;`Mol0s8tP6F$W3gY7XN0Z&;uJ-urZ9`pM zk_4@MC%U!7dA!s>TUj4F5Ugc@*L}qh{dmL>SLcu|vzYmW4$&n4hGuB|*TO|JDquU* z4~qn35&(SV5jHDZ!_stMz4ked)Uj**ioO>nG%= z-l8k?4;vX%7B~IES8NV3f`o?)EWbZ`5ex+IeL`}$N_#^Uc>ls9zBg*#1FdF+*mr^V zdhUFpCdEcq+u8zJWP>G!BtBM;L)&fa-}Ahx=hffdkQ>4dsNdK%T>r_=jP^i(+Ob5Sm{8(!gB4JZHCP7w;)-k2(-4&5 zvW~~|sG=t^ONxDYc}}Xt*p-)9r-cc+LaH4eYen;wz$mMZ&bW7d#r(SFG;?!(K~55z zN^C7CJ=>`!oI^E!adFX8_T$yph0l3^7Moh&f|T$}L*s#brs2|A+j9wX6B3TkCF9fe z!SK23Y)No<|HHC{?E0_rhkK5gIZesWq-NmE|yAvKK*t)2WFeZcK6iEO{!u-FBhzgL2KZh=GI(&2<>3Dx zUvC{%Wz=<#>Jg+n6p-%jMm%(bAl*neNH>S>knU6(>5>u`|jiWe&6qo zd+)gaIb@7;&hzZO*Pd(6xz?(XeRBKO91_cMp3sB3+EX?r@(-ffQvrx(UpQ(%O$47w zj`H2>9pjcB)?{YNmr$9lbtbZm&Ra-{-=$f4<&xZ|M*7y6b>%0OENj~g`Wxa=%cY%p zb_E-^oez7$Ek8faqSSZYALt5PZ@wz`&5>)(q*w7fl=3{tU~G9;&bV|EFH^AEc^8^T zH@-8P9ngB}d&R;z%{%l*0T+_ZiElza3WXD^@ z_PRK0?tc5b*FP95BKeWGRVyaA zA2YFFY?w8R@5|Q!K`eiRd$c9rZTq9NlYt0A41evp9!9 z0J@f0QCoZY=i|u}s;SQG4^m3`6KC&SR#>!C1kr|$yQ2864{QBO2jz#5+(=MvJcLOP zjzf7{uCNl)_)O|5Z+VaI-y>0U70VK9)Wq{W6EAg%e%yk4fSMQ`{VbI53|b9(Exr52aW{OA>(*TjN_tI{uG=oU+vXV`Hk9wclneC~e}ZqC&!t zZr>fdkI?_|dro-k!PfBS2hRQZ(aUkuPlmGCGyJW|DXCuyy^Ov*DEUgZJFP{i4)k2j z>OW;Yvd?UI={zREv0s(Jv2jU_C^o~xsx)XVgH;l+UR$!3GkcHG8%IVYpWkRT4WZjW zL%H4v-ap&AP=yuJ$Nn(7!V_caL)B?Mb{5J#-TAji=KVnkZ&y`F z@-^dlql`ysyk8OP3_@n^&ga%)5jVOzk3vGBV-4!Efw5Dq>xsaGKh>J~s$|1p`ieu> zF`@Q(Z~W0sZ}__2D|+Y+Ch7R%R^iyEsx1Cq^KAq4S8O_{+xi@Y)t?V6N{z?|jCl1T z`ql(Xa$(ox#y@tF);nd$aWe<~-)nnYuX@sr`>&`YH!JBqx$^pzkUH~drOo2i&YW)U zRQbMdDIFvuBg5(8?xOc}I+S=~N;#RmS zW3=OvP`}}rb-e2+AT35b{MA*8%;Rg%_nYDJX`T-sM%1t~qCYK(|DC{<`rC-|bwA~B zO26vBWbcM|BHj>ty+ih+p`d7r=2GaTMni=>8)E{*Z-EqRS@!{Dufo?=AC`fr6G0m{ z1aUJoh;IC#cFk8uh#XA@4@D69Tr~M-N8?rA1yo#;^eHe(HYG5Zl=_XT)G|TTu{(nL z55P-=)Gvi@DKdGH?RzD2r6NrVDb0<<>P7%&D;Vh+ASUu>uszp9pRMhd zBX7|y<1}+v#5D4|r)|HuUfj4cBuzB?eA`4~+_j1!P9V1V~F>0OZY?G@6FiCvl}B|AR} z*}ekdKg#P>>XDPLSLyht8}dsNF6 z4Kx_7LTWcK0AO7#d*pdW+cpN#v`zt2;o9dvoxu{Ag2l>tX@C!_NHIcvP;Q!Keprwp zNAepze?{y`%Lh{uGl&mOVtu0_@YC=oPF+Wkye%H6Sx+*t8OwRJixQnYmCGkQ%;~~| zuE+j0Mfex84}bY;Zmf~BKdsc)qr)HkCJQI*5{&N+s}`GJngZ6F#3)r9{e_{c-MH-?b@*BZ=H`bH z+AwLnpTgBGk2^Dk$sH~2m~+c!%50yB{g^;|ywjuzi+d`iLR@BkrLq&yqoEnUK{=e96BqO94*Pg^d!F16auM5w)?~Lr zwPGV=+y%R&f;H9roh3M;-pGm{dsCER*R0Y`BJ}h5#FoA!U7t9qf5Z) z?rGz*86{4UT-B@ue-TpEhJB*p%4eTO!^15dLPkd3(nP&k-+JTI`z?6q+-Pi0a$9^F z+jAq5KL1Izr^wyuRK?cd%u`tWtNZ5ww~Jiv6vgKU1bBUJSDh*7X$k+>n|=0|7FP!I zd+=ieg8r_2y8DIxh1*X;8;_wvV=o!~PZK8O=|NB3t=U?#N8w(tN({tGc zxSsr!;3*s!ET-wZZnqt}uD`jbl5;BEKUdm6esk*RE?^%IOc59s)R85u~0BFGE%w z<8wVufBk;=Lh{Biw2@g$!o>eEkKGb-p3Dp)=Bq8_LFvDC{SExox!~}8SUuA0{iY)N zNrmEL+T5ci+vzlK#XL@64sqMeBh22=#g&1i*TZ=Ew;b;iJYq@Cf zJLVp3cGMNAFe=n2aNXbH9us*2h8|V==hIwd|5v0SWH&^Af`nOFZ*_#q^6T%KVz4!X04<(bEPsKXb{*+hzqwRf7n*{}K<*UxkDEmmOPA2yQdot5+YJ z=kHZRKd*<87*7_<5#U<+U7y0RUfp7IadF|p2p9DP+-xw^)J&Y3NeWu)QI*y#k)+ex z($&H{+ED5Ke`7N}A?!SeZrs}*@Jc!1zQ%x8+7u5&Y1kzG)AHxwVtBS2Jw%e zD!Y>~r5cB?HZ;XY1ZM(rwjo$Ke{v`A+Rpc26>YlZk+rL2jbareGoK8RhMgPLlr=F$ z!z+nSS3a|!{+>ef7kh6vzd<|xL55#o%>zkHqMi4(X~@m5UldjG09y5%G_9il*2bCqhQ_ilX{D z;ggi7iMdZay2X&*uAWa(;k+dfWp$NRLVK z{EhqpNQHiA%s)_|Iyi`-Rh;k&<&rBjS@|uYm->|tvwpIo7rB`yyu^P{Q_?%xDmxnA z`OaZx%x>L%Lri@H&}%OkwH-)SKBRXBq^HsQ8hZS%1KNJ+2Qs>;OJgl{l2{kQSf4bl z_6&+N5+bylIQ?{JBj-*V6=!!L^k#Gs9vY|b=*Ug!@ z_Nre8JPk4I=e^n$wIL;ya3Mtf!_8!1C!s8lFIifp83h_i;PS3;1GVHGTrGJQAFy077lJ-gy9l=RLr2!FleVEzf}*U{1;iLoU-n)$({MWcFXG1gWNb7-a%FWdbKl*^Z2=D0PHt}e%K`y|Ha zrBv^EwpCY^y6sD-*^0d*&SGu-aJ1L1cg$N@U-R1f%)GZ9!F+mDdu4^2(O2stRjC`-8QFCo%P#n%3xh zT^Bq8H1+=Y%E*cibMCYJ7CYhFrURlurUk1fzYA%89`wkT{Y-({W-mN{>efn@7n(}R zMbLs!GpjB4Bwz`QWyLszOJ^alnjPYrC?4!!y6=b|&^!eM_Z1n+9i=oFYXO%Eda;=? zm6v`N-@fI<_$fy*reVF4S=5t0|GPAVs8D{asQA0L5F?7-E@f38nNkMS*tKK zc(2Bu{q%B+lBh1PFBilcnV#?3U~6^+a=i0UP-pv|$dMCj4En_%0SpFNj++LO*Kp@|1ChB7EZ8)euWsy8c| z4dy>*2wR72gVi0MXrki6Rtjqh7oVM(;h|F|92#HP$wj#NKdbACr&RjAd`po-%aXo8 zo^%IE?!&7?4GOm!V&-wNA2*#YYAm}`v&M0mClNo*bB{_&?3;(7mF)yQ4;uHBvcn zQ`FTX_hCH?|M5@j>^J!-q`%Cb#=_lL@|>7i=a`d_Yf0`uI*F}c!7&#nCnW5lkDmGRgxoIZcQzHnz*nm?wMVO)EqJ>R8G%_Co%4tOJ_r z&mDtU&GUZRs`n`0cG0`U2ZnOp6eo@|l<0&=^s${p?+Y%zLk*)~tuqE3piBN%bI5qNs)DscAWO$A<%-wTADjJQ*q zQ_V~1_3Qg*5vBwWQgq|L8>iS2yOO3Gr=FL!arXf%`h%8)HMX7=u`^gvo#plXjlY(? zYJ1^i)zQgJmxRAuQ_`2}Xs(s&gF&iXlm_WDlAgfR=zOKAC&%X82!sXZ{GSNBwe{pk z{-)smQ^h7Yj0vobQJ6p+kLV8>QT1G%HG=_FjQM43>(s}@vQprIkl12W%txv58~{m+ z9Gu*j;~HNYouhz;eA_VM?^bxwLpjkp=pz@Vtwm8(E$2du{WzW3R}tyR(<)_)w=eRZ z)v!TeV3JLl!(pjK^hNf>qx*MOG6H6Khi;h(5xNr2%Tbt5j`Pk6cypmD)vA_0U^7ZaZQ$vSiD?E=8N%pPf4+oYH*DaJon~8E(hZ_W zNXkNT=`?uyr`ifVTFXpAQ!}x}tx9^iVu*z&Gt36}6J6)TCSjFQ<%w*Q=fuVfh0iZz zoa-$xjG@wK0v25}PLg?q;8Kd#K94gML}t;cL&)_x6!g4Y4<99QB;|dUg#Q>Ut#l?#C)La%(sQTfXo6g5wImeq=?r8Y<;ct%aIcpp-FGCursG7Hq(?PZ~z>{It8>F3q-akCdSA5@m;n z$HM>2Z1&t5ahsT0Jhq$5Mbc@NdaWB)AH!s)kr5(|*F<-0vq7TdFtN}l6hE}_>u?Tr zUXNd_#L(wwRja3?1HPX5aiLQ@N2FVO#8VqF?0r#n!cU^29yN$7h4W=f+_H|#2>%#1o zW-+xwY-AK7)39xM;!uRPL6a9%r=LqSX7jMAMEmGv_u&Avc_x^7kef?#(ZXk)WW=YS zFxT@Kxg#)SOYKnFk4^_@oNJ(JJ|rdvoq=(O(ex)J* z&UM}@g&QhSAm*3dEV^ovlN!WU_7!`%5ja47dYC@{P3_X!+Y>X5!%cYPW!Hp#RyoE@YZ($hwm9B#f z0qx-7>z5!2Ontb-W4Ut2dEv7`UXbG{hhdWpVCpwoBiS#mcpHNVnIY7=kOM@e{IdmM zW%ZzQ4h2vA9ub_%_vXc|c`?uzlyIf`Ik!bjdgZFuc8=2ksYjoxdp0UHb^!7?*5cL4 zcJ!?0fRVc!cM+%r?=Dp>GCl@f+ML$9+vZV5GEsOGA}&8%0>w32t2?8hA)pDwSZqj($wb2%IpAf)hqUf0rDkjj^aW{H|REE|y#hN=uPJ zlPM(jF3?tyOvDq*qECq%avc4r!|wz%3l1xa6G8=(UkSS+XKF{%p^F&JPUU*TecqI3 z`-}I;X|%Ok!WCf18bU^aqY-!59wEP5^*0XbGB5le?^*m&8V;^b)|wBd3SK-!ySu=R zJS??AH=rawW`@-N{>oCj!`m&Cc|4QH8iGXXiapM`a8NryhS!`jYlf)%iLdwA9AX;r z?QiC?jv*%E)n$4d@BUhSyeRCfVpiN)wYJs-5_|MM-D{BX#mfvRD_^q{uwb~LzJtM6 z2p%`zyRNk^#&qY3=eM*xj}zEHPUp6alPxgm1-GNr~;)rZiYB$AACBGvBXqBdw}raw`vYp z6aB>FjfGcy>LhmPIGYi= z`#9+Nq6-)Rdzx4MnH|K6_Eij))1fEU+=6j5(UuT}dfDK5k3Zvwx={gg7x<=TOlQK#W*x{rT>udp`SjqI3=omla$J>H*m^<*QkN!EkV z*s#a^VtMIad~$%BAeLY7f6oMG%q9Zr;J~IIaCdgJ z!*?7fCv`PJ-eBc+33^gKvy~MvB!#CBLPAd2d~nEy4r_JUj$)Ufnq*pNb3b4rOI0?? z1_C=V52i+JwhWsSU#B`Zyezri3wEZQp`okqYO}l@&LaB&= z5A`a>7?gQH!-(;R5Z~K{iAW?DY)xIpIGkC(yGQ;6(LtG1-Vt>t1}1LmQL6-2w)I9~ z81qJ2>jaQTN=H}<1HXM(f9Lp9?W7@(WD8F9I%=hG)UhZwRS4*+us`{0L{+0vQ7GDE z{T>s^w58s-Q5hd0r(#h74v2rxoQQUsYir!W6=c_$+$g_&!qeRR9=>d*|z^efgi_*aGMFF=SbR>+@6WFb#JFYoq%|7Np&3;0ifPkFsvEl_}l4N-w zd1kxsqBnN=;ZjWw%WZ0Unp}{`xNgwK=IcRL1EdN}GLKv_I5gtLStGDYeSV~HtdM`K zfbjd0pJcB6TM;4_0Vp1ARTvkh=4ltcR*sva-fcV4nHDR?1+}7+3)8a zyY3`MwzuPgMg=vDU^5@SigHlg=}O%0q<=`b!6(&(tD{F)kU-6lJt zXMSc7R<xeJ;B* zeLjK=89gu&1Gk^iAYNBU?dYweqJ^aDIwp zK!3Ufg7D+c49U2Jt^r!BXDEePN%*w)Y=As~=d9ymg+y8O%FJr9sj5XI_8!+C)EKDm zvaiztdman{Y7AzIZTxzCIIdU28Z7ED66lmr~XC zqL)VCOwI=ir8$lYxgW_}k6`L?1TZci}N@*h%L!{g5o;^jP5G zN{o-%Ks~dB{$dV-*DC4C3GQl=W-z?9Q{UAup5fapmJ~}4ZNItA_wJ^%25WbxN=Rg_2#~8@E2yaObFl#c7y~!ltpoHG zvS0l3(mFhn}XWdkBio-8eyLc+NSalX>A45d{18=C=`unEc&4tm7kHduXi}ghl z44T#FYfbLd1!#Vn?aW)0Mrvie+|A@ags^715}s@5z`(WtD62l8Z%KjKUOb7@HT&w$ zbC4yWSx0jZ|25{WGYOAlvKWOI@uw|g{jihBN>ZKiLUXjUeuoCQw-Wbh&s!3>>4k2b z$NeyZ!cH;2HPA^=6Ii)PT%$-3v2{&9D$P&?yu!M zRb~(+>Mt_V&}wYYqc2Eb?L)za5Rs_M0(?L$3Xf2uzR-szWS$K>{{=1-cl_$Izw@jp ztvgF$7lNc=r>zjRy$|9V$PMybougr6-65rg5Zr@1&}Mvf)eVf-f4&l$B_6BgnmlX>Iu*K07>+7i0fc zJd$nqcNn(6mBOga$%A0i~hk z`BUW}99cGm{urVs>>a1?cX<*KT%CU}wbmy(gXpG-ATKY!c6&00L^UDgwwGMV@%!AL z8)gwOhc~xD?}A{G?=L$XOpIsXoX5HKAn?pAh-+#lvjy9&v0jKsC}?DS6UKRJjS7ls zNNqyQ9Q=Q@H2^zNd4=HwWuad|l3$VqrM(H1+08pr{Jp|^IbAE^ArrszfFmi$LyAd7 z%_mfAf+8mG<@mcI`^gOU%cRIgnkyUhEqkcpvJ1_ z6hb8}A z%blu5%WUVTAR80Vpz$}yOs4tRFLH3q^L}BmrDdr_PEqVxsbpLoK038UK!JimVW3C95 zLlMc}!jnSIP;Sa#PFQL}?@-QLkj$4;0F0+G{&vw!+!$Enpa{e`L#KSY0JRxVb|Ylc zLT}Z4@vO`*AuzXj$L_Ff#LvW_#RCKL_yX?%3}HcIoTHHz&&nbMe8m&QXYZfFUx7tw z#j+~hcsT~CFudHU5dIZ~F*Fy{Gsw3$s}Et*6ay98$8mZ z!S#*dJQAAEYd?8ue_;*%FAd*|-5(gl>sKNbOb#7fVH#Y#Z zKS{irDpC0~HA_Y}&f>hDwvh)dO7i!)_q}vmw0%wXYZw6+2Hl@NchZNsS6_`pJEL(J zxU)=v8kXP`<6k^qEE$930%!H+xK1k*<{uzo0+4LD!P%|c6!^-*Zf@O#e^O)C4?Td#Cf7DF1E zM9W#1lLd!5%g{;7n4S)j!UfUPkhDZp`i$V@gAlb#>l29t4g6nKjHi>zyXCm0mmKY5 z)*QuS1PN;F1aMg8xs_qr$Ld>+w60JrKbHG)y>ZqMZ}YBc$!IMF@tK5!D$JHxr&^qC zp;f2%qtokIy#i+&8P@m9iTRgY1`-C^X&=xp)xYb#y~5E}ly(j&MF9dXcyF7BU+592VNGu%VKBeBLgT(=(BT z;wXo(f1LWbDg`S*TVF1}BOj%~P%i_TMDe#X3;;t`c6Jp>c?AVf{uY*AcXJ3&{FbBH zW#A$bHh507$DvYku~0cVlx)BAdH65`F)#`xi8y*;PXy2Evk7A6wF&zSTG{H>cNJI8 zhQexhO?sJ^Z+TziRziz%(^7x+?2QC`Newc`jgUXrpsWILVHx}-LDY+i1SRQ!4N|4? zGmayF>euD(`tOEVzjoL1Fbo0Sh<^6C7a{X&u!0*Ih=F^z*oWtsBN^OC_aNf+*f%dX zEiOiV1b-MapY?D-iGf}^m zeb7p*ASj7T8%9U5+3aFKyTJy|VpN#BQ@rVVKG}AJo&claa~(pu;SnPg#{duEEj4vj zVsXP1ed4vKdOxAuCkIoQy}oMgEpO3pX|Yo3COx0)N^!k!#HnIm@uGT!@|6tV^_lFYw) z0=&7bDe0@~8boIX}(*Yi~b-u1E}I8(h&u5UiJ{*l=p(HN)kwll>sR_-w(ycF;$jI!rIh1dw;0kf5tg#Z(NPm9*AX#N!)khcxpzY$ zizQm#%$6Fm75!xx1Z6(gMY50o+a#V(qQK*` zORXet0N&Z7b9m{<0qg(Xuu`*u8pSY!^$)I(u_oFR1W_MY5=Lct@vOYtF9(rBz}>0) zoQcH+UKS_8wCW*Nf$7X?>IZ zA7m4e+ewkk^ED1d5YGzL8inK&wBE^y(H*`&g<0@;aq+Aoh3b~r;jx1l_AQAXZlibn z?Q{Mc#j)dz@P$+NW9fcGz*k}C^RG_ycCdf`U1zq-b-^75KEVP7r%U|;nMa6Ke%Qs> zerCUh(LNC0_K?T-}PlAzRNWi_s7l|#NbP0uRh1T2dvtEjS*;njg~KB|*>^&`fl@he|z z=!vvn0?A*Qfmm9exLntnOs00o=$ceF#nHQ8*^-PA1oPfp>oLy@6Tn1_?lprK@TGo% zA)WoP2SGO*6TRdsoUaWB-`g0@+l;@8T&drRyOX-*{Ig$dw%XvQu+DcjHaO@}L-dzR zLVq{MgJ|kSY;ErsQQ~at-9meuU!+~eg!tbJhb9SJIP)>diQUtC_^9$Ty_2Ky;?5Zl z6b}Z8nuRb_clc7HE>ZWTTumxGx%Lpio6N#W^KUcNu79b;BqZRin!?m7V=z9jkS`66 zx(-^1R5N_z(v~K(tx>+i{jXIu6Fdu2;zbNp036`ZpFL{P-R)i{&{xef@)!D%d9bGK zn(b5~Sh18bXfjjK>~z@ma4#2);vpi414fQutt~osg=RT+wg!>^`4*{Qf9ef1kE6{b zl^#-$rN;(su4S=w9ttG?L$uY#zsN><(w4e`<#DpVd>cXkAp$&dnlDv80 zOUP2#OdCQ9O@@2w|KrA%yiUw}?01vNiJ$nP_WO5AXiSFH(rCPHnyzj29vv}oF)_Pc z|7oU(0qp>C3=0!B-%9_{@k;71_jeqA|#%8_iDm8tb$q*o}Zuvg04q zNW#&Vre-_Q!g^}$LEK>T?olTfw8MF7SDV96)#LO>yJfJ@r|Q2+TuaO!9E%!2R{%lZ z5gc3KxPWOC#tKYw|5MBM{(C1^Ne5xYK+kZ-5p*3Ks*xw#ya^Wu`#W4t0Nz4Uvwx82F$}z6gQZS6pn7P2 zMjlD$d;$kUlEo8Qf%|*?fT5AtNZS*IN#jeK>{`qv++m91+1Gl~>6%w`y1QY9gM;Qy zAhX}=tH-ZA@a$#xpvz>zEDm5xJkM3{&^iqbucnklnxTG*p zKMa?fU=G$FyITJJUO;|gTRnM-Ub@_GI!6owjC!=uBuy}a-Y1hfkyrq>WWmC5i|<&2 zx@%mK$L|LIxe&jZ6;oBm(3{gZH8!zUFfL{=%Qj#Lq7_uOqenK_I8ZY8OLAn%PeX){ z)i{pNt^M*n7P<4SgAeKAP!3#229s0%zgmE79&yehq=0%Y5?L-P30E~D&}2SCKv_+i z?z*_JMZ9lLm|d;}$fC%Wt!See>VHMfwVda|vXeuqPV+k!#*?-3tvN2AD*ze}y!N+| zOalu!ATl)xPyCt#`5t!EqeOGSL$ps=Lk`9zuz2Bj8z4(b9NQo7&wdJL{?yJj@tHX7 z?k9oO`Me?zbfd(zFW&L$iSX|M-uoyI|GAr2-w`qL{j(~99mE=`^%{iAks=_tR)4#n zGX=q1TTp6~vN`dH`Co~V%aWVYpna4-&erbH_RiOzRTOJddaJHmHWW$o*GLhWXWpC0 zZY&QAXjS6=a&gL~@-lK;S8OI+tA!hx=O4Cn&aC_dTy5Q`eyCV)2tK!Ci6ApHW3hYD zk+>`SI3`vbAAek%&)!<_o3Bane1N=hV;%~UbBs|_baD-RhmfKS4BPX2#TBpKwieF2c$U!n+@MXqwo!77&|Y`X&* zLK@sUpw;nV8*~kHzK4PJenI95=52!1vZxLT+Y^-VtS7%whJEU~C*A`u1GGp}aJ>A` zrwjXdMN4HmG;~7o`N&5*?M!qt^>=rucZ(up6l)9%TG9a%exek{K7XHvrHnWd&3ED% z$6!hPP8Mi_v^zdST-h#W3)gf)jq_=4Ua2IgCwSA}-(e$3x>O37(*iLHoW9JFq_@%O zSY63_th%P54(po)R!?8E{nPv2BA7wKGyYquJ~TZ1vk=r1eFf4qF7Vh=!ZE18`JAaz z322{%a>^$TxJPJDD1vmLe|P)&f-Q9Hrsb#y&3QV^TfqQwAxQRFx_Qk1eBN?H$;g$& z3ob>$Q)0D3T&qEIibXL$)>R$I;@|C_t?bqWtvUkvAqA$FvAu(J96C~Bd$Sc-));yc z16xQE;ak)NCh`h$+15ykOtNknoX*dI3Dy5UZCFb0_bg6T|F>S3B!;Et$MFb`7)}Ka z+}ptAj{n6!32;mDD*tH#y zPyf~d)PSSo0rK7&lirvZG(b~OLC3BT; zsQWQ8uAGG;6HP4s#O4lT%Q0SxvHE;_15PX|Y`bWe+30x;plL#IonIp&eanx~%5Qz{ z{C^`s0r(PbTH=O-FI*T969|K_g0f%w@Wnjmq#8oD>qq`!fS_5ej{@Xj9(|Wl-h|v- zLl5Dei~&S&lh5Z}`7?SU^X&0g*@5*4sTi*-s4;tqYprm^H}8?<>g#fx_ZIe<2Z*dBhfl0yxoTFV+bN(+IwoL?YoG=na0iNA_gJ=EUo@RgEr{r+iQq-<9Ls{}Y8O_aRj)A(t z4O_U+{25!3c`lYX%`hQqx~Uk5v?K*Ju20DL#jL&9aP321tG`E*3Pl{Rw6>L1vJV>L zR0sxbOT?0%<{Ur7$)-0-iC1-!@V#p!#It%z9^9HrasJmAoM4F_8EFc15IK5Yn`A_H z0|FeKz5NK{d%asy?HQkPMP%iJ?(L9{BWnn9MTM2x?c89B@9?`+A`IO`;G6=I#qS6& znvBx32LI|r;x5yfQp~?<$dF7jQ_RN+E|Plzi>7EVcDezua=AT}!*j=Eyspvjsqtl& zEJ69bpMS$|3_0G;(4=t_h*B;!FgKRNc{dFymaE^p8DD@zKVFvP?b}Tj27<^-kw;4+ z)Cn=jz93`(Tt()8x&93q=|V6Pdk*d^aQ;egv=PlNUL+m+zI6L|R>M2v+q`8SMqa?Z z;BO#(9H$v46UF*i361Ei8fudZ3Sd2TWledtMeDMG!ys<9>28iLsF0)1gvToNUNS^W z7~W*580&RAo+D0Wnz%YdGdZJa2BHf0bret!lH5IVGu@fO=ZaC?qtjS%@K0T<@k{id zA+fKV+^zYYH8egLOMabRXQOjU5K1Ex_YDXJbwmb9?iP+;>4Ww##WqQ8!2BDjJ%}-y#)>tr`3AawXg6F4MFLA2m%h zbD-eX53#Sqwqmz}RLyDvn&5Y{v@dNn3Qs1+ z0F>#Hto)D;M-l7s4z9W^d48GUGSwi$SBtXXV?4MXpq>=Eip@)D-alT#X!g8FecxG< zW(g2rLA`vGca3{BK{*Clq&2sQ&VbMVrV$l%Bj%#lC2r!ZyFWOPPkE;$1j6KP*-YLeAA z35*{~Pz2lvU9L}O4r&v3NZO6>fg z!_aNCdyGNC7xQ!E!p;VmaMKLEQQ-Cg2i3Xyd8hqsd(=6!#GaUkp#f=}~@L6xSoMdo!>VdHFOwcb54^va^b}BS- zI&I=uy{~)%$1WCM2^OU)%H-NJFQv1Lc-(8h7X^8KozF79YkyYI-hiw|%+o)zCwB8V zAjBfa{rE*8>W0)*AxeP5IK?DUA3FUfO+|64;HN2rnU)6#FnfO=GC!%dAu$HKe_* zD%PIk`FKf(j{Xm0*(HVB1*>7o2Ys>jA$_LluR?|&Vh`xdD{o#0dxANO!EB^i-N!zU zV+3tpSjS-E7*G6lqmjWK{fUHfkv(n+?OQR9X0&0>{ zPyZ_TqLmmTmRR>J{kHRbi1k=YBTvoXR$85${b)cUa97AV-W2Rh8OrH|3U>PQzbtq@ zK0O6L${4uq%Q*Q6F;#8NAeR4uDG;I+TDZHI_HeGh+!BB>RJph$eB|GQLt%_A{5_{$ zSrqKe$+=cR&Ch;^^v4brXhd`EOcbC`>Lp7M-zz&tiJ*7h4(qAYeWbFX+E8||I;8S~ zs1T>$&k@CB@6QRr{)_~e>8QLDUq}k3C2+v!CYtWFQSV(Sk`^&C0F5Pu`5yaZ)HmY0 zO_L!~s06&gaiq1_MMK8q7md|j*(V33 z{+VM}qh%+Esp~g(rf~a9uceR*XxB9>mj9wQ!n~Jx*pjkR6%x|H^o!~%37?(3i4T){ zK}(mB`i6}*1R{S2|4AAWfX&5gt#JG3csfAjo$j4Pr?ht}2yZ+ZcSZYfBNHGqH=GMH z%EVboFRr2zPA)_rR_PBZd$v6Co3FTM^R*Y0`=@wW)7%zQ_hEcPB5x0It&F@GO+`ve zX~Oad*>0pVQv? zBB@7>=MSzU^bxGk;O?B5;sV=h6?~PCrbVx}sU-g|!rn3{uCVDA9h?LR7F+|t-Q6`1 zJh%lP+yjB&?(R0Y6WrZ{4iMafI|L1SH}8ACb57m5RhOSoGga)}Pe0wO*IKQ3q$r2e zkFa3@`SjbJOc$s*BABwn@B!+LB(eJGjGfrmr@`^_?PBZA4tpm^JFx`@uMv2yo!G;T z&d+aEE?0sO%luG<#5fPq`SkQj0s-5QEv&l~NkK3wfJr{C4o`}!Mx`SvQ+CANL->!5 zyc>-HtMpg@kIruw)T&6Q<<8!a5Dl%R^0KG9YqwM!ge8R^f0aNZvX(g=bQtL+e8VRN z;mhU@Pl-6zf-h2KyeXLZ7uRi#Ms$8C&a$GM>zRM3v)5nuFRm8W(1UXuZdm1D%y3cza%DCfiMJH|VuICLmTaT*b4KSMb+FqHq$5Ovo0GmETe5?cy z-Wd82&1`Sdx#z|a0rYV$vOmm~zGm@UQQd=f(HSFY7 z=?xJyv(V~hVc#61}CXc@7O{l@L0gZ^Un>e$MF7Oxi|xvj5y7(+1-= zMxxDn(6Mv=ocDv4F0RSu8z86cI@3fY(LD`BymO~=tEK3?m7qa$>LpjJuY=&Dko zwgVfFPERk6BK}Skm#T|ah0ySorGfu)VDp?XP?lz9-h5JQq3;l+?Vxe7A@(%+Gb{){ z5*-wIGI1HHU`}F4~O>kw%Ld>?Wf6&_u63jcv(AtI>7j z{*Ab@QW)uKL#F={fm-|Ta;!Qm8Pj+$!6kccGrp45N~@a<2tDc)(et*dt!X$c`nQAI zth#1T%U(7LK+;u4qpT4CthRwca0l%_DuA2RCTXmSZb(=yhXLO4OJPd07S7h-qyZ!h zXGYs(G5&~Xgm*OQ)1KlNPkr!&Mt{bFKT5v4AvAIJ<~Qt=rf&c8UK2l+EF5Nb{lVPD zEP*``@X6c{EYbeIWne!=XTTS1$*;|XJSR2>P~V5~RjricQ@qwXF+R;$MbGw>@fK#h zVGJQ&>~ddabFjeSEO#?8qfRsof65KK9!7t9Wnt(L_|YK~Oo;TGPR$N$X0DJ_ zSZJZ~1B|4n2aUcU97_{K`Zoz+cAKc1M1K(GB56lqWNDvOZO1SoRY=c;xE20K+ih#_ zrU6wmV%n;{Sa<@(eV2{`Xl(@$3yWz-QfR@adaeXNhV?OtbF12i0565JrnUnGP?M zy9HLJ)$OCB02ZTf`WLgk#7nA+F`@WGD0`#x!-)xi#!{&1H&dLV&sjOQ{^juZaMuEJ zS7uc!p{zt@svQjmsm~Bg#(w~+1>`5H^e_VJlck1gryV(beitynN-?Cscr8Ghk$?PM zrEj)8l&kSCH0W^Rtv1ArwmxAU# zlRq3*5Gf;lg|{|-#NAzHsL@R=I4yB`#3PH-B;_)?4@1&cAQ2FZcCyXMDi+VjcY-aN zSdNmIz2PoCG+4t!9UVz6tHEsTVNA7yi@sPOlFpw5TkCV&PXPAYX5nK%E!rC!f>9Eu6e*9->t{mSlb z?NIUX?e&M%x_`o2=yWBr(dsg&SCYH0St)b19`0UYzrP9w1eR~fY7#bdZIrhK|vBnRGQ=_E12uLy`+%?=MgJ}9lnPmN%TH~ zyD~pzn^DW}<5EZlIY%A*I5s+{yj(g?cj23#AFhh_hhsX zze+>meS^;tU;R7dhXHaSeAaUnH|w8Yw39LbA%z;ib;VyhdieLa16)AGGH`wV(a=et z)_Jp*PJZJdDJbu9l^?I~4H9m6l0|c@l;Y!W)Wxg{WEClR|Jr#qg14WGc#Gd(T*Y`# zAJw`z{`V|)V5(k(M?`D{%&+@Rirm3B+ONTC3)JoyqFngdGU+t>&fOK2V^8qnJUjHV z-5t8O<0Wyg@@T7l=RHXvPDsEqD=0S4hSoS9M}<*hKe>v>6=%yHC3BHZk4due9^fkp zUEz%;X=yrOO|w-q?A`%?s#_8S)PdQAW-UI;uP zm4<37wmIX7e~N{DPg23%pO`_q^WQB(Mh-9Atw{+JVHtLh{Mmk1qL?8-2@&k!LpEZW9NJiU>vdJ&7tS4664v_P(&u@Nhly8=VwX-QguqzO zt$lSaD^Zekd#6YY$H1?sX8V6QZ%!3UukG!suM>=-#^LZwZTs%@8sNF6pAX z-1e8}hmd%L=nWw-BBan9Qm17`$I>uCF6WE)AyljAU1;MLj?sS7gakn+{&t_)$B?NYTEhK`|6 zizvo-eTN33EE$&!2fJ_FHobW2iC4JLwCl0IHeOZ3NZe311gQ9wOF zfSot=5vm8Uk2fBAZmqlkR3`yJ!)Jw@ek2`s`Cs%JagXf{L40VEUDlv=LaRe*c_K8T zHg1{hrXZ1jeo6M)Or@XI?*iMd3#mrRWi@Qw^IEmUwPINL9-6ldG9LwFT`Cl3;BBjX)z?S<7 z5Ed_){^QhKz5cts@oG;a(ciV6%1Zmcj72PLYI9;+fUWDXT1JU26&XdJ74drs7!A>e z{*BdW1zz*Ec8272;M(%nKRfv(aLnwmvl61%;Fgva3Q<5>VRXDu{zOWyIW)PB6NvT*%t zgY5ki3x&mW5c+Rz8gte)jRK!T2DNtWg^-NG^V%{eeE?ow=Mo>XMqS@ zK~a>86%d}{ly13v33Ay({CH-J=moCJoM}W2;BbJ9=kCB}{z{-i5A$Byk4B}28Y9|^ z!`}nL+s#*xIp+yJ8s_^12f#Pm^=P9YLD3%=U-4V{9^|^Lw#`2_=Go{?iJPtfh&p5T z<>j=p`ADei2mKU*NqN?2l8?_)le6ND-n~B558?iF5t7 zzE2vJ3!YGFJ?w=EP#Oq2EUz}-J0a=0US6LMsVRLqkcYMH$z+krvdevZke?Jo^0LmW z$7*sIG)7oTm1psFoHwA#3%|mQI058Vr9V`h-4nDf=@s3z{(X-BR$`EO4MA< zfFZeopZR2!c)QnSm=aIV-nu#58VP(cI%8nQ5OH~|!ZBQ>wD)$L%rU{@VG)#Rahp7T{b|`d{f^J!!n@b40p zz|WcUKMm2bO(GmXgteSt5SoD*pdoovo5WtdqxsW7;ZwZBh8-C?`QC6nx9j1T3vdR9 z#k(X%?djx056EKCxlU^5KVk(vui&HSqN~#9yuU4^)imif!c!5**6LrGH3ChAo+_=% z(Hp)h(U+ZqO_J|pAkVdJrorNvocGxke6<`HHi#I>D#NLaYR=l@ z{ygZMzd@n`J#P>Z-Ofgr=~&d-va?D*@4OvhW|Z`V4E$Me7{=151{)ntccp~Ll!ypI zZ~k&md5jYwz(AjyeR2@5_9c!wwd6k~(B!i;RqcH!bv&Fhw!Q#-+YUx{MqN^x)+u1b z7}v_)J})@_abJ*t1KXi;dYNs^Kl0e}Ii^rowDdba20EEX%lFY2LkhzFlpm49`eX~Z zMI)k}*JXnMjX6fvI{jLA7XkTjwW!h5MKsv(Tx7`joG$N- zzH5`ip`~+|SFJB1R1cjqXAD1=2u=_M89DY`)W`^eNhr#oT7Voy%%|F@COox+pu7z90W#Yoe6=Fn)s@7@lAXBmRh)ZOg2iob;?iG9Net^>7RS0UmD(CsT>WW z3iG9rs5?t!o4xIzg$21C?ti+sT9o5&zXj=Uom43CCK){jUeMX@q7t)IvUUu8Li+r3 zU~-SK0OwL9KSqc1BS^WNPwFWxM{%0$7$!u^^k>Hfzj~EgfD@j(8ZNji^Eqwlw`Kpu z4yp3(hmHe%)6m~)y$F$Z2klavuKn(EDRgjXIl^Kf-f@Zqqoc~w5o#qkD5;RQ2fW#R`i5NCI85d_7fv!&7CQ~k626|5GWU-WIo|! z|2oCIVf5`X1*H97dg+0M4rrcb!fKRwqIEf@6C4V1%)W>myh@y%k&GJy2++)&M1v5% z_d|-i9^IupjE8z2HWpj(V?WCH^g-h_*}6_l7z?n>zPJk%r{BM#G=p@$M%?{dxe^Vy zod;A-blSD1-lsbQ{XyKmH|LuY7ugrBTJAL;{#D}O;;9clix&HIP+m-ybvTE_o80Di zfHs}*5uwq~4xxirgpA8*M3-~pw+tz{!_s;f+L8p_k=&Wt5nCbN1qXdJ&_(QGK5iX7 zkF9Y*V(=5r(onWqmtngk8?eW9ocg-tF+{yz0QCe0!j#INfmIu_w7vqvu4nE=N#yzO z@slo}C~G$vJR)hALs7lY-8m-O$y`X|1b{#0AwVj&*`+sAhm)96gV%lRDY zB?bd9<=UZ$&pUWG0JuYsNn^zS+HG+v&A28-P0y zZwg_9P{Q@_MAdc&ioMf7zTckKC<%vwJ*7D>5Z;>nso2_@lCNt##Jk8^y+W11&g9>@ z5=bNQRPyk|xdH{%#xQuI9N7vt1-`qmV0Tf5BQ0(x7|u>+MqRD>aD$M=ly`5Fv#yM3 zlu?Xu(p7SEzBdwAsm)GkSZ1thOCrokXDvrUgLd#Adm-0#p4FZRb@@4B!Q3U&U$e`c z_}SYWICNeT^3c&X2Q)31aqS*c1-_07Aoe`W@HJ0fEypWhbJk~lIdre2ZPJ`7Sf)HD< z(Sh%25YjPZxry}e-E$G*z?pJIinfIJ*X09qgJPGyzx^7T=5b`!*GetC|&v$!4nD%s-pMx}Y>>c1kqwcf*&9H@3tJ^lEL zYG(Aa#*Xqq?Gb)q4fj*+x2n`zx=O8c0?&Yf;>M0P)th!cDSH@b(O5D(!tCMvE~uO9 z<1&rMYMQO@nAUW+mE)W;J+H9 z?=97wiR(aeeT$?GLN?Uutov4pgraq2CoGXH8pm41-Cd;WPbJiz_5%(Ty{^>0b0rEW zIyHCB>4Hv-RPu3|gOgdzVYrV75FBHcPI@v8ywzB=YpCB-z0nDO>-Q%_v*FA*>CdDx zbZX9wl5;2oYF%ILyV(L{LMLqbn~Je^p!81B6?-3@ZUpF~ z<#;IFC<0X6-M|Y58XH&s?PMvQ&9>VY2L9aU;pPNB4df1z{xItWn57(S#fT)^rhD)L z6Xo~zJYLy6_6F$!Zp0#7tREYUU70x#QBfX6;x!SDxV3Y%a&7L~#n8?;QtIQrxON4b_67*)d#x3EFK zl0J>qeWnVYZ=5$U1c-RnZ!9gE910d9KTlg5jtBGC2)qrSGMztqkDgv{hL%?1s`7`J zxuc!!K(X?01Ob}px}x;*pLn#M45d8*nAjgadZ)7*iKpTl1V;PaC^b2xg9wdNpU|R= zL;lALV4!Egqes`Wz2Snd%wK!2V*so_})ZbG@lLYzjLQs}q@O}4) zjeFHgl*P6W8;v%Jfap=wmBMj{G%)#Oj6UvbyfENgQ}lY|Wkle7&{py(Um{G_VJ*Qf zmS&=;4!)?EM?oI2{2@1WtkTfHpF3_?RlJx_Q-h?#2?PDxnXqocR+vuoxBlD8=3^L# z<>HD|B9qcPtL3IPM$;tHcQ2u&PvFt{j1uLryx*;w7K)&A5h78~ItLglY-o@{6m~`- zJ;~X{%`eOR!I}Fc>r&|_{@)N1dxh<-H!Q84cP@0}q&!jV4ya|!0r$#Sirfx=_8keQ ztU;?rF+;v868ljre2I)2Mvt*^F5ok7%bz0jb9cV<=OU{)H=?6yEIhOJi;~@U5RR8xe7 zsKR6uOHi}JAhIqu+m>beHDw>U@ddoFhWJL!W=mOqP}FqOB0hX$W=2cZM2^iGg0zD9&UXVC znEiMkj8N3h<_$V_-P-@bhJ2iU2EsPnorV8Z^h_YG3U8-)+ol zYrrak6Ciu|BN4qb%@bW;2>*V6Wl66L@wHi(Z0Fez`dk272aY-wixZ7fcERsbwTKX% zsXQqmmv<;Mn#475MIg*{=s*c}n=FUI6xOoEj&-5qn@Kvwsd!e?X@AodzR4qzZ+tn6-x^~bc@ zY`VAmvO)6i^OI0gpcNNt`yI7Zr3?X9trq)Jb?P!pHQ(EY%v5uP<`12!efE(J&>%RB z8mUb_H$@AMTe4`~*fwF=Jg(GCnZlav;Wq?qp|yJxzZ3#s!5)sCUNg9dGr2@m@@bhm zZYQ!Qg!CXNJoI#}RRPZPrjY4~+hc#uJn}JyI-~Khnzi`v zZ*hGG-@pl{v1YMb_(Z`DzAJ1N27&jE(W-5TOvv2^-RFK^+CP7~kVQH6 zEBdMHtG6)Hl8$$G9xPD3hm;!k8Fcc2=F^?gEop0an$m1;z4W)S`q1@3NqZ2X`()Nr znn~()9vM1uo^j!N@kB=}N;8d<);gMXS5Z|Z|FGqCjFqD_bYFFjtc)hJM>?(q9QG%X z!QKvdPpELCp)JqWq~KK@NASY#}cGaLc@tX8eL&eH*(*y4&l(y zy+L}c;EvG_1s`ZU*h+uww>Z-3)gBi_*)8idI0#3*ohaT3reEReb{vozzra!U1fP z&=30vFna420j=LI3?$9K?%bTi1o9y|Nq9paaRbG79`I-}l`rAx2tgyF&bDyh_^DkN zESE%2)x8X~Sp9vd>$1LINkyLTDAh|uZBZ;k0y5&LIJPZA-%(IjYSo= z9*9TxxBCigc|(xrJ|wkV3Sy!$b5RwhP8l=;Md#x2g3sFQGm!`*8A{s+LG@g8(rQym z6KHwqY=vR7{i^b_3lT@fzc@^_RV!{P%2kRV*kI~A=5QyT3n|nJt&*u+>Qq@0+i>q|GuX!di4N$h&0~3sUZNi+nvoGtq zS6-UKV&1nJ0_9P)al&jc^vQ$UT{)~#TPY)F>&7!H_k+}Xb0KOLpSbNQ$Su#i&ENkD zPZLSPZ&O8IEMvP?yH(|MX;nVSoGam&?2eL%Qb?KQDVgiBJ!Vh%k|r|y(#>zvlCI>M zKN-mlK8v-r-2XJwAeh0V{L)*_WlTL_;ix-_f-MT?^CN4!Do6^L zN;p*^fv1Gr+aob;!b3!u%wxD)=Mun8wXHRK(mA8I&e`P~m^H!2-CKwOWzkxorA&L~ zh_I2uF6jVW^`zQ|O`UE$6aXMLignmn?J(%@z!F%&Yq47nHG$!U`OvPe^`;4P6U508 z)-7<>PPt6bfiM=br_mc|)lT$6@wrwiq&Qbj=3n|&cD3xxy*{cvb3 ztDs9@v7{^jJ?CoInXA+o57f)X*xT$d8nib)4v1EQQ0)AG5N>deRSD zMQM^7+Mx6%rp(xT-;SoDNPHRV z<=s>O*1;I3^BK{nPy5#Nl_;wqD!IohU5a8$KZBaNn=K6QpW=0@(@4Rj5$YPfTbaw^ zENGB4UEU8wt_$)a9#B@D)^iTI30)}}MN*}-T64?}sNn-O)4^SgOsQ+i4e@$K=ij9- z*e2tV$sZLQ{)&@=MUL0xg#=u3FT0+llgc%5?3*>Ww8C(;s0UrcvD51Opv3+-0}uq0({a@774O<>)sT{ZrFj{K;7h{xOM zoPnyH#Y(vAm~V}**F@$Ze*PPX(ZmWqxM;lPF>|l1wg!L64!o?VhPo`!g;K7n4{|ad z2gvG|W>`^2Lw1ns%S?T$zIfa4YWnT8pRQA!*HPAAQqC(CVG{sYdq&!B(GV93A_QR( z^RKx9&NNT6K@2pRx*&p+p%^kHnP}oS*py*VP&&&e_-q6qt9BXGYEwLG0T!`>jsBg; zzTYC?nke7&(B%9O2O!Vf@0Ys>7Zfv^qk#73qad&SR9;ZOltdkRPYWf+DMCoMg})DG!eaMez~{qyyqDPN+|p+H7c{=X zVvj%WQ5&p(XL<|V5+%`Urr*BP{(*->2Vc;jG_QvpW8&xNxSfhht+7yRYwWo7 zMa(A*DM4q~f2yuM+Arw}HXtpf+v)9&eBB{&mE$_(tO>c*Awq#I- zi_Wa<3$fan@X5V5k{7oR=E=eh$?XgK^)|lq4ud8ZMD2)z#cQs>ozeXtVZ3tynE7wk z_e56OPr|GyqX5 zP>{PZp(zXP@PQ3^1_$zUzE7_iE6+}xjhXY1CUYwgB6^uG8>H&65`v#9%I775HM5e$ zOKNj2G<+Jhls@=b{YyO32Px=|KHU_*LY_Huk*T%5`sB|7=l<0W&7gEPl%8u*6a^5P zBb422Vn3FWi@TSoB|N59Gn^tPU_vl*^`0a?hYr$`U{B zNaC=WAmoZWU2Bm)U@tD7cQV~;WV2-P{QBf{SmDb*OZ2@kw@q}++AX@>7Y3lzK zugvkTF~;3!_$|tZNjNAJPqaT5=83AhYY$gZf{)p%ataa2Drd(p_?fIn;BHYeH(`GHO%xIZ*`47B3f$dB)UzWvLVTnqfoV2<-!5=rr zn61H{;9+FF8XcTNXl0#EojRqdpT`GVg>WOBs`?6J34FltI*T3< z)87Z)&+ssrBdCP=^PSJ#-`!HuZ(O&(#}p_wUfb>}Btp`_ z5$a^KiaiA5Yyjmg)st=BqhwQQAylVTppGW9S{oK(X@Iy=Od=;xJZXDr+n#VpR4H(L zj|3=&?#&ZhD>i$m>=-dtn6frse~AsjFHU~1@|P+gl0%q^Jz_+&fsUB-B-UDDj4*Al zd~78J8x!18{&|I&C{9=ItxbV4o<&|*D!L_zqd*Xv%wniW_T${I{fmkCxRw>mA3vaI zb)4PEPGvKZ{twRibhB(A(`>iO-`ng!{sAPof#F;XDDUDOv9GYGmBLHX`CJrHz1?C;{aKs7%fDW!REbz{1sb1_Lw&x^3omuEjVcoJ4Q|Z2hEZYD7!XGnCWnV#%#Bc(t>IxHI7jwr zg3wSJ(8}khr~gG*b_c2?@>>28jSw^#PK)5RrVH&PY1KKl1fjX^!eAhis|7Qrm8>A) z(48$K5YsV!eC@TOwnAc%L+2o7YLtlSxx{#IECwnMf*>@OcRWA!^D*Iq9T;ovX=kjp zOV)>b64{*1r!G&k1UP->^zNfO_PzNYPy)2w5QroDbRit(zhwWt-7dCj5P3hz7P$-$ z{Vd?l&~!8xcgSVS2qHX0C*$omu)jPQ&h5J*qLIFfX!^D#o5;&n*vqB-$E)n0Bgu_H z`_*tDsS1gf2@O3(_n?JW=fOPAC$tU$al&_==Do%A@XU3fe>5(*~L#LFz;W z#!;=wh=fNO$fN~Rto-}iAEpzCzRBFqh-bd>u53s!^O68m)FZfCr#@!3U?8Gdsr}Ww z&2&7*>?_kiY4UY`$f`%xd7xIU0;WcgOZn^7can>h@WmRP;Kv^ih^2MQ`g6T6 z3gq8RI;L7O3MA;f{h0_v=>J4{LlBS5sM{DBODsyp0t?wR;&$0rK3s14=^pk$ngGOR zKLof|+IGJaHP{(R3Oa!wl7&re1WZ!G0Bn61Mj6cs@T;?Sd=@`pg#@NjjPW?pk|924(E$;@{p}c3 z+10OY6dV@P@490oe(ybBi~k)9Q*+tfrh^#*fSgZZ?%7uuHCdM{lU-1$#vP$GRe~U) zS%aOE>C&0P!n-$EPJ*rAMj_TSnLCz;y);d-=t@qh>zuTnR?Z4+!KEw5ke@hLu4zIt zFp>GTk9TkAwEvnWCca2TnH#_*2Lox#wcuKVov9pOaz-(`wUp2nTZlHFF2a;l$sbz( zY@T(x*R|~zTzPeFT4vDE${qG_qC?)#v;hmAQ212`%;nu<=`ysEYqIfxgz?CD;+iVq z1S%^Q)a;>u1p_Cdup`7&G8=-&jRYBO370kqtQ(TlerO#@JP@Pax18~{g%6ZZl&DVH zKM=62D4qq&^_J@(C)|J40H%q@o48NruPt{59c2YMo+!#@={26Oxxbo@nV<{k3l5`J zY*o#?T&?M^kE}1{Klx$0au6UNmhl?HsLf-r`q?GM*uBUV7XbnP=TNoaG0OAz9B@zh z_NMp-1oL@uSp>s&0!ue&$@xLy9{Q<8(*$&KQ7v2M?oyOYt<4O)G1! z@*ggbXFPp+;l4kT4T5i3I>3(9`cpQZT8z7RhxOhN&-%uz+l_gem3JZe0qWe_X|5hn z>TzbIxI!z&+S&Z^%KM}lQO*c#FNCT$W2ZemN$oJw}n5qn$`o_n2_T(KA*VtvFoat-*gkpxw4WJlHo~l#3>xMr9Ts5@pb-F9E51ivRe%tR@Z_)Qx4#oyG) z`|1!5tNdX`2LR@TOhr&cB?iYZDW#D`*uj6KVf2j(IG*|>q4gHc#&_{6kopL zQmJC+|C}{)THvfX!f-xY@@c;Dw%A=7eO?vXYoR|mMxT`vxwIh&KDliy)Ay7&n%7lsplTIMsGyRp!TYz8+|6cJDK zpZ>5d7>JSQF5qb*_6D2!ef9C-Y`MROLFW(cZf=__Vu5cJ?zw5Tz)7=R@6>P96UV(D z%@Sxb9bnX~$_%{&+@nf?!8mN75nB#}CUY-7=oH3L$o2632U#v(G4dMzG>`!ZSnWlr zD_7%+%wjJSLy8mA0F=MB<+2RHSOBG0HT4YBERy7eobDjS{6$qrYn}Jffg4*2KX3ywT%s*u=zeS;nk)-I4 zq#uWe*OspN8mE$wGmUI0~C&Xam33Onbg`> z`)exQ!!q^q<2}n%Ro2uIpxAaqueC)?M}chKR8*KO4kJMCTB~i0^cf7!=7G{GuT+*z z^1vdSN(29k>xPJ8Ud;@Yj;_?n4hvwpaF^{gE}GfLp@0#9;5`?0YgQftuuIr<5dMR1B7Wm#!&JVORBVV0g@b>1ixk_j2l zQ`)BWQCt4o)H23^eBCvz#zIjM6a-MM;v2|o%G8N)H!9RJ38BsM->0~aT(unub5uD! zzt~nP8fXEuyP~0@hQ|B8(rH+P$!FK7Ic+X3AW@#Ne>RXsX&*KTUxi9w2iT3Uj^w$? z!hx;&|I(RUE6?@WWiRO_Z*dbc{smj2Efp;8kh_)*Qs};2T+!irekWPS+c941KzK~5 zVb-cwhR@AK4AK5C4kvSpZ1@83`&ECr6%u0# zpNF8iBT=E8>+j6=1Tm@B{3LUG497&^8S!pVkmefG?M%$?mkPm zOa0pH$CAq)Y3QJMqjx`UM3Kj1V^&)%Yze7wOaCoegQD@6T-na8{Vm)9?|ZT9txWl( zYaP{?In6ieMysCOU~ephWD4{B7~ zIO<8HRYg^io-*C4y%cbEZ%=}b5jMS&_xAe99Wd_)LJ2dT@$}7{c&vIDpFE+AZ_BP0 z0t?VWvV{oU<9a^YY-V1Ln_22GKXDPRl$Bv^dAH)8_W&%=KC%yv8(u`4kr>{?Ks_2L zg(ArdIst4*VKF}7>Nvc!iL(KCpaYQvGktJq7yZg1WDPg`@)_Kf?jAd1X_uvWm;$zd zlUgE$)Y~^7N$^ekB5(!AK}%2fM?amMK_zCSpw?IZ1kyWV3>J2G2+X;^458q7^$F)W z3<>Z#hHhdICqT;+HGeqL+H)r9qz|2&mXNQj>}5 zXa93~JH|u{ns=ueXlciHFqc5vrj_J^Q6)&-$tOMhY98?Dmo7GQBdGZ@!~gs+hAF*< zo6YBu_X#1oo-?Ae&XL#s!TWcNUx+usJSAXQl?zwf4eV9UqHZCNk>@p-^lh;$&N*Pe zr^lsugnp4F^>C#$M6|p?jB^KJj>x7ls!}e$G=A?B2BFWe(a6lxzBM_;5bz}JV*3G$ ze!4$zDo!@NLu^sVZ2}OX)lnSnq`4=4-!3NH?=LUV*m7XM7>vcQEqUZ4XtuK01}w`s z+&xx?u>5UrxRyqlXzt#UU*5REB>eGAn)3Qf1SYf0$S?$@zcX?v9GT$D;EHMZ0&^Cc&JDyU*6AP+4AkTl*#Y%?7&gWkC!&rc4PT{ zGL`8DGm&A+oxnQ=CJj)5oz&=MV_r zKiySNpz@jXyx&9vedoWuwEhvdS6RukblIRx+*yxiN0!^OcjowyVp3NZD3O>rt5BOJ zW2=F_CMZyyGI>lPgL?{K>DX96prlzS5CRU2Y~cQ!54a_XC+&t~I==vx4yfdc2L(nh z*aAvVv@xu*J!aoQ78XSI+;Bew)71TocP3c`X!NCVdGETDqlXVVasTqga@jw1SV zJnf9JglSGQ`xe>$J@e@^!6!(x7)#xxM!&lq zKU0eVZb?|Mu)b)I%`qD2jk(%So6h0OxjCY7>mh^C&iZQ#=CRmEA!ftv@75^5-4&EJ zI-uS!1C#^lF^G@ooMeLT)gP7m6+8bIoV1A}>UqB5*S{`?+f2x1r5uJuMXO%+r8WsU z+hUkBT+4dh`=PSmL$||&YiBZN9sm12o<1lFh6X9Aw*AL@QDZo2(ha=bv9czD_v6A` zcO6sj`_0CMIFL66%GvjG@BS^@?O)MQWmJ7nvuaXfm&$GnnlWKcGP@c>Xh;*HY4oxU z=Pm>CH2FSXhm2&U~+xyuT*wZBcuhS1J)z3%lTjgbQ36V%B3080ua&SSDm|2 zdV7J`9%KyE1lb=eS2XI(GMj%$Ee=hY)4Pt)N0 zz<>X7w7Z0W^pZUOrqb6yH~T+g(n-+D|%R(aFhF*qZ zKQMPWo1u8!1ll+BcSQo~8o$yxP)a!t3*B5#IsZ8CeXL$9nKB4fO4{RWlHSDl{JoXz zBnV#xICT-uK=D*+rOuOD>nawQ!QzhoXEBvgoBg{%Fwp=U-61U457ju!1CF%aIhb*F z-Q(^xobBs3box^odrl}h@BZ);u{A;5aAOJb|7`r{8^!A9=FeGI;c2~|1`;pXG% zRN8_7ZYarPzxJMdyD-HdPcjTEOhyN=J1*6#F*f=u)q?=V$7h2gsekDXL{nm&Vjevk zF1sZs6h!PAH-^49HGd*cgJI%wH~ut-AKq6WA1x|FgT9z7I1e{on`_HzvfGS5Y<`o8 ze>@nOw^Y-_d?BY(MUouU2d1}hvcpgSZa~Z8hq-y*>>vM0kmvhTiNz0L!vJL~iUc)8liQ05Iv}bdyeG$Z97Dw-ewmzoP>p2-CidqzEG}b ziS6<0CaTZ#1&tTCD{~g#H~;uz1*rxdPKcJ{pM`7cpH{{IR|{@L84|?jZ2+Yum%vo8 zn_Y>uXs8q8Se+}%#|T}Adt^VvAzE$uo`E}g7Z^>#|G9y1zL;m72^_$Li}JqSlHCBe z&S^?@sd|?|E{PefVShT?xPx%?RD1dKHX_CH;`sZQP8~CV6d%C$jq2<}z(gJO=fVI_ z9|+Ii?L6G9<(f#@D}VsLRAWYzCF~eK;-#F1MfK_U>MR#!&}1>D>*fAZ$?Bg9WiFT1 zYUW?O-5S;|#Jj-xxQ|8bGL7H<(d)GCbBttip|}!3WwsZ%H#q2sfGc}J_E!qaq*E+^ z#JrmCOssra7*fDhjP>*2HPO{Lo;CP#(Eea8tLSbgfJ<#Kwf6OZom{k7a=w@QtS*|!*C z60xw~Pqn|hi_$qP^SD?{l#=Fal%m^d5G&d6AGcqLSY4mV9=(p)Y3@&XW~wbr%bTor zMc`1U282Wo*V>-0Hzn#z+(z+|bekDlLMLuKM2%4EfF`Lbe#-Y9Rk7dZF zY+_<7!0g%GkT*U#2Ar191OG>-Wdx{(naJXEFSgtM`@=xr8maCp6LJOmD`?`@@*#`j zhCt{79bk>2+!fMf4i~GQ@y5^YP>Fbd$(FSjj^paP9quNV==NhJ6e)Oxjo?bNh2*$0 zeMUh4%3n}S0>P%oeqSJmy9n~>j3~u*n`0|Aa@tX#_p&`~P($o1<+SpmSTf$MJ)+-= zEaL!Mc97JWE~6YRbWmbee*e4Ixa>NhaCvLFF2twlh!#4XG*)ztwDCR8YWd%bSMqrs z;=wm#{^Fd!W{03P3-t=6)yr92=w=ZD$Y#a9G*u08vSyc8$XN#EKtiS%wKiQV^VRzN za*Tjww@|u%Lhl+5rA7->Mk{%uP;w#y&)fgwD+%r`UF{y98OLUtkZaMi@>p(d37iE zKHd!HwXtfO^s^w)TY_zjp4hm$YkGg?(t`oM4^KU4*;MlVxh33PS}_t+C)flxQMGyr ztP|pNrk-gpZ%>xVSu|B{H#%?AdP6aSe-UKUeuj9xNInIUqWF{VItI|9vYH)muR}q25iB%jU+#Lx!$#o@UC*pApr35g z#Wub9R7wPosl>PS>v}z4E;U*6-F)T_+4>tPG7dQb@)}Fy_ZLSdl#_}h&lUO-q_|0X zHe}P!-d>RV`m&)iZY>Hk;q`RF0et@0qbW>ZR@}d8Oy6Zc39n2eVA240-(V^r^?TF# z85k)&k6Qo#zeHyfapSPyF#$2s5R2}+ARtbYfGAzef+*pm03Uj${r;FNkQyJAVE8xU zVdmnCj{}fm4`*IsQ|Qi{Y!2J7ES}MK=>uRzz9Hp@~e)8tLeMr305aB<{xVG z13?|dKhM^f=Ky3>IhIr`tBw1}GqnCxcn{xI$iS_e}RK!a@J~*>3A{f1GjS z7c9T7%0mqF1i<0ThO2o@cDraF>Bo1+LM-wbmk=)c{wt&7`b)R5w4E6v3C;2Fm- zU6IqVN@oq00|+vm$Re|&R(g~$@hh=UylQ3pff(O}(bHPhJ$BehLeBlvuW>5~|U z**LmCI$L42hG?R1ogw6IZRl)nkS#}LcWf~RBPc3w7m+_-+h(_5?F+0D?`>N5J&}Ox z$i^63^Y}7GJemB^iC(pM;ASdT%`c7k3s%68BDOD+HTGsQ@wEGG(r2>AicQAs2Z5k_ zcSo6nsdsMQ6}Hl@IgvskT`IMCUGZ0h)pl2Z{tC5Vxl2#vETgMrZwkX+8@roJ%$Z6p zkl6nE2Ea8Rwr!n#aPeOsx7qh9Xr8s0t}jyuZ=92;RGVWqdmf`W*{sK+5b%Ds(%B>eAHhMsd9GWtLFTJZUUyY~BUVgdI|#7Nqn z6Be@fyB^68Rdo|dO=1;L-MGM;Hi*$BY7-Qa1HsDo-G?=iMl}y0Q@?>crdwpLfMXS8 zK2zMq3WAeurWkO0EmlpFGFFc{{^a(KqZ<~&n{0`Ftx+@ zTZGldqdd1_GZ5acnQ(5={!+M+P_7O;sW&tiJUYY%ybL8uguaGLLBSWRjj9H!<2IL* zOCNp;$qqHqdO^>>@m2&OcaEqJt55&R&8L=YZDI}L8;C1#-0COb@km(mTuEr`;zv7y zmBp8`acHG?5;fR3#jCfO$N-eC)T2hr?u+@O55-VcY%W^|RuLpX$z(b|qF_3o8Qy3n zGmAD9pd9%v_*nhTQO&mIGkYk?R+d@I`m?`4!o+q;zR>*e&@FFVwr8qGmt3CZ;;-0g zMRqhQI3EWSe)uv)4o0%+d43vPuS@4!Y6plzFC(;*k;7Vaz0;e2i&P?o5h~?s$U~2A zb48HgeXrNR<~l2Rcb8>wiM=s+pm_6B@HVU);Vn2~9lb(MY<^y_Llql59?LUTrze#+ zL>}%E_Cdn5iHH;hikawbd~FCX3U;HE=D&o+6NeYV8VF7A@#g)4w+8N-9shs7-?*Nx zaB;ie(zQLEwg8h&E=taiWl@0ghPn6Sm`eFBJ4@dKSRf=uWW6UpVlrrdZ-rz+fxcX>w@5CI;tD_p;I+GMkq^|M zldtZXqj>zjrz=fjm;x0-1p>c&=QWXBC)LBzy+#|?-iQR;WU$9y#DjmH6(Rl`g@Px> zv;k@$-Auk^GST}BPtr>YB0F#ZOTi{X#)6@gsI@YMFuAW2jv>oyf1JY6DWpV=OVyK_ zRLZcuA6ClfAf{g1qjnv333g9P=<6ZlP-MO)Ers{GqnHGy?i+Zu3BR0r+nhRLEysf(?+)2h)*Z3DCr)mMo^- zteULH7>9`LG+O28PDjfkV_mh;+xf_clgXf7CnLAbEiv+_`DRs%gX=-%+osaFhnPm{ z!k?A{VTLK{mD&ZX_*%z17ulR@OJC$zV*>2=hQ&tB=aTsyk?PW02^=T?$%Ds#@*EcB z_fv}M)3x35?9dWPfMw2#CkZe? zVpUwJ@lC+=}rEk_|`C*W;0(~v>@%V{@^c;g#S&8-BViCTqT~9&~` zC*19+1n~EA1Mq3B82b5!6S%^itV%0pi?n|$&CHX^kuv+dFY6zHE$41(zU78j(0_I` zh&eBx=gT=jy~$pl|5;XP+TL6hSlK?oBlCMl55M05{eb;XuAOm{XHdX9A5Zohui7Yz zNI=(M?)8y&ZCvCHcq3Z@I9daoCb0b{i$c-`T`(is!gekIE(f#C?m)+2|2Wfu7~hTh zq%RUt#8@pJ4>jjxfHCdesny~I2ttHfoG-XL4?q|?iIL9J?XpdYL0?#%K_5bko(+u4 zZy|XfQ7IgJpPxD|oIqcY46yu4lnDb-R+nRGnKMOcA)(h>%tw7q4q=&ZbN>m!t>>tn z@&q8u(pLkDuZ0b}n-_l>_l~^D)!fZA&W;~p2oi~5d7SDNMwdEwu#~6L>tgspb%MjbC8%)De3M(m&Ti} zHmWCFx$o+4zdJ5*?1H%S(sDLU`3ZlvJeU+|xF=+6?>n+5W zs<+GUA1m&(a`uIYvHLG9lotm3p;8|rl{Q0Q?-=|x#>eX07_HmxB2`ffp3MJ%6R-~7;I##h!z~yGul;<&>tsDA z-arEh4j%#SNHdn4OkZsE7o7S(Dp{l0Mni#?W+pMHRF`HF_|#Dzy@s73_R)S35$Xaa z|NO^MJmoRFZr4? z?)n0ga2iJLKaWX3OIkMAIU3>nq2*sc;0dO)|3HuY%&*z1rUZKrGt-OH<(^=xc+76Oo+&nbioS2}?vOq%Drt|9Pd}9hofP6R2oyPW zytzF8cdK6Y|6!|ctQp3MTdd#QWk7IDSx#rn7s3a|!Y-|fN?;SoN!q*8dJEw*Iv>3B zq`b3G5XEh{aRzniZ2K?TUsC7stV98uq&U)3sg`Dk>S>rj=fWe(JjxZ$@nJT>)262CmMuc z-+s)sG5HKmq4|ZY5YXyIu4w7C6q z?MQWn3 zO&=nXA1J5%en;6Yw5@xKQciG@vq2@x&t}T?)?m1^dQNvd)1Tq_^7D=5N)lhqDU0AO z`mPHt$JI{Jccj}>69v=#y%YSaBlcPAH6L@!tCwT6Wz#rXXYY-!4P)*j?V(qi~(O!d~R5oQV!NlPeD74Lup5{rj7Z_ zaULqUoC8(qyRM{BN;+9uFkEaJTv6lcu=Vc{Js(N6iBb-aCCfx6L-+!NBBzg*Ke^fr za09uYH<(TNwd*trkrD8EjzHqP7521xVk{fP3Sr=~dGorY*zb;xdOL=Y15l<2sb3A| zu-)zW96nDuMk9~RS)xZaY&5x#3d^A%clFGt! zMHl2K(47ba*y+hO7J(+2{R_AzTQ+#LIe={R{5wN5*gF4`uJS*Y2Soq-@}L6lf3540 zb2g(NIMMKW-nR%-o9Yz!t^l&i2#+X{0yCkH-&(9vrWRSHLjeIt4yT}6r7zT-hfOHY zdF~g{Q3%qf)u{RPqMo<`0-uX>c|A9|#BF}w4>^O9=eUjsxW&VsXQ1Y79GdF!wB_so z7~vCWRB33`ey5a6thKtrXp0y?JUh7qaROT!P`L1%tG}c$hT_Dm(wTQj*KmwTH$i`c z%Zty`L@YU;udtv{2=&jKMd-4O2IHhCn0zz=;`?JIUh9f#g?8#xp{#|oadK=bD$`hJ zv{k{vx3b30C7E$5{bS3Uqgi8l`|nT_Vd@MLwEGNLXOup8ku8n+KZjdwqG1^y%@xr@ z8@$^>O`Lf^*BR)kb<1}2bW1( z*v)8{OIjQXM=k&TsW<2(5qoI_pN)Eq~$N94qie{k$Dct)1ymmT&gh zB!!8<5j|_y8>wcpaF6V&M!_dK`BySc2wK8i%j3(j0KIe)f^D!}gO{Ao$n55?Z+`we zkDjSACx|V8V11tZ7<`i5E9lRm-S`TT5ZqOJ&2c@7gOhZpPi#JB&!c_v?0UwcRQ=!of`D>6P;sD*tcs@}zSjC)& zh`gPQtvC>ey=tS$F6o*4IX%US`PU^cp7x7ZMAUu|UGc))bspUH;G$U>YxLGO zmYn4Y)CPPisH~#;?)%a_v%-}w(v+lK_lmgh2jM+ z-JS_8;(CNzlAxtLA(yr1Y2;HXP9+SM%pVBV&|h9(#Ix2&2*4IFDEIsFl_*@0CTYV+ z<*dpFM^C1-`yVp6V>l7cq}yW-?6dgX2Gm+L6e~uLWpIj(@-!A0AZ#Q78< zT${x3iSa8NxgZ_Ft#Pygvqm%oH!OB}zqD^4u=>nS2gv2IIaQv{04Vp4GyA zPf$WXNyYX>kFDHt`b>y%@8}L}ckgJCNZ-Ve(>V&Fba+ZQedHV@_j*Xcu3@DI09Tm~ zRFSS|9nU%OiuWO%23z?+xhy2hY32JI%ncMlmrvcq>m=zsZnx)Oj*>thR)eqq0$5O9;} zI<5ghJh~7Eur}=f9>jCJd^9ABLE~Sap>ou6g<&9CCbIcKIbB-Fsb4pRlBty`gay4_ zxQw^kter8WXx1E9jfj@A5e{zQXE$(VQyvM1_%dM1XD-8?MA(KFKwbmmRMry*pc_ zynx_0{~q0Zb2@kj8AZK6N%A+(-BFhT@Y^_`W%GHtqaH~F-xB4dDi=D$g}+S}(I(sd z9h8xjdTr3A3`rhX2~1%)l=$2e9ot+5wp`VG|oJm6`!aL-(AZpW~v{^u-& z@xRYP|GSX#QJTUQiIb#Uw+%pQ>11p+0|EgZg%X72L*}9gAt;^lH5Ugp=VJrM8UVr) zL4iT5Uq7C!2!B)YQJmz8LigLQ)M-fqauG;~@jA-^?R>#A&(O!h1l@PPPf$b{+UNuc zDTid5N|4Uz5c+$tRSA*jb?h|^2zeXI2hJ8D%Z|_e_SaY@Pk7b@kTys4^I*dF$gWe& zav7SDIWP*?n$~#bc|Beq*34oUh#?Kl?mvGBWto1FeX*RMC1(&X;ipjLZvB!hNgLe z0(47;H49^bVo4p>@Pf7G2OB7GNF=-S-Y3+yo6pHmh(G8=*F7FE-&f?@zwmw3>3mU( z0$7aH=yglU+SE&-<#rVqZL>UXan&kn~{%| zm*q|l?}j^+TlUHi`bC0E3=)S}ZF|uem$3*oHM2^7ib77HVW8g`t?=Bi(OzwN1xO=q zpen#;`Uv26T`~VB+rxA3|3KmaIEjEB41*mrB&s?I>^%rpA4yOUkczYmm~DdKBm)76 zZ@DN7$w#_ev!2v)7Rd68bQyu5n|hBab73YPrfP_V1X(WD6j)9p;VZ)-dh|N`V}EwM zKPs!FvfJv-3xl1A`C$F}{2&CP`{f2KOv`0?dhGJ{{8-BwdqHNbVYJO-G0c zFYfCvvZowzr{`(|2>AVUSD!T^e^BWQ^3z8UlemYGZz4riDN90t^%yjSSj1&=4jB7# zvcAA^GajuJ6$BlG*c;ul%OtX6_JtCSGXid|%66;f2_}?{H}i}IAda!gd7D13Zz!S0 zo06x&GQa(#MXHr!fu7KvHZp}jYsz% zdtlk00ygUp&6|nG$|ycG^+%qfY!BJ|E=%`c=?ydNcE7yx$?lh@EBDJK?4Z8{q2{4m z+IYH*aEIHGm7v#zWpTTHYn?|S;2Z$yen7xkh(@pG_|g1~`R|W&`>TE8XPXC`{F$IrWsPCNFyNDBhKgHhU z833Ez(ZBmH5(_y%)J-%oukeffC;J6m&B-|E7fd&=kMsG4(1l=6qX6H8Ouhjv#a1bh zLZdbzYgy2HD*+N=TEu`m^;^&^LG=cK1(jMH4~0Xtf<&QfO~)yWAg`P|uaFM4>^3TQ zf%3eD1U8=_`uqu1cX%msZ&1QF0BHvxGP$h^5+UD8xK6QNeyELLGE_hiB6V3H8T4li z7J#U83Efg&_T?jUCkF*|ym<+ycYK#Xq#WBhv5qcQT_3wPnG}Z1c^M<*RfdhotL>Lp z9K}K|9tln~!a;xon1%xg;{bTz2_|L0J%CFJS9eCDKeYfeO>Mk1GgKELhjR}-P4_~V zSE<2C@&ag6d9L_W-h(S*Q>LYLX2tzgUJ2B+=z7Ib46n{vLDvV;0%^MFQqWr$_(((?X$b94QGG@leFtlG@pFJ@ZAi6LaJx`uN2pYM z-sQ;yxF+>MF|KRbggwt>EY5J@Z)y@AISPD)5$w73dhUYyVBTHZ7{_8`{b8IrK%$eV z810q#|NPPOUv>Ek$f~k6(E4KF_9K_abPPWXn=xylH<>;I4Wuq;8QrTU=VN+)D|~L* zdkq;9^j#F>93zXNtN#daPeJrC?+VOFUPxEaZ)-r-o7ZviVAO3*F`q3ZXLTt*w6_JZ zTH1-paF8*)V^Tqlp9hDEwLh z2XTdvF9vWQmA%{?bAW9H^&U+|=QGfV$&s_aCe`e3aC*SDa51pSl~wn#0>NsUU!O2S z<>!CPN}EqMVD?pEWPdNuCY3*ap#vey{@WM1Mz5JYum|8eVs=fz5g9fguO%qt z>hKY$=5){!(DX2R_bKG>>qVJtut(zmhm)v#*?)Vz04rI99n%YLG7Fp9$+85LBmnl?;ErXMye;Acdhkt_VerNNHJp5ngFFwm@x=tfV@m zycyR=Z7saJ>2&;;eyQz-=Yo@N*sedXU*XCN5M#-Y#nD=Pp6to zcSE=!-)cfHc0Zhku{w|oA3moA1=jPd4jNLUlQ*Q4-zY5o$p7Q6oev7Ux$+RFurZ9h zt}vuREf$m?yDB=^z{&=s$w(x?{ogK@^4lM8h~w~<8WIswKc7q%vMV>tLz=RI17s)I zb3FMU1B*~!5PjAip|SLZ@K``9ON z&;P)|=W(8LW*X%K_;M8^)bl22-@?TiU*rSPZOva)n6J;nMMc(t7D})s#xC-!ZsxCR zXYz5&6QM_zr6s2|u-fj+GV0-2BwPaV;_$OX&44pTi+-Kw^}J!T^P#-k^ZAFqu+NzW zebGN^16%x<7&;D4-=~ArBvLt8niRPcxoWSeZ>rmiCnY?Ua48?h9bPJoUccYCb)u8% zW!Z-&owZ0gwEv-V+MUwN?D6%UsNE)_09#xV-^dS+>e;^z534yAwqwGPP7z$tQ~mS zzyA87H8>4gT2wAFyB?7O!k6zazAQGw#~de5o03!Ld|ShU2mO0YEOR{FcZdtcSiRXy z(Wmfne%K<%V*%q>8Or}r*A11ztugvU%xJ$uH51#mJ7pqC#K1cd_2Lqc=D8A5QTvlD zifk|rVS|zKZ~rk8o%8W(xDczCP?GD`@%E`F9)~#+z1B1Co89I|)e^qA;JD8O*B{_S8cQQ&Z`JUD?&R%Wc~ zB;5n#^Qi0obfm}n`xIib#OURhN|R=(>5*x}Y{VM4*?PS>{CajouS6-mE}=XI(#(mV zeY634e5D`|DtdZ!IJ!h~R8*x^PBWfOJ8{(RLB#IA=8a?Dz35as;{cfV)6UalOH9}V z_&EYoKk#rP;jxB5LPGXF5YvG%3n5rtd`NC_8jvXrP?10yCSTvMVSwu(5}^(l^+Ax8 z#b{uIiRhfBM@`B{j07TwBn!potL3=pKTMZhl8}x@s72OTV~qaG-Qg1Z-65q{4v0Fy zg)jYgf?lg3qzDj3uAU#0eZ8Rsd&2o!M}#W1B{#N}?{Z6uH9P3h(@je}fkE?>+}VKV0K#a=$4r z{W;-@+I33g17c}y*j<7m56WI^@NfkEql&z_rKI;5f7x`bN#5b;M_@jw4vlAlWf$Y= zjni0v5DK=qKcOIRm8V-QBBE@&&c*JJwz8T}xqRzgc4u-;3G8^i4>uW4ky`hv^&L_# zq0%+HDF4@6{1e3@3OSGI_*VKmFlRms89wfqvHJS9G~ruV^)4u?G=V>!4t=c9JDTVK z_C_FZNLQ#pX{RAxsPf7egYWN4b5`>is!*oKIsyH_;MiuSiPrJB54_hEP2qg`tygA8 zWgj@hFD0m{O+Vh=-|Sfx;J$6%H57DA<(8{cnKh@V6e_x9^R159y?h2OT~_Z?x&8&K z)Jmv@rKgBc-~^Wd*_^mRA5>URJ{2GRDf%y0GR~Gcj3)O^*lil+2`kEC3mR?%0YJ!9 zjzDS=c%{ki(odt{8)hhm$(Rn1Iza~QvW*-?YecXx6!JUvoTgJv-cs6eB^49HAa1rq zdH)Xxui*+bgMiF)~#H`Gm_B@rw0UV`&AT?!XO*g6ha1}PcJ z0Tqp;ZQ>3X-*XOZQ^mCZ;4p2-qF)OpO`NW_NasNsNsC_kTd#5=<`bAwgVVjgT>yX7 z=ihmEw=}Q8rxLDR;HL7T3n99YM@v@G>Eg(TMfQBBBCICzx+PNIhczqNI{HP4>=kyA zJLd{x%`Mm~&wfw7E|#C30t63|aP(GmkuDyXguMTJlgE;rAHq;tJtb8i%#Shv$wykf zpIm^Nj9G~_bV8!M)QLhf{JWgvo{jcz(Xe(W?xU~AEM5)U_8`P&9y;Y8Xk=_gJMZ3fJf{f)v_e|o?h7`sab;?(-0mgZH6Zx8; zoKyA3EpBbboLt>`so)jZV5!<+p(8robP`+X1FQsC_kkv;oBZEyqdZ0zTAIcxeUq2`gEDMJy@=<43G zo=(<3nf(tNg+vzWA35QEwUcn5P4TNJS>O!uZU!mM$p8zaZ_r_S)4D;wxVsS@v~70h z3ppQAk|5$K-^$oEpA-gIY}hDwEV`6RX@@98;G6X8niQhx-{~$s%9NEM(MHs=K_m4w z{rn;l&x=!Uv*l((l`=)$=Dk<*!#F{xD7Mdg1r0ydZ)HbWnHr!qFa3LZsa?xAGIDBS zG$8457-?m_;lD%DzuYXQ-Y%enBV4{g7NsUfTKExqgbpFU>68){?{c>}? zmu`a)8*~|q4*o8J(o4Xu(1~XQRz?doxm5>}3=V;;$b5jx?hGs@a0xAW8bGQ}-esK^ zoR%M2JLVl>(%C>*2^g3W2^G*grjgxNj-$BofpZwo;$%QQrgP{mY@n%N&akB3TADBx zLs*s!IH;YVJKlQ=2#U%*E5%_>xZ64UhN1Fp?4iPMKl+#UAG03G@H-io3vY0K*=gBy zLJ!T;My0}sO!*zNH_`d3Fr#9&M!Eh%_ei&1AJg%X=}2~CYwl3`#OqW0_DZ}lFpXsGQeQF~ju2ir10RvY8U79~Ug6tDxIs>5-D*Yq`o^;> zx@B~K$h%6?vW_Lv9_u7eIe2RPI)PDP)*ZROTPHQAi~GW z!ytT$k*ova2+2b)xp94iN`k)Q<9t;6{)=c*S6$mKaG?_x0yq@Ec88OPXJln!MF~ku zff!=B%J#=nQ4bPkM8^1YI7Wp>L6z?SHyU^@{0nx^%rK0cwCC&Ur-wb2=^c31I_zD+ z^B-4!K>JF_DvfGs2(k=rceBkx9moiGfI&3hERF;n5}BVg49N;cQT?56@Ehvv;}u+i z_$>3Io5^SZt@-h&49dG2?ja~sw2j)kupqvO-WQgEJRwA3z9QjjwXXAU=YRd!i4waJ zs7eMg`kC(uw$B-OU2Q$7$ZqL;_w)~b6jj;j-`oU?%XT_^@@NY{e0D&g?7O3q&M}k8 z^Zv^!@Qf_TTK7N@=_NatC$@2oRt2s3)tchMmKhScps|8aF#k-0DkyFBdqhQ3ZtJg6 z)P-$38VNu+s*PEkrktJ``@?fkx9nl2UkpEx1uqiZ!Fl7(1Z4|0be zj||95`~_kv{V7C>Y}rp$)J8UKTw|iq;2!X3ruly$v&wAFNSr>W7C+_SwVG)2R~s%N z1`GQ0qs$9vgCzQWi3NOewW8JZeGR|8$U5#AU{gHcktR4C3-c*udo({NAk+ui;~9Pm zz4qZdc`R8%pg~L}pOC|pDkCa;e?q9aQ#e24aXS`Ua(@^j#foCpgF+_WK+Sy3FB0m4 z^aBFY(JzsX6JcVRfyabk21yvwFN^_F{E(XWjxF-YRSqU6hkdzvpGNph)EH;tQlNA~ zI7D1zrV9x06D4lAhVVP9XgSRTB3wm8O+KSBu>-I3ay-sJWm7-Vp6;0tdl5~dLcZVQ zK?EYzlfI!MWYBm6g8)CHK6eA*8hH3UWy$SfH`xL^z@yhu7R@81Tk^12qqeY4M4m}j zYJH>;I_;4VEHmSubQNO?pV*5j+5M(~B?JDm`+ZT?VMx{iQvIHA$iJut|FU!6{ImUZ zQ0(5NK{SyxpZizygL{uy1Tpo3IIp8K=rK+t>85i%>Goo~pt+wFPK3iRC$d6Bi&BofuPqt(k%k-ZTSWyk>8C==-@6s=s2PEcqP(_ zIr>Fu8{WF_NP!H0vu#;poh^AK5xtj%%JxnRN6VrnCwnNVu;uUQ6q(>u+p5sy$LWOa z#2hZ&a*n)04o_M5wIrt3)=BRKR-}caNegNx{B7n?F{f{l;R>>hYky8Ii3J}_@T)dV zJ#dv=J_O#s`PJY;95H{W+glC0qJX!n)8BK}yfo!KS@O{K_^l~Gq%%_romi2S{iWZ9 z8sRL7-YK0kkURgqAIz^xT>JS}l)K1L?P#fk!lpe2aV6e@ksK14(^|7S`t`{Wj{en< z<7DMm7Hv2)3g+1GI1W|;LOlYRhWBZ?#Qs#;Ta-c~qP=2pmMZSs%k^2b7WhyA&PXKW zvfh5;Mt*MRu&Mu`_M@~*(JY4JFiZVI|EB2U$rbYT`kA>K!$5~Un}ag_8dC0FU!a-5 z+1E!RL$KNj@#9;(Q~CI*h6gNwp`%|@$2)A?eR z@nP_I`9^m|@Ht3T)Lk&7=OaZ3K-s1Xw4;3qN}ShcH0w z{4*P-R6CT)`@A;t_Z3 zEZ!#mf>G#1@(B;mL&vz_Uy~9v<6YDKOaBxPAsL6IF5>I(ejY6i&$o4&ih!@b$Grxo z5khfKQ_%#qwg-FSX9cAe9Zwn(^Pz+9{UgDKbd)|$q$jb}X!kF7>*VN9z<;$M43oBS z`7bb+G#$%qZ2})^p{<;|H`{1NlmF$X0a2YNwS0S-odd&{nl{#lUCcT=v2(`t)KqO6 zJr>IPKSj(yIjKZgCWCK-*M)A{#dg0%PiQ>&hWqk< zpy!EBwOAC(+{qNC)JyS)r^U}&*$nO6_z7rD>%RpTY<*^3l|10#G((lQQa{ICSxhQ* zeqE872aNZpI?fx5G0O|TSuL7-?#%`s^eQGtLZRDe*n=c%ydbT5Eok)FfRN+yZ4vlVVLc`X3j|WSQzsasKyNXaq+(Tvg1UmyZS%!%|=Spg3QN_F@)Fsg(hC8ljg3Nl*j@ zX--UZvk>WzZ`4DisINehq3-2Mx17OpgH-_Aq+sU{5`zUn*Z2@WlqEz+q)8)y_d{g+ zvYQ*+tiY%nB1F|Kr>&i#9gLO6Vj3mW(c5f>NuvTk1Hr!#1K|Rox~ry;{Du-rz}eK> zsGcs7+S*8V-3fNr|H-}K&fT%I49{9fa;;1SRS|boGE+eL{h+WwIm07+VuI;f2U7%g zt0%sURyK<>i9p~Q-nJh3eH6<_e$J+lkBAe`^zxiBFvzi(HT};VRvR#MpRFD~^z)lN z*E*YCNN?3XZE8k4&WHM4?I&bO&qtBeD{7Z4dlPV!_GjU2*x@!h%0BriKP{hi1{z%w zhBF;bF?@B^sBk;`+tw{>E^n>b#0rUM-)qU77a(Bh8qlW(xKoQ9^ro8@94tw{x?UPR zbRh|7XIc_WFI=x(4WWYxu}p@@2w8pA^6c50uNJQo@Z!NwbXey95jacgut;`d z5JFx`+PZ1y-GWxs#nAmO+|q!bf0c$+cqvoy1CL^-w$M7)`0e~3L_6c0%|}XQ)*tvz zXUnbux94x!ye$!1jrc_f`+GSQ^NIQRe`W!M>R)P!3rxKR#PvM=n#g+lpE!0p9*X10 z*(Nf^**$Q{vzWc4waWiO<`a7PP3A5+q|BS=)ce;D zI*NhzI~}dgN;5PjLu&Y!rCa3^^M_)~8%TUHXF>S7(K2*s0ZaJF937^yVfo-&%8B)L z>pV;cln+d)Y&<`4)fMuE_)C|s@rSZl5aW~F50m8^Cu036scdrTX6ofJX|#I2D2Y^a zB_7BB$cW4Q?`4O4$I{D9s1qH0SWxsk8_$=OWWsUx*7D;7Fy6@WjhOMy_~#qYXt(s=h1U|lt_A{kcF57L2{9=&$4ZZv9~*}>RQLo7 z1&FQxIaW!64j;obAqBqbC!0K<`@rkC8}S0~lfBie$rgJFJL}WjJKVFZGjn_4Rd@EY z&$7YMXW~I<5?(qVgKuK-715C0(*&i*VzqhO&W^N(zNRWge2L+`lsWdtr*V-RBjW)R z@_1ydo$_VRIrQR?rf1!sLnaeX8}}@Vf{d0;Wv0qA98w1V$jtov8|Sz=OYSL3`0r%d zNo9aX_B<6@Q;GtPhfgIkEKyz2w)s`6k}kiaL-A$0-iBN^ygXzq&H?RIvx{t^!n2N9 zwM0F{>t3Jjh1n~;?ru+LHZaG}%N}}ZY&00Wg(yvq_gnvhZl+zGywp)QqviE;-c8b5 zOL)7RF=n5Q^ryTFZ&WqYJi4N$+z+hU38it7&3vj_d1qwP_Y2K`zf4puoA*DZfVM-} z_`Faq6Gki7zYAS?tckpX-~)R0Mn+sW833H^MzaYvYcJ!605rYEi0 zUwR6Q^iTX$MZ(oh=Ka%L1D%H09*ivN_fsKjNbzNt<}e^bHOoS1ur8=lfY|Y|SAoM# z3jd-e2&&Epu2! zG;U1#kk_!=?eR5JHX+otk&tmNFD_9#wi+wNf*N zWikmX^71y2Xg?>P-{?d$m ziOybQS$ye$!I)8_ga4%U+`IugEjUoNfCQ4*ddwZ%>=MH@{GqwyL|2YQDCd=W{N7jS zBh1kh8CJkI1(`qz=MX##E?$2eWihcr!f($Hn(CNL{Hieg%M_dBCh|Q3^Fa(2_eC!^ zD;!!#V*LW@LW3*P))8{YEO@7dT)Tx_zGc^9Ib8ZuwVQ3rc!Rc;8FEgQT84%t3}e^VnV(R1u7z_(A8oQ({~eaGC~wWj<@_v9Xzr2JKc}p#fn|$FB1kA(FAFU_#Cw*^L8Da zG3ID^(6R;pT$<+cKGa^Q48G(!i>!w+dnzS&xnA>`V~^n>Urw;yT`^?1x>gtuFx{-G zULOY#khWQUpC#*!l3(qFhF&^9HBY{#wP6@jR5r(!QzrUM+$x~YRQb_41J8+$bXs84 z1BFbcVcg{E(=4MzWq!s3GgGRH6zN1xM^aI`7nd8%$y%Bu@)CAo;e1j-W-IKXgn#k` zVUmv53)jr^-^q#wi>3R)ueaHOrz4WpCn-TrX{ia%&A!Pa07pWrf z=>{@BMQc?AJH3E-J>&3aHpmkr9Eo%B~?1|1(FU|kyOQZa5DLXK`!=h)EZcwRSA>-i?X+ls`72Og>|tg5$TXF z=`QK+?k+((r5hLBp|o^^G>h&Klw5R|pmcYihu?nRefHVo8)tv#KgTm1_{ja-SIlc( zb0T5t0s)`Wa0wCLzva_pU}DQFzwJ-Ywe{PqP$2S5plQXL51;X2!3nOY_G-z@74Ex>x) zhq*1$6foRhe?cim>cM&|MzWT_=Am0dYf(8!oE02b#TDl_QMPo1RW$Aw_U0w7)4u29 z9P*x&K^s#AC`iy(wH6CoEX+yPU+3V@DlzIyWpB~xB4PYz4KeB3mngc&nKsiMiJz|b{=G(+Ty8libP*HhS``iRlvZuS%gZ7+G!H^ zB;G2(A+zf5kInW#dhptO|B&?bbwq!-K6w`^6|J6NF4C{-u`04N=wcfhCGT>mC8@_h=nLxNR?rGuIv#TSbHZtyIJ z0UD2&N#%(#^wTjLRt*`h(RjP7)vlx|%QWgdW7MDn9_7}I^Hb-Q0x~0{(&WaZV#k4U zUVD)JVhh~)?!yyX5eWacM3IIL>znWWCPiC6ZU~`_-QYU|2nSJf!FzOBC+cS!4nAz;@`%JZ%BB4+7C(I zcv;Sy7;6xk#fb2`9_Q@iMjP7wF}Ziz=BA>ea$bCMJlfn=hX)7A^mO_AQ?}Xj{Yv?K zE&ZuVq8l>ZXzOP-=B1~(Pggu~@KC~Q6FlSnnLRBsx3}iJJBGY;mD)}yCyLXX=QY~l zLq2#)7=^o=O{)F?92?pCCnK1{!H=tz%Eo&kGuyS%7=oeSuV=m-E zi94PeWOOxg%qy?AkOJI;68v2Jm&BVr3APP;ZB@r>9~IxYazLKuI^@lXoBG>62Sj~i z&;iACv)$>N!2?(xC&6XL3I>2odLMO9U9DnTmB!UnkP}llH2?td)@a4zcGXVjc((9J z4i|&gBtXZ#Ar~Y}s#M9cndMJm?!1*NQ>o8Gx89aZsWK7kRg@?F{+$d{!rbT9fj1kK zg*5b2A%LCM#-WdAl^b;gT0F95{vA$Nd>Tnp9*8Bu0u>t+zS1bQ!LflAuo=$(wW;w) zr1a|VxJB?gpYfT>^RwOMw?i*e3md~@a(1at_7R%Iu-Z;xt7@Z`&^{`@vqhXv%Qo^j zoRjTTxJxVY=MQ;&uuNWi0fJg&>P=Oj6i&5>^V91}RjlXLh_9QqwQiIPKjpFrf0L;}_~5E*6qE3hT$vPeq_OV9SrV7ji)mjH`qG7NlqmzDai? z7^6xT;m>jkw1_sA2H^V;51Jo;UnG^9^zOhq2m>?Loxaex}t^p~3V zV`(nbZX7cr&zaDKs?ugP^7gf#-hQDr2zA^ItKUz~vGWvx%S^0pbq!J$LIAW_e%9AA zx0~&8+ho$Ee+IvP1JukUTRjenG8a8$z(!}u+NwvQa3vEqaIaRkb-VqMPC6&4G%M3#-#F^j`CM=NWYuwT=j(^>p5uOBcJk*! zIBdC%G?>;{7jaO)dZ4e-0RgSDKUtbkhMx};@1Y|T@2~O5$4%t^Dq6$&iaDVminWXlU-_x$h_5B3DJt}>aPtEkrX9`6)$$JyIMZ$0g4?Ha!8k^r4dk{l%&RAjG z9vj;E@G7awpnFGI9dG7KoQU`nqcJqmi0Gb(Nw+%u_p2Y{ng@E5f3&&sdnBWN61dCb zF&72+-v#MGZ6)1U%2+NZ2qB?JZE%Q}lXs3Wg>frh#zpf+lZ7s1kju49LF>bP-l@xg zgVIUVSyiE|zY#)@-oNqANB7J;ZT^1Rz1fCyojhnfMhWr8;Pb!dS@s>d;0$2?nl}A? zGG28ILxwc}>bpoM`;UaLSt?q{f zIP#t^Pg}N5V?|fenZM0A7CgpX6a@erl63ZNp6d`xI!B=k`CC91qGWRVz-l$Co%6Vv zd}@8+Nfp=VJ?cOnJjzbip$E*N-}{;&e|wj^Lc-y^q%DUR`gV(#PCf@&Y5e-d%iGf- zmZ8IMfHHVQsH`nzC`PJ)SM7c+9{{(P0W$foeY~A#fl)xTGCCxgLz%j4wk=*l1R}ye zoS^(on=}WQxTsRBo}TCr+$gL;k90L>c=&q?l8oS(Y#S6b4tfMMI<#I=_p*-z0~YI( z>=Rcf7DkAKTGXo9)z1vCy2#h0PY(@2{iud?>R2dAKYLpFaf?8b&M*h+t>4$aAHILd ziz>}g?=LSJVerxgQ!mPC{~A@7MNxa~nctaPm(c5_THU=tq?N85p4#|vMym|5YcpEJ z@V+c+R(n)9>WPr!>gRFPZO^9es7@S1b^xdI^pg%*9+?I@<2+ArX2iZv^2CnP<-2bf zw0g3bL6!54B9n&X4tz9=oVif8^O8lrZi(Z)k$vGv2>CnEXH;km09ncYcZYTtOPE%c zwqpZ2tsHz1+%XIb7GaKz$A<&Qvej$M-SMpt^cn@#kwjv~KjAQ?Kc7?$PZlfWYa5@Qkdw>1%cK}7>37>Qo-ptC^^n_F> zvkkgh(m9#sjJVChox0>LP_wu6DjJg6J^g<*@5~#20SyN=-5aq=>l93=@2G=bH-=V| zgO4(A+0eM{?F-qfk#7uEqsl(P5|hpuw(m#fiG4J&dQc~2o4f> zJL`;@i4dqp$_!sYou%4+lA?j(j;71A zpb;qfY+S*Zu2z}GEj>@-OFgzR!bHR|_X^str&w=~R=uXG=grTrtBXHIZWhD|<+BJ~ zCo;FRVF|5kDy>>X`4W(Oo>@-k?CS2(EJlfW}xh|h02_4dSfcg57Pej+=Y?#H>S ze78!s^J4P~^;MSO=E%qFterFL`dwJ)g3)Axl<*a3f z2xo^GL`q$V{Wrw#ml(&b|IjBEV^p=tXqLGYh|iRt?;vh$r***5qDj{&-aD`-F@K*~ zIH{LkpewrWwZ;&Z9pZUqRU8#L?m3yI-p5wrY!Lmm&T>u-UJzofJnPcq^-Bq@)#>qK z)|m6tZwUf!yVU)XvV?}M=m_9c+I+~obvk~@c3kb=+50yx`zZ5Kz>E$rSgP#%1FXc~ zETwNm{)sQCB|wdok-#rfNI&?2rhn%_Y7Vkvkj^+L=taczqBnJ?A4ss`@}f7-HEp(f zudJRibTjn0@?%pK*=V3VSRZTka%VdiY_w{NIC5uBh$SPi{55Vu^GqegWupXl_8Vnf zqz)XLGub08fabGI%4v_SGPKOQSmd#DovKT&7`(8Q$TlC%kiGZ5`vspjSz;z@+)lBH zCeh+|vFn@_>z$Z?G@9F=dB4|`t5nzCLNprXzbZPpVmzd3rd#rZ3CB0#!%s9O6wHH^ z6WEv(Fc}Gq^!ZE{z3QKu%r5o)?@kHM|qKRd)UN*zSgYSSYY%}KDbjKv63 zNKM+##*^M_#y5Ps;cGYB5KX^5v{~o?S`Qyx3>$B8-poH)HCl_L#`YG0oP>?V6iFqi zvm@dl9YsG!E_+?K@G@f}8a|cfi z0VYcvav1OwYuP08=Qem@>}zMSc36i9JPK~XK$x(irj6!TB{T|HERm3cBWgfIk^c?q zbjO+iIqdZBm~?jR&1a$YWky27`CxD1D^5VbbMF3l8rwVf3B61spFZYxO_3bdaq9{4 z8|#||)CX_y*1%;ygK==n22`(lP_~neJY)%gt6_E8SF>mCOb zaYGt>f3e3tF34RF8Zg>sqx$BNDrR490M&EXQQX^3)m7=N_+`y({5fTAc$i-Z?nvfE z6}lbZ-t^lIRyKa|-Yw~3RK;TNFO%z8X5)BoJE;rdk+|lqtR=6NOD0fsq)NE=|9!5^ zz7q3L)_s&OM*P1UI zxfO&MK^JkJA3gKcn5050IJlnKH8wfvHFh1B3%1X+PrVjbYXweVY|l21IujACkVDMx z9Zenk5Z=%6Qmf5X)_XT;U861W){QS*q}K|6h~JH3p&W!|6elp5BA5I`z(M|REdVAz zJ@5j_3Xb#1(#`oGETML2<8y*|sVncro{JLo_sC;}7tY6bDnN`_Y4_P_VOT&9uhREf zR(!wW7DZu~5qPUf4^SIawNi}6gBJ`JRWYpg7Yk0XcJ6+u^DLBcqWXUVYnJAQOa4iG znxumDpB<*uJ>3pep*y+tGA);^{J@iq-J>y_NB9G9T$zeZjao#BZAQ2I3y< zp+ziYh3|K{&O@&XtkDXeV9{%V)VW%ria z>1o#|Gcu!0=?G6)~^z!f#x02wmlh z87PhLi!U%FK4&D87pqWR1`Z

bYR~QoBBrfa^W(p#A*lnKTB)f7>uR^;}t<vhL zc|DI!0BOMN;mwGUWZ=Yby}NJ8B&3$Ziicc359=zy{OqrI(asPqy3k(JPe{GAz(nar zdxV)WrL;O>QK+N6d-Sc?Yh#y@9BeR?r`qj(*9m2O`xMK5!g+X6!|AmIFiAe1R3mdL z%rhG1G76r%lF|zJ)WpRN5|A4Z+jXzQLVlt zSz7D;>ZmPX)2dG&Mw;JB(j3RPIwg4#1YLxFDBwlY_QryI+P}KNn=Q69B2v0ZWmS{( zx<&r#l38WPFs>ITyd1aSepqq}5(35~7kE;WER+7PV5d>uf|uxA4l^3T6LCrn1^ej0@6$DqTyHsNt#HAAeeO6f!y=F7?yjJCX?A zY<^{1k8`8Az|g6)K=pgLn4Kz=$D=oBa@y5SmE-!?=wgqNyDH$bO^TArs_)iaiBf+t z!_^_gNwIGXptozC+Pr2qBd zSmU`;&=EQiTj$~(xJ#O{lra7qibvvGxk_UB>Kn%O`CdB>XSwr){9UGLY;Em!qx|{J zz9+%mgVwmUx-|{z&b%yPqh(Z=HDwMm6mZmtLT#gQJzB}sYq0ZL;I{7u2vZi}pGA5@ z9c!F`QQ$uxUbJ(5vHw5MORvJ zge$r`TOFH1aPRq34*j=?e~x zw2dgsfu@{Vj0I_M8`9dngKeB&9R*Chiy@qVB~WI>P^f$_TgN{-zrQpjg&7%SEb#2E>IQEWq& z>2JS)1U+B$m^ssscp)LRxBdhRWkTeo-vYbY*TzK#3bP^yxyt2y@ZN3A_EkEH_Kk@= zb?p#hd0lj>sezq}9@wSX81q#+?W31Z-w>zyZ*7f-CB5_Q_(zQrt94FF*qgI-J7O05 zZgn2CN&Bm-oJxL;?bi8pLX`>@OCl%+WHoum_H(`;IMMkuz$j@boy(k^C+hz9ck8N-;o4AvlDcbmtDd>^yrATjd~szls^6mSsWbuw?2<&TK-Abi@7?z|r$MS(1Pgp< zhW;lvEjsDhvsK0{whRKd+xZ%8o+q4F1AHbJJUGtW-plDI{_4;Znj$=Ei zdrZD>k~r=>ugJbqZN-|fg;CzaVNsL?jYPE0@=+Gga3&rZ>R(IVwPQ!|0*x|8Bq+bp z&;_o-sNoB44~ zEEHqxbs^fSe;l)oC-TuKZ}4mngKSz-U^&~>;Lq{%LqfBXndW-yDZ;9sdY^6GMujAatl~< z7))bGvFxQoVgK71ur+z+hlnvV*m8L3Fm-XLzk|;rkev)(C=nf-(9ZT}ZvEK7yAJ0; z34-x5zlsj~W+CE~1QbFaV`tI_CWS6A7I5@dI?lLUjUG@oCyofCnp8LeWtCWWoo3AToAFi+=p63?{99WQxPg2@(lte_j-(OLEt+3?{`9L`*1P*jYK#)Ej)cBpqQZte1=$XkOFl-o^60;tr&h_raL**kI1B zD&480j)85mDrH!nIK%EmA7vuHBgsj(>Ps@6%;aJkl7gjs_E$cok~zdeuBov!M3X<| zmn%sf#;~NZ{uJ2pVGd7Pj9r@q0QC!eZ=6UkKEd6szQBbuw)wC0klWV;BE(gDc2fl= z>&gk%po*QO5v#6h%M#Y3z_N!>mZLbY6e(0fr9TW>IqNKO1%T$#r;eG=V+)wWyw1#c zLtwq06A6N=F=hf`zhd7o{rB`zaEqF`c5^432ygCHfqkX-f|qkX($qCljW`GR2tGI!IhoqG&XeJi zdSx^?E-M{)B90?w`$oUQrd`8VqtR?Hd&rz!F{wR}oF`k$U5Qiwerf;gijc?SaY^Lu zux64j>_NiyA<+I>o~22mz&3uw^S}vMb}>s6S;P-Wyp3HIS@a;uH)yU^JNkaAb~m)w zqA0i5H=M?Stj<%@5hysy45n7jQg|lO0P5lDfSuHY{RS{`Uvd-e6R)<|mCOInl2VfD z|K5v~)u*LDQ^L5$>l>7jUovVv-Q(3+|IPr*$?$ZmS`ARWViK0l{gnU`#gRxvs`${@eMwC8z)JoohUDDSKO;#mJeB`U>b~yQ>2Z z>i$Dcug3H3yW`{#H|8>F!8-x5cXBDN$X^Sem~s;y5GMbk5|;Ys%a_OwF_gYw(do2a z_P%vn7=!wbvp2K~BSRW2Jp|exmuS<4{vgr=9gUkQs6t{#4I1MSQ?AlXp;@6RuJaZ; zs!W?UhhCxEJ*k3-R_yh3&j2>-Tg^ zD`1>IW6xQzI(a^XCj85Fso}|1nQL!KNo+I8w0vwUle>;h6`1g`w-nzUK;6!8>9z8j z(5+H0BkCuM6w~BTQU!V!tjE~B+|7q@*;{<`V+V2xl72ofF9GLCc(?c_ohK-N?3z}a zkEPws1&7{fPMKZKg23Om53b@het|EPihMu~B(n{r_kn{(A)1`&Rl4nyFzK+TJB%=(Jc+Ufg>U+b@3# zmW5!sipKO#HSMw)S!F7Qh^pj@Jy=!Izn3LrL1cmZr`$2dOnk0)NRvM4feFIvNITQT zz6#wc%<#x>{$xK%V;Jp=8BXV7F!Y|rnJT?QC0f>$D)OXfktzD*|9+j}c00!7Bc8Ov z*WvDQ<5X(hjEIJXt!(M+fMZ!;K`B8E#nZ16FWW!4!ES+ZFZIu8X4D%}A+gz2XG%jC zbv%HD!LaeSU12BLR$$G{^f$6zt20h}FTdiW;2Bz$m{*2*+c?}Ct-gK`&_`?sw0*N^ zwXI5UwNYG8q#wz$2zzKZi-s%4AduG7y4k0BozRQ8SBwn_?LIqcC53HU8s|_(3nMW~ zV4BX3!1Cz;C6YJ$V-jXYVpQwT$i%x9dycFKMWSZstz;nyBmY>V){89ul0ByP%~A7r z3-1o5xOkXMjZG6odavBN`^C@q3lyQ2N1vjZz4OehcPE-69YoTz0pAhDt0fn-XYT+^ z*|fiJbP-2sfvV^1@aBclA+79T%C68xc;@4$L&5vZ{`LV(D;m;*laad1J{DC^MDC_>& z{>O9X;C9PJLdR{uGT(oyYgDxMc-8BCK)G|G=z@^e7;^8rUYH^yvA`ay^6mV@6F_CK zmmi&!W5IBAMQ<|RS1yaJ`kmlZ=+}q-Y8lpurCejSoBC`sBbb5=WfX~IQ)Q88v4{z? zl*I4?W~+HOb&rKlf0^RTo5bKtXS}v1-lZ7Q+qQ`!`2Y?sg`}M~u|>*v5xa&O*_C@M zF@>Y+&@CJzVY`vR>8Z7eP_d4keHh2B(OsB;v17IBYW*fZZiZCT#jZeaRvKak#;^B> z?Qv8T<)Rx>Q-6vo2UE>Zg0^am&`zhO^OfGM{IZPtv1JU%`7QZkVVTn>ViwTlAq8YU zqC5=YlL&dr{H*0$LW|*j^-2|@qpV2U;V#SHP9)3g_xCjcn(;4JIYOnhOIMdf$)pX_ z{f`8ax-o9|poG%XjJ?hE)_UK)4+Iy>+jXM2!fW;Z}6n>eF=rdB8C(}Kx zzjL9W+iS$u*Zwbn)EoiF%(qEtphho_Tb@_L94bHGn;k=0lGz9arkyeTU0T`Lkn8rl zos)>~J7e(tH&+K(ZTxx^T;-pFZITso?0WJ_*@Y-xe{0&Ep|Mr@aFCcCGt$xZ5DWv4e`kR)*rkSdkB>jL~z~mx!}FL2jO~y>>W|1<2wZG z37PbL!j+shO(RU3O9S>?Nk~IvN~YlSSlKXV7(txA;g4!y;s$_ zU0R%#H*4e*Gnm4Xjms=isMLy}#~S_roNWQ*&l(vTJ)PaaRT|v&J892q?Y+L$NjjWD zH#!T{KX6x*R}4?{Bc>wvk=aC-@24=*(Z$v)J{B>iP!QNzL{l)a8b^D-FzF|dcy%eu zTu|uq)4~7R)E!ouw2GEWQ#~PLbOS)T+ zH@h$B4yLGKDp4*ntpZPEZdF8<%eI*V&*hDN$Elui@L;6nQ1d0Yh+B>a{}$V~BjoST zdU8(mWGOFQrAS2Qa&#+XBCZyb4N>pCVq&QE5JvZVj!pElx!kFCrqX+d@$AVoPhh%6R?*Mx&?ykv=OdQZG+a(g_}A z63dKM);?T@Q`PAMVxkgnarCL8aVmzDjaJH;y z0dfI8Q)Vz!Us`WlvehtQ4uyWGcH&b~sp=2%z}O?(ubcekFEz$oZ=0oM+rKEWN$dV< zH^4pnu~oNWy?>=IS*%!?$ys1YBfXl%2r7;PFr3S?eKevQPy0VW?X3D2UY(ja&`}~eOy3wgzFDU z{Mg*tUM88&#aZsdq--j0&1-7k#rys1AV%UMG0AB>OZ^Lf@Ee}1Q zfc&g9z-PB6z0~w?9^DdsqGkGqIcgpT4cLSOoQ5*8@&BX5*X)E3;w+8zGHcqjA z#gn||?X1w|(&=~8rm;CaEIKS5*Fd?cm$U%2rW9I4ixTmce8xEh@gxMmwndfyrJ0ua zpPT7qks+N2Yke04^tl$Z_c3=a9DYK!hzH9lGHnHzq%^^dL&>O?l4L)XGS_e64#~Gm z`{=4lkmJgb2Fj_kd&U*>g1EiP6XQhK+>}Rv!BML938lH=co*KZa)2yrxWKPbd*tT z^wsj--hHJ}@0yUBK>+A~3$+04B@+3%`)$$TeBDt-e7s258{*$?+-JwD7>^h4GD)5p zo=w;e=G`G`qW=Z#dHp|wJ@4d`VLc%zfMfhzDz`(ilSTPSQm%g8#}R68p9M9&CVn)a zJWa){UQ?rzhN;Wy%||t`6feeF-6aIJQ=WXIm4^HtW(v7EO%H5WcW{RuWY)r|(xM-5 zXM_qmq1~LFDAk&ustxdY?F^pD4H!(NMVvqHQUklD@$j39&5GHENI=ahFEf=uDXrG* z{O8}TRPiSU6mIO?|9zlK)YAX$E&!MaE@kgb8o@suluA9fI@a^TA$7})`xFL`=P^2y zace?y`LdYHB~I?NKH7M!ptRL0B;R#YCQqV2*w7mI@63h3VN=%N$e#hpIL6QFW4jX8a3FZ(K;U5VGk7OmiF*& z-@k${vB`SS$oG}yD|T9XYwb>zByfM+>$>a)kRI?1u_NB{OX_g+JWm0pB}=vAQ&9z|qW?y-N><2psmz1+e?&6=J9q*05kvluTa4$OrzmVu z{YQmoQXxl7Q^gpC2}X|F-i#{X{%-wuI1O-wU`l8G0MZ0@Gbzk4WZb|)59fg1j|&Yz zexaP+tHtT#27X8#z$L}c6$Q&sT}b^4Lwow4uRS2?mjft_sR4zCS>6ZWRU#a!m2|*B zTfYKm-?ThE-th(=BLNN2H-tAtGH62AIWb6lt)bm-9S_!f!;txJpf=NyfNiT&|6U_b zKkmQZBjNw`Jpk*(ybryh;{XrP^uN9b3HW)EpJL#Q%w*iLWbgWLd$9v(>f`_#m(A?o zs*xlJUqXHX6&SHb|CxPw4g!__Pr%At zcc*D^s^mAi=)jE}L%|n?-kBGth7g%2XLvsuQ~5kd|Y z8z=YY>q=GgrGgPJ&%FySs6jrl3i-i(nV{_00X{-mg2Rz#TA28o*G#LN-K_`F}tJsNJm#30( z?#b%*MUikr+O>(`TANkdyiB7Y2-oY!h0Yn35h+!{)68W3GJovU)5mqyjETk;rjp21?AAU|$u`jjIq5 z4&DG&Um!J_fuyi#{%}(of)`xc!lnq}>^owA$d9rN?FGsf*SBt3_LJ6`TgfQo9tPht zMD!A9Dm5Y*NE2U!IA+hrh7uCQ-6p4M-A^FZ}19^?|I{!y_->Z;&&Z)>tBQntVX!G|QPQ zkX*wC-erOAvs-eR9_WDaz;n?qs%t*>Mc&v#gS|WtBC#iH=YH*w6~OX$_bVg&T+T=V zX7zI!&ETCXR?ZIJpX6mxWBSga1CeK`FbtDQ zqZ0`_1O95B88Ro{Dk6!2mXpxb}@HDH8oOUR?oqnZat|6Rad=2Sa zJfIdaU+NesVX6b`Nge1eGDnF-lYPDzy#=hl63qXOe*4uNw(KEvy$0u1qo_16$OI}5 zd?D6RBY11*vFt|wquc>O+63?>ovJkCpKmKr%H-Q=s0ZG44k2I=Sc{KQO)VMQKt?{l zV&Sm<3*AX{VifHc_({Cq8YObCVwtV!wB;0S^+1Y2renB} zg6Eovj#>+I@%ax8cLUy*7&2E#j5^hyNqmn*^=dyN1CcZi&{L8bh$Fo^=!rlD4QFEPLK^mdEr$nE+r+T4npOB&DFO$(GTxi zLZ4?eP>d1JOQ+SICr{((5FldLz)&pH(5^uyFZC6Ru<6u-UmgFhN4_801~eG0foWhO zm#tQrTH&kbO%T`pIW~X|Mt88(tnUBz*#-iTqmXfT?wjfY+Cr7f{rOUhk#tkUvx29) zBjE_1Uuhi03M0S1(!>pCBBTZJP62gtp39hEN_-riox-}PV6)3s(b<@=@ordLQz;EK zHRhnPQ#br6I$S6DK!R|9|F|?@D!z>Q>oe#5BnIkYqhqmT6s{=a0gxp_3jiC3edZG5 zG7Y*iVCL}?nDZ-t*Q)s9n#OKk0(e9^Pd@W+0DWgAUs~f10l-yN%QuL5=CoT8zNH#I z(B?-Z4iv!N2fw_FnqxL80IGmx3-UE3`)e@X!V5+S+ut-w@7Eni%dIKS zn?K3-q@S6JpHG0!qmUFrzVK@Q^8wJw77(#k>Ni%-!6kR$`@LqY>3@6w*-^&_X_CLy zZBGi%*tZ5sI4)+zun>F?Tyy13e7F)k#2NX?$x2o(?8gle{_mRKsjb&QL{5p!hUtJB z7u62nTZwtxr&?*i3(CSz>2d~2Ts6m#S2zW)s|}i+-{p3$v9R`^PdE)Q_(H0>@O46C zwN0W=B0s@1)=1&y9e*}=-oU}{C1FME-yYiRXQ<8)^l5cLD*ReT<)^2MnkZ`H;fY6!`Xi=Y~fT*1; z(yPN75lRJ4c#T^mW$^C(deVl1a@j$R#KT4>Q-rK$!f+aa@@JGNioDrO619_IxBg$LHDNcx)m*;aPI zt5TP(Opl2_DaOou)-go9YBC7u*zMjIc_=JxDn{qoewVuBN+Ml?K&lY~tx1wb%>^NK zMfn4q`ru%_cJKNEQ4+|EvW0fdH$T!fXgOE5G8jU*IG!cW672N-Klf z&N6#%#Jslx+7dClCGWk8U@I1M8q!bFipoUrZs{8B6sWVU9QCxJ{}lmjOQ-$$A^q=NgfRH90mQvIw9 z29Jg=tt@a7Tp$_^3||Z;0`5-`1k~T9RkjXHW(ileION-5K`*jMY?gGZd-|=h6bJj`Z<~hc(r4|nwDnmHO1=< zoT&Jq`$XzI7v(zXb67U^kMEA$Ui{fW&Z}H^4wqAVLdV z3@DBcoOrZ$v^aVIzU8%%Nn0K%>m!@dxC6R2Ox(~4aX)X>>GOFY5h96>304@dk>zlz z==I}T)8-d>1LE_v!)zd|{qlIRaRCIYl=>gX%T2!{8MXoGH9Ypy##MhTJCVNMK7sWI z6ThzkZN-oG6t6JBC4!6~0w9;_f>6!6H|f8iOR(#tPB|ql1#2@0cc~HFmp=Y@4RMbT z`04Gj`5kI`Me=|XUq|iXqKqptb9Ht_yTP2Z?KZ z3`vM+B#cLtYnZ0%KvZWu7elys8APNiN*0ff49-L7!wW}@=O+_Hj{)}-n+?PQVjXzC z&o47VV!IFx2?_l0#wmd;?_IhU zfEp4~pfuuLesCW4B2QP;{c2}Y+FwI9fJud9kqZWrVajc6`$*S zG$9P>Z!}83*gHD!EhvzuOa<%xE!QpKmCiNv)Xf)R{IUj=Odc8AC+0{*U#2O`mTN4c zIkc(ai5ZGuQaR1>3ljYjK&W|JXPiGzFP|fcrD)6W7ro0>M&U?27?t9~O2Cs}L~M^d zCxr_7eDKDkvTzbDmBMmx8(;v;4u1fw+{gdKS5OpB#G!CSueqGJ%>FecD8cAF5NQP^OPrWS~BnNG;hmj`B&`H$pSkXw_lKnqp_d?TE|5Jr}g9rd8+1b9sh_psg|dL(H^=NRi?VErj6oSk?FP7TyS{smQ4Je*rB z5vNET@3e3QS}Zl z5~Ct4Vx<^-rRtnI6TVF&(M$gOwK^R}Qi#lZ6Q8SP5BVvwb9DQ#lb6vDEIU9BQC$y0 zNiqJJ?ad}R#T&%k-gAB;&-H13I@7WZ`^IJJQ^40D4A9`9*}1*g8`9~5ce(+VxFp|T z!XLob+yTG4!RubAI67~*n0Vr*$EqLAHRT|hDum3BEo+=~n?=D4UN)Hay+T^By#$GA zTV6JE%QnSZ>22-n$WOsmToRt-J(xli6t5=<*o-jJLQW!S#UzeCw3+&n$)RnWA2hBe zTM0eqG9)gO=oEin7N?k#$RqfE23Bz@;(i8-N>;67LC^INJTROu8rc9OlaB51Q|bne z8Qdx<5ny9{g!V?_Y27CrF2Nd~lBdX}kCuXrE$i41{>|~r4P=b?E<)~O!GY^u5A;pm z{&#sd=S_HKs*fUmz)fyl3-727lO&@A=4704_QybnRFuC~z414N)K_ki+g?^IIt@(F zLHcem@e46;6fR5h=AJ1sr%0Yuny)w9=sPPz7>}~8O|+pK(6O-22S03CL zat;J`q>sem-0{l51{L~SB>Z@dOPK&96I9W7^KIb_(ZE1E1YQ(1EZZ}3OgC7LD8+yu zsE>B0z4zf|AMr^b^m@n{VUs{>gUU*E8YliVd>!PU zd5ZlyVt75;if%0vZvMl?%m1jdQqQ_0XYEV@mMQ~a>Q^MPP@v_ z+q|1*eq0h@DY>$sa(GqXUi`2EV00Avh z8fBFp$lVx4Lxs*mWK~M7V`(sTlizB9_IPac^ieh!ZjMhA52&tu1^q@e&nK}5qra7Y z|7aGU_9&Wn-qye~+oDVR2dL)rA6~kBdaG;${Sp#yc3aT!{X&E5c951*2w#&hx!!gm zOP{a6Kha&#w9xOw$7;~hc;fHHH^W}(8>1+n0wG|sZGJVtz^oqavAOTLQJ=_9<*eN( zo+7^C%YO3XLSbr&VfjUJ3;RI40pSh$ohxWzd#Ac6f1hCkNqjg|@A3{UpU1T|qX*)i zYx%J$RXjH6OrzmS&}AC6*dR?^2I$_{{Go=k!i@;OR7oL1lu?2oTlok^Rkdg~)7=mj zG+UiwAN;>3{)_7Ui^&lS7cwac;50fe_=pyoWNRP@-eon?3kXmO8O?O{50~ih>=QgU z2Rl=U3PZgboC?c5=WGgyy^834h((9w=|4#<--E&>#6D6KaheWN;_eMVB_FzCB(K;C zHH4eYY-o!fb6?9~D@aQQrwg)^fWmcpHFSyQ+Fnl9d@@;}<=*My+>?0|O>Rd9QN&`j zCPn`8wy}KW0FQrQ*GI+Mt|CU{Y~}uePPjQA`Arl3r`C>5my>k6qk)_OTA6J>Z|0Ms`#^z^&MBObK53PM|OvI;SJk)H-qE&t)^`* zeXv|TA6MyF#rr`wR)jBq8pI7{KP0~uj+FCLGK~G=Q%Kn5tZlK2|9>1ozWyOzsGiaS zHGTt_Km}GUNAQn^jRzn^_X!XU!DheY5Ca1{#8+S<9cQ4U!U#<7jx=lj24tdNwZb(p zK)mY5|6=T|qpJMct>H~0-Q6YKAkC(wK^g=JflUh1DIuLoNr!?6n=a`T1q3!A9fEX8 zrzqdT-}5}@ocE0JzTfa4$H0Mm-}hSAn%A83noFMVoe+JsA&EEL4LDD1bJ_O_VxH7@ z-2Ms3%rk2Bb}=!+UdxgmaYe0mrHCd}OtTtpL}Yx^LB@qyaa&0@gA!{rrQDlk$>6a6 z%AK$)3uat}_m13~=u^EGO|%Bey|}ZVTIY|*U*b&B_I-N}548hZg8&4Fqt}u-5i?m! zv;(2#go5Y=NYP`ECSk)91bCpT`GMMDF8<9tC}7RB*U2Dmtl%E^IrHFMXFV8V>ha-j zdhrqMk9Y$l3%eSGgoRjjFe#bJ0bnuVHj&##Z-^`h6641VEqy&tK%xt(+Y>os(IV5D52*cEqUKxQ zJ3F1DLx<_1J&~<8_eJr2k3Nw{EPwX23NS_uBeiL|M((@&XG|ge{1l(n`0*6Ae*jasASQWHCg@rR zkkn5zM&cu?U4{n;ps183CqKIC4;d9Ab&h^~(l+cH?w$GMS`p3Lqg4#{5h}=e7JlZ! zw#>{oG-6hN@dN%yj&GWV&sp){a@y#%YC_^Gmd3V0dFxQ*qR`V*mFb6Fo~pg#Mw;B& ze+q**r}ca)6S@f>mK3~mn$Z-Sg5)&9aheUqLxnMfn#H!q2lF}}`Tu->KhoWngya5) z{bq5-v$r9;gj^ibRs3_X_iioKEkUcQVz$f(R6L*fl~^zl%~m~=#C|7Ru%kE*7fS$| zO5H1Z;sx59M^WVC$Y@v}`Ko7Jh#yF4457r>y$v6KAsf{Q!MW~; zePp97im5IkWg!Udi)Yr>s0W; z{b?eLNOfx8ue@d|^!`eXL+?^;Js3fCVyw#H)wb@ylt#Hj&pd_i8>{7sdRQNutf~LnAGxr4++S0AVq8zh}YSbQBI53?jw+xZ5xhBP$svATJi(9pOZkN(m}#Z+{=ilPMieoQYq*L02>$e zFxz(MJu|9dg(+)Xq_w${25Wz+H9W(+8;$%+{ZYYD)P@yp1L)WEKr_#0f7cKA z@tt>BSbWSS4>(viD~`Visu-{r>QPs6*v{}MM519 z;%d*LRlMW{0@o$ZgXemO`JUX`%<;l=+M~ct4egyv)R=2~W`uADU9?PFGS4ODRpnQ9 zC+j_Mryyk5p);`sOCVdH>qD4|j<{c_w{Fv`6?!zz(Ie=T_otcXlL->$lzTOikF(`} z(CgG=4^;;_>B3S>J0`br|69>%I3)2h(||z4qtQB)s?DJheee zcAklc5GI3qlY@hF^Sj41Z>Ci7d#6?jeYu}EG147;)d<6X8aa6O5qooPYi5}~h(x^f z+l@$cJg)zvB;ink=k0cso(*}&a!(wZTh4EU)TBGT3-*@syE9Bbl+8ilL~8pIyQ+dGF@{9N6b6NqaoiYjhMjm5OY zP}CUfjoSg-$30yoAYrDHCr}F6A##0|R6zt+#*=KlG2DCyskLQB7E=}Oze;`F*oSMR zAjNm)0O$+ywFm#c7Pjny%mx+!YLBA@={4@!-oTV6D>{)24RohM(3VKAG?IgwI; zL)1LN1>BMkLv)Zeg+FjXz6^_wA$+UDbf>CTwkWRnHzx>NTZj(eo|hB4=V|UQ8SNB! zFfN*1GxWp|ybJt-^79E`nFftM_Uh#@VjF%AfAe<}zt`H&Hwzz8?5LbfR(0bSoWwtJS2D3hXJcY8J z*7COIWbrN!D1t=s;pRwgP~4}BgYQo)bVh|yIbI38-_ibOCB{Zf$(qYLek+y3mpX)) zyMH3B{i6@?z#6`#+Bt#2Iq)64R4TAyjw845?Iwb?)a_;3jpk_*vuPv`Bl9Ovw04H+ z_g=#WT=*@m>M)-Q12kfHPZ%Fbq-7DtKBD877|DvDV30NS)lz-Fg~@k+w5qUjZQ_fY zz;r+8C3i~sj225L(o~ygl7Md@>!)Qv;+=FKNA{X>dISWfX@LupG7|Qz3)4jJ<}w*0 z1C0Sa4DRpzp6SycF44@smcn605J(H|U zGTz6fb+2;-WJV+pr2f)tfSRbyggnG1Z~)U;58W}eC~$mm>jBM=Kolz2V%Y&?iA>X>j!jugJsaj&y|}o6xFH9e#)D27vHm&R;WS0_;Xl z883c(iu*iXI)KZ6<%5AZBzMVc z-l7BY<9VQDp(KO=>4AHO2Pp31YGSWz$^f`K5c!@DvTa!rfRkQqDyl;Vf0a^n;gq-o z&k|y$T=sb2tj`sP+qAIJT2$ut>1)B=$xfLZL3-|^ei;xm3+|@U;}tLR0#YSqjd_V*}`Al(3KTJISuO6Scqs}h#9RNyT8V} z(dZ3dHTeCo#V&qf?KqY0?G-C;Gx_&->4CffhA}j-r>xth9;1XNjl`W4wP#`&0rtY0 zQ7h&l3f0;>hO+j9Zo^83uv&fZN><08^tRTO`GP3BuPmOLLsomhWE)+r(^^W(~0?+75jo%vNd;Q2pSSq>S!#H*pv?@q(Ww> zo>u|`;akeKp2n2a*VP40!RBFShqR~waj{2UN)uNU1NwF_mZZb)Y|^L~5Z%sdIa{r$ zJW$?6n$qFnVO!oM|E^nB7*)+0IyVDE*vm${#5#LjBKw&ivrAiwUE?IOKGB!H2b(w= z$$t8VCg~EHZJPEyGwQVb;o%su1PX7$UWrJ)u%$?WPbN))Qi1b&baI=tKwY7yq+T?p zgnwwfG+TWtvCDm27O6nh&dU2yRvTldP`P6z^D6`OiI~C!TT#DwXP6D$k`qI;816l< zpW;+KNSr43nenI?RmVKPjS>8M=X&N*f-?;_C>(e++noMI76tw!@6GP}uU(QKo>))C zCi@rvdh#W!PEd_5g#~~7_cXEfPD&kNZ39U6~_If zBS(6Roji*pNx%bA-yU#LyLQOec_YZ>Y?X*-{k7O0sYToS=fc3xrYJgp%!P#oUTNlN zQnRPvQ+%Kf5Tu#so}LXHqIOu#6|So%D-c}B5#pEI&^56&*?D>_ntRWN&#MaukB(g) z#|le%YVRh^N~6|L)-+$LQ}CgT(R*?T)m+ht2)`{o`kGidGveLCNKR>%nZtllSQ3BR zg_r?t0!dpx`sOi1Ok~yb_bU|5)5#BeE=l&xecEXX%1-U1~^8A=Sv`jK3G^``6sn-DF8EmIi+%Q90G&Yna@hB_H~G<_LB4Vhx%n=dXDZuT-V zG!(?g)8it8WO3|9%T%D?lzT#$$crs1)GcpeVyV!hz@yzcq=))5Lh0y0>_-Mhq!22@fA8$=i^(@`nH zo)2^K^)v(EqjUGLe=;AbKQ-8>SfT{msi!#*VWMB1_lN-zoH>c0Gw4b zp`yNgGgw%n!9tbMhE_^}6+x>vhV71t{aB6cO@|xW-p83l9>1*+>n%!)h>!Q=RRyzH z+8r!N{fXD0<#LF@OuNsjseYG}{8OO`KXUil6CV(a-m(la6{-=78=I|4ygk=dhrpgP z46p%Tm%P}(VQzqTxch~6NlAwa+3whO%KE^fFk~}j^M_QM-i3r}ZoI;Twmu4n1~2)$ z4t1MD13Kp}X5xfCD)p%+d`~jI!nrcOAv`InvakPDk=Tn7Gqq1SOqp(Rs)PNs?kfAi zyI`5}$o?#L>Q?x0==oKNC!5PtXVI_uj8g3h72G2{O%M!3Y-}t=U4B0O0wB`dhIpEc zc{SYX)v&(;J8Oj7xKEIZm?GwT=ErD~+toq8yupreRW5TCa~Nl-lL~r?k(JA9jGgZd zE^IplI#P{}*5Ct0Mm=nD-j@KzB84{JyLS(2l@D~&P|1TfJqRPujsLv*?;k&`3l5)@ zp{p})@6V}rg|(j&o?u}MH=yn9hZ-7sVL3X+%?i3d85P~eK}Uo^;7vdKmwclILf8gU zME|^|O%TVaC_jHRL;zi7PT17(d+4*D+0#wcFFn{yn3S&NqR#_@_*d*yIhvcKMD7CeVz;y~#vwoOh)dM`W@&+g>N?<050fy8nm4$`FU1m@I^ zB1*#OE=cjENYo!mjr2B+fi2j=^ObobGUrw4;|0^d%;IdDHb2e-UBsu%aooRaf%y6Z;?OGAfZ5ZwDPpUn?2~P`0u&S-KJS#Z+*2AqijqQI4vr zs)8V#Z5IrgHp{Cz!hnspE0I=`###xi_vj(Sjhap=+hR1h_WJ3)E9E3Q+}rEv&aR1x zIyqagCktZwLj1X*7Ivvyxl&&-yUbg?d89zY2epXThlhlS5c(S>9icYMVe9MbcLChg z^^@l~HfZgY3I)SF!&?eBRnw3*>+Lv6Qi(#oJ)X1b2O;UEA2A2ZJTkAubPyC~J4ZWX zU(;N0tC~ID2#rkn8#8Ms870|`w7}v;+BQb4lH40DGDMC~ePhT7De%LsH-t_okj*8Q zGo)gng5a!MO(gQjss*)9!$jru6&e;c_ALIYm6a8q6#-I#c%LmSm=*KOOsfyy4HyxE zk`em ztSRv~KY!q*+Z%BYJ828nsbSWSW*un3zt4<5ZsrA8g3%H+P>o4b9_Ug7Fc(H19=a7M)`KIVbQJs+y_D>f z1moq5986U#WW7?Uir3C+OI+&|#Edz$COdu;D9{~BN3z*}o>d)5x7`2lk3_34YEWiY zV~sXZ`RscKx96t?teid;h%iQbHIHINd_2y_o8uu9Ug1ZXW$v%Z=Tzu}P!oyfm*h1# zzBzXxuUV;wWU-Dmp)1DzG`xc=;KUYpwU>tUW%*V7^LUJ8oqUILd+Ao4+j*VVwvUK3 zA+Yn0LVY=!R(B62!oV!!>twl6FYswu0p#(#v=E=TY=EslUj-J8mxC#DKc6yZ5!eXQK)Qt7Fw z{CuB*897D@me6RoYfAzFuP1Bu=M)MQo;m%m#dTH}IlUmhConR#TdSN$$2SI!+$2eb zud^h7my~gX*FX-)L;@#NRM2~$muX<_5mw3z?YjNB@dFcHkwh{os!cpc2(bQyo>UtO zFV5E4n~Z}))2B{NXRi3{=OXpp+3V)xGyNJItXF$wVXUN!ew&LU4vPJSKQYu?PW$&>4kba!{RvdRJ4 zLgyiwplG64(-&lLCfu1;6rMj9JgLkTaUxSxQ88mtz;XReR&Z!>{OI)T%(ifi$LkX^ zbfWXQe^>aQFD1ourucVn8dRi)=>Mbo0I(3SKxsWvQhkT3dde!)OCSi->9aN#BW6gw z`f;sUI}F+Ra{@I+tM~qRno=4O1`b7+^#a;Vp%asw@B}hRUWZjLdI4wEt5>h?stM-3 zdjctDIK*TylRHTA=WHB>?o>sTXYxNauu=)WBI1J=5MOuJa<^sG0bH3~9U<@#jX(n? zpRchoWB8_?h$Pe1&q+~02@(F}DQ|~e4?dsivJx>T^<7UDc}`%zVX~b4tOMzow;T?K zXN3NNbpS^Rw@Q9_xl{Y9VpMeq2ewY7G-tYe^I7S`2(g!FrgdnjAtf)oz_p^kUumj0 zG|=6Zhsn2@PK5t^OF;4iZ<$DpN@~qeHJJ>E8pm*Locu%|4KWEcS}5oZcs&0Qw%5a= z!J|JswYj&46`a?-ys_sr=1*%|E#h4pY4q&(dHVEJIhO{VkeRkf9KecoUmjb#=^0xB zA0T&Noo2QuKS*7Rj*hN|WdTj2Hb*{(gPk3-wyy5#gD*K*sZA(2>F-K<$a6dqPaWHt zXPv*vS^!^w&>7?7Miij2d%kI-MoXS5_`utYr2mD5j}L$AnUy;F-g~~sNRqsbHq+1k z1-YnI@7q3z;gJ%qdX$n-sMc_LrK1d+=Oq1HO?(xR|FG>tL|g$}LVKrfW4?v1icYgU z_CupcTgjT~oc({Qs!ziIEMZEfzLqpNv6-XrUfEaUF&ih&mXd*-mKo@>E<|zGl_SYYWA}NOYfXe5IrSJ~*X7O%Y=fLKx?eMD+qtmFa*b_lq#g zvs$jok3CE-5g1uY8${t_$f(hpxe}zM64nYBNVIoTawm#TF!&ZZ8KMs2SFr!RlOB&& zqAwPs@6#tFq&iU75vC!W4SzOeZ>^ds-xd|soAZ?MNFh%7GV(SP|6iL(3IV}{mg6Qv zgry}-JPu||^F&&N!RvQ&^Ylw-NU6Aj*QAKF`R`UYNnPY%q9Qt`^(vGm01PSOd$OAP z;&|Zx3Lp&QGdU-9&lJ@0kP1w!c?3kcxZ*le5y2rfWkH6 zyQ*Vgg7V^b4i+tN#gy3>An7SNh`@txg;Pn5C=rr<*YfXwA`#3^(l4|>#<|wWU3-_- zQ!+|i&{9h#9n5k-NB-bG=Jpy&E=IlWfh%1CHs9v4#_*FT`E*t3=bKqiA4;2{8YSX| zSBrIC_py8ORLR#@>zW#)dAi)piD{v^n!(kb(EokvfD*N2hK(_WD;_K@Evdm?3F-*HMEhFbbw58488sQ<4rHI+)9s1jHIiOl zQotb$4>07!MK}$(KX%+o|2B?&&|}Ge9-l5R<}vGHGo0b|x4d7xPtSm6rq$yIf1c=Q5!qAxb5osie`@4x zw#@A6i8(LIa=U*aGH|r_Ut}INFSXl9^}qH2k9fc|>pA?NN38juN1X8v6h?1V3-4@R ze_*h$_S}A?wDIK?v(>G9PjH=+^1c$b#f?e5%IuYO20!4=2Q?< z6J64q8i{+}6$3AiUz@%;bgoy0xO`oGg(wLp*EgD$SGE?Q`JRhLu9d3L%5Ow*2BT3t z!-O58Z$h%P3y9-=#bW)84APAh9OVUQ7GI%}6LT_Fc9O^Yv(SY}`irn@OLk@9j*aD- zRNDt(owf_0UgP81S~nbLQA4=~%R{PeJy4GY%dLKNt}@Pj5z=Y>x#8^OVjcGv@!Xv* z00b2V=p8IMM1R)rd^;pm9N=+oVsqcg_iPIG%&!p{0s4miA~ZIwoYJPRcJE^dM(iH= z4dv@VU_7@}a{*{zI5Pw=B-*;+SU7>eLO?)C`2nE_s15L^fP)9%-6{Nn^^%O9MpE)+ z#8hA@yTUi5q)15pec7MIzLAsPzR90|dNgyR$waegg1K0E7^*w*qg!^ScdcRoseajw z{LQhNHYU!|(^V&eRCRGPOlnzG%YudV-uW)g+gNmM`jc`kvpPo8lO>0lRz?hEP5dY> zl}Lfy`nxB8xQ|&nkvhi6N#Qrt@+e*Pyc~tkzL$5eE$` zY+9>q$+prTfW7HQ){le8e1F8;zvD}ETRu?jWjqPKr>m=rr3v;Lyauu^S6=q|m{K=F zi9EKD2-K;5WDsLXEd|mdsvRcSw#JL&O5WRwDB%hJ-c3s}L|1NxVFBr z^AlMH$#wbr6w2er#cmU2u~c6ka)y4-$WuvbVfyH=VYcO-mfF#a|G7aaNm8O7V)|L^ zNva8+0qG>pGB)P9eviyRIUjbYk_3BIwrIW?Pgr8+04sv+8@8A3NgE+WW{N46QZp$5 z)jD=ssowb~BDQc(SMm)sI>BV)1o5%w<3k1Y)kF$UoZKxQ9OAlR@Vzh!Cz#vetS24h zQVbZIGk=`heIJ4Q(5S;(G&GZM{S68qiWffC5%QB%oATZ)P556_%m%5C{Hfg@@Sc%Q z?{^~aXzI`5S^I=s_{yZ&+xGQYEGRfD?qKr!rN?vrPS4ctu=aQh9_O!)Uq8dYe$^Dz-`T`)3s4Gj8rwDbo!z>Y}5Yeiet%YVG z9bRv+og{B28y0(*Sj zbdqF!X-kMBf4IHDQNxBrr3axJc>C_^V0ZzalDy3F^erL(VLIM2%ch~^jSAb|_Z)Qc z(R4X0E!;32EsBk8=~+axn3#<^v1NK>AZe|o!4Ou zXBbb2a5!(~MU|5v|1!!QRa`|E5ocd!yd>Gd=1=db!oP4`a4HRs_tze?)s4wO*U1dp zqBVSsjkKR5aNiVyLr2~nQsar^V*cm)9=N==<6hk=BM~;wAv69}hure>Z>M60A)yoZ zJb!IWGA9StBA0Z`(tgKY{`^T5E?M0!WRWpKFv|mxMk}AazHojR?u;r`&gm%dZVIhO zBA}N?&zhm2eQj8UVgAqeiIu~1oO%QCu?}&+9=+M|@$@V4d^RHEwD%OZYKdx}M;ik3 zH&Q~FzC#0o12QEgrN#xACR5PRh*#RJ#t1Rg)9YOj1;VWMOw%)Z6bJ{0G;8VR!NPps z!l!RP^ghMZUd?S)E9Z%{i9X7YscE1)8+dDc4u(~g&cHSB)lvrfVqM>K^QH8pO0EIo z`}SCK(tcOu4RcakGa%Qy054n`#@T>^j9d_K@zV`R96~yIZ5_^6SGC`Q>4Q!Q;pVOB zHNXKzV>goH@Bvf6!ni=m7_mJmPh0{gI>KNQhvwtZ2H&+lUj>ClnW+ z4^R#`ZE&fO5wM%46N2P<6Vz#KXMBa9iPm{`pZ|P)=|#_0YmKs4*+}jA}TxAi7TUOI2S(c@8N$``;CpA^5Qc}5=Thdnihf|3-sS;-( zoXI2uD@J9AApS(}d2@u~71Wb&`!iSV*q`qw-L#>l$y%;BzVOPjP-$vG1T~zjl4_AG zj-#*I{ull;x+t60Sj(AqS_1ue%K|!=DQ~!1*prTrm=CnDBAoh_!^-#2G6Ks8(!a`} zyZ>&>{EhW|aAl!TYUWf4%YUZuejZN2t2`&NeA_PbQ704voKv~n6G9DmNKIXf;4|?o z`-4c5%d>2f(h8(4k065jq(eJ8*I!L*3|r`KpT1a9NN^0Ud6juA0^&GZ@!Hk-5SLQh z@OKJEzKfc@jdK6f?i)&*1kD-vCU!b*_#A|zsDBA6SI4{Xb%17(t#N8pqfYpW-)YJg zETo>j|F)1yvFuf2)%67I%t2;ZCV9*q@ zznt!Qda3pjuf~=CETIG4Dz2aE4N8HoMFY?GlDF?uWfzWo$-iiEMUYnIhSa{MI#rA7 zW|Te&C^xpBUDXne!P;pye`($xJgmJ4ttg?>0f4RL$PCWs&Nt~ zAS8(r5%#EbH{;eP)o-dH$umOLqlM#YO|$@G`RCcJu|6Y>^iK)Fu4L4aPA9$-npMc$ z`2k;7$*5l8e432g1-A5z80VHwm{k_ZpsH2jT6?4ClIr2Hm$KQX!p$sG+O!^(jE?6x zf+=MyV@ivJk56sH&cP@}fH6!gU5D573CYg&^6_B}hjkc0v*d9siJE4in}dS`I^=<0{+PWd@RajE{X#(Y!(qLitVZv00~p^Z@csIMF>aJ8 zSiso0p!n|&H&SbLuSN@L3~<5nk78B0v9;j$)6!1K)7*f-oCMlDN3Urv=$ ze4!eVkY7@8KT|ed4yu{m8#i@)QKEn8K8E(u`>`ZNVqyFOO2vt7Fb;IWG_oU0`&8-5ue{+d}qGhMPJ#XRG|yL&2Gt$c)4`) z!km$NDJjCj_3{!qQcI$4!NbPRJ1p3EKjkox$y`+Lz;(C-oh%zJGX-V?) zac@FaB-Hvl1B#iRmhdt9LbC*MB@HdkgGK6!5OL}kJ$}Dhx*Kem>z&D(8ZL^8zxuZ+0^X77WnE!_ z>i-4g2LHLTFF2LGyJ2ZbFgus+d4Wjlz`iVFs0OO?m|USzIvvQjjWDwCQ?kOw!m_4@ zMOZ3@%ZqTd9e~IwJH}uoB_$1`V~rvmfjLZk8Zm;Rw{K0pP>g-?h*{cSS-1Z52(7#t z8rJ~v3rkN+t08baW@Aa3Iz9EgW2tGRdU|#zrJAwPnxCdx%Dtd9wGiM!wfQq}8iM`Z%zEXbd#1OZwk$6?WBo=CBsi4a82~ z4Q!Q1a6e%(J}kTAeU08l8X+UJ>^~wagR*x<`hlUjqi}pkJ%6Z2OCoV?wd!CREoP zny|uGR)^I7@AgfzQDo#s2q^Z7Xb0ztmEIWBdxV{JjL=Ub4A2n>lz1eVIc~p^HC;_Y zK5N%f{_`;fE*f5yBtD6$ju+^Et~oH2HL&%{+`{^`xNd!jw0@iGU_hCF!To=+OkDgy z%S}rjJ)_UFVpy)g_fphuAt*Rm#e^7JXJ4KsON-8HMYcDsLeHlzp z{m|7C1xh81r`mN?vGMfexm>e>5C*F2}LKM}6Tb?AhMN<9x#u8mdcMWGOVs2ft%U2?;SmI}la1nfLPi4c~srucDgjE(dVy zEM(zYhvw$og~{K?*2>qOBu)yCa~S#Ny~ zmcIU~;MgRtHOm9EgPl6Q@U{w{QwMzaj-YOzsBfD>hH=s{<@eJ)mtnR!| zN@2v|a@6|MnYX;Ho)*uFtD0oGX1jSQe?S9`JK$S~uG++eZ(KhADZu`vGW5eUuW(G9 z7w=Z5tB3{a1!*HSu(~-jb>mAGEp-dg+G${Cegy_^?U(3fA9v+EbP!6MGMI(oHGi+Zq)q424Y`+ zk&6NgWgIakV8BBTlcp$01zo_p`O9xFIivS?V^-2~3pFt~Or-q=DonvnR4W|yLgHUe z$0Jx6qjoHFEFeg^)MNcBcDmm4KKl@P<7&PJW~DFR3foPW^9QwoRG$2;3kqm3vJX^)bO?VhGWpv_&{ghCIfWPUG2r`#2ltcjU|3 z!MyatEGnmAHbcN>6-9>t8^BK9Tbch0=V1oII=C&`C`zGkm}!6+;AJ_Y$FrI>L^;pC z+^EX#-at@LkV%pIvD8~o{{L1kX)=aq!dRPk`3Ed)Y=YW)Set=|-5A;LerB?qay^k=XuITqlPH^4w@oES0B74gJOT8d0P2%1P3~#S`l_UZVIQ`#_RnV7d9ia z5%gdMET6zed><5YpZld{rJ;LqlUwEnacQjWRumg2I|Qpk29^h?VlJ2LNRI?Q1ZCs# zB4m1q%MdB${H;XKg~|llglLf7;=$$bUqHEfr5wAhDlxB{nP&{KO~|{=?fc;H*)Cpj#H8EqHww{_NBH2~8KmlW`&7|}AHR0| zxchyvo4#8g~TBiI7!IK zg`V+cWt5p1@BFsHh9s0-j@ey<^e@QZ3uUY9)USSkuwGtg{Y^E$;7xdP59~(a3bZYJ z`Upq8w8i>*;q;jocddp^KaOs0or5OdyItcDs9_7aJzx^t1!!sAy_sW}kgrRoU1_j9 zIQgZHar|GC+AMLS8dlf286J+CEYChilxlLu}(aa@x$}0tBE=aQB#PI>~}P#7Bws3h5i@M)NG-n zA=&b{&WgL@^&%*-$DGT9x>n0Oh7U6Ke{A0}SZ<@frAtsL7{@QSPw$m`e4F`Od<)6A z+jCE^Y`aIfzI_yENm`2vuumtX9=+7Cc z)SNa%e>|nlgp!_&k_)NUk#o%lzX*PeJHyN=Uu4Atyz4q)4IuD7rn`Xd24<$BR9I>5 zT-Z4}lSog0%4s$mr!cD;x-%R6vpA`)HMaks%-EiJM2gfhyX;t1=bu5T6%h7x4|VO+ zr%w$J1F(uC&|ac(*FliXQh&LD7>un39qgK#n!|5;+H(5(skUJn@gAoyYy6N|Kc zJ>5(rp3J-lnP(b!NhUhI8<)hkM#uSqM(*YZEGjeS3AEiV3&z1VGXJHO33ZIdkDmRc zT@^^rk!+DtpBhiKsrV}^I@SKS$NEMlNCMETLW>~#%q#9e#lL$x6UlY1)nk&Pn10ZF zg=`cfBZF9~P!7G4u4G09Zah{Ifru?4fa2WTu#+#gK|Tfsn8HPorT>okJP^7AKeX9oog5{1!2&aUfl=pu zZ#$S2vtWo;>1YCRs0#-hk#Py6{%)XDzExZGT7u}y zMgp#<^-S1no149vLiTqwD38QN$st%A94^0yTK{n4>bX>4WZaiRP&EeGsJKx-NdNw3 z^q&<>2LtHkQbtH-y45HCv+?tvTgTic>UxO{|4=vsK;&5uRn~f7VqyYS>ZVx%whZRS z5o9IdA`mLS{C40cg1d#3mvglp^;6@WSi!6^WL|8jNc|yCBS^b zn}o&8lFM$H#^d+sHnM)DZO+|a*roFylfhi;=zqCK~D6NU*;g2fL6-lM1`w)#m{)Q;FSqbBxWU zAZgu)`387b=f%w4g-~z<{ETqqhSJsO=Vi6&ZTl%+5i0fE8hZHA`ZL}oUry$4N99_y z`-|6CmZX9stHaB=-5rr%Y*kiq2B>Y|i>O#JP-Xt#-jzFbaWwK@`kKdD8i_t~=r5Av z`NnW^!3Ts z7ajsFJCN`3vQwtF<0TGrI71V-yMAhj2G;zwY%0)X`p()CfMV9`^pj>IzV1J6mpWc5oB!qAt5`>dq4Kll zbBEOa;AH=FYvL+@rF?Pd9a^Bnw?(jzZ4q2)xcJBaB2ff+_*IjW3l{vApE76cflYs_ zG~>Gz?RLE^X6>QJ2D&Jlc$+nwTAN5N_2w=6r+wC90Sx2|QsxC7w+xuD?^4@g+~ygE zeonp(&wc0b$i@Dum%h)A$f}JQ`3}$N3|kf*v(mqM&Ao z^v;eP>WB8{h!BX1)zCUK2E;UW&m0<5?9U6oj6_6(-Y?~C=z!bxAo{mg86)=2%JH2m~yr&5Lo(j z6AKn=7EJ&YCJ2-NijlUH@_w?21F62Yh|~CD#o?jUBh-tIZ;LEqDiZs@x0&F65AFqq z=U6T5iAC>HzcXkUnA@27&@oW4GMVD=SDt2H@81~C#*L-esc3430iv&&dmnlt1#Sf7 zCZzvxx>H^y4=4`q@k#+Y7>Qtu^(QQ2kfv4+@tn|n@YfPz_+#V84{e;BCin`)$TQlVZw&o($I zinSQwZ6KJM!d1rI!c{x&vgc3L})?0z^9kw*x@cFFNj1p=@3iAK=%jtP(i=} zX~3jj?=kb12YBI(7e_pPe(%g@J-ROfGHUF}qKgfh1t6wC5k*9LgF4~d5kq{xXQl=> zqf;jTHD3PyKon1ukicGjSmUm#0o=*#ngj+Dby_p&olmw%Kb<#CQrFC5ZET!0#;6kr zlDANY%d~DXnI6OIXaVt(e%CMI?YhPa?Vj~j!9cm4qk{}_v>4$IyV{!fF}o$b{WBSh3UP?Ew|UBx?l z8rTr5yv1`axiElHBdjde6cf)UlrrvDc(%&#=DwX@@mq_MQMX@Su2&p|eAS3t;LTT6!q4oEQ5w@JgSHcr=hSedb&lF%_Zx#EY9Fe+6w$T&5B*Q*{_ql} z3b7eV=s7s5n=`kfou>A5?{LSapY4pQxcSXBKYXyg5E9G%C`ushDP`GHF?sMW)zn?t zH`${g)(R#X67AIrv|J5aY4gT()`$JhMS_#&M!7OqHf&sl+vyt*9#xjYsmA%OoaL3# zXvB?>ZHss4Md|5{D8`0}^FB6kU5Y}Aic3m*ejRj(>-!zP@6Lii*aZ^6}5s$=EqKkXkObmI|c&c+OxI5GsQ=3s*K+sbmJ68Eov|i02~w zfFdG?kg5H%AkoJl{hics5j{dMEls~pEqx>bCuN)z&Y1N!%I>3U9od5^zs&XJ)r0=y zD+=jYFTc}h|CX(>Q(y6lTAC9A+QA4t3ex0E{lP!N*BAj%jZsMSCjt(~roQk!w=To{V%hd&x0&t;Va&`d`>9g{V%Dx<3bX@cSzzlJJ`DI=Mrua62B}^8D}dD@$|pJ@7_w zkeWh-1w%6OiTw+e6q4ljJ@)~tcJc>rhpLLMXi3^&R7bfHFqC)Rc#~6+IJEKcapO;t zS=_?HB#+G}Wsf)55fqSpFx&$j=pM}x@7(|qg`zs8=}U5x5-+|7P>?zEM#jURpRa#i zY)6-$Gwd$lB}FhMLn$Hlco>%1E~mTB*Twst*GhU4@5N)-q*3*|bV+O{!eQz@UU`r`e3Xf6C(10Q8YpWpL|;sJTA^r+tEom=T_&xs zymsidei$4%(a65rOxMOniNwim-h(qJa;xB?c8BWn>LJpNcPWIoBtr<1z4nA`730XAAnIKs!Jbt;Vh8b zv$>7upmr-4n)14blb!FATF;xC`AgVC{M94>O%k#OD^Z5*P{x zF=5e_NP9~rSDnPw2h)KR{8GsgaqiOPTuzp&_V=otTtq!P7O2_UXNV89>rv-mzb;gh za}BY87MIYR!D6}enA+oN&gbt)L&X^~%=5Po5ajb^o4>e5P#&Cz;0Z)7-K6StvNs{e z9{qo8y=7RHVe|I8=ClrC+z-xH*^+sPu5DoGb!e|R%l2cs!k~Kv^?$nbcIMw zjvf@_kns7*mmrEVOBvMSN<_vp`>@haL4sWUAnqZD*E*@clG)7%A@g=<`xO>>RnCLH zSsT)o@|%EN8$YF8BjJ%v;_ZRz&ghZd*>od)hz^$a2yK4 zcoAmAc%+gPrmFPtSV*DPHWlhCPCiyQt}R+?Vktot_c5@V9_qzNB}TLdmonEtO?&vk z2f|u^wsONs-{r&n+X|3_?wBQQ#!7rr^ih7!`*HH|O-MipN?7Zv?lS#LA6~=irRURT z!wr%>g1L7G_a4N{ms=Vr`82XYkfz-}ugKqBR#olr&CP24<#gHZKXuCNvqk-~`&reY zpq?7uMBg4UV5V=P~*{aZemKHl|3s*Cox?{*%Eh(GoQZ4 z>mKzg{|WkdNR=zn_5e?r^*!`Ow_1{X7_|^}-k)|_ts3D!N$A2pQIbu+-Jn7JB1-}!VJ7iQT&_T( zRz*#iL_G0lI;dQSUi--Dir%Up^6D}ilGo~w$nx9YOAHKeEjKDkR$z`~bCOo%wJ)_;twqx?(w;D(Jn5S9I|;xdKZPD1iF~1U93vtlHvw*-5mjn-b~ZaO zav~J<1)tl_2z}aMW|IMtV_dObU0D1uz>3ywGioVdhAcsGQ~1#ly4ulPV*~oe;3TdP z`;B|+m75u{P7)ZSonD8H01~Y2{5-a&dZ?G1bp$G|X6yk`oWZuJD6=%DY@Zrj-$%}X z7qZUdM{*Z!;^}VgA;V&DjNhc zh&sx>kF`mbZwFzbGeT|Oj-Z0Qg_?u!drWP7$OMcurI_3X=;h-t3^Ab^J`&9~UwppF z3HH59_wOgPP`33$Gr{k)ALry?S0y$_;*G6W=T>0;(__&+hKKNaJHx+zP8>hW(aE`2 zO?XjHGpB`KMEG6U`r6W&op?aZx==*DX&ZoKM7BA{zW#tKzS|WWDOH2@FT(^wUtGBM$mgA5;-91V zga(UgXHq1NKylMWp|`l%g0FCUzU_ScSYir?Ltmxm%tq1V?T+w$;2nI5`>Q1scW2|o z#_2ICk=&;T<|~sNy-wcH>>Xh4L~v*~oZ>7Cj*>dWX3|*`Kt{8n*o>fTq4-mi-^OZN zTfT$+aB)bjqN^QgMDk6Fv6xgCoD3K|UQicb8wupV;C#JTy@DzR~u zA=Gtt%V6(pj=obMH&0352Pcsa-bD6R3jWeC{nA<7Tcf`i=bP;>A5|-1Uu{y3aMLUT zoAW|7(y-H#c3pY@%pO+N!I3fo2sV9EaBuLpD(A`RkAlOsP6!F@x##u4jEQTA_nAGp_tl9Utm=yKQN|0dpdEJs0)_ceRvXRq z1YUq23C7-~(QuJEla?}paC%DE68D?kI?Yz7NTdxb-D@j}=c@-t71EQC1m{p|@M{a^ z!%PDDbQbSYC(Fms8s&kXGCF#Xe~tA)(F1$AZi`25Ns6=?HFtABPE$P=lbs(3(CS-|B%wI2qQ-@cuj!^VwQWfDeD7k@e-Jw;N>?G4}J+B z>YSqz9nRg~>e5^0#0+aoJD>!e9;e1gXgczcNOIqisqDSSSa+z?07dV_nJ*WyVQ*}t ze1<2iIp0#MO&C)4m_cmHr{J{ps^_`}?*f`kjPsLFBNe>S-2oRGa>1q0=C0LAw;+hc zpS*`@i^F8U#vvG%wn!o988Velnj6JmAtYzWA;0V}5cu3~AJr-RrLFSqPj{xK5PE#%64T!A5N&NIqJ|6{yTj@f9n?0A%vVuC@I#X6-Nb`6Y4lhB3{*5*s zv?5IU5R`G4337U~_3%ZbnE*q(S}E;JzR(l=yzv^XMvmXfeE8{2Znj#mx6jEQV>S@d zBR3zX<|7L8Qm`JBMG~|nK&Ju51W`}PRs-bmFcn16r}XOoSi|yoN>{LvQBn1>4FkdU zTt7OFAw@zI^NJ)+PZ?r)lHG()Mm1_i7Pk(Qkw@cJn zz&D;+dk#`TBwK~A$f%D$k5z-`LptcDRssm`cC2<-BwSggy;NQ~pyjv{BRIG}kYexonkox$DtI0NJQ0EoKM6F+uic1Z4_G^JCO2*bP=be}d ziTe$z;Dw$D44Q2YLjXK?)!l=aI%B9iZ8y*Fu&xcH_!B$EPVyy(z4~U>r5a!YqI(R2 zz^{-FwFw+PC1{;InmitaCk5SmumqZS*pI$&VERvzojTkZv0C@bW>vy0*Lug{4k%65 z?x@3ETn{206fz03of$&l0I&KX*VuS~^dueh(d#0O6tfhik;{vI*2RyVNs-MaW|xv({Ma3-dqA=I6rQYO^5ovr zX>erb;NG@&9mFs*M4<3bb8l79@^JuYc+qlIY$J;}?jR;5Cv(67V}jCSq_6USi=ZK# zwT>J8E)3OwRd{`Wl!w@InAuDImS?a|#O+~2MU{>3^efZzC8Cb&%o6f`zG*0ppM3D)@OB<&i?hJQ>~74{%d#jJo`F$*meSCVCD=d zh0Qy|Q3f^>;f@$7UXoT*=p9a%NZn|DOu{#svyf_)d_-Iv-vvk!F;-{1k#o4$kEA7?qkZK*v z7KY62FoIeU2LJdAhE+guBO|>4ysbqrVAc)+2r4BLnSu7-{$oiKTZ;kENr6D0y+PXq z5%dub!n(2Nv>!Wc|LTr}yK5vuqBWwv9Z2;tz<#%ceUC?qAYI}eA9J%mC5iAQs9qpc zlNlV-rA?&ijuO=Qzr%dmy&OS z64~wfwncCbZRnevP-fs9Sm)Zm{?1w#<-VaZ^iHO0?UBXUYMueMkvivkY z=BwasIjl!A`UYepL6RRZEuK) z6rz0mJyIzA=2zB^9EzKM$S-=u*U3T4`COax&snIVKAAmjCqRo*?{dz{f-5;T1f$U> zR`81kR0!XcyB}(M#K94o*5>W^6LJu%<7Mc2=f_~5j5?!(9mK|Fa2qv-JO_6)SWxix zCis|cW3N3!6xTQ=_tOECbY8*s*V&JhzNo$99s2h8TMk^Fe74m{isk2&K1wIUf#>V8 z{w3rF_H4E)ha1P7@DOyHQGW`8_Y#zA>TZ*VF_fmOnQnc`2I1?S4#eVcEoa)Y2*LA1>gU_-IQwc;rv`!EdOmg;)>nF?i9Ced_^{RVd!_C?n-Ii~*6t5C zmif;n&IeEyuxi-s4Q4umz@m_*-3GVZ{q#Knae?thA3t)}%84&sJ1iKXHTKrf+YKP% zaSlgae$T&o1ZA``L~mnPofxq7Xw7<8nl8!0I~HR^fPtg^fO*G+eBxk@`~Ye>IiPSQ zc2e-m5F#h4t#0OVQL;vrnRZj);btIjo z|B&fJ{s7P}-T?NsewhLMrU#z6r^5EI*`KozLLPpFv1u-~nTa6{7Zda8$QA1L zVdvWqHCANkGzUjtt)0@RwaJkZ_xtgG*Q7{O9~iYUwuUO4X+6#8Txe=k22Hb|X&qcK zL~Ep9IX=i1niHj%7(VIYFULD~A2l;&6BSV-Z2Z%4Kyx1u0-QI+91;DL*(Sb`-?DJ( z2Nvrc!ok78+)8m9mJRsYoxbhRxl^%nEVuuKlcC?OAQH?a^bEx@iq84?xXtj7CL<`4 z&ok*QU3vXE;n9?QSo-gLzmkqjOA8&csw{@ZL1(wYE9D^h-G9;hvpZ z-ZEZ?z0kCK8u6=mu-!gc2p57pJD(2jNY^A>r+&l8u+5eB%8duKZBpITSDVcg1d~E? z4}DmJgx@-E8PLW!G5%0|A;j+Jmi>u*t6FbvQBV%ft3p&N7V;P@ zc3F+t(ee1?ga`mwrEU{mq5PU?5BsvzAEqJ4qe5-KuQz;Dg|(~9?`q@vW12sCY<$fk zyWfEJT;u#`pjmJ3fMHpt{QR1VF;hcr12|fcjwAk!k1fj#7de1NS;rSr7vLBbf3KOL z!Us%N=wVutCrK&jI}`VHSYDNL3CMO1AK})Danl!G7!8|Ha0K!LhdXdd2eik3Ol!p%PNY>V> zJnPc0YHHXjfi{Bl-hTjw-Mh<=y8(CI=lh#Mw>K(PAqzu}gtJwFFVlJ8Uzd_h!0l6* zz?|=Us49+0+%HZy_tl!;;AwE;Ab43bA;3I+9QSx1hF^5w)VWlhC{QAaHzuF!-=j!g zlowS-A+SMkRK12zY2NpeYlAPLc!Ihi$w{ZA1{~y@?}8cm4vqxg#3L3;Rao_eaq`#5 zyS!3mX`PtZ(J>9VZIZ!1|_x< z|G~}{kW|R5w(%b{kuF0GuYS4kS}O^baZ zQ*Zq5fx$^q|D-?3>__qTd*RZ=5ZcT9VvLQwU$rp^OYvlIh@r+0s|&L4P4~7I-#t8s zm=Ln+puD~AalViDlN5jeZUo}H@=0+!>?A()-8^%`5Mh?LbDa_~9iOm=-n1Y3U9-JE z4sY)2eXO~_M0~DfChzTn{Cwp?KG_+Ht-i;of6jE&exwZykxRC}kGvd#Pm<6Zey#T? z?{Rs%In=pn91ZzpixNsQ;0DcyDp}~elxFirg#^{7!JNH;lK55?_9_)$y58mu+rAr7 z18qV0nMY6&DU!S6>t{rpBe(JokLyKw^Jxex;Zz0zRF;HGlO+TJG4*%L=FVT&C}4AK z^s{3~HpGG?kZ=;?r_{IKf;VwMSdkHu>sn))y}Cb1@#T}$aANSBG=d=kpb>K|#JFY`sI5b+%E4K=5+l48EXA0i z#7KExhfxi?pHCj%bx|3^`}ec`OE7}&@r77EST~i5aI}&s^eR3Ke|J{!N(O(FLi&Dd zsMNhPk1lm?G+cyW$42^%E{i@;!#iebN@N-@M^$?qt37?mnDd!Vq$O(VF;als;)c}Y zzUCz^)u4Ar_%)9v$d9#&x!vt7Kst9L%FU}*#eLfW;rG|Bh_`GL&{D8D`}qclD3goa z!ntxkKpI3$1}k6rysncXdGdsmcOXt0LtgQ%dT*5`B`OET;wI_=HwQ}tHh14hwCPwt z(qC0*_pkCv?>od~RFK!ftpRHJAJ1HU7d<_~7^=nr^0pPF6&M!&m5`8tP!o%yJHYWe zR{WjncdALDAvA=ajY7s?pwl}ZSv(Im*QJqAj4Neum-lW}&7<*3OhHaA0AQDacDY@H zB|#Q3_IT+!2v_8z!^6gPY6i8{)h!u1>Wn;a{?c7Tk~w4n?J^wRIEi#?3!Si@G8@Y5 z4^kL!a$z8dh8}yQY>Fx$6QhOr>c9T|Hw)#Y0UM^_F0M9oMvA*@sPspg#go~%h@ghv z!>Ax3Z2dp6u;WCD)Wfs#Dnk)UfG(GJ6Te>}^iA&nv=n_bNPf+0wG%hOLn1hkPJ5!i zUGRlscz6o-mQ$^<`yPO#=+^$(1K)Ew2K-B^F$_Ae@Vv( zluG!W#{gp4RKbe!K^E?PD$O?Dxs}e=-bYXEH96lNP`N9&KonB|r(;k|ia=e$v>82@ z+oHxTSHrf`H|_^fgt8~&&vw=S_N=MS50><}SVW=Yu#2OJO2?nXhWsi1LnxMC@CrMI zWwhX@jrmzrB;ixs%$|1sXD}0fwo&r=*^lMukSt3Z30}R|C8V)PO_)h4$|1?fn{0Se zvDWQy*#kaqjp5y*~Eqy!h-7Q*lfX(Ha+sUT(;{Cpi4N( zL)1hib(2(rMAXxy)PWx|@Pl9_` zM_s&-Td2>Uc>|(QXy3-cM1vx$NZmq2LvaB{=(+I-D7y^E$S!7U?{-wLHUbUCGU7)$ zef{1XR=lY#yk7e5m`~>26K%j99Ztd|H~`e*e+S9dAF@XM$a6@n)z}(M>GqKh1e>QI}$u z;fv)nX3lTc%|8hAo&uUTW{Y^70CJ13?iVz~bhAy+5fpu<#)t&VTMhsEcXM zT}jWP3KkT!v~Fy3l2`n%Gi;bTGnSsMla091} zy5es9B^fw>468437drPC>=HC*9OgrROz`|}A}baOE-`-Dx$K+pFjGxXN}x1TqG9~s z0ocnXf-PPxehCU~}r)rtth6vK5?+#`S|v&d6coL(=d{3mj%5prgh*XiHJ`r8`$mkg`E6GNe_@)jb)^S zc|_9hqkFOpOCdx-abIcX^SpdFR_ITA8{>lzNzyET(Kbdv6pKY&g2bmTA4wsh=33!w zz4NMyBP&GX3-QKwm-aKu)}GF|jIX$f0oVY>pgG8J9^n8{se6xwP5JJ2_#<0(sec&0 zs}}h1fe;!b@_27Nq*NM8Cwe&U7(NR;UVpCuz81_6V=&Mcti3$b;-BPF#{@$+Tglm9 zf7p(B#a_Yfnt$qsx%MJjXM2n@X6^Z^YZuUMvJ!SF;)egDlevv?WZe}Dx`KH0U1XJ-I+Tupw=TF981X+F6m)K+94gx&- zp^Zm>AHtJcq<}1d_5f+lAl%LO;9}9!aS35S*Y!eXd(hnf*9I37V3$W0zREo^u0vBFD(^Ws&0f$;p9$~}9#Rc?*1fPee#nim8 zt9_gtZOKI}SX!Dq>5{V;S=Pd{n!$%b6$*t`IsPM;Jl)AB-}u%af{c(prB9fOH5MkV`dDp9RxX>jm~^RI z7Vk@w6(r>CD@DpXvncn|xx~pSf=b^#*P#`d``Qft@7(uRB#z?f$}BJW!(c$h)E z*{v7Y5fQscrC4q(PC@BxnsWIoUv>hdyRhDWFS1OUKpeW>z7fUt-wP)hr%ROu{kY*X zAQtb36G?Xo+{N>BcodViW(7rqsY|X_stk*I;^e4K1X|2=h&^5UY9bPuux^Isesx_; z6T<$PsCoVhNDF%@*P+XHwU5f$Z;e7kGe`ZtOruz=uLB{w> zolHbPE@aemrUZXI>-L4!UpfS10373y`-ISa=+jKd>Yd_)LHwo&{%~KOAsMmvZ>FjF zn=H#QcNSh&0CLnyKq8b;vHR^d*S@!EFWb1_2@EN5T6RQT@7;$d<9p}+5l%7YY^bmA zY9EumQVO(2(VJD+7GsUsp^`=&Qb*Bmzb);ROLfqB(eX0om0Tt&Qk{!|V;A%BWH(@7 zJ|~#`>#tY0^xdADo}HKX+TZ&Kgnc{^&5<<^^f~|X*RA7GFLxarUS`g2WMwboKIJ(7 z4D_h~=!N5F+9lrwEtW4F7*ML14bY?tJU*B!)OG=`p@5!@1AugcqT-(TUPc|t!pGdk z+!E9O?t%Wp^S4&Em&BQE%EagHPrnbW)2_lbMd#HNUUIE~cz>Z1`~M#Gz2fk40^SYU zqG~!RLmpFYEeW(GDP>Syz6>VB~IApdDju?zsddLHsOCY1?l5_&YFnSLZ-W z|ENCy@Tpf}s6_9|>={$=qvyT`%%SJPxfQ&F7DkOgBs7$~8qf{TpuK|kyvgcdMo4q5 zb3s8jZLjl1Qh*L#t};Z_07*X?I=6PQQO!b0skiW#1yusx+jB= z_Q?03)8}X**!>W)B=>X1=e=EHm7ySvfwuPM=De5Dl?_tduUI||r=$wNUA^bX7{+kj zGBdm^`EoAmfY|q_F+^g)*N3quyZtu{$f)&A5vyWYYGT8R|5v7g-+QS1BPi%p&iDmKscgj1LMd z;9o8Oj@ukBrUAE;^b0i7L807}mj?pu6s4X`le z2s+UhddV44iN3(pgRp{xcIaYYeF%p)+OB-e(8c=m*l)#)vf|0@j}+BVfSIE=zMxqc zC&jCa#-j>31!Z)C%yn4tOet!2w>*i=^bZhZ$W-^2#f0fL@jda<=UQsMwk0|TK$FIi zTAn;u`8_n(Q4YJ|>Vf^KrTZL8NqxUca#P>s3{NCsXAwqhbaFDR4P44e=|KPof8>Qk zkAjJ29Dr{L{6C@M2LH&Hm#6&r98Rw0*w63XHN_Rf`xBSA*-eslZ_6?=PsQOxei>Oz zK~U8n>2#_Ut(`3U)ft46GqWaNe^|giF{`ly!ntfUS}iiZ|FXt9h9d3JN`G?DTa5n2 zA($BcD`57o6F=igC`8Crh4zL<^?1$lE@zd&`%v1l0#oH8PC1^nE0$jt|9djZc=p5L z;zZ70jfPl>uJ1X2>OHNNaUGkVU6Oq2NtryJx@=-yQ^crs$SdLpl2X+=Xzet+{IA+h z|FU4%rqSCxH)nI-T~H9wpOc0fw)7@xc?HmE{;uN=PQ1x)bNZklkFL-3?*M|Z zUgOXEaAzOu?8^b-!KDrig`;>0<~{w4&i&zbG`2M{>YQ4cGdH)q_^ za{@0^HV#=6Q(~W>>8q>onXrz3XXE305%1*JOC4N~1k{s0J2S?d%T!A_WDcfnyhUtP zGI4bz;Lp+3l8{A2@1PWCj<$3D$dcQh#j?!#kgo-m%TR<3grveWVAYajV_vyDrf`jl#S`{{10D<{XzVTwEe zFE~z62tWV_4@1R?Rx|GIRZab29zh!ro`qZb)>go4Qkl=PZdNUCrEJtMv>p6907^&M z^T|hc;`zjb8FKw-6k@`NTz%JGhj^yZJHuD4Yo_I=Set_?>X!nYmqvG(F4X&jE-ga{ zpH&Pav5)$3<8&Wc9|4}ar1&*!>MthNf~%z+FLeBVE@d&iBvHFg&xx8Jg;@B4T@Igqk~*ChC`DO%j-l zex5NO=`n?sg$~WdaIWtlj;5&{9bl_P)%A6$4?BN?Xhl!14d{>9r6`^L&kmxIg z43@oRnTD_<`V+H*37slJ#8kX6;8hZf^`zA}0ylq@$0=A-lx{>&gDmPFKHpI41xOjP zl2-jYkYo45m`9|}Im>szrfPJEk+(2zb_C@UI&d;?V;{~tO*KR6^@CBOd|FTfU)10} z0^WRKN}bxxUQN(at=XvmI|X{T3To1K2>*;(U*`fq>;@s%00r17jTEqZsK2VG$It`SgIU!N4upt!oc%#ILy>F(frs6z?hGy%jc9PZZf zx$-9~-912geb|Dx`>Bh~Ff+)fb5C5%%N*cUemD9xe$5FE7Lj%wvs#nhPJAqb`U)FLN zE7588&p98P$4uUHj|AbF=6?L5`vp&vPs*+ZJiUSlFx)a`K!wo0!NJ>JDho1xt-FQ`go&al`(4x(tPFoo5`T zKD`^$(yuQ=MD?A7YJ&19R{euB%H;)YReafNKowg+85bzl~*xw6iH zx`AKvN0x~W7q%Z)#HY%oa3mtXSg76`0qo6V#N51!rHW7wczgZJG1jzp%_6?-^wcdL z%E|Wj0NA~NoY&AdL~E)rNh4XWlm1s8Im&7{Dvp?j`+5)j+WW2 zUSpmsi}%ZBQj{}khqsoW#jrNCNJ*FKb?_{MPG41WjpQj`8e6Q_=bi27d|oxQ1wg3J z2bR7~0~<85+1fMG>5+16EHMbg9~VNL?oJlJEaE7G#esG1EOE(MLT*Ks`mKW50M-0Y zo5%HM0ah-Dtq(_l=PtKQqf`O_G(^Txu}?Skrb08N0DDeJ=qPBg1`A+IxN~<&YAMC~ zQHh&%&V4>2tAp2F@9_E7<{+l=W zysFKXNE#|>pK_d`4Z<~tvaHgxB#h%xfGaIEmerW%m%Mc%T-diGYPE&q9Camx=xC+exSR#O)mMVN7Q{DpW_uvl4lgnGo$!z5&@o=q9(62{4E z6b0VT9_?BkvaOkdM1I^YJ0hK(YC6gMpl#U~$NOP7ICRsiVrNMn(EV7aIqjYj(tnd` zzflhB_gAZ|83zI0Z9Dc^e}`J*abtV9EueSOa6He?SLa=KpFEGeTCe1S5_xh%&?+j_ z%>t_Njd|VSciveujpz~ox6qrn43~-GZMM>37w&t{?{9a#-yHq8zmOoOe|>HaE>3&} zayzoMfBH7v_;;^iDBDInAD(-#ZdQp!BZ}3r&*PH@IZ-`KEc{Z?$n|&00?|R%TsQKk z#QCpGz3O0+G&l1dqKJr*I@9wUM6}+r)a4-b3a=c0>YsnIjd!_UA}@Z-?~~w`p4$g7 z{9b9yw)&F(U1Sepn_A3rl``3`KA2Qcb2_7`nkUUJ-bdVzN%}|EW(n+EMh-z`r}H9xRzYzxXiuN&w#yB zR-=W~jDR&W#~lXK7L>i`YCFwn>!qTq+pgOCAd~<=#{LeCmQW2Q~ z-nLQaJYC>5KV=jDMyaU9N@`^3Q}%oQ>Zac)U4u2oj66%T@i1?Ne{4$zC-aHkBk0zD zB*c5(juWVz@~=#VC-pXL+`6mv81D$XMAXl2DI11`_{LM9Rx>D~tR-s~S20GfMM@ex zXUIrT<{`Jmm}g`hU(+`pV7L(SJiDH+Tg(|+xw~yTTARS!17%_F!{Cp}r^y705K8K^ zm)!cA{hp3k6neWsIG)43M!SLZ#`iFZprsV=Ti)qAue0XCXtCpk8tPBKG%Lgs5*RWE z^ofHVCbp%ej2h{-B@wU4q10r1V`38X+BDOsKIWMl4OtbIY41gCWm$KNu{!~JAbtVWIIc~4aQn^B*$>femPGzMp7;f2JYGi1mAWI3K zOkCWNVeZY?BAsd)YcTS6%DZ8}6J7Qi6u-=}i7{E>K#8~hD%lddtAti0ikggivtsVJ zyA`2r*5v^yCq1R}^a53rFo+9{wyWs_(Ihg_#5}qbI`Lt471WRoci60-HVUt7GPG@v z^R>n!J&n{j9u92|)oAl0DyBUdhGZG(=oGu!Y~Ol*M;`9^zHP?4^mDbGvtOC<`=w!c zIpJryY5DZbG%7yh4x(stLfedz1%ytOtUGq`vl>V6K-Z*M8Wbv(%0fg71`&sHI@4L8 zY|5*&Tw*z~Eqd!MS19Ue*>+R^2p+0?HXZi0&1bFMW1aY)T9Zts& zru>Hurq3{F)xW^yB;lnGC?2>rHXUn!uQ7y2esOnR(hI^Wy0PkI23dbHC9v$@-OprX z69s!r;c6HdM~jV)7Flm3fOL7=vdyyonKLU4#! zR*^8Tq44=(xa{sZh2fKdhGu;>xp~un0xl1HsS3LIJdm!Eop4!kP#M&dd79lF2$sU# ztwP^1qxPTc^m`j(%>=qr&6?B#bmW*AG*ICh=Uh&O7lIcsGqsoI#>0{a_L_)e!VN-uF=4F5#ZsNV%mlYLnV9Hjj>V zpmQam(+>|h1${7i2#qN~*9oIT%5a$>lLh0mbhJCNIY)hAWA|-E2%LaBwiwEJ*vI6_ zI|N1TDt^5!BPwNsM@9GRgw|#U!tkpXI<9ZT=Oq#PU^m7(N%=QAKm2Zr#6h-#&2y!~ zZPv0;66v%=KyKPcrLH;d(^V|#}Sea&aa9Pc)Hicc< zyK%*ux&P*l7i3yX3&~|(a<+kic+~wDk0%9U8TlA_40qz5*kU+#n;QvEr7o3BC{ksU z4(;HOPICCr{3q?I^5Vues4gcWCkYSvf!{w{5pchj2!9?e!!cu1ke{#|8XZ97){X~0 zvWJI}6;iIt$u*SEmAMYNO*I)=<2t9M8eJH`sCN1b%22m=zM%0>nSW8$ZnF043cmeb z_x}F`;9{5YN1Bs^W)g&U_EXL$rO_NIZLTAx?2ho;J4?Qm{*R|4eaa4P{x)YIU34a& z+g6X80f@&+M|nQiy{<%7rA3zLp!GXO;HZU#%$=_^d6~b=kPP6uc#u?f@8u2kDgU?_ zm}OnQLT2)xYxi;kU?MoXdwT<)oa%ak2>$Bkafi3&`}fp#%5No#$1?3S@!CoUTZ}9F zDuDJ_04)%Xg6>EiYH=;?|L^;9@b(EUM!6>NZ+`=~{~2Vd*f(jTb5+Y7p9+KE{-x9p z5-M8{-;SC`52y85MGo&t#IiUdxV69*Q;#jku7N2&>(Qw-9X2JH&_H-^MhwsuZ;xgp z$=x>spgR#F6aET{{>AdSIG*TRKXZn3jFx(|H`EKyfg8~(?O2$Hx`DfKuPJ8n)gS4Y z&|HC(d1sg@2g%OBu`Z-nSD7%nQGGF*BB0UYPF`mL{8<}}Biern-PjCcVbvVqak%Vp zGof{D;bkE9x)RECFc%4Qmyb^Lb(0CIQ%R;_*^~UxGGA2rr=voVL0iqErqAQ%- z=_+cb6oGsy20A@<%(3)w3gpcH`C#4_%xN}`H}CpkghS%9VIdnC5GL_qK|VZ^TMzeS z{`2o2;bm@X-gZ`zK@2QYQISHo17{+=MHM;@y=2)fXQiNLL`R@qNK--Cv{o9Tm7db%)+H+ZI! zegmrn5V2un&#V2yf>2)d>CguQQae)SPM7guzD~hfajk{h5#XQLj537$8y#&}g}I{K zf?j;7s7Ms9I=GbCRmEXtvR)uPUA7SxbJRZvjXz51(6NaCdm#=`E zX62KdkzN4;JE~(hblrv>Aown^{IvG99mIf?OJrW?WB(0v%s1>2GcekC0i*tcYaGaH zFS7qRA@l0o5&o4lMGZ*_4m;v;y@%T=bJsQCe|g>TpSl|Eq;d2H@v_1JA*Tb~dr@|h+q;%60- zVIfzpek<*Xd@^ebKUvxAj`=8faYW5}NVIybM-`wg9JJmC*u;|qSf-Zc5S%}Jkpz=o zC>t~Kv{2mJSZGK5Zx)zi1O7drcz&A^`#x;u}Bzj2Rs*>`=w(AkLX)YYDY6G{jhPRDIhjh1>bgcsyTY$>`|v(|FkU zf1P9O{~JKd1qgy}XNmw^#Kncpu%k^+eH==i6TpemM7c_qsTCpvq-ZvYfOx5BA#aB2 zFUwW=B+5Et9%s85qLRPFi*m2+FzH37 zSER4@vHne5+%_i$gI-|-3nTf`I48dGF2b$0o6+&n;#DIuasQ@99h`F?ZG5pr)}9!x z7~z{z*-bWkPV=3t()2XoyE9=Na%_y4leOmWeHQ_y$z&FVvNlAKLh_%64{0OMArWY_ zj9r`2Pl(fxq^}b4{kqkuSSs$B`=2O&6{uu?jt6ZtM?L&krSn|D&LjO&WHuXKoBIB} zInNH2|J_Zlju@n-VNdP+&W479{ORsgL3@Vm@V_7bAqUX=4plowN+iX=_WI#BXcRfZ zMoSE%XfU(6zZ-UugKg=zorhuP?O*EdG_ShF=4_75uxYuzYZ{*90( zW#YK#8`J-s23uU4<5e-wkpELN6&vIZ|KBIX|Gp^Q2Sug_)bGql19TX3dwi3uXb5wj z&=C~qP1dd&obQ*pEJkkdG+a}$XRdEn4XgB z$s$_XqHooWug$iK72UUw;>|pSSWD#ie*17u)IxUyemf~bkn=_W0{(Y9bu79JL~F)L zeB)-5!5EiTdjJHSR^8<)uEq&H?#{%L))k%9WgSb6N{2*Wq`qxRU)7EoI)ms)MhM^1 z2GMDQk-k*o{i7>C%+h;KE^wow2&w=mtlSn5%F<&w zx~vcL23{4v+!Ex#)ePVZV1e8;ncIrEHd__2M=nF6|7*N}$=N&0>xapD(_!6g-DJts z#?R71CQdfXxQ}x^fuwEhFcx2R5U+b@20jS^^=yja;lJ8d#`geuIYx1(|8k)FN}Ubz zp9j=pF4Kvp**ms>MVc_Q&EC)18^4!g$aJAVy;VrU%=xsZMzLj~^=I;8E~0?F&sZ7Z zt!`hCu9295`%fqOqf$zE4(Dd{FYU%%yg9`BrWP|Cpwy^6d|^K909g7f-)sJDEeEJ} zD5i42H*Qi0VxRCJhOMKZd*DHZBBt>V&Zeg9_r~q(63cBHa^j@fG>?TUv!JRqlri{B zC{g=_nx_^)We{5I_OxJNST$p$)Wx8qK7gh7yJ{Qt-1;n`w2J5@KWT4_rpNR-Xi7Z2 z%{D}vqDiBb(V!zy%Wv~{21K6T<`v?82L0-+tx!IfUnhH z-ShA5)}Hfzb(l4CC3Sa#d5Zz@=K??cy&chk2(!I|qIk@m(UjvZH)xxcdQ=i*_6ls% zvh2IRSxq&ZDsaI0wtKa!5q1B()~<_ZxpD)t^u58VpY*$nJbV6XqpCAdBqoYk{RwHu z`oA6~0&Kbn;Ymn`KhEbSKY znwlDQo17GJ~3Y+`0#2eS2-CDH`7W=Z?!&BsRxKSxv5us5p@0AL+ITo6sh%s2Fp1 zwy$NHe-tiuhH1K}S+>WNuBktmiI(|zbi7r}?wm5JHv8~hyErbp^O)iWhZ z2n%k2Kvs_owk|&Je-44>|9uD`-cKFuTYVhS+qqr~ft*!Ukq;+PVzdm_RC<(9^rY2WhweHz2n8(%-mOc7Hd+fWtX=xH7xy!+6Z;p+LYmukJcDlx36FsUgZbw$ z(Zh25nxYQ;Q>y@T>z^;3PS(Yv5?A}1kHN$A2??N#*8ZnByOT<5(m3eJYfbX~8(_>r z%yafML+;@2je=ZpjZJp)5`F#L*Cg|wJ7*7RE~DlvW93)?!DazbnWOrKv`EaC;rUp0 z_LY?r2UW;Q|QD@MxHZP2BNb_A}PAkM?>|9Al)9}+SrTqtXgn;_@Yc;r3!N%?y; zFb6MwNSdi;gx}>h*6f#-#lZHFtHxziX(nwY!t#q$;ENO2IMsl)7u47VEWyo ztv^-P!(3^RiRH}+;RJpaj`@)moSz*Y-LIb?i9*B$j|DSR+Fp&=o zpO})#-aS*S+EC_n-zt9~Rkx>sIUiybC(_w71zz<(c|YwgOmlZdOT-fT4od|D8lvP? zY5Yv7w)JIb6nH|Wts}!e4KMD6&ohADOk@&}qi|hLFCqRr2~EU)H&(lK-N#90lK+q> zZzb34)EX_FN<)^dpr>LKNdvqzFtD0Y+)BV`yKwJ`#Nf%{vgHM2+n5i=kr++_aWhgl zI!^K6)GREtFu0~-&@#@Bsp1n3<@b)J0>t+CZWsyreb=^@$)@ClfWJ1L!6j_ZX&k@e z!1PVl?jqs1ZyArBitf{p>O>QkCfs$M*~|`8&mNZkhiy~-1$>aGZUpIzgv+5BxRq4R zCSS+**{EI>ccvI8as&c9AGWP5zl+EmmcWy_6~I!fk6UoX@2490v6QShXpW)(Gk(Z|kmBx! z0_TU96_glA%MfhUb$_TUOV~NKdMV@>7`}o1l={uCl+){#+tC5CfPg*;6Y>TRy`4HC zb66W`E>CjPNI!t%u1=sJq>m0!HUNFZpwwZBSWSR4-E@rOCvr}E0} zO^Q5-hDjR0VzsCw$0Oo)!`D}^BLbnR2(%2O0g2nJG!-rJ&C%5a!8D(>9y69} z*u1Zox(p;7<|2}3OO6BCwtpHtTiN1SIzGTS4Y!!JB9Z9Cr^!br#!m~g6i%TcuLpEi zKuuwv`6;z0p3nsUiD&KF%NDS;;Lkm@Zjt7)^d`Z#eLrc5qRmb%fvCKXk_P_QVylzR ztry?GSAEnxe@pug{RLvd-$%=^^6z}`3(AhlUz#Byyyv?#1)5(Vy${9%A1J9Cdhl5$ukupnn zKJV(}OZOGsEgBTTVi;>(*d`OCUktJE-xsg%f`=2ayAMBWW5`6YPz@=mzi4uhO{Og# z-q|3TN8yHE8z&&0Kvm^m(ONIj7=dGv^Lr%BqZs!t6xowW_G%3hGXJ&4+jPW*5ax zI35H0ms!IevC@9BBDwUJqDIg(dFD3z=JSfw2=V>0}qB$kKmhmPC4P?HL#(8v?w zxlMwD&QH%%-H&7K|sS9MRQt2)%(|^TKTNpza@c^L!FNo*A zI@NMjnzB29u}@J81b;*0;lZr##0ts-e-e5%(gYR~6}rd^M0FtRFP6`iUKB%Ck*Jfr zMd4ii#2b;^L!H5g86s4zUME6bOGxjA^!~J6d|Qkscv>^}XuoDyYSJ91Elp5r& zn!G>0u`Wada4Mz&#n>|bM3-#i(R{HQY=;#&vlXro6gt@gk=v2dX_na8hTKP?z;U|} zZ!u1750v`r-;Qqud+n5(t+@;Pg7$dCrMU?h=I00rz_CHdg`WIPGn60Z4(c+N&tMC$ zebo@&2o$TNZSXsG7~xepc+%Uu3fs`;G3-ii5Uf!NGS;HW_L%@*up>h9Ht`E%=N2ukT7J9g;2nhZOMnVk z{t;OUgu)oH4g+a_7a(AE`gvMDkOZnt?pp+zvAy)0Vo+;m1_|;|;|{iexKqlQP~#u! zYgKN=cjU@KTJHvQAnv?^D?N$rRec?Ut{Lmb z_@wIr=jpR_SnFIkCR)0fLXmB>_y|Z6d5zXf3Gc%+kj0E;k){dH7jpQXf{97*@i$)L zV3@APsbI2c4z$eByeiVX;lq1kUedy_|L5?t&4)HiLtFbQ&Bg;@rlW-Zb14nqFqX}$ zCh2i~K;q>-RkHp5`-i!Z*V(_0#0ts~ciaekEz`)ZK665vvuogFN;~6CzVxZJ3qRM% zXPLib>-Y6}i!4#``V)B(n?RGh+rqk%Bw@_X|4)C(Pv4VBtf(EU|5=vl?oeVNXE`FH zkt@ju`~ZdWVh%r+#i3IyOyo1Fiw$k@-)^YrYk!J27 zu?8=fJM+m#`kV3u?`hFjI<=|b6UGwLP$5X`%gJB%hCd^FSeOU0AQk)8vCk3 zss+*TaJAo-zkXjtpt7ErYTs~zu8g@>&@C`fgS;v1a_l?f+5(Vm;tbO zzZ6(YV_2xK{A<4&FTi6povdnJmj+^LYqZjQwNUa%+~!*x&2WTNLf2e3yqzZa?*>~< zbG=gBxC&wroG)nZGkK7InQrXBhdKeVF*rhZ z_{K8_-ehVGM z2QTQ*2s7;xzKq>n+wT)yj$HV_ohs;|JODPIFpnf>8|gpI+2MR{`SU)%Z16WLIit$^ zpIw}yO?Pu{6hNWbh;zDjMADwNRveaoW=vgx@ho{ezJA71n9+^C&DUyk`BBbZ9{QqF ztbCG{SV=PgGwI;yU~%!|&ov=jDbeYCq(s`O?495P^U0h&@dex$3lIvgq3+BUtVid> zO#8@Rr`FD$x2Za++=Bq~Oj8P7&wZQo3ZV0nwlm*&LCH zn+G6;@)e}ZTSczGS~Y6=6M#|GuyqB_^C+E8#!kvEd{H;`|K2Ls0L8GS{glivatSey zA`||gQF44G0xrZ`Fy{2ttxR@@UW+nRlU+mp>+p9!Q37<}951t?xcPtSzU^C2pmyRv z3t$%dQ$@yF#)kmK#c$}B8vH}a)JBCbL$5|756ow;kFQ@#FCHZddkI21xvEM~Gq~q- zjq;f?Ix54_Mc6E$I?P?gKzVADaYpaxJc4F@vEBB>jD?SdsJW3PAg} zTjzTOSY77-7VFr6m!70Tr>%*8xeGYlZGJ@P?lz>g`aO_G3pKgckf0YFT{n^wU;;T- z!S@ff%By1Xy9H*I5gtbM85%)|hEQUkG;9VI-a~zy8QQPqUz}`9fq^qBw1JHKrpEW+ z!qa1Vqw7Vgq=`S0@z<6#2|TaL^vCeSS)qPcJ=78jp1ZCihQB>?MAFbhm8bQ|EnyMzy-2_ z@v6zm%2YZ|Hl0p>xS&E_CZh#aPY#g+KPd#qj2NkAcY!m}kr#3FUc5Dqwlhs*%e@zZ zzDD8+L;mctAgc*V9)spDVAkRHD_5V-xzXI*;gjJzGI0k`RW3Av z0@sak>&v>lAstU{yyMpWEcA`R=OGoQ$@d4`3e;`iM=*`1znv{m)) z9prV!EzjVv3fM;$Ut5T3Z@Hd(3DKMcSy4=Iyy(4rXWItaQB`h|`-RZ$5{4wS_V;B-o?%QWFMONM3&03?_?K|T~Qvu7MThE55 z1O4}faNmP=HrBmvF013mv`@bVQ7OH?NyCpCf*3em2~lv{!gTNd>2>Obp{}+;^4p8c zm8;g-b4aVG+5F>1$lE<>v4&H)Y?9yZ-YTdixnzLN4c}nhigr}|PfK}nH6d66%;0IY z=|W198hTI+w$NZ*HoNu{LaS{$=C5VTLCXM&J0dZqL;e}q=QkJ6-431^n}j16XxKTAHi4H zlYy>I`Fi591=v(XqBt9F>|YlF7k0(TgRy%v@Xg^=W`8VLW*~Pxg0brX7IpPh5@-^f zttSdbnagEMj176r&)V9iQi*-V;fSWATJO9ic_u5qRTo!tuE`4`6HApe+K8U;xS(^L z9-no+hIlxFYz7i2N+oh7N=?_6wc)CT%_~nr3LIc<fU%2%RF@^hSD zxo6`fez!P>q_GqDhk(H1KltZ=a(bWlx3yykBLUpKrqB{P9B~lJt!j;i=&7@;UGMBD zMP%xr;|uZn$D86d+X8+reX0paHQPhR~2Jxw!PO>NxRrC&6(W;-Z___nE{y zny&g+fXj+aGs8Na`ADvt9XbEq81423bE(JfSEoJUiZCOxELl$mi9|7^Gw~?yh=#kQ z-Jg8@Qks&G=G$nu=^noyFwn2m3dQrrcBagT+aEbzMl$8id%Xi+k;M&j9sh{c8wYJG zPb{B(x8Fh$uawFbwL8cjAh=zJv`gwuf${LhRHYSx!O9}~K-RX5+TNr$lo@benRbZ3HW$dCE$1v}e=l8S zGPF%W13>dLr0(Xr^wfL0cg(T>cUB14SQ$K=NYraFU9ryrD z660tV>li09z-juVLl}4sMcY zc@VUGVCR!Z)q0zdC)>u$GuhLwLXmK&l}7)9ovr-SnFWzqsdRX_6y7I&P zNR3jB=NT3ldIz0gp#kcw-CgAPo72#onsOCS;|KcjuGSNZ>%QiVsTMxWnim_+l~26* zHm`pRejaDvO}5o^NL8(53F~;`5lVsN)(+0ZPP>4Pr&KG!G5!)@C_@<+@;*W@O6prl z`i`YZ?BAq!3xfB~(I8u#PUB_D>aA1E)~OuWG%Q$e`iPu4fBDH17>=Y2+)E<8^DG9N zU8#A|9AMqAAa?SGx2YlE>x8ZU7zFjBbGtVUjw-!~=lr=G`a62DG7$=t8E-_))Lu`J zV1Kr~>G2J$WWxp)Gph}zP@v#+waBLDOd1g#yGANcA7RfT30=~V9zU4}2VVebV^Sqq z4u}RxYQLbUvKsS&{h*!~N%pqMNXk?X95FpvH<-g3D56i7BTN*f!SNQ2RtqU4{+=g| zsoHQ(r5FdjJQDEj!f!xamNd|hqzqDRE* zM~mL-0vIU+5DE{Nb8i!LJU@oLNf#y3D-QXNOGd@4#68E8NeqYaO#-};(G$N`5`jNJ zb=yHWnUvk3$%?)BFh11vCS_(1I$E`2#0aQmrO?4x;r+aZNoF*Oe>Qa$%l8MOw%ln2 zN~bgg)ARla4QM@n>sxGi%a8$)&{yN=hrs%}&%e>*y+L_d{<3`6x(RxgX?<)nr6Vn+Z&?>N>^c2*! z3QlFZ{G7pN_iHkhLtUHRMJ3{QDU7Su?hM$o?|w)`gPro^+r@xr$G7WB*Q}Gj9@~aA zKUnmBnzL_xtF9)9}#SxxHcb(NHftbY`<0Y1N>2wu2SvAYv!8rhyj4$+9<@MH~cGBJtS z-8SL9S4giN>?Nzfq7XndTF(j3 zOz^B{VTMU8ztlf`LVWH<&XJbd`8 zl{Qh!aziD43QBELm*9sTiyMOynyGw^xt>ji8{!PRG=9ng*1esO#Z`lu#h(DWOwA|f zrKARcUWm2+i|7Ac(DWVDu#R&DcD}uO#{1#0fUlX^%_sdBJMRr2y6Bi(V#-o8nB^`i ze~u@Ra^Q14+p}82fn=M1ht;DA+zW|E8$jb0hle0Ql)PtI2WU-3(xl+F1S@giDup;t zFh?L2v}JHAtc6dud)3BZLTY57FN!c9kFrjXLl67xJ0~nR+c7!+u1-@BcpzuDnEoogA{?iZpqz*8ee!E23${cSNGV6NZ%-Q0|k5T~o5*I|Vbunw>$wn(R zi6ELPc(1)=O8G2mjsVz#hgwSQG%Fp&dfRTl3btd;#`WGroF!F&fuw<%n9%Ks z-INH46Y3^Q6IKsP+CV*?d&=)DfSD^A9fEU-Gny{pea4yz&&Iv2-+H0DPLqt2jaG)@ z`3Ws5Yu)|zt4Gn9qiW^?N83+V&IJp04!p!D@#^C6D1*T5Zn7Te+x360Ns|SPyFZ6?PU1ygyGUZ6pdh zUvfCH?}-`eto9&uf7%kZmR+5+xAatt09PD*6f1?sb?$%=d~cbJo-VEFG@=8Py-ADoa4ekPgU`b*oa zR6s#;lSTATpIKZ^#XW{ztZTAdOqSfq8p_A*>H0;u9+ldBd)no>`U~;S;*4fXIv$Bm zhHt==#KCWDvvi0uP!%Y(;6oF^W)J577b;q7^yf=$&`sg~#ux_m1=uoh1>X8wN)X}h z>i<}kM)2=>F_O9$p7GRThwmpaw1JHc^ouLN9 z97uUh@xy}Tfal`}68-85>dy@t-(H_aROO?X@ddoNqH#Zq2&~rnEElEfN-OZ+s9M93 zo<@X{P3kiRn8Bas`!IDqJ7}Ua$%_-v0|QFlc8=#MO%G^t+q9Pw30DekL;nkb6B%Iq zCZew_NDpuot41RvQ@eC$A9t96sByPm!+|GelF z&jHv>Yxbcmz{L^Ze>2xxzo-YVtyds@tuPU%&UlT@C+N3Izv*~0!1JHaAIat`w?6ZN zncNo7%XjpNH_|NKMns=D zJ_rTowqEXma#8PtGKf5w*Ve08V|7>fU7#}_o$xkRbhr*~*<=0sB&U?l%_0@C##W6dAw~-F2qvhFgK4^&4LHRNKEk{%~ZJ4 z6;eR1_!FYR$Z|YC)DY6#`|&zoI1pB98al2U@^_H9%;Yd&1PkL2@HG>BID-f z8{wHk5r)ye1G5$3iH_?>!VbW%Al_FkE@hP6?6r@`3)UbY0OGw7(@lWf(mH$(M%#?iU*l8SBM0idO=wNMmHua#zKWYC@^L-zj*gFB!p6 zKe2!1C6l2=IWQ{$=$Vi=EVP|-xwm{EslQqkgls*1rqa~$$wwHKSTk#q&rg zuMP^@w+QSIH9jcMLzznuxfgOlt8mBHHZ4?Y{5;6C>x^%lKqn|ytI3I?o`HF~0yYtU z@x;}IVFRQ9%CdrSVHyvi5)gVS>Q@esSAy6T(GOZ5_A*U8$mThD-=42_^eSn|xvW4Z z_dBtv7TLr;_xbu5G-6bQ!oAV&ZLBbNlRGnX6`#HI0h$)iUm8 zI8}~)o4u~fk_iicwwoVC?aVKqOL8P}*6ov#2K_W!saJV> z2F3mb6fM}Z#WMDR&F>|Db$??33vED0auY2SDC=}yi>f&xuXC8&MFpj54#76qtD2IU(sLK^3jDrasXbvy5S)ZJ~OIjRsjZB3~t6}>YN_P z=6ds`WpLPuquuTL`Tjwq)gC{P_kG?HYQ2k*-%hnfTUQv!cKY#WYkn&Bi8%;>3KiE)KgiNy1O#om57xyl#5+D6#P^Bi}xMIeOv zf#r0fa#zgNvGi-=N7V|_Wki=w@7jx`5aJogCbW9rTRFy?2aZ(56`~|{aAG+hK-Gz3 z(+F8T|3pmw_F5-|Lc~SVX#)}L2em_xt}pWRTLj~12&Hk~036!@i_1XtuS_QEZ7q-Ovnuv+W`Uqm(wS&e5Xho6(v@1ZGaeX4q4zENUuQ(M%~;(fViSOe=9a@xBYO z(qE2fw?GU41O$l6sd7@8|3#Wr{<9WDv6=@&7&H?Yv)^wvRNsfeZI`VtCZD5`+RP16 zKELN9e1)_e-2W)#36Qi*?&M`@{>@k6uetAM>r2VN3+VVf>_R^t%~gz;*8$>P6-Zlk z^XHeljS>}w-NuZ|bT+?`Zg`%WwD~aoPms<4%YlQ${A<3*bsv}Dk3JBX1EKp`iavyh zQkMTu!`H=Fi=wpKvP}FSr;LF?fyJ<(HTr3uFVeqq903hT48lvwPma)>Uw06X9_Kji zK>Cy}(g^`iPRW;)r6Od7j#dBpiRXJv@w@gy~THg!Sp$#a_63Gt28>tSp`Dvl}<1aAEXpRwll`f)W~ zmtjWkEmk6z(0lq~diVB*C7Zz)KTRHQxVXS%_!uc~w^D?=UG6iQO-RXM9Jg)U}~2Sv(rZvEOzcu|}Y&ZAy{tMTUnHDwv@S~1~70PN(vz1G!eq5@za z&-aIzmf|3xf-!ZXjlj{&vn*937BX>ME8K6>7e6oy&#gYHLOJgWJmA%rzz(ezoHY}^Rx{op?w2`NIe@%Cxa?fs7q&*4dyGH(`>3^umYh9c#X^%rQrrxEzViY6t}E#W^&jZ&CAk7fikL-Lxk{L?tcTRagi=) zvFan_ez0wHU(3ml%MDP7pfhVjFvY`k+_r*kA1Mwl6ECrZ0<+9=+(W9{FQqZUqtR^J zPMV&>HmEazfBb67a(ouM4JE(ZN(b9RvtZfc?H_YHqPa$@*z@HgFYexe$FA?6{!*^HEkk;_1F9P_UZa6)~HnT1neZi zq^o5W%kzV82DD6z`s4Q0i`aW;M)df!1&)6(6E11Z1n0W)*I5xE?dhP!w}!_6|4d+A zf&Pi_vCycDk*PH{h^8NY)?3i9aJFll$oa53N5^8q+bt2&bj%7m1AenFVt3>n-<(=u z>d-v^7N_0vnVfpwP&F5INdP&HQ0My#yY-4o``{(tk<-Wt+hqlYYRxN=(`9;zQr`=B zXuKd2hoKN*hswUAvk2d|Bm8!)CjTlC_yo|?8O;7gOo7REYqRQ47Y0igbI0ZQ71)22 z%Qxd~7T0J6g{#lCq6dOLEd2DxZOhbkpfekUosb_I@%)+55aKl|~kg#bJ- zSyp-5|2I?_5iYj@;@bJ8u`X)gv%ZU0;_hszc#Myu<4O`BpkjS96}=)(nJw6Ao7~Zf zao1iSG75X>vzi;u! z^F$#DUi?^@1QgbVF9gt))zDNhO+$djVJr4P=M%_N?|z@Y-E>pffmtiT0Ur63c%XY> zRGaFZnG}ur&P=kU_Tw=BC-S%<@sq+wsaW>=k@L5&JcaQOQ}VybLw`eM4npk=#HP}C zRB1KKIGgCY6v)qSFK5=hH!k zRdqc3>Lk1u@TdeGmf+$wJ)QFP7J+EGAQ}LOv?qGLF9fo~^G>%0dohrQ)Zm5lpS)%q z@ZTBLtb7)|nx*~d{akb1&-!A)2iM`0q<5h8T<+P6S8{&Ppkmy?CJS(IY8P^S;sxBz z{Gg*~FSnm5+~J1Bu5gA$Zs*+1#*vnF#6yW33y;lBz(RpGdT4mkXjiD2`?}vTvOSkz ztc$NJ13U$u&4n1eT=ws0Vt;BUTIdRjpUHuxUK ze8L(Vg|8KAk=CT>hn4SK@s$25f7o=7lW*19!f4K(e(Sq(NT0g)vCw7&2+;e&dZfDK z6E@ev4D@6s9bMPMamZvpdpMu|<5sOx7CTxk$drAVMZ-tQ6PD0DT>M5gxpwEA@4%d;l@&&z#N3Z8xz8%MA)z5mQm6=5^lfZ}Wu4F_KWzd0|lq8#7PMys%F zG_s>1Cm(aW`Ca@+s@d>;1WpGYsSK0~s>K6!soJ#8c~E-{d~cauICrM-RS%{hb1 zrG0Wc;~kX^7E~8J432%|JU92V7f_>#QjvbsRxj+zSFX-Z#vF8>5c-6}ncPyM8c6zJ zZ4yd~oMOVN2y{UXkerTNmPe!baHCl^;#4^<_Ap0Yd!2PMThw|E@_dT8MBUcqz#>^L zg``(MrQ>$Fg84TkOB3dQj&~ezykD+o6zao4)!qf-(Z<;`m#s#0xp$NUsdocfYhd?ZT#C&8XMV%1@0=Aqquu$5*?@?KR zP8w$9u{3yeuvEfoji@;&yTPV+8@^pyZ?u9ASNKArlkz1Mh8a}g+#3pLShK5BU&&a$ z%P?s41qu9j$yz_%GKq@-%LaK+G_%j1MTVfy4TD@JpBSWZ8h9#iO2Fx$PC{wd@@pY0 zq))Q{VayDrLXlmlkC;(9j>^;URni7!C}&A=JGPYAIz>IM!r=_+)bHjh0Z41d1g}Pp!>NYp5h-)EN{D9q zcuGa?&g^S?JX`nzqNl&>k#;wPU~&~7%I#Vt>fgo=RKC)(f98iA)cU=|?h>`Bx!yT} zlDh|wF`O5z=E+%tNyKJe&_0~c_!-Q$eA$i{5AA_wc*0N14)MFd7!G9>DLinyf>W}; z?jd$>uh=De`?rg7=4b33P<@N1dn5=%Qgt1RWP#t;5v6M~)cY33o81Xl?&XR2(Ct*W z5s}czo)uWBVK?Bz%`wkS->O**e#)tS?9va_s2eGB#)}eYt6claCsI{;kKBqqs`shw z`z!M$Z%&g1_E6>E%rNlz2!!6Nb*hmPRR-5Tq>4kh0MX`t)gViaZKZ-ZsqMeAZV@`!5p@K6A>d`-Z-iGhEQ*5Pm>dmpr z6~5qnN7FZAwk8E~MuSKVEaG*bP=uRe2ziGaq}>pENGdHt0L=*TP)y?B-O-#3_NxRD z!f_0MHO1t3-)fW(BK^2KSy6*jhqa1vbYCmOw`-4Q-bN;q1_b0bJiAFM@KbisZM80_ zM4C||eUB!%U$Kin(tSfOHZwe7(?QmSGG9yswT>jyQPJZb2K!;Vw0%)T@#p!+oVwAF z4}ikqU|tvcY}a*nHj8O;j7SvR3{JaF-H$j_ZT8FZ(2sbA6KO3Rm^|(+JAFfyoj7lfeAIKkzfyiONW%A_>$ z3(mwP4pwOGaX+KtHAzOH{D*jLlgxlW=O8;{mv@BC_~*dTpQ!u;@@HSSs^Z47e%J{U zhuZ_-yRoY;2{a;n(*DsKl8Wx9D-8(SbxIg@;ln5+L5($tq-DJlBh4g86X*!oRJmJ@ z55|c1XL$SeubL-$)^)t#jwb5~vcKQ!4jr)Wl>Yq{&VLS&B!)nEr9F3-jSH8RABheU z09%D~5G-l_#J3XpY|^bbU`j@JJ6if#qIHZs{s3^l$R$IB(6QTV0Or>}ws0kL$;IC5 zI8js#vr}NG!mOKLWAe}C`cC=|&=hN)04n{UKX-K`AfiD%p9eA0w8igO^$J{D&`Xs< zIOzQ!6zKtiCWVziSBAoeN2IkV%0_Fv<6m(y4JaqOwl{nXPz!tY0eND06Wv6&ofMe{74BM zFxhS7O%2`49DqbD^tdg)dF&5I6xWS^=@*D<6sxm=k^Hewm9-$m8q zi4HXsXF-0gRSu_*tCw!6P8l2gIhs%Y4nAn%8H5YNG))}f7!1?FR7J#}HPq%a+R-yh z0QL4J2PPgc!IQAHBpB33=Kb?uTZS+`fBZr+sXDZIb9)IIeujVc6m>m=a-qTYt5lYH zLCQ{^VIxb3zv|}-SyXkQbCuDin7m|7UMrJ8SRXgf#l4{qtq85O$|nkZ>gnk>27fDg z6ZFo4F23ijJK{zBM;woPwgalZ{7N;fo06o>Pl84^nM1^e1kdwsi?`Mp{kIQF9I0SU zoaff4}a3x_%WD!=1u&X#vP!ZZyv~l8si}vswX@v#BSo3J~0*! z0|nD=TiIk1DsDu}OC}A3eV1G~#f8cDHp~yZasKR;lWD;>o&G5c=s4QnMKJ{$G04!` zPhj*?C-LZ%54OWfSdie%ukNUcKQ<2P)CJPB7*<@(0VRibG!Hu~e4fVA58O`pVIaLC z&kz-sNM0HB9FTOPj#V`MZW$>?@f)Q1kpxz3P3MfULF-H`8~0PDdI zLZK(Kb<0-YZu*QKVYWi=qm+CzMjJ^JX)&`6couGQB5Gqa{k=tizR2_<_?LZa1#@HU zovac7@YwuGcl;}%Cwti^HxZoF&q*rWq1BlkkVQE^uc3D#m%uK4+WCW{m&9-YP@rA& z2g2C1mBXhD^}KJAWoN$Vqhum}S`2}?0vdB)p6ByElZD~sMyts6b3Zt@>w~HKpA_Tp zullr<0k*X$GyLbmA^Ld#z6@JIgU{i!4A}5}4y!dA%dM&7a$zw0FwEnyC>K>qqkz&ps7pS%k)<3O~K$0;#mhvmd?i^@1R1efH9&Rt{=Uq!M9F3r3qm z#q+E@tuaut{JDWG`h_6@OB|fKIk763M-NBKW!P&4$B)bE$W;Gxt9o8&0608$uVxT# z(tZq3{p=mJ7TgB^8^hGy*hIAYw)n%EClG}f0R)91mF8;xPccVEiihV%RBHLQ99b8oG&C-KD-~2fX?O8hy z{&5XIyvu2@j0ou=3BXt6AD!H+-}%8jH@Y$>2cg+CGbAiJS+xZ_@3Jh}4cg!wf0CNp zRWCt$zz$?dn<||3KL!TcCCJXk?-oeVWcGnhqUkoaq}Hs z1?o4IzsH0LHDE^-lA``nz)Us4svcJo_TzcZuiZLINF_57xSoJYS_|wCOPn`x{VX9k z?I^f6IlRl={uxE716*;K1U0Yt8MSn1#+vx=<`DK>k-$cOiGkm6gAOphd((l(eXS?>i&+n1PBQvNd@#M1)X9CB*DmYL<)!xcWlOz@wAuxcclkk<1PwN2-1MCZNO_vuJi>DlJtbDuV&V9BxhhDXI=SjW6#8Q}}-X|DH*r5i|QN<IkEYF;Aj<7XBd-2lXrn|7LMPwq7~Ms z|JDHoak!Rlj(O+tW&=IyhkN_UVS5JJ$U;q|!HTz}O;|(v6i3-CQ z;iP39+xJd-vKi>t?Ll$A7Pagn0tUZ#C9ZeH2aiR|htZ*JLgomx6Gy<6Y)~#*ENNz+ z#{~3~Pl4WqL|lFo_E1v!Um!bpRY_XG>ZJ%cJ^AE)`-S$DB33dF7m1pi_Y8@2ND#*{|Mv_iCHFo9&b#S! zjsr8`8#FKj`mnBYc_1>4bbo|$UxI&^{cQ4((=ee$5pXN>7%!>=0}hXSyzSRta}^7U z&BAY!dGRwAC~#8|XSFfFsza24E9_fwp}WJ)!<-o(0G*;DK&cj=T-wme)QU_S`)l}h zt|`HlG=6@xbrSi(2Z(>lr7J!)pwXga*F-W@IM}-z!=o(k;_8g<AUn{F4D&>Ja1GkItu6`R(%bz(&c?NYD`j9D^Mc+i0SNVkyYs%=at3pdy41hnt~Je z@PnT*rP9q=ilJtQ_v!3qt5Vz-LI$4y4_9v)5M|W03&SwL(A^;^0)o;FLw8DppoD;Q zh=d?RNr!|qNQ2Vdp(5QNEl9T@((&!_d7tk+=lpU+nS0-R@3q#o>RQWxf0%!WzCL~* zX{3*&Y8*`IkL2!hT%-qO?j~n{7nxrE0F~TAey17k!;Fgv{*E3{h!9mYE;={YS4Ec% z>P>Ty9oQdRPnjsk6Ez6Y%4`eX*N_cdJXq=@$5l$5zy8?SHz5WbrWZ#q>TeU;x?cex(~SOQtah{wJapggOlUw-Ay^@g)~cbUD6 zE2|3yvZAf4YbFS>g+&B|(gV=%F}<;t=+)_kvYTsxM*DDAtJUksv7f7ElPfc+7DIr9la-U9uAL=e-8x~g;Zq>%j? z9W&a%HT)ha;JM=etv7kn!%t5o6XYpUDjk;cT{g&=m(az(OteYGX3~~@`T1or7HafR zsrWnPF@p3Ac{QsoalReUYQM~yU{TQor2m^sJ9%;DM{Kh`nTlLI29DK2+ecX{PbaL_ckgV1dFL3kQR`C6nG4=2Ic_67In*Ltx z7eXC~KIn*3NM0AX#1&k=5C?=4TM?Y)ajcPobx*^ubGH#J0s8zEsup^v;>hl+nsH=Y z=l?!uQA!==-oJU2TgNgsNbU!*Sv7lh*419BQtzUXSN%BwM=FF{BvGAsSHi#Z-9KGgH5q#~Em2TqcJo5z|CDJwI?+UhC`wezEzk#^ zyBZ^=fu!Ao|3Y0wunr_oSZ3eK9!6&{Plyo3R8-F!zpNUeE;U<x7 z()+mr9HF&bTK_rTueE{-{|;Ag`LPA9j-Ig^QhOt1`hVg&T`i9lC~!_jbCQAYfM!Nm z;L(G@#lDc(=EsORVTfdRcydC+9l4&+H(TxZ9Te(%_2tjlm-{j9dhAN9$JGFL;Dq}b zaSxRTlA?3SNRh;*EZ6_b#<=+t{obM4FZlCgv+wkC?MnR5EIZz)_X5Dlp2GFxr>0%96Q}jC7-pZ@m$hzLlBTv za_dV6_7l5It4JZX4P1fTw`+vuB%5L$T5H>xlKcD-bWvwa!DgMk|4=yEQ?ud(6yq(n zO|cKkN7XWTnrHox zcG#3gtv|9JZ(Co6Mf3W{_&ZdN2~W^0oAc^Lc9$dd@9sKnp~7|=+y(k17;a3EJ2DhT z#cXQk)oBe%pG8dS56?kTmkyoalcj_bcUmXcu+NA$1r-7vw&|YqML1~K=|4SUsg)^9 z7fy^rSpztAebg5u|rZ1I}go7@A>TJ(B&)Zan{d^HEPSU@RkQ zmrAI0^kD#`^yxp^l%}Y~V!56+>pF<81d~{?jRaBxd5mfRasZgL=t}?TyOxoS5r=9y#n0G1BPw>#ol9u4X7wZOOt1U!hiC!{$V5ZjO0QJQFvj5k9i zNh;(~e<^<71)9qeY~i8Y$a&oNw=Ci|2S$sP&{zFxP=q$uOGe` zkCe3;t!A|^i03fkeRuk)AvtZ@NVP!4;rkQuFEsKrxhw(q&6Iu5Q2Ql$-BDZ#E7nVF!1xH91Nyr1)l%;iaue62-@_dU?o zoxv^M%8~ing^b?2K4Fp-_^mS;^oC&YAYl98s{ZrhaH$PzLR5gxt(Vc`xR0%6ovf1C z1~B=PWo9(+5JcUR%Ib-tk_GeU{NszgtHa^(jkxJQ!mLP84<=nIFn&&N3EOR|h)UJ- z1L&%|9)O&ZI3$z(ixXiLZ{m+7?4Cj->^R!6`hN&%iZT_R$Zn=<-8+t< z_YNY>8Nd<8$_epzb8@3vEgQA@UAUsnglG)>?!Ckku(3eWcMV!QPJWob#rmvYW?cKz z|4RP1GUHKav36du%}CY>7>2F`IMy2M-Z9LGuf6-ZUG*Ht?y_URtBA9fXSaCgzTt5|WxrC^fXUMbw0_l}LvU^VcL5tHv& zss5k%!ZigN=0xvD^Pt52)m_sX>P*kuwqE}k>q+>T&3VOEttpIUrfm>$cn zsYq9;nDt{8rVki^F~YYS_JVtEiwdpv%?kKSgSIACL`7 zNPkN4%_%G1)j8i-z_Lctj^6fUz0E{<(rcXZiX3LlxBFa%Yv3fkm;oP3=Y_vJ`e-Ak zG9N|7M}#H$#E*`LA6Zo&bEU&}GXDr?cn89o$EvTwC-*=Gwg{VygS0sLzr!!+9eAQL2ai<%hh{!LohVE>i9@g0yu+W! z&4U}+m+=U%CY2|t^cL<`aiip_LBNJq2jsupq6NW2wD#eE) zl+0aDa;LUb>yV+s?=BdpJPZ$vP>XYatOSfcUzx)~=a%UMbeLa^Bn|3io~+CNW*Q*o zXv;|LzeZR3VmBpMto`_Rn1}Og0MjBLc-NZY+x)G@|6AqrH6H#pqALq$y2S6jyC(u{G4Gm#9b=sZ)c)oDdqf@6Gs^&y4+bTW(L}3p z<#c>0k2^u)-|(kCmE_0}KmE2Ep@!lbuNlQ3ui~+XiWrb`KRlSOXr6*9w$S?j;yULO z!-v^w!^!(%lDIjDcDFhXD(L8E2W^K6iDN+#2Q3~<96N}3CsA68s1J2v?N=iDGH=op z4w2h56Dc0zm%6QMbu1}xPf(e!)?emFp-4YD)g{;b&-bgG`$kzeV%!r;$7PbF%6x06 za(Gl+bAdChGitw2LH9YlRPtvWn zQpQx}4aba#0I!HxcbsV%LX9~oNQ%?|*6h+rj~ zo|$dJ$5**5dF|TXF=k;CTu-i|v$T@G$)O8FHVrVbT5Wf!pNAOaqQGezzEe?x2wu`& zNm(o!@aLFhkrd!e^KmLACo0zcr6jEXr2yy7m_fL&^~OIAP2;zd;WDm`d*h}goJpa)B)RCistiI`Y zS!iG17)X=M z8SLz6_o>)vJD9yx@7VTVyqQYfE;j%Jamp%uO3q$QwQ=+65!$Bk9l&cUgxM^oW%T?h zE(u>fm(QCqDCDl3@jb**Y`epmDC(a0^@#rvwV_0RXRavAYX6k?!a*;5@W#37vzb9D zmPy#gxnmh`Dw;#(7BEDKN~XIWVkA@L@^`C z!|i$>GVesf#SR+ol{|QMl^;Kc5OkW+=(Ept(T;?*#5|6KHFS0uj>#vHi^k+M)Yqq- z%%S5DJe*cxO+He7s-(hnctjvKG)n&oBrk|kG6hp*aOkdd-}!fZQGr~ z*Oxj}nmq{(H>vV@d9xO)9=rA>HN$mW(KVoMzsZtCV%E!kk-oJ{82uoRD$*!8cc#Y7 z{Hv+&O-j~N`GY5|lAN)ZjjmfviyF!zZ!i13UH*ROsQgQAV3MoXsJ>X=E-Fz<`Ly|~ ziq~&1N4y?|nvSNPUW4Vj2p;LDUbdwlaswI6#(S?4x0w#-YVgRi+WmF}{zkZ2h+Cd2 ztcShnxUN0JPvt)_Rs48JLR=!+Gm*E=K`MK%n$ z-*ZeHGC3_C@nBnNTPX2sI?-(zyDYtnLEn+6ki;LfslBWDe#ZWhX4n3#>y!IQ+6a9E z&9%((8+s$@!y0tn7;2Mk)7_p2%m@rPt%qsU!lBn4K0mQ7x8UWgRmqe(t)!nJl5&3P zmgcD{v88N-YSF&7H8B6`yRM(pcoUu4bpGhwoy{t)r2)p2L0 zJ@tC$>#3VUhs^e<r(i0kI2ou?P66Q_?%ob)TOF6Q|50W_fwilgBX2>iV=q^& zD7+g4|MJV?{BUES`JU%)`kv~$gGBr?Qtp=@pZy?uL9{}CPQlq+JTCK{=wzB*KcQvR zpjoe=~r5866qK?|uPo7=95mhrWNVQC;c$@e|e?FCa)S$DNouTI`0B`Due9MY25A4~AF- z^c7`YGq$HfKPWG-TlIPF7vm79|JBhY{HjXroz44n?Wu45$Lk!eZ$i z+}pe9+IdmoVt$imJ@Z?ORZGNV`08_pV(QYoO!c(A1a7+1ft5v$e3SW`@zy^7zi5<8 z?&lA?iV(^g8dA*!W|_>ElD=v<#J2P@8+`hH>boPhCV3nVOX$PN#n?r|NiH3$$~m(J zDL3tZRCI?whnS999{pw9`Ba%gBVRhKlvWo>CX`TO<(TTU`SOUO^rBqr&bR+`AZBsB z`i*SKwYyQ(=ks>m(^AXyG~p+Iqo;h!)um;rL@Mhte06=dmkRPOrH$L9Q^VjG3~&e+ z+I{xu`x;r|M)%cHp8fc1$1nd{9m2K97Z-Y)M=6IA8OfPt`Y@YoF_)b0YJISa)y>f0 zuX~!Y+{~-s7F&Mw+iwxcCAFij#y-2GpOhc%>@!VrO zp_owrsPL_$tixUX5FOR&Zt<=$AcnERm%u9IFj zA1!|-X`4_zY3Pai_E1Kfc0y7kTVS3ppsifNT&?XTiF?_Vn~%qyXVMm%A}>*f8t11L zm3iHt`ki(w4LzUSuk@HOEct3~H=4^TI_=kY>R*{X+9|Nz?9vzeRlm>{oIm7K|FL(X z;Zv=Q|J?z`g@baHq>kI_=$odsK}!d*1g!Lb?cnR6l6-o{_wP6*&^2R8=G-<4fuk&t>hJ>gg7!B=#ekfDIhHtsg! zBtCn_vq&y-^Lx4N*pW@z`8!ehMlFV}KPd{iCiZE$t|oEbjx60&dmA);p4GW;U!T-h zs_wgPPi9nOTXgxn*PCdIJUVftE2xF6H+*rx>f}gbJ%^hv3LV%7svs1s+NjjhT1piN z7guG?6Df!`%jmP8vNZ;M35VC3>!$;Zm>>FQ6%StdcM0~;yzhJJ_W;QO0y=Jei}_a9=8h^ zS*=0wrTmc!iteC9HeXC) zn;to^)=YJmkeiFm{=k+g)^zS^jOYnm>^}0SEJ-zj9%hHb8)45B=U&_Y82J9_Yyfkh zq3@UTYr}hX8$I+=!Rze5KFj~P8CR2N(Dl!mCwp*b$91auTs6M1Ur)BC5bv*hRs`Y9 z%p!yF{MpMrw%n_mRlmnRKNUs4GiL;JUDaz#Ls@Tv*f4{ zQ7d9wc4+!jy=WAsSTYKVT@^VIP;{?86*!eR{hdg|<8NP}yxWAme6_0l%}c#X&0*1I zx#n7JMY+LY_Wm+RbEZSR(J4zM=@xPxz1Q&S$su@?BcKv(2w#f2WWm?OKa7mne@dyP3wpImdBpqLM=}6a1uqnwTo7C?heRmwvq0O zd%bUN;(A^FdrR4?Im2xd<>wD_Cxfr&3Z(rXF$EGRDyTk_UnIpQ1*_*s9DOAbEM?1x z881r9f=Ch~MhQEKwijDo4;1D~9QSt^4755&SnVG4RDa4h?Vobxdz0*??Drv^IdkU= zYBt3wADTC-tz>MQpx%dzUHg86f{ObWF1riT(j_%BN`?}Z#q6wn+!OUjp>LH}2<~k= zY3a{>PS_K73C5nC0fal8Q&fIcWOdOD`l5-SDqEt`Ypf9x*D6}b+Ad69uyW^%gkHIVk4~ zIUIZGoOW0(9*xj|vza*?PO_6;WXO%%SH=Q2i|n@dHvU>qB5~B;G;&fMCjT^glRnkh zFzV{&8BhPZCiWZlNve<3>~j62(pKx$a-tXt6>(2D*^K0Cajj?n(A>3tG4{;Pw3wz- z8&_1>*}RQFNfeoZ2mo`>KfXYbZX+6{^P$kEe?gV5`*CR3jo8*dw1T-zRt za#b}P^UNQD{m{8kI^Xq%M-@qU>HR16t8>pY*ZTQNPX)*X5uthB*>7)~dYqO!5pJL? z$AmVO5hBO%2I`r#m%RQ-sGs8QCi~$}j_MbhoJK8n9}*jbjRz^@^cKa;`>O=>lD?_v zzD-h_->Ji>z?zS-I%#rDces@kb-pl+MGsb2_Gg=&lP9wE?gFXTwP!BB$0wPnqCW99 z;a|d2>Kw)lo9msq7mw>zvpu8NQf?*kUpJhNo5sITlVeCHWI!aV*HGOW0(JpCZxD@L zTmN8=Z|+UUKaOck2Os!*|F3+`%8xyYr7IQsTbrs&y%rg|yW+U#iigQdnY4<_$yRPu zm{L#kB272Cm|0BUcahIT-Dk@98>MJy>sKB&ucKKv+(O_~_k@1LgOYfVtaMRQK8BL6 zueNd& z(V7bFGI589-B?Y>Lt&m#m;IE;=X=l|JUBMv=o`c?%0I+ zx*tm=C*^f;@r^3sCiQD_qo`l?CVRl5+C-t9|7z&^}cuuOqxvfr3lgY zI*gicTa8|f{N(0{3GmpSG0nQRCfL@Gewz%ClO-Qey{}*lX%h=g?$+nOSH=Nj@Xl95 zGfb$8sw$alyB<2m9LqPnw1MWui)tNoEaR3}v7dj|oAgf|)h!mhEHW1HtYTb#cez#) zVO3oEP}}#tUApbP@ZJzdMc&JWdX~|bsf|{T%T->HT~e1m8oGY5Ls{c?`%~>kXF)F0 z^1w8u0^+)+IubLKue!v|Zrf;pzcnc>-^!(H|Esl>za|Q1g$`jEt+t@vbtIhGH`CE! z0%LZs{)oHpt6Dxi5x1=msNX99z}c2?oaOhzil52-gu28PwfY`XNix= z=3inAgnlklP+Rr0^y{01GM&Qf=CyU9_eVCbl5jM)wD+|Wz*!L<|DFmCRaq|j^0X^rflj3L5tMRCn|)&-EG=+2Zi~l zOiB_zMELt`~KjV?1er_0)B3<>xCdxb7EBhw1|GqyA&H_9r@;eZW1n)hT?Agygu{=d9b=_|5YcwTJsCpPjQx0+Qa@&`wL3u#+}ZgKx1*a&7uro zH3s^>?>s?(S%UO|_zL(BmU8iM#DXZZ+Hg1zAL)f;LGzJ-fmg{m%G71Qjy&x9Lf2S* zUAG2CtTxlA3o{00Z-rme)1@1S7N*M6Pc*3UM2Z`I=S1(6pQrDDl14xK{e9{C!@G zD98M};UgkcCPHbTp5fi!l?223VC9mPNS;2nubkrzGtN~+$Uug^$akFYaVoa?XGlHS zoAP?ZWrD#N$;rK7CC}Mr#u(QGrH*+vWTemH{OIBP!|DAo(DLnJl|6lYh2x`I)D1D! zPw&1zN*PjPTx_($1cyXF3Ne)rfEt$%i+@*#=7t0GXFH5jR!WUZYzs)#rV2orhju*o z>unxTf7_>{rS&J;QJfNlD-)Fy?$7?}?v|OYwvKEo=csX*<)xDHZ!O;$FNhQeMSptX z-x7z+YfTTXzUyC{UP&A{@yH((KQMDNK1?usDP)|!aaABc;WticBNop=M9fWOQA)?Y zIcCWtZam~zp=Za=nIBG4vCtNEI3jYjk?^0IvX9~w*M7ME`J3mme!X~yDLYL(yg%~_M`d)RhKbc}u`C5=%v@BJvi!T z!0w*y5-$H(I>Dz?GQ;`%@_1u-eoQ}Uzr9}NuDtEadbix|!bi5`p|R0Ju?UV6RTzPA zy|+q7q1*gOOk8*1hEK16gj%}bkirlxk43O4;HVJ@|6#lt+wHxPYIDD=Vj7=4&K1VM zR5y!2&Qpx4$;Kc-G&7+L8h>Ae7fR>%*q*`Bg9%Oyh_2txUjfw;@0K9L4$^ zB5AO&NbcK8m144#h&Xs$qXAYIBC+U7-$S1)*bBeu0^nnId zqg0Zl2_al&>fc7rHrO+T_m8YjSSRZ5rw2|08&JND6Y-kzc>6nLtDvMR_B?+QAEsww z_Ng6vh1`NoU(6v`NcD z6vhi#K#;G`Dt9Iz<9#<%=;sdUvut;Yd0>z}=!I>bXbR8=-puj}&rH z)ULYry}-C|`-U-FS!O8c_>;B3n(W8|>#1cCZt=TH9p20Lb~MR1qa^ioO@)sJo0N17 zM0uPRnKVTkxZ=>}>JlxlUM1uaZ2zKXQ=jzFLHjm-$0S!ENy79OgT0wY93Ni#S=qP2zjUhLRC0FPl;02(6#)#kNa0BkZ0s0`|r zzKi~!HwUlX&K6&*y1KfuR4x+@E2+u8jP4wcdjFnyaT|**gMrJW9@h%|cesl-@@t`l zuynZZPDv3|n>=0v^p!P%K`O+RflN)oD7o68%|IVt^88ufz`#JkNB7SnG2l08-=eyl z%Q090Z5lZfEH{`jlrA(tAa%_E!zK@rgK%Nj#61^`gAY_<lB$9fQ49wVNLvj09DTVdHt1tzE$Aqathm>GIv14^5K9z~; zPJQ~PZwweQI5=I$;S7;#z~GBF1mytE~sDuFsE}OF?n{F&Leu&)UNamgz={0G=xr3@@LES)q5zz&w+b zc%n%21G2Aims0Xg3QseJ-&F|g=zt7$`M&i`a zZY)=xVcsj4kIT%jx4BJ#6BPI0jk$VeBM(_ANseJV-|ij)QWPs`BC z0+v+qg+U*GS{PL#*xmq`l>`z#`$vetnc}p;w+$3;t29-yjOyTvh(##| zfRP|XD@d)htpL`r4|>dq&VA*m#l0%8Pl}|}&j@Fwz(2|{5F;1zm46flmTL;ucT}K7 z0{OEslmIH$%?W!SV~0zQ=03(@uU~22Ew9lQK5RQP69H8}5F?gSyjkkmXqk|ux}2ejV4 zM{nD0t;PgRK_`BT7*I!ZLu=D>HWd1hb(S%kbv6zuW8!atSuQ1pj>xIPi z+Afw+H=oO0K`&z4S6`yb!r*?)%WR|Y_nCl(ejHTsE$$bpr(3Obhwy@tseX&a+RRPA zyK&Y*(Et7E{``*yySg0$pc~tK>S8LxVhF^4!?(xs6w3s)7#fg{uF0RFF&~(&e9lwX zZ)9%2!r6Re2LG86onWvTq^LtqGSc8pA3T1MOuV4<;ll^}JTcF`2aWOg?_F-azy{ST zKY#wwVYd34nL(0Ky+bM}{V+3Nv_*{>dW^)y+z?h)R!>Z7B_vD!3=LDw%|%|UBr)x3 z&yfJ_${WxSxt<%z{T>64I@n$f?g*k;B!tvT%nDI)0CtGxU__80p21oHTahPb8e>|J zT@F7{uNpL&`zZ&YXQ>fr2ZK2vca6da!~YLj`erjWQ)S62CG|6zC!qxvU^CO|T?5QJ z8!^MbOPD_$hDq-Q2VFjeVj<=X3M*<{w|mjBugn4RUqHCO&b%7nith#n%^0Y+Kt1)L zV$u@+x&3)W<-w zU6d$k-gNR1p-KTov7s1rVJd?nl|u<}yt}=DK=wa$To;_irm70NdL7^tJDZaZ`G zLERt}PygX82@}l808Yq#rZ`Td9!g5x5DtgGKqfQzHrSu)zlW<*9Pj;L3J9}od3`wj zLy^{1ZaI^UW%OYpaN@KGQNlwPZGNL+5aU@9=b~NSkjT6My72Ox_05OpXIpkfndHm~ zFhuB68!*Kuy7l&Q7eq4b_tEgWpyT^!TtK&hS#R+TOFpV21TKM(@a_Xe25=kUK3amJ z@P!Ie$8o?q3r72sILky{EQSj3kN_XJ=LfupS&IVDDMMX!aZB7chjE$Wjw&Gd3=YtG zr$r8ABmMo6Sr&?&{>b+kwWNCcrEMRDN0?&$Ba`=Vp^#C>>3+rYcqne7#T?0die*|a>(6r zecW7XDfXW&Ek;G9U-^tms}FQYoEvmm1Im`KCJLgt0tkgOY>fB?e}CY*|5XSCECu|; z@KH=-mO{olg@A7Y+QQ#s@7?pVQFscg1=JiGS4M${0&$*UC^IEbiYDOU}RK{HMi;tmSvE|umY$T&{`tY6yt zQDGjTID8NWZaB&>%(JMLdfetaS_M&0Y zGaV>@c>+|Str;S6mMLpYYX9wb6MxWg$d%^n9skywE==-*L)4-|JwNQ$RzH`J?IeJb zG=c%3>zBI~6-Qe&;1#sDkvetoY7exxkqX;=^K}J{MLt3_*F*}8&s%>ZLPs#;LVo+Y zf;cgoHFciLeROjT{wxru!JkH0GJ_yZZ#a*8oE!?+-_>1k$q^5L~$iKiT4 z*3Tu#H8yM~z#P8&!b_wqQz7s#+|=i;m%qOd2%rWmG63F5{BbVpaJU9Tz-pARG2Stl zd*ONcVAcHY(g0ON5cJIN(|q>+v!Q39O@gR*-_&S2Xx=^1RzAN6{pk0N%s{t=2M+>=Li`1RyaIDZfe5~RgM zU4TFycJDnxTax=~gsrhiB}ygaWSVrCnFizch<--@`z$ytTf*lg4hgh?tF>(Wx%6&l zMDj}Q>J2AwLlw7wQ%!)FZN6v^Wz{@BmK@7%YFNNMfJXfJQ$|>!glaFr1@q}OOz2Wv zd*pwFb1^&OpRD@$KXwZW{vT0lSXkOo4~=s#XGLpnP_}#*uv`CZL*>gU4w?`Wp|P-D z3gos2-Ns5@DrI^7!kVqMOP~eyqi$f5p9-*-K_2QxOhAf2>mq?@BZjBO$_#5Dj1~;q z6r8)?yJ_OoB!9)%0%j}xy<*D>D36AJi6)K-)yNRmR{Z*{{4@9m%_}-pwo35*#Yk6d zE*R-s=ovgs(r}-79fLFyiucpLd=5rwcP3QB=p(fKVCoj00T>aItc?CC@?vMO^HW0I zsb%-h`kfDc6SwDcx zT)=oQf&WHyj%&pPX+rvZrm1!@2{0=Rr0KTI8ndyyaST} z=9=Yr8(qKi@Ff5r%~1>TM1P9UfBntY$9EoI`4;%I{Z3~uA5hcg^ARCT! z;@kNNClVGw7#yhhnj$+B*`J)?*G5D{CJ<(bHCI}8r-v?w)=)5`QVxyy2 zlC;w82>#B%{oAXIWeA@U zbT$j@0|6y@}NHAmtaB)Pz?fZ*Q4rB8+r(S-^&4;lV^~0<42veAj_~`?-h?TlwYYBwLUFEu<6!jgo9-nR7$8-~ z;DzE@V1c;^5)bR$IFE3n)i6`TMkwfG`okP9)d2$|@wrkdZ<8iY|$EdwbTgJAxvIrl3V64lcbeVY8eOz*acro~$yH@IETNA>zfkz`a z88htMv`#MSCWvyB3bW#Y2sexPU3#FV#_+!wq@)sfsaxElwPHhbr*v^~G2ZI!p1Fxe z^!Q&aY5BD?h)R5v^B3^BP=n?o-O}O_!hL2)6EbkLkanncj|;0-ExSu}Cbm~qxH$xh zx%4}al!dhg4bP7edox4C9U9Z_nt(&R54yVIRk@pjg_@=B2UW{41QEhdGJxsA$_*+9 zglL#t=YSrC2?=rUH+9qY1E6!lZLlQ{IG$v1Gzbf$CzH&;o)=KOJ~P{jhCIU37mz`mtLHDLN|Q1GN}Hq5 z0acIDuxt?9>LrFsUjLQ_HL3-|YS4YMGqj6Qr5bP%Wx#RpCpv|u5b5s^IqCwm}| z4Us%hGg3u@*g_g}A9rHbNJ3#pFg5(LR)usA1T8|0l&bJP8B|OMlQm0#I02ZW!EirV zImZzIk}U5Hs;+1O&w`bdsW?X2T1|fXCTw0#prFU(b6Q(XJHCS%Z1P4O{%wLM0ciMf zG)|4u_+opNTCq;lq|27bAja4Sn2y`U`iFE5NfJ~L8RTy#bR$Eh&F)wZ#5ws*q*Vxn zUNmd~5K*UnigQ^2_cNae+!Q;g6k4!VN2wi3r2%h$=x+qYdR$%h_c%eGYldqe!3eWlz*?EB{ ziQr86{~f$auOo|DfOZeageqMs=~zOV(1uP-Oie%EF!_Nu4SW-XftL^10w8xvY#A%gfJgp8>7NE%y3gQg9`-MKP8>O%6celuk`* z;G%&$fnx52L;f%h$@#imgA&}R%$EJ88VQU8SJVHhhgmkHhj-6@HsZrz<4NeW0q^EU z56)WTj+g;OjzI_L%*aU(xnW9h(CP0N_+TB63&pT)UKOu?(Av{SPM)&5I9@;9pb?sH z#G)5=$>}`uIy+cIM_{B0I+ng3eNI7lj69Y}DaedW^wZBE$aP6|>j9z@_y25;e0bD; zF7viM#XNl31hbu7It%Zfr1S77#rK5b#>s zNWiZ9f+@`ttZ$=rEm05H=Ir*_Z#gSpTR=b-(PM5dHhQf9z@4n*#Tv;4a3TMudM!ky zTVllRj(!ZPr$Jy~V`HBj5MSp~uQ}A3XLlSnZzPO;P@8X_Gtq0Q-0v0e(|d0G(|Ws; z@=$E=p!Yp?Zb4Z_KTvx}O_IhS>rKf{_f__l=Q~H$S2vUT^!0R8+pi*>a>`r@ zFdE&m1#m|Kp9zEN5CbG&=j0o5B}NRw!S22oeA4`xpUuqW@*#HkziihR2PHBDIRQ>@ zEl|0A;{~dJiFoz`x|j%^Ij$|wsGNS0sBeTyX`WF!=U^(|E0%8X+kIdL(EHJ^g>nov$n_fik+%GKj4^{x8CZSzLN-*JE7{v&8LF+E;pGp zIN=IqMPS+7`-s%#{0~YVd_gYkIf;UMI6N7-umLvVfzY!%R1}oEJ2TY@vj5JG-^;|z zAkm}v_6WG9(mslWM7sH5Dny(dfnkPlZHM2pRSbrmp=4xTDkL~e955ywXcADnZ66lb z)9*~!hC`JFXRU;0_u8ap)LxiLIKKyJo3kY~8!nPM+>p*M1X2${YrKHF|Is&!0+|qO z=^>y_Z$a(|MKcrpweR-kO5*U9EYa_GDCQ!-jca)Wa!BKiu9m4hmZ58dsT1a85`{2m zbmvFecWWR>Q)sr->K$^(E+F3--igds(TRB`g5lDti_Nbx-+7o2h1o%W{ra^A_(FNy z*Mvks3UVCGMFx>DU$qevwj$UA$XIwMm`C^AKXQNUsn2!+NYQDwdDfuzEmvH>$P{r` za9i&uBU7@{fpBq5Gpb{Dp^pPtua+(Q4nGNmoiL|5h;6023PRclQvk9y1(w$BWJ}w9 zZ9-Ej31DCEr16FbxGfBqMoP-X`LBPVizFzy#|zPJh)==-)IxzkrCEP^U|%SbM|h(m!&7R{mvHI3RIG? zU!thwprd~v_hfL{o8j4>D2j!ref$1Bof#ws6G8Y{_2}0(I@VWDh+uf&K{8(+F;D74 zxR^gz50Bv?f`g$eV&iteI40g#PZNHc8jGh<*KPsg6A zt9#MJ=cd{29=LykO+zgiItkS3b~>jJ1V&mP81^veyd^cO8)T z;ay7WJpA?1z26Y_8$MtMP<|ZX;1~VhHKIV%o;;ld$ZmV;(+8K>oI)5bHGeOmt>9ZC zJ2|jnC)oL4ZuJn!_!L~&p=#o#V7|H*NF#a=+iHER>UeW;yh%Gj7ZLF9jsN!#LNMG5 z!X@}az_dmO-MBGs(o-j@6J&qa&W~0LZ$l`|9)KJoekm90Xkef!&Rt|DD6M0s6ZIez zs$<$*2E*Dbszy&7w4tP^@^B)p|Ng;C2vNzg&iSU%rJDp-MYT0%ZG|m>Uh*s!NJZ;$ zgzjfUo?`q&I1V;9mjh$KTJjnhz)T^U;$Zmaqqr7;2hNNX2p&KXOz>RN^v|qlHsQpq zkh0SC|NAN9FmS}Q1fqvUg_Uvl&iA z(j_9$FyC~?vp>0iu~##aZ~voQa$pNpo1EQ2di0*Cz|Y>Zs$vL0^uQ%G z2z{iM^XnbGj}mAyUr{z&!N7(ZBLS<5`jm_lY`Yy!BH479l3=UxR3jQJ`_98^}*qU!(MUWR5uEeKchw5;OlJ@ES1t0W1EjQo2K$$?Es z0%{slKofaZoL?xi0E+2rZ2~{Hs(PT66M##_`_*a<_q0{5-jyrvIpOk^BgnEG0}lH1 z!)7qPK^ah5@D5(T%0)1OIEfUc^&W`d768#k=#A_AA2&uH3QmqQW6qF+8SDrL8g1kd zCxSfkQ<|u|sLx3;NQHba9)awC1X01`?V@ZIfV|Mka{qI7Flw;oz)&^L2ax~y#^;J~ ztlCQ1QoKSYaAyjb8pr1NdCcn_W-WecPnACvv;9HFI-2{+I?ve@tdDphKci~I?Cs4f zB(k-s6nHrq2SDG_A4!99Q>5D?9bH%mL^=Zgk`XNKCF7@kj{iOhfn4sgq4n4IU&n93 znC~?Zau1K#25To9h^r4e2aUYeAffy|E{*}2oAub8<|;-7YKzp=cm@XnavozDZ%+>* zAGOc0P*wii{lrAb0Rf5WxT#MD>E|{;Op<|!!@hn_V_KGjWAU?5HGr&*}W#{lM05FyjHT(=R@(@PyC~&QCjg6;i!0)sYjh(G1 zJOi2~aNHHNH0pemK~I3A)RiG0@(fh(j5j!&Cp6{*-90S(zt00YEf}fOpdO<8FMv){ z@C!X>qgDfjS%x9}^vE>SdJuu+W1d1-#sbKr=O}=k(Q>hS{shKNqq7Uz2Fb)7Fk)&dgBcl%_~Aa*#rfYo-X;T!9^dUR7-|H9 zPm@tcAqC5j@YUDImUOIkegQ_vZ#~Qa;laLEpVx?{ zbQv}X>d%D!j^{&M=*^oqT7PvuRdS5#;LLhVa=Y~+WlP)w2sUmK zxra2I%$7T6t6Wz);IRJ7I|5Zu<@lt)&rz2sOaY3VaK?zhA?UAn{>d%QUlWaK27qJ& z80tqwCQ2|i$rRKHz?(I|?S+4U8D|OKf6r&5!LI)~1+-dzXYpZ5&PxNAhDH9_VSC&0 zia6`&``qK7ElLS&chO({I2Tsk4_WumLU}8+cAAIy$M>y&nrmEoQf&&>=A4#5xE?gt zy1Znpe0%4gD)qJ<)UAB|3S&xz=M+MXbCm2vguofQ%gUbt1029aNDu9Gh~j1D=GFk= zv&3NdKmuDVJU8i~(Ri@4*D?YV_ak{$R$bCFLQ0TyZv|B4h1k0u8-rAuuSjc<8V9@x z&d~*kj4%U6^i$d&ctB4PYD0$Zj0o+Hzus+qy>!H`c=@>Lc6`U{!5`#yUxV##gw+Ch zNeg!ON|Xx%OQzOmgjlACzA6Pn&&4*_UxEAnFV76D7CJvM01Sc3I2UjrRQEc!4k{U3 z03`4RIsD*!)CI5HWvwUB1wzACuPR(RQQa)SR+JJEZ_w)Hiu5Z}d8-x~h91KNNK@-S zinA#!18u==ifM&ibeFKG9Mwwv=n$}GzISfFP#BiFdxOMFy!-&zuFK;|!|bKcHK%J>8p~Sa87LWdW}ztvqlp176R( zck9-L;WvO(sVwYqARXj_Agtk`r1CDYpzoz51les#J(geG7rHnl(m9FEWhmoIqLAg<^Qf(`{B#K-uyLKur5 zHV88P-@m?GYq19jmC)r&;Pwm(=5d(Re{OODPMWOJ+9DuqSOb0j$)JxOYd!-r&Z9BP z41mY`d|Xs>Q=|x6puPF;I7#iTqV8rDM#i*U2nxz+An{I`So{roV}9G@sJhf3dGJ`1 zclSqyv#Bt|ue1be-2C*kn6eRs!K!e1L|{(hKfPA*R$hF7=OK~s!ka%sJR$~}x?3euP=M?wq6qI#jzq#6P$V#z~v+>Ljrx)b69fi9*VwG(v zC2k{{JR3u*&07QJ@wM=~u@cA--v39}S4Ty;^=%VFcc(CPgEUBkG$`FAEj@&^GIYw& z64FQs$j~hz4I(1lsYnPY1}NVi&w1W&J?s6xf6h7UtaUsy_uPB$>vz>IGP4u~Kc$KJ zcP9<7a3&BFOs$&W$dKS}VWELMWmXXE`+b>DT6_h7H`%sQH4latuH#*dsbTc(Bf$Qm z2J8U@SE}6QGnmtxJ;P%w$ZVdX-_r>35WsH4EVjsg0o_$G_*4a|7|X++7t8JqR7Rfn z{K#D||A1@FN5Mhl`K#&sCvez!0tmhTMAbee`AlKuqclDKNy%J*3Uu~3CP|TP4sF~+ zR=c!+E=Clrr2&V{(nKA-q1RHU-52S%QpjD$l*_TM0ou-E_W7!}NUHcSDm1UtJC8sS4Hx08hMLye?!{;9T6idXQ1$HcJ1(zWL z6{VPf@6p`z#>#7AeYOU!VMacD&l$ADje>Hkf^U$C!M47;9| zITfofoJNB4@Gb=a6+dlLz~1>sy%JwcGR)p{YCyG^tnK+{s>dT`{Qx+}8pm(a0;`l$ zp(5Mo2GCR~3(D;4X$GcQc;WkZRobFd0Yb4Odw&jxM%-}#Lh8PA9)sW2z67Gvg42N& zjSRlRT8`hS=bajkxhtxqo@K^&fASMQYjo%jFy0WRDLcQ##381|AVMa@ zd6C+J##a$oQ#W1}2!yVvyp{zro{0NAKgE+22oAz0GtPgze+}F?&H=U>KaNIP(t!4e zKLDfX#_Ci0H^c3p!W6dy+AywmT0OaRUvVLWFPtd|$oPG1orn<{Vd{#nm9UkIdw|p6 zfV&Im|1v05@#uw>Ei5?Nia;XPN(;9+Puv~p=t+K{_|S{4Q;E0D>y<#nWn*&seP;9x0KaTD^{3C0 z%E6!%f<*2;#AKO)4*P6fAJ#Syra^KCF`L0u6k@07c?m3k^n2uq_mHB>poOK2_elJhLtjYhnJ+e^|S8P9tfGc;F4pZq*g6+YP z0z#hdI}i;y9PQe6o-cg^&)F`fi_5FbEm2Vn3>SGf#tCA<+fV#!6#WLd=I3=#Tj+fF zCnj`*3O`svzA7-ZRe+M5G3Om45T+Fhy~_n2<=w5+vFVP+UA3ZQUs|msl(&iVpeDlr zlCA13Y2AM>Yt;39J=6)e z(c>DVbjoTtGDLCTZB)BK_$1>fr<~G;CVw4Vq_e$EjSI(?*0>g375bDKoOazm^jE>Esyyhskjo<;KGS-!!{!7EE-kR0b)O7o&X~T zbG#$^p5JP3cxXmSdb>S$1yIs4g8}?X)=F`x(gCDF-jHfO>>3#Tlo{JV*8|rA`R-tyZ0X9DJC0Ha(Ew%LkG(haCrnqPnE3Pd?6+#4b9> zSFypQ!vT2Gxt4wf7aMvGG~8jHLuwlHd2u}~dN&m-sU#}C?9xBY>8yxlT%Xgk{?*Hb zwSpEK5vZ}z-`#}>Da|{+vst9lcv+|7`Q^X z;*-^iJ0)YFneg;Z0T{*@eBD5bI(B#9))jI>P*kvxTNAd$K2HxD)_dJShtFqaz;X!v zB>(%X;f@LY=yx=H(M%XS&Fh{FGtc(;I|40DK6kEsSB7=Jt_5z0+hBN&U@pZSz5?c| zgVl&1zFWHw78V;zHhw@vBm~NE%TeFqr@FxGA=^!vAehC z4hLwvAdR*x?Cv4SyRAoOpSsPQKXT9bqyg-T0ruBsMZH-OI26rX4J{}#2qzz12yxk> zM>56k7M-U3R2nanq(k+JV^v9UE<*%sPaEGfIhHf!wYr*ioOLecPOeu0^ z0j4iBb`-1gJ66v@F&G!+NfluO--+oy0j|c&3W#e}Jb*4jQ&Jhgqcm}pOr`=5iqx?E zW-GD7{tdng@z#L4mGGSXy}ewyC-R2vS7&y~5jPy#+5X|roS*c){{yV9Isx|nJ-}(= zETe~!GGJwjxEt~I8&DB-8AVoaC)}tGVB*TI@*cN4yufTw7~*2%K^J7Yo+eG5;+%>9 zHca91CnJj@>(-7l@esve&xc1u2@tqiK_h`s_e}dbxdG;a*>Joh8q5t=_DwQSh&!xT zbuUhve@NKO;quP%%4#ubQ5|56HPf}84g69ki)TfR;(fGfocFI&?@_sW{TN!#14z$r zkT?AbfEM;&`ty=rgjRZGA76(@arLfy5fU!tD{9w1loC20c;_P4LUYnsPFduW%7y^@ zp!O#>r=t+a0%t6#uwJ`nEO+}>EQ5#y_noU^QbprAFjAO<`H9QRRFCNH!v|b&HO4x? zkUmQz=Vl&2+$PGT172=7w>#yQRxbl)PuG3{XkbWfGT z;yY1d_RK}F(`VLyh8>w5yoT;Z`C9-vRKIzN#0xs9_gsn4x1e$vh+?-Bh^m_UD{fwY?Y1d3o`WSMWr4*KxY%*zCOF~%J3dvo)YD=%|iP168cLd?*$q`*%&bD*g zZfV*3j~bb@Zr;t=w2!14SPjTMRa`d25U8llYW!UCG*|3lI?88a0PgVedtaG=DRL=I zc620450Aws+K#sCu4?)59GGd|R;Z=WYZkLjpfZwfR|My;d=CYyD<(PL zObbs8<8#JKTn>7rD7rwv{ORh5w!wl@lfKhf`=`MMcl$TlQe|1ImfD>g`}m%9Whk=E z(=RBPmE(yG-v*xRjFilB+;*6<8yfp}4m zAWkxUS5tYwJXgV~a+j~bx<>L5*Ug|@FWc32YENd6QGE4?$X#DyoT6{C|UvQgSNX5H-=16SMhdCK3HhzB_?38mlqfvxTf z(?gttS)yhHnTk=115W-x42#`myJyt@T%2T zfS9E)a*KSAs&V6$D|E((i-QcqL7gR5@^$mW_w1{?)Qj)s4BiNbX0z4cQ`;tR7)D;0F}Un!=)c|#B;WUF4S938IA)wht@O+ zaO6v?uU~xlg300#NvHjHEkdHAnmpP^wmecj<5T5AFL7(oRx%Zv(qr^RU~HOuYQIVs%pH5xq+6^gk(Ncipl%Oh zE>ESR`Ynnf^yX490gf=pc%ViDaADK6X|2*oPrDTIr?0zp><&k7*8}gRno?X%O1rC9 z;}BS~Oi!+1engHtS*x^Rx74Rnmx=v4ee!P2Gc%EX`I7$3KRMU^_Vz^e?7W(Wt>_)p zYo>t7s#I7Yodr#Lvcb`pyvjO4T$sfdnt9I{BSrdT1*XiIY9=&sz?fx_g#@pblmIJ? zSgJF}hsHWFSKqnW{jMuyZv@evh!F7cN!95{v zH~G7vBLEBy1_m!Dz>pU5Y83|;&k!l#qCx?yA1Oxg#QBBk6KrGbFTtoSDV)!1RJB=& zC07opGqizNkCdSa-=y!k&HlfF5BRY2`krcD)1}R3frnK{ zl_rH#@SE@NI4TOk7#fAjIJr&(`*NaeZQy<;{B1>^{@sZO-Hkw#PXIYMZ}N$;r!;HTP9( zz_(aZFcb6ZC2nCSi10sqeT9f6&80OvliT;S@`ei?O54wtydL~OC20h{VHsOQ+$4_Q z3fF)gPa1nc8^@NH5P1^SiS!RwlJxyAg&F5j25txzY?sM()%Pv5 zzsEy*PLbt)-3nRiUALchoj|5%A=Rwl)Z^XCHTk$ovwbXf1tZ!!#07EB9Cdsu}S;Ac&;VkJ6eO!3v>5X_6>Nn%-qt(u$n z{fIbP3bR5wzvVY(pvqR}Aa6J*5WVAAJ(2x(-61^Ca-+jUr-_#cKX>r#%*E+4(9&r- z`nhk=lSrfb97E!xJ8W02BefV)c*QNp?o-`~Sl1GZmqu{v+sziYO~^Js9Rb6GBk<1+ z_e2)=wm+)j!RSXZ7`I)WR*|blsldulEo9>{=PdEy?G!IBwIvp&a+oBFqryP9<8u`e z^u2SD{8SO_&Jgi(l5x>wDZ3$NKI_ir;M8Au;?%oTjyY{5D{Mj1nsg5M9Pprzmy`8YM+qCk`=Sj1%E8-q-tlz$Q?gYDamqTY88ZTzW z9q+OPXv&A{-5vV=N>-sGd%onZ+--U>N}~d1uUots) z+v@+x;61h@jaLfyTfMd=!P4oorXFD6R~c0yEq$pjJ^Vx8&$!OwL-H-(r>D1YC2{4* z8G;z@@ zx~vX`6G`3hakKwiUb@1;Uj@1tf3c6T;*mLVLm4v>jJ$;2x<#;g@#1OhI^iyPY%4ctA zPY0Y>5cKURGDFNgdq6^BRF^tMD!$e%-fF$SP15pF1(?l2)fO3wKu zaG`w_TBA{jaZ{a`OT(VOcF?He0${45WCD#1+SUgJpQW);T&{` z5?s(Gi6PjnSAkLz%5z8dlti~Gay}$aN@Xb~5uUtT3nw# z6@R5TJjNU9BtmvKJZ%hK1p8 zw`kUZPqd%=@!(ocYb9s*_T!i-aAjY6Iwx(Hs*ESbfER8IVZ{9cDKcXOxe-aP z=R;qimRya7dQTzwS>x}Ty@t8L|Le`i0c>f3%0u+;f*4Pl8bE^jd`5*NSmU;f^b>2u ztyKOn$KY=V$o&Q)x$H)i0U=<)60dpgjU-1Mg%J6DjQS5g6)k7ndht z4Io}*Gr*WqQ?{i-$H8bfCrgcP2NS87u$M`df=|USU{BOlb4vtIlOnPSap#R&2fsg#$XsI4 z<QJ~%krH{eHeej?*^bZeVH@n-(Ng}5agP$0&Z#|Ac8K1VFRn^hquQr`tpM$S?QpyL*oI)oml!^(@5;BE#}&^w7J2+yQ1` zaTar0lI8!XDUHI2ibHS>nI!C8JNz+VF;TWd&ir-2Sh~+~RcPmA17jJ!u%r)B{!-d^2DcPENDz4QNL*N2`&1} z!BoCW>zIMoreAW>{LhPne9z&y0qiIpy8>Feeh}7lXbn7Ce6X)=3-%%r!A3?X<`@u2 z2!$18xxb)q0OCEDmJ>la^t1*9KaSLI;Xc62b`)kulQEmchE*)u)>(-2m?O*keNh1d zQ1TGPk=S3Vnw0n3QD%pLK!=u%X5MZfPbutGpHdX|@ZQtkfJI0h%f@!LL(|8|w!?e` zx)#VVdPUXt<0qKtsCk}%qm6+uJ^|~JR%y?iBKeOiGuvkdNle<40RkF9v6Z0%hbiG{ z)oHS1ozSX`+xzz?`F|ZSV&Wtd3TsHB0AsN*HN34<@mT@jobmey4ZZ6uAkP+?Tc0 zPnjhKe;e}|S+)@Mc5@s*^d||c7 z#0`LiUQGF1Kd56l#UY}1obK{zuJ#-f!~m3uXrD?|#qnvx;&adCx4@jzJhEz=3Y7R4 z!LI$@neTtx+6|muX>Vn@;_dSW2m8Mo$(Dm>c=w<)yp@I@uBLWxRjM*|*>gCr&(*o3 z+Sa(*tSiECyx^`?m@`*KHWBU zX;gj~B?py&0Zu8pq`>HH4ccu+*n48(VIm>NXitP7J3F+Htok$P_V@e=+)_jjVmcuU zLSDn*dOYLzBtyzzR@$A_77}MCSh{Mjj-w3%VRu42G~w40cX1&!%xW4W@dSs;qzKvv zC<-1LA-0VjA$@y2&E!{*21dM)r80in>&MBJoroA($ON#|dAG9M`bZO{gN~WJ*^`VO z#YlJCE@COxe^YsZGY2(-Uc|sHB0XSMldXU;*u;6Ld{hO-+F6A7CWbH`-RS!raDVVR zg0BhAfpmghdE-j+YfS2h(ywa;9Fm78t{<(+kFprHSWc>_cU&7HW zcd2b1U((U4BrD&8JAt$Sp%PX$Ff>XG-C`dHE=-sC*v7`}0xksiB$hosav%7iZqq9I zz~!(Let=Cm-!qshdJCKAERDqc6!gyqd5!n8CYW z^8)vWO2laeib}i4J(X+u!ukd5hoKsQk6kj$359z5TVuJ<`W)6B={EY-uUzwSlx1IT zvE+jtIOXpSY#hlUbKOQAWb)PyuUY)!`^2K@!gxX%_0EXG=OTWH+Rjuo z#L&fSE|2zZrf>}tC_aqt9)?DGrjvhQaAar48iL-i6tR#GdXn}IIYc)b8vbMIBrVhN zT4}8-?rd&o@~{HhWtjTKtAx`4Fgwuy!)Dw~l*P+bR3Yqoa)9^q&+`w+$L}1}L28X>xa z51rF$6~WCB_1e|n_~Mu)uO9<5OvQ+A*WnJO-2u5Z$2lwRc|9v7h`}@`o}R88xX~V) z%t{D$e@IXF8z{B#P$BIF>ZmircZGNe;ey88J=&AXWD1oMv5>E>W#dYp-dAYT`1Na- zVpBR*wfNL28pqDMfU|?Mmii=eGiDhtfW}J`By(?@mA;77L(?`gX*Y~SMGri!4E1A5 zkjmkZU{t7+?gJBT*{i{03Vv9diYiWa)s`MTl6jAUmva?E|4T`Ta$_u#gac7Wy({I` z3O1gpm!b}8CGp$%$wi?j(oyJcit(h$NkNumSYwIX_+<*2X0+pC!%-!^cCxfy&kNXA zp_>-2d^q}sB?x;3uNShp2i+=-60q&NP1EfeHhxp$*b`$JTnq)$LcmY=FmkT!kBV+g zI^y?gK`}R`gAu&_IHah7`$$jCw|ADyA<}U_EXEFdLgu7>kejXl*+uL{;sw`wLyXVwq|F|?s-Stt zknFeKXUwiJArE}a;WS7b7|O$->e@0O7l*cDc{?CShq2ZC9GKrwV=BpP2gyx1!!Mip)!B1b#l-HeuG5hSfsSz$_b9ut2UP zPP#v!RUAX7=Xnp9%CFgq@x^X_e0(=U_8Kb>?X0~AurR%(P=7Q{rTB>inHCEJh|i}u zEB4Tq{0=f9uPdm1{8s$uJ8IA?5Wya4-^xgan8I3`l_FZ8z>P_&H+O3bz6(0sHbsO8}$p@$X-SM+HMY>@rA;WvF7SGtwv{ zAos^JVD@91=Bx3E*R^4JKL-{8a3%fy(C3b)tvkrzpAQmbGN5~IyEz62908aL_GGS~ zX_zw&M|Rk4S@>XzJ(206Z#!VVJ9WR3UYtrC-+uoqhbo6raBsRm>(#0p$vE7ai{n1r z!-O?5GF|40#LZ^;_CI#MmCXN<)BY|?Tyh|&L_&0ua8l3`G&OkN^@Ii*w?R^DhO1E= zQVZup0?h{+qdU&**8MT~7_sc)>>IFj;YYWO9s@hCM1tBPj<5=nggg}ix6CS)FnD;8 zc*Q>k(I~u|&SvFrF1h;bXE^6OvK6huim4D+A)db$!^OX@%FZvfW$B-V>Ym2^RhRc^ zK3xJcSDIPMM!l=PDq!)BJ>G`{TPwm19HZH8Gj4-7tctAYB3bEFS^40odW{N&tm@%~ zZGdbgFGvN5-6dGc8XI7UPfPd#ut(1M-oymTa|KE!2+Q-b@kB9(+E+RQYJZaaGzZpe zmB`o*2JlX5h@>Qy`Ki*PV;N$)O~up$~!Ke|K1 z6N{-r{sN#9WB@DD3*(&S`*&1;@;^h(Ujeaj?Q!$`Yw;)ldzj{u{=eK-%dg>>d%(3W z&6U)=Qt;ls2w1$~$jcJ2O);T9kYWi#bv!D);W!YDhq9|y@fq-L7Zm2(fM_(Tb>yV? z%8SRQJ1C@3>+UcUAzW==ckUjr)oY2&gNP5;&oJ`U#~?kJ&ZbVx12{^Ufa3aW z>SNj26os>dY~}A;Hfz&uhdggNp{=s5(qkFRmu#rs;rWPE3jQS(G?O^y&@vc08hD8E z?Rj(=iFS?Rt62%JYU}yyr}95Yp1PN}SzCW%3ZVpz*(lr2-0T%QkS5`PBOH~#n`VB5brTBQQ%M{qEIoLUUmQi7)Zh=r{x;sEd z7XaY{@ufQ~FXA&Uft?&TnoH>fCeE(SehAX2`QxC02~|Fs4tS2-MYr;<5xK9Zdks-V z5P6c`CmE0^sWK)4PD27SYeZ87m0F$8~h6qMWqsD_}0%1v+>d$jr{SXWgDh zSyD~s0Y8B%&3ibESN%7Hu=V%NeKt%_xAUa+ou3%xs!H|eSjF-c@Wyx=`xpH^UzAuC zH@6jTuHPHh09Bfb=2=`ko!5-!hgt9iPU*#-Q8y8xx;fU4ytZC&sa3?!q@IP{eNhbirEssWOK zCMYz`aLdcrrwzEK22x&w^1MADex`912C=T{x`!qz#;T(^<%lLI0!}8}K9bi1_~eH_ zq)Bmt&mrQZUtF?`L|N3-;IZ~Z=lG^mz=TdqMO&XXRHt%=1$XLlQ#MJv4QVmY6>*A= z#zu9Y#SqZY8yo=vJ_j_Q&@O0oG0)k9;+iaA23_Qy)@xbwf3M5=3HLYRxi1KFh=7dz zgCk%;6X$IKsfT<{R7w~%mK>W_8>oU-Di+djIK+$y;LFPUS%L@!A-rQj5IsPBc{K9~ zg=Z`BjUSrkwORXa71|7}*#xOkS@CTZsx-IJFy5&)y+c``T>V+147V+3M=|)*iqp%L z(a2>0AWw65?YR5O`o*v<#9l@|RYJYcAZT=iOG<&4H3}M(Ua)&EzSF%SBEJA>cH?2= z`^L)s4KBCw#aup1uKGm5!X1l*Xzt;PY4D9o$Pc!b=nQ(EgFlvP+^%ZI=J8J_iiSzS zwvcab;0;dkSnpms8o;7#yH7|n1%C5!P@AYg(MKpb(;tOXl+l=Tfw@QLwTW;=6`NJ7 zdo%&bXdKw8J(g9R_H|6`Gh&ABt30Sfr=K?d7rr&bXwLlH>A9tCXxSvy@WdceU6MKN z&bV7LjH{noa%s`wTS4~)WCz~p{(c1bxiJue)6GSNkvsiA*|)ZV^be&&-=AN#nS7?x zU*Y3ZbUTxj&3`;VHtSB{iHCh{iNTIlwP{0PGnfK*EVNczf$+Ix45lAF$lp}hHRi3q z@q@XT_b3ZDYH#*3Q}qQKX4+lbE0cdl8VzFz;LzB1K&H5^?S-4da?mdow7}P>pcqI# zfwar-DJYWdo+rjJ%29ka;j{C#^-jRyv)8|mfWfNAsK#ilD_%co4fK{cDntTzI_1ts+0=5egK^$1 z6f0pG4x20FwZSC!6r$M^h+h|jQ~{^Hidd@U$P3&>aF{u>ZT+DKXTlYKnYTB0dz`@n zQG!MrKn*K3ALaii8y=I!fpU{s2S*#$Dq1866ExV$zuOc zv%_yF26_RHsPsK;nJS(@+HsV$Y?_S>bGV&OL5=@45SI-=Fo7dw!*wLq)~n(-b3z+ zs#5C)AUA5Ym^Cywr8v?qTsr1D|q&=iq~o>o1~K zC700tdEL5m7Xhn4i*_$dzq?vSqilO_1q{m2j$V~qQ((v&ruqjm{md&@=I;HMv7HhA zH%?VfkAbRz@F`1nPtRkuV?()+P{}XdY}Gbei$~Jo0qRijFdUD<0L7*AvH^fqx|Xaf zLd=a&kj7OhKJzc2b$8t62U>48*adyReK2J(;QcahP|2Ry0VMp%AvwRxP2J+95hz;6 z18`=PQo)mTil56;bE%wKLe+`cHI!GM@MZEeX>6sNvp-3D37j}KU}!=Z)fnm_4~U;@ z_@xW86ql3QrpLH951cP(gLKpsB|W z+~FFZ^B`5QS)*Bs$KX!V*RZ&OlkShf%WtoD01gEa_SgvU0sn$dPau5sWq>KDc6fcBP z2i^)KaHRkCD1Z0MRkEKAQG(Z81{2wQ@kF**sbC5FA#3c=7<(|jQ~I4L{Bh$zlWY%Z z#+ai4rS3ap&-&MH(xbWrL{7U)gS746!vx?_Fo_`?I{dUkL$v;+p* zD1a&mmTCI4;rhNmAuJR=`YYDH>n)k@%IMIppunnCvh&#?9(tbM87?qIt+=#h{rwahsWaF%*1B)h$ zI^P?dQe_jYWIZAlDE73yz%Ta&%Vs&%KUAx@qb?B`%KqghX3;Ueull8aJzg=b0lE^g zTCIu^q?8#BHnQjP;TAT*dsI7}V-goo*FUPj???P`rOi7Byb%OTcQi@uK;UY{5!l^O zHIT4jlky%K+pu?zpJjoHu?kpLPHZ+jT-Jx6274AHAt)LI^f2mz&bH4%GE5WDxopz@ z$ChMY5&%8})exX;XRp=eGiQIoqU(h7NtUhr8o&A_hsK}M+`a6P8qJtxtn^HReO-$* z;WYM{$o@gorGJu-LH|A2p+L8P1e#IU|>l~q&5EJX$+vTAZlhB zuLJwACO~wUc>|!i*;T~UfR<(%VBnxbP)ajhr3yv}J}`M#qIxd3)#N|aS(>5}KZkX; zee2v~vMJYHXFm^$CO`()Ydrp4Q0y=nVnF6x`NLaC?d`49W()S*~ z&W-9`Mtu_4;L%c3rFz9BIIwb0)5$rsMs+3E9PN|R93zG(ziM9vpYIAU$%lUZxK({X zrwEa_&)ic2r0|0FeFKcEQY6`l!|;7dc_UwiDVU6=jFZR454 z)OKAf#rE|0#M=JECqNt7BwhOkH1K90DB73k%)8GYRZ`$ggkE;hIep~I$TI7n z+J3n+`DT(OlsZA?)@gsl`J*T60XzNUH7;L}dBblqZwenzv>xL)k=7nCZJ0UQM4-%cVQVQk0ykFsi>yYJ{H zph58H@bIo}>a9Ae7#+k&czbwfJakBy<@v$Dkq$uq&w|$=bh^p_!U??Ao&LhS;xgHE z-+=NX$lbNd@4epbJ8~;<6buwXq`vR$*bQ0a^8=|xZ3PQno`O-d^ufnF*s8CoQ0{q? zVsu)^d4;|sKp}Z%2Ys$ROlL^cvgD)A^K{{l^!&YziL{@#G4J~Fig7VlLyJnN$e8!T zgiwGW4=0t|ehz=OJNKh)jRL^UXqn4jJzI;;tSQh9>6w2`YI(#SwU>iw0*E=h^DnPr zk${>u|9a(BJN@{*eBpW4;a=u1fkew=njFR(PDv>Tp8XeQjvFseFN2pa*Q=})D!&F* z_wpC?Y7h5xzGLj&|KnTlm5melJQLPXm#N(##P=bevV|#=-BVzl^)ObUS5HDf!Sjv9 z1kZw;neK33ua<i95bQ-jU*`I@vWsaYoIV z;U9$fS)FQjwbSpS4ZsGsqmYvWL4(_a$Ut}24zp3MuX=me*TcHa$gDSF>V-o-XlW+N z!JMZKY>Q_x+VFh`DYBJ1h{aIx!qa~n3uf`rF?fK9iV zo>(=4nc6|C4JPud&+!u-=C z1y3iaHTMWfQC3ZgujY63Z#oRq3P}R9UXy$b4>43R=+1AHCwk)HSQns-S*?EHlc>``d(6#K$rOpx$y|C!Ynp5M@hAmOTMD_mnN~TXCfVpAmgE zh$Jd@LhMPUs`_35=7Cf5NNF>Cm04d?Nq$T1BlYSNaKX58vhYj5AJpfvk}%1BWEq0C zf+7;v@z9$Rq17%JgIw@=xpcA?6%}1TP=M&Th|RuzDhH@ch~(q?=0o&Uf^CPp5!%1_ zXzwey*+tT;aws86!w3b>z$dw;aBb)QBN39J!`x$uw2U@hk(r=xp{}Xwd%VnI*th5$ z`HhAz=L7zt?P^uYJ*L!*6m8GpcE<^VO^Xa ze}`FFC%>~BrJh4aGo44Xmo{(BXVHhAaNg)4Evmhh-oZc1*Z=kAZKg{4%6R6|Rb7aQ z<&D#wVQb%^_eoB_OEc0}SgS(JbH$P}$M>XDSm1wtgfv}?*vC=w=9=G8Fjvts8D)!m z*>Ow3Uu~GkTp+aVjb$S8hSGK9p8K>$uB4etz619jjJxyvSoKMp@o*cEs zw`&<(l$ORqE)XuZo*52tsvP5m&_o$T4))*CZILlb|JxMg&Yc7m(H7ph|p6jVd~y zO1GI}C8PDAAGvuWdi)d`Pt0ruqe|BxHTSif2Mh-thJ#Q$XBiM zk92n%Jo0|pgmiVz`pi`CY(;Y1#M95(1l*N;6mgzn$Vpo;0nKqGUEy*R zgu`z;qTVcxCU^2Ik4ZRTNm1V`i+dLAw;sJYuGmH9Pvm?4VQh@=6tXb`o}uU27eYeN zTPM}_bGD2sc#*<6fv{`3#+E$cw_2oCjZ=+W_DsE5dPbx#C$P&i;dwc1(uH8*k_z0A7Vsx?dI=+U#b`&cx6KBcrnQxR;KeKFhFJiw9`rTE<90D z1N=VfGha}J@c1->j0Tn#(!WtPB)@D`o-2L9EpFdY!l08dt+U|Z?$?}CWW{1~JY%MM z4us%i6oai%tn|@8yt6f-Ean%!z4Pw@Or+g1QxdzD(V>l9hltHuZH>OvRDSbXVp8jy zcH`Eb+GeqqQ7#$1zMlO`fvlzI@AIRP)p6y;6|;w5HH`V@O_fMGC~)DAs;T^rdDSPy zyN+At7l(;#+^y8}AMty1n3*(^ zd++W4Y2SG7oRDZ&X9kcuZ5;6B_>^DpMH?X7} z`#q$_cbdZjmH4Gw$|AbIeWR`~KX~yaz|F!21sAgJ|1ctT)!g)2KOgN@>*X<+= zA{XrN8g@Hz|C%>|n@ODH@zHI9u@upkRN!M;#&y2}yDStHzxO{`T^dR_!@Z;jar&^Bc!&(^{`DByb#8^^2h+k``)l&8@M z3CU9xYiaLjZ~kO5)@5E65p})X!Lsh_&JSHsFp|^E1<7G@sLrpTx;^aR*HvvmILphd zEwqjIy!?hRM!L8cWR*kvRA1f77`)mR_&$Da!rzu|@?r|2;`LkjaYKK_g`s$b+0ts^ zpKvnQuad{LDcSCynaXb08c9}Ggz&LZxlthsOfb-pKFio+U;n6F5Ot_Cb}&U9#;4vb zMvkz`sP{DwYt8t~`>QBpkFD4twIj+CGN3VuGVCuMpyZH{Y&J1@W zm0MrR3Z&F?2N4G!Bq^5lVVZ#J3i-)B&yojy_l-(JxityhEbwsM%;=;pXeJ&~h9GqA zw-sq;6~8pjA*cRS1<7M@1;b-W=@p6tb|y1>We4{^U0-wTB=Ap1UG4y`MdN}~FZd=Z zzQ#p`&p6YtU^D;Gz)}BAII4RJ+$cEftN(JPyNV^wY;5%>QnP3V#M2<8qy)N+AQ0>8OE8$@E&A_yZ;JxPSj z@~b;1q&`jpRe!bkzf7Gz{O{aLzr@-lYdym1MQ*I`SQj0Arur!CdQo}l)lg!JbHW(5 zPW$bIs}jElH1}4V-YBg&QreiVnJ+^Q-=>R)P`W06TXj8t^3^x@R*OK(N32BZ zEm?t;d8RC$BCkwq6Oq`aY?lXF7k(D(*1BVs++AP$oalbU`(rz4;J^l@?|?T8)!8=7{Lg9=&)dyjDXa^f#`wi5!}<7?+P;nk zA@XJuC!jp&ryrF8;%v^p5vaE0E$bkt;<9a}pCOgg!sz#g=08T?ag#0O@YOkQIxdPy&Sa6UAM zc`WLEAiJqY$tAl$Izr%6@oE4@!?4+Y^Y;+N{ZSy5upx5RAr_#r3TyTS;IGa>_+I;6 zsU}Ac&!G_?9R%Ed5vXE@cr2yYwVlYf3sg3|zwd6Oas+Lt;}+^>nW&n|7zYA$B*`>r zie7Gh$aalg`?q$f7`L1m(B9f~@K~hBDk8c!rB=NALxs)Ho2p4X%AaHA`jFU!@0n zj(Y^IU6Kc49t;{J#2G>p>tY9TV6tWB&A#WZ1%%yBy}`D1!uZX13LQWNE~xtW6T}yP z^J1^^rPM?7^51Jz752v%>2iT46I5j1heSm@zjA`$9wxN|tFM0CfbZU|19L60&mo^| zlhrS6SwHiCFiy}iuUK#=l7-Db@{;AXB{t*13FOeY$WQEW-qO)weP!{eKD9~2VPAm@ zaWj%CkghvHZmn}lXP}~GIb=;;>pd#4%Fwj`-ZQIBIG^4qz>!p=lnv;y=tt?nWefC;*s0+F_-NFlZ z>Dr!w^ojAs+%ezx$*J0}L@I{II#qFWyF1UtZ^Gk&1NrC->o@(@r7t75P28n)=~6%?M6mXZ|K0uUKKs1m4KVYIbMABB*Y&+XU$x?X)jPn5_66JG0hn0| z4N~%_AQSH1b=)VmkUh}(DF@pVh`bXJABZQ-;>jWe`mhg(?ZIfGdC*OKFkkcw`Doz3 zX#oAi!=F7an;x6y0xTR7pW{OB^-K-2D8}b7j&z8oP@rMt+1@DvtQ)&tfgD?M-^B=? z=y;(tW=fmuvx+6QH))NK?KiC|l{Hm~k46%+h>D%APr+FMmmR1NTq_eFE`)oWC$1kv z-n3=5Tb6G*nCYur?9cH0e36@UA$+O+dm_gFgAI=RLgNF07o!g1e%sZG=&#eU*E3{} zNFGy8)+tGn48t}0S#?S%^s8E9Yg0r&6g5J` zD+;xn?j{YfCwM}MdX{?-+ergn<((9O0#l*-^2ip51kwQRaSWs!Og4b{8+|bBOp9QQ zdtdwT)jtqS0HFH2gAutuOB^9C4->4$ z7*hN6@&xx3$qLm1npT0wT6eB_TzOuDNp{o8!VMo>jf@?{B;Qkv=u9RDdW_r=&0n_o zrSyz;;6Zn5PICr5LG!1FWTqxYzt((owc2)_Eab&iuRX=L4}`1s>Y1M)D8k(0GN%}nP}T1X?XW^Z;61CVW5q?DpU13Ko8Gf15v zRKZc{0s;XFZ(W#V8hJb4!OWYjb}klWa%Lu`Oma)9mccXPi9E@+lZ-#kZTesSoU!H& z(!G+hzK?8!pI3gkyuXoX%31qza0bpo9T9Va1fr}1`7J+6uX-BW(~qn-BNYe_#@TZ1 z7cwx7yKX>*um(wo&jQ9;YDQ2WY=e36Yb2II=^&CY-lNyPooCBqMj8p7yj0lm)c|xT zS4O^|Up{@m_drHhE01S2Crt}ZB)FdxE@NK(_o2u!SlIBkTulUy+r8qP6LjRfkKpI$ zR92#(#M-98ZLXJkf>ht{YgEGH9ANc!;dR(xrG4Ke!lbj2lj{c3jw8 zvK8f?{T6eoaeCtRDCYGPGib6(&%Ilq?E84ex}UN0dH?Gl!TGNjg!6-nxWF=OZ|>Z< zL8YxS#S*SL$#=AUa#L-gzAm?wGpokSv-4WZ`wJLMYj?~THpphJ z)niM`<_)w))y0VLl2#HA+zg!*LNPbUUNLE3+lxOz#<$Bdc*BZZigXU0F<#h|UUU>< z{I-vq7zbsOU63HH{tQrtJbE^nNdSlMRMbLEd|YwL(-E*{u;^_hiQ(%aRVyfGdGW=$ zi1cvDJuq*!*&B`r(=vN@FeTXzod<2j)NlAHllI72wVQ7B!_pWjU=d=;LgBBVU!<=a zMIhah6WnbZmXMRKvD=eK>{<|~I|`75IDLwAFD_mI@>=HIM;=|N18Tu%ajH-Y+y1a9 zMpTdrcU=OMMeaZ*k^SP7U#|=(tajsa4koiPZ@&ZQ!x%_3u`v4flM)uv3X;PXSUK1U zWdcA;Dn<19lb*9recSlKCDb5$lYt#z5*(wd)_ zz*o7k3|{kb$~hB&83icO>x+}^2Z-RvEHi9E`LIYMBw86krxGz+V`u>F2To{~SqO^2 z{JyHY70%6%|FnVsfiZdik@2Vi5?|_aQV{YI9~S^HDQo53p_HscU{db{F)fBOvKS@O zN~grRf*v@_Ychqfb=5RsN5j1oXon{#H2glTK`F6>R0=^w?vt=eRo)pTD-1$wI>2Bn z6`vx>7C9|vsL^q5r#J(>a}%VTMEf;Pnk_V}jK@J#w3s#|R~f*6#dDXSl!HuR6t%DL zW8%mjYNxa3Gh?o!AEA6w){kSGqkpNZMM#w7p`%Hlf(-@mTR5=L<=@JLPZ~}5z>paR zNaXEhPT3J+*z|9ll-2H80$_6>dLMAs_UVVePM(Rn68A=8E=>PP60v*MIQ~O!B9$V} zLI0b_D8~S(Xrt}+RXMIkKn>@>$z!3i5EF_M4&8^K8zTkc56X4hz%*!h@eGvIqS|;F z+@>tR=#tJ*NY4W6#!I#tX{pB)N^@?P+f`FedNB&WqmX=2t(#H!Vb`9?ZG}?E&f;$P zAf@1EyN~c%?>LR# z&baCAJ9TrCkn(>jJ$2%LdLcjz`#;cY|E^ZlWf&D)iHo1d4s?v1)1D48_#fqYBjM5c z-`y6&n){ubfK^v1Z&>rqWj|dJHavF4K1m(g*8@2`zm+mO9fqZ zZrgmOQVoiviNRox*E>G#O|ENaukr-%lW~*-RsMs~-jlrTzS`nC&_BVKge9Gkkd*wE zrjQg|$oePkaOHuReFjDqH6#geXNP;(3m?zOHxuy#>q}Gu`|Ge^AAJ^CW=O zfIjoHo=d!FbdZ`YL9|s36eBD$_%U3tQY5xH$qSg^v{O#gr|+8AH)EbEZm})--JAzb zlXvJmxP3}w1n~aCqIF2Gn8GSp2y^#$xrJduU(uEStnY27$13f%75WUM6P0HF&qo0N zr1Gp4In3MM!)0FPG=NGwYJo^Fr=u=Lqim3sFnU135fpPdVK(D z)vm5F23BX&{a0nCLYcz|^#KD{Lc5G7eicV2bSSsJ98r#YpfF+r9!4zz4z1T9PeKjk zby{w4GR$tk_`2qiJf98n-6C-yoN1E82f8cWG`B&>tI+`)!gQ(_(cw`=5|=_VyN*?$ zGQ;_%5WopzixQQjKyjYQlQd${&gTMQg|XFU9rJ9mI;DlVdqoNF>9O(}>_7!J;FXF8 zw}z7?Y)=d^@suDNpV3o`z_nf+Aqg(y=G)4D{Uvn#?aP?~tX<9B&RXEr-ENEA;tpEZ z_eyP8k|~}hy{5<#&3i^@(VWJqNN z8_os%uub~kqkQ9IUnb^b{*{uthrX#+gRdFO#T+Cdn-5szo0Bu;kWC3oB9!-E=of%k zf-Z;XgYm6Ga6KjxpNfq6sucSkokAN$0{wd52Ly{vK9E1k1%JGdtNr}j^xJc;x^*Dt z2Xuz)zv)r`r)ZVS47A`3lQ)3tkFsbBfH(RS3HHWBvqTzX|NM9W)CVl>IwFos8Zv)= zSXJm2M~Dy8je~{Xnp7KhcJFt-d;)F`;#Fx&6NmKIUQ{KUr(Le_#T${`r#?h>+|^>w zKyOA6N=ate9*%Cwgx$WMqOKvG2IymK@8}PaitAJc;-L{RxI}6Jo5t-2yEaG zZRw<;VoSgWVLChk*^FFIx>o|WFwAtZIUpxEC@PKoR>PSXSx!F%X!qTJD@i>v|A_wn z(oh8+ES(3n@<&{i!7 zo;U#tP-y^c?{Zy(X19UP8c_;f8!gUme93*_?uLVyf`V~+PX4S{i}8XY%PY&(Ap8yk zFNBoa1?By4}ZC}E-Uy<{OujO8<Y-&_E0%_xxFa05thinh${ptU;I z+e^@**Eh0WRk&8nt!LMJn_1BKyi?oW#*`qKeeZAn`X+Ck7}wF`EU zvCLnBN5=^4tcRJRWSD5l9<@gkCdO;LX*UN<%2yBur2xL&9N)GXa~tUQ`D-|hF9 z(S~#5cd8e`;G`lR>6qr+9Pb!niX*gPnj`w$@*BSQk{^B8))>F}6+5U!wBEoqwj?D_ynNYt5K=sZCS z0UJNX3`G9fiqh!)_doe+sTihmm%W&l=oxK6^v4;;T_sF`2Amde%~Sq=QH1Dtv`$jj z?Xv%5%(zB7r)H-~;P{eFu7~YGBKrViN`C+Ck-?fCW71AH}@ui#)xSPt%vcB5(?! z?2AD9rk5ZKDkKpmG6!DCk9cA0;4Gri&AS#Fu*bxl$u&GS9*=H44@^0@6iWAkT*T;4 z0PEX9g*8F0QZr9t@$!Q^XwR}Fqnr_`;~J2i=az3EInKke7gk^rH7uRJ@_cgZJjb1F zBopr^{VI4oZB)1)*{R4LrsN`JxRXD88^^;Lt!epZ4C|hNfCBkPyW=sYWZ(V`U+7y2 zf`yJG(HmuT$gWqEn5-txL7{fo7Q00($zng8*)c6pjGW~v$)Z|X{Xsr)BUf6()T{n# zZ{t^)Y`GhpHV=nN+<(jeVa-KXPCS?8&T4v=vT;lFC3&+G2MMW-@aNjG1Lc({v6s0H z3@Tqx$m9?DT$&s8pj$;zx(qBPCk$OWesThZ_^&)eE&7wTigSah`+pN!Rsb}9{q zC513UHT^pkEyPQ<>UQ7Yl_KiZjBeDU;)GZ9d+0#$j?$q#=) zT#Cjp+l)e>6Q}^XV0*!c0diiCd$RN0sU!vq?dvMxH|f`V8?7_4Kf4cdza~r(e#~zH zek&T3_Zcb${is;aMLljPJB^T2hmb2I0OiE{%ya)QL()gg=CB`4?t^gaC7u=D93$G- z@#={x@&UWQVlrkJm^ly9W4y3bhSt!SO%o>nYD)FifQx+k(Np7NkVOU4V6^A4u~POE zJ(1olj3+p8yIMn3DLnq?Tr!-)Z&Z`e7G8|S)wQWkZ)6=rL{_Os$M~8}{O27*C;H!f zU}8u+M$u?LGCvS+Y=0A`J)}w*%z@>OHD3wVg?!7TCrO&Xq0_|BB6iQ$0AMpZ!1a18 z$O^8$!0+VOmADTbKsa+m*ejg}t6qSx&&seVMI#|XFMT+fWo0I#JNR^P{D-XnG77n3 zOH0ocEu=6aOiTVb78~180wk0eJ7y)l>`^}aPZS!-zAk|HKh!7IK^Y1qCSjlVR}IFe zN10c7p7S@qOEPD5iV1{Hbv5M(j-OAyI|4dlZ`Pd2wz63twO-AS&L&JS64e5x*8Ky; zeX`X#fRAdrA+L$Ni^gZd6T2|fqf-dhO=tx{4Qhl5FqsQVdfqP3rma*Io?)N#1YcnV z1m-=Hol^dhtm&|;tCvaT7tW}5kSrUpwU>47bp1#`4ar(?azXZBHc#K9<5Oi-%)h<<7gQ37sc_qksbZ%4T}6&`~oalg#yV<)%?ta;h#C?~d70b1AQhcNNB8`eHKTWMx(5%M#_aRJ!j9 z7Uj=dM3)x#RSnRMY|NdB?2c01zg@?@Qjsx^lWP+hZrL+6vU68;V}DVn-pBjqjl51d znUXX~Y-YQw=56ItsrcvzyO~Ga?`oHh*dXymv*iWw1Nnu4KJnSBdTg!wkc$jmJ2u}S zf)J&S@gEk@-0*w8JP*SkT*xrujmO*{msK310^K*6lZr>l)NO*01f}49G4_&N_rhc& z9=iO*d%l*bwgSI{N{G<|_#j?rDDYV}0fcyaRB$M?r;7+ey!f^6bxs_M9zdC4n0&Dd zjIo#h^1f%r>XwRCM_(r<>Fy)BP%DMR&GRq|_?Sj=Hgl-;#00yXp%O#UdL(;s)y?^_ zu$b|iWW+jeiW{n^4F17a{OL3z@kdwVa@Hl}d=^f{NEYfxdX?slYPg(F47J_U_ZN>n z9-Q`#>7%D8gd9t^2O4GvK_60 z4Z@mD;6SG8Q-O|&UwJ8<0(_-ug>B&H><4i*`?9vv6CNGtDC5i$0Y6-gCnIE^n>Arb zRXO5Y;0Y~nPd7=BqIh2+)TL7+6&LWf%`EvLq1}8=me^O_YXZ*k#X9aSH@XI&%g5iZ zeMjiQnw$4iIHYGQO=3HHD!nRfYG%$T9BX20AJ@$2(%P9gde~4A+0`m+MSBPWIAgsp z2R0N$-Nb{OIHS&wl>;K&VftigjL*SB(C8XgnWaq;VxnvTRFf+`FJ-}L&+N0uH)ySL zFs9J*7ZlTNN@ym+1NU62%8I~M8RERx7X2e|&Nz)2WfLQE@9^&+ydY&CbYcuMO+)v7 z3gGuh(qt7`GSg_9Qeflh_!K6-X;Vpl_;x3h{)Oau#wvT%%R8}(B zyX!A@Sl4n|a21z1DH*r~5cj!+({_)iJ)Qnub9D!H+nARH;ZCGgPRCrRe;xQ$%*Meb zr{N@4OHNY!DZIg=KS3yW_bl%ep_lse#RIh0?jH*>=Gi} zL>GDe$oX*~`ZxdisX+aZ%gS|AhH5$A=jFMRaHiAgUj7Ew+tq$k&0k~-4Ak6;{84PW z#GxEMm5J3KAa1g`h<8dM0%az_upMb%dcAY@r`% z%HST=mEnOKF9l==U-qZglmjtAa!JESczAYQ582uc7D4p+dXd}$qG?<&l6QAqT-o~g z826?%P2$C$d>8(YBY5z__jtn^35r4l5Bu9o7|9x(J`$Lz2t?^D;K%uJ-Is%h&E1JL zGm~JK(Ge+-^95Mq_+dB3+=nl{_>~{0DVwV}?28%`t6*06e}e#eXikvIDgi*tv^gRHk!Usiv;;op0G0$ z>X|Uo$UrzutY#cP?R|2qK|pie&BXFz6Z2PD2jn8Po#Kln=No<|dPXNdOimvbjrmUU zF~)hWNbjAyLUED$=S}eX#l_(m|ELE?D3b|KKk%w)MQ#Wa0Sg+=%iq@-lb1+H*;r*> zZ9|wXpi$1gQ>L{e<9*WXdjj@2C3*SKsSB=Am>2}5GFZy{+{`$-2~gY4$N|UUHDJ;*l0KLqm122oww@FR8IYUcYEnX!ga%t9Yo; zf|sO#?EJG9fI&@T* zPM>^dy#u4+%Z!sk%jH0go!ZRNukn8p7`tdeWKxCGw&kFRhwsF#W&gV z1-ksvEJaBQcjB)`eUB0&_~rV&af{>~6Ed6dhr%tjJJDix(cBJy>+hM*b(G;FEr&A5 z?k}~tdr48LL`xo~P+9*}DnG;v52dnG6-pWCZ}E8!L?3JhsH`~%PISmZK&)seooJJEpX0S2xKhrWSR8@34m^+&jfa z#oS;WEOskFnJ=M{Z3pm@d@}Sc%J6CUzJOVn87Vd|8`h9nkC1Md$TKzVQcf59s6OxJ zl$|Va+x$8eeEfVdd8>xouh!neoGoycnFNlOXNX+w9Bb(t!i<(nM0EOv@m${1kM``b z5i@Oi?d4!5Hjrbl&kWk`V<5ezT#@=;X z?H&ka5* zsOwn(apx2J4ed_B`I&>gBkf(cuWzX34pF={oFng(_^jD)O6G&Rc|8RTks%ijpLamr zhS$|A!Uv&^>qXj=U`HS?(dh}Z!6`C+#p$)JoKmkY48I~JGcyXLOX3Fw*|>a9n4j^x zdUel$p`_cp?V}$joU!GB?3ER_X+KQtK3FrUBbgoDX&~TcoslpE0pF|{L}@3i`0sk5 z$#(M1YX0?6{jVCLA~=J>s8vZh^(uN03jEo;C;rnEc9o1Thcx}sf#LH_4p%StpuQwH z!XZn~6_{$@%qkuJc$g?%el2Dm>#s< zX{x|D$lXK_kDXl-Q~l9_8lqIfTobry#4h0ZJzKJP*4r+P2`|1%kmW))gN-9dOBQzV z=ku&6d2yIGUATKCrVw1Ru0uU6zVdsjC-qiN2%6sgiTY*75z(Q7brX)tssLL^jarlg z6WB@OEiO0Y^qKi(W#7=TNO{NRy>Hy#w>}AJ1@XYTxWiZLjf9-#|DXH=sOdMHF3rf(2NGKOz7JVAINwi$oJ_)t{ z*8jUn0xn@F>GmgEd+*!wg}$5uN}KeuVHl&53BKJcm*;VgXvS87Oms8D8N77FOt~ma zVbcj)DGjIbsqUli1g+*;{R4fq!ZzDH@Gy(Y?!Bw6Vf82>IC;&}z4N-qjU3bef`H~I zab4k&?Qa$vhAId5qu;FF@Dtaj1Zb)b-1SwP~!0*VW1?^He@cpw!rbfg0}IO_fImi@o;rE>E3zW{{b^rT8mtuI8`H24ln~EE z84%9o1CDlcAbrRrCz3=E0!|Fu9LRc&Bk)YfWN_nt$x8JcT*3pB|Kfmhth~?!z zswvr4w_e7(BvL%Dei)be0g~}#nOIes9U});M$C{)LSSb<=G#x@)N_>v636FLE*p*y zeyfNZa}ix|(t0W@+Dr}AQK#Pa(B>}m<x44-9Twu3SGnRJgGN0#2l5IF44@L_KVs9HopNaYIi+Kn~I) z^bYq_$x)Po30zsgDNCHlIlrnE-&GWHw*NKN zb`Vjv!TP-We>xqlX*7Q(3F|UH=imsTyzC@+dUGB3pr2?eSSusaVxyUd^gDRSwn%A5 z?&_w;{$Ttei!_8s_ekQ~RoGsBJ;45nMqn69O7r{M>_+}Bg&8St%OlRmzb^OetMBMl zTetqabe>h*ueE~IP-|r4f;5ZXU$>VHfc`rbWGm_pD{CU~-VO*44==!q1hc}HSv3V{ z_!NL$=o-o}Y3IuX-t{gUS+h1vuxRB*feHJV;k?J+-#3iH!X(I{@}#K^^K?cWBA!6Z z>l5)uzt7xY3XFd=^M+UX0*LKY{eEu%&;<;OgA@1h9;Ponqs{}Uh|@|i;L-s|?UTwv zblSuFu-Px2z=5vz{Bu$gzBgZ%GpKAs!TxzaBssmE9K{>xKIB=FHC2^k6cYGk)ukvG z{BYyKhtrd81FH7J(hJu7Mn)#0$EZY&jn=m+wOoPX&<(nVuDh|bJTo&coba+E52wlo zGS%Ak7cdr7w6&EdUbh~-Y|`|YD#=7tQMhzcDBg}osWi!ec{)d`1)Xe=nctso(1oA% zL(mt=tEOp^90N@!^umepOYH2dj$=!}6l^6NVQ5Gim0K{l30ls>09`p-tc(Y&xnv5t z1X=1(KDxMIPJ)7;o}k{Jw|aOR;8`jy<$XqvFsDqmJq7H<(DwHBwXCof0f>YGSU_o9 zuyXXBp&Rjti5VVJ%sV`*E%Q}@onP^I3dJ|LQ9^+r8M?Cq8AH4Dw>XUj8>haRVdT)8bckGj6;%^ubn0Uk!lMbdX>kM8+B;xYD=#k( zL0Z-)^QChz`Az$*#KEguehJgam!bg8NwkCgy#Yajf|x20emVlsJF_j0Ml)tibuk39 z2yg*$b=t?ykgs?AJ(>Cw$m@4m!M{!)DhPcejlSN4-s-#c6~&Ta;S=ug5>4naPsVRC z?Y4!Fg+d`NZ{|s9c%MIV*#;wK#XDA5TTI370 z1au9|d+GsF9Q!w(^cB8In+x~)=s>(w1ki(o0-_EC+?pXk?K^1ut}v^+ILPz3vj9+$ zqT=Jj+uGW2P=5n-=)o&Y<^z|-N)n!~Q-PCJ!z^VC622dw>w1uJut{XFlk<=7BzZda z3GMMtT4hE{Q@gX+h$E-RJQAdi?KCvv4kBx=AH9DD1uHcI8)bZ1nP(4A=0|~{Xq)dn zWOJ;S<$};53@jnSq45}C-000Ti{-BVHSg=T#Micla>J@=6!Hs%sgNU;gPcrq+v;kV zlM}J{NGG0C-1Kj%u?-W}?>5_?jg^!0T<~{N3(4;@7|V3k3bVNc-9e z=`^F@s1iYs{JE+h-qZ+(VC@)WnunszoDZKAE4nYC>&k@Ue~Mz^=GJ4t-?^E=D?{7y zR;mXhN&a}ZgL2ZtevFbtSh?3$?%-QGo>8XT<%H)EoJvC~T%)f=X*W<`(Vy7Sw8<_T z8@~-OsvjlXDaoJ~TQ~k+EC4gIkQ`bk*6yBhsf{S9^{>uwk-(QI5rk=sv(u~xyPucw z;Npek{4~FqD-`penntjm0{1I$d2|A5iTO(9qy=sMG^=a!F}v zVSD?d6q65H;!CuFSGqrbng$`}+Q`5}vmZ#K5sL^x0RaRAM2V|Z6j!V_< zZ;WtKZ0Qj_9>jP;cZ5-RN(Hs_h?i7zTiWAt9qvcdc~xc?l8oD=FdfrODs-Niu8c>{D z+9|K@?mN?uR<#~<_mm4t*5Io@qwcME5FV<3A)nZHJ?F0?s2fI)PyDC-piD6ItGaUX zS9mPx(*j1;7G*0U0_B~uOWKtL4PDPZ-d^>Ud7MpX7CDF1g`KpovEq;2@W7POOEQKU zqXd`Gd?R7GB>Fln+R?#Q^OWRzPUS;eM$=Bb`)ywl#owxA;zshX$OMbXhKy+zB3#nB z-#xG9#ni1jkKr}&H*C-m_N%PmrGJN(PiuQpq8Gn9StR@WiU5QI$wdh#o#&ExDp zqm0_35`f_riIk?=eJD?6nmq3~q7{D)6K^a7tE2!cb-Z}sU30aLgXVoT#b5s(S_Csp zrx2Uttvi^o6+iBJk*?yN(FO0NwL9BRxK{U?0|tp3M(A@d*@q%yDe(x?+);pO5r!m7Rz)LYgU?jwVl#RKF@@OKYQuJ zSlKnmo@OXKDP_*smi^)>N2D9yRP>6&fG5ujzf6&*`QZ-cTk3r#x%csk(St^dXV`SWEGZQ?E}~903ag`&S>2R)_PiSdkKUXs`qx9ID5f_ zhL!-x*b2Z(F2GtGFV9i;{?0)sr~sUr%pxAqO8Myc<+495s4;2j*VC2`2l^8gUOude zUx3lZ4+2P3UE<04A)JQQI&X*QUj9PCq9_f;021(093ti~qVTP#ZpI7e>0EL8@8Vos z8iyXmktP?e#okyq-GE|Of(zaf5%gr!bEg0XEa==JC-nxL@@K);lBC7wcLj!x`LD+WF@EyosFUlK5dFm=;;H3|$c z39!W=A#3p2(TUFhlT%;o91p@o2 z@=|XCE>wZ?=#gKN?x;Qh6_(iMgrWvYx&fOI4ut^p+qZA+AW88XfV70Tw{3!PNQw{R zmlgosHzigpz_2ji+0yt6?hq(53OZr4M}*-i3rDere}+DNvhodNK-@~RKfYIrxxGbn zw?@_a_64#@`{J9;*|c2mCZ>>lG>jVS-Q*0XQUiVIF})=_Lcgj+N5Pyx=EPt)WGv_) zeE1Oaxe%r4Lu)3-@^Ks2%r`LAtJn6fAFQIU#=DbvZP@koIk${dlTl8N;snw}xrO>? zsIyfSGK&(Y$B1h6!;B-Mx}U0bS>`NBPH9`QMN)>_&dWJgJLECtaQ1UwD!vzzVEd(~ zxuW|rzn71h%Q!g(xayf;nt5-G717KuvHI$ zgmiO?{0iDKnKHhI<8yLPOVZv%$t#Va#;bGydAQ25)`cH7UYqsJqQs9}0^}VuB`Da4 z(MKQ?+F&?1&0gda{K)1=fg@;+EL?xDcN2kcV?wjz)oYR;d5tAhvdne8uRQfO?2#5* z6*P1VtX&#`W-ryc=C}~Jl0nV@MQ;eSUeFl7FniPv4GkGYv6FBY?7}>ACA`%9>bb!w zn21$N!NsL^GFIuMeH!r{$V^Y)$tU61yAO^i7SKvWu$1*15K1lRW*#o){a>es6vh{F zNV-0#WEg^MB%|&-T$H$fjw{StqBS;%cZ&@2cWtw+29S#X`@_7#lezbsK^bb|>grm$ zD3ETQnilB{dPjCk)mf7jNUi%x$BW%TOkYh}yR@qu4!?(5LL5%L82?&v!g9ggGfD~6 z>{WQn)>4Ucypa7N4fFS55fScDwLxvs+*N4c5>=t;7?&;}A8_~4$C~;2E;K}TCO>@E z^2K8ilE6_n`8){7pzQh%uS9s@+n%F&f`wq}lx7|!CGDiTq=_e)@%Tj++Nx4vaT-}X z7K4sUb8ap*us7lZ3VG7g{7(T+)2k@?;o!nZp`dU)kq>{!HkgYyBfkxJFK7pgK0t^x z2T*@ACe(Ei1m#fBvE@LC?Aky_{;!VdWka&pHU@)QaJfn%>7#JZlzhsWS@nKZ*d`uu&iUzI?_TOeNk1LeOdyFM98 zC0xpWp6RxuKO9;aoB#d*fJoRM{9%_y0C&K(GneTIObj2{I)<@Sxl@2O+~EGZse&vP5StZ~6q^hw#- zG5>1gv=|9Potj1^b+_jCloU^o=#}f_zTBOLO*|bq`!_7dc5Qecd_`J>b@Y&XVxu!& zdUnI_Cx^(#$5ONN3kx8QRoepS;6PcQwj-qQ%lL@9*i1NWFU~!uBJsx$A7ig35U%j) zSvR;%J^)_$l#jf`1%C6xRT=_%OBlX^pudUU<^2WS_hy#v9Jn3^>z@v|Zh9?ud{k)- zCa9OU007sN?SmNtPK%9SDd3+6psxO+WCUfDa6|&eCm70iL{)&G)KpHz#p0*3W!jB1 zimLG7s%!CaRHP+Y%<~vn*glw)fh;bBhsds0_RM2g^TThc?||>?UoJbn<6IFMvbOU- z@4x^xMQB|e?|o{qU=aIa1tvm1AcBE;Rh}=!i0Xj3C?*$V0`v(ln=xMIpKbNfX-i)d zkH*#lk$C~N5}?9*5nf1XD#B*lHuovOaYYs`C`iK+75Ir?a#yFPk>@P5r`sp41tzoNJ< zFE3LESCDM1nF{y_NaLU^i$=Dt`}_M#c!Jz;A^IOe`d_^S zj$<8I*^=B(t*|3deQ^yWr2k${IS^6m&4Mwpu#DrsxRXLgm-U$1^88)duprz*+%Xj0HeyO1hGiu?i@iA()nB##Jvl-Y-D1m96x=q zf)4hvdTj9hPD2whKL1;iw_yLzN=yg#iLQRj~^p;h?8-l@w>zHsD6M+*T02}=-vF;6tLwPnuxtbyjg zk3$7Md?hXg-Yf(1&3VtBuQaUx-sJby+N&;Us}_9M78zv%JK0}fQICC79{xJs*`~Jh zTS03LA^2(m1uk<2OyuA{$B2T8it}=NRVo+_=w^VZY2c*L14OJH$bWG3buctnhw1!~ zJ(#5yaUzGq;STnsh7+)3Jn*lt+E|@*umA0-lmES|{ss2`>s^h(JUloUv-B`&7w-bQ zm@VV#U}+k53`6Rw>T29|NY+3Dnj_=aG^`yBl9iS9!|vM0-jTg~@MY=! zruDfy>SvFHiN*bJTWV%zw(yvrps}$Ryt}F#R=Z|)z|;C3wc7O+B#B2Gn5v$v_q9Y^X218mqB{iJ(KDdYV*iNX7+dHsXjI9TF(o6#a>=BPY zf4#VXkl5yw#-Y2J%g0*IXZrb@)(p^vW(!0MFt=kUpGWXn#PD&V#(nLY}6i%bTz*|}7NM_yz9oaCDENTp2tEmk%>az_DO3s-ydY9m zbiO2U!L0O2IXSU_Mox-}iRmtWh2{^;kk$v6nVIC;kKlEET41|tB0|tkBb$Cmg=9Mx zN@f2opkZ5pZh%?GoIy_SWhV^;4J~|hbbMIx8!xLUp6@Lh8wZb)t}Z`TY>bYX85|gk zzH7YY)vsQ6dfY8-<(xM_Dz3c#G*QOpbXlDgzPCYPFSD~=qG-Ay;1I_>(U7EjWh~^^ z8(zFzS~|15jPWNZ=wLh&d(~5}GK#^teFDypGe9DKK$Z%o zZHEUBYdO_^yMv6Kto0uy=SZ>E=BED>@qZ?TNfk`W>E{5h{& zt}iBJ%Cy^`adeA{AtWdYcKx8Zz*2}!OG&>Nb`=#b=cHjputd7ueSN7B7o3V;aK)3h z?f~lI1GtV!boVG&aFtwL>r<HJd~zmDWHoyzKkRl{ZXNyv4~m{bJhUK{w@CE;Y3n54&OKuqqO zXg;g_cYhWhfmM9P!k=&H;bYW585xCoOiS)rjO$;Qa}~z)wI{T%c%~lU6Ob}pDkR%d zfe~959C16%c0<7&H8V5A+lJHG^Y`-2UyIQshJ(BILKg-;Ai|o=LJO#I(Vq`Ggm;l1E zTj-wO>kwzzLSCiHmpK&AdZ-Q@A6qTXdiIO-+WPM*S2e!dd1oI))i;|N4WW*&K5JyQ zy@aQ|WmU~9ndwsHlINvx&M)pK`dm@6P~c@1UMwszKK#1L;DJfknEH0%rj>tHIt0zC z>TvuDlau3=r_ENrdGH=Vr&$bKjwcrOj9{KC>Nu3}TOBdlE&A;xJxTU=4P9Eo-AHw- z&pV%9pd8Px;5l1nd$gB9?)P+&K=gH-z|lht$P&X}j0_K3g7dWt;;? z+YxEG)Zvc;pdphKJJP%WT?R;6o6|_L-j6>PwBO%M_MLciz%#PO1~jhg=?I+=sVbqsT&)Mlnimve$o0x2NB)N=Twv8ufV%)9U=4-V;YF`mXUk#_Nj8 z{jpmv(t<}1TkjQ)KkWCc;hGgoFO3_fRGlnW*5Xh${v>>DB9g5@>T<0XZQxfb_ z(O;4QHK3`aN#-cmPS%GBHMM7^Mm5RHWzWe<%#h`0lcBl2;GYTputAvH^C)h?UQsMa zgwEJF)LD}2D58pS(7I5`v3Bmc+aCCCbh9OdHP8woFt z-Mu}|maw23Y6y(;N8zL~%2liW180aI=kxTmtfn&YfLAuc{ za04OZTez%$gc8gOLehS{?#E*LRRGNAej@44RrkOzbKFx}%uNeGp>t8|>y)Nw&s4pg zip!IcK_6`8A}BG&%w8m8GD#}HQ!uB{rRULTd@IU7>%(HM$!im+^zL93CtQjT+hm?t zN%&C9Ms2~2x?eSze2)LAH9P{8RX#m)jvf99lwN8D$@a~Kr9}>Bx2KlWlIBskk&O+P zt_k$5Hg(6){?E5Nowe7lGY9bmM2jmQ`ku!H%ix3WjZ_X*ciUGfa+@^wnm&KWeDRx~ z-HG#AK2w|h0*ez+#vBeT40M9#&Hzt6ZXAL}KKMcz=~<6(oo^iI??)U&k4dlR>;{9z z@dh;HWT8wQ;`4R8&&85Ir$WgBpoI?cr@>IXyzA31U%&ss*jtB1*>%ywFbq;kHxkkz z-93_$(p{26cZUo}cL)lIgp`zYOG+apjY@}zG?Mb2dEV#!)pvdImtG@|-uF3YpSAa1 zYi$MQkj1oE?mMyY^skl#Tnefgm_zpk=vr$~gBLUM>?OlTpW8x5BYeTZy{JZk(jlFon)!JguX43bp+9j#NiQ{)0Dm{j z>}z9qm;36g#(+Qaza&#QWXdBHv*ztStSyN2Q2n_d+nwx^Qc6bSV#3qO2jNl(%|&A+ zEGm5(#AYQ10ALhwz+le=X7b!FZTmu=v1C`GMcj*mFU-aYFZ#3zUJE*sBIjTj9vR~~ zTK|{O>5YkKaGYBM$%2BqdZPc#M#TANGV?q@7TOY$dJa8JM2Vowkqp#J?YvQZ``x#^ zJy=I>rK}{k2k28j%R^NUlQ9BKNMd6IC-oWunOx%Sec4OM*c^ zv-a9dLny2i=qw^e2rS6&go(_cHSpA^6b`Mbbf5KxnsG)r%_D2owM6C#_&8F1(zI)8 zL4V*(v|}SKe_I^PF8(Tf^nFn~Uo2ayjWk)Yx($DVFWLZw+;W@nZY3$2p)12gg%om8=E3I^->tba>?4Tw|U4Ze&a+1QHG2z(_ zQizv;2*{y!V-`g~rx*t~uQ8oU!!om`SQtnJ$GdG?=udJ;y1*%zf|}AYuBI{qk6oCj}DL)Y3b6(+zEe zN8SB8eeVc2D0l_eO3BQ^q-x}&h7J zH7KW4c4Z$;sZHtV?mT7h#EyfeK52Qccb@t90%1;OODwTNdgv`~GJ26dCM&OBr5w3)%a>zsJg5T0p%b z0A_4m#QjFm3(fl7`0{}zs0D!L{fqfqP#;T$F;r9UF$f8f0#heB05iSG%)vv;A#=pq z=q`XGV2tcxBK!V7ZU`WksJu)YhKM z|0bU#sP4Z5*UAr};b5z7af(Op^*QLvRTt;&+&LB50#spXugkt&*tXYd8Jm4*4_zXJ zdwjE$+(^jC(vEj+v|Ty4HKhMFM7Kgh;(iK<^<55@z&BR{s(&68fwD$C)% zP2!)tjm4L3`H~*x&Ol@_Rbs+}U`8b%XN#U__y?)kLB_o1Sgmy^6qKW|_@_IH>p$=o z)sW6HKeEckf=}AAS^aEqddv+Vmd^eoC}m}YOpWi`;a>VD1)l-ZR+y9SqRRAk3N^H{ zA|UKr-SZf~{9J^ruWDU*ON;8Se!V5FL4ZtP1r7z(MVn2v4*~iMPyF8lKcq*JGeNBO z7+~9dArNs}LD>Qv$h+NT9iee-1Ss(i;V*;AUO`pL`38aKjVgQYofN_KIyd&>{Q`$~ zsBagEpTkUeWp2rjA63RSh1jnx_outJ|2GqEw$Fn_}fXs{@Rrmc{3flECYev+W z1a>@yw5NvAWifm@aU3G$OFP?=3{B8Dp%SaxsyJ3IQt7s}lE10AXGhBq)Z|0znJr%j zytF^LNFZ<{M4}lixbWLZsotG;Ai?L->yn552cNGjVBUao2e{F_BEH-GoYCH-KK^Mx zSrF0sW}E`mlpeNmcyY9T)Qv$)M^7)4DO9VuhzX292m!$^^Hpc7&;AF_jr_`iag(mo z8M}`s8d--Ls6oLGb{4BJ(a`;wWrfD}grszgrHnc77FF|2X9b5OAoGqa=J^xA!~=t$Al{o}vHkb%-OD1X(`W46j}Oho zr>h?N-VwMCnt3y6hE6IZm|t}vm3bdAlei?JU*HwTj9aG|ESvik^pWH}qe6qozfuz$ z&i#>K_hCC&mzxN#@KL8Pnt~dUee?o%HJy%tXV&-C)brLKI=5f7nVlbkkV@&`L{wd@ zu0o*=uHyz<=W!0A)uMSPWG>*O;cf68yF6|lkAC)G_qq` zhVmts%gbCw@ppTBupWSHQKi`!~7GI|Rn%vQjX4aM_P|tuHtXY$4s}pyen%%2) z(zSB<>PV>9L16=8H@5CO51XRKb{&6IkZu$fv0H!7x+Oars@|5LF%8FzgH-KKL+^ZL z?^AwV|CvMDL|`t06rlPXe0R=u1STSWpg^@y_apx4Hz`ry-w#vM(k8)pG*c81aJd>0 z34p~x3VA>&01F6c^`8W;0=>YZdkpy&n~5MTEiHb)W$XwH#(N}@Q=Zi_@_j}Sum0Ds zhwW_v!k?COAJb;BZ5t-d%iJfN5klX#VWd^HmZ2bcE>X8Zo^B@igb7YGSs{_5LiicH zylkuotxv2A1!w59I9Uj>)tqgaGOXeqM&GhSA{M`%Po^uh)Uv70eV}SwzNvi{FYxTS z6kE26S4Klth5~2*o%gf6$=uK0`_>qU;$bC*xe5i@?(p;WijH5Sn4jA9&a}%T;z|1Z zHDlwG1+a|cc?FHi8fH!Y<}z$z^TH`B$}%@}s55*KHRJP~rS@r3baR;*qaTl znwJ#Gw_5K=uNqKU{kL4ugb$j8j}`MefOoDp4%Do2fDZOO9dtay zMo$14kUPmRvxNJl!~`g`A^=?~pUW?=axvgx_21VomvbyvneKVPO+&k>hIx)o1?oD#d1uy+SFf znK-Io{4ulG=(YlZyR!p5Uu?m1!w-&~cJB4UrrzPus~H1ta7Kvz;7vEa;- zufw+C^jw9V&hBw4sQBlNdLo7D=%7Mui;7|1EDAYI_}deKQ}!BKiGsP>2qQUA>USV8!PB?}fmI2T zAYpg*A6c~p6*)pI5`aN(r{29T4`fZ~2Dr8Rfw^w52p*7Oorb}VTX6iIFFbORq95cx zLZK!#3gf*!!Kbji`EoNdP1J?}sHWIQ{+;2dL&&;BvDuNW1ixu^G+6@%B@vw22~%WK1CxE))T|z0<0@*9^xkEBv;@H#YEVU8eE?%yD)_gnho~$h|18$A+Gmx?wyV6pP0~DBuB%&+k4T_zX1E?8?uIYdF2h5d{VG zLG41Q@2Z}W`X}DU#N=7|@cVM-p))?-ADes|Xi88rw<7s#R+PAspv^EW0e?qB0t4p0 zQE04;-y2@$ilXE&D~d^ao8c^_)U1tsH?CMI&!m|fMki*^m&ux67{{Pla*7a#08I&= z8E0EOZlWDq@OQ8^b)zUe<&lqgrk+P3w{zCoOLjErx}B2g7~0t>Ow-)T`6FKthq^!rDWh4?xGdAjAP|1I&&i`lqDli`t zSrTf9>0@%I>L&B|RG6^s7lVWPfk3AJYt;^aoZ*;j&cIiW@9}oUjzoy_F&6W+(HKU; zSC4}C#XhfXxm#^EFa!UR!Fy>m(+w%`HF^j%3iv-*ztD_9{o$n28{0We=jSW;^`zI> zin7qIGmlTTCo5xCu0fn;sxN2fGs zqf1!p+Kl9xJi>d?h>`urZm_>NA=d4r=n@l$Z&+j)jk9h=n3zhdNQf^F?Zt9J5Q;=TVP;CTFHz^-%7r=Vt_ct8RTWsmH6*#@~DgNqilzCyW$!Km6(# z<`w$c@YSld*6FcbWX+P+({3fhcpFzylH$nNq6&9*MLE6qsf&Uhc`3nX;fwjsy#yZb zKaUoxHNp9nG&B93^%%;yB}*{5@7zd_SWavXLaSE&*1x#);HWXGY=je($|E6)yqhFU zP?F!)*0uo9Xj=^)fE;lCo=_OUp$QUhBUyXM5d;9aJORzUC!MUG9v-7{zvrrH3knNu zm?Y@ufEZ!yIhw84MV~4De60-*lKeZGBRXxWS~cwhKu>k^URB#iKw0!~&v_&B z{Ro$1L7rHiL&A(hoo>nh+AFH8F>-FKh_n7h;kfs(D%tb2_19#HV8)^){bJ#ZYzjh} zO_nLY=T~nyF>)?q$V`8}^A|M!a1-w;o)SKnkxM%BJ2a1uRwB?8s(`y=$&45&+q|dH zg^EO~DENjE)A$yUF?)>@xp>$&*-(iuq}*olg^Ed*U^H^U z^X!Xu;|9jtZV@?!b2iA2zUmo6jh?5Vp3J=+Xr+F`N`ihM)>syG_igGvXST5SXT8H5 zZzOf#=b%F1Am1CE~#=@Y%fdon8==cIy z8xbm8Epq3;AqU)tO0ErxH2!kJzgunBFJA(R7y=lcS!@iZOikAk^8N;65ObCDGhJqvfZ5&@ISVe%o-3X!>1!OKZ5N|6~%oR=y*2H_-;UZB-L%j;9k4jj%`x2 z%>8K2uq2VpWy4B*c)x^WdbQG{oev2Gvj&wy#w1Jpv$LX3DW3Jz=R4Dsq20TmtUJa)f04@{t`+qxgWEh2G;SzWGP!2Zu~qYSEve=)6<|Lw ze|>8s0|uEAE}geNO0k*+r@$A+{~QccRGD%FbFi~EC=vj{t%`91*i{ykv2$}j`bHizE9p0He2i=S^DQ$=g}&0oocC7g7zEctXpZ1KhE_6^iSi1a0Ugd#ID&& zW1FXxDTZ?`k_Yhpf??=!|15eCV5rg@6fPdi7_;fTx^p^+dJM1h0h!x zH_Qkj6y7l%R7*#_6Q=4=X=}voBTG7stgcStXneVS>IS0l$>D=WPk+_C1uQx|Bm-xg zArBuez)ymT`QX7T(UHFgcYpEHJL6>@mr+vPcFVt<1I#-fx0NpC2Ms_zfexH185&)G zBm!~rQ1RiUGRvmALS6;|D?!l+(sfwCc~NIerhIePpo1Dy11?683z(3{o@2PWtrV^M z6KtwCYt4hfw)ib^_g}ktllDMjgsdQAy4m2-=B?uXeK3c2|Abr;574egexzj#yo_E3 zrXI=!251&MU40HSc+I_)kRbEpi?c}$lSD}rlHw5M`tj`u_%d}Ae-Y5u5}`#EL=@+~ zjTRVoFMjDW)yujWQb3O?SFh_cubgLNZT<8ul0}gvV6XK?`L`9+3(*BOZajlxqQzQY z3`r!;e?{ArhLkHQj- zI&x1%7c;Uft&^xX+_(1j;=ou2E5Led282f&T!E=FZ?HU*k_t4=&!0ah7j#I^V0Qtl z^N+F?IdNVB%qxd4HC9$RSy>I^=Y_HC?(QCY#!9s<-(I-i`ix#klC`uj!RPZZh0lf+ z-?P%>6?}=`E>5!Mc9(ztXV|XZNhsJXGuRbvNy#shTIvod#46$XK}W z9IA@A|1l%zHoDK~>Wjj%Z`nbo;g)=s?ZHOyc@GZv;QQZS7{@WK0|%)q)uz&<6qgDb z%#0Tw+F+!Z5-%=vzIdVZ!G5d*KpBeE1;Ej*bcquhMkvr`A&33Si<^ThRNeycuK?7> zKr(?kvJAHO7r|uO(aH-@-cva+^u70Tu&bq$d9_@c@bkg2mY0ihXt_8^tOiHKZ@yHs z;~778I1gy_)odF(@H%dR|M_&;KbXdH}#i+5$eVf&+tVlkuYL2Dy=F7chj zstShwvav*o5J&;+*Ly0jkc#Et-*GOL!{g(bm!+k{1$B6eDdMr;0;aa%#_583f4&j; zXl!;X(gqU)TH-79PQZ@_R;)6l45g~c5{yXQI2b*-20X;7RMW4?QNPh0g5nh7gTxeV zM9>Zq;(t=el5T<~OI|e^zG9Zx&Bb5MZo@5#YY)74|7UJg6)%&=$5=7`t3+OeEmbLQ z>J&Zfm!|Xo;~{+vNjNmG<0hi5Qc}<)n2;2L&?Y94e1~^e%~R%1%OaOkR(Mu6^D~Gr zE>6yzDseCzY;2h(Frw5^xz0>6S=`oUP43y%L+7}Fw=<&EgNcCnu3=UZi+e1xw@}B@ z)7#7Yya&4MF~Yg^F)y}Xhvnr70B$fjIjFv11*WRY95s@hhb$n~z$fXs81ve`@ltLm zK^JS*W61vuDQ(0Yj;-Vo#&^(@J&^gV-6#Sos-WXz_WyBMnZcBuI+SNY&nxv~JzY2u zK1XZvH-9?sK7(1n6Qu%4-bgEf? zF4}N$QuZo4U&-U2R8KfoGBHeG=(-ueq4_VoMM*Tk*%!5U{zlygIXhQip_&}2I5(WXip z7@_fav68zK!V+wOv<;ax1qKB>ln6k27&N~CS?M;9I`!|bz@(L_r3K*6zR#a`CuEk? zC@-%LqMs3vp!Kj>-LH>V?eKilE|e={V# zdU?fq**hVAMln)pk4^Snc9huKM=~0w{`$AQ4wSZpLJm56-*S=@=01sXUW*HIch5aD zFJ#^FRp@g07lay)hoYh#9`@pmoN{g{NR`!LCKG)IT0z+7`~Mb`+!&|;KOjM91W~M& z>U{i|O4NgqRi_U0SV*MjU{Y1uz_*^!l?|hAxrN}&kCb%)t`+bh2<3w#Q5`H%<)^CBmVpowN z)lvUH0P2@>m`bxoD2ue@BN_`tY4MYb-!(Lt53najGHR$@kVVv%%X_o6m#0kvlKPbQ zBaL#0gw6nrizHHc155?$Tm3nJK6U7D6g21c4O`nke#8g5L@o##T9I#5XJ;ofT-QI( z#?CJMsSrI4m~zCWM!h@;dH|?b?Tmaa!1lKpK-e=U_NA?U%p6tq-u(k=n2#O-B01A^|U~jfd1p#%oayIXb_49^5 zzvJxV)1NaqVBl$*b=Daq<{^X3@5UvUy5Eo#J5Z#7xx4cM-;%-2sgXy04}OPr)!K{z z2@Cs>wJTG3TO0YfPb)g4^DdD5u1Ny@^75X5lg3M9?s3`BcRb&~)9^WD!>sV?V_ir& z@d{$Co$Z_cg+9ONDE@EfI54khn{<8iV0>kMNFwD^FWz+2?7&Qic#CDApE@40z$xLT zP2J3R4)dgYT`_!#ghD~1-@FyDJ8u9T*7|Rl_(40-11h5o2^NSmcp`&OK`jyb#+H`! zQg=6=8WH1w-A)At*Zb&W&bug}=yWaeDE-$9AauRyM+lnk#{CBmY~;vMqZkfnT$gx# zopG^o9>A&7lJdXdfVNOEXmxJ|d5Emc=#+v~K|3O%`9DizsA@12A!)^-xp&OC zwg5HqL7=++&59qpuI(s zZ>{vv5NY_~Jg5A&kAHKk6+uv0E{OZ8d4}H3WnERTH{5?Rxx=YeOw> zlZ4{1`|8Bt+V$SeZO4(uJ1q&f5m?iLwvf&bur$R7x9$zz=`_)QcQ@nUppO=;*t~E9 zK=4B3Zr}3z_l)hphMF&lQn#&;T|;N+)nbeUa!>&buMVWI_x#Is_1?o+R4n<)QL_nN zi+O)&ABcf%$x8pXQ;I>Nw7b|)9&*3Dn5i!F22t(fj{|wILy;MU>Mnnn9{?Z|KqubZ zx^n$I2wSXlQuPeNnYMpnwh%&3!>J3ImOsm$me;lCUj9~yndY*soHNr#WY9{FETv&m zxXdwtZ`pO%_i`fo$3*aLw!(Wu4-dXleW;4_p;bxr|J*>@r0v;tEjNp@J7F@3|H{w>-B53xp-PjwuNQ&KCVs3CsDX5_T_T$@P2q9lhj2 z=wAi~_jR;=Cu$?B|5lBV5rZ=L{o^0Gh-WX9Qkrp{JyH2-s7#!QzN7y0`cc>S@cS*zV^G&^tEgxJV+lOAG-w2rcR=xdB?fbX z`|j}2ZP{nZIk#D&$pma@v()(`KBlhJ^u#NZ-yc$TYDQCvlBIjL)x2`X0;n#TcD0Q5 zp8uH5mPKwqNx&92rWTEWIZh3e`F6}xd`g9J$ea1_BAv|c4shl>q|lhw#0#UDzmtoW zQ7@iVn7?IvY)zdIQ+Q@bz88u^KT-VP2fH9RlhAae=m=xkq(Vj*87HzUCyE|bcbroe zt7S85=|;Pjdm%el;h>dBG#KFlWI(CtdcS}pI~q_2<{=$M@?ua@s;5hy4g6ZD)6}vg z*`rsuxV{c&_b_w+Op>+Ul0x8S2L){Mh|h6kRv?3W*k6{GAmOor33Sab?{2oG0loUX zx5vS8fhYoF(cCWnlA+rD4Te#qyZ>6Z85VWrc|ZfQ{PSTRS=D>AQT#wKReaL@uOHkc z$%EmfF17~Pfy{ptK$oZgT6fFzr1^5v;Hx9ChWc#ry?@u6S@vZzD6!;(W7d-?I;Blz z{!pcXJ3BwLk9z%YS+{^wtbqEe>CGduGL6N49HL!Qm&+b*U9S}3_cB@KFAN%4YToEQ-gO;2EdcORC8^vFLE?$f>yJP6;?}&+~FY5Q6pBib;zDhHE34BeG-K$XU zrBIE|{h%U9K(k&A%j4?TK(4VHyEBjyVov1@36;3kk$8G zGX#t9+?I}|@afCed-TP!%^T0t60L_?^xEID&NKkmxZh|AtHWPmwUuoOtlFZ#OuOeb zvG@P!H~^%p+149y8VZmhh$^R!Xy8|0*LglFWE0_DvVj~C>ec22=!rO%&s}Lp_5byyx*h6@Y|XYKy!>(4mo7oH1jPoiyN zbF7(IM@we4pb>%j_9G}>pbq^FL$HRt?V$SP!a)H73G3kT@i6wltAf2yo@Jq;wAP_V+lili|I^w5`*l#Sj_@rV zNSf7!WJ+wo@t;n3UgreVX#z`bmTW3eTw1BeG2#4nUkrQj8IsGzJ)Tgapz1`Dka+Y^ec%Wp**WbSiL(bLp3?K?JrBA#;U3GENgr>xG%)MA^dn>2?a`l#o5ll|huvTW5-Hc*dCIp}g6X-oZGWE<^l8x_d#VuN zZ$wx;F-yrba@Q&4fzPz$E-KW2R{9lZep_(LG$jD0WuP|(+;bD?n_YO z^dLKIV?n+~mS2|pm8(E++zrIKZXkv(X1}M*5`9@mX;D*J%Fc9bV800y_dBHP?Zt%C z!~9Q>evnEvpi{0!;mx4CVNz?2g=`?WY*&>`E7rstE(*v2KJmNnI>~OFP$R`c&f=u5^YyApCzuV2Bn<7 zraJkdw<9(GTK^*$B&hMaIww006BFY}oV9iO0JysRJrCmskS}?_$)@r#qogD?8JEd~ z8dD(igo91Cq&^#{nzS`FG}Kay`&a-U!cz6!)z#jpbszyK_JF5)u*dB8{q5 z3xOjfR}F%NQ#KS+jh*fzDA;|w(bThjyKe`IepW@m9sa2N+RcLKGmvaSDI3iBKe1Jj zQ!uo*3`=yrW3$@5olg|F0lwID$M@kc?D1V*G^R1$f>Ifw*(~F0=xihzx>W;fLk&h2 zC4rQ$FTl{Af2b>rfzVQ~>3r$;3?IT)_@%cxs{dQfzbxLnt4O&QL$6MQ8Xa=rwiQr4 z**XOU2X{lmDR8BZ@S&{nY3<{swK^s0iQ`M_k9b6#DS(OwnsTPIf}F{xo;^27#EU3S zC?!40d8X9iJpvh#gWee;dS17#kW*h&^TS zbg|$WL2kiKbQoeDoxZd*lzxisZRSX)rNcoTcTV7V%W@XyZ^=5_nTiG8?KYy$kTjuC zJ>nAQ?{`|BIc5}Xf63MIbNJSLO`3k8e6`mipCfXFepVSF)@Zs{55L=~!tqyTMp*Re zV+xZ`pcpL-U#|B59XKkuW8ChyA5SEHEpSTqFrg7cabPp5jmGi5jk`9Y>D^!IEqXUa zj_V({)(oY3(wLu8XdnmhvD%ip#+9|d=qtm&MIwlW9~`0>@$?gypFVWChKm;Xqo^tN zntRGZB|ecB8@lODfZ}wDIzonBx7-lv+GCLnGIJ76Lucu$%UEqb#z(x`)Qxk%ZQH+FKavpAEVN7^vSo!OxQ6j+fqMm31 zh7}-8R{LO9{rHyP5C#14)Qh=3>tay$dlpu6 z@7b=tn4{cZyBuavY58@t$I&f1H{%wEwo7*XI8@q=m|}l)!FOt?eaTLOVFLI4!U%kc;vA3Yj44+CrI`~E$5i{mf>Ifhg|hep<1ZrabUXZ{sv2|K#F$|= zqoDn(-J_cp3q0)^MslWa9e^7bY~g^Y2mzl=`9GjHA-}a(k;VZA;(z?~V4{ z?YyV@21iR$BC8dt6MuiVia^whOQx71c;@Gla}}FKin7__KTC~l*O(UmsMB8+Sf_xg zai$`L=lHl`0q>dUcnhC;=fDSaAt%-wrIxc1DTS)x0jR}amLS|?Mgx5NKb^uJnN|rL zORX{_KZRn#j6Yf+G~H(n?T$Ri>T6z02SI@o9yUnJqx^&l#R(^mQ5I98J-BLD74tR= z`bT>{ToAww$%9=r(72@5Z)*MyNQ+t*PZgeuLRapgo&A{xDld>`8e$-){m-M9hj;((auzznnSF@YlBkH*J?r>%)+Yh>U7X0(c+qB`H(!>+D z{RUzh(YeiC=5|q(%l&>4!85ubK6}x>owz0OAJ}rXFWGWdMKq-(qo}Mn6$PXr|Hi8u z%=+bt*f;u7pxuhb`DeYV;R_D4f0dCt)pg5a21t z8w6ZoCYoG-m}q3@7)JoMYBzZ1e4_>jx38LtoAa^Yr(1UmLZjec`bv*b;2a^}u(fH7 z!}4ZPSxtGR6Q#;=8VFu%g-5JGCiL!V(3tVA9|%>dN35%J+|xgk_!4j7>G<*aXRg;1 zw{{e}k=d9g;n|qP@xDZew`_;Qz0%#s^4EEC3pZ|mh!XPeyU-tY4unL|+PQ|`h#Xcd zy(DsXZYXe?O5k;d9scH5dH#36m_e9;RY!!je$Rkf{Znb|9#@GCbdkO>a80zx4QeF$ zeKQT)9%|Hi&NqAzk1GHgD)B+JSS}r@`Qik@nzeE#C`Non{d$@M4)D4eZyVPurPDs}@P8G2#)piBakwtT zhxlV}$j>xTC@a|-T`txA0>4@sba+XlY1ny{qHM2U*+jTKe-Yfr4{`qE11Zhu3VYVb zQ2GjwYhMfFr_d98+!C*!RtHsRhm$WdyXU5bpZvWE#kr)#8RIIA-wm=QLQU zQYR9(D^j#F*gz{~gfp#BZE+Q2m$^YLfwf_LI-{`s_5|;#B#PU)Ac-5cy&9_lYbOOl zK`f+KaIrYylBm&D8GZ5o>Ia$uUd+Z{C7g%-hfE*W-|0Zdhr5`l_jvP4f{O_}Pozw6 zXg}2w`Z$YYYN9NLa&$(e2V<0O9Ob!kPZ}Z%l>cH?J^1VKMa{M?d0lZFJD*Ex{I0o3>T>=GvndRGx)nFWN@*oU~n=ZC8SmE{@g_W zmk6uwC$}!R1PUeA&DEH~YcU6kUyE7Tf|S&ssjwa{cap zTvSSkGlwKqXe}kIuw}eH1=ZOI$sPm)$oP>CR6@>+P~aLiCgfR|r0Td~!co74-%u!J zR1ZfT%MzS6d@;5~z^Ye4KB0gQcz7FkSooFZol@!C<}`|88w0e~K5}wG)yf&onW?D| z;8q<0M!d>^23}nshN})e><4kHo#*i?YW>b^9E7Gol&H`Fr4mKmRif453C*Q^k=|7)z z8|v#uPIH&vi*WnNLS6r9;Rj}_8r2C#b%Y+dFgdH~Clq2%O%(V#!Z{m{&f@j3g-M){ zjXTDdR@-!ngr4ggwdL0q$6~Y3v}INFSy7+@spH18>-Bo02i*d@m}qc}Nui{8s%zng zU1vWqmpP1U=;r+YBw;Ge_=PG3=gi3m}9HoChaXWQ@%_*&zEs=Bz>?i4yUsgC%fO$KkZFdyWU zGyefKNJo4AI5BGD(glDZpYm^zr-b1+vY9c{5_76sOlnRXr+&~Vu{Sg^`Juk#b-9i{ z@iY>pj8@77?L^8TDs&-)##zCe38h-+@!7Gr!;|>T?Z>+X?j$Sfabd+Fs=CmeRK0uvFYS;;9c2FNBMHuY$(}Y5aXH zGjU(BL9jakt>=ov08+@QHs}K+vurKC!yA?1BCBYZEbM92Rwf^G*nl|>fq?xK6d1iou$r07E-KYmwJ3$`P8CTcq&04F z8R?L2d{H|a!Sa416)g6w&=fd$0sATumga}Mg%}g`y_U1vv0Q32XLZ8vV>~hf%V>&V zz*NM!{Ri7itJt;DD%tQ2^Nu<~-#OI?q!W{u;-@)$*Vpa@rYas@8IwF|Iy}|l ztgfy%?D`bICz`vaP8=y}TD zQrk(XW?}%gtJgn%H|v9gX35y;aU_(3+ZK*0Xmp)mY4V9OF}k1QGNmH?MrjN7^S_a8 z$icaqMO>~64Ze_1(vEke0DgC*Fnr<;O@xLjFr?{KnP7qX)W&L-fNmNLA_h;#xixR` zuqoBA&V8expgnNIii?{feTn%n1#{xmK1&Q~ghxYt1=b7FK?@HkkirxxEPJbO>P><) zb;Ksat-3C}JYMq#15VursC~P@+~j<%B4vMGsG*r1Uf!ya>4o!*b`?d4MkykKo{kgsJ#fAX_Q8_Il2U2rp4Dn$lstGRR6C{u7FLa-V~ z7215w?A;yvd2Bt$@!w6R_k-zs2p`y6_0|g0d&9qw7_5#VXXMmQC{#guSo(5<2z2Dw zoxm_$xg?ZYNvo<@uxRKVA0LM0rZ@!~bC%yk$kn zLSFR|86@nE*#h}p!Tzr3KlxourO!rCF+y-8lY9FIq2d>Z#SCyVe4K)m+;6T^{ z3awZW80UiYOK%JT7_s|}T`m@w=FUFh;?1NCojZ+on@kIW20?3iAOettViv;JfF1kp z2vczXi~9gI#+>;JNkF;$8Xque<9~Do=0kUbG9xCK4{dE$5t1m`u-#~m!{euGlcPVe8iE9!vl1Q{ppHaq9+u_QqVE)wXdVC#h(A#NwoZ*TbYzZZpGy`&@ya;uBh%p~TF z151KP;JL`;&3P)<{s+Z~esUzdcIe&&rbMA8InqNj$tXXWltb_SV!b^YAh5EfbLey7 zDJ!^nto00zZ3IB(rf{bXVEd@#5{h=!T|L)^n<8UDc>%= z8`}l6vHAMdW-!l`vnOqPz(M1w-TJ4gbS|UoZ7|*8-W9^MNSUM=aA;OR6X%}M^(d_r zL<-Wv0jbw!LBLP8#@5Hs;a17Oc7sG+>T+XQchc)?WT?E+b&c=Aca2)>A+M(adbFr9 zu#yz*mADxak?EfsZ{1`XhzFzYfGN(#TFwK>H;#E_YA+6>IvPj%r#)j+6AC_;^CMIH zzTQkHX~Ib|CMx;360^RvXs33e+A!{gGiBt4nfmam49D&(P0yreQoS6zEsH5NEQo`y zu3?+<3L4^ah36IzpvAxGT^EC{> z!?1rMi2gtU=_w@EFduVq9zzTf>t zSt-Bk(xTELP8BQw!|ZU`bDVcao!Wo#74Xx_)o5^-IsGt9a#R0z9Z0bKO+`9o?W^EZ z3}#r;5KgfTO$OfrQU1)V|M85%^f80|jtm?ZsixKcxd@VABeCMWOQRK-AR~W6vap=* z3##ik7#Cn$a2x9kf`u%g@JJ*Yf=s~G7>)iot@kHrORaSmkn3Z%-=pX8VjNjA4hGA z>;a&GA{{O?%Cw4{G#}yR*~HlnimEfB!5Ajuecl$*asdnPZMN*Y%kdX0;B0;oMI7c{ zVP0Tbi$I_S0Gi_<2dD$D#nUIiPLCfn4kdcUJZo3Lx>*4H?N2H<`k^sf)RWo7b4>7t z(*d~1>9wSMGzA5!C?0IT=&ex`huT|Ym~kTJ{2t9nPgR{zv7+MBf!tgw0^tAi`R%H+ ztsWBT7^a*B@G~I6YJBOI<=Y{Y3BV~sdnSD=aOYNRZw!h65k$`q3Yg-v$ca z)H$;_Eqx;LIrx^zfR!)szD6?z+91*CPMiDO^mB~1wuzB$&v(Ji?v>n6DI)l@ev;2)+LQC zjCM3qItX;i`M{q*g1**$y`MriZPNp!+b&`MGqNJ@Rn2-ohSdN4xFCTziEm`n8$)Cg zYAZjxHx`eK2>bd6Qm67&Lw;K%z9N*}9Rc;>YqS4BbY!R5l`UR{7e0LG1P4 zO?rdoB51{u2MVGN4Y&ZJF27vS0HYP;&y=Su65?{AKY^X#!KGeoeVsCQ`k7+ueXWg6 z{G?BQ;jwJK%w;}&HB6&Ji&vyZ2HV8nNUhXx^N>N=o?W;ysu5V^vYNIEobF6AJWG@* z{PUBuX&HhPo}{yC(w8+w>1X&L7iUxr`gaw^ax8|75=Gw%x>Mz>!?OrSn3j@|2S#CX ztjM9|EcmO?&qtAQ)#`#AlWBa=xu&g zwf4{B7IDlgS$JsFsAB5vc-z0~-Z@sZQ zc1c%?RZY~AHTIeog9Ah2Va4rjYMW z(YK%Gz@XpV7il$fPJ z#5~$&zFA@^F4lYd`;N9jF|V-xFVfyRD$A~0AARTs=?>}c2I($oX$k3WgOrk%l9o8RuVz4&=V?HP@Wiyh7V)q)-MeDYiC(!JgRT z>HMv9R~Iz{Z9^%Zqpl3Cafz|RJj)bD7)-(TfQPY-LvMOMFYPFUmn&=ReYepM>!e|wldsVZ zVRp^HKOc)XFYKCmf%Lp?Kjq*4iR}o}m9PHqClcj!1XT@u0TV240HY#>|G+sReoBZ^ zcVKvY^+hD3U3s4Z*d4_xniT5!-#-3xMjY6APF>lREu>AdI1t}qI2FYL{n{H)s(D63 zz!d{A>h#mB39?&Txk4X!zjCw3PHa7B!*8(pmYVZjfj0@-3x4}bbu?8bMOE|1Od-Ez zJYL2KVr?1($xb%6md91qtI3de6~~fQ>Y7J=CGd=*A>LL~K1fbaV_xdvGEsZP5l(}a zi8ar6`@VB)vq2QibieV6gCTAF0e9;2&+#I)fwEP(xmxpttI@%)zXfAiQqZ9N#HzVj z+VrUxH$NMh?9unf%Lyr6#ivK-NL&(pbiYz0NRa!}ZoczqGa+r>zAmN+eR8K`aXvy| z(CWhFSL4R^yVepX-ht*R+XpWAC8K9yKErWSaVS-Y#lB#y2D-A53s@!{aE_%~9l#)c z{RL#U#;G_E^Skw{}E9iYVQ1>WYleS7wtsI%*tkHODRu$obl(Q6kCjM=X)vDwk|(blBBjB~!D1|5o-s%N zt&e!YT51E9GcbF2HS=U%j%F{{KGW6zmM`Yki|JsoHs|ymZxMEfNbI#tY=KUmd4SAo zIbI0rBNj*+^COxd_}Lg_K@-iOOVXtZ%}mQBeWST&UE>1@+7h3VhPdeSzS_!E#c&N~ zlwx@(UxqSHrv!!*tSd4UqZ$_MF_L}dMuE56Y@@h2+nP~s&v^amisw}~HP5Y#BHnuLWd=3e9(7xoop`ak33}uuHyHdl^>By53qZy_?I{}()?%j&|k#%3pLulp<_MVi_4irv9WGflC zMX;A~!R_$#Hf!d|7+OjiDu~5q!9TC4>ew^NZpGIq1?)}VCms_m3_s&hd5%?IwHK_o zo*fz1-`mf6nn&xRNDh|@S}DSAQM1Fpy$TgTWd3+HBWgs}@Qtb)SQTK}141Net`? z|GPO0vayAtHXx81U5MYfpVL7TsV7V-c?#&DcCV*l`4X@x@lTM;QnSF~tL<$22e%uG zQ}mZ^dt?Un0mQxMG`dwa)FWaRT)JAvS`Ve4xLPo{QBBd|ZNIa{TJe$1x`|;|Jgv2( zRDdT{H~QQ#y#~jRsU$2hJ20^t7sn75JF%CnFRu5}ic@qszfYybi$y2D+?}HY8o6?) zgghvFS!121oesg$W96GKZ8kFI?Ce*#Wz04irf%P;pe$m@gtOZeQs2n>W=SmmX8_rt z=`{{Y3AV;Ig|93}sdjBvRYudeQf{7XMAt~RSRVK-q$tYF*QP0Z)2} zP4DVbO$HIHGj?bOg+!VUl(YfL0&DXN>`r@`{#P}-0b3k*+n1T?gm9qY{Ch}laqXH4 zD9@cZc+qp6mzoIydXJM6_#jg431T0G;0`-Ty`k@dBwRUy(^o1t?9Gc*i=_68Tk;&z zL%xzU1tI(C{|W|=ZD~!*E&_?NKOe6!8JU|MO_itM%)_0X=3DpRkt5~!z!=`V+WQ>U z&*}+3`GP}@;y*kRRLohQ^ka=CPUhAK`z8BO=kk5u!y`k3?8LJ21zhA=Jn!DSm5*b# zUhi=U%{2DrS*LVu*sLNo2046-Ohs#>8B~M?WnYU2{X~>g#B_ZTXZ>Pvv2mu3aa?rL z1k6(LY(8E607w?|p)f!ElJktJxiQ)lC$`uXCY5LvjfaBElOW2@OI>aPC>9dJt9sMRFG`6y_;#T zu9NY_i(^q@O-MLMvIB!qB}`Rg2}xkEwI;f_9=f+KGj@urEtOA;qHoe}q>*Tza)bXj**VWvnm9S^I=cua|C zmrAPslzS9uxu0UF|Eon$H%`#G9OBS>T<_sqq|Wo}>9rcipt_>?x3&+a7-Z8G>tx}V z`!jqU4?u{0dejQY=wey^S0K=%^?jZU8yOdc4gAMXj8tfto^D(~ zj4qt%S2lO*=#;@={O=5q^FL2M3^@7UGXMa~%`f++=6=TS2cX*?eXWi&fPICa^@;LL zy#v!`u1X54cvn84s3ZhjzdqGTyWzXSJ}EWz=Q5f1tNv}Ln24;5_zv$B@=m6(11Iq2 zGxd45nOvm=3W2#lYjq)qTu>B+{&c>7qDI^Y8p%@OelGfmK08n-gwqIW&8_U~nwqy< zyi7%TRFhfOdKXG9#Qr`t+-~x8D^gOSn6GFRA3legr#FmzFlFI3pLZ}gn{i<_;FetU zQ*i(K#hPZx?h1tuaYo6(P045}Xx$@1nKosFGoMorM5q(GelaQ=I=#^Z^`&7kii&L{p%lU46z$97I2 zWD$=s32;v=f}`J_JWl1gi0)*{Ny;ER_3EwuqXEA@N-41Q_XInH}HfFu;QHSGl1O;|uqpDp;lq-s?B95b2Il&q@l zYF=FLxT2JO$rY>swiCvHZej$TIMH$fJTZ&^|4$6;6taD%3oeI2n}H3>_84PEvZB2_ z+CH~9LHT0|zCEqdcC3?TH_;n#rIrUd#l(m0%pEnbTU6YtK02^x6N!fO&_8j~R5Q18y;qh7Tf#TW^`; z#NAipx$>BODo1zReuq){!74_}S-x!ph2CN3 zHyn<%l<+f!-l$Rwb^8#^+iwCVcXg(k6$Wr5(~Trq!1P1EV+PE^oq&@ptgb@>JSHia zN7>Q=;C9NH1#%~&e$&aCMRq5YmVh0?axl+moA^ppMTWfOIe3cO60z`7>j|pSf0rIw z4Gm5ayOI6j-qt04|Hn`uw#2iXDhI=;<>=B{wm{p813oGL#+!|10JL7;f4G>p77V)T z`E=a<2_xzZ8D5?IAe?XixSKkmyuPZFq=Ha~Km?Mn8H*qZ31w+Me_BzyRWZ6_69&hv zK&S;rZ4}|8XpeYb*O$4cU- z^|#WM&3E=_o%RUEZ=rrr-hUL-D6~Z*9<`s#VbGeCvn^eTA)R@y;Ts!*0(Amx%N;nO zMK=U~$LSw<{;az&74~BovXYatAXW(#oKY06{%t{qqldUaZpQF<~H?sslRD%ucTY|65AsZLpmc?=lzkxCSJ*` zaf|iHh28 zBs93N?7Xofx_+AHY0Eshg;FY8rD@I?t`ZHviBxG`zLL7A?W2OpXFGxEcAdQQlb{bA z8IQMx6XqcEboj+xli#<5X*kG_-+u7E${Y!Tv#Lp!JltzJ_Nfo$1ZgMhDM|}5M{-{& zQKm}_8nEetV;Czl)c4)Q5`PqEaDICqy8dw~uDy{)0;#FLu(9L^zWjBnu)0@5e(1D< z?oa-(DjSW7f!)${4>hF>y>~>i+$P#2b+BrVEy)z}kdQcg8|mQhGh}L?I`X;#Qqoh$ z7)7~MhIQiEJ9RLlaFcHWi$;Lm9CWYxRp=_{r{@+@$RIs))G>fei^A*Wa9M@ew%TcihHAvZk4Ea(9&= zu$Ty#JHOfLvbI0#9Bvw)E%u`S^%A%9Jbx+R2-Zze^Y|OKA!q0kwg#{x`Agq9OWl5}Rn_a(y0_+P)L<1)gk`A;U9V$mY_zo4 zUb&BfKGI7H#YQ}+xL~V#AP!vSiIiiDMXtbR8VWRaO5eRx>bn7R-~fs)(vO0Z``s%+ z%@+TTk8wB~+uK8oMg}kHP4RQIX8aNkC0oXR`7cpF>%U;BppvKX&e?nxT>F{D>Lntg zUgAcnl=B-ON##xJRg6SINDMQV2?f+T1a#Kx)AC+;#H5FR`E5zMnB~rQ`Q{y>1=dG_&+>T07jhTuurvZ*hzlbnT?H zaN1TyGhI3Q(N97ou0-uumu*Vl_unL~gomyCfy8DAhGPrOsjKE&(L`Ad8swlr3yXAP z^vE?=y$`&fce~A!t9;R|*}|kt%nl&s(@J+xbd-*qklzJoylk;=o>)EU6RwV70`^Wv z3<}h%;kUpXtjyKXWwcy?ZJF<9+mD}y!J6b1U(i^AVrDZCw>$#Mo)>3YkzjFy1rnx4 zP_X*bNhuh8iDv{2bc$TAx#;>p)9QMJ?=uy(z2pVlORv zf`YA7U8u8hxwAc1)(j~F!Me|U(vL6|b$tde#SCytX`NrZy`}XRyp`+6m9N$ceBaTa z>8-2p=ME^J4v6y|(>H5%(IeBAew2~{;>TUe>3|o_6GUL%FH3rh0jM!cxq#OKElpT* zdW;j+@l1dccZc5t-LKZj=d+9VnTBwlQB;Fr0%M=CP6C8 zSSUIPA96nWZ)!PZ7~>09Yc#6aV}CQh5E|efd?O#BIGHBnj29jG^R~~Gi>RJaZbXGH zo@0{HAeF(`C8{}IdQ2O^;~q#+1}HJizuu*y+^Yy=>D{$@_!dc@P{Df zX)gaL0MV$?E<9@^lscGbLcU8Q!i%Sd+J0P3i-{#gkSI}e5u`QE5Xm{ek8ZGgb5)g~ zkzgh5?VmPR%%UoyRmK??pc1~H`zz&LnpDQ2oyeJP6cU_SB~BzKQ^}O_j*7r1R2?VM zB5<6iO+?S_dA6Jk*iKt02c%w=QZ-`gu;S;|KA$)55XGO^&Ln@%Ys*nwtyz%!aCvfd#q8i23NU*xY@k@s3go(xt)sISGfcYZqQ&lr7w##T@D&4ALuE;Ts zCtd$Spa1pVmjeH%ky$CdLRlRA8E=oD3^{8vTa7a_*ozN+b3(}-bJ^irxt092dv{YD z$HXs*#@T+REsU<%s?aB8U^V#bZsYf9)IK{dbWnXHvNDf_aYSrO4*Mk3l43G8D@uf> zz?fDE(=o29`lc$%O*2f^6aB4kaN1Z_bG5~6b~72-GZlOH$Y$y-eWSaQ{nDYGa<3NF zLK1H}t-E}C4*^S7wsAPhv{Sl&&oKarV0CsB?9ogQ)3+Tq;2P366L%L-%;W=TZ!*QJBy2M=_d z0)!gjP*jd?44sHQ4GznUz8cnU!^$FnYA)d3rjg zS?3-Kq{<#5d#g(_?%kjT+M-cn#yr@(8T_Da?lFSW1vQ~6$?AurV#nAt6)_%o6t5II z87`nWGvoIx1>5>5C-$&AYp*Q(#NWqAS%Mjx!p6Zlwrk8vB-5F^GXHiXcl@v%8laPK zB|p(nxPh0L571QgzQN~7Fopz!WO0-722ej*fK3>;WzRb&Ee?Pu{UvR6A=vSl(ofuU zJ;W$`&d6pLlG8e~!@y9S+&NpC;t-c;;jIrXm22aXV?9q|N9e(as@y%=Rb`_{Y_Avd z;kmX}_EVoUKQtl-LfsB_ePZ&N7kWgj0){D2O8XJydq!KrbZ?eJL1!;zG-48eZ4&dx z{?hPK1~N5jPH({FBW8-0aU}slKX<@dP`AQ>$F%OFNINRKVI3mCnXzE*fgWBlV5!m? zqBeoA9)c+|>8g155l@7T@~%oSF?@jjAQOkss62ph!D=}%Dn zWvq~L4#@~*eMw9+9NBi{sKf?tE)fp3*U3c2YF}!b0^fxoUHIDV+_(Mf1z@Dya$w{e zRE#)KaTTT<=iSF(EvJT-^Zqv~KR#BDk_T+>wHfr2*PGuX?0WyC;6EeTMSOCP7gWApq-CqBLDlw&kqnX|XclJO#S(+h zG@m$;{-;+MjNvs`jeIY$QllUD)mU+?NXHm7bc9)mI$MI! z7_UToT{7K&hpID)^ofV#Dv#WE$LJP+o9Km4?N?r6hBb88@Bbc=_HQPy|93y9Rn5f* zn-G7*xNuG$!U_$Zk5XZDp4jKfgTJLeQyNxlCS}K8Hy~+|Z(1&OnqeE)7e10CLX}_W z%c6y$F-a3zd|PSn8+KM*@*k6MTGQB#?(m6H1Zaf+(u$4?X(VcIhP@dk(povRzT$$I zUrTX?Yql28>I@>v$|90118(hlq zuoyy^BaF#wHI=)2L7UBDSWQKHm6roe0D1oF^(HSZy)^M_TqO^F{Jk*P7f>1rA3#xEaYSO%)1L>2%aP zl~nH`RO6LesCLRzxrD$uWGUlsud!Oi_eh0mBKoB3brF$3ldGXKj2Sr}6Q_gL!kk=Rp#2yOs$n3%%4+uQ6x zSE?d<1!6Qlt+pIF(wy#N6$C0025C|)EA&faHBz^GK3CaoQw|S(vR7{j^*$;u(OZFX zWvymp_!FHrVTZ>bVci0bje*6x@kId^ey?lgXieAwU-cDg`a9P@?e{003&&k=UAMna zYJEwc?T;vcYJ}?_E}X<=$$7@7{W`hsL$D(#ST*fHoI<}6@mx^rn}Vk0Hhh&^iud?? zFrZ;C^{#6^$*62e_$(M7Ve64SdastY3B)ZU66upLwny385AQ)nwKNDU(inQ%F}DV0 z9Ur?eicJ9cST-va^Ob{om9@C5$@B$*7Iu-pCH3mlHFgu5g`+I@eBnyIOQ0-Knj)$V zXE@067Is8LC;|G*R}7An9kfehgNoX4sfz+YRE((+$ob87|cq z&vzeMfpRPl@ESF%&Dt*5PVgdNY)1g>?LGGS=n!WC&PE3~8{f(_mA>iR>^@Vm^h)5y zU;2_@hxv_d$@Sg3+p@5c`s`c5)!RO*I&@>=g!=J*D~#`1O2Ws!|7mCYx&F4ZaJd}M zn=x(FKB5~7-s9^iF4;79Sr4U#s8+SW$5ooDCjAg#l=$5SuyrKnm&Dwiix{H49Sg9xJB8*u!Q8InwrFcc|3%`>SN z$dhEh1>DmJ2nZPKvJF8R0Mn-vNakSdlp2s_9FVvFj)2|Z5!jRjYrbr|B z0bV|=eNd>Cv>N|E-ud3+-MF$wn(TBM;?HG2>kp&xx)h4k8VxJezlxNgn`&rGu$Z8X zGyRbZ9;d2?^$s+2gQ^&ahV3cEWu9w;h#6Evig_KlB0oc^N)YabSV<`PmD35v` z^)|Jw<+_9Ec!XcfHGF?303}73Uvh&$7{_;$-Q;Tx3W(@B>cM*}F-A z!N9upwdrgl2h=04(xe4AT-Z2@0zEh+pOF3{pXD*MH0`$AW4HcM!8Y-KHy354BTQmy ze_tJ|Ink%+SjRe6jFgNDmH#TXKbn($G6kCpLos3`STJkex6{DMvp+UqYPSEmQT+V&1s3)v?w$lX6!W!?X-T5m7IL}fnX=f`_pkBZ*perO5jKKo*z3_ zEQ8gzIWW~dzARfL2R~tJxlvtV7{G7&zd*pb6j%z72v119=!Pp6cn*Te(Loqa1W5Io z`HgxBA%k#mE|uvpQMCzy!2;a8i%%AD(Sq>C1>A30`|VoTn)!A|e3Cu$2z-HozCIcqn0zwofn6?$dzR8Np`nhqwv8z;6_{F8=y;>l=$ z!xjxA-)J)KG;$Dn=ZWL)zPrG37CV4-3261`H*g{LBfV~pbtiSndPlI1&F@2|E}}nv z(FeMUcop#=My7U> z@VBoy<&6>e9UHnN$&e-~uqbaBK8&%=?@akwDr{4bCVqKQ5SumT_21V~=u`Is?y1wM z+R@G^$y9u{5C<$Wp|_u0+7iykcvfk3a0Y6*$H9OSrFg<={uVaplSvN+LKVHbVp zrk!gFOp6@U6$>|d=2c4mT`v#)7u9&XU4{t)=VRqUSm9I5-L<=Mtb=uEFBJKGcfri$ zFe)ESL<%{KVyg=J_~eZTTH7ovJ_2TIvIcC&B4F}MmXfTl8&GUq|NJ=V|L zdDuo5@d1eA(uco*d#hX>P8v7i@@*S?YWDO?;)aU)m4K;i4aC67snn&zhJR=x3eF0? z1bTrX(S|veW1=yLycrGUz-LqJvz!f;x5rHw7`)V;&{rxNKgxM)QnBFu!Cwl^V&X$( z{NuoayO9W%88Z0 z_4o_W`qff&^44y`+j#~i;Nl<{Z$}Jqie`omOVd^CEGa0q_#FPiK`iFRVFD&c5E!?W z3)k)RJMs3K(y#Io!r*_T*lzBhl6N_MH%z`^)9WqMN?%rP0KeMuemUCeXgEa9R z1QBwyKgm^J&_(bvI`1ffMid;M=1Ix5uZR%-k}C=fCf9XoG}SLGbyb$T$>e0Ic(UBR z;XlwOiwTe~RX7VjB-^_u#97xsQdlvR?LNWj@lfTzKkK*whi{~!RiqDN7eV#|CmA5^t zmtE6H*p&VK&xZweKcwnlYHWOGgei?z#=^pVoOx_9wZdjGGeVhxd$!F6(JR*r&hU1W z9TxrU&f2IZZN=jK+IE~k`0@u|uES?51E5!d4E_0qFLA$v&#Ya@-Ozr3$6P5)7z!`Ma1`kCxL4>a5dp3|fw^tt`<+E{X1`+2VkYHQtmDvbLLXVwkDRj+W ziRF9c3t!2h82Hb4IE@^s^zEeE_B z+wu@+z^%c|Y6GU9QbM@?kX@9|D9{}~*KfC3+HH`$@_OseH~ltVeFKNWHGp4KSc%AM zWZzDyzCCnE?<(Elbmm#zk$)rE`4fxW;(EmN2LL%AGwT%-By&C^YQAcrb`YxIQt|kO z+R%o2B;n>OWYDl`I~z|Px)ZW)T<66iOg8hZX}#BRZQOCQI)y^kyZK}@PH>aFon zdq`laEeP4&X<1AzmQ^;k6_8h<6rG7JRmdqP_T0nY!{~psg!k6k5;RYuKBV79 zGKFw#MzYXMo4ta8QKqd)LMo=5`PIQ3EUtssZ2`1iQid9XFCoLPP}0NcfYHa!CHt5b zkM^-I;BS#L{Ji_Y5OITzhoh()rgx^Hz=Qfd?t-6Uq;DLmg#6iKnc+1gmTu!|lNcBZ zhL#R97aUS6;rPj%y+O8y#}9i^TStL)S~Ca-lHzx$5=X|m5*<0{4Ve%`vtRv8cLO4o zZV<4F4mi;9PS57^q7~W`4grsz!mc3b_pgE$@DaJKg%NiFf@EJNC7}ArI@3x8y!WqD z)S&@?&5z2E_!#GLMz4xwmK{G;CAsgP`|ECU*H*X&n9N$uH)KImlxqAOHEU_)-JV8b z5VC1jdyLm}f^X9nJqcrlQ>F#p=7G_j>EBY>Lu?}?(V_z%u;RLKbyCXp*Kp;%j-jOu zlnXv5RR&L6A1DxH@_S)G<*D+U+!zrFF#?oH)YFLH)=TB^VArv`={2b)!bo4qkxwN1 z9udyY)TNp71Fe!g-99U|PD;AYqs^sO3;c@bEpb-vGybNlyoh5jdAx{*QMNz6WCfmX zw-;}Rs@ldIA{!NEEzcMl*A){wr4JjsV-UbEBBp%q-OI zK5+&bBelf_p`C{EmS*$dLN7J(AR8=?^OQDE?(csRiohWWGjln!pI#X(16dfi@t%Wo|pZvO-l{y z8T+`XAkNfrZy!S)cA$ADpqWza&7NN0o314HI-Dnr3ilz)06}ih?%#l>&AUTXlnV^3 zPeA(J>AbByoE^XR0ce_EKw^q3$BnG{^;n){UuFwC6hF<80Oq{tKthKzS!qn{0_1FwG-=;;fSP`KqS^bElZQ(9 z66{pLhoZ8$waK4tDSE4v95@XZ^$6pUG9G7;k7wV)YU7(Dn*g#`*9)W3t8*p2>pKlO zx!=DTb%n%VMMR~z7ZOLK2PqUxmz&3Yd>KG7S!y~|%P{Z)&d%5O82yV^Ye?|P;X-5A zO5i<~YnueAgr6^Cp(TRPp*OWj;E=jscN%MkQ6iU;9#W%;wkRW&RFYtUl-3LI!Zs34 z+58plm^5xThJeJdI)?2mF&IEE!w)qj9VeM;ZT;h~zbQk!%kEuH&*NS^dBT2WY{kwj zmN=6k$K^1_rL7JHj0fgpLen2F6nksRzQlies%qr3*2s%{HTOt`j4|o`YfTX%BUbsh zky*L?mskzd8BZ^Ev*_sd3_`Lfs8S|x)s*`s21w)IEB~tCzF7y9J|16Akl@@j{5#FPs zDlw`7l-&EjcjJ4P);5TBNI?qw%~09_uSZI9K@))F_bOCR-lV_|cs_!BJCyn>RdL8X zEb6&n2A0nG&ss6c8_G;`>&VCP_M6|0$4BgQ;HKOK_@-S~yK)oOAS^)Qs=#fnOKSi0 z{coN=if%_{e-I*td0^@_CVwDwT8Na(VcJaaaGvw01IT5uKt6W};8-Tic&Ltw3O2ms zk1hP9TZRWTYf<+XfP5$IKc}v!l>|>oHHC(H4KExuY4MK3)?Y!0N0jKjWNws73O1R& zu;(EEOJ@p~_d)#gW97pFl(`0{mV;#WmT)6%L*+jn9*x1)wW>W<;%=f6k00KD)AF{; zz6Av;hEWaiZ+((8f{?*^Od4iBd@JLAJ&UiIr;W5DX~|6yh!=Ndu7#1CfIK8FQGVIVwRd{`@kGaL$P?Q|3OIu6oH>`{LM)REm_MOE*dMgwJWVb z6oX&26MO4_gW(FTAU085`SG@FEe=?=I_Y}Aix7HBDd5UFXwmi%4w=KT>JcSE^%2Tp z>sDNmyDZWtaq6-}*#YT100hPklid0UgAlJ9tZgP=0HafJZj}xBbPKvCqV?cQ zpd`9Ypp=LcI})aGiw}^E8;Y;y z62yg{cqz@RCw1_XwJGSZ-(4T$fK+VXAE=*U1m!fn!guWAR7klAoJP;Xrd`>dw~Iq@ zXy*0kNr5vBcT{=EnvYT8#^b*ns#(CHVi)r#)s5bYlIM&=deJbhl8Lw6Zq0VlRxei3 zrKR2{B#C<~AwW>VL4^kk&q0vNHZ#+#jqCl6T3zSU1fx|rHgD8H~ z3k~Q->2!~M%B^6J&J-(Epk;6D4i`Yy>2qHxBx`Z$ln|JXl&huIxGJ-b@a8&V)OHjI z+n-<{V^RY@w_(9DQbUxFuT>{BXDSU*fY35bMv?NnV~Jjs(Ucn83=g=ydSZ_8)nH3q zy4rovO6Iz*wEtElxu$%dHr>i`2*-cd{qYF#2D?9VwEd6nG9EvZ*&%hng>Ue3nOERh zs`h|^R?!d{MyqXl#q+}CJlPNw1?3a+zZI(1W=8T}q3e@HV(37A(~d3MRi{li_XgZVK_n}MS^qHdZ8;3 z;S-L|XT>5&onrWgHEL0Zo;I`D3WEGaDd(JHO@T2MhdH$!3?ACeriA4|p9*rc)F%?- zRF&{{gRp-9iL9v6L&jeNfjQh^(_3U5D$M)a%Z-&Ub9pv%WKzIYgbG1}|2BG>oGnym zbq9Q+SAcge15*UW(vdYOWCFnir5F4h$p1sa5kG-&nP||B+YRmjFU9^$LTIxUOrH;J zEVHjnDE1x`z|0*R$>;Y^em`z0oDpV*$zL8j6X1cwmv#jqL)P%0ThhQtg3m^=h}t93 z-0|)k_*-^7?RDtQCv-z2rfYwd$e%!+uXzub;o8-)ma`=i{ zv4&DPt+*olI& ztE5n{7FVT76(_d)@@qlglh&#kX$oh7{XR(D>wX`+3*;^T6#`ROSEyhpC{SC5=JYGbTvh`!gGHEH`^_{88|f9>;GW z>iG#f=>ZooD{1kRBn7@Zx>Hn`dCcPDH$X+13;M7~wjZ=u0a*O^jnmKR9Z+F7Viyf) zxWT@lS!jnULH2sh>ti)pF8T$NKm@)Pp*}0x)m56gWwkV0_mP(fZg<(SnZzaV53(!Iy203GW1wVRYNV_F9r~-aj{LIC5A~tc8 zHelyHhPMlS|AFmUoMQEukqiI$Vbk_UE2Ba;AWt*POEB-AcxPWS?Fi=6^$(8e#PIfB>n%; zz1)~fwZK1`f8~^I`8tukc(8nt<<{5kPM zXIQUeE=A>I1)!^084gRnXTfas-t(s>==6Tj|J_8PaOF?%@XH@rtGsLFzB0 zlkH>GwTA_<7o?`D*JGasfW`BR!QLo z{3EbJ@o^Cg%O>+DHP-xYw`^yt(VoVII|!6_&pvw|r0vkx2|6;-^yOE(5kdalEw!d@GAOLCtafT67$_Z@)^ zP-b_&px*^UB< zTn81!D_N8qHDKFD90)5q`u=mK@1m4(At^|HrSeXxQodP#bqg5Z5~LK8ifP3g;+`oy zLHKh$pNX0EL>#!Ie=_)}g7BN2d~Ky#k~6gL4)bm7zF8<14+O%MG2Xbyrz?Q^4iwhqSrNDCiU@O=#}npBRLB9Da_wJBh&&1qS=4>$Pc zUKTm8eMD(SP?uLH3i6R&n09;kPAmhLMb_fHPh@a0G0N?Edsl-G*PZlK42jC$@+-u` zK)QSah7bHPdxJnGD2jdLpUafBct!yC=Va+@A|Athvm@_hs4HhJYQH+z%-~yrn?ngT zNSS~%T|a^K_`g!xGx=FFJ_wpW5_L~8ljVd7bbg==-00&1U$U#aN=vYHZRI++mx=#& z)U7p70R$47h_qp(00R)~VM>IpJs3oCo7A#Vpt>Qw2)ScB4ZyYcyqSwXHgOzeQfQC` zHG=pkv*Q?_-By?!-sjUT@CW8lLpsADn1>*sGESahZB z2b^->3sHjx;Ux%OEigbSnZ9HJ#cG!;{rj=PGE|&OtHsp7wKOO~L4er=l?DZ&X zk}3*45p)+%?z$Z5VXAH1OkG)fFcw>*wMJP>7^8bhKsGlGnb-Sbf+OVJ9^!lZBIiVI zzW{6hp4+KUOV{-7B8ZPmoZ5dhqM((io9St;qY>d_Cy?Wc7}K(}h^yG) z9Q5YyBGePtZZKX7pAdOJ1_o9}AV3G#GAyy4+J)vChzZ|k@okBiYXOuM(qMyXdEfMc zMSO8XdiHAog4doqSHrFwS{&XT;;qkq*nU$=m@Q))X5VzV@|4`S7`kH0uR%lfpM5SJ zb^@L~Gq_!_G<{Ju>2vW%c{o!lz8$B_eTR-(>29kpXLaq4IuvV15&U5XL+W!z@^t1T z{#&P8Qw2(}k+M9A1i;#&JBnHwcd(+*7{7AHwDNx8*a)V8<23(z1C+N? zQh^Q%3zrezz$Cr2B+<1zT5J-BkDXR4*v&(98`B*J(MUXfHYO&mYNUoiPuR&vCK_}y zkY!NRk@CD3P*2`$#0vdF!Kv&7hA!2o8o3s8F`EOUpsC3glE@{=_L;XQ)k<<9%{Ute z0_x@Ox)>9vVh*FSl>AXOAW-NB@Pi`E9ShLv3qsRb14L(!G5c6w=^@d(+Uf#{H53*W zdkHoVh}>i4R7GD(!s^)u(t^=MO?W<5F5*7;Js$(Z?CI09q>Z(f$5PXG`*Gf{mDcH}9yy9VT6_(8rN92#_RrC4CX>h@Qp(rp+p)eY=+ExAdpn9D z{Qe#t{%!6$Z@r;gom6gD-z&eml%eRX4Jd|pD574O`fU}t*6U2s40??Xxp+H7vdTY#=p`|DK?;3BIsR}sK8I+1tBj}( zyB&KeYVta06*B<^tDe?EIM~JT!SPr*OvEM zY1>u7b0AxK(X#!e2~{!mGDziXWUz~Bb3}2w1X4H-31Z4=0E3%apX|K+4~|ZOF=X-C z_>z3dw*eo6_ZzLf5a**ea>8^*?a)FLutNs>ba4@0PWw`*Zy(O=OH=R&AsCtZ;l{jH zh@BUp^v4@pG&0%B&Ya$I+wXE#w^YbM70J(+R5B-v9K)Sftl$h1yiPs*AynfX@bw_S zUZnCV|C0nF?~(#jF+c2+meqK3L_6JcV1d%<7{cf&=LuC=9vCw!C78$;ZUeR@JNtk6 z;?3|vCjTyv)N?=TH3{3weu>UZ;!c{9tuWvh_&H}|1s8nYcfns8 z6Mu9monvPzYMWH3&Huq;0(?>%O*q|SUtfGU^k@DiwkbZ?C3MADX{;q_y!UNT@!5b* zB5h~6EsceYpSm7dzJuweDW_qV&4|j;RN`!5d-)`K_V3?WOUtNbD;dqG-Vh|3->Z3| zcU}5e#{ndU&;BXamu0zOh92*RJyR)jO}RSih-?tidnJRUN{=%f&; zcOG)n-_EV|6J??n3WOzrn2aPd>R5${wqNVf5tPA#)`J8v&m>Z*H(-}fN*|3bco_8x z%%?Lxvpf@Xil=u3E|Fk#e3qjX;LJD{-toYDJ!}q7++e;@Pk`>zT>*m|}(c z_Dg|%ljk{I#_w!y2TZk|-HKC+1`FB1N%s+?ZEF&#HvtETcCUVtMj%0#GX*I*o!+>| zAwt`qLFOTg22I`a+lVq^z4F3X9DwDjvY%ywvy$Cjwrm zE*>hegdP2qEY6djz+o=qM)vr3`IKU|KhofynYbgxH$HZYRgi}^D@2HyK%9ZK_kqwr z;d<8wfBxaeH%AuSu|DgCxE>9;uW4b3%jui?hzczJ9#r5?Q_reEjHXkLY-CTK5Reo294Ei(DQI@vlChaMTjgpJ| zCq`%`cCNEdSQqC`kjO3OeQK)5ECfzwUe0 z@&ba#qR>?6B&;tS?0E`2yKF~3{Aip*E+R19hbXfDJp#?u*iT#dP<2IQhEO1 z$&%bk)_LhQ2;AvO)C%Mu?9C3Gzbx5I6aQbNy>(QUUDqv6BT~{OARrw|Bi)^XbV%Pw zE8UHBw^GvG(jX-vozf|t(s}mn^E~f+zHj`-`F`&?e>uiI9D{vb*WPQ-HP>2mei?%C zeT$$wI%@$0>ed%H=oOIf&eHo2ntv-!Dezhl(8ddlUTPnkOJ3MAPFusBZstCav`mWRm3j#8NJD_h)QsT^2 z90EwB8N-_Tzd-$dQXXcH^8!88&QgaucDaV1{Bji?8Y1N$$Y% zFBa!^(EQ!i!w%H0uJx*yJ|ik&dZB%*)~@HdFFfM)llbb;ic{8fx5SSvH~pDmdCHf{ z_1lvZb=Gm@c~}y!%u2mXNI)fo!uis8*Jq-P6~t%{iw=FzD~S3dOU@S zSM|)Oq;Ec&va4kLMTj+R*AY9Qg8=DgJmEHE{Z_kCoL6$b0? z&gZUP3;^GSpgQZhRK1tp_gCgfm5?Nb7P4u;IRAKW@&M+XjF%%FaOsp`pz_b9I$wi! zCX3lCq&9$b->(=8s4x0c&(qOhgb``hD;eGbMe}r5num?F+x{&8>%%|oV5UZ9(-aOx z&sm^B!{D){zFcf%{-^@;x~8yKaKht=NI;w%9prp{I~I^+hn{?Echy!7dPm!^LzA8? zC%PVsQ!x^F+`vz~DkCfFqo%6YcY$+G@6uzpe3pDR(}kj3FKswcObvtYbpe06?LuBo za=tb5J*%&9uyDkRGJ28z)AJ^$$#*W<4^K3CoCR^esp!;T06C|gXD+H?p>G66QLeso zyfQ?a;T~}0fw8_g(0jT5O=@`@Q!pr-=lcE5r;ABQc?~Rw^^teEqF;WnAX1IK3L7lP zyPE|bPj|Z|skc)PEy24*lRmUG;h=Y<;$HI{J#vU-r0GSg|G7xpH~8f}^-W4)nly%) z)VcUb8A)J-5T2=Yp?}-j(6;1nw^sTvHOcL~w1+9=cXP1Qao8mlgzjf)qqYlEKrTM^ zcI8HOL&06;!)*AuRycj*D6Q)Nvm<){%=HI`b|x?~MKh`}m68s>0~s4b0=gmw>UaRVmTwlep`w(cCMhXcD z4Sz=tO}l@bt}r30LTm3Y{#|<)G!LRWNq4Lvl+4*S_X*w3WZO5Wa0GKRBf5ZL4&&%8 zwRkU43k%y9AVVfRK6)~30&QpSi-a7ENmZjJfwESOyt)$)v=&B69H8y1sVqM9}mhR6kAd!zuxe82&M zXZzi)lp`8lb7S>E;&-Q8Qq*3;EoC0wEqwiIBfM}@p4e$ZM$pUgX>sQ%Wb)pqKs}#k z$#@{?7RA8h62;@>0NHM~;bC>uEYL3WD2fODxUbdUm0^`sps@onsn(ZLBFv83;&f8; zDgF5x_lK;NkciB~II%3I14$VvNvX^jCgypet3ZvFvrU@S?CYE5SWA{*zELh%3y0)O zLrP8aa9;{(N@kb{;qLCX+s7GRm#p$>{0XqdsI#>}i~eM0(a{|FoNy7J+s7lV=L|>0 z?pD~tb*f-F$Tql~$Fl*DC>B94~x?r8&G-ZKjXa)1QPRRR(!FP65Q9%Z5sk< z>DGESXoi}#v4r(*3d?a_Y$O^HGaUN@9sl%afvNTL>rg)enZjnbHo%l4hEhI7 ztH*u%i%(kqDd4&L_=vin;et{g2Iz~63AcHbv^6gj8feRKtfTYKg!&T~NlpbhUKR>^ z*>b>d8H4HcKl3x>M?R1>i_VRod_d@OlaBn5-PYsxuE|{M!hw8rH zS`9&p!sPr$g@XM;@QXq73-Rz!q~97L$1s=^j|#d7yXU&#F<)Z6$X<Q1@QDE-a~NlKE&DXABvE zqXeZ-G8dy#*(XbwbnMvbndV^;=I z4(>maz)cVsA3Um*oJG_ypm0jGqMmv(hx9_=7&G}iDyf^rY@-H;<{&hX?}gXZ@n(g3 zG~{#B9wI>)DLbaXiJjPL0L94BIi&kyU?pgvoPuqCR>w9#4C$$vhw-Zg@sI%dI3JV6 zh(xAJlo}eN4={I)9D?n8^D3fUMA{bu6;H~py{AQNE8WcnDpg6`%0h*yzt}_1|0IBW{RoahXoTbp0+W~sL&`lqKidig;l)vxGzv$!eb`LEvRkFxmN~hnD8kQ{?8wN^yhfAau>yQo zGRf?_a~JbG>BsFYN;P9Ak0HH2m+$57=UBcjJw)+g}vQMiXp?(y=IT?b0cjal%|Kyb7c38I9$ z5s2?c8TJ2w`#EKy}7XBx3t^@4*lDn@K>zxH>uoab3p>Tb`b(?{eJV?+8)wLN<> zl%sj4Qey8fU-qA#_syLBWJy{y30WMS*h&d_lb;x(bm||Fdv1Os`(1(LF>~(L_CWkJ za=~uG>mh@!1P0eoE_K^TwSZ=}+?+Y#pB&a&8i@D&2DrH;j0K5-yY#KK3 zeVZ^{+9)^QlliW!r3a@JuLp7XUlWKKilj7OPgF#cPyOsAUGja$M33kDx#~B9WRMWy z;!NBUdecCld~>>zMq8njv+qfumTYVyOODcT2KMgKbJeyb$wt;d2JIS&6w-`)N$dVIYz&or6*iP|7L_d-(?JJ((fF`(wKa`h8E;kzC+PpTUL0NQs7%#?zp3Wv8kQxtq`F!>z@;~yrsh< zR;zlElU-Vd9Ice+iJBD|C{D~%0<;L&3n}Nt-=3V-n()?3VTF_02RU?_OFnxJC!IO? znQz+VTgpoP)7X~#2KsgJ&th+Z^62SfT9&4`0DprOdC?apDro%h%=`F#^B&u2+IxzP zU_$Z+Njx;COK^_+x)I5heQla@>AvY{(I8IlRu~+;h@7Kk9zq>W?y<=NP-cpWws=p` zLg+gh8%TdfQrj1}y zT^T1&E9n}SJ`1Mi@;z^$9I?Sl+xq0TS~;fs*KkVE|sO7G-8r$2$MAr%M*uw&sSU*{4fZE+bjx zT;ANUuKbc$-K8;>67Jy;A}8X$gh~r|db+mcy2oRbxTe~wOO}L=jC540t@O62r8jJ< zEs?_$!qg}0;Y4BUJX4i)rdPhXPM;OqnnQeG!*F7#+Sx}agE{h_wgSQQqbToN_$J~rvnfU{!Y{p&+>?r{ zVX|I1wTu&AZJ$R7zN!;27Px*wpK#nKUb*{gT=NRG7S(wDN)%P)EA^3-k-T)&?JkK$ zmCSgeg2j_?^?a%)tDW7jb{bxmhNsknRYGo!F<@S#g>ZNs}3hgI@t8 zXm=+`Z9jI}-n09fc_OE<2C}951{n-vlLp3Iw^W#C0>jY?R%7H~3=ryqF^OWn5kaE1 zi69HpztK3{f7tQjg3Bz>@5N?BjQ0%Cp)|UG`vXCH2(tk~(u<(YAVOjD64w$q#}Zn? zp)R2((clFt_m{F$tDg1+P834c-!)f4w_26FtMXmkP67(Y4hv#zpe8g*$Ns3ok?~I1 zBc&N(`bZ_e%5nak%pGm^Bj@E1kH5i}LX;YwH7w`D`_-@)nH3ckG%VPLo!~CsKlpPO zHv=4nh67nmJ(EPyZ!f&Sa8S`W=96~%C63>EgB7bevnw=}l3GeGN2Mwm|26PhP*$C) zbNcwoT?3Ia!kVDS$LotDleXFXHQH0bfR88~5%}Jc5#VM}q3XY1754#ud~{G`Nj~#u ze8k>c&)?Js=073`7u*Wsn$cAnFIT7>915Mklv_r>tJ1ZU`M@WIy^sC*OM8<5uA-H7 zKfQ0Mj0Jyg9c<~w!@5R#00CS3CwUhwZxGmz4YyUSKTA5z*d#0QO?1|We}u*Xm!vo($lX>fcp4u=ty*#Ya=J_+ zWoqJrSaN|xSshbHKRi2H>%w%Mc)-CU4~Zd$c4N*d>oH`0TeC)9N1FmArsn+W2@%Z* zhxRQIC)jfm5p*|UIIuJ$tK!_qp6tHA+EjX2rnn8R^Ab&_`dVSLY7T!%Cb>c5Vj;~y zZK57|-bReq2iwy?oH+dccvZM7!_TZ1AS_~ou=s}5jTP=ouITZ!`7IH7q9E9xtdheG zWg{0BNpn+mPti>jk}Gy<__JC)sVq+WX3njyM9XDI$CK#2XY>mll47Urd5m!!b$0Hh z6UAzg-AmdV&EZ^w2* zIk8oq4a5gsw=5Gu7BzHclRGJYH|>ZGXNI9LsQiWSXJ?yODukalva`y-dA=4fM?Le5 z&tL7{Jk%@9@|a^$^F1VZF#ltF8oN)YRFA|Gq7LSQl<{c>d0{ zeOKvY!#e@CbqpnaC*EY319}ZAT<9kpfoNQ;_YvqN<3uy>k=S}4+5zSRE zc&$jhI=StO7qTi>Yn1CN9^r^C%&mLj{XE<4O9}QbNrSo$TvgKkS?pN=7{?Oj!-XtN zQbLUk_X(oI_g-!=#&LrCVl-s)!)jNEva&J^@SS?ckGEc-K{Jv~F;cg83)AmJFsktd z=oX2r-LOfMNSrivl&}4ib!19li>Xp8%{keXX~w-Tx^9cxI}|jPCa+WIPfFG+way+J z8+&L;Dl_{!(6wAVG0mR=|0U|R>6bfKmgaaauGsj1`@x%?-whQOoo274w3;y7+r5wv zUof(A<;m`I?T?>EF%&*U9k36jWjMyWVaxfo84g=hhhm%fQkyVM6?H<*JwsbsJn zV{@yba;0VW*cW#!g&L8zr^*BMYtp&j1q7;fzQ%Uf+E;j|s)#q68sCG119@DaHlLGx zbhZ-wJ}@Vi5qZ`}Q1#|VK~W^TYoYZSt(y`(5UqLwA9btBrJ$otLeeH=JO8%3@ZMyR zjheJA!ZbKn(3A)j2l7$H2G4vQfin>&xy{`sJEf*tRt0H$M=k3@(=r|F{@R5H)h_etlx6 z8F^nOx2x`-EG$P&7tImP{p^HBDsQ_)@ipc55L}rMPcuM>L~^93cV$!!>@QYqKn(3P z?u<|RN&ke6_bO(aEj7}G;YvYgu5e9GMWt?)eQCOQG>a!YIVZW*vMfksSNT?{bv1Rm zGsHv55r4LA=v`%>G=GGew`K=I>Or}Spq!i>)MF}0irf=d8>l)cQSWh%g4FKU0ohgB z9!s%;#e{tTctXcMzg3(bE(mGLNLZtu;cbJJN)LiIjU>`E5dpZGx&y6jE=}Q>xvwjS z>ikeR(!8QOh2yABXueaD(rr&CQObRfrs9WJFi;~_`mA(UGhppt{c2V^p4D(|-&M?$ zW`v&~&B*8(PNL#G;Bv6!WQom{fpy%Y`%UkAT6zlmr?npe60xb!>+;uO*Z=pLD6#*8 zHC4rQ3gd7?*JSzknnw8LGq=NJ*9w0ZW;iuOlrc+@gs{>8y++(Y>>L4bDhfY)y~evD zn)@nyiDj?Pa(l;EIWTb`*9;>rJ03gvdNyAy2Sz2c!%;|Vhe>w8^}k~H1P4B)!eO-s zAvHqt^%&5sfSs3LKA*D-6 zh4$Z*=;=gqVVla&xVSGWgS#0M(?6N8Hy{^>+qWK8ea;OIig@+m71nT<%vDsdupn)B z`?gtN%;o32&7e!J%6BaAmZzvuL#6-g zfh59#q!1G6n(KnEgilbl$Bh9Z;-GH#lg2 zP=e#mLP3e)F2pFh^VCcDLRT@_gS0(X^E zw7s#eR9NtOSoHF|B$q^CzBlcubyo55u4*Bb;G%fDL&$U6W-D#kwFpEfgNfp#&2dgs z!DazGnvoy5*}^IxLwuhS_;T%9zB;>}iaS}3$B8=T70aPfNJY40pd^EO@qq?a)l{W7 zR6a`sAA$DiJ~HhmJJCs`5|Kw>wi+Ecroz>em>U`zPGz%s%>RD*;f;H!xAKD#9TOeS zQpBp8dQ#18#gOCy5H6p+$gNW4!hZfaW(n^-!B9k%H2=)5TW%(?`JXUn`agv^yehvb z0ro1?&Gl`|LMfyRI=@P)pZSv_YLaZM+ zxH8)zOd(VUM3A)%S9r3FaYCnkIG&1W44yn^;B%hYj>V0Z%AYmo?1*Wrs`i=%)x$iO zQNWdGZ5ZbjIlnwB4g^lrw;uVC$d2j$;uDF+;Q+sh%At7qQK`f94=119ss^kr!5*uk zk=Jz89#YOQ&(!jPbQwJxaY{T0n$y=o{6K&&=H(?o_wYCav4^HH6D(RyhF6-wUd*Oh(r%&-`x29yL%nHl)u2NxVqqO=dH zX5;^TO&iWAB$m*uP#ZuKlH;H8h@el-h+T7Wb>^^u8G zwZGhsQrjg5+wk?y=l%BvOZiW1u}$Ag_t2E-_3>va8igeNAF_iqQB`!xAG{FJJKu?v1nQFU8eDX>27S`F~J0vY|Be)yY9bE z=7&k}W2NJjTqx7ucO^KQGDOTzrEsA~6EU=5cMTYB27)$&k>KW57XtQ-JfvOlw7^P>aGxC|EkV=&2uRN??iwW#)*;A7RoqoXxrOW?A z{KSW{Y7acrN^J+CDL=>2pEAfeHuuoHhHjTAFMK4%V^XIEYwLm0_Q<{@4G@vcGwEM$ ziBZrT{6HC~LAD4na=ReRQSIgY$tRt8=e*%mZ@xoJH>>=ZbdKAXPss)`825U5#|Uol zAZ#b!UgVasyczodREQzdWqN0%6C$X8ISEv>&&gi+uBtbY{!%n5J%ubY&i zi0=(q57(qd?1{=(#&Dq|=6(PI#y+lSc0kdACc=j~vO{ZIz$>=9FFbz=uyGba6!1ui9v5ZsYxls6O&!lSuAoc#hcOnn0JXn4zw7hv>%?2D)hX@PU8isV&3bR zzf#4NP*#S4hrV)tl#H=({_GzE*SqZZ;h2Ro=8m!0+bJHdLjqX|Rdcq}W&?dubt?C| zXD@y8M5_M2l*}$#>dvCioIme<*4t<@?B7E_I^`1@EB+}(T7x`qFRKArs0wrv&XCW% zS}srl9;d8!K(@RSh_xC}-87n=g8E-4)U^W4$Mc`={baXH8!oqNY#RDcZ!Ux8Hc|mM zj`*Y`+H8@(^Nf%6C>-LeLFR(~76w*S%I62VaOO{2#jv>w*CX9*E@J2)8}3I*vW1xE z>z~xd2l_IO6)+fSJ~EMmS9ld?rn&R;btGDaTHhN9`AjUDfUvgea9vuf>>rZTiE-q^ zKCg)_l=hN6SMc&K(#}^&Na)l&4~pB9VglmNt(jhCS(UO8vnMtzNUHr_R*^MYmgxwS z^|X(BLEpnpS|>D2fS-F^%*(Z};x$_-xMq-JNiB*H3D6SIu9)wMFytcD=JE zAVU@Km;J^UmoY<9yCRil-b|8_t!eRUM)?_gOFmbMb=O6G6islFjj+dvxHHaOb zLb01(Rp!TlN_1^0n$v6>4JdMnXZ1}CiTUQ(1FZ$vQ>2l2r3ep1c$HL_7NEoJ&us{G zJP6do$jUm2_+K|IUy6MdJ5=0=rcLkRlo5?OKe^K9KQ9j0!lO@eUHYjysyz|hlkrCJ zUQPezWw)RGOAF4&HJGk(Uzi;@y*wGtVrXVbhJ{t_QU#~3CX^z$`vDqkU{XyKY)(M4 z4*X=E>FQEYEcHWlBH$BeVnnL2E~b`TUfg|^+v*Dmqw}9hB+GqYy~V#)Y?7fbSB71; zFOrTMu$q9ZT=Tcmr&+IWhy?9%N`fhq0m_3|1!;-$nGH93_cTod?^OT&{Fnd1^IIp) z`28L(XYrjjE5rR=$M-?ZH2|5Z#RZ6P|CDV1ldglrugK0IyS&;wb|3R>DNDZH7D z_2EmJU@{9D+&3{!h8_SH6x;lb>lOh%^4Sp_1yA2J4>rx02yr4DuElij2~Y*??0iOp z#AdkNrSQzgXRoF-c9N{FcN*88n%p z18LgdmlfcZM7g+cLHs4T(=^vnAweMa<8xlYwCgOke0%lIBjE#gjinCKck&8&-mftu z<5W$5j*8|U^r)Qsw@^Hm=4|=&cg!iEWXk+2_7pbFZC&w5ma&0t7mLlVM&1M^r>0fwDHBD+(i5#czF1hDIfM+W{# z=-9AuIFOL7pgolAIDLL&h|wz*W$8SM8A|BlOmk4E6G)k0k>< zMxa|=kvM}Y^J_}R2{*Q2b63?I+3n7){I1=1X{W+ATV|R!8Y;?s;X)e=odXaRlqyC+ zy@-$}fZ|Z6rBC|hGIi#T?F~M&Wi=>3r4_!_JDd40ykNmVXO{+_gi6cI-hJ0jb82sO zUr_*|%X&tY#q8MZ%;nuHW3Kz07}L{ZCOV9<-%UI++)}N_0a*h!ige= zvm{YnBAtygqjE`h>J_czc$u66AR#FGgCRS|!hbwTHF>8agTR zQ&x=?w-rA7Fqbt8jyFZ@Y{8UHbDiq~tAfQZt!}|wK|q)G8@>ip$^IxfH0_|V30~@{ zy*Q!9E877C?RfSPt`V2xyZARB5dK`aHk~s_rBomF|HY@&pgMLyf5LJ5Hwql+>Rskt z69vVV`e@TO`)2?SJ}vlFF+58%@L_@-v{Je}9ejXptzG+Qf|q6l?i;T3b_i|y zGwjn(ABGA8(9qWz@)MGUjp0hUkX&M0v-K+LO+@Vfr;7p~5APdrm&>#@xR5ldU67?2 zc}H^0RQ>`7AvFkX$uqspgu1y60{=r_4>R#2k;R--e28PB6N)H$KJS~~b_)q=pVuo{{kA8QwWbW#d7EL~- z87V^4$nDSYI6-L!-kVSrQfjl7kFxMX$`m*d`OL|lk%4s$hj%x+;6VzV@HRSU)YsWj zPX5iwP#fmhp{YPQ88zu&dE_3F^@puD%T0%^a^mnQ!gAu=CQ46Io{wl8b>bwk9%Ygy$$%P`2vfSaY;BC2o^}KwgQ<#DNqo!@xOIvvB8B4 z3j^W<^^uw|=REV(5BA{zkoj_CN-U`AR2m^RpH1o5_0#7Ir*T`-RT24c|n)?-MnxqM5{4Iai@NCQo z@@$7!elK;t5$!9KXK}Fcvg#idbh_8Wulx zWY=5npqH2bhq_EAGn~tTZh$ej=UT^rq%OH%h7^SW^e~AKK1l>C`RoaJ3PeBp3hdUN zS00ykF6TeZDPELo&*37M`Qyd;LUD$d-CsCEWOb%73nA~`5zB+SiLhU`3uB$yk3vOC z!2XKs-D}V#+v`6hklL=2k1w1g$eg90-a=h9Rdzd$H&8aVaC8!2Yk1C=mTCXn?d4vp zD8yi-=Dk#r!>mD@f0O}Z_WU6^Lx4aNSMi9=@9WQY(+wXAZo7W-eQST%f1PEMhpNjH z7()Md7Ku7eWtGMOzPOxr)N@8MnvvK^@}~-%01B3-`c(2#wLe0D1{dL%Z{#e1q?UO0 zK4{yVtTNu%13z;(dpCSU{%Y1d_RRj7@A=r>Qb)(|R}oLvt75%Pr0Atb(*ug{Z z?1Xr>(yVh4JVi=;)BI3l#wYK2#HYz`j?97E{FA=z@NuBjfY$}RdZK;TYkh4DRT`#% z26is7&b-UVoA-54U~}XpJET&a@CeJaXpm{R5r=;sKlaCGLR$o zcgOuUgJrIqssyaQ5RL>*4|K2Yd|Ht(o<0JFd4C|>5$rg49xl8N<%eG7H0(#4XKpw> zyx%4Cm8SshCfQP6eX-OpT~AEvLC;ARU>!#_qBqE5DoEK({tO2GDdv~15J73;!FhYj z9!7NRV49I5ydexd;&#`skKnA8gnWZk?l&7ukA^;2X6Z8B3YoaT-dY{$TYNJnvXNAN z{^%< z<0~VYk)8L4)0z7q(8F+W86pt+GC)tC?8l-~pYet7QRGU`M@5iH)vRxHxk25yp9n$% zsjnwH_m8B4lCV7n%5!vQel-+yzUK$k18727SOIm2V642e*-NdmBKcx!{yKZC&;{GV4? zzQcdi4nIgoMDep;*u)X4t|C?r^N&r_^)8fhP(9gf zF(ylZ);PCl)6m(dZ;8J(=0b5 zNHFKux+K#FnJK1nZL=uu2jegp0)|Sg?xn5ODrsS|%U`CaLeD0gI$6!@f^mj+hsy1E zqaTl%V^lwu_y?8*+I7;#lWU}#Z zKLk5_r&$<-2oCS?)jc(|O~rOlexaM^t^yp_ZHKsB9a=gr2hGnx#j(cWlU8gC^hmaN zYLLXRgMdKyfwU3|pR7izj3u9`BY=@VBoc0$0H|I#uu}I#wdZ;_@ran)M!vj?(}I$q zERvHj_yfo#VW=sw1Ya5eSwX05su)-K86MWt9&}KV02TufU?bnwL9d)e05>^7V+${u zQf1ChuV6^w}eRmiy$C9 z*YU#pYCs&JH|^2$zdIhQ^2>Q%OSVhGeD!GsE(wM}4XxtcB!jT10y!cgA~d>Q{v7r~ zYP%Y=UH(WF>4Sv64mYJud!Pg;IEhQ302&CTn##TJc3l|zDGxI|&xz*Uc1uBI#vHjE z&K~j10=YZ+Q<^nOGI>L08ow?TEZ@A&Pse*~E;rkp@dBWw$ZiaAp-U3RHQu1@J;|W3 zPHQHD8`&&^Vp}73&HR@bt~1X@qmWQxlJlfi;Bb-HdcF7K!ob!~>b}B99JKiLs--)f z5rMKq@l$S=K^72T2g>e?c1PKK)u*L|Ip>BEv^imo9VMSpJX+p$hh{Q^qQy)(Ku)Q% z!mx80cv1%DDWs|A5nF)!C#ECimlY@=RRCojVt5|QkC;A%s`(w!6k<^N2{>`Kl>`O= z^f~=6^rsSVnyb&fu}7-BV~~jgE6=x29;v@PduaXDhXb{ppPGrO~KRYji^}gC1v9#>&?mhxs8iBt1K4&tJ zdL5LvTxu17=6o*!3Fjukm>2C~L#vP~9)R??auL*dZ~9;1V8HIJ(W~aQCur!SSkKm= zY%)oe#HdXmXhTc9b?+|gHw}Lko#Q@a-ghf&D%0qH$9-P7Dzg#n$o0N~ERjY{rlj{x zE~N(ppVC{Cdkr3L+1oDvmNJdUMfOfm{#P#u%KJZQfw70c?Ki!s6KnFjwp%P+1>opC z6>O2{W%9$ZTQZv$ef-D1P7v8TwyC;<;PyAw3@X%)EHtA;fvIsrdLn^cqM zZO1f#S%`99^-<1^RFU{1P&5wA2-7k#!hr=&yQRS-kDIU7v6IhwBr8MOZ7#30J}(!j zqGpP8^3OC@BHY_=dT8?6=F`FR?9`mwwYOu8h&(YwC*JrrWZi@iQj6Yty(|TpE<|~H z8YYnC>xMXy?mH#@1hRSCK(IGDs$)DD2|Lasp2{&y+eUa{iOK9&E;K9KY?0K8p7kK} z8RDed`q6STQm^9u3Vt%DC#T25nc@udWz$`x6^%aoK72%4K8{DBqbUP<&TtWq&YzOd zFL37cR$49={cmqQc&@j;crurV3X7S3lDP)@u}`PU^iYAeBM+2)sHyZBR*?ryu10X` z4c&uO&`K%~I3O#@#*raktC)_#B?*EKiP$6*f06_Ui>18@``R)7mDxvO*GVKT($(%f z!gFvpb%4Msi2mY5Y;`k2$dpYv(nyt_;DoI*rrocfzcOSE@A}_w%~!*Ia`iNr5qo0c zGJ~KkP+6ia#Tv(C5N(4;s-Vr`lks4nK{xXiG5&BCJ#5p)CyAB`pLJ!A>-qFf$PDo# zzcRA;#66G^47{DA`6P+xllJu1Ok8QJ(VMX#iPrp;ib#i|$aM)XMeyIadvY5V$Tk23 zLERF}MzUmZe?o9g^#CX{HjZR+ve6vcrpqknpSh7iA zFb;&$>2)0^-=tSyLhXeHQI0PX+p9NPb2aM%sgy32TW~BWWGMVGyYfwP8?MYE1#HbS?kZ zWEBPiQwI-Dj}}}-Lk>k^I1mE7_;ZCP#cMQpilra{i0KY zRf%t(!}2qZ&ocz#1KE$eQ7gM%puL`{G;AtcB24>R3`)(Vq2vjgt>`^)bNX(eg+HOJ zriK8zAhH(B)3}{qG^QgC!T<)VW&K|aR+T5g?L{YfHx2Aurt2=4Q+&buc(v~?l&ZD= zYI~P4OT3vZWvN2&RH%}wrow4~iIF}yA?#~=D56fLiz>?C>AP7<@>yq#$L?oVu!Gtg zH^j}wR@|lKb=orsuZTG0~5pbK=?o5BuMr25_;Yd zINSqJgV9gGBSBnrS`ky+05sv+$p6Rfo88cFwCM{LmUm~c6bCn*VqMItS@+5uIr8!VwG#@E)o}IO$UBdQT(x7 zp#=lU@}GI}pj~&re+~NoX$!*zIpTlXmr}LF1I>>y)IQ9ZE1=YX>I25n02cY!#ZPnc zdf>E2k>)Hgu)^{ykDAaI_$B;il~Y^|0+haDO=@fVj+8Jq?+~<^iho4n)Ak)9^om^TB~r#${X}=Li+d^)cQC zvqWX%-o1dLA{%@=>HQ=7RpC3cqE8v2fSsVPY?OD{wjmb!JM&Ub*)=pEdIQYN9s7zL za&Z5YCjRd#T1uY;Lr@cy0J5&ZHlXfwbY$Q=dP6inX*-z83A*ToEtyv2IRv4&nh;J} zkm~q0sqGdqmD5skdIZS(zi>lDm*I%l;0i-RDJm{+P5p@Te59=57qKd9Yfn%b&a)WD zy~^)n@)f``KKY~`ryhO_sCFUQHXkF^?w3>#`Hd^`F(16!EqMTinE0u&rjvd~l*7iy zU7%#~1Ffc`qoX9y@V_H%$P1D8^yzXcZ`BknsG?$p=je`cFwYzH=?i*(g3dQYMBZZt)u^v19YIk0u%>00puC|gmoWj@>(ImWDe zwZW*i&OD0CFQ*RvnkfxUrF!c~%> zPJo98+ZuWe6>0!=Bgi?Cjaj=C|DzJ!hS;N4Aex#9x}^UXXKx)=b-MSB!lJvoJES|M zyA`BCx=RHC>5y(v=`KkLY3Y&{q@=sMB=p>CoY{MyeO>1~@ALjK*UYtM#T<8KU)O8%=1+JH+=z;16gh$KhR|9#oqhhEh(6LUg5!6F&VkGKTY|2$W?|oz(5ol@ z3`c{P=W{;Y@>p8@y7i?a=E}BsR-lE2L0W-Ozrzs}nqF{}V359)qdubyM(&&88C5))Y z!4Bte9=Qwc$`RZDJ$sBL@GX{oWA_r+jG}(bwXRB%Vf76-;fq8u2xj1vpu+kTK50LS zww%rlqcq4x`7#>8E9%BbI!fMs*>kYSS4KNN&Ryvy1XgXPr3QP^1DB8AKB|4*fGrqo zJ-d1hKbqC@b1OX;km^58>(6Aa?_UUgIiaGOg9;aqe*LdA<$IL6g`IwpACYOew#c*P z`n9%;6h6jiuh)7I%1?E;p#iBQrUNX2vbGMTaG%oJs#j|efI0Gb18gRx$Fjs%Z}+Rh zfR8=NcTi=rj%5hrxe}?;Z(s^~s4zW@Ss-O=~0k@VG~ zbmzP|dyX*A7`oih7V|Ymh)@}ZP3N3t#!zJu&w}v<*x&~rP4*+)fq}MjRN)?Z<(fOO z0J>rPKE(|KIvh+RGx%R095;5`RvY7bd!7GaVR6Bee#SLLN$0UbWru`U<*x?&I2~TR z7iWH15G}bV31d3fqwf*-_?DSke-;>h(vI)R@AEEyjg$EG!Q^KfuqETAQbnYdwKUTP$eJ2>MWQ1%nUw)+^V-M`t3I1M$j&C5nD3_Jpe-Tb3L` z%OH?2(}a}*zW+lOOZ-C?L*I+|ec~^+0cKNY-WQ@qRawT?g-#AuGPH-p9&q!Of23B- zQD8vD@-A`Zj|#!!|44YBrne!8x$Dl=cjO0A*)6H&*Z<&*eIr@O zNA8Oik`dk3kf+A~+jiiF_-~bhXUsD=6sDC3bnbt=0B}P<OJ3U)A9!sreUbX>wPZt6Y#PDIVYbO}%2|3Pet>jn{cl9FtFDvakHl$MrwYK7 z)0>fbN0&)xj)x6!xknlncTPx#)V^lC;uEW#-TdH@ZoX5nkhiSEiw82#dX_bEPwB9M>GPp%zjiVLB zWU&h0X;All9BvYmhlg$=hc424sxHjXXOOr}>i^nR;1$p>CfJ^B8Q>ZWp@L83>EbY(xZbb|pFxp@A6-IwF=2fmeD zD>H&_Xil&dqJzO&&|B%j(+F9Q6~*-QK$YrZlBeFsW`6Vtj^4(#Xcxpkln)9|?#E@Y zLeV%H11itU9kekKqpRG^{l&l<=nDfUOaboV^d9)MmN)0foI@r6fo-at7ex%Vu?K6D z)J0JI&eNAAfoFBiCw*bEbE@Ar3I=hQHi%~l#>kdwl0Wo6<9{_Ho5HJJjeeS8hMJrt zWX{39iQ-PDYZS(lll+!YlrwP$8PC6cnI z@lFM9Dgi!xYFa>ELAZAFvxKRbveF%|KvzdWN;|5=tKZ1!HHlA{aUI4#6VwKf2{>>3 ztcFu`OMa$d{~I|JHUCp|TY>7t9A2b>FG*zAH`NGb+u|}Zs$DBTEB&MQX2n1IGmQ%g zBCOs`J8Kp3x9v;+Je~XLYbd8J`H=aZmY|Cf1IY6k+m!lsKaw6Kv9Pn9^{?6ebW40u zr7eV=3Z4J=7gaI11h-U$0cBbaUp8@IHJBn25_T?p?zcfPYPd>ss8o`b@%meJi9-}v zg!4hrh4OH`0l+$G6+sdgH{-qoG+`ZJp`>jv>;g+A7#+=rNI-)CEb_`h4X{MThK+66 z|Fv)WJ8!P=20QWnd!(C>jESpE**-i;t0+5FQ`oMACmF&opHG!&mC6aH_q&OJ&D-;^ zq&DI<`w4aswKc{axYK|Ns?i8lO^xLxnMTzKec*Gj+4~zmj{E~XPjp^(QM|Kiu3Ssb zutt!rN)@WzwtiXjH^d;vbmw9Y0gtX+r6}oj9eOS``0$U~`-A19`U1Ac5ix=ZTrUpC zN(!emnnh@H=`WK%bM+lt!almM_GD?j{JQVDW*3~Ra_wv@uh83r)0cB^YjyU%_@#C# z6^3!U<|oIetpmgrFJA3T;#hrY{iyY|N1x!RTi^T4`Kp|_;eGEYv|c=>mv*=Fhuc$86>^A!7ORoWuUDh-)v%!?CV>p%9?vVceQ-8ixFd`Hz-C1fA zZM&@p^AEg)>5n>(I{bk-7l9zs(g+O6X#~!5cv=OR*ONWERf10Q@IY3ZOJdzPI@cA7 zX14pSP`l_+@BRyrMqH<~6kwfkgJ!s?24$XueeD5HH;w`0Q{3TbF(PBT@J^r05w@N#xK|qQ-VP+YoCdev#i&cwnOoJCu+(r7S8GrX)xJ za;_NAb7HT-(`hA-A<0_mUDSh>N95EE0~y z*dNhr(QlfR;-3OaI%#%3ltr?`Z54M4B^db3ZWV0L(~diy5OSKRM#Tdk?Q6bCD?k;y zSaIH;qCaepD2YI7S0nZ}b9~SW83IBBxnJ?NQfCNCgg$MYJ6Z|tIaj6LT6Zd!rer@$ zerP`Rx1Fi6Qt8kwKV9w!gtO}-8I@e8`-!|lMg8llp&fSgpjTht9SnwJQ25BV1(f3D zu_r>QBapn*Jm(oxtbj*auG?*LDcAKRr)5&l&wy@Bu!n~V@^cL<(HUKP|A8%v_C@)! zD{N%8dfiFABZdbn%O~%O zQh(i;=I_tw^5t1LKgXl+EiT@tc0*y1?o2YNNmt{|lt;SKPo{T4<_J=+P~qT_%CYlF z5V`{^u+V_a8_8#Jr#Ra$2-u%b?wj9`l8lB-@IJ3ov&z1GTf~#OmkiXIeV;|RpB0k1 zlN0=$D@($g_&Zl(2aZf(xix+`;j(ArU0d`tZcVBheKgGi&0-b>_D)9umB zT~lD+7!gtmbOHDy7F6}&9bopREXe%c#M>^@^ZPp#KESd&t7$?N^h!2_pN~9#-{~;c zikvRsSl~i*KkCw^Kt}AdWjF4=OzAAGj}U&5V{Z!nid0+>=IqUllW^~5?|0ISHz^AVVl>N$+JC{?Q-VEbW&B^c!itD zCFB4xJ??3AmB(8`K1=X{ckMg_n{;tFT2G`K4{C-!@}sf;tQdMgcH5%7xIwuP%u9)YjG5zonrh zV0PMA3U1T2b|7PEyA2igVn+HRcgrwX*J9Kvz~X_fT1CaqzMM?YzWLPab2vic&YWPg z^6uQMqF|Bc_kdpJbzLDG#E4k2_Yrr6*S{VJhi^}wGB4{myY{56gi&7d3SH?kNz#3y zvylr+mrVLHD1TvUUzPTx%q2!uN7k#1y!SE1Sy^%gRw>=6@|;b+;SmvVxdAIF;9@kd zq?mXBsnfCz%_fd^DRh!FDg!+XXlFdJ^00iO%x0_2I)?mhW%sVOXFpk(UF=Lm=zokr zscROeEL*%)9hEHesCJB3gc5w@ML+u8pZ3?4?H$b3?(!ydzR--8q>zT0k?JN(-mh$l zwZ_@$-=xSzl9@oWcZlvFpoekK;4>`jh!zf&_DcAcq> zP5*3*X=CyV^Pqlgsfdk#m2un?k#sFtnS6fMA$sJdZa|ujt~!OtutdpnkwmWu<+158 zD*~`eqEQKP`OBB5=0)PhV=CHiha9`Q8(w?p1aT^S>z%~H2I{d!*?EzE2DAoXY8Ku} zN2wz}m@;tw1|~R&Hks~()Cq6<0bv6LSmyY$D|CM6hC~OT1^#ep&-73gKWbbz>Sa@* z1_-X#pD!Q~5iD~ZLu860>N0v3W1gwf`bfHqS6-0Fj*4A%y8ey!k2$kbP6{)$p= z0aDc*{rt$JXc*8@ilMT~fT;F$e6^+rn)cROm?vP@L_DC4MjK<%??^y{k?_3TEnW?P z!&H2AKu%7lNrdeQrgAI5Gd(|ju6caP$d(Sz_U*fBW^|y|d<)j0?MY2EQ<oclkmBI9!e!K*>)WIIjRR^S;Mj zAfRUAqE1!1qOiu3B$c@*dJM zxTEz8t-azN+am=`gT136+moB;e!T>`Fn%a&7%jdBfWLk+=Kl*16%EaayBJ!g!68F& z?*LIZ>+`N#spp1J3*yhBSF;v&c+u1XN>e~UH;zjf8VClne1V_McX>YCF5u5re<^ta z!RWuHJp9o)Xp@bJV0^7acBGm^L!U;-tJDQ7mVyESVL1j&aVzAjzmw#zt(nh)G-5tI z49Jh8=xw&7U*Qe-gr?;)h!E@_LDlvno+iCh_3J zA~wXr&&y|?4a8b(Q}9=4Corbt$UugtN`Us?^jtMb22hT_0YpUiqZx!<*Sp5!hZYcD z*<`pOwt_|V1kV=oEX>V|DfS>k?i~k+jg8PiI9oG2ijL94R|G>K7 zjH0*=v4B&{?LhV#iS588zRMD%*4j@x`__Hr_=`UBkI#gY@e2B70u{N2Rg_z}w)3sR z|PpMXGzn0D!dDX{5r1tnDRa)GXGF9LM~JpBpi@4){^?sW00c-v(Xq~m*TF0zpUs0nayKp=KtPJz}`3vwjX_GAeZu(U>T_!PL{{a_RgiK1S zg|D$XK`<>N%<#07a6?bbX-`G`t}pyS;9Cq!T?x+L;T&Hc3<7NE0Th<(f*G2`_<--N zJSaWIdJ?Ca%lfshDC^Q{s>03La>LjQ+HH-~n5P1B&K$Jf;xng~_`1k|^f7gExr`+lmdSO+wVR-~f{(?L z&#?x{`5EuRUL*mYt$Shd%eqX}oK4&Qz9}~Pk$0S?>Y}&1tdu?{QU*EXf`lD#uiLwg ze_z0{Ee)@|g#e4FaYg`-azW~&VJdY!0eIG({rlbq>czo6V8KouidIkq1AUJ z@;1nGkCT_(`YALG43vu+udu%;f3J6#*=siU1p~KmoIr_Rr*T{WrV_dsm$4`hH4Xz^ zE?*q>X)#n+2|9`6yHL~Bi|Q5aZ%`c2PCT*P6Tq6!{9XCJL%lWaA$@yx(Dy<_K_d>7 zr9A<-d>6pGp97UvM08wmBKUV(#Uu2rK(G(~G|-+ND}h+dCY193srK?F1%*qYR2M{^ zIsv)dumMv(mK;Sx>=b?lk*djK<(TBl$9b?X9*pG1=C^<`H!!>kAn&sl=YX32Ayx%4 z!kp#zR=IxUUJvdSw7yuIops(FfeGVWi(O3b{QV_s*;Q<%!U3~>e$ZyWBi$T>lCbEJfVN<^a9ao1JeCo0V=M#=wThYJdVcq#;djepcZ?wmLgOE zj}bdw6k)(;{_|Tr0`{g@XzC1q(?H2J{&BN@_ZH?z4-nV?3W_ZI|ERl!q5kt7H>c}= z;d=n9d$JGwHCh3KLjM8AQ2$L+G|qrEJaiA0=WgNqmCrw%hni-Diq+Dsk@*&e=?&xl z*h0@gz4_Y~dPNOBu)vc67;?t6QXV|3y2#~@`2NgCSx%)_J#@n)^Hm>X$yB#jQ~ajc zp|nD&ofCUbtB9A?H~0DdS1q9g&z$iGYf+%~&{?E~2RGxPQ|I$}XMO1(hrX7TlwXPN z;bMxHV(KVv{DxSJRZS;x?_^lfv0kio&$CmcO#M<-QhxZRoDzwZ`oydMQAWvfSv4nm z+sW$>$(~PHACi|O{Gz!%B}kk7UO19}w7vK8@2+^K@4RQnJSC&+H;esTRG%+sqduTM)j(ZSN z)!JfuU00}puP8c3+Jq!p9M)3JXP$JjhZ~wTl}8zZy_t3*?P8#DJ1NZm-m8k^hx87! z`{9^g@xR@+AI_(O<2ue5hBQ+Yr~X(oZzfQQX!(BRVBi;CgrO1SG3;km_@vQ4A+Moe z%W9P!2*Iz3g8qSY{OZ#hei!@KE}`vtKiLz_oAN4yB7 z?r0yl_4{^Z=i&hQSuV}69nwuv2Xtn=o$c>OJ3k0-K-*07CJ3rco77c`xd0gZ3tU?$ z7vzb|RwHqsNK^+D=!Ft0@YIE1Z?BKDp9+C&L5$3o6yJffu%wJHDCC*-kihDPw=PBzAo0>%C>~O3JX?~`>_=Ep0s91_(5=&q5VODvnJdK{MSID<-Pf(xY*|tVVs=bLhg!Ttev8CN%SC;sj zK2wqzO(0SOD?C0N1M6wR5J@n;Xw$Ob08>z8R4N|YZr<@>9Nf z_ddHp)fIrTHDo8jN%-rx?$8zbebX<|1wuJpCI%sVAho3`wuAwb(yzQ&xrR?*{?(7u zV|snGxC(q_0%740wFQl)Z9soyApQJwt=APe5q5TCNjQr8e1EdZtrP5&1~>^$QA=NZ z?>^W^ECZWk=uc#Yhj4Sxse`!mng}G8>#u6cvmfd-6aN~d`VpyT1zKNc z?fI(ll%B54#0!fUBK*>#Mhg0K;rmh(w7!3hV#nt+x#?qgw|mn%Q3UDtF8p*TP59NV z++u>q$u=}ZUGdc0!@FM1ExumH$P{r` zR+L@s`T({VqLc8zgOGs)ejv1Mw&3Z1O-9msCPTs*O|raCaP+=dlEfH*3I!Yp3pGvQ zLFQ5on;{_c3#Mcu4z!0(Kk8{Lg3+NWH^k1d>5uQ(4H2Omm|FqhYfVu$Z>xgMQzYU> zKWIAjNgD~tP;gJ^$;0YeoMP+Vv!en@^MGRupkS1^k zGR_e69@~l0(>yN}FU^{g#+B&{P}BTr(}10tGk|cJ>!5j=Q^xnTQ(_WP=i-2Dy~jny z++iGu?N+sz~-8WWci})xVu00V21~&Qe<-`(-x}n=@<$2bhrai}1cL z)SJs~UEa-tcrovnFdCuYFm*tcs85UR4vga$^+|8BaMZk-^%C9+yC z380beVv~Dh-1$cCNlKxf<{Kqzf&w|_ua=))XOg&=snb0*0UrPS5vrT4K58eaq_1e+ zsaX67WlLHAV7gjYy=G|+tyH;<69yzBqoV8}ePA(ZYWNLE!m@&{*iholb0`xR`6mEr zRfAKZ)0-e1MLtLrA$t%|CPJX)Cg9nsy(*UqvH&GswLG)qw6(pA#G*yfh(aqln<%@itE)g26_iIcW$Zc1s2 z=m3MCRgiFl0B4BeYT5sHxeGmXxU)+A*$a*-H7^`VctzYP33oVHiOgS`_O}g04YF-v z>)2ZT;mi-QKa8bDv{ax^9j@ZxIV{CD+TIH0(>E`!nixxx-6BJ`7ym-w2F;H}c=w)Z z$``dhSt8^Klpi=@X@DNPhC4yW{k}h{iMN0OUtLN2$p*rXOFV}HwSbUX0Ta#&SHyXM zvy=58>~n#+)LZNhesm^43;1~IKc*M zFvf0q_MREWsFG+wkA=cruN1?bH0%7HH_T7Vd1#}pSP+Ou@`!$z1+BD@w-Iq3Sfa9s z{^d_~`g^Vgvx7O3<35xS*liqROCM9PM7|=E!`IX-2oGlt56`1kR*>{o5Mh>+^Fs?# z)RMv}yk3$)DM)~^J82hescm_!zp}!yfoqNOy4Gc@4d`NzosU~|=k*XvifTH74Fh27 z7F%AELH1^<;sC*fnwq*Jr?fBMAuxeX0Tm|ZSf*`N6DCdYAn2Qv{tC{gfHTQAvH}Lc zINcy1sy{=-9ktE-vgY?%s=e+}8`@JHA+?f@PYhCsPRH~eW)Or3sUTUHgy#(EGSw4Y zxk!6_rGz_y;i^pK-JTT#ZfRm()~mUvTSj6{R?$nzXVH|!KlwS@-e=Xl^Q(m8dol-+voQ=DPPbq(;dfOF}gW37*tZR2nxg2dErW;HRr+beWKax z#3R(Cw9TqE-%H6A8-H@V^f846jTvpB8{z?DkSiU^<+3Ek=QMvox$x0YUs9jCiM_+5 z0FbS^Ls0PI1)SzLo+*rI5Q6URL<^`-lmii7(O$S%lKyY9GJSPw)Ue+by(XY1I^P%;6oJX zq0PeD{yf`+4n9u68m!`3cb4811WE$5p8o!PU_SwoR2ejjE46C}@0^Db%Z`x40yEM% zr{X`Cu8V?)A5fIq8?^gL2Y#aS$*V4!jIVHNweP(z$`SAbw)Ec=3E16>C-^stzt{-Z zWm5}v_3jm5W=KY$#6TmL@H5)r11(T+0&=1X?m3wSm~~QN(3R-0q7}HJsiyKE7>PIN zls#kdW>{TaeXiO5q)_?f{9NUegOW20yNCzbnoAhk9RsS)%j%DtDox#jFMZ>aiZQTg z6sSi~28go<#uWT!h@_*<;9(Sv$+dIv3_E10EYC#U=0cPV6zI3;BCl*JJG*jXUT=LX z9knb?SD$fw5hnGb95Y~m$-;gy>xba)sOqwZvJ|SaB8=K_fpqJk_+E@+aTPu) z3~u3(cfsfnJ(wK{>*ROzI**s*jJDj!t1^YZMLQ@kiC)dSDH^kl@u|l0ZRj^!O=^8s zujkP7?$#&1AsyH=Bu0SC!q-M35vhVxqv|z|ZT^s7{xmIZNNG3qEi253Io9`!66jOZ zdRZi5=F?6{N*bAyLtUkp4jFGXCrift3Woz71f=@Z4dh0S~yPPE7f*%G73cdgJY3jp8TPsHs z$k)Z%IG`VpM-A3?(9NnuIO!_{RZ@j@h_y+{47@)>s7{Ki6|7aDsDUJ`?Q6c6JxxMI zZbD+5&S||E1#ccU@e0W@a9H9mzNby z>>IeP^K@+w_NS#ikY%593@}Zi?fUd|FA>}qn7S~x!+BWhqv3t%;{;FrwyAr6k9?4O zbaMZNLKX$*BtCekx^L5XOWDAcEvRwai*NCJ@CKC#BIxulBN4z2eDqn0C711FNAEd^ z6rX7_KB8{tzr-j>De^&com~fG_fyatbT2PU$ZGOgOGnid_W9R~n5g z;PabnXA7%?#ie`&EO>Z$V!rR5uh_YOvIiX);iI6Sn3%yMi?*3W-cS zS0}iG3`%PxQ5z&(UAX{BH5BZ%@lw+%xZ~dibw^Af9FjR0s4zb(pM9u#Gvb*7H{$o` zK6OEfV)Zm^ z0f$Z=8Rif;^q732{?E_iiXidJiiMR`nJvxV7%*FKMLZ5}c8H}awDqY&4XFhL2!VW4 z9`GhL0YeSHiL?@x92Ymp=jR!X@9hQ43yktQaHzVc=e&*<%xn`A#Vf)@&kj5fZ7WH8 zppR8C)-B8AJ@loBBC+wr^)Ahqj424oNCr>z1>6U8C~ZqEsJ^p8FB9cBqYix%g7c)YjFAy}<#J<$xDDN>b5G>a5ZM*!9e3cm z5na_w0w+(ZI{FnP;g;^dK^jc}?UJos4w_Du%bwB&%+xlN6@6#X>fX zkl|_*t^*!@pA};tW4EG=pPO`O3=>#<~bZH@ia zzwfcN>wiT`7MOn>GFV$BtGOwf_h`CK%U>29@=~-h>!#d{#c|7!;~L*ecu;f~oqw72 zphr(ga678_0lp|j#9v>NA~Vwq9GRHbkPpnVSxd1$8tRozcd1|E;$-#x)D4lL56qrNq>u6byEiV!7QMT1foK>}@i2)#naCo7XIR z#ts~mXUi_@o;vr(QV{c6M>9v6G`gC#(69s*!SLNVHIFs$yrH4Y`FHd+{`|j)zHtTt zD$~DS@9@3*MHEq?P_=sNj+Ihf-}1;!;pOWM5b= zi4q1GKM?GibcwmDT}a=v6JPifxvxLg;T)YbizvK^KEG&&zBt4OvO|AL?v^Sm$XM*9 zY-Jmap&*FA!+)m3BW@iCTwb34oPMd-$N+C<#@*=tBTU-K`Dw{4yXBw{ncR5~-RtQe zCu5#ueqSU>Z*jAku@#u~WL|yf>C941T9;FaC3qAsE*6inr8iL_L7cCc^P!uX7$ehI zRIv2c^BtX6q3S+$rhFHMF@IvMSij%Ccg$&zIRX0*gNsBguh7NmX=-A{Eb^RehZlM- z&UTs8G_g*J3cpp!482QtODY)Wp4nt6+0^gZBovjIKdF0n-Xv$!rqaUg{w?G-<0t>2 z9?l#S%dO&}5;jEv$vQh6ec9tEH`b=iw+0%86a|C}Oiqr4sXTTeasktPD9^kSy%ork zG?%hxLw+>krUYQ0!H9cvBfe2L?vEiu!AI?{Vjg$k#faP-3!!00%W-`ExWI~t-@U*9 zt*57FrGCbwyVdJFUijq}e!V)#reC(Bu3;Y3q`puM2x6bN1vThh*&iPhRjT!kgm8Vj z_AtFNZexZ9K5(v6W|Gzy^n)`gq3N_fe{RG<0bFVItcLOb$m~lm>Y2sGp!wXZD^j_y zyCwiKPWi6zyV$i;VEm)p50`f&J`Ma3l#9a5af+Lti_#PKrzNDsVfXqdTq^ky?uUJ* zOBcM@)ID}9vFd~!VH4I(^AVm?R2R(KPC}pI)IH*;)+$^fVtz zm8+x~$#Ij}+dw|j>k}E|^}awL-fq^m@xpvi-^V^jd(0hS#;caxw;P#xQL5%=xR~K| zWXtrD($@r+)hWYPqC{hMwnivPWoBuI_jNFbT=$ZEgTO-0ez8vzwaAvhFL8=c_cwHI zHy0QfigeybMqhUHIq3|h*sG*P3Ok>y^0qS~p%Vu{_Zfgy@rqiHv|&JnLfMBfbey#H zrs0ma`R_FN?ti!p=^`6hpo0SBGfA*erPW*tRq{O2NmE(Kkec?4wCMf?a+$}Hq~ihm zM5crWoBjJ}nbnMj_`nw_>83xM)D&nXrNhwB@jGn8bLZ6}Q1tXV5i_3-KbpS{W(h2= z&6?13OH$>_^l^D$0}uRz7Ka?3<*@WQiPKZwCdWLXE5l1(dadqcv)Gw;(&Co*NpZI$ zozh<_v(#rqV*Y;M0|{GtHctG5TMI836`CRA1;zz(Nko#hCTh78ZhEg*Nd10slCR>N zN>2nh?w$@#&P^PAdhx{VG6R*p8gQVsB!kdU3kqalZ9%a{JX1yivo_lL=Lj}^%>Eyu z8_0-&xkD)IFqSF0O@Q_UCQa8d_(_+2;LVNa@S-Q~O93Ov;Su-(wbx|5X#IJ9yF!*j zJ&H*J^CAAP5@WB3sbe0G!w0g7Pfa;1tHvu1@LHae|9u{FCI6hqoHzRlyo#IPH1fTN zVU>}PVcCx+oSRM=*Rv3+w={ejJUYPqR(*jJ$@bB7y~W(Z6nHSQeDisNrLcT56uI&4 z9x_KT&jfAK&pm^?Yp6(L zs2CJk7}E3IP0%_wzG?Gc3fSbTOf(|0l?O@9tF3P-yo4W8rY?Wue}XZ5!w$dA(wbCN zEY`&Hd~GO@nM%7O-r=n5nSC|Pw$$DEn8ZYbi|Nhn6EDc{h^`X=l-3;e{kxlP8IdMn+Ge%6m3f)cCtf-NpPnlbbp6;kgt4srWoUQRPo9zOD6hpW3#=K)!Urk8lPo>63N4NzcF0sK zA2MolG)BklE5$3uCLoC*n{hcomv-D@^BYKE5{!f=l#4ZLDo@ly>3J0rt;C`G;%!_vXKi*TJLBgpoJqB>rwtZAfb-5P90uhS*&EzZO-}1yX3@6v=XR919V{} z+uD|a;BuJ*N8O$23Nlnewh!jd`i}Jt8=U0;89IUg(I&iNZWt@=^LDp+n^xcD?t*{U zCG~OteRDxc|4)v``qMy(n)e{X<}5ZB$`vwPe+2n{K{RLUb+x3WHlH{9Y;Y#iz?;z6 zpr7XBc?Re_wNHcLQ~8F2zI@on*u!O~=ylCAfI(5g;;&D~P0ULQ70Vx~bzoZ4@*i($ zO1dexk)?hhCRx^i`8t2>*HKExeH`cQ!fAc_d@<# zB5QHuu#x1ckrG|9qHiYQYgB0{cO z*n6$*2oc5ikK3lR2S%mEF75hwZB|$3Qx}4*qQ3N%XoTU;>XP#kjmSKncM*Q8aBg;j zH<>c@p1WkZf3|fO>($;80hJGr4XmUiijyJcJWWFVSQ5}wUs~k;aO-|j70lUw&%+9uR{QCcCxl>> zc9&TLNBKYXbzlk*`$%RaRU5S6e|JzY279v#^^x?|F!^7j{C^@%e(p)IQHOXPFG%mt z&m;(z#R{mzD=AQyyCzWYGQWk@6bd9G+>s8zhvqO`6tB0ACOIB?gVUpGb5oDB@D>SA zKb_8AoLAjAcg{3=v&h0&rashSb0uLmxGWZJ?73}7y`sR#c8FAOTgDHFp^>RndtVsw zd6=`Pp;B*Bfk}k}FM}qMCjEywytuu8L?79Cf|lyzNdeMe#?T*qp7vG0JP3+nGMU8= zUu=%bJy$}6Fxj*1uZ)laj)ZSK775a2xdQ|&7A(cCJdJ(Merm#R3^=D6 zrYBI5NLskmzZY#^gU?B|(4#4HFDP1O(~orPeoO?1M#KOUSEe3c(bVjKA80R#qCKP} za((cFY^=_{(yIYQpKxw!YGWe*%4T=!O&t?gfPnXj>lyYV4^xwSkDuUmkX1!FpJEV=FVokheQrnXFxH);fAs_VGX1i|Hkzr4=30n)3^PTXI);&z?F&1y9qMbBcK9)!@oL3y>FqyQsG^acgW>Sz z;CmuCc?>B#)LPYW4eZr23BNn>FKM;d4FkL`vjyC?v*i($8HhRDCw*D=Rj>ACP3$R9 z_(s@t9mHDJ1N&g+XKCnI;!H&Dzc`egm{{aoZJe&pD&;1bju;2Lv>DAc>OK1)WFzw8 ziBK_OAIW6Y+qQ>BmE(z+k7DBmi**ipz+nHQ&jj**Dn7mFRM^vg9?l{p(G8g7FffUb0F$30_$_-}7R>lJoPewAT;sRepVc4)WqO}B9L_Y!*k4q$8#SrI z*r7_U&xB3Z7aOX7!;mqkR6G~iFgZ84_p>41s%&D@W5o~epLuwN{T)BgptJ*$kZf}T zjEnk#P3#lZ4@bahvjYJeP~zSKB4**NF_`v0OwYk$UC|}I`&T>r@;mMST))5A(B#_; zR98puw;$9-$N~2b<8tpio>z%bh$>}9M`pWZaWT1q@QV$oE&h0|jYcxFmNcf9?HHhmV)I1w41%JX=v8Z9A9fis`={j9jzbnZrc zADq)uH?XNX3U+FHJ*h9g-@Yf|{X)}I8|H-z!WVcsn;IRwLp)fu6{QuKL)hwNNTlc0 zpJ#SdF2my_x=qP`OP0Mi?Lo@MnCihPgCEyZz`Pqy5yZ0QFpe)-Q-1<$TRtF>-mr40 zz8I>$Zll(lq(*vHMC`E|Ypc9MMMx^Y+MpfMu`R%3JuLfXE$-mLq^oR0rWQ#Sx%g`! zBb)i`iMb`4H>Xg{{zLGbse{RBZxh-pP zwe9`E#=-S}j4`$e^0uz(pcsK&6swn#T_E_q-H??4r6HsZc<8*7?M?L~+Y~MmVhDO_t`s{-C~8a5mEB z;Su_x$mzxnEY!vP8wJ!n2ZR#e}3{ld=KkiLd)je{ikVYI)}A)A4y9 zu_S~YERXop@ojvB8Dafi9%)^XC6oE~lEraC7oV4S`ZWyZu|&Y_BJq$!3gFyIhZ3ZpX2> z01UCpY1@YDzXbc6=29i%x{C9}`dJpHqdb6$45McAO75sjTd#;(SMBQF3@Mg(Tni>I&-zxr*8)@5xANZ6yWH?b}o7UyPffS_cp9~kH5i(tDKfxK*<4s#(AHymdN5Dq?Az70;?8$ zc2t;vqz2Qb7YO%{&OdNlz{mHQaDu^UC!k|d+3&msUw~5%aeQ8NDKNjdY9dSP?$m*~ zK=j6z3**#_CBP$9tfdI+F6g7nQRn&yY0k{bFWT*QC^S2oNwGD=`?(C<0ev|~8=d*5 z$Jzd98lOJaJ5VcR841Zt&hO?kF)QfF!zNym-+X}`RDy{q^V{Nlf=t95ryIJbGvM)f zw#^BNwwo?KEFddR-dEmhmKEiT)k{LNU&;E&tj3UFrkEns1O#-igH!>RGu3jVR&su- zs+en*3?7A1azw6z!?e<$Z7_Aj29uzn`s6unSfhi6n!VR>hCY09zNS^C2DN;Q?5AKE)hi+F#sVcOD_*&QR>F-bC1_O7jDdl%V4`^kZEY6M*bB&W7m=$+vbh z-DZ4Sx^}Pml6h7ux!&gL#N*5u67jf&^RCha4ov0ikP@!F`u%ZUp9=(vHX!WKSryQ5 zzHTpv&)ZauiqzPWAF)BD&d*kDWmRap_KAXNi7L*+@i;|P))@=DAX!XusaJXeXZypy zEr7}RAJA5le@DaCRjO6dI53*$+`aNDJF0-0SAs+` z`ikIo7LTnLI#Wy({U|dptc0;e5Afyp<@jP+mm&S=W!@c9o7Q=Rh&KB-q*FJP*_(3% zwp_ofJfAlmceXcv4eu9xO^&K;sxw~qlh?Ct#*lVQpOhtDRv)&G#B{9xm+_kulZJ)_c51T6 z6ujgqDAjA-crXT}kqB8L{)~YMek-()eoTta<9)%E<@N>|xZQ<{*I0GOQ(B(Or+INy z!jac+F1oxRj6Yh>c$zOfgHh2p@UI#Py)aD2OhdeQzrerAk4fhy*?gpf-3(&bhd@dY z!va~yIqn3}{~ylYGAOR7-PUcK0D<7H3BfHvgEbx;f;$9vcXx*%!2^w3a1Rc_12itd z-QD#pvcG-Lz5Dz)Rre27R1qrlT5GO3-!aBBq`879m~e%>s}`KS#xFH^6!;6NP=9|R zxbuU7`)z8~yo06AjpCG%huTw#FfBW5-&`1}wERN$e2-m<{p zFlp9;fz9h{EpbJkJRo9h2&Bgv#hXgGM>XT9#1v{wWPu-1_9N}pw^+dN>pfGy0$h=3 zbBqZhN{2>6?gZ}=J_Uz{Mmkr@0l|r00QnIDAalZ`%z;dE663(rABgznlWznxqSoAY z{m}Gn95Yik<9t>@R--xlt1UGr()qDtTJsOA-vETaD(8x!w!Qqb}<}#@qdSxql z#$m`{ABx;UbJ!gJz(2IzaZOq%V0a7Mf8G#H01?4i2hzj}*(**>Oxhj=ncujqeo3S` zuCYfoL6#->mj2OzzU|op!D7-A?X9T{&P7m=*3G4_puyhux?(`fM2I}81cfTd+-zQ4 zNsSBrI;_BjS%jMhBmKNHGjo$(O|)*FV=U(owO`tO|V}sIabk7!8YREr19fhIm=wkD%voCtFI(FfB@fafxDim#91Uu3$Yq;!D z<<|+Wa?Cc#B4Mu0nSuqKc9&!;ftyxyx&@C={Be22PVuu}MS3A@#aEdM{7vw`^Um@5 zN7E5em@^x108rNY?|!`i)wcNWp3IwW_S8u>;4~7F#!etnom^w!+ihyYj^E!wCZJzv zRpSKV_#I{Tsshgvd$PUwVWCU1sURXPs>%Qs&1>dh?chHOY7C9Jpjuu%7FFIw_eAg5 zf6BdyEsL0J>dW+DaG@bIpF_=!oZal3U4Waq?9FG3(HyU;^Mb5#R=8N2Pf!gu{km#9j#=bTLlz-UN^vnoiuFV!yAM_t9~i`uroMcS?%qPe`es!_#|P?wd%p?>@fBDg>)9ZR zoE=t;Ib?y$uikkbK=$yE?b5-Y=)m^T3##fcPN67xx4RMICE#n0t@S{YTe>6gFoNKr zv+I_V2jTsCDf}q1(6v6}<*#?!*owb|d1UVPr0erQXs*`x7`cLEjD8!Z8*wT>jqIi3 z$)oMF(EGC5t-I16J#tAmKTn{O4@TCbE{{DZD$WFX72q>!BkOoxf?ZHfH$;GKs1u}n zzarf3b3qNii^A;1U6wCyrr9N(&@&Pa(cbYfOZ_xDxS7BU#!!GXEpfcqI{3Rs(beQNC^PP4i5e2=ueOj2Wk3H|+a$&w;4Cio zH!FfoJ91@TiuZ%*Kom?jQtncM{2K*{PXmEb%k>(l8IT9-MLKe=hRj6-R-0Bb!UJMK z2Q<7<4Zq%v0j|sz=nZh7GZ3{OJ|*;(d^DGbH+l3@bmFAL3OMzikEr+f?2UJ`nXG4b zv7^`m5Z%NH>Kn@A`NsjrB888_8u`eVy=(_SV$Z;J`$lGlAe}EU8a_or!2LU31XoR# zhhb-!HhDJTx}SQBVug&R7y-O)H`A9n*wVudG7j<<4U5{OA7&*3YmjD46#V@LpgKP+b zC+Jv2I?-qbLk?`zJCVq}z_0$+u{<6S0*2^{)s@ab*bcyxlR`j4&YkxHNEN;U4#QXj z0mgHgE-&dqicE%4=o7H|xHX$Jk9&j*5 z14Wz0P*wC2=F;d-q+PnLPPy#NH`8ot@&>i(cwix`f!4R=W?IYGWw|PrgHA;RXjbTi zX9$q#Rbb!tOGi8YJf(zTZ>G<9@c7$kLi!%bVDA6OPwpB@-+x}Z1bmP{t)%|`b`S(;%-*eETd zQmH5~LE=tJ2P{C&c7+5k2Pc>%Ew_j&e6Rb4e&`Bi#R}%J+|4J$kiC()9sVS+EVmJt z;Bb7UU8-ySCkGbZ>%WG6@L01vj$?d-3B8=-SBhwrnPZw23L`~Aj{?2_83dko(!gyP zQ-xwTVBoTH2bLf5Ml4tc3X;dvlj`CZFU6)$jIMZ@R?Z>Hw1Wm(h4X{T6f0Rub)O?uh)g5`@*=O49RZ0!VW-8pC z#hoR|(1QNQvoz13;0vRrcb#Jm<=^$kE8nGF0%H~L5OuzoebU%px6aJmnU`8O$~uRi zCF_IAET~?ou`G1_=DI@h=tg&k7K!NRu+7l*h$@RR+KBg{+TJ?tdeM?3lo%^*N;kvb zVU32h#`&0lO3#@*7Mx8)Vg=#a*xf ziUEhrEGogR`?TL3@)F2!%XsMq0rW0BP#XMV9uiTY4v9yvt|qWs6JPBLw!gbDhv5cn zj~F!anWBK{>I67hM`(Wq%-v!@VQCLHdh-LY0G$AQldr({U4~JHE_!aO2vy71XJB5R zhsJNiA8zpNlZHX$4H{QK;v@K)1YdN8g@!R*#F+c~4-hGNsW4exngw`F*@B^-z~lzE zRJ9CCpLnQ9xi}sg2P10B^Q9U|^BoLZP5)oZzXk0I+2Y16Tv73x+kb5Q-&~}>Nl#BZ z-tvv^nb?951k3q+@Cjn_tM@)vjVdIuY*Ut5Y3(G>btto7hA~r;h;lpuV{1ZRL&+^? z49`y3Xtf#NkjD}QkSF_w+Tt_yl=A;t+nnL;RFe-IT_4Th3-u<8f<76+BzuibwPLa(16Y2WESd*qg!1C(X z88~4}5)DF!!ET;D`A8z9&%3-+iNyH~%8$uw_&0*CnsVH>Av62q?d3|a~S<4 zQ1o~3;-^Kr z7TPq^FIs^af?7KP2VAMhYm*tf_v<_8X_iB{@Z{0L{Y`GL6YNV~I6i4C{TRj;o5FMR zJi+)W97vkEYG+iQnNhP1)bQJ}OQOoNOqDSR5vE{V*cSS!(YGDE5|AJm+@}NCyby{? zQ?hNaA&+}RL0?Zo)ycM(8F^yPrHSK8Ycg=3D&I>sz$s&1lI-JqNdJlBP$Sx)14^~N zV^5`Ta)HE)$CAkpx{4p!Xf^n;)2?a#`P)NRp!KDVc6>2MuKrjmA95B$H_ckBZnh#t z8r>i9bV-#}L55nYKf0T`;inZ(k(Xrz;5+g~ss`s2Vz0P@I_h=ySyq29S%x8J8^IZc zJ{Ya%;8uU>3=&rUKIMo>q>GP7y_yX9L-xb^k zI=yU7^s8JoZVB~rZUVmT%$nWJT@iQF;Nh8(1ce^wf zdsCJMje!KZ-^?$O$6qPQ{+$3Mwli%#N(9VvO%22+Y&kR2XP<)JPhI-P;v<_DQhbu=t>_nhNHZx4^u>n@1x&w)wskP5x6pE_*3 zLt`W+t%9uQzIFnlf)SCG=9EnKtq*uvc z#9IS92@H0k?j_yQ0NA zCqpe0Vei197(AKD4(XDU{c(b%zr1yI-I6r;p=UUs73MelSvPj$EyuB!f+J(;{R8Wzj){&c*^2VLv$ zwZzZ(eH3l5aVUme{n}Zei5YR%f*7RQs=Y}HA0}khEkN|2Kes+SD+trRH0C)vZ)%pj47vOcbeE`vk2-x*G z?~DRFv65O1x)E1ZJ#J16HU@HO#FGPeQel{`zGyW$pAJsCO-_yXfp@k_avQ$hFI75ACa(QdP_e2nEhb{_^0VM;6V0TCrvt`>e z=DHv;HK=_yMV;^R!O?BMTcCeue_s*Aw5keQ|9pwjM!<)a%&eW4VN$X31LHn|SaqAk z%!*;xzxPlnG0^I!FKe|U)7vYpA8r^W<@9ysrS=lWU*LB$W}2Rd$P!4M@%*g&%WJzf zHN)GTJU!}qN#3@IwC{V1E1Ts>lodlHzQX03ij_ccJ@h$vgboby~)xmyGFh5y|-^ZXPUI7POK6IBR(};QE@W2 z)AEf+s9cCOjHzJJtxKWT!lO(qw4P!OJBA38*kXT9!gJQ#cuPUkfZQI165CVkjR{Yt zQ(%$VB#KNeZ*&Qk7KxdmE-?}wE<}Z}yF2#uyjxkBW_Wl0^F&-)x>!h1TR|m=PV;;1 z{D9PU{RQ}ZXiG%b6h1|fb|*d?3DbkpzvaSVt(aLzT;Sze_Vg`vZGAVbvY(?~O;yW_ zVj*&(jQ}GOo1bBOcAe5()+7^#W^lUEv5@&@S5m5ZK#VCcK1|nH|RjMDMMiN~Zh$9g~hi>xdoaOcSI+ zw1vA`Yla4TOd^Zk z(n{QGL=T7Uk!UDX0XWoV*7A~OWJU3ha-Ry~)0j;bjp@L!qjzt-Q5;F_qrw5_hR9$aY!v0P5S&B;WtLnLEuH`oH_A^wlTQ29nNjm#>2$b~LVLivv3C>|h z6Cp3`>Vk;+njD{MSwdp4bR9=1GgLcdy6YM5UU?sia5qKHxFz^qN@n>A8DtI>u?h=$ z2^hl;s^rb^T|UZ^8IZUBjXf*)osAFU!=&rBMC;R0qy1Lj@o!RD(W9tNoV4Qnt`G1* z(mZB%sq`rd1*CJ?LXNOwbCnv(i!^IK;Yy$2HjPd{+f1ID{MVQ2yd%MveF)tUCPSa3-yKzw`XFZT@XG+2R-Px9bQB-)F#;pNCk$ zzON0BySK^dN4!64H)z~@Tl2w)ojYBP^gP-O5^-C`45V+gl`f3FeyS}zMP(@w_n z+JH^jX5}WgVoFX`z=jvjjJkBQLr@TDJ(*|wH?;mSpy*c-;)lgoVC;_R#%`*O555YY zYLAp6x}+YX?@s{sZIb%jj+5jXu~4P@G0YqR!b2d&9hp?JuY^_~!w} zl(!lvT13XU@x5n`J@qgzsWyc_Do*7L5&r8HJHjis^)R~)4=}r-LK~7Tuoe|1|kYq7(OmVcX zwd!-~dn)(G;@m#AqQ3h^IfKaHe$-;s$6p)fiNq4LRF7GsQmrrp*CwCEQ)OCqvnzXuZu@@#xig~)2}myuGy7m{VKwxH>%tM z`HUUD95ucASeWBcMg%Wqrnh!euLol`GOe;bU=8!Mut^2`@b(MPAD*D~gLh;!D*<>; zaLD$S&PumPzqd-iJSfqI2xP#@r7z{7=d(SDPL}Y^E?pqkiof#%ia9c zjBJPk)icsQ=A#gW|6iY1y=hIIR2Rp?r^br2#jJh|p!tzy=C$VVLNVmk1eQ(lb+rh> zj+pY2QzM;ZT(ISIcu8oR@^1nf;_XC&jTG3|oJ8Ux8HMgQ-;dpZMyvDFaNDi_FhWb=l#6w%GiiiunF{BJz>-5=xP>!jm%N_nPG{NB%2L0Y>vJMOOg(3Qny|2egytOc?=JJtG-j|nI^d~FMnk(%RCjB8 zaHrGLMsPY>dQB~rjKy{P6ENL0dN+s!g~X6VfKc$a`8S&iyx8LWn%Ze6FR@@^4FVRE zJ9_!t&mm4t)ozjW1xcmqLinXRuO5K#^5VO_fl|hf%rR{RzsOf^`#_LXUUhYHspg>V zp!wJHe~H>OFtvPQrWgkCK?z&-12(x*aH=_ zQo6nlwvC?yeBYjNsw`g!$qh=nJeg&Hpt^hHK@m7E=KR_)P*quhDBppP5wQX_3{38Qrj9qW?|o z?5GbOC=mDt((L@QJNC5hkZ3zH)-5*}zk(lXH|-GvQ~RZQ$ZbiKV)0iGzF3&mUmt4h zU58*4R4?8qR$Q_$B@mH@Z1V(kSb|17*fH~G7on4w$|`#~w0I|Q&HgvR&jq0#I1Ly1}iGl;F3=O*%2%eFPx{d_B&&qUAnkpmdfyS=RR z+)ZmVLq0(N7yw*RO`eGcfii$FRA5@6aB*et^NzT^KxAz#Y+A_AQ^xkr>f70-=);Y``U)4;ph0(nH|rMZ&sMiueyYKA z_+QHU_9@`pe+;bu9y8?j*&vwpJgFbGhWU-pgcn9Zf3vdP*EHrv-h&W-Z9LI_T9GQbG z9&_ta{xz};BP$Pajl8;p8D399FtxXEuIsaMOgv}bR5+}=KX3kC0s%+7jApCTm0YL4 z;7seoGpv)8k8FP8l;sRGqy-N)aVJB2fn4oH;2!LQPN6IP^wT96l&qHMzO-#HYDuFx z0<;ljJM#FuBn}nBA>ij;#JY4#t=Kwi}p zg}$o|xS0<0sy*2#@d2ytk#1A_dBHPC!40P}6Y=l0Kc{-*eCpgcsO z!A6I&S-JSjnn+wE@=FmJFgnKG{?kpKNHVTjsV@Z3^W{(OZs(Re`gzE%94+9WL5r`= zMBBxxnu^(Mz9NG^&VL;R#Qa=wr^eK=&2%n)RpbFlGSs!x`7G{rY&!V_AyYXn)dH&@ zdJZV!(OQEm%`CZ0Cb=x87qK(c5=4)a-;{Oo+^Xk0Vr*JZ9amiAO-FL;q8xRKG9(5a zcepM%I9T%(x($A6r^1c#jUAX~)`+gl?6Z&dIMvb{`f;{9|1ino3?t8v&NpIcUa5N{ z!>m(3W;!CAoMX>Kv^Z}USe|wEIoEuz_nm~h;9eV_gLy!DyDUoB0Bx-?|0g;5_9$!iyZV1 zTd#pV3}f>OieGOxFyoMJ0Z65=khM3m==gP8O6wV>K($>jP;T*C{2EC0WMD)mTmwE1 zVn77{4PeQ7&J*dBM0Kf?hV-^fhtsIzS0~1V3}^d;Ye@{@Ng{WEFE#N2J-DA^hEs> z%O8!Aw}0?K)w_zlSJm0BXsM8C85EhYRSX#4me~7zJbDAvRTs9HnHL3^QVh-vh^deb zUmK(EW=uR+YJuj}y0629;eYn|Rg_XX`t2eNv+sB6XKAZB1lvh@=e%7jf^^s)EQw}% zCgw(w^adlP;)3t(w-n9*xQ z1MQX4cJvuok&Z|Ba?`$m#1rQkZ96ij6otXu36#ld#RW#W-+tAPWz*nW0fOhkoZF1+ zPk@c$bJU$@nrTAbu&Su*s@@uzqzm{ z#OF~(cxR%vXA7^n-wb1SmnY36kCi%WUIikJkrBadEWGdh&s7|j+Z$_U><~mgH6{18 zaXy?^2}<`&4gq-%oW{xmX74tJJ52g~U|p_R=3zJi?qnB0_fU=E2`OU521iCqb|hwc zVtSr)T73C>ho`*xMb$d7#se07Z#_V_+SB`i*GE-2NJQ#(WOUk@n#-^ESB9X+YX%yU z&isi}VU?$hcn}f3*=+S0V{o4?1f?KR%D==VskBt(&`Ecuz_@TA{V<0%?*vN)wd}@- z?9`ps@kN@M_-f^f07<=4C`+_`xmXYUGK09lw)a)p{ixtGfj$+ZDyh>mUT4%EK~(IQ&aEMSE+JPRz&#B zb5kiaM<9U&B0BT3MX%J6-H9SMCT|b|uOP0LPD+*BH19;-U2~v+u9!P)@6!MSA{+xSuQ-mSAPjYSPzX19jzNVRe=YCIBf=9Ct}|D&uF!l zZ@sP((Ev$yx@HW!n+6%+R(!h6*|F5TKd* z{+jwll5mI(UGPf5Ji(?!WcOm#?*fBRwEt!r$x(2 zXrBxdqXXcKIRrI+=sVPQXp;hrV2uPppvf-XEjK=_K5G<%0Z_?4f!INP3m|yt{OW-A zZJ2}M-WB3Ni}8f0zC$3K5aeBLb!F5nH$|F%U@=t^u6NONRsg@1vJJ3byUwG|%k5V| zJ`;30zt~~n;KUSRp7MXRL+!%oLgVDp*u&%dJo}P~HvPr9=#!f9%Ze9&5pb~rS>Kk` zyVQZL*<%X%7#f%YG{+V;@AR%)(QMm>?}t*^f_WN* zuwF|xEZy*&@D9}M{e}g06+dhEUh=^y4_~opG>8gKFmxJ?gQn!dJ)GZ%HqFt;z!!X0 zmCh1ix%^B#q>`5FOss|cKKcE4#!!ZsS90}K^hPS$FS!D{K4uAbdZe_f;8^7s>de~5 z_}1DvLrqdi!x!?1t=gf)hcbB&h&}EIb2md6ROTo)QWo-Un1EyvGh%o-SOJE-gC!jU zhCBa_G94JC>?pV-O`!~nl04uy-6~C~)+s|(`wbPNa%O%W!4wl$%7FU9W%idVX6}vO zrZS|EhT z`^YNo;j_J4VtrQnkh5}*yc2POmQJ|n<@>oL!QmX|a&%m9{srLWw$+a8V{wc%?7Oow zx;PhFRCQM@v)8@Zp7i_ut!8Yt@q+!zHay20W=|k1vWp8YjuQI^6o7l()TpI&x#@c{ zV6bo0QI~dK>1uCAf2kv-^1@U25=)+x*0@VTQ1nzm&4AE%cdKpq^+vhp7 zUsVUf!6^R5+ka9gR5{_?J+BU6_P=vjs>aRaBdeSmj(*IHthRFjfV0a@%qP z{%d48GcM_Q(ggcuQPEr`PT6Cwu$V{HN)d88o0I6(ce3@-w>#y3HgbJle%PCsLCFL#6D`b3e_XF^=9a$)mb`7K1b{R`;*j@I56rnKKllWW#7cOxz*h zTDQDHh5m;3%A!{-gsmuG)Mh8R(gKpsa2>AbH*&)tlv5u>aa85rjg}kQQzh1b!_h)$ zGTzr70WozZAkk|Lm*JE5VqJW9{+OsRP#JVq=;F6VeRMeW1Yu zZaecjUYt_70f3xMK3 zn-IdEV_eHA@-Sv=aw_v;ZJf1^9+OY+QZp~f4ka}wcDN2K6y71#?6o8b98C=%PAyOe z2B(S%3IYT%*6$W>*$ArT+TklJ`igFSe<#;*kPaRJxSnlX#pK@$M-_5_+oLsz4z3|i zO**^0H#rj;dDvBK7K2I5rGgp14>uMK?$uvhQ{ryqfaS{BAx-wHuZ6EQrm!s`xgB5K zEf?D0#|_XS?iMYtHT{M6j-^DnS6MsU;qRXQL(>^-PM0mALCmhVH!XRJU@&`E_rcQ zGaVtZ^J%y}-_8Xr<0IYio7%t*G0qxT9y$D@gS=}$cl*Q@ z!oAx!PTiRMpVgOnY77xEZX1fLNSQHCKc0gEPQX(fT{8Cv*m~hDxWL!C%QQK;GT(<& ziCU|%Ro?(`m>zbu~(ktI^6RC;kR}-JoqFGzltn8&KJ9B}!s{!8ES9)5X=qQU=(3iw(Pe++L4`9BuP$Kv<(42t%Q% z;P`a*Em@@v!bqz;p4Bq=$DQZrdDnp1Y8|fELmEN4fC$kBQyL+H)J=j~8u@tsOb#*k zh2vot_E$7G-*AUSMR^?e!AB7g*{D{FgNw48_E0iS%1);Rjby@TOXd$^X3UNhys5M; z39_`m9p%JHr%`wj`vJ^^KygRby~mhKL9e9^ZBub)w+ajCieeL=t^tQW|G2? ze_6ybD|6ycD_kvj@H+eK z|9NiSN~9TT;I7Ne^u0VcL|`R`u#Ek*yeZ}|_`gt=#Yowj3VrT_p< zsacn%*9`xdHVzvndZ4wo8XM18mNZC^CeA4Z(-d3+amy5W4n8A)jT)2iuRMqIb?4ti zN5g=G2`rS)k$O=lf)wQ78q<9JU$7h(*Vlh^UEO5qmS)cWcjnT4**TZp!wz_@9d=q@ z)7m-M{|3}!Orh=9`zrsSq@hk>QBgRru7JZ%@ZUo!Z04 z)~i+gAM8-ZmRA~C0iO=#&_a9;zw{4a&TvS@1RTB1F)t5V@n-8R@qt7^gkr9^y9xpD zdIBJYthyX7pfKL^|3(4%{NJO*BS0L18^zvU?B9;zb@Pe=DFOR>%g2`t!gnDJzEBH1 zS_SZf=`p)4?*ale$81=T=@t*jT;-^C(wiaGBQ*Sev6 zw@@jR(l&qn?Q;k3ojP_VG#@zt0rYh@W(VsnA<^)+Y_7cbcWwz5ag^?t(e?Av%I-fAHS28+-N>syTgw4&!VN}l`76y2mdrNV?J@Ye*0DgoPIFppf9m_= zW$08Q2@m3E3p75Srf_06M45!&4cW2q50=9+@g(^z%0^+am)+g(S5}YfER#h_s6zQt zRAoP<7%rf}`|~FWwY%5S?VU-oz7VT?6t`kSq#M&AdhsS$h)%)2&(UNmauZ2_PoRLX zP>Jw9U&6m3U7;9nMvd-Q7vMkkJM{{3Z6^O|k+13~*XV7T3 zO|SFI4**7B>3Ijv2o)K+Jssi@ylN1LxPF!|O{-GCe&@YFZoXUh_HsqBz-H|aG?3Z4 z=#54DI6l&8lV;?_K&&q_&*_uzc-s zUxWG!hur+RNPu)`h>pUiFh0ib_cmBbpA)V)m#jN89WRFU?k5=E3B4z^uY^Ti1){fu zg}L@Xl!WhkFw$*zSdK_gaQtSaW}D6Sk21+p9e_uP!(T$;kK>7K2U%_4`@Z5lF`CCo zk{GKU>368}A!CVY-YKlo+Sp|bK^w4yHm$N}ZaHQs9?yiI_saJTQV)$a_Dr~4Pdou$ zT7YMgGs94^{wG`s9?}}b-0Dg3Hu%W`X;`BAc6je`W?9_inbGZeA~ud+O-`s`kIQkv z=%@M#?x0G!A0>7#1>gh9E5%qcJExB!6$r4Xv-~>yd#qoW6CY{7S@-osK7TL9GYLbx zrvJRWK#c!e#J5Pp14wYXr}n3fQY-42Uhc2{OklI1{xu8Hhn6p=x7r>3;bGLSBgr57 z?p&SD6Uzl-7PlIgBC#@e)}v=IFlY>mw$7n-Xr${fuW75GewF;uWIgn_t#3ziU2(lC z4c!vAh5h1B@x)D~s~&6zvtjqNASN#EYr%FIr)aaQbl!dGD6&sj)W{p!55r$J2Fs{Z zw=}pz*D_LT*vt>ys@(;mQZEqprc3)u#YD8XxW z*RLPNiug6iUjG8LH#=EppU(ZWV^5}k>Y2S9DImhje(Dl%4yQ&!!Ov~V9yMjzJ1N)Oa*7Q%&ZwvP?qTouveymsPc*@Z0Vpf zV<_fC?^}xIE$`oF8nHEnBP&3=q7Wy&9GlVFHmy>23!Gms#fYbl=h{gdkg2`Jb=nuB z(0)%ZBLDp2;r7vMfQins4teo#L!xy%BR4)(h<}W$CqRU@>9n9Z69bueXijnb0o7T` z-?mG=sLW4fO=nwEugwP&K=Xv-o~IbgdB`B94^*&@1-{|Nkx3LGXN2Y2_v=8IrBwO2 z6_fhESpaAo`@xZHu6I^Oqp5IuGCvZL!G^>n)UMlIVv+i*$>*ZXztwo#LvZz2#O`Nw zg%DcLDB#R;5i#(Wi~&KE)4$O7=?MwZ=LEBh?z!Wyw@2;}@!~f+C2{xsy-_1y>wUCe zeEHH^?hry%4|Za-E*9Mub-z2(Pfl-+n_~;8<4vYYPzqah4*&mPTeUa+lRjNxX8d6e zXuCi}L|nR#;Iv&rYkRY{zCH%vEoyBR@7j#>I1yb%xbux8yL zCfwGzs5S4L*8*Y9c#)&_mRq?#81?(n>5Wpa6y!|`7d>=Gd{5z9LuE}lTQVijbx&8l z!`HW73-r7)=)Wg_OT|dyEwCF~m}PojT`oWs_2K3~W22D7tLc2po+s#B4m7_RnGF5a zIRR_;xLm-|FKsZeMxm1D{DsI5&*zf9KrSWV*}Dddi%vOS1zGx1fz$@#Tl z*va3P+jDAI=vUeem+47D^|lR5oIE>)wg-ZTg{v9W3_c%Na&OKz745entjj+@?C5g` z_b#?pi)UPWTo!((N;C6rVoE1ZYz=I5H=a&A1=&K?`yrFIoe}P86+98O)2)8o+ ztgDm6sZ4AhMUyi&%?>5|cSB^W36B*jxoSgLXEGm`yCZQ1i%A8-!oxemFa)y~EYe){ zhx_*RCof*XpXFs!3M9%i!whXPcLfkSzEQ5`79iwD6AOSv;I+Jmh`}ErLh4GEW%}h` zgXRmCIrZ;u!nwUxDnSG+I*ouD&*Xo0{n&bI8+B(zJMx1+6bR^1MU{-(ahKL0X?l6V zULDQ_Mz@#sQJ$gmd7ZEd@7C$PtE7!kz>M1(Z2(73mjy8$f@--@7D;&A2=Ior{P;F@ zuP+B}6<2QUts|0V3=XE~*qJWkzRb=MI1Y0BEiTAk%RVw6Au2z5s^r z1W568rBOTsrRnO`Mz4U9-Vh+~x;|%`_1(&bUp}4lHPB@g1KJ1Vp-3POw4B_s0S_%# zn4JOQwIKkP^j&qnwkQBRXLUJHhsl%F{31w$;D_hTgu-CL{QFq&If9BhBD)XbLyMy> z${5^!V97;&A*T0SKu$^+ybf=a!RMg;?$2oa6nSu9M2_3sm;U%@N7qycm|#=;Z}pMu zLRec&p{^CBUmmc-U0E2Ippq#z3G|>9_Zwd(e)_$+f@*g4;FzP|d1TFN7X#);EO6Kw z2}a>4fq>(LH&T0g(B5r;KPsM2m;L?ii5YrL24>wnU)$%SFm*|!*Hi76lP5Fu`dp8M z0JTvdtJTo^#x=oJs<0pa3;tU)zJq(KHi`aZU!Yh_ExR&^qSY4BtJMX!8kOZ8x#*WK zcam?#4ZHW8%}$gJz4#nh)hiaCZTjTS9ug$#G&vOVO`Jcn5SPhoIzAuq&o($>s6<8k zRoIFrmZ_7qm+1y(wLNBuAa7^wbFH?QJ?r;$RRqtvR5-tJpS#IJ+vKr?{tSm`@k8;v z+M~eleGlf^uuVrZf>~gCGT_@)7w7xIO-n}1Zzj&(1)%NOjl84Opgo)fq%+nOn)#)a zZN^^i+z~Fl!+_T2E6C}DtYVV|F50adR0pQ^q*B3a{MgXy{+^ug+4rg(ex+b7l3dy) zou1Ci|Qr&kE`lA%RbC-#OBcGPZ;30}SwYm#kx90|Dv#+?L z&vj$E@;-N`RkXtPkkiL_Q*+{+!a?zd^+1Q3^}YSw4A62%3AfuRwTm?KJS~!69Cx3T zomAF9?3<-p_X*QwW)lUkE6n3r^xDGg5Fb-&5rjY47GDjk zFCx|wG8eZ4cVjvGSt_C_eq=9+>TfE@*lv2uKSue}Ij}mB`6_t)A!76;f zUj9)$@)FBWY+r}}%+1`54CWo|q)~z&sCIu*)UVawBiX0sO~};_#;(FLbik0}tvEa8 z)KD(Zhk~`}BJg%`4rOV7c8^DDF03k5435NMNGiN9NLz z(T>LO+wLw^Uo-i90&?jma-IMb zO?SF1k->9;0sjF~l1b06ERJyOe(EQj)D*FBHGgLOEq9b?8^(ihzqvqU=JAXE!*-j{ zJC^p|D^ZT%1W>JgW4GCAPH;A(_=1i4HGi%3G|HD%L=1mSGb8xh%O3?yE948JkeQCQ zmgX&n(bh0O9kHFI2K>A}%W!mH<(Gwz$*uU*V2wfCriY=DRfhP((?vvgp0DdK-jvVy zD$@rFfM&Gt9(E&N>mH)_#++0%q5~**25QC>0Osn`DN}u>^Qu|HUr`E_it%BZlO$XO zOZ9>5`nT`z9*iE%#+*OP80BgKN?hht#kJL-rHS&qW&+s9+?Mrl*R1uF`E@r1*m z%OSPy5y2i&?VB{NDLj)9RdQ{miZBUy6M&E^5ETmnq}SgMiN7Kb-fxQ>B+} z8dxuo@1S#;8*tB%8}<`FJ`->}$89{R;cC^ALUBf1b(W=>81p|v7}(+y|jlR~wo^zF=ZJ_r5O)LJa+lPz$o z5ylsWAS!NnKFLkVbkv3XmY1C;1gTexKlNbJ;K4sPjc~>B|9oCaQ}0xRs~15Y%bHv) zkEw9ibVq;&;q{>X$1F-JQ(uyTi1J4(FIJ0w|HpcHX3XJmR^<;mwFh_elx5(01tCmN z#{4^pq7p;{C8W$zT_tfUoIYHMdnR@#le@<;XTmZ#f(evi&1{)uCdR;ePkxRC(r{(u zbLys`kO`z2%(-X3Ig4`CeQFs>$WKBHa?P2K{&p(`u_~X7PQN)YkhlZFgJ~9C3?He8 z!q}sLajz$diFH$f67#fAn{XqhU|8}9_v?6Y;P#@ zDz0e8o$rKqkf7Q92ak9MH`t{cXMfU?DPw&2drYhda*~J!+%ggS;dGqs?{28*^lXLX zqn6)Ygj%Pn={3Q75}s%z%^jHFKvQ9t$W$pvXTcq0S>rvsVLef#jA$FCYav_p8D~e- zNJwnw(W|}!YNaSrUh_29c&u<=AVY&Gaxrze_?}E$yJM$WE;% zGnw!BC0#HjI&7?2v4AHLKb%ETs|86!3(_!zmmtuu*GKcZujGcvt$R~0C@^0pw^Lr< z2OS#C)Xg(O;J=c5=tb60vVJ~lnc(#@&j$|H@Yu0aQ^3exB8u&xA-|aLo3!Ss%pd|W z%LVzGB`Ic1Rbh6fC{%pSP;iyQY3givWR$)PPc3FdvoM;VYi-t=2+w3-2kf2fCb?!d$kka=XL|VJGCR>-++!# z+l>)wMdX?Vjk-a{Av@I5pZ%0vYGP2KhXvRIgMd)Q*aOA~v?QN{zqxZ7*~G{%J5C-h zbL9GgK!0)gFWbHQ{_bYxXIB(N`3IFwAR7uo&~N?^h^qRBqV@~4QNgWAGCCeZlbc8j z3|QjVM9!)1>Az8)+~viG`$!8(jdUs&3j&YS;N65D-#Q{bGxfbLHi~6vdum>ai(559 z@+XSogFb<5jauV3hi){i=+G-SYcNdHpb|AvqW<3|ON6zs^9Q)VOLLdM`oXL=sPI%7 z`as(vN4N@(g`Ew2OQ)GK^zJE8#v`M;8HRVJG9}=A7>UZby)fhJQ4`wo~ zhik$LjwHC@_II<_^=g+$dT3F`X5jEtBrU~={$9y8vf`vgyim=&vS3!hYIHEUKoy~cpAoqscRL==F_b+>!2Db+xU zxEhHG_Z<^Wc){PT*n_|OqtB;HKm>PYB4qhE$};Y7Hh4QXR?9l)HQV3detIsK`!SKE zRF%8+3*b-NMB6{Lk58BlT^@we-Iga{bA|1~n<|a6aEtPw_kJ9=Kv? ziJ$M_tN-JTbZ@_;qFUAK+y2u2X35g#5_YO^>2d1szfWqWyz^NNxmz#dX*l+t&;M41 zO{?ndq%vuWZwDsAGV~fNYs$6P%I4uBwj2D6F6|ut&gE4(2=Uo63_rj#3T5JD1?;&S z+oAo?tf-wZ`ioCz?3xaU-&>4_YfaqDae7C}lmVD89UEemC}l64#>Ci*k5fVL)FH%I z=mLg2G>K-Wz;I(<1!eQ^7xM9P)hQ_BpX~HHi+cSJ6!hXCjjP^}GzdUJpa8an0Sg8g&ERmoF<6! zkDql);xUF?K-F9?^{=rk4;@(tfOhUAa+5?=DI0xvp(O+IfaqsHvDHR_o#F&_gx*oS zEsDNoZ?1+$pyE0W_0b-&F(!r!h`WFiK|-@5-+;t0sB`;BmK!k8UhDscBkGgf$wzfa z9Ij00(`#B+oY}7242vDwo~Q&@Q!6)@FCa#`>Vp;Qya7Cn2CS@iN;BP|35@RB0!Ww$ zs-DTlAZdDG=imO9^AKnMomkEAlfM*jlf4Z1wzH5w2dgzXNv`nM2Nk0>4@X2+CMEf( zdmJ^!d~7!M8yqN;*1?ZHjvj#BEVR@8+l_8Dt@kxg>je`M_A2Te&vYs2L-x|pzeMXl z4wfs4+eF@Uj;h>fqb+Xu(cY%zUpMHTQ7180ZhUq7*qPSelFXrxlf2jpcIZM3$ zhj^;s`6ptm(6Z*Ff~SoVMR0c9s<)B9^_#}p!)rCX^pt)QD;gFowM{=F?dlcfKxBiU} z_cvmN3EqGuK`c`%%!xyhXa+JfOuTR{srGT*MZZ@P#)vR9n0nSnohiXFO;Zx6%5ZS;)}E? z)PTvjz_aBuVfWRQ)2-<~z%i>Avra69u4r%;cN62{A6Y`#9EfVK*0`Qbi8(NZI8~e3 zWLL`lYvg>!G8)o?!~D};&9R&i+Z*SEnbNd4y4m)gkOEZp#2%JZO{p92!t%tzGz}K?dO&fWuu9307&Ducxp@L0GsP*9|8s>C2kl|pq!u;= zUhEG^xQ(J-cq6||k?<>y*pcv)VV=)phAV}W4t;Ag88^@aQMFMBI*y{j814ZVmcyh1 zO{uf@c^^x&=q7k_5cV@~37Pyo?NFz0Ck=}F>j76I*@yc(hUZFAPoM^+?~Gr^ZNq)$ z*r*aGBh+(Ec9G@VN!#fzVgEikg!^t4|HyURc5&HPBLlwhr&(UPp%|a9pHbb!Lr*ci zgLY^|!=00)%xXwl6P;t8sb-;d!PWT=(s`H0zS0YnE48@%jqw(D*?b33f}2koFa=S| zn}yx*WL~Y!C!ASEi)$^d)w?QVrw!XDyY*RF zRD67CVm?~BN3O6qZ0eWrf_Uev_1{O?@vfG7pxKy%N!g(JoN$(3=QP3~H7N_FFy$Hu zG`k6gZ>uYDVDn=@;+!YPFuPmzTv-G)F}%B{v6Rug%$NG?17n=3XZ6pPD)xUO)Zrgl zEiE#yue{r~x<8GTS)U$i^LY-6V(7;v)1ngu6zU|3(1a@)zL%(3N4Z{p>$^Pok(>I# z>NR($VW=xW*bOZzhjvEEIQW15A3Edgf1@+h1tIVsKRzsP<7PkrRKqj;lb{W7E9?p_ z#^joP5+0sA^UdL8B|%XhvD_LfBOqCqtba%p`MbncV%-ySL}F<96Grqa z8EsyBoK}O>b*AnaGMWlgMM-x7?`t$Mw8d*(fa}_eZjeF>i^yi2r23V?Y3d4Qj9G++ zb9*0qUJnI@W0G?*X~>#vgJmb`Wxl+Pa`e!w*+u$wW1_{|oxTzT z))m)(%R%O61dd$oLnIf7Uc40)8bc_fNGTK0M@a8AcP8%$chYW+$@bk#Ep;i63tBKV z1>7lZ{%r=+NO+MPll0>(#GJ=g`8mk@DCLjtf5)IgS^A>)8DGzEf;M2K)xU#|hZvn_u_>U&v?D4*s)&*aAS)}Bb8jW`Mo{Kx^mc6v&dkYmtErx~g zbwv5r=w)LfYtj+&|0a?EU)jm$(dngyNZEu7Vvus@*n}@`u8=dCrcB8elVEhZFbn)_ z_R*X1sVSS&yUCZjR&sFeGJcLVU~B^&(}3X9^z$-ANRA8<^EG)a%faRw`M+q$YrDVt z`OaEPpjlk5OWBviwe6k$vlrgzCX({nF_%g__X?zRInI-&U4_G3DDG;gyq$%K**QN9Hsdg)PVKPlb3%>53W-TH>yIT8GU>r{g=e1HAIdY$CG4 zL76Q9_zk|E40kNwwMSXcX?DoP)s^rO?Fsd#?eUn+%MjKi`cj_XwonT*b)K>9M}FeI zk)mq6bQu-AgpH%A#7$`?FN3H5+5SrZ`cj)U?+5)DTak-4l)yYa|6M+A{) z)|fgaFStuFIe#ZJPD@ldgc-HjqU#0RQ@(@lu=&1Ge5OXP4{h3a74NJ=UzFQZa)&O& zxNj~9_uN*f@Y`HXT4i#9fHvvBeVmzof$=b*7Q@!kPZLU z=aJiP|3##?emXSvSko}lfOhT|tUIB#!5lm1PZbCe=68>yYah1V64z;3zg9c|g8EarP-_DVTKRD79Gn=^WQ9MUAwrNaWATAeh1`eh z#uw{9tZvCm^eAP#*p)_xsL$r75gJeIV&u1Otme@ukl%RcV|Th{2w}ad7xPdfVdTQqYO4agVm|Ge#_Waq(Vj3r*wmj95e{lSx}Xu zZ#Hpw&n&;o+&!|r`ZWJR)U@qdbN@wAAT|*R{{P!mDz;M(qJKXylhL+%HGuw(``yi9 zP53*(zrqLPN;-)Uv<`T-PO;6u--*iB$B`j}Izm$nn{y-?(w?y=P)tcE(D?J$;AfdD z%GCWhGxA-D=t95z4yqf4idymWj52n`ZL zyN`*r`{xH|O0NFsY&6?m^dsISq58Wr%cZri#9Bwu6&UA)Z*!W(wO8tp9v z{g#;*1b;a8RMa2NKy;TcsymivZGMFdxU{NB#$?j~*NlwUYDsj?nOrGIICQ2%b*qh{9GF+xy~)y+N|?Qo>5krK;ANKZ#)hp@E68q+he$W#;5VM6ws%`ShpsGhQ-ec2IG zzShIHwVG_}UH#b5iM+JLISv2RvQF_X9b|G9gx{$JfwLwDIV z0m}4n^vUUbnUJRYWRD{J>Vif}bIo-!pW$Qq{}gW@jsN#7LR}ExX8)UrAOt{r)fhEC z!TdOgO6<)O)p#x@y)?2NRu6TJkawIQZos{Z&$j=G|4#My;G10&5S3yfA)gYVy`cnK z-F{8W{&6l#ppaP0yU7nlFfr7pzbvu_C?`sB8_debzM{Z$W&G$^2tw%%ZV8Hp_J=U4 zZ~T4?w*-WTcr2nAoedF9AKxiWH`s9kIc$_^Wojs5PvGot5>6K_x2=;l)!39*0ZiTh zJ%do;X6J&o7B*=%PkN#}eK*|UfzA{$x@vPDfc-COr9@5cXf)YMnhCWUwHHn+8TOM_ zkUUr@S%$vzb4KD^J%Hf#hz6n3N31QwAPcI^F725-^Oge3*nK2I-nrkh5k4`2!F6>N ztHD<_1)>lR;3{ros$PA4UI=w z+)C}vfjLD)gD@tPsWrbeDfD>|{-E4b*C`o^rv3S=<4*N@XW9A;CjeQEy~@MSxO)|- z+GeJMX40|W*5!^}rt{r)jCZ&9?M6y<{z`kGr5q!EA&nWclTLtxb- z75bI4YepJlHIoQW{Ay~dZ0FFJ=2U#>$wfK+;nFarg;V+7+sB$MW8y7E^nYmaU%HD* zdW?Z#Kp^u5EtBK&1F(a0G|RD_u{Mo?gkd{Wo79VXRRN6YW5f=c!yr!SHdF?*2jm$n z)%I4Kem^e7B+*xvh>G5M`ZTh%u>Jmer}a6TYd2K7xc_^js0}upaLH$TT(!}AnUT2% z=m0ge+UB!)R6q{=PFsUp&vC^|UgX9A4cZeM#<{la?dNpTw8{DTiP=%7fnA&N>(5UM(x@{qKkfLsqR+kTIwfnq zZ^e(e?rnuI)}Dyhrrd@*_vH5a4Cq)hKbvsCFNi+Q;8`ALb$QtHn=c5FQz>4CR}gi2 z&M)uNG%(l=kU+;cNkOXb^j;F$m3d;5Muh2!xUN830xUW)KfHGM<7KK_u&B32e@1+j zxn3K(Ol~EHnv>3=o@8~1o;2R_CPaRx@tKm$w6C^#p9hsFEdmSm>K6}z3*;!yOL+Eo zrCKYt${T_`L&Z)_0QmXj6?!GnNn`tek`*_5NCA0?yL?e2tX`M2s2uNYjy2 zG}tk$Ba>yuliiXC*%s7N;GGD7oCDgfX0jJcveGi~OV1+m$1@x~X0s=#ys~{LDrv** zn|?g2hlE}(LoD3Ak#N~8vN0O+Q_s&~<%l|oQ zvx<`chb|D9bM}Bmq{5Y`aas?%acY4qc#DVKZ1?tMTE1V#*ife0S8M&Lj|K*o^!w9a z<-bIbrL*Up$^m)gv`?Q@0f|zF%dzxwfMa~r_9++%#2kQm+fIqV;g8c%q&HHTE`V7@ zSRCybs|^>}A|u9ZJdzkk{X_CqSy#l7TX=Tp)MQNQ&O17iOc#{6xUf#$8jwuReq!VP z!Ab{L?NHG&?dpSN*=K#m9*2OmAMFZ=q{s_UV#=mP?iP{b{i?Dpd5&)UUKQWQ zpn^05A#eXx?}SlxrxdZY_GX#?wprRcKDEcIr>Z`MW5`_>yvwv_cu2pcd*pe)@*7p6nJg^UWw<(0sAfVnFV^Y8B!`*ryCRf zW1vsKoa2CPTe;FzW+GF4voE_d;k)-%b!Y6ttt3suZ{VHk`ZvHo5^8bVJODJPI?#@kn zcmIa!f?#OzBa9t@oiTnh_VvX9!Gb3(X*^Y7m*`&bXW!)F+;*7B3SduA0I=22Rz`te zl`KFfzgMi<@@kal6GHltJB9=DHd$Rp#|+K*Tb*o71Z*S{kI^cz6DrPOSc3~3`v}u> zStv1g=Zg_Xgn`MDY&M7Y-Yn_MJV`8IgIm&jdz?Kbnn@LQ5&^_k)Zrlvs8M2<-}&PR zForLI1bIanI^TjC!ExB{pX{fvXKivDJ|#SI(#S;aX6iVa(Ji!YBDkghc2=PDe!f|DwPO zP>_EtZt-}`#kz)xUndI>Cv^T*!!s;34Dd+%T#rPGuD zx49UPx!`Q`4MRb3K?Nd}5cGCQD>&i|lS%J0|F4sr_dl?;M&?n5|3b~PlCj^*r1}#w zofE8Lym0iC{DfB2fli;tjl${pu1?2y?k(5sIsCMHZGoG1q`XauS=X z5>Dz!<*rhlzDJy$ z+3RD|fbY+B)d9mTZI9XGhBZi%cE65@z65&lOuY(~hc83mlRsd8;_ z`^~<@eRMWd(8N7DoKsGKQ?I-e2NDbDLjEuZ0rV^gaEOoqI{@Pj2WlW|uaAIx_Q-1% z;rM8Y?Ul!ai4IRS2ewStqHnRD45qUZC?lkfdWj{45@zMcxE%iEU;*K9tL%lU9`Oa@ zFuzeCLOL}>F`Puhv8Y_MVov&z4j-1j;X1Ha>$C+HrTn?6vXx65#SK4j=y%h7`UeuE z$u=q?PsOF^d^++xQ#(pt(PjFA^QBLS6O!$UJ3Xb4^B6ZxNT^eKbRQ(> zReYXu$`Pg&8+0PDaVDlp9iVG(`yp9hD#moc1dZvLOD%9qZ=M;96#vL`)^pNQtPm4$ zuVM+|LDgva^^A)k`0i|~_2#)$_UZ~8(-pUrIv3mZzs;5_FT@>a4qovJn65Kd ztcwTw^p-=DbAYnbzz|S{I&O)Ls-{$_)Iz%*9gTe5=6i%01F66921V=z+wiSik~^hvTkj+Yoj(W&h9)LaB&VI(S@r}` z;Tu+sB&cBETBU4K4wj=Nu>}5>x^8@)qJ8_T=CL$Lu`!WxO*Jc8V5>E=jdP)#r~0Dr zv%T#3J^MlY&T+0PE>p`pXvv|972}JvZgR!UBw3Z_39%_uBC~hJ3`^gRR;MEtCNTR& zmPjS#kI;NC$>!@)m?L|4aAYLh-e_ab+YF-!$K*h@5lXT~U{V)O`Ijh7X0g z0)LzQHBESF>zcGH!7ZSfYaK^bEYu$Nb)i26F92Y!pU?wwH9QmuVAE!Tt(VMM%_7)^ z&77ZtKBPP!{r9(Q!lS#Lm)b2?E?PYCmNC)ch&&ZEYZpApn9b7Q$WrHKG0Gpdwqqd*u?cp|{>Z6SL7c?7#c?x~%@e zlO3Z8_sxA-T;B?8`Tl|AnSMAG z=HMA|_TrPkSC=mqZE%G+A_fQ&=1LwPm<`ZKUmTT2VBYV!jEV9>uN_~GVSsH-pN1<% zlc+X0O>kVUrE7{-Bnhj*9X}()4Y8`K) zOTUiRWUs|;(%?hWqpax${bQ9Q+NYvu#16kUMK`jg{MZexX0i%r{%L@6ixTB6^u=p<}D2tv8OC9N;%np9or7w=OFoc+O}cYfW`4$fiHH4 zq0Vk~!XM~xj*(ce9e&EmPiUgQd_qfKY4T--_*(Bznrx^pxxV93fAMIUI_fsLP92Gi zg4)0J=Xt{UpG5sh+$Ym%M6N!M?Q-{rw*-CzdY;nHx+!5s`C#O>uWLZU_uuUkiTWsx4$I2-ZS5U(+yug|zAmZ2SgIV$@Qg~2~4 z3l=KKmR9wHyRJb-DHo!)U&Oaa$J{&(N@%e7Il=rNHt#jfn9V6eY{)-DQPtYmsuyWd z(nDWj60_jMlF(=0)96bmyk0fg64C>lu5Y5;6x|wIfWDo8~3XJHbJ#mR}|br>)9cPf@>B?_Y;ht8TQyoV*cIorX-ZQE zZ9SSyoRfY|&^YSwYYks!&t%#3rd!c%5zHi?Fon4X5E{0rU$ddQ-nKp%m3Fs5tW|!K ze^9#!zzGV^i(*oG7k+clL7x|V{L6HjnBVz&E=oE|KM3toyZw@#?fqWiMhkCy$)s_p z>y~N7euOPb`Z~FB9h(!$FzRa=7DuAC$QDsn6_qT0rVnUa0w37P3J(kRr8wv>dq}2= zaN2~4XG}9(^HT*cIJ#VIQ~F2(o%n5gs&$RZ$7V5#-^!eR9jUy0BmLo(0h42YmeNl) z>i5z^WLh#O90@6^hpKKQrTn`qUpgZ2Dkq6JlerT2kU36eFb<1`MJj0J*h^*(2mm)Ht*xfY^M@HnO$M)x1ha2#5o z7gGKB-0qOI;8a=qRaPfK15rc&%Vt;CbqzJ(xc<|CmdLh#1E!m9svXe@HdfaU2ip?% zob(AQSQCDe5R&Vn(RL8v%VvROjCvw$3diwsLdYPjP<_6R4ww%)VwZJ$t;`Y<%0xe} zp>33 z-v=}!Zss<7YS&1&V{Iob76J==LI*cz$dn;tL^DmRTq zqKvrXkeLbNPEoRS{97epOu*ROMF|MR4?wE8w!KJ20;`nrvEX0*UI3z+1DpT)Rm3oq zG^=MaRR*!si=c4%RH^HkUR^k!6SuHQ3{-}hP{w>@K^q&khQ&uUfeo~dk6NoX)zNA| z`7&Q85-EVdil|NO3ncbSHMGaR2HXM{qm_=08!Y=;6vU1b1c-BM+Q%nyqCDrB{BcZGeJXc+&8cHUIAB0%{N_cZ zW)82-_{KoA`qIt1^XWT$R6QaVHOyxN?<0Y{uz(3;i0u6`e-s!gDP4w3`|*Gj_AD@B zz+3H%Bq8NVMy3q}t~%V9e7W~n-VW{)9d}^=76IixqkR>i!pJ-?PbY{2re#X2?9Xk8 zRDmM|m#yOx!Oe@vRHnZm5{Q?Xaj*wOV5?oVIl~-2_ zYL!MRR2b`RUagOh{w$pds5TzhYSp7EAJ(Nq;u8_B1Hu-a1pO6#_;1CtJh{&a@38}| zJ~LC(D#^vA@MVk{Urm>?Tc5}2N2C(bl=7@fkw=uaZb-lC0h5GG)>_N|C1Gult!jF` zaZFW#P}MGwi?%6hvi=MRoc^(FXf$C8o(vM@&D>qc@C9u< zr>yT+HFNP5xR)%$`D6?xqE|b-wPpWIi z1bNnNy5s}Iylqy2VQ~ zfvP>0yT6+Fv2nfzW%(qJCz1LoAH3H3DiriJeGw!&AL!#BtBa zf$BO^CDwR8BVOrxIxPZy71yUO8}0DJ-jI&v6V`Du7>USkHAG3Xd(f4;nr^p$4=?4H zb^01-fL3Jt_2++X6M1E9sW+4M4CA(l*q`&#J5jh&k)zWA>FpQ%c_Oe#u*%y6-qkRgA8#XGbb?|w)xu@$ngE!&7g00$ z25TsZ{lny$G1zg1#9ROKQfb%J2hNP;<7X~lGQtnrD;hk=P@)xKt6z3&MH-xMHyY;B zpmHTouu6SPEI0YZ(sf+FLPF`TKp}ec^mWp?Q?|0&*OKeDGRNBV+SJ6)UNY|~Rb0N} zW)7kxA2h%*(b2bcC{&13yip}!w{hiSe7Po*Cck(XIoBJJ;5HURk3~F+!9d3Q0Z8k4 z!8bC40<#EfnjhMT_>n-~11N>yfSl*6Q!5xbm>?x)F<&#-rsy3rm_RTi16V+kx9c*@ z%}zeNLK#xTF;O6S>flv6M3xku%== z0xxrNvTZ~ulj7|NQ+_^H!#kuyv4UD2p0V>QV^zazK~v~mA)RqNJ>v&Z60I0EJ@QXB zN=t2}G*i>~_p%6a4eF&V*cLmrH`)~>Q8lGgV;OLm{DR5?TLdb1NNG-`V_Ox&9*Y@L*(h7%8i>i?z_DACbR z9^GJ9{-ln1&7C7c|8MA;3ZqR1SH~u3`n4GWA6p|`NUhahk=o!Rozouc_Rkyrxi|xc zH6Trc3QV#6-_b4mEJR*)7T42fQ;U2%C%o3A*Pe4ri?vp*6ttd0yHq7o~o9QicswWT2Jnlk@Lc=d_${7dT&64X+{ zfO;{%Nt<5#e4^g2-QilCpWyXyy*$xi5v6u2a9fkj8SevuTX*!`U4Eo4j+nK(T)W6> ztW={wi~JQWpyK?BlN+h!zhnSH@|ZP(p0kFa;LmRrU^>x>( z`RQop{Xo*W$Na%UAuMLS4VSc!6AWplSZJxl^Ka+bq|mv;g$uq~W%neO?9-X?ry$q8 z0U_jPpCD21!>1PuQeykj&&!58yY3*3^as56An)xlu-C>=qMHES;=`i!=8>C&QQQ(G z?S*XzWg-dd7~7!|8PuSZ4?=P;GUra2LN9r#&?{sSfcYrkZ%IzIIsE z39jzt?6H0Q3bEgyXEW!p54dHrO;l(zmWK)}AQID2gx=cFXTR0FRYVJKxXCx?+W>Sm zhbTvbM6QGm6Atko8D1;-mrmlOoc#?VQ4Pbm>I0NXB9rx93NsJh3~cl}&O>HzP;`z> zM&FgIhCuW*^nQkxy|F^fOk_;RzcTiu(-hN<6b zT~CH47O~CO6SO(;J+t1JEIc~&u zy>zEl>E?RgQ=(38{X02ZyMZ9mrmVmuaRa^q4;B4LI%cZ!ka?lh{YLxMQ;n>-0?*&2 zhnufTn*Kb5NI|w<1vJAB7{KuD*AtgP3_)GRXXsCF=+uE*v#|Pj5@60=%?^Zb$xQnu zt4*zMFOLrQTMpztav9rH&ZbPQPC~~-l!i|d_mhN|it_xon@aNw43ejBW8S4TW9Rbu zSQiwSn53$Bx1KpmkmMuFyguR{XE*nwCMdIfkTxgM>PSf|x$Kf~=P|-w{L+~#azsf! zt;7qD6k5^2O_tWiP&4ErCy6Fw4Uj?ka)&nN0KamenKy31xE90#FfkZH*IepH;i za@TB07sRytFPI<4 zKKK-iM%*KOj*JwL7nw_jF_3VXkkl=m>MzTHvf_nn9b_`Ej&PG$ZoQVxR5C28p1sEP zlL7*4osHUDo=M)UoFG5@`Q_D=-d=dmVB^lVnQiEAj+Q~DG+E~=E9jnc`GK;0->c7$ z(@g_Ps^GH;!&ZDMl*9eE?ZA&zx?i~cN2Q%9>rV3dg-*v!`Ef(QJv(+tmV>nw-e5F# zB6>WSeTDZA3L?EPsmCwDurqG8FW?2#b^dB&(Sxk0bPK*gDK-4femF*KByGHr$zg_L z{R$#wpH}wt;lDT@x<`&@O9NQ+_}v#GN}%u`Ot(d47!Ii0vs!R-AiD~=kmOtDDu_8W?VZy2rWzN#c;f6Rs({mm!+f21N z%L~qvl_YzE3N6|yO>~-!4hcG!OI`T^pL*Msf{D*xr8oix zuFUw-yNTGwg_o$L%3Zi84oob4w+uyxz$p)!$E#N(!tSpifKmhvk4iAyWoA?HFYU+~ zaQ+6aY9l5KSt^Fb}Xw>=pYiNwcbJRsi5u*8!8oAu~74MRyI92L5oE(OeF%!HU)J<@nOucM z%w2R^zA^E@Y#%~7Vlc|!5UYpY$BP$~gg$KfrEs+FxSRY*qpQjCM>KpO#c705uvc zwb;>%gFoiT-7^dizocEQFU#4r_CGi>1@^%?S)cm+u*>WyS?z>8?r#J9zdSt2ykFMq z&w2KI(7YY9tdrTAuEJR*ALa0-Gq5Kamvz{C6 z$#A`N?SofGmui^x#$vqCtu+40q1k$%9%X1<>$6v<+5-*4DpbAzEfc)G(X241P#GQ{wPGDGXewo%i z1qe1vBs{ur}2uBaI0wV_u&*4h8mYrNTPJk@v-RL}|CD z@j^%z-swI@{nxaS?HbN(o)=q^PnZavgf|47*jE<;j4iTUn@J^GxVp5dw|Xa{Kt5J( zu{~t@X^j*uR;($Qn<%hrrZWMREXV;ph*4V-j6A<@fy}mElWAtaMvB1igPJ4vJ@4wC zLR1h2v)wZ%OLx-pPucf-#k_tpH}33Ld0Z}8m+O8N{~f~TfH5B@A+(K>MDuKd%^W?V zE*<%L{HcHWDHE=N2Bi>P=u9iA->_tou9Ca*5;yUVN**I6utiPklb}Mt@3E5lK`4_X zs=908y7K|p1$o?cqP_Nwb!n`v{WoQx)h!WRoI@L7aQ&;#dSPI~gY{*=i}!4V8MzWc z`0+Bwe29aYXxEW@E6yR+k-dogGGW1mKk6B^8pVnmB?C}k5PA=B9%(q%=io<*x?QqS z$ONDID8p%`ywLtu>63>(xJpQ#biYPAY=SiRAm@8g4H_oh1Un)&f;Jd1WHAFW2CU2S zBJ%3>J)~qn8bb=>)S6yZ7od{x2(iQ$xq#?oEcl0x4G^vmC~Mo|!ZBbeia;6^KTxG# z`B4$i^0+sS7@F#SvEWDlWsagftGjYW5xzxTIp}YWR;it52H@k8DDC;+)I48==f9a)9)5kfmLqLLo zHO-iUs}e6&Wqx%fZ8>k5Ut4u-zl67bNW~qvaM~>tt9m&p_&NJTn7nz!? z{+1Bc_=%*hGu~ELE0thf1eS-!mv6{=MTzNf;SZm+@J)OJGRjz-Lf6JtTf=1~@mO3z z-10x)X;p^0nu%2L@qX+G9TLwJa~01K_r#vk3rhH%#!NV3G9a_9dliRKAoI|O{%^&^ zE)SfDNz+Oebcj-}(?+Fy<49eSQk^`S;ZZjFCXOh+A3C_m7T)0THyERzt-__ZmtU*d z4DjPLz2P+tUD+z6?pOq=rm;l3OcSh?0V*Qs@AHFB`Q-OV!g$-71)gO;S$zixJe75x zkY2JDFrm3TpK~t&HnzTw<^T>ca=;GI@|dQK-4kvD4m|m1z{bagk^6umd}LMmQXx=D zdUX#IkZ^yhR0@-tFjA5BZln3AT#b5TJQP^_!V_2~3YvTd6mThExNiAf)aK1&$3!^G z|2@lCu#s)jp7KG5v(zK+2@ewdWRN3872jyL3p+g$L)>eV@oGXVhJ={YATGaL@#8n? zTzekzIFu)2dsNX}&&zd5&a2Jqh%o#%f0->P6C{y^PgdGN&^{5#&$XNTi*Ut5gW8U9 zs$O*)<|_c(?Yg2Z$rPQ0uyNsIPFF^#w(Ia@h37&xCXH-wN6-%b;qTJ?rMNNd^cM--puL;*%7O<8bHM#G9@+z| z@`HHC64|LQXrX1{!521pa$-Ge0+NMkmpABz<*zC}xMM;qjh89iZmQOCo))3@H(C;R zrd=h#gR=3pYK;gtM>|+;+cw<(;YrSflUHx}UM?{ZB;aj+js%QM1)h?lr7%QQ^Ol>T z!ls@_p9rF65Gy4{B+J0hh6Lol$lfghoX0O`O|w+AylS&%+-9qwVJq*a)nNu3zzOy} zqsly4cbPsL=J8)DfD|yjbQm(Mg*#zb%uSM3r!&fKs6dcBtK@qLfYx@P@Gga~i_~9u z*mHA4t16+4dJCxZqhLS*7YS^r)5fphe_xjVH~6Ut1pyd2#kMrpZut?yu7AQ7xknc9 zp-i!vW#~-`m%SKB%z4lM{v0638ovL0j_F~i$r!u%=h7%JH*AN_`jt9{qwoYMP8@Y^ zn$a-;BW-?-)MQl#g=3KGV}ud}o#%FKIU6q3d$B&1cX~epe)3|m7&My?m|rybP<;c7 z<8CE}PZ!NnV)tviGRagK)uP3CZ=*kYhujk@LGvQ7R&b#bVfQrNjxTjZr@B`^{FeXy zXD@(Th`M%kRhfBP&@a~ydt(BGaJl%4*S1KF+2k=2+97|XiYFJ>r%P?gTI!oi6ce~> z^B?A{gRK=f@`g2LF$V@!k#w}p-34ICa&j@%z9b@ib@c?m?mQII!x$- z+Y{&8fm>nNM4QEhZ|7^7r9ZqAXp_)Uh1Vdermo{=F!HinU<${Q{a@EWNw62lq_zy{ zYPphZ4ZMXJaq7{3fVn14F7IklsgUzzC!jrvs7Y551=>rjwk4OP&oO7LZV`zcbY1kf zHJxM6H0AOJ1-%TSWW04cp|f;>Cmt&Jk%LH;;=$>DjxOe;WKkm34-kUO63H_HR^Y9x zD%&ZnbzjVTfWNx4_gF((ms?!emCx;RbC+6@6>}-FhY0uz1WP+o6~D;E{uwHUx8J!S z>TXMnGO7V0)L3pE!4y0ms?EEz-rx8}*#A3O!tXr1<=f&;TosW8BR%>hB-j2+gs%{2 z?@!DAn|6Rb+P}R|kmr!m4*F;_K3gjdt|ur`)9YqW9h&_fH%SF+#w*61^OSOikNiZ@ z@PG_gn})_nHYUxy+0tDf*Dz4Tp6LZrrRBc+=6WoAW9Eac@%h)T^{ia?=)`yCoU9un z>$zVp{JKjxeH=Ka2LbjBF(bMC$OT5Dg|wXT(l^22@0Txg+m>d8-BG+_l{FtoUv zfuw!|j{RYcgkCij*HA1I)3o@!%KqJr2i1wss-UyaM79*-)z$i^9`Wc&Sg&+ml-i0e zi<5KBjAx{`#Yp~Rd29?Uon4vtSCf`lt(-YyijfJTXCVt*EFD#BIq|SC`7PC(QFI?uwi zqhcixAfz1;jDfsNlf@bPEXN1kH z^6rM6kTl>B^H#+|aog1O&xEy4t*VyKV9x2A&*5jmhkPJoK@Ivrf5W!v}VR3=X;^0vCw=ud;kB^ctMHmgLB1 zDp0;u;q(OvY8#&Oc`j(#olN+}(pwy{;MSw(*jWM`=*3#A#0!+8h#EfT zQ$-pvgkPDaOZ7g5((!J@Uzz0X<^t=v-ktvZ91SuQLjWbYe3)^UTBIB~8(viT7MLQe z+)rxPamiTimp zC!52FulHh+lZ%{EKe}ym)EiWqPMXTL7?Gg;*LSTCz;XqpC+Q;16@5eO#BWF*Mz6$( zTBAQ2;_L#P$r|w^bw->g7!P-?D5u11m`;A4p~D8XC#%2bV)LLs69v{*I}p{sT3Z&G zwld1=GQ8|dm+&Uiecej&#U8tNiDC=|9%6Zo6IN*(0RQ872%heXT8$~e1+EeH@c6hr zv_Ex3rHA;*NHAK@%JxOepspXjsedVzRl5lJd=@^KcK^uM74J@!JSezQ{0@Pv4ua{6 zL9_}VAPC3HH^(=VR}E>&&`bW{w+{7nCO#b$h$TmYDjrFsr!G_)sp&Uc32pRUdXYoP zPl@^?S{@G?7Evj`pHF_>`sfS9>kXLnB^g#2y_V?bNp1a}z;S5cF_XD@<^9R#hcRmF zl7y45gi$my-!aSj{=qcd=@B8sukLE^9~LkiUK6$M>Aq$rJTKBoU)X#B_JXhZj~eSL z=OtmE?F?@pWq0j>kDQ=haIhKsQJ!{R<1IIFHvea~&s`thTBTH)mh`b! zWlcMV@((>voH@|VNM(F0=<;}aE_uj0!>^2y#K}wOq)IhD?x(QsI+YY}T#`}^zS@}} zPzUZR|HgBJF{K-v(R%2V!9Y`1Q{gN3GRf09XWREqi6 zxD(dTKc%51gp0hw`-9P>{_yeeb+_97>qn_aN?_uIQ<}`L35JXF92@zFsQoeyB` zzf*$POq>zfKPSg@Etf7?_+IR@>*qhse-R%ZI9@A5tlu8Mkgw{E;!LEDq%3NPKWec8 z{K)Y-KHsy@ORJy1)iVus4)nL<%k;97kqaG-#mp|h*(bnEZAX#n8A4rT48^P54p%r` zE?(FIwQ~^FT!Rz&@aJcUWCG5ZKxTJp`HFW4q*_271Yj2dCEN5_XwjajJ=Id^LjPym zunAgm4NaciH^mqjSc!W%h+{v;0N@|FVVrn+rQ$cFRbi)0+u4^ojJ6ATo<>$Axlv}H zVl-~e=)sSwHSeCTo^4#w z76b;2j1UDk6M~B(sW$=3MlyIptGGxruECqb*`0q&K%dq8L4&OSCO`{znDa*4t|~We z!W2Neg1fYkO$~&0zDMT!@l-gv(1$4S3O<|Vorn6gcEjQJin(@W$jR>~BChr?m60tt zP1;a_CuIowbY^Jf)4>CTiUDgx2)$bH($2k%#mEGq;9FiatNw(L3+q8Qrn=p6v>NJn zOaZ}s#Zq)=FGmIilDW)6mPEWQ=og+}y`2AIOy<+VG2jHd{C2TnnMMTs5=C(8%M!^p zjn-S88@D_NCm@MbNJD%?RaYz-L1jXtm$^t`*L2_N39UC#K%RUja7bylCaS9R4Yi|J z@UXsHD>3_z2%xDce%tF@>U_DDF(y2nmqb-|bc~IODzA zT(+BXX+QamV(d4%c;zfkv?4Rb0nJ-nmXzc%AAz^=B9Ur)zP8MtH7#j=0H^7=Z>L4D z4HQE-MDPWnoVRs)Aocrv0Z|&nL?Z6n+!u$K!tsd85#vs|K_VIp)|%(E491SiNC46A zb~vK&gGHos@n|)&uR!gtU;PrP4vgodP2HT4jC$wnDnCa>BIf=0gPGr(+2_gu3)V$% zG$749>)J1n2>O{?#PlsRLC*~~+&QWw(*||uchVxcID@Pg@pAw4qc}Q4vsJL%=;N~j zd0mCvR&gDT&&1@@@8AHNWohzza|}`LDRdj2)a_ieMJDPZyGIuxZ9Dd>{NJ8|%bI#2%g)mDDX{{Ds}*+Cv?6 zza&^hXF2oIODl_v_dSvQ3aS_y$)oGLS*I%JP5Ud-FXq(5KzVih-QI9g{ zo*^7=2z`JW`N8Qr3x^-mB(IG=XUYCOX}7jU`{&kVt7sGvo{B`+gE`Drr{vuQE#iBb zxVzmiSum4kXtTSmsXGU~^19@}Kk`d%w~@`LtnLCF&i6MtZDyZakyg4lBE^TNxbbZ| zvi%Dal3HZXw7(-4sguW5u72V`b6rB}a8cs_R?fQBCYr)sY%s&+=7a51fnMra+-H+Z z7MM_T$P-tH{*MSVRk?^SRf*}ji9V!g5XdmUzC(c)<@lm+MiB3R{qA)6JH%B%J;io7 zSo8p6fqd{u?sG^rB(4+x`@r)H2--=t-X{-Ud@=5St$#PKhGx48Wxo$PbDB&GZ}QL4q%c)|RnuSxRq>?>rT zjdoZsTI0_opNsuVU75;lJt&zcA59Ge+o@aljTI<69dZwk8F_~`dJBNd1X1#`x95%D z>PdAx^m?^;Hg=f6?bN-jp? zl3T(_V#Z+pie@KFj$N>Sm7;R;C!Fr&BDDuIlu0QLmrNps%077nOUX*RAl92@8~!y8 zyKxgPztj99*yt8pRCrI&$fjOy$TwGGjX}U^Ecl3QUH)y9WbKhm3&uOpnbIPMajf`c zz_ty@=ZN12Et8^~2m`q3b$_XR6EY9zC28u;kU56=Ec#^$N3WfM3c7pS=!+0C)z#W8 zCG_rf#Om?V7xf`R#mrmognmFyY&Yd+?QA8Q#f8G52{}&D!DX0JcLS|OShwfS6F$gB zM~~Y&cmC%OTaCqvycE5MaJ&m@2NQ6w@kTC5{YO;HCF~7M81X$m(!)g;It4Xd;|4^e zAsaVIs#A1Mew&ouwmp*^4DJ`^^k9N%{iMgxSdQ&*04BxHm<`?NRBXSzMcf9-S^IIOwtOB@*9?M$D6C8 z!9)@-8ryuO#Ei%M)%lG3pkT1ZOl;citi76)m{BTWwr1?I+o3(?>-rh1B<5wI>VsZn zwi7>mMWZSId430Umx#J$ok!E8RL8KNx#?5YqXX&QMbxkMKc@Z4Wx^bU?7g63=s+A4 zXPva<>w?sl9*0GIuYu6uCvr$>d zDLskFfPF_wi-=2Xd9w!>RHHM^z>aGR)6#}X|7WuUk%?yX=KHRa zPb!}|(fR(h+22o|)OgL3>Qunj1~?(;jKy-$=yLy5^OrMyd-_VjoRZ z*O{TY@PEfhSMtmke@e(MY_xVpG7F_ng$G2<#7~JfBi)`BqqOhxX%rt5Fc}3W=U#qV z+n`k?77}I(RuQz`hYmzlz1n8^+;NIaAr?+8z^W9QhAf&X@XRAH2RFaVZs5q3s>u|G z*Ksa}4(hf!=-Hk-{uAgHfXYj{`q&+&@(Mn-?UN^V}z>LtOsH>>>Y@m6f zLn=e*T{+~+?nYQqJ`ows53M7x;=A-jAskvmu$^v*7&4eNHy1Dl^0lsNymWeFB>cLJ zb5f03eXRB7z^o1HRF>GbF3w0SPggA}s4ca8Se@zbTge2Sb1AF0GY{H)&1&HW`64O;pWX2TW!2*6ZN`M*#Gn*yt4QF{Is5}T=7h| z2!mjW>dJQm@jS&(R(ZSNyX0SydCa?+P&AxM)?c`)^1OW;(VUvBJ0S%joK>KU^h4Qc zb0Vl2b_BjsJMwSuCXE_?>BLNnRUgN+|5u6v8tbM6PUjz2uff))aa?r(o% zu4WPi&@08EnzE#j*(Z0PaN8&{y#p#7*9@x9{7z(*7y6(DXFuTsRqJ0pr|*Za$5Ou; zd}mJe&kejEZI!Q}Y`=Qt=GXN_!}Djao~tTD@WM%aFwcDbWYCu;j&<<6r6# z;=R?gY;I8tWoSnf9P&tMYw3`@H@|x?GyRw~P}ybmO9@j9=c&fR_Slo>3|M+Dh;FjA zWZ%3v0?)D9eDEI3S`ci<%)+(4aQ-s1wW8eZk2&IpoXF}H>9X_WE5@ex!;T~k&h6Ql z1T;?p=QA+e!WuABfdE7FlSTKjF3=T*3P`6GBLCi`RzB#|(on||9&iZs&>~un6^|S9 z0kmOd?eX06=(ig(#(rw(1|W`x%0u)E1*hG0YK_7WRV@zz#xAz5*%$|onHSr|J^dv8w2HTfJpT1=wwo~uTV*Q@`^_*vip8%5bzNzM^kQL8jgN5;OS)hKg*W{odUM zD#Jg%-^SBXa(6?3!+iG)Wvj*H<hqm_}J1nTH==B|N7u9iCp5zrd zT%e(w#2^V(^Gg%fM1C6<`brx5=@&5^q$6c4q+<-RempjDk3`vgl%3@7z$8m5mR2qs zC{*z5Ea^!AV|bQ&{Lj~d(ZI|CZzr$13us5c7LTflOK>b}3)ArC?ax{gS;+!kPB@!;TEeGe7nOR*l;v`-hRmxi^Al z!f2xzFO(nR8ACkmT>4Dh;+KY1-5M0CP=Gar18ksM+uh~DeJm}WqrW4p@0QDv@%9Cv zp#Q$^Epd=j*D;dc^TMuD6MpnH2e04^tu7*Jzx#IjQe%sP;g%ATpw-u6do%-9Gr_n7 z8sJRw$~>}k0`^bh#HxVK1V)^K{k&0_X+!$ri3n^Zza#!sp4@+B5yBi#akWmTYT)6J zo#K+8*3D8$IwrT#Y(FKp38>Yf?*-DZw#2t{IlrDSO%-cd0N22(fIp}2ASpnc^0&7| z)YZWkW+MC)^o=@u07;~I_Zze}?D6tVq*=pl`0Wq}-&N4gj5r@y9CUXbL#Oc3q1sIE zPY5E4Iha194w`%wGlVZZ2juZPJofNtt5w&VdK?pxjREs zk}!_(0wOQpSG`J8_3D>$KIh{Kl^m{=6aXrJGhgp;CXWET5Wiv%^uG-BhuT)ztI6}1 z1pLxcXI_x&RmD|~gT!4`jizRH)<$u~Zz}(cL-JG>l7aFb6Da+03nGN~K<$jr4u5uKEOt!AdDDMOtojbfeBA@>uvAvzYgYS}W z8FQo^BMn#wGWzEo_5km2FVY3lAQ4vv86A#zTfF zV4ez27h|Tu3KafX;cugeqlh6?`oL*zs#FgLjGUp;e*}zGLRhp4AB|IXfxW*vbJ_Wk zQY72#Ymn%)NJ1tow9-Q9Pda1b0&e)Z)@Z-k`f^+6+J<#vB8l6{KdBm&PUlSvuHbc* zn@`NQAYPOY?_?ACQo)}vWtH(1N<>6ff}~FqyMum$&ySiRofV7E;>&Mk#ZPJ-_D?tI zMgKoY${j#t$8iqWHuYuP@W9v%c^_o194-N>w@R!C9NSW0OPldKU&m@P%T>qR}XKsl> zpw#_fF@_11v9Kipxod)sZ}_tm+*@`)Oi&844ILzjQDIhj%hT?fyJ@CUdFYP6)uvNn zmynG(gkb%d9o>_`lqwcOU1d;TIU{sl76~7s-ESBCD2jW86}1L;NVfVWWmW9n@ zJNn_>e;rob2QkTU+cD8IObYRUxyEYVcY*8C0_MQ%B!=C9+l)#9w(pww#Gcv=lVle! z^Vz>~9-##3mDsc@fOZl_juE$Zh?3f;w!1Q#@L@r(Cc?hxjGwHFTm#{fj3HBmEMFK5DpRP;9bx zDO8PzKk6?3M{Y@0i8Nyu)97H*F;V3k8I*6epUiCjI&1_2nqy(-Kzyb*6T2Y37g^Xe+hu zBmV`t0~5Ww;}+T$2a}~~xmnLH9gjow%lfcTB;!|ja&j~1_58V<)$lW>chSWge5T=`r!Q2^*V2Cje8*}Yi`qT2`GjRehc!P6- zdIUpw0RnS3iw=R$V9e@x{xmC?MpFxbZW)*t{O&@dOX&q-edt&B>_fKK%JzFviK2|m zf)wRt^M;Ad4h!)ON0)0a#9Ti0JoeXhDj!@-A>Q|ym{}6P)fihiJMGQXF-p2nqF`RH zwPE&64dukR+AJ!7Pn9@gT>2t{4hi1+sBF^s5gB6QeRwfkk6@x3!l07WUyy9d`u6Qx z4trc8qC)()59?1JaDgO5hAo>AimnbH9=zf?YjWC&h@%Mj?RHvpnSthNv=xUYa#sa0 zUddWhA+{59lgY+{WvU3K%fFwr^c|Da(qYe&i#9PXwF85C>l9?jeG`vIR)?k}BCi?C zg3y6|@Ql!m#+WpCh^tn}#Dr#wuxAN~uYKIV4cvL|t}o(2iW&O+%rez%q7uBf5YCd| z5gV;#TQEd6_rnv3j;IoIF()i&nP)Wu&J16m(ca-JpO<4LbLn+->y8|WgRTHFO(AvOSvDn{~iRQ#~s~36D9Bo zcW%+2x{NKjeC6kRW$T}krur^uYEwRh0js-VrR^K8U< zJ#}{3CR%Mfon%dx{bzj?8hEV?j|B}VaNMpcg*V*RR}jM}57zBeMcUij%de@?WT>!? z4-G{i6>v^RANaZd?j36L#f;IWz}G)mW+h=XI;5neTPk;DU`ZfIZ^&cWjT)Xix9l=T zMn?Y9kDZ@4WZ~pYa$agtfP_lBEm99>HG?(1G9a?>t-oLcP<+fe+lk&363 zAu`ln8yOr6x^6y;cg&^TH5lNOHt1R(kzMAOw+1$8jV-L7z&2AUxA)dJ;s5!4h^%)b z5pUxV52wdYr`|u`Gh^m)zFaQkyw|r`vR!4C4E}EXW$W3ipR$0qc~8&Cm@0SOjKLNY zA5XW?;1nPKLYReG8=8jtsF!<33WC(~5h36)1)qJw)4F&CW8?JUbRjM0rnT@2y3ovG zT}d{m4bs)}GyhA+M5b!nu}mlDm9{Q$q2dBRzk*z=LL$7ZfpZvQ)9$95Df0ii4RCm0 zFOs&XnXzPxft4t0*vEivh_@y4h2T-HibyjRCTx9_-szKw95ggihoRm}p0kuS(H*dZ z072Rr@f#ESU4%P|MxQq-AP^g{HWq!qk1lyFi5&BeB>@H~j*CEyE z{d;ad`7LKA!wyKzbHx9A4*1l{glVt9oM2=~hq}(f6nG4FaVO|=WQU)xpEdRG=f4gG zpDmj(8^NXRS2#L~H`X6}6y*tVM~ z88(p;wcB@oa$mH4=<5rP2FIWW>K6%T%%v}t1W=mv4o%&Ki?;}1cS`FxX@N%w<|E7_ z9>z@UU>L%eOR}M#+=W@B0|RAkHa?@TWn$2_6vT3UH0Mo@$!ld2KG;DA|Lao2a@qC( zMcMbYNHN$R(!B$JJTWt9nw{Y5ZwfmTL)?cUM9kh%R1pFMP%6l>diB{ns79g(W4$*M z^^kk%`6MykSwq91=pK|{IrfDe%56&sPZ|*T(^M-uuI+ty3zD_(Wz`NGf(zkOnSOt2 zbt;E@2Y%UueJOC>b-wCaXV%r%5Hnl<*Bh>rY&JR`Ktj~OA zHKfM0VGCi51iSP60mjDD*r^c?VTbSEzkg!^4{XZvdwY9*UJh0|@k!Put)PqQ&z?IK zH}?JOR534UMf80%RLQ zSHIRXT3x-4mrXJQFAbV^a-*;(K|2B1^BxRHZtm`zAt+Qa;Mu+FPe(_y(~u!Er90?U zKBjs`Tq{9MM%S1hli@AnQh7_tOOm~xm@@GZUNUaBqkL1E>6aboRkLy|dW6K+9MDgU zG;5-5ShY5&90D_zEYZaz~AjL$>!$&|*{kRt07Csx$FZ64q|vat9Nod zmfhPYFR01Kt18mCj((R=TY1dY4mUCH*ECvM?u2%(jBd|B%Pspeu1CjR&;N{S zg*q*WH5tQYpDXM;L!*y{?WVk@6a5P4S|%KTw_06`yQB?O;W=g>L?O39GxYd_yVo5>6e z404>nY7hJA%N1-5kmmwbPb3B9$4at@2qUqwwgG1gg}m(xdkfyMxzydJvl?7jXKpxFDSbadO%e6X&bO}P z2$)Mxz9=)|yxWWt{o}l5DzsNd<+ed8rem$=e~1Nb;&%~i(tUrrw^1X36ZgRajYBOv zqoT$BXEdkSzp|^ZrFC7jq=qD7ouolzCq-YbT+|zKiDxfaaIrHGIdFBQK1s1b-x5IK z$}zyuwG!P<1@x1iDR3OE1`}BDserbyuD+B~#H+vW$!qpSqKu~%c~oTC29*@>`pcur zNMrnW`YfI&x6<_1rErH0#0tud%j1I(OPRccF3WukxF*7^WW|{sW})LPskj6pyQSj1 zsYj5d37L%+6a~fHydcDbLzD!g^9kJ_rz-UnuQWr`?^^HiDWA;bip@spN`AZWQ`vqZ z)@l%Rm9IFOlFHflVz0cUW!}Lh_&a>U#J`1wK91w(;VB=Mk5( z4-I(CptJ)Gk9eQgi%Lu5K`js&?bJHkz-5yUpUR~@%?~S|WLUG|=V=PYiT^2i=GuH~ z+b-uF+(5chbaUI!vC*~pGdnyG55$UI4Ie0kCQ6hem?z;z{8HY#Ic$ue%qZJQiAgi& zt*Yv>A^GnuJc2$SdIPCF>Q6e&MQ+}~?g=X|2!KQ&KnmN_FA7eNZ}<^NkSs(I86328 zeUCCx9(A^^re;9L)qglj3Iiz=%Y)UJVtKOMk2si5` z47SSmc-CXpI9QT)>w6C*5$bRsmC2Hdjs|}p5X;L~C|pC@Mu$#Z%d0F`uvX_eX&V>i z7b>VY#=)18`Ze1+2uW*)(OW{=-YC$>Prt&&Kaw{S*eyo#He zD!YN^M>Y>fqI!ul1I`fT^2eHVi*`SCDcwgFOBT!Gbd0c|sge0+UOupNcA_#@zz7ar zvZ1q>L3!{4AqXW84%rHzs@y@>*3%mT^LjZNm%P%fY;34zJa{EB5I)#x<0^Z^bsxwy4MP;A?^5=3BrlS?@Yy^P{F_ zs|t?nW`hy2^eb!f`j~2L?LC)@bQMcy&c_)zGMXQ?uZ?MZMGo9Xqj%lTEB%$K=tAzt z9+sHck~iSr{fG%G$GbyqtChCed8GvY-mmM1LwQqop)FT&YjlzJmlDkchnQx=y(Y^v zVKhD-B|VFnjP;aKvD;LN*z^`k!HvUa`RlU8cB2FP5tBZ#dtg_?=F|0y8g!)xrhG{M zOfT0wUE|KtrJeVbfX_j@EZt8I`TBj*9h`&veY0r^EBR)e2nw7!6SsWbGT;36kh9K`rq#~olwVuuEtCa?nZ(JR&*k;sH-#(gRNNG?!&2)L!VA( z;%80pqc*rZ6%YGbJ#Is|8R$nV-hzq+U2cds{HozmtbQ6cc7Qx2YqnncE_GNpE21lyFx-TIiT$eUqorwBi z#ZFrIKD_>5wJ#hEx>(AVScgVKc}Z;4gH?NEr;J{ic9KhqST||+?)WvTKSdTxZb$lTQ;4(cdP$)CiJN+na?bF=uMQ?W*ys&5-vZJezTw+QF{wU zLzoo-wXU!id_jcH7Ld# zN3Gm;o#DAmri&(w#uIv*Wl|(kEGhWnzG%Nqj^;kN(ieKb3CpjcFm+5Dgdg`Rx!;LY2yR%rk^Xa% za@c+Hn8AoCOOUf*=5weTCKEauQ_A&B!;w*nJ%~zNCoznCRIHzYHiVRvq5{vpq&)} ziCN`jUjBp^JZ>veYe`4o9{{%%vv|Dkv z{_^_HVJ$}&2bGjoZ%N>}CVYcW!0n>VpFdyaPWQ>L`vo1%%}1TSZ%Rn2gx8Q!K~~zn zV&J~2#1v;WtVrdTp0*@m5}nJnXqsH7j=?d16?+nf=HyG>bq z`@+GA2RUmXa$o8Fk}tT86e=SN*CIeFKf`g90%$^u^X{Pfw}+R^zr6s!jcCLYO{6}A z*wjZV({GOjSk$UY4r?LS1?wY=p_WU|YODhPsyMcaCh3xrlBs+?*afB%c7Z8G2*7Cu zDRvW}>&gB7N%1X47o{?S?YkjFJ~Zje$dDn4MUJetDh1>xlpK1CJd0?_Wk(#|aPtvk zj&V;OwAe9oMArpUPJ6!iTaUB(Z@i0tJS~sZ)xT{`l|D2$<_-_b=vC+F9ZQoZtbWhwsHw+Gn@?~tI^u#PZbLHb@nNz zT$05!zclrFpcTz9>#a!0`VQSSL9*EWe2Lq7rQsftX3p!q)xfvm$ ztUBL&O^xT_9^7!ba5sp^fw}e>vw*_QL;11(kKmKpZ!XTsn@@=P(TMX0i%*t$EGV}7 zPv%^fhho3EY|w7-n0!ogbWG}W7{Qspws*#smUEaYaN7ZM{R}5i6lk@lP#>&5HuF@D z*e}f#*tmcsTL{4l*4Sj)OW$ z3P+j^>PY}NoQmrS6+L56#d&7b&+<&8`xu4zAGhnhb)^DB6P@@+Ohi|}h^OXX-^m#` zILb#35eq_=Wjp;CjJv`FM4{q=zNPGGdeoL`YU;s5@e~k>Uk<;mhqkgxHqRaWUMdWm zp8g?U*nItjxY+f3Q4`|HaJq>}IW<8&eDlap9n|985E~@uXZ6?Pr3)K3Z{<#G_URNd zmQ+%NjDzKmE5TVwxLDB{Jf@;L7fmE(508K{$mH%`+X zO(-7Vmse(>8#00wnqXQezvb>kC^kc;<8xpzOGy^=O!Pu^BYbG=8C>cTqo%>2>f_L7 z@;!zx?LQko^y3xo_V{_V6EE0zmXsqyoOCp1aXet}{G9Og_zGYpmG>>)FxYAgh9+(! zL6;nmgkuqgEx%I=HK$1G{lx=)^-fk4dJ_%=P5V+HbOu1t%M$@v9o8yhuxo9PhbT;n|Zmx&!*>2>F3zU=*EWv*^&z% zuDGd?^eQWlW0fqY>FbrexyL6YM-Og5n`NbUu%h?c7Co1T&r?&1zt3A5OUaWFRK36o zUU;VdL=rt31%jHrDI@z#UH zr%gYkj!H;8ok#1V|A%} zP-2(-YumF2J3R#-p9*s>kqyIkllo5(bKEwLUgWi^Vqf#q#hB)f(9q~50@YxC*Roi| zk+PvzcvtPeeq*6}J0_54k#LIfM5kM-EYH)JwIwn=u9hiGnH|z$a;`H^L_AcdNq>Iy zz~R=z5foU-I~DaIodo9-kIWNOqo)qFx51(=qKw7~th#FnBk*JkuPPSB;@y8b*z-I@ zQb@KaHalNK4mz4KL~@V5?L(SRW@x%at+#G73~3_Lo2!G#g2{2nvQud7f^ArHvKAD; zb$VW5`{Vo&&zLCPC$Y=Vjw>Bv)P>pZ*+~C^JV)RSZEOma5rVqWm^>))&Yy#=1YgN) zUi-reSOderCZV%oo&~Ggx;pL8C{Dfa@bC!_orQvWSrn3diV8T=?mpp zUE^EZrGpF;FkD#4z5Fnf2Lvqo`ru#9P_`tb!f{mlS3S>6EinJQq>JOD8=k>@7VAFr zF{T$HWIHH1O`OIZa;Jc*x_IthfGNhP`#ob?%g?H}DF3QJvHPp1P9&xnznmXjI37_7 z90v=B_GlBoR?8O!6)|(It|QhzKMyl%9P(7*%Q0NY*>fzq>^**%ZCWyoABJx)WFf0a zyEP?Ph6N{NYINVebV7nm`49)ME-DBpIeF%#`)d_`5Tuzdmc^D_9(uhTEPEAwy6_r1 z_CpTZsKb_$dvH>+7E5VIfBc6vw741KGnC#9Wb73V<~Y2^FE>xWVkXDklKF1uT*Niz zy^zoNoPAnZaQ==!1dbtOdzoXC`?UAS-qc3oNR;@fG>I_E-FoiV(xRcQw`0x6{q`Ma z@>&>Iva9;&`P*}MTXldPj8?txNRxT>s+xd^XfPNaF<@4SoZTUK2D#cp2Nv<%kx~)s z024phdfdSVOz18an;Kxau!j)R^6Q5{8qD>N=`=Jn&SS54CkosqRRsnjo&+%mJlQLX z0EijeHagWj0+d9Tl#D3fT-(4RYz{A|zs+iRf7;fy&C_;7Sv8pG14Is?LS1yB+Q`}t zezA^Taq~?0aPPSi(z%-$$z{{Y9}e^&hC*aZfuW~Jr>rXb@N;CVd^1K)sJIBntdFyG zk_AdkxE^atPk&ayIM6hm3Vu4NxXF=>jB4v=OSnDVfAh&HhSI^Fp+wPJK!^~!?L9vkJ&2DYr z33P#BZCol2^RQ{x@P1SnEo5?~S-`gB*2Qt?50{@^>*!3;#XaWl~B?u4qzrO-H zo<-9risPH>Nz^{za5vu3i7spomWEAzhs9#-JV^jo45taG9kuLp0xsm0|Keh>Ie_K6 zFVCg-i~6Qg+^v6@_Uv!}!-8PTA`&yup%4Aqz>nKx$I;A`lEQAueQz?BNd3|4BY(L3 zR+M~j3iS`Y`-G{fWRm2QZ_5X>%#fbEzg&oFOInD}TZe>Fh;(h-G}5esOu- zkjm>(XHBAbf^cXb32A`G`||^0I=pA9?%sUGiw8(zvs-r#@vNJMyg$F@*P-tIkkCbb zlE}WaWWIb-xE~!Lxf~30ydybLB zq%}ao3xy-Weprl~Q8uL00?xKQyMCg?H{z(xYj`N5#CiG`42w)CV8V|1Wx%8TTUA?Z z$9s+<2k%eVx-xtIR_lXGa61>(DDiSHm9we(w^uKmLo%D-@eVbb`jpWAl33STuN+ z17gpwy}(^?Z4{vX&|4pS>%md&}nesRxh0MP%)s`kl*U|zo zsLO_wPE5G4(}Jr@l!d{M!0EX3zf23f(jqyU3P(GmGx=3>>5VmlzdaN(mt- zNF7%{F$_8anr|;EUwgI+U_p5$-EVAuW#>Zi7JOfqlH$Kmhjw;; zzN~ZGFL{fP-%d^tm4=NpBRZp>%$o&X=nM=F+85hPnt@7wX2=qWs?6cKVQOLh=^@}| zTRnOBE_HGKO%`pHPmg;cNWpMv3qk~MOfZQ$*>1AXLW}f(^)VcHID8eVAF5$UMLsBm z`N3qVM7V&tDhm@OWHCKhq#i|hT(@eV(l)Pt4UY}O(XCRkA6KH)tB*n5vC z?u=IxX90`S=uZ3s3yQ6!xbz%^O-ODhnDVp%FvUbLXdqk0KY=eLft(A_^h5*y|Jfpx zCjsjBzdpTw2bg#wUP+PXG@d-*jb}$9`2Rji+|KlWQTRNCc6WDIwZfns(%Y!MHHnIb zR^75_F$fb1u>OA$3RLJ`|r#B%63OQumCLg{}WqEmQATi7AOVFR_gFa-K0cBjLBv} z&5w|f>LJn1#6W<@{^Q3FZnuqoqJ893Ik4^Y`u?vM!M?QvlqbAqMI}zKP!^%R>hk&~ ztpLSPUj94H!e6cy?0HHgpfh6x2QBe-SM9YuAI>~*8p(i*&a9%m_$=&R!M>1I68?I= zNtra(Ihp#l@4=Tujq}s!DYXSjDH@qhHKMR^2?ZoDyVHR2W!=(0>92~7D21{l*~t$w z#hc%+UE?0a#s}Rtf{3@V7CpmtbrLqBZ4GHy1cNo^+M~I8v@4(_}!z1u+J>&ZX zyk1LO+k1OuYZWCNkPzfy5z2<;ToNQ6nUF0iSjvf&iy~42GMUzJIKLzqE<8rS%+5{_ zg!>v$%M9XFW}!xcbttI<@P^fz^Mgi(Zy6wN#l8z>NgD&9KZG>(7O2sdr0MCk^BY<; zU?gI-DIDL+&5a=*dJa1I&`_N3NF#@QyO>;%-vi{YlO{run}25BbDz&SkCZlDwB&SX zu)cTF)G=ie@HpPU!Owl~+Pa=hp4Ol^n3d$6KE@_iUd#4G*iZzaQn9*A(0N2;5r z0|(_b{&}jjmZ5Nw(s1T-_9bLfEzg^(TGImz!rz29!ts}XK#}}0qw}riJcB~$?4c(A zcc&mqUb|7?dZXVAsgkAG$3tr1roOi^tSESm2EnzlwV+E|q8fWhl*x-Am5zMB?8zF zCTJfcp|`!`*YZ7w-bA|o?344=OrNFBd+{OJ{R?(N|6Eu|9HEbEI}NIo>BEBkz>)@A z@~vI=n$(72*T~u338v9iHxU{lZ;SnWSLTQE=$;JcnziZku6Pk}7guuo?!wKnz5kK2 zv|2yW;KzJpxipbzqk)%GP1gcD{!=d1mdBlKioPjb6&Js%htu*&tZ6~)@#IgQ*!kr6y5Z>8%cWqiou5VLwf-(a5PmaR>gFYE2}T0N zfVx8|P;7AwaTSzZa70o#fJ#mzXgX|In0nzm0-p-Dtg?4VW~pwcH#z|UL4SGUGHYe~ zW$K%7s)H1(%k;D~wxMLM->fi$JTYl}nTlPw3I(d^U3TmIVrx<}5Xw>koby(_mc!kB_kL z1%j2^Wn77Vk}cvgKMOF0CQcjl-ebjZa5HNA8Z*FcAc;uJ?ts+V)NuS}!QsR*{~`W*fbS7#X(W#Dc5nW4L+ zhn5EEk{Uv~yF)-)O1g#yfgz+p7(k^{KtLp=yE{f&K#*?k_&?{|``q)HFT?Y`vuE$U z*81&6{fl}VSFBr?3DbC(3+vGs;oB*f6>kr*2Bsdepz^*jy_!x8Qp{?{l5jOJ> zX|JqLHX{-y&cg?@n69fy7i?&NEuyG)-N0AMuT4G$xJqeNq&^4%jFcxjVHfO~R9huD z@t7%ke$k)dcs=06%S{TA4WeZJe8?X&l}c{}s<76dmV%*are^DN+OxBJKJsZ`sz z%GL6A6FE#iv}?>?hyd~FC}|_=+s=Wf*sY=R>A^ZdEv}(?ov7jCnGOu>i!W=!Gp+@# z5=zWX8V+*(%r}gz)@8{de{iuK2>xqF(1SF6l0uvvcppbE%{BiR&jCgyKfdGs*LkX9 zulQ{yp8ksGdHM9-Dgch`ZJEOwDUpAIRChcEv>-dA=K=y>rz;A9OF(F!gaJ~;P`!Lx z1{4HQU^_UxP^t`d;AMcJ`PiONUT;{f7WARh()}&1upZ2mpPE~|9 zn|baM#zmCm`UiEmCm|8d#pE+hhg7r32x98? zi1XLy5*+ejjXY~>>|zHMz;r60XLw|-$Kk%C=SWSUSQoy`%={UAaEF6^?A4>_MCKmq zh@C?N*k6Hw($BU&vZ;X*CEyYLi2O&JA-|%+fEz6ZJ0ivXp~o7{cLuE=1&G*nb5Zns z#AcTn;GTuatV@1nw7d7;pP~bdpqi;IJe~b|@SI})VRx6Kvv`dHG+}}o5PGIYaT2Br zz0~Fa(n|X|kqDM$;JiJG;a&T3qiY!Y)pAFo4NvUoHT*^k%PV2l+iXXyzvJyqi2qaF z5rt}lnRfwExbs+r7{v6(zdGpN4E9b{WR9hb5>@cX6D6TVK*VM`p0~%8;Gi0qQw9r} z!v%HR4sNPHBywB$DYT$s=37B?}4L9wKC>*VRqkLR>zc5tJ(=-)2P z5c+=2^5DskKHhRXJ#CFa3B0bk3OG-rO7Pi+O7g5Gdc5ul$9cQY^U(jCGr9ZTd3?Q9 zM+N)$c9HmPg_K%n{obrA6neLvM?GzKSSC4P-$IBT{0*+81~u4Kdfl3 zY23z*IGr=~2b42?Gj(4a??zg%IMh2KLsO6dpCdCvoTNr~(%(uaO?bifq2`QrSY7-# zZyfkvDm%(TFK1UI08LhYd9MZ>pnTD@Z$i^qsY5G4C-KSgPlBjxK}1M(?9O(Q^E=SsYdG5GO4_6Nrb&E2IX72*Ye9(^O7@J=|XeM{SmNc4csXdP_40ZW-?LZAo<7#On zW15-sU#cP$&5QE{8f(BFDkdc3GKP42i0h?>Mjgv}%mEQdJ=p7A)#jfTF|zsFf89#; z`@J(*^}p#j7%f;l1ZnE6sA;N34tuE0Amqc+R;2kXOf(@j0SZE#pCDaTO2FthA;@>b z6X_~u)XYF|PlrXXe<}3wfo4oG$@{-%rWc%yo@jpvEZ-jX^+9&G6}B4-YyfznN)18& z3LAB3dD`aq5le|hb{T-Y@EKy(dAi=NKgT3;`4KP1lB%zpd@PF>JkeVF!>05=%*7D~n&&C0 zqvzD;;PfBf!#G<%`kUhLzQ2cMQLvJ)3d={~dsb?$oP5Q;`F>g>muKN2{p4FP63W<* zH8J!aP(r)Rb|dCJvD1l;iR)&IUmv?ih(^%TB6Eqfbh~T&0^m|Z)bTmIm0mZBfg3Uy zHTBxso=5A7!jxWT)IxMRbrZBORZzN#jE}AC0{MMTF;S`?r(ex0eVn9ny zz#@0FAytosVm?KQ#0ln{8q{Zta!_4Z#Y*YmgRY^HzjcI}v=^IaEkgsD`=3Tf=Vy=~ zCzoMU1i9oV3wUng)!GFN+Q|MCUFh$jE@_VBLW=31ZRwhD@&a|>unnIG?M|B-R2XwY z5LN5l!&SPGbc#UrK6k%*G(OwcmsuD`QR1N~ZRS3QIZKMLm){NJ>Bonnjo-j}gGTM~ zCd_6hc0c}GR=n)txP6Q7wKI|;vgRlE&!PKWW68=m#=S{~%Uu88kZ6W z&$-bh2p=5F+a8D!_T51z)!4^gF+p(<8=w=uySzA&ikBB%=WR6I4L8n>vUnR;#-L+j z#DjF$FqHi$wmW;z=%?kHp-r+0v_OA(7Kl{V&LK!KQXNyeQcFA~Kj@XT-`uO0U1Yoc z_b$hyD{MY~GUgB8tW)3LJ67jx?dlD!#E2;U4FPnCB&=Av*n4LIj_yrw8RZ*dzS0>6 zRv&THiuEXtzeV4w1iU*amQ^vY*)2Ra{#K>W`+IKDX2YB83t;Iq>9h{#4IQQL8%=xf zfVdxbCoi{wzWb}$W6A2W%{4n)!~2F;Ki_8V51AgFiTDmYS#8c#1OSC;0l^wJqZ)^R zjOm(d<*WG%SXLOyI)f}zk zh`uMawei{>3uEn*z2B^ox4mpU@X>|zvxoj~D)M^CD=!a}^mnjnvp)RpMf!eplB`6S9)o z6dB%q6({~fGKc(XLy%(hcfAZDtKBjBevK5fpb2uRZk(!|mg!nVmn#n})i3kjr96B% zg0kCxDE+)>*g+`r14Gl#W@GG^WmgKFb@K8RQ2p$ikI`pdxpUyR=#aH>8&d*n0)fxXG zfX+cRcJGb2+_)E!)267KR+$9vwuG0`tug0D%0E5ydBNM2G31(VZY=!Z5ArdaBg;YS zEe(9PhV=qSrGdxNDTCMkHy+x*oBw}mrTav;2G7j8BaeSNZC}Z4Wk(1S8$-GN)=eG6sF^oh0 zLoVvGKV|7(L+3Tgo&rlwiNEiJA=G`!kWaNa4nV-NZyEOUayf)OB@eH0L0`Z~qXpD5 zt#QDBoSf1K^4Go`CgxdfiF(b-Q+A>Nmuf=XwjTM1bEHY$a{CKtGVgReWBAH~TVLPb z!~$9VHX04~KB-2T@sivdy&C%AOc?HznVU}wQzv^(upj2b2JO4|!LfHuDFDn>x7Kvb zFuI&>+WX3G(t7^t4L1KSzSZ*mp8#nz_T3J8PM6L$LMQSR#|BSzX;0b`B;UB=J-Qe(V_-Rf-S7(eOqPl=st?@$iTYRAdygNnU6vY@ zk=FXWn#7KjmzK0jDhQpe$^Am9o5o4Dj`-5LfY;+6nq0wd@w1W?F`MGqn{LZ63$DW| zM$P=^U~DZYJTO3m5+z_eM3a z*?bv*$~U9!bPYkgF53+L(TQ~YQN~k0*Dau^I{nbno?i9f)6pgwoQs^>q7Q6l)h8?f ziJQE>c3Z9#M5MS$ygT01FZklDPajwVk)74s{W)>9Q=d7cvk_}(dXSWm78~xtI3i-u zMn{h)9v|`$BDF)y)e&kN`D_yBaIlJ)hMKD*(60d-!+Taf4vJC1f0lF8w$B=wvG&c_ zAN2SKVLi6u660i=Y8jEdnCOHO=1d2UBwUf7P7By!!5?2q<({=mJk9*pkIXNh2%`vI z{vb+re zwL@j!t}k3qJCbnf*yZxna`*uor&ewvByja+QGJaFUu#7IL}k?O*6{kXfzWnNg`q_H zmYvv_H=ElfEv03eji8tsWV;qq^YQKCXU*f7N(9BaE_DJL`^B)M& z+kVjwpY=Q8Gmz!QKg(Hcbu=}M@1xKkht<_aT8nDT1M|=@82(0YidwC5@f9R;?qrQh z-7HaHxaKhNE0#VJz}w((Dt`Z0a=NkIxVQOjyH2rh)~_t{Dh#ncmj&?&y5*%KBlD1H zxll8aXsYa+@sXSc?-Mia-!oWFB2+i>34swAiF07-JALNg?V<~*tAmE<&&@&Ym)!$>$3=gd<8!ipL$%G z*72{Q%hU`|C9b68nI)C)5~0f+TRxi|@r)gz?=Vw0`o(k4A^}G|B|d|b9U7@$t4C9L z{os=*@xf>@RN;nJ{hB)t%lYO^^Xhc5?z3!LQOi(u^54lZTyhF|$xd4@1j=!|;wQhW zCeCZmebUV&&lA5`tuLa=#g`wt`G%V+@A4a-Beq0#SSEY@$vl7{7G=gcS<`ZnMfwFt z-wv9BABXSER#zyG0hjR32Qhw-*g>?a-@8&2_>m%h!#RuCgc?CZdO_xpf9T;RF5**F zdHu`CA8!?!ouVQYQ47uM-Gf)7^xH%%uBSN`e_Jm( zAW&iCq2r*pj%h=!I>s$eIPB%@#34J_XvqlaEku`39Bz|xkb6}mx`>jxT$ViAtg2e6 zx)XKyCt9suSV2yoRvUi$(5<@+FV51&c>Hx|SW<576dO@32`CLZUaezJ$1+<=O>V-8 zsO=^2?Sf)OT6dL3n3R%W=UO|+HA92A9LAmRTREtNQN2s9I zX<$HC*4IrOy+9ZvZ!N)4Yttiovjp(C9u~ai`4*$9ABVlCEsB=*^Un*=)9zd8_y_SEDQAotH>15i2h9R#MN$lFRb# z@FqkYVFrjQz`o!+8=KHo3BJ*kD3WyvCn}Y4p0s@HDzUQ!pR2+@FpTH(43^~$tuCx$ zFk+NFBqrC1mr4*q1pkaU7!&i70^{_nP`443!_5g@vI2;?JU+V2Bp->tHi2 ztI~1o+uuDey;dBd=pN^j)z~NTzm7wn0&fm1pE!_FTa1fUV~L41i~FwNWL=g@yC43_O1>QaTMbB5JC@kYQI(vZg9(=p2#)x2-7h!P+_SgMlUJw) z9lPGXT5&*(*_<_@I^ew7jTNVL!2NJf*ZrKlqfRrxHhM4;XBxTb&s%Uhusa(^s^&}a zc}86ebzaF!R?tm8;L=)^_perVC~yLQO@2d5Yy$hrU^p*kemI}o8|oWeYYa&Ru{|p? ztzo-3CK57!M7(W0VD9&wq>0G&Q|1n~XYI?6uP40e`D#M|f{aQ|$O1+~uHO}4n}VT_ zzfz8!kLR$BMFVrwUh6`sX{lNEBYvg%E{7X`PmAu)pC++U&A{~2)=!%o%t-GY2qb&K z)K@|~V|jMr)ESdm)$wdNe7A|grGb{#E`=jo+ACKGM8=nPu9#3#Kk_2;!LD>jUGWar zdyDUHw*~*kz;iU*E31-`-kMz{J7ts&9lG@9f;Au4(XcnLlqIg8;Ns-nqaty16|!!w zRY||~pmVO%3Zw>5Vm^!Ff01y7b!H(bLPX(923*=;d9yEejRkPy7yf#EB*(-&#vB6< z`%|&N+<={HlK0flMd^tf_V&Zh+Ya9TUrSKCqaSe)w6=?UXy&GG#V|9m*!82kY^>Qs z#in`8=W%3zP=yqq#9j7V<$I7A^5F)@=}(1S*R;FG%FfW8W73HSaZ-0Qhh8E7)2=!4 z=ifLmphyn_JkoMz6QG?6Y)$=ZCyl*53m^-E^=XQ=|1?CzB$pFIKVx?mQ%UO8DicHQgcx~i=;)h0h9Fi&#R585WbqwwIM(=!!n#;q3SFO zloyinW5C7tszN>7Adpdf-XH%I8=4AN{1mQ<${I=65a(Y^d#AUiD_TLn2g0;@a?!Nv zKfn%Oi~v?K&KMOog=@Br@y*IaP<1k+1Qi1Dm+aROM${m|Zb_H`x^o(p1)4ina-1iJ z46g!oGtyQ@VzYVV*st#%%U+bsq5_Jahr^Hrl7Y&%^~G=RRg?;wJqSs1q0*Du)S&78 z%5hgGwGFrU+iXbOXO4 z2XdJ?D)IJ5z`vRLHOfbUUKa{fqts9gOal`2YX?3008Jhi3aWB)mxHIG4XkUQUfeXf zv!k|cip?i>ki=OFmE-S@+|j--TQ9_3xdO)n^;(*|CjclitUC{-e?%_Cf3_4oMF+$s z{bsj+v&!~RboaVF(dh#wGOk#3>)?iF5wOpUG^3>q^K{(9tC7bq*psAgOYhGLp3;yW z1n|2b#m~4NJt9`_WZGwD1eKiLtG^H<(<-RUcLR*EZbk*X%kZUz_W2uRE^nWSd>i(~ ziPw8k`*VgDQXjxR)_W)XnBg_&9c+89?5JV1#SK*ux9lij8gVA0rHova`Rhi$#)3hY z{`@>z=V?sFi{(eC>yk+V0~$5Dt+29G0n#nHhSWS0#F+IpS|sGQl{fQ7ip3zsCFCN4 z3w2bXZ3je*+I80v3UM@Eu3G*>FIt0R<@WkIYz585m5RzPc7QXrl3 zyL7v&SYsavhi^3ufIfh=shV4I_ddQ?`YrnvoCsZot1geQIWulCPQi-8{CAP z=V_YWnHLozF&VG3FP;-90u7-CdLMYsFef-X0VSBPVg;#~fsjw)?T4)Jgoy+|#5&rw zE*3q6M=+$ev^3@_0VQ%Gw+x*=!txwTAb}3V4%?rmFmJ?_7a}?M62^`_7&)qB3~=g7}av79sf*ADiGhj`$QS7$(@@@s~4S6F1d%G ze-hxp4wZTa4bbtI6&|6Ijut5lj}{dP120dW#c932^$e+0U3DtN8^>x(@Sltr1}F>7 zpll=%ra|M+O1ody)i^+kKh>HDX`(1;^FWxIddsl#N;5skF`*96|9o_B*ttEmaMU`9 zy>yJYLUAqGpnYD8I;EQr)rR`1$R%Xju!h@=QYC!!Q!9k1h1ZB(5wDQgHJ9u$^!tDb zhB*&5gh71GO?d*6?WN3!7e`Y)>ImVP5<`YUmj1CoRvnnt}J2m-GVEkH| zjE)0N`fI;MqCRa-nN`~n*684kJTk0AeM<_G3kJw1vdL1pfj0q_TI4X9DwO<1`*%u> z=kE_#`Y&2u1-yDwE`;3t8$A!ceifFN2lhYd!q|=?w{Tzkwgb5i>2^Pz&}wz|Ua`4b z1GW6R!PAM;>KqsdXt~@{iGy_pU9_&g@@hTxtMvC?Jg}%Dv)MJ-@J&q z;#x2{w$|-0Zz9`#Q_jSX8*~cES_S$oD znW#h9y7A$`xn?%ZFvsmwv$!~d4>F3MMUTYr8r-{^W<35dQ3FvU5H0qH3&WnU1Ejvf zIBt@H+v}$xzg~@T;9?vX`0nxTRs&A@v_}#PlS@WF%OUpgii=#5a`K=Eo{rs)#B98< z#J}3n{jYYk*a^3sjhgG|IAx}N!pwfiRijLs?D00bHRE(&K&}>i7N)%8qLJI-IJ>zI z5@57SfQihuMr0l3MmRkSFyBIQW1AGmiU^IZ2c=+qCWc>J>f=dG`#B}YH-YePSAFt{;1r0B~Oih$S->t zplHx-T8h7cN2_>kJz>rc(4<`Q_;lZC^NE9D;w9lr2MA+i_7o>>@LZ8}w*ZJ;mMcK3 zunefhODEo^qy@avz5phjMApV6zu(S7L68xIuvGVmbx|V-hOrvej6QesPO?0sd%e$X zh|W{$2@o91zN9EufxeLoWCS;{suf-lFY9Q68cfVs{eFow8AaB8>2`*#y^gd*m7huR z+Q3k1Z(`rD3wq7Eq3w=6uf`*(r+-mOX-#=_N(D>~&o5OiHAMeNVKV4l(w|H7pbyaC zvlV17BBON>o`tPv$F7Lt%gTdPqbC)9t^H~neLMwDr~rwK3#?Pj+(3$Bn*Ngrp?N8W zXPsD3*n2N;EI!AAE2>@famk<^U%&e(w$5J9ZjtI&47X*ne)i99H&#`Hu>h3Vj?r+> z>KS1ycTUiixa$-;kRE6#)b@vCp8=iHpTIo`wGGuSGVw4z=WGog!zkh^7fCKLvGGF@ zsz4D#cPTfaQx@Pv?T+Y1O!OS83JWeyk z=5~ccl99hK^Ab{AeS1;>GBo#2|A)G1wpO;6aB>wEd+4&$(Q%A4vGg@*dERIB#jj(- zTB1b)IbsA>Khk*o*zx@H-}U0}3xq1Pul?E$D$*6f_jnn6Zi{cgbn#!fo{EsFAqhuk zyKy?Z#*?YL@nKV-x$4#>c|Rb8-**?W?tAWZcl*N}88~b4yjVw9?Lc7BE`FuL8D64> zyDF}IGqqeR6z1H|j=m2FrQr6MOnEfoU$tk|L*^qinoBM6QqdhOp6TZPHp6ZPHN5tN z2iJNx9o8%X;n14x>)Or}vFNWCn2H)~5E>WICkTe*h*6k(SZyq%95Y`c64%GfMoREk zWL~2M(LN;WR)e}QL$kt#A%e{AS9jY2fLhe zFhb%L{%{RxD)suzfNsqBRti&@XF;!amR}T79sJM?^OC z_*eMXl4^+hf)tjl#DqSZ!?M>jBaQPgBs+|fHsYx~)Ds&c#wZAu-H#~dDDVJE8_KJg zt(Z{UjrZOL7itT(&;{8}ficV3exouVvib4|LRGl$rXYZl(HZPElY<$4OiY`x%5sKu zG9UiRYH7l5??#to!g0~Y##LzXgA>WJCrK)r{vN&ifJjGueXg-3P)IK4;h$9d@BdEB zk1{h2K?ojOw`)Z1;6^oBt3^RSy=u0{LKhQu8ol^+?RV^a~9knG-8Ys9+lIWZQ z^>6DFAB$dRU*ps_3!I7X{6QF(=l;Xy`{3Ub$U&jx#K`vKc!=IcVGM05GX1*rup06p zI@fb_dmocIFekBn$F-YOlQyEaWw*s6g11kh{00 zQTH}uw2h@b0gKzK;+hAdIF*W2C&PEp)hO$Szy4$ zPvEtCF{oQ0K#d5cOdSNa_@o1DwV619e<8?;;tbozl`pO*yToP=Zi5d<+}As&vSRZQCd6cu0?9ciivGJ@R2NP^v(nYm&U#9w zfN{NhtG(h;T1kbikus>dD-U}6@pD5J1YYz?Ti2M4#2csoOdy+<$)2o?F5 zxdp6EGQq^wM+o*u&iaMmEVzj0&un77fS&M4uOw$#ywZHWkbqH2M6sF3V~G2xAMbes z(3O&GI0Wp-=sElIIiYymaP9T2-fz+=VN$mG6|laa!>adi=jJq}fjv?F$Bf z&sJ>1#Q$g=oA2y9PS3;!i8?IXkx*2^Me3#ss{KxD}k&hGaq3_X>1|I53Z|3bSpY{u}~BvXS5K zjb)_CIn`zxR_@G_Ra+j841965RX zW*^8bpZn<2b7J>H>Y+C zwc*5zFiyGmVxgB^LPFN8l7mJHKlH~0)bd;TBR>GD(66+lHa-rWMGq3fJ=9xR&R=Za ziggJddR3{9mJ&$bQdFxns+Fq!Xy)XYXGPuY;oPUm7`-#Wj6*Vdb1{odEyXhg)_tqsVSATr$IJp}&UP!l_ z_yfOo`uTeK3u`(*J+fc{3oDFT$)g zm<+I_5jhIvMGQ;)Xba#Sr|=B#Mb5s2$W0IP4=BB*poJpHV&cqMy3M5-r6^nq6QQ?< zGk=mT2L#FTh4X-D!})he1tdc-xiO9QoaN!2!sjZlPkpwvQLPH4CN{9;?)IX7)`|oe zT!0|OzF=1p2M+<1Zr*$^T#3EWSDl}`?@w}0BA|UxojwJ?iaZ>=rj-H^T1AA(gL}+$ zqpz}vd>liX1*{ABVIE#CsIm=;d$I$%`zPH39V#9$7dNN`0K8y`$Sy5`(l%_GCAv$! z&47$c^-RYWU|@BN#<+Jr3pkjhe!u1%>5r;;UpVUSr}GzDR4#RA6W9&;J?hOX*|U&w z5)X;B91->5=&My}XZiF?0UtF#Qr^rI7Vq2}fXlv>rN$%h_MMeq3i1F``B%wCWQHF| zS4Om}3>|dW4d<4XhsEc+L|W%z+##Ap`G5C`orMUt`w_EvqME6WP;Rp972z$}12S)pnm6l$&3$1ZVlqle4bD^3Ey(`LaK(XJPaKB>L-)>koQ15usQMj-wOt|2%Hd^{$EIt} zBQ%Pm{+543xG*YrOgvX+SirOn$TngaHPZddD@SPGoh;p>KGfn3@(!ATxCcI1bV4i( z4To%CYw!cJxVWhT47|^E9Lt*fZFeA@=Z}`SN1y!sFf*&33h@~n(OF|Rtym1$rh>CO z2HkgFX`QV*yS@8QcnZm~giZgWE>7lOG*hB782L5SA##Gq2(`IWCgsE$YNAacm>ZCA=H)`WkK0rnW++7nu>ZC{@ zxb*r3Kid9;ITjQ**e6y8Q*S}!3|5vbl{Ss?UYcWtaI;tu;H8-kN}X8dAhm)$5W*=D zA20w^{cj-<6i*M~-$D(%h4|7Ah)^^bt|kY7S6`M;LCawea@}L*);O^o8GpVKu8gLO zNhHSF%{La7<24eb4wZ$=R`#{<{Y=@2Wp*fPFno?~UnytQ>uJGg_Di7Op_@0a8$Y0s zEK6h;;-GjU50xX}mQ}b^t(B*NeorG^`XyQ>ctG@y@&Lyb?DaYLQprdj&mrs#0V~(X zvvvwH3#3IYYtXC_lPEJ(d(>Za8S%uf zSdrP;!=Bg+yBxgSwF~veZ$BrYBTNR40N|6;=wTm}e!Yz}=I0tC_0dDY-C|Hn?2L1d zoahvdmBH1d`zjw?-pfiPiva?J^U8{Jb1w0cyP z8suWmgbNem5UHehI=?S;&x;E87u^(-A-|*7(8CT~H+s zhx0fYh+MNe6V9GHY;t9@Cb&mq+#%7it|n$5{pFZFC! zvO7hGAbOeKRR<3pV+TU`+LC39cppi^~YZXfd*#~@eo8YO^BAg`Qt=!a=w!U=4dJVu^6o2 zA|Y^!WxuY>Q>Bki2%@X99|O^{LUNr_kx>%e5)E(s5&>WXlJVz%-tpaTEc5%x_hmLF zjJa|N2L=`l!(-aMz>k~HH2q3?d61CfG1G2V)*v?|kc4&WcEmx^=<6lO!6%45Xea)m zeruQb;`m;_HR`H5wQeHZns%I0Eux=f4;#H_*DDbyt`)Y|21-c);#f zx>gDF5%6%E14?7(Em}EoSn8UB9R-FAYM`|=`T*YsUSSgUSr5g6w45YXJZ9CN-Op9p zK>BY~a{Fn!#6x0ijRIegJEIa}$=R%hk4+8*$uVbemJxlsZb1uHC59}%xLfhUW+{7GGJW6N+CCkEO>-;*>Shv-$neXGuZck>Ytgi1hk#kBZ4n zAjX~jHHYLxR%ra1J`3*W7!u2AQ2)61FEa`Yn zE98}0=4c%+?P3LIHJvIvjcl`m1LfQ0k1+}qwFHCm!v426J+!{E<1-% z$;)-tL`J-t&Z{6hKLvF)I?)8s@W(Ttj}9;qP!YKG-zFb0p=^O8+LCctBDKg6G9|{1 zi;J8Sg-#-u0p^+hWSd_?*B!2i=Vxw`Tx_n|2Mp;Q!-cseq!jGM8-F}RsW?o z8^NP`>R}(9*Mjwx$lP@k=_fDBPtx_KL2WLP_-6ipGz0yZ=EiXiI5NMXAFSY@@Tim0K*$`(Vf<3K<7-v{fzt^C)xKL zob}i5Nr~4XSrb(xJk{i1goAJ2Wg{<;wBJpbcRwM)2#=O1lAsunI*>}3XhGQY6cMJ7 z*2VMtyj~esb0$HYiRl26{tJ{RVBrvu;-5H!LNC++KS3k| zaHCl!2YOq62Fe2x6T)XFI4F`qvXl7c?Cx6l3$vJ6MAV>h>PYv#E&-TBG1-GYh~J$F zScJ0^&uGOjtt6n9>JulPwVp_81EvE~$!IWps`hri;n-2-sl26^=oHH;RELc-e@{elme@gj0o+G(A`Qm9S9r zmw5yr*r41jUn4jczH7!h_AwWSu!+!C+qs%(*{fFz4yOb%3+dHjKX}sUS+ae)oQd<= z$}>(PI8WBKf^8~Tj*u$0y;z$s`;GAjr2U1PKUdhYDr1!tDXyb! zvloNuteCDuLbVdk5N*xTeJiydsAIEP6Cyv3MWq8=r)&}r%4C%%F%ZbKD7wwgS0bm; zN2li~40$BCD=*lo;AKKH1Yw2$0YW_7U1chNKmyLw)6xczh~pV_khZpV7@rb>g0`h4 zPiW{>;Eh9+`okZBKZZ>9iQ#m@PaZ$Tqs}HpUtz6Dp5()UIjMd21j$A}Isz(C{+o>8 zTlxQ=06=cBfD=wGc&9U|K{q>@I#^v8pK)NA9|B*9qwoG!@1;{MzB~N%Z=0n|KBqc8 z1sGM8brSffi4@ZN)KR5Y|K`=YhjKXy6J~ytL7j>*vl*Z zCt$GtnG0j*OT3c|CV;NBcP*s3C;24^Z|V6V96!?G-u<)B?oZL1-Z(G%lMSIp8IXb| zji@{58B9)wGmQl9@pc45(Q5m;`g3P;xROKhPEvbTf5M!xNS2B9;W>;+?Kbps4Rxu$ zy!FarrF>b$JNGv=d=E93O(EjEXuO1yF6X$6&tI4>L?KFoM4qWH6XB)sL%|D}|iaZ@-^a;$7wa zn9o5*;CD=r1uR5p-h}2i>lYLdR@|3jysE@-i6M-iyuHcw_!B{`4A8g`W;)Zlo>OLw zEYL4N{+`IvK+)5uX_l)BImQq|5LuW;!^a+I@f6h04{a~ztwKtK2&72**=x=aVa$@I z317z%c)VkJ0?{F>N&HKI)?TqoxcQu+O25H7?c?ORT^kM#NR4rdOO^Zy`$tS3pPz#9 zX`O-=_p22`CrewZ8x%tPQc5MOGd-f8J^Qe2fTj+dN1L4@a8Z|6v=rjcJBSuzs?=zh zb71N?wIBFinO~#&DTmTyG}w$}YwEE!@qXD`e^#f2j5v|oLEk)%^IU8Kzr_5D--eX| zG-o&S@4iKGN`K>4b7UCXLHe&oc_{H{xL2&N<7-07pRJOf7mfo)&A!_5tQg*e7*%d6 zopbB?6e6R~2PtX6ZQfvb6d5V*uen0nqw^ z=jp&_xE!*vFM+t7=vL!d5E|X%EQ{81M~O=ELnqT}Tr7?9e1=7tPXrhn46_8!yU-!n zR$tJPUu()6(ThbSUN23_Yi|+?sWxC52sG9ivv2MtOEl2`9?9sKA9@E%F)E zQjT`MjL=wP)BF$&m{JZ?35rTY*Tu_kIU;_ys?)Pjq`8bnSJ%&or_Y1oW9t0QF&C4* z;@x3GT+fimMujg*Qhl6bFAB(E&jZ>tGFR!W^8~ycig5MhWGYy#`Mj zyt{7ohYA9h>`2~2OsmXA*FR$lTB2}kA_V8RGi?rmj15_R{tp-P}f0t!_308CYA*SbVOSO$w9-Vax?tA~eR)?~V zN%O5t?00?#qGX=&tJO@C-?WUUGN|AlZq0~-%V@W?%>(NEmNK!G^FW@N7afmgeq6WA z^EWdW5OfF4)pqlQ3_EBOg~iCr{8ZXfkl=dMnJ&n|6b2)8$u7INX8bGr_!aKerblq2w;K|Rx+iNZD7a~&b~?@B$6rD zv4;~St-Sutl=52M_M$!x>QnF>gf}3InsfY{?F3H8(@EvRRE~2J{;)xC4lJro+E1TB zG>NcJz17hbEy1-SW^@}Cre^*TY?gVP5Sc!3BouUk(ZcJ}nY!iB$ zX~OY@d5Bp_sE(diB;_KP|1>2QuqsmaA}JPhvhe!S#mP!)9{=NSoVgcF@8boF;7}qL8RmGF-SVv0-jg=2rihB5yPY5?$k;MCHfiWBZ9qBg7A#J-tDmI_V5+51D`fOE>5y^_S$-{hx8Gjur9@AQK3BTpTKFdAlXY>U^! z0tQGZvF@3Y>Uv(^Lxh4h+rJ34*P+#V==i(l^u^Ip#I#4SSGgAye)wo;oXh>F(R9h% z*P6ao{0Kjeo0H&OElott8Bw51N+@{oa@Eo*Dc=hOtG3sDX$mj?`CVuNB#27*7~$4lJ!<}Q{Za9UEDTR zKe)`*Zt|3?T-}Xj=s{pHb?A4-rek{1t9%VEvDNb_?`Q78p9PN^izl{Odq4AYmJfy3 ztpx3tR+q!s^jQpoQ+0miztewK|G{4>c2W7|zUn*07@Pj6r>c~dfo%az@F%+F%zu>aIYp9edkJP zZ%5@M%5jE&A6zfdGy#l~x`2@3n-S6oGJIt%A=TFniv04$O_&@{U^)a0nSm9B;rMy} z11Yc>K4xv~FW9m8%(ln!Bw+^HjigUezRp)R4b8EG<99{zNUiJ(E@ZbT_a}sEK?q19 zUA%WCz(2EERL1SB+w@Z6g9~R-!)M2#noy$hYB^)j0}u)$q9fEf1svpZ>HD9PMN4nC zDfclp0(!77nAaFmhG50@CuC*x9C)ihKATbm1qD$fB)=$J5WuIzSTnhUtHGs)i03CDb;oyN$fZZ;51z=1IlLTB5OukeChj~Y((^sTr9qn117=F_7O zWzu(}veb>~!5be>dY;q^wr`BD@3BDC0Kvl^XpzPM>yvjZH2eW5DwCsz5C_6#)T%Dp zne@P@>RpiX2L%hA$BZ?e-kE1Y4ItgrUcT z-vUg~Ca*3Ix~tLWRmm^`IFM1ohO;pYcQ(^oP7q`r$B|F@`3YApo4-UNGdM*9PvR_a zJ_jrxbZw7jAN2Jx3T=K+_#08_x>7uf67?+n_gmh;ZCla5y32pOvK#+{r5Xv;zfJql z-}~Rle|33=n6}2-83{5bpBH35iv2Qq{c`@Kctl{bi?nm9@;B#83wnpFIA;SL+p+S> zS9wMbTSLvh-|D@pDkpN)+gyV6l*#P14Rcfm#xM3Cf3(;2^$%nusua$T|8mMSMb30o zcg*8Do>lq3&wtNg@QlZRw1jIFz4gr@&YQb?mv!dH`Qh%R_=D=4RPT-0gMFn}3Ljo> zME_a!H4Lh&;U665@ONS^j86^3ni*jGyxsERi-+~g>US2SS`r3lcYM*m&%eb)lFh`K zGTY=v4#kn`N|O;!Y4KGQhT-t)sHx_GI2{;@gXD49lkaq$m`Bx%O^o&g_P^Q5lO!L~ z2D-J`hg|KaJ>=)lx%VlVtw)Kt;&I;kZ*z>!FGAveA$bpJZ@h*Pv)ugnh(mHiT3U;P z4Vr`ZB<>=HWFkPJF;W$JiL18$Q>xi)n(!W7e9{V;2F-ZY3k@v!08%MA-^jJc0C1o{ zQT&*&VG8bnAG=Dm>EGw}|O9idt04SmEjO(;}kkUWPx?+!F`e zpcL;)LrAsJu~YHdBFFcd61y;piikis>a@~Z1cU!DJkc3t2^3pK8Bmts=!w9$?IU4E z&#|noo*e}QVfkxl_^8=T8e&UnFtWtwgO`j-ZY#pGVbm%7wMCu)hGw**UlH~mJEzi420s@r<7ej-S28qc%(chb95*+AKSA(a4N17 z11>a4W#|Ia8sYA8$mFoEdpe=uwRyW;d1>o==s5Q$uzxm|Di9OQC^OxSeOa`d*@B%tUD-?o1^ z2|XVCgX6Fxd9+!{)~FKV%}`}la7!Q}^>C<`_cG|1h`7#{h)qs)$kv8@Z1`tAmt!#P zx!`YNJ1KJxW?!kA(ihxWHuvQyB5W$0C~BS`j#^?zbeXfx4Vn^&(>W_+<|CU?_rFNG zWyf)ArW2=?n>Vy`eDL~RU2di{=Avkj%Adr0^rc1&=mA#yao)XQy?Vt~^2T~4Ng5vU zBVZRUJ0|i-r=gx6u~$2wdI|1R#BF5TF%0;(o3kayd1UM(eS6CoMmZ9UE8eD_ z5Xn1h`dJiFaP?ah{;r74uo^87-cF9(WKEtqKVHi%KQ6UcCAmDDmWdMnh+h-vsNvhK zlA9_5bywJ)XFf(2FqO5LDvv?QM9@;wH(}1;LK;N?_Z&ey_WAKwRITSBK{;smX^_sO zeHeYFnaQrY)8!-Uo% z@zX&Vq><=r$H%gVPx75KVNR=F*`bl@AW51L5dIzL8Gt2~)iP(}6qu?dyTDIx$zOGP zIQNzZ@xs5Bp7ts+pC~cfG+aK6d=$*um>Ichloto7jgay6wKis}#ik|&z4SZHKTYh` zXhezGf+{0q5-vU9+iQ54uVtkzW7MMGiJusqnuHlCC!sABTr?MvnXp6Y_-o6>sdL3( z)0?uPBp>vS$Qo-e7QEb#9E zRM*7wXmN!Q7@xWL4DGqdgFLCb5;fH>nj(SJOX(!F?%1(kREU>0$}XE$guae;GwG7H zmyQ)Y!qaGX;qe2R7I3@z1gB|ciO|;wF6}40fYIH%@vkqb=#ycz`S0sOmk|PeARnDl z-HeYNm$|nTAy3yv&V5? zMK&Gkvn3^HKy4tm1CW&0qZv=sT9+%(0+)QdbVVB~MN9*K>1B0&Q#z8QI;C*%6{5XZ z3q_u#is*be=)`mL8#d`e9{%9IIkcg@&1jzw4~nzv=Jk=JSn)m^tM%@85S7i_vi~Sf z^03hm#3Aw*6EC>JAVk%~8NQb-HL;Ji4=E-9T;g;%pPvmhx_^R@O|@d*E{Q?mhc(%2i> zdd9(T9bS6u;ahh61}{Op>{5H29@{f+_2oGcraa1(Pn&U|xRDxS}wf!izH2Zu*k9c?P>a(-_S1)mJ@RpQX88!$Ez0YJUMNzKavf8UWBNuA!||^ zqTFaJ$&U`w$n5E&e8OrV65Z9avFx1=(9E@WS(m)7 zKbENNA**DiHHo%2L4my_J`%JN2DsoHR6vK_W#h(i%(KS_c}Mcoh@% z?UVjjVNKa{$?p?NX1~ZKPBu6CP}qbvPdJ~S^^55!;j-+vn{5pDMcpBEO@!$I-J~du zm}VQ(S{7QH&;z{qJlE?2m1Y})Aav7}QZp^~1gKcXgHk{=i5>gRt01QX6y=a~1{qdz z%2Un_s(@Y38uDa0m8|x92;0KXrv&OprsO;;D6lh(MsAbl2yEWAiTxqoMM(b?WO{mf z9T}?Q{`GuHMM!AR@~V3EhTDYfXw)tEtWMBr@?g8;wpZdeG~{OV-b()fQ*d_`Fn=>? zxG+&I5xQ*ZYXXLsgsh9oiZHi^lUx*S`-QdlrL!fWctNw1lh$NEQ2mLoi>PGh*O^XH z$?7gFl1$~QJod12Oyv;J;hWZDNuTP?6<8d-$k8&3O1BEzWxAxQaiv>OjL~W?n9YcrW`AQ{Vv3T z?f@|WqO=6>@-wl42CO_m`B-a!FPO>(m!~k|rjch(d2zmwJqNQ7bW$66(I*NQpu@IA zbE+UO(z!1>ePTmk>`$#96dp^suYFFg^E}iS0fVEv7x{7}F9V{jbGR(>@X&3CxbHk% z+aOg(Di|dUAHX$}6a@Lgt*_`pg^jcD7)BGlzd~X_m-yG)aEvSTw#*N_iuPD^I#@qY zCnl6`?Pn6GfELGVs1so1*e(>fY^$X~(;j)>NI5+C>F6rbb00he`VtfKs}>i{Ybc7a z=W`P_U7Zu>^*Kg#I6I>3uhYELR`I zU)N-IE%$?Od$RxQE=k}KNsej3II4MJjn#J%pLLXl0_qhe(fNjR0rD&z!}@@b&7vhxy~~JiN^Mc&E~C&b-gJ9;VNFIznSb{7Kda?b`mJ9Yw@w^HI_c?7 zagcldIO=fc93GF%G}^uRneoz1OGV!8gj~^jllfMgQitXb_L4JsTB`F2gPlAvB}E=C z?cM$-WHJ&_TF{sg^?J*2uF0Zpm&z6rZoQYV_b?OZ1ZnT{`1kM8eAe%-$3tjbJFd4| z91Q=Do@<~W#M*HTZSs;JMbn@1kl0-k@qX& z2552-p%ehPuorj&;I5H7$t)5}vwy&fd$I8vtO(YHbn*oe2Em0aWK=I5mzEa3zvL9C z76+aj7{BO_+Qtke{Z1{5YKaOsD8e8Z-5dF8%zCKsD8UPYeEFamw`iJo0M~QliQq^& z-On&IAXd1xgo5BjW99%OR`Eq2prX2rlKVk^88aV5@(ryPkkyTpp8=s#XYr8;ipy?o zc?{;uKj?l$6gYxvQS!OKY96K-9iTX}P2Q^-{Yn zPmY!w0{VgIt6Z7D#*mWPdwS%B-e}zB^{^!nard|?6x9_}U#l@VCEGd0J5c646R(C4 z*l!9>l%^xwAaF;$vY{wSs?JfkB0$}CPw@wm0KPtWs9axOAZW&R1BHR)ZF4u6ZlP9q zh%WYZ&<_X0&wStN-+GTUDn;z0aFq&O+zCut!tfpXmz=4J1Zu7h^i8;@R9)=4v7FZ$ zdhOqWV%pD90o=SMo@|`#AGcko*$;F`N!o2Y-Tw@c5Pbg09eHvD6|tDTa$a&=VbDHQ zII%xE)?2$X7Vro9c;OVjgye^2NBH*)E^NnPqb=#X8^anG`w5P}Eqp%xsxJFE{xu|3 zp`{GW(fNfXXjff1$xFHR;7WV@+vPPJEtyFVBWt^h7|9OPtbO#Gpd>s7&CKRxOV(zo z2M&53aXe&YDpsjHmB=OO@Cs-V4<%5Ttm| z62c%a2_of};UsV>og-%mc&|r#A5^%wWE1vm;k~3VNG65@?DdqHq*g&1`wy#GipB&6 ze#B#KUJNt3<`uo)h!wUu3LMmuHps#-4v1f6dUs{$D^YS+#0g$tuhq-*pM&yaK*s|&+TH?b$5JiJg|>0w;s@CATG==3*&qvq zJU}r)_;~Cdhf7r@fNpEkuMwzZ0(X~#@=NccpM5+9(pm~kLMTjAmaAsbX3V%1-#+#g z|Ah+^h~~5B$f;|my+&6=bH}T^CuJ@30_cJ;YH18s<2j&-3O}N3|9tJE4{~|<)O1R| z@u}R3Bj@KN+1M%o8`cr6W;|0d^MgAAZob`=T?#TclkR_)-$dh53P)h`e9fw85vBYIeW$YA98obrE8v~I#GfDdB&*7D7i506~j0qiJy+~mDH@}>)|lrWZq>$ zjb>j=-toVGKSJL}Xe>ALzKh=v=%}M_yDpqvKTZEwu`94@TBiM6dX&b{3%bdk6Zd8VGS?cG&VxFZ`kg=K9b73%4mvO=n5|7ciydi<&9$5FLWVD(X->`u8 zRxba94p3dAJH_z@9j?h*>ORf4fQOd9EaFqcjp?2&Je9M$Ueboq=9#0P!qDdtXf%d! z3_3I~K&EI3PADT)*1s0JpZ}906su&ofGiOOi-NGq@(i%V@&;S5kPON{?&Y(>;kz#+qb@$^;O^J@`_Y}+``or8WOmT+4a1O(H4X58-jK@4! z--|Ld!6nYggG*L3JO(Jt8$-K;h#%!m=NH~4go*$#JV@xJI#a>t+ZT`ceDX@@nmI%+ ztZN(+F$_#o@59S@4goOe*a0d`FezMkxx`bnEs|HLJVO?Qv)QF_j(cP>E9sh;7$bVa z7%J~$BhO3VH*hla#9Hq#m+2Q>AV@Q5KV{h8I7Dnc%FlJC(<@e|87M=F?ZzZNlkhuYescGu_1Yzd4 z4mQkrIfXf9GdH5CUgYgB*0cD_x0>F13XugRdo2arn960p(z{A+*XQ3hDr;lP3^a`~ zSTZg$>eW&as^bqXjFF=BMJjt7bN^4{bbQ*1R@xnMdpcHY(CT`c(8BAlZgGY9?1phL zf7M4wa&#>iV5uaT$T*SKQ>}qn-0E{wPYyYl{|wE$9yfxHPIbmEZ51?|mP-Xnp%9LBUSg2C)qVoit{ z_2(R;`VGxEO2Ujf0=pR$c+89>F$g?5de|VtE=t+13>}e;{9fk)?SNcSiXpRLlXL;g{pm6tjfkm$9XCpEW;7$=pukhs!Hb25W-O4QQZh&|$EW z6+q~n&Cp)ALapWV4l{7%>KPQ4c)6#0ah0HVljS6dx=_TM88r+_v_>~90YSL^zRIV) z35>KZK_d$73zZ>{Q!S`^VNjwbTt<{CAE~SZ4`@Q87;rUqY%QfkoeU&X@1xXm zviS23Yqbxx#gOEDt~2}ol+^GPmTYHdN#MV+ zR9)*oWcB{fSc;RKcctAW*y#T?miqhrl|hmx<&<7A&erheGfMALuMW2fgBZPWf*Gw% z<$(!>S?W?cYz4J(X`PA&V)C#mzQ_*+(tud987!^D0%@+ar3rJf{iN|qoe=d!YD^H*_hExsT&(|{kj?tc>$ zxXLRl`2zPJgI@T~DRG)HAt@UbD%2M4L!1+sqkl5Mm=H0{)5;}zyEu+FZyw-c*2n<) zpFwUJszX2{HfuJ9^L#_Nf<36+zmesMj_8jIMwDu@FxD?u??;Kq-6J~IW91sCQ4 zxG+!x_>}!Uf^sNSK8ExcdCyv6&D%hR1V|Z3be-ZWaA@sMHI0{PuC!9i;2$&gYJR2( zH>+##ee~f9<^p+L@4VE_@Wwo`8H)+PeiVBrR#Zw3;Lt@oW&^nIKal4Y&d&?h3^Xgr zqxlh&!!RWh4emZv(?+$!1Q4`7U=7i$%!vE&FmBS&VsxIvUYP;>`yL5?QZr`1!uu&J zgkn9wY>7kkd{K5PKRE&}jG$7^*GZW47d>NtFPA>~qT~u`IJ9}@0*HFhDX`xNUQ_y= z*Oo&%Ha^l~$xYO`aky+yzfWlPPyEwR}#|o7`OwYL%eE;Ci-b$rhcEGLGrz zC=}B+Zx5}9xH?X}d#N0(KN?%w6r1S0HJKfU$(ny}RJgxe0CqhLZ;O(f zegY0x1Zz>=@1z1zQhqtaz!TV7E@FM0rds^Zc`N$#SbYf1_U27rwthS0yhVLz%c#Yclqe zHfi?B9RiffJ8iBFMZX0V&%pw|ARSe<58n=IT2Fa$L?vjwH$bGtcF~_Tf=`nR8X6j0 z4~>yrkq>b7cb;7;78eIueLGsWX*3qw*9v{M@a-}7z+5WQAPp!J zt-);-Lz?;pO%rjLa>YAZBBt@XvA5Xzz0*JiQWqyT86@qS+?^>n&g8!BK9ZrDdop=u zpz_@Jk#j!M;A22b>Hty*RTzel=TmcQ(1u(96Dt=rZz4U%#8}&Qr2dr0q1nRq$(9r2 z5fD$%l1L-=&dN!(KPjDfY*VV?MBq|kEL95N0JoI;i=X-2D~%ylPKAXS_qiMaJkrsp ziVW3ujkkIug@(tF6FVQR2f*MntI-MBr6Luz+-K1;LWseLwA_w{zEI2Pz0?F;o@sv4 zskN#u$pGs#21caPCR4-7;j~arDoKQV{dJ^6S8qPe)~0n;{bu*cl9kxWGbrj8vdbq)j7whOLe>vkck@ zAFwaYG=*b{HUR=#y~)~+3%g#U6o^mZqihb40ZJo`4djk48e@|$Fd z#GSR|V01*BQ-g(l7558?MMKcC%d7B<(ws1Egm_7*y6&S6Q*u@gNzMOKs#ko+djiqB>{3t$Xt(ith( zeLax{VDQ$u$Gqf|HhMeM%UNGs8fA&#ISY&=T>=99D@~m8?pTm*I1la@ztYGi0XgUaHVvM zbnn?2?FGa=X^{NhzL>lh@i)R}zbC0By17My*r^s;-OMR<@{DBL&ujLvKL_4Bb(bV- zC@b{_AiWTDHve04-gcCtCnX>w0|``0{S%C_WSNV{VMz@Cix=b^siA(Vx(AuKErlO= znG0DzSf$08#?o*jRty=HKj@aQXp^PHXe?ej4Y)*TognM8Lhoq59W(M(S)Wv(ZMt|e zUuC32<}v_O^R&=YhN5qS8{J&*ZFlp2R@kF|$GQJ%uS_>hp2>iTVM)O%vmJ`oGX-Ec zKL>4rp2s{>(S+~WqZx2sPezY`glU~3r01=o-$svfoEDoLGaEm@AMr>a<6->5^-UocdTbvn}mB+kz zRk7wT-voNNQ;4lv?h`lYc>iRHx&|a2A)JDKyi~g9^Dxm9PVO=-DDx3?2^b8# z2GXhW#d;WLOBHXj>^9HVa(#9;sXCw9CGWkTv64;;reiwBc@kyT9g9qt70TdZ$9r=u zafEuVnTrdN$%u8xR};@1TBz9-hLwU|%*x7< z9DyXKL>i&3AcuD(C3W@JD3b=4niZ1PgK$;2xFx}Z{peGG!s$DR?Atb#g|&qLHRO;! z;q{Dy^hR7vORyR*T76wMt2am*q-`$J(W9`huxDN=G%+#0Qh81yjOQph?X#At0HT?5 ztVOu2-Ndz0YVl7PY zIZVEXnIED_AH?-Im#PCxgVdaiuwep+?S(t-H#BPso24@Md&gw7c5nVS3jlcpQPRuH zE8GCg_a=Ewz53@P@|-R1FidqdMb2Ndwlm0uNQH|i21;PF(%Pm<>@=UTU6R-Z zD$57CwLaUYvM`U%dTrf2Uf!pim++gbwRkwi&GUb2?#Mu8=E}@s^80Gg{p;^Pf1}FTJH9>t zy6eZ;eakDwSG9NKadMX>* z2@hXa(u2zy7@4&kBmu!VPkVIdWq?iKJutE0eJ&?YM=*}Mse}kG4?(Lg!gK)VgJLIxOH}LCNv`FZ zrNo!5FEn`%VnM0LxCe$-L~u-67PemSWtH2@4~Ag9ON9J=7XN^_!BteCTWL2f0`CF! z#X992diRTcu<6AxT44(Xs-#PhKcbQcXA)?#5+L* zmA}gw+T!<@q<~=|kpxQKL^v;zZx^|~`n_xrU~q?VZi7BT|D8WPk0b7W!5Xd|$G7oj z>M1~5LjZ%&@bfIYM&~u-W#uykJ!ATTrRLV$k@B%>Z$#AE1t#pdt8FAPQv%uVzhBA^ zt3!Lbl~03LlPuK;{TR5C@lq9qf>PD88$P$w;QiBaQK z+J7LEPV@C_OrrY-(t-E1sHq_MIGi>@En9fD_+y;QOwWwh*E_cd?z;mGbg&lN=u+lc zjzDSmBCDeqeBVpG(olql`e)}5qZyzv<#&|e*)e{V$M`?A%shqr{JP22-dq9cxU+dV zW5})(V^pnon|SdPS!n*FMmxz|G#8(xn9VRp_I~5Xt`=X<`IEF+%CsX~RH;lpls*_K z{E>K1$%0C-7EE~hhZ-0Wtu>Hm&`1ZPv;QgF;>jZls9 zS2m|$%F8fkJN;Ls^ZcQ9p~QdP{vVaDmvq7VpNF~PzKeRZ10xV#;t{?o;1Bq`EY9A2TG#}qMcVnW0bNOc@ar2?MnDv3 zOhw}>lpV^OOwqQU%BnV>t#~xDAM#@7mSr=EG56qF8S{UIU?HtXw{)H$RzIYk#CvN4 z4yTFVqQqxgWEcWisTR@;3y>oc#Nzw7Da@bXO-qTrjWAv`FtdHWvg6D!h>0=2jLl29 z90wk~%urrzq=^PZm^t1QsaNBOo1B#A<4r0_j_}zfJ8*jyM(;NM#t58L~XpmSLt)eD0f8oxsKu-lr`xxkuz4EFth%2 z{_&>qaPpRZLt}cwICfLD51ADy{8Rrr9}hc|U*3yPKUaDjm98{<<<`d;7uSj!ye!aT z&H3?0tx9n4+ODM{r64fV)L~lbL+&|=wdcObqAhaj^Un_S(00<&i6aqQhpV;tMpSY` zY#gWaI!$EhqD{J1HzYIN{)QjcEz$G|L5PL417ZKQnat!NCGv|iwti#u;y>zvwanO_ zS$Yhltb*a??&2@atHb5{brTsXjLvQ?Qj*(7O zWEm3WB{MUH&VU=2*J)`UY(<~e>1l$lekFjHvre0}Wp;IlKZFy7tkEeC=BJNPyaVF< z#U}`##)h-e}|Iug&&?_*qrqrzA$4sa4tsirAB&OaP>k>YtFX&!5&13!@d zo$Df0R=p`97>ve{JY)^k!FgL|^NQ!e&X`rlJT&(T5)*4-bi5+0kcdXQV*e>xe+x2^ zNS`%&SMwmBXKEc?En9kX0uirEt}7;8NKomiYTHLmsS~DNd-qz&_6dPC)e7U@Y$&x2 zh$=GZoEE^bU{qV>BkkKrQC^D86V0gqSyA^*-FrQxkADTiOc);f=}72vU-q7l6zP}* zJzFMkK|!OQ!aV}XHbR4g46YWN30ld7g=w(HrCeBY0$P4c!313uZ=8Qrb}srUH%OS@ z)u~gIGP6y{bLo&?2n59c(+6^)4n=6ak(S&g?n3e}*#jYzvP{zf%lKt~rvrVHayZZnD#tG#)IY zMkPd!cerH8>e7^14Rj6ZTWeVK`!uwI>OYs!?{9JM*J#mS^Qq0=-N|6*E=$-t(VgOk z%(0uTC-9ol=DwaXrX(ZbMoQ0XW&TK0+^7BD8x{2$fxn1a>_;P3uRpG0E-x>8TOrH$ zf1TT!s1Dsat=(U~Jo@>1IY)Q=lG=|}L(vyfTLeE!(gHDXdN@8T>O1kR?D(^TFd_e1 zI`JqrK86KD-Q0^V0KaI@?^*u$AgJN51axRV#U3spzzKYTDz|Kn&R|NTgtVF@=Q1psdp)dC50x#cluZMF6<>bgjV)XUbxU(zmJCkLI=c0 zD91Le>iIEhI(n_Rr-RSd)M9HHXnz>nQfh`{h+9tA!ZBrhjzL>+!COK3lmXV`Xt|?x zOhoj!Z@#TD^PdGT!y}+jlmeV$M9_*R_<9Y>bZqe^86r4$+C{d2di*TS5e|QT*dS9Z zhT+|r^;!2l1@KDl&YL|6Cj=ab^K8gfm?$55!p4u_*;f#`0RIx7HMg{hO{2`d*yXlF zrh%ai&NM6C;{uUI@pEf7i2Npnn*}b!ysS>mzUi#sO9BrxGCeH#pkdFeJGlUt9nE53 z;Yl_mcAGNp9>zq8z?_JYgkfDpEKk3#S}%RE*!jXTtxndY911xWByiX*rn;Av7^G)- zs+Nc9D+WtWs0zdgcneFWo*Dl7Qk=Kd2|SWPsFHnRABI4v^s!yky`*;eM76UZEAjPZ zEbm&~XFn#%Pn8mFmdgRpEzlhHKL^e)+nepi#>cV*k}14W%pMJRcsoy4o=Jzs@kw=CCyx<+k zer6g}&Z}Q~i*)FOHDz?o5@(hxldx33qGjhX$jGU&(Q6y6Ts>`b;c14MX&aXKk6u>D zO?`XA+WtR=?U&*6f}cCskj0~TXGY^&7X#)uQ==y){4ma-oSPEW$;=$*y%jB?dy|dH zLLv@njaZHjUKuhzf&HzD!mgUbmMlYZ8U3s_TML&z#?3gqJ)#A_(P=*7h`5qt$ShWn z<7PFtx4q6}P;c#O`Ya>h)!5|w?O)S^^RHU)c?Ewo2>ShPCu%fXB)an|t`noZz1`c& z<^RA1BiLbe@!{ln_lhx0>UIA!`HiZ1e@y@+rLkLpup)4e=nqcNK4W*yflvlAg5-C0 z`-A;ytd3ZRz$=QV82D;P|0WHH0WJy$<%7`u05QlD|0hb`gd~0HA7Sr=) zsLpaCiG5_lWGk8s`ESGbvH+oye)2|b3U44Eu6D@(O_{)fvgs`#p5cT3uA3)1`ZxNV z0b~lU4>ya#o^8e)eDCy%L3~jz^rADS`9L>?^u|Z6iIN!HfHw=3e;Fh`Vf&C66E#)raxfJAF_w;Cs%uv!Z?H6?QJNG^3z>!kqYC z{#pz$z?>-N-q{`|KCtG%45uI6Za|Ew1FoMr75 za(kipRIzeNA$FLArS!m%#Odjt%%Ca7+PaGyON%C0b|LdeCzwpr0vNG^7Q>%JM%c*8w|GD`M zw@DP4wa$&f%pWO1Qr9Ve`+&>qGqhG0^V2rH^@{WwTuOua{sb)%$xi8y|G+T)?eF7# zGZA;E>vGHS&3vbH;3gw&-AB>KpuJD&$)KglGw+%`KFR+&tC^WK8Q?zs@145G>J-kq z&02Byqs1o$DM^;6iXB@VU_&+pka$&ivwSqw82{exaH)@aR?&ljBzraN$qyKK1{s5m*W6= z0`hX7<98F5*}2D5;sIH>pkeJt`CkSOL;GlI>nH;gG|o>4Sg0lCY*dWXl>@YfG+%iq zZcGbq_?sxBpMn(piCs`|7C&PM#^LLjpZ+MDI_8d>`a~J?T#o3-E5VNR$v&4lWHW@@>M0`zQl==m>bVt6q?sUB0yF||cmxC&Swy2p-kqJD@GwK)m8Sz1fk zG*vlo@EddazQl6z4I+5+u9MJVNfrplx*}#Y2n|%`Fpmoq~i}U3m*T)HmtMiW%U8^fiA|t8?(w|JU;)Kb3$JiUz ze@+V)PYaW+;UNgz+<%fxjjB855H5R!4YUSzzNlmRIJEe$WE5^c%d_yDew|40+03)W zU4_F=IVeZ(=#$SlckhOG0r#BVw@V#26LF12d1TGV;cS?wAR4=_Q4Lo1aDD#&$EI{a z6sPu*w^fYDHL=>BcU2cJDmteC_3}oK1#g#bCp%IH#$K}o^G{t@q(c2amwF0yi8*dq z%zgx%>v;HA$Ci|Uo$FHKHJ}oQy{etMezh4f5REhgXbp`gR;5=n%IUgFIcUKBz4Ef*^cW}Kk zR>@c*Yb=InkJW1lZ5*Tz;eCWo7`f^fQ|1Xs&N|=?$OGC7>Vw48QF^4ICH{-+1V)tx z_EqgKbbr(qM#M;q*LnZUU3VaxP^&Y1fYtB=9CL1;)Z^;ogsC&kT`k9azUxw7#WriA zOX&O}vLxQq|3af4YV1q_n=uQo@msSQDpljurEuXZ1D_K)ztWKrEHl5uf;C?5`F|U# zJ!3pZyGJk=PO+dqla)rcJ5YDD&M%lN_{=3tdwJ680AF7VzUgC>6AdR zx$~!^D!qQZL@I>VK8kcfy64Ss0eg_@!9DCTqi`Iqw6LGA`H&>w0A0h)#!*$0*}!+J zVFs(Ypm-1lZDC(k;%y1Nswzn5Qx=q$DeA zlm9gsT@@akch&v4+gxY8F*on^T*N)S9__m=x(($jXhMH*PWW1BHu}B-9p3j9Wk#Dd zn`Djl9L!Iyvda{f>`(r6AN_)S;J0O;Cre98ZYutp5oJ4(WSeNo63f*kHGoPnf z`+g`Aydlyhwv(eX@59sd8XZc5v)cA|{aEJrK2*XdeyKf5+~9I7%5y9Ov$SRojSuXI ziMIIH2_&Ux5bHX!c}Dx_neqlI(=dcZ&x@t|uQfT38D{5IO3RBYD&#WE{qWzV0y!E9nG})09y{H!&Jp z_F`N3H<9towYvgoKdz2eAC6k;xRC??y;x^Ol6+kC-*{hKTE|2PCPX?l4j2g?InG}I zJ>j@eY*jT#;mL$pjTtXGDr-g|+E0e1xIKTl5KyAwUaieguO2%w1?1!0MR7r zd=fa=L7paLtq~-Dj*r_^?#+}_hV&rL#AD<;vIjZwYDQmzbXrKX(B!~rfC#|2%Lv5} zOAxi^7V&yh(auy~$qMzhvn6#hDz7TxD2(RkvQEJJvPjQJpblq=dt!6zzn4+ZCjQsE z+e-eM&@F|-ZhwO+Ckj&mQSJ6Y@%8V7BQPKQDc%x_Y8Ih$g9GpNNd7wjdM{ylPjM83 z7(t{D<%0mZ5R=W+e1b{y8Ku11y0CCoI%0Q~4^~6{yt|3zXrX23=%Ygk=Z_vuv?j-x z5_Mw+=&b*tM+!+(8b) zmrQwm99o|5%+A+MEg71pcwnwKKk<9}YI3$I2b>+>iR$iK_OP=@h?0`H20!A zUGb6R3snS%`lMR4;h%Rov&?Hm=>v#7pLz94VpZ>|=vepSr+f;T=f$@DiE*R=EUrVb zBTHgiF(EjqLG_jTTmI@Kt*f6N7lm!QEqA6pBg!91Nc0D4D*w$wCQTL$X4W#HlHvpp zzb&G=I;D$t`Y?0Zen!QGc-#5jqE>bbh*ZzW1nx^n(Jn%~w~>oz`&G~PdvZE&Dm$@4 zPFX_;eLFXkRQ7u~mJPghG{zOYZFGe<_x3_LV3Zj@i8|^oqMnLgC&EHMpUKG)Op^zo z@=QUZ+MQ4YV*F^V1R`>4m!4D`#@z&c6*UYPD(w+F{w`h< zDuH0Af+*MGQ8>9&lchjxi5Nc!56>`|^6|5VAdR+@&!)0BL6QgpIzlI3 z=AC#>7t`bYTGitFE2^h9T8c@%Zb;+fru*c)vIJWv=#2yRH19kc#W-Ez)XzVvCg+OM zcj$@F<_dS<*(rK51h|Tuq)s_?2kZ&KzPKUbg&C)NibM0DPf|T|PQcK^(A{0q9fJ%Z-5?=D4<)FGfW!dOEr^IHBMpLtfQkY`h$uCLfFLD^bV-+I ze*bgMvz~XXHLqB+=f3Z~uj~43rTaWOT;3uxp28M-y59Ra><-H@;G?XTmC-7ahBdQ5#aTVtnsC&?uEAdyC_Ow z%T1uf6z6*_?k_R0kgaLPx#RZZ_>H;_&EVi=ACur8bCb%zY-fq;)p>#O zEfYG6%ZIRTN;3-&zr?E%!0@*;HJT?3;Yw1$; zw)pxpqK{D~lJv3fKHtJy?^}|aAN|lm5r4mQFU}RECuCa`-Rb=6VfKf7@sQb{st$qN z*s98$-LRA+>f^}oMAgBjEqgczc6NT=of&3$Jzp3$EGr#R5o3DlWNpwFn`AUW{;taB`?)^^)@T ztGv`Z04l5zz=EaTjjdDwNYOnZe;JT~-Y)LC2OOb+=f#r|uxrIN`Xq(C*J&c@6gW{$ zh_K(PoCLI7i#xltFo=J&vZ$qgf;HQ)jTo!di(cR_enCgvN;2ZQiNC3y_!WtzHHWrg z75sZFC%TKbmK_ToKYOWSEh*X-4EJT{sDx>vP4s8Q1&^uM~kLk;?d(0Y2CEFcs|JGe%EUzNPT zi#YUwP=*T>NnfUe4tNn5nzB!Yr!K9QDi^WDv{px$q!(YR#&{|`sgX~1t8{|5`OY+d zQ!dPd?~D?=EQyoyv2^XF#?yB!(Bpq+p;=m|e=PG%e>E+{%AX~M%f!gOR*0v6|NH{h zWpw54goeBKDI$_QS>sL~w=69&1@NK$cD`01=PlA7%sAW9rkB`U8ix@#6IAN5n)V2d z8CHx-U8K2SI73GCs_fVZ&rCQuGDcy>i|Vx{N}qzdzlTu>4-M+iK18F@JnH{Whv83M zqQXLtKh8TVrH)U;EpbJJdcSw=j|n<9)9acN@e^#dNq0y+s2P-7Wc7n5HqScfH$2+` zZ9iyLTYdEJvA;xdY?N2sfBJVDsbn?vWMr?lo}v8UT9<<59d+N~pLw#w_l)*JUqBBu z>!SMqwtTP;pnlngHEFms_{*7fP(J$K(|<3`qpdvb@bf`w9j#T&FLfqK%iP5<3%(<> zmt8l{ln+gw6~A4r7-wz1sts)-9_{L)mj4^0%Df%@%EIDURCH}OY&lqcD=EZMZ$Rn8n~s>Q6l16^u&|5WrI#*83k+PmSe;9z z&%Za^ez?cq3iVpl*Bl#B282#$1 zRKK=Z)95dUqMY(zyhHhqF&4(vMWQdJX#Bfr0D64SLZ-(PPag>}c>N6o2AL+KOY=jj z5Gnqo^LBA6Rt{iT26=Ey88v0;SCG%4o+ojEqj)~Ix{j&puCCn2nnO|%HwS)ACuA`OJ~J*+Ym!G{59hM0 zC7o0$hDA*Ffr7BwPWQ0-F}6o7r301ztZ`i(*=$)ijZg zk#J0bdr_$hPyD#uDjvwklli+uQ{z9h%&oK0rwPq|lL;NC&pXI9G&BxHInIIJy=D|x02EqfRpXNW}xTxZPG zd-vGsq6O;BM`3oQFT`75@Qy*dddPnurQ2bvq+4Adbap%Wlcp7P z14{S^bV#F2D)kO~febFO811)bOJCH-#_iLJxUbYd0Th5T(~XkrNH`i|;fdwofrep4 z(br2R8rdua*v%ZdfK*9@|~r^x&P0uKJZ)7ltd zLMN&3CdR1$h9n4)SWjTyeW7nAWg^5Av+z(pHVS1Aw1FqJxhVC?XL+lG58 z-a3In)4yJ{s>z~uakq~T`*Qk?0G7wf^rq9x<_6VfN)cL`{Kp|Fxm8E*U?hs+iSDN= znhTO%H!GmL(b6^6t#3gq7{ue~q11k?Xg+jK%l<)C#4$g-A5QXZl+<4!^JwN3Wn4Hy zBCI%0e#7xQnRQpkR4jbWo#r^Xh_uW@@_vj2*uQGM=!nIdjpmqi)`8pTHVlOF{L+FyYqXIFxyM2 zX=sOgt^0R9s?W7<@3YmnqXDA8oUYPmAp2XRZrsMVN+Xq4fXPI#gw^o=TBC7|xQ)%H z*B-pqg*~H{HwxO2|A}Y!hRwDN%bZA@_p>YY>L*%yo~$^Hgt)s$BY%;^7Im5pQ$+Q!_Z_9s>K+n~8wAT4FFJW5Fj#6F zl@&OQa0dsxN5{+aCFHA;FYpv6cH`rCumJ)*VQ#)0Nh;$YRRO#%ipFXKOWFsq3^llc zF3Ta~{R$Ts+r%zV(8lNsPRxR3d8jcmS$rydSr}3@nRJYl=lKUIEXvrXM~|=7e#iHE z>P1}u8|CvEb#Rpn0dk_(cJYc`1YEDpY;g+^93{$lUtQE=MP1ec941pB*u025ESG2U zr;=KvY6N;XMTl z%qm!FglmC1VAnt~F>qUM{=;M0i;+o11uJS^a9X80o?5X~6{N{=zQ7uKZgKa+aFVQ6kQk>#U^l`K-)NK!V)#GR^=UFn+9n8lj+an z*1c4hKTDjIRe(T9O<6@FD%X`h3a0mFtTlQnQF2^Io)31ZB#1WL`HM^EZa}E;(d;o^ z^(3Zg%`f9;-exL**UX-f_ScS%yZd{k%Qr2tPLmp!MOGEfkR1Fh0&aM4;x{w!n%5`pXQGCkMT93m3Q_w7OA2V&)Dw~-#;hqmE|X4&lu{gT-)q^_j^p| zyh+JPxj{LFdh;*vm{6+pBqVZ8OkudLzqRr$(P=*W`H89xF88ilDuZX}t^X-zMwg=h z##MjH5zqe2u>bP{sFfDprdiiMQx#zOG+Ifm?r!d;TNLwXlDCjey5jmlp~Kl(O9cP0i42B>yRb(R@7 zG(qG^p%6_#l0bd2hz{paV>7LK$qQ-%bxsf-qdJNNO1}u4jWr$DZ|zh(r4@k&6}O1KfP{&-TR0ec3R`SeNVq3l z@njy3aA4^;g}C)*PpDj)McgPOb4VLjMPW})9x1Ht(%tHPc?(p*xKFKk*MxhNl!u31 zg3wQllRVKDe8Jysk89`Em5>!0>4cj^%+@(8CL1mxN&mTOSt%!<6%!?JZzqduJ1Fw~ zk5BvpD~=@*iPW-bNBchvo+|<)y~{;y1!5Etkc-idnpu4h9(OA~+YHT;*)(dJUH&4< zv$SDs50$mt;Imna)6M!@u5bAWicMg`Tn?xpOsaze&UMlWkD?`}&z0gQ+mD(Ti5y}T zE@RCV#f)@HoW_MgDB6YW-)_#{Zs1Hp_c(OEZ0(IjLEmM|hnraEccUvpv&^TrN1x`j zZKy|942r)#pNhqnSl7NiJq$Z?fAYfRc9dZaCMJ#Kv6$_X59BYcW&bFQ{rx6u^CS4> zPgR$B_28V8frn*)yjo7=->{hbdc=jpjffL)2_eU zSyr*yAysbq%>mJwpFe3|MjAdi$*fR{_kMGeX=p0K4HJNPHl%-qzq3gk{A4T*IfBQI zH-FGeecY6Vc)$C3*T}y8XEe8@wvCygCz0U#E#L#eG8Dju5S{o&%?tT_gxfP$e_a19 zD{3gkU;#S^bLVWW*;oi?I7^Xq62S3m@wvTtCs+fsJ0K&nDq|4*7yuz0MCmB1pt$^A zGW8(k6?2Lp5N`1qtq4i}qD8VSl6}+;!zsTH+#-tBamHWhQNSo*3$OyzO}jMj`ROP` z9!Vwg(_CQ@C^*Q6pyj(@_$QX5Mw}!|tu|7i5Xa4e%(Lb?iB&){;+X>g=|#1uBw7>e znuleHidXSD7?pKk+0|&DbxhBPQk5F#vtDUJoqMf;og37F_2fV`fo|_o0h!J7SPhyC ztgV~_I(l?}v*FfJXg|oVU5Z^1>|Z~2rV}p1-ItiDqMxAXMj22M;@=z1ffw8KY!!!> zqx|-UtPK`;aBqks;dL3C?T4GaffL}bW15p9*y<3=i8OQ0aa~U1^xp*PvFXv;sI~8G znA;WS94@q{eFYfxoz^IETR?kSCmN2KzINBy0g@h7;&K+CWCwG|?J7bw1l(qASZi2? zCLwNq)q{`gJFdceLsM&qD-_OtwT2!DA2rNz0W?_&@KdKvZ+)}h8Gf6Mi(S5lc8O=x zUnDGoY1aHJY+qQ>yw@w`+W^=Wa)YNaSWdTq>ALBW7Jpg)+dI_Wb?(&inaG(&PorB9 zldFFH%W!NuG-~HKYWS-8Z@N4N8T+^WJMOw_FY zT9RqEFU&q4E@`RIIdVly>bu-+m^th@L_GOm{Hj2*05*8MzUIf)9UZz|zWb9}()GSs z#TLuORM6?_L#fGpja`rbPr*tlW_Av?c>gO!{Q83ja~dr{-^Ny#V_;nM%|3Ol>(#r& znUIP%}y}z8U4|IFZ6Vpw}IHw4M`5hL!FALD0d$+(2po0Ni|>M0VMfuAOG`5 zqPa3sfG*S~6=3-1s~MSPprktw=kp81DV-e zKSep7uGGU7W%VdHHvSTY2hp_!#yH^X5XC=?)o`XfH}VkK`ZM*MZBx1(^_0Ss9AA~v z6tDvI*?XV!iy7KUy+Ph{Y2`P2}l}SUz*O+Vk%TIgWZdZ$m~6f@X=9A7l{aX{Vim38Z`?CHi^7Ikl;G+DnHe_*A`Rmma4Ps;L{kvpa1K)9X zF8beP4D2+5;~6YLPp|vZ@%Rz5yD09&@X4rfJTwlIW{(UbRO>7%>g z+o>%FB=;~peLCCs+135m7PvIB?V*<5`s-?#qx%X8Px=$O*)+V>kJC&RZwl9qe2f|DgoXlfcc&!E%T?t^h*u?1YSQQ(%K3iU>VrqQ!6wLnV8|M z2rY@eI5T8K6Svx9v$H%iHtAfp4M%f{cChA19&|g20V=_nHJse%=6|BRd*$BB@197c zTI+}ZKw;yKXhC{vgv1V(2gdXP6HNr`%*PUL{UNt)H1OAnVF&I=1)h#fO8-T(V9V&| z<%l9NY%eLb6}ZLN&hw{_TkY>L=B9Y22fmy4DRCf{5tHY;g6+8B(@FN_Q9Z;-XZiZK zOuwd~f+tTTh=xLm!$Uvc^18Q7{4k%K3;f9|ttwdlmp8>}H`h0Lz_lg9>JPTK#lS?$ z_6K?HFUc_C!a>@of5*{UKZD0ij>1$-2Q;MyI5^oSRZaOp^}X}jl4PnHwLZIi7^|^X z_}!&}FBX6C^HEp#Tru-fui@C1@Ug1Je^G}y`CVb-5>e|mJ#zM`6B3RSxM-g7pDUhc zPk+40I3mQp zE9$guu!^$HY7yQuU%I!GIl(0Dw-|tGJ9N2OG-s^+g}t|6xvfPkQzt+?k6w13vLozy z9AH_U?kXedh0Uv9v?!VBo82uG4gUbN2v4ZM4L>`O4B&<5ET?QNXXANMTApBeHB}bStMLX z9-#X%(6|?1bEF!QbRlV;8;J;UD%I%mvA7nE>R(K4qW7#QK2gq+JJEmf*fl`Pam+*- zYHq;5tm*?pe+#LSFZ$q;>;|5qCD--fm^4-s0H>cxfT>x3T>rS|b9N6&KGkWvLw^Cn za@d_WP@dO6Yh6=IyuIzXAe`+481o&G$)UI$V`?`OC>V-4IgEWsd@YLdSQ|LaHD&|4 z{yg*#kgl2F)sR>o4z>XhTRmS6It{nFZU_|XCReqg+DD|S6f04uUWZgx()Eez)7{+e zKIZ5YMMMKxM2Teg`i@0e3FRk!O$$#HZlnV%TW$?*Bl!58eceI5mbvZt-HVTdwPY3T zG$jOXjF-{Pw4foxOGS`Klxb|(z=-CJ28HOaH_{R{#(FHyR3D^4t_)sjpaS+y|53fA z!5z?jn3^W?3h9v#=cbPnh`uwEx#2H+%Q510pJXq#cdl+bYu0m`MGdq}yo0eU&Aba# z;s;?akf(38&flIXsyaT^$kN!&{oe+FA8#TpsUH&2)*Pl9hvT~qnUTgHDIREuavI$f zfi#sJ+@836M9gfkDSO%jTQf6nY@pLLm!-^3bin{cjej9+Fri8PGRS)`xEKeJ;t>F7 zH%as^%>Roaj2Q>PXXwMi#;Jn9Sip%7z5p@+ut1qI=xgXQOVHOzVTiu~3-}RFvENTV zf;vT|Gd{s56Vb_ny88l82{2~|AXR)1;=)0gF<(l6`Vz&O&f!lJ>AT}#QpZ0EQ+D+; ztuoDEXp3({d6hvh?zqH|*Pbox71fYYTsJRnndFAQhTIL;YuJpcNMlK8`)*2`eF|3u zsfe6?IRlObDX1-krW>8=)&u8NFhE=%ds~r4KyavQ(qGWUL)o2z{ZK5BHwC}YP9T$y zpyu$Uui;~(oOyO8him?n(CjD*P@m_Ar?#Pn5G0c&4Y67_@#}o8gMV)taHA6>0s_G) z)o(z}-1weg)ZH19C;s|&osxXrm2p>Y_h1u2-FBqI%i=uDt$-n3$FJfIwZJ#h=_Hr6 zr8ck?_QQAZol;OBjO10X!tFgpAN$!?ivGT~(1KnC_w+`ZpZ4=+hXxf+ypDSxAU2NP zLklk~vAf(Us7j?2_SIW^|lHNr9nkBO4buVM&a6Xu< z@t*kE%u^E#^HBS)-$rvX@=Nl*-&{O>$I8;lF5Vjmj7uMrP~_Yxo_pd&CJZ+pZrSQ7(F6t)s2oXU+UR_*q)R9Y~30r zA+)S{s|G#lEA0jw=A6;{j@E@o4V;58GUdf28iKQ*G6w|EB!XNv5HO0~4Ab{K9#h=yjOvAZ#(c-{JGqoieO`(K5={Grv6n2D!iagkUlJokuM4Y!_l zxkh6QGI8!E%e)M$IrY>Zte?|H#hnFyha3NJnRn!>$gcT&&0h2W%`w|P@!Ze-oBQ)= zNY7nt{B2n9DzpI9NEj5yC$DPsTg@aupT8G!8`0*Xee%LCCZ5J%Byo6ZWPCGghqZ1M zZ3WS!KS`7)HFyoMMN;=ctm&!}sCY8tTJF&{d{HjuxWE5_ub4&!V6jM?3oPIhS6hSk zKGVU3+5ZWUiOt@l`iX)^PmOg+6$ZTS3S*z+oCrA)0*6t{-;0b=p-~T-33wO(pyPaY z@ob}RnB9c~cKO64;_iO~&9A4DPrvpst491bPek`DpFCxUPf=~Yi2?Rw-cmV1vsmYM^AP6UBZ0g|LK zK0|Uc;!I$He#0DuK%&SErJZS=N$=$j(c~EidSr(z(8G2gVYtfcKwSBTuvK88G=3E% z!ozW-j(Q4$ujxde->up06W}WXv{ymqX>>QsDl0T&3htXMJ*GFOvwvj@zyrJt#myh3 zL$O9p+5F!@060Rq=WkU!Qi~?j5*&A`eS=+ZC+Cbc|Q4rGlDjo4z#rr^Xr|RFDPWdA@&a6B|V62znv+m8zWyt>M zTpTGO-*)X%%%wl17f@v58hI#Mc` z0CE600R|K~2#%$}Q}Y!O$az3+S&2+z6>>OmQ2gCKY2Bb><5YcfaxPj8(n2@=9&KFC zzG(OL?HXNA^Uv~mW2*o?Rs>J|+p39!HdeH-rh*jX<{FLesl@**$ZK$;5YEuMB4Ra`q2 z%S{xBp6@RsK%dXt2luPY>}-~we51FC*BT+;Bn~C5U&`Z>gCi20<)M+|+C`1$m}$W$ zA)`0@?$V?gBFWS`)llZ3nG=XJC?o%6k``5vXu2(brb*%0&iDtjNOGRi+vQJX74m}l zX8qHZZY;U*@!sS&UR9{^f7#fsZ!wLJL40O$R~7rcXO#nRMGo7$Z7Kfxtg7&`s_2Eu zGgF@zea+LlYBNoLH;TYi7u%mkJ4GJ%7HoT(rlJZR5AW-bgPLbb&41Jh*V+QBFRSG zkd^Y_$J5`T`}m$7)cdpb*WFw1=DFJczSw!B;5Wfxw&Lyh7RL4Tzrt|3{q6dHg`xkI zcvf%<>i_X&`uyLik;4A=@#o$Dgb=fBn)$h_>x z6-dWE&aZ>X?VZAO?w8_Suhiw_2(U8kTFye2;!Rl`c$HU=rfod)1waQls20&Rt=V`G z94K1VLVq2ZS+L1gCZi0c`c3A6P==!)CB)Sd|dxQYoIN zoyy-qrCnM`dNJ1m-VNe5?M|*!3PL;_!+cj9nvW#{sT}|64pjdCEPxS}64|LKDa^7( z#I|E`H&%X1)TSIhU-0j~M)_TOaPg|I4T@>jr}ev6qD6mzu&L~)4j>;d4D&hgWaH<0 zTIsj`n5v#G%g}>wYyKLxSBM%byy^DIRcqtnBAjUhMhdD{@xYbO~JHVP3uvSvRUg?Y~DwvU2=Jn zbNS`@r}jeH52k&q|Ai$=&I-9OTP9 zvT&Iq-5CuR7G6&#{MBv%TVd1ja&b#u5)~Aw``#E{%QkZ(m(1XlT56dH(ZH`Gjq( z>-t+3-1k542aH|=m@ixpYa;BRd5VaWm9yhgN4-FL={_}{{oOh7P~8K%s;BD9f4*Fx z#aJJ&I%TlXA?c;f<9P&;xylk12VBX%s^s~v#DMrCEcf^yuOWb;k}_ibrnICoD!c-C zE_Qpgc!z!Jt9cgqf@#G2XT`%L*)dmSJXoO7LgkD+$hP>?>T_mf?BpZj>zm1+FGy>v zD_Nn$kpDByCW$>r0V{~Yy)x0?me{Q83oDN4ZJuCO#m-^bOM@PV7#!HnjmSb+r%BSN z$)`d429D4RAjmw?U68t3azdbAHRe}6BQl2{Di_@IK=C25HPkES<6Ce{kV`?*%k!yH zO6#&M^KAHh)eAB6$`^RUdn)fKNnGC%Cm{3DMVDc?Ve9#Gg6aufHrbi~$p?!y-zy)0d#TTC!n zHPC+bnC`2}3hC0|r(ahl;Je+iPf#0@bhKVv^>Zw1_*D~)Q6b0gN0%R4^jfcd%xki60n7mB3ZpxV+N){}e@m-TH@;c3=Va>sj0gFDt&ru zH#;uK2WblDcoOD2wn^5+ ztBZ?t{up%orPDR(>=GAy^GNg5^)J89!!2PLk13mYv^s&30dNjofYH$Ej`M->-5T|A zm6BX%Ho8HvWF?<88Fh@I-3JM>wkSkeXFF2l8eSO#aq&wk%ioq1x^*q=<__-L<_ z%aH{G!1ZDlTEzWNKsf<689Iy?)Ze3h)}yj5U_!^u^v+j1q&7~^ntk%G7vw;2$AfMcG*KFzrwUdJCr$QYQ#qv1oQg zQ+mgyz}F8{-unuGLkEu54+nZxFDGxuy-K%h)p8fkQ*PJNFYx`Vdm{Vj-qpC5r@-zLkLpZjy++f>s;bV3#-ijoC9)GHI`r$)t0>fxdK)vHXpMd`k6 zasb6G(d>mIyT_O#r}RHf_W^<9VkA=BnZWrGzyjnIbo$yG51A(r8~1glE@Yo#{~tgFv&CX1ngP#NapWqyB|t z6zfPF;BaN%#OegUsu}O4`KezmZ4sOXzlzn`iwR+6@8u%ys#%NVn??j%F8?YIa4xtO z8yI^F(`I4rC(qZt%dlE5(iv~HW?maZx>?E_0WB^TMIwOS74lVm50;S#jn=?dXI`H` zw;kQ%;+={mcf*A6lDSd$K{M|&S0yOWZ>A`a;KJ2^ zp(jg84yZdL!o(j~PuIQ^Fy*R5OU2S6XsOH?P}b^H!shVUl5Lk#nV%g~lq7+rQyjT% z%joRg#Pk!@9IeSr(6P1?EPe{4Z~1Z|6MHdl{*^dKln{jQWRCXrMI&1X9*UWgmQjb_ zlP^+!@_nZJVr0!Vl``H(75*NoJh(OL1?WaX!$!20%sNmHs3FVHw?Is3j3tYor7y_9 zUIk}huh2`X`FSKZ47oCy#zX&?6i#{IJ{J_Y%F~7oXJ^dr?FA!6w?RBK zBAojRLx{4rl_c4a2`b0xzXDE{6=|TqocWXDQ%p$nKdcV`{_^lA+VyLe z%Zy4eb;K;tPSiKFYhZf0tkQ93K$E?R@gxv-kdw_dN#YOHY9)>l?i)i5!1Z5il*_rk zpq_&WSZh!(R-lm*3pIwU1GQij`8PD2sKe=h^aSie6a>#CCfvID?jy$;*GhG)I8%}e zZbWj9y`pv)r3nDWS2G`^{dz&x#8OhtiF{QX>$Lg-H5n>AD32{#?;F4>P4&xMyC{K= zDzRFruW0(-dvO$#QX|~r+9=-p?qj;Mq#T}&xJ%U^LiFG1cV3$=LHhw#ob(n3kGQi_ zu)@>Ao($|$_PCR8QwQX@hSatE1%}DBR2gBn_^v{fJC3srEb=UoLvE->yEmeQdM^0@ zJS`J)klH>#uV|wD{@!a~+L`vTOzACAOlrVp*9{6`wpaXO8TU*sWhaw1OK268-pHH! z_t-7)A?^e61zUNw4L1_0&=A;)vf_#&V_W5E2db(;MZs>ZHr7jL)Z02cz`vd#j158U zAXR-&H>e1W%U3=m#0sbsCzimvud;s{WBxJm>=@h;4Lba8_l0Kl-$~}b{Feu%!r$9i zh`Dwx)ZZ^ifH?D`op18|l@UBRh{Da*z#h;h@!f@zwnE85vtT+PYDLOTGUnAq7jr%M z6^Vlf8a2+emJz;cw@mjD3FgAHj6)=1+_98+D5uzAwz4qRYVI?=NaKqu6598UfuLQ3 zq7qZ09jB;E6GZg`WX}5-_K;7=G#%o@_uEtfBw7ed+Uw$8U{^$Q*8%#iq)5j%3XNDH zl}PCdpI>T`ZuAR}T2x5egE|VR*ug)^IYziti4Y3?lVOG86oW|}di=K}z8pQ0GG#Lg zK06=`DZgO;oUd^8T@5VZ=p;E5&530}U}|FWO1{x5>bHVcumzBm(UK0VRqK<7Ko-u`nUos#3lceR9J}90;Cch?4&)O-%)_K(M-n!^t zQ5~IN9W#K-U7hzb3Fla>Vyo!H)0HVhA_e|3)l9#!#*9m;Gpb%nEh-&Z85GW4ue!79 zpqx#!b2B&N)}1$&%J30o+s8_Xq{Y9Pq}J%rQ03XE^Q|=^ZaCW4=d0*X>eU337OVrfExpe&$Nz`nGpVVlYC#^F2THMV?+LADiF7!KCg@C?9% zM8NNHObEx)36PmG?4TGv?UjCQ4uY$a0{HlgIc|_H142~d`(7SG73U4^o<;r`#zIgR zNOm<*v-4V*NzG5C$Kc@>3RB?7K9G_H6lh^*LwGCAw+&!&Iao>i0Gr=rVj}dr)H5-X zNbLyRj`EvWQ5?Bq14Q`CNxa6m4!eBn;TsP)9*U$-GTj2r8w~2)$5CtnudykeiN#vM z{n;_A(jq}NW5|FJF4_c!?xeY~wlu~_XrCoJ(u`sP7vq#F%NsSye?*}1{;nq-q1(XfpI-u4|`w*M`8ic_* zzvv(FMOjzdI9Etds3h=0*)%84crQ-)isYC4-xb~9`zOCIiY|CxHOA$zq ziT@R%v>3!mL3De7Y_M{yx2!v#4`qvyo4M|E+82r8bNPZ9I_3C60b6h^BJ4>gJ>}t_ zO4*6RUo1G1LXfgKzFHhpc@3k%iLcI5j57?m1=SKs%V<_(Zc z>ssAH@knj-vCB%=#Z-c>eWw93m`c1a(Vvn~9-AsF8akgpk+S~<3_jcu3*^IUMZm9% zpLSnBhQJ%U*_OrsDq)*CYAe^+?drFAmVf~wUI6&vk9Agl`k=-@nB(AE(=;F%e302| z0|bSftyNTbYn}gCs|}3wTT*rFufn(tE zHLO0#fI}Npf1?72Odco}q($(6^G601IL*kh=2!}P1W0iA)%nBRTZZSTUXX86Qc1mZ z4P2xt8EisyoSg-I66EYmGB`1d8c4X3@~R=w_qv`Ly^jjBNai5HS*nu63EQivc< zQU9OYw44GIWa@fhSmX(l)qlXDs)c}F)Aj5Bntp(q9W{c=Q6P}~NMwkos_ z=3Alj zcNM04j~hH_79I@nDgeun8kwlHJNRpDW5S5|ah(k+qkCsUig3Rrl8Y`qo)x-u<|t|A zXaA-Kjml9)Q{OmJ87zO`9sK+Yk-WZvZ!bEd&Ni$m{uB%euDyEgvPITp-lxTOGiGmZ z@1XH%n?&Tl^Lev>M}hzHxOTBH`9`&tS&BzsQp`&-;uSKUL;UMdo((pXM`S;i9UUEw zkB!j}hr!)g zzPbI;Mc>%`B+3F7==&&4XYph*349NLP9>e#o#vROzk?w2nQTBHub;NU|8`ZKU0C` zqI#KFpt#c9`&|Zs<7>O*8=cx9sl9jMaKADWdTNouRfyzBwC;Hⅆ}zGJfKO>=U(1 znMvu_ZN)UzFiAA@9JK5;VWh^fw; z=lO91oYS5;FusfWnDcr{gQR(tcD%vNbEYgZ5u!&P5$_N1ZIL#ZpREt+!}Tz6C-9J`%U#_Mk3kkA)hBQm8~jJ!O!^ znmeDIgRh*j=eqB&+y$B_I9Xq=GjD_V1aeGcYD|20v_wRfKwZ#eoo-_{0&5>-Mtfgh zdtO=#UZ{u7Rt-g5_6Ur~{y%bT)BD&5E8I&+E>$PK5049810 zecTQ1r@eOAN&;~bCOCWN3+Gy6zm$eL&R93+Pe&3jNYmhS{9oeiVfK{*!QeXoQOdY1 z*Yb@-h#o)<2;#X%RYW(1$Z7z^Sy5~r|3~|NQ3w+c*e94v9a6&}J!v(>amDOSoES!-yS`u4`aIJFO4S!nPgNq$z z5#H-5hy7Tw67by;2MO-_*mTycuZl0e*iFMMO{=3LNW8$ZAbqFLW(#a*AVTPlZO6~P z%O>)lZUOR5tacws9=%cs0PtPcoMA)tLyj4@`lUs?MKfYNP&*Cf93QCY-I* z0NIK6-LMRhMH=trIMe6M;cCfq7WX$K$k)Uy%NCp2cDjLXv(v;c$Ld`D6_DRESEb1t zAGU>A%!3^O*N zW8mSYt(*+$fm+GTUWXk(mCw9X9)6NxiUqb=EXP^Yf@HNo1O6%4e>$<91Jt7~Ynj2r z`IC(~k8`3F{&_M#*Cr?xG_Y@#7XQkpEFkeoCa!=OFR&|DxCN06{+rorxA3&%!n5^v zp5y2Z;TJ@PNH6zL*iH;+`cn~@iFgS;A=b@(1&;IC{B(9DJ%Iyak}a$K>aecDvp;X^ zSo7EuMv6JNGpsU37!^DZqJad1cm`*e39Vw zRhCtVG9U%m7=mbMV|0xIY)pd`r2p2s`f8)skycnQf$C*qxL{$b*Y;?qPrSufnudtH5|NI`(OxVU{ zWMiH-53nd~mj@QV+of-M#Oz7KKq#{!?yg#)r($x4ee#q-C_K|QJ@!k>=jDvW8j-~N z6k${&DsGX!Qn^d*O?cW4Dnlx*Tc&WH^dHAGbk`5Y_IEe7S>dRie;n{edr!X|T7TN` z%K3mNlJ()<|HIT==>`GG(IFtl0XVs0`K`1fdD#K;Q_? zx}^f5M*u;ct`W~>pCbz2QXm$k3}81hhIER>y#%k0w}I`Q8waHI120Q20H@H^wu%d z(?nqcN~wqX0W1fkb2CU_t>xL#?l_4bndns(KaBuPd!qcdjPSa^N)e72?ZW!X9vy%F zij+~1xFu6%5qxxxK7?39evgyVd6ikpI3ya?d^2Jqu#0^9dOkqDa*e}*K(jpHhU`+u zFXrGjM?opSF5?#iFZg~(Ne2HKRq5P+VSA8xtt*FaM*EO|xi9Q)6zQa~b8@L!xl9Jh zoK=PTq(r{w$u7(Ct~}vW)yX{E8gg0cUl;6*bs7rRDmxc5VhQowv)JZg-T6N=!x35a zZ9hKw@4X{c%osAr=3$KG#?6jo55FEW{^%_T{U7j2etffA&=MIalHn{`ILl8 z!iXpV7L|7_ujvV-=ro>9n1X4HoaPQ#(>DCLpG>AXKOJ{t3Joi}Zpmgg4(kS7V;@6} zzcCKA2{E-If4(xDBq6gVlHr?0*_hqL!vWe=G#j$|HpI*k$=NKjQHhtQ6T_q{Xq0-d z^NRDQP_e}}h(dkEHLsSe>p#w~A>xH}1J$wmu($s2jShLvDaiw!WS@}n# zSYq)G?9)fLc?kA)BA`6J>KBJvRtbPz>Z|C}l^tw=hZ5p%I+hT0ZzoYPyhi{o3Rx+1 zY0+6H2(jIYXUPPXR5+QIfyzfqheXy@X7a0$Z$MR|aK|Y3lpB$<=roxMk&5(A8!6G4 zd?0hV>dw&RIQLcSq@db~%B)ZME_IC)T+ZyylYB6GH?%^{X4EeOPIdL)mxg7K~Ed zfGz7U;QJccbk*H;9L_m7>kjJX z9xwohznhbF&wqD-@-69!o;Fr(!<5Ju^p*V>#EVDA{B}NfIq%Rgg zrE8FUAVo1W7^WPwxzT0`pmlpLkEjEe;3L=3MEph8f7D9(>c36G-c}VvK;TZ-uRM~> zZM5KZ&Tt|!GyFU(k&n-|w$=BB_Mk=Zj7#M0kh34Tp2$pyYt7GfF)2!Gh3de#5Gb~e z5y7WQw;87f#i2bLxO9=NTcc#tkbF~=fcf?;F1BN6zeIA$79sG3hL?&K8GB`pueI?> zOPqp0%@v(j-?{YSGNFfB@u{UZUr(Cz>hF8kw*gC;Dsngu{^d`%xBH1J84&m_|Z4>lO zJhN|lu(doNTC_+M*T2(@OrY`{dLX%u9VmLb0XuyYy~24z{j>woU{PKBKxEj@o8D}j zG!zJ)&X{uXd}_kr;o~0iVZ)DzK%M-17aTM||DvG8$1#fu4u>VUyM5`0-HIL3M@L8U z7^}eB9irf{_qW2=pHuMhR~Iwj-TlS@WicmoODXrcTnv1Rj1WCt zcxpr8xo%JYdh#rld39DsyxpeHvsgxBJne{%;Kse{d$6h~o<@nBSzaUBIgvwBxjQeT z$Y+etq}u@l6guJxjR{{f0~baBtqR9Ck}y5L@bGie-Uc3`gkJ$7?-MwaNE+C@xKf^M z2~1e5HgYUQt*k|pARUU$APC@<`5ik!=3iq5KW)!ysQBsPR=&c$SM0B2DuXE#jLO)W z@-ChDSD9E)b;|DbE12AmL_LFCN%O+XdTENTK*KZ1RCWXl9yE`NHHIfB*rPO=nvKHi zi5oJ#k>y(7a>97|O)b8%k2oWTAicV+AAC*#tTc&BV%1`^lMP+0K8r=GYGV1$j0o+s zceh9q*7~enSbM7Oa$HMkXY77xsGNI;IDD1MgKd*CFuR{;g<&eEGLhk@>;m%XM;q@^ z^3&K24yQsLoBPqenq%JVV}b72=2@HWW{#bv!``T&`*7i}t#VJHf7s4I8)mWajT=S! zJimK&Mwfa19#{Mr9vYwl3z(Ijt8)gI2_oQVM*frTj4{|UJ5Z_OxTsl*_r)V>2 zBs|NbJ+`#BJj2;l7lFDPe5n2dbp)}FE8{>O3MfYt;45JpV~(P$19tgm(t91 zop>I0g> ze!>YGT`kOtK=}>&Bkzk}KENuHrQBhb}co8XbonR(AsNVgP0GCS# ze@v3I33c6E8O&Sm)=FUBz6A^L&xY7=QFVtYx+oy#lzz7!)*bB{a|quXi3&}Q=YOA& zsXe*F|B2LER*(ql&sUx1hYTau9leG&tsT5{8zZB#1(w1Pq&4IQz=@VykX7j2w4kda zQ4s-1fRF7+1gNFG9imt|}7#s_v*-{&Q0{v>nn@3E4Q zOZKFeK&}Y#YO1SJjRGmrU21dCZvsoaodVA#@KgsPu*T~extCI-)90WekiRlg>?_>Z z_*oqRycI+jE$&36@KqfZCnBcSeg?L>q&DP%fNd_c`y7}i=Vpmtk; zE(p)n{Sx^CQUI88H5IiyY+ff->z@$2uMC&}#0%?=>S5kjRha;0AkT{ZoJui*f+5*#8Vm1^hz>P#@XS7KhOlQA9;VqGN zc8l?2$AVG1IX3qdfo%w0Mptly^7%9MTS46vwrfQ!NA5u{v5T${xzU6;3FpDLnnWHW zUb=ps#R`H>+{GMw(mo=^&8Oq6zhqw=GW&4@31VAtgG;=N&n@3ASrizIYSM&Q*vtp% z4!BkNb`y?uGDHyUt3B%H`>{ynzkd##rR~lm9yCIeO3)CS(!|Vd9P~gFY|F23uU1!A z`-4n#;iZ{3tA%+Xp`jeJw_&(w?v53i4nU3@rQ&9{7BLF=ka~x?mU2!*?`K~VM-LpFSC)%O4w}~GMlvLi3ieBX} z3DaHvMn`T9!($!cd+0-DFDUCriHr?-P0o^uwG`b;81f0p&7h>`RQw|{rUKrk+s~-~g8{@G0N{AO?AsxSZp; zUBW8l{gE~$T%On`(Hpeednv)A{Kwt-mEQd0=mzw=`@bjn$Xc~MvC=`YE6%Px?HiZT zg*$Q0d;ShFvk3ku74z$j?bBTDZ#Td?9QC|4yUf`#KhKgKD@|pWYl;!KZ7I8GyfWYE z{PUxK%uI)C;<3YQxhn`Ls=K8#8F35KeG~jqx@UXA=B()N@1phK;ZO&q)qKCN6AQ-O zHRNwoUNVF-XozmrVnXEwir}Nrd?(Wsf9Pn7x1kbM5HzI5lm?-AGiikn8gHpY zlxi&u7a<+Mb7M^otNqx{8!K##zY~2z2(?gz0IN`4vZh2aeRl+}M>)U_5I1NtTE$%w zQ-D1^VWOI-Q`2271Clct**lD$drzWZ+)w>=yfk!bVAj*J3<7vte4**}_xDLcM{>tR zj3HU3pRP8%6g>iB&t{A7S2OSS1Z|Pn8LhK(@Dd@$q;pI7%nXhxIhP&$T?emv0kmHP zq*5Ycaj^JeD?Qr}5?*}iNr7lL{8bC9Z&~SG?_*qK5-IqT(8>2i* zSn4xbZm)+@dORnF4e3gZ z=`oy$$ojjoOckFNWI0TGtc^PtO2_l z!sfTu*(v!b3;e zz5`)yWM=l%7r)101z&xb>=D24T#AHHdWv9;+M*P*opa$_it|If6m2in!6>4%4%zcq z0GckVc33?63SLCN+i)gur`YIFGiEwgqX(atuO`AOC=RG2%Nq-4%psjo&3mc{61URu z_Kfqs9IVB}Q_6klpm;02byK;0OQ`1jWVWT;-M#m0nMr}Z z7cDuALO2-v2)(C`j4#|u*7Fc*X^<7gUl~OQ6iU0*^VIBrRm0v&o^+`(?#b7RiU9xNx6GBKY}hAi=ePf1j1(_A@!m+U6sDZTi$r$y zI3*>eG6-ahte@>hMxIRfsAZl18UEWQ$|;ZVjKA$gEeuD%q^XQPo6RGagx(Q-TwkkJ zjiOo7Be6x06t3J-r)an_Sp_8m2KB!bt6?({fan8&xjs9VYa?SpuSWPGC>Vz~*!5xr z?BIK)K_}zcLMO0XK&1s`B+4I;H>rC>iGHq=Y`m{hsK=k}9rKoK(GiyIIeM zXTKUG?{HI^+LD(223l5sTo))yL`pGQBg*(LZWLgy$knj5V$ zn-Ke23?cByxP8oSI!2Yzzl0d){Mb*&MIsPiaDFCy@xg1Zw$rCPRCNP2hSL{V8^E?C7 zzy`KrFNVQ`*XSZZd}_Iy>gKKW!mXfyrV&2IKhHku7*fhYU1_=s1yLa=idg%M^4t;=UrGO%({U&1K6IWE(6hL=hWig z%QB4h-3o(U**Q6{RQ_2&!}okjFR1d;g`ShO?g*Md?eZcvD;bwoVF8$r!Y0O~@^E)xE)@sMur2-KVkdk0k_6oudpwQ&06uXCu zFPbY!<{pf5JS1r#?!}i&YdlqE`k2Ksk?l53Kjo(+CYqbyW#ttYD=`o20BY|pA;%f^ zkccQHTyfA244bAF7kR%xJ&T-PYG^u0STH|V?5_4lh`38U{aM{x3N|$kk3CmoLx;{D1pk^+p zM(Q*kZmp*tKMqe8cv-SUlYPH@;4G$=mEG@)rd>)g?3JgYGCA;({GKs2Am;kS)9zi-3FRj4@0 zd-Yr!%!L-^dgcw|xcPEp^|QnAHHvQvM@} zRJIQAL;PpR&wEQRQueV$DSjU)0aE<+ajIQ5_HVlE{v;1gd5G}ue;|G7-Vpg>RkveU zcaiYMDwS0LCJ;XxMD*0}rv^R=0*U9Ie@R=@%_MIa zzg*??VYfM%J(UclVNmAwkDz{wPtTM3VkZS7x&5 z6PSyV8bZc1{ncj^0_yHkDx+M{8FcJ{K*Uv2^f#rWMnNEP7BY^bpoXFd(FJ00A6p}b zC-)%J85VdsV{VEwi|vEx!iM+{pPIy-n)LfRWI`eiVX_I$dvOOL^i! zgO3fjCi&)~66>UC*sX%CnuWx7f3~Sgl2ppBh^Yr_WD8c%dx)FznQlTWrhP%w4j_&qM8YFQZ-7G<&)m2_=vI9g&5sdId{Hltl3*_1fF&AKn zPcYMRf^Y#oWe=fL54GMFg~a;3E&3U|VfNV^4Ml54>#*s%;BK0=PGg&kDsExh?Sm}& zl4m8W^KdW3ApFzQ=tE5>Uadj60uUN&NWU}k+xoc}UTcB&_I2!IT-!(eFCTWGNb69_ z(*{gH2Y?14&s6RyX!^HtM?K?obf{xve-;Ay!$+35xWC5(N3lFN&a8zEx0jSRy&--L ztUiFmONtQ`o*h1Q3zoRnqSf^5EnRh)E#Z21H8qo8I0?Bf?2?#SIG#(M&Q}JV0wM^~ z>Z?uz43NwvEDAkU&)+mLvoBHRAWg!dCTH=4Qg8pFo9$*=i6hDir7X@yL`*@c(fUNQ zk0+J8KB!o&3F}ynaBM$HpJrpILgH+nFSuu933wO{N-M@f%UlpFHks0Rp6=ZDb_J=! z(}tA@5&ufHJt`61ae-w@Lzl2;vLKhoEsnOVj*?Fnd&yqDvJi?m_!XrF#jY=)vo~NA zw_f?Yne?ZK^iTc`925CNdvt|KhNNG>tT(t3w?5bL3>X}K4OddA%F%z`npL)J)%li$ z&_=>8@NIMp{^!rCx=Ce0U~UJ^Q~wDIt8%8xmU6BAKVN7ty$&U^xpnfj<87DS;#lae zrz2_8dB;yrpV_&I_Ex1JpQD+)&|PnyUlj`-Eu{-{t%|u@n@UIZeLGt)EFETraVT8% z^3TXn_1}FaqtR3x{NA9Kyx-rTtE8$RBprB^1GtWKvz(~uxLCQ;u<$J5YolU$7p_q z(k3llcZ0JIxCC0vQvmGV5D~ONuxd;GXm9Z3Z)nMgq9w-iwL1Uv2JIf)@R>7|6TyR^ z^?fWqFVdKt4Xe}c=gy|lYaXyf=A=SNU@9|Zc zAXf6hpIZ0HS-~iSu57Dn*pgo|5mC^0`7R)a(TSHGQbQp6%c>mmUVT(RFVbZ{yqnns zW4-q7(ciVlMu&7$M7I<#H`B*BWaw;5=7_q`pjN6SXu>~|>xSvU*?YmV=#kDoh z*P*7;_>9>v1t)1)HN(NLEX|RAL{mBZttCG08l8+Q`{)zr#OfaE+{4RbB~sjB8|G#F zKS$MaN7c_vC9(O*!EUv(Rs9hGqHyY^zN_DN8GknIu%$(K(Gs{1UT7*oZ;a$^;!+kl zOH~VChlhHAw|mI(h94#22kq}w6|;*OU^v*zHMtOU3IUa!3N92euhqB}CWe>3aYwhD z(#_3Bo$c&?KALEE_p$o5v+=n;bm8~b!sk{$&(H{0$2qgz?d><_p%HHFo<(W9?0L`^`4KLWIHg6udygeK>$ot6>2ZAkCX;z;GyoA`yJn?(@ zqryYZ#F&Ocgs%b8_N7?0BBR6x?fF&EO=4gxEa68s-=mrMCeWA)kHUg^vjk8=WW(`IXBsjrf0I*hz*1S0%c_TB(;3cf&~i|T{C zaWRFaz-cFWsQ9dT>mk*L5^->T)EEb>0-Cl@)5#l>*x$wuHSuq`sGyTkq=-`5IASb} zNJf%?hN8+;S9sJ$dDu5@t{JNT=_ggh997z_H_V& zD^hw$jhp?SohcQcQ*Z9vwnbI=tqte;AeFS`O1vN~5R@ng=cHAUn0KvFyJxj{iAK^R zZ!-LvJhKIX(Q`d2csPh35Y}FYdozgZb|$)@YCL(He-fAZN*K@t5V7BRGaySWBrH6p zys)2aP5I1w&Z^EveweYLH3^FcAuQL$k7s6c1xh*IMa-c<6s4aSpkmu}S5N8eaBPfR zIG>PjtWa&`?Ccyj$a}2CpEvqv!tu-kB!EC@2UnUPHZPMfZ5i^R3WlRw?-`vBFNw=Yt-&fFa zSqacjQ-5R^zR_xHO604^zb0K$rC`3Ne~BI-#sZJtD|p?OjnY((<%cN{93<$jm2VWwXoqM2V}wr(d~QGh$$55kiGbs*}D8rtfndBYH|R500o?Wg3-OQhLz5EExI$-pvVIQ z*7JDF!=(`8yn#Jh>z;O8pN-3w5d8QHH~gm)MN6ZpdYT{3pv>k~ccwIF#!qyW*X5Rm zbDM=Y`qUQP?Vl79Sqf|~b10hh{yq;FBo08Tw@otzJYS-x^BItB6Z%mLsPot51SP)1 zE^%(l?1WL*x$w|8oU^t1mfT(7lJ8o$zuw2%dLIovW>8Mr;Pifh z;+Qhd4<@x5eW2K{y;HeX>g81MW_yI$Z(Cws-qjG8fR27;IPgKzc_9Ir`r~Me{>=A% zHlepWao;m{>_{Lppkq-hdc+W{`Y7_wE~6{LVon@ou%=hCtVvWQ!VfO0U??E zoeG*~TDxT}X`UMMFyn{k^Cj|a*xguK_!3{}2~*#MB&!w?8xUKi@aNLkm@JaJ{ddIA zjbEav&}4xypxQ*dgivv$R_v*{0Rw%j*0WRrIe<)xhRT|hQcjNppH4ReP5gustV}Y7 z6MY}=%csv{*(aR@l3m|BmVAItBmX99cVwfkbSINlKp~;u7;>R`HhwB8P;dz;55O{lL5796|w{_n@c&e;`NKViA46wYUYp#)vYk_OY}7NLUKt{^yg z@unqXqS?CXZ)ZEVvXYi`gnSEar5s2#R$F78*1BC!n9WpMK6vfL1#R=FoiT-`e$01C zPLt?M&Pu!=eJeP;>Elaa)^|KZa3e}rYgXfixoYTlpAl((&Iw_F%u$@sY@U zGlEyOlPsMpeu&}3}lix{Wbu+UEm(22?be|@vF%;{vYx3 zHGq6Sk!gs)L!l{Hi7CRHjS~`pwnwMOj|yW1*;MY^>aFLGHk)ZcOIp#MNGwDJEm-T= zVkY(2|N54w*)^OAUJc6H4tHQbgoSR#2qsAoovUZM7h`UO&D zbcBE_p>d#=piQjXej=+z|RJ!1_o5#Ifr=5V>^)oFF&F3W z`>W&-%k;vaKZHgek&!HMV2c46AA;uJ+E{=5DI`7m)(){zO^Wtzyg+Ay0|anfnKCAr zZAYKx*2C#%=&bA~S*wU>W#NDG31fx@TyreFoDfC8BkydSog7 z$Z>8Ij`_f?N0bPOpu0uJdDjq?A*WyR{VEMviNz9w=X6@t^ovlIPtxem3T)RTih6Eh zHjgnxpHl*nE#FbsANM-ZC1CUc^^;|{$#j!zHR6@ou4LMwhr<$uAZ&f(0L|aPKWn($ zV0S=iD{z!`;_(vk8Zd9VwX71DE*tY{Z#Jb;AmJ?;)A6+QbIWTyu89!1woB`^nNk0nRy66kCTSLy<9kUEQoD_GS zu7mf8zL1p?EqsfTuuu>!n0IGQkeA#&{ zz#jNAHN10)Ui6s&M4`O1yBMhZ9$yAUjQWf~Lq~4wM*sx09BxsR6df(Aq+Y#%QBplg zbVT0xqPYGdBZTzM9oaMt?qg+JRsnSl@o!c~G;5l9W-NY;xI0h%7~_c+@^ZT5tX37F zHslxEF+<02hQVBJ=DbzteEdv5~}_<|^l+A*lxQy|ek*9rNqk4p<&^;;Y!4Q6NGrAfv|TS}nB zY_f?f4x<_6d9&WPc$))g{(4&oBil!K%I#m2TNM2sl5)+Hk9;Nbcj8{nKv{W#!PuAn zg;Zt^>=V~>=}0Cg`_x};tE%J=13!sYmQhHy5nAI17uQV{3pub8gQnB8U{HW$ca#*^ z(PQ_*=g!#u8z)?5@gtfI@@W@IGOQbOLz_Ef$M-zMQ+cXum6E(iR}C>pkCz69-es6*AdB21<~o?6BD+ zZCU*g`gnwzH}-8YMYu1g8o|vnU4f0_%J+pfI!%%s!T2lq%6dMrRb9;8D{)u0rd|Ur zIXeDn;eE^cW$c0qRs@SJvcLXnmA9K5tVz7kb$lAV>F{f(idPeCg~r%5Zt|0?$DrQP zd&dqRmR8drv>5EBX^SO|aEtV*aG9;5-^u9g7c~$6=w@E4FkB_a;7ydUoWqvde1^E3t*{D)b!clF6vP(AL(2O($(DR#e& zMvBiPrdI-PCXZCAqrJWH7<+TrPrPvV`L4>A-!nh9&MniY8}~gsxOtXAR2YKXoD^b? zhK+tjhm~DvGPax+6?r!lU3M%E#a7D#M(!!U%=zQ>Q(>u4bbo^wtVm5G_j^cl|1I7d z_p3jr@Zi}1>-)4upN?WnMg{NkzNd^GtA1X`uQUE-!JNey+5YER)MY0c_abEGcU`#+ zfbO?w(bpTvEn?dZ_pitOxTB&=cUNb;e6&Q9+B7BNRz~9eZ+`nx(&B2+ny=*y`J^4i z&U;tv^S(vQPm4O*RAHNpsJ{yD841;dQ#UQHu_!;?Q1hw;(qoa%s){8afo#Ft)RW+4 z*pdZ_w$}&zal_9Uddwv(llT`CjO+KDU}BZK_g0jG?HW>g#b=2gr>nKz@?7^G6@nsT zAO{3sk%e5Pellz)D>C0Xf4%7EL*ftd0;3Lo-?+Ew`FhwR86MoppiGdPs-aHg`Xw>? z5UcE)gi=CC{`5@>zCKs_wg~m%$Jy6~rZm|3#$Oycx`O3No!rS>R^{JUUsdWbS&-Zt zS2fxH=Jdb~4Gqf_+t-c$al>8&4|o0)pa=h7wxHu;QjlY`*ZS zcbY2e;aBbs4W+8Ix&1LuTTtkd%{3akrH=)6 z%TlzlVgz^tSb4Dd(o91U0!qn84c0H~bkO|f%l@+X=oT~w&T+QF$Xp5|22eP`y_wda za{PA^xvLZUZjk~8Eml4rDbJO&zLL7RXZ!^7awVtQQX>E*kEWBr)b){^;P*GeRK1$zE z;uqKT+`a@gnri)SQmv^9PeXCW_$WaKMrgwQxYuIp5A|zAX&*!ZHWjK-x* zqMJ*YxT52}#3=l5idfUK4U;69%PJShF3Qe2NEqIKKe{ox&ZB$vxEY_u16^ec9BL_nG52d5uGRqX&fJf=c(h^L>7gMOnm zBAOO8yCHru(2jowxYY~?n2Wl{t(C?8Gz%?r)?R)KwFZ&{Ha=Hv&_DZl4Y2XAONJ;T z`!VdOaR1hy!Z+uT&iAQoUy47Hjvsw*IW$5d`sB(Zo?p*3z~*Z-FWj8$&BBYSOC=Sk z(!*+@_pS@hQ@V(hlg-S!LnHqi30Tr!Y00{L9-=3c(n_W@v{_T60-rAXi~=|ft3#05 z>d$T6_w{Ivkz%}G2eELcZsv*P@=4g@X8aN&L6q@>3 zXs)r&;H+`dk|Z2aNNG5NxoLg^1&W!@zDKb2)bex~d)$EWTHRB1go!n}!&1pxF*phr zn72P!)_p?tCSw@ysHt{$a%Q6N*?}=c!JHu+?Y8}+x(p#>(nQr{8V$AC_dLMiM%{2_ zLkbX4FLu)tA=#qSH{I+Q(b=+^8Q~${Filw zDKgp zzm9X;ZlI2@($|mOo}chAho@jRr@nLf#%$7or3N`IrIRo$)tbkv!id&MZ1Qt1eJ`Y; zA(>$b$~sZZ@`Uts|Cy!gw^!uJh7=tQF)k8Yl;n3IBEHFiu*52m7GqOH;&eUSerxjU zF&7l3;d=9x1@Duij@fsYY7~pr0JcL}(PQY+`sBULZID*S6IjbxaGS;iU=Ot!72doB zdB7~jl^P@Vj&h!~7xn@0eNAI6EjQ0}wmh}l9ZGD%Trn|k`!qW49xeau?3{7#-UyEg(vgD zErkcn?LuW!*Wa}{B+-ORaiiiSK<6##6qn)5S#GbVop{Ujm%(pky>=$7M}zXhT+gL` zX;&y;m!0*V_-r%Tah0p(@OO2m=hKrTiTw25kMKVl#4VSv`}0NewpwFgWoQ8cbt+-! z--wxT2_E(%qsZ_@#L@FH3diI3*3ox-eH-*tT6gb(iAT_2SWkrck_1}F_hq;Z2bmXr zo6~n8?!R%bF^=mQ3lJEaCBQE8u$Vgl&BLZSPs~t7a|M>13Ek`E6)5U|t@UedInxRL zL`?))bN{b>LhFsZ!h?YNQAKMayvE#jZVz)se2fQUFVwoXZi&LoZY0X%o5H?uln_=s zI%b*%S4={1{mF&1fZuz-L>Q4+do0fAg;$^L+jgUf=bC<;GV`dG=V*ky_2Bo`Wl&Nr z8aXjpRcr1~a{XzQm`zfI{&Ewrr`Ng`^NQ6ogv_sw}|0ChhFa^Y9u* z$)@>WyB}{xH47!*2u=Apeph=-RwthYIUE=AeQrm3=SMX{YJ;V}M&_o;8g)UAOJFni z!uRXS$LT!92J;UrBKq!zUFq;;3DTh$eS9r4+e+(In9xoQ`^}L2Iu>sW+pkh9_uZ9; zzkKFKXO^NvA5^R%dZtbKwxJPo=2oZl5C;wCqix2a)E@O(yfGX++l#u2pM5n$MiLw{ zZEOz~m6a`ikY8>{VUj}3^*nle?V?mbWL);$tb0|l8@?O54&ofca)0=VGn0Lyq;S9sIv)x{ttA!gH8&|w) z-}YV~HIOtK4qZ67W47?UKU?v6>(A%O7Z|A0cMYOo!^0XwV&>$i?k5*Jr~C{Y0A*ZK zlePlUa!+GR->`$@6kOAIX5QQON8=%3Yj801;Gz~JJU5MrcNl7h<(`p$Mtjr2|bgL(Tq{s#gj=~$kE1$)R3iI zIrtVv*J=YlU3B^yOc;-U4@Yu)f|`R1XFEZa7z`$S^xjK3kmYtoV1_7y70ZU5ACs#O zvi3Lmxtp?iR38VKJ!cJgHRA;z7#JuUj`;aq6;4C+hZ% ze<^%G31DCl_WiPB+#r-$(ji}>ZSVcJ$doOWt0X98&#$t6WoSq)WLDLvJtRi!*WJJ8 zuN`)CS+UzGWLIE=c;|lEz;MaK?d&f632(KA@qKiIt1X z=)#*jq(^xnjaqrD!GGt?YDJIg&i(4TI`7D!?Vovepo)|HZ(vUK;@$D8B2ucVT%DPo z?(T15ay!C;z-oG@K33iKqUZ9T>IzHE=wy_>8}X}ky<9Tf{p<18%VA^5;68)&zwc~G zqsH(;SD|$n(&37KulZ??ne*vXJ?m2cqp7s>dU>?EvhZzo-C*Y&6=7vDXU_Tmy)TB* zx-%lY?7laYs5X6 zeVrUd@I(FouC^^_p1j$)xnEit2Q5@ET(I(C;OGBD)y2o(CBE0T-+fs4t4-Mc0{g^A z6RdkO*LgPQ4>nJZd3usmsz~vHFdU4xsrBD#4WvP9eOf>vghMo`3UAlOn!EmmFxvoL;=i9~LRS_zZ@AL9v z6UM8dPxS>G^$@>*0S3xp;Mp*S!C(O%_m^{B(37v7-TMD0UTNVCefSFk%OP0ei?=Ux z?dp0?)i=vnI%R{K_J2xa;FlwKG||p&-=&L{r@UXyXIZ{*us_|3`dtT}G_eq-6{;BRaU|%7rhjpV%(i&jhVu`b|$z0MiszP zV)LNduQ$V5pH@CSy*zepH9v#wcK8u^1!|lJZYmGU3fj?dR zdw$6Ccb{h+^hokj`tq1zl;;fWKrSpSoGmtnUR}Dl9en%di!auBiJK`WCr1}7Bbr0N zLelNzwDa#NC|s3szBPP)`2CQ2xe!|LaHazOD-#BK3QIQRHsyF8t3TO0B~8QXWQ}h93+nR+@k$)Aa<$ zKX=&K+4UX4J81MBDUaPtACSH79ulIIw(-XI%lp}29=T-joF>Uywv2;W$;u6MfbIg_q=*^_gi5R;Z;1p15)vRJcMiArF1q>; z+#h;=IcuFcXXeaav-f;^&+KoMRk5sW?BvO4xs*3?ao1&UFH~yTNhyEh@*3XBOiE@J z#Sv5j9C?92ATF>uE}k9jBz?%8&8-JXubA3K8pNXA+r2wkvVwU*aPWk;^`Kx@;*+wH_ z;mU{Uw_5HTx*^s9zT|K09V?5V#crnfs-^Gum)2_eWgBNQhVvc2(BdtnKLEvlYm{zl z4|UaTEj5d3e^atb|F#C|X$K)R`dW&2{k)Xa1LryMQmA4c%fJbC$fc~6nyWmykj`lfP%W?Nl@GI5ptZ>o$1 z7iWMg3kxY3p~h6QSP|J3F_da8$fhu-tFF*GA}u;EZr+u4??>1Q5d5$X%d+5R9{>j` z*=%cC$I~$v&tLSOg=`H4z*1oDk^bNL8-}6Igv^2417>N&)X|?N+RU|aE=kq=PC@>* za4Wmb-n7;YC;V%afZy*sct_E&<4g-0bpXsZJz|z`Ph!c^?`D0Bow9sh|0z>c5(@0Z z6;9n6burwpQ~2qOPZuOUu3^82qjA#RsH~&sE>I#(7wI7P`TFGpvI=Q*--V^8R<4Ja zLXap$sR64TTAmmUzqiZ4Y2;+?!?0FHU}fMFPLCgiRjO3NRJRcoFvLf7U*H4j+}heZ zLYFbS2_t5Tr`(>@O-$&poO#3*8d#X(F>>xPEK1|5TM!A`j?2rCDZi%o)!h7P3M?VD zPuw`uup8!Sc)&Na>N2bE*y0K@l@9r(JlS*`1qp=0aOyW+UZioSu;BW_rpbtz(vHYd zAkRg~#oBj?ySd9FR#0ko_P1kgUSPNcu;%f5KB9ril17e6;5fpWMtQB&8JY5B1oA=k zenbYRajM}u6dwR}&B8&($&lceg9fC3#ET}MP~n5xLNMnGf{EmyhQSNlV#b}^IwM_% zKcA>>L$Zr??Y6WluyK>PCbl>|Ff!m5G76|x-s9JaT~{kwt~Q)6C_9LTPB9&ieYKx^ zeEo*JYezCvj0`H>XYi~J{M5HYdi2`qNwrWPnuuU&)Ita!x+pGuCQKPm7fd~-&sxy~ zFe23?Oz25vo-(x{z{P#|_+o^3;eH__R6WNn$Jzw6bS{MI7Jr-QDbXe^Y?=IZX*qHnNH@qx!`V;!* z=O^gbl7w_{1Y`-(hC5>F=tmveL>gbXxjYBy5-koTPzKZIlj*8tp1lu05|&JlAqXY+ zONOTNVE~y2ZRkXdS<+pN;{>)D>Z#%+@xe>YAAH<>HSX40>KOU`t!zPvEm`w+?Rr4n zy2^&Zz}uDJX84&2_FqAYcZa+6Q*m*ug#L)aAntuBGDD-H>ymMC<+ts-a{2< z@iU=2R%Q)p1`SDM<&F@esB>wPAH^OiqYj+ZLscjPMz9H1A@^r4DfZQPV6ogIihXqi z_-^c8=RvelKPF(MPv`q_hJz-ohkY`cyUS%n)Sk4`svya^qobNR5l3Q8o7nj{0p>c9 z#g`f=k2Q%=N<<^2qTE1`(-K;BCVG<%AB5DW{Z--SOGkTeyt%pKAg;0=)B75%z2FR+ zFuV?D_q=07xbe7#mB!Z^lax;WBSUy)waM#9)3bMS+}iNuB6FYoyFn(Ul_3CZVI)1E z7l7lhLNc7`?EHHCwozg%rCUD%hv}N?y|X3?*F~mCp26Is#jD`o7^=ZQ72xg$BM5`$0 zH89qjv`UMmBkA)O93S5Tuj*yx?C8XkXW}VF+ z?|L;dnlZgVncRDeWwVh5Zf}re<%Yas*-T_Xm>j7D4Q)&;Z_pIAhwME+T|~R}&;!e_ zV^A>B9VOcs6(;Un_0pJ4Y^pD=%Z^WHVVgzfi0xE8Yg+2)6h8!<}7uu;LD{s;U}3#fP%%vh6BF`j-av?L~V3bIblxYW;T| ie-_T)^7y}%hZgELX0|j~+J&wG*FhZK^ZwUijDG`6=k0s| literal 0 HcmV?d00001 diff --git a/models/tacotron2/NeMo/docs/source/asr/speech_classification/intro.rst b/models/tacotron2/NeMo/docs/source/asr/speech_classification/intro.rst new file mode 100644 index 0000000000..6a1a2f9ee7 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speech_classification/intro.rst @@ -0,0 +1,47 @@ +Speech Classification +================================== +Speech Classification refers to a set of tasks or problems of getting a program to automatically classify input utterance or audio segment into categories, +such as Speech Command Recognition (multi-class), Voice Activity Detection (binary or multi-class), and Audio Sentiment Classification (typically multi-class), etc. + +**Speech Command Recognition** is the task of classifying an input audio pattern into a discrete set of classes. +It is a subset of Automatic Speech Recognition, sometimes referred to as Key Word Spotting, in which a model is constantly analyzing speech patterns to detect certain "command" classes. +Upon detection of these commands, a specific action can be taken by the system. +It is often the objective of command recognition models to be small and efficient so that they can be deployed onto low-power sensors and remain active for long durations of time. + + +**Voice Activity Detection (VAD)** also known as speech activity detection or speech detection, is the task of predicting which parts of input audio contain speech versus background noise. +It is an essential first step for a variety of speech-based applications including Automatic Speech Recognition. +It serves to determine which samples to be sent to the model and when to close the microphone. + + +The full documentation tree is as follows: + +.. toctree:: + :maxdepth: 8 + + models + datasets + results + configs + + +Resource and Documentation Guide +-------------------------------- + +Hands-on speech classification tutorial notebooks can be found under ``/tutorials/asr/``. +There are training and offline & online microphone inference tutorials for Speech Command Detection and Voice Activity Detection tasks. +This and most other tutorials can be run on Google Colab by specifying the link to the notebooks' GitHub pages on Colab. + +If you are looking for information about a particular Speech Classification model or would like to find out more about the model +architectures available in the `nemo_asr` collection, check out the :doc:`Models <./models>` page. + +Documentation on dataset preprocessing can be found on the :doc:`Datasets <./datasets>` page. +NeMo includes preprocessing scripts for several common ASR datasets, and this page contains instructions on running +those scripts. +It also includes guidance for creating your own NeMo-compatible dataset, if you have your own data. + +Information about how to load model checkpoints (either local files or pretrained ones from NGC), perform inference, as well as a list +of the checkpoints available on NGC are located on the :doc:`Checkpoints <./results>` page. + +Documentation for configuration files specific to the ``nemo_asr`` models can be found on the +:doc:`Configuration Files <./configs>` page. diff --git a/models/tacotron2/NeMo/docs/source/asr/speech_classification/models.rst b/models/tacotron2/NeMo/docs/source/asr/speech_classification/models.rst new file mode 100644 index 0000000000..b8c0248ec5 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speech_classification/models.rst @@ -0,0 +1,71 @@ +Models +====== + +This page gives a brief overview of the models that NeMo's Speech Classification collection currently supports. +For Speech Classification, we support Speech Command (Keyword) Detection and Voice Activity Detection (VAD). + +Each of these models can be used with the example ASR scripts (in the ``/examples/asr`` directory) by +specifying the model architecture in the config file used. +Examples of config files for each model can be found in the ``/examples/asr/conf`` directory. + +For more information about the config files and how they should be structured, see the :doc:`./configs` page. + +Pretrained checkpoints for all of these models, as well as instructions on how to load them, can be found on the :doc:`./results` page. +You can use the available checkpoints for immediate inference, or fine-tune them on your own datasets. +The Checkpoints page also contains benchmark results for the available ASR models. + +.. _MatchboxNet_model: + +MatchboxNet (Speech Commands) +------------------------------ + +MatchboxNet :cite:`sc-models-matchboxnet` is an end-to-end neural network for speech command recognition based on `QuartzNet <../models.html#QuartzNet>`__. + +Similarly to QuartzNet, the MatchboxNet family of models are denoted as MatchBoxNet_[BxRxC] where B is the number of blocks, and R is the number of convolutional sub-blocks within a block, and C is the number of channels. Each sub-block contains a 1-D *separable* convolution, batch normalization, ReLU, and dropout: + + .. image:: images/matchboxnet_vertical.png + :align: center + :alt: MatchboxNet model + :scale: 50% + +It can reach state-of-the art accuracy on the Google Speech Commands dataset while having significantly fewer parameters than similar models. +The `_v1` and `_v2` are denoted for models trained on `v1` (30-way classification) and `v2` (35-way classification) datasets; +And we use _subset_task to represent (10+2)-way subset (10 specific classes + other remaining classes + silence) classification task. + +MatchboxNet models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecClassificationModel` class. + +.. note:: + For model details and deep understanding about Speech Command Detedction training, inference, finetuning and etc., + please refer to ``/tutorials/asr/03_Speech_Commands.ipynb`` and ``/tutorials/asr/04_Online_Offline_Speech_Commands_Demo.ipynb``. + + + +.. _MarbleNet_model: + +MarbleNet (VAD) +------------------ + +MarbleNet :cite:`sc-models-marblenet` an end-to-end neural network for speech command recognition based on :ref:`MatchboxNet_model`, + +Similarly to MatchboxNet, the MarbleNet family of models are denoted as MarbleNet_[BxRxC] where B is the number of blocks, and R is the number of convolutional sub-blocks within a block, and C is the number of channels. Each sub-block contains a 1-D *separable* convolution, batch normalization, ReLU, and dropout: + + .. image:: images/marblenet_vertical.png + :align: center + :alt: MarbleNet model + :scale: 30% + +It can reach state-of-the art performance on the difficult `AVA speech dataset `_ while having significantly fewer parameters than similar models even training on simple data. +MarbleNet models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecClassificationModel` class. + +.. note:: + For model details and deep understanding about VAD training, inference, postprocessing, threshold tuning and etc., + please refer to ``/tutorials/asr/06_Voice_Activiy_Detection.ipynb`` and ``/tutorials/asr/07_Online_Offline_Microphone_VAD_Demo.ipynb``. + + +References +---------------- + +.. bibliography:: ../asr_all.bib + :style: plain + :labelprefix: SC-MODELS + :keyprefix: sc-models- \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/asr/speech_classification/results.rst b/models/tacotron2/NeMo/docs/source/asr/speech_classification/results.rst new file mode 100644 index 0000000000..6dd65b9906 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/asr/speech_classification/results.rst @@ -0,0 +1,105 @@ +Checkpoints +=========== + +There are two main ways to load pretrained checkpoints in NeMo: + +* Using the :code:`restore_from()` method to load a local checkpoint file (`.nemo`), or +* Using the :code:`from_pretrained()` method to download and set up a checkpoint from NGC. + +See the following sections for instructions and examples for each. + +Note that these instructions are for loading fully trained checkpoints for evaluation or fine-tuning. +For resuming an unfinished training experiment, please use the experiment manager to do so by setting the +``resume_if_exists`` flag to True. + +Loading Local Checkpoints +------------------------- + +NeMo will automatically save checkpoints of a model you are training in a `.nemo` format. +You can also manually save your models at any point using :code:`model.save_to(.nemo)`. + +If you have a local ``.nemo`` checkpoint that you'd like to load, simply use the :code:`restore_from()` method: + +.. code-block:: python + + import nemo.collections.asr as nemo_asr + model = nemo_asr.models..restore_from(restore_path="") + +Where the model base class is the ASR model class of the original checkpoint, or the general `ASRModel` class. + + +Transcribing/Inference +----------------------- + +The audio files should be 16KHz monochannel wav files. + +**Transcribe speech command segment:** + +You may perform inference and transcribe a sample of speech after loading the model by using its 'transcribe()' method: + +.. code-block:: python + + mbn_model = nemo_asr.models.EncDecClassificationModel.from_pretrained(model_name="") + mbn_model.transcribe([list of audio files], batch_size=BATCH_SIZE, logprobs=False) + +Setting argument ``logprobs`` to True would return the log probabilities instead of transcriptions. You may find more details in `Modules <../api.html#modules>`__. + +Learn how to fine tune on your own data or on subset classes in ``/tutorials/asr/03_Speech_Commands.ipynb`` +f + +**Run VAD inference:** + +.. code-block:: bash + + python examples/speaker_recognition/vad_infer.py --vad_model="vad_marblenet" --dataset= --out_dir='frame/demo' --time_length=0.63 + +Have a look at scripts under ``/scripts/voice_activity_detection`` for posterior processing and threshold tuning. + + +NGC Pretrained Checkpoints +-------------------------- + +The Speech Classification collection has checkpoints of several models trained on various datasets for a variety of tasks. +These checkpoints are obtainable via NGC `NeMo Automatic Speech Recognition collection `_. +The model cards on NGC contain more information about each of the checkpoints available. + +The tables below list the Speech Classification models available from NGC, and the models can be accessed via the +:code:`from_pretrained()` method inside the ASR Model class. + +In general, you can load any of these models with code in the following format. + +.. code-block:: python + + import nemo.collections.asr as nemo_asr + model = nemo_asr.models.EncDecClassificationModel.from_pretrained(model_name="") + +Where the model name is the value under "Model Name" entry in the tables below. + +For example, to load the MatchboxNet3x2x64_v1 model for speech command detection, run: + +.. code-block:: python + + model = nemo_asr.models.EncDecClassificationModel.from_pretrained(model_name="commandrecognition_en_matchboxnet3x2x64_v1") + +You can also call :code:`from_pretrained()` from the specific model class (such as :code:`EncDecClassificationModel` +for MatchboxNet and MarbleNet) if you will need to access specific model functionality. + +If you would like to programatically list the models available for a particular base class, you can use the +:code:`list_available_models()` method. + +.. code-block:: python + + nemo_asr.models..list_available_models() + + +Speech Classification Models +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. tabularcolumns:: 30 30 40 + +.. csv-table:: + :file: data/classification_results.csv + :header-rows: 1 + :class: longtable + :widths: 1 1 1 + diff --git a/models/tacotron2/NeMo/docs/source/common/intro.rst b/models/tacotron2/NeMo/docs/source/common/intro.rst new file mode 100644 index 0000000000..61d160b5c6 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/common/intro.rst @@ -0,0 +1,39 @@ +Common Collection +================= + +The common collection contains things that could be used across all collections. + +Tokenizers +---------- +.. automodule:: nemo.collections.common.tokenizers.AutoTokenizer + :special-members: __init__ +.. automodule:: nemo.collections.common.tokenizers.SentencePieceTokenizer + :special-members: __init__ +.. automodule:: nemo.collections.common.tokenizers.TokenizerSpec + :special-members: __init__ + + +Losses +------ +.. automodule:: nemo.collections.common.losses.AggregatorLoss + :special-members: __init__ + +.. automodule:: nemo.collections.common.losses.CrossEntropyLoss + :special-members: __init__ + +.. automodule:: nemo.collections.common.losses.MSELoss + :special-members: __init__ + +.. automodule:: nemo.collections.common.losses.SmoothedCrossEntropyLoss + :special-members: __init__ +.. automodule:: nemo.collections.common.losses.SpanningLoss + :special-members: __init__ + + +Metrics +------- + +.. autoclass:: nemo.collections.common.metrics.Perplexity + :show-inheritance: + :members: + :undoc-members: diff --git a/models/tacotron2/NeMo/docs/source/conf.py b/models/tacotron2/NeMo/docs/source/conf.py new file mode 100644 index 0000000000..7dfba6e3e4 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/conf.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import sys +import glob + +import sphinx_rtd_theme + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + +sys.path.insert(0, os.path.abspath("../..")) +sys.path.insert(0, os.path.abspath("../../nemo")) +sys.path.insert(0, os.path.abspath("../../nemo_text_processing")) + +from package_info import __version__ + + +autodoc_mock_imports = [ + 'torch', + 'torch.nn', + 'torch.utils', + 'torch.optim', + 'torch.utils.data', + 'torch.utils.data.sampler', + 'torchvision', + 'torchvision.models', + 'torchtext', + 'ruamel.yaml', # ruamel.yaml has ., which is troublesome for this regex + 'hydra', # hydra-core in requirements, hydra during import + 'dateutil', # part of core python + 'transformers.tokenization_bert', # has ., troublesome for this regex + 'megatron', # megatron-lm in requirements, megatron in import + 'sklearn', + 'nemo_text_processing.inverse_text_normalization', # Not installed automatically + 'nemo_text_processing.text_normalization', # Not installed automatically + 'attr', # attrdict in requirements, attr in import + 'torchmetrics', # inherited from PTL +] + +_skipped_autodoc_mock_imports = ['wrapt', 'numpy'] + +for req_path in sorted(list(glob.glob("../../requirements/*.txt"))): + if "docs.txt" in req_path: + continue + + req_file = os.path.abspath(os.path.expanduser(req_path)) + with open(req_file, 'r') as f: + for line in f: + line = line.replace("\n", "") + req = re.search(r"([a-zA-Z0-9-_]*)", line) + if req: + req = req.group(1) + req = req.replace("-", "_") + + if req not in autodoc_mock_imports: + if req in _skipped_autodoc_mock_imports: + print(f"Skipping req : `{req}` (lib {line})") + continue + + autodoc_mock_imports.append(req) + print(f"Adding req : `{req}` to autodoc mock requirements (lib {line})") + else: + print(f"`{req}` already added to autodoc mock requirements (lib {line})") + +# +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", + "sphinx.ext.githubpages", + "sphinxcontrib.bibtex", + "sphinx.ext.inheritance_diagram", + "sphinx.ext.intersphinx", + "sphinx.ext.autosectionlabel", + "sphinxcontrib.bibtex", + "sphinx_rtd_theme", +] + +bibtex_bibfiles = [ + 'asr/asr_all.bib', + 'nlp/nlp_all.bib', + 'tools/tools_all.bib', + 'nemo_text_processing/textprocessing_all.bib', + 'tts_all.bib', +] + +intersphinx_mapping = { + 'pytorch': ('https://pytorch.org/docs/stable', None), + 'pytorch-lightning': ('https://pytorch-lightning.readthedocs.io/en/latest/', None), +} + +# Set default flags for all classes. +autodoc_default_options = {'members': None, 'undoc-members': None, 'show-inheritance': True} + +locale_dirs = ['locale/'] # path is example but recommended. +gettext_compact = False # optional. + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "NVIDIA NeMo" +copyright = "2021-, NVIDIA CORPORATION" +author = "NVIDIA CORPORATION" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. + + +# The short X.Y version. +# version = "0.10.0" +version = __version__ +# The full version, including alpha/beta/rc tags. +# release = "0.9.0" +release = __version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "default" + +### Previous NeMo theme +# # NVIDIA theme settings. +# html_theme = 'nvidia_theme' + +# html_theme_path = ["."] + +# html_theme_options = { +# 'display_version': True, +# 'project_version': version, +# 'project_name': project, +# 'logo_path': None, +# 'logo_only': True, +# } +# html_title = 'Introduction' + +# html_logo = html_theme_options["logo_path"] + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "nemodoc" + +### from TLT conf.py +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +html_theme = "sphinx_rtd_theme" +html_logo = os.path.join('nv_logo.png') + +html_theme_options = { + 'logo_only': True, + 'display_version': True, + 'prev_next_buttons_location': 'bottom', + 'style_external_links': False, + 'style_nav_header_background': '#000000', + # Toc options + 'collapse_navigation': False, + 'sticky_navigation': False, + # 'navigation_depth': 10, + 'includehidden': False, + 'titles_only': False, +} + + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". + +html_favicon = 'favicon.ico' + +html_static_path = ['_static'] + +html_last_updated_fmt = '' + + +def setup(app): + app.add_css_file('css/custom.css') + app.add_js_file('js/pk_scripts.js') + + +# html_css_files = [ +# './custom.css', +# ] + +# html_js_files = [ +# './pk_scripts.js', +# ] diff --git a/models/tacotron2/NeMo/docs/source/core/core.rst b/models/tacotron2/NeMo/docs/source/core/core.rst new file mode 100644 index 0000000000..1e053bd4dd --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/core/core.rst @@ -0,0 +1,963 @@ +NeMo Models +=========== + +Basics +------ + +NeMo models contain everything needed to train and reproduce Conversational AI models: + +- neural network architectures +- datasets/data loaders +- data preprocessing/postprocessing +- data augmentors +- optimizers and schedulers +- tokenizers +- language models + +NeMo uses `Hydra `_ for configuring both NeMo models and the PyTorch Lightning Trainer. + +.. note:: Every NeMo model has an example configuration file and training script that can be found `here `_. + +The end result of using NeMo, `Pytorch Lightning `_, and Hydra is that NeMo models all have the same look and feel and are also fully compatible with the PyTorch ecosystem. + +Pretrained +---------- + +NeMo comes with many pretrained models for each of our collections: ASR, NLP, and TTS. Every pretrained NeMo model can be downloaded +and used with the ``from_pretrained()`` method. + +As an example, we can instantiate QuartzNet with the following: + +.. code-block:: Python + + import nemo.collections.asr as nemo_asr + + model = nemo_asr.models.EncDecCTCModel.from_pretrained(model_name="QuartzNet15x5Base-En") + +To see all available pretrained models for a specific NeMo model, use the ``list_available_models()`` method. + +.. code-block:: Python + + nemo_asr.model.EncDecCTCModel.list_available_models() + +For detailed information on the available pretrained models, refer to the collections documentation: + +- :ref:`Automatic Speech Recognition (ASR)` +- :ref:`Natural Language Processing (NLP)` +- :ref:`Speech Synthesis (TTS)` + +Training +-------- + +NeMo leverages `PyTorch Lightning `_ for model training. PyTorch Lightning lets NeMo decouple the +conversational AI code from the PyTorch training code. This means that NeMo users can focus on their domain (ASR, NLP, TTS) and +build complex AI applications without having to rewrite boiler plate code for PyTorch training. + +When using PyTorch Lightning, NeMo users can automatically train with: + +- multi-GPU/multi-node +- mixed precision +- model checkpointing +- logging +- early stopping +- and more + +The two main aspects of the Lightning API are the `LightningModule `_ +and the `Trainer `_. + +PyTorch Lightning ``LightningModule`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Every NeMo model is a ``LightningModule`` which is an ``nn.module``. This means that NeMo models are compatible with the PyTorch +ecosystem and can be plugged into existing PyTorch workflows. + +Creating a NeMo model is similar to any other PyTorch workflow. We start by initializing our model architecture, then define the forward pass: + +.. code-block:: python + + class TextClassificationModel(NLPModel, Exportable): + ... + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + """Initializes the BERTTextClassifier model.""" + ... + super().__init__(cfg=cfg, trainer=trainer) + + # instantiate a BERT based encoder + self.bert_model = get_lm_model( + pretrained_model_name=cfg.language_model.pretrained_model_name, + config_file=cfg.language_model.config_file, + config_dict=cfg.language_model.config, + checkpoint_file=cfg.language_model.lm_checkpoint, + vocab_file=cfg.tokenizer.vocab_file, + ) + + # instantiate the FFN for classification + self.classifier = SequenceClassifier( + hidden_size=self.bert_model.config.hidden_size, + num_classes=cfg.dataset.num_classes, + num_layers=cfg.classifier_head.num_output_layers, + activation='relu', + log_softmax=False, + dropout=cfg.classifier_head.fc_dropout, + use_transformer_init=True, + idx_conditioned_on=0, + ) + +.. code-block:: python + + def forward(self, input_ids, token_type_ids, attention_mask): + """ + No special modification required for Lightning, define it as you normally would + in the `nn.Module` in vanilla PyTorch. + """ + hidden_states = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask + ) + logits = self.classifier(hidden_states=hidden_states) + return logits + +The ``LightningModule`` organizes PyTorch code so that across all NeMo models we have a similar look and feel. +For example, the training logic can be found in ``training_step``: + +.. code-block:: python + + def training_step(self, batch, batch_idx): + """ + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. + """ + # forward pass + input_ids, input_type_ids, input_mask, labels = batch + logits = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + + train_loss = self.loss(logits=logits, labels=labels) + + lr = self._optimizer.param_groups[0]['lr'] + + self.log('train_loss', train_loss) + self.log('lr', lr, prog_bar=True) + + return { + 'loss': train_loss, + 'lr': lr, + } + +While validation logic can be found in ``validation_step``: + +.. code-block:: python + + def validation_step(self, batch, batch_idx): + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. + """ + if self.testing: + prefix = 'test' + else: + prefix = 'val' + + input_ids, input_type_ids, input_mask, labels = batch + logits = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + + val_loss = self.loss(logits=logits, labels=labels) + + preds = torch.argmax(logits, axis=-1) + + tp, fn, fp, _ = self.classification_report(preds, labels) + + return {'val_loss': val_loss, 'tp': tp, 'fn': fn, 'fp': fp} + +PyTorch Lightning then handles all of the boiler plate code needed for training. Virtually any aspect of training can be customized +via PyTorch Lightning `hooks `_, +`Plugins `_, +`callbacks `_, or by overriding `methods `_. + +For more domain-specific information, see: + +- :ref:`Automatic Speech Recognition (ASR)` +- :ref:`Natural Language Processing (NLP)` +- :ref:`Speech Synthesis (TTS)` + +PyTorch Lightning Trainer +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since every NeMo model is a ``LightningModule``, we can automatically take advantage of the PyTorch Lightning ``Trainer``. Every NeMo +`example `_ training script uses the ``Trainer`` object to fit the model. + +First, instantiate the model and trainer, then call ``.fit``: + +.. code-block:: python + + # We first instantiate the trainer based on the model configuration. + # See the model configuration documentation for details. + trainer = pl.Trainer(**cfg.trainer) + + # Then pass the model configuration and trainer object into the NeMo model + model = TextClassificationModel(cfg.model, trainer=trainer) + + # Now we can train with by calling .fit + trainer.fit(model) + + # Or we can run the test loop on test data by calling + trainer.test(model=model) + +All `trainer flags `_ can be set from from the +NeMo configuration. + + +Configuration +------------- + +Hydra is an open-source Python framework that simplifies configuration for complex applications that must bring together many different +software libraries. Conversational AI model training is a great example of such an application. To train a conversational AI model, we +must be able to configure: + +- neural network architectures +- training and optimization algorithms +- data pre/post processing +- data augmentation +- experiment logging/visualization +- model checkpointing + +For an introduction to using Hydra, refer to the `Hydra Tutorials `_. + +With Hydra, we can configure everything needed for NeMo with three interfaces: + +- Command Line (CLI) +- Configuration Files (YAML) +- Dataclasses (Python) + +YAML +~~~~ + +NeMo provides YAML configuration files for all of our `example `_ training scripts. +YAML files make it easy to experiment with different model and training configurations. + +Every NeMo example YAML has the same underlying configuration structure: + +- trainer +- exp_manager +- model + +Model configuration always contain ``train_ds``, ``validation_ds``, ``test_ds``, and ``optim``. Model architectures vary across +domains, therefore, refer to the ASR, NLP, and TTS Collections documentation for more detailed information on Model architecture configuration. + +A NeMo configuration file should look similar to the following: + +.. code-block:: yaml + + # PyTorch Lightning Trainer configuration + # any argument of the Trainer object can be set here + trainer: + gpus: 1 # number of gpus per node + num_nodes: 1 # number of nodes + max_epochs: 10 # how many training epochs to run + val_check_interval: 1.0 # run validation after every epoch + + # Experiment logging configuration + exp_manager: + exp_dir: /path/to/my/nemo/experiments + name: name_of_my_experiment + create_tensorboard_logger: True + create_wandb_logger: True + + # Model configuration + # model network architecture, train/val/test datasets, data augmentation, and optimization + model: + train_ds: + manifest_filepath: /path/to/my/train/manifest.json + batch_size: 256 + shuffle: True + validation_ds: + manifest_filepath: /path/to/my/validation/manifest.json + batch_size: 32 + shuffle: False + test_ds: + manifest_filepath: /path/to/my/test/manifest.json + batch_size: 32 + shuffle: False + optim: + name: novograd + lr: .01 + betas: [0.8, 0.5] + weight_decay: 0.001 + # network architecture can vary greatly depending on the domain + encoder: + ... + decoder: + ... + +More specific details about configuration files for each collection can be found on the following pages: + +:ref:`NeMo ASR Configuration Files` + +CLI +~~~ + +With NeMo and Hydra, every aspect of model training can be modified from the command-line. This is extremely helpful for running lots +of experiments on compute clusters or for quickly testing parameters while developing. + +All NeMo `examples `_ come with instructions on how to +run the training/inference script from the command-line (see `here `_ +for an example). + +With Hydra, arguments are set using the ``=`` operator: + +.. code-block:: bash + + python examples/asr/speech_to_text.py \ + model.train_ds.manifest_filepath=/path/to/my/train/manifest.json \ + model.validation_ds.manifest_filepath=/path/to/my/validation/manifest.json \ + trainer.gpus=2 \ + trainer.max_epochs=50 + +We can use the ``+`` operator to add arguments from the CLI: + +.. code-block:: bash + + python examples/asr/speech_to_text.py \ + model.train_ds.manifest_filepath=/path/to/my/train/manifest.json \ + model.validation_ds.manifest_filepath=/path/to/my/validation/manifest.json \ + trainer.gpus=2 \ + trainer.max_epochs=50 \ + +trainer.fast_dev_run=true + +We can use the ``~`` operator to remove configurations: + +.. code-block:: bash + + python examples/asr/speech_to_text.py \ + model.train_ds.manifest_filepath=/path/to/my/train/manifest.json \ + model.validation_ds.manifest_filepath=/path/to/my/validation/manifest.json \ + ~model.test_ds \ + trainer.gpus=2 \ + trainer.max_epochs=50 \ + +trainer.fast_dev_run=true + +We can specify configuration files using the ``--config-path`` and ``--config-name`` flags: + +.. code-block:: bash + + python examples/asr/speech_to_text.py \ + --config-path=conf/quartznet \ + --config-name=quartznet_15x5 \ + model.train_ds.manifest_filepath=/path/to/my/train/manifest.json \ + model.validation_ds.manifest_filepath=/path/to/my/validation/manifest.json \ + ~model.test_ds \ + trainer.gpus=2 \ + trainer.max_epochs=50 \ + +trainer.fast_dev_run=true + +Dataclasses +~~~~~~~~~~~ + +Dataclasses allow NeMo to ship model configurations as part of the NeMo library and also enables pure Python configuration of NeMo models. +With Hydra, dataclasses can be used to create `structured configs `_ for the conversational AI application. + +As an example, refer to the code block below for an *Attenion is All You Need* machine translation model. The model configuration can +be instantiated and modified like any Python `Dataclass `_. + +.. code-block:: Python + + from nemo.collections.nlp.models.machine_translation.mt_enc_dec_config import AAYNBaseConfig + + cfg = AAYNBaseConfig() + + # modify the number of layers in the encoder + cfg.encoder.num_layers = 8 + + # modify the training batch size + cfg.train_ds.tokens_in_batch = 8192 + +.. note:: Configuration with Hydra always has the following precedence CLI > YAML > Dataclass + +Optimization +------------ + +Optimizers and learning rate schedules are configurable across all NeMo models and have their own namespace. Here is a sample YAML +configuration for a Novograd optimizer with Cosine Annealing learning rate schedule. + +.. code-block:: yaml + + optim: + name: novograd + lr: 0.01 + + # optimizer arguments + betas: [0.8, 0.25] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + + # Optional arguments + max_steps: null # computed at runtime or explicitly set here + monitor: val_loss + reduce_on_plateau: false + + # scheduler config override + warmup_steps: 1000 + warmup_ratio: null + min_lr: 1e-9: + +.. note:: `NeMo Examples `_ has optimizer and scheduler configurations for +every NeMo model. + +Optimizers can be configured from the CLI as well: + +.. code-block:: bash + + python examples/asr/speech_to_text.py \ + --config-path=conf/quartznet \ + --config-name=quartznet_15x5 \ + ... + # train with the adam optimizer + model.optim=adam \ + # change the learning rate + model.optim.lr=.0004 \ + # modify betas + model.optim.betas=[.8, .5] + +Optimizers +~~~~~~~~~~ + +``name`` corresponds to the lowercase name of the optimizer. To view a list of available optimizers, run: + +.. code-block:: Python + + from nemo.core.optim.optimizers import AVAILABLE_OPTIMIZERS + + for name, opt in AVAILABLE_OPTIMIZERS.items(): + print(f'name: {name}, opt: {opt}') + +.. code-block:: bash + + name: sgd opt: + name: adam opt: + name: adamw opt: + name: adadelta opt: + name: adamax opt: + name: adagrad opt: + name: rmsprop opt: + name: rprop opt: + name: novograd opt: + +Optimizer Params +~~~~~~~~~~~~~~~~ + +Optimizer params can vary between optimizers but the ``lr`` param is required for all optimizers. To see the available params for an +optimizer, we can look at its corresponding dataclass. + +.. code-block:: python + + from nemo.core.config.optimizers import NovogradParams + + print(NovogradParams()) + +.. code-block:: bash + + NovogradParams(lr='???', betas=(0.95, 0.98), eps=1e-08, weight_decay=0, grad_averaging=False, amsgrad=False, luc=False, luc_trust=0.001, luc_eps=1e-08) + +``'???'`` indicates that the lr argument is required. + +Register Optimizer +~~~~~~~~~~~~~~~~~~ + +To register a new optimizer to be used with NeMo, run: + +.. autofunction:: nemo.core.optim.optimizers.register_optimizer + +Learning Rate Schedulers +~~~~~~~~~~~~~~~~~~~~~~~~ + +Learning rate schedulers can be optionally configured under the ``optim.sched`` namespace. + +``name`` corresponds to the name of the learning rate schedule. To view a list of available schedulers, run: + +.. code-block:: Python + + from nemo.core.optim.lr_scheduler import AVAILABLE_SCHEDULERS + + for name, opt in AVAILABLE_SCHEDULERS.items(): + print(f'name: {name}, schedule: {opt}') + +.. code-block:: bash + + name: WarmupPolicy, schedule: + name: WarmupHoldPolicy, schedule: + name: SquareAnnealing, schedule: + name: CosineAnnealing, schedule: + name: NoamAnnealing, schedule: + name: WarmupAnnealing, schedule: + name: InverseSquareRootAnnealing, schedule: + name: SquareRootAnnealing, schedule: + name: PolynomialDecayAnnealing, schedule: + name: PolynomialHoldDecayAnnealing, schedule: + name: StepLR, schedule: + name: ExponentialLR, schedule: + name: ReduceLROnPlateau, schedule: + name: CyclicLR, schedule: + +Scheduler Params +~~~~~~~~~~~~~~~~ + +To see the available params for a scheduler, we can look at its corresponding dataclass: + +.. code-block:: Python + + from nemo.core.config.schedulers import CosineAnnealingParams + + print(CosineAnnealingParams()) + +.. code-block:: bash + + CosineAnnealingParams(last_epoch=-1, warmup_steps=None, warmup_ratio=None, min_lr=0.0) + +Register scheduler +~~~~~~~~~~~~~~~~~~ + +To register a new scheduler to be used with NeMo, run: + +.. autofunction:: nemo.core.optim.lr_scheduler.register_scheduler + +Save and Restore +---------------- + +NeMo models all come with ``.save_to`` and ``.restore_from`` methods. + +Save +~~~~ + +To save a NeMo model, run: + +.. code-block:: Python + + model.save_to('/path/to/model.nemo') + +Everything needed to use the trained model is packaged and saved in the ``.nemo`` file. For example, in the NLP domain, ``.nemo`` files +include the necessary tokenizer models and/or vocabulary files, etc. + +.. note:: A ``.nemo`` file is simply an archive like any other ``.tar`` file. + +Restore +~~~~~~~ + +To restore a NeMo model, run: + +.. code-block:: Python + + model.restore_from('/path/to/model.nemo') + +When using the PyTorch Lightning Trainer, a PyTorch Lightning checkpoint is created. These are mainly used within NeMo to auto-resume +training. Since NeMo models are ``LightningModules``, the PyTorch Lightning method ``load_from_checkpoint`` is available. Note that +``load_from_checkpoint`` won't necessarily work out-of-the-box for all models as some models require more artifacts than just the +checkpoint to be restored. For these models, the user will have to override ``load_from_checkpoint`` if they want to use it. + +It's highly recommended to use ``restore_from`` to load NeMo models. + +Register Artifacts +------------------ + +Conversational AI models can be complicated to restore as more information is needed than just the checkpoint weights in order to use the model. +NeMo models can save additional artifacts in the .nemo file by calling ``.register_artifact``. +When restoring NeMo models using ``.restore_from`` or ``.from_pretrained``, any artifacts that were registered will be available automatically. + +As an example, consider an NLP model that requires a trained tokenizer model. +The tokenizer model file can be automatically added to the .nemo file with the following: + +.. code-block:: python + + self.encoder_tokenizer = get_nmt_tokenizer( + ... + tokenizer_model=self.register_artifact(config_path='encoder_tokenizer.tokenizer_model', + src='/path/to/tokenizer.model', + verify_src_exists=True), + ) + +By default, ``.register_artifact`` will always return a path. If the model is being restored from a .nemo file, +then that path will be to the artifact in the .nemo file. Otherwise, ``.register_artifact`` will return the local path specified by the user. + +``config_path`` is the artifact key. It usually corresponds to a model configuration but does not have to. +The model config that is packaged with the .nemo file will be updated according to the ``config_path`` key. +In the above example, the model config will have + +.. code-block:: YAML + + encoder_tokenizer: + ... + tokenizer_model: nemo:4978b28103264263a03439aaa6560e5e_tokenizer.model + +``src`` is the path to the artifact and the base-name of the path will be used when packaging the artifact in the .nemo file. +Each artifact will have a hash prepended to the basename of ``src`` in the .nemo file. This is to prevent collisions with basenames +base-names that are identical (say when there are two or more tokenizers, both called `tokenizer.model`). +The resulting .nemo file will then have the following file: + +.. code-block:: bash + + 4978b28103264263a03439aaa6560e5e_tokenizer.model + +If ``verify_src_exists`` is set to ``False``, then the artifact is optional. This means that ``.register_artifact`` will return ``None`` +if the ``src`` cannot be found. + + + +Experiment Manager +================== + +NeMo's Experiment Manager leverages PyTorch Lightning for model checkpointing, TensorBoard Logging, and Weights and Biases logging. The +Experiment Manager is included by default in all NeMo example scripts. + +To use the experiment manager simply call :class:`~nemo.utils.exp_manager.exp_manager` and pass in the PyTorch Lightning ``Trainer``. + +.. code-block:: python + + exp_manager(trainer, cfg.get("exp_manager", None)) + +And is configurable via YAML with Hydra. + +.. code-block:: bash + + exp_manager: + exp_dir: /path/to/my/experiments + name: my_experiment_name + create_tensorboard_logger: True + create_checkpoint_callback: True + +Optionally, launch TensorBoard to view the training results in ``./nemo_experiments`` (by default). + +.. code-block:: bash + + tensorboard --bind_all --logdir nemo_experiments + +.. + +If ``create_checkpoint_callback`` is set to ``True``, then NeMo automatically creates checkpoints during training +using PyTorch Lightning's `ModelCheckpoint `_. +We can configure the ``ModelCheckpoint`` via YAML or CLI. + +.. code-block:: yaml + + exp_manager: + ... + # configure the PyTorch Lightning ModelCheckpoint using checkpoint_call_back_params + # any ModelCheckpoint argument can be set here + + # save the best checkpoints based on this metric + checkpoint_callback_params.monitor=val_loss + + # choose how many total checkpoints to save + checkpoint_callback_params.save_top_k=5 + +We can auto-resume training as well by configuring the ``exp_manager``. Being able to auto-resume is important when doing long training +runs that are premptible or may be shut down before the training procedure has completed. To auto-resume training, set the following +via YAML or CLI: + +.. code-block:: yaml + + exp_manager: + ... + # resume training if checkpoints already exist + resume_if_exists: True + + # to start training with no existing checkpoints + resume_ignore_no_checkpoint: True + + # by default experiments will be versioned by datetime + # we can set our own version with + exp_manager.version: my_experiment_version + +Neural Modules +============== + +NeMo is built around Neural Modules, conceptual blocks of neural networks that take typed inputs and produce typed outputs. Such +modules typically represent data layers, encoders, decoders, language models, loss functions, or methods of combining activations. +NeMo makes it easy to combine and re-use these building blocks while providing a level of semantic correctness checking via its neural +type system. + +.. note:: *All Neural Modules inherit from ``torch.nn.Module`` and are therefore compatible with the PyTorch ecosystem.* + +There are 3 types on Neural Modules: + + - Regular modules + - Dataset/IterableDataset + - Losses + +Every Neural Module in NeMo must inherit from `nemo.core.classes.module.NeuralModule` class. + +.. autoclass:: nemo.core.classes.module.NeuralModule + +Every Neural Modules inherits the ``nemo.core.classes.common.Typing`` interface and needs to define neural types for its inputs and outputs. +This is done by defining two properties: ``input_types`` and ``output_types``. Each property should return an ordered dictionary of +"port name"->"port neural type" pairs. Here is the example from :class:`~nemo.collections.asr.modules.ConvASREncoder` class: + +.. code-block:: python + + @property + def input_types(self): + return OrderedDict( + { + "audio_signal": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "length": NeuralType(tuple('B'), LengthsType()), + } + ) + + @property + def output_types(self): + return OrderedDict( + { + "outputs": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + "encoded_lengths": NeuralType(tuple('B'), LengthsType()), + } + ) + + @typecheck() + def forward(self, audio_signal, length=None): + ... + +The code snippet above means that ``nemo.collections.asr.modules.conv_asr.ConvASREncoder`` expects two arguments: + * First one, named ``audio_signal`` of shape ``[batch, dimension, time]`` with elements representing spectrogram values. + * Second one, named ``length`` of shape ``[batch]`` with elements representing lengths of corresponding signals. + +It also means that ``.forward(...)`` and ``__call__(...)`` methods each produce two outputs: + * First one, of shape ``[batch, dimension, time]`` but with elements representing encoded representation (``AcousticEncodedRepresentation`` class). + * Second one, of shape ``[batch]``, corresponding to their lengths. + +.. tip:: It is a good practice to define types and add ``@typecheck()`` decorator to your ``.forward()`` method after your module is ready for use by others. + +.. note:: The outputs of ``.forward(...)`` method will always be of type ``torch.Tensor`` or container of tensors and will work with any other Pytorch code. The type information is attached to every output tensor. +If tensors without types is passed to your module, it will not fail, however the types will not be checked. Thus, it is recommended to define input/output types for all your modules, starting with +data layers and add ``@typecheck()`` decorator to them. + +.. note:: To temporarily disable typechecking, you can enclose your code in ```with typecheck.disable_checks():``` statement. + +Neural Types +============ + +Motivation +---------- + +Neural Types describe the semantics, axis order, and dimensions of a tensor. The purpose of this type system is to catch semantic and +dimensionality errors during model creation and facilitate module re-use. + +.. image:: whyntypes.gif + :width: 900 + :alt: Neural Types Motivation + +``NeuralType`` class +-------------------- + +Neural Types perform semantic checks for modules and models inputs/outputs. They contain information about: + + - Semantics of what is stored in the tensors. For example, logits, logprobs, audiosignal, embeddings, etc. + - Axes layout, semantic and (optionally) dimensionality. For example: ``[Batch, Time, Channel]`` + +Types are implemented in ``nemo.core.neural_types.NeuralType`` class. When you instantiate an instance of this class, you +are expected to include both *axes* information and *element type* information. + +.. autoclass:: nemo.core.neural_types.NeuralType + +Type Comparison Results +----------------------- + +When comparing two neural types, the following comparison results are generated. + +.. autoclass:: nemo.core.neural_types.NeuralTypeComparisonResult + +Examples +-------- + +Long vs short notation +~~~~~~~~~~~~~~~~~~~~~~ + +NeMo's ``NeuralType`` class allows you to express axis semantics information in long and short form. Consider these two equivalent types. Both encoder 3 dimensional tensors and both contain elements of type ``AcousticEncodedRepresentation`` (this type is a typical output of ASR encoders). + +.. code-block:: python + + long_version = NeuralType( + axes=(AxisType(AxisKind.Batch, None), AxisType(AxisKind.Dimension, None), AxisType(AxisKind.Time, None)), + elements_type=AcousticEncodedRepresentation(), + ) + short_version = NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()) + assert long_version.compare(short_version) == NeuralTypeComparisonResult.SAME + +Transpose same +~~~~~~~~~~~~~~ + +Often it is useful to know if a simple transposition will solve type incompatibility. This is the case if the comparison result of two types equals ``nemo.core.neural_types.NeuralTypeComparisonResult.TRANSPOSE_SAME``. + +.. code-block:: python + + type1 = NeuralType(axes=('B', 'T', 'C')) + type2 = NeuralType(axes=('T', 'B', 'C')) + assert type1.compare(type2) == NeuralTypeComparisonResult.TRANSPOSE_SAME + assert type2.compare(type1) == NeuralTypeComparisonResult.TRANSPOSE_SAME + +Note that in this example, we dropped ``elements_type`` argument of ``NeuralType`` constructor. If not supplied, the element type is ``VoidType``. + +``VoidType`` for elements +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sometimes it is useful to express that elements' types don't matter but axes layout do. ``VoidType`` for elements can be used to express this. + +.. note:: ``VoidType`` is compatible with every other elements' type but not the other way around. See the following code snippet below for details. + +.. code-block:: python + + btc_spctr = NeuralType(('B', 'T', 'C'), SpectrogramType()) + btc_spct_bad = NeuralType(('B', 'T'), SpectrogramType()) + # Note the VoidType for elements here + btc_void = NeuralType(('B', 'T', 'C'), VoidType()) + + # This is true because VoidType is compatible with every other element type (SpectrogramType in this case) + # And axes layout between btc_void and btc_spctr is the same + assert btc_void.compare(btc_spctr) == NeuralTypeComparisonResult.SAME + # These two types are incompatible because even though VoidType is used for elements on one side, + # the axes layout is different + assert btc_void.compare(btc_spct_bad) == NeuralTypeComparisonResult.INCOMPATIBLE + # Note that even though VoidType is compatible with every other type, other types are not compatible with VoidType! + # It is one-way compatibility + assert btc_spctr.compare(btc_void) == NeuralTypeComparisonResult.INCOMPATIBLE + +Element type inheritance +~~~~~~~~~~~~~~~~~~~~~~~~ + +Neural types in NeMo support Python inheritance between element types. Consider an example where you want to develop a Neural Module which performs data augmentation for all kinds of spectrograms. +In ASR, two types of spectrograms are frequently used: mel and mfcc. To express this, we will create 3 classes to express +element's types: ``SpectrogramType``, ``MelSpectrogramType(SpectrogramType)``, ``MFCCSpectrogramType(SpectrogramType)``. + +.. code-block:: python + + input = NeuralType(('B', 'D', 'T'), SpectrogramType()) + out1 = NeuralType(('B', 'D', 'T'), MelSpectrogramType()) + out2 = NeuralType(('B', 'D', 'T'), MFCCSpectrogramType()) + + # MelSpectrogram and MFCCSpectrogram are not interchangeable. + assert out1.compare(out2) == NeuralTypeComparisonResult.INCOMPATIBLE + assert out2.compare(out1) == NeuralTypeComparisonResult.INCOMPATIBLE + # Type comparison detects that MFCC/MelSpectrogramType is a kind of SpectrogramType and can be accepted. + assert input.compare(out1) == NeuralTypeComparisonResult.GREATER + assert input.compare(out2) == NeuralTypeComparisonResult.GREATER + +Custom element types +~~~~~~~~~~~~~~~~~~~~ + +It is possible to create user-defined element types to express the semantics of elements in your tensors. To do so, the user will need to inherit and implement abstract methods of the ``nemo.core.neural_types.elements.ElementType`` class + +.. autoclass:: nemo.core.neural_types.elements.ElementType + +Note that element types can be parametrized. Consider this example where it distinguishes between audio sampled at 8Khz and 16Khz. + +.. code-block:: python + + audio16K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(16000)) + audio8K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(8000)) + + assert audio8K.compare(audio16K) == NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS + assert audio16K.compare(audio8K) == NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS + +Enforcing dimensions +~~~~~~~~~~~~~~~~~~~~ + +In addition to specifying tensor layout and elements' semantics, neural types also allow you to enforce tensor dimensions. +The user will have to use long notations to specify dimensions. Short notations only allows you to specify axes semantics and assumes +arbitrary dimensions. + +.. code-block:: python + + type1 = NeuralType( + (AxisType(AxisKind.Batch, 64), AxisType(AxisKind.Time, 10), AxisType(AxisKind.Dimension, 128)), + SpectrogramType(), + ) + type2 = NeuralType(('B', 'T', 'C'), SpectrogramType()) + + # type2 will accept elements of type1 because their axes semantics match and type2 does not care about dimensions + assert type2.compare(type1), NeuralTypeComparisonResult.SAME + # type1 will not accept elements of type2 because it need dimensions to match strictly. + assert type1.compare(type2), NeuralTypeComparisonResult.DIM_INCOMPATIBLE + +Generic Axis kind +~~~~~~~~~~~~~~~~~ + +Sometimes (especially in the case of loss modules) it is useful to be able to specify a "generic" axis kind which will make it +compatible with any other kind of axis. This is easy to express with Neural Types by using ``nemo.core.neural_types.axes.AxisKind.Any`` for axes. + +.. code-block:: python + + type1 = NeuralType(('B', 'Any', 'Any'), SpectrogramType()) + type2 = NeuralType(('B', 'T', 'C'), SpectrogramType()) + type3 = NeuralType(('B', 'C', 'T'), SpectrogramType()) + + # type1 will accept elements of type2 and type3 because it only cares about element kind (SpectrogramType) + # number of axes (3) and that first one corresponds to batch + assert type1.compare(type2) == NeuralTypeComparisonResult.SAME + assert type1.compare(type3) == NeuralTypeComparisonResult.INCOMPATIBLE + +Container types +~~~~~~~~~~~~~~~ + +The NeMo-type system understands Python containers (lists). If your module returns a nested list of typed tensors, the way to express it is by +using Python list notation and Neural Types together when defining your input/output types. + +The example below shows how to express that your module returns single output ("out") which is list of lists of two dimensional tensors of shape ``[batch, dimension]`` containing logits. + +.. code-block:: python + + @property + def output_types(self): + return { + "out": [[NeuralType(('B', 'D'), LogitsType())]], + } + +Core APIs +========= + +Base class for all NeMo models +------------------------------ + +.. autoclass:: nemo.core.ModelPT + :show-inheritance: + :members: + :member-order: bysource + :undoc-members: cfg, num_weights + :exclude-members: set_eff_save, use_eff_save, teardown + +Base Neural Module class +------------------------ + +.. autoclass:: nemo.core.NeuralModule + :show-inheritance: + :members: + :member-order: bysource + +Neural Type classes +------------------- + +.. autoclass:: nemo.core.neural_types.NeuralType + :show-inheritance: + :members: + :member-order: bysource + +.. autoclass:: nemo.core.neural_types.axes.AxisType + :show-inheritance: + :members: + :member-order: bysource + +.. autoclass:: nemo.core.neural_types.elements.ElementType + :show-inheritance: + :members: + :member-order: bysource + +.. autoclass:: nemo.core.neural_types.comparison.NeuralTypeComparisonResult + :show-inheritance: + :members: + :member-order: bysource + +Experiment manager +------------------ + +.. autoclass:: nemo.utils.exp_manager.exp_manager + :show-inheritance: + :members: + :member-order: bysource + diff --git a/models/tacotron2/NeMo/docs/source/core/whyntypes.gif b/models/tacotron2/NeMo/docs/source/core/whyntypes.gif new file mode 100644 index 0000000000000000000000000000000000000000..56b5ab154391c95d393e5b6d48f2170cba260f07 GIT binary patch literal 309073 zcmeFYXHb*zy0`lzp@fbg5Q>PPbg4=Q5fzYbLz)=rAVLJBgQ17edr0WL6MEfPUTe3r_dYXc&Y5@RLo$;*`H;-y&i(sc*I(nlx}5BbCy+%@9PpPL ziA4IRN8k#HM(^I^M|W-;+>;WMfPnryd*zsniV^?=qJO#KFK+?Bq~Mt1flY?owAz`7 znuV5AhR`L=!wf_1onw~wK$%Cksli|a223qQ{uAE(zj&RCg!B46vt-_BbztBSt z4RaNTH1rWY->Q!z^-3o<>QCun23Gc)en*e2AD*?57+`{Onnre>C`=KdcWU(r$}S{- z-`L(OJhiy7Z~DhE6^F2bmWhKm8e7uTKeKj1%_*X&ZTiwD0$18RFuQ(A!zHSuW9H}^ znO4>^IJa>|d+EBe?z2~ZQR(HaL-U)z>A1yI^vs?7qcbYnh8MQZ=`V}l(tqy!HYT&O zedO~ti9sTZ7(hnDp@fpi?hK*jRPcpL=5&Q$6ftQ*N#=G(k+>AS&!AFyy>VByu}a}m z`Ta?vrcJ(7(glO5H(t)Pgi99=rz`oKoKf8<8qK~N!J&-4Q9PcngH!OMzFG3I_;IO8 zEBa>XM7deB_it*MvZ*So0jzR_O!-Wm<7|^3jcmnSqucsSYlLj&LaXoT$!{7t{9;Ei z4W~+^T-DP1NKQq6TKVd4eTgEbZISXd%R_03KIgOwwX0)!+BlUcg}Sv*Wv0#kbc*#G z)3q;W+oBX3w&q)WPS5F-2s>Z8A~d$YAH*rXrB`m+|M9WZv^`q6`CxOd*@r~0 z(sH!>Z2)&GMy2)S*T!t~TZUV0XUF^Nv+Xgr+RuNVot~0N3~(fvUK1Wb!Ht3kQj7J% zgXmSDFM^r$G+%_UnxkHXaya$A2;=sLTA+BNH7&yVGf)<2;fh|12(dP(Wu(-wre&1O z0?IO4VXN0NM&%r86{|*n-zrX>JKQQ>ORUc-K~IIsI`N_2ed{Dc^Kk2A6Q@4w6mx$n z8;nKteVbI9jBp#QLq(qr&Z&*cHqB-DzHPeuLbz>)_g0^6rvEvWT~;8ymR)uzH`*=- zE!J+Cti0`D=cC6~=j&Q8*KM)&mtQl6Jc~v{Er~RsK!aU+t{fyJV ztA+)C8YjY1w6;^@az=zx(^|!VQ}b3EjdRQ1u(osS!9s*{+sW2|bNl%@4T1<}&_Q%i zT#iI^Qi~5F-qGKpeci>Ruk-pn>+{Ih-5kz?uY0)P(z^8W#^|{8@n=T5^b1!Gx(tZ5 z)4C2yjp(=z$$XA<9ah*LbRAJ4(YlSQG3dIDsb7wA8`lyaa{Hini|)3ZB<&gWVQ#+lGIli-5UR`wZoB^8eAZ3xj3w^yG%E4F>NNyIFK zrYJ{DZ=Oq^qeA9rk(*YzN##_T`@p7m`y$>`M+`TzTI)UJ)ckpe;Hy_TyM5H+KeByF zLIwl|1&4%&p~BG-kx|hxv2pPUiAl*Rm{cq-Ej=SMD?2AQFTbF$sJNuGth}NUUsYXG zTUXyeXl!b3X>DsKc67e$df(mC+t)uZI5a#mIyU~{ zvif6fePeTLduMm==l-vQ!=vMq)3e{_BmhFqrHRk#2!b*ToAu(e--R*qX{Bja<-Cu8 zNjuE-R*`ahVtDSLx$alz^(6>CDlzM;&L2pTu#Hf$Hv|Ap#uMfz z208SBYI04gKOPq9Wb*nk-u&^H;z5kVyjF+MlS*(s`kq4a%MaD;6*2&4z2lR5qmHI0 zFW(5xSGbO8DJveV&k-y@FRwe4IGDAE9STd(dV;^ccl*5j-e(eFUY&4!&bC*G8B3KJ^e^_ z=WCS5gnv177fBG#ppszRcvA9}aM(p#Z6^mHq3{-}5=PUaKA7f)amf-&MRHtOyBGld$Z_`|z?neIfY|R6Kgra*gFXDH+ ze&u9#XUlyryqSY8biJ}ylw}^VW>@HP*B&Zl%`pSzDOMMj7br-mH5Pe!(i-~0#Ne8? zstlhGPsKL z{J{IKVqR&TK1Pp)QZ1AdT#8@igDH(F8dvCjU=3w!A_6V(9KQInz;T~qKe}z)LMfMp zkVBHJg6~Wk&rk2~F%rCIRK@(290k1u6%onkX_F1}9`TngmwI3|!qa&&3)(;5-Kh`} z_Z9Z(tS;ecP}oc7|2e%g9>=XEGCcZz4cBw!)U$Blj-~Re;Vg%AH~#07Mbgtd^&*mu zFDwh~TIdT~>*j3U>0ArM*OznmO(iSuO^1XOrj2Rd`t9DG2AaRQq?T#sk;CTmyJV0A zA$iwKxqfJs|MHo=?O+PdET}LWSR-=0_9}w2G0gB$fALTxcusmPo$dO~)o3M!rW|a$ zxp(FD7(WNclowCW=GB7uO((pmeSWZ*hVV+sa*O_&%isk#Ks>g^In6T|NeE`dq@c6SB6 zA4FULEHBic?n+4^BP`~YSo7`P%eVj$HoiIR?SyVQ1!Sb%MLzCuRF7mvT#TbzF3)w< z_iD+zu`Xv{Aw@=A6vPWrXtUg_l=u5+VB>M9w5yl4d)0-U_Tv51tgdMYfeg@6iQ!yd zx!F?H^=+6EaaZ$2<-_|-G9HFU((49fU&oB%VM=U^wG_#&A3j{2 z`kw5Y6qWptzPnUZms^1-oxP6D!Gmb`E-S86f`)2!eDVw#zieS~Kezt{+#9#=d~ezN zybBT%8?C^6F*^E&|cLlUaV`!889N18i zx@`YgwD*244_&dQhSqpIc0W&0w^-|G|9EoKe!fIhv5u|Q2kgv#f!t8Bo=5+OjFbIB zRk{*`Fs+X{9KVV*bxR(m^nWZ+_*HBWRr0t{>r;uzuM(r762pf6PZi$3O5t>+Mm<^+ z)!1KUcDkh|ll>F*O~1+!QKe=pT9ZvPzbZV3O3e@ZC)-YbRU+xi;FQ`^og4>vlx~>? z+rU(}!a-GRRGF24_H@6=K{aNm%tm%#df5A*CX24zPD6WU9D7h(q+9OrbYNzp>7WiD zRqkl3Jv%dVP)`^tck&pRT{t;tAktMJ!nEg>I1UNDx)m-d19QtHg~P_ts0z12?fEs6 z!=|aB3ipPA`7Q6m=0&)w z&*vwH?EpOkisO;budY_|rJoHs4 z4V@(>+)S-EzZK(H?_exlfOJROzc@!*ss!avt|; z>Q%?541VKRJnl1yu1+k}`7UgF+;22oo!l_^UCifr08U?n>Csu1!W|FV>D6E-2bX1< zkB1P^HEAn4D+;s6!=A%68Ha-_DyPRINc!3=O5Ig8&J)roO0PDDZD>_p@nkGEx;9Th z_lK70$v9@Xwm^31hn~;Lhb;QKA`RWOhq#lEMS685PlwhFn@>LBqwC6Sb=OU1PbLV% zbrl{%>*l8?lSKM@e3t1%Fu?5;_38gbbW21?xut3>CDt{eSO2wrjyU< z>>_;wp+|Si1$R2Pcf?=!-V^u!hV|6ri;2zOFC}|7&ra^v%eJyZx9iTr7xoG=%YXJc z7a=Gn*3<)z&_g_A?rUEdV+dUWdb?=TvnALFp<8x%H`?dyD-T0suZG@UJnrn9pnhY& z{4g=8`RuzyOyi)f-cRi8*|OY7QThTjg znoVE&+19*pIDFW4`g>ceu%4J&PJAJrHU+xdP|5tyS?$8VEM>ue_Ae?bDrRP8E-o%s zUJx4}6vju+b&clQwQJl07kC6NUJ+!yD#Rrq!XqSlRa{*BU+r%Hw7~smcjTl+zW>A4 zxxe4$NdLh)H(_?YjhS42PK1=oZlUllvu)4_ck!t-!KGtm@8bg0v}Z0s{boLlwykHf;B2#N9-H0`7ifMXKEd7FwS+ zWIW3`aGdK+pi!w(Aa`Qw%Y_^XKWp4t10|QD{5UFhF4UDaw>iy{fu~2SUW$)#-P_!9 zc(1|M%0Zy|Ong6k`O4cmi_d+ZZX(^*vuz??FW2=fkUVN7?K)j~7YpXSs5?Wsty%VZ z;BP;a8+?An;-dNN?eh?UOTO=qJvx!!Zwh{P+)h`$@|t0v@^Z-hdnhD=D_Y=1Aay(@ z!@gXco-Hb)fsH=|0YUbt?%e+|(X7&Y^_r=gQo^P%&nr5Lx ziu1&7(NhUVbWtf-Wq98Ek%q=Pxxa{F??k3y{BmZ5QI#e~RynCoVL04E-D+{NyNi5A zYQcenP1(2X)xaVemAl!v@tWBR z-1VR*6{8Yf?gh9tn!d(j$xcS?Qi%j|g~;nf7xme)PI$V>R#$t~iF132lTu(kiw4Gl zApXu|FDOsUcc7=xrD-pbUB*nLwwzxhZH!`oah|vkX#aDpjN@h6z#g{jP3_?tS7}XK zC~NJc+?hzl)Ijk-&5Ufb<9emq=N!Ix>b;E|d+)|)nG1_jD345UwzbKsif3~*UtV6r zu~mw)DanH4R}`H6LM`a7M)gvpZM%ssX-SiMo+k9z4EBNNdB4Vm6D{3EJk1-fA5QQK z6>H-JGY9jX^%gciiwOio&uKu_%?4xV^fuUjHt{@^rmUgL*TaY{NSDz4add?FA70;II8S|&E3eMEQz<;Hy^x0-bs5YXngt2 zb(hCCW^}ny55$tMZz7jw##i&M@})fZ@%&=mqVVQq^BtSb>rAr`%8f4eneFt&@C2tZ z&r6rXG6vAACD#PWjcz`q(0hUOwXp;T*EgoC$1Ze!D(7}>q8xr0eZ4nt%Fe#GEWa>? zusD`*a81W4^)xOjFi)xIM&dl9eBk8-TBtfgK+JXdXEuDC^9T%T2o+`O1br||OAQ5GM zP-1`E8eL;ci35@I^{e<()TB3F%xU!P*H3f*sGmD%d%fXqW2P3fCP}9_6}YDoD|KI$ zv}dN4WHOACaV!2nYcl029Px~0G)Jb{Ug+i>amN0~1;1&EOBbjoQ^r{e_}@j-I0}d) zHZ529|JIc)wd>=5LLUKZF2z!_>qQ1$G(?BQIdXsKO}d{{R-9!Za$%w}B1ew`OU}ew z%KY@i_<_QgJLUUDjgvL`0{HQ1hRn1A1;f_m#K(zczLCGCJScv_moCh*T&H<3SdfF} z5dIFsW#7cM{_fejA6z>Eb2QIKjuT1u-oF3XR_sMkVol>bonX`)c) zmx{zi+N(QbKcb?m5-;PCdJ$Qlgx!(FVZL^kUFCa%WPU*`M3+@EO$IaR-D&YUfjlwr z(K@|@thhldcm#Zujt5kpId6Z5?{rGuy}Ddu)A5Cm5BZRoOtGKenpK0z*cY*mgn6A; z58gFR*bW~+8RzZ!Dol@3EDSb)n?DE+gMBm`i zCg2Y?kKpX1r}WJEISA$}#ms%{-bGzfu~y}jpW&0J?hwjw=KHgC5{HlKGtKs1eP$>1e%zRAIqta=~|vg4I2PH8H_jgkYViV7;SY16asIxsb<3A%>nIMwk#2LWtQ^i1|?n z92ROJ7iwh`YU3GdhY58cggQ=zIvs@~U|}wDVQxlY?w(indVUeA3kzGcS-JX%Xn8Z4KAglE(gCTfNdH8&Nta1^x&i(Zn8{$>=t>>0g^iC!Z_Z%jpR z9YycJV)o=>_Kji=JY$Y9F(-tWv#FT#qZoia7Azl2W*kf56$?#`rEZL+osOkHj%8$z zW0H@%XdK7t6$eX=<7kZInvUZ>j^kmE=arAYY8=n+6)%_?FWeX}Ivp=|952D1ASIu0 z!#F|4D?u(bL7_20X*xmWI6;*?QB6Mau5qHeSE6QWqE=&~&UB*QaiRfx(nI;A$Hqy9 zUP&aQ)FhL}B(v!x^W!8qd$NUmvXybNjaRZ=YO+IPvg35J({VC_J;g;n#mzXy-7Cd2 zHO0Fz#dkWz|2XAz6?k)C_qgx5>r(`^0WieMD)C-+GDtZ-=UL)c1uz^V^5eOL0V1{l zpv}V2@(2~|ayo!h%dT)_OY$Hda46ZQ@)A>NvjD1CfY!j4d?&RaPH^gu@l7e90EBIP z!4Xr1y`hT0Q@swf$Ju7{XIB~J-#3P+$xD)7p1l`e=H-Yjc8Pe9n$^eBPxZ)35FMeJ1kg*DE?vEPRa8{;#*G`-6hQn6G=hp0ii(ON%CurCOyalL zC2p}u+~Sl}WtG0oDWk@xrlux;_nP88Q56j-RSoIenm6xhDQRfm*3!ABudn~toJvmO z5%@;{{dai6e-c1>Aw~}b4G^d|;O-3bABW)H%X+70tQ5V; zLkgW0pKIJmZFDcQkYB)F)~F|n?!iVKu2cdtRrS}yI?17S>SZ4HzFdq^Y&N24+_EIP zOy?*&dRB*z`<^nvB#^2BOw3Xr+Ft#7`lm(kOaT}d$v}&cyR?8 zuaRawdru>s!J3hua1oyL{NsNnKN6lal`3D!{E*YE=nNa8VsLJH4QSH z_v6iD<8$MqCh$`8=V%6^fze4`O4cTgOjjl>Vj+^Fy6{B|=@;eKKppI27UHbYV?F{w zXS5hp)JkENauq(k7(YrEYaX=?I)(%3?P+|RvTVKPDx){u`a7>G8l4k00iPX_be2fz5-1r zMf1uPS(c#Vd9IsuRU2#~N~5!0HAvTLe9rUZm~ko}9^$a}vyP_D$_&@TR6=DePQ_w? zyAB&s-diH?niu>VEeWgC)R*947Pb=6L_dXB;VcPo%dD$|TKOFpey(o36UErQHmY(T zgm+2Qu9W3*F`Nl@@*{AU>J5Y=FBda!+alUcXXHC8=j%$gTW^liI(bP@4%!a#4W0-M zs51?ACbF$a2~fRL7!vN7>o1c+wpQTl-d%lr>Ds54mh4SfupTwndK6K*8_Wa6R*omGIf_9%rik$y(X&igw)a}pLOqidDe%d-3j6dqLfMU3SY3543AVfxN!5Kv4E^)8W;PSM32h)T&y?CO ze=&l+O5~DS7=0yW1^XJvCVgp{F+!c|d0duTat&RTmeAKP7k*o3t>0r*tpPJ%)Gch0 zK8b!1vip&<;{)-I`G5v-I*aXjT<_-eXxQh|h$Fq&OYf?(5||Vi1X;eP9Nv8Jz%2DS zaZRAXaQT9BQWq!O_ekR#E#8@&EF-`%zmbD_k3#6~{K~gwZrf+0H;0Ske;*L~Om=!@ z6iTDPOkUcJ4;;h92?arBiT9lzN<7PCX20}>;`W3*!Vz`K;NbsB`}3Qh&cSKcC^Cvp zSMHwyQ6Kr_1n!GUG&0NXxv<9EFxGv__|ow0*9B!u%EkaGDS+{J`kT8cgQJopI&UVw zw2h3X(m{R^F~hSTc;DHma-R1jx<2L^XPpkT!Vczw?nHt^-A1MBLp11yLRf#^*IiW> za*|EzFbGl~%JAN|eq&*)`rE774>Xvx_YpU?;8iQ|>nC#E5$^4f%MMOt!pPz6X;U;F z)(ioYaf}qZq$A($&iXe84CMLr;x5|I&K!x zOx63j*;P9}GcDz0^}1bO67g(iVN?!(j2V^UvSuXC<)|D?xRtOxbE2M?2T9x4>9$K= zb$j-L7sf&;4ZY6cu3cXVsYxO*ayzZ-cops6g>|BvBwjbKH0=fuUX+)7{q_$OkX@V(9y7LHAzKqG@lP ztQypR9?E)rIZqJs+DKV#i#zIVf~itq!>!QV=^>twKvvnl`wzOrXZA~%1B|_z*1U{; zS(U7wAUx~}l)gRCZ=QtBdAWMYZe-X&P4RH*jdZWd1x&wcMV6h8Jwh1{+=(1E`UD^*SbzSE3<#N}u`kwUm>_&R;< zzr+&;7XYr>s7)tr5QK73hEj?cOc#rzWPu|aCg+3cxZof@BUM5Yum#;$g;3&Un-Beh zprQ7ngB!$ z38p>rS>hldiOhH;xt%H*O!WxNv$KbOZnnUPm!smv1GETWjvh`J^)o=k44eYn^$p~t z*x`K|9zQbrF!>LudLU+`&)Of8a@@4yRXGjUk8kEDAM5nMc`p^$?&$l_bHR6z3~3OH z%BK2EF>o!78F;1uIjx=L;pv>U#wAE-lV9ASMlF6rXf(~q^Ee|Z&jD@?g}zUEUL3ug zJG_hp!h7?Bu*u&)^cW5))MMN-6I9m=*a8+cR1ZdOrTr`mYk@L49SvU2I!GG%O%ogK zLvQC#rhOVLR7W*{gT)l$hTZdXkYL5>Rt@s->QO5+ zS>@=9Pumeg%sE(?czzIZy8ry}_@_MM@4ct(n|wQm?{`N)!LFZA0Z;JWgbxW^i1;?? zWpL1C)9}-cS8adH|YOkPH-LdiW-T%};X$fY^b_ z1za=>oke#WDtM~;F)s!r4Hs~e7)`g$io73C_&o-V0NPW zeM(;)O6UEjVFgb^uc-!5c?4`zgenO-JIi^C?u6Rdq4<%gP!vi4aB`Oedl&j5r~EDo z23q1lu!g|h15^SGIp#rOXyhMLh)9Pa<6#u6s-QfmYpKyc0_gvf0E%SC1j=KAjWMBK z7*r|--H3^t#zY@uV%byU-lQan{mdnSH@~;##&>>#&pKkamIf; zfHHDqGbv7>!0f-L& znzJE51+M6M@)RUt?WRV=2fh|YFf?jQ!R+A7H+N< zovju-t(M@dky5O=VOk^OQzPe7?eSjN?H<3%X^kppt(s!(UDH~1pIS{^tyXib&TOsT zX{`Zg-9yE?$EI~-hCX#hxH=QXTI2;T)6+UQXT61Dy_IRbjZeKDuHK=!-f_0x*oW18 zz1~H!!OgV6-KW75*WlgU;5*yk!P4N*NeEOV1e+2wy7`z+<}o1qm{HDk z5yLIx3%86fxw*OhwIuk@2=+g{MoxlX{YQ)N-)a%o1|deDxFm?bfJoEG8pSV5dg}I-Wq}ELiXi3`FK^&N9DVnPB-+U-h+H6xa#Dd31dp z2;+nMD}lwAK@vM92u_*6l_6jQTM-EZ>;`b(E1?UXQ)JHF7)WAhIxhId za2Kb(`M^5Sg1PMe=&&zt&!xBgB@U6%^eg;FpRC4KM*3Z+iIw6N?us1m_3=X4Qg?>= zy5ptXx`Zqeub*Y})Tb0G1V7|2~6i)31(oOPLNwV4Y({;|UD zgUx1pMtfo5scHE4^2JmKO6YR3v8X4%so#!r3i$1zJgeL=>K zg&Wy}I7YL=g}0t?fXF)q!JZ6Tf%$GuKTuXFENj=;Q%*MDcP8`ctXRiWOe`9qUsWJ; zUh%0jev7-7HZkjWbBUebS|LZSFhGMz=v@|c!k+*2wNaK!IZs`t0&@**ylC>TwcJ;W zJ^keMqQJ7?M==!!TXd-~a(rdJ6XB+!_-NaNW#3XmMKf6ZA&)U=Dnh&_PX2 zEPt>poL9;RwQFJ~b&^zH`A-k;Vjv;|wLO<$GC)n)LFHPD4C0@}=W%#Ma_3Y~X44zV&j5j#E=>8{` zEZZdS`V~KV^+h`qxduizhdZ*mJK5p)L@Nd#l`59BUU8)}&D8iHwB2v*jb3|hu_CcG zZtkN~Fe?Zv7PqD#ZJ;aW$hBV&{8xFpklL?{p?6pPp!im z_;Kxz6wUus*Uq-N6Od32i_ByXg@c{wH|usr5``Q56SW^USxllgh0Ypco(+l|ls53f z5G7;XY#th%o8`k!Lqna%R|;O?>|cMC#AKI* zwa{HEKuX<(ui-7PO2(CY%3zF&$@XDmah2xB;3t%Cs#4ad%JmpO@Eif7iZ0~DaDzu5 zFRcH>w-aN{{!G|;%zU3?cbS|Dt(pZIG~PtTsq76lkFXEl3j$h4*A+2z<8tH z#@*GogS*d7iM5Hx{~CNM{4A{Q&CQ1q?SZ*JGm?n!=SwB6zn~Ksh5G(DNwJq9#H{U;tBYr5%^<=Hu;gX`MYXEUPO3d0;3g3 zknVmD%(pkOE%sWr;jzlY#=TgXno;sbyCLy5Eo71_wvcImz?oa;p)n0pX#pP+MEIHh zp=umbCDilMCdx?~;J)EeRdaW1PfWq_8jJm?zSNtSxUBfkuWgDIZlM?TmcHgA`;8^U zh8|gx? zT~3DkUvEfWBVW2I;&v}XZ&#c`P&KeCjMqW=BXY=!Pd`X_{G#a3jOo<1mr`9xia%Cw zj7SNIsNXC}N=9UrC@>k1SU!@x?HurNNRO zb3mWx_;RhcQ*#gw4x*3=cxz02eyV_> z*z^LIgR^tWlYh(KjR~8x*zJUMDW6yRznWY6*?IJyoxdYG zX3(1AoyWkVVMM|o@lk|wx=Th3lvw3xDMBI)APhUO8i0)q&T~*)PlD!Zr6;=}7H^OX{PhVjO-S#uoG8UlxCimgwS8aNouf?^e zCm(YcYKsJ(nNzJCzT5gP9)2%v0W5yrY1A$1Xw-d;Dwg}ZHbg~Wageso>*Q~^+!+Xj9o|X7B zv|f3*^k3@wY4`bV9d z=Q}9>FHG;Sy3&Ti;l=!BOI0x!k1yGSw$jQ-UAVU|A1(U|lKgQTXLfYF)m;Z|xJN7Rvf>&a4fR^chfU%$S!OVXo|67dxc`D^!wFoNV zY_E&|u0`-^j86WW79q8mLz(JtT7=@n3;Af$g@}-k=y&_*^81kmuzyCWYX1^~LjUa5 z)YR1fm&?n?$0s5pA}uZbCj{NOb4N={>+$2qCMG79mXY-?W&V?i9=u_G{6@gggUirE_^Ib*6R)dg-Z!3k3&VZIE&QbY{QRunDq6q2 zYl~E}L#n(Cyz3aK^(yFLaB%SJ5Ixt>$8Mo!?qP5bl!*rl?iK#b2W^2yqx~c81EL&* zV_ZXHyuxBV!eTwqaemPW!7+)b#Kgq-q{tLZ5(blk#bW>KiR}M2vi-k*0VN5k`v38x zW`FM{|M%78Po+jR4)gtWrIVHLcj!yn^<~pFb{|Tf4b+#D(0I#f3D3B zG*o_WM?z@0bqM$`?@-Jl=7WT)uRXDR+8H{H)!zp&(k~YV8*5fZvhGB1>o(Q?_)zqy z)O@I^ZhaDOIgp{-T(3Yuu;y}re=(q*YYhk{rY|uv%+Zm3vB^4(rTrG3EGynMD~sUi zNqIRrd}nNT36;#y-{$AzvG+a2gVN$fK;R)2EZB%-=b7`FmPn&tsT`d3%Kv5>&(Qd;ftgF^U4}LX48CK~+wJEkv zKED>24@lE95u<|Ojo(wxQ@sk3pV2q+gBZl5tl~liRlo5#RrFnDbQ-^}36W(|MP(>a zgh3fBLxsMj$BL@1rV4J^qY`eHG+F~xWp>3G&q8^=C(}SJR`Nr~iynYwS>ezupo=OP zkd?x(CaUua^~TA%-$#)fR~SJ{fFiB=sEST%WErlaxYRYCRf-?5R>6ue_T|WzX7~#) zEmsFcYW+DPuQt8n6hMDcRQBaPx|a~8RhkbXfQi?AS|6U;0ZoR@*4U8@cCJb#4jNUU z4eRg#YCPP##j`8ZAaRAjF9gL0vhrG!>O**Zb2}rY%qp>*p`z6}V+*j7Dl`91xLVD3Pa1?>cCjC;gF)+x<-(%bJHd#?Knfk^eH^`Vm)di%^xrg8r!Yc_! ziyo?1U16ii;di3p8QmT}z|!u6=Ns9PV4>boB#23W&YF58!x5q0O?}Lo$IN+7yD9>68Da@Mc9|BpjhjeOM1Fk$ts9#X{r)V!G5C(u=P)djr&_3yS-bMYT z5d_vYiWudgtg~vEty^=MQv95pA%OaIf0R-7YSs_34q>k$hWiYAtQpjr(BHe|Z-S5< zfSn-0|H>BQQBW`q%Y)AFd^DphEq9_zZ@bqehu5%{UhpqIJp96{`X27drn)9H>X*2= z`Ny%E-lA(49V)dXt}gHmUIip37(F%~k+DS>sm`~G8wKRVFn*G2pV{2D8y5+#+Hmi_ zCS3J6vbO6Y=SLAaiw7jv8Z?h!V{&iWO|i&}uOG(USiTDM>L7!CA9~Xl_N2;(hocR? zzeOhEjp^`?Ju|}_2$Xu%1xM%6z!wsAKeH;Sq~DT1UJ}>uQ5E>@5|^Kw>fWZI=mLVj zQB~z-G+zCP6Eu@5)yh@NrV}~2uPk}ttx4K*$VKnmML4sWDR10` z_P%!V8%Pk10WUC4C3%BB#&Z1z56|^lgXi+Gl;s=oZOKQ9BjjjY5Gq6Ex6lPPcMUBi zc+zTQv7Y^p*J+}pL<{k3;U{10oQhbl@pBhHHAqSi4~0Lm-P99&$b;7cn&gGDADa+0 zs-q3apb6yUUm8!|JW8N6>smMDH!N!`dVWK9eOmz|{PaFmu}(UQi4$4M^Jc`JnDe8I zeiS+EZD^l+@$T7G$?1LzTOCYmoXw~G>EUk43>oXO8tIPPI#2*|Ktd`U&TYf*T7Lbn4P8ec1zURdYYIQzA?u#61l(xFrqw`EH01CRM!WZ7eqEut zC>k3}z*cN&N-^H)Dzdq2QYA_%4_Qk?f69g5Pu}P^57o>R>NC=cpSHB& zznjNb^@GKx??cew>!=rH*JLNuf9r3DyXSXLlYRzawp3T^_Q_6N z>4Y(NSc2~%DINM!jrK#Tzh;a7(3)mW1EZsvIABDZy&JBd<`znpM$=*R^ts#kTNbm7 zUvl-g^x$1YeTcZ!|vEmZ5*X4kV$=U{4M8tA`Nnb3~x#aiOqpP%f2jb!Cxn8x>!@|+9GUE zva`ydq++(SHN-akdwL|0t0Jqh<*+4FWA9(+v|=|Ta)+9NjJ@FxtdUJ zjBD#eD%tb(xi2RV+RGX;C>BK!Bhp5@dy=1@v=|UTt-4FUeXf$bWq^J6$u@o9h1sYBx(m)#sk@4 zrM-gm-(&{2WC#4>fpFPbhZ7-e1mu6JCX@cgO)d&j*nx~>K~xxnH3eyI@PL$gptn>Z z8b#C!NJx4Xm~%AvJ|?7e6v7k>3POMwvLGdl5C#C89t&aE0htg&ivTbK011)+y(pp< zGYHKff*4dGj)btBQ3!(s=m9E>Ar{1L09G}Cu*1Q^h(Cbq4@U+CjY7D9@IWMlTLQ#P z1W6*Hk^q=F7A4OJaXAWSkA(y>LbxPAEQ_E5PgFPo#k2@wM}UKQ!0h;Nt5uLMAtD10 zW>5`Rl>jmBgfa7gRaHY7b^xXwz)BDlgonKKM8CsC_Su0$RUv(Ud}RXI^CAodNiX3RwJ13+{zM>7DazP56GxDV$YL`8y@yrF09ZfN;M1OzZw|`L@@(o$~NyC@3`$XO8jMl)O`HWDWn!+)O& zZWnb}6#Kn5IB+y(1QykcfhZUxxERF;A)~$JpiF%U{jiX&BS3jHa(6W%5fz3Igt`=g z1*Q|@;Bg}IP%CJp#Bt16Z|p}*k{BZ|_c6dZidGha(iTS?dO{xEhnm4cf5T$9?NNHi z$&vU-Q}(1|xj1lfq$GRnwH<&B5B{SjE5e~F#HjwFuoqNhwDysKHIX+}NulmSP$y4t z+Y!cdH5uWR8U}=ls6t&lArI|iJg7kHU!p-jA~<)T@kcHO5f*QP=}PD9cVJ2B%%IX$;Na^f@mvZ}wiDo+90_WNN@r=x;J%-&iifBdL6sMC zgYl4nVyKF0-hGaki+y?F5)fZwDD-}s;tX!8DE@sGL>QkKya-`KfL4t2F;uyF9JD^E zp)Z=iOo&{IHJq?P0g(|RU;r7Hhe|J|zWPtl@PErq#+H;cm6XktRGgIHIZCS)N^4C@ z>%B_}*wUt^(w3Rhwv$pKM_H#rS(iy!w|7}DwyeLYY;dM*_@r!PQ6_eus6b<8Q6q;+4W^4W{X!w7V z;pRHC**f$8fDAjG)+0C@{w*2yKW#w%YcfnCM9%&l8cz0U#Nhr04Rbc-C^qH&4H_<+ zZK^nJ`X8g=37^&}T)dSX!fERwXWNou+c(p;WuLZHT-#c6+s16$)@j=gXZxOF z`@U)WflvDpuKlFB{cN`V{InhTV__)~$;^lpzC>smk-CLQJ4d8HBQkPzFe!ChH0xmX z?SQ3qaI|!A&2?~}b?|U?@+x&+HS6T}?G#Mw6mIDho$C}k>y+SnC#Cf6h8gLdjPEbh&zrS99MnbxJ%(xo%krFYh4!1ey2()-6|@4u`=l;L?M zEpK=S--qu&Zc2bJGJ>qkKyqH4JzoNXfo^d;SP%&oPzBL10^*EdDg%Im@iH|WbS0LD zVibVF0ZN|B8P=bmQ|IDXe0V3Gt49toEDDeOo(L?nRK;{6b1L^Wk z;Lkt$Ss0NQMwLL@!^Z;_lmK(9f|4V7ly;;gTzWLX{jg5Y-^_dgbG_iS{#Zn;0I^?S z2YhoBBnb!Ao=JO{{W(uD;5FC>z(M4Qp<8#P{BIIFX{i)ONnqzluqYDDVE}rQ#zXvI z3`GEj>Oh_jK#>JpSsbPsHKvXQHYgM?IYISS${sD%Dxjp+baEq>jE=quRd% zh2Q}Q;~10j*u`gKtbSv#^f8XsF|PSB?%!iPm&SRO$FDvc=l2^IOdl6+9T%M+7yCW_ zzu0@vs3zNWQTKh*NDwg~(n~_`A|Qek1JVUVumA!g9Rv-CGyy|3^w2wo4uTkp2nZOF zUL#5`2BcaLv4NuOeDhmn&UMz@XRSTv9^>q@=YRe%_`!Q$&wX9Dg1}2f&6mn{FI7Tc zia!vGZt&h5d#Uy7B~IX#j^-;}yI1<5uM7)b8MnVOnSW*W>y^2{xP|8UdAsq8q2pEs z<2G_~>zywz{~C7?m~hmbaJHLp4V`c=nDA_$@S2~nl9RFjFyX5?>1Q|TA37OWFd5uF z88Scl7$FEtc zz+9*1T$kNkPv~4s!e|D*U~EcoaBwh$e_>%^X=&-dhJ==umj4C`FA=F*xCt$B8ieF7S?3~Q)LgI5J9d#oqxTgqtq$+!+iF&1}T~C*3Z*TwKP4E!= z)a3u4cA)-iaBlzahXelSbnZ_$_X@$XE#CPA7Y;#~^CCN|vj!ueNDicYC+1l=EdE2V z2ms)mg+V00k@+qGJ(8Jno)xo>}181St^6a zhuucH)8S-%he@fn3-w;?>U5|!C>9tqtTFJB|Md+VuSq0V2-q^VUspm8 z`PkVagJvP3w=bJdQ`vU)SO9Y1J_sx@H6ID2v+aR&p^#1AxUD%r|X;nCTgbcdl5UpTUUJedp8ro4W$$45VZTtwFnJCuZe)#N#$2g zs6`+N#965zf!jBo|Uj?Y2~r{6BX>`N&Rop)lEBZIhq8Dijog(^3-@e zJP5ki#9Yh-3NAFYb|q+;1&Q4?`eH0wY;JO(I{V0o%~cAsXp0QHo??q`BtA8F-?SN! zovq03)knzrKfCDf@xYk!uJ*28*O?nfnr^wtP`FrS-V`NY6vwkmD(00n7!M5=HumWZ zkiA!RyrP??=K0<=>ojL>KE5bi-F2U*`i!Z|Lj6yl1okwOuDT+X9eT9f<=N$@=jT-F z(NDKoO2S2iYg}GDf?uxa>2D2|d8CJCrjAlP*B=yK!pp*jb*5}#eL|Eo1w-Sx`CU=c z*A6R_niA*b*p5XOkS8eJRvU?0!3Vw_mMzhw&|LUzqq@Qj=B!;TV%!YN>%@HV54 z_4q@;U1f9TN!IX(XWi^Yxtm>K_&s!7Za6Acdx_o z&WS?dbe(UM9{V3q)_vwz1y0%$gf1Qj^a0|t+TkAj<@)r{+!Y-<`h~~X0ag?rtiXxw zY>Zjo9FwCJS@yVV*yI81-1VDEG!I(TD8)9q@+IGx@_O*r?FIH&f7qn#{t(Jj!;x>Y zLiQJ!5W<&$M&DiQg>lw}S}HArZqISdNvv@nTMrgbR3XNa1jC=<%*&jC3X+Q>f%kavbmMcI2r^m zz2sdoR_!^huN@x(BgokK<#BGaGaqhy!(*_)a@(3sUFdvSgjJ6E0*aoeGRH4jMph8U zGlQ+NR!5V!I+nl2T1qIARjs>BbZtIEEuJKFC-jeaH~Gd~mYtSQR_N&UQR3@8DLh-= zaMmW(GS`wQxzHqbj;eW>#1DV1vteK^m%4t7Tf|+-@*>umRr~aQL$#^no=%njedWst zxi!@lsp#+CpNj^$?`7Mnv^76SLo#JVn8)K@6>M)N@v}>vO$ij3B$0^v+4Sf1yWNP` z?K{8bmNIx>73FQqRdC;WOXCcV8VASuPQJOMhw`s$8eF*cQg`abi(wkYYT5jLC(lJLy#cQ$_utDLRfoOwyjWxsxDd+z&Svyv ziT32xO9O_tUwEj1t@@dCI!Xc1?0s|ZT4N1Sx~0*Y?8COy zOm5iu0qbn$hqof4rr@m@wj%v zTPOO2ROv(}GQ=g(^KM?>v$;Zc6hyJc&(A=Jj=OMKZ##LSw7hu7dB@#ELEe#VulULI zs;5xfr>Ha^Gma~!W%phlXdo-g_39EHd{cArRg;}4IJb&=_^zjczcNO1Cq1(2(94&Lm3jLjCa1E zMe)~Ik+V?tN*r65fS1UZ;E6{g{zop_fyg;3?+*+LuHU*hGdIJ!bHsN3t#BjX{I2|o+EcjeQ9_GgCu;y(n<;ikq$`n`{jJ&av9 zJ@kYP$9k(cIsffq(hj?G!PllMbMFujfvKk*es#C=Klq*4>AE!ku|oao!lM1o?2Aya zc`al`huAvb&3diqGIfhBKU+z44^6A! zN#Y^w#^qnock&?v;Ky%qC+xo!A9{k&IYeZlcYJGcJNfY!d8cb%Ysd7gD%RW#_Dtj- z7Y_J)-GKf7OTqzvuN$!ce@Qst?`a$R|CMmSf7rPAU-LEujX*YZ9vdy^FdbmUXsvw` zj0d?3i-6>Fyh{zI!>se}h4XG+1qfIthISQp6&S(7kmi6+d6s}e-c=Fqky!4#;f%*- z-dHR@sR;OeB>VEJ&;bXaufKpXldrW%0?~y<@N7PZg7_49X1aWT3V^@^gcN?-Gk{0m z!7CPo<$_LImZTuDTy5abZU95u*#2*O8@$CnVx}s-|5-TT?|Bn*9Kc)o%BM2#ufhQ*Q#b6JQ6K3kCocU8`o3{wruv@Qe9o<~ z&vnO}W|{jlwq752rxwGtA2L3#V(T{AeS(aH_;Ct>8z36%_e9r~?6Kl{;pXwc9eZEJ?F)1ixo!dnSpe~z2gonVu7p-E29$V#9g-2yL2vg?@a3Yv zGYji0!-!b%Qe+8|ol%$e`glREDFt10z9EOm(qF}-+{{HdN?f?{<%27py2+@!+9Yar!1GA#W0 zhwhwtkeIuW-5A7z>o0uy%j{Z1rT*dAx_+rjmbHeP_4x-JCgyKRx?ef{e&Jz4ZZ0qb zt0wq|AZrd@nmuvPgwNKDcP`*{HYDrlv;eo@m9l42EMFfzt#^AwxaBn8IqdtgpavBx zofDA{sGe!qpKc{;*fRyBJ(2b8`&N0CrTk+?Fq5Zow)K46n7CUI{N8<@XZP6>s;bm) z8~?0IxHeH&eD1Q_aUkutbZKj*NLxL!R^3P%w8w!RuOmZoO15W8w-q^l>FLehU*_3+P?p?jh63l#St|D@uody zl4H?jHOD18AJI6x7k_0+$9mTmLjD@rcn&4|)RI=!vB9nJUq z9=Dg6qLiD{rd)}4al(&iuy4|{A=x)0W>Ca{v(L5R7sQojUCQIMPHTT><-Yegk;eIe zmo+8*X34&&^tCJAKc}th8?v5zB_Z%W6NUan1ws86F!nD+w^ApR)5Yq7&M;@~XapEA z$DtGB28Nn#*CWo_=dub4?Ui$|4>l-8jyYMrRiI4msUhJniFZ`bDH& zbiDUnQEE8pv|XFNf*QNKM}mdM@&bF1nt46o_cYE~zLQDG_n2$Mi4eNH3vOsBXd^oz z;oiFaYP+TDIbxgN+&(u`>D^z!YC)xxR>k!zBtgz}Am@ z)r1mDabRpcA~-d{a#A;YUDj&&q<@;GG@m5g5cq1<~S@>fbDmvp3KWjb-FEXM1eGEsUBG5y>$PVvlfU#TgyLBtjt zEf6~(WwDsf8?V2qY~6TIf3fmRjY?%!MS5jMeHpEy)$z)YFB4*;QzP7f|K$ZQGmvx8 zs)zZ!e8;sK=OZ%Ky%sV6)I7(jCuYLs*mWSo&g&unVZc|Ko&$?|)p6bFcKMt!;t$k77KS<=bttaNC+mv!u`(f6m9j?%%Zk~J?q$vN#SP?LxQWf=AuVmg z_NR=zzL;p=;WEzV7y_K&=dnI?W0@2rsWs`KGT> zxYI%CkZk@KSAMOeLww{mkKHKp_i+Lgb`HL~GU%a|owK85S*z5F`2u;1Wj?Ln{5o3l zY>sF9276k(SS3w(W$?d|A(rJ}R@m*&$8RH*0Z{z>G}EP+`pj6;3ac%7&=B~{+SRtA z_zr+raDuSDIq+a<5S%MeiFskqMJ=8H;l_pMSyNq=e&!(TdZ@XdG$M|dv<$e5YZs4u ztsgIO7(R^O_hXWzxUwwgyx=B(fiY{k>pry}69B&6GdQ@;9?}9Uwr|WW3-L6&M;LNQ zYAm+XT#=t`9W5QX#jFUdUgWZY#Qkne*E8L?G|Mw|pY>}77I<&Xl?&^bc+_|abIo0Z zZ@kmhuSD;``Ze|2ux{2zC{fCL(>u3k*0r>o4ba{`Oh4y~MyLmrf4#pMK$!CkyVY^7 zU^6U_Fj@Xh<-w))8{E#;^aT0l;?J_Kwm&aV9$Y(t+Ok`}sy+uplL9JFd0$IRv!?ZL z639Ykz5xMGpn|=D8B!14YRBgfqx(L$N`-=fk-V!1p9c2-V!3AYJ%8~y)8j!)^v0QF z%ov(RpzSMN{}|3?^G5ySvHB<6>z6)BC~GfRSj+rEf=+|Ch4?6x6B2V}dB(|3)@oB$lVg=asZh>$`){MX$&V{$||S?I62bt2OI!uS8BTlYsc z;ABBSP->D-iRz51LVt-uRmo|oRP4WwbZJl3WP%zFhcqZOG&F=h;w3y5B0R=MJl2S47K?1wjA@mQY1JVZrrZXwEqf*kU1h_QWRh zp9;fn6#Wl{AxZCrkI)MVQLgmRpWhh8_M;!=W1^#wXhDr^2dIfC=bJ;j)PwfT!5`Wm z8RXx3c5r(E%BoBQ7$4)0i7>RN;vB47yZ-f^RntCx;ism60O!YUCT$pps-a=04NRwryLx(zk36#MVy}NGu2Wf zHXt}4F_|BQA@!ZYP#MF1t+(G?VH)4(Zk9#0P2a#SFxNMXY6S!+k7+dAb%Qi(l5ur_ zG7VV>J3n)M=&aJPlHX^Dh9UertFSnX!*GiLIx+)|+{koq{p zU>4S1To*4^!xli%KQQDpK|t&Vf_&}8(n45y{l?5t7kCTt@eL%#%Uh~FWJaTDP5f|u zYw|8!Q@kOHW{To;Yzh;8JJxVregFMI;`EK|jwgbZgR!dDj<53Ypa5uJl9b4&q4>?b zfk?wV@t~+`LE&YCQ&H2&=p85aANUjHkifGnZI|gEIb1dB+J_7ff%Lha^w!Y_XYa(( z7Ct-QJqq{@-H5?YFde|^7~w|^csZi>ry`?)u=t2*+SpG~B9RLM+>Q^VGB3KNM6nUj zD@yk^x|YeDao4OOsjr{pA6M7oW{Y*=%{}z<`E$eUx1o_l%$9g|hi)UuYms>|n^XVE zk(0-6j~s^aRWbUpCYF1D&mYJRqNq4VxBjwlYew%)r%bo?dWE5cMvd88DC1-A8K zm1vbRrDo^^kg~W-^-5O~H}_+sv#0A98f*2~H&dO)L?v!m=^IAH0x2-6Xm?;#i2$sC;V=_X%$b$%yd#~=c( z9vzuJM1;s6Z-7x^Ew+3C{)f8Z{;g?{+K460as?oBgu?b_=!)9thO$sqnjaA8rhen|eMnlERsBGQY!{U*0Lw#+}O)n&U5TJQL}5Y=a>7ruWH zRen-|44zT4nh{N20e+YtUi@$_-aTuxGnUOmd(fJr&&}5{fZ}`HxG*AnDtqo5@*aC? zu#O$uViNqAP`FQo?mm0*ds>;fs@Xn}C@?pRq^6tVj^GnmF^UEy2{_3roSAI)(*^T=C>@l#Ps{vQ+OjB-+AGfXCqA3goyTI%)I;6Ui3n^{~+E}i31Msk_n zKK^2GsY;vijt@fCtwcetFS$}r%0yg^Ta&=c!iycY0U*A`fmdO?hSJoZa&Le`T!ol@5&Bx86f_6kvv)~!7EJZ36C)uxmo?t@!)_dBH zc%6ytL`r|uAI=)HGI8Q*k|JSd(yaM*YY3adQP>MUfv<+56!hz1 zBt4C&j$Pf{=~c1@-?3*md7+J<%XS|R`b_x2Q@YN+wd$zmQWy2RL0nkl1FoIzm}!@) z4LsDz#C#!a`@?7PjfVnufy{txzaM}(X-(Lg`2p12AUOM<02FKpJ;qw!N1v*coEthOE`GbvO74)ClZ10i9K1b9m0twf7+R%B_s1t0(oma>vVQm-c; z-$w?B9;F@^Us)CSQjV-yyVF!E^)1}!V5!L6@jJ{@XvkJTBya}G;j*jZ*U5xBOGH|@ zBZFL{;z(|M0Z1Puq}@)qi0vhMImuU@_;!N zw56(9I3(Vq?87h6PYE85RdTf)10Xg2EW+MGTUsB}G6ADvgFk5uno4)UT4z zgVX7Sco>&?sv`!%4WtMYppG;c7bSxq4}R?fBezHnQi!wW8JAl!PB|bgs2P`cVFt7e zE(}DNkfuk;5Ta%9;^5XnnRciQeo~?%4St~t?zlzbCMB8WWM^ZbOj7$Km=w|`2=r)y z+W{$8J#y?6Na|F$9U8_(06!Zt6qXbR5?q^p1orL|8f{0*}nBS#ql0k?J z%SlRx5?#}X3I+$&W!S*@_*hKQu=88EvxHL7|4_I4r^4`mlEQE?q}wXL+orACcCOog zuiJsY$5Erl*|x_uq{ltK$Fr@+Yp&<|UJstX*H@$0&$ib;q&G0XH@K}gWUe=Cub05z z7opJ?W!o1M(nrkii)-sknCna0>m%{^r)u=4+xBOM^k?Vy=eG6d&-EAX^^^GriZupG zZ3oIj1}gIh?zaup%nj7;4N&+8>oo=&Z3mk|2AlH-TiXWP<_0_V2C4i*of<=3wnIH3 zLw)%}18qY?b3-G0Lp1*3F^%C@w!;%4!&CXgGi}3jbHfXJ`@?kpktL0hW!sU}kdgKL zk+*Fl@8?D~_eQq(N4GUbzu1m`3mM(XAN|obx;Hoadv6pFpg}ZgFgqF|l!h#zF}Kq) zSrB%lhv`-Pz(v=Z3oW)BUU)DvDwg)0@KI9a8#&(1IS5+scZrHib7~K zM1`P&!hr`F$0~3_C^Xz;3phxKd+r6r$f5KdV7d-4Lo!T*028E)VJW}?3<#wINB}&H z2b`fGiUu&)WzII=0X)EJ{-Q-AK9veJ#*e9)LvbXi90fE+Lk>U?_Msy377A!GAXpEq zJclVw0LpUTSq|lBp$SKWaN7LBuX(z_!jk5~vfaXJ=)!uz z!rS(R_wx&zzZSLxUTMo{{8DUAV`O3(YH0lcNgWMj911E zI?Do`^*0?YxX7-x$YH<88Meq>xX9bF$iJ{C_j>` zuWPOO*{}JBtpygYv9qofaKcr#)(C>@5nAg}_UkcW>%_wKxQ_LNh4rN0>mn46Yg(8JUofkeZsBpPye` zT+HCA3~cK6Sm0JO5Y#Lh*n+s-$`aZt6ZV8X;xE9I(9Rjz!5i}w6GIh_rE0}K6Hn-r zP3SzG+;uFa`*=#XNq2X5dartBuVrSRR#qP_yYEu&fNtTCY04?Y3$Vlm^RppB- z_h0x`k2%-Ay#4a!%i32SlyUEdiJJ`*kxi4trm4WkQ?V`6p-*NbpUkFp%*8*Q%VxmS zXA8-l3x&^Lr**w9>Y``#F6Q(umiI3eE-ft$Etia}+<&q9;N==+eEre*M(fPGws-H| z&41{6^Kt0S*2v1Im+RY;?>^6c{`~pF*Tt=GtDnAaeBOEgeRq5J=eK_#V`Xqvw|{cg ze;ijm$;{xY^^_$v`kfg0;y3E0 zmWJ|%>H(Vz!}qRi{u+uyu?yn}xFY?Z1~Pp@Guk1vt*?V$SMBe_Xc%bYBdJW!pscFs zXRyU%irTQ)y-WO}dtl(gogu8FgP((>>OI&ITPtEPun?dRKO{-ULD|d$5bS6L5&&g$ zzB>B{S0(%lS5;e}+zx}QI_j@u0oZ!rzi?H2%y;p8 zC;dv%0;cSKVyX!*dw4RM#aE)>N}-S2y(laC8!!sZOaPJM!Z<*h-Pc`|U6r{a-tyw~ zM!qxi79lr+VC)`s+}&su7O}_US*q^&g^-NoE|vtuS#=1>9HK3yH_kFg!14g3VIPwg z0^@{jfS6yqrL`*ER||aclr9M7w5r5*wN_On`-UAL6yzdi`U;NoFWqm635$1r!kAY3 z9we}Cdp{(er;AkIwPiLcUlLTSN$z2dkOc-Jsx|;udp?P52s)0=ti`E7*w2Q@@>S~{ z7rk8dH1a*BJQs6#Nfa<=9;*RkxPeMx0g=5=52J|hOOkFHlRuBBr@hV3-s2qeu5#S= zQXbLcn{RjsND?l`cR^^>7AE2Ha@p^aHLC)wAVscC2oTQr=R1xJK#6xKcO24L;S^wSLt z`d)u!UGiMJG0N=~l>SDh4|bv2)NnN#Z{N43D`L{04?5rOp#?Kh`tn+jS}3wJ5qVmK zBNgdx3tI!%4A;LT9VCoa{;a&$?_@mKxSc}4iVHiWn;Cw&*t_s^AEUo1^IJH*;j+E( zuvuu3{>qc1eb2N8xI@}(-n?F3k61r|qp$4FenGQ6bxQ0HQ+_l5zUMr_c+o68j)3{( zu8(Z28wzkzw5uj@ojh|+;hbZ`IoNrQv&MK4=#lrLc^E~#)()?9~12}!nGqOP?r5? z(T7YXHoJ)0zrtw-&u`9VLrB#%5AYGL8cu5?3iUdoR0CF3-7cbD*Ged!T@FSNCQEiV zy}*Yo%baIDge{Kg1oZ?QY}s5g`-gsl}-@@ z&P3-4z@ewXU9MqmNH_fk_(`@)_!&+vY*?x&IG3-Kb5R=HX~<*y^tvJULiml;wHt80 z^zwXFqU_!WBdyC(W*5XY?=eFve?+A*J{T*N_DFFK`+h|W>tFQl?iv;r$p}dc<-!NB z#4H`Lv{Cu{XV#3UkdQO08Q+)y;dUcH$*Eiv(t<2WG6IZ-uX5JF2z@~pJfnc;zc`Qd z>+s70NnIPdkc|Stb8j!6FSe4RJmmrw`^8Cv$u(O7!Q2NJTG0&G%o3h zoxksO_*&Nk>$7JPGraVCJ;KwlMll6m7yWaZ2vJQp8)9gw;#h6_&#n@V<`y5l(%>)k z+cqMgmg2Hd+UwJtfoU;2;@8y92-Z1z7hk$Bx$kx-_S{K!X^7{~*=XC0*%KEbtyAaA zn)t_YP`)qd;~3e2FV0G`vEPMXJbxj#_!Vr7KH+Do+r=@uOXPmSd?nz1T{5rw8H2q5 zG)JWkCqkpg>aBn2_rni(zgr8+bu)ENIFgJpwF#5Yj&_Rl=$T>57~Bbc(@M z2kkzysrR{DNHL*sIW3z)o3GqvbrtjV%&9HZ#0rXqyJ?PoG!&eDn1=OP*0wUrw~?za zW>a%{U$24#P2ns3GV=JD;zaFi~IRMS8??v1~umU}e!JJv-B#nQhM* zsKo2p3SzT~Vx#8rWN`gDu(JAY?$E=NY(3?h#rsAc2Ilha5_w;1v5Joe?AY7~or~3l zCf^!|t^5%4xI(xtg*h+KD*8bzmZ5r#dj&0AOcILd4d zXMf~R!491i^S?oqlD=VvFzndQ7~T4X_G6r_oG&4dhLpnzzE5N}k z=7`>@1d2JF4~>X)fV)zduaMydIG7X$F*22SAC2H+q+QJSA>4HMeGHu20a`$Xb7P`GPJpQ45mp3x*?tl$DuoXXWi>*o zF1G?y7<0H`3uveY z*QaN2qaj!t+zkgaWdn92K&kNFh&8=ep4$+%#AuA+=B~kG_G-_kcw?BsXtl9#ufL=VV)`>Mt^E#(L>3#v*5xfpM6=1?>VVBP{a(4dexlnpp{gbtk00ev{(=KzqE z68pdwijqU=nZvm6!;HyL1I!EKEs%u*9994h5r8k@KpZ=WB3wt3q@LCRj0>O-65>+H zP~$BqBhRJ`ypX1XCq7^t%wM4#!cYpbY!raQ0bGlbQS}Wg3>+{KKwaH}dL=?t$xstK zTwx0)L>RY?G|~Dn&XZ_zXB$LrO(GS*`!7R_^CwbhaARN}rh|d$Z9yeS;1L0k13<8v zgPeaL+c+j1$N;y@0GQ&Ya%2l++XAEu&P9a|+H67PaFE(i$QU#8*vmmz9Ks1d`&f=r zKroLo2iVPl|6&?urhR^H{%^Ue-Mmw=5I~a!U;e*yRhT^;5k^NA(!ZDrzI*U@xM~u; z{{>y(-?(bnl4RkMG=r-aE?^e^6)qchESvm=tE&E% zH)Q<_R}EXSDqOMITu8eFJ!cMOa8*aGRcHHE*T3Sb*MG0#1=oE4imL{9tc5JBh5i03 zu4=#L2)VP=CbvANyk4{ZeQ)?*An7&0?Y{s?|8xrfgrrVRPJcqu;NajtA!%A#8UvCt zmQQPHY8a68RkoyhFh$m&0zGhmfFs5>+?R5*NwJaV>Z^jsQ~OS;~ur+w<+VHb(7u=Q#Tu@q8?AjKAs6`nN4h+4P)TZw)v>` z`837`LFendPI~fldSUlsdiP?{n>TL;mkWniN=H^7zF52ea=muqZR6y-b_O1uec!gQ z`FvrspZ>A;&8Okj?U(DHC%%07^8V}G=C`G-@5|deYoB-DefzQX4pvms zKNONaxO)fuuWJiy}{d z1=zC;9uF`K?+*p+(>OnIIrHK|yL6%LbsHVS;9O%6WoLS`*NKSJ-+}Xq&bslTm%{rg`aW#;hbLC z%8)Um0s&`0nb%haXpwKN#&$peulq6EM)aZ>?gJ7PxG|JG5a@f-T_mUns3d3m-WM(I zG(JEmi6k756+W^TXIPw@bfz3~mOK7EW&lEcUrAymh&TCxn#dS#G(xVIN8V5)2RpUa|N?Rq}GQEXg3 zq8kvP^g^BK00Gk6?k)=)Bp+DLje#*!Tecu1U`$m=RS}yo^3y$y0ZGBZbK71BNb{!5 zO~ySb?^`5YrmXD&xki@&Mh&;{U=K%}B<-fyiuV{R9qs3G7T(s|7^_ISy^l47?0(*iH%WSg z@yz?=^(xctyw+Z4M$mm8>e1J=-&~1ih8+Mm?sfvt6v#FoOJ-_GNqmDew1^b=euyc0M#Bzx|&ASWpG*ATVrt6Hsz{Lq3}Ggsqjz?|@2 z#(^%Yifnv#D6$88_#p=v{-%3W7%l=}2i!Wgkv?}iu$rvY6g zkwoSE1=H46xGv)_`05Il838Ox-LRHq$8soD>IZ4w2#+)bNc?!>U7jel;U5 zk!grDGfnlre^ummQ;im?-lp8LP_TQVD-ZLZzGUqI%P52B_O)TIO{Ugj>o_>*%uOSrW9zDWK zqa{(d+ao-xK;zAVM?cXdmaY2ugT)EXKiuGzIkY>TgHca(B?RQqFYN6f*fE>CV{9~9#1wZf5x^~M zPlcl6!(15A9PzriQAcm*3+Xq)sSrHn4u3lZYWm`l_9KC<-$&74&Q;xMl09g?z`Eo=~X++ zU31VVGvs>mS>#OgurXY7qnX59ttimImga&TlD`%+baC9|(h>KIpC%Qfc$sUijLYs9 zmi5Qx7CAs^!!`{Q6Mowl(47e~rL45`OpUU4FncQ^dvQc7)Mmn<5+AhC)gs5{5&DXQ zqxjvvG9U|r_{)Mj!QkvDR#!dvCc`3cq_^PyI zs&~pZSU=0mo+cKL}&Rc@KpsQP{nK5=B?lDrE`WgMMsz_Hd z^%W83Ab5YG|E@xjq8#HxA~gKWr%z(WYwo=qW(3G(I4<$>s~5Z}Rn`O1&fmFwR z%~d*?p^_KkDUUnZ3U0(2ob>$(Wg7tzFU%8>jY4qTRN8~cym^KmgUAB*uufO@B2y%wyzI=#MX!uzN>O#f_JL;SEQ(2|BcNzVIsfa zqM`sBc!8@Yj81-j_f5ym<`1#RkT*xec6j9oXGip29t?3Iw9d9;PnOpw?YebFof3GR z*7s}Z&SeQG_x?_AmTE7P4|GzWdAING{O6(@d;Sba`oQzm$YK5!A4V{#=al$i=+-9O zp}wiiz9^oSfAY z$2P0nbv*BHK{y=|Oe6;c zbt0!+BJXa=>NH2OXN9j#c!`&2N^U7Y(J`k3(Z?u+l#(kCNyHz|h{bq#q8`gJ2Vy)P zevHc3RBRpUw--IKt$%IvuN{}V`>9q#}fd^#21DS6-m1-z<&yg+4k zpuh|8Flij(Id@_W1Bv1pNEE`2PN+n~xfwZd+8;!8Om#o0(j3YSz!^w%nt?=ViG?%> zHz{FRHMt1$2NI10lDYAWo7YfR0$dt^>6t_Ls8DGdBoc>UQbS$_pcon`hlUf;a3%A^ zQ#ANFa*_aJtCR|lP=GTwfEZh)g1gC}8i@}Bl~#Z;%FNQ3lsP2yi)vaS3C2wzMW7+v z1dtC6QFlO?Q_?hI!8ryJMI$tJQ*@-#aSq4}lr%m{nk5N#E|x)8A$AT314)?g%?tx+^VdL2@7IKmxh(um~KK0VS_`WVvp|OW>0tXi2Ib2z9BH>uR~_gv2;0 z1mqYo7EaZW+EQr|^jw3S^q7`(ovQq3N)9(IrGO{P)ggOA3VALEK5m3`267x6 zlK%&L=NZ;ix~S2;)7(MCU=2lxs0<)Fu~Ri-qiV!PHwc0T)QqDJC~Blh2@nZ2w19L` zqXN>2h|(ceIz~lBYQ#p*-GDmkappMZ&OK+&T=It>$*<%I&y)47Z@p>iP((kg+BAlN z3vTgd|11@$b!ol~U33$g$c~%u+5(h08gf9VG)GZP_tADFhK0e1M9XEQVM_ z@DBH!jPHlSpNw&xSL7^@Ez!lKsmNn#u@sEW(Y6a$*OXnnF8p z$vd-=j#xAko9N0z?odYfsYKufDQ@%>JQMAXOR%PgvWCRR%eP?v_#Gb!Gl zO!Q_tN(YD19)q5nX}>~GIT~yofl|)^F+TA29I#7>f>lcvRJMv)9Y{y%!Y5T6l%W8r zPpH`}0L6J@+>;4Rbp;qO_XOzhc}fk;J!wY4L*9XWq#;nFK|!wNAQ$pLsV;)UJP<29 zh^3%F95|T-&f)>L-=Z|00Fyb@UY&Q(;V^4)4>UMPZ2@Yh5V2JNYD58WBMqX%`+wycoap~qcSWDh0{G2uirAV$`LUy3<8Y7=HhY#cv2QCrT;>e&JP6STH z`S6iCQEBM40)!$7F`t4kCWzgiSH72oiL9zWk2~!)hCbsh8xlY3XI~m_;Ld5hs{I}P z?5IZh?LS&S|M9DS_BSBuC;9Y1bN^2(x_(1G?J$xPP7!>9q@Uzd`E(oHSMuq|eq_sM z^69B3(53nE{-4RG1vZa+u0B~dCZB2)Jk@G>dgp-SDjIqf@mKlu6D0jrJ{^aopUJ16 zLsAoo17($qO7oUisya7VRDR%|lYsA(L*bSheCZyRLw}!prl+U>zv7<#LvQ~?^f@7i z{>i(6Vp;!`dkTMxdk+7_xM%bqH%r41IDwL!Yn>%4e*r`XB2%7J7FvVR8gPEU!e2_`n2&5wTICs)8AF-ui=c* z);mWtWe0Kmw&;a6sf9=hLQZ%Goy-nu8Rwqr!SU))&lvagPl<5J4nLqCbWt5rOJVia zp^oTkUv)JqOSgu00UUraIebD-pNOuY*${UpL5C7R2*0(SQQ=*dg9Q7JSwwE+e}>y2YENK*Mas zM%XhG@Vqe$N(_8GYC4xCRhg*hln%M|rU8S#XA>F5BmS5I7% z&uVRpkfEgYnk8)#Tr2|68wjHYsA|LIiQT~5bo1QI6S-LtxjDJHd5yURgSkAJydurK zV)MM)C-O=o^2&4bDjV~v2lMzc`FAz*@0;iU*Y5`YJ-FvN8XCTI-#jF82&16|LuI>h zAbjcmg}6>61trO;j(0&b1AmF2?E&akiQ+ZWw*dobHSZ#^fmsH_L)E9Bg7Bq#!7DUW zH4mF#h2w4qwafd&cVpd`J$Q{$TLwNgbUyW(AIGRc2~mV$RMH;sS;FG_;Q3PddI*6B zspw8DqXq#Sk`S6s6(~ZKJIKA>SQNMmw{r26@;*mJ+4tz&`+z&%$6s(Sl~DuIj!mM} zT(e&y-v{YOl}o}yga@GN1p&P()MYf4K+wS>9R& z)PF&Onf1R}gs3cZM2i9=~{5wgN2iK{E8 zF@c^CU?UC~*J8yf7F#&;B)Gu2E^P?+c-rE}Gl~GoQ-)iZfaGSm8JrqO@r>j_=WVi& zM#(Yl$>-#Fc$6HA07-u}ltcq8=ALmyOt(56aYRgs0@`-nnlJZ(9+$%Om{h)#GEa46MR(*BbmX;k6pVE6@0KL1$%4XZ1)YU#{!!>aP0|ovm9tA4GRG6m&JUaJpJXx&(6FPgZxgj;z((+1(c1 z-Coe$+0xxT(k+zhiSg{}wd#3ys;4iyXP}^GsHIC^r)OABh*%>;?G|EA3xy+#fdHZS zBcX&yh?9Rc?NIyeW5`{&I(*D4>B3jCk6y`%UJ>Mb71s35-`%Ttx>qTtSEaC5^-=F) zQ7=*c_0l!3m+yYP^7L!9nAaMGueEx5`5UX6mR1@3TxD^wszDa|up{|B&={6w$4jlB zL*o-CPE5$M6VUjN?TMuR{+y*~0vi9OEIR>>|2$cC0vb=qvVTcgb^;pzS+eW|H2&|A zWhbEVud?g}H2&nvPC#SC1T>z2#uLz(otqumm>oTs&5_B8)69uC&q+LyGa<{~+HP^{ zKM9RpayD|y(<;kDOt9Xp^13SkTx!j5JR3>@CkxM<9|yf;P}R*Uv9kj6RJk+_$kM{D zqOBz9R3g_Z(k0}bsPd{~r7&0(c3Y?N<&?E`r0_NzN{5(}Gk&mSrmA@jQ>4@HR?}4r0Zrn;>QTc60 z#fl^wNT7Ja7>k=zy%ljtf?~aeybDG2W^(VOs8;V{-rYr022d-q#jAf9w)PgPn%%3) z1@9%RR{M~^Q$Z$Ly(lKMQtf2-7ILU(S7Eg1vQ5ONd$P%7-%ngK-RvWID~%v>LD#kRty2M4x@WPu-z5$di!ZF z$~>IV^{Og%Bkukg%R0}KbzV`TI-mSHMsuD2a2<0_eV}%IkY#djfH=7jVb(?ao3pA^vN~u{vFqt z*b<;B7$wH!FNm>3^tfv*0=%|8t{xtfjfqc}Ro5;$T&D057rz+A#f@L#;?c|kEE^M_ zjmgFzT;t@gh_Ou_OpIv~FETz7V>I){H;Mw^2aWyz525k-{~KsbuKL%Z@xz9uq6UN4 z4fX`*^{3x>;(nVO&zLdes}k*hFtPwocm0(k3lrRUf*b#vA`27Tc!C>G$i{#3$if6S zp5Vs+EV3{m8~>dn3lrRUf*b$ZS(xC)6Wn-$8&7cK;*BoFx{<~DdBuiJ#hZtU$+EYM zwf+@utS*U!7rvauy+i2;7srxh8X8S3bvG<+dw_+rr8-b)8f{eUWed?b@{+Tar56RI zip%8LhVs#LgqUDeuw&T;df6H$bfRN&weG*%a9cCw#6L11jDxFC;Db$XVQULMC}`@l?n7R>S1>j!FdUJ2|`%u zS|qCQq@iy~uDoYhb)HhS_o~aiZHp#zy&0tH=S`)j?``&bi}u(9__RUr;8!ZS!iTr< zSRFK2S#xH{y2Zu>EiCsU@=*uGMi>w*+n#a*-v5k@&Ek`ctEV($!&HF?E%|U8@Yyr4 z`z*oD#i%Fpj*NEYta-rVuDe@pP$sj~3#I`tt{NO)z*ot?`#@ZIr}6%K*5EWFIgLyg zZ!%H3>K>bpFl@eWWGTCZcYk&-dh1DtJ=4H|r_e6K1O2YaiA@hEa~|%|erRj?(C*~J zeNhjo`41hNA08ZjNSjl8SiAO!W$m$(wN6pB$Mb7lnrlxD*TS-~t9G6H|CSs7V`O2t zc$6d$G*&o$Aj!93l8mc|HRcetv_BTEP^(?VYx+W0CN+PdDJ zYp{5H{xjLws_WsYuDWsA*gvha^-gE=xNL0I{ruEN*%%g&ACAh#y{miPe3Xr2Fx>+! zJtHGMfc&^@+^t{KgN_l36@HM7`$iT+Q(w*4{c6_fSEJmxXG}a6l>Y(Q_|fatqSqw( zH#%$HtUvUcf4iaS>L^1Fyxy2P=eGg!^y$<85g`Agc>K*u@4r^7C@wDktxE3%KxPF6 zhlGZOvm+v-qGLF*ap&U`5|ffsQsMESOP9G5_iiNzl_>cdgXjc0AZbvtlri5GM3V| zREx3+fII<^|LZEf6XNlNcswBsNXwY84Z;iiujqM6Dw8 z;9w?A=K5jH>qpG5A3Jf~DdPI^-0Lol*G~;zr^{ryYG%2cXL+8;@`}jv$rWWW8ngTd zvzRj3ftuMt=Gh@9vcn>>|Gxrc23C#&>Sz_~bZ(5MfE$NyJ|CmIk@Wd(3B?9Y#W6y( zgyOB2ykan(9P0#wS-q`{TTeN+gm0H9(Gg0AZz~BMrfpQTR=xe0blZ#Xki7ufId2b> z`6GgBa7eu55$6_y2)b&4E$Geq#zc*o*tH32q`M`jnx>e+k%mdzDwe4eP+04yr2%VaAQ1B>PdR>OZ5K%OH$CKA_IWYvy~ z#B3PP$MJ9CC;a=nVT#z{ zdr#rXHXI6b=`;E(Z23fg-wl6=Pt4f-7^c7Lv;`mNFTDw-zmZ?iU(v7lclpQkgk|Rz z(~Dn5i&+m~k(ey;EcjQE_?g*nRC=ie&mCKy|IU=z!4D#Fo9Fk5#~itK-_iQSZ`3Lz z+FQ4_XM9vVk%kohBv1^&O*DtZsLjn*e<$d9#( z&jIpitpfTCAdl86)_ew#$7>ajgnUe0Q*W)ovbwV~>ey@kD@PB0%wPSZkZ$z!MyJf&))*;Qz|Cwh0bA!GR|@ zFmj?=JW(y4s1{FDi*qgw=5S?luWJ4$s>K!=yNEoLA+OLAyNkvHJ0*7$+ZH!6$)1+5}ExVM{ZxM1i8;mi%HD`N3b5uDsbR?b7oim#rgyr37WH1clB-YA00yPJnlNl(C;^ML%H(9TzZjd^-VCh&|^ zte+;SstO3712JgupewKiw=nz{c}W`D`suCR3`y0G{oSeSfY>(ZkZj4^8%1z}S2SDy z!JegWCt*)^mdM|^iEi3Xczfy`v(%?aQacZ7eh8Q{pg^rMnXP4uw=KE7hlU)2A}Y(I zZWOO#mZx^k-NG?s*eqo&FAlg_LPC@z%1Y81Ku%r-9M9G81g3Nk@Hyk#smQYFEu@Y?pIWxPtkt$=8yi;QcReymcw8?90_eY;A5dp@IcG-7CECk^=5x-}*?B=UMy#4ji|-J>KdA2mas^_x@hB_;ZmMt`=v&82RSMYVj8w z_;OdvrOwt@osZyZ@#iA(Xtj8)=DY6CMPj&G9HZDh^s7h=bKq|MCq0usio`Gn9vE3H z_Txv980NrJJ4P`wr~G@X#h;7ByWeaKc+FRAY?{_+u&puhedE>dXULkGn&UBwiRHV_ zw@uyeo(SK!_V&Ga-QV$UKsXVjm~e?FT;d6r*m-Wm>?hfL%r;|tpkZS9?$8-GU4SV| z^ExYK;Jx5JWqwXFq!O2QKy&%}{+7MBbx<{uyOVc5_ji&kwA^=?q|YM&N^7MgDnzy( z9I0#i8y9afIgAUt-=O$Ih%U6*DDXh4^}G|AX{)hUCt?)nrj$z6lXrE0nntQ*t7C%| zbW-+C zcw+gEjxa}){3aN(?+j0ijm6hYw7f1_ohwT0R%JFN+ib=LwD@wb7fD7P^4`Rj91@?y zLyq8#6Z1|vi95G7n0tAwD|+#M8z<+=94vKqzM&A*hSzeAq{J$&ntS{tSBF+s zwf2R;10>UtVZ)uQ^YWGXNQVu0?RMj{>yeZ1Y>(dLZt%nG;?|t~Pq!#eM2sgQ#!-AU zo-na|H?e#-v3xhNeD~+L#EmmH;_`|X=8Z;nU*|e(HHp5OSK)9rl&dE1m1i?fcgG^T zcg<(rcOXs{<_kLVSTrP#x1b?6|DNWGRosHkM%i6L)E3$rp?O}V{eIHnf;O31&69Ui zUgt}|nO#9)p&b-?q_E#%Rn>&l-J)s$bP2RWWZ2`O~Hn(==&1}~E z(A<^eEd}W-z%=0PS!kWsZHw3&Xj+LAWAiSq!4x5SCV2@Guzhk9`q{9=>{SuGc2=Wv z(-j9ER46iabN3~edNO9h8+2ijW#N&s)2EFlcwb3^?Y*~u>XZo^at&^pwGp_T(o}wF zsGKWX0e2_hh%xLBXGK=P3^}izwMtAdyBT82bb0@>;(TGg!<)$sCNTV!>6UUg?vb@xy;T%VX=$iE#i z{uV=)u^!#tviNv&IkLtT7J|Cw|*Tme2bTqqz6UM&|rpjH2fc#V8V74*vWpMnN2nQIvh5BY&`w z~z$Fc$E9L({Y`7U|lr}7q_&X>&3&vz?W zcBXKqTcMOYPxfrlT%V#f-ZvL8igo;oRd3z8^z={lqJZHQjt$!d@U1h}JsR)P&%`3I+;%1skb8a6ls zYz9%eb*?0Uy+LqpkV=$t)-d%#ljKNVu>hV%Q9z)0im^r;6~OqzD)fX9PZyQMScDca zif~kix?9|Wt$v0O!B*zQYg69-k^DkR36ItnP(B&)bMQaFn@zQfP1aqFVK6l!TUeEAR+WwxYUu z5pDr$qn7m<1bu2RJ;$zzyaZTdA9<&>O!CR9d^eM-{#v&nm!-#8j~14qwk>o0f+4dX zFX{wUr`dP=x*RA5ZFMf`#)@xmKK#%pX9}s;&AlpHYVLGb4Q4-l#p|~IvLDN4+bZ;K zczHL;|Ac6+E~5W+x38gp27wy!%JF`P{r1&EMZwOgQoJ7@U3zEdwOO|Sz>`iC2(Gs; z*mKIAY4QehZp{y|ho+@0HJ*F$EU{(5lQTC@L`d-yueJo2(e1d>4@vgWH1-W=8zgVI zl9%vS{gUD0%9aTa`7h}qFI^`Ydz5UdLuH81+oo-lRl3|vAjZ;U$_Cpd4gM4q>WXLo z84TIMvvTJH!*6)V>{tqtUuAaf;lm@nJ8(seh4B*n?UT|~PZNfo%^ULZGQoB;DwfG0 zqwS2t%`$MiDSH&pmL}+Je5jC{PAY48OH|grkfU!y*J)PXu#T3(3V?c?lElhi-@Qu_ zYfdqkWH#?ief5RDgqISp7dEb=FUu&Xvglu)-KdrFJ_h}8rpO}p&k7pbad`K=P!ze; zhh=Apu}c?KR#u)oWAN(L0{s{JN@i^SFJ{PWkp9d3#4yH=EyjyEYp1YfT`a4-B;=2m zY*~3VbMBJt@>|$io~KvVf6owJqxhlIsFo>L-iwyCSUj5EGdFEsbVrKKBy`crV@c=K z501=@G@Qh_Gn1y9>++=q49L& z7(hneRTKGtsu|ZOiy86#qWK6slhLXc5KF+|`JfvRfTtl;ct|@sS_MFDAR}aWNEJT9 zjf9z`4w(Z;0v}w2Lwn)S%gBCf_~@-1|2Z7^^=MZjdJz{?ARy)X13-1A3<0S^M7eX3 zDui=Q5b}*0E1HSI)0wU~1fCAc;1HS=44KQ)@&=o-KsOvltN)yiauA6E?clOxxU6kV z)K+hBE&*Xl!EEBP=Hh~-=xCC1(ANE|jmnTMmsJR1G_iim>qEi-WOtZMRVR+;50U~dW!^2`p>Ns08}hU59?qNGCYC}9tk zp{?QQ2YQe#5Kf^4q^Uy@{j6%!7zQr5#hd-JRLF87RF@s9SBAtB5PD<|<~YNXhMGeM zrT7tr%8+L`(j5qI_XdacpdfXW0VUR*gTM;{-buxI5#i4~>Tnt3NLj31c8Gs?P**Y{ zfP@h$vAUe1S2~NLO}H#;T+nPE2C_XiUpeNSa@=||NLU)^CWx&pirZ;;GcI6(ne-RP;P@tTq+1l*>}# za*k(5$pA=Ob_`DnLBs_gpkgMAAr=w*gTe#le86plM0X-eQ!mDi%d+AmQrYOU;Y?rs zpeD6s|1yN8GQu2-aATsJ*ny^|&< ztf?u~obZOM&|AkLTUwes4g;k^~qr9B8w2CZBZ00op~DwRX!ZbC9kuSFEUY zU|jxmrsopWvXob?kF^!Vo5zL;}tBM|Egy*p7@ zE>q~-rWo<_npH&aE`uC^QHbDYy*{Tti)^JNI0YF4C^}`DXa3|NsfvYw+|Ug;n`_al za?PGNEyLjWjzY1#8%bC4?e;9lO^3q_Aj(kmF`Zjk6m=(awT0{|Ze~?c=E~_{08-E4 zf|r)X!m!-`RnYjDO9>n_KAuZ(=hZc*y#QtB01>XTQ>Xe#v|DrL%+ z!R~R8MOlbT8O)H`d1aAJWzj=r9NF?Xt@3z_@5F%!khmqi6C8Jic1KgU>{mV~7|B%F&RLWbiA;7*c(>irn~xV=QR=qE|fT z7-IqY2gjIy-G1CL<~4qDjGz6EV@zo@yZC5y{Eq+lbHy0*Vf+s8x+l0bW*JY<6Fq^* zrz74xkp8F{x4@b)?l;EoxUHj4ZoF@0; z@ah*wtX>>D^};FoyNB>Lx5;n+(faw1?1Gn(Eia=-Ucxm&Yt} zP8_a30|0aTQP^F8-FHLc?-7l^35oOa@?cK<_k+ZLE+K|NGYpCMPC(-8mzr~(Ubi<_ zT)+INkdB-z_h&%jIll#og-d?}66gQLAaTeagT!`!J|sT!w}HeKe-IMy{X>vANq6c`<;;3{39gZ`vDS%j6>q`2}t}JHAC^`BB_XlOzXuYV{}v<; z`R{?mW|R2OEn7E82Aw4?2Ih--&o=L}a%L}K=4l#iEKYIT`?rO}A%8naoI3%Dk9%>B zGwel{Gfn=Aka!OFII){ar|G`o|#g-hUMm%l$_n@!s!*#Ad$%iOv4)kl5_64T-ULKo}Bhc8q3nSNmSe zavH5IL{co;7NoB;PanP^nbP|6t1q=RnfWg85H`g}PUdGpWAuyeJeY(IhE{>EweXNnIbN%R`kFmpOBX{jm(O4vRU6x0q z&siAUCLK9piw%&;HrHISk)Q2t16kXppJQgTeKX}TM(cjY#+&CHJCSqyjtYoN_t(ry zTj;w#0un4&_Xs2TML-wKb1ykv7boTw4=!^!fgUg9wq1t`C+DT9Wy5H;YSR20KkeP2 z02QswNs=vw zqs8kj{%O48 zr25T#@TTUsMc~%Pd*fN+z)vDDxAAvY=@@hF)r@6{doTW)C7#1={*)ztl3zbQN(V>? z;4JZq;g*l1bd9hFeD+%&Fg!{}fUCs((U!!Hjx7csJYW%GJ_TV+5W9c#zAg8u=10Vf zCbq7(YTbCMRX4g-zo6ByrFHX2D_QQD@#<&Wt)A^X^~@ytH>1TS<4xk;KLCl3j>aU~ zm~t-zSHBFhdKq%+Wmxpf50Lm$TkET~n9pJrA0aW^l4!rWx+C4HBlA>8)>yPSMzKBb z3rO6q|D>a&ptG{&10>EFSzNyOv(-BZ;}_#i;wRsTReZgA2gfRgzF55*ix!{m;VafQ zO{+E7RvTDSd-Q_~ybEyn4$Ql5-8$F>-mzncg@uKUjg6zD<9L(U&CQL$V0^Pl?39E4 zyY#UAT5U&uUB~r~j*f=Tyq4~pE!|~;p4*RmN}s-}eBOJv?e&8D=YzHUg`wq{ZldTwGS@vIOb*I$6R0lEjie0r z>qDPDMR22B@9QXxPh zBj5}xOANjeQ%f}Gg1j&4O`S7EHgSMGX6)*OPKt9F0c~j`+E`>ehc%a{xQtE=RfEI z{}aqh{YT8}{r7NzQ~!3D*L=$NxWMP|e+L(M4}Im_Or5oxG#WQ3&dpkRcF*=>#`JZ+ z_9T-3)0mfu_S+!7gyQAPL@(jVWF}1w+Xwzp%xeQov0-BJ)fiVPXc-d5je;&p7PH}x z>A*Bhv6^2RX-o$uA@D-q6vCJe3}8O%N#M|KLa-6mfrVo_Fa`bGBp{BA!3)3&r7;~? zfTVKJDmc^z5<*5WrUTP3Sarw}K+40Umw@&npqJ77)&Lk|p8qK66{1~9A9P@X6ccIs znGQ@k*8(BmnXsZcC_Lvg9higJ%4erM4s7EcN`*-;)8AF-uaSb$);mWtWe0Kmw&;a6seRETrVufi>cR2q zP)`=Pg9Q1fM7U&!A5agvs1B*6uzKrINA$uIVHcPXu1|tGv)JqOSgu00UUraIebD-p zNOvYC${UpL5C1s@}us)3g`7k_~m`wr@&jj=PBV6fW>O^S&XWMj!oG~L9V7@o~U<4m|$vB3Mb7h9$ z1tE^!K|Qb(OcP;xbwBhZ%=j!@L5v43Sb-3iuMS0(MJndRyAc8wt3wK@@j4n%pdLHK zJ7_nbwTa8fu8(f*W`jgw;s7>sE`Z$TokSIIgOItFt456)zuO9e0513Xf9 zVQibuAj73&=9#f=I&JkV%QIu!biZ84ntlg_x9M18tU1I1>z|t)>6^)vku(LAolCH{ zJ*-0c*~t+(7K7P1$sEt6IpGA1L?$-XJU6ofOpI(-mNApD<)(ev3BDZo-hqJ*Abd(_tRWg{7eIZxgZqF#W zPA?cG*9v*TKxS)Qu;Jvi&c2M5sYyErV^9O>eVW&8Nogk83@Q%u*o_Dw~wkqmF(0%piOV`OrMocQ&ApEy~^^k2x_$FE2G_2g{QyOCm*;g1$UF9Zh9|)moT` zGKp5Si|ka49ibX)ftJGgQAsx~Czr=M{xE(5kZPE_+q|$zD##LtkfNl?_6>#?GA#5Da+Xz5hDh8#3l&p_X7o<`N0b5=o>%6MxsRW@8@S)}) zYZd=GFSKmeo&1-`$4#tO$Lc+)=|S1ZH%b9gLX<0jX~17D%i{GhaF+~#jkpJ+#}1g> z6vJYmD+AFS)b4bv&mf@Qpn8!oXf4lgD*zk*7)wsVX^q=!`~C64hd&u{pX>Rr5sT9p zjjX)hKDr;m1co6kB54T2bQT`+DA$2F1v4Z z04I?Lao(TVH3hgJi@n$s^ae`C3m;I)sgya1uH0M9dKR^rwWTbOzyN28)e7oR%lBh8 z;iA?8{?}d>0?wGVw5GKHGHJNBsXxXD$6#J8(#~!aINj1V3QuSVYW2bxlFsd;XsUw5yfe?Moa|LHJV(uU!mZO7~lqU7?-g zzZOW_KM=Ey!|d~_U7ix8tsGvCtz97d_@KI=m@1gj#VTyOS8o*bW*_E$J70|nx{0M} z5?fZXF^7(~v>D-|qTmA>(dY?HawH^5!}d0_R0&XrGUJ>Iv{UH&%M@ly7+90$hZjD| z$Orc^G3%IMVHWyL>60z{;s1huy9v*zB#eGwQFrQ)Ny&O=j6?HXB}+`)al}=9485T$<{)Nvd3&x8V}E&4q)N?C z7hgt3r;Be6N^g0VJO@=H%kZ_VX~HrCPliyr$fkYh!%~Uc3Bfh0WxE{Oh!V;>@4~x` zBH3L#M~4}ECR>Hocx;v^f z-pXCCTaAl@bKgZDRvgW5{c@H}6b?hzE3%~M40}%N^hfE)8cF!T>ZzQV)afkidel4F zUj5ux2G+eQyK|K;&sZqxjcV-G179zHl(jTqhJ$&&K?l;UGu@0_W+8L)U3L1M#yZMm zq#01NUnW1a*@}<(;qL2Hm3I&XcpOB@Gq8T?J2zR zyqR@L?7Z=sw;Q&;bz$UPh5$Oy&E?T~Y1y|S4h>Evr-O~1cvsD%qSdZ9F!kEx5TZ!LJWyetop&32Z zSv3!KbNmL|Ig^I{3x2#6*k5|g@ZnGSXfLUx6DhA4Yi1Z3uzyGib0x&JFe+szRi$|` zGQvuGf?u2AfZ;-hHG#1ofac3kcjEDyRJAo!cVo|hbqu*hc2vuqKLmJgV(ewCR@hIq zjq_j1uw@uZ1@ET%$JyIZZ9I+CJcB~Q*c+&8&r(VFB%|2#3HV46b+66xLL$|EK3*!F zUFXSYY-(F!EIY zWX`j>>t?Q4jXR1xXYI?LTEO0ZFK#2|hnXYG6Z&1Pkvt_gG4#!u&kc1|s9H+f?kO}* z+PK1vG`|mZbJFtJH&Mj|9V7Ny>Sh<*nYmM>!evvepKhNngIzJ-@&TgGHE9b+UM)rS znCJ8J%gl9FKPkz)*m#}x>yBLDHeZmiZGG0}Au6}YAZtqNmn7JBJtAb7Ifh$uEV`nB~ z&ZOKpD@)EI+A)8w58n60TPZR5iI1w?^(g7nkKbD^-*3${!5Mj-U2CJh)EYXltcv0_ zX{jT#$_-yB^jTqx*TF5OBwO3A9I_Tv?5RuVG4|sYBvLSYy~#UwkjiHu$Z`ZJ>y^6( zrX%JhXa@!P;dV?Gi zyUl+v&MX zJ{dbN{tPB&ZomO4V$`$zOii~F)^vkzt|`k%V- z{xM@!ci&Uys?~k1LAwt2Kl^3FrT%#<>URG*K2YHge5RH`lg{1-F2ky<0@GC7= z)HrucqB2U%y|-A!?3K@%bx*oi(ow3%U-?qBcFovb>uT!$im}ho#^*M3eeD5(4V7_T z_C;UhlB<^(M=H5WBmLd^h($H zOJMBc;}usamk`qrBDVS2trJx3>u;_(vl){TpRo8B zZ=GGIs*zV(6NqO`Npnwb#tbSXE)7*)^K4%=^1`0PP7W zPa*rIuJV}~C@;Gs)x?83Q%48kYy0-X)_kdzuCyIo_KqafHJ@EI{+=YU~O`nRGj1g-z`$ccHrp7bVjwELR^R+tKD9V=+LqqDZ4@9iG2eKH-<`o3_dJ8*g~AV93zDqf_0 z87QU*Ftg7gx)bPbKWMFgrhh$qMm1Mr9-$f9*LPEc0V4MK((&}ZVq)H9X$Bn3kre=9 zUc6%uC>Apsv*@aFOMqE4Bn`hd6Ho(Cb}mm9ZeB)ch^#JxvP5vQ2^}E;EOMAiu%2h$ ze_}@6kEfFw8yB7kcV42#3kh`qrl0PwEOZlwU5l_UU*QHB8kREDde6D<_O4uDg$W+m$X5d_D!T7rA{&%y zoUJy>wH^-0d{%D6yI_97lie8kB(Cdvb?CuR8Ak4lw=GwsgW<(o?- z&oIWD)f}c1Uf}1`TkT77g68ljmvb!_OG5Ozqg5SC3x~T8XRj?jU+~P*9O$9POn%g# z+rg8OTkLlE(Br8!eJ5l|0CS7h<9Vk$ZkvzH+4$(4<4jtA^|g7EaWs0{rY!?^BSzjN zp5^hhYX%p(iq;zS1oPy`eQj z&oxE;{D6WtNn3{7%|!!sH3jdk*9>=`5Df~Z7WS2F8R?A>4Yh449H^}sd6z3176ud! zz1SieXcUd~)fA4ruMv$5!r3l79ic`?nb0v#bSR82mPHq@qf7MBariUS)XvN>IWx=Y z3_k3Pbk-T!x-)WpX9###1vS_CCa#K3u1aC9Dp{_ob*_v1T#0zMrD|@=P25&Gxv7P@ zX=J%+)w!+ib0gv1b=2I~o49Xua@P%W*Uxe{taIPo=T63Z7^`_~H}Tl%@I@Wm}?DM4I&mLAgd&K1IF{iUmVP}tLopq@@ zd#dj&9q;9;=H+hU<>}<*73SrW<;AG;^6&Fv;=Kdayn{@G>%%CB$Of(sHTo~+^^fR(4SwfGA z(q*CygeYw;>RB6V&ZjgXP6W=N1CwThSUxa?4isU~Gq}Jc+)qHb#vH!i3m$R{(I38k zs>4Ap*4*NAN2&lyr!~!xxk%%6^nfdpJrMqFU zZy3D2*gb;x6GkaCkebcCi&QW#YQ4QHqP4!VN{ZRs`=qKvY zt)|h>oukbk+THPv?yQgQ?vEBq#q_GjyfKY==N!`)9+Mkv)sYf2(jNmza}XLFlo=dw z;6TT%p1(beZQw`@aB$MG(==jdn8gab58d4#E1eT7+Yl=^5KEAbt1ojNR*q9V9;d{P zQ^|=_)sKlx!YpWrTdHw>x!L)Z$Iq({#8GKTQyOyhz3}H5Kop)FxE)eZkDj~c!CK#!7L}iq9MU*Ab}#CxJM(=RwE%U;e6!FnQLP`68AT( z1$c=E%Gc7Qla82Kez=fo^=Fz&7cN{#Pfs7O7XH4bG?px!Kc*=y{E#gCQd0^mU-K!V@#IHSiB-_-^R=n;B>bbP6c3w9{O>iD9)4~r!4o)dCQK!wO?jl= zwZy6jwV&IY%C05V#E{&?=IEy<-#M?#Qn4&gPq~*wKCd68pOO0Dg2^>%b9u&v+RK!i zK64B*)9SD7yO(NNG4`WZ^y{zQKj}g$Y%0-4tA(XD#1Mm`eY2`R4CnDoOwTh19>OsT2vEsAH@T^GLD<(ht>4 zEsOEoYMS_q+74 z>hb??Or;m7FRFzTrqZ8WE&M@9MB+TxIpLOY>F3VxzRdI*xZxG|EmKMM4_-+9+*FeO z(o|CZXevol;Mm~rm`Z|=rVVRTsU*h-RlhWq z<}eY;W2TZS2b@blj+;u-1f())G+e0sm8k@bh6|+ugyttxDfw$tN&2IyM8F}&O(g;? z*o+XgUOiYX1*~wwX!rZt@L6~Fu?~7;i9kR{oxdwF2rmq>pxLOEnO6s9aPT>bqLUqeR z4PN@;0gMeXVw*DCP3SL8M!WOFu4jjamxUaw524ao8p;7i#86o4DaT^9Q&8?a{}lD0 z<;qZ6c#wf}n2}yki5|2cP8JH!MJb0;X<_%2p>5gdyL!;RY`;D)uqZoxZFXcY>?mbN zxdI;?r3WwBqmB{|p-<$X#vLU-sGSm#rwpA9N4haF+TPK4Zj3uAd_4~}Qy)^MgMfMr z!3R>HgVpMx)h3}bDo{s#hyyGrg~5W-n4e@4t!65U9u<^0qk__0eryh$OAL>kYxY4< znxh}Dr2)mNha2>xwVhEF`y$%2qtv7jR9x`I6pXPC_=67;PeSb3AM1`oXxD=t^u+dk z@duBOwhPtzqZCN7cBz=9e3mkYbHY1H8bI!&#^hrWL}KtysTi?#P?h8#lpQ$72i#6h z^dzCQrD9wOv3S@dYELw$hfVHJ-uTk*>_FH;*h}LF%Ml@cUJ~pmX`78YN(c6b+N5Ik z3y`~{5a&`caa1VE2Q()^#`=j7?THcLY!~mevouV(K4eSbcr!6oY>YJ_GGPBjZ&*+w zAUsH6>&?>UvPI{UOvT3qr6wsvlh~z$1IX_EV}epvgfuR#vpsPW{o)N+P{NH0N4#0CiFi*uDd=iY1Wz1AuF zI@kHjkNnA#>*c+lagPDftf>`=24e}3vs9=Dwib~Ivj#xO8QE%Wm1=0Hgny|u9eg75 z@w_ZZ4GF!3e%#>)vqwWu6YJElU{xYm6A3ZFLNC_DMAM;~gk-(N1_NX8&x{yVN`uqe z26-w(DZ0LX4tg35*0OChmj$U(AysxvCucDsvng31#gZI*H?v6)3DGWZEG~g+QsN%S z*GnQ9keiFik>yRl{b(l2mxOpnc~T+w}uLx+6&lG z+TSu4QlmR1YryA8ji3CPe~JGtL;6<-q@DA(g$I^Aa%*%v?x)*Di_fqzV4M zkovA+6Aul@__dJgXbN&}E5-a;NOgF!L2EwI-h*L>93K6(kV=l`Op@z06z;|#+r;kn znl1IJ`*YTB_MWZi{Onb5E}+lQhtn0?XtUHeW}kOzv(Lq$=QXX%HK0GY97y+0^jPW_ zbKrcx(eEqVX{|MItG)9TAoF(nfSz}0Z5NPfEew43Yxg+B2N94pIAPCuO1N7Pn{J0q z6N>>`lcCq<`|??Z{(}%B59S55y&`6dwL{gJAP%y7E-kHaq3>ncP}Uy8g@(_>3^!}_ zu3?JN>?D`6VX>I1FaCWnVWCW6>|fzA$Z$kJTQnO*-5+8{=_8wh>`7qq2*@6rPB3MR zJ&Wm7BMf=A{zAvDl>$$ZexYMiK5eTQm>n9rAlw}^pXiBwtht#jNP*NDLl@vo)N06t zl&@I>Jxy;_po8r(u(Qbe{k@5WZOGI*)McTsQbA~d{o`7FzjSy^cw|Z=m<1!kmNdz6KATb@a-Gdd8^NG%X zVXnwPn$3g_AY;fD=88@9AeVk=n>OE{KD+?^w9u!lFya>e=WT4OsU*Ub!$yP3y|yD9 ztxDNVH#<5%Yvd&`VKo5g7&d*W70RX<^TwXhwm^beV#kl}fm4}~6`Aa848mx@Sj#eX zkTcrqGY=f6k!4RK_p{T}=e+>rFEagtm}k}3MwxJ1jrO15Hi9<7H+#xc5IQCk`_2Po zjfGvLCaaPgw1*(e0T6pC(2fAZx=q!&F#Wr0+KQ!0@U&O$=^|HxZf56?^6cHVrvVE8$kV^ z(+94u1mS1g$_t&)m7-T+U)!H92*!G07Fr!R>}a!^3h8W}!)dZ$q~HpZ;G&_P*>RSM zqcgy9!0XG2H#t{5-VIb;8+ES>FljoSLRoDJXp_m#)}4oH2~>#EvY7OnDDtr=;GsCF zK&B=~O?E!15|*N!RS-K5yua`gmC%|FE-iy?Z71mvW9{GN>IrNlRxO*C7OqCb+7RZ8 zI_AC5GgcjiPb*-W*!aEe-=VkWs-d~3GxRCV_8jhU+8J4}%1-$viUA>iNPMzAOD-eR z(((;j-fPdl4~pxrT@oBHV~*oL{(|G44JA<4Gw7@4rE$(3Q(yds&W(Z22tbxl(1e9t z_9N*Eo|_^^Rt|N7orNH49v_t6r(Gt%x=WyY7Rjy4Nng@H4`mw;eW2gcdB?mg+kc)k zSOP?vlw$x7&x}=iAS>QRwmrStYsX5Piz|N5B*D^PV#Fj5G*nvl!`;@xFGkS$ooJok z$~`g`uC$kj{&c3QsH74F>g*Qzlr~adOMw zJn72N-jdFv7gfNOBk8(qtM2p)CHbNN!BS27Hiq@f@sk{JBE6g2!;XtUmgNq;Mc_(S z5ep|A^@r_JbFY6=njzUTuTWCiO2JjQ1Zlo3{!;d$_?R}-F|X61^6~bTo?fB~@zxAi zFS{Teud_jyyd;A>9fNeejtP2r8lXn)t-Uy=x+yIofeE-wn_;)VC}II z%CEul{MG`H29Pv_lcFGASkf>slRz*v;clE0cfD-$GA|glYqc1o_L|v*?VnzM$<%BQ zWkGoZ!k6NV?iZfoEjTQYMJiHzxt3%lZVXDYaW8$9;;?!4RVu!5?yw9t#CDL|PS_}d z{}kYjF4BBU931GrRX3LA_qz1WBb}LzH#rwSwHRyPKYNXxAwCCa3Ko2leZxQi1d4)t z2U&Y<6f)WMjiNUEUE>2@)}9^(!IHZVV?zAUBo+g5SwtDi!I-x=bZKLwviY^3Td+si z?kN8?@$MoEg*}?QxPp&vFahI{5F9+jYHh?DE0yx`zF7_rV8 zDtzkgS5=b}=Qb~hD2;iC@fHgeND4Y|LsSo}-F@m)JX#`kFZt-JqRmP11NvtTxy98G zb9SQJpb3eug1li$o}?O*v8Sqgw**n(#st;N0I1KfT@btuQ)WGRWz-XOAh(~3Q^?GG z%q{iGEgAkpca8d^3GGIx%jh~iuaiLvQNa=mWik54q;`(+7@kiIw04#xE=L=spXR;A>YAg-B$ZC* z8F8Oq|9pzuBxEJD5#>RFpj>fc5>3lN(Yo-n?|4ty`}`Jgwe@Y|$WivwsKMzQ1Rfohg6}~wK@p}^XobP(&_)28Sq&P}CG>MU*x*yBs^DDEokn`*m)Pig znseRUo!GD|N%DcOHi?3x*vEB&a>jDtN4avVj0EM1LU55u|3OTuim+C$67yYux`B9K#%A%Gwu{`7D>S8qu0~ABsS*fCD;u(xXPT_1tXlq zLkv~&8;WkK(a(4{pAI>Fg9ttSE{G3#kXtooj71Cs6NnMx$WaP&LaD9l7dWd{B=jn& zvcE(jRvK8!a?LtD^aNMB3@5en*j?3Zk=y+F5ZO^Xw`kjk!(xj1isl@;gKo%8=d#ck zdemX9d4V!gn2HBlwbI#)14Gw4cO_q_ADJd--J((8DF=B)&pmRxR32T{g@$A_m zmB@vxUvR=N<5n6B`3zxa4Gz3gV3ETyyg0!G{g7Ph)`>1{y`x{ygS@bF`~&;GcV5fH zA+)F|Drr}pXD|jQ5Z4ah)-pJe;u@$nbSEYPtw?&;JSnhywmIXxym6ezbP31~er}9% zlTNwlszT_MHi<^R#$S~GxMjkQSK)ZqgXO%RTgoZp9NunLYO=F7(&%vA)7-`?-z`H| zH%2bi!m5We?@k1=F*sy)^tD){=U`i4;cuuxdv*38pu*l3x9A@F*d+DHxgp#mf_kq$ zW(zJ@pnJlUFXWspGT~C?2ksl(y$72wM`Wb0*r;u>V0SmT6%QCX+F$pw9k#b*LmlxMMHEegwR8i5n0;M`-xI(l{s^3sz)rXqH z#2{L#2GS3_%nz{6APutLpmj`n3r!4=uhayjRE=fCYkDNB)?n7La&R=Z`?92Bcz=qZ zc1|)h9!1t`L#x-4EFu-$Ex~q}g}BHhp0bIu+=HWCG3w7>viAdU>d*ATW;{oZOw5&^ z7M*lrFBv_55!A&cetrJu?m^S^j@QWTG^NqA~0+QYmaIrum2$_WhL$1CsG+ zEO7V2s$Mj^)KB&Zq&nj%wuiNLO?QtXcmV{KPuJku6U~EJRT+9snH=W8S4%KB9;e+q zfIq;wEwY9=4PUd1aV1kQ=heU%QH`+Xl9%N7G3x z+J1F0ou}5=Z=DZr?FYb4lf_hp`L2q=uXyMmaJL;I`En$8v0qj>m>8zT(d+9#jcic! zX(RUY9=aCNbb}rlQ`5kE)74>hd=M;40&*7s(v*aL%pKS|Y*%HQAh`|2xRLm8Dds;f zh^u6$Jk}GWypEnzsXu)gew3~M$N3fC{+^(kEPwXU>9M7rY#FO)=Py0akD3`Qp1Hvu z@af_CxnVzruMOc0$ya0i`o|-G3vq!V3`j5)R0~p-m#yOVN}$A=DXU*^*RguSf}=+O zyYO$(T zo5ew|D^unqq{nY$`zgM{#qVAzQ3>NdzQ2ixQhV0KnHX-g{}%rdk?e)PQDs`@BYC22 zL5Y`1!DS!tj{gxeDrMYK?FjC>-?scc4$<}_+tB?fX!Ymvja#4h=@CsLD+jr<=8eUu zF2y@MRONA45!a3LpH89Qe$bZ+%IKvO#&!F7oN z=4Y`XRDH$|Tpul|LwWnZtJRS&pJoqhC^2}^ts|!?_cGrsq>@D_- zRLVkMR}TW+83<~62t&QO->U~ve<^7v`HNIaz`>-{gfFcH&z|*Er`|a-3B=$a))+BQ z+5=@hP^fN*l}vE>GjUmD@Ohh%^CZa0n!8eH5X{$PM;Fx92tI{`+LIVTiFByhD{)bZ zqrF=Aw}XTu$3kvI_*^92f{WVj_AG|&Xor= z21Ht(hMmEJkFWDEN2BLFBFQp$gN;NL00FrXK8c^97txL?c#sqdq)-DkuYo@Bi)x&^ zWjc!%M29!2xw`@!nafH}1IY821J7fj=kZW`0E}-YdOXc9A0=G49n=2|`ueipbXshE zYIIL7FpeDSd<*uW7#IeRd!6P-Q4@aZ7xz=u@j`zbz2x>1TzDB3|1&hC&N+T}`{qXT ziK6Cs**^F<3kf3YzDJFfwjw2;zvaUeR@mAGZ1d4RljOfL!cLq{K$V`&GmMu@Opxuf zKJq$2d`N%j`!TTkLmjt-5Im?M;^C2Wd7&M?rLBkUA+YNHW75Wn8tL)|-;UKjPSjq5 zVrd{Y7O)6{xeY4s5XLKlxnpds>4x->qQ{@HP4@RqJgy5K^GL1>R%0JX#-~f0+VYCh zlALXiFI&S;-%0U79JblvQKgcu7%M$fg*P5c^@|j{?aV7mjJ;#5#G(rK+fEH>5tiHF zxn`T@1`Y|sCP(e0>FeUKr(H?LN=u((lhf1nuce7fr)HGOo1?+UYQR0)=|wF(YBzY$ zB-3I6C4t$Dno^$5BDgv&y*6Don2^~j%UU`FZ?0!#&Yl(00%rA=A|Bs?&z;X|_JtYH z?1y)<>E<{1kvlofxDBz(;-b`A)`)C;aSOj>AJJ7g}7%mkdAxIFM{0uZ+zgcSeG4gnR(KfVlhc88Rr|P7FpM1Ic+LEMS5gU<*K;AmqR`0EwFq zEosI1USe{-WdO{AruNt0Z8b4gbC1ixTKU`K4B6oj4OqE0ooQSN5o3_y(C?b;A~ce zpIrHxNh}wogo|D^;w*m91dv7pEU*HmO{$wp3Ky8Y882a9!74#P1P2lT#TN0G6@%HL z&eT@N?lNkcC#&F50Co%jBm@NE0ktOLw&+Tsy0Qa5B5X1F3vz(Bc6EBdvJ)oahD?Ed z4FFUFc%cM@-~k@3VjxOgJgS-}llAlFV_mU|1wjA=4S*7ANcd{1Zm|JdgrR)xwqSz} zcb#q~>v?a4#SdX^?}qEB$F`ZQ-mHyswU2Gv>gL1{2m;yOyU`%6%vH0BpvkI)X|&me zWd%MXmvm{8KmA|cejumm`hwvp$5n8Kw_+qecX2T85mTYXZt32X@ zM!AP(d0Kf;WmMCWU~48C#x7Jp&Wa$nmY+bj0C25Ei-AYVE8R8PvZxKN@*?qgMx_G~ zk&(&T6x}9X(zXy)&|8kcq3djal-n}bPIuw4KZGS8wu{TaWJXFrl=^2v2#@F{Af@9l zNMv!bL%g|!r>*(5H=w1U;b5pzKH5)-hKm!|Kro4+bKsH1I zSqT6ZGJqS`=;hs=NP5B>Ll^>TnAmT8bPsc>Qgu&w(6l%I0`OqF z7>ows#RK`VKu$p0#5KUCe2-y3uW?0h(rKVXOcMmq$|l^e>cePu8ojMVYCf&SClK(& ze5qe8qd#Jn;%k;K=;T7rCs6r;dohC- zXL~_dpe-8q*VIhbkS!Ld90d?mfZ10(Nl+NFJY6J>hp>|Yf6bNe4YL!#zNU3-S5TF5jTS#t@kCpwr98UFg#@)lH@c3Dj`{fT z5c<53V}6=r6T1D!D#Ub0#-1y4*fF7TxJjk22~e8e_9ByYj~&qO z)6G3r!fQS$cc}`xFFb8RS(N8Hv^6Q_IA-85b>i-n%$TWT+teO&7xkT(AkR~+MJm;@ z+aOB>N}4w5m^SN}cF1@-U|pl7D00^a;;328#xZ3uHidgWV;SD4#R0I2Ei{atVdrRD zU77K^;xi2#pHvX3ME6Yuve~|Gu;&0kFumR#vy88;(m|8(j8Om*{8S$BRCAiP`y`-a z&V?TcL_V8ZqQdC?F^=;oH^GO*$I%tAGgsyxiMVHEH*oFG7d*e1D^j2DxKQ?9p9Kx5 zy0?(GzYy5AQ15vBX2&3a3~1|!rC!ISUJjtd*y3=<(&+Q0@%<&L z$nz=f=hKeQpWS;tm;HR9%dhV(uV*iBb}YYpzWiZ-nJ)5T zOZ&wa#~0u3z1YrvvD@)t@A-?L`!4{ZG>{Gr;zWbpr@g zrxpJDE2u{+f}JbE%PXQkSJ0xX;ySC6PODP)S7jcp%5|u$(}VP2kCXNKXe$qOdi2HKIAf1`Xn*SD{`AHDXAk!0a`qRV>@U68UuNvn z4*gs``E%{!&({xruIK#ReDd?%i=S_~L?VX=o3$BVE;7D7U~K0wcAqfzUNC-g!2y35 z+Wt8YPU7?O0w6Q<@(T)!ic3n%$}1|Xs%vWN9@jTCre%|x+S)rhpHRBGdwTo&2L_|~ z09-@k6V%D6r_(dfX6Hz3t@D51F0EzF$$kB|?b7n~jnBVtm+tNNe)YQQ)}mWO?h1zU z$~g?y&E^``Q;cG_=7b78DJAR7_ZA| zyOv;F^lwIs{vDg;p?|4t@AEG*;@yA}a++_MWL~!k8!%)ddI8j zw>*=gaf92pl_xb>n0uGa9Uck*4l>(XMi_*Z(n1H308?s*)Q*Y8jQ{XRRM=ZCp z{%W&4_1~85PyM}Y?>YDHlW@B)`|da)SvyqS8Zi zgS3+(CJ`;hUfkBaSZs#?H^A_%Eg#_W{k`Kgde>MV1yTO7&C=w*+AIP8qRkTZZ`dp| z|C!CQ=-;zhI{z=)EaxaSQu^Fu*sqS!W@ij{hZkV~ZM4YqzuGKA|5cl%bFgETwK_J) zCt8}P7`Rrg4edFresAO?;}y#2>MesDhC&4P`nyh?waNpQz`|?CZErW2NieK0FQz9T zvLu%3q}#O2_$)SlOU*bYMono~3BENJ?eW;F@-UD!?f*U_UiAMvo2BMIwpkARnGv`A zpKO+i908Yzt&1IE8bx(+JD0mdPKvxp9NZs$U@r|uF}4R%)dpCc!Lx>}JA>iRKj;3o zcMD1cRC-O1^9sMZ#zoxiKPklvrhkCq>lzE4a^t`_;Y$y-N)hK5zIXjbHyGUe9@)6v zSA6DPi6|M!f%cKUAoKc`KYK2A?(;Eyx4Wqq?rxt(z&+rsaF?*WfH5dOOZ*;b>WD87 z!0OpG*D#UJhc`*xuFn61%`)wO&t@t0H%1)uM@F3Z%VvpT+AN7s%-?L5LjNWs?gHR& zA^vW&{N?C_1>=6%EL|wjY&=8;|GUkSDO$mSg3-S-;z%fp0M4dEQAh@O+9s}q2}lE= z!PH-jxEa%CSp!1hez#d-VJONkMtn8`q)md^5%%&_N;;p#1M; z`(uPy9XiyK5+_Uq$!?)dg4h0Bn%&8qz+XnO+?e7o_L7HPNFR}(I^slrX|Lml=wsj zrY;+MOg$xy4x%^5y_bPAHJB!{uuG&weMG7|B~BXybH_rQY~ff!tS23c{Y8Yw1p!h} zct{Wij3R(+hEhD~jOfF^6XD0xnID(nb|;-lgquXBEH$UO?4+DZhZ&^9ba$XGgp6cc zI1|ES7VHDn;pOV^5Kox5Y>Fo~@wjZf88Ia+GQAUlP)~<2iSX==M2sw)ERd?#l4_ft z*v5pv>1oV}eZrj-&zjT$S-5&BwBH!+_&X8axRa)}llgiGKCYhS8JVn%fe*>T^XHl0 z2eV+G^DQzaln67V0M+L+s%7E+k>Fq=Y;hy;y)it3S+FMpS>)g{L?FZO5l<9cjGVl= z5#u#PMhK(?BT_#`fa3A6EqLO`v}~31Y+F*IE1n#Rk3rMPOy6ewd?v45elVI5FQ*RY zGRarafS0DjB4m@CNQs6uF)6licY2lz0^~_a3dR(mh`A?-Kyk{WFaSt>2pCE%-1NwI zYE4za!p}s4#3fCN{iXC z&wVXey`@q8#jg6>-RkX(>fP4ry}y_3+qgYwP>T)l2`pF{305J44FFI(QdJBcYDIxy z$zaxzniF)eArk69fPtf{J#f&I$kMU-EHMJu5?vc%2i3vXNoOX@vO(RDbsZisJ0#SQ zPB|Hi&|MLvb2ZgA(T>^jI7KaWPH4>D}ni*5oc!1il1O+FX#?*1Fg{AK6+W1~bEet4ErJkzivSl-$-f z+}3_`v9O!cP8I5y((IVdOzwOOcf~`dY&(_~J7U>DW+Z5_H`}s(=WGAYSNK0^ z=Z7DiW3nI(EY!M>t$RK~`)B7a4TKKgUPe?^O$-(q37MGB8X zhd8Z$&9qg60x=`izb=P{3O+tplVRHeS}tpsTxu3|Xc7TH7JnqllcDeBTUoWbenpGM zN;%KdN{_eqw8(=jOxf}mfsR<|Yu%n;jD*FJpn-LNZQ#K;(Il$mIXWE3p=!6>XBXp-xFcr1sBcx=|+7Wn(T;V%rze zKFh{rjg+A?U@jXaFQanUyhe=D^M#3(E_m2wOn1^K^gIs4rC6baPELAPgwn3+(@1^^ z&k3p_bBRnEMaJsy0ioMNn>Y|lrZ2}A9U*I4T5j%g#=tR3N2 z%pIsvWTg%^{^Wc?XynXTB+OtV4Mj*kjLxzk0!09IB7|hCp@O*968E73h}Wcfdhr*d zWW%hPJ$Rl6wn;n|s5O)nWY=!fQmQ}v?DL(ni(^w>*^FdS0g(D#(0k)VXThq#r70cR zxdex4v+Vg6Z%*!t`E>2hB6KTe9%`NqJAt18WY6ekBzsVx>UWY2GGIrs@we_mJ!?SS zMorcVaChor5CursPkf-5W`*vny*FD?RwOSE$L-Fp*^EGj7dx|yTR2h!Dw432SRA^9 zOK|>{;-I>1eX1kZBgf@jd(IWlWwR??Rp{3GsHECfP>Es^FdK%kr&isYQ;)2CeC36@ z@nYDO=b3l2u{he6z>@mT?9258cl=T!N8Xc|7#t;WEW3C@WcdyU8OjOS5}@e~4VcNn zo0oDc<>s3>mQm}Qi*LTZ9g3*-2TFN|G?T_Z-nTloW}CkD?0MEJ zR*+v=({gO3vHyxJ=PS-7Rw2xr8y0V_t|9mdZ&J`N14J8Mkq6hK(`+%*Z~PZ*>Ae>n zN>z`v`k6qL1j+Y|p;PNGrG?gZV}W~D63^44W#zIY{FCFn(%kVao&-<@f;6@>YmA3k z(xc0>nNL}mF`i`GSrMxa(a(Y{tS{j$$axlvI^;=}aoakivnv#CS1RvYS^>^yZGoW*7{iozIJ}H(aAreakYrA#|h-Dv3 z9wlcE#~T0&6|?IRM?Wp@WKVRmPF|IH`Uv67T_aV!z@8t8?}c0YvlCComyW4_v5ow4a^tg!Y164E^+t>@CuRH2o_Tiu$!Zy43rxo4 z`XTs;#?Lz601R8jg^EgW-U+ZmveacmvflsTQatFO|2iE1G{js2|3>UzU70dd`U}nn=7gJ5WXId ze;Y!s2pQ$5r~((=e*zzRh1`!`^_%m+4ml#?f`41I&4z(F5--=3y!TI@t$u0IF?wmQ z=e5)ByF-tQ8O~uC?LTUY>&_tRC#Fo;;O9I=P@!Sr5s?uxp2kt}k;r9S_gLnVFA%T{rx7X6u1FTHD=2((%NFdbl!#QMR7qF)pK4 zi6=bOLN9ot^3I2~67MX|ES z2siAByGOyzlW|y>>>4R* zTV}$iVct4gzd@aLfzXp{y>u;1t-p^2r7)w*K1U;N7O1yC^f>T8p1hMb-PzuqpWgqdD8l1YxC7TCD_5hqmLQpb_N8iFjo1YM+K}m!mv#D~%T+gyDhsoQv+YcU& zC7>pi^ zs_^p?FeD&CR4=*K5OUlBE9%IjCHV?{;UUASFjBQgY_y*A)R2Jb;7yn~VKFo??dl8V z?;jd&Ox{p%M@9t-NNTuSA->1vDj+{z>yS~Ghg9~R2}gg7-12+B`$PLsbGN4A`=y#C zF^9M=WtgqR^P6H9c_nU%<15;S<}-V%kDrS)lHQb(Tl+Bn{c%a=k!~Dd0Rn)XSrxjr z=Pqo3(Z#2dxDJP&@*7g%_EIaE@^#40G-p*C}+{WL86A%uJ6 z6S~Z*PmaBr)i4rwTA&{T!^DyVP{0Um8fE?6-X#GD8dEHuU#g7J#?MO->y>ImPC=1k%e$yO*T3N;bt{M2GcnM z4-}H~<8Dt!rwx-sx$}790noaKng_UE3kduI1At{_70l?Feihz<+uMsm z+PP4f<~*>pTheWhnc`7KNBG?@@_evyVwmUQD>`V5C0_)FXVdR8yBPqi@emw_i_(XV z-gwP6^afEH@H9dPoFVNL3WttWAq7VDxTqNDk*w2tD*ouXwW3q0l^>%>>qG%GVls+T zBE!fhoKpjJ@VF#0Sb zwqBA?Dg4E#AckM)dZK08ZTYvV4a$s92K>WR)wi+|91`q-t5mGA2$J8-N~IUka1K7= zQf)AHz3i_AISk^M?jkynxTlG+xb_GP%)u-leB@3k7{7_N z3f|0y3#?kijPV{ep+6^YP9Pzib9%uZHiU@a#%}7Rhh9BMAqW;NWCR8tT#W=%rK;Q2 zr118McGZVnNLaBjKCYGi(!KB8GI8e*#{eIf8gPS^+y`RMODv`ApKHN#`YKS8EN`pL z@?jZ(KxKC-;!MG8n1@VY(wSu09o!IdLZSC24jKBY;9w^T8+6mv> zfqj9w-do~S)^D0K;lAP=tokge9a)d|-_H+4A|Wg)yqpbHd_=@pT*W(V)D5yF62JteqfshA1y?Rw7$NX!;$cQ z1F}$4LQZhv6xoci!Gq9-H%l{9pi)@B=MggBt399}?_M|%Gpbq_*DjdfC#l;yr37Xu zk4C&C2hp5A_7m^S<*^#8uy*VxeA|WSYAYlf!OpxFXlPN_JUuF5%#sS6V{CT-19qs(v7i9X zPPv590;P)J^o=mmjm1^tZjxgDCy*`eVnLunm^Ivt=x_=YTx$tiysc;@ z#v{2BEbYt;jlsf`jG_(rU6i0!X)xxnOq>KU0UE^=K$VX1s;(GZM1{du8SLn0Ps|K% z?}VfhKM{;d6Wr(t3S>dtP7`y( zLPWdezpvakhaGch7SUEcguUrQJ}eS^_RPdr_W?Qx-_5hMA_DKWsKOj8$_*}%0@*^t zZ^9*AofVt>LK5b#-%&dgK?x_(!js89Vbqhk$OojF<004N>cv^}Q_OC>iSX`^?(h?I z!3S4=lD0ht->kQ9f?ew!a3cGGxq5lFu7Rc3II;-`1qz}@Y8=ID;?|YL$iP5s7^uM` zx@|5#?{=)#1%v?>n!5%yAVKg{=*w?>+Pd229(-#2P!|9UM`0*;Z^O^ogxdOs9;F!t z9Ehotkoj#jIBVSPIv`*a_9vLkGmS|xFy_yhy5t#p_?BZM}toD@!?eAFWsx;N>5sm1}YaS&i6$T>pYd9xY{Ef7jg^Q5QMHyi@$-N?c@0G<%AVr!ad|Oe0K;i9DVo?pd2-ISD*QPP zK6T15zR$S6H2X6?BxmYgkc1`7P8oIKxcQ|-av?#nP{HBYp@G^g$vkseK~xxk^MbX5 zFbQ(CC3{ZY+l&O6EV*f(%croda5ULX-76L)=O2YWs1@w9o3E6hl>8}AcqAA6R-KrO z$kOA>iO1$p*EKXw>uDn|M}d5;TZC_{DvRqePX0Vgo|KUiSVJgN(Nw9+d2}B0*H^-5 z&}|Rz(}G;)JDOdyW)qUE?j-0q(Wu(*wqe3;e-=&;w5b6WK3V`ZLrN$tsAVgaCNHQ_ zZ5_lZkP}|nb1jg1EK!CClg%r5g}0AY);lHm+dYlINOIXA%+QRakx$BGPq95m$wcD{ zL+a1ju;%ppUH@u(MS!JfqKNY$0h6Pwt55P%%{TAb&JCr59}|j2YAb>U!>g?Gc=Mwp zk>wqj{8<+}^H95GbvCD(qfal~n|&j4CRLbALUgl6SYaBW0JKG+ZCit?(N=*}7yN-OU|?jF zbj&6qApG+EaQK%yaxNrD;jEnbjGs2rJf!$~h*!xa7yXNL?BmCeBZk_(${@k|%=iCm;e2m%!blPE zxdmJ|_+tRDoP5r{y0e>-n7id|xSs6r^Lea6ke4K_;h?I}K6Yi&c{lRG^F-LAX+*~< zqmP0apEB-^hQoc8ax!k-ANph~@#UW1Ege}65v5~KUMT79ZC1bdRydO9dy%XPtFXz# za<&Y#h$BHB1em3F?W#RnP($>`3#OK|vJ+T8TWmo_O*eBYtMwF|tsyyPm&qr8a6MDn zmVoyaV_hY*HDGItd3xv6yN?8O$@jZQ(Sc&@hIMKUGnNg~jR=lyxF+nT`m?Z}A9pWY za}6!gG%{oB4Q>4D&JbMKDa%F^T)2IoVWHw!iNjX#7%cRxH3%;tk3RSCr&|B=mZl4) z(S7N`eJ!@bT7z$IrC#4D+c=GGpg?@PL_*L_Vc4c1i7JBaVAhtVYRsT->ChwU&?8|t zRA`U--HxMQL1jMdrhR0ov&WN`%7FUB6}DlaW5dk~-QMVP;@EzLpcuaKo(6SiZ(=Wo zdb)x-ti-EQ!vG}@Qu8NS51LMXd`G<>I5}A{Sx@icdYLHf$aMngU#l_I@oq98aOz&g zR4V~yIxAL8J*{!2DhG7iAba}4#?(*r)3(9s zHZ0V6uS_2R3D9C^UID^5W{&LFDVe-n&uSM<>ve?aodr44Wpb^W3ihFE=tda)A`O5tyOJPZA`pDz# z#m@&`5*!d>6ETxLoPj zzZmi2r%2@6J+X0ZCL=E9gkLK7R9cr!!-*tUCKw{`$1aQs%8-`$I#=#n!i)(sCZl}9 zKj#m;6=cy_-MK)kbVkW6uijIFSzy2&B5X1M$fNhGm$YYoAi+|xFSBp2=+3>=(OGLH zIyupx%u5<&%yYFy5c49$j!rYEnv}CZNu79=&q6Ctpx7+GYCyvdI~{vWfCZW{=B+SL zD;(604)uTkDxjnI2Y&ME@^$Pumpppyy2Y9UvSBgnO|Rpd?||F)$FJMz2;#NZZSJd* z$SV=!3y$7g;mhmY=T~))Y&gaZIeBkXDCX>tXET;vLp!+=er~A6jTQWy|85$1`}U@t z!Nwjfpu%D;1OK#keCE|XE`9W?hH+U#Ey2d7w`x@g5a&D5v9*jEP>jVpTo3I#DRDUN z%E|2b}ybn`*2L@9+wjK?W@j}NgdAC&JRYT zBemES`tq@b`&@@CKg!&P$wY%Tt3Hz1S2GB!fZrrbVn6(_p(B@}C#fKjJqBpDiXQ57 zxrg$SXF@h;iBlLqaquL~m1ugH_(tSpxAJNBL)14C)ne@Af<|v;L*lZW?TImn5)~B{v)utl$P<_2CfHn_AP!-^Cr&KLW$gew zzlV`NW}W0Z#el0(zvSl3?*{B*JgN$-cR%5FlCjxUCl`GzhaaE38)nS_ow)y_8M7PQ z_ajpWGReBPeY&o%IB7YM)vZS#Q~XR`mwh*#JMA=z2stn6I;rM15^;z`m${R284J%0gh ztB}CLnHeCb^W-nkHr|;R{;Yt7$L0mIK$K)PFd%OC?pJ}R=M>8`5DR!MABtZRVuH50 z7;e3Nc|ICYBN!=BC&&0?fka;7VKU-#V_sS;n$o0>g&%;6wmxAI^3hX$y>XoZ6QHDe>U68mCROjI_ zz}K2fW>!keH6jN5k_+2L_#2r~s_rV>jFTK?z@X-s=6s%<1_XSuLLQB%#ir7p#$hkd!pPPRO z?ju}>)8JqPO^2JoD=0Mt{~p}>w--eJEBLXy;aZl%-r+LXhrW>evl%kmA`l8F5(^;u znu+1z{P;*c`w-t#x~q|{>V6!-VcUGzqn>HF<&!Y-U{+AUVE7;eAc+q;?(7Gl06351 zGQjE!s6+%88VdlzweUd?txWh}7iLsQ7iEArwADEP#ir1Uixh+x=d+QH0%#BQX?V~iiV;@K<|i+VhMy^L+=7rCIA}U~u*Z|Qy+-t49 zX0N^1KF>4ztbOL3ImwGK!;8EKFhhR7>vMgDjianKfyE|^l(o*W4uU~tY5JOA%ybDn z>%bUg2mq{#)0 z7KVK&#-2a}ZAnkR(S1EE5gis?3L>N6%AACr1sQpFDFMkyM4Cri^v~`tbw3yT%{NKhGQ{=OiDvFO_JnujJqxc*j9LH8==t zw#NsYw0*eH8m;+2WT59{BCGaAlu>$c7ya4Wtr~&Sb!ohOYgZq7(5~{)?@v-{C7abd zP86>_Do9=JGyu1C9@7{P6CJ(%G-;LwX?XOT@H&=Ay+yXLGbvlXQMk?EL)(_$r@M}B zYcv2_r4Gi}minHw(bDP4Kz3@8qAw7ZlxO^BYSz6Nj$1LI_!Z zd(Op7ab+f8*x;?2vxQX zD&Ulme2g$w8NlrlnTOC|yP8FFU@4&z%&o%NhZA^dR0)|NTj;Uen`(49$g6rEP?cSH@8bs zXW7sPxZ>KYBlqU9(=iIV=1M0DN|ar51G&}pfwnZzVCo!T zSLF4iPc9Nhd*2Sxe)}*B=t4M+5#D113E(n3nP^dqY>Vpvmz*p;rtXbv3jwo%TTSZW z@23V662>XpRH6?&ujv*Sq~s>%z|0x-r@N>!hE)-aNXq*cqsDI&7S{ z-UYR4h}M5=JJC9QzD~Ho?8p5I*BNp|*%|x5V=4!uJVg&R?COmCzHy*mZ=ko5e%Jvf z!5(bF>Nln~fMoYxoD#aQYfsE`&pTyr%m-Ffh27X-&K3V@$EcT}f*WrBD!qn`a%@wp zJ$)QA?fw9M=Wb0%Br@im+B2R{usQSFBIPxF_O*KUt--9WKX$LxEp(r4VPOK#1%B~- zb$VjUlOp9K*F8kilELa*m5#h9yVLlouHs(y`CTC;Mtx7GpEbCkugA0DnE;`1;Acj$9U_{J-YSWEAI$CdPgjRke5mi{9{ zE7|upUgJ_*2K*hr`c@adx!6PO-eiyC_r`)>?&&l@A;X)CJ%)bV|M}|^O{|S0;I!Hy_nRB%)HW|YyxMK_`!gf8Z9&axt>5GKO73vm zo4vzpL*c)_FvZ&cEkRV})V_S=pC^d^g#{wszal{tLZgu3zQjnCIC1Q%w<{5{jRDfm z1vxMxO{L&cBeo1C%z+9u-&~W*{L^DPkRd1rAjkzIQ6M=oM4SlvyT{b0?r@?7zEGko z`$6_lko&1n8!FV612tws_f10O{!-y-O9#XWAOs*U&HzM400ecX69FWIv2jCN%aQ3R z4G>2*9Z!UqF(4XD(1}H0@?h*=3p`eTFYwTmo&M@EIlS2LD$~mpq%R6umJ7jeZt@_* zv{5iQW}Jy0NR+k-G9A<1q8uc2RWXT;y&h3tj)rB-77!<3Dbnc*#{*7kW>Rm9HWv104{G5XKumF*kI*Nk4XSKbA(u^lzFtV|K@zA zdaC-W;I8us7zVWWPU^;F>hIN5K!yRvGoT~}Jd}aRV+gh|gr^vyYYdc3ngl*gl9aYJ zG!31XCfkxGU!A6T9${DRqKHpdCZ(%}rmN?rYqq3oO{HtErQ>8WbnqE^q>Kf!1omWx zVM~VbRK~uw41!FiIX=^plxe+mI~|*rvA-qLek#*(Et4pd<&4jAA!U7V({|0va&O7< zoXR@7mPMAyRz=DlP{}?KnoY^e4rs{^n#z6(#n~=p2jg?jk#a7C=7b!`5Z{s$I+b%} zEr%wP8`+or)feJMhQ#INCbZ-xP33L|hZr(>>G-@%QeJjwUT$7qeoJ2AR9^8~9#bZt zh0iY|<^My1Xe#f^mlTAmV54fm3=Ft^vS2i;pmnMsS{m^eNh!dWlz*mn_>yunG0T^f zn#3xc`p@qlo)2C-HpiEgktjeJW_M!UFSDu@NuWjXCFSTspZNUVcDmc<2qiu9lFo&U z7dv{Phjb%Du?5oeEhF$l&i=qOUsC=NLAL=I^3=Fk8M}Cuak6r26+)AB{gjcO<=Wiw zZgc3cIA2n-n#`?MzcPMH)?nhMo$ya}9nyGwN$FoT9B|qlJKz_!63!wbXvVr5XL#d{5pQiI_5RmN zr(eq+y!FZ9``_;l|N0ot`^gb|u-fDFd!>N4vFP+*ePa0c_j|lw+|~upLPtm#L`b?tYz>J(=SIjjN61e`Y+H@M@FnH{ zXC)=?e_m1^*a4!&BvlK={hOq;RTmVb1qy;dng7?4axK3>rl1jD&_pV@6I#%mSI~kt zP3C$+xlp!D;UX6rhgY|v44DSKt7{zi-oD)+FgpynuHB`yN__A;(=VgpvpDYHvY zQ7S_lh6?&YGL?5#ienp|DFupG5l*PRZMn>uEqu=T>Q(3gN?zaFe5pfLm)?ZLFGu-JN7q*Qb z!wSNIml}VN)FrTzx>eSpCVbAx4`W(cP^&Xi5uZXT1^d5@jgbYPSKA4=D@9BV)vlH= ziGJBJV?ooseSgq%O|NCfxcbL6*fS}fX9=Hk{yg^6s#H0}9?_Ivoh3b>r*M1X9Ya;o zX^zi1k$^j_=tKaYb2?r`gq?H0z=|;H^#DZyA{fxjk>ha~KIi0zG5KLkei)M<#^i@F z`C&|c7?U5yx`hKEnSclmbYmgnQ>X8A z7t)Dp03HEk5cH<6o*){qZ9D;mtM{Jdw=3k?q?9J#W zTbc#LBoYq@G3ZGR@hm3b-<()^7?d!Ha36v0$3V>*AhJ}D^axZR0PR@)#|9`nQj!e0 zy8>5Zi`ihlb6}Jg|km1Bn7j z(g3zN2#JaOdk?gL86}5GSFBcA<7kLt~5X7X*F;jn6$;#o96E}mQ zjO4!uLxq-r$+R?N0}s%2+pLR&gb1LmY{15uF3tgjXc;MN`aovZTd2>C&yrhEf>1Vw z?41ppQO0j*2Xh@Dn~o>kd_C$kd%S{EfSHtV7t{|9wZycAryx!@=mq_`^K zhKdy7%5G%;Ik~ z%`RSUDSkgy{Bfn+zdrmp>7y9UTI!G=tzI};wpMC3CC z6PSPPkD_EtBn(R=-AlHHm7w!WWLrz*r%Sf2mtbUBiiWJs{-~HOOFf^Z*~-$IW@)dp zaI&R3hNXJ_CD=2pJ^7`Et)<4(rTf-P39@D8hGml^ZOgDSoBT4{)-wC)GRO5YNkQ2I zuyPmoa@VkOxBPPV*7D8#s6l@@S+>H*u)@#1;)Etk{7boiYemp>#p(45YEAi{`B7Bt zg|Nzy{K~M_%1hIgSJo?OvQ?3WRZ;F$F=174`Be$6RY}uTDeF}X+3Iw|>P+|Q?6B(G z{ObJvtyN#16d68CDqgQ{+*$+GuPFpe74mp(!JC}{PmX8FCjEq!_dQ?9-R`cXs z?YLjv)9{9;m+PJd+?+VwI6-ZiJl8ZCb$2TM?sRC&%;nkH*|z6V?awprKaYR-BB|p= zK?f(b>t$xw%aZQ-yjOgx`tcL@^U9a6e5yLJ&3g2VTs)tu@~JBD0^GM}==n~R3e?J& zPgSqp$8N>0T&&|$)q9p|3OiAwM~~j{5+qWvkyBNYbAN=DYengtLeCrC?CQ@z%+)x1t<^ti9NtU_Mm3~0eAH!mO*J2Km zDlUqzcNa=UU02~@gAPmZsVermQt`WL-yUd({)Ngk{ORA=LVE$`K{T6BRT29Q0_Ca@ ze5wj3(ZB=(bb1n^-T1r>cCa%BQM)s>-LTe5%T)s(h--r>cCa%BQM) zs>-LTe5%T)s(h--r>cCa%BQM)s>-LTe5%T)s(h--r>cCa%BQM)s>-LTe5%T)s(h-- zr>d{e)_khUr>cCa%BQM)s>-LTe5%T)s{c<>Ra2hGgPNyqL5F`wR?vR`c)87 z2omK2TZqsvQCDTV|Kz6L7?7I|&=yVG{>-cLLV)Wd;X8>{oAQwAaw%cre-#i|GNHti*qTpMJps!{~eXc~+irFrM2o^)b^E^U|AOhiR zX9WWSX!L}5l;L+zFxBp^i#0h7@TM9bpqL82j3739ZFV2`PDXm_oXnv{A4B*)-vb00 z0MZCZ`pd|4_Oasm=#!J4!&ZP(L#z|ZA?sX((bl~OF2sJ6qNk3S1f!AH8}*_Pkrs;x zCx-fo5L6mr2Z(buIWE>B*S#>8cUYiA5l#>R<8%!SKsTZgZm7M2Bk>No@t41Og2+e4 zA^Tx~V&IriRU<6{rPz!+0=!PRu?aAJz~d<3kOVxw1ej)^T{;nmRCch@UYAE>#tndu z2p|Y;W`)`b;iyQE0~)gt;>6_f+_+Kn2{|I@Mo7y0$&`<)DO_~wib3jEx76<;sjIoE z>&>Yflc~Q~Q-PK+3K{N8WWeIZK{Eb-^|7LPAO{8`Ei1Zm^`B_iO&=^q0Est%^id!w zCIGGiLJBGbV6J+a+c;fn3bCIGwV^_7IZ$IJRACA##|DIPJWyLYAWi@w0C{l+ zATk2{oxtkL*vc_7k~W2FL)zbktL3I}-JOMO07Myp6$%h<03^vEVVbfN8~D?>va?7B zEIdvkeNoWJJP5u4;y{LJqhNBhY!f|@C=Ec+9H|G=9VB#Bfjovx1ZF(vN?%UmU(M=B z#->?CAt4eR6;UQ#gbiFg3rMnoW)w)?3WOqpQN@x|do1 zm8!Zs0B2ct>t#gQa%aOosj9f_ZpVN>sjBC6`O)?A1Gc96G?+fI;zU>lCBGuz?^IQ` zGT5;4oO|W}ma2vsnhqdOuuv(f>)rsSI-|yWQ>*6NLP8?|0J{c73ILmw7}$IqhHyYJ zn_q>6g|}?kA}=qmtgNi9t*xi0XJlk#X=!O^XXoPL;^E=p>+2g76m;&~x$yAtsHmv_ zo=Z)9WVVS?ySlpgDD}@b|KA=0JQ^jE9u*xE8yBCDn3SB7%1BGk$jr*l`OC-UqT*}J z5>{zhc|~PabxrN{8+G*!Hydv?-M(|Txux}9Tl@V7?1vqXI=i}idi(kx4-5_skBp8z z8Grh0VsdJFX7>5q3(m{=R|~J-yj@&ce)s;v{|{Vhy?i5a9wk022hCzTETGV{TcIy2 zIMB2$3-Q_S^gCzvQuTUr0w+YRM?cH{qf1qAh(}5&5@-a_r80>H@u!%5l-2du?JYgm zu)_yYvlvMm;2}iR00n||?bb1a#;;1hRC@61;+)XqJr`{v`XWHrE{WR^A4W%2{WumL z>B6Ofh4fGnv5_bc2!h0+K#6zGxNQF5ZO8~TNRny;A!uR{JP5&=f)FPNV&cFAZ5kpA zaEtFvH4`N*q#a=F9SY~92dbF4!8OXP4qG8wDV-*yr6{$LV8hB!v8w<(q7&a^`oZk?RSZ#@>YWQ? z8nhCDY`_l$VQ3I|w{9Q(+R+%aVrL`b=lJ#vmI%g_V}@nPaiI6AjW-`E^z1M~5DnbiPSFBHm_iHrF;GEv3m41)#tgO_=wOqE zeo``|>;cf@gDXjJaP!928B-6ZFhWIu|lt?C6Ca(v1wo7D&&xjKB{$`vcRgmpk8sv?TmLCSG#r z1o@fQ*^~&oJP&@B&?-V$p+^nkWwBc#4H;``o~qrpy1S5QKSi+h(pY6mI1g(BUiAM^a8nKHL*t+n^N1*3Aur&ZOmZ4izQ>8EPEmBQ~_--m2;uQ*h^t?OCKt2_ou z)zZ5?z%?%5*V5H+znPn0(tcZ%jyUTaSYyqv+xYX%$VATAm&IlWTjx${z@U|!ijd`QLsXgXVT`zMMa%xz2*VI}#k`@^d8 zMy>`tj{2`%YT$pvr8Y;tU^SUrt$tN=$H%q_%A&J#)I0ZGD2hfExr zd-dEof*D2(`K%L_KhI?5M56Oyv(*Dym&a_*NDv0{@yIMi^XlUgY{3WwTq|1dI-$nj z>#bF--_Ag?ak-83y)R03(Sx=ylci>cV6BeAE%A)w*a7Xtm;u+P_D7^BuAe7#qGScH zAA73BD=zPgXO)m4BZKa@o;^Faav0T-w4+RP#6_S<>ndjBwbDjHoE!3E(qmmYNbw2h zJonouGTBEpwzvF%^)tnLO@zL=Z#WH;+E!a%n)!2$f3-^;XE&%?hz}CJc_rFAS+Dm` z!$;nDEm(ieNY|49$wmE&l@)}Poj&SzzWgPx!8p>3tll?yp61mp|bBJ*Hng!{jgu&+JZ=jPbPYyya#bcjIVU zN5w}i#%z_ZBQkX^vEgoqc+ns6`}4&&gcaD3-zevbwksjRp^MWVanTkXZ+{$Ow%$}g z21vKt+t;t%*>lhSR_nRy64kxfu*(hR)V@Kpx14AYkp{{A^**C?`#?J+&>O(e z5gK(V-XCSBCf&Kvdh(T%s+`ClSD|wvH-GSS=R99*`SdrJTKSJIRs4VFQV)q-I!J|w1BaGa5r~GJ13WO+sJl(h&d3gHA!#}u|y&FH`@2qxJI$f^X#srCA zg4B+vzqt@DTtb-qqGRC~mUAR*bw3h8K?ovzVpx|3VdO%^%ZxGYUa`HYc{_ zCVB%R-c)$IK~f_DhQ`3@1XxFvpmzhThzM09!aEI;uVdh70+@~gqq&LI01Qor6mei^ zfCm|OOR2&^&|D~e1T<46Xva-1W`fZK*c-Q077mVPgC47=PK1DsX>d0x*c1ny?uDo^ z!F1xEF4cqcr%TlYU~vF!H#OOW10%9i6Wz5Ba=;UPcc(^k+!*mT2Lr${Mfaz6dU^p-e z5o)-W;Wmjm5I$~cu$W{jK$H=fi<{W3I7-Qha zanPeAgef(d!hxA$l0G3)B7tl)5gLhspvhpjzHADYm#EB0iR7l}$8Ww|nqO#MCNtSK zK6|z~*Jmx;Djseb4>wzac~kSVNQh5yphE;i1Rha~M?{g}fg0HqLWaIZsvRvmCO+>z z5{Zw8MzK?hmNIY}2&PPqc}or{FXP?;gg-YIJ(89dnoViQ>Cr&o^I+Wv5MC|Gt8t(^ zYq`d21#kKgL-@jz@mV@JM4twtv=4#?z&6Bd-{X^_scTCUMS6KfZj1~c zGBcW-gyAx8;1Q`)1zX%p=@?$BCLSSXTdHMJ+(JDhn8MDa zZ+*JZZT+bt;b=UmHt= zx7s3nXxHL5{{+I|$H)crlh@;MFfF(zg6BePFfgx%q{I=}OW9kt z1z9>Hb$xLuc5=7zd0;!BQQt1ng9ydZGMs;AU~v#TpeC9FJ2`#ZD-e8S4LVJ^bK)mN zEBtn491PD+36i^|$c6aWWt|Va<9YAyi5->H@C5$`XyCiMcD3L`#5*zfc+E)#Svc;U z2q8Hi4$OlFb0fpt7%(pk+`0fkq?ISjwX%#_cL#zQaxE3&aAzD;mkK_2uSuQ&*1Rwv9R^bWxLF>6*brdMd-wY9wVj_S?_jr$$hD6dwT~BM-G7Jh zAwtJU?XxrO$wFW|8muxn-!k6|F{(j5oT33q`|( zy8X0T_fh|JsU&vV*Id?8LFZsgr&%4vIQP1v} zqz8-Khp}zFSKfh(goH|WAo}d!6av_S(MjUmz%yW04J{!_FdG`o2Xprj29C+gP$s~9 zMqs{$VAgDY{qp8BY(S%jMB4oAW63u_kBz{b)+%~!;DK7;!t4E3YzPSl_h2+N7qWJ3 zKs-=zE5hBzw*K?)`+b1NHh73lL53}gSJo^KAvJVsqXq))Q;xD>-WaH%XV-UjFE{D_ zTVl;;&*6hcb&Z%-jBV}`o}G!z>7Fd`We?*ulKj}oZPU45G?EG**I*LhK1)@v{+&zp zCT^}G?8t0`xe&l&JFnv~S()!E&?a?VhFPx=*CHF3VhW=+@yQnJAlD?&=F&oKpb^gm zZ&w)8+?gDr@#s^~u(e718^Xx`_Hi4H!ZrC;+)0>C{0$RAs_9f&bo|r7P`Kq%E}EL9 zj45=WffRs71!|T{Us+0P)rr0`XuzmLUgcMtEbGFj8wi#^;jUsb$ha>v(!I^CrP`wZ z*_Y6oV}oM>MZBzSGGKfF_``t=Z`rzt*|B{ZlWCsg_C-@IfuiEqr}9khS72IkeK3b) zxIXzQQ1sL+Kg+)Xh7ZlMOn_?>QZEF+C`@pN&0SY5#PN}tzq`~!n68_bC$862?AU=I zu1_quJ%;wr+%KwZ5y`o9J=2t(?1ZTjlbyP-vlmZl%JC8__L?j45Pf-a&i>THI!tR* zLgvjHaMjLCP%<3nF;aJV5+8r#)~Ofx12ZwFo)?4_nG!iGGPC%#iC2qh$H}u9BCH2V zNlxsH!J^7xg}G1>ChRw4MTTSE*JH1VXr3*(p*eL|WDbp)vvinSem;NRL)2_(-ug1H zsvg&xm|wFS7>!tdfpLHFBN=q#*EDb7#d2=V7y_I*km{8T2oyR<@_h`3<7{YeT zGB4v{V~elU4kp!P!oFzLlS&qvUAD3K!<-xU7Gt#ZVch7I`dw05S{_UXP5kMof2iYBc zG-u@PQ9Q1WDdE2@j?6Q(*^G1H$PJkt$%o(cw_g#BdBm+%M9qz&x{Z9 zt=&Z#rm59~_^+h+uck|1Z0+t^J!rDweKpnSa&~%l_`!S)bp^sCmV_XsXa}C%e_I-U z*Wt?J)5hOPR7ld#j{3SLE9&?0KuFvW2+xJukSi@VZzPD&BEBBNCc$2$QHhkv>b%pzF zw+b~7*|h#Z)!R?vdlsQ}!Jr$F3O{n}E^4UxL(50I9aWny)rZl!o!oSc{4R=lJ!(MY zdNHKp$^%5-Yt*mA`H)F}LZ25hh5pA3DH#qC$vDwa^&vcKqW+a*`@r#yM{ka9W<6g(lqg>C>InI|GP6=e<$sm9gi3$6sKr^AD%qgjiQ+3J#Z*PpRzaF zm4aruMDL{>LpQ{ZKFL74%yhp0@P4TM^Gfr_zbzA7{qno*2mqjPB^;<8($6;Jo0acP`4}v1;CTGG+8q5^}W_{Ll3rwCU40O)?;B_(<3a3Ma|m{wiUe zfDl&54v|r^iZ0pa)!}Cr>un>w+bz<26&}&MV`*s~2Pj(N3AgOpQ-{ zl$9^V@Pgk5)k6hV-jwY9yGy0PrTsCBXqn8C$ZjvSBuhuro5!oXce>G5@{L&R+g?8C z4K~`qPSPjYkFt0?PhAZy5$U;eTO=!?+G2rgF9UUXsxFp35p&lQ-*%-bDs|-7Zb14k9#GoH=job?a5@1fCR>8N6ap( zI4UYc8q52tWV@@z#YiuaUaIm6A+D|%**RrJThZl)r%`H0H!=lxCVxD>P;q6+(oZYB z%)ke$+QI2KkM8~PC@(tXA}J{v=Q~J>Da^OR)>M?fs@%x!J!$-nikZjib7La}N#`!u zi0aC(Ks-A9uu=QmgdYd0_Kuww+mZh}|8Wo$h1C{!oCJeTAvR#gV-6`QUOSporIe+w znY&8A*n^OGC8L}Gws54FY^5gP+4OF)q7hA`*t5bTF+1GaJbG2)s?J8L>tE8RvUd4o z$DwCvbbG9JT>MeZ6I#)8`{wS*r&dm!U~#-k-%RCQBvqY-BhXB5xTKG^;0`61DkYer z!MlN5l-*yH;vUQ1{$j=pjw2p|V~fy%HfVn|i(GM&Sz{wgL_GRK%Wq2yPT7WXBVN~e z3i55|EvW`Ok){L8k5=#BJUZBw&nlewb{S#d;<@)P<9jo}b4a z+oE(q@!0jYzNv!Fdi_K3HfqbN8VetXe%z`m&}I{XX($KgG%t5%CpU)U^-p5wm1{_YsfzEM*QY zZvMMVRql6kpU!sW#O0a=*Z-n~ZBC9VwiQ5OsS34Sw zfDP86WIDlOLzw8Dh5OC*!ZDKz6;?SnJ|!>~2{IVuXo9FxzD>}6QA5HWRCa`*-Rgm7 zj)!Beg}goc9P?1xbr}hji4)gYb!@CQC^GX65vZP-r^ft4XvKF3k34k6Y;Blms4=0+dJDTPx9#ypZU{jDxus zcf8wkTbK74y0v#?&pQo>h>~!`{1D-mJZh`Gi(V)4_94V$pL)x|vt7rvC8vD3dIGB~ zbDZS4Ci6A{ci~f2cRd3}Epy3NyBxHGn%q)TwqQ)Q-e$sQT*^(sCL0yaiU>}7ZL#t(7zl*1}Xpj*|J;*i}Z zdXf(-&1U~P)Ityq)?#NKyl5~XWy%90bp1wD*CFpcR6*KLqsKRoR8ySqTK3`oB6RkoBc^;?!qn-l$ z6m~>1X<$!dN}FtH7sjl0OasEx9f*6yjO2KK>ZXNGvIGz43AX=A`@Rk}GttUWhdX_c zx!s~?{#J3`rFbz3DzsYywYbChP`SRMYxt>c2F5buX(>{5zqsx~DKd>6x<(6wUqUB; zzLom0R&j|rt2kmL{&ar7z!e77hK=bd|9&`a=H=|Zcr(E|s@b|_gmC0tr{hitqf;&C z;F<g2HaW4LxBu(t+vKI~C;*iG_Pf*C>q1BF*16bJ~ipG%qQ`9qarLd{}(W03R+AG)N~ zwlSX=PK0U&Vu*XupaP+ijWmjhKn5c;S1rGBuCDdSS_>f0kDrTSoST_H(81VQ`V#EM zIaU^-72}Gqqj_3^={L>bGZ%NdU?r4a($&1RqLtyXnKp@*TYa!FmpJ(5VVNQgY74SS z%7pFK*|O)QP@ECe^w*uAeNgv?!wP>_8etWAYz>oE8Y;b1(71E zv`ksT;|4FqhLlBJEE5z#xB$LQNxV0i%DR|rd<1EMgO!ZKENIYUY}l*sQYL054iqT^ zILw<1b7J#!J5~{g+@eXr(b^oFOOi>A%3CpyY?b3c8lqDXKV-LYy zmyq(r6LT>qn0qufU8+7+^P*YmCpX>SirJA9>PU-#z0`n?*QwOdolY8mWD%6ibQ37V zm@ncOap^-*+-*mY+oY=zb>sw7mBZ#%F4l)#HObpN2ryL4VeUg0$HV%0i%2iZHXmpG z+GYbZMkabBm%`0$y1gA_ac;~iZ)9|vFAAOx7``<#DMrSh?UVkktISd|V6Ea!3CPYn zysX#i$m2Mef5W~yvIJ#B>v*0+#Mwg36RmH}WXt zVI{=?*5lYeU8*MY+Ug#}b0T8Q$}6?&KvQ+m7jo3KvCEN42jT9z=p*_L$1|Aa)Scy8 zo;urmZWb!D95iInF@We1S5J8w^mI$nBtFoN1|6+B?@%J8wW_r%%MTxrjMfZKz(^a@ zgVsxTr|r)A%#wec26>04l^_evrLLtCu8k}j8tykYM)@azgI!zX&&}&9n)6J5J2OYs zRb>{CIwP17Reu&&!r{%Sm=V}2C2+s2n8TXU!wLIgWx?Y#*bvR8KIEcx+Qo1IQGbk` z1pzUT39~~fX-yjx?XJ$6HW(p!DmFm(1{6(JLz@URH7Z<#RrZ?fk*sTOCHu!^!-#lh zO)uQqj$v5JC=kvjIkO8e#PX=7LvDiCxBb|~Fs2HvgO{e0n>6)1{LiUt8JX|e_ z=0I*yD-~{DkL-=Db7gHQO^ip?w&O}CeB2$P-RJN^UJbjR9JxI4R>3JpUQB7*a)Z3q zI8qBlLSsm+k@Xmth}~a64`X5{@lbCnSecC7j?k&r4BFLI6FSeUsZR)8ST~t%DN_lk z-JYo_@Tlb8I>Dt3T`tpQ+a!%6h$R0K&E}lqyvRi!>!cZB6ZSb_Q@SH1zAVwK%7e)!GQ>O55zxp zshdNHs%(VoaYcRN{=iI_-q|9D$3`+j*X{&Z)G+lTG@TU4;QZ(ig@`lWWXTm!M0|o; z(lYW=tpAl*#Me+wZyL0GLet@Ch%w3`s`6}9K-F;{i=$k^ty{O^X)XDQ=W@@OcR*UB z{hKx(o_`6tnc8r-h1R$+!Bcw?VE}B+%}2EH1^Yj0iSgCi{Vi<9$ zC1qmgwkC05*GfgZ8l9JwabdOhh>wb553>Y=p|2Rm{gHLiUBWty!@f*u7b?7X9NBJV z^GP=UbN=OlSVZvdYx(D|^nE5NeZ3rVVV{Pf!fu^^aH&U`d*IjISOn3Qo)$$Ec(ZtM zF!1KQhfw70#7{@;h@2XILWmooEWe>+^HkRO5v0gadC0K^2XgcF?;ojv5<;! z@4E{*XV+?$_G4~0K!YDDL}Bj65bj1Q)lo^kg)2sTl6r%y`-(^Uisk*#(U0(9?Ymxs zYl4LBx|piY`kAveAPd?{p?5ibj&$}e-?kWPiz4tcW4tjj~$>U%@k=|M=p}gfqD)2g#U0WE|UWaIepKJ2d9F%KfFiY{W@f)+%s!G zygE?%`TEH<&znE(HOy&)zpl$-$#*Urj;w7AZzPTEzdq8xdy6iU2hQppDIFD*wj2HQ ze&kBT=;-y)ChkMAR~hnNV*04?n}%cU??*31j9tDy)=GxkJsx8KU<+n^xJQLS(G$b= zC&tg8?ECeEpfGN3GH!UP?i%=_WzqPNrLo_bC--{C@8MwH8#U%!$R%Up%_~5-$W!fK zO}jI!Ja^XogG;>()u)2%e;iXIL-x~Pmafn2e?2qOfxPpcFsT?fZ;vuJ2YYfRqP!+! zE>D^aPBfF@k$&Bps2acbf8JS~$bmV5rV>7kmRc$+l5u&LZ(d#3plnQ)-4Jr&Je^zu zKR!5p{j!j(pt1#R>c+3kgYwF%+{d?yX1=8HIzmo~X=0#Terf2J&$RCB-#!n1VK+P7 z=WgMH>HjrL53U89SZU*+E(AY+V3II4HBvO=IV-Ec7<&FIv*4bR8g1s)ui2wG*!~fS zxC0o=h!`0`|9K;_PnV>7}y~KjKqP3e|UYVz49k{^z__W!qDcMvT)XM!2@-Brtod2*V`Y! z#VbQ+-S^3oO%~m*>|rop#tlt-1&YPaEp{B9H`8A7O6fZqxO9EzwKe8M{+w_0eX+FP zO9m-}WxuC>*hO5txa@AZw86P_-EkqG{N(1))7O{9%>SuNWrC9&-xD8kelRlnQ(pbt zsXkmcR#@jdZ}LI=)>K^F2W`=}4i7)b?kwsgvpFAic3&355#GJN|8jJnXzTrtHUp1u z5?*rWbf&L}Z9n)){WM%X9K2lji77mvPn`$;P_9b;_|uJx8iMU(gB9NMz!P=c=pg?` z>{k-Q8j-W2^5kJ@Q;rYKPMP*r!L&nnzwq{vw@UTcw#TA6BX2aW&7s~r+;yxswhd`Q zoY37aCu6!ftMM5x_PlyuyJ>1?Z!IEn-&dK&l@1)_QOeiOQ{xhgUvJue&cZD_4@)-< zd{Mvpt@GD3bokrt&~uM-x&KX<+788ypc9f-+Yv}j+FZ)8kk13;cHEDQ;c3mnHQ?lr z7{;eUQ!y(ZV(-Y;l511zm)3EVy7Kz<&&0JXLQ$RROwhspTc%+#);#duD?giY>-4Ul z1=`S2!Hw1Zjh*#E4fP1y3mXH+J}$xD4W6q4kNJ*KW9IU zl8F!%GDw98S=#!$Fxu zk(ky2W)KKU^Z@^yxefG=p^9#``)!jzpi0kYIYWO3k#qTd@;it%hVX+1W`vYUu3F*PmD--$Zs|rvyIGzB|g6*;7UMX_C zxkps6nb~~2FFr(s98MvVhe*Gd8)@arVb)vTn+(jxAK7h;8Mw#I@!5$r%=Uv;0-o!i`77PiO0!p zsGO5z^WL||b-@@EfCu&j_2ou-wqw2+_i4;>QR8tcv6i-IuQN zP#tUxe-D1u-d(C_A;Jl$1Q_l&42>gOUbegsK09rSz+paOkun^n?-4R-Fz>@rDHb@6 zW<~=J$?Bv7FO&Cf2cX7Z!5YAi$AjU^U<{in;lE)AS~j$X0s>kX_MsSi0u8h!J^e=a zL6r+wMRDMN#HBieleP~h!O@xzLZ+AchR4{-Kr5NU6;now{~@=2kj~u z{r)7SRW<3#b=qk`1MP6KdT=P`}(FwxQ5Pm^Y8kcLOU39n;`)LUc=JCm~I8-?2p zKD2H5eY)%DwnhV>^%?VL;X^sgGhNWd!BSs}B?l|Zgp9XoHo@oCzLS3zUr= zOdPu23L#|u@fp-(V%xdoAL+~Sd3H<>Jr+r7_|GmCp}6QMdESVl0^Ur**`NYW3CYI@ zW0e7%n8-YY2HVvvngdG-m0&&-%08UHOQTB21ldB5nm>;q0n@g9-Av1J0H}MB zYNRbW*`>mT|HP$Q#?io{Q~+klHbv{6s7B)lf)5-*_q+Tr_U<#RsdQib{z@eYEr4KX z8mcrk2-4M1L_kVZqzM>`NKpr|p;$r-p@-ff^rrMKY6R&WkuFAxAhw8&raU+^XZFlK zyUjj(pZ%ZbByVyhFY+d9UF%x+{rjvh34G4VgkqB=ox zoOaj3U5AHMk;ZPOw~~<#M~C7RXJrAD(a=;GI^X0HOfpl*C;>?VTLS(TwVh8&u3aj& zq|pzRY<)7fxpb$y(?jaqwIRZ4E>y29P-Wg|fH+A+N0=%qvZwBR}ipPZA4uk49y z`9X1|WVZD}eHPZ^qQ?i@nR6q2kJBVAZ=c|d3J1#wNLW70si}BZl6N=$(r)ie&5kGI z!(}!$0x%;cWFe;XylP`>Sa6;`w=YwL|T zAp~2kTbD(Ipvg<4dFY@?RaKrUC!cX~@x4^8>VjA20Z`)jV-u}Qi~2o_7OMU|Mo$Wg zPn&q0v;~37yQ)jB^t`gy1i+Az>ark{*N(z#qwS_O6$w4BT{PFm`Xg$p@=e~jpI93o z?X9V)?Rj&~XKjMPRa@6)^42S5ZF0f1wt?kR|9O}C#rvUW{(-OOi>6_`H2N1>jq=##<-ATuw3@ctHIk69?mCI}GFb@GMC zpdbX0oE2!}gJl+0x1UHGiWsl4?e&jh{;G|Vhs>Q8)M=6y~2!YN1=ZwI|8DB&AF0Ckl zTzA8-!0qWJ_xwC5J;Js_-BtICqVt8*Kl?bPr z@1}&@c|oeHBNfs}9LpGX3YRHP*4^MxJtg#nYiN(OUwd6x_h?w}a#%kmd{8ZX$SQo; zJA5=Te7r7vax{EuIh=usm{yCJwTgJ*9kGxYu~_%_T&f^-z!QE73x~T=F=Ev77BFQ7 zNRUYtq(f1H)5Q%&(8cC3NgCQbE>3Zp?x`lHnF5IhMY zLj&n=($phA(^S1*(Nxo)XeuM_grTmT8`2Sg1_VRYXb@upY$pH{ru~Vgdbxe4sZ!Xa z=#CQs9}`F~6G9m<6Eswt8H2}xd4g})PXqi!&|jsgKw!G8whaK4$AMeb!ABVESwQN; z!UHY@ct0`?xq{k#2?4``RO9~5KBVX zKjBgXH4gM_id3veD9Q<76l}WG&5CNg&F=%Ly1Kgm4v@;KqyoQvu|VofXWHS~lBajQ zH}8|XYD=FzAVJu-8P}CPZ>I3c+jrNM&$Lk`4yPN}SIl)KDtSzG*H_N>(R4$%nKV?r zd|Ytsr{SiNGN-O|le;wZr?rh zP~>TBA2A`^@IHU2j*5hdlShq-NTk;pu=ZXSNM(Uk7D#1*R2E1ztM9bzU5dZEK8K5} z?>=q%A$el`h1y16eeadt59w>`^LXxt{vgwjS;Aiz49pq^6M8@9X?}f4h-?_jH~my} z;_ISSU&C;1@23)*1PZS z)R*2b_t(C@BXKt}*vyujgugKaz33x~{B2fu^VQVkx9zH7s^(R_NNcw}qRs<)XpsIG6{+oYmkAuS?^jShplo?N^9 z!Rl_AIhgqHdQusmuHQ`_wR4*-Qwm*91+wQ++pYWol9k8zGG<4w=Hp`mtm=?g+T22dYGC@pAzFb~j_L4C+aVA}bj zVz9D_h^(W4Cp=L6N95|LMxJ>O-VDi|PTbB;+f2411ZZd!j?7ODv~NG#f5j=s3nu3% z$&cO?v08Vc3@8lfI~`!}Wy4FDz*&;mW{dTkBB$WON%tAVOGi#1k@5@T@h9E;8a)tM1vpn@IiO}d-`QG^!{=v9TV217WT+0tkpZLJu$5F z-vCm<>eS870Q@Qykwit+Q#lHqP7~mc=qR3G9!~Ys$MKMDRFF^-$e0k72ZQsM1%y&z z#zd$#0M(#@wt8-U4?$A_6cgBj1_=`&JOt4B1pw6#Y#~5ur=fMNA=}*a_oAcYi5pOT zPpCc>YElLG;C^h0HW zA<6($jRY1#$8nKFiAlg}0`M1nDmC6^MaT6=pBnLNpGrvhC7pU*Fy^~Yr6;9c4be+X zJXM((|C3Lp#cujkkrnXwbSeQP&<}8Us04tHh|H*`}?S>04tH}$`DQyHvo>TkNKrayO6 zP5;H+ROX{UbyL~?PjypMS>4p%(@iyFbyHd0)PKI4I`BX1rovd=R8}|jkI+p``ajW4 zHT}oBsh9swHZblHx~U(3e>c_iPu)~4RyXzkb~knT58X{IXLVCq-PFI`P4#AV zQ(4{A{}rbCdUes_B#692OKg->TL%^B^KLz6&0{I!3&r%45V64f;7GN|>Q}*T1!okK zM!c_N=v@OFRPfyt+JvU~aKKsZBj2sJfC_$XoZXoTdMqdPS`WBA%uv?#ztNJR*`(J< zkBzzMCo7BVJBW4|%vz-}Wxq|5b5*K8AOnGO-0@K54IVkU7a#XNdtv{nlaQrDS{i|# zZt`+Voe5Ig2=cgyz)7}(G81n|)BBSjw`HuSI|AT#>Fnp?(}$2Z#R2A*TU?em$fvyr z%uiHUAkEOK58FsfJ5VjPi2D%Nc>1j^OE)kS&qwo!7+n+)XcOhB0afxV&sWhy`|nwh zITr`kF0QbOt|VWX6YWeOD&9IX}{YZV;t9h{gLoLm>2I=b0S zT@I#ULbB9Ca;!q~yh92SLyGD`ZjXkPEQgf+rU)QNin>sbfbu=ci z3MBmUyG|ARMW>d~EZUhBo&W=6Rsv%B6I?f2gzpvv^lHq`wopxXUYK$V!;p7p4mpgwn(gQh=^fD zPDhN)u_D`!irlUQoKt72VP#%VirR(6$&SkWz6`uSO4O*TU?AHdM$WFYs_=0>A=@a; zsJiG$u~nt})6VMRky2uFi0I*(+hdikkBjWOYVJ(bdT#vh2vr6&>JKPXpa1_tsG9zP zgsK`#sQv+k>P41NWeHW5P-O{KmQek}3RUKRRH!mpLX{;{|MNoC?4J{=%zsL#uKlZp z>IzG!G8$Pzl_gaF5JJ`L-yl@YSVEO0RR5b$WwM0oZ-lDZ&qCGgUo2D^zY?lJ|EGj% zI!mbj9-(T=5~?hr`p*m1_y04Y8psl=ETQ^G5UQ+@Dl4SQ3aPR}s#^R z@K1_Pa$nPu^0Uw~^pc{`W8S^DpYM~RPoJdr(nN{rQEK6mk?$^aYKr^#)j~yJ{YH!P zldE9>#K-a-Me*6x8lG)KoZiF>mbv*ROf7ul_P~P=^C%i_#gU`o_*oQcgXuljt#E>^bs?Jtq7jLp_BH6u; zyeU-AEt5SlL0)P>7p;OWdk1+Z23@NQx;`3ob2*5Fq5MB7R1qQHW2n@KmlU2k8P}g; zs7r|;d00$1#7!B~?K?p#EhYBnQ{n$b+1NYCm(+ z>pW$^7UE{FRMM0V@Gt;&bnO1%gZLj~s4l;bq4vlA5<>;ijPHRB00Q`;VM60U0|(i7 zVm;y~ZYr_KP4SU`pPPD;d;-*DEaZ`M9?#g#=T4k8>dKtC*5(0(=sSc;yUrC`0 zrZ%akKC(`2y_(vdl-gOJ+C7%qyOP>3m^P@MHe{VPd^K$}DQ&zyZE`Gasv}MA65?>K z`Ck&MV;M{886T}PK3~n)uw&V&xggaT?bu4K0+(`(Gi|QtMrqhD?$1 zKMB$mk--J{*xhco5L#pl>JWAP`zX;XEte?2)uZ8MSja+k^UkTOmT>I5r;x9t= zTApiio_j;y&q8%I_XA(b1!)x6Am0cMTnWk7sL8*+n(y(mHnjxYe~&i(SM^Xsnhu0L z(g}Yg{~t`E&dkhgR#E?-N}_J|P~H6e0|LoGl;Duiu<(dTYE*PgY+QUoVp4KSYFc^* zEi)@SCpRy@ps=X;_MMW_vhs?`s_L5By84E@jrZ<9pg(MCe$>+1*51+C)!ozE*FP}$ zc<9OS$mrPk#N^YdXN>35GqZCq<`-TrzIy%U?K|fCr4JuJeg3k%vbwhZ_1gvjL5UcY z(H{CE_~dNc%QBmT&=N+e2IW~Te^6~&E>c09Ch$xOLzC!svjohf0Q7m@uY0IW<+cQm zk!|_|=Y@mi2n9Hqwa;&!_GFx#+7}*-rCDlLTg0 zM*9Z&Z2^HGNE{jzT65lflXn{s4KScBo(2%S92T(w!5?)s;K8G?6fk}l2@wZa1-D0P zb6L+u8Rq@I9;%DbRSjzj48ILe$pkQ-L~$^bE}dzjFAmD_k|X8DldsF=WrDSEY1m6T z?%O$Rlhu8(Dao?*oPF*e0V;yPCK=*~1|(sJT~?iA3mTAkQotF&8BiATk_YN{H57~8 z2Loq{$V-NdOfrUjM?QWKxeM_2(r}fy+5{BP61|Efvzm<9N!k7+b_H?%^$>%swKA7y*pU#r6piDWKQV73B{_+vE}L_(h9+ zsM>t$hunan75Ca7vW)V2^{!k|8qj28CxPqkxhephH1^qKG89FxXM(A~pxSn|Jz`PGCV{h-%ooFfO!C!i) zZleT6;K1qBf!Pk8Y1MoFRy`{t{;}8hT#k7r-wr*ds>R z;j)JPY4iQ-O>tMS#%FnFIPM(v`MLfgPDY@D}blDt0aiHzhUg+w5~fs6Ve_Dz1a%RwguzKaJqcdwA6eLjO8Q&;6uc zV!W$+N~&&al`cRay_P336x0w^54rtY#rR6MfP80K`ov)pfLQ10`%8%P|2A#vP+;*Q z!kg!TovRn#h>ithv<11kHk<=#f|W-4yAD}-p%aJ3rFX;|AWTi7&lWjI_|)9qkF2oC zJC?q!Ww?GOiHeribGYYjL2>{3@|KVDL^;^{kP&(hPuBl>5A`g%c>1h4mFoQhRIJyh zLsE=3(MvoaYDB3wbmy_)!{hG>@&Nj;toeb{H`i6lbk3vF{pfzrvZ{0rE`Owc+fsz3 zjGL)HQ8J0zFw*78yhO0paS!8=IJVEEu3gy%V2{zHip06(ubq!g@WUj-CJVb2ZVVcn z=f!s?t0UvYbqY`M(owz`xI&=vZG6#byZM8EPn)KYyDgp=Tk^YFydT*UAcVSo@`=Jm zdTvK>UM3OJ-(yobJbZEKIJzlJK8LH{oUKaX7IuA3Vm*Xng?t;sc5RiFoIokLvs|I7mc2Ej-L-BjG#N zTfCh6lIw%~gf2;``hJh?igNvgR^gWn*f9TwZRY8MKKFArV%^RG@((9>mh(1TmIFdB zA0q0G(#xcEd3x}l>_Uyr^jAB_;r%b{W)Jej2H|!AlBnRVC*Ig!O;)5fY97{qROs(U zqYqJ_#BD{x9YMIvJv+O*y90d>zke@fH1FEB#gqg-0mFx%7#a|YDzxqlbpC?LblUdJ z(iBoKxxCH6->$?$9q)H0bO&!fXts18tof+=jjjV5a~Okjm-R9>6e#!$ukRBqly9;5 z=AKuvzEW*0v0-s~A}I=)Dlu8Hvla9uI<;_%v36B#1!j0TaY7kGUX4X zO*J`xxCNc!EdR2hG-dmY@159fiS$>6PHoU8mCN~2>gT?Rr5dx*j$r6Wzy_A);tpld znuBKUU#NQ%jJQ6t`{uIHPLf&g!5Gf#r$`MG_rx^vDk4@+H~l4T>HrzhN3FC6J^hgR z!}?_y3Ap1v)XOI_cb=QH+Im2m55jy4BUaSp+Gj>U2qLGVwF;5P$G<%MfvMNN{w26( zxh3E1M!_~3h!cBeho#Jm%RU^L_|Xr0be*pySYBH`ghaR^PzXnI*7Y73F;)Bqb#Rv# zFJL}=-ia)FKl&22$;pAFB37fLeMn>CptM+>N|2-Z<)b8BziSlQwXoA0cKnI(LUU=D z=YiCOTkuizO)>%s7<7O1y(;CUd1eh5*Uw%j*miQ%E10aIGZ}cmI)T6gXGaL3L3V@GkiZ2c7z@wLITS$ zV178*5(2EqhyrA(Q1u8I95jFik-q!s>;4#nV31jTmYV6I|#K`*3gWBb0xQA@K8$L!wHqw|BPYzDHk3^~`K?CR!X)j}NvIv@Bf=+#cbyCb-Lxd|c z5z`+Pbv53tETK&np^mZ3GXQ2NFsa*5%VrFO*tveni}Ioqy-YguuR$=bwuP?3ZG53KXxNhP94E*l&zqF zC`^J=WMl2AG5Td;@zw|zW~wq0>_(6E$K_y1nW`j^I6a*VfYm!dfu!6;*KGTS1Z6zJ zEEp_CPZQEWC{kl&=oy|@(+;eFtyWV@f*J;0R!aflGVomMB!ZLiEuvTTeJ7wA7UVJAf|i8A}!OgdD+^Q*2=Si4{9i;$+!j&ghDJu5cSP zOy9Fy6$erFglGUzLmbS$EG(1(doEOIloF@ZU(!K|I4oSPUI;b@DpU@K+7h9Pq?n^? zF=BX#F;En^d3KCf+xdVkk+APQRCcI>Pj!AGOoJYAMz~TOfI1$IyW~-0d$-m_KHt+P z#FY;9cvXA&F4&wIoivNRhM zeZW-V`aCiCF+5b61ipB;N`wGb!a=X~yk zQekzW_usDF?~w&-;9(Yb*_l2MmQx$vVfO3^kR2Z3*xxjn0x!K*;fRm5z8ZJAsA^|Yqur}V zVWE#Mb;WF7t)E`ZYtU@gT5sV<<+{e+>a5Uk!1H0|-4q*o)W^iEmnn~W>L2YW26xw2 zkyjqr9d6^|XgfLq(xru8X>7S<++c6pN)CIlzjoW{2vtGlxW!tRXpf=Pj5jvE} zfSIl2wOjqHP4)XBHh8!#wW=;PYc~hf77N$M*H$!kUV7c>1auh~LJU%34x=}6>J%V0 zWv#o>-5$mfC;MTJIH-ng%V&B!Gwl96q3FGB-x1A{3Ty+`DDkB_JrDB#!cXIxV!L6-31IG>w-vE*v9I$m z`$}3g;${#Teq}Un(E)?taNSjqMHuKb9=;o>&?p0M7af$_8SX9H{O(Mj{=R!J@coDG z4H?L$u81_?uEGq0@9f7%9vI6B41UrR4A*;^i1Cb*!ls&%K%zi}sArscM@~dTfy;l> zrmpx}@o{!=;<-JRwC{c%wgEQ=Gf9K15}yERPqdTcTx%hzCTW#A@*S_=h) zOaFu)9pw2n=GM+cb8Jibjgi|$dGhiI!s^JZRTs2#;(l6wJ!iu8+p&0hIN|SV(*|(C z&R9?y9CxC>MJ8N3sEk_$r#S82Vr4^Uu#n? z=#n5qr=!hS4pBFmc}H%nmU9Y&ozgRzT6{Wv=>(Vd%W3@^8wI7fhS21qJ-~qP;xnwx zvoGPGJ732)x}Pm377Zf6vE7jt3gO;ipp(=B>q5rYB)CZ9GxL(X+N78_9kH+!hCpx2 zbkD4H((Lfl)HxK`yQp?5{Enf|b6KuAu1SN#5bLTI8j zZfM?T+?x65q;28eT@BtwFhwES4MW)A!psig+0}5+`q`M{%urdmR0*HB2=_!6V!fLu z_%@QdxeQ=Pgdbss7N>1qW#NWIs`dTb;p$M`RQUM9B+-=ilQumLEcUEKYFk%&yn0%` z|JAvY>G>X%6(D=H4cQuKpU^))db45<*Djj6krjXL{^)2zf>G<^hJ`F2h@%_4P^h2@ zQ~b$pWIE}YIkjNDXT&iHM)OG0ecfuB2DfL@^P`exUyC$r41 z7i*EnR^QBp-cjsshH`Y&Pk@@&V%~@}w5G)zh%D?;|7acj@xaRuMu%$;KBzL-_;^6J z<*3>4@dwjIo=XrKA=4YlPcj_7KcDSWYjU&eoYrR>($Csd-}Cd34}{VS(g0wF#C$yx z^e_>6YWq9`3%5qY+#~N_w(gg>yOA|<7n-LI8(#+75c4y;M^ayR?5DvT(XcOfmWztR zO|jv2xLjwEx8=;GoW7bno=eP6iqf7Q$g0%`(hknNpX2Zivudq~47laVT6AySoBFqh zbQ*Z6{U{iLU}^w*$I4Z(ZeP6=!a~B4O5F`AJ6vIgtm+#yVn-Zv6UF?v zW=oNAGYp9gR&v`%4e|?!OG*w>baN96)OR!9uBe0&bF-1(@9HL%lJaY9>PpyM-qcLp z??wo8b=nZYSi6abId+@JsJf{ITDoDfjt8o_5iw;!1CL`c$0r`We)GEb-utDxw?8l1 zE`RveXaxXhTqXm`UbO{bOKlCH11jsst$yjDI!p4f$vD;}3M!X$Vs?wVc@G?M?T|fZ zqvovWxF-a~5hhZ-F76V7;1Er`E~a!UP;9&XBUkMpUrK_awV%T>#!E6!$Fzz#m9!TjU?PLHQs@oZBOx@NC-|*8c2qr9rWMPeb)9#{nhpjj*}NA_KZ>(YmGcnbWe%vH9f7n!QoDif6HZQem7s{U42?1GhGY-V(; zolKaX>4EZ71r9r{NlVF^S@dcI_TI=1hyp^JKNw7B-B+6K!5MHPDctzxVqNN4HdL zutXg_EA5GZHUpYeM8DrT-0x$h?7*jjdWt`)|YnAr8iO`78XRIDM|4V zuEnysfE_2-V^KT9-=3PyyZKVjSs^+{%~4FcmDzL&)6VuNDbV|hbyy(IsmGd}T69pX zC@*^^e?76?P3zNlZK}!)dWf>Qc$vbbECPm}XmV8y*l*3zF5~b)6Po%vQW+{L)zPeuU|ADX$dpYqO=6I7t?m=D|`}l%-I8HhGTAfy9?@j zA_MlGaw_x8Z~>mHqjwi%E3^a!bnb;yOI^c5b#@GP2p(>~^c!t@ap0nY3p&JCP*TIi z9Qh?YQvv<%{5=_UdFY)Mvmord;3e-jt84qV*EMPGd^1@#DP|k-Pzi1=@${0|Nj{0o z;>6pH9b+kvN>z>p8|+w=mYaRs`=zoVWmgjc7>5FIvuWY;>n(6A+zhqUT(dt&yHf`_xw80^g6<7 zT~o&6^2j4n{eD7E5@8CWIxuStWbe5rkeyUt_G^lM@>PTyeu6fKBd%$M>v>4N`EzJs zzaPaBFR-sj{6gRYNuW`={BA5=Qbrcab1u$cJCp}yjP_@8`EJRjWk*~Y3Qg>!1@dI^ zMn(*R?_8(B6!kzB&HY)xg>u06BOHLpj3^j0Q@n3`_%1I`2=B{;$<$>*b~wjgaeb2C zy>Tz-+DCbQcrS_UCUtf{7I%c7g5zEEKFwhaKs2HuWI~88toz~%_KtaE;q}Lq{gC7x z_JIgkPYGJETZg+J2iui;NJrTRJ35EdUwA45!dNUKp@M!q%k`#S!_C{Ba7AbG-fs&=2y$7{p6>_)_i&!EfrU6? z)wNIN}^~1V{_$4$m(N=F2_>6o(nCfc5NARVw$rp z7wO=(T&&V*WV7KoTTpB3KA@LMywzf|>r9nZWW=^OfrXb)KK&+v>kf0jTBbp)fKY|4CG)g}BSgJhg83IXi6nuP$5nh5p6NRUTPkX!r0sbZ*#=yNs# z`##5CK47wl_2DMj05hFKB8Sp7?d zDe*9Ay!TUz%$HJE*t=_%Tf(~cRzx%kWw%IbHw-F5Hk7(4Gc-Sj!@E|})zK`Jp)%^; z*Qn2{Q0;vRF$Qq6H-gpm$~yDn)8^?5VNiBOUi5;z*0Ed_aSNX}+tjoKhoXA*E2vYeVOdm*|R-PpNpMUMmKxX150m&#Atlm-shU$`v8@bX~W7r&zj zXEMYYf|4*?k?2k8HO#s3CVjc1xd;estGqX@x+8hu@~)4)hd38#KP1oJP}UBOe%q+o z6V&SH0NwL?`SI}aE1HeOdBdyVUHQqhD|#Euz@Vu@6!Uf*+KbSat~JMBD<|HK)P0=B2+M&f?%~_}T<;`?jF{fw zz}C6po)FemB^3e1^94u)wQIM4bAtap~cGh|aM2%j!s%A!^2T>V8akMk!{sWolThua@fb()y;;8I+AR_;n9;H^boimaqy* z0c^98Bn6aPwFN>pGzSOT6BXoXftGY$7d5!b5ri!-$8KD-vp0gkfDQ5S1;>gfT;y$w z5-@Yv?V1T&A8zC8$^HQ^Q~d1N#%*n2A-@+d;Y)2M;Fo4{USB!lMC=dF_4a$lwbd zTfZquW=W`JEvw`4$VW9BadR?A7d*_hY<~%n*R5aSRFaABg;eZgg->f*gZ3aN3#9VI>O$ynC}MR!2OI)-@i zyizA+B2#tvGa~UB{Rc{p=pUz_Pn!GqmX26zRJq`(Ro2+O)95L>bM7dC5y(EKD z?<@LISA5vGT(O7s@QCglm@!&HVO%Xuu`q62t>4;KoDNlWPaCa=R^dr9B)Du=&K&VX zxRTC6p&xE&_y*I8+Tr@fRE=zE3P-&4F?tG?kQ-2S%nFs!>V4sp^;to-yuLiHXir>* zlD00@dvCVM!{y9CCZy6c|8Hnh`?B4SEpLp>i<%{fa7%1kEE7=}LMni)F<9#czf!EZ zui^*LLvqllA=HrsmLiI6N9-w-JF~l`=<4)FQE7SMhdr zL2g(``JMpbEm=Z`vP5eSwi=xbrh5@JytZ8Q+}_Eton+G;T!BkL4F6dV^&@25F=wj> zyR^^29aesOq6PO0pF9B4572=e%`^pjAZ$(`Z10Myj3|q`5x!ZLV~B)0=Nj%Vr6V-h z4m!f(l(;J4kVg(o`9hjWm!=?lMvc2}5lz`w&P>$vXKm_8+_D7n4G!56wurnQboyox z;^S30M=CUTM9$=i_dc{qK>mdQ_kvT7x+j_V%F4=MQhjph#l-VEO^}Ab(^cyaFFgm9 zN0!yrlPcCnHg-JoRRaVPlhKV5plzc<;rKz23>u=ldH;Vti2qdvnM#Sw%;CN$5C*_A zvbkC+jxI{#t`*-UG^Yg}&qDb@XQ)~~^-z_l$DL_gkCWi(L&$pv4c-YQzfZo=9fa^w z%t*d;v*W$B#K#-nm-owRh$`-R@Jvx>eX;b}=iII=uamUB@Y_~dc&_@kdT}(^)f0Zi zqkQ@VyI*zaJIljI7)7dhZ)ns;t&yNScOZk4*}{{MLn(9oC*iFEP}#=iO?2 zc|Px>QT3)>{@n7dE+I^i7>9m^TJ@77)jMjCoXZGJ_$BpWa`W0X%kxfw1)2uN?2iI# zKDlfNjjt4?VLcsryoT{GalC|-IU)=XJ8A(Y3d&=TMSoLkom$d##MQV=-nyy9E~VA} z>T<$`m7`4!6*xkE+B@7J+n0&)GYL_jJ}NG|{f7 z)Y~5E^XMEAG_l9Spw|1I$e-zWznk}PtvI0QpjuIVrt8Mzu5^W-pRW-Gyhzh z;vpAQ*|X_poH!q;cPJcmDg^ZY#FsYsRv=q@J(0g~W(8H#GWJ z*8A4O`VZai?^NVdF53XdwfAQaptc?!c=x*hrtiSO?SU%hL++Uv5j$>Gv`@Ll;Jw!a z*L??X+#YQ3gdgr2qyk_)TCmTFJhimP8uuP+J$=0Y>tp<1>!IRd4(mm_0Q9;R$L1Cw zob$=9uT_dM2W@v2nTiHEq_O+*G^vN2=MIOelE9^3PRe*f45%_*8qe$=L(a6ji(d32HU`u9BxmwcERXe{K-K(?NwI1!h0qx{yY ztn2z%&K>sSj3=Wj;I1R%w{Nfup(OQ4V|TvB9ubk;!R)F`oA?m5(d2!WTOJ4H`zouF zJJGPSbNdqb+2P6Y4jWxZZ0FZWf3ITjzJt5*(BpV#S6~!BIM$ywVLK@#O6`66H8$n0 zgbZn7=Ii81Jj{Rr;jsgYQGNUSG2eHDHhZWHSo0U56VX$5>{=%%&%TKUzgidT-N!(r zi`f$=bKVzLq%jDhad)Eh(Ko}#dxT`Dll=Fe-#h|0^n7B}%`ft8Y|ASlw*Av9mW(?N z7@4W*8;WpK9ORxTy9@wLe=~h*-;*^oL^^yX?aFiQ(V6}GXB$ZN_6*qOCXEvAsag$q zb0frt$W~&An9Gb4u^HodI&fYg-m^ydVu@j212d>V`j$K>h)EeYe{bD>U+XU96qh&z{Hs& zos|c!ku5gBsyEl_aI5|;YbpDo1E}@oLluuo*~>~1Mwi#SPriK#d({)kZu&9EZ~NEb zXA6c*(EeLr-yBDZm90&_8*m^(q=_JDBIKpOx28)^7f-T-pOrp&_U+~w5C`4WPzPZD zZ4cGj?XL{lrdn}>fz3hNY!K(4dZ<>yXbuzt#lbC5SpMUnZR`%e*0%kw%|8v=mY_ko zPo99d90tA*+D1C?A%=6e#CZUklOGz32F}jl zCti!)EbLrT%Cb1~wok7%TE;L6MJ<#lh+c9VP z{M|j&aoSqyLt(x1EzpIYY$rE8hJz3dGSnzn1)o~^Ok7L9EN&%~T#?BbId;1Nf{*** zJ*aJ;(W95A{hSz%!zV9d2K}td#FijHXzZbo^_OO%<>Mls26HIl3#MBxX)>h12x+j4+&&W9r^Uv} zqDndvE-AvWt(@l%cyRQFhatubnC-iE=5c~})bJ`CM?erP%_@9m2De^NqMio|&K;$R z3xnDMtIkKa3pt4#_7$xP82I^Y_;!E~UqO2+5)SWigEkn1+7K*Og69kKDV zIgr8ZLzbw=rWcjdm0Pkn?XHEp4iBj!jonOdB_kb<4#g?X$^s~(p{X)-zR4w+WTuc& z0+Ix_Lik(Mc0MV&cB$BkMn6=t^~v1k(w**352sZa(yO7d<}M&YTa|dHV!sR5(~hK*I7_ zPEEzTlDxa|mv(z+YIZysA1<@85r7#p9x59QI5tSZoagRtUG9vhC*#T*T#6q|PpKY0 z{LlhJk`j@1wA_+=u*_i#hcw(njZ@@YNYLXEm8B4!!!2IGb$Qzm_6=rp0#X;`Vmu=> zG1}_rdtORsGMdLfGEn+v(zeikm7|3tEySVn=#|(~Ky9Ui5V0rrM&rF$n033j*K3)u z)JJnWSGtHb;he2LCp?G<Vb{1t7=FN|QT zb?dT-5HxvdG!Gp#sjA9T<>WIiF20w_RbBAvJOD}@e{7<4X;Hst(L&X~$LL8x@o5u} zlSB}>ysNt8O3y2MO#lojsV)mLdF?2?Hrj4lQ<2c~+C_71tUscrD&ORd`-!#j(cYSx z+MYM(eAXryT(xy=CU3n`)+QHBYa2#--d?`D_LLb>+c<;dKVWtgdNmFO#gf{u~`y_h^Ud`;Zgs)0n=xR@L735kBiPSnm4vL#9j7 zDeJQ`X7!zxy-V?T*XMAN_1&jUKO|4AzYyPeO6t9G{{z*_Pk<*;xi{!7$2N~zK7RTj z5McT-57NAF+p|oz_2tK+yVLcM0@848x`TB6&^1hz{r! zMCXjn-~9jByU(zu_BG-EE0q8d0ty5K485vB*n-%CAYF}$N)sa>5Nx3+p&CjGJrpUS zghnRKXN6)B_c7gKGY^wSEJ z2Z0v0`y`^%ud7QgFW=8nlz0&8CvoyJr$k{wG9dZKyk{O`wjfkOSl2@hPYoFR(eb^Y zDOB10f=HgJa&bL=bMd1Lu#NA9ueIGA5b}8hy~=`yl~Fc#ljt>14*0WQM|3 zJ6H8*^=N&=&FyzCMt)>@^5-hgYJK%{`AKAC6^p#HxqEa#LQs!>^SlN{2uou@U+gF% zO%&uPMQoXZLWfHlg>SMCmnMg!Gs5MX!sSQ96_>*?=m=$_2vz$CL2as;5ux1_p*tF} zZ8-vqj@)4sY4HD4ZAy)Vdtl+uyyDSH@irc?-*_O|#dujRbj<$b**`XzI&z^F7^q1N zM2-ZK;X-!;AYnEjjsr>5fOh`V4WFAI8*}~ zHT=7tbORSakhUJdfkZKu!h;r@NeKxxki*=BU3iEI6Qaci$;<pf4Xmo?tm+k3jciu!uB>{8tcJj>rp&CC z=B&2o|3+IShV zK#W+0#w~?={(c{-c~Frh>+klV{wuZVSkad+na1iOPc%wC!+>+%B_FPrc#oBs{3R)M z`M*j^BO@c@;^I#sB^Tx|w|T2#Z@blTe5uC+^qcSuEc=tp+S zQaK9rE=+W{>OY#47Cbms#64Ne_2zQ9B@dm-AD^js>|goBz4~cz&C}58XWq9)F4c~Z z>PP+SN23~_M>mcIHopiRA0Ka@h-#lm>6nb}nu_b5%IfAN-g%jN=VgB1bmm`^(!%?1 zZgOYKAJ5%*_O@>1UHj;}#&_@DjV-iIe(0I}Fv$DZ_wwVwt53s!zr%ETacSn$@|$m~ z@4v4uuB?6I^Z9~4@c;6)o|OKlFECwCO4pOp^`vw?DfJG?1dqFGnNYrfIJU=N&j#;{ z^m_Y8YEFS`4EhdI9$QVTdIsN@H3I7Wa&G@XF_DY4u^S5F+*dteWtrS--gl!~%`_CY zq8_@q+&C?Q_fm>@UbboSa!|2u)QstKF^VJQNf%3YXr2cC#CC2>3UMS zo|OIE?;s@U1xFPE!=Y-z7*)Y&RHI5dGy<*uO%w3^Z3M;C$N`aD~|kpYcCY^G$;34 zb@0#EPM23l`VU`CqEF17JJLGl8Fn@AxHA-IZ*un(n<`6SLvZXEL;)M7Lcj@Kz1bv9 z!+z35Db1P>92g8{!z!vcLQ;9)jmDE#8;|UP6Bl<6c`|8=YkN1O^aJQe7+Ozh7gG92 zq_lQ26oZCra2{CM^oR>e-mJ7i6J@64vKTOqDEQWg>OpIsSz#TQ|H9h8Yj*EcxY7j+ zD3KDO2js0jeF6J=GjZ>I&oe)gruHqN@!{_dCI*?L0zV>9`#!~XetzZoOTpi;n=jLg zg7w9MgtrLT%6crVWpj^^3kLp~3)#+wIa0s|7H}7h&#LnD3I<6dh#FhQ)Y7huE+yN#UCrptl+badh}bO`in|zYpml5tKl|LJ%X% zAR`7e&mxG1f>?VY+zALM8BSyi7CHzh){URpWP%Z#RwGzb0;XYU&peT2CHM(m;1B^W zvJ8s=V1kVl=LiT3FQj@h+=T$YbqbL{4vJWWUGw52ZjaDBD3PvZjFQiqI zERh4+M?jpw%S75U?2JOtOnR`TkJD&a`chawCGgQK-GvlTL81RJu06o6Vb(=;L%-i3DTnw zW2ON9zxJ66nve*hvT2>8AbCQ#5hbcg15sy@K){DJOC+2i#~YW$Z)}X!9-)4u1ing# zf53$)mNH8ifiZN@U7`3#_K8pCd}5a9`CbTlHgX~$d_*Qlp)52@0y0Pr@?ayklwTwA zg7aj8z70U*${=v7;1iT&8it?R&xTM52!|#bFMug0oPc$pYVM-Vlu;#Dd^WAb5=lw2 zS~SH|@IF$|AY-$VZ2SSMWVcM%-O=l~#k5?ZB*mq8QQ?#tPYEIl^41G}c_mF#6Y&<2 zf~G*`(&KFLslNt7saRO3Fcjy7+)juwXF^X~F^@ctIqHz!Wf*etc}A{GlEZV76A>wG zb>&QS5Xn9`X&$D-gu7!9p1YEX_+%oRY=n-ESY%ktCHIlSPBBBtvXCTVthh|(oELT5 zAOpvN&h>{557JSF!SVxvQ31$C4Tz~#LiT7DaWN~40goVM?U7+24OwUGGZh0^Xil~p z3O2+G6fxopmS61a+gfdk!ZFHtBH z;fjqDDT5u1hGW=Z8XJb^Bx?wMb~u;^k1FH{GowWIkdZrZLB2xQ<1%1)W-vrMRC+Y3 zv@wr}N3yMQzr6{2DxCRX&PP8SmF8j{Y!J>elZG#DDJ$Hq?UxvcoM`k(6e-_mST5mNE|qkxpTh?wc_UA) zh;3Oc&wN{s_AE~cs*vd~i*LDFHCAC)TJhygMQzK~8oA24ph~w>m4pUR+gN40+_7w& zs05Z+L8T=*r$W@zqpG}> zGgif@l&lY{)zI(GDh6^rYU(}TR8j#5DndJ1^@gcX&C7~Rm=jZe1T@d`LpYI%oErJr znl12Kam;OyMHH#tAOn#8=`9>v3UlYcEUln+6exjH$Csf%mib~EG>b}%sQtV;oKF2} zW5l#J0^WM@+kGR2V+Igz{<8MXCdX78Mz%4iA(s= zq(TQD#nO7nwX;GnMd7UdLCCDE4)N0NcbYY4iJ6;srMVK>jT+hLXwaHdNyqELyFyEhG|WyYH8Q!Z_bxbLJ1dehkB~gNE(|r`>K}G z@%TioY&ee5i^r#J3+Z@@sQpN)C580Zl`=nP1(z75m3gx7m(hlR2qF#={5)ejj~?=a z41Keg+7(9o zF3X4BuqslF?U>lrb14oKn9V}hu%OX7Z3}lJ;@FI!qV{ZfuNESGjdXK4ZV1zrL<}8j z9_F=z?+>I%Yo_X83O48rwI3Xc!?=|IRZQkyyAp&dy;rcig&ZD3ZcF&ID{p&T9(XnU zQ2_FzZs@{%?w2@_mNs0c=D|nVl&*Q7ic**@GgxGZEopntBm;i8HJ3;t+m!`LiGZJk z)Mle0f+ZV!GT>KBDTh|tCdZw5x(WpxNeJ#1QOdCPnXuYy)PSzI4ncV)qo3GPROcXZ zY&F0s=0QegKko&fF&+mBT`i>%hQSq~Z#4rB?xOX`lvuqNn?3cUWn_S>5lM_0_O^eT zXOnP4E1XS!@EM=%aq5x!ga@+b*@YYO*jN~+R-felTP(^znDY!tx=$qePMoSWv9FL# zuR&>y_TwO$MnUD4qf$y0X+!05p5qHlt?_4(w_iI=A*%MB;T zJSN}f6wZb6FTVRZ$(fiOg*SXXJ@q|wYWd>Sl^B0z5^Uw?6rjih@8dyTc<`{P&-X|n zd;ABPJh2Wjm3F`1KJhkv*dQwAfAZ){xsDAI34R+pUSd8VWpe#w6{poZko!^o0Y1~Z z`;e}}{#B#XJAR37&F};7cx5CeIvCeC@LDO#+s=trT2n@y7c;g5cuIXn8}* zMR=U{)nT^ygv}CxA&sRszS{}2bj8`IeZI8GIkMty zxg~IrCV!qBR-n+&$e5gMbed0}oDZm;m+PM|NSMtF^U=eWmWIua(S78x(KixiuMvIp zNHKL`vmk9B1II+NVgU&*jd*`|pO4|< z!ZWeiM`9HRo8OO!`8*f<5c*+A3Iv`y{DCYM_K^)4tNw8Q(uY3EhsAAk3m=L}_8-ZI zON^URR}*G4A41TdJ_aZjDe*o+ugz{!S`?m|)zd85kT`o7yC`wylk~@l!Ft)f^R>~h=)gKZw_4}+pc}x|A*S$O)C#rsU)0eHecB(#gJNdkgw^0(U zU*uf)7E+I`ae2JsF!ib9UqA0PQz-c=7xwi~($~ZLC%>Q2BklY)7k@M^@|#rd=iuIN z;=lG7Y`H)mo&#d{5TL4x^nguH1yda1nFDu zgRAw8Q(v1n6?7IEh3f5N#8B18{^`PPro==u zjYpFs2;=9R^-SL)4{`hdws8As!Oa7WYf|3Kl5nNXYdcG{=%~zr7|r7q&V#kZchkRG zN)CC&6|tySj?lAx zHPzp*t(j}vT>JX&X$8|?v530a2W->WhSEU4sr!Y;|5)Cl35B|TG-!9&8J=D3_ipmu zW&{QzZ*N`t$m@ySa-^ee<;&dT z$}=8R;|MI}5%}ACxc%u;o0Hbu}y5hw#oJ8T99W7kdpq9%3{d$9JZc+mL|&21MY zprWdASg`Q1`U93X=boLv>E)S`f3riX_85ZSPTYhs0YPOhEOsF@susJEh6{^5C=1}j(cv-)7Y21J0;?e|p_Quj=)G}CqiXyaanHME#7gw^@7((2W>RgE83-_`VQ`>{|z za_PtWrrGKrAKI2a{P@Tb+P=2fD|316(}2d!HI2{1A6*!8OXl0#$x*odKfjH73Q@j2 z5BL~*dHmY;U&}M;mw&C4K*5003RL*iKW5iBIXStxxm~z$!QbCMI5_w}X4eSdn&=+2 zxZVTty*j5|c(MaO2+cn3$EG2Lv7zP-(je@ui z*hdc}nus~fBw3Ywf->FHtV%-fgK$7%6cm`jxUsY}$UZ|3C84%sfdDHGt2Lt}0SI?t z0>@-Mvj!Oq;iKsokYqxrQvN^wBeIaZBI#go%}SiBT7!B z>daGJes1w#bV)t|!oBBM_w1Sf=M$*zIGrLfuANZ5?lsKXYn8PahQ08IxV!qBA-NvM zG97QD&JtYJ=>x@9>vqk$U9)c21T9+#BfJnI2v=G`$UPV#TRD{ZXq&GjVE4?&ji%6% ze1+NVc9ybJso{W$h4G7n>Pk&J>8^HHk5NqjAGK?+VDw^aISPg*LDub>b-U&tvTIa0 zxikR0YY;@G6ufWfW{M^!j}(}*e+6v6nq3_Y+CoS}qe_UJ zJcSzY47x~fk&VW}PC6n?nJKTe5YQM9fmiHO0Sn1wQer?ydGTc93~go#26tTni?HI9 zywbeB)w^`JR@!;1V!^}b&(Y-()~o<;L_!S65r;5!ERK6yocKI*F}jjYfVWs9+$ed` z6nJ|KoIuEmBvwV^U}zMWN=Y|xtlY>d$Z|ju2v8#{C|4LfEONu;Eo7RAG-mSum)kWe z0Mymy`jzty&aI6eI;EuG7%vX={JTb*Rbp z2*Kj5@htewpjua4szc!Qs}=QIGuuwT>y3-;y>d5Y%WCuV+me>uJ$u&fAhX4SMEl%z zTlSN>@>{bUIZ0nL3g)wV?=|;oRDy?_>uD>Ur)~Phkp0JAfK1u(0d04#*tWPh_tD}y z=N7x@Z3AIHz&WC#g*u2qCpZxgHe>cW@@^QjVdgc>w9b0R#L zTNSuqoON>*T(AHe*6Md4LLHd}@*F6E2RpJ-GGGrszXzOMHDu0(IN}6T5%o>k1>2EO zXAImN*I3&&bY)@44Y+G*1+mOZu|e^RnsgzKHS2cGx?Qtw*R0z$|8LqgDsisEb}5EL zBkC#Qf1h15gg`1}ged8sqbhB2>Gd+9hcnXk93svxqkUCN4jis0Jj#bXlg!j$3Lzqvcmc=L>tSDf?fbn(F#?#?0`tTvkVmmNsR zwC|yrY;J6(?u&zNgdBArNPH@xxHWEI>T#XrA5YZY*_N%qH}NeuQ$UVq5r|YQjzH~@ z1`*=c?V5GFX5Fs&Pq1rTp}9@?)u!&e$F~xXs&OOao@T&w6%N20(2(CQJ%wy#hKmO2 zMd;My8>H|if*IWI6|t8wrFJi&RBi;r6(_y7LfMZxN0DwbxYe4!}QERCwsUBfZr`3{3{%QXpZRGE@qLd z&WA5Azu3GtAEwz<0MT$y4e)weI?QjU2YuC%g5Rgmh-w%2VX#M~7+A@-WKX2+x?S^M zW!LC~v1;HAPr4}1Oo=CyczhcJ3wO3Hglr2ehY;T4>>}UhAY`Wx$KBhgX1zGUe)~W$ zS3Y_m(#xL2h-~QQo=f)bL2ZWOFqGS2tKwv**t36VEHX*ldaysdX?g!}aNyRgX+@m|T#J`jzW4LKwk z*`k=cHK$c-66W-k*r0G64Sd@xwskbIARvCvDZ!8tEPn=O#)J~Muvg!u_L}THS4or0+=<2N7HS2cGzss&s z;Xn<&b4Ht?^*D+q1+G<4^qSxluYcg6+~rW?@Mw0$0NmV`XX1se-vqfwRZ z*uoK4$3s-d$z7r@HQOJb3LTkMIFc?euA=z1Mqc-cur9~}jd5s+xQVd~*ZTtMqR~gK zpsp0K8bN6bVn@05h3$7L0;l;EH)GDvtnM9eE>iQZ+>)v-)RW)3iaXszCt@JCc0_I7 zR3K+jpwM`*&-zUCx?S^MZP%!vz+NQy(et;aokSyU$9_CzbCg$Mh$B1TidZ$>f>YT& zk0Ck6s`nhTq;zEFvNRk>1YaeQX;SNL+^tf{-qBs%+akqv`aEJ5K}tyTTBF-fkKWd3 z5Zlv-*N6;xBH)`2#OIy)iYQr}VzTDVKMGl7KD&hMK2UEeRsM#UnaIj-1 z!30?y%<<%3Mtzf?ce`R6JjSnu?GaVm^WS9Gr~rCr4Qp>-xEuQTZm#YBtNdJcLdn`fw0GL(w)}9yL5Iwz*eRM~GvAw>`(NO9@%TOFO8uI^Ngh`#=PUOj(i z5Y;G9@zbXuv9k|fj89A^p52IeRwN{8KQ;gMU8-Gwzl!6+m#^Qx_Z$>@1{Q+M6v7DO zn=NbD`B;eqKXs&dAmd1sO072k(_s|qoTR?@_~^Z;Zf~haqxMGHs{JXNP8Sy~eDelV zvAYhJqqqM)X{+JDHupAxKu}^<80PtxQQ4ONKd@^OIXKnUr7?K5@E%nsJ4zE%Mr`?h zn1u8o;(~lF-gpB);tAarQN^ht1^3|RzE4qtFB0wr(f1r3I#1WcCHq3dMp&B53Q!I> z8V68(ZIuWJX+rb?rzqcpCezVw>vqk$UE`v-Zr5Pg_VMd>&AMH)ZrA*$*fsC1_1r-s zW{W1`{uS!}0l%6PO55WBwdxI6yH{ zi?TNR9vr#x5;HT$!dd7jrboGt1S#D86V_S|7RmnbRSca!jk6FzG~EB)u6aigZyoJF z{Jl2o=Yp5Ot{J)i{pObsqPS79r%Kge)wq`^+l<{$W2=`kuW`&AEUc%5jx2X?{M`0b2 zzrH1&>3XaGKw|X8ukTF{_QJor({y$7g|;(WhfNp33aT$fJaE0T>NkJ95#I-|PmKOj zDu8x&=U97WM2JP5Sw@XxM4U`U0YL_UAk0!0nUyQcE_ zLFX^6Bg)%(1U@hjr~^Q$q-!V%3L=AIZh>(@K}41*iKURw1PTNcE{zE1pM@Kf!sW?i zO`hK-dvOJ$h}98AM*ysv5fMrU@05vn$Bw`ng)Jln=-Ee_dqTG@N9=5hY+6#V;zZ&Q zJ_gLl`5HgLaGDN2P%w?9PXP%gv+PW+iPAyF+$i%>m?_rJ0SB$|ggP+Ed)Y9%Ti|^J z|KqsG$XNw_iKq+9VurY=Nsb?p0KIMqA#x#h8E^wK$(2JT65u1HFsURcj)3&yz-ngw zj04C1;`%;hfTk@Y|73t#7$9vNmGWZD#-F&IKzqzVaL!!Y4q6c80hZydbdfrt~BNp3t6 z6$|ccN*7>2<%n1hOn|~-rYn#1#pp7T1JlzGSKulb*dxT6QWQ9#FZOYLq*yf1Ppm(- zZ!y+uG+AJVG-hU{H1p*h>_f$xq9&e)97D$_7)511M`mduxTUc~o>(h_;irWN6AnGY z02R)$h(Hz*lXa%-dLR)okO7aiOsrwh=$Z(D4|I8%F7SawwGl>4`1MS>0VC6vl(N}A z7wrh!uF^WdE_?^?0MnrtycN_GkxUx<#$yC@5g6=j6*{Ps4C=)pMyyL zN`_Gk8~Qe-RR9Jp(l0YMZ&+moYUYRz+OBt zn;3rH-OPv`a>aMC#aYH|e_@7Vso!>hrNc~(SS%KoD{)$-4mPGmFw>OE^Y;hAYP}>b zvnmo+NaZov)YTY6VQ7wJWXx!E@(O9+i2qfiyjPj<9lTO%G-xHQvTqT#xU14{WU~hk z=724~PcK^(3O&Px3Dgc*di)&{9t)=8k&Ko^D!T-YEl`okt{Y&9 zWn30uLoyB$LL-T;DAv|e*p8ULVnb}C0trlG!t4le#JiFcI6ibxhSiUO;4-832Bm!5 zc}2A{=8ISIwG5cTih7fBj_>Z?yungEaAg><} z^TN~Ar(?ehWnFCs3(+G|L|F+MDcMJ1RT*LB&tr46SosbtG>O%k-f(s`(~Fre7${Z1 zl|^9MpJd%q831i5R9pXZr{e<{F4(Oc@!pyjH<%ihU*3OSV z#3na$>;hyDqrfXmnFMA{%N(pZknP~uo)ZkobIANch93?>iY=tqup)B2I%dyjV#WjL zFJaT^VV(AAb}gI@RmBKz_OsRoBB5N774odLShp>hqS>J}z*1ZgRa)zTw8>PUDPFv` z=phc8RDotz|ROij)di zyH`_w6PO$h-zo#?EaT_3h^7wrr_o4F5sOW~p}~WUi+_WdqS}P%YeP;fnI&0Dxpt4- zNXOSopJr!?&rXqsD;GkjAr2ZXVJQkk+<^?1mK5afxb67mpZwf~pG3bNr!H|J`?ujj#BfPwiTvzp+d|TZyb7r(J6XUpPNlmwg%TjL`H_ z)b`Um4v3B>ZA7EjWQ1mRuP{HOlGVVqfb|ng%Ngx* z?Xo_e2!G9Nu2xQf*Hi>S*(ig9#uWNnN*OlajcSys+k9;R>GRzx(7i0cfEj^zd_+!0 zP`3UKH_<{z716N$vMl+Zv3NH7#_D7A+5{&Msn(G#8`3iu`cUsut-d0p{H@qWg@Qn< zf{kk=$O~A?dH7`pRClb|cJUP`4Dm&_z>b|HIIgF8qRraFBz}rzRge>n0%naOzSCbK zk3(HupztRKWfz14rn%Wna)1yl!)jXK6+%Q|r&-ezd$TqvO!_2%{C;t^>Lwe6_4qw6 z(cC5WZKKQw*jwr zXO{WNWJFS8Ekt~Lm+}?t8~Q^Z#r(-S6+EhD&R0}Qbv+DC zxWg8ndpxv|Z#5Tc+{|96{79J>7b-q0+h9|be<$XB`d*4=5b{e?xDK{{wN3D!#mo7T z$2_mcpj}Km??Ih+$)B~GMaLjzG9#;E@MYWNGfh!A4$PGRjUD~ylfzQZqNtHZMm!!R z{LVlP{K-Hu3~T0NWGLkecX!>As+^YFUHikR&qi^f!mD375oE0c*I|D?8(8c4F2<2L z^GvQmH;cpjC05>iFK4bu--%y`0Y1p*T-IYkJqSreZq(M1x5^>11Rg~0MsVUq1er1B zJ(%*H&bVwJcZvxg|3nW+kDfRSce^PoJ(S{VLqB)zd(qDapM}8w@9sg~BRJiX5EA!K zG%7kEd2Fs(eVP;+))MRVI?0B5b$!d@k#4FhenOF(tq1392pW4|vKKowm>I+Erz$biFf6!> z_;=zwX^;-Ov%i)gUMRoVRxZmLytcM9UMg+`_B*m9-2;qu{v1l$roBOTH^ky(9l|69 zc<}&`>0FC$?~Z8EfH5>lq8s+^{7R=*VZRBCLJfrmnFG+)!lAQ z!m~4n1c$aMdY%c79zS{dY-IHL*zbq7(e}Xft5>4LouKK>#IvyJH$->3R3%;(gAQ*4XeL5BM~w-CG0AbcDYghx`zhnM*3GU%{70~5n{HExBC*uK?53P48TY75e^lAd&rM0e3jvS zckp&)*X9$w?jUe?aclGb&BoCxof~seTwWRH{7Y1G#sXsS2>C#*} z?GHvohu09SPCR0J-Mxa?dOJh^{N!-+7X#6aa~Df&Tk7W@|JYO6z+h!&wAgY~ebgsFRqor}Uqa@)CSJ2ZOx<&LJ4=fk$dIlJ8Z)2ani&N;z1OUO zDytyjg^8>qcjfLN6wkh-V`!Sw2Ak8orlXVMmLDTEtsix%(GDIv$&X9n+IJr^FEZ(k zPnxmp-Lc1&rvf*a8&!sD0!OJ@+B@fGQt$~^XHxANq+T29*<~J=rpPkD=_1w*$P5=( z_MOWM;S|h9M>D_IS)@L#(%&4P5lc<{JRlP}uzY|j%8*z zCl&AZX^so*8jefdyPC~#r-4Jg>NdhYeLLS()w!DKysv+9^8EYy0hVF6)X3F$?;CkE zoexbjCx297?64IJ)dSh@KC~`cn<@qG!^(bar(yNFTY9P^KJq(()>e%`l_oh7$6=1W ztuyT^=XDp#LjGObYtY%7y`WeJ-V6k@O0o3GED zgHQHd$-8>GR@T}kNBDFELjs1<({;8&=m8Q3LtIBL8Xm34J$c|>fSWTMbu??Qm9X;0 zV>B5DJH3OOj^mT4GX6_jS7t>eH-iK#;BWole;z)Qa>GG$(?r{lhZpD$ z%CHUJ-K`MC?nds#9Hm}hxa-LVnho#!xw;@8@y20SL+3B4s+4EpS3l2g=YMfNcys>` zS4?-uePzh&pC2jrg3p|qrsKYcb|Jp$4pq`BD?O zNA*#)h#?Ln;=hR1aP$f2?-7=`so9cuA_6WwQ-FTopq0IRJ}!0l8MU3^eLKgt#tH?R zNKB9$zd1)}_zxF893nII-#)wqX@idavK&ahHGtX9;5#d|mekqkeom5j%rqH;4OZVd zD0_WawWIcHqM7%x3qy!54T}|BtJTQlBL6X^?JpzkT0Wo3p0Pm2ELj(QdMH_6RlLpg z4mFGfB)0C6bJCWGCCK?5lW;ai+L3LGh$ivk$u9b(!Wvr_+%%qyv(VK{sK;5(66;HLRK2>sF;~=o%vRJChyYDk9YI~$w$VV-b1x|EhE$YK9`M>$BQ>dwk`O0+ zeUV`zGnN_bsO*lsZhuYt(*uU1zY?UqSE&B#!${aA$(V&QOQ~tD70$m16j`E*A9{4z z4rDHWQLBTEk3Ko{wPX}eRP|>c$u>Su&?_rD!lyej(vN|mdTQQn9ZaYS+O{0OH_G}7 zucXZJTZKeVqACu5uqsz{gTmK}QyMYhiii_cwQ{Bo=G^1;4XNr%a}`cm=b%>0KGml0 z6CFMUZDX0c=7!vv-*ilR)DHxLA$1oH<&aL^G5&sw8DHi8<75;NL%!FSP1r)sZ4wYg-*4Ca96VuflOWXly`kwT9Of0gNo3}GBj=nMI=9=& z7lAYAqDo4~0Fcdll$yJ;t$jX%KywucxR?>nctC;zF(4>BQ!74JvO57BwPbph0f0(& z0|?AgTWkn8F!8hVa)c+iH|Hkce4amb;1Z!@3AlXi)R!nIK@EToeDvDDc@CPr1qReD zVDeRRn~{Y24yuU{1Va!>{M0FNl1|^{7V$no1tf~5oK(b6h96z+R*`m-IG~#V@3-H2 zxLF7+eG3?4@%o&dFGmRj?zagh12b5TvK-C{^rUgXGJ7lKqUh^ zg<+2;`M}t(w?pnI+7mgL3%mpn!XA9*+l|a{BY_P@xQ~0eBVhL)mmT8WP|H4%q)%(6 z>PJr9i~1nU`ltsS7)X~_I0;;SojoUIoq5}?y8b{=NGgzV6(th3BP-BJ>d;LR1a42F zA9}xG;}*5Y(xaE`ptoMk^V2e)A~wYf*V+JJc@Aj5hfsApX!;=>XtuAR#J@6eu@=t? zwa=h{bEhd9o+}^_f&x}2y!SkA+ES?CuZ%qHY!=)axS9F&C<*|HW^ka=6pB}3G9a3r zC6}#uv3%o806^k~-*qf~)@6FjZ7Bw@(#y(U_l!Y}*g$430FvamkL+XJjz4MiHikfm ztdj=aeMVLi**CHKZX-}|h~lM;0l;NH6v3$GIc=VI;uiAL9YLDr@ZzR@Uz7_kvd%A|RQJzEj^U${KW#z?iANM`8UlMAd z%&d>V=qU6R41lb;7#0GQ0c2y~3Fev$)iAe@O8S-~Pv6E@4pok@08Q`W|DJN3d?Yfg}m*Zl56(7cEv~c zjJ%@+`Ek;(84X+B5|P|&RGS-GG@tong1?V=P|-gty#tjlh8uY?RkzliH>rLS_^A#g z8D3FcB7=pXF*Tzgaf?rrhv<0qoGIjob7K#Ze!>_I28_HBWJk4 zt0l_3CuN2k0&WC}W2u9sXVR%sD5EIzl*^kKAxmrzCr`+OQjZ%W;Y6G-K)AG{Gj~N=(M{v)#UyXZW z30!UTx5q@BN{@S}srP^!H-iZXTY`Eq{3n;=QTpVVk%SM+2~W*KHlkt;(h}a16R&K9 zA0$9_)_6&HBrckIEXBl?^vX|+S#%0*&V5?WYKYS(F* z%p}F=?VBB(6;_f|=eM0z2Q)L2wVRW*qr=ozlCiQWJ9ecQIHV*{;2v1Gad}Goxs-kL zx+n1vMJC8B6Le@X#Zp%LEFLuR2I0tsT410iH4r%xNQMjD34nyzfH)2$Q3GGb4zh0Wa5xEjb|?DbuEhzS#-#RgrQ2deL6Xubi2@lX&7B#Hw` zbAX@Up-2=+tOiKimDx}(ecfS$8w%Q%3E5QxIYfYML%}v@Ws;fRb*41Dgv&BzaC^c#t#~KoY=R79b=AV8I|lI1q}- zkXQt6Y9a<0Icv`qoKR3*99S6xR>i>fD~UaOo+na)^2CXVjiroW#Y*h~NjxAOl#gcR z%eCaokL4?_=40dvl#TfXs*VNfK?Rzu0_~Op-LZmgs|8rO!X3ti29AY0g9?pUg~ly~ zds+%4cOn4b(zH>LnPZW8P?068$hxJ-cC6^|Y7t(p_?U6Aon!Gyk$57i=u}Iw^H{OV zYB52s#MQXO-Lb?YNbtp3$C#1}VBB)XDP6?rV4>KUcFi&kj86`E9)IA8-gmESd}d;l>%?PW3`eaSJiD?)$3T*7gROC zsv2sk8XnWEy1!b*6{&hfsStI%k)ka2#QVk=O9Nouc(HneG<}0-Ts`AhJ-iX|2VUp` zcmleB;B^AR1JMMZkx1mmjT`0VIU|_I&_ii&YGaDNlJ3G674_x!znZx8$Ky`QtMc zkNqp3xK}?7u6Y_-{mlE;$fep5QvGOD{d50@=YfsSqnlrZj*pN3X%==)Mt4ocbx&n= z^Ahj8Ouh3mzi&G8)vH%SGdcHO7v6t!ll!|_c;i1}7OuSizPh-w_Kh#VM*s6(LVSwH zKfOIqNM!!?_B@l7m7SBD$IchbR27$$mX-fyrs`I0UH$Eb#-`?$*0%OPW~zF6@AUN# z4E}z5e*XdY;ose!Pw`$(znXddW_C_6Q?>B^pVG3cf83q}5D|I7OjTDTVxzY6z~5-u zTLpJw(Snhxfm?-OgUJf#Om1&?4iSpr=UF(VWRc1LfA2{d|H=A}yK&j1(EyT1ikh7SgU@C8V@Y7hfHsjVAvl4m&xhcSc`;D` zDqhIy`iJns64`$MFFYga%-R)>$zo}7iuQVc1(*oDkZMc>3Q&dFc&xgombD1uC{YB! z9Wapd)d8BfwNxr~fPo7Yw9i(&d>UF*qZIc@cpE_WHTF^oYyisG8NL;&1>M%7l)^}g zsIIcqT8w1@E5W7&Y>SRvzzl&o3uR0(ih9G*?wbV!=s(;o zBuW7{wR`CT8`z>VEE-gV)4T{~0{;>(yZ{F`ebDh!Ie@x&g=r>~{&{HY$gW(d*dF8z zFa;j`?f%48lp6E^_G~zqBL11u;xUZZ1NM7nKb#qqm^N&Ww7<795}6vZ<7&H(hd zeng;BmdtcB7k>Phr{F%}<<u-;iaaW!`IRB_LA(*dZ310c~A=WnGHpV;a(}gE#o_71zx~~UT$a1z% zRe$$j9JcHL(750>UgKLe*RUw2{a1L>evSOglI1&&DP5vtC)K9IdxEoMxh1ue*9R#c zHFD2zo(M@+SD+h&^K+O}{B2jY?Y7}bwI-Q*Qvgp{&>`@2jS%%+;*oty8*gp`g}=J< zKv^)Cp)};#@fqQBtwTv#>2pA}=fU$5R?e=O+7_mO8G^y&NdJVIm|!pITvdE>Mo9z? zywtkvped=eNx#fIaPQ^tNK1ezPh!wzZKBnl5L8>r5ZVP*7k3+&txTTlI{^R$;Q&YFV(G)-j1?(*HKY}q<%^KBOPzNm~RW9k}dmWacLk}p=ARS9mW+9zD= zSaLj9@y@g6SD8$dy4l%w?~@GgpYzv(-Cx`S3+6IVJ0uF+!CEWseS1FLiiqVs`ZCva z$og0>>Aw;$^h8xoU$kQ~$!|cFW)BWf^pX#mW$ah5W;9!QOPt(@!?nvw0Os!;qyW#b zwVgEwd_;01$|9x;>JJ=T{>uEdsRF|oJF@Gk{-GrP4pWKzD|m+k-f>8k<9iP^4H!7G zMN)P%<*$2`{_sgnk3h3v?yXCY5k&NL%nAW+dJR%Pw~R@{}x`z`p57>m;Xk*aF|o0ZYpsP_w977?Gf|U zAvrkr&FPtg5~+0THb7M*TE^+UOCU>+*|yupyth2kgUxxufD#ldo(AmqSs zn@^wAtY^LYHyx&conW|lrza2PlFA*1sP0SXd^g3ZQ->jC^<|$v)A{t)&ed!o`62F(~TH8 zwewEOJn*LRogFng@%q{$lD+nAfvAeBN1bl9{@;cd(g9&JDaOpWDlb6Gm5Dt=cj9fG zDsPEw4HQYl*6-iJ2b>q(cHcHBS4^{lw-nUmU`DlNTce;%#M3+8)w}7Y7og*h_BAPJQq?nu zcqRisilrky>-xCNy`IA5F^aBgCX2_rn)Wq29;Y{4$mp&7qQ+RL_OwByze;Ae_lgu> zNi-`E`Gb<>fd8TGD!U_tQ6VdHF7!B~-VeXevL4)*MfhVLT3I!2wjKHZ5nectx^f%x zO6dOYox`tOUyFUk*DI8Ks_`^~^d7xgm7yE>4_{;@z{GqyO4Gn38Sl*FHHh?!9#C2-z%Zd-Bz`S=?j!%M}*#_m+2(kbxf)Z zPO4@m)pjJ+&m`U3NMcJRH|Zp|I3~9RCwDNDA9N&lGn3r`LhdBkL!Fc+925h9#o(a5 z)q?H}XfXkzNPzX}q}Bsq7&It`55(+-7xEw&BDk0X#qf4OFC5dV(cs-b-xvmPzFN?N zpISl#VQ|p*j_GA+7={WQQBI#W2kDVv_C(N0G-R$CtVjdJ5MTnja3>CEjSjM*!EN}U z=?<_f53&{vR3w62aBvbXU4{pe-5q~hFy5MjP7&e9h#--VscPhO!bgND6$n>>OEc0j zY~V36%$k<9xsr+@z^A0LDFEon{Pf-0vw@W?5+Q4|11Q6RQV5VUiCOlXEVGZ9AH70u1UFtXiouubLcE4r|?YA~4qGvMaL@xar*FjpS9cO#9b z40jI!8sXpsDo}xzbIA|(SZW9KQ5pW#F^j^_@^s42VWnDxxHpfvS2vTU3lDWFie{uc5Q}0tIrH>%46!h+GwY#lNeZW^Co$I%0gj;- zMWYM!sJq2$kZ-zo1|8wv#GK8Jg6Ar`*Meh6u$kGcRSS5+CfJ+`1gpXoslXlc65%Ac zEEUKJ$sK3r!V4H=73P)AG(|q#6PJEyGi^#YYswG035S84;RAy}|Xi@31rr%3`Pm3D|5GM!FF@MZ|BHohdpz1f;ODWBBD5QmHl-D90#?=Kyt_;OA=7 z4i{8}V!^SzY(*|`Hl#F$2iojlo%e?ghXB=x6=OpM(xhArry?r^h^2v+XUa9ml_Xrn zt^svZxg;9E!f>HZ1r-kLJcM5*h6j12TooMx+8hL)%Lm5rvNu;ib~Ra$DzWr&VU*Y{~slxF?}576AL15BDb5d2^vHD$K_Pa9b)+ zmIS|rEx)(LQYy&r=R%FBSq8i!8B+aFL4`iQco#~|bB2Gdu6Vtc0yFD;l_xO7ozv(spaBln++PJYB6!)M}+7{-*&DJM^H347^H24@AVoilE z4Z~g0PzxUT7=aD71Rp0rY#2~4PUE^W%#H(2dQdbn3)JR8kkZw5Y_J@%$xNlm8V%b! zpS;Uy*$^OTOE3Wq)~13_k|88Ebfg3B!fH^}Z!trF-xs8;n75oROh3SYV7HrEm{21g z=vZfq&Ma_u&YtfKcO_QZUuspL=2){*-M3p$lA1_iN%mAQ)~OA*4Lr$i-T2sM@6vi> zyY+HGTa_csg4lM;vPBsUy|CRL>(XKN5#(Ff&Ia!6#@j*d$-6KY)PW5p^P3ZSkaPYW z1znv*bxjJ4j-*R_-2qSoG+2?_W-JXN@WD33YJ_e_v2<6{rH(1=gX(pt1pr*_(xE|y z*fBbrrMr7SwK=Myo`iJ|7IqJJb({Hhr2$|@fZ|d8hc7O5$S@%GfCp`)hYRx$UyVE* z?}FO{x?a0Hdf!zfM+MvRTe_%^zHC3@Z@0clhTC(}*TWvaM}mJ;BRA$B|GV_)H~%r{ zI48bK*n`mmDdG%m1N*!F!&6x>Acpe~9e7o$NN7z+wlcM$r>iI4r6;fucoO$eClTa8 zd{ijS5$x_&>n>VlXRFIRY2Mz&dv*ZhOnV zxKB?4yRu}GI_C>2i|cx|T>HUA!q+eLJJ)v}=Wz-O3mw=QKYYu0+fPQYeYh}?U42uW z-(&l_0cgO0Wg+ksEA_HWf3s!3OZU^S$-MYpoa6qO8&hk8Bq6KL@;ESRa9Ayf>o?#X`)*;>5vn1!-sZH za~+{jM-+)^uB;uXJO61x-P0)7@t0x3CH~{9^)1Uf$in&37{8J&{VD^$lrAi6y(qm` z8Mfe9mx8#{>)2T_S4u(WCd`fh3}Mx+WgQI#PHsUxiLkvzgHAXop2+P=s+k|XgW+;N zuhpH^P5Z0@7dXv@ea@QMD3|8+VTdpf4hS0tRwhnL{+)aloTn17=TrCdkxRmxqtA(- zv&8ly3%4`R@v+qmGcqs{K>g0_Fx6)#ENV_|z2qZm&&ob?i^!Ip1s>G{t?FjE);8R- zn!e$`TfC?@t|$8)#tNj=dP$cwfG2L zn1O^nV8s^>E$okg|&Or1L3t1M67+&+Q)YqG`+0lEm zuch3L<__=muZ?e*3hw5KT;R-nUc_lUUIt|2A3djS2^Lvamu-Qd5x|Lt3>!D*YQs+Y zn%kUk#N4WK?Xy8(;%K^+T54c8(3Sh9)q0}V4<@$u%KSoQYkvA_XZFA0*NC*rmqILJ zWcG=Yg^`a9GPtS@?Bb+qi(ufQ?wFty^}TP}`@m&{B>R0XX^DETh;hoTU_;5p=l34XukekU8@13hfrh^D7Ii;q%&9JU?I-3)}EGnUHUG|=~c2` z{vnlG!RomiWuh8`3g`t@`(pvnrwR|2_xD7G3@0*XFaP#QH zosGLmUmi{-^=frLj+5!avb%Z<=QowV^E->QZ)fYKXK9{l!TNnSD*R5At2wFrE-&-k4q8IBcu)`wN)d1h<#(XHs=F%5n>9cG9W?B36OJN z-|+ZQdmhAT=V4IIgiLbTvt-C!1axi?HO zXV=fh>fbUCEn#;&l$1d2xkTp9B~eo)k~^Z*GC`|9i4Sil*UCC>kmWopk$^0kM z{AM@WYAm7t?5NP4NN~ZIM{vYD=-=d}j%jbkperKn>f<@T%dm|%$?kvX>77MbCXUu(YYUPuW*sUI0R%{-Z_IPm3wA1zg3jH6B@z%KyU$o9S(>7SH^TW*ehVv5xj8N83~66e;e}wiG>W z%8}@E6pRtYF8ToE@qww06=2PVvg+7<}@fqOlxM^3u6GsvafnAWw7iGzroJ6+Op`Ne>d<;lmndX>}x`ccA*_{QBju#{ocY-ht-octLRZ;>BQQj(=9negaBNNzo{ zCBRyu#ej^##I1~{UP9&1J{gIoxfQ|hzkSM3unZjNIM_jD0;NdHaYP*ixc5BYc@O8| z@cgfz2m7EIoG6)LCU8e#-U5EuMI(Tqunx*6V7$^d%T&wrQ^m%zMLa1fhm1$hd^{os zv%Rl0TwWfqC?hh9Op4g82km+B?LB5~yf!HL0oy=k)K^6Tb&uJFslT`%&&Ny;17IC= z30bR~W9lSm`xNwKM{Ba{$A0LDeaNDr;dwhhlfjs1j{LMHEFPXm+lbv|s+22d5Q$vZ z+wF3l z(rRY}qqAPHgxeA2U}fshjknhP{GR^n=g#S-99(_VzfXg?aP5hvm3kB$gLkSh*O+H3 zD?1L>bfoNLp>yw(V>H+tX=};`Pw`sc#~?hOEpvok{?OyVK6<>~6puCGnuV)ea-;Q1 z&_u2op<*{UFSH!~uu~k_F{Yu9eqhHFg}053na4q_@77Fzr>>hQ|jM zNqq5!OSP_vCgId@AB)a=v6o$)dQJaWl|HyBDrYY!*t-JlbGvGVmDw$|#@gU?KUCD1 zUqy+>-S~5LkYyi`+ba6u@EEK?;@Pki#UEp^#9ch`gc{-ko=S#1JVe zH#sboYJu5vp!vpu7y65$O6>;BQACl0Sh&2C(UJ*}3FR85lFOCaTA~%NYCM{e@O*&9 zH^<&a1=LmCcWJlSTZMZEzBI|YLfHTcRx7}RbyZKtFyvx|9q7|=u5U>}Hcr)h2X7=} zgdziCGK6*Dmlko?YXDBHfe)RvKLf>PcB|OF1w}18#l#;{Oa9~ND@6$}4UXOb$p0zt zHpiPBPrANO!;l)MbaYbiq>d^>Rw_y#TnfHCTg>16GR;N0z|(gMRGfQW-{bbuC{=!!Nw z>$&6-6e!T%lu?RmyOw`Vhl^8z7~neb!cv*sxZIXXWXxLs!4L&Y?H%kMk;5XDf@bzw z#>6!A`u!r@h7VLW8z|^RJo8=ycWT8cH9_>!1>s{EQJSMyOE!v3)t$^!rM+-IjJ+}9 znP|~Fy8b9UK8+WMk3-*hQyK`4zeEm+nld&!eurc7OdQ+|6)`;O#<0d38XmYli7U73 z@G)8cEpxDsa1!ZiryOrGd!D}EjbrQOOR3Wt>K%d8 zfTYw_2wF3WWDm?PSO>|=qthUFn<8{X3mZV5iX!rFVU1T+WXCw!d8nlu(NbA?y||2p zF&N>UL99d{u4N&$7bUPp`!`IxFP;6;oau<8LOiAyHQ_rqnxzunQ9_KT##b$7saW>jkEjCJEA@K|; zB`uyzJ&a0Wa!sv@YR1^{g&vBL_T{Zj>p%f3U`zxK)G2e&2yVZoT;?GVSw?(bZqAFH z#{KT}srgLHYvU{sFl-EG9d9r4tk)~KVQy#4L=^ev9;_{P;AmvpZ&vt4qx$wk-Hxhf zpMEIYxCf2^utelzakL4hhZ8`DbC>{_>JQMl12I}DEcfy7cz1I=xa2T@;HmQyf*FQI zSce}|p7@X|c8p0Hx7u&WR^q%Hc=^OeQe)^@dZ_s1shfd61@Sp&*lSXLhrw@9MrYv{ zhf(2S&gF9I?+ohqPR2Q2iPUPzQJ==jNiu1-H;X>Ja>66BJ_nUuIiYNg*q*7Dkl(X? zoPOs*wYn+&^W}YQUPjl+7kVTi*O|T22Y{;`Z}CCj>w~%uYpg)4C7rTkZ*Zn0du{0e ztG^IDT9`JXbNc8{ocJd%*lN@NsH4DM#=VFmr$@Dw>_NS4_iK`vg!IJ2N_u9ua_ybgo_@C^P-p!X96oNHM z?j8v_@Ik1!gLUP5+xtd=zV7#KJ6~lNZ|9-yY|nolxAEzeeqBgC^Jp|pj#aqy{L8@4 z6)pN9(P%H8z>BBGFu1&NPW17XgrP_$-;S)W%*psoRYo8xgG{CLirX}L9Hlx&{@zpe zFWuwcy29D3*5r)0v9zl*{DY&MgY+JdJxwnCzN)k2sj4FZTTi7fvO~4bih&H!@L+UJ zv?-Zo8nA3?eDxfO8%3c(FnpjXTh#N1sRI=+?jfQT8?CC0s^mrn=L+(w!c~_*Kgwci zC8924uv{~^Yzrc|9bp}VIF}`2!@nBC*G`C%eQ4lm<0~|BmQ1Qb5aBovG-S2v;KKS| zEcqhjY;;c#Qr^Ul_H4(sy;NzY#SRY@R3j^*@G^aP)P$n6K)Dp>*(EvF*}8Wz%39)` z^7MRmT)=_YsdI3XYJ^!e!tf}ZcUa?Ayw1F*WLlX@J1KF78uvRdE`d(ZS%(6Y<&Q5Z z5T98URQU2tLq=NTY5Py1qGYw%#7_1e$DUI`Qu{0Uz&1zYxSYTcQxHvx9v_@?NHGc3 zjxaz&V_KjFWQZFJ+6oiLX{rORMjcs(?7IG}(ORiYgv?6RDNwW;+RVL=?m!h+BNEcZ zZ)2$D2SqM~DXL>Tpf%AT1#>8BJ(^4viN-1Banp{!jE?0(T&VhyH=x%_%>?f6Jy9x7 z=BS$~?!b@tBZfPq#(k83m0j3SIMn~dbJv%t%{m~UcuJe6?6{u%fT=9seRK?X3f~pen`Pd7yiaWLk#0`0ee>WC)7quXY8uJ zZ%y3sS&290@^PL=FDvVuB%!7cXT}(SZM0yvB-ogOG|U6WewO(VBE_&Eu*gSXk8vC!dol{urDhNmI#)r_@JMs@6d#0r8bQkaftd!3tN~e5OI0Uo7rY;mnSk z8|Z=_$<0IyUF&F+bkrD&)+*bM!x?u*Dc0d6l!-;tN6@JK@cTzxuDmlV)hU?d z(@NzHC}+<+_6dxp1zH@I1|qyB*P#vaio^H!W_K!k8{N663GV`UR(nH^SrnA&+F0zm zteMw%rD9N}gE705)lt?Pl@j>`gq?`i*naPSh8WqzLpG(^bSk(N8xhU*?DVti%11oP zFX{D$eO(fqY1+rH%Dg^&~^ArtVX+hml5z5RT%7p;8 z^1(@vBwPg67Iw|HG?pQF>?KYSQffJTSIi$C99n9a4M`EhM19N%fkI=4fOvnu!|yy5 zL&}(pvRiG({=)pDsVEy!m1xgXr)WVMotn;0iN_gm)44qb{$3`^zQmh{;(XyWJ;cA7 zgYieC`%Uh2h-j2MgjSi!?3>>E{hQ)b{@%OnJt&|92ImmxQO}T%)M^H1E8Rlxg}4(z zc&=Ow8wj~xm7@d?)q{txR_)?hJ!cCf1-;_*h!~#A`utPXkn~=bd?@Ou> zUSFmP2MKQRoGb9V;O{q2O;?u9CpqT$!2u(Yp+A)Wd3Of6S+T4x% z@u;+KP*dV9X>DoODsZcIkXkL~oPUMP6X-WjUF3FPEwv5PjzON&yKEUGad6*3ADP#J zQRX9X-P#z>GQohiZu|6{^LZVXA;7BPM5RoiA`f(m2U!5d9+j5+rWhA%Tl(#~cnudC zeI2FE1Mhih)gVxi5O$=Zrf}UB@`$6?aIKE=lBg%K19QRe`eN0#IZkpf>lC)?YMzD^ zQKYmzueR68K&}&gD;$Q(>s+XKPMx5@-TF~~?>*8|=sxJF!w=O2(?pU9^UcAsB$!m& z{rkyq&!7}6S|s1tP@52B&u0}=AG`9plP=g@MLr~UBy+3M zo^ilik3`FbBF*6P`f$S+39;y=wFJcZ;M=fxBhdje_PCKk_-&E*QsUik#---LN~v;- zd)s^h|2>F}hRQMU2*^E$`xo)F%zH}N=(*nBXZr5wW^DH&=(>px7sac%Glm55P zN=K6i)XD}T|5Szl$-8akTs%Ke>SXD%YfPoV1}?E0&6osgkK7%uA00S8I^o(kxwcna1gy-Dd%h;6&#ye(J=Wzq z<_;VeQyWj^Jo=u=SPmD`1_aX{jNb_ucLPp%s7)|-#aPRUA`Z}i6mNd893aaDo!~;; zxUdK=yqGKakPC9Fi~n{VCA%}Z&wpZ(HPLJ}$v{K#sM1qBFnPFML0->w;0p^TXj!HU!#rX|>f43(XFCLYlP69rH9JeO3O6ts>~5d}FDUz~pU z!fNpaRQ*L75$4^}r_38mB$_*sp+{~(2_7#=>f>=<%5rQC_p$pnwFk9ar~HKwcI0Q^ zVNm1!scVbyOIB_l%gm9nqVS0c2avVH4Yw3_+0%bGes;OxWNV z8IUYFvYIbxz}F!P;;=^~6l;wH! zJM#F>>mk?o-qzjoyZfz#_s5OjF@4_!YP>4uOV2=Wz97S2t%|D+4zm`U_GuQC2$e*BnzDxJo(9#AfAS4@HN;)$)ld#x zOzaO{n`(d_XMyCnKsENK(ZIggrzj2FhnJ6}P6r4*?EX9?1S=E=%?W==(5H^Efu#Xo zK$#uS0bs)T7q)D`r-NUu|MU&9!TOiK=1hJT`}O%xNwv+)Hqy3w*%Rmk~5L z?%obR!>tZ4u-+W{xsdo?{I$f1aTJ#Q1uHijba_qgRKa*5g1}x=NZ->~f)c*-E$_r^ z`@3(VtK9Pj!&+a3cth(A)bF_sc2X1HT{^y@L_ZNy8Y_6`w7@7bZzvN@b94;4D@f2f!|3x8yRV?B?6nfjA#)R zYR}fb;|!scwuq=1rbnuc^q)h`I7$)+7{t4%`KRd-t7MYZ42bf?n+P&-z^{L`44D2 zDiuBS0stD}?)qRkq}FCRhlT$FZA&D9@kk*UI$0DVh$xnj&nL97i3!n*7hqb; ziV#ofG(f!X`vmU02mK8=PI1~Cnjv(N1G=mU$i-M|lFQCX1)C-Tzb>wLdWW3{=U~*c zC8y|%f0^zPfFJuo-~;DH3;+;w0Pt3dtO*fl48xJa05>Y`S}AXR=P-Q-`elzmteSXq z*-8~ztysd%qNwMJ)MtSw)a{;w_rO_$_OU;d5178)6#Gz`vSsY6b;jZu5ddw`NeW8c zQUc@qI9T(xpac&%3P)tJn8tS|Dr}K_JN8K?p>OC|kAY zuOv=P*-d4*J#0WD096svhzRJ0x+(^S^r2XfV|9lr=*_vkG?|Z;MpGr zFIad0fMB3kjd_Hs;c?+!g?|UX4UKOUe8wpmEfKnupZxrcOI8A$QIya2jqmtT5G6aR z>7(#vIOYfT$7EOic5OyW#yRs5bF-Hhqw`0(l1C}sYLpja3l=SXgHYAL4OXAIuWz;`siL^ zv9?)J?)Tdk`69qz3d0(1llgt==P)@TP){Mcabd8Ln{VLusY*Y4r4B6hu5PJ+{J44R z{l`smlX|}+A7FCj^Xugw{N4!0_aGb5#oj*SWeSEHCkDsH+spJbSw2^`MvO02nMVPP z15H;Gf+x6L-ewxWO7-LZgX^nRpnvq4={-7Pk8>NYOGr?2u~GuG(T`W;R$THoj<(Y$ zv1Mlken3*2_5tQG_UjG7E6INEW{wGho$1i-;$3u%DEhB#n9L z{)zD3{u~YHobG%-E1l0|`?BP-E7a`D=FvD?7~3``Jlno}+=brY(&4 zU$Ro^%UzP(!=~_3c>x-2kL|3wO_JvAme_dp{$QGve-kKJF%~F803scCZ=~!I-|p=XQV1XLXw z+^h}$rkS#;9ed(kNLje##XEmSP4vDaNb27v?zj$l5j7z@CvHpLC~Y7q>Rb;w6?@EJ4oB1>+p!rB-I zfziKJ5rt|=QNuKBu)5dL%Or_cDgy;dL1KpRW%=TyUe)k%)UazgcD0;aRa*r@;NJ1N zA0%VD*#YPNvZe>+Up665U;LqPDW;ngr;5544yJj|7x_*v9im=@q1VlT05q{eB56~F)_l48BNA+#m zhmCgYBjeQJ;+2<{9lWgSqI>GBou=?>f56QPnXcQX`_Flu`OWcDx-`JA<357lI(IGy3%9T^Ag^Q)rzpmzrEJm`3V-o= zl9ASO_rPZW-0{{VbG)VU*w{y#u&w7T;nr$1>s7nLtryict+ftgtM*-6FWG6W^*+|0 zoaeWuI>!Gyyl{JKnj_rC?%#wS)W*a0xjjO;pTP>mC}bfzMxuxSkyGn}9h>PvXX4^C zP}}ns4|~zB{M2jH(GYzWNJAbUD9PMj#1L-qmt@YB?7I0uhdIyq4{+fijhqPn?&&%)oDy@S@r31i1IyToNXG{bYak@i? z39&&g$2DZ=oufWB-wWpdyw};8=p4T_{=H=T&j;e32i!|vj%#f2v{Y~ZMkE2^O%(AT zRbT785PJmt(JiBKcH+TQ`+tWQUid02ZYO-xnZprOIMg*KW&_UqRQ>^9{cH3#Kc;m4 z#EM0SA^gku??+D;1YF>M_QVr6diO*@3GiNlKVPoPHT+#?q<1eLxbo#@4j$zO59f6O zVC`mxf_7EiD! z|1NhC;_E;C`y(=aVyE*k5hO|q7x?*sg9xYnSt{T4S4q*n-cy>4uIWS38(=9sX>Ku4+QSwN=AGk8QyHzb)rR}+oRPtqA^tPQX7;ExZT2v+7&i2fM|L1XmuZ$ z14q<=4YlAyUC`0YtI!rkj2idGMJiM!50dW#d8c%{bvJDWaC?`9vL_(_o0{5%-)sRe z65zJ!N_0n6&_98w_+a|*F(pq1dN~55kS#WAPN|O-=OX0FSZYV4oe>G6iNw zKo**V6t>QLtS|~1 zcFz%6=nE^}NGxT>?(#wyc#>MwK2P3d3Mbx<3~K@G2Jwo604*51-S9>m^Q+Ay7@2?sA3*kCKnFuI zYIK5rZKONq=e*`AxR7$3H*;L8V&4*i@^p|7I#Mrk8Dlz06*`#LjMN7!$@_d@9n%@s zB(M}cWjFq(kO{FS!;TS<_XYCmtKr93VRjukT{8?nr>yc|ppGtbMUxKx4Ls?WOH54D zs)5@Q=&j`JW?FU&D|^i`8KaAQ@DvoQk{27SK;A5H@{{*OGucjs&OW#22|*Vbxhb8g zpFc3sA*t^r!$~~wz(?>w^E?a}NZ~^C*mj>*hBDF5{{B{<1yk69&lUyVKb%Mmjy4NDsre3tio=KE9j@7hG~X%2+$N=6yO+0r9)nmOGalQqM<3xxYCQkyN+Hk z)i>jQd+BRV@q1V)23LBoJ*=5psvvM#H!)0x1)uhVsW>E+GO;ZdNCn-y-#+G)sW3kU z-%+c%)BCjSH5!yUcokT{vJVZ%T+G<@!NSc_#(AtC2$&5Gxr+wRNG6F(HUm25(>TroniOY{CczM) z%+lu2x>ht;NHB;@DsA*ls!swPx|&onlTq&+NB~^xmtwAHBIQDXZ#aq8L`)VORuc^V z0fYTGiX5tid!U&QKF}WT^1SH!CwkOnNl=zfUNTQ8IuW_VO%XS%KFLmcK3DcV1V}ZP z7z+iq=pc7Rew)?6#Rv77elSe{>s~%pjt91Z-&E#CVHm)ZRCI=NpoMZR{1)^SgK7_m z&prkTS_#r)-D;+R^rWtmSQYk(a$l0_2>M9mEXw^BOCayYv5~8~mPj#+Ad*k>`=j)g zNua>iP9T<$VKP$EI2WX!j2*7IcK7-WPUmJILOp z63YkYtpjhUChbLDLG5m}b=%5)+FTRbwjCmqXEU6)+P13q{iU~SUJC8TJt*_pT|Qeb z+yc9EA7tbKEs=G|Wa`;%ArXDZi51BHZKw&gnWTKbnGJT~LznpB4zzepbh_HrZpooL zWH$6-81MiiL1DACnT+w#xt13k-D7}ssZKbnhdeeIr7{w85*}-49*aTK%^i_g#ty_4 z4ZH1_rgAJHC^*h37&sFKRrt`UP>{g2tXRFztPHEWjYUC=I4yczeEY6eeuOJ8dC3=M z!b=&hhA*3@3~z=1xCz#zAy4YW#Xbm4;@+TeN-L_9Be#SOtwZfdl^uZex5tpT6O)tp z?6(h~JE6&)9mx-?;T)+vyaiI$57y}mg9lfi?DhN?~W9tsEk+m28gEk(5N3#{R| z7Nl@k{TKcOMO*g9jf6gTPP&%|{2`TfIB{TQ1?*5)7q!WxI3>J#AE>X&?#YYR_kVap zvvbX{wC6WaHY8;tq_LTs;mOaa2u@LO$~>UbtE$`C^Sf39a8&{OpoO;6b{Cr;vCt{u z{PdIMy{5u_vPjyp+M6)w`%jarpM*^KSdO`4?xz<9d8~2l^v5Y!=!eb1*EPvVUCd*d(Q69~d(Oa| zQ)OBGsO{?0@wEXBevv1uICY~)?O?H1SKcf$X~qxM|M5;MZm0Aj0fG^nDCkZr8XdKn z%~p*l@(%4^^{89$N!({WvGBAbKA@0VGvPK2_XbQ3GW*|Ew##5Z#ntd@TP;~1I+Zz* zr3KV+nX8wQskxtm9?RTB{R!NAqN8JNvU?_ypjy6ktQ|voHXr^>AtgemD4CDxG!B5B zQcOfaz{hj5edwU zm8b6phTc=ALm*!2JFy~nGd0@e9&-kxqo}|G@zG*OJlGYbZ@)zpc(o`$vZh1UAK%8z3fx+V1(=7R?j!WYVu*-$T^phE_eWh=>8e&cv>q%1GY8!*#~ZWz{S zSat55E@+q!of+P0c+CKU{h$~5bFDb2H!J4v+*+d0+LeYPOfCFD$=gCaG&U{Lw>#PE z_Nhvnpj6BCz^a)uF}I9cfV)ebVK#V%39Iz)zXqch;SL zl18wWTwuCmlB7-V#V#mlzV)HS*UJ`nDMXm4`Y%(pQLtb_!6v8%v*)y-#!Vy5n2po-rfNgoGE$#E3d`ks{Y=*M;*Y)E4j&jeGf+)6hDs95O76` z4Q4mIFya3cZFA}WXF5W8-hV{fEbA(EmA7Y{2IXX&i#NuLg2kv&Trd=JoGC>++LSK+ zkpA#-&y!k6nQ{5GF!)@bMDfpx;WnhE30$#v)#zixu-qcu08uj`zJ8kJn`G%jhEi~Vjp~Vm5 z#ch9-f=jxMFA~#4ELK3M!+Q11D6Om~O{9fPRkz`6+qNoF`RU@*tu;F@>vF%@<~(?} z92e7hbHB>_yC^R5U^&oci>zFm_1A7&{n;Y=N#P2G<3>2>I~|uNw;Sc7 zOcH7HuOfl66^{oexlSv*tc|XBib|#qjc$;<4Qm4&ZtKC z`U9QkGm07!N2bE7*S(DQ=_jlArH@lZSuV0|bFlc$}B0{ObMVCy5kE2Z?4K_ALaS1?zU zd(J%_(i;nJGB}WZ(zzsF1T^qk;-q=A3mre8G;WT;c}QaI7{oL9jLVWSN)|-8>4nBQ z&m}Y^DnnTC+vJBb2mPB2;oG+4srPz~5{47?^lac8sUzY`M!SHfN3K^@=8Zw2hqU!k z=%8{_|J7Q<*;UFPs8qC$H!2&0ni((>rI~B4nwNxs`&hMeUh+qs58$wPsIs5~=T}3o z?MJE!ydYP!PvY^7Z!uzM&+@MWH7$}?mF)(9t`FL3-ttk)w-4AOpMR~)c4Vx2!J=O) z9rS(po^1a2F*O7R86yC(`fcUp4k@>eJlagdMNarLK zx*dflcG3*hk3Wr8Z-F3OH$dW|Mntf16yByInmN-Fw@_r`S&PKa2AlPYs($oRvMLpP z{^V#Rf(#-Q0ZPS7X~X#j5b#0AgWWj536oTnS#*MwSMNO)H;VK+$@n)aBTfGE2c(^2 zS@UQGedL^iOH%bBQ|5aAqT-O6=gioA@Qb+pZY&vZsj@6c&z(arY-CP|4ePB$flBYL zUkX@DNB+~ymG@gldZK3;T5WCUS`DXzRy$swWTFjC^uA?05|aX;IPgPg;u%pXT8Szn zTVUlQz<$USUhf+}!7bnaaszn0=mt=(2r6Qnl!`rm3wN&IjQ~apu48-+YD}}q!|>jb z3G&%{2pfZ15{zX8~oLb3#8Qp{M#82N545t~F3 zns}LA5$`ynj=cIznY1dfFS%cL$H?*I18)tnW1&eTWpL7=7VV6xOt+IpS7W}~S_v2t z4AG{(Nr|Z%Vv0hC8&g&{TpiA3zKf*LfyRo4B9bo==)8DgAsA?8Vr*xUA!YpH7sTVc@#Wv^>!w zm8_me>`XK>_nY^uyu)6>J0Ix$R%0ZO5JdZqPYHiP{`lkgtL^tN zZa1{U-LE`duqXN;d;l)|vPbd(RZk>q#=@w>Fa!NQMlIX}E^&DMBM(oOhzA&Qf|ifI zm`c2S*W3j|>L)q@nL>aa*ifJBv@VD-nu(JQe|f^Zemb06Z7M#~i>lh9o7V;b zvz7x;;EV)Q@?ufMPSNs<&+rd*o|oZQQOBq<#K5%nUZCZzQiKe=M*{sZ=3ob>_tYGy zEXo5)2*3x7{dy_vNMNkH#m6*C)VQF|E*&IJxOCgyI77n4wF46N*=|Hj$d4l46Zm0y z_d#@UaEvkyDA=d+zq&dTcc|Nc0e|N^i!rt_wxQA(vK7V_AjK-QR23gv?TD(c= z9b_=Jv6U<}sEl1i3N@rGp|O;uXsk)rBq2HTUgwYX&x%aOZqP65&e;tiOLcu2>pDMZfvoiUy5dk)l~yPq5z zGa?+7DSNJ@BI}!7vzYk?rt+$i)<$@86H?i)Wv%?1RuxN@I-iukqy3jhhgg=MaCl-q zImL91_{;uiW+64sjcJDbgSEQAiZCF##?xY~TL0KYqKgm?NgpniJUG`fwtEq7-GxVJ z(X~BdLX1BNjUANH;r(FW$(4-~pp7fsNbAs*2QUP|6LV@1b|CFlIw+NWZK9u~__{K3 zCLW7?M7Edz^FRmko6ITOC?JF*rP#&vpUwg*5e+tf=}GVPYm1?O&m@e6K7 z+J&eU4qSYA3scswJld~YbOnNrJbsUM4Hc!n(nmupN=C2mxUCQ3wv?=I>p~+z082$U zTbvZAQt4Pl8Y)=>0Oh#0v(sanzXZZf>IM7m7#&;IzOhCzBDm=`g9Z*FW3lSIr?MVQ zQPfc}M?*>PtBNR}7^fiQG}Epk9b#0gJ>4sFBu@RvV@FhGkana45-#2qLu2_PO#Q=- z)CpB!Tm>>c=Q&2dFXFxt$mi<><-;IC)z_{mOD;)Z1L7#uAbB-a!fii)qacJC*Brn> z0+wA^l7e+A)|u_sNvcwM++Y_Tv@`aE`Ov|xOx3B-h>cLBfteVAwo3$i8kq+BM%D3W zCAekXL^UV8$f9{ZG&n+|*f`3Ykt8J_ix$y$(1D7g$~Xx>xcHMNacBM9#UZ-F?3)Jb z2d`$HjBrIBgv349iMzAzMn^`)&q~QPgKu3WRqGiUREPv6`hqTF(=g=+TdQnU+dm-k z^bbXnuaFNHHbJnU;P+vwF^*XH`w8aTH)pj{eUSNRj9@CAD^*bgF{dS-ca-n#6>e|; zCs#q?)kS18!D0p)LuXQBLt{-cH9qTOUvU%@$FWQV8s7%m(~&V@anZfW7A>G+ywjK7 z6q{!$wu13C9jfC$Rk1Q!AJCWE(|nM%a9QY;kxLi@oQBse(0h06$0U0=k_BF&+v9{Yoc*vn){ymL9`MCw$M{M=4WP@I4P? zjYkSu%Y_!i-4rInV;sb7WGhCW_INDf%cD&B!2BtR9+6qr*9zs_3*Ey}uATB8ozZo5 z8H%rt;BBM~XvPvHGD_5I;iSK=_de{}czDU(AY??6U{u7oa=@3tmlsk)jSM5?*i}D@ zT$-JtHjU2z$cTN;?zLnWzhXzuv0YZ;m_`PU1?4S%(p2P`3`Zy zm9Vo*N}m6ZypYvdOtz2Bc0ZUKRlJD>O}Q{@V3!Lm*NH@YucMt%mr`xR-U(ARrunsv z?38V1!H&3<#_LE!o~Ll(66S95mMhS(z_Ol^hld#QBOB#YBPsmViiLW)Ln_3~hxJTd zLSY%)jdi$guhOK=m{u?Bn3Ff zA+n^yA0{={E+4UD9{EW<4-2eZkhujI=Zw3{Pa0LL9#8Q{BkCm(E|f>`bEuKHhnur| zReoGs=%t^{z@GCt8Z9pF%fF?=!*Sor)fyOEXXUrUpV)3?_)*!~`%$u0Q2S#wzvQav z=qMSAt@nzMxr2smmd&dxfA6LI6Zi!@R+>UKzGcG2DG(LU?Rr`ommF3CHQ?^(sun^c zR*E(aHG9(n8{c1CFp`t<$16 zWbEOAraahnmEz4IsiU{c!w)*hWjfz74ydGfYob%S+w1YipV6-rO^btf&Neh|)$cpb zQV6L^!2LVPasc3V0NkDm2b?N4>qV}8xX4NkOj^UNRy$6tJs)_Bv3!;j%{}L8?A(-N zR6_b^^7@N%X9PLq`D1Ia#rWA4@+z4Lr#CdQJZwl@?D+I^_D9zuw>v5hh}~h8wFxhF zuikXZZ$LD1z9=EAfm(_q=ao~;%KN~oL?>l>{y zk;daToG;Cgg*UCGHm|OlmDrO&=X0+;O$_~5`;HKy2hgBdI)tALBCw#xaPY8o8JJ6+st`hUkA&O>?Nq0?r7XRC{ z9jBDcgV~3`pILN!3U^OGhSUo8(2rgj!kwOJ?3w@6b6# zB51&C|7F`z)+j~#{jPZRRS~NPmdABjd2$yGtt#W{=C)6XL%W50=X&K&kjkZ-ddrCq zW#9DPOO%)8<>9(j5oFX;$3FE7g7$)E_IGn?Or!{uUaf9PY=ns1seYqTWaoM0-G=_j zkNx%vbYhcyt)N<=$AEF(ljxFxjQu^76;N+spp19BD5TWsRKG#5pxfbRLdH^lL+@=P zSuD;4Uy*@(9IL?5Cy)pc&5*8h<&rLBlZD(~->(qs-KG@*KpQ9gTyaoyRxz~K!>!_D&HPuvey@WFc`eJ2TK2dZ8d zMwFRdKn3KDlu-Ow#DR+KK5ru2g*8;8QA#<5I+XV@6BwZr`d=LW)Oy&-I|8M4>Qe^r zQP;e$Gmm?68EVSuc&AYurGaWXHJZYCA#(Wr2)UaY2a0Wv)(j68(zsI#Z|)PgUz6T> zN0cw@;j1-JR_M=6&L|!C?&slm=Z(g;G$L{%#?GyOrpOgc|Ni`~QIMI$`%yoVM|~&U zW0S-BEZ)=Ol=aHLJQ+Zw_#t1gqs%qk$)nb=Fb<+N( z9Yk}=($2~VJ?T8wK}E-njZP;{@KC=e2L>S&&6&nT;&AzlP0dH0+-bS?>0rAKchYER zg`SH1u7=V{i3dg37Q0MY{ITspr1~?<7qBZ{vx#=^F{fsd^Sv9`lYdEs&Pd9n5jY!88a^p{omne_Z-| zRNN;QCO`g*G-=eUKtceK<)|smc^B!$>%OO!7VEvxkUM${Pyq75YblirVt9ZspA_W* z7@na*5A3@pVRUiH(ioCezYKbb-~*V1)bac?ji1%TYIFOye4zxc7;2kdCTJjDQwr5~Te(~2Eq*j^KY62o!^ZH5zF<{P%2$`%vYH(J!`{Gv? zzI~UUzrIu^9P&MhXp%@+uTB2hvIwL-SW8+qOsM=0P!Fw~g0t_>gPe^!TBQc61g}b@ zE=}L^B@#necxyVZe_}xx5EQ0vBvOCgD2QL_4*y;e{#dK@Uge_M-lfWyNIY*HEL?9{ zgfssFV6T5NIA0S9P*%lnsN=5@jbHsGnFR&O*R}UO=sWL=J6|)F$*#kgQN3f*8UBe48K~cO*4VHIW5?W(p?yipP7~p2ju# z1gWDSFd_U=R~n6T5)_Kx;TLB-OR5a184S@d)nxRS{$LeV_hI(2e%N#u(NRRr3<6%3 zNHps<&Ws5;8!f};Hq@^?)pk@4p#b;p3c@gy1S&ngi`G_?V%spA8VO_xbpu3}j-F+x zfTxr=BlYKc+O1hpyCdS@%jMN?Ck0G?-I@a=DQ=+jF&qF%jL{joDV*)-h8Hr#Lx9+Y zKBD2G8ovW@b=tAr9Il7-hA$4B^adWsvnLqkQHlgOx%1urGrzO zeHh?w^i^pm^xT_brK@%p8|^ppbN79@mm!Qp!9v#33S=*ov=(`UA#jmsTfMp2LZ-zW z8;!}x;^b$|8QjZ>)^muV!)^uA55GKpR=v0Y%>`g(b#l$Dr1r}IJ|sK!6X_9sIuI{S z7zdyn#QyoJR>U_zB6K|aTXoan*#|I%Ytl4tJnfK7&BgG?=OW;|Krs_BEbgAFtM54n zDMoJS{-S)qoQ7ApyPy9=f4*UrzM~(Ng#Xd?ai6LkhMFqFFq=5`PkjBCN~`&OpzxV_ zBY%s3q%_YzCNEtI_fn8u9YIQbo4C5dxcE4y5^nvcHW+_8x$)gC(HzLd))Q-YD&HEO zaANP|oWC0xX^bkAw0U+YA+9LrvNhnYV5Ud)$4kkSi(VD8Y@BXoVAa+>%;Eg>0dH;X zNOph{l34EfMMbYc9J5)7Jkg)_)99(oc5UnSw&D8N>w%!qMkY{}mUlx&;8D&6%aAh) z``?i7_~Y2!*J^D#@CnnHR%GbR0tF?q*bqqZ+mHzKbpYg$?^32GIz+A z^Xu>KNdH0#oA4r6byERJt|5?-92qw}I2sjwQ20@c?x*y-s(7;^SxI%g+~*UPBvFcT}rbjoo>~Px3+siRT00^ z?`SyO!P_X0e{jIxTK!C|Q;Cj5{uRs}FF}(72gQMk2h&Ge>YF|1(l9%ZGk9A0SD$!^ zrYn6+$k$3agMghst@!jtz_l#5dUfGn2!SY(nC)6+zzWh$QyJlv!ZMV+~PA9(ms<`yAZReh_3y>J&@;lm0+8Pu1A5Q#AcUTj7BuwJ)AFd$T$R zr1T4@t`qkN+7FrfWE_EO5{80DY8RL(IVrPs7bsENuFj!3jJCGZQ4tz z0>%bfPQ3lu^--8O;Fdg(S80<4fniuQ0SeDqxr_$VAJ7mKDngJAO3=amE!jVx9LR3= zIYfftB_olRu4baiyp#K=fU5+S15LJszy^+86bS-AGxD{@5K-6M!%ZB#A&J2SHW<=cakE_lovYl4<*ywcaslNTRYg+R3Z(l$W!^*#+`kLlLNI~pIf6(G-=?b_tV@j+Xe?pX`t&0Z!VP+?>D zR~wJbzW)c>u04u3KV$zNpl!vOEZ_eN+Mcs`TX3L>58B3(=Y4l=4Ye!lDca=C2N-V+ zbCc_y6g8b$C*+I__0-ip?Oq5z?%|`$HY2r>7cb>(eVVm<`h2k4cJc3qt!X8`L#s@^339*=AG3F%%2(Td`0YYHKZfwp41JlG< zfZ7Zop2Zoiut~6CAT|+s5lq4~av)>Gz1)X6mVj6i!b{kVI1-e^So?idUDG|%Sh8iv zVs^^ullRHy1IHiB9@Gi#>NTAE9Jz|BG-vXET-#kRfuc`ZNday-`x^9UJMedqPeqBFtl*n;24S!75%@TQVj^Rfge5W01_+xTnrO4#tM0SS{ zFd@wV{bF);rpm=1E*e2`U7g_*MZlGOK97ce^(X!PECKjO5j<(oQS!wW=KN zXcRk>iRV~@NJl2ik`1y{wYPwv8nW?36ASS;$ZJr@#xtx(JK|)pVKb7UKhl*{2JH^| zPxfRo63@6IiG*q=KiTb!Tb0S8A(Bm?Rq?0eosw-5I}y)R7LDp|y3Y&=eWNh&1^J3+QH_*CNnxv^}dM=*(!ok?QU7|(WcVWzhy*e5+~ zDzYj9Rq;$5)5Qfgn}vOmvyxmF(pdT8^cUS-gDO`f|2`y$M@KTAp?;OBIL2uDvCqeHz{h?d2R-&fIrOm`s=BfpP>!9sw+|BNr}9 zemoIaFni0Xl*Q_biVCafFR0a(5ZG4!n(FF^wS_cgp#FwhY-x3h)l~XkZONb65jt-8 znKz85NNN7L^rAW@^%E!6#R8%EH6hhQ9{RkpL$f3jLi;3UpRjb6HY!t)H&Z5In9YU3uyUYgbm-SgWYYf9T*K z&92a0R`G;wU7HV=tlE0o!5<|jqc9UWoW-a1r$)%3T?aq@0)83-+dNP;zb?!L*%wf* zs1Nv457`@LErsOsQymUa;CG|AJSqVnW;hF7fhnZYQ$iDFuxkBh#5y64(&@U zhz(0dDRhp*gx;0LI9}saiE3!5C_ZG-iH(sOP6DJ^ zSP-iXIT0!rf^E2woB3HneYn2`2{k``&xi3}$!shC+*rfR1`R(m)tR{Z%4r9SfPT#6 za~9J`au-X+5a&GQbE_faT6}KhKc`lPCJ`4Hv&irn_3rP=Xg;HM;Td&(i;OqVsC+)X zr&T?lPQCX6nU=p$W^QPcIe0$Zy?d3+@4nQ3M!iBl_rR617oJm_9=clgdH7n{56`KK z@tk_VfBd_8PCfXsAIV{l-zZ;y@+Lf|X5=V5rylprTKQ)@qaF_|#WU(jct$-H�&2 z8TE`8?v&JvS^4fuzmVlX`584de&u300^1n@wSa7=j!%$y!{Dt+Uu)y zW!~fvSD`&>jhZ@lo~h1=`igwQLOKtJTYW<^VJW5W; z*H5!7ykLiBofNOCt9JBazC{g&L1czkY4xfZM3vYg zqpTUsxQm-L%K8j4coAyHre4LK{Zj>BIjWRnn#%Cc)%(C%wl5rN2g_uAkDC@6y+B?m z8G8IxC<4v8q~r=8QwF6%(+*qU?yO|`KY8!q?^6onje#1@3hh&>u}7JJbMz_q zu9P}pdj2Ab;rx8J`*42l?@bomf2kxMzDD+awU;qun#!|`;en?Mt=OWx5>%R z2YZe)vFC7bCf@RLZigMyvg73le3~MF^=X9NsWc0^lAoYXj?~=+^;+E0rtZX>S$!oI z7*wmLPS{0iNkP3*aI|Puk3s7sv{%>4Xuct>tCBi(PO|e(vwU52H)H1FCdrNJfiZ}%iUA<-;Y2pRe^c)67Z&vUgVpA%^xF19)HiN!Q40k zzTJTQDd0VnV@2bcOI3ls8*mWX&O&|*@FL28L9XXksku^$#(6oIzU}^d&MfG;7WV%L z4Afcde^F;bm0W6P$Q}WFAG%&ezNtvpU_bRWHJHBj{^wFPuyq;mCm=j0IA3lBP6M*Q zBfvwzdf;ZD1vn6x80tY#%%v*Pwij?Q!1?zf5Y|6bdF%Z5oF?deQN<{T`ID$03A%p( z373Xmf8GUu4mbvU=B0bk50-MN(P&?c{8r?D0tSxn!1Zi+|8uFMq33<*G_rq2IRzS? zi$ckBYC-ovzXSLwkVpNu$Oq1I#*^#pa)9TWrIY7ONBb)9_97qI{K=)3K>lxl8N;_w z_P_KCv$>QF91YM{_A6H8OfEGFwE4ifz^j1KWAfjCb}UeM@GFvMC6}5CTnqdaFmfi} za{N8#o9MUpU614MQ0MD@omlpqN0m+;AI{swpnVNE2G}2U9zg5FdG0yi23`)ynEBBK z`V7NbGWVR%L+)wtg~y__|DLlK_}){zdfssUgyVhy+Vr^U_J>iYznm{xzwYzw;l<^T0cC)+jFBiG*r z%B4Po@~=bfohb7ihEDW9kTde({AslNUz~x6UXyPtAw!!J0Sn;0?{Ofk$CQix_nZXy z{))buIl%cJp7Y;RZPevCe*lWV;Zi#O;qj(kt^s?SE~Curq7BgNelRySg2wphds{%) zzG)u<|2?My{P%%B7*Ef+Ib^2+?cq7QywGL}gZ+OAGP?l0uLXK@DfTJo1LwTwoK^6( zm^UmpupIcGOC6wWP*#I`uRV=xC|?q4=X};*nJO@#P&|J)b_c0uz0O{ax{k5d{|D5A zxkScXY8kLCd%nPbeQz|^HOGjt3%>b~OI?Y&?!S)D#YOGfKJQDECEu0G_wa}3oD6y& zd^B<7et}<~3zFxYA8HT#Q>q;H|03wteGhy%qo7#NQ#WLvK6Jqr*IHZg>G|_})XxKg zz8fXnhk6e0bH@KCQ4ZE5@_EkJK^tn$m!ALnT3-rTy;sm<%sFs^F|YDm>K{Q1uLtDW z2zUVZX~q6fNbBbM7uNke>bnfk{YB8;1gTWKNCmNk*tcIRH1%oJXFXlNJK=)tg z_n^L8m<@HV&(o0~2k^f0ugDJtjsjfN>8I|Wo-^#jHK6mbJV>6i6!Z^3*S?yZaT&Ti zIR1fFj{70ND@q6T=)HeCa-6r9NX6^Vz_I6?TU;c?xt2?<0S5AJZGC9}X-q!@o`am% zr5RJ+2i5{#1APke@b#;C^q6s-{~2&7_%x5!$$JgQN#Dx@y&rA-!@=VH+;h$WUV|^; zIr#x-F93IeemZilk$h%Yh#|W{uCT3QQ*Eam*V=;3|xTn8Ge~#!1^u7xsM6ovxd7S zhi?zA>+SP@V3&rzeoMt~duboN|K?f@Vq~uQ@HI1ZxzsgLT3^}!@EppexW`=mcf{FY z^W}0m0_6ylBT$Y&IRZOc1oU@7NAmx5C^^Lc*Wp*V@Z{3p+obdSB7DOXbrJP@FPkk_ z{SM5gh}gZL_tiDYCgi=1wzMMeX_h>HXC|L_8~Hmk`IT-RgK z+sZt1(YckMrs(2RF~k z^!p~h3k#n4M&`%x_W6Fvb9jc=3A_oMRPfnz_+5l~z&8NC@3;^6Ghoh}hRe5X-*Y0+ zb2h+tGStKV-VIy=@SMFBSOV+~j0yQffBy+s1N=8Ilnc!S?k#SjL%tKLhCVRnM7=@@2rOz;s{^a5!)SpwD>q z`Bx70dCFJwq2J|v8T7}2a6F3bYrVhqd7ehjGhZ9v*z(MlXN*??mjfJwgMm>%4X_g6 zyH)D%1I`74^KBgu>T&?{J}$7Y$NLeqQJ(Q^0Qe4SH9#6;F&h{d{}(|!P<2(eKZrW{ z37$^}XX2i-6Y7kiIbRRkZw1}RE%Ei802;^bEkOTYtHJgu!*?4O0z5+<4O|D9@$Nx+ zFW?k_dW;R`+P>$+P-bjNdk&x;#?;LHrDiBW3-|vQsNV)qj`rBbm=P-geIBa)Gyasa ze?52&{a^gPFZF5cTR4WsKIhyFfVR?rvDFsxmDg`6yT+F3KgY??T7COsY@O|q;0Ch0N`d2>m+~hg3(LFFX_PLI&1?VgLai;GB$MP)T_dfmC$a(z* z-&66Nsi<=tsGEHHdj*{PdhQP2e;U($I|;d2*Vmw|cWa)*b>wK^hrk)Y=Yd0ks{wQE zKY_CT7QSAWZ%2J7+w?b>|0bAvOz%V+Y%%2*-uPTP~L)@c%Ia`n+MlXOCF@J~odNt52G(-dMfW<3_B!SQ}i~Z^MN#94j=7 z6>T18MeRzfa&**MDHE)o<{0>0YjYGjz-;skuu!P-oPsyH7+}sR_AA~wqU95sS`Hg8 z?MF2Z3-|ES->>aF{VQ0@J^+TkSKoHZqCx~NI)eM4Zvz_uzK>lD>;>!%aDT8KFnesh z7905+mvFE1bD#^@1NaVbm0y1ic?&QW*crGGco*mw%3u5q+RXyIFYgIF4^Z}gU={EM zfX^SVs4^Y)oCAOlmArmldmr^_z_Y-&fI|TK$bA~^?+(x|W5PX|zMs>+-d9lnbl{J` zfdJ+CosuZPd*gose9pKApk58i-<4v2Hv-hpdpG&7^UEh7?*YCFd=~f}P%O{>+_aQ- z&IQDBk&ebENAKi<+T=(AD^^8 nWL_H` for installation details. + +Additional requirements can be found in `setup.sh `_. + +.. toctree:: + :maxdepth: 1 + + text_normalization + inverse_text_normalization + api + + diff --git a/models/tacotron2/NeMo/docs/source/nemo_text_processing/inverse_text_normalization.rst b/models/tacotron2/NeMo/docs/source/nemo_text_processing/inverse_text_normalization.rst new file mode 100644 index 0000000000..0e828c78bf --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nemo_text_processing/inverse_text_normalization.rst @@ -0,0 +1,90 @@ +Inverse Text Normalization +========================== + +Inverse text normalization (ITN) is a part of the Automatic Speech Recognition (ASR) post-processing pipeline. +ITN is the task of converting the raw spoken output of the ASR model into its written form to improve text readability. + +For example, +`"in nineteen seventy"` -> `"in 1975"` +and `"it costs one hundred and twenty three dollars"` -> `"it costs $123"`. + +NeMo ITN :cite:`textprocessing-itn-zhang2021nemo` is based on WFST-grammars :cite:`textprocessing-itn-Mohri2009`. We also provide a deployment route to C++ using `Sparrowhawk `_ :cite:`textprocessing-itn-sparrowhawk` -- an open-source version of Google Kestrel :cite:`textprocessing-itn-ebden2015kestrel`. +See :doc:`Text Procesing Deployment <../tools/text_processing_deployment>` for details. + +.. note:: + + For more details, see the tutorial `NeMo/tutorials/text_processing/Inverse_Text_Normalization.ipynb `__ in `Google's Colab `_. + + + + + + +Classes +---------------------------------- + + +The base class for every grammar is :class:`GraphFst`. +This tool is designed as a two-stage application: 1. `classification` of the input into semiotic tokens and 2. `verbalization` into written form. +For every stage and every semiotic token class there is a corresponding grammar, e.g. :class:`taggers.CardinalFst` +and :class:`verbalizers.CardinalFst`. +Together, they compose the final grammars :class:`ClassifyFst` and +:class:`VerbalizeFinalFst` that are compiled into WFST and used for inference. + + + + + + +.. autoclass:: nemo_text_processing.inverse_text_normalization.ClassifyFst + :show-inheritance: + :members: + +.. autoclass:: nemo_text_processing.inverse_text_normalization.VerbalizeFinalFst + :show-inheritance: + :members: + + +Prediction +---------------------------------- + +Example prediction run: + +.. code:: + + python run_prediction.py --input= --output= [--verbose] + + +The input is expected to be lower-cased. Punctuation are outputted with separating spaces after semiotic tokens, e.g. `"i see, it is ten o'clock..."` -> `"I see, it is 10:00 . . ."`. +Inner-sentence white-space characters in the input are not maintained. + + + +Data Cleaning for Evaluation +---------------------------------- + +.. code:: + + python clean_eval_data.py --input= + + + +Evaluation +---------------------------------- + +Example evaluation run on (cleaned) `Google's text normalization dataset `__ :cite:`textprocessing-itn-sproat2016rnn`: + +.. code:: + + python run_evaluation.py --input=./en_with_types/output-00001-of-00100 [--cat CLASS_CATEGORY] [--filter] + + + + +References +---------- + +.. bibliography:: textprocessing_all.bib + :style: plain + :labelprefix: TEXTPROCESSING-ITN + :keyprefix: textprocessing-itn- \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/nemo_text_processing/text_normalization.rst b/models/tacotron2/NeMo/docs/source/nemo_text_processing/text_normalization.rst new file mode 100644 index 0000000000..3f225cacd7 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nemo_text_processing/text_normalization.rst @@ -0,0 +1,78 @@ +Text Normalization +================== + +NeMo Text Normalization converts text from written form into its verbalized form. It is used as a preprocessing step before Text to Speech (TTS). It could also be used for preprocessing Automatic Speech Recognition (ASR) training transcripts. + + +For example, +`"at 10:00"` -> `"at ten o'clock"` +and `"it weighs 10kg."` -> `"it weights ten kilograms ."`. + + +NeMo Text Normalization :cite:`textprocessing-norm-zhang2021nemo` is based on WFST-grammars :cite:`textprocessing-norm-Mohri2009`. We also provide a deployment route to C++ using `Sparrowhawk `_ :cite:`textprocessing-norm-sparrowhawk` -- an open-source version of Google Kestrel :cite:`textprocessing-norm-ebden2015kestrel`. +See :doc:`Text Procesing Deployment <../tools/text_processing_deployment>` for details. + + +.. note:: + + For more details, see the tutorial `NeMo/tutorials/text_processing/Text_Normalization.ipynb `__ in `Google's Colab `_. + + + + + + +Classes +---------------------------------- + + +The base class for every grammar is :class:`GraphFst`. +This tool is designed as a two-stage application: 1. `classification` of the input into semiotic tokens and 2. `verbalization` into written form. +For every stage and every semiotic token class there is a corresponding grammar, e.g. :class:`taggers.CardinalFst` +and :class:`verbalizers.CardinalFst`. +Together, they compose the final grammars :class:`ClassifyFst` and +:class:`VerbalizeFinalFst` that are compiled into WFST and used for inference. + + + + +.. autoclass:: nemo_text_processing.text_normalization.ClassifyFst + :show-inheritance: + :members: + +.. autoclass:: nemo_text_processing.text_normalization.VerbalizeFinalFst + :show-inheritance: + :members: + + +Prediction +---------------------------------- + +Example prediction run: + +.. code:: + + python run_prediction.py <--input INPUT_TEXT_FILE> <--output OUTPUT_PATH> [--input_case INPUT_CASE] + +``INPUT_CASE`` specifies whether to treat the input as lower-cased or case sensitive. By default treat the input as cased since this is more informative, especially for abbreviations. Punctuation are outputted with separating spaces after semiotic tokens, e.g. `"I see, it is 10:00..."` -> `"I see, it is ten o'clock . . ."`. +Inner-sentence white-space characters in the input are not maintained. + + +Evaluation +---------------------------------- + +Example evaluation run on `Google's text normalization dataset `__ :cite:`textprocessing-norm-sproat2016rnn`: + +.. code:: + + python run_evaluation.py --input=./en_with_types/output-00001-of-00100 [--cat CLASS_CATEGORY] [--input_case INPUT_CASE] + + + +References +---------- + +.. bibliography:: textprocessing_all.bib + :style: plain + :labelprefix: TEXTPROCESSING-NORM + :keyprefix: textprocessing-norm- \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/nemo_text_processing/textprocessing_all.bib b/models/tacotron2/NeMo/docs/source/nemo_text_processing/textprocessing_all.bib new file mode 100644 index 0000000000..63eb101cae --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nemo_text_processing/textprocessing_all.bib @@ -0,0 +1,58 @@ +@article{ebden2015kestrel, + title={The Kestrel TTS text normalization system}, + author={Ebden, Peter and Sproat, Richard}, + journal={Natural Language Engineering}, + volume={21}, + number={3}, + pages={333}, + year={2015}, + publisher={Cambridge University Press} +} + + +@article{sproat2016rnn, + title={RNN approaches to text normalization: A challenge}, + author={Sproat, Richard and Jaitly, Navdeep}, + journal={arXiv preprint arXiv:1611.00068}, + year={2016} +} + +@book{taylor2009text, + title={Text-to-speech synthesis}, + author={Taylor, Paul}, + year={2009}, + publisher={Cambridge university press} +} + +@misc{zhang2021nemo, + title={NeMo Inverse Text Normalization: From Development To Production}, + author={Yang Zhang and Evelina Bakhturina and Kyle Gorman and Boris Ginsburg}, + year={2021}, + eprint={2104.05055}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} + +@Inbook{Mohri2009, +author="Mohri, Mehryar", +editor="Droste, Manfred +and Kuich, Werner +and Vogler, Heiko", +title="Weighted Automata Algorithms", +bookTitle="Handbook of Weighted Automata", +year="2009", +publisher="Springer Berlin Heidelberg", +address="Berlin, Heidelberg", +pages="213--254", +abstract="Weighted automata and transducers are widely used in modern applications in bioinformatics and text, speech, and image processing. This chapter describes several fundamental weighted automata and shortest-distance algorithms including composition, determinization, minimization, and synchronization, as well as single-source and all-pairs shortest distance algorithms over general semirings. It presents the pseudocode of these algorithms, gives an analysis of their running time complexity, and illustrates their use in some simple cases. Many other complex weighted automata and transducer algorithms used in practice can be obtained by combining these core algorithms.", +isbn="978-3-642-01492-5", +doi="10.1007/978-3-642-01492-5_6", +url="https://doi.org/10.1007/978-3-642-01492-5_6" +} + +@inproceedings{sparrowhawk, + title = {TTS for Low Resource Languages: A Bangla Synthesizer}, + author = {Alexander Gutkin and Linne Ha and Martin Jansche and Knot Pipatsrisawat and Richard Sproat}, + booktitle = {10th Language Resources and Evaluation Conference}, + year = {2016}, +} \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/nlp/api.rst b/models/tacotron2/NeMo/docs/source/nlp/api.rst new file mode 100755 index 0000000000..18a81d76e3 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/api.rst @@ -0,0 +1,74 @@ +NeMo NLP collection API +======================= + +Model Classes +------------- + +.. autoclass:: nemo.collections.nlp.models.TextClassificationModel + :show-inheritance: + :members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact, classifytext + +.. autoclass:: nemo.collections.nlp.models.GLUEModel + :show-inheritance: + :members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact + +.. autoclass:: nemo.collections.nlp.models.PunctuationCapitalizationModel + :show-inheritance: + :members: add_punctuation_capitalization, setup_training_data, setup_optimization, setup_validation_data, setup_test_data, multi_validation_epoch_end, register_artifact + +.. autoclass:: nemo.collections.nlp.models.TokenClassificationModel + :show-inheritance: + :members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact + +.. autoclass:: nemo.collections.nlp.models.QAModel + :show-inheritance: + :members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, inference, validation_epoch_end, test_epoch_end + +.. autoclass:: nemo.collections.nlp.models.BERTLMModel + :show-inheritance: + :members: setup_training_data, setup_optimization + +Modules +------- + +.. autoclass:: nemo.collections.nlp.modules.BertModule + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.nlp.modules.MegatronBertEncoder + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.nlp.modules.AlbertEncoder + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.nlp.modules.BertEncoder + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.nlp.modules.DistilBertEncoder + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.nlp.modules.RobertaEncoder + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.nlp.modules.SequenceClassifier + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.nlp.modules.SequenceRegression + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.nlp.modules.SequenceTokenClassifier + :show-inheritance: + :members: + +.. autofunction:: nemo.collections.nlp.modules.get_lm_model + +.. autofunction:: nemo.collections.nlp.modules.get_pretrained_lm_models_list + +.. autofunction:: nemo.collections.nlp.modules.get_megatron_lm_models_list \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/nlp/bert_pretraining.rst b/models/tacotron2/NeMo/docs/source/nlp/bert_pretraining.rst new file mode 100755 index 0000000000..eee23bd550 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/bert_pretraining.rst @@ -0,0 +1,133 @@ +.. _bert_pretraining: + +BERT +==== + +BERT is an autoencoding language model with a final loss composed of: + +- masked language model loss +- next sentence prediction + +The model architecture is published in `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding `__ :cite:`nlp-bert-devlin2018bert`. +The model is originally trained on English Wikipedia and BookCorpus. BERT is often used as a language model encoder for downstream tasks, for example, :ref:`token_classification`, :ref:`text_classification`, :ref:`question_answering`, etc. +Domain-specific BERT models can be advantageous for a wide range of applications. One notable application is the domain-specific BERT in a biomedical setting, +e.g. BioBERT :cite:`nlp-bert-lee2019biobert` or its improved derivative BioMegatron :cite:`nlp-bert-shin2020biomegatron`. For the latter, refer to :ref:`megatron_finetuning`. + +Quick Start Guide +----------------- + +.. code-block:: python + + from nemo.collections.nlp.models import BERTLMModel + + # to get the list of pre-trained models + BERTLMModel.list_available_models() + + # Download and load the pre-trained BERT-based model + model = BERTLMModel.from_pretrained("bertbaseuncased") + +Available Models +^^^^^^^^^^^^^^^^ + +.. list-table:: *Pretrained Models* + :widths: 5 10 + :header-rows: 1 + + * - Model + - Pretrained Checkpoint + * - BERT-base uncased + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:bertbaseuncased + * - BERT-large uncased + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:bertlargeuncased + +.. _dataset_bert_pretraining: + +Data Input for the BERT model +----------------------------- + +Data preprocessing can be either done on-the-fly during training or offline before training. The latter is optimized and recommended +for large text corpora. This was also used in the original paper to train the model on Wikipedia and BookCorpus. For on-the-fly data +processing, provide text files with sentences for training and validation, where words are separated by spaces, i.e.: ``[WORD] [SPACE] [WORD] [SPACE] [WORD]``. +To use this pipeline in training, use the dedicated configuration file `NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_preprocessed_config.yaml`. + +To process data offline in advance, refer to the `BERT Quick Start Guide `__. +To recreate the original Wikipedia and BookCorpus datasets, follow steps 1-5 in the Quick Start Guide and run the script ``./data/create_datasets_from_start.sh`` inside the Docker container. +The ``downloaded`` folder should include two sub folders ``lower_case_[0,1]_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5`` +and ``lower_case_[0,1]_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5``, containing sequences of length 128 with a maximum of 20 masked tokens +and sequences of length 512 with a maximum of 80 masked tokens respectively. To use this pipeline in training, use the dedicated configuration file ``NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_text_config.yaml`` +and specify the path to the created hd5f files. + + +Training the BERT model +----------------------- + +Example of model configuration for on-the-fly data preprocessing: `NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_text_config.yaml `__. +Example of model configuration for offline data preprocessing: `NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_preprocessed_config.yaml `__. + +The specification can be grouped into three categories: + +- Parameters that describe the training process: **trainer** +- Parameters that describe the datasets: **model.train_ds**, **model.validation_ds** +- Parameters that describe the model: **model**, **model.tokenizer**, **model.language_model** + +More details about parameters in the config file can be found below: + ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Description** | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.only_mlm_loss** | bool | Only uses masked language model without next sentence prediction. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **train_ds.data_file** | string | Name of the text file or hdf5 data directory. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **train_ds.num_samples** | integer | Number of samples to use from the training dataset, ``-1`` - to use all. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ + +More details about parameters for offline data preprocessing can be found below: + ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Description** | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **train_ds.max_predictions_per_seq** | integer | Maximum number of masked tokens in a sequence in the preprocessed data. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ + +More details about parameters for online data preprocessing can be found below: + ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Description** | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.max_seq_length** | integer | The maximum total input sequence length after tokenization. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.mask_prob** | float | Probability of masking a token in the input text during data processing. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.short_seq_prob** | float | Probability of having a sequence shorter than the maximum sequence length. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ + +.. note:: + + For offline data preprocessing, **model.tokenizer** is null. For downstream task, use the same tokenizer that was used for + offline preprocessing. For online data preprocessing, **model.tokenizer** needs to be specified. See also :ref:`nlp_model` for + details. + +Example of the command for training the model: + +.. code:: + + python bert_pretraining.py \ + model.train_ds.data_file= \ + trainer.max_epochs= \ + trainer.gpus=[] + + +Fine-tuning on Downstream Tasks +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To use a trained BERT model checkpoint on a NeMo NLP downstream task, e.g. :ref:`question_answering`, specify +:code:`model.language_model.lm_checkpoint=`. + +References +---------- + +.. bibliography:: nlp_all.bib + :style: plain + :labelprefix: NLP-BERT + :keyprefix: nlp-bert- \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/nlp/glue_benchmark.rst b/models/tacotron2/NeMo/docs/source/nlp/glue_benchmark.rst new file mode 100644 index 0000000000..37e0912520 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/glue_benchmark.rst @@ -0,0 +1,10 @@ +.. _glue_benchmark: + +GLUE Benchmark +============== + +We recommend you try the GLUE Benchmark model in a Jupyter notebook (can run on `Google's Colab `_): `NeMo/tutorials/nlp/GLUE_Benchmark.ipynb `__. + +Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator). + +An example script on how to train the model can be found here: `NeMo/examples/nlp/glue_benchmark/glue_benchmark.py `__. diff --git a/models/tacotron2/NeMo/docs/source/nlp/information_retrieval.rst b/models/tacotron2/NeMo/docs/source/nlp/information_retrieval.rst new file mode 100644 index 0000000000..e4c047a40b --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/information_retrieval.rst @@ -0,0 +1,10 @@ +.. _information_retrieval: + +Information Retrieval +===================== + +We recommend you try the Information Retrieval model in a Jupyter notebook (can run on `Google's Colab `_): `NeMo/tutorials/nlp/Information_Retrieval_MSMARCO.ipynb `__. + +Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for hardware the accelerator), + +An example script on how to train the model can be found here: `NeMo/examples/nlp/information_retrieval `__. diff --git a/models/tacotron2/NeMo/docs/source/nlp/intro.rst b/models/tacotron2/NeMo/docs/source/nlp/intro.rst new file mode 100755 index 0000000000..7703dd06b3 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/intro.rst @@ -0,0 +1,15 @@ +Natural Language Processing (NLP) +================================= + +.. toctree:: + :maxdepth: 2 + + models + megatron_finetuning + api + + +Example scripts for the NLP collection can be found under `NeMo/example/nlp/ `__. + +NLP notebook-tutorials are located under `NeMo/tutorials/nlp/ `__. Most NeMo +tutorials can be run on `Google's Colab `_.), with more details in the :ref:`tutorials` section. diff --git a/models/tacotron2/NeMo/docs/source/nlp/joint_intent_slot.rst b/models/tacotron2/NeMo/docs/source/nlp/joint_intent_slot.rst new file mode 100644 index 0000000000..5dc00430a2 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/joint_intent_slot.rst @@ -0,0 +1,244 @@ +.. _intent_slot: + +Joint Intent and Slot Classification +==================================== + +Joint Intent and Slot classification is a NLU task for classifying an intent and detecting all +relevant slots (Entities) for the intent in a query. For example, in the query ``What is the weather in Santa Clara tomorrow morning?``, +we would like to classify the query as a ``weather intent``, detect ``Santa Clara`` as a `location slot`, +and ``tomorrow morning`` as a ``date_time slot``. Intent and Slot names are usually task-specific and +defined as labels in the training data. This is a fundamental step that is executed in any +task-driven conversational assistant. + +Our BERT-based model implementation allows you to train and detect both of these tasks together. + +.. note:: + + We recommend you try the Joint Intent and Slot Classification model in a Jupyter notebook (can run on `Google's Colab `_.): `NeMo/tutorials/nlp/Joint_Intent_and_Slot_Classification.ipynb `__. + + Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator). + + An example script on how to train the model can be found here: `NeMo/examples/nlp/intent_slot_classification `__. + + +NeMo Data Format +---------------- + +When training the model, the dataset should be first converted to the required data format, which requires the following files: + +- :code:`dict.intents.csv` - A list of all intent names in the data. One line per an intent name. The index of the intent line + (starting from ``0``) is used to identify the appropriate intent in ``train.tsv`` and ``test.tsv`` files. + +.. code:: + + weather + alarm + meeting + ... + +- :code:`dict.slots.csv` - A list of all slot names in the data. One line per slot name. The index of the slot line + (starting from ``0``) is used to identify the appropriate slots in the queries in ``train_slot.tsv`` and ``test_slot.tsv`` files. + In the last line of this dictionary ``O`` slot name is used to identify all ``out of scope`` slots, which are usually the majority of the tokens + in the queries. + +.. code:: + + date + time + city + ... + O + +- :code:`train.tsv/test.tsv` - A list of original queries, one per line, with the intent number + separated by a tab (e.g. "what alarms do i have set right now 0"). Intent numbers are + set according to the intent line in the intent dictionary file (:code:`dict.intents.csv`), + starting from ``0``. The first line in these files should contain the header line ``sentence + label``. + +- :code:`train_slot.tvs/test_slot.tsv` - A list that contains one line per query, when each word from the original text queries + is replaced by a token number from the slots dictionary file (``dict.slots.csv``), counted starting from ``0``. All the words + which do not contain a relevant slot are replaced by ``out-of scope`` token number, which is also a part of the slot dictionary file, + usually as the last entry there. For example a line from these files should look similar to: "54 0 0 54 54 12 12" (the numbers are + separated by a space). These files do not contain a header line. + + +Dataset Conversion +------------------ + +To convert to the format of the model data, use the ``import_datasets`` utility, which implements +the conversion for the Assistant dataset. Download the dataset `here `_ or you can +write your own converter for the format that you are using for data annotation. + +For a dataset that follows your own annotation format, we recommend using one text file for all +samples of the same intent, with the name of the file as the name of the intent. Use one line per +query, with brackets to define slot names. This is very similar to the assistant format, and you can +adapt this converter utility or your own format with small changes: + +:: + + did i set an alarm to [alarm_type : wake up] in the [timeofday : morning] + +Run the ``dataset_converter`` command: + +.. code:: + + python examples/nlp/intent_slot_classification/data/import_datasets.py + --source_data_dir=`source_data_dir` \ + --target_data_dir=`target_data_dir` \ + --dataset_name=['assistant'|'snips'|'atis'] + +- :code:`source_data_dir`: the directory location of the your dataset +- :code:`target_data_dir`: the directory location where the converted dataset should be saved +- :code:`dataset_name`: one of the implemented dataset names + +After conversion, ``target_data_dir`` should contain the following files: + +.. code:: + + . + |--target_data_dir + |-- dict.intents.csv + |-- dict.slots.csv + |-- train.tsv + |-- train_slots.tsv + |-- test.tsv + |-- test_slots.tsv + +Model Training +-------------- + +This is a pretrained BERT based model with 2 linear classifier heads on the top of it, one for classifying an intent of the query and +another for classifying slots for each token of the query. This model is trained with the combined loss function on the Intent and Slot +classification task on the given dataset. The model architecture is based on the paper `BERT for Joint Intent Classification and Slot Filling `__:cite:`nlp-jis-chen2019bert`. + +For each query, the model classifies it as one the intents from the intent dictionary and for each word of the query it will classify +it as one of the slots from the slot dictionary, including out of scope slot for all the remaining words in the query which does not +fall in another slot category. Out of scope slot (``O``) is a part of slot dictionary that the model is trained on. + +Example of model configuration file for training the model can be found at: `NeMo/examples/nlp/intent_slot_classification/conf/intent_slot_classification.yaml `__. +In the configuration file, define the parameters of the training and the model, although most of the default values will work well. + +The specification can be roughly grouped into three categories: + +- Parameters that describe the training process: **trainer** +- Parameters that describe the model: **model** +- Parameters that describe the datasets: **model.train_ds**, **model.validation_ds**, **model.test_ds**, + +More details about parameters in the spec file can be found below: + ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Default** | **Description** | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **model.data_dir** | string | -- | The path of the data converted to the specified format. | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **model.class_balancing** | string | ``null`` | Choose from ``[null, weighted_loss]``. The ``weighted_los``s enables weighted class balancing of the loss. | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **model.intent_loss_weight** | float | ``0.6`` | The elation of intent-to-slot loss in the total loss. | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **model.pad_label** | integer | ``-1`` | A value to pad the inputs. | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **model.ignore_extra_tokens** | boolean | ``false`` | A flag that specifies whether to ignore extra tokens. | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **model.ignore_start_end** | boolean | ``true`` | A flag that specifies whether to not use the first and last token for slot training. | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **model.head.num_output_layers** | integer | ``2`` | The number of fully connected layers of the classifier on top of the BERT model. | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **model.head.fc_dropout** | float | ``0.1`` | The dropout ratio of the fully connected layers. | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **training_ds.prefix** | string | ``train`` | A prefix for the training file names. | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **validation_ds.prefix** | string | ``dev`` | A prefix for the validation file names. | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ +| **test_ds.prefix** | string | ``test`` | A prefix for the test file names. | ++-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ + +For additional config parameters common to all NLP models, refer to the `nlp_model doc `__. + +The following is an example of the command for training the model: + +.. code:: + + python examples/nlp/intent_slot_classification/intent_slot_classification.py + model.data_dir= \ + trainer.max_epochs= \ + trainer.gpus=[] + + +Required Arguments for Training +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- :code:`model.data_dir`: the dataset directory + + +Optional Arguments +^^^^^^^^^^^^^^^^^^ + +Most of the default parameters in the existing configuration file are already set appropriately, however, there are some parameters +you may want to experiment with. + +- ``trainer.max_epochs``: the number of training epochs (reasonable to be between 10 to 100) +- ``model.class_balancing`` - value ``weighted_loss`` may help to train the model when there is unbalanced set of classes +- ``model.intent_loss_weight`` - a number between 0 to 1 that defines a weight of the intent lost versus a slot loss during training. A +default value 0.6 gives a slight preference for the intent lose optimization. + +Training Procedure +^^^^^^^^^^^^^^^^^^ + +At the start of evaluation, NeMo will print out a log of the experiment specification, a summary of the training dataset, and the +model architecture. + +As the model starts training, you should see a progress bar per epoch. During training, after each epoch, NeMo will display accuracy +metrics on the validation dataset for every intent and slot separately, as well as the total accuracy. You can expect these numbers +to grow up to 50-100 epochs, depending on the size of the trained data. Since this is a joint iIntent and slot training, usually +intent's accuracy will grow first for the initial 10-20 epochs, and after that, slot's accuracy will start improving as well. + +At the end of training, NeMo saves the best checkpoint on the validation dataset at the path specified by the experiment spec file +before finishing. + +.. code:: + + GPU available: True, used: True + TPU available: None, using: 0 TPU cores + LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2] + [NeMo W 2021-01-28 14:52:19 exp_manager:299] There was no checkpoint folder at checkpoint_dir :results/checkpoints. Training from scratch. + [NeMo I 2021-01-28 14:52:19 exp_manager:186] Experiments will be logged at results + ... + label precision recall f1 support + weather.weather (label_id: 0) 0.00 0.00 0.00 128 + weather.temperature (label_id: 1) 0.00 0.00 0.00 0 + weather.temperature_yes_no (label_id: 2) 0.00 0.00 0.00 0 + weather.rainfall (label_id: 3) 0.00 0.00 0.00 0 + weather.rainfall_yes_no (label_id: 4) 0.00 0.00 0.00 0 + weather.snow (label_id: 5) 0.00 0.00 0.00 0 + weather.snow_yes_no (label_id: 6) 0.00 0.00 0.00 0 + weather.humidity (label_id: 7) 0.00 0.00 0.00 0 + weather.humidity_yes_no (label_id: 8) 0.00 0.00 0.00 0 + weather.windspeed (label_id: 9) 0.00 0.00 0.00 0 + weather.sunny (label_id: 10) 0.00 0.00 0.00 0 + weather.cloudy (label_id: 11) 0.00 0.00 0.00 0 + weather.alert (label_id: 12) 0.00 0.00 0.00 0 + context.weather (label_id: 13) 0.00 0.00 0.00 0 + context.continue (label_id: 14) 0.00 0.00 0.00 0 + context.navigation (label_id: 15) 0.00 0.00 0.00 0 + context.rating (label_id: 16) 0.00 0.00 0.00 0 + context.distance (label_id: 17) 0.00 0.00 0.00 0 + ------------------- + micro avg 0.00 0.00 0.00 128 + macro avg 0.00 0.00 0.00 128 + weighted avg 0.00 0.00 0.00 128 + +Model Evaluation and Inference +------------------------------ + +There is no separate script for the evaluation and inference of this model in NeMo, however, inside of the example file `examples/nlp/intent_slot_classification/intent_slot_classification.py` +after the training part is finished, you can see the code that evaluates the trained model on an evaluation test set and then an example of doing inference using a list of given queries. + +For the deployment in the production environment, refer to `NVIDIA Jarvis `__ and `NVIDIA TLT documentation `__. + +References +---------- + +.. bibliography:: nlp_all.bib + :style: plain + :labelprefix: NLP-JIS + :keyprefix: nlp-jis- \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/nlp/language_modeling.rst b/models/tacotron2/NeMo/docs/source/nlp/language_modeling.rst new file mode 100644 index 0000000000..5565e5d191 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/language_modeling.rst @@ -0,0 +1,6 @@ +.. _language_modeling: + +Language Modeling +================= + +An example script on how to train the model can be found here: `NeMo/examples/nlp/language_modeling `__. diff --git a/models/tacotron2/NeMo/docs/source/nlp/machine_translation.rst b/models/tacotron2/NeMo/docs/source/nlp/machine_translation.rst new file mode 100644 index 0000000000..8f3ed7d53c --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/machine_translation.rst @@ -0,0 +1,466 @@ +.. _machine_translation: + +Machine Translation Models +========================== +Machine translation is the task of translating text from one language to another. For example, from English to Spanish. Models are +based on the Transformer sequence-to-sequence architecture :cite:`nlp-machine_translation-vaswani2017attention`. + +An example script on how to train the model can be found here: `NeMo/examples/nlp/machine_translation/enc_dec_nmt.py `__. +The default configuration file for the model can be found at: `NeMo/examples/nlp/machine_translation/conf/aayn_base.yaml `__. + +Quick Start Guide +----------------- + +.. code-block:: python + + from nemo.collections.nlp.models import MTEncDecModel + + # To get the list of pre-trained models + MTEncDecModel.list_available_models() + + # Download and load the a pre-trained to translate from English to Spanish + model = MTEncDecModel.from_pretrained("nmt_en_es_transformer12x2") + + # Translate a sentence or list of sentences + translations = model.translate(["Hello!"], source_lang="en", target_lang="es") + +Available Models +^^^^^^^^^^^^^^^^ + +.. list-table:: *Pretrained Models* + :widths: 5 10 + :header-rows: 1 + + * - Model + - Pretrained Checkpoint + * - English -> German + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_de_transformer12x2 + * - German -> English + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_de_en_transformer12x2 + * - English -> Spanish + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_es_transformer12x2 + * - Spanish -> English + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_es_en_transformer12x2 + * - English -> French + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_fr_transformer12x2 + * - French -> English + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_fr_en_transformer12x2 + * - English -> Russian + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_ru_transformer6x6 + * - Russian -> English + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_ru_en_transformer6x6 + * - English -> Chinese + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_zh_transformer6x6 + * - Chinese -> English + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_zh_en_transformer6x6 + +Data Format +----------- + +Supervised machine translation models require parallel corpora which comprises many examples of sentences in a source language and +their corresponding translation in a target language. We use parallel data formatted as seperate text files for source and target +languages where sentences in corresponding files are aligned like in the table below. + +.. list-table:: *Parallel Coprus* + :widths: 10 10 + :header-rows: 1 + + * - train.english.txt + - train.spanish.txt + * - Hello . + - Hola . + * - Thank you . + - Gracias . + * - You can now translate from English to Spanish in NeMo . + - Ahora puedes traducir del inglés al español en NeMo . + +It is common practice to apply data cleaning, normalization, and tokenization to the data prior to training a translation model and +NeMo expects already cleaned, normalized, and tokenized data. The only data pre-processing NeMo does is subword tokenization with BPE +:cite:`nlp-machine_translation-sennrich2015neural`. + +Data Cleaning, Normalization & Tokenization +------------------------------------------- + +We recommend applying the following steps to clean, normalize, and tokenize your data. All pre-trained models released, apply these data pre-processing steps. + +#. Language ID filtering - This step filters out examples from your training dataset that aren't in the correct language. For example, + many datasets contain examples where source and target sentences are in the same language. You can use a pre-trained language ID + classifier from `fastText `__. You can then run our script using the + ``lid.176.bin`` model downloaded from the fastText website. + + .. code :: + + python NeMo/scripts/neural_machine_translation/filter_by_language.py \ + --input-src train.en \ + --input-tgt train.es \ + --output-src train_lang_filtered.en \ + --output-tgt train_lang_filtered.es \ + --source-lang en \ + --target-lang es \ + --removed-src train_noise.en \ + --removed-tgt train_noise.de \ + --fasttext-model lid.176.bin + +#. Length filtering - We filter out sentences from the data that are below a minimum length (1) or exceed a maximum length (250). We + also filter out sentences where the ratio between source and target lengths exceeds 1.3 except for English <-> Chinese models. + `Moses `__ is a statistical machine translation toolkit that contains many useful + pre-processing scripts. + + .. code :: + + perl mosesdecoder/scripts/training/clean-corpus-n.perl -ratio 1.3 train en es train.filter 1 250 + +#. Data cleaning - While language ID filtering can sometimes help with filtering out noisy sentences that contain too many puncutations, + it does not help in cases where the translations are potentially incorrect, disfluent, or incomplete. We use `bicleaner `__ + a tool to identify such sentences. It trains a classifier based on many features included pre-trained language model fluency, word + alignment scores from a word-alignment model like `Giza++ `__ etc. We use their available + pre-trained models wherever possible and train models ourselves using their framework for remaining languages. The following script + applies a pre-trained bicleaner model to the data and pick sentences that are clean with probability > 0.5. + + .. code :: + + awk '{print "-\t-"}' train.en \ + | paste -d "\t" - train.filter.en train.filter.es \ + | bicleaner-classify - - > train.en-es.bicleaner.score + +#. Data deduplication - We use `bifixer `__ (which uses xxHash) to hash the source and target + sentences based on which we remove duplicate entries from the file. You may want to do something similar to remove training examples + that are in the test dataset. + + .. code :: + + cat train.en-es.bicleaner.score \ + | parallel -j 25 --pipe -k -l 30000 python bifixer.py --ignore-segmentation -q - - en es \ + > train.en-es.bifixer.score + + awk -F awk -F "\t" '!seen[$6]++' train.en-es.bifixer.score > train.en-es.bifixer.dedup.score + +#. Filter out data that bifixer assigns probability < 0.5 to. + + .. code :: + + awk -F "\t" '{ if ($5>0.5) {print $3}}' train.en-es.bifixer.dedup.score > train.cleaned.en + awk -F "\t" '{ if ($5>0.5) {print $4}}' train.en-es.bifixer.dedup.score > train.cleaned.es + +#. Punctuation Normalization - Punctuation, especially things like quotes can be written in different ways. + It's often useful to normalize the way they appear in text. We use the moses punctuation normalizer on all languages except Chinese. + + .. code :: + + perl mosesdecoder/scripts/tokenizer/normalize-punctuation.perl -l es < train.cleaned.es > train.normalized.es + perl mosesdecoder/scripts/tokenizer/normalize-punctuation.perl -l en < train.cleaned.en > train.normalized.en + + For example: + + .. code :: + + Before - Aquí se encuentran joyerías como Tiffany`s entre negocios tradicionales suizos como la confitería Sprüngli. + After - Aquí se encuentran joyerías como Tiffany's entre negocios tradicionales suizos como la confitería Sprüngli. + +#. Tokenization and word segmentation for Chinese - Naturally written text often contains punctuation markers like commas, full-stops + and apostrophes that are attached to words. Tokenization by just splitting a string on spaces will result in separate token IDs for + very similar items like ``NeMo`` and ``NeMo.``. Tokenization splits punctuation from the word to create two separate tokens. In the + previous example ``NeMo.`` becomes ``NeMo .`` which when split by space, results in two tokens and adressess the earlier problem. + + For example: + + .. code :: + + Before - Especialmente porque se enfrentará "a Mathieu (Debuchy), Yohan (Cabaye) y Adil (Rami) ", recuerda. + After - Especialmente porque se enfrentará " a Mathieu ( Debuchy ) , Yohan ( Cabaye ) y Adil ( Rami ) " , recuerda . + + We use the Moses tokenizer for all languages except Chinese. + + .. code :: + + perl mosesdecoder/scripts/tokenizer/tokenizer.perl -l es -no-escape < train.normalized.es > train.tokenized.es + perl mosesdecoder/scripts/tokenizer/tokenizer.perl -l en -no-escape < train.normalized.en > train.tokenized.en + + For languages like Chinese where there is no explicit marker like spaces that separate words, we use `Jieba `__ to segment a string into words that are space separated. + + For example: + + .. code :: + + Before - 同时，卫生局认为有必要接种的其他人员，包括公共部门，卫生局将主动联络有关机构取得名单后由卫生中心安排接种。 + After - 同时 ， 卫生局 认为 有 必要 接种 的 其他 人员 ， 包括 公共部门 ， 卫生局 将 主动 联络 有关 机构 取得 名单 后 由 卫生 中心 安排 接种 。 + +Training a BPE Tokenization +--------------------------- + +Byte-pair encoding (BPE) :cite:`nlp-machine_translation-sennrich2015neural` is a sub-word tokenization algorithm that is commonly used +to reduce the large vocabulary size of datasets by splitting words into frequently occuring sub-words. Currently, mMachine translation +only supports the `YouTokenToMe `__ BPE tokenizer. One can set the tokenization configuration +as follows: + ++-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Default** | **Description** | ++-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+ +| **model.{encoder_tokenizer,decoder_tokenizer}.tokenizer_name** | str | ``yttm`` | BPE library name. Only supports ``yttm`` for now. | ++-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+ +| **model.{encoder_tokenizer,decoder_tokenizer}.tokenizer_model** | str | ``null`` | Path to an existing YTTM BPE model. If ``null``, will train one from scratch on the provided data. | ++-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+ +| **model.{encoder_tokenizer,decoder_tokenizer}.vocab_size** | int | ``null`` | Desired vocabulary size after BPE tokenization. | ++-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+ +| **model.{encoder_tokenizer,decoder_tokenizer}.bpe_dropout** | float | ``null`` | BPE dropout probability. :cite:`nlp-machine_translation-provilkov2019bpe`. | ++-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+ +| **model.{encoder_tokenizer,decoder_tokenizer}.vocab_file** | str | ``null`` | Path to pre-computed vocab file if exists. | ++-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+ +| **model.shared_tokenizer** | bool | ``True`` | Whether to share the tokenizer between the encoder and decoder. | ++-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+ + + +Applying BPE Tokenization, Batching, Bucketing and Padding +---------------------------------------------------------- + +Given BPE tokenizers, and a cleaned parallel corpus, the following steps are applied to create a `TranslationDataset `__ object. + +#. Text to IDs - This performs subword tokenization with the BPE model on an input string and maps it to a sequence of tokens for the + source and target text. + +#. Bucketing - Sentences vary in length and when creating minibatches, we'd like sentences in them to have roughly the same length to + minimize the number of ```` tokens and to maximize computational efficiency. This step groups sentences roughly the same length + into buckets. + +#. Batching and padding - Creates minibatches with a maximum number of tokens specified by ``model.{train_ds,validation_ds,test_ds}.tokens_in_batch`` + from buckets and pads, so they can be packed into a tensor. + +Datasets can be configured as follows: + ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Default** | **Description** | ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.src_file_name** | str | ``null`` | Path to the source language file. | ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.tgt_file_name** | str | ``null`` | Path to the target language file. | ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.tokens_in_batch** | int | ``512`` | Maximum number of tokens per minibatch. | ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.clean** | bool | ``true`` | Whether to clean the dataset by discarding examples that are greater than ``max_seq_length``. | ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.max_seq_length** | int | ``512`` | Maximum sequence to be used with the ``clean`` argument above. | ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.shuffle** | bool | ``true`` | Whether to shuffle minibatches in the PyTorch DataLoader. | ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.num_samples** | int | ``-1`` | Number of samples to use. ``-1`` for the entire dataset. | ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.drop_last** | bool | ``false`` | Drop last minibatch if it is not of equal size to the others. | ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.pin_memory** | bool | ``false`` | Whether to pin memory in the PyTorch DataLoader. | ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.num_workers** | int | ``8`` | Number of workers for the PyTorch DataLoader. | ++-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+ + + +Tarred Datasets for Large Corpora +--------------------------------- + +When training with ``DistributedDataParallel``, each process has its own copy of the dataset. For large datasets, this may not always +fit in CPU memory. `Webdatasets `__ circumvents this problem by efficiently iterating over +tar files stored on disk. Each tar file can contain hundreds to thousands of pickle files, each containing a single minibatch. + +We recommend using this method when working with datasets with > 1 million sentence pairs. + +Tarred datasets can be configured as follows: + ++-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Default** | **Description** | ++-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.use_tarred_dataset** | bool | ``false`` | Whether to use tarred datasets. | ++-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.tar_files** | str | ``null`` | String specifying path to all tar files. Example with 100 tarfiles ``/path/to/tarfiles._OP_1..100_CL_.tar``. | ++-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.metadata_file** | str | ``null`` | Path to JSON metadata file that contains only a single entry for the total number of batches in the dataset. | ++-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.lines_per_dataset_fragment** | int | ``1000000`` | | ++-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.num_batches_per_tarfile** | int | ``100`` | Maximum sequence to be used with the ``clean`` argument above. | ++-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.tar_shuffle_n** | int | ``100`` | Whether to cache IDs to avoid re-tokenizing data. This will be deprecated in favor of tarred datasets. | ++-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+ +| **model.{train_ds,validation_ds,test_ds}.shard_strategy** | str | ``scatter`` | Whether to cache IDs in each of the nodes in multi-node training. | ++-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+ +| **model.preproc_out_dir** | str | ``null`` | Path to folder that contains processed tar files or directory where new tar files are written. | ++-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+ + +Tarred datasets can be created in two ways: + +#. Using the Hydra config and `training script `__. + + For example: + + .. code :: + + python examples/nlp/machine_translation/enc_dec_nmt.py \ + -cn aayn_base \ + do_training=false \ + model.preproc_out_dir=/path/to/preproc_dir \ + model.train_ds.use_tarred_dataset=true \ + model.train_ds.lines_per_dataset_fragment=1000000 \ + model.train_ds.num_batches_per_tarfile=200 \ + model.train_ds.src_file_name=train.tokenized.en \ + model.train_ds.tgt_file_name=train.tokenized.es \ + model.validation_ds.src_file_name=validation.tokenized.en \ + model.validation_ds.tgt_file_name=validation.tokenized.es \ + model.encoder_tokenizer.vocab_size=32000 \ + model.decoder_tokenizer.vocab_size=32000 \ + ~model.test_ds \ + trainer.gpus=[0,1,2,3] \ + +trainer.fast_dev_run=true \ + exp_manager=null \ + + The above script processes the parallel tokenized text files into tarred datasets that are written to ``/path/to/preproc_dir``. Since + ``do_training`` is set to ``False``, the above script only creates tarred datasets and then exits. If ``do_training`` is set ``True``, + then one of two things happen: + + (a) If no tar files are present in ``model.preproc_out_dir``, the script first creates those files and then commences training. + (b) If tar files are already present in ``model.preproc_out_dir``, the script starts training from the provided tar files. + +#. Using a separate script without Hydra. + + Tarred datasets for parallel corpora can also be created with a script that doesn't require specifying a configs via Hydra and + just uses Python argparse. + + For example: + + .. code :: + + python examples/nlp/machine_translation/create_tarred_parallel_dataset.py \ + --shared_tokenizer \ + --clean \ + --bpe_dropout 0.1 \ + --src_fname train.tokenized.en \ + --tgt_fname train.tokenized.es \ + --out_dir /path/to/preproc_dir \ + --vocab_size 32000 \ + --max_seq_length 512 \ + --min_seq_length 1 \ + --tokens_in_batch 8192 \ + --lines_per_dataset_fragment 1000000 \ + --num_batches_per_tarfile 200 + + You can then set `model.preproc_out_dir=/path/to/preproc_dir` and `model.train_ds.use_tarred_dataset=true` to train with this data. + +Model Configuration and Training +-------------------------------- + +The overall model consists of an encoder, decoder, and classification head. Encoders and decoders have the following configuration +options: + ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Default** | **Description** | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.max_sequence_length** | int | ``512`` | Maximum sequence length of positional encodings. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.embedding_dropout** | float | ``0.1`` | Path to JSON metadata file that contains only a single entry for the total number of batches in the dataset. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.learn_positional_encodings** | bool | ``false`` | If ``True``, this is a regular learnable embedding layer. If ``False``, fixes position encodings to sinusoidal. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.hidden_size** | int | ``512`` | Size of the transformer hidden states. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.num_layers** | int | ``6`` | Number of transformer layers. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.inner_size** | int | ``2048`` | Size of the hidden states within the feedforward layers. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.num_attention_heads** | int | ``8`` | Number of attention heads. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.ffn_dropout** | float | ``0.1`` | Dropout probability within the feedforward layers. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.attn_score_dropout** | float | ``0.1`` | Dropout probability of the attention scores before softmax normalization. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.attn_layer_dropout** | float | ``0.1`` | Dropout probability of the attention query, key, and value projection activations. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.hidden_act** | str | ``relu`` | Activation function throughout the network. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.mask_future** | bool | ``false``, ``true`` | Whether to mask future timesteps for attention. Defaults to ``True`` for decoder and ``False`` for encoder. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ +| **model.{encoder,decoder}.pre_ln** | bool | ``false`` | Whether to apply layer-normalization before (``true``) or after (``false``) a sub-layer. | ++-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+ + +Our pre-trained models are optimized with Adam, with a maximum learning of 0.0004, beta of (0.9, 0.98), an inverse square root learning +rate schedule from :cite:`nlp-machine_translation-vaswani2017attention`. The **model.optim** section sets the optimization parameters. + +The following script creates tarred datasets based on the provided parallel corpus and train a model based on the ``base`` configuration +from :cite:`nlp-machine_translation-vaswani2017attention`. + +.. code :: + + python examples/nlp/machine_translation/enc_dec_nmt.py \ + -cn aayn_base \ + do_training=true \ + trainer.gpus=8 \ + ~trainer.max_epochs \ + +trainer.max_steps=100000 \ + +trainer.val_check_interval=1000 \ + +exp_manager.exp_dir=/path/to/store/results \ + +exp_manager.create_checkpoint_callback=True \ + +exp_manager.checkpoint_callback_params.monitor=val_sacreBLEU \ + +exp_manager.checkpoint_callback_params.mode=max \ + +exp_manager.checkpoint_callback_params.save_top_k=5 \ + model.preproc_out_dir=/path/to/preproc_dir \ + model.train_ds.use_tarred_dataset=true \ + model.train_ds.lines_per_dataset_fragment=1000000 \ + model.train_ds.num_batches_per_tarfile=200 \ + model.train_ds.src_file_name=train.tokenized.en \ + model.train_ds.tgt_file_name=train.tokenized.es \ + model.validation_ds.src_file_name=validation.tokenized.en \ + model.validation_ds.tgt_file_name=validation.tokenized.es \ + model.encoder_tokenizer.vocab_size=32000 \ + model.decoder_tokenizer.vocab_size=32000 \ + ~model.test_ds \ + +The trainer keeps track of the sacreBLEU score :cite:`nlp-machine_translation-post2018call` on the provided validation set and saves +the checkpoints that had the top 5 (by default) sacreBLEU scores. + +At the end of training, a ``.nemo`` file is written to the result directory which allows to run inference on a test set. + +Multi-Validation +---------------- + +To run validation on multiple datasets, specify ``validation_ds.src_file_name`` and ``validation_ds.tgt_file_name`` with a list of file paths: + +.. code-block:: bash + + model.validation_ds.src_file_name=[/data/wmt13-en-de.src,/data/wmt14-en-de.src] \ + model.validation_ds.tgt_file_name=[/data/wmt13-en-de.ref,/data/wmt14-en-de.ref] \ + +When using ``val_loss`` or ``val_sacreBLEU`` for the ``exp_manager.checkpoint_callback_params.monitor`` +then the 0th indexed dataset will be used as the monitor. + +To use other indexes, append the index: + +.. code-block:: bash + + exp_manager.checkpoint_callback_params.monitor=val_sacreBLEU_dl_index_1 + +Multiple test datasets work exactly the same way as validation datasets, simply replace ``validation_ds`` by ``test_ds`` in the above examples. + +Model Inference +--------------- + +To generate translations on a test set and compute sacreBLEU scores, run the inference script: + +.. code :: + + python examples/nlp/machine_translation/nmt_transformer_infer.py \ + --model /path/to/model.nemo \ + --srctext test.en \ + --tgtout test.en-es.translations \ + --batch_size 128 \ + --source_lang en \ + --target_lang es + +The ``--srctext`` file must be provided before tokenization and normalization. The resulting ``--tgtout`` file is detokenized and +can be used to compute sacreBLEU scores. + +.. code :: + + cat test.en-es.translations | sacrebleu test.es + +References +---------- + +.. bibliography:: nlp_all.bib + :style: plain + :labelprefix: nlp-machine_translation + :keyprefix: nlp-machine_translation- diff --git a/models/tacotron2/NeMo/docs/source/nlp/megatron_finetuning.rst b/models/tacotron2/NeMo/docs/source/nlp/megatron_finetuning.rst new file mode 100644 index 0000000000..fec82f2d1d --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/megatron_finetuning.rst @@ -0,0 +1,216 @@ +.. _megatron_finetuning: + +Megatron-LM for Downstream Tasks +================================ + +Megatron-LM :cite:`nlp-megatron-shoeybi2019megatron` is a large, powerful transformer developed by the Applied Deep Learning Research +team at NVIDIA. Unlike BERT, the position of the layer normalization and the residual connection in the model architecture (similar to +GPT-2 architucture) are swapped, which allows the models to continue to improve as they were scaled up. This model reaches higher +scores compared to BERT on a range of Natural Language Processing (NLP) tasks. More details on efficient, model-parallel and multi-node +pre-training of GPT and BERT using mixed precision can be found in the `Megatron-LM GitHub repo `_. + + +Fine-tuning +----------- + +Pre-trained Megatron-LM (BERT) can be used for most of the NLP downstream tasks from `NeMo/examples/nlp `_. +Specify the ``model.language_model.pretrained_model_name`` parameter: + +.. code-block:: bash + + model.language_model.pretrained_model_name=megatron-bert-345m-uncased + +Available pre-trained Megatron-LM models: + +- ``megatron-bert-345m-cased`` +- ``megatron-bert-345m-uncased`` +- ``biomegatron-bert-345m-uncased`` +- ``biomegatron-bert-345m-cased`` + +For example, to fine-tune SQuAD v1.1 with Megatron-LM, run: + +.. code-block:: bash + + python question_answering_squad.py \ + model.train_ds.file= \ + model.validation_ds= \ + model.language_model.pretrained_model_name=megatron-bert-345m-uncased + +If you have a different checkpoint or model configuration (pre-trained with `Megatron-LM GitHub repo `_), +use ``model.language_model.pretrained_model_name=megatron-bert-uncased`` or ``model.language_model.pretrained_model_name=megatron-bert-cased`` +and specify ``--bert_config`` and ``--bert_checkpoint`` for your model. + +.. note:: + Megatron-LM has its own set of training arguments (including tokenizer) that are ignored during fine-tuning in NeMo. Use downstream + task training scripts for all NeMo supported arguments. + +BioMegatron +----------- + +BioMegatron has the same network architecture as the Megatron-LM, but is pretrained on a different dataset - `PubMed `_, +a large biomedical text corpus, which achieves better performance in biomedical downstream tasks than the original Megatron-LM. + +Examples of using BioMegatron on biomedical downstream tasks can be found at (can be executed with `Google's Colab `_): +`NeMo/tutorials/nlp/Relation_Extraction-BioMegatron.ipynb `__ and `NeMo/tutorials/nlp/Token_Classification-BioMegatron.ipynb `__. + +Model Parallelism +----------------- + +`Megatron-LM `_ is a highly optimized and efficient library for training large language models. +With Megatron model parallelism, language models can be trained with billions of weights and then used in NeMo for downstream tasks. + +NeMo handles pretrained model parallel checkpoints from Megatron-LM automatically and model parallel models in NeMo have the all +the same features as other NeMo Models. + +.. note:: + + Currently, NeMo only supports tensor model parallelism. + +Training +^^^^^^^^ + +All of the necessary logic to train model parallel models in NeMo with PyTorch Lightning is contained in the ``NLPDDPPlugin``. +The ``NLPDDPPlugin`` subclasses the PyTorch Lightning training type plugin ``DDPPlugin``. +See `plugins `_ for more information on PyTorch Lightning Plugins. + +To enable model parallel training in NeMo: + +.. code-block:: python + + trainer = Trainer(plugins=[NLPDDPPlugin()], **cfg.trainer) + +Megatron-LM checkpoints have a specific format. One checkpoint is saved for each model parallel rank: + +.. code-block:: bash + + iter_0080000/ + ├── mp_rank_00 + │ └── model_optim_rng.pt + └── mp_rank_01 + └── model_optim_rng.pt + + +To start fine-tuning from a Megatron-LM checkpoint, simply pass the path to the Megatron-LM checkpoint +via the language model config: + +.. code-block:: bash + + model.language_model.lm_checkpoint=/raid/megatron/bert/iter_0080000 \ + +We also need to input the model configuration. This can be done via json: + +.. code-block:: json + + { + "hidden-size": 1024, + "num-attention-heads": 16, + "num-layers": 24, + "max-seq-length": 512 + } + +And input via command line: + +.. code-block:: bash + + model.language_model.config_file=/raid/data/megatron/bert/config.json \ + +Or the model configuration can be input via YAML: + +.. code-block:: YAML + + model: + language_model: + config: + hidden_size: 1024 + num_attention_heads: 16 + num_layers: 24 + max_position_embeddings: 512 + +Additionally, Megatron-LM requires a vocab file: + +.. code-block:: bash + + model.tokenizer.vocab_file=/path/to/vocab.txt + +If using the Megatron-LM default tokenizer for training BERT the vocab file can be omitted: + +.. code-block:: bash + + # uncased model + model.tokenizer.tokenizer_name=megatron-bert-uncased + +.. code-block:: bash + + # cased model + model.tokenizer.tokenizer_name=megatron-bert-uncased + +Auto-Resume +^^^^^^^^^^^ + +Resuming training with NeMo experiment manager and PyTorch Lightning works exactly the same as other NeMo models. +While training with PTL, model parallel checkpoint will be saved and loaded properly. + +.. code-block:: bash + + checkpoints/ + ├── mp_rank_00 + │ ├── mp_autoresume-last.ckpt + │ ├── mp_autoresume---val_loss=0.35-epoch=0.ckpt + │ ├── mp_autoresume---val_loss=0.38-epoch=1.ckpt + │ └── mp_autoresume---val_loss=0.39-epoch=2.ckpt + └── mp_rank_01 + ├── mp_autoresume-last.ckpt + ├── mp_autoresume---val_loss=0.35-epoch=0.ckpt + ├── mp_autoresume---val_loss=0.38-epoch=1.ckpt + └── mp_autoresume---val_loss=0.39-epoch=2.ckpt + +Save and Restore +^^^^^^^^^^^^^^^^ + +Model parallel .nemo files behave the same as all other .nemo files. Calling ``.save_to`` will save +a checkpoint for each model parallel rank inside the .nemo file: + +.. code-block:: bash + + text_class_350m + ├── megatron-bert-uncased_encoder_config.json + ├── megatron_checkpoint_version.json + ├── model_config.yaml + ├── mp_rank_00 + │ └── model_weights.ckpt + ├── mp_rank_01 + │ └── model_weights.ckpt + ├── tokenizer_vocab_dict.json + └── tokenizer.vocab_file + +When restoring a model parallel .nemo file, we must pass in the ``Trainer`` as model parallel requires DDP: + +.. code-block:: python + + model = TokenClassificationModel.restore_from(cfg.pretrained_model, trainer=trainer) + +Evaluation +^^^^^^^^^^ + +Since model parallel models always require more than one GPU, the ``Trainer`` is needed for evaluation: + +.. code-block:: python + + trainer = pl.Trainer(plugins=[NLPDDPPlugin()], **cfg.trainer) + + model = TextClassificationModel.restore_from(cfg.model.nemo_path, trainer=trainer) + model.setup_test_data(test_data_config=cfg.model.test_ds) + + trainer.test(model=model, ckpt_path=None) + + + + + +References +---------- + +.. bibliography:: nlp_all.bib + :style: plain + :labelprefix: NLP-MEGATRON + :keyprefix: nlp-megatron- \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/nlp/models.rst b/models/tacotron2/NeMo/docs/source/nlp/models.rst new file mode 100755 index 0000000000..c6b9adbad0 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/models.rst @@ -0,0 +1,22 @@ +.. _nlp_models: + +Models +====== + +NeMo's NLP collection supports the following models: + +.. toctree:: + :maxdepth: 1 + + punctuation_and_capitalization + token_classification + joint_intent_slot + text_classification + bert_pretraining + language_modeling + question_answering + sgd_qa + glue_benchmark + information_retrieval + nlp_model + machine_translation diff --git a/models/tacotron2/NeMo/docs/source/nlp/nlp_all.bib b/models/tacotron2/NeMo/docs/source/nlp/nlp_all.bib new file mode 100644 index 0000000000..f291310941 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/nlp_all.bib @@ -0,0 +1,109 @@ +@article{devlin2018bert, + title={Bert: Pre-training of deep bidirectional transformers for language understanding}, + author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina}, + journal={arXiv preprint arXiv:1810.04805}, + year={2018} +} + +@article{shoeybi2019megatron, + title={Megatron-lm: Training multi-billion parameter language models using model parallelism}, + author={Shoeybi, Mohammad and Patwary, Mostofa and Puri, Raul and LeGresley, Patrick and Casper, Jared and Catanzaro, Bryan}, + journal={arXiv preprint arXiv:1909.08053}, + year={2019} +} + +@InProceedings{maas2011, + author = {Maas, Andrew L. and Daly, Raymond E. and Pham, Peter T. and Huang, Dan and Ng, Andrew Y. and Potts, Christopher}, + title = {Learning Word Vectors for Sentiment Analysis}, + booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies}, + month = {June}, + year = {2011}, + address = {Portland, Oregon, USA}, + publisher = {Association for Computational Linguistics}, + pages = {142--150}, + url = {http://www.aclweb.org/anthology/P11-1015} +} + +@inproceedings{socher2013, + title = "Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank", + author = "Socher, Richard and Perelygin, Alex and Wu, Jean and Chuang, Jason and Manning, Christopher D. and Ng, Andrew and Potts, Christopher", + booktitle = "Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing", + month = oct, + year = "2013", + address = "Seattle, Washington, USA", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/D13-1170", + pages = "1631--1642", +} + +@article{lim2018chemical, + title={Chemical--gene relation extraction using recursive neural network}, + author={Lim, Sangrak and Kang, Jaewoo}, + journal={Database}, + volume={2018}, + year={2018}, + publisher={Oxford Academic} +} + +@inproceedings{li2007scalable, + title={Scalable term selection for text categorization}, + author={Li, Jingyang and Sun, Maosong}, + booktitle={Proceedings of the 2007 Joint Conference on Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP-CoNLL)}, + pages={774--782}, + year={2007} +} + +@misc{lee2019biobert, + title={BioBERT: a pre-trained biomedical language representation model for biomedical text mining}, + author={Jinhyuk Lee and Wonjin Yoon and Sungdong Kim and Donghyeon Kim and Sunkyu Kim and Chan Ho So and Jaewoo Kang}, + year={2019}, + eprint={1901.08746}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} + +@misc{shin2020biomegatron, + title={BioMegatron: Larger Biomedical Domain Language Model}, + author={Hoo-Chang Shin and Yang Zhang and Evelina Bakhturina and Raul Puri and Mostofa Patwary and Mohammad Shoeybi and Raghav Mani}, + year={2020}, + eprint={2010.06060}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} + +@article{vaswani2017attention, + title={Attention is all you need}, + author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, Lukasz and Polosukhin, Illia}, + journal={arXiv preprint arXiv:1706.03762}, + year={2017} +} + +@article{sennrich2015neural, + title={Neural machine translation of rare words with subword units}, + author={Sennrich, Rico and Haddow, Barry and Birch, Alexandra}, + journal={arXiv preprint arXiv:1508.07909}, + year={2015} +} + +@article{provilkov2019bpe, + title={Bpe-dropout: Simple and effective subword regularization}, + author={Provilkov, Ivan and Emelianenko, Dmitrii and Voita, Elena}, + journal={arXiv preprint arXiv:1910.13267}, + year={2019} +} + +@article{post2018call, + title={A call for clarity in reporting BLEU scores}, + author={Post, Matt}, + journal={arXiv preprint arXiv:1804.08771}, + year={2018} +} + +@misc{zhang2021sgdqa, + title={SGD-QA: Fast Schema-Guided Dialogue State Tracking for Unseen Services}, + author={Yang Zhang and Vahid Noroozi and Evelina Bakhturina and Boris Ginsburg}, + year={2021}, + eprint={2105.08049}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/nlp/nlp_model.rst b/models/tacotron2/NeMo/docs/source/nlp/nlp_model.rst new file mode 100755 index 0000000000..8c47518c40 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/nlp_model.rst @@ -0,0 +1,43 @@ +.. _nlp_model: + +Model NLP +========= + +The config file for NLP models contain three main sections: + + - ``trainer``: contains the configs for PTL training. For more information, refer to :doc:`../core/core` and `PTL Trainer class API '. + - ``exp_manager``: the configs of the experiment manager. For more information, refer to :doc:`../core/core`. + - ``model``: contains the configs of the datasets, model architecture, tokenizer, optimizer, scheduler, etc. + +The following sub-sections of the model section are shared among most of the NLP models. + + - ``tokenizer``: specifies the tokenizer + - ``language_model``: specifies the underlying model to be used as the encoder + - ``optim``: the configs of the optimizer and scheduler :doc:`../core/core` + +The ``tokenizer`` and ``language_model`` sections have the following parameters: + ++------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Description** | ++------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.tokenizer.tokenizer_name** | string | Tokenizer name will be filled automatically based on ``model.language_model.pretrained_model_name``. | ++------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.tokenizer.vocab_file** | string | Path to tokenizer vocabulary. | ++------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.tokenizer.tokenizer_model** | string | Path to tokenizer model (only for sentencepiece tokenizer). | ++------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.language_model.pretrained_model_name** | string | Pre-trained language model name, for example: ``bert-base-cased`` or ``bert-base-uncased``. | ++------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.language_model.lm_checkpoint** | string | Path to the pre-trained language model checkpoint. | ++------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.language_model.config_file** | string | Path to the pre-trained language model config file. | ++------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.language_model.config** | dictionary | Config of the pre-trained language model. | ++------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ + +The parameter **model.language_model.pretrained_model_name** can be one of the following: + + - ``megatron-bert-345m-uncased``, ``megatron-bert-345m-cased``, ``biomegatron-bert-345m-uncased``, ``biomegatron-bert-345m-cased``, ``bert-base-uncased``, ``bert-large-uncased``, ``bert-base-cased``, ``bert-large-cased`` + - ``distilbert-base-uncased``, ``distilbert-base-cased`` + - ``roberta-base``, ``roberta-large``, ``distilroberta-base`` + - ``albert-base-v1``, ``albert-large-v1``, ``albert-xlarge-v1``, ``albert-xxlarge-v1``, ``albert-base-v2``, ``albert-large-v2``, ``albert-xlarge-v2``, ``albert-xxlarge-v2`` diff --git a/models/tacotron2/NeMo/docs/source/nlp/punctuation_and_capitalization.rst b/models/tacotron2/NeMo/docs/source/nlp/punctuation_and_capitalization.rst new file mode 100755 index 0000000000..4cce7b34ce --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/punctuation_and_capitalization.rst @@ -0,0 +1,300 @@ +.. _punctuation_and_capitalization: + +Punctuation and Capitalization Model +==================================== + +Automatic Speech Recognition (ASR) systems typically generate text with no punctuation and capitalization of the words. +There are two issues with non-punctuated ASR output: + +- it could be difficult to read and understand +- models for some downstream tasks, such as named entity recognition, machine translation, or text-to-speech, are usually trained +on punctuated datasets and using raw ASR output as the input to these models could deteriorate their performance + +Quick Start Guide +----------------- + +.. code-block:: python + + from nemo.collections.nlp.models import PunctuationCapitalizationModel + + # to get the list of pre-trained models + PunctuationCapitalizationModel.list_available_models() + + # Download and load the pre-trained BERT-based model + model = PunctuationCapitalizationModel.from_pretrained("punctuation_en_bert") + + # try the model on a few examples + model.add_punctuation_capitalization(['how are you', 'great how about you']) + +Model Description +----------------- + +For each word in the input text, the Punctuation and Capitalization model: + +- predicts a punctuation mark that should follow the word (if any). By default, the model supports commas, periods, and question marks. +- predicts if the word should be capitalized or not + +In the Punctuation and Capitalization model, we are jointly training two token-level classifiers on top of a pre-trained +language model, such as `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding `__ :cite:`nlp-punct-devlin2018bert`. + +.. note:: + + We recommend you try this model in a Jupyter notebook (run on `Google's Colab `_.): `NeMo/tutorials/nlp/Punctuation_and_Capitalization.ipynb `__. + + Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator). + + An example script on how to train the model can be found at: `NeMo/examples/nlp/token_classification/punctuation_capitalization_train.py `__. + + An example script on how to run evaluation and inference can be found at: `NeMo/examples/nlp/token_classification/punctuation_capitalization_evaluate.py `__. + + The default configuration file for the model can be found at: `NeMo/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml `__. + +.. _raw_data_format_punct: + +Raw Data Format +--------------- + +The Punctuation and Capitalization model can work with any text dataset, although it is recommended to balance the data, especially +for the punctuation task. Before pre-processing the data to the format expected by the model, the data should be split into ``train.txt`` +and ``dev.txt`` (and optionally ``test.txt``). Each line in the ``train.txt/dev.txt/test.txt`` should represent one or more full +and/or truncated sentences. + +Example of the ``train.txt``/``dev.txt`` file: + +.. code:: + + When is the next flight to New York? + The next flight is ... + .... + + +The ``source_data_dir`` structure should look similar to the following: + +.. code:: + + . + |--sourced_data_dir + |-- dev.txt + |-- train.txt + +NeMo Data Format +---------------- + +The Punctuation and Capitalization model expects the data in the following format: + +The training and evaluation data is divided into 2 files: +- ``text.txt`` +- ``labels.txt`` + +Each line of the ``text.txt`` file contains text sequences, where words are separated with spaces. + +[WORD] [SPACE] [WORD] [SPACE] [WORD], for example: + + :: + + when is the next flight to new york + the next flight is ... + ... + +The ``labels.txt`` file contains corresponding labels for each word in ``text.txt``, the labels are separated with spaces. +Each label in ``labels.txt`` file consists of 2 symbols: + +- the first symbol of the label indicates what punctuation mark should follow the word (where ``O`` means no punctuation needed) +- the second symbol determines if a word needs to be capitalized or not (where ``U`` indicates that the word should be upper cased, +and ``O`` - no capitalization needed) + +By default, the following punctuation marks are considered: commas, periods, and question marks; the remaining punctuation marks were +removed from the data. This can be changed by introducing new labels in the ``labels.txt`` files. + +Each line of the ``labels.txt`` should follow the format: ``[LABEL] [SPACE] [LABEL] [SPACE] [LABEL]`` (for ``labels.txt``). For example, +labels for the above ``text.txt`` file should be: + + :: + + OU OO OO OO OO OO OU ?U + OU OO OO OO ... + ... + +The complete list of all possible labels used in this tutorial are: + +- ``OO`` +- ``O`` +- ``.O`` +- ``?O`` +- ``OU`` +- +- ``U`` +- ``.U`` +- ``?U`` + +Converting Raw Data to NeMo Format +---------------------------------- + +To pre-process the raw text data, stored under :code:`sourced_data_dir` (see the :ref:`raw_data_format_punct` +section), run the following command: + +.. code:: + + python examples/nlp/token_classification/data/prepare_data_for_punctuation_capitalization.py \ + -s + -o + + +Required Argument for Dataset Conversion +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- :code:`-s` or :code:`--source_file`: path to the raw file +- :code:`-o` or :code:`--output_dir` - path to the directory to store the converted files + +After the conversion, the :code:`output_dir` should contain :code:`labels_*.txt` and :code:`text_*.txt` files. The default names +for the training and evaluation in the :code:`conf/punctuation_capitalization_config.yaml` are the following: + +.. code:: + + . + |--output_dir + |-- labels_dev.txt + |-- labels_train.txt + |-- text_dev.txt + |-- text_train.txt + +Training Punctuation and Capitalization Model +--------------------------------------------- + +The language model is initialized with the pre-trained model from `HuggingFace Transformers `__, +unless the user provides a pre-trained checkpoint for the language model. Example of model configuration file for training the model can be found at: `NeMo/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml `__. + +The specification is roughly grouped into the following categories: + +- Parameters that describe the training process: **trainer** +- Parameters that describe the datasets: **model.dataset**, **model.train_ds**, **model.validation_ds** +- Parameters that describe the model: **model** + +More details about parameters in the config file can be found below and in the `model's config file `__: + ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Description** | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **pretrained_model** | string | Path to the pre-trained model ``.nemo`` file or pre-trained model name. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.dataset.data_dir** | string | Path to the data converted to the specified above format. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.punct_head.punct_num_fc_layers** | integer | Number of fully connected layers. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.punct_head.fc_dropout** | float | Activation to use between fully connected layers. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.punct_head.activation** | string | Dropout to apply to the input hidden states. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.punct_head.use_transrormer_init** | bool | Whether to initialize the weights of the classifier head with the same approach used in Transformer. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.capit_head.punct_num_fc_layers** | integer | Number of fully connected layers. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.capit_head.fc_dropout** | float | Dropout to apply to the input hidden states. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.capit_head.activation** | string | Activation function to use between fully connected layers. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.capit_head.use_transrormer_init** | bool | Whether to initialize the weights of the classifier head with the same approach used in Transformer. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **training_ds.text_file** | string | Name of the text training file located at ``data_dir``. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **training_ds.labels_file** | string | Name of the labels training file located at ``data_dir``, such as ``labels_train.txt``. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **training_ds.num_samples** | integer | Number of samples to use from the training dataset, ``-1`` - to use all. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **validation_ds.text_file** | string | Name of the text file for evaluation, located at ``data_dir``. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **validation_ds.labels_file** | string | Name of the labels dev file located at ``data_dir``, such as ``labels_dev.txt``. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **validation_ds.num_samples** | integer | Number of samples to use from the dev set, ``-1`` - to use all. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ + +For more information, refer to the :ref:`nlp_model` section. + +To train the model from scratch, run: + +.. code:: + + python examples/nlp/token_classification/punctuation_and_capitalization_train.py \ + model.dataset.data_dir= \ + trainer.gpus=[0,1] \ + optim.name=adam \ + optim.lr=0.0001 + +The above command will start model training on GPUs 0 and 1 with Adam optimizer and learning rate of 0.0001; and the trained model is +stored in the ``nemo_experiments/Punctuation_and_Capitalization`` folder. + +To train from the pre-trained model, run: + +.. code:: + + python examples/nlp/token_classification/punctuation_and_capitalization_train.py \ + model.dataset.data_dir= \ + pretrained_model= + + +Required Arguments for Training +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* :code:`model.dataset.data_dir`: Path to the `data_dir` with the pre-processed data files. + + +.. note:: + + All parameters defined in the configuration file can be changed with command arguments. For example, the sample config file + mentioned above has :code:`validation_ds.batch_size` set to ``64``. However, if you see that the GPU utilization can be + optimized further by using a larger batch size, you may override to the desired value by adding the field :code:`validation_ds.batch_size=128` + over the command-line. You can repeat this with any of the parameters defined in the sample configuration file. + +Inference +--------- + +An example script on how to run inference on a few examples, can be found at `examples/nlp/token_classification/punctuation_capitalization_evaluate.py `_. + +To start inference with a pre-trained model on a few examples, run: + +.. code:: + + python punctuation_capitalization_evaluate.py \ + pretrained_model= + + +Model Evaluation +---------------- + +An example script on how to evaluate the pre-trained model, can be found at `examples/nlp/token_classification/punctuation_capitalization_evaluate.py `_. + +To start evaluation of the pre-trained model, run: + +.. code:: + + python punctuation_capitalization_evaluate.py \ + model.dataset.data_dir= \ + pretrained_model=punctuation_en_bert \ + model.test_ds.text_file= \ + model.test_ds.labels_file= + + +Required Arguments +^^^^^^^^^^^^^^^^^^ + +- :code:`pretrained_model`: pretrained Punctuation and Capitalization model from ``list_available_models()`` or path to a ``.nemo`` +file. For example: ``punctuation_en_bert`` or ``your_model.nemo``. +- :code:`model.dataset.data_dir`: path to the directory that containes :code:`model.test_ds.text_file` and :code:`model.test_ds.labels_file` + +During evaluation of the :code:`test_ds`, the script generates two classification reports: one for capitalization task and another +one for punctuation task. This classification reports include the following metrics: + +- :code:`Precision` +- :code:`Recall` +- :code:`F1` + +More details about these metrics can be found `here `__. + +References +---------- + +.. bibliography:: nlp_all.bib + :style: plain + :labelprefix: NLP-PUNCT + :keyprefix: nlp-punct- + diff --git a/models/tacotron2/NeMo/docs/source/nlp/question_answering.rst b/models/tacotron2/NeMo/docs/source/nlp/question_answering.rst new file mode 100644 index 0000000000..567be7dc62 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/question_answering.rst @@ -0,0 +1,300 @@ +.. _question_answering: + +Question Answering model +======================== + +With Question Answering, or Reading Comprehension, given a question and a passage of content (context) that may contain an answer for +the question, the model predicts the span within the text with a start and end position indicating the answer to the question. For +datasets like SQuAD 2.0, this model supports cases when the answer is not contained in the content. + +For every word in the context of a given question, the model is trained to predict: + +- The likelihood this word is the start of the span. +- The likelihood this word is the end of the span. + +The model chooses the start and end words with maximal probabilities. When the content does not contain the answer, we would like the +start and end span to be set for the first token. + +A pretrained BERT encoder with two span prediction heads is used for the prediction start and the end position of the answer. The span +predictions are token classifiers consisting of a single linear layer. + +Quick Start Guide +----------------- + +.. code-block:: python + + from nemo.collections.nlp.models import QAModel + + # to get the list of pre-trained models + QAModel.list_available_models() + + # Download and load the pre-trained BERT-based model + model = QAModel.from_pretrained("qa_squadv1.1_bertbase") + + # try the model on a few examples + model.inference(test_file) + + +.. note:: + + We recommend you try Question Answering model in a Jupyter notebook (can run on `Google's Colab `_.): + `NeMo/tutorials/nlp/Question_Answering_Squad.ipynb `__. + + Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator). + + An example script on how to train and evaluate the model can be found here: `NeMo/examples/nlp/question_answering/question_answering_squad.py `__. + + The default configuration file for the model can be found at: `NeMo/examples/nlp/question_answering/conf/question_answering_squad.yaml `__. + + + +Available models +^^^^^^^^^^^^^^^^ + +.. list-table:: *Pretrained Models* + :widths: 5 10 + :header-rows: 1 + + * - Model + - Pretrained Checkpoint + * - qa_squadv1.1_bertbase + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv1_1_bertbase + * - qa_squadv2.0_bertbase + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv2_0_bertbase + * - qa_squadv1.1_bertlarge + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv1_1_bertlarge + * - qa_squadv2.0_bertlarge + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv2_0_bertlarge + * - qa_squadv1.1_megatron_cased + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv1_1_megatron_cased + * - qa_squadv2.0_megatron_cased + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv2_0_megatron_cased + * - qa_squadv1.1_megatron_uncased + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv1_1_megatron_uncased + * - qa_squadv2.0_megatron_uncased + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv2_0_megatron_uncased + + +.. _dataset_question_answering: + +Data Format +----------- + +This model expects the dataset in `SQuAD format`_ (i.e., a JSON file for each dataset split). The code snippet below shows an example +of the training file. Each title has one or multiple paragraph entries, each consisting of the "context" and question-answer entries. +Each question-answer entry has: + +- A question +- A globally unique id +- The Boolean flag ``is_impossible``, which shows whether a question is answerable or not: + - if the question is answerable, one answer entry contains the text span and its starting character index in the context + - if the question is not answerable, an empty ``answers`` list is provided + +.. _SQuAD format: https://rajpurkar.github.io/SQuAD-explorer/ + +The evaluation files (for validation and testing) follow the above format, except that it can provide more than one answer to the +same question. The inference file also follows the above format, except that it does not require the ``answers`` and ``is_impossible`` +keywords. + +The following is an example of the data format (JSON file): + +.. code:: + + { + "data": [ + { + "title": "Super_Bowl_50", + "paragraphs": [ + { + "context": "Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24\u201310 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the \"golden anniversary\" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as \"Super Bowl L\"), so that the logo could prominently feature the Arabic numerals 50.", + "qas": [ + { + "question": "Where did Super Bowl 50 take place?", + "is_impossible": "false", + "id": "56be4db0acb8001400a502ee", + "answers": [ + { + "answer_start": "403", + "text": "Santa Clara, California" + } + ] + }, + { + "question": "What was the winning score of the Super Bowl 50?", + "is_impossible": "true", + "id": "56be4db0acb8001400a502ez", + "answers": [ + ] + } + ] + } + ] + } + ] + } + + +Dataset Download +---------------- + +To perform training of the Question Answering model on the SQuAD dataset, you must first download it from `here +`_ or run: + +.. code:: + + python get_squad.py + +There are two versions: + +- SQuAD version 1.1, which does not contain questions without the answer and has 100,000+ question-answer pairs on 500+ articles. +- SQuAD version 2.0, which combines the 100,000 questions from SQuAD 1.1 with over 50,000 unanswerable questions. To do well with + SQuAD 2.0, a system must not only answer questions when possible, but also determine when no answer is supported by the paragraph + and abstain from answering. + +After downloading the files, you should have a :code:`squad` data folder that contains the following four files for training and +evaluation: + +.. code:: + + . + |--squad + |-- v1.1/train-v1.1.json + |-- v1.1/dev-v1.1.json + |-- v2.0/train-v2.0.json + |-- v2.0/dev-v2.0.json + + +.. _model_training_question_answering: + +Model Training +-------------- + +In the Question Answering Model, we are training a span prediction head on top of a pre-trained language model, such as +`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding `__ :cite:`nlp-qa-devlin2018bert`. +Unless the user provides a pre-trained checkpoint for the language model, the language model is initialized with the pre-trained model +from `HuggingFace Transformers `__. + +Example of model configuration file for training the model can be found at: `NeMo/examples/nlp/question_answering/conf/question_answering_squad_config.yaml `__. + +The specification can be grouped into three categories: + +- Parameters that describe the training process: **trainer** +- Parameters that describe the datasets: **model.dataset**, **model.train_ds**, **model.validation_ds**, **model.test_ds** +- Parameters that describe the model: **model** + +More details about parameters in the spec file can be found below: + ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Description** | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **pretrained_model** | string | Pretrained QA model model from ``list_available_models()`` or path to a ``.nemo`` file. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **do_training** | bool | If ``true``, starts training, otherwise, skips training and continues with evaluation/inference. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.dataset.version_2_with_negative** | bool | Set to ``true`` to allow examples without an answer, e.g. for SQuAD v2.0. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.dataset.do_lower_case** | bool | If ``true``, converts text to lower case, only import for inference/evaluation. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.dataset.use_cache** | bool | If ``true``, either loads all preprocessed data from cache or saves preprocessed data for future use. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **training_ds.file** | string | The training file path. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **training_ds.num_samples** | integer | The number of samples to use from the training dataset (use ``-1`` to specify all samples). | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **validation_ds.file** | string | The validation file path. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **validation_ds.num_samples** | integer | The number of samples to use from the validation dataset (use ``-1`` to specify all samples). | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **test_ds.file** | string | The test file path (optional). | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **test_ds.num_samples** | integer | The number of samples to use from the test dataset (use ``-1`` to specify all samples). | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ + +Example of the command for training the model: + +.. code:: + + python question_answering_squad.py \ + model.train_ds.file= \ + model.validation_ds.file= \ + model.dataset.version_2_with_negative= \ + model.dataset.do_lower_case= \ + trainer.max_epochs= \ + trainer.gpus=[] + +.. Note:: + + The first time you train, it will take an extra 5-10 minutes to process the dataset. For future training runs, it will use the + processed dataset if :code:`model.dataset.use_cache=true`, which is automatically cached in the files in the same directory as + the data. + +Required Arguments for Training +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- :code:`model.train_ds.file`: path to the training file in JSON format +- :code:`model.validation_ds.file`: path to the validation file in JSON format + +Fine-tuning Procedure +^^^^^^^^^^^^^^^^^^^^^ + +Fine-tuning procedure and logs look similar to what's described in the Model Training section, with the addition of the model +that is initially loaded from a previously trained checkpoint, e.g. by specifying :code:`pretrained_model=`. + +Inference +--------- + +An example script on how to run inference can be found at `examples/nlp/question_answering/question_answering_squad.py `_. + +To run inference with the pre-trained model, run: + +.. code:: + + python question_answering_squad.py \ + pretrained_model= \ + model.dataset.version_2_with_negative= \ + model.dataset.do_lower_case= \ + do_training=false \ + model.validation_ds.file= + +Required Arguments for inference: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- :code:`pretrained_model`: pretrained QA Model model from ``list_available_models()`` or path to a ``.nemo`` file + +Model Evaluation +---------------- + +An example script on how to evaluate the pre-trained model, can be found at `examples/nlp/question_answering/question_answering_squad.py `_. + +To run evaluation of the pre-trained model, run: + +.. code:: + + python question_answering_squad.py \ + pretrained_model= \ + model.dataset.version_2_with_negative= \ + model.dataset.do_lower_case= \ + do_training=false \ + model.test_ds.file= + + +Required Arguments: +^^^^^^^^^^^^^^^^^^^ + +- :code:`pretrained_model`: pretrained QA model from `list_available_models()`` or path to a ``.nemo`` file +- :code:`model.test_ds.file`: path to test file + +During evaluation of the :code:`test_ds`, the script generates the following metrics: + +- :code:`Exact Match (EM)` +- :code:`F1` + +More details about these metrics can be found `here `__. + +References +---------- + +.. bibliography:: nlp_all.bib + :style: plain + :labelprefix: NLP-QA + :keyprefix: nlp-qa- \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/nlp/sgd_qa.rst b/models/tacotron2/NeMo/docs/source/nlp/sgd_qa.rst new file mode 100644 index 0000000000..351ffe520e --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/sgd_qa.rst @@ -0,0 +1,17 @@ +.. _sgd_qa: + +Dialogue State Tracking - SGD-QA Model +====================================== + +More details can be found in the paper +`SGD-QA: Fast Schema-Guided Dialogue State Tracking for Unseen Services `__ :cite:`nlp-sgdqa-zhang2021sgdqa`. +An example script on how to train the model can be found here: `NeMo/examples/nlp/dialogue_state_tracking/sgd_qa.py `__. + + +References +---------- + +.. bibliography:: nlp_all.bib + :style: plain + :labelprefix: NLP-SGDQA + :keyprefix: nlp-sgdqa- \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/nlp/text_classification.rst b/models/tacotron2/NeMo/docs/source/nlp/text_classification.rst new file mode 100644 index 0000000000..ea90c8503a --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/text_classification.rst @@ -0,0 +1,169 @@ +.. _text_classification: + +Text Classification model +========================= + +Text Classification is a sequence classification model based on BERT-based encoders. It can be used for a +variety of tasks like text classification, sentiment analysis, domain/intent detection for dialogue systems, etc. +The model takes a text input and predicts a label/class for the whole sequence. Megatron-LM and most of the BERT-based encoders +supported by HuggingFace including BERT, RoBERTa, and DistilBERT. + +An example script on how to train the model can be found here: `NeMo/examples/nlp/text_classification/text_classification_with_bert.py `__. +The default configuration file for the model can be found at: `NeMo/examples/nlp/text_classification/conf/text_classification_config.yaml `__. + +There is also a Jupyter notebook which explains how to work with this model. We recommend you try this model in the Jupyter notebook (can run on `Google's Colab `_.): +`NeMo/tutorials/nlp/Text_Classification_Sentiment_Analysis.ipynb `__. +This tutorial shows an example of how run the Text Classification model on a sentiment analysis task. You may connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator) to run the notebook. + +Data Format +----------- + +The Text Classification model uses a simple text format as the dataset. It requires the data to be stored in TAB separated files +(``.tsv``) with two columns: sentence and label. Each line of the data file contains text sequences, where words are separated with spaces and the label is separated with ``[TAB]``, i.e.: + +.. code:: + + [WORD][SPACE][WORD][SPACE][WORD][TAB][LABEL] + +Labels need to be integers starting from ``0``. Some examples taken from the SST2 dataset, which is a two-class dataset for sentiment analysis: + +.. code:: + + saw how bad this movie was 0 + lend some dignity to a dumb story 0 + the greatest musicians 1 + +You may need separate files for train, validation, and test with this format. + +Dataset Conversion +------------------ + +If your dataset is stored in another format, you need to convert it to NeMo's format to use this model. There are some conversion +scripts available for the following datasets: + +- SST2 :cite:`nlp-textclassify-socher2013` +- IMDB :cite:`nlp-textclassify-maas2011` +- ChemProt :cite:`nlp-textclassify-lim2018chemical` +- THUCnews :cite:`nlp-textclassify-li2007scalable` + +You can convert them from their original format to NeMo's format. To convert the original datasets to NeMo's format, use the ``examples/text_classification/data/import_datasets.py`` script: + +.. code:: python + + python import_datasets.py \ + --dataset_name DATASET_NAME \ + --target_data_dir TARGET_PATH \ + --source_data_dir SOURCE_PATH + +It reads the dataset specified by ``DATASET_NAME`` from ``SOURCE_PATH`` and converts it to NeMo's format. It then saves the new +dataset at ``TARGET_PATH``. + +Arguments: + +- ``dataset_name``: name of the dataset to convert (``sst-2``, ``chemprot``, ``imdb``, and ``thucnews`` are currently supported) +- ``source_data_dir``: directory of your dataset +- ``target_data_dir``: directory to save the converted dataset + +After the conversion, the ``TARGET_PATH`` should contain the following files: + +.. code:: + + . + |--TARGET_PATH + |-- train.tsv + |-- dev.tsv + |-- test.tsv + +Some datasets do not have the test set or their test set does not have any labels, therefore, the corresponding file may be missing. + +Model Training +-------------- + +You may find an example of a config file to be used for training of the Text Classification model at `NeMo/examples/nlp/text_classification/conf/text_classification_config.yaml `__. +You can change any of these parameters directly from the config file or update them with the command-line arguments. + +The config file of the Text Classification model contains three main sections of ``trainer``, ``exp_manager``, and ``model``. You can +find more details about the ``trainer`` and ``exp_manager`` at :doc:`./nlp_model`. Some sub-sections of the model section including +``tokenizer``, ``language_model``, and ``optim`` are shared among most of the NLP models. The details of these sections can be found +at :doc:`./nlp_model`. + +Example of a command for training a Text Classification model on two GPUs for 50 epochs: + +.. code:: + + python examples/nlp/text_classification/text_classification_with_bert.py \ + model.training_ds.file_path= \ + model.validation_ds.file_path= \ + trainer.max_epochs=50 \ + trainer.gpus=[0,1] \ + optim.name=adam \ + optim.lr=0.0001 \ + model.nemo_path= + +At the start of each training experiment, there is a printed log of the experiment specification including any parameters added or +overridden via the command-line. It also shows additional information, such as which GPUs are available, where logs are saved, +and some samples from the datasets with their corresponding inputs to the model. It also provides some stats on the lengths of +sequences in the dataset. + +After each epoch, you should see a summary table of metrics on the validation set which include the following metrics: + +- :code:`Precision` +- :code:`Recall` +- :code:`F1` + +At the end of training, NeMo saves the last checkpoint at the path specified by ``NEMO_FILE_PATH`` in ``.nemo`` format. + +Model Arguments +^^^^^^^^^^^^^^^ + +The following table lists some of the model's parameters you can use in the config files or set them from the command-line when +training a model: + ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Default** | **Description** | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **model.class_labels.class_labels_file** | string | ``null`` | Path to an optional file containing the labels; each line is the string label corresponding to a label. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **model.dataset.num_classes** | int | ``Required`` | Number of the categories or classes, ``0`` < ``Label`` < ``num_classes``. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **model.dataset.do_lower_case** | boolean | ``true`` for uncased models, ``false`` for cased | Specifies if inputs should be made lower case, would be set automatically if pre-trained model is used. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **model.dataset.max_seq_length** | int | ``256`` | Maximum length of the input sequences. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **model.dataset.class_balancing** | string | ``null`` | ``null`` or ``weighted_loss``. ``weighted_loss`` enables the weighted class balancing to handle unbalanced classes. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **model.dataset.use_cache** | boolean | ``false`` | Uses a cache to store the processed dataset, you can use it for large datasets for speed up. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **model.classifier_head.num_output_layers** | integer | ``2`` | Number of fully connected layers of the classifier on top of the BERT model. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **model.classifier_head.fc_dropout** | float | ``0.1`` | Dropout ratio of the fully connected layers. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **{training,validation,test}_ds.file_path** | string | ``Required`` | Path of the training ``.tsv`` file. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **{training,validation,test}_ds.batch_size** | integer | ``32`` | Data loader's batch size. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **{training,validation,test}_ds.num_workers** | integer | ``2`` | Number of worker threads for data loader. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **{training,validation,test}_ds.shuffle** | boolean | ``true`` (training), ``false`` (test and validation) | Shuffles data for each epoch. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **{training,validation,test}_ds.drop_last** | boolean | ``false`` | Specifies if last batch of data needs to get dropped if it is smaller than batch size. | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **{training,validation,test}_ds.pin_memory** | boolean | ``false`` | Enables pin_memory of PyTorch's data loader to enhance speed | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ +| **{training,validation,test}_ds.num_samples** | integer | ``-1`` | Number of samples to be used from the dataset; -1 means all samples | ++-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+ + + +Model Evaluation and Inference +------------------------------ + +After saving the model in ``.nemo`` format, you can load the model and perform evaluation or inference on the model. You can find +some examples in the example script: `NeMo/examples/nlp/text_classification/text_classification_with_bert.py `__. + +References +---------- + +.. bibliography:: nlp_all.bib + :style: plain + :labelprefix: NLP-TEXTCLASSIFY + :keyprefix: nlp-textclassify- diff --git a/models/tacotron2/NeMo/docs/source/nlp/token_classification.rst b/models/tacotron2/NeMo/docs/source/nlp/token_classification.rst new file mode 100755 index 0000000000..cab12eda12 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/nlp/token_classification.rst @@ -0,0 +1,219 @@ +.. _token_classification: + +Token Classification (Named Entity Recognition) Model +===================================================== + +Token Classification model supports named entity recognition (NER) and other token level classification tasks, as long as the data +follows the format specified below. + +We're going to use NER task throughout this section. NER, also referred to as entity chunking, identification or extraction, is the +task of detecting and classifying key information (entities) in text. In other words, a NER model takes a piece of text as input and +for each word in the text, the model identifies a category the word belongs to. For example, in a sentence: ``Mary lives in Santa Clara +and works at NVIDIA``, the model should detect that ``Mary`` is a person, ``Santa Clara`` is a location and ``NVIDIA`` is a company. + +Quick Start Guide +----------------- + +.. code-block:: python + + from nemo.collections.nlp.models import TokenClassificationModel + + # to get the list of pre-trained models + TokenClassificationModel.list_available_models() + + # Download and load the pre-trained BERT-based model + model = TokenClassificationModel.from_pretrained("ner_en_bert") + + # try the model on a few examples + model.add_predictions(['we bought four shirts from the nvidia gear store in santa clara.', 'NVIDIA is a company.']) + +.. note:: + + We recommend you try this model in a Jupyter notebook (run on `Google's Colab `_.): + `NeMo/tutorials/nlp/Token_Classification_Named_Entity_Recognition.ipynb `__. + + Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator). + + An example script on how to train the model can be found here: `NeMo/examples/nlp/token_classification/token_classification_train.py `__. + + An example script on how to run evaluation and inference can be found here: `NeMo/examples/nlp/token_classification/token_classification_evaluate.py `__. + + The default configuration file for the model can be found here: `NeMo/examples/nlp/token_classification/conf/token_classification_config.yaml `__. + +.. _dataset_token_classification: + +Data Input for Token Classification Model +----------------------------------------- + +For pre-training or fine-tuning of the model, the data should be split into 2 files: + +- ``text.txt`` +- ``labels.txt`` + +Each line of the ``text.txt`` file contains text sequences, where words are separated with spaces, i.e.: ``[WORD] [SPACE] [WORD] [SPACE] [WORD]``. +The ``labels.txt`` file contains corresponding labels for each word in ``text.txt``, the labels are separated with spaces, i.e.: ``[LABEL] [SPACE] [LABEL] [SPACE] [LABEL]``. +Example of a ``text.txt`` file: + + Jennifer is from New York City . + She likes ... + ... + +Corresponding ``labels.txt`` file: + + B-PER O O B-LOC I-LOC I-LOC O + O O ... + ... + +Dataset Conversion +------------------ + +To convert an `IOB format `__ (short for inside, outside, beginning) data to the format required for training, use +`examples/nlp/token_classification/data/import_from_iob_format.py `_. + +.. code:: + + # For conversion from IOB format, for example, for CoNLL-2003 dataset: + python import_from_iob_format.py --data_file= + +Convert Dataset Required Arguments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- :code:`--data_file`: path to the file to convert from IOB to NeMo format + +After running the above command, the data directory, where the :code:`--data_file` is stored, should contain :code:`text_*.txt` and :code:`labels_*.txt` files. +The default names for the training and evaluation in the :code:`conf/token_classification_config.yaml` are the following: + +.. code:: + + . + |--data_dir + |-- labels_dev.txt + |-- labels_train.txt + |-- text_dev.txt + |-- text_train.txt + + +Training The Token Classification model +--------------------------------------- + +In the Token Classification model, we are jointly training a classifier on top of a pre-trained language model, such as +`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding `__ :cite:`nlp-ner-devlin2018bert`. +Unless the user provides a pre-trained checkpoint for the language model, the language model is initialized with the pre-trained model +from `HuggingFace Transformers `__. + +Example of model configuration file for training the model can be found at: `NeMo/examples/nlp/token_classification/conf/token_classification_config.yaml `__. + +The specification can be roughly grouped into three categories: + +- Parameters that describe the training process: **trainer** +- Parameters that describe the datasets: **model.dataset**, **model.train_ds**, **model.validation_ds** +- Parameters that describe the model: **model** + +More details about parameters in the spec file can be found below: + ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **Parameter** | **Data Type** | **Description** | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.dataset.data_dir** | string | Path to the data converted to the specified above format. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.head.num_fc_layers** | integer | Number of fully connected layers. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.head.fc_dropout** | float | Dropout to apply to the input hidden states. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.head.activation** | string | Activation to use between fully connected layers. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **model.punct_head.use_transrormer_init** | bool | Whether to initialize the weights of the classifier head with the same approach used in Transformer. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **training_ds.text_file** | string | Name of the text training file located at ``data_dir``. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **training_ds.labels_file** | string | Name of the labels training file located at ``data_dir``. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **training_ds.num_samples** | integer | Number of samples to use from the training dataset, ``-1`` - to use all. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **validation_ds.text_file** | string | Name of the text file for evaluation, located at ``data_dir``. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **validation_ds.labels_file** | string | Name of the labels dev file located at ``data_dir``. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ +| **validation_ds.num_samples** | integer | Number of samples to use from the dev set, ``-1`` - to use all. | ++-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ + +For more information, see :ref:`nlp_model`. + +Example of the command for training the model: + +.. code:: + + python token_classification_train.py \ + model.dataset.data_dir= \ + trainer.max_epochs= \ + trainer.gpus=[] + + +Required Arguments for Training +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- :code:`model.dataset.data_dir`: path to the directory with pre-processed data. + +.. note:: + + While the arguments are defined in the spec file, if you want to override these parameter definitions in the spec file and + experiment with them, use the command-line to define the parameter. For example, the sample spec file mentioned above has + :code:`validation_ds.batch_size` set to ``64``. However, if the GPU utilization can be optimized further by + using a larger batch size, override it to the desired value by adding the field :code:`validation_ds.batch_size=128` from + the command-line. You can repeat this with any of the parameters defined in the sample spec file. + +Inference +--------- + +An example script on how to run inference can be found at `examples/nlp/token_classification/token_classification_evaluate.py `_. + +To run inference with the pre-trained model, run: + +.. code:: + + python token_classification_evaluate.py \ + pretrained_model= + +Required Arguments for Inference +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- :code:`pretrained_model`: pretrained Token Classification model from ``list_available_models()`` or path to a ``.nemo`` file. For example, ``ner_en_bert`` or ``your_model.nemo`` + +Model Evaluation +---------------- + +An example script on how to evaluate the pre-trained model can be found at `examples/nlp/token_classification/token_classification_evaluate.py `_. + +To start evaluation of the pre-trained model, run: + +.. code:: + + python token_classification_evaluate.py \ + model.dataset.data_dir= \ + pretrained_model=ner_en_bert \ + model.test_ds.text_file= \ + model.test_ds.labels_file= \ + model.dataset.max_seq_length=512 + + +Required Arguments +^^^^^^^^^^^^^^^^^^ + +- :code:`pretrained_model`: pretrained Token Classification model from ``list_available_models()`` or path to a ``.nemo`` file. For example, ``ner_en_bert`` or ``your_model.nemo`` +- :code:`model.dataset.data_dir`: path to the directory that containes :code:`model.test_ds.text_file` and :code:`model.test_ds.labels_file` + +During evaluation of the :code:`test_ds`, the script generates a classification report that includes the following metrics: + +- :code:`Precision` +- :code:`Recall` +- :code:`F1` + +For more information, see `here `__. + +References +---------- + +.. bibliography:: nlp_all.bib + :style: plain + :labelprefix: NLP-NER + :keyprefix: nlp-ner- \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/nv_logo.png b/models/tacotron2/NeMo/docs/source/nv_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..e764a71852c6f4e22e150621aaf6e1d8b35183a8 GIT binary patch literal 109116 zcmYIvWmKC@v~_WJcXxLv?k>Rz#jONu(c;CuxVsdBJ1y=Gg+g#EQrwCb|3cq;zxyL= zvBHyCGxO|o=Inj;L}_U#VxW?uzIpQoLs>~q`^_6z(3>|fW5|fmZyFix(BHiI{6<+$ zTKCiPaVOFTW=O2G(nA{f-=AZ7h#hDt;ITHX4#xvF&rLhLs&(I*2OUn;9c zqT4Bi2*z9Fu;~XGRWwdJxu&+$cYgoAvlY9$G_!L(v*x|y%M{I2+kLowz7gqxK~6}K zQ)%ev&J>_mdXp{o=7;GXn}dz@`LEs(BrHm?Pp^Lre-pn!Fz_{S%%C^uk=zsy_&i@0Q|m{aEkFQzM_(Y6!iNa;M2c z_rEa_-|nWwony2z)n4`1RoafD^^Iak;%ip<4*B|>%&u=cR?)s+!JXoTEFa+Z(s8rn(}0)E<=eyZA8_8yiRc2cCgdU#7c3he(l@SaGs{D`h`m+ zX~~BEyMzGa=kmXLcTDhmEF(_PDT-32V)VIG7WFNNZeRoF#kM3qZe(?c@9+b_eT97$ zeG@2Tud7D?5SwL_%^RTnqw1-wQ!?;s50(5W6Fx#Rz2F&DnyA<2+QA@t4UHY#YuzEAbnJdR*HDo6j&zl}v? zJ1Qb8{K6bt(G)@ zQ5?0mgzxSg!Tvq|&+41|?+o=ql#w`iVbThD{FXkG|;;ZMdhyjebU-Q76U0Ecw*6^GLe z&MH<%QiC+NoE%0%oC%%LPJS~a&fO%LZq&H2Y5UB_M~QM$&DFMd8CUBNT!yH z1akX`GX-uU3St*h2~u@QKt~OvJF>b<#%_)<`}h)Y`W#$Yyl2;~_ycbRS+K&|=}VST zG4q(YxPvf`&+#IU#{4S?tf(WG&yAKYJv*c$fy2$X0-J~~w>Eg2|CTJ|;I9CY`}DWu z0ju$L=VfKh?Zj@xdkY&T_xYWQmIAj*m@$Kc{(oq)`Wy)y?VLa8x3%V4d=B43%L8mf z2k};jvxNwD9zV~*ty~1UEvOlbifD?-o{I}KwlIWTR{=)gEfJ4&8b1{Md%kW!qY z8&U?4D=Yx*YC0$9FJIxx8Lms&v;WwGoR4Yxe~?+dr-=hocs+UF_j{YIT}yc~K}Xe; zia_-QTRTPn55TPoqJV$I=SjZmoWu|Vwn+qDH^~k{zxX%!$VEqJ=jr1j{{t6vL2$tf zC^GPR@Qu3oc6Eip?QiO=$!<0uh6KvV8k{U*Ko*R~&u*~@FB0Do~n zcEh@KWL7Gkz-!2Vmk+Agdj2zSB+Ixm{z)P3-%K6kocb^21M@q6{7Em=aqMQ;i&6-7 zgV%k$`GA;wR6wkzl~uT`^6|gbe+vbF>;*t)`O;4R$#|vrKQOK8%T>F(>f-|NkO6p$ zdY*Fa%cn4j5&`OWhCByd_Rg>1^m~Fa%f}JGna3;g+yD-nMX1$3eaT@0BLBb>t4R|0 z?-hmELH^$R`FA~6kAZ0b#>r9;iQoTW%vrXGP;^NVlI&A*tIoJ~=6jfW;4%9j2;jN` z3>6`a)uaFWtY%Ruo4+X*?SXR{>vm+PSopt%Zd6@g%W;%njsDN1e~>g2)0WpjYR_&w z#Q%E)Yw^D0YV=; zU%c+`VY1zSNwZ}N%wV>J6D$5N zZQTnL4`sJPhw+zC>-nDzVlCR$M{pTT3I4`YsNVU@w!~#Cw=>G;=Aiy{X=Z^v3ukG7 z?ci^(K>R3*+q_bLrF_Qp)f?0Fe(N=`4w7H9!|D5?-MgZiexyx8kvu>##?THtWh9u% zwKJjg-kDCD%?Y}NnY=Csq5<#=2x3Ez2xMWqS>psO_x4{%0P!TtS z!-8VDwXvFUUwQKo8h+{?fAG6?hKlSJjyHzTxOv^N?G^?QEDb0i6|DTx4wgb0O^{J0 z(1ud&d+iru0hPDS9>L(~MJp7N7+lwI?1Tb6MU;Jmd(n>g0YTi02UhWf4rSytrx;1E zz73UmEQK9gvZ?8s-NDV_&UW!QHL z-XBv;;t%lIOREUeu}a6{^h)sCD&DUn=RM!{T?-|?hLDiO6E;!D?^iUgB!f=^7A^=DupIr?F|fps!ab%wnV;xT z8eXOl=}WTkiqUY3o!U@$7k8UIz+ts^lg%d&J5k4P{4F9A%>U5a&ty_}JxsZHQ$gg} z$v}J4$wfCdbzkt9gT*STS8x$5qeMWX3&>31S6MaG=Yl98liTj=PREj`;@IOLmtSuE zf+Gu<_VF=%F($a{ica7|BHj#2z?+;E+d%V{Cdu$|}j21j+biOt|hRifsJdH!oUtiY)T>@@r0$aD?+CH^$&81;5# z)`8HVDA0+gRW!@OVu9Su)Xds9&G_HY4C9i2E&D$S>9#80tEaV={he0h7S7%_nJKo0#ADLm!!98eiRDZE(aaD#RUq>h_mt;o zOZ&S0v&&lonL;&r&isT1Ua&+9=BtMjy@e{rr zy?|JrE;h0iOr9>b0d%{fjM{2GBlRCIe1><8e8mYGF+WujEuhA8NivGHA1a@XUQe~J z2mB4)E2gMLi)-IV%f<}&b@ukta$xhL&IS>_A}3pSu_3AJv(aWJ-S z7ud5~Csi{H>s^HK?kgd7xJ|H269M8{Hb#HxZr`Fe0(SrojT0F;w@oqJu7NO4W!GNN z+{SI+ZhB|kH38RgfMT#e)yRkQ9C(6!Wi1atUlah_PC@-ti6<66Mcb&YZb z$?aImWvUl?rfm#dd&mDPOs{a`#Tv#rBXY8!2r#RD1EuS zp(7~5i<{Eq59OZ?NjXFB9#{b55FA##00C-rcoJpqee+`&MDD}W>_kF^wYKG{wq%Lo z(a$3+b+jEJu2RHFaI&SlJtll^=3}y)8J#hYnjWxK4~0@A+3>&1{I$H-kvyCR;RC?! zwA8^)l;Q75i4beH2N(9-c3g3Ldr3m6FHLU^}iYt9fys z^YO?+JtQ$vpuInsS^HRPr!gcjheCyE+2XheHlt|_fzPYe0x*riM){|&pDViVqy!#G zLi^V*`?rSF+;W}B6Dyl+-A+SIZhaIG8iD8U$6!ICsC2E>F5JH0s}n9RCr zYom!Y1m!BSP3YwbiIM9gS-l0e7iw>beQA6~r>y9$f1vMHvDb+-P@B~S`zBz7PfmmY zE-x)UrXtlJHCxL4DvHPEr-C%yjnsc_`-ls(R0otNS3%ZJN1U^25(N}XrURseN)iQ9 zo(puo@@BKf@%mPo$AayBTsjn^Yo~nwD>>2--w^9%ps@SIvW({{)EBI}f!U8SoZfQg zELq4^@!gh(1V(t(635J9dSsM9ioLMm~aJ9qJVYM za_^aTmpKvXK}a*$&Bm}+xabS3SP^oQmN3T7ggf4iJ$j&!!dVJ#S7%Cg;e7?iheRwu zfN4+#8Y+#=ga^$kBMx}DrIn~i!DETmb>c+ol25)U+?CpLVz|jyfx|?k!fy8N^XYBS zki_A8ft$iBC`^xpKsx!f|0u=x#*myk_8J@#k)|ofi{+w*OwrYBDxc*`u@jz32~luZ zQJ4B-Qc=t7!|n38sSG9xvrFpKCD+;4(zvni9|Yfc@3;!J>U zk6b`v@q>pZg{M`^)rC@RNekZX#>f16o7T}kEb0#;8OldG_?l-{no)y&{Bm>wimN^J zloABm*n|Utr3}4P-0_?Pj_OYe5w%nQCCNY4)Sz`PmMb}qDx7B9AZ&rTGv4K?$$=a& z4KIrFxSBE~^oSnhlMW{ih_PFS_1BKyo4op0+rAd}Pq=((@sm`d#lnFIKwe@G3lyEe<`p8{ver1z6G&5!dQ7TJ`smEC$X1Cpd&pyzzo_(jl4KH* z+lG{S76c!&hAdK}Hj-Xl5c?xM(e1)~t3rku&BtPp2j-@ess9;^9(;h09opB?Dt~3DwdX`NKMYF`#a(0Gygvun0bi-Cf?GICLYD-5mL$L zWl}M>TtS4J-Efd0aAtH>gLbK{Ny(_JuDp2UBEE%O?vL2WC-X7(fk&=xQ8{%62j(gV zr7S@KcQ+L!N+35eN;w}%Z;lPT!xxkWv6+XpJCsAJ59v9ZYonQ2t;r?K;a8KJ0Hz^djCT;@I>=nkdHkM4ef}eF!`}qE z?hoz0C|9EH0brJ|HPNQDQv%t)>H8lZRXN>05%AfW&_c{;nA$N*_4Mbqg2tG0VM|B| zxopQR<*G+3`!HZo#Q-xX>Y3OA2uuPGwTfYQ0vI0)O&hap`09_1g#e~&R_IM)Aoz&v zF@NjcwHQ=vhc2!+-c(X^A_yQl39!?ptdjvx=<+B}mzDwm2!>`vZADz<3;X_XInh^|D$57Z96ayRq;p&oQH;GF zkHuvSZ<#YdxX5MFuS?H~Igf*{ESCkFdVdmTjkiV^Mg0v``&9{tpR~)5$ruB=G`I1Pb z9+j7O9&FfGvcw~)E5s~JU?@LoltRPk7kx`$D1z=W-JzVE*XEcZ0q53FA^`K}gcjt# zpvy;m6*`d-{CzMg3*c)SqH?Y)sjc!O<5!Ky=(@_B10Pv$iaqQZx9H!Y-5$7XlR7c_ zo|beJfap}i`{wke5@rGj240_nz*2CWb}*x{rcr{Nc0Mav(w2BHpNgM9l=ee|Hy4-6 z&*z=k^^Hy!7BDa8_JIH z;{i$SK_A`fvL`!rP{{WHS-nO^QlsMEYH512>OlD&xstJ#!5Vjl>p$AT8~}8jRQN;Z z;Q>7|ei`sH#?_fq5d)CR5`_kId9$HbWcytKXJv=o&}Cl#z)udN^MxXOVB_FPZOM)P zX?-MNdjdX!3GcDE(P9xFN$nlrO`{i4?+j3gbjT%8{qxuqvUfYPtP`6o*hq$3`d$2~ zfR!^6X;fKeQ-#9KJ;>TjHuiJbiCu>v&z!`q0>cYC?fLat!u5Z-%ZVtaC9d60&8WA@ zMsZcKJZ`o({xPn3lHo;v;qc*Ua~x8^V)_GJ&V!c-s1MI)ZA!6-!nH`uZ#+SIqj^kt1t&-)j8qdtJq=DE1%5>245$Vmok9DQ@W>=4l2xpkNem^q}%qN zA9!}__*Zg%d%+ZW6253S=!oG;i)=$SFIi_EU0$B~UOltDH&7 z0JVBPxheb;md`oK z&BX^#g`y>kz)lxWE1QBZ=^hj_W=EXK>X%7ijUyChLmH7d#Q>Ba$s#%%cwTh z4{QNVCgkL}x1l5isDLNY)j!9ovBVW$Yr&B_>}rCKq%xciEAZem>V5peF4gchPL#tM zrQ@UDmIGD>5wvOZ-!0aAX_>E#sYAxJ7kPUQN8b1bAnpkBQGa8GyI4h(>so-nqqIWD zB$D+Ef@!$KNJzB`Rcl9yC6p;u)790VW#_w2s^()9K{7nAOcUX&Fv9?y8HNMGJdGbZ zCD^=@Ew~4wfV_VunzDKizJ+j$k=dRw%cKPED?;4=jyP;b914cl=WTH;SewrUUQBZ*pGl0&5~&0zJosK%Tw(Khe<6*eX2& z;w@j0y7Z$z=sv5TFx-7>cG}tiEoa~eRorfms?vcvQ?O_|SLdTtw9CJ+n%r3QV?7O`6K{g!A-*0FvP5#hmJSMLF45+%VcKKS?Xb?W&bmEu zl=`z=%YjE%p^0S^_^-(gy?hYV88Exn5U@vb9ro)j_NqfrZ*qhAxxW#&bpwHPcNwu< zfW&)CLi~6p;&gnN36gZQYzv_yO<6rT z@*}y*43EYNjmS}n4&KN&$IpJ0gVhisLvjMVXi zRxsS2HMl@uSDm-(Hlw?8z=s^C2-Q=R@lTO2*Y)owFZKOx-}|hy@rCZ{g)+J7@PA@C zx_bg%crrVAJ<-tc<#BWuxH&q8C z$23P30!_o$&i+~qJ22E84B{Hz>avJ^QVT+W!I}`+4<60Fw3rDOBQ{P8Vb0Bn$f)Z> zz*~S8&$kGRJ~F}8wMum-!@Q2Z>cJE)N_Cg&D4yNwG<~OD2#aCo)Sg|FQrW-1y%>Iu z&>X2d_zsIDUfHu;=2v$CKu8NCFnq_*cbuz{(OQkoD3%ezoOvvTpSsMmV(pQU*aYZH z>#9}DhCm8zR?r>CbQaD$$0M_POUBQ?(k2|gxQ*!#JY}O=VA71Wo6(&`X+G|A02ky< zA3X2ZOZut!ef8BMznd|f_tg`%E{ajP*QvT^2ull7(%+2+Jneoe|&$$&sD5G{E)P3fo^80Rx5c z{!m>4OBwP*2x0Lm2_5T6D=0Po&fLBli%>V3+bSKc)XAgoZz?A7Z2_FZL<^mtl^f1> zI#F+DrAeqEO5OHwXilfbc`3T+Mv^%Z&HMKCMDr<4X1T@7uFs24c@djTp>;UNeyX2y zT_WpJ#RA5aW#&z_&7a&*X<~LT)o-r^9Z5%oU8X=JpG1`+c}Qf*SD!;S5eGhf+Q z48^^LF?5?uNOeL)qjAnOQn=dYpIH|QVNnR>b(aITV=h`_}vUu|S9$8k6e6H=f8;U2GC(k`DJ5bu7OS=9)TEc=v?H1qhl zy#C_%K=;7ogt0f6xxc0xB28YEPHjL z`Hcpb8t^o@-HzZh|0-slHrXP%QRsk?W%wb~KHDW!&Px=_jGk{3#$)4oEKn@Rsy67CJyIWisIm1^ zP9*7+$%m#_ou-C>$wP=T9%Mdc9A_huNlpjLil}-$xcocghvE0k4#E7p&1n%T$EQSO zEGi<4n=YAwI%wf#4Wh(n)gSRN?r{`YNBpWOOQlkH@LT15ZEmlU@1i4jO2YYQWsLcy z8}IQad2MAb8oQH0Wc3z0>0gE?VQTh&DO(3gD4;b)cK|HFxc^_Lt0o(4W5}|5!m-?3 zx9VYjrdgs{DWUVWzj`ci&f|_+t$svx0G& z-{ay&X5r6D%;E{$dxZis)er|0*(BhPWOcN39EKfWtHfCQl0Yb@8~foXsyW<_6>y=O zg2!vTc6Py|pwr?3d0GJ9;RvZimk?dk1|7gS=?i#pRBuXOL0OK2k%q>@;72@T(9Na* zUy=f!yT?iBi02SsMFmMTx^N3m@@CpeoTYCyL}TEbZ> zR&s3;vhgv!|6iW&%>TuPc(0by===6!irxxUc1OSi8-#rA!L~s9QC#HbV||KICZ4 zDDs6drXI=iD-L7w;Q>kd19dy544o^<)3M_6tlRWvlv*Xu@trO6h0Xz<=xTR0vhAs7 zZ%`v`QE3P{p3EvN$4=)Geqh{Ht>&t}+AS#~N$0TXp03$-KHA!pZ<~wD)aXLD|H7F9 zgz-NX&=vM^dY}CE@hOiYX1wzOcJC`;bic`rmOV$qHlLt|7E6FjY}(WP>i&?BP*P`e$R7-l zhxR*#<;bvQOq9|$KRpX@YpOzQwTsf=+K~ap{?;Fh7#T8qha35&Cu!LbZEf%uo|3la zjZjFMPYtd{pG$ZM0w%jt61L6xoGNbnHZ{VFO$H;g^bzV`#7R86S^6uAVc8{i!lKwv zEza?gK1D1UPUQ*6Fa5+`uqytlV5^==GXDAJwv9H%O5C^4VEYZ7qP$eAa4{M*MlnNp zyajw5S5|5e7n+t=7&%=-tG>WdnIv8WI&306+^0MMw51&T+97b}Uk>!tz?5uv9_uQz zt1-Ney!1WF`$)0j1-u`8=G&?bo}39R|7%wl7TpD+m)Xw*%MU<0MOg~I+0KN@SC;Ry z!!Ynh>Klbc7R;a`X|kQ4$HN<5`wOvGu?Iis6BYE>kA5fTKjBPi#BLG8>vQA@W|_?} zY~VweOZJ_T!{7sv0%Tbd@<aK-sfvsywOJc)jM zO;q^U0gt3H4t{Al_^xCMhczL-6Z-Itu&3^!(PFy9u}*|N^6zJP*Tz7xV%wQSf&S{G zd+xg9ERk_!%y@zedA8V$6R9fiWjHMHy|C!1M{H1GmJjb+odT|EPw7JMj-dutICC3_ zf$X2io`vqog}r}X3I8uxWO)>C-ICn~XFdStg5pENqvOMBmMg94+GqzEajb&#b)^Vt z810eQ{zCMmX5Z?p!7zieY{EU?#+4N$2@Uo(q8|G%AXQS`_N_k2e>jWwllkG=Bf5w= zf2>xwIKkq_szV$?_K;silBE9AL(a)vTOkhgjR75Rb*x-yZNSl^IL6OoP%fl?s&+~7 zi}=zzP93dwcB)Y0r=2q{#*(-sTZKi&njaA#RWEt~*0KX-T**v)bFR!D_|%f^U*#qf za%==bddoI-NzDvehti{RTFozOC;dhu=J5~1D z)cqLuv28Q-c684nTGIradAqK8)FSq0G^yBFBiiLO!1z5FgdL!WJWExvGBR3{dJj}V zX7{MbOP>H|rLw>s@Y;&o;Lp5sLM+~4ZCo1n<6ohd)0z}M35ba5tN9e`m~ri#(NO+o zh+`6}zPn0M+Q50)6GHi5Z=M}u@)Ee>)Zy?xO8prnvj%1o?7;*ADsbrE$2PpQFH~Nb zjtgvIP`!XmN;zyZORl@*jiLk>k7;&76ZTbMRq8@NP|E~W#PE^}Gxg@($}hUFVjShW z^{;lyN7Cq!BQKL>=)AWunk*>|kOczPvRU#R_650R20yd-S)KgKPrzvOYA>t;ex|5J z)$-0$hlH2Yt5=pR$0|p`mMtW=w(8YNn!S|PO#M6TW`=5(8TyAOywu+5AC<3Mw_U-c zu7Ho&pY_xuE*`W**FKLbtkIk26SpC0RhB6*QpR*zElMhs3HpQJ9^!5zrzNOVfQKW? z9-;K}W`Cqj(s8p5SI=|a9OdJfZvo$uYr)s_de#wG7kSzEh#=~_@-fl`Nv**ar7aex z1%ubNC@JW-0dbKIGgLsXG_>A#lpPZVdCM*)NxB@uB!_RU!L9WDg;@9lu@s!4!mF5^ z-5BhGc#VZBBhD4#j^W014;IraRvvWDB=x54y|==@PBX*mc72@}U+@GyjBi_E1#%Cs z+>)o^0Ndg~zyBn{L|Tm;iA32yOQ}?A3+J`9YS30?ia*<-xQDSO${uD;Y8SYJ*`L5M z^#!|w{v}gGM_=g3y@QlKlu_F_%-^~#7TA|+Wfm7VmrO9pC{TX^P_GPiC1~Qw%gQjxTJa}gm1ghkZ z;$;Rouypd9wi~%qPo8|5*M+D@qqcR)akL-W$h9d?b_hD;mE#6{^$*2A%D<(xJ!Sv= zAhg#%T`uyA%qjwQ3AROSS9Ceoaf8;RB`-4TknE7cI>I+i)O^0i&+sXf%(IiOzqgUs zCt}MtLu33fEU)&@-;KissQ=DwKHf-4eE1a#8+TemOY+fQulqIyG^3_;1CCw9{mQRm zvIKv3%k0QWGcB=Z!Gq;E_>BbkiEw-9sdv81g?I5h`P48UT+TR7Fu-W)E7mlW_eEby zEMC^2$5eE3cdxAsG)t5j-WPuC?oKtUOiMU z3>*Ic5^DtM8}rk9Ga^+MM#+@!9Lw#PzV?*>_B<@V-Is>bq?e;}hC424*M@>{NNNQ+ zBAxXbMp(1y*U#%1)JaMZ+^aU+JlW2^*k}-8ocg#0A)|`mxn`!q!p*`G{9Y1}DAr8> zjn()?uYfR`^mQw@udR}PX}rrR4^oixY?wG31k2E~RI!NB>kM#Y52sufM~H*hyqmO2 zseJ%OND``YSAe|Lm6s1ySZuykk{Yj@=?+O~k_!1z)g~{eRV^A_5wqiellg|s?5eL~ zxbwWnlL?aoJu3xHK7b7MAggI~HTJ6ZvZu?r6iR~=3Q2U6&bQINp#urkQKQMcQZ%{+DNfpj^LIEwwHs4MZaSWnVqM#R z$|U`N8)}`dD+1P-&Ax(DeV>$S^n!j&iu{6%|9v861fQYdI@O&r13`gKW(`@*n((JLXX2qkmg*U{2>sox@ zQPlT>=6@Ej8*Q6h0NonJD(hs|k(#F)1gv_Dehy7Trd2x>3 z*b~@TQ$u7CPPD~~$u=u{>FISDop6ytF?%Hiwc5#&0D-F|ahf~?y(r)e0!;J;Xv&5E zU|mQd2h)R9khrn#4dvo)uxy!X!PYw-^42P9nH@KrzNzs0xBF2*ex9RJ`-VX?A(m^0 zAwJ+gZ%Vw28RAKL_oK~^1+&1!m+SmMlgHoPLUBIlkq^xB9=i9zMJ=+=qN#ewE#ReU zXPfg2_mq3esU8L3aB5c7< zsr3cPBpS#Fb}7R%;~XV>7TClfb%>L_!1jBynr&^Zk;;?!AHv|;jTL`kXcJ^xl(39! z{pyR67Pr&=P0=;ddh!(2y%tD#hT$bE7}ZUiK84@qJes%VV3_iVCOauR0Y%eMn;$Zv z&R8lPBoOagShWmSZ2LQ?Y~V{021&TF)5Ti;lg2?LP%zd=CURL6qBv7_1i;hd z(q*xI7f(H00gRvyNVWWZV8~@y`vYG23lh$sG;93Dmol3tdUq+Xlb5W2JG!DMlC-ck zD%8mpG|9!x9S{Ylz%_yZ^~)$xtD_7rl-$rC%DH2}z2g+9TNenSkris{oBz?2*=v%8 z@mwzgt-@hGLRCY6o3fOy;x&;Zug?bJKDpyxIkkW)M?HlhKkpzJ4jDfbLEV)aB=Uo5QSSC2gV5nc4*D+hPQd5KwVNY%-T>p z8%ubEnf9-Bs8H4oftg=b3+){Umq!nerKc@DlUHM!p!_t-d3g%cF-A-6T7 zj*U@c*p7_Z&Sh^RMCHIPEYejEU7v}?qM~P)Y1md$ip1WNYK_KbD^H`>qpLQA=?cbj zj)&#CcxMW+i0Dsz)6s09r#OgCALN((!8(`CXG&<}N)m+3CWJ_9YQh33qg|KOrnIhK zl}|w>&_47f{@a7YP%g}OJ{Twy>Y9{O-wy&`8yqxZ$(?AU1miUe1NvjLzJS9l<<9xp)dw|b| z@kM5ZW(jm_l>;m7ab83h_n(kNU>{75pCPC)C}N*Fq-re3?h-eT^>c{G)Sn&glVYmK~?G=J$7{(@)mfGg-U^>Bg^NXh65 z@n{_oFm@L$$uN}fnGGLpJTR7Rfcb;oskg~1D$z=GsFx|Tk?Ca;JGR-uae3%Q#1%Zs z?z(dOy5OwdYSDZ`TC$ix}T2S`MdwBUkd_Hutj}`Ce9G1xXFCChC>iBdeUz|0A{lO{;X1epIV@5K~46&iHR@@bYi1(;BQw3 zC=bT6;6^S_vOn!d0(CVZ#KeH)dNWnJ)K8MZbm=y7`@vYj`FBw9h3s?#2;aw@mHD-+ zkdb0ZhyjD>su3pNv&8Del2n6G413It9}A zPT!3lRl&-SzFGnkHO?vS!6n+3j2J)N_`J6ay{9nh)$mm1WKXqZd5(Qdg1!ovX4E$# z@9->9e`yMtOpH=;eiWp)p4N4BV^a5LJSfjpvmF>Q^3DJly~rZmbQr^~Xx#nU+{UG8 zhOP2nhI7dK&_AYVQg?wv#9qrK!(RT8b_`?^$#Cs#=(C7{C9xGmut68^o#a!XVtr{~ zeiR>V@eu!1cgw_k?3<6BKP48a6%*SNej})YZjJm+Ow+!J;8?y zOYeZayh2y}+utTuhdjG9Ys;#E%7e(w4FxS()o}P!$l5O}oOy}g;e&;=#Er~YtFn#> zM=d4H#aOxrvl%jk0}1bCZT2-FRU!%(-rUj$p^n5NvoPZz_{Aq6if$f5XuC-eFbm-; zsr#aKPs3)Gn8u++a91~?zwCzOi;ylS^fJ(^P~xTX=5zgbVaa^UIjfoNqkR25hV12T zh=+1_%>nj{jIYUbHXq{@_4bhB!{<8X&Y#6HcxnWH_Jr7{qy)88W?7qvw$SuVCm49Jfa>R8=&e+V;`W}S)M%!-See8?f_+)s^_l!gr?rQ zO%cf|f9-IBcr9ZFp_H(9b#+C^yHu`yIcpjjo$c7Rr{)>b^|L5^x55_l>$LdDrdq$l z87#P}glT&Zhw)_=Rm`{1s;)YYI6zA~(vWxB*v%&a;3MXtsM%#qm3g_Q^CD{wz$XLU zuVOPe0-X@WGcv=}yhsKm^~fr_Y97u*QYD7N6=w7B1Y1-rudxp$s*G|5pC^YAGTO?q zQ+PzQsEu7<0kfK!r0y*`%3DJf)O^0R+1sR!hnM&&M%3u_$qS2UD__!O?jjgIG&K?* zE<+tnGk5|bjKUO3w{J_H*e@x0)?Y7A#KXk6blnY^-ah`QFWeJcYrMcYE5?t|VC!an zF?{y$X#89Dq4JzaX8Ktg!Md3psCLdz>%k^6s2;%};EO}FtVWIFTD?ehL-Ozf~Ty7g#7)xo9{;LQy^W{?5?+yx- zDH`rle?rO6

e4ZKSTN^tbC!)=4Sln6f|8eyN#2*0~xds;F_`!xC#YH`IQcpldf z5~wDaqBasHjpt^uctuVuL5BoYLlmIkkYTUB?z`xRB`tB0*kq{9M~2oaTGuQDHmUVL z#a9`5npBi_z^%gaestP=Er+GWQ-A5ZySEsoPK&;eX11TEDJv7I0vIvX)Tda$dV0Uh zU>zUoiRQ&lgN)j9xc_#&!&Y4EHK!$lVWjZZR!tiXJy=G_+y`84vI2%*&##@kD8O*f@ zTEm-Sl{v=>5QMY^?<5BHEg2YEd?_QdYLGf#%ZTUhcd+-F%FMKor*A&1hiR!rhU>H> zs*h9vpNW%^jlBA7zb76Wah<}G!tjA-Kaq@1uOTtrXgh<$Q)J3^+4+{<^U*A%h8dB6 zEI~k2Z|NCn72)=-MuPf+!w}{q4Dt1U723fM^kQ{J-bJ9A&;NjB)@5hS9_?WxD&7nIG_!@i%#%X)pJ?Rz2vcu`lM;+K&r+ zBOw1>{+@|I2?M0h|MuhTA${U(D2CR8a80J&u#*GC)xMaqfpt*OXcCs$UD;OaA?H1! z)uk#WKZLUV`x`&22AV1LvbTIX{2$4XO5x|&0JCG|h`G`JtBpLxNQ2&>M~zVpPie+i z?T>FUnmN}oLRtrmpeqq;W(vJ6!V+ran_E}c_9l&DG9opes5g9%0orMXUoq1&A3kc* zRFewL>WGvNg3ag@%ExY=INkaA0WMQIL`5aey<+}z=`Mez{bVyWsyvwtxigY?CVA{=!C3THH-Uvof+zq15suIMtHroVmz5`NI~d{)@G~&jd@D`=Ra6ccPXq)`S>Zs!(96 zAtTw#3nHUCxuZFJ4eOzPd}2i>5ukB%P2$D=i`Uq8exG{>SNu<-=|9&GE&qHyelf6; z?sUBDvbNUoCVw*a{$Ox@(j-4Ue<|wfe_?AQ0qpxy1}`{l&Ju4tE>7X+aH7c(SL|Pz zK(EBm+g7PdiIcF%Ob)^`)X^UYEw!6aucyMLIkFM+Z6~{+JlXm{jkbhS^EBwJ?n~lU zSObZ;xoT`WeO~FyTCkRo4%mSzXy;wW6y6~((Y#+3_1If8=;BhXIF>YS!F}2hIeAie zMw*Gc+&cWT@J%hcjQ|4o?Hd|U!(HjskbU^>sQSq=&sak?PqFOF6pfCX`2^HA->69) zs{NUw41ZA>*^a||l4T!qkXYHouPAUo;tL+_k0hl*+TT+_mWx~t$HA4Bb{Rg7cP~8` zTBUsJ8a4GNdIqWv3f@(wTJ}>o)eB+G>mSdp$`^CF2)2439cUZxa)5i#y==9k(W$kMj zySF%i^ulRDQHD~huSO3AFz&iN+DshY_S#fOc{mo6I<&FINl5)vBVm3tXljIfX=hDw zoTl2mgOl|bL&nxC5b5^=RAIi-h*3;E^SR0L@6ScN$@WApRq(S^m>J**+V zI~}FZI^2=jRnL!a^ovh;B$&BOp*KabOF0Q3L5=2}eL0)z3aiq@V>(F!92*N6$jt_N zQ5ri|Vw*vNEha~36@Yh}(i-8;S@dQqE-xR8?;BBXRW9eJeHNJdTXCElf2k3o$*l6e zp{o~6!2?ce?iD;mzZo4n_8MOo#I{b!WM`&F2_8{$I!5cK3c`jPhKg2&@dD`8q%hkB zc;a?$Drqw-)cZCjTOSt&K+J8&&lnWo{f|q-I(N~{UcW^@na>=xTH`D@(i?X8N?JdQ zCSz=VNU1T4aHi!vUk1_*cT4_|^z(l@`^RNh5|n+riatSBJA4Us+c7ErYPD(Im0w); z^Rvxvs#a46_8Yh_`|o5dGKZAhkkzvTJsOb?+rd;{k$pF==@cedKNrqkB(A#agbz=o zN-vyw@I_^5@Gm3?pD6NKSDmr_c~)2+hc-`U?T~Dmp!Ss?5u;?e+;-h)^O79ByF~P0 z;WDYgVfVK)@dc!|iV_CD&TUAf$3Iu$a~$43LbS~sEfWtG3zayejlc=m&A~AApm2PK zpRu286UnJ$V>3#))Mqd3J-Y~8bMzfO%M9^d`;&aIj(w-F^zu`1d4aHsoaqVM1?F6O z9y%SFCVVl&u|zjjC?Ac!JWss;b?H-xyBRLk7=tZ{McyeRA?p;d=uttuNtPV~ZR?_t z$h#AAXC#-(xo`qhNB+*7jfQEq@Zp`Bq#`9R>`7yY1mM%i$jI&(8YqlDBz4u2;>&~+ zcq|(`g+;3Tv6}c_we2V#debJmh^rfv?(jvD-(d)IZ%9D%Qy$lg;iLVp;H&Jn6=jf$ z0l|lLP2`1u28*ZnT^T6+c~V(oj(xS6QH%Pf*wC&WGTRpvx?%?fy}+9-w1@=)376^tmh7u1|&NEhJpje3<;Jy z6Qf#{K;lA9eNEA()>>p0gj^7vyF^t9Xj>#4h3(@15%rcqZ8YxpccHkuLvg1-A-EI> zuEC+Wd(jptP~3}K@L<8+-Jw9S;9lH|J3QRq-+$(ryvnQ0>~40?ey(%Qg@cGUUY=gf zf4pl`X*lYmL;k}Af1t0(x^Gxeu=<&hGtiQo5DI}G9FImg@_{D>;d=lpa+t^;&!66+ z*BDgF2u=rn45VqX4 zors*Ae)E7b3Is(~2uWh!47J=xJv4a;SlhCQgCupa3*oG?9E9DPyQ0)&=f3EK^A}|H zDmF~jdRz{$_ct@Bq%|)gvi*vzJ{Ls1c@BPh4UU@w5(-QkDX3u#K~;3ZicVGehM0r+ z%0YzWzvIk-{NZYta|h?q(=RmqR*c-jUeb~#2X60tU>M##v%yOF=@PTok=DXnVB_*s zwW`A$PxhElB`#8qjjP3OVHnmMF&fJ5Q07dyoxL|hk^KR6k&$=sKo`{JIFC>kiV4S$ zI2hZgvON5_t5Z6I*^f~JJ?ctIdxmL+{*E&0?+18sc?Y}?1GcKttd~v~dyiofUAqx5 z{gSDJlNlS6qIifiX!rdabcp53UPTbU#l}@Dv;Lh-FUoM`yhKs#`YmxDYmX*0D}Vmj z_0;(<JY)lLQYJf9q~0^0R|QcfB1;XJt=CpIzHUyW5ReBN2Xq~Uwkvd ziwBV%(GRc*f#9)48y){F`GRJ$Lr1mn42DM*4^N!6Thdq@YZn5&D#^m|LiG|JnXX%H zSKEmO3VKLVYN9B8uCGyTh^tY>36IwftjYnif+VIx^lJT9J-nVHeT81HDy{?n zFHE#zmDhq^{?vIlR+O4NKeGFxQS0(5w8Ho*xS}*WoIYWSc>hm((vRG=7qblCon4Xc zdQnXv*RDWo^Le&bQ%Qa;hzOuns{i93o}ivPiVd6Qx)8}tyCH_#4H}oN2d=JzJN&1H zAVe0gYv-CVmE~++;1flA4GZa;VZcL;rA8u8=8|aIW+- zW$A^+ohCVh>x#YzPA<7gV#f8s0OLWw(>4twp@e+Z(o8YS5VB@^q6yGcj}JgT)Gks8 zcLw5dr?Xj&a8(ljXnNNN%?P!qBE6|A(Wo6vT2#AfKK_B;{Pd0F{IeK)DS^qg`L=+e zg&x%vQ%A4DHv(^EF8X?lOLW`C>fnRPP9@(+?f|O?q3XTMlENP+ZW|IdJ9a`H`tu?v zDKO8?7Y=D^jHbL_l;oe()-_~0&V@u(Fy`rgtuXD+_3ZJJd29=+CTVQ&eyScqfqs~ zCWS4P=&G)lC82LMK4{V#pH+RaRz6D_f3P3tmO9_3AmJN;!xm+rvs;eB1=z#_eeM?AZ9Wr@ zDHHT5^ZiiuTW7*=*mn4`lAu6t);)oT=-Z_MXM90g-ooBpM2x6wK+`>(j%ZTzWzvII zYVRSJsWen~(etM`d1&@%fppLb^-9+J(b(qAZXdixd6*$%&JLeVr3Fxv!^vD0W_}c(R zb}QBRwe0&NUE=)N57*x8i`c?%k)>Gk>&$^Mz2cw&N`895P8#Xy>%UhjF!)j^v-hj5 z)dKfL(cS#8~8iT9Z6i^+^0=N7?`^`=D&r zC*Ff5^`W}&xI?=7FO;}3Gijq+kd*b~Fit@NDXhLtK3Gjt$sdC*LZ@XFyzn&*T2i?C zCPUk(h!L{(YvNH4)a~h6>6N%Wg79nh#?6>tp#(O@R)$Q1VyW$FQ~e6(5?UTP$x=Ra zc4j%@;G%V+2^hEW*l(iRQwIK4vS6^qg135qPr6SE@lVklgLm6tJRW=0KBA_50rGVB zFzy+6MnxhNDMv0(2quwc!j=Nca-J&C!5pPw+LZnSEeT!cu?WsYdBl5F&1t_=Cp^#t zQF+d1*xY|rIYY|mrLm3sKdY_1bTMWd*t&9|odyWu;YDj7%L-8sLVf7dF4}^T#*;xZ zumvmGF7G#kxAJG{&qx2WHIiaw#+wYDN|J~$5NC=hP7nH^7j0m4;54uwM%e_vwKh24 z_EPAoZ-pvz4p+a@A@;t(!2|l&buBCAN?gEUO!!MUe~0W&n3HGw zq8muY9x%}0?uJAs{WZiLJSM$=iMvaAU-U^$*GJojahrbcEE6D*b z+(R`q>HcY656^H#2ZOWoSZna&fT95bzcyNb0FK@9$HZ}Unj^dmSBak|Ro*6ZeoRUR zLlgq+BcZ2ATNT28P$2`1RKd%8T9O}ELmK6O=g9X~o&C&v|ObEeRfE{yQt9CS^; z=P91!oe~jPEWvGXt;Gewd_PiSwR84$ekUMlW5zDPT{M}PfW_1CT@yE%k}br@vauX= z2=H9pXvol!@WgQ0xbm5u%)mQ%8hv&pa=OL+dl0nr@xz0C=oHP*dunxwB?S*xzPWsN z4Ij-;b0B7&XasPa#J*jXotQ{LZ-ce5M?^AoPV^u^>O+Kcl9lALW_pqpjZCRi{S|>4 z&o~BqolL`uq~;9fXt|i{mjJcIK#Oyd`yZNqjNMBg(741m;C%u@4Eim;)%mn02kX{l zd^)^3uSKux7G*|etY8oBs2YW0k}$q1?xQ_Z42P?|m#UT+t^R)mdgwc~N{(2(+p(_n z%V4A}kCnsda@}9usTt%y!^%53!{V4t%>Mrc$9h6np*T7XvFRE|-VkrD8%*HeXRCmM zo`9c!_W5&FdIl;}0OzwRa3DWzph$HC&gmD!pGYf71(y0stp^HP^?bX&X)ZIx%VR`z z+eDTU@#{$DL_2-Wh4QaT*iO%ZB_?-t)~oC83_>dF{nO8#r@VnW4^B6tY?LEJI?-yFUMu7YM%B+ zE*ejHiY#m2GgF^faiGL|rh_YGksY9NA5T_SURsHZoS1(;&Jmbc#oRBi{Z?{&5*T98 zTCh&R`m9TtPQ7X=Tg&&o3mq#fc@F-umj%6uA{R zH)A;PQw3z2vLRRSN%Kh{d~>j|O9_@XUitG%(3>ibW=@a8%$DG*UQn`&_zkT5qh(JM z^cKJdOFoRJ(&Uo2SgY|Rt!@3ZyoML?j5hFS{4~ij+o}S`L!Adz_0Wl%WT4KqB%&MG zHzk|@byU)ymqS6&W~*A5QHcpX0tFJOqmsSEIKmrIYQVm;Vx;nb(;TC|6t``AqEYM>2VF*_n!QZAVmWXD&o zTX$h&w|8P}R&+DjK89a-{>J^r5LCq;@gQW^J3nfSv{FLmF$w>@byWQ~qGpnodbB-T zE;jym^04+i$|Aq~1le0HRoym5{HV}{*)Z&n!Kdhtwf~8eTai4A#8JO2Q?7J`Z~R6c zA3a$UFU=EBTp`cT%?eRORlfJ)G~3l?>MgO5P%*U%^!Oz)uhGvpPtM{?RymRo=1mG_ zM;gAg2o&i>>pPnWAYG0!d7&Xb_WUmnA;)ctX;&3O^aZqgP9pakT=QxS@>NP4EAh}9 z`T55WD(p_m6LVF{Czp0$Ys#bp_C+-}&XnZykU=ZCLIu z+C35*-m51lz8#m=3PS{_I6qj4nOAW%mjU0^^kV7lr62kZyNe!TfZ_Oa2*WZe?#?gP zeZCQ$w`h^XqmTF4H_tuMG{WC$;YhWvF3DX7AFuPw_u|DzG1x_s66H&Fh=H6%ZBHOV zjY;2la#cjc85#a@w~bXyip0R?B9~7SIgxg2UP31mDpcU3v{JubTeD)T+7v>td9B2} zMH~H%Ya6!3o{Z8|SIAM=fAq@S#7tu++UHa+TikrcWuoK|>7I45+~0Wlv@i5q1p+B- za5JCPOi0E~zE)L3)^3jaP`W=G{h1*hUbvMAs^O}uKQJB|+5V7MG=MCuDu>cE@rw)W zF-;zNigt{YrS3i0pj1FwKR!w@I!$_B28YWOqM5^*9l`xY*~B9)&(nZUX?Y`+b! zk(B;<9IhdH2SCF{W;tPv`8?w*>J8nA;xf2r#=rw7H-^#moUAv3L-s-yEES?Gb4vtG}k3sZY07DhkS$Xo90v`o%~KlZP>U2aq!c*PmoAZeuFM$24_QoSL) zHrWj=nD`(Pq`oj9>xAL`PCui@;bdD6I5*}@~{80{7gf2dBHm`5^ zJJ!fY&5KQtuP^`=4VB0H+OS;Qewppf?J$k0G7>7c@cEHD-z+@&=idaf1u1i0dL<7= zJU+T+WXpiB<&>j8zZkjq^fCdH5_w z5bO4n7Y3I-O(EInphm6Nvq{sxkcEnsiW)B!jxw-%iDuHP&h+2Re|PV>`fnOWR1n^O z3F5>LzXQ${T}}RV2yV1xsBBvWjGM}9sPGPeM-FQEZS^QRChWj5PbsY`?vJF;A3mAu ziD&d7T!cH2aL>RAuj84_3tP~&{gtxOt1(hk^NqieJ5S41IAoE}pIJ@E7BKpj&n(a45BqF>&V^E-}Y%!Jn>exB}<=(?IAK}w?tZiEwYmaHIG)U)2J zLqAa>FNjcCdb^Pv*}OC!dcSzj2$m#Cy@mddVmLLud$;wL#C|E3tGQX&D6Oq7F1l#{ z(a@-MuME|Cv9C^EoJJr^OIB=$(^uM8j=F|Qd*dI#lFu|D+KbF|@L&#HD-@A)BRqW~ z%?cNr4`_t7zW>tDiL`}iCXz!)1RyE#7RQuK_M}3Km_em*(oD*>u`81$i(FQ?lqAl* ze{@9E{ImmGu@1qxChJHi$VY4Ish%QUWz6mr2V-^*oD6>UDLyddTW7sO+-5){&p zE6RbxpTBD1e$X`1+}IFA#Y!-5xkl*w%Zf{s9L43LCH6MwK@Te44gC583#HGsduKFM zQZoEfL@zi^kLxIKn4jj@%c>vR;FrBbxJr*kIy(-koT~UK0G>yfT-y*Y4JH6do%rfj~m59{|yxXQW zok5p}w;jrh@TqdD8rW@Odb_84W_!-N|5hL6|GoDW zc^(;X%`K+nQ z3sM3mg~Vp$ltuHuQWeTfD}+N29`xEPRe3THg3D!u>6Wrm7rr+`B_2a~DTK2Ev&4Id zi!r`CMD#kw&XZ?kO`IWCO|Z_Wp&gv-3)wKFHS(g=Q9*;>cO`6+zb?E}eL;GbBj3C$ zB%tW(?nM9X?au%CW`_4*RrI_FsjSVScdV_C%U3=P;*nV`-&c^$posRd%jv`px8do1 zkVQ$=1(}BrRTudo8GKwCQSeDsDZ*jVz^^eeqq;@)F{bZHxP&vrygkF-(eWuHww3Wo zVQ~uw5A2vj;$MGOaF#~fjB?o;F(8y)UDba9@`*}r9oyM1oulM@xmf%mPrj>zQ!R*m z^g3x*agqSQWEHX)XbYL2AN1-UkAMo|0;~nL^kWL(p8S*j@2bRM70=#(^$?a}uPv7e z4^jXW;$(CXkuIJqG?SeOsiH=V`3@W?1bqjB8EW9AwXZ}8S*70}#p zG%ds_Sg3|Qe)IjfYc9g6PFi*}PO%3m#3;|}TQ)zs3H|2ABddeAM5t_zeDBk>hw~_% z2aY2OPJ1-et38|6LtAP*YG{s^%h48*$*wDD^XQcalWAcvHs*sNvBu=UZf>1%5Fam0 z26FQY>2ZB}Bd=;SID;y@`}mO-#}PwaAAsYk;jMTkS&zqEC&SgHeP|~Oi^K)kv!+gP zB;-`@hAnu&E!?#$b<5#6*+KQJaT;h&U*m@T*P@Y<=;L{nVw&8CP#^M2E)~GTOE)?M zo=8F7B0^GM~0KZMr z!&A)8^SpZK-fPsqdn}M!D9<1lZ7o~JLD;!YV>`jC%Y5O;YoN_Z{Og*62gQ?gwxvbM zb@`eg!!*Nwi(tWX_@_Yauj5EUn>T|I8@Z_;B1^y3go#|TbpEi^O)d%Bz9z((dK$Fa zCwenAr)rw~u6U(V-~wtS2M^__;0bwad0yb*@qDA)GoOS-m8|m8$%DCKwR1CzhP~mu z-9Ad#2pgyPW0-_eTuNi(_| zDQS3no=vD%U&^xR)-wI}(I=QD>a*C3BW7bqTxer9l7LHLe7}n<6(uPf z)hL}Z(qjL?AatB_(9}aV8n+S0Q%zvW+$iUZyT1`7Wbm}5En(a+WM`xQ^{G|3)MR%R zP<=FXEVLkGZ|xR-7SCQOdZUtesxzIgs9%pVn+HJ33J2r=chV{uASvT?c8d}b`hrV+ z0FbH{8c0I`l=1BZzX87wui;a}P;4YrOZ6S7tPvkjKEx&idyxGMEA5g5R_NdWj-9E1 zDSe&XSg%i{z=}2!(3WA`I3J|whgZ1%D-N(?SrSI!g(aKaNBXB39K_*W^Cz1?Imkf%%g75V}?r|G@#!jS<~Xw>K~4+^WMF+_aIA_WyRv(ET6=^ zd`WN2h&kG7s!%(#D|kyOz8)cecliJi*q`srYssh+s?DpR#{Q5Br4(0@xFjWP{m90P%}29!+U~A}I*~*3bEP6p z1u--u;AM{+zRIURG?iDF9nibAu9yH=)|3gn&}gniJ-yp*4e=%}9cr{<0gTS^&++}~ z6oyCVXE2#B`;_^a+o};O<8dML<}eIgAh1n9M|Ls1W|{Bxih2IzpDwmeWB12-P%Z-e zx3t~G(qc1vo@{;TRr?SY!8YQ5)SCI^>vsnZCzEN6HnWl*Mq>W<1zHol#sRF*%9SrL zj)Piq?gKdx*|3L9ra1r(L6ta$KdJtq)mkMW8554}K3#C9q68tr zb72x0dLwfB@7m#1d#*F$sWbxnPgqx;3e13*ZJ#Q-tQqXe%1A(pl1G-`I`*1R4HaW| zT79mFJFLRnFQC*+c8gJWUlZw3eE!y=g{^6s9$m7suM>yBG8Mv=S!H0yhRss4x`i%` z1X8k0ExMnAmf8AU?z_sYf?FB&;Otd~tDota>_}|u@f$>RA`)`M+c*fB>@W>nTQ)YG4DjV<$C)|(lNHvbf?t~1}sYf@*OSEE8Mo-dS1zzdh!n$F? zrFkmXi4LKh;Td3`VfWmmuxSmokV`QsAnc|+Irv7AjBwaGq@?{iqzeDn4-MB!ideVw zZJB=vA^l#lnd)yAZ!OQeBY7DzyEpN77#Lo zMXVpRXQ>#E%gvU9cZH+ck6HCc1veG`C;cqY0qgsmlC_aS+S@n9@y! zZ>+yYFj8o-O%dos;BQ?G+m!R$CBD0)gzUaaLjrkWgBbyyBp}VQ@0o2ewtvzC6G4o0 z3sOcky97QToAsNX#&MXaSl*g-nV} zb}Lvz<#nHCK$4T%>+k3*j|g+{f|uu{O?Ki*>OE8g8dog#fl~{kEcw(KUu@FjA-#fZ zI|yZ8NlKky*af+ZU_t!87!T_+RK~nRl4qs)}+C-02*|OL|sjubum9t9M%4Lj@|oxMI7W zoR-PK4EP|swKxq=HfoNaK2L2ug3YQQ0fbmN3X(@50zP7R?NVSqDMQY$1QR)05BX`N z?Nl{VfiajL-aEt?+q`MN5w6!wp|o$-*9~tr^4*oZ!3=sYbUVZDEfZ=H$2hf{cHBoQh})bDOf&x@CaNkE zUG2$s%hEepF{Pfv8v(chb6mm(qklS)%uC@Ge1rO3Otf~2iv~OR4cYNH4U+aB=?MTz?rby{<=LElmJUzJ-79)yD|KLD@Bm02G*7x{zc<)v5 z>i?9R0!dw}7Kh(P+*AJ_9sCcJuNCkosi5=>IyRM^@eX;(LB?u9I73pJK&m0~WrUdpF1m|k(#h5YBCS0o}r>POB>~vg#SD-}NV0d?A&~^2F3w`GP*WgMw z+u`gX{Z@JkbA(%x_!TpwV)KJmb&kQM%;x%Q7U#c^K{0W}^qIuX4NfxCo>u0btxMeF zDe~C#KO8*V@klgwGo+x5zVIcZw_GkFX!Hhc<{tDt>pWFVh|N7)RRVe0S@@y2y9S%w zbZXE$&^OD|!7w!he)zOZVeU$ElE{wWwhS1OvwjAWL z<&mBdg^ph1>i!HspATfGvvsr){ua90#KZhGz^qx`#@wRjAfG_w<^-Xij5SXMRpoA_ z1rt()|JFJadAZByF?ZD^?q!7H!eLGHn z>_vxKrxTpJxl5f6{*t^uDi8BKZ%^&|hX|UXqqKR52E|;B(=?k(W3J3ogsh6EY)*16 zC>E=Re7h1m*#g+sw}8fDZG#P`ulGWm+^1$rblc{=jTXdxAw8Y^gw3~d@F}Oi0)F_0 z|HS1D1eI;I*a1%l9|b`?8aa$0@{#N1wkaTMQv59YoXT}E^c&hU3+{nnGD6k5A#5O5 zH}z=t4azAaQjuSY$3p$Wm9cvNI^ue6j|g~lH0BRzR_WUy7goi`i}_3MuIqL|I@O6= zWqJ;7CbR^u;xWj(J_Pw-Nkd=CAiH+bF#1s%McyXeBl*R%{9x=>G4@5m0Obh==v-Ap z4!Zbdc!Ax43FoU%6~)AB>gJ^&NQ2_PJ1_>Ilu(CPP;{aB%EZu3L-pbNxZw3VH|D^J zraNRX{)|H)n@Na@;cter@De$9xj!pvY_^8gFu%XhM`CBX-ZrKYR=Td?z`D{i|#g+L9YDyCV? zxofDs*W8CHP~k(90oabC?%3Lzv-Q;-|&TL#>EVAI> z5A`mEEAOAOK)u_!H?0^WCA>75ry(oOCr59K<@H;jD!q3KsaaAX(oUy`=$`r)(u&%qmrFu`}11pB;?Xc_nH#_OL!m$@N$?;Vq(r2d&$iyl9@!2fyzcQ{OaM1Fj4* z7+8SYCdpm8$Q?)fmTF>T<_T0Qz7M8|P?`HdJMGyJ(~q8kCLoa4ujy~*`i|%%yt_l$ z;iThZUrj72NH_}DK-!&X$Y^ZhTDHl9rGew-)@Tni|;v-xklY!|7o_2bAL!ZB z2jxas$DgIy*R-*}ix?s2g%7Pv(#MAT$y=`5FpR8mlR(d;v;eCfHGn)TkHj$vHLK+arAds8F(N@NYreTD zfsJ}t&2Q-6>_u5Uzt}#|@db<`jdrY{o2+*Ym3r!Iob9;jHk$^$ zuVAg$+h(X)g6u)?FglvMVGna9Jb8|WNsqZHil)m*YMIKnknQM}PC<2kt)6W}K=r-! zjgxRzC`LlPqC-WdRA5#7hf11^psBu0zTJAW(X?ITdv*WL#0U3K9+}TDvOOg2N&$o0 zpzDc2aa-}!2R31rSLG(kTJV8YTSZnF9c-KB`fNWdPSQq$$?le-G_26)C{Jj2d}cpk zaHMij1%@}T_4(pK!E@W5d*KBwBVz_l-k5rkivmvW>qlGLP@EEoQycQ0^L$Q?1Roia zXDS`P=2yWe<@dG=7jHPU0oJnu005^YVUA3@();o)7i}A8cC`jN;@5&Q9gG^^RS78@ z9V-7Ypl10dqOxHPnVkO;!W+2OG4aMatC(P0S!~&0GpUkpvgSr167Q2?6@9URzRN~usuXOlo@rTvl;UI<^LQp4avxr(A zKa3_DxJ4{T_2?&B?o*k~9w;JK<&0SE?WnMo5ans$3zOO{M>2lQ!JLc) z$<}A~5Q{u^jOo!lERPGHf;p;3m6O!-t*7E=5arwqiVC*O;9D!{O$v(Fjr?8M zf5%uS2pU5BB6v|ADZBd>#eZv<<=X$lqwc*_V{ANcedILZ?=~i`tB<_k$-%-S_@Uj@ z{Oa>Ckem{xWbU*zi#8Q!jr#N618NkXw8_CQAsey$p2SzW@d*7`fruQf6X5uM>r9*2 zP7nXM#O?@JACb7!e4_n{;%9q*_{NU3pAAe2T-IJ3Fc=~=$;A?%w~yTnkBY^OzMQZA z;j24@@et|`2`%{noMFOrKF&vKzMeu+YAr-7H5bEmaaUFYa^yhe8A13!;Dy`E<|RH*NE}{1{*N$*D*Jw7#+Yd!56YMqCGM)d-|0VVU63!n3@j;2N#DTF za*B*5e-DqUewj|NQeTTiwildRJIC_}lKySyA>*_xz{}ol^B*!AMOOku^3XwrJ-G>g+@THBrbG|q<3WfOE$kh% zo}6&|1a}0jE-IClGq2BV5DJWTe6|c<0hfqJszeTxF$L|PM0~{gWLs=0uG|9|$4wJy zLbJp78|}FHbr$fswvsb;=hD+H+(Q04t%RZqIu2I*>Sr6_;u?=)&`Fum#I4pZoB!kE zQF1UVZph8aLh?B0fd9PSXV}FPaoA-Dx$?6VmGewDFJd4*AO>Z|6Av&?ot?wKs)N}i zqALZgII2Z*^bLgWBu)9O6LueC03deS5Wep4r7|>!hir@)A}uLqA{^kjIUSG@)TXa3 zwxyPzLOxn`-%bk>_7g<(afdVp<2Kld)5+n>!Y;;-1(hJ$eZx(BWs&#|)>@5ojJN_I z{E2J{*ze0HQd~g3jH0_Z9%zdiG8iX19{y*DOgfEemEt#@9hHWf=Sq%plOZqwr$mrzkg^o5)nP_oxjz3s`GCS)mn}5PlL^heZ z-EzM3r8>83gM#e+72{$5_A9$4oeHy_fBZVi&Hu`}a^s0mX{@hoE8O7>u3w|qtR)86 zCRih^6;*Od&s)p8afHk5dC@PUmA|TB5S>JR-fCAc@B6N}H7ViXQoms<_-Gy6D<)?| z9GIakZSq(aE!UYIh<|)9{3p9>w^JZ2Te28jQ!nR|a&*r#`LWfoqtPYnf@D- z{ooGOiEF6Ne!)O!XTzHbY-{R-JS{gS{yq~%Qk&G`9zX#A zOv4BjWdbz$QRC`NF1>TgBD;GN`f{8oCP-Qjq`&IUf^@wEux^i|B(e_8Ey6|7okrw# zh|rj0We-=)d>NlqI}*Mu*n2I0zmV`)QGLnE=!&77LaR5H!v}lgBe**5t1MW`BKf}j z?s}2AdGfy(&p}@k`j2?zYeGV!#O1Mr<{O&DPS)EGkPpSbdV%hVqs6FY4!!srVa3pl zya$1pT>OUjT^8g^%PwgRB5!=-0|M(%81x`3(J4xr@MSH5($o`5hZa_?eNx9;j|tsM z^2Pyj5YuHg7rgxqH2#_Jp^bic z5WH<@JeP!MpI(RY8DOXyO`?o`8#I|JBGH`n8XrVaEQsT;liOMdsmLftFUH}E6$^si zcN-rB=?lJ_IW;*tDnL?g&zri85J-u_Z|=};kAn)9sQ&L7Gboc{|5hdEe+X(2%o93` zM2wYQGi>H~g8L>9D%Om>hKVYe_OA>_9mOz)`lYw7EqQC%oPjn^J3P32K>(QT%XvH~ z&Rv-?TLt#&F!-0Xd8bEz;hrwZ7iTr6W5vJjb3MN?FE76megGo0#@22jn$QPb{$4@0U+=!63Jed&y;o!UoZY8l0`?|-Akh=7>WPSA(Ve9T z$~cSt{6oRyhk7<%>`#-`)S=n~rB((6o>W-(95E^Ah#08MWFK@-kLC01#b;Qb?yN{| zs9K?c(-3sVTz%$2=%lNU4wypwr*k)kC9oR3R01Mzm6sj{^ zD7y%3f~?@-e}jBm4b)F-6G9{JO5OGjIDR+nm%eOxaEite`VsuAw?Vyfk8{S+g>ioe zc+$Va>z=LH^BjhcVBBr3>zLkKOZBBd(cfyYZ+iSneAa&Y4t%Tgn5&P~o2dnOi`tRi zbQE@JKoF?12yneuu^Wqn|9|l|KRKC!c1S|@>^Gm;n)KD$Rexjh#dzd3 zV)$m*Up8QvC`j%ft$*je9?+$26vv8RbC&}c<2%QlRAK_GNW^UzUr#mlun(kFq*Cof zy)v1KQ-{!dJCdpD9F(X&-_qH=`uKL>0th#E7eGUdI!ib?v2gvfUW$ zq`LZ~j~to#H;Tgz?8uwL>^wAxQn05fq0DYlbO4M8taFLZyU$z8E_R41%6I6_u@U{b zb&Cwvvv2tfM}K^{lpsDQcR{h>N%t48RZn>_+CHmZ%q1m7QrSTte( zg5J3gtPU|DC{7a_>Lf+t2R3AA1+;jK>OEB+CxU%MrPJ@<@Mv0NHCf7WWbab(U*}`< zkn+{qy}M2MrXG9IGQ8%FG+XWzmayAXl(}_a5m^^X&e{KlAMx|02Ot3E2ehjB1`)Lr zFX!c0{4QAwp9KG}NBkAuP_{Z9*wX@U+HV&K51pcri|ygAUueai#-txNrs|VY&@)pNyeS>!ztZE0Tp;5f@_rP)4Ne#{()(@I<51VVx*VTIk@I!tv8RYZc zb&LOSKwAfjPN(I`1aBcIQSuOuxJ|)@peZIoA*AXW$&j7n%eTOC$b)7Gjc1v06?wkh z$5=W~VS;}=K7xX_<^b6Oa@J*Fvp?afi=!c^i0a^whoZ|fw+4XIRQsFc_lV35K zCKPq6#gG)gGi-LZIRpt+eF~hYd<|6SOxNqv`h4a+r;%xembtOQ1-|Lw1V^kV@<+*F-;ojk6TwQ$m z3RW>-BV$QQpS?+YwIHc8I~W|PJ%*%K$3!jMKji_HNtvaX-Nu41W0>&vy?7w>rtUApX4<-$sjk&}HFQBYgstd#e+ zLbaCfU#1&R?R8|8VNgOK>;gd-eQr`Nroe3=*ihNc$DY2f!PtDe3l6LE?Vze-e?LR# z`E*KXl?dg zZGMm2ppNL+rhFSvl<740=UF5kYRVh?B(=CMc!y5d*h$})>0`ecYS}>T)sB$5U6V17 zo&J|H3%PXmWkfrW+ACV{M#p>6bOe2VT@SOVBIa$?1D^A~FXk`b-*eONVpaI|*zM&6 z1T7{?Q(EeEfrLI(L$=S!4=<0UKg7-+-2Xch+<1^8+1cK}^6`s%gG;N`FV8}MyE&7@ zsYDB(O75d?)}cEEiO4Fj;14?jC~IV#njq`xelg^LKku@hS#(#4f{utGBi}-Hb~qys zW1QNiZ~!b5;%a9Lm(vq(8W{ndf3`Q}!ypUcvNx8rhW+4%t-wXi z>dg}RF@xw5V_^_Ezs51A=X`N8)N%j}@t6$*d)nSSvIsn6!++^aHQDZhKh%);ayUbc z;Uqf1Uf;gr9g`w|$n~J1r6cdz%%)&toiQ^%32RpkYFs0b?!S8&v_@{j#bz_JK|r(Rd>qb~#gf@gg{Y?yC|7QF&~4ffvna(IFvlyznHn z)eR9q8@!XQ5u$w;Zpa+PGK$Y+Gk6^RrD@sdy{1VQV1j@7VE9EDVr2^C@&X2%{Tl&n?Z!>h=^L@mp6)?x|6d zH!~E(Lu*4@@lg+jqUcb>=^JM|PUPLf1eC_XAE07mp0hkq*;0W#@&ko^fXyIp-HQQkj;rxhP@Zp$vfbsdnEQU1RMEVW#mY-mC4>b zC7VAdQ+PZ?{`i0Fj=%gRC+)?-LN|2b-%0;jV>|Qf@)S==mlg7K+yS9F+^T_G0$+OO zK)WwH08f%7Ygiwq%4Ur<{@V~yKRB?t!`?q4^kEKlUAJnK1=fgp zD(C6z8qlMRffJP&aNb1NQN&@-w3#vkrXay_r^t(-Fh-_w@1X;MHdV>{rZ2P$RYT@u zbs4RJt1@sRkv$L+7jG4u#x5MD%)1zc?qNek&d1|A%t==(ypgXKA%e|oyplli;*`yY z78ufqqkAzzLa>$0v|3cfNg$e`4jHpY^F1;e`|1~Lt+&-@!5u`R5;>pmBgiV1->!Ry zV3#N%MV~#JnB%Z&GU}RxRi+S&ad9p@?|`U9J`h-7{OrUT#8WIYR`Q>Rr;{G2jBnw{ zZRTNrHQlE=={p&U5Pmu@g!j!Ej4wyK`b*quQ#7Dk~uSwYOZ z@S2VB#aOAkcia?3`?1^DbdFFyq2X;=1>lQxIG8ZNj#E>{*Lkmq>)2O0w67~kItHWa zz1^WUx(7^#wR+J}*%u7N6uzx>=##C0&Q@G=nVT}uSD|muKFZD0gUCULb;H-#MXuQ_ z+}FL>_^p_UX3Vqs?(dfZJdAdyMlshT`{k)=Fjs`g0r66v0uk*J!8r$pp535-?Z0Dcak3<1PoA%7*`sskaJiv+bgF zTZ+3EcPXx+6fN%V?i$?PU5giYx8PQs;!YsA6pCwcU-`ea_I2&^9Obq7jB$@K52p~s zWmgKT5(|YzO0dS>637`qRkZNeWJowNDC)k<;!_hljDhqG_WrUS{@kwxf6=62n;*0M z^bmQ3K1b3Aq?d^4KjYmg5@0H#{po>zZJ9}PH}2*?F?}SNdLmvsNc4X-0{*|rq+zk@ zQR0bfoE?GVaVh4Ye8cc@ewth=aO=h9kJ6M?S}$PX$Os<`ka^sc)zxBB@ZeNLEakdXgd~|sPkVSNy0g)%LCB;T+{@P~ zG&O=)p?YK$@NEzYn@%=Aar=sl0HnN-&}ZD^dn$=-AR}v3TvV%xZE_)L^lCZLq`t1i z=4h(Cx*z^`&@5|De8hqv2 z;xeo^hNLxbK-8Ic?WMqdqGvz)|GB1dEU%SovPUhpQMB6CtuIdY4jWAkfy21gBY(MN zpwc=Okh_4Ho=?ZEY)l|`xtQ=sWBpLq$CO5U)w}c~`im^>+=Rs1U>wLPSY`9|s0j;D zxtCfFY5vVoPz`!UovIkuB^Or{feyK-kt_RtH)ioX1NhlGU3{EhBe4~mrJYuvc^|EX zazg-dHI;ECOG?PNGAww&vL8J@P+4#~sa`F)`cMXG47thV)5-Gc^6GS+Qh!|rE)2Aj znMr^u%>lb*2#ox*r3hb>ELLP6=gx7uq%bzAVESVd-QE(;7?X3mbD> z9=}>j*?Xdv2wfIEGW^lP5|PNEF2vu_ChQJ{K*VxJSnqn|vUA3k8$#1fIsc31P%r&H zQ3ecR;>qU9H(OX$NSr<%f?a4pQb8^v@bM(^CQ4Bh{~y9UNC?%kqq4-nQ1==W0JUMn zrwFGXTrhyh&5BCFjF4wNVuvcP^Et=ix2sWxvv;0c&39S4c}=-2kxpT*j`@N8R)w)2 z?I0A5W0->Wx#DFzAo-_f{xze7LHD+Xj2DG833EQllqcqMi1nv6a4I1l$lOs-;KrP! zleIhw3d;YzH&M+mokj^R54rli6_Xotz%9wB3c5ew6qqFng(Du2$fB=NfM<=J@A1`N zM;T9lB8g2{Ft#BscICAxA)TW8B-j#ON6c7wEih6vQUuyA{1P^xzW`j#rEK4AU>A7X zAb^;j><}vf^EL3r+Pw&b^C2vj|5PmqRjMuNwM$pxGmq3*)68)4@B_2XQ|7kMExww-8GQG4GaZjhlIZQKn_eSOgHuH&8lT@Ky= zS~PHX%3zHp)NuLqy!L3QIJbJxIWBZcH zZeMV!Lg34is{7~mGo>w!iJ2`1kxC9(rqhdS8Wd0FwDVSp7()?N&P5HX9{?s1IpZa| z#Ya>`qbsVp3_=e@qCpC5nl>rQ%P1)an*CGv;e|)tW3x9CGP94FvLG8QU5VJPYuk}= zI*bw=z(!T<*8(TvI=`afY=W$n;Iyo-KF~K$=92NoqleLP!3B5d|49A+97fub?;BKb zx|QYMyFWr)l=^P_LT-0F$%M$>%QqZp-5zyFkT`}23)*6!q>!BaA;Y4#A0gzPHfq)h>xbU%~^m@D1g zYblr4s_>W3ST1K8@z&r3Exdu|wQ7+rO7V?cZL)&$Z)m0(JOiwFS*@S_5(uYi6&?z& z(lOUNSyB0o)K;F15q@0TSQASVn=ibf>Ks5A83m(W=McAqACr95$VRb0(pIL>_0ZfOXv>DqDS~TU3LF9x z6AP?n>GNsH)`IlBedoEu=-QNilX|uY2S&ML)Kzl@;M0XvG)y(0G;wq1%k)P$PF#N1 zLw&)+E>uRT%?S&8@9mvO!-DFtpToEN);LLC!Y6rT!j;ic7=Q;kj&-HetVh81d?wqLA-tE@xFlZm&>boo+ewa~ z=Vv*^S;S8v74CXB=fl3QYRz=NsjVd3;FBr}S#V7ku8~E0SYcZlkT0JE%0k1}R6}*j ztcv^0wEiI?y8OHsSh9Io2lpj!+&Z16aC7I%h>fIiHybIgjGe;jd^w`2{gznKqT3r; zA#8W{*Yb|^mErI>e{vSl;9&V)I3tAa?isO3j#~z1gk`|WU7)sx1cd>5B&-I!uy*Mq zlzn%MQ9vTe(?x5=(_$E*O89DdrLI8ch8`6HxUP8Lb1CALg?g0@~w29 z#A>=gNH?a$?tyU)UO6+(?8*=jDIhymo;MA6H_DKGvRN>Zm*q z|5e?C!Ps6TS6;x!=P)2;ik^=*=`Ih(BCBzXIT+l4r_F*B2SR&LCZ($z(B|VamDreF}?H=!(&{LjvMU>*=};E;!I+j46=bw(?t=)AcdCfQx9v9T!AIZ-z$F zZaIs$Ub#O&jg`Y|WRG_<(D0sKpY(_RzVBwQ@TYUoPlX4 zb1M@}M!FlC?`!sK^l#m-eldz!t)l~8lv~Xt5@ISrZAxZrDuh|n<4%EUaY0F}LtZkf z+R?fcdmqmtklWv;8F?S_B(0^4=0CQ`Gr-BU87I<~%5Ky%V8aT%^vIG1q#Q&dJ}#+x zG=6IkS5Wfzhy&by6P3$OdbMkUAB$W5H>%4-n2AE|Dfj8Sj&NZ6$+;#fmuF0sX*omo z1iB;z$9&sK337H1{ooDlBCKPd7)24Cci3ac$d`TcM1XRQW&Aq&vZr!geGtQS)%H34 zF%;0n2HjiPk=~SyVb{XZTKP1*snThm{H8m8Ta(3$d)%KVv8PI2x~YVh3aITbhMpLU zNKJnFJK{+ora%yOI^igPcGM6p`3Z_xKS@(J-#3ZTjJFmjdnOv{Sy$>YG2rR`;&K~U zG9oBa-UjVeQ@>05QrVXwA`^hcX=e$L-uauHDEU~Wy)K`gHF$9s9&#QUVBF6 z8{IGkgd}o{O0+~$$kaL0*|rnP#MO8gvf>4uvDt>z$=QSy`U3FPJaP+ewcVT5Ln~U@ zi&lRtUQF3h>wqXlLC>nPaJ?Q>w!c-2inR4F)+cDg$2uHr(zkCVH~xi5an~ecOi9~< zB+o;T>4oI_8xqyEQxpaSpO-(D&MUE7nDeS6yWIxT*v|7}0><2kK(c}D-DH;!zo6na zJM#*5F?{6R8#i?ik-Gyk_t&)cVTZr^^@*%As1G?4GRlbA4SZ=TX`H_Q! zgOIAJYBfK8V#lh7X#@VjauwM7L;7)}o5N7TG=fgGFI?L9D$yI_tE z--z~MkqqB?CBUdIe<%x;>$g5>+zM$sQz-Tgn8GFwXdh>~c#{H`fCPI_?nUYKI$_~6 z3g*EjjX1(YflX-7;yKEMcUbM+sZ4Bmj0$#K!4)b@Q5Er5Zy236ZWvqsn(c4Q2L7r4 z=27igSnaZ!zec1!Ue{h5FQDNXg!~E)njth32B;y1%MJRQx56AV`A}jAdN4Dp z)lKu#CL%23Vz%@$KhDLQvG-

-Bu)ej8~;ac=ddSso-fDBvSgIF!}_28|QOWON|^ zx+M8Ms(X+|V=`BUNU8mIECuyPd!1U|^nx^%jpY+2HD17+&QY2Ag8MZP(Y_-!NYuI^ z`fj|n=F?Z4QuKAH30IcHHA8A*U1HDu4L#lUrybu&8) za1vV$8gzZ!LcLTL#PDa>w+TI)6A2cl9niuX-zn;(W4+IbZMo3hzpexUhQt zz@h-{mvZxsyG=M*EG{3QwE>IEHLY3~czGn7GT4B;?47LDFPWMTc5e z=YjPr^ZAjM)kZdTeg^rLT2}KE;+9;9UF9^!Og%W%s)%5hLG)FqYzjJBL#7@uiJ}q1 zM0}j;?1DMM62dT{{od{>CCJQA5b7@s3_8tZm=&7s+f+LJ##=v6(tLwx&$}a14Iy{g z)|HE`yPza8fdblAyn~e*6V)1nkqCeJGy&^}!aa<7_ZTO6(Qk5cj{h7{-|Y#i)QH3; z`0)}HW39d@075IGp$Fzt*qvbcR7}N2wBmWD(^7G9nPlclqKw~E*5k## zD3oex1g^NkXUFS$i}NAn%Sty-GKPePD2!Hb-b+6J+Ask$HUx!z!_Zb_Ut6S)lIpbe zit}~oD#Xnvt~K|+HY~LJq+pF zudP)IF1}y_U7*2{oyQEuGXYJ;YvX?S3Z_~z&vG!$BXU*waLEwqb1Br&(*S+!^`FWZ z2UhEUn=H7>SU++9<~iiEO2KMm(H)A1kWCoV^eqFjs(Wlp)_Tj6UVO4s`GMgB?)g~$m|LKg zxOmdW40DlI)czd}oQ_e^p?$4$7A%Q@}V<;d`u!U=_T=G4I3a*m^_sY)U zr`px{ugEGTQo+*Kqr=GK{02qT{;%9q>el2LL`WNTGCN5LWOj_wm|0s&IAxRd!9%Hx z|LnMJ$KCwtRg}bfSsS>Tl{+>J+;%$;jB}vK>h@3?_u?OlB)fSNwK{S=NJd^y|GRKN zrr81pc9^sn<_jP|QoUPpPw7dP4=1&!?397QU0*oAyah*KpqOd(pl1A;vl*k{MUrUM z*E`kY46+N=quIGP^^qLD_U2=+lx2Q7@J|~(8niktIwEWLHYj#Tgf*dswr3cm1>IVM zlVVog{5ydo;*8fPoZmXd!stj4xUp_O&yZv91jFt;Sj1l%OOc#AMvK{D-%HZ=8*f|{ z?^#I~^%C1x6m+&qtQCRI8B(!LXFg_UFI_Vya zRMN*D=EB#0k`dh>#QmSUWhYjRPY14}S~We#!U8}*Zv`Zt4t4Y-X605@8iQFmLgA;D zSvQCZo`%g0LnmxMG32PFAQB1o!m^mDD{0EraQ(!W``-SB&y_Qi1c8G19Dnf}7!vLe zTI2F_G4+gX2+u z_?>2OZu7Ucu%FBCwm>Pup*BvFUD*}q<}|tbVmZqdh80rvRx+Zlo~k9V9JXn6IS&fS zpz|Z%nj_QlKVu@TgCmRQ zu6CnyF7nX^60ZubEO6C}iUz7tjjt4`-0^m(rGu%61uWOetCsyYg=a zNxegc{On0W1N2^QBMQMwD05ZeT{2RF<@V)K=rq^Sy+4MX?zQDoEnn41vi#k$4E=h7 z?*QK?j=0wwXcaHv{r}_^LXHY2LfXW&5Nt5>jpJmW3RpEhF5oZroh3q`%K^EUZ*lG( zGl&SG4vrW80_bOg-~?dfA*T{L`f@=hUlmNi;{DKuZ;=)>miyjt#??y5ML;w42Y-bcd zd+YBOl5%~QtLq{hKVkQ@_v;};W03?W7@3Qo&C~GytLBEDZ{bkd!2QtVBf$|muHBpT zJgbj)a1*O|2lJw}t^dE4c8%a~d3%;L&?td<@hAq;DC*2zf} z)QJ)Ax2aAsU`aj>fXe28+xkpV?0STu-QO&38*Mvh9}6J(s@J$H^XA&$?MbbRS+)uF zmF+7+6}GgXnMroy*$?)pKI#Jb6*@rO-iniaPyhdhH#31qtrC%|evys@$%yP;6HNAFE>%MYG_kbw#o^YzhuDT^knX0$Za9|IiDoF2v?^YE}U7>JkKgZ ziPOQ!U#$Y3B#BhA77W1YV*u$w_v2wq`JP&$)?Ya^Fe^*tqUD#-;I6gt^s0j9DoN|J zgF}jiLm7~r5w2xDa#PB$UTOO3JvV>D+L_M*$?CwBk=Ti@fE2n@FP;Bzjg7O#R35R~ zo|i1?u|IsP&_)!?3twfE3!cj?-QC|w_zP}^ z(WJsBVP|^tganpwLW{dqM)Oh%QK+^3(SAI-6M5vTn=A>M{OsSv+JEu!lFh8!viFEi zROZ&@%tKM??;1QyqtHGJ2K|Zj##rpPKRGtOnBj)uBXJ8zaJM5GYuLWt5C}6Y{jUjx z*X&fhR*FUMxT$hP5qtwSDx;t7mqreW(iO&T@%ic9617_UD6a4q`XhQ9f)tyNNNa2SSwMr6N0ocO=aBXO(Rwx|4R7;BVq*ahY zqE`_h_$q6I(yMs_Md{ExE(iSIL-9gMhTBE~)WC0yFesRK#jBIDT*vdlaK7%~;q zINr*3t?Fh8sEF(e`wW*MrySm@q1s%1-Tc9u^6uw9ii))`9)6`?rG3*{7VS0`0I;~f z`STu8lS~J9O-4oCWWe=n3&_nA*Q0wnYBgN4v6N#SKBh7u<6qGv)6eNBzVG69cF95c z3p_tUipk4%X6wDoo#&_0Fjf2@n(?}dgs-<5X2n_CGZ210E^XsLL%MHK39e}7)i=h# zoF6|`Ki)E=hGK@FCaag+jd9f}P-%L>I9xU!?03RxI6>~d2t2vp)%b^4-ZC~HsgM`{ z5^-bkII-bhQN?A;TS3YRG#J7sh2dju7@VXh61XZu#>kIaxP|giKX`dQk4}F^Nk{zw zzl%uY;O0`Dp(Ty~Q`e4Fmhw5o;aD+I2?-njB5Gh>kYo9}Op?#7_H`eo_Z(vd1 z@e+Y`6Js<1-VY`$nT6jW(KT4!OcLD7abBj>5orY-v_Q9cg90rroNF8?_Aro}ZF=gI z5klRosCK}{V_kp}T_^GND@`{X}}i)~iB(L5R=y2M@;AynW>un$))$d855I`a0J z%2o4GkH6AsSB8(0*$0>LhXmE4T|*qhk4ggOB0AKS$H0_`59Ve3MxA0jofJ){+*yJd z5iLrz#yk5y^93%1zvs^>vtjKR$Zbj*N%ebc8WXzxEQFcYi6Rtc=(qd2WDYySh;;D1 z(Xt;ZP2IfFh~iJlaw;+dLS7*DuZ+!H$ze&qS%iPAkcd<8eER@!HX2K&{w=OJLZmX` zhGB9lkjy>hls$q8wBgQo)QD-D>9R$J{8>F;f_NkkWj4$e2^0TG0XgUxUPl$0ia1hYjFb8j=-uK@p8CyLDuP7!1E6~pcQ-h|2 z)wd6&7_9k1XVhiAwCTd5YE6YQZtRYMdM(_TKZgpL@3$Bcidg~d@uw7$Dl`mYL z*$2(4E{_r=LmvdRuJ+bdH<2YW(e-Qbn>IvA@Y||;bhma?MS~iT1XTG?Z^UYu(~;LP zd`jPvGMTFR&D0Q|6qYcv0D7bi_d}>R(gujy zwE18s?uShlJk@CM=5KYV*PIq+Ly}B5H9sevy+jq(Nl5`pbiG;jTz*FpInvqwcbUkG z$7Nz3+zk~6^YX1hJ91yfw%MMk0}97*vBnF688FmSY7sEVJW;bU%pBj@JHk152b#P@1lmiC#wq`ykLMv z4w2735tV*XxKQj)m->95E(@Gsb9v=c8T^8eRbqI@c`;`UIg#-KaUpx!)r9*jVzexc zW{H52Z+G-V`2c=%EG3fv9?PgB42j3mc5&cU*x%}Yl=La!)^~qW%R!Y|@!y(~9xUFH zA9!iO`q|x>jyw)bdHWM)m$ObIuRZoDk(ss}UL=z&%);07^1B?iNB3~gJp&osOr8}) zGKstJJpVIVD^^R=`ouco)<-Lrixh{2cYNwmST}J3W#hGKB<=Sz-t|r)n3Vm+@68<2 z0F$!Mb=7?yC%VCG+Y#Tx5sHpJRB$nx+zGaTtmOpP^W*yyAjzN$(v|ktT*DF%eNry=i5uPX1G3 z0BT9WD>i;;HI1duwgDWEKyClLfwf9ls~n#9^+D^yysQO1Cbq;J=uK8Q%!~OXZ}*A6 z+GHPETKHs&jNbh<8tWBTubK!T%zaB=KnP@(&}H$}=tS2z{!(fQXUBTX57t`7Q@gnnN`61{~^}x$!eBHB*S#Q+=q^S!BQON33}*L(MiyEzC7w zD*K)om`-8N9y*szlf1N^Yk9H8y;zyt=P?b|cY|_#J8|`z$m(qw1L(7vWQjmQ)6J}^ z#PfA+LC`gQG!%}keMw77dZ(KY1F)z!(Oo-rPx;mhq|>$f@KhDbD93z=&G+nO9sI`NtRO6XNINcf)dGm z$)a+Ffwg*C`pgK!BC4Xf1Z)P;va-iGWPT&LzK9F)o%2_Bc3%k= z`7A97bo_qXd|F1Rf}Ja)x-u5zo@+KIkQHAHJG%EsIdFVO5G_IePzpbycrAu68PI{3 z1?o6%b;DJhebGZ|vZ18UvJMLBLb4g7WuwuW%05^a@AXIE4un=&Nab? znf(94xOEQ5&9VZ@@h}sZbMWu3V+{Q%i9$H9kkE^XKp`Kx$2NIk;j@;g&SoA-uXy6yppr zpOT?kK5|Zbk?4XHGRa#S%zc}<$>yL*qy9fm)4e-yW-J3BtU{xBN6A0y4H?xuopv4G zL{AixiCyK}> zkLpX%0t{SD*ob-rbqmA)DKUY7aoUb}R{vjxR}W{4t@ILZFhAy^ildyufbYxsYZB8* ze)N05#ags5DS!5ho?6?BXSWAjHVP);#^!ITV{11Tk2MxgyI!)rnQv~pu9==2x6V^l z@`=bJLUYEu_c+j-5KX?j8+qNZ)!9eDvZSz}(wOG=(0PdN*X|GG?*ht4g$u9lD5AI; zYnRN_KeWxKzqk0k#?19Enc6fHJl?LoD@uZEP1LJjNqD@ioo*IzPraqWyF4KM1NQF9 z5^01Hj`PfI?vU{NV9!K`+6)-@Ah^pEEQYIyTaVU zi5+e5V?^k$wrf#>(?*D`^i*AlF@V}88ImlTj%=<_0;DVA(GHtYq`L~B>L0cX;_SvI zazIXcU^j)FAQwJ1M~yd$iOC6fS#`(owpZPGj>8HD!ik@G{D(u}C@g^tS(w??K9ILZ zmTlnsxNg4gyxnfayuV#Oj|Y;zYQF+qSrz7@H7O0)HG!FwT?yh4LOw!n6XXLJ#=Z$? zfP#?BS2x%f<98qVzmR%pwGuT=bW}1oHo#?!v3i?BV_L#of!?k#3W06|JEY?t^8G~W z)Ym_*)6phm<%j{p=o9yRozz@QR0)#%{wjj%tQ#L(B-B`&s-}WXgqU5^bMtAl%){MT zhdndR-P)YxaCHj0GT6=^cK-OoI}$tvgW*9dqYhXi2s2It{66RKb z)gwrA6_bc3Z_Wg;@7C!qKe>>!=EjMQIO2-x8D;gw-dVs%V7SJS{1vB}IPBbaH^r$m zlgevFJ=6B|vEgjKR|+AmXMpv#4BEcZq4|1a4bt<}zuZeQIm!akIHBzW^u?dCBwU#W zg{T-Md#>z_me8D&0HRmNF#7D%vV};Hh;-X`@ zCmwOe#&9`%#S8ul`(@sw=hv-UIT@O3d>_wnu9V>o8pNKzqu4)AfCUl4hT1=!VZQuY zg?e^Cg!m;Y_LkN%H_Hw=GTrahn{JG+1M^l;`X?WO4DiH?=B@@l?)%`p;BA;SpL7zw zQ6CfPS+^vk!zB^mdEX?l0vZp}v35YB;+s#HQrQUX@QS=tj+p;;&2Q=QXb_KFyOC~~ zP=j6rF#pb>F*M(O>vXWhr|eh}_t_@3e8nVp!F@%4uR3fti)oZHfl5{XAcmNT;bt&F7gz=@}syGt>6-Nu$b zJ152Pyf+iv0<&23;IB`J4)r6NA;!~9hWvavv^gh4aUeX%oEQ~Z#Xrnhbd+EPHND=E zh4ZQMj&-G0HiaZN-QErrH}{0YN7w(p-9VPA2t;HWI0u(2#aStW*69wmo6Y0t(cW_} z>brbQua zM^*D3YTS#hqs3cj_J@sS8l)|c;bPm%H?V?<&l8|>B)sT}1mP;Hb2fvb>1`H8^<}MX zluo)TwW!2z!I(p4h1p-t9Hw)_iV*wH>Z%!zy#-jBq$xX_p`1fROa(>52xRqLQM0vc zQM~xnwp;`%T}aWZYjG6@lELcXIJ0>>;(imp?|OEwR=He@noVY}I{Ugi@gIL(!2!U_ z`nOJ{4|fx2r(_LAMsXafq>=V)Wr%8yiI^{GhOtdSYuGAySXy$$F^g5B_BB`rkZCUI z&&CXVSw&~>V-Ct(S#6?A9 z=dKR6qv2lhve@SG9KL>~{jU3Le%Kz=$2S4b>%#~KTupJB2_{gF_!RbHNhSl%?PS5S z_q7caAmSNLtU20PIuALRdAmPZ5cwBv;7su`QLr{iY~m``+H3?p>~s6Dz6#a|o}+h*t0T1C#khkWB|XAl`8 zELefGdoy!1Y8hl^;n5TRUvD2o8fBH^epC>`LaUnE?2GQ|p!tn_?t1((7SzE)qY}iG zk9V)EQHN>?I4$y?jRg~|W{cuP!>q%&o$|>lJd4%2b5VZHd0L3lOPD}D5SL~I_!JXt z^RX&iq{WXaFBCwGOKp4aTC|ifFao96A7B5%q+Yzyvvm5)ezZ&Cj-}I5OPDCVWVKM# zW)BI49ucXbmL;~IpbsB1g3@o}jjwnEB^@oWM&VME8UCSZF+Ui)f}9#gR}Z6zlD=bn z{ZL=ALmIl^P#;{{g?%}S#6zC%5*?c$=Z90u&00Eqsv_;$BhDo;J^w(x-{ux<8;kIl zLKKoIvc!Yk;^D`${L5;(%y>(WkzaYp_^Qo1u!*aeznSV>#@=N-T`p=uKYHtl|Fx1t?Y-EMP<}` z&4}MU1;Fy&)eZ0K<8ie$qxg5{E&ga>D+F!_v6L+l9#(@uZ(>-7p4VTO#0wusgj7uS z8;M|eKDR@qKj76#ilh#rz|he$6<$#GQJh3e81+Td)a3#+Dk@hB+eNlw`mV^Qxp$7Cgpz}%s>SC-`OWS^&8*KRAt-;#J!lGtuFB)I7m`#Ql zcL&6_J^HPe8@I{*MB{NOPJGlK@pmp|^}#Pjv)`KaO-mBsGrM(NIA&MEze zUh|4iQQH3h8DJ}4cPPBaUo!=_47YmtX9^C5kgNi+EAe+3tYPnC|M7Z{(RiTHCGhcm z$I@hjefalhkn}jMZ+c4JT`-Mr5{=1wC2XOI1Kt-d=w2JMC4%_G%L)a$?{+uwrX38#f{l*_o zlPr3W1xSSntO#560!1Ov_LdQ+YYt9T+Zu1pTv}PrPWSi%6GihfpmoT3>2|Er;pxRV z@p3)JksOhXoq$?2m>fUj6d5(P=j(ukUAsCZcDFl#sFWl1>0>24v61rjxfrzj(49rT zvxTVYgsGsBjv^|#bjKX=Yoy8`1*zpog$hch)OS!QA;(1_I{^9APb1unyd-)Lb47#$ zOm;#tYUZ^sOJUn#`unKrN34u8QC|73JKz=96UU@4zWbWUI#jOyzglIdnK=S104q~*Z=eQ_NjTKRs#{A+F-qS!rB^KYm#3GBt__f-9ORKm$Y&-naf$ ziVOv8Jc=|u*uKYJT9U0~h5&#lRG>2rIP}1uR7DZLPgbA`y$DRS?gg}Y0XL+$J(x;m zaKPT~%<*LKQ0kMN|Ht^V|An0RqP|8@$J zz8Sw%j9%v;9P(U`LY9JPUl+Wm<7GaDy@JIFCq0ju84w>ta|WC3NMoDZodN|G<0@X2 zgDnt>*nKrvpc(IWn2^FCi9aIXpB<&YbbC0l0Bz8A(h^AM+$NPci}GLUg2XRRqUUcM zU3G~>%P-~ILskqiikaf}rf`u>_}D+N5MZcm1^HBVi~7M=aq*ur)DJt0`rkYkK)dFj znAh>4jjEW7MuZz+f~$09)5XxL=fIA_df~5A;XeYMy;1h5`6^*7H;9A(MiQ!s)&3GH ze1$L_NGJmS6CWRtQNfx`y7CI1Ca%Gdmib+0``U$HN}J7{cLM`wJ6mM0(vK5!iF>`q zG4-XMKNCRuNYLe@Rr^agJyD%E#!m`=4g65E^OXjMFK8$MOo#vuHm^_fyxscVD(pUN zKFl^^-0^o@@;xocmdv%mw_B!RXB+U01(J_p>9I1rKCJK~PuLNUZQj^q{kI43O#4en zsh|T9Gw0^RvC5RguXFu*mM5wG-R?Fvr_ZLlt^@WN@-diSJLDG{rIAG$zKU#pR}sgHL~89Oi+s%o z&N}JR+Tdc}*yG?-sBa$A#QG2(7zLuKciO16hYhR5uep<2)x*_#C|1GVgxeX@cwO+z z49p{#Yrk+2O0pphfHE!iaH8W^Xt5+i{CGfJ_5#IbEfyB~rOdu%WQO#~4Zm`_A5Cql z{eC8b_8h`N&+OFs_`qpV$h4wb1~mZ zoC03yQppHADo$_mJQp4Be#K@2HmMgk)SpXH^8N`itbbu#qM-gH6tRuIH7dE4UdPudMaV6gcL=0t?UN*r=oNb z=xtl1R7qGm9gqK_BpeggRpBd^{HQ3*7Fm#)iA$m@Vg8h`oD7M8Do0UVg+tDxxa` zHZ*TmCAq=wh{K7oY2PgHPFBQ-J}MYCcDS_He8Xu7kIFjKc%H)o%n6yeyG$@M=y}jk ztcehAM{rDpf}R>^>;gVcFwyfj+ePDvfaUx3xF*}rsUfDa_!i1EKpcC;2`Z!6 zO^SPW0Zuc=L$)TMXyC?k@ITU`P6-Cq;bl_FVZ{K;)YCpm5Aj!(v><^ zFcBd(L(;zcAy3f6EKAVE z_rVO3I*?Uf1iG%}rOss3d8G)h5blo}LYn$9hpkhK{_)qd&@|immts{^w~T-stHGPC z-v@;UrZa2bXYwh)q;krVmB)jyr`~d*_Vw1Fr)XH__fnXJy9nd)I!x58KlG!o0+<)O z5y_xW0v{(URx5f(lC2KyCkvwUD2~PnDv9ML(Z4Z5uJ0Cx&MGy*Xud51Elg2|r;0p1e`W@whT(t>7T)1zX*_cKw zArOhZ^9U{nOMd#~hW;oCq`v#urX61e>Ruox_S z8ld%vKbN8w7#=vn&cV>!$o@$!kqU&WkY85t==m3PZsTEMLhQYz=eNz`9<*W8Z3=DM zITV&%Ly1-xA-@u((Lipii|Rj?N(Sww(VPlrm(acX(6C4?qN=dcI%O8#uFMWwSt?;Dq*j?*88t^l%rPAM_0q_YDH zPlanX=={noJc3rOW9*oR%=RIY6c&FNn#p1(9-j4!F?%OO2af{zszr7Q<6%8d;8fP` z>yY^gW!5g6;bQ}e`cL`bB4#l9eyo%#SG6N1m*DKbY<9`hN!`Gfj?qBWWnCPPUb+p5~Z8 zsgmv2&OE)h*h68gFA;Lna9=(<(U^<|SvWoZ&6)%ks)XkT2@0b6ZfP0Mn5Z8m_KkiM zQGWKVcu89k`1d@;S}Ijbz)6V9zm`M#gWxc*m9SWLWk-fhflFb(yL_ve%-+O@sOFK{KZWwbZtjB=XM`7G z!~ci9jTK%l06+9^Z`@#~mBn)3`DGjVyz%o1(HJ}hqp&LCVm>t9MPL8VKh0?zMk$53jh!fV_Y6woTKxo8)@KSbClZp@l2mJ@GJ4xgIU;;aT z35$9ybImmgHX`hMo{c2R5%D$NN1ul4t=dYZ00AL`Z0U)5bDi&nuo#wepy1b^lH$bq zRwpP#cBTim>*MT{OKWgDscP#4ctn^#kfj~Xw_tT*w6hJ^Xs1n=w9penXC0f#;S~@; z#7XSWH4-Lklxh03wkyv@f$Ek(2_Eg>zt{j;qk{JGG9hw6bQYoI%eJWBsM`Sy4Uj_x zhLq-M(UScX)Wxj9#W)x!0nU@ww|-Hl=Iq-*d4N}ow&ucYnaF(N9(6Ba33Ka!PwxJ% zg9L}3qhb7t!^rF)`(rqB@C>u;?+USmC36oGE4*a>J#^u&vnX_j!j?b1p zOq0im$&o)NcaD7WncA*RxG|0GFDnIktv@A3daNpPxMLLr-2}hO*3u^{h8Biu((|%P zN62^E?_*sCLdq~F*GWsTsd}s$3XhUHuBPy4%A80Vb-r{7zb-B6HAMSvxbfzV>(p1uT-&1_IW`MV@~3~; zq%VcVH!f-Ke8NviDK7D6x6B7Ev(-za@sfJez=?pkwn>Gbzxa^3JV!w%?%|DAZepHP zGAsAG&aCt1hWR(l;^4`+|6TEl@d6*DX8D`y9R#fI?<%ckrBfz4UyYOR4Tgrd#*OR+ zOZ*HrlG~L%)p3$>1(?(q)3=NkZ4$**aA%XSXMG{ZiA*K5ttYpzb^(T=YhNP^FIZ?( zSAN*3rb{T_N9lfLolO?eU$QhapF%g8Li3iLxxHa5MURXiDz_w4%K4o0-#U%GHiqw-p#D3DQaSGL)5@vImQ8>ye6j@<5 zb=3Rs`Feg%yI@rUPQeew8DYjZ^O*9>1T^v%yS52 z{!K@_Rc6~#CRu zA8B;pTw81H>zr=1=mh6A|C-58&hW2P)N`gqdU_HK(UgMeVoU>ja&Yz>YpJwfZz8U9 zGkq95M+{=yquI0KU`qqP%yQXw0E@YB?)0BK;zsBVV`VK6cFMk4gr#9Hg3hmCPoFV; z@$lx2ODYd|KK*ZM`S!Icd>WV`zs4Ud1D<5xU8Mv*IRz^&DI3{?MysZvaB?m?e`j(rs`sLx6XW`NoTG_ zwwy!bJ_FQ}JiHM+i=4Qi(s&=I*~>L@=e{2{7A3&e>s#ajB}Hd2a%IZ6<|HOvB}p2S zvCONL)D8TfA&cmVf5YbSZV=3-6M~&po#D~U?gzRURTv&A-DkNiy8%4(s2i7%^_j7# zTomv}vRQa<%KieX<@YpV0ddo!?iDkpQ|`qGr}TEjhhNEySKSWtN6h-2j^maOZ4T{u z94JZ|^~%lZ-Q=O?mxH6UadGbg=xhInuD5K8vu&FIg9i5mcPF^JGq^hhm_dU}aDsb+ zySog*WpHhP`6DjSPZLmUtXCQiiIHyzu+&Thi=woN#%UE zwivi+ph1Y#d=%|;1TA%#Q2dTNb08I9<5*0V1mig!iqz6OOncvF4n|J>N?&Y$P)%ilLb@%e91WAPawZUrR96~1o1>a*-x`GzeRtmuY73}V_S=8UyT#svRs{w{c_{i zbd>2DR3eW3JOvJ!j5M$;lw{$Yr|lDERwWD-7@t^!x*Pq>+teh=F$7|HcGebl5G$M6xsJ>)Bv;8x$c;6hd)Yj+>@cX;P_vw+B zuLDttb5cS7IyllsjSaUrXyZu!6lY?LSiWMbemluMec_?ruOzN2vx2M*2U2P5Ack%< zueZ|4E&DM@eB;jvDoNa*j|tHEHvcqjI$eKScD*%l>X;L=s+R~(hyEODhy&vptYJcO z=8FfEXDX+@DowY6$2|FBZ137TV-Xd6KLZ#; z6P51q4r~Hns}FW6BK7XH6}C}zm`~YF)aO#a$FTJ5aYD5CKoa#z%i;Q;le&TULxXc) zCkq<2n=WTOy_!T6ymc19_AFAjW+o2W6Pp{kk`TNlpT znX0Up#TA)dA7x_Km;{fbpc|QuttAwdMod-VqYFJ)7 zSftd##fSPso6`mX@zYHs3~O@GEcA3#G(AX}jRtrO$FyE23_l}E2B^t?JT#hJdW?;# zk#F(!BwWG2h$!&CdlYOGq;aavFL^$RQ?HzP85O8;eIG6E5i$Fy@a=#xTfFel(n#4(i&$SLgv!M%(4k3Os7 z-rv8X6dw@S#99kquM`+YGFl_YoJj7UlIgu~P}(dc4QR{J5I!s3cU1S}H1W9ji2F8B za=~KFf89nE`nekWs>fwJP||su$-|5-)S=_vQ7hHnh58hI-*fiF&)iI(LuZ3tiVIKj zmLG^ADCV--m0F!Dt<&;UH|ft8hOA@-3teG(pBL7t>ObR(*!CfO=3sVztibeAuvgt^ zUGBJ~weZ0wE+)^& z;LYs%sw9{nIdb+oFr@3%3(I5sPW#!IGFsbJ`s;-GlGDkRbv|-?>(j}Ze;%>=bXN<} zVo$P?!@nbd3|tLe{rWcR77hNC<~>hOGjkOtOk8Ol{M~jFzQWk}z*ne|+02DHkYzr? z_YC9bzG0#4#1P#B*<73UPpv1+$j8?X&KbFo7#>Zm%C22 z0~!!G3QcGVPq_Ar&@i0zF>1fjPm!803cA9}a$QB&j2loWOz$LBndRJLF^M;R^5c8w z*5A!!b!gyiBNc|BPyb0Dp3;3ZfSuM(iIIt^)4jN*QxnZB0~YFGsrn|#Ii?=)YmrL5 z+!k9}c?#^tqo|?wvGV%X4fd55(IaWiVPpL9~zk2gWwx_c^omKc>+zKPE1<#9UMug88$+T4zIbf48OdrBYjmWtC zYWZ`1`=0~J#s{L>hDC!(P}2D~mb~he;%Y7C!p}4-QD?JOAYTsolPIlEGc;ovdE@3a zv-$$^mzT$cW$28m+YeN+O;27u0KAkS&D%Y<7J(Zus7g4^eXF3OWs}9g|`YH!|cE z84@i$@5uY7_Y*VOl_3nLS!JH%A3#{?X@pa!}5T-X8I4mjeTpm{tFH*&UN%Oe)K-I9c|({Gt~|b zijn2#EP$*(wCEXh3ercL_{NtFHf-CLOEBQ@wHB^Rmi_bm$=;}GxM+SS)tplh z71ARs)5o{dhKeM`J$_3?e^qPA46J#!84%@I+6T~TBmej+ecuzk_VPasD3B_CN)|mj#Z59Dzpa98 zZ}SEeH#q1PPjd~_&)T+CM-$0-eDcI!>DNbIS_^x-rngt5A zacV?%#4x&d@+?0PyQW0+ZXeUadqn;MLXb;CDVr>}u&yVEr*Z}m_!i?_D@Py3k$XZq9yB{;L0m^hGz^O=3i z`6$8|k_Cr$Cr-)l+jat=5mhbNr%weGl@lQ4UpWPMaMl!*423{>sZprADvnGCfqK2G z%6wF3JQ?u=6|35l4+w7!i*EB%(1dDv&AxszF1N(26ZK#;j0y3Q*<55s#w5_CAZ3`! z=6Q++8QNJxp;1WbC%F1GQMFziU7CeSvN54*7pEz%YxIVzTG@rmG}%cq87v54{NE(G ze5(XkJXf}ku7r@`=n#?XzkJeLP6{~hvN-+APR(~N-zb{EJuN}^kDeQJterPiH?U0- zpQ0g05Rs9!2yO+BO9nC{;TJBv*33#66@$ve_=yT*$nCA1>DUv5JO9hHtb$023awBLXttt4_{&3 zyOIki1L1OO-M(}7(EeQI+UqP)KJQQDiqDm|ABpGZ{NMWl@bxhUkWX{|1D&%CuWJ38 zs2jCu(n=mYi`81+vnS=KzPX-YW7vX1IFF5=mDg2iW_xo7>>Yo8brO_lJdfeXEGo*C zax7Vl-o)R;4Hs~81`}Ka*QMcR<-7nqczF)rF8P z=^Spl)AlVwEHFoFu0ROx?ePI3m(~rGd?Z%lJ6pJaF5ciRLu+104!`Xt7Rdd@swak` z*SheM^0JHbR+e>@amc=upyetH2F@w7F+fi=hvv`vpVn=Y=V1@suqrI3jMX$}A)Y2| zNtpM2;7|}j8#C;$3ms&J@<6b6Ze=v4jQkrz3i8Uq@}!7>?Fc>>DZue4etpFiASy(VaLlwz)dMyH0+kuedPh zgBofMe@o>7jVwC4oS7-7{E3zmk4+O4bEd_N&W_zMzOs+`@68%%{vJs+=M(T}{8@Q* z79!LSw`aohfYN($@K+8* z89K&ah|fO)pm@O0qFC{Z$L{s$l3OXEGk+yvM8+)3%;H#GhRseUN5SLwbZ8qxUlfiH zf!Te4gXkJNabNpF?Kp>KL#i8#gI-1U z)9X4f4u2M_%y!M@9or>Dtaf^6H=m|*D(Ml1g61T*|E1R4_tTf~ks|)FQ%(dP%jQR? zlp|P8Oi~b67W@^dt;?iMC4Gsy!AvI#$+ScO?K)~pvW(Ve$^YXG!Oa3MxM{3*&IWCj zY*!Lmrm2JBe_Jh1w@^@tHU}fG%pzW+G2J!wKZdzHZZ zG%!KRF-0nIsc_QuLA}tAM><>q{K9~oOil+{dF33hjP8lngu^=CMKD=TbBp6tzDS@Q z!ktZdO(WY-t3yW45H!OiPO-s`lpvdizL9MqSoR+dSPok_l(BQPXOIdng;1zUxH?hWUFH`q@5GD|< zMk)F)0_irY0LAJW!WFv1k-sf6TDlI3S!yT&$1ioW`>XIjH7B~9cD~wy_ThM7sQ<@& z6D_iJONi@=p0W_i(F&%`$>+DM)P}cKD&f_Hwm_yZib>wQ!x({gsbRk+mUKqAKC`3(La?&CdP zk^FPqz%gDw({4x5*091|QOdgiAD}(=v=5@n_%6epr2-pt|Bg2;eX8Cw^@2Jl+Bid5 zGi58*_GQz9AEC`yv8Nwh9ZWzP0G!Qc^e1?#;a9J>Dny+NO_iMeCaHDf=%&;7PgVp= zDUZzX17KXD_j4%1M?Af^8rV!iiIsnJZ4Zq}Qc|`k8Djg*_wR~0pSi<<2N%F4=&NuP zW-*Ea7t@kLfAwurbz++#g$hiye z?2dy%d8Uti`4&_9$*#v(P&I(%-g<91-^mv*UkiwCtt`W7?U4#)_Xv@0WwnTo%JRED z4#rDNM`sG&yuTL47r(0Rvr&GuoP%&+3vN4QTYl1co}Jmj*ONm^Gf6h1rNpcON7g7> zkNLQIAyO>U+;>s4S9TkDEFIzB-L25Ht^g5#?*2lo-_dYNt}!k?_-wl`COfCOD~112 z#GT_Ah9Bx}>W*}!4q>^SP3&s*7X?HXKPpJx4LM|-zKfWimO~$cdwiRC^xwg^XjT+I zV+^^+%zPM9G=kqT?0@cR5x@9-3fGUuCV(`fCGe$)(;Vju*|i5)aSAs^d7gf{7Uo7{ z+Dm?^+qqfoG5Qlwa{${?O+4z@8|EyAmadLsrh^FVZKW75i|Jc$)cEy{2&$JRXAU#-Oc1^qn2{B*c@Pp5fJq z1|te^)Mu59MQ4xHv!_g%wl*AbEW*+JE4%G>YNX~>+nsfbG5mLIg4Smmo;Qq8IG-u3 zpaAVsdWK}BOZQh3%D7`h_qYg%K(rq3 zX;Y(jvKV!}_^jiG04Y+dsEaL@-K5&IF?7JYQZff&$$;TFXtT)^G^`{RWkMeE__3ho zCDqxzu(0RCR;8u$#OwF%z-s>pnCaMd zXAb7HGh04ngpf)y`>Q>o1WH%#6b(X(4O}h(xGT!dBA&8kisbWtQ~^>LRsAGOKc+TO z4kvAg!C&LE`jtzB{} z%EH9lj_cz<$e1h$XOZ$j0F88TG$&%u2PXbd6J3Yv%-h&z01khiF&K55nnuogNUxbZ zserd{Iq;diS^`hRoK7O1P_S7OsH_r&2$4thmf%}_<4*}0#5lW`M(@B=cDSpf6*5+#qte!kVg{*C0tkm$Tw~b44tLgTBF?Y=79{esvS6MrZk> zbjP$4_ip+ydQtRe{!nSj!dRnEh04qSW4RlWlQ`C!Bzi}8`zBx-OeBJkl1E6SK8$Z#nPcy)j^ zUV)MrHCz{NuJ?vY$ormRw)M=U%(wWbf1?&px%tnww8)(p*&IV=Nn%aIon8ib06G0E zox|q~zKN#M*%yWTB#3)V#oI`O9ETqU&b+4H(xioM4Ncp14T{jFMp-Tn$CHXX#^Fct zmLBx{qQr_B6nSNH?=LKmcP7V?2WQr@W4Eh?bes|1xmE)aI7eKon7cZmU`+dCDfQ)t zHU2YiS@~xdUXr0|gM)RV2m`lFQ>>gtG)|U{Bht2cG=njVw=Oxr1h)krbZHc;Kj1cB zG*`6j{=j9+C2eV+*=O&I?L2fbE%uo#;;rqZqrb&Rr4)>&PyHg%5?3nvDdS(+xHyMh z#g$B`bouv8$lN|_s6y_7tcKR?H{pD>LS7Y_0?=&`97nUSA-saA!z}NY<--KcnDII6 zL{^Ik@icJ{z%q@Y@wK0X*=@Tu-`g{sqm(ILNE?}j^ma5&g%58GzIF5U8~eBeMo&}* zlFvCOwGwBPQ9g8V=8VcBrAEuf386%Tjvye4zJN$H+!u3Bv2Pk55Ni*~ge%d)R4{vo zz0N!ihPD@zId@{tQ9u$>c%OM>L9}ZWx?-QHm4Dl_6)$fk1o@H|<(=TVs|c!1p+P1z zNUV4ycNs$ig1We|DEebKzBg8Dn@`W;CazObD9`uPSe57tjgTe&(O)D>s~StvLN{5{ z3f8NH#sHf7UPGeN_XPptY7?E04U)FCIhFD>#foG=K1{LX{QF^?pt%Y9uUsLU6lGNV zR6Zihbrf{Yw5RumPf0F5G?mA1YGX34>onBZr)LDbk<&g}j38rNACpVf-is!}8OMQ0 zYjg~t7`2K5SAx!3m4(BR(>cjVMLtvTUuRX-*9^^EC?qhL%o^xD5RT7f%cux2L+7%! z#c~h7b?zTVcw-x^#(1&roBQ{EA(X|h`xxlmd1ZDhxq!X7uDlXM9oeAI!E@2*L{N{t zz~VB(-Get@x~_QQlzIHemNxY%p?z+K<>^k`EmxaskzlA;TPuUB@zyrPU(=zuJfVdomhLe_YthaIeYS1F=QG%dxk=qiwV`rRng z8aUva>|?qcNS&L_tVk@E7(#?D)sa13IWfw~VtT@?{?r@;NgBI?_Lc5ByD~$5$0nY& z*GybBBf5Rl^5>eK`z+tY!>TIRc;}F|6HIF4)}C6YBI0hJKJp!l5iwmc`{$y%a8f zd`X#y%zv5P;NG(?S~{NH2!R{_k2L*G&08 z!P=5`1m^FBIQ?tr2&nV}{4rvD_1xPsTI7kNeBo+-__5eOtMkjtl(uSr=cbpQXw$va zqcC#JAA~)*pK@9$K~0pX3D$o)+RMsUb>JTRaF}`W>B0BNiq=BL^T6wRWSkCZptXA4 z$`vo##jkH!E=gzxz>WB;BMF!d;ex>~|2}(T0>M)FMraah;JbY9o9YbYLw)3#Tkay# z@^$E3EqE$SvQL zyI3X->oj|TQb*JpDQm%tEg$~!gt)4fm^<~e|5P?aS#Lf%hjGuandq>V@L@iiV~!Tc zzvLG~%P>G0Ba_2AS<{3c&)a#j;BafEQ9|Y(Y zB`Td(?Z6qpujsyIE?IomDhT9S8@;hJE&x;y8C&1p zmW;xt9zp#%D-*((t3jBqfOJ(fV;N==X@@PayY~xe)!D(jT1a#cmhELFB)U*u*l}(I z9%f>aI12upUv`6|s$G@A1*7d}2CHB7^Y8=TV&f>`3ZhgQL=c+ev^8Dr>g_*+!sF`A4VyMUyGgt^ekN!PwNA z78y$x8qsxM8A7~TmAJ-d0KgfMpwkmb^hX#H6+~ZNiWCahO3rGkL>b5~7sBr-q@r z2bjOW=vagh7Pc|y3EGISl+c(l_vM5bK)X?t1f4} z$pwFcN=SDCp(F%ekZ2(s7C@h%+(_xxoobfJNNg@$1|j(tEtAKCoyuo*S=$uOWEo~8 zUzEUG7K_d?qUX1)X3b6>B7~k0Tc}LDk1)oYh8DlRyzO8y_y2ga{|_|#pY-Odu6jhx zG2kM?xd5;AYg{8@SU7CmZsjJmuRl~d__t|-ZLn*@o#D>t^&bJn@GtC{&-7YN&Z_-H z>Mp+-6BeOcwb-Rctw6Q{6nbVPiB84#$Yl-pnM~@t;2Q%yk1?jTQln5Gqy`F34-J@+ zHZmL4Uuvo~a24<>VL=Ax+P5mIL^NT4Gv0N&Ue)LPc(0D!Z8As1QB(cpK()flzmhWB7Tk18V!d zl&iYPggf=0xckr&!ES*06iaLfKeY|@5C7%?Xeh6sYk|%rbbimqg=FwIONcsoW&y{jZ0<fbBwKtPn9vv%-xgo9^U*JgA(Yi}83-{rIb%Jg?yNZ<=m#EJ!O{D? zHy@}!WP(hCUbEy>V+Wld%XKmE#fhi#*<hn9CNT<3%VRo>zWbF5ImaK0wvLW7$talxiJqM+UHCKV~N?(bCYsaYbHjikJaMQ-u&UcUfBP)gSq5%n=k!IbW5i=-Mi$BKyu7=!tgvL5rLWL`RXuoI+Av#O} zg0`Q~sAb&k$@-YWM9}U9))%ctwPa;o(hjeaF&Rv%pF*$Ppa(5kk_9Pyw9}t4{B*y5 zeQ5LvO8oJ2ieJKDi1lw4*cyI=spNvDtveealSsfc$q_4yT^%q_C&`kS`;g%Xb_u)M#DW-SX+heKAAonj<#f9G9A|Xqg|Gi zz3)Y2Ss(BY(>tZYwVd)Yvpklpw@w#3993DZ$IZv7U8rMl`o7b^v~4PM4_fi)J)(w= z_cQ-fL?;q?WIX#pz#CZf!F2<|!)12&?vsvnbK6`_tr4cSnD zqX^9*O9y2YNFF(t0wCSS6k_lRV{nftZVybH{7UBsmabg39r?7Fd?lSj>LBDvz=&G< zH7dz#XOPja*2_~ztG04 z$LnbFkM$gv?;Qk~{pA7?Jcjb8(r{p(IH@@A#2z#C0jU!i9slG3csYpUuYPaR8<1pm zuAH&SNTMm%MdX6Po~fCE$cz-3c^d?Esi}#l!d5*;*bFgkWb`xvp_~w-aPH-|1VoR` zN+@u`xz5dpG?J-6p|yhuHA2=2hm1MkX{<7K*I>*xs)-2~(Cu4iTdau@(q})~*5jbq z7U=ivn}zW11@%*~p;=1fJ38pR;2;zEb!tR$?*;@YfKRu<4ygiBWR-)N%m#27H6L*c z`l;r^jI$SX8S5?n_Z*{a1?mqzli$Tr*m}YcfObS|Jtg#pY9?$T!sZ@&!Qz*PXl!{EobaQ+H z)be9==726)TOB~*7eYuoC@AIY?qAqT)%g40id$>}C+*|1*qNk1a$}v2Xjts0LNXl< z-5Y5V(HB^<{=ohrut%2M-~hA=nb(!fB6yz0UK#g`Tz;#0u0gcTO+fx`W5!-Ch**f9 z+N1nk0jg&dmBY9bFIq?MtVtYo(;93rs7@SFQtToAiKaf5WF`NDw*&FIJ}_ZJf#Fvh zMNwu5P_fw{0s_TJmk!UVe4UPHsuOmlqh3>GAU#&Ub76Fn31I7G#CC^8(+epXCu&$xrv*Ivj~ z%lj9F7H#$E+Z!570}fQ;Np5zn2t~hx29Yw92n1Q+gB>wv_CUzgmhPoIrH8-M@K0~2 zHA_^@4bWFieoSCws3ceLf(mt(<$Deqv2oy_098MxSthI#S=wwo_)D zHWioksZAlH;y6XF74zV3DItXRex&Befe^ZT_JU<#nnCU48d8e?0Z4BVxWdIzMJs1N zcE^sNM~)Qr>p2}iQ(93FGFMa$vr8!Q`TOZ#y7{Hh*n<1g1d5|FdhNU8iXhI@fLIYF zG`FEoutc5IM^bOV0$6gW7=a~;Te>zof&L2lcc%ezXXxplpG zC#6QVS`iSOn6wXfqPCuM1lJjnE0KzE{l^1X{_r{qaF*W+T>t-sIYAy57o$$r^sV* zY9{(E=GPmma)irv9!x=gu-RoB(U~_jj4hciXv7y~9NGq*zrZ@pt!sb@jzjm@*+Mi` z8g~SG&YN&6iY6?-OwZA|!GwCxH^&eqG^}2n{r-(WzrA&=gX01L>OqD*~ji+jTNTy zC79TO1);brl`|`CqyN66`CF~?w?3U4jwKTHw&AUET<*;1zzsFeWuLuRL5d{TAi-|9 zPtaqaAMPIjtjxh)w;a86cb(KsVh2PKxAaaQWb>=tNY8$dN+pzF{WfrN(Yz?SJmZ9# zz*elB9zFagUqRz)P1OG)z5KU#+9XMm9;Gg9f0|AmV_*{^Rcl;TA58b~&S5RJ(zQe$ z<=eEc_VRUuu(39`7vLWn=fLEkUbQ;^iEXJEY2nZE!q76JD+%$Ea{)WT#F?XTH*dvY z;o0Sjbm)fT@0urFCgu*4KxYq`?Gy`;8sZc3k-Z!(U-|rqrMGjzpkl7G%6!bf;|1md zciPfP16DthC&T41Wll6%3-CD>Kjrs#Kx1v5o+I@w7c#Kqj0T0ed^!OBLJFN}OR;+} zmulf3J&6zn{#2GfG=9O)5C*PrUMmW=Jkh z?AHhK9pyz%NF(9maN?{zC4 z^~FEaU-hWm0#mkg^D)rH4`1pQ`^;b61>PRP+Gb=?wVX9?&uBwKZb3+vsUVG$U-Uhm zCf~pu`FAXpzGfboqDz{EP+48HlI30yE<_|;d-?BISRIZ!USCJNlJlRv(Q~6n;32wu z52^G&0kdgv?H0UF&jMb3F1VI_6}*5yb^ypTEFlkIzOj?(ds}u4ymb#K*B}S^x!3^* z0d3Xx`;0M(o1~r&o5>=l%P;UuAYnakG~QvkC~LTHRMNSZn1q3$kZ~$#?F%P#=?H|V z=1D6=v4cd0wizwhoiqshpP%!Tu50e65mcJs?A)`#pW;ME=8zJv>a7qWS8rzu!)#G@ zZVZs$kWBoWKST1#5*7&BsHAP(9;Gp*fMeMp$t^OX9QbWvF77H7cab5GYZfwmi)KQ|M3Y0nNw8wV+!p3 zTRzFfLsUIROrGw z*Xi)sGW~C=i6L;gCBS$6JI5P4>EQDilyAbuIgMAdp{jhO^T57(c>sz=&BLZknl@?p zT2o5Ta&j@dCjj{-R}Z%K4F>7{F1T-(0uT4=#6nIY%C}iU|5S60Sz6V?+jxw%d+x_{ zHbFzA4L{>M?^h6C{?f3<5trbHMBaYjlLMiSSbV^Fc!Juaa{`GV*7E&wK*O+1jvZZ0FBLXO3tHtL)>5pAj_C=r4M;i{N&P4Ns{1pc&OGt z{~D*3HoV3y;Zq_r{xJgc;*>i&G(jL=XFQK%nYN~l#;KVW$(pPe9T%1zs6nW-@;O*v z8_Y5kv3g2$n_WD$&*)b*Ctax=uE2gmyJ4pJ5Ti3<9hN$Wa@<;iOK)S)cpB(@ zlrN+5%k{+9UCt?E*)rlSAqxn4S4EhI=!h?bcK{DF<@Nm4!n9RKYyRfIk2~t@+$p1g z9!QHrIK~t~>qanoH=4}VI#5Y~H^$Mwn8h-HwfInc8j6k9yfr0rGca!$5f% z?*>-DJhKtedd=5@JiAWn&?LBDJrHqnn{CZBaJ?>;Kov~OkbA3r63;Ik&;Qg`1e~-V z<~#<02RQ_G7JMHgTJP|9QPT!Ga?SFw>ns}tR1zd&Z1Gcv7uIqj}xKrrIbH#N|XX1*Sd3?b(HS3~QWS4ozXwo08b ziIBN35bCGc>(|}Ewb%Ero~k#QJY6YM=WpU|YT>O0QSjR20{}M}EH34nQ5y6+DM7DM zR_gh^QH1oM- z6yx?0-8V**agKij%jePG(mn~h0L?#Cvn{n?bF?mom1Yl z415XL>dS4(GcCep0&HLMQ}Dpk!(`tGFFVCV+}+{SgnU-=q`>+P)zctsa)uK`)M%8U z=M$$Fc6<&}aOPzmlMgwCSa*36^5|~s(StRxD?21HkY7AD&R*|ANYpsiKf*cQ_AOgL z&pt#Df8CYWyhXYIhDY;HHc-Hic_c(a+>)=1=>ajs5lX81liZ-3ef5~G6PZP?MqiD$ zI8|l*F89(KkC{U|_3G2z%;NtoIto}k9UkZ-2BQh~BB3ihStv5dv4RqdITCUS`$z># zPM2HHu`G{WUK2BpI$~v6Yqy>5`EPat^EpZpk&-e5g4F7Y@vh=-hU*{nzy8J9nj&?B z$>yc8k(;gIL(2SnpS!qCZ^hjlp8u~Nsz$@_K)%i_U^nw1pz3G`x-}EXyGY2V#nS<@ zB$4?-c<}p~NU%=#KNNQ*=ApYgqd7~sfHUKWPCzC`A^kuaOANJ-1vnj~d!6o2uM9pp zPO~x$n!@nHpL=&bf=c_5F*qs};Lj53fJ@lwi7usX_6R_H>4_QTfT;uD!ImyUrfpi6QoYM;|mVmMU z6C+sSv4$St@M@bc)zAnUm@hdkf7lnkLjnriaFoglo-xI5V$z-E%6JRCIql9TunkpW z_(6lXUM2>uOb6mweKGCz0lj?^;)j+;l#w$@rtkBFquR47#MAHy%h17jJ-Rb z|KbH2;?$J*%lc!y3UIA(U~se1%WbgWE9f&lEG0LUBxNhljio=P^OV&*unMKR1H6rl ziQ&!ZLkJ`;#3ogn?akt99z+RKM8Lg+TVUw1*eCan#DETR8#T8OMawwppK&rX2%*+2 zls9OpfncMB+pUR1_E>+YP52eb|G44Nieu4y+CK&P#BIAW=HO7(uuV!<)=c;de9!nD z?Ga0XL8Shbeu{gOkE)EX_em^0)k~tEg__Q3AAjIlVyl>~#9>1)xbIVtvx)|!-&gM$ zJx7YsQ8sB&pGluG7eMQ)RSjN?W1r(spusH5yf71VWyN-zK zM{~*7Iot3T$I1JiI|GJe7_Gh1&p6u|`E8*9>VL)p<40tmx#vaK-V#EjH6)2;EMoY( zxq%@Yub3T+Sd*t^a!78}y|drUv*o=>xYD(+fSc9d(-oc_M_!Vs+^rkQm-ednV%yM9 z5U&?(bqu>R1V{&sONk+IyDhg1I& zY~2PM1m4G@mk(F|F+s1RK|eg^X+e4aXitIdO6?Wwmrg@SN60ly&$&db{)H~3JeOQZ zPUoX%u*zRqqdbhilGWI>?5=rR$C-#BBs!h?LU!YU(_!IV+O25dR5-LDoXvnO-LGr z-7CT%DusSuU@+-t)llEyk6osZOfF`8f-^&)~t^h(2vh8E;V zfA>|YH7YDeT@(2`QBV|^kjutsNU=5fFZYjM2VKR7cS8E;QHZ1zz2D039Z0Hb<<)r2P51N!e=V>E=I&LvWUJ6dJE{; z0aO52OvB>qq zhS>j+XIxc8pf3qVUm)fUONhvMcdO>82woj_um4@I87H%8LH+{P4eXX3r<6i0iQcq>{gT9O2bscEQ|UOrDHFIB zH46y5duVinRg=qiQ-&L>Y<8!^(K2S}P`GT+q5Vw=DZl8Is>6D5>`eVMDo-wSBx<-* zKH-+CA22%l*N7E+jNx8!syiw^!?|2?U|wO7rIH_drwUX0^5}AOlfXvri1OmGI$2kpDuVawaL)Db+R!*A zck9vprFRo^M)<4JMC9?9a6S~&S7s9`TPZ5%nX^rqbUM<^LGW3);&Q^LwP_2=HlJrs zW!z@E6%Q-7T`5%*TZD>o1nDC+_>fH9?_wLar?=btQ7!^LL7lNfG#%oez4D+Mj;AOl z%H4;Z`cdGYFbbtryby$sR3VTq9pHv0%9*DWlwSkWh=1Lhj977Y*pe2lqz_TG6$6Sufa&Sm!$Ze)mDbCwmFZ$cGs3&0k5X^daozg4D7rRwI#P=Two%YWcKzYuy z!H0Ldgorv@(nbuFcg+qZhT3qW;#n@|U=#B8TZh#btFLWI{=OcAhO~8MGzi{IywCxb za}uT;{iQmmox(D^_(%>*_GvraiV(?3kC%$z3rcA=1>p@|j|CbE*X8($5OPj>D?$%NBk|N zF14FTgBa=ipIY*^9zH;`YD1CB&Dfh1!e~99hb+fn5810!wBiOI#k?m}NV@u@0Q2(E1fZY%}w5ebbMrTNt$JF>)=x z^3d-jY|=0hl?ND=#nxL9?dzc)ClX z?>wmkOos*{4O$<&Aj{$k_zEtee(q)OmkM5!!f)Cl|6vsWS5_}MZW6#t3HDOO`kbaM zdN`!?3I*S#!<3r}C50#d()1Cdg%hq4s;t`0;$x)VPFd9LUUoR#y&euSzP+n#5kEQT zC_l9Es50VXB_V^p31$O<{guId9)Ksc|Ud;-^?uw^l#qd;A16| zm*F#s>9A_~6~MwO<}M8Vk$g0wtG{n>g^lNCFzbdPBW^pi1l8fW%W$_+&U0I-8PYN~jBCkiqm9xqR%0Zf_r7##voIHfP24Nw_!=a?lgwuAB^s>^AnL3J$&t{{aZWj)d0* zYEs-QgSo(Klm#aE?ozq1n?O)|YniIV2Tk&B%sb2pTTknsC|$Kp~F@L zqE!=B82OBpNRxCg{23Q`jqe&_3#+A!a{?8}#~!z-J;O`}nxp1%3iAfo&8F^FJj@u1 zZ#g?SY1lNySII~7hf{ zFSXc!LUzjsZ*oYi890Y@jZx{V!n(%JR;@&I?+I~t?ecqYonUFt-M}?iE~WbAYn0!! zurFw=rGx;!JGYMCKUmH5^6^`FF`=F~Y1h>4hqqX&@#DM&(}ibnh-Nm57&4a}`}ACD z%j5Az)CzjSv=$hraHbkKHxi8nxQ#^^d)&WZrOOa42E(vaRttO6s<>ADs1V{#V*AQ= z@c^GQwf5!-2tj>`fjYCPkL_eR;>vx`>XC^{9}<$eZtagAQpykCppwcTZKM#;}L#qF+j z18?`&7mTod_h$_TRJ0$F_gjm3`4MHfCU8y9Xd~)9zq2;5qW0q9?~Vw<-}Jx#E@Va;Vq$GY$H}$5Rfl&Hft?<(B4t1 z_2!JE)}HaR0NahC$X$Qb?ux=+2Zf{%b#9NM{}iOICT#bOC?PYURKLVVv9vUu=s{34 z%FUyt`8gC<84=>%%^5k1)}| z^$Ho{D|s`Se3_TFRLA$2#S1Yht`G4nci!9my=`?rfBsTk!4?uPXYjC`7M(_Y7mVFg zd)M*Y?lnS~fwbC3ioE<@=GmWhBcXb|5QE!#Vj;!wZ{L z6|x%6F`7757tu=odsZz^C9^-RWT4Zl%wx(i2aE1k}wFBzIQPm`B7;L0z)fSX7DO%>*% zA8D&aCg-d4xkVp7^dSo=Z*VkMW~rGS%cbXO5(zwb-7=GPbkY~G_0w(U8CAD91R+c@(I9JDasaU!4}wIRZDL+>|VQ6ys$KdqXD&V(|KpTCf9QygRuV z)1pkVVp^#nt<+!rfLA6(`iiqN!FGKnISxI?cG=_4w+thVnqZ}mHzf&m|5@y|GAxkJ zV6QuDX=aK>q!~$%$$@k2hBhx!(c50GyLefQkZV3;F_Oz-IoW|?xe*OWnQ2rH^gkSm zs5qA1S7`X8(K^G<=fhf}>M3NhI9r(Yj*5^jC2=pm*rdzCgN(R$_U_$h%#A&|n+cwC z0y@Kz#{7P*TIY=Pi2a{xulR#6wSV9%$5XNi=B^e$o|%}Y9(gC*0&lDN7>h~ggc(!M zblp_M{74#BOO7q@{xq`pMQeV->h9gZ23_S}NIRyY^leq|_c&Iz&Dl7`Q$x z;12|DP`l(%h(rT>Q1jxh+zU_{y9lMeVgA^3L6R{A>W&KO24i%x@kN4Tk#YxqDr83Y zIsbKYU(6%k+nD`xYS1=pfE>+eL7x_J2q&UvIF4>P2dkNo+^{aDaeqh6xrkYjt6X&R zKA6bY7#S_+UpO`@F#%BZHjHHH;`MfC4pe*j>|t=G0q^9?0&fgX4I_`n0b)5lnPXh7 zlqg%uv>qkOBOkwHJWI(zB-_G*R0k0#)CI8N{@-yvZMQj*Zf0Lcoc|Jk4oWA-->ja4 z0YtN8+jk8%tlwA$Wt~6w5`|U}uw8|QBRLp0MOmiObH`87v3*up&&Qj3{~d$$N9pz_ zDkL(c&_K&3qh|{*1+Yi!S8?v*EBzMp`$~432Sl1Qw`W|`Xa7n0sR6) z)CE*^8Ipw5y@TaAu70CpV62qY*%r^?Y@e0LA4l$^=7o0Znzg|?&|kn*dmD_OAp+?? zm$x-Idk45SpC0%7U2VaE6+ zL~Z%7mUGtb2G5_-eo~}ao=xKQ7eCna_ZQIM-M8 zCD!WJHNf8prQvzKNYu>2ETZykYNG^X|`6d|F_@0n{Mm4OFI5yYC z_UbEBCIudGi*(d?QZol}0z%UugFlVzHcz-lrd9X4k1IfpkXG-;O)^5jx_a-i58k4e zLFgA(&d!S7#07!o?qZx~Bquu?<&licIi6F_74yG~?h27#YAS2vL5)lMb0I(Mz65OqTQLeQno0ZLY^Hx(%1tu!(m9=+rWIEG( zUGu+O|BYm3Kw!h=2f!|{k;SAT8EmLi&>_6XgOLit@9L+tBu{d;y2a#yx%ztn}9@w||CvWInF*>x;P2nzarfyV>>l+BXBvwXQf{A8b6`+{a< zysv<>_fSxF?yiRjUJd0Ydr)`06p68e=?svE7H9}2+S9RgeevFojoe@uv>;9&w z@r`9AR)lj1J?(d4xzEiNZ!cuB@fw`p$quV7ILa>sGCbW zk$42p{5F2;pv=zhYmAMIjmbq;k^ESTLzGNRlC;97TRxf^YV6Dy>D8x=%om~+&Xu2Q z2scqp1ehZ?4?*JQtqW!soICs@WY#Jyn(hWMjR8_eVibICe@>*1D9-#DnBQM}7}RS} za9*6;-90|aod+grT5}M^=ZHE>^)8W+r+z}yvQPLU2Zz0}V@>xvY;Tk&3ym42&7DNL zlhl914(n>9{HUW@We?hAJ7d1#*_qHZZzi6Yerg(=v|^+nd54;)0PbRRBOrPLJ>Eol z;I0MNATxAD>)iWn`kp`UYDk4ZD7_auhwO@CPp+a%C`)GG@~%cEcXuzQ8N>T5vT#7} z!@|P8^ME_?%wl*oj|Zmre06dpKUq!a(=5JCfHP)7=1bNkk;`w{CvP_#@;;vCqKGj1 zGn2}86R|vf+!IHv-PDQJ)EDmWXjREG$%!v|HzyxWB7>erm_rS*0x z?gFb~M1CFPNw7yej(9=ABq!*#4fUd(ogKSyC{cCoKdQ4<#Uuq&dMnkzHa}cM5ZYgf zx9zP>IS0_6{>UgzJ*gVTH29$CiJ0VygMx{elDjo-y@QL=%v9xf&$$9(Rf>?gzvMc; z;9xOv^NC&0coLv1nGW1j8CVu`1frJm6XYgwilFmp+}Drf3s|8m8*n5Jz*;g zsJsZ_eGXWANy?@VEfmLMgM62BN$opO7U(#0>xW}2?Sc?{;4%dTHH#O5*&%%v93i%m20S{DGFLqGB$fTJc0$hG~_uX?K!E68aQSe?noL=1h z&m*D9bw%b>9C8TOG}h6jCj7sBGwL4&Fa3eS8Stl%fkOq8hOaZ?|8cdbA*rfvpP(Pf zR}8zRQU5)4L4z1E@I_J3;q>p_i;Ja_VW|Y**75K`V8)_mx;Yv0re&6rN=^As|J(R~ zk%tC6$^Ui?FlZdCH;^`k2qS)uRR-VIsEtb5|LutrWZvC>ilmYd)i*_a5s|MW%!5<3 zlXY>E%H}T}#GYMfe6>l-by1rQF2{Sl_2A$t@XbA;8Wl9}oB8@;8wd%hY5l)pEWt)@ zx(tv~^@S45vOS1}2QNk>i$8`Gh&uJrSRLD+|M%oe1>gb=(#iiGg2V*UhUp~Qd4t}KZBDtzqaso1z>M6hxZ{~9ZOleYcV1=Y7w>u0 zrrsX!@4ko3|BdH#q14302*YTQ+W~ee|3^U{*QARkKjvSy7h*ouQRbDq$F^MAn1~$* zcP%gfP-rG3Zyq<*bYbP2fm0~ntQh~#UBE;dC5=?QeG!3pjYhO|9OgF-2->UFd!FU3 z{hDh??#NQv85lU8vot32+FJ0M7bTspoAHr(sF#2EpBWwXmSm2gYQ=GN#cSU8+3)j& zPEBER90e)}_hNCjc}1iE)PNaW0Mla!>k5PFFFsd>hul+R!<9Xe1io%z_92~W|DIcZ z^+j#Xc&+{bN`rVf2xGzJM(@VV5fLLbf$lK!KXV2L)T<_esQNCUCOw^@)>e69kl;NJ zc5hR!Gx$E00p1rQ3$;^~)f%{N3OyK9#60t;7@psV+RJKz;)WR8NZ)HwEc{=TX_!I0 zrcikkhDir=3d&0DcpigEI8dIwP#@dLecxtbjMEZuWRKg8Ia1A3_txww{)8(_L05Rg z_U0En25qr|r)|-&1qaa&RNl*Pes3N?$K)SW=)yZGVyv8#cFf|H7&xP|e%f&5!Vl$% zA&N+oHqiF2*}(F>p$(Z0g^?jF!X7#a-E1^lMntebIL)A^jTWH|7N5H4SvxB8#vQFw)M9NFmRr`~MU4HGSk%7cz= z1B&Vr8D0MC0;X}CQ1@Pk-y{=Vw3ADXp`;ak9I*i(p{uC5wU+<)6_&}ZJ7be}*6ZY$ z+_++!n9i%K7>7AuAD(EU%UW$U&oYCN0dst6HO_db7D=xprhe{Jc6;A0QvR1G%l z$P`a}5pgoJfjp$dyf%~+L?1RDx6=*Dv}2w69{b(^!U?2gR9=SEvH$*(!U)OQv6Pc6 zdRZyHo@uMkhKmwo8P_?MPo}QTI`nlM6C@ytLnQ2Gz$B-K9{%_Da+SAMw)NI<}?e)>e$LDCH=SAWxW|8YgclT&b_x+EX z!pSjU) zM}N7h0k=^Ixb^DAb9*jh3^$3!>oeX~9}jQn#-7A@B4lT0?;~x@x-3+i&R-p!ygZ-0 zczt39dAqcJi_2E93%WCWI1+#$ntel+g57CssR-xH4)w9v8tSmTYewWq7 z?b~0%Cxf&LRmOzTffp0}{NK&Gg#4H4Z5;{O%{sZbxID(#Ro- zgU`nppFV@1FAB($r&RQM32^vWM6MUD*Ihf$hDqo;SB!3fvxRba2b5r=GuiRS^EQMtDLy3wd0&s@{*}B>so!eU+7VL zQzFPuPycsH{N-*EwjlU^-(_3;`7B9P^~*}L^Gd?w?N(B1YD$VX3TM0R&x|T2Iew=_ ztebgb*FS^iadHTRN&04fwp{0JyZNM1GBs8w>s=4+ZEb$-9UTDzf`V;=0s;czMcyH6 zk>rPk&X)X3ZA0NP`XT!Qaql3qI{bR9uXztE)qH z?RoM>i!azeT5j?zgkLRVCRW+wb~LaMlAyeIS??4iSmrz~NfJ|%VwsM*G*r9|@Liwp zc{*~%=BQq^fbrpKIkH$q=-sg*Av#IF6;-`zA=@wvMEFI#oyBQAj84ooSfgF8#vZh<;!#`+6s+sSf$j_Q3a_E?7Xk`C(cloDQRyx>`KC(A5F22+%ew1b3 zN`I$MW+AdnMLJbxAQWvBLf$BsM03r|p*JQ-cZey3=?2T`zB%~el=m=(Z$h>u(wsqDvuj4Pv%u% z_9q{0u>CK4-l*NojEsNCa**xEyNjWRn-lRC$A#)fpN$}XWVG{pX6>?Z4iX3-hy>{) z?*Q8x3EEsM@Oe7;<>>|=)co=q?t1z^3Tl`T*sLOogEF5sYJ@%)&yE`aZbx1j4#2gk8<0GOl&-n-? z_&Xa5i&p8A<>OxW!?E!Dm@H`yu*h&ZIwP+c|GbsJOsQ6m8gYD(~U-2M9(ADH{) z`>&Cxe*I>p3wGL9)P@JX$Zr3&BT?PdK3K-5A&_4PApAx%AM}Ta;ab#DZa=m%n(Rp2zdJ^^A~ONT9<}3CN?%!Q3^&Z=>D>8&r7;klseanoftF0g+eiM`HQfR3JNd% z@`eJ$Z+QhM;l}h|5DBN#qn6;gVLmYQ5XfxqO4vpkjQ1tgYW*P^RN3_|J-eg0QS8xt{rvC<*X(rhI9=>wQKL z&Vb3C!1ng`9{7zf5+X`Z>TPUQvI^lhEE<+C$nZ-eDbdjGc*;%H&~|*B(}&PAGQLPc zm_?ue5`gyJa}ayr9@cZ; zboWn#+^{KOFBz5Kb4_V+9p4Wpk_+FNzufEpnXA-~n>}J!5TS_@ z$HlnOYr=^O ziyAEtl}FswtT1>ziss*P_ud?>WQIDS98f0C#>dCM#o8AR4-4BI4Mu0IJEV^#c1k0? zVqk{)%D9KYU(h|SH- ze`#rH+^$_$^NHBBauF-b%R;k6A9SQtRg*}Nn#iy|c!qNSK8R_3Xqdh<^}V9m0vp$&YRU zpb}7RP-o>QD;s$Y^!4mF&gWAISrS@taj^%0WpY>By$)nx+=44k|T*c2O_%}?nJs&5txfuzE4M^hG-k*vVD_A6!k@Jw)s*}uq0in<^6xkV5 z7G2jaQ;HDC=AK_nit87ehN2HwL7`9+6$NB8G&E8Ui(UuC2j!$ZJW&$(73-5ffBxJ( zUQ8x&5`dJNdt@M{u(`OXQs@ns_OP0@=(O##3{pLsgJA;+?xMpu>INkobaZr(+S=N; zOFF~i)%=v;NB6!T>F*{rR!U5fXp~k8k|YkOpx^(oKo?7{!I*fUVR2&(4Gnm!G7n(X z>!hR5ki);E_6%!mO0f15*p4bhkf^=YU!03F>GZ8ToGDed1$BdMZRv;ibpi0)i4-w{ zfaQ2FwiQOEKtV_t>jMEq855H{?ih@X*Ew62vvsmy4xWok9UUDtF)&E^Q|Sa>tnZ9Y zEQseAD&y`HSfCN`H(mcy>bFYDn_-9CDd2^UW|D zw3cG`cVGAekUzZPklgXMRP7$63-kfz)MGOch3*Hv_M|^zx0)xh-cwG+2JrzPfUMhY zBFFjVc0^of!oR@=i|=}u6qQ#S0)b2w?IL@tEhGY17-4JPf>5Lj!0&zAIhhpZ2FRS+ zxrNwPxt$(tr-qURLDz=I>s;AJI0zUy}Rb}V%yaN>{-7PI^Ge7)j2C+vMV8Zt)AZ8Q4$ zL&`0MX6ldc0=Vvh?;}WJzJ|26wq61_i=%;hozgqMd;bw za-#@9lOh)?qo45K&4Zusb}K;FU?nA`IUrtHB)*!5*MxEg$%m|jo)KSbE|Nv%eWDVi_M1FX*+jYD1BZMLBh5)j1(3*rtbU7e%>;xBLxDWy zmX;({x~#lBrBRpvmCHFzo3corR!x_cR8^=5$VUQF`t~N-$6wT$V)z4^VIfU&r6vmR zrGnIkv!a5+;=0ZV*u}-=!_D#9jexK)ic;;$UGh#ER|XpF{w+u9Q)qj=`q& z*k^J@AF4?BYMPEntRl<`T?t8T8XAnnEqB@Ie$vp>n}_Fp00Z+qeQP-39U^A{YPUQe zmh6$5#ar@~@eE(64Df_q-UPg!iHy!&$|C~>G!1jQ zEB>-Xikk!aq^bQDr$}$L-u5RN7HF?%U21(_QHD~{N)1ulx*mJ*lW`x(?`Vo?VD5bR z@M$aQ5Ih7VNiZA015*#B=$r*$d84?*{7+o0ubk@T6GTA_h7#eCW;eI{}PPxp0sKx$^vqhwx=LG9K&!g10CT~ znJTGf!K5tLjT7(rnukWEhW1#A-ZDV#ov=W}+xx186VgO}o`EE78V$phNS>aaapxBo zak-nfz-tQ$^xm6w-d?Y^dAe#Ez-igk$Kj@{rJPd8nMN3?A`~)|Xw3E~znHA_WMAy= zJO^fF`82z%u3iCyB?jC4{uhJ!sFYHSpo+*f%-xF@KBc!L_{0;7cK+HuRDOKGz6e#=FE;4RqUh= z&aI0%ee!c>6XqPjgQ~Qi=rpyt|u%H-0CqYUT2r{WExQJzPqLOSp<9FQOt)Xjupy3C%XP7{^az~_i_g`1VtiEDn0yIEfK!()0{nuE{bqUQadXi9>< z$|~B#5`(@6H60RTaUQ`W5_9U(X&Jp>HSX=OWMjAf+C0lw?3g(7U->FxT(DS(J;PBARJ>q8s7xM#rw?08o+|kdC|)6B5?K z3_TaGJ#KoKrK7{(94!(kMB9r2#iO(qvKms8Dvb=UxE+L+=?jCPBvRS`U`cy3T~7b> z0?>;pvw0$Cg<{eBKq#);XJl*Y;Bzag~q&P+@ z_Run^nO%i@XQ-m;2gp&A0f-1Io0&6C(Vr z`Dz`O43*0#k$%rQJ&8_IGH(z;vQqF8P{_D&YY%9Xnb_Fa?nrQRIymh4DJmkG1Q0g) z!USr$sxg$NSIRjE#BE}Mh#yx+Tl@<^d}T@T@Y2E?>+3%iFRoyH>bUysJNK_a(xRwr zP@?X#fuec0gdZh7?-&43LuzVj4aqhZfsROUeDqMa(;KIi=Bmm2s{`)ORBxJiiQVZV zNdjL!(jk9l!lBy45U8uH*AsrQ-yTi2CSWsZz5L1EqoUvNVr^~hrJTV!eRaG>x^;uH zzsQ|{gIugQl}IglAep8L8RLA;t=G^Fc+S-})>JJ*$Zhrn)KP`-=eUp|Pa;33H9GI1 z>O}cMJ7iw^^&VzaUTpLD1s8G=Uyj+7Lr>)qaV;Gf$UcRG;0V~yukARJ&}>w?-@uq{ z34%!M3N-{w%qm4ON*Oi$%Mp0GSW?YpDH|1Ut{=-O6yW6dTux4j11^1 z508$fa~VL8qY~>$rm?fyOS!i}^X!j4Pn*ca8KNLno96{@V@2sjN=*VLKsPoA*135g z`}uLj60ygQm<{$DZ5mo98Ifb*;?DN{{P`1qM-p%NhZRv2&J+7R`O3GiU*S+|4&wGV zH{S6+bL4GrkEQA=2n%-@#6D44+q@hJ=&26bgtg}-=RWL4AKfV-H7$GJDs_Uj2|z$L zaqz?g0pdg}jg)HDSxuCNw)OLvq^I=|tFvqAy16evszFN28#`1`ATxUxXry+$L~0E9 zMTI`ognnZvVi>tFgt*V|N}mOi?I{5G#=!<)r9*W^EyEhltG`AeD0TC(&!z=}zE}U| z>H+a>o`|Pr@L?KdFH}i6Gh01^HbE$T zG@>ZqjwWY%xW)KDpRRWSLV<4{bfvrWRWa4gOzrC;wYz#{3Ej`db~dSm2C<1G)qt%8 z{$flrfzRJ;UI_8<_EsDY7whJp9&Sd$R(u&7o=?(9z{OV$Q z7XNTHS%!fZ3*7ECjEVLXMdpy5BuO~~@GE_Em;s7|i|b>3@2e`Mdi<=f4LO_M4d0*~ zBJd}QIK|<4_Ei0@3gLx+C~4H#Dw(`T=52e}&?;hB06LWSZP9fh29JFDI~c)yR~M{W zl1cMM2Tcl`Ln^Z&&UBeXg&M+e6C+C#I{d}B@jd{;ey=ia*&!e#)Q8MHe>^F~SOO%f zAU!?300+qtxLEAQ!T_cC@=aVXK+A`r!jS^Z7_O-hqRjW2phrNf(Fd%Zu6ux`57C7Q z)6EPz!_}q9aD*-oxDzC9AGK_cjtfSYJtsoaBw=| zN3(Z#cl(b|Pfdd?+GJcdG5MjPp}Oj#S30^13NZxh(!ze{Hful%{m9|%wK&4e%sj{M zy8iw7xZ~uqPC`NOCva2-*aEk-^80t_(D3jM8ij~2WBgBEp6jqZg&(F|&)>(9uK5o3 zv}12vKS`;s#=qUNV;0AH7#{ukQv_e910SZ^@^N3w)Zf3ocuO5? z$L|lI|DXWW1&6lIt1o4uz2pZq$DyVp8$}f&GH=uR3qY$oBO@ZQYxa@Lhcpi5^5VJd z<0Es#Jlz06c^~iq)&&i|vOqy#JW%T!+OxN=SRiH|#a*py^*6{kj7Gwbr#l_YiO za{)zTQdWNva4&YbK3rONFw>1qN3tllE`@sLB8=7AO&pkg4@}7@D z5D)aRof3}$tK^TD=g<9Zs)+kRRh&wvfp04GNjdai;)l@?l>vK%msMmAfUrXcULP?F z_4&z|xNmjevSL)tNqZywWX0=qy?N^maM7jO)-NA` zEWx7+jcsI#Ax|%rfA7@~;M|=}&4Hmf!WyQfo?#?Z)Y(L0ZiIs$mWqj_+p5A+BYl;7 z3ceA;ZtivOt_rGRw^sZ|AalE2q#g2Ae1QOV_s8Z11RzrYXEp)Ks6kp=|MGG>TCn-s ztp+0BXy04|DqBDrMM*w=L0xHb)PEHKR}vEwTO>!<&Q4F6-ohAcpPZPBeB7H#0nC}#B zZ(v6gmUJ8WLg3AEu<$&o^hsID)6KJ~g&F!*q14;aFPlKvx8SyN_WeKFNKKbvp?cJlJ zs&Mlrt^SSJ4Snn;Oqyf=WHB+Z<>#jd$AZ9f#iLnu@v03Nz{Rvan+9dE`ZBi%_ir7y z)cN6YauB|PM+O5!!>V#3_VK*S1)_h`M3^4#j9;JiV10Mf`}+D0fThqweuBKq0{F?V zsX2i+_-&<_MRWooJm?2M;z zp$(DAiL(i9RCryfcWy>##~vR84>7y!VPa7AiLp?z2&d~l!ki(TeZsEErg~m|Klajv z9tHz%!i?jTz(+$P6?cu1WL8((bSku<3mLNnbEsVaD|`S^4%w z9oXW#W#FpVBA#<(u#p6D;Xp-BXveUdGy-bV@TMlNgsYy8PMQkRY=I9H^hplmmc7irGV)UB4*wG z3&{0G4v6oM{fB&!y7IGQ4GSJ9>XH*8Q6Vdmi#XiOFQ(_AG_?&$#!KaLE~s01Rd|P*8&Uf-k3N zEMdbkYxNsJb0~yYBIjkXG%d+=CPP;&#VTQn$t5!L_R{`P726;d_;^y-WHWLc94$o} zRis8K9815CtO*PLwm7r#SuTTU_ie1diD`o603@s`hU28#XPqRf#{pRD#ejIYi79ql zcs^Amy?n4xbHK_R|Bjy{d*|V#XJL4F*zC;;5vI3S)v$)!(kzo+eKYNpzip716GapL zJ3W;+AmD_9?msqbQM`Xt5uuj~d#d>8?Jal08D)momnD;QS_K$5C3vXcPv$2|%FZJ+X&FZgPYz3b3wofXQ>3BB7hic3bA>E*-J&005MySnW!8G>z(lLhr#570GgxqE?u{v z1949v-$eh)6V%oK0A$hdLy5Z-KxfiIlUc)M>C8P(mYMJbRze+W?KA>&D3934Rc!<6 z<*S}AFq6d2ZqI-jS{c&Klp53MnD*N}agfL?z&j>O3-$&Zb852aRt=saGp3!D=Gu>K zF`IOLe!j&F@VL0LaCOVozG%@Ow3a=~yJ9fRg{<`ezDG?E8yj2f_I_63-Ii&?eA8fZ zz5GP|*Ru40n`70Ez&npnt*4~7Sk_$0wUTTt_4O>}S2|8gY^(;gg?j+fGE%wKvVNS% z;@n#1o)k%a&5S~eN^Kc5J1>}!~9wnqMYoj6N)zk^EVKzj5ZG}nl-<@ z$V}Tm#+LrRhy|v9<@W8AT>a<)Y?ve+8r;3}T01YdsA$(7AU&jC@n&Uoz5u!*i-b;E zM_Zfajy;X)Ohw8#C3dG%O@s!lu#)R~^z_u}!x2hiY5Mn*=m8-y z8Sre$cSg+^Aw1U+Nr)}XD11TM(KZuB!XbEA8f+>?Ng?_XIxGt-D1oVrw1O^Br*l~Xy8uo!xy&JY^4 zE3~c=t;37VuOcQyGvHY0fkQA!PO@| zk#2-&*FwWS+@T|Kd}5+XZu3_IwJ7LdzUrM>r|%iLzWxH;`Cx={CrFfjBwDK`t(6g{ z0ZBBI-8?@9D4_|s*qIO_jcM-`C!#;9&@OL$Ra|BAzy58ciiZ-hMSEktl|AkdwI0jgxm`@s8bKr*79coe4(#bVa)`}ZQNzm@Xyhm6psbE3 z;w<(B?&_}>KFDK{GOOS(6$vV%T3(jy37(?0M=PD4D#&=TL=UkOD6wICLrhJaxw(MXQO#RFrW%kVirG!lfTh5rfxHv%)?)xMhk>3@+~#I$?d~wge`$UA1M& z69(E(sFUtlhm6aOA5s(I6BF6wiUWwOhvG#YJr40ED()Zq5Gf0zYrXc_G|P@_)4boNQKO9+ zyo5#9H5CDQ#EbJr=4f5Z(?FFh5>bs)WpkV9{4;gw+U;=5=lb3JPl%zJ4IfNUv0Fa1 zhQe9UAR{&MKLAZ<{Mp({vse{h0Q8C1{1>!PWd_62riwPJCu5f@l|^9epUJHJ<}0N%fj zJn}#RAf7D?!Un)rbOB<(!_)JuwTA@=KBVukXH({zHcS!oZ)DZ=fFTx7${~m_V)mOk zA8qiGqABCUnhyE?bolZUe~L zwwErU^h7+Hz{lKZ^-A&*Fd^zmvUWC{*I?Q z6U|AHVbEMrVN@q;o)aR0b+AtXD94N?xBEi^|2mW2>@&)1MOFLv@2p3Ff9u+}Ni3g> zbhh?@LrzCi4j%wzMf~o?t@qua4ng!mX@xRRJ4+%Yf)D$e@r&ALw@${zz4_Mf;=53jbn`R z=+s2|p*Q7V8!tyn;+L=Tgp~@7L!T{X1O6B_fxO7t0J$Iq@G;MUGBCw=3lGLznspfH zB0WDA7WnbeX84Tu${o=%^8yr;FcTt3hO-179v&vi0kt;=mr-r|&vf3UmKps|*x$df zkx?(<0i4BcLDf8UgIOZ`D=AU`G+-Gpyk7-q7a&eJYkl!5sG%}-=2$!iL`^8+qaOmx zXQuTVWgr;k&ks*e={O*dK?Gi}q9`w@1Y|9A=pbh>mjOpOV5f5 zg)`1X+Tu-IrjCb8tQp*@A_U3Rf$IICIh7i@!agog;ge2WFhIabm6j+Q_kF*9Qnbdv zz`$1^Iar`0l{Hz}lwcZ?Y#Ex!6TE;M?_;?I8}UXg11y-$*WW+?O)`{gZL{sEDC@zMv} z?79(5q_Ub!CB&UzO$UW+;Sm%2krM11*K#0Z$~nt}>um%21(Xq!bG7yK%5Epnm}WHK zwx05{_G8n?c2ANA_vW(l%~qz_1E4l%khL;26YSjPNIsY1{4X)tXHX)P9+FvEDeRHI za+U}bGOoo~e*FXqMMH|dOKz#HLjwcW>Lw;8#N(45yrLi?j1bZaASI`V1PQDI1TLsC zJ0Z;%$8^8zNQDl+R(=9a`0{Lo@+ACf3jt}%qx2AS`4Xs3v^M|8bZkt=`B!0+V_~)o z?!iY+K|x^?sAY8>c+nmziBn4gYCzU0cpRoE6gxeT61e$Q-^2G9LPV+LjwE;YGHgO) z3X+j{_n)GL70%+UmFh^zCBF!c54sPAJN#>v^0%SJcT80zLXnOP_wVZHcsK*RUHd!{ zzxvD9a)G(+g!=8L_p#Hx%5$0~q8H=bOwdGte#KTsj}w3_l*0jugUbq7gZFD4_9a^P z#)A+A@4;g!^gck=F`feQ(wA*roW8G0$h=PvCo3)N7l3s~_&ey`!SeIYGbI_J?_n$< zwKQcqwIT#rx^03fTZ#oL^onpqp9eR1b>ef#wrU892;@D3gq9+xVi!HWh4Lcv@hh5S z5>99Cl-J7AGI#Uj<;QJNHv^9K7+`Dg%^SYsWRv+8@s#dwsR4N{Mx_UIW5!d1PZ0-i zV<$dVR%)brKJv=GZYe68v{?lTm#DL9S}1fVAYxmTjv1CPKDB;YMH$kH++$F_1BYM5 zK0fyilVYSjKnr0&RSOO8{L7@+!*{%zG~hH32b4(CtRQYw)TXElh;En$Zs>ePV@E== zO8uLJB93p-!y0W}Y#(`x7h!I`PTtVmRMyCh#~PrMEpl{c>>e*SRbB%6wQf@^S(_-y zrcfC`;QwyX0P9FsLZTxLT^)~T0zwPz{?dt(@v3MZBb~ zl$2CtO-)U!SVi+W-W3W%St zz5{A$ODpQ0OBOIuYXG~F9rd0ye{h+O#6IZ5cdyUZ9Lh!3p?9 zl{-okqE}(IR252u(Gae>hTRO&>74=06sJd`I3TmUL+?zY6Q{t<`?&~kC8*c z(|{?I`dYAItS_yZ`7DnejzAVapAF=3tL#Md9C5QEx-i9q^efn;hNxx~=`c$OLp6SNmn4lia$r5Dns-=-XqX!!d zm~7%SQx?es>TQ2D`kj}ZC0SeLgoh7=0s8BT0{v0VST***JMW$tF8{1%5zIAE13iLA=HF-gLp!l z=CTJvaEdBf=D;_@zvShW0j!0OWCfSMNvZl}b8TyD3l;Y5+xm*_v{pyH&;62ljt$hT zQ0NszH}nrbJ#=0nA_wW8*JmQ61OJ&chlW%l4a8VNHIvdXvw~>=?j=zIYhpK0n7gjU zoh|VulH@StA3Zc5sQ6I0%^L*1uwV$8D}8Y%jI=re3=;usTibLRY*1=C7Klo&oEi+2 z8lseXUk>*w_teoLBKcQn2m>Gg1cYmpBqM1gUdw>NvZ>hn4dqQ!)bQqn_}ExxnQ!}= z;i^&I>z#h|9QGt#8je6Q4Io>BM}Q*k>nEUsu^yRs>xDLA87Kz*_?lYWk!!_zD{p1Q?=(1`DAkPJDitAq};Dxj@M+JTw4WX?+Rg-*Tqt3l1)e@5#jNwU*E$Mo-9V2=boTr;sOO$y1#GVrI>4IkyhAMuec>v_-fLCPrPC80lW$3T4Y4sj$>ZYR`@ziD+u{Dm+||`pJTST% zy)Kh6(X93{#rf$eg&nQbqrS@g{s&PBDSmwN1lJ2B3mr_|-7)Ly>+>g5luzX{G-@iev&L%)ZbzlUYRQ>AboNRsvq@|3@TQyd0Wj?Veev{*3THi&&O+bc zKN#jqa-W&gkdYY_2+o=CBJ^+RE#eYYXLnb^L~zsQzFYLo6yvx>GPelWvy%^OwcbCh zb4oGx-fiJdNE~G49F(!=%Mzu*1Q znVKH>CgvTeYi@6X5F)czS1;riL&+%(rhKD7fvDGhcl?Z;Ku;)0m^o}zc{e2l`8tZV zvf^NgH9eWKPi816{G-G!6z=SGL{3J2UO*V){^-e*#SNI#l~e^@iZ{=+XDC&Q3ky?F z;a?_Eato*9IB+|v;=d!}3uF~-jii^?+Ss4emDftzpNxsf?EPB%Si^2mja{hYgh0V= zmU&e(qO9y%Pcu)&A1oHDv|W*Zm&GfHX&a@=p6m!mk;?kCq+2DOPMSu}ob0cB06bra z1n1J^?jP!Bx{}^jeKPLEeF-#$A)s-;&HY@qqKxZnZ)G8yt0!Jqlkafsry@pxrC8uC zo4esMD=j<`>O{p*A=Lut$3u*unz0{0?k24*DXM+d<4VjQuA#sKFn65N#7?WTFJ)Qi zf)Y$QY_O@_Oi+x3LkRp;&C8cBmjM=X98NNsqY;b!B-F|LA1Q*BP=|?%jFFMi`{HDO z_VryW-Vfy|RT|e^92~Edl$9yFniQ(?nzRwT|kTo%U%*1BK-J|`jpa`A#Zo|UDLWr##?VBWL6W5AW!vuMd z>C~aUiut~P`T$pZkWba>yQ8jTw;st6;@E#)*wkyG3p-1tkFW1ttdB{Ols;#f?D%0t#mM+9DPc3ve4R@z z;k_rr?OL?fEkY=f3=EW%A~b%ksvLd?=o=dwEg<t&9(q8_5qStV7HKomxYmxEf+bDarY^e7HYoIw!~FBEmVP*Gl$JvMhq zNG3=<^^IaPb%=aC5Cg;qD~e8I>JC~%ue>jodTl^O(b$*FlpNA%>4jvJ_Z`);Z~7P& zY3^pC%9HX4Ngk2OuB^`MWoi4-tR?Vplg&DKUnv`d-{tIh=Gikqs#bw3z1vZthN$AJ zv&dAIAa*98HIUb$Uuls%TQ6k8z`%H2W!?Ua>5qHSq#e)nwUg!k-whBB;{o@wfYJL1 zk0!P37sbICb>-$q)l!gRO8m$acDbQ>@vdo)dDS)4K0MMb%5D#c3vptBSD@1DbRD`~ zS4iP0t3bQpl zBf3;oTV2@y^Ug?chL)|7oM~7Q8)>ksHU$*07<--GtGRze$o9h0FBT>?^uL+=?U^+4 zambwVsUAJrUI({(BHGG##wRTP8}D9o=DEr8o?db3c+z zOIU6H2hi%-CBQZ}qd*goUjrM1`rs9u!VuJSFa*=)qX*5O91VzlUiNz2k5GCfZu4;+ z2Yn4KV^z1{GBGmKlw11bXE?9ZcYWe)CHB$k(9{?bc8BgHWBQU%SnB?y zVnk*=dv~V~I1*{Tf+L|>|Iex8=6&7L`6kc8_aH22Rvm0Bm>G4Il!^09y?f;I?JQm4 z%>$AATXZ7P&@SA1(ZiaKbYf@xI&U$EddNO_^o`uj#)g@bgk1k?5009vs}PPW$@Lb` zG&^oSXR6sSXTX=aBdq;KBIio357N6ap1!7CL=&SD%%o}iw8CnOv02aGvU{cwndsjL zLT-|m7xU{+w})PPGQTmVY5Z@Lk$@OMfl>=&z>bfn#Txq-n^X)=@QA5_{D(;PyEj6A zNQjF1lW3m20FM%@@HvyO?z?KTJ7Eoya+jYC|MiKBH&AwS)~(OC`YqNv4pBaKbX+vH zS4qaZ?g+>7G>KQc>BpUdmc~BH5pVg1{pv@S;=u0sJcL{@3X?o$F%$I^^s%gS8e#ke zN`J%rIu$Ln2ptO7oWCg!)}$NjuJ;a~Soe>#@dY9OEOK{ddZ49Yrw50UBdI4ysHjSS zy$j+P=p!X{ULRak+5gUJ?-$Xld-m1HE~p}5neEU8-SuZ%a7f5&=JARZEZaL_MAJ(7EY3drR$lQ@1?CZl4>Rn$2vxm-yt@FuuIfS^v#4-B$ z4fTxj1fjiK(1>&5X)0i*bMj0v%<);5BW$E*WbpXlfs# zxWQTQ5J#o%Cab*nZ$3pSJ$#-%t#AU-vS3f|8Uo)hD2~>@#<;Cp(b<7viQ7CGoD;gk1HWn5Z zrB&+sII4&K`M^IBM4Iz5eE3ixT1woXSGhB#%dVVPKKlVE{FR3RtFPR4KmAKuzT~IS zWG=1?I@&IKD;*Memtm=rYd5X#XTGU`%LmO6a7}9KNS_uh);W)L3OvSlc4u}C@jVcT z@+nkK9POm%WlPD>LH`-nt2<)@qE5CS*>E?g4^G0PWb@i2y)^gIgCWp6C#%4bXw>9O zo&ghUiI1me^YfykX=OS&PpU%x>DSuZdgmd9P8hz(0N4ds1b#e?jEofLSY*;jh0oL? z&dQ4ZAco`V_J3Mg-R!YVIojU(h8QQ8XZ#;C)Fqqkv^)y5R%NGRGFW;JP1KWQsPE5! z3nCp96m!M(Hf`)>vX*}1YP=JSacsI8HPWlxqm$kC$UkHV0iNiNf>W$~PLm5cEtx{# zI!DBAQC=0nN-9y$DaU4#qHLdkt99rb?7!IglvYdtQ!Ek5yYa>eZS-_z;0k*$<|1xv z%KAHj2yd7hl=~|>?)g!ev4j0*>Gml6?$(MJqjM$70J?P>S*^ntt5?2t;i4(O^W`~4 zg8;wf7m=+<+FnbZ)2qoU>%Lj+y#)vFh|Ex{NOLeO@Q`ku0LTdZXCVp+_zokHVV15p z;7W}3Sm;%^Iz%w?f62h$5(}nm3A*PhZUouClk<9NjPp7Fdvm7A1iIZt6~+LEQ4RjK zIzoQRs)`E3ch@66M0mP~y@_;BK)<=;0@g{=HkF$kN;jSGABiJAXkNlv6aW^&EfgHjd>xUt!|6C-bv)?%#iCQ_1pst`h5(q*{!Y?2(sWBVwXcL@c zY$g3U{hd2`1lvctqPg^kd@LsdD-BzAd#CVq=k3IMAH}Lk^QBmo{Q)fD`MSJp*JID# zvvv%03FnJ{HIBV@K+Z74Wk&H0oH+WXVzkW@9^k{wjJC z6BF{B+s;c5DmC41zu47cpg280#|k-`cI-=r6R4cHOBinkld|t2P(S-E40nWL?N9JI zNb>wirgLoTR8LeXapR$MN}ko-OjIfz$JEj+-AHqKJn)7jT1fiL(BIXi<1+>8k_0=M zV6{d15XtyX4~1eq*nCGIQz$+xENls2A2C1hu}g9+Qq@YO*!aDlf$7BE-rhSp6??Dk zc=#3x88+o`aJu`P3qkj5a$l-R{1(|kje^yyyCa`HyWAGD+nLsEnTY=T^fUxp1{vUKIxdR3TCB39yS9jl$@N z_vg3B6*`<}VXyVbr>$!#6s9Vzm|Aw; zjT2dB9hr2>chc+I?-N7}N-AsdB5i_uSjH3dg2Ht10$%C{VcR3kzkI?M1F7UgtF=Oo zU4BIz4>hgR+28lg>1KeS5p; z7j-D`NizZe)+ktE>t@TZQ`;Mz3!U8Vx{ciQLh8OIbncWNe6NzsxKM2q+1=ag{$V$w zj-Af}e)x1l8;webVYNT`0)*P1ju(RdLmaffIf2cAqB^LB0FRsTcdrw^A70WJ^|z<0 zRm7(u`0N}U(>BZ~B7w`QEwCQH%2luC5;HX|Z6EgzUxJln;1{+6N!qUS^>d9Xx&0zR z%Uu`UO}Kl<|GL+BZvqCE?ug81e}Lunm(WlD{W9q4zq24s&aj?lVzu~o?n7o*O*DqY z*n9UU*8Gvb-i9Xj6lT19`4f%KlAoBu+m_x06?pMV{fEq_STt9iLm&n$v~w=#$lX2P zl45&=Q){mTFps3TeiEZoWSKbd;|nLom+idoZG#bd4hw9he&B`e21&Jxk!oVT6TiEE z&meUuchg)&vaFc$2W__rdF*;2(G<>iR3R!s_VyT2qZ8B9(@S7p$v#?1mf7UzF=kW#6;32Q46udv z*3M2MjY%W7G~cV<>ZS;(^r^J7?uQ0_t~yD2LF;zN18+Z|_#{%m6u*k-Db3dn7c+`U z$F47J!L47TFKqDst?f7<676GKM*ELwVqg=56`L#yQRyjM*SG}qZt(vtdtS>_1q;R5~pJqpQ@4;6}XT^@9-18 z(;|;tv}8ce&wd#<1vPJx5z)&6{-S7ZB$53IdL zKu=YAcz6)^{-sZ}HV|Z8UY83Ew#XH9RWB_QyGkcTWHf|Y^;#Q{BSuC>R9>!s&_i6& zTua~ztElv$v-e2tl*Q{BcwNO>)0vklDJtSeYH?CZ>loW#eXwjM-97AfAks@^dp+@E zzRtI@=;n)*)G_QJJgL)muG*L(lnt!`oY)Lafrp5Xqbo$oeuDV~I3h>bqu;(Ep2s?5 zEG92(6vabEnqI4QX@~pR*zBx?6U?nRTH}SB{a4*M8xD?+zjrn^#2C6~^eYrNKPVL! zsSqTz{DJ(9QF#l>)M2$EHL-BccT9Y^guU`RAO=0|v>yVcP0#z0D|COd8+9hl;2w4p zKN^52ZcFBFj%`-&aWng`M|Bw%DDC5EAaZ$SWh~_pfbobxq|KOdSVgjK5Va%%_^)l% z_1x^Tvab%NrW!hgU)ps`|cQ$ zE!eYjUuI=llpdUT0Nn@)h@-P^^|s)aIb7q}JpZ4H&*p-n-g)dXljb*v#HGpyEWVaPoaz>`K{xM$ zgl-4U$#Ln(6-c#5`0)*jHOtUajAv>dY0$S)Tf9~d`53AAT`2bJQz4FfbiDuNN$)P| z_TTi%jplL!$hk#_7p${eVdIPkXH84kS# zss?Zt)nP9j;!e3_?`vUtCCaP}jKcqb?f_6qkJa!BciETd3DIo>I^wDcqU`@U)lnlm z7dIzoaaj{4@6n}gNX=6#n zhO)ojeioVBw0;f-EOQHsSqoJ_I$M0r0pav__VnD}Eg@1LY`<+%xxPygRZn_fN`og> zw&KPtPtxrdz5BXxt1VTx-bGe%uT@^%xA`3X6r3hP71Ptq-Q^G=VUj(!GL)_hAdwB` zXn(&m-k?E3JDpa@X1<^$@-8oWoquWMO!zGWD^3gVy;7X~;tVSNH3#&iHA+u})7MWn zeIoc6Dk#GX@zq+pN8b?b8j=J$UWUGhkC7&pX5 zPjH#Ue&;^zitx+WMbV{C*l#2EV!BbKk%Wx9HMIluh);XZ}M6zs7K>%9_e5+Blo zE&w0?{Uf_v9whhPSi#Z${>Lj~kE@xDZed|9qLVz8GdO?usetjN5eUUdE6#7WnrGAQ5M5RU zEk#mQ|1kW>u6(mr%1>V=W;uavlZ34kk|9es>?0J9=;FBF{bq8{sOIv0H-CA~F4Sa+cb3-6^zP zS~rinTd=A3L%SV+{tnWAx6!mUu`n>F)t={XC(Wg~iIgqo~^g z1brTdiYykrXu+<}yyszBYIb!+gwUvupe1l;LC(QswkzSiHo&0jeSa5vJtzShyq3FT zBSFk;`o$AN4{5Na(JLS7Mlhk>)nf~$l-v9FtW9m6Li&JHIEeRMas$@><8^d&)ca+; zjc826@*tr}u^GH<267Ud2GW)PjYSYG?&K66?fJC%ZnwXR{ z6?4liGdp{pSA3b^-HMftb9)U6OszdErLkgIj!*-P>9n>7IuupwmakgFz1rKwdb!ebvLc-0+a z!(^IP5`k(*gj|LNcN(#73OzqObcyUfG48%Szg-N0NODV)l-){P4e%>0+11x zDC%}HN8D$#7xq(=f+7{I^fGF&5m3a`sk^vrp~uB*6&`xUJ6sWP&9Z&F@n2AZJ{{^{ zG@qWFD27d2HG)%!{-a2#ZRi4|o=N=sJ;eXi$485KcgcY^9>e(qS?psGqDbBHvZU|f zpLmJP80ya>%&iu$<=mX(5IDlHAbYU?sYy{kXy0E9e?AGLjnS8+4>8MkNu6(GtH*~E z;e-(aAAn)|S{VJAv|8d(p97ZAn>Rj~tncV&unE>M@4b=exB0=*_^Hh1VNV3)i0G76 z=y>N^LlZEJpbb_zN97uc>#Lo+TpUxoME2jqB4Vs~txIN>tDYdUk_#~;lg(uuNKkQC zwc9zv;4uR1J71Wfk@@U##qN20p^RzaZ3YGgJ`zNwa!zoFo@t%jnl9-0bMvVXsE!A^ ziY89@+;n6*-JB1d&ueYFq6(lLE<8IJQ6by-#hUP`wV*ijp8^LI?IYdZQnfT(@!|af zX?sB#nRZg=w@DwmlNd2s9_v1GbTk3^&W>E_iQKIe>@R-aw2)Mg0AGxsOJLxcnD7i! zPIyk#CYUkoN##;{btT-GVqXc@eWz1!m}aPx`y!Wc+U30IbtAxlSIBZDraDxE3h3C1u_0(-dpF8%XRk)h(roCUf-3U6`-Bjy@NWrdOo`H&@x=Hy#l*y1+`zc?_67^Biilr;>l_LmP6v<3-%D(_=OcdAB;l(hfiMuRnf#{s%h2^zxA_ly?@U z$iZ}74TPpV#1&{w7DmYvxGu@QS!7dcD9L0LJ}@14D^|M+Aqp8rrNUgvGr;*4l9H18 z%34^ZwO(r7=piP3@}$GBCn7m64tJc#ZngXPi>4;Z!|j>?TzACGE4lDn*%onc)`9fbfDUB(# z3jYyG8q-gL+HTl=z9sTYTjakkWh47W1@|DV00(gYNy3kE;UjwaiMJAu>dW1NqC; z65$g1Dk77BukJ^ICe|IV9tS z?CjK?fL!RhM=WWz{>A3zJ z1FQ0_sONG9?a|~(E!ac~>7XTJP6;P;Nve=fPcGG&(W=W2Hgke)v{iZwp47L(TRV0{ zB8>`OUVl@7(a7lXu7_2tOvb`IM``19ad=1u$h}o47y5+0duW5R(@$q!H1vf}w4EW7 zU1;SKzk8eBMQ)yfFtBxWRV=|qDM~Zc2m4kjYx~(|xdqC+*)4&!x_O-$Q+Wi=M-x^K zi9mdZoHfWhAg4U5hpIb3B1r00-_Enl8rE&|kd7{k)D-{k*Rks4&y%#~-5x1Ss_~{F zmF^!+|F_@$EtMKr6@#+8dk8DN%9pvMQ}~|0k608&T~d&nG=-e5zBaFRd1q)S98ax+ ze3Gtu;6I|=T`7{BNxCw0;yBJbBnT77l0Q4DGHpW|C12fZ?aHI7p>rA6p>|j<3Afxn zLoS^eP`2~#AnA+hT~p0$eVBQ^YutdykWaKnZvNJDSr$aFY7fTU3gV)@kY-M(1n~Ux zGQ@w3@KCU^5#+Ubt;P}Z#HMr%3~f{syN0P{zRAeQ;6MJSwexVF)q(xM4@a{7&AlZL zLD|Vx?$4%a`7c9MP=F7=vKyEdZ~QNxZE6t%F=KsWZaZ)C1o*NU5O-yOfylmNL$hGd zNhiZqS#ppBG03CrwlVrH?P1VPJxxGBz^VGIO#OK~kF*S=i(>U6)PZIbL00SiInc!u z78}aS`UiOrlli9T6cL%kF3U*|E_;??Q_`IM91_Gx{nM-Yti2&}1tcz`WSq;=?6J>P zsrmc;BXMGSXT4+=p?s*S4zs|XGW^{mD=B_~Zmuw?`lw@I3iKa2WFCl-q@f=&JK zUWA9&0>j=gHuJ!Ulx@4hFaZG7sltgVkhpzCHp*<<7y_ce6vJr+PMQYGOY7NOrHqu)k_l+a1l$ug zj#(iPDamSz&HF0&vkl5#jmmr>cYdQ9X3|Js(3Jh^moE{^^JQgpB|!`pm3wj_y}3|D zdCtd%<@xyd1_@m&E+LvWDv@5y>o0gO8j2wJacyMgJX$JRUW z9LdgC8+%uw{ZS=#VJxC4Qp0?EXlCLuQF7{5t~^mf(#=}#$F}n5lWFk^xVR>NZH9U; zgick177y0CSm1;{3Mk`!5!l*$D9@40KLy;pIx1P9%K6W$)h|6E4-yUZ7%b|=rQaS- zKaHFL{Lz!K`GZxf$Pxf6Lm1BuU7gRrJBM<_KRn#3YOf3e;qh(NqE&-M+$TNyiTTQT zTmiQuetty!Fv|tt_E5QViibQ_12H2bdVhk8ox!uHhqt7CA)=W=33Qj<>y(p znXU<$Vt)qKOjQO`sf9I?E@=BlN|Z@s%Gyoo6ZX2Hcj!nwyEuugskxWkB>qnN^2v{L z@XjvEb=Df>4-#!d9N0S8ZjrGy^y-pKQUNEYUs3Nqn=Q%L5=C^&Prjpf-9;XC#-?Iw zt3L{w;DY!O=zTt@7Y-K}S3pgPA1QPk*UZa_9M*Pt{8*e75?(j3>^eh1ik}8?Vicbt z9uW&2v7cMa}@tE}ya|?FUjcD4o0KG8IF8%dMuwfnc-_KyeQX;Cdnuisl-oHx@e2u}1}- ze8d+1h&SRG+JG4UMzwwnDFTlyn>=n*QH;DNDMahk90HEq1XA+8kMFqxks%;LNJYo~ zD3H~gJH>{pr$i~mmRc@8jH~`tf#0BKg8BRoQHJN&p`ofVFc zW83+5sA%a+%dGRiOnH&N>e{&_Ju1TICtrM(AM(VBt)r*?E#Z9so`pT+#3tTM&UU!r z+8~HPRYBQKYQ7cfkG;pi)UAHto2`K)sh+lB>Nrwa`*AbUI6DHFg@S49daUvLg5!G` z6M7`=(iJ|)fy?^mn;^dwzI^e>&265l2Xg1;V!;dWdO76D;SL3i|>#dK3eLsmsrc{Wyg`}CJFMuv7Y8Pr}{vG``J88Kg#><0y( zKHFrap)n{{RIU_%T%mC%<-ca-%8!VzeJPg}g z2no##a+D5d{uH!(nzJjG3%`4R1`tzXd1>h*(dP^dT57*lOmzwbO^DrquXnjGbtqk~@(6Ms#p^NbrU1j?G=?Ne`3c(hZV&7E8;Hm|QCK zBuA`I|MpRej5s!9Lw{<)Whg`b-H9{mUlwkQze(Wq*O5FXK5wcgOG-&HewnnUFWpVg z>MTe8Xs@p1YJ)_0Ylo;I=*El4qIPe9R!j;C3oUF2AG}DB!S^Ok6Xtjuv&%X) zC+A_r*KgmxsYsswO8f5Z<&^=Ef@4aMmr;&#QC)$)nj0daLoIOr<==XHO0q3{0%py+ z6+3jYgVPg1lQ<{UXsm9%Yy$zpd2->k`>uMEw{iZ9a58^+LN>imoa7{=j6@!H9?RPN z(z|`_$9C`jC>!DdtEu?{6XnCU`uh4PkPSYLOQ1@#(&lKTfB$BdquEU3E%YfW$P|8r zqCX`7&oZ#EwqBeX9`-nb0L?*I6x{>ijiX!eUrB6(L1QKp*92gpn!R>+FTIuF_d0y3 zn4x!iZgzd}^7I}5ZKFs)Z?p=Md)ln5EPj_)CihM6$&d**X~s6y=I;AX&+l<*w}R>B+{Z@eKwtIGT)%>=@!n zc7tKH{ysW7Dgx=Tm2$t<)+kwH@+hQ^=e%9edSpk{e|mp=7`X9{S76vjloCcyR}_mp zD~O0z!pl(9!aJS)dzF#&4;q1pu!u;e@dxu&;)2J?$Bk@y20nhw1&_-wKSm8t-8Qq< zxTM3qG5+_entAg>dHMY?!Ut(q6A`5yDy<}vB*1-qHT@hM@K8yq=LgUg6HKBWKTkkY zH-WCWbBDNoY!AN=P!X-|eY0kV{?s0u70S2+9KZjD#9V8NO+q$}a=S{e6fqcxX=wh- zdik~RAr!UX z!}S~GRu}|V*9lW=8HN5dO^`JX+V>Y?b^iU8kV$voa%_sw`@SR7(m|z4uCl>6m%I1s z=qmTUm4yAq5H5A6t2XKhxXmH}-LO0hS52a5W9_-Y?p@XMN~ccfBy|o?Jjo2tRAFPA+5U?`%Ki zWG0y$DzTin=K8_J{=(0PWlut8-3GQMrY&r5hHx4;Ic=*aLeLKmX3n zuO#{{de1~>t$|994*p6N323Gl$y=klHc03xNC?muOiWGr zlh{%fh>*(>Wps zY@c5Fa&uB+oH3YMTZhuy5}jJ0pFabp589`xvLQI zvuk*<58$nM;quh>^F2ElpgH{?i~%B|1&%ynAn3Eg$RDc6%7*a>2_c0bZUm_`2F}{V z@%gFVZzsv27lkXkq1QnwP`noy{N_?bZ98vD-+p}=^*hg^Fi=q|xs>2_sqGfJ>)1Z5 zWRuI3ShVUvpRYX{uw2{P;f{^^hK5X3YC@sYch9)$_!%$q)O{WH75+keZ5BkJqML(q z-CLi~1X)`l=G(XN*m|k8S{2>IJT#BT3LH3Cl8A*Y3q<7E5cck^I}Oa5q$;9z7kUo~ z2ndQnQSb(KaKo^!2dp4^C_!Hfsk)8MRs;K;UOOi{`vK6A2L;w&{Je^;Am;ZQTEhk_ zv2xw`R9*nZ^t$2_5f%qRwivzI<|isDvM2YuBFe=gJL`zf@F0;%W&sLF6vK_muKD@- zn<69RjEwHQw{Lf?OjXLzQ3@_OqTS%fCnVI~|N9r0s#RcuOxS`DW5OG$Y?VZ7{ZQ!E z$@EflbF)P#M8M}pzdAJv`rOVpi_z64Q#k;-6^95TBYv?ejkr?}gycO09?eycqG~6( z_dgvSmGwoZ?|O6y!RKtfp+fWR?J+l%89nf*Tw_R#kK5oo+bNp3|3PsyN%uyF@r-!t ztjd+$xzm70Os3SLwMM}$@BV)fI!6@lr;1i%D+^BxBE{{=25{Jj8QDbl8M`BJ$XbZW zT*`hR(a?An)fD9nxwN;LEmA6d(N1S@h$yYxLwO&fQv#v(Cf7WcApezG|`@lAEIjc9+27f`43sbn#%kE zv@xu(ZeKWnRfxM21ldeO;JnXSJpbn=A^o&6J{i1O z-k&$N!R8@VFq1}9a+hmN1myLZK8!c=!TJ$x=}{q8-WQnV5FmX(#|Cq`#S zH0=a14wqi+x(>P3fg65+{Y3BAR9kLLcPTd~;3?gTZ>;qIKvR39vJgFW%G>XL4g=EbfZtk3@zId84vUakd?|z>Pd(8fEStK=^bi zL|~--GOPvxiCa0|xgv-@+Euzxj{RaR<)C#yu}vj1Sq`r4es=L?ac4IcePJ9x_6-_YF} zuI2Ur+=)ZRd8f!p_(|khETzsGpRSaO){q|ErjlZJb~=v$gne!(5J?&S1?M9J4WF1i z5Qq1*v&HKrn3?l541*?!<9OtRU!|6JIWM0><1iS~8}mlGw#kSmcXum}fj%Ff{DU(W zgN$~AAJytHdF(}wGi!UFci2UaUP>V)EiG+SpArNb?|65g*w?di&cA;3>af$BO>L{- z)6{Ld{I6m)Z=Q|Fid++uxR{_zkJ&O`6I@+g4KjbHWG}#0*$mZyzz~Rs2kS}<%Btp6Pz4(4ABsRlO4#zPQY*TNYx zmEd}cc7gCg!j>JulmD2jmzf3k+rjLy(jnlOk)Cd7U;oR)IM-ZA5-ERu()qjHYZW)D zF348j*G@a;Z+k6s7SG4bwDN!b0X;9Zz1*K?6&~ove;67W-ZTEb@7HkU>?3+4+x!;p z0x`+P;tavU!a7c1lDVQydHzokeYpQm(Oexm?RT-8()=T}a30IyJbrGcR7_mxC!qHg z61j_667-E^g{TM}CyWI9DG|wU!kGzIikmxVwoFDVe~SEOmtIWa*jJ;(1b-T4-6i&* z1{cBjLBrwtF1XBd&P4B>Y^mGk9IvGPwS=> z9v3MKRohD)bq0XPA_&f6yqj1hwnD!$cW#1oRoDwB`GBJ|%MU8ZgP2T(T^ZA%_YiK| zS?94NZ#i#|?3E30R>jsfNWaC_Tb07lK|k+{=Rt^|7ChcV^~3gL9czBVK` zC{=84W2z)vgP1J=5b|D7{kXF_&vCBanLg@L9Prgo+NPYzTu_zv%Eq&mlA@AcbOEHN z^rqPns{sF~-oIT?{oV&PT5EYh!7{HDo*jv2EVf*fXw?+KnsP5-S~E6tRW?;Z$dc-6 z0hH|9;CXc={k#Z*Jo53O z9=r=px|jdfz(#2F^4Z;jo@>pTtgDdVM!pba-MW+zEzzUKqw0CO{x_j(=!%2!2@FYU z?A$h}D@H0qpomx{`G@iyCO>Z*+uYjnG(uuTUE%&r{n|FA{I&SkrfKHhbM?i-r{qvz zRW@^L2_H57K9R7vuKOgTd~#Gi(FOx9eqBUNDv4zDJL4%~w{rmR|+uGW;03o;qj~Jemi>8jwZemi> zAJSjVv44j}SMBg@2?lHB;{umE1&1PE>dYpb1owcAkV9Bl*o1gWV?gh0aeO?I3;ih* zn2Y|w8Ci(Sd+|c(8&+2Sn`AkILKGD~4HzG2p+96#QIWG{zs>4ZB*4Sgek+=wc1(0X?X+osNVrZv|FURN;}AP@hLcO;Kd}l)P}crGpJVx^ z4BY$b+Z)5DJG7O3^WeQs+vZr@_wcW@yauNF`lFkE&@Q(DRi0UkIc`*8<*j@%QEBB8 z@k-~x>P}r8VrVepDXxLHS$*zTAKkZVNcytS!0{57MW$HVRH1=akXo+01h{|h$kW%f=Rfdg^%>(gH;yPwZ19ih;j zlQ?AgKVXkL$h7eB51SwAb5pal**k3nQIcMVp5Va!t{9pe^)O5x z=}0e&Lgd<#GD9V);6k(OpCeAu#CcShMT_+HxRfnFafVF8#N+YsCF@uCLr=dG#n7$= zuZ2@}W_x z8FLPHBsN=7ZOADs{#|NjW%yHDk%QPcg2(g{=XOkHN(NguS!+wn40c3$G;LT8SGbvm zsgf$|bX43Xyv+fm~_jtx!^t@JQ26;inSe zU)}os{eR=0aa-gRva0Mj%AA|=Qj!RXh~)S{o7TP~4vCq;;oQtl#UGsRQvXb2NyScK z?c+5yTxlVFv2iMZpujk)!@gnf;PEu6xWncPE5QJ!3&v43gVB87L)5aXS=06c?|F^C-wi8RYom&&B#Y&*KtmQ^Z-MS$n- z-AaD|RmOuo>QM{CJ0B+Ha#2 zyNgVE3YR@`j}i)4%sTk8q=Js9FE1~HN;Ll))!1Pqe$Acfo7o`gQkMFk$v-g%i8Vad zJTbl6=nFS?1ft#%f!HhntLe!vDEB`k>@H?!XU8Q=OL79g$?k|YRj*JdekU9;e=$^^ zkmrOqd&Eq2Gfh}CE*ZVRAnXJnyH)|6#E@{QwcOQg(du?G<8(gL>Mz)5@xKx*r1p#*TkYI4%r#Q#_zcTlm%O199V$`{&$W~0-Bbp! zL%iUY$lPz(KJcQ%KHXZ4C7dK#NhWZwt*uS;*ma3(F5_D(9*_N|9dJRy_2oN4$>e74 z!xf!>Osk9`U?kCyeIe0)eHBz#xW<$!KOmu?nJHA3`aaX!DPGX{PnVo$I>#c zCGAM=dGGV`@N9s0a1)!5(7ix#QBJWYOtpD{WIQ%g8~L;VcChTy4Mi<3gt*>V>`V+R z;sI|m!{)Jq+RcixvW@=1LE!-)5V}d~0fLYs=QOtjO7mU)t_;1|-ysp#so^&N{pCu? zH5xNB%FP(h6=ms?jjs84R$EzFIi9my_3ny6D$r0$2*oc54<8L<>iHNcn<~cST`-7b zxc?ZI;Y8T1o1iZDGc_<6Q&QPngy_+pXkdwyDCeM}mna z>>Ks%reXesDS1|02*{K=c;2%z08`RCRphXoxcpTEIT<^ML0xC+HiSJstI&&Gp)!z( zM7_I!Svh%Yw2-u*WHAycB(g8`y2&cMwf1t6cve)<)*!?Ws;}_NS7-mzL*8Fwy2F?m z_B8w8?_YDB*?t~FwCC|@x=oZXq(TTN0U;C2DTy#U!sbLpQqTKW9Bu4ODu+f(SYj)=9uxpyW4L_9%yh|`numm-BB?8BB%YFw zHbv15%!|Bw-bGg|dRb!L#g7#UbRuNCN>z|&?Qcpi;r%pgrE&UHtaT37i(Kz2UO%4 z3WJ^2}&UHaVM;)Y&r$%S6=R4Xcukyj7nx0qpp z$YW5tZCxETw)^~)n@$>*nG^5geg?oF)jM;`JhzRmgHNkuCSTRhULNi;<$tLaOE?m;y*8%HctdZ+ z#&E<(BJt1^wY8IQHTh+kdAc|or|_v7g+J2VO%;?3W^gU!;pZnR56~KtI({4IB!O<= zI!yR8Ww%ANMrxmj^y=unCdrWx=ya7Afd$Q5esagJ?1wICe82Jv5UJi3skahwqzt68s7>R13v`39rxw=vp+qfR`2*DDyx6J&D|;Snc% zwK*r>^P|6%weegW;$0KV-uTymUbQvjg|(P7;vb%_OGo>9wD#9`AjDiHDqEIBB6a z>MATM*3L$tw??(z=QZn^6W(r%`srP*p{|}_Ba`^EwQ6{I^H+6DW>wXbbpVK|ZWIon z1j#w5J0KQO^I2C6tc=U_%b>YwM{ml^$yvZ$zx8<9p6{o33+Sq6q(D$iXj8o7as4>( zCxRqLhvg=jWn8J&8-b3=+1a2OozSx}p_kJlHX}wh4lXyQepTUk&@R+6-1xhx*xb=- z!S^7Y%J)YuBuYMFj&-y7KUH04P*YvIrFS9$Ly?FOnpEjU5C{Z96GfEX3DS|`OD{?2 zMWh=L=@^P49g*IH^ngec0U=Ro5l}kbyg$DC-FxQwd1m&^nSJ)`XRY_8S3Iyv zHLZ;vZQZ#$yX;|mURUWzW~r;lq|QAhTrpiD%;)iAf*6f@IPXXawJYVGOPwxf3soWX z6V#ISpENQiz@#27@z(n}zPzeDzyrpD61-wE4st4`qY2a)`wW)h=XL7Z9VeTUqD~em z_I+tWi|FmzDE8)2Q`#$}JUSZgJSD%6L?_TkH<+(5B>nCO(U%nfy;+eJa=a+>F9tcw zdh_$27Tt?i9JXGNqYu~#Y1Q%!`D(8w@Hr*?Qc^p&9f6;yC452u0{F`7WE&oHN}<hEc zQc5yyCg{%6WM}e;t^TfOH^!R756W`u_W69RxQSQ-%5e@QJ-RiB% zMPEd*7P!-@A6=3~0@fJKVW3}83d4Ih`qVu|BqcIsv#6BU1 z{0f{hSuy#9;hhXsaAk04rik>Ma1J7kS6P}9XAB-g1z?uKwYUIfuh7xTG2YhD!X1k2 zC!!yN-tRZlM8>0bZj7}ulT9|%uC#O|6Ld8|AfbR>>WLH^-!@@df}83tORqC^cr}mz zC_9R|ij0mJjKj$rx9>4t1oQ6o)YSdo#`n1`a>kqvS(%yYhOcqSI|@+oL1|CpP5qbm zeX*($sc5sDrulJ;s1Uityvj8i>V~UvOBP!5bK_*=20^w;I|hE*VIUbmc(w%i}Vuh)>AoVNRg0XBZd&7H2RaA&OSxa z0^_&vwbw6w*LBK{%oINv1|<3;4v!tzph?$Kbyy_Ty^)tv z2(G1xaODMV$$l)sd%#AKI#kD)Udo##v+K03BNu96ye47aHrMwpV7i$u#IIh z;m}r@#lSs7UHpO`8l67^u!V%ryY_?iUcnZE`)uDc1)h}hzg5!*BK`BU2PW_l;@xq?xRAAFHH?+us(x`+p~MibabjRoWu<_fRH0pOc$_u1D!-(amk(VtnWiCrl7}mVY>6e(wK>46>cQ&na-}YX(6fus)#80h5JSix z9TEUeLB29dk(zT5PR;7xv0+crH$wx$h7AhOs*sLl`~CYN4sAZ`l6N+iwcew(Za}o= z7-byfk>IPyl^m3YZ(Px5YCw;g62~IL3-77!6r?%6Q_q^w*<$B`wU8tT$Qw;gk<2F> z%Qmyo`T)lARO(c#Navv>XJ4=gcIWlm%AxS0B;w%sL@O6eA z37DY}%*x~v(#}kTQ+#)!n(v3r8qlQv0Hi)fUV`J_!J&F?aX0F9MNL<@pesaZ7uG=O zpN-VkL{+c=fdn%dOK@YMkyGN>4wht#LmX?c5~d8N`ah8h-1~!jBd4Nn#GExzOi(XU z>@a|dCCDPKp&K#m;7;Q8j5~;a3-QQF>l|GgKEdL|V$D}1c3s*M{uo{U8RMyhe~MpcGoBv7}5O$EPkLCMh%cUuDFIO!R%`o1l=ZOM|1A(X{bh4!!Y0VO+ROk#d} zVs%`mq~D|WiRQnza3v3C0+sQ(cT=A2*U=KcU*0`bPZ-cfJ?C9}dvqV&>RkKhX+9ts z0?@>F<|#OEnGMe0mb(m(3)D2)^GX@Z_gGHLl$w_)Rc_m0bHS~i%p9;9&g#SazwdEo z0$yZhZ3=c^Dpiau;;ZlJKvU`>-Dr8#-eSUEy%N%Q4zk>M_3MuaJW6>6^X|b|!aZ!} z?tTk^_~tZoeBIvGS4j_lD(D#nh5}(#*#*;4VbAr};NCeGR#1eM#!E4-B#9Qc`$F|s zOgWgx*zmnH4PQ?n-Fy5fpO8_Rp(XYJz;#HwC6u|}wthZ|Gi}jVX_=C%HEhj{dHv>b zrqQXjNlSGu0)LY*q{=%52~bk4>^*6s1~%ux($fQlD64GZZc0o0>X3smD!2}|RQ9ei zv&s;!-lOfM5*TY7{5u?>F!u6a{C_rP9$1(7J$`bVb5Ile!_I5q*lor(snl%^lBogn zf$)qwKA2Y-jNsn(iuF8o6!9MZ{&;S+5?8bO&A0_0J5iQU=mMl7o1GHG7<^Sme1P@q zzlF~4&}ghKapozq{qz6_-6S-g21tTkk%A(sAJG8eqpRhLsVw%T&Kh*CykZDeo*BV% z>K0r-CzxZX!GwEE>aZi!xugeZr$~i@yQ`YfOI7>C;1~xpT%?P?i;uRaP4TmoG^=O_ zcISFSX;lXx_>+hDPM+j8kDt#Huc?jbC!d#RXuRWKh6QE+dyV=zbKmdbd>38j4C?{z^V<@RIGT$ENT>-LNjkMJS6Q=Ko8 z!HO+5@*XQ=*smjGowQ2qarfV|?t@$1=K)(}SeG*P`hN3xCYzM(7;tsMwsl6 zyI07O^DAV-lEV&i7T6uKYsnV|{)mFPDe-_AfH)ygBL(+|HSDGu9CnT)!87~siTXW6 z@ZLq=Ib1&j9fT(K4@NzRDzw~0gqUYgaM^6ook&RI;X~+3*T&3L!`gIqtk5ZzEBz+Zg4c;>(@-w{_vM&jV+Qc zK<*Gw2oYC_Np)e;mCt2{x23E~+SQwPpHe^X*ATe3>u)X`P}9P=WU6hcvu|%A_Yq># zrcc(e)z2^r@AndE_c^oM<4qMtQw|<=hSnI#S{QRp1wMHlTwQupJEt{wHlV(uopM1n zSb&kLSq*LfoLNbmwH#t0WOm%i)MsBWU-#9J6moD}E9j6>ad^i|5vJl9o|;o8{YSH_ zJYfwMpE)u_$g+QtnHrqfR_{Q)4a)fIA_^kWI0ex>~icCSpTw6{X}%TqI{Rb+Q} z^eEZvWs&qWm)v;ON!-4@$OZy^iX?-u)vDE)m(`fb`A)r6ZGPV(vwM4%CX-n@fhs1l zt*CX?B`h&Xlu=$w#>k%a7pFVpokK6@L38lv5ae z>-?0Gl2w7PMAe?eEkfi0gD<94>ZT`3{gb)83tCmSO=brKb|h1@uC?l4GcvF>a_?xK zDlz2!{p4lbB?wR)y}fO#{WrFwGj4ZWNVE|HX1yWQomU+9rZxGkQX17pO3n|kYa)G^ z+11=KG|Kr*zOVq%;=JoJ99t1UQ{qfP0zbCudX14#2z{f(p3Eo_=d*$1;b^)F(GJA5 zkXulFF8OD*ncw*oMfpvqw^lyq_h+f^ClK8BHPr#S2A?Bpf$)XVDRsMWctsVO7m zXz-VRK7Z=z>8Y`-AWzqiT(E||Jjm34uA(>H7YQ>|#ve#FbN|(<_gTgUJQsv?M1mmM z1xF{L4Vx*T2PHX{(k++(_nvvuKypX2V@cg^ve)m-jFZSq=LR@X&4d$J5djpdGk;7Q z3%-`{G}8*gN-~zkNy@`x$La9`D|*E9GTlB#qDQe#oYGGR6z+$W;Uwk_CQEfQ7c24O zD|gCwU8IbRKP-qdOLm>MKY$e>_4}5Q?9oQDXy`cqmjjDa^Xg|jj=Q+G4O;d2cgs&2 zv(w>zxwZ4oUvru5B-n!Z@3Gw~oU}s6{;KVBUXx+J8d%RZ8Jy<0-$B&>kXqcPhFdsT z&k?M+Om1UK7*=?-?Nm$m=l;)%L&4=B0n}4*TkngH>x{}7U42VrW4#0d^T`1?GiuVB zH1EdrGV3OTPUU`oqd9zE_@GVolR)~4q9I$$V4t$SgNH=OivRDxYbOGQaNq_Oef8j^ zd@OoC*cE8#j;e>tVv&juVp9W=Qh}Q$5`+SnG-g?+D6p(CXdKGPTsk!J3w2o6@f&w` z{-fO1cs}52Am5T-h9HYZ@Tf`4ygUIy#fPw1s(G<7a8KnW12BL%QYaj7!#jgVStonI zxht-W#jkE`h+%kiL*Tx58JuZ7|8)t#Ltu0)s^$56;)7z#xc1D;XZ4BpelTR&gFxJI zMRJ^W8ePK#uw2PbwiUQqX4$8Ibe_#y#3lB75MF-8@8NBGNjCYx++BTW8f^nD&nSDj zqz+r7!kRv(HJd_OzW*bia=-0fH2+y5L;m(Nf_v$Zja`A@u*-+^Z6~@Bt(21@Da6Rd zI)P)2X(!>_lD)Z?a6cKNv(We0ika_7!6}d+K`?`$2Yqqw5YIw%Pv8eSVvKNR!3?ez z;lb}V8af;(`3QzD9dQ`dO^en%Px#w{fjG)d?#t+oymwz~qD|va`TJB<7p0txyQRFc zd2-(WMJD6Prie(M@&$xL*dC;dj~Px!66cYY^>W!G5l)c9EtitIT287Kp|%xtP9w^( zlwFVo`BpB)d@M&m{XO)l!;?YVPaFDq1`Py(7F#M>6zgZpgr;0ql@Yo7IJ}L2XEk}= zNrKBV3X;hq&=OEOc;+eA$dGB3$j`(f1yR+>Q^Q*YJeA2FLcG=s99{?$*|AIIesL4p zp^_94Ul-@(>?Uv@w{S1LA&JxGrLRdtPA;=jTJbS+xQdx9k;223y8MO{4?qzikz@;1XP&(l3beET{`Lf{6 zrcfg%-q8TgBf_w*FXEXyF_b&vD<-eBeW7Y-A?QizObb#%mITqAW(o6-BZ0=V=&ONU z8!S*7XN7`BQ%>d@M<+zQ(d@ERV*Xqjej%}5E1KqRW)p+c6P>{pfC4NyNBIJMTgf{gonqc=`m?qGD`>{U%({k)2l*FLwa?26@)p=qkcm+-`Y&(TuO< z=c^~0(~OhA(&$WZIFrtILEOut7zDv7ZtOHXA*s**1G?5-K;G!sdd%sDEL?=CZ%dfd zZkKZ@$mQy1+m5vQdFyPKpDtM0=0iaaDb6<+BgPtT(Wlr=@Xsgbkpn8>e;TFK@}-SJ zZ5AcM&*(IvvGff#xgUDMdSag?A=G`WZgJYI7j(PlX^t!%{D2*@=X9xd_M#=jSQRs? z{~r_dA99FG*i)IkR>Xf(UHI82AD( zfp6?R^c8A#;r3SF*)eJ&3H+Jso<=;`-n>!Z(;7}Eie^T+6UhGrtY{)>KQ~|fQJ0Kt`pTv%uWe{VOFris5fE74yP6WDc4V~ zZFUl7^6zh=Tj5wMB(q%T)+66GYrW$`qwfEf7v_cQO0$r+t-$U3_ScDD{4c9s&>Ec8 z!yo?sKd&hWd*`-2HD0Fc*HR7m?;7OAT;c0V*&VlEVSc=b`rlH`tMjmz-2c01`^}bi So>i5C{21t(z-qM}(f`_ as well as +a `Docker container on NGC `_. It's assumed the user has +already installed NeMo by following the :ref:`quick-start-guide` instructions. + +The conversational AI pipeline consists of three major stages: + +- Automatic Speech Recognition (ASR) +- Natural Language Processing (NLP) or Natural Language Understanding (NLU) +- Text-to-Speech (TTS) or voice synthesis + +As you talk to a computer, the ASR phase converts the audio signal into text, the NLP stage interprets the question +and generates a smart response, and finally the TTS phase converts the text into speech signals to generate audio for +the user. The toolkit enables development and training of deep learning models involved in conversational AI and easily +chain them together. + +Why NeMo? +--------- + +Deep learning model development for conversational AI is complex. It involves defining, building, and training several +models in specific domains; experimenting several times to get high accuracy, fine tuning on multiple tasks and domain +specific data, ensuring training performance and making sure the models are ready for deployment to inference applications. +Neural modules are logical blocks of AI applications which take some typed inputs and produce certain typed outputs. By +separating a model into its essential components in a building block manner, NeMo helps researchers develop state-of-the-art +accuracy models for domain specific data faster and easier. + +Collections of modules for core tasks as well as specific to speech recognition, natural language, speech synthesis help +develop modular, flexible, and reusable pipelines. + +A neural module’s inputs/outputs have a neural type, that describes the semantics, the axis order and meaning, and the dimensions +of the input/output tensors. This typing allows neural modules to be safely chained together to build models for applications. + +NeMo can be used to train new models or perform transfer learning on existing pre-trained models. Pre-trained weights per module +(such as encoder, decoder) help accelerate model training for domain specific data. + +ASR, NLP and TTS pre-trained models are trained on multiple datasets (including some languages such as Mandarin) and optimized +for high accuracy. They can be used for transfer learning as well. + +NeMo supports developing models that work with Mandarin Chinese data. Tutorials help users train or fine tune models for +conversational AI with the Mandarin Chinese language. The export method provided in NeMo makes it easy to transform a trained +model into inference ready format for deployment. + +A key area of development in the toolkit is interoperability with other tools used by speech researchers. Data layer for Kaldi +compatibility is one such example. + +NeMo, PyTorch Lightning, And Hydra +---------------------------------- + +Conversational AI architectures are typically very large and require a lot of data and compute for training. NeMo uses +`Pytorch Lightning `_ for easy and performant multi-GPU/multi-node +mixed precision training. + +Pytorch Lightning is a high-performance PyTorch wrapper that organizes PyTorch code, scales model training, and reduces +boilerplate. PyTorch Lightning has two main components, the ``LightningModule`` and the Trainer. The ``LightningModule`` is +used to organize PyTorch code so that deep learning experiments can be easily understood and reproduced. The Pytorch Lightning +Trainer is then able to take the ``LightningModule`` and automate everything needed for deep learning training. + +NeMo models are LightningModules that come equipped with all supporting infrastructure for training and reproducibility. This +includes the deep learning model architecture, data preprocessing, optimizer, check-pointing and experiment logging. NeMo +models, like LightningModules, are also PyTorch modules and are fully compatible with the broader PyTorch ecosystem. Any NeMo +model can be taken and plugged into any PyTorch workflow. + +Configuring conversational AI applications is difficult due to the need to bring together many different Python libraries into +one end-to-end system. NeMo uses Hydra for configuring both NeMo models and the PyTorch Lightning Trainer. `Hydra `_ +is a flexible solution that makes it easy to configure all of these libraries from a configuration file or from the command-line. + +Every NeMo model has an example configuration file and a corresponding script that contains all configurations needed for training +to state-of-the-art accuracy. NeMo models have the same look and feel so that it is easy to do conversational AI research across +multiple domains. + +Using Optimized Pretrained Models With NeMo +------------------------------------------- + +`NVIDIA GPU Cloud (NGC) `_ is a software repository that has containers and models optimized +for deep learning. NGC hosts many conversational AI models developed with NeMo that have been trained to state-of-the-art accuracy +on large datasets. NeMo models on NGC can be automatically downloaded and used for transfer learning tasks. Pretrained models +are the quickest way to get started with conversational AI on your own data. NeMo has many `example scripts `_ +and `Jupyter Notebook tutorials `_ showing step-by-step how to fine-tune pretrained NeMo +models on your own domain-specific datasets. + +For BERT based models, the model weights provided are ready for +downstream NLU tasks. For speech models, it can be helpful to start with a pretrained model and then continue pretraining on your +own domain-specific data. Jasper and QuartzNet base model pretrained weights have been known to be very efficient when used as +base models. For an easy to follow guide on transfer learning and building domain specific ASR models, you can follow this `blog `_. +All pre-trained NeMo models can be found on the `NGC NeMo Collection `_. Everything needed to quickly get started +with NeMo ASR, NLP, and TTS models is there. + +Pre-trained models are packaged as a ``.nemo`` file and contain the PyTorch checkpoint along with everything needed to use the model. +NeMo models are trained to state-of-the-art accuracy and trained on multiple datasets so that they are robust to small differences +in data. NeMo contains a large variety of models such as speaker identification and Megatron BERT and the best models in speech and +language are constantly being added as they become available. NeMo is the premier toolkit for conversational AI model building and +training. + +For a list of supported models, refer to the :ref:`tutorials` section. + +ASR Guidance +------------ + +This section is to help guide your decision making by answering our most asked ASR questions. + +**Q: Is there a way to add domain specific vocabulary in NeMo? If so, how do I do that?** +A: QuartzNet and Jasper models are character-based. So pretrained models we provide for these two output lowercase English +letters and ‘. Users can re-retrain them on vocabulary with upper case letters and punctuation symbols. + +**Q: When training, there are “Reference” lines and “Decoded” lines that are printed out. It seems like the reference line should +be the “truth” line and the decoded line should be what the ASR is transcribing. Why do I see that even the reference lines do not +appear to be correct?** +A: Because our pre-trained models can only output lowercase letters and apostrophe, everything else is dropped. So the model will +transcribe 10 as ten. The best way forward is to prepare the training data first by transforming everything to lowercase and convert +the numbers from digit representation to word representation using a simple library such as `inflect `_. Then, add the uppercase letters +and punctuation back using the NLP punctuation model. Here is an example of how this is incorporated: `NeMo voice swap demo `_. + +**Q: What languages are supported in NeMo currently?** +A: Along with English, we provide pre-trained models for Zh, Es, Fr, De, Ru, It, Ca and Pl languages. +For more information, see `NeMo Speech Models `_. + +Data Augmentation +----------------- + +Data augmentation in ASR is invaluable. It comes at the cost of increased training time if samples are augmented during training +time. To save training time, it is recommended to pre-process the dataset offline for a one time preprocessing cost and then train +the dataset on this augmented training set. + +For example, processing a single sample involves: + +- Speed perturbation +- Time stretch perturbation (sample level) +- Noise perturbation +- Impulse perturbation +- Time stretch augmentation (batch level, neural module) + +A simple tutorial guides users on how to use these utilities provided in `GitHub: NeMo `_. + +Speech Data Explorer +-------------------- + +Speech data explorer is a `Dash-based tool `_ for interactive exploration of ASR/TTS datasets. + +Speech data explorer collects: + +- dataset statistics (alphabet, vocabulary, and duration-based histograms) +- navigation across datasets (sorting and filtering) +- inspections of individual utterances (waveform, spectrogram, and audio player) +- errors analysis (word error rate, character error rate, word match rate, mean word accuracy, and diff) + +In order to use the tool, it needs to be installed separately. Perform the steps `here `_ to install speech data explorer. + +Using Kaldi Formatted Data +-------------------------- + +The `Kaldi Speech Recognition Toolkit `_ project began in 2009 at `Johns Hopkins University `. It is a toolkit written in C++. If +researchers have used Kaldi and have datasets that are formatted to be used with the toolkit; they can use NeMo to develop models +based on that data. + +To load Kaldi-formatted data, you can simply use ``KaldiFeatureDataLayer`` instead of ``AudioToTextDataLayer``. The ``KaldiFeatureDataLayer`` +takes in the argument ``kaldi_dir`` instead of a ``manifest_filepath``. The ``manifest_filepath`` argument should be set to the directory +that contains the files ``feats.scp`` and ``text``. + +Using Speech Command Recognition Task For ASR Models +---------------------------------------------------- + +Speech Command Recognition is the task of classifying an input audio pattern into a set of discrete classes. It is a subset of ASR, +sometimes referred to as Key Word Spotting, in which a model is constantly analyzing speech patterns to detect certain ``action`` classes. + +Upon detection of these commands, a specific action can be taken. An example Jupyter notebook provided in NeMo shows how to train a +QuartzNet model with a modified decoder head trained on a speech commands dataset. + +.. note:: It is preferred that you use absolute paths to ``data_dir`` when preprocessing the dataset. + +NLP Fine-Tuning BERT +-------------------- + +BERT, or Bidirectional Encoder Representations from Transformers, is a neural approach to pre-train language representations which +obtains near state-of-the-art results on a wide array of Natural Language Processing (NLP) tasks, including the GLUE benchmark and +SQuAD Question & Answering dataset. + +BERT model checkpoints (`BERT-large-uncased `_ and `BERT-base-uncased `_) are provided can be used for either fine tuning BERT on your custom +dataset, or fine tuning downstream tasks, including GLUE benchmark tasks, Question & Answering tasks, Joint Intent & Slot detection, +Punctuation and Capitalization, Named Entity Recognition, and Speech Recognition post processing model to correct mistakes. + +.. note:: Almost all NLP examples also support RoBERTa and ALBERT models for downstream fine-tuning tasks (see the list of all +supported models by calling ``nemo.collections.nlp.modules.common.lm_utils.get_pretrained_lm_models_list()``. The user needs to specify +the name of the model desired while running the example scripts. + +BioMegatron Medical BERT +------------------------ + +BioMegatron is a large language model (Megatron-LM) trained on larger domain text corpus (PubMed abstract + full-text-commercial). +It achieves state-of-the-art results for certain tasks such as Relationship Extraction, Named Entity Recognition and Question & +Answering. Follow these tutorials to learn how to train and fine tune BioMegatron; pretrained models are provided on NGC: + +- `Relation Extraction BioMegatron `_ +- `Token Classification BioMegatron `_ + +Efficient Training With NeMo +---------------------------- + +Using Mixed Precision +^^^^^^^^^^^^^^^^^^^^^ + +Mixed precision accelerates training speed while protecting against noticeable loss. Tensor Cores is a specific hardware unit that +comes starting with the Volta and Turing architectures to accelerate large matrix to matrix multiply-add operations by operating them +on half precision inputs and returning the result in full precision. + +Neural networks which usually use massive matrix multiplications can be significantly sped up with mixed precision and Tensor Cores. +However, some neural network layers are numerically more sensitive than others. Apex AMP is an NVIDIA library that maximizes the +benefit of mixed precision and Tensor Cores usage for a given network. + +Multi-GPU Training +^^^^^^^^^^^^^^^^^^ + +This section is to help guide your decision making by answering our most asked multi-GPU training questions. + +**Q: Why is multi-GPU training preferred over other types of training?** +A: Multi-GPU training can reduce the total training time by distributing the workload onto multiple compute instances. This is +particularly important for large neural networks which would otherwise take weeks to train until convergence. Since NeMo supports +multi-GPU training, no code change is needed to move from single to multi-GPU training, only a slight change in your launch command +is required. + +**Q: What are the advantages of mixed precision training?** +A: Mixed precision accelerates training speed while protecting against noticeable loss in precision. Tensor Cores is a specific +hardware unit that comes starting with the Volta and Turing architectures to accelerate large matrix multiply-add operations by +operating on half precision inputs and returning the result in full precision in order to prevent loss in precision. Neural +networks which usually use massive matrix multiplications can be significantly sped up with mixed precision and Tensor Cores. +However, some neural network layers are numerically more sensitive than others. Apex AMP is a NVIDIA library that maximizes the +benefit of mixed precision and Tensor Core usage for a given network. + +**Q: What is the difference between multi-GPU and multi-node training?** +A: Multi-node is an abstraction of multi-GPU training, which requires a distributed compute cluster, where each node can have multiple +GPUs. Multi-node training is needed to scale training beyond a single node to large amounts of GPUs. + +From the framework perspective, nothing changes from moving to multi-node training. However, a master address and port needs to be set +up for inter-node communication. Multi-GPU training will then be launched on each node with passed information. You might also consider +the underlying inter-node network topology and type to achieve full performance, such as HPC-style hardware such as NVLink, InfiniBand +networking, or Ethernet. + + +Recommendations For Optimization And FAQs +----------------------------------------- + +This section is to help guide your decision making by answering our most asked NeMo questions. + +**Q: Are there areas where performance can be increased?** +A: You should try using mixed precision for improved performance. Note that typically when using mixed precision, memory consumption +is decreased and larger batch sizes could be used to further improve the performance. + +When fine-tuning ASR models on your data, it is almost always possible to take advantage of NeMo's pre-trained modules. Even if you +have a different target vocabulary, or even a different language; you can still try starting with pre-trained weights from Jasper or +QuartzNet ``encoder`` and only adjust the ``decoder`` for your needs. + +**Q: What is the recommended sampling rate for ASR?** +A: The released models are based on 16 KHz audio, therefore, ensure you use models with 16 KHz audio. Reduced performance should be +expected for any audio that is up-sampled from a sampling frequency less than 16 KHz data. + +**Q: How do we use this toolkit for audio with different types of compression and frequency than the training domain for ASR?** +A: You have to match the compression and frequency. + +**Q: How do you replace the 6-gram out of the ASR model with a custom language model? What is the language format supported in NeMo?** +A: NeMo’s Beam Search decoder with Levenberg-Marquardt (LM) neural module supports the KenLM language model. + +- You should retrain the KenLM language model on your own dataset. Refer to `KenLM’s documentation `_. +- If you want to use a different language model, other than KenLM, you will need to implement a corresponding decoder module. +- Transformer-XL example is present in OS2S. It would need to be updated to work with NeMo. `Here is the code `_. + +**Q: How do I use text-to-speech (TTS)?** +A: +- Obtain speech data ideally at 22050 Hz or alternatively at a higher sample rate and then down sample to 22050 Hz. + - If less than 22050 Hz and above 16000 Hz: + - Retrain WaveGlow on your own dataset. + - Tweak the spectrogram generation parameters, namely the ``window_size`` and the ``window_stride`` for their fourier + transforms. + - For below 16000 Hz, look into obtaining new data. +- In terms of bitrate/quantization, the general advice is the higher the better. We have not experimented enough to state how much + this impacts quality. +- For the amount of data, again the more the better, and the more diverse in terms of phonemes the better. Aim for around 20 hours + of speech after filtering for silences and non-speech audio. +- Most open speech datasets are in ~10 second format so training Tacotron 2 on audio on the order of 10s - 20s per sample is known + to work. Additionally, the longer the speech samples, the more difficult it will be to train Tacotron 2. +- Audio files should be clean. There should be little background noise or music. Data recorded from a studio mic is likely to be easier + to train compared to data captured using a phone. +- To ensure pronunciation of words are accurate; the technical challenge is related to the dataset, text to phonetic spelling is + required, use phonetic alphabet (notation) that has the name correctly pronounced. +- Here are some example parameters you can use to train Tacotron 2: + - Tacotron 2 requires a single speaker dataset + - Use AMP level O0 + - Trim long silences in the beginning and end + - ``optimizer="adam"`` + - ``beta1 = 0.9`` + - ``beta2 = 0.999`` + - ``lr=0.001 (constant)`` + - ``amp_opt_level="O0"`` + - ``weight_decay=1e-6`` + - ``batch_size=48 (per GPU)`` + - ``trim_silence=True`` + +Resources +--------- + +Ensure you are familiar with the following resources for NeMo. + +- Developer blogs + - `How to Build Domain Specific Automatic Speech Recognition Models on GPUs `_ + - `Develop Smaller Speech Recognition Models with NVIDIA’s NeMo Framework `_ + - `Neural Modules for Fast Development of Speech and Language Models `_ + +- Domain specific, transfer learning, Docker container with Jupyter Notebooks + - `Domain Specific NeMo ASR Application `_ + diff --git a/models/tacotron2/NeMo/docs/source/starthere/intro.rst b/models/tacotron2/NeMo/docs/source/starthere/intro.rst new file mode 100644 index 0000000000..d80a3dfac9 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/starthere/intro.rst @@ -0,0 +1,157 @@ +Introduction +============ + +.. # define a hard line break for html +.. |br| raw:: html + +
+ +.. _dummy_header: + +`NVIDIA NeMo `_ is a toolkit for building new state-of-the-art +conversational AI models. NeMo has separate collections for Automatic Speech Recognition (ASR), +Natural Language Processing (NLP), and Text-to-Speech (TTS) models. Each collection consists of +prebuilt modules that include everything needed to train on your data. +Every module can easily be customized, extended, and composed to create new conversational AI +model architectures. + +Conversational AI architectures are typically large and require a lot of data and compute +for training. NeMo uses `PyTorch Lightning `_ for easy and performant multi-GPU/multi-node +mixed-precision training. + +.. raw:: html + +

+ +
+ +For more information and questions, visit the `NVIDIA NeMo Discussion Board `_. + +Prerequisites +------------- + +Before you begin using NeMo, it's assumed you meet the following prerequisites. + +#. You have Python version 3.6, 3.7 or 3.8. + +#. You have Pytorch version 1.8.1. + +#. You have access to an NVIDIA GPU for training. + +.. _quick_start_guide: + +Quick Start Guide +----------------- + +This NeMo Quick Start Guide is a starting point for users who want to try out NeMo; specifically, this guide enables users to quickly get started with the NeMo fundamentals by walking you through an example audio translator and voice swap. + +If you're new to NeMo, the best way to get started is to take a look at the following tutorials: + +* `Text Classification (Sentiment Analysis) `__ - demonstrates the Text Classification model using the NeMo NLP collection. +* `NeMo Primer `__ - introduces NeMo, PyTorch Lightning, and OmegaConf, and shows how to use, modify, save, and restore NeMo models. +* `NeMo Models `__ - explains the fundamental concepts of the NeMo model. +* `NeMo voice swap demo `__ - demonstrates how to swap a voice in the audio fragment with a computer generated one using NeMo. + +Below we is the code snippet of Audio Translator application. + +.. code-block:: python + + # Import NeMo and it's ASR, NLP and TTS collections + import nemo + # Import Speech Recognition collection + import nemo.collections.asr as nemo_asr + # Import Natural Language Processing colleciton + import nemo.collections.nlp as nemo_nlp + # Import Speech Synthesis collection + import nemo.collections.tts as nemo_tts + + # Next, we instantiate all the necessary models directly from NVIDIA NGC + # Speech Recognition model - QuartzNet trained on Russian part of MCV 6.0 + quartznet = nemo_asr.models.EncDecCTCModel.from_pretrained(model_name="stt_ru_quartznet15x5").cuda() + # Neural Machine Translation model + nmt_model = nemo_nlp.models.MTEncDecModel.from_pretrained(model_name='nmt_ru_en_transformer6x6').cuda() + # Spectrogram generator which takes text as an input and produces spectrogram + spectrogram_generator = nemo_tts.models.Tacotron2Model.from_pretrained(model_name="tts_en_tacotron2").cuda() + # Vocoder model which takes spectrogram and produces actual audio + vocoder = nemo_tts.models.WaveGlowModel.from_pretrained(model_name="tts_waveglow_88m").cuda() + # Transcribe an audio file + # IMPORTANT: The audio must be mono with 16Khz sampling rate + # Get example from: https://nemo-public.s3.us-east-2.amazonaws.com/mcv-samples-ru/common_voice_ru_19034087.wav + russian_text = quartznet.transcribe(['Path_to_audio_file']) + print(russian_text) + # You should see russian text here. Let's translate it to English + english_text = nmt_model.translate(russian_text) + print(english_text) + # After this you should see English translation + # Let's convert it into audio + # A helper function which combines Tacotron2 and WaveGlow to go directly from + # text to audio + def text_to_audio(text): + parsed = spectrogram_generator.parse(text) + spectrogram = spectrogram_generator.generate_spectrogram(tokens=parsed) + audio = vocoder.convert_spectrogram_to_audio(spec=spectrogram) + return audio.to('cpu').numpy() + audio = text_to_audio(english_text[0]) + + +Installation +------------ + +Pip +~~~ +Use this installation mode if you want the latest released version. + +.. code-block:: bash + + apt-get update && apt-get install -y libsndfile1 ffmpeg + pip install Cython + pip install nemo_toolkit[all] + +Pip from source +~~~~~~~~~~~~~~~ +Use this installation mode if you want the version from a particular GitHub branch (for example, ``main``). + +.. code-block:: bash + + apt-get update && apt-get install -y libsndfile1 ffmpeg + pip install Cython + python -m pip install git+https://github.com/NVIDIA/NeMo.git@{BRANCH}#egg=nemo_toolkit[all] + # For v1.0.2, replace {BRANCH} with v1.0.2 like so: + # python -m pip install git+https://github.com/NVIDIA/NeMo.git@v1.0.2#egg=nemo_toolkit[all] + +From source +~~~~~~~~~~~ +Use this installation mode if you are contributing to NeMo. + +.. code-block:: bash + + apt-get update && apt-get install -y libsndfile1 ffmpeg + git clone https://github.com/NVIDIA/NeMo + cd NeMo + ./reinstall.sh + +Docker containers +~~~~~~~~~~~~~~~~~ +If you chose to work with the ``main`` branch, we recommend using `NVIDIA's PyTorch container version 21.05-py3 `_, then install from GitHub. + +.. code-block:: bash + + docker run --gpus all -it --rm -v :/NeMo --shm-size=8g \ + -p 8888:8888 -p 6006:6006 --ulimit memlock=-1 --ulimit \ + stack=67108864 --device=/dev/snd nvcr.io/nvidia/pytorch:21.05-py3 + + +FAQ +--- +Have a look at our `discussions board `_ and feel free to post a question or start a discussion. + + +Contributing +------------ + +We welcome community contributions! Refer to the `CONTRIBUTING.md `_ file for the process. + +License +------- + +NeMo is under `Apache 2.0 license `_. \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/starthere/tutorials.rst b/models/tacotron2/NeMo/docs/source/starthere/tutorials.rst new file mode 100644 index 0000000000..76b4dd744e --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/starthere/tutorials.rst @@ -0,0 +1,117 @@ +.. _tutorials: + +Tutorials +========= + +The best way to get started with NeMo is to start wtih one of our tutorials. + +Most NeMo tutorials can be run on `Google's Colab `_. + +To run a tutorial: + +#. Click the **Colab** link (see table below). +#. Connect to an instance with a GPU. For example, click **Runtime** > **Change runtime type** and select **GPU** for the hardware accelerator. + +.. list-table:: **Tutorials** + :widths: 15 25 25 + :header-rows: 1 + + * - Domain + - Title + - GitHub URL + * - General + - Getting Started: Exploring Nemo Fundamentals + - `NeMo Fundamentals `_ + * - General + - Getting Started: Sample Conversational AI application + - `Audio translator example `_ + * - General + - Getting Started: Voice swap application + - `Voice swap example `_ + * - General + - Exploring NeMo Model Construction + - `NeMo Models `_ + * - ASR + - ASR with NeMo + - `ASR with NeMo `_ + * - ASR + - ASR with Subword Tokenization + - `ASR with Subword Tokenization `_ + * - ASR + - Offline ASR Inference with Beam Search and External Language Model Rescoring + - `Offline ASR `_ + * - ASR + - Online ASR inference with Microphone + - `Online ASR Microphone `_ + * - ASR + - Fine-tuning CTC Models on New Languages + - `ASR CTC Language Fine-Tuning `_ + * - ASR + - Speech Commands + - `Speech Commands `_ + * - ASR + - Online and Offline Speech Commands Inference + - `Online Offline Microphone Speech Commands `_ + * - ASR + - Voice Activiy Detection (VAD) + - `Voice Activiy Detection `_ + * - ASR + - Online and Offline VAD Inference + - `Online Offline Microphone VAD `_ + * - ASR + - Speaker Recognition and Verification + - `Speaker Recognition and Verification `_ + * - ASR + - Speaker Diarization Inference + - `Speaker Diarization Inference `_ + * - ASR + - ASR with Speaker Diarization + - `ASR with Speaker Diarization `_ + * - ASR + - Online Noise Augmentation + - `Online Noise Augmentation `_ + * - NLP + - Using Pretrained Language Models for Downstream Tasks + - `Pretrained Language Models for Downstream Tasks `_ + * - NLP + - Exploring NeMo NLP Tokenizers + - `NLP Tokenizers `_ + * - NLP + - Text Classification (Sentiment Analysis) with BERT + - `Text Classification (Sentiment Analysis) `_ + * - NLP + - Question Answering with SQuAD + - `Question Answering Squad `_ + * - NLP + - Token Classification (Named Entity Recognition) + - `Token Classification: Named Entity Recognition `_ + * - NLP + - Joint Intent Classification and Slot Filling + - `Joint Intent and Slot Classification `_ + * - NLP + - GLUE Benchmark + - `GLUE Benchmark `_ + * - NLP + - Punctuation and Capitalization + - `Punctuation and Capitalization `_ + * - NLP + - Named Entity Recognition - BioMegatron + - `Named Entity Recognition - BioMegatron `_ + * - NLP + - Relation Extraction - BioMegatron + - `Relation Extraction - BioMegatron `_ + * - TTS + - Speech Synthesis + - `TTS Inference `_ + * - TTS + - Speech Synthesis + - `Tacotron2 Training `_ + * - Tools + - CTC Segmentation + - `CTC Segmentation `_ + * - Text Processing + - Text Normalization for TTS + - `Text Normalization `_ + * - Text Processing + - Inverse Text Normalization for ASR + - `Inverse Text Normalization `_ \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/tools/ctc_segmentation.rst b/models/tacotron2/NeMo/docs/source/tools/ctc_segmentation.rst new file mode 100644 index 0000000000..80e4f6cf85 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/tools/ctc_segmentation.rst @@ -0,0 +1,20 @@ +Dataset Creation Tool Based on CTC-Segmentation +=============================================== + +This tool provides functionality to align long audio files and the corresponding transcripts into shorter fragments +that are suitable for an Automatic Speech Recognition (ASR) model training. + +More details could be found in `NeMo/tutorials/tools/CTC_Segmentation_Tutorial.ipynb `__ (can be executed with `Google's Colab `_). + +The tool is based on the `CTC-Segmentation `__ package and +`CTC-Segmentation of Large Corpora for German End-to-end Speech Recognition +`__ :cite:`tools-kurzinger2020ctc` + + +References +---------- + +.. bibliography:: tools_all.bib + :style: plain + :labelprefix: TOOLS + :keyprefix: tools- diff --git a/models/tacotron2/NeMo/docs/source/tools/intro.rst b/models/tacotron2/NeMo/docs/source/tools/intro.rst new file mode 100644 index 0000000000..ff7a61fcb1 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/tools/intro.rst @@ -0,0 +1,14 @@ +Tools +===== + +NeMo provides a set of tools useful for developing Automatic Speech Recognitions (ASR) and Text-to-Speech (TTS) models: \ +`https://github.com/NVIDIA/NeMo/tree/main/tools `__ . + +.. toctree:: + :maxdepth: 1 + + ctc_segmentation + speech_data_explorer + text_processing_deployment + + diff --git a/models/tacotron2/NeMo/docs/source/tools/speech_data_explorer.rst b/models/tacotron2/NeMo/docs/source/tools/speech_data_explorer.rst new file mode 100644 index 0000000000..92d1d0752a --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/tools/speech_data_explorer.rst @@ -0,0 +1,33 @@ +Speech Data Explorer +==================== + +.. note:: + + The tool could be found under `NeMo/tools/speech_data_explorer `__. + +`Dash `__-based tool for interactive exploration of ASR/TTS datasets. + +Features: + +* dataset's statistics (alphabet, vocabulary, duration-based histograms) +* navigation across dataset (sorting, filtering) +* inspection of individual utterances (waveform, spectrogram, audio player) +* errors' analysis (Word Error Rate, Character Error Rate, Word Match Rate, Mean Word Accuracy, diff) + +Please make sure that requirements are installed. Then run: + +.. code:: + + python data_explorer.py path_to_manifest.json + + +JSON manifest file should contain the following fields: + +* `audio_filepath` (path to audio file) +* `duration` (duration of the audio file in seconds) +* `text` (reference transcript) + +Errors' analysis requires "pred_text" (ASR transcript) for all utterances. + +Any additional field will be parsed and displayed in 'Samples' tab. + diff --git a/models/tacotron2/NeMo/docs/source/tools/text_processing_deployment.rst b/models/tacotron2/NeMo/docs/source/tools/text_processing_deployment.rst new file mode 100644 index 0000000000..3b277bdfcf --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/tools/text_processing_deployment.rst @@ -0,0 +1,78 @@ +NeMo Text Processing Deployment +=============================================== + +This tool deploys :doc:`NeMo Inverse Text Normalization (ITN) <../nemo_text_processing/inverse_text_normalization>` and :doc:`NeMo Text Normalization (TN) <../nemo_text_processing/text_normalization>` for production :cite:`tools-itn_deploy-zhang2021nemo`. +It uses `Sparrowhawk `_ :cite:`tools-itn_deploy-sparrowhawk` -- an open-source version of Google Kestrel :cite:`tools-itn_deploy-ebden2015kestrel`. + +Requirements +------------------------ + +:doc:`nemo_text_processing <../nemo_text_processing/intro>` package + + +Usage +------------ + +Starts docker container with production backend with plugged in grammars. This is entry point script. + +Arguments: +^^^^^^^^^ +* ``GRAMMARS`` - ``tn_grammars`` or ``itn_grammars`` to export either TN or ITN grammars from :doc:`nemo_text_processing <../nemo_text_processing/intro>`. +* ``INPUT_CASE`` - ``cased`` or ``lower_cased`` (lower_cased is supported only in TN grammars). +* ``MODE`` - choose ``test`` to run test on the grammars inside the container. + +For example: + + +.. code-block:: bash + + # to export ITN grammars + bash export_grammar.sh --GRAMMARS=itn_grammars + + # to export and test TN grammars + bash export_grammar.sh --GRAMMARS=itn_grammars --INPUT_CASE=cased --MODE=test + +This script runs the following steps in sequence: + +Exports grammar `ClassifyFst` and `VerbalizeFst` from :doc:`nemo_text_processing <../nemo_text_processing/intro>` to `OUTPUT_DIR/classify/tokenize_and_classify.far` and `OUTPUT_DIR/verbalize/verbalize.far` respectively. + +.. code-block:: bash + + python pynini_export.py <--output_dir OUTPUT_DIR> <--grammars GRAMMARS> <--input_case INPUT_CASE> + +Builds C++ production backend docker + +.. code-block:: bash + + bash docker/build.sh + + +Plugs in grammars into production backend by mounting grammar directory `classify/` and `verbalize/` with sparrowhawk grammar directory inside docker. Returns docker prompt + +.. code-block:: bash + + # to launch container with the exported grammars + bash docker/launch.sh + + # to launch container with the exported grammars and run tests on TN grammars + bash docker/launch.sh test_tn_grammars + + # to launch container with the exported grammars and run tests on ITN grammars + bash docker/launch.sh test_itn_grammars + + +Runs TN or ITN in docker container: + +.. code-block:: bash + + echo "two dollars fifty" | ../../src/bin/normalizer_main --config=sparrowhawk_configuration.ascii_proto + +This returns $2.50 for ITN. + +References +---------- + +.. bibliography:: tools_all.bib + :style: plain + :labelprefix: TOOLS-ITN_DEPLOY + :keyprefix: tools-itn_deploy- diff --git a/models/tacotron2/NeMo/docs/source/tools/tools_all.bib b/models/tacotron2/NeMo/docs/source/tools/tools_all.bib new file mode 100644 index 0000000000..4b86ee2b74 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/tools/tools_all.bib @@ -0,0 +1,49 @@ +@inproceedings{kurzinger2020ctc, + title={CTC-segmentation of large corpora for german end-to-end speech recognition}, + author={K{\"u}rzinger, Ludwig and Winkelbauer, Dominik and Li, Lujun and Watzel, Tobias and Rigoll, Gerhard}, + booktitle={International Conference on Speech and Computer}, + pages={267--278}, + year={2020}, + organization={Springer} +} + +@article{ebden2015kestrel, + title={The Kestrel TTS text normalization system}, + author={Ebden, Peter and Sproat, Richard}, + journal={Natural Language Engineering}, + volume={21}, + number={3}, + pages={333}, + year={2015}, + publisher={Cambridge University Press} +} + +@article{sproat2016rnn, + title={RNN approaches to text normalization: A challenge}, + author={Sproat, Richard and Jaitly, Navdeep}, + journal={arXiv preprint arXiv:1611.00068}, + year={2016} +} + +@book{taylor2009text, + title={Text-to-speech synthesis}, + author={Taylor, Paul}, + year={2009}, + publisher={Cambridge university press} +} + +@misc{zhang2021nemo, + title={NeMo Inverse Text Normalization: From Development To Production}, + author={Yang Zhang and Evelina Bakhturina and Kyle Gorman and Boris Ginsburg}, + year={2021}, + eprint={2104.05055}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} + +@inproceedings{sparrowhawk, + title = {TTS for Low Resource Languages: A Bangla Synthesizer}, + author = {Alexander Gutkin and Linne Ha and Martin Jansche and Knot Pipatsrisawat and Richard Sproat}, + booktitle = {10th Language Resources and Evaluation Conference}, + year = {2016}, +} \ No newline at end of file diff --git a/models/tacotron2/NeMo/docs/source/tts/api.rst b/models/tacotron2/NeMo/docs/source/tts/api.rst new file mode 100644 index 0000000000..2255af4110 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/tts/api.rst @@ -0,0 +1,47 @@ +NeMo TTS Collection API +======================= + +TTS Base Classes +---------------- + +The classes below are the base of the TTS pipeline. +To read more about them, see the `Base Classes <./intro.html#Base Classes>`__ section of the intro page. + +.. autoclass:: nemo.collections.tts.models.base.SpectrogramGenerator + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.tts.models.base.Vocoder + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.tts.models.base.TextToWaveform + :show-inheritance: + :members: + +TTS Datasets +------------ + +.. autoclass:: nemo.collections.tts.data.datalayers.AudioDataset + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.tts.data.datalayers.MelAudioDataset + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.tts.data.datalayers.SplicedAudioDataset + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.tts.data.datalayers.NoisySpecsDataset + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.asr.data.audio_to_text.FastPitchDataset + :show-inheritance: + :members: + +.. autoclass:: nemo.collections.tts.data.datalayers.FastSpeech2Dataset + :show-inheritance: + :members: diff --git a/models/tacotron2/NeMo/docs/source/tts/datasets.rst b/models/tacotron2/NeMo/docs/source/tts/datasets.rst new file mode 100644 index 0000000000..ef3236e703 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/tts/datasets.rst @@ -0,0 +1,187 @@ +TTS Datasets +============ + +Most TTS scripts work out-of-the-box with the LJ Speech dataset. +However, some models may require supplementary data for training and validation. +The following sections contain details of required data formats and instructions for running preprocessing scripts for +such models. + + +FastSpeech 2 +------------ + +The FastSpeech 2 model converts from phonemes to Mel Spectrograms and predicts phoneme durations, pitches, and +energies. +Therefore, in addition to the manifest and audio samples, it requires some supplementary files and data: + +* A **mappings file** for converting from words to phonemes and phonemes to indices +* **Phoneme durations** (for each audio sample) +* **Pitch per frame** (for each training audio sample) +* **Energy per frame** (for each training audio sample) +* (Optional) **Ignore list** for filtering out samples with OOV words + +The ``FastSpeech2Dataset`` uses the manifest format shared with all other NeMo speech tasks. +Each line of the manifest should describe one sample, and should be in the following format: + +.. code:: + + {"audio_filepath": "/data/wavs/audio.wav", "text": "an unused transcription", "duration": 23.147} + +See the documentation on :ref:`Preparing Custom ASR Data` for more details on the NeMo speech manifest format. + +.. note:: + The ``FastSpeech2Dataset`` ignores the ``"text"`` field of the manifest, since the model reads phoneme indices from + the supplementary duration files instead. + +The **mappings file** should be a JSON-formatted file that contains dictionaries ``word2phones`` and ``phone2idx``. +``word2phones`` is a mapping that converts from your vocabulary to phonemes. +For example, one entry in ``word2phones`` might be ``"nemo": ["N", "EH1", "M", "OW0"]``. +``phone2idx`` is an (arbitrary) mapping that converts each phoneme to an index for prediction. +It is also used to determine the ``pad_id``, which is set to the length of ``phone2idx``. + +Together, these two dictionaries are also used for inference, as an input sentence is first normalized, then converted +to phonemes using ``word2phones``, and then converted from phonemes to indices using ``phone2idx``. + +For each audio sample, there should be a corresponding **phoneme durations** file that contains ``token_duration`` +(duration per phoneme in frames) and ``text_encoded`` (index of each phoneme corresponding to ``phone2idx``). + +For each sample, there should also be a **pitch file** that contains the pitches (F0) per frame, and an **energy file** +that contains the energies (L2-norm of STFT frame amplitudes) per frame. + +The **ignore list** is a pickled list of file base names (with no extension) that tells the ``FastSpeech2Dataset`` +which audio samples to discard. +You should use this to exclude samples from your manifest that contain OOV words. + + +Directory Structure +^^^^^^^^^^^^^^^^^^^ + +While the mappings file and ignore list paths are passed in to the ``FastSpeech2Dataset`` directly, the +``FastSpeech2Dataset`` infers the path to the supplementary files corresponding to each audio sample based on the +audio samples' paths found in the manifest. +For each sample, the paths to the corresponding duration, pitch, and energy files are inferred by replacing ``wavs/`` +with ``phoneme_durations/``, ``pitches/``, and ``energies``, and swapping out the file extension (``.wav``) with +``.pt``, ``.npy``, and ``.npy`` respectively. + +For example, given manifest audio path ``/data/LJSpeech/wavs/LJ001-0001.wav``, the inferred duration and phonemes file +path would be ``/data/LJSpeech/phoneme_durations/LJ001-0001.pt``. + +Your directory structure should therefore look something like this: + +.. code:: + + data/ + ├── manifest.json + ├── mappings.json + ├── ignore_file.pkl + ├── energies/ + │   ├── basename_1.npy + │   ├── basename_2.npy + │   ├── ... + │   └── basename_n.npy + ├── phoneme_durations/ + │   ├── basename_1.pt + │   ├── basename_2.pt + │   ├── ... + │   └── basename_n.pt + ├── pitches/ + │   ├── basename_1.npy + │   ├── basename_2.npy + │   ├── ... + │   └── basename_n.npy + └── wavs/ +    ├── basename_1.wav +    ├── basename_2.wav +    ├── ... +    └── basename_n.wav + + +LJ Speech Dataset +^^^^^^^^^^^^^^^^^ + +NeMo comes with scripts for preprocessing the LJ Speech dataset for use with FastSpeech 2, which can be found in the +``/scripts/dataset_processing/ljspeech/`` directory. +These scripts assume that you have downloaded the LJ Speech dataset and extracted it to ````. + +They perform the following dataset preparation steps: + +* Create manifest files and ``.txt`` files with normalized transcripts +* Extract pitches +* Extract energies +* Downloads the `CMU Pronouncing Dictionary `_ to convert normalized + transcripts to phonemes +* Uses the `Montreal Forced Aligner `_ (MFA) to + perform alignment +* Calculates durations based on the alignment +* Generates the mappings file and ignore file for the dataset + +To run the scripts, follow the steps below. + +#. Download and extract the `LJ Speech dataset `_ to ````. + +#. Create the manifest files and normalized text files (for MFA to discover later) by running: + + .. code-block:: bash + + python create_manifests_and_textfiles.py --ljspeech_base= + +#. Extract pitches and energies from the audio files: + + .. code-block:: bash + + python extract_ljspeech_energy_pitch.py --ljspeech_base= + + This will write the extracted pitches and energies to ``/pitches/`` and + ``/energies/``. + +#. Run the phoneme extraction and duration calculation script. + This script will set up a Conda environment to download and install MFA and its dependencies, so make sure that you + have Anaconda or Miniconda before running the script. + + Also note that MFA does sometimes have trouble finding OpenBlas, so you may have to manually install it with the + command ``sudo apt-get install libopenblas-dev``. + + .. code-block:: bash + + ./extract_ljspeech_phonemes_and_durs.sh + + This script takes additional options ``--skip_env_setup`` if you have already set up the ``aligner`` environment, + and ``--g2p_dict=`` if you have already downloaded CMU dict or another G2P dictionary you prefer. + + The alignment step will take a while to run, so be prepared to wait upwards of an hour. + + In addition to alignments and durations, this script will create the mappings and ignore files as well. + It will also generate some intermediate files that you are free to delete, as you only need the files listed above. + +When the following steps are finished, your ```` directory will look like this. +(Starred files and directories indicate files from intermediate steps that are safe to remove, but that you may want +to keep for bookkeeping purposes.) + +.. code:: + + / + ├── alignments/ * + ├── cmudict.dict * + ├── energies/ + │   ├── LJ001-0001.npy + │   └── ... + ├── ljs_audio_text_test_filelist.txt * + ├── ljs_audio_text_train_filelist.txt * + ├── ljs_audio_text_val_filelist.txt * + ├── ljspeech_test.json + ├── ljspeech_train.json + ├── ljspeech_val.json + ├── mappings.json + ├── metadata.csv + ├── phoneme_durations/ + │   ├── LJ001-0001.pt + │   └── ... + ├── pitches/ + │   ├── LJ001-0001.npy + │   └── ... + ├── README + ├── uncommented_cmudict.dict * + ├── wavs/ + │   ├── LJ001-0001.wav + │   └── ... + └── wavs_to_ignore.pkl diff --git a/models/tacotron2/NeMo/docs/source/tts/intro.rst b/models/tacotron2/NeMo/docs/source/tts/intro.rst new file mode 100644 index 0000000000..7aeb5e1e38 --- /dev/null +++ b/models/tacotron2/NeMo/docs/source/tts/intro.rst @@ -0,0 +1,153 @@ +Speech Synthesis (TTS) +====================== + +.. toctree:: + :maxdepth: 4 + :hidden: + + datasets + api + +Speech Synthesis or Text-to-Speech (TTS) involves turning text into human speech. The NeMo TTS collection currently +supports two pipelines for TTS: + +1) The two stage pipeline. First, a model is used to generate a mel spectrogram from text. Second, a model is used +to generate audio from a mel spectrogram. +2) The "end-to-end" approach that uses one model to generate audio straight from text. + +Quick Start: + +.. code-block:: python + + import soundfile as sf + from nemo.collections.tts.models.base import SpectrogramGenerator, Vocoder + + # Download and load the pretrained tacotron2 model + spec_generator = SpectrogramGenerator.from_pretrained("tts_en_tacotron2") + # Download and load the pretrained waveglow model + vocoder = Vocoder.from_pretrained("tts_waveglow_88m") + + # All spectrogram generators start by parsing raw strings to a tokenized version of the string + parsed = spec_generator.parse("You can type your sentence here to get nemo to produce speech.") + # They then take the tokenized string and produce a spectrogram + spectrogram = spec_generator.generate_spectrogram(tokens=parsed) + # Finally, a vocoder converts the spectrogram to audio + audio = vocoder.convert_spectrogram_to_audio(spec=spectrogram) + + # Save the audio to disk in a file called speech.wav + # Note vocoder return a batch of audio. In this example, we just take the first and only sample. + sf.write("speech.wav", audio.to('cpu').detach().numpy()[0], 22050) + +.. note:: + + For an interactive version of the quick start above, refer to the TTS inference notebook that can be found on the + github readme. + +Available Models +################ + +NeMo supports a variety of models that can be used for TTS. For beginners, we recommend starting with the Tacotron2 + +WaveGlow combination. Then we suggest trying using (TalkNet or FastPitch) + HiFiGAN if you want to continue exploring +the two stage pipeline, or the FastPitch_HifiGan_E2E model for the end-to-end pipeline. + +.. list-table:: *TTS Models* + :widths: 5 5 10 25 + :header-rows: 1 + + * - Model + - Base Class + - Pretrained Checkpoint + - Description + * - Tacotron2 + - :class:`SpectrogramGenerator` + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_en_tacotron2 + - LSTM encoder decoder based model that generates spectrograms + * - GlowTTS + - :class:`SpectrogramGenerator` + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_en_glowtts + - Glow-based spectrogram generator + * - FastSpeech 2 + - :class:`SpectrogramGenerator` + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_en_fastspeech_2 + - Non-autoregressive transformer-based spectrogram generator that predicts duration, energy, and pitch + * - FastPitch + - :class:`SpectrogramGenerator` + - (Coming soon) + - Non-autoregressive transformer-based spectrogram generator that predicts duration and pitch + * - TalkNet + - :class:`SpectrogramGenerator` + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_en_talknet + - Non-autoregressive convolution-based spectrogram generator + * - WaveGlow + - :class:`Vocoder` + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_waveglow_88m + - Glow-based vocoder + * - SqueezeWave + - :class:`Vocoder` + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_squeezewave + - Glow-based vocoder based on WaveGlow but with fewer parameters + * - UniGlow + - :class:`Vocoder` + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_uniglow + - Glow-based vocoder based on WaveGlow but shares 1 set of parameters across all flow steps + * - MelGAN + - :class:`Vocoder` + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_melgan + - GAN-based vocoder + * - HiFiGAN + - :class:`Vocoder` + - https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_hifigan + - GAN-based vocoder + * - FastPitch_HifiGan_E2E + - :class:`TextToWaveform` + - (Coming soon) + - End-to-end model based on composing FastPitch and HiFiGAN + * - FastSpeech2_HifiGan_E2E + - :class:`TextToWaveform` + - (Coming soon) + - End-to-end model based on composing FastSpeech2 and HiFiGAN + +Base Classes +############ + +Two Stage Pipeline +****************** + +NeMo TTS has two base classes corresponding to the two stage pipeline: + + - :class:`SpectrogramGenerator` + - :class:`Vocoder` + +The :class:`SpectrogramGenerator` class has two important +functions: :py:meth:`parse` which +accepts raw python strings and returns a torch.tensor that represents tokenized text ready to pass to, and +:py:meth:`generate_spectrogram` which +accepts a batch of tokenized text and returns a torch.tensor that represents a batch of spectrograms + +The :class:`Vocoder` class has one important functions +:py:meth:`convert_spectrogram_to_audio` which +accepts a batch of spectrograms and returns a torch.tensor that represents a batch of raw audio. + +End-to-End Pipeline +******************* + +Correspondingly, NeMo TTS has one base class for the end-to-end pipeline: + + - :class:`TextToWaveform` + +Similarly to the SpectrogramGenerator, :class:`TextToWaveform` +implements two functions: :py:meth:`parse` which +accepts raw python strings and returns a torch.tensor that represents tokenized text ready to pass to, and +:py:meth:`generate_spectrogram` which +accepts a batch of tokenized text and returns a torch.tensor that represents a batch of audio. + +TTS Training +############ + +Training of TTS models can be done using the scripts inside the NeMo ``examples/tts folders``. The majority of the TTS +YAML configurations should work out of the box with the LJSpeech dataset. If you want to train on other data, it is +recommended that you walk through the Tacotron 2 Training notebook. Please pay special attention to the sample rate and +FFT parameters for new data. + +Some models, like FastSpeech 2, require supplementary data such as phoneme durations, pitches, and energies. +For more information about how to preprocess datasets for such models, see the `TTS Datasets <./datasets.html>`__ page. diff --git a/models/tacotron2/NeMo/docs/update_docs_docker.sh b/models/tacotron2/NeMo/docs/update_docs_docker.sh new file mode 100755 index 0000000000..c7a45f8705 --- /dev/null +++ b/models/tacotron2/NeMo/docs/update_docs_docker.sh @@ -0,0 +1,3 @@ +cd ../ +docker run --rm -v $PWD:/workspace python:3.7 /bin/bash -c "cd /workspace && \ +pip install -r requirements/requirements_docs.txt && cd docs/ && rm -rf build && make clean && make html && make html" diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/Dockerfile b/models/tacotron2/NeMo/examples/asr/asr_webapp/Dockerfile new file mode 100644 index 0000000000..8e5ee21a7f --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/Dockerfile @@ -0,0 +1,41 @@ +# syntax=docker/dockerfile:experimental + +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE=nvcr.io/nvidia/nemo:1.0.0rc1 + +# build an image that includes only the nemo dependencies, ensures that dependencies +# are included first for optimal caching, and useful for building a development +# image (by specifying build target as `nemo-deps`) +FROM ${BASE_IMAGE} as nemo-deps + +# copy asr service source into a temp directory +WORKDIR /tmp/nemo-service +COPY . . + +# override nemo installation with dependency from requirements.txt +RUN /bin/echo "export BASE_IMAGE=${BASE_IMAGE}" >> /root/.bashrc +RUN cd /tmp/nemo-service && pip install -r "requirements.txt" + +# copy webapp into container for end user +WORKDIR /workspace/nemo-service +COPY . /workspace/nemo-service/ + +# pre-import some asr dependencies which would take time on each worker +RUN python -c "import nemo.collections.asr as nemo_asr" + +# expose port 8000 and launch gunicorn +EXPOSE 8000 +CMD ["gunicorn", "wsgi:app"] diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/README.rst b/models/tacotron2/NeMo/examples/asr/asr_webapp/README.rst new file mode 100644 index 0000000000..1f05b5f27e --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/README.rst @@ -0,0 +1,69 @@ +**ASR EVALUATION SERVICE ** +=========================== + +Usage +----- + +A simple service that will load a pre-trained model from NeMo's ASR collection and after uploading some files, will transcribe them. Any pre-trained model (or model which has been uploaded to the service) can be selected and if the local / docker environment supports GPUs then it will be possible to transcribe the audio segments with GPU. + +All uploaded files are immediately deleted after transcriptions are obtained. + + .. note:: + + When using `Gunicorn `_, you might notice that each pretrained checkpoint takes a long time for the first transcription. This is because that worker is instantiating the model into memory and then moving the model onto the GPU. For large models, this step might take a significant amount of time. However, this step is cached, and subsequent transcription requests should be much faster (especially if a GPU is utilized). + +From inside the ``asr_webapp`` directory, there are three options to run the ASR Service to transcribe audio - + +Local Setup +----------- + +1) Install all the dependencies (``pip install -r requirements.txt``). Note : It will utilize the latest branch set in the requirements and will override current nemo installation. + +2) Simply run ``python asr_service.py``. This will launch a single service worker that can be run locally. + +3) The app will run at ``127.0.0.1:5000`` by default. + +Gunicorn Setup +-------------- + +1) Follow Step (1) of the ``Local Setup`` section. + +2) Edit the configuration of ``gunicorn`` inside ``gunicorn.conf.py`` if you wish to change the port or number of workers (though this can be achieved via command line overrides as well). + +3) Simply run ``gunicorn wsgi:app``. This will launch two workers (default) in a gunicorn environment. + +4) The app will run at ``0.0.0.0:8000`` by default. + +Docker Setup +------------ + +The cleanest approach of the three, and requires simply building and running a docker container. + +1) Build the docker by executing ``bash docker_container_build.sh``. This will build a container using the latest branch of nemo. + +2) Run the container by executing ``bash docker_container_run.sh``. This will run a detached container that can be used by visiting ``0.0.0.0:8000`` on a modern browser. + +Note About Uploading Models +--------------------------- + +Uploading models is a useful method to evaluate models quickly without having to write scripts. +By design, models uploaded via the browser are visible to **all** users who have access to the web app - so that there only needs to be one user who uploads the model and everyone else who can access this service will be able to evaluate with that model. + +This also means that models are uploaded indefinitely - and cannot be removed easily without explicitly deleting the ``models`` directory or by shutting down the container (if a container is used). + +There are several reasons this approach was taken - + +* It allows for easy evaluation amongst peers without duplicating model uploads (which may be very big). +* When using ``gunicorn``, every worker has access to a central cache of models which can be loaded into memory. +* Model upload mode is meant to be used in a local, protected environment. + +If you wish to disable model uploading, please open ``templates/main.html`` and comment out the following section: + +.. code-block:: html + + + +
+ NeMo File : + + diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/asr_service.py b/models/tacotron2/NeMo/examples/asr/asr_webapp/asr_service.py new file mode 100644 index 0000000000..1c53d1593c --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/asr_service.py @@ -0,0 +1,374 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit +import glob +import os +import shutil +import time +from html import unescape +from uuid import uuid4 + +import model_api +import torch +import werkzeug +from flask import Flask, make_response, render_template, request, url_for +from flask_cors import CORS +from werkzeug.utils import secure_filename + +from nemo.utils import logging + +app = Flask(__name__) +CORS(app) + +# Upload folder for audio files; models are stored in permanent cache +# which gets deleted once the container shuts down +app.config[f'UPLOAD_FOLDER'] = f"tmp/" + + +@app.route('/initialize_model', methods=['POST']) +def initialize_model(): + """ + API Endpoint to instantiate a model + + Loads ASR model by its pretrained checkpoint name or upload ASR model that is provided by the user, + then load that checkpoint into the cache. + + Loading of the model into cache is done once per worker. Number of workers should be limited + so as not to exhaust the GPU memory available on device (if GPU is being used). + """ + logging.info("Starting ASR service") + if torch.cuda.is_available(): + logging.info("CUDA is available. Running on GPU") + else: + logging.info("CUDA is not available. Defaulting to CPUs") + + # get form fields + model_name = request.form['model_names_select'] + use_gpu_if_available = request.form.get('use_gpu_ckbx', "off") + + # get nemo model from user (if not none) + nemo_model_file = request.files.get('nemo_model', '') + + # if nemo model is not None, upload it to model cache + if nemo_model_file != '': + model_name = _store_model(nemo_model_file) + + # Alert user that model has been uploaded into the model cache, + # and they should refresh the page to access the model + result = render_template( + 'toast_msg.html', toast_message=f"Model {model_name} has been uploaded. " f"Refresh page !", timeout=5000 + ) + + else: + # Alert user that model has been loaded onto a workers memory + result = render_template( + 'toast_msg.html', toast_message=f"Model {model_name} has been initialized !", timeout=2000 + ) + + # Load model into memory cache + model_api.initialize_model(model_name=model_name) + + # reset file banner + reset_nemo_model_file_script = """ + + """ + + result = result + reset_nemo_model_file_script + result = make_response(result) + + # set cookies + result.set_cookie("model_name", model_name) + result.set_cookie("use_gpu", use_gpu_if_available) + return result + + +def _store_model(nemo_model_file): + """ + Preserve the model supplied by user into permanent cache + This cache needs to be manually deleted (if run locally), or gets deleted automatically + (when the container gets shutdown / killed). + + Args: + nemo_model_file: User path to .nemo checkpoint. + + Returns: + A file name (with a .nemo) at the end - to signify this is an uploaded checkpoint. + """ + filename = secure_filename(nemo_model_file.filename) + file_basename = os.path.basename(filename) + model_dir = os.path.splitext(file_basename)[0] + + model_store = os.path.join('models', model_dir) + if not os.path.exists(model_store): + os.makedirs(model_store) + + # upload model + model_path = os.path.join(model_store, filename) + nemo_model_file.save(model_path) + return file_basename + + +@app.route('/upload_audio_files', methods=['POST']) +def upload_audio_files(): + """ + API Endpoint to upload audio files for inference. + + The uploaded files must be wav files, 16 KHz sample rate, mono-channel audio samples. + """ + # Try to get one or more files from form + try: + f = request.files.getlist('file') + except werkzeug.exceptions.BadRequestKeyError: + f = None + + # If user did not select any file to upload, notify them. + if f is None or len(f) == 0: + toast = render_template('toast_msg.html', toast_message="No file has been selected to upload !", timeout=2000) + result = render_template('updates/upload_files_failed.html', pre_exec=toast, url=url_for('upload_audio_files')) + result = unescape(result) + return result + + # temporary id to store data + uuid = str(uuid4()) + data_store = os.path.join(app.config[f'UPLOAD_FOLDER'], uuid) + + # If the user attempt to upload another set of files without first transcribing them, + # delete the old cache of files and create a new cache entirely + _remove_older_files_if_exists() + + # Save each file into this unique cache + for fn in f: + filename = secure_filename(fn.filename) + if not os.path.exists(data_store): + os.makedirs(data_store) + + fn.save(os.path.join(data_store, filename)) + logging.info(f"Saving file : {fn.filename}") + + # Update user that N files were uploaded. + msg = f"{len(f)} file(s) uploaded. Click to upload more !" + toast = render_template('toast_msg.html', toast_message=f"{len(f)} file(s) uploaded !", timeout=2000) + result = render_template( + 'updates/upload_files_successful.html', pre_exec=toast, msg=msg, url=url_for('upload_audio_files') + ) + result = unescape(result) + + result = make_response(result) + result.set_cookie("uuid", uuid) + return result + + +def _remove_older_files_if_exists(): + """ + Helper method to prevent cache leakage when user attempts to upload another set of files + without first transcribing the files already uploaded. + """ + # remove old data store (if exists) + old_uuid = request.cookies.get('uuid', '') + if old_uuid is not None and old_uuid != '': + # delete old data store + old_data_store = os.path.join(app.config[f'UPLOAD_FOLDER'], old_uuid) + + logging.info("Tried uploading more data without using old uploaded data. Purging data cache.") + shutil.rmtree(old_data_store, ignore_errors=True) + + +@app.route('/remove_audio_files', methods=['POST']) +def remove_audio_files(): + """ + API Endpoint for removing audio files + + # Note: Sometimes data may persist due to set of circumstances: + + - User uploads audio then closes app without transcribing anything + + In such a case, the files will be deleted when gunicorn shutsdown, or container is stopped. + However the data may not be automatically deleted if the flast server is used as is. + """ + # Get the unique cache id from cookie + uuid = request.cookies.get("uuid", "") + data_store = os.path.join(app.config[f'UPLOAD_FOLDER'], uuid) + + # If the data does not exist (cache is empty), notify user + if not os.path.exists(data_store) or uuid == "": + files_dont_exist = render_template( + 'toast_msg.html', toast_message="No files have been uploaded !", timeout=2000 + ) + result = render_template( + 'updates/remove_files.html', pre_exec=files_dont_exist, url=url_for('remove_audio_files') + ) + result = unescape(result) + return result + + else: + # delete data that exists in cache + shutil.rmtree(data_store, ignore_errors=True) + + logging.info("Removed all data") + + # Notify user that cache was deleted. + toast = render_template('toast_msg.html', toast_message="All files removed !", timeout=2000) + result = render_template('updates/remove_files.html', pre_exec=toast, url=url_for('remove_audio_files')) + result = unescape(result) + + result = make_response(result) + result.set_cookie("uuid", '', expires=0) + return result + + +@app.route('/transcribe', methods=['POST']) +def transcribe(): + """ + API Endpoint to transcribe a set of audio files. + + The files are sorted according to their name, so order may not be same as upload order. + + Utilizing the cached info inside the cookies, a model with selected name will be loaded into memory, + and maybe onto a GPU (if it is supported on the device). + + Then the transcription api will be called from the model_api. If all is successful, a template is updated + with results. If some issue occurs (memory ran out, file is invalid format), notify the user. + """ + # load model name from cookie + model_name = request.cookies.get('model_name') + logging.info(f"Model name : {model_name}") + + # If model name is not selected via Load Model, notify user. + if model_name is None or model_name == '': + result = render_template('toast_msg.html', toast_message="Model has not been initialized !", timeout=2000) + return result + + # load whether gpu should be used + use_gpu_if_available = request.cookies.get('use_gpu') == 'on' + gpu_used = torch.cuda.is_available() and use_gpu_if_available + + # Load audio from paths + uuid = request.cookies.get("uuid", "") + data_store = os.path.join(app.config[f'UPLOAD_FOLDER'], uuid) + + files = list(glob.glob(os.path.join(data_store, "*.wav"))) + + # If no files found in cache, notify user + if len(files) == 0: + result = render_template('toast_msg.html', toast_message="No audio files were found !", timeout=2000) + return result + + # transcribe file via model api + t1 = time.time() + transcriptions = model_api.transcribe_all(files, model_name, use_gpu_if_available=use_gpu_if_available) + t2 = time.time() + + # delete all transcribed files immediately + for fp in files: + try: + os.remove(fp) + except FileNotFoundError: + logging.info(f"Failed to delete transcribed file : {os.path.basename(fp)}") + + # delete temporary transcription directory + shutil.rmtree(data_store, ignore_errors=True) + + # If something happened during transcription, and it failed, notify user. + if type(transcriptions) == str and transcriptions == model_api.TAG_ERROR_DURING_TRANSCRIPTION: + toast = render_template( + 'toast_msg.html', + toast_message=f"Failed to transcribe files due to unknown reason. " + f"Please provide 16 KHz Monochannel wav files onle.", + timeout=5000, + ) + transcriptions = ["" for _ in range(len(files))] + + else: + # Transcriptions obtained successfully, notify user. + toast = render_template( + 'toast_msg.html', + toast_message=f"Transcribed {len(files)} files using {model_name} (gpu={gpu_used}), " + f"in {(t2 - t1): 0.2f} s", + timeout=5000, + ) + + # Write results to data table + results = [] + for filename, transcript in zip(files, transcriptions): + results.append(dict(filename=os.path.basename(filename), transcription=transcript)) + + result = render_template('transcripts.html', transcripts=results) + result = toast + result + result = unescape(result) + + result = make_response(result) + result.set_cookie("uuid", "", expires=0) + return result + + +def remove_tmp_dir_at_exit(): + """ + Helper method to attempt a deletion of audio file cache on flask api exit. + Gunicorn and Docker container (based on gunicorn) will delete any remaining files on + shutdown of the gunicorn server or the docker container. + + This is a patch that might not always work for Flask server, but in general should ensure + that local audio file cache is deleted. + + This does *not* impact the model cache. Flask and Gunicorn servers will *never* delete uploaded models. + Docker container will delete models *only* when the container is killed (since models are uploaded to + local storage path inside container). + """ + try: + uuid = request.cookies.get("uuid", "") + + if uuid is not None or uuid != "": + cache_dir = os.path.join(os.path.join(app.config[f'UPLOAD_FOLDER'], uuid)) + logging.info(f"Removing cache file for worker : {os.getpid()}") + + if os.path.exists(cache_dir): + shutil.rmtree(cache_dir, ignore_errors=True) + logging.info(f"Deleted tmp folder : {cache_dir}") + + except RuntimeError: + # Working outside of request context (probably shutdown) + # simply delete entire tmp folder + shutil.rmtree(app.config[f'UPLOAD_FOLDER'], ignore_errors=True) + + +@app.route('/') +def main(): + """ + API Endpoint for ASR Service. + """ + nemo_model_names, local_model_names = model_api.get_model_names() + model_names = [] + model_names.extend(local_model_names) # prioritize local models + model_names.extend(nemo_model_names) # attach all other pretrained models + + # page initializations + result = render_template('main.html', model_names=model_names) + result = make_response(result) + + # Reset cookies + result.set_cookie("model_name", '', expires=0) # model name from pretrained model list + result.set_cookie("use_gpu", '', expires=0) # flag to use gpu (if available) + result.set_cookie("uuid", '', expires=0) # session id + return result + + +# Register hook to delete file cache (for flask server only) +atexit.register(remove_tmp_dir_at_exit) + + +if __name__ == '__main__': + app.run(debug=True) diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/docker_container_build.sh b/models/tacotron2/NeMo/examples/asr/asr_webapp/docker_container_build.sh new file mode 100644 index 0000000000..b94987e710 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/docker_container_build.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# build container +DOCKER_BUILDKIT=1 docker build -f Dockerfile --progress=plain -t nemo-asr-service:latest . + diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/docker_container_run.sh b/models/tacotron2/NeMo/examples/asr/asr_webapp/docker_container_run.sh new file mode 100644 index 0000000000..6ea348c47f --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/docker_container_run.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# run container +docker run -d --gpus=all -p 8000:8000 --shm-size=8g --ulimit memlock=-1 --ulimit \ + stack=67108864 nemo-asr-service:latest + diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/gunicorn.conf.py b/models/tacotron2/NeMo/examples/asr/asr_webapp/gunicorn.conf.py new file mode 100644 index 0000000000..1fc0e361d4 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/gunicorn.conf.py @@ -0,0 +1,36 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://docs.gunicorn.org/en/stable/settings.html + +# NOTE: Do not import nemo / torch code here +# Gunicorn creates forked processes - and CUDA cannot be used in forked multiprocessing environment. + +import shutil + +# General config +bind = "0.0.0.0:8000" +workers = 2 + +# Worker specific config +worker_connections = 1000 +timeout = 180 # 3 minutes of timeout + + +def on_exit(server): + # delete tmp dir + print("Shutting down server ...") + print("Deleteing tmp directory ...") + shutil.rmtree('tmp/', ignore_errors=True) + print("Tmp directory deleted !") diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/model_api.py b/models/tacotron2/NeMo/examples/asr/asr_webapp/model_api.py new file mode 100644 index 0000000000..e14bcb3a8a --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/model_api.py @@ -0,0 +1,131 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import glob +import os + +import torch + +import nemo.collections.asr as nemo_asr +from nemo.utils import logging, model_utils + +# setup AMP (optional) +if torch.cuda.is_available() and hasattr(torch.cuda, 'amp') and hasattr(torch.cuda.amp, 'autocast'): + logging.info("AMP enabled!\n") + autocast = torch.cuda.amp.autocast +else: + + @contextlib.contextmanager + def autocast(): + yield + + +MODEL_CACHE = {} + +# Special tags for fallbacks / user notifications +TAG_ERROR_DURING_TRANSCRIPTION = "" + + +def get_model_names(): + # Populate local copy of models + local_model_paths = glob.glob(os.path.join('models', "**", "*.nemo"), recursive=True) + local_model_names = list(sorted([os.path.basename(path) for path in local_model_paths])) + + # Populate with pretrained nemo checkpoint list + nemo_model_names = set() + for model_info in nemo_asr.models.ASRModel.list_available_models(): + for superclass in model_info.class_.mro(): + if 'CTC' in superclass.__name__ or 'RNNT' in superclass.__name__: + nemo_model_names.add(model_info.pretrained_model_name) + break + nemo_model_names = list(sorted(nemo_model_names)) + return nemo_model_names, local_model_names + + +def initialize_model(model_name): + # load model + if model_name not in MODEL_CACHE: + if '.nemo' in model_name: + # use local model + model_name_no_ext = os.path.splitext(model_name)[0] + model_path = os.path.join('models', model_name_no_ext, model_name) + + # Extract config + model_cfg = nemo_asr.models.ASRModel.restore_from(restore_path=model_path, return_config=True) + classpath = model_cfg.target # original class path + imported_class = model_utils.import_class_by_path(classpath) # type: ASRModel + logging.info(f"Restoring local model : {imported_class.__name__}") + + # load model from checkpoint + model = imported_class.restore_from(restore_path=model_path, map_location='cpu') # type: ASRModel + + else: + # use pretrained model + model = nemo_asr.models.ASRModel.from_pretrained(model_name, map_location='cpu') + + model.freeze() + + # cache model + MODEL_CACHE[model_name] = model + + model = MODEL_CACHE[model_name] + return model + + +def transcribe_all(filepaths, model_name, use_gpu_if_available=True): + # instantiate model + if model_name in MODEL_CACHE: + model = MODEL_CACHE[model_name] + else: + model = initialize_model(model_name) + + if torch.cuda.is_available() and use_gpu_if_available: + model = model.cuda() + + # transcribe audio + logging.info("Begin transcribing audio...") + try: + with autocast(): + with torch.no_grad(): + transcriptions = model.transcribe(filepaths, batch_size=32) + + except RuntimeError: + # Purge the cache to clear some memory + MODEL_CACHE.clear() + + logging.info("Ran out of memory on device - performing inference on CPU for now") + + try: + model = model.cpu() + + with torch.no_grad(): + transcriptions = model.transcribe(filepaths, batch_size=32) + + except Exception as e: + logging.info(f"Exception {e} occured while attemting to transcribe audio. Returning error message") + return TAG_ERROR_DURING_TRANSCRIPTION + + logging.info(f"Finished transcribing {len(filepaths)} files !") + + # If RNNT models transcribe, they return a tuple (greedy, beam_scores) + if type(transcriptions[0]) == list and len(transcriptions) == 2: + # get greedy transcriptions only + transcriptions = transcriptions[0] + + # Force onto CPU + model = model.cpu() + MODEL_CACHE[model_name] = model + + return transcriptions diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/requirements.txt b/models/tacotron2/NeMo/examples/asr/asr_webapp/requirements.txt new file mode 100644 index 0000000000..b08f825d02 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/requirements.txt @@ -0,0 +1,4 @@ +-e git+git://github.com/NVIDIA/NeMo.git@v1.0.2#egg=nemo_toolkit[all] +flask +flask-cors +gunicorn diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/main.html b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/main.html new file mode 100644 index 0000000000..b6bafcb2da --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/main.html @@ -0,0 +1,150 @@ + + + + + + + + Automatic Speech Recognition + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/toast_msg.html b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/toast_msg.html new file mode 100644 index 0000000000..794d0e7a4a --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/toast_msg.html @@ -0,0 +1,28 @@ + + + \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/transcripts.html b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/transcripts.html new file mode 100644 index 0000000000..5da42c7a22 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/transcripts.html @@ -0,0 +1,43 @@ + + + + + + + + + + + {% for transcript in transcripts %} + + + + + {% endfor %} + +
FilenameTranscript
+
+ {{ transcript.filename }} +
+ 		+
+ +
+
\ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/remove_files.html b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/remove_files.html new file mode 100644 index 0000000000..185f5d8b84 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/remove_files.html @@ -0,0 +1,22 @@ + + +{{ pre_exec }} + \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/upload_files_failed.html b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/upload_files_failed.html new file mode 100644 index 0000000000..61ad36ba33 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/upload_files_failed.html @@ -0,0 +1,22 @@ + + +{{ pre_exec }} + \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/upload_files_successful.html b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/upload_files_successful.html new file mode 100644 index 0000000000..c3d15da2d4 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/templates/updates/upload_files_successful.html @@ -0,0 +1,22 @@ + + +{{ pre_exec }} + \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/asr/asr_webapp/wsgi.py b/models/tacotron2/NeMo/examples/asr/asr_webapp/wsgi.py new file mode 100644 index 0000000000..64410091e4 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/asr_webapp/wsgi.py @@ -0,0 +1,18 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from asr_service import app + +if __name__ == '__main__': + app.run() diff --git a/models/tacotron2/NeMo/examples/asr/conf/citrinet/citrinet_384.yaml b/models/tacotron2/NeMo/examples/asr/conf/citrinet/citrinet_384.yaml new file mode 100644 index 0000000000..1ba7085208 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/citrinet/citrinet_384.yaml @@ -0,0 +1,428 @@ +# This config contains the default values for training a Citrinet model with CTC loss and BPE-based vocabulary. +# Default learning parameters in this config are set for effective batch size of 1k on 32 GPUs. +# To train it with smaller batch sizes, you may need to re-tune the learning parameters or use higher accumulate_grad_batches. +# If training for a short time, you can also reduce weight decay to 0. + +# Training Recipe +# This model can be trained using the default settings in this config with FP32 precision. +# When training under AMP, increase `warmup_steps` to 5000 for stable training. +# In order to create Citrinet-C, find-replace `filters: 384` with `filters: C`. +# When reducing the receptive field of these models, it is advised to reduce the amount of augmentation +# for larger models from 10x time masking to 5x or 2x time masking. +# For further details regarding Citrinet, visit - https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/configs.html#citrinet + +name: &name "Citrinet-384-8x-Stride" + +model: + sample_rate: &sample_rate 16000 + + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 32 + trim_silence: false + max_duration: 16.7 + shuffle: true + is_tarred: false + tarred_audio_filepaths: null + use_start_end_token: false + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 32 + shuffle: false + use_start_end_token: false + + test_ds: + manifest_filepath: null + sample_rate: 16000 + batch_size: 32 + shuffle: false + use_start_end_token: false + + model_defaults: + repeat: 5 + dropout: 0.0 + separable: true + se: true + se_context_size: -1 + kernel_size_factor: 1.0 + + tokenizer: + dir: ??? # path to directory which contains either tokenizer.model (bpe) or vocab.txt (for wpe) + type: ??? # Can be either bpe or wpe + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + sample_rate: *sample_rate + normalize: "per_feature" + window_size: 0.025 + window_stride: 0.01 + window: "hann" + features: &n_mels 80 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + pad_to: 16 + stft_conv: false + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + freq_masks: 2 + time_masks: 2 + freq_width: 27 + time_width: 0.05 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: 384 + repeat: 1 + kernel: [5] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [11] + stride: [2] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [13] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [15] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [17] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [19] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [21] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [13] + stride: [2] # *stride + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [15] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [17] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [19] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [21] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [23] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [25] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [25] + stride: [2] # stride + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [27] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [29] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [31] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [33] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [35] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [37] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 384 + repeat: ${model.model_defaults.repeat} + kernel: [39] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: &enc_final 640 + repeat: 1 + kernel: [41] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: *enc_final + num_classes: -1 # filled with vocabulary size from tokenizer at runtime + vocabulary: [] # filled with vocabulary from tokenizer at runtime + + optim: + name: novograd + lr: 0.05 + + # optimizer arguments + betas: [0.8, 0.25] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + + # scheduler config override + warmup_steps: 1000 + warmup_ratio: null + min_lr: 1e-5 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 100 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager + log_every_n_steps: 100 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + check_val_every_n_epoch: 1 + precision: 32 + sync_batchnorm: false + benchmark: false + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: true + create_checkpoint_callback: true + checkpoint_callback_params: + monitor: "val_wer" + mode: "min" + save_top_k: 3 + create_wandb_logger: false + wandb_logger_kwargs: + name: null + project: null + entity: null + resume_if_exists: false + resume_ignore_no_checkpoint: false + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/asr/conf/citrinet/citrinet_512.yaml b/models/tacotron2/NeMo/examples/asr/conf/citrinet/citrinet_512.yaml new file mode 100644 index 0000000000..2aab672008 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/citrinet/citrinet_512.yaml @@ -0,0 +1,428 @@ +# This config contains the default values for training a Citrinet model with CTC loss and BPE-based vocabulary. +# Default learning parameters in this config are set for effective batch size of 1k on 32 GPUs. +# To train it with smaller batch sizes, you may need to re-tune the learning parameters or use higher accumulate_grad_batches. +# If training for a short time, you can also reduce weight decay to 0. + +# Training Recipe +# This model can be trained using the default settings in this config with FP32 precision. +# When training under AMP, increase `warmup_steps` to 5000 for stable training. +# In order to create Citrinet-C, find-replace `filters: 384` with `filters: C`. +# When reducing the receptive field of these models, it is advised to reduce the amount of augmentation +# for larger models from 10x time masking to 5x or 2x time masking. +# For further details regarding Citrinet, visit - https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/configs.html#citrinet + +name: &name "Citrinet-512-8x-Stride" + +model: + sample_rate: &sample_rate 16000 + + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 32 + trim_silence: false + max_duration: 16.7 + shuffle: true + is_tarred: false + tarred_audio_filepaths: null + use_start_end_token: false + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 32 + shuffle: false + use_start_end_token: false + + test_ds: + manifest_filepath: null + sample_rate: 16000 + batch_size: 32 + shuffle: false + use_start_end_token: false + + model_defaults: + repeat: 5 + dropout: 0.1 + separable: true + se: true + se_context_size: -1 + kernel_size_factor: 1.0 + + tokenizer: + dir: ??? # path to directory which contains either tokenizer.model (bpe) or vocab.txt (for wpe) + type: ??? # Can be either bpe or wpe + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + sample_rate: *sample_rate + normalize: "per_feature" + window_size: 0.025 + window_stride: 0.01 + window: "hann" + features: &n_mels 80 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + pad_to: 16 + stft_conv: false + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + freq_masks: 2 + time_masks: 10 + freq_width: 27 + time_width: 0.05 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: 512 + repeat: 1 + kernel: [5] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [11] + stride: [2] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [13] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [15] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [17] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [19] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [21] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [13] + stride: [2] # *stride + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [15] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [17] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [19] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [21] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [23] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [25] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [25] + stride: [2] # stride + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [27] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [29] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [31] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [33] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [35] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [37] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: 512 + repeat: ${model.model_defaults.repeat} + kernel: [39] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + - filters: &enc_final 640 + repeat: 1 + kernel: [41] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + kernel_size_factor: ${model.model_defaults.kernel_size_factor} + + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: *enc_final + num_classes: -1 # filled with vocabulary size from tokenizer at runtime + vocabulary: [] # filled with vocabulary from tokenizer at runtime + + optim: + name: novograd + lr: 0.05 + + # optimizer arguments + betas: [0.8, 0.25] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + + # scheduler config override + warmup_steps: 1000 + warmup_ratio: null + min_lr: 1e-5 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 100 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager + log_every_n_steps: 100 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + check_val_every_n_epoch: 1 + precision: 32 + sync_batchnorm: false + benchmark: false + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: true + create_checkpoint_callback: true + checkpoint_callback_params: + monitor: "val_wer" + mode: "min" + save_top_k: 3 + create_wandb_logger: false + wandb_logger_kwargs: + name: null + project: null + entity: null + resume_if_exists: false + resume_ignore_no_checkpoint: false + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/asr/conf/citrinet/config_bpe.yaml b/models/tacotron2/NeMo/examples/asr/conf/citrinet/config_bpe.yaml new file mode 100644 index 0000000000..4b7a7bfb8e --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/citrinet/config_bpe.yaml @@ -0,0 +1,183 @@ +name: &name "ContextNet5x1" +sample_rate: &sample_rate 16000 +repeat: &repeat 1 +dropout: &dropout 0.0 +separable: &separable true + +model: + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 32 + trim_silence: True + max_duration: 16.7 + shuffle: True + is_tarred: False + tarred_audio_filepaths: null + shard_strategy: "scatter" + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 32 + shuffle: False + + tokenizer: + dir: ??? # path to directory which contains either tokenizer.model (bpe) or vocab.txt (for wpe) + type: ??? # Can be either bpe or wpe + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.02 + sample_rate: *sample_rate + window_stride: 0.01 + window: "hann" + features: &n_mels 64 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + rect_freq: 50 + rect_masks: 5 + rect_time: 120 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: 128 + repeat: 1 + kernel: [11] + stride: [1] + dilation: [1] + dropout: *dropout + residual: true + separable: *separable + se: true + se_context_size: -1 + + - filters: 256 + repeat: *repeat + kernel: [13] + stride: [1] + dilation: [1] + dropout: *dropout + residual: true + separable: *separable + se: true + se_context_size: -1 + + - filters: 256 + repeat: *repeat + kernel: [15] + stride: [1] + dilation: [1] + dropout: *dropout + residual: true + separable: *separable + se: true + se_context_size: -1 + + - filters: 256 + repeat: *repeat + kernel: [17] + stride: [1] + dilation: [1] + dropout: *dropout + residual: true + separable: *separable + se: true + se_context_size: -1 + + - filters: 256 + repeat: *repeat + kernel: [19] + stride: [1] + dilation: [1] + dropout: *dropout + residual: true + separable: *separable + se: true + se_context_size: -1 + + - filters: 256 + repeat: 1 + kernel: [21] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: *separable + se: true + se_context_size: -1 + + - filters: &enc_feat_out 1024 + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: *separable + se: true + se_context_size: -1 + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: 1024 + num_classes: -1 # filled with vocabulary size from tokenizer at runtime + vocabulary: [] # filled with vocabulary from tokenizer at runtime + + optim: + name: adam + # _target_: nemo.core.optim.optimizers.Adam + lr: .1 + + # optimizer arguments + betas: [0.9, 0.999] + weight_decay: 0.0001 + + # scheduler setup + sched: + name: CosineAnnealing + + # scheduler config override + warmup_steps: null + warmup_ratio: 0.05 + min_lr: 1e-6 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 5 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True + create_wandb_logger: False + wandb_logger_kwargs: + name: null + project: null + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/asr/conf/config.yaml b/models/tacotron2/NeMo/examples/asr/conf/config.yaml new file mode 100644 index 0000000000..f18a23fc38 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/config.yaml @@ -0,0 +1,183 @@ +name: &name "QuartzNet15x5" +sample_rate: &sample_rate 16000 +repeat: &repeat 1 +dropout: &dropout 0.0 +separable: &separable true +labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"] + +model: + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: *labels + batch_size: 32 + trim_silence: True + max_duration: 16.7 + shuffle: True + is_tarred: False + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: *labels + batch_size: 32 + shuffle: False + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.02 + sample_rate: *sample_rate + window_stride: 0.01 + window: "hann" + features: &n_mels 64 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + stft_conv: false + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + rect_freq: 50 + rect_masks: 5 + rect_time: 120 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: 128 + repeat: 1 + kernel: [11] + stride: [1] + dilation: [1] + dropout: *dropout + residual: true + separable: *separable + se: true + se_context_size: -1 + + - filters: 256 + repeat: *repeat + kernel: [13] + stride: [1] + dilation: [1] + dropout: *dropout + residual: true + separable: *separable + se: true + se_context_size: -1 + + - filters: 256 + repeat: *repeat + kernel: [15] + stride: [1] + dilation: [1] + dropout: *dropout + residual: true + separable: *separable + se: true + se_context_size: -1 + + - filters: 256 + repeat: *repeat + kernel: [17] + stride: [1] + dilation: [1] + dropout: *dropout + residual: true + separable: *separable + se: true + se_context_size: -1 + + - filters: 256 + repeat: *repeat + kernel: [19] + stride: [1] + dilation: [1] + dropout: *dropout + residual: true + separable: *separable + se: true + se_context_size: -1 + + - filters: 256 + repeat: 1 + kernel: [21] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: *separable + se: true + se_context_size: -1 + + - filters: &enc_feat_out 1024 + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: *separable + se: true + se_context_size: -1 + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: 1024 + num_classes: 28 + vocabulary: *labels + + optim: + name: novograd + # _target_: nemo.core.optim.optimizers.Novograd + lr: .01 + # optimizer arguments + betas: [0.8, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + + # pytorch lightning args + monitor: val_loss + reduce_on_plateau: false + + # Scheduler params + warmup_steps: null + warmup_ratio: null + min_lr: 0.0 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 5 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/asr/conf/conformer/conformer_ctc_bpe.yaml b/models/tacotron2/NeMo/examples/asr/conf/conformer/conformer_ctc_bpe.yaml new file mode 100644 index 0000000000..1560fc36ed --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/conformer/conformer_ctc_bpe.yaml @@ -0,0 +1,189 @@ +# It contains the default values for training a Conformer-CTC ASR model, large size (~120M) with CTC loss and sub-word encoding. + +# Architecture and training config: +# Default learning parameters in this config are set for effective batch size of 2K. To train it with smaller effective +# batch sizes, you may need to re-tune the learning parameters or use higher accumulate_grad_batches. +# Here are the recommended configs for different variants of Conformer-CTC, other parameters are the same as in this config file. +# One extra layer (compared to original paper) is added to the medium and large variants to compensate for replacing the LSTM decoder with a linear one. +# +# +-------------+---------+---------+----------+------------+-----+ +# | Model | d_model | n_heads | n_layers | time_masks | lr | +# +=============+=========+========+===========+============+=====+ +# | Small (13M)| 176 | 4 | 16 | 5 | 5.0 | +# +-------------+---------+--------+-----------+------------+-----+ +# | Medium (30M)| 256 | 4 | 18 | 5 | 5.0 | +# +-------------+---------+--------+-----------+------------+-----+ +# | Large (121M)| 512 | 8 | 18 | 10 | 2.0 | +# +---------------------------------------------------------------+ +# +# If you do not want to train with AMP, you may use weight decay of 0.0 or reduce the number of time maskings to 2 +# with time_width=100. It may help when you want to train for fewer epochs and need faster convergence. +# With weight_decay=0.0, learning rate may need to get reduced to 2.0. + +# You may find more info about Conformer-CTC here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/models.html#conformer-ctc +# Pre-trained models of Conformer-CTC can be found here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/results.html +# The checkpoint of the large model trained on LibriSpeech with this recipe can be found here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large_ls + +name: "Conformer-CTC-BPE" + +model: + sample_rate: 16000 + log_prediction: true # enables logging sample predictions in the output during training + ctc_reduction: 'mean_batch' + + train_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + batch_size: 16 # you may increase batch_size if your memory allows + shuffle: true + num_workers: 8 + pin_memory: true + use_start_end_token: false + trim_silence: false + max_duration: 16.7 # it is set for LibriSpeech, you may need to update it for your dataset + min_duration: 0.1 + + validation_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + batch_size: 16 # you may increase batch_size if your memory allows + shuffle: false + num_workers: 8 + pin_memory: true + use_start_end_token: false + + test_ds: + manifest_filepath: null + sample_rate: ${model.sample_rate} + batch_size: 16 # you may increase batch_size if your memory allows + shuffle: false + num_workers: 8 + pin_memory: true + use_start_end_token: false + + # recommend small vocab size of 128 or 256 when using 4x sub-sampling + # you may find more detail on how to train a tokenizer at: /scripts/tokenizers/process_asr_text_tokenizer.py + tokenizer: + dir: ??? # path to directory which contains either tokenizer.model (bpe) or vocab.txt (wpe) + type: wpe # Can be either bpe (SentencePiece tokenizer) or wpe (WordPiece tokenizer) + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + sample_rate: ${model.sample_rate} + normalize: "per_feature" + window_size: 0.025 + window_stride: 0.01 + window: "hann" + features: 80 + n_fft: 512 + log: true + frame_splicing: 1 + dither: 0.00001 + pad_to: 0 + pad_value: 0.0 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + freq_masks: 2 # set to zero to disable it + # you may use lower time_masks for smaller models to have a faster convergence + time_masks: 10 # set to zero to disable it + freq_width: 27 + time_width: 0.05 + + encoder: + _target_: nemo.collections.asr.modules.ConformerEncoder + feat_in: ${model.preprocessor.features} + feat_out: -1 # you may set it if you need different output size other than the default d_model + n_layers: 18 + d_model: 512 + + # Sub-sampling params + subsampling: striding # vggnet or striding, vggnet may give better results but needs more memory + subsampling_factor: 4 # must be power of 2 + subsampling_conv_channels: -1 # -1 sets it to d_model + + # Feed forward module's params + ff_expansion_factor: 4 + + # Multi-headed Attention Module's params + self_attention_model: rel_pos # rel_pos or abs_pos + n_heads: 8 # may need to be lower for smaller d_models + # [left, right] specifies the number of steps to be seen from left and right of each step in self-attention + att_context_size: [-1, -1] # -1 means unlimited context + xscaling: true # scales up the input embeddings by sqrt(d_model) + untie_biases: true # unties the biases of the TransformerXL layers + pos_emb_max_len: 5000 + + # Convolution module's params + conv_kernel_size: 31 + + ### regularization + dropout: 0.1 # The dropout used in most of the Conformer Modules + dropout_emb: 0.0 # The dropout used for embeddings + dropout_att: 0.1 # The dropout for multi-headed attention modules + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: null + num_classes: -1 + vocabulary: [] + + optim: + name: adamw + lr: 2.0 + # optimizer arguments + betas: [0.9, 0.98] + # less necessity for weight_decay as we already have large augmentations with SpecAug + # you may need weight_decay for large models, stable AMP training, small datasets, or when lower augmentations are used + # weight decay of 0.0 with lr of 2.0 also works fine + weight_decay: 1e-3 + + # scheduler setup + sched: + name: NoamAnnealing + d_model: ${model.encoder.d_model} + # scheduler config override + warmup_steps: 10000 + warmup_ratio: null + min_lr: 1e-6 + +trainer: + gpus: -1 # number of GPUs, -1 would use all available GPUs + num_nodes: 1 + max_epochs: 1000 + max_steps: null # computed at runtime if not set + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + accelerator: ddp + accumulate_grad_batches: 1 + gradient_clip_val: 0.0 + amp_level: O0 # O1/O2 for mixed precision + precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. + log_every_n_steps: 10 # Interval of logging. + progress_bar_refresh_rate: 10 + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs + sync_batchnorm: true + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: true + create_checkpoint_callback: true + checkpoint_callback_params: + # in case of multiple validation sets, first one is used + monitor: "val_wer" + mode: "min" + save_top_k: 3 + + # you need to set these two to True to continue the training + resume_if_exists: false + resume_ignore_no_checkpoint: false + + # You may use this section to create a W&B logger + create_wandb_logger: false + wandb_logger_kwargs: + name: null + project: null diff --git a/models/tacotron2/NeMo/examples/asr/conf/conformer/conformer_ctc_char.yaml b/models/tacotron2/NeMo/examples/asr/conf/conformer/conformer_ctc_char.yaml new file mode 100644 index 0000000000..735b075d6b --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/conformer/conformer_ctc_char.yaml @@ -0,0 +1,165 @@ +# It contains the default values for training a Conformer-CTC ASR model, large size (~120M) with CTC loss and char-based vocabulary. +# Char-based encoding may give lower accuracy than sub-word encoding (conformer_ctc_bpe.yaml). + +# You may find more detail on Conformer-CTC at `examples/asr/conf/conformer/conformer_ctc_bpe.yaml` + +name: "Conformer-CTC-Char" + +model: + sample_rate: 16000 + labels: [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"] + log_prediction: true # enables logging sample predictions in the output during training + ctc_reduction: 'mean_batch' + + train_ds: + manifest_filepath: ??? + labels: ${model.labels} + sample_rate: ${model.sample_rate} + batch_size: 16 # you may increase batch_size if your memory allows + shuffle: true + num_workers: 8 + pin_memory: true + trim_silence: false + max_duration: 16.7 # it is set for LibriSpeech, you may need to update it for your dataset + min_duration: 0.1 + + validation_ds: + manifest_filepath: ??? + labels: ${model.labels} + sample_rate: ${model.sample_rate} + batch_size: 16 # you may increase batch_size if your memory allows + shuffle: false + num_workers: 8 + pin_memory: true + + test_ds: + manifest_filepath: null + labels: ${model.labels} + sample_rate: ${model.sample_rate} + batch_size: 16 # you may increase batch_size if your memory allows + shuffle: false + num_workers: 8 + pin_memory: true + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + sample_rate: ${model.sample_rate} + normalize: "per_feature" + window_size: 0.025 + window_stride: 0.01 + window: "hann" + features: 80 + n_fft: 512 + log: true + frame_splicing: 1 + dither: 0.00001 + pad_to: 0 + pad_value: 0.0 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + freq_masks: 2 # set to zero to disable it + # you may use lower time_masks for smaller models to have a faster convergence + time_masks: 10 # set to zero to disable it + freq_width: 27 + time_width: 0.05 + + encoder: + _target_: nemo.collections.asr.modules.ConformerEncoder + feat_in: ${model.preprocessor.features} + feat_out: -1 # you may set it if you need different output size other than the default d_model + n_layers: 16 + d_model: 256 + + # Sub-sampling params + subsampling: striding # vggnet or striding, vggnet may give better results but needs more memory + subsampling_factor: 4 # must be power of 2 + subsampling_conv_channels: -1 # set to -1 to make it equal to the d_model + + # Feed forward module's params + ff_expansion_factor: 4 + + # Multi-headed Attention Module's params + self_attention_model: rel_pos # rel_pos or abs_pos + n_heads: 8 # may need to be lower for smaller d_models + # [left, right] specifies the number of steps to be seen from left and right of each step in self-attention + att_context_size: [-1, -1] # -1 means unlimited context + xscaling: true # scales up the input embeddings by sqrt(d_model) + untie_biases: true # unties the biases of the TransformerXL layers + pos_emb_max_len: 5000 + + # Convolution module's params + conv_kernel_size: 31 + + ### regularization + dropout: 0.1 # The dropout used in most of the Conformer Modules + dropout_emb: 0.0 # The dropout used for embeddings + dropout_att: 0.1 # The dropout for multi-headed attention modules + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: null + num_classes: -1 + vocabulary: ${model.labels} + + optim: + name: adamw + lr: 2.0 + # optimizer arguments + betas: [0.9, 0.98] + # less necessity for weight_decay as we already have large augmentations with SpecAug + # you may need weight_decay for large models, stable AMP training, small datasets, or when lower augmentations are used + # weight decay of 0.0 with lr of 2.0 also works fine + weight_decay: 1e-3 + + # scheduler setup + sched: + name: NoamAnnealing + d_model: ${model.encoder.d_model} + # scheduler config override + warmup_steps: 10000 + warmup_ratio: null + min_lr: 1e-6 + +trainer: + gpus: -1 # number of GPUs, -1 would use all available GPUs + num_nodes: 1 + max_epochs: 1000 + max_steps: null # computed at runtime if not set + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + accelerator: ddp + accumulate_grad_batches: 1 + gradient_clip_val: 0.0 + amp_level: O0 # O1/O2 for mixed precision + precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. + log_every_n_steps: 10 # Interval of logging. + progress_bar_refresh_rate: 10 + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs + sync_batchnorm: true + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager + + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: true + create_checkpoint_callback: true + checkpoint_callback_params: + # in case of multiple validation sets, first one is used + monitor: "val_wer" + mode: "min" + save_top_k: 3 + + # you need to set these two to True to continue the training + resume_if_exists: false + resume_ignore_no_checkpoint: false + + # You may use this section to create a W&B logger + create_wandb_logger: false + wandb_logger_kwargs: + name: null + project: null diff --git a/models/tacotron2/NeMo/examples/asr/conf/jasper/jasper_10x5dr.yaml b/models/tacotron2/NeMo/examples/asr/conf/jasper/jasper_10x5dr.yaml new file mode 100644 index 0000000000..2f5309aeed --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/jasper/jasper_10x5dr.yaml @@ -0,0 +1,208 @@ +name: &name "Jasper10x5" + +model: + sample_rate: &sample_rate 16000 + labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"] + + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: *labels + batch_size: 32 + trim_silence: True + max_duration: 16.7 + shuffle: True + is_tarred: False + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: *labels + batch_size: 32 + shuffle: False + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.02 + sample_rate: *sample_rate + window_stride: 0.01 + window: "hann" + features: &n_mels 64 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + rect_freq: 50 + rect_masks: 5 + rect_time: 120 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + jasper: + - dilation: [1] + dropout: 0.2 + filters: 256 + kernel: [11] + repeat: 1 + residual: false + stride: [2] + - dilation: [1] + dropout: 0.2 + filters: 256 + kernel: [11] + repeat: 5 + residual: true + residual_dense: true + stride: [1] + - dilation: [1] + dropout: 0.2 + filters: 256 + kernel: [11] + repeat: 5 + residual: true + residual_dense: true + stride: [1] + - dilation: [1] + dropout: 0.2 + filters: 384 + kernel: [13] + repeat: 5 + residual: true + residual_dense: true + stride: [1] + - dilation: [1] + dropout: 0.2 + filters: 384 + kernel: [13] + repeat: 5 + residual: true + residual_dense: true + stride: [1] + - dilation: [1] + dropout: 0.2 + filters: 512 + kernel: [17] + repeat: 5 + residual: true + residual_dense: true + stride: [1] + - dilation: [1] + dropout: 0.2 + filters: 512 + kernel: [17] + repeat: 5 + residual: true + residual_dense: true + stride: [1] + - dilation: [1] + dropout: 0.3 + filters: 640 + kernel: [21] + repeat: 5 + residual: true + residual_dense: true + stride: [1] + - dilation: [1] + dropout: 0.3 + filters: 640 + kernel: [21] + repeat: 5 + residual: true + residual_dense: true + stride: [1] + - dilation: [1] + dropout: 0.3 + filters: 768 + kernel: [25] + repeat: 5 + residual: true + residual_dense: true + stride: [1] + - dilation: [1] + dropout: 0.3 + filters: 768 + kernel: [25] + repeat: 5 + residual: true + residual_dense: true + stride: [1] + - dilation: [2] + dropout: 0.4 + filters: 896 + kernel: [29] + repeat: 1 + residual: false + stride: [1] + - dilation: [1] + dropout: 0.4 + filters: 1024 + kernel: [1] + repeat: 1 + residual: false + stride: [1] + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: 1024 + num_classes: 28 + vocabulary: *labels + + optim: + name: novograd + # _target_: nemo.core.optim.optimizers.Novograd + lr: .01 + # optimizer arguments + betas: [0.8, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + + # pytorch lightning args + # monitor: val_loss + # reduce_on_plateau: false + + # Scheduler params + warmup_steps: null + warmup_ratio: null + min_lr: 0.0 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 5 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True + create_wandb_logger: False + wandb_logger_kwargs: + name: null + project: null + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/asr/conf/marblenet/marblenet_3x2x64.yaml b/models/tacotron2/NeMo/examples/asr/conf/marblenet/marblenet_3x2x64.yaml new file mode 100644 index 0000000000..e3d34cb1f3 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/marblenet/marblenet_3x2x64.yaml @@ -0,0 +1,180 @@ +name: &name "MarbleNet-3x2x64" + +model: + sample_rate: 16000 + repeat: 2 + dropout: 0.0 + kernel_size_factor: 1.0 + + labels: ['background', 'speech'] + + train_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + labels: ${model.labels} + batch_size: 128 + shuffle: True + is_tarred: False + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + augmentor: + shift: + prob: 1.0 + min_shift_ms: -5.0 + max_shift_ms: 5.0 + white_noise: + prob: 1.0 + min_level: -90 + max_level: -46 + + validation_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + labels: ${model.labels} + batch_size: 128 + shuffle: False + val_loss_idx: 0 + + test_ds: + manifest_filepath: null + sample_rate: ${model.sample_rate} + labels: ${model.labels} + batch_size: 128 + shuffle: False + test_loss_idx: 0 + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMFCCPreprocessor + window_size: 0.025 + window_stride: 0.01 + window: "hann" + n_mels: &n_mels 64 + n_mfcc: *n_mels + n_fft: 512 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + freq_masks: 2 + time_masks: 2 + freq_width: 15 + time_width: 25 + rect_masks: 5 + rect_time: 25 + rect_freq: 15 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: 128 + repeat: 1 + kernel: [11] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: false + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 64 + repeat: ${model.repeat} + kernel: [13] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 64 + repeat: ${model.repeat} + kernel: [15] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 64 + repeat: ${model.repeat} + kernel: [17] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 128 + repeat: 1 + kernel: [29] + stride: [1] + dilation: [2] + dropout: ${model.dropout} + residual: false + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: &enc_final_filters 128 + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: false + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoderClassification + feat_in: *enc_final_filters + return_logits: true + pooling_type: 'avg' + + optim: + name: sgd + lr: 0.01 + # optimizer arguments + weight_decay: 0.001 + momentum: 0.9 + + # scheduler setup + sched: + name: PolynomialHoldDecayAnnealing + # Scheduler params + power: 2.0 + warmup_ratio: 0.05 + hold_ratio: 0.45 + min_lr: 0.001 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 150 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True + create_wandb_logger: False + wandb_logger_kwargs: + name: null + project: null + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v1.yaml b/models/tacotron2/NeMo/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v1.yaml new file mode 100644 index 0000000000..3016539bc1 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v1.yaml @@ -0,0 +1,193 @@ +name: &name "MatchboxNet-3x1x64-v1" + +model: + sample_rate: 16000 + timesteps: 128 + repeat: 1 + dropout: 0.0 + kernel_size_factor: 1.0 + + labels_full: ['bed', 'bird', 'cat', 'dog', 'down', 'eight', 'five', 'four', 'go', 'happy', 'house', 'left', 'marvin', + 'nine', 'no', 'off', 'on', 'one', 'right', 'seven', 'sheila', 'six', 'stop', 'three', 'tree', 'two', 'up', + 'wow', 'yes', 'zero'] + + labels_subset: ["yes", "no", "up", "down", "left", "right", "on", "off", "stop", "go", "unknown", "silence"] + + labels: ${model.labels_full} + + train_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + labels: ${model.labels} + batch_size: 128 + shuffle: True + is_tarred: False + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + augmentor: + shift: + prob: 1.0 + min_shift_ms: -5.0 + max_shift_ms: 5.0 + white_noise: + prob: 1.0 + min_level: -90 + max_level: -46 + + validation_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + labels: ${model.labels} + batch_size: 128 + shuffle: False + val_loss_idx: 0 + + test_ds: + manifest_filepath: null + sample_rate: ${model.sample_rate} + labels: ${model.labels} + batch_size: 128 + shuffle: False + test_loss_idx: 0 + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMFCCPreprocessor + window_size: 0.025 + window_stride: 0.01 + window: "hann" + n_mels: &n_mels 64 + n_mfcc: *n_mels + n_fft: 512 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + freq_masks: 2 + time_masks: 2 + freq_width: 15 + time_width: 25 + rect_masks: 5 + rect_time: 25 + rect_freq: 15 + + crop_or_pad_augment: + _target_: nemo.collections.asr.modules.CropOrPadSpectrogramAugmentation + audio_length: ${model.timesteps} + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: 128 + repeat: 1 + kernel: [11] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: false + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 64 + repeat: ${model.repeat} + kernel: [13] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 64 + repeat: ${model.repeat} + kernel: [15] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 64 + repeat: ${model.repeat} + kernel: [17] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 128 + repeat: 1 + kernel: [29] + stride: [1] + dilation: [2] + dropout: ${model.dropout} + residual: false + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: &enc_final_filters 128 + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: false + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoderClassification + feat_in: *enc_final_filters + return_logits: true + pooling_type: 'avg' + + optim: + name: novograd + # _target_: nemo.core.optim.optimizers.Novograd + lr: 0.05 + # optimizer arguments + betas: [0.95, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: PolynomialHoldDecayAnnealing + + # Scheduler params + power: 2.0 + warmup_ratio: 0.05 + hold_ratio: 0.45 + min_lr: 0.001 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 200 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True + create_wandb_logger: False + wandb_logger_kwargs: + name: null + project: null + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v2.yaml b/models/tacotron2/NeMo/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v2.yaml new file mode 100644 index 0000000000..5e088b065b --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v2.yaml @@ -0,0 +1,193 @@ +name: &name "MatchboxNet-3x1x64-v2" + +model: + sample_rate: 16000 + timesteps: 128 + repeat: 1 + dropout: 0.0 + kernel_size_factor: 1.0 + + labels_full: ['visual', 'wow', 'learn', 'backward', 'dog', 'two', 'left', 'happy', 'nine', 'go', 'up', 'bed', 'stop', + 'one', 'zero', 'tree', 'seven', 'on', 'four', 'bird', 'right', 'eight', 'no', 'six', 'forward', 'house', + 'marvin', 'sheila', 'five', 'off', 'three', 'down', 'cat', 'follow', 'yes'] + + labels_subset: ["yes", "no", "up", "down", "left", "right", "on", "off", "stop", "go", "unknown", "silence"] + + labels: ${model.labels_full} + + train_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + labels: ${model.labels} + batch_size: 128 + shuffle: True + is_tarred: False + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + augmentor: + shift: + prob: 1.0 + min_shift_ms: -5.0 + max_shift_ms: 5.0 + white_noise: + prob: 1.0 + min_level: -90 + max_level: -46 + + validation_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + labels: ${model.labels} + batch_size: 128 + shuffle: False + val_loss_idx: 0 + + test_ds: + manifest_filepath: null + sample_rate: ${model.sample_rate} + labels: ${model.labels} + batch_size: 128 + shuffle: False + test_loss_idx: 0 + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMFCCPreprocessor + window_size: 0.025 + window_stride: 0.01 + window: "hann" + n_mels: &n_mels 64 + n_mfcc: *n_mels + n_fft: 512 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + freq_masks: 2 + time_masks: 2 + freq_width: 15 + time_width: 25 + rect_masks: 5 + rect_time: 25 + rect_freq: 15 + + crop_or_pad_augment: + _target_: nemo.collections.asr.modules.CropOrPadSpectrogramAugmentation + audio_length: ${model.timesteps} + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: 128 + repeat: 1 + kernel: [11] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: false + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 64 + repeat: ${model.repeat} + kernel: [13] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 64 + repeat: ${model.repeat} + kernel: [15] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 64 + repeat: ${model.repeat} + kernel: [17] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: 128 + repeat: 1 + kernel: [29] + stride: [1] + dilation: [2] + dropout: ${model.dropout} + residual: false + separable: true + kernel_size_factor: ${model.kernel_size_factor} + + - filters: &enc_final_filters 128 + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: false + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoderClassification + feat_in: *enc_final_filters + return_logits: true + pooling_type: 'avg' + + optim: + name: novograd + # _target_: nemo.core.optim.optimizers.Novograd + lr: 0.05 + # optimizer arguments + betas: [0.95, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: PolynomialHoldDecayAnnealing + + # Scheduler params + power: 2.0 + warmup_ratio: 0.05 + hold_ratio: 0.45 + min_lr: 0.001 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 200 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True + create_wandb_logger: False + wandb_logger_kwargs: + name: null + project: null + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/asr/conf/quartznet/quartznet_15x5.yaml b/models/tacotron2/NeMo/examples/asr/conf/quartznet/quartznet_15x5.yaml new file mode 100644 index 0000000000..ad44ed16f0 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/quartznet/quartznet_15x5.yaml @@ -0,0 +1,281 @@ +name: &name "QuartzNet15x5" + +model: + sample_rate: &sample_rate 16000 + repeat: &repeat 5 + dropout: &dropout 0.0 + separable: &separable true + labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"] + + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: *labels + batch_size: 32 + trim_silence: True + max_duration: 16.7 + shuffle: True + is_tarred: False + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: *labels + batch_size: 32 + shuffle: False + + test_ds: + manifest_filepath: null + sample_rate: 16000 + labels: *labels + batch_size: 32 + shuffle: False + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.02 + sample_rate: *sample_rate + window_stride: 0.01 + window: "hann" + features: &n_mels 64 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + rect_freq: 50 + rect_masks: 5 + rect_time: 120 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: 1 + residual: false + separable: *separable + stride: [2] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [2] + dropout: *dropout + filters: 512 + kernel: [87] + repeat: 1 + residual: false + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: &enc_filters 1024 + kernel: [1] + repeat: 1 + residual: false + stride: [1] + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: *enc_filters + num_classes: 28 + vocabulary: *labels + + optim: + name: novograd + # _target_: nemo.core.optim.optimizers.Novograd + lr: .01 + # optimizer arguments + betas: [0.8, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + + # pytorch lightning args + # monitor: val_loss + # reduce_on_plateau: false + + # Scheduler params + warmup_steps: null + warmup_ratio: null + min_lr: 0.0 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 5 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True + checkpoint_callback_params: + monitor: "val_wer" + mode: "min" + create_wandb_logger: False + wandb_logger_kwargs: + name: null + project: null + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/asr/conf/quartznet/quartznet_15x5_zh.yaml b/models/tacotron2/NeMo/examples/asr/conf/quartznet/quartznet_15x5_zh.yaml new file mode 100644 index 0000000000..576ef81b6d --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/conf/quartznet/quartznet_15x5_zh.yaml @@ -0,0 +1,483 @@ +name: &name "QuartzNet15x5" + +model: + sample_rate: &sample_rate 16000 + repeat: &repeat 5 + dropout: &dropout 0.0 + separable: &separable true + labels: &labels [' ', '''', A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, + S, T, U, V, W, X, Y, Z, 㶧, 䶮, 一, 丁, 七, 万, 丈, 三, 上, 下, 不, 与, 丐, 丑, 专, 且, 丕, + 世, 丘, 丙, 业, 丛, 东, 丝, 丞, 丢, 两, 严, 丧, 个, 丫, 中, 丰, 串, 临, 丸, 丹, 为, 主, 丽, 举, 乃, + 久, 么, 义, 之, 乌, 乍, 乎, 乏, 乐, 乒, 乓, 乔, 乖, 乘, 乙, 九, 乞, 也, 习, 乡, 书, 买, 乱, 乳, 乾, + 了, 予, 争, 事, 二, 于, 亏, 云, 互, 五, 井, 亘, 亚, 些, 亟, 亡, 亢, 交, 亥, 亦, 产, 亨, 亩, 享, 京, + 亭, 亮, 亲, 亳, 亵, 人, 亿, 什, 仁, 仄, 仅, 仆, 仇, 今, 介, 仍, 从, 仑, 仓, 仔, 仕, 他, 仗, 付, 仙, + 仞, 仟, 仡, 代, 令, 以, 仨, 仪, 们, 仰, 仲, 件, 价, 任, 份, 仿, 企, 伉, 伊, 伍, 伎, 伏, 伐, 休, 众, + 优, 伙, 会, 伞, 伟, 传, 伢, 伤, 伦, 伪, 伫, 伯, 估, 伴, 伶, 伸, 伺, 似, 伽, 佃, 但, 位, 低, 住, 佐, + 佑, 体, 何, 佗, 佘, 余, 佚, 佛, 作, 佝, 佟, 你, 佣, 佩, 佬, 佯, 佰, 佳, 佶, 佻, 佼, 使, 侃, 侄, 侈, + 例, 侍, 侏, 侑, 侗, 供, 依, 侠, 侣, 侥, 侦, 侧, 侨, 侬, 侮, 侯, 侵, 便, 促, 俄, 俊, 俎, 俏, 俐, 俑, + 俗, 俘, 俚, 保, 俞, 俟, 信, 俨, 俩, 俪, 俭, 修, 俯, 俱, 俸, 俺, 俾, 倌, 倍, 倒, 倔, 倘, 候, 倚, 倜, + 借, 倡, 倦, 倩, 倪, 倭, 债, 值, 倾, 偃, 假, 偈, 偌, 偎, 偏, 偓, 偕, 做, 停, 健, 偶, 偷, 偻, 偿, 傀, + 傅, 傍, 傣, 傥, 储, 催, 傲, 傻, 像, 僚, 僧, 僮, 僵, 僻, 儋, 儒, 儡, 儿, 兀, 允, 元, 兄, 充, 兆, 先, + 光, 克, 免, 兑, 兔, 兖, 党, 兜, 兢, 入, 全, 八, 公, 六, 兮, 兰, 共, 关, 兴, 兵, 其, 具, 典, 兹, 养, + 兼, 兽, 冀, 内, 冈, 冉, 册, 再, 冒, 冕, 冗, 写, 军, 农, 冠, 冢, 冤, 冥, 冬, 冯, 冰, 冲, 决, 况, 冶, + 冷, 冻, 冼, 冽, 净, 凄, 准, 凇, 凉, 凋, 凌, 减, 凑, 凛, 凝, 几, 凡, 凤, 凭, 凯, 凰, 凳, 凶, 凸, 凹, + 出, 击, 函, 凿, 刀, 刁, 刃, 分, 切, 刊, 刍, 刎, 刑, 划, 列, 刘, 则, 刚, 创, 初, 删, 判, 刨, 利, 别, + 刮, 到, 制, 刷, 券, 刹, 刺, 刻, 刽, 剁, 剂, 剃, 削, 剌, 前, 剐, 剑, 剔, 剖, 剜, 剥, 剧, 剩, 剪, 副, + 割, 剽, 剿, 劈, 力, 劝, 办, 功, 加, 务, 劣, 动, 助, 努, 劫, 劭, 励, 劲, 劳, 劵, 劾, 势, 勃, 勇, 勉, + 勋, 勐, 勒, 勘, 募, 勤, 勺, 勾, 勿, 匀, 包, 匆, 匈, 匏, 匕, 化, 北, 匙, 匝, 匠, 匡, 匣, 匪, 匮, 匹, + 区, 医, 匾, 匿, 十, 千, 升, 午, 卉, 半, 华, 协, 卑, 卒, 卓, 单, 卖, 南, 博, 卜, 卞, 占, 卡, 卢, 卤, + 卦, 卧, 卫, 卯, 印, 危, 卲, 即, 却, 卵, 卷, 卸, 卿, 厂, 厄, 厅, 历, 厉, 压, 厌, 厕, 厘, 厚, 厝, 原, + 厢, 厥, 厦, 厨, 厩, 厮, 去, 县, 叁, 参, 又, 叉, 及, 友, 双, 反, 发, 叔, 取, 受, 变, 叙, 叛, 叠, 口, + 古, 句, 另, 叨, 叩, 只, 叫, 召, 叭, 叮, 可, 台, 叱, 史, 右, 叵, 叶, 号, 司, 叹, 叼, 叽, 吁, 吃, 各, + 吆, 合, 吉, 吊, 吋, 同, 名, 后, 吏, 吐, 向, 吒, 吓, 吕, 吖, 吗, 君, 吝, 吞, 吟, 吠, 否, 吧, 吨, 吩, + 含, 听, 吭, 吮, 启, 吱, 吴, 吵, 吸, 吹, 吻, 吼, 吾, 呀, 呃, 呆, 呈, 告, 呐, 呕, 呗, 员, 呛, 呜, 呢, + 呦, 周, 呱, 呲, 味, 呵, 呷, 呸, 呻, 呼, 命, 咀, 咂, 咄, 咆, 咋, 和, 咎, 咏, 咐, 咒, 咔, 咕, 咖, 咘, + 咙, 咚, 咝, 咣, 咤, 咦, 咧, 咨, 咩, 咪, 咫, 咬, 咭, 咯, 咱, 咳, 咸, 咻, 咽, 哀, 品, 哂, 哄, 哆, 哇, + 哈, 哉, 响, 哎, 哐, 哑, 哒, 哔, 哕, 哗, 哟, 哥, 哦, 哨, 哩, 哪, 哭, 哮, 哲, 哺, 哼, 哽, 唁, 唆, 唇, + 唉, 唏, 唐, 唑, 唛, 唠, 唢, 唤, 唧, 唬, 售, 唯, 唰, 唱, 唳, 唷, 唾, 啃, 啄, 商, 啊, 啕, 啖, 啜, 啡, + 啤, 啥, 啦, 啧, 啪, 啬, 啰, 啲, 啵, 啶, 啸, 啼, 啾, 喀, 喁, 喂, 喃, 善, 喆, 喇, 喉, 喊, 喋, 喔, 喘, + 喜, 喝, 喟, 喧, 喱, 喳, 喵, 喷, 喻, 喽, 嗄, 嗅, 嗑, 嗒, 嗓, 嗔, 嗖, 嗜, 嗝, 嗡, 嗣, 嗤, 嗦, 嗨, 嗪, + 嗫, 嗬, 嗯, 嗲, 嗷, 嗽, 嘀, 嘈, 嘉, 嘎, 嘏, 嘘, 嘛, 嘞, 嘟, 嘣, 嘭, 嘱, 嘲, 嘴, 嘶, 嘹, 嘻, 嘿, 噌, + 噎, 噗, 噘, 噙, 噜, 噢, 噤, 器, 噩, 噪, 噬, 噱, 噶, 噻, 噼, 嚎, 嚏, 嚓, 嚣, 嚷, 嚼, 囊, 囍, 囔, 囗, + 囚, 四, 回, 因, 团, 囤, 囧, 囫, 园, 囯, 困, 囱, 围, 囵, 囹, 固, 国, 图, 圃, 圄, 圆, 圈, 土, 圣, 在, + 圩, 圪, 圭, 地, 圳, 圹, 场, 圻, 圾, 址, 坂, 均, 坊, 坍, 坎, 坏, 坐, 坑, 块, 坚, 坛, 坝, 坞, 坟, 坠, + 坡, 坤, 坦, 坨, 坩, 坪, 坭, 坯, 坳, 坷, 坻, 垂, 垃, 垄, 垅, 型, 垌, 垒, 垚, 垛, 垡, 垢, 垣, 垤, 垦, + 垩, 垫, 垭, 垮, 埂, 埃, 埇, 埋, 城, 埔, 埕, 埚, 埝, 域, 埠, 埭, 埸, 培, 基, 堀, 堂, 堃, 堆, 堇, 堕, + 堡, 堤, 堪, 堰, 堵, 堺, 塌, 塍, 塑, 塔, 塘, 塞, 填, 塬, 塾, 境, 墅, 墉, 墓, 増, 墙, 增, 墟, 墨, 墩, + 壁, 壑, 壕, 壤, 士, 壬, 壮, 声, 壳, 壶, 壹, 处, 备, 复, 夏, 夔, 夕, 外, 夙, 多, 夜, 够, 大, 天, 太, + 夫, 夭, 央, 夯, 失, 头, 夷, 夸, 夹, 夺, 奁, 奂, 奄, 奇, 奈, 奉, 奋, 奎, 奏, 契, 奔, 奕, 奖, 套, 奘, + 奚, 奠, 奢, 奥, 女, 奴, 奶, 奸, 她, 好, 如, 妃, 妄, 妆, 妇, 妈, 妊, 妍, 妒, 妓, 妖, 妙, 妞, 妤, 妥, + 妨, 妩, 妪, 妫, 妮, 妯, 妲, 妹, 妻, 妾, 姆, 姊, 始, 姐, 姑, 姓, 委, 姗, 姚, 姜, 姝, 姣, 姥, 姨, 姬, + 姻, 姿, 威, 娃, 娄, 娅, 娆, 娇, 娈, 娉, 娌, 娓, 娘, 娜, 娟, 娠, 娣, 娥, 娩, 娱, 娲, 娴, 娶, 娼, 婀, + 婆, 婉, 婊, 婕, 婚, 婢, 婧, 婪, 婴, 婵, 婶, 婷, 婺, 婿, 媒, 媚, 媛, 媞, 媲, 媳, 嫁, 嫂, 嫉, 嫌, 嫒, + 嫔, 嫖, 嫚, 嫡, 嫣, 嫦, 嫩, 嫫, 嬅, 嬉, 嬗, 嬛, 嬴, 嬷, 孀, 子, 孑, 孔, 孕, 字, 存, 孙, 孚, 孛, 孜, + 孝, 孟, 孢, 季, 孤, 学, 孩, 孪, 孬, 孰, 孱, 孳, 孵, 孺, 孽, 宁, 它, 宅, 宇, 守, 安, 宋, 完, 宏, 宓, + 宕, 宗, 官, 宙, 定, 宛, 宜, 宝, 实, 宠, 审, 客, 宣, 室, 宥, 宦, 宪, 宫, 宰, 害, 宴, 宵, 家, 宸, 容, + 宽, 宾, 宿, 寂, 寄, 寅, 密, 寇, 富, 寐, 寒, 寓, 寝, 寞, 察, 寡, 寥, 寨, 寮, 寰, 寸, 对, 寺, 寻, 导, + 寿, 封, 射, 尅, 将, 尉, 尊, 小, 少, 尔, 尕, 尖, 尘, 尚, 尝, 尤, 尧, 尬, 就, 尴, 尸, 尹, 尺, 尼, 尽, + 尾, 尿, 局, 屁, 层, 居, 屈, 屉, 届, 屋, 屌, 屎, 屏, 屐, 屑, 展, 属, 屠, 屡, 履, 屯, 山, 屹, 屿, 岁, + 岂, 岌, 岐, 岑, 岔, 岖, 岗, 岚, 岛, 岩, 岬, 岭, 岱, 岳, 岷, 岸, 峁, 峋, 峒, 峙, 峡, 峥, 峦, 峨, 峪, + 峭, 峰, 峻, 崂, 崃, 崆, 崇, 崎, 崔, 崖, 崛, 崧, 崩, 崭, 崮, 崴, 崽, 嵇, 嵊, 嵋, 嵌, 嵘, 嵛, 嵩, 嵬, + 嶂, 嶙, 嶝, 巅, 巍, 川, 州, 巡, 巢, 工, 左, 巧, 巨, 巩, 巫, 差, 己, 已, 巳, 巴, 巷, 巾, 币, 市, 布, + 帅, 帆, 师, 希, 帐, 帕, 帖, 帘, 帚, 帛, 帜, 帝, 带, 帧, 席, 帮, 帷, 常, 帼, 帽, 幂, 幄, 幅, 幌, 幔, + 幕, 幡, 幢, 干, 平, 年, 并, 幸, 幺, 幻, 幼, 幽, 广, 庄, 庆, 庇, 床, 序, 庐, 库, 应, 底, 庖, 店, 庙, + 庚, 府, 庞, 废, 度, 座, 庭, 庵, 庶, 康, 庸, 庹, 庾, 廉, 廊, 廓, 廖, 延, 廷, 建, 开, 异, 弃, 弄, 弈, + 弊, 弋, 式, 弑, 弓, 引, 弗, 弘, 弛, 弟, 张, 弥, 弦, 弧, 弩, 弭, 弯, 弱, 弹, 强, 弼, 归, 当, 录, 彗, + 彝, 形, 彤, 彦, 彩, 彪, 彬, 彭, 彰, 影, 彷, 役, 彻, 彼, 往, 征, 径, 待, 徇, 很, 徉, 徊, 律, 徐, 徒, + 得, 徘, 徙, 徜, 御, 徨, 循, 微, 德, 徽, 心, 必, 忆, 忌, 忍, 忏, 忐, 忑, 忒, 忖, 志, 忘, 忙, 忠, 忡, + 忤, 忧, 忪, 快, 忱, 念, 忻, 忽, 忿, 怀, 态, 怂, 怄, 怅, 怆, 怎, 怒, 怕, 怖, 怜, 思, 怠, 怡, 急, 怦, + 性, 怨, 怪, 怫, 怯, 怵, 总, 怼, 怿, 恁, 恃, 恋, 恍, 恐, 恒, 恕, 恙, 恢, 恣, 恤, 恨, 恩, 恪, 恬, 恭, + 息, 恰, 恳, 恶, 恸, 恺, 恻, 恼, 恿, 悄, 悉, 悌, 悍, 悔, 悖, 悚, 悟, 悠, 患, 悦, 您, 悬, 悭, 悯, 悱, + 悲, 悴, 悸, 悻, 悼, 情, 惆, 惊, 惋, 惑, 惕, 惚, 惜, 惟, 惠, 惦, 惧, 惨, 惩, 惫, 惬, 惭, 惮, 惯, 惰, + 想, 惶, 惹, 惺, 愁, 愈, 愉, 意, 愕, 愚, 感, 愣, 愤, 愧, 愫, 愿, 慈, 慌, 慎, 慑, 慕, 慢, 慧, 慨, 慰, + 慵, 慷, 憋, 憎, 憔, 憧, 憨, 憩, 憬, 憷, 憾, 懂, 懈, 懊, 懋, 懑, 懒, 懦, 懵, 懿, 戈, 戊, 戌, 戍, 戎, + 戏, 成, 我, 戒, 或, 戗, 战, 戚, 戛, 戟, 截, 戬, 戮, 戳, 戴, 户, 戾, 房, 所, 扁, 扇, 扈, 扉, 手, 才, + 扎, 扑, 扒, 打, 扔, 托, 扛, 扞, 扣, 扦, 执, 扩, 扪, 扫, 扬, 扭, 扮, 扯, 扰, 扳, 扶, 批, 扼, 找, 承, + 技, 抄, 抉, 把, 抑, 抒, 抓, 投, 抖, 抗, 折, 抚, 抛, 抠, 抡, 抢, 护, 报, 抨, 披, 抬, 抱, 抵, 抹, 押, + 抽, 抿, 拂, 拄, 担, 拆, 拇, 拈, 拉, 拌, 拍, 拎, 拐, 拒, 拓, 拔, 拖, 拗, 拘, 拙, 拚, 招, 拜, 拟, 拢, + 拣, 拥, 拦, 拧, 拨, 择, 括, 拭, 拮, 拯, 拱, 拳, 拴, 拷, 拼, 拽, 拾, 拿, 持, 挂, 指, 按, 挎, 挑, 挖, + 挚, 挛, 挝, 挞, 挟, 挠, 挡, 挣, 挤, 挥, 挨, 挪, 挫, 振, 挺, 挽, 捂, 捅, 捆, 捉, 捋, 捌, 捍, 捎, 捏, + 捐, 捕, 捞, 损, 捡, 换, 捣, 捧, 据, 捶, 捷, 捺, 捻, 掀, 掂, 掇, 授, 掉, 掌, 掏, 掐, 排, 掖, 掘, 掠, + 探, 掣, 接, 控, 推, 掩, 措, 掬, 掮, 掰, 掳, 掴, 掷, 掸, 掺, 揄, 揉, 揍, 描, 提, 插, 握, 揣, 揩, 揪, + 揭, 援, 揶, 揽, 搀, 搁, 搂, 搅, 搏, 搐, 搓, 搔, 搜, 搞, 搡, 搧, 搪, 搬, 搭, 携, 搽, 摁, 摄, 摆, 摇, + 摈, 摊, 摒, 摔, 摘, 摞, 摧, 摩, 摸, 摹, 撂, 撅, 撇, 撑, 撒, 撕, 撞, 撤, 撩, 撬, 播, 撮, 撰, 撵, 撸, + 撺, 撼, 擀, 擂, 擅, 操, 擎, 擒, 擘, 擞, 擢, 擦, 攀, 攒, 攘, 攥, 攫, 支, 收, 攸, 改, 攻, 放, 政, 故, + 效, 敌, 敏, 救, 敕, 敖, 教, 敛, 敝, 敞, 敢, 散, 敦, 敬, 数, 敲, 整, 敷, 文, 斋, 斌, 斐, 斑, 斓, 斗, + 料, 斛, 斜, 斟, 斡, 斤, 斥, 斧, 斩, 断, 斯, 新, 方, 施, 旁, 旅, 旋, 旌, 族, 旖, 旗, 无, 既, 日, 旦, + 旧, 旨, 早, 旬, 旭, 旮, 旯, 旱, 时, 旷, 旺, 旻, 昀, 昂, 昆, 昊, 昌, 明, 昏, 易, 昔, 昕, 昙, 昝, 星, + 映, 春, 昧, 昨, 昭, 是, 昱, 昴, 昵, 昶, 昼, 显, 晃, 晋, 晌, 晏, 晒, 晓, 晔, 晕, 晖, 晗, 晚, 晞, 晟, + 晤, 晦, 晨, 普, 景, 晰, 晴, 晶, 晷, 智, 晾, 暂, 暄, 暇, 暌, 暑, 暖, 暗, 暧, 暨, 暮, 暴, 暹, 暾, 曈, + 曙, 曜, 曝, 曦, 曰, 曲, 曳, 更, 曹, 曼, 曾, 替, 最, 月, 有, 朋, 服, 朐, 朔, 朕, 朗, 望, 朝, 期, 朦, + 木, 未, 末, 本, 札, 术, 朱, 朴, 朵, 机, 朽, 杀, 杂, 权, 杆, 杈, 杉, 李, 杏, 材, 村, 杓, 杖, 杜, 杞, + 束, 杠, 条, 来, 杨, 杭, 杯, 杰, 杳, 杵, 杷, 松, 板, 极, 构, 枇, 枉, 枋, 析, 枕, 林, 枚, 果, 枝, 枞, + 枢, 枣, 枥, 枪, 枫, 枭, 枯, 枰, 枳, 架, 枷, 枸, 柃, 柄, 柏, 某, 柑, 柒, 染, 柔, 柘, 柚, 柜, 柞, 柠, + 查, 柩, 柬, 柯, 柱, 柳, 柴, 柿, 栀, 栅, 标, 栈, 栋, 栌, 栎, 栏, 树, 栓, 栖, 栗, 校, 栩, 株, 样, 核, + 根, 格, 栽, 栾, 桁, 桂, 桃, 框, 案, 桉, 桌, 桎, 桐, 桑, 桓, 桔, 桠, 桢, 档, 桥, 桦, 桨, 桩, 桴, 桶, + 桷, 梁, 梅, 梆, 梏, 梓, 梗, 梢, 梦, 梧, 梨, 梭, 梯, 械, 梳, 梵, 检, 棂, 棉, 棋, 棍, 棒, 棕, 棘, 棚, + 棠, 棣, 森, 棱, 棵, 棺, 椁, 椅, 椋, 植, 椎, 椒, 椟, 椤, 椭, 椰, 椴, 椹, 椿, 楂, 楔, 楚, 楞, 楠, 楣, + 楷, 楸, 楼, 概, 榄, 榆, 榈, 榉, 榔, 榕, 榛, 榜, 榨, 榫, 榭, 榴, 榷, 榻, 槃, 槌, 槎, 槐, 槛, 槟, 槭, + 槽, 槿, 樊, 樟, 模, 樨, 横, 樯, 樱, 樵, 樽, 樾, 橄, 橇, 橐, 橘, 橙, 橡, 橱, 檀, 檐, 檗, 檬, 欠, 次, + 欢, 欣, 欧, 欲, 欸, 欺, 款, 歆, 歇, 歉, 歌, 歙, 止, 正, 此, 步, 武, 歧, 歩, 歪, 歹, 死, 歼, 殁, 殃, + 殆, 殇, 殉, 殊, 残, 殒, 殓, 殖, 殚, 殡, 殴, 段, 殷, 殿, 毁, 毂, 毅, 毋, 母, 每, 毒, 毓, 比, 毕, 毗, + 毙, 毛, 毡, 毫, 毯, 毽, 氏, 民, 氓, 气, 氚, 氛, 氟, 氢, 氤, 氦, 氧, 氨, 氪, 氮, 氯, 氰, 氲, 水, 永, + 汀, 汁, 求, 汇, 汉, 汊, 汐, 汕, 汗, 汛, 汝, 汞, 江, 池, 污, 汤, 汨, 汩, 汪, 汰, 汲, 汴, 汶, 汹, 汽, + 汾, 沁, 沂, 沃, 沅, 沈, 沉, 沌, 沏, 沐, 沓, 沙, 沛, 沟, 没, 沢, 沣, 沥, 沦, 沧, 沪, 沫, 沭, 沮, 沱, + 河, 沸, 油, 治, 沼, 沽, 沾, 沿, 泄, 泉, 泊, 泌, 泓, 泔, 法, 泖, 泗, 泛, 泞, 泠, 泡, 波, 泣, 泥, 注, + 泪, 泫, 泮, 泯, 泰, 泱, 泳, 泵, 泷, 泸, 泺, 泻, 泼, 泽, 泾, 洁, 洋, 洒, 洗, 洙, 洛, 洞, 津, 洪, 洮, + 洱, 洲, 洵, 洹, 洺, 活, 洼, 洽, 派, 流, 浃, 浅, 浆, 浇, 浈, 浊, 测, 济, 浏, 浐, 浑, 浒, 浓, 浔, 浙, + 浚, 浜, 浠, 浣, 浦, 浩, 浪, 浮, 浴, 海, 浸, 涂, 涅, 消, 涉, 涌, 涎, 涑, 涓, 涕, 涛, 涝, 涞, 涟, 涠, + 涡, 涣, 涤, 润, 涧, 涨, 涩, 涪, 涮, 涯, 液, 涵, 涸, 涿, 淀, 淄, 淅, 淆, 淇, 淋, 淌, 淑, 淖, 淘, 淝, + 淞, 淡, 淤, 淦, 淫, 淬, 淮, 深, 淳, 混, 淹, 添, 淼, 清, 渊, 渌, 渍, 渎, 渐, 渑, 渔, 渗, 渚, 渝, 渠, + 渡, 渣, 渤, 渥, 温, 渭, 港, 渲, 渴, 游, 渺, 湃, 湄, 湉, 湍, 湎, 湖, 湘, 湛, 湫, 湾, 湿, 溃, 溅, 溆, + 溉, 溏, 源, 溜, 溟, 溢, 溥, 溧, 溪, 溯, 溶, 溺, 滁, 滇, 滋, 滑, 滔, 滕, 滘, 滚, 滞, 满, 滢, 滤, 滥, + 滦, 滨, 滩, 滴, 滹, 漂, 漆, 漉, 漏, 漓, 演, 漕, 漠, 漩, 漪, 漫, 漭, 漯, 漱, 漳, 漾, 潆, 潇, 潋, 潍, + 潘, 潜, 潞, 潢, 潦, 潭, 潮, 潸, 潺, 潼, 澄, 澈, 澍, 澎, 澜, 澡, 澧, 澳, 澶, 激, 濂, 濑, 濒, 濠, 濡, + 濮, 濯, 瀑, 瀚, 瀛, 灌, 灏, 灞, 火, 灭, 灯, 灰, 灵, 灶, 灸, 灼, 灾, 灿, 炀, 炅, 炉, 炊, 炎, 炒, 炔, + 炕, 炖, 炙, 炜, 炫, 炬, 炭, 炮, 炯, 炳, 炷, 炸, 点, 炼, 炽, 烀, 烁, 烂, 烃, 烈, 烊, 烘, 烙, 烛, 烟, + 烤, 烦, 烧, 烨, 烩, 烫, 烬, 热, 烯, 烷, 烹, 烽, 焉, 焊, 焓, 焕, 焖, 焗, 焘, 焙, 焚, 焦, 焯, 焰, 焱, + 然, 煊, 煌, 煎, 煜, 煞, 煤, 煦, 照, 煨, 煮, 煲, 煳, 煽, 熄, 熊, 熏, 熔, 熙, 熟, 熠, 熨, 熬, 熵, 熹, + 燃, 燊, 燎, 燕, 燥, 燮, 爆, 爪, 爬, 爱, 爵, 父, 爷, 爸, 爹, 爽, 片, 版, 牌, 牍, 牒, 牙, 牛, 牟, 牠, + 牡, 牢, 牧, 物, 牲, 牵, 特, 牺, 牾, 犀, 犁, 犄, 犇, 犊, 犒, 犟, 犬, 犯, 状, 犷, 犸, 犹, 狂, 狄, 狈, + 狐, 狒, 狗, 狙, 狞, 狠, 狡, 狩, 独, 狭, 狮, 狰, 狱, 狸, 狼, 猁, 猎, 猖, 猛, 猜, 猝, 猥, 猩, 猪, 猫, + 猬, 献, 猴, 猷, 猹, 猾, 猿, 獒, 獗, 獭, 獾, 玄, 率, 玉, 王, 玑, 玖, 玛, 玟, 玥, 玩, 玫, 玮, 环, 现, + 玲, 玳, 玷, 玹, 玺, 玻, 珀, 珂, 珈, 珉, 珊, 珍, 珏, 珑, 珙, 珞, 珠, 珥, 班, 珮, 珲, 珺, 球, 琅, 理, + 琉, 琊, 琏, 琐, 琛, 琢, 琤, 琥, 琦, 琨, 琪, 琬, 琮, 琰, 琳, 琴, 琵, 琶, 琼, 瑁, 瑄, 瑕, 瑙, 瑚, 瑛, + 瑜, 瑞, 瑟, 瑠, 瑭, 瑰, 瑶, 瑷, 瑾, 璀, 璃, 璇, 璋, 璐, 璞, 璟, 璧, 璨, 瓜, 瓢, 瓣, 瓦, 瓮, 瓯, 瓶, + 瓷, 甄, 甘, 甚, 甜, 生, 甥, 用, 甩, 甫, 甬, 甭, 田, 由, 甲, 申, 电, 男, 甸, 町, 画, 畅, 畈, 畊, 界, + 畏, 畔, 留, 畜, 略, 番, 畴, 畸, 畿, 疃, 疆, 疏, 疑, 疖, 疗, 疙, 疚, 疝, 疟, 疡, 疣, 疤, 疫, 疮, 疯, + 疱, 疲, 疴, 疵, 疸, 疹, 疼, 疽, 疾, 病, 症, 痉, 痊, 痍, 痒, 痔, 痕, 痘, 痛, 痞, 痢, 痣, 痧, 痨, 痪, + 痫, 痰, 痱, 痴, 痹, 痼, 瘀, 瘁, 瘙, 瘟, 瘠, 瘢, 瘤, 瘦, 瘩, 瘪, 瘫, 瘳, 瘴, 瘸, 瘾, 癌, 癖, 癜, 癞, + 癣, 癫, 登, 白, 百, 皂, 的, 皆, 皇, 皋, 皎, 皑, 皓, 皖, 皙, 皮, 皱, 皿, 盂, 盅, 盆, 盈, 益, 盎, 盏, + 盐, 监, 盒, 盔, 盖, 盗, 盘, 盛, 盟, 目, 盯, 盱, 盲, 直, 相, 盹, 盼, 盾, 省, 眈, 眉, 看, 眙, 真, 眠, + 眨, 眩, 眬, 眯, 眶, 眷, 眸, 眺, 眼, 着, 睁, 睇, 睐, 睑, 睛, 睡, 睢, 督, 睦, 睫, 睬, 睹, 睽, 睾, 睿, + 瞄, 瞅, 瞌, 瞎, 瞑, 瞒, 瞟, 瞠, 瞥, 瞧, 瞩, 瞪, 瞬, 瞭, 瞰, 瞳, 瞻, 瞿, 矍, 矗, 矛, 矜, 矢, 矣, 知, + 矩, 矫, 矬, 短, 矮, 石, 矶, 矸, 矽, 矾, 矿, 砀, 码, 砂, 砌, 砍, 砒, 研, 砖, 砚, 砝, 砣, 砥, 砭, 砰, + 破, 砷, 砸, 砺, 砼, 砾, 础, 硅, 硌, 硒, 硕, 硖, 硚, 硝, 硫, 硬, 确, 硼, 碉, 碌, 碍, 碎, 碑, 碓, 碗, + 碘, 碚, 碜, 碟, 碣, 碧, 碰, 碱, 碳, 碴, 碾, 磁, 磅, 磊, 磋, 磐, 磕, 磨, 磴, 磷, 磺, 礁, 示, 礼, 社, + 祀, 祁, 祈, 祉, 祎, 祐, 祖, 祚, 祛, 祝, 神, 祟, 祠, 祢, 祥, 票, 祭, 祯, 祷, 祸, 祺, 禀, 禁, 禄, 禅, + 福, 禧, 禹, 禺, 离, 禽, 禾, 秀, 私, 秃, 秆, 秉, 秋, 种, 科, 秒, 秘, 租, 秣, 秤, 秦, 秧, 秩, 积, 称, + 秸, 移, 秽, 稀, 程, 稍, 税, 稔, 稚, 稞, 稠, 稣, 稳, 稷, 稹, 稻, 稼, 稽, 稿, 穆, 穗, 穴, 究, 穷, 穹, + 空, 穿, 突, 窃, 窄, 窈, 窍, 窑, 窒, 窕, 窖, 窗, 窘, 窜, 窝, 窟, 窠, 窥, 窦, 窨, 窿, 立, 竖, 站, 竞, + 竟, 章, 竣, 童, 竭, 端, 竹, 竺, 竽, 竿, 笃, 笆, 笈, 笋, 笑, 笔, 笙, 笛, 笠, 符, 笨, 第, 笳, 笸, 笼, + 等, 筋, 筏, 筐, 筑, 筒, 答, 策, 筛, 筝, 筠, 筱, 筵, 筷, 筹, 签, 简, 箍, 箔, 箕, 算, 管, 箩, 箫, 箭, + 箱, 箴, 篁, 篆, 篇, 篑, 篓, 篝, 篡, 篦, 篪, 篮, 篱, 篷, 篼, 簇, 簋, 簧, 簪, 簸, 簿, 籁, 籍, 米, 类, + 籼, 籽, 粉, 粑, 粒, 粕, 粗, 粘, 粟, 粤, 粥, 粪, 粮, 粱, 粲, 粳, 粹, 粼, 粽, 精, 糊, 糕, 糖, 糗, 糙, + 糟, 糠, 糯, 系, 紊, 素, 索, 紧, 紫, 累, 絮, 綦, 繁, 纂, 纠, 纡, 红, 纣, 纤, 约, 级, 纨, 纪, 纫, 纬, + 纭, 纯, 纰, 纱, 纲, 纳, 纵, 纶, 纷, 纸, 纹, 纺, 纽, 纾, 线, 绀, 练, 组, 绅, 细, 织, 终, 绉, 绊, 绋, + 绌, 绍, 绎, 经, 绑, 绒, 结, 绔, 绕, 绘, 给, 绚, 绛, 络, 绝, 绞, 统, 绢, 绣, 绥, 继, 绩, 绪, 绫, 续, + 绮, 绯, 绰, 绳, 维, 绵, 绷, 绸, 绻, 综, 绽, 绿, 缀, 缄, 缅, 缆, 缇, 缉, 缎, 缓, 缔, 缕, 编, 缘, 缙, + 缚, 缛, 缜, 缝, 缠, 缢, 缤, 缨, 缩, 缪, 缬, 缭, 缮, 缰, 缱, 缴, 缸, 缺, 罂, 罄, 罐, 网, 罔, 罕, 罗, + 罚, 罡, 罢, 罩, 罪, 置, 署, 罹, 羁, 羊, 羌, 美, 羔, 羚, 羞, 羡, 群, 羧, 羯, 羲, 羸, 羹, 羽, 羿, 翁, + 翅, 翊, 翌, 翎, 翔, 翘, 翟, 翠, 翡, 翩, 翰, 翱, 翻, 翼, 耀, 老, 考, 耄, 者, 耆, 耋, 而, 耍, 耐, 耒, + 耕, 耗, 耘, 耙, 耜, 耪, 耳, 耶, 耷, 耸, 耻, 耽, 耿, 聂, 聆, 聊, 聋, 职, 联, 聘, 聚, 聪, 肃, 肆, 肇, + 肉, 肋, 肌, 肖, 肘, 肚, 肛, 肝, 肠, 股, 肢, 肤, 肥, 肩, 肪, 肮, 肯, 肱, 育, 肴, 肺, 肾, 肿, 胀, 胁, + 胃, 胆, 背, 胎, 胖, 胗, 胚, 胛, 胜, 胞, 胡, 胤, 胥, 胧, 胫, 胭, 胯, 胰, 胱, 胳, 胶, 胸, 胺, 能, 脂, + 脆, 脉, 脊, 脍, 脏, 脐, 脑, 脓, 脖, 脚, 脯, 脱, 脸, 脾, 腆, 腈, 腊, 腋, 腌, 腐, 腑, 腓, 腔, 腕, 腥, + 腩, 腭, 腮, 腰, 腱, 腴, 腹, 腺, 腻, 腼, 腾, 腿, 膀, 膈, 膊, 膏, 膑, 膛, 膜, 膝, 膨, 膳, 膺, 臀, 臂, + 臃, 臆, 臊, 臣, 臧, 自, 臬, 臭, 至, 致, 臻, 臼, 舀, 舅, 舆, 舌, 舍, 舐, 舒, 舔, 舛, 舜, 舞, 舟, 航, + 舫, 般, 舰, 舱, 舵, 舶, 舷, 舸, 船, 艇, 艋, 艘, 艮, 良, 艰, 色, 艳, 艺, 艾, 艿, 节, 芊, 芋, 芍, 芒, + 芗, 芙, 芜, 芝, 芥, 芦, 芩, 芪, 芬, 芭, 芮, 芯, 花, 芳, 芷, 芸, 芹, 芽, 芾, 苇, 苋, 苍, 苏, 苑, 苓, + 苔, 苗, 苛, 苞, 苟, 苡, 苣, 若, 苦, 苫, 苯, 英, 苷, 苹, 茁, 茂, 范, 茄, 茅, 茆, 茉, 茌, 茎, 茗, 茛, + 茜, 茧, 茨, 茫, 茬, 茯, 茱, 茳, 茴, 茵, 茶, 茸, 茹, 茼, 荀, 荃, 荆, 荇, 草, 荏, 荐, 荒, 荔, 荚, 荛, + 荞, 荟, 荠, 荡, 荣, 荤, 荧, 荨, 荫, 药, 荷, 荸, 荻, 荼, 莅, 莆, 莉, 莎, 莒, 莓, 莘, 莜, 莞, 莠, 莪, + 莫, 莱, 莲, 莴, 获, 莹, 莺, 莽, 菀, 菁, 菅, 菇, 菊, 菌, 菏, 菖, 菘, 菜, 菠, 菡, 菩, 菱, 菲, 萃, 萄, + 萋, 萌, 萍, 萎, 萝, 萤, 营, 萦, 萧, 萨, 萱, 萸, 落, 葆, 著, 葚, 葛, 葡, 董, 葩, 葫, 葬, 葱, 葳, 葵, + 葺, 蒂, 蒋, 蒙, 蒜, 蒯, 蒲, 蒸, 蒿, 蓁, 蓄, 蓉, 蓓, 蓝, 蓟, 蓥, 蓦, 蓬, 蓼, 蔑, 蔓, 蔗, 蔚, 蔡, 蔫, + 蔬, 蔷, 蔺, 蔻, 蔼, 蔽, 蕃, 蕉, 蕊, 蕙, 蕨, 蕲, 蕴, 蕾, 薄, 薇, 薏, 薛, 薪, 薯, 薰, 薷, 藁, 藉, 藏, + 藐, 藓, 藕, 藜, 藠, 藤, 藩, 藻, 藿, 蘑, 蘸, 虎, 虏, 虐, 虑, 虔, 虚, 虞, 虫, 虱, 虹, 虻, 虽, 虾, 蚀, + 蚁, 蚂, 蚊, 蚌, 蚓, 蚕, 蚝, 蚣, 蚤, 蚪, 蚬, 蚯, 蚱, 蚴, 蛀, 蛆, 蛇, 蛉, 蛊, 蛋, 蛎, 蛐, 蛔, 蛙, 蛛, + 蛟, 蛤, 蛮, 蛰, 蛳, 蛹, 蛾, 蜀, 蜂, 蜃, 蜇, 蜈, 蜊, 蜍, 蜒, 蜓, 蜕, 蜗, 蜘, 蜚, 蜜, 蜡, 蜢, 蜥, 蜱, + 蜴, 蜷, 蜻, 蜿, 蝇, 蝈, 蝉, 蝌, 蝎, 蝗, 蝙, 蝠, 蝮, 蝴, 蝶, 蝽, 螂, 螃, 螈, 融, 螨, 螳, 螺, 蟀, 蟆, + 蟊, 蟋, 蟑, 蟒, 蟠, 蟹, 蟾, 蠊, 蠕, 蠡, 蠢, 血, 衅, 行, 衍, 衔, 街, 衙, 衡, 衢, 衣, 补, 表, 衩, 衫, + 衬, 衮, 衰, 衲, 衷, 袁, 袂, 袄, 袅, 袈, 袋, 袍, 袒, 袖, 袜, 被, 袭, 袱, 裁, 裂, 装, 裆, 裔, 裕, 裘, + 裙, 裟, 裤, 裨, 裱, 裳, 裴, 裸, 裹, 褂, 褐, 褒, 褓, 褔, 褚, 褛, 褥, 褪, 褴, 褶, 襁, 襄, 襟, 西, 要, + 覃, 覆, 见, 观, 规, 觅, 视, 览, 觉, 觊, 觎, 觐, 觑, 角, 觞, 解, 觥, 触, 言, 訾, 詹, 誉, 誓, 警, 譬, + 计, 订, 讣, 认, 讥, 讧, 讨, 让, 讪, 训, 议, 讯, 记, 讲, 讳, 讴, 讶, 讷, 许, 讹, 论, 讼, 讽, 设, 访, + 诀, 证, 诃, 评, 诅, 识, 诈, 诉, 诊, 诋, 词, 诏, 译, 诓, 试, 诗, 诘, 诙, 诚, 诛, 话, 诞, 诟, 诠, 诡, + 询, 诣, 诤, 该, 详, 诧, 诩, 诫, 诬, 语, 误, 诱, 诲, 说, 诵, 诶, 请, 诸, 诹, 诺, 读, 诽, 课, 诿, 谀, + 谁, 调, 谅, 谆, 谈, 谊, 谋, 谌, 谍, 谎, 谏, 谐, 谑, 谓, 谕, 谖, 谘, 谙, 谚, 谛, 谜, 谟, 谢, 谣, 谤, + 谦, 谧, 谨, 谩, 谬, 谭, 谮, 谯, 谱, 谴, 谶, 谷, 豁, 豆, 豇, 豉, 豌, 豚, 象, 豢, 豪, 豫, 豹, 豺, 貂, + 貅, 貉, 貌, 貔, 贝, 贞, 负, 贡, 财, 责, 贤, 败, 账, 货, 质, 贩, 贪, 贫, 贬, 购, 贮, 贯, 贰, 贱, 贲, + 贴, 贵, 贷, 贸, 费, 贺, 贻, 贼, 贾, 贿, 赁, 赂, 赃, 资, 赅, 赈, 赉, 赊, 赋, 赌, 赎, 赏, 赐, 赓, 赔, + 赖, 赘, 赚, 赛, 赝, 赞, 赠, 赡, 赢, 赣, 赤, 赦, 赫, 走, 赳, 赴, 赵, 赶, 起, 趁, 超, 越, 趋, 趟, 趣, + 足, 趴, 趵, 趸, 趺, 趾, 跃, 跄, 跆, 跋, 跌, 跎, 跑, 跚, 跛, 距, 跟, 跤, 跨, 跪, 跬, 路, 跳, 践, 跶, + 跷, 跹, 跺, 跻, 踉, 踊, 踌, 踏, 踝, 踞, 踢, 踩, 踪, 踮, 踯, 踱, 踵, 踹, 踺, 蹁, 蹂, 蹄, 蹈, 蹉, 蹊, + 蹋, 蹒, 蹚, 蹦, 蹩, 蹬, 蹭, 蹲, 蹴, 蹶, 蹼, 蹿, 躁, 躅, 躇, 躏, 身, 躬, 躯, 躲, 躺, 车, 轧, 轨, 轩, + 轫, 转, 轮, 软, 轰, 轱, 轲, 轳, 轴, 轶, 轸, 轻, 轼, 载, 轿, 较, 辄, 辅, 辆, 辈, 辉, 辊, 辍, 辐, 辑, + 输, 辕, 辖, 辗, 辘, 辙, 辛, 辜, 辞, 辟, 辣, 辨, 辩, 辫, 辰, 辱, 边, 辽, 达, 迁, 迂, 迄, 迅, 过, 迈, + 迎, 运, 近, 返, 还, 这, 进, 远, 违, 连, 迟, 迢, 迥, 迦, 迩, 迪, 迫, 迭, 述, 迷, 迸, 迹, 追, 退, 送, + 适, 逃, 逅, 逆, 选, 逊, 逋, 逍, 透, 逐, 逑, 递, 途, 逗, 通, 逛, 逝, 逞, 速, 造, 逡, 逢, 逮, 逯, 逵, + 逸, 逻, 逼, 逾, 遁, 遂, 遇, 遍, 遏, 遐, 遑, 道, 遗, 遛, 遢, 遣, 遥, 遨, 遭, 遮, 遴, 遵, 避, 邀, 邂, + 邃, 邋, 邑, 邓, 邕, 邙, 邛, 邝, 邡, 邢, 那, 邦, 邪, 邬, 邮, 邯, 邰, 邱, 邳, 邵, 邸, 邹, 邺, 邻, 郁, + 郅, 郇, 郊, 郎, 郑, 郓, 郜, 郝, 郡, 郧, 部, 郫, 郭, 郯, 郴, 郸, 都, 鄂, 鄙, 鄞, 鄢, 鄱, 酉, 酊, 酋, + 酌, 配, 酐, 酒, 酗, 酚, 酝, 酞, 酣, 酥, 酩, 酪, 酬, 酮, 酯, 酰, 酱, 酵, 酶, 酷, 酸, 酿, 醇, 醉, 醋, + 醍, 醐, 醒, 醛, 醺, 采, 釉, 释, 里, 重, 野, 量, 金, 釜, 鉴, 銮, 鏖, 鑫, 钇, 针, 钉, 钊, 钎, 钏, 钐, + 钒, 钓, 钗, 钙, 钛, 钜, 钝, 钞, 钟, 钠, 钢, 钣, 钥, 钦, 钧, 钨, 钩, 钮, 钯, 钰, 钱, 钲, 钳, 钴, 钵, + 钻, 钼, 钾, 钿, 铀, 铁, 铂, 铃, 铄, 铅, 铆, 铉, 铋, 铍, 铎, 铐, 铑, 铖, 铛, 铜, 铝, 铟, 铠, 铡, 铣, + 铤, 铧, 铨, 铩, 铬, 铭, 铮, 铰, 铲, 银, 铷, 铸, 铺, 链, 铿, 销, 锁, 锂, 锄, 锅, 锆, 锈, 锉, 锋, 锌, + 锏, 锐, 锑, 锒, 错, 锚, 锟, 锡, 锢, 锣, 锤, 锥, 锦, 锨, 锭, 键, 锯, 锰, 锲, 锴, 锵, 锷, 锹, 锻, 镀, + 镁, 镂, 镇, 镉, 镊, 镌, 镍, 镏, 镐, 镑, 镔, 镕, 镖, 镜, 镣, 镭, 镯, 镰, 镳, 镶, 长, 门, 闩, 闪, 闫, + 闭, 问, 闯, 闰, 闲, 闳, 间, 闵, 闷, 闸, 闹, 闺, 闻, 闽, 闾, 阀, 阁, 阂, 阄, 阅, 阆, 阉, 阎, 阐, 阑, + 阔, 阕, 阖, 阙, 阚, 阜, 队, 阡, 阪, 阮, 阱, 防, 阳, 阴, 阵, 阶, 阻, 阿, 陀, 陂, 附, 际, 陆, 陇, 陈, + 陉, 陋, 陌, 降, 限, 陕, 陛, 陡, 院, 除, 陨, 险, 陪, 陬, 陵, 陶, 陷, 隅, 隆, 隋, 隍, 随, 隐, 隔, 隗, + 隘, 隙, 障, 隧, 隶, 隼, 隽, 难, 雀, 雁, 雄, 雅, 集, 雇, 雉, 雌, 雍, 雏, 雒, 雕, 雨, 雪, 雯, 雳, 零, + 雷, 雹, 雾, 需, 霁, 霄, 霆, 震, 霈, 霉, 霍, 霎, 霏, 霓, 霖, 霜, 霞, 霪, 露, 霸, 霹, 霾, 靑, 青, 靓, + 靖, 静, 靛, 非, 靠, 靡, 面, 革, 靳, 靴, 靶, 鞅, 鞋, 鞍, 鞑, 鞘, 鞠, 鞭, 韦, 韧, 韩, 韫, 韬, 韭, 音, + 韵, 韶, 页, 顶, 顷, 项, 顺, 须, 顽, 顾, 顿, 颀, 颁, 颂, 预, 颅, 领, 颇, 颈, 颊, 颌, 颍, 颐, 频, 颓, + 颖, 颗, 题, 颚, 颜, 额, 颠, 颢, 颤, 颦, 颧, 风, 飒, 飓, 飘, 飙, 飚, 飞, 食, 飧, 餍, 餐, 餮, 饕, 饥, + 饨, 饪, 饭, 饮, 饯, 饰, 饱, 饲, 饴, 饵, 饶, 饷, 饺, 饼, 饽, 饿, 馀, 馁, 馄, 馅, 馆, 馈, 馊, 馋, 馍, + 馏, 馑, 馒, 馕, 首, 馗, 香, 馥, 馨, 马, 驭, 驮, 驯, 驰, 驱, 驳, 驴, 驶, 驷, 驸, 驹, 驻, 驼, 驾, 驿, + 骁, 骂, 骄, 骅, 骆, 骇, 骈, 骊, 骋, 验, 骏, 骐, 骑, 骓, 骗, 骚, 骛, 骜, 骝, 骞, 骠, 骡, 骤, 骥, 骨, + 骰, 骷, 骸, 骺, 骼, 髂, 髅, 髋, 髌, 髓, 高, 髦, 髯, 鬃, 鬓, 鬟, 鬼, 魁, 魂, 魄, 魅, 魇, 魉, 魍, 魏, + 魔, 魟, 鱼, 鱿, 鲁, 鲅, 鲈, 鲍, 鲑, 鲜, 鲟, 鲠, 鲢, 鲤, 鲨, 鲫, 鲭, 鲳, 鲶, 鲷, 鲸, 鲼, 鳃, 鳄, 鳅, + 鳌, 鳍, 鳕, 鳖, 鳗, 鳝, 鳞, 鳟, 鸟, 鸠, 鸡, 鸢, 鸣, 鸥, 鸦, 鸩, 鸪, 鸫, 鸭, 鸯, 鸳, 鸵, 鸽, 鸾, 鸿, + 鹁, 鹂, 鹃, 鹅, 鹉, 鹊, 鹌, 鹏, 鹑, 鹜, 鹞, 鹤, 鹦, 鹧, 鹫, 鹭, 鹰, 鹳, 鹿, 麂, 麋, 麒, 麓, 麝, 麟, + 麦, 麸, 麻, 麾, 黄, 黍, 黎, 黏, 黑, 黔, 默, 黛, 黝, 黟, 黯, 鼎, 鼓, 鼠, 鼬, 鼹, 鼻, 鼾, 齐, 齿, 龃, + 龄, 龅, 龈, 龉, 龊, 龌, 龙, 龚, 龟, "\U0002B5AF", "\U0002B689"] + + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: *labels + batch_size: 32 + trim_silence: True + normalize: False + max_duration: 16.7 + shuffle: True + is_tarred: False + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + normalize: False + labels: *labels + batch_size: 32 + shuffle: False + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.02 + sample_rate: *sample_rate + window_stride: 0.01 + window: "hann" + features: &n_mels 64 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + rect_freq: 50 + rect_masks: 5 + rect_time: 120 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: 1 + residual: false + separable: *separable + stride: [2] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [2] + dropout: *dropout + filters: 512 + kernel: [87] + repeat: 1 + residual: false + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: &enc_filters 1024 + kernel: [1] + repeat: 1 + residual: false + stride: [1] + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: *enc_filters + num_classes: 5206 + vocabulary: *labels + + optim: + name: novograd + # _target_: nemo.core.optim.optimizers.Novograd + lr: .01 + # optimizer arguments + betas: [0.8, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + + # pytorch lightning args + # monitor: val_loss + # reduce_on_plateau: false + + # Scheduler params + warmup_steps: null + warmup_ratio: null + min_lr: 0.0 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 5 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True + checkpoint_callback_params: + monitor: "val_wer" + mode: "min" + create_wandb_logger: False + wandb_logger_kwargs: + name: null + project: null + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/asr/experimental/conformer/conformer_rnnt_bpe.yaml b/models/tacotron2/NeMo/examples/asr/experimental/conformer/conformer_rnnt_bpe.yaml new file mode 100644 index 0000000000..e7c2beac47 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/conformer/conformer_rnnt_bpe.yaml @@ -0,0 +1,201 @@ +# It contains the default values for training a Conformer ASR model medium size with RNNT loss and BPE-based vocabulary. +# Default learning parameters in this config are set for effective batch size of 2K. +# To train it with smaller batch sizes, you may need to re-tune the learning parameters or use higher accumulate_grad_batches. + +# You may find more info about Conformer-CTC here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/models.html#conformer-ctc +# Pre-trained models of Conformer-CTC can be found here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/results.html + +name: "Conformer-RNNT-BPE" + +model: + sample_rate: &sample_rate 16000 + compute_eval_loss: false # disables loss evaluation for validation and test data + log_prediction: true # enables logging sample predictions in the output during training + ctc_reduction: 'mean_batch' + + model_defaults: + se: true + se_context_size: -1 + # encoder / decoder / joint values + enc_hidden: ${model.encoder.d_model} + pred_hidden: 640 + joint_hidden: 640 + + train_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + batch_size: 16 # you may increase batch_size if your memory allows + shuffle: true + num_workers: 8 + pin_memory: false + use_start_end_token: true + trim_silence: false + max_duration: 16.7 # it is set for LibriSpeech, you may need to update it for your dataset + min_duration: 0.1 + + validation_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + batch_size: 16 + shuffle: false + num_workers: 8 + pin_memory: false + use_start_end_token: true + + test_ds: + manifest_filepath: null + sample_rate: ${model.sample_rate} + batch_size: 16 + shuffle: false + num_workers: 8 + pin_memory: false + use_start_end_token: true + + # you may find more detail on how to train a tokenizer at: /scripts/tokenizers/process_asr_text_tokenizer.py + tokenizer: + dir: ??? # path to directory which contains either tokenizer.model (bpe) or vocab.txt (for wpe) + type: wpe # Can be either bpe (SentencePiece tokenizer) or wpe (WordPiece tokenizer) + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + sample_rate: *sample_rate + normalize: "per_feature" + window_size: 0.025 + window_stride: 0.01 + window: "hann" + features: 80 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + pad_to: 0 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + freq_masks: 2 # set to zero to disable it + time_masks: 10 # set to zero to disable it + freq_width: 27 + time_width: 0.05 + + encoder: + _target_: nemo.collections.asr.modules.ConformerEncoder + feat_in: ${model.preprocessor.features} + feat_out: -1 # you may set it if you need different output size other than the default d_model + n_layers: 16 + d_model: 256 + + # Sub-sampling params + subsampling: striding # vggnet or striding + subsampling_factor: 4 # must be power of 2 + subsampling_conv_channels: -1 # set to -1 to make it equal to the d_model + + # Feed forward module's params + ff_expansion_factor: 4 + + # Multi-headed Attention Module's params + self_attention_model: rel_pos # rel_pos or abs_pos + n_heads: 4 + xscaling: true # scales up the inputs by sqrt(d_model) + untie_biases: true # unties the biases of the TransformerXL layers + pos_emb_max_len: 5000 + + # Convolution module's params + conv_kernel_size: 31 + + ### regularization + dropout: 0.1 # The dropout used in most of the Conformer Modules + dropout_emb: 0.0 # The dropout used for embeddings + dropout_att: 0.1 # The dropout for multi-headed attention modules + + decoder: + _target_: nemo.collections.asr.modules.RNNTDecoder + normalization_mode: null + random_state_sampling: false + blank_as_pad: true + + prednet: + pred_hidden: ${model.model_defaults.pred_hidden} + pred_rnn_layers: 1 + t_max: null + dropout: 0.1 + + joint: + _target_: nemo.collections.asr.modules.RNNTJoint + log_softmax: null # 'null' would set it automatically according to CPU/GPU device + + # fused mode + experimental_fuse_loss_wer: true + fused_batch_size: 16 + + jointnet: + joint_hidden: ${model.model_defaults.joint_hidden} + activation: "relu" + dropout: 0.1 + + decoding: + strategy: "greedy_batch" + + # greedy strategy config + greedy: + max_symbols: 30 + + # beam strategy config + beam: + beam_size: 2 + score_norm: true + tsd_max_sym_exp: 50 # for Time Synchronous Decoding + alsd_max_target_len: 2.0 # for Alignment-Length Synchronous Decoding + + optim: + name: adam + lr: 5.0 + # optimizer arguments + betas: [0.9, 0.98] + weight_decay: 1e-6 + + # scheduler setup + sched: + name: NoamAnnealing + d_model: ${model.encoder.d_model} + # scheduler config override + warmup_steps: 10000 + warmup_ratio: null + min_lr: 1e-6 + +trainer: + gpus: -1 # number of GPUs, -1 would use all available GPUs + num_nodes: 1 + max_epochs: 1000 + max_steps: null # computed at runtime if not set + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + accelerator: ddp + accumulate_grad_batches: 1 + gradient_clip_val: 0.0 + amp_level: O0 # O1/O2 for mixed precision + precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. + log_every_n_steps: 10 # Interval of logging. + progress_bar_refresh_rate: 10 + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs + sync_batchnorm: true + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager + + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: true + create_checkpoint_callback: true + checkpoint_callback_params: + # in case of multiple validation sets, first one is used + monitor: "val_wer" + mode: "min" + save_top_k: 3 + resume_if_exists: false + resume_ignore_no_checkpoint: false + + create_wandb_logger: false + wandb_logger_kwargs: + name: null + project: null diff --git a/models/tacotron2/NeMo/examples/asr/experimental/contextnet_rnnt/config_rnnt.yaml b/models/tacotron2/NeMo/examples/asr/experimental/contextnet_rnnt/config_rnnt.yaml new file mode 100644 index 0000000000..306ffe094c --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/contextnet_rnnt/config_rnnt.yaml @@ -0,0 +1,237 @@ +name: &name "ConvRNNT5x1" + +model: + sample_rate: 16000 + compute_eval_loss: true + + labels: [ " ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'" ] + + train_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + batch_size: 32 + trim_silence: true + max_duration: 16.7 + labels: ${model.labels} + shuffle: true + is_tarred: false + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + + validation_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + batch_size: 32 + shuffle: false + labels: ${model.labels} + + test_ds: + manifest_filepath: null + sample_rate: ${model.sample_rate} + batch_size: 32 + shuffle: false + labels: ${model.labels} + + model_defaults: + repeat: 5 + dropout: 0.0 + separable: true + se: true + se_context_size: -1 + # encoder / decoder / joint values + enc_hidden: 1024 + pred_hidden: 320 + joint_hidden: 320 + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.02 + sample_rate: ${model.sample_rate} + window_stride: 0.01 + window: "hann" + features: &n_mels 80 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + rect_freq: 50 + rect_masks: 5 + rect_time: 120 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: 128 + repeat: 1 + kernel: [11] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: 256 + repeat: ${model.model_defaults.repeat} + kernel: [13] + stride: [2] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + + - filters: 256 + repeat: ${model.model_defaults.repeat} + kernel: [15] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: 256 + repeat: ${model.model_defaults.repeat} + kernel: [17] + stride: [2] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + + - filters: 256 + repeat: ${model.model_defaults.repeat} + kernel: [19] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: 256 + repeat: 1 + kernel: [21] + stride: [2] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + + - filters: ${model.model_defaults.enc_hidden} + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + decoder: + _target_: nemo.collections.asr.modules.RNNTDecoder + normalization_mode: null + random_state_sampling: false + blank_as_pad: true + + prednet: + pred_hidden: ${model.model_defaults.pred_hidden} + pred_rnn_layers: 1 + t_max: null + dropout: 0.0 + + joint: + _target_: nemo.collections.asr.modules.RNNTJoint + log_softmax: null # sets it according to cpu/gpu device + + # fused mode + experimental_fuse_loss_wer: false + fused_batch_size: 1 + + jointnet: + joint_hidden: ${model.model_defaults.joint_hidden} + activation: "relu" + dropout: 0.0 + + decoding: + strategy: "greedy_batch" + + # greedy strategy config + greedy: + max_symbols: 30 + + # beam strategy config + beam: + beam_size: 2 + score_norm: true + tsd_max_sym_exp: 50 # for Time Synchronous Decoding + alsd_max_target_len: 2.0 # for Alignment-Length Synchronous Decoding + + optim: + name: adam + # _target_: nemo.core.optim.optimizers.Adam + lr: .1 + + # optimizer arguments + betas: [0.9, 0.999] + weight_decay: 0.0001 + + # scheduler setup + sched: + name: CosineAnnealing + + # scheduler config override + warmup_steps: null + warmup_ratio: 0.05 + min_lr: 1e-6 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 5 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + precision: 32 + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True + checkpoint_callback_params: + monitor: "val_wer" + mode: "min" + create_wandb_logger: False + wandb_logger_kwargs: + name: null + project: null diff --git a/models/tacotron2/NeMo/examples/asr/experimental/contextnet_rnnt/config_rnnt_bpe.yaml b/models/tacotron2/NeMo/examples/asr/experimental/contextnet_rnnt/config_rnnt_bpe.yaml new file mode 100644 index 0000000000..c259ba581c --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/contextnet_rnnt/config_rnnt_bpe.yaml @@ -0,0 +1,238 @@ +name: &name "ConvRNNTBPE5x1" + +model: + sample_rate: 16000 + compute_eval_loss: true + + tokenizer: + dir: ??? # path to directory which contains either tokenizer.model (bpe) or vocab.txt (for wpe) + type: ??? # Can be either bpe or wpe + + train_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + batch_size: 32 + trim_silence: true + max_duration: 16.7 + labels: [] + shuffle: true + is_tarred: false + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + + validation_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + batch_size: 32 + shuffle: false + labels: [] + + test_ds: + manifest_filepath: null + sample_rate: ${model.sample_rate} + batch_size: 32 + shuffle: false + labels: [] + + model_defaults: + repeat: 5 + dropout: 0.0 + separable: true + se: true + se_context_size: -1 + # encoder / decoder / joint values + enc_hidden: 1024 + pred_hidden: 320 + joint_hidden: 320 + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.02 + sample_rate: ${model.sample_rate} + window_stride: 0.01 + window: "hann" + features: &n_mels 80 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + rect_freq: 50 + rect_masks: 5 + rect_time: 120 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: 128 + repeat: 1 + kernel: [11] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: 256 + repeat: ${model.model_defaults.repeat} + kernel: [13] + stride: [2] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + + - filters: 256 + repeat: ${model.model_defaults.repeat} + kernel: [15] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: 256 + repeat: ${model.model_defaults.repeat} + kernel: [17] + stride: [2] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + + - filters: 256 + repeat: ${model.model_defaults.repeat} + kernel: [19] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: 256 + repeat: 1 + kernel: [21] + stride: [2] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + stride_last: true + residual_mode: "stride_add" + + - filters: ${model.model_defaults.enc_hidden} + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + decoder: + _target_: nemo.collections.asr.modules.RNNTDecoder + normalization_mode: null + random_state_sampling: false + blank_as_pad: true + + prednet: + pred_hidden: ${model.model_defaults.pred_hidden} + pred_rnn_layers: 1 + t_max: null + dropout: 0.0 + + joint: + _target_: nemo.collections.asr.modules.RNNTJoint + log_softmax: null # sets it according to cpu/gpu device + + # fused mode + experimental_fuse_loss_wer: false + fused_batch_size: 1 + + jointnet: + joint_hidden: ${model.model_defaults.joint_hidden} + activation: "relu" + dropout: 0.0 + + decoding: + strategy: "greedy_batch" + + # greedy strategy config + greedy: + max_symbols: 30 + + # beam strategy config + beam: + beam_size: 2 + score_norm: true + tsd_max_sym_exp: 50 # for Time Synchronous Decoding + alsd_max_target_len: 2.0 # for Alignment-Length Synchronous Decoding + + optim: + name: adam + # _target_: nemo.core.optim.optimizers.Adam + lr: .1 + + # optimizer arguments + betas: [0.9, 0.999] + weight_decay: 0.0001 + + # scheduler setup + sched: + name: CosineAnnealing + + # scheduler config override + warmup_steps: null + warmup_ratio: 0.05 + min_lr: 1e-6 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 5 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + precision: 32 + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True + checkpoint_callback_params: + monitor: "val_wer" + mode: "min" + create_wandb_logger: False + wandb_logger_kwargs: + name: null + project: null diff --git a/models/tacotron2/NeMo/examples/asr/experimental/quartznet/quartznet_15x5_aug.yaml b/models/tacotron2/NeMo/examples/asr/experimental/quartznet/quartznet_15x5_aug.yaml new file mode 100644 index 0000000000..8ebc2f477c --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/quartznet/quartznet_15x5_aug.yaml @@ -0,0 +1,290 @@ +name: &name "QuartzNet15x5" + +model: + sample_rate: &sample_rate 16000 + repeat: &repeat 5 + dropout: &dropout 0.0 + separable: &separable true + labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"] + + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: *labels + batch_size: 32 + trim_silence: True + max_duration: 16.7 + shuffle: True + is_tarred: False + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + augmentor: + rir_noise_aug: + prob: 0.5 + rir_manifest_path: ??? + rir_tar_filepaths: ??? + rir_prob: 0.5 + noise_manifest_paths: ??? + noise_tar_filepaths: ??? + min_snr_db: [0,0] + max_snr_db: [30,30] + orig_sample_rate: ??? + bg_noise_manifest_paths: ??? + bg_noise_tar_filepaths: ??? + bg_min_snr_db: [10,10] + bg_max_snr_db: [40,40] + bg_orig_sample_rate: ??? + transcode_aug: + prob: 0.1 + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: *labels + batch_size: 32 + shuffle: False + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.02 + sample_rate: *sample_rate + window_stride: 0.01 + window: "hann" + features: &n_mels 64 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + stft_conv: false + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + rect_freq: 50 + rect_masks: 5 + rect_time: 120 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: 1 + residual: false + separable: *separable + stride: [2] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [2] + dropout: *dropout + filters: 512 + kernel: [87] + repeat: 1 + residual: false + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: &enc_filters 1024 + kernel: [1] + repeat: 1 + residual: false + stride: [1] + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: *enc_filters + num_classes: 28 + vocabulary: *labels + + optim: + name: novograd + # _target_: nemo.core.optim.optimizers.Novograd + lr: .01 + # optimizer arguments + betas: [0.8, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + + # pytorch lightning args + # monitor: val_loss + # reduce_on_plateau: false + + # Scheduler params + warmup_steps: null + warmup_ratio: null + min_lr: 0.0 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + max_epochs: 5 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True + create_wandb_logger: False + wandb_logger_kwargs: + name: null + project: null + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/asr/experimental/sclite/speech_to_text_sclite.py b/models/tacotron2/NeMo/examples/asr/experimental/sclite/speech_to_text_sclite.py new file mode 100644 index 0000000000..b90258212d --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/sclite/speech_to_text_sclite.py @@ -0,0 +1,158 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script is based on speech_to_text_infer.py and allows you to score the hypotheses +with sclite. A local installation from https://github.com/usnistgov/SCTK is required. +Hypotheses and references are first saved in trn format and are scored after applying a glm +file (if provided). +""" + +import errno +import json +import os +import subprocess +from argparse import ArgumentParser + +import torch + +from nemo.collections.asr.metrics.wer import WER +from nemo.collections.asr.models import EncDecCTCModel +from nemo.utils import logging + +try: + from torch.cuda.amp import autocast +except ImportError: + from contextlib import contextmanager + + @contextmanager + def autocast(enabled=None): + yield + + +def score_with_sctk(sctk_dir, ref_fname, hyp_fname, out_dir, glm=""): + sclite_path = os.path.join(sctk_dir, "bin", "sclite") + if not os.path.exists(sclite_path): + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), sclite_path) + # apply glm + if os.path.exists(glm): + rfilter_path = os.path.join(sctk_dir, "bin", "rfilter1") + if not os.path.exists(rfilter_path): + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), rfilter_path) + hypglm = os.path.join(out_dir, os.path.basename(hyp_fname)) + ".glm" + rfilt_cmd = [rfilter_path] + [glm] + with open(hypglm, "w") as hypf, open(hyp_fname, "r") as hyp_in: + subprocess.run(rfilt_cmd, stdin=hyp_in, stdout=hypf) + refglm = os.path.join(out_dir, os.path.basename(ref_fname)) + ".glm" + with open(refglm, "w") as reff, open(ref_fname, "r") as ref_in: + subprocess.run(rfilt_cmd, stdin=ref_in, stdout=reff) + else: + refglm = ref_fname + hypglm = hyp_fname + + _ = subprocess.check_output(f"{sclite_path} -h {hypglm} -r {refglm} -i wsj -o all", shell=True) + + +can_gpu = torch.cuda.is_available() + + +def get_utt_info(manifest_path): + info_list = [] + with open(manifest_path, "r") as utt_f: + for line in utt_f: + utt = json.loads(line) + info_list.append(utt) + + return info_list + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "--asr_model", type=str, default="QuartzNet15x5Base-En", required=False, help="Pass: 'QuartzNet15x5Base-En'", + ) + parser.add_argument("--dataset", type=str, required=True, help="path to evaluation data") + parser.add_argument("--batch_size", type=int, default=4) + parser.add_argument( + "--dont_normalize_text", + default=False, + action='store_true', + help="Turn off trasnscript normalization. Recommended for non-English.", + ) + parser.add_argument("--out_dir", type=str, required=True, help="Destination dir for output files") + parser.add_argument("--sctk_dir", type=str, required=False, default="", help="Path to sctk root dir") + parser.add_argument("--glm", type=str, required=False, default="", help="Path to glm file") + args = parser.parse_args() + torch.set_grad_enabled(False) + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + use_sctk = os.path.exists(args.sctk_dir) + + if args.asr_model.endswith('.nemo'): + logging.info(f"Using local ASR model from {args.asr_model}") + asr_model = EncDecCTCModel.restore_from(restore_path=args.asr_model) + else: + logging.info(f"Using NGC cloud ASR model {args.asr_model}") + asr_model = EncDecCTCModel.from_pretrained(model_name=args.asr_model) + asr_model.setup_test_data( + test_data_config={ + 'sample_rate': 16000, + 'manifest_filepath': args.dataset, + 'labels': asr_model.decoder.vocabulary, + 'batch_size': args.batch_size, + 'normalize_transcripts': not args.dont_normalize_text, + } + ) + if can_gpu: + asr_model = asr_model.cuda() + asr_model.eval() + labels_map = dict([(i, asr_model.decoder.vocabulary[i]) for i in range(len(asr_model.decoder.vocabulary))]) + + wer = WER(vocabulary=asr_model.decoder.vocabulary) + hypotheses = [] + references = [] + all_log_probs = [] + for test_batch in asr_model.test_dataloader(): + if can_gpu: + test_batch = [x.cuda() for x in test_batch] + with autocast(): + log_probs, encoded_len, greedy_predictions = asr_model( + input_signal=test_batch[0], input_signal_length=test_batch[1] + ) + for r in log_probs.cpu().numpy(): + all_log_probs.append(r) + hypotheses += wer.ctc_decoder_predictions_tensor(greedy_predictions) + for batch_ind in range(greedy_predictions.shape[0]): + reference = ''.join([labels_map[c] for c in test_batch[2][batch_ind].cpu().detach().numpy()]) + references.append(reference) + del test_batch + + info_list = get_utt_info(args.dataset) + hypfile = os.path.join(args.out_dir, "hyp.trn") + reffile = os.path.join(args.out_dir, "ref.trn") + with open(hypfile, "w") as hyp_f, open(reffile, "w") as ref_f: + for i in range(len(hypotheses)): + utt_id = os.path.splitext(os.path.basename(info_list[i]['audio_filepath']))[0] + # rfilter in sctk likes each transcript to have a space at the beginning + hyp_f.write(" " + hypotheses[i] + " (" + utt_id + ")" + "\n") + ref_f.write(" " + references[i] + " (" + utt_id + ")" + "\n") + + if use_sctk: + score_with_sctk(args.sctk_dir, reffile, hypfile, args.out_dir, glm=args.glm) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/experimental/structured/conf/quartznet_15x5.yaml b/models/tacotron2/NeMo/examples/asr/experimental/structured/conf/quartznet_15x5.yaml new file mode 100644 index 0000000000..5bb9925c29 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/structured/conf/quartznet_15x5.yaml @@ -0,0 +1,235 @@ +name: &name "QuartzNet15x5" + +model: + sample_rate: &sample_rate 16000 + repeat: &repeat 5 + dropout: &dropout 0.0 + separable: &separable true + labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"] + + train_ds: + manifest_filepath: ??? + sample_rate: *sample_rate + labels: *labels + batch_size: 32 + max_duration: 16.7 + is_tarred: False + tarred_audio_filepaths: null + + validation_ds: + labels: *labels + manifest_filepath: ??? + sample_rate: *sample_rate + batch_size: 32 + + preprocessor: + sample_rate: *sample_rate + window_size: 0.02 + window_stride: 0.01 + features: &n_mels 64 + + spec_augment: + rect_freq: 50 + rect_masks: 5 + rect_time: 120 + + encoder: + feat_in: *n_mels + activation: relu + + jasper: + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: 1 + residual: false + separable: *separable + stride: [2] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [33] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 256 + kernel: [39] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [51] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [63] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: 512 + kernel: [75] + repeat: *repeat + residual: true + separable: *separable + stride: [1] + + - dilation: [2] + dropout: *dropout + filters: 512 + kernel: [87] + repeat: 1 + residual: false + separable: *separable + stride: [1] + + - dilation: [1] + dropout: *dropout + filters: &enc_filters 1024 + kernel: [1] + repeat: 1 + residual: false + stride: [1] + + decoder: + feat_in: *enc_filters + num_classes: 28 + vocabulary: *labels + + optim: + name: novograd + lr: .01 + + # optimizer arguments + betas: [0.8, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + + # Scheduler params + warmup_steps: null + warmup_ratio: null + min_lr: 0.0 + +trainer: + gpus: 0 # number of gpus + max_epochs: 5 + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + name: *name diff --git a/models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_hybrid.py b/models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_hybrid.py new file mode 100644 index 0000000000..e396c447e4 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_hybrid.py @@ -0,0 +1,56 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.asr.models import EncDecCTCModel, configs +from nemo.core.config import hydra_runner +from nemo.utils.config_utils import update_model_config +from nemo.utils.exp_manager import exp_manager + + +""" +python speech_to_text_hybrid.py \ + --config-path="conf/quartznet" \ + --config-name="quartznet_15x5" \ + model.train_ds.manifest_filepath="/home/smajumdar/PycharmProjects/NeMo-som/examples/asr/an4/train_manifest.json" \ + model.validation_ds.manifest_filepath="/home/smajumdar/PycharmProjects/NeMo-som/examples/asr/an4/test_manifest.json" \ + trainer.gpus=1 +""" + + +@hydra_runner(config_path="conf/quartznet", config_name="quartznet_15x5") +def main(cfg): + # Generate default asr model config + asr_model_config = configs.EncDecCTCModelConfig() + + # Merge hydra updates with model config + # `drop_missing_subconfig=True` is necessary here. Without it, while the data class will instantiate and be added + # to the config, it contains test_ds.sample_rate = MISSING and test_ds.labels = MISSING. + # This will raise a OmegaConf MissingMandatoryValue error when processing the dataloaders inside + # model_utils.resolve_test_dataloaders(model=self) (used for multi data loader support). + # In general, any operation that tries to use a DictConfig with MISSING in it will fail, + # other than explicit update operations to change MISSING to some actual value. + asr_model_config = update_model_config(asr_model_config, cfg, drop_missing_subconfigs=True) + + # From here on out, its a general OmegaConf DictConfig, directly usable by our code. + trainer = pl.Trainer(**asr_model_config.trainer) + exp_manager(trainer, asr_model_config.get("exp_manager", None)) + asr_model = EncDecCTCModel(cfg=asr_model_config.model, trainer=trainer) + + trainer.fit(asr_model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_structured.py b/models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_structured.py new file mode 100644 index 0000000000..23f33f04f3 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_structured.py @@ -0,0 +1,146 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import asdict + +import pytorch_lightning as pl + +import nemo.collections.asr as nemo_asr +from nemo.collections.asr.models import EncDecCTCModel, configs +from nemo.utils.exp_manager import exp_manager + +""" +python speech_to_text_structured.py +""" + +# Generate default asr model config +cfg = configs.EncDecCTCModelConfig() + +# set global values +cfg.model.repeat = 5 +cfg.model.separable = True + +# fmt: off +LABELS = [ + " ", "a", "b", "c", "d", "e", + "f", "g", "h", "i", "j", "k", + "l", "m", "n", "o", "p", "q", + "r", "s", "t", "u", "v", "w", + "x", "y", "z", "'", +] +# fmt: on + +qn_15x5 = [ + nemo_asr.modules.conv_asr.JasperEncoderConfig( + filters=256, + repeat=1, + kernel=[33], + stride=[2], + separable=cfg.model.separable, + dilation=[1], + dropout=cfg.model.dropout, + residual=False, + ), + nemo_asr.modules.conv_asr.JasperEncoderConfig( + filters=256, + repeat=1, + kernel=[33], + stride=[1], + separable=cfg.model.separable, + dilation=[1], + dropout=cfg.model.dropout, + residual=True, + ), + # ... repeat 14 more times + nemo_asr.modules.conv_asr.JasperEncoderConfig( + filters=1024, repeat=1, kernel=[1], stride=[1], dilation=[1], dropout=cfg.model.dropout, residual=False, + ), +] + + +def main(): + # Update values + # MODEL UPDATES + cfg.name = "Mini QuartzNet" + cfg.model.labels = LABELS + + # train ds + cfg.model.train_ds.manifest_filepath = "" + cfg.model.train_ds.labels = LABELS + cfg.model.train_ds.sample_rate = cfg.model.sample_rate + + # validation ds + cfg.model.validation_ds.manifest_filepath = "" + cfg.model.validation_ds.labels = LABELS + cfg.model.validation_ds.sample_rate = cfg.model.sample_rate + + # del `test_ds` does not work! + # Refer - https://stackoverflow.com/questions/58119758/how-to-remove-dataclass-attributes + # Hydra/OmegaConf dont allow custom .asdict() methods either + # For now, explicitly set parameters + cfg.model.test_ds.sample_rate = cfg.model.sample_rate + cfg.model.test_ds.labels = cfg.model.labels + + # preprocessor + cfg.model.preprocessor.sample_rate = cfg.model.sample_rate + + # spec aug + cfg.model.spec_augment.rect_masks = 5 + cfg.model.spec_augment.rect_freq = 50 + cfg.model.spec_augment.rect_time = 120 + + # encoder + cfg.model.encoder.feat_in = cfg.model.preprocessor.features + cfg.model.encoder.activation = 'relu' + cfg.model.encoder.jasper = qn_15x5 + + # decoder + cfg.model.decoder.feat_in = qn_15x5[-1].filters + cfg.model.decoder.num_classes = len(LABELS) + cfg.model.decoder.vocabulary = LABELS + + # optim + cfg.model.optim.name = 'novograd' + cfg.model.optim.lr = 0.01 + + # `betas` dont exist inside the base config, + # so they cannot be added as such! + # Same for `weight_decay`. + cfg.model.optim.betas = [0.8, 0.5] + cfg.model.optim.weight_decay = 0.001 + + # sched + # As parameters such as warmup_steps and warmup_ratio + # dont exist inside the shell config, these values are not added! + cfg.model.optim.sched.name = "CosineAnnealing" + cfg.model.optim.sched.warmup_steps = None + cfg.model.optim.sched.warmup_ratio = 0.01 + + # Trainer config + cfg.trainer.gpus = 1 + cfg.trainer.max_epochs = 5 + + # Exp Manager config + cfg.exp_manager.name = cfg.name + + # Note usage of asdict + trainer = pl.Trainer(**asdict(cfg.trainer)) + exp_manager(trainer, asdict(cfg.exp_manager)) + asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer) + + trainer.fit(asr_model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_structured_v2.py b/models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_structured_v2.py new file mode 100644 index 0000000000..1fc9cc8341 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/structured/speech_to_text_structured_v2.py @@ -0,0 +1,90 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import asdict + +import pytorch_lightning as pl + +from nemo.collections.asr.models import EncDecCTCModel, configs +from nemo.core.config import modelPT, optimizers, schedulers +from nemo.utils.exp_manager import exp_manager + +""" +python speech_to_text_structured_v2.py +""" + +# fmt: off +LABELS = [ + " ", "a", "b", "c", "d", "e", + "f", "g", "h", "i", "j", "k", + "l", "m", "n", "o", "p", "q", + "r", "s", "t", "u", "v", "w", + "x", "y", "z", "'", +] + +optim_cfg = optimizers.NovogradParams( + lr=0.01, + betas=(0.8, 0.5), + weight_decay=0.001 +) + +sched_cfg = schedulers.CosineAnnealingParams( + warmup_steps=None, + warmup_ratio=None, + min_lr=0.0, +) +# fmt: on + + +def main(): + # NeMo Model config + cfg = modelPT.NemoConfig(name='Custom QuartzNet') + + # Generate default asr model config + builder = configs.EncDecCTCModelConfigBuilder(name='quartznet_15x5') + + # set model global values + builder.set_labels(LABELS) + builder.set_optim(cfg=optim_cfg, sched_cfg=sched_cfg) + + model_cfg = builder.build() + + # set the model config to the NeMo Model + cfg.model = model_cfg + + # Update values + # MODEL UPDATES + # train ds + model_cfg.train_ds.manifest_filepath = "" + + # validation ds + model_cfg.validation_ds.manifest_filepath = "" + + # Trainer config + cfg.trainer.gpus = 1 + cfg.trainer.max_epochs = 5 + + # Exp Manager config + cfg.exp_manager.name = cfg.name + + # Note usage of asdict + trainer = pl.Trainer(**asdict(cfg.trainer)) + exp_manager(trainer, asdict(cfg.exp_manager)) + asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer) + + trainer.fit(asr_model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/experimental/wav2vec/configs/wav2vec_pretrain.yaml b/models/tacotron2/NeMo/examples/asr/experimental/wav2vec/configs/wav2vec_pretrain.yaml new file mode 100644 index 0000000000..00268fcefe --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/wav2vec/configs/wav2vec_pretrain.yaml @@ -0,0 +1,123 @@ +name: &name "Wav2vec_Pretrain" +# For more details on the model parameters, see nemo/collections/asr/models/wav2vec/wav2vec_config.py +# These parameters are based off the FairSeq implementation. +# See here: https://github.com/pytorch/fairseq/blob/master/examples/wav2vec/config/pretraining/wav2vec2_base_librispeech.yaml +model: + transformer_encoder: + dropout: 0.1 # Dropout probability for the transformer + conv: # Config for convolutional model that generates positional embeddings required for attention layer + conv_pos: 128 # Number of filters for convolutional positional embeddings + conv_pos_groups: 16 # Number of groups for convolutional positional embeddings + encoder: + encoder_layers: 12 # Number of encoder layers in transformer model + encoder_layerdrop: 0.05 # Probability of dropping transformer layers + embedding_dim: 768 # Encoder embedding dim + ffn_embedding_dim: 3072 # Encoder embedding dim for feed forward + num_attention_heads: 8 # Number of encoder attention heads + dropout: 0.1 # Dropout probability for transformer encoder + activation_fn: gelu # Activation for transformer + layer_norm_first: False # Apply layer norm first within the transformer + loss: + prob_ppl_weight: 0.1 # Weight applied to quantized prob perplexity loss + feature_loss_weight: 0 # Weight applied to feature L2 Norm + quantizer: + quantize_targets: True # Quantize targets + quantize_input: False # Quantize inputs + same_quantizer: False # Use the same quantizer for inputs and targets (If both enabled) + latent_vars: 320 # Number of latent variables in each group of the codebook + latent_groups: 2 # Number of groups within the codebook + latent_dim: 0 # If greater than 0, use the dim for latest variables else inferred by final dim / latent groups + latent_temp: [2, 0.5, 0.999995] # Quantize temperature (start, stop, decay factor) + conv_feature_encoder: # Config for feature extractor taking raw audio waveform and converting to features + extractor_mode: default # Mode for feature extractor. [Default, layer_norm] + conv_bias: False # Include bias in convolution feature extractor model + masking: + mask_prob: 0.65 # Probability of replacing token with mask + mask_type: static # Used to select configuration to compute mask lengths [static, uniform, normal, poisson] + mask_other: 0 # Secondary mask used for complex distributions (see help in compute_mask_indices) + mask_length: 10 # Length of mask when masking time steps + no_mask_overlap: False # Whether to allow masks to overlap + mask_min_space: 1 # Minimum space beetween spans (if no overlap is enabled) + mask_channel_prob: 0 # Probability of replacing a feature with 0 + mask_channel_type: static # Used to select configuration to compute mask channel lengths [static, uniform, normal, poisson] + mask_channel_other: 0 # Secondary mask argument (used for more complex distributions (see help in compute_mask_indices) + mask_channel_length: 10 # Length of masks for features (channels) + no_mask_channel_overlap: False # Whether to allow channel masks to overlap + mask_channel_min_space: 1 # Minimum space between spans (if no overlap is enabled) + dropout_input: 0.1 # Dropout applied to input raw features + dropout_features: 0.1 # Dropout applied to the features generator by convolutions + final_dim: 0 # Project final representations and targets to this dimension + n_negatives: 100 # Number of negatives to sample from the same audio sample + cross_sample_negatives: 0 # Number of negatives to sample from any sample in the batch + codebook_negatives: 0 # Number of negative examples in codebook + negatives_from_everywhere: False # Sample negatives from everywhere, not just masked states + logit_temp: 0.1 # Temperature to divide logits by + target_glu: False # Adds project and applies GLU to targets + feature_grad_mult: 0.1 # Multiply extracted feature gradients + + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 32 + shuffle: True + num_workers: 4 + labels: [ ] # No vocab needed for pre-training + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 32 + shuffle: False + num_workers: 1 + labels: [ ] + test_ds: + manifest_filepath: null + sample_rate: 16000 + batch_size: 32 + shuffle: False + num_workers: 4 + labels: [ ] + optim: + name: adamw + lr: 0.0005 + eps: 1e-06 + # optimizer arguments + betas: [ 0.9, 0.98 ] + weight_decay: 0.01 + + # scheduler setup + sched: + name: PolynomialDecayAnnealing + min_lr: 0.000 + # Scheduler params + warmup_steps: null + warmup_ratio: 0.1 +trainer: + gpus: 0 # number of gpus + num_nodes: 1 + max_epochs: 100 + max_steps: null # computed at runtime if not set + val_check_interval: 0.5 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + accelerator: ddp + accumulate_grad_batches: 1 + gradient_clip_val: 0.0 + amp_level: O0 # O1/O2 for mixed precision + precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. + log_every_n_steps: 10 # Interval of logging. + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs + sync_batchnorm: true + + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: true + create_checkpoint_callback: true + create_wandb_logger: false + wandb_logger_kwargs: + name: null + project: null + resume_if_exists: false + resume_ignore_no_checkpoint: false \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/asr/experimental/wav2vec/configs/wav2vec_pretrain_large.yaml b/models/tacotron2/NeMo/examples/asr/experimental/wav2vec/configs/wav2vec_pretrain_large.yaml new file mode 100644 index 0000000000..4a92a281c4 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/wav2vec/configs/wav2vec_pretrain_large.yaml @@ -0,0 +1,123 @@ +name: &name "Wav2vec_large_Pretrain" +# For more details on the model parameters, see nemo/collections/asr/models/wav2vec/wav2vec_config.py +# These parameters are based off the FairSeq implementation. +# See here: https://github.com/pytorch/fairseq/blob/master/examples/wav2vec/config/pretraining/wav2vec2_large_librivox.yaml +model: + transformer_encoder: + dropout: 0.0 # Dropout probability for the transformer + conv: # Config for convolutional model that generates positional embeddings required for attention layer + conv_pos: 128 # Number of filters for convolutional positional embeddings + conv_pos_groups: 16 # Number of groups for convolutional positional embeddings + encoder: + encoder_layers: 24 # Number of encoder layers in transformer model + encoder_layerdrop: 0.0 # Probability of dropping transformer layers + embedding_dim: 1024 # Encoder embedding dim + ffn_embedding_dim: 4096 # Encoder embedding dim for feed forward + num_attention_heads: 16 # Number of encoder attention heads + dropout: 0.0 # Dropout probability for transformer encoder + activation_fn: gelu # Activation for transformer + layer_norm_first: True # Apply layer norm first within the transformer + loss: + prob_ppl_weight: 0.1 # Weight applied to quantized prob perplexity loss + feature_loss_weight: 0 # Weight applied to feature L2 Norm + quantizer: + quantize_targets: True # Quantize targets + quantize_input: False # Quantize inputs + same_quantizer: False # Use the same quantizer for inputs and targets (If both enabled) + latent_vars: 320 # Number of latent variables in each group of the codebook + latent_groups: 2 # Number of groups within the codebook + latent_dim: 0 # If greater than 0, use the dim for latest variables else inferred by final dim / latent groups + latent_temp: [2, 0.1, 0.999995] # Quantize temperature (start, stop, decay factor) + conv_feature_encoder: # Config for feature extractor taking raw audio waveform and converting to features + extractor_mode: layer_norm # Mode for feature extractor. [default, layer_norm] + conv_bias: True # Include bias in convolution feature extractor model + masking: + mask_prob: 0.65 # Probability of replacing token with mask + mask_type: static # Used to select configuration to compute mask lengths [static, uniform, normal, poisson] + mask_other: 0 # Secondary mask used for complex distributions (see help in compute_mask_indices) + mask_length: 10 # Length of mask when masking time steps + no_mask_overlap: False # Whether to allow masks to overlap + mask_min_space: 1 # Minimum space beetween spans (if no overlap is enabled) + mask_channel_prob: 0 # Probability of replacing a feature with 0 + mask_channel_type: static # Used to select configuration to compute mask channel lengths [static, uniform, normal, poisson] + mask_channel_other: 0 # Secondary mask argument (used for more complex distributions (see help in compute_mask_indices) + mask_channel_length: 10 # Length of masks for features (channels) + no_mask_channel_overlap: False # Whether to allow channel masks to overlap + mask_channel_min_space: 1 # Minimum space between spans (if no overlap is enabled) + dropout_input: 0.0 # Dropout applied to input raw features + dropout_features: 0.0 # Dropout applied to the features generator by convolutions + final_dim: 768 # Project final representations and targets to this dimension + n_negatives: 100 # Number of negatives to sample from the same audio sample + cross_sample_negatives: 0 # Number of negatives to sample from any sample in the batch + codebook_negatives: 0 # Number of negative examples in codebook + negatives_from_everywhere: False # Sample negatives from everywhere, not just masked states + logit_temp: 0.1 # Temperature to divide logits by + target_glu: False # Adds project and applies GLU to targets + feature_grad_mult: 1.0 # Multiply extracted feature gradients + + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 32 + shuffle: True + num_workers: 4 + labels: [ ] # No vocab needed for pre-training + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 32 + shuffle: False + num_workers: 1 + labels: [ ] + test_ds: + manifest_filepath: null + sample_rate: 16000 + batch_size: 32 + shuffle: False + num_workers: 4 + labels: [ ] + optim: + name: adamw + lr: 0.0005 + eps: 1e-06 + # optimizer arguments + betas: [ 0.9, 0.98 ] + weight_decay: 0.01 + + # scheduler setup + sched: + name: PolynomialDecayAnnealing + min_lr: 0.000 + # Scheduler params + warmup_steps: null + warmup_ratio: 0.1 +trainer: + gpus: 0 # number of gpus + num_nodes: 1 + max_epochs: 100 + max_steps: null # computed at runtime if not set + val_check_interval: 0.5 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + accelerator: ddp + accumulate_grad_batches: 1 + gradient_clip_val: 0.0 + amp_level: O0 # O1/O2 for mixed precision + precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. + log_every_n_steps: 10 # Interval of logging. + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs + sync_batchnorm: true + + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: true + create_checkpoint_callback: true + create_wandb_logger: false + wandb_logger_kwargs: + name: null + project: null + resume_if_exists: false + resume_ignore_no_checkpoint: false \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/asr/experimental/wav2vec/wav2vec_pretrain.py b/models/tacotron2/NeMo/examples/asr/experimental/wav2vec/wav2vec_pretrain.py new file mode 100644 index 0000000000..2e5c1db320 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/experimental/wav2vec/wav2vec_pretrain.py @@ -0,0 +1,97 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.asr.models.wav2vec.wav2vec_model import Wav2VecEncoderModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +""" +Pre-train a wav2vec 2.0 transformer model on audio. Uses a contrastive loss function to pre-train on unlabelled audio, +using a task similar to masked language modeling in NLP. In wav2vec, we mask portions of the audio +and the model is trained by minimising the distance of the ground truth for the masked section, +using the ground truth quantized codebook representation. Distractors are obtained from other time steps. +See :class:`Wav2VecCriterion` for more information. + +Reference: https://arxiv.org/abs/2006.11477 + + python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \ + model.train_ds.manifest_filepath="./examples/asr/train.tsv" \ + model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \ + trainer.gpus=1 \ + trainer.max_epochs=100 + +Basic run (on CPU for 50 epochs): + python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \ + model.train_ds.manifest_filepath="./examples/asr/train.tsv" \ + model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \ + trainer.gpus=1 \ + trainer.max_epochs=50 + +Using wav2vec-large with mixed precision: + python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \ + --config-name=wav2vec_pretrain_large \ + model.train_ds.manifest_filepath="./examples/asr/train.tsv" \ + model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \ + trainer.gpus=1 \ + trainer.max_epochs=100 \ + trainer.precision=16 + +Add PyTorch Lightning Trainer arguments from CLI: + python wav2vec.py \ + ... \ + +trainer.fast_dev_run=true + +Hydra logs will be found in "$(./outputs/$(date +"%y-%m-%d")/$(date +"%H-%M-%S")/.hydra)" +PTL logs will be found in "$(./outputs/$(date +"%y-%m-%d")/$(date +"%H-%M-%S")/lightning_logs)" + +Override some args of optimizer: + python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \ + model.train_ds.manifest_filepath="./examples/asr/train.tsv" \ + model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \ + trainer.gpus=2 \ + trainer.max_epochs=2 \ + model.optim.args.params.betas=[0.8,0.5] \ + model.optim.args.params.weight_decay=0.0001 + +Override optimizer entirely + python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \ + model.train_ds.manifest_filepath="./examples/asr/train.tsv" \ + model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \ + trainer.gpus=2 \ + trainer.max_epochs=2 \ + ~model.optim.args \ + +model.optim.args.betas=[0.8,0.5]\ + +model.optim.args.weight_decay=0.0005 + +""" + + +@hydra_runner(config_path="configs", config_name="wav2vec_pretrain") +def main(cfg: DictConfig): + logging.info("Application config\n" + OmegaConf.to_yaml(cfg)) + + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + + wav2vec_encoder_model = Wav2VecEncoderModel(cfg=cfg.model, trainer=trainer) + trainer.fit(wav2vec_encoder_model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_calibrate.py b/models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_calibrate.py new file mode 100644 index 0000000000..264806c7b1 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_calibrate.py @@ -0,0 +1,160 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Script for calibrating a pretrained ASR model for quantization +""" + +from argparse import ArgumentParser + +import torch +from omegaconf import open_dict + +from nemo.collections.asr.models import EncDecCTCModel +from nemo.utils import logging + +try: + from pytorch_quantization import calib + from pytorch_quantization import nn as quant_nn + from pytorch_quantization import quant_modules + from pytorch_quantization.tensor_quant import QuantDescriptor +except ImportError: + raise ImportError( + "pytorch-quantization is not installed. Install from " + "https://github.com/NVIDIA/TensorRT/tree/master/tools/pytorch-quantization." + ) + +try: + from torch.cuda.amp import autocast +except ImportError: + from contextlib import contextmanager + + @contextmanager + def autocast(enabled=None): + yield + + +can_gpu = torch.cuda.is_available() + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "--asr_model", type=str, default="QuartzNet15x5Base-En", required=True, help="Pass: 'QuartzNet15x5Base-En'", + ) + parser.add_argument("--dataset", type=str, required=True, help="path to evaluation data") + parser.add_argument("--batch_size", type=int, default=256) + parser.add_argument( + "--dont_normalize_text", + default=False, + action='store_false', + help="Turn off trasnscript normalization. Recommended for non-English.", + ) + parser.add_argument('--num_calib_batch', default=1, type=int, help="Number of batches for calibration.") + parser.add_argument('--calibrator', type=str, choices=["max", "histogram"], default="max") + parser.add_argument('--percentile', nargs='+', type=float, default=[99.9, 99.99, 99.999, 99.9999]) + parser.add_argument("--amp", action="store_true", help="Use AMP in calibration.") + parser.set_defaults(amp=False) + + args = parser.parse_args() + torch.set_grad_enabled(False) + + # Initialize quantization + quant_desc_input = QuantDescriptor(calib_method=args.calibrator) + quant_nn.QuantConv2d.set_default_quant_desc_input(quant_desc_input) + quant_nn.QuantConvTranspose2d.set_default_quant_desc_input(quant_desc_input) + quant_nn.QuantLinear.set_default_quant_desc_input(quant_desc_input) + + if args.asr_model.endswith('.nemo'): + logging.info(f"Using local ASR model from {args.asr_model}") + asr_model_cfg = EncDecCTCModel.restore_from(restore_path=args.asr_model, return_config=True) + with open_dict(asr_model_cfg): + asr_model_cfg.encoder.quantize = True + asr_model = EncDecCTCModel.restore_from(restore_path=args.asr_model, override_config_path=asr_model_cfg) + + else: + logging.info(f"Using NGC cloud ASR model {args.asr_model}") + asr_model_cfg = EncDecCTCModel.from_pretrained(model_name=args.asr_model, return_config=True) + with open_dict(asr_model_cfg): + asr_model_cfg.encoder.quantize = True + asr_model = EncDecCTCModel.from_pretrained(model_name=args.asr_model, override_config_path=asr_model_cfg) + + asr_model.setup_test_data( + test_data_config={ + 'sample_rate': 16000, + 'manifest_filepath': args.dataset, + 'labels': asr_model.decoder.vocabulary, + 'batch_size': args.batch_size, + 'normalize_transcripts': args.dont_normalize_text, + 'shuffle': True, + } + ) + asr_model.preprocessor.featurizer.dither = 0.0 + asr_model.preprocessor.featurizer.pad_to = 0 + if can_gpu: + asr_model = asr_model.cuda() + asr_model.eval() + + # Enable calibrators + for name, module in asr_model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + if module._calibrator is not None: + module.disable_quant() + module.enable_calib() + else: + module.disable() + + for i, test_batch in enumerate(asr_model.test_dataloader()): + if can_gpu: + test_batch = [x.cuda() for x in test_batch] + if args.amp: + with autocast(): + _ = asr_model(input_signal=test_batch[0], input_signal_length=test_batch[1]) + else: + _ = asr_model(input_signal=test_batch[0], input_signal_length=test_batch[1]) + if i >= args.num_calib_batch: + break + + # Save calibrated model(s) + model_name = args.asr_model.replace(".nemo", "") if args.asr_model.endswith(".nemo") else args.asr_model + if not args.calibrator == "histogram": + compute_amax(asr_model, method="max") + asr_model.save_to(F"{model_name}-max-{args.num_calib_batch*args.batch_size}.nemo") + else: + for percentile in args.percentile: + print(F"{percentile} percentile calibration") + compute_amax(asr_model, method="percentile") + asr_model.save_to(F"{model_name}-percentile-{percentile}-{args.num_calib_batch*args.batch_size}.nemo") + + for method in ["mse", "entropy"]: + print(F"{method} calibration") + compute_amax(asr_model, method=method) + asr_model.save_to(F"{model_name}-{method}-{args.num_calib_batch*args.batch_size}.nemo") + + +def compute_amax(model, **kwargs): + for name, module in model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + if module._calibrator is not None: + if isinstance(module._calibrator, calib.MaxCalibrator): + module.load_calib_amax() + else: + module.load_calib_amax(**kwargs) + print(F"{name:40}: {module}") + if can_gpu: + model.cuda() + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_quant_infer.py b/models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_quant_infer.py new file mode 100644 index 0000000000..b072fe8e90 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_quant_infer.py @@ -0,0 +1,216 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Script for post training quantization of ASR models +""" + +import collections +from argparse import ArgumentParser +from pprint import pprint + +import torch +from omegaconf import open_dict + +from nemo.collections.asr.metrics.wer import WER, word_error_rate +from nemo.collections.asr.models import EncDecCTCModel +from nemo.utils import logging + +try: + from pytorch_quantization import nn as quant_nn + from pytorch_quantization import quant_modules +except ImportError: + raise ImportError( + "pytorch-quantization is not installed. Install from " + "https://github.com/NVIDIA/TensorRT/tree/master/tools/pytorch-quantization." + ) + + +try: + from torch.cuda.amp import autocast +except ImportError: + from contextlib import contextmanager + + @contextmanager + def autocast(enabled=None): + yield + + +can_gpu = torch.cuda.is_available() + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "--asr_model", type=str, default="QuartzNet15x5Base-En", required=True, help="Pass: 'QuartzNet15x5Base-En'", + ) + parser.add_argument("--dataset", type=str, required=True, help="path to evaluation data") + parser.add_argument("--wer_target", type=float, default=None, help="used by test") + parser.add_argument("--batch_size", type=int, default=4) + parser.add_argument("--wer_tolerance", type=float, default=1.0, help="used by test") + parser.add_argument( + "--dont_normalize_text", + default=False, + action='store_false', + help="Turn off trasnscript normalization. Recommended for non-English.", + ) + parser.add_argument( + "--use_cer", default=False, action='store_true', help="Use Character Error Rate as the evaluation metric" + ) + parser.add_argument('--sensitivity', action="store_true", help="Perform sensitivity analysis") + parser.add_argument('--onnx', action="store_true", help="Export to ONNX") + parser.add_argument('--quant-disable-keyword', type=str, nargs='+', help='disable quantizers by keyword') + args = parser.parse_args() + torch.set_grad_enabled(False) + + quant_modules.initialize() + + if args.asr_model.endswith('.nemo'): + logging.info(f"Using local ASR model from {args.asr_model}") + asr_model_cfg = EncDecCTCModel.restore_from(restore_path=args.asr_model, return_config=True) + with open_dict(asr_model_cfg): + asr_model_cfg.encoder.quantize = True + asr_model = EncDecCTCModel.restore_from(restore_path=args.asr_model, override_config_path=asr_model_cfg) + + else: + logging.info(f"Using NGC cloud ASR model {args.asr_model}") + asr_model_cfg = EncDecCTCModel.from_pretrained(model_name=args.asr_model, return_config=True) + with open_dict(asr_model_cfg): + asr_model_cfg.encoder.quantize = True + asr_model = EncDecCTCModel.from_pretrained(model_name=args.asr_model, override_config_path=asr_model_cfg) + asr_model.setup_test_data( + test_data_config={ + 'sample_rate': 16000, + 'manifest_filepath': args.dataset, + 'labels': asr_model.decoder.vocabulary, + 'batch_size': args.batch_size, + 'normalize_transcripts': args.dont_normalize_text, + } + ) + asr_model.preprocessor.featurizer.dither = 0.0 + asr_model.preprocessor.featurizer.pad_to = 0 + if can_gpu: + asr_model = asr_model.cuda() + asr_model.eval() + + if args.quant_disable_keyword: + for name, module in asr_model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + for keyword in args.quant_disable_keyword: + if keyword in name: + logging.warning(F"Disable {name}") + module.disable() + + labels_map = dict([(i, asr_model.decoder.vocabulary[i]) for i in range(len(asr_model.decoder.vocabulary))]) + wer = WER(vocabulary=asr_model.decoder.vocabulary, use_cer=args.use_cer) + wer_quant = evaluate(asr_model, labels_map, wer) + logging.info(f'Got WER of {wer_quant}. Tolerance was {args.wer_tolerance}') + + if args.sensitivity: + if wer_quant < args.wer_tolerance: + logging.info("Tolerance is already met. Skip sensitivity analyasis.") + return + quant_layer_names = [] + for name, module in asr_model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + module.disable() + layer_name = name.replace("._input_quantizer", "").replace("._weight_quantizer", "") + if layer_name not in quant_layer_names: + quant_layer_names.append(layer_name) + logging.info(F"{len(quant_layer_names)} quantized layers found.") + + # Build sensitivity profile + quant_layer_sensitivity = {} + for i, quant_layer in enumerate(quant_layer_names): + logging.info(F"Enable {quant_layer}") + for name, module in asr_model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer) and quant_layer in name: + module.enable() + logging.info(F"{name:40}: {module}") + + # Eval the model + wer_value = evaluate(asr_model, labels_map, wer) + logging.info(F"WER: {wer_value}") + quant_layer_sensitivity[quant_layer] = args.wer_tolerance - wer_value + + for name, module in asr_model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer) and quant_layer in name: + module.disable() + logging.info(F"{name:40}: {module}") + + # Skip most sensitive layers until WER target is met + for name, module in asr_model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + module.enable() + quant_layer_sensitivity = collections.OrderedDict(sorted(quant_layer_sensitivity.items(), key=lambda x: x[1])) + pprint(quant_layer_sensitivity) + skipped_layers = [] + for quant_layer, _ in quant_layer_sensitivity.items(): + for name, module in asr_model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + if quant_layer in name: + logging.info(F"Disable {name}") + if not quant_layer in skipped_layers: + skipped_layers.append(quant_layer) + module.disable() + wer_value = evaluate(asr_model, labels_map, wer) + if wer_value <= args.wer_tolerance: + logging.info( + F"WER tolerance {args.wer_tolerance} is met by skipping {len(skipped_layers)} sensitive layers." + ) + print(skipped_layers) + export_onnx(args, asr_model) + return + raise ValueError(f"WER tolerance {args.wer_tolerance} can not be met with any layer quantized!") + + export_onnx(args, asr_model) + + +def export_onnx(args, asr_model): + if args.onnx: + if args.asr_model.endswith("nemo"): + onnx_name = args.asr_model.replace(".nemo", ".onnx") + else: + onnx_name = args.asr_model + logging.info(F"Export to {onnx_name}") + quant_nn.TensorQuantizer.use_fb_fake_quant = True + asr_model.export(onnx_name, onnx_opset_version=13) + quant_nn.TensorQuantizer.use_fb_fake_quant = False + + +def evaluate(asr_model, labels_map, wer): + # Eval the model + hypotheses = [] + references = [] + for test_batch in asr_model.test_dataloader(): + if can_gpu: + test_batch = [x.cuda() for x in test_batch] + with autocast(): + log_probs, encoded_len, greedy_predictions = asr_model( + input_signal=test_batch[0], input_signal_length=test_batch[1] + ) + hypotheses += wer.ctc_decoder_predictions_tensor(greedy_predictions) + for batch_ind in range(greedy_predictions.shape[0]): + seq_len = test_batch[3][batch_ind].cpu().detach().numpy() + seq_ids = test_batch[2][batch_ind].cpu().detach().numpy() + reference = ''.join([labels_map[c] for c in seq_ids[0:seq_len]]) + references.append(reference) + del test_batch + wer_value = word_error_rate(hypotheses=hypotheses, references=references, use_cer=wer.use_cer) + + return wer_value + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_quant_infer_trt.py b/models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_quant_infer_trt.py new file mode 100644 index 0000000000..747bd5bd43 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/quantization/speech_to_text_quant_infer_trt.py @@ -0,0 +1,226 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Script for inference ASR models using TensorRT +""" + +import collections +import os +import time +from argparse import ArgumentParser +from pprint import pprint + +import numpy as np +import pycuda.autoinit +import pycuda.driver as cuda +import tensorrt as trt +import torch +from omegaconf import open_dict + +from nemo.collections.asr.metrics.wer import WER, word_error_rate +from nemo.collections.asr.models import EncDecCTCModel +from nemo.utils import logging + +TRT_LOGGER = trt.Logger() + + +can_gpu = torch.cuda.is_available() + +try: + from torch.cuda.amp import autocast +except ImportError: + from contextlib import contextmanager + + @contextmanager + def autocast(enabled=None): + yield + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "--asr_model", type=str, default="QuartzNet15x5Base-En", required=True, help="Pass: 'QuartzNet15x5Base-En'", + ) + parser.add_argument( + "--asr_onnx", + type=str, + default="./QuartzNet15x5Base-En-max-32.onnx", + help="Pass: 'QuartzNet15x5Base-En-max-32.onnx'", + ) + parser.add_argument("--dataset", type=str, required=True, help="path to evaluation data") + parser.add_argument("--batch_size", type=int, default=4) + parser.add_argument( + "--dont_normalize_text", + default=False, + action='store_false', + help="Turn off trasnscript normalization. Recommended for non-English.", + ) + parser.add_argument( + "--use_cer", default=False, action='store_true', help="Use Character Error Rate as the evaluation metric" + ) + parser.add_argument('--qat', action="store_true", help="Use onnx file exported from QAT tools") + args = parser.parse_args() + torch.set_grad_enabled(False) + + if args.asr_model.endswith('.nemo'): + logging.info(f"Using local ASR model from {args.asr_model}") + asr_model_cfg = EncDecCTCModel.restore_from(restore_path=args.asr_model, return_config=True) + with open_dict(asr_model_cfg): + asr_model_cfg.encoder.quantize = True + asr_model = EncDecCTCModel.restore_from(restore_path=args.asr_model, override_config_path=asr_model_cfg) + + else: + logging.info(f"Using NGC cloud ASR model {args.asr_model}") + asr_model_cfg = EncDecCTCModel.from_pretrained(model_name=args.asr_model, return_config=True) + with open_dict(asr_model_cfg): + asr_model_cfg.encoder.quantize = True + asr_model = EncDecCTCModel.from_pretrained(model_name=args.asr_model, override_config_path=asr_model_cfg) + asr_model.setup_test_data( + test_data_config={ + 'sample_rate': 16000, + 'manifest_filepath': args.dataset, + 'labels': asr_model.decoder.vocabulary, + 'batch_size': args.batch_size, + 'normalize_transcripts': args.dont_normalize_text, + } + ) + asr_model.preprocessor.featurizer.dither = 0.0 + asr_model.preprocessor.featurizer.pad_to = 0 + if can_gpu: + asr_model = asr_model.cuda() + asr_model.eval() + labels_map = dict([(i, asr_model.decoder.vocabulary[i]) for i in range(len(asr_model.decoder.vocabulary))]) + wer = WER(vocabulary=asr_model.decoder.vocabulary, use_cer=args.use_cer) + wer_result = evaluate(asr_model, args.asr_onnx, labels_map, wer, args.qat) + logging.info(f'Got WER of {wer_result}.') + + +def get_min_max_input_shape(asr_model): + max_shape = (1, 64, 1) + min_shape = (64, 64, 99999) + for test_batch in asr_model.test_dataloader(): + test_batch = [x.cuda() for x in test_batch] + processed_signal, processed_signal_length = asr_model.preprocessor( + input_signal=test_batch[0], length=test_batch[1] + ) + shape = processed_signal.cpu().numpy().shape + if shape[0] > max_shape[0]: + max_shape = (shape[0], *max_shape[1:]) + if shape[0] < min_shape[0]: + min_shape = (shape[0], *min_shape[1:]) + if shape[2] > max_shape[2]: + max_shape = (*max_shape[0:2], shape[2]) + if shape[2] < min_shape[2]: + min_shape = (*min_shape[0:2], shape[2]) + return min_shape, max_shape + + +def build_trt_engine(asr_model, onnx_path, qat): + trt_engine_path = "{}.trt".format(onnx_path) + if os.path.exists(trt_engine_path): + return trt_engine_path + + min_input_shape, max_input_shape = get_min_max_input_shape(asr_model) + workspace_size = 512 + with trt.Builder(TRT_LOGGER) as builder: + network_flags = 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH) + if qat: + network_flags |= 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_PRECISION) + with builder.create_network(flags=network_flags) as network, trt.OnnxParser( + network, TRT_LOGGER + ) as parser, builder.create_builder_config() as builder_config: + parser.parse_from_file(onnx_path) + builder_config.max_workspace_size = workspace_size * (1024 * 1024) + if qat: + builder_config.set_flag(trt.BuilderFlag.INT8) + + profile = builder.create_optimization_profile() + profile.set_shape("audio_signal", min=min_input_shape, opt=max_input_shape, max=max_input_shape) + builder_config.add_optimization_profile(profile) + + engine = builder.build_engine(network, builder_config) + serialized_engine = engine.serialize() + with open(trt_engine_path, "wb") as fout: + fout.write(serialized_engine) + return trt_engine_path + + +def trt_inference(stream, trt_ctx, d_input, d_output, input_signal, input_signal_length): + print("infer with shape: {}".format(input_signal.shape)) + + trt_ctx.set_binding_shape(0, input_signal.shape) + assert trt_ctx.all_binding_shapes_specified + + h_output = cuda.pagelocked_empty(tuple(trt_ctx.get_binding_shape(1)), dtype=np.float32) + + h_input_signal = cuda.register_host_memory(np.ascontiguousarray(input_signal.cpu().numpy().ravel())) + cuda.memcpy_htod_async(d_input, h_input_signal, stream) + trt_ctx.execute_async_v2(bindings=[int(d_input), int(d_output)], stream_handle=stream.handle) + cuda.memcpy_dtoh_async(h_output, d_output, stream) + stream.synchronize() + + greedy_predictions = torch.tensor(h_output).argmax(dim=-1, keepdim=False) + return greedy_predictions + + +def evaluate(asr_model, asr_onnx, labels_map, wer, qat): + # Eval the model + hypotheses = [] + references = [] + stream = cuda.Stream() + vocabulary_size = len(labels_map) + 1 + engine_file_path = build_trt_engine(asr_model, asr_onnx, qat) + with open(engine_file_path, 'rb') as f, trt.Runtime(TRT_LOGGER) as runtime: + trt_engine = runtime.deserialize_cuda_engine(f.read()) + trt_ctx = trt_engine.create_execution_context() + + profile_shape = trt_engine.get_profile_shape(profile_index=0, binding=0) + print("profile shape min:{}, opt:{}, max:{}".format(profile_shape[0], profile_shape[1], profile_shape[2])) + max_input_shape = profile_shape[2] + input_nbytes = trt.volume(max_input_shape) * trt.float32.itemsize + d_input = cuda.mem_alloc(input_nbytes) + max_output_shape = [max_input_shape[0], vocabulary_size, (max_input_shape[-1] + 1) // 2] + output_nbytes = trt.volume(max_output_shape) * trt.float32.itemsize + d_output = cuda.mem_alloc(output_nbytes) + + for test_batch in asr_model.test_dataloader(): + if can_gpu: + test_batch = [x.cuda() for x in test_batch] + processed_signal, processed_signal_length = asr_model.preprocessor( + input_signal=test_batch[0], length=test_batch[1] + ) + + greedy_predictions = trt_inference( + stream, + trt_ctx, + d_input, + d_output, + input_signal=processed_signal, + input_signal_length=processed_signal_length, + ) + hypotheses += wer.ctc_decoder_predictions_tensor(greedy_predictions) + for batch_ind in range(greedy_predictions.shape[0]): + seq_len = test_batch[3][batch_ind].cpu().detach().numpy() + seq_ids = test_batch[2][batch_ind].cpu().detach().numpy() + reference = ''.join([labels_map[c] for c in seq_ids[0:seq_len]]) + references.append(reference) + del test_batch + wer_value = word_error_rate(hypotheses=hypotheses, references=references, use_cer=wer.use_cer) + + return wer_value + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/speech_to_label.py b/models/tacotron2/NeMo/examples/asr/speech_to_label.py new file mode 100644 index 0000000000..369b0f026b --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/speech_to_label.py @@ -0,0 +1,146 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +# Task 1: Speech Command + +## Preparing the dataset +Use the `process_speech_commands_data.py` script under /scripts/dataset_processing in order to prepare the dataset. + +```sh +python /scripts/dataset_processing/process_speech_commands_data.py \ + --data_root= \ + --data_version= \ + --class_split= \ + --rebalance \ + --log +``` + +## Train to convergence +```sh +python speech_to_label.py \ + # (Optional: --config-path= --config-name=) \ + model.train_ds.manifest_filepath="" \ + model.validation_ds.manifest_filepath=["",""] \ + trainer.gpus=2 \ + trainer.accelerator="ddp" \ + trainer.max_epochs=200 \ + exp_manager.create_wandb_logger=True \ + exp_manager.wandb_logger_kwargs.name="MatchboxNet-3x1x64-v1" \ + exp_manager.wandb_logger_kwargs.project="MatchboxNet-v1" \ + +trainer.precision=16 \ + +trainer.amp_level=O1 # needed if using PyTorch < 1.6 +``` + + +# Task 2: Voice Activity Detection + +## Preparing the dataset +Use the `process_vad_data.py` script under /scripts/dataset_processing in order to prepare the dataset. + +```sh +python process_vad_data.py \ + --out_dir= \ + --speech_data_root= \ + --background_data_root= \ + --rebalance_method=<'under' or 'over' of 'fixed'> \ + --log + (Optional --demo (for demonstration in tutorial). If you want to use your own background noise data, make sure to delete --demo) +``` + +## Train to convergence +```sh +python speech_to_label.py \ + --config-path= + --config-name= \ + model.train_ds.manifest_filepath="" \ + model.validation_ds.manifest_filepath=["",""] \ + trainer.gpus=2 \ + trainer.accelerator="ddp" \ + trainer.max_epochs=200 \ + exp_manager.create_wandb_logger=True \ + exp_manager.wandb_logger_kwargs.name="MatchboxNet-3x1x64-vad" \ + exp_manager.wandb_logger_kwargs.project="MatchboxNet-vad" \ + +trainer.precision=16 \ + +trainer.amp_level=O1 # needed if using PyTorch < 1.6 +``` + +# Optional: Use tarred dataset to speed up data loading. Apply to both tasks. +## Prepare tarred dataset. + Prepare ONE manifest that contains all training data you would like to include. Validation should use non-tarred dataset. + Note that it's possible that tarred datasets impacts validation scores because it drop values in order to have same amount of files per tarfile; + Scores might be off since some data is missing. + + Use the `convert_to_tarred_audio_dataset.py` script under /scripts/speech_recognition in order to prepare tarred audio dataset. + For details, please see TarredAudioToClassificationLabelDataset in /nemo/collections/asr/data/audio_to_label.py + +python speech_to_label.py \ + --config-path= + --config-name= \ + model.train_ds.manifest_filepath= \ + model.train_ds.is_tarred=True \ + model.train_ds.tarred_audio_filepaths= \ + +model.train_ds.num_worker= \ + model.validation_ds.manifest_filepath=\ + trainer.gpus=2 \ + trainer.accelerator="ddp" \ + trainer.max_epochs=200 \ + exp_manager.create_wandb_logger=True \ + exp_manager.wandb_logger_kwargs.name="MatchboxNet-3x1x64-vad" \ + exp_manager.wandb_logger_kwargs.project="MatchboxNet-vad" \ + +trainer.precision=16 \ + +trainer.amp_level=O1 # needed if using PyTorch < 1.6 + +# Fine-tune a model + +For documentation on fine-tuning this model, please visit - +https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/configs.html#fine-tuning-configurations + +# Pretrained Models + +For documentation on existing pretrained models, please visit - +https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/speech_classification/results.html# + +""" +import pytorch_lightning as pl +from omegaconf import OmegaConf + +from nemo.collections.asr.models import EncDecClassificationModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf/matchboxnet", config_name="matchboxnet_3x1x64_v1") +def main(cfg): + logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}') + + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + asr_model = EncDecClassificationModel(cfg=cfg.model, trainer=trainer) + + # Initialize the weights of the model from another model, if provided via config + asr_model.maybe_init_from_pretrained_checkpoint(cfg) + + trainer.fit(asr_model) + + if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: + gpu = 1 if cfg.trainer.gpus != 0 else 0 + trainer = pl.Trainer(gpus=gpu) + if asr_model.prepare_test(trainer): + trainer.test(asr_model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/speech_to_text.py b/models/tacotron2/NeMo/examples/asr/speech_to_text.py new file mode 100644 index 0000000000..ced4cb8e3a --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/speech_to_text.py @@ -0,0 +1,105 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +# Training the model + +Basic run (on CPU for 50 epochs): + python examples/asr/speech_to_text.py \ + # (Optional: --config-path= --config-name=) \ + model.train_ds.manifest_filepath="" \ + model.validation_ds.manifest_filepath="" \ + trainer.gpus=0 \ + trainer.max_epochs=50 + + +Add PyTorch Lightning Trainer arguments from CLI: + python speech_to_text.py \ + ... \ + +trainer.fast_dev_run=true + +Hydra logs will be found in "$(./outputs/$(date +"%y-%m-%d")/$(date +"%H-%M-%S")/.hydra)" +PTL logs will be found in "$(./outputs/$(date +"%y-%m-%d")/$(date +"%H-%M-%S")/lightning_logs)" + +Override some args of optimizer: + python speech_to_text.py \ + # (Optional: --config-path= --config-name=) \ + model.train_ds.manifest_filepath="./an4/train_manifest.json" \ + model.validation_ds.manifest_filepath="./an4/test_manifest.json" \ + trainer.gpus=2 \ + trainer.max_epochs=2 \ + model.optim.args.betas=[0.8,0.5] \ + model.optim.args.weight_decay=0.0001 + +Override optimizer entirely + python speech_to_text.py \ + # (Optional: --config-path= --config-name=) \ + model.train_ds.manifest_filepath="./an4/train_manifest.json" \ + model.validation_ds.manifest_filepath="./an4/test_manifest.json" \ + trainer.gpus=2 \ + trainer.max_epochs=2 \ + model.optim.name=adamw \ + model.optim.lr=0.001 \ + ~model.optim.args \ + +model.optim.args.betas=[0.8,0.5]\ + +model.optim.args.weight_decay=0.0005 + +# Fine-tune a model + +For documentation on fine-tuning this model, please visit - +https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/configs.html#fine-tuning-configurations + +# Pretrained Models + +For documentation on existing pretrained models, please visit - +https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/results.html + +""" + +import pytorch_lightning as pl +from omegaconf import OmegaConf + +from nemo.collections.asr.models import EncDecCTCModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="config") +def main(cfg): + logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}') + + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer) + + # Initialize the weights of the model from another model, if provided via config + asr_model.maybe_init_from_pretrained_checkpoint(cfg) + + trainer.fit(asr_model) + + if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: + gpu = 1 if cfg.trainer.gpus != 0 else 0 + test_trainer = pl.Trainer( + gpus=gpu, + precision=trainer.precision, + amp_level=trainer.accelerator_connector.amp_level, + amp_backend=cfg.trainer.get("amp_backend", "native"), + ) + if asr_model.prepare_test(test_trainer): + test_trainer.test(asr_model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/speech_to_text_bpe.py b/models/tacotron2/NeMo/examples/asr/speech_to_text_bpe.py new file mode 100644 index 0000000000..090535f3ac --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/speech_to_text_bpe.py @@ -0,0 +1,101 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +# Preparing the Tokenizer for the dataset +Use the `process_asr_text_tokenizer.py` script under /scripts/tokenizers/ in order to prepare the tokenizer. + +```sh +python /scripts/tokenizers/process_asr_text_tokenizer.py \ + --manifest= + OR + --data_file= \ + --data_root="" \ + --vocab_size= \ + --tokenizer=<"spe" or "wpe"> \ + --no_lower_case \ + --spe_type=<"unigram", "bpe", "char" or "word"> \ + --spe_character_coverage=1.0 \ + --log +``` + +# Training the model +```sh +python speech_to_text_bpe.py \ + # (Optional: --config-path= --config-name=) \ + model.train_ds.manifest_filepath= \ + model.validation_ds.manifest_filepath= \ + model.tokenizer.dir= \ + model.tokenizer.type= \ + trainer.gpus=-1 \ + trainer.accelerator="ddp" \ + trainer.max_epochs=100 \ + model.optim.name="adamw" \ + model.optim.lr=0.001 \ + model.optim.betas=[0.9,0.999] \ + model.optim.weight_decay=0.0001 \ + model.optim.sched.warmup_steps=2000 + exp_manager.create_wandb_logger=True \ + exp_manager.wandb_logger_kwargs.name="" \ + exp_manager.wandb_logger_kwargs.project="" +``` + +# Fine-tune a model + +For documentation on fine-tuning this model, please visit - +https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/configs.html#fine-tuning-configurations + +# Pretrained Models + +For documentation on existing pretrained models, please visit - +https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/results.html + +""" + +import pytorch_lightning as pl +from omegaconf import OmegaConf + +from nemo.collections.asr.models.ctc_bpe_models import EncDecCTCModelBPE +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf/citrinet/", config_name="config_bpe") +def main(cfg): + logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}') + + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + asr_model = EncDecCTCModelBPE(cfg=cfg.model, trainer=trainer) + + # Initialize the weights of the model from another model, if provided via config + asr_model.maybe_init_from_pretrained_checkpoint(cfg) + + trainer.fit(asr_model) + + if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: + gpu = 1 if cfg.trainer.gpus != 0 else 0 + test_trainer = pl.Trainer( + gpus=gpu, + precision=trainer.precision, + amp_level=trainer.accelerator_connector.amp_level, + amp_backend=cfg.trainer.get("amp_backend", "native"), + ) + if asr_model.prepare_test(test_trainer): + test_trainer.test(asr_model) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/asr/speech_to_text_infer.py b/models/tacotron2/NeMo/examples/asr/speech_to_text_infer.py new file mode 100644 index 0000000000..bbba1a6119 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/speech_to_text_infer.py @@ -0,0 +1,114 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script serves three goals: + (1) Demonstrate how to use NeMo Models outside of PytorchLightning + (2) Shows example of batch ASR inference + (3) Serves as CI test for pre-trained checkpoint +""" + +from argparse import ArgumentParser + +import torch + +from nemo.collections.asr.metrics.wer import WER, word_error_rate +from nemo.collections.asr.models import EncDecCTCModel +from nemo.utils import logging + +try: + from torch.cuda.amp import autocast +except ImportError: + from contextlib import contextmanager + + @contextmanager + def autocast(enabled=None): + yield + + +can_gpu = torch.cuda.is_available() + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "--asr_model", type=str, default="QuartzNet15x5Base-En", required=True, help="Pass: 'QuartzNet15x5Base-En'", + ) + parser.add_argument("--dataset", type=str, required=True, help="path to evaluation data") + parser.add_argument("--batch_size", type=int, default=4) + parser.add_argument("--wer_tolerance", type=float, default=1.0, help="used by test") + parser.add_argument( + "--dont_normalize_text", + default=False, + action='store_true', + help="Turn off trasnscript normalization. Recommended for non-English.", + ) + parser.add_argument( + "--use_cer", default=False, action='store_true', help="Use Character Error Rate as the evaluation metric" + ) + args = parser.parse_args() + torch.set_grad_enabled(False) + + if args.asr_model.endswith('.nemo'): + logging.info(f"Using local ASR model from {args.asr_model}") + asr_model = EncDecCTCModel.restore_from(restore_path=args.asr_model) + else: + logging.info(f"Using NGC cloud ASR model {args.asr_model}") + asr_model = EncDecCTCModel.from_pretrained(model_name=args.asr_model) + + asr_model.preprocessor.featurizer.pad_to = 0 + asr_model.preprocessor.featurizer.dither = 0.0 + + asr_model.setup_test_data( + test_data_config={ + 'sample_rate': 16000, + 'manifest_filepath': args.dataset, + 'labels': asr_model.decoder.vocabulary, + 'batch_size': args.batch_size, + 'normalize_transcripts': not args.dont_normalize_text, + } + ) + if can_gpu: + asr_model = asr_model.cuda() + asr_model.eval() + labels_map = dict([(i, asr_model.decoder.vocabulary[i]) for i in range(len(asr_model.decoder.vocabulary))]) + wer = WER(vocabulary=asr_model.decoder.vocabulary) + hypotheses = [] + references = [] + for test_batch in asr_model.test_dataloader(): + if can_gpu: + test_batch = [x.cuda() for x in test_batch] + with autocast(): + log_probs, encoded_len, greedy_predictions = asr_model( + input_signal=test_batch[0], input_signal_length=test_batch[1] + ) + hypotheses += wer.ctc_decoder_predictions_tensor(greedy_predictions) + for batch_ind in range(greedy_predictions.shape[0]): + seq_len = test_batch[3][batch_ind].cpu().detach().numpy() + seq_ids = test_batch[2][batch_ind].cpu().detach().numpy() + reference = ''.join([labels_map[c] for c in seq_ids[0:seq_len]]) + references.append(reference) + del test_batch + + wer_value = word_error_rate(hypotheses=hypotheses, references=references, use_cer=args.use_cer) + if not args.use_cer: + if wer_value > args.wer_tolerance: + raise ValueError(f"got wer of {wer_value}. it was higher than {args.wer_tolerance}") + logging.info(f'Got WER of {wer_value}. Tolerance was {args.wer_tolerance}') + else: + logging.info(f'Got CER of {wer_value}') + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/speech_to_text_rnnt.py b/models/tacotron2/NeMo/examples/asr/speech_to_text_rnnt.py new file mode 100644 index 0000000000..39d3be9ca7 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/speech_to_text_rnnt.py @@ -0,0 +1,104 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +# Training the model + +Basic run (on CPU for 50 epochs): + python examples/asr/speech_to_text_rnnt.py \ + # (Optional: --config-path= --config-name=) \ + model.train_ds.manifest_filepath="" \ + model.validation_ds.manifest_filepath="" \ + trainer.gpus=0 \ + trainer.max_epochs=50 + + +Add PyTorch Lightning Trainer arguments from CLI: + python speech_to_text_rnnt.py \ + ... \ + +trainer.fast_dev_run=true + +Hydra logs will be found in "$(./outputs/$(date +"%y-%m-%d")/$(date +"%H-%M-%S")/.hydra)" +PTL logs will be found in "$(./outputs/$(date +"%y-%m-%d")/$(date +"%H-%M-%S")/lightning_logs)" + +Override some args of optimizer: + python speech_to_text_rnnt.py \ + --config-path="experimental/contextnet_rnnt" \ + --config-name="config_rnnt" \ + model.train_ds.manifest_filepath="./an4/train_manifest.json" \ + model.validation_ds.manifest_filepath="./an4/test_manifest.json" \ + trainer.gpus=2 \ + trainer.precision=16 \ + trainer.max_epochs=2 \ + model.optim.betas=[0.8,0.5] \ + model.optim.weight_decay=0.0001 + +Override optimizer entirely + python speech_to_text_rnnt.py \ + --config-path="experimental/contextnet_rnnt" \ + --config-name="config_rnnt" \ + model.train_ds.manifest_filepath="./an4/train_manifest.json" \ + model.validation_ds.manifest_filepath="./an4/test_manifest.json" \ + trainer.gpus=2 \ + trainer.precision=16 \ + trainer.max_epochs=2 \ + model.optim.name=adamw \ + model.optim.lr=0.001 \ + ~model.optim.args \ + +model.optim.args.betas=[0.8,0.5]\ + +model.optim.args.weight_decay=0.0005 + +# Fine-tune a model + +For documentation on fine-tuning this model, please visit - +https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/configs.html#fine-tuning-configurations + +""" + +import pytorch_lightning as pl +from omegaconf import OmegaConf + +from nemo.collections.asr.models import EncDecRNNTModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="experimental/contextnet_rnnt", config_name="config_rnnt") +def main(cfg): + logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}') + + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + asr_model = EncDecRNNTModel(cfg=cfg.model, trainer=trainer) + + # Initialize the weights of the model from another model, if provided via config + asr_model.maybe_init_from_pretrained_checkpoint(cfg) + + trainer.fit(asr_model) + + if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: + gpu = 1 if cfg.trainer.gpus != 0 else 0 + test_trainer = pl.Trainer( + gpus=gpu, + precision=trainer.precision, + amp_level=trainer.accelerator_connector.amp_level, + amp_backend=cfg.trainer.get("amp_backend", "native"), + ) + if asr_model.prepare_test(test_trainer): + test_trainer.test(asr_model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/speech_to_text_rnnt_bpe.py b/models/tacotron2/NeMo/examples/asr/speech_to_text_rnnt_bpe.py new file mode 100644 index 0000000000..91e41deaf0 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/speech_to_text_rnnt_bpe.py @@ -0,0 +1,96 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +# Preparing the Tokenizer for the dataset +Use the `process_asr_text_tokenizer.py` script under /scripts/tokenizers/ in order to prepare the tokenizer. + +```sh +python /scripts/tokenizers/process_asr_text_tokenizer.py \ + --manifest= + OR + --data_file= \ + --data_root="" \ + --vocab_size= \ + --tokenizer=<"spe" or "wpe"> \ + --no_lower_case \ + --spe_type=<"unigram", "bpe", "char" or "word"> \ + --spe_character_coverage=1.0 \ + --log +``` + +# Training the model +```sh +python speech_to_text_rnnt_bpe.py \ + # (Optional: --config-path= --config-name=) \ + model.train_ds.manifest_filepath= \ + model.validation_ds.manifest_filepath= \ + model.tokenizer.dir= \ + model.tokenizer.type= \ + trainer.gpus=-1 \ + trainer.accelerator="ddp" \ + trainer.max_epochs=100 \ + model.optim.name="adamw" \ + model.optim.lr=0.001 \ + model.optim.betas=[0.9,0.999] \ + model.optim.weight_decay=0.0001 \ + model.optim.sched.warmup_steps=2000 + exp_manager.create_wandb_logger=True \ + exp_manager.wandb_logger_kwargs.name="" \ + exp_manager.wandb_logger_kwargs.project="" +``` + +# Fine-tune a model + +For documentation on fine-tuning this model, please visit - +https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/configs.html#fine-tuning-configurations + +""" + +import pytorch_lightning as pl +from omegaconf import OmegaConf + +from nemo.collections.asr.models import EncDecRNNTBPEModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="experimental/contextnet_rnnt", config_name="config_rnnt_bpe") +def main(cfg): + logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}') + + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + asr_model = EncDecRNNTBPEModel(cfg=cfg.model, trainer=trainer) + + # Initialize the weights of the model from another model, if provided via config + asr_model.maybe_init_from_pretrained_checkpoint(cfg) + + trainer.fit(asr_model) + + if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: + gpu = 1 if cfg.trainer.gpus != 0 else 0 + test_trainer = pl.Trainer( + gpus=gpu, + precision=trainer.precision, + amp_level=trainer.accelerator_connector.amp_level, + amp_backend=cfg.trainer.get("amp_backend", "native"), + ) + if asr_model.prepare_test(test_trainer): + test_trainer.test(asr_model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/transcribe_speech.py b/models/tacotron2/NeMo/examples/asr/transcribe_speech.py new file mode 100644 index 0000000000..b186c22757 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/transcribe_speech.py @@ -0,0 +1,167 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import glob +import json +import os +from dataclasses import dataclass +from typing import Optional + +import pytorch_lightning as pl +import torch +from omegaconf import OmegaConf + +from nemo.collections.asr.metrics.rnnt_wer import RNNTDecodingConfig +from nemo.collections.asr.models import ASRModel +from nemo.core.config import hydra_runner +from nemo.utils import logging, model_utils + + +""" +# Transcribe audio +# Arguments +# model_path: path to .nemo ASR checkpoint +# pretrained_name: name of pretrained ASR model (from NGC registry) +# audio_dir: path to directory with audio files +# dataset_manifest: path to dataset JSON manifest file (in NeMo format) +# +# ASR model can be specified by either "model_path" or "pretrained_name". +# Data for transcription can be defined with either "audio_dir" or "dataset_manifest". +# Results are returned in a JSON manifest file. + +python transcribe_speech.py \ + model_path=null \ + pretrained_name=null \ + audio_dir="" \ + dataset_manifest="" \ + output_filename="" +""" + + +@dataclass +class TranscriptionConfig: + # Required configs + model_path: Optional[str] = None # Path to a .nemo file + pretrained_name: Optional[str] = None # Name of a pretrained model + audio_dir: Optional[str] = None # Path to a directory which contains audio files + dataset_manifest: Optional[str] = None # Path to dataset's JSON manifest + + # General configs + output_filename: Optional[str] = None + batch_size: int = 32 + cuda: Optional[bool] = None # will switch to cuda if available, defaults to CPU otherwise + amp: bool = False + audio_type: str = "wav" + + # decoding strategy for RNNT models + rnnt_decoding: RNNTDecodingConfig = RNNTDecodingConfig() + + +@hydra_runner(config_name="TranscriptionConfig", schema=TranscriptionConfig) +def main(cfg: TranscriptionConfig): + logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}') + + if cfg.model_path is None and cfg.pretrained_name is None: + raise ValueError("Both cfg.model_path and cfg.pretrained_name cannot be None!") + if cfg.audio_dir is None and cfg.dataset_manifest is None: + raise ValueError("Both cfg.audio_dir and cfg.dataset_manifest cannot be None!") + + # setup GPU + if cfg.cuda is None: + cfg.cuda = torch.cuda.is_available() + + if type(cfg.cuda) == int: + device_id = int(cfg.cuda) + else: + device_id = 0 + + device = torch.device(f'cuda:{device_id}' if cfg.cuda else 'cpu') + + # setup model + if cfg.model_path is not None: + # restore model from .nemo file path + model_cfg = ASRModel.restore_from(restore_path=cfg.model_path, return_config=True) + classpath = model_cfg.target # original class path + imported_class = model_utils.import_class_by_path(classpath) # type: ASRModel + logging.info(f"Restoring model : {imported_class.__name__}") + asr_model = imported_class.restore_from(restore_path=cfg.model_path, map_location=device) # type: ASRModel + model_name = os.path.splitext(os.path.basename(cfg.model_path))[0] + else: + # restore model by name + asr_model = ASRModel.from_pretrained(model_name=cfg.pretrained_name, map_location=device) # type: ASRModel + model_name = cfg.pretrained_name + + trainer = pl.Trainer(gpus=int(cfg.cuda)) + asr_model.set_trainer(trainer) + asr_model = asr_model.eval() + + # Setup decoding strategy + if hasattr(asr_model, 'change_decoding_strategy'): + asr_model.change_decoding_strategy(cfg.rnnt_decoding) + + # get audio filenames + if cfg.audio_dir is not None: + filepaths = list(glob.glob(os.path.join(cfg.audio_dir, f"*.{cfg.audio_type}"))) + else: + # get filenames from manifest + filepaths = [] + with open(cfg.dataset_manifest, 'r') as f: + for line in f: + item = json.loads(line) + filepaths.append(item['audio_filepath']) + logging.info(f"\nTranscribing {len(filepaths)} files...\n") + + # setup AMP (optional) + if cfg.amp and torch.cuda.is_available() and hasattr(torch.cuda, 'amp') and hasattr(torch.cuda.amp, 'autocast'): + logging.info("AMP enabled!\n") + autocast = torch.cuda.amp.autocast + else: + + @contextlib.contextmanager + def autocast(): + yield + + # transcribe audio + with autocast(): + with torch.no_grad(): + transcriptions = asr_model.transcribe(filepaths, batch_size=cfg.batch_size) + logging.info(f"Finished transcribing {len(filepaths)} files !") + + if cfg.output_filename is None: + # create default output filename + if cfg.audio_dir is not None: + cfg.output_filename = os.path.dirname(os.path.join(cfg.audio_dir, '.')) + '.json' + else: + cfg.output_filename = cfg.dataset_manifest.replace('.json', f'_{model_name}.json') + + logging.info(f"Writing transcriptions into file: {cfg.output_filename}") + + with open(cfg.output_filename, 'w', encoding='utf-8') as f: + if cfg.audio_dir is not None: + for idx, text in enumerate(transcriptions): + item = {'audio_filepath': filepaths[idx], 'pred_text': text} + f.write(json.dumps(item) + "\n") + else: + with open(cfg.dataset_manifest, 'r') as fr: + for idx, line in enumerate(fr): + item = json.loads(line) + item['pred_text'] = transcriptions[idx] + f.write(json.dumps(item) + "\n") + + logging.info("Finished writing predictions !") + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/asr/vad_infer.py b/models/tacotron2/NeMo/examples/asr/vad_infer.py new file mode 100644 index 0000000000..ba036029f2 --- /dev/null +++ b/models/tacotron2/NeMo/examples/asr/vad_infer.py @@ -0,0 +1,171 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +During inference, we perform frame-level prediction by two approaches: + 1) shift the window of length time_length (e.g. 0.63s) by shift_length (e.g. 10ms) to generate the frame and use the prediction of the window to represent the label for the frame; + [this script demonstrate how to do this approach] + 2) generate predictions with overlapping input segments. Then a smoothing filter is applied to decide the label for a frame spanned by multiple segments. + [get frame level prediction by this script and use vad_overlap_posterior.py in NeMo/scripts/voice_activity_detection + One can also find posterior about converting frame level prediction + to speech/no-speech segment in start and end times format in that script.] + + Image https://raw.githubusercontent.com/NVIDIA/NeMo/main/tutorials/asr/images/vad_post_overlap_diagram.png + will help you understand this method. + +Usage: +python vad_infer.py --vad_model="vad_marblenet" --dataset= --out_dir='frame/demo' --time_length=0.63 + +""" + + +import json +import logging +import os +from argparse import ArgumentParser + +import torch + +from nemo.collections.asr.models import EncDecClassificationModel +from nemo.collections.asr.parts.utils.vad_utils import get_vad_stream_status, prepare_manifest +from nemo.utils import logging + +try: + from torch.cuda.amp import autocast +except ImportError: + from contextlib import contextmanager + + @contextmanager + def autocast(enabled=None): + yield + + +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "--vad_model", type=str, default="MatchboxNet-VAD-3x2", required=False, help="Pass: 'MatchboxNet-VAD-3x2'" + ) + parser.add_argument( + "--dataset", + type=str, + required=True, + help="Path of json file of evaluation data. Audio files should have unique names.", + ) + parser.add_argument("--out_dir", type=str, default="vad_frame", help="Dir of your vad outputs") + parser.add_argument("--time_length", type=float, default=0.63) + parser.add_argument("--shift_length", type=float, default=0.01) + parser.add_argument("--normalize_audio", type=bool, default=False) + parser.add_argument("--num_workers", type=float, default=20) + parser.add_argument("--split_duration", type=float, default=400) + parser.add_argument( + "--dont_auto_split", + default=False, + action='store_true', + help="Whether to automatically split manifest entry by split_duration to avoid potential CUDA out of memory issue.", + ) + + args = parser.parse_args() + + torch.set_grad_enabled(False) + + if args.vad_model.endswith('.nemo'): + logging.info(f"Using local VAD model from {args.vad_model}") + vad_model = EncDecClassificationModel.restore_from(restore_path=args.vad_model) + else: + logging.info(f"Using NGC cloud VAD model {args.vad_model}") + vad_model = EncDecClassificationModel.from_pretrained(model_name=args.vad_model) + + if not os.path.exists(args.out_dir): + os.mkdir(args.out_dir) + + # Prepare manifest for streaming VAD + manifest_vad_input = args.dataset + if not args.dont_auto_split: + logging.info("Split long audio file to avoid CUDA memory issue") + logging.debug("Try smaller split_duration if you still have CUDA memory issue") + config = { + 'manifest_filepath': manifest_vad_input, + 'time_length': args.time_length, + 'split_duration': args.split_duration, + 'num_workers': args.num_workers, + } + manifest_vad_input = prepare_manifest(config) + else: + logging.warning( + "If you encounter CUDA memory issue, try splitting manifest entry by split_duration to avoid it." + ) + + # setup_test_data + vad_model.setup_test_data( + test_data_config={ + 'vad_stream': True, + 'sample_rate': 16000, + 'manifest_filepath': manifest_vad_input, + 'labels': ['infer',], + 'num_workers': args.num_workers, + 'shuffle': False, + 'time_length': args.time_length, + 'shift_length': args.shift_length, + 'trim_silence': False, + 'normalize_audio': args.normalize_audio, + } + ) + + vad_model = vad_model.to(device) + vad_model.eval() + + time_unit = int(args.time_length / args.shift_length) + trunc = int(time_unit / 2) + trunc_l = time_unit - trunc + all_len = 0 + + data = [] + for line in open(args.dataset, 'r'): + file = json.loads(line)['audio_filepath'].split("/")[-1] + data.append(file.split(".wav")[0]) + logging.info(f"Inference on {len(data)} audio files/json lines!") + + status = get_vad_stream_status(data) + for i, test_batch in enumerate(vad_model.test_dataloader()): + test_batch = [x.to(device) for x in test_batch] + with autocast(): + log_probs = vad_model(input_signal=test_batch[0], input_signal_length=test_batch[1]) + probs = torch.softmax(log_probs, dim=-1) + pred = probs[:, 1] + + if status[i] == 'start': + to_save = pred[:-trunc] + elif status[i] == 'next': + to_save = pred[trunc:-trunc_l] + elif status[i] == 'end': + to_save = pred[trunc_l:] + else: + to_save = pred + + all_len += len(to_save) + outpath = os.path.join(args.out_dir, data[i] + ".frame") + with open(outpath, "a") as fout: + for f in range(len(to_save)): + fout.write('{0:0.4f}\n'.format(to_save[f])) + del test_batch + if status[i] == 'end' or status[i] == 'single': + logging.debug(f"Overall length of prediction of {data[i]} is {all_len}!") + all_len = 0 + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/cv/mnist_lenet5_image_classification_pure_lightning.py b/models/tacotron2/NeMo/examples/cv/mnist_lenet5_image_classification_pure_lightning.py new file mode 100644 index 0000000000..ff03a9e146 --- /dev/null +++ b/models/tacotron2/NeMo/examples/cv/mnist_lenet5_image_classification_pure_lightning.py @@ -0,0 +1,65 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass + +# from hydra.utils import instantiate +import pytorch_lightning as ptl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.cv.models import MNISTLeNet5, MNISTLeNet5Config +from nemo.core.config import Config, TrainerConfig, hydra_runner +from nemo.utils import logging + + +@dataclass +class AppConfig(Config): + """ + This is structured config for this application. + + Args: + name: Description of the application. + trainer: configuration of the trainer. + model: configuation of the model. + """ + + name: str = "Training of a LeNet-5 Model using a pure PyTorchLightning approach - using DDP on 2 GPUs." + trainer: TrainerConfig = TrainerConfig(gpus=2, accelerator="ddp") + model: MNISTLeNet5Config = MNISTLeNet5Config() + + +@hydra_runner(schema=AppConfig, config_name="AppConfig") +def main(cfg: DictConfig): + # Show configuration - user can modify every parameter from command line! + logging.info("Application config\n" + OmegaConf.to_yaml(cfg)) + + # The "model" - with dataloader/dataset inside of it. + lenet5 = MNISTLeNet5(cfg.model) + + # Setup train data loader and optimizer + lenet5.setup_training_data() + + # Setup optimizer and scheduler + lenet5.setup_optimization() + + # Create trainer. + trainer = ptl.Trainer(**(cfg.trainer)) + # trainer = instantiate(cfg.trainer) + + # Train. + trainer.fit(model=lenet5) + + +if __name__ == "__main__": + main() # TODO: No cfg in function call, and no hydra runner diff --git a/models/tacotron2/NeMo/examples/nlp/dialogue_state_tracking/conf/sgdqa_config.yaml b/models/tacotron2/NeMo/examples/nlp/dialogue_state_tracking/conf/sgdqa_config.yaml new file mode 100644 index 0000000000..7ecc5652a6 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/dialogue_state_tracking/conf/sgdqa_config.yaml @@ -0,0 +1,115 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +pretrained_model: null # pretrained QAModel model from list_available_models() +do_training: true # training mode, for testing change to false +trainer: + gpus: 8 # number of GPUs (0 for CPU), or list of the GPUs to use e.g. [0, 1] + num_nodes: 1 + max_epochs: 3 + max_steps: null # precedence over max_epochs + accumulate_grad_batches: 1 # accumulates grads every k batches + gradient_clip_val: 1.0 + amp_level: O1 # O1/O2 for mixed precision + precision: 16 # Should be set to 16 for O1 and O2 to enable the AMP. + accelerator: ddp + log_every_n_steps: 5 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + +model: + nemo_path: null # filename to save the model and associated artifacts to .nemo file + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # or sentencepiece + vocab_file: null # path to vocab file + tokenizer_model: null # only used if tokenizer is sentencepiece + special_tokens: null + + language_model: + pretrained_model_name: bert-base-cased + lm_checkpoint: null + config_file: null # json file, precedence over config + config: null + + encoder: + dropout: 0.1 + + dataset: + max_seq_length: 128 # the maximum length BERT supports is 512 + max_num_cat_slot: 6 # maximum number of different categorical slots per service in dataset + max_num_noncat_slot: 12 # maximum number of different non-categorical slots per service in dataset + max_value_per_cat_slot: 12 # maximum number of different categorical slot values per service in dataset + max_num_intent: 4 # maximum number of different intents per service in dataset + task_name: sgd_single_domain # or from [sgd_all, sgd_all_single, sgd_multi_domain, debug_sample] + state_tracker: nemotracker # or baseline + data_dir: ??? + dialogues_example_dir: ??? # cache location to store or load preprocessed data from. To use cache, specify `use_cache` + subsample: false # balances negative and positive training examples for improved performance + use_cache: false # uses a cache to store the processed dataset, you may use it for large datasets for speed up + use_fuzzy_match: true # Whether to use fuzzy string matching when comparing non-categorical slot values. Should be set to False when conducting multiwoz style evaluation. + joint_acc_across_turn: false # Whether to compute joint goal accuracy across turn instead of across service. Should be set to True when conducting multiwoz style evaluation. + + train_ds: + ds_item: "train" + batch_size: 128 + shuffle: true + num_workers: 3 + drop_last: false + pin_memory: false + + validation_ds: + ds_item: ["dev"] # ["dev", "test"] + batch_size: 8 + shuffle: false + num_workers: 3 + drop_last: false + pin_memory: false + + test_ds: + ds_item: ["test"] + batch_size: 8 + shuffle: false + num_workers: 3 + drop_last: false + pin_memory: false + + optim: + name: adamw + lr: 1e-4 + # optimizer arguments + betas: [0.9, 0.999] + weight_decay: 0.01 + + # scheduler setup + sched: + name: PolynomialDecayAnnealing + # Scheduler params + warmup_steps: null + warmup_ratio: 0.02 + last_epoch: -1 + # pytorch lightning args + monitor: val_loss + reduce_on_plateau: false + +exp_manager: + exp_dir: null # exp_dir for your experiment, if None, defaults to "./nemo_experiments" + name: "SGDQA" # The name of your model + create_tensorboard_logger: True # Whether you want exp_manger to create a tb logger + create_checkpoint_callback: True # Whether you want exp_manager to create a modelcheckpoint callback + resume_if_exists: false + resume_ignore_no_checkpoint: false diff --git a/models/tacotron2/NeMo/examples/nlp/dialogue_state_tracking/sgd_qa.py b/models/tacotron2/NeMo/examples/nlp/dialogue_state_tracking/sgd_qa.py new file mode 100644 index 0000000000..85dbae19af --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/dialogue_state_tracking/sgd_qa.py @@ -0,0 +1,145 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script contains an example of how to train and test the NeMo SGD-QA Model (https://arxiv.org/abs/2105.08049). +The SGD-QA model is a fast multi-pass schema-guided state-tracking model, that is trained on the Google schema-guided state tracking dataset (https://arxiv.org/abs/1909.05855). +The model takes dialogue as input and outputs the dialogue state, which includes slot-value pairs. +The model consists of two components: a neural natural language understanding model (NLU), and a rule-based state tracker. +The NLU takes in a dialogue turn and different schema (entity) information options and outputs their match score. The state tracker takes the highest rated entities and composes +the dialogue state across turns. + +***Data format*** +The SGD-QA requires a JSON schema and dialogue files for each dataset split. +In the following we will show an example for a service entry in the schema file. +* service_name +* description +* slots + * name + * description + * is_categorical + * possible values +* intents + * name + * description + * required_slots (not used) + * is_transactional (not used) + * optional_slots (not used) + * result_slots (not used) + + +In the following we will show an example for a dialogue. More information at https://arxiv.org/abs/1909.05855 +* dialogue_id +* services +* turns + * frames + * actions + * act + * slot + * values + * service + * slots + * exclusive_end + * slot + * start + * state + * active_intent + * requeste_slots + * slot_values + * speaker - [USER, SYSTEM] + * utterance + + +***Downloading the dataset*** +# git clone https://github.com/google-research-datasets/dstc8-schema-guided-dialogue.git + +***Setting the configs*** +The model and the PT trainer are defined in a config file that declares multiple important sections. +The most important ones are: + model: All arguments that are related to the Model - language model, SGD-QA encoder and decoder, loss, optimizer, + schedulers, and datasets/data loaders. + trainer: Any argument to be passed to PyTorch Lightning including number of epochs, number of GPUs, + precision level, etc. + +This script uses the `/examples/nlp/dialogue_state_tracking/conf/sgdqa_config.yaml` config file +by default. You may update the config file from the file directly. The other option is to set another config file via command-line arguments by `--config-name=CONFIG_FILE_PATH'. + + +***Model Training*** +# python sgd_qa.py + do_training=True + model.dataset.data_dir= + model.dataset.dialogues_example_dir= + model.validation_ds.ds_item= + trainer.max_epochs= + trainer.gpus=[] + + +***Model Evaluation*** +# python sgd_qa.py + do_training=False + model.test_ds.ds_item= + +To load a pretrained checkpoint from the cloud prior to training (e.g. for fine-tuning) or evaluation you can set cfg.from_pretrained=. You can find all pretrained model names by using +SGDQAModel.list_available_models(). To load a local checkpoint use model.restore_from() + +""" + +import os + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models.dialogue_state_tracking.sgdqa_model import SGDQAModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="sgdqa_config") +def main(cfg: DictConfig) -> None: + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + + if cfg.pretrained_model or (cfg.model.nemo_path and os.path.exists(cfg.model.nemo_path)): + if cfg.pretrained_model: + logging.info(f'Loading pretrained model {cfg.pretrained_model}') + model = SGDQAModel.from_pretrained(cfg.pretrained_model) + else: + logging.info(f'Restoring model from {cfg.model.nemo_path}') + model = SGDQAModel.restore_from(cfg.model.nemo_path) + if cfg.do_training: + model.setup_training_data(train_data_config=cfg.model.train_ds) + model.setup_multiple_validation_data(val_data_config=cfg.model.validation_ds) + else: + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + model = SGDQAModel(cfg.model, trainer=trainer) + + if cfg.do_training: + trainer.fit(model) + if cfg.model.nemo_path: + model.save_to(cfg.model.nemo_path) + + if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.ds_item is not None: + gpu = 1 if cfg.trainer.gpus != 0 else 0 + trainer = pl.Trainer(gpus=gpu) + model.setup_multiple_test_data(test_data_config=cfg.model.test_ds) + if model.prepare_test(trainer): + trainer.test(model) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/entity_linking/build_index.py b/models/tacotron2/NeMo/examples/nlp/entity_linking/build_index.py new file mode 100644 index 0000000000..eeba5c8313 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/entity_linking/build_index.py @@ -0,0 +1,201 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pickle as pkl +import random +from argparse import ArgumentParser + +import h5py +import numpy as np +import torch +from omegaconf import DictConfig, OmegaConf +from sklearn.decomposition import PCA +from tqdm import tqdm + +from nemo.collections.nlp.models import EntityLinkingModel +from nemo.utils import logging + +try: + import faiss +except ModuleNotFoundError: + logging.warning("Faiss is required for building the index. Please install faiss-gpu") + +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + +def build_index(cfg: DictConfig, model: object): + """ + Builds faiss index from index dataset specified in the config. + + Args: + cfg (DictConfig): Config file specifying index parameters + model (object): Encoder model + """ + + # Get index dataset embeddings + # PCA model exists and index embeddings have already been PCAed, no need to re-extract/PCA them + if cfg.apply_pca and os.path.isfile(cfg.pca.pca_save_name) and os.path.isfile(cfg.pca_embeddings_save_name): + logging.info("Loading reduced dimensionality embeddings") + embeddings = h5py.File(cfg.pca_embeddings_save_name, "r") + embeddings = embeddings[cfg.index_ds.name][:] + + elif os.path.isfile(cfg.embedding_save_name): + logging.info("Loading previously extracted index dataset embeddings") + embeddings = h5py.File(cfg.embedding_save_name, "r") + embeddings = embeddings[cfg.index_ds.name][:] + + else: + logging.info("Encoding index dataset, this may take a while") + index_dataloader = model.setup_dataloader(cfg.index_ds, is_index_data=True) + embeddings, concept_ids = get_index_embeddings(cfg, index_dataloader, model) + + # Create pca model to reduce dimensionality of index dataset and decrease memory footprint + if cfg.apply_pca: + + # Need to train PCA model and apply PCA transformation with newly trained model + if not os.path.isfile(cfg.pca.pca_save_name): + logging.info("Fitting PCA model for embedding dimensionality reduction") + pca_train_set = random.sample(list(embeddings), k=int(len(embeddings) * cfg.pca.sample_fraction)) + pca = PCA(n_components=cfg.pca.output_dim) + pca.fit(pca_train_set) + pkl.dump(pca, open(cfg.pca.pca_save_name, "wb")) + embeddings = reduce_embedding_dim(pca, embeddings, cfg) + + # PCA model already trained, just need to reduce dimensionality of all embeddings + elif not os.path.isfile(cfg.pca_embeddings_save_name): + pca = pkl.load(open(cfg.pca.pca_save_name, "rb")) + embeddings = reduce_embedding_dim(pca, embeddings, cfg) + + # Build faiss index from embeddings + logging.info(f"Training index with embedding dim size {cfg.dims} using {faiss.get_num_gpus()} gpus") + quantizer = faiss.IndexFlatL2(cfg.dims) + index = faiss.IndexIVFFlat(quantizer, cfg.dims, cfg.nlist) + index = faiss.index_cpu_to_all_gpus(index) + index.train(embeddings) + + logging.info("Adding dataset embeddings to index") + for i in tqdm(range(0, embeddings.shape[0], cfg.index_batch_size)): + index.add(embeddings[i : i + cfg.index_batch_size]) + + logging.info("Saving index") + faiss.write_index(faiss.index_gpu_to_cpu(index), cfg.index_save_name) + logging.info("Index built and saved") + + +def reduce_embedding_dim(pca, embeddings, cfg): + """Apply PCA transformation to index dataset embeddings""" + + logging.info("Applying PCA transformation to entire index dataset") + embeddings = np.array(pca.transform(embeddings), dtype=np.float32) + emb_file = h5py.File(cfg.pca_embeddings_save_name, "w") + emb_file.create_dataset(cfg.index_ds.name, data=embeddings) + emb_file.close() + + return embeddings + + +def get_index_embeddings(cfg: DictConfig, dataloader: object, model: object): + """Use entity linking encoder to get embeddings for full index dataset""" + embeddings = [] + concept_ids = [] + + with torch.no_grad(): + for batch in tqdm(dataloader): + input_ids, token_type_ids, input_mask, batch_concept_ids = batch + input_ids = input_ids.to(device) + token_type_ids = token_type_ids.to(device) + input_mask = input_mask.to(device) + batch_embeddings = model.forward( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=input_mask + ) + + embeddings.extend(batch_embeddings.detach().cpu().numpy()) + concept_ids.extend(batch_concept_ids.numpy()) + + emb_file = h5py.File(cfg.embedding_save_name, "w") + emb_file.create_dataset(cfg.index_ds.name, data=embeddings) + emb_file.close() + + pkl.dump(concept_ids, open(cfg.concept_id_save_name, "wb")) + + return embeddings, concept_ids + + +def load_model(cfg: DictConfig, restore: bool): + """ + Loads encoder model. + + Args: + cfg: Config file specifying model parameters + restore: Whether to restore model weights trained + by the user. Otherwise will load weights + used before self alignment pretraining. + """ + + if restore: + model = EntityLinkingModel.restore_from(cfg.nemo_path) + else: + cfg.train_ds = None + cfg.validation_ds = None + cfg.test_ds = None + model = EntityLinkingModel(cfg) + + model = model.to(device) + + return model + + +def main(cfg: DictConfig, restore: bool): + """ + Builds new index if one hasn't been built yet. + + Args: + cfg: Config file specifying index parameters + restore: Whether to restore model weights trained + by the user. Otherwise will load weights + used before self alignment pretraining. + """ + + logging.info("Loading entity linking encoder model") + model = load_model(cfg.model, restore) + + if not os.path.isfile(cfg.index.index_save_name) or ( + cfg.apply_pca and not os.path.isfile(cfg.index.pca.pca_save_name) + ): + logging.info("Building index") + build_index(cfg.index, model) + else: + logging.info("Index and pca model (if required) already exists. Skipping build index step.") + + if not os.path.isfile(cfg.index.idx_to_id): + logging.info("Mapping entity index postions to ids") + map_idx_to_ids(cfg.index) + else: + logging.info("Map from concept index to id already exists. Skipping mapping step.") + + +if __name__ == '__main__': + parser = ArgumentParser() + parser.add_argument( + "--restore", action="store_true", help="Whether to restore encoder model weights from nemo path" + ) + parser.add_argument("--project_dir", required=False, type=str, default=".") + parser.add_argument("--cfg", required=False, type=str, default="./conf/umls_medical_entity_linking_config.yaml") + args = parser.parse_args() + + cfg = OmegaConf.load(args.cfg) + cfg.project_dir = args.project_dir + + main(cfg, args.restore) diff --git a/models/tacotron2/NeMo/examples/nlp/entity_linking/conf/tiny_example_entity_linking_config.yaml b/models/tacotron2/NeMo/examples/nlp/entity_linking/conf/tiny_example_entity_linking_config.yaml new file mode 100644 index 0000000000..36a0273060 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/entity_linking/conf/tiny_example_entity_linking_config.yaml @@ -0,0 +1,90 @@ +project_dir: null +name: SelfAlignmentPretrainingForMedicalEntityLinking +trainer: + gpus: 1 + num_nodes: 1 + max_epochs: 2 + max_steps: null + accumulate_grad_batches: 1 + precision: 16 + amp_level: O1 + accelerator: ddp + gradient_clip_val: 0.0 + log_every_n_steps: 1 + val_check_interval: 2 + checkpoint_callback: false + logger: false +model: + nemo_path: ??? + max_seq_length: 128 + language_model: + pretrained_model_name: bert-base-uncased + config_file: null + config: null + lm_checkpoint: null + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} + vocab_file: null + tokenizer_model: null + do_lower_case: true + loss_params: null + train_ds: + data_file: ??? + max_seq_length: ${model.max_seq_length} + batch_size: 8 + shuffle: true + num_workers: 2 + pin_memory: false + drop_last: false + validation_ds: + data_file: ??? + max_seq_length: ${model.max_seq_length} + batch_size: 8 + shuffle: false + num_workers: 2 + pin_memory: false + drop_last: false + optim: + name: adam + lr: 3.0e-05 + weight_decay: 0.0 + sched: + name: CosineAnnealing + warmup_steps: null + warmup_ratio: 0.1 + min_lr: 0.0 + last_epoch: -1 +index: + dims: 768 + nlist: 2 + top_n: 3 + query_num_factor: 20 + index_save_name: ??? + index_batch_size: 10 + index_ds: + name: tiny_example + data_file: ??? + max_seq_length: ${model.max_seq_length} + batch_size: 100 + shuffle: false + num_workers: 2 + pin_memory: false + drop_last: false + idx_to_id: ${project_dir}/idx_to_id.pkl + id_to_string: ${project_dir}/id_to_string.pkl + concept_id_save_name: ${project_dir}/tiny_example_concept_ids.pkl + embedding_save_name: ${project_dir}/tiny_example_concept_embeddings.hdf5 + pca_embeddings_save_name: null + apply_pca: false + pca: null +exp_manager: + exp_dir: . + name: ${project_dir}/SelfAlignmentPretrainingTinyExample + create_tensorboard_logger: true + create_checkpoint_callback: true +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/nlp/entity_linking/conf/umls_medical_entity_linking_config.yaml b/models/tacotron2/NeMo/examples/nlp/entity_linking/conf/umls_medical_entity_linking_config.yaml new file mode 100644 index 0000000000..5b6709e722 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/entity_linking/conf/umls_medical_entity_linking_config.yaml @@ -0,0 +1,95 @@ +project_dir: ??? +name: SelfAlignmentPretrainingForMedicalEntityLinking +trainer: + gpus: 1 + num_nodes: 1 + max_epochs: 2 + max_steps: null + accumulate_grad_batches: 1 + precision: 16 + amp_level: O1 + accelerator: ddp + gradient_clip_val: 0.0 + log_every_n_steps: 1 + val_check_interval: 1000 + checkpoint_callback: false + logger: false +model: + nemo_path: ${project_dir}/sap_bert_umls.nemo + raw_data: ${project_dir}/data/MRCONSO.RRF + max_seq_length: 128 + language_model: + pretrained_model_name: bert-base-uncased + config_file: null + config: null + lm_checkpoint: null + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} + vocab_file: null + tokenizer_model: null + do_lower_case: true + train_ds: + data_file: ${project_dir}/data/umls_train_pairs.tsv + max_seq_length: ${model.max_seq_length} + batch_size: 128 + shuffle: true + num_workers: 2 + pin_memory: false + drop_last: false + validation_ds: + data_file: ${project_dir}/data/umls_validation_pairs.tsv + max_seq_length: ${model.max_seq_length} + batch_size: 128 + shuffle: false + num_workers: 2 + pin_memory: false + drop_last: false + optim: + name: adam + lr: 3.0e-05 + weight_decay: 0.0 + sched: + name: CosineAnnealing + warmup_steps: null + warmup_ratio: 0.1 + min_lr: 0.0 + last_epoch: -1 +index: + dims: 256 + nlist: 300 + top_n: 5 + query_num_factor: 20 + index_save_name: ${project_dir}/medical_entity_linking_index + index_batch_size: 1000 + raw_data: ${model.raw_data} + index_ds: + name: umls + data_file: ${project_dir}/data/umls_index_concepts.tsv + max_seq_length: ${model.max_seq_length} + batch_size: 128 + shuffle: false + num_workers: 2 + pin_memory: false + drop_last: false + idx_to_id: ${project_dir}/data/idx_to_id.pkl + id_to_string: ${project_dir}/data/id_to_string.pkl + concept_id_save_name: ${project_dir}/data/concept_ids.pkl + embedding_save_name: ${project_dir}/data/medical_concept_embeddings.hdf5 + pca_embeddings_save_name: ${project_dir}/data/medical_concept_reduced_${index.dims}dim_embeddings.hdf5 + apply_pca: true + pca: + input_dim: 756 + output_dim: ${index.dims} + sample_fraction: 0.5 + pca_save_name: ${project_dir}/${index.pca.input_dim}_to_${index.pca.output_dim}_pca_model.pkl +exp_manager: + exp_dir: ${project_dir}/medical_entity_linking_experiments + name: sap_bert_umls + create_tensorboard_logger: true + create_checkpoint_callback: true +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/nlp/entity_linking/data/umls_dataset_processing.py b/models/tacotron2/NeMo/examples/nlp/entity_linking/data/umls_dataset_processing.py new file mode 100644 index 0000000000..03a17da3c0 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/entity_linking/data/umls_dataset_processing.py @@ -0,0 +1,189 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import pickle as pkl +import random +from argparse import ArgumentParser + +import pandas as pd +from omegaconf import OmegaConf +from tqdm import tqdm + +# Info on these headers can be found here on the UMLS website https://www.ncbi.nlm.nih.gov/books/NBK9685/ +# section 3.3.4 Table 1 +HEADERS = [ + 'CUI', + 'LAT', + 'TS', + 'LUI', + 'STT', + 'SUI', + 'ISPREF', + 'AUI', + 'SAUI', + 'SCUI', + 'SDUI', + 'SAB', + 'TTY', + 'CODE', + 'STR', + 'SRL', + 'SUPPRESS', + 'CVF', +] + + +def process_umls_training_dataset(data_path, train_save_name, val_save_name, max_pairs, train_split, headers): + """ + Generates and saves UMLS self alignment pretraining train and validation data. Takes the raw .RRF UMLS + data file and creates different pair combinations for entities with the same CUI. Each row in the output + will be formatted as 'CUI EntitySynonym1 EntitySynonym2' with each item in a row separated by tabs. + Saves two .tsv output files, one for the train split and one for the validation split. + Only data marked as English is added to the train and val splits. + + Arguments: + data_path (str): path to MRCONSO.RRF UMLS data file + train_save_name (str): path to where training data will be saved + val_save_name (str): path to where validation data will be saved + max_pairs (int): max number of pairs for any one CUI added to the train + or validation splits + train_split (float): precentage of raw data to be added to train set split + headers (list): column lables within MRCONSO.RRF + """ + + print("Loading training data file...") + df = pd.read_table(data_path, names=headers, index_col=False, delimiter='|') + train_file = open(train_save_name, 'w') + val_file = open(val_save_name, 'w') + + cui = df["CUI"].iloc[0] + names = [] + random.seed(2021) + + for idx in tqdm(range(len(df))): + # Address incorrectly formatted data + if type(df["STR"].iloc[idx]) != str or "|" in df["STR"].iloc[idx]: + continue + + # Collect all english concept strings matching the current CUI + if df["CUI"].iloc[idx] == cui and df["LAT"].iloc[idx] == "ENG": + concept_string = df["STR"].iloc[idx] + names.append(concept_string) + + else: + # Pair off concept synonyms to make training and val sets + pairs = list(itertools.combinations(names, 2)) + + if len(pairs) == 0: + # Not enough concepts gathered to make a pair + cui = df["CUI"].iloc[idx] + names = [df["STR"].iloc[idx]] + continue + + # Removing leading C to convert label string to int + cui = int(cui[1:]) + random.shuffle(pairs) + + # Keep up to max pairs number pairs for any one concept + for pair in pairs[:max_pairs]: + + # Want concepts in train and val splits to be randomly selected and mutually exclusive + add_to_train = random.random() + + if add_to_train <= train_split: + train_file.write(f'{cui}\t{pair[0]}\t{pair[1]}\n') + else: + val_file.write(f'{cui}\t{pair[0]}\t{pair[1]}\n') + + # Switch to next concept + cui = df["CUI"].iloc[idx] + names = [df["STR"].iloc[idx]] + + train_file.close() + val_file.close() + print("Finished making training and validation data") + + +def process_umls_index_dataset(data_path, data_savename, id2string_savename, headers): + """ + Generates data file needed to build a UMLS index and a hash table mapping each + CUI to one canonical concept string. Takes the raw .RRF data file and saves + a .tsv indec concept file as well as the a .pkl file of cui to concept string + mappings. Only data marked as English is added to the index data file. + + Arguments: + data_path (str): path to MRCONSO.RRF UMLS data file + data_savename (str): path to where .tsv index data will be saved + id2string_savename (str): path to where .pkl cui to string mapping will + be saved + headers (list): column lables within MRCONSO.RRF + """ + + print("Loading index data file...") + df = pd.read_table(data_path, names=headers, index_col=False, delimiter='|') + id2string = {} + + with open(data_savename, "w") as outfile: + for idx, row in tqdm(df.iterrows(), total=df.shape[0]): + # Address incorrectly formatted data + if type(row["STR"]) != str or "|" in row["STR"]: + continue + + cui = row["CUI"] + sent = row["STR"] + + # Removing leading C to convert label string to int + cui = int(cui[1:]) + + # Only keeping english concepts + if row["LAT"] == "ENG": + outfile.write(f'{cui}\t{sent}\n') + + # Matching each cui to one canonical string represention + if cui not in id2string and ":" not in sent: + id2string[cui] = sent + + outfile.close() + pkl.dump(id2string, open(id2string_savename, "wb")) + print("Finished saving index data and id to concept mapping") + + +if __name__ == '__main__': + parser = ArgumentParser() + parser.add_argument("--index", action="store_true", help="Whether to process data for building an index") + parser.add_argument("--project_dir", required=False, type=str, default=".") + parser.add_argument("--cfg", required=False, type=str, default="conf/umls_medical_entity_linking_config.yaml") + parser.add_argument( + "--max_pairs", required=False, type=int, default=50, help="Max number of train pairs for a single concepts" + ) + parser.add_argument( + "--train_split", required=False, type=float, default=0.99, help="Precentage of data to add to train set" + ) + + args = parser.parse_args() + cfg = OmegaConf.load(args.cfg) + cfg.project_dir = args.project_dir + + if args.index: + process_umls_index_dataset(cfg.index.raw_data, cfg.index.index_ds.data_file, cfg.index.id_to_string, HEADERS) + else: + process_umls_training_dataset( + cfg.model.raw_data, + cfg.model.train_ds.data_file, + cfg.model.validation_ds.data_file, + args.max_pairs, + args.train_split, + HEADERS, + ) diff --git a/models/tacotron2/NeMo/examples/nlp/entity_linking/query_index.py b/models/tacotron2/NeMo/examples/nlp/entity_linking/query_index.py new file mode 100644 index 0000000000..6cb51a7de1 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/entity_linking/query_index.py @@ -0,0 +1,166 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pickle as pkl +from argparse import ArgumentParser +from collections import OrderedDict +from typing import Dict + +import numpy as np +import torch +from build_index import load_model +from omegaconf import DictConfig, OmegaConf + +from nemo.utils import logging + +try: + import faiss +except ModuleNotFoundError: + logging.warning("Faiss is required for building the index. Please install faiss-gpu") + +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + +def get_query_embedding(query, model): + """Use entity linking encoder to get embedding for index query""" + model_input = model.tokenizer( + query, + add_special_tokens=True, + padding=True, + truncation=True, + max_length=512, + return_token_type_ids=True, + return_attention_mask=True, + ) + + query_emb = model.forward( + input_ids=torch.LongTensor([model_input["input_ids"]]).to(device), + token_type_ids=torch.LongTensor([model_input["token_type_ids"]]).to(device), + attention_mask=torch.LongTensor([model_input["attention_mask"]]).to(device), + ) + + return query_emb + + +def query_index( + query: str, cfg: DictConfig, model: object, index: object, pca: object, idx2id: dict, id2string: dict, +) -> Dict: + + """ + Query the nearest neighbor index of entities to find the + concepts in the index dataset that are most similar to the + query. + + Args: + query (str): entity to look up in the index + cfg (DictConfig): config object to specifiy query parameters + model (EntityLinkingModel): entity linking encoder model + index (object): faiss index + pca (object): sklearn pca transformation to be applied to queries + idx2id (dict): dictionary mapping unique concept dataset index to + its CUI + id2string (dict): dictionary mapping each unqiue CUI to a + representative english description of + the concept + Returns: + A dictionary with the concept ids of the index's most similar + entities as the keys and a tuple containing the string + representation of that concept and its cosine similarity to + the query as the values. + """ + query_emb = get_query_embedding(query, model).detach().cpu().numpy() + + if cfg.apply_pca: + query_emb = pca.transform(query_emb) + + dist, neighbors = index.search(query_emb.astype(np.float32), cfg.query_num_factor * cfg.top_n) + dist, neighbors = dist[0], neighbors[0] + unique_ids = OrderedDict() + neighbor_idx = 0 + + # Many of nearest neighbors could map to the same concept id, their idx is their unique identifier + while len(unique_ids) < cfg.top_n and neighbor_idx < len(neighbors): + concept_id_idx = neighbors[neighbor_idx] + concept_id = idx2id[concept_id_idx] + + # Only want one instance of each unique concept + if concept_id not in unique_ids: + concept = id2string[concept_id] + unique_ids[concept_id] = (concept, 1 - dist[neighbor_idx]) + + neighbor_idx += 1 + + unique_ids = dict(unique_ids) + + return unique_ids + + +def main(cfg: DictConfig, restore: bool): + """ + Loads faiss index and allows commandline queries + to the index. Builds new index if one hasn't been built yet. + + Args: + cfg: Config file specifying index parameters + restore: Whether to restore model weights trained + by the user. Otherwise will load weights + used before self alignment pretraining. + """ + + if not os.path.isfile(cfg.index.index_save_name) or ( + cfg.apply_pca and not os.path.isfile(cfg.index.pca.pca_save_name) or not os.path.isfile(cfg.index.idx_to_id) + ): + logging.warning("Either no index and/or no mapping from entity idx to ids exists. Please run `build_index.py`") + return + + logging.info("Loading entity linking encoder model") + model = load_model(cfg.model, restore) + + logging.info("Loading index and associated files") + index = faiss.read_index(cfg.index.index_save_name) + idx2id = pkl.load(open(cfg.index.idx_to_id, "rb")) + id2string = pkl.load(open(cfg.index.id_to_string, "rb")) # Should be created during dataset prep + + if cfg.index.apply_pca: + pca = pkl.load(open(cfg.index.pca.pca_save_name, "rb")) + + while True: + query = input("enter index query: ") + output = query_index(query, cfg.top_n, cfg.index, model, index, pca, idx2id, id2string) + + if query == "exit": + break + + for concept_id in output: + concept_details = output[concept_id] + concept_id = "C" + str(concept_id).zfill(7) + print(concept_id, concept_details) + + print("----------------\n") + + +if __name__ == '__main__': + parser = ArgumentParser() + parser.add_argument( + "--restore", action="store_true", help="Whether to restore encoder model weights from nemo path" + ) + parser.add_argument("--project_dir", required=False, type=str, default=".") + parser.add_argument("--cfg", required=False, type=str, default="./conf/umls_medical_entity_linking_config.yaml") + args = parser.parse_args() + + cfg = OmegaConf.load(args.cfg) + cfg.project_dir = args.project_dir + + main(cfg, args.restore) diff --git a/models/tacotron2/NeMo/examples/nlp/entity_linking/self_alignment_pretraining.py b/models/tacotron2/NeMo/examples/nlp/entity_linking/self_alignment_pretraining.py new file mode 100644 index 0000000000..0d0e0d3683 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/entity_linking/self_alignment_pretraining.py @@ -0,0 +1,49 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Please see tutorial at Nemo/tutorials/nlp/Entity_Linking_Medical.ipynb for +# more information on entity linking and self alignment pretraining. + +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer + +from nemo.collections.nlp.models import EntityLinkingModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="umls_medical_entity_linking_config.yaml") +def main(cfg: DictConfig) -> None: + logging.info(f"\nConfig Params:\n{OmegaConf.to_yaml(cfg)}") + trainer = Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + + logging.info(f"Loading weights from pretrained model {cfg.model.language_model.pretrained_model_name}") + model = EntityLinkingModel(cfg=cfg.model, trainer=trainer) + logging.info("===========================================================================================") + logging.info('Starting training...') + trainer.fit(model) + logging.info('Training finished!') + logging.info("===========================================================================================") + + if cfg.model.nemo_path: + # '.nemo' file contains the last checkpoint and the params to initialize the model + model.save_to(cfg.model.nemo_path) + logging.info(f'Model is saved into `.nemo` file: {cfg.model.nemo_path}') + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/glue_benchmark/glue_benchmark.py b/models/tacotron2/NeMo/examples/nlp/glue_benchmark/glue_benchmark.py new file mode 100644 index 0000000000..3242986749 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/glue_benchmark/glue_benchmark.py @@ -0,0 +1,73 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +## Tasks +This script works with all GLUE Benchmark tasks, more details about the GLUE Benchmark could be found at +https://gluebenchmark.com/ + +More details on how to use this script could be found in tutorials/nlp/GLUE_Benchmark.ipynb + +## Model Training + +To train GLUEModel with the default config file, run: + python glue_benchmark.py \ + model.dataset.data_dir= \ + model.task_name=TASK_NAME \ + trainer.max_epochs= \ + trainer.gpus="[] + +Supported task names: +["cola", "sst-2", "mrpc", "sts-b", "qqp", "mnli", "qnli", "rte", "wnli"] +Note, MNLI task includes both matched and mismatched dev sets +""" + +import os + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models import GLUEModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_name="glue_benchmark_config") +def main(cfg: DictConfig) -> None: + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(**cfg.trainer) + exp_manager_cfg = cfg.get("exp_manager", None) + + if exp_manager_cfg: + exp_manager_cfg.name = cfg.model.task_name + logging.info(f'Setting task_name to {exp_manager_cfg.name} in exp_manager') + exp_manager(trainer, exp_manager_cfg) + + if cfg.model.nemo_path and os.path.exists(cfg.model.nemo_path): + model = GLUEModel.restore_from(cfg.model.nemo_path) + logging.info(f'Restoring model from {cfg.model.nemo_path}') + model.update_data_dir(data_dir=cfg.model.dataset.data_dir) + model.setup_training_data() + model.setup_multiple_validation_data() + trainer.fit(model) + else: + model = GLUEModel(cfg.model, trainer=trainer) + trainer.fit(model) + if cfg.model.nemo_path: + model.save_to(cfg.model.nemo_path) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/glue_benchmark/glue_benchmark_config.yaml b/models/tacotron2/NeMo/examples/nlp/glue_benchmark/glue_benchmark_config.yaml new file mode 100644 index 0000000000..23b5522ae3 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/glue_benchmark/glue_benchmark_config.yaml @@ -0,0 +1,82 @@ +# GLUE Benchmark with pre-trained BERT models +supported_tasks: &supported_tasks ['cola', 'sst-2', 'mrpc', 'sts-b', 'qqp', 'mnli', 'qnli', 'rte', 'wnli'] + +trainer: + gpus: 1 # the number of gpus, 0 for CPU + num_nodes: 1 + max_epochs: 3 + max_steps: null # precedence over max_epochs + accumulate_grad_batches: 1 # accumulates grads every k batches + amp_level: O0 # O1/O2 for mixed precision + precision: 16 + accelerator: ddp + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + +model: + task_name: &task_name mrpc # choose from: ["cola", "sst-2", "mrpc", "sts-b", "qqp", "mnli", "qnli", "rte", "wnli"] GLUE task name, MNLI includes both matched and mismatched dev sets + supported_tasks: *supported_tasks + output_dir: null # dir to write write predictions + nemo_path: null # filename to save the model and associated artifacts to .nemo file + dataset: + data_dir: ??? # /path/to/data + max_seq_length: 128 + use_cache: true + + # shared across dataloaders: + num_workers: 2 + pin_memory: false + drop_last: false + + train_ds: + ds_item: 'train.tsv' + shuffle: true + num_samples: -1 + batch_size: 32 + + validation_ds: + ds_item: 'dev.tsv' # for MNLI 'dev_matched.tsv' and 'dev_mismatched.tsv' will de used + shuffle: false + num_samples: -1 + batch_size: 32 + + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # or sentencepiece + vocab_file: null # path to vocab file + tokenizer_model: null # only used if tokenizer is sentencepiece + special_tokens: null # only necessary for adding transformer/bert-specific special tokens to tokenizer if the tokenizer does not already have these inherently. + + language_model: + pretrained_model_name: bert-base-uncased + lm_checkpoint: null + config_file: null # json file, precedence over config + config: null + + optim: + name: adam + lr: 5e-5 + weight_decay: 0.00 + + sched: + name: WarmupAnnealing + # Scheduler params + warmup_steps: null + warmup_ratio: 0.1 + last_epoch: -1 + + # pytorch lightning args + monitor: val_loss + reduce_on_plateau: false + +exp_manager: + exp_dir: null # exp_dir for your experiment, if None, defaults to "./NeMo_experiments" + name: *task_name # The name of your model + create_tensorboard_logger: True # Whether you want exp_manger to create a tb logger + create_checkpoint_callback: True # Whether you want exp_manager to create a modelcheckpoint callback + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/nlp/information_retrieval/bert_dpr.py b/models/tacotron2/NeMo/examples/nlp/information_retrieval/bert_dpr.py new file mode 100644 index 0000000000..2d9cd962ff --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/information_retrieval/bert_dpr.py @@ -0,0 +1,35 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models import BertDPRModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="bert_ir_config") +def main(cfg: DictConfig) -> None: + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + bert_dpr_model = BertDPRModel(cfg.model, trainer=trainer) + trainer.fit(bert_dpr_model) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/information_retrieval/bert_joint_ir.py b/models/tacotron2/NeMo/examples/nlp/information_retrieval/bert_joint_ir.py new file mode 100644 index 0000000000..1bb164e580 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/information_retrieval/bert_joint_ir.py @@ -0,0 +1,35 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models import BertJointIRModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="bert_ir_config") +def main(cfg: DictConfig) -> None: + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + bert_joint_ir_model = BertJointIRModel(cfg.model, trainer=trainer) + trainer.fit(bert_joint_ir_model) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/information_retrieval/conf/bert_ir_config.yaml b/models/tacotron2/NeMo/examples/nlp/information_retrieval/conf/bert_ir_config.yaml new file mode 100644 index 0000000000..e4fe72b349 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/information_retrieval/conf/bert_ir_config.yaml @@ -0,0 +1,99 @@ +# Fine-tuning BERT model for information retrieval +name: &name BertIR +trainer: + gpus: 1 # the number of gpus, 0 for CPU, or list with gpu indices + num_nodes: 1 + max_epochs: 2 # the number of training epochs + max_steps: null # precedence over max_epochs + accumulate_grad_batches: 1 # accumulates grads every k batches + precision: 16 # 16 to use AMP + amp_level: O1 # O1 or O2 if using AMP + accelerator: ddp + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 0.05 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + checkpoint_callback: false # provided by exp_manager + logger: false # provided by exp_manager + +model: + nemo_path: null # exported .nemo path + + language_model: + pretrained_model_name: bert-base-uncased + sim_score_dropout: 0.1 + lm_checkpoint: null + config: + attention_probs_dropout_prob: 0.1 + hidden_act: gelu + hidden_dropout_prob: 0.1 + hidden_size: 768 + initializer_range: 0.02 + intermediate_size: 3072 + max_position_embeddings: 512 + num_attention_heads: 12 + num_hidden_layers: 12 + type_vocab_size: 2 + vocab_size: 30522 + config_file: null # json file, precedence over config + + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec + vocab_file: null # path to vocab file + tokenizer_model: null # tokenizer model for sentencepiece + special_tokens: null + + train_ds: + passages: null # path to file with passages and their indices + queries: null # path to file with training queries and their indices + query_to_passages: null + # path to file with training examples which have the form of + # (query_id, relevant_passage_id, irrelevant_passage_1_id, ..., irrelevant_passage_n_id) + num_negatives: 10 + batch_size: 6 + psg_cache_format: npz + shuffle: true + num_samples: -1 # number of samples to be considered, -1 means all the dataset + num_workers: 1 + drop_last: false + pin_memory: false + + validation_ds: + passages: null # path to file with passages and their indices + queries: null # path to file with validation queries and their indices + query_to_passages: null # path to file with passages to re-rank for each validation query + num_negatives: 10 + batch_size: 6 + psg_cache_format: pkl + shuffle: false + num_samples: -1 # number of samples to be considered, -1 means all the dataset + num_workers: 1 + drop_last: false + pin_memory: false + + optim: + name: adam + lr: 1e-5 + betas: [0.9, 0.999] + weight_decay: 0 + + sched: + name: WarmupAnnealing + warmup_steps: null + warmup_ratio: 0.05 + last_epoch: -1 + + # pytorch lightning args + monitor: val_loss + reduce_on_plateau: false + +exp_manager: + exp_dir: null # where to store logs and checkpoints + name: *name # name of experiment + create_tensorboard_logger: True + create_checkpoint_callback: True + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/nlp/information_retrieval/construct_random_negatives.py b/models/tacotron2/NeMo/examples/nlp/information_retrieval/construct_random_negatives.py new file mode 100644 index 0000000000..36da96f503 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/information_retrieval/construct_random_negatives.py @@ -0,0 +1,49 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +import numpy as np +import pandas as pd + + +def construct_negatives(input_file, output_file, num_passages, num_negatives): + qrels = pd.read_csv(input_file, delimiter="\t", header=None) + with open(output_file, "w") as f: + for i in range(len(qrels)): + query_id, rel_passage_id = qrels[0][i], qrels[2][i] + negatives = np.random.randint(num_passages, size=num_negatives) + output_ids = [query_id, rel_passage_id] + negatives.tolist() + output_str = [str(id_) for id_ in output_ids] + print("\t".join(output_str), file=f) + + +def main(): + parser = argparse.ArgumentParser(description="Negative passages construction") + parser.add_argument("--data", type=str, default="msmarco_dataset", help="path to folder with data") + parser.add_argument("--num_passages", type=int, default=8841823, help="total number of passages") + parser.add_argument("--num_negatives", type=int, default=10, help="number of negatives per positive") + args = parser.parse_args() + + for mode in ["train", "dev"]: + construct_negatives( + input_file=f"{args.data}/qrels.{mode}.tsv", + output_file=f"{args.data}/query2passages.{mode}.tsv", + num_passages=args.num_passages, + num_negatives=args.num_negatives, + ) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/information_retrieval/get_msmarco.sh b/models/tacotron2/NeMo/examples/nlp/information_retrieval/get_msmarco.sh new file mode 100644 index 0000000000..0bf286c3f7 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/information_retrieval/get_msmarco.sh @@ -0,0 +1,22 @@ +echo "=== Acquiring MSMARCO dataset ===" +echo "---" + +mkdir -p msmarco_dataset +cd msmarco_dataset + +echo "- Downloading passages" +wget --quiet --continue https://msmarco.blob.core.windows.net/msmarcoranking/collection.tar.gz +tar -xzvf collection.tar.gz +rm collection.tar.gz + +echo "- Downloading queries" +wget --quiet --continue https://msmarco.blob.core.windows.net/msmarcoranking/queries.tar.gz +tar -xzvf queries.tar.gz +rm queries.tar.gz +rm queries.eval.tsv + +echo "- Downloading relevance labels" +wget --quiet --continue https://msmarco.blob.core.windows.net/msmarcoranking/qrels.train.tsv +wget --quiet --continue https://msmarco.blob.core.windows.net/msmarcoranking/qrels.dev.tsv + +echo "---" \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/conf/intent_slot_classification_config.yaml b/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/conf/intent_slot_classification_config.yaml new file mode 100644 index 0000000000..502458c1d3 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/conf/intent_slot_classification_config.yaml @@ -0,0 +1,111 @@ +# Intent and Slot classification with pretrained BERT models + +trainer: + gpus: 1 # the number of gpus, 0 for CPU + num_nodes: 1 + max_epochs: 50 + max_steps: null # precedence over max_epochs + accumulate_grad_batches: 1 # accumulates grads every k batches + amp_level: O0 # O1/O2 for mixed precision + precision: 32 # Should be set to 16 for O1 and O2 amp_level to enable the AMP. + accelerator: ddp + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager + +model: + nemo_path: null # filename to save the model and associated artifacts to .nemo file + data_dir: ??? # /path/to/data + class_labels: + intent_labels_file: intent_labels.csv + slot_labels_file: slot_labels.csv + class_balancing: null # or weighted_loss + intent_loss_weight: 0.6 # relation of intent to slot loss in total loss (between 0 to 1) + pad_label: -1 # if -1 not slot token will be used + ignore_extra_tokens: false + ignore_start_end: true # do not use first and last token for slot training + + train_ds: + prefix: train + batch_size: 32 + shuffle: true + num_samples: -1 + num_workers: 2 + drop_last: false + pin_memory: false + + validation_ds: + prefix: test + batch_size: 32 + shuffle: false + num_samples: -1 + num_workers: 2 + drop_last: false + pin_memory: false + + test_ds: + prefix: test + batch_size: 32 + shuffle: false + num_samples: -1 + num_workers: 2 + drop_last: false + pin_memory: false + + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # or sentencepiece + vocab_file: null # path to vocab file + tokenizer_model: null # only used if tokenizer is sentencepiece + special_tokens: null + + language_model: + max_seq_length: 50 + pretrained_model_name: bert-base-uncased + lm_checkpoint: null + config_file: null # json file, precedence over config + config: null + + head: + num_output_layers: 2 + fc_dropout: 0.1 + + optim: + name: adam + lr: 2e-5 + args: + name: auto + params: + weight_decay: 0.01 + + sched: + name: WarmupAnnealing + iters_per_batch: null # computed at runtime + max_steps: null # computed at runtime or explicitly set here + + # pytorch lightning args + monitor: val_loss + reduce_on_plateau: false + + # scheduler config override + args: + name: auto + params: + warmup_steps: null + warmup_ratio: 0.1 + last_epoch: -1 + +exp_manager: + exp_dir: null # exp_dir for your experiment, if None, defaults to "./nemo_experiments" + name: "IntentSlot" # The name of your model + create_tensorboard_logger: true # Whether you want exp_manger to create a tb logger + create_checkpoint_callback: true # Whether you want exp_manager to create a modelcheckpoint callback + +hydra: + run: + dir: . + job_logging: + root: + handlers: null \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/data/assistant_utils.py b/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/data/assistant_utils.py new file mode 100644 index 0000000000..f47094e276 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/data/assistant_utils.py @@ -0,0 +1,151 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import shutil + +from nemo.collections.nlp.data.data_utils.data_preprocessing import DATABASE_EXISTS_TMP, if_exist, write_files +from nemo.utils import logging + + +def copy_input_files(infold): + """ Put training files in convenient place for conversion to our format. """ + our_infold = infold + "/dataset" + + if os.path.exists(our_infold + "/trainset") and os.path.exists(our_infold + "/testset"): + logging.info("Input folders exists") + return + + logging.info(f"Copying files to input folder: {our_infold}") + os.makedirs(infold, exist_ok=True) + + old_infold = ( + infold + '/CrossValidation/autoGeneFromRealAnno/autoGene_2018_03_22-13_01_25_169/CrossValidation/KFold_1' + ) + if not os.path.exists(our_infold + "/trainset"): + shutil.copytree(old_infold + '/trainset', our_infold + '/trainset') + + if not os.path.exists(our_infold + "/testset"): + shutil.copytree(old_infold + '/testset/csv', our_infold + '/testset') + + +def get_intents(infold): + """ Get list of intents from file names. """ + intents = [f[:-4] for f in os.listdir(infold)] + intents.sort() + print(f'Found {len(intents)} intents') + return intents + + +def get_intent_queries(infold, intent_names, mode): + """ Get list of queries with their corresponding intent number. """ + intent_queries = ['sentence\tlabel\n'] + + for index, intent in enumerate(intent_names): + queries = open(f'{infold}/{mode}set/{intent}.csv', 'r').readlines() + for query in queries[1:]: + phrases = query.split(";") + intent_query = phrases[4][1:-1] + "\t" + str(index) + intent_queries.append(intent_query) + + return intent_queries + + +def get_slots(infold, modes): + """ + Find a lost of unique slot types in training and testing data. + We use a single slot type name both for starting and continuation tokes (not using B-, I- notation). + """ + slots = set() + + for mode in modes: + path = f'{infold}/{mode}set' + for filename in os.listdir(path): + lines = open(f'{path}/{filename}', 'r').readlines() + for line in lines[1:]: + query = line.split(";")[3] + slot_phrases = re.findall('\[.*?\]', query) + for slot_phrase in slot_phrases: + slot = slot_phrase.split(" : ")[0][1:] + slots.add(slot) + + slots = sorted(slots) + slots.append("O") + print(f'Found {len(slots)} slot types') + return slots + + +def get_slot_queries(infold, slot_dict, mode, intent_names): + """ Convert each word in a query to corresponding slot number. """ + slot_queries = [] + outside_slot = len(slot_dict) - 1 + + # keep the same order of files/queries as for intents + for intent in intent_names: + lines = open(f'{infold}/{mode}set/{intent}.csv', 'r').readlines() + for line in lines[1:]: + slot_query = "" + query = line.split(";")[3] + words = query.split(" ") + current_slot = outside_slot + for word in words: + if word[0] == "[": + current_slot = slot_dict[word[1:]] + elif word[0] == ":": + continue + else: + slot_query += str(current_slot) + " " + if word[-1] == ']': + current_slot = outside_slot + + slot_queries.append(slot_query.strip()) + + return slot_queries + + +def process_assistant(infold, outfold, modes=['train', 'test']): + """ + https://github.com/xliuhw/NLU-Evaluation-Data - this dataset includes + about 25 thousand examples with 66 various multi-domain intents and 57 entity types. + """ + if if_exist(outfold, [f'{mode}_slots.tsv' for mode in modes]): + logging.info(DATABASE_EXISTS_TMP.format('robot', outfold)) + return outfold + + logging.info(f'Processing assistant commands dataset and store at {outfold}') + os.makedirs(outfold, exist_ok=True) + + # copy train/test files to the convenient directory to work with + copy_input_files(infold) + infold += "/dataset" + + # get list of intents from train folder (test folder supposed to be the same) + intent_names = get_intents(infold + "/trainset") + write_files(intent_names, f'{outfold}/dict.intents.csv') + + # get all train and test queries with their intent + for mode in modes: + intent_queries = get_intent_queries(infold, intent_names, mode) + write_files(intent_queries, f'{outfold}/{mode}.tsv') + + # get list of all unique slots in training and testing files + slot_types = get_slots(infold, modes) + write_files(slot_types, f'{outfold}/dict.slots.csv') + + # create files of slot queries + slot_dict = {k: v for v, k in enumerate(slot_types)} + for mode in modes: + slot_queries = get_slot_queries(infold, slot_dict, mode, intent_names) + write_files(slot_queries, f'{outfold}/{mode}_slots.tsv') diff --git a/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/data/import_datasets.py b/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/data/import_datasets.py new file mode 100755 index 0000000000..9c58ded287 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/data/import_datasets.py @@ -0,0 +1,265 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import shutil +from os.path import exists + +from assistant_utils import process_assistant + +from nemo.collections.nlp.data.data_utils.data_preprocessing import ( + DATABASE_EXISTS_TMP, + MODE_EXISTS_TMP, + create_dataset, + get_dataset, + get_vocab, + if_exist, +) +from nemo.utils import logging + + +def ids2text(ids, vocab): + return ' '.join([vocab[int(id_)] for id_ in ids]) + + +def process_atis(infold, outfold, modes=['train', 'test'], do_lower_case=False): + """ MSFT's dataset, processed by Kaggle + https://www.kaggle.com/siddhadev/atis-dataset-from-ms-cntk + """ + vocab = get_vocab(f'{infold}/atis.dict.vocab.csv') + + if if_exist(outfold, [f'{mode}.tsv' for mode in modes]): + logging.info(DATABASE_EXISTS_TMP.format('ATIS', outfold)) + return outfold + logging.info(f'Processing ATIS dataset and storing at {outfold}.') + + os.makedirs(outfold, exist_ok=True) + + outfiles = {} + for mode in modes: + outfiles[mode] = open(os.path.join(outfold, mode + '.tsv'), 'w') + outfiles[mode].write('sentence\tlabel\n') + outfiles[mode + '_slots'] = open(f'{outfold}/{mode}_slots.tsv', 'w') + + queries = open(f'{infold}/atis.{mode}.query.csv', 'r').readlines() + intents = open(f'{infold}/atis.{mode}.intent.csv', 'r').readlines() + slots = open(f'{infold}/atis.{mode}.slots.csv', 'r').readlines() + + for i, query in enumerate(queries): + sentence = ids2text(query.strip().split()[1:-1], vocab) + if do_lower_case: + sentence = sentence.lower() + outfiles[mode].write(f'{sentence}\t{intents[i].strip()}\n') + slot = ' '.join(slots[i].strip().split()[1:-1]) + outfiles[mode + '_slots'].write(slot + '\n') + + shutil.copyfile(f'{infold}/atis.dict.intent.csv', f'{outfold}/dict.intents.csv') + shutil.copyfile(f'{infold}/atis.dict.slots.csv', f'{outfold}/dict.slots.csv') + for mode in modes: + outfiles[mode].close() + + +def process_snips(infold, outfold, do_lower_case, modes=['train', 'test'], dev_split=0.1): + if not os.path.exists(infold): + link = 'https://github.com/snipsco/spoken-language-understanding-research-datasets' + raise ValueError(f'Data not found at {infold}. ' f'You may request to download the SNIPS dataset from {link}.') + + exist = True + for dataset in ['light', 'speak', 'all']: + if if_exist(f'{outfold}/{dataset}', [f'{mode}.tsv' for mode in modes]): + logging.info(DATABASE_EXISTS_TMP.format('SNIPS-' + dataset, outfold)) + else: + exist = False + if exist: + return outfold + + logging.info(f'Processing SNIPS dataset and storing at folders "speak", "light" and "all" under {outfold}.') + logging.info( + f'Processing and importing "smart-speaker-en-close-field" -> "speak" and "smart-speaker-en-close-field" -> "light".' + ) + + os.makedirs(outfold, exist_ok=True) + + speak_dir = 'smart-speaker-en-close-field' + light_dir = 'smart-lights-en-close-field' + + light_files = [f'{infold}/{light_dir}/dataset.json'] + speak_files = [f'{infold}/{speak_dir}/training_dataset.json'] + speak_files.append(f'{infold}/{speak_dir}/test_dataset.json') + + light_train, light_dev, light_slots, light_intents = get_dataset(light_files, dev_split) + speak_train, speak_dev, speak_slots, speak_intents = get_dataset(speak_files) + + create_dataset(light_train, light_dev, light_slots, light_intents, do_lower_case, f'{outfold}/light') + create_dataset(speak_train, speak_dev, speak_slots, speak_intents, do_lower_case, f'{outfold}/speak') + create_dataset( + light_train + speak_train, + light_dev + speak_dev, + light_slots | speak_slots, + light_intents | speak_intents, + do_lower_case, + f'{outfold}/all', + ) + + +def process_jarvis_datasets( + infold, outfold, modes=['train', 'test', 'dev'], do_lower_case=False, ignore_prev_intent=False +): + """ process and convert Jarvis datasets into NeMo's BIO format + """ + dataset_name = "jarvis" + if if_exist(outfold, ['dict.intents.csv', 'dict.slots.csv']): + logging.info(DATABASE_EXISTS_TMP.format(dataset_name, outfold)) + return outfold + + logging.info(f'Processing {dataset_name} dataset and storing at {outfold}') + + os.makedirs(outfold, exist_ok=True) + + outfiles = {} + intents_list = {} + slots_list = {} + slots_list_all = {} + + outfiles['dict_intents'] = open(f'{outfold}/dict.intents.csv', 'w') + outfiles['dict_slots'] = open(f'{outfold}/dict.slots.csv', 'w') + + outfiles['dict_slots'].write('O\n') + slots_list["O"] = 0 + slots_list_all["O"] = 0 + + for mode in modes: + if if_exist(outfold, [f'{mode}.tsv']): + logging.info(MODE_EXISTS_TMP.format(mode, dataset_name, outfold, mode)) + continue + + if not if_exist(infold, [f'{mode}.tsv']): + logging.info(f'{mode} mode of {dataset_name}' f' is skipped as it was not found.') + continue + + outfiles[mode] = open(os.path.join(outfold, mode + '.tsv'), 'w') + outfiles[mode].write('sentence\tlabel\n') + outfiles[mode + '_slots'] = open(f'{outfold}/{mode}_slots.tsv', 'w') + + queries = open(f'{infold}/{mode}.tsv', 'r').readlines() + + for i, query in enumerate(queries): + line_splits = query.strip().split("\t") + if len(line_splits) == 3: + intent_str, slot_tags_str, sentence = line_splits + else: + intent_str, sentence = line_splits + slot_tags_str = "" + + if intent_str not in intents_list: + intents_list[intent_str] = len(intents_list) + outfiles['dict_intents'].write(f'{intent_str}\n') + + if ignore_prev_intent: + start_token = 2 + else: + start_token = 1 + + if do_lower_case: + sentence = sentence.lower() + sentence_cld = " ".join(sentence.strip().split()[start_token:-1]) + outfiles[mode].write(f'{sentence_cld}\t' f'{str(intents_list[intent_str])}\n') + + slot_tags_list = [] + if slot_tags_str.strip(): + slot_tags = slot_tags_str.strip().split(",") + for st in slot_tags: + if not st.strip(): + continue + [start_i, end_i, slot_name] = st.strip().split(":") + slot_tags_list.append([int(start_i), int(end_i), slot_name]) + if slot_name not in slots_list: + slots_list[slot_name] = len(slots_list) + slots_list_all[f'B-{slot_name}'] = len(slots_list_all) + slots_list_all[f'I-{slot_name}'] = len(slots_list_all) + outfiles['dict_slots'].write(f'B-{slot_name}\n') + outfiles['dict_slots'].write(f'I-{slot_name}\n') + + slot_tags_list.sort(key=lambda x: x[0]) + slots = [] + processed_index = 0 + for tag_start, tag_end, tag_str in slot_tags_list: + if tag_start > processed_index: + words_list = sentence[processed_index:tag_start].strip().split() + slots.extend([str(slots_list_all['O'])] * len(words_list)) + words_list = sentence[tag_start:tag_end].strip().split() + slots.append(str(slots_list_all[f'B-{tag_str}'])) + slots.extend([str(slots_list_all[f'I-{tag_str}'])] * (len(words_list) - 1)) + processed_index = tag_end + + if processed_index < len(sentence): + words_list = sentence[processed_index:].strip().split() + slots.extend([str(slots_list_all['O'])] * len(words_list)) + + slots = slots[1:-1] + slot = ' '.join(slots) + outfiles[mode + '_slots'].write(slot + '\n') + + outfiles[mode + '_slots'].close() + outfiles[mode].close() + + outfiles['dict_slots'].close() + outfiles['dict_intents'].close() + + return outfold + + +if __name__ == "__main__": + # Parse the command-line arguments. + parser = argparse.ArgumentParser(description="Process and convert datasets into NeMo\'s format.") + parser.add_argument( + "--dataset_name", required=True, type=str, choices=['atis', 'snips', 'jarvis', 'assistant'], + ) + parser.add_argument( + "--source_data_dir", required=True, type=str, help='path to the folder containing the dataset files' + ) + parser.add_argument("--target_data_dir", required=True, type=str, help='path to save the processed dataset') + parser.add_argument("--do_lower_case", action='store_true') + parser.add_argument( + "--ignore_prev_intent", + action='store_true', + help='ignores previous intent while importing datasets in jarvis\'s format', + ) + + args = parser.parse_args() + + dataset_name = args.dataset_name + source_dir = args.source_data_dir + target_dir = args.target_data_dir + + if not exists(source_dir): + raise FileNotFoundError(f"{source_dir} does not exist.") + + if dataset_name == 'atis': + process_atis(infold=source_dir, outfold=target_dir, do_lower_case=args.do_lower_case) + elif dataset_name == 'snips': + process_snips(infold=source_dir, outfold=target_dir, do_lower_case=args.do_lower_case) + elif dataset_name == 'jarvis': + process_jarvis_datasets( + infold=source_dir, + outfold=target_dir, + modes=["train", "test", "dev"], + do_lower_case=args.do_lower_case, + ignore_prev_intent=args.ignore_prev_intent, + ) + elif dataset_name == 'assistant': + process_assistant(infold=source_dir, outfold=target_dir) + else: + raise ValueError(f'Dataset {dataset_name} is not supported.') diff --git a/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/intent_slot_classification.py b/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/intent_slot_classification.py new file mode 100644 index 0000000000..39b4b7c5dd --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/intent_slot_classification/intent_slot_classification.py @@ -0,0 +1,85 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models import IntentSlotClassificationModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="intent_slot_classification_config") +def main(cfg: DictConfig) -> None: + logging.info(f'Config Params:\n {OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + + # initialize the model using the config file + model = IntentSlotClassificationModel(cfg.model, trainer=trainer) + + # training + logging.info("================================================================================================") + logging.info('Starting training...') + trainer.fit(model) + logging.info('Training finished!') + + # Stop further testing as fast_dev_run does not save checkpoints + if trainer.fast_dev_run: + return + + # after model training is done, you can load the model from the saved checkpoint + # and evaluate it on a data file or on given queries. + logging.info("================================================================================================") + logging.info("Starting the testing of the trained model on test set...") + logging.info("We will load the latest model saved checkpoint from the training...") + + # for evaluation and inference you can load the previously trained model saved in .nemo file + # like this in your code, but we will just reuse the trained model here + # eval_model = IntentSlotClassificationModel.restore_from(restore_path=checkpoint_path) + eval_model = model + + # we will setup testing data reusing the same config (test section) + eval_model.update_data_dir_for_testing(data_dir=cfg.model.data_dir) + eval_model.setup_test_data(test_data_config=cfg.model.test_ds) + + trainer.test(model=eval_model, ckpt_path=None, verbose=False) + logging.info("Testing finished!") + + # run an inference on a few examples + logging.info("======================================================================================") + logging.info("Evaluate the model on the given queries...") + + # this will work well if you train the model on Assistant dataset + # for your own dataset change the examples appropriately + queries = [ + 'set alarm for seven thirty am', + 'lower volume by fifty percent', + 'what is my schedule for tomorrow', + ] + + pred_intents, pred_slots = eval_model.predict_from_examples(queries, cfg.model.test_ds) + + logging.info('The prediction results of some sample queries with the trained model:') + for query, intent, slots in zip(queries, pred_intents, pred_slots): + logging.info(f'Query : {query}') + logging.info(f'Predicted Intent: {intent}') + logging.info(f'Predicted Slots: {slots}') + + logging.info("Inference finished!") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/language_modeling/bert_pretraining.py b/models/tacotron2/NeMo/examples/nlp/language_modeling/bert_pretraining.py new file mode 100644 index 0000000000..7760b73709 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/language_modeling/bert_pretraining.py @@ -0,0 +1,38 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning.plugins import DDPPlugin + +from nemo.collections.nlp.models.language_modeling import BERTLMModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="bert_pretraining_from_text_config") +def main(cfg: DictConfig) -> None: + logging.info(f'Config:\n {OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(plugins=[DDPPlugin(find_unused_parameters=True)], **cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + bert_model = BERTLMModel(cfg.model, trainer=trainer) + trainer.fit(bert_model) + if cfg.model.nemo_path: + bert_model.save_to(cfg.model.nemo_path) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_preprocessed_config.yaml b/models/tacotron2/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_preprocessed_config.yaml new file mode 100644 index 0000000000..cc516c1e85 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_preprocessed_config.yaml @@ -0,0 +1,80 @@ +# BERT Pretraining from Preprocessed (tokenized) data +name: &name PretrainingBERTFromPreprocessed +trainer: + gpus: 8 # the number of gpus, 0 for CPU, or list with gpu indices + num_nodes: 1 + max_steps: 2285714 # precedence over max_epochs + num_sanity_val_steps: 0 # needed for bert pretraining from preproc + replace_sampler_ddp: false # needed for bert pretraining from preproc + accumulate_grad_batches: 1 # accumulates grads every k batches + precision: 16 # 16 to use AMP + amp_level: O1 # O1 or O2 if using AMP + accelerator: ddp + gradient_clip_val: 1.0 + log_every_n_steps: 1 + val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch + checkpoint_callback: false # provided by exp_manager + logger: false # provided by exp_manager + +model: + nemo_path: null # exported .nemo path + only_mlm_loss: true # only use masked language model without next sentence prediction + num_tok_classification_layers: 1 # number of token classification head output layers + num_seq_classification_layers: 2 # number of sequence classification head output layers + + + language_model: + pretrained_model_name: bert-base-uncased # huggingface model name + lm_checkpoint: null + config: + attention_probs_dropout_prob: 0.1 + hidden_act: gelu + hidden_dropout_prob: 0.1 + hidden_size: 768 + initializer_range: 0.02 + intermediate_size: 3072 + max_position_embeddings: 512 + num_attention_heads: 12 + num_hidden_layers: 12 + type_vocab_size: 2 + vocab_size: 30522 + config_file: null # json file, precedence over config + + tokenizer: null + + train_ds: + data_file: null # path to hdf5 file (or directory) + max_predictions_per_seq: 80 + batch_size: 16 + shuffle: true + num_samples: -1 + num_workers: 2 + drop_last: false + pin_memory: false + + optim: + name: adamw + lr: 0.4375e-4 + weight_decay: 0.01 + + sched: + name: SquareRootAnnealing + warmup_steps: null + warmup_ratio: 0.01 + min_lr: 0.0 + last_epoch: -1 + + +exp_manager: + exp_dir: null # where to store logs and checkpoints + name: *name # name of experiment + create_tensorboard_logger: True + create_checkpoint_callback: True + + +hydra: + run: + dir: . + job_logging: + root: + handlers: null \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_text_config.yaml b/models/tacotron2/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_text_config.yaml new file mode 100644 index 0000000000..02df49edf5 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_text_config.yaml @@ -0,0 +1,109 @@ +# BERT Pretraining from Text +name: &name PretrainingBERTFromText +trainer: + gpus: 1 # the number of gpus, 0 for CPU, or list with gpu indices + num_nodes: 1 + max_epochs: 2 # the number of training epochs + max_steps: null # precedence over max_epochs + accumulate_grad_batches: 1 # accumulates grads every k batches + precision: 16 # 16 to use AMP + amp_level: O1 # O1 or O2 if using AMP + accelerator: ddp + gradient_clip_val: 0.0 + log_every_n_steps: 1 + val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch + checkpoint_callback: false # provided by exp_manager + logger: false # provided by exp_manager + +model: + nemo_path: null # exported .nemo path + only_mlm_loss: false # only use masked language model without next sentence prediction + num_tok_classification_layers: 1 # number of token classification head output layers + num_seq_classification_layers: 2 # number of sequence classification head output layers + max_seq_length: 128 + # The maximum total input sequence length after tokenization. Sequences longer than this + # will be truncated, and sequences shorter than this will be padded. + mask_prob: 0.15 + # Probability of masking a token in the input text during data processing. + short_seq_prob: 0.1 + # Probability of having a sequence shorter than the maximum sequence length `max_seq_length` in data processing.", + + language_model: + pretrained_model_name: bert-base-uncased + lm_checkpoint: null + config: + attention_probs_dropout_prob: 0.1 + hidden_act: gelu + hidden_dropout_prob: 0.1 + hidden_size: 768 + initializer_range: 0.02 + intermediate_size: 3072 + max_position_embeddings: 512 + num_attention_heads: 12 + num_hidden_layers: 12 + type_vocab_size: 2 + vocab_size: 30522 + config_file: null # json file, precedence over config + + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec + vocab_file: null # path to vocab file + tokenizer_model: null # tokenizer model for sentencepiece + special_tokens: # only necessary for adding transformer/bert-specific special tokens to tokenizer if the tokenizer does not already have these inherently. + unk_token: '[UNK]' + sep_token: '[SEP]' + pad_token: '[PAD]' + bos_token: '[CLS]' + mask_token: '[MASK]' + eos_token: '[SEP]' + cls_token: '[CLS]' + + train_ds: + data_file: null # path to data file + max_seq_length: ${model.max_seq_length} + mask_prob: ${model.mask_prob} + short_seq_prob: ${model.short_seq_prob} + batch_size: 16 # per GPU + shuffle: true + num_samples: -1 + num_workers: 2 + drop_last: false + pin_memory: false + + validation_ds: + data_file: null # path to data file + max_seq_length: ${model.max_seq_length} + mask_prob: ${model.mask_prob} + short_seq_prob: ${model.short_seq_prob} + batch_size: 16 # per GPU + shuffle: false + num_samples: -1 + num_workers: 2 + drop_last: false + pin_memory: false + + optim: + name: adamw + lr: 3e-5 + weight_decay: 0.0 + + sched: + name: CosineAnnealing + warmup_steps: null + warmup_ratio: 0.1 + min_lr: 0.0 + last_epoch: -1 + + +exp_manager: + exp_dir: null # where to store logs and checkpoints + name: *name # name of experiment + create_tensorboard_logger: True + create_checkpoint_callback: True + +hydra: + run: + dir: . + job_logging: + root: + handlers: null \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/language_modeling/conf/transformer_lm_config.yaml b/models/tacotron2/NeMo/examples/nlp/language_modeling/conf/transformer_lm_config.yaml new file mode 100644 index 0000000000..cb968bdc06 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/language_modeling/conf/transformer_lm_config.yaml @@ -0,0 +1,96 @@ +name: TransformerLanguageModel +do_training: True # set to False if only preprocessing data + +model: + label_smoothing: 0.0 + preproc_out_dir: null # path to store data preprocessing outputs + + train_ds: + use_tarred_dataset: true + tar_files: ??? # path to tarred files + metadata_file: ??? # metadata for tarred dataset + shard_strategy: scatter + tokens_in_batch: 4096 + tar_shuffle_n: 256 + + validation_ds: + file_name: ??? # path to file with validation data + tokens_in_batch: 512 + clean: false + shuffle: false + num_samples: -1 + drop_last: false + pin_memory: false + num_workers: 8 + + test_ds: + file_name: ??? # path to file with test data + tokens_in_batch: 512 + clean: false + shuffle: false + num_samples: -1 + drop_last: false + pin_memory: false + num_workers: 8 + + optim: + name: adam + lr: 0.001 + betas: + - 0.9 + - 0.98 + weight_decay: 0.0 + sched: + name: InverseSquareRootAnnealing + min_lr: 0.0 + last_epoch: -1 + warmup_ratio: 0.1 + + tokenizer: + tokenizer_name: yttm + tokenizer_model: ??? + vocab_file: null + special_tokens: null + training_sample_size: null # valid for sentencepiece tokenizer + + encoder: + library: nemo + model_name: null + pretrained: false + max_sequence_length: 512 + num_token_types: 0 + embedding_dropout: 0.1 + learn_positional_encodings: false + hidden_size: 512 + num_layers: 6 + inner_size: 2048 + num_attention_heads: 8 + ffn_dropout: 0.1 + attn_score_dropout: 0.1 + attn_layer_dropout: 0.1 + hidden_act: relu + mask_future: true + pre_ln: false + + head: + num_layers: 1 + activation: relu + log_softmax: true + dropout: 0.0 + use_transformer_init: true + +trainer: + gpus: 4 + num_nodes: 1 + max_epochs: 200 + amp_level: O2 # O1/O2 for mixed precision + precision: 16 # Should be set to 16 for O1 and O2, default is 16 as PT ignores it when am_level is O0 + accelerator: ddp + checkpoint_callback: False + logger: False + log_every_n_steps: 50 # Interval of logging. + check_val_every_n_epoch: 1 + +exp_manager: + name: TransformerLM + files_to_copy: [] \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/language_modeling/convert_weights_to_nemo1.0.py b/models/tacotron2/NeMo/examples/nlp/language_modeling/convert_weights_to_nemo1.0.py new file mode 100644 index 0000000000..b58b33942e --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/language_modeling/convert_weights_to_nemo1.0.py @@ -0,0 +1,61 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Converts BERT NeMo0.* checkpoints to NeMo1.0 format. +""" + +from argparse import ArgumentParser + +import torch + +parser = ArgumentParser() +parser.add_argument("--bert_encoder", required=True, help="path to BERT encoder, e.g. /../BERT-STEP-2285714.pt") +parser.add_argument( + "--bert_token_classifier", + required=True, + help="path to BERT token classifier, e.g. /../BertTokenClassifier-STEP-2285714.pt", +) +parser.add_argument( + "--bert_sequence_classifier", + required=False, + default=None, + help="path to BERT sequence classifier, e.g /../SequenceClassifier-STEP-2285714.pt", +) +parser.add_argument( + "--output_path", required=False, default="converted_model.pt", help="output path to newly converted model" +) +args = parser.parse_args() + +bert_in = torch.load(args.bert_encoder) +tok_in = torch.load(args.bert_token_classifier) +if args.bert_sequence_classifier: + seq_in = torch.load(args.bert_sequence_classifier) + +new_dict = {} +new_model = {"state_dict": new_dict} +for k in bert_in: + new_name = k.replace("bert.", "bert_model.") + new_dict[new_name] = bert_in[k] + +for k in tok_in: + new_name = "mlm_classifier." + k + new_dict[new_name] = tok_in[k] + +if args.bert_sequence_classifier: + for k in seq_in: + new_name = "nsp_classifier." + k + new_dict[new_name] = seq_in[k] + +torch.save(new_model, args.output_path) diff --git a/models/tacotron2/NeMo/examples/nlp/language_modeling/get_wkt2.sh b/models/tacotron2/NeMo/examples/nlp/language_modeling/get_wkt2.sh new file mode 100644 index 0000000000..33b55b2a73 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/language_modeling/get_wkt2.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +""" +This file is adapted from +https://github.com/salesforce/awd-lstm-lm/blob/master/getdata.sh +Copyright by the AWD LSTM authors. +""" +DATA_DIR=$1 +echo "- Downloading WikiText-2" + +wget --continue -P $DATA_DIR https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip +unzip -q $DATA_DIR/wikitext-2-v1.zip -d $DATA_DIR +cd $DATA_DIR/wikitext-2 +mv wiki.train.tokens train.txt +sed -i -e "s//[UNK]/g" train.txt +mv wiki.valid.tokens valid.txt +sed -i -e "s//[UNK]/g" valid.txt +mv wiki.test.tokens test.txt +sed -i -e "s//[UNK]/g" test.txt +cd .. +rm wikitext-2-v1.zip + +echo "- WikiText-2 saved at $DATA_DIR/wikitext-2" diff --git a/models/tacotron2/NeMo/examples/nlp/language_modeling/transformer_lm.py b/models/tacotron2/NeMo/examples/nlp/language_modeling/transformer_lm.py new file mode 100644 index 0000000000..caaa0e0d29 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/language_modeling/transformer_lm.py @@ -0,0 +1,35 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models.language_modeling import TransformerLMModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="transformer_lm_config") +def main(cfg: DictConfig) -> None: + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + transformer_lm = TransformerLMModel(cfg.model, trainer=trainer) + trainer.fit(transformer_lm) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/conf/aayn_base.yaml b/models/tacotron2/NeMo/examples/nlp/machine_translation/conf/aayn_base.yaml new file mode 100644 index 0000000000..a932f63152 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/conf/aayn_base.yaml @@ -0,0 +1,156 @@ +name: AttentionIsAllYouNeed +do_training: True # set to False if only preprocessing data +do_testing: False # set to True to run evaluation on test data after training + +model: + beam_size: 4 + len_pen: 0.6 + multilingual: False + max_generation_delta: 5 + label_smoothing: 0.1 + shared_tokenizer: True # train tokenizer model across src and tgt train data + preproc_out_dir: null # path to store data preprocessing outputs + src_language: 'en' + tgt_language: 'de' + + train_ds: + src_file_name: null + tgt_file_name: null + use_tarred_dataset: False # if true tar_file_name and meta_file_name will be used (or created automatically) + # config for preprocessing training data and creating a tarred datset automatically + tar_file_prefix: parallel # prefix for tar file names + tar_files: null # if data has already been preprocessed (rest of config ignored) + metadata_file: null # metadata for tarred dataset + lines_per_dataset_fragment: 1000000 # Number of lines to consider for bucketing and padding + num_batches_per_tarfile: 100 # Number of batches (pickle files) within each tarfile + tar_shuffle_n: 100 # How many samples to look ahead and load to be shuffled + shard_strategy: scatter # tarred dataset shard distribution strategy + n_preproc_jobs: -2 # number of processes to use for data preprocessing (-2 means all but 2) + tokens_in_batch: 512 + clean: true + max_seq_length: 512 + shuffle: true + num_samples: -1 + drop_last: false + pin_memory: false + num_workers: 8 + concat_sampling_technique: temperature # only used with ConcatTranslationDataset + concat_sampling_temperature: 5 # only used with ConcatTranslationDataset + concat_sampling_probabilities: null # only used with ConcatTranslationDataset + + validation_ds: + src_file_name: ??? + tgt_file_name: ??? + tokens_in_batch: 512 + clean: false + max_seq_length: 512 + shuffle: false + num_samples: -1 + drop_last: false + pin_memory: false + num_workers: 8 + + test_ds: + src_file_name: ??? + tgt_file_name: ??? + tokens_in_batch: 512 + clean: false + max_seq_length: 512 + shuffle: false + num_samples: -1 + drop_last: false + pin_memory: false + num_workers: 8 + + optim: + name: adam + lr: 0.001 + betas: + - 0.9 + - 0.98 + weight_decay: 0.0 + sched: + name: InverseSquareRootAnnealing + min_lr: 0.0 + last_epoch: -1 + warmup_ratio: 0.1 + + encoder_tokenizer: + library: yttm + tokenizer_model: null + vocab_size: null # vocab size for training bpe + bpe_dropout: null + vocab_file: null + special_tokens: null + training_sample_size: null # valid for sentencepiece tokenizer + + decoder_tokenizer: + library: yttm + tokenizer_model: null + vocab_size: null # vocab size for training bpe + bpe_dropout: null + vocab_file: null + special_tokens: null + training_sample_size: null # valid for sentencepiece tokenizer + + encoder: + library: nemo + model_name: null + pretrained: false + max_sequence_length: 512 + num_token_types: 0 + embedding_dropout: 0.1 + learn_positional_encodings: false + hidden_size: 512 + num_layers: 6 + inner_size: 2048 + num_attention_heads: 8 + ffn_dropout: 0.1 + attn_score_dropout: 0.1 + attn_layer_dropout: 0.1 + hidden_act: relu + mask_future: false + pre_ln: false + pre_ln_final_layer_norm: true + + decoder: + library: nemo + model_name: null + pretrained: false + max_sequence_length: 512 + num_token_types: 0 + embedding_dropout: 0.1 + learn_positional_encodings: false + hidden_size: 512 + inner_size: 2048 + num_layers: 6 + num_attention_heads: 8 + ffn_dropout: 0.1 + attn_score_dropout: 0.1 + attn_layer_dropout: 0.1 + hidden_act: relu + pre_ln: false + pre_ln_final_layer_norm: true + + head: + num_layers: 1 + activation: relu + log_softmax: true + dropout: 0.0 + use_transformer_init: true + +trainer: + gpus: 4 + num_nodes: 1 + max_epochs: 200 + amp_level: O2 # O1/O2 for mixed precision + precision: 16 # Should be set to 16 for O1 and O2, default is 16 as PT ignores it when am_level is O0 + accelerator: ddp + checkpoint_callback: False + logger: False + log_every_n_steps: 50 # Interval of logging. + check_val_every_n_epoch: 1 + +exp_manager: + name: AAYNBase + files_to_copy: [] \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/conf/huggingface.yaml b/models/tacotron2/NeMo/examples/nlp/machine_translation/conf/huggingface.yaml new file mode 100644 index 0000000000..abb5e9a844 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/conf/huggingface.yaml @@ -0,0 +1,132 @@ +name: HuggingFaceEncoder +do_training: True # set to False if only preprocessing data +do_testing: False # set to True to run evaluation on test data after training + +model: + beam_size: 4 + len_pen: 0.6 + max_generation_delta: 5 + label_smoothing: 0.1 + shared_tokenizer: false + preproc_out_dir: null + src_language: 'en' + tgt_language: 'de' + + train_ds: + src_file_name: null + tgt_file_name: null + use_tarred_dataset: False # if true tar_file_name and meta_file_name will be used (or created automatically) + # config for preprocessing training data and creating a tarred datset automatically + tar_file_prefix: parallel # prefix for tar file names + tar_files: null # if data has already been preprocessed (rest of config ignored) + metadata_file: null # metadata for tarred dataset + lines_per_dataset_fragment: 1000000 # Number of lines to consider for bucketing and padding + num_batches_per_tarfile: 100 # Number of batches (pickle files) within each tarfile + tar_shuffle_n: 100 # How many samples to look ahead and load to be shuffled + shard_strategy: scatter # tarred dataset shard distribution strategy + n_preproc_jobs: -2 # number of processes to use for data preprocessing (-2 means all but 2) + tokens_in_batch: 512 + clean: true + max_seq_length: 512 + shuffle: true + num_samples: -1 + drop_last: false + pin_memory: false + num_workers: 8 + + validation_ds: + src_file_name: null + tgt_file_name: null + tokens_in_batch: 512 + clean: false + max_seq_length: 512 + shuffle: false + num_samples: -1 + drop_last: false + pin_memory: false + num_workers: 8 + + test_ds: + src_file_name: null + tgt_file_name: null + tokens_in_batch: 512 + clean: false + max_seq_length: 512 + shuffle: false + num_samples: -1 + drop_last: false + pin_memory: false + num_workers: 8 + + optim: + name: adam + lr: 0.001 + betas: + - 0.9 + - 0.98 + weight_decay: 0.0 + sched: + name: InverseSquareRootAnnealing + min_lr: 0.0 + last_epoch: -1 + warmup_ratio: 0.1 + + encoder_tokenizer: + library: huggingface + tokenizer_model: null + vocab_file: null + special_tokens: null + vocab_size: null + + decoder_tokenizer: + library: yttm + tokenizer_model: null + vocab_file: null + special_tokens: null + vocab_size: null + + encoder: + library: huggingface + model_name: bert-base-uncased + pretrained: false + + decoder: + library: nemo + model_name: null + pretrained: false + max_sequence_length: 512 + num_token_types: 2 + embedding_dropout: 0.1 + learn_positional_encodings: false + hidden_size: 512 + inner_size: 2048 + num_layers: 6 + num_attention_heads: 8 + ffn_dropout: 0.1 + attn_score_dropout: 0.1 + attn_layer_dropout: 0.1 + hidden_act: relu + pre_ln: false + + head: + num_layers: 1 + activation: relu + log_softmax: true + dropout: 0.0 + use_transformer_init: true + +trainer: + gpus: 4 + num_nodes: 1 + max_epochs: 200 + amp_level: O2 # O1/O2 for mixed precision + precision: 16 # Should be set to 16 for O1 and O2, default is 16 as PT ignores it when am_level is O0 + accelerator: ddp + checkpoint_callback: False + logger: False + log_every_n_steps: 50 # Interval of logging. + check_val_every_n_epoch: 1 + +exp_manager: + name: HuggingFaceEncoder + files_to_copy: [] \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/create_tarred_monolingual_dataset.py b/models/tacotron2/NeMo/examples/nlp/machine_translation/create_tarred_monolingual_dataset.py new file mode 100644 index 0000000000..5ad5f789d7 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/create_tarred_monolingual_dataset.py @@ -0,0 +1,68 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os + +from nemo.collections.nlp.data.machine_translation.preproc_mt_data import MTDataPreproc + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='NMT dataset pre-processing') + parser.add_argument('--tokenizer_model', type=str, required=True, help='Path to tokenizer model') + parser.add_argument('--tokenizer_name', type=str, default='yttm', help='BPE Tokenizer Name, Options: [yttm]') + parser.add_argument('--bpe_droput', type=float, default=0.0, help='BPE dropout to use') + parser.add_argument('--clean', action="store_true", help='Whether to clean dataset based on length diff') + parser.add_argument('--pkl_file_prefix', type=str, default='parallel', help='Prefix for tar and pickle files') + parser.add_argument('--fname', type=str, required=True, help='Path to monolingual data file') + parser.add_argument('--out_dir', type=str, required=True, help='Path to store dataloader and tokenizer models') + parser.add_argument('--max_seq_length', type=int, default=512, help='Max Sequence Length') + parser.add_argument('--min_seq_length', type=int, default=1, help='Min Sequence Length') + parser.add_argument('--tokens_in_batch', type=int, default=16000, help='# Tokens per batch per GPU') + parser.add_argument( + '--lines_per_dataset_fragment', + type=int, + default=1000000, + help='Number of lines to consider for bucketing and padding', + ) + parser.add_argument( + '--num_batches_per_tarfile', + type=int, + default=1000, + help='Number of batches (pickle files) within each tarfile', + ) + + args = parser.parse_args() + if not os.path.exists(args.out_dir): + os.mkdir(args.out_dir) + if not os.path.exists(args.tokenizer_model): + assert FileNotFoundError("Could not find tokenizer model %s" % (args.tokenizer)) + + tokenizer_model = MTDataPreproc.get_monolingual_tokenizer( + tokenizer_name=args.tokenizer_name, tokenizer_model=args.tokenizer_model, bpe_dropout=args.bpe_droput + ) + + MTDataPreproc.preprocess_monolingual_dataset( + clean=args.clean, + fname=args.fname, + out_dir=args.out_dir, + tokenizer=tokenizer_model, + max_seq_length=args.max_seq_length, + min_seq_length=args.min_seq_length, + tokens_in_batch=args.tokens_in_batch, + lines_per_dataset_fragment=args.lines_per_dataset_fragment, + num_batches_per_tarfile=args.num_batches_per_tarfile, + pkl_file_prefix=args.pkl_file_prefix, + global_rank=0, + world_size=1, + ) diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/create_tarred_parallel_dataset.py b/models/tacotron2/NeMo/examples/nlp/machine_translation/create_tarred_parallel_dataset.py new file mode 100644 index 0000000000..d2cd59b37a --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/create_tarred_parallel_dataset.py @@ -0,0 +1,143 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os + +from nemo.collections.nlp.data.machine_translation.preproc_mt_data import MTDataPreproc + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='NMT dataset pre-processing') + parser.add_argument('--shared_tokenizer', action="store_true", help='Whether to share encoder/decoder tokenizers') + parser.add_argument('--clean', action="store_true", help='Whether to clean dataset based on length diff') + parser.add_argument('--tar_file_prefix', type=str, default='parallel', help='Prefix for tar files') + parser.add_argument('--bpe_dropout', type=float, default=0.1, help='Whether to share encoder/decoder tokenizers') + parser.add_argument('--src_fname', type=str, required=True, help='Path to the source file') + parser.add_argument('--tgt_fname', type=str, required=True, help='Path to the target file') + parser.add_argument('--out_dir', type=str, required=True, help='Path to store dataloader and tokenizer models') + parser.add_argument('--encoder_model_name', type=str, default=None, help='For use with pretrained encoders') + parser.add_argument( + '--decoder_model_name', type=str, default=None, help='For use with pretrained decoders (not yet supported)' + ) + parser.add_argument( + '--encoder_tokenizer_model', type=str, default='None', help='Path to pre-trained encoder tokenizer model' + ) + parser.add_argument( + '--encoder_tokenizer_name', type=str, default='yttm', help='Encoder BPE Tokenizer Name, Options: [yttm]' + ) + parser.add_argument('--encoder_tokenizer_vocab_size', type=int, default=32000, help='Encoder Vocab size after BPE') + parser.add_argument( + '--encoder_tokenizer_coverage', type=float, default=0.999, help='Encoder Character coverage for BPE' + ) + parser.add_argument('--encoder_tokenizer_bpe_dropout', type=float, default=0.1, help='Encoder BPE dropout prob') + parser.add_argument( + '--decoder_tokenizer_model', type=str, default='None', help='Path to pre-trained decoder tokenizer model' + ) + parser.add_argument( + '--decoder_tokenizer_name', type=str, default='yttm', help='Encoder BPE Tokenizer Name, Options: [yttm]' + ) + parser.add_argument('--decoder_tokenizer_vocab_size', type=int, default=32000, help='Encoder Vocab size after BPE') + parser.add_argument( + '--decoder_tokenizer_coverage', type=float, default=0.999, help='Encoder Character coverage for BPE' + ) + parser.add_argument('--decoder_tokenizer_bpe_dropout', type=float, default=0.1, help='Encoder BPE dropout prob') + parser.add_argument('--max_seq_length', type=int, default=512, help='Max Sequence Length') + parser.add_argument('--min_seq_length', type=int, default=1, help='Min Sequence Length') + parser.add_argument('--tokens_in_batch', type=int, default=16000, help='# Tokens per batch per GPU') + parser.add_argument('--coverage', type=float, default=0.999, help='BPE character coverage [0-1]') + parser.add_argument( + '--lines_per_dataset_fragment', + type=int, + default=1000000, + help='Number of lines to consider for bucketing and padding', + ) + parser.add_argument( + '--num_batches_per_tarfile', + type=int, + default=1000, + help='Number of batches (pickle files) within each tarfile', + ) + parser.add_argument( + '--n_preproc_jobs', type=int, default=-2, help='Number of processes to use for creating the tarred dataset.', + ) + + args = parser.parse_args() + if not os.path.exists(args.out_dir): + os.mkdir(args.out_dir) + + if ( + args.encoder_tokenizer_model != 'None' + and args.decoder_tokenizer_model == 'None' + or args.decoder_tokenizer_model != 'None' + and args.encoder_tokenizer_model == 'None' + ): + if args.shared_tokenizer: + raise ValueError( + ''' + If using a pre-trained shared tokenizer, + both encoder and decoder tokenizers must be the same + ''' + ) + else: + raise ValueError('Both encoder and decoder pre-trained tokenizer models must be specified') + + if args.encoder_tokenizer_model == 'None' and args.decoder_tokenizer_model == 'None': + encoder_tokenizer_model, decoder_tokenizer_model = MTDataPreproc.train_tokenizers( + out_dir=args.out_dir, + src_fname=args.src_fname, + tgt_fname=args.tgt_fname, + shared_tokenizer=args.shared_tokenizer, + encoder_tokenizer_name=args.encoder_tokenizer_name, + encoder_tokenizer_vocab_size=args.encoder_tokenizer_vocab_size, + encoder_tokenizer_coverage=args.encoder_tokenizer_coverage, + decoder_tokenizer_name=args.decoder_tokenizer_name, + decoder_tokenizer_vocab_size=args.decoder_tokenizer_vocab_size, + decoder_tokenizer_coverage=args.decoder_tokenizer_coverage, + global_rank=0, + ) + else: + encoder_tokenizer_model, decoder_tokenizer_model = args.encoder_tokenizer_model, args.decoder_tokenizer_model + + encoder_tokenizer, decoder_tokenizer = MTDataPreproc.get_enc_dec_tokenizers( + encoder_tokenizer_name=args.encoder_tokenizer_name, + encoder_tokenizer_model=encoder_tokenizer_model, + encoder_bpe_dropout=args.encoder_tokenizer_bpe_dropout, + decoder_tokenizer_name=args.decoder_tokenizer_name, + decoder_tokenizer_model=decoder_tokenizer_model, + decoder_bpe_dropout=args.decoder_tokenizer_bpe_dropout, + ) + + _, _ = MTDataPreproc.preprocess_parallel_dataset( + clean=args.clean, + src_fname=args.src_fname, + tgt_fname=args.tgt_fname, + out_dir=args.out_dir, + encoder_tokenizer_name=args.encoder_tokenizer_name, + encoder_model_name=args.encoder_model_name, + encoder_tokenizer_model=encoder_tokenizer_model, + encoder_bpe_dropout=args.encoder_tokenizer_bpe_dropout, + decoder_tokenizer_name=args.decoder_tokenizer_name, + decoder_model_name=args.decoder_model_name, + decoder_tokenizer_model=decoder_tokenizer_model, + decoder_bpe_dropout=args.decoder_tokenizer_bpe_dropout, + max_seq_length=args.max_seq_length, + min_seq_length=args.min_seq_length, + tokens_in_batch=args.tokens_in_batch, + lines_per_dataset_fragment=args.lines_per_dataset_fragment, + num_batches_per_tarfile=args.num_batches_per_tarfile, + tar_file_prefix=args.tar_file_prefix, + global_rank=0, + world_size=1, + n_jobs=args.n_preproc_jobs, + ) diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/enc_dec_nmt.py b/models/tacotron2/NeMo/examples/nlp/machine_translation/enc_dec_nmt.py new file mode 100644 index 0000000000..1cfb8bcfe5 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/enc_dec_nmt.py @@ -0,0 +1,137 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional + +from omegaconf import OmegaConf +from pytorch_lightning import Trainer + +from nemo.collections.nlp.data.machine_translation.preproc_mt_data import MTDataPreproc +from nemo.collections.nlp.models.machine_translation.mt_enc_dec_config import MTEncDecModelConfig +from nemo.collections.nlp.models.machine_translation.mt_enc_dec_model import MTEncDecModel +from nemo.core.config import hydra_runner +from nemo.core.config.modelPT import NemoConfig +from nemo.core.config.pytorch_lightning import TrainerConfig +from nemo.utils import logging +from nemo.utils.config_utils import update_model_config +from nemo.utils.exp_manager import ExpManagerConfig, exp_manager + + +""" +Usage: + 1. If you need to start docker and install NeMo, otherwise skip this step: + + a. ```docker run --gpus all -it --rm -v /home/okuchaiev/repos/NeMo/:/NeMo -p 6006:6006 -v /mnt:/mnt --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 --device=/dev/snd nvcr.io/nvidia/pytorch:20.11-py3``` + b. ```cd /NeMo``` + c. ```./reinstall.sh``` + + 2. Train a new tokenizer (or use pre-trained one): + ```yttm bpe --data /mnt/D1/Data/NMT/wmt16_de_en/train.clean.en-de.shuffled.common --model tokenizer.BPE.8192.model --vocab_size 8192``` + +(To use WANDB, optionally, do login first) +``wandb login [YOUR WANDB login]`` + + 3. Start training: + + + (This example for "base" model on 2 GPUs for 150000 steps with batch size of 12500 tokens per GPU) + + python enc_dec_nmt.py \ + --config-path=conf \ + --config-name=aayn_base \ + trainer.gpus=[0,1] \ + ~trainer.max_epochs \ + +trainer.max_steps=150000 \ + model.beam_size=4 \ + model.max_generation_delta=5 \ + model.label_smoothing=0.1 \ + model.encoder_tokenizer.tokenizer_model=tokenizer.BPE.8192.model \ + model.decoder_tokenizer.tokenizer_model=tokenizer.BPE.8192.model \ + model.encoder.num_layers=6 \ + model.encoder.hidden_size=512 \ + model.encoder.inner_size=2048 \ + model.encoder.num_attention_heads=8 \ + model.encoder.ffn_dropout=0.1 \ + model.decoder.num_layers=6 \ + model.decoder.hidden_size=512 \ + model.decoder.inner_size=2048 \ + model.decoder.num_attention_heads=8 \ + model.decoder.ffn_dropout=0.1 \ + model.train_ds.src_file_name=/mnt/D1/Data/NMT/wmt16_de_en/train.clean.de.shuffled \ + model.train_ds.tgt_file_name=/mnt/D1/Data/NMT/wmt16_de_en/train.clean.en.shuffled \ + model.train_ds.tokens_in_batch=12500 \ + model.validation_ds.src_file_name=/mnt/D1/Data/NMT/wmt16_de_en/wmt14-en-de.ref \ + model.validation_ds.tgt_file_name=/mnt/D1/Data/NMT/wmt16_de_en/wmt14-en-de.src \ + model.validation_ds.tokens_in_batch=8192 \ + model.test_ds.src_file_name=/mnt/D1/Data/NMT/wmt16_de_en/wmt14-en-de.ref \ + model.test_ds.tgt_file_name=/mnt/D1/Data/NMT/wmt16_de_en/wmt14-en-de.src \ + model.optim.lr=0.001 \ + model.optim.sched.warmup_ratio=0.05 \ + +exp_manager.create_wandb_logger=True \ + +exp_manager.wandb_logger_kwargs.name=TEST-nmt-base \ + +exp_manager.wandb_logger_kwargs.project=nmt-de-en \ + +exp_manager.create_checkpoint_callback=True \ + +exp_manager.checkpoint_callback_params.monitor=val_sacreBLEU \ + +exp_manager.exp_dir=nmt_base \ + +exp_manager.checkpoint_callback_params.mode=max +""" + + +@dataclass +class MTEncDecConfig(NemoConfig): + name: Optional[str] = 'MTEncDec' + do_training: bool = True + do_testing: bool = False + model: MTEncDecModelConfig = MTEncDecModelConfig() + trainer: Optional[TrainerConfig] = TrainerConfig() + exp_manager: Optional[ExpManagerConfig] = ExpManagerConfig(name='MTEncDec', files_to_copy=[]) + + +@hydra_runner(config_path="conf", config_name="aayn_base") +def main(cfg: MTEncDecConfig) -> None: + # merge default config with user specified config + default_cfg = MTEncDecConfig() + cfg = update_model_config(default_cfg, cfg) + logging.info("\n\n************** Experiment configuration ***********") + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + + # training is managed by PyTorch Lightning + trainer = Trainer(**cfg.trainer) + + # tokenizers will be trained and and tarred training data will be created if needed + # model config is then updated + if cfg.model.preproc_out_dir is not None: + MTDataPreproc(cfg=cfg.model, trainer=trainer) + + # experiment logs, checkpoints, and auto-resume are managed by exp_manager and PyTorch Lightning + exp_manager(trainer, cfg.exp_manager) + + # everything needed to train translation models is encapsulated in the NeMo MTEncdDecModel + mt_model = MTEncDecModel(cfg.model, trainer=trainer) + + logging.info("\n\n************** Model parameters and their sizes ***********") + for name, param in mt_model.named_parameters(): + print(name, param.size()) + logging.info("***********************************************************\n\n") + + if cfg.do_training: + trainer.fit(mt_model) + + if cfg.do_testing: + trainer.test(mt_model) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_transformer_infer.py b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_transformer_infer.py new file mode 100644 index 0000000000..91f2fe9e94 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_transformer_infer.py @@ -0,0 +1,122 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Given NMT model's .nemo file, this script can be used to translate text. +USAGE Example: +1. Obtain text file in src language. You can use sacrebleu to obtain standard test sets like so: + sacrebleu -t wmt14 -l de-en --echo src > wmt14-de-en.src +2. Translate: + python nmt_transformer_infer.py --model=[Path to .nemo file] --srctext=wmt14-de-en.src --tgtout=wmt14-de-en.pre +""" + + +from argparse import ArgumentParser + +import torch + +import nemo.collections.nlp as nemo_nlp +from nemo.collections.nlp.modules.common.transformer import ( + BeamSearchSequenceGenerator, + BeamSearchSequenceGeneratorWithLanguageModel, +) +from nemo.utils import logging + + +def main(): + parser = ArgumentParser() + parser.add_argument("--model", type=str, required=True, help="") + parser.add_argument("--srctext", type=str, required=True, help="") + parser.add_argument("--tgtout", type=str, required=True, help="") + parser.add_argument("--batch_size", type=int, default=256, help="") + parser.add_argument("--beam_size", type=int, default=4, help="") + parser.add_argument("--len_pen", type=float, default=0.6, help="") + parser.add_argument("--max_delta_length", type=int, default=5, help="") + parser.add_argument("--target_lang", type=str, default=None, help="") + parser.add_argument("--source_lang", type=str, default=None, help="") + # shallow fusion specific parameters + parser.add_argument("--lm_model", type=str, default=None, help="") + parser.add_argument("--fusion_coef", type=float, default=0.0, help="") + + args = parser.parse_args() + torch.set_grad_enabled(False) + if args.model.endswith(".nemo"): + logging.info("Attempting to initialize from .nemo file") + model = nemo_nlp.models.machine_translation.MTEncDecModel.restore_from(restore_path=args.model) + src_text = [] + tgt_text = [] + else: + raise NotImplemented(f"Only support .nemo files, but got: {args.model}") + + if torch.cuda.is_available(): + model = model.cuda() + + if args.lm_model is not None: + lm_model = nemo_nlp.models.language_modeling.TransformerLMModel.restore_from(restore_path=args.lm_model).eval() + model.beam_search = BeamSearchSequenceGeneratorWithLanguageModel( + embedding=model.decoder.embedding, + decoder=model.decoder.decoder, + log_softmax=model.log_softmax, + bos=model.decoder_tokenizer.bos_id, + pad=model.decoder_tokenizer.pad_id, + eos=model.decoder_tokenizer.eos_id, + language_model=lm_model, + fusion_coef=args.fusion_coef, + max_sequence_length=model.decoder.max_sequence_length, + beam_size=args.beam_size, + len_pen=args.len_pen, + max_delta_length=args.max_delta_length, + ) + else: + model.beam_search = BeamSearchSequenceGenerator( + embedding=model.decoder.embedding, + decoder=model.decoder.decoder, + log_softmax=model.log_softmax, + bos=model.decoder_tokenizer.bos_id, + pad=model.decoder_tokenizer.pad_id, + eos=model.decoder_tokenizer.eos_id, + max_sequence_length=model.decoder.max_sequence_length, + beam_size=args.beam_size, + len_pen=args.len_pen, + max_delta_length=args.max_delta_length, + ) + + logging.info(f"Translating: {args.srctext}") + + count = 0 + with open(args.srctext, 'r') as src_f: + for line in src_f: + src_text.append(line.strip()) + if len(src_text) == args.batch_size: + res = model.translate(text=src_text, source_lang=args.source_lang, target_lang=args.target_lang) + if len(res) != len(src_text): + print(len(res)) + print(len(src_text)) + print(res) + print(src_text) + tgt_text += res + src_text = [] + count += 1 + # if count % 300 == 0: + # print(f"Translated {count} sentences") + if len(src_text) > 0: + tgt_text += model.translate(text=src_text, source_lang=args.source_lang, target_lang=args.target_lang) + + with open(args.tgtout, 'w') as tgt_f: + for line in tgt_text: + tgt_f.write(line + "\n") + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/README.rst b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/README.rst new file mode 100644 index 0000000000..dd453d165b --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/README.rst @@ -0,0 +1,7 @@ +**STEPS 2 RUN** +=============== +0. (Make sure you have Flask installed - ``pip install -r requirements.txt``) +1. Edit "config.json" file to only contain models you need. If model's location starts with "NGC/" - it will load this model from NVIDIA's NGC. Otherwise, specify full path to .nemo file. +4. To run: ``python nmt_service.py`` +5. To translate: ``http://127.0.0.1:5000/translate?text=Frohe%20Weihnachten`` (here %20 means space) +6. Run web UI: ``python -m http.server`` \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/config.json b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/config.json new file mode 100644 index 0000000000..938d91a0f3 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/config.json @@ -0,0 +1,12 @@ +{ + "en-de": "NGC/nmt_en_de_transformer12x2", + "de-en": "NGC/nmt_de_en_transformer12x2", + "en-es": "NGC/nmt_en_es_transformer12x2", + "es-en": "NGC/nmt_es_en_transformer12x2", + "en-ru": "NGC/nmt_en_ru_transformer6x6", + "ru-en": "NGC/nmt_ru_en_transformer6x6", + "en-fr": "NGC/nmt_en_fr_transformer12x2", + "fr-en": "NGC/nmt_fr_en_transformer12x2", + "en-zh": "NGC/nmt_en_zh_transformer6x6", + "zh-en": "NGC/nmt_zh_en_transformer6x6" +} diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/index.html b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/index.html new file mode 100644 index 0000000000..2f639d5860 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/index.html @@ -0,0 +1,140 @@ + + + + + + Neural Machine Translation Demo App + + + + + + + + + +
+

Neural Machine Translation Demo

+

+
+

Running NeMo commit id: v1.0.2

+
+

+

+

+ Select translation direction: + + + + + + + + + + + + + + + + + + + + + + + +
+ + +             + + +             + + +             + + +
+ + +             + + +             + + +             + + +             + + +             + + +
+
+
+ + + + + + + + + + +
+ +
+ +
+ +
+
+

+
+ + + + diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/nmt_service.py b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/nmt_service.py new file mode 100644 index 0000000000..3661fcdb4a --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/nmt_service.py @@ -0,0 +1,100 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import time + +import flask +import torch +from flask import Flask, json, request +from flask_cors import CORS + +import nemo.collections.nlp as nemo_nlp +from nemo.utils import logging + +MODELS_DICT = {} + +model = None +api = Flask(__name__) +CORS(api) + + +def initialize(config_file_path: str): + """ + Loads 'language-pair to NMT model mapping' + """ + __MODELS_DICT = None + + logging.info("Starting NMT service") + logging.info(f"I will attempt to load all the models listed in {config_file_path}.") + logging.info(f"Edit {config_file_path} to disable models you don't need.") + if torch.cuda.is_available(): + logging.info("CUDA is available. Running on GPU") + else: + logging.info("CUDA is not available. Defaulting to CPUs") + + # read config + with open(config_file_path) as f: + __MODELS_DICT = json.load(f) + + if __MODELS_DICT is not None: + for key, value in __MODELS_DICT.items(): + logging.info(f"Loading model for {key} from file: {value}") + if value.startswith("NGC/"): + model = nemo_nlp.models.machine_translation.MTEncDecModel.from_pretrained(model_name=value[4:]) + else: + model = nemo_nlp.models.machine_translation.MTEncDecModel.restore_from(restore_path=value) + if torch.cuda.is_available(): + model = model.cuda() + MODELS_DICT[key] = model + else: + raise ValueError("Did not find the config.json or it was empty") + logging.info("NMT service started") + + +@api.route('/translate', methods=['GET', 'POST', 'OPTIONS']) +def get_translation(): + try: + time_s = time.time() + langpair = request.args["langpair"] + src = request.args["text"] + do_moses = request.args.get('do_moses', False) + if langpair in MODELS_DICT: + if do_moses: + result = MODELS_DICT[langpair].translate( + [src], source_lang=langpair.split('-')[0], target_lang=langpair.split('-')[1] + ) + else: + result = MODELS_DICT[langpair].translate([src]) + + duration = time.time() - time_s + logging.info( + f"Translated in {duration}. Input was: {request.args['text']} <############> Translation was: {result[0]}" + ) + res = {'translation': result[0]} + response = flask.jsonify(res) + response.headers.add('Access-Control-Allow-Origin', '*') + return response + + else: + logging.error(f"Got the following langpair: {langpair} which was not found") + except Exception as ex: + res = {'translation': str(ex)} + response = flask.jsonify(res) + response.headers.add('Access-Control-Allow-Origin', '*') + return res + + +if __name__ == '__main__': + initialize('config.json') + api.run(host='0.0.0.0') diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/requirements.txt b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/requirements.txt new file mode 100644 index 0000000000..cd6e0e7486 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/requirements.txt @@ -0,0 +1,3 @@ +nemo_toolkit[all]>=1.0.0rc1 +flask +flask_cors \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/style.css b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/style.css new file mode 100644 index 0000000000..58f23342f5 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/nmt_webapp/style.css @@ -0,0 +1,111 @@ +@import url("https://fonts.googleapis.com/css?family=Spartan&display=swap"); +* { + margin: 0; + padding: 0; +} + +body { + /*background: url("https://img.wallpapersafari.com/desktop/1680/1050/65/18/8AYkSt.jpg") center center;*/ + background: url("https://images5.alphacoders.com/550/550948.jpg") center center; + background-size: cover; + width: 100vw; + height: 100vh; + display: grid; + align-items: center; + justify-items: center; + font-size: 12pt; + font-family: "Spartan"; + color: #ffffff; +} + +.contact-us { + /*background: #1f1d1d;*/ + padding: 50px 100px; + border-top: 10px solid #229955; +} + +label, input, textarea { + display: block; + width: 100%; + font-size: 12pt; + line-height: 24pt; + font-family: "Spartan"; +} + +input { + margin-bottom: 24pt; +} + +h1 { + color: #ffffff; + font-weight: bold; + font-size: 22pt; + line-height: 24pt; + font-style: italic; + margin: 0 0 0.5em 0; +} + +span { + font-size: 8pt; +} + +em { + color: #229955; + font-weight: bold; +} + +input, textarea { + border: none; + border: 1px solid rgba(0, 0, 0, 0.1); + border-radius: 2px; + background: #f8f4e5; + padding-left: 5px; + outline: none; +} + +input:focus, textarea:focus { + border: 1px solid #6bd4b1; +} + +textarea { + resize: none; +} + +fieldset { + overflow: hidden + } + +button { + display: block; + float: left; + line-height: 24pt; + padding: 0 20px; + border: none; + background: #f45702; + color: white; + letter-spacing: 2px; + transition: 0.2s all ease-in-out; + border-bottom: 2px solid transparent; + outline: none; +} +button:hover { + background: inherit; + color: #ea6153; + border-bottom: 2px solid #f45702; +} + +::selection { + background: #ffc7b8; +} + +input:-webkit-autofill, +input:-webkit-autofill:hover, +input:-webkit-autofill:focus, +textarea:-webkit-autofill, +textarea:-webkit-autofill:hover, +textarea:-webkit-autofill:focus { + border: 1px solid #6bd4b1; + -webkit-text-fill-color: #2A293E; + -webkit-box-shadow: 0 0 0px 1000px #f8f4e5 inset; + transition: background-color 5000s ease-in-out 0s; +} \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/machine_translation/translate_ddp.py b/models/tacotron2/NeMo/examples/nlp/machine_translation/translate_ddp.py new file mode 100644 index 0000000000..bf057df9a1 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/machine_translation/translate_ddp.py @@ -0,0 +1,122 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from argparse import ArgumentParser + +import torch +import torch.multiprocessing as mp +from torch.utils.data import DataLoader + +from nemo.collections.nlp.data.language_modeling import TarredSentenceDataset +from nemo.collections.nlp.data.machine_translation import TarredTranslationDataset +from nemo.collections.nlp.models.machine_translation.mt_enc_dec_model import MTEncDecModel +from nemo.utils import logging + + +def get_args(): + parser = ArgumentParser(description='Batch translation of sentences from a pre-trained model on multiple GPUs') + parser.add_argument("--model", type=str, required=True, help="Path to the .nemo translation model file") + parser.add_argument( + "--text2translate", type=str, required=True, help="Path to the pre-processed tarfiles for translation" + ) + parser.add_argument("--result_dir", type=str, required=True, help="Folder to write translation results") + parser.add_argument( + "--twoside", action="store_true", help="Set flag when translating the source side of a parallel dataset" + ) + parser.add_argument( + '--metadata_path', type=str, required=True, help="Path to the JSON file that contains dataset info" + ) + parser.add_argument('--topk', type=int, default=500, help="Value of k for topk sampling") + parser.add_argument('--src_language', type=str, required=True, help="Source lang ID for detokenization") + parser.add_argument('--tgt_language', type=str, required=True, help="Target lang ID for detokenization") + parser.add_argument( + '--reverse_lang_direction', + action="store_true", + help="Reverse source and target language direction for parallel dataset", + ) + parser.add_argument('--n_gpus', type=int, default=-1, help="Number of GPUs to use") + args = parser.parse_args() + return args + + +def translate(rank, world_size, args): + if args.model.endswith(".nemo"): + logging.info("Attempting to initialize from .nemo file") + model = MTEncDecModel.restore_from(restore_path=args.model, map_location=f"cuda:{rank}") + elif args.model.endswith(".ckpt"): + logging.info("Attempting to initialize from .ckpt file") + model = MTEncDecModel.load_from_checkpoint(checkpoint_path=args.model, map_location=f"cuda:{rank}") + model.replace_beam_with_sampling(topk=args.topk) + model.eval() + if args.twoside: + dataset = TarredTranslationDataset( + text_tar_filepaths=args.text2translate, + metadata_path=args.metadata_path, + encoder_tokenizer=model.encoder_tokenizer, + decoder_tokenizer=model.decoder_tokenizer, + shuffle_n=100, + shard_strategy="scatter", + world_size=world_size, + global_rank=rank, + reverse_lang_direction=args.reverse_lang_direction, + ) + else: + dataset = TarredSentenceDataset( + text_tar_filepaths=args.text2translate, + metadata_path=args.metadata_path, + tokenizer=model.encoder_tokenizer, + shuffle_n=100, + shard_strategy="scatter", + world_size=world_size, + global_rank=rank, + ) + loader = DataLoader(dataset, batch_size=1) + result_dir = os.path.join(args.result_dir, f'rank{rank}') + os.makedirs(result_dir, exist_ok=True) + originals_file_name = os.path.join(result_dir, 'originals.txt') + translations_file_name = os.path.join(result_dir, 'translations.txt') + num_translated_sentences = 0 + + with open(originals_file_name, 'w') as of, open(translations_file_name, 'w') as tf: + for batch_idx, batch in enumerate(loader): + for i in range(len(batch)): + if batch[i].ndim == 3: + batch[i] = batch[i].squeeze(dim=0) + batch[i] = batch[i].to(rank) + if args.twoside: + src_ids, src_mask, _, _, _ = batch + else: + src_ids, src_mask = batch + if batch_idx % 100 == 0: + logging.info( + f"{batch_idx} batches ({num_translated_sentences} sentences) were translated by process with " + f"rank {rank}" + ) + num_translated_sentences += len(src_ids) + inputs, translations = model.batch_translate(src=src_ids, src_mask=src_mask) + for src, translation in zip(inputs, translations): + of.write(src + '\n') + tf.write(translation + '\n') + + +def main() -> None: + args = get_args() + world_size = torch.cuda.device_count() if args.n_gpus == -1 else args.n_gpus + mp.spawn(translate, args=(world_size, args), nprocs=world_size, join=True) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/question_answering/conf/question_answering_squad_config.yaml b/models/tacotron2/NeMo/examples/nlp/question_answering/conf/question_answering_squad_config.yaml new file mode 100644 index 0000000000..929409d53d --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/question_answering/conf/question_answering_squad_config.yaml @@ -0,0 +1,135 @@ +# Question Answering with SQUAD +name: &name QA + +pretrained_model: null # pretrained QAModel model from list_available_models() +do_training: true # training mode, for testing change to false +trainer: + gpus: 1 # the number of gpus, 0 for CPU, or list with gpu indices + num_nodes: 1 + max_epochs: 2 # the number of training epochs + max_steps: null # precedence over max_epochs + accumulate_grad_batches: 1 # accumulates grads every k batches + precision: 16 # 16 to use AMP + amp_level: O1 # O1 or O2 if using AMP + accelerator: ddp + gradient_clip_val: 0.0 + val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch + checkpoint_callback: false # provided by exp_manager + logger: false # provided by exp_manager + num_sanity_val_steps: 0 + log_every_n_steps: 1 # Interval of logging. + +model: + nemo_path: null # exported .nemo path + dataset: + version_2_with_negative: false + # If true, the examples contain some that do not have an answer. + doc_stride: 128 + # When splitting up a long document into chunks, + # how much stride to take between chunks. + max_query_length: 64 + # The maximum number of tokens for the question. + # Questions longer than this will be truncated to + # this length. + max_seq_length: 384 + # The maximum total input sequence length after + # WordPiece tokenization. Sequences longer than this + # will be truncated, and sequences shorter than this + # will be padded. + max_answer_length: 30 + # The maximum length of an answer that can be + # generated. This is needed because the start + # and end predictions are not conditioned + # on one another. + null_score_diff_threshold: 0.0 + # If null_score - best_non_null is greater than the threshold predict null. + n_best_size: 20 + # The total number of n-best predictions to generate at testing. + use_cache: true + do_lower_case: true + # if true does lower case + + num_workers: 2 + pin_memory: false + drop_last: false + + train_ds: + file: null # .json file + batch_size: 24 # per GPU + shuffle: true + num_samples: -1 + # Default values for the following params are retrieved from dataset config section, but you may override them + num_workers: ${model.dataset.num_workers} + drop_last: ${model.dataset.drop_last} + pin_memory: ${model.dataset.pin_memory} + + validation_ds: + file: null # .json file + batch_size: 24 # per GPU + shuffle: false + num_samples: -1 + # Default values for the following params are retrieved from dataset config section, but you may override them + num_workers: ${model.dataset.num_workers} + drop_last: ${model.dataset.drop_last} + pin_memory: ${model.dataset.pin_memory} + + test_ds: + file: null # .json file + batch_size: 24 # per GPU + shuffle: false + num_samples: -1 + # Default values for the following params are retrieved from dataset config section, but you may override them + num_workers: ${model.dataset.num_workers} + drop_last: ${model.dataset.drop_last} + pin_memory: ${model.dataset.pin_memory} + + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec + vocab_file: null # path to vocab file + tokenizer_model: null # only used if tokenizer is sentencepiece + special_tokens: null # expand the following to a dictionary if special tokens need to be added. + # only necessary for adding transformer/bert-specific special tokens to tokenizer if the tokenizer does not already have these inherently. + + language_model: + pretrained_model_name: bert-base-uncased # BERT-like model name + lm_checkpoint: null + config_file: null # json file, precedence over config + config: null # if specified initializes model from scratch + + token_classifier: + num_layers: 1 + dropout: 0.0 + num_classes: 2 + activation: relu + log_softmax: false + use_transformer_init: true + + + optim: + name: adamw + lr: 3e-5 + weight_decay: 0.0 + sched: + name: SquareRootAnnealing + + # pytorch lightning args + monitor: val_loss + reduce_on_plateau: false + + # scheduler config override + warmup_steps: null + warmup_ratio: 0.0 + last_epoch: -1 + +exp_manager: + exp_dir: null # where to store logs and checkpoints + name: *name # name of experiment + create_tensorboard_logger: True + create_checkpoint_callback: True + +hydra: + run: + dir: . + job_logging: + root: + handlers: null \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/question_answering/get_squad.py b/models/tacotron2/NeMo/examples/nlp/question_answering/get_squad.py new file mode 100755 index 0000000000..255b040c36 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/question_answering/get_squad.py @@ -0,0 +1,68 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import urllib.request + +from nemo.utils import logging + + +class SquadDownloader: + def __init__(self, save_path): + self.save_path = save_path + '/squad' + + if not os.path.exists(self.save_path): + os.makedirs(self.save_path) + + if not os.path.exists(self.save_path + '/v1.1'): + os.makedirs(self.save_path + '/v1.1') + + if not os.path.exists(self.save_path + '/v2.0'): + os.makedirs(self.save_path + '/v2.0') + + self.download_urls = { + 'https://rajpurkar.github.io/SQuAD-explorer' '/dataset/train-v1.1.json': 'v1.1/train-v1.1.json', + 'https://rajpurkar.github.io/SQuAD-explorer' '/dataset/dev-v1.1.json': 'v1.1/dev-v1.1.json', + 'https://rajpurkar.github.io/SQuAD-explorer' '/dataset/train-v2.0.json': 'v2.0/train-v2.0.json', + 'https://rajpurkar.github.io/SQuAD-explorer' '/dataset/dev-v2.0.json': 'v2.0/dev-v2.0.json', + } + + def download(self): + for item in self.download_urls: + url = item + file = self.download_urls[item] + + logging.info('Downloading: %s', url) + if os.path.isfile(self.save_path + '/' + file): + logging.info('** Download file already exists, skipping download') + else: + response = urllib.request.urlopen(url) + with open(self.save_path + '/' + file, "wb") as handle: + handle.write(response.read()) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Download Squad') + parser.add_argument( + '--destDir', + type=str, + required=False, + help='directory to store data', + default=os.path.split(os.path.abspath(__file__))[0], + ) + args = parser.parse_args() + logging.info(args.destDir) + squad_dl = SquadDownloader(args.destDir) + squad_dl.download() diff --git a/models/tacotron2/NeMo/examples/nlp/question_answering/question_answering_squad.py b/models/tacotron2/NeMo/examples/nlp/question_answering/question_answering_squad.py new file mode 100644 index 0000000000..269b4bfe48 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/question_answering/question_answering_squad.py @@ -0,0 +1,134 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +This script contains an example on how to train, evaluate and perform inference with the question answering model. +The QAModel in NeMo supports extractive question answering problems for data in the SQuAD (https://rajpurkar.github.io/SQuAD-explorer/) format. + +***Data format*** +The QAModel requires a JSON file for each dataset split. +In the following we will show example for a training file. Each title has one or multiple paragraph entries, each consisting of the text - "context", and question-answer entries. Each question-answer entry has: +* a question +* a globally unique id +* a boolean flag "is_impossible" which shows if the question is answerable or not +* in case the question is answerable one answer entry, which contains the text span and its starting character index in the context. If not answerable, the "answers" list is empty + +The evaluation file follows the above format except for it can provide more than one answers to the same question. +The inference file follows the above format except for it does not require the "answers" and "is_impossible" keywords. + + +***Downloading the dataset*** +Run ./NeMo/examples/nlp/question_answering/get_squad.py to download the SQuAD dataset: + +# python get_squad.py --destDir= + +***Setting the configs*** +The model and the PT trainer are defined in a config file which declares multiple important sections. +The most important ones are: + model: All arguments that are related to the Model - language model, tokenizer, token classifier, optimizer, + schedulers, and datasets/data loaders. + trainer: Any argument to be passed to PyTorch Lightning including number of epochs, number of GPUs, + precision level, etc. + +This script uses the `/examples/nlp/question_answering/conf/question_answering_squad_config.yaml` config file +by default. You may update the config file from the file directly. The other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'. + + +***Model Training*** +# python question_answering_squad.py + model.train_ds.file= + model.validation_ds= + trainer.max_epochs= + trainer.gpus=[] + + +***Model Evaluation*** +Set `do_training=False` in the script and run: + +# python question_answering_squad.py + model.test_file= + +To load a pretrained checkpoint from cloud prior to training (e.g. for fine-tuning) or evaluation you can set cfg.from_pretrained=, +e.g. MODEL_NAME='BERTBaseUncasedSQuADv1.1'. You can find all pretrained model names by using +QAModel.list_available_models(). To load a local checkpoint use qa_model.restore_from() + + +***Model Inference*** +For inference use + qa_model.inference( + file=, + batch_size=, + output_nbest_file=, + output_prediction_file= + ) + +More details on how to use this script can be found in +./NeMo/tutorials/nlp/Question_Answering_Squad.ipynb +""" + +import os + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models.question_answering.qa_model import QAModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="question_answering_squad_config") +def main(cfg: DictConfig) -> None: + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(**cfg.trainer) + exp_dir = exp_manager(trainer, cfg.get("exp_manager", None)) + + if not cfg.pretrained_model: + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + model = QAModel(cfg.model, trainer=trainer) + else: + logging.info(f'Loading pretrained model {cfg.pretrained_model}') + model = QAModel.from_pretrained(cfg.pretrained_model) + if cfg.do_training: + model.setup_training_data(train_data_config=cfg.model.train_ds) + model.setup_validation_data(val_data_config=cfg.model.validation_ds) + + if cfg.do_training: + trainer.fit(model) + if cfg.model.nemo_path: + model.save_to(cfg.model.nemo_path) + + if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.file is not None: + trainer.test(model) + + # change to path if you want results to be written to file e.g. os.path.join(exp_dir, "output_nbest_file.txt") + output_nbest_file = None + # change to path if you want results to be written to file e.g. os.path.join(exp_dir, "output_prediction_file.txt") + output_prediction_file = None + inference_samples = 5 # for test purposes. To use entire inference dataset set to -1 + all_preds, all_nbests = model.inference( + file=cfg.model.validation_ds.file, + batch_size=1, + num_samples=inference_samples, + output_nbest_file=output_nbest_file, + output_prediction_file=output_prediction_file, + ) + + for _, item in all_preds.items(): + print(f"question: {item[0]} answer: {item[1]}") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/text2sparql/conf/text2sparql_config.yaml b/models/tacotron2/NeMo/examples/nlp/text2sparql/conf/text2sparql_config.yaml new file mode 100644 index 0000000000..1603279ce8 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/text2sparql/conf/text2sparql_config.yaml @@ -0,0 +1,105 @@ +# Text2Sparql with BART + +name: &name Text2Sparql + +trainer: + gpus: 1 # the number of gpus, 0 for CPU, or list with gpu indices + num_nodes: 1 + max_epochs: 2 # the number of training epochs + max_steps: null # precedence over max_epochs + accumulate_grad_batches: 1 # accumulates grads every k batches + accelerator: ddp + gradient_clip_val: 0.0 + log_every_n_steps: 1 + val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch + checkpoint_callback: false # provided by exp_manager + logger: false # provided by exp_manager + +model: + nemo_path: null # exported .nemo path + max_seq_length: 150 + batch_size: 16 + convert_labels: true # true if Bart, false otherwise (converts pad_id to -100 for masked loss) + data_dir: null + + language_model: + pretrained_model_name: facebook/bart-base # huggingface end-to-end model name + pretrained_encoder_model_name: null # huggingface encoder model name + pretrained_decoder_model_name: null # huggingface decoder model name + lm_checkpoint: null + config: null + config_file: null # json file, precedence over config + + encoder_tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec + vocab_file: null # path to vocab file + tokenizer_model: null # tokenizer model for sentencepiece + special_tokens: null + add_special_tokens: true + + decoder_tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec + vocab_file: null # path to vocab file + tokenizer_model: null # tokenizer model for sentencepiece + special_tokens: null + add_special_tokens: true + + train_ds: + filepath: ${model.data_dir}/train.tsv # path to data file + shuffle: true + num_samples: -1 + num_workers: 2 + drop_last: false + pin_memory: false + + validation_ds: + filepath: ${model.data_dir}/test_easy.tsv # path to data file + shuffle: false + num_samples: -1 + num_workers: 2 + drop_last: false + pin_memory: false + + test_ds: + filepath: ${model.data_dir}/test_hard.tsv # path to data file + shuffle: false + num_samples: -1 + num_workers: 2 + drop_last: false + pin_memory: false + + optim: + name: adamw + lr: 4e-5 + weight_decay: 0.0 + + sched: + name: CosineAnnealing + warmup_steps: null + warmup_ratio: 0.06 + min_lr: 0.0 + last_epoch: -1 + + generate: + max_length: ${model.max_seq_length} + num_beams: 1 + length_penalty: 2.0 + early_stopping: true + repetition_penalty: 1.0 + do_sample: false + top_k: null + top_p: null + num_return_sequences: 1 + +exp_manager: + exp_dir: null # where to store logs and checkpoints + name: *name # name of experiment + create_tensorboard_logger: True + create_checkpoint_callback: True + +hydra: + run: + dir: . + job_logging: + root: + handlers: null \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/nlp/text2sparql/data/import_datasets.py b/models/tacotron2/NeMo/examples/nlp/text2sparql/data/import_datasets.py new file mode 100644 index 0000000000..ce5f86393b --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/text2sparql/data/import_datasets.py @@ -0,0 +1,134 @@ +# Copyright (c) 2020, MeetKai Inc. All rights reserved. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script downloads Text2Sparql data and processes it into NeMo's neural machine translation dataset format. + +Text2Sparql data consists of 3 files which are saved to source_data_dir: + - train_queries_v3.tsv + - test_easy_queries_v3.tsv + - test_hard_queries_v3.tsv + +After processing, the script saves them to the target_data_dir as: + - train.tsv + - test_easy.tsv + - test_hard.tsv + + +You may run it with: + +python import_datasets \ + --source_data_dir ./text2sparql_src \ + --target_data_dir ./text2sparql_tgt +""" + +import argparse +import csv +import os +from urllib.request import Request, urlopen + +from nemo.collections.nlp.data.data_utils.data_preprocessing import MODE_EXISTS_TMP, if_exist +from nemo.utils import logging + +base_url = "https://m.meetkai.com/public_datasets/knowledge/" +prefix_map = { + "train_queries_v3.tsv": "train.tsv", + "test_easy_queries_v3.tsv": "test_easy.tsv", + "test_hard_queries_v3.tsv": "test_hard.tsv", +} + + +def download_text2sparql(infold: str): + """Downloads text2sparql train, test_easy, and test_hard data + + Args: + infold: save directory path + """ + os.makedirs(infold, exist_ok=True) + + for prefix in prefix_map: + url = base_url + prefix + + logging.info(f"Downloading: {url}") + if if_exist(infold, [prefix]): + logging.info("** Download file already exists, skipping download") + else: + req = Request(url, headers={"User-Agent": "Mozilla/5.0"}) + with open(os.path.join(infold, prefix), "wb") as handle: + handle.write(urlopen(req, timeout=20).read()) + + +def process_text2sparql(infold: str, outfold: str, do_lower_case: bool): + """ Process and convert MeetKai's text2sparql datasets to NeMo's neural machine translation format. + + Args: + infold: directory path to raw text2sparql data containing + train.tsv, test_easy.tsv, test_hard.tsv + outfold: output directory path to save formatted data for NeuralMachineTranslationDataset + the first line is header (sentence [tab] label) + each line should be [sentence][tab][label] + do_lower_case: if true, convert all sentences and labels to lower + """ + logging.info(f"Processing Text2Sparql dataset and storing at: {outfold}") + + os.makedirs(outfold, exist_ok=True) + + dataset_name = "Text2Sparql" + for prefix in prefix_map: + input_file = os.path.join(infold, prefix) + output_file = os.path.join(outfold, prefix_map[prefix]) + + if if_exist(outfold, [prefix_map[prefix]]): + logging.info(f"** {MODE_EXISTS_TMP.format(prefix_map[prefix], dataset_name, output_file)}") + continue + + if not if_exist(infold, [prefix]): + logging.info(f"** {prefix} of {dataset_name}" f" is skipped as it was not found") + continue + + assert input_file != output_file, "input file cannot equal output file" + with open(input_file, "r") as in_file: + with open(output_file, "w") as out_file: + reader = csv.reader(in_file, delimiter="\t") + + # replace headers + out_file.write("sentence\tlabel\n") + next(reader) + + for line in reader: + sentence = line[0] + label = line[1] + if do_lower_case: + sentence = sentence.lower() + label = label.lower() + out_file.write(f"{sentence}\t{label}\n") + + +if __name__ == "__main__": + # Parse the command-line arguments. + parser = argparse.ArgumentParser(description="Process and convert datasets into NeMo's format") + parser.add_argument( + "--source_data_dir", required=True, type=str, help="Path to the folder containing the dataset files" + ) + parser.add_argument("--target_data_dir", required=True, type=str, help="Path to save the processed dataset") + parser.add_argument("--do_lower_case", action="store_true") + args = parser.parse_args() + + source_dir = args.source_data_dir + target_dir = args.target_data_dir + do_lower_case = args.do_lower_case + + download_text2sparql(infold=source_dir) + process_text2sparql(infold=source_dir, outfold=target_dir, do_lower_case=do_lower_case) diff --git a/models/tacotron2/NeMo/examples/nlp/text2sparql/evaluate_text2sparql.py b/models/tacotron2/NeMo/examples/nlp/text2sparql/evaluate_text2sparql.py new file mode 100644 index 0000000000..050d142e99 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/text2sparql/evaluate_text2sparql.py @@ -0,0 +1,72 @@ +# Copyright (c) 2020, MeetKai Inc. All rights reserved. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script contains an example on how to evaluate a NeuralMachineTranslationModel. +To load the example Text2Sparql dataset, please refer to ./data/import_datasets.py. +To train a model, please refer to text2sparql.py. + + +***Setting the configs*** +This script uses the `/examples/nlp/text2sparql/conf/text2sparql_config.yaml` config file by default. +You may update the config file from the file directly or by using the command line arguments. +Another other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'. + +Please refer to text2sparql.py for detailed instructions on setting the configuration. + + +***How to run the script?*** +- To reload and evaluate the model, run: + +python evaluate_text2sparql.py \ + model.test_ds.filepath="$TGT_DATA_DIR"/test_easy.tsv \ + model.batch_size=16 \ + model.nemo_path=./NeMo_logs/bart.nemo \ + exp_manager.exp_dir=./NeMo_logs +""" + +import os + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models.neural_machine_translation import NeuralMachineTranslationModel +from nemo.core.config import hydra_runner +from nemo.utils import logging + + +@hydra_runner(config_path="conf", config_name="text2sparql_config") +def main(cfg: DictConfig) -> None: + logging.info(f"Config:\n {OmegaConf.to_yaml(cfg)}") + trainer = pl.Trainer(gpus=cfg.trainer.gpus) + nmt_model = NeuralMachineTranslationModel.restore_from(restore_path=cfg.model.nemo_path) + nmt_model.setup_test_data(cfg.model.test_ds) + results = trainer.test(nmt_model) + + with open(cfg.model.test_ds.filepath, "r") as f: + lines = f.readlines() + + lines[0] = lines[0].strip() + f"\tpredictions\n" + for i, res in enumerate(results[0]["texts"]): + lines[i + 1] = lines[i + 1].strip() + f"\t{res}\n" + + savepath = os.path.join(cfg.exp_manager.exp_dir, os.path.basename(cfg.model.test_ds.filepath)) + with open(savepath, "w") as f: + f.writelines(lines) + logging.info(f"Predictions saved to {savepath}") + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/examples/nlp/text2sparql/text2sparql.py b/models/tacotron2/NeMo/examples/nlp/text2sparql/text2sparql.py new file mode 100644 index 0000000000..062094ccd3 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/text2sparql/text2sparql.py @@ -0,0 +1,112 @@ +# Copyright (c) 2020, MeetKai Inc. All rights reserved. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script contains an example on how to train and save a NeuralMachineTranslationModel. +NeuralMachineTranslationModel in NeMo supports sequence to sequence problems such as language translation +and text summarization, provided the data follows the format specified below. + + +***Data format*** +NeuralMachineTranslationModel requires the data to be stored in TAB separated files (.tsv) with two columns of +sentence and label, where the first line is a header of format: + sentence[TAB]label +And each line is of the format: + [SENTENCE][TAB][LABEL] + +If your dataset is stored in another format, you need to convert it to this format to use a +NeuralMachineTranslationModel. + + +***Setting the configs*** +This script uses the `/examples/nlp/text2sparql/conf/text2sparql_config.yaml` config file by default. +You may update the config file from the file directly or by using the command line arguments. +Another other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'. + +A NeuralMachineTranslationModel's config file declares multiple import sections. They are: + - trainer: Arguments to be passed to PyTorch Lightning. + - model: All arguments that relate to the Model - language_model, tokenizers, datasets, optimizer, generate. + - exp_manager: Arguments to be passed to NeMo's experiment manager. + - hydra: Arguments to be passed to Hydra. + +If using text2sparql_config.yaml, you must first update the following fields in the config: + - model.nemo_path: Model save path. Eg. [PATH]/bart.nemo + - model.data_dir: Path to data directory. Alternatively, you can adjust the file paths directly: + - model.train_ds.filepath + - model.validation_ds.filepath + - model.test_ds.filepath + - exp_manager.exp_dir: Directory to log results from the experiment. + +It is highly recommended to also adjust these parameters as necessary: + - trainer.gpus: Set to 0 to use CPU. Otherwise the number denotes the number of GPUs. + - trainer.max_epochs: Maximum number of epochs to train for. + - model.batch_size: 8 is sufficient to train a decent Bart model for Text2Sparql. + - model.max_seq_length: Maximum (tokenized) sequence length. 150 is sufficient for Text2Sparql. + - model.language_model.pretrained_model_name: End2end pretrained model name from huggingface. + - model.encoder_tokenizer.tokenizer_name: Pretrained tokenizer name from huggingface. + - model.decoder_tokenizer.tokenizer_name: The same as above, as the tokenizer will handle encoding and decoding. + - model.optim.lr: Learning rate. + +You can also specify an encoder and decoder rather than using an end2end model like Bart by defining these parameters: + - model.language_model.pretrained_encoder_model_name: Pretrained huggingface encoder model name. + - model.encoder_tokenizer.tokenizer_name: Pretrained huggingface encoder tokenizer name. + - model.language_model.pretrained_decoder_model_name: Pretrained huggingface decoder model name. + - model.decoder_tokenizer.tokenizer_name: Pretrained huggingface decoder tokenizer name. + - model.language_model.pretrained_model_name: Set this to null. + + +***How to run the script?*** +- First, download the data to TGT_DATA_DIR (see ./data/import_datasets.py): + +SRC_DATA_DIR=./data/text2sparql_src +TGT_DATA_DIR=./data/text2sparql_tgt + +python ./data/import_datasets.py \ + --source_data_dir $SRC_DATA_DIR \ + --target_data_dir $TGT_DATA_DIR + +- And run the following to train and save the model: + +python text2sparql.py \ + model.train_ds.filepath="$TGT_DATA_DIR"/train.tsv \ + model.validation_ds.filepath="$TGT_DATA_DIR"/test_easy.tsv \ + model.test_ds.filepath="$TGT_DATA_DIR"/test_hard.tsv \ + model.batch_size=16 \ + model.nemo_path=./NeMo_logs/bart.nemo \ + exp_manager.exp_dir=./NeMo_logs +""" + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models.neural_machine_translation import NeuralMachineTranslationModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="text2sparql_config") +def main(cfg: DictConfig) -> None: + logging.info(f"Config:\n {OmegaConf.to_yaml(cfg)}") + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + nmt_model = NeuralMachineTranslationModel(cfg.model, trainer=trainer) + trainer.fit(nmt_model) + if cfg.model.nemo_path: + nmt_model.save_to(cfg.model.nemo_path) + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/examples/nlp/text_classification/conf/text_classification_config.yaml b/models/tacotron2/NeMo/examples/nlp/text_classification/conf/text_classification_config.yaml new file mode 100644 index 0000000000..8470991ea8 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/text_classification/conf/text_classification_config.yaml @@ -0,0 +1,119 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for text classification with pre-trained BERT models + +trainer: + gpus: 1 # number of GPUs (0 for CPU), or list of the GPUs to use e.g. [0, 1] + num_nodes: 1 + max_epochs: 100 + max_steps: null # precedence over max_epochs + accumulate_grad_batches: 1 # accumulates grads every k batches + gradient_clip_val: 0.0 + amp_level: O0 # O1/O2 for mixed precision + precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. + accelerator: ddp + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + +model: + nemo_path: text_classification_model.nemo # filename to save the model and associated artifacts to .nemo file + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # or sentencepiece + vocab_file: null # path to vocab file + tokenizer_model: null # only used if tokenizer is sentencepiece + special_tokens: null + + language_model: + pretrained_model_name: bert-base-uncased + lm_checkpoint: null + config_file: null # json file, precedence over config + config: null + + classifier_head: + num_output_layers: 2 + fc_dropout: 0.1 + + class_labels: + class_labels_file : null # optional to specify a file containing the list of the labels + + dataset: + num_classes: ??? # The number of classes. 0 < Label ", "") + outfiles[mode].write(f'{review}\t{label}\n') + + for mode in modes: + outfiles[mode].close() + + class_labels_file = open(os.path.join(outfold, 'label_ids.tsv'), 'w') + class_labels_file.write('negative\npositive\n') + class_labels_file.close() + + +def process_sst2(infold, outfold, uncased, splits=['train', 'dev']): + """Process sst2 dataset.""" + # "test" split doesn't have labels, so it is skipped + if not os.path.exists(infold): + link = 'https://dl.fbaipublicfiles.com/glue/data/SST-2.zip' + raise ValueError( + f'Data not found at {infold}. Please download SST-2 dataset from `{link}` and ' + f'extract it into the folder specified by `source_data_dir` argument.' + ) + + logging.info(f'Processing SST-2 dataset') + os.makedirs(outfold, exist_ok=True) + + def _read_tsv(input_file, quotechar=None): + """Read a tab separated value file.""" + with open(input_file, "r") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + lines.append(line) + return lines + + for split in splits: + # Load input file. + input_file = os.path.join(infold, split + '.tsv') + lines = _read_tsv(input_file) + # Create output. + outfile = open(os.path.join(outfold, split + '.tsv'), 'w') + + # Copy lines, skip the header (line 0). + for line in lines[1:]: + text = line[0] + label = line[1] + # Lowercase when required. + if uncased: + text = text.lower() + # Write output. + outfile.write(f'{text}\t{label}\n') + # Close file. + outfile.close() + + class_labels_file = open(os.path.join(outfold, 'label_ids.tsv'), 'w') + class_labels_file.write('negative\npositive\n') + class_labels_file.close() + + logging.info(f'Result stored at {outfold}') + + +def process_chemprot(source_dir, target_dir, uncased, modes=['train', 'test', 'dev']): + if not os.path.exists(source_dir): + link = 'https://github.com/arwhirang/recursive_chemprot/tree/master/Demo/tree_LSTM/data' + raise ValueError(f'Data not found at {source_dir}. ' f'Please download ChemProt from {link}.') + + logging.info(f'Processing Chemprot dataset and store at {target_dir}') + os.makedirs(target_dir, exist_ok=True) + + naming_map = {'train': 'trainingPosit_chem', 'test': 'testPosit_chem', 'dev': 'developPosit_chem'} + + def _read_tsv(input_file, quotechar=None): + """Reads a tab separated value file.""" + with open(input_file, "r") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + lines.append(line) + return lines + + outfiles = {} + label_mapping = {} + out_label_mapping = open(os.path.join(target_dir, 'label_mapping.tsv'), 'w') + for mode in modes: + outfiles[mode] = open(os.path.join(target_dir, mode + '.tsv'), 'w') + input_file = os.path.join(source_dir, naming_map[mode]) + lines = _read_tsv(input_file) + for line in lines: + text = line[1] + label = line[2] + if label == "True": + label = line[3] + if uncased: + text = text.lower() + if label not in label_mapping: + out_label_mapping.write(f'{label}\t{len(label_mapping)}\n') + label_mapping[label] = len(label_mapping) + label = label_mapping[label] + outfiles[mode].write(f'{text}\t{label}\n') + for mode in modes: + outfiles[mode].close() + out_label_mapping.close() + + +def process_thucnews(infold, outfold): + modes = ['train', 'test'] + train_size = 0.8 + if not os.path.exists(infold): + link = 'thuctc.thunlp.org/' + raise ValueError(f'Data not found at {infold}. ' f'Please download THUCNews from {link}.') + + logging.info(f'Processing THUCNews dataset and store at {outfold}') + os.makedirs(outfold, exist_ok=True) + + outfiles = {} + for mode in modes: + outfiles[mode] = open(os.path.join(outfold, mode + '.tsv'), 'a+', encoding='utf-8') + categories = ['体育', '娱乐', '家居', '彩票', '房产', '教育', '时尚', '时政', '星座', '游戏', '社会', '科技', '股票', '财经'] + for category in categories: + label = categories.index(category) + category_files = glob.glob(f'{infold}/{category}/*.txt') + test_num = int(len(category_files) * (1 - train_size)) + test_files = category_files[:test_num] + train_files = category_files[test_num:] + + for mode in modes: + logging.info(f'Processing {mode} data of the category {category}') + if mode == 'test': + files = test_files + else: + files = train_files + + if len(files) == 0: + logging.info(f'Skipping category {category} for {mode} mode') + continue + + for file in tqdm.tqdm(files): + with open(file, 'r', encoding='utf-8') as f: + news = f.read().strip().replace('\r', '') + news = news.replace('\n', '').replace('\t', ' ') + outfiles[mode].write(f'{news}\t{label}\n') + for mode in modes: + outfiles[mode].close() + + +if __name__ == "__main__": + # Parse the command-line arguments. + parser = argparse.ArgumentParser(description="Process and convert datasets into NeMo\'s format.") + parser.add_argument("--dataset_name", required=True, type=str, choices=['imdb', 'thucnews', 'chemprot']) + parser.add_argument( + "--source_data_dir", required=True, type=str, help='The path to the folder containing the dataset files.' + ) + parser.add_argument("--target_data_dir", required=True, type=str) + parser.add_argument("--do_lower_case", action='store_true') + args = parser.parse_args() + + dataset_name = args.dataset_name + do_lower_case = args.do_lower_case + source_dir = args.source_data_dir + target_dir = args.target_data_dir + + if not exists(source_dir): + raise FileNotFoundError(f"{source_dir} does not exist.") + + if dataset_name == 'imdb': + process_imdb(source_dir, target_dir, do_lower_case) + elif dataset_name == 'thucnews': + process_thucnews(source_dir, target_dir) + elif dataset_name == "chemprot": + process_chemprot(source_dir, target_dir, do_lower_case) + elif dataset_name == "sst-2": + process_sst2(source_dir, target_dir, do_lower_case) + else: + raise ValueError( + f'Dataset {dataset_name} is not supported.' + + "Please make sure that you build the preprocessing process for it. " + + "NeMo's format assumes that a data file has a header and each line of the file follows " + + "the format: text [TAB] label. Label is assumed to be an integer." + ) diff --git a/models/tacotron2/NeMo/examples/nlp/text_classification/model_parallel_text_classification_evaluation.py b/models/tacotron2/NeMo/examples/nlp/text_classification/model_parallel_text_classification_evaluation.py new file mode 100644 index 0000000000..255a27ee79 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/text_classification/model_parallel_text_classification_evaluation.py @@ -0,0 +1,41 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script runs model parallel text classification evaluation. +""" +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models.text_classification import TextClassificationModel +from nemo.collections.nlp.parts.nlp_overrides import NLPDDPPlugin +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="text_classification_config") +def main(cfg: DictConfig) -> None: + logging.info(f'\nConfig Params:\n{OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(plugins=[NLPDDPPlugin()], **cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + # TODO: can we drop strict=False + model = TextClassificationModel.restore_from(cfg.model.nemo_path, trainer=trainer, strict=False) + model.setup_test_data(test_data_config=cfg.model.test_ds) + + trainer.test(model=model, ckpt_path=None) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/text_classification/text_classification_with_bert.py b/models/tacotron2/NeMo/examples/nlp/text_classification/text_classification_with_bert.py new file mode 100644 index 0000000000..1cad923381 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/text_classification/text_classification_with_bert.py @@ -0,0 +1,154 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script contains an example on how to train, evaluate and perform inference with the TextClassificationModel. +TextClassificationModel in NeMo supports text classification problems such as sentiment analysis or +domain/intent detection for dialogue systems, as long as the data follows the format specified below. + +***Data format*** +TextClassificationModel requires the data to be stored in TAB separated files (.tsv) with two columns of sentence and +label. Each line of the data file contains text sequences, where words are separated with spaces and label separated +with [TAB], i.e.: + +[WORD][SPACE][WORD][SPACE][WORD][TAB][LABEL] + +For example: + +hide new secretions from the parental units[TAB]0 +that loves its characters and communicates something rather beautiful about human nature[TAB]1 +... + +If your dataset is stored in another format, you need to convert it to this format to use the TextClassificationModel. + + +***Setting the configs*** +The model and the PT trainer are defined in a config file which declares multiple important sections. +The most important ones are: + model: All arguments that are related to the Model - language model, tokenizer, head classifier, optimizer, + schedulers, and datasets/data loaders. + trainer: Any argument to be passed to PyTorch Lightning including number of epochs, number of GPUs, + precision level, etc. + +This script uses the `/examples/nlp/text_classification/conf/text_classification_config.yaml` default config file +by default. You may update the config file from the file directly or by using the command line arguments. +Other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'. + +You first need to set the num_classes in the config file which specifies the number of classes in the dataset. +Notice that some config lines, including `model.dataset.classes_num`, have `???` as their value, this means that values +for these fields are required to be specified by the user. We need to specify and set the `model.train_ds.file_name`, +`model.validation_ds.file_name`, and `model.test_ds.file_name` in the config file to the paths of the train, validation, + and test files if they exist. We may do it by updating the config file or by setting them from the command line. + + +***How to run the script?*** +For example the following would train a model for 50 epochs in 2 GPUs on a classification task with 2 classes: + +# python text_classification_with_bert.py + model.dataset.num_classes=2 + model.train_ds=PATH_TO_TRAIN_FILE + model.validation_ds=PATH_TO_VAL_FILE + trainer.max_epochs=50 + trainer.gpus=2 + +This script would also reload the last checkpoint after the training is done and does evaluation on the dev set, +then performs inference on some sample queries. + +By default, this script uses examples/nlp/text_classification/conf/text_classifciation_config.py config file, and +you may update all the params in the config file from the command line. You may also use another config file like this: + +# python text_classification_with_bert.py --config-name==PATH_TO_CONFIG_FILE + model.dataset.num_classes=2 + model.train_ds=PATH_TO_TRAIN_FILE + model.validation_ds=PATH_TO_VAL_FILE + trainer.max_epochs=50 + trainer.gpus=2 + +***Load a saved model*** +This script would save the model after training into '.nemo' checkpoint file specified by nemo_path of the model config. +You may restore the saved model like this: + model = TextClassificationModel.restore_from(restore_path=NEMO_FILE_PATH) + +***Evaluation a saved model on another dataset*** +# If you wanted to evaluate the saved model on another dataset, you may restore the model and create a new data loader: + eval_model = TextClassificationModel.restore_from(restore_path=checkpoint_path) + +# Then, you may create a dataloader config for evaluation: + eval_config = OmegaConf.create( + {'file_path': cfg.model.test_ds.file_path, 'batch_size': 64, 'shuffle': False, 'num_workers': 3} + ) + eval_model.setup_test_data(test_data_config=eval_config) + +# You need to create a new trainer: + eval_trainer = pl.Trainer(gpus=1) + eval_model.set_trainer(eval_trainer) + eval_trainer.test(model=eval_model, verbose=False) +""" +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models.text_classification import TextClassificationModel +from nemo.collections.nlp.parts.nlp_overrides import NLPDDPPlugin +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="text_classification_config") +def main(cfg: DictConfig) -> None: + logging.info(f'\nConfig Params:\n{OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(plugins=[NLPDDPPlugin()], **cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + + if not cfg.model.train_ds.file_path: + raise ValueError("'train_ds.file_path' need to be set for the training!") + + model = TextClassificationModel(cfg.model, trainer=trainer) + logging.info("===========================================================================================") + logging.info('Starting training...') + trainer.fit(model) + logging.info('Training finished!') + logging.info("===========================================================================================") + + if cfg.model.nemo_path: + # '.nemo' file contains the last checkpoint and the params to initialize the model + model.save_to(cfg.model.nemo_path) + logging.info(f'Model is saved into `.nemo` file: {cfg.model.nemo_path}') + + # We evaluate the trained model on the test set if test_ds is set in the config file + if cfg.model.test_ds.file_path: + logging.info("===========================================================================================") + logging.info("Starting the testing of the trained model on test set...") + trainer.test(model=model, ckpt_path=None, verbose=False) + logging.info("Testing finished!") + logging.info("===========================================================================================") + + # perform inference on a list of queries. + if "infer_samples" in cfg.model and cfg.model.infer_samples: + logging.info("===========================================================================================") + logging.info("Starting the inference on some sample queries...") + + # max_seq_length=512 is the maximum length BERT supports. + results = model.classifytext(queries=cfg.model.infer_samples, batch_size=16, max_seq_length=512) + logging.info('The prediction results of some sample queries with the trained model:') + for query, result in zip(cfg.model.infer_samples, results): + logging.info(f'Query : {query}') + logging.info(f'Predicted label: {result}') + + logging.info("Inference finished!") + logging.info("===========================================================================================") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml b/models/tacotron2/NeMo/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml new file mode 100644 index 0000000000..0d27907751 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml @@ -0,0 +1,133 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Punctuation and capitalization model with pretrained BERT-like models + +pretrained_model: null # pretrained Punctuation and Capitalization model from list_available_models(), for example: +# punctuation_en_bert or punctuation_en_distilbert +# or your_model.nemo +trainer: + gpus: 1 # the number of gpus, 0 for CPU + num_nodes: 1 + max_epochs: 3 + max_steps: null # precedence over max_epochs + accumulate_grad_batches: 1 # accumulates grads every k batches + gradient_clip_val: 0.0 + amp_level: O0 # O1/O2 for mixed precision + precision: 16 # Should be set to 16 for O1 and O2, default is 16 as PT ignores it when am_level is O0 + accelerator: ddp + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + +exp_manager: + exp_dir: null # exp_dir for your experiment, if None, defaults to "./nemo_experiments" + name: Punctuation_and_Capitalization # The name of your model + create_tensorboard_logger: true # Whether you want exp_manger to create a tb logger + create_checkpoint_callback: true # Whether you want exp_manager to create a model checkpoint callback + +model: + punct_label_ids: null # will be filled during training + capit_label_ids: null # will be filled during training + class_labels: + punct_labels_file: punct_label_ids.csv + capit_labels_file: capit_label_ids.csv + dataset: + data_dir: ??? # /path/to/training/data + # parameters for dataset preprocessing + max_seq_length: 128 + pad_label: 'O' + ignore_extra_tokens: false + ignore_start_end: false + use_cache: true + # shared among dataloaders + num_workers: 2 + pin_memory: false + drop_last: false + + train_ds: + text_file: text_train.txt + labels_file: labels_train.txt + shuffle: true + num_samples: -1 + batch_size: 64 + + validation_ds: + # if evaluation data is not in the model.dataset.data_dir as the training data or multiple datasets are used for + # evaluation is needed, specify ds_item, otherwise by default data_dir is used + ds_item: null # expected format: [PATH_TO_DEV1,PATH_TO_DEV2] (Note no space between the paths and square brackets) + text_file: text_dev.txt + labels_file: labels_dev.txt + shuffle: false + num_samples: -1 + batch_size: 64 + + test_ds: + # if evaluation data is not in the model.dataset.data_dir as the training data or multiple datasets are used for + # evaluation is needed, specify ds_item, otherwise by default data_dir is used + ds_item: null # expected format: [PATH_TO_DEV1,PATH_TO_DEV2] (Note no space between the paths and square brackets) + text_file: text_dev.txt + labels_file: labels_dev.txt + shuffle: false + num_samples: -1 + batch_size: 64 + + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # or sentencepiece + vocab_file: null # path to vocab file + tokenizer_model: null # only used if tokenizer is sentencepiece + special_tokens: null + + language_model: + pretrained_model_name: bert-base-uncased + lm_checkpoint: null + config_file: null # json file, precedence over config + config: null + + punct_head: + punct_num_fc_layers: 1 + fc_dropout: 0.1 + activation: 'relu' + use_transformer_init: True + + capit_head: + capit_num_fc_layers: 1 + fc_dropout: 0.1 + activation: 'relu' + use_transformer_init: true + + optim: + name: adam + lr: 1e-4 + weight_decay: 0.00 + + sched: + name: WarmupAnnealing + # Scheduler params + warmup_steps: null + warmup_ratio: 0.1 + last_epoch: -1 + + # pytorch lightning args + monitor: val_loss + reduce_on_plateau: false + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/nlp/token_classification/conf/token_classification_config.yaml b/models/tacotron2/NeMo/examples/nlp/token_classification/conf/token_classification_config.yaml new file mode 100644 index 0000000000..8cb162a60d --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/token_classification/conf/token_classification_config.yaml @@ -0,0 +1,119 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Token Classification tasks (for example, Named Entity Recognition) with pretrained BERT-like models + +pretrained_model: null # pretrained TokenClassification model from list_available_models() or path to a .nemo file, +# for example: ner_en_bert or your_model.nemo +trainer: + gpus: 1 # the number of gpus, 0 for CPU + num_nodes: 1 + max_epochs: 5 + max_steps: null # precedence over max_epochs + accumulate_grad_batches: 1 # accumulates grads every k batches + gradient_clip_val: 0.0 + amp_level: O0 # O1/O2 for mixed precision + precision: 16 # Should be set to 16 for O1 and O2, default is 16 as PT ignores it when am_level is O0 + accelerator: ddp + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + +exp_manager: + exp_dir: null # exp_dir for your experiment, if None, defaults to "./nemo_experiments" + name: token_classification_model # The name of your model + create_tensorboard_logger: true # Whether you want exp_manager to create a tb logger + create_checkpoint_callback: true # Whether you want exp_manager to create a model checkpoint callback + +model: + label_ids: null # will be filled during training + class_labels: + class_labels_file: label_ids.csv # will be generated during training and saved in .nemo file + dataset: + data_dir: ??? # /path/to/data + class_balancing: null # choose from [null, weighted_loss]. Weighted_loss enables the weighted class balancing of the loss, may be used for handling unbalanced classes + max_seq_length: 128 + pad_label: 'O' + ignore_extra_tokens: false + ignore_start_end: false + use_cache: true + # shared among dataloaders + num_workers: 2 + pin_memory: false + drop_last: false + + train_ds: + text_file: text_train.txt + labels_file: labels_train.txt + shuffle: true + num_samples: -1 + batch_size: 64 + + validation_ds: + text_file: text_dev.txt + labels_file: labels_dev.txt + shuffle: false + num_samples: -1 + batch_size: 64 + + test_ds: + text_file: text_dev.txt + labels_file: labels_dev.txt + shuffle: false + num_samples: -1 + batch_size: 64 + + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # or sentencepiece + vocab_file: null # path to vocab file + tokenizer_model: null # only used if tokenizer is sentencepiece + special_tokens: null + + language_model: + pretrained_model_name: bert-base-uncased + lm_checkpoint: null + config_file: null # json file, precedence over config + config: null + + + head: + num_fc_layers: 2 + fc_dropout: 0.5 + activation: 'relu' + use_transformer_init: True + + optim: + name: adam + lr: 5e-5 + weight_decay: 0.00 + + sched: + name: WarmupAnnealing + # Scheduler params + warmup_steps: null + warmup_ratio: 0.1 + last_epoch: -1 + + # pytorch lightning args + monitor: val_loss + reduce_on_plateau: false + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/models/tacotron2/NeMo/examples/nlp/token_classification/data/get_tatoeba_data.py b/models/tacotron2/NeMo/examples/nlp/token_classification/data/get_tatoeba_data.py new file mode 100644 index 0000000000..d895e81543 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/token_classification/data/get_tatoeba_data.py @@ -0,0 +1,242 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import random +import re +import string +import subprocess + +from nemo.utils import logging + +URL = {'tatoeba': 'https://downloads.tatoeba.org/exports/sentences.csv'} + + +def __maybe_download_file(destination: str, source: str): + """ + Downloads source to destination if not exists. + If exists, skips download + Args: + destination: local filepath + source: url of resource + """ + source = URL[source] + if not os.path.exists(destination): + logging.info(f'Downloading {source} to {destination}') + subprocess.run(['wget', '-O', destination, source]) + else: + logging.info(f'{destination} found. Skipping download') + + +def __process_english_sentences( + in_file: str, out_file: str, percent_to_cut: float = 0, num_to_combine: int = 1, num_samples: int = -1 +): + """ + Extract English sentences from the Tatoeba dataset. + + Expected in_file format + that + contrain letters and punctuation marks (,.?). + Chop and combine sentences. + Args: + in_file: local filepath to the tatoeba dataset. + Format: id [TAB] region_name [TAB] sentence, + for example: "1276\teng\tLet's try something.\n" + out_file: local filepath to the clean dataset + percent_to_cut: Percent of sentences to cut in the middle + to get examples of incomplete sentences. + This could be useful since ASR output not always + represents a complete sentence + num_to_combine: Number of sentences to combine into + a single example + num_samples: Number of samples in the final dataset + """ + if not os.path.exists(in_file): + raise FileNotFoundError(f'{in_file} not found.') + + in_file = open(in_file, 'r') + out_file = open(out_file, 'w') + lines_to_combine = [] + samples_count = 0 + + for line in in_file: + line = line.split('\t') + # use only English sentences + if line[1] == 'eng': + line = line[2].strip() + if re.match("^[A-Z][A-Za-z.,'?\s]+$", line): # nopep8 + # chop some sentences in the middle + if percent_to_cut > 0: + line = line.split() + if random.random() < percent_to_cut: + line = line[: len(line) // 2] + line = ' '.join(line) + + # combine multiple sentences into a single example + # to make it harder for the model to learn eos punctuation + if len(lines_to_combine) >= num_to_combine: + if samples_count == num_samples: + return + out_file.write(' '.join(lines_to_combine) + '\n') + lines_to_combine = [] + samples_count += 1 + lines_to_combine.append(line) + + if len(lines_to_combine) > 0 and (samples_count < num_samples or num_samples < 0): + out_file.write(' '.join(lines_to_combine) + '\n') + + +def __split_into_train_dev(in_file: str, train_file: str, dev_file: str, percent_dev: float): + """ + Create train and dev split of the dataset. + Args: + in_file: local filepath to the dataset + train_file: local filepath to the train dataset + dev_file: local filepath to the dev dataset + percent_dev: Percent of the sentences in the dev set + """ + if not os.path.exists(in_file): + raise FileNotFoundError(f'{in_file} not found.') + + lines = open(in_file, 'r').readlines() + train_file = open(train_file, 'w') + dev_file = open(dev_file, 'w') + + dev_size = int(len(lines) * percent_dev) + train_file.write(' '.join(lines[:-dev_size])) + dev_file.write(' '.join(lines[-dev_size:])) + + +def remove_punctuation(word: str): + """ + Removes all punctuation marks from a word except for ' + that is often a part of word: don't, it's, and so on + """ + all_punct_marks = string.punctuation.replace("'", '') + return re.sub('[' + all_punct_marks + ']', '', word) + + +def create_text_and_labels(output_dir: str, file_path: str, punct_marks: str = ',.?'): + """ + Create datasets for training and evaluation. + + Args: + output_dir: path to the output data directory + file_path: path to file name + punct_marks: supported punctuation marks + + The data will be split into 2 files: text.txt and labels.txt. \ + Each line of the text.txt file contains text sequences, where words\ + are separated with spaces. The labels.txt file contains \ + corresponding labels for each word in text.txt, the labels are \ + separated with spaces. Each line of the files should follow the \ + format: \ + [WORD] [SPACE] [WORD] [SPACE] [WORD] (for text.txt) and \ + [LABEL] [SPACE] [LABEL] [SPACE] [LABEL] (for labels.txt).' + """ + if not os.path.exists(file_path): + raise ValueError(f'{file_path} not found') + + os.makedirs(output_dir, exist_ok=True) + + base_name = os.path.basename(file_path) + labels_file = os.path.join(output_dir, 'labels_' + base_name) + text_file = os.path.join(output_dir, 'text_' + base_name) + + with open(file_path, 'r') as f: + with open(text_file, 'w') as text_f: + with open(labels_file, 'w') as labels_f: + for line in f: + line = line.split() + text = '' + labels = '' + for word in line: + label = word[-1] if word[-1] in punct_marks else 'O' + word = remove_punctuation(word) + if len(word) > 0: + if word[0].isupper(): + label += 'U' + else: + label += 'O' + + word = word.lower() + text += word + ' ' + labels += label + ' ' + + text_f.write(text.strip() + '\n') + labels_f.write(labels.strip() + '\n') + + print(f'{text_file} and {labels_file} created from {file_path}.') + + +def __delete_file(file_to_del: str): + """ + Deletes the file + Args: + file_to_del: local filepath to the file to delete + """ + if os.path.exists(file_to_del): + os.remove(file_to_del) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Prepare tatoeba dataset') + parser.add_argument("--data_dir", required=True, type=str) + parser.add_argument("--dataset", default='tatoeba', type=str) + parser.add_argument("--num_samples", default=-1, type=int, help='-1 to use the whole dataset') + parser.add_argument("--percent_to_cut", default=0, type=float, help='Percent of sentences to cut in the middle') + parser.add_argument( + "--num_lines_to_combine", default=1, type=int, help='Number of lines to combine into single example' + ) + parser.add_argument("--percent_dev", default=0.2, type=float, help='Size of the dev set, float') + parser.add_argument("--clean_dir", action='store_true') + args = parser.parse_args() + + if not os.path.exists(args.data_dir): + os.makedirs(args.data_dir) + + if args.dataset != 'tatoeba': + raise ValueError("Unsupported dataset.") + + logging.info(f'Downloading tatoeba dataset') + tatoeba_dataset = os.path.join(args.data_dir, 'sentences.csv') + __maybe_download_file(tatoeba_dataset, args.dataset) + + logging.info(f'Processing English sentences...') + clean_eng_sentences = os.path.join(args.data_dir, 'clean_eng_sentences.txt') + __process_english_sentences( + tatoeba_dataset, clean_eng_sentences, args.percent_to_cut, args.num_lines_to_combine, args.num_samples + ) + + train_file = os.path.join(args.data_dir, 'train.txt') + dev_file = os.path.join(args.data_dir, 'dev.txt') + + logging.info( + f'Splitting the {args.dataset} dataset into train and dev sets' + ' and creating labels and text files' + ) + __split_into_train_dev(clean_eng_sentences, train_file, dev_file, args.percent_dev) + + logging.info(f'Creating text and label files for training') + create_text_and_labels(args.data_dir, os.path.join(args.data_dir, 'train.txt')) + create_text_and_labels(args.data_dir, os.path.join(args.data_dir, 'dev.txt')) + + if args.clean_dir: + logging.info(f'Cleaning up {args.data_dir}') + __delete_file(clean_eng_sentences) + __delete_file(tatoeba_dataset) + __delete_file(train_file) + __delete_file(dev_file) + logging.info(f'Processing of the {args.dataset} is complete') diff --git a/models/tacotron2/NeMo/examples/nlp/token_classification/data/import_from_iob_format.py b/models/tacotron2/NeMo/examples/nlp/token_classification/data/import_from_iob_format.py new file mode 100644 index 0000000000..4a6f15442b --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/token_classification/data/import_from_iob_format.py @@ -0,0 +1,124 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os + +from nemo.utils import logging + + +def __convert_data(in_file: str, out_text_f: str, out_labels_f: str, max_length: int): + """ + Convert data from the IOB format to NeMo accepted format described below. + in_file should be in the IOB format, see example here: + https://www.clips.uantwerpen.be/conll2003/ner/. + + Args: + in_file: input file name + out_text_f: output file with text + out_labels_f: output file with labels + max_length: use -1 to leave the examples' length as is, otherwise long examples will be split into multiple + examples + After the conversion, the dataset is split into 2 files: text.txt + and labels.txt. + Each line of the text.txt file contains text sequences, where words + are separated with spaces. The labels.txt file contains corresponding + labels for each word in text.txt, the labels are separated with spaces. + Each line of the files should follow the format: + [WORD] [SPACE] [WORD] [SPACE] [WORD] (for text.txt) and + [LABEL] [SPACE] [LABEL] [SPACE] [LABEL] (for labels.txt). + + """ + in_file = open(in_file, 'r') + + if max_length == -1: + with open(out_text_f, 'w') as out_text, open(out_labels_f, 'w') as out_labels: + for line in in_file: + if line == '\n': + out_text.write(line) + out_labels.write(line) + else: + line = line.split() + out_text.write(line[0] + ' ') + out_labels.write(line[-1] + ' ') + + else: + words = [] + labels = [] + with open(out_text_f, 'w') as out_text, open(out_labels_f, 'w') as out_labels: + lines = in_file.readlines() + for line_id, line in enumerate(lines): + logging.info(f"{line_id} {len(lines)}") + contends = line.strip() + if len(contends) == 0: + assert len(words) == len(labels) + if len(words) > max_length: + # split if the sentence is longer than max_length + while len(words) > max_length: + tmplabel = labels[:max_length] + for iidx in range(len(tmplabel)): + if tmplabel.pop() == 'O': + break + l = ' '.join([label for label in labels[: len(tmplabel) + 1] if len(label) > 0]) + w = ' '.join([word for word in words[: len(tmplabel) + 1] if len(word) > 0]) + + out_text.write(w + "\n") + out_labels.write(l + "\n") + words = words[len(tmplabel) + 1 :] + labels = labels[len(tmplabel) + 1 :] + + if len(words) == 0: + continue + l = ' '.join([label for label in labels if len(label) > 0]) + w = ' '.join([word for word in words if len(word) > 0]) + + out_text.write(w + "\n") + out_labels.write(l + "\n") + words = [] + labels = [] + continue + + word = line.strip().split()[0] + label = line.strip().split()[-1] + words.append(word) + labels.append(label) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='Convert data from IOB format to the format compatible with \ + nlp/examples/token_classification/scripts/token_classification_train.py and \ + token_classification_evaluate.py' + ) + parser.add_argument("--data_file", required=True, type=str, help='path to a file in IOB format') + parser.add_argument( + "--max_length", + default=-1, + type=int, + help='use -1 to leave the examples\'s length as is, ' + 'otherwise long examples will be split into multiple examples', + ) + args = parser.parse_args() + + data_dir, basename = os.path.split(args.data_file) + prefix = os.path.splitext(basename)[0] + if not os.path.exists(args.data_file): + raise FileNotFoundError(f"{args.data_file} not found") + + logging.info(f'Processing {args.data_file}') + out_text = os.path.join(data_dir, 'text_' + prefix + '.txt') + out_labels = os.path.join(data_dir, 'labels_' + prefix + '.txt') + + __convert_data(args.data_file, out_text, out_labels, args.max_length) + logging.info(f'Processing of the {args.data_file} is complete') diff --git a/models/tacotron2/NeMo/examples/nlp/token_classification/data/prepare_data_for_punctuation_capitalization.py b/models/tacotron2/NeMo/examples/nlp/token_classification/data/prepare_data_for_punctuation_capitalization.py new file mode 100644 index 0000000000..f31e6fc391 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/token_classification/data/prepare_data_for_punctuation_capitalization.py @@ -0,0 +1,99 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The script converts raw text to the NeMo format for punctuation and capitalization task. + +Raw Data Format +--------------- + +The Punctuation and Capitalization model can work with any text dataset, although it is recommended to balance the data, especially for the punctuation task. +Before pre-processing the data to the format expected by the model, the data should be split into train.txt and dev.txt (and optionally test.txt). +Each line in the **train.txt/dev.txt/test.txt** should represent one or more full and/or truncated sentences. + +Example of the train.txt/dev.txt file: + When is the next flight to New York? + The next flight is ... + .... + + +The `source_data_dir` structure should look like this: + . + |--sourced_data_dir + |-- dev.txt + |-- train.txt + + + +NeMo Data Format for training the model +--------------------------------------- + +The punctuation and capitalization model expects the data in the following format: + +The training and evaluation data is divided into 2 files: text.txt and labels.txt. \ +Each line of the **text.txt** file contains text sequences, where words are separated with spaces, i.e. + +[WORD] [SPACE] [WORD] [SPACE] [WORD], for example: + when is the next flight to new york + the next flight is ... + ... + +The **labels.txt** file contains corresponding labels for each word in text.txt, the labels are separated with spaces. \ +Each label in labels.txt file consists of 2 symbols: + +* the first symbol of the label indicates what punctuation mark should follow the word (where O means no punctuation needed); +* the second symbol determines if a word needs to be capitalized or not (where U indicates that the word should be upper cased, and O - no capitalization needed.) + +By default the following punctuation marks are considered: commas, periods, and question marks; the rest punctuation marks were removed from the data. +This can be changed by introducing new labels in the labels.txt files + +Each line of the labels.txt should follow the format: [LABEL] [SPACE] [LABEL] [SPACE] [LABEL] (for labels.txt). \ +For example, labels for the above text.txt file should be: + + OU OO OO OO OO OO OU ?U + OU OO OO OO ... + ... + +The complete list of all possible labels for this task used in this tutorial is: OO, ,O, .O, ?O, OU, ,U, .U, ?U. + +Converting Raw data to NeMo format +---------------------------------- + +To pre-process the raw text data, stored under :code:`sourced_data_dir` (see the :ref:`raw_data_format_punct` +section), run the following command: + + python examples/nlp/token_classification/data/prepare_data_for_punctuation_capitalization.py \ + -s + -o + +""" + +import argparse +import os + +from get_tatoeba_data import create_text_and_labels + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Prepare data for punctuation and capitalization tasks') + parser.add_argument("-s", "--source_file", required=True, type=str, help="Path to the source file") + parser.add_argument("-o", "--output_dir", required=True, type=str, help="Path to the output directory") + args = parser.parse_args() + + if not os.path.exists(args.source_file): + raise ValueError(f'{args.source_file} was not found') + + os.makedirs(args.output_dir, exist_ok=True) + create_text_and_labels(args.output_dir, args.source_file) + + print(f'Processing of the {args.source_file} is complete') diff --git a/models/tacotron2/NeMo/examples/nlp/token_classification/punctuation_capitalization_evaluate.py b/models/tacotron2/NeMo/examples/nlp/token_classification/punctuation_capitalization_evaluate.py new file mode 100644 index 0000000000..b91b332a87 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/token_classification/punctuation_capitalization_evaluate.py @@ -0,0 +1,127 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytorch_lightning as pl +from omegaconf import DictConfig + +from nemo.collections.nlp.models import PunctuationCapitalizationModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +""" +This script shows how to perform evaluation and runs inference of a few examples. + +More details on the task and data format could be found in tutorials/nlp/Punctuation_and_Capitalization.ipynb + +*** Setting the configs *** + +This script uses the `/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml` config file +by default. You may update the config file from the file directly. +The other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'. + +For more details about the config files and different ways of model restoration, see tutorials/00_NeMo_Primer.ipynb + + +*** Model Evaluation *** + + python punctuation_capitalization_evaluate.py \ + model.dataset.data_dir= \ + pretrained_model=punctuation_en_distilbert + + - a directory that contains test_ds.text_file and test_ds.labels_file (see the config) +pretrained_model - pretrained PunctuationCapitalizationModel model from list_available_models() or + path to a .nemo file, for example: punctuation_en_bert or your_model.nemo + +""" + + +@hydra_runner(config_path="conf", config_name="punctuation_capitalization_config") +def main(cfg: DictConfig) -> None: + logging.info( + 'During evaluation/testing, it is currently advisable to construct a new Trainer with single GPU and \ + no DDP to obtain accurate results' + ) + + if not hasattr(cfg.model, 'test_ds'): + raise ValueError(f'model.test_ds was not found in the config, skipping evaluation') + else: + gpu = 1 if cfg.trainer.gpus != 0 else 0 + + trainer = pl.Trainer( + gpus=gpu, + precision=cfg.trainer.precision, + amp_level=cfg.trainer.amp_level, + logger=False, + checkpoint_callback=False, + ) + exp_dir = exp_manager(trainer, cfg.exp_manager) + + if not cfg.pretrained_model: + raise ValueError( + 'To run evaluation and inference script a pre-trained model or .nemo file must be provided.' + f'Choose from {PunctuationCapitalizationModel.list_available_models()} or "pretrained_model"="your_model.nemo"' + ) + + if os.path.exists(cfg.pretrained_model): + model = PunctuationCapitalizationModel.restore_from(cfg.pretrained_model) + elif cfg.pretrained_model in PunctuationCapitalizationModel.get_available_model_names(): + model = PunctuationCapitalizationModel.from_pretrained(cfg.pretrained_model) + else: + raise ValueError( + f'Provide path to the pre-trained .nemo file or choose from {PunctuationCapitalizationModel.list_available_models()}' + ) + + data_dir = cfg.model.dataset.get('data_dir', None) + + if data_dir is None: + logging.error( + 'No dataset directory provided. Skipping evaluation. ' + 'To run evaluation on a file, specify path to the directory that contains test_ds.text_file and test_ds.labels_file with "model.dataset.data_dir" argument.' + ) + elif not os.path.exists(data_dir): + logging.error(f'{data_dir} is not found, skipping evaluation on the test set.') + else: + model.update_data_dir(data_dir=data_dir) + model._cfg.dataset = cfg.model.dataset + + if not hasattr(cfg.model, 'test_ds'): + logging.error(f'model.test_ds was not found in the config, skipping evaluation') + elif model.prepare_test(trainer): + model.setup_test_data(cfg.model.test_ds) + trainer.test(model) + else: + logging.error('Skipping the evaluation. The trainer is not setup properly.') + + # run an inference on a few examples + queries = [ + 'we bought four shirts one pen and a mug from the nvidia gear store in santa clara', + 'what can i do for you today', + 'how are you', + ] + + inference_results = model.add_punctuation_capitalization(queries, batch_size=len(queries), max_seq_length=512) + + for query, result in zip(queries, inference_results): + logging.info(f'Query : {query}') + logging.info(f'Result: {result.strip()}\n') + + logging.info(f'Results are saved at {exp_dir}') + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/token_classification/punctuation_capitalization_train.py b/models/tacotron2/NeMo/examples/nlp/token_classification/punctuation_capitalization_train.py new file mode 100644 index 0000000000..ae55448d3d --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/token_classification/punctuation_capitalization_train.py @@ -0,0 +1,101 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models import PunctuationCapitalizationModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +""" +This script show how to train a Punctuation and Capitalization Model. +More details on the task and data format could be found in tutorials/nlp/Punctuation_and_Capitalization.ipynb + +*** Setting the configs *** + +The model and the PT trainer are defined in a config file which declares multiple important sections. +The most important ones are: + model: All arguments that are related to the Model - language model, tokenizer, token classifier, optimizer, + schedulers, and datasets/data loaders. + trainer: Any argument to be passed to PyTorch Lightning including number of epochs, number of GPUs, + precision level, etc. +This script uses the `/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml` config file +by default. You may update the config file from the file directly. +The other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'. + +For more details about the config files and different ways of model restoration, see tutorials/00_NeMo_Primer.ipynb + +*** Model training *** + +To run this script and train the model from scratch, use: + python punctuation_and_capitalization_train.py \ + model.dataset.data_dir= + +To use one of the pretrained versions of the model and finetune it, run: + python punctuation_and_capitalization.py \ + pretrained_model=punctuation_en_bert \ + model.dataset.data_dir= + + - a directory that contains test_ds.text_file and test_ds.labels_file (see the config) + pretrained_model - pretrained PunctuationCapitalization model from list_available_models() or + path to a .nemo file, for example: punctuation_en_bert or model.nemo + +""" + + +@hydra_runner(config_path="conf", config_name="punctuation_capitalization_config") +def main(cfg: DictConfig) -> None: + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + + if not cfg.pretrained_model: + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + model = PunctuationCapitalizationModel(cfg.model, trainer=trainer) + else: + if os.path.exists(cfg.pretrained_model): + model = PunctuationCapitalizationModel.restore_from(cfg.pretrained_model) + elif cfg.pretrained_model in PunctuationCapitalizationModel.get_available_model_names(): + model = PunctuationCapitalizationModel.from_pretrained(cfg.pretrained_model) + else: + raise ValueError( + f'Provide path to the pre-trained .nemo file or choose from {PunctuationCapitalizationModel.list_available_models()}' + ) + + data_dir = cfg.model.dataset.get('data_dir', None) + if data_dir: + if not os.path.exists(data_dir): + raise ValueError(f'{data_dir} is not found at') + + # we can also do finetuning of the pretrained model but we would need to update the data dir + model.update_data_dir(data_dir) + # setup train and validation Pytorch DataLoaders + model.setup_training_data() + model.setup_validation_data() + logging.info(f'Using config file of the pretrained model') + else: + raise ValueError( + 'Specify a valid dataset directory that contains test_ds.text_file and test_ds.labels_file \ + with "model.dataset.data_dir" argument' + ) + + trainer.fit(model) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/token_classification/token_classification_evaluate.py b/models/tacotron2/NeMo/examples/nlp/token_classification/token_classification_evaluate.py new file mode 100644 index 0000000000..4a888f50d7 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/token_classification/token_classification_evaluate.py @@ -0,0 +1,137 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytorch_lightning as pl +from omegaconf import DictConfig + +from nemo.collections.nlp.models import TokenClassificationModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +""" +This script shows how to perform evaluation and runs inference of a few examples. + +More details on Token Classification model could be found in tutorials/nlp/Token_Classification_Named_Entity_Recognition.ipynb + +*** Setting the configs *** + +This script uses the `/examples/nlp/token_classification/conf/token_classification_config.yaml` config file +by default. You may update the config file from the file directly. +The other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'. + +For more details about the config files and different ways of model restoration, see tutorials/00_NeMo_Primer.ipynb + +*** Model Evaluation *** + +The script runs two types of evaluation: + * model.test() - this eval will use the config setting for evaluation such as model.dataset.max_seq_length + * model.evaluate_from_file(): + * disregards model.dataset.max_seq_length and evaluates all the tokens, BERT max seq length - 512 tokens after tokenization + * creates confusion matrix + * saves predictions and labels (if provided) + +To run the script: + + python token_classification_evaluate.py \ + model.dataset.data_dir= \ + pretrained_model=ner_en_bert + + - a directory that contains test_ds.text_file and test_ds.labels_file (see the config) +pretrained_model - pretrained TokenClassification model from list_available_models() or + path to a .nemo file, for example: ner_en_bert or your_model.nemo + +""" + + +@hydra_runner(config_path="conf", config_name="token_classification_config") +def main(cfg: DictConfig) -> None: + logging.info( + 'During evaluation/testing, it is currently advisable to construct a new Trainer with single GPU and \ + no DDP to obtain accurate results' + ) + + if not hasattr(cfg.model, 'test_ds'): + raise ValueError(f'model.test_ds was not found in the config, skipping evaluation') + else: + gpu = 1 if cfg.trainer.gpus != 0 else 0 + + trainer = pl.Trainer( + gpus=gpu, + precision=cfg.trainer.precision, + amp_level=cfg.trainer.amp_level, + logger=False, + checkpoint_callback=False, + ) + exp_dir = exp_manager(trainer, cfg.exp_manager) + + if not cfg.pretrained_model: + raise ValueError( + 'To run evaluation and inference script a pre-trained model or .nemo file must be provided.' + f'Choose from {TokenClassificationModel.list_available_models()} or "pretrained_model"="your_model.nemo"' + ) + + if os.path.exists(cfg.pretrained_model): + model = TokenClassificationModel.restore_from(cfg.pretrained_model) + elif cfg.pretrained_model in TokenClassificationModel.get_available_model_names(): + model = TokenClassificationModel.from_pretrained(cfg.pretrained_model) + else: + raise ValueError( + f'Provide path to the pre-trained .nemo checkpoint or choose from {TokenClassificationModel.list_available_models()}' + ) + + data_dir = cfg.model.dataset.get('data_dir', None) + if data_dir is None: + logging.error( + 'No dataset directory provided. Skipping evaluation. ' + 'To run evaluation on a file, specify path to the directory that contains test_ds.text_file and test_ds.labels_file with "model.dataset.data_dir" argument.' + ) + elif not os.path.exists(data_dir): + logging.error(f'{data_dir} is not found, skipping evaluation on the test set.') + else: + model.update_data_dir(data_dir=data_dir) + model._cfg.dataset = cfg.model.dataset + + if not hasattr(cfg.model, 'test_ds'): + logging.error(f'model.test_ds was not found in the config, skipping evaluation') + elif model.prepare_test(trainer): + model.setup_test_data(cfg.model.test_ds) + trainer.test(model) + + model.evaluate_from_file( + text_file=os.path.join(data_dir, cfg.model.test_ds.text_file), + labels_file=os.path.join(data_dir, cfg.model.test_ds.labels_file), + output_dir=exp_dir, + add_confusion_matrix=True, + normalize_confusion_matrix=True, + ) + else: + logging.error('Skipping the evaluation. The trainer is not setup properly.') + + # run an inference on a few examples + queries = ['we bought four shirts from the nvidia gear store in santa clara.', 'Nvidia is a company.'] + results = model.add_predictions(queries, output_file='predictions.txt') + + for query, result in zip(queries, results): + logging.info(f'Query : {query}') + logging.info(f'Result: {result.strip()}\n') + + logging.info(f'Results are saved at {exp_dir}') + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/nlp/token_classification/token_classification_train.py b/models/tacotron2/NeMo/examples/nlp/token_classification/token_classification_train.py new file mode 100644 index 0000000000..f4ffb61c55 --- /dev/null +++ b/models/tacotron2/NeMo/examples/nlp/token_classification/token_classification_train.py @@ -0,0 +1,147 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models import TokenClassificationModel +from nemo.collections.nlp.parts.nlp_overrides import NLPDDPPlugin +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +""" +This scripts shows how to train a Token Classification model. + +The Token Classification model supports Named Entity Recognition task and other token level classification tasks, +as long as the data follows the format specified below. + +More details on how to use this script could be found in +tutorials/nlp/Token_Classification_Named_Entity_Recognition.ipynb + +*** Data Format *** +Token Classification Model requires the data to be split into 2 files: text.txt and labels.txt. +Each line of the text.txt file contains text sequences, where words are separated with spaces, i.e.: +[WORD] [SPACE] [WORD] [SPACE] [WORD]. +The labels.txt file contains corresponding labels for each word in text.txt, the labels are separated with spaces, i.e.: +[LABEL] [SPACE] [LABEL] [SPACE] [LABEL]. + +Example of a text.txt file: +Jennifer is from New York City . +She likes ... +... + +Corresponding labels.txt file: +B-PER O O B-LOC I-LOC I-LOC O +O O ... +... + +*** Preparing the dataset *** + +To convert an IOB format data to the format required for training, run +examples/nlp/token_classification/data/import_from_iob_format.py on your train and dev files, as follows: + +python examples/nlp/token_classification/data/import_from_iob_format.py --data_file PATH_TO_IOB_FORMAT_DATAFILE + +*** Setting the configs *** + +The model and the PT trainer are defined in a config file which declares multiple important sections. +The most important ones are: + model: All arguments that are related to the Model - language model, tokenizer, token classifier, optimizer, + schedulers, and datasets/data loaders. + trainer: Any argument to be passed to PyTorch Lightning including number of epochs, number of GPUs, + precision level, etc. +This script uses the `/examples/nlp/token_classification/conf/token_classification_config.yaml` config file +by default. You may update the config file from the file directly. +The other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'. + +For more details about the config files and different ways of model restoration, see tutorials/00_NeMo_Primer.ipynb + +*** Model Training *** + +To train TokenClassification model from scratch with the default config file, run: + + python token_classification_train.py \ + model.dataset.data_dir= \ + trainer.max_epochs= \ + trainer.gpus=[] + +To use one of the pretrained versions of the model specify a `pretrained_model` arg with either +TokenClassification model from list_available_models() or path to a .nemo file, for example: +ner_en_bert or model.nemo, run: + + python token_classification_train.py pretrained_model=ner_en_bert + +To use one of the pretrained versions of the model and fine-tune it, run: + + python token_classification_train.py \ + model.dataset.data_dir= \ + pretrained_model=ner_en_bert + + - a directory that contains test_ds.text_file and test_ds.labels_file (see the config) +pretrained_model - pretrained TokenClassification model from list_available_models() or + path to a .nemo file, for example: ner_en_bert or model.nemo + +For more ways of restoring a pre-trained model, see tutorials/00_NeMo_Primer.ipynb +""" + + +@hydra_runner(config_path="conf", config_name="token_classification_config") +def main(cfg: DictConfig) -> None: + trainer = pl.Trainer(plugins=[NLPDDPPlugin()], **cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + + if not cfg.pretrained_model: + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + model = TokenClassificationModel(cfg.model, trainer=trainer) + else: + if os.path.exists(cfg.pretrained_model): + # TODO: can we drop strict=False? + model = TokenClassificationModel.restore_from(cfg.pretrained_model, trainer=trainer, strict=False) + elif cfg.pretrained_model in TokenClassificationModel.get_available_model_names(): + model = TokenClassificationModel.from_pretrained(cfg.pretrained_model) + else: + raise ValueError( + f'Provide path to the pre-trained .nemo file or choose from {TokenClassificationModel.list_available_models()}' + ) + + data_dir = cfg.model.dataset.get('data_dir', None) + if data_dir: + if not os.path.exists(data_dir): + raise ValueError(f'{data_dir} is not found at') + + # we can also do finetuning of the pretrained model but it will require + # setup the data dir to get class weights statistics + model.update_data_dir(data_dir=data_dir) + # finally, setup train and validation Pytorch DataLoaders + model.setup_training_data() + model.setup_validation_data() + # then we're setting up loss, use model.dataset.class_balancing, + # if you want to add class weights to the CrossEntropyLoss + model.setup_loss(class_balancing=cfg.model.dataset.class_balancing) + logging.info(f'Using config file of the pretrained model') + else: + raise ValueError( + 'Specify a valid dataset directory that contains test_ds.text_file and test_ds.labels_file \ + with "model.dataset.data_dir" argument' + ) + + trainer.fit(model) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/speaker_recognition/README.md b/models/tacotron2/NeMo/examples/speaker_recognition/README.md new file mode 100644 index 0000000000..4e321b3553 --- /dev/null +++ b/models/tacotron2/NeMo/examples/speaker_recognition/README.md @@ -0,0 +1,21 @@ +# Speaker Recognition + +Documentation section for speaker related tasks can be found at: + - [Speaker Recognition and Verification](https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/speaker_recognition/intro.html) + - [Speaker Diarization](https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/speaker_diarization/intro.html) + +This folder contains scripts for performing various speaker related tasks as breiefly explained below: + - **speaker_reco.py** +: Training script for speaker verification and recognition + - **speaker_reco_infer.py** +: Inference for speaker recognition. Can be used to map labels using trained model on test set with known speaker labels from train set. (speaker id) + - **extract_speaker_embeddings.py** +: Extract speaker embeddings using trained model for speaker verification purposes. + - **speaker_reco_finetune.py** +: Sample script to demonstrate on how to finetune a already trained model for specific sub tasks. + - **voxceleb_eval.py** +: Helper script to evaluate speaker embeddings on voxceleb trail files. + - **speaker_diarize.py** +: Inference script for speaker diarization - knowing who spoke when. + - **conf/** +: Folder containing configuration files for training speaker tasks. We currently only support inference for speaker diarization. diff --git a/models/tacotron2/NeMo/examples/speaker_recognition/conf/SpeakerNet_recognition_3x2x512.yaml b/models/tacotron2/NeMo/examples/speaker_recognition/conf/SpeakerNet_recognition_3x2x512.yaml new file mode 100644 index 0000000000..4bba3a780d --- /dev/null +++ b/models/tacotron2/NeMo/examples/speaker_recognition/conf/SpeakerNet_recognition_3x2x512.yaml @@ -0,0 +1,157 @@ +name: &name "SpeakerNet" +sample_rate: &sample_rate 16000 +repeat: &rep 2 +dropout: &drop 0.5 +separable: &separable True +n_filters: &n_filters 512 + +model: + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: null + batch_size: 64 + shuffle: True + time_length: 8 + is_tarred: False + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: null + batch_size: 128 + shuffle: False + time_length: 8 + + test_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: null + batch_size: 1 + shuffle: False + time_length: 8 + embedding_dir: '.' + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.02 + sample_rate: *sample_rate + window_stride: 0.01 + window: "hann" + features: &n_mels 64 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: *n_filters + repeat: 1 + kernel: [3] + stride: [1] + dilation: [1] + dropout: *drop + residual: true + separable: *separable + + - filters: *n_filters + repeat: *rep + kernel: [7] + stride: [1] + dilation: [1] + dropout: *drop + residual: true + separable: *separable + + - filters: *n_filters + repeat: *rep + kernel: [11] + stride: [1] + dilation: [1] + dropout: *drop + residual: true + separable: *separable + + - filters: *n_filters + repeat: *rep + kernel: [15] + stride: [1] + dilation: [1] + dropout: *drop + residual: true + separable: *separable + + - filters: &enc_feat_out 1500 + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: *separable + + decoder: + _target_: nemo.collections.asr.modules.SpeakerDecoder + feat_in: *enc_feat_out + num_classes: 7205 + pool_mode: 'xvector' + emb_sizes: 512,512 + angular: False + + loss: + scale: 30 + margin: 0.2 + + optim: + name: novograd + # _target_: nemo.core.optim.optimizers.Novograd + lr: .008 + # optimizer arguments + args: + name: auto + # _target_: nemo.core.config.optimizers.NovogradParams + betas: [0.95, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + iters_per_batch: 1 # computed at runtime + max_steps: null # computed at runtime or explicitly set here + + # scheduler config override + args: + name: auto + # _target_: nemo.core.config.schedulers.CosineAnnealingParams + warmup_steps: null + warmup_ratio: 0.1 + min_lr: 0.0 + last_epoch: -1 + +trainer: + gpus: 1 # number of gpus + max_epochs: 200 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + amp_level: O0 + deterministic: True + checkpoint_callback: False + logger: False + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/speaker_recognition/conf/SpeakerNet_verification_3x2x512.yaml b/models/tacotron2/NeMo/examples/speaker_recognition/conf/SpeakerNet_verification_3x2x512.yaml new file mode 100644 index 0000000000..9ce23f1f8d --- /dev/null +++ b/models/tacotron2/NeMo/examples/speaker_recognition/conf/SpeakerNet_verification_3x2x512.yaml @@ -0,0 +1,145 @@ +name: &name "SpeakerNet" +sample_rate: &sample_rate 16000 +repeat: &rep 2 +dropout: &drop 0.5 +separable: &separable True +n_filters: &n_filters 512 + +model: + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: null + batch_size: 64 + shuffle: True + time_length: 8 + is_tarred: False + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: null + batch_size: 128 + shuffle: False + time_length: 8 + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.02 + sample_rate: *sample_rate + window_stride: 0.01 + window: "hann" + features: &n_mels 64 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: *n_filters + repeat: 1 + kernel: [3] + stride: [1] + dilation: [1] + dropout: *drop + residual: true + separable: *separable + + - filters: *n_filters + repeat: *rep + kernel: [7] + stride: [1] + dilation: [1] + dropout: *drop + residual: true + separable: *separable + + - filters: *n_filters + repeat: *rep + kernel: [11] + stride: [1] + dilation: [1] + dropout: *drop + residual: true + separable: *separable + + - filters: *n_filters + repeat: *rep + kernel: [15] + stride: [1] + dilation: [1] + dropout: *drop + residual: true + separable: *separable + + - filters: &enc_feat_out 1500 + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: *separable + + decoder: + _target_: nemo.collections.asr.modules.SpeakerDecoder + feat_in: *enc_feat_out + num_classes: 7205 + pool_mode: 'xvector' + emb_sizes: 256 + angular: True + + loss: + scale: 30 + margin: 0.2 + + optim: + name: sgd + lr: .006 + # optimizer arguments + args: + name: auto + weight_decay: 0.001 + momentum: 0.9 + + # scheduler setup + sched: + name: CosineAnnealing + iters_per_batch: 1 # computed at runtime + max_steps: null # computed at runtime or explicitly set here + + # scheduler config override + args: + name: auto + warmup_steps: null + warmup_ratio: 0.1 + min_lr: 0.0 + last_epoch: -1 + +trainer: + gpus: 1 # number of gpus + max_epochs: 200 + max_steps: null # computed at runtime if not set + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + amp_level: O0 + deterministic: True + checkpoint_callback: False + logger: False + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/speaker_recognition/conf/speaker_diarization.yaml b/models/tacotron2/NeMo/examples/speaker_recognition/conf/speaker_diarization.yaml new file mode 100644 index 0000000000..482bf4682a --- /dev/null +++ b/models/tacotron2/NeMo/examples/speaker_recognition/conf/speaker_diarization.yaml @@ -0,0 +1,27 @@ +name: &name "ClusterDiarizer" + +num_workers: 4 +sample_rate: &sample_rate 16000 + +diarizer: + oracle_num_speakers: ??? # oracle number of speakers or pass null if not known. [int or path to file containing uniq-id with num of speakers of that session] + max_num_speakers: 20 # max number of speakers for each recording. If oracle num speakers is passed this value is ignored. + out_dir: ??? + paths2audio_files: ??? #file containing paths to audio files for inference + path2groundtruth_rttm_files: null #file containing paths to ground truth rttm files for score computation or null if no reference rttms are present + + vad: + model_path: null #.nemo local model path or pretrained model name or none + window_length_in_sec: 0.15 + shift_length_in_sec: 0.01 + threshold: 0.7 # tune threshold on dev set. Check /scripts/voice_activity_detection/vad_tune_threshold.py + vad_decision_smoothing: True + smoothing_params: + method: "median" + overlap: 0.875 + + speaker_embeddings: + oracle_vad_manifest: null + model_path: ??? #.nemo local model path or pretrained model name (speakerverification_speakernet or speakerdiarization_speakernet) + window_length_in_sec: 1.5 # window length in sec for speaker embedding extraction + shift_length_in_sec: 0.75 # shift length in sec for speaker embedding extraction diff --git a/models/tacotron2/NeMo/examples/speaker_recognition/extract_speaker_embeddings.py b/models/tacotron2/NeMo/examples/speaker_recognition/extract_speaker_embeddings.py new file mode 100644 index 0000000000..ce7ce0ac5f --- /dev/null +++ b/models/tacotron2/NeMo/examples/speaker_recognition/extract_speaker_embeddings.py @@ -0,0 +1,101 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This is a helper script to extract speaker embeddings based on manifest file +Usage: +python extract_speaker_embeddings.py --manifest=/path/to/manifest/file' +--model_path='/path/to/.nemo/file'(optional) +--embedding_dir='/path/to/embedding/directory' + +Args: +--manifest: path to manifest file containing audio_file paths for which embeddings need to be extracted +--model_path(optional): path to .nemo speaker verification model file to extract embeddings, if not passed SpeakerNet-M model would + be downloaded from NGC and used to extract embeddings +--embeddings_dir(optional): path to directory where embeddings need to stored default:'./' + + +""" + +from argparse import ArgumentParser + +import pytorch_lightning as pl +import torch +from omegaconf import OmegaConf + +from nemo.collections.asr.models.label_models import ExtractSpeakerEmbeddingsModel +from nemo.utils import logging + +try: + from torch.cuda.amp import autocast +except ImportError: + from contextlib import contextmanager + + @contextmanager + def autocast(enabled=None): + yield + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "--manifest", type=str, required=True, help="Path to manifest file", + ) + parser.add_argument( + "--model_path", + type=str, + default='speakerverification_speakernet', + required=False, + help="path to .nemo speaker verification model file to extract embeddings, if not passed SpeakerNet-M model would be downloaded from NGC and used to extract embeddings", + ) + parser.add_argument( + "--embedding_dir", + type=str, + default='./', + required=False, + help="path to directory where embeddings need to stored default:'./'", + ) + args = parser.parse_args() + torch.set_grad_enabled(False) + + if args.model_path.endswith('.nemo'): + logging.info(f"Using local speaker model from {args.model_path}") + speaker_model = ExtractSpeakerEmbeddingsModel.restore_from(restore_path=args.model_path) + else: + speaker_model = ExtractSpeakerEmbeddingsModel.from_pretrained(model_name="speakerverification_speakernet") + logging.info(f"using pretrained speaker verification model from NGC") + + num_gpus = 1 if torch.cuda.is_available() else 0 + if not num_gpus: + logging.warning("Running model on CPU, for faster performance it is adviced to use atleast one NVIDIA GPUs") + + trainer = pl.Trainer(gpus=num_gpus, accelerator=None) + + test_config = OmegaConf.create( + dict( + manifest_filepath=args.manifest, + sample_rate=16000, + labels=None, + batch_size=1, + shuffle=False, + time_length=8, + embedding_dir=args.embedding_dir, + ) + ) + speaker_model.setup_test_data(test_config) + trainer.test(speaker_model) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/speaker_recognition/speaker_diarize.py b/models/tacotron2/NeMo/examples/speaker_recognition/speaker_diarize.py new file mode 100644 index 0000000000..a9013134c7 --- /dev/null +++ b/models/tacotron2/NeMo/examples/speaker_recognition/speaker_diarize.py @@ -0,0 +1,49 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from omegaconf import OmegaConf +from pytorch_lightning import seed_everything + +from nemo.collections.asr.models import ClusteringDiarizer +from nemo.core.config import hydra_runner +from nemo.utils import logging + +""" +This script demonstrates how to use run speaker diarization. +Usage: + python speaker_diarize.py \ + diarizer.paths2audio_files= \ + diarizer.path2groundtruth_rttm_files=<(Optional) either list of rttm file paths or file containing paths to rttm files> \ + diarizer.vad.model_path="vad_telephony_marblenet" \ + diarizer.vad.threshold=0.7 \ + diarizer.speaker_embeddings.model_path="speakerverification_speakernet" \ + diarizer.out_dir="demo" + +Check out whole parameters in ./conf/speaker_diarization.yaml and their meanings. +For details, have a look at /tutorials/speaker_recognition/Speaker_Diarization_Inference.ipynb +""" + +seed_everything(42) + + +@hydra_runner(config_path="conf", config_name="speaker_diarization.yaml") +def main(cfg): + + logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}') + sd_model = ClusteringDiarizer(cfg=cfg) + sd_model.diarize() + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco.py b/models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco.py new file mode 100644 index 0000000000..90bb75388b --- /dev/null +++ b/models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco.py @@ -0,0 +1,73 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytorch_lightning as pl +from omegaconf import ListConfig, OmegaConf +from pytorch_lightning import seed_everything + +from nemo.collections.asr.models import EncDecSpeakerLabelModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + +""" +Basic run (on GPU for 10 epochs for 2 class training): +EXP_NAME=sample_run +python ./speaker_reco.py --config-path='conf' --config-name='SpeakerNet_recognition_3x2x512.yaml' \ + trainer.max_epochs=10 \ + model.train_ds.batch_size=64 model.validation_ds.batch_size=64 \ + model.train_ds.manifest_filepath="" model.validation_ds.manifest_filepath="" \ + model.test_ds.manifest_filepath="" \ + trainer.gpus=1 \ + model.decoder.params.num_classes=2 \ + exp_manager.name=$EXP_NAME +exp_manager.use_datetime_version=False \ + exp_manager.exp_dir='./speaker_exps' + +See https://github.com/NVIDIA/NeMo/blob/main/tutorials/speaker_recognition/Speaker_Recognition_Verification.ipynb for notebook tutorial + +Optional: Use tarred dataset to speech up data loading. + Prepare ONE manifest that contains all training data you would like to include. Validation should use non-tarred dataset. + Note that it's possible that tarred datasets impacts validation scores because it drop values in order to have same amount of files per tarfile; + Scores might be off since some data is missing. + + Use the `convert_to_tarred_audio_dataset.py` script under /speech_recognition/scripts in order to prepare tarred audio dataset. + For details, please see TarredAudioToClassificationLabelDataset in /nemo/collections/asr/data/audio_to_label.py +""" + +seed_everything(42) + + +@hydra_runner(config_path="conf", config_name="SpeakerNet_recognition_3x2x512.yaml") +def main(cfg): + + logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(**cfg.trainer) + log_dir = exp_manager(trainer, cfg.get("exp_manager", None)) + speaker_model = EncDecSpeakerLabelModel(cfg=cfg.model, trainer=trainer) + trainer.fit(speaker_model) + if not trainer.fast_dev_run: + model_path = os.path.join(log_dir, '..', 'spkr.nemo') + speaker_model.save_to(model_path) + + if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: + gpu = 1 if cfg.trainer.gpus != 0 else 0 + trainer = pl.Trainer(gpus=gpu) + if speaker_model.prepare_test(trainer): + trainer.test(speaker_model) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco_finetune.py b/models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco_finetune.py new file mode 100644 index 0000000000..c8ed4d1835 --- /dev/null +++ b/models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco_finetune.py @@ -0,0 +1,98 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script demonstrates how to use pretrained .nemo speaker recognition model to finetune on your domain set +Usage: +python speaker_reco_finetune.py --pretrained_model='/path/to/.nemo_or_.ckpt/file' --finetune_config_file=/path/to/finetune/config/yaml/file' + +in finetune config file make sure to change +- train manifest +- validation manifest +- decoder num classes +- learning rate +- num of epochs + +for finetuning tips see: https://github.com/NVIDIA/NeMo/blob/main/tutorials/speaker_recognition/Speaker_Recognition_Verification.ipynb +""" + + +from argparse import ArgumentParser + +import pytorch_lightning as pl +from omegaconf import OmegaConf + +from nemo.collections.asr.models import EncDecSpeakerLabelModel +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "--pretrained_model", + type=str, + default="speakerverification_speakernet", + required=False, + help="Pass your trained .nemo model", + ) + parser.add_argument( + "--finetune_config_file", + type=str, + required=True, + help="path to speakernet config yaml file to load train, validation dataset and also for trainer parameters", + ) + + parser.add_argument( + "--freeze_encoder", + type=bool, + required=False, + default=True, + help="True if speakernet encoder paramteres needs to be frozen while finetuning", + ) + + args = parser.parse_args() + + if args.pretrained_model.endswith('.nemo'): + logging.info(f"Using local speaker model from {args.pretrained_model}") + speaker_model = EncDecSpeakerLabelModel.restore_from(restore_path=args.pretrained_model) + elif args.pretrained_model.endswith('.ckpt'): + logging.info(f"Using local speaker model from checkpoint {args.pretrained_model}") + speaker_model = EncDecSpeakerLabelModel.load_from_checkpoint(checkpoint_path=args.pretrained_model) + else: + logging.info("Using pretrained speaker recognition model from NGC") + speaker_model = EncDecSpeakerLabelModel.from_pretrained(model_name=args.pretrained_model) + + finetune_config = OmegaConf.load(args.finetune_config_file) + + if 'test_ds' in finetune_config.model and finetune_config.model.test_ds is not None: + finetune_config.model.test_ds = None + logging.warning("Removing test ds") + + speaker_model.setup_finetune_model(finetune_config.model) + finetune_trainer = pl.Trainer(**finetune_config.trainer) + speaker_model.set_trainer(finetune_trainer) + + _ = exp_manager(finetune_trainer, finetune_config.get('exp_manager', None)) + speaker_model.setup_optimization(finetune_config.optim) + + if args.freeze_encoder: + for param in speaker_model.encoder.parameters(): + param.requires_grad = False + + finetune_trainer.fit(speaker_model) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco_infer.py b/models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco_infer.py new file mode 100644 index 0000000000..bc3ceaf642 --- /dev/null +++ b/models/tacotron2/NeMo/examples/speaker_recognition/speaker_reco_infer.py @@ -0,0 +1,139 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script serves two goals: + (1) Demonstrate how to use NeMo Models outside of PytorchLightning + (2) Shows example of batch speaker recognition inference +Usage: +python speaker_reco_infer.py --spkr_model='/path/to/.nemo/file' --train_manifest=/path/to/train/manifest/file' +--test_manifest=/path/to/train/manifest/file' --batch_size=32 +train_manifest file is used to map the labels from which model was trained so it is mandatory to +pass the train manifest file + +for finetuning tips see: https://colab.research.google.com/github/NVIDIA/NeMo/blob/main/tutorials/speaker_recognition/Speaker_Recognition_Verification.ipynb +""" + + +import json +import os +import sys +from argparse import ArgumentParser + +import numpy as np +import torch +from tqdm import tqdm + +from nemo.collections.asr.models import EncDecSpeakerLabelModel +from nemo.utils import logging + +try: + from torch.cuda.amp import autocast +except ImportError: + from contextlib import contextmanager + + @contextmanager + def autocast(enabled=None): + yield + + +can_gpu = torch.cuda.is_available() + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "--spkr_model", + type=str, + default="speakerrecognition_speakernet", + required=True, + help="Pass your trained .nemo model", + ) + parser.add_argument( + "--train_manifest", type=str, required=True, help="path to train manifest file to match labels" + ) + parser.add_argument( + "--test_manifest", type=str, required=True, help="path to test manifest file to perform inference" + ) + parser.add_argument("--batch_size", type=int, default=32) + args = parser.parse_args() + torch.set_grad_enabled(False) + + if args.spkr_model.endswith('.nemo'): + logging.info(f"Using local speaker model from {args.spkr_model}") + speaker_model = EncDecSpeakerLabelModel.restore_from(restore_path=args.spkr_model) + else: + logging.error(f"Please pass a trained .nemo file") + sys.exit() + + labels = [] + with open(args.train_manifest, 'rb') as f: + lines = f.readlines() + for line in lines: + line = line.strip() + item = json.loads(line) + labels.append(item['label']) + + labels_map = sorted(set(labels)) + label2id, id2label = {}, {} + for label_id, label in enumerate(labels_map): + label2id[label] = label_id + id2label[label_id] = label + + speaker_model.setup_test_data( + test_data_layer_params={ + 'sample_rate': 16000, + 'manifest_filepath': args.test_manifest, + 'labels': labels_map, + 'batch_size': args.batch_size, + 'trim_silence': False, + 'shuffle': False, + } + ) + if can_gpu: + speaker_model = speaker_model.cuda() + speaker_model.eval() + + speaker_model.test_dataloader() + all_labels = [] + all_logits = [] + for test_batch in tqdm(speaker_model.test_dataloader()): + if can_gpu: + test_batch = [x.cuda() for x in test_batch] + with autocast(): + audio_signal, audio_signal_len, labels, _ = test_batch + logits, _ = speaker_model.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + + all_logits.extend(logits.cpu().numpy()) + all_labels.extend(labels.cpu().numpy()) + + all_logits, true_labels = np.asarray(all_logits), np.asarray(all_labels) + infer_labels = all_logits.argmax(axis=1) + + out_manifest = os.path.basename(args.test_manifest).split('.')[0] + '_infer.json' + out_manifest = os.path.join(os.path.dirname(args.test_manifest), out_manifest) + with open(args.test_manifest, 'rb') as f1, open(out_manifest, 'w') as f2: + lines = f1.readlines() + for idx, line in enumerate(lines): + line = line.strip() + item = json.loads(line) + item['infer'] = id2label[infer_labels[idx]] + json.dump(item, f2) + f2.write('\n') + + logging.info("Inference labels have been written to {} manifest file".format(out_manifest)) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/examples/speaker_recognition/voxceleb_eval.py b/models/tacotron2/NeMo/examples/speaker_recognition/voxceleb_eval.py new file mode 100644 index 0000000000..cdfd1db033 --- /dev/null +++ b/models/tacotron2/NeMo/examples/speaker_recognition/voxceleb_eval.py @@ -0,0 +1,110 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import pickle as pkl +import sys + +import numpy as np +from scipy.interpolate import interp1d +from scipy.optimize import brentq +from sklearn.metrics import roc_curve +from tqdm import tqdm + + +""" +This script faciliates to get EER % based on cosine-smilarity +for Voxceleb dataset. + +Args: + trial_file str: path to voxceleb trial file + emb : path to pickle file of embeddings dictionary (generated from spkr_get_emb.py) + save_kaldi_emb: if required pass this argument to save kaldi embeddings for KALDI PLDA training later + Note: order of audio files in manifest file should match the embeddings +""" + + +def get_acc(trial_file='', emb='', save_kaldi_emb=False): + + trial_score = open('trial_score.txt', 'w') + dirname = os.path.dirname(trial_file) + emb = pkl.load(open(emb, 'rb')) + trial_embs = [] + keys = [] + all_scores = [] + all_keys = [] + + # for each trials in trial file + with open(trial_file, 'r') as f: + tmp_file = f.readlines() + for line in tqdm(tmp_file): + line = line.strip() + truth, x_speaker, y_speaker = line.split() + + x_speaker = x_speaker.split('/') + x_speaker = '@'.join(x_speaker) + + y_speaker = y_speaker.split('/') + y_speaker = '@'.join(y_speaker) + + X = emb[x_speaker] + Y = emb[y_speaker] + + if save_kaldi_emb and x_speaker not in keys: + keys.append(x_speaker) + trial_embs.extend([X]) + + if save_kaldi_emb and y_speaker not in keys: + keys.append(y_speaker) + trial_embs.extend([Y]) + + score = (X @ Y.T) / (((X @ X.T) * (Y @ Y.T)) ** 0.5) + score = (score + 1) / 2 + + all_scores.append(score) + trial_score.write(str(score) + "\t" + truth) + truth = int(truth) + all_keys.append(truth) + + trial_score.write('\n') + trial_score.close() + + if save_kaldi_emb: + np.save(dirname + '/all_embs_voxceleb.npy', np.asarray(trial_embs)) + np.save(dirname + '/all_ids_voxceleb.npy', np.asarray(keys)) + print("Saved KALDI PLDA related embeddings to {}".format(dirname)) + + return np.asarray(all_scores), np.asarray(all_keys) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--trial_file", help="path to voxceleb trial file", type=str, required=True) + parser.add_argument("--emb", help="path to numpy file of embeddings", type=str, required=True) + parser.add_argument( + "--save_kaldi_emb", + help=":save kaldi embeddings for KALDI PLDA training later", + required=False, + action='store_true', + ) + + args = parser.parse_args() + trial_file, emb, save_kaldi_emb = args.trial_file, args.emb, args.save_kaldi_emb + + y_score, y = get_acc(trial_file=trial_file, emb=emb, save_kaldi_emb=save_kaldi_emb) + fpr, tpr, thresholds = roc_curve(y, y_score, pos_label=1) + + eer = brentq(lambda x: 1.0 - x - interp1d(fpr, tpr)(x), 0.0, 1.0) + sys.stdout.write("{0:.2f}\n".format(eer * 100)) diff --git a/models/tacotron2/NeMo/examples/tts/aligner.py b/models/tacotron2/NeMo/examples/tts/aligner.py new file mode 100644 index 0000000000..e32c0444ca --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/aligner.py @@ -0,0 +1,33 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import AlignerModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path='conf', config_name='aligner') +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get('exp_manager', None)) + model = AlignerModel(cfg=cfg.model, trainer=trainer) + trainer.callbacks.extend([pl.callbacks.LearningRateMonitor(), LogEpochTimeCallback()]) # noqa + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/conf/aligner.yaml b/models/tacotron2/NeMo/examples/tts/conf/aligner.yaml new file mode 100644 index 0000000000..801b5770eb --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/aligner.yaml @@ -0,0 +1,123 @@ +name: Aligner +sample_rate: 22050 + +train_dataset: ??? +validation_datasets: ??? +attn_prior_folder: ??? + +model: + d_char: 512 + n_mels: 80 + bin_start_ratio: 0.3 + + train_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharWithPriorDataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.1 + int_values: false + normalize: false + sample_rate: ${sample_rate} + trim: false + attn_prior_folder: ${attn_prior_folder} + n_window_stride: 256 + vocab: + notation: phonemes + punct: true + spaces: true + stresses: false + add_blank_at: None + pad_with_space: True + dataloader_params: + drop_last: false + shuffle: true + batch_size: 64 + num_workers: 4 + + validation_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharWithPriorDataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: 0.1 + int_values: false + normalize: false + sample_rate: ${sample_rate} + trim: false + attn_prior_folder: ${attn_prior_folder} + n_window_stride: 256 + vocab: + notation: phonemes + punct: true + spaces: true + stresses: false + add_blank_at: None + pad_with_space: True + dataloader_params: + drop_last: false + shuffle: false + batch_size: 64 + num_workers: 1 + + preprocessor: + _target_: "nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor" + dither: 0.0 + features: ${model.n_mels} + frame_splicing: 1 + highfreq: 8000 + log: true + log_zero_guard_type: clamp + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: 1024 + n_window_size: 1024 + window_size: False + n_window_stride: 256 + window_stride: False + normalize: null + pad_to: 1 + pad_value: -11.52 + preemph: null + sample_rate: ${sample_rate} + stft_conv: false + window: hann + + alignment_encoder: + _target_: "nemo.collections.tts.modules.aligner.AlignmentEncoder" + n_mel_channels: ${model.n_mels} + n_text_channels: ${model.d_char} + n_att_channels: ${model.n_mels} + + optim: + name: adam + lr: 1e-3 + weight_decay: 1e-6 + + sched: + name: CosineAnnealing + min_lr: 5e-5 + warmup_ratio: 0.35 + +trainer: + gpus: 1 + max_epochs: ??? + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False + logger: False + gradient_clip_val: 1.0 + flush_logs_every_n_steps: 1000 + log_every_n_steps: 200 + check_val_every_n_epoch: 25 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: False + create_checkpoint_callback: True + checkpoint_callback_params: + monitor: "val_forward_sum_loss" + mode: "min" \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/tts/conf/degli.yaml b/models/tacotron2/NeMo/examples/tts/conf/degli.yaml new file mode 100644 index 0000000000..5327694142 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/degli.yaml @@ -0,0 +1,107 @@ +name: "DeepGriffinLim" +sample_rate: 22050 +n_fft: 1024 +hop_length: 256 +destination: "/tmp" + +model: + train_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.NoisySpecsDataset" + destination: ${destination} + subdir: "/train" + n_fft: ${n_fft} + hop_length: ${hop_length} + num_snr: 1 + dataloader_params: + drop_last: false + shuffle: true + batch_size: 16 + num_workers: 1 + + validation_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.NoisySpecsDataset" + destination: ${destination} + subdir: "/valid" + n_fft: ${n_fft} + hop_length: ${hop_length} + num_snr: 1 + + dataloader_params: + drop_last: false + shuffle: false + batch_size: 16 + num_workers: 1 + + preprocessor: + valid_filelist: none + train_filelist: none + n_fft: ${n_fft} + hop_length: ${hop_length} + num_snr: 1 + destination: ${destination} + + degli: + _target_: nemo.collections.tts.modules.degli.DegliModule + n_fft: ${n_fft} + hop_length: ${hop_length} + depth: 1 + out_all_block: True + tiny: False # overrides all below configurations if true. + layers: 6 + k_x: 3 + k_y: 3 + s_x: 1 + s_y: 2 + widening: 16 + use_bn: False + linear_finalizer: True + convGlu: False + act: relu + act2: selu + glu_bn: True + use_weight_norm: True + + + optim: + name: novograd + lr: 0.01 + weight_decay: 0.001 + + sched: + name: ReduceLROnPlateau + mode: min + factor: 0.6 + patience: 5 + verbose: True + threshold: 0.01 + threshold_mode: 'rel' + cooldown: 0 + min_lr: 0.00001 + eps: 1.e-08 + + train_params: + repeat_training: 2 + repeat_validation: 32 + sampling_rate: ${sample_rate} + +trainer: + gpus: 1 # number of gpus + max_epochs: 200 + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + flush_logs_every_n_steps: 2000 + log_every_n_steps: 200 + check_val_every_n_epoch: 5 + precision: 32 + gradient_clip_val: 4.0 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/tts/conf/ed_mel2spec.yaml b/models/tacotron2/NeMo/examples/tts/conf/ed_mel2spec.yaml new file mode 100755 index 0000000000..d98464e1b9 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/ed_mel2spec.yaml @@ -0,0 +1,103 @@ +name: "EncoderDecoderMelToSpec" +sample_rate: 22050 +n_fft: 1024 +hop_length: 256 +destination: "/tmp" + +model: + train_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.NoisySpecsDataset" + destination: ${destination} + subdir: "/train" + n_fft: ${n_fft} + hop_length: ${hop_length} + num_snr: 1 + dataloader_params: + drop_last: false + shuffle: true + batch_size: 16 + num_workers: 1 + + validation_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.NoisySpecsDataset" + destination: ${destination} + subdir: "/valid" + n_fft: ${n_fft} + hop_length: ${hop_length} + num_snr: 1 + + dataloader_params: + drop_last: false + shuffle: false + batch_size: 16 + num_workers: 1 + + preprocessor: + valid_filelist: none + train_filelist: none + n_fft: ${n_fft} + hop_length: ${hop_length} + num_snr: 1 + destination: ${destination} + + mel2spec: + _target_: nemo.collections.tts.modules.ed_mel2spec.EDMel2SpecModule + n_fft: ${n_fft} + hop_length: ${hop_length} + mel_fmin: 0 + mel_fmax: null + mel_freq: 80 + layers: 8 + sampling_rate: ${sample_rate} + widening: 16 + use_batchnorm: True + droprate: 0.2 + num_dropout: 3 + pre_final_lin: True + act1: lrelu + act2: relu + use_weight_norm: False + + optim: + name: adam + lr: 5.e-4 + weight_decay: 1.e-3 + + sched: + name: ReduceLROnPlateau + mode: min + factor: 0.8 + patience: 5 + verbose: True + threshold: 0.01 + threshold_mode: 'rel' + cooldown: 0 + min_lr: 5.e-5 + eps: 1.e-08 + + train_params: + lreg_factor: 1.0 + loss_mode: 3 + validate_scores: True # Slow! + +trainer: + gpus: 1 # number of gpus + max_epochs: 200 + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + flush_logs_every_n_steps: 2000 + log_every_n_steps: 200 + check_val_every_n_epoch: 10 + precision: 32 + gradient_clip_val: 4.0 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/tts/conf/fastpitch.yaml b/models/tacotron2/NeMo/examples/tts/conf/fastpitch.yaml new file mode 100644 index 0000000000..8336d7e502 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/fastpitch.yaml @@ -0,0 +1,144 @@ +name: "FastPitch" +sample_rate: 22050 + +labels: [' ', '!', "'", '(', ')', ',', '-', '.', ':', ';', '?', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'] +train_dataset: ??? +validation_datasets: ??? +test_datasets: null + +model: + labels: ${labels} + n_speakers: 1 + symbols_embedding_dim: 384 + max_token_duration: 75 + n_mel_channels: 80 + pitch_embedding_kernel_size: 3 + + train_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.FastPitchDataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.1 + trim: false + int_values: false + sample_rate: ${sample_rate} + dataloader_params: + drop_last: false + shuffle: true + batch_size: 48 + num_workers: 12 + + + validation_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.FastPitchDataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: null + int_values: false + sample_rate: ${sample_rate} + trim: false + dataloader_params: + drop_last: false + shuffle: false + batch_size: 48 + num_workers: 8 + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + dither: 0.0 + features: ${model.n_mel_channels} + frame_splicing: 1 + highfreq: 8000 + log: true + log_zero_guard_type: add + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: 1024 + n_window_size: 1024 + n_window_stride: 256 + normalize: null + pad_to: 1 + pad_value: 0 + preemph: null + sample_rate: ${sample_rate} + window: hann + window_size: null + window_stride: null + + input_fft: + _target_: nemo.collections.tts.modules.transformer.FFTransformerEncoder + n_layer: 6 + n_head: 1 + d_model: ${model.symbols_embedding_dim} + d_head: 64 + d_inner: 1536 + kernel_size: 3 + dropout: 0.1 + dropatt: 0.1 + dropemb: 0.0 + n_embed: 148 # NOTE Should match # of tokens in `symbol_set` + d_embed: ${model.symbols_embedding_dim} + padding_idx: 0 + + output_fft: + _target_: nemo.collections.tts.modules.transformer.FFTransformerDecoder + n_layer: 6 + n_head: 1 + d_model: ${model.symbols_embedding_dim} + d_head: 64 + d_inner: 1536 + kernel_size: 3 + dropout: 0.1 + dropatt: 0.1 + dropemb: 0.0 + + duration_predictor: + _target_: nemo.collections.tts.modules.fastpitch.TemporalPredictor + input_size: ${model.symbols_embedding_dim} + kernel_size: 3 + filter_size: 256 + dropout: 0.1 + n_layers: 2 + + pitch_predictor: + _target_: nemo.collections.tts.modules.fastpitch.TemporalPredictor + input_size: ${model.symbols_embedding_dim} + kernel_size: 3 + filter_size: 256 + dropout: 0.1 + n_layers: 2 + + optim: + name: lamb + lr: 1e-1 + betas: [0.9, 0.98] + weight_decay: 1e-6 + + sched: + name: NoamAnnealing + warmup_steps: 1000 + last_epoch: -1 + d_model: 1 # Disable scaling based on model dim + +trainer: + gpus: -1 # number of gpus + max_epochs: 1500 + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + gradient_clip_val: 1000.0 + flush_logs_every_n_steps: 1000 + log_every_n_steps: 100 + check_val_every_n_epoch: 5 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/tts/conf/fastpitch_align.yaml b/models/tacotron2/NeMo/examples/tts/conf/fastpitch_align.yaml new file mode 100644 index 0000000000..4b26618b9b --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/fastpitch_align.yaml @@ -0,0 +1,185 @@ +name: "FastPitch" +sample_rate: 22050 +train_dataset: ??? +validation_datasets: ??? +prior_folder: ??? + +model: + learn_alignment: true + n_speakers: 1 + symbols_embedding_dim: 384 + max_token_duration: 75 + n_mel_channels: 80 + pitch_embedding_kernel_size: 3 + n_window_size: 1024 + n_window_stride: 256 + pitch_fmin: 80 + pitch_fmax: 640 + pitch_avg: 211.27540199742586 + pitch_std: 52.1851002822779 + + train_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharWithPriorAndPitchDataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.1 + int_values: false + normalize: true + sample_rate: ${sample_rate} + trim: false + sup_data_path: ${prior_folder} + n_window_stride: ${model.n_window_stride} + n_window_size: ${model.n_window_size} + pitch_fmin: ${model.pitch_fmin} + pitch_fmax: ${model.pitch_fmax} + pitch_avg: ${model.pitch_avg} + pitch_std: ${model.pitch_std} + vocab: + notation: phonemes + punct: true + spaces: true + stresses: true + add_blank_at: None + pad_with_space: True + chars: true + improved_version_g2p: true + dataloader_params: + drop_last: false + shuffle: true + batch_size: 32 + num_workers: 12 + + + validation_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharWithPriorAndPitchDataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: null + int_values: false + normalize: true + sample_rate: ${sample_rate} + trim: false + sup_data_path: ${prior_folder} + n_window_stride: ${model.n_window_stride} + n_window_size: ${model.n_window_size} + pitch_fmin: ${model.pitch_fmin} + pitch_fmax: ${model.pitch_fmax} + pitch_avg: ${model.pitch_avg} + pitch_std: ${model.pitch_std} + vocab: + notation: phonemes + punct: true + spaces: true + stresses: true + add_blank_at: None + pad_with_space: True + chars: true + improved_version_g2p: true + dataloader_params: + drop_last: false + shuffle: false + batch_size: 32 + num_workers: 8 + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + dither: 0.0 + features: ${model.n_mel_channels} + frame_splicing: 1 + highfreq: null + log: true + log_zero_guard_type: add + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: ${model.n_window_size} + n_window_size: ${model.n_window_size} + n_window_stride: ${model.n_window_stride} + normalize: null + pad_to: 1 + pad_value: 0 + preemph: null + sample_rate: ${sample_rate} + window: hann + window_size: null + window_stride: null + + input_fft: #n_embed and padding_idx are added by the model + _target_: nemo.collections.tts.modules.transformer.FFTransformerEncoder + n_layer: 6 + n_head: 1 + d_model: ${model.symbols_embedding_dim} + d_head: 64 + d_inner: 1536 + kernel_size: 3 + dropout: 0.1 + dropatt: 0.1 + dropemb: 0.0 + d_embed: ${model.symbols_embedding_dim} + + output_fft: + _target_: nemo.collections.tts.modules.transformer.FFTransformerDecoder + n_layer: 6 + n_head: 1 + d_model: ${model.symbols_embedding_dim} + d_head: 64 + d_inner: 1536 + kernel_size: 3 + dropout: 0.1 + dropatt: 0.1 + dropemb: 0.0 + + alignment_module: + _target_: nemo.collections.tts.modules.aligner.AlignmentEncoder + n_text_channels: ${model.symbols_embedding_dim} + + duration_predictor: + _target_: nemo.collections.tts.modules.fastpitch.TemporalPredictor + input_size: ${model.symbols_embedding_dim} + kernel_size: 3 + filter_size: 256 + dropout: 0.1 + n_layers: 2 + + pitch_predictor: + _target_: nemo.collections.tts.modules.fastpitch.TemporalPredictor + input_size: ${model.symbols_embedding_dim} + kernel_size: 3 + filter_size: 256 + dropout: 0.1 + n_layers: 2 + + optim: + name: lamb + lr: 1e-1 + betas: [0.9, 0.98] + weight_decay: 1e-6 + + sched: + name: NoamAnnealing + warmup_steps: 1000 + last_epoch: -1 + d_model: 1 # Disable scaling based on model dim + +trainer: + gpus: -1 # number of gpus + max_epochs: 1500 + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + gradient_clip_val: 1000.0 + flush_logs_every_n_steps: 1000 + log_every_n_steps: 100 + check_val_every_n_epoch: 5 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True + checkpoint_callback_params: + monitor: v_loss diff --git a/models/tacotron2/NeMo/examples/tts/conf/fastpitch_hifigan_e2e.yaml b/models/tacotron2/NeMo/examples/tts/conf/fastpitch_hifigan_e2e.yaml new file mode 100644 index 0000000000..c0570ee031 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/fastpitch_hifigan_e2e.yaml @@ -0,0 +1,138 @@ +name: "FastPitch_HiFiGAN_E2E" + +labels: [' ', '!', "'", '(', ')', ',', '-', '.', ':', ';', '?', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'] +train_dataset: ??? +validation_datasets: ??? +test_datasets: null + +model: + sample_rate: 22050 + splice_length: 64 + lr: 3e-2 + labels: ${labels} + n_speakers: 1 + symbols_embedding_dim: 384 + max_token_duration: 75 + n_mel_channels: 80 + pitch_embedding_kernel_size: 3 + mel_loss_coeff: 40 + hop_size: 256 + + train_ds: + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.1 + sample_rate: ${model.sample_rate} + trim: false + parser: null + drop_last: true + shuffle: true + batch_size: 64 + num_workers: 12 + + validation_ds: + manifest_filepath: ${validation_datasets} + sample_rate: ${model.sample_rate} + trim: false + parser: null + drop_last: false + shuffle: false + batch_size: 64 + num_workers: 8 + + preprocessor: + _target_: nemo.collections.asr.parts.preprocessing.features.FilterbankFeatures + dither: 0.0 + nfilt: ${model.n_mel_channels} + frame_splicing: 1 + highfreq: 8000 + log: true + log_zero_guard_type: clamp + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: 1024 + n_window_size: 1024 + n_window_stride: ${model.hop_size} + normalize: null + pad_to: 1 + pad_value: 0 + preemph: null + sample_rate: ${model.sample_rate} + window: hann + exact_pad: true + use_grads: false + + input_fft: + _target_: nemo.collections.tts.modules.transformer.FFTransformerEncoder + n_layer: 6 + n_head: 1 + d_model: ${model.symbols_embedding_dim} + d_head: 64 + d_inner: 1536 + kernel_size: 3 + dropout: 0.1 + dropatt: 0.1 + dropemb: 0.0 + n_embed: 148 # NOTE Should match # of tokens in `symbol_set` + d_embed: ${model.symbols_embedding_dim} + padding_idx: 0 + + output_fft: + _target_: nemo.collections.tts.modules.transformer.FFTransformerDecoder + n_layer: 6 + n_head: 1 + d_model: ${model.symbols_embedding_dim} + d_head: 64 + d_inner: 1536 + kernel_size: 3 + dropout: 0.1 + dropatt: 0.1 + dropemb: 0.0 + + duration_predictor: + _target_: nemo.collections.tts.modules.fastpitch.TemporalPredictor + input_size: ${model.symbols_embedding_dim} + kernel_size: 3 + filter_size: 256 + dropout: 0.1 + n_layers: 2 + + pitch_predictor: + _target_: nemo.collections.tts.modules.fastpitch.TemporalPredictor + input_size: ${model.symbols_embedding_dim} + kernel_size: 3 + filter_size: 256 + dropout: 0.1 + n_layers: 2 + + generator: + _target_: nemo.collections.tts.modules.hifigan_modules.Generator + upsample_kernel_sizes: [16,16,4,4] + upsample_rates: [8,8,2,2] + upsample_initial_channel: 512 + resblock_kernel_sizes: [3,7,11] + resblock_dilation_sizes: [[1,3,5], [1,3,5], [1,3,5]] + resblock: 1 + initial_input_size: 384 + +trainer: + gpus: -1 # number of gpus + max_epochs: 1500 + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + gradient_clip_val: 1000.0 + flush_logs_every_n_steps: 1000 + log_every_n_steps: 100 + check_val_every_n_epoch: 5 + precision: 16 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/tts/conf/fastspeech2.yaml b/models/tacotron2/NeMo/examples/tts/conf/fastspeech2.yaml new file mode 100644 index 0000000000..ed1dea2373 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/fastspeech2.yaml @@ -0,0 +1,144 @@ +name: FastSpeech2 +sample_rate: 22050 +n_mels: 80 + +# Dataset info +train_dataset: ??? +validation_datasets: ??? +mappings_file: ??? # JSON file with word->phone and phone->idx mappings +ignore_file: ??? + +model: + add_pitch_predictor: True + add_energy_predictor: True + d_model: 256 + duration_coeff: 0.25 + mappings_filepath: ${mappings_file} + + train_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.FastSpeech2Dataset" + manifest_filepath: ${train_dataset} + mappings_filepath: ${mappings_file} + sample_rate: ${sample_rate} + max_duration: null + min_duration: 0.1 + ignore_file: ${ignore_file} + trim: false + dataloader_params: + drop_last: false + shuffle: true + batch_size: 64 + num_workers: 4 + + validation_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.FastSpeech2Dataset" + manifest_filepath: ${validation_datasets} + mappings_filepath: ${mappings_file} + sample_rate: ${sample_rate} + ignore_file: ${ignore_file} + trim: false + load_supplementary_values: false + dataloader_params: + drop_last: false + shuffle: false + batch_size: 64 + num_workers: 4 + + preprocessor: + _target_: nemo.collections.asr.parts.preprocessing.features.FilterbankFeatures + dither: 0.0 + nfilt: ${n_mels} + frame_splicing: 1 + highfreq: 8000 + log: true + log_zero_guard_type: clamp + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: 1024 + n_window_size: 1024 + n_window_stride: 256 + normalize: null + pad_to: 16 + pad_value: -11.52 + preemph: null + sample_rate: ${sample_rate} + window: hann + + # FFTransformer encoder + encoder: + _target_: nemo.collections.tts.modules.fastspeech2.FastSpeech2Encoder + d_model: ${model.d_model} + n_layers: 4 + n_attn_heads: 2 + d_attn_head: 256 + d_inner: 1024 + kernel_size: 9 + dropout: 0.1 + attn_dropout: 0.1 + n_embed: 74 # Should match number of tokens in symbol set +1 (pad_idx) + padding_idx: 73 + + # FFTransformer mel spec decoder + decoder: + _target_: nemo.collections.tts.modules.fastspeech2.MelSpecDecoder + d_model: ${model.d_model} + d_out: ${n_mels} + n_layers: 4 + n_attn_heads: 2 + d_attn_head: 256 + d_inner: 1024 + kernel_size: 9 + dropout: 0.1 + attn_dropout: 0.1 + + # VarianceAdaptor + variance_adaptor: + _target_: nemo.collections.tts.modules.fastspeech2.VarianceAdaptor + d_model: ${model.d_model} + dropout: 0.2 + dur_d_hidden: 256 + dur_kernel_size: 3 + pitch: ${model.add_pitch_predictor} + log_pitch: True + n_f0_bins: 256 + pitch_kernel_size: 3 + pitch_min: 80.0 + pitch_max: 800.0 + energy: ${model.add_energy_predictor} + n_energy_bins: 256 + energy_kernel_size: 3 + energy_min: 0.0 + energy_max: 600.0 + + optim: + name: adam + betas: [0.9,0.98] + lr: 1e-3 + weight_decay: 1e-6 + # scheduler setup + sched: + name: NoamAnnealing + warmup_steps: 4000 + min_lr: 1e-5 + d_model: ${model.d_model} + +trainer: + gpus: 1 # number of gpus + max_epochs: ??? + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + flush_logs_every_n_steps: 1000 + log_every_n_steps: 200 + check_val_every_n_epoch: 25 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/tts/conf/fastspeech2_hifigan_e2e.yaml b/models/tacotron2/NeMo/examples/tts/conf/fastspeech2_hifigan_e2e.yaml new file mode 100644 index 0000000000..95acf3d769 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/fastspeech2_hifigan_e2e.yaml @@ -0,0 +1,136 @@ +name: FastSpeech2_HiFiGAN_E2E + +# Dataset info +train_dataset: ??? +validation_datasets: ??? +mappings_file: ??? # JSON file with word->phone and phone->idx mappings +ignore_file: ??? + + +model: + lr: 0.5 + splice_length: 64 # Length in frames + mel_loss_coeff: 45. + pitch_loss_coeff: 0.003 + energy_loss_coeff: 0.03 + add_energy_predictor: True + add_pitch_predictor: True + mappings_filepath: ${mappings_file} + sample_rate: 22050 + n_mels: 80 + hop_size: 256 + + train_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.FastSpeech2Dataset" + manifest_filepath: ${train_dataset} + mappings_filepath: ${mappings_file} + sample_rate: ${model.sample_rate} + max_duration: null + min_duration: 0.75 + ignore_file: ${ignore_file} + trim: false + dataloader_params: + drop_last: false + shuffle: true + batch_size: 64 + num_workers: 4 + + validation_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.FastSpeech2Dataset" + manifest_filepath: ${validation_datasets} + mappings_filepath: ${mappings_file} + sample_rate: ${model.sample_rate} + ignore_file: ${ignore_file} + trim: false + load_supplementary_values: false + dataloader_params: + drop_last: false + shuffle: false + batch_size: 64 + num_workers: 4 + + preprocessor: + _target_: nemo.collections.asr.parts.preprocessing.features.FilterbankFeatures + dither: 0.0 + nfilt: ${n_mels} + frame_splicing: 1 + highfreq: 8000 + log: true + log_zero_guard_type: clamp + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: 1024 + n_window_size: 1024 + n_window_stride: ${model.hop_size} + normalize: null + pad_to: 0 + preemph: null + sample_rate: ${model.sample_rate} + window: hann + + # FFTransformer encoder + encoder: + _target_: nemo.collections.tts.modules.fastspeech2.FastSpeech2Encoder + d_model: 256 + n_layers: 4 + n_attn_heads: 2 + d_attn_head: 256 + d_inner: 1024 + kernel_size: 9 + dropout: 0.1 + attn_dropout: 0.1 + n_embed: 74 # Should match number of tokens in symbol set +1 (pad_idx) + padding_idx: 73 + + # VarianceAdaptor + variance_adaptor: + _target_: nemo.collections.tts.modules.fastspeech2.VarianceAdaptor + d_model: 256 + dropout: 0.2 + dur_d_hidden: 256 + dur_kernel_size: 3 + + pitch: ${model.add_pitch_predictor} + log_pitch: True + n_f0_bins: 256 + pitch_kernel_size: 3 + pitch_min: 80.0 + pitch_max: 800.0 + energy: ${model.add_energy_predictor} + n_energy_bins: 256 + energy_kernel_size: 3 + energy_min: 0.0 + energy_max: 600.0 + + generator: + _target_: nemo.collections.tts.modules.hifigan_modules.Generator + upsample_kernel_sizes: [16,16,4,4] + upsample_rates: [8,8,2,2] + upsample_initial_channel: 512 + resblock_kernel_sizes: [3,7,11] + resblock_dilation_sizes: [[1,3,5], [1,3,5], [1,3,5]] + resblock: 1 + initial_input_size: 256 + apply_weight_init_conv_pre: true + +trainer: + gpus: 1 # number of gpus + max_epochs: ??? + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 200 + check_val_every_n_epoch: 25 + move_metrics_to_cpu: False + precision: 16 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/tts/conf/glow_tts.yaml b/models/tacotron2/NeMo/examples/tts/conf/glow_tts.yaml new file mode 100644 index 0000000000..f082d9f1f4 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/glow_tts.yaml @@ -0,0 +1,143 @@ +name: "GlowTTS" +sample_rate: 22050 +n_fft: 1024 +n_mels: 80 +fmax: 8000 +pad_value: -11.52 +gin_channels: 0 + +train_dataset: ??? +validation_datasets: ??? +test_datasets: null +cmu_dict_path: null + +model: + + n_speakers: 1 + gin_channels: ${gin_channels} + + train_ds: + manifest_filepath: ${train_dataset} + sample_rate: ${sample_rate} + batch_size: 32 + max_duration: null + min_duration: 0.1 + trim: false + shuffle: True + num_workers: 8 + cmu_dict_path: ${cmu_dict_path} + + + validation_ds: + manifest_filepath: ${validation_datasets} + sample_rate: ${sample_rate} + batch_size: 8 + max_duration: null + min_duration: 0.1 + trim: false + shuffle: False + num_workers: 8 + cmu_dict_path: ${cmu_dict_path} + + test_ds: + manifest_filepath: ${test_datasets} + sample_rate: ${sample_rate} + batch_size: 1 + max_duration: null + min_duration: 0.1 + trim: false + shuffle: False + num_workers: 8 + cmu_dict_path: ${cmu_dict_path} + + parser: + _target_: nemo.collections.tts.modules.glow_tts_parser.GlowTTSParser + cmu_dict_path: ${cmu_dict_path} + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + dither: 0.0 + features: ${n_mels} + frame_splicing: 1 + highfreq: ${fmax} + log: true + log_zero_guard_type: add + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: ${n_fft} + n_window_size: 1024 + n_window_stride: 256 + normalize: null + pad_to: 16 + pad_value: ${pad_value} + preemph: null + sample_rate: ${sample_rate} + window: hann + window_size: null + window_stride: null + + encoder: + _target_: nemo.collections.tts.modules.glow_tts.TextEncoder + n_vocab: 148 + out_channels: ${n_mels} + hidden_channels: 192 + filter_channels: 768 + filter_channels_dp: 256 + kernel_size: 3 + p_dropout: 0.1 + n_layers: 6 + n_heads: 2 + window_size: 4 + prenet: true + mean_only: true + gin_channels: ${gin_channels} + + decoder: + _target_: nemo.collections.tts.modules.glow_tts.FlowSpecDecoder + in_channels: ${n_mels} + hidden_channels: 192 + kernel_size: 5 + n_blocks: 12 + n_layers: 4 + n_sqz: 2 + n_split: 4 + sigmoid_scale: false + p_dropout: 0.05 + dilation_rate: 1 + gin_channels: ${gin_channels} + + optim: + name: adam + lr: 1e-3 + # optimizer arguments + betas: [0.9, 0.98] + weight_decay: 0 + + # scheduler setup + sched: + name: CosineAnnealing + + # Scheduler params + warmup_steps: 3000 + min_lr: 1e-4 + last_epoch: -1 + +trainer: + gpus: -1 # number of gpus + max_epochs: 350 + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + gradient_clip_val: 1.0 + flush_logs_every_n_steps: 1000 + log_every_n_steps: 50 + check_val_every_n_epoch: 5 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/tts/conf/hifigan/hifigan.yaml b/models/tacotron2/NeMo/examples/tts/conf/hifigan/hifigan.yaml new file mode 100644 index 0000000000..9412ced669 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/hifigan/hifigan.yaml @@ -0,0 +1,67 @@ +name: "HifiGan" +train_dataset: ??? +validation_datasets: ??? + +defaults: + - model/generator: v1 + - model/train_ds: train_ds + - model/validation_ds: val_ds + +model: + preprocessor: + _target_: nemo.collections.asr.parts.preprocessing.features.FilterbankFeatures + dither: 0.0 + frame_splicing: 1 + nfilt: 80 + highfreq: 8000 + log: true + log_zero_guard_type: clamp + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: 1024 + n_window_size: 1024 + n_window_stride: 256 + normalize: null + pad_to: 0 + pad_value: -11.52 + preemph: null + sample_rate: 22050 + window: hann + use_grads: false + exact_pad: true + + optim: + _target_: torch.optim.AdamW + lr: 0.0002 + betas: [0.8, 0.99] + + sched: + name: CosineAnnealing + min_lr: 1e-5 + warmup_ratio: 0.02 + + max_steps: 25000000 + l1_loss_factor: 45 + denoise_strength: 0.0025 + +trainer: + gpus: -1 # number of gpus + max_steps: ${model.max_steps} + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + flush_logs_every_n_steps: 200 + log_every_n_steps: 100 + check_val_every_n_epoch: 10 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True + checkpoint_callback_params: + monitor: "val_loss" + mode: "min" diff --git a/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v1.yaml b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v1.yaml new file mode 100644 index 0000000000..30b819e979 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v1.yaml @@ -0,0 +1,8 @@ +# @package _group_ +_target_: nemo.collections.tts.modules.hifigan_modules.Generator +resblock: 1 +upsample_rates: [8,8,2,2] +upsample_kernel_sizes: [16,16,4,4] +upsample_initial_channel: 512 +resblock_kernel_sizes: [3,7,11] +resblock_dilation_sizes: [[1,3,5], [1,3,5], [1,3,5]] diff --git a/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v2.yaml b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v2.yaml new file mode 100644 index 0000000000..ef33801d5c --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v2.yaml @@ -0,0 +1,8 @@ +# @package _group_ +_target_: nemo.collections.tts.modules.hifigan_modules.Generator +resblock: 1 +upsample_rates: [8,8,2,2] +upsample_kernel_sizes: [16,16,4,4] +upsample_initial_channel: 128 +resblock_kernel_sizes: [3,7,11] +resblock_dilation_sizes: [[1,3,5], [1,3,5], [1,3,5]] diff --git a/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v3.yaml b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v3.yaml new file mode 100644 index 0000000000..734f0a330b --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/generator/v3.yaml @@ -0,0 +1,8 @@ +# @package _group_ +_target_: nemo.collections.tts.modules.hifigan_modules.Generator +resblock: 2 +upsample_rates: [8,8,4] +upsample_kernel_sizes: [16,16,8] +upsample_initial_channel: 256 +resblock_kernel_sizes: [3,5,7] +resblock_dilation_sizes: [[1,2], [2,6], [3,12]] diff --git a/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/train_ds/train_ds.yaml b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/train_ds/train_ds.yaml new file mode 100644 index 0000000000..d7bfce88c2 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/train_ds/train_ds.yaml @@ -0,0 +1,13 @@ +# @package _group_ +dataset: + _target_: "nemo.collections.tts.data.datalayers.AudioDataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.75 + n_segments: 8192 + trim: false +dataloader_params: + drop_last: false + shuffle: true + batch_size: 16 + num_workers: 4 diff --git a/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/train_ds/train_ds_finetune.yaml b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/train_ds/train_ds_finetune.yaml new file mode 100644 index 0000000000..195f3b2324 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/train_ds/train_ds_finetune.yaml @@ -0,0 +1,11 @@ +# @package _group_ +dataset: + _target_: "nemo.collections.tts.data.datalayers.MelAudioDataset" + manifest_filepath: ${train_dataset} + min_duration: 0.75 + n_segments: 8192 +dataloader_params: + drop_last: false + shuffle: true + batch_size: 16 + num_workers: 4 diff --git a/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/validation_ds/val_ds.yaml b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/validation_ds/val_ds.yaml new file mode 100644 index 0000000000..cfe6003837 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/validation_ds/val_ds.yaml @@ -0,0 +1,13 @@ +# @package _group_ +dataset: + _target_: "nemo.collections.tts.data.datalayers.AudioDataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: null + n_segments: -1 + trim: false +dataloader_params: + drop_last: false + shuffle: false + batch_size: 16 + num_workers: 1 diff --git a/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/validation_ds/val_ds_finetune.yaml b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/validation_ds/val_ds_finetune.yaml new file mode 100644 index 0000000000..ad9e2b9714 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/hifigan/model/validation_ds/val_ds_finetune.yaml @@ -0,0 +1,11 @@ +# @package _group_ +dataset: + _target_: "nemo.collections.tts.data.datalayers.MelAudioDataset" + manifest_filepath: ${validation_datasets} + min_duration: 3 + n_segments: 66048 +dataloader_params: + drop_last: false + shuffle: false + batch_size: 16 + num_workers: 4 diff --git a/models/tacotron2/NeMo/examples/tts/conf/melgan.yaml b/models/tacotron2/NeMo/examples/tts/conf/melgan.yaml new file mode 100644 index 0000000000..92599889d2 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/melgan.yaml @@ -0,0 +1,108 @@ +name: "MelGan" +sample_rate: 22050 +n_fft: 1024 +fmax: 8000 +pad_value: -11.52 +train_dataset: ??? +validation_datasets: ??? +n_mels: 80 + +model: + train_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.AudioDataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.75 + n_segments: 16384 + trim: false + dataloader_params: + drop_last: false + shuffle: true + batch_size: 64 + num_workers: 4 + + + validation_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.AudioDataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: null + n_segments: -1 + trim: false + dataloader_params: + drop_last: false + shuffle: false + batch_size: 13 + num_workers: 4 + + preprocessor: + _target_: nemo.collections.asr.parts.preprocessing.features.FilterbankFeatures + dither: 0.0 + frame_splicing: 1 + nfilt: ${n_mels} + highfreq: ${fmax} + log: true + log_zero_guard_type: clamp + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: ${n_fft} + + n_window_size: 1024 + n_window_stride: 256 + + normalize: null + pad_to: 0 + pad_value: ${pad_value} + preemph: null + sample_rate: ${sample_rate} + window: hann + exact_pad: true + + generator: + _target_: nemo.collections.tts.modules.melgan_modules.MelGANGenerator + in_channels: ${n_mels} # Number of input channels. + out_channels: 1 # Number of output channels. + kernel_size: 7 # Kernel size of initial and final conv layers. + channels: 512 # Initial number of channels for conv layers. + upsample_scales: [8, 4, 4, 2] # List of Upsampling scales. + stack_kernel_size: 3 # Kernel size of dilated conv layers in residual stack. + stacks: 4 # Number of stacks in a single residual stack module. + use_weight_norm: True # Whether to use weight normalization. + + discriminator: + _target_: nemo.collections.tts.modules.melgan_modules.MelGANMultiScaleDiscriminator + scales: 3 + out_channels: 1 # Number of output channels. + kernel_sizes: [5, 3] # List of kernel size. + channels: 16 # Number of channels of the initial conv layer. + max_downsample_channels: 512 # Maximum number of channels of downsampling layers. + downsample_scales: [4, 4, 4] # List of downsampling scales. + use_weight_norm: True # Whether to use weight norm. + use_spectral_norm: False # Whether to use spectral norm. + + loss: + _target_: nemo.collections.tts.losses.stftlosses.MultiResolutionSTFTLoss + fft_sizes: [1024, 2048, 512] + hop_sizes: [120, 240, 50] + win_lengths: [600, 1200, 240] + window: "hann_window" + +trainer: + gpus: 1 # number of gpus + max_epochs: ??? + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + log_every_n_steps: 1000 + check_val_every_n_epoch: 150 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/tts/conf/squeezewave.yaml b/models/tacotron2/NeMo/examples/tts/conf/squeezewave.yaml new file mode 100644 index 0000000000..9fc620f401 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/squeezewave.yaml @@ -0,0 +1,104 @@ +name: "SqueezeWave" +sample_rate: 22050 +n_fft: 1024 +n_mels: 80 +fmax: 8000 +pad_value: -11.52 +train_dataset: ??? +validation_datasets: ??? + +model: + sigma: 1.0 + train_ds: + dataset: + # Use SplicedAudioDataset to use all of training data + _target_: "nemo.collections.tts.data.datalayers.AudioDataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.1 + n_segments: 16384 + trim: false + dataloader_params: + drop_last: true + pin_memory: true + shuffle: true + batch_size: 96 + num_workers: 4 + + validation_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.AudioDataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: 0.1 + n_segments: -1 + trim: false + # This value is hard-coded to reflect n_window_stride * lcm(2, pad_to) + truncate_to: 512 + dataloader_params: + drop_last: false + shuffle: false + batch_size: 1 + num_workers: 4 + + preprocessor: + _target_: nemo.collections.asr.parts.preprocessing.features.FilterbankFeatures + dither: 0.0 + nfilt: ${n_mels} + frame_splicing: 1 + highfreq: ${fmax} + log: true + log_zero_guard_type: clamp + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: ${n_fft} + + # SqueezeWave is currently hardcoded to these values for window size and stride + # Changing these parameters are not recommended + n_window_size: 1024 + n_window_stride: 256 + + normalize: null + pad_to: 1 + pad_value: ${pad_value} + preemph: null + sample_rate: ${sample_rate} + window: hann + exact_pad: true + + squeezewave: + _target_: nemo.collections.tts.modules.squeezewave.SqueezeWaveModule + n_early_every: 2 + n_early_size: 16 + n_flows: 12 + n_group: 128 + n_mel_channels: ${n_mels} + n_wn_channels: 256 + n_wn_layers: 8 + wn_kernel_size: 3 + + optim: + name: adam + lr: 4e-4 + +trainer: + gpus: 1 # number of gpus + max_epochs: 10000 + max_steps: 600000 + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager + flush_logs_every_n_steps: 1000 + log_every_n_steps: 200 + check_val_every_n_epoch: 25 + precision: 16 + amp_level: O1 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: true + create_checkpoint_callback: true diff --git a/models/tacotron2/NeMo/examples/tts/conf/tacotron2.yaml b/models/tacotron2/NeMo/examples/tts/conf/tacotron2.yaml new file mode 100644 index 0000000000..11b37cf4d1 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/tacotron2.yaml @@ -0,0 +1,140 @@ +name: Tacotron2 +sample_rate: 22050 +# , , will be added by the tacotron2.py script +labels: [' ', '!', '"', "'", '(', ')', ',', '-', '.', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', ']', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z'] +n_fft: 1024 +n_mels: 80 +fmax: 8000 +n_stride: 256 +pad_value: -11.52 +train_dataset: ??? +validation_datasets: ??? + +model: + labels: ${labels} + train_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharDataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.1 + trim: false + int_values: false + normalize: true + sample_rate: ${sample_rate} + # bos_id: 66 + # eos_id: 67 + # pad_id: 68 These parameters are added automatically in Tacotron2 + dataloader_params: + drop_last: false + shuffle: true + batch_size: 48 + num_workers: 4 + + + validation_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharDataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: 0.1 + int_values: false + normalize: true + sample_rate: ${sample_rate} + trim: false + # bos_id: 66 + # eos_id: 67 + # pad_id: 68 These parameters are added automatically in Tacotron2 + dataloader_params: + drop_last: false + shuffle: false + batch_size: 48 + num_workers: 8 + + preprocessor: + _target_: nemo.collections.asr.parts.preprocessing.features.FilterbankFeatures + dither: 0.0 + nfilt: ${n_mels} + frame_splicing: 1 + highfreq: ${fmax} + log: true + log_zero_guard_type: clamp + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: ${n_fft} + n_window_size: 1024 + n_window_stride: ${n_stride} + normalize: null + pad_to: 16 + pad_value: ${pad_value} + preemph: null + sample_rate: ${sample_rate} + window: hann + + encoder: + _target_: nemo.collections.tts.modules.tacotron2.Encoder + encoder_kernel_size: 5 + encoder_n_convolutions: 3 + encoder_embedding_dim: 512 + + decoder: + _target_: nemo.collections.tts.modules.tacotron2.Decoder + decoder_rnn_dim: 1024 + encoder_embedding_dim: ${model.encoder.encoder_embedding_dim} + gate_threshold: 0.5 + max_decoder_steps: 1000 + n_frames_per_step: 1 # currently only 1 is supported + n_mel_channels: ${n_mels} + p_attention_dropout: 0.1 + p_decoder_dropout: 0.1 + prenet_dim: 256 + prenet_p_dropout: 0.5 + # Attention parameters + attention_dim: 128 + attention_rnn_dim: 1024 + # AttentionLocation Layer parameters + attention_location_kernel_size: 31 + attention_location_n_filters: 32 + early_stopping: true + + postnet: + _target_: nemo.collections.tts.modules.tacotron2.Postnet + n_mel_channels: ${n_mels} + p_dropout: 0.5 + postnet_embedding_dim: 512 + postnet_kernel_size: 5 + postnet_n_convolutions: 5 + + optim: + name: adam + lr: 1e-3 + weight_decay: 1e-6 + + # scheduler setup + sched: + name: CosineAnnealing + min_lr: 1e-5 + + +trainer: + gpus: 1 # number of gpus + max_epochs: ??? + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + gradient_clip_val: 1.0 + flush_logs_every_n_steps: 1000 + log_every_n_steps: 200 + check_val_every_n_epoch: 25 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/tts/conf/talknet-durs.yaml b/models/tacotron2/NeMo/examples/tts/conf/talknet-durs.yaml new file mode 100644 index 0000000000..aaa977d376 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/talknet-durs.yaml @@ -0,0 +1,175 @@ +name: TalkNetDurs +sample_rate: 22050 + +train_dataset: ??? +validation_datasets: ??? +durs_file: ??? +f0_file: ??? + +model: + d_char: 128 + dropout: 0.1 + separable: true + + train_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.1 + int_values: false + load_audio: false + normalize: false + sample_rate: ${sample_rate} + trim: false + durs_file: ${durs_file} + f0_file: ${f0_file} + blanking: true + vocab: + notation: phonemes + punct: true + spaces: true + stresses: false + add_blank_at: last + dataloader_params: + drop_last: false + shuffle: true + batch_size: 64 + num_workers: 4 + + validation_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: 0.1 + int_values: false + load_audio: false + normalize: false + sample_rate: ${sample_rate} + trim: false + durs_file: ${durs_file} + f0_file: ${f0_file} + blanking: true + vocab: + notation: phonemes + punct: true + spaces: true + stresses: false + add_blank_at: last + dataloader_params: + drop_last: false + shuffle: false + batch_size: 64 + num_workers: 1 + + model: + _target_: "nemo.collections.asr.modules.ConvASREncoder" + feat_in: ${model.d_char} + activation: relu + conv_mask: true + + jasper: + - filters: 256 + repeat: 1 + kernel: [3] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + + - filters: 256 + repeat: 5 + kernel: [5] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: True + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [7] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: True + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [9] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: True + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [11] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: True + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [13] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: True + se_reduction_ratio: 8 + + - filters: 512 + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + + optim: + name: adam + lr: 1e-3 + weight_decay: 1e-6 + + sched: + name: CosineAnnealing + min_lr: 1e-5 + warmup_ratio: 0.02 + +trainer: + gpus: 1 + max_epochs: ??? + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False + logger: False + gradient_clip_val: 1.0 + flush_logs_every_n_steps: 1000 + log_every_n_steps: 200 + check_val_every_n_epoch: 25 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True + checkpoint_callback_params: + monitor: "val_acc" + mode: "max" \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/tts/conf/talknet-pitch.yaml b/models/tacotron2/NeMo/examples/tts/conf/talknet-pitch.yaml new file mode 100644 index 0000000000..722c429045 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/talknet-pitch.yaml @@ -0,0 +1,179 @@ +name: TalkNetPitch +sample_rate: 22050 + +train_dataset: ??? +validation_datasets: ??? +durs_file: ??? +f0_file: ??? + +model: + d_char: 128 + dropout: 0.1 + separable: true + + # F0 stats (depends on dataset, current for LJSpeech) + f0_mean: 178.7767791748047 + f0_std: 33.51659393310547 + + train_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.1 + int_values: false + load_audio: false + normalize: false + sample_rate: ${sample_rate} + trim: false + durs_file: ${durs_file} + f0_file: ${f0_file} + blanking: true + vocab: + notation: phonemes + punct: true + spaces: true + stresses: false + add_blank_at: last + dataloader_params: + drop_last: false + shuffle: true + batch_size: 64 + num_workers: 4 + + validation_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: 0.1 + int_values: false + load_audio: false + normalize: false + sample_rate: ${sample_rate} + trim: false + durs_file: ${durs_file} + f0_file: ${f0_file} + blanking: true + vocab: + notation: phonemes + punct: true + spaces: true + stresses: false + add_blank_at: last + dataloader_params: + drop_last: false + shuffle: false + batch_size: 64 + num_workers: 1 + + model: + _target_: "nemo.collections.asr.modules.ConvASREncoder" + feat_in: ${model.d_char} + activation: relu + conv_mask: true + + jasper: + - filters: 256 + repeat: 1 + kernel: [3] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + + - filters: 256 + repeat: 5 + kernel: [5] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: True + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [7] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: True + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [9] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: True + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [11] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: True + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [13] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: True + se_reduction_ratio: 8 + + - filters: 512 + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + + optim: + name: adam + lr: 1e-3 + weight_decay: 1e-6 + + sched: + name: CosineAnnealing + min_lr: 1e-5 + warmup_ratio: 0.02 + +trainer: + gpus: 1 + max_epochs: ??? + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False + logger: False + gradient_clip_val: 1.0 + flush_logs_every_n_steps: 1000 + log_every_n_steps: 200 + check_val_every_n_epoch: 25 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True + checkpoint_callback_params: + monitor: "val_body_mae" + mode: "min" \ No newline at end of file diff --git a/models/tacotron2/NeMo/examples/tts/conf/talknet-spect.yaml b/models/tacotron2/NeMo/examples/tts/conf/talknet-spect.yaml new file mode 100644 index 0000000000..e476b4d62b --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/talknet-spect.yaml @@ -0,0 +1,244 @@ +name: TalkNetSpect +sample_rate: 22050 + +train_dataset: ??? +validation_datasets: ??? +durs_file: ??? +f0_file: ??? + +model: + d_char: 128 + n_mels: 80 + dropout: 0.1 + separable: true + + train_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.1 + int_values: false + load_audio: true + normalize: false + sample_rate: ${sample_rate} + trim: false + durs_file: ${durs_file} + f0_file: ${f0_file} + blanking: true + vocab: + notation: phonemes + punct: true + spaces: true + stresses: false + add_blank_at: last + dataloader_params: + drop_last: false + shuffle: true + batch_size: 64 + num_workers: 4 + + validation_ds: + dataset: + _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: 0.1 + int_values: false + load_audio: true + normalize: false + sample_rate: ${sample_rate} + trim: false + durs_file: ${durs_file} + f0_file: ${f0_file} + blanking: true + vocab: + notation: phonemes + punct: true + spaces: true + stresses: false + add_blank_at: last + dataloader_params: + drop_last: false + shuffle: false + batch_size: 64 + num_workers: 1 + + preprocessor: + _target_: "nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor" + dither: 0.0 + features: ${model.n_mels} + frame_splicing: 1 + highfreq: 8000 + log: true + log_zero_guard_type: clamp + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: 1024 + n_window_size: 1024 + window_size: False + n_window_stride: 256 + window_stride: False + normalize: null + pad_to: 1 + pad_value: -11.52 + preemph: null + sample_rate: ${sample_rate} + stft_conv: false + window: hann + + model: + _target_: "nemo.collections.asr.modules.ConvASREncoder" + feat_in: ${model.d_char} + activation: relu + conv_mask: true + + jasper: + - filters: 256 + repeat: 3 + kernel: [3] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + + - filters: 256 + repeat: 5 + kernel: [5] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: true + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [7] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: true + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [9] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: true + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [13] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: true + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [15] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: true + se_reduction_ratio: 8 + + - filters: 256 + repeat: 5 + kernel: [17] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: true + se_reduction_ratio: 8 + + - filters: 512 + repeat: 5 + kernel: [21] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: true + se_reduction_ratio: 8 + + - filters: 512 + repeat: 5 + kernel: [23] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: true + se_reduction_ratio: 8 + + - filters: 512 + repeat: 5 + kernel: [25] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + separable: ${model.separable} + se: true + se_reduction_ratio: 8 + + - filters: 1024 + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: ${model.dropout} + residual: true + + optim: + name: adam + lr: 1e-3 + weight_decay: 1e-6 + + sched: + name: CosineAnnealing + min_lr: 1e-5 + warmup_ratio: 0.02 + +trainer: + gpus: 1 + max_epochs: ??? + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False + logger: False + gradient_clip_val: 1.0 + flush_logs_every_n_steps: 1000 + log_every_n_steps: 200 + check_val_every_n_epoch: 25 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True + checkpoint_callback_params: + monitor: "val_loss" + mode: "min" diff --git a/models/tacotron2/NeMo/examples/tts/conf/uniglow.yaml b/models/tacotron2/NeMo/examples/tts/conf/uniglow.yaml new file mode 100644 index 0000000000..b74584d799 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/uniglow.yaml @@ -0,0 +1,96 @@ +name: "UniGlow" +sample_rate: 22050 +n_fft: 1024 +n_mels: 80 +fmax: 8000 +pad_value: -11.52 +train_dataset: ??? +validation_datasets: ??? +resume_from_ckpt: null + +model: + sigma: 1.0 + train_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.AudioDataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.1 + n_segments: 16384 + trim: false + dataloader_params: + drop_last: false + shuffle: true + batch_size: 12 + num_workers: 4 + + validation_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.AudioDataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: 0.1 + n_segments: 49152 # 3 secs + trim: false + dataloader_params: + drop_last: false + shuffle: false + batch_size: 12 + num_workers: 4 + + preprocessor: + _target_: nemo.collections.asr.parts.preprocessing.features.FilterbankFeatures + dither: 0.0 + nfilt: ${n_mels} + frame_splicing: 1 + highfreq: ${fmax} + log: true + log_zero_guard_type: clamp + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: ${n_fft} + + # UniGlow is currently hardcoded to these values for window size and stride + # Changing these parameters are not recommended + n_window_size: 1024 + n_window_stride: 256 + + normalize: null + pad_to: 0 #16 + pad_value: ${pad_value} + preemph: null + sample_rate: ${sample_rate} + window: hann + + uniglow: + n_flows: 16 + n_group: 8 + n_mel_channels: ${n_mels} + n_wn_channels: 256 + n_wn_layers: 8 + wn_kernel_size: 3 + stft_loss_coef: 0.1 + + optim: + name: adam + lr: 1e-4 + +trainer: + gpus: 1 # number of gpus + max_epochs: ??? + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + flush_logs_every_n_steps: 1000 + log_every_n_steps: 200 + check_val_every_n_epoch: 25 + precision: 16 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/tts/conf/waveglow.yaml b/models/tacotron2/NeMo/examples/tts/conf/waveglow.yaml new file mode 100644 index 0000000000..83f97a0474 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/conf/waveglow.yaml @@ -0,0 +1,97 @@ +name: "WaveGlow" +sample_rate: 22050 +n_fft: 1024 +n_mels: 80 +fmax: 8000 +pad_value: -11.52 +train_dataset: ??? +validation_datasets: ??? + +model: + sigma: 1.0 + train_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.AudioDataset" + manifest_filepath: ${train_dataset} + max_duration: null + min_duration: 0.1 + n_segments: 16000 + trim: false + dataloader_params: + drop_last: false + shuffle: true + batch_size: 12 + num_workers: 4 + + validation_ds: + dataset: + _target_: "nemo.collections.tts.data.datalayers.AudioDataset" + manifest_filepath: ${validation_datasets} + max_duration: null + min_duration: 0.1 + n_segments: -1 + trim: false + dataloader_params: + drop_last: false + shuffle: false + batch_size: 8 + num_workers: 4 + + preprocessor: + _target_: nemo.collections.asr.parts.preprocessing.features.FilterbankFeatures + dither: 0.0 + nfilt: ${n_mels} + frame_splicing: 1 + highfreq: ${fmax} + log: true + log_zero_guard_type: clamp + log_zero_guard_value: 1e-05 + lowfreq: 0 + mag_power: 1.0 + n_fft: ${n_fft} + + # Waveglow is currently hardcoded to these values for window size and stride + # Changing these parameters are not recommended + n_window_size: 1024 + n_window_stride: 256 + + normalize: null + pad_to: 16 + pad_value: ${pad_value} + preemph: null + sample_rate: ${sample_rate} + window: hann + + waveglow: + _target_: nemo.collections.tts.modules.waveglow.WaveGlowModule + n_early_every: 4 + n_early_size: 2 + n_flows: 12 + n_group: 8 + n_mel_channels: ${n_mels} + n_wn_channels: 256 + n_wn_layers: 8 + wn_kernel_size: 3 + + optim: + name: adam + lr: 1e-4 + +trainer: + gpus: 1 # number of gpus + max_epochs: ??? + num_nodes: 1 + accelerator: ddp + accumulate_grad_batches: 1 + checkpoint_callback: False # Provided by exp_manager + logger: False # Provided by exp_manager + flush_logs_every_n_steps: 1000 + log_every_n_steps: 200 + check_val_every_n_epoch: 25 + precision: 16 + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/models/tacotron2/NeMo/examples/tts/degli.py b/models/tacotron2/NeMo/examples/tts/degli.py new file mode 100755 index 0000000000..b38b4eb159 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/degli.py @@ -0,0 +1,36 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.data.datalayers import preprocess_linear_specs_dataset +from nemo.collections.tts.models import DegliModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="degli") +def main(cfg): + preprocess_linear_specs_dataset(**cfg.model.preprocessor) + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + model = DegliModel(cfg=cfg.model, trainer=trainer) + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([epoch_time_logger]) + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/ed_mel2spec.py b/models/tacotron2/NeMo/examples/tts/ed_mel2spec.py new file mode 100755 index 0000000000..8815018407 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/ed_mel2spec.py @@ -0,0 +1,36 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.data.datalayers import preprocess_linear_specs_dataset +from nemo.collections.tts.models import EDMel2SpecModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="ed_mel2spec") +def main(cfg): + preprocess_linear_specs_dataset(**cfg.model.preprocessor) + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + model = EDMel2SpecModel(cfg=cfg.model, trainer=trainer) + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([epoch_time_logger]) + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/fastpitch.py b/models/tacotron2/NeMo/examples/tts/fastpitch.py new file mode 100644 index 0000000000..3afb305a99 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/fastpitch.py @@ -0,0 +1,35 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import FastPitchModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="fastpitch") +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + model = FastPitchModel(cfg=cfg.model, trainer=trainer) + lr_logger = pl.callbacks.LearningRateMonitor() + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([lr_logger, epoch_time_logger]) + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/fastpitch_hifigan_e2e.py b/models/tacotron2/NeMo/examples/tts/fastpitch_hifigan_e2e.py new file mode 100644 index 0000000000..f8b52537be --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/fastpitch_hifigan_e2e.py @@ -0,0 +1,36 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl +from pytorch_lightning.plugins import DDPPlugin + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models.fastpitch_hifigan_e2e import FastPitchHifiGanE2EModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="fastpitch_hifigan_e2e") +def main(cfg): + trainer = pl.Trainer(plugins=[DDPPlugin(find_unused_parameters=True)], **cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + model = FastPitchHifiGanE2EModel(cfg=cfg.model, trainer=trainer) + lr_logger = pl.callbacks.LearningRateMonitor() + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([lr_logger, epoch_time_logger]) + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/fastspeech2.py b/models/tacotron2/NeMo/examples/tts/fastspeech2.py new file mode 100644 index 0000000000..0b95dff3bb --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/fastspeech2.py @@ -0,0 +1,35 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import FastSpeech2Model +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="fastspeech2") +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + model = FastSpeech2Model(cfg=cfg.model, trainer=trainer) + lr_logger = pl.callbacks.LearningRateMonitor() + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([lr_logger, epoch_time_logger]) + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/fastspeech2_hifigan_e2e.py b/models/tacotron2/NeMo/examples/tts/fastspeech2_hifigan_e2e.py new file mode 100644 index 0000000000..4122eefcc4 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/fastspeech2_hifigan_e2e.py @@ -0,0 +1,35 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models.fastspeech2_hifigan_e2e import FastSpeech2HifiGanE2EModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="fastspeech2_hifigan_e2e") +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + model = FastSpeech2HifiGanE2EModel(cfg=cfg.model, trainer=trainer) + lr_logger = pl.callbacks.LearningRateMonitor() + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([lr_logger, epoch_time_logger]) + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/glow_tts.py b/models/tacotron2/NeMo/examples/tts/glow_tts.py new file mode 100644 index 0000000000..7d797ff2fe --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/glow_tts.py @@ -0,0 +1,35 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import GlowTTSModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="glow_tts") +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + model = GlowTTSModel(cfg=cfg.model, trainer=trainer) + lr_logger = pl.callbacks.LearningRateMonitor() + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([lr_logger, epoch_time_logger]) + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/hifigan.py b/models/tacotron2/NeMo/examples/tts/hifigan.py new file mode 100644 index 0000000000..5c3406a2f2 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/hifigan.py @@ -0,0 +1,31 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.tts.models import HifiGanModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf/hifigan", config_name="hifigan") +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + model = HifiGanModel(cfg=cfg.model, trainer=trainer) + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/melgan.py b/models/tacotron2/NeMo/examples/tts/melgan.py new file mode 100644 index 0000000000..946f2b76e8 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/melgan.py @@ -0,0 +1,35 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl +from pytorch_lightning.plugins import DDPPlugin + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import MelGanModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="melgan") +def main(cfg): + trainer = pl.Trainer(plugins=[DDPPlugin(find_unused_parameters=True)], **cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + model = MelGanModel(cfg=cfg.model, trainer=trainer) + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([epoch_time_logger]) + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/squeezewave.py b/models/tacotron2/NeMo/examples/tts/squeezewave.py new file mode 100755 index 0000000000..d7a5ad8a1d --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/squeezewave.py @@ -0,0 +1,34 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import SqueezeWaveModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="squeezewave") +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + model = SqueezeWaveModel(cfg=cfg.model, trainer=trainer) + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([epoch_time_logger]) + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/tacotron2.py b/models/tacotron2/NeMo/examples/tts/tacotron2.py new file mode 100755 index 0000000000..a5446c35f7 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/tacotron2.py @@ -0,0 +1,44 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import Tacotron2Model +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +# hydra_runner is a thin NeMo wrapper around Hydra +# It looks for a config named tacotron2.yaml inside the conf folder +# Hydra parses the yaml and returns it as a Omegaconf DictConfig +@hydra_runner(config_path="conf", config_name="tacotron2") +def main(cfg): + # Define the Lightning trainer + trainer = pl.Trainer(**cfg.trainer) + # exp_manager is a NeMo construct that helps with logging and checkpointing + exp_manager(trainer, cfg.get("exp_manager", None)) + # Define the Tacotron 2 model, this will construct the model as well as + # define the training and validation dataloaders + model = Tacotron2Model(cfg=cfg.model, trainer=trainer) + # Let's add a few more callbacks + lr_logger = pl.callbacks.LearningRateMonitor() + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([lr_logger, epoch_time_logger]) + # Call lightning trainer's fit() to train the model + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/talknet_durs.py b/models/tacotron2/NeMo/examples/tts/talknet_durs.py new file mode 100644 index 0000000000..f3543f69a2 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/talknet_durs.py @@ -0,0 +1,33 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import TalkNetDursModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path='conf', config_name='talknet-durs') +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get('exp_manager', None)) + model = TalkNetDursModel(cfg=cfg.model, trainer=trainer) + trainer.callbacks.extend([pl.callbacks.LearningRateMonitor(), LogEpochTimeCallback()]) # noqa + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/talknet_pitch.py b/models/tacotron2/NeMo/examples/tts/talknet_pitch.py new file mode 100644 index 0000000000..6836d63dc9 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/talknet_pitch.py @@ -0,0 +1,33 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import TalkNetPitchModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path='conf', config_name='talknet-pitch') +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get('exp_manager', None)) + model = TalkNetPitchModel(cfg=cfg.model, trainer=trainer) + trainer.callbacks.extend([pl.callbacks.LearningRateMonitor(), LogEpochTimeCallback()]) # noqa + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/talknet_spect.py b/models/tacotron2/NeMo/examples/tts/talknet_spect.py new file mode 100644 index 0000000000..619403c30f --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/talknet_spect.py @@ -0,0 +1,33 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import TalkNetSpectModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path='conf', config_name='talknet-spect') +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get('exp_manager', None)) + model = TalkNetSpectModel(cfg=cfg.model, trainer=trainer) + trainer.callbacks.extend([pl.callbacks.LearningRateMonitor(), LogEpochTimeCallback()]) # noqa + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/test_tts_infer.py b/models/tacotron2/NeMo/examples/tts/test_tts_infer.py new file mode 100644 index 0000000000..ce98fef1f3 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/test_tts_infer.py @@ -0,0 +1,153 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script is used as a CI test and shows how to chain TTS and ASR models +""" + +from argparse import ArgumentParser +from math import ceil +from pathlib import Path + +import librosa +import soundfile +import torch + +from nemo.collections.asr.metrics.wer import word_error_rate +from nemo.collections.asr.models import EncDecCTCModel +from nemo.collections.common.parts.preprocessing import parsers +from nemo.collections.tts.models.base import SpectrogramGenerator, Vocoder +from nemo.utils import logging + +LIST_OF_TEST_STRINGS = [ + "Hey, this is a test of the speech synthesis system.", + "roupell received the announcement with a cheerful countenance.", + "with thirteen dollars, eighty-seven cents when considerably greater resources were available to him.", + "Two other witnesses were able to offer partial descriptions of a man they saw in the southeast corner window.", + "'just to steady their legs a little' in other words, to add his weight to that of the hanging bodies.", + "The discussion above has already set forth examples of his expression of hatred for the United States.", + "At two:thirty-eight p.m., Eastern Standard Time, Lyndon Baines Johnson took the oath of office as the thirty-sixth President of the United States.", + "or, quote, other high government officials in the nature of a complaint coupled with an expressed or implied determination to use a means.", + "As for my return entrance visa please consider it separately. End quote.", + "it appears that Marina Oswald also complained that her husband was not able to provide more material things for her.", + "appeared in The Dallas Times Herald on November fifteen, nineteen sixty-three.", + "The only exit from the office in the direction Oswald was moving was through the door to the front stairway.", +] + + +def main(): + parser = ArgumentParser() + parser.add_argument( + "--asr_model", + type=str, + default="QuartzNet15x5Base-En", + choices=[x.pretrained_model_name for x in EncDecCTCModel.list_available_models()], + ) + parser.add_argument( + "--tts_model_spec", + type=str, + default="tts_en_tacotron2", + choices=[x.pretrained_model_name for x in SpectrogramGenerator.list_available_models()], + ) + parser.add_argument( + "--tts_model_vocoder", + type=str, + default="tts_waveglow_88m", + choices=[x.pretrained_model_name for x in Vocoder.list_available_models()], + ) + parser.add_argument("--wer_tolerance", type=float, default=1.0, help="used by test") + parser.add_argument("--trim", action="store_true") + parser.add_argument("--debug", action="store_true") + args = parser.parse_args() + torch.set_grad_enabled(False) + + if args.debug: + logging.set_verbosity(logging.DEBUG) + + logging.info(f"Using NGC cloud ASR model {args.asr_model}") + asr_model = EncDecCTCModel.from_pretrained(model_name=args.asr_model) + logging.info(f"Using NGC cloud TTS Spectrogram Generator model {args.tts_model_spec}") + tts_model_spec = SpectrogramGenerator.from_pretrained(model_name=args.tts_model_spec) + logging.info(f"Using NGC cloud TTS Vocoder model {args.tts_model_vocoder}") + tts_model_vocoder = Vocoder.from_pretrained(model_name=args.tts_model_vocoder) + models = [asr_model, tts_model_spec, tts_model_vocoder] + + if torch.cuda.is_available(): + for i, m in enumerate(models): + models[i] = m.cuda() + for m in models: + m.eval() + + asr_model, tts_model_spec, tts_model_vocoder = models + + parser = parsers.make_parser( + labels=asr_model.decoder.vocabulary, name="en", unk_id=-1, blank_id=-1, do_normalize=True, + ) + labels_map = dict([(i, asr_model.decoder.vocabulary[i]) for i in range(len(asr_model.decoder.vocabulary))]) + + tts_input = [] + asr_references = [] + longest_tts_input = 0 + for test_str in LIST_OF_TEST_STRINGS: + tts_parsed_input = tts_model_spec.parse(test_str) + if len(tts_parsed_input[0]) > longest_tts_input: + longest_tts_input = len(tts_parsed_input[0]) + tts_input.append(tts_parsed_input.squeeze()) + + asr_parsed = parser(test_str) + asr_parsed = ''.join([labels_map[c] for c in asr_parsed]) + asr_references.append(asr_parsed) + + # Pad TTS Inputs + for i, text in enumerate(tts_input): + pad = (0, longest_tts_input - len(text)) + tts_input[i] = torch.nn.functional.pad(text, pad, value=68) + + logging.debug(tts_input) + + # Do TTS + tts_input = torch.stack(tts_input) + if torch.cuda.is_available(): + tts_input = tts_input.cuda() + specs = tts_model_spec.generate_spectrogram(tokens=tts_input) + audio = [] + step = ceil(len(specs) / 4) + for i in range(4): + audio.append(tts_model_vocoder.convert_spectrogram_to_audio(spec=specs[i * step : i * step + step])) + + audio = [item for sublist in audio for item in sublist] + audio_file_paths = [] + # Save audio + logging.debug(f"args.trim: {args.trim}") + for i, aud in enumerate(audio): + aud = aud.cpu().numpy() + if args.trim: + aud = librosa.effects.trim(aud, top_db=40)[0] + soundfile.write(f"{i}.wav", aud, samplerate=22050) + audio_file_paths.append(str(Path(f"{i}.wav"))) + + # Do ASR + hypotheses = asr_model.transcribe(audio_file_paths) + for i, _ in enumerate(hypotheses): + logging.debug(f"{i}") + logging.debug(f"ref:'{asr_references[i]}'") + logging.debug(f"hyp:'{hypotheses[i]}'") + wer_value = word_error_rate(hypotheses=hypotheses, references=asr_references) + if wer_value > args.wer_tolerance: + raise ValueError(f"Got WER of {wer_value}. It was higher than {args.wer_tolerance}") + logging.info(f'Got WER of {wer_value}. Tolerance was {args.wer_tolerance}') + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/uniglow.py b/models/tacotron2/NeMo/examples/tts/uniglow.py new file mode 100755 index 0000000000..39a4be5987 --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/uniglow.py @@ -0,0 +1,44 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import UniGlowModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="uniglow") +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([epoch_time_logger]) + exp_manager(trainer, cfg.get("exp_manager", None)) + + if cfg.resume_from_ckpt is None: + logging.info("Training UniGlow from scratch") + model = UniGlowModel(cfg=cfg.model, trainer=trainer) + else: + logging.info("Fine-tuning UniGlow from {cfg.resume_from_ckpt}") + model = UniGlowModel.restore_from(cfg.resume_from_ckpt) + model.setup_training_data(cfg.model.train_ds) + model.setup_validation_data(cfg.model.validation_ds) + + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/examples/tts/waveglow.py b/models/tacotron2/NeMo/examples/tts/waveglow.py new file mode 100755 index 0000000000..66b13491ab --- /dev/null +++ b/models/tacotron2/NeMo/examples/tts/waveglow.py @@ -0,0 +1,34 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytorch_lightning as pl + +from nemo.collections.common.callbacks import LogEpochTimeCallback +from nemo.collections.tts.models import WaveGlowModel +from nemo.core.config import hydra_runner +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="waveglow") +def main(cfg): + trainer = pl.Trainer(**cfg.trainer) + exp_manager(trainer, cfg.get("exp_manager", None)) + model = WaveGlowModel(cfg=cfg.model, trainer=trainer) + epoch_time_logger = LogEpochTimeCallback() + trainer.callbacks.extend([epoch_time_logger]) + trainer.fit(model) + + +if __name__ == '__main__': + main() # noqa pylint: disable=no-value-for-parameter diff --git a/models/tacotron2/NeMo/external/get_collections.py b/models/tacotron2/NeMo/external/get_collections.py new file mode 100644 index 0000000000..803db2afd2 --- /dev/null +++ b/models/tacotron2/NeMo/external/get_collections.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Script responsible for generation of a JSON file with list of NeMo collections. """ + +import argparse +import importlib +import json +import os + +import nemo +from nemo.utils import logging + + +def process_collection(id, col): + """ Helper function processing the collection. + + Args: + id: (short) name of the collection. + col: a collection (python module). + """ + return { + "id": id, + "name": col.__name__, + "description": col.__description__, + "version": col.__version__, + "author": col.__author__, + } + + +def main(): + """ Main function generating a JSON file with list of NeMo collections. """ + # Parse filename. + parser = argparse.ArgumentParser() + parser.add_argument('--filename', help='Name of the output JSON file', type=str, default="collections.json") + args = parser.parse_args() + + # Get collections directory. + colletions_dir = os.path.dirname(nemo.collections.__file__) + logging.info('Analysing collections in `{}`'.format(colletions_dir)) + + # Generate list of NeMo collections - from the list of collection subfolders. + collections = {} + for sub_dir in os.listdir(colletions_dir): + # Skip cache. + if sub_dir == "__pycache__": + continue + # Check if it is a directory. + if os.path.isdir(os.path.join(colletions_dir, sub_dir)): + collections[sub_dir] = "nemo.collections." + sub_dir + + output_list = [] + # Iterate over all collections. + for key, val in collections.items(): + # Try to get module specification. + module_spec = importlib.util.find_spec(val) + if module_spec is None: + logging.warning(" * Failed to process `{}`".format(val)) + else: + try: + # Import the module from the module specification. + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) + # Add to list. + output_list.append(process_collection(key, module)) + logging.info(" * Processed `{}`".format(val)) + except AttributeError: + logging.warning(" * Failed to process `{}`".format(val)) + + # Export to JSON. + with open(args.filename, 'w') as outfile: + json.dump(output_list, outfile) + + logging.info('Finshed the analysis, results exported to `{}`.'.format(args.filename)) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/external/get_modules.py b/models/tacotron2/NeMo/external/get_modules.py new file mode 100644 index 0000000000..ba8f2f60d8 --- /dev/null +++ b/models/tacotron2/NeMo/external/get_modules.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Script responsible for generation of a JSON file containing list of modules of a given collection. """ + +import argparse +import importlib +import inspect +import json +import os + +import nemo +from nemo.utils import logging + + +def process_member(name, obj, module_list): + """ Helper function processing the passed object and, if ok, adding a record to the module list. + + Args: + name: name of the member + obj: member (class/function etc.) + module_list: list of modules that (probably) will be expanded. + """ + # It is not a class - skip it. + if not inspect.isclass(obj): + return + + # Check inheritance - we know that all our datasets/modules/losses inherit from Serialization, + # Btw. Serialization is also required by this script. + if not issubclass(obj, nemo.core.Serialization): + return + + logging.info(" * Processing `{}`".format(str(obj))) + + module_list.append( + { + "name": name, + "cls": str(obj), + # Temporary solution: mockup arguments. + "arguments": [ + "jasper", + "activation", + "feat_in", + "normalization_mode", + "residual_mode", + "norm_groups", + "conv_mask", + "frame_splicing", + "init_mode", + ], + # Temporary solution: mockup input types. + "input_types": { + "audio_signal": "axes: (batch, dimension, time); elements_type: MelSpectrogramType", + "length": "axes: (batch,); elements_type: LengthType", + }, + # Temporary solution: mockup output types. + "output_types": { + "encoder_output": "axes: (batch, dimension, time); elements_type: AcousticEncodedRepresentation" + }, + } + ) + + +def main(): + """ Main function analysing the indicated NeMo collection and generating a JSON file with module descriptions. """ + # Parse filename. + parser = argparse.ArgumentParser() + parser.add_argument('--collection', help='ID of the collection', type=str) + parser.add_argument('--filename', help='Name of the output JSON file', type=str, default="modules.json") + args = parser.parse_args() + + # Get collections directory. + colletions_dir = os.path.dirname(nemo.collections.__file__) + logging.info('Analysing collections in `{}`'.format(colletions_dir)) + + # Generate list of NeMo collections - from the list of collection subfolders. + collections = {} + for sub_dir in os.listdir(colletions_dir): + # Skip cache. + if sub_dir == "__pycache__": + continue + # Check if it is a directory. + if os.path.isdir(os.path.join(colletions_dir, sub_dir)): + collections[sub_dir] = "nemo.collections." + sub_dir + + # Check the collection. + if args.collection not in collections.keys(): + logging.error("Coudn't process the incidated `{}` collection".format(args.collection)) + logging.info( + "Please select one of the existing collections using `--collection [{}]`".format("|".join(collections)) + ) + exit(-1) + + # Load the collection specification. + collection_spec = importlib.util.find_spec(collections[args.collection]) + if collection_spec is None: + logging.error("Failed to load the `{}` collection".format(val)) + + # Import the module from the module specification. + collection = importlib.util.module_from_spec(collection_spec) + collection_spec.loader.exec_module(collection) + + module_list = [] + # Iterate over the packages in the indicated collection. + logging.info("Analysing the `{}` collection".format(args.collection)) + + try: # Datasets in dataset folder + logging.info("Analysing the 'data' package") + for name, obj in inspect.getmembers(collection.data): + process_member(name, obj, module_list) + except AttributeError as e: + logging.info(" * No datasets found") + + try: # Datasets in dataset folder + logging.info("Analysing the 'datasets' package") + for name, obj in inspect.getmembers(collection.datasets): + process_member(name, obj, module_list) + except AttributeError as e: + logging.info(" * No datasets found") + + try: # Modules + logging.info("Analysing the 'modules' package") + for name, obj in inspect.getmembers(collection.modules): + process_member(name, obj, module_list) + except AttributeError as e: + logging.info(" * No modules found") + + try: # Losses + logging.info("Analysing the 'losses' package") + for name, obj in inspect.getmembers(collection.losses): + process_member(name, obj, module_list) + except AttributeError as e: + logging.info(" * No losses found") + + # Add prefix - only for default name. + filename = args.filename if args.filename != "modules.json" else args.collection + "_" + args.filename + # Export to JSON. + with open(filename, 'w') as outfile: + json.dump(module_list, outfile) + + logging.info( + 'Finished analysis of the `{}` collection, results exported to `{}`.'.format(args.collection, filename) + ) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/nemo/README.md b/models/tacotron2/NeMo/nemo/README.md new file mode 100644 index 0000000000..2db456547a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/README.md @@ -0,0 +1,9 @@ +NeMo (**Ne**ural **Mo**dules) is a toolkit for creating AI applications built around **neural modules**, conceptual blocks of neural networks that take *typed* inputs and produce *typed* outputs. + +**NeMo Core** provides common APIs all modules and models have to implement. + +**NeMo Collections** + +* ASR - collection of modules and models for building speech recognition networks +* TTS - collection of modules and models for building speech synthesis networks +* NLP - collection of modules and models for building NLP networks diff --git a/models/tacotron2/NeMo/nemo/__init__.py b/models/tacotron2/NeMo/nemo/__init__.py new file mode 100644 index 0000000000..5b9fedbd1c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.package_info import ( + __contact_emails__, + __contact_names__, + __description__, + __download_url__, + __homepage__, + __keywords__, + __license__, + __package_name__, + __repository_url__, + __shortversion__, + __version__, +) diff --git a/models/tacotron2/NeMo/nemo/collections/__init__.py b/models/tacotron2/NeMo/nemo/collections/__init__.py new file mode 100644 index 0000000000..9e32500719 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/asr/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/__init__.py new file mode 100644 index 0000000000..cd14a43950 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.asr import data, losses, models, modules +from nemo.package_info import __version__ + +# Set collection version equal to NeMo version. +__version = __version__ + +# Authorship. +__author__ = "NVIDIA Corporation" + +# Set collection name. +__description__ = "Automatic Speech Recognition collection" diff --git a/models/tacotron2/NeMo/nemo/collections/asr/data/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/data/__init__.py new file mode 100644 index 0000000000..9e32500719 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_label.py b/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_label.py new file mode 100644 index 0000000000..5b4ed6d4e3 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_label.py @@ -0,0 +1,921 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import math +import os +from typing import Dict, List, Optional, Union + +import braceexpand +import torch +import webdataset as wd + +from nemo.collections.asr.parts.preprocessing.segment import available_formats as valid_sf_formats +from nemo.collections.common.parts.preprocessing import collections +from nemo.core.classes import Dataset, IterableDataset +from nemo.core.neural_types import AudioSignal, LabelsType, LengthsType, NeuralType, RegressionValuesType +from nemo.utils import logging + +# List of valid file formats (prioritized by order of importance) +VALID_FILE_FORMATS = ';'.join(['wav', 'mp3', 'flac'] + [fmt.lower() for fmt in valid_sf_formats.keys()]) + + +def repeat_signal(signal, sig_len, required_length): + """repeat signal to make short signal to have required_length + Args: + signal (FloatTensor): input signal + sig_len (LongTensor): length of input signal + required_length(float) : length of generated signal + Returns: + signal (FloatTensor): generated signal of required_length by repeating itself. + """ + repeat = required_length // sig_len + rem = required_length % sig_len + sub = signal[-rem:] if rem > 0 else torch.tensor([]) + rep_sig = torch.cat(repeat * [signal]) + signal = torch.cat((rep_sig, sub)) + return signal + + +def normalize(signal): + """normalize signal + Args: + signal(FloatTensor): signal to be normalized. + """ + signal_minusmean = signal - signal.mean() + return signal_minusmean / signal_minusmean.abs().max() + + +def count_occurence(manifest_file_id): + """Count number of wav files in Dict manifest_file_id. Use for _TarredAudioToLabelDataset. + Args: + manifest_file_id (Dict): Dict of files and their corresponding id. {'A-sub0' : 1, ..., 'S-sub10':100} + Returns: + count (Dict): Dict of wav files {'A' : 2, ..., 'S':10} + """ + count = dict() + for i in manifest_file_id: + audio_filename = i.split("-sub")[0] + count[audio_filename] = count.get(audio_filename, 0) + 1 + return count + + +def _speech_collate_fn(batch, pad_id): + """collate batch of audio sig, audio len, tokens, tokens len + Args: + batch (Optional[FloatTensor], Optional[LongTensor], LongTensor, + LongTensor): A tuple of tuples of signal, signal lengths, + encoded tokens, and encoded tokens length. This collate func + assumes the signals are 1d torch tensors (i.e. mono audio). + """ + _, audio_lengths, _, tokens_lengths = zip(*batch) + max_audio_len = 0 + has_audio = audio_lengths[0] is not None + if has_audio: + max_audio_len = max(audio_lengths).item() + max_tokens_len = max(tokens_lengths).item() + + audio_signal, tokens = [], [] + for sig, sig_len, tokens_i, tokens_i_len in batch: + if has_audio: + sig_len = sig_len.item() + if sig_len < max_audio_len: + pad = (0, max_audio_len - sig_len) + sig = torch.nn.functional.pad(sig, pad) + audio_signal.append(sig) + tokens_i_len = tokens_i_len.item() + if tokens_i_len < max_tokens_len: + pad = (0, max_tokens_len - tokens_i_len) + tokens_i = torch.nn.functional.pad(tokens_i, pad, value=pad_id) + tokens.append(tokens_i) + + if has_audio: + audio_signal = torch.stack(audio_signal) + audio_lengths = torch.stack(audio_lengths) + else: + audio_signal, audio_lengths = None, None + tokens = torch.stack(tokens) + tokens_lengths = torch.stack(tokens_lengths) + + return audio_signal, audio_lengths, tokens, tokens_lengths + + +def _fixed_seq_collate_fn(self, batch): + """collate batch of audio sig, audio len, tokens, tokens len + Args: + batch (Optional[FloatTensor], Optional[LongTensor], LongTensor, + LongTensor): A tuple of tuples of signal, signal lengths, + encoded tokens, and encoded tokens length. This collate func + assumes the signals are 1d torch tensors (i.e. mono audio). + """ + fixed_length = self.featurizer.sample_rate * self.time_length + _, audio_lengths, _, tokens_lengths = zip(*batch) + + has_audio = audio_lengths[0] is not None + fixed_length = int(min(fixed_length, max(audio_lengths))) + + audio_signal, tokens, new_audio_lengths = [], [], [] + for sig, sig_len, tokens_i, _ in batch: + if has_audio: + sig_len = sig_len.item() + chunck_len = sig_len - fixed_length + + if chunck_len < 0: + repeat = fixed_length // sig_len + rem = fixed_length % sig_len + sub = sig[-rem:] if rem > 0 else torch.tensor([]) + rep_sig = torch.cat(repeat * [sig]) + signal = torch.cat((rep_sig, sub)) + new_audio_lengths.append(torch.tensor(fixed_length)) + else: + start_idx = torch.randint(0, chunck_len, (1,)) if chunck_len else torch.tensor(0) + end_idx = start_idx + fixed_length + signal = sig[start_idx:end_idx] + new_audio_lengths.append(torch.tensor(fixed_length)) + + audio_signal.append(signal) + tokens.append(tokens_i) + + if has_audio: + audio_signal = torch.stack(audio_signal) + audio_lengths = torch.stack(new_audio_lengths) + else: + audio_signal, audio_lengths = None, None + tokens = torch.stack(tokens) + tokens_lengths = torch.stack(tokens_lengths) + + return audio_signal, audio_lengths, tokens, tokens_lengths + + +def _sliced_seq_collate_fn(self, batch): + """collate batch of audio sig, audio len, tokens, tokens len + Args: + batch (Optional[FloatTensor], Optional[LongTensor], LongTensor, + LongTensor): A tuple of tuples of signal, signal lengths, + encoded tokens, and encoded tokens length. This collate func + assumes the signals are 1d torch tensors (i.e. mono audio). + """ + slice_length = self.featurizer.sample_rate * self.time_length + _, audio_lengths, _, tokens_lengths = zip(*batch) + shift = self.shift_length * self.featurizer.sample_rate + has_audio = audio_lengths[0] is not None + + audio_signal, num_slices, tokens, audio_lengths = [], [], [], [] + for sig, sig_len, tokens_i, _ in batch: + if has_audio: + sig_len = sig_len.item() + dur = sig_len / self.featurizer.sample_rate + base = math.ceil((dur - self.time_length) / self.shift_length) + slice_length = int(slice_length) + shift = int(shift) + slices = 1 if base < 0 else base + 1 + for slice_id in range(slices): + start_idx = slice_id * shift + end_idx = start_idx + slice_length + signal = sig[start_idx:end_idx] + if len(signal) < slice_length: + signal = repeat_signal(signal, len(signal), slice_length) + audio_signal.append(signal) + + num_slices.append(slices) + tokens.extend([tokens_i] * slices) + audio_lengths.extend([slice_length] * slices) + + if has_audio: + audio_signal = torch.stack(audio_signal) + audio_lengths = torch.tensor(audio_lengths) + else: + audio_signal, audio_lengths = None, None + tokens = torch.stack(tokens) + tokens_lengths = torch.tensor(num_slices) # each embedding length + + return audio_signal, audio_lengths, tokens, tokens_lengths + + +def _vad_frame_seq_collate_fn(self, batch): + """collate batch of audio sig, audio len, tokens, tokens len + Args: + batch (Optional[FloatTensor], Optional[LongTensor], LongTensor, + LongTensor): A tuple of tuples of signal, signal lengths, + encoded tokens, and encoded tokens length. This collate func + assumes the signals are 1d torch tensors (i.e. mono audio). + batch size equals to 1. + """ + slice_length = int(self.featurizer.sample_rate * self.time_length) + _, audio_lengths, _, tokens_lengths = zip(*batch) + slice_length = min(slice_length, max(audio_lengths)) + shift = int(self.featurizer.sample_rate * self.shift_length) + has_audio = audio_lengths[0] is not None + + audio_signal, num_slices, tokens, audio_lengths = [], [], [], [] + + append_len_start = slice_length // 2 + append_len_end = slice_length - slice_length // 2 + for sig, sig_len, tokens_i, _ in batch: + if self.normalize_audio: + sig = normalize(sig) + start = torch.zeros(append_len_start) + end = torch.zeros(append_len_end) + sig = torch.cat((start, sig, end)) + sig_len += slice_length + + if has_audio: + slices = (sig_len - slice_length) // shift + for slice_id in range(slices): + start_idx = slice_id * shift + end_idx = start_idx + slice_length + signal = sig[start_idx:end_idx] + audio_signal.append(signal) + + num_slices.append(slices) + tokens.extend([tokens_i] * slices) + audio_lengths.extend([slice_length] * slices) + + if has_audio: + audio_signal = torch.stack(audio_signal) + audio_lengths = torch.tensor(audio_lengths) + else: + audio_signal, audio_lengths = None, None + + tokens = torch.stack(tokens) + tokens_lengths = torch.tensor(num_slices) + return audio_signal, audio_lengths, tokens, tokens_lengths + + +class _AudioLabelDataset(Dataset): + """ + Dataset that loads tensors via a json file containing paths to audio files, + labels, and durations and offsets(in seconds). Each new line is a + different sample. Example below: + and their target labels. JSON files should be of the following format:: + {"audio_filepath": "/path/to/audio_wav_0.wav", "duration": time_in_sec_0, "label": \ +target_label_0, "offset": offset_in_sec_0} + ... + {"audio_filepath": "/path/to/audio_wav_n.wav", "duration": time_in_sec_n, "label": \ +target_label_n, "offset": offset_in_sec_n} + Args: + manifest_filepath (str): Dataset parameter. Path to JSON containing data. + labels (list): Dataset parameter. List of target classes that can be output by the speaker recognition model. + featurizer + min_duration (float): Dataset parameter. All training files which have a duration less than min_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to 0.1. + max_duration (float): Dataset parameter. + All training files which have a duration more than max_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to None. + trim (bool): Whether to use trim silence from beginning and end of audio signal using librosa.effects.trim(). + Defaults to False. + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + + output_types = { + 'audio_signal': NeuralType( + ('B', 'T'), + AudioSignal(freq=self._sample_rate) + if self is not None and hasattr(self, '_sample_rate') + else AudioSignal(), + ), + 'a_sig_length': NeuralType(tuple('B'), LengthsType()), + } + + if self.is_regression_task: + output_types.update( + { + 'targets': NeuralType(tuple('B'), RegressionValuesType()), + 'targets_length': NeuralType(tuple('B'), LengthsType()), + } + ) + else: + + output_types.update( + {'label': NeuralType(tuple('B'), LabelsType()), 'label_length': NeuralType(tuple('B'), LengthsType()),} + ) + + return output_types + + def __init__( + self, + *, + manifest_filepath: str, + labels: List[str], + featurizer, + min_duration: Optional[float] = 0.1, + max_duration: Optional[float] = None, + trim: bool = False, + is_regression_task: bool = False, + ): + super().__init__() + self.collection = collections.ASRSpeechLabel( + manifests_files=manifest_filepath.split(','), + min_duration=min_duration, + max_duration=max_duration, + is_regression_task=is_regression_task, + ) + + self.featurizer = featurizer + self.trim = trim + self.is_regression_task = is_regression_task + + if not is_regression_task: + self.labels = labels if labels else self.collection.uniq_labels + self.num_classes = len(self.labels) if self.labels is not None else 1 + self.label2id, self.id2label = {}, {} + for label_id, label in enumerate(self.labels): + self.label2id[label] = label_id + self.id2label[label_id] = label + + for idx in range(len(self.labels[:5])): + logging.debug(" label id {} and its mapped label {}".format(idx, self.id2label[idx])) + + else: + self.labels = [] + self.num_classes = 1 + + def __len__(self): + return len(self.collection) + + def __getitem__(self, index): + sample = self.collection[index] + + offset = sample.offset + + if offset is None: + offset = 0 + + features = self.featurizer.process(sample.audio_file, offset=offset, duration=sample.duration, trim=self.trim) + f, fl = features, torch.tensor(features.shape[0]).long() + + if not self.is_regression_task: + t = torch.tensor(self.label2id[sample.label]).long() + else: + t = torch.tensor(sample.label).float() + + tl = torch.tensor(1).long() # For compatibility with collate_fn used later + + return f, fl, t, tl + + +# Ported from https://github.com/NVIDIA/OpenSeq2Seq/blob/master/open_seq2seq/data/speech2text/speech_commands.py +class AudioToClassificationLabelDataset(_AudioLabelDataset): + """ + Dataset that loads tensors via a json file containing paths to audio + files, command class, and durations (in seconds). Each new line is a + different sample. Example below: + {"audio_filepath": "/path/to/audio_wav_0.wav", "duration": time_in_sec_0, "label": \ + target_label_0, "offset": offset_in_sec_0} + ... + {"audio_filepath": "/path/to/audio_wav_n.wav", "duration": time_in_sec_n, "label": \ + target_label_n, "offset": offset_in_sec_n} + Args: + manifest_filepath: Path to manifest json as described above. Can + be comma-separated paths. + labels (Optional[list]): String containing all the possible labels to map to + if None then automatically picks from ASRSpeechLabel collection. + featurizer: Initialized featurizer class that converts paths of + audio to feature tensors + max_duration: If audio exceeds this length, do not include in dataset + min_duration: If audio is less than this length, do not include + in dataset + trim: Boolean flag whether to trim the audio + """ + + # self.labels = labels if labels else self.collection.uniq_labels + # self.num_commands = len(self.labels) + + def _collate_fn(self, batch): + return _speech_collate_fn(batch, pad_id=0) + + +class AudioToSpeechLabelDataset(_AudioLabelDataset): + """ + Dataset that loads tensors via a json file containing paths to audio + files, command class, and durations (in seconds). Each new line is a + different sample. Example below: + {"audio_filepath": "/path/to/audio_wav_0.wav", "duration": time_in_sec_0, "label": \ + target_label_0, "offset": offset_in_sec_0} + ... + {"audio_filepath": "/path/to/audio_wav_n.wav", "duration": time_in_sec_n, "label": \ + target_label_n, "offset": offset_in_sec_n} + Args: + manifest_filepath (str): Path to manifest json as described above. Can + be comma-separated paths. + labels (Optional[list]): String containing all the possible labels to map to + if None then automatically picks from ASRSpeechLabel collection. + min_duration (float): Dataset parameter. + All training files which have a duration less than min_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to 0.1. + max_duration (float): Dataset parameter. + All training files which have a duration more than max_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to None. + trim (bool): Whether to use trim silence from beginning and end + of audio signal using librosa.effects.trim(). + Defaults to False. + time_length (float): time length of slice (in seconds) + Use this for speaker recognition and VAD tasks. + shift_length (float): amount of shift of window for generating the frame for VAD task in a batch + Use this for VAD task during inference. + normalize_audio (bool): Whether to normalize audio signal. + Defaults to False. + is_regression_task (bool): Whether the dataset is for a regression task instead of classification + """ + + def __init__( + self, + *, + manifest_filepath: str, + labels: List[str], + featurizer, + min_duration: Optional[float] = 0.1, + max_duration: Optional[float] = None, + trim: bool = False, + time_length: Optional[float] = 8, + shift_length: Optional[float] = 1, + normalize_audio: bool = False, + is_regression_task: bool = False, + ): + logging.info("Time length considered for collate func is {}".format(time_length)) + logging.info("Shift length considered for collate func is {}".format(shift_length)) + self.time_length = time_length + self.shift_length = shift_length + self.normalize_audio = normalize_audio + + super().__init__( + manifest_filepath=manifest_filepath, + labels=labels, + featurizer=featurizer, + min_duration=min_duration, + max_duration=max_duration, + trim=trim, + is_regression_task=is_regression_task, + ) + + def fixed_seq_collate_fn(self, batch): + return _fixed_seq_collate_fn(self, batch) + + def sliced_seq_collate_fn(self, batch): + return _sliced_seq_collate_fn(self, batch) + + def vad_frame_seq_collate_fn(self, batch): + return _vad_frame_seq_collate_fn(self, batch) + + +class _TarredAudioLabelDataset(IterableDataset): + """ + A similar Dataset to the AudioLabelDataSet, but which loads tarred audio files. + + Accepts a single comma-separated JSON manifest file (in the same style as for the AudioToSpeechLabelDataset), + as well as the path(s) to the tarball(s) containing the wav files. Each line of the manifest should + contain the information for one audio file, including at least the label and name of the audio + file within the tarball. + + Valid formats for the audio_tar_filepaths argument include: + (1) a single string that can be brace-expanded, e.g. 'path/to/audio.tar' or 'path/to/audio_{1..100}.tar.gz', or + (2) a list of file paths that will not be brace-expanded, e.g. ['audio_1.tar', 'audio_2.tar', ...]. + + Note: For brace expansion in (1), there may be cases where `{x..y}` syntax cannot be used due to shell interference. + This occurs most commonly inside SLURM scripts. Therefore we provide a few equivalent replacements. + Supported opening braces - { <=> (, [, < and the special tag _OP_. + Supported closing braces - } <=> ), ], > and the special tag _CL_. + For SLURM based tasks, we suggest the use of the special tags for ease of use. + + See the documentation for more information about accepted data and input formats. + + If using multiple processes the number of shards should be divisible by the number of workers to ensure an + even split among workers. If it is not divisible, logging will give a warning but training will proceed. + In addition, if using mutiprocessing, each shard MUST HAVE THE SAME NUMBER OF ENTRIES after filtering + is applied. We currently do not check for this, but your program may hang if the shards are uneven! + + Notice that a few arguments are different from the AudioLabelDataSet; for example, shuffle (bool) has been + replaced by shuffle_n (int). + + Additionally, please note that the len() of this DataLayer is assumed to be the length of the manifest + after filtering. An incorrect manifest length may lead to some DataLoader issues down the line. + + Args: + audio_tar_filepaths: Either a list of audio tarball filepaths, or a + string (can be brace-expandable). + manifest_filepath (str): Path to the manifest. + labels (list): Dataset parameter. + List of target classes that can be output by the speaker recognition model. + featurizer + shuffle_n (int): How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + Defaults to 0. + min_duration (float): Dataset parameter. + All training files which have a duration less than min_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to 0.1. + max_duration (float): Dataset parameter. + All training files which have a duration more than max_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to None. + trim(bool): Whether to use trim silence from beginning and end + of audio signal using librosa.effects.trim(). + Defaults to False. + time_length (float): time length of slice (in seconds) # Pass this only for speaker recognition and VAD task + shift_length (float): amount of shift of window for generating the frame for VAD task. in a batch # Pass this only for VAD task during inference. + normalize_audio (bool): Whether to normalize audio signal. Defaults to False. + shard_strategy (str): Tarred dataset shard distribution strategy chosen as a str value during ddp. + - `scatter`: The default shard strategy applied by WebDataset, where each node gets + a unique set of shards, which are permanently pre-allocated and never changed at runtime. + - `replicate`: Optional shard strategy, where each node gets all of the set of shards + available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. + The benefit of replication is that it allows each node to sample data points from the entire + dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. + + Note: Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. + global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. + world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. + is_regression_task (bool): Whether it is a regression task. Defualts to False. + """ + + def __init__( + self, + *, + audio_tar_filepaths: Union[str, List[str]], + manifest_filepath: str, + labels: List[str], + featurizer, + shuffle_n: int = 0, + min_duration: Optional[float] = 0.1, + max_duration: Optional[float] = None, + trim: bool = False, + shard_strategy: str = "scatter", + global_rank: int = 0, + world_size: int = 0, + is_regression_task: bool = False, + ): + self.collection = collections.ASRSpeechLabel( + manifests_files=manifest_filepath.split(','), + min_duration=min_duration, + max_duration=max_duration, + index_by_file_id=True, # Must set this so the manifest lines can be indexed by file ID + ) + + self.file_occurence = count_occurence(self.collection.mapping) + + self.featurizer = featurizer + self.trim = trim + + self.labels = labels if labels else self.collection.uniq_labels + self.num_classes = len(self.labels) + + self.label2id, self.id2label = {}, {} + for label_id, label in enumerate(self.labels): + self.label2id[label] = label_id + self.id2label[label_id] = label + + for idx in range(len(self.labels[:5])): + logging.debug(" label id {} and its mapped label {}".format(idx, self.id2label[idx])) + + valid_shard_strategies = ['scatter', 'replicate'] + if shard_strategy not in valid_shard_strategies: + raise ValueError(f"`shard_strategy` must be one of {valid_shard_strategies}") + + if isinstance(audio_tar_filepaths, str): + # Replace '(' and '[' with '{' + brace_keys_open = ['(', '[', '<', '_OP_'] + for bkey in brace_keys_open: + if bkey in audio_tar_filepaths: + audio_tar_filepaths = audio_tar_filepaths.replace(bkey, "{") + + # Replace ')' and ']' with '}' + brace_keys_close = [')', ']', '>', '_CL_'] + for bkey in brace_keys_close: + if bkey in audio_tar_filepaths: + audio_tar_filepaths = audio_tar_filepaths.replace(bkey, "}") + + # Check for distributed and partition shards accordingly + if world_size > 1: + if isinstance(audio_tar_filepaths, str): + # Brace expand + audio_tar_filepaths = list(braceexpand.braceexpand(audio_tar_filepaths)) + + if shard_strategy == 'scatter': + logging.info("All tarred dataset shards will be scattered evenly across all nodes.") + + if len(audio_tar_filepaths) % world_size != 0: + logging.warning( + f"Number of shards in tarred dataset ({len(audio_tar_filepaths)}) is not divisible " + f"by number of distributed workers ({world_size})." + ) + + begin_idx = (len(audio_tar_filepaths) // world_size) * global_rank + end_idx = begin_idx + (len(audio_tar_filepaths) // world_size) + audio_tar_filepaths = audio_tar_filepaths[begin_idx:end_idx] + logging.info( + "Partitioning tarred dataset: process (%d) taking shards [%d, %d)", global_rank, begin_idx, end_idx + ) + + elif shard_strategy == 'replicate': + logging.info("All tarred dataset shards will be replicated across all nodes.") + + else: + raise ValueError(f"Invalid shard strategy ! Allowed values are : {valid_shard_strategies}") + + # Put together WebDataset + self._dataset = wd.WebDataset(urls=audio_tar_filepaths, nodesplitter=None) + + if shuffle_n > 0: + self._dataset = self._dataset.shuffle(shuffle_n) + else: + logging.info("WebDataset will not shuffle files within the tar files.") + + self._dataset = ( + self._dataset.rename(audio=VALID_FILE_FORMATS, key='__key__') + .to_tuple('audio', 'key') + .pipe(self._filter) + .map(f=self._build_sample) + ) + + def _filter(self, iterator): + """This function is used to remove samples that have been filtered out by ASRSpeechLabel already. + Otherwise, we would get a KeyError as _build_sample attempts to find the manifest entry for a sample + that was filtered out (e.g. for duration). + Note that if using multi-GPU training, filtering may lead to an imbalance in samples in each shard, + which may make your code hang as one process will finish before the other. + """ + + class TarredAudioFilter: + def __init__(self, collection, file_occurence): + self.iterator = iterator + self.collection = collection + self.file_occurence = file_occurence + self._iterable = None + + def __iter__(self): + self._iterable = self._internal_generator() + return self + + def __next__(self): + try: + values = next(self._iterable) + except StopIteration: + # reset generator + self._iterable = self._internal_generator() + values = next(self._iterable) + + return values + + def _internal_generator(self): + """ + WebDataset requires an Iterator, but we require an iterable that yields 1-or-more + values per value inside self.iterator. + + Therefore wrap the iterator with a generator function that will yield 1-or-more + values per sample in the iterator. + """ + for _, tup in enumerate(self.iterator): + audio_bytes, audio_filename = tup + + file_id, _ = os.path.splitext(os.path.basename(audio_filename)) + if audio_filename in self.file_occurence: + for j in range(0, self.file_occurence[file_id]): + if j == 0: + audio_filename = file_id + else: + audio_filename = file_id + "-sub" + str(j) + yield audio_bytes, audio_filename + + return TarredAudioFilter(self.collection, self.file_occurence) + + def _build_sample(self, tup): + """Builds the training sample by combining the data from the WebDataset with the manifest info. + """ + audio_bytes, audio_filename = tup + # Grab manifest entry from self.collection + file_id, _ = os.path.splitext(os.path.basename(audio_filename)) + + manifest_idx = self.collection.mapping[file_id] + manifest_entry = self.collection[manifest_idx] + + offset = manifest_entry.offset + if offset is None: + offset = 0 + + # Convert audio bytes to IO stream for processing (for SoundFile to read) + audio_filestream = io.BytesIO(audio_bytes) + features = self.featurizer.process( + audio_filestream, offset=offset, duration=manifest_entry.duration, trim=self.trim, + ) + + audio_filestream.close() + + # Audio features + f, fl = features, torch.tensor(features.shape[0]).long() + + t = self.label2id[manifest_entry.label] + tl = 1 # For compatibility with collate_fn used later + + return f, fl, torch.tensor(t).long(), torch.tensor(tl).long() + + def __iter__(self): + return self._dataset.__iter__() + + def __len__(self): + return len(self.collection) + + +class TarredAudioToClassificationLabelDataset(_TarredAudioLabelDataset): + """ + A similar Dataset to the AudioToClassificationLabelDataset, but which loads tarred audio files. + + Accepts a single comma-separated JSON manifest file (in the same style as for the AudioToClassificationLabelDataset), + as well as the path(s) to the tarball(s) containing the wav files. Each line of the manifest should + contain the information for one audio file, including at least the transcript and name of the audio + file within the tarball. + + Valid formats for the audio_tar_filepaths argument include: + (1) a single string that can be brace-expanded, e.g. 'path/to/audio.tar' or 'path/to/audio_{1..100}.tar.gz', or + (2) a list of file paths that will not be brace-expanded, e.g. ['audio_1.tar', 'audio_2.tar', ...]. + + See the WebDataset documentation for more information about accepted data and input formats. + + If using multiple processes the number of shards should be divisible by the number of workers to ensure an + even split among workers. If it is not divisible, logging will give a warning but training will proceed. + In addition, if using mutiprocessing, each shard MUST HAVE THE SAME NUMBER OF ENTRIES after filtering + is applied. We currently do not check for this, but your program may hang if the shards are uneven! + + Notice that a few arguments are different from the AudioToBPEDataset; for example, shuffle (bool) has been + replaced by shuffle_n (int). + + Additionally, please note that the len() of this DataLayer is assumed to be the length of the manifest + after filtering. An incorrect manifest length may lead to some DataLoader issues down the line. + + Args: + audio_tar_filepaths: Either a list of audio tarball filepaths, or a + string (can be brace-expandable). + manifest_filepath (str): Path to the manifest. + labels (list): Dataset parameter. + List of target classes that can be output by the speaker recognition model. + featurizer + shuffle_n (int): How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + Defaults to 0. + min_duration (float): Dataset parameter. + All training files which have a duration less than min_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to 0.1. + max_duration (float): Dataset parameter. + All training files which have a duration more than max_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to None. + trim(bool): Whether to use trim silence from beginning and end + of audio signal using librosa.effects.trim(). + Defaults to False. + shard_strategy (str): Tarred dataset shard distribution strategy chosen as a str value during ddp. + - `scatter`: The default shard strategy applied by WebDataset, where each node gets + a unique set of shards, which are permanently pre-allocated and never changed at runtime. + - `replicate`: Optional shard strategy, where each node gets all of the set of shards + available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. + The benefit of replication is that it allows each node to sample data points from the entire + dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. + + Note: Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. + global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. + world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. + is_regression_task (bool): Whether it is a regression task. Defualts to False. + """ + + # self.labels = labels if labels else self.collection.uniq_labels + # self.num_commands = len(self.labels) + + def _collate_fn(self, batch): + return _speech_collate_fn(batch, pad_id=0) + + +class TarredAudioToSpeechLabelDataset(_TarredAudioLabelDataset): + """ + A similar Dataset to the AudioToSpeechLabelDataset, but which loads tarred audio files. + + Accepts a single comma-separated JSON manifest file (in the same style as for the AudioToSpeechLabelDataset), + as well as the path(s) to the tarball(s) containing the wav files. Each line of the manifest should + contain the information for one audio file, including at least the transcript and name of the audio + file within the tarball. + + Valid formats for the audio_tar_filepaths argument include: + (1) a single string that can be brace-expanded, e.g. 'path/to/audio.tar' or 'path/to/audio_{1..100}.tar.gz', or + (2) a list of file paths that will not be brace-expanded, e.g. ['audio_1.tar', 'audio_2.tar', ...]. + + See the WebDataset documentation for more information about accepted data and input formats. + + If using multiple processes the number of shards should be divisible by the number of workers to ensure an + even split among workers. If it is not divisible, logging will give a warning but training will proceed. + In addition, if using mutiprocessing, each shard MUST HAVE THE SAME NUMBER OF ENTRIES after filtering + is applied. We currently do not check for this, but your program may hang if the shards are uneven! + + Notice that a few arguments are different from the AudioToBPEDataset; for example, shuffle (bool) has been + replaced by shuffle_n (int). + + Additionally, please note that the len() of this DataLayer is assumed to be the length of the manifest + after filtering. An incorrect manifest length may lead to some DataLoader issues down the line. + + Args: + audio_tar_filepaths: Either a list of audio tarball filepaths, or a + string (can be brace-expandable). + manifest_filepath (str): Path to the manifest. + labels (list): Dataset parameter. + List of target classes that can be output by the speaker recognition model. + featurizer + shuffle_n (int): How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + Defaults to 0. + min_duration (float): Dataset parameter. + All training files which have a duration less than min_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to 0.1. + max_duration (float): Dataset parameter. + All training files which have a duration more than max_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to None. + trim(bool): Whether to use trim silence from beginning and end + of audio signal using librosa.effects.trim(). + Defaults to False. + time_length (float): time length of slice (in seconds) # Pass this only for speaker recognition and VAD task + shift_length (float): amount of shift of window for generating the frame for VAD task. in a batch # Pass this only for VAD task during inference. + normalize_audio (bool): Whether to normalize audio signal. Defaults to False. + shard_strategy (str): Tarred dataset shard distribution strategy chosen as a str value during ddp. + - `scatter`: The default shard strategy applied by WebDataset, where each node gets + a unique set of shards, which are permanently pre-allocated and never changed at runtime. + - `replicate`: Optional shard strategy, where each node gets all of the set of shards + available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. + The benefit of replication is that it allows each node to sample data points from the entire + dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. + + Note: Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. + global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. + world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. + """ + + def __init__( + self, + *, + audio_tar_filepaths: Union[str, List[str]], + manifest_filepath: str, + labels: List[str], + featurizer, + shuffle_n: int = 0, + min_duration: Optional[float] = 0.1, + max_duration: Optional[float] = None, + trim: bool = False, + time_length: Optional[float] = 8, + shift_length: Optional[float] = 1, + normalize_audio: bool = False, + shard_strategy: str = "scatter", + global_rank: int = 0, + world_size: int = 0, + ): + logging.info("Time length considered for collate func is {}".format(time_length)) + logging.info("Shift length considered for collate func is {}".format(shift_length)) + self.time_length = time_length + self.shift_length = shift_length + self.normalize_audio = normalize_audio + + super().__init__( + audio_tar_filepaths=audio_tar_filepaths, + manifest_filepath=manifest_filepath, + labels=labels, + featurizer=featurizer, + shuffle_n=shuffle_n, + min_duration=min_duration, + max_duration=max_duration, + trim=trim, + shard_strategy=shard_strategy, + global_rank=global_rank, + world_size=world_size, + ) + + def fixed_seq_collate_fn(self, batch): + return _fixed_seq_collate_fn(self, batch) + + def sliced_seq_collate_fn(self, batch): + return _sliced_seq_collate_fn(self, batch) + + def vad_frame_seq_collate_fn(self, batch): + return _vad_frame_seq_collate_fn(self, batch) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_label_dataset.py b/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_label_dataset.py new file mode 100644 index 0000000000..c14818d0c0 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_label_dataset.py @@ -0,0 +1,129 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.asr.data import audio_to_label + + +def get_classification_label_dataset(featurizer, config: dict) -> audio_to_label.AudioToClassificationLabelDataset: + """ + Instantiates a Classification AudioLabelDataset. + + Args: + config: Config of the AudioToClassificationLabelDataset. + + Returns: + An instance of AudioToClassificationLabelDataset. + """ + dataset = audio_to_label.AudioToClassificationLabelDataset( + manifest_filepath=config['manifest_filepath'], + labels=config['labels'], + featurizer=featurizer, + max_duration=config.get('max_duration', None), + min_duration=config.get('min_duration', None), + trim=config.get('trim_silence', False), + is_regression_task=config.get('is_regression_task', False), + ) + return dataset + + +def get_speech_label_dataset(featurizer, config: dict) -> audio_to_label.AudioToSpeechLabelDataset: + """ + Instantiates a Speech Label (e.g. VAD, speaker recognition) AudioLabelDataset. + + Args: + config: Config of the AudioToSpeechLabelDataSet. + + Returns: + An instance of AudioToSpeechLabelDataset. + """ + dataset = audio_to_label.AudioToSpeechLabelDataset( + manifest_filepath=config['manifest_filepath'], + labels=config['labels'], + featurizer=featurizer, + max_duration=config.get('max_duration', None), + min_duration=config.get('min_duration', None), + trim=config.get('trim_silence', False), + time_length=config.get('time_length', 0.31), + shift_length=config.get('shift_length', 0.01), + normalize_audio=config.get('normalize_audio', False), + ) + return dataset + + +def get_tarred_classification_label_dataset( + featurizer, config: dict, shuffle_n: int, global_rank: int, world_size: int +) -> audio_to_label.TarredAudioToClassificationLabelDataset: + """ + Instantiates a Classification TarredAudioLabelDataset. + + Args: + config: Config of the TarredAudioToClassificationLabelDataset. + shuffle_n: How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + global_rank: Global rank of this device. + world_size: Global world size in the training method. + + Returns: + An instance of TarredAudioToClassificationLabelDataset. + """ + dataset = audio_to_label.TarredAudioToClassificationLabelDataset( + audio_tar_filepaths=config['tarred_audio_filepaths'], + manifest_filepath=config['manifest_filepath'], + labels=config['labels'], + featurizer=featurizer, + shuffle_n=shuffle_n, + max_duration=config.get('max_duration', None), + min_duration=config.get('min_duration', None), + trim=config.get('trim_silence', False), + shard_strategy=config.get('tarred_shard_strategy', 'scatter'), + global_rank=global_rank, + world_size=world_size, + is_regression_task=config.get('is_regression_task', False), + ) + return dataset + + +def get_tarred_speech_label_dataset( + featurizer, config: dict, shuffle_n: int, global_rank: int, world_size: int, +) -> audio_to_label.TarredAudioToSpeechLabelDataset: + """ + InInstantiates a Speech Label (e.g. VAD, speaker recognition) TarredAudioLabelDataset. + + Args: + config: Config of the TarredAudioToSpeechLabelDataset. + shuffle_n: How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + global_rank: Global rank of this device. + world_size: Global world size in the training method. + + Returns: + An instance of TarredAudioToSpeechLabelDataset. + """ + dataset = audio_to_label.TarredAudioToSpeechLabelDataset( + audio_tar_filepaths=config['tarred_audio_filepaths'], + manifest_filepath=config['manifest_filepath'], + labels=config['labels'], + featurizer=featurizer, + shuffle_n=shuffle_n, + max_duration=config.get('max_duration', None), + min_duration=config.get('min_duration', None), + trim=config.get('trim_silence', False), + time_length=config.get('time_length', 0.31), + shift_length=config.get('shift_length', 0.01), + normalize_audio=config.get('normalize_audio', False), + shard_strategy=config.get('tarred_shard_strategy', 'scatter'), + global_rank=global_rank, + world_size=world_size, + ) + return dataset diff --git a/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text.py b/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text.py new file mode 100644 index 0000000000..cdff89d76f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text.py @@ -0,0 +1,1330 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import math +import os +from pathlib import Path +from typing import Callable, Dict, List, Optional, Union + +import braceexpand +import librosa +import numpy as np +import torch +import webdataset as wd +from scipy.stats import betabinom +from torch.nn import functional as F + +from nemo.collections.asr.data import vocabs +from nemo.collections.asr.parts.preprocessing.features import WaveformFeaturizer +from nemo.collections.common.parts.preprocessing import collections, parsers +from nemo.core.classes import Dataset, IterableDataset +from nemo.core.neural_types import * +from nemo.core.neural_types.elements import ProbsType +from nemo.utils import logging + +__all__ = [ + 'AudioToCharDataset', + 'AudioToCharWithDursF0Dataset', + 'AudioToCharWithPriorDataset', + 'AudioToBPEDataset', + 'TarredAudioToCharDataset', + 'TarredAudioToBPEDataset', +] + + +def _speech_collate_fn(batch, pad_id): + """collate batch of audio sig, audio len, tokens, tokens len + Args: + batch (Optional[FloatTensor], Optional[LongTensor], LongTensor, + LongTensor): A tuple of tuples of signal, signal lengths, + encoded tokens, and encoded tokens length. This collate func + assumes the signals are 1d torch tensors (i.e. mono audio). + """ + _, audio_lengths, _, tokens_lengths = zip(*batch) + max_audio_len = 0 + has_audio = audio_lengths[0] is not None + if has_audio: + max_audio_len = max(audio_lengths).item() + max_tokens_len = max(tokens_lengths).item() + + audio_signal, tokens = [], [] + for sig, sig_len, tokens_i, tokens_i_len in batch: + if has_audio: + sig_len = sig_len.item() + if sig_len < max_audio_len: + pad = (0, max_audio_len - sig_len) + sig = torch.nn.functional.pad(sig, pad) + audio_signal.append(sig) + tokens_i_len = tokens_i_len.item() + if tokens_i_len < max_tokens_len: + pad = (0, max_tokens_len - tokens_i_len) + tokens_i = torch.nn.functional.pad(tokens_i, pad, value=pad_id) + tokens.append(tokens_i) + + if has_audio: + audio_signal = torch.stack(audio_signal) + audio_lengths = torch.stack(audio_lengths) + else: + audio_signal, audio_lengths = None, None + tokens = torch.stack(tokens) + tokens_lengths = torch.stack(tokens_lengths) + + return audio_signal, audio_lengths, tokens, tokens_lengths + + +class _AudioTextDataset(Dataset): + """ + Dataset that loads tensors via a json file containing paths to audio files, transcripts, and durations (in seconds). + Each new line is a different sample. Example below: + {"audio_filepath": "/path/to/audio.wav", "text_filepath": "/path/to/audio.txt", "duration": 23.147} + ... + {"audio_filepath": "/path/to/audio.wav", "text": "the transcription", "offset": 301.75, "duration": 0.82, "utt": + "utterance_id", "ctm_utt": "en_4156", "side": "A"} + Args: + manifest_filepath: Path to manifest json as described above. Can be comma-separated paths. + labels: String containing all the possible characters to map to + sample_rate (int): Sample rate to resample loaded audio to + int_values (bool): If true, load samples as 32-bit integers. Defauts to False. + augmentor (nemo.collections.asr.parts.perturb.AudioAugmentor): An AudioAugmentor object used to augment loaded + audio + max_duration: If audio exceeds this length, do not include in dataset + min_duration: If audio is less than this length, do not include in dataset + max_utts: Limit number of utterances + blank_index: blank character index, default = -1 + unk_index: unk_character index, default = -1 + normalize: whether to normalize transcript text (default): True + bos_id: Id of beginning of sequence symbol to append if not None + eos_id: Id of end of sequence symbol to append if not None + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + 'audio_signal': NeuralType(('B', 'T'), AudioSignal()), + 'a_sig_length': NeuralType(tuple('B'), LengthsType()), + 'transcripts': NeuralType(('B', 'T'), LabelsType()), + 'transcript_length': NeuralType(tuple('B'), LengthsType()), + } + + def __init__( + self, + manifest_filepath: str, + parser: Union[str, Callable], + sample_rate: int, + int_values: bool = False, + augmentor: 'nemo.collections.asr.parts.perturb.AudioAugmentor' = None, + max_duration: Optional[int] = None, + min_duration: Optional[int] = None, + max_utts: int = 0, + trim: bool = False, + bos_id: Optional[int] = None, + eos_id: Optional[int] = None, + pad_id: int = 0, + ): + self.parser = parser + + self.collection = collections.ASRAudioText( + manifests_files=manifest_filepath.split(','), + parser=parser, + min_duration=min_duration, + max_duration=max_duration, + max_number=max_utts, + ) + + self.featurizer = WaveformFeaturizer(sample_rate=sample_rate, int_values=int_values, augmentor=augmentor) + self.trim = trim + self.eos_id = eos_id + self.bos_id = bos_id + self.pad_id = pad_id + + def __getitem__(self, index): + sample = self.collection[index] + offset = sample.offset + + if offset is None: + offset = 0 + + features = self.featurizer.process( + sample.audio_file, offset=offset, duration=sample.duration, trim=self.trim, orig_sr=sample.orig_sr + ) + f, fl = features, torch.tensor(features.shape[0]).long() + + t, tl = sample.text_tokens, len(sample.text_tokens) + if self.bos_id is not None: + t = [self.bos_id] + t + tl += 1 + if self.eos_id is not None: + t = t + [self.eos_id] + tl += 1 + + output = f, fl, torch.tensor(t).long(), torch.tensor(tl).long() + + return output + + def __len__(self): + return len(self.collection) + + def _collate_fn(self, batch): + return _speech_collate_fn(batch, pad_id=self.pad_id) + + +class AudioToCharDataset(_AudioTextDataset): + """ + Dataset that loads tensors via a json file containing paths to audio + files, transcripts, and durations (in seconds). Each new line is a + different sample. Example below: + {"audio_filepath": "/path/to/audio.wav", "text_filepath": + "/path/to/audio.txt", "duration": 23.147} + ... + {"audio_filepath": "/path/to/audio.wav", "text": "the + transcription", "offset": 301.75, "duration": 0.82, "utt": + "utterance_id", "ctm_utt": "en_4156", "side": "A"} + Args: + manifest_filepath: Path to manifest json as described above. Can + be comma-separated paths. + labels: String containing all the possible characters to map to + sample_rate (int): Sample rate to resample loaded audio to + int_values (bool): If true, load samples as 32-bit integers. Defauts to False. + augmentor (nemo.collections.asr.parts.perturb.AudioAugmentor): An AudioAugmentor + object used to augment loaded audio + max_duration: If audio exceeds this length, do not include in dataset + min_duration: If audio is less than this length, do not include + in dataset + max_utts: Limit number of utterances + blank_index: blank character index, default = -1 + unk_index: unk_character index, default = -1 + normalize: whether to normalize transcript text (default): True + bos_id: Id of beginning of sequence symbol to append if not None + eos_id: Id of end of sequence symbol to append if not None + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + 'audio_signal': NeuralType(('B', 'T'), AudioSignal()), + 'a_sig_length': NeuralType(tuple('B'), LengthsType()), + 'transcripts': NeuralType(('B', 'T'), LabelsType()), + 'transcript_length': NeuralType(tuple('B'), LengthsType()), + } + + def __init__( + self, + manifest_filepath: str, + labels: Union[str, List[str]], + sample_rate: int, + int_values: bool = False, + augmentor: 'nemo.collections.asr.parts.perturb.AudioAugmentor' = None, + max_duration: Optional[float] = None, + min_duration: Optional[float] = None, + max_utts: int = 0, + blank_index: int = -1, + unk_index: int = -1, + normalize: bool = True, + trim: bool = False, + bos_id: Optional[int] = None, + eos_id: Optional[int] = None, + pad_id: int = 0, + parser: Union[str, Callable] = 'en', + ): + self.labels = labels + + parser = parsers.make_parser( + labels=labels, name=parser, unk_id=unk_index, blank_id=blank_index, do_normalize=normalize + ) + + super().__init__( + manifest_filepath=manifest_filepath, + parser=parser, + sample_rate=sample_rate, + int_values=int_values, + augmentor=augmentor, + max_duration=max_duration, + min_duration=min_duration, + max_utts=max_utts, + trim=trim, + bos_id=bos_id, + eos_id=eos_id, + pad_id=pad_id, + ) + + +class AudioToCharWithDursF0Dataset(AudioToCharDataset): + """ + Dataset that loads tensors via a json file containing paths to audio + files, transcripts, and durations (in seconds). + Each new line is a different sample. Example below: + {"audio_filepath": "/path/to/audio_1.wav", "text": "the + transcription", "offset": 301.75, "duration": 0.82} + ... + {"audio_filepath": "/path/to/audio_n.wav", "text": "the + transcription", "offset": 301.75, "duration": 0.82} + + Additionally, user provides path to precomputed durations via "durs_file" arg, which is a pickled python dict, + mapping example tag to it's durations tensor, and name of method that was received durations via "durs_type". + Tag is a unique example identifier, which is a wav filename without suffix. Durations depend on "durs_type". + If durs_type == "asr_based" then durations are an additional dict of two tensors: graphemes/phonemes durations and blanks durations with + "blanks" and "tokens" keys respectively. + Example below: + {'LJ050-0234': {'blanks': `blanks_durs_tensor`, 'tokens': `tokens_durs_tensor`}, + ...} + If durs_type == "aligner_based" then durations are just tensor graphemes/phonemes durations. + Example below: + {'LJ050-0234': `tokens_durs_tensor`}, + ...} + + Additionally, F0 statistics is passed precomputed along mel length via "f0_file" arg, which is a pickled python + dict, mapping example tag to it's f0 tensor. + Example below: + {'LJ050-0234': `f0_tensor`, + ...} + + Args: + **kwargs: Passed to AudioToCharDataset constructor. + durs_file (str): String path to pickled durations location. + durs_type (str): Type of durations. Currently supported durations are "asr-based" and "aligned-based". + f0_file (str): String path to pickled f0 statistics location. + blanking (bool): Boolean flag indicate whether add or not blanks between text graphemes. + load_audio(bool): Boolean flag indicate whether do or not load audio. + vocab: Vocabulary config (parser + set of graphemes to use). Constructor propagates these to + `self.make_vocab` function call to build a complete vocabulary. + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports.""" + return { + 'audio': NeuralType(('B', 'T'), AudioSignal()), + 'audio_len': NeuralType(('B',), LengthsType()), + 'text': NeuralType(('B', 'T'), LabelsType()), + 'text_len': NeuralType(('B',), LengthsType()), + 'durs': NeuralType(('B', 'T'), LengthsType()), + 'f0': NeuralType(('B', 'T'), FloatType()), + 'f0_mask': NeuralType(('B', 'T'), MaskType()), + } + + @staticmethod + def make_vocab( + notation='chars', + punct=True, + spaces=False, + stresses=False, + chars=False, + add_blank_at="last_but_one", + pad_with_space=False, + improved_version_g2p=False, + phoneme_dict_path=None, + ): + """Constructs vocabulary from given parameters. + + Args: + notation (str): Either 'chars' or 'phonemes' as general notation. + punct (bool): True if reserve grapheme for basic punctuation. + spaces (bool): True if prepend spaces to every punctuation symbol. + stresses (bool): True if use phonemes codes with stresses (0-2). + chars (bool): True if additionally use chars together with phonemes. + add_blank_at: add blank to labels in the specified order ("last" or "last_but_one"), + if None then no blank in labels. + pad_with_space (bool): True if pad text with spaces at the beginning and at the end. + improved_version_g2p (bool): True if use new version of g2p. + phoneme_dict_path (str): path to phoneme dict file (like CMU Pronouncing dictionary). If it's None then cmudict.dict() will be used. + + Returns: + (vocabs.Base) Vocabulary + """ + if notation == 'chars': + vocab = vocabs.Chars(punct=punct, spaces=spaces, add_blank_at=add_blank_at) + elif notation == 'phonemes': + vocab = vocabs.Phonemes( + punct=punct, + stresses=stresses, + spaces=spaces, + chars=chars, + add_blank_at=add_blank_at, + pad_with_space=pad_with_space, + improved_version_g2p=improved_version_g2p, + phoneme_dict_path=phoneme_dict_path, + ) + else: + raise ValueError("Unsupported vocab type.") + return vocab + + def __init__(self, **kwargs): + durs_file = kwargs.pop('durs_file', None) + durs_type = kwargs.pop('durs_type', "asr-based") + f0_file = kwargs.pop('f0_file', None) + self.blanking = kwargs.pop('blanking', False) + self.load_audio = kwargs.pop('load_audio', True) + self.vocab = self.make_vocab(**kwargs.pop('vocab', {})) + kwargs.setdefault('labels', []) # For compatibility. + super().__init__(**kwargs) + + tags = [] + self.id2enc_text = {} + for i, e in enumerate(self.collection): + tag = os.path.splitext(os.path.basename(e.audio_file))[0] + tags.append(tag) + # cache vocab encoding + self.id2enc_text[i] = self.vocab.encode(e.text_raw) + + if durs_file: + tag2durs = torch.load(durs_file) + durs = [] + for tag in tags: + tag_durs = tag2durs[tag] + if durs_type == "asr-based": + durs.append(self.interleave(tag_durs['blanks'], tag_durs['tokens'])) + elif durs_type == "aligner-based": + durs.append(tag_durs) + else: + raise NotImplementedError( + f"{durs_type} duration type is not supported. Use asr-based or align-based." + ) + self.durs = durs + if f0_file: + tag2f0 = torch.load(f0_file) + self.f0 = [tag2f0[tag] for tag in tags] + + def __getitem__(self, item): + audio, audio_len = None, None + if self.load_audio: + audio, audio_len, _, _ = super().__getitem__(item) # noqa + + text = self.id2enc_text[item] + text, text_len = torch.tensor(text).long(), torch.tensor(len(text)).long() + durs, f0 = self.durs[item], self.f0[item] + return ( + audio, + audio_len, + text, + text_len, + durs, + f0, + ) + + @staticmethod + def merge(tensors, dim=0, value=0, dtype=None): + """Merges list of tensors into one.""" + tensors = [tensor if isinstance(tensor, torch.Tensor) else torch.tensor(tensor) for tensor in tensors] + dim = dim if dim != -1 else len(tensors[0].shape) - 1 + dtype = tensors[0].dtype if dtype is None else dtype + max_len = max(tensor.shape[dim] for tensor in tensors) + new_tensors = [] + for tensor in tensors: + pad = (2 * len(tensor.shape)) * [0] + pad[-2 * dim - 1] = max_len - tensor.shape[dim] + new_tensors.append(F.pad(tensor, pad=pad, value=value)) + return torch.stack(new_tensors).to(dtype=dtype) + + @classmethod + def repeat_merge(cls, x, reps, pad): + """Repeats `x` values according to `reps` tensor and merges.""" + return cls.merge( + tensors=[torch.repeat_interleave(text1, durs1) for text1, durs1 in zip(x, reps)], value=pad, dtype=x.dtype, + ) + + @staticmethod + def make_mask(lengths, max_length=None): + """Makes mask from list of lengths.""" + device = lengths.device if torch.is_tensor(lengths) else 'cpu' + lengths = lengths if torch.is_tensor(lengths) else torch.tensor(lengths) + max_length = max_length or torch.max(lengths) + start = torch.tensor(0).int() + indices = torch.arange(start=start, end=max_length, device=device) # noqa + mask = indices.lt(lengths.view(-1, 1)) + + return mask + + @staticmethod + def interleave(x, y): + """Interleave two tensors.""" + xy = torch.stack([x[:-1], y], dim=1).view(-1) + xy = F.pad(xy, pad=[0, 1], value=x[-1]) + return xy + + def _collate_fn(self, batch): + batch = list(zip(*batch)) + + asr_batch = _speech_collate_fn(list(zip(*batch[:4])), pad_id=self.vocab.pad) + audio, audio_len, text, text_len = asr_batch + + if self.blanking: + text = [ + self.interleave( + x=torch.empty(len(t) + 1, dtype=torch.long, device=t.device).fill_(self.vocab.blank), y=t, + ) + for t in text + ] + text = self.merge(text, value=self.vocab.pad, dtype=torch.long) + text_len = text_len * 2 + 1 + + durs, f0 = batch[4:] + durs = self.merge(durs, dtype=torch.long) + f0_mask = self.make_mask([f.shape[-1] for f in f0]) # noqa + f0 = self.merge(f0, dtype=torch.float) + + return ( + audio, + audio_len, + text, + text_len, + durs, + f0, + f0_mask, + ) + + +class AudioToCharWithPriorDataset(AudioToCharDataset): + """ + Dataset that loads tensors via a json file containing paths to audio + files, transcripts, durations (in seconds). + Each new line is a different sample. Example below: + {"audio_filepath": "/path/to/audio_1.wav", "text": "the + transcription", "offset": 301.75, "duration": 0.82} + ... + {"audio_filepath": "/path/to/audio_n.wav", "text": "the + transcription", "offset": 301.75, "duration": 0.82} + + Additionally, user provides path to folder with precomputed attention priors via "attn_prior_folder" arg. + This folder should contain saved numpy array as attention prior along mel and text lengths. + If folder is empty, attention priors will be calculated and saved in specified folder. + Every name of saved numpy array is a unique example identifier, which is a wav filename without suffix. + + Args: + **kwargs: Passed to AudioToCharDataset constructor. + attn_prior_folder (str): String path to folder with precomputed attention priors. + n_window_stride (int): Stride of window for fft in samples. Need to generate prior. + vocab: Vocabulary config (parser + set of graphemes to use). Constructor propagates these to + `self.make_vocab` function call to build a complete vocabulary. + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports.""" + return { + 'audio': NeuralType(('B', 'T'), AudioSignal()), + 'audio_len': NeuralType(('B',), LengthsType()), + 'text': NeuralType(('B', 'T'), LabelsType()), + 'text_len': NeuralType(('B',), LengthsType()), + 'attn_prior': NeuralType(('B', 'T', 'D'), ProbsType()), + } + + def __init__(self, attn_prior_folder, n_window_stride=256, **kwargs): + self.vocab = AudioToCharWithDursF0Dataset.make_vocab(**kwargs.pop('vocab', {})) + kwargs.setdefault('labels', []) # For compatibility. + super().__init__(**kwargs) + + Path(attn_prior_folder).mkdir(parents=True, exist_ok=True) + self.attn_prior_folder = attn_prior_folder + + self.n_window_stride = n_window_stride + self.id2enc_text = {} + for i, e in enumerate(self.collection): + # cache vocab encoding + self.id2enc_text[i] = self.vocab.encode(e.text_raw) + + @staticmethod + def beta_binomial_prior_distribution(phoneme_count, mel_count, scaling_factor=1.0): + x = np.arange(0, phoneme_count) + mel_text_probs = [] + for i in range(1, mel_count + 1): + a, b = scaling_factor * i, scaling_factor * (mel_count + 1 - i) + mel_i_prob = betabinom(phoneme_count, a, b).pmf(x) + mel_text_probs.append(mel_i_prob) + return np.array(mel_text_probs) + + def __getitem__(self, item): + audio, audio_len, _, _ = super().__getitem__(item) # noqa + + text = self.id2enc_text[item] + + attn_prior_path = ( + Path(self.attn_prior_folder) + / f"ap_tl{len(text)}_al_{math.ceil((audio_len.item() + 1) / self.n_window_stride)}.npy" + ) + + if attn_prior_path.exists(): + attn_prior = np.load(attn_prior_path) + else: + attn_prior = self.beta_binomial_prior_distribution( + len(text), math.ceil((audio_len.item() + 1) / self.n_window_stride) + ) + np.save(attn_prior_path, attn_prior) + + text, text_len, attn_prior = ( + torch.tensor(text).long(), + torch.tensor(len(text)).long(), + torch.tensor(attn_prior), + ) + + return audio, audio_len, text, text_len, attn_prior + + def _collate_fn(self, batch): + batch = list(zip(*batch)) + + asr_batch = _speech_collate_fn(list(zip(*batch[:4])), pad_id=self.vocab.pad) + audio, audio_len, text, text_len = asr_batch + attn_prior_list = batch[4] + + attn_prior = torch.zeros( + len(attn_prior_list), + max([attn_prior_i.shape[0] for attn_prior_i in attn_prior_list]), + max([attn_prior_i.shape[1] for attn_prior_i in attn_prior_list]), + ) + + for i, attn_prior_i in enumerate(attn_prior_list): + attn_prior[i, : attn_prior_i.shape[0], : attn_prior_i.shape[1]] = attn_prior_i + + return audio, audio_len, text, text_len, attn_prior + + +class AudioToCharWithPriorAndPitchDataset(AudioToCharWithPriorDataset): + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports.""" + return { + 'audio': NeuralType(('B', 'T'), AudioSignal()), + 'audio_len': NeuralType(('B',), LengthsType()), + 'text': NeuralType(('B', 'T'), LabelsType()), + 'text_len': NeuralType(('B',), LengthsType()), + 'attn_prior': NeuralType(('B', 'T', 'D'), ProbsType()), + 'pitch': NeuralType(('B', 'T'), RegressionValuesType()), + } + + def __init__(self, sup_data_path, pitch_fmin, pitch_fmax, n_window_size, pitch_avg, pitch_std, **kwargs): + self.pitch_fmin = pitch_fmin + self.pitch_fmax = pitch_fmax + self.n_window_size = n_window_size + self.pitch_avg = pitch_avg + self.pitch_std = pitch_std + super().__init__(attn_prior_folder=sup_data_path, **kwargs) + + def __getitem__(self, item): + audio, audio_len, text, text_len, attn_prior = super().__getitem__(item) + tag = Path(self.collection[item].audio_file).stem + pitch_path = ( + Path(self.attn_prior_folder) + / f"{tag}_pitch_pyin_fmin{self.pitch_fmin}_fmax{self.pitch_fmax}_fl{self.n_window_size}.npy" + ) + + if pitch_path.exists(): + pitch = np.load(pitch_path) + else: + pitch, _, _ = librosa.pyin( + audio.numpy(), + fmin=self.pitch_fmin, + fmax=self.pitch_fmax, + frame_length=self.n_window_size, + sr=self.featurizer.sample_rate, + fill_na=0.0, + ) + np.save(pitch_path, pitch) + pitch -= self.pitch_avg + pitch[pitch == -self.pitch_avg] = 0.0 # Zero out values that were perviously zero + pitch /= self.pitch_std + + return audio, audio_len, text, text_len, attn_prior, torch.tensor(pitch) + + def _collate_fn(self, batch): + batch = list(zip(*batch)) + audio, audio_len, text, text_len, attn_prior = super()._collate_fn(list(zip(*batch[:5]))) + pitch_list = batch[5] + + pitch = torch.zeros(len(pitch_list), max([pitch.shape[0] for pitch in pitch_list])) + + for i, pitch_i in enumerate(pitch_list): + pitch[i, : pitch_i.shape[0]] = pitch_i + + return audio, audio_len, text, text_len, attn_prior, pitch + + +class FastPitchDataset(_AudioTextDataset): + """ + Dataset used for FastPitch that has both duration and pitch information per input char. + See https://github.com/NVIDIA/NeMo/pull/1799 for information on how to extract duration and pitch information. + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports.""" + return { + 'audio': NeuralType(('B', 'T'), AudioSignal()), + 'audio_len': NeuralType(('B',), LengthsType()), + 'text': NeuralType(('B', 'T'), LabelsType()), + 'text_len': NeuralType(('B',), LengthsType()), + 'durs': NeuralType(('B', 'T'), TokenDurationType()), + 'pitch': NeuralType(('B', 'T'), RegressionValuesType()), + 'speakers': NeuralType(('B',), Index()), + } + + def __getitem__(self, item): + audio, audio_len, text, text_len = super().__getitem__(item) # noqa + + audio_path = self.collection[item].audio_file + durs_path = audio_path.replace('/wavs/', '/fastpitch/durations/').replace('.wav', '.pt') + pitch_path = audio_path.replace('/wavs/', '/fastpitch/pitch_char/').replace('.wav', '.pt') + speaker = None + if self.collection[item].speaker is not None: + speaker = torch.zeros_like(text_len).fill_(self.collection[item].speaker) + + return (audio, audio_len, text, text_len, torch.load(durs_path), torch.load(pitch_path), speaker, audio_path) + + def _collate_fn(self, batch): + asr_batch = list(zip(*batch))[:4] + asr_batch = _speech_collate_fn(list(zip(*asr_batch)), pad_id=self.pad_id) + audio, audio_len, text, text_len = asr_batch + + max_tokens_len = text.size(1) + durs, pitch, speakers = [], [], [] + for _, _, _, tokens_i_len, durs_i, pitch_i, speaker_i, path_i in batch: + pad = (0, max_tokens_len - tokens_i_len.item()) + assert len(durs_i) == len(pitch_i), f"{len(durs_i)}, {len(pitch_i)}: {path_i}" + assert len(durs_i) == tokens_i_len, f"{len(durs_i)}, {tokens_i_len}: {path_i}" + durs.append(F.pad(durs_i, pad, value=self.pad_id)) + pitch.append(F.pad(pitch_i, pad, value=self.pad_id)) + speakers.append(speaker_i) + + return ( + audio, + audio_len, + text, + text_len, + torch.stack(durs).to(audio.dtype), + torch.stack(pitch).to(audio.dtype), + torch.stack(speakers).to(audio.dtype) if speakers[0] is not None else None, + ) + + +class AudioToBPEDataset(_AudioTextDataset): + """ + Dataset that loads tensors via a json file containing paths to audio + files, transcripts, and durations (in seconds). Each new line is a + different sample. Example below: + {"audio_filepath": "/path/to/audio.wav", "text_filepath": + "/path/to/audio.txt", "duration": 23.147} + ... + {"audio_filepath": "/path/to/audio.wav", "text": "the + transcription", "offset": 301.75, "duration": 0.82, "utt": + "utterance_id", "ctm_utt": "en_4156", "side": "A"} + + In practice, the dataset and manifest used for character encoding and byte pair encoding + are exactly the same. The only difference lies in how the dataset tokenizes the text in + the manifest. + + Args: + manifest_filepath: Path to manifest json as described above. Can + be comma-separated paths. + tokenizer: A subclass of the Tokenizer wrapper found in the common collection, + nemo.collections.common.tokenizers.TokenizerSpec. ASR Models support a subset of + all available tokenizers. + sample_rate (int): Sample rate to resample loaded audio to + int_values (bool): If true, load samples as 32-bit integers. Defauts to False. + augmentor (nemo.collections.asr.parts.perturb.AudioAugmentor): An AudioAugmentor + object used to augment loaded audio + max_duration: If audio exceeds this length, do not include in dataset + min_duration: If audio is less than this length, do not include + in dataset + max_utts: Limit number of utterances + trim: Whether to trim silence segments + use_start_end_token: Boolean which dictates whether to add [BOS] and [EOS] + tokens to beginning and ending of speech respectively. + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + 'audio_signal': NeuralType(('B', 'T'), AudioSignal()), + 'a_sig_length': NeuralType(tuple('B'), LengthsType()), + 'transcripts': NeuralType(('B', 'T'), LabelsType()), + 'transcript_length': NeuralType(tuple('B'), LengthsType()), + } + + def __init__( + self, + manifest_filepath: str, + tokenizer: 'nemo.collections.common.tokenizers.TokenizerSpec', + sample_rate: int, + int_values: bool = False, + augmentor: 'nemo.collections.asr.parts.perturb.AudioAugmentor' = None, + max_duration: Optional[int] = None, + min_duration: Optional[int] = None, + max_utts: int = 0, + trim: bool = False, + use_start_end_token: bool = True, + ): + if use_start_end_token and hasattr(tokenizer, 'bos_token'): + bos_id = tokenizer.bos_id + else: + bos_id = None + + if use_start_end_token and hasattr(tokenizer, 'eos_token'): + eos_id = tokenizer.eos_id + else: + eos_id = None + + if hasattr(tokenizer, 'pad_token'): + pad_id = tokenizer.pad_id + else: + pad_id = 0 + + class TokenizerWrapper: + def __init__(self, tokenizer): + self._tokenizer = tokenizer + + def __call__(self, text): + t = self._tokenizer.text_to_ids(text) + return t + + super().__init__( + manifest_filepath=manifest_filepath, + parser=TokenizerWrapper(tokenizer), + sample_rate=sample_rate, + int_values=int_values, + augmentor=augmentor, + max_duration=max_duration, + min_duration=min_duration, + max_utts=max_utts, + bos_id=bos_id, + eos_id=eos_id, + pad_id=pad_id, + trim=trim, + ) + + +class _TarredAudioToTextDataset(IterableDataset): + """ + A similar Dataset to the AudioToCharDataset/AudioToBPEDataset, but which loads tarred audio files. + + Accepts a single comma-separated JSON manifest file (in the same style as for the AudioToCharDataset/AudioToBPEDataset), + as well as the path(s) to the tarball(s) containing the wav files. Each line of the manifest should + contain the information for one audio file, including at least the transcript and name of the audio + file within the tarball. + + Valid formats for the audio_tar_filepaths argument include: + (1) a single string that can be brace-expanded, e.g. 'path/to/audio.tar' or 'path/to/audio_{1..100}.tar.gz', or + (2) a list of file paths that will not be brace-expanded, e.g. ['audio_1.tar', 'audio_2.tar', ...]. + + Note: For brace expansion in (1), there may be cases where `{x..y}` syntax cannot be used due to shell interference. + This occurs most commonly inside SLURM scripts. Therefore we provide a few equivalent replacements. + Supported opening braces - { <=> (, [, < and the special tag _OP_. + Supported closing braces - } <=> ), ], > and the special tag _CL_. + For SLURM based tasks, we suggest the use of the special tags for ease of use. + + See the WebDataset documentation for more information about accepted data and input formats. + + If using multiple workers the number of shards should be divisible by world_size to ensure an + even split among workers. If it is not divisible, logging will give a warning but training will proceed. + In addition, if using mutiprocessing, each shard MUST HAVE THE SAME NUMBER OF ENTRIES after filtering + is applied. We currently do not check for this, but your program may hang if the shards are uneven! + + Notice that a few arguments are different from the AudioToCharDataset; for example, shuffle (bool) has been + replaced by shuffle_n (int). + + Additionally, please note that the len() of this DataLayer is assumed to be the length of the manifest + after filtering. An incorrect manifest length may lead to some DataLoader issues down the line. + + Args: + audio_tar_filepaths: Either a list of audio tarball filepaths, or a + string (can be brace-expandable). + manifest_filepath (str): Path to the manifest. + parser (callable): A callable which is used to pre-process the text output. + sample_rate (int): Sample rate to resample loaded audio to + int_values (bool): If true, load samples as 32-bit integers. Defauts to False. + augmentor (nemo.collections.asr.parts.perturb.AudioAugmentor): An AudioAugmentor + object used to augment loaded audio + shuffle_n (int): How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + Defaults to 0. + min_duration (float): Dataset parameter. + All training files which have a duration less than min_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to 0.1. + max_duration (float): Dataset parameter. + All training files which have a duration more than max_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to None. + max_utts (int): Limit number of utterances. 0 means no maximum. + blank_index (int): Blank character index, defaults to -1. + unk_index (int): Unknown character index, defaults to -1. + normalize (bool): Dataset parameter. + Whether to use automatic text cleaning. + It is highly recommended to manually clean text for best results. + Defaults to True. + trim (bool): Whether to use trim silence from beginning and end + of audio signal using librosa.effects.trim(). + Defaults to False. + bos_id (id): Dataset parameter. + Beginning of string symbol id used for seq2seq models. + Defaults to None. + eos_id (id): Dataset parameter. + End of string symbol id used for seq2seq models. + Defaults to None. + pad_id (id): Token used to pad when collating samples in batches. + If this is None, pads using 0s. + Defaults to None. + shard_strategy (str): Tarred dataset shard distribution strategy chosen as a str value during ddp. + - `scatter`: The default shard strategy applied by WebDataset, where each node gets + a unique set of shards, which are permanently pre-allocated and never changed at runtime. + - `replicate`: Optional shard strategy, where each node gets all of the set of shards + available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. + The benefit of replication is that it allows each node to sample data points from the entire + dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. + + Note: Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. + global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. + world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. + """ + + def __init__( + self, + audio_tar_filepaths: Union[str, List[str]], + manifest_filepath: str, + parser: Callable, + sample_rate: int, + int_values: bool = False, + augmentor: Optional['nemo.collections.asr.parts.perturb.AudioAugmentor'] = None, + shuffle_n: int = 0, + min_duration: Optional[float] = None, + max_duration: Optional[float] = None, + max_utts: int = 0, + trim: bool = False, + bos_id: Optional[int] = None, + eos_id: Optional[int] = None, + pad_id: int = 0, + shard_strategy: str = "scatter", + global_rank: int = 0, + world_size: int = 0, + ): + self.collection = collections.ASRAudioText( + manifests_files=manifest_filepath.split(','), + parser=parser, + min_duration=min_duration, + max_duration=max_duration, + max_number=max_utts, + index_by_file_id=True, # Must set this so the manifest lines can be indexed by file ID + ) + + self.featurizer = WaveformFeaturizer(sample_rate=sample_rate, int_values=int_values, augmentor=augmentor) + self.trim = trim + self.eos_id = eos_id + self.bos_id = bos_id + self.pad_id = pad_id + + valid_shard_strategies = ['scatter', 'replicate'] + if shard_strategy not in valid_shard_strategies: + raise ValueError(f"`shard_strategy` must be one of {valid_shard_strategies}") + + if isinstance(audio_tar_filepaths, str): + # Replace '(' and '[' with '{' + brace_keys_open = ['(', '[', '<', '_OP_'] + for bkey in brace_keys_open: + if bkey in audio_tar_filepaths: + audio_tar_filepaths = audio_tar_filepaths.replace(bkey, "{") + + # Replace ')' and ']' with '}' + brace_keys_close = [')', ']', '>', '_CL_'] + for bkey in brace_keys_close: + if bkey in audio_tar_filepaths: + audio_tar_filepaths = audio_tar_filepaths.replace(bkey, "}") + + # Check for distributed and partition shards accordingly + if world_size > 1: + if isinstance(audio_tar_filepaths, str): + # Brace expand + audio_tar_filepaths = list(braceexpand.braceexpand(audio_tar_filepaths)) + + if shard_strategy == 'scatter': + logging.info("All tarred dataset shards will be scattered evenly across all nodes.") + + if len(audio_tar_filepaths) % world_size != 0: + logging.warning( + f"Number of shards in tarred dataset ({len(audio_tar_filepaths)}) is not divisible " + f"by number of distributed workers ({world_size})." + ) + + begin_idx = (len(audio_tar_filepaths) // world_size) * global_rank + end_idx = begin_idx + (len(audio_tar_filepaths) // world_size) + audio_tar_filepaths = audio_tar_filepaths[begin_idx:end_idx] + logging.info( + "Partitioning tarred dataset: process (%d) taking shards [%d, %d)", global_rank, begin_idx, end_idx + ) + + elif shard_strategy == 'replicate': + logging.info("All tarred dataset shards will be replicated across all nodes.") + + else: + raise ValueError(f"Invalid shard strategy ! Allowed values are : {valid_shard_strategies}") + + # Put together WebDataset + self._dataset = wd.WebDataset(urls=audio_tar_filepaths, nodesplitter=None) + + if shuffle_n > 0: + self._dataset = self._dataset.shuffle(shuffle_n) + else: + logging.info("WebDataset will not shuffle files within the tar files.") + + self._dataset = ( + self._dataset.rename(audio='wav', key='__key__') + .to_tuple('audio', 'key') + .pipe(self._filter) + .map(f=self._build_sample) + ) + + def _filter(self, iterator): + """This function is used to remove samples that have been filtered out by ASRAudioText already. + Otherwise, we would get a KeyError as _build_sample attempts to find the manifest entry for a sample + that was filtered out (e.g. for duration). + Note that if using multi-GPU training, filtering may lead to an imbalance in samples in each shard, + which may make your code hang as one process will finish before the other. + """ + + class TarredAudioFilter: + def __init__(self, collection): + self.iterator = iterator + self.collection = collection + + def __iter__(self): + return self + + def __next__(self): + while True: + audio_bytes, audio_filename = next(self.iterator) + file_id, _ = os.path.splitext(os.path.basename(audio_filename)) + if file_id in self.collection.mapping: + return audio_bytes, audio_filename + + return TarredAudioFilter(self.collection) + + def _collate_fn(self, batch): + return _speech_collate_fn(batch, self.pad_id) + + def _build_sample(self, tup): + """Builds the training sample by combining the data from the WebDataset with the manifest info. + """ + audio_bytes, audio_filename = tup + + # Grab manifest entry from self.collection + file_id, _ = os.path.splitext(os.path.basename(audio_filename)) + manifest_idx = self.collection.mapping[file_id] + manifest_entry = self.collection[manifest_idx] + + offset = manifest_entry.offset + if offset is None: + offset = 0 + + # Convert audio bytes to IO stream for processing (for SoundFile to read) + audio_filestream = io.BytesIO(audio_bytes) + features = self.featurizer.process( + audio_filestream, + offset=offset, + duration=manifest_entry.duration, + trim=self.trim, + orig_sr=manifest_entry.orig_sr, + ) + audio_filestream.close() + + # Audio features + f, fl = features, torch.tensor(features.shape[0]).long() + + # Text features + t, tl = manifest_entry.text_tokens, len(manifest_entry.text_tokens) + if self.bos_id is not None: + t = [self.bos_id] + t + tl += 1 + if self.eos_id is not None: + t = t + [self.eos_id] + tl += 1 + + return f, fl, torch.tensor(t).long(), torch.tensor(tl).long() + + def __iter__(self): + return self._dataset.__iter__() + + def __len__(self): + return len(self.collection) + + +class TarredAudioToCharDataset(_TarredAudioToTextDataset): + """ + A similar Dataset to the AudioToCharDataset, but which loads tarred audio files. + + Accepts a single comma-separated JSON manifest file (in the same style as for the AudioToCharDataset), + as well as the path(s) to the tarball(s) containing the wav files. Each line of the manifest should + contain the information for one audio file, including at least the transcript and name of the audio + file within the tarball. + + Valid formats for the audio_tar_filepaths argument include: + (1) a single string that can be brace-expanded, e.g. 'path/to/audio.tar' or 'path/to/audio_{1..100}.tar.gz', or + (2) a list of file paths that will not be brace-expanded, e.g. ['audio_1.tar', 'audio_2.tar', ...]. + + See the WebDataset documentation for more information about accepted data and input formats. + + If using multiple workers the number of shards should be divisible by world_size to ensure an + even split among workers. If it is not divisible, logging will give a warning but training will proceed. + In addition, if using mutiprocessing, each shard MUST HAVE THE SAME NUMBER OF ENTRIES after filtering + is applied. We currently do not check for this, but your program may hang if the shards are uneven! + + Notice that a few arguments are different from the AudioToCharDataset; for example, shuffle (bool) has been + replaced by shuffle_n (int). + + Additionally, please note that the len() of this DataLayer is assumed to be the length of the manifest + after filtering. An incorrect manifest length may lead to some DataLoader issues down the line. + + Args: + audio_tar_filepaths: Either a list of audio tarball filepaths, or a + string (can be brace-expandable). + manifest_filepath (str): Path to the manifest. + labels (list): List of characters that can be output by the ASR model. + For Jasper, this is the 28 character set {a-z '}. The CTC blank + symbol is automatically added later for models using ctc. + sample_rate (int): Sample rate to resample loaded audio to + int_values (bool): If true, load samples as 32-bit integers. Defauts to False. + augmentor (nemo.collections.asr.parts.perturb.AudioAugmentor): An AudioAugmentor + object used to augment loaded audio + shuffle_n (int): How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + Defaults to 0. + min_duration (float): Dataset parameter. + All training files which have a duration less than min_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to 0.1. + max_duration (float): Dataset parameter. + All training files which have a duration more than max_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to None. + max_utts (int): Limit number of utterances. 0 means no maximum. + blank_index (int): Blank character index, defaults to -1. + unk_index (int): Unknown character index, defaults to -1. + normalize (bool): Dataset parameter. + Whether to use automatic text cleaning. + It is highly recommended to manually clean text for best results. + Defaults to True. + trim (bool): Whether to use trim silence from beginning and end + of audio signal using librosa.effects.trim(). + Defaults to False. + bos_id (id): Dataset parameter. + Beginning of string symbol id used for seq2seq models. + Defaults to None. + eos_id (id): Dataset parameter. + End of string symbol id used for seq2seq models. + Defaults to None. + pad_id (id): Token used to pad when collating samples in batches. + If this is None, pads using 0s. + Defaults to None. + shard_strategy (str): Tarred dataset shard distribution strategy chosen as a str value during ddp. + - `scatter`: The default shard strategy applied by WebDataset, where each node gets + a unique set of shards, which are permanently pre-allocated and never changed at runtime. + - `replicate`: Optional shard strategy, where each node gets all of the set of shards + available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. + The benefit of replication is that it allows each node to sample data points from the entire + dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. + + Note: Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. + global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. + world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. + """ + + def __init__( + self, + audio_tar_filepaths: Union[str, List[str]], + manifest_filepath: str, + labels: List[str], + sample_rate: int, + int_values: bool = False, + augmentor: Optional['nemo.collections.asr.parts.perturb.AudioAugmentor'] = None, + shuffle_n: int = 0, + min_duration: Optional[float] = None, + max_duration: Optional[float] = None, + max_utts: int = 0, + blank_index: int = -1, + unk_index: int = -1, + normalize: bool = True, + trim: bool = False, + bos_id: Optional[int] = None, + eos_id: Optional[int] = None, + parser: Optional[str] = 'en', + pad_id: int = 0, + shard_strategy: str = "scatter", + global_rank: int = 0, + world_size: int = 0, + ): + self.labels = labels + + parser = parsers.make_parser( + labels=labels, name=parser, unk_id=unk_index, blank_id=blank_index, do_normalize=normalize + ) + + super().__init__( + audio_tar_filepaths=audio_tar_filepaths, + manifest_filepath=manifest_filepath, + parser=parser, + sample_rate=sample_rate, + int_values=int_values, + augmentor=augmentor, + shuffle_n=shuffle_n, + min_duration=min_duration, + max_duration=max_duration, + max_utts=max_utts, + trim=trim, + bos_id=bos_id, + eos_id=eos_id, + pad_id=pad_id, + shard_strategy=shard_strategy, + global_rank=global_rank, + world_size=world_size, + ) + + +class TarredAudioToBPEDataset(_TarredAudioToTextDataset): + """ + A similar Dataset to the AudioToBPEDataset, but which loads tarred audio files. + + Accepts a single comma-separated JSON manifest file (in the same style as for the AudioToBPEDataset), + as well as the path(s) to the tarball(s) containing the wav files. Each line of the manifest should + contain the information for one audio file, including at least the transcript and name of the audio + file within the tarball. + + Valid formats for the audio_tar_filepaths argument include: + (1) a single string that can be brace-expanded, e.g. 'path/to/audio.tar' or 'path/to/audio_{1..100}.tar.gz', or + (2) a list of file paths that will not be brace-expanded, e.g. ['audio_1.tar', 'audio_2.tar', ...]. + + See the WebDataset documentation for more information about accepted data and input formats. + + If using multiple workers the number of shards should be divisible by world_size to ensure an + even split among workers. If it is not divisible, logging will give a warning but training will proceed. + In addition, if using mutiprocessing, each shard MUST HAVE THE SAME NUMBER OF ENTRIES after filtering + is applied. We currently do not check for this, but your program may hang if the shards are uneven! + + Notice that a few arguments are different from the AudioToBPEDataset; for example, shuffle (bool) has been + replaced by shuffle_n (int). + + Additionally, please note that the len() of this DataLayer is assumed to be the length of the manifest + after filtering. An incorrect manifest length may lead to some DataLoader issues down the line. + + Args: + audio_tar_filepaths: Either a list of audio tarball filepaths, or a + string (can be brace-expandable). + manifest_filepath (str): Path to the manifest. + tokenizer (TokenizerSpec): Either a Word Piece Encoding tokenizer (BERT), + or a Sentence Piece Encoding tokenizer (BPE). The CTC blank + symbol is automatically added later for models using ctc. + sample_rate (int): Sample rate to resample loaded audio to + int_values (bool): If true, load samples as 32-bit integers. Defauts to False. + augmentor (nemo.collections.asr.parts.perturb.AudioAugmentor): An AudioAugmentor + object used to augment loaded audio + shuffle_n (int): How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + Defaults to 0. + min_duration (float): Dataset parameter. + All training files which have a duration less than min_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to 0.1. + max_duration (float): Dataset parameter. + All training files which have a duration more than max_duration + are dropped. Note: Duration is read from the manifest JSON. + Defaults to None. + max_utts (int): Limit number of utterances. 0 means no maximum. + trim (bool): Whether to use trim silence from beginning and end + of audio signal using librosa.effects.trim(). + Defaults to False. + pad_id (id): Token used to pad when collating samples in batches. + If this is None, pads using 0s. + Defaults to None. + shard_strategy (str): Tarred dataset shard distribution strategy chosen as a str value during ddp. + - `scatter`: The default shard strategy applied by WebDataset, where each node gets + a unique set of shards, which are permanently pre-allocated and never changed at runtime. + - `replicate`: Optional shard strategy, where each node gets all of the set of shards + available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. + The benefit of replication is that it allows each node to sample data points from the entire + dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. + + Note: Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. + global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. + world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. + """ + + def __init__( + self, + audio_tar_filepaths: Union[str, List[str]], + manifest_filepath: str, + tokenizer: 'nemo.collections.common.tokenizers.TokenizerSpec', + sample_rate: int, + int_values: bool = False, + augmentor: Optional['nemo.collections.asr.parts.perturb.AudioAugmentor'] = None, + shuffle_n: int = 0, + min_duration: Optional[float] = None, + max_duration: Optional[float] = None, + max_utts: int = 0, + trim: bool = False, + use_start_end_token: bool = True, + shard_strategy: str = "scatter", + global_rank: int = 0, + world_size: int = 0, + ): + if use_start_end_token and hasattr(tokenizer, 'bos_token'): + bos_id = tokenizer.bos_id + else: + bos_id = None + + if use_start_end_token and hasattr(tokenizer, 'eos_token'): + eos_id = tokenizer.eos_id + else: + eos_id = None + + if hasattr(tokenizer, 'pad_token'): + pad_id = tokenizer.pad_id + else: + pad_id = 0 + + class TokenizerWrapper: + def __init__(self, tokenizer): + self._tokenizer = tokenizer + + def __call__(self, text): + t = self._tokenizer.text_to_ids(text) + return t + + super().__init__( + audio_tar_filepaths=audio_tar_filepaths, + manifest_filepath=manifest_filepath, + parser=TokenizerWrapper(tokenizer), + sample_rate=sample_rate, + int_values=int_values, + augmentor=augmentor, + shuffle_n=shuffle_n, + min_duration=min_duration, + max_duration=max_duration, + max_utts=max_utts, + trim=trim, + bos_id=bos_id, + eos_id=eos_id, + pad_id=pad_id, + shard_strategy=shard_strategy, + global_rank=global_rank, + world_size=world_size, + ) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text_dali.py b/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text_dali.py new file mode 100644 index 0000000000..4d0e6642b9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text_dali.py @@ -0,0 +1,444 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from collections.abc import Iterator +from typing import Callable, List, Optional, Union + +import torch +from omegaconf import DictConfig + +from nemo.collections.common.parts.preprocessing import parsers +from nemo.utils.decorators import experimental + +try: + import nvidia.dali as dali + from nvidia.dali.pipeline import Pipeline + from nvidia.dali.plugin.pytorch import DALIGenericIterator as DALIPytorchIterator + from nvidia.dali.plugin.pytorch import LastBatchPolicy as LastBatchPolicy + + HAVE_DALI = True +except (ImportError, ModuleNotFoundError): + HAVE_DALI = False + +__all__ = [ + 'AudioToCharDALIDataset', +] + + +class DALIOutputs(object): + def __init__(self, out_dict): + self._has_processed_signal = 'processed_signal' in out_dict and 'processed_signal_len' in out_dict + if not self._has_processed_signal: + assert 'audio' in out_dict and 'audio_len' in out_dict + assert 'transcript' in out_dict and 'transcript_len' in out_dict + if self._has_processed_signal: + self._outs = ( + out_dict['processed_signal'], + out_dict['processed_signal_len'].reshape(-1), + out_dict['transcript'], + out_dict['transcript_len'].reshape(-1), + ) + else: + self._outs = ( + out_dict['audio'], + out_dict['audio_len'].reshape(-1), + out_dict['transcript'], + out_dict['transcript_len'].reshape(-1), + ) + + @property + def has_processed_signal(self): + return self._has_processed_signal + + def __getitem__(self, key): + return self._outs[key] + + def __len__(self): + return len(self._outs) + + +@experimental +class AudioToCharDALIDataset(Iterator): + """ + NVIDIA DALI pipeline that loads tensors via one or more manifest files where each line containing a sample descriptor in JSON, + including audio files, transcripts, and durations (in seconds). + Here's an example: + {"audio_filepath": "/path/to/audio.wav", "text_filepath": "/path/to/audio.txt", "duration": 23.147} + ... + {"audio_filepath": "/path/to/audio.wav", "text": "the transcription", "offset": 301.75, "duration": 0.82, "utt": + "utterance_id", "ctm_utt": "en_4156", "side": "A"} + Args: + manifest_filepath: Path to manifest file with the format described above. Can be comma-separated paths. + labels: String containing all the possible characters to map to. + sample_rate (int): Sample rate to resample loaded audio to. + batch_size (int): Number of samples in a batch. + num_threads (int): Number of CPU processing threads to be created by the DALI pipeline. + max_duration (float): Determines the maximum allowed duration, in seconds, of the loaded audio files. + min_duration (float): Determines the minimum allowed duration, in seconds, of the loaded audio files. + blank_index (int): blank character index, default = -1 + unk_index (int): unk_character index, default = -1 + normalize (bool): whether to normalize transcript text (default): True + bos_id (int): Id of beginning of sequence symbol to append if not None + eos_id (int): Id of end of sequence symbol to append if not None + trim (bool): If True, it will extract the nonsilent region of the loaded audio signal. + shuffle (bool): If set to True, the dataset will shuffled after loading. + drop_last (bool): If set to True, the last batch will be dropped if incomplete. This will be the case when the shard size is not divisible by the batch size. + If set to False and the size of dataset is not divisible by the batch size, then the last batch will be smaller. + device (str): Determines the device type to be used for preprocessing. Allowed values are: 'cpu', 'gpu'. + device_id (int): Index of the GPU to be used (local_rank). Only applicable when device == 'gpu'. Defaults to 0. + global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. + world_size (int): Total number of processes, used for partitioning shards. Defaults to 1. + preprocessor_cfg (DictConfig): Preprocessor configuration. Supports AudioToMelSpectrogramPreprocessor and AudioToMFCCPreprocessor. + """ + + def __init__( + self, + manifest_filepath: str, + device: str, + batch_size: int, + labels: Union[str, List[str]], + sample_rate: int = 16000, + num_threads: int = 4, + max_duration: float = 0.0, + min_duration: float = 0.0, + blank_index: int = -1, + unk_index: int = -1, + normalize: bool = True, + bos_id: Optional[int] = None, + eos_id: Optional[int] = None, + trim: bool = False, + shuffle: bool = True, + drop_last: bool = False, + parser: Union[str, Callable] = 'en', + device_id: int = 0, + global_rank: int = 0, + world_size: int = 1, + preprocessor_cfg: DictConfig = None, + ): + self.drop_last = drop_last # used by lr_scheduler + if not HAVE_DALI: + raise ModuleNotFoundError( + f"{self} requires NVIDIA DALI to be installed. " + f"See: https://docs.nvidia.com/deeplearning/dali/user-guide/docs/installation.html#id1" + ) + + if device not in ('cpu', 'gpu'): + raise ValueError( + f"{self} received an unexpected device argument {device}. Supported values are: 'cpu', 'gpu'" + ) + + self.batch_size = batch_size # Used by NeMo + + self.device = device + self.device_id = device_id + + if world_size > 1: + self.shard_id = global_rank + self.num_shards = world_size + else: + self.shard_id = None + self.num_shards = None + + self.labels = labels + if self.labels is None or len(self.labels) == 0: + raise ValueError(f"{self} expects non empty labels list") + + self.parser = parsers.make_parser( + labels=labels, name=parser, unk_id=unk_index, blank_id=blank_index, do_normalize=normalize, + ) + + self.eos_id = eos_id + self.bos_id = bos_id + self.sample_rate = sample_rate + + self.pipe = Pipeline( + batch_size=batch_size, + num_threads=num_threads, + device_id=self.device_id, + exec_async=True, + exec_pipelined=True, + ) + + has_preprocessor = preprocessor_cfg is not None + if has_preprocessor: + if preprocessor_cfg._target_ == "nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor": + feature_type = "mel_spectrogram" + elif preprocessor_cfg._target_ == "nemo.collections.asr.modules.AudioToMFCCPreprocessor": + feature_type = "mfcc" + else: + raise ValueError( + f"{self} received an unexpected preprocessor configuration: {preprocessor_cfg._target_}." + f" Supported preprocessors are: AudioToMelSpectrogramPreprocessor, AudioToMFCCPreprocessor" + ) + + # Default values taken from AudioToMelSpectrogramPreprocessor + params = preprocessor_cfg + self.dither = params['dither'] if 'dither' in params else 0.0 + self.preemph = params['preemph'] if 'preemph' in params else 0.97 + self.window_size_sec = params['window_size'] if 'window_size' in params else 0.02 + self.window_stride_sec = params['window_stride'] if 'window_stride' in params else 0.01 + self.sample_rate = params['sample_rate'] if 'sample_rate' in params else sample_rate + self.window_size = int(self.window_size_sec * self.sample_rate) + self.window_stride = int(self.window_size_sec * self.sample_rate) + + normalize = params['normalize'] if 'normalize' in params else 'per_feature' + if normalize == 'per_feature': # Each freq channel independently + self.normalization_axes = (1,) + elif normalize == 'all_features': + self.normalization_axes = (0, 1) + else: + raise ValueError( + f"{self} received {normalize} for the normalize parameter." + f" It must be either 'per_feature' or 'all_features'." + ) + + self.window = None + window_name = params['window'] if 'window' in params else None + torch_windows = { + 'hamming': torch.hamming_window, + 'blackman': torch.blackman_window, + 'bartlett': torch.bartlett_window, + } + if window_name is None or window_name == 'hann': + self.window = None # Hann is DALI's default + elif window_name == 'ones': + self.window = torch.ones(self.window_size) + else: + try: + window_fn = torch_windows.get(window_name, None) + self.window = window_fn(self.window_size, periodic=False) + except: + raise ValueError( + f"{self} received {window_name} for the window parameter." + f" It must be one of: ('hann', 'ones', 'hamming', 'blackman', 'bartlett', None)." + f" None is equivalent to 'hann'." + ) + + self.n_fft = params['n_fft'] if 'n_fft' in params else None # None means default + self.n_mels = params['n_mels'] if 'n_mels' in params else 64 + self.n_mfcc = params['n_mfcc'] if 'n_mfcc' in params else 64 + + features = params['features'] if 'features' in params else 0 + if features > 0: + if feature_type == 'mel_spectrogram': + self.n_mels = features + elif feature_type == 'mfcc': + self.n_mfcc = features + + # TODO Implement frame splicing + if 'frame_splicing' in params: + assert params['frame_splicing'] == 1, "Frame splicing is not implemented" + + self.freq_low = params['lowfreq'] if 'lowfreq' in params else 0.0 + self.freq_high = params['highfreq'] if 'highfreq' in params else self.sample_rate / 2.0 + self.log_features = params['log'] if 'log' in params else True + + # We want to avoid taking the log of zero + # There are two options: either adding or clamping to a small value + + self.log_zero_guard_type = params['log_zero_guard_type'] if 'log_zero_guard_type' in params else 'add' + if self.log_zero_guard_type not in ["add", "clamp"]: + raise ValueError( + f"{self} received {self.log_zero_guard_type} for the " + f"log_zero_guard_type parameter. It must be either 'add' or " + f"'clamp'." + ) + + self.log_zero_guard_value = params['log_zero_guard_value'] if 'log_zero_guard_value' in params else 1e-05 + if isinstance(self.log_zero_guard_value, str): + if self.log_zero_guard_value == "tiny": + self.log_zero_guard_value = torch.finfo(torch.float32).tiny + elif self.log_zero_guard_value == "eps": + self.log_zero_guard_value = torch.finfo(torch.float32).eps + else: + raise ValueError( + f"{self} received {self.log_zero_guard_value} for the log_zero_guard_type parameter." + f"It must be either a number, 'tiny', or 'eps'" + ) + + self.mag_power = params['mag_power'] if 'mag_power' in params else 2 + if self.mag_power != 1.0 and self.mag_power != 2.0: + raise ValueError( + f"{self} received {self.mag_power} for the mag_power parameter." f" It must be either 1.0 or 2.0." + ) + + self.pad_to = params['pad_to'] if 'pad_to' in params else 16 + self.pad_value = params['pad_value'] if 'pad_value' in params else 0.0 + + with self.pipe: + audio, transcript = dali.fn.nemo_asr_reader( + name="Reader", + manifest_filepaths=manifest_filepath.split(','), + dtype=dali.types.FLOAT, + downmix=True, + sample_rate=float(self.sample_rate), + min_duration=min_duration, + max_duration=max_duration, + read_sample_rate=False, + read_text=True, + random_shuffle=shuffle, + shard_id=self.shard_id, + num_shards=self.num_shards, + pad_last_batch=False, + ) + + transcript_len = dali.fn.shapes(dali.fn.reshape(transcript, shape=[-1])) + transcript = dali.fn.pad(transcript) + + # Extract nonsilent region, if necessary + if trim: + # Need to extract non-silent region before moving to the GPU + roi_start, roi_len = dali.fn.nonsilent_region(audio, cutoff_db=-60) + audio = audio.gpu() if self.device == 'gpu' else audio + audio = dali.fn.slice( + audio, roi_start, roi_len, normalized_anchor=False, normalized_shape=False, axes=[0] + ) + else: + audio = audio.gpu() if self.device == 'gpu' else audio + + if not has_preprocessor: + # No preprocessing, the output is the audio signal + audio_len = dali.fn.shapes(dali.fn.reshape(audio, shape=[-1])) + audio = dali.fn.pad(audio) + self.pipe.set_outputs(audio, audio_len, transcript, transcript_len) + else: + # Additive gaussian noise (dither) + if self.dither > 0.0: + gaussian_noise = dali.fn.normal_distribution(device=self.device) + audio = audio + self.dither * gaussian_noise + + # Preemphasis filter + if self.preemph > 0.0: + audio = dali.fn.preemphasis_filter(audio, preemph_coeff=self.preemph) + + # Power spectrogram + spec = dali.fn.spectrogram( + audio, nfft=self.n_fft, window_length=self.window_size, window_step=self.window_stride + ) + + if feature_type == 'mel_spectrogram' or feature_type == 'mfcc': + # Spectrogram to Mel Spectrogram + spec = dali.fn.mel_filter_bank( + spec, + sample_rate=self.sample_rate, + nfilter=self.n_mels, + normalize=True, + freq_low=self.freq_low, + freq_high=self.freq_high, + ) + # Mel Spectrogram to MFCC + if feature_type == 'mfcc': + spec = dali.fn.mfcc(spec, n_mfcc=self.n_mfcc) + + # Logarithm + if self.log_zero_guard_type == 'add': + spec = spec + self.log_zero_guard_value + + spec = dali.fn.to_decibels( + spec, multiplier=math.log(10), reference=1.0, cutoff_db=math.log(self.log_zero_guard_value) + ) + + # Normalization + spec = dali.fn.normalize(spec, axes=self.normalization_axes) + + # Extracting the length of the spectrogram + spec_len = dali.fn.slice(dali.fn.shapes(spec), 1, 1, axes=(0,)) + + # Pads feature dimension to be a multiple of `pad_to` and the temporal dimension to be as big as the largest sample (shape -1) + spec = dali.fn.pad(spec, fill_value=self.pad_value, axes=(0, 1), align=(self.pad_to, 1), shape=(1, -1)) + self.pipe.set_outputs(spec, spec_len, transcript, transcript_len) + # Building DALI pipeline + self.pipe.build() + + if has_preprocessor: + output_names = ['processed_signal', 'processed_signal_len', 'transcript_raw', 'transcript_raw_len'] + else: + output_names = ['audio', 'audio_len', 'transcript_raw', 'transcript_raw_len'] + + last_batch_policy = LastBatchPolicy.DROP if drop_last else LastBatchPolicy.PARTIAL + self._iter = DALIPytorchIterator( + [self.pipe], + output_map=output_names, + reader_name="Reader", + last_batch_policy=last_batch_policy, + dynamic_shape=True, + auto_reset=True, + ) + + # TODO come up with a better solution + class DummyDataset: + def __init__(self, parent): + self.parent = parent + + def __len__(self): + return self.parent.size + + self.dataset = DummyDataset(self) # Used by NeMo + + def reset(self): + self._iter.reset() + + def __iter__(self): + return self + + def next(self): + return self.__next__() + + @property + def size(self): + return self._iter.size + + def __len__(self): + return len(self._iter) + + def __next__(self): + outputs = self._iter.next() + assert len(outputs) == 1 + dali_out = outputs[0] + text_raw_len = dali_out['transcript_raw_len'].numpy() + text_raw = dali_out['transcript_raw'].numpy() + + out = {} + out_names = ['processed_signal', 'processed_signal_len', 'audio', 'audio_len'] + for out_name in out_names: + if out_name in dali_out: + out[out_name] = dali_out[out_name].detach().clone() + + text_tokens = [] + text_tokens_len = [] + max_len = 0 + batch_size = text_raw.shape[0] + for i, text in enumerate(text_raw): + n = text_raw_len[i][0] + tbytes = str(text[:n].tobytes(), encoding='utf8') + ttokens = self.parser(tbytes) + if self.bos_id is not None: + ttokens = [self.bos_id] + ttokens + if self.eos_id is not None: + ttokens = ttokens + [self.eos_id] + ttokens_len = len(ttokens) + text_tokens_len.append(ttokens_len) + text_tokens.append(ttokens) + if ttokens_len > max_len: + max_len = ttokens_len + + transcript_out = torch.zeros(batch_size, max_len, dtype=torch.long) + for i, n in enumerate(text_tokens_len): + transcript_out[i, :n] = torch.tensor(text_tokens[i], dtype=torch.long) + transcript_len_out = torch.tensor(text_tokens_len, dtype=torch.long) + + out['transcript'] = transcript_out + out['transcript_len'] = transcript_len_out + return DALIOutputs(out) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text_dataset.py b/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text_dataset.py new file mode 100644 index 0000000000..dc1b3fdae8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/data/audio_to_text_dataset.py @@ -0,0 +1,247 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch +from omegaconf import DictConfig, open_dict + +from nemo.collections.asr.data import audio_to_text, audio_to_text_dali +from nemo.utils import logging + + +def inject_dataloader_value_from_model_config(model_cfg: dict, dataloader_cfg: dict, key: str): + """ + Extracts the label set provided at the top level of the model, and propagates it to the dataloader + config. + + Args: + model_cfg: A DictConfig representing the model's config. + dataloader_cfg: A DictConfig representing the individual data loader + key: A str value representing a key in the model_cfg whose value will be propagated to the + dataloader config. + """ + if key not in model_cfg: + logging.info( + f"Model level config does not container `{key}`, please explicitly provide `{key}` to the dataloaders." + ) + return + + # If key exists in the data loader config (either set explicitly or as a placeholder (via None)) + if key in dataloader_cfg: + # Dataloader `labels` is provided and is non-null + if dataloader_cfg[key] is not None and model_cfg[key] != dataloader_cfg[key]: + # Model level `labels` dont match Dataloader level `labels` + logging.warning( + f'`{key}` is explicitly provided to the data loader, and is different from ' + f'the `{key}` provided at the model level config.\n' + f'If this is incorrect, please set the dataloader\'s `{key}` to None.' + ) + + else: + # Dataloader `key` is None or values match + # Propagate from model level `key` (even if they match) + with open_dict(dataloader_cfg): + dataloader_cfg[key] = model_cfg[key] + + else: + # If key key doesnt even exist in dataloader_cfg, inject it explicitly + with open_dict(dataloader_cfg): + dataloader_cfg[key] = model_cfg[key] + + +def get_char_dataset(config: dict, augmentor: Optional['AudioAugmentor'] = None) -> audio_to_text.AudioToCharDataset: + """ + Instantiates a Character Encoding based AudioToCharDataset. + + Args: + config: Config of the AudioToCharDataset. + augmentor: Optional AudioAugmentor object for augmentations on audio data. + + Returns: + An instance of AudioToCharDataset. + """ + dataset = audio_to_text.AudioToCharDataset( + manifest_filepath=config['manifest_filepath'], + labels=config['labels'], + sample_rate=config['sample_rate'], + int_values=config.get('int_values', False), + augmentor=augmentor, + max_duration=config.get('max_duration', None), + min_duration=config.get('min_duration', None), + max_utts=config.get('max_utts', 0), + blank_index=config.get('blank_index', -1), + unk_index=config.get('unk_index', -1), + normalize=config.get('normalize_transcripts', False), + trim=config.get('trim_silence', False), + parser=config.get('parser', 'en'), + ) + return dataset + + +def get_bpe_dataset( + config: dict, tokenizer: 'TokenizerSpec', augmentor: Optional['AudioAugmentor'] = None +) -> audio_to_text.AudioToBPEDataset: + """ + Instantiates a Byte Pair Encoding / Word Piece Encoding based AudioToBPEDataset. + + Args: + config: Config of the AudioToBPEDataset. + tokenizer: An instance of a TokenizerSpec object. + augmentor: Optional AudioAugmentor object for augmentations on audio data. + + Returns: + An instance of AudioToBPEDataset. + """ + dataset = audio_to_text.AudioToBPEDataset( + manifest_filepath=config['manifest_filepath'], + tokenizer=tokenizer, + sample_rate=config['sample_rate'], + int_values=config.get('int_values', False), + augmentor=augmentor, + max_duration=config.get('max_duration', None), + min_duration=config.get('min_duration', None), + max_utts=config.get('max_utts', 0), + trim=config.get('trim_silence', False), + use_start_end_token=config.get('use_start_end_token', True), + ) + return dataset + + +def get_tarred_char_dataset( + config: dict, shuffle_n: int, global_rank: int, world_size: int, augmentor: Optional['AudioAugmentor'] = None +) -> audio_to_text.TarredAudioToCharDataset: + """ + Instantiates a Character Encoding based TarredAudioToCharDataset. + + Args: + config: Config of the TarredAudioToCharDataset. + shuffle_n: How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + global_rank: Global rank of this device. + world_size: Global world size in the training method. + augmentor: Optional AudioAugmentor object for augmentations on audio data. + + Returns: + An instance of TarredAudioToCharDataset. + """ + dataset = audio_to_text.TarredAudioToCharDataset( + audio_tar_filepaths=config['tarred_audio_filepaths'], + manifest_filepath=config['manifest_filepath'], + labels=config['labels'], + sample_rate=config['sample_rate'], + int_values=config.get('int_values', False), + augmentor=augmentor, + shuffle_n=shuffle_n, + max_duration=config.get('max_duration', None), + min_duration=config.get('min_duration', None), + max_utts=config.get('max_utts', 0), + blank_index=config.get('blank_index', -1), + unk_index=config.get('unk_index', -1), + normalize=config.get('normalize_transcripts', False), + trim=config.get('trim_silence', False), + parser=config.get('parser', 'en'), + shard_strategy=config.get('tarred_shard_strategy', 'scatter'), + global_rank=global_rank, + world_size=world_size, + ) + return dataset + + +def get_tarred_bpe_dataset( + config: dict, + tokenizer: 'TokenizerSpec', + shuffle_n: int, + global_rank: int, + world_size: int, + augmentor: Optional['AudioAugmentor'] = None, +) -> audio_to_text.TarredAudioToBPEDataset: + """ + Instantiates a Byte Pair Encoding / Word Piece Encoding based TarredAudioToBPEDataset. + + Args: + config: Config of the TarredAudioToBPEDataset. + tokenizer: An instance of a TokenizerSpec object. + shuffle_n: How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + global_rank: Global rank of this device. + world_size: Global world size in the training method. + augmentor: Optional AudioAugmentor object for augmentations on audio data. + + Returns: + An instance of TarredAudioToBPEDataset. + """ + dataset = audio_to_text.TarredAudioToBPEDataset( + audio_tar_filepaths=config['tarred_audio_filepaths'], + manifest_filepath=config['manifest_filepath'], + tokenizer=tokenizer, + sample_rate=config['sample_rate'], + int_values=config.get('int_values', False), + augmentor=augmentor, + shuffle_n=shuffle_n, + max_duration=config.get('max_duration', None), + min_duration=config.get('min_duration', None), + max_utts=config.get('max_utts', 0), + trim=config.get('trim_silence', False), + use_start_end_token=config.get('use_start_end_token', True), + shard_strategy=config.get('tarred_shard_strategy', 'scatter'), + global_rank=global_rank, + world_size=world_size, + ) + return dataset + + +def get_dali_char_dataset( + config: dict, + shuffle: bool, + device_id: int, + global_rank: int, + world_size: int, + preprocessor_cfg: Optional[DictConfig] = None, +) -> audio_to_text_dali.AudioToCharDALIDataset: + """ + Instantiates a Character Encoding based AudioToCharDALIDataset. + + Args: + config: Config of the AudioToCharDALIDataset. + shuffle: Bool flag whether to shuffle the dataset. + device_id: Index of the GPU to be used (local_rank). Only applicable when device == 'gpu'. Defaults to 0. + global_rank: Global rank of this device. + world_size: Global world size in the training method. + augmentor: Optional AudioAugmentor object for augmentations on audio data. + + Returns: + An instance of AudioToCharDALIDataset. + """ + device = 'gpu' if torch.cuda.is_available() else 'cpu' + dataset = audio_to_text_dali.AudioToCharDALIDataset( + manifest_filepath=config['manifest_filepath'], + device=device, + batch_size=config['batch_size'], + labels=config['labels'], + sample_rate=config['sample_rate'], + max_duration=config.get('max_duration', None), + min_duration=config.get('min_duration', None), + blank_index=config.get('blank_index', -1), + unk_index=config.get('unk_index', -1), + normalize=config.get('normalize_transcripts', False), + trim=config.get('trim_silence', False), + parser=config.get('parser', 'en'), + shuffle=shuffle, + device_id=device_id, + global_rank=global_rank, + world_size=world_size, + preprocessor_cfg=preprocessor_cfg, + ) + return dataset diff --git a/models/tacotron2/NeMo/nemo/collections/asr/data/feature_to_label.py b/models/tacotron2/NeMo/nemo/collections/asr/data/feature_to_label.py new file mode 100644 index 0000000000..dcba068d7a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/data/feature_to_label.py @@ -0,0 +1,139 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, List, Optional + +import torch + +from nemo.collections.common.parts.preprocessing import collections +from nemo.core.classes import Dataset +from nemo.core.neural_types import AcousticEncodedRepresentation, LabelsType, LengthsType, NeuralType +from nemo.utils import logging + + +def _feature_collate_fn(batch): + """collate batch of feat sig, feat len, tokens, tokens len + Args: + batch (FloatTensor, LongTensor, LongTensor, LongTensor): A tuple of tuples of feature, feature lengths, + encoded tokens, and encoded tokens length. + """ + _, feat_lengths, _, tokens_lengths = zip(*batch) + feat_signal, tokens = [], [] + for sig, sig_len, tokens_i, tokens_i_len in batch: + feat_signal.append(sig) + tokens.append(tokens_i) + + feat_signal = torch.stack(feat_signal) + feat_lengths = torch.stack(feat_lengths) + + tokens = torch.stack(tokens) + tokens_lengths = torch.stack(tokens_lengths) + + return feat_signal, feat_lengths, tokens, tokens_lengths + + +class _FeatureSeqSpeakerLabelDataset(Dataset): + """ + Dataset that loads tensors via a json file containing paths to feature files, sequences of labels. + Each new line is a different sample. Example below: + and their target labels. JSON files should be of the following format: + {"feature_filepath": "/path/to/feature_0.p", "seq_label": speakerA speakerB SpeakerA ....} \ + ... + {"feature_filepath": "/path/to/feature_n.p", "seq_label": target_seq_label_n} + target_seq_label_n is the string of sequence of speaker label, separated by space. + + Args: + manifest_filepath (str): Dataset parameter. Path to JSON containing data. + labels (Optional[list]): Dataset parameter. List of unique labels collected from all samples. + feature_loader : Dataset parameter. Feature loader to load (external) feature. + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + # TODO output type for external features + output_types = { + 'external_feat': NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + 'feat_length': NeuralType(tuple('B'), LengthsType()), + } + + if self.is_speaker_emb: + output_types.update( + { + 'embs': NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + 'embs_length': NeuralType(tuple('B'), LengthsType()), + 'label': NeuralType(('B', 'T'), LabelsType()), + 'label_length': NeuralType(tuple('B'), LengthsType()), + } + ) + else: + output_types.update( + {'label': NeuralType(('B', 'T'), LabelsType()), 'label_length': NeuralType(tuple('B'), LengthsType()),} + ) + + return output_types + + def __init__( + self, *, manifest_filepath: str, labels: List[str], feature_loader, is_speaker_emb: bool = False, + ): + super().__init__() + self.collection = collections.ASRFeatureSequenceLabel(manifests_files=manifest_filepath.split(','),) + + self.feature_loader = feature_loader + self.labels = labels if labels else self.collection.uniq_labels + self.is_speaker_emb = is_speaker_emb + + self.label2id, self.id2label = {}, {} + for label_id, label in enumerate(self.labels): + self.label2id[label] = label_id + self.id2label[label_id] = label + + for idx in range(len(self.labels[:5])): + logging.debug(" label id {} and its mapped label {}".format(idx, self.id2label[idx])) + + def __len__(self): + return len(self.collection) + + def __getitem__(self, index): + sample = self.collection[index] + + features = self.feature_loader.process(sample.feature_file) + f, fl = features, torch.tensor(features.shape[0]).long() + + t = torch.tensor(sample.seq_label).float() + tl = torch.tensor(len(sample.seq_label)).long() + + return f, fl, t, tl + + +class FeatureToSeqSpeakerLabelDataset(_FeatureSeqSpeakerLabelDataset): + """ + Dataset that loads tensors via a json file containing paths to feature + files and sequence of speakers. Each new line is a + different sample. Example below: + {"feature_filepath": "/path/to/feature_0.p", "seq_label": speakerA speakerB SpeakerA ....} \ + ... + {"feature_filepath": "/path/to/feature_n.p", "seq_label": target_seq_label_n} + target_seq_label_n is the string of sequence of speaker label, separated by space. + + Args: + manifest_filepath (str): Path to manifest json as described above. Canbe comma-separated paths. + labels (Optional[list]): String containing all the possible labels to map to + if None then automatically picks from ASRFeatureSequenceLabel collection. + feature_loader, Feature load to loader (external) feature. + + """ + + def _collate_fn(self, batch): + return _feature_collate_fn(batch) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/data/feature_to_label_dataset.py b/models/tacotron2/NeMo/nemo/collections/asr/data/feature_to_label_dataset.py new file mode 100644 index 0000000000..b88cba52a7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/data/feature_to_label_dataset.py @@ -0,0 +1,31 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nemo.collections.asr.data import feature_to_label + + +def get_feature_seq_speakerlabel_dataset( + feature_loader, config: dict +) -> feature_to_label.FeatureToSeqSpeakerLabelDataset: + """ + Instantiates a FeatureSeqSpeakerLabelDataset. + Args: + config: Config of the FeatureToSeqSpeakerLabelDataset. + + Returns: + An instance of FeatureToSeqSpeakerLabelDataset. + """ + dataset = feature_to_label.FeatureToSeqSpeakerLabelDataset( + manifest_filepath=config['manifest_filepath'], labels=config['labels'], feature_loader=feature_loader, + ) + return dataset diff --git a/models/tacotron2/NeMo/nemo/collections/asr/data/vocabs.py b/models/tacotron2/NeMo/nemo/collections/asr/data/vocabs.py new file mode 100644 index 0000000000..38beb20b3b --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/data/vocabs.py @@ -0,0 +1,352 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import itertools +import re +import string +import unicodedata +from builtins import str as unicode +from typing import List + +import nltk + +from nemo.collections.common.parts.preprocessing import parsers + +try: + import g2p_en # noqa + + _g2p = g2p_en.G2p() + _g2p.variables = None + + HAVE_G2P = True +except (FileNotFoundError, LookupError): + HAVE_G2P = False + +_words_re = re.compile("([a-z\-]+'[a-z\-]+|[a-z\-]+)|([^a-z{}]+)") + + +def _text_preprocessing(text): + text = unicode(text) + text = ''.join(char for char in unicodedata.normalize('NFD', text) if unicodedata.category(char) != 'Mn') + text = text.lower() + text = re.sub("[^ a-z'\".,?!()\[\]:;\-]", "", text) + return text + + +def _word_tokenize(text): + words = _words_re.findall(text) + words = [re.sub(r'\s(\d)', r'\1', word[1].upper()) if word[0] == '' else word[0] for word in words] + return words + + +class G2p: + def __init__( + self, + phoneme_dict_path=None, + use_seq2seq_for_oov=False, + ignore_ambiguous_words=True, + text_preprocessing_func=_text_preprocessing, + word_tokenize_func=_word_tokenize, + ): + # Download NLTK datasets if this class is to be instantiated + try: + nltk.data.find('taggers/averaged_perceptron_tagger.zip') + except LookupError: + nltk.download('averaged_perceptron_tagger', quiet=True) + try: + nltk.data.find('corpora/cmudict.zip') + except LookupError: + nltk.download('cmudict', quiet=True) + + self.homograph2features = _g2p.homograph2features + self.g2p_dict = self._construct_grapheme2phoneme_dict(phoneme_dict_path) + self.use_seq2seq_for_oov = use_seq2seq_for_oov + self.ignore_ambiguous_words = ignore_ambiguous_words + + self.text_preprocessing_func = text_preprocessing_func + self.word_tokenize_func = word_tokenize_func + + @staticmethod + def _construct_grapheme2phoneme_dict(phoneme_dict_path=None, encoding='latin-1'): + if phoneme_dict_path is None: + from nltk.corpus import cmudict + + return cmudict.dict() + + _alt_re = re.compile(r'\([0-9]+\)') + g2p_dict = {} + with open(phoneme_dict_path, encoding=encoding) as file: + for line in file: + if len(line) and ('A' <= line[0] <= 'Z' or line[0] == "'"): + parts = line.split(' ') + word = re.sub(_alt_re, '', parts[0]) + word = word.lower() + + pronunciation = parts[1].strip().split(" ") + if word in g2p_dict: + g2p_dict[word].append(pronunciation) + else: + g2p_dict[word] = [pronunciation] + return g2p_dict + + def handle_ambiguous(self, word): + if not self.ignore_ambiguous_words or len(self.g2p_dict[word]) == 1: + return True + return False + + def __call__(self, text): + text = self.text_preprocessing_func(text) + words = self.word_tokenize_func(text) + words_and_pos_tags = nltk.pos_tag(words) + + prons = [] + for word, pos in words_and_pos_tags: + word_by_hyphen = word.split("-") + + # punctuation + if re.search("[a-zA-Z]", word) is None: + pron = list(word) + # homograph + elif word in self.homograph2features: + pron1, pron2, pos1 = self.homograph2features[word] + if pos.startswith(pos1): + pron = pron1 + else: + pron = pron2 + # `'s` suffix + elif ( + len(word) > 2 + and word.endswith("'s") + and (word not in self.g2p_dict) + and (word[:-2] in self.g2p_dict) + and self.handle_ambiguous(word[:-2]) + ): + pron = self.g2p_dict[word[:-2]][0] + ["Z"] + # `s` suffix + elif ( + len(word) > 1 + and word.endswith("s") + and (word not in self.g2p_dict) + and (word[:-1] in self.g2p_dict) + and self.handle_ambiguous(word[:-1]) + ): + pron = self.g2p_dict[word[:-1]][0] + ["Z"] + # g2p dict + elif word in self.g2p_dict and self.handle_ambiguous(word): + pron = self.g2p_dict[word][0] + # word with hyphens + elif len(word_by_hyphen) > 1 and all( + [sub_word in self.g2p_dict and self.handle_ambiguous(sub_word) for sub_word in word_by_hyphen] + ): + pron = [] + for sub_word in word_by_hyphen: + pron.extend(self.g2p_dict[sub_word][0]) + pron.extend(["-"]) + pron.pop() + else: + if self.use_seq2seq_for_oov: + # run gru-based seq2seq model from _g2p + pron = _g2p.predict(word) + else: + pron = word + + prons.extend(pron) + + return prons + + +class Base(abc.ABC): + """Vocabulary for turning str text to list of int tokens.""" + + # fmt: off + PUNCT = ( # Derived from LJSpeech and "/" additionally + ',', '.', '!', '?', '-', + ':', ';', '/', '"', '(', + ')', '[', ']', '{', '}', + ) + # fmt: on + PAD, BLANK, OOV = '', '', '' + + def __init__(self, labels, *, pad=PAD, blank=BLANK, oov=OOV, sep='', add_blank_at="last_but_one"): + super().__init__() + + labels = list(labels) + self.pad, labels = len(labels), labels + [pad] # Padding + + if add_blank_at is not None: + self.blank, labels = len(labels), labels + [blank] # Reserved for blank from QN + else: + # use add_blank_at=None only for ASR where blank is added automatically + self.blank = -1 + + self.oov, labels = len(labels), labels + [oov] # Out Of Vocabulary + + if add_blank_at == "last": + labels[-1], labels[-2] = labels[-2], labels[-1] + self.oov, self.blank = self.blank, self.oov + + self.labels = labels + self.sep = sep + + self._util_ids = {self.pad, self.blank, self.oov} + self._label2id = {l: i for i, l in enumerate(labels)} + self._id2label = labels + + def __call__(self, text: str) -> List[int]: + return self.encode(text) + + @abc.abstractmethod + def encode(self, text: str) -> List[int]: + """Turns str text into int tokens.""" + pass + + def decode(self, tokens: List[int]) -> str: + """Turns ints tokens into str text.""" + return self.sep.join(self._id2label[t] for t in tokens if t not in self._util_ids) + + +class Chars(Base): + """Chars vocabulary.""" + + def __init__( + self, punct=True, spaces=False, apostrophe=True, add_blank_at="last_but_one", + ): + labels = [] + self.space, labels = len(labels), labels + [' '] # Space + labels.extend(string.ascii_lowercase) + if apostrophe: + labels.append("'") # Apostrophe for saving "don't" and "Joe's" + + if punct: + labels.extend(self.PUNCT) + + super().__init__(labels, add_blank_at=add_blank_at) + + self.punct = punct + self.spaces = spaces + + self._parser = parsers.ENCharParser(labels) + + def encode(self, text): + """See base class.""" + text = self._parser._normalize(text) # noqa + + if self.spaces: + for p in set(text) & set(self.PUNCT): + text = text.replace(p, f' {p} ') + text = text.strip().replace(' ', ' ') + + return self._parser._tokenize(text) # noqa + + +class Phonemes(Base): + """Phonemes vocabulary.""" + + # fmt: off + VOWELS = ( + 'AA', 'AE', 'AH', 'AO', 'AW', + 'AY', 'EH', 'ER', 'EY', 'IH', + 'IY', 'OW', 'OY', 'UH', 'UW', + ) + CONSONANTS = ( + 'B', 'CH', 'D', 'DH', 'F', 'G', + 'HH', 'JH', 'K', 'L', 'M', 'N', + 'NG', 'P', 'R', 'S', 'SH', 'T', + 'TH', 'V', 'W', 'Y', 'Z', 'ZH', + ) + # fmt: on + + def __init__( + self, + punct=True, + stresses=False, + spaces=True, + chars=False, + *, + space=' ', + silence=None, + apostrophe=True, + oov=Base.OOV, + sep='|', # To be able to distinguish between 2/3 letters codes. + add_blank_at="last_but_one", + pad_with_space=False, + improved_version_g2p=False, + phoneme_dict_path=None, + ): + labels = [] + self.space, labels = len(labels), labels + [space] # Space + + if silence is not None: + self.silence, labels = len(labels), labels + [silence] # Silence + + labels.extend(self.CONSONANTS) + vowels = list(self.VOWELS) + + if stresses: + vowels = [f'{p}{s}' for p, s in itertools.product(vowels, (0, 1, 2))] + labels.extend(vowels) + + if chars: + labels.extend(string.ascii_lowercase) + + if apostrophe: + labels.append("'") # Apostrophe + + if punct: + labels.extend(self.PUNCT) + + super().__init__(labels, oov=oov, sep=sep, add_blank_at=add_blank_at) + + self.punct = punct + self.stresses = stresses + self.spaces = spaces + self.pad_with_space = pad_with_space + + if improved_version_g2p: + self.g2p = G2p(phoneme_dict_path) + else: + self.g2p = _g2p + + def encode(self, text): + """See base class.""" + ps, space, labels = [], self.labels[self.space], set(self.labels) + + for p in self.g2p(text): # noqa + # Remove stress + if p.isalnum() and len(p) == 3 and not self.stresses: + p = p[:2] + + # Add space if last one isn't one + if p == space and len(ps) > 0 and ps[-1] != space: + ps.append(p) + + # Add next phoneme + if (p.isalnum() or p == "'") and p in labels: + ps.append(p) + + # Add punct and remove space if needed + if (p in self.PUNCT) and self.punct: + if not self.spaces and len(ps) > 0 and ps[-1] == space: + ps.pop() + ps.append(p) + + # Remove trailing spaces + while ps[-1] == space: + ps.pop() + + if self.pad_with_space: + ps = [space] + ps + [space] + + return [self._label2id[p] for p in ps] diff --git a/models/tacotron2/NeMo/nemo/collections/asr/losses/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/losses/__init__.py new file mode 100644 index 0000000000..d5d8200ead --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/losses/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.asr.losses.angularloss import AngularSoftmaxLoss +from nemo.collections.asr.losses.ctc import CTCLoss diff --git a/models/tacotron2/NeMo/nemo/collections/asr/losses/angularloss.py b/models/tacotron2/NeMo/nemo/collections/asr/losses/angularloss.py new file mode 100644 index 0000000000..e2aee9bba6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/losses/angularloss.py @@ -0,0 +1,68 @@ +# ! /usr/bin/python +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from nemo.core.classes import Loss, Typing, typecheck +from nemo.core.neural_types import LabelsType, LogitsType, LossType, NeuralType + +__all__ = ['AngularSoftmaxLoss'] + + +class AngularSoftmaxLoss(Loss, Typing): + """ + Computes ArcFace Angular softmax angle loss + reference: https://openaccess.thecvf.com/content_CVPR_2019/papers/Deng_ArcFace_Additive_Angular_Margin_Loss_for_Deep_Face_Recognition_CVPR_2019_paper.pdf + args: + scale: scale value for cosine angle + margin: margin value added to cosine angle + """ + + @property + def input_types(self): + """Input types definitions for AnguarLoss. + """ + return { + "logits": NeuralType(('B', 'D'), LogitsType()), + "labels": NeuralType(('B',), LabelsType()), + } + + @property + def output_types(self): + """Output types definitions for AngularLoss. + loss: + NeuralType(None) + """ + return {"loss": NeuralType(elements_type=LossType())} + + def __init__(self, scale=20.0, margin=1.35): + super().__init__() + + self.eps = 1e-7 + self.scale = scale + self.margin = margin + + @typecheck() + def forward(self, logits, labels): + numerator = self.scale * torch.cos( + torch.acos(torch.clamp(torch.diagonal(logits.transpose(0, 1)[labels]), -1.0 + self.eps, 1 - self.eps)) + + self.margin + ) + excl = torch.cat( + [torch.cat((logits[i, :y], logits[i, y + 1 :])).unsqueeze(0) for i, y in enumerate(labels)], dim=0 + ) + denominator = torch.exp(numerator) + torch.sum(torch.exp(self.scale * excl), dim=1) + L = numerator - torch.log(denominator) + return -torch.mean(L) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/losses/ctc.py b/models/tacotron2/NeMo/nemo/collections/asr/losses/ctc.py new file mode 100644 index 0000000000..1b35447b45 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/losses/ctc.py @@ -0,0 +1,70 @@ +# ! /usr/bin/python +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torch import nn + +from nemo.core.classes import Serialization, Typing, typecheck +from nemo.core.neural_types import LabelsType, LengthsType, LogprobsType, LossType, NeuralType + +__all__ = ['CTCLoss'] + + +class CTCLoss(nn.CTCLoss, Serialization, Typing): + @property + def input_types(self): + """Input types definitions for CTCLoss. + """ + return { + "log_probs": NeuralType(('B', 'T', 'D'), LogprobsType()), + "targets": NeuralType(('B', 'T'), LabelsType()), + "input_lengths": NeuralType(tuple('B'), LengthsType()), + "target_lengths": NeuralType(tuple('B'), LengthsType()), + } + + @property + def output_types(self): + """Output types definitions for CTCLoss. + loss: + NeuralType(None) + """ + return {"loss": NeuralType(elements_type=LossType())} + + def __init__(self, num_classes, zero_infinity=False, reduction='mean_batch'): + self._blank = num_classes + # Don't forget to properly call base constructor + if reduction == 'mean_batch': + ctc_reduction = 'none' + self._apply_batch_mean = True + elif reduction in ['sum', 'mean', 'none']: + ctc_reduction = reduction + self._apply_batch_mean = False + super().__init__(blank=self._blank, reduction=ctc_reduction, zero_infinity=zero_infinity) + + @typecheck() + def forward(self, log_probs, targets, input_lengths, target_lengths): + # override forward implementation + # custom logic, if necessary + input_lengths = input_lengths.long() + target_lengths = target_lengths.long() + targets = targets.long() + # here we transpose because we expect [B, T, D] while PyTorch assumes [T, B, D] + log_probs = log_probs.transpose(1, 0) + loss = super().forward( + log_probs=log_probs, targets=targets, input_lengths=input_lengths, target_lengths=target_lengths + ) + if self._apply_batch_mean: + loss = torch.mean(loss) + return loss diff --git a/models/tacotron2/NeMo/nemo/collections/asr/losses/rnnt.py b/models/tacotron2/NeMo/nemo/collections/asr/losses/rnnt.py new file mode 100644 index 0000000000..24496565ef --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/losses/rnnt.py @@ -0,0 +1,288 @@ +# ! /usr/bin/python +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2018-2019, Mingkun Huang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import operator +from dataclasses import dataclass +from typing import Optional + +import torch + +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types import LabelsType, LengthsType, LogprobsType, LossType, NeuralType +from nemo.core.utils.numba_utils import NUMBA_INSTALLATION_MESSAGE +from nemo.utils import logging, model_utils + +try: + import warprnnt_pytorch as warprnnt + + WARP_RNNT_AVAILABLE = True +except (ImportError, ModuleNotFoundError): + WARP_RNNT_AVAILABLE = False + +try: + from nemo.collections.asr.parts.numba.rnnt_loss import RNNTLossNumba + + NUMBA_RNNT_AVAILABLE = True +except (ImportError, ModuleNotFoundError): + NUMBA_RNNT_AVAILABLE = False + + +WARP_RNNT_INSTALLATION_MESSAGE = ( + "Could not import `warprnnt_pytorch`.\n" + "Please visit https://github.com/HawkAaron/warp-transducer " + "and follow the steps in the readme to build and install the " + "pytorch bindings for RNNT Loss, or use the provided docker " + "container that supports RNN-T loss." +) + + +@dataclass +class RNNTLossConfig: + loss_name: str + lib_name: str + is_available: bool = False + installation_msg: str = "" + min_version: Optional[str] = None + + +# Resolved list of available RNNT losses +RNNT_LOSS_RESOLVER = { + "warprnnt": RNNTLossConfig( + loss_name="warprnnt", + lib_name="warprnnt_pytorch", + is_available=WARP_RNNT_AVAILABLE, + installation_msg=WARP_RNNT_INSTALLATION_MESSAGE, + ), + "warprnnt_numba": RNNTLossConfig( + loss_name="warprnnt_numba", + lib_name="numba", + min_version='0.53.0', + is_available=NUMBA_RNNT_AVAILABLE, + installation_msg=NUMBA_INSTALLATION_MESSAGE, + ), +} + +RNNT_LOSS_RESOLVER['default'] = RNNT_LOSS_RESOLVER['warprnnt_numba'] + + +def _warn_unused_additional_kwargs(loss_name, kwargs): + if len(kwargs) > 0: + logging.warning( + f"Loss function `{loss_name}` was provided with following additional kwargs,\n" + f"however they were ignored as it is unused.\n" + f"{kwargs}" + ) + + +def resolve_rnnt_default_loss_name() -> str: + return RNNT_LOSS_RESOLVER['default'].loss_name + + +def resolve_rnnt_loss(loss_name: str, blank_idx: int, loss_kwargs: dict = None) -> torch.nn.Module: + loss_function_names = list(RNNT_LOSS_RESOLVER.keys()) + + if loss_name not in loss_function_names: + raise ValueError( + f"Provided `loss_name` {loss_name} not in list of available RNNT losses \n" f"{loss_function_names}" + ) + + all_available_losses = {name: config for name, config in RNNT_LOSS_RESOLVER.items() if config.is_available} + + loss_config = RNNT_LOSS_RESOLVER[loss_name] # type: RNNTLossConfig + + # Re-raise import error with installation message + if not loss_config.is_available: + msg = ( + f"Installed RNNT losses are : {list(all_available_losses.keys())}.\n" + f"****************************************************************\n" + f"To install the selected loss function, please follow the steps below:\n" + f"{loss_config.installation_msg}" + ) + raise ImportError(msg) + + # Library version check + if loss_config.min_version is not None: + ver_matched, msg = model_utils.check_lib_version( + loss_config.lib_name, checked_version=loss_config.min_version, operator=operator.ge + ) + + if ver_matched is False: + msg = ( + f"{msg}\n" + f"****************************************************************\n" + f"To update the selected loss function, please follow the steps below:\n" + f"{loss_config.installation_msg}" + ) + raise RuntimeError(msg) + + # Resolve loss functions sequentially + loss_kwargs = {} if loss_kwargs is None else loss_kwargs + + # Get actual loss name for `default` + if loss_name == 'default': + loss_name = loss_config.loss_name + + """ + Resolve RNNT loss functions + """ + if loss_name == 'warprnnt': + loss_func = warprnnt.RNNTLoss(blank=blank_idx, reduction='none') + _warn_unused_additional_kwargs(loss_name, loss_kwargs) + + elif loss_name == 'warprnnt_numba': + loss_func = RNNTLossNumba(blank=blank_idx, reduction='none') + _warn_unused_additional_kwargs(loss_name, loss_kwargs) + + else: + raise ValueError( + f"Invalid value of `loss_name`: {loss_name}. Allowed loss names are :" f"{loss_function_names}" + ) + + return loss_func + + +class RNNTLoss(Loss): + @property + def input_types(self): + """Input types definitions for CTCLoss. + """ + return { + "log_probs": NeuralType(('B', 'T', 'D', 'D'), LogprobsType()), + "targets": NeuralType(('B', 'T'), LabelsType()), + "input_lengths": NeuralType(tuple('B'), LengthsType()), + "target_lengths": NeuralType(tuple('B'), LengthsType()), + } + + @property + def output_types(self): + """Output types definitions for CTCLoss. + loss: + NeuralType(None) + """ + return {"loss": NeuralType(elements_type=LossType())} + + def __init__(self, num_classes, reduction: str = 'mean_batch', loss_name: str = "default", loss_kwargs=None): + """ + RNN-T Loss function based on https://github.com/HawkAaron/warp-transducer. + Optionally, can utilize a numba implementation of the same loss without having to compile the loss, + albiet there is a small speed penalty for JIT numba compile. + + Note: + Requires the pytorch bindings to be installed prior to calling this class. + + Warning: + In the case that GPU memory is exhausted in order to compute RNNTLoss, it might cause + a core dump at the cuda level with the following error message. + + ``` + ... + costs = costs.to(acts.device) + RuntimeError: CUDA error: an illegal memory access was encountered + terminate called after throwing an instance of 'c10::Error' + ``` + + Please kill all remaining python processes after this point, and use a smaller batch size + for train, validation and test sets so that CUDA memory is not exhausted. + + Args: + num_classes: Number of target classes for the joint network to predict. + (Excluding the RNN-T blank token). + + reduction: Type of reduction to perform on loss. Possibly values are `mean`, `sum` or None. + None will return a torch vector comprising the individual loss values of the batch. + + loss_name: String that is resolved into an RNNT loss function. Available list of losses + is ininitialized in `RNNT_LOSS_RESOLVER` dictionary. + + loss_kwargs: Optional Dict of (str, value) pairs that are passed to the instantiated loss + function. + """ + super(RNNTLoss, self).__init__() + + if reduction not in [None, 'mean', 'sum', 'mean_batch']: + raise ValueError('`reduction` must be one of [mean, sum, mean_batch]') + + self._blank = num_classes + self.reduction = reduction + self._loss = resolve_rnnt_loss(loss_name, blank_idx=self._blank, loss_kwargs=loss_kwargs) + + @typecheck() + def forward(self, log_probs, targets, input_lengths, target_lengths): + # Cast to int 32 + targets = targets.int() + input_lengths = input_lengths.int() + target_lengths = target_lengths.int() + + max_logit_len = input_lengths.max() + max_targets_len = target_lengths.max() + + # Force cast joint to float32 + # TODO: Remove once Numba supports FP16 + if log_probs.dtype != torch.float32: + logits_orig = log_probs + log_probs = log_probs.float() + del logits_orig # save memory *before* computing the loss + + # Ensure that shape mismatch does not occur due to padding + # Due to padding and subsequent downsampling, it may be possible that + # max sequence length computed does not match the actual max sequence length + # of the log_probs tensor, therefore we increment the input_lengths by the difference. + # This difference is generally small. + if log_probs.shape[1] != max_logit_len: + log_probs = log_probs.narrow(dim=1, start=0, length=max_logit_len).contiguous() + + # Reduce transcript length to correct alignment if additional padding was applied. + # Transcript: [B, L] -> [B, L']; If L' < L + if targets.shape[1] != max_targets_len: + targets = targets.narrow(dim=1, start=0, length=max_targets_len) + + # Loss reduction can be dynamic, so set it prior to call + if self.reduction != 'mean_batch': + self._loss.reduction = self.reduction + + # Compute RNNT loss + loss = self._loss(acts=log_probs, labels=targets, act_lens=input_lengths, label_lens=target_lengths) + + # Loss reduction can be dynamic, so reset it after call + if self.reduction != 'mean_batch': + self._loss.reduction = 'none' + + # Loss reduction only for mean_batch mode + if self.reduction == 'mean_batch': + loss = torch.mean(loss) + + # del new variables that may have been created + del ( + log_probs, + targets, + input_lengths, + target_lengths, + ) + + return loss diff --git a/models/tacotron2/NeMo/nemo/collections/asr/losses/wav2vecloss.py b/models/tacotron2/NeMo/nemo/collections/asr/losses/wav2vecloss.py new file mode 100644 index 0000000000..8459103814 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/losses/wav2vecloss.py @@ -0,0 +1,118 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import torch +import torch.nn.functional as F + +from nemo.core import Loss, typecheck +from nemo.core.neural_types import EncodedRepresentation, LossType, NeuralType + + +class Wav2VecLoss(Loss): + @property + def input_types(self): + """Input types definitions for Wav2VecLoss. + """ + return { + "logits": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "targets": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "negatives": NeuralType(('N', 'B', 'T', 'D'), EncodedRepresentation()), + "prob_ppl_loss": NeuralType(elements_type=LossType()), + "feature_loss": NeuralType(elements_type=LossType()), + } + + @property + def output_types(self): + """Output types definitions for Wav2VecLoss. + """ + return { + "loss": NeuralType(elements_type=LossType()), + "feature_loss": NeuralType(elements_type=LossType()), + "prob_ppl_loss": NeuralType(elements_type=LossType()), + } + + def __init__(self, feature_loss_weight: float, prob_ppl_weight: float, logit_temp: float, reduce: bool = True): + """ + Compute the contrastive loss with respect to the model outputs and sampled negatives from quantizer codebooks. + Args: + feature_loss_weight: Feature penalty weight (L2 Norm) + prob_ppl_weight: Perplexity Loss with respect to probabilities during quantization + logit_temp: Temperature normalization applied in loss. + reduce: Reduce loss via sum reduction (Default true) + """ + super().__init__() + self.feature_loss_weight = feature_loss_weight + self.prob_ppl_weight = prob_ppl_weight + self.logit_temp = logit_temp + self.reduce = reduce + + @typecheck() + def forward( + self, + logits: torch.tensor, + targets: torch.tensor, + negatives: torch.tensor, + prob_ppl_loss: torch.tensor, + feature_loss: torch.tensor, + ) -> [torch.tensor, torch.tensor, torch.tensor]: + """ + Args: + logits: Model activations + targets: The true target quantized representations + negatives: Sampled negatives from the quantizer codebooks. Sampled from all other timesteps. + feature_loss: Feature penalty (L2 Norm) + prob_ppl_loss: Perplexity Loss with respect to probs in quantization + Returns: + output loss values, feature loss, prob_ppl loss (after scaling). + """ + + # Calculate similarity between logits and all targets, returning FxBxT + similarity_scores = self._calculate_similarity(logits, negatives, targets) + + # Create targets of size B*T + similarity_targets = logits.new_zeros(similarity_scores.size(1) * similarity_scores.size(2), dtype=torch.long) + + # Transpose similarity scores to (T*B)xF for loss + similarity_scores = similarity_scores.transpose(0, 2) + similarity_scores = similarity_scores.reshape(-1, similarity_scores.size(-1)) + + loss = F.cross_entropy(similarity_scores, similarity_targets, reduction="sum" if self.reduce else "none") + + sample_size = similarity_targets.numel() + + if self.prob_ppl_weight != 0: + prob_ppl_loss = self.prob_ppl_weight * prob_ppl_loss * sample_size + loss += prob_ppl_loss + + if self.feature_loss_weight != 0: + feature_loss = self.feature_loss_weight * feature_loss * sample_size + loss += feature_loss + + return loss, feature_loss, prob_ppl_loss + + def _calculate_similarity(self, logits, negatives, targets): + neg_is_pos = (targets == negatives).all(-1) + targets = targets.unsqueeze(0) + targets = torch.cat([targets, negatives], dim=0) + logits = torch.cosine_similarity(logits.float(), targets.float(), dim=-1).type_as(logits) + logits /= self.logit_temp + if neg_is_pos.any(): + logits[1:][neg_is_pos] = float("-inf") + return logits diff --git a/models/tacotron2/NeMo/nemo/collections/asr/metrics/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/metrics/__init__.py new file mode 100644 index 0000000000..9e32500719 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/metrics/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/asr/metrics/rnnt_wer.py b/models/tacotron2/NeMo/nemo/collections/asr/metrics/rnnt_wer.py new file mode 100644 index 0000000000..17861a3f57 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/metrics/rnnt_wer.py @@ -0,0 +1,478 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import List, Optional, Union + +import editdistance +import torch +from torchmetrics import Metric + +from nemo.collections.asr.parts.submodules import rnnt_beam_decoding as beam_decode +from nemo.collections.asr.parts.submodules import rnnt_greedy_decoding as greedy_decode +from nemo.collections.asr.parts.utils.rnnt_utils import Hypothesis, NBestHypotheses +from nemo.utils import logging + +__all__ = ['RNNTDecoding', 'RNNTWER'] + + +class AbstractRNNTDecoding(ABC): + """ + Used for performing RNN-T auto-regressive decoding of the Decoder+Joint network given the encoder state. + + Args: + decoding_cfg: A dict-like object which contains the following key-value pairs. + strategy: str value which represents the type of decoding that can occur. + Possible values are : + - greedy, greedy_batch (for greedy decoding). + - beam, tsd, alsd (for beam search decoding). + + compute_hypothesis_token_set: A bool flag, which determines whether to compute a list of decoded + tokens as well as the decoded string. Default is False in order to avoid double decoding + unless required. + + preserve_alignments: Bool flag which preserves the history of logprobs generated during + decoding (sample / batched). When set to true, the Hypothesis will contain + the non-null value for `logprobs` in it. Here, `logprobs` is a List of torch.Tensors. + + In order to obtain this hypothesis, please utilize `rnnt_decoder_predictions_tensor` function + with the `return_hypotheses` flag set to True. + + The length of the list corresponds to the Acoustic Length (T). + Each value in the list (Ti) is a torch.Tensor (U), representing 1 or more targets from a vocabulary. + U is the number of target tokens for the current timestep Ti. + + The config may further contain the following sub-dictionaries: + "greedy": + max_symbols: int, describing the maximum number of target tokens to decode per + timestep during greedy decoding. Setting to larger values allows longer sentences + to be decoded, at the cost of increased execution time. + + "beam": + beam_size: int, defining the beam size for beam search. Must be >= 1. + If beam_size == 1, will perform cached greedy search. This might be slightly different + results compared to the greedy search above. + + score_norm: optional bool, whether to normalize the returned beam score in the hypotheses. + Set to True by default. + + return_best_hypothesis: optional bool, whether to return just the best hypothesis or all of the + hypotheses after beam search has concluded. This flag is set by default. + + tsd_max_sym_exp: optional int, determines number of symmetric expansions of the target symbols + per timestep of the acoustic model. Larger values will allow longer sentences to be decoded, + at increased cost to execution time. + + alsd_max_target_len: optional int or float, determines the potential maximum target sequence length. + If an integer is provided, it can decode sequences of that particular maximum length. + If a float is provided, it can decode sequences of int(alsd_max_target_len * seq_len), + where seq_len is the length of the acoustic model output (T). + + NOTE: + If a float is provided, it can be greater than 1! + By default, a float of 2.0 is used so that a target sequence can be at most twice + as long as the acoustic model output length T. + + decoder: The Decoder/Prediction network module. + joint: The Joint network module. + blank_id: The id of the RNNT blank token. + """ + + def __init__(self, decoding_cfg, decoder, joint, blank_id: int): + super(AbstractRNNTDecoding, self).__init__() + self.cfg = decoding_cfg + self.blank_id = blank_id + self.compute_hypothesis_token_set = self.cfg.get("compute_hypothesis_token_set", False) + self.preserve_alignments = self.cfg.get('preserve_alignments', False) + + possible_strategies = ['greedy', 'greedy_batch', 'beam', 'tsd', 'alsd'] + if self.cfg.strategy not in possible_strategies: + raise ValueError(f"Decoding strategy must be one of {possible_strategies}") + + if self.cfg.strategy == 'greedy': + self.decoding = greedy_decode.GreedyRNNTInfer( + decoder_model=decoder, + joint_model=joint, + blank_index=self.blank_id, + max_symbols_per_step=self.cfg.greedy.get('max_symbols', None), + preserve_alignments=self.preserve_alignments, + ) + + elif self.cfg.strategy == 'greedy_batch': + self.decoding = greedy_decode.GreedyBatchedRNNTInfer( + decoder_model=decoder, + joint_model=joint, + blank_index=self.blank_id, + max_symbols_per_step=self.cfg.greedy.get('max_symbols', None), + preserve_alignments=self.preserve_alignments, + ) + + elif self.cfg.strategy == 'beam': + + self.decoding = beam_decode.BeamRNNTInfer( + decoder_model=decoder, + joint_model=joint, + beam_size=self.cfg.beam.beam_size, + return_best_hypothesis=decoding_cfg.beam.get('return_best_hypothesis', True), + search_type='default', + score_norm=self.cfg.beam.get('score_norm', True), + preserve_alignments=self.preserve_alignments, + ) + + elif self.cfg.strategy == 'tsd': + + self.decoding = beam_decode.BeamRNNTInfer( + decoder_model=decoder, + joint_model=joint, + beam_size=self.cfg.beam.beam_size, + return_best_hypothesis=decoding_cfg.beam.get('return_best_hypothesis', True), + search_type='tsd', + score_norm=self.cfg.beam.get('score_norm', True), + tsd_max_sym_exp_per_step=self.cfg.beam.get('tsd_max_sym_exp', 50), + preserve_alignments=self.preserve_alignments, + ) + + elif self.cfg.strategy == 'alsd': + + self.decoding = beam_decode.BeamRNNTInfer( + decoder_model=decoder, + joint_model=joint, + beam_size=self.cfg.beam.beam_size, + return_best_hypothesis=decoding_cfg.beam.get('return_best_hypothesis', True), + search_type='alsd', + score_norm=self.cfg.beam.get('score_norm', True), + alsd_max_target_len=self.cfg.beam.get('alsd_max_target_len', 2), + preserve_alignments=self.preserve_alignments, + ) + + def rnnt_decoder_predictions_tensor( + self, encoder_output: torch.Tensor, encoded_lengths: torch.Tensor, return_hypotheses: bool = False + ) -> (List[str], Optional[List[List[str]]], Optional[Union[Hypothesis, NBestHypotheses]]): + """ + Decode an encoder output by autoregressive decoding of the Decoder+Joint networks. + + Args: + encoder_output: torch.Tensor of shape [B, D, T]. + encoded_lengths: torch.Tensor containing lengths of the padded encoder outputs. Shape [B]. + return_hypotheses: bool. If set to True it will return list of Hypothesis or NBestHypotheses + + Returns: + If `return_best_hypothesis` is set: + A tuple (hypotheses, None): + hypotheses - list of Hypothesis (best hypothesis per sample). + Look at rnnt_utils.Hypothesis for more information. + + If `return_best_hypothesis` is not set: + A tuple(hypotheses, all_hypotheses) + hypotheses - list of Hypothesis (best hypothesis per sample). + Look at rnnt_utils.Hypothesis for more information. + all_hypotheses - list of NBestHypotheses. Each NBestHypotheses further contains a sorted + list of all the hypotheses of the model per sample. + Look at rnnt_utils.NBestHypotheses for more information. + """ + # Compute hypotheses + with torch.no_grad(): + hypotheses_list = self.decoding( + encoder_output=encoder_output, encoded_lengths=encoded_lengths + ) # type: [List[Hypothesis]] + + # extract the hypotheses + hypotheses_list = hypotheses_list[0] # type: List[Hypothesis] + + prediction_list = hypotheses_list + + if isinstance(prediction_list[0], NBestHypotheses): + hypotheses = [] + all_hypotheses = [] + for nbest_hyp in prediction_list: # type: NBestHypotheses + n_hyps = nbest_hyp.n_best_hypotheses # Extract all hypotheses for this sample + decoded_hyps = self.decode_hypothesis(n_hyps) # type: List[str] + hypotheses.append(decoded_hyps[0]) # best hypothesis + all_hypotheses.append(decoded_hyps) + if return_hypotheses: + return hypotheses, all_hypotheses + best_hyp_text = [h.text for h in hypotheses] + all_hyp_text = [h.text for hh in all_hypotheses for h in hh] + return best_hyp_text, all_hyp_text + else: + hypotheses = self.decode_hypothesis(prediction_list) # type: List[str] + if return_hypotheses: + return hypotheses, None + best_hyp_text = [h.text for h in hypotheses] + return best_hyp_text, None + + def decode_hypothesis(self, hypotheses_list: List[Hypothesis]) -> List[Union[Hypothesis, NBestHypotheses]]: + """ + Decode a list of hypotheses into a list of strings. + + Args: + hypotheses_list: List of Hypothesis. + + Returns: + A list of strings. + """ + for ind in range(len(hypotheses_list)): + # Extract the integer encoded hypothesis + prediction = hypotheses_list[ind].y_sequence + + if type(prediction) != list: + prediction = prediction.tolist() + + # RNN-T sample level is already preprocessed by implicit CTC decoding + # Simply remove any blank tokens + prediction = [p for p in prediction if p != self.blank_id] + + # De-tokenize the integer tokens + hypothesis = self.decode_tokens_to_str(prediction) + hypotheses_list[ind].text = hypothesis + + if self.compute_hypothesis_token_set: + hypotheses_list[ind].tokens = self.decode_ids_to_tokens(prediction) + return hypotheses_list + + @abstractmethod + def decode_tokens_to_str(self, tokens: List[int]) -> str: + """ + Implemented by subclass in order to decoder a token id list into a string. + + Args: + tokens: List of int representing the token ids. + + Returns: + A decoded string. + """ + raise NotImplementedError() + + @abstractmethod + def decode_ids_to_tokens(self, tokens: List[int]) -> List[str]: + """ + Implemented by subclass in order to decode a token id list into a token list. + A token list is the string representation of each token id. + + Args: + tokens: List of int representing the token ids. + + Returns: + A list of decoded tokens. + """ + raise NotImplementedError() + + +class RNNTDecoding(AbstractRNNTDecoding): + """ + Used for performing RNN-T auto-regressive decoding of the Decoder+Joint network given the encoder state. + + Args: + decoding_cfg: A dict-like object which contains the following key-value pairs. + strategy: str value which represents the type of decoding that can occur. + Possible values are : + - greedy, greedy_batch (for greedy decoding). + - beam, tsd, alsd (for beam search decoding). + + compute_hypothesis_token_set: A bool flag, which determines whether to compute a list of decoded + tokens as well as the decoded string. Default is False in order to avoid double decoding + unless required. + + preserve_alignments: Bool flag which preserves the history of logprobs generated during + decoding (sample / batched). When set to true, the Hypothesis will contain + the non-null value for `logprobs` in it. Here, `logprobs` is a List of torch.Tensors. + + In order to obtain this hypothesis, please utilize `rnnt_decoder_predictions_tensor` function + with the `return_hypotheses` flag set to True. + + The length of the list corresponds to the Acoustic Length (T). + Each value in the list (Ti) is a torch.Tensor (U), representing 1 or more targets from a vocabulary. + U is the number of target tokens for the current timestep Ti. + + The config may further contain the following sub-dictionaries: + "greedy": + max_symbols: int, describing the maximum number of target tokens to decode per + timestep during greedy decoding. Setting to larger values allows longer sentences + to be decoded, at the cost of increased execution time. + + "beam": + beam_size: int, defining the beam size for beam search. Must be >= 1. + If beam_size == 1, will perform cached greedy search. This might be slightly different + results compared to the greedy search above. + + score_norm: optional bool, whether to normalize the returned beam score in the hypotheses. + Set to True by default. + + return_best_hypothesis: optional bool, whether to return just the best hypothesis or all of the + hypotheses after beam search has concluded. This flag is set by default. + + tsd_max_sym_exp: optional int, determines number of symmetric expansions of the target symbols + per timestep of the acoustic model. Larger values will allow longer sentences to be decoded, + at increased cost to execution time. + + alsd_max_target_len: optional int or float, determines the potential maximum target sequence length. + If an integer is provided, it can decode sequences of that particular maximum length. + If a float is provided, it can decode sequences of int(alsd_max_target_len * seq_len), + where seq_len is the length of the acoustic model output (T). + + NOTE: + If a float is provided, it can be greater than 1! + By default, a float of 2.0 is used so that a target sequence can be at most twice + as long as the acoustic model output length T. + + decoder: The Decoder/Prediction network module. + joint: The Joint network module. + vocabulary: The vocabulary (excluding the RNNT blank token) which will be used for decoding. + """ + + def __init__( + self, decoding_cfg, decoder, joint, vocabulary, + ): + blank_id = len(vocabulary) + self.labels_map = dict([(i, vocabulary[i]) for i in range(len(vocabulary))]) + + super(RNNTDecoding, self).__init__(decoding_cfg=decoding_cfg, decoder=decoder, joint=joint, blank_id=blank_id) + + def decode_tokens_to_str(self, tokens: List[int]) -> str: + """ + Implemented by subclass in order to decoder a token list into a string. + + Args: + tokens: List of int representing the token ids. + + Returns: + A decoded string. + """ + hypothesis = ''.join(self.decode_ids_to_tokens(tokens)) + return hypothesis + + def decode_ids_to_tokens(self, tokens: List[int]) -> List[str]: + """ + Implemented by subclass in order to decode a token id list into a token list. + A token list is the string representation of each token id. + + Args: + tokens: List of int representing the token ids. + + Returns: + A list of decoded tokens. + """ + token_list = [self.labels_map[c] for c in tokens if c != self.blank_id] + return token_list + + +class RNNTWER(Metric): + """ + This metric computes numerator and denominator for Overall Word Error Rate (WER) between prediction and reference texts. + When doing distributed training/evaluation the result of res=WER(predictions, targets, target_lengths) calls + will be all-reduced between all workers using SUM operations. + Here contains two numbers res=[wer_numerator, wer_denominator]. WER=wer_numerator/wer_denominator. + + If used with PytorchLightning LightningModule, include wer_numerator and wer_denominators inside validation_step results. + Then aggregate (sum) then at the end of validation epoch to correctly compute validation WER. + + Example: + def validation_step(self, batch, batch_idx): + ... + wer_num, wer_denom = self.__wer(predictions, transcript, transcript_len) + return {'val_loss': loss_value, 'val_wer_num': wer_num, 'val_wer_denom': wer_denom} + + def validation_epoch_end(self, outputs): + ... + wer_num = torch.stack([x['val_wer_num'] for x in outputs]).sum() + wer_denom = torch.stack([x['val_wer_denom'] for x in outputs]).sum() + tensorboard_logs = {'validation_loss': val_loss_mean, 'validation_avg_wer': wer_num / wer_denom} + return {'val_loss': val_loss_mean, 'log': tensorboard_logs} + + Args: + decoding: RNNTDecoding object that will perform autoregressive decoding of the RNNT model. + batch_dim_index: Index of the batch dimension. + use_cer: Whether to use Character Error Rate isntead of Word Error Rate. + log_prediction: Whether to log a single decoded sample per call. + + Returns: + res: a torch.Tensor object with two elements: [wer_numerator, wer_denominator]. To correctly compute average + text word error rate, compute wer=wer_numerator/wer_denominator + """ + + def __init__( + self, decoding: RNNTDecoding, batch_dim_index=0, use_cer=False, log_prediction=True, dist_sync_on_step=False + ): + super(RNNTWER, self).__init__(dist_sync_on_step=dist_sync_on_step, compute_on_step=False) + self.decoding = decoding + self.batch_dim_index = batch_dim_index + self.use_cer = use_cer + self.log_prediction = log_prediction + self.blank_id = self.decoding.blank_id + self.labels_map = self.decoding.labels_map + + self.add_state("scores", default=torch.tensor(0), dist_reduce_fx='sum', persistent=False) + self.add_state("words", default=torch.tensor(0), dist_reduce_fx='sum', persistent=False) + + def update( + self, + encoder_output: torch.Tensor, + encoded_lengths: torch.Tensor, + targets: torch.Tensor, + target_lengths: torch.Tensor, + ) -> torch.Tensor: + words = 0.0 + scores = 0.0 + references = [] + with torch.no_grad(): + # prediction_cpu_tensor = tensors[0].long().cpu() + targets_cpu_tensor = targets.long().cpu() + tgt_lenths_cpu_tensor = target_lengths.long().cpu() + + # iterate over batch + for ind in range(targets_cpu_tensor.shape[self.batch_dim_index]): + tgt_len = tgt_lenths_cpu_tensor[ind].item() + target = targets_cpu_tensor[ind][:tgt_len].numpy().tolist() + + reference = self.decoding.decode_tokens_to_str(target) + references.append(reference) + + hypotheses, _ = self.decoding.rnnt_decoder_predictions_tensor(encoder_output, encoded_lengths) + + if self.log_prediction: + logging.info(f"\n") + logging.info(f"reference :{references[0]}") + logging.info(f"predicted :{hypotheses[0]}") + + for h, r in zip(hypotheses, references): + if self.use_cer: + h_list = list(h) + r_list = list(r) + else: + h_list = h.split() + r_list = r.split() + words += len(r_list) + # Compute Levenshtein's distance + scores += editdistance.eval(h_list, r_list) + + self.scores += torch.tensor(scores, device=self.scores.device, dtype=self.scores.dtype) + self.words += torch.tensor(words, device=self.words.device, dtype=self.words.dtype) + # return torch.tensor([scores, words]).to(predictions.device) + + def compute(self): + wer = self.scores.float() / self.words + return wer, self.scores.detach(), self.words.detach() + + +@dataclass +class RNNTDecodingConfig: + strategy: str = "greedy_batch" + compute_hypothesis_token_set: bool = False + + # greedy decoding config + greedy: greedy_decode.GreedyRNNTInferConfig = greedy_decode.GreedyRNNTInferConfig() + + # beam decoding config + beam: beam_decode.BeamRNNTInferConfig = beam_decode.BeamRNNTInferConfig(beam_size=4) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/metrics/rnnt_wer_bpe.py b/models/tacotron2/NeMo/nemo/collections/asr/metrics/rnnt_wer_bpe.py new file mode 100644 index 0000000000..0f679592a1 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/metrics/rnnt_wer_bpe.py @@ -0,0 +1,242 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import List + +import editdistance +import torch +from torchmetrics import Metric + +from nemo.collections.asr.metrics.rnnt_wer import AbstractRNNTDecoding +from nemo.collections.asr.parts.submodules import rnnt_beam_decoding as beam_decode +from nemo.collections.asr.parts.submodules import rnnt_greedy_decoding as greedy_decode +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec +from nemo.utils import logging + +__all__ = ['RNNTBPEDecoding', 'RNNTBPEWER'] + + +class RNNTBPEDecoding(AbstractRNNTDecoding): + """ + Used for performing RNN-T auto-regressive decoding of the Decoder+Joint network given the encoder state. + + Args: + decoding_cfg: A dict-like object which contains the following key-value pairs. + strategy: str value which represents the type of decoding that can occur. + Possible values are : + - greedy, greedy_batch (for greedy decoding). + - beam, tsd, alsd (for beam search decoding). + + compute_hypothesis_token_set: A bool flag, which determines whether to compute a list of decoded + tokens as well as the decoded string. Default is False in order to avoid double decoding + unless required. + + preserve_alignments: Bool flag which preserves the history of logprobs generated during + decoding (sample / batched). When set to true, the Hypothesis will contain + the non-null value for `logprobs` in it. Here, `logprobs` is a List of torch.Tensors. + + In order to obtain this hypothesis, please utilize `rnnt_decoder_predictions_tensor` function + with the `return_hypotheses` flag set to True. + + The length of the list corresponds to the Acoustic Length (T). + Each value in the list (Ti) is a torch.Tensor (U), representing 1 or more targets from a vocabulary. + U is the number of target tokens for the current timestep Ti. + + The config may further contain the following sub-dictionaries: + "greedy": + max_symbols: int, describing the maximum number of target tokens to decode per + timestep during greedy decoding. Setting to larger values allows longer sentences + to be decoded, at the cost of increased execution time. + + "beam": + beam_size: int, defining the beam size for beam search. Must be >= 1. + If beam_size == 1, will perform cached greedy search. This might be slightly different + results compared to the greedy search above. + + score_norm: optional bool, whether to normalize the returned beam score in the hypotheses. + Set to True by default. + + return_best_hypothesis: optional bool, whether to return just the best hypothesis or all of the + hypotheses after beam search has concluded. + + tsd_max_sym_exp: optional int, determines number of symmetric expansions of the target symbols + per timestep of the acoustic model. Larger values will allow longer sentences to be decoded, + at increased cost to execution time. + + alsd_max_target_len: optional int or float, determines the potential maximum target sequence length. + If an integer is provided, it can decode sequences of that particular maximum length. + If a float is provided, it can decode sequences of int(alsd_max_target_len * seq_len), + where seq_len is the length of the acoustic model output (T). + + NOTE: + If a float is provided, it can be greater than 1! + By default, a float of 2.0 is used so that a target sequence can be at most twice + as long as the acoustic model output length T. + + decoder: The Decoder/Prediction network module. + joint: The Joint network module. + tokenizer: The tokenizer which will be used for decoding. + """ + + def __init__(self, decoding_cfg, decoder, joint, tokenizer: TokenizerSpec): + blank_id = tokenizer.tokenizer.vocab_size + self.tokenizer = tokenizer + + super(RNNTBPEDecoding, self).__init__( + decoding_cfg=decoding_cfg, decoder=decoder, joint=joint, blank_id=blank_id + ) + + def decode_tokens_to_str(self, tokens: List[int]) -> str: + """ + Implemented by subclass in order to decoder a token list into a string. + + Args: + tokens: List of int representing the token ids. + + Returns: + A decoded string. + """ + hypothesis = self.tokenizer.ids_to_text(tokens) + return hypothesis + + def decode_ids_to_tokens(self, tokens: List[int]) -> List[str]: + """ + Implemented by subclass in order to decode a token id list into a token list. + A token list is the string representation of each token id. + + Args: + tokens: List of int representing the token ids. + + Returns: + A list of decoded tokens. + """ + token_list = self.tokenizer.ids_to_tokens(tokens) + return token_list + + +class RNNTBPEWER(Metric): + """ + This metric computes numerator and denominator for Overall Word Error Rate (WER) between prediction and reference texts. + When doing distributed training/evaluation the result of res=WER(predictions, targets, target_lengths) calls + will be all-reduced between all workers using SUM operations. + Here contains two numbers res=[wer_numerator, wer_denominator]. WER=wer_numerator/wer_denominator. + + If used with PytorchLightning LightningModule, include wer_numerator and wer_denominators inside validation_step results. + Then aggregate (sum) then at the end of validation epoch to correctly compute validation WER. + + Example: + def validation_step(self, batch, batch_idx): + ... + wer_num, wer_denom = self.__wer(predictions, transcript, transcript_len) + return {'val_loss': loss_value, 'val_wer_num': wer_num, 'val_wer_denom': wer_denom} + + def validation_epoch_end(self, outputs): + ... + wer_num = torch.stack([x['val_wer_num'] for x in outputs]).sum() + wer_denom = torch.stack([x['val_wer_denom'] for x in outputs]).sum() + tensorboard_logs = {'validation_loss': val_loss_mean, 'validation_avg_wer': wer_num / wer_denom} + return {'val_loss': val_loss_mean, 'log': tensorboard_logs} + + Args: + decoding: RNNTBPEDecoding object that will perform autoregressive decoding of the RNNT model. + batch_dim_index: Index of the batch dimension. + use_cer: Whether to use Character Error Rate isntead of Word Error Rate. + log_prediction: Whether to log a single decoded sample per call. + + Returns: + res: a torch.Tensor object with two elements: [wer_numerator, wer_denominator]. To correctly compute average + text word error rate, compute wer=wer_numerator/wer_denominator + """ + + def __init__( + self, + decoding: RNNTBPEDecoding, + batch_dim_index=0, + use_cer: bool = False, + log_prediction: bool = True, + dist_sync_on_step=False, + ): + super(RNNTBPEWER, self).__init__(dist_sync_on_step=dist_sync_on_step, compute_on_step=False) + self.decoding = decoding + self.batch_dim_index = batch_dim_index + self.use_cer = use_cer + self.log_prediction = log_prediction + self.blank_id = self.decoding.blank_id + self.tokenizer = self.decoding.tokenizer + + self.add_state("scores", default=torch.tensor(0), dist_reduce_fx='sum', persistent=False) + self.add_state("words", default=torch.tensor(0), dist_reduce_fx='sum', persistent=False) + + def update( + self, + encoder_output: torch.Tensor, + encoded_lengths: torch.Tensor, + targets: torch.Tensor, + target_lengths: torch.Tensor, + ) -> torch.Tensor: + words = 0.0 + scores = 0.0 + references = [] + with torch.no_grad(): + # prediction_cpu_tensor = tensors[0].long().cpu() + targets_cpu_tensor = targets.long().cpu() + tgt_lenths_cpu_tensor = target_lengths.long().cpu() + + # iterate over batch + for ind in range(targets_cpu_tensor.shape[self.batch_dim_index]): + tgt_len = tgt_lenths_cpu_tensor[ind].item() + target = targets_cpu_tensor[ind][:tgt_len].numpy().tolist() + reference = self.decoding.decode_tokens_to_str(target) + references.append(reference) + + hypotheses, _ = self.decoding.rnnt_decoder_predictions_tensor(encoder_output, encoded_lengths) + + if self.log_prediction: + logging.info(f"\n") + logging.info(f"reference :{references[0]}") + logging.info(f"predicted :{hypotheses[0]}") + + for h, r in zip(hypotheses, references): + if self.use_cer: + h_list = list(h) + r_list = list(r) + else: + h_list = h.split() + r_list = r.split() + words += len(r_list) + # Compute Levenshtein's distance + scores += editdistance.eval(h_list, r_list) + + del hypotheses + + self.scores += torch.tensor(scores, device=self.scores.device, dtype=self.scores.dtype) + self.words += torch.tensor(words, device=self.words.device, dtype=self.words.dtype) + # return torch.tensor([scores, words]).to(predictions.device) + + def compute(self): + wer = self.scores.float() / self.words + return wer, self.scores.detach(), self.words.detach() + + +@dataclass +class RNNTBPEDecodingConfig: + strategy: str = "greedy_batch" + compute_hypothesis_token_set: bool = False + + # greedy decoding config + greedy: greedy_decode.GreedyRNNTInferConfig = greedy_decode.GreedyRNNTInferConfig() + + # beam decoding config + beam: beam_decode.BeamRNNTInferConfig = beam_decode.BeamRNNTInferConfig(beam_size=4) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/metrics/wer.py b/models/tacotron2/NeMo/nemo/collections/asr/metrics/wer.py new file mode 100644 index 0000000000..e3d62f4281 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/metrics/wer.py @@ -0,0 +1,247 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import editdistance +import torch +from torchmetrics import Metric + +from nemo.collections.asr.parts.utils.rnnt_utils import Hypothesis +from nemo.utils import logging + +__all__ = ['word_error_rate', 'WER'] + + +def word_error_rate(hypotheses: List[str], references: List[str], use_cer=False) -> float: + """ + Computes Average Word Error rate between two texts represented as + corresponding lists of string. Hypotheses and references must have same + length. + Args: + hypotheses: list of hypotheses + references: list of references + use_cer: bool, set True to enable cer + Returns: + (float) average word error rate + """ + scores = 0 + words = 0 + if len(hypotheses) != len(references): + raise ValueError( + "In word error rate calculation, hypotheses and reference" + " lists must have the same number of elements. But I got:" + "{0} and {1} correspondingly".format(len(hypotheses), len(references)) + ) + for h, r in zip(hypotheses, references): + if use_cer: + h_list = list(h) + r_list = list(r) + else: + h_list = h.split() + r_list = r.split() + words += len(r_list) + scores += editdistance.eval(h_list, r_list) + if words != 0: + wer = 1.0 * scores / words + else: + wer = float('inf') + return wer + + +class WER(Metric): + """ + This metric computes numerator and denominator for Overall Word Error Rate (WER) between prediction and reference texts. + When doing distributed training/evaluation the result of res=WER(predictions, targets, target_lengths) calls + will be all-reduced between all workers using SUM operations. + Here contains two numbers res=[wer_numerator, wer_denominator]. WER=wer_numerator/wer_denominator. + + If used with PytorchLightning LightningModule, include wer_numerator and wer_denominators inside validation_step results. + Then aggregate (sum) then at the end of validation epoch to correctly compute validation WER. + + Example: + def validation_step(self, batch, batch_idx): + ... + wer_num, wer_denom = self.__wer(predictions, transcript, transcript_len) + return {'val_loss': loss_value, 'val_wer_num': wer_num, 'val_wer_denom': wer_denom} + + def validation_epoch_end(self, outputs): + ... + wer_num = torch.stack([x['val_wer_num'] for x in outputs]).sum() + wer_denom = torch.stack([x['val_wer_denom'] for x in outputs]).sum() + tensorboard_logs = {'validation_loss': val_loss_mean, 'validation_avg_wer': wer_num / wer_denom} + return {'val_loss': val_loss_mean, 'log': tensorboard_logs} + + Args: + vocabulary: List of strings that describes the vocabulary of the dataset. + batch_dim_index: Index of the batch dimension. + use_cer: Whether to use Character Error Rate instead of Word Error Rate. + ctc_decode: Whether to use CTC decoding or not. Currently, must be set. + log_prediction: Whether to log a single decoded sample per call. + + Returns: + res: a torch.Tensor object with two elements: [wer_numerator, wer_denominator]. To correctly compute average + text word error rate, compute wer=wer_numerator/wer_denominator + """ + + def __init__( + self, + vocabulary, + batch_dim_index=0, + use_cer=False, + ctc_decode=True, + log_prediction=True, + dist_sync_on_step=False, + ): + super().__init__(dist_sync_on_step=dist_sync_on_step, compute_on_step=False) + self.batch_dim_index = batch_dim_index + self.blank_id = len(vocabulary) + self.labels_map = dict([(i, vocabulary[i]) for i in range(len(vocabulary))]) + self.use_cer = use_cer + self.ctc_decode = ctc_decode + self.log_prediction = log_prediction + + self.add_state("scores", default=torch.tensor(0), dist_reduce_fx='sum', persistent=False) + self.add_state("words", default=torch.tensor(0), dist_reduce_fx='sum', persistent=False) + + def ctc_decoder_predictions_tensor( + self, predictions: torch.Tensor, predictions_len: torch.Tensor = None, return_hypotheses: bool = False, + ) -> List[str]: + """ + Decodes a sequence of labels to words + + Args: + predictions: A torch.Tensor of shape [Batch, Time] of integer indices that correspond + to the index of some character in the label set. + predictions_len: Optional tensor of length `Batch` which contains the integer lengths + of the sequence in the padded `predictions` tensor. + return_hypotheses: Bool flag whether to return just the decoding predictions of the model + or a Hypothesis object that holds information such as the decoded `text`, + the `alignment` of emited by the CTC Model, and the `length` of the sequence (if available). + May also contain the log-probabilities of the decoder (if this method is called via + transcribe()) + + Returns: + Either a list of str which represent the CTC decoded strings per sample, + or a list of Hypothesis objects containing additional information. + """ + hypotheses = [] + # Drop predictions to CPU + prediction_cpu_tensor = predictions.long().cpu() + # iterate over batch + for ind in range(prediction_cpu_tensor.shape[self.batch_dim_index]): + prediction = prediction_cpu_tensor[ind].detach().numpy().tolist() + if predictions_len is not None: + prediction = prediction[: predictions_len[ind]] + # CTC decoding procedure + decoded_prediction = [] + previous = self.blank_id + for p in prediction: + if (p != previous or previous == self.blank_id) and p != self.blank_id: + decoded_prediction.append(p) + previous = p + + text = self.decode_tokens_to_str(decoded_prediction) + + if not return_hypotheses: + hypothesis = text + else: + hypothesis = Hypothesis( + y_sequence=None, + score=-1.0, + text=text, + alignments=prediction, + length=predictions_len[ind] if predictions_len is not None else 0, + ) + + hypotheses.append(hypothesis) + return hypotheses + + def decode_tokens_to_str(self, tokens: List[int]) -> str: + """ + Implemented in order to decoder a token list into a string. + + Args: + tokens: List of int representing the token ids. + + Returns: + A decoded string. + """ + hypothesis = ''.join(self.decode_ids_to_tokens(tokens)) + return hypothesis + + def decode_ids_to_tokens(self, tokens: List[int]) -> List[str]: + """ + Implemented in order to decode a token id list into a token list. + A token list is the string representation of each token id. + + Args: + tokens: List of int representing the token ids. + + Returns: + A list of decoded tokens. + """ + token_list = [self.labels_map[c] for c in tokens if c != self.blank_id] + return token_list + + def update( + self, + predictions: torch.Tensor, + targets: torch.Tensor, + target_lengths: torch.Tensor, + predictions_lengths: torch.Tensor = None, + ) -> torch.Tensor: + words = 0.0 + scores = 0.0 + references = [] + with torch.no_grad(): + # prediction_cpu_tensor = tensors[0].long().cpu() + targets_cpu_tensor = targets.long().cpu() + tgt_lenths_cpu_tensor = target_lengths.long().cpu() + + # iterate over batch + for ind in range(targets_cpu_tensor.shape[self.batch_dim_index]): + tgt_len = tgt_lenths_cpu_tensor[ind].item() + target = targets_cpu_tensor[ind][:tgt_len].numpy().tolist() + reference = self.decode_tokens_to_str(target) + references.append(reference) + if self.ctc_decode: + hypotheses = self.ctc_decoder_predictions_tensor(predictions, predictions_lengths) + else: + raise NotImplementedError("Implement me if you need non-CTC decode on predictions") + + if self.log_prediction: + logging.info(f"\n") + logging.info(f"reference:{references[0]}") + logging.info(f"predicted:{hypotheses[0]}") + + for h, r in zip(hypotheses, references): + if self.use_cer: + h_list = list(h) + r_list = list(r) + else: + h_list = h.split() + r_list = r.split() + words += len(r_list) + # Compute Levenstein's distance + scores += editdistance.eval(h_list, r_list) + + self.scores = torch.tensor(scores, device=self.scores.device, dtype=self.scores.dtype) + self.words = torch.tensor(words, device=self.words.device, dtype=self.words.dtype) + # return torch.tensor([scores, words]).to(predictions.device) + + def compute(self): + scores = self.scores.detach().float() + words = self.words.detach().float() + return scores / words, scores, words diff --git a/models/tacotron2/NeMo/nemo/collections/asr/metrics/wer_bpe.py b/models/tacotron2/NeMo/nemo/collections/asr/metrics/wer_bpe.py new file mode 100644 index 0000000000..a0e7d38556 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/metrics/wer_bpe.py @@ -0,0 +1,210 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import editdistance +import torch +from torchmetrics import Metric + +from nemo.collections.asr.parts.utils.rnnt_utils import Hypothesis +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec +from nemo.utils import logging + + +class WERBPE(Metric): + """ + This metric computes numerator and denominator for Overall Word Error Rate for BPE tokens (WER-BPE) between prediction and reference texts. + When doing distributed training/evaluation the result of res=WERBPE(predictions, targets, target_lengths) calls + will be all-reduced between all workers using SUM operations. + Here contains two numbers res=[wer_numerator, wer_denominator]. WERBPE=wer_numerator/wer_denominator. + + If used with PytorchLightning LightningModule, include wer_numerator and wer_denominators inside validation_step results. + Then aggregate (sum) then at the end of validation epoch to correctly compute validation WER. + + Example: + def validation_step(self, batch, batch_idx): + ... + wer_num, wer_denom = self.__wer(predictions, transcript, transcript_len) + return {'val_loss': loss_value, 'val_wer_num': wer_num, 'val_wer_denom': wer_denom} + + def validation_epoch_end(self, outputs): + ... + wer_num = torch.stack([x['val_wer_num'] for x in outputs]).sum() + wer_denom = torch.stack([x['val_wer_denom'] for x in outputs]).sum() + tensorboard_logs = {'validation_loss': val_loss_mean, 'validation_avg_wer': wer_num / wer_denom} + return {'val_loss': val_loss_mean, 'log': tensorboard_logs} + + Args: + vocabulary: NeMo tokenizer object, which inherits from TokenizerSpec. + batch_dim_index: Index of the batch dimension. + use_cer: Whether to compute word-error-rate or character-error-rate. + ctc_decode: Whether to perform CTC decode. + log_prediction: Whether to log a single decoded sample per call. + + Returns: + res: a torch.Tensor object with two elements: [wer_numerator, wer_denominators]. To correctly compute average + text word error rate, compute wer=wer_numerator/wer_denominators + """ + + def __init__( + self, + tokenizer: TokenizerSpec, + batch_dim_index=0, + use_cer=False, + ctc_decode=True, + log_prediction=True, + dist_sync_on_step=False, + ): + super().__init__(dist_sync_on_step=dist_sync_on_step, compute_on_step=False) + self.tokenizer = tokenizer + self.batch_dim_index = batch_dim_index + self.blank_id = tokenizer.tokenizer.vocab_size + self.use_cer = use_cer + self.ctc_decode = ctc_decode + self.log_prediction = log_prediction + + self.add_state("scores", default=torch.tensor(0), dist_reduce_fx='sum', persistent=False) + self.add_state("words", default=torch.tensor(0), dist_reduce_fx='sum', persistent=False) + + def ctc_decoder_predictions_tensor( + self, predictions: torch.Tensor, predictions_len: torch.Tensor = None, return_hypotheses: bool = False + ) -> List[str]: + """ + Decodes a sequence of labels to words + + Args: + predictions: A torch.Tensor of shape [Batch, Time] of integer indices that correspond + to the index of some character in the vocabulary of the tokenizer. + predictions_len: Optional tensor of length `Batch` which contains the integer lengths + of the sequence in the padded `predictions` tensor. + return_hypotheses: Bool flag whether to return just the decoding predictions of the model + or a Hypothesis object that holds information such as the decoded `text`, + the `alignment` of emited by the CTC Model, and the `length` of the sequence (if available). + May also contain the log-probabilities of the decoder (if this method is called via + transcribe()) inside `y_sequence`, otherwise it is set None as it is a duplicate of + `alignments`. + + Returns: + Either a list of str which represent the CTC decoded strings per sample, + or a list of Hypothesis objects containing additional information. + """ + hypotheses = [] + # Drop predictions to CPU + prediction_cpu_tensor = predictions.long().cpu() + # iterate over batch + for ind in range(prediction_cpu_tensor.shape[self.batch_dim_index]): + prediction = prediction_cpu_tensor[ind].detach().numpy().tolist() + if predictions_len is not None: + prediction = prediction[: predictions_len[ind]] + # CTC decoding procedure + decoded_prediction = [] + previous = self.blank_id + for p in prediction: + if (p != previous or previous == self.blank_id) and p != self.blank_id: + decoded_prediction.append(p) + previous = p + + text = self.decode_tokens_to_str(decoded_prediction) + + if not return_hypotheses: + hypothesis = text + else: + hypothesis = Hypothesis( + y_sequence=None, # logprob info added by transcribe method + score=-1.0, + text=text, + alignments=prediction, + length=predictions_len[ind] if predictions_len is not None else 0, + ) + hypotheses.append(hypothesis) + return hypotheses + + def decode_tokens_to_str(self, tokens: List[int]) -> str: + """ + Implemented in order to decoder a token list into a string. + + Args: + tokens: List of int representing the token ids. + + Returns: + A decoded string. + """ + hypothesis = self.tokenizer.ids_to_text(tokens) + return hypothesis + + def decode_ids_to_tokens(self, tokens: List[int]) -> List[str]: + """ + Implemented in order to decode a token id list into a token list. + A token list is the string representation of each token id. + + Args: + tokens: List of int representing the token ids. + + Returns: + A list of decoded tokens. + """ + token_list = self.tokenizer.ids_to_tokens(tokens) + return token_list + + def update( + self, + predictions: torch.Tensor, + targets: torch.Tensor, + target_lengths: torch.Tensor, + predictions_lengths: torch.Tensor = None, + ): + words = 0.0 + scores = 0.0 + references = [] + with torch.no_grad(): + # prediction_cpu_tensor = tensors[0].long().cpu() + targets_cpu_tensor = targets.long().cpu() + tgt_lenths_cpu_tensor = target_lengths.long().cpu() + + # iterate over batch + for ind in range(targets_cpu_tensor.shape[self.batch_dim_index]): + tgt_len = tgt_lenths_cpu_tensor[ind].item() + target = targets_cpu_tensor[ind][:tgt_len].numpy().tolist() + reference = self.decode_tokens_to_str(target) + references.append(reference) + if self.ctc_decode: + hypotheses = self.ctc_decoder_predictions_tensor(predictions, predictions_lengths) + else: + raise NotImplementedError("Implement me if you need non-CTC decode on predictions") + + if self.log_prediction: + logging.info(f"\n") + logging.info(f"reference:{references[0]}") + logging.info(f"predicted:{hypotheses[0]}") + + for h, r in zip(hypotheses, references): + if self.use_cer: + h_list = list(h) + r_list = list(r) + else: + h_list = h.split() + r_list = r.split() + words += len(r_list) + # Compute Levenstein's distance + scores += editdistance.eval(h_list, r_list) + + self.scores = torch.tensor(scores, device=self.scores.device, dtype=self.scores.dtype) + self.words = torch.tensor(words, device=self.words.device, dtype=self.words.dtype) + # return torch.tensor([scores, words]).to(predictions.device) + + def compute(self): + scores = self.scores.detach().float() + words = self.words.detach().float() + return scores / words, scores, words diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/models/__init__.py new file mode 100644 index 0000000000..05808b1e03 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.asr.models.asr_model import ASRModel +from nemo.collections.asr.models.classification_models import EncDecClassificationModel +from nemo.collections.asr.models.clustering_diarizer import ClusteringDiarizer +from nemo.collections.asr.models.ctc_bpe_models import EncDecCTCModelBPE +from nemo.collections.asr.models.ctc_models import EncDecCTCModel +from nemo.collections.asr.models.label_models import EncDecSpeakerLabelModel, ExtractSpeakerEmbeddingsModel +from nemo.collections.asr.models.rnnt_bpe_models import EncDecRNNTBPEModel +from nemo.collections.asr.models.rnnt_models import EncDecRNNTModel diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/asr_model.py b/models/tacotron2/NeMo/nemo/collections/asr/models/asr_model.py new file mode 100644 index 0000000000..bdabfc29d9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/asr_model.py @@ -0,0 +1,91 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from abc import ABC, abstractmethod +from typing import List, Optional, Union + +import torch +from omegaconf import OmegaConf + +from nemo.core.classes import ModelPT +from nemo.core.classes.exportable import Exportable +from nemo.utils import model_utils + +__all__ = ['ASRModel'] + + +class ASRModel(ModelPT, ABC): + @abstractmethod + def transcribe(self, paths2audio_files: List[str], batch_size: int = 4) -> List[str]: + """ + Takes paths to audio files and returns text transcription + Args: + paths2audio_files: paths to audio fragment to be transcribed + + Returns: + transcription texts + """ + pass + + def multi_validation_epoch_end(self, outputs, dataloader_idx: int = 0): + val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean() + wer_num = torch.stack([x['val_wer_num'] for x in outputs]).sum() + wer_denom = torch.stack([x['val_wer_denom'] for x in outputs]).sum() + tensorboard_logs = {'val_loss': val_loss_mean, 'val_wer': wer_num / wer_denom} + return {'val_loss': val_loss_mean, 'log': tensorboard_logs} + + def multi_test_epoch_end(self, outputs, dataloader_idx: int = 0): + val_loss_mean = torch.stack([x['test_loss'] for x in outputs]).mean() + wer_num = torch.stack([x['test_wer_num'] for x in outputs]).sum() + wer_denom = torch.stack([x['test_wer_denom'] for x in outputs]).sum() + tensorboard_logs = {'test_loss': val_loss_mean, 'test_wer': wer_num / wer_denom} + return {'test_loss': val_loss_mean, 'log': tensorboard_logs} + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + # recursively walk the subclasses to generate pretrained model info + list_of_models = model_utils.resolve_subclass_pretrained_model_info(cls) + return list_of_models + + +class ExportableEncDecModel(Exportable): + """ + Simple utiliy mix-in to export models that consist of encoder/decoder pair + plus pre/post processor, but have to be exported as encoder/decoder pair only + (covers most ASR classes) + """ + + @property + def input_module(self): + return self.encoder + + @property + def output_module(self): + return self.decoder + + def forward_for_export(self, input, length=None): + encoder_output = self.input_module(input, length) + if isinstance(encoder_output, tuple): + return self.output_module(encoder_output[0]) + else: + return self.output_module(encoder_output) + + def _prepare_for_export(self, **kwargs): + self.input_module._prepare_for_export(**kwargs) + self.output_module._prepare_for_export(**kwargs) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/classification_models.py b/models/tacotron2/NeMo/nemo/collections/asr/models/classification_models.py new file mode 100644 index 0000000000..894454c2dd --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/classification_models.py @@ -0,0 +1,750 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import json +import os +import tempfile +from abc import abstractmethod +from math import ceil +from typing import Dict, List, Optional, Union + +import torch +from omegaconf import DictConfig, ListConfig, OmegaConf +from pytorch_lightning import Trainer +from torchmetrics.regression import MeanAbsoluteError, MeanSquaredError + +from nemo.collections.asr.data import audio_to_label_dataset +from nemo.collections.asr.models.asr_model import ASRModel, ExportableEncDecModel +from nemo.collections.asr.parts.preprocessing.features import WaveformFeaturizer +from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations +from nemo.collections.common.losses import CrossEntropyLoss, MSELoss +from nemo.collections.common.metrics import TopKClassificationAccuracy +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types import * +from nemo.utils import logging, model_utils + +__all__ = ['EncDecClassificationModel', 'EncDecRegressionModel'] + + +class _EncDecBaseModel(ASRModel, ExportableEncDecModel): + """Encoder decoder Classification models.""" + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + # Get global rank and total number of GPU workers for IterableDataset partitioning, if applicable + # Global_rank and local_rank is set by LightningModule in Lightning 1.2.0 + self.world_size = 1 + if trainer is not None: + self.world_size = trainer.num_nodes * trainer.num_gpus + + # Convert config to a DictConfig + cfg = model_utils.convert_model_config_to_dict_config(cfg) + + # Convert config to support Hydra 1.0+ instantiation + cfg = model_utils.maybe_update_config_version(cfg) + + self.is_regression_task = cfg.get('is_regression_task', False) + # Change labels if needed + self._update_decoder_config(cfg.labels, cfg.decoder) + super().__init__(cfg=cfg, trainer=trainer) + + if hasattr(self._cfg, 'spec_augment') and self._cfg.spec_augment is not None: + self.spec_augmentation = ASRModel.from_config_dict(self._cfg.spec_augment) + else: + self.spec_augmentation = None + if hasattr(self._cfg, 'crop_or_pad_augment') and self._cfg.crop_or_pad_augment is not None: + self.crop_or_pad = ASRModel.from_config_dict(self._cfg.crop_or_pad_augment) + else: + self.crop_or_pad = None + + self.preprocessor = self._setup_preprocessor() + self.encoder = self._setup_encoder() + self.decoder = self._setup_decoder() + self.loss = self._setup_loss() + self._setup_metrics() + + @abstractmethod + def _setup_preprocessor(self): + """ + Setup preprocessor for audio data + Returns: Preprocessor + + """ + pass + + @abstractmethod + def _setup_encoder(self): + """ + Setup encoder for the Encoder-Decoder network + Returns: Encoder + """ + pass + + @abstractmethod + def _setup_decoder(self): + """ + Setup decoder for the Encoder-Decoder network + Returns: Decoder + """ + pass + + @abstractmethod + def _setup_loss(self): + """ + Setup loss function for training + Returns: Loss function + + """ + pass + + @abstractmethod + def _setup_metrics(self): + """ + Setup metrics to be tracked in addition to loss + Returns: void + + """ + pass + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + if hasattr(self.preprocessor, '_sample_rate'): + audio_eltype = AudioSignal(freq=self.preprocessor._sample_rate) + else: + audio_eltype = AudioSignal() + return { + "input_signal": NeuralType(('B', 'T'), audio_eltype), + "input_signal_length": NeuralType(tuple('B'), LengthsType()), + } + + @property + @abstractmethod + def output_types(self) -> Optional[Dict[str, NeuralType]]: + pass + + def forward(self, input_signal, input_signal_length): + processed_signal, processed_signal_len = self.preprocessor( + input_signal=input_signal, length=input_signal_length, + ) + # Crop or pad is always applied + if self.crop_or_pad is not None: + processed_signal, processed_signal_len = self.crop_or_pad( + input_signal=processed_signal, length=processed_signal_len + ) + # Spec augment is not applied during evaluation/testing + if self.spec_augmentation is not None and self.training: + processed_signal = self.spec_augmentation(input_spec=processed_signal, length=processed_signal_len) + encoded, encoded_len = self.encoder(audio_signal=processed_signal, length=processed_signal_len) + logits = self.decoder(encoder_output=encoded) + return logits + + def setup_training_data(self, train_data_config: Optional[Union[DictConfig, Dict]]): + if 'shuffle' not in train_data_config: + train_data_config['shuffle'] = True + # preserve config + self._update_dataset_config(dataset_name='train', config=train_data_config) + + self._train_dl = self._setup_dataloader_from_config(config=DictConfig(train_data_config)) + + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'is_tarred' in train_data_config and train_data_config['is_tarred']: + # We also need to check if limit_train_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # training batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if isinstance(self._trainer.limit_train_batches, float): + self._trainer.limit_train_batches = int( + self._trainer.limit_train_batches + * ceil((len(self._train_dl.dataset) / self.world_size) / train_data_config['batch_size']) + ) + + def setup_validation_data(self, val_data_config: Optional[Union[DictConfig, Dict]]): + if 'shuffle' not in val_data_config: + val_data_config['shuffle'] = False + + # preserve config + self._update_dataset_config(dataset_name='validation', config=val_data_config) + + self._validation_dl = self._setup_dataloader_from_config(config=DictConfig(val_data_config)) + + def setup_test_data(self, test_data_config: Optional[Union[DictConfig, Dict]]): + if 'shuffle' not in test_data_config: + test_data_config['shuffle'] = False + + # preserve config + self._update_dataset_config(dataset_name='test', config=test_data_config) + + self._test_dl = self._setup_dataloader_from_config(config=DictConfig(test_data_config)) + + def test_dataloader(self): + if self._test_dl is not None: + return self._test_dl + + def _setup_dataloader_from_config(self, config: DictConfig): + + OmegaConf.set_struct(config, False) + config.is_regression_task = self.is_regression_task + OmegaConf.set_struct(config, True) + + if 'augmentor' in config: + augmentor = process_augmentations(config['augmentor']) + else: + augmentor = None + + featurizer = WaveformFeaturizer( + sample_rate=config['sample_rate'], int_values=config.get('int_values', False), augmentor=augmentor + ) + shuffle = config['shuffle'] + + # Instantiate tarred dataset loader or normal dataset loader + if config.get('is_tarred', False): + if ('tarred_audio_filepaths' in config and config['tarred_audio_filepaths'] is None) or ( + 'manifest_filepath' in config and config['manifest_filepath'] is None + ): + logging.warning( + "Could not load dataset as `manifest_filepath` is None or " + f"`tarred_audio_filepaths` is None. Provided config : {config}" + ) + return None + + if 'vad_stream' in config and config['vad_stream']: + logging.warning("VAD inference does not support tarred dataset now") + return None + + shuffle_n = config.get('shuffle_n', 4 * config['batch_size']) if shuffle else 0 + dataset = audio_to_label_dataset.get_tarred_classification_label_dataset( + featurizer=featurizer, + config=OmegaConf.to_container(config), + shuffle_n=shuffle_n, + global_rank=self.global_rank, + world_size=self.world_size, + ) + shuffle = False + batch_size = config['batch_size'] + collate_func = dataset.collate_fn + + else: + if 'manifest_filepath' in config and config['manifest_filepath'] is None: + logging.warning(f"Could not load dataset as `manifest_filepath` is None. Provided config : {config}") + return None + + if 'vad_stream' in config and config['vad_stream']: + logging.info("Perform streaming frame-level VAD") + dataset = audio_to_label_dataset.get_speech_label_dataset( + featurizer=featurizer, config=OmegaConf.to_container(config) + ) + batch_size = 1 + collate_func = dataset.vad_frame_seq_collate_fn + else: + dataset = audio_to_label_dataset.get_classification_label_dataset( + featurizer=featurizer, config=OmegaConf.to_container(config) + ) + batch_size = config['batch_size'] + collate_func = dataset.collate_fn + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=batch_size, + collate_fn=collate_func, + drop_last=config.get('drop_last', False), + shuffle=shuffle, + num_workers=config.get('num_workers', 0), + pin_memory=config.get('pin_memory', False), + ) + + @torch.no_grad() + def transcribe(self, paths2audio_files: List[str], batch_size: int = 4, logprobs=False) -> List[str]: + """ + Generate class labels for provided audio files. Use this method for debugging and prototyping. + + Args: + paths2audio_files: (a list) of paths to audio files. \ + Recommended length per file is approximately 1 second. + batch_size: (int) batch size to use during inference. \ + Bigger will result in better throughput performance but would use more memory. + logprobs: (bool) pass True to get log probabilities instead of class labels. + + Returns: + + A list of transcriptions (or raw log probabilities if logprobs is True) in the same order as paths2audio_files + """ + if paths2audio_files is None or len(paths2audio_files) == 0: + return [] + # We will store transcriptions here + labels = [] + # Model's mode and device + mode = self.training + device = next(self.parameters()).device + dither_value = self.preprocessor.featurizer.dither + pad_to_value = self.preprocessor.featurizer.pad_to + + try: + self.preprocessor.featurizer.dither = 0.0 + self.preprocessor.featurizer.pad_to = 0 + # Switch model to evaluation mode + self.eval() + logging_level = logging.get_verbosity() + logging.set_verbosity(logging.WARNING) + # Work in tmp directory - will store manifest file there + with tempfile.TemporaryDirectory() as tmpdir: + with open(os.path.join(tmpdir, 'manifest.json'), 'w') as fp: + for audio_file in paths2audio_files: + label = 0.0 if self.is_regression_task else self.cfg.labels[0] + entry = {'audio_filepath': audio_file, 'duration': 100000.0, 'label': label} + fp.write(json.dumps(entry) + '\n') + + config = {'paths2audio_files': paths2audio_files, 'batch_size': batch_size, 'temp_dir': tmpdir} + + temporary_datalayer = self._setup_transcribe_dataloader(config) + for test_batch in temporary_datalayer: + logits = self.forward( + input_signal=test_batch[0].to(device), input_signal_length=test_batch[1].to(device) + ) + if logprobs: + # dump log probs per file + for idx in range(logits.shape[0]): + lg = logits[idx] + labels.append(lg.cpu().numpy()) + else: + labels_k = [] + top_ks = self._accuracy.top_k + for top_k_i in top_ks: + # replace top k value with current top k + self._accuracy.top_k = top_k_i + labels_k_i = self._accuracy.top_k_predicted_labels(logits) + labels_k.append(labels_k_i) + + # convenience: if only one top_k, pop out the nested list + if len(top_ks) == 1: + labels_k = labels_k[0] + + labels += labels_k + # reset top k to orignal value + self._accuracy.top_k = top_ks + del test_batch + finally: + # set mode back to its original value + self.train(mode=mode) + self.preprocessor.featurizer.dither = dither_value + self.preprocessor.featurizer.pad_to = pad_to_value + logging.set_verbosity(logging_level) + return labels + + def _setup_transcribe_dataloader(self, config: Dict) -> 'torch.utils.data.DataLoader': + """ + Setup function for a temporary data loader which wraps the provided audio file. + + Args: + config: A python dictionary which contains the following keys: + + Returns: + A pytorch DataLoader for the given audio file(s). + """ + dl_config = { + 'manifest_filepath': os.path.join(config['temp_dir'], 'manifest.json'), + 'sample_rate': self.preprocessor._sample_rate, + 'labels': self.cfg.labels, + 'batch_size': min(config['batch_size'], len(config['paths2audio_files'])), + 'trim_silence': False, + 'shuffle': False, + } + + temporary_datalayer = self._setup_dataloader_from_config(config=DictConfig(dl_config)) + return temporary_datalayer + + @abstractmethod + def _update_decoder_config(self, labels, cfg): + pass + + +class EncDecClassificationModel(_EncDecBaseModel): + """Encoder decoder Classification models.""" + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + + if cfg.get("is_regression_task", False): + raise ValueError(f"EndDecClassificationModel requires the flag is_regression_task to be set as false") + + super().__init__(cfg=cfg, trainer=trainer) + + def _setup_preprocessor(self): + return EncDecClassificationModel.from_config_dict(self._cfg.preprocessor) + + def _setup_encoder(self): + return EncDecClassificationModel.from_config_dict(self._cfg.encoder) + + def _setup_decoder(self): + return EncDecClassificationModel.from_config_dict(self._cfg.decoder) + + def _setup_loss(self): + return CrossEntropyLoss() + + def _setup_metrics(self): + self._accuracy = TopKClassificationAccuracy(dist_sync_on_step=True) + + @classmethod + def list_available_models(cls) -> Optional[List[PretrainedModelInfo]]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + results = [] + + model = PretrainedModelInfo( + pretrained_model_name="vad_telephony_marblenet", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:vad_telephony_marblenet", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/vad_telephony_marblenet/versions/1.0.0rc1/files/vad_telephony_marblenet.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="vad_marblenet", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:vad_marblenet", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/vad_marblenet/versions/1.0.0rc1/files/vad_marblenet.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="commandrecognition_en_matchboxnet3x1x64_v1", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x1x64_v1", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/commandrecognition_en_matchboxnet3x1x64_v1/versions/1.0.0rc1/files/commandrecognition_en_matchboxnet3x1x64_v1.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="commandrecognition_en_matchboxnet3x2x64_v1", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x2x64_v1", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/commandrecognition_en_matchboxnet3x2x64_v1/versions/1.0.0rc1/files/commandrecognition_en_matchboxnet3x2x64_v1.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="commandrecognition_en_matchboxnet3x1x64_v2", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x1x64_v2", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/commandrecognition_en_matchboxnet3x1x64_v2/versions/1.0.0rc1/files/commandrecognition_en_matchboxnet3x1x64_v2.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="commandrecognition_en_matchboxnet3x2x64_v2", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x2x64_v2", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/commandrecognition_en_matchboxnet3x2x64_v2/versions/1.0.0rc1/files/commandrecognition_en_matchboxnet3x2x64_v2.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="commandrecognition_en_matchboxnet3x1x64_v2_subset_task", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x1x64_v2_subset_task", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/commandrecognition_en_matchboxnet3x1x64_v2_subset_task/versions/1.0.0rc1/files/commandrecognition_en_matchboxnet3x1x64_v2_subset_task.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="commandrecognition_en_matchboxnet3x2x64_v2_subset_task", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x2x64_v2_subset_task", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/commandrecognition_en_matchboxnet3x2x64_v2_subset_task/versions/1.0.0rc1/files/commandrecognition_en_matchboxnet3x2x64_v2_subset_task.nemo", + ) + results.append(model) + return results + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return {"outputs": NeuralType(('B', 'D'), LogitsType())} + + # PTL-specific methods + def training_step(self, batch, batch_nb): + self.training_step_end() + audio_signal, audio_signal_len, labels, labels_len = batch + logits = self.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + loss_value = self.loss(logits=logits, labels=labels) + + self.log('train_loss', loss_value) + self.log('learning_rate', self._optimizer.param_groups[0]['lr']) + + self._accuracy(logits=logits, labels=labels) + topk_scores = self._accuracy.compute() + self._accuracy.reset() + + for top_k, score in zip(self._accuracy.top_k, topk_scores): + self.log('training_batch_accuracy_top@{}'.format(top_k), score) + + return { + 'loss': loss_value, + } + + def validation_step(self, batch, batch_idx, dataloader_idx=0): + audio_signal, audio_signal_len, labels, labels_len = batch + logits = self.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + loss_value = self.loss(logits=logits, labels=labels) + acc = self._accuracy(logits=logits, labels=labels) + correct_counts, total_counts = self._accuracy.correct_counts_k, self._accuracy.total_counts_k + return { + 'val_loss': loss_value, + 'val_correct_counts': correct_counts, + 'val_total_counts': total_counts, + 'val_acc': acc, + } + + def test_step(self, batch, batch_idx, dataloader_idx=0): + audio_signal, audio_signal_len, labels, labels_len = batch + logits = self.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + loss_value = self.loss(logits=logits, labels=labels) + acc = self._accuracy(logits=logits, labels=labels) + correct_counts, total_counts = self._accuracy.correct_counts_k, self._accuracy.total_counts_k + return { + 'test_loss': loss_value, + 'test_correct_counts': correct_counts, + 'test_total_counts': total_counts, + 'test_acc': acc, + } + + def multi_validation_epoch_end(self, outputs, dataloader_idx: int = 0): + val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean() + correct_counts = torch.stack([x['val_correct_counts'] for x in outputs]).sum(axis=0) + total_counts = torch.stack([x['val_total_counts'] for x in outputs]).sum(axis=0) + + self._accuracy.correct_counts_k = correct_counts + self._accuracy.total_counts_k = total_counts + topk_scores = self._accuracy.compute() + self._accuracy.reset() + + tensorboard_log = {'val_loss': val_loss_mean} + for top_k, score in zip(self._accuracy.top_k, topk_scores): + tensorboard_log['val_epoch_top@{}'.format(top_k)] = score + + return {'log': tensorboard_log} + + def multi_test_epoch_end(self, outputs, dataloader_idx: int = 0): + test_loss_mean = torch.stack([x['test_loss'] for x in outputs]).mean() + correct_counts = torch.stack([x['test_correct_counts'].unsqueeze(0) for x in outputs]).sum(axis=0) + total_counts = torch.stack([x['test_total_counts'].unsqueeze(0) for x in outputs]).sum(axis=0) + + self._accuracy.correct_counts_k = correct_counts + self._accuracy.total_counts_k = total_counts + topk_scores = self._accuracy.compute() + self._accuracy.reset() + + tensorboard_log = {'test_loss': test_loss_mean} + for top_k, score in zip(self._accuracy.top_k, topk_scores): + tensorboard_log['test_epoch_top@{}'.format(top_k)] = score + + return {'log': tensorboard_log} + + @typecheck() + def forward(self, input_signal, input_signal_length): + logits = super().forward(input_signal=input_signal, input_signal_length=input_signal_length) + return logits + + def change_labels(self, new_labels: List[str]): + """ + Changes labels used by the decoder model. Use this method when fine-tuning on from pre-trained model. + This method changes only decoder and leaves encoder and pre-processing modules unchanged. For example, you would + use it if you want to use pretrained encoder when fine-tuning on a data in another dataset. + + If new_labels == self.decoder.vocabulary then nothing will be changed. + + Args: + + new_labels: list with new labels. Must contain at least 2 elements. Typically, \ + this is set of labels for the dataset. + + Returns: None + + """ + if new_labels is not None and not isinstance(new_labels, ListConfig): + new_labels = ListConfig(new_labels) + + if self._cfg.labels == new_labels: + logging.warning( + f"Old labels ({self._cfg.labels}) and new labels ({new_labels}) match. Not changing anything" + ) + else: + if new_labels is None or len(new_labels) == 0: + raise ValueError(f'New labels must be non-empty list of labels. But I got: {new_labels}') + + # Update config + self._cfg.labels = new_labels + + decoder_config = self.decoder.to_config_dict() + new_decoder_config = copy.deepcopy(decoder_config) + self._update_decoder_config(new_labels, new_decoder_config) + del self.decoder + self.decoder = EncDecClassificationModel.from_config_dict(new_decoder_config) + + OmegaConf.set_struct(self._cfg.decoder, False) + self._cfg.decoder = new_decoder_config + OmegaConf.set_struct(self._cfg.decoder, True) + + if 'train_ds' in self._cfg and self._cfg.train_ds is not None: + self._cfg.train_ds.labels = new_labels + + if 'validation_ds' in self._cfg and self._cfg.validation_ds is not None: + self._cfg.validation_ds.labels = new_labels + + if 'test_ds' in self._cfg and self._cfg.test_ds is not None: + self._cfg.test_ds.labels = new_labels + + logging.info(f"Changed decoder output to {self.decoder.num_classes} labels.") + + def _update_decoder_config(self, labels, cfg): + """ + Update the number of classes in the decoder based on labels provided. + + Args: + labels: The current labels of the model + cfg: The config of the decoder which will be updated. + """ + OmegaConf.set_struct(cfg, False) + + if 'params' in cfg: + cfg.params.num_classes = len(labels) + else: + cfg.num_classes = len(labels) + + OmegaConf.set_struct(cfg, True) + + +class EncDecRegressionModel(_EncDecBaseModel): + """Encoder decoder class for speech regression models. + Model class creates training, validation methods for setting up data + performing model forward pass. + """ + + @classmethod + def list_available_models(cls) -> List[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + result = [] + + return result + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + if not cfg.get('is_regression_task', False): + raise ValueError(f"EndDecRegressionModel requires the flag is_regression_task to be set as true") + super().__init__(cfg=cfg, trainer=trainer) + + def _setup_preprocessor(self): + return EncDecRegressionModel.from_config_dict(self._cfg.preprocessor) + + def _setup_encoder(self): + return EncDecRegressionModel.from_config_dict(self._cfg.encoder) + + def _setup_decoder(self): + return EncDecRegressionModel.from_config_dict(self._cfg.decoder) + + def _setup_loss(self): + return MSELoss() + + def _setup_metrics(self): + self._mse = MeanSquaredError() + self._mae = MeanAbsoluteError() + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return {"preds": NeuralType(tuple('B'), RegressionValuesType())} + + @typecheck() + def forward(self, input_signal, input_signal_length): + logits = super().forward(input_signal=input_signal, input_signal_length=input_signal_length) + return logits.view(-1) + + # PTL-specific methods + def training_step(self, batch, batch_idx): + self.training_step_end() + audio_signal, audio_signal_len, targets, targets_len = batch + logits = self.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + loss = self.loss(preds=logits, labels=targets) + train_mse = self._mse(preds=logits, target=targets) + train_mae = self._mae(preds=logits, target=targets) + + tensorboard_logs = { + 'train_loss': loss, + 'train_mse': train_mse, + 'train_mae': train_mae, + 'learning_rate': self._optimizer.param_groups[0]['lr'], + } + + self.log_dict(tensorboard_logs) + return {'loss': loss} + + def validation_step(self, batch, batch_idx, dataloader_idx: int = 0): + audio_signal, audio_signal_len, targets, targets_len = batch + logits = self.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + loss_value = self.loss(preds=logits, labels=targets) + val_mse = self._mse(preds=logits, target=targets) + val_mae = self._mae(preds=logits, target=targets) + + return {'val_loss': loss_value, 'val_mse': val_mse, 'val_mae': val_mae} + + def test_step(self, batch, batch_idx, dataloader_idx: int = 0): + logs = self.validation_step(batch, batch_idx, dataloader_idx) + + return {'test_loss': logs['val_loss'], 'test_mse': logs['test_mse'], 'test_mae': logs['val_mae']} + + def multi_validation_epoch_end(self, outputs, dataloader_idx: int = 0): + val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean() + val_mse = self._mse.compute() + self._mse.reset() + val_mae = self._mae.compute() + self._mae.reset() + + tensorboard_logs = {'val_loss': val_loss_mean, 'val_mse': val_mse, 'val_mae': val_mae} + + return {'val_loss': val_loss_mean, 'val_mse': val_mse, 'val_mae': val_mae, 'log': tensorboard_logs} + + def multi_test_epoch_end(self, outputs, dataloader_idx: int = 0): + test_loss_mean = torch.stack([x['test_loss'] for x in outputs]).mean() + test_mse = self._mse.compute() + self._mse.reset() + test_mae = self._mae.compute() + self._mae.reset() + + tensorboard_logs = {'test_loss': test_loss_mean, 'test_mse': test_mse, 'test_mae': test_mae} + + return {'test_loss': test_loss_mean, 'test_mse': test_mse, 'test_mae': test_mae, 'log': tensorboard_logs} + + @torch.no_grad() + def transcribe(self, paths2audio_files: List[str], batch_size: int = 4) -> List[float]: + """ + Generate class labels for provided audio files. Use this method for debugging and prototyping. + + Args: + paths2audio_files: (a list) of paths to audio files. \ + Recommended length per file is approximately 1 second. + batch_size: (int) batch size to use during inference. \ + Bigger will result in better throughput performance but would use more memory. + + Returns: + + A list of predictions in the same order as paths2audio_files + """ + predictions = super().transcribe(paths2audio_files, batch_size, logprobs=True) + return [float(pred) for pred in predictions] + + def _update_decoder_config(self, labels, cfg): + + OmegaConf.set_struct(cfg, False) + + if 'params' in cfg: + cfg.params.num_classes = 1 + else: + cfg.num_classes = 1 + + OmegaConf.set_struct(cfg, True) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/clustering_diarizer.py b/models/tacotron2/NeMo/nemo/collections/asr/models/clustering_diarizer.py new file mode 100644 index 0000000000..3eb59debfc --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/clustering_diarizer.py @@ -0,0 +1,439 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import pickle as pkl +import shutil +import tarfile +import tempfile +from collections import defaultdict +from typing import List, Optional + +import torch +from omegaconf import DictConfig, OmegaConf, open_dict +from omegaconf.listconfig import ListConfig +from pytorch_lightning.utilities import rank_zero_only +from tqdm import tqdm + +from nemo.collections.asr.models.classification_models import EncDecClassificationModel +from nemo.collections.asr.models.label_models import ExtractSpeakerEmbeddingsModel +from nemo.collections.asr.parts.mixins.mixins import DiarizationMixin +from nemo.collections.asr.parts.utils.speaker_utils import audio_rttm_map, perform_diarization, write_rttm2manifest +from nemo.collections.asr.parts.utils.vad_utils import ( + generate_overlap_vad_seq, + generate_vad_segment_table, + get_vad_stream_status, + prepare_manifest, +) +from nemo.core.classes import Model +from nemo.utils import logging, model_utils +from nemo.utils.exp_manager import NotFoundError + +try: + from torch.cuda.amp import autocast +except ImportError: + from contextlib import contextmanager + + @contextmanager + def autocast(enabled=None): + yield + + +__all__ = ['ClusteringDiarizer'] + +_MODEL_CONFIG_YAML = "model_config.yaml" +_VAD_MODEL = "vad_model.nemo" +_SPEAKER_MODEL = "speaker_model.nemo" + + +def get_available_model_names(class_name): + available_models = class_name.list_available_models() + return list(map(lambda x: x.pretrained_model_name, available_models)) + + +class ClusteringDiarizer(Model, DiarizationMixin): + def __init__(self, cfg: DictConfig): + cfg = model_utils.convert_model_config_to_dict_config(cfg) + # Convert config to support Hydra 1.0+ instantiation + cfg = model_utils.maybe_update_config_version(cfg) + self._cfg = cfg + self._out_dir = self._cfg.diarizer.out_dir + if not os.path.exists(self._out_dir): + os.mkdir(self._out_dir) + + # init vad model + self.has_vad_model = False + self.has_vad_model_to_save = False + + self._speaker_manifest_path = self._cfg.diarizer.speaker_embeddings.oracle_vad_manifest + self.AUDIO_RTTM_MAP = None + self.paths2audio_files = self._cfg.diarizer.paths2audio_files + + if self._cfg.diarizer.vad.model_path is not None: + self._init_vad_model() + self._vad_dir = os.path.join(self._out_dir, 'vad_outputs') + self._vad_out_file = os.path.join(self._vad_dir, "vad_out.json") + shutil.rmtree(self._vad_dir, ignore_errors=True) + os.makedirs(self._vad_dir) + + # init speaker model + self._init_speaker_model() + + if self._cfg.diarizer.get('num_speakers', None): + self._num_speakers = self._cfg.diarizer.num_speakers + logging.warning("in next release num_speakers will be changed to oracle_num_speakers") + else: + self._num_speakers = self._cfg.diarizer.oracle_num_speakers + + if self._cfg.diarizer.get('max_num_speakers', None): + self.max_num_speakers = self._cfg.diarizer.max_num_speakers + else: + self.max_num_speakers = 8 + + self._device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + @classmethod + def list_available_models(cls): + pass + + def _init_speaker_model(self): + model_path = self._cfg.diarizer.speaker_embeddings.model_path + if model_path is not None and model_path.endswith('.nemo'): + self._speaker_model = ExtractSpeakerEmbeddingsModel.restore_from(model_path) + logging.info("Speaker Model restored locally from {}".format(model_path)) + else: + if model_path not in get_available_model_names(ExtractSpeakerEmbeddingsModel): + logging.warning( + "requested {} model name not available in pretrained models, instead".format(model_path) + ) + model_path = "speakerdiarization_speakernet" + logging.info("Loading pretrained {} model from NGC".format(model_path)) + self._speaker_model = ExtractSpeakerEmbeddingsModel.from_pretrained(model_name=model_path) + + self._speaker_dir = os.path.join(self._out_dir, 'speaker_outputs') + + def set_vad_model(self, vad_config): + with open_dict(self._cfg): + self._cfg.diarizer.vad = vad_config + self._init_vad_model() + + def _init_vad_model(self): + model_path = self._cfg.diarizer.vad.model_path + if model_path.endswith('.nemo'): + self._vad_model = EncDecClassificationModel.restore_from(model_path) + logging.info("VAD model loaded locally from {}".format(model_path)) + else: + if model_path not in get_available_model_names(EncDecClassificationModel): + logging.warning( + "requested {} model name not available in pretrained models, instead".format(model_path) + ) + model_path = "vad_telephony_marblenet" + logging.info("Loading pretrained {} model from NGC".format(model_path)) + self._vad_model = EncDecClassificationModel.from_pretrained(model_name=model_path) + + self._vad_window_length_in_sec = self._cfg.diarizer.vad.window_length_in_sec + self._vad_shift_length_in_sec = self._cfg.diarizer.vad.shift_length_in_sec + self.has_vad_model_to_save = True + self.has_vad_model = True + + def _setup_vad_test_data(self, manifest_vad_input): + vad_dl_config = { + 'manifest_filepath': manifest_vad_input, + 'sample_rate': self._cfg.sample_rate, + 'vad_stream': True, + 'labels': ['infer',], + 'time_length': self._vad_window_length_in_sec, + 'shift_length': self._vad_shift_length_in_sec, + 'trim_silence': False, + 'num_workers': self._cfg.num_workers, + } + self._vad_model.setup_test_data(test_data_config=vad_dl_config) + + def _setup_spkr_test_data(self, manifest_file): + spk_dl_config = { + 'manifest_filepath': manifest_file, + 'sample_rate': self._cfg.sample_rate, + 'batch_size': 1, + 'time_length': self._cfg.diarizer.speaker_embeddings.window_length_in_sec, + 'shift_length': self._cfg.diarizer.speaker_embeddings.shift_length_in_sec, + 'trim_silence': False, + 'embedding_dir': self._speaker_dir, + 'labels': None, + 'task': "diarization", + 'num_workers': self._cfg.num_workers, + } + self._speaker_model.setup_test_data(spk_dl_config) + + def _run_vad(self, manifest_file): + self._vad_model = self._vad_model.to(self._device) + self._vad_model.eval() + + time_unit = int(self._vad_window_length_in_sec / self._vad_shift_length_in_sec) + trunc = int(time_unit / 2) + trunc_l = time_unit - trunc + all_len = 0 + data = [] + for line in open(manifest_file, 'r'): + file = os.path.basename(json.loads(line)['audio_filepath']) + data.append(os.path.splitext(file)[0]) + + status = get_vad_stream_status(data) + for i, test_batch in enumerate(tqdm(self._vad_model.test_dataloader())): + test_batch = [x.to(self._device) for x in test_batch] + with autocast(): + log_probs = self._vad_model(input_signal=test_batch[0], input_signal_length=test_batch[1]) + probs = torch.softmax(log_probs, dim=-1) + pred = probs[:, 1] + if status[i] == 'start': + to_save = pred[:-trunc] + elif status[i] == 'next': + to_save = pred[trunc:-trunc_l] + elif status[i] == 'end': + to_save = pred[trunc_l:] + else: + to_save = pred + all_len += len(to_save) + outpath = os.path.join(self._vad_dir, data[i] + ".frame") + with open(outpath, "a") as fout: + for f in range(len(to_save)): + fout.write('{0:0.4f}\n'.format(to_save[f])) + del test_batch + if status[i] == 'end' or status[i] == 'single': + all_len = 0 + + if not self._cfg.diarizer.vad.vad_decision_smoothing: + # Shift the window by 10ms to generate the frame and use the prediction of the window to represent the label for the frame; + self.vad_pred_dir = self._vad_dir + + else: + # Generate predictions with overlapping input segments. Then a smoothing filter is applied to decide the label for a frame spanned by multiple segments. + # smoothing_method would be either in majority vote (median) or average (mean) + logging.info("Generating predictions with overlapping input segments") + smoothing_pred_dir = generate_overlap_vad_seq( + frame_pred_dir=self._vad_dir, + smoothing_method=self._cfg.diarizer.vad.smoothing_params.method, + overlap=self._cfg.diarizer.vad.smoothing_params.overlap, + seg_len=self._vad_window_length_in_sec, + shift_len=self._vad_shift_length_in_sec, + num_workers=self._cfg.num_workers, + ) + self.vad_pred_dir = smoothing_pred_dir + + logging.info("Converting frame level prediction to speech/no-speech segment in start and end times format.") + table_out_dir = generate_vad_segment_table( + vad_pred_dir=self.vad_pred_dir, + threshold=self._cfg.diarizer.vad.threshold, + shift_len=self._vad_shift_length_in_sec, + num_workers=self._cfg.num_workers, + ) + + vad_table_list = [os.path.join(table_out_dir, key + ".txt") for key in self.AUDIO_RTTM_MAP] + write_rttm2manifest(self._cfg.diarizer.paths2audio_files, vad_table_list, self._vad_out_file) + self._speaker_manifest_path = self._vad_out_file + + def _extract_embeddings(self, manifest_file): + logging.info("Extracting embeddings for Diarization") + self._setup_spkr_test_data(manifest_file) + uniq_names = [] + out_embeddings = defaultdict(list) + self._speaker_model = self._speaker_model.to(self._device) + self._speaker_model.eval() + with open(manifest_file, 'r') as manifest: + for line in manifest.readlines(): + line = line.strip() + dic = json.loads(line) + uniq_names.append(dic['audio_filepath'].split('/')[-1].rsplit('.', 1)[0]) + + for i, test_batch in enumerate(tqdm(self._speaker_model.test_dataloader())): + test_batch = [x.to(self._device) for x in test_batch] + audio_signal, audio_signal_len, labels, slices = test_batch + with autocast(): + _, embs = self._speaker_model.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + emb_shape = embs.shape[-1] + embs = embs.view(-1, emb_shape).type(torch.float32) + embs = embs.cpu().detach().numpy() + out_embeddings[uniq_names[i]].extend(embs) + del test_batch + + embedding_dir = os.path.join(self._speaker_dir, 'embeddings') + if not os.path.exists(embedding_dir): + os.makedirs(embedding_dir, exist_ok=True) + + prefix = manifest_file.split('/')[-1].rsplit('.', 1)[-2] + + name = os.path.join(embedding_dir, prefix) + self._embeddings_file = name + '_embeddings.pkl' + pkl.dump(out_embeddings, open(self._embeddings_file, 'wb')) + logging.info("Saved embedding files to {}".format(embedding_dir)) + + def path2audio_files_to_manifest(self, paths2audio_files): + mfst_file = os.path.join(self._out_dir, 'manifest.json') + with open(mfst_file, 'w') as fp: + for audio_file in paths2audio_files: + audio_file = audio_file.strip() + entry = {'audio_filepath': audio_file, 'offset': 0.0, 'duration': None, 'text': '-', 'label': 'infer'} + fp.write(json.dumps(entry) + '\n') + + return mfst_file + + def diarize(self, paths2audio_files: List[str] = None, batch_size: int = 1): + """ + """ + + if paths2audio_files: + self.paths2audio_files = paths2audio_files + else: + if self._cfg.diarizer.paths2audio_files is None: + raise ValueError("Pass path2audio files either through config or to diarize method") + else: + self.paths2audio_files = self._cfg.diarizer.paths2audio_files + + if type(self.paths2audio_files) is str and os.path.isfile(self.paths2audio_files): + paths2audio_files = [] + with open(self.paths2audio_files, 'r') as path2file: + for audiofile in path2file.readlines(): + audiofile = audiofile.strip() + paths2audio_files.append(audiofile) + + elif type(self.paths2audio_files) in [list, ListConfig]: + paths2audio_files = list(self.paths2audio_files) + + else: + raise ValueError("paths2audio_files must be of type list or path to file containing audio files") + + self.AUDIO_RTTM_MAP = audio_rttm_map(paths2audio_files, self._cfg.diarizer.path2groundtruth_rttm_files) + + if self.has_vad_model: + logging.info("Performing VAD") + mfst_file = self.path2audio_files_to_manifest(paths2audio_files) + self._dont_auto_split = False + self._split_duration = 50 + manifest_vad_input = mfst_file + + if not self._dont_auto_split: + logging.info("Split long audio file to avoid CUDA memory issue") + logging.debug("Try smaller split_duration if you still have CUDA memory issue") + config = { + 'manifest_filepath': mfst_file, + 'time_length': self._vad_window_length_in_sec, + 'split_duration': self._split_duration, + 'num_workers': self._cfg.num_workers, + } + manifest_vad_input = prepare_manifest(config) + else: + logging.warning( + "If you encounter CUDA memory issue, try splitting manifest entry by split_duration to avoid it." + ) + + self._setup_vad_test_data(manifest_vad_input) + self._run_vad(manifest_vad_input) + else: + if not os.path.exists(self._speaker_manifest_path): + raise NotFoundError("Oracle VAD based manifest file not found") + + self._extract_embeddings(self._speaker_manifest_path) + out_rttm_dir = os.path.join(self._out_dir, 'pred_rttms') + os.makedirs(out_rttm_dir, exist_ok=True) + + perform_diarization( + embeddings_file=self._embeddings_file, + reco2num=self._num_speakers, + manifest_path=self._speaker_manifest_path, + sample_rate=self._cfg.sample_rate, + window=self._cfg.diarizer.speaker_embeddings.window_length_in_sec, + shift=self._cfg.diarizer.speaker_embeddings.shift_length_in_sec, + audio_rttm_map=self.AUDIO_RTTM_MAP, + out_rttm_dir=out_rttm_dir, + max_num_speakers=self.max_num_speakers, + ) + + @staticmethod + def __make_nemo_file_from_folder(filename, source_dir): + with tarfile.open(filename, "w:gz") as tar: + tar.add(source_dir, arcname="./") + + @rank_zero_only + def save_to(self, save_path: str): + """ + Saves model instance (weights and configuration) into EFF archive or . + You can use "restore_from" method to fully restore instance from .nemo file. + + .nemo file is an archive (tar.gz) with the following: + model_config.yaml - model configuration in .yaml format. You can deserialize this into cfg argument for model's constructor + model_wights.chpt - model checkpoint + + Args: + save_path: Path to .nemo file where model instance should be saved + """ + + with tempfile.TemporaryDirectory() as tmpdir: + config_yaml = os.path.join(tmpdir, _MODEL_CONFIG_YAML) + spkr_model = os.path.join(tmpdir, _SPEAKER_MODEL) + + self.to_config_file(path2yaml_file=config_yaml) + if self.has_vad_model: + vad_model = os.path.join(tmpdir, _VAD_MODEL) + self._vad_model.save_to(vad_model) + self._speaker_model.save_to(spkr_model) + self.__make_nemo_file_from_folder(filename=save_path, source_dir=tmpdir) + + @staticmethod + def __unpack_nemo_file(path2file: str, out_folder: str) -> str: + if not os.path.exists(path2file): + raise FileNotFoundError(f"{path2file} does not exist") + tar = tarfile.open(path2file, "r:gz") + tar.extractall(path=out_folder) + tar.close() + return out_folder + + @classmethod + def restore_from( + cls, + restore_path: str, + override_config_path: Optional[str] = None, + map_location: Optional[torch.device] = None, + strict: bool = False, + ): + # Get path where the command is executed - the artifacts will be "retrieved" there + # (original .nemo behavior) + cwd = os.getcwd() + + with tempfile.TemporaryDirectory() as tmpdir: + try: + cls.__unpack_nemo_file(path2file=restore_path, out_folder=tmpdir) + os.chdir(tmpdir) + if override_config_path is None: + config_yaml = os.path.join(tmpdir, _MODEL_CONFIG_YAML) + else: + config_yaml = override_config_path + conf = OmegaConf.load(config_yaml) + if os.path.exists(os.path.join(tmpdir, _VAD_MODEL)): + conf.diarizer.vad.model_path = os.path.join(tmpdir, _VAD_MODEL) + else: + logging.info( + f'Model {cls.__name__} does not contain a VAD model. A VAD model or manifest file with' + f'speech segments need for diarization with this model' + ) + + conf.diarizer.speaker_embeddings.model_path = os.path.join(tmpdir, _SPEAKER_MODEL) + conf.restore_map_location = map_location + OmegaConf.set_struct(conf, True) + instance = cls(cfg=conf) + + logging.info(f'Model {cls.__name__} was successfully restored from {restore_path}.') + finally: + os.chdir(cwd) + + return instance diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/configs/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/models/configs/__init__.py new file mode 100644 index 0000000000..1dba1be349 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/configs/__init__.py @@ -0,0 +1,46 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.asr.models.configs.classification_models_config import ( + EncDecClassificationConfig, + EncDecClassificationDatasetConfig, + EncDecClassificationModelConfig, +) +from nemo.collections.asr.models.configs.ctc_models_config import ( + EncDecCTCConfig, + EncDecCTCDatasetConfig, + EncDecCTCModelConfig, +) +from nemo.collections.asr.models.configs.matchboxnet_config import ( + EncDecClassificationModelConfigBuilder, + MatchboxNetModelConfig, + MatchboxNetVADModelConfig, +) +from nemo.collections.asr.models.configs.quartznet_config import ( + EncDecCTCModelConfigBuilder, + JasperModelConfig, + QuartzNetModelConfig, +) +from nemo.collections.asr.modules.audio_preprocessing import ( + AudioToMelSpectrogramPreprocessorConfig, + AudioToMFCCPreprocessorConfig, + CropOrPadSpectrogramAugmentationConfig, + SpectrogramAugmentationConfig, +) +from nemo.collections.asr.modules.conv_asr import ( + ConvASRDecoderClassificationConfig, + ConvASRDecoderConfig, + ConvASREncoderConfig, + JasperEncoderConfig, +) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/configs/classification_models_config.py b/models/tacotron2/NeMo/nemo/collections/asr/models/configs/classification_models_config.py new file mode 100644 index 0000000000..c71e821eb6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/configs/classification_models_config.py @@ -0,0 +1,95 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from omegaconf import MISSING + +import nemo.core.classes.dataset +from nemo.collections.asr.modules.audio_preprocessing import ( + AudioToMFCCPreprocessorConfig, + CropOrPadSpectrogramAugmentationConfig, + SpectrogramAugmentationConfig, +) +from nemo.collections.asr.modules.conv_asr import ConvASRDecoderClassificationConfig, ConvASREncoderConfig +from nemo.core.config import modelPT as model_cfg + + +@dataclass +class EncDecClassificationDatasetConfig(nemo.core.classes.dataset.DatasetConfig): + manifest_filepath: Optional[str] = None + sample_rate: int = MISSING + labels: List[str] = MISSING + trim_silence: bool = False + + # Tarred dataset support + is_tarred: bool = False + tarred_audio_filepaths: Optional[str] = None + tarred_shard_strategy: str = "scatter" + shuffle_n: int = 0 + + # Optional + int_values: Optional[int] = None + augmentor: Optional[Dict[str, Any]] = None + max_duration: Optional[float] = None + min_duration: Optional[float] = None + + # VAD Optional + vad_stream: Optional[bool] = None + time_length: float = 0.31 + shift_length: float = 0.01 + normalize_audio: bool = False + is_regression_task: bool = False + + +@dataclass +class EncDecClassificationConfig(model_cfg.ModelConfig): + # Model global arguments + sample_rate: int = 16000 + repeat: int = 1 + dropout: float = 0.0 + separable: bool = True + kernel_size_factor: float = 1.0 + labels: List[str] = MISSING + timesteps: int = MISSING + + # Dataset configs + train_ds: EncDecClassificationDatasetConfig = EncDecClassificationDatasetConfig( + manifest_filepath=None, shuffle=True, trim_silence=False + ) + validation_ds: EncDecClassificationDatasetConfig = EncDecClassificationDatasetConfig( + manifest_filepath=None, shuffle=False + ) + test_ds: EncDecClassificationDatasetConfig = EncDecClassificationDatasetConfig( + manifest_filepath=None, shuffle=False + ) + + # Optimizer / Scheduler config + optim: Optional[model_cfg.OptimConfig] = model_cfg.OptimConfig(sched=model_cfg.SchedConfig()) + + # Model component configs + preprocessor: AudioToMFCCPreprocessorConfig = AudioToMFCCPreprocessorConfig() + spec_augment: Optional[SpectrogramAugmentationConfig] = SpectrogramAugmentationConfig() + crop_or_pad_augment: Optional[CropOrPadSpectrogramAugmentationConfig] = CropOrPadSpectrogramAugmentationConfig( + audio_length=timesteps + ) + + encoder: ConvASREncoderConfig = ConvASREncoderConfig() + decoder: ConvASRDecoderClassificationConfig = ConvASRDecoderClassificationConfig() + + +@dataclass +class EncDecClassificationModelConfig(model_cfg.NemoConfig): + model: EncDecClassificationConfig = EncDecClassificationConfig() diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/configs/ctc_models_config.py b/models/tacotron2/NeMo/nemo/collections/asr/models/configs/ctc_models_config.py new file mode 100644 index 0000000000..91d79a4c0e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/configs/ctc_models_config.py @@ -0,0 +1,85 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from omegaconf import MISSING + +import nemo.core.classes.dataset +from nemo.collections.asr.modules.audio_preprocessing import ( + AudioToMelSpectrogramPreprocessorConfig, + SpectrogramAugmentationConfig, +) +from nemo.collections.asr.modules.conv_asr import ConvASRDecoderConfig, ConvASREncoderConfig +from nemo.core.config import modelPT as model_cfg + + +@dataclass +class EncDecCTCDatasetConfig(nemo.core.classes.dataset.DatasetConfig): + manifest_filepath: Optional[str] = None + sample_rate: int = MISSING + labels: List[str] = MISSING + trim_silence: bool = False + + # Tarred dataset support + is_tarred: bool = False + tarred_audio_filepaths: Optional[str] = None + tarred_shard_strategy: str = "scatter" + shuffle_n: int = 0 + + # Optional + int_values: Optional[int] = None + augmentor: Optional[Dict[str, Any]] = None + max_duration: Optional[float] = None + min_duration: Optional[float] = None + max_utts: int = 0 + blank_index: int = -1 + unk_index: int = -1 + normalize: bool = False + trim: bool = True + parser: Optional[str] = 'en' + eos_id: Optional[int] = None + bos_id: Optional[int] = None + pad_id: int = 0 + use_start_end_token: bool = False + + +@dataclass +class EncDecCTCConfig(model_cfg.ModelConfig): + # Model global arguments + sample_rate: int = 16000 + repeat: int = 1 + dropout: float = 0.0 + separable: bool = False + labels: List[str] = MISSING + + # Dataset configs + train_ds: EncDecCTCDatasetConfig = EncDecCTCDatasetConfig(manifest_filepath=None, shuffle=True) + validation_ds: EncDecCTCDatasetConfig = EncDecCTCDatasetConfig(manifest_filepath=None, shuffle=False) + test_ds: EncDecCTCDatasetConfig = EncDecCTCDatasetConfig(manifest_filepath=None, shuffle=False) + + # Optimizer / Scheduler config + optim: Optional[model_cfg.OptimConfig] = model_cfg.OptimConfig(sched=model_cfg.SchedConfig()) + + # Model component configs + preprocessor: AudioToMelSpectrogramPreprocessorConfig = AudioToMelSpectrogramPreprocessorConfig() + spec_augment: Optional[SpectrogramAugmentationConfig] = SpectrogramAugmentationConfig() + encoder: ConvASREncoderConfig = ConvASREncoderConfig() + decoder: ConvASRDecoderConfig = ConvASRDecoderConfig() + + +@dataclass +class EncDecCTCModelConfig(model_cfg.NemoConfig): + model: EncDecCTCConfig = EncDecCTCConfig() diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/configs/matchboxnet_config.py b/models/tacotron2/NeMo/nemo/collections/asr/models/configs/matchboxnet_config.py new file mode 100644 index 0000000000..55a8b9fede --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/configs/matchboxnet_config.py @@ -0,0 +1,253 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Any, Callable, List, Optional + +from omegaconf import MISSING + +from nemo.collections.asr.models.configs import classification_models_config as clf_cfg +from nemo.collections.asr.modules.audio_preprocessing import ( + AudioToMFCCPreprocessorConfig, + CropOrPadSpectrogramAugmentationConfig, + SpectrogramAugmentationConfig, +) +from nemo.collections.asr.modules.conv_asr import ( + ConvASRDecoderClassificationConfig, + ConvASREncoderConfig, + JasperEncoderConfig, +) +from nemo.core.config import modelPT as model_cfg + + +# fmt: off +def matchboxnet_3x1x64(): + config = [ + JasperEncoderConfig(filters=128, repeat=1, kernel=[11], stride=[1], dilation=[1], dropout=0.0, + residual=False, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=64, repeat=1, kernel=[13], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=64, repeat=1, kernel=[15], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=64, repeat=1, kernel=[17], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=128, repeat=1, kernel=[29], stride=[1], dilation=[2], dropout=0.0, + residual=False, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=128, repeat=1, kernel=[1], stride=[1], dilation=[1], dropout=0.0, + residual=False, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False) + ] + return config + + +def matchboxnet_3x1x64_vad(): + config = [ + JasperEncoderConfig(filters=128, repeat=1, kernel=[11], stride=[1], dilation=[1], dropout=0.0, + residual=False, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=64, repeat=1, kernel=[13], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=64, repeat=1, kernel=[15], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=64, repeat=1, kernel=[17], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=128, repeat=1, kernel=[29], stride=[1], dilation=[2], dropout=0.0, + residual=False, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=128, repeat=1, kernel=[1], stride=[1], dilation=[1], dropout=0.0, + residual=False, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False) + ] + return config + + +# fmt: on + + +@dataclass +class MatchboxNetModelConfig(clf_cfg.EncDecClassificationConfig): + # Model global arguments + sample_rate: int = 16000 + repeat: int = 1 + dropout: float = 0.0 + separable: bool = True + kernel_size_factor: float = 1.0 + timesteps: int = 128 + labels: List[str] = MISSING + + # Dataset configs + train_ds: clf_cfg.EncDecClassificationDatasetConfig = clf_cfg.EncDecClassificationDatasetConfig( + manifest_filepath=None, shuffle=True, trim_silence=False + ) + validation_ds: clf_cfg.EncDecClassificationDatasetConfig = clf_cfg.EncDecClassificationDatasetConfig( + manifest_filepath=None, shuffle=False + ) + test_ds: clf_cfg.EncDecClassificationDatasetConfig = clf_cfg.EncDecClassificationDatasetConfig( + manifest_filepath=None, shuffle=False + ) + + # Optimizer / Scheduler config + optim: Optional[model_cfg.OptimConfig] = model_cfg.OptimConfig(sched=model_cfg.SchedConfig()) + + # Model general component configs + preprocessor: AudioToMFCCPreprocessorConfig = AudioToMFCCPreprocessorConfig(window_size=0.025) + spec_augment: Optional[SpectrogramAugmentationConfig] = SpectrogramAugmentationConfig( + freq_masks=2, time_masks=2, freq_width=15, time_width=25, rect_masks=5, rect_time=25, rect_freq=15 + ) + crop_or_pad_augment: Optional[CropOrPadSpectrogramAugmentationConfig] = CropOrPadSpectrogramAugmentationConfig( + audio_length=128 + ) + + encoder: ConvASREncoderConfig = ConvASREncoderConfig(activation="relu") + decoder: ConvASRDecoderClassificationConfig = ConvASRDecoderClassificationConfig() + + +@dataclass +class MatchboxNetVADModelConfig(MatchboxNetModelConfig): + timesteps: int = 64 + labels: List[str] = field(default_factory=lambda: ['background', 'speech']) + + crop_or_pad_augment: Optional[CropOrPadSpectrogramAugmentationConfig] = None + + +class EncDecClassificationModelConfigBuilder(model_cfg.ModelConfigBuilder): + VALID_CONFIGS = ['matchboxnet_3x1x64', 'matchboxnet_3x1x64_vad'] + + def __init__(self, name: str = 'matchboxnet_3x1x64', encoder_cfg_func: Optional[Callable[[], List[Any]]] = None): + if name not in EncDecClassificationModelConfigBuilder.VALID_CONFIGS: + raise ValueError("`name` must be one of : \n" f"{EncDecClassificationModelConfigBuilder.VALID_CONFIGS}") + + self.name = name + + if 'matchboxnet_3x1x64_vad' in name: + if encoder_cfg_func is None: + encoder_cfg_func = matchboxnet_3x1x64_vad + + model_cfg = MatchboxNetVADModelConfig( + repeat=1, + separable=True, + encoder=ConvASREncoderConfig(jasper=encoder_cfg_func(), activation="relu"), + decoder=ConvASRDecoderClassificationConfig(), + ) + + elif 'matchboxnet_3x1x64' in name: + if encoder_cfg_func is None: + encoder_cfg_func = matchboxnet_3x1x64 + + model_cfg = MatchboxNetModelConfig( + repeat=1, + separable=False, + spec_augment=SpectrogramAugmentationConfig(rect_masks=5, rect_freq=50, rect_time=120), + encoder=ConvASREncoderConfig(jasper=encoder_cfg_func(), activation="relu"), + decoder=ConvASRDecoderClassificationConfig(), + ) + + else: + raise ValueError(f"Invalid config name submitted to {self.__class__.__name__}") + + super(EncDecClassificationModelConfigBuilder, self).__init__(model_cfg) + self.model_cfg: clf_cfg.EncDecClassificationConfig = model_cfg # enable type hinting + + def set_labels(self, labels: List[str]): + self.model_cfg.labels = labels + + def set_separable(self, separable: bool): + self.model_cfg.separable = separable + + def set_repeat(self, repeat: int): + self.model_cfg.repeat = repeat + + def set_sample_rate(self, sample_rate: int): + self.model_cfg.sample_rate = sample_rate + + def set_dropout(self, dropout: float = 0.0): + self.model_cfg.dropout = dropout + + def set_timesteps(self, timesteps: int): + self.model_cfg.timesteps = timesteps + + def set_is_regression_task(self, is_regression_task: bool): + self.model_cfg.is_regression_task = is_regression_task + + # Note: Autocomplete for users wont work without these overrides + # But practically it is not needed since python will infer at runtime + + # def set_train_ds(self, cfg: Optional[clf_cfg.EncDecClassificationDatasetConfig] = None): + # super().set_train_ds(cfg) + # + # def set_validation_ds(self, cfg: Optional[clf_cfg.EncDecClassificationDatasetConfig] = None): + # super().set_validation_ds(cfg) + # + # def set_test_ds(self, cfg: Optional[clf_cfg.EncDecClassificationDatasetConfig] = None): + # super().set_test_ds(cfg) + + def _finalize_cfg(self): + # propagate labels + self.model_cfg.train_ds.labels = self.model_cfg.labels + self.model_cfg.validation_ds.labels = self.model_cfg.labels + self.model_cfg.test_ds.labels = self.model_cfg.labels + self.model_cfg.decoder.vocabulary = self.model_cfg.labels + + # propagate num classes + self.model_cfg.decoder.num_classes = len(self.model_cfg.labels) + + # propagate sample rate + self.model_cfg.sample_rate = self.model_cfg.sample_rate + self.model_cfg.preprocessor.sample_rate = self.model_cfg.sample_rate + self.model_cfg.train_ds.sample_rate = self.model_cfg.sample_rate + self.model_cfg.validation_ds.sample_rate = self.model_cfg.sample_rate + self.model_cfg.test_ds.sample_rate = self.model_cfg.sample_rate + + # propagate filters + self.model_cfg.encoder.feat_in = self.model_cfg.preprocessor.features + self.model_cfg.decoder.feat_in = self.model_cfg.encoder.jasper[-1].filters + + # propagate timeteps + if self.model_cfg.crop_or_pad_augment is not None: + self.model_cfg.crop_or_pad_augment.audio_length = self.model_cfg.timesteps + + # propagate separable + for layer in self.model_cfg.encoder.jasper[:-1]: # type: JasperEncoderConfig + layer.separable = self.model_cfg.separable + + # propagate repeat + for layer in self.model_cfg.encoder.jasper[1:-2]: # type: JasperEncoderConfig + layer.repeat = self.model_cfg.repeat + + # propagate dropout + for layer in self.model_cfg.encoder.jasper: # type: JasperEncoderConfig + layer.dropout = self.model_cfg.dropout + + def build(self) -> clf_cfg.EncDecClassificationConfig: + return super().build() diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/configs/quartznet_config.py b/models/tacotron2/NeMo/nemo/collections/asr/models/configs/quartznet_config.py new file mode 100644 index 0000000000..1850fce17a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/configs/quartznet_config.py @@ -0,0 +1,308 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Callable, List, Optional + +from omegaconf import MISSING + +from nemo.collections.asr.models.configs import ctc_models_config as ctc_cfg +from nemo.collections.asr.modules.audio_preprocessing import ( + AudioToMelSpectrogramPreprocessorConfig, + SpectrogramAugmentationConfig, +) +from nemo.collections.asr.modules.conv_asr import ConvASRDecoderConfig, ConvASREncoderConfig, JasperEncoderConfig +from nemo.core.config import modelPT as model_cfg + + +# fmt: off +def qn_15x5(): + config = [ + JasperEncoderConfig(filters=256, repeat=1, kernel=[33], stride=[2], dilation=[1], dropout=0.0, + residual=False, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=256, repeat=5, kernel=[33], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=256, repeat=5, kernel=[33], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=256, repeat=5, kernel=[33], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=256, repeat=5, kernel=[39], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=256, repeat=5, kernel=[39], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=256, repeat=5, kernel=[39], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=5, kernel=[51], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=5, kernel=[51], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=5, kernel=[51], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=5, kernel=[63], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=5, kernel=[63], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=5, kernel=[63], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=5, kernel=[75], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=5, kernel=[75], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=5, kernel=[75], stride=[1], dilation=[1], dropout=0.0, + residual=True, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=1, kernel=[87], stride=[1], dilation=[2], dropout=0.0, + residual=False, groups=1, separable=True, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=1024, repeat=1, kernel=[1], stride=[1], dilation=[1], dropout=0.0, + residual=False, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False) + ] + return config + + +def jasper_10x5_dr(): + config = [ + JasperEncoderConfig(filters=256, repeat=1, kernel=[11], stride=[2], dilation=[1], dropout=0.2, + residual=False, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=256, repeat=5, kernel=[11], stride=[1], dilation=[1], dropout=0.2, + residual=True, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=True, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=256, repeat=5, kernel=[11], stride=[1], dilation=[1], dropout=0.2, + residual=True, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=True, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=384, repeat=5, kernel=[13], stride=[1], dilation=[1], dropout=0.2, + residual=True, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=True, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=384, repeat=5, kernel=[13], stride=[1], dilation=[1], dropout=0.2, + residual=True, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=True, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=5, kernel=[17], stride=[1], dilation=[1], dropout=0.2, + residual=True, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=True, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=512, repeat=5, kernel=[17], stride=[1], dilation=[1], dropout=0.2, + residual=True, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=True, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=640, repeat=5, kernel=[21], stride=[1], dilation=[1], dropout=0.3, + residual=True, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=True, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=640, repeat=5, kernel=[21], stride=[1], dilation=[1], dropout=0.3, + residual=True, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=True, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=768, repeat=5, kernel=[25], stride=[1], dilation=[1], dropout=0.3, + residual=True, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=True, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=768, repeat=5, kernel=[25], stride=[1], dilation=[1], dropout=0.3, + residual=True, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=True, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=896, repeat=1, kernel=[29], stride=[1], dilation=[2], dropout=0.4, + residual=False, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False), + JasperEncoderConfig(filters=1024, repeat=1, kernel=[1], stride=[1], dilation=[1], dropout=0.4, + residual=False, groups=1, separable=False, heads=-1, residual_mode='add', + residual_dense=False, se=False, se_reduction_ratio=8, se_context_size=-1, + se_interpolation_mode='nearest', kernel_size_factor=1.0, stride_last=False) + ] + return config +# fmt: on + + +@dataclass +class JasperModelConfig(ctc_cfg.EncDecCTCConfig): + # Model global arguments + sample_rate: int = 16000 + repeat: int = 1 + dropout: float = 0.0 + separable: bool = False + labels: List[str] = MISSING + + # Dataset configs + train_ds: ctc_cfg.EncDecCTCDatasetConfig = ctc_cfg.EncDecCTCDatasetConfig( + manifest_filepath=None, shuffle=True, trim_silence=True + ) + validation_ds: ctc_cfg.EncDecCTCDatasetConfig = ctc_cfg.EncDecCTCDatasetConfig( + manifest_filepath=None, shuffle=False + ) + test_ds: ctc_cfg.EncDecCTCDatasetConfig = ctc_cfg.EncDecCTCDatasetConfig(manifest_filepath=None, shuffle=False) + + # Optimizer / Scheduler config + optim: Optional[model_cfg.OptimConfig] = model_cfg.OptimConfig(sched=model_cfg.SchedConfig()) + + # Model general component configs + preprocessor: AudioToMelSpectrogramPreprocessorConfig = AudioToMelSpectrogramPreprocessorConfig() + spec_augment: Optional[SpectrogramAugmentationConfig] = SpectrogramAugmentationConfig() + encoder: ConvASREncoderConfig = ConvASREncoderConfig(activation="relu") + decoder: ConvASRDecoderConfig = ConvASRDecoderConfig() + + +@dataclass +class QuartzNetModelConfig(JasperModelConfig): + separable: bool = True + + +class EncDecCTCModelConfigBuilder(model_cfg.ModelConfigBuilder): + VALID_CONFIGS = ['quartznet_15x5', 'quartznet_15x5_zh', 'jasper_10x5dr'] + + def __init__(self, name: str = 'quartznet_15x5', encoder_cfg_func: Optional[Callable[[], List[Any]]] = None): + if name not in EncDecCTCModelConfigBuilder.VALID_CONFIGS: + raise ValueError("`name` must be one of : \n" f"{EncDecCTCModelConfigBuilder.VALID_CONFIGS}") + + self.name = name + + if 'quartznet_15x5' in name: + if encoder_cfg_func is None: + encoder_cfg_func = qn_15x5 + + model_cfg = QuartzNetModelConfig( + repeat=5, + separable=True, + spec_augment=SpectrogramAugmentationConfig(rect_masks=5, rect_freq=50, rect_time=120), + encoder=ConvASREncoderConfig(jasper=encoder_cfg_func(), activation="relu"), + decoder=ConvASRDecoderConfig(), + ) + + elif 'jasper_10x5' in name: + if encoder_cfg_func is None: + encoder_cfg_func = jasper_10x5_dr + + model_cfg = JasperModelConfig( + repeat=5, + separable=False, + spec_augment=SpectrogramAugmentationConfig(rect_masks=5, rect_freq=50, rect_time=120), + encoder=ConvASREncoderConfig(jasper=encoder_cfg_func(), activation="relu"), + decoder=ConvASRDecoderConfig(), + ) + + else: + raise ValueError(f"Invalid config name submitted to {self.__class__.__name__}") + + super(EncDecCTCModelConfigBuilder, self).__init__(model_cfg) + self.model_cfg: ctc_cfg.EncDecCTCConfig = model_cfg # enable type hinting + + if 'zh' in name: + self.set_dataset_normalize(normalize=False) + + def set_labels(self, labels: List[str]): + self.model_cfg.labels = labels + + def set_separable(self, separable: bool): + self.model_cfg.separable = separable + + def set_repeat(self, repeat: int): + self.model_cfg.repeat = repeat + + def set_sample_rate(self, sample_rate: int): + self.model_cfg.sample_rate = sample_rate + + def set_dropout(self, dropout: float = 0.0): + self.model_cfg.dropout = dropout + + def set_dataset_normalize(self, normalize: bool): + self.model_cfg.train_ds.normalize = normalize + self.model_cfg.validation_ds.normalize = normalize + self.model_cfg.test_ds.normalize = normalize + + # Note: Autocomplete for users wont work without these overrides + # But practically it is not needed since python will infer at runtime + + # def set_train_ds(self, cfg: Optional[ctc_cfg.EncDecCTCDatasetConfig] = None): + # super().set_train_ds(cfg) + # + # def set_validation_ds(self, cfg: Optional[ctc_cfg.EncDecCTCDatasetConfig] = None): + # super().set_validation_ds(cfg) + # + # def set_test_ds(self, cfg: Optional[ctc_cfg.EncDecCTCDatasetConfig] = None): + # super().set_test_ds(cfg) + + def _finalize_cfg(self): + # propagate labels + self.model_cfg.train_ds.labels = self.model_cfg.labels + self.model_cfg.validation_ds.labels = self.model_cfg.labels + self.model_cfg.test_ds.labels = self.model_cfg.labels + self.model_cfg.decoder.vocabulary = self.model_cfg.labels + + # propagate num classes + self.model_cfg.decoder.num_classes = len(self.model_cfg.labels) + + # propagate sample rate + self.model_cfg.sample_rate = self.model_cfg.sample_rate + self.model_cfg.preprocessor.sample_rate = self.model_cfg.sample_rate + self.model_cfg.train_ds.sample_rate = self.model_cfg.sample_rate + self.model_cfg.validation_ds.sample_rate = self.model_cfg.sample_rate + self.model_cfg.test_ds.sample_rate = self.model_cfg.sample_rate + + # propagate filters + self.model_cfg.encoder.feat_in = self.model_cfg.preprocessor.features + self.model_cfg.decoder.feat_in = self.model_cfg.encoder.jasper[-1].filters + + # propagate separable + for layer in self.model_cfg.encoder.jasper[:-1]: # type: JasperEncoderConfig + layer.separable = self.model_cfg.separable + + # propagate repeat + for layer in self.model_cfg.encoder.jasper[1:-2]: # type: JasperEncoderConfig + layer.repeat = self.model_cfg.repeat + + # propagate dropout + for layer in self.model_cfg.encoder.jasper: # type: JasperEncoderConfig + layer.dropout = self.model_cfg.dropout + + def build(self) -> ctc_cfg.EncDecCTCConfig: + return super().build() diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/ctc_bpe_models.py b/models/tacotron2/NeMo/nemo/collections/asr/models/ctc_bpe_models.py new file mode 100644 index 0000000000..8a0ea82985 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/ctc_bpe_models.py @@ -0,0 +1,304 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import os +from typing import Dict, Optional + +import torch +from omegaconf import DictConfig, ListConfig, OmegaConf, open_dict + +from nemo.collections.asr.data import audio_to_text_dataset +from nemo.collections.asr.losses.ctc import CTCLoss +from nemo.collections.asr.metrics.wer_bpe import WERBPE +from nemo.collections.asr.models.ctc_models import EncDecCTCModel +from nemo.collections.asr.parts.mixins import ASRBPEMixin +from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations +from nemo.core.classes.common import PretrainedModelInfo +from nemo.utils import logging, model_utils + +__all__ = ['EncDecCTCModelBPE'] + + +class EncDecCTCModelBPE(EncDecCTCModel, ASRBPEMixin): + """Encoder decoder CTC-based models with Byte Pair Encoding.""" + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + results = [] + + model = PretrainedModelInfo( + pretrained_model_name="stt_en_citrinet_256", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_256", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_en_citrinet_256/versions/1.0.0rc1/files/stt_en_citrinet_256.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_en_citrinet_512", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_512", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_en_citrinet_512/versions/1.0.0rc1/files/stt_en_citrinet_512.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_en_citrinet_1024", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_1024", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_en_citrinet_1024/versions/1.0.0rc1/files/stt_en_citrinet_1024.nemo", + ) + + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_es_citrinet_512", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_es_citrinet_512", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_es_citrinet_512/versions/1.0.0/files/stt_es_citrinet_512.nemo", + ) + + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_en_conformer_ctc_small", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_small", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_en_conformer_ctc_small/versions/1.0.0/files/stt_en_conformer_ctc_small.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_en_conformer_ctc_medium", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_medium", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_en_conformer_ctc_medium/versions/1.0.0/files/stt_en_conformer_ctc_medium.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_en_conformer_ctc_large", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_en_conformer_ctc_large/versions/1.0.0/files/stt_en_conformer_ctc_large.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_en_conformer_ctc_small_ls", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_small_ls", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_en_conformer_ctc_small_ls/versions/1.0.0/files/stt_en_conformer_ctc_small_ls.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_en_conformer_ctc_medium_ls", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_medium_ls", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_en_conformer_ctc_medium_ls/versions/1.0.0/files/stt_en_conformer_ctc_medium_ls.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_en_conformer_ctc_large_ls", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large_ls", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_en_conformer_ctc_large_ls/versions/1.0.0/files/stt_en_conformer_ctc_large_ls.nemo", + ) + results.append(model) + + return results + + def __init__(self, cfg: DictConfig, trainer=None): + # Convert to Hydra 1.0 compatible DictConfig + cfg = model_utils.convert_model_config_to_dict_config(cfg) + cfg = model_utils.maybe_update_config_version(cfg) + + if 'tokenizer' not in cfg: + raise ValueError("`cfg` must have `tokenizer` config to create a tokenizer !") + + # Setup the tokenizer + self._setup_tokenizer(cfg.tokenizer) + + # Initialize a dummy vocabulary + vocabulary = self.tokenizer.tokenizer.get_vocab() + + # Set the new vocabulary + with open_dict(cfg): + cfg.decoder.vocabulary = ListConfig(list(vocabulary.keys())) + + # Override number of classes if placeholder provided + num_classes = cfg.decoder["num_classes"] + + if num_classes < 1: + logging.info( + "\nReplacing placeholder number of classes ({}) with actual number of classes - {}".format( + num_classes, len(vocabulary) + ) + ) + cfg.decoder["num_classes"] = len(vocabulary) + + super().__init__(cfg=cfg, trainer=trainer) + + # Setup metric objects + self._wer = WERBPE( + tokenizer=self.tokenizer, + batch_dim_index=0, + use_cer=self._cfg.get('use_cer', False), + ctc_decode=True, + dist_sync_on_step=True, + log_prediction=self._cfg.get("log_prediction", False), + ) + + def _setup_dataloader_from_config(self, config: Optional[Dict]): + if 'augmentor' in config: + augmentor = process_augmentations(config['augmentor']) + else: + augmentor = None + + shuffle = config['shuffle'] + + # Instantiate tarred dataset loader or normal dataset loader + if config.get('is_tarred', False): + if ('tarred_audio_filepaths' in config and config['tarred_audio_filepaths'] is None) or ( + 'manifest_filepath' in config and config['manifest_filepath'] is None + ): + logging.warning( + "Could not load dataset as `manifest_filepath` was None or " + f"`tarred_audio_filepaths` is None. Provided config : {config}" + ) + return None + + shuffle_n = config.get('shuffle_n', 4 * config['batch_size']) if shuffle else 0 + dataset = audio_to_text_dataset.get_tarred_bpe_dataset( + config=config, + tokenizer=self.tokenizer, + shuffle_n=shuffle_n, + global_rank=self.global_rank, + world_size=self.world_size, + augmentor=augmentor, + ) + shuffle = False + else: + if 'manifest_filepath' in config and config['manifest_filepath'] is None: + logging.warning(f"Could not load dataset as `manifest_filepath` was None. Provided config : {config}") + return None + + dataset = audio_to_text_dataset.get_bpe_dataset( + config=config, tokenizer=self.tokenizer, augmentor=augmentor + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=config['batch_size'], + collate_fn=dataset.collate_fn, + drop_last=config.get('drop_last', False), + shuffle=shuffle, + num_workers=config.get('num_workers', 0), + pin_memory=config.get('pin_memory', False), + ) + + def _setup_transcribe_dataloader(self, config: Dict) -> 'torch.utils.data.DataLoader': + """ + Setup function for a temporary data loader which wraps the provided audio file. + + Args: + config: A python dictionary which contains the following keys: + paths2audio_files: (a list) of paths to audio files. The files should be relatively short fragments. \ + Recommended length per file is between 5 and 25 seconds. + batch_size: (int) batch size to use during inference. \ + Bigger will result in better throughput performance but would use more memory. + temp_dir: (str) A temporary directory where the audio manifest is temporarily + stored. + + Returns: + A pytorch DataLoader for the given audio file(s). + """ + dl_config = { + 'manifest_filepath': os.path.join(config['temp_dir'], 'manifest.json'), + 'sample_rate': self.preprocessor._sample_rate, + 'batch_size': min(config['batch_size'], len(config['paths2audio_files'])), + 'shuffle': False, + } + + temporary_datalayer = self._setup_dataloader_from_config(config=DictConfig(dl_config)) + return temporary_datalayer + + def change_vocabulary(self, new_tokenizer_dir: str, new_tokenizer_type: str): + """ + Changes vocabulary of the tokenizer used during CTC decoding process. + Use this method when fine-tuning on from pre-trained model. + This method changes only decoder and leaves encoder and pre-processing modules unchanged. For example, you would + use it if you want to use pretrained encoder when fine-tuning on a data in another language, or when you'd need + model to learn capitalization, punctuation and/or special characters. + + Args: + new_tokenizer_dir: Path to the new tokenizer directory. + new_tokenizer_type: Either `bpe` or `wpe`. `bpe` is used for SentencePiece tokenizers, + whereas `wpe` is used for `BertTokenizer`. + + Returns: None + + """ + if not os.path.isdir(new_tokenizer_dir): + raise NotADirectoryError( + f'New tokenizer dir must be non-empty path to a directory. But I got: {new_tokenizer_dir}' + ) + + if new_tokenizer_type.lower() not in ('bpe', 'wpe'): + raise ValueError(f'New tokenizer type must be either `bpe` or `wpe`') + + tokenizer_cfg = OmegaConf.create({'dir': new_tokenizer_dir, 'type': new_tokenizer_type}) + + # Setup the tokenizer + self._setup_tokenizer(tokenizer_cfg) + + # Initialize a dummy vocabulary + vocabulary = self.tokenizer.tokenizer.get_vocab() + + # Set the new vocabulary + decoder_config = copy.deepcopy(self.decoder.to_config_dict()) + decoder_config.vocabulary = ListConfig(list(vocabulary.keys())) + + decoder_num_classes = decoder_config['num_classes'] + + # Override number of classes if placeholder provided + logging.info( + "\nReplacing old number of classes ({}) with new number of classes - {}".format( + decoder_num_classes, len(vocabulary) + ) + ) + + decoder_config['num_classes'] = len(vocabulary) + + del self.decoder + self.decoder = EncDecCTCModelBPE.from_config_dict(decoder_config) + del self.loss + self.loss = CTCLoss( + num_classes=self.decoder.num_classes_with_blank - 1, + zero_infinity=True, + reduction=self._cfg.get("ctc_reduction", "mean_batch"), + ) + self._wer = WERBPE( + tokenizer=self.tokenizer, + batch_dim_index=0, + use_cer=self._cfg.get('use_cer', False), + ctc_decode=True, + log_prediction=self._cfg.get("log_prediction", False), + ) + + # Update config + OmegaConf.set_struct(self._cfg.decoder, False) + self._cfg.decoder = decoder_config + OmegaConf.set_struct(self._cfg.decoder, True) + + logging.info(f"Changed tokenizer to {self.decoder.vocabulary} vocabulary.") diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/ctc_models.py b/models/tacotron2/NeMo/nemo/collections/asr/models/ctc_models.py new file mode 100644 index 0000000000..c3cbec0ad2 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/ctc_models.py @@ -0,0 +1,649 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import json +import os +import tempfile +from math import ceil +from typing import Dict, List, Optional, Union + +import torch +from omegaconf import DictConfig, OmegaConf, open_dict +from pytorch_lightning import Trainer +from tqdm.auto import tqdm + +from nemo.collections.asr.data import audio_to_text_dataset +from nemo.collections.asr.data.audio_to_text_dali import DALIOutputs +from nemo.collections.asr.losses.ctc import CTCLoss +from nemo.collections.asr.metrics.wer import WER +from nemo.collections.asr.models.asr_model import ASRModel, ExportableEncDecModel +from nemo.collections.asr.parts.mixins import ASRModuleMixin +from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types import AudioSignal, LabelsType, LengthsType, LogprobsType, NeuralType, SpectrogramType +from nemo.utils import logging + +__all__ = ['EncDecCTCModel'] + + +class EncDecCTCModel(ASRModel, ExportableEncDecModel, ASRModuleMixin): + """Base class for encoder decoder CTC-based models.""" + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + results = [] + + model = PretrainedModelInfo( + pretrained_model_name="QuartzNet15x5Base-En", + description="QuartzNet15x5 model trained on six datasets: LibriSpeech, Mozilla Common Voice (validated clips from en_1488h_2019-12-10), WSJ, Fisher, Switchboard, and NSC Singapore English. It was trained with Apex/Amp optimization level O1 for 600 epochs. The model achieves a WER of 3.79% on LibriSpeech dev-clean, and a WER of 10.05% on dev-other. Please visit https://ngc.nvidia.com/catalog/models/nvidia:nemospeechmodels for further details.", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemospeechmodels/versions/1.0.0a5/files/QuartzNet15x5Base-En.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_en_quartznet15x5", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_quartznet15x5", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_en_quartznet15x5/versions/1.0.0rc1/files/stt_en_quartznet15x5.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_zh_quartznet15x5", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_quartznet15x5", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_zh_quartznet15x5/versions/1.0.0rc1/files/stt_zh_quartznet15x5.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_en_jasper10x5dr", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_jasper10x5dr", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_en_jasper10x5dr/versions/1.0.0rc1/files/stt_en_jasper10x5dr.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_ca_quartznet15x5", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ca_quartznet15x5", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_ca_quartznet15x5/versions/1.0.0rc1/files/stt_ca_quartznet15x5.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_it_quartznet15x5", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_it_quartznet15x5", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_it_quartznet15x5/versions/1.0.0rc1/files/stt_it_quartznet15x5.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_fr_quartznet15x5", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_fr_quartznet15x5", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_fr_quartznet15x5/versions/1.0.0rc1/files/stt_fr_quartznet15x5.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_es_quartznet15x5", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_es_quartznet15x5", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_es_quartznet15x5/versions/1.0.0rc1/files/stt_es_quartznet15x5.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_de_quartznet15x5", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_de_quartznet15x5", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_de_quartznet15x5/versions/1.0.0rc1/files/stt_de_quartznet15x5.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_pl_quartznet15x5", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_pl_quartznet15x5", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_pl_quartznet15x5/versions/1.0.0rc1/files/stt_pl_quartznet15x5.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_ru_quartznet15x5", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ru_quartznet15x5", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_ru_quartznet15x5/versions/1.0.0rc1/files/stt_ru_quartznet15x5.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="stt_zh_citrinet_512", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_citrinet_512", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_zh_citrinet_512/versions/1.0.0rc1/files/stt_zh_citrinet_512.nemo", + ) + results.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="asr_talknet_aligner", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:asr_talknet_aligner", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/asr_talknet_aligner/versions/1.0.0rc1/files/qn5x5_libri_tts_phonemes.nemo", + ) + results.append(model) + + return results + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + # Get global rank and total number of GPU workers for IterableDataset partitioning, if applicable + # Global_rank and local_rank is set by LightningModule in Lightning 1.2.0 + self.world_size = 1 + if trainer is not None: + self.world_size = trainer.num_nodes * trainer.num_gpus + + super().__init__(cfg=cfg, trainer=trainer) + self.preprocessor = EncDecCTCModel.from_config_dict(self._cfg.preprocessor) + self.encoder = EncDecCTCModel.from_config_dict(self._cfg.encoder) + + with open_dict(self._cfg): + if "feat_in" not in self._cfg.decoder or ( + not self._cfg.decoder.feat_in and hasattr(self.encoder, '_feat_out') + ): + self._cfg.decoder.feat_in = self.encoder._feat_out + if "feat_in" not in self._cfg.decoder or not self._cfg.decoder.feat_in: + raise ValueError("param feat_in of the decoder's config is not set!") + + self.decoder = EncDecCTCModel.from_config_dict(self._cfg.decoder) + + self.loss = CTCLoss( + num_classes=self.decoder.num_classes_with_blank - 1, + zero_infinity=True, + reduction=self._cfg.get("ctc_reduction", "mean_batch"), + ) + + if hasattr(self._cfg, 'spec_augment') and self._cfg.spec_augment is not None: + self.spec_augmentation = EncDecCTCModel.from_config_dict(self._cfg.spec_augment) + else: + self.spec_augmentation = None + + # Setup metric objects + self._wer = WER( + vocabulary=self.decoder.vocabulary, + batch_dim_index=0, + use_cer=self._cfg.get('use_cer', False), + ctc_decode=True, + dist_sync_on_step=True, + log_prediction=self._cfg.get("log_prediction", False), + ) + + @torch.no_grad() + def transcribe( + self, + paths2audio_files: List[str], + batch_size: int = 4, + logprobs: bool = False, + return_hypotheses: bool = False, + ) -> List[str]: + """ + Uses greedy decoding to transcribe audio files. Use this method for debugging and prototyping. + + Args: + paths2audio_files: (a list) of paths to audio files. \ + Recommended length per file is between 5 and 25 seconds. \ + But it is possible to pass a few hours long file if enough GPU memory is available. + batch_size: (int) batch size to use during inference. + Bigger will result in better throughput performance but would use more memory. + logprobs: (bool) pass True to get log probabilities instead of transcripts. + return_hypotheses: (bool) Either return hypotheses or text + With hypotheses can do some postprocessing like getting timestamp or rescoring + + Returns: + A list of transcriptions (or raw log probabilities if logprobs is True) in the same order as paths2audio_files + """ + if paths2audio_files is None or len(paths2audio_files) == 0: + return {} + + if return_hypotheses and logprobs: + raise ValueError( + "Either `return_hypotheses` or `logprobs` can be True at any given time." + "Returned hypotheses will contain the logprobs." + ) + + # We will store transcriptions here + hypotheses = [] + # Model's mode and device + mode = self.training + device = next(self.parameters()).device + dither_value = self.preprocessor.featurizer.dither + pad_to_value = self.preprocessor.featurizer.pad_to + + try: + self.preprocessor.featurizer.dither = 0.0 + self.preprocessor.featurizer.pad_to = 0 + # Switch model to evaluation mode + self.eval() + # Freeze the encoder and decoder modules + self.encoder.freeze() + self.decoder.freeze() + logging_level = logging.get_verbosity() + logging.set_verbosity(logging.WARNING) + # Work in tmp directory - will store manifest file there + with tempfile.TemporaryDirectory() as tmpdir: + with open(os.path.join(tmpdir, 'manifest.json'), 'w') as fp: + for audio_file in paths2audio_files: + entry = {'audio_filepath': audio_file, 'duration': 100000, 'text': 'nothing'} + fp.write(json.dumps(entry) + '\n') + + config = {'paths2audio_files': paths2audio_files, 'batch_size': batch_size, 'temp_dir': tmpdir} + + temporary_datalayer = self._setup_transcribe_dataloader(config) + for test_batch in tqdm(temporary_datalayer, desc="Transcribing"): + logits, logits_len, greedy_predictions = self.forward( + input_signal=test_batch[0].to(device), input_signal_length=test_batch[1].to(device) + ) + if logprobs: + # dump log probs per file + for idx in range(logits.shape[0]): + lg = logits[idx][: logits_len[idx]] + hypotheses.append(lg.cpu().numpy()) + else: + current_hypotheses = self._wer.ctc_decoder_predictions_tensor( + greedy_predictions, predictions_len=logits_len, return_hypotheses=return_hypotheses, + ) + + if return_hypotheses: + # dump log probs per file + for idx in range(logits.shape[0]): + current_hypotheses[idx].y_sequence = logits[idx][: logits_len[idx]] + + hypotheses += current_hypotheses + + del greedy_predictions + del logits + del test_batch + finally: + # set mode back to its original value + self.train(mode=mode) + self.preprocessor.featurizer.dither = dither_value + self.preprocessor.featurizer.pad_to = pad_to_value + if mode is True: + self.encoder.unfreeze() + self.decoder.unfreeze() + logging.set_verbosity(logging_level) + return hypotheses + + def change_vocabulary(self, new_vocabulary: List[str]): + """ + Changes vocabulary used during CTC decoding process. Use this method when fine-tuning on from pre-trained model. + This method changes only decoder and leaves encoder and pre-processing modules unchanged. For example, you would + use it if you want to use pretrained encoder when fine-tuning on a data in another language, or when you'd need + model to learn capitalization, punctuation and/or special characters. + + If new_vocabulary == self.decoder.vocabulary then nothing will be changed. + + Args: + + new_vocabulary: list with new vocabulary. Must contain at least 2 elements. Typically, \ + this is target alphabet. + + Returns: None + + """ + if self.decoder.vocabulary == new_vocabulary: + logging.warning(f"Old {self.decoder.vocabulary} and new {new_vocabulary} match. Not changing anything.") + else: + if new_vocabulary is None or len(new_vocabulary) == 0: + raise ValueError(f'New vocabulary must be non-empty list of chars. But I got: {new_vocabulary}') + decoder_config = self.decoder.to_config_dict() + new_decoder_config = copy.deepcopy(decoder_config) + new_decoder_config['vocabulary'] = new_vocabulary + new_decoder_config['num_classes'] = len(new_vocabulary) + + del self.decoder + self.decoder = EncDecCTCModel.from_config_dict(new_decoder_config) + del self.loss + self.loss = CTCLoss( + num_classes=self.decoder.num_classes_with_blank - 1, + zero_infinity=True, + reduction=self._cfg.get("ctc_reduction", "mean_batch"), + ) + self._wer = WER( + vocabulary=self.decoder.vocabulary, + batch_dim_index=0, + use_cer=self._cfg.get('use_cer', False), + ctc_decode=True, + dist_sync_on_step=True, + log_prediction=self._cfg.get("log_prediction", False), + ) + + # Update config + OmegaConf.set_struct(self._cfg.decoder, False) + self._cfg.decoder = new_decoder_config + OmegaConf.set_struct(self._cfg.decoder, True) + + ds_keys = ['train_ds', 'validation_ds', 'test_ds'] + for key in ds_keys: + if key in self.cfg: + with open_dict(self.cfg[key]): + self.cfg[key]['labels'] = OmegaConf.create(new_vocabulary) + + logging.info(f"Changed decoder to output to {self.decoder.vocabulary} vocabulary.") + + def _setup_dataloader_from_config(self, config: Optional[Dict]): + if 'augmentor' in config: + augmentor = process_augmentations(config['augmentor']) + else: + augmentor = None + + # Automatically inject args from model config to dataloader config + audio_to_text_dataset.inject_dataloader_value_from_model_config(self.cfg, config, key='sample_rate') + audio_to_text_dataset.inject_dataloader_value_from_model_config(self.cfg, config, key='labels') + + shuffle = config['shuffle'] + device = 'gpu' if torch.cuda.is_available() else 'cpu' + if config.get('use_dali', False): + device_id = self.local_rank if device == 'gpu' else None + dataset = audio_to_text_dataset.get_dali_char_dataset( + config=config, + shuffle=shuffle, + device_id=device_id, + global_rank=self.global_rank, + world_size=self.world_size, + preprocessor_cfg=self._cfg.preprocessor, + ) + return dataset + + # Instantiate tarred dataset loader or normal dataset loader + if config.get('is_tarred', False): + if ('tarred_audio_filepaths' in config and config['tarred_audio_filepaths'] is None) or ( + 'manifest_filepath' in config and config['manifest_filepath'] is None + ): + logging.warning( + "Could not load dataset as `manifest_filepath` was None or " + f"`tarred_audio_filepaths` is None. Provided config : {config}" + ) + return None + + shuffle_n = config.get('shuffle_n', 4 * config['batch_size']) if shuffle else 0 + dataset = audio_to_text_dataset.get_tarred_char_dataset( + config=config, + shuffle_n=shuffle_n, + global_rank=self.global_rank, + world_size=self.world_size, + augmentor=augmentor, + ) + shuffle = False + else: + if 'manifest_filepath' in config and config['manifest_filepath'] is None: + logging.warning(f"Could not load dataset as `manifest_filepath` was None. Provided config : {config}") + return None + + dataset = audio_to_text_dataset.get_char_dataset(config=config, augmentor=augmentor) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=config['batch_size'], + collate_fn=dataset.collate_fn, + drop_last=config.get('drop_last', False), + shuffle=shuffle, + num_workers=config.get('num_workers', 0), + pin_memory=config.get('pin_memory', False), + ) + + def setup_training_data(self, train_data_config: Optional[Union[DictConfig, Dict]]): + """ + Sets up the training data loader via a Dict-like object. + + Args: + train_data_config: A config that contains the information regarding construction + of an ASR Training dataset. + + Supported Datasets: + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text_dali.AudioToCharDALIDataset` + """ + if 'shuffle' not in train_data_config: + train_data_config['shuffle'] = True + + # preserve config + self._update_dataset_config(dataset_name='train', config=train_data_config) + + self._train_dl = self._setup_dataloader_from_config(config=train_data_config) + + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'is_tarred' in train_data_config and train_data_config['is_tarred']: + # We also need to check if limit_train_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # training batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if isinstance(self._trainer.limit_train_batches, float): + self._trainer.limit_train_batches = int( + self._trainer.limit_train_batches + * ceil((len(self._train_dl.dataset) / self.world_size) / train_data_config['batch_size']) + ) + + def setup_validation_data(self, val_data_config: Optional[Union[DictConfig, Dict]]): + """ + Sets up the validation data loader via a Dict-like object. + + Args: + val_data_config: A config that contains the information regarding construction + of an ASR Training dataset. + + Supported Datasets: + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text_dali.AudioToCharDALIDataset` + """ + if 'shuffle' not in val_data_config: + val_data_config['shuffle'] = False + + # preserve config + self._update_dataset_config(dataset_name='validation', config=val_data_config) + + self._validation_dl = self._setup_dataloader_from_config(config=val_data_config) + + def setup_test_data(self, test_data_config: Optional[Union[DictConfig, Dict]]): + """ + Sets up the test data loader via a Dict-like object. + + Args: + test_data_config: A config that contains the information regarding construction + of an ASR Training dataset. + + Supported Datasets: + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text_dali.AudioToCharDALIDataset` + """ + if 'shuffle' not in test_data_config: + test_data_config['shuffle'] = False + + # preserve config + self._update_dataset_config(dataset_name='test', config=test_data_config) + + self._test_dl = self._setup_dataloader_from_config(config=test_data_config) + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + if hasattr(self.preprocessor, '_sample_rate'): + input_signal_eltype = AudioSignal(freq=self.preprocessor._sample_rate) + else: + input_signal_eltype = AudioSignal() + return { + "input_signal": NeuralType(('B', 'T'), input_signal_eltype, optional=True), + "input_signal_length": NeuralType(tuple('B'), LengthsType(), optional=True), + "processed_signal": NeuralType(('B', 'D', 'T'), SpectrogramType(), optional=True), + "processed_signal_length": NeuralType(tuple('B'), LengthsType(), optional=True), + } + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "outputs": NeuralType(('B', 'T', 'D'), LogprobsType()), + "encoded_lengths": NeuralType(tuple('B'), LengthsType()), + "greedy_predictions": NeuralType(('B', 'T'), LabelsType()), + } + + @typecheck() + def forward( + self, input_signal=None, input_signal_length=None, processed_signal=None, processed_signal_length=None + ): + """ + Forward pass of the model. + + Args: + input_signal: Tensor that represents a batch of raw audio signals, + of shape [B, T]. T here represents timesteps, with 1 second of audio represented as + `self.sample_rate` number of floating point values. + input_signal_length: Vector of length B, that contains the individual lengths of the audio + sequences. + processed_signal: Tensor that represents a batch of processed audio signals, + of shape (B, D, T) that has undergone processing via some DALI preprocessor. + processed_signal_length: Vector of length B, that contains the individual lengths of the + processed audio sequences. + + Returns: + A tuple of 3 elements - + 1) The log probabilities tensor of shape [B, T, D]. + 2) The lengths of the acoustic sequence after propagation through the encoder, of shape [B]. + 3) The greedy token predictions of the model of shape [B, T] (via argmax) + """ + has_input_signal = input_signal is not None and input_signal_length is not None + has_processed_signal = processed_signal is not None and processed_signal_length is not None + if (has_input_signal ^ has_processed_signal) == False: + raise ValueError( + f"{self} Arguments ``input_signal`` and ``input_signal_length`` are mutually exclusive " + " with ``processed_signal`` and ``processed_signal_len`` arguments." + ) + + if not has_processed_signal: + processed_signal, processed_signal_length = self.preprocessor( + input_signal=input_signal, length=input_signal_length, + ) + + if self.spec_augmentation is not None and self.training: + processed_signal = self.spec_augmentation(input_spec=processed_signal, length=processed_signal_length) + + encoded, encoded_len = self.encoder(audio_signal=processed_signal, length=processed_signal_length) + log_probs = self.decoder(encoder_output=encoded) + greedy_predictions = log_probs.argmax(dim=-1, keepdim=False) + + return log_probs, encoded_len, greedy_predictions + + # PTL-specific methods + def training_step(self, batch, batch_nb): + signal, signal_len, transcript, transcript_len = batch + if isinstance(batch, DALIOutputs) and batch.has_processed_signal: + log_probs, encoded_len, predictions = self.forward( + processed_signal=signal, processed_signal_length=signal_len + ) + else: + log_probs, encoded_len, predictions = self.forward(input_signal=signal, input_signal_length=signal_len) + + loss_value = self.loss( + log_probs=log_probs, targets=transcript, input_lengths=encoded_len, target_lengths=transcript_len + ) + + tensorboard_logs = {'train_loss': loss_value, 'learning_rate': self._optimizer.param_groups[0]['lr']} + + if hasattr(self, '_trainer') and self._trainer is not None: + log_every_n_steps = self._trainer.log_every_n_steps + else: + log_every_n_steps = 1 + + if (batch_nb + 1) % log_every_n_steps == 0: + self._wer.update( + predictions=predictions, + targets=transcript, + target_lengths=transcript_len, + predictions_lengths=encoded_len, + ) + wer, _, _ = self._wer.compute() + self._wer.reset() + tensorboard_logs.update({'training_batch_wer': wer}) + + return {'loss': loss_value, 'log': tensorboard_logs} + + def validation_step(self, batch, batch_idx, dataloader_idx=0): + signal, signal_len, transcript, transcript_len = batch + if isinstance(batch, DALIOutputs) and batch.has_processed_signal: + log_probs, encoded_len, predictions = self.forward( + processed_signal=signal, processed_signal_length=signal_len + ) + else: + log_probs, encoded_len, predictions = self.forward(input_signal=signal, input_signal_length=signal_len) + + loss_value = self.loss( + log_probs=log_probs, targets=transcript, input_lengths=encoded_len, target_lengths=transcript_len + ) + self._wer.update( + predictions=predictions, targets=transcript, target_lengths=transcript_len, predictions_lengths=encoded_len + ) + wer, wer_num, wer_denom = self._wer.compute() + self._wer.reset() + return { + 'val_loss': loss_value, + 'val_wer_num': wer_num, + 'val_wer_denom': wer_denom, + 'val_wer': wer, + } + + def test_step(self, batch, batch_idx, dataloader_idx=0): + logs = self.validation_step(batch, batch_idx, dataloader_idx=dataloader_idx) + test_logs = { + 'test_loss': logs['val_loss'], + 'test_wer_num': logs['val_wer_num'], + 'test_wer_denom': logs['val_wer_denom'], + 'test_wer': logs['val_wer'], + } + return test_logs + + def test_dataloader(self): + if self._test_dl is not None: + return self._test_dl + + def _setup_transcribe_dataloader(self, config: Dict) -> 'torch.utils.data.DataLoader': + """ + Setup function for a temporary data loader which wraps the provided audio file. + + Args: + config: A python dictionary which contains the following keys: + paths2audio_files: (a list) of paths to audio files. The files should be relatively short fragments. \ + Recommended length per file is between 5 and 25 seconds. + batch_size: (int) batch size to use during inference. \ + Bigger will result in better throughput performance but would use more memory. + temp_dir: (str) A temporary directory where the audio manifest is temporarily + stored. + + Returns: + A pytorch DataLoader for the given audio file(s). + """ + dl_config = { + 'manifest_filepath': os.path.join(config['temp_dir'], 'manifest.json'), + 'sample_rate': self.preprocessor._sample_rate, + 'labels': self.decoder.vocabulary, + 'batch_size': min(config['batch_size'], len(config['paths2audio_files'])), + 'trim_silence': True, + 'shuffle': False, + } + + temporary_datalayer = self._setup_dataloader_from_config(config=DictConfig(dl_config)) + return temporary_datalayer diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/label_models.py b/models/tacotron2/NeMo/nemo/collections/asr/models/label_models.py new file mode 100644 index 0000000000..8ae5707ad1 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/label_models.py @@ -0,0 +1,409 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import json +import os +import pickle as pkl +from typing import Dict, List, Optional, Union + +import librosa +import torch +from omegaconf import DictConfig +from omegaconf.omegaconf import open_dict +from pytorch_lightning import Trainer + +from nemo.collections.asr.data.audio_to_label import AudioToSpeechLabelDataset +from nemo.collections.asr.losses.angularloss import AngularSoftmaxLoss +from nemo.collections.asr.models.asr_model import ExportableEncDecModel +from nemo.collections.asr.parts.preprocessing.features import WaveformFeaturizer +from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations +from nemo.collections.common.losses import CrossEntropyLoss as CELoss +from nemo.collections.common.metrics import TopKClassificationAccuracy +from nemo.core.classes import ModelPT +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types import * +from nemo.utils import logging + +__all__ = ['EncDecSpeakerLabelModel', 'ExtractSpeakerEmbeddingsModel'] + + +class EncDecSpeakerLabelModel(ModelPT, ExportableEncDecModel): + """Encoder decoder class for speaker label models. + Model class creates training, validation methods for setting up data + performing model forward pass. + Expects config dict for + * preprocessor + * Jasper/Quartznet Encoder + * Speaker Decoder + """ + + @classmethod + def list_available_models(cls) -> List[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + result = [] + model = PretrainedModelInfo( + pretrained_model_name="speakerrecognition_speakernet", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/speakerrecognition_speakernet/versions/1.0.0rc1/files/speakerrecognition_speakernet.nemo", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerrecognition_speakernet", + ) + result.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="speakerverification_speakernet", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/speakerverification_speakernet/versions/1.0.0rc1/files/speakerverification_speakernet.nemo", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerverification_speakernet", + ) + result.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="speakerdiarization_speakernet", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/speakerdiarization_speakernet/versions/1.0.0rc1/files/speakerdiarization_speakernet.nemo", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerdiarization_speakernet", + ) + result.append(model) + + return result + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + super().__init__(cfg=cfg, trainer=trainer) + self.preprocessor = EncDecSpeakerLabelModel.from_config_dict(cfg.preprocessor) + self.encoder = EncDecSpeakerLabelModel.from_config_dict(cfg.encoder) + self.decoder = EncDecSpeakerLabelModel.from_config_dict(cfg.decoder) + if 'angular' in cfg.decoder and cfg.decoder['angular']: + logging.info("Training with Angular Softmax Loss") + scale = cfg.loss.scale + margin = cfg.loss.margin + self.loss = AngularSoftmaxLoss(scale=scale, margin=margin) + else: + logging.info("Training with Softmax-CrossEntropy loss") + self.loss = CELoss() + self.task = None + self._accuracy = TopKClassificationAccuracy(top_k=[1]) + + def __setup_dataloader_from_config(self, config: Optional[Dict]): + if 'augmentor' in config: + augmentor = process_augmentations(config['augmentor']) + else: + augmentor = None + + featurizer = WaveformFeaturizer( + sample_rate=config['sample_rate'], int_values=config.get('int_values', False), augmentor=augmentor + ) + self.dataset = AudioToSpeechLabelDataset( + manifest_filepath=config['manifest_filepath'], + labels=config['labels'], + featurizer=featurizer, + max_duration=config.get('max_duration', None), + min_duration=config.get('min_duration', None), + trim=False, + time_length=config.get('time_length', 8), + shift_length=config.get('shift_length', 0.75), + ) + + if self.task == 'diarization': + logging.info("Setting up diarization parameters") + _collate_func = self.dataset.sliced_seq_collate_fn + batch_size = 1 + shuffle = False + else: + logging.info("Setting up identification parameters") + _collate_func = self.dataset.fixed_seq_collate_fn + batch_size = config['batch_size'] + shuffle = config.get('shuffle', False) + + return torch.utils.data.DataLoader( + dataset=self.dataset, + batch_size=batch_size, + collate_fn=_collate_func, + drop_last=config.get('drop_last', False), + shuffle=shuffle, + num_workers=config.get('num_workers', 0), + pin_memory=config.get('pin_memory', False), + ) + + def setup_training_data(self, train_data_layer_config: Optional[Union[DictConfig, Dict]]): + if 'shuffle' not in train_data_layer_config: + train_data_layer_config['shuffle'] = True + self.task = 'identification' + self._train_dl = self.__setup_dataloader_from_config(config=train_data_layer_config) + + def setup_validation_data(self, val_data_layer_config: Optional[Union[DictConfig, Dict]]): + val_data_layer_config['labels'] = self.dataset.labels + self.task = 'identification' + self._validation_dl = self.__setup_dataloader_from_config(config=val_data_layer_config) + + def setup_test_data(self, test_data_layer_params: Optional[Union[DictConfig, Dict]]): + if hasattr(self, 'dataset'): + test_data_layer_params['labels'] = self.dataset.labels + + if 'task' in test_data_layer_params and test_data_layer_params['task']: + self.task = test_data_layer_params['task'].lower() + self.time_length = test_data_layer_params.get('time_length', 1.5) + self.shift_length = test_data_layer_params.get('shift_length', 0.75) + else: + self.task = 'identification' + + self.embedding_dir = test_data_layer_params.get('embedding_dir', './') + self._test_dl = self.__setup_dataloader_from_config(config=test_data_layer_params) + self.test_manifest = test_data_layer_params.get('manifest_filepath', None) + + def test_dataloader(self): + if self._test_dl is not None: + return self._test_dl + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + if hasattr(self.preprocessor, '_sample_rate'): + audio_eltype = AudioSignal(freq=self.preprocessor._sample_rate) + else: + audio_eltype = AudioSignal() + return { + "input_signal": NeuralType(('B', 'T'), audio_eltype), + "input_signal_length": NeuralType(tuple('B'), LengthsType()), + } + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "logits": NeuralType(('B', 'D'), LogitsType()), + "embs": NeuralType(('B', 'D'), AcousticEncodedRepresentation()), + } + + @typecheck() + def forward(self, input_signal, input_signal_length): + processed_signal, processed_signal_len = self.preprocessor( + input_signal=input_signal, length=input_signal_length, + ) + + encoded, _ = self.encoder(audio_signal=processed_signal, length=processed_signal_len) + logits, embs = self.decoder(encoder_output=encoded) + return logits, embs + + # PTL-specific methods + def training_step(self, batch, batch_idx): + audio_signal, audio_signal_len, labels, _ = batch + logits, _ = self.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + loss = self.loss(logits=logits, labels=labels) + + self.log('loss', loss) + self.log('learning_rate', self._optimizer.param_groups[0]['lr']) + + self._accuracy(logits=logits, labels=labels) + top_k = self._accuracy.compute() + self._accuracy.reset() + for i, top_i in enumerate(top_k): + self.log(f'training_batch_accuracy_top@{i}', top_i) + + return {'loss': loss} + + def validation_step(self, batch, batch_idx, dataloader_idx: int = 0): + audio_signal, audio_signal_len, labels, _ = batch + logits, _ = self.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + loss_value = self.loss(logits=logits, labels=labels) + acc_top_k = self._accuracy(logits=logits, labels=labels) + correct_counts, total_counts = self._accuracy.correct_counts_k, self._accuracy.total_counts_k + + return { + 'val_loss': loss_value, + 'val_correct_counts': correct_counts, + 'val_total_counts': total_counts, + 'val_acc_top_k': acc_top_k, + } + + def multi_validation_epoch_end(self, outputs, dataloader_idx: int = 0): + val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean() + correct_counts = torch.stack([x['val_correct_counts'] for x in outputs]).sum(axis=0) + total_counts = torch.stack([x['val_total_counts'] for x in outputs]).sum(axis=0) + + self._accuracy.correct_counts_k = correct_counts + self._accuracy.total_counts_k = total_counts + topk_scores = self._accuracy.compute() + self._accuracy.reset() + + logging.info("val_loss: {:.3f}".format(val_loss_mean)) + self.log('val_loss', val_loss_mean) + for top_k, score in zip(self._accuracy.top_k, topk_scores): + self.log('val_epoch_accuracy_top@{}'.format(top_k), score) + + return { + 'val_loss': val_loss_mean, + 'val_acc_top_k': topk_scores, + } + + def test_step(self, batch, batch_idx, dataloader_idx: int = 0): + audio_signal, audio_signal_len, labels, _ = batch + logits, _ = self.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + loss_value = self.loss(logits=logits, labels=labels) + acc_top_k = self._accuracy(logits=logits, labels=labels) + correct_counts, total_counts = self._accuracy.correct_counts_k, self._accuracy.total_counts_k + + return { + 'test_loss': loss_value, + 'test_correct_counts': correct_counts, + 'test_total_counts': total_counts, + 'test_acc_top_k': acc_top_k, + } + + def multi_test_epoch_end(self, outputs, dataloader_idx: int = 0): + test_loss_mean = torch.stack([x['test_loss'] for x in outputs]).mean() + correct_counts = torch.stack([x['test_correct_counts'] for x in outputs]).sum(axis=0) + total_counts = torch.stack([x['test_total_counts'] for x in outputs]).sum(axis=0) + + self._accuracy.correct_counts_k = correct_counts + self._accuracy.total_counts_k = total_counts + topk_scores = self._accuracy.compute() + self._accuracy.reset() + + logging.info("test_loss: {:.3f}".format(test_loss_mean)) + self.log('test_loss', test_loss_mean) + for top_k, score in zip(self._accuracy.top_k, topk_scores): + self.log('test_epoch_accuracy_top@{}'.format(top_k), score) + + return { + 'test_loss': test_loss_mean, + 'test_acc_top_k': topk_scores, + } + + def setup_finetune_model(self, model_config: DictConfig): + """ + setup_finetune_model method sets up training data, validation data and test data with new + provided config, this checks for the previous labels set up during training from scratch, if None, + it sets up labels for provided finetune data from manifest files + Args: + model_config: cfg which has train_ds, optional validation_ds, optional test_ds and + mandatory encoder and decoder model params + make sure you set num_classes correctly for finetune data + Returns: None + """ + if hasattr(self, 'dataset'): + scratch_labels = self.dataset.labels + else: + scratch_labels = None + + logging.info("Setting up data loaders with manifests provided from model_config") + + if 'train_ds' in model_config and model_config.train_ds is not None: + self.setup_training_data(model_config.train_ds) + else: + raise KeyError("train_ds is not found in model_config but you need it for fine tuning") + + if self.dataset.labels is None or len(self.dataset.labels) == 0: + raise ValueError(f'New labels must be non-empty list of labels. But I got: {self.dataset.labels}') + + if 'validation_ds' in model_config and model_config.validation_ds is not None: + self.setup_multiple_validation_data(model_config.validation_ds) + + if 'test_ds' in model_config and model_config.test_ds is not None: + self.setup_multiple_test_data(model_config.test_ds) + + if scratch_labels == self.dataset.labels: # checking for new finetune dataset labels + logging.warning( + "Trained dataset labels are same as finetune dataset labels -- continuing change of decoder parameters" + ) + elif scratch_labels is None: + logging.warning( + "Either you provided a dummy manifest file during training from scratch or you restored from a pretrained nemo file" + ) + + decoder_config = model_config.decoder + new_decoder_config = copy.deepcopy(decoder_config) + if new_decoder_config['num_classes'] != len(self.dataset.labels): + raise ValueError( + "number of classes provided {} is not same as number of different labels in finetuning data: {}".format( + new_decoder_config['num_classes'], len(self.dataset.labels) + ) + ) + + del self.decoder + self.decoder = EncDecSpeakerLabelModel.from_config_dict(new_decoder_config) + + with open_dict(self._cfg.decoder): + self._cfg.decoder = new_decoder_config + + logging.info(f"Changed decoder output to # {self.decoder._num_classes} classes.") + + @torch.no_grad() + def get_embedding(self, path2audio_file): + audio, sr = librosa.load(path2audio_file, sr=None) + target_sr = self._cfg.train_ds.get('sample_rate', 16000) + if sr != target_sr: + audio = librosa.core.resample(audio, sr, target_sr) + audio_length = audio.shape[0] + device = self.device + audio_signal, audio_signal_len = ( + torch.tensor([audio], device=device), + torch.tensor([audio_length], device=device), + ) + mode = self.training + self.freeze() + + _, embs = self.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + + self.train(mode=mode) + if mode is True: + self.unfreeze() + del audio_signal, audio_signal_len + return embs + + +class ExtractSpeakerEmbeddingsModel(EncDecSpeakerLabelModel): + """ + This Model class facilitates extraction of speaker embeddings from a pretrained model. + Respective embedding file is saved in self.embedding dir passed through cfg + """ + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + super().__init__(cfg=cfg, trainer=trainer) + + def test_step(self, batch, batch_ix): + audio_signal, audio_signal_len, labels, slices = batch + _, embs = self.forward(input_signal=audio_signal, input_signal_length=audio_signal_len) + return {'embs': embs, 'labels': labels, 'slices': slices} + + def test_epoch_end(self, outputs): + embs = torch.cat([x['embs'] for x in outputs]) + slices = torch.cat([x['slices'] for x in outputs]) + emb_shape = embs.shape[-1] + embs = embs.view(-1, emb_shape).cpu().numpy() + out_embeddings = {} + start_idx = 0 + with open(self.test_manifest, 'r') as manifest: + for idx, line in enumerate(manifest.readlines()): + line = line.strip() + dic = json.loads(line) + structure = dic['audio_filepath'].split('/')[-3:] + uniq_name = '@'.join(structure) + if uniq_name in out_embeddings: + raise KeyError("Embeddings for label {} already present in emb dictionary".format(uniq_name)) + num_slices = slices[idx] + end_idx = start_idx + num_slices + out_embeddings[uniq_name] = embs[start_idx:end_idx].mean(axis=0) + start_idx = end_idx + + embedding_dir = os.path.join(self.embedding_dir, 'embeddings') + if not os.path.exists(embedding_dir): + os.mkdir(embedding_dir) + + prefix = self.test_manifest.split('/')[-1].split('.')[-2] + + name = os.path.join(embedding_dir, prefix) + pkl.dump(out_embeddings, open(name + '_embeddings.pkl', 'wb')) + logging.info("Saved embedding files to {}".format(embedding_dir)) + + return {} diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/rnnt_bpe_models.py b/models/tacotron2/NeMo/nemo/collections/asr/models/rnnt_bpe_models.py new file mode 100644 index 0000000000..da69c9d02d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/rnnt_bpe_models.py @@ -0,0 +1,290 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import os +from typing import Dict, Optional + +import torch +from omegaconf import DictConfig, ListConfig, OmegaConf, open_dict +from pytorch_lightning import Trainer + +from nemo.collections.asr.data import audio_to_text_dataset +from nemo.collections.asr.losses.rnnt import RNNTLoss +from nemo.collections.asr.metrics.rnnt_wer_bpe import RNNTBPEWER, RNNTBPEDecoding +from nemo.collections.asr.models.rnnt_models import EncDecRNNTModel +from nemo.collections.asr.parts.mixins import ASRBPEMixin +from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations +from nemo.core.classes.common import PretrainedModelInfo +from nemo.utils import logging, model_utils + + +class EncDecRNNTBPEModel(EncDecRNNTModel, ASRBPEMixin): + """Base class for encoder decoder RNNT-based models with subword tokenization.""" + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + result = [] + return result + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + # Convert to Hydra 1.0 compatible DictConfig + cfg = model_utils.convert_model_config_to_dict_config(cfg) + cfg = model_utils.maybe_update_config_version(cfg) + + # Tokenizer is necessary for this model + if 'tokenizer' not in cfg: + raise ValueError("`cfg` must have `tokenizer` config to create a tokenizer !") + + if not isinstance(cfg, DictConfig): + cfg = OmegaConf.create(cfg) + + # Setup the tokenizer + self._setup_tokenizer(cfg.tokenizer) + + # Initialize a dummy vocabulary + vocabulary = self.tokenizer.tokenizer.get_vocab() + + # Set the new vocabulary + with open_dict(cfg): + cfg.labels = ListConfig(list(vocabulary)) + + with open_dict(cfg.decoder): + cfg.decoder.vocab_size = len(vocabulary) + + with open_dict(cfg.joint): + cfg.joint.num_classes = len(vocabulary) + cfg.joint.vocabulary = ListConfig(list(vocabulary)) + cfg.joint.jointnet.encoder_hidden = cfg.model_defaults.enc_hidden + cfg.joint.jointnet.pred_hidden = cfg.model_defaults.pred_hidden + + super().__init__(cfg=cfg, trainer=trainer) + + # Setup decoding object + self.decoding = RNNTBPEDecoding( + decoding_cfg=self.cfg.decoding, decoder=self.decoder, joint=self.joint, tokenizer=self.tokenizer, + ) + + # Setup wer object + self.wer = RNNTBPEWER( + decoding=self.decoding, + batch_dim_index=0, + use_cer=self._cfg.get('use_cer', False), + log_prediction=self._cfg.get('log_prediction', True), + dist_sync_on_step=True, + ) + + # Setup fused Joint step if flag is set + if self.joint.fuse_loss_wer: + self.joint.set_loss(self.loss) + self.joint.set_wer(self.wer) + + def change_vocabulary( + self, new_tokenizer_dir: str, new_tokenizer_type: str, decoding_cfg: Optional[DictConfig] = None + ): + """ + Changes vocabulary used during RNNT decoding process. Use this method when fine-tuning on from pre-trained model. + This method changes only decoder and leaves encoder and pre-processing modules unchanged. For example, you would + use it if you want to use pretrained encoder when fine-tuning on data in another language, or when you'd need + model to learn capitalization, punctuation and/or special characters. + + Args: + new_tokenizer_dir: Directory path to tokenizer. + new_tokenizer_type: Type of tokenizer. Can be either `bpe` or `wpe`. + decoding_cfg: A config for the decoder, which is optional. If the decoding type + needs to be changed (from say Greedy to Beam decoding etc), the config can be passed here. + + Returns: None + + """ + if not os.path.isdir(new_tokenizer_dir): + raise NotADirectoryError( + f'New tokenizer dir must be non-empty path to a directory. But I got: {new_tokenizer_dir}' + ) + + if new_tokenizer_type.lower() not in ('bpe', 'wpe'): + raise ValueError(f'New tokenizer type must be either `bpe` or `wpe`') + + tokenizer_cfg = OmegaConf.create({'dir': new_tokenizer_dir, 'type': new_tokenizer_type}) + + # Setup the tokenizer + self._setup_tokenizer(tokenizer_cfg) + + # Initialize a dummy vocabulary + vocabulary = self.tokenizer.tokenizer.get_vocab() + + joint_config = self.joint.to_config_dict() + new_joint_config = copy.deepcopy(joint_config) + new_joint_config['vocabulary'] = ListConfig(list(vocabulary.keys())) + new_joint_config['num_classes'] = len(vocabulary) + del self.joint + self.joint = EncDecRNNTBPEModel.from_config_dict(new_joint_config) + + decoder_config = self.decoder.to_config_dict() + new_decoder_config = copy.deepcopy(decoder_config) + new_decoder_config.vocab_size = len(vocabulary) + del self.decoder + self.decoder = EncDecRNNTBPEModel.from_config_dict(new_decoder_config) + + del self.loss + self.loss = RNNTLoss(num_classes=self.joint.num_classes_with_blank - 1) + + if decoding_cfg is None: + # Assume same decoding config as before + decoding_cfg = self.cfg.decoding + + self.decoding = RNNTBPEDecoding( + decoding_cfg=decoding_cfg, decoder=self.decoder, joint=self.joint, tokenizer=self.tokenizer, + ) + + self.wer = RNNTBPEWER( + decoding=self.decoding, + batch_dim_index=self.wer.batch_dim_index, + use_cer=self.wer.use_cer, + log_prediction=self.wer.log_prediction, + dist_sync_on_step=True, + ) + + # Setup fused Joint step + if self.joint.fuse_loss_wer: + self.joint.set_loss(self.loss) + self.joint.set_wer(self.wer) + + # Update config + with open_dict(self.cfg.joint): + self.cfg.joint = new_joint_config + + with open_dict(self.cfg.decoder): + self.cfg.decoder = new_decoder_config + + with open_dict(self.cfg.decoding): + self.cfg.decoding = decoding_cfg + + logging.info(f"Changed decoder to output to {self.joint.vocabulary} vocabulary.") + + def change_decoding_strategy(self, decoding_cfg: DictConfig): + """ + Changes decoding strategy used during RNNT decoding process. + + Args: + decoding_cfg: A config for the decoder, which is optional. If the decoding type + needs to be changed (from say Greedy to Beam decoding etc), the config can be passed here. + """ + if decoding_cfg is None: + # Assume same decoding config as before + logging.info("No `decoding_cfg` passed when changing decoding strategy, using internal config") + decoding_cfg = self.cfg.decoding + + self.decoding = RNNTBPEDecoding( + decoding_cfg=decoding_cfg, decoder=self.decoder, joint=self.joint, tokenizer=self.tokenizer, + ) + + self.wer = RNNTBPEWER( + decoding=self.decoding, + batch_dim_index=self.wer.batch_dim_index, + use_cer=self.wer.use_cer, + log_prediction=self.wer.log_prediction, + dist_sync_on_step=True, + ) + + # Setup fused Joint step + if self.joint.fuse_loss_wer: + self.joint.set_loss(self.loss) + self.joint.set_wer(self.wer) + + # Update config + with open_dict(self.cfg.decoding): + self.cfg.decoding = decoding_cfg + + logging.info(f"Changed decoding strategy to \n{OmegaConf.to_yaml(self.cfg.decoding)}") + + def _setup_dataloader_from_config(self, config: Optional[Dict]): + if 'augmentor' in config: + augmentor = process_augmentations(config['augmentor']) + else: + augmentor = None + + shuffle = config['shuffle'] + + # Instantiate tarred dataset loader or normal dataset loader + if config.get('is_tarred', False): + if ('tarred_audio_filepaths' in config and config['tarred_audio_filepaths'] is None) or ( + 'manifest_filepath' in config and config['manifest_filepath'] is None + ): + logging.warning( + "Could not load dataset as `manifest_filepath` was None or " + f"`tarred_audio_filepaths` is None. Provided config : {config}" + ) + return None + + shuffle_n = config.get('shuffle_n', 4 * config['batch_size']) if shuffle else 0 + dataset = audio_to_text_dataset.get_tarred_bpe_dataset( + config=config, + tokenizer=self.tokenizer, + shuffle_n=shuffle_n, + global_rank=self.global_rank, + world_size=self.world_size, + augmentor=augmentor, + ) + shuffle = False + else: + if 'manifest_filepath' in config and config['manifest_filepath'] is None: + logging.warning(f"Could not load dataset as `manifest_filepath` was None. Provided config : {config}") + return None + + dataset = audio_to_text_dataset.get_bpe_dataset( + config=config, tokenizer=self.tokenizer, augmentor=augmentor + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=config['batch_size'], + collate_fn=dataset.collate_fn, + drop_last=config.get('drop_last', False), + shuffle=shuffle, + num_workers=config.get('num_workers', 0), + pin_memory=config.get('pin_memory', False), + ) + + def _setup_transcribe_dataloader(self, config: Dict) -> 'torch.utils.data.DataLoader': + """ + Setup function for a temporary data loader which wraps the provided audio file. + + Args: + config: A python dictionary which contains the following keys: + paths2audio_files: (a list) of paths to audio files. The files should be relatively short fragments. \ + Recommended length per file is between 5 and 25 seconds. + batch_size: (int) batch size to use during inference. \ + Bigger will result in better throughput performance but would use more memory. + temp_dir: (str) A temporary directory where the audio manifest is temporarily + stored. + + Returns: + A pytorch DataLoader for the given audio file(s). + """ + dl_config = { + 'manifest_filepath': os.path.join(config['temp_dir'], 'manifest.json'), + 'sample_rate': self.preprocessor._sample_rate, + 'batch_size': min(config['batch_size'], len(config['paths2audio_files'])), + 'shuffle': False, + } + + temporary_datalayer = self._setup_dataloader_from_config(config=DictConfig(dl_config)) + return temporary_datalayer diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/rnnt_models.py b/models/tacotron2/NeMo/nemo/collections/asr/models/rnnt_models.py new file mode 100644 index 0000000000..4f3725de58 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/rnnt_models.py @@ -0,0 +1,835 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import json +import os +import tempfile +from math import ceil +from typing import Dict, List, Optional, Union + +import torch +from omegaconf import DictConfig, OmegaConf, open_dict +from pytorch_lightning import Trainer +from tqdm.auto import tqdm + +from nemo.collections.asr.data import audio_to_text_dataset +from nemo.collections.asr.data.audio_to_text_dali import DALIOutputs +from nemo.collections.asr.losses.rnnt import RNNTLoss, resolve_rnnt_default_loss_name +from nemo.collections.asr.metrics.rnnt_wer import RNNTWER, RNNTDecoding +from nemo.collections.asr.models.asr_model import ASRModel +from nemo.collections.asr.parts.mixins import ASRModuleMixin +from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types import AcousticEncodedRepresentation, AudioSignal, LengthsType, NeuralType, SpectrogramType +from nemo.utils import logging + + +class EncDecRNNTModel(ASRModel, ASRModuleMixin): + """Base class for encoder decoder RNNT-based models.""" + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + result = [] + return result + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + # Get global rank and total number of GPU workers for IterableDataset partitioning, if applicable + # Global_rank and local_rank is set by LightningModule in Lightning 1.2.0 + self.world_size = 1 + if trainer is not None: + self.world_size = trainer.num_nodes * trainer.num_gpus + + super().__init__(cfg=cfg, trainer=trainer) + + # Initialize components + self.preprocessor = EncDecRNNTModel.from_config_dict(self.cfg.preprocessor) + self.encoder = EncDecRNNTModel.from_config_dict(self.cfg.encoder) + + # Update config values required by components dynamically + with open_dict(self.cfg.decoder): + self.cfg.decoder.vocab_size = len(self.cfg.labels) + + with open_dict(self.cfg.joint): + self.cfg.joint.num_classes = len(self.cfg.labels) + self.cfg.joint.vocabulary = self.cfg.labels + self.cfg.joint.jointnet.encoder_hidden = self.cfg.model_defaults.enc_hidden + self.cfg.joint.jointnet.pred_hidden = self.cfg.model_defaults.pred_hidden + + self.decoder = EncDecRNNTModel.from_config_dict(self.cfg.decoder) + self.joint = EncDecRNNTModel.from_config_dict(self.cfg.joint) + + # Setup RNNT Loss + loss_name, loss_kwargs = self.extract_rnnt_loss_cfg(self.cfg.get("loss", None)) + + self.loss = RNNTLoss( + num_classes=self.joint.num_classes_with_blank - 1, loss_name=loss_name, loss_kwargs=loss_kwargs + ) + + if hasattr(self.cfg, 'spec_augment') and self._cfg.spec_augment is not None: + self.spec_augmentation = EncDecRNNTModel.from_config_dict(self.cfg.spec_augment) + else: + self.spec_augmentation = None + + # Setup decoding objects + self.decoding = RNNTDecoding( + decoding_cfg=self.cfg.decoding, decoder=self.decoder, joint=self.joint, vocabulary=self.joint.vocabulary, + ) + # Setup WER calculation + self.wer = RNNTWER( + decoding=self.decoding, + batch_dim_index=0, + use_cer=self._cfg.get('use_cer', False), + log_prediction=self._cfg.get('log_prediction', True), + dist_sync_on_step=True, + ) + + # Whether to compute loss during evaluation + if 'compute_eval_loss' in self.cfg: + self.compute_eval_loss = self.cfg.compute_eval_loss + else: + self.compute_eval_loss = True + + # Setup fused Joint step if flag is set + if self.joint.fuse_loss_wer: + self.joint.set_loss(self.loss) + self.joint.set_wer(self.wer) + + self.setup_optim_normalization() + + def setup_optim_normalization(self): + """ + Helper method to setup normalization of certain parts of the model prior to the optimization step. + + Supported pre-optimization normalizations are as follows: + + .. code-block:: yaml + + # Variation Noise injection + model: + variational_noise: + std: 0.0 + start_step: 0 + + # Joint - Length normalization + model: + normalize_joint_txu: false + + # Encoder Network - gradient normalization + model: + normalize_encoder_norm: false + + # Decoder / Prediction Network - gradient normalization + model: + normalize_decoder_norm: false + + # Joint - gradient normalization + model: + normalize_joint_norm: false + """ + # setting up the variational noise for the decoder + if hasattr(self.cfg, 'variational_noise'): + self._optim_variational_noise_std = self.cfg['variational_noise'].get('std', 0) + self._optim_variational_noise_start = self.cfg['variational_noise'].get('start_step', 0) + else: + self._optim_variational_noise_std = 0 + self._optim_variational_noise_start = 0 + + # Setup normalized gradients for model joint by T x U scaling factor (joint length normalization) + self._optim_normalize_joint_txu = self.cfg.get('normalize_joint_txu', False) + self._optim_normalize_txu = None + + # Setup normalized encoder norm for model + self._optim_normalize_encoder_norm = self.cfg.get('normalize_encoder_norm', False) + + # Setup normalized decoder norm for model + self._optim_normalize_decoder_norm = self.cfg.get('normalize_decoder_norm', False) + + # Setup normalized joint norm for model + self._optim_normalize_joint_norm = self.cfg.get('normalize_joint_norm', False) + + def extract_rnnt_loss_cfg(self, cfg: Optional[DictConfig]): + """ + Helper method to extract the rnnt loss name, and potentially its kwargs + to be passed. + + Args: + cfg: Should contain `loss_name` as a string which is resolved to a RNNT loss name. + If the default should be used, then `default` can be used. + Optionally, one can pass additional kwargs to the loss function. The subdict + should have a keyname as follows : `{loss_name}_kwargs`. + + Note that whichever loss_name is selected, that corresponding kwargs will be + selected. For the "default" case, the "{resolved_default}_kwargs" will be used. + + Examples: + .. code-block:: yaml + + loss_name: "default" + warprnnt_numba_kwargs: + kwargs2: some_other_val + + Returns: + A tuple, the resolved loss name as well as its kwargs (if found). + """ + if cfg is None: + cfg = DictConfig({}) + + loss_name = cfg.get("loss_name", "default") + + if loss_name == "default": + loss_name = resolve_rnnt_default_loss_name() + + loss_kwargs = cfg.get(f"{loss_name}_kwargs", None) + + logging.info(f"Using RNNT Loss : {loss_name}\n" f"Loss {loss_name}_kwargs: {loss_kwargs}") + + return loss_name, loss_kwargs + + @torch.no_grad() + def transcribe( + self, paths2audio_files: List[str], batch_size: int = 4, return_hypotheses: bool = False + ) -> List[str]: + """ + Uses greedy decoding to transcribe audio files. Use this method for debugging and prototyping. + + Args: + + paths2audio_files: (a list) of paths to audio files. \ + Recommended length per file is between 5 and 25 seconds. \ + But it is possible to pass a few hours long file if enough GPU memory is available. + batch_size: (int) batch size to use during inference. \ + Bigger will result in better throughput performance but would use more memory. + return_hypotheses: (bool) Either return hypotheses or text + With hypotheses can do some postprocessing like getting timestamp or rescoring + Returns: + + A list of transcriptions in the same order as paths2audio_files + """ + if paths2audio_files is None or len(paths2audio_files) == 0: + return {} + # We will store transcriptions here + hypotheses = [] + # Model's mode and device + mode = self.training + device = next(self.parameters()).device + try: + # Switch model to evaluation mode + self.eval() + # Freeze the encoder and decoder modules + self.encoder.freeze() + self.decoder.freeze() + self.joint.freeze() + logging_level = logging.get_verbosity() + logging.set_verbosity(logging.WARNING) + # Work in tmp directory - will store manifest file there + with tempfile.TemporaryDirectory() as tmpdir: + with open(os.path.join(tmpdir, 'manifest.json'), 'w') as fp: + for audio_file in paths2audio_files: + entry = {'audio_filepath': audio_file, 'duration': 100000, 'text': 'nothing'} + fp.write(json.dumps(entry) + '\n') + + config = {'paths2audio_files': paths2audio_files, 'batch_size': batch_size, 'temp_dir': tmpdir} + + temporary_datalayer = self._setup_transcribe_dataloader(config) + for test_batch in tqdm(temporary_datalayer, desc="Transcribing"): + encoded, encoded_len = self.forward( + input_signal=test_batch[0].to(device), input_signal_length=test_batch[1].to(device) + ) + hypotheses += self.decoding.rnnt_decoder_predictions_tensor( + encoded, encoded_len, return_hypotheses=return_hypotheses + ) + del encoded + del test_batch + finally: + # set mode back to its original value + self.train(mode=mode) + logging.set_verbosity(logging_level) + if mode is True: + self.encoder.unfreeze() + self.decoder.unfreeze() + self.joint.unfreeze() + return hypotheses + + def change_vocabulary(self, new_vocabulary: List[str], decoding_cfg: Optional[DictConfig] = None): + """ + Changes vocabulary used during RNNT decoding process. Use this method when fine-tuning a pre-trained model. + This method changes only decoder and leaves encoder and pre-processing modules unchanged. For example, you would + use it if you want to use pretrained encoder when fine-tuning on data in another language, or when you'd need + model to learn capitalization, punctuation and/or special characters. + + Args: + new_vocabulary: list with new vocabulary. Must contain at least 2 elements. Typically, \ + this is target alphabet. + decoding_cfg: A config for the decoder, which is optional. If the decoding type + needs to be changed (from say Greedy to Beam decoding etc), the config can be passed here. + + Returns: None + + """ + if self.joint.vocabulary == new_vocabulary: + logging.warning(f"Old {self.joint.vocabulary} and new {new_vocabulary} match. Not changing anything.") + else: + if new_vocabulary is None or len(new_vocabulary) == 0: + raise ValueError(f'New vocabulary must be non-empty list of chars. But I got: {new_vocabulary}') + + joint_config = self.joint.to_config_dict() + new_joint_config = copy.deepcopy(joint_config) + new_joint_config['vocabulary'] = new_vocabulary + new_joint_config['num_classes'] = len(new_vocabulary) + del self.joint + self.joint = EncDecRNNTModel.from_config_dict(new_joint_config) + + decoder_config = self.decoder.to_config_dict() + new_decoder_config = copy.deepcopy(decoder_config) + new_decoder_config.vocab_size = len(new_vocabulary) + del self.decoder + self.decoder = EncDecRNNTModel.from_config_dict(new_decoder_config) + + del self.loss + loss_name, loss_kwargs = self.extract_rnnt_loss_cfg(self.cfg.get('loss', None)) + self.loss = RNNTLoss( + num_classes=self.joint.num_classes_with_blank - 1, loss_name=loss_name, loss_kwargs=loss_kwargs + ) + + if decoding_cfg is None: + # Assume same decoding config as before + decoding_cfg = self.cfg.decoding + + self.decoding = RNNTDecoding( + decoding_cfg=decoding_cfg, decoder=self.decoder, joint=self.joint, vocabulary=self.joint.vocabulary, + ) + + self.wer = RNNTWER( + decoding=self.decoding, + batch_dim_index=self.wer.batch_dim_index, + use_cer=self.wer.use_cer, + log_prediction=self.wer.log_prediction, + dist_sync_on_step=True, + ) + + # Setup fused Joint step + if self.joint.fuse_loss_wer: + self.joint.set_loss(self.loss) + self.joint.set_wer(self.wer) + + # Update config + with open_dict(self.cfg.joint): + self.cfg.joint = new_joint_config + + with open_dict(self.cfg.decoder): + self.cfg.decoder = new_decoder_config + + with open_dict(self.cfg.decoding): + self.cfg.decoding = decoding_cfg + + ds_keys = ['train_ds', 'validation_ds', 'test_ds'] + for key in ds_keys: + if key in self.cfg: + with open_dict(self.cfg[key]): + self.cfg[key]['labels'] = OmegaConf.create(new_vocabulary) + + logging.info(f"Changed decoder to output to {self.joint.vocabulary} vocabulary.") + + def change_decoding_strategy(self, decoding_cfg: DictConfig): + """ + Changes decoding strategy used during RNNT decoding process. + + Args: + decoding_cfg: A config for the decoder, which is optional. If the decoding type + needs to be changed (from say Greedy to Beam decoding etc), the config can be passed here. + """ + if decoding_cfg is None: + # Assume same decoding config as before + logging.info("No `decoding_cfg` passed when changing decoding strategy, using internal config") + decoding_cfg = self.cfg.decoding + + self.decoding = RNNTDecoding( + decoding_cfg=decoding_cfg, decoder=self.decoder, joint=self.joint, vocabulary=self.joint.vocabulary, + ) + + self.wer = RNNTWER( + decoding=self.decoding, + batch_dim_index=self.wer.batch_dim_index, + use_cer=self.wer.use_cer, + log_prediction=self.wer.log_prediction, + dist_sync_on_step=True, + ) + + # Setup fused Joint step + if self.joint.fuse_loss_wer: + self.joint.set_loss(self.loss) + self.joint.set_wer(self.wer) + + # Update config + with open_dict(self.cfg.decoding): + self.cfg.decoding = decoding_cfg + + logging.info(f"Changed decoding strategy to \n{OmegaConf.to_yaml(self.cfg.decoding)}") + + def _setup_dataloader_from_config(self, config: Optional[Dict]): + if 'augmentor' in config: + augmentor = process_augmentations(config['augmentor']) + else: + augmentor = None + + # Automatically inject args from model config to dataloader config + audio_to_text_dataset.inject_dataloader_value_from_model_config(self.cfg, config, key='sample_rate') + audio_to_text_dataset.inject_dataloader_value_from_model_config(self.cfg, config, key='labels') + + shuffle = config['shuffle'] + device = 'gpu' if torch.cuda.is_available() else 'cpu' + if config.get('use_dali', False): + device_id = self.local_rank if device == 'gpu' else None + dataset = audio_to_text_dataset.get_dali_char_dataset( + config=config, + shuffle=shuffle, + device_id=device_id, + global_rank=self.global_rank, + world_size=self.world_size, + preprocessor_cfg=self._cfg.preprocessor, + ) + return dataset + + # Instantiate tarred dataset loader or normal dataset loader + if config.get('is_tarred', False): + if ('tarred_audio_filepaths' in config and config['tarred_audio_filepaths'] is None) or ( + 'manifest_filepath' in config and config['manifest_filepath'] is None + ): + logging.warning( + "Could not load dataset as `manifest_filepath` was None or " + f"`tarred_audio_filepaths` is None. Provided config : {config}" + ) + return None + + shuffle_n = config.get('shuffle_n', 4 * config['batch_size']) if shuffle else 0 + dataset = audio_to_text_dataset.get_tarred_char_dataset( + config=config, + shuffle_n=shuffle_n, + global_rank=self.global_rank, + world_size=self.world_size, + augmentor=augmentor, + ) + shuffle = False + else: + if 'manifest_filepath' in config and config['manifest_filepath'] is None: + logging.warning(f"Could not load dataset as `manifest_filepath` was None. Provided config : {config}") + return None + + dataset = audio_to_text_dataset.get_char_dataset(config=config, augmentor=augmentor) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=config['batch_size'], + collate_fn=dataset.collate_fn, + drop_last=config.get('drop_last', False), + shuffle=shuffle, + num_workers=config.get('num_workers', 0), + pin_memory=config.get('pin_memory', False), + ) + + def setup_training_data(self, train_data_config: Optional[Union[DictConfig, Dict]]): + """ + Sets up the training data loader via a Dict-like object. + + Args: + train_data_config: A config that contains the information regarding construction + of an ASR Training dataset. + + Supported Datasets: + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text_dali.AudioToCharDALIDataset` + """ + if 'shuffle' not in train_data_config: + train_data_config['shuffle'] = True + + # preserve config + self._update_dataset_config(dataset_name='train', config=train_data_config) + + self._train_dl = self._setup_dataloader_from_config(config=train_data_config) + + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'is_tarred' in train_data_config and train_data_config['is_tarred']: + # We also need to check if limit_train_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # training batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if isinstance(self._trainer.limit_train_batches, float): + self._trainer.limit_train_batches = int( + self._trainer.limit_train_batches + * ceil((len(self._train_dl.dataset) / self.world_size) / train_data_config['batch_size']) + ) + + def setup_validation_data(self, val_data_config: Optional[Union[DictConfig, Dict]]): + """ + Sets up the validation data loader via a Dict-like object. + + Args: + val_data_config: A config that contains the information regarding construction + of an ASR Training dataset. + + Supported Datasets: + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text_dali.AudioToCharDALIDataset` + """ + if 'shuffle' not in val_data_config: + val_data_config['shuffle'] = False + + # preserve config + self._update_dataset_config(dataset_name='validation', config=val_data_config) + + self._validation_dl = self._setup_dataloader_from_config(config=val_data_config) + + def setup_test_data(self, test_data_config: Optional[Union[DictConfig, Dict]]): + """ + Sets up the test data loader via a Dict-like object. + + Args: + test_data_config: A config that contains the information regarding construction + of an ASR Training dataset. + + Supported Datasets: + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.AudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToCharDataset` + - :class:`~nemo.collections.asr.data.audio_to_text.TarredAudioToBPEDataset` + - :class:`~nemo.collections.asr.data.audio_to_text_dali.AudioToCharDALIDataset` + """ + if 'shuffle' not in test_data_config: + test_data_config['shuffle'] = False + + # preserve config + self._update_dataset_config(dataset_name='test', config=test_data_config) + + self._test_dl = self._setup_dataloader_from_config(config=test_data_config) + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + if hasattr(self.preprocessor, '_sample_rate'): + input_signal_eltype = AudioSignal(freq=self.preprocessor._sample_rate) + else: + input_signal_eltype = AudioSignal() + + return { + "input_signal": NeuralType(('B', 'T'), input_signal_eltype, optional=True), + "input_signal_length": NeuralType(tuple('B'), LengthsType(), optional=True), + "processed_signal": NeuralType(('B', 'D', 'T'), SpectrogramType(), optional=True), + "processed_signal_length": NeuralType(tuple('B'), LengthsType(), optional=True), + } + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "outputs": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + "encoded_lengths": NeuralType(tuple('B'), LengthsType()), + } + + @typecheck() + def forward( + self, input_signal=None, input_signal_length=None, processed_signal=None, processed_signal_length=None + ): + """ + Forward pass of the model. Note that for RNNT Models, the forward pass of the model is a 3 step process, + and this method only performs the first step - forward of the acoustic model. + + Please refer to the `training_step` in order to see the full `forward` step for training - which + performs the forward of the acoustic model, the prediction network and then the joint network. + Finally, it computes the loss and possibly compute the detokenized text via the `decoding` step. + + Please refer to the `validation_step` in order to see the full `forward` step for inference - which + performs the forward of the acoustic model, the prediction network and then the joint network. + Finally, it computes the decoded tokens via the `decoding` step and possibly compute the batch metrics. + + Args: + input_signal: Tensor that represents a batch of raw audio signals, + of shape [B, T]. T here represents timesteps, with 1 second of audio represented as + `self.sample_rate` number of floating point values. + input_signal_length: Vector of length B, that contains the individual lengths of the audio + sequences. + processed_signal: Tensor that represents a batch of processed audio signals, + of shape (B, D, T) that has undergone processing via some DALI preprocessor. + processed_signal_length: Vector of length B, that contains the individual lengths of the + processed audio sequences. + + Returns: + A tuple of 2 elements - + 1) The log probabilities tensor of shape [B, T, D]. + 2) The lengths of the acoustic sequence after propagation through the encoder, of shape [B]. + """ + has_input_signal = input_signal is not None and input_signal_length is not None + has_processed_signal = processed_signal is not None and processed_signal_length is not None + if (has_input_signal ^ has_processed_signal) is False: + raise ValueError( + f"{self} Arguments ``input_signal`` and ``input_signal_length`` are mutually exclusive " + " with ``processed_signal`` and ``processed_signal_len`` arguments." + ) + + if not has_processed_signal: + processed_signal, processed_signal_length = self.preprocessor( + input_signal=input_signal, length=input_signal_length, + ) + + # Spec augment is not applied during evaluation/testing + if self.spec_augmentation is not None and self.training: + processed_signal = self.spec_augmentation(input_spec=processed_signal, length=processed_signal_length) + + encoded, encoded_len = self.encoder(audio_signal=processed_signal, length=processed_signal_length) + return encoded, encoded_len + + # PTL-specific methods + def training_step(self, batch, batch_nb): + signal, signal_len, transcript, transcript_len = batch + + # forward() only performs encoder forward + if isinstance(batch, DALIOutputs) and batch.has_processed_signal: + encoded, encoded_len = self.forward(processed_signal=signal, processed_signal_length=signal_len) + else: + encoded, encoded_len = self.forward(input_signal=signal, input_signal_length=signal_len) + del signal + + # During training, loss must be computed, so decoder forward is necessary + decoder, target_length = self.decoder(targets=transcript, target_length=transcript_len) + + if hasattr(self, '_trainer') and self._trainer is not None: + log_every_n_steps = self._trainer.log_every_n_steps + sample_id = self._trainer.global_step + + else: + log_every_n_steps = 1 + sample_id = batch_nb + + # If experimental fused Joint-Loss-WER is not used + if not self.joint.fuse_loss_wer: + # Compute full joint and loss + joint = self.joint(encoder_outputs=encoded, decoder_outputs=decoder) + loss_value = self.loss( + log_probs=joint, targets=transcript, input_lengths=encoded_len, target_lengths=target_length + ) + + tensorboard_logs = {'train_loss': loss_value, 'learning_rate': self._optimizer.param_groups[0]['lr']} + + if (sample_id + 1) % log_every_n_steps == 0: + self.wer.update(encoded, encoded_len, transcript, transcript_len) + _, scores, words = self.wer.compute() + self.wer.reset() + tensorboard_logs.update({'training_batch_wer': scores.float() / words}) + + else: + # If experimental fused Joint-Loss-WER is used + if (sample_id + 1) % log_every_n_steps == 0: + compute_wer = True + else: + compute_wer = False + + # Fused joint step + loss_value, wer, _, _ = self.joint( + encoder_outputs=encoded, + decoder_outputs=decoder, + encoder_lengths=encoded_len, + transcripts=transcript, + transcript_lengths=transcript_len, + compute_wer=compute_wer, + ) + + tensorboard_logs = {'train_loss': loss_value, 'learning_rate': self._optimizer.param_groups[0]['lr']} + + if compute_wer: + tensorboard_logs.update({'training_batch_wer': wer}) + + # Log items + self.log_dict(tensorboard_logs) + + # Preserve batch acoustic model T and language model U parameters if normalizing + if self._optim_normalize_joint_txu: + self._optim_normalize_txu = [encoded_len.max(), transcript_len.max()] + + return {'loss': loss_value} + + def validation_step(self, batch, batch_idx, dataloader_idx=0): + signal, signal_len, transcript, transcript_len = batch + + # forward() only performs encoder forward + if isinstance(batch, DALIOutputs) and batch.has_processed_signal: + encoded, encoded_len = self.forward(processed_signal=signal, processed_signal_length=signal_len) + else: + encoded, encoded_len = self.forward(input_signal=signal, input_signal_length=signal_len) + del signal + + tensorboard_logs = {} + + # If experimental fused Joint-Loss-WER is not used + if not self.joint.fuse_loss_wer: + if self.compute_eval_loss: + decoder, target_length = self.decoder(targets=transcript, target_length=transcript_len) + joint = self.joint(encoder_outputs=encoded, decoder_outputs=decoder) + + loss_value = self.loss( + log_probs=joint, targets=transcript, input_lengths=encoded_len, target_lengths=target_length + ) + + tensorboard_logs['val_loss'] = loss_value + + self.wer.update(encoded, encoded_len, transcript, transcript_len) + wer, wer_num, wer_denom = self.wer.compute() + self.wer.reset() + + tensorboard_logs['val_wer_num'] = wer_num + tensorboard_logs['val_wer_denom'] = wer_denom + tensorboard_logs['val_wer'] = wer + + else: + # If experimental fused Joint-Loss-WER is used + compute_wer = True + + if self.compute_eval_loss: + decoded, target_len = self.decoder(targets=transcript, target_length=transcript_len) + else: + decoded = None + target_len = transcript_len + + # Fused joint step + loss_value, wer, wer_num, wer_denom = self.joint( + encoder_outputs=encoded, + decoder_outputs=decoded, + encoder_lengths=encoded_len, + transcripts=transcript, + transcript_lengths=target_len, + compute_wer=compute_wer, + ) + + if loss_value is not None: + tensorboard_logs['val_loss'] = loss_value + + tensorboard_logs['val_wer_num'] = wer_num + tensorboard_logs['val_wer_denom'] = wer_denom + tensorboard_logs['val_wer'] = wer + + return tensorboard_logs + + def test_step(self, batch, batch_idx, dataloader_idx=0): + logs = self.validation_step(batch, batch_idx, dataloader_idx=dataloader_idx) + test_logs = { + 'test_wer_num': logs['val_wer_num'], + 'test_wer_denom': logs['val_wer_denom'], + # 'test_wer': logs['val_wer'], + } + if 'val_loss' in logs: + test_logs['test_loss'] = logs['val_loss'] + return test_logs + + def multi_validation_epoch_end(self, outputs, dataloader_idx: int = 0): + if self.compute_eval_loss: + val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean() + val_loss_log = {'val_loss': val_loss_mean} + else: + val_loss_log = {} + wer_num = torch.stack([x['val_wer_num'] for x in outputs]).sum() + wer_denom = torch.stack([x['val_wer_denom'] for x in outputs]).sum() + tensorboard_logs = {**val_loss_log, 'val_wer': wer_num.float() / wer_denom} + return {**val_loss_log, 'log': tensorboard_logs} + + def multi_test_epoch_end(self, outputs, dataloader_idx: int = 0): + if self.compute_eval_loss: + test_loss_mean = torch.stack([x['test_loss'] for x in outputs]).mean() + test_loss_log = {'test_loss': test_loss_mean} + else: + test_loss_log = {} + wer_num = torch.stack([x['test_wer_num'] for x in outputs]).sum() + wer_denom = torch.stack([x['test_wer_denom'] for x in outputs]).sum() + tensorboard_logs = {**test_loss_log, 'test_wer': wer_num.float() / wer_denom} + return {**test_loss_log, 'log': tensorboard_logs} + + def _setup_transcribe_dataloader(self, config: Dict) -> 'torch.utils.data.DataLoader': + """ + Setup function for a temporary data loader which wraps the provided audio file. + + Args: + config: A python dictionary which contains the following keys: + paths2audio_files: (a list) of paths to audio files. The files should be relatively short fragments. \ + Recommended length per file is between 5 and 25 seconds. + batch_size: (int) batch size to use during inference. \ + Bigger will result in better throughput performance but would use more memory. + temp_dir: (str) A temporary directory where the audio manifest is temporarily + stored. + + Returns: + A pytorch DataLoader for the given audio file(s). + """ + dl_config = { + 'manifest_filepath': os.path.join(config['temp_dir'], 'manifest.json'), + 'sample_rate': self.preprocessor._sample_rate, + 'labels': self.joint.vocabulary, + 'batch_size': min(config['batch_size'], len(config['paths2audio_files'])), + 'trim_silence': True, + 'shuffle': False, + } + + temporary_datalayer = self._setup_dataloader_from_config(config=DictConfig(dl_config)) + return temporary_datalayer + + def on_after_backward(self): + super().on_after_backward() + if self._optim_variational_noise_std > 0 and self.global_step >= self._optim_variational_noise_start: + for param_name, param in self.decoder.named_parameters(): + if param.grad is not None: + noise = torch.normal( + mean=0.0, + std=self._optim_variational_noise_std, + size=param.size(), + device=param.device, + dtype=param.dtype, + ) + param.grad.data.add_(noise) + + if self._optim_normalize_joint_txu: + T, U = self._optim_normalize_txu + if T is not None and U is not None: + for param_name, param in self.encoder.named_parameters(): + if param.grad is not None: + param.grad.data.div_(U) + + for param_name, param in self.decoder.named_parameters(): + if param.grad is not None: + param.grad.data.div_(T) + + if self._optim_normalize_encoder_norm: + for param_name, param in self.encoder.named_parameters(): + if param.grad is not None: + norm = param.grad.norm() + param.grad.data.div_(norm) + + if self._optim_normalize_decoder_norm: + for param_name, param in self.decoder.named_parameters(): + if param.grad is not None: + norm = param.grad.norm() + param.grad.data.div_(norm) + + if self._optim_normalize_joint_norm: + for param_name, param in self.joint.named_parameters(): + if param.grad is not None: + norm = param.grad.norm() + param.grad.data.div_(norm) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/__init__.py new file mode 100644 index 0000000000..0aa406b8d7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nemo.collections.asr.models.wav2vec.wav2vec_model import Wav2VecEncoderModel diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/wav2vec_config.py b/models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/wav2vec_config.py new file mode 100644 index 0000000000..8691df4f48 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/wav2vec_config.py @@ -0,0 +1,180 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from dataclasses import dataclass, field +from enum import Enum, auto +from typing import Any, Dict, List, Optional + + +class Wav2VecActivationType(Enum): + relu = 'relu' + gelu = 'gelu' + + +class Wav2VecMaskType(Enum): + """ + Used to select configuration to compute mask lengths + static = fixed size + uniform = sample from uniform distribution [mask_other, mask_length*2] + normal = sample from normal distribution with mean mask_length and stdev mask_other. mask is min 1 element + poisson = sample from possion distribution with lambda = mask length + """ + + static = auto() + uniform = auto() + normal = auto() + poisson = auto() + + +class Wav2VecConvExtractorMode(Enum): + """ + Mode for feature extractor. default has a single group norm with d groups in the first conv block, + whereas layer_norm has layer norms in every block. + """ + + default = auto() + layer_norm = auto() + + +@dataclass +class ConvConfig: + conv_pos: int = field(default=128, metadata={'help': 'Number of filters for convolutional positional embeddings'}) + conv_pos_groups: int = field( + default=16, metadata={'help': 'Number of groups for convolutional positional embeddings'} + ) + + +@dataclass +class Wav2VecTransformerEncoderConfig: + encoder_layers: int = field(default=12, metadata={'help': 'Number of encoder layers in transformer model'}) + encoder_layerdrop: float = field(default=0.05, metadata={'help': 'Probability of dropping transformer layers'}) + embedding_dim: int = field(default=768, metadata={'help': 'Encoder embedding dim'}) + ffn_embedding_dim: int = field(default=3072, metadata={'help': 'Encoder embedding dim for feed forward'}) + num_attention_heads: int = field(default=8, metadata={'help': 'Number of encoder attention heads'}) + dropout: float = field(default=0.1, metadata={'help': 'Dropout probability for transformer encoder'}) + activation_fn: Wav2VecActivationType = field( + default=Wav2VecActivationType.gelu, metadata={'help': 'Activation for transformer'} + ) + layer_norm_first: bool = field(default=False, metadata={'help': 'Apply layer norm first within the transformer'}) + + +@dataclass +class Wav2VecTransformerConfig: + dropout: float = field(default=0.1, metadata={'help': 'Dropout probability for the transformer'}) + conv: ConvConfig = ConvConfig() + encoder: Wav2VecTransformerEncoderConfig = Wav2VecTransformerEncoderConfig() + + +@dataclass +class QuantizerConfig: + quantize_targets: bool = field(default=True, metadata={'help': 'Use quantized targets'}) + quantize_input: bool = field(default=False, metadata={'help': 'Use quantized inputs'}) + same_quantizer: bool = field(default=False, metadata={'help': 'Use the same quantizer for inputs and targets'}) + latent_vars: int = field( + default=320, metadata={'help': 'Number of latent variables in each group of the codebook'} + ) + latent_groups: int = field(default=2, metadata={'help': 'Number of groups within the codebook'}) + latent_dim: int = field( + default=0, + metadata={ + 'help': 'If greater than 0, use dim for latent variables, else infered by final_dim / latent_groups' + }, + ) + latent_temp: tuple = field( + default=(2, 0.5, 0.999995), metadata={'help': 'Quantize temperature (start, stop, decay factor)'} + ) + + +@dataclass +class ConvFeatureEncoderConfig: + extractor_mode: Wav2VecConvExtractorMode = field(default=Wav2VecConvExtractorMode.default) + conv_bias: bool = field(default=False, metadata={'help': 'Include bias in convolution feature extractor model'}) + conv_feature_layers: List = field( + default_factory=lambda: [(512, 10, 5)] + [(512, 3, 2)] * 4 + [(512, 2, 2)] + [(512, 2, 2)], + metadata={'help': 'convolutional feature extraction layers [(dim, kernel_size, stride), ...'}, + ) + + +@dataclass +class LossConfig: + prob_ppl_weight: float = field(default=0.1, metadata={'help': 'Weight applied to quantized prob perplexity loss'}) + feature_loss_weight: float = field(default=0, metadata={'help': 'Weight applied to feature L2 Norm'}) + + +@dataclass +class Wav2VecMaskingConfig: + mask_prob: float = field(default=0.65, metadata={'help': 'Probability of replacing token with mask'}) + mask_type: Wav2VecMaskType = field(default=Wav2VecMaskType.static,) + mask_other: int = field( + default=0, + metadata={'help': 'Secondary mask used for complex distributions (see help in compute_mask_indices)'}, + ) + mask_length: int = field(default=10, metadata={'help': 'Length of mask when masking time steps'}) + no_mask_overlap: bool = field(default=False, metadata={'help': 'Whether to allow masks to overlap'}) + mask_min_space: int = field( + default=1, metadata={'help': 'Minimum space beetween spans (if no overlap is enabled)'} + ) + mask_channel_prob: float = field(default=0, metadata={'help': 'Probability of replacing a feature with 0'}) + mask_channel_type: Wav2VecMaskType = field(default=Wav2VecMaskType.static,) + mask_channel_other: int = field( + default=0, + metadata={ + 'help': 'Secondary mask argument (used for more complex distributions (see help in compute_mask_indices)' + }, + ) + mask_channel_length: int = field(default=10, metadata={'help': 'Length of masks for features (channels)'}) + no_mask_channel_overlap: bool = field( + default=False, metadata={'help': 'Whether to allow channel masks to overlap'} + ) + mask_channel_min_space: int = field( + default=1, metadata={'help': 'Minimum space between spans (if no overlap is enabled)'} + ) + + +@dataclass +class Wav2VecEncoderModelConfig: + loss: LossConfig = LossConfig() + quantizer: QuantizerConfig = QuantizerConfig() + conv_feature_encoder: ConvFeatureEncoderConfig = ConvFeatureEncoderConfig() + transformer_encoder: Wav2VecTransformerConfig = Wav2VecTransformerConfig() + masking: Wav2VecMaskingConfig = Wav2VecMaskingConfig() + + dropout_input: float = field(default=0.1, metadata={'help': 'Dropout applied to input raw features'}) + dropout_features: float = field( + default=0.1, metadata={'help': 'Dropout applied to the features generator by convolutions'} + ) + final_dim: int = field(default=0, metadata={'help': 'Project final representations and targets to this dimension'}) + n_negatives: int = field( + default=100, metadata={'help': 'Number of negatives to sample from the same audio sample'} + ) + cross_sample_negatives: int = field( + default=0, metadata={'help': 'Number of negatives to sample from any sample in the batch'} + ) + codebook_negatives: int = field(default=0, metadata={'help': 'Number of negative examples in codebook'}) + negatives_from_everywhere: bool = field( + default=False, metadata={'help': 'Sample negatives from everywhere, not just masked states'} + ) + logit_temp: float = field(default=0.1, metadata={'help': 'Temperature to divide logits by'}) + target_glu: bool = field(default=False, metadata={'help': 'Adds project and applies GLU to targets'}) + feature_grad_mult: float = field(default=0.1, metadata={'help': 'Multiply extracted feature gradients'}) + + train_ds: Optional[Dict[Any, Any]] = None + validation_ds: Optional[Dict[Any, Any]] = None + test_ds: Optional[Dict[Any, Any]] = None + optim: Optional[Dict[Any, Any]] = None diff --git a/models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/wav2vec_model.py b/models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/wav2vec_model.py new file mode 100644 index 0000000000..dd33c2e017 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/models/wav2vec/wav2vec_model.py @@ -0,0 +1,504 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# Copyright (c) Facebook, Inc. and its affiliates. +# +import logging +from math import ceil +from typing import Dict, Optional, Union + +import torch +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer +from torch import nn + +from nemo.collections.asr.data import audio_to_text_dataset +from nemo.collections.asr.losses.wav2vecloss import Wav2VecLoss +from nemo.collections.asr.models.wav2vec.wav2vec_config import Wav2VecEncoderModelConfig +from nemo.collections.asr.modules.wav2vec_modules import GumbelVectorQuantizer, compute_mask_indices +from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations +from nemo.collections.asr.parts.submodules.wav2vec import ConvFeatureEncoder, GradMultiply, Wav2VecTransformerEncoder +from nemo.core import ModelPT +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types import AudioSignal, EncodedRepresentation, LossType, MaskType, NeuralType +from nemo.core.neural_types.elements import BoolType, FloatType + + +def buffered_arange(max): + if not hasattr(buffered_arange, "buf"): + buffered_arange.buf = torch.LongTensor() + if max > buffered_arange.buf.numel(): + buffered_arange.buf.resize_(max) + torch.arange(max, out=buffered_arange.buf) + return buffered_arange.buf[:max] + + +class Wav2VecEncoderModel(ModelPT): + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + # Get global rank and total number of GPU workers for IterableDataset partitioning, if applicable + self.global_rank = 0 + self.world_size = 0 + self.local_rank = 0 + if trainer is not None: + self.global_rank = (trainer.node_rank * trainer.num_gpus) + trainer.local_rank + self.world_size = trainer.num_nodes * trainer.num_gpus + self.local_rank = trainer.local_rank + + super().__init__(cfg=cfg, trainer=trainer) + + schema = OmegaConf.structured(Wav2VecEncoderModelConfig) + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + + cfg = OmegaConf.create(OmegaConf.to_container(cfg, resolve=True)) + cfg = OmegaConf.merge(schema, cfg) + + feature_enc_layers = cfg.conv_feature_encoder.conv_feature_layers + self.embed = feature_enc_layers[-1][0] # Select last conv output layer dimension + + self.feature_extractor = ConvFeatureEncoder( + conv_layers=feature_enc_layers, + mode=cfg.conv_feature_encoder.extractor_mode, + conv_bias=cfg.conv_feature_encoder.conv_bias, + ) + + encoder_embed_dim = cfg.transformer_encoder.encoder.embedding_dim + self.post_extract_proj = ( + nn.Linear(self.embed, encoder_embed_dim) + if self.embed != encoder_embed_dim and not cfg.quantizer.quantize_input + else None + ) + + self.mask_cfg = cfg.masking + + self.dropout_input = nn.Dropout(cfg.dropout_input) + self.dropout_features = nn.Dropout(cfg.dropout_features) + + self.feature_grad_mult = cfg.feature_grad_mult + + self.quantizer = None + self.input_quantizer = None + + self.n_negatives = cfg.n_negatives + self.cross_sample_negatives = cfg.cross_sample_negatives + self.codebook_negatives = cfg.codebook_negatives + self.negatives_from_everywhere = cfg.negatives_from_everywhere + + final_dim = cfg.final_dim if cfg.final_dim > 0 else encoder_embed_dim + self.final_dim = final_dim + if cfg.quantizer.quantize_targets: + vq_dim = cfg.quantizer.latent_dim if cfg.quantizer.latent_dim > 0 else final_dim + self.quantizer = GumbelVectorQuantizer( + dim=self.embed, + num_vars=cfg.quantizer.latent_vars, + temp=cfg.quantizer.latent_temp, + groups=cfg.quantizer.latent_groups, + combine_groups=False, + vq_dim=vq_dim, + time_first=True, + ) + self.project_q = nn.Linear(vq_dim, final_dim) + else: + self.project_q = nn.Linear(self.embed, final_dim) + + if cfg.quantizer.quantize_input: + if cfg.quantizer.same_quantizer and self.quantizer is not None: + vq_dim = final_dim + self.input_quantizer = self.quantizer + else: + vq_dim = cfg.quantizer.latent_dim if cfg.quantizer.latent_dim > 0 else encoder_embed_dim + self.input_quantizer = GumbelVectorQuantizer( + dim=self.embed, + num_vars=cfg.quantizer.latent_vars, + temp=cfg.quantizer.latent_temp, + groups=cfg.quantizer.latent_groups, + combine_groups=False, + vq_dim=vq_dim, + time_first=True, + ) + self.project_inp = nn.Linear(vq_dim, encoder_embed_dim) + + self.mask_emb = nn.Parameter(torch.FloatTensor(encoder_embed_dim).uniform_()) + + self.encoder = Wav2VecTransformerEncoder(cfg.transformer_encoder) + self.layer_norm = nn.LayerNorm(self.embed) + + self.target_glu = None + if cfg.target_glu: + self.target_glu = nn.Sequential(nn.Linear(final_dim, final_dim * 2), nn.GLU()) + + self.final_proj = nn.Linear(encoder_embed_dim, final_dim) + self.loss = Wav2VecLoss( + feature_loss_weight=cfg.loss.feature_loss_weight, + prob_ppl_weight=cfg.loss.prob_ppl_weight, + logit_temp=cfg.logit_temp, + ) + + def training_step(self, batch, batch_idx): + loss, feature_loss, prob_ppl_loss = self._step(batch) + + self.log('learning_rate', self._optimizer.param_groups[0]['lr']) + self.log('loss', loss, on_epoch=True) + self.log('feature_loss', feature_loss, on_epoch=True, sync_dist=True) + self.log('prob_ppl_loss', prob_ppl_loss, on_epoch=True, sync_dist=True) + return {'loss': loss} + + def validation_step(self, batch, batch_idx, dataloader_idx=0): + loss, feature_loss, prob_ppl_loss = self._step(batch) + self.log('val_loss', loss, prog_bar=True, on_epoch=True, sync_dist=True) + + def test_step(self, batch, batch_idx, dataloader_idx=0): + loss, feature_loss, prob_ppl_loss = self._step(batch) + self.log('test_loss', loss, prog_bar=True, on_epoch=True, sync_dist=True) + + def _step(self, batch): + audio_signal, audio_lengths, _, _ = batch + + padding_mask = self._create_padding_mask(audio_lengths) + + self._update_quantizer_temp() + logits, targets, sampled_negatives, _, features_penalty, prob_ppl_loss, _ = self( + source=audio_signal, padding_mask=padding_mask + ) + loss, feature_loss, prob_ppl_loss = self.loss( + logits=logits, + targets=targets, + negatives=sampled_negatives, + prob_ppl_loss=prob_ppl_loss, + feature_loss=features_penalty, + ) + return loss, feature_loss, prob_ppl_loss + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + return None + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "source": NeuralType(('B', 'T'), AudioSignal()), + "padding_mask": NeuralType(('B', 'T'), MaskType(), optional=True), + "mask": NeuralType(elements_type=BoolType(), optional=True), + "features_only": NeuralType(elements_type=BoolType(), optional=True), + } + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "logits": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "targets": NeuralType(('B', 'T', 'D'), EncodedRepresentation(), optional=True), + "sampled_negatives": NeuralType(('N', 'B', 'T', 'D'), EncodedRepresentation(), optional=True), + "padding_mask": NeuralType(('B', 'T'), MaskType(), optional=True), + "features_penalty": NeuralType(elements_type=LossType(), optional=True), + "prob_ppl_loss": NeuralType(elements_type=LossType(), optional=True), + "cur_codebook_temp": NeuralType(elements_type=FloatType(), optional=True), + } + + @typecheck() + def forward(self, source, padding_mask=None, mask=True, features_only=False) -> tuple: + prob_ppl_loss, cur_codebook_temp = None, None + + if self.feature_grad_mult > 0: + features = self.feature_extractor(source) + if self.feature_grad_mult != 1.0: + features = GradMultiply.apply(features, self.feature_grad_mult) + else: + with torch.no_grad(): + features = self.feature_extractor(source) + + features_penalty = features.float().pow(2).mean() # L2 Norm on features + + features = features.transpose(1, 2) + features = self.layer_norm(features) + unmasked_features = features.clone() + + if padding_mask is not None: + extra = padding_mask.size(1) % features.size(1) + if extra > 0: + padding_mask = padding_mask[:, :-extra] + padding_mask = padding_mask.view(padding_mask.size(0), features.size(1), -1) + padding_mask = padding_mask.all(-1) + + if self.post_extract_proj is not None: + features = self.post_extract_proj(features) + + features = self.dropout_input(features) + unmasked_features = self.dropout_features(unmasked_features) + + if self.input_quantizer: + features, prob_ppl_loss, cur_codebook_temp = self.input_quantizer(features) + features = self.project_inp(features) + if mask: + logits, mask_indices = self.apply_mask(features, padding_mask) + if mask_indices is not None: + targets = unmasked_features[mask_indices].view( + unmasked_features.size(0), -1, unmasked_features.size(-1) + ) + else: + targets = unmasked_features + else: + logits = features + targets = unmasked_features + mask_indices = None + + logits = self.encoder(logits, padding_mask=padding_mask) + + if features_only: + return logits, padding_mask + + if self.quantizer: + targets, prob_ppl_loss, cur_codebook_temp = self.quantizer(targets) + targets = self.project_q(targets) + + if self.negatives_from_everywhere: + neg_cands, *_ = self.quantizer(unmasked_features) + sampled_negatives, _ = self.sample_negatives(neg_cands, targets.size(1)) + sampled_negatives = self.project_q(sampled_negatives) + else: + sampled_negatives, _ = self.sample_negatives(targets, targets.size(1)) + + if self.codebook_negatives > 0: + cb_negs = self.quantizer.sample_from_codebook( + targets.size(0) * targets.size(1), self.codebook_negatives + ) + cb_negs = cb_negs.view( + self.codebook_negatives, targets.size(0), targets.size(1), -1 + ) # order doesnt matter + cb_negs = self.project_q(cb_negs) + sampled_negatives = torch.cat([sampled_negatives, cb_negs], dim=0) + else: + targets = self.project_q(targets) + + if self.negatives_from_everywhere: + sampled_negatives, _ = self.sample_negatives(unmasked_features, targets.size(1)) + sampled_negatives = self.project_q(sampled_negatives) + else: + sampled_negatives, _ = self.sample_negatives(targets, targets.size(1)) + + logits = logits[mask_indices].view(logits.size(0), -1, logits.size(-1)) + + if self.target_glu: + targets = self.target_glu(targets) + sampled_negatives = self.target_glu(sampled_negatives) + + logits = self.final_proj(logits) + + return logits, targets, sampled_negatives, padding_mask, features_penalty, prob_ppl_loss, cur_codebook_temp + + def extract_features(self, source, audio_lengths, mask=False): + padding_mask = self._create_padding_mask(audio_lengths) + return self(source=source, padding_mask=padding_mask, mask=mask, features_only=True) + + def remove_pretraining_modules(self): + self.quantizer = None + self.project_q = None + self.target_glu = None + self.final_proj = None + + def _update_quantizer_temp(self): + if self.quantizer: + self.quantizer.set_num_updates(self.trainer.global_step) + if self.input_quantizer: + self.input_quantizer.set_num_updates(self.trainer.global_step) + + def apply_mask(self, x, padding_mask): + B, T, C = x.shape + if self.mask_cfg.mask_prob > 0: + mask_indices = compute_mask_indices( + (B, T), + padding_mask, + self.mask_cfg.mask_prob, + self.mask_cfg.mask_length, + self.mask_cfg.mask_type, + self.mask_cfg.mask_other, + min_masks=2, + no_overlap=self.mask_cfg.no_mask_overlap, + min_space=self.mask_cfg.mask_min_space, + ) + mask_indices = torch.from_numpy(mask_indices).to(x.device) + mask_emb = self.mask_emb.type_as(x) + x[mask_indices] = mask_emb + else: + mask_indices = None + + if self.mask_cfg.mask_channel_prob > 0: + mask_channel_indices = compute_mask_indices( + (B, C), + None, + self.mask_cfg.mask_channel_prob, + self.mask_cfg.mask_channel_length, + self.mask_cfg.mask_channel_type, + self.mask_cfg.mask_channel_other, + no_overlap=self.mask_cfg.no_mask_channel_overlap, + min_space=self.mask_cfg.mask_channel_min_space, + ) + mask_channel_indices = torch.from_numpy(mask_channel_indices).to(x.device).unsqueeze(1).expand(-1, T, -1) + x[mask_channel_indices] = 0 + + return x, mask_indices + + def sample_negatives(self, y, num): + + if self.n_negatives == 0 and self.cross_sample_negatives == 0: + return y.new(0) + + bsz, tsz, fsz = y.shape + y = y.view(-1, fsz) # BTC => (BxT)C + + cross_high = tsz * bsz + high = tsz + with torch.no_grad(): + assert high > 1, f"{bsz, tsz, fsz}" + + if self.n_negatives > 0: + tszs = buffered_arange(num).unsqueeze(-1).expand(-1, self.n_negatives).flatten() + + neg_idxs = torch.randint(low=0, high=high - 1, size=(bsz, self.n_negatives * num)) + neg_idxs[neg_idxs >= tszs] += 1 + + if self.cross_sample_negatives > 0: + tszs = buffered_arange(num).unsqueeze(-1).expand(-1, self.cross_sample_negatives).flatten() + + cross_neg_idxs = torch.randint( + low=0, high=cross_high - 1, size=(bsz, self.cross_sample_negatives * num), + ) + cross_neg_idxs[cross_neg_idxs >= tszs] += 1 + + if self.n_negatives > 0: + for i in range(1, bsz): + neg_idxs[i] += i * high + else: + neg_idxs = cross_neg_idxs + + if self.cross_sample_negatives > 0 and self.n_negatives > 0: + neg_idxs = torch.cat([neg_idxs, cross_neg_idxs], dim=1) + + negs = y[neg_idxs.view(-1)] + negs = negs.view(bsz, num, self.n_negatives + self.cross_sample_negatives, fsz).permute( + 2, 0, 1, 3 + ) # to NxBxTxC + return negs, neg_idxs + + def _create_padding_mask(self, audio_lengths): + # Broadcast to vectorize creating the padding mask + max_len = max(audio_lengths) + padding_mask = torch.arange(max_len, device=self.device) + padding_mask = padding_mask.expand(len(audio_lengths), max_len) < audio_lengths.unsqueeze(1) + # Negate to false where no padding + padding_mask = ~padding_mask + return padding_mask + + def setup_training_data(self, train_data_config: Optional[Union[DictConfig, Dict]]): + if 'shuffle' not in train_data_config: + train_data_config['shuffle'] = True + + # preserve config + self._update_dataset_config(dataset_name='train', config=train_data_config) + + self._train_dl = self._setup_dataloader_from_config(config=train_data_config) + + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'is_tarred' in train_data_config and train_data_config['is_tarred']: + # We also need to check if limit_train_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # training batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if isinstance(self._trainer.limit_train_batches, float): + self._trainer.limit_train_batches = int( + self._trainer.limit_train_batches + * ceil((len(self._train_dl.dataset) / self.world_size) / train_data_config['batch_size']) + ) + + def setup_validation_data(self, val_data_config: Optional[Union[DictConfig, Dict]]): + if 'shuffle' not in val_data_config: + val_data_config['shuffle'] = False + + # preserve config + self._update_dataset_config(dataset_name='validation', config=val_data_config) + + self._validation_dl = self._setup_dataloader_from_config(config=val_data_config) + + def setup_test_data(self, test_data_config: Optional[Union[DictConfig, Dict]]): + if 'shuffle' not in test_data_config: + test_data_config['shuffle'] = False + + # preserve config + self._update_dataset_config(dataset_name='test', config=test_data_config) + + self._test_dl = self._setup_dataloader_from_config(config=test_data_config) + + def _setup_dataloader_from_config(self, config: Optional[Dict]): + + if 'augmentor' in config: + augmentor = process_augmentations(config['augmentor']) + else: + augmentor = None + + shuffle = config['shuffle'] + device = 'gpu' if torch.cuda.is_available() else 'cpu' + if config.get('use_dali', False): + device_id = self.local_rank if device == 'gpu' else None + dataset = audio_to_text_dataset.get_dali_char_dataset( + config=config, + shuffle=shuffle, + device_id=device_id, + global_rank=self.global_rank, + world_size=self.world_size, + preprocessor_cfg=self._cfg.preprocessor, + ) + return dataset + + # Instantiate tarred dataset loader or normal dataset loader + if config.get('is_tarred', False): + if ('tarred_audio_filepaths' in config and config['tarred_audio_filepaths'] is None) or ( + 'manifest_filepath' in config and config['manifest_filepath'] is None + ): + logging.warning( + "Could not load dataset as `manifest_filepath` was None or " + f"`tarred_audio_filepaths` is None. Provided config : {config}" + ) + return None + + shuffle_n = config.get('shuffle_n', 4 * config['batch_size']) if shuffle else 0 + dataset = audio_to_text_dataset.get_tarred_char_dataset( + config=config, + shuffle_n=shuffle_n, + global_rank=self.global_rank, + world_size=self.world_size, + augmentor=augmentor, + ) + shuffle = False + else: + if 'manifest_filepath' in config and config['manifest_filepath'] is None: + logging.warning(f"Could not load dataset as `manifest_filepath` was None. Provided config : {config}") + return None + + dataset = audio_to_text_dataset.get_char_dataset(config=config, augmentor=augmentor) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=config['batch_size'], + collate_fn=dataset.collate_fn, + drop_last=config.get('drop_last', False), + shuffle=shuffle, + num_workers=config.get('num_workers', 0), + pin_memory=config.get('pin_memory', False), + ) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/modules/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/modules/__init__.py new file mode 100644 index 0000000000..15edf3874a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/modules/__init__.py @@ -0,0 +1,30 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.asr.modules.audio_preprocessing import ( + AudioToMelSpectrogramPreprocessor, + AudioToMFCCPreprocessor, + CropOrPadSpectrogramAugmentation, + SpectrogramAugmentation, +) +from nemo.collections.asr.modules.beam_search_decoder import BeamSearchDecoderWithLM +from nemo.collections.asr.modules.conformer_encoder import ConformerEncoder +from nemo.collections.asr.modules.conv_asr import ( + ConvASRDecoder, + ConvASRDecoderClassification, + ConvASREncoder, + SpeakerDecoder, +) +from nemo.collections.asr.modules.lstm_decoder import LSTMDecoder +from nemo.collections.asr.modules.rnnt import RNNTDecoder, RNNTJoint diff --git a/models/tacotron2/NeMo/nemo/collections/asr/modules/audio_preprocessing.py b/models/tacotron2/NeMo/nemo/collections/asr/modules/audio_preprocessing.py new file mode 100644 index 0000000000..f7ad62d8db --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/modules/audio_preprocessing.py @@ -0,0 +1,643 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, Optional + +import torch +from packaging import version + +from nemo.collections.asr.parts.numba.spec_augment import SpecAugmentNumba, spec_augment_launch_heuristics +from nemo.collections.asr.parts.preprocessing.features import FilterbankFeatures +from nemo.collections.asr.parts.submodules.spectr_augment import SpecAugment, SpecCutout +from nemo.core.classes import NeuralModule, typecheck +from nemo.core.neural_types import ( + AudioSignal, + LengthsType, + MelSpectrogramType, + MFCCSpectrogramType, + NeuralType, + SpectrogramType, +) +from nemo.core.utils import numba_utils +from nemo.core.utils.numba_utils import __NUMBA_MINIMUM_VERSION__ +from nemo.utils import logging + +try: + import torchaudio + import torchaudio.functional + import torchaudio.transforms + + TORCHAUDIO_VERSION = version.parse(torchaudio.__version__) + TORCHAUDIO_VERSION_MIN = version.parse('0.5') + + HAVE_TORCHAUDIO = True +except ModuleNotFoundError: + HAVE_TORCHAUDIO = False + +__all__ = [ + 'AudioToMelSpectrogramPreprocessor', + 'AudioToMFCCPreprocessor', + 'SpectrogramAugmentation', + 'CropOrPadSpectrogramAugmentation', +] + + +class AudioPreprocessor(NeuralModule, ABC): + """ + An interface for Neural Modules that performs audio pre-processing, + transforming the wav files to features. + """ + + def __init__(self, win_length, hop_length): + super().__init__() + + self.win_length = win_length + self.hop_length = hop_length + + self.torch_windows = { + 'hann': torch.hann_window, + 'hamming': torch.hamming_window, + 'blackman': torch.blackman_window, + 'bartlett': torch.bartlett_window, + 'ones': torch.ones, + None: torch.ones, + } + + @typecheck() + @torch.no_grad() + def forward(self, input_signal, length): + processed_signal, processed_length = self.get_features(input_signal, length) + + return processed_signal, processed_length + + @abstractmethod + def get_features(self, input_signal, length): + # Called by forward(). Subclasses should implement this. + pass + + +class AudioToMelSpectrogramPreprocessor(AudioPreprocessor): + """Featurizer module that converts wavs to mel spectrograms. + We don't use torchaudio's implementation here because the original + implementation is not the same, so for the sake of backwards-compatibility + this will use the old FilterbankFeatures for now. + Args: + sample_rate (int): Sample rate of the input audio data. + Defaults to 16000 + window_size (float): Size of window for fft in seconds + Defaults to 0.02 + window_stride (float): Stride of window for fft in seconds + Defaults to 0.01 + n_window_size (int): Size of window for fft in samples + Defaults to None. Use one of window_size or n_window_size. + n_window_stride (int): Stride of window for fft in samples + Defaults to None. Use one of window_stride or n_window_stride. + window (str): Windowing function for fft. can be one of ['hann', + 'hamming', 'blackman', 'bartlett'] + Defaults to "hann" + normalize (str): Can be one of ['per_feature', 'all_features']; all + other options disable feature normalization. 'all_features' + normalizes the entire spectrogram to be mean 0 with std 1. + 'pre_features' normalizes per channel / freq instead. + Defaults to "per_feature" + n_fft (int): Length of FT window. If None, it uses the smallest power + of 2 that is larger than n_window_size. + Defaults to None + preemph (float): Amount of pre emphasis to add to audio. Can be + disabled by passing None. + Defaults to 0.97 + features (int): Number of mel spectrogram freq bins to output. + Defaults to 64 + lowfreq (int): Lower bound on mel basis in Hz. + Defaults to 0 + highfreq (int): Lower bound on mel basis in Hz. + Defaults to None + log (bool): Log features. + Defaults to True + log_zero_guard_type(str): Need to avoid taking the log of zero. There + are two options: "add" or "clamp". + Defaults to "add". + log_zero_guard_value(float, or str): Add or clamp requires the number + to add with or clamp to. log_zero_guard_value can either be a float + or "tiny" or "eps". torch.finfo is used if "tiny" or "eps" is + passed. + Defaults to 2**-24. + dither (float): Amount of white-noise dithering. + Defaults to 1e-5 + pad_to (int): Ensures that the output size of the time dimension is + a multiple of pad_to. + Defaults to 16 + frame_splicing (int): Defaults to 1 + exact_pad (bool): If True, sets stft center to False and adds padding, such that num_frames = audio_length + // hop_length. Defaults to False. + stft_exact_pad (bool): If True, uses pytorch_stft and convolutions with + padding such that num_frames = num_samples / hop_length. If False, + stft_conv will be used to determine how stft will be performed. + Defaults to False. TODO:This feature is deprecated and will be removed in 1.1.0 + stft_conv (bool): If True, uses pytorch_stft and convolutions. If + False, uses torch.stft. TODO:This feature is deprecated and will be removed in 1.1.0 + Defaults to False + pad_value (float): The value that shorter mels are padded with. + Defaults to 0 + mag_power (float): The power that the linear spectrogram is raised to + prior to multiplication with mel basis. + Defaults to 2 for a power spec + """ + + def save_to(self, save_path: str): + pass + + @classmethod + def restore_from(cls, restore_path: str): + pass + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "input_signal": NeuralType(('B', 'T'), AudioSignal(freq=self._sample_rate)), + "length": NeuralType(tuple('B'), LengthsType()), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + processed_signal: + 0: AxisType(BatchTag) + 1: AxisType(MelSpectrogramSignalTag) + 2: AxisType(ProcessedTimeTag) + processed_length: + 0: AxisType(BatchTag) + """ + return { + "processed_signal": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "processed_length": NeuralType(tuple('B'), LengthsType()), + } + + def __init__( + self, + sample_rate=16000, + window_size=0.02, + window_stride=0.01, + n_window_size=None, + n_window_stride=None, + window="hann", + normalize="per_feature", + n_fft=None, + preemph=0.97, + features=64, + lowfreq=0, + highfreq=None, + log=True, + log_zero_guard_type="add", + log_zero_guard_value=2 ** -24, + dither=1e-5, + pad_to=16, + frame_splicing=1, + exact_pad=False, + stft_exact_pad=False, + stft_conv=False, + pad_value=0, + mag_power=2.0, + ): + super().__init__(n_window_size, n_window_stride) + + self._sample_rate = sample_rate + if window_size and n_window_size: + raise ValueError(f"{self} received both window_size and " f"n_window_size. Only one should be specified.") + if window_stride and n_window_stride: + raise ValueError( + f"{self} received both window_stride and " f"n_window_stride. Only one should be specified." + ) + if window_size: + n_window_size = int(window_size * self._sample_rate) + if window_stride: + n_window_stride = int(window_stride * self._sample_rate) + + self.featurizer = FilterbankFeatures( + sample_rate=self._sample_rate, + n_window_size=n_window_size, + n_window_stride=n_window_stride, + window=window, + normalize=normalize, + n_fft=n_fft, + preemph=preemph, + nfilt=features, + lowfreq=lowfreq, + highfreq=highfreq, + log=log, + log_zero_guard_type=log_zero_guard_type, + log_zero_guard_value=log_zero_guard_value, + dither=dither, + pad_to=pad_to, + frame_splicing=frame_splicing, + exact_pad=exact_pad, + stft_exact_pad=stft_exact_pad, + stft_conv=stft_conv, + pad_value=pad_value, + mag_power=mag_power, + ) + + def get_features(self, input_signal, length): + return self.featurizer(input_signal, length) + + @property + def filter_banks(self): + return self.featurizer.filter_banks + + +class AudioToMFCCPreprocessor(AudioPreprocessor): + """Preprocessor that converts wavs to MFCCs. + Uses torchaudio.transforms.MFCC. + Args: + sample_rate: The sample rate of the audio. + Defaults to 16000. + window_size: Size of window for fft in seconds. Used to calculate the + win_length arg for mel spectrogram. + Defaults to 0.02 + window_stride: Stride of window for fft in seconds. Used to caculate + the hop_length arg for mel spect. + Defaults to 0.01 + n_window_size: Size of window for fft in samples + Defaults to None. Use one of window_size or n_window_size. + n_window_stride: Stride of window for fft in samples + Defaults to None. Use one of window_stride or n_window_stride. + window: Windowing function for fft. can be one of ['hann', + 'hamming', 'blackman', 'bartlett', 'none', 'null']. + Defaults to 'hann' + n_fft: Length of FT window. If None, it uses the smallest power of 2 + that is larger than n_window_size. + Defaults to None + lowfreq (int): Lower bound on mel basis in Hz. + Defaults to 0 + highfreq (int): Lower bound on mel basis in Hz. + Defaults to None + n_mels: Number of mel filterbanks. + Defaults to 64 + n_mfcc: Number of coefficients to retain + Defaults to 64 + dct_type: Type of discrete cosine transform to use + norm: Type of norm to use + log: Whether to use log-mel spectrograms instead of db-scaled. + Defaults to True. + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "input_signal": NeuralType(('B', 'T'), AudioSignal(freq=self._sample_rate)), + "length": NeuralType(tuple('B'), LengthsType()), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return { + "processed_signal": NeuralType(('B', 'D', 'T'), MFCCSpectrogramType()), + "processed_length": NeuralType(tuple('B'), LengthsType()), + } + + def save_to(self, save_path: str): + pass + + @classmethod + def restore_from(cls, restore_path: str): + pass + + def __init__( + self, + sample_rate=16000, + window_size=0.02, + window_stride=0.01, + n_window_size=None, + n_window_stride=None, + window='hann', + n_fft=None, + lowfreq=0.0, + highfreq=None, + n_mels=64, + n_mfcc=64, + dct_type=2, + norm='ortho', + log=True, + ): + self._sample_rate = sample_rate + if not HAVE_TORCHAUDIO: + logging.error('Could not import torchaudio. Some features might not work.') + + raise ModuleNotFoundError( + "torchaudio is not installed but is necessary for " + "AudioToMFCCPreprocessor. We recommend you try " + "building it from source for the PyTorch version you have." + ) + if window_size and n_window_size: + raise ValueError(f"{self} received both window_size and " f"n_window_size. Only one should be specified.") + if window_stride and n_window_stride: + raise ValueError( + f"{self} received both window_stride and " f"n_window_stride. Only one should be specified." + ) + # Get win_length (n_window_size) and hop_length (n_window_stride) + if window_size: + n_window_size = int(window_size * self._sample_rate) + if window_stride: + n_window_stride = int(window_stride * self._sample_rate) + + super().__init__(n_window_size, n_window_stride) + + mel_kwargs = {} + + mel_kwargs['f_min'] = lowfreq + mel_kwargs['f_max'] = highfreq + mel_kwargs['n_mels'] = n_mels + + mel_kwargs['n_fft'] = n_fft or 2 ** math.ceil(math.log2(n_window_size)) + + mel_kwargs['win_length'] = n_window_size + mel_kwargs['hop_length'] = n_window_stride + + # Set window_fn. None defaults to torch.ones. + window_fn = self.torch_windows.get(window, None) + if window_fn is None: + raise ValueError( + f"Window argument for AudioProcessor is invalid: {window}." + f"For no window function, use 'ones' or None." + ) + mel_kwargs['window_fn'] = window_fn + + # Use torchaudio's implementation of MFCCs as featurizer + self.featurizer = torchaudio.transforms.MFCC( + sample_rate=self._sample_rate, + n_mfcc=n_mfcc, + dct_type=dct_type, + norm=norm, + log_mels=log, + melkwargs=mel_kwargs, + ) + + def get_features(self, input_signal, length): + features = self.featurizer(input_signal) + seq_len = torch.ceil(length.to(torch.float32) / self.hop_length).to(dtype=torch.long) + return features, seq_len + + +class SpectrogramAugmentation(NeuralModule): + """ + Performs time and freq cuts in one of two ways. + SpecAugment zeroes out vertical and horizontal sections as described in + SpecAugment (https://arxiv.org/abs/1904.08779). Arguments for use with + SpecAugment are `freq_masks`, `time_masks`, `freq_width`, and `time_width`. + SpecCutout zeroes out rectangulars as described in Cutout + (https://arxiv.org/abs/1708.04552). Arguments for use with Cutout are + `rect_masks`, `rect_freq`, and `rect_time`. + Args: + freq_masks (int): how many frequency segments should be cut. + Defaults to 0. + time_masks (int): how many time segments should be cut + Defaults to 0. + freq_width (int): maximum number of frequencies to be cut in one + segment. + Defaults to 10. + time_width (int): maximum number of time steps to be cut in one + segment + Defaults to 10. + rect_masks (int): how many rectangular masks should be cut + Defaults to 0. + rect_freq (int): maximum size of cut rectangles along the frequency + dimension + Defaults to 5. + rect_time (int): maximum size of cut rectangles along the time + dimension + Defaults to 25. + """ + + @property + def input_types(self): + """Returns definitions of module input types + """ + return { + "input_spec": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "length": NeuralType(tuple('B'), LengthsType(), optional=True), + } + + @property + def output_types(self): + """Returns definitions of module output types + """ + return {"augmented_spec": NeuralType(('B', 'D', 'T'), SpectrogramType())} + + def __init__( + self, + freq_masks=0, + time_masks=0, + freq_width=10, + time_width=10, + rect_masks=0, + rect_time=5, + rect_freq=20, + rng=None, + mask_value=0.0, + use_numba_spec_augment: bool = True, + ): + super().__init__() + + if rect_masks > 0: + self.spec_cutout = SpecCutout(rect_masks=rect_masks, rect_time=rect_time, rect_freq=rect_freq, rng=rng,) + # self.spec_cutout.to(self._device) + else: + self.spec_cutout = lambda input_spec: input_spec + + if freq_masks + time_masks > 0: + self.spec_augment = SpecAugment( + freq_masks=freq_masks, + time_masks=time_masks, + freq_width=freq_width, + time_width=time_width, + rng=rng, + mask_value=mask_value, + ) + else: + self.spec_augment = lambda input_spec: input_spec + + # Check if numba is supported, and use a Numba kernel if it is + if use_numba_spec_augment and numba_utils.numba_cuda_is_supported(__NUMBA_MINIMUM_VERSION__): + self.spec_augment_numba = SpecAugmentNumba( + freq_masks=freq_masks, + time_masks=time_masks, + freq_width=freq_width, + time_width=time_width, + rng=rng, + mask_value=mask_value, + ) + else: + self.spec_augment_numba = None + + @typecheck() + def forward(self, input_spec, length=None): + augmented_spec = self.spec_cutout(input_spec=input_spec) + + # To run the Numba kernel, correct numba version is required as well as + # tensor must be on GPU and length must be provided + if self.spec_augment_numba is not None and spec_augment_launch_heuristics(augmented_spec, length): + augmented_spec = self.spec_augment_numba(input_spec=augmented_spec, length=length) + else: + augmented_spec = self.spec_augment(input_spec=augmented_spec) + return augmented_spec + + +class CropOrPadSpectrogramAugmentation(NeuralModule): + """ + Pad or Crop the incoming Spectrogram to a certain shape. + Args: + audio_length (int): the final number of timesteps that is required. + The signal will be either padded or cropped temporally to this + size. + """ + + def __init__(self, audio_length): + super(CropOrPadSpectrogramAugmentation, self).__init__() + self.audio_length = audio_length + + @typecheck() + @torch.no_grad() + def forward(self, input_signal, length): + image = input_signal + num_images = image.shape[0] + + audio_length = self.audio_length + image_len = image.shape[-1] + + # Crop long signal + if image_len > audio_length: # randomly slice + cutout_images = [] + offset = torch.randint(low=0, high=image_len - audio_length + 1, size=[num_images]) + + for idx, offset in enumerate(offset): + cutout_images.append(image[idx : idx + 1, :, offset : offset + audio_length]) + + image = torch.cat(cutout_images, dim=0) + del cutout_images + + else: # symmetrically pad short signal with zeros + pad_left = (audio_length - image_len) // 2 + pad_right = (audio_length - image_len) // 2 + + if (audio_length - image_len) % 2 == 1: + pad_right += 1 + + image = torch.nn.functional.pad(image, [pad_left, pad_right], mode="constant", value=0) + + # Replace dynamic length sequences with static number of timesteps + length = (length * 0) + audio_length + + return image, length + + @property + def input_types(self): + """Returns definitions of module output ports. + """ + return { + "input_signal": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "length": NeuralType(tuple('B'), LengthsType()), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return { + "processed_signal": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "processed_length": NeuralType(tuple('B'), LengthsType()), + } + + def save_to(self, save_path: str): + pass + + @classmethod + def restore_from(cls, restore_path: str): + pass + + +@dataclass +class AudioToMelSpectrogramPreprocessorConfig: + _target_: str = "nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor" + sample_rate: int = 16000 + window_size: float = 0.02 + window_stride: float = 0.01 + n_window_size: Optional[int] = None + n_window_stride: Optional[int] = None + window: str = "hann" + normalize: str = "per_feature" + n_fft: Optional[int] = None + preemph: float = 0.97 + features: int = 64 + lowfreq: int = 0 + highfreq: Optional[int] = None + log: bool = True + log_zero_guard_type: str = "add" + log_zero_guard_value: float = 2 ** -24 + dither: float = 1e-5 + pad_to: int = 16 + frame_splicing: int = 1 + exact_pad: bool = False + stft_exact_pad: bool = False + stft_conv: bool = False + pad_value: int = 0 + mag_power: float = 2.0 + + +@dataclass +class AudioToMFCCPreprocessorConfig: + _target_: str = 'nemo.collections.asr.modules.AudioToMFCCPreprocessor' + sample_rate: int = 16000 + window_size: float = 0.02 + window_stride: float = 0.01 + n_window_size: Optional[int] = None + n_window_stride: Optional[int] = None + window: str = 'hann' + n_fft: Optional[int] = None + lowfreq: Optional[float] = 0.0 + highfreq: Optional[float] = None + n_mels: int = 64 + n_mfcc: int = 64 + dct_type: int = 2 + norm: str = 'ortho' + log: bool = True + + +@dataclass +class SpectrogramAugmentationConfig: + _target_: str = "nemo.collections.asr.modules.SpectrogramAugmentation" + freq_masks: int = 0 + time_masks: int = 0 + freq_width: int = 0 + time_width: Optional[Any] = 0 + rect_masks: int = 0 + rect_time: int = 0 + rect_freq: int = 0 + mask_value: float = 0 + rng: Optional[Any] = None # random.Random() type + use_numba_spec_augment: bool = True + + +@dataclass +class CropOrPadSpectrogramAugmentationConfig: + audio_length: int + _target_: str = "nemo.collections.asr.modules.CropOrPadSpectrogramAugmentation" diff --git a/models/tacotron2/NeMo/nemo/collections/asr/modules/beam_search_decoder.py b/models/tacotron2/NeMo/nemo/collections/asr/modules/beam_search_decoder.py new file mode 100644 index 0000000000..b39804ae8e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/modules/beam_search_decoder.py @@ -0,0 +1,103 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from nemo.core.classes import NeuralModule, typecheck +from nemo.core.neural_types import LengthsType, LogprobsType, NeuralType, PredictionsType + + +class BeamSearchDecoderWithLM(NeuralModule): + """Neural Module that does CTC beam search with a N-gram language model. + It takes a batch of log_probabilities. Note the bigger the batch, the + better as processing is parallelized. Outputs a list of size batch_size. + Each element in the list is a list of size beam_search, and each element + in that list is a tuple of (final_log_prob, hyp_string). + Args: + vocab (list): List of characters that can be output by the ASR model. For English, this is the 28 character set + {a-z '}. The CTC blank symbol is automatically added. + beam_width (int): Size of beams to keep and expand upon. Larger beams result in more accurate but slower + predictions + alpha (float): The amount of importance to place on the N-gram language model. Larger alpha means more + importance on the LM and less importance on the acoustic model. + beta (float): A penalty term given to longer word sequences. Larger beta will result in shorter sequences. + lm_path (str): Path to N-gram language model + num_cpus (int): Number of CPUs to use + cutoff_prob (float): Cutoff probability in vocabulary pruning, default 1.0, no pruning + cutoff_top_n (int): Cutoff number in pruning, only top cutoff_top_n characters with highest probs in + vocabulary will be used in beam search, default 40. + input_tensor (bool): Set to True if you intend to pass PyTorch Tensors, set to False if you intend to pass + NumPy arrays. + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "log_probs": NeuralType(('B', 'T', 'D'), LogprobsType()), + "log_probs_length": NeuralType(tuple('B'), LengthsType()), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return {"predictions": NeuralType(('B', 'T'), PredictionsType())} + + def __init__( + self, vocab, beam_width, alpha, beta, lm_path, num_cpus, cutoff_prob=1.0, cutoff_top_n=40, input_tensor=False + ): + + try: + from ctc_decoders import Scorer, ctc_beam_search_decoder_batch + except ModuleNotFoundError: + raise ModuleNotFoundError( + "BeamSearchDecoderWithLM requires the installation of ctc_decoders " + "from scripts/asr_language_modeling/ngram_lm/install_beamsearch_decoders.sh" + ) + + super().__init__() + + if lm_path is not None: + self.scorer = Scorer(alpha, beta, model_path=lm_path, vocabulary=vocab) + else: + self.scorer = None + self.beam_search_func = ctc_beam_search_decoder_batch + self.vocab = vocab + self.beam_width = beam_width + self.num_cpus = num_cpus + self.cutoff_prob = cutoff_prob + self.cutoff_top_n = cutoff_top_n + self.input_tensor = input_tensor + + @typecheck(ignore_collections=True) + @torch.no_grad() + def forward(self, log_probs, log_probs_length): + probs_list = log_probs + if self.input_tensor: + probs = torch.exp(log_probs) + probs_list = [] + for i, prob in enumerate(probs): + probs_list.append(prob[: log_probs_length[i], :]) + res = self.beam_search_func( + probs_list, + self.vocab, + beam_size=self.beam_width, + num_processes=self.num_cpus, + ext_scoring_func=self.scorer, + cutoff_prob=self.cutoff_prob, + cutoff_top_n=self.cutoff_top_n, + ) + return res diff --git a/models/tacotron2/NeMo/nemo/collections/asr/modules/conformer_encoder.py b/models/tacotron2/NeMo/nemo/collections/asr/modules/conformer_encoder.py new file mode 100644 index 0000000000..0f82e172bb --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/modules/conformer_encoder.py @@ -0,0 +1,255 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from collections import OrderedDict + +import torch +import torch.nn as nn + +from nemo.collections.asr.parts.submodules.conformer_modules import ConformerLayer +from nemo.collections.asr.parts.submodules.multi_head_attention import PositionalEncoding, RelPositionalEncoding +from nemo.collections.asr.parts.submodules.subsampling import ConvSubsampling +from nemo.core.classes.common import typecheck +from nemo.core.classes.exportable import Exportable +from nemo.core.classes.module import NeuralModule +from nemo.core.neural_types import AcousticEncodedRepresentation, LengthsType, NeuralType, SpectrogramType + +__all__ = ['ConformerEncoder'] + + +class ConformerEncoder(NeuralModule, Exportable): + """ + The encoder for ASR model of Conformer. + Based on this paper: + 'Conformer: Convolution-augmented Transformer for Speech Recognition' by Anmol Gulati et al. + https://arxiv.org/abs/2005.08100 + + Args: + feat_in (int): the size of feature channels + n_layers (int): number of layers of ConformerBlock + d_model (int): the hidden size of the model + feat_out (int): the size of the output features + Defaults to -1 (means feat_out is d_model) + subsampling (str): the method of subsampling, choices=['vggnet', 'striding'] + Defaults to striding. + subsampling_factor (int): the subsampling factor which should be power of 2 + Defaults to 4. + subsampling_conv_channels (int): the size of the convolutions in the subsampling module + Defaults to -1 which would set it to d_model. + ff_expansion_factor (int): the expansion factor in feed forward layers + Defaults to 4. + self_attention_model (str): type of the attention layer and positional encoding + 'rel_pos': relative positional embedding and Transformer-XL + 'abs_pos': absolute positional embedding and Transformer + default is rel_pos. + pos_emb_max_len (int): the maximum length of positional embeddings + Defaulst to 5000 + n_heads (int): number of heads in multi-headed attention layers + Defaults to 4. + xscaling (bool): enables scaling the inputs to the multi-headed attention layers by sqrt(d_model) + Defaults to True. + untie_biases (bool): whether to not share (untie) the bias weights between layers of Transformer-XL + Defaults to True. + conv_kernel_size (int): the size of the convolutions in the convolutional modules + Defaults to 31. + dropout (float): the dropout rate used in all layers except the attention layers + Defaults to 0.1. + dropout_emb (float): the dropout rate used for the positional embeddings + Defaults to 0.1. + dropout_att (float): the dropout rate used for the attention layer + Defaults to 0.0. + """ + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + input_example = torch.randn(16, self._feat_in, 256).to(next(self.parameters()).device) + input_example_length = torch.randint(0, 256, (16,)).to(next(self.parameters()).device) + return tuple([input_example, input_example_length]) + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return OrderedDict( + { + "audio_signal": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "length": NeuralType(tuple('B'), LengthsType()), + } + ) + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return OrderedDict( + { + "outputs": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + "encoded_lengths": NeuralType(tuple('B'), LengthsType()), + } + ) + + def __init__( + self, + feat_in, + n_layers, + d_model, + feat_out=-1, + subsampling='striding', + subsampling_factor=4, + subsampling_conv_channels=-1, + ff_expansion_factor=4, + self_attention_model='rel_pos', + n_heads=4, + att_context_size=None, + xscaling=True, + untie_biases=True, + pos_emb_max_len=5000, + conv_kernel_size=31, + dropout=0.1, + dropout_emb=0.1, + dropout_att=0.0, + ): + super().__init__() + + d_ff = d_model * ff_expansion_factor + self.d_model = d_model + self._feat_in = feat_in + self.scale = math.sqrt(self.d_model) + if att_context_size: + self.att_context_size = att_context_size + else: + self.att_context_size = [-1, -1] + + if xscaling: + self.xscale = math.sqrt(d_model) + else: + self.xscale = None + + if subsampling_conv_channels == -1: + subsampling_conv_channels = d_model + if subsampling and subsampling_factor > 1: + self.pre_encode = ConvSubsampling( + subsampling=subsampling, + subsampling_factor=subsampling_factor, + feat_in=feat_in, + feat_out=d_model, + conv_channels=subsampling_conv_channels, + activation=nn.ReLU(), + ) + self._feat_out = d_model + else: + self._feat_out = d_model + self.pre_encode = nn.Linear(feat_in, d_model) + + if not untie_biases and self_attention_model == "rel_pos": + d_head = d_model // n_heads + pos_bias_u = nn.Parameter(torch.Tensor(n_heads, d_head)) + pos_bias_v = nn.Parameter(torch.Tensor(n_heads, d_head)) + nn.init.zeros_(pos_bias_u) + nn.init.zeros_(pos_bias_v) + else: + pos_bias_u = None + pos_bias_v = None + + if self_attention_model == "rel_pos": + self.pos_enc = RelPositionalEncoding( + d_model=d_model, + dropout_rate=dropout, + max_len=pos_emb_max_len, + xscale=self.xscale, + dropout_rate_emb=dropout_emb, + ) + elif self_attention_model == "abs_pos": + pos_bias_u = None + pos_bias_v = None + self.pos_enc = PositionalEncoding( + d_model=d_model, dropout_rate=dropout, max_len=pos_emb_max_len, xscale=self.xscale + ) + else: + raise ValueError(f"Not valid self_attention_model: '{self_attention_model}'!") + + self.layers = nn.ModuleList() + for i in range(n_layers): + layer = ConformerLayer( + d_model=d_model, + d_ff=d_ff, + self_attention_model=self_attention_model, + n_heads=n_heads, + conv_kernel_size=conv_kernel_size, + dropout=dropout, + dropout_att=dropout_att, + pos_bias_u=pos_bias_u, + pos_bias_v=pos_bias_v, + ) + self.layers.append(layer) + + if feat_out > 0 and feat_out != self.output_dim: + self.out_proj = nn.Linear(self.feat_out, feat_out) + self._feat_out = feat_out + else: + self.out_proj = None + self._feat_out = d_model + + @typecheck() + def forward(self, audio_signal, length=None): + if length is None: + length = torch.tensor(audio_signal.size(-1)).repeat(audio_signal.size(0)).to(audio_signal) + audio_signal = torch.transpose(audio_signal, 1, 2) + + if isinstance(self.pre_encode, ConvSubsampling): + audio_signal, length = self.pre_encode(audio_signal, length) + else: + audio_signal = self.pre_encode(audio_signal) + + audio_signal, pos_emb = self.pos_enc(audio_signal) + bs, xmax, idim = audio_signal.size() + + # Create the self-attention and padding masks + pad_mask = self.make_pad_mask(length, max_time=xmax, device=audio_signal.device) + att_mask = pad_mask.unsqueeze(1).repeat([1, xmax, 1]) + att_mask = att_mask & att_mask.transpose(1, 2) + if self.att_context_size[0] >= 0: + att_mask = att_mask.triu(diagonal=-self.att_context_size[0]) + if self.att_context_size[1] >= 0: + att_mask = att_mask.tril(diagonal=self.att_context_size[1]) + att_mask = ~att_mask + pad_mask = ~pad_mask + + for lth, layer in enumerate(self.layers): + audio_signal = layer(x=audio_signal, att_mask=att_mask, pos_emb=pos_emb, pad_mask=pad_mask) + + if self.out_proj is not None: + audio_signal = self.out_proj(audio_signal) + + audio_signal = torch.transpose(audio_signal, 1, 2) + return audio_signal, length + + @staticmethod + def make_pad_mask(seq_lens, max_time, device=None): + """Make masking for padding.""" + bs = seq_lens.size(0) + seq_range = torch.arange(0, max_time, dtype=torch.int32) + seq_range_expand = seq_range.unsqueeze(0).expand(bs, max_time) + seq_lens = seq_lens.type(seq_range_expand.dtype).to(seq_range_expand.device) + seq_length_expand = seq_lens.unsqueeze(-1) + mask = seq_range_expand < seq_length_expand + + if device: + mask = mask.to(device) + return mask diff --git a/models/tacotron2/NeMo/nemo/collections/asr/modules/conv_asr.py b/models/tacotron2/NeMo/nemo/collections/asr/modules/conv_asr.py new file mode 100644 index 0000000000..7f2767019d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/modules/conv_asr.py @@ -0,0 +1,504 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import List, Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +from omegaconf import MISSING, ListConfig, OmegaConf + +from nemo.collections.asr.parts.submodules.jasper import ( + JasperBlock, + MaskedConv1d, + StatsPoolLayer, + init_weights, + jasper_activations, +) +from nemo.core.classes.common import typecheck +from nemo.core.classes.exportable import Exportable +from nemo.core.classes.module import NeuralModule +from nemo.core.neural_types import ( + AcousticEncodedRepresentation, + LengthsType, + LogitsType, + LogprobsType, + NeuralType, + SpectrogramType, +) +from nemo.utils import logging + +__all__ = ['ConvASRDecoder', 'ConvASREncoder', 'ConvASRDecoderClassification'] + + +class ConvASREncoder(NeuralModule, Exportable): + """ + Convolutional encoder for ASR models. With this class you can implement JasperNet and QuartzNet models. + + Based on these papers: + https://arxiv.org/pdf/1904.03288.pdf + https://arxiv.org/pdf/1910.10261.pdf + """ + + def _prepare_for_export(self, **kwargs): + m_count = 0 + for m in self.modules(): + if isinstance(m, MaskedConv1d): + m.use_mask = False + m_count += 1 + Exportable._prepare_for_export(self, **kwargs) + logging.warning(f"Turned off {m_count} masked convolutions") + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + input_example = torch.randn(16, self._feat_in, 256).to(next(self.parameters()).device) + return tuple([input_example]) + + @property + def disabled_deployment_input_names(self): + """Implement this method to return a set of input names disabled for export""" + return set(["length"]) + + @property + def disabled_deployment_output_names(self): + """Implement this method to return a set of output names disabled for export""" + return set(["encoded_lengths"]) + + def save_to(self, save_path: str): + pass + + @classmethod + def restore_from(cls, restore_path: str): + pass + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return OrderedDict( + { + "audio_signal": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "length": NeuralType(tuple('B'), LengthsType()), + } + ) + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return OrderedDict( + { + "outputs": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + "encoded_lengths": NeuralType(tuple('B'), LengthsType()), + } + ) + + def __init__( + self, + jasper, + activation: str, + feat_in: int, + normalization_mode: str = "batch", + residual_mode: str = "add", + norm_groups: int = -1, + conv_mask: bool = True, + frame_splicing: int = 1, + init_mode: Optional[str] = 'xavier_uniform', + quantize: bool = False, + ): + super().__init__() + if isinstance(jasper, ListConfig): + jasper = OmegaConf.to_container(jasper) + + activation = jasper_activations[activation]() + + # If the activation can be executed in place, do so. + if hasattr(activation, 'inplace'): + activation.inplace = True + + feat_in = feat_in * frame_splicing + + self._feat_in = feat_in + + residual_panes = [] + encoder_layers = [] + self.dense_residual = False + for lcfg in jasper: + dense_res = [] + if lcfg.get('residual_dense', False): + residual_panes.append(feat_in) + dense_res = residual_panes + self.dense_residual = True + groups = lcfg.get('groups', 1) + separable = lcfg.get('separable', False) + heads = lcfg.get('heads', -1) + residual_mode = lcfg.get('residual_mode', residual_mode) + se = lcfg.get('se', False) + se_reduction_ratio = lcfg.get('se_reduction_ratio', 8) + se_context_window = lcfg.get('se_context_size', -1) + se_interpolation_mode = lcfg.get('se_interpolation_mode', 'nearest') + kernel_size_factor = lcfg.get('kernel_size_factor', 1.0) + stride_last = lcfg.get('stride_last', False) + future_context = lcfg.get('future_context', -1) + encoder_layers.append( + JasperBlock( + feat_in, + lcfg['filters'], + repeat=lcfg['repeat'], + kernel_size=lcfg['kernel'], + stride=lcfg['stride'], + dilation=lcfg['dilation'], + dropout=lcfg['dropout'], + residual=lcfg['residual'], + groups=groups, + separable=separable, + heads=heads, + residual_mode=residual_mode, + normalization=normalization_mode, + norm_groups=norm_groups, + activation=activation, + residual_panes=dense_res, + conv_mask=conv_mask, + se=se, + se_reduction_ratio=se_reduction_ratio, + se_context_window=se_context_window, + se_interpolation_mode=se_interpolation_mode, + kernel_size_factor=kernel_size_factor, + stride_last=stride_last, + future_context=future_context, + quantize=quantize, + ) + ) + feat_in = lcfg['filters'] + + self._feat_out = feat_in + + self.encoder = torch.nn.Sequential(*encoder_layers) + self.apply(lambda x: init_weights(x, mode=init_mode)) + + @typecheck() + def forward(self, audio_signal, length=None): + s_input, length = self.encoder(([audio_signal], length)) + if length is None: + return s_input[-1] + + return s_input[-1], length + + +class ConvASRDecoder(NeuralModule, Exportable): + """Simple ASR Decoder for use with CTC-based models such as JasperNet and QuartzNet + + Based on these papers: + https://arxiv.org/pdf/1904.03288.pdf + https://arxiv.org/pdf/1910.10261.pdf + https://arxiv.org/pdf/2005.04290.pdf + """ + + def save_to(self, save_path: str): + pass + + @classmethod + def restore_from(cls, restore_path: str): + pass + + @property + def input_types(self): + return OrderedDict({"encoder_output": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation())}) + + @property + def output_types(self): + return OrderedDict({"logprobs": NeuralType(('B', 'T', 'D'), LogprobsType())}) + + def __init__(self, feat_in, num_classes, init_mode="xavier_uniform", vocabulary=None): + super().__init__() + + if vocabulary is not None: + if num_classes != len(vocabulary): + raise ValueError( + f"If vocabulary is specified, it's length should be equal to the num_classes. Instead got: num_classes={num_classes} and len(vocabulary)={len(vocabulary)}" + ) + self.__vocabulary = vocabulary + self._feat_in = feat_in + # Add 1 for blank char + self._num_classes = num_classes + 1 + + self.decoder_layers = torch.nn.Sequential( + torch.nn.Conv1d(self._feat_in, self._num_classes, kernel_size=1, bias=True) + ) + self.apply(lambda x: init_weights(x, mode=init_mode)) + + @typecheck() + def forward(self, encoder_output): + return torch.nn.functional.log_softmax(self.decoder_layers(encoder_output).transpose(1, 2), dim=-1) + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + bs = 8 + seq = 64 + input_example = torch.randn(bs, self._feat_in, seq).to(next(self.parameters()).device) + return tuple([input_example]) + + def _prepare_for_export(self, **kwargs): + m_count = 0 + for m in self.modules(): + if type(m).__name__ == "MaskedConv1d": + m.use_mask = False + m_count += 1 + if m_count > 0: + logging.warning(f"Turned off {m_count} masked convolutions") + Exportable._prepare_for_export(self, **kwargs) + + @property + def vocabulary(self): + return self.__vocabulary + + @property + def num_classes_with_blank(self): + return self._num_classes + + +class ConvASRDecoderClassification(NeuralModule, Exportable): + """Simple ASR Decoder for use with classification models such as JasperNet and QuartzNet + + Based on these papers: + https://arxiv.org/pdf/2005.04290.pdf + """ + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + input_example = torch.randn(16, self._feat_in, 128).to(next(self.parameters()).device) + return tuple([input_example]) + + @property + def input_types(self): + return OrderedDict({"encoder_output": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation())}) + + @property + def output_types(self): + return OrderedDict({"logits": NeuralType(('B', 'D'), LogitsType())}) + + def __init__( + self, + feat_in: int, + num_classes: int, + init_mode: Optional[str] = "xavier_uniform", + return_logits: bool = True, + pooling_type='avg', + ): + super().__init__() + + self._feat_in = feat_in + self._return_logits = return_logits + self._num_classes = num_classes + + if pooling_type == 'avg': + self.pooling = torch.nn.AdaptiveAvgPool1d(1) + elif pooling_type == 'max': + self.pooling = torch.nn.AdaptiveMaxPool1d(1) + else: + raise ValueError('Pooling type chosen is not valid. Must be either `avg` or `max`') + + self.decoder_layers = torch.nn.Sequential(torch.nn.Linear(self._feat_in, self._num_classes, bias=True)) + self.apply(lambda x: init_weights(x, mode=init_mode)) + + @typecheck() + def forward(self, encoder_output): + batch, in_channels, timesteps = encoder_output.size() + + encoder_output = self.pooling(encoder_output).view(batch, in_channels) # [B, C] + logits = self.decoder_layers(encoder_output) # [B, num_classes] + + if self._return_logits: + return logits + + return torch.nn.functional.softmax(logits, dim=-1) + + @property + def num_classes(self): + return self._num_classes + + +class SpeakerDecoder(NeuralModule, Exportable): + """ + Speaker Decoder creates the final neural layers that maps from the outputs + of Jasper Encoder to the embedding layer followed by speaker based softmax loss. + Args: + feat_in (int): Number of channels being input to this module + num_classes (int): Number of unique speakers in dataset + emb_sizes (list) : shapes of intermediate embedding layers (we consider speaker embbeddings from 1st of this layers) + Defaults to [1024,1024] + pool_mode (str) : Pooling stratergy type. options are 'gram','xvector','superVector'. + Defaults to 'xvector' + init_mode (str): Describes how neural network parameters are + initialized. Options are ['xavier_uniform', 'xavier_normal', + 'kaiming_uniform','kaiming_normal']. + Defaults to "xavier_uniform". + """ + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + input_example = torch.randn(16, self.input_feat_in, 256).to(next(self.parameters()).device) + return tuple([input_example]) + + @property + def input_types(self): + return OrderedDict({"encoder_output": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation())}) + + @property + def output_types(self): + return OrderedDict( + { + "logits": NeuralType(('B', 'D'), LogitsType()), + "embs": NeuralType(('B', 'D'), AcousticEncodedRepresentation()), + } + ) + + def __init__( + self, feat_in, num_classes, emb_sizes=None, pool_mode='xvector', angular=False, init_mode="xavier_uniform", + ): + super().__init__() + self.angular = angular + self.emb_id = 2 + if self.angular: + bias = False + else: + bias = True + + if type(emb_sizes) is str: + emb_sizes = emb_sizes.split(',') + elif type(emb_sizes) is int: + emb_sizes = [emb_sizes] + else: + emb_sizes = [512, 512] + + self.input_feat_in = feat_in + self._num_classes = num_classes + self._pooling = StatsPoolLayer(feat_in=feat_in, pool_mode=pool_mode) + self._feat_in = self._pooling.feat_in + + shapes = [self._feat_in] + for size in emb_sizes: + shapes.append(int(size)) + + emb_layers = [] + for shape_in, shape_out in zip(shapes[:-1], shapes[1:]): + layer = self.affineLayer(shape_in, shape_out, learn_mean=False) + emb_layers.append(layer) + + self.emb_layers = nn.ModuleList(emb_layers) + + self.final = nn.Linear(shapes[-1], self._num_classes, bias=bias) + + self.apply(lambda x: init_weights(x, mode=init_mode)) + + def affineLayer(self, inp_shape, out_shape, learn_mean=True): + layer = nn.Sequential( + nn.Linear(inp_shape, out_shape), + nn.BatchNorm1d(out_shape, affine=learn_mean, track_running_stats=True), + nn.ReLU(), + ) + + return layer + + @typecheck() + def forward(self, encoder_output): + pool = self._pooling(encoder_output) + embs = [] + + for layer in self.emb_layers: + pool, emb = layer(pool), layer[: self.emb_id](pool) + embs.append(emb) + + if self.angular: + for W in self.final.parameters(): + W = F.normalize(W, p=2, dim=1) + pool = F.normalize(pool, p=2, dim=1) + + out = self.final(pool) + + return out, embs[-1] + + +@dataclass +class JasperEncoderConfig: + filters: int = MISSING + repeat: int = MISSING + kernel: List[int] = MISSING + stride: List[int] = MISSING + dilation: List[int] = MISSING + dropout: float = MISSING + residual: bool = MISSING + + # Optional arguments + groups: int = 1 + separable: bool = False + heads: int = -1 + residual_mode: str = "add" + residual_dense: bool = False + se: bool = False + se_reduction_ratio: int = 8 + se_context_size: int = -1 + se_interpolation_mode: str = 'nearest' + kernel_size_factor: float = 1.0 + stride_last: bool = False + + +@dataclass +class ConvASREncoderConfig: + _target_: str = 'nemo.collections.asr.modules.ConvASREncoder' + jasper: Optional[JasperEncoderConfig] = field(default_factory=list) + activation: str = MISSING + feat_in: int = MISSING + normalization_mode: str = "batch" + residual_mode: str = "add" + norm_groups: int = -1 + conv_mask: bool = True + frame_splicing: int = 1 + init_mode: Optional[str] = "xavier_uniform" + + +@dataclass +class ConvASRDecoderConfig: + _target_: str = 'nemo.collections.asr.modules.ConvASRDecoder' + feat_in: int = MISSING + num_classes: int = MISSING + init_mode: Optional[str] = "xavier_uniform" + vocabulary: Optional[List[str]] = field(default_factory=list) + + +@dataclass +class ConvASRDecoderClassificationConfig: + _target_: str = 'nemo.collections.asr.modules.ConvASRDecoderClassification' + feat_in: int = MISSING + num_classes: int = MISSING + init_mode: Optional[str] = "xavier_uniform" + return_logits: bool = True + pooling_type: str = 'avg' diff --git a/models/tacotron2/NeMo/nemo/collections/asr/modules/lstm_decoder.py b/models/tacotron2/NeMo/nemo/collections/asr/modules/lstm_decoder.py new file mode 100644 index 0000000000..2cfb61524f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/modules/lstm_decoder.py @@ -0,0 +1,95 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import OrderedDict + +import torch +import torch.nn as nn + +from nemo.core.classes.common import typecheck +from nemo.core.classes.exportable import Exportable +from nemo.core.classes.module import NeuralModule +from nemo.core.neural_types import AcousticEncodedRepresentation, LogprobsType, NeuralType + +__all__ = ['LSTMDecoder'] + + +class LSTMDecoder(NeuralModule, Exportable): + """ + Simple LSTM Decoder for ASR models + Args: + feat_in (int): size of the input features + num_classes (int): the size of the vocabulary + lstm_hidden_size (int): hidden size of the LSTM layers + vocabulary (vocab): The vocabulary + bidirectional (bool): default is False. Whether LSTMs are bidirectional or not + num_layers (int): default is 1. Number of LSTM layers stacked + """ + + @property + def input_types(self): + return OrderedDict({"encoder_output": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation())}) + + @property + def output_types(self): + return OrderedDict({"logprobs": NeuralType(('B', 'T', 'D'), LogprobsType())}) + + def __init__(self, feat_in, num_classes, lstm_hidden_size, vocabulary=None, bidirectional=False, num_layers=1): + super().__init__() + + if vocabulary is not None: + if num_classes != len(vocabulary): + raise ValueError( + f"If vocabulary is specified, it's length should be equal to the num_classes. " + f"Instead got: num_classes={num_classes} and len(vocabulary)={len(vocabulary)}" + ) + self.__vocabulary = vocabulary + self._feat_in = feat_in + # Add 1 for blank char + self._num_classes = num_classes + 1 + + self.lstm_layer = nn.LSTM( + input_size=feat_in, + hidden_size=lstm_hidden_size, + num_layers=num_layers, + batch_first=True, + bidirectional=bidirectional, + ) + self.linear_layer = torch.nn.Linear(in_features=lstm_hidden_size, out_features=self._num_classes) + + @typecheck() + def forward(self, encoder_output): + output = encoder_output.transpose(1, 2) + output, _ = self.lstm_layer(output) + output = self.linear_layer(output) + return torch.nn.functional.log_softmax(output, dim=-1) + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + bs = 8 + seq = 64 + input_example = torch.randn(bs, self._feat_in, seq).to(next(self.parameters()).device) + return tuple([input_example]) + + @property + def vocabulary(self): + return self.__vocabulary + + @property + def num_classes_with_blank(self): + return self._num_classes diff --git a/models/tacotron2/NeMo/nemo/collections/asr/modules/rnnt.py b/models/tacotron2/NeMo/nemo/collections/asr/modules/rnnt.py new file mode 100644 index 0000000000..abe7d6ca6c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/modules/rnnt.py @@ -0,0 +1,958 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2017 Johns Hopkins University (Shinji Watanabe) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch + +from nemo.collections.asr.modules import rnnt_abstract +from nemo.collections.asr.parts.utils import rnnt_utils +from nemo.collections.common.parts import rnn +from nemo.core.classes import typecheck +from nemo.core.neural_types import ( + AcousticEncodedRepresentation, + ElementType, + EmbeddedTextType, + LabelsType, + LengthsType, + LogprobsType, + LossType, + NeuralType, +) +from nemo.utils import logging + + +class RNNTDecoder(rnnt_abstract.AbstractRNNTDecoder): + """A Recurrent Neural Network Transducer Decoder / Prediction Network (RNN-T Prediction Network). + An RNN-T Decoder/Prediction network, comprised of a stateful LSTM model. + + Args: + prednet: A dict-like object which contains the following key-value pairs. + pred_hidden: int specifying the hidden dimension of the prediction net. + pred_rnn_layers: int specifying the number of rnn layers. + + Optionally, it may also contain the following: + forget_gate_bias: float, set by default to 1.0, which constructs a forget gate + initialized to 1.0. + Reference: + [An Empirical Exploration of Recurrent Network Architectures](http://proceedings.mlr.press/v37/jozefowicz15.pdf) + t_max: int value, set to None by default. If an int is specified, performs Chrono Initialization + of the LSTM network, based on the maximum number of timesteps `t_max` expected during the course + of training. + Reference: + [Can recurrent neural networks warp time?](https://openreview.net/forum?id=SJcKhk-Ab) + weights_init_scale: Float scale of the weights after initialization. Setting to lower than one + sometimes helps reduce variance between runs. + hidden_hidden_bias_scale: Float scale for the hidden-to-hidden bias scale. Set to 0.0 for + the default behaviour. + dropout: float, set to 0.0 by default. Optional dropout applied at the end of the final LSTM RNN layer. + + vocab_size: int, specifying the vocabulary size of the embedding layer of the Prediction network, + excluding the RNNT blank token. + + normalization_mode: Can be either None, 'batch' or 'layer'. By default, is set to None. + Defines the type of normalization applied to the RNN layer. + + random_state_sampling: bool, set to False by default. When set, provides normal-distribution + sampled state tensors instead of zero tensors during training. + Reference: + [Recognizing long-form speech using streaming end-to-end models](https://arxiv.org/abs/1910.11455) + + blank_as_pad: bool, set to True by default. When set, will add a token to the Embedding layer of this + prediction network, and will treat this token as a pad token. In essence, the RNNT pad token will + be treated as a pad token, and the embedding layer will return a zero tensor for this token. + + It is set by default as it enables various batch optimizations required for batched beam search. + Therefore, it is not recommended to disable this flag. + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "targets": NeuralType(('B', 'T'), LabelsType()), + "target_length": NeuralType(tuple('B'), LengthsType()), + "states": NeuralType(('D', 'B', 'D'), ElementType(), optional=True), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return { + "outputs": NeuralType(('B', 'D', 'T'), EmbeddedTextType()), + "encoded_lengths": NeuralType(tuple('B'), LengthsType()), + } + + def __init__( + self, + prednet: Dict[str, Any], + vocab_size: int, + normalization_mode: Optional[str] = None, + random_state_sampling: bool = False, + blank_as_pad: bool = True, + ): + # Required arguments + self.pred_hidden = prednet['pred_hidden'] + self.pred_rnn_layers = prednet["pred_rnn_layers"] + self.blank_idx = vocab_size + + # Initialize the model (blank token increases vocab size by 1) + super().__init__(vocab_size=vocab_size, blank_idx=self.blank_idx, blank_as_pad=blank_as_pad) + + # Optional arguments + forget_gate_bias = prednet.get('forget_gate_bias', 1.0) + t_max = prednet.get('t_max', None) + weights_init_scale = prednet.get('weights_init_scale', 1.0) + hidden_hidden_bias_scale = prednet.get('hidden_hidden_bias_scale', 0.0) + dropout = prednet.get('dropout', 0.0) + self.random_state_sampling = random_state_sampling + + self.prediction = self._predict( + vocab_size=vocab_size, # add 1 for blank symbol + pred_n_hidden=self.pred_hidden, + pred_rnn_layers=self.pred_rnn_layers, + forget_gate_bias=forget_gate_bias, + t_max=t_max, + norm=normalization_mode, + weights_init_scale=weights_init_scale, + hidden_hidden_bias_scale=hidden_hidden_bias_scale, + dropout=dropout, + ) + + @typecheck() + def forward(self, targets, target_length, states=None): + # y: (B, U) + y = rnn.label_collate(targets) + + # state maintenance is unnecessary during training forward call + # to get state, use .predict() method. + g, _ = self.predict(y, state=states, add_sos=True) # (B, U, D) + g = g.transpose(1, 2) # (B, D, U) + + return g, target_length + + def predict( + self, + y: Optional[torch.Tensor] = None, + state: Optional[List[torch.Tensor]] = None, + add_sos: bool = True, + batch_size: Optional[int] = None, + ) -> (torch.Tensor, List[torch.Tensor]): + """ + Stateful prediction of scores and state for a (possibly null) tokenset. + This method takes various cases into consideration : + - No token, no state - used for priming the RNN + - No token, state provided - used for blank token scoring + - Given token, states - used for scores + new states + + Here: + B - batch size + U - label length + H - Hidden dimension size of RNN + L - Number of RNN layers + + Args: + y: Optional torch tensor of shape [B, U] of dtype long which will be passed to the Embedding. + If None, creates a zero tensor of shape [B, 1, H] which mimics output of pad-token on Embedding. + + state: An optional list of states for the RNN. Eg: For LSTM, it is the state list length is 2. + Each state must be a tensor of shape [L, B, H]. + If None, and during training mode and `random_state_sampling` is set, will sample a + normal distribution tensor of the above shape. Otherwise, None will be passed to the RNN. + + add_sos: bool flag, whether a zero vector describing a "start of signal" token should be + prepended to the above "y" tensor. When set, output size is (B, U + 1, H). + + batch_size: An optional int, specifying the batch size of the `y` tensor. + Can be infered if `y` and `state` is None. But if both are None, then batch_size cannot be None. + + Returns: + A tuple (g, hid) such that - + + If add_sos is False: + g: (B, U, H) + hid: (h, c) where h is the final sequence hidden state and c is the final cell state: + h (tensor), shape (L, B, H) + c (tensor), shape (L, B, H) + + If add_sos is True: + g: (B, U + 1, H) + hid: (h, c) where h is the final sequence hidden state and c is the final cell state: + h (tensor), shape (L, B, H) + c (tensor), shape (L, B, H) + + """ + # Get device and dtype of current module + _p = next(self.parameters()) + device = _p.device + dtype = _p.dtype + + # If y is not None, it is of shape [B, U] with dtype long. + if y is not None: + if y.device != device: + y = y.to(device) + + # (B, U) -> (B, U, H) + y = self.prediction["embed"](y) + else: + # Y is not provided, assume zero tensor with shape [B, 1, H] is required + # Emulates output of embedding of pad token. + if batch_size is None: + B = 1 if state is None else state[0].size(1) + else: + B = batch_size + + y = torch.zeros((B, 1, self.pred_hidden), device=device, dtype=dtype) + + # Prepend blank "start of sequence" symbol (zero tensor) + if add_sos: + B, U, H = y.shape + start = torch.zeros((B, 1, H), device=y.device, dtype=y.dtype) + y = torch.cat([start, y], dim=1).contiguous() # (B, U + 1, H) + else: + start = None # makes del call later easier + + # If in training mode, and random_state_sampling is set, + # initialize state to random normal distribution tensor. + if state is None: + if self.random_state_sampling and self.training: + state = self.initialize_state(y) + + # Forward step through RNN + y = y.transpose(0, 1) # (U + 1, B, H) + g, hid = self.prediction["dec_rnn"](y, state) + g = g.transpose(0, 1) # (B, U + 1, H) + + del y, start, state + return g, hid + + def _predict( + self, + vocab_size, + pred_n_hidden, + pred_rnn_layers, + forget_gate_bias, + t_max, + norm, + weights_init_scale, + hidden_hidden_bias_scale, + dropout, + ): + """ + Prepare the trainable parameters of the Prediction Network. + + Args: + vocab_size: Vocab size (excluding the blank token). + pred_n_hidden: Hidden size of the RNNs. + pred_rnn_layers: Number of RNN layers. + forget_gate_bias: Whether to perform unit forget gate bias. + t_max: Whether to perform Chrono LSTM init. + norm: Type of normalization to perform in RNN. + weights_init_scale: Float scale of the weights after initialization. Setting to lower than one + sometimes helps reduce variance between runs. + hidden_hidden_bias_scale: Float scale for the hidden-to-hidden bias scale. Set to 0.0 for + the default behaviour. + dropout: Whether to apply dropout to RNN. + """ + if self.blank_as_pad: + embed = torch.nn.Embedding(vocab_size + 1, pred_n_hidden, padding_idx=self.blank_idx) + else: + embed = torch.nn.Embedding(vocab_size, pred_n_hidden) + + layers = torch.nn.ModuleDict( + { + "embed": embed, + "dec_rnn": rnn.rnn( + input_size=pred_n_hidden, + hidden_size=pred_n_hidden, + num_layers=pred_rnn_layers, + norm=norm, + forget_gate_bias=forget_gate_bias, + t_max=t_max, + dropout=dropout, + weights_init_scale=weights_init_scale, + hidden_hidden_bias_scale=hidden_hidden_bias_scale, + ), + } + ) + return layers + + def initialize_state(self, y: torch.Tensor) -> List[torch.Tensor]: + """ + Initialize the state of the RNN layers, with same dtype and device as input `y`. + + Args: + y: A torch.Tensor whose device the generated states will be placed on. + + Returns: + List of torch.Tensor, each of shape [L, B, H], where + L = Number of RNN layers + B = Batch size + H = Hidden size of RNN. + """ + batch = y.size(0) + if self.random_state_sampling and self.training: + state = [ + torch.randn(self.pred_rnn_layers, batch, self.pred_hidden, dtype=y.dtype, device=y.device), + torch.randn(self.pred_rnn_layers, batch, self.pred_hidden, dtype=y.dtype, device=y.device), + ] + + else: + state = [ + torch.zeros(self.pred_rnn_layers, batch, self.pred_hidden, dtype=y.dtype, device=y.device), + torch.zeros(self.pred_rnn_layers, batch, self.pred_hidden, dtype=y.dtype, device=y.device), + ] + return state + + def score_hypothesis( + self, hypothesis: rnnt_utils.Hypothesis, cache: Dict[Tuple[int], Any] + ) -> (torch.Tensor, List[torch.Tensor], torch.Tensor): + """ + Similar to the predict() method, instead this method scores a Hypothesis during beam search. + Hypothesis is a dataclass representing one hypothesis in a Beam Search. + + Args: + hypothesis: Refer to rnnt_utils.Hypothesis. + cache: Dict which contains a cache to avoid duplicate computations. + + Returns: + Returns a tuple (y, states, lm_token) such that: + y is a torch.Tensor of shape [1, 1, H] representing the score of the last token in the Hypothesis. + state is a list of RNN states, each of shape [L, 1, H]. + lm_token is the final integer token of the hypothesis. + """ + if hypothesis.dec_state is not None: + device = hypothesis.dec_state[0].device + else: + _p = next(self.parameters()) + device = _p.device + + # parse "blank" tokens in hypothesis + if len(hypothesis.y_sequence) > 0 and hypothesis.y_sequence[-1] == self.blank_idx: + blank_state = True + else: + blank_state = False + + # Convert last token of hypothesis to torch.Tensor + target = torch.full([1, 1], fill_value=hypothesis.y_sequence[-1], device=device, dtype=torch.long) + lm_token = target[:, -1] # [1] + + # Convert current hypothesis into a tuple to preserve in cache + sequence = tuple(hypothesis.y_sequence) + + if sequence in cache: + y, new_state = cache[sequence] + else: + # Obtain score for target token and new states + if blank_state: + y, new_state = self.predict(None, state=None, add_sos=False, batch_size=1) # [1, 1, H] + + else: + y, new_state = self.predict( + target, state=hypothesis.dec_state, add_sos=False, batch_size=1 + ) # [1, 1, H] + + y = y[:, -1:, :] # Extract just last state : [1, 1, H] + cache[sequence] = (y, new_state) + + return y, new_state, lm_token + + def batch_score_hypothesis( + self, hypotheses: List[rnnt_utils.Hypothesis], cache: Dict[Tuple[int], Any], batch_states: List[torch.Tensor] + ) -> (torch.Tensor, List[torch.Tensor], torch.Tensor): + """ + Used for batched beam search algorithms. Similar to score_hypothesis method. + + Args: + hypothesis: List of Hypotheses. Refer to rnnt_utils.Hypothesis. + cache: Dict which contains a cache to avoid duplicate computations. + batch_states: List of torch.Tensor which represent the states of the RNN for this batch. + Each state is of shape [L, B, H] + + Returns: + Returns a tuple (b_y, b_states, lm_tokens) such that: + b_y is a torch.Tensor of shape [B, 1, H] representing the scores of the last tokens in the Hypotheses. + b_state is a list of list of RNN states, each of shape [L, B, H]. + Represented as B x List[states]. + lm_token is a list of the final integer tokens of the hypotheses in the batch. + """ + final_batch = len(hypotheses) + + if final_batch == 0: + raise ValueError("No hypotheses was provided for the batch!") + + _p = next(self.parameters()) + device = _p.device + dtype = _p.dtype + + tokens = [] + process = [] + done = [None for _ in range(final_batch)] + + # For each hypothesis, cache the last token of the sequence and the current states + for i, hyp in enumerate(hypotheses): + sequence = tuple(hyp.y_sequence) + + if sequence in cache: + done[i] = cache[sequence] + else: + tokens.append(hyp.y_sequence[-1]) + process.append((sequence, hyp.dec_state)) + + if process: + batch = len(process) + + # convert list of tokens to torch.Tensor, then reshape. + tokens = torch.tensor(tokens, device=device, dtype=torch.long).view(batch, -1) + dec_states = self.initialize_state(tokens.to(dtype=dtype)) # [L, B, H] + dec_states = self.batch_initialize_states(dec_states, [d_state for seq, d_state in process]) + + y, dec_states = self.predict( + tokens, state=dec_states, add_sos=False, batch_size=batch + ) # [B, 1, H], List([L, 1, H]) + + # Update done states and cache shared by entire batch. + j = 0 + for i in range(final_batch): + if done[i] is None: + # Select sample's state from the batch state list + new_state = self.batch_select_state(dec_states, j) + + # Cache [1, H] scores of the current y_j, and its corresponding state + done[i] = (y[j], new_state) + cache[process[j][0]] = (y[j], new_state) + + j += 1 + + # Set the incoming batch states with the new states obtained from `done`. + batch_states = self.batch_initialize_states(batch_states, [d_state for y_j, d_state in done]) + + # Create batch of all output scores + # List[1, 1, H] -> [B, 1, H] + batch_y = torch.stack([y_j for y_j, d_state in done]) + + # Extract the last tokens from all hypotheses and convert to a tensor + lm_tokens = torch.tensor([h.y_sequence[-1] for h in hypotheses], device=device, dtype=torch.long).view( + final_batch + ) + + return batch_y, batch_states, lm_tokens + + def batch_initialize_states(self, batch_states: List[torch.Tensor], decoder_states: List[List[torch.Tensor]]): + """ + Create batch of decoder states. + + Args: + batch_states (list): batch of decoder states + ([L x (B, H)], [L x (B, H)]) + + decoder_states (list of list): list of decoder states + [B x ([L x (1, H)], [L x (1, H)])] + + Returns: + batch_states (tuple): batch of decoder states + ([L x (B, H)], [L x (B, H)]) + """ + # LSTM has 2 states + for layer in range(self.pred_rnn_layers): + for state_id in range(len(batch_states)): + batch_states[state_id][layer] = torch.stack([s[state_id][layer] for s in decoder_states]) + + return batch_states + + def batch_select_state(self, batch_states: List[torch.Tensor], idx: int) -> List[List[torch.Tensor]]: + """Get decoder state from batch of states, for given id. + + Args: + batch_states (list): batch of decoder states + ([L x (B, H)], [L x (B, H)]) + + idx (int): index to extract state from batch of states + + Returns: + (tuple): decoder states for given id + ([L x (1, H)], [L x (1, H)]) + """ + state_list = [] + for state_id in range(len(batch_states)): + states = [batch_states[state_id][layer][idx] for layer in range(self.pred_rnn_layers)] + state_list.append(states) + + return state_list + + +class RNNTJoint(rnnt_abstract.AbstractRNNTJoint): + """A Recurrent Neural Network Transducer Joint Network (RNN-T Joint Network). + An RNN-T Joint network, comprised of a feedforward model. + + Args: + jointnet: A dict-like object which contains the following key-value pairs. + encoder_hidden: int specifying the hidden dimension of the encoder net. + pred_hidden: int specifying the hidden dimension of the prediction net. + joint_hidden: int specifying the hidden dimension of the joint net + activation: Activation function used in the joint step. Can be one of + ['relu', 'tanh', 'sigmoid']. + + Optionally, it may also contain the following: + dropout: float, set to 0.0 by default. Optional dropout applied at the end of the joint net. + + num_classes: int, specifying the vocabulary size that the joint network must predict, + excluding the RNNT blank token. + + vocabulary: Optional list of strings/tokens that comprise the vocabulary of the joint network. + Unused and kept only for easy access for character based encoding RNNT models. + + log_softmax: Optional bool, set to None by default. If set as None, will compute the log_softmax() + based on the value provided. + + preserve_memory: Optional bool, set to False by default. If the model crashes due to the memory + intensive joint step, one might try this flag to empty the tensor cache in pytorch. + + Warning: This will make the forward-backward pass much slower than normal. + It also might not fix the OOM if the GPU simply does not have enough memory to compute the joint. + + experimental_fuse_loss_wer: Optional bool, set to False by default. + NOTE: This is an experimental feature that attempts to trade of compute time for memory preservation. + There may be undetermined effects to convergence behaviour. + + Fuses the joint forward, loss forward and + wer forward steps. In doing so, it trades of speed for memory conservation by creating sub-batches + of the provided batch of inputs, and performs Joint forward, loss forward and wer forward (optional), + all on sub-batches, then collates results to be exactly equal to results from the entire batch. + + When this flag is set, prior to calling forward, the fields `loss` and `wer` (either one) *must* + be set using the `RNNTJoint.set_loss()` or `RNNTJoint.set_wer()` methods. + + Further, when this flag is set, the following argument `fused_batch_size` *must* be provided + as a non negative integer. This value refers to the size of the sub-batch. + + When the flag is set, the input and output signature of `forward()` of this method changes. + Input - in addition to `encoder_outputs` (mandatory argument), the following arguments can be provided. + - decoder_outputs (optional). Required if loss computation is required. + - encoder_lengths (required) + - transcripts (optional). Required for wer calculation. + - transcript_lengths (optional). Required for wer calculation. + - compute_wer (bool, default false). Whether to compute WER or not for the fused batch. + + Output - instead of the usual `joint` log prob tensor, the following results can be returned. + - loss (optional). Returned if decoder_outputs, transcripts and transript_lengths are not None. + - wer_numerator + wer_denominator (optional). Returned if transcripts, transcripts_lengths are provided + and compute_wer is set. + + fused_batch_size: Optional int, required if `fuse_loss_wer` flag is set. Determines the size of the + sub-batches. Should be any value below the actual batch size per GPU. + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "encoder_outputs": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + "decoder_outputs": NeuralType(('B', 'D', 'T'), EmbeddedTextType()), + "encoder_lengths": NeuralType(tuple('B'), LengthsType(), optional=True), + "transcripts": NeuralType(('B', 'T'), LabelsType(), optional=True), + "transcript_lengths": NeuralType(tuple('B'), LengthsType(), optional=True), + "compute_wer": NeuralType(optional=True), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + if not self._fuse_loss_wer: + return { + "outputs": NeuralType(('B', 'T', 'D', 'D'), LogprobsType()), + } + + else: + return { + "loss": NeuralType(elements_type=LossType(), optional=True), + "wer": NeuralType(elements_type=ElementType(), optional=True), + "wer_numer": NeuralType(elements_type=ElementType(), optional=True), + "wer_denom": NeuralType(elements_type=ElementType(), optional=True), + } + + def __init__( + self, + jointnet: Dict[str, Any], + num_classes: int, + vocabulary: Optional[List] = None, + log_softmax: Optional[bool] = None, + preserve_memory: bool = False, + experimental_fuse_loss_wer: bool = False, + fused_batch_size: Optional[int] = None, + ): + super().__init__() + + self.vocabulary = vocabulary + + self._vocab_size = num_classes + self._num_classes = num_classes + 1 # add 1 for blank symbol + + self._fuse_loss_wer = experimental_fuse_loss_wer + self._fused_batch_size = fused_batch_size + + if experimental_fuse_loss_wer and (fused_batch_size is None): + raise ValueError("If `fuse_loss_wer` is set, then `fused_batch_size` cannot be None!") + + if experimental_fuse_loss_wer: + logging.warning( + "\nFused joint step is an experimental technique. Please be aware that it " + "may have unintended side effects!\n" + ) + + self._loss = None + self._wer = None + + # Log softmax should be applied explicitly only for CPU + self.log_softmax = log_softmax + self.preserve_memory = preserve_memory + + if preserve_memory: + logging.warning( + "`preserve_memory` was set for the Joint Model. Please be aware this will severely impact " + "the forward-backward step time. It also might not solve OOM issues if the GPU simply " + "does not have enough memory to compute the joint." + ) + + # Required arguments + self.encoder_hidden = jointnet['encoder_hidden'] + self.pred_hidden = jointnet['pred_hidden'] + self.joint_hidden = jointnet['joint_hidden'] + self.activation = jointnet['activation'] + + # Optional arguments + dropout = jointnet.get('dropout', 0.0) + + self.pred, self.enc, self.joint_net = self._joint_net( + num_classes=self._num_classes, # add 1 for blank symbol + pred_n_hidden=self.pred_hidden, + enc_n_hidden=self.encoder_hidden, + joint_n_hidden=self.joint_hidden, + activation=self.activation, + dropout=dropout, + ) + + @typecheck() + def forward( + self, + encoder_outputs: torch.Tensor, + decoder_outputs: Optional[torch.Tensor], + encoder_lengths: Optional[torch.Tensor] = None, + transcripts: Optional[torch.Tensor] = None, + transcript_lengths: Optional[torch.Tensor] = None, + compute_wer: bool = False, + ) -> Union[torch.Tensor, List[Optional[torch.Tensor]]]: + # encoder = (B, D, T) + # decoder = (B, D, U) if passed, else None + encoder_outputs = encoder_outputs.transpose(1, 2) # (B, T, D) + + if decoder_outputs is not None: + decoder_outputs = decoder_outputs.transpose(1, 2) # (B, U, D) + + if not self._fuse_loss_wer: + if decoder_outputs is None: + raise ValueError( + "decoder_outputs passed is None, and `fuse_loss_wer` is not set. " + "decoder_outputs can only be None for fused step!" + ) + + out = self.joint(encoder_outputs, decoder_outputs) # [B, T, U, V + 1] + return out + + else: + # At least the loss module must be supplied during fused joint + if self._loss is None or self._wer is None: + raise ValueError("`fuse_loss_wer` flag is set, but `loss` and `wer` modules were not provided! ") + + # If fused joint step is required, fused batch size is required as well + if self._fused_batch_size is None: + raise ValueError("If `experimental_fuse_loss_wer` is set, then `fused_batch_size` cannot be None!") + + # When using fused joint step, both encoder and transcript lengths must be provided + if (encoder_lengths is None) or (transcript_lengths is None): + raise ValueError( + "`experimental_fuse_loss_wer` is set, therefore encoder and target lengths " + "must be provided as well!" + ) + + losses = [] + wer_numer_list = [] + wer_denom_list = [] + batch_size = int(encoder_outputs.size(0)) # actual batch size + + # Iterate over batch using fused_batch_size steps + for batch_idx in range(0, batch_size, self._fused_batch_size): + begin = batch_idx + end = min(begin + self._fused_batch_size, batch_size) + + # Extract the sub batch inputs + # sub_enc = encoder_outputs[begin:end, ...] + # sub_transcripts = transcripts[begin:end, ...] + sub_enc = encoder_outputs.narrow(dim=0, start=begin, length=end - begin) + sub_transcripts = transcripts.narrow(dim=0, start=begin, length=end - begin) + + sub_enc_lens = encoder_lengths[begin:end] + sub_transcript_lens = transcript_lengths[begin:end] + + # Sub transcripts does not need the full padding of the entire batch + # Therefore reduce the decoder time steps to match + max_sub_enc_length = sub_enc_lens.max() + max_sub_transcript_length = sub_transcript_lens.max() + + if decoder_outputs is not None: + # Reduce encoder length to preserve computation + # Encoder: [sub-batch, T, D] -> [sub-batch, T', D]; T' < T + if sub_enc.shape[1] != max_sub_enc_length: + sub_enc = sub_enc.narrow(dim=1, start=0, length=max_sub_enc_length) + + # sub_dec = decoder_outputs[begin:end, ...] # [sub-batch, U, D] + sub_dec = decoder_outputs.narrow(dim=0, start=begin, length=end - begin) # [sub-batch, U, D] + + # Reduce decoder length to preserve computation + # Decoder: [sub-batch, U, D] -> [sub-batch, U', D]; U' < U + if sub_dec.shape[1] != max_sub_transcript_length + 1: + sub_dec = sub_dec.narrow(dim=1, start=0, length=max_sub_transcript_length + 1) + + # Perform joint => [sub-batch, T', U', V + 1] + sub_joint = self.joint(sub_enc, sub_dec) + + del sub_dec + + # Reduce transcript length to correct alignment + # Transcript: [sub-batch, L] -> [sub-batch, L']; L' <= L + if sub_transcripts.shape[1] != max_sub_transcript_length: + sub_transcripts = sub_transcripts.narrow(dim=1, start=0, length=max_sub_transcript_length) + + # Compute sub batch loss + # preserve loss reduction type + loss_reduction = self.loss.reduction + + # override loss reduction to sum + self.loss.reduction = None + + # compute and preserve loss + loss_batch = self.loss( + log_probs=sub_joint, + targets=sub_transcripts, + input_lengths=sub_enc_lens, + target_lengths=sub_transcript_lens, + ) + losses.append(loss_batch) + + # reset loss reduction type + self.loss.reduction = loss_reduction + + else: + losses = None + + # Compute WER for sub batch + if compute_wer: + sub_enc = sub_enc.transpose(1, 2) # [B, T, D] -> [B, D, T] + sub_enc = sub_enc.detach() + sub_transcripts = sub_transcripts.detach() + + original_log_prediction = self.wer.log_prediction + if batch_idx == 0: + self.wer.log_prediction = True + else: + self.wer.log_prediction = False + + # Compute the wer (with logging for just 1st sub-batch) + self.wer.update(sub_enc, sub_enc_lens, sub_transcripts, sub_transcript_lens) + wer, wer_num, wer_denom = self.wer.compute() + self.wer.reset() + + wer_numer_list.append(wer_num) + wer_denom_list.append(wer_denom) + + # Reset logging default + self.wer.log_prediction = original_log_prediction + + else: + wer = None + + del sub_enc, sub_transcripts, sub_enc_lens, sub_transcript_lens + + # Collect sub batch loss results + if losses is not None: + losses = torch.cat(losses, 0) + losses = losses.mean() # global batch size average + else: + losses = None + + # Collect sub batch wer results + if compute_wer: + wer_num = torch.tensor(wer_numer_list, dtype=torch.long) + wer_denom = torch.tensor(wer_denom_list, dtype=torch.long) + + wer_num = wer_num.sum() # global sum of correct words/chars + wer_denom = wer_denom.sum() # global sum of all words/chars + else: + wer_num = None + wer_denom = None + + return losses, wer, wer_num, wer_denom + + def joint(self, f: torch.Tensor, g: torch.Tensor) -> torch.Tensor: + """ + Compute the joint step of the network. + + Here, + B = Batch size + T = Acoustic model timesteps + U = Target sequence length + H1, H2 = Hidden dimensions of the Encoder / Decoder respectively + H = Hidden dimension of the Joint hidden step. + V = Vocabulary size of the Decoder (excluding the RNNT blank token). + + NOTE: + The implementation of this model is slightly modified from the original paper. + The original paper proposes the following steps : + (enc, dec) -> Expand + Concat + Sum [B, T, U, H1+H2] -> Forward through joint hidden [B, T, U, H] -- *1 + *1 -> Forward through joint final [B, T, U, V + 1]. + + We instead split the joint hidden into joint_hidden_enc and joint_hidden_dec and act as follows: + enc -> Forward through joint_hidden_enc -> Expand [B, T, 1, H] -- *1 + dec -> Forward through joint_hidden_dec -> Expand [B, 1, U, H] -- *2 + (*1, *2) -> Sum [B, T, U, H] -> Forward through joint final [B, T, U, V + 1]. + + Args: + f: Output of the Encoder model. A torch.Tensor of shape [B, T, H1] + g: Output of the Decoder model. A torch.Tensor of shape [B, U, H2] + + Returns: + Logits / log softmaxed tensor of shape (B, T, U, V + 1). + """ + # f = [B, T, H1] + f = self.enc(f) + f.unsqueeze_(dim=2) # (B, T, 1, H) + + # g = [B, U, H2] + g = self.pred(g) + g.unsqueeze_(dim=1) # (B, 1, U, H) + + inp = f + g # [B, T, U, H] + + del f, g + + res = self.joint_net(inp) # [B, T, U, V + 1] + + del inp + + if self.preserve_memory: + torch.cuda.empty_cache() + + # If log_softmax is automatic + if self.log_softmax is None: + if not res.is_cuda: # Use log softmax only if on CPU + res = res.log_softmax(dim=-1) + else: + if self.log_softmax: + res = res.log_softmax(dim=-1) + + return res + + def _joint_net(self, num_classes, pred_n_hidden, enc_n_hidden, joint_n_hidden, activation, dropout): + """ + Prepare the trainable modules of the Joint Network + + Args: + num_classes: Number of output classes (vocab size) excluding the RNNT blank token. + pred_n_hidden: Hidden size of the prediction network. + enc_n_hidden: Hidden size of the encoder network. + joint_n_hidden: Hidden size of the joint network. + activation: Activation of the joint. Can be one of [relu, tanh, sigmoid] + dropout: Dropout value to apply to joint. + """ + pred = torch.nn.Linear(pred_n_hidden, joint_n_hidden) + enc = torch.nn.Linear(enc_n_hidden, joint_n_hidden) + + if activation not in ['relu', 'sigmoid', 'tanh']: + raise ValueError("Unsupported activation for joint step - please pass one of " "[relu, sigmoid, tanh]") + + activation = activation.lower() + + if activation == 'relu': + activation = torch.nn.ReLU(inplace=True) + elif activation == 'sigmoid': + activation = torch.nn.Sigmoid() + elif activation == 'tanh': + activation = torch.nn.Tanh() + + layers = ( + [activation] + + ([torch.nn.Dropout(p=dropout)] if dropout else []) + + [torch.nn.Linear(joint_n_hidden, num_classes)] + ) + return pred, enc, torch.nn.Sequential(*layers) + + @property + def num_classes_with_blank(self): + return self._num_classes + + @property + def loss(self): + return self._loss + + def set_loss(self, loss): + if not self._fuse_loss_wer: + raise ValueError("Attempting to set loss module even though `fuse_loss_wer` is not set!") + + self._loss = loss + + @property + def wer(self): + return self._wer + + def set_wer(self, wer): + if not self._fuse_loss_wer: + raise ValueError("Attempting to set WER module even though `fuse_loss_wer` is not set!") + + self._wer = wer + + @property + def fuse_loss_wer(self): + return self._fuse_loss_wer + + def set_fuse_loss_wer(self, fuse_loss_wer): + self._fuse_loss_wer = fuse_loss_wer + + if self._fuse_loss_wer is False: + self._loss = None + self._wer = None + + @property + def fused_batch_size(self): + return self._fuse_loss_wer + + def set_fused_batch_size(self, fused_batch_size): + self._fused_batch_size = fused_batch_size diff --git a/models/tacotron2/NeMo/nemo/collections/asr/modules/rnnt_abstract.py b/models/tacotron2/NeMo/nemo/collections/asr/modules/rnnt_abstract.py new file mode 100644 index 0000000000..3f7f9805af --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/modules/rnnt_abstract.py @@ -0,0 +1,236 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional, Tuple + +import torch + +from nemo.collections.asr.parts.utils.rnnt_utils import Hypothesis +from nemo.core import NeuralModule + + +class AbstractRNNTJoint(NeuralModule, ABC): + """ + An abstract RNNT Joint framework, which can possibly integrate with GreedyRNNTInfer and BeamRNNTInfer classes. + Represents the abstract RNNT Joint network, which accepts the acoustic model and prediction network + embeddings in order to compute the joint of the two prior to decoding the output sequence. + """ + + @abstractmethod + def joint(self, f: torch.Tensor, g: torch.Tensor) -> torch.Tensor: + """ + Compute the joint step of the network. + + Here, + B = Batch size + T = Acoustic model timesteps + U = Target sequence length + H1, H2 = Hidden dimensions of the Encoder / Decoder respectively + H = Hidden dimension of the Joint hidden step. + V = Vocabulary size of the Decoder (excluding the RNNT blank token). + + NOTE: + The implementation of this model is slightly modified from the original paper. + The original paper proposes the following steps : + (enc, dec) -> Expand + Concat + Sum [B, T, U, H1+H2] -> Forward through joint hidden [B, T, U, H] -- *1 + *1 -> Forward through joint final [B, T, U, V + 1]. + + We instead split the joint hidden into joint_hidden_enc and joint_hidden_dec and act as follows: + enc -> Forward through joint_hidden_enc -> Expand [B, T, 1, H] -- *1 + dec -> Forward through joint_hidden_dec -> Expand [B, 1, U, H] -- *2 + (*1, *2) -> Sum [B, T, U, H] -> Forward through joint final [B, T, U, V + 1]. + + Args: + f: Output of the Encoder model. A torch.Tensor of shape [B, T, H1] + g: Output of the Decoder model. A torch.Tensor of shape [B, U, H2] + + Returns: + Logits / log softmaxed tensor of shape (B, T, U, V + 1). + """ + raise NotImplementedError() + + @property + def num_classes_with_blank(self): + raise NotImplementedError() + + +class AbstractRNNTDecoder(NeuralModule, ABC): + """ + An abstract RNNT Decoder framework, which can possibly integrate with GreedyRNNTInfer and BeamRNNTInfer classes. + Represents the abstract RNNT Prediction/Decoder stateful network, which performs autoregressive decoding + in order to construct the output sequence. + + Args: + vocab_size: Size of the vocabulary, excluding the RNNT blank token. + blank_idx: Index of the blank token. Can be 0 or size(vocabulary). + blank_as_pad: Bool flag, whether to allocate an additional token in the Embedding layer + of this module in order to treat all RNNT `blank` tokens as pad tokens, thereby letting + the Embedding layer batch tokens more efficiently. + + It is mandatory to use this for certain Beam RNNT Infer methods - such as TSD, ALSD. + It is also more efficient to use greedy batch decoding with this flag. + """ + + def __init__(self, vocab_size, blank_idx, blank_as_pad): + super().__init__() + + self.vocab_size = vocab_size + self.blank_idx = blank_idx # first or last index of vocabulary + self.blank_as_pad = blank_as_pad + + if blank_idx not in [0, vocab_size]: + raise ValueError("`blank_idx` must be either 0 or the final token of the vocabulary") + + @abstractmethod + def predict( + self, + y: Optional[torch.Tensor] = None, + state: Optional[torch.Tensor] = None, + add_sos: bool = False, + batch_size: Optional[int] = None, + ) -> (torch.Tensor, List[torch.Tensor]): + """ + Stateful prediction of scores and state for a (possibly null) tokenset. + This method takes various cases into consideration : + - No token, no state - used for priming the RNN + - No token, state provided - used for blank token scoring + - Given token, states - used for scores + new states + + Here: + B - batch size + U - label length + H - Hidden dimension size of RNN + L - Number of RNN layers + + Args: + y: Optional torch tensor of shape [B, U] of dtype long which will be passed to the Embedding. + If None, creates a zero tensor of shape [B, 1, H] which mimics output of pad-token on Embedding. + + state: An optional list of states for the RNN. Eg: For LSTM, it is the state list length is 2. + Each state must be a tensor of shape [L, B, H]. + If None, and during training mode and `random_state_sampling` is set, will sample a + normal distribution tensor of the above shape. Otherwise, None will be passed to the RNN. + + add_sos: bool flag, whether a zero vector describing a "start of signal" token should be + prepended to the above "y" tensor. When set, output size is (B, U + 1, H). + + batch_size: An optional int, specifying the batch size of the `y` tensor. + Can be infered if `y` and `state` is None. But if both are None, then batch_size cannot be None. + + Returns: + A tuple (g, hid) such that - + + If add_sos is False: + g: (B, U, H) + hid: (h, c) where h is the final sequence hidden state and c is the final cell state: + h (tensor), shape (L, B, H) + c (tensor), shape (L, B, H) + + If add_sos is True: + g: (B, U + 1, H) + hid: (h, c) where h is the final sequence hidden state and c is the final cell state: + h (tensor), shape (L, B, H) + c (tensor), shape (L, B, H) + + """ + raise NotImplementedError() + + @abstractmethod + def initialize_state(self, y: torch.Tensor) -> List[torch.Tensor]: + """ + Initialize the state of the RNN layers, with same dtype and device as input `y`. + + Args: + y: A torch.Tensor whose device the generated states will be placed on. + + Returns: + List of torch.Tensor, each of shape [L, B, H], where + L = Number of RNN layers + B = Batch size + H = Hidden size of RNN. + """ + raise NotImplementedError() + + @abstractmethod + def score_hypothesis( + self, hypothesis: Hypothesis, cache: Dict[Tuple[int], Any] + ) -> (torch.Tensor, List[torch.Tensor], torch.Tensor): + """ + Similar to the predict() method, instead this method scores a Hypothesis during beam search. + Hypothesis is a dataclass representing one hypothesis in a Beam Search. + + Args: + hypothesis: Refer to rnnt_utils.Hypothesis. + cache: Dict which contains a cache to avoid duplicate computations. + + Returns: + Returns a tuple (y, states, lm_token) such that: + y is a torch.Tensor of shape [1, 1, H] representing the score of the last token in the Hypothesis. + state is a list of RNN states, each of shape [L, 1, H]. + lm_token is the final integer token of the hypothesis. + """ + raise NotImplementedError() + + def batch_score_hypothesis( + self, hypotheses: List[Hypothesis], cache: Dict[Tuple[int], Any], batch_states: List[torch.Tensor] + ) -> (torch.Tensor, List[torch.Tensor], torch.Tensor): + """ + Used for batched beam search algorithms. Similar to score_hypothesis method. + + Args: + hypothesis: List of Hypotheses. Refer to rnnt_utils.Hypothesis. + cache: Dict which contains a cache to avoid duplicate computations. + batch_states: List of torch.Tensor which represent the states of the RNN for this batch. + Each state is of shape [L, B, H] + + Returns: + Returns a tuple (b_y, b_states, lm_tokens) such that: + b_y is a torch.Tensor of shape [B, 1, H] representing the scores of the last tokens in the Hypotheses. + b_state is a list of list of RNN states, each of shape [L, B, H]. + Represented as B x List[states]. + lm_token is a list of the final integer tokens of the hypotheses in the batch. + """ + raise NotImplementedError() + + def batch_initialize_states(self, batch_states: List[torch.Tensor], decoder_states: List[List[torch.Tensor]]): + """ + Create batch of decoder states. + + Args: + batch_states (list): batch of decoder states + ([L x (B, H)], [L x (B, H)]) + + decoder_states (list of list): list of decoder states + [B x ([L x (1, H)], [L x (1, H)])] + + Returns: + batch_states (tuple): batch of decoder states + ([L x (B, H)], [L x (B, H)]) + """ + raise NotImplementedError() + + def batch_select_state(self, batch_states: List[torch.Tensor], idx: int) -> List[List[torch.Tensor]]: + """Get decoder state from batch of states, for given id. + + Args: + batch_states (list): batch of decoder states + ([L x (B, H)], [L x (B, H)]) + + idx (int): index to extract state from batch of states + + Returns: + (tuple): decoder states for given id + ([L x (1, H)], [L x (1, H)]) + """ + raise NotImplementedError() diff --git a/models/tacotron2/NeMo/nemo/collections/asr/modules/wav2vec_modules.py b/models/tacotron2/NeMo/nemo/collections/asr/modules/wav2vec_modules.py new file mode 100644 index 0000000000..bf89500f33 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/modules/wav2vec_modules.py @@ -0,0 +1,306 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Optional, Tuple + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from nemo.collections.asr.models.wav2vec.wav2vec_config import Wav2VecMaskType +from nemo.core import NeuralModule +from nemo.core.neural_types import EncodedRepresentation, LossType, NeuralType + + +class GumbelVectorQuantizer(NeuralModule): + def __init__( + self, + dim, + num_vars, + temp, + groups, + combine_groups, + vq_dim, + time_first, + activation=nn.GELU(), + weight_proj_depth=1, + weight_proj_factor=1, + ): + """Vector quantization using gumbel softmax + + Args: + dim: input dimension (channels) + num_vars: number of quantized vectors per group + temp: temperature for training. this should be a tuple of 3 elements: (start, stop, decay factor) + groups: number of groups for vector quantization + combine_groups: whether to use the vectors for all groups + vq_dim: dimensionality of the resulting quantized vector + time_first: if true, expect input in BxTxC format, otherwise in BxCxT + activation: what activation to use (should be a module). this is only used if weight_proj_depth is > 1 + weight_proj_depth: number of layers (with activation in between) to project input before computing logits + weight_proj_factor: this is used only if weight_proj_depth is > 1. scales the inner dimensionality of + projections by this factor + """ + super().__init__() + + self.groups = groups + self.combine_groups = combine_groups + self.input_dim = dim + self.num_vars = num_vars + self.time_first = time_first + + assert vq_dim % groups == 0, f"dim {vq_dim} must be divisible by groups {groups} for concatenation" + + var_dim = vq_dim // groups + num_groups = groups if not combine_groups else 1 + + self.vars = nn.Parameter(torch.FloatTensor(1, num_groups * num_vars, var_dim)) + nn.init.uniform_(self.vars) + + if weight_proj_depth > 1: + + def block(input_dim, output_dim): + return nn.Sequential(nn.Linear(input_dim, output_dim), activation) + + inner_dim = self.input_dim * weight_proj_factor + self.weight_proj = nn.Sequential( + *[block(self.input_dim if i == 0 else inner_dim, inner_dim) for i in range(weight_proj_depth - 1)], + nn.Linear(inner_dim, groups * num_vars), + ) + else: + self.weight_proj = nn.Linear(self.input_dim, groups * num_vars) + nn.init.normal_(self.weight_proj.weight, mean=0, std=1) + nn.init.zeros_(self.weight_proj.bias) + + assert len(temp) == 3, "Quantize temperature should be a tuple of 3 elements: (start, stop, decay factor)" + + self.max_temp, self.min_temp, self.temp_decay = temp + self.curr_temp = self.max_temp + self.codebook_indices = None + + def set_num_updates(self, num_updates): + self.curr_temp = max(self.max_temp * self.temp_decay ** num_updates, self.min_temp) + + def get_codebook_indices(self): + if self.codebook_indices is None: + from itertools import product + + p = [range(self.num_vars)] * self.groups + inds = list(product(*p)) + self.codebook_indices = torch.tensor(inds, dtype=torch.long, device=self.vars.device).flatten() + + if not self.combine_groups: + self.codebook_indices = self.codebook_indices.view(self.num_vars ** self.groups, -1) + for b in range(1, self.groups): + self.codebook_indices[:, b] += self.num_vars * b + self.codebook_indices = self.codebook_indices.flatten() + return self.codebook_indices + + def sample_from_codebook(self, b, n): + indices = self.get_codebook_indices() + indices = indices.view(-1, self.groups) + cb_size = indices.size(0) + assert n < cb_size, f"sample size {n} is greater than size of codebook {cb_size}" + sample_idx = torch.randint(low=0, high=cb_size, size=(b * n,)) + indices = indices[sample_idx] + + z = self.vars.squeeze(0).index_select(0, indices.flatten()).view(b, n, -1) + return z + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + if self.time_first: + return {"x": NeuralType(('B', 'T', 'D'), EncodedRepresentation())} + return {"x": NeuralType(('B', 'D', 'T'), EncodedRepresentation())} + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + if self.time_first: + return { + "x": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "quantize_prob_ppl": NeuralType(elements_type=LossType()), + } + return { + "x": NeuralType(('B', 'D', 'T'), EncodedRepresentation()), + "quantize_prob_ppl": NeuralType(elements_type=LossType()), + } + + def forward(self, x): + + if not self.time_first: + x = x.transpose(1, 2) + + bsz, tsz, fsz = x.shape + x = x.reshape(-1, fsz) + x = self.weight_proj(x) + x = x.view(bsz * tsz * self.groups, -1) + + _, k = x.max(-1) + hard_x = x.new_zeros(*x.shape).scatter_(-1, k.view(-1, 1), 1.0).view(bsz * tsz, self.groups, -1) + + # Calculate quantize prob perplexity + num_vars = self.num_vars * self.groups + avg_probs = torch.softmax(x.view(bsz * tsz, self.groups, -1).float(), dim=-1).mean(dim=0) + quantize_prob_ppl = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-7), dim=-1)).sum() + quantize_prob_ppl = (num_vars - quantize_prob_ppl) / num_vars + + if self.training: + x = F.gumbel_softmax(x.float(), tau=self.curr_temp, hard=True).type_as(x) + else: + x = hard_x + + x = x.view(bsz * tsz, -1) + + vars = self.vars + if self.combine_groups: + vars = vars.repeat(1, self.groups, 1) + + x = x.unsqueeze(-1) * vars + x = x.view(bsz * tsz, self.groups, self.num_vars, -1) + x = x.sum(-2) + x = x.view(bsz, tsz, -1) + + cur_codebook_temp = self.curr_temp + + if not self.time_first: + x = x.transpose(1, 2) # BTC -> BCT + + return x, quantize_prob_ppl, cur_codebook_temp + + +def compute_mask_indices( + shape: Tuple[int, int], + padding_mask: Optional[torch.Tensor], + mask_prob: float, + mask_length: int, + mask_type: Wav2VecMaskType = Wav2VecMaskType.static, + mask_other: float = 0.0, + min_masks: int = 0, + no_overlap: bool = False, + min_space: int = 0, +) -> np.ndarray: + """ + Computes random mask spans for a given shape + Args: + shape: the the shape for which to compute masks. + should be of size 2 where first element is batch size and 2nd is timesteps + padding_mask: optional padding mask of the same size as shape, which will prevent masking padded elements + mask_prob: probability for each token to be chosen as start of the span to be masked. this will be multiplied by + number of timesteps divided by length of mask span to mask approximately this percentage of all elements. + however due to overlaps, the actual number will be smaller (unless no_overlap is True) + mask_type: how to compute mask lengths + static = fixed size + uniform = sample from uniform distribution [mask_other, mask_length*2] + normal = sample from normal distribution with mean mask_length and stdev mask_other. mask is min 1 element + poisson = sample from possion distribution with lambda = mask length + min_masks: minimum number of masked spans + no_overlap: if false, will switch to an alternative recursive algorithm that prevents spans from overlapping + min_space: only used if no_overlap is True, this is how many elements to keep unmasked between spans + """ + + bsz, all_sz = shape + mask = np.full((bsz, all_sz), False) + + all_num_mask = int( + # add a random number for probabilistic rounding + mask_prob * all_sz / float(mask_length) + + np.random.rand() + ) + + all_num_mask = max(min_masks, all_num_mask) + + mask_idcs = [] + for i in range(bsz): + if padding_mask is not None: + sz = all_sz - padding_mask[i].long().sum().item() + num_mask = int( + # add a random number for probabilistic rounding + mask_prob * sz / float(mask_length) + + np.random.rand() + ) + num_mask = max(min_masks, num_mask) + else: + sz = all_sz + num_mask = all_num_mask + + if mask_type.value is Wav2VecMaskType.static.value: + lengths = np.full(num_mask, mask_length) + elif mask_type.value is Wav2VecMaskType.uniform: + lengths = np.random.randint(mask_other, mask_length * 2 + 1, size=num_mask) + elif mask_type.value is Wav2VecMaskType.normal.value: + lengths = np.random.normal(mask_length, mask_other, size=num_mask) + lengths = [max(1, int(round(x))) for x in lengths] + elif mask_type.value is Wav2VecMaskType.poisson.value: + lengths = np.random.poisson(mask_length, size=num_mask) + lengths = [int(round(x)) for x in lengths] + else: + raise Exception("unknown mask selection " + str(mask_type)) + + if sum(lengths) == 0: + lengths[0] = min(mask_length, sz - 1) + + if no_overlap: + mask_idc = [] + + def arrange(s, e, length, keep_length): + span_start = np.random.randint(s, e - length) + mask_idc.extend(span_start + i for i in range(length)) + + new_parts = [] + if span_start - s - min_space >= keep_length: + new_parts.append((s, span_start - min_space + 1)) + if e - span_start - keep_length - min_space > keep_length: + new_parts.append((span_start + length + min_space, e)) + return new_parts + + parts = [(0, sz)] + min_length = min(lengths) + for length in sorted(lengths, reverse=True): + lens = np.fromiter((e - s if e - s >= length + min_space else 0 for s, e in parts), np.int) + l_sum = np.sum(lens) + if l_sum == 0: + break + probs = lens / np.sum(lens) + c = np.random.choice(len(parts), p=probs) + s, e = parts.pop(c) + parts.extend(arrange(s, e, length, min_length)) + mask_idc = np.asarray(mask_idc) + else: + min_len = min(lengths) + if sz - min_len <= num_mask: + min_len = sz - num_mask - 1 + + mask_idc = np.random.choice(sz - min_len, num_mask, replace=False) + + mask_idc = np.asarray([mask_idc[j] + offset for j in range(len(mask_idc)) for offset in range(lengths[j])]) + + mask_idcs.append(np.unique(mask_idc[mask_idc < sz])) + + min_len = min([len(m) for m in mask_idcs]) + for i, mask_idc in enumerate(mask_idcs): + if len(mask_idc) > min_len: + mask_idc = np.random.choice(mask_idc, min_len, replace=False) + mask[i, mask_idc] = True + + return mask diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/__init__.py new file mode 100644 index 0000000000..9e32500719 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/features.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/features.py new file mode 100644 index 0000000000..06665c413a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/features.py @@ -0,0 +1,39 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright (c) 2018 Ryan Leary +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# This file contains code artifacts adapted from https://github.com/ryanleary/patter + +""" +ALIAS FILE for backward compatibility +""" +from nemo.collections.asr.parts.preprocessing.features import * diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/mixins/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/mixins/__init__.py new file mode 100644 index 0000000000..42caf13f66 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/mixins/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.asr.parts.mixins.mixins import ASRBPEMixin, ASRModuleMixin, DiarizationMixin diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/mixins/mixins.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/mixins/mixins.py new file mode 100644 index 0000000000..35e689b04e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/mixins/mixins.py @@ -0,0 +1,193 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from abc import ABC, abstractmethod +from typing import List + +from omegaconf import DictConfig, OmegaConf, open_dict + +from nemo.collections.asr.parts.utils import asr_module_utils +from nemo.collections.common import tokenizers +from nemo.utils import logging + + +class ASRBPEMixin(ABC): + """ ASR BPE Mixin class that sets up a Tokenizer via a config + + This mixin class adds the method `_setup_tokenizer(...)`, which can be used by ASR models + which depend on subword tokenization. + + The setup_tokenizer method adds the following parameters to the class - + - tokenizer_cfg: The resolved config supplied to the tokenizer (with `dir` and `type` arguments). + - tokenizer_dir: The directory path to the tokenizer vocabulary + additional metadata. + - tokenizer_type: The type of the tokenizer. Currently supports `bpe` and `wpe`. + - vocab_path: Resolved path to the vocabulary text file. + + In addition to these variables, the method will also instantiate and preserve a tokenizer + (subclass of TokenizerSpec) if successful, and assign it to self.tokenizer. + """ + + def _setup_tokenizer(self, tokenizer_cfg: DictConfig): + # Prevent tokenizer parallelism (unless user has explicitly set it) + if 'TOKENIZERS_PARALLELISM' not in os.environ: + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + + self.tokenizer_cfg = OmegaConf.to_container(tokenizer_cfg, resolve=True) # type: dict + self.tokenizer_dir = self.tokenizer_cfg.pop('dir') # Remove tokenizer directory + self.tokenizer_type = self.tokenizer_cfg.pop('type').lower() # Remove tokenizer_type + + self.hf_tokenizer_kwargs = self.tokenizer_cfg.pop("hf_kwargs", {}) # Remove HF tokenizer kwargs + + # Preserve config + if hasattr(self, 'cfg') and 'tokenizer' in self.cfg: + self.cfg.tokenizer.dir = self.tokenizer_dir + self.cfg.tokenizer.type = self.tokenizer_type + + if 'hf_kwargs' in tokenizer_cfg: + with open_dict(self.cfg.tokenizer): + self.cfg.tokenizer.hf_kwargs = tokenizer_cfg.get('hf_kwargs') + + if self.tokenizer_type not in ['bpe', 'wpe']: + raise ValueError( + "`tokenizer.type` must be either `bpe` for SentencePiece tokenizer or " + "`wpe` for BERT based tokenizer" + ) + + if self.tokenizer_type == 'bpe': + # This is a BPE Tokenizer + if 'model_path' in self.tokenizer_cfg: + model_path = self.tokenizer_cfg.get('model_path') + else: + model_path = os.path.join(self.tokenizer_dir, 'tokenizer.model') + model_path = self.register_artifact('tokenizer.model_path', model_path) + self.model_path = model_path + + if 'special_tokens' in self.tokenizer_cfg: + special_tokens = self.tokenizer_cfg['special_tokens'] + else: + special_tokens = None + + # Update special tokens + self.tokenizer = tokenizers.SentencePieceTokenizer( + model_path=model_path, special_tokens=special_tokens, legacy=True + ) + + if 'vocab_path' in self.tokenizer_cfg: + vocab_path = self.tokenizer_cfg.get('vocab_path') + else: + vocab_path = os.path.join(self.tokenizer_dir, 'vocab.txt') + vocab_path = self.register_artifact('tokenizer.vocab_path', vocab_path) + self.vocab_path = vocab_path + + try: + if 'spe_tokenizer_vocab' in self.tokenizer_cfg: + spe_vocab_path = self.tokenizer_cfg.get('spe_tokenizer_vocab') + else: + spe_vocab_path = os.path.join(self.tokenizer_dir, 'tokenizer.vocab') + spe_vocab_path = self.register_artifact('tokenizer.spe_tokenizer_vocab', spe_vocab_path) + self.spe_vocab_path = spe_vocab_path + except FileNotFoundError: + # fallback case for older checkpoints that did not preserve the tokenizer.vocab + self.spe_vocab_path = None + + vocabulary = {'': 0} + with open(vocab_path) as f: + for i, piece in enumerate(f): + piece = piece.replace('\n', '') + vocabulary[piece] = i + 1 + + # wrapper method to get vocabulary conveniently + def get_vocab(): + return vocabulary + + # attach utility values to the tokenizer wrapper + self.tokenizer.tokenizer.vocab_size = len(vocabulary) + self.tokenizer.tokenizer.get_vocab = get_vocab + self.tokenizer.tokenizer.all_special_tokens = self.tokenizer.special_token_to_id + + else: + # This is a WPE Tokenizer + # If path from previous registration exists, remove it + if 'vocab_path' in self.tokenizer_cfg: + vocab_path = self.tokenizer_cfg.get('vocab_path') + else: + vocab_path = os.path.join(self.tokenizer_dir, 'vocab.txt') + vocab_path = self.register_artifact('tokenizer.vocab_path', vocab_path) + self.vocab_path = vocab_path + + # If path from previous registration exists, remove it + if 'vocab_path' in self.tokenizer_cfg: + self.tokenizer_cfg.pop('vocab_path') + + self.tokenizer = tokenizers.AutoTokenizer( + pretrained_model_name='bert-base-cased', + vocab_file=self.vocab_path, + mask_token=self.hf_tokenizer_kwargs.get('mask_token', None), + bos_token=self.hf_tokenizer_kwargs.get('bos_token', None), + eos_token=self.hf_tokenizer_kwargs.get('eos_token', None), + pad_token=self.hf_tokenizer_kwargs.get('pad_token', None), + sep_token=self.hf_tokenizer_kwargs.get('sep_token', None), + cls_token=self.hf_tokenizer_kwargs.get('cls_token', None), + unk_token=self.hf_tokenizer_kwargs.get('unk_token', None), + use_fast=self.hf_tokenizer_kwargs.get('use_fast', False), + ) + + logging.info( + "Tokenizer {} initialized with {} tokens".format( + self.tokenizer.__class__.__name__, self.tokenizer.vocab_size + ) + ) + + +class ASRModuleMixin(ABC): + """ + ASRModuleMixin is a mixin class added to ASR models in order to add methods that are specific + to a particular instantiation of a module inside of an ASRModel. + + Each method should first check that the module is present within the subclass, and support additional + functionality if the corresponding module is present. + """ + + def change_conv_asr_se_context_window(self, context_window: int, update_config: bool = True): + """ + Update the context window of the SqueezeExcitation module if the provided model contains an + `encoder` which is an instance of `ConvASREncoder`. + + Args: + context_window: An integer representing the number of input timeframes that will be used + to compute the context. Each timeframe corresponds to a single window stride of the + STFT features. + + Say the window_stride = 0.01s, then a context window of 128 represents 128 * 0.01 s + of context to compute the Squeeze step. + update_config: Whether to update the config or not with the new context window. + """ + asr_module_utils.change_conv_asr_se_context_window( + self, context_window=context_window, update_config=update_config + ) + + +class DiarizationMixin(ABC): + @abstractmethod + def diarize(self, paths2audio_files: List[str], batch_size: int = 1) -> List[str]: + """ + Takes paths to audio files and returns speaker labels + Args: + paths2audio_files: paths to audio fragment to be transcribed + + Returns: + Speaker labels + """ + pass diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/__init__.py new file mode 100644 index 0000000000..77a23cf78c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.asr.parts.numba.rnnt_loss.rnnt_pytorch import RNNTLossNumba diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/__init__.py new file mode 100644 index 0000000000..77c5b60fdf --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.asr.parts.numba.rnnt_loss.rnnt import rnnt_loss_cpu, rnnt_loss_gpu +from nemo.collections.asr.parts.numba.rnnt_loss.rnnt_pytorch import RNNTLossNumba diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt.py new file mode 100644 index 0000000000..5caf2fea2f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt.py @@ -0,0 +1,225 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2018-2019, Mingkun Huang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing + +import torch +from numba import cuda + +from nemo.collections.asr.parts.numba.rnnt_loss.utils import global_constants, rnnt_helper +from nemo.collections.asr.parts.numba.rnnt_loss.utils.cpu_utils import cpu_rnnt +from nemo.collections.asr.parts.numba.rnnt_loss.utils.cuda_utils import gpu_rnnt + + +def rnnt_loss_cpu( + acts: torch.Tensor, + labels: torch.Tensor, + input_lengths: torch.Tensor, + label_lengths: torch.Tensor, + costs: torch.Tensor, + grads: torch.Tensor, + blank_label: int, + num_threads: int, +): + """ + Wrapper method for accessing CPU RNNT loss. + + CPU implementation ported from [HawkAaron/warp-transducer](https://github.com/HawkAaron/warp-transducer). + + Args: + acts: Activation tensor of shape [B, T, U, V+1]. + labels: Ground truth labels of shape [B, U]. + input_lengths: Lengths of the acoustic sequence as a vector of ints [B]. + label_lengths: Lengths of the target sequence as a vector of ints [B]. + costs: Zero vector of length [B] in which costs will be set. + grads: Zero tensor of shape [B, T, U, V+1] where the gradient will be set. + blank_label: Index of the blank token in the vocabulary. + num_threads: Number of threads for OpenMP. + """ + # aliases + log_probs = acts + flat_labels = labels + + minibatch_size = log_probs.shape[0] + maxT = log_probs.shape[1] + maxU = log_probs.shape[2] + alphabet_size = log_probs.shape[3] + + if num_threads < 0: + num_threads = multiprocessing.cpu_count() + + num_threads = max(1, num_threads) # have to use at least 1 thread + + gpu_size, status = rnnt_helper.get_workspace_size(maxT, maxU, minibatch_size, gpu=False) + if status != global_constants.RNNTStatus.RNNT_STATUS_SUCCESS: + raise RuntimeError("Invalid parameter passed when calculating working space memory") + + cpu_workspace = torch.zeros(gpu_size, device=log_probs.device, dtype=log_probs.dtype, requires_grad=False) + + ### VIEW TENSORS AS VECTORS FOR POINTER INDEXING ### + log_probs, acts_shape = rnnt_helper.flatten_tensor(log_probs) + flat_labels, labels_shape = rnnt_helper.flatten_tensor(flat_labels) + + wrapper = cpu_rnnt.CPURNNT( + minibatch=minibatch_size, + maxT=maxT, + maxU=maxU, + alphabet_size=alphabet_size, + workspace=cpu_workspace, + blank=blank_label, + num_threads=num_threads, + batch_first=True, + ) + + if grads is None: + status = wrapper.score_forward( + log_probs=log_probs.data, + costs=costs, + flat_labels=flat_labels.data, + label_lengths=label_lengths.data, + input_lengths=input_lengths.data, + ) + + if status != global_constants.RNNTStatus.RNNT_STATUS_SUCCESS: + raise RuntimeError("Could not calculate forward scores") + + else: + ### FLATTEN GRAD TENSOR ### + grads, grads_shape = rnnt_helper.flatten_tensor(grads) + + status = wrapper.cost_and_grad( + log_probs=log_probs.data, + grads=grads.data, + costs=costs, + flat_labels=flat_labels.data, + label_lengths=label_lengths.data, + input_lengths=input_lengths.data, + ) + + if status != global_constants.RNNTStatus.RNNT_STATUS_SUCCESS: + raise RuntimeError("Could not calculate forward scores") + + del cpu_workspace, wrapper + return True + + +def rnnt_loss_gpu( + acts: torch.Tensor, + labels: torch.Tensor, + input_lengths: torch.Tensor, + label_lengths: torch.Tensor, + costs: torch.Tensor, + grads: torch.Tensor, + blank_label: int, + num_threads: int, +): + """ + Wrapper method for accessing GPU RNNT loss. + + CUDA implementation ported from [HawkAaron/warp-transducer](https://github.com/HawkAaron/warp-transducer). + + Args: + acts: Activation tensor of shape [B, T, U, V+1]. + labels: Ground truth labels of shape [B, U]. + input_lengths: Lengths of the acoustic sequence as a vector of ints [B]. + label_lengths: Lengths of the target sequence as a vector of ints [B]. + costs: Zero vector of length [B] in which costs will be set. + grads: Zero tensor of shape [B, T, U, V+1] where the gradient will be set. + blank_label: Index of the blank token in the vocabulary. + num_threads: Number of threads for OpenMP. + """ + minibatch_size = acts.shape[0] + maxT = acts.shape[1] + maxU = acts.shape[2] + alphabet_size = acts.shape[3] + + if hasattr(cuda, 'external_stream'): + stream = cuda.external_stream(torch.cuda.current_stream(acts.device).cuda_stream) + else: + stream = cuda.default_stream() + + if num_threads < 0: + num_threads = multiprocessing.cpu_count() + + num_threads = max(1, num_threads) # have to use at least 1 thread + + gpu_size, status = rnnt_helper.get_workspace_size(maxT, maxU, minibatch_size, gpu=True) + if status != global_constants.RNNTStatus.RNNT_STATUS_SUCCESS: + raise RuntimeError("Invalid parameter passed when calculating working space memory") + + # Select GPU index + cuda.select_device(acts.device.index) + gpu_workspace = torch.zeros(gpu_size, device=acts.device, dtype=acts.dtype, requires_grad=False) + + ### VIEW TENSORS AS VECTORS FOR POINTER INDEXING ### + acts, acts_shape = rnnt_helper.flatten_tensor(acts) + + ### REPRESENT THE CUDA ARRAY INTERFACE OF COSTS VECTOR ### + costs_repr = cuda.as_cuda_array(costs, sync=False) # NO COPY OF DATA, JUST CHANGE REPRESENTATION + + wrapper = gpu_rnnt.GPURNNT( + minibatch=minibatch_size, + maxT=maxT, + maxU=maxU, + alphabet_size=alphabet_size, + workspace=gpu_workspace, + blank=blank_label, + num_threads=num_threads, + stream=stream, + ) + + if grads is None: + status = wrapper.score_forward( + acts=acts.data, + costs=costs_repr, + pad_labels=labels.data, + label_lengths=label_lengths.data, + input_lengths=input_lengths.data, + ) + + if status != global_constants.RNNTStatus.RNNT_STATUS_SUCCESS: + raise RuntimeError("Could not calculate forward scores") + + else: + ### FLATTEN GRAD TENSOR ### + grads, grads_shape = rnnt_helper.flatten_tensor(grads) + + status = wrapper.cost_and_grad( + acts=acts.data, + grads=grads.data, + costs=costs_repr, + pad_labels=labels.data, + label_lengths=label_lengths.data, + input_lengths=input_lengths.data, + ) + + if status != global_constants.RNNTStatus.RNNT_STATUS_SUCCESS: + raise RuntimeError("Could not calculate forward scores") + + del gpu_workspace, wrapper + return True diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt_numpy.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt_numpy.py new file mode 100644 index 0000000000..aa814718cd --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt_numpy.py @@ -0,0 +1,213 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2018-2019, Mingkun Huang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import numpy as np +import torch +from torch.autograd import Function, Variable +from torch.nn import Module + + +def _assert_no_grad(tensor): + assert not tensor.requires_grad, ( + "gradients only computed for log_probs - please " "mark other tensors as not requiring gradients" + ) + + +def forward_pass(log_probs, labels, blank): + """ + Computes probability of the forward variable alpha. + + Args: + log_probs: Tensor of shape [T, U, V+1] + labels: Labels of shape [B, U] + blank: Index of the blank token. + + Returns: + A tuple of the forward variable probabilities - alpha of shape [T, U] + and the log likelihood of this forward step. + """ + T, U, _ = log_probs.shape + alphas = np.zeros((T, U), dtype='f') + + for t in range(1, T): + alphas[t, 0] = alphas[t - 1, 0] + log_probs[t - 1, 0, blank] + + for u in range(1, U): + alphas[0, u] = alphas[0, u - 1] + log_probs[0, u - 1, labels[u - 1]] + for t in range(1, T): + for u in range(1, U): + no_emit = alphas[t - 1, u] + log_probs[t - 1, u, blank] + emit = alphas[t, u - 1] + log_probs[t, u - 1, labels[u - 1]] + alphas[t, u] = np.logaddexp(emit, no_emit) + + loglike = alphas[T - 1, U - 1] + log_probs[T - 1, U - 1, blank] + return alphas, loglike + + +def backward_pass(log_probs, labels, blank): + """ + Computes probability of the backward variable beta. + + Args: + log_probs: Tensor of shape [T, U, V+1] + labels: Labels of shape [B, U] + blank: Index of the blank token. + + Returns: + A tuple of the backward variable probabilities - beta of shape [T, U] + and the log likelihood of this backward step. + """ + T, U, _ = log_probs.shape + betas = np.zeros((T, U), dtype='f') + betas[T - 1, U - 1] = log_probs[T - 1, U - 1, blank] + + for t in reversed(range(T - 1)): + betas[t, U - 1] = betas[t + 1, U - 1] + log_probs[t, U - 1, blank] + + for u in reversed(range(U - 1)): + betas[T - 1, u] = betas[T - 1, u + 1] + log_probs[T - 1, u, labels[u]] + + for t in reversed(range(T - 1)): + for u in reversed(range(U - 1)): + no_emit = betas[t + 1, u] + log_probs[t, u, blank] + emit = betas[t, u + 1] + log_probs[t, u, labels[u]] + betas[t, u] = np.logaddexp(emit, no_emit) + + return betas, betas[0, 0] + + +def compute_gradient(log_probs, alphas, betas, labels, blank): + """ + Computes the gradients of the log_probs with respect to the log probability of this step occuring. + + Args: + Args: + log_probs: Tensor of shape [T, U, V+1] + alphas: Tensor of shape [T, U] which represents the forward variable. + betas: Tensor of shape [T, U] which represents the backward variable. + labels: Labels of shape [B, U] + blank: Index of the blank token. + + Returns: + Gradients of shape [T, U, V+1] with respect to the forward log probability + """ + T, U, _ = log_probs.shape + grads = np.full(log_probs.shape, -float("inf")) + log_like = betas[0, 0] + + # // grad to last blank transition + grads[T - 1, U - 1, blank] = alphas[T - 1, U - 1] + + grads[: T - 1, :, blank] = alphas[: T - 1, :] + betas[1:, :] + for u, l in enumerate(labels): + grads[:, u, l] = alphas[:, u] + betas[:, u + 1] + + grads = -np.exp(grads + log_probs - log_like) + return grads + + +def transduce(log_probs, labels, blank=0): + """ + Args: + log_probs: 3D array with shape + [input len, output len + 1, vocab size] + labels: 1D array with shape [output time steps] + Returns: + float: The negative log-likelihood + 3D array: Gradients with respect to the + unnormalized input actications + """ + alphas, ll_forward = forward_pass(log_probs, labels, blank) + betas, ll_backward = backward_pass(log_probs, labels, blank) + grads = compute_gradient(log_probs, alphas, betas, labels, blank) + return -ll_forward, grads + + +def transduce_batch(log_probs, labels, flen, glen, blank=0): + """ + Compute the transducer loss of the batch. + + Args: + log_probs: [B, T, U, V+1]. Activation matrix normalized with log-softmax. + labels: [B, U+1] - ground truth labels with padded as blank token in the beginning. + flen: Length vector of the acoustic sequence. + glen: Length vector of the target sequence. + blank: Id of the blank token. + + Returns: + Batch of transducer forward log probabilities (loss) and the gradients of the activation matrix. + """ + grads = np.zeros_like(log_probs) + costs = [] + for b in range(log_probs.shape[0]): + t = int(flen[b]) + u = int(glen[b]) + 1 + ll, g = transduce(log_probs[b, :t, :u, :], labels[b, : u - 1], blank) + grads[b, :t, :u, :] = g + costs.append(ll) + return costs, grads + + +class _RNNT(Function): + @staticmethod + def forward(ctx, acts, labels, act_lens, label_lens, blank): + costs, grads = transduce_batch( + acts.detach().cpu().numpy(), labels.cpu().numpy(), act_lens.cpu().numpy(), label_lens.cpu().numpy(), blank, + ) + + costs = torch.FloatTensor([sum(costs)]) + grads = torch.Tensor(grads).to(acts) + + ctx.grads = Variable(grads) + return costs + + @staticmethod + def backward(ctx, grad_output): + return ctx.grads, None, None, None, None + + +class RNNTLoss(Module): + """ + Parameters: + `blank_label` (int): default 0 - label index of blank token + """ + + def __init__(self, blank: int = 0): + super(RNNTLoss, self).__init__() + self.blank = blank + self.rnnt = _RNNT.apply + + def forward(self, acts, labels, act_lens, label_lens): + assert len(labels.size()) == 2 + _assert_no_grad(labels) + _assert_no_grad(act_lens) + _assert_no_grad(label_lens) + + acts = torch.nn.functional.log_softmax(acts, -1) + return self.rnnt(acts, labels, act_lens, label_lens, self.blank) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt_pytorch.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt_pytorch.py new file mode 100644 index 0000000000..016f2dce9d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/rnnt_pytorch.py @@ -0,0 +1,186 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2018-2019, Mingkun Huang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import torch +from torch.autograd import Function +from torch.nn import Module + +from nemo.collections.asr.parts.numba.rnnt_loss import rnnt + +__all__ = ['rnnt_loss', 'RNNTLossNumba'] + + +class _RNNTNumba(Function): + @staticmethod + def forward(ctx, acts, labels, act_lens, label_lens, blank, reduction): + """ + log_probs: Tensor of (batch x seqLength x labelLength x outputDim) containing output from network + labels: 2 dimensional Tensor containing all the targets of the batch with zero padded + act_lens: Tensor of size (batch) containing size of each output sequence from the network + label_lens: Tensor of (batch) containing label length of each example + """ + is_cuda = acts.is_cuda + + certify_inputs(acts, labels, act_lens, label_lens) + + loss_func = rnnt.rnnt_loss_gpu if is_cuda else rnnt.rnnt_loss_cpu + grads = torch.zeros_like(acts) if acts.requires_grad else None + minibatch_size = acts.size(0) + costs = torch.zeros(minibatch_size, device=acts.device, dtype=acts.dtype) + + loss_func( + acts, + labels=labels, + input_lengths=act_lens, + label_lengths=label_lens, + costs=costs, + grads=grads, + blank_label=blank, + num_threads=0, + ) + + if reduction in ['sum', 'mean']: + costs = costs.sum().unsqueeze_(-1) + if reduction == 'mean': + costs /= minibatch_size + + if grads is not None: + grads /= minibatch_size + + # costs = costs.to(log_probs.device) + ctx.grads = grads + + return costs + + @staticmethod + def backward(ctx, grad_output): + if grad_output is not None and ctx.grads is not None: + grad_output = grad_output.view(-1, 1, 1, 1).to(ctx.grads) + return ctx.grads.mul_(grad_output), None, None, None, None, None + + +def rnnt_loss(acts, labels, act_lens, label_lens, blank=0, reduction='mean'): + """RNN Transducer Loss + Args: + acts: Tensor of (batch x seqLength x labelLength x outputDim) containing output from network + labels: 2 dimensional Tensor containing all the targets of the batch with zero padded + act_lens: Tensor of size (batch) containing size of each output sequence from the network + label_lens: Tensor of (batch) containing label length of each example + blank (int, optional): blank label. Default: 0. + reduction (string, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'none': no reduction will be applied, + 'mean': the output losses will be divided by the target lengths and + then the mean over the batch is taken. Default: 'mean' + """ + if not acts.is_cuda: + acts = torch.nn.functional.log_softmax(acts, -1) + + return _RNNTNumba.apply(acts, labels, act_lens, label_lens, blank, reduction) + + +class RNNTLossNumba(Module): + """ + Parameters: + blank (int, optional): blank label. Default: 0. + reduction (string, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'none': no reduction will be applied, + 'mean': the output losses will be divided by the target lengths and + then the mean over the batch is taken. Default: 'mean' + """ + + def __init__(self, blank=0, reduction='mean'): + super(RNNTLossNumba, self).__init__() + self.blank = blank + self.reduction = reduction + self.loss = _RNNTNumba.apply + + def forward(self, acts, labels, act_lens, label_lens): + """ + log_probs: Tensor of (batch x seqLength x labelLength x outputDim) containing output from network + labels: 2 dimensional Tensor containing all the targets of the batch with zero padded + act_lens: Tensor of size (batch) containing size of each output sequence from the network + label_lens: Tensor of (batch) containing label length of each example + """ + if not acts.is_cuda: + # NOTE manually done log_softmax for CPU version, + # log_softmax is computed within GPU version. + acts = torch.nn.functional.log_softmax(acts, -1) + + return self.loss(acts, labels, act_lens, label_lens, self.blank, self.reduction) + + +def check_type(var, t, name): + if var.dtype is not t: + raise TypeError("{} must be {}".format(name, t)) + + +def check_contiguous(var, name): + if not var.is_contiguous(): + raise ValueError("{} must be contiguous".format(name)) + + +def check_dim(var, dim, name): + if len(var.shape) != dim: + raise ValueError("{} must be {}D".format(name, dim)) + + +def certify_inputs(log_probs, labels, lengths, label_lengths): + # check_type(log_probs, torch.float32, "log_probs") + check_type(labels, torch.int32, "labels") + check_type(label_lengths, torch.int32, "label_lengths") + check_type(lengths, torch.int32, "lengths") + check_contiguous(log_probs, "log_probs") + check_contiguous(labels, "labels") + check_contiguous(label_lengths, "label_lengths") + check_contiguous(lengths, "lengths") + + if lengths.shape[0] != log_probs.shape[0]: + raise ValueError( + f"Must have a length per example. " + f"Given lengths dim: {lengths.shape[0]}, " + f"Log probs dim : {log_probs.shape[0]}" + ) + if label_lengths.shape[0] != log_probs.shape[0]: + raise ValueError( + "Must have a label length per example. " + f"Given label lengths dim : {label_lengths.shape[0]}, " + f"Log probs dim : {log_probs.shape[0]}" + ) + + check_dim(log_probs, 4, "log_probs") + check_dim(labels, 2, "labels") + check_dim(lengths, 1, "lenghts") + check_dim(label_lengths, 1, "label_lenghts") + max_T = torch.max(lengths) + max_U = torch.max(label_lengths) + T, U = log_probs.shape[1:3] + if T != max_T: + raise ValueError(f"Input length mismatch! Given T: {T}, Expected max T from input lengths: {max_T}") + if U != max_U + 1: + raise ValueError(f"Output length mismatch! Given U: {U}, Expected max U from target lengths: {max_U} + 1") diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cpu_utils/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cpu_utils/__init__.py new file mode 100644 index 0000000000..1b4bbd40df --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cpu_utils/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2018-2019, Mingkun Huang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cpu_utils/cpu_rnnt.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cpu_utils/cpu_rnnt.py new file mode 100644 index 0000000000..3e4cac7a09 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cpu_utils/cpu_rnnt.py @@ -0,0 +1,378 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2018-2019, Mingkun Huang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import multiprocessing +from typing import Optional + +import numba +import torch + +from nemo.collections.asr.parts.numba.rnnt_loss.utils import global_constants + + +def log_sum_exp(a: torch.Tensor, b: torch.Tensor): + """ + Logsumexp with safety checks for infs. + """ + if torch.isinf(a): + return b + + if torch.isinf(b): + return a + + if a > b: + return math.log1p(math.exp(b - a)) + a + else: + return math.log1p(math.exp(a - b)) + b + + +class CpuRNNT_index: + def __init__(self, U: int, maxU: int, minibatch: int, alphabet_size: int, batch_first: bool): + """ + A placeholder Index computation class that emits the resolved index in a flattened tensor, + mimicing pointer indexing in CUDA kernels on the CPU. + + Args: + U: Length of the current target sample (without padding). + maxU: Max Length of the padded target samples. + minibatch: Minibatch index + alphabet_size: Size of the vocabulary including RNNT blank - V+1. + batch_first: Bool flag determining if batch index is first or third. + """ + super(CpuRNNT_index, self).__init__() + self.U = U + self.maxU = maxU + self.minibatch = minibatch + self.alphabet_size = alphabet_size + self.batch_first = batch_first + + def __call__(self, t: int, u: int, v: Optional[int] = None): + # if indexing all the values of the vocabulary, then only t, u are provided + if v is None: + return t * self.U + u + else: + # otherwise, t, u, v are provided to index particular value in the vocabulary. + if self.batch_first: + return (t * self.maxU + u) * self.alphabet_size + v + else: + return (t * self.maxU + u) * self.minibatch * self.alphabet_size + v + + +class CpuRNNT_metadata: + def __init__( + self, + T: int, + U: int, + workspace: torch.Tensor, + bytes_used: int, + blank: int, + labels: torch.Tensor, + log_probs: torch.Tensor, + idx: CpuRNNT_index, + ): + """ + Metadata for CPU based RNNT loss calculation. Holds the working space memory. + + Args: + T: Length of the acoustic sequence (without padding). + U: Length of the target sequence (without padding). + workspace: Working space memory for the CPU. + bytes_used: Number of bytes currently used for indexing the working space memory. Generally 0. + blank: Index of the blank token in the vocabulary. + labels: Ground truth padded labels matrix of shape [B, U] + log_probs: Log probs / activation matrix of flattented shape [B, T, U, V+1] + idx: + """ + super(CpuRNNT_metadata, self).__init__() + + self.alphas = workspace[bytes_used : bytes_used + T * U] + bytes_used += T * U + + self.betas = workspace[bytes_used : bytes_used + T * U] + bytes_used += T * U + + self.log_probs2 = workspace[bytes_used : bytes_used + T * U * 2] # // only store blank & label + bytes_used += T * U * 2 + + self.bytes_used = bytes_used + + self.setup_probs(T, U, labels, blank, log_probs, idx) + + def setup_probs( + self, T: int, U: int, labels: torch.Tensor, blank: int, log_probs: torch.Tensor, idx: CpuRNNT_index + ): + # initialize the log probs memory for blank and label token. + for t in range(T): + for u in range(U): + offset = (t * U + u) * 2 # mult with 2 is for selecting either blank or label token. Odd idx is blank. + self.log_probs2[offset] = log_probs[idx(t, u, blank)] + # // labels do not have first blank + if u < U - 1: + self.log_probs2[offset + 1] = log_probs[idx(t, u, labels[u])] + + +class CPURNNT: + def __init__( + self, + minibatch: int, + maxT: int, + maxU: int, + alphabet_size: int, + workspace: torch.Tensor, + blank: int, + num_threads: int, + batch_first: bool, + ): + """ + Helper class to compute the Transducer Loss on CPU. + + Args: + minibatch: Size of the minibatch b. + maxT: The maximum possible acoustic sequence length. Represents T in the logprobs tensor. + maxU: The maximum possible target sequence length. Represents U in the logprobs tensor. + alphabet_size: The vocabulary dimension V+1 (inclusive of RNNT blank). + workspace: An allocated chunk of memory that will be sliced off and reshaped into required + blocks used as working memory. + blank: Index of the RNNT blank token in the vocabulary. Generally the first or last token in the vocab. + num_threads: Number of OMP threads to launch. + batch_first: Bool that decides if batch dimension is first or third. + """ + self.minibatch_ = minibatch + self.maxT_ = maxT + self.maxU_ = maxU + self.alphabet_size_ = alphabet_size + self.workspace = workspace # a flat vector of floatX numbers that represents allocated memory slices + self.blank_ = blank + self.num_threads_ = num_threads + self.batch_first = batch_first + + if num_threads > 0: + numba.set_num_threads(min(multiprocessing.cpu_count(), num_threads)) + else: + self.num_threads_ = numba.get_num_threads() + + def cost_and_grad_kernel( + self, + log_probs: torch.Tensor, + grad: torch.Tensor, + labels: torch.Tensor, + mb: int, + T: int, + U: int, + bytes_used: int, + ): + idx = CpuRNNT_index(U, self.maxU_, self.minibatch_, self.alphabet_size_, self.batch_first) + rnntm = CpuRNNT_metadata(T, U, self.workspace, bytes_used, self.blank_, labels, log_probs, idx) + + if self.batch_first: + # zero grads + grad *= 0.0 + + llForward = self.compute_alphas(rnntm.log_probs2, T, U, rnntm.alphas) + llBackward = self.compute_betas_and_grads( + grad, rnntm.log_probs2, T, U, rnntm.alphas, rnntm.betas, labels, llForward + ) + + diff = (llForward - llBackward).abs() + if diff > 0.1: + print(f"WARNING: Forward backward likelihood mismatch : {diff}") + + return -llForward + + def compute_alphas(self, log_probs: torch.Tensor, T: int, U: int, alphas: torch.Tensor): + """ + Compute the probability of the forward variable alpha. + + Args: + log_probs: Flattened tensor [B, T, U, V+1] + T: Length of the acoustic sequence T (not padded). + U: Length of the target sequence U (not padded). + alphas: Working space memory for alpha of shape [B, T, U]. + + Returns: + Loglikelihood of the forward variable alpha. + """ + idx = CpuRNNT_index(U, self.maxU_, self.minibatch_, self.alphabet_size_, self.batch_first) + + alphas[0] = 0 + for t in range(T): + for u in range(U): + if u == 0 and t > 0: + alphas[idx(t, 0)] = alphas[idx(t - 1, 0)] + log_probs[idx(t - 1, 0) * 2] + + if t == 0 and u > 0: + alphas[idx(0, u)] = alphas[idx(0, u - 1)] + log_probs[idx(0, u - 1) * 2 + 1] + + if t > 0 and u > 0: + no_emit = alphas[idx(t - 1, u)] + log_probs[idx(t - 1, u) * 2] + emit = alphas[idx(t, u - 1)] + log_probs[idx(t, u - 1) * 2 + 1] + alphas[idx(t, u)] = log_sum_exp(emit, no_emit) + + loglike = alphas[idx(T - 1, U - 1)] + log_probs[idx(T - 1, U - 1) * 2] + return loglike + + def compute_betas_and_grads( + self, + grad: torch.Tensor, + log_probs: torch.Tensor, + T: int, + U: int, + alphas: torch.Tensor, + betas: torch.Tensor, + labels: torch.Tensor, + logll: torch.Tensor, + ): + """ + Compute backward variable beta as well as gradients of the activation matrix wrt loglikelihood + of forward variable. + + Args: + grad: Working space memory of flattened shape [B, T, U, V+1] + log_probs: Activatio tensor of flattented shape [B, T, U, V+1] + T: Length of the acoustic sequence T (not padded). + U: Length of the target sequence U (not padded). + alphas: Working space memory for alpha of shape [B, T, U]. + betas: Working space memory for alpha of shape [B, T, U]. + labels: Ground truth label of shape [B, U] + logll: Loglikelihood of the forward variable. + + Returns: + Loglikelihood of the forward variable and inplace updates the grad tensor. + """ + idx = CpuRNNT_index(U, self.maxU_, self.minibatch_, self.alphabet_size_, self.batch_first) + betas[idx(T - 1, U - 1)] = log_probs[idx(T - 1, U - 1) * 2] + + for t in range(T - 1, -1, -1): + for u in range(U - 1, -1, -1): + if (u == U - 1) and (t < T - 1): + betas[idx(t, U - 1)] = betas[idx(t + 1, U - 1)] + log_probs[idx(t, U - 1) * 2] + + if (t == T - 1) and (u < U - 1): + betas[idx(T - 1, u)] = betas[idx(T - 1, u + 1)] + log_probs[idx(T - 1, u) * 2 + 1] + + if (t < T - 1) and (u < U - 1): + no_emit = betas[idx(t + 1, u)] + log_probs[idx(t, u) * 2] + emit = betas[idx(t, u + 1)] + log_probs[idx(t, u) * 2 + 1] + betas[idx(t, u)] = log_sum_exp(emit, no_emit) + + loglike = betas[0] + # // Gradients w.r.t. log probabilities + for t in range(T): + for u in range(U): + if t < T - 1: + g = alphas[idx(t, u)] + betas[idx(t + 1, u)] + grad[idx(t, u, self.blank_)] = -torch.exp(log_probs[idx(t, u) * 2] + g - loglike) + + if u < U - 1: + g = alphas[idx(t, u)] + betas[idx(t, u + 1)] + grad[idx(t, u, labels[u])] = -torch.exp(log_probs[idx(t, u) * 2 + 1] + g - loglike) + + # // gradient to the last blank transition + grad[idx(T - 1, U - 1, self.blank_)] = -torch.exp( + log_probs[idx(T - 1, U - 1) * 2] + alphas[idx(T - 1, U - 1)] - loglike + ) + + return loglike + + def cost_and_grad( + self, + log_probs: torch.Tensor, + grads: torch.Tensor, + costs: torch.Tensor, + flat_labels: torch.Tensor, + label_lengths: torch.Tensor, + input_lengths: torch.Tensor, + ) -> global_constants.RNNTStatus: + # // per minibatch memory + per_minibatch_bytes = 0 + + # // alphas & betas + per_minibatch_bytes += self.maxT_ * self.maxU_ * 2 + + # // blank & label log probability cache + per_minibatch_bytes += self.maxT_ * self.maxU_ * 2 + + for mb in range(self.minibatch_): + T = input_lengths[mb] # // Length of utterance (time) + U = label_lengths[mb] + 1 # // Number of labels in transcription + batch_size = self.alphabet_size_ + if self.batch_first: + batch_size = self.maxT_ * self.maxU_ * self.alphabet_size_ + + costs[mb] = self.cost_and_grad_kernel( + log_probs[(mb * batch_size) :], + grads[(mb * batch_size) :], + flat_labels[(mb * (self.maxU_ - 1)) :], + mb, + T, + U, + mb * per_minibatch_bytes, + ) + + return global_constants.RNNTStatus.RNNT_STATUS_SUCCESS + + def score_forward( + self, + log_probs: torch.Tensor, + costs: torch.Tensor, + flat_labels: torch.Tensor, + label_lengths: torch.Tensor, + input_lengths: torch.Tensor, + ): + # // per minibatch memory + per_minibatch_bytes = 0 + + # // alphas & betas + per_minibatch_bytes += self.maxT_ * self.maxU_ * 2 + + # // blank & label log probability cache + per_minibatch_bytes += self.maxT_ * self.maxU_ * 2 + + for mb in range(self.minibatch_): + T = input_lengths[mb] # // Length of utterance (time) + U = label_lengths[mb] + 1 # // Number of labels in transcription + batch_size = self.alphabet_size_ + if self.batch_first: + batch_size = self.maxT_ * self.maxU_ * self.alphabet_size_ + + idx = CpuRNNT_index(U, self.maxU_, self.minibatch_, self.alphabet_size_, self.batch_first) + rnntm = CpuRNNT_metadata( + T, + U, + self.workspace, + mb * per_minibatch_bytes, + self.blank_, + flat_labels[(mb * (self.maxU_ - 1)) :], + log_probs[(mb * batch_size) :], + idx, + ) + + costs[mb] = -self.compute_alphas(rnntm.log_probs2, T, U, rnntm.alphas) + + return global_constants.RNNTStatus.RNNT_STATUS_SUCCESS diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/__init__.py new file mode 100644 index 0000000000..1b4bbd40df --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2018-2019, Mingkun Huang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/gpu_rnnt.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/gpu_rnnt.py new file mode 100644 index 0000000000..ce940c3b39 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/gpu_rnnt.py @@ -0,0 +1,254 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2018-2019, Mingkun Huang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing +from typing import Optional + +import numba +import torch +from numba import cuda + +from nemo.collections.asr.parts.numba.rnnt_loss.utils import global_constants +from nemo.collections.asr.parts.numba.rnnt_loss.utils.cuda_utils import gpu_rnnt_kernel, reduce + + +class GPURNNT: + def __init__( + self, minibatch: int, maxT: int, maxU: int, alphabet_size: int, workspace, blank: int, num_threads: int, stream + ): + """ + Helper class to launch the CUDA Kernels to compute the Transducer Loss. + + Args: + minibatch: Int representing the batch size. + maxT: The maximum possible acoustic sequence length. Represents T in the logprobs tensor. + maxU: The maximum possible target sequence length. Represents U in the logprobs tensor. + alphabet_size: The vocabulary dimension V+1 (inclusive of RNNT blank). + workspace: An allocated chunk of memory that will be sliced off and reshaped into required + blocks used as working memory. + blank: Index of the RNNT blank token in the vocabulary. Generally the first or last token in the vocab. + num_threads: Number of OMP threads to launch. + stream: Numba Cuda Stream. + """ + self.minibatch_ = minibatch + self.maxT_ = maxT + self.maxU_ = maxU + self.alphabet_size_ = alphabet_size + self.gpu_workspace = cuda.as_cuda_array( + workspace + ) # a flat vector of floatX numbers that represents allocated memory slices + self.blank_ = blank + self.num_threads_ = num_threads + self.stream_ = stream # type: cuda.cudadrv.driver.Stream + + if num_threads > 0: + numba.set_num_threads(min(multiprocessing.cpu_count(), num_threads)) + else: + self.num_threads_ = numba.get_num_threads() + + def log_softmax(self, acts: torch.Tensor, denom: torch.Tensor): + """ + Computes the log softmax denominator of the input activation tensor + and stores the result in denom. + + Args: + acts: Activation tensor of shape [B, T, U, V+1]. The input must be represented as a flat tensor + of shape [B * T * U * (V+1)] to allow pointer indexing. + denom: A zero tensor of same shape as acts. + + Updates: + This kernel inplace updates the `denom` tensor + """ + # // trans_acts + pred_acts -> log_softmax denominator + reduce.reduce_max( + acts, + denom, + rows=self.alphabet_size_, + cols=self.minibatch_ * self.maxT_ * self.maxU_, + minus=False, + stream=self.stream_, + ) + + reduce.reduce_exp( + acts, + denom, + rows=self.alphabet_size_, + cols=self.minibatch_ * self.maxT_ * self.maxU_, + minus=True, + stream=self.stream_, + ) + + def compute_cost_and_score( + self, + acts: torch.Tensor, + grads: Optional[torch.Tensor], + costs: torch.Tensor, + labels: torch.Tensor, + label_lengths: torch.Tensor, + input_lengths: torch.Tensor, + ) -> global_constants.RNNTStatus: + """ + Compute both the loss and the gradients. + + Args: + acts: A flattened tensor of shape [B, T, U, V+1] representing the activation matrix. + grad: A flattented zero tensor of same shape as acts. + costs: A zero vector of length B which will be updated inplace with the log probability costs. + flat_labels: A flattened matrix of labels of shape [B, U] + label_lengths: A vector of length B that contains the original lengths of the acoustic sequence. + input_lengths: A vector of length B that contains the original lengths of the target sequence. + + Updates: + This will launch kernels that will update inline the following variables: + - grads: Gradients of the activation matrix wrt the costs vector. + - costs: Negative log likelihood of the forward variable. + + Returns: + An enum that either represents a successful RNNT operation or failure. + """ + training = grads is not None + used_offset = 0 + + # // denom + denom = self.gpu_workspace[used_offset : used_offset + self.maxT_ * self.maxU_ * self.minibatch_] + used_offset += self.maxT_ * self.maxU_ * self.minibatch_ + + # // alphas & betas + alphas = self.gpu_workspace[used_offset : used_offset + self.maxT_ * self.maxU_ * self.minibatch_] + used_offset += self.maxT_ * self.maxU_ * self.minibatch_ + betas = self.gpu_workspace[used_offset : used_offset + self.maxT_ * self.maxU_ * self.minibatch_] + used_offset += self.maxT_ * self.maxU_ * self.minibatch_ + + # // logllh + llForward = self.gpu_workspace[used_offset : used_offset + self.minibatch_] + used_offset += self.minibatch_ + llBackward = self.gpu_workspace[used_offset : used_offset + self.minibatch_] + used_offset += self.minibatch_ + + if training: + grads *= 0.0 # zero grads + + ######## START EXECUTION ######## + self.log_softmax(acts, denom) + + # Compute alphas + gpu_rnnt_kernel.compute_alphas_kernel[self.minibatch_, self.maxU_, self.stream_, 0]( + acts, + denom, + alphas, + llForward, + input_lengths, + label_lengths, + labels, + self.minibatch_, + self.maxT_, + self.maxU_, + self.alphabet_size_, + self.blank_, + ) + + if training: + # Compute betas + gpu_rnnt_kernel.compute_betas_kernel[self.minibatch_, self.maxU_, self.stream_, 0]( + acts, + denom, + betas, + llBackward, + input_lengths, + label_lengths, + labels, + self.minibatch_, + self.maxT_, + self.maxU_, + self.alphabet_size_, + self.blank_, + ) + + # Compute gradient + grad_blocks_per_grid = self.minibatch_ * self.maxT_ * self.maxU_ + grad_threads_per_block = gpu_rnnt_kernel.GPU_RNNT_THREAD_SIZE + gpu_rnnt_kernel.compute_grad_kernel[grad_blocks_per_grid, grad_threads_per_block, self.stream_, 0]( + grads, + acts, + denom, + alphas, + betas, + llForward, + input_lengths, + label_lengths, + labels, + self.minibatch_, + self.maxT_, + self.maxU_, + self.alphabet_size_, + self.blank_, + ) + + # // cost + costs.copy_to_device(llForward, stream=self.stream_) + self.stream_.synchronize() + + # compute negative log likelihood. + for mb in range(self.minibatch_): + costs[mb] = -costs[mb] + + return global_constants.RNNTStatus.RNNT_STATUS_SUCCESS + + def cost_and_grad( + self, + acts: torch.Tensor, + grads: torch.Tensor, + costs: torch.Tensor, + pad_labels: torch.Tensor, + label_lengths: torch.Tensor, + input_lengths: torch.Tensor, + ): + if ( + acts is None + or grads is None + or costs is None + or pad_labels is None + or label_lengths is None + or input_lengths is None + ): + return global_constants.RNNTStatus.RNNT_STATUS_INVALID_VALUE + + return self.compute_cost_and_score(acts, grads, costs, pad_labels, label_lengths, input_lengths) + + def score_forward( + self, + acts: torch.Tensor, + costs: torch.Tensor, + pad_labels: torch.Tensor, + label_lengths: torch.Tensor, + input_lengths: torch.Tensor, + ): + if acts is None or costs is None or pad_labels is None or label_lengths is None or input_lengths is None: + return global_constants.RNNTStatus.RNNT_STATUS_INVALID_VALUE + + return self.compute_cost_and_score(acts, None, costs, pad_labels, label_lengths, input_lengths) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/gpu_rnnt_kernel.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/gpu_rnnt_kernel.py new file mode 100644 index 0000000000..7f7d0227aa --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/cuda_utils/gpu_rnnt_kernel.py @@ -0,0 +1,365 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2018-2019, Mingkun Huang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import torch +from numba import cuda + +from nemo.collections.asr.parts.numba.rnnt_loss.utils import rnnt_helper + +GPU_RNNT_THREAD_SIZE = 128 + + +@cuda.jit(device=True, inline=True) +def logp( + denom: torch.Tensor, acts: torch.Tensor, maxT: int, maxU: int, alphabet_size: int, mb: int, t: int, u: int, v: int +): + """ + Compute the sum of log probability from the activation tensor and its denominator. + + Args: + denom: Tensor of shape [B, T, U] flattened. Represents the denominator of the logprobs activation tensor + across entire vocabulary. + acts: Tensor of shape [B, T, U, V+1] flattened. Represents the logprobs activation tensor. + maxT: The maximum possible acoustic sequence length. Represents T in the logprobs tensor. + maxU: The maximum possible target sequence length. Represents U in the logprobs tensor. + alphabet_size: The vocabulary dimension V+1 (inclusive of RNNT blank). + mb: Batch indexer. + t: Acoustic sequence timestep indexer. + u: Target sequence timestep indexer. + v: Vocabulary token indexer. + + Returns: + The sum of logprobs[mb, t, u, v] + denom[mb, t, u] + """ + col = (mb * maxT + t) * maxU + u + return denom[col] + acts[col * alphabet_size + v] + + +@cuda.jit() +def compute_alphas_kernel( + acts: torch.Tensor, + denom: torch.Tensor, + alphas: torch.Tensor, + llForward: torch.Tensor, + xlen: torch.Tensor, + ylen: torch.Tensor, + mlabels: torch.Tensor, # [B] + minibatch: int, + maxT: int, + maxU: int, + alphabet_size: int, + blank_: int, +): + """ + Compute alpha (forward variable) probabilities over the transduction step. + + Args: + acts: Tensor of shape [B, T, U, V+1] flattened. Represents the logprobs activation tensor. + denom: Tensor of shape [B, T, U] flattened. Represents the denominator of the logprobs activation tensor + across entire vocabulary. + alphas: Zero tensor of shape [B, T, U]. Will be updated inside the kernel with the forward variable + probabilities. + llForward: Zero tensor of shape [B]. Represents the log-likelihood of the forward pass. + Returned as the forward pass loss that is reduced by the optimizer. + xlen: Vector of length B which contains the actual acoustic sequence lengths in the padded + activation tensor. + ylen: Vector of length B which contains the actual target sequence lengths in the padded + activation tensor. + mlabels: Matrix of shape [B, U+1] (+1 here is due to token - usually the RNNT blank). + The matrix contains the padded target transcription that must be predicted. + minibatch: Int representing the batch size. + maxT: The maximum possible acoustic sequence length. Represents T in the logprobs tensor. + maxU: The maximum possible target sequence length. Represents U in the logprobs tensor. + alphabet_size: The vocabulary dimension V+1 (inclusive of RNNT blank). + blank_: Index of the RNNT blank token in the vocabulary. Generally the first or last token in the vocab. + + Updates: + Kernel inplace updates the following inputs: + - alphas: forward variable scores. + - llForward: log-likelihood of forward variable. + """ + # // launch B blocks, each block has U threads + b = cuda.blockIdx.x # // batch id + u = cuda.threadIdx.x # label id, u + T = xlen[b] # select AM length of current sample + U = ylen[b] + 1 # select target length of current sample, +1 for the blank token + + labels: torch.Tensor = mlabels[b] # mb label start point, equivalent to mlabels + b * (maxU - 1) + offset = b * maxT * maxU # pointer indexing offset + + # alphas += offset # pointer offset, ignored since we explicitly add offset + + # Initilize alpha[b, t=0, u=0] for all b in B + if u == 0: + alphas[offset] = 0 + + # sync until all alphas are initialized + cuda.syncthreads() + + # Ordinary alpha calculations, broadcast across B=b and U=u + # Look up forward variable calculation from rnnt_numpy.forward_pass() + for n in range(1, T + U - 1): + t = n - u + + if u == 0: + # for t in range(1, T) step to initialize alphas[b, t, 0] + if t > 0 and t < T: + alphas[offset + t * maxU + u] = alphas[offset + (t - 1) * maxU + u] + logp( + denom, acts, maxT, maxU, alphabet_size, b, t - 1, 0, blank_ + ) + elif u < U: + # for u in range(1, U) step to initialize alphas[b, 0, u] + if t == 0: + alphas[offset + u] = alphas[offset + u - 1] + logp( + denom, acts, maxT, maxU, alphabet_size, b, 0, u - 1, labels[u - 1] + ) + + # for t in range(1, T) for u in range(1, U) step to compute alphas[b, t, u] + elif t > 0 and t < T: + no_emit = alphas[offset + (t - 1) * maxU + u] + logp( + denom, acts, maxT, maxU, alphabet_size, b, t - 1, u, blank_ + ) + emit = alphas[offset + t * maxU + u - 1] + logp( + denom, acts, maxT, maxU, alphabet_size, b, t, u - 1, labels[u - 1] + ) + + alphas[offset + t * maxU + u] = rnnt_helper.log_sum_exp(emit, no_emit) + + # sync across all B=b and U=u + cuda.syncthreads() + + # After final sync, alphas[b, T-1, U - 1] + logprobs[b, T-1, U-1, blank] + denom[b, T-1, U-1] gives + # log-likelihood of forward pass. + if u == 0: + loglike = alphas[offset + (T - 1) * maxU + U - 1] + logp( + denom, acts, maxT, maxU, alphabet_size, b, T - 1, U - 1, blank_ + ) + llForward[b] = loglike + + +@cuda.jit() +def compute_betas_kernel( + acts: torch.Tensor, + denom: torch.Tensor, + betas: torch.Tensor, + llBackward: torch.Tensor, + xlen: torch.Tensor, + ylen: torch.Tensor, + mlabels: torch.Tensor, # [B, U] + minibatch: int, + maxT: int, + maxU: int, + alphabet_size: int, + blank_: int, +): + """ + Compute beta (backward variable) probabilities over the transduction step. + + Args: + acts: Tensor of shape [B, T, U, V+1] flattened. Represents the logprobs activation tensor. + denom: Tensor of shape [B, T, U] flattened. Represents the denominator of the logprobs activation tensor + across entire vocabulary. + betas: Zero tensor of shape [B, T, U]. Will be updated inside the kernel with the backward variable + probabilities. + llBackward: Zero tensor of shape [B]. Represents the log-likelihood of the backward pass. + Returned as the backward pass loss that is reduced by the optimizer. + xlen: Vector of length B which contains the actual acoustic sequence lengths in the padded + activation tensor. + ylen: Vector of length B which contains the actual target sequence lengths in the padded + activation tensor. + mlabels: Matrix of shape [B, U+1] (+1 here is due to token - usually the RNNT blank). + The matrix contains the padded target transcription that must be predicted. + minibatch: Int representing the batch size. + maxT: The maximum possible acoustic sequence length. Represents T in the logprobs tensor. + maxU: The maximum possible target sequence length. Represents U in the logprobs tensor. + alphabet_size: The vocabulary dimension V+1 (inclusive of RNNT blank). + blank_: Index of the RNNT blank token in the vocabulary. Generally the first or last token in the vocab. + + Updates: + Kernel inplace updates the following inputs: + - betas: backward variable scores. + - llBackward: log-likelihood of backward variable. + """ + # // launch B blocks, each block has U threads + b = cuda.blockIdx.x # // batch id + u = cuda.threadIdx.x # label id, u + T = xlen[b] # select AM length of current sample + U = ylen[b] + 1 # select target length of current sample, +1 for the blank token + + labels: torch.Tensor = mlabels[b] # mb label start point, equivalent to mlabels + b * (maxU - 1) + offset = b * maxT * maxU # pointer indexing offset + + # betas += offset # pointer offset, ignored since we explicitly add offset + + # Initilize beta[b, t=T-1, u=U-1] for all b in B with log_probs[b, t=T-1, u=U-1, blank] + if u == 0: + betas[offset + (T - 1) * maxU + U - 1] = logp(denom, acts, maxT, maxU, alphabet_size, b, T - 1, U - 1, blank_) + + # sync until all betas are initialized + cuda.syncthreads() + + # Ordinary beta calculations, broadcast across B=b and U=u + # Look up backward variable calculation from rnnt_numpy.backward_pass() + for n in range(T + U - 2, -1, -1): + t = n - u + + if u == (U - 1): + # for t in reversed(range(T - 1)) step to initialize betas[b, t, U-1] + if t >= 0 and t < (T - 1): + betas[offset + t * maxU + U - 1] = betas[offset + (t + 1) * maxU + U - 1] + logp( + denom, acts, maxT, maxU, alphabet_size, b, t, U - 1, blank_ + ) + elif u < U: + if t == T - 1: + # for u in reversed(range(U - 1)) step to initialize betas[b, T-1, u] + betas[offset + (T - 1) * maxU + u] = betas[offset + (T - 1) * maxU + u + 1] + logp( + denom, acts, maxT, maxU, alphabet_size, b, T - 1, u, labels[u] + ) + elif (t >= 0) and (t < T - 1): + # for t in reversed(range(T - 1)) for u in reversed(range(U - 1)) step to compute betas[b, t, u] + no_emit = betas[offset + (t + 1) * maxU + u] + logp( + denom, acts, maxT, maxU, alphabet_size, b, t, u, blank_ + ) + emit = betas[offset + t * maxU + u + 1] + logp( + denom, acts, maxT, maxU, alphabet_size, b, t, u, labels[u] + ) + betas[offset + t * maxU + u] = rnnt_helper.log_sum_exp(emit, no_emit) + + # sync across all B=b and U=u + cuda.syncthreads() + + # After final sync, betas[b, 0, 0] gives + # log-likelihood of backward pass. + if u == 0: + llBackward[b] = betas[offset] + + +@cuda.jit() +def compute_grad_kernel( + grads: torch.Tensor, + acts: torch.Tensor, + denom: torch.Tensor, + alphas: torch.Tensor, + betas: torch.Tensor, + logll: torch.Tensor, + xlen: torch.Tensor, + ylen: torch.Tensor, + mlabels: torch.Tensor, # [B, U] + minibatch: int, + maxT: int, + maxU: int, + alphabet_size: int, + blank_: int, +): + """ + Compute gradients over the transduction step. + + Args: + grads: Zero Tensor of shape [B, T, U, V+1]. Is updated by this kernel to contain the gradients + of this batch of samples. + acts: Tensor of shape [B, T, U, V+1] flattened. Represents the logprobs activation tensor. + denom: Tensor of shape [B, T, U] flattened. Represents the denominator of the logprobs activation tensor + across entire vocabulary. + alphas: Alpha variable, contains forward probabilities. A tensor of shape [B, T, U]. + betas: Beta varoable, contains backward probabilities. A tensor of shape [B, T, U]. + logll: Log-likelihood of the forward variable, represented as a vector of shape [B]. + Represents the log-likelihood of the forward pass. + xlen: Vector of length B which contains the actual acoustic sequence lengths in the padded + activation tensor. + ylen: Vector of length B which contains the actual target sequence lengths in the padded + activation tensor. + mlabels: Matrix of shape [B, U+1] (+1 here is due to token - usually the RNNT blank). + The matrix contains the padded target transcription that must be predicted. + minibatch: Int representing the batch size. + maxT: The maximum possible acoustic sequence length. Represents T in the logprobs tensor. + maxU: The maximum possible target sequence length. Represents U in the logprobs tensor. + alphabet_size: The vocabulary dimension V+1 (inclusive of RNNT blank). + blank_: Index of the RNNT blank token in the vocabulary. Generally the first or last token in the vocab. + + Updates: + Kernel inplace updates the following inputs: + - grads: Gradients with respect to the log likelihood (logll). + """ + # Kernel call: + # blocks_per_grid = minibatch (b) * maxT (t) * maxU (u) + # threads_per_block = constant buffer size of parallel threads (v :: Constant) + tid = cuda.threadIdx.x # represents v, taking steps of some constant size + idx = tid # index of v < V+1; in steps of constant buffer size + col = cuda.blockIdx.x # represents a fused index of b * t * u + + # Decompose original indices from fused `col` + u = col % maxU # (b * t * u) % u = u + bt = (col - u) // maxU # (b * t * u - u) // U = b * t + t = bt % maxT # (b * t) % t = t + mb = (bt - t) // maxT # (b * t - t) // T = b + + # constants + T = xlen[mb] # select AM length of current sample + U = ylen[mb] + 1 # select target length of current sample, +1 for the blank token + labels: torch.Tensor = mlabels[mb] # labels = mlabels + mb * (maxU - 1); + + # Buffered gradient calculations, broadcast across B=b, T=t and U=u, looped over V with some constant stride. + # Look up gradient calculation from rnnt_numpy.compute_gradient() + if t < T and u < U: + # For cuda kernels, maximum number of threads per block is limited to some value. + # However, it may be the case that vocabulary size is larger than this limit + # To work around this, an arbitrary thread buffer size is chosen such that, + # 1) each element within the thread pool operates independently of the other + # 2) An inner while loop moves the index of each buffer element by the size of the buffer itself, + # such that all elements of the vocabulary size are covered in (V + 1 // thread_buffer) number of steps. + # As such, each thread will perform the while loop at least (V + 1 // thread_buffer) number of times + while idx < alphabet_size: + # remember, `col` represents the tri-index [b, t, u] + # therefore; logpk = denom[b, t, u] + acts[b, t, u, v] + logpk = denom[col] + acts[col * alphabet_size + idx] + # initialize the grad of the sample acts[b, t, u, v] + grad = math.exp(alphas[col] + betas[col] + logpk - logll[mb]) + + # // grad to last blank transition + # grad[b, T-1, U-1, v=blank] -= exp(alphas[b, t, u) + logpk - logll[b]) + if (idx == blank_) and (t == T - 1) and (u == U - 1): + grad -= math.exp(alphas[col] + logpk - logll[mb]) + + # grad of blank across t < T; + # grad[b, t= warp_size: + if (tid + offset) < count and tid < offset: + # Read from the right half and store to the left half. + if R_opid == 0: + x = rnnt_helper.add(x, storage[offset + tid]) + else: + x = rnnt_helper.maximum(x, storage[offset + tid]) + + storage[tid] = x + + cuda.syncthreads() + offset = offset // 2 + + offset = warp_size // 2 + while offset > 0: + # warp reduction and sync + shuff = cuda.shfl_down_sync(0xFFFFFFFF, x, offset) + + if (tid + offset < count) and (tid < offset): + if R_opid == 0: + x = rnnt_helper.add(x, shuff) + else: + x = rnnt_helper.maximum(x, shuff) + + offset = offset // 2 + + return x + + +@cuda.jit() +def _reduce_rows(I_opid: int, R_opid: int, acts, output, num_rows: int): + """ + CUDA Warp reduction kernel which reduces via the R_Op.Maximum + + Reduces the input data such that I_Op = Identity and R_op = Maximum. + The result is stored in the blockIdx, and is stored as an identity op. + + Note: + Efficient warp occurs at input shapes of 2 ^ K. + + References: + - Warp Primitives [https://developer.nvidia.com/blog/using-cuda-warp-level-primitives/] + + Args: + I_opid: Operator ID for input. See I_Op for more information. For this kernel, + the Identity op is chosen in general, and therefore the input is reduced in place + without scaling. + R_opid: Operator ID for reduction. See R_Op for more information. + For this kernel, generally Maximum op is chosen. It reduces the kernel via max. + acts: Flatened activation matrix of shape [B * T * U * (V+1)]. + output: Flatened output matrix of shape [B * T * U * (V+1)]. Data will be overwritten. + num_rows: Vocabulary size (including blank token) - V+1. + """ + tid = cuda.threadIdx.x + idx = tid + col = cuda.blockIdx.x + + # allocate shared thread memory + storage = cuda.shared.array(shape=(CTA_REDUCE_SIZE,), dtype=acts.dtype) + + max = output[col] + + # // Each block works on a column + if idx < num_rows: + curr = acts[col * num_rows + idx] - max + if I_opid == 0: + curr = rnnt_helper.exponential(curr) + else: + curr = rnnt_helper.identity(curr) + + idx += CTA_REDUCE_SIZE + + while idx < num_rows: + activation_ = acts[col * num_rows + idx] - max + + if I_opid == 0 and R_opid == 0: + curr = rnnt_helper.add(curr, rnnt_helper.exponential(activation_)) + elif I_opid == 0 and R_opid == 1: + curr = rnnt_helper.maximum(curr, rnnt_helper.exponential(activation_)) + elif I_opid == 1 and R_opid == 0: + curr = rnnt_helper.add(curr, rnnt_helper.identity(activation_)) + else: + curr = rnnt_helper.maximum(curr, rnnt_helper.identity(activation_)) + + idx += CTA_REDUCE_SIZE + + # // Sum thread-totals over the CTA. + curr = CTAReduce(tid, curr, storage, num_rows, R_opid) + + # // Store result in out (inplace, I_op: identity) + if tid == 0: + output[col] = curr + + +@cuda.jit() +def _reduce_minus(I_opid: int, R_opid: int, acts, output, num_rows: int): + """ + CUDA Warp reduction kernel which reduces via the R_Op.Add + + Reduces the input data such that I_Op = Exponential and R_op = Add. + The result is stored in the blockIdx, and is stored as an exp op. + + Note: + Efficient warp occurs at input shapes of 2 ^ K. + + References: + - Warp Primitives [https://developer.nvidia.com/blog/using-cuda-warp-level-primitives/] + + Args: + I_opid: Operator ID for input. See I_Op for more information. For this kernel, + the Exponential op is chosen in general, and therefore the input is reduced in place + with scaling. + R_opid: Operator ID for reduction. See R_Op for more information. + For this kernel, generally Add op is chosen. It reduces the kernel via summation. + acts: Flatened activation matrix of shape [B * T * U * (V+1)]. + output: Flatened output matrix of shape [B * T * U * (V+1)]. Data will be overwritten. + num_rows: Vocabulary size (including blank token) - V+1. + """ + tid = cuda.threadIdx.x + idx = tid + col = cuda.blockIdx.x + + # allocate shared thread memory + storage = cuda.shared.array(shape=(CTA_REDUCE_SIZE,), dtype=acts.dtype) + + max = output[col] + + # // Each block works on a column + if idx < num_rows: + curr = acts[col * num_rows + idx] - max + if I_opid == 0: + curr = rnnt_helper.exponential(curr) + else: + curr = rnnt_helper.identity(curr) + + idx += CTA_REDUCE_SIZE + + while idx < num_rows: + activation_ = acts[col * num_rows + idx] - max + + if I_opid == 0 and R_opid == 0: + curr = rnnt_helper.add(curr, rnnt_helper.exponential(activation_)) + elif I_opid == 0 and R_opid == 1: + curr = rnnt_helper.maximum(curr, rnnt_helper.exponential(activation_)) + elif I_opid == 1 and R_opid == 0: + curr = rnnt_helper.add(curr, rnnt_helper.identity(activation_)) + else: + curr = rnnt_helper.maximum(curr, rnnt_helper.identity(activation_)) + + idx += CTA_REDUCE_SIZE + + # // Sum thread-totals over the CTA. + curr = CTAReduce(tid, curr, storage, num_rows, R_opid) + + # // Store result in out (inplace, I_op: exponential) + if tid == 0: + output[col] = -max - math.log(curr) + + +def ReduceHelper( + I_opid: int, + R_opid: int, + acts: torch.Tensor, + output: torch.Tensor, + num_rows: int, + num_cols: int, + minus: bool, + stream, +): + """ + CUDA Warp reduction kernel helper which reduces via the R_Op.Add and writes + the result to `output` according to I_op id. + + The result is stored in the blockIdx. + + Note: + Efficient warp occurs at input shapes of 2 ^ K. + + References: + - Warp Primitives [https://developer.nvidia.com/blog/using-cuda-warp-level-primitives/] + + Args: + I_opid: Operator ID for input. See I_Op for more information. + R_opid: Operator ID for reduction. See R_Op for more information. + acts: Flatened activation matrix of shape [B * T * U * (V+1)]. + output: Flatened output matrix of shape [B * T * U * (V+1)]. Data will be overwritten. + num_rows: Vocabulary size (including blank token) - V+1. + Represents the number of threads per block. + num_cols: Flattened shape of activation matrix, without vocabulary dimension (B * T * U). + Represents number of blocks per grid. + minus: Bool flag whether to add or subtract as reduction. + If minus is set; calls _reduce_minus, else calls _reduce_rows kernel. + stream: CUDA Stream. + """ + if minus: + grid_size = num_cols + # call kernel + _reduce_minus[grid_size, CTA_REDUCE_SIZE, stream, 0](I_opid, R_opid, acts, output, num_rows) + + else: + grid_size = num_cols + # call kernel + _reduce_rows[grid_size, CTA_REDUCE_SIZE, stream, 0](I_opid, R_opid, acts, output, num_rows) + + return True + + +def reduce_exp(acts: torch.Tensor, denom, rows: int, cols: int, minus: bool, stream): + """ + Helper method to call the Warp Reduction Kernel to perform `exp` reduction. + + Note: + Efficient warp occurs at input shapes of 2 ^ K. + + References: + - Warp Primitives [https://developer.nvidia.com/blog/using-cuda-warp-level-primitives/] + + Args: + acts: Flatened activation matrix of shape [B * T * U * (V+1)]. + output: Flatened output matrix of shape [B * T * U * (V+1)]. Data will be overwritten. + rows: Vocabulary size (including blank token) - V+1. + Represents the number of threads per block. + cols: Flattened shape of activation matrix, without vocabulary dimension (B * T * U). + Represents number of blocks per grid. + minus: Bool flag whether to add or subtract as reduction. + If minus is set; calls _reduce_minus, else calls _reduce_rows kernel. + stream: CUDA Stream. + """ + return ReduceHelper( + I_opid=I_Op.EXPONENTIAL.value, + R_opid=R_Op.ADD.value, + acts=acts, + output=denom, + num_rows=rows, + num_cols=cols, + minus=minus, + stream=stream, + ) + + +def reduce_max(acts: torch.Tensor, denom, rows: int, cols: int, minus: bool, stream): + """ + Helper method to call the Warp Reduction Kernel to perform `max` reduction. + + Note: + Efficient warp occurs at input shapes of 2 ^ K. + + References: + - Warp Primitives [https://developer.nvidia.com/blog/using-cuda-warp-level-primitives/] + + Args: + acts: Flatened activation matrix of shape [B * T * U * (V+1)]. + output: Flatened output matrix of shape [B * T * U * (V+1)]. Data will be overwritten. + rows: Vocabulary size (including blank token) - V+1. + Represents the number of threads per block. + cols: Flattened shape of activation matrix, without vocabulary dimension (B * T * U). + Represents number of blocks per grid. + minus: Bool flag whether to add or subtract as reduction. + If minus is set; calls _reduce_minus, else calls _reduce_rows kernel. + stream: CUDA Stream. + """ + return ReduceHelper( + I_opid=I_Op.IDENTITY.value, + R_opid=R_Op.MAXIMUM.value, + acts=acts, + output=denom, + num_rows=rows, + num_cols=cols, + minus=minus, + stream=stream, + ) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/global_constants.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/global_constants.py new file mode 100644 index 0000000000..cc30475303 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/global_constants.py @@ -0,0 +1,68 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2018-2019, Mingkun Huang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import enum + +import numpy as np +from numba import float32 + +# Internal globals +_THREADS_PER_BLOCK = 32 +_WARP_SIZE = 32 +_DTYPE = float32 + +# Constants +FP32_INF = np.inf +FP32_NEG_INF = -np.inf +THRESHOLD = 1e-1 + +""" +Getters +""" + + +def threads_per_block(): + global _THREADS_PER_BLOCK + return _THREADS_PER_BLOCK + + +def warp_size(): + global _WARP_SIZE + return _WARP_SIZE + + +def dtype(): + global _DTYPE + return _DTYPE + + +# RNNT STATUS +class RNNTStatus(enum.Enum): + RNNT_STATUS_SUCCESS = 0 + RNNT_STATUS_INVALID_VALUE = 1 diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/rnnt_helper.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/rnnt_helper.py new file mode 100644 index 0000000000..c27d3fad2e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/rnnt_loss/utils/rnnt_helper.py @@ -0,0 +1,129 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2018-2019, Mingkun Huang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import math +from typing import Optional + +import torch +from numba import cuda + +from nemo.collections.asr.parts.numba.rnnt_loss.utils import global_constants + +threshold = global_constants.THRESHOLD + + +@cuda.jit(device=True, inline=True) +def log_sum_exp(a: float, b: float): + if a == global_constants.FP32_NEG_INF: + return b + + if b == global_constants.FP32_NEG_INF: + return a + + if a > b: + return math.log1p(math.exp(b - a)) + a + else: + return math.log1p(math.exp(a - b)) + b + + +@cuda.jit(device=True, inline=True) +def div_up(x: int, y: int): + return (x + y - 1) // y + + +@cuda.jit(device=True) +def maximum(x, y): + if x < y: + return y + else: + return x + + +@cuda.jit(device=True) +def add(x, y): + return x + y + + +@cuda.jit(device=True) +def identity(x): + return x + + +@cuda.jit(device=True) +def negate(x): + return -x + + +@cuda.jit(device=True) +def exponential(x): + return math.exp(x) + + +@cuda.jit(device=True) +def log_plus(p1: float, p2: float): + if p1 == global_constants.FP32_NEG_INF: + return p2 + + if p2 == global_constants.FP32_NEG_INF: + return p1 + + result = math.log1p(math.exp(-math.fabs(p1 - p2))) + maximum(p1, p2) + return result + + +def get_workspace_size( + maxT: int, maxU: int, minibatch: int, gpu: bool +) -> (Optional[int], global_constants.RNNTStatus): + + if minibatch <= 0 or maxT <= 0 or maxU <= 0: + return (None, global_constants.RNNTStatus.RNNT_STATUS_INVALID_VALUE) + + # per minibatch memory + per_minibatch_size = 0 + + # alphas & betas + per_minibatch_size += maxT * maxU * 2 + + if not gpu: + # // blank & label log probability cache + per_minibatch_size += maxT * maxU * 2 + else: + # // softmax denominator + per_minibatch_size += maxT * maxU + # // forward - backward loglikelihood + per_minibatch_size += 2 + + size = per_minibatch_size * minibatch + return (size, global_constants.RNNTStatus.RNNT_STATUS_SUCCESS) + + +def flatten_tensor(x: torch.Tensor): + original_shape = x.shape + x = x.view([-1]) + return x, original_shape diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/spec_augment/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/spec_augment/__init__.py new file mode 100644 index 0000000000..17a22fcf81 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/spec_augment/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.asr.parts.numba.spec_augment.spec_aug_numba import ( + SpecAugmentNumba, + spec_augment_launch_heuristics, +) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/spec_augment/spec_aug_numba.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/spec_augment/spec_aug_numba.py new file mode 100644 index 0000000000..2edf2e8d7b --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/numba/spec_augment/spec_aug_numba.py @@ -0,0 +1,290 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +from numba import cuda + +from nemo.core.classes import Typing, typecheck +from nemo.core.neural_types import LengthsType, NeuralType, SpectrogramType +from nemo.utils import logging + +MAX_THREAD_BUFFER = 512 + + +@cuda.jit() +def spec_augment_kernel( + x: torch.Tensor, + x_len: torch.Tensor, + freq_starts: torch.Tensor, + freq_widths: torch.Tensor, + time_starts: torch.Tensor, + time_widths: torch.Tensor, + mask_value: float, +): + """ + Numba CUDA kernel to perform SpecAugment in-place on the GPU. + Parallelize over freq and time axis, parallel threads over batch. + Sequential over masks (adaptive in time). + + Args: + x: Pytorch tensor of shape [B, F, T] with the acoustic features. + x_len: Pytorch tensor of shape [B] with the lengths of the padded sequence. + freq_starts: Pytorch tensor of shape [B, M_f] with the start indices of freq masks. + freq_widths: Pytorch tensor of shape [B, M_f] with the width of freq masks. + time_starts: Pytorch tensor of shape [B, M_t] with the start indices of time masks. + time_widths: Pytorch tensor of shape [B, M_t] with the width of time masks. + mask_value: Float value that will be used as mask value. + """ + f = cuda.blockIdx.x # indexes the Freq dim + t = cuda.blockIdx.y # indexes the Time dim + tid = cuda.threadIdx.x # index of the current mask + threads_per_block = cuda.blockDim.x + + # Compute the number of masks over freq axis + len_f = freq_starts.shape[1] + # For all samples in the batch, apply the freq mask + for bidx in range(0, x.shape[0], threads_per_block): + # Resolve the index of the batch (case where more masks than MAX_THREAD_BUFFER) + bm_idx = bidx * threads_per_block + tid + + # For `len_f` number of freq masks that must be applied + for fidx in range(0, len_f): + # If resolved freq mask index < total number of freq masks + if fidx < len_f: + # Access the start index and width of this freq mask + f_start = freq_starts[bm_idx, fidx] + f_width = freq_widths[bm_idx, fidx] + + # If block idx `f` >= start and < (start + width) of this freq mask + if f >= f_start and f < (f_start + f_width): + x[bm_idx, f, t] = mask_value + + # Compute the number of masks over time axis + len_t = time_starts.shape[1] + # For all samples in the batch, apply the time mask + for b_idx in range(0, x.shape[0], threads_per_block): + # Resolve the index of the batch (case where more masks than MAX_THREAD_BUFFER) + bm_idx = bidx * threads_per_block + tid + + # For `len_t` number of freq masks that must be applied + for tidx in range(0, len_t): + # If resolved time mask index < total number of time masks + if tidx < len_t: + # Access the start index and width of this time mask + t_start = time_starts[bm_idx, tidx] + t_width = time_widths[bm_idx, tidx] + + # If block idx `t` >= start and < (start + width) of this time mask + if t >= t_start and t < (t_start + t_width): + # Current block idx `t` < current seq length x_len[b] + # This ensure that we mask only upto the length of that sample + # Everything after that index is padded value so unnecessary to mask + if t < x_len[bm_idx]: + x[bm_idx, f, t] = mask_value + + +def spec_augment_launch_heuristics(x: torch.Tensor, length: torch.Tensor): + """ + Heuristics to determins whether pytorch implementation or numba implementation is selected. + Assumes numba cuda is supported. + + Args: + x: Torch tensor of shape [B, F, T] + length: Optional, Torch of tensor of shape [B] - containing lengths of the tensor. + + Returns: + True if numba kernel should be selected, else False + """ + if not x.is_cuda: + return False + + if length is None: + return False + + if x.shape[0] < 8: + return False + + return True + + +def launch_spec_augment_kernel( + x: torch.Tensor, + x_len: torch.Tensor, + freq_starts: torch.Tensor, + freq_lengths: torch.Tensor, + time_starts: torch.Tensor, + time_lengths: torch.Tensor, + freq_masks: int, + time_masks: int, + mask_value: float, +): + """ + Helper method to launch the SpecAugment kernel + + Args: + x: Pytorch tensor of shape [B, F, T] with the acoustic features. + x_len: Pytorch tensor of shape [B] with the lengths of the padded sequence. + freq_starts: Pytorch tensor of shape [B, M_f] with the start indices of freq masks. + freq_widths: Pytorch tensor of shape [B, M_f] with the width of freq masks. + time_starts: Pytorch tensor of shape [B, M_t] with the start indices of time masks. + time_widths: Pytorch tensor of shape [B, M_t] with the width of time masks. + freq_masks: Int value that determines the number of time masks. + time_masks: Int value that determines the number of freq masks. + mask_value: Float value that will be used as mask value. + + Returns: + The spec augmented tensor 'x' + """ + # Setup CUDA stream + sh = x.shape + stream = cuda.external_stream(torch.cuda.current_stream(x.device).cuda_stream) + + if time_masks > 0 or freq_masks > 0: + # Parallelize over freq and time axis, parallel threads over batch + # Sequential over masks (adaptive in time). + blocks_per_grid = [sh[1], sh[2]] + # threads_per_block = min(MAX_THREAD_BUFFER, max(freq_masks, time_masks)) + threads_per_block = min(MAX_THREAD_BUFFER, x.shape[0]) + + # Numba does not support fp16, force cast to fp32 temporarily at the expense of memory + original_dtype = x.dtype + cast_x = False + if x.dtype == torch.float16: + x = x.float() + cast_x = True + + # Launch CUDA kernel + spec_augment_kernel[blocks_per_grid, threads_per_block, stream, 0]( + x, x_len, freq_starts, freq_lengths, time_starts, time_lengths, mask_value + ) + torch.cuda.synchronize() + + # Recast back to original dtype if earlier cast was performed + if cast_x: + x = x.to(dtype=original_dtype) + + return x + + +class SpecAugmentNumba(nn.Module, Typing): + """ + Zeroes out(cuts) random continuous horisontal or + vertical segments of the spectrogram as described in + SpecAugment (https://arxiv.org/abs/1904.08779). + + Utilizes a Numba CUDA kernel to perform inplace edit of the input without loops. + Parallelize over freq and time axis, parallel threads over batch. + Sequential over masks (adaptive in time). + + Args: + freq_masks - how many frequency segments should be cut + time_masks - how many time segments should be cut + freq_width - maximum number of frequencies to be cut in one segment + time_width - maximum number of time steps to be cut in one segment. + Can be a positive integer or a float value in the range [0, 1]. + If positive integer value, defines maximum number of time steps + to be cut in one segment. + If a float value, defines maximum percentage of timesteps that + are cut adaptively. + rng: Ignored. + """ + + @property + def input_types(self): + """Returns definitions of module input types + """ + return { + "input_spec": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "length": NeuralType(tuple('B'), LengthsType()), + } + + @property + def output_types(self): + """Returns definitions of module output types + """ + return {"augmented_spec": NeuralType(('B', 'D', 'T'), SpectrogramType())} + + def __init__( + self, freq_masks=0, time_masks=0, freq_width=10, time_width=0.1, rng=None, mask_value=0.0, + ): + super().__init__() + # Message to mention that numba specaugment kernel will be available + # if input device is CUDA and lengths are provided + logging.debug("Numba SpecAugment kernel is available") + + self.freq_masks = freq_masks + self.time_masks = time_masks + + self.freq_width = freq_width + self.time_width = time_width + + self.mask_value = mask_value + + # Unused + self.rng = rng + if self.rng is not None: + logging.warning("`rng` was supplied to SpecAugmentNumba, but it is not used.") + + if isinstance(time_width, int): + self.adaptive_temporal_width = False + else: + if time_width > 1.0 or time_width < 0.0: + raise ValueError('If `time_width` is a float value, must be in range [0, 1]') + + self.adaptive_temporal_width = True + + @typecheck() + @torch.no_grad() + def forward(self, input_spec, length): + sh = input_spec.shape + bs = sh[0] + + if self.adaptive_temporal_width: + time_width = max(1, int(sh[2] * self.time_width)) + else: + time_width = self.time_width + + # Construct the freq and time masks as well as start positions + if self.freq_masks > 0: + freq_starts = torch.randint( + 0, sh[1] - self.freq_width + 1, size=[bs, self.freq_masks], device=input_spec.device + ) + freq_lengths = torch.randint(0, self.freq_width + 1, size=[bs, self.freq_masks], device=input_spec.device) + else: + freq_starts = torch.zeros([bs, 1], dtype=torch.int64, device=input_spec.device) + freq_lengths = torch.zeros([bs, 1], dtype=torch.int64, device=input_spec.device) + + if self.time_masks > 0: + time_starts = torch.randint( + 0, sh[2] - time_width + 1, size=[bs, self.time_masks], device=input_spec.device + ) + time_lengths = torch.randint(0, time_width + 1, size=[bs, self.time_masks], device=input_spec.device) + else: + time_starts = torch.zeros([bs, 1], dtype=torch.int64, device=input_spec.device) + time_lengths = torch.zeros([bs, 1], dtype=torch.int64, device=input_spec.device) + + x = launch_spec_augment_kernel( + input_spec, + length, + freq_starts=freq_starts, + freq_lengths=freq_lengths, + time_starts=time_starts, + time_lengths=time_lengths, + freq_masks=self.freq_masks, + time_masks=self.time_masks, + mask_value=self.mask_value, + ) + + return x diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/__init__.py new file mode 100644 index 0000000000..c467ec3327 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/__init__.py @@ -0,0 +1,41 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.asr.parts.preprocessing.feature_loader import ExternalFeatureLoader +from nemo.collections.asr.parts.preprocessing.features import ( + STFT, + FeaturizerFactory, + FilterbankFeatures, + STFTExactPad, + STFTPatch, + WaveformFeaturizer, +) +from nemo.collections.asr.parts.preprocessing.perturb import ( + AudioAugmentor, + AugmentationDataset, + GainPerturbation, + ImpulsePerturbation, + NoisePerturbation, + Perturbation, + RirAndNoisePerturbation, + ShiftPerturbation, + SpeedPerturbation, + TimeStretchPerturbation, + TranscodePerturbation, + WhiteNoisePerturbation, + perturbation_types, + process_augmentations, + register_perturbation, +) +from nemo.collections.asr.parts.preprocessing.segment import AudioSegment diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/feature_loader.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/feature_loader.py new file mode 100644 index 0000000000..18866b6e2c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/feature_loader.py @@ -0,0 +1,69 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np +import torch + + +class ExternalFeatureLoader(object): + """Feature loader that load external features store in certain format. + Currently support pickle, npy and npz format. + """ + + def __init__(self, file_path, sample_rate=16000, augmentor=None): + """Load feature samples from file_path. + file_path (str) is the path of the file that stores feature/sample. + """ + self._file_path = file_path + self.augmentor = augmentor + + def load_feature_from_file(self, file_path): + """Load samples from file_path and convert it to be of type float32 + file_path (str) is the path of the file that stores feature/sample. + """ + # load pickle/npy/npz file + try: + samples = np.load(file_path, allow_pickle=True) + return self._convert_samples_to_float32(samples) + except: + raise TypeError( + "Error open feature files. Probably not supported the format yet. Support pkl, npz, and npy format now. " + ) + # TODO load other type of files such as kaldi io ark + + @staticmethod + def _convert_samples_to_float32(samples): + """Convert sample type to float32. + Integers will be scaled to [-1, 1] in float32. + """ + float32_samples = samples.astype('float32') + if samples.dtype in np.sctypes['int']: + bits = np.iinfo(samples.dtype).bits + float32_samples *= 1.0 / 2 ** (bits - 1) + elif samples.dtype in np.sctypes['float']: + pass + else: + raise TypeError("Unsupported sample type: %s." % samples.dtype) + return float32_samples + + def process(self, file_path, offset=0, duration=0, trim=False, orig_sr=None): + feature = self.load_feature_from_file(file_path) + return self.process_segment(feature) + + def process_segment(self, feature_segment): + if self.augmentor: + # augmentor for external features. Here possible augmentor for external embedding feature is Diaconis Augmentation and might be implemented later + self.augmentor.perturb(feature_segment) + return torch.tensor(feature_segment, dtype=torch.float) + + return torch.tensor(feature_segment, dtype=torch.float) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/features.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/features.py new file mode 100644 index 0000000000..6cf0c07aa7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/features.py @@ -0,0 +1,432 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright (c) 2018 Ryan Leary +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# This file contains code artifacts adapted from https://github.com/ryanleary/patter +import math + +import librosa +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from librosa.util import tiny +from torch.autograd import Variable +from torch_stft import STFT + +from nemo.collections.asr.parts.preprocessing.perturb import AudioAugmentor +from nemo.collections.asr.parts.preprocessing.segment import AudioSegment +from nemo.collections.common.parts.patch_utils import stft_patch +from nemo.utils import logging + +CONSTANT = 1e-5 + + +def normalize_batch(x, seq_len, normalize_type): + if normalize_type == "per_feature": + x_mean = torch.zeros((seq_len.shape[0], x.shape[1]), dtype=x.dtype, device=x.device) + x_std = torch.zeros((seq_len.shape[0], x.shape[1]), dtype=x.dtype, device=x.device) + for i in range(x.shape[0]): + if x[i, :, : seq_len[i]].shape[1] == 1: + raise ValueError( + "normalize_batch with `per_feature` normalize_type received a tensor of length 1. This will result " + "in torch.std() returning nan" + ) + x_mean[i, :] = x[i, :, : seq_len[i]].mean(dim=1) + x_std[i, :] = x[i, :, : seq_len[i]].std(dim=1) + # make sure x_std is not zero + x_std += CONSTANT + return (x - x_mean.unsqueeze(2)) / x_std.unsqueeze(2) + elif normalize_type == "all_features": + x_mean = torch.zeros(seq_len.shape, dtype=x.dtype, device=x.device) + x_std = torch.zeros(seq_len.shape, dtype=x.dtype, device=x.device) + for i in range(x.shape[0]): + x_mean[i] = x[i, :, : seq_len[i].item()].mean() + x_std[i] = x[i, :, : seq_len[i].item()].std() + # make sure x_std is not zero + x_std += CONSTANT + return (x - x_mean.view(-1, 1, 1)) / x_std.view(-1, 1, 1) + elif "fixed_mean" in normalize_type and "fixed_std" in normalize_type: + x_mean = torch.tensor(normalize_type["fixed_mean"], device=x.device) + x_std = torch.tensor(normalize_type["fixed_std"], device=x.device) + return (x - x_mean.view(x.shape[0], x.shape[1]).unsqueeze(2)) / x_std.view(x.shape[0], x.shape[1]).unsqueeze(2) + else: + return x + + +def splice_frames(x, frame_splicing): + """ Stacks frames together across feature dim + + input is batch_size, feature_dim, num_frames + output is batch_size, feature_dim*frame_splicing, num_frames + + """ + seq = [x] + for n in range(1, frame_splicing): + seq.append(torch.cat([x[:, :, :n], x[:, :, n:]], dim=2)) + return torch.cat(seq, dim=1) + + +class WaveformFeaturizer(object): + def __init__(self, sample_rate=16000, int_values=False, augmentor=None): + self.augmentor = augmentor if augmentor is not None else AudioAugmentor() + self.sample_rate = sample_rate + self.int_values = int_values + + def max_augmentation_length(self, length): + return self.augmentor.max_augmentation_length(length) + + def process(self, file_path, offset=0, duration=0, trim=False, orig_sr=None): + audio = AudioSegment.from_file( + file_path, + target_sr=self.sample_rate, + int_values=self.int_values, + offset=offset, + duration=duration, + trim=trim, + orig_sr=orig_sr, + ) + return self.process_segment(audio) + + def process_segment(self, audio_segment): + self.augmentor.perturb(audio_segment) + return torch.tensor(audio_segment.samples, dtype=torch.float) + + @classmethod + def from_config(cls, input_config, perturbation_configs=None): + if perturbation_configs is not None: + aa = AudioAugmentor.from_config(perturbation_configs) + else: + aa = None + + sample_rate = input_config.get("sample_rate", 16000) + int_values = input_config.get("int_values", False) + + return cls(sample_rate=sample_rate, int_values=int_values, augmentor=aa) + + +class FeaturizerFactory(object): + def __init__(self): + pass + + @classmethod + def from_config(cls, input_cfg, perturbation_configs=None): + return WaveformFeaturizer.from_config(input_cfg, perturbation_configs=perturbation_configs) + + +# Create helper class to patch forward func for use with AMP +class STFTPatch(STFT): + def forward(self, input_data): + return super().transform(input_data)[0] + + +# Create helper class for STFT that yields num_frames = num_samples // hop_length +class STFTExactPad(STFTPatch): + """adapted from Prem Seetharaman's https://github.com/pseeth/pytorch-stft""" + + def __init__(self, *params, **kw_params): + super().__init__(*params, **kw_params) + self.pad_amount = (self.filter_length - self.hop_length) // 2 + + def inverse(self, magnitude, phase): + recombine_magnitude_phase = torch.cat([magnitude * torch.cos(phase), magnitude * torch.sin(phase)], dim=1) + + inverse_transform = F.conv_transpose1d( + recombine_magnitude_phase, + Variable(self.inverse_basis, requires_grad=False), + stride=self.hop_length, + padding=0, + ) + + if self.window is not None: + window_sum = librosa.filters.window_sumsquare( + self.window, + magnitude.size(-1), + hop_length=self.hop_length, + win_length=self.win_length, + n_fft=self.filter_length, + dtype=np.float32, + ) + # remove modulation effects + approx_nonzero_indices = torch.from_numpy(np.where(window_sum > tiny(window_sum))[0]) + window_sum = torch.autograd.Variable(torch.from_numpy(window_sum), requires_grad=False).to( + magnitude.device + ) + inverse_transform[..., approx_nonzero_indices] /= window_sum[approx_nonzero_indices] + + # scale by hop ratio + inverse_transform *= self.filter_length / self.hop_length + + inverse_transform = inverse_transform[..., self.pad_amount :] + inverse_transform = inverse_transform[..., : -self.pad_amount :] + inverse_transform = inverse_transform.squeeze(1) + + return inverse_transform + + +class FilterbankFeatures(nn.Module): + """Featurizer that converts wavs to Mel Spectrograms. + See AudioToMelSpectrogramPreprocessor for args. + """ + + def __init__( + self, + sample_rate=16000, + n_window_size=320, + n_window_stride=160, + window="hann", + normalize="per_feature", + n_fft=None, + preemph=0.97, + nfilt=64, + lowfreq=0, + highfreq=None, + log=True, + log_zero_guard_type="add", + log_zero_guard_value=2 ** -24, + dither=CONSTANT, + pad_to=16, + max_duration=16.7, + frame_splicing=1, + exact_pad=False, + stft_exact_pad=False, # TODO: Remove this in 1.1.0 + stft_conv=False, # TODO: Remove this in 1.1.0 + pad_value=0, + mag_power=2.0, + use_grads=False, + ): + super().__init__() + if stft_conv or stft_exact_pad: + logging.warning( + "Using torch_stft is deprecated and will be removed in 1.1.0. Please set stft_conv and stft_exact_pad " + "to False for FilterbankFeatures and AudioToMelSpectrogramPreprocessor. Please set exact_pad to True " + "as needed." + ) + if (exact_pad or stft_exact_pad) and n_window_stride % 2 == 1: + raise NotImplementedError( + f"{self} received exact_pad == True, but hop_size was odd. If audio_length % hop_size == 0. Then the " + "returned spectrogram would not be of length audio_length // hop_size. Please use an even hop_size." + ) + self.log_zero_guard_value = log_zero_guard_value + if ( + n_window_size is None + or n_window_stride is None + or not isinstance(n_window_size, int) + or not isinstance(n_window_stride, int) + or n_window_size <= 0 + or n_window_stride <= 0 + ): + raise ValueError( + f"{self} got an invalid value for either n_window_size or " + f"n_window_stride. Both must be positive ints." + ) + logging.info(f"PADDING: {pad_to}") + + self.win_length = n_window_size + self.hop_length = n_window_stride + self.n_fft = n_fft or 2 ** math.ceil(math.log2(self.win_length)) + self.stft_pad_amount = (self.n_fft - self.hop_length) // 2 if exact_pad else None + self.stft_exact_pad = stft_exact_pad + self.stft_conv = stft_conv + + if stft_conv: + logging.info("STFT using conv") + if stft_exact_pad: + logging.info("STFT using exact pad") + self.stft = STFTExactPad(self.n_fft, self.hop_length, self.win_length, window) + else: + self.stft = STFTPatch(self.n_fft, self.hop_length, self.win_length, window) + else: + logging.info("STFT using torch") + if exact_pad: + logging.info("STFT using exact pad") + torch_windows = { + 'hann': torch.hann_window, + 'hamming': torch.hamming_window, + 'blackman': torch.blackman_window, + 'bartlett': torch.bartlett_window, + 'none': None, + } + window_fn = torch_windows.get(window, None) + window_tensor = window_fn(self.win_length, periodic=False) if window_fn else None + self.register_buffer("window", window_tensor) + self.stft = lambda x: stft_patch( + x, + n_fft=self.n_fft, + hop_length=self.hop_length, + win_length=self.win_length, + center=False if exact_pad else True, + window=self.window.to(dtype=torch.float), + return_complex=False, + ) + + self.normalize = normalize + self.log = log + self.dither = dither + self.frame_splicing = frame_splicing + self.nfilt = nfilt + self.preemph = preemph + self.pad_to = pad_to + highfreq = highfreq or sample_rate / 2 + + filterbanks = torch.tensor( + librosa.filters.mel(sample_rate, self.n_fft, n_mels=nfilt, fmin=lowfreq, fmax=highfreq), dtype=torch.float + ).unsqueeze(0) + self.register_buffer("fb", filterbanks) + + # Calculate maximum sequence length + max_length = self.get_seq_len(torch.tensor(max_duration * sample_rate, dtype=torch.float)) + max_pad = pad_to - (max_length % pad_to) if pad_to > 0 else 0 + self.max_length = max_length + max_pad + self.pad_value = pad_value + self.mag_power = mag_power + + # We want to avoid taking the log of zero + # There are two options: either adding or clamping to a small value + if log_zero_guard_type not in ["add", "clamp"]: + raise ValueError( + f"{self} received {log_zero_guard_type} for the " + f"log_zero_guard_type parameter. It must be either 'add' or " + f"'clamp'." + ) + + self.use_grads = use_grads + if not use_grads: + self.forward = torch.no_grad()(self.forward) + + # log_zero_guard_value is the the small we want to use, we support + # an actual number, or "tiny", or "eps" + self.log_zero_guard_type = log_zero_guard_type + logging.debug(f"sr: {sample_rate}") + logging.debug(f"n_fft: {self.n_fft}") + logging.debug(f"win_length: {self.win_length}") + logging.debug(f"hop_length: {self.hop_length}") + logging.debug(f"n_mels: {nfilt}") + logging.debug(f"fmin: {lowfreq}") + logging.debug(f"fmax: {highfreq}") + logging.debug(f"using grads: {use_grads}") + + def log_zero_guard_value_fn(self, x): + if isinstance(self.log_zero_guard_value, str): + if self.log_zero_guard_value == "tiny": + return torch.finfo(x.dtype).tiny + elif self.log_zero_guard_value == "eps": + return torch.finfo(x.dtype).eps + else: + raise ValueError( + f"{self} received {self.log_zero_guard_value} for the " + f"log_zero_guard_type parameter. It must be either a " + f"number, 'tiny', or 'eps'" + ) + else: + return self.log_zero_guard_value + + def get_seq_len(self, seq_len): + if isinstance(self.stft, STFT): + pad_amount = self.stft.pad_amount * 2 + else: + # Assuming that center is True is stft_pad_amount = 0 + pad_amount = self.stft_pad_amount * 2 if self.stft_pad_amount is not None else self.n_fft // 2 * 2 + seq_len = torch.floor((seq_len + pad_amount - self.n_fft) / self.hop_length) + 1 + return seq_len.to(dtype=torch.long) + + @property + def filter_banks(self): + return self.fb + + def forward(self, x, seq_len): + seq_len = self.get_seq_len(seq_len.float()) + + if self.stft_pad_amount is not None: + x = torch.nn.functional.pad( + x.unsqueeze(1), (self.stft_pad_amount, self.stft_pad_amount), "reflect" + ).squeeze(1) + + # dither (only in training mode for eval determinism) + if self.training and self.dither > 0: + x += self.dither * torch.randn_like(x) + + # do preemphasis + if self.preemph is not None: + x = torch.cat((x[:, 0].unsqueeze(1), x[:, 1:] - self.preemph * x[:, :-1]), dim=1) + + # disable autocast to get full range of stft values + with torch.cuda.amp.autocast(enabled=False): + x = self.stft(x) + + # torch returns real, imag; so convert to magnitude + if not self.stft_conv: + # guard is needed for sqrt if grads are passed through + guard = 0 if not self.use_grads else CONSTANT + if x.dtype in [torch.cfloat, torch.cdouble]: + x = torch.view_as_real(x) + x = torch.sqrt(x.pow(2).sum(-1) + guard) + + # get power spectrum + if self.mag_power != 1.0: + x = x.pow(self.mag_power) + + # dot with filterbank energies + x = torch.matmul(self.fb.to(x.dtype), x) + + # log features if required + if self.log: + if self.log_zero_guard_type == "add": + x = torch.log(x + self.log_zero_guard_value_fn(x)) + elif self.log_zero_guard_type == "clamp": + x = torch.log(torch.clamp(x, min=self.log_zero_guard_value_fn(x))) + else: + raise ValueError("log_zero_guard_type was not understood") + + # frame splicing if required + if self.frame_splicing > 1: + x = splice_frames(x, self.frame_splicing) + + # normalize if required + if self.normalize: + x = normalize_batch(x, seq_len, normalize_type=self.normalize) + + # mask to zero any values beyond seq_len in batch, pad to multiple of `pad_to` (for efficiency) + max_len = x.size(-1) + mask = torch.arange(max_len).to(x.device) + mask = mask.expand(x.size(0), max_len) >= seq_len.unsqueeze(1) + x = x.masked_fill(mask.unsqueeze(1).type(torch.bool).to(device=x.device), self.pad_value) + del mask + pad_to = self.pad_to + if pad_to == "max": + x = nn.functional.pad(x, (0, self.max_length - x.size(-1)), value=self.pad_value) + elif pad_to > 0: + pad_amt = x.size(-1) % pad_to + if pad_amt != 0: + x = nn.functional.pad(x, (0, pad_to - pad_amt), value=self.pad_value) + + return x, seq_len diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/perturb.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/perturb.py new file mode 100644 index 0000000000..812efc3eab --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/perturb.py @@ -0,0 +1,895 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright (c) 2018 Ryan Leary +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# This file contains code artifacts adapted from https://github.com/ryanleary/patter +import copy +import io +import os +import random +import subprocess +from tempfile import NamedTemporaryFile +from typing import List, Optional, Union + +import librosa +import numpy as np +import soundfile as sf +import webdataset as wd +from omegaconf import DictConfig, OmegaConf +from scipy import signal +from torch.utils.data import IterableDataset + +from nemo.collections.asr.parts.preprocessing.segment import AudioSegment +from nemo.collections.common.parts.preprocessing import collections, parsers +from nemo.utils import logging + +try: + from nemo.collections.asr.parts.utils import numba_utils + + HAVE_NUMBA = True +except (ImportError, ModuleNotFoundError): + HAVE_NUMBA = False + + +def read_one_audiosegment(manifest, target_sr, rng, tarred_audio=False, audio_dataset=None): + + if tarred_audio: + if audio_dataset is None: + raise TypeError("Expected augmentation dataset but got None") + audio_file, file_id = next(audio_dataset) + manifest_idx = manifest.mapping[file_id] + manifest_entry = manifest[manifest_idx] + + offset = 0 if manifest_entry.offset is None else manifest_entry.offset + duration = 0 if manifest_entry.duration is None else manifest_entry.duration + + else: + audio_record = rng.sample(manifest.data, 1)[0] + audio_file = audio_record.audio_file + offset = 0 if audio_record.offset is None else audio_record.offset + duration = 0 if audio_record.duration is None else audio_record.duration + + return AudioSegment.from_file(audio_file, target_sr=target_sr, offset=offset, duration=duration) + + +class Perturbation(object): + def max_augmentation_length(self, length): + return length + + def perturb(self, data): + raise NotImplementedError + + +class SpeedPerturbation(Perturbation): + """ + Performs Speed Augmentation by re-sampling the data to a different sampling rate, + which does not preserve pitch. + + Note: This is a very slow operation for online augmentation. If space allows, + it is preferable to pre-compute and save the files to augment the dataset. + + Args: + sr: Original sampling rate. + resample_type: Type of resampling operation that will be performed. + For better speed using `resampy`'s fast resampling method, use `resample_type='kaiser_fast'`. + For high-quality resampling, set `resample_type='kaiser_best'`. + To use `scipy.signal.resample`, set `resample_type='fft'` or `resample_type='scipy'` + min_speed_rate: Minimum sampling rate modifier. + max_speed_rate: Maximum sampling rate modifier. + num_rates: Number of discrete rates to allow. Can be a positive or negative + integer. + If a positive integer greater than 0 is provided, the range of + speed rates will be discretized into `num_rates` values. + If a negative integer or 0 is provided, the full range of speed rates + will be sampled uniformly. + Note: If a positive integer is provided and the resultant discretized + range of rates contains the value '1.0', then those samples with rate=1.0, + will not be augmented at all and simply skipped. This is to unnecessary + augmentation and increase computation time. Effective augmentation chance + in such a case is = `prob * (num_rates - 1 / num_rates) * 100`% chance + where `prob` is the global probability of a sample being augmented. + rng: Random seed number. + """ + + def __init__(self, sr, resample_type, min_speed_rate=0.9, max_speed_rate=1.1, num_rates=5, rng=None): + + min_rate = min(min_speed_rate, max_speed_rate) + if min_rate < 0.0: + raise ValueError("Minimum sampling rate modifier must be > 0.") + + if resample_type not in ('kaiser_best', 'kaiser_fast', 'fft', 'scipy'): + raise ValueError("Supported `resample_type` values are ('kaiser_best', 'kaiser_fast', 'fft', 'scipy')") + + self._sr = sr + self._min_rate = min_speed_rate + self._max_rate = max_speed_rate + self._num_rates = num_rates + if num_rates > 0: + self._rates = np.linspace(self._min_rate, self._max_rate, self._num_rates, endpoint=True) + self._res_type = resample_type + self._rng = random.Random() if rng is None else rng + + def max_augmentation_length(self, length): + return length * self._max_rate + + def perturb(self, data): + # Select speed rate either from choice or random sample + if self._num_rates < 0: + speed_rate = self._rng.uniform(self._min_rate, self._max_rate) + else: + speed_rate = self._rng.choice(self._rates) + + # Skip perturbation in case of identity speed rate + if speed_rate == 1.0: + return + + new_sr = int(self._sr * speed_rate) + data._samples = librosa.core.resample(data._samples, self._sr, new_sr, res_type=self._res_type) + + +class TimeStretchPerturbation(Perturbation): + """ + Time-stretch an audio series by a fixed rate while preserving pitch, based on [1, 2]. + + Note: + This is a simplified implementation, intended primarily for reference and pedagogical purposes. + It makes no attempt to handle transients, and is likely to produce audible artifacts. + + Reference + [1] [Ellis, D. P. W. “A phase vocoder in Matlab.” Columbia University, 2002.] + (http://www.ee.columbia.edu/~dpwe/resources/matlab/pvoc/) + [2] [librosa.effects.time_stretch] + (https://librosa.github.io/librosa/generated/librosa.effects.time_stretch.html) + + Args: + min_speed_rate: Minimum sampling rate modifier. + max_speed_rate: Maximum sampling rate modifier. + num_rates: Number of discrete rates to allow. Can be a positive or negative + integer. + If a positive integer greater than 0 is provided, the range of + speed rates will be discretized into `num_rates` values. + If a negative integer or 0 is provided, the full range of speed rates + will be sampled uniformly. + Note: If a positive integer is provided and the resultant discretized + range of rates contains the value '1.0', then those samples with rate=1.0, + will not be augmented at all and simply skipped. This is to avoid unnecessary + augmentation and increase computation time. Effective augmentation chance + in such a case is = `prob * (num_rates - 1 / num_rates) * 100`% chance + where `prob` is the global probability of a sample being augmented. + n_fft: Number of fft filters to be computed. + rng: Random seed number. + """ + + def __init__(self, min_speed_rate=0.9, max_speed_rate=1.1, num_rates=5, n_fft=512, rng=None): + + min_rate = min(min_speed_rate, max_speed_rate) + if min_rate < 0.0: + raise ValueError("Minimum sampling rate modifier must be > 0.") + + self._min_rate = min_speed_rate + self._max_rate = max_speed_rate + self._num_rates = num_rates + if num_rates > 0: + self._rates = np.linspace(self._min_rate, self._max_rate, self._num_rates, endpoint=True) + self._rng = random.Random() if rng is None else rng + + # Pre-compute constants + self._n_fft = int(n_fft) + self._hop_length = int(n_fft // 2) + + # Pre-allocate buffers + self._phi_advance_fast = np.linspace(0, np.pi * self._hop_length, self._hop_length + 1) + self._scale_buffer_fast = np.empty(self._hop_length + 1, dtype=np.float32) + + self._phi_advance_slow = np.linspace(0, np.pi * self._n_fft, self._n_fft + 1) + self._scale_buffer_slow = np.empty(self._n_fft + 1, dtype=np.float32) + + def max_augmentation_length(self, length): + return length * self._max_rate + + def perturb(self, data): + # Select speed rate either from choice or random sample + if self._num_rates < 0: + speed_rate = self._rng.uniform(self._min_rate, self._max_rate) + else: + speed_rate = self._rng.choice(self._rates) + + # Skip perturbation in case of identity speed rate + if speed_rate == 1.0: + return + + # Increase `n_fft` based on task (speed up or slow down audio) + # This greatly reduces upper bound of maximum time taken + # to compute slowed down audio segments. + if speed_rate >= 1.0: # Speed up audio + fft_multiplier = 1 + phi_advance = self._phi_advance_fast + scale_buffer = self._scale_buffer_fast + + else: # Slow down audio + fft_multiplier = 2 + phi_advance = self._phi_advance_slow + scale_buffer = self._scale_buffer_slow + + n_fft = int(self._n_fft * fft_multiplier) + hop_length = int(self._hop_length * fft_multiplier) + + # Perform short-term Fourier transform (STFT) + stft = librosa.core.stft(data._samples, n_fft=n_fft, hop_length=hop_length) + + # Stretch by phase vocoding + if HAVE_NUMBA: + stft_stretch = numba_utils.phase_vocoder(stft, speed_rate, phi_advance, scale_buffer) + + else: + stft_stretch = librosa.core.phase_vocoder(stft, speed_rate, hop_length) + + # Predict the length of y_stretch + len_stretch = int(round(len(data._samples) / speed_rate)) + + # Invert the STFT + y_stretch = librosa.core.istft( + stft_stretch, dtype=data._samples.dtype, hop_length=hop_length, length=len_stretch + ) + + data._samples = y_stretch + + +class GainPerturbation(Perturbation): + """ + Applies random gain to the audio. + + Args: + min_gain_dbfs (float): Min gain level in dB + max_gain_dbfs (float): Max gain level in dB + rng: Random number generator + """ + + def __init__(self, min_gain_dbfs=-10, max_gain_dbfs=10, rng=None): + self._min_gain_dbfs = min_gain_dbfs + self._max_gain_dbfs = max_gain_dbfs + self._rng = random.Random() if rng is None else rng + + def perturb(self, data): + gain = self._rng.uniform(self._min_gain_dbfs, self._max_gain_dbfs) + # logging.debug("gain: %d", gain) + data._samples = data._samples * (10.0 ** (gain / 20.0)) + + +class ImpulsePerturbation(Perturbation): + """ + Convolves audio with a Room Impulse Response. + + Args: + manifest_path (list): Manifest file for RIRs + audio_tar_filepaths (list): Tar files, if RIR audio files are tarred + shuffle_n (int): Shuffle parameter for shuffling buffered files from the tar files + shift_impulse (bool): Shift impulse response to adjust for delay at the beginning + """ + + def __init__(self, manifest_path=None, rng=None, audio_tar_filepaths=None, shuffle_n=128, shift_impulse=False): + self._manifest = collections.ASRAudioText(manifest_path, parser=parsers.make_parser([]), index_by_file_id=True) + self._audiodataset = None + self._tarred_audio = False + self._shift_impulse = shift_impulse + self._data_iterator = None + + if audio_tar_filepaths: + self._tarred_audio = True + self._audiodataset = AugmentationDataset(manifest_path, audio_tar_filepaths, shuffle_n) + self._data_iterator = iter(self._audiodataset) + + self._rng = random.Random() if rng is None else rng + + def perturb(self, data): + impulse = read_one_audiosegment( + self._manifest, + data.sample_rate, + self._rng, + tarred_audio=self._tarred_audio, + audio_dataset=self._data_iterator, + ) + if not self._shift_impulse: + impulse_norm = (impulse.samples - min(impulse.samples)) / (max(impulse.samples) - min(impulse.samples)) + data._samples = signal.fftconvolve(data._samples, impulse_norm, "same") + else: + # Find peak and shift peak to left + impulse_norm = (impulse.samples - min(impulse.samples)) / (max(impulse.samples) - min(impulse.samples)) + max_ind = np.argmax(np.abs(impulse_norm)) + + impulse_resp = impulse_norm[max_ind:] + delay_after = len(impulse_resp) + data._samples = signal.fftconvolve(data._samples, impulse_resp, "full")[:-delay_after] + + +class ShiftPerturbation(Perturbation): + """ + Perturbs audio by shifting the audio in time by a random amount between min_shift_ms and max_shift_ms. + The final length of the audio is kept unaltered by padding the audio with zeros. + + + Args: + min_shift_ms (float): Minimum time in milliseconds by which audio will be shifted + max_shift_ms (float): Maximum time in milliseconds by which audio will be shifted + rng: Random number generator + """ + + def __init__(self, min_shift_ms=-5.0, max_shift_ms=5.0, rng=None): + self._min_shift_ms = min_shift_ms + self._max_shift_ms = max_shift_ms + self._rng = random.Random() if rng is None else rng + + def perturb(self, data): + shift_ms = self._rng.uniform(self._min_shift_ms, self._max_shift_ms) + if abs(shift_ms) / 1000 > data.duration: + # TODO: do something smarter than just ignore this condition + return + shift_samples = int(shift_ms * data.sample_rate // 1000) + # logging.debug("shift: %s", shift_samples) + if shift_samples < 0: + data._samples[-shift_samples:] = data._samples[:shift_samples] + data._samples[:-shift_samples] = 0 + elif shift_samples > 0: + data._samples[:-shift_samples] = data._samples[shift_samples:] + data._samples[-shift_samples:] = 0 + + +class NoisePerturbation(Perturbation): + """ + Perturbation that adds noise to input audio. + + Args: + manifest_path (str): Manifest file with paths to noise files + min_snr_db (float): Minimum SNR of audio after noise is added + max_snr_db (float): Maximum SNR of audio after noise is added + max_gain_db (float): Maximum gain that can be applied on the noise sample + audio_tar_filepaths (list) : Tar files, if noise audio files are tarred + shuffle_n (int): Shuffle parameter for shuffling buffered files from the tar files + orig_sr (int): Original sampling rate of the noise files + rng: Random number generator + """ + + def __init__( + self, + manifest_path=None, + min_snr_db=10, + max_snr_db=50, + max_gain_db=300.0, + rng=None, + audio_tar_filepaths=None, + shuffle_n=100, + orig_sr=16000, + ): + self._manifest = collections.ASRAudioText(manifest_path, parser=parsers.make_parser([]), index_by_file_id=True) + self._audiodataset = None + self._tarred_audio = False + self._orig_sr = orig_sr + self._data_iterator = None + + if audio_tar_filepaths: + self._tarred_audio = True + self._audiodataset = AugmentationDataset(manifest_path, audio_tar_filepaths, shuffle_n) + self._data_iterator = iter(self._audiodataset) + + self._rng = random.Random() if rng is None else rng + self._min_snr_db = min_snr_db + self._max_snr_db = max_snr_db + self._max_gain_db = max_gain_db + + @property + def orig_sr(self): + return self._orig_sr + + def get_one_noise_sample(self, target_sr): + return read_one_audiosegment( + self._manifest, target_sr, self._rng, tarred_audio=self._tarred_audio, audio_dataset=self._data_iterator + ) + + def perturb(self, data): + noise = read_one_audiosegment( + self._manifest, + data.sample_rate, + self._rng, + tarred_audio=self._tarred_audio, + audio_dataset=self._data_iterator, + ) + self.perturb_with_input_noise(data, noise) + + def perturb_with_input_noise(self, data, noise, data_rms=None): + snr_db = self._rng.uniform(self._min_snr_db, self._max_snr_db) + if data_rms is None: + data_rms = data.rms_db + noise_gain_db = min(data_rms - noise.rms_db - snr_db, self._max_gain_db) + # logging.debug("noise: %s %s %s", snr_db, noise_gain_db, noise_record.audio_file) + + # calculate noise segment to use + start_time = self._rng.uniform(0.0, noise.duration - data.duration) + if noise.duration > (start_time + data.duration): + noise.subsegment(start_time=start_time, end_time=start_time + data.duration) + + # adjust gain for snr purposes and superimpose + noise.gain_db(noise_gain_db) + + if noise._samples.shape[0] < data._samples.shape[0]: + noise_idx = self._rng.randint(0, data._samples.shape[0] - noise._samples.shape[0]) + data._samples[noise_idx : noise_idx + noise._samples.shape[0]] += noise._samples + + else: + data._samples += noise._samples + + def perturb_with_foreground_noise( + self, data, noise, data_rms=None, max_noise_dur=2, max_additions=1, + ): + snr_db = self._rng.uniform(self._min_snr_db, self._max_snr_db) + if not data_rms: + data_rms = data.rms_db + + noise_gain_db = min(data_rms - noise.rms_db - snr_db, self._max_gain_db) + n_additions = self._rng.randint(1, max_additions) + + for i in range(n_additions): + noise_dur = self._rng.uniform(0.0, max_noise_dur) + start_time = self._rng.uniform(0.0, noise.duration) + start_sample = int(round(start_time * noise.sample_rate)) + end_sample = int(round(min(noise.duration, (start_time + noise_dur)) * noise.sample_rate)) + noise_samples = np.copy(noise._samples[start_sample:end_sample]) + # adjust gain for snr purposes and superimpose + noise_samples *= 10.0 ** (noise_gain_db / 20.0) + + if noise_samples.shape[0] > data._samples.shape[0]: + noise_samples = noise_samples[0 : data._samples.shape[0]] + + noise_idx = self._rng.randint(0, data._samples.shape[0] - noise_samples.shape[0]) + data._samples[noise_idx : noise_idx + noise_samples.shape[0]] += noise_samples + + +class WhiteNoisePerturbation(Perturbation): + """ + Perturbation that adds white noise to an audio file in the training dataset. + + Args: + min_level (int): Minimum level in dB at which white noise should be added + max_level (int): Maximum level in dB at which white noise should be added + rng: Random number generator + """ + + def __init__(self, min_level=-90, max_level=-46, rng=None): + self.min_level = int(min_level) + self.max_level = int(max_level) + self._rng = np.random.RandomState() if rng is None else rng + + def perturb(self, data): + noise_level_db = self._rng.randint(self.min_level, self.max_level, dtype='int32') + noise_signal = self._rng.randn(data._samples.shape[0]) * (10.0 ** (noise_level_db / 20.0)) + data._samples += noise_signal + + +class RirAndNoisePerturbation(Perturbation): + """ + RIR augmentation with additive foreground and background noise. + In this implementation audio data is augmented by first convolving the audio with a Room Impulse Response + and then adding foreground noise and background noise at various SNRs. RIR, foreground and background noises + should either be supplied with a manifest file or as tarred audio files (faster). + + Different sets of noise audio files based on the original sampling rate of the noise. This is useful while + training a mixed sample rate model. For example, when training a mixed model with 8 kHz and 16 kHz audio with a + target sampling rate of 16 kHz, one would want to augment 8 kHz data with 8 kHz noise rather than 16 kHz noise. + + Args: + rir_manifest_path: Manifest file for RIRs + rir_tar_filepaths: Tar files, if RIR audio files are tarred + rir_prob: Probability of applying a RIR + noise_manifest_paths: Foreground noise manifest path + min_snr_db: Min SNR for foreground noise + max_snr_db: Max SNR for background noise, + noise_tar_filepaths: Tar files, if noise files are tarred + apply_noise_rir: Whether to convolve foreground noise with a a random RIR + orig_sample_rate: Original sampling rate of foreground noise audio + max_additions: Max number of times foreground noise is added to an utterance, + max_duration: Max duration of foreground noise + bg_noise_manifest_paths: Background noise manifest path + bg_min_snr_db: Min SNR for background noise + bg_max_snr_db: Max SNR for background noise + bg_noise_tar_filepaths: Tar files, if noise files are tarred + bg_orig_sample_rate: Original sampling rate of background noise audio + + """ + + def __init__( + self, + rir_manifest_path=None, + rir_prob=0.5, + noise_manifest_paths=None, + min_snr_db=0, + max_snr_db=50, + rir_tar_filepaths=None, + rir_shuffle_n=100, + noise_tar_filepaths=None, + apply_noise_rir=False, + orig_sample_rate=None, + max_additions=5, + max_duration=2.0, + bg_noise_manifest_paths=None, + bg_min_snr_db=10, + bg_max_snr_db=50, + bg_noise_tar_filepaths=None, + bg_orig_sample_rate=None, + ): + + logging.info("Called Rir aug init") + self._rir_prob = rir_prob + self._rng = random.Random() + self._rir_perturber = ImpulsePerturbation( + manifest_path=rir_manifest_path, + audio_tar_filepaths=rir_tar_filepaths, + shuffle_n=rir_shuffle_n, + shift_impulse=True, + ) + self._fg_noise_perturbers = {} + self._bg_noise_perturbers = {} + if noise_manifest_paths: + for i in range(len(noise_manifest_paths)): + if orig_sample_rate is None: + orig_sr = 16000 + else: + orig_sr = orig_sample_rate[i] + self._fg_noise_perturbers[orig_sr] = NoisePerturbation( + manifest_path=noise_manifest_paths[i], + min_snr_db=min_snr_db[i], + max_snr_db=max_snr_db[i], + audio_tar_filepaths=noise_tar_filepaths[i], + orig_sr=orig_sr, + ) + self._max_additions = max_additions + self._max_duration = max_duration + if bg_noise_manifest_paths: + for i in range(len(bg_noise_manifest_paths)): + if bg_orig_sample_rate is None: + orig_sr = 16000 + else: + orig_sr = bg_orig_sample_rate[i] + self._bg_noise_perturbers[orig_sr] = NoisePerturbation( + manifest_path=bg_noise_manifest_paths[i], + min_snr_db=bg_min_snr_db[i], + max_snr_db=bg_max_snr_db[i], + audio_tar_filepaths=bg_noise_tar_filepaths[i], + orig_sr=orig_sr, + ) + + self._apply_noise_rir = apply_noise_rir + + def perturb(self, data): + prob = self._rng.uniform(0.0, 1.0) + + if prob < self._rir_prob: + self._rir_perturber.perturb(data) + + orig_sr = data.orig_sr + if orig_sr not in self._fg_noise_perturbers: + orig_sr = max(self._fg_noise_perturbers.keys()) + fg_perturber = self._fg_noise_perturbers[orig_sr] + + orig_sr = data.orig_sr + if orig_sr not in self._bg_noise_perturbers: + orig_sr = max(self._bg_noise_perturbers.keys()) + bg_perturber = self._bg_noise_perturbers[orig_sr] + + data_rms = data.rms_db + noise = fg_perturber.get_one_noise_sample(data.sample_rate) + if self._apply_noise_rir: + self._rir_perturber.perturb(noise) + fg_perturber.perturb_with_foreground_noise( + data, noise, data_rms=data_rms, max_noise_dur=self._max_duration, max_additions=self._max_additions + ) + noise = bg_perturber.get_one_noise_sample(data.sample_rate) + bg_perturber.perturb_with_input_noise(data, noise, data_rms=data_rms) + + +class TranscodePerturbation(Perturbation): + """ + Audio codec augmentation. This implementation uses sox to transcode audio with low rate audio codecs, + so users need to make sure that the installed sox version supports the codecs used here (G711 and amr-nb). + + Args: + rng: Random number generator + """ + + def __init__(self, rng=None): + self._rng = np.random.RandomState() if rng is None else rng + self._codecs = ["g711", "amr-nb"] + + def perturb(self, data): + att_factor = 0.8 + max_level = np.max(np.abs(data._samples)) + norm_factor = att_factor / max_level + norm_samples = norm_factor * data._samples + orig_f = NamedTemporaryFile(suffix=".wav") + sf.write(orig_f.name, norm_samples.transpose(), 16000) + + codec_ind = random.randint(0, len(self._codecs) - 1) + if self._codecs[codec_ind] == "amr-nb": + transcoded_f = NamedTemporaryFile(suffix="_amr.wav") + rates = list(range(0, 8)) + rate = rates[random.randint(0, len(rates) - 1)] + _ = subprocess.check_output( + f"sox {orig_f.name} -V0 -C {rate} -t amr-nb - | sox -t amr-nb - -V0 -b 16 -r 16000 {transcoded_f.name}", + shell=True, + ) + elif self._codecs[codec_ind] == "g711": + transcoded_f = NamedTemporaryFile(suffix="_g711.wav") + _ = subprocess.check_output( + f"sox {orig_f.name} -V0 -r 8000 -c 1 -e a-law {transcoded_f.name}", shell=True + ) + + new_data = AudioSegment.from_file(transcoded_f.name, target_sr=16000) + data._samples = new_data._samples[0 : data._samples.shape[0]] + return + + +perturbation_types = { + "speed": SpeedPerturbation, + "time_stretch": TimeStretchPerturbation, + "gain": GainPerturbation, + "impulse": ImpulsePerturbation, + "shift": ShiftPerturbation, + "noise": NoisePerturbation, + "white_noise": WhiteNoisePerturbation, + "rir_noise_aug": RirAndNoisePerturbation, + "transcode_aug": TranscodePerturbation, +} + + +def register_perturbation(name: str, perturbation: Perturbation): + if name in perturbation_types.keys(): + raise KeyError( + f"Perturbation with the name {name} exists. " f"Type of perturbation : {perturbation_types[name]}." + ) + + perturbation_types[name] = perturbation + + +class AudioAugmentor(object): + def __init__(self, perturbations=None, rng=None): + self._rng = random.Random() if rng is None else rng + self._pipeline = perturbations if perturbations is not None else [] + + def perturb(self, segment): + for (prob, p) in self._pipeline: + if self._rng.random() < prob: + p.perturb(segment) + return + + def max_augmentation_length(self, length): + newlen = length + for (prob, p) in self._pipeline: + newlen = p.max_augmentation_length(newlen) + return newlen + + @classmethod + def from_config(cls, config): + ptbs = [] + for p in config: + if p['aug_type'] not in perturbation_types: + logging.warning("%s perturbation not known. Skipping.", p['aug_type']) + continue + perturbation = perturbation_types[p['aug_type']] + ptbs.append((p['prob'], perturbation(**p['cfg']))) + return cls(perturbations=ptbs) + + +def process_augmentations(augmenter) -> Optional[AudioAugmentor]: + """Process list of online data augmentations. + Accepts either an AudioAugmentor object with pre-defined augmentations, + or a dictionary that points to augmentations that have been defined. + If a dictionary is passed, must follow the below structure: + Dict[str, Dict[str, Any]]: Which refers to a dictionary of string + names for augmentations, defined in `asr/parts/perturb.py`. + The inner dictionary may contain key-value arguments of the specific + augmentation, along with an essential key `prob`. `prob` declares the + probability of the augmentation being applied, and must be a float + value in the range [0, 1]. + # Example in YAML config file + Augmentations are generally applied only during training, so we can add + these augmentations to our yaml config file, and modify the behaviour + for training and evaluation. + ```yaml + AudioToSpeechLabelDataLayer: + ... # Parameters shared between train and evaluation time + train: + augmentor: + shift: + prob: 0.5 + min_shift_ms: -5.0 + max_shift_ms: 5.0 + white_noise: + prob: 1.0 + min_level: -90 + max_level: -46 + ... + eval: + ... + ``` + Then in the training script, + ```python + import copy + from ruamel.yaml import YAML + yaml = YAML(typ="safe") + with open(model_config) as f: + params = yaml.load(f) + # Train Config for Data Loader + train_dl_params = copy.deepcopy(params["AudioToTextDataLayer"]) + train_dl_params.update(params["AudioToTextDataLayer"]["train"]) + del train_dl_params["train"] + del train_dl_params["eval"] + data_layer_train = nemo_asr.AudioToTextDataLayer( + ..., + **train_dl_params, + ) + # Evaluation Config for Data Loader + eval_dl_params = copy.deepcopy(params["AudioToTextDataLayer"]) + eval_dl_params.update(params["AudioToTextDataLayer"]["eval"]) + del eval_dl_params["train"] + del eval_dl_params["eval"] + data_layer_eval = nemo_asr.AudioToTextDataLayer( + ..., + **eval_dl_params, + ) + ``` + # Registering your own Augmentations + To register custom augmentations to obtain the above convenience of + the declaring the augmentations in YAML, you can put additional keys in + `perturbation_types` dictionary as follows. + ```python + from nemo.collections.asr.parts import perturb + # Define your own perturbation here + class CustomPerturbation(perturb.Perturbation): + ... + perturb.register_perturbation(name_of_perturbation, CustomPerturbation) + ``` + Args: + augmenter: AudioAugmentor object or + dictionary of str -> kwargs (dict) which is parsed and used + to initialize an AudioAugmentor. + Note: It is crucial that each individual augmentation has + a keyword `prob`, that defines a float probability in the + the range [0, 1] of this augmentation being applied. + If this keyword is not present, then the augmentation is + disabled and a warning is logged. + Returns: AudioAugmentor object + """ + if augmenter is None: + return None + + if isinstance(augmenter, AudioAugmentor): + return augmenter + + if not type(augmenter) in {dict, DictConfig}: + raise ValueError("Cannot parse augmenter. Must be a dict or an AudioAugmentor object ") + + if isinstance(augmenter, DictConfig): + augmenter = OmegaConf.to_container(augmenter, resolve=True) + + augmenter = copy.deepcopy(augmenter) + + augmentations = [] + for augment_name, augment_kwargs in augmenter.items(): + prob = augment_kwargs.get('prob', None) + + if prob is None: + raise KeyError( + f'Augmentation "{augment_name}" will not be applied as ' + f'keyword argument "prob" was not defined for this augmentation.' + ) + + else: + _ = augment_kwargs.pop('prob') + + if prob < 0.0 or prob > 1.0: + raise ValueError("`prob` must be a float value between 0 and 1.") + + try: + augmentation = perturbation_types[augment_name](**augment_kwargs) + augmentations.append([prob, augmentation]) + except KeyError: + raise KeyError(f"Invalid perturbation name. Allowed values : {perturbation_types.keys()}") + + augmenter = AudioAugmentor(perturbations=augmentations) + return augmenter + + +class AugmentationDataset(IterableDataset): + """ + A class that loads tarred audio files and cycles over the files in the dataset. + + Accepts a single comma-separated JSON manifest file (in the same style as for the AudioToCharDataset/AudioToBPEDataset), + as well as the path(s) to the tarball(s) containing the wav files. Each line of the manifest should + contain the information for one audio file, including at least the transcript and name of the audio + file within the tarball. + + Valid formats for the audio_tar_filepaths argument include: + (1) a single string that can be brace-expanded, e.g. 'path/to/audio.tar' or 'path/to/audio_{1..100}.tar.gz', or + (2) a list of file paths that will not be brace-expanded, e.g. ['audio_1.tar', 'audio_2.tar', ...]. + + Note: For brace expansion in (1), there may be cases where `{x..y}` syntax cannot be used due to shell interference. + This occurs most commonly inside SLURM scripts. Therefore we provide a few equivalent replacements. + Supported opening braces - { <=> (, [, < and the special tag _OP_. + Supported closing braces - } <=> ), ], > and the special tag _CL_. + For SLURM based tasks, we suggest the use of the special tags for ease of use. + + See the WebDataset documentation for more information about accepted data and input formats. + """ + + def __init__(self, manifest_path: str, tar_filepaths: Union[str, List[str]], shuffle_n: int = 128): + self._manifest = collections.ASRAudioText(manifest_path, parser=parsers.make_parser([]), index_by_file_id=True) + + if isinstance(tar_filepaths, str): + # Replace '(' and '[' with '{' + brace_keys_open = ['(', '[', '<', '_OP_'] + for bkey in brace_keys_open: + if bkey in tar_filepaths: + tar_filepaths = tar_filepaths.replace(bkey, "{") + + # Replace ')' and ']' with '}' + brace_keys_close = [')', ']', '>', '_CL_'] + for bkey in brace_keys_close: + if bkey in tar_filepaths: + tar_filepaths = tar_filepaths.replace(bkey, "}") + + self.audio_dataset = wd.WebDataset(urls=tar_filepaths, nodesplitter=None) + + if shuffle_n > 0: + self.audio_dataset = self.audio_dataset.shuffle(shuffle_n) + else: + logging.info("WebDataset will not shuffle files within the tar files.") + + self.audio_dataset = self.audio_dataset.rename(audio='wav', key='__key__').to_tuple('audio', 'key') + self.audio_iter = iter(self.audio_dataset) + + def __len__(self): + return len(self._manifest) + + def __iter__(self): + return self + + def __next__(self): + while True: + try: + audio_bytes, audio_filename = next(self.audio_iter) + + except StopIteration: + self.audio_iter = iter(self.audio_dataset) + audio_bytes, audio_filename = next(self.audio_iter) + file_id, _ = os.path.splitext(os.path.basename(audio_filename)) + + # Convert audio bytes to IO stream for processing (for SoundFile to read) + audio_file = io.BytesIO(audio_bytes) + return audio_file, file_id diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/segment.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/segment.py new file mode 100644 index 0000000000..b573109471 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/preprocessing/segment.py @@ -0,0 +1,270 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright (c) 2018 Ryan Leary +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# This file contains code artifacts adapted from https://github.com/ryanleary/patter + +import os +import random + +import librosa +import numpy as np +import soundfile as sf +from kaldiio.matio import read_kaldi +from kaldiio.utils import open_like_kaldi +from pydub import AudioSegment as Audio +from pydub.exceptions import CouldntDecodeError + +from nemo.utils import logging + +available_formats = sf.available_formats() +sf_supported_formats = ["." + i.lower() for i in available_formats.keys()] + + +class AudioSegment(object): + """Monaural audio segment abstraction. + :param samples: Audio samples [num_samples x num_channels]. + :type samples: ndarray.float32 + :param sample_rate: Audio sample rate. + :type sample_rate: int + :raises TypeError: If the sample data type is not float or int. + """ + + def __init__(self, samples, sample_rate, target_sr=None, trim=False, trim_db=60, orig_sr=None): + """Create audio segment from samples. + Samples are convert float32 internally, with int scaled to [-1, 1]. + """ + samples = self._convert_samples_to_float32(samples) + if target_sr is not None and target_sr != sample_rate: + samples = librosa.core.resample(samples, sample_rate, target_sr) + sample_rate = target_sr + if trim: + samples, _ = librosa.effects.trim(samples, trim_db) + self._samples = samples + self._sample_rate = sample_rate + if self._samples.ndim >= 2: + self._samples = np.mean(self._samples, 1) + + self._orig_sr = orig_sr if orig_sr is not None else sample_rate + + def __eq__(self, other): + """Return whether two objects are equal.""" + if type(other) is not type(self): + return False + if self._sample_rate != other._sample_rate: + return False + if self._samples.shape != other._samples.shape: + return False + if np.any(self.samples != other._samples): + return False + return True + + def __ne__(self, other): + """Return whether two objects are unequal.""" + return not self.__eq__(other) + + def __str__(self): + """Return human-readable representation of segment.""" + return "%s: num_samples=%d, sample_rate=%d, duration=%.2fsec, rms=%.2fdB" % ( + type(self), + self.num_samples, + self.sample_rate, + self.duration, + self.rms_db, + ) + + @staticmethod + def _convert_samples_to_float32(samples): + """Convert sample type to float32. + Audio sample type is usually integer or float-point. + Integers will be scaled to [-1, 1] in float32. + """ + float32_samples = samples.astype('float32') + if samples.dtype in np.sctypes['int']: + bits = np.iinfo(samples.dtype).bits + float32_samples *= 1.0 / 2 ** (bits - 1) + elif samples.dtype in np.sctypes['float']: + pass + else: + raise TypeError("Unsupported sample type: %s." % samples.dtype) + return float32_samples + + @classmethod + def from_file( + cls, audio_file, target_sr=None, int_values=False, offset=0, duration=0, trim=False, orig_sr=None, + ): + """ + Load a file supported by librosa and return as an AudioSegment. + :param audio_file: path of file to load + :param target_sr: the desired sample rate + :param int_values: if true, load samples as 32-bit integers + :param offset: offset in seconds when loading audio + :param duration: duration in seconds when loading audio + :return: numpy array of samples + """ + samples = None + if not isinstance(audio_file, str) or os.path.splitext(audio_file)[-1] in sf_supported_formats: + try: + with sf.SoundFile(audio_file, 'r') as f: + dtype = 'int32' if int_values else 'float32' + sample_rate = f.samplerate + if offset > 0: + f.seek(int(offset * sample_rate)) + if duration > 0: + samples = f.read(int(duration * sample_rate), dtype=dtype) + else: + samples = f.read(dtype=dtype) + samples = samples.transpose() + except RuntimeError as e: + logging.error( + f"Loading {audio_file} via SoundFile raised RuntimeError: `{e}`. " + f"NeMo will fallback to loading via pydub." + ) + elif isinstance(audio_file, str) and audio_file.strip()[-1] == "|": + f = open_like_kaldi(audio_file, "rb") + sample_rate, samples = read_kaldi(f) + if offset > 0: + samples = samples[int(offset * sample_rate) :] + if duration > 0: + samples = samples[: int(duration * sample_rate)] + if not int_values: + abs_max_value = np.abs(samples).max() + samples = np.array(samples, dtype=np.float) / abs_max_value + + if samples is None: + try: + samples = Audio.from_file(audio_file) + sample_rate = samples.frame_rate + if offset > 0: + # pydub does things in milliseconds + seconds = offset * 1000 + samples = samples[int(seconds * sample_rate) :] + if duration > 0: + seconds = duration * 1000 + samples = samples[: int(seconds)] + samples = np.array(samples.get_array_of_samples()) + except CouldntDecodeError as e: + logging.error(f"Loading {audio_file} via pydub raised CouldntDecodeError: `{e}`.") + + return cls(samples, sample_rate, target_sr=target_sr, trim=trim, orig_sr=orig_sr) + + @classmethod + def segment_from_file(cls, audio_file, target_sr=None, n_segments=0, trim=False, orig_sr=None): + """Grabs n_segments number of samples from audio_file randomly from the + file as opposed to at a specified offset. + + Note that audio_file can be either the file path, or a file-like object. + """ + try: + with sf.SoundFile(audio_file, 'r') as f: + sample_rate = f.samplerate + if n_segments > 0 and len(f) > n_segments: + max_audio_start = len(f) - n_segments + audio_start = random.randint(0, max_audio_start) + f.seek(audio_start) + samples = f.read(n_segments, dtype='float32') + else: + samples = f.read(dtype='float32') + samples = samples.transpose() + except RuntimeError as e: + logging.error(f"Loading {audio_file} via SoundFile raised RuntimeError: `{e}`.") + + samples = samples.transpose() + return cls(samples, sample_rate, target_sr=target_sr, trim=trim, orig_sr=orig_sr) + + @property + def samples(self): + return self._samples.copy() + + @property + def sample_rate(self): + return self._sample_rate + + @property + def num_samples(self): + return self._samples.shape[0] + + @property + def duration(self): + return self._samples.shape[0] / float(self._sample_rate) + + @property + def rms_db(self): + mean_square = np.mean(self._samples ** 2) + return 10 * np.log10(mean_square) + + @property + def orig_sr(self): + return self._orig_sr + + def gain_db(self, gain): + self._samples *= 10.0 ** (gain / 20.0) + + def pad(self, pad_size, symmetric=False): + """Add zero padding to the sample. The pad size is given in number + of samples. + If symmetric=True, `pad_size` will be added to both sides. If false, + `pad_size` + zeros will be added only to the end. + """ + self._samples = np.pad(self._samples, (pad_size if symmetric else 0, pad_size), mode='constant',) + + def subsegment(self, start_time=None, end_time=None): + """Cut the AudioSegment between given boundaries. + Note that this is an in-place transformation. + :param start_time: Beginning of subsegment in seconds. + :type start_time: float + :param end_time: End of subsegment in seconds. + :type end_time: float + :raise ValueError: If start_time or end_time is incorrectly set, + e.g. out + of bounds in time. + """ + start_time = 0.0 if start_time is None else start_time + end_time = self.duration if end_time is None else end_time + if start_time < 0.0: + start_time = self.duration + start_time + if end_time < 0.0: + end_time = self.duration + end_time + if start_time < 0.0: + raise ValueError("The slice start position (%f s) is out of bounds." % start_time) + if end_time < 0.0: + raise ValueError("The slice end position (%f s) is out of bounds." % end_time) + if start_time > end_time: + raise ValueError( + "The slice start position (%f s) is later than the end position (%f s)." % (start_time, end_time) + ) + if end_time > self.duration: + raise ValueError("The slice end position (%f s) is out of bounds (> %f s)" % (end_time, self.duration)) + start_sample = int(round(start_time * self._sample_rate)) + end_sample = int(round(end_time * self._sample_rate)) + self._samples = self._samples[start_sample:end_sample] diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/conformer_modules.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/conformer_modules.py new file mode 100644 index 0000000000..b0d3f7440c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/conformer_modules.py @@ -0,0 +1,190 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import torch +from torch import nn as nn +from torch.nn import LayerNorm + +from nemo.collections.asr.parts.submodules.multi_head_attention import ( + MultiHeadAttention, + RelPositionMultiHeadAttention, +) +from nemo.collections.asr.parts.utils.activations import Swish + +__all__ = ['ConformerConvolution', 'ConformerFeedForward', 'ConformerLayer'] + + +class ConformerLayer(torch.nn.Module): + """A single block of the Conformer encoder. + + Args: + d_model (int): input dimension of MultiheadAttentionMechanism and PositionwiseFeedForward + d_ff (int): hidden dimension of PositionwiseFeedForward + n_heads (int): number of heads for multi-head attention + conv_kernel_size (int): kernel size for depthwise convolution in convolution module + dropout (float): dropout probabilities for linear layers + dropout_att (float): dropout probabilities for attention distributions + """ + + def __init__( + self, + d_model, + d_ff, + self_attention_model='rel_pos', + n_heads=4, + conv_kernel_size=31, + dropout=0.1, + dropout_att=0.1, + pos_bias_u=None, + pos_bias_v=None, + ): + super(ConformerLayer, self).__init__() + + self.self_attention_model = self_attention_model + self.n_heads = n_heads + self.fc_factor = 0.5 + + # first feed forward module + self.norm_feed_forward1 = LayerNorm(d_model) + self.feed_forward1 = ConformerFeedForward(d_model=d_model, d_ff=d_ff, dropout=dropout) + + # convolution module + self.norm_conv = LayerNorm(d_model) + self.conv = ConformerConvolution(d_model=d_model, kernel_size=conv_kernel_size) + + # multi-headed self-attention module + self.norm_self_att = LayerNorm(d_model) + if self_attention_model == 'rel_pos': + self.self_attn = RelPositionMultiHeadAttention( + n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att, pos_bias_u=pos_bias_u, pos_bias_v=pos_bias_v + ) + elif self_attention_model == 'abs_pos': + self.self_attn = MultiHeadAttention(n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att) + else: + raise ValueError( + f"'{self_attention_model}' is not not a valid value for 'self_attention_model', " + f"valid values can be from ['rel_pos', 'abs_pos']" + ) + + # second feed forward module + self.norm_feed_forward2 = LayerNorm(d_model) + self.feed_forward2 = ConformerFeedForward(d_model=d_model, d_ff=d_ff, dropout=dropout) + + self.dropout = nn.Dropout(dropout) + self.norm_out = LayerNorm(d_model) + + def forward(self, x, att_mask=None, pos_emb=None, pad_mask=None): + """ + Args: + x (torch.Tensor): input signals (B, T, d_model) + att_mask (torch.Tensor): attention masks(B, T, T) + pos_emb (torch.Tensor): (L, 1, d_model) + pad_mask (torch.tensor): padding mask + Returns: + x (torch.Tensor): (B, T, d_model) + """ + residual = x + x = self.norm_feed_forward1(x) + x = self.feed_forward1(x) + x = self.fc_factor * self.dropout(x) + residual + + residual = x + x = self.norm_self_att(x) + if self.self_attention_model == 'rel_pos': + x = self.self_attn(query=x, key=x, value=x, mask=att_mask, pos_emb=pos_emb) + elif self.self_attention_model == 'abs_pos': + x = self.self_attn(query=x, key=x, value=x, mask=att_mask) + else: + x = None + x = self.dropout(x) + residual + + residual = x + x = self.norm_conv(x) + x = self.conv(x, pad_mask) + x = self.dropout(x) + residual + + residual = x + x = self.norm_feed_forward2(x) + x = self.feed_forward2(x) + x = self.fc_factor * self.dropout(x) + residual + + x = self.norm_out(x) + return x + + +class ConformerConvolution(nn.Module): + """The convolution module for the Conformer model. + Args: + d_model (int): hidden dimension + kernel_size (int): kernel size for depthwise convolution + """ + + def __init__(self, d_model, kernel_size): + super(ConformerConvolution, self).__init__() + assert (kernel_size - 1) % 2 == 0 + self.d_model = d_model + + self.pointwise_conv1 = nn.Conv1d( + in_channels=d_model, out_channels=d_model * 2, kernel_size=1, stride=1, padding=0, bias=True + ) + self.depthwise_conv = nn.Conv1d( + in_channels=d_model, + out_channels=d_model, + kernel_size=kernel_size, + stride=1, + padding=(kernel_size - 1) // 2, + groups=d_model, + bias=True, + ) + self.batch_norm = nn.BatchNorm1d(d_model) + + self.activation = Swish() + self.pointwise_conv2 = nn.Conv1d( + in_channels=d_model, out_channels=d_model, kernel_size=1, stride=1, padding=0, bias=True + ) + + def forward(self, x, pad_mask=None): + x = x.transpose(1, 2) + x = self.pointwise_conv1(x) + x = nn.functional.glu(x, dim=1) + + if pad_mask is not None: + x.masked_fill_(pad_mask.unsqueeze(1), 0.0) + + x = self.depthwise_conv(x) + x = self.batch_norm(x) + x = self.activation(x) + x = self.pointwise_conv2(x) + x = x.transpose(1, 2) + return x + + +class ConformerFeedForward(nn.Module): + """ + feed-forward module of Conformer model. + """ + + def __init__(self, d_model, d_ff, dropout, activation=Swish()): + super(ConformerFeedForward, self).__init__() + self.linear1 = nn.Linear(d_model, d_ff) + self.activation = activation + self.dropout = nn.Dropout(p=dropout) + self.linear2 = nn.Linear(d_ff, d_model) + + def forward(self, x): + x = self.linear1(x) + x = self.activation(x) + x = self.dropout(x) + x = self.linear2(x) + return x diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/jasper.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/jasper.py new file mode 100644 index 0000000000..5cd5e00b6c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/jasper.py @@ -0,0 +1,960 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Callable, List, Optional, Tuple + +import torch +import torch.nn as nn +from torch import Tensor +from torch.nn.init import _calculate_correct_fan +from torch.nn.modules.utils import _single + +from nemo.collections.asr.parts.utils.activations import Swish +from nemo.utils import logging + +try: + from pytorch_quantization import calib + from pytorch_quantization import nn as quant_nn + from pytorch_quantization import quant_modules + from pytorch_quantization.tensor_quant import QuantDescriptor + + PYTORCH_QUANTIZATION_AVAILABLE = True +except ImportError: + PYTORCH_QUANTIZATION_AVAILABLE = False + +jasper_activations = {"hardtanh": nn.Hardtanh, "relu": nn.ReLU, "selu": nn.SELU, "swish": Swish, "silu": nn.SiLU} + + +def tds_uniform_(tensor, mode='fan_in'): + """ + Uniform Initialization from the paper [Sequence-to-Sequence Speech Recognition with Time-Depth Separable Convolutions](https://www.isca-speech.org/archive/Interspeech_2019/pdfs/2460.pdf) + Normalized to - + + .. math:: + \text{bound} = \text{2} \times \sqrt{\frac{1}{\text{fan\_mode}}} + + Args: + tensor: an n-dimensional `torch.Tensor` + mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'`` + preserves the magnitude of the variance of the weights in the + forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the + backwards pass. + """ + fan = _calculate_correct_fan(tensor, mode) + gain = 2.0 # sqrt(4.0) = 2 + std = gain / math.sqrt(fan) # sqrt(4.0 / fan_in) + bound = std # Calculate uniform bounds from standard deviation + with torch.no_grad(): + return tensor.uniform_(-bound, bound) + + +def tds_normal_(tensor, mode='fan_in'): + """ + Normal Initialization from the paper [Sequence-to-Sequence Speech Recognition with Time-Depth Separable Convolutions](https://www.isca-speech.org/archive/Interspeech_2019/pdfs/2460.pdf) + Normalized to - + + .. math:: + \text{bound} = \text{2} \times \sqrt{\frac{1}{\text{fan\_mode}}} + + Args: + tensor: an n-dimensional `torch.Tensor` + mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'`` + preserves the magnitude of the variance of the weights in the + forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the + backwards pass. + """ + fan = _calculate_correct_fan(tensor, mode) + gain = 2.0 + std = gain / math.sqrt(fan) # sqrt(4.0 / fan_in) + bound = std # Calculate uniform bounds from standard deviation + with torch.no_grad(): + return tensor.normal_(0.0, bound) + + +def init_weights(m, mode: Optional[str] = 'xavier_uniform'): + if isinstance(m, MaskedConv1d): + init_weights(m.conv, mode) + if isinstance(m, (nn.Conv1d, nn.Linear)): + if mode is not None: + if mode == 'xavier_uniform': + nn.init.xavier_uniform_(m.weight, gain=1.0) + elif mode == 'xavier_normal': + nn.init.xavier_normal_(m.weight, gain=1.0) + elif mode == 'kaiming_uniform': + nn.init.kaiming_uniform_(m.weight, nonlinearity="relu") + elif mode == 'kaiming_normal': + nn.init.kaiming_normal_(m.weight, nonlinearity="relu") + elif mode == 'tds_uniform': + tds_uniform_(m.weight) + elif mode == 'tds_normal': + tds_normal_(m.weight) + else: + raise ValueError("Unknown Initialization mode: {0}".format(mode)) + elif isinstance(m, nn.BatchNorm1d): + if m.track_running_stats: + m.running_mean.zero_() + m.running_var.fill_(1) + m.num_batches_tracked.zero_() + if m.affine: + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + + +def compute_new_kernel_size(kernel_size, kernel_width): + new_kernel_size = max(int(kernel_size * kernel_width), 1) + # If kernel is even shape, round up to make it odd + if new_kernel_size % 2 == 0: + new_kernel_size += 1 + return new_kernel_size + + +def get_same_padding(kernel_size, stride, dilation) -> int: + if stride > 1 and dilation > 1: + raise ValueError("Only stride OR dilation may be greater than 1") + return (dilation * (kernel_size - 1)) // 2 + + +def get_asymtric_padding(kernel_size, stride, dilation, future_context): + if stride > 1 and dilation > 1: + raise ValueError("Only stride OR dilation may be greater than 1") + + left_context = kernel_size - 1 - future_context + right_context = future_context + + symmetric_padding = get_same_padding(kernel_size, stride, dilation) + + if kernel_size <= future_context: + # kernel size is smaller than future context, equivalent to using entire context of kernel + # simply return symmetric padding for this scenario + logging.warning( + f"Future context window is larger than the kernel size!\n" + f"Left context = {left_context} | Right context = greater than {right_context} | " + f"Kernel size = {kernel_size}\n" + f"Switching to symmetric padding (left context = right context = {symmetric_padding})" + ) + return symmetric_padding + + if left_context < symmetric_padding: + logging.warning( + f"Future context window is larger than half the kernel size!\n" + f"Conv layer therefore uses more future information than past to compute its output!\n" + f"Left context = {left_context} | Right context = {right_context} | " + f"Kernel size = {kernel_size}" + ) + + if dilation > 1: + left_context = dilation * kernel_size - 1 - dilation * future_context + right_context = dilation * future_context + return (left_context, right_context) + + return (left_context, right_context) + + +class StatsPoolLayer(nn.Module): + def __init__(self, feat_in, pool_mode='xvector'): + super().__init__() + self.feat_in = 0 + if pool_mode == 'gram': + gram = True + super_vector = False + elif pool_mode == 'superVector': + gram = True + super_vector = True + else: + gram = False + super_vector = False + + if gram: + self.feat_in += feat_in ** 2 + else: + self.feat_in += 2 * feat_in + + if super_vector and gram: + self.feat_in += 2 * feat_in + + self.gram = gram + self.super = super_vector + + def forward(self, encoder_output): + + mean = encoder_output.mean(dim=-1) # Time Axis + std = encoder_output.std(dim=-1) + + pooled = torch.cat([mean, std], dim=-1) + + if self.gram: + time_len = encoder_output.shape[-1] + # encoder_output = encoder_output + cov = encoder_output.bmm(encoder_output.transpose(2, 1)) # cov matrix + cov = cov.view(cov.shape[0], -1) / time_len + + if self.gram and not self.super: + return cov + + if self.super and self.gram: + pooled = torch.cat([pooled, cov], dim=-1) + + return pooled + + +class MaskedConv1d(nn.Module): + __constants__ = ["use_conv_mask", "real_out_channels", "heads"] + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + heads=-1, + bias=False, + use_mask=True, + quantize=False, + ): + super(MaskedConv1d, self).__init__() + + if not (heads == -1 or groups == in_channels): + raise ValueError("Only use heads for depthwise convolutions") + + self.real_out_channels = out_channels + if heads != -1: + in_channels = heads + out_channels = heads + groups = heads + + # preserve original padding + self._padding = padding + + # if padding is a tuple/list, it is considered as asymmetric padding + if type(padding) in (tuple, list): + self.pad_layer = nn.ConstantPad1d(padding, value=0.0) + # reset padding for conv since pad_layer will handle this + padding = 0 + else: + self.pad_layer = None + + if PYTORCH_QUANTIZATION_AVAILABLE and quantize: + self.conv = quant_nn.QuantConv1d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias, + ) + elif not PYTORCH_QUANTIZATION_AVAILABLE and quantize: + raise ImportError( + "pytorch-quantization is not installed. Install from " + "https://github.com/NVIDIA/TensorRT/tree/master/tools/pytorch-quantization." + ) + else: + self.conv = nn.Conv1d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias, + ) + self.use_mask = use_mask + self.heads = heads + + # Calculations for "same" padding cache + self.same_padding = (self.conv.stride[0] == 1) and ( + 2 * self.conv.padding[0] == self.conv.dilation[0] * (self.conv.kernel_size[0] - 1) + ) + if self.pad_layer is None: + self.same_padding_asymmetric = False + else: + self.same_padding_asymmetric = (self.conv.stride[0] == 1) and ( + sum(self._padding) == self.conv.dilation[0] * (self.conv.kernel_size[0] - 1) + ) + + # `self.lens` caches consecutive integers from 0 to `self.max_len` that are used to compute the mask for a + # batch. Recomputed to bigger size as needed. Stored on a device of the latest batch lens. + if self.use_mask: + self.max_len = 0 + self.lens = None + + def get_seq_len(self, lens): + if self.same_padding or self.same_padding_asymmetric: + return lens + + if self.pad_layer is None: + return ( + lens + 2 * self.conv.padding[0] - self.conv.dilation[0] * (self.conv.kernel_size[0] - 1) - 1 + ) // self.conv.stride[0] + 1 + else: + return ( + lens + sum(self._padding) - self.conv.dilation[0] * (self.conv.kernel_size[0] - 1) - 1 + ) // self.conv.stride[0] + 1 + + def forward(self, x, lens): + if self.use_mask: + max_len = x.size(2) + if max_len > self.max_len: + self.lens = torch.arange(max_len) + self.max_len = max_len + + self.lens = self.lens.to(lens.device) + mask = self.lens[:max_len].unsqueeze(0) < lens.unsqueeze(1) + x = x * mask.unsqueeze(1).to(device=x.device) + lens = self.get_seq_len(lens) + + # asymmtric pad if necessary + if self.pad_layer is not None: + x = self.pad_layer(x) + + sh = x.shape + if self.heads != -1: + x = x.view(-1, self.heads, sh[-1]) + + out = self.conv(x) + + if self.heads != -1: + out = out.view(sh[0], self.real_out_channels, -1) + + return out, lens + + +class GroupShuffle(nn.Module): + def __init__(self, groups, channels): + super(GroupShuffle, self).__init__() + + self.groups = groups + self.channels_per_group = channels // groups + + def forward(self, x): + sh = x.shape + + x = x.view(-1, self.groups, self.channels_per_group, sh[-1]) + + x = torch.transpose(x, 1, 2).contiguous() + + x = x.view(-1, self.groups * self.channels_per_group, sh[-1]) + + return x + + +class SqueezeExcite(nn.Module): + def __init__( + self, + channels: int, + reduction_ratio: int, + context_window: int = -1, + interpolation_mode: str = 'nearest', + activation: Optional[Callable] = None, + quantize: bool = False, + ): + """ + Squeeze-and-Excitation sub-module. + + Args: + channels: Input number of channels. + reduction_ratio: Reduction ratio for "squeeze" layer. + context_window: Integer number of timesteps that the context + should be computed over, using stride 1 average pooling. + If value < 1, then global context is computed. + interpolation_mode: Interpolation mode of timestep dimension. + Used only if context window is > 1. + The modes available for resizing are: `nearest`, `linear` (3D-only), + `bilinear`, `area` + activation: Intermediate activation function used. Must be a + callable activation function. + """ + super(SqueezeExcite, self).__init__() + self.interpolation_mode = interpolation_mode + self._quantize = quantize + + self.pool = None # prepare a placeholder which will be updated + self.change_context_window(context_window=context_window) + + if activation is None: + activation = nn.ReLU(inplace=True) + + if PYTORCH_QUANTIZATION_AVAILABLE and quantize: + self.fc = nn.Sequential( + quant_nn.QuantLinear(channels, channels // reduction_ratio, bias=False), + activation, + quant_nn.QuantLinear(channels // reduction_ratio, channels, bias=False), + ) + elif not PYTORCH_QUANTIZATION_AVAILABLE and quantize: + raise ImportError( + "pytorch-quantization is not installed. Install from " + "https://github.com/NVIDIA/TensorRT/tree/master/tools/pytorch-quantization." + ) + else: + self.fc = nn.Sequential( + nn.Linear(channels, channels // reduction_ratio, bias=False), + activation, + nn.Linear(channels // reduction_ratio, channels, bias=False), + ) + self.gap = nn.AdaptiveAvgPool1d(1) + + def forward(self, x): + # The use of negative indices on the transpose allow for expanded SqueezeExcite + batch, channels, timesteps = x.size()[:3] + # Computes in float32 to avoid instabilities during training with AMP. + with torch.cuda.amp.autocast(enabled=False): + x = x.float() + if timesteps < self.context_window: + y = self.gap(x) + else: + y = self.pool(x) # [B, C, T - context_window + 1] + y = y.transpose(1, -1) # [B, T - context_window + 1, C] + y = self.fc(y) # [B, T - context_window + 1, C] + y = y.transpose(1, -1) # [B, C, T - context_window + 1] + + if self.context_window > 0: + y = torch.nn.functional.interpolate(y, size=timesteps, mode=self.interpolation_mode) + + y = torch.sigmoid(y) + return x * y + + def change_context_window(self, context_window: int): + """ + Update the context window of the SqueezeExcitation module, in-place if possible. + + Will update the pooling layer to either nn.AdaptiveAvgPool1d() (for global SE) or nn.AvgPool1d() + (for limited context SE). + + If only the context window is changing but still a limited SE context block - then + the earlier instance of nn.AvgPool1d() will be updated. + + Args: + context_window: An integer representing the number of input timeframes that will be used + to compute the context. Each timeframe corresponds to a single window stride of the + STFT features. + + Say the window_stride = 0.01s, then a context window of 128 represents 128 * 0.01 s + of context to compute the Squeeze step. + """ + if hasattr(self, 'context_window'): + logging.info(f"Changing Squeeze-Excitation context window from {self.context_window} to {context_window}") + + self.context_window = int(context_window) + + if self.context_window <= 0: + if PYTORCH_QUANTIZATION_AVAILABLE and self._quantize: + if not isinstance(self.pool, quant_nn.QuantAdaptiveAvgPool1d(1)): + self.pool = quant_nn.QuantAdaptiveAvgPool1d(1) # context window = T + + elif not PYTORCH_QUANTIZATION_AVAILABLE and self._quantize: + raise ImportError( + "pytorch-quantization is not installed. Install from " + "https://github.com/NVIDIA/TensorRT/tree/master/tools/pytorch-quantization." + ) + + else: + if not isinstance(self.pool, nn.AdaptiveAvgPool1d): + self.pool = nn.AdaptiveAvgPool1d(1) # context window = T + else: + if PYTORCH_QUANTIZATION_AVAILABLE and self._quantize: + if not isinstance(self.pool, quant_nn.QuantAvgPool1d): + self.pool = quant_nn.QuantAvgPool1d(self.context_window, stride=1) + + elif not PYTORCH_QUANTIZATION_AVAILABLE and self._quantize: + raise ImportError( + "pytorch-quantization is not installed. Install from " + "https://github.com/NVIDIA/TensorRT/tree/master/tools/pytorch-quantization." + ) + + else: + if not isinstance(self.pool, nn.AvgPool1d): + self.pool = nn.AvgPool1d(self.context_window, stride=1) + else: + # update the context window + self.pool.kernel_size = _single(self.context_window) + + +class JasperBlock(nn.Module): + """ + Constructs a single "Jasper" block. With modified parameters, also constructs other blocks for models + such as `QuartzNet` and `Citrinet`. + + - For `Jasper` : `separable` flag should be False + - For `QuartzNet` : `separable` flag should be True + - For `Citrinet` : `separable` flag and `se` flag should be True + + Note that above are general distinctions, each model has intricate differences that expand over + multiple such blocks. + + For further information about the differences between models which use JasperBlock, please review + the configs for ASR models found in the ASR examples directory. + + Args: + inplanes: Number of input channels. + planes: Number of output channels. + repeat: Number of repeated sub-blocks (R) for this block. + kernel_size: Convolution kernel size across all repeated sub-blocks. + kernel_size_factor: Floating point scale value that is multiplied with kernel size, + then rounded down to nearest odd integer to compose the kernel size. Defaults to 1.0. + stride: Stride of the convolutional layers. + dilation: Integer which defined dilation factor of kernel. Note that when dilation > 1, stride must + be equal to 1. + padding: String representing type of padding. Currently only supports "same" padding, + which symmetrically pads the input tensor with zeros. + dropout: Floating point value, determins percentage of output that is zeroed out. + activation: String representing activation functions. Valid activation functions are : + {"hardtanh": nn.Hardtanh, "relu": nn.ReLU, "selu": nn.SELU, "swish": Swish}. + Defaults to "relu". + residual: Bool that determined whether a residual branch should be added or not. + All residual branches are constructed using a pointwise convolution kernel, that may or may not + perform strided convolution depending on the parameter `residual_mode`. + groups: Number of groups for Grouped Convolutions. Defaults to 1. + separable: Bool flag that describes whether Time-Channel depthwise separable convolution should be + constructed, or ordinary convolution should be constructed. + heads: Number of "heads" for the masked convolution. Defaults to -1, which disables it. + normalization: String that represents type of normalization performed. Can be one of + "batch", "group", "instance" or "layer" to compute BatchNorm1D, GroupNorm1D, InstanceNorm or + LayerNorm (which are special cases of GroupNorm1D). + norm_groups: Number of groups used for GroupNorm (if `normalization` == "group"). + residual_mode: String argument which describes whether the residual branch should be simply + added ("add") or should first stride, then add ("stride_add"). Required when performing stride on + parallel branch as well as utilizing residual add. + residual_panes: Number of residual panes, used for Jasper-DR models. Please refer to the paper. + conv_mask: Bool flag which determines whether to utilize masked convolutions or not. In general, + it should be set to True. + se: Bool flag that determines whether Squeeze-and-Excitation layer should be used. + se_reduction_ratio: Integer value, which determines to what extend the hidden dimension of the SE + intermediate step should be reduced. Larger values reduce number of parameters, but also limit + the effectiveness of SE layers. + se_context_window: Integer value determining the number of timesteps that should be utilized in order + to compute the averaged context window. Defaults to -1, which means it uses global context - such + that all timesteps are averaged. If any positive integer is used, it will utilize limited context + window of that size. + se_interpolation_mode: String used for interpolation mode of timestep dimension for SE blocks. + Used only if context window is > 1. + The modes available for resizing are: `nearest`, `linear` (3D-only), + `bilinear`, `area`. + stride_last: Bool flag that determines whether all repeated blocks should stride at once, + (stride of S^R when this flag is False) or just the last repeated block should stride + (stride of S when this flag is True). + future_context: Int value that determins how many "right" / "future" context frames will be utilized + when calculating the output of the conv kernel. All calculations are done for odd kernel sizes only. + + By default, this is -1, which is recomputed as the symmetric padding case. + + When future_context >= 0, will compute the asymmetric padding as follows : + (left context, right context) = [K - 1 - future_context, future_context] + + Determining an exact formula to limit future context is dependent on global layout of the model. + As such, we provide both "local" and "global" guidelines below. + + Local context limit (should always be enforced) + - future context should be <= half the kernel size for any given layer + - future context > kernel size defaults to symmetric kernel + - future context of layer = number of future frames * width of each frame (dependent on stride) + + Global context limit (should be carefully considered) + - future context should be layed out in an ever reducing pattern. Initial layers should restrict + future context less than later layers, since shallow depth (and reduced stride) means each frame uses + less amounts of future context. + - Beyond a certain point, future context should remain static for a given stride level. This is + the upper bound of the amount of future context that can be provided to the model on a global scale. + - future context is calculated (roughly) as - (2 ^ stride) * (K // 2) number of future frames. + This resultant value should be bound to some global maximum number of future seconds of audio (in ms). + + Note: In the special case where K < future_context, it is assumed that the kernel is too small to limit + its future context, so symmetric padding is used instead. + + Note: There is no explicit limitation on the amount of future context used, as long as + K > future_context constraint is maintained. This might lead to cases where future_context is + more than half the actual kernel size K! In such cases, the conv layer is utilizing more of the future + context than its current and past context to compute the output. While this is possible to do, + it is not recommended and the layer will raise a warning to notify the user of such cases. + It is advised to simply use symmetric padding for such cases. + + Example: + Say we have a model that performs 8x stride and receives spectrogram frames with stride of 0.01s. + Say we wish to upper bound future context to 80 ms. + + Layer ID, Kernel Size, Stride, Future Context, Global Context + 0, K=5, S=1, FC=8, GC= 2 * (2^0) = 2 * 0.01 ms (special case, K < FC so use symmetric pad) + 1, K=7, S=1, FC=3, GC= 3 * (2^0) = 3 * 0.01 ms (note that symmetric pad here uses 3 FC frames!) + 2, K=11, S=2, FC=4, GC= 4 * (2^1) = 8 * 0.01 ms (note that symmetric pad here uses 5 FC frames!) + 3, K=15, S=1, FC=4, GC= 4 * (2^1) = 8 * 0.01 ms (note that symmetric pad here uses 7 FC frames!) + 4, K=21, S=2, FC=2, GC= 2 * (2^2) = 8 * 0.01 ms (note that symmetric pad here uses 10 FC frames!) + 5, K=25, S=2, FC=1, GC= 1 * (2^3) = 8 * 0.01 ms (note that symmetric pad here uses 14 FC frames!) + 6, K=29, S=1, FC=1, GC= 1 * (2^3) = 8 * 0.01 ms ... + quantize: Bool flag whether to quantize the Convolutional blocks. + """ + + __constants__ = ["conv_mask", "separable", "residual_mode", "res", "mconv"] + + def __init__( + self, + inplanes, + planes, + repeat=3, + kernel_size=11, + kernel_size_factor=1, + stride=1, + dilation=1, + padding='same', + dropout=0.2, + activation=None, + residual=True, + groups=1, + separable=False, + heads=-1, + normalization="batch", + norm_groups=1, + residual_mode='add', + residual_panes=[], + conv_mask=False, + se=False, + se_reduction_ratio=16, + se_context_window=-1, + se_interpolation_mode='nearest', + stride_last=False, + future_context: int = -1, + quantize=False, + ): + super(JasperBlock, self).__init__() + + if padding != "same": + raise ValueError("currently only 'same' padding is supported") + + kernel_size_factor = float(kernel_size_factor) + if type(kernel_size) in (list, tuple): + kernel_size = [compute_new_kernel_size(k, kernel_size_factor) for k in kernel_size] + else: + kernel_size = compute_new_kernel_size(kernel_size, kernel_size_factor) + + if future_context < 0: + padding_val = get_same_padding(kernel_size[0], stride[0], dilation[0]) + else: + padding_val = get_asymtric_padding(kernel_size[0], stride[0], dilation[0], future_context) + + self.conv_mask = conv_mask + self.separable = separable + self.residual_mode = residual_mode + self.se = se + self.quantize = quantize + + inplanes_loop = inplanes + conv = nn.ModuleList() + + for _ in range(repeat - 1): + # Stride last means only the last convolution in block will have stride + if stride_last: + stride_val = [1] + else: + stride_val = stride + + conv.extend( + self._get_conv_bn_layer( + inplanes_loop, + planes, + kernel_size=kernel_size, + stride=stride_val, + dilation=dilation, + padding=padding_val, + groups=groups, + heads=heads, + separable=separable, + normalization=normalization, + norm_groups=norm_groups, + quantize=quantize, + ) + ) + + conv.extend(self._get_act_dropout_layer(drop_prob=dropout, activation=activation)) + + inplanes_loop = planes + + conv.extend( + self._get_conv_bn_layer( + inplanes_loop, + planes, + kernel_size=kernel_size, + stride=stride, + dilation=dilation, + padding=padding_val, + groups=groups, + heads=heads, + separable=separable, + normalization=normalization, + norm_groups=norm_groups, + quantize=quantize, + ) + ) + + if se: + conv.append( + SqueezeExcite( + planes, + reduction_ratio=se_reduction_ratio, + context_window=se_context_window, + interpolation_mode=se_interpolation_mode, + activation=activation, + quantize=quantize, + ) + ) + + self.mconv = conv + + res_panes = residual_panes.copy() + self.dense_residual = residual + + if residual: + res_list = nn.ModuleList() + + if residual_mode == 'stride_add': + stride_val = stride + else: + stride_val = [1] + + if len(residual_panes) == 0: + res_panes = [inplanes] + self.dense_residual = False + for ip in res_panes: + res = nn.ModuleList( + self._get_conv_bn_layer( + ip, + planes, + kernel_size=1, + normalization=normalization, + norm_groups=norm_groups, + stride=stride_val, + quantize=quantize, + ) + ) + + res_list.append(res) + + self.res = res_list + if PYTORCH_QUANTIZATION_AVAILABLE and self.quantize: + self.residual_quantizer = quant_nn.TensorQuantizer(quant_nn.QuantConv2d.default_quant_desc_input) + elif not PYTORCH_QUANTIZATION_AVAILABLE and quantize: + raise ImportError( + "pytorch-quantization is not installed. Install from " + "https://github.com/NVIDIA/TensorRT/tree/master/tools/pytorch-quantization." + ) + else: + self.res = None + + self.mout = nn.Sequential(*self._get_act_dropout_layer(drop_prob=dropout, activation=activation)) + + def _get_conv( + self, + in_channels, + out_channels, + kernel_size=11, + stride=1, + dilation=1, + padding=0, + bias=False, + groups=1, + heads=-1, + separable=False, + quantize=False, + ): + use_mask = self.conv_mask + if use_mask: + return MaskedConv1d( + in_channels, + out_channels, + kernel_size, + stride=stride, + dilation=dilation, + padding=padding, + bias=bias, + groups=groups, + heads=heads, + use_mask=use_mask, + quantize=quantize, + ) + else: + if PYTORCH_QUANTIZATION_AVAILABLE and quantize: + return quant_nn.QuantConv1d( + in_channels, + out_channels, + kernel_size, + stride=stride, + dilation=dilation, + padding=padding, + bias=bias, + groups=groups, + ) + elif not PYTORCH_QUANTIZATION_AVAILABLE and quantize: + raise ImportError( + "pytorch-quantization is not installed. Install from " + "https://github.com/NVIDIA/TensorRT/tree/master/tools/pytorch-quantization." + ) + else: + return nn.Conv1d( + in_channels, + out_channels, + kernel_size, + stride=stride, + dilation=dilation, + padding=padding, + bias=bias, + groups=groups, + ) + + def _get_conv_bn_layer( + self, + in_channels, + out_channels, + kernel_size=11, + stride=1, + dilation=1, + padding=0, + bias=False, + groups=1, + heads=-1, + separable=False, + normalization="batch", + norm_groups=1, + quantize=False, + ): + if norm_groups == -1: + norm_groups = out_channels + + if separable: + layers = [ + self._get_conv( + in_channels, + in_channels, + kernel_size, + stride=stride, + dilation=dilation, + padding=padding, + bias=bias, + groups=in_channels, + heads=heads, + quantize=quantize, + ), + self._get_conv( + in_channels, + out_channels, + kernel_size=1, + stride=1, + dilation=1, + padding=0, + bias=bias, + groups=groups, + quantize=quantize, + ), + ] + else: + layers = [ + self._get_conv( + in_channels, + out_channels, + kernel_size, + stride=stride, + dilation=dilation, + padding=padding, + bias=bias, + groups=groups, + quantize=quantize, + ) + ] + + if normalization == "group": + layers.append(nn.GroupNorm(num_groups=norm_groups, num_channels=out_channels)) + elif normalization == "instance": + layers.append(nn.GroupNorm(num_groups=out_channels, num_channels=out_channels)) + elif normalization == "layer": + layers.append(nn.GroupNorm(num_groups=1, num_channels=out_channels)) + elif normalization == "batch": + layers.append(nn.BatchNorm1d(out_channels, eps=1e-3, momentum=0.1)) + else: + raise ValueError( + f"Normalization method ({normalization}) does not match" f" one of [batch, layer, group, instance]." + ) + + if groups > 1: + layers.append(GroupShuffle(groups, out_channels)) + return layers + + def _get_act_dropout_layer(self, drop_prob=0.2, activation=None): + if activation is None: + activation = nn.Hardtanh(min_val=0.0, max_val=20.0) + layers = [activation, nn.Dropout(p=drop_prob)] + return layers + + def forward(self, input_: Tuple[List[Tensor], Optional[Tensor]]): + """ + Forward pass of the module. + + Args: + input_: The input is a tuple of two values - the preprocessed audio signal as well as the lengths + of the audio signal. The audio signal is padded to the shape [B, D, T] and the lengths are + a torch vector of length B. + + Returns: + The output of the block after processing the input through `repeat` number of sub-blocks, + as well as the lengths of the encoded audio after padding/striding. + """ + # type: (Tuple[List[Tensor], Optional[Tensor]]) -> Tuple[List[Tensor], Optional[Tensor]] # nopep8 + lens_orig = None + xs = input_[0] + if len(input_) == 2: + xs, lens_orig = input_ + + # compute forward convolutions + out = xs[-1] + + lens = lens_orig + for i, l in enumerate(self.mconv): + # if we're doing masked convolutions, we need to pass in and + # possibly update the sequence lengths + # if (i % 4) == 0 and self.conv_mask: + if isinstance(l, MaskedConv1d): + out, lens = l(out, lens) + else: + out = l(out) + + # compute the residuals + if self.res is not None: + for i, layer in enumerate(self.res): + res_out = xs[i] + for j, res_layer in enumerate(layer): + if isinstance(res_layer, MaskedConv1d): + res_out, _ = res_layer(res_out, lens_orig) + else: + res_out = res_layer(res_out) + + if self.residual_mode == 'add' or self.residual_mode == 'stride_add': + if PYTORCH_QUANTIZATION_AVAILABLE and self.quantize: + out = self.residual_quantizer(out) + res_out + elif not PYTORCH_QUANTIZATION_AVAILABLE and self.quantize: + raise ImportError( + "pytorch-quantization is not installed. Install from " + "https://github.com/NVIDIA/TensorRT/tree/master/tools/pytorch-quantization." + ) + else: + out = out + res_out + else: + out = torch.max(out, res_out) + + # compute the output + out = self.mout(out) + if self.res is not None and self.dense_residual: + return xs + [out], lens + + return [out], lens diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/multi_head_attention.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/multi_head_attention.py new file mode 100644 index 0000000000..f34a746f68 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/multi_head_attention.py @@ -0,0 +1,340 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright 2017 Johns Hopkins University (Shinji Watanabe) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Part of this code is adopted from https://github.com/espnet/espnet +""" + +import math + +import numpy as np +import torch +import torch.nn as nn + +__all__ = [ + 'RelPositionMultiHeadAttention', + 'RelPositionalEncoding', + 'PositionalEncoding', +] + + +class MultiHeadAttention(nn.Module): + """Multi-Head Attention layer of Transformer. + Args: + n_head (int): number of heads + n_feat (int): size of the features + dropout_rate (float): dropout rate + """ + + def __init__(self, n_head, n_feat, dropout_rate): + """Construct an MultiHeadedAttention object.""" + super(MultiHeadAttention, self).__init__() + assert n_feat % n_head == 0 + # We assume d_v always equals d_k + self.d_k = n_feat // n_head + self.h = n_head + self.linear_q = nn.Linear(n_feat, n_feat) + self.linear_k = nn.Linear(n_feat, n_feat) + self.linear_v = nn.Linear(n_feat, n_feat) + self.linear_out = nn.Linear(n_feat, n_feat) + self.attn = None + self.dropout = nn.Dropout(p=dropout_rate) + + def forward_qkv(self, query, key, value): + """Transforms query, key and value. + Args: + query (torch.Tensor): (batch, time1, size) + key (torch.Tensor): (batch, time2, size) + value (torch.Tensor): (batch, time2, size) + returns: + q (torch.Tensor): (batch, head, time1, size) + k (torch.Tensor): (batch, head, time2, size) + v (torch.Tensor): (batch, head, time2, size) + """ + n_batch = query.size(0) + q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k) + k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k) + v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + return q, k, v + + def forward_attention(self, value, scores, mask): + """Compute attention context vector. + Args: + value (torch.Tensor): (batch, time2, size) + scores(torch.Tensor): (batch, time1, time2) + mask(torch.Tensor): (batch, time1, time2) + returns: + value (torch.Tensor): transformed `value` (batch, time2, d_model) weighted by the attention scores + """ + n_batch = value.size(0) + if mask is not None: + mask = mask.unsqueeze(1) # (batch, 1, time1, time2) + if scores.dtype == torch.float16: + dtype = np.float16 + else: + dtype = np.float32 + min_value = np.finfo(dtype).min + scores = scores.masked_fill(mask, min_value) + self.attn = torch.softmax(scores, dim=-1).masked_fill(mask, 0.0) # (batch, head, time1, time2) + else: + self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) + + p_attn = self.dropout(self.attn) + x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) + x = x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) # (batch, time1, d_model) + + return self.linear_out(x) # (batch, time1, d_model) + + def forward(self, query, key, value, mask, pos_emb=None): + """Compute 'Scaled Dot Product Attention'. + Args: + query (torch.Tensor): (batch, time1, size) + key (torch.Tensor): (batch, time2, size) + value(torch.Tensor): (batch, time2, size) + mask (torch.Tensor): (batch, time1, time2) + returns: + output (torch.Tensor): transformed `value` (batch, time1, d_model) weighted by the query dot key attention + """ + q, k, v = self.forward_qkv(query, key, value) + scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k) + return self.forward_attention(v, scores, mask) + + +class RelPositionMultiHeadAttention(MultiHeadAttention): + """Multi-Head Attention layer of Transformer-XL with support of relative positional encoding. + Paper: https://arxiv.org/abs/1901.02860 + Args: + n_head (int): number of heads + n_feat (int): size of the features + dropout_rate (float): dropout rate + """ + + def __init__(self, n_head, n_feat, dropout_rate, pos_bias_u, pos_bias_v): + """Construct an RelPositionMultiHeadedAttention object.""" + super().__init__(n_head, n_feat, dropout_rate) + # linear transformation for positional encoding + self.linear_pos = nn.Linear(n_feat, n_feat, bias=False) + # these two learnable biases are used in matrix c and matrix d + # as described in https://arxiv.org/abs/1901.02860 Section 3.3 + if pos_bias_u is None or pos_bias_v is None: + self.pos_bias_u = nn.Parameter(torch.FloatTensor(self.h, self.d_k)) + self.pos_bias_v = nn.Parameter(torch.FloatTensor(self.h, self.d_k)) + # nn.init.normal_(self.pos_bias_u, 0.0, 0.02) + # nn.init.normal_(self.pos_bias_v, 0.0, 0.02) + nn.init.zeros_(self.pos_bias_u) + nn.init.zeros_(self.pos_bias_v) + else: + self.pos_bias_u = pos_bias_u + self.pos_bias_v = pos_bias_v + + def rel_shift(self, x): + """Compute relative positional encoding. + Args: + x (torch.Tensor): (batch, nheads, time, 2*time-1) + """ + x_size = x.size() # (b, h, t1, t2) + qlen = x_size[-2] + pos_len = x_size[-1] + + # need to add a column of zeros on the left side of last two dimensions to perform the relative shifting + x = torch.nn.functional.pad(x, pad=(1, 0, 0, 0)) # (b, h, t1, t2+1) + x = x.view(*x.size()[:-2], pos_len + 1, qlen) # (b, h, t2+1, t1) + # need to drop the first row + x = x[:, :, 1:].view(*x_size) # (b, h, t1, t2) + return x + + def forward(self, query, key, value, mask, pos_emb): + """Compute 'Scaled Dot Product Attention' with rel. positional encoding. + Args: + query (torch.Tensor): (batch, time1, size) + key (torch.Tensor): (batch, time2, size) + value(torch.Tensor): (batch, time2, size) + mask (torch.Tensor): (batch, time1, time2) + pos_emb (torch.Tensor) : (batch, time1, size) + Returns: + output (torch.Tensor): transformed `value` (batch, time1, d_model) weighted by the query dot key attention + """ + q, k, v = self.forward_qkv(query, key, value) + q = q.transpose(1, 2) # (batch, time1, head, d_k) + + n_batch_pos = pos_emb.size(0) + p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) + p = p.transpose(1, 2) # (batch, head, time1, d_k) + + # (batch, head, time1, d_k) + q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) + # (batch, head, time1, d_k) + q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) + + # compute attention score + # first compute matrix a and matrix c + # as described in https://arxiv.org/abs/1901.02860 Section 3.3 + # (batch, head, time1, time2) + matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) + + # compute matrix b and matrix d + # (batch, head, time1, time2) + matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) + matrix_bd = self.rel_shift(matrix_bd) + # drops extra elements in the matrix_bd if there are any left from rel_shift() to match the matrix_ac's size + matrix_bd = matrix_bd[:, :, :, : matrix_ac.size(-1)] + + scores = (matrix_ac + matrix_bd) / math.sqrt(self.d_k) # (batch, head, time1, time2) + + return self.forward_attention(v, scores, mask) + + +class PositionalEncoding(torch.nn.Module): + """Fixed sinusoidal positional encoding. + Args: + d_model (int): embedding dim + dropout_rate (float): dropout rate + max_len (int): maximum input length + xscale (bool): whether to scale the input by sqrt(d_model) + dropout_rate_emb (float): dropout rate for the positional embeddings + """ + + def __init__(self, d_model, dropout_rate, max_len=5000, xscale=None, dropout_rate_emb=0.0): + """Construct an PositionalEncoding object.""" + super(PositionalEncoding, self).__init__() + self.d_model = d_model + self.xscale = xscale + self.dropout = torch.nn.Dropout(p=dropout_rate) + self.extend_pe(length=max_len) + if dropout_rate_emb > 0: + self.dropout_emb = nn.Dropout(dropout_rate_emb) + else: + self.dropout_emb = None + + def create_pe(self, positions): + pos_length = positions.size(0) + pe = torch.zeros(pos_length, self.d_model) + div_term = torch.exp( + torch.arange(0, self.d_model, 2, dtype=torch.float32) * -(math.log(10000.0) / self.d_model) + ) + pe[:, 0::2] = torch.sin(positions * div_term) + pe[:, 1::2] = torch.cos(positions * div_term) + pe = pe.unsqueeze(0) + return pe + + def extend_pe(self, length): + """Reset and extend the positional encodings if needed.""" + if hasattr(self, 'pe') and self.pe.size(1) >= length: + return + positions = torch.arange(0, length, dtype=torch.float32).unsqueeze(1) + pe = self.create_pe(positions=positions) + if not hasattr(self, 'pe'): + self.register_buffer('pe', pe, persistent=False) + self.pe = pe + + def forward(self, x: torch.Tensor): + """Adds positional encoding. + Args: + x (torch.Tensor): Input. Its shape is (batch, time, feature_size) + Returns: + x+pos_emb (torch.Tensor): Its shape is (batch, time, feature_size) + pos_emb (torch.Tensor): Its shape is (1, time, feature_size) + """ + self.extend_pe(x) + if self.pe.dtype != x.dtype or self.pe.device != x.device: + self.pe = self.pe.to(device=x.device, dtype=x.dtype) + + if self.xscale: + x = x * self.xscale + pos_emb = self.pe[:, : x.size(1)] + if self.dropout_emb: + pos_emb = self.dropout_emb(pos_emb) + x = x + pos_emb + return self.dropout(x), pos_emb + + +class RelPositionalEncoding(PositionalEncoding): + """Relative positional encoding for TransformerXL's layers + See : Appendix B in https://arxiv.org/abs/1901.02860 + Args: + d_model (int): embedding dim + dropout_rate (float): dropout rate + max_len (int): maximum input length + xscale (bool): whether to scale the input by sqrt(d_model) + dropout_rate_emb (float): dropout rate for the positional embeddings + """ + + def __init__(self, d_model, dropout_rate, max_len=5000, xscale=None, dropout_rate_emb=0.0): + super().__init__(d_model, dropout_rate, max_len, xscale=xscale) + + if dropout_rate_emb > 0: + self.dropout_emb = nn.Dropout(dropout_rate_emb) + else: + self.dropout_emb = None + + self.max_len = max_len + + def extend_pe(self, length): + """Reset and extend the positional encodings if needed.""" + needed_size = 2 * (length - 1) + 1 + if hasattr(self, 'pe') and self.pe.size(1) >= needed_size: + return + # positions would be from negative numbers to positive + # positive positions would be used for left positions and negative for right positions + positions = torch.arange(length - 1, -length, -1, dtype=torch.float32).unsqueeze(1) + pe = self.create_pe(positions=positions) + if not hasattr(self, 'pe'): + self.register_buffer('pe', pe, persistent=False) + self.pe = pe + + def forward(self, x): + """Compute positional encoding. + Args: + x (torch.Tensor): Input. Its shape is (batch, time, feature_size) + Returns: + x (torch.Tensor): Its shape is (batch, time, feature_size) + pos_emb (torch.Tensor): Its shape is (1, time, feature_size) + """ + self.extend_pe(length=x.size(1)) + if self.pe.dtype != x.dtype or self.pe.device != x.device: + self.pe = self.pe.to(device=x.device, dtype=x.dtype) + + if self.xscale: + x = x * self.xscale + + # center_pos would be the index of position 0 + # negative positions would be used for right and positive for left tokens + # for input of length L, 2*L-1 positions are needed, positions from (L-1) to -(L-1) + center_pos = self.pe.size(1) // 2 + start_pos = center_pos - x.size(1) + 1 + end_pos = center_pos + x.size(1) + pos_emb = self.pe[:, start_pos:end_pos] + if self.dropout_emb: + pos_emb = self.dropout_emb(pos_emb) + return self.dropout(x), pos_emb diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/rnnt_beam_decoding.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/rnnt_beam_decoding.py new file mode 100644 index 0000000000..203aa750f6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/rnnt_beam_decoding.py @@ -0,0 +1,791 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2017 Johns Hopkins University (Shinji Watanabe) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +from dataclasses import dataclass +from typing import List, Optional, Union + +import numpy as np +import torch +from tqdm import tqdm + +from nemo.collections.asr.modules import rnnt_abstract +from nemo.collections.asr.parts.utils.rnnt_utils import Hypothesis, NBestHypotheses +from nemo.core.classes import Typing, typecheck +from nemo.core.neural_types import AcousticEncodedRepresentation, HypothesisType, LengthsType, NeuralType + + +class BeamRNNTInfer(Typing): + """ + Beam Search implementation ported from ESPNet implementation - + https://github.com/espnet/espnet/blob/master/espnet/nets/beam_search_transducer.py + + Sequence level beam decoding or batched-beam decoding, performed auto-repressively + depending on the search type chosen. + + Args: + decoder_model: rnnt_utils.AbstractRNNTDecoder implementation. + joint_model: rnnt_utils.AbstractRNNTJoint implementation. + + beam_size: number of beams for beam search. Must be a positive integer >= 1. + If beam size is 1, defaults to stateful greedy search. + This greedy search might result in slightly different results than + the greedy results obtained by GreedyRNNTInfer due to implementation differences. + + For accurate greedy results, please use GreedyRNNTInfer or GreedyBatchedRNNTInfer. + + search_type: str representing the type of beam search to perform. + Must be one of ['beam', 'tsd', 'alsd']. 'nsc' is currently not supported. + + Algoritm used: + `beam` - basic beam search strategy. Larger beams generally result in better decoding, + however the time required for the search also grows steadily. + + `tsd` - time synchronous decoding. Please refer to the paper: + [Alignment-Length Synchronous Decoding for RNN Transducer](https://ieeexplore.ieee.org/document/9053040) + for details on the algorithm implemented. + + Time synchronous decoding (TSD) execution time grows by the factor T * max_symmetric_expansions. + For longer sequences, T is greater, and can therefore take a long time for beams to obtain + good results. This also requires greater memory to execute. + + `alsd` - alignment-length synchronous decoding. Please refer to the paper: + [Alignment-Length Synchronous Decoding for RNN Transducer](https://ieeexplore.ieee.org/document/9053040) + for details on the algorithm implemented. + + Alignment-length synchronous decoding (ALSD) execution time is faster than TSD, with growth + factor of T + U_max, where U_max is the maximum target length expected during execution. + + Generally, T + U_max < T * max_symmetric_expansions. However, ALSD beams are non-unique, + therefore it is required to use larger beam sizes to achieve the same (or close to the same) + decoding accuracy as TSD. + + For a given decoding accuracy, it is possible to attain faster decoding via ALSD than TSD. + + score_norm: bool, whether to normalize the scores of the log probabilities. + + return_best_hypothesis: bool, decides whether to return a single hypothesis (the best out of N), + or return all N hypothesis (sorted with best score first). The container class changes based + this flag - + When set to True (default), returns a single Hypothesis. + When set to False, returns a NBestHypotheses container, which contains a list of Hypothesis. + + # The following arguments are specific to the chosen `search_type` + + tsd_max_sym_exp_per_step: Used for `search_type=tsd`. The maximum symmetric expansions allowed + per timestep during beam search. Larger values should be used to attempt decoding of longer + sequences, but this in turn increases execution time and memory usage. + + alsd_max_target_len: Used for `search_type=alsd`. The maximum expected target sequence length + during beam search. Larger values allow decoding of longer sequences at the expense of + execution time and memory. + + # The following two flags are placeholders and unused until `nsc` implementation is stabilized. + + nsc_max_timesteps_expansion: Unused int. + + nsc_prefix_alpha: Unused int. + + preserve_alignments: Bool flag which preserves the history of alignments generated during + beam decoding (sample). When set to true, the Hypothesis will contain + the non-null value for `alignments` in it. Here, `alignments` is a List of List of ints. + + The length of the list corresponds to the Acoustic Length (T). + Each value in the list (Ti) is a torch.Tensor (U), representing 1 or more targets from a vocabulary. + U is the number of target tokens for the current timestep Ti. + + NOTE: `preserve_alignments` is an invalid argument for any `search_type` + other than basic beam search. + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "encoder_output": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + "encoded_lengths": NeuralType(tuple('B'), LengthsType()), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return {"predictions": [NeuralType(elements_type=HypothesisType())]} + + def __init__( + self, + decoder_model: rnnt_abstract.AbstractRNNTDecoder, + joint_model: rnnt_abstract.AbstractRNNTJoint, + beam_size: int, + search_type: str = 'default', + score_norm: bool = True, + return_best_hypothesis: bool = True, + tsd_max_sym_exp_per_step: Optional[int] = 50, + alsd_max_target_len: Union[int, float] = 1.0, + nsc_max_timesteps_expansion: int = 1, + nsc_prefix_alpha: int = 1, + preserve_alignments: bool = False, + ): + self.decoder = decoder_model + self.joint = joint_model + + self.blank = decoder_model.blank_idx + self.vocab_size = decoder_model.vocab_size + self.search_type = search_type + self.return_best_hypothesis = return_best_hypothesis + + if beam_size < 1: + raise ValueError("Beam search size cannot be less than 1!") + + self.beam_size = beam_size + self.score_norm = score_norm + + if self.beam_size == 1: + self.search_algorithm = self.greedy_search + elif search_type == "default": + self.search_algorithm = self.default_beam_search + elif search_type == "tsd": + self.search_algorithm = self.time_sync_decoding + elif search_type == "alsd": + self.search_algorithm = self.align_length_sync_decoding + elif search_type == "nsc": + raise NotImplementedError("`nsc` (Constrained Beam Search) has not been implemented.") + # self.search_algorithm = self.nsc_beam_search + else: + raise NotImplementedError( + f"The search type ({search_type}) supplied is not supported!\n" + f"Please use one of : (default, tsd, alsd, nsc)" + ) + + if tsd_max_sym_exp_per_step is None: + tsd_max_sym_exp_per_step = -1 + + if search_type in ['tsd', 'alsd', 'nsc'] and not self.decoder.blank_as_pad: + raise ValueError( + f"Search type was chosen as '{search_type}', however the decoder module provided " + f"does not support the `blank` token as a pad value. {search_type} requires " + f"the blank token as pad value support in order to perform batched beam search." + f"Please chose one of the other beam search methods, or re-train your model " + f"with this support." + ) + + self.tsd_max_symmetric_expansion_per_step = tsd_max_sym_exp_per_step + self.alsd_max_target_length = alsd_max_target_len + self.nsc_max_timesteps_expansion = nsc_max_timesteps_expansion + self.nsc_prefix_alpha = nsc_prefix_alpha + self.preserve_alignments = preserve_alignments + + @typecheck() + def __call__( + self, encoder_output: torch.Tensor, encoded_lengths: torch.Tensor + ) -> Union[Hypothesis, NBestHypotheses]: + """Perform general beam search. + + Args: + encoder_output: Encoded speech features (B, T_max, D_enc) + encoded_lengths: Lengths of the encoder outputs + + Returns: + Either a list containing a single Hypothesis (when `return_best_hypothesis=True`, + otherwise a list containing a single NBestHypotheses, which itself contains a list of + Hypothesis. This list is sorted such that the best hypothesis is the first element. + """ + # Preserve decoder and joint training state + decoder_training_state = self.decoder.training + joint_training_state = self.joint.training + + with torch.no_grad(): + # Apply optional preprocessing + encoder_output = encoder_output.transpose(1, 2) # (B, T, D) + + self.decoder.eval() + self.joint.eval() + + hypotheses = [] + with tqdm( + range(encoder_output.size(0)), + desc='Beam search progress:', + total=encoder_output.size(0), + unit='sample', + ) as idx_gen: + + # Freeze the decoder and joint to prevent recording of gradients + # during the beam loop. + with self.decoder.as_frozen(), self.joint.as_frozen(): + + # Decode every sample in the batch independently. + for batch_idx in idx_gen: + inseq = encoder_output[batch_idx : batch_idx + 1, :, :] # [1, T, D] + logitlen = encoded_lengths[batch_idx] + + # Execute the specific search strategy + nbest_hyps = self.search_algorithm(inseq, logitlen) # sorted list of hypothesis + + # Pack the result + if self.return_best_hypothesis: + best_hypothesis = nbest_hyps[0] # type: Hypothesis + else: + best_hypothesis = NBestHypotheses(nbest_hyps) # type: NBestHypotheses + hypotheses.append(best_hypothesis) + + self.decoder.train(decoder_training_state) + self.joint.train(joint_training_state) + + return (hypotheses,) + + def sort_nbest(self, hyps: List[Hypothesis]) -> List[Hypothesis]: + """Sort hypotheses by score or score given sequence length. + + Args: + hyps: list of hypotheses + + Return: + hyps: sorted list of hypotheses + """ + if self.score_norm: + return sorted(hyps, key=lambda x: x.score / len(x.y_sequence), reverse=True) + else: + return sorted(hyps, key=lambda x: x.score, reverse=True) + + def greedy_search(self, h: torch.Tensor, encoded_lengths: torch.Tensor) -> List[Hypothesis]: + """Greedy search implementation for transducer. + Generic case when beam size = 1. Results might differ slightly due to implementation details + as compared to `GreedyRNNTInfer` and `GreedyBatchRNNTInfer`. + + Args: + h: Encoded speech features (1, T_max, D_enc) + + Returns: + hyp: 1-best decoding results + """ + if self.preserve_alignments: + # Alignments is a 2-dimensional dangling list representing T x U + alignments = [[]] + else: + alignments = None + + # Initialize zero state vectors + dec_state = self.decoder.initialize_state(h) + + # Construct initial hypothesis + hyp = Hypothesis( + score=0.0, y_sequence=[self.blank], dec_state=dec_state, timestep=[-1], length=encoded_lengths + ) + cache = {} + + # Initialize state and first token + y, state, _ = self.decoder.score_hypothesis(hyp, cache) + + for i in range(int(encoded_lengths)): + hi = h[:, i : i + 1, :] # [1, 1, D] + + not_blank = True + symbols_added = 0 + + while not_blank: + ytu = torch.log_softmax(self.joint.joint(hi, y), dim=-1) # [1, 1, 1, V + 1] + ytu = ytu[0, 0, 0, :] # [V + 1] + + # max() requires float + if ytu.dtype != torch.float32: + ytu = ytu.float() + + logp, pred = torch.max(ytu, dim=-1) # [1, 1] + pred = pred.item() + + if self.preserve_alignments: + # insert logits into last timestep + alignments[-1].append(pred) + + if pred == self.blank: + not_blank = False + + if self.preserve_alignments: + # convert Ti-th logits into a torch array + alignments.append([]) # blank buffer for next timestep + else: + # Update state and current sequence + hyp.y_sequence.append(int(pred)) + hyp.score += float(logp) + hyp.dec_state = state + hyp.timestep.append(i) + + # Compute next state and token + y, state, _ = self.decoder.score_hypothesis(hyp, cache) + symbols_added += 1 + + # Remove trailing empty list of alignments + if self.preserve_alignments: + if len(alignments[-1]) == 0: + del alignments[-1] + + # attach alignments to hypothesis + hyp.alignments = alignments + + return [hyp] + + def default_beam_search(self, h: torch.Tensor, encoded_lengths: torch.Tensor) -> List[Hypothesis]: + """Beam search implementation. + + Args: + x: Encoded speech features (1, T_max, D_enc) + + Returns: + nbest_hyps: N-best decoding results + """ + # Initialize states + beam = min(self.beam_size, self.vocab_size) + beam_k = min(beam, (self.vocab_size - 1)) + blank_tensor = torch.tensor([self.blank], device=h.device, dtype=torch.long) + + # Precompute some constants for blank position + ids = list(range(self.vocab_size + 1)) + ids.remove(self.blank) + + # Used when blank token is first vs last token + if self.blank == 0: + index_incr = 1 + else: + index_incr = 0 + + # Initialize zero vector states + dec_state = self.decoder.initialize_state(h) + + # Initialize first hypothesis for the beam (blank) + kept_hyps = [Hypothesis(score=0.0, y_sequence=[self.blank], dec_state=dec_state, timestep=[-1], length=0)] + cache = {} + + if self.preserve_alignments: + kept_hyps[0].alignments = [[]] + + for i in range(int(encoded_lengths)): + hi = h[:, i : i + 1, :] # [1, 1, D] + hyps = kept_hyps + kept_hyps = [] + + while True: + max_hyp = max(hyps, key=lambda x: x.score) + hyps.remove(max_hyp) + + # update decoder state and get next score + y, state, lm_tokens = self.decoder.score_hypothesis(max_hyp, cache) # [1, 1, D] + + # get next token + ytu = torch.log_softmax(self.joint.joint(hi, y), dim=-1) # [1, 1, 1, V + 1] + ytu = ytu[0, 0, 0, :] # [V + 1] + + # remove blank token before top k + top_k = ytu[ids].topk(beam_k, dim=-1) + + # Two possible steps - blank token or non-blank token predicted + ytu = ( + torch.cat((top_k[0], ytu[self.blank].unsqueeze(0))), + torch.cat((top_k[1] + index_incr, blank_tensor)), + ) + + # for each possible step + for logp, k in zip(*ytu): + # construct hypothesis for step + new_hyp = Hypothesis( + score=(max_hyp.score + float(logp)), + y_sequence=max_hyp.y_sequence[:], + dec_state=max_hyp.dec_state, + lm_state=max_hyp.lm_state, + timestep=max_hyp.timestep[:], + length=encoded_lengths, + ) + + if self.preserve_alignments: + new_hyp.alignments = copy.deepcopy(max_hyp.alignments) + + # if current token is blank, dont update sequence, just store the current hypothesis + if k == self.blank: + kept_hyps.append(new_hyp) + else: + # if non-blank token was predicted, update state and sequence and then search more hypothesis + new_hyp.dec_state = state + new_hyp.y_sequence.append(int(k)) + new_hyp.timestep.append(i) + + hyps.append(new_hyp) + + if self.preserve_alignments: + if k == self.blank: + new_hyp.alignments[-1].append(self.blank) + else: + new_hyp.alignments[-1].append(new_hyp.y_sequence[-1]) + + # keep those hypothesis that have scores greater than next search generation + hyps_max = float(max(hyps, key=lambda x: x.score).score) + kept_most_prob = sorted([hyp for hyp in kept_hyps if hyp.score > hyps_max], key=lambda x: x.score,) + + # If enough hypothesis have scores greater than next search generation, + # stop beam search. + if len(kept_most_prob) >= beam: + if self.preserve_alignments: + # convert Ti-th logits into a torch array + for kept_h in kept_most_prob: + kept_h.alignments.append([]) # blank buffer for next timestep + + kept_hyps = kept_most_prob + break + + # Remove trailing empty list of alignments + if self.preserve_alignments: + for h in kept_hyps: + if len(h.alignments[-1]) == 0: + del h.alignments[-1] + + return self.sort_nbest(kept_hyps) + + def time_sync_decoding(self, h: torch.Tensor, encoded_lengths: torch.Tensor) -> List[Hypothesis]: + """Time synchronous beam search implementation. + Based on https://ieeexplore.ieee.org/document/9053040 + + Args: + h: Encoded speech features (1, T_max, D_enc) + + Returns: + nbest_hyps: N-best decoding results + """ + if self.preserve_alignments: + raise NotImplementedError("`preseve_alignments` is not implemented for Time-Synchronous Decoding.") + + # Precompute some constants for blank position + ids = list(range(self.vocab_size + 1)) + ids.remove(self.blank) + + # Used when blank token is first vs last token + if self.blank == 0: + index_incr = 1 + else: + index_incr = 0 + + # prepare the batched beam states + beam = min(self.beam_size, self.vocab_size) + beam_state = self.decoder.initialize_state( + torch.zeros(beam, device=h.device, dtype=h.dtype) + ) # [L, B, H], [L, B, H] (for LSTMs) + + # Initialize first hypothesis for the beam (blank) + B = [ + Hypothesis( + y_sequence=[self.blank], + score=0.0, + dec_state=self.decoder.batch_select_state(beam_state, 0), + timestep=[-1], + length=0, + ) + ] + cache = {} + + for i in range(int(encoded_lengths)): + hi = h[:, i : i + 1, :] + + # Update caches + A = [] + C = B + + h_enc = hi + + # For a limited number of symmetric expansions per timestep "i" + for v in range(self.tsd_max_symmetric_expansion_per_step): + D = [] + + # Decode a batch of beam states and scores + beam_y, beam_state, beam_lm_tokens = self.decoder.batch_score_hypothesis(C, cache, beam_state) + + # Extract the log probabilities and the predicted tokens + beam_logp = torch.log_softmax(self.joint.joint(h_enc, beam_y), dim=-1) # [B, 1, 1, V + 1] + beam_logp = beam_logp[:, 0, 0, :] # [B, V + 1] + beam_topk = beam_logp[:, ids].topk(beam, dim=-1) + + seq_A = [h.y_sequence for h in A] + + for j, hyp in enumerate(C): + # create a new hypothesis in A + if hyp.y_sequence not in seq_A: + # If the sequence is not in seq_A, add it as the blank token + # In this step, we dont add a token but simply update score + A.append( + Hypothesis( + score=(hyp.score + float(beam_logp[j, self.blank])), + y_sequence=hyp.y_sequence[:], + dec_state=hyp.dec_state, + lm_state=hyp.lm_state, + timestep=hyp.timestep[:], + length=encoded_lengths, + ) + ) + else: + # merge the existing blank hypothesis score with current score. + dict_pos = seq_A.index(hyp.y_sequence) + + A[dict_pos].score = np.logaddexp( + A[dict_pos].score, (hyp.score + float(beam_logp[j, self.blank])) + ) + + if v < self.tsd_max_symmetric_expansion_per_step: + for j, hyp in enumerate(C): + # for each current hypothesis j + # extract the top token score and top token id for the jth hypothesis + for logp, k in zip(beam_topk[0][j], beam_topk[1][j] + index_incr): + # create new hypothesis and store in D + # Note: This loop does *not* include the blank token! + new_hyp = Hypothesis( + score=(hyp.score + float(logp)), + y_sequence=(hyp.y_sequence + [int(k)]), + dec_state=self.decoder.batch_select_state(beam_state, j), + lm_state=hyp.lm_state, + timestep=hyp.timestep[:] + [i], + length=encoded_lengths, + ) + + D.append(new_hyp) + + # Prune beam + C = sorted(D, key=lambda x: x.score, reverse=True)[:beam] + + # Prune beam + B = sorted(A, key=lambda x: x.score, reverse=True)[:beam] + + return self.sort_nbest(B) + + def align_length_sync_decoding(self, h: torch.Tensor, encoded_lengths: torch.Tensor) -> List[Hypothesis]: + """Alignment-length synchronous beam search implementation. + Based on https://ieeexplore.ieee.org/document/9053040 + + Args: + h: Encoded speech features (1, T_max, D_enc) + + Returns: + nbest_hyps: N-best decoding results + """ + if self.preserve_alignments: + raise NotImplementedError( + "`preseve_alignments` is not implemented for Alignment-length Synchronous Decoding." + ) + + # Precompute some constants for blank position + ids = list(range(self.vocab_size + 1)) + ids.remove(self.blank) + + # Used when blank token is first vs last token + if self.blank == 0: + index_incr = 1 + else: + index_incr = 0 + + # prepare the batched beam states + beam = min(self.beam_size, self.vocab_size) + + h = h[0] # [T, D] + h_length = int(encoded_lengths) + beam_state = self.decoder.initialize_state( + torch.zeros(beam, device=h.device, dtype=h.dtype) + ) # [L, B, H], [L, B, H] for LSTMS + + # compute u_max as either a specific static limit, + # or a multiple of current `h_length` dynamically. + if type(self.alsd_max_target_length) == float: + u_max = int(self.alsd_max_target_length * h_length) + else: + u_max = int(self.alsd_max_target_length) + + # Initialize first hypothesis for the beam (blank) + B = [ + Hypothesis( + y_sequence=[self.blank], + score=0.0, + dec_state=self.decoder.batch_select_state(beam_state, 0), + timestep=[-1], + length=0, + ) + ] + + final = [] + cache = {} + + # ALSD runs for T + U_max steps + for i in range(h_length + u_max): + # Update caches + A = [] + B_ = [] + h_states = [] + + # preserve the list of batch indices which are added into the list + # and those which are removed from the list + # This is necessary to perform state updates in the correct batch indices later + batch_ids = list(range(len(B))) # initialize as a list of all batch ids + batch_removal_ids = [] # update with sample ids which are removed + + for bid, hyp in enumerate(B): + u = len(hyp.y_sequence) - 1 + t = i - u + 1 + + if t > (h_length - 1): + batch_removal_ids.append(bid) + continue + + B_.append(hyp) + h_states.append((t, h[t])) + + if B_: + # Compute the subset of batch ids which were *not* removed from the list above + sub_batch_ids = None + if len(B_) != beam: + sub_batch_ids = batch_ids + for id in batch_removal_ids: + # sub_batch_ids contains list of ids *that were not removed* + sub_batch_ids.remove(id) + + # extract the states of the sub batch only. + beam_state_ = [beam_state[state_id][:, sub_batch_ids, :] for state_id in range(len(beam_state))] + else: + # If entire batch was used (none were removed), simply take all the states + beam_state_ = beam_state + + # Decode a batch/sub-batch of beam states and scores + beam_y, beam_state_, beam_lm_tokens = self.decoder.batch_score_hypothesis(B_, cache, beam_state_) + + # If only a subset of batch ids were updated (some were removed) + if sub_batch_ids is not None: + # For each state in the RNN (2 for LSTM) + for state_id in range(len(beam_state)): + # Update the current batch states with the sub-batch states (in the correct indices) + # These indices are specified by sub_batch_ids, the ids of samples which were updated. + beam_state[state_id][:, sub_batch_ids, :] = beam_state_[state_id][...] + else: + # If entire batch was updated, simply update all the states + beam_state = beam_state_ + + # h_states = list of [t, h[t]] + # so h[1] here is a h[t] of shape [D] + # Simply stack all of the h[t] within the sub_batch/batch (T <= beam) + h_enc = torch.stack([h[1] for h in h_states]) # [T=beam, D] + h_enc = h_enc.unsqueeze(1) # [B=beam, T=1, D]; batch over the beams + + # Extract the log probabilities and the predicted tokens + beam_logp = torch.log_softmax(self.joint.joint(h_enc, beam_y), dim=-1) # [B=beam, 1, 1, V + 1] + beam_logp = beam_logp[:, 0, 0, :] # [B=beam, V + 1] + beam_topk = beam_logp[:, ids].topk(beam, dim=-1) + + for j, hyp in enumerate(B_): + # For all updated samples in the batch, add it as the blank token + # In this step, we dont add a token but simply update score + new_hyp = Hypothesis( + score=(hyp.score + float(beam_logp[j, self.blank])), + y_sequence=hyp.y_sequence[:], + dec_state=hyp.dec_state, + lm_state=hyp.lm_state, + timestep=hyp.timestep[:], + length=i, + ) + + # Add blank prediction to A + A.append(new_hyp) + + # If the prediction "timestep" t has reached the length of the input sequence + # we can add it to the "finished" hypothesis list. + if h_states[j][0] == (h_length - 1): + final.append(new_hyp) + + # Here, we carefully select the indices of the states that we want to preserve + # for the next token (non-blank) update. + if sub_batch_ids is not None: + h_states_idx = sub_batch_ids[j] + else: + h_states_idx = j + + # for each current hypothesis j + # extract the top token score and top token id for the jth hypothesis + for logp, k in zip(beam_topk[0][j], beam_topk[1][j] + index_incr): + # create new hypothesis and store in A + # Note: This loop does *not* include the blank token! + new_hyp = Hypothesis( + score=(hyp.score + float(logp)), + y_sequence=(hyp.y_sequence[:] + [int(k)]), + dec_state=self.decoder.batch_select_state(beam_state, h_states_idx), + lm_state=hyp.lm_state, + timestep=hyp.timestep[:] + [i], + length=i, + ) + + A.append(new_hyp) + + # Prune and recombine same hypothesis + # This may cause next beam to be smaller than max beam size + # Therefore larger beam sizes may be required for better decoding. + B = sorted(A, key=lambda x: x.score, reverse=True)[:beam] + B = self.recombine_hypotheses(B) + + # If B_ is empty list, then we may be able to early exit + elif len(batch_ids) == len(batch_removal_ids): + break + + if final: + return self.sort_nbest(final) + else: + return B + + def recombine_hypotheses(self, hypotheses: List[Hypothesis]) -> List[Hypothesis]: + """Recombine hypotheses with equivalent output sequence. + + Args: + hypotheses (list): list of hypotheses + + Returns: + final (list): list of recombined hypotheses + """ + final = [] + + for hyp in hypotheses: + seq_final = [f.y_sequence for f in final if f.y_sequence] + + if hyp.y_sequence in seq_final: + seq_pos = seq_final.index(hyp.y_sequence) + + final[seq_pos].score = np.logaddexp(final[seq_pos].score, hyp.score) + else: + final.append(hyp) + + return hypotheses + + +@dataclass +class BeamRNNTInferConfig: + beam_size: int + search_type: str = 'default' + score_norm: bool = True + return_best_hypothesis: bool = True + tsd_max_sym_exp_per_step: Optional[int] = 50 + alsd_max_target_len: float = 1.0 + nsc_max_timesteps_expansion: int = 1 + nsc_prefix_alpha: int = 1 + preserve_alignments: bool = False diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py new file mode 100644 index 0000000000..52124b9033 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py @@ -0,0 +1,731 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2017 Johns Hopkins University (Shinji Watanabe) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import List, Optional, Union + +import torch + +from nemo.collections.asr.modules import rnnt_abstract +from nemo.collections.asr.parts.utils import rnnt_utils +from nemo.collections.common.parts.rnn import label_collate +from nemo.core.classes import Typing, typecheck +from nemo.core.neural_types import AcousticEncodedRepresentation, HypothesisType, LengthsType, NeuralType + + +def pack_hypotheses( + hypotheses: List[List[int]], + timesteps: List[List[int]], + logitlen: torch.Tensor, + alignments: Optional[List[List[int]]] = None, +) -> List[rnnt_utils.Hypothesis]: + logitlen_cpu = logitlen.to("cpu") + return [ + rnnt_utils.Hypothesis( + y_sequence=torch.tensor(sent, dtype=torch.long), + score=-1.0, + timestep=timestep, + length=length, + alignments=alignments[idx] if alignments is not None else None, + ) + for idx, (sent, timestep, length) in enumerate(zip(hypotheses, timesteps, logitlen_cpu)) + ] + + +class _GreedyRNNTInfer(Typing): + """A greedy transducer decoder. + + Provides a common abstraction for sample level and batch level greedy decoding. + + Args: + decoder_model: rnnt_utils.AbstractRNNTDecoder implementation. + joint_model: rnnt_utils.AbstractRNNTJoint implementation. + blank_index: int index of the blank token. Can be 0 or len(vocabulary). + max_symbols_per_step: Optional int. The maximum number of symbols that can be added + to a sequence in a single time step; if set to None then there is + no limit. + preserve_alignments: Bool flag which preserves the history of alignments generated during + greedy decoding (sample / batched). When set to true, the Hypothesis will contain + the non-null value for `alignments` in it. Here, `alignments` is a List of List of ints. + + The length of the list corresponds to the Acoustic Length (T). + Each value in the list (Ti) is a torch.Tensor (U), representing 1 or more targets from a vocabulary. + U is the number of target tokens for the current timestep Ti. + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "encoder_output": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + "encoded_lengths": NeuralType(tuple('B'), LengthsType()), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return {"predictions": [NeuralType(elements_type=HypothesisType())]} + + def __init__( + self, + decoder_model: rnnt_abstract.AbstractRNNTDecoder, + joint_model: rnnt_abstract.AbstractRNNTJoint, + blank_index: int, + max_symbols_per_step: Optional[int] = None, + preserve_alignments: bool = False, + ): + super().__init__() + self.decoder = decoder_model + self.joint = joint_model + + self._blank_index = blank_index + self._SOS = blank_index # Start of single index + self.max_symbols = max_symbols_per_step + self.preserve_alignments = preserve_alignments + + def __call__(self, *args, **kwargs): + return self.forward(*args, **kwargs) + + @torch.no_grad() + def _pred_step( + self, + label: Union[torch.Tensor, int], + hidden: Optional[torch.Tensor], + add_sos: bool = False, + batch_size: Optional[int] = None, + ) -> (torch.Tensor, torch.Tensor): + """ + Common prediction step based on the AbstractRNNTDecoder implementation. + + Args: + label: (int/torch.Tensor): Label or "Start-of-Signal" token. + hidden: (Optional torch.Tensor): RNN State vector + add_sos (bool): Whether to add a zero vector at the begging as "start of sentence" token. + batch_size: Batch size of the output tensor. + + Returns: + g: (B, U, H) if add_sos is false, else (B, U + 1, H) + hid: (h, c) where h is the final sequence hidden state and c is + the final cell state: + h (tensor), shape (L, B, H) + c (tensor), shape (L, B, H) + """ + if isinstance(label, torch.Tensor): + # label: [batch, 1] + if label.dtype != torch.long: + label = label.long() + + else: + # Label is an integer + if label == self._SOS: + return self.decoder.predict(None, hidden, add_sos=add_sos, batch_size=batch_size) + + label = label_collate([[label]]) + + # output: [B, 1, K] + return self.decoder.predict(label, hidden, add_sos=add_sos, batch_size=batch_size) + + def _joint_step(self, enc, pred, log_normalize: Optional[bool] = None): + """ + Common joint step based on AbstractRNNTJoint implementation. + + Args: + enc: Output of the Encoder model. A torch.Tensor of shape [B, 1, H1] + pred: Output of the Decoder model. A torch.Tensor of shape [B, 1, H2] + log_normalize: Whether to log normalize or not. None will log normalize only for CPU. + + Returns: + logits of shape (B, T=1, U=1, V + 1) + """ + with torch.no_grad(): + logits = self.joint.joint(enc, pred) + + if log_normalize is None: + if not logits.is_cuda: # Use log softmax only if on CPU + logits = logits.log_softmax(dim=len(logits.shape) - 1) + else: + if log_normalize: + logits = logits.log_softmax(dim=len(logits.shape) - 1) + + return logits + + +class GreedyRNNTInfer(_GreedyRNNTInfer): + """A greedy transducer decoder. + + Sequence level greedy decoding, performed auto-repressively. + + Args: + decoder_model: rnnt_utils.AbstractRNNTDecoder implementation. + joint_model: rnnt_utils.AbstractRNNTJoint implementation. + blank_index: int index of the blank token. Can be 0 or len(vocabulary). + max_symbols_per_step: Optional int. The maximum number of symbols that can be added + to a sequence in a single time step; if set to None then there is + no limit. + preserve_alignments: Bool flag which preserves the history of alignments generated during + greedy decoding (sample / batched). When set to true, the Hypothesis will contain + the non-null value for `alignments` in it. Here, `alignments` is a List of List of ints. + + The length of the list corresponds to the Acoustic Length (T). + Each value in the list (Ti) is a torch.Tensor (U), representing 1 or more targets from a vocabulary. + U is the number of target tokens for the current timestep Ti. + """ + + def __init__( + self, + decoder_model: rnnt_abstract.AbstractRNNTDecoder, + joint_model: rnnt_abstract.AbstractRNNTJoint, + blank_index: int, + max_symbols_per_step: Optional[int] = None, + preserve_alignments: bool = False, + ): + super().__init__( + decoder_model=decoder_model, + joint_model=joint_model, + blank_index=blank_index, + max_symbols_per_step=max_symbols_per_step, + preserve_alignments=preserve_alignments, + ) + + @typecheck() + def forward(self, encoder_output: torch.Tensor, encoded_lengths: torch.Tensor): + """Returns a list of hypotheses given an input batch of the encoder hidden embedding. + Output token is generated auto-repressively. + + Args: + encoder_output: A tensor of size (batch, features, timesteps). + encoded_lengths: list of int representing the length of each sequence + output sequence. + + Returns: + packed list containing batch number of sentences (Hypotheses). + """ + # Preserve decoder and joint training state + decoder_training_state = self.decoder.training + joint_training_state = self.joint.training + + with torch.no_grad(): + # Apply optional preprocessing + encoder_output = encoder_output.transpose(1, 2) # (B, T, D) + + self.decoder.eval() + self.joint.eval() + + hypotheses = [] + timesteps = [] + alignments = [] if self.preserve_alignments else None + # Process each sequence independently + with self.decoder.as_frozen(), self.joint.as_frozen(): + for batch_idx in range(encoder_output.size(0)): + inseq = encoder_output[batch_idx, :, :].unsqueeze(1) # [T, 1, D] + logitlen = encoded_lengths[batch_idx] + sentence, timestep, alignment = self._greedy_decode(inseq, logitlen) + hypotheses.append(sentence) + timesteps.append(timestep) + + if self.preserve_alignments: + alignments.append(alignment) + + # Pack results into Hypotheses + packed_result = pack_hypotheses(hypotheses, timesteps, encoded_lengths, alignments=alignments) + + self.decoder.train(decoder_training_state) + self.joint.train(joint_training_state) + + return (packed_result,) + + @torch.no_grad() + def _greedy_decode(self, x: torch.Tensor, out_len: torch.Tensor): + # x: [T, 1, D] + # out_len: [seq_len] + + # Initialize blank state and empty label set + hidden = None + label = [] + timesteps = [] + + if self.preserve_alignments: + # Alignments is a 2-dimensional dangling list representing T x U + alignments = [[]] + else: + alignments = None + + # For timestep t in X_t + for time_idx in range(out_len): + # Extract encoder embedding at timestep t + # f = x[time_idx, :, :].unsqueeze(0) # [1, 1, D] + f = x.narrow(dim=0, start=time_idx, length=1) + + # Setup exit flags and counter + not_blank = True + symbols_added = 0 + + # While blank is not predicted, or we dont run out of max symbols per timestep + while not_blank and (self.max_symbols is None or symbols_added < self.max_symbols): + # In the first timestep, we initialize the network with RNNT Blank + # In later timesteps, we provide previous predicted label as input. + last_label = self._SOS if label == [] else label[-1] + + # Perform prediction network and joint network steps. + g, hidden_prime = self._pred_step(last_label, hidden) + logp = self._joint_step(f, g, log_normalize=None)[0, 0, 0, :] + + del g + + # torch.max(0) op doesnt exist for FP 16. + if logp.dtype != torch.float32: + logp = logp.float() + + # get index k, of max prob + v, k = logp.max(0) + k = k.item() # K is the label at timestep t_s in inner loop, s >= 0. + + if self.preserve_alignments: + # insert logits into last timestep + alignments[-1].append(k) + + del logp + + # If blank token is predicted, exit inner loop, move onto next timestep t + if k == self._blank_index: + not_blank = False + + if self.preserve_alignments: + # convert Ti-th logits into a torch array + alignments.append([]) # blank buffer for next timestep + else: + # Append token to label set, update RNN state. + label.append(k) + timesteps.append(time_idx) + hidden = hidden_prime + + # Increment token counter. + symbols_added += 1 + + # Remove trailing empty list of Alignments + if self.preserve_alignments: + if len(alignments[-1]) == 0: + del alignments[-1] + + return label, timesteps, alignments + + +class GreedyBatchedRNNTInfer(_GreedyRNNTInfer): + """A batch level greedy transducer decoder. + + Batch level greedy decoding, performed auto-repressively. + + Args: + decoder_model: rnnt_utils.AbstractRNNTDecoder implementation. + joint_model: rnnt_utils.AbstractRNNTJoint implementation. + blank_index: int index of the blank token. Can be 0 or len(vocabulary). + max_symbols_per_step: Optional int. The maximum number of symbols that can be added + to a sequence in a single time step; if set to None then there is + no limit. + preserve_alignments: Bool flag which preserves the history of alignments generated during + greedy decoding (sample / batched). When set to true, the Hypothesis will contain + the non-null value for `alignments` in it. Here, `alignments` is a List of List of ints. + + The length of the list corresponds to the Acoustic Length (T). + Each value in the list (Ti) is a torch.Tensor (U), representing 1 or more targets from a vocabulary. + U is the number of target tokens for the current timestep Ti. + """ + + def __init__( + self, + decoder_model: rnnt_abstract.AbstractRNNTDecoder, + joint_model: rnnt_abstract.AbstractRNNTJoint, + blank_index: int, + max_symbols_per_step: Optional[int] = None, + preserve_alignments: bool = False, + ): + super().__init__( + decoder_model=decoder_model, + joint_model=joint_model, + blank_index=blank_index, + max_symbols_per_step=max_symbols_per_step, + preserve_alignments=preserve_alignments, + ) + + # Depending on availability of `blank_as_pad` support + # switch between more efficient batch decoding technique + if self.decoder.blank_as_pad: + self._greedy_decode = self._greedy_decode_blank_as_pad + else: + self._greedy_decode = self._greedy_decode_masked + + @typecheck() + def forward(self, encoder_output: torch.Tensor, encoded_lengths: torch.Tensor): + """Returns a list of hypotheses given an input batch of the encoder hidden embedding. + Output token is generated auto-repressively. + + Args: + encoder_output: A tensor of size (batch, features, timesteps). + encoded_lengths: list of int representing the length of each sequence + output sequence. + + Returns: + packed list containing batch number of sentences (Hypotheses). + """ + # Preserve decoder and joint training state + decoder_training_state = self.decoder.training + joint_training_state = self.joint.training + + with torch.no_grad(): + # Apply optional preprocessing + encoder_output = encoder_output.transpose(1, 2) # (B, T, D) + logitlen = encoded_lengths + + self.decoder.eval() + self.joint.eval() + + with self.decoder.as_frozen(), self.joint.as_frozen(): + inseq = encoder_output # [B, T, D] + hypotheses, timesteps, alignments = self._greedy_decode(inseq, logitlen, device=inseq.device) + + # Pack the hypotheses results + packed_result = pack_hypotheses(hypotheses, timesteps, logitlen, alignments=alignments) + + del hypotheses, timesteps + + self.decoder.train(decoder_training_state) + self.joint.train(joint_training_state) + + return (packed_result,) + + def _greedy_decode_blank_as_pad(self, x: torch.Tensor, out_len: torch.Tensor, device: torch.device): + with torch.no_grad(): + # x: [B, T, D] + # out_len: [B] + # device: torch.device + + # Initialize state + hidden = None + batchsize = x.shape[0] + + # Output string buffer + label = [[] for _ in range(batchsize)] + timesteps = [[] for _ in range(batchsize)] + + # If alignments need to be preserved, register a danling list to hold the values + if self.preserve_alignments: + # alignments is a 3-dimensional dangling list representing B x T x U + alignments = [] + for _ in range(batchsize): + alignments.append([[]]) + else: + alignments = None + + # Last Label buffer + Last Label without blank buffer + # batch level equivalent of the last_label + last_label = torch.full([batchsize, 1], fill_value=self._blank_index, dtype=torch.long, device=device) + + # Mask buffers + blank_mask = torch.full([batchsize], fill_value=0, dtype=torch.bool, device=device) + + # Get max sequence length + max_out_len = out_len.max() + for time_idx in range(max_out_len): + f = x.narrow(dim=1, start=time_idx, length=1) # [B, 1, D] + + # Prepare t timestamp batch variables + not_blank = True + symbols_added = 0 + + # Reset blank mask + blank_mask.mul_(False) + + # Update blank mask with time mask + # Batch: [B, T, D], but Bi may have seq len < max(seq_lens_in_batch) + # Forcibly mask with "blank" tokens, for all sample where current time step T > seq_len + blank_mask = time_idx >= out_len + # Start inner loop + while not_blank and (self.max_symbols is None or symbols_added < self.max_symbols): + + # Batch prediction and joint network steps + # If very first prediction step, submit SOS tag (blank) to pred_step. + # This feeds a zero tensor as input to AbstractRNNTDecoder to prime the state + if time_idx == 0 and symbols_added == 0: + g, hidden_prime = self._pred_step(self._SOS, hidden, batch_size=batchsize) + else: + # Perform batch step prediction of decoder, getting new states and scores ("g") + g, hidden_prime = self._pred_step(last_label, hidden, batch_size=batchsize) + + # Batched joint step - Output = [B, V + 1] + logp = self._joint_step(f, g, log_normalize=None)[:, 0, 0, :] + + if logp.dtype != torch.float32: + logp = logp.float() + + # Get index k, of max prob for batch + v, k = logp.max(1) + del v, g + + # Update blank mask with current predicted blanks + # This is accumulating blanks over all time steps T and all target steps min(max_symbols, U) + k_is_blank = k == self._blank_index + blank_mask.bitwise_or_(k_is_blank) + + del k_is_blank + + # If preserving alignments, check if sequence length of sample has been reached + # before adding alignment + if self.preserve_alignments: + # Insert ids into last timestep per sample + logp_vals = logp.to('cpu').max(1)[1] + for batch_idx in range(batchsize): + if time_idx < out_len[batch_idx]: + alignments[batch_idx][-1].append(logp_vals[batch_idx]) + del logp_vals + del logp + + # If all samples predict / have predicted prior blanks, exit loop early + # This is equivalent to if single sample predicted k + if blank_mask.all(): + not_blank = False + + # If preserving alignments, convert the current Uj alignments into a torch.Tensor + # Then preserve U at current timestep Ti + # Finally, forward the timestep history to Ti+1 for that sample + # All of this should only be done iff the current time index <= sample-level AM length. + # Otherwise ignore and move to next sample / next timestep. + if self.preserve_alignments: + + # convert Ti-th logits into a torch array + for batch_idx in range(batchsize): + + # this checks if current timestep <= sample-level AM length + # If current timestep > sample-level AM length, no alignments will be added + # Therefore the list of Uj alignments is empty here. + if len(alignments[batch_idx][-1]) > 0: + alignments[batch_idx].append([]) # blank buffer for next timestep + else: + # Collect batch indices where blanks occurred now/past + blank_indices = [] + if hidden is not None: + blank_indices = (blank_mask == 1).nonzero(as_tuple=False) + + # Recover prior state for all samples which predicted blank now/past + if hidden is not None: + # LSTM has 2 states + for state_id in range(len(hidden)): + hidden_prime[state_id][:, blank_indices, :] = hidden[state_id][:, blank_indices, :] + + # Recover prior predicted label for all samples which predicted blank now/past + k[blank_indices] = last_label[blank_indices, 0] + + # Update new label and hidden state for next iteration + last_label = k.clone().view(-1, 1) + hidden = hidden_prime + + # Update predicted labels, accounting for time mask + # If blank was predicted even once, now or in the past, + # Force the current predicted label to also be blank + # This ensures that blanks propogate across all timesteps + # once they have occured (normally stopping condition of sample level loop). + for kidx, ki in enumerate(k): + if blank_mask[kidx] == 0: + label[kidx].append(ki) + timesteps[kidx].append(time_idx) + + symbols_added += 1 + + # Remove trailing empty list of alignments at T_{am-len} x Uj + if self.preserve_alignments: + for batch_idx in range(batchsize): + if len(alignments[batch_idx][-1]) == 0: + del alignments[batch_idx][-1] + + return label, timesteps, alignments + + @torch.no_grad() + def _greedy_decode_masked(self, x: torch.Tensor, out_len: torch.Tensor, device: torch.device): + # x: [B, T, D] + # out_len: [B] + # device: torch.device + + # Initialize state + hidden = None + batchsize = x.shape[0] + + # Output string buffer + label = [[] for _ in range(batchsize)] + timesteps = [[] for _ in range(batchsize)] + + # If alignments need to be preserved, register a danling list to hold the values + if self.preserve_alignments: + # alignments is a 3-dimensional dangling list representing B x T x U + alignments = [] + for _ in range(batchsize): + alignments.append([[]]) + else: + alignments = None + + # Last Label buffer + Last Label without blank buffer + # batch level equivalent of the last_label + last_label = torch.full([batchsize, 1], fill_value=self._blank_index, dtype=torch.long, device=device) + last_label_without_blank = last_label.clone() + + # Mask buffers + blank_mask = torch.full([batchsize], fill_value=0, dtype=torch.bool, device=device) + + # Get max sequence length + max_out_len = out_len.max() + for time_idx in range(max_out_len): + f = x.narrow(dim=1, start=time_idx, length=1) # [B, 1, D] + + # Prepare t timestamp batch variables + not_blank = True + symbols_added = 0 + + # Reset blank mask + blank_mask.mul_(False) + + # Update blank mask with time mask + # Batch: [B, T, D], but Bi may have seq len < max(seq_lens_in_batch) + # Forcibly mask with "blank" tokens, for all sample where current time step T > seq_len + blank_mask = time_idx >= out_len + + # Start inner loop + while not_blank and (self.max_symbols is None or symbols_added < self.max_symbols): + # Batch prediction and joint network steps + # If very first prediction step, submit SOS tag (blank) to pred_step. + # This feeds a zero tensor as input to AbstractRNNTDecoder to prime the state + if time_idx == 0 and symbols_added == 0: + g, hidden_prime = self._pred_step(self._SOS, hidden, batch_size=batchsize) + else: + # Set a dummy label for the blank value + # This value will be overwritten by "blank" again the last label update below + # This is done as vocabulary of prediction network does not contain "blank" token of RNNT + last_label_without_blank_mask = last_label == self._blank_index + last_label_without_blank[last_label_without_blank_mask] = 0 # temp change of label + last_label_without_blank[~last_label_without_blank_mask] = last_label[ + ~last_label_without_blank_mask + ] + + # Perform batch step prediction of decoder, getting new states and scores ("g") + g, hidden_prime = self._pred_step(last_label_without_blank, hidden, batch_size=batchsize) + + # Batched joint step - Output = [B, V + 1] + logp = self._joint_step(f, g, log_normalize=None)[:, 0, 0, :] + + if logp.dtype != torch.float32: + logp = logp.float() + + # Get index k, of max prob for batch + v, k = logp.max(1) + del v, g + + # Update blank mask with current predicted blanks + # This is accumulating blanks over all time steps T and all target steps min(max_symbols, U) + k_is_blank = k == self._blank_index + blank_mask.bitwise_or_(k_is_blank) + + # If preserving alignments, check if sequence length of sample has been reached + # before adding alignment + if self.preserve_alignments: + # Insert ids into last timestep per sample + logp_vals = logp.to('cpu').max(1)[1] + for batch_idx in range(batchsize): + if time_idx < out_len[batch_idx]: + alignments[batch_idx][-1].append(logp_vals[batch_idx]) + del logp_vals + del logp + + # If all samples predict / have predicted prior blanks, exit loop early + # This is equivalent to if single sample predicted k + if blank_mask.all(): + not_blank = False + + # If preserving alignments, convert the current Uj alignments into a torch.Tensor + # Then preserve U at current timestep Ti + # Finally, forward the timestep history to Ti+1 for that sample + # All of this should only be done iff the current time index <= sample-level AM length. + # Otherwise ignore and move to next sample / next timestep. + if self.preserve_alignments: + + # convert Ti-th logits into a torch array + for batch_idx in range(batchsize): + + # this checks if current timestep <= sample-level AM length + # If current timestep > sample-level AM length, no alignments will be added + # Therefore the list of Uj alignments is empty here. + if len(alignments[batch_idx][-1]) > 0: + alignments[batch_idx].append([]) # blank buffer for next timestep + else: + # Collect batch indices where blanks occurred now/past + blank_indices = [] + if hidden is not None: + blank_indices = (blank_mask == 1).nonzero(as_tuple=False) + + # Recover prior state for all samples which predicted blank now/past + if hidden is not None: + # LSTM has 2 states + for state_id in range(len(hidden)): + hidden_prime[state_id][:, blank_indices, :] = hidden[state_id][:, blank_indices, :] + + # Recover prior predicted label for all samples which predicted blank now/past + k[blank_indices] = last_label[blank_indices, 0] + + # Update new label and hidden state for next iteration + last_label = k.view(-1, 1) + hidden = hidden_prime + + # Update predicted labels, accounting for time mask + # If blank was predicted even once, now or in the past, + # Force the current predicted label to also be blank + # This ensures that blanks propogate across all timesteps + # once they have occured (normally stopping condition of sample level loop). + for kidx, ki in enumerate(k): + if blank_mask[kidx] == 0: + label[kidx].append(ki) + timesteps[kidx].append(time_idx) + + symbols_added += 1 + + # Remove trailing empty list of alignments at T_{am-len} x Uj + if self.preserve_alignments: + for batch_idx in range(batchsize): + if len(alignments[batch_idx][-1]) == 0: + del alignments[batch_idx][-1] + + return label, timesteps, alignments + + +@dataclass +class GreedyRNNTInferConfig: + max_symbols_per_step: Optional[int] = None + preserve_alignments: bool = False + + +@dataclass +class GreedyBatchedRNNTInferConfig: + max_symbols_per_step: Optional[int] = None + preserve_alignments: bool = False diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/spectr_augment.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/spectr_augment.py new file mode 100644 index 0000000000..ef9821e3a3 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/spectr_augment.py @@ -0,0 +1,152 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random + +import torch +import torch.nn as nn + +from nemo.core.classes import Typing, typecheck +from nemo.core.neural_types import NeuralType, SpectrogramType + + +class SpecAugment(nn.Module, Typing): + """ + Zeroes out(cuts) random continuous horisontal or + vertical segments of the spectrogram as described in + SpecAugment (https://arxiv.org/abs/1904.08779). + + params: + freq_masks - how many frequency segments should be cut + time_masks - how many time segments should be cut + freq_width - maximum number of frequencies to be cut in one segment + time_width - maximum number of time steps to be cut in one segment. + Can be a positive integer or a float value in the range [0, 1]. + If positive integer value, defines maximum number of time steps + to be cut in one segment. + If a float value, defines maximum percentage of timesteps that + are cut adaptively. + """ + + @property + def input_types(self): + """Returns definitions of module input types + """ + return {"input_spec": NeuralType(('B', 'D', 'T'), SpectrogramType())} + + @property + def output_types(self): + """Returns definitions of module output types + """ + return {"augmented_spec": NeuralType(('B', 'D', 'T'), SpectrogramType())} + + def __init__( + self, freq_masks=0, time_masks=0, freq_width=10, time_width=10, rng=None, mask_value=0.0, + ): + super(SpecAugment, self).__init__() + + self._rng = random.Random() if rng is None else rng + + self.freq_masks = freq_masks + self.time_masks = time_masks + + self.freq_width = freq_width + self.time_width = time_width + + self.mask_value = mask_value + + if isinstance(time_width, int): + self.adaptive_temporal_width = False + else: + if time_width > 1.0 or time_width < 0.0: + raise ValueError("If `time_width` is a float value, must be in range [0, 1]") + + self.adaptive_temporal_width = True + + @typecheck() + @torch.no_grad() + def forward(self, input_spec): + sh = input_spec.shape + + if self.adaptive_temporal_width: + time_width = max(1, int(sh[2] * self.time_width)) + else: + time_width = self.time_width + + for idx in range(sh[0]): + for i in range(self.freq_masks): + x_left = self._rng.randint(0, sh[1] - self.freq_width) + + w = self._rng.randint(0, self.freq_width) + + input_spec[idx, x_left : x_left + w, :] = self.mask_value + + for i in range(self.time_masks): + y_left = self._rng.randint(0, sh[2] - time_width) + + w = self._rng.randint(0, time_width) + + input_spec[idx, :, y_left : y_left + w] = self.mask_value + + return input_spec + + +class SpecCutout(nn.Module, Typing): + """ + Zeroes out(cuts) random rectangles in the spectrogram + as described in (https://arxiv.org/abs/1708.04552). + + params: + rect_masks - how many rectangular masks should be cut + rect_freq - maximum size of cut rectangles along the frequency dimension + rect_time - maximum size of cut rectangles along the time dimension + """ + + @property + def input_types(self): + """Returns definitions of module input types + """ + return {"input_spec": NeuralType(('B', 'D', 'T'), SpectrogramType())} + + @property + def output_types(self): + """Returns definitions of module output types + """ + return {"augmented_spec": NeuralType(('B', 'D', 'T'), SpectrogramType())} + + def __init__(self, rect_masks=0, rect_time=5, rect_freq=20, rng=None): + super(SpecCutout, self).__init__() + + self._rng = random.Random() if rng is None else rng + + self.rect_masks = rect_masks + self.rect_time = rect_time + self.rect_freq = rect_freq + + @typecheck() + @torch.no_grad() + def forward(self, input_spec): + sh = input_spec.shape + + for idx in range(sh[0]): + for i in range(self.rect_masks): + rect_x = self._rng.randint(0, sh[1] - self.rect_freq) + rect_y = self._rng.randint(0, sh[2] - self.rect_time) + + w_x = self._rng.randint(0, self.rect_freq) + w_y = self._rng.randint(0, self.rect_time) + + input_spec[idx, rect_x : rect_x + w_x, rect_y : rect_y + w_y] = 0.0 + + return input_spec diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/subsampling.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/subsampling.py new file mode 100644 index 0000000000..85f31611b8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/subsampling.py @@ -0,0 +1,138 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import torch +import torch.nn as nn + + +class ConvSubsampling(torch.nn.Module): + """Convolutional subsampling which supports VGGNet and striding approach introduced in: + VGGNet Subsampling: https://arxiv.org/pdf/1910.12977.pdf + Striding Subsampling: + "Speech-Transformer: A No-Recurrence Sequence-to-Sequence Model for Speech Recognition" by Linhao Dong et al. + Args: + subsampling (str): The subsampling technique from {"vggnet", "striding"} + subsampling_factor (int): The subsampling factor which should be a power of 2 + feat_in (int): size of the input features + feat_out (int): size of the output features + conv_channels (int): Number of channels for the convolution layers. + activation (Module): activation function, default is nn.ReLU() + """ + + def __init__(self, subsampling, subsampling_factor, feat_in, feat_out, conv_channels, activation=nn.ReLU()): + super(ConvSubsampling, self).__init__() + self._subsampling = subsampling + + if subsampling_factor % 2 != 0: + raise ValueError("Sampling factor should be a multiply of 2!") + self._sampling_num = int(math.log(subsampling_factor, 2)) + + in_channels = 1 + layers = [] + if subsampling == 'vggnet': + self._padding = 0 + self._stride = 2 + self._kernel_size = 2 + self._ceil_mode = True + + for i in range(self._sampling_num): + layers.append( + torch.nn.Conv2d( + in_channels=in_channels, out_channels=conv_channels, kernel_size=3, stride=1, padding=1 + ) + ) + layers.append(activation) + layers.append( + torch.nn.Conv2d( + in_channels=conv_channels, out_channels=conv_channels, kernel_size=3, stride=1, padding=1 + ) + ) + layers.append(activation) + layers.append( + torch.nn.MaxPool2d( + kernel_size=self._kernel_size, + stride=self._stride, + padding=self._padding, + ceil_mode=self._ceil_mode, + ) + ) + in_channels = conv_channels + elif subsampling == 'striding': + self._padding = 1 + self._stride = 2 + self._kernel_size = 3 + self._ceil_mode = False + + for i in range(self._sampling_num): + layers.append( + torch.nn.Conv2d( + in_channels=in_channels, + out_channels=conv_channels, + kernel_size=self._kernel_size, + stride=self._stride, + padding=self._padding, + ) + ) + layers.append(activation) + in_channels = conv_channels + else: + raise ValueError(f"Not valid sub-sampling: {subsampling}!") + + in_length = feat_in + for i in range(self._sampling_num): + out_length = calc_length( + length=int(in_length), + padding=self._padding, + kernel_size=self._kernel_size, + stride=self._stride, + ceil_mode=self._ceil_mode, + ) + in_length = out_length + + self.out = torch.nn.Linear(conv_channels * out_length, feat_out) + self.conv = torch.nn.Sequential(*layers) + + def forward(self, x, lengths): + x = x.unsqueeze(1) + x = self.conv(x) + b, c, t, f = x.size() + x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) + + # TODO: improve the performance of length calculation + new_lengths = lengths + for i in range(self._sampling_num): + new_lengths = [ + calc_length( + length=int(length), + padding=self._padding, + kernel_size=self._kernel_size, + stride=self._stride, + ceil_mode=self._ceil_mode, + ) + for length in new_lengths + ] + + new_lengths = torch.IntTensor(new_lengths).to(lengths.device) + return x, new_lengths + + +def calc_length(length, padding, kernel_size, stride, ceil_mode): + """ Calculates the output length of a Tensor passed through a convolution or max pooling layer""" + if ceil_mode: + length = math.ceil((length + (2 * padding) - (kernel_size - 1) - 1) / float(stride) + 1) + else: + length = math.floor((length + (2 * padding) - (kernel_size - 1) - 1) / float(stride) + 1) + return length diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/wav2vec.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/wav2vec.py new file mode 100644 index 0000000000..843001d602 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/submodules/wav2vec.py @@ -0,0 +1,222 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import math +from typing import List, Tuple + +import torch +from torch import nn + +from nemo.collections.asr.models.wav2vec.wav2vec_config import Wav2VecConvExtractorMode, Wav2VecTransformerConfig + + +class TransposeLast(torch.nn.Module): + """ + Transposes last dimension. Useful for adding to a sequential block. + """ + + def forward(self, x): + return x.transpose(-2, -1) + + +class SamePad(torch.nn.Module): + def __init__(self, kernel_size): + super().__init__() + self.remove = kernel_size % 2 == 0 + + def forward(self, x): + if self.remove: + x = x[:, :, :-1] + return x + + +class ConvFeatureEncoder(nn.Module): + """ + Converts input raw audio into features for downstream transformer model. + Uses 1D convolutional blocks with GeLU activation. + """ + + def __init__( + self, + conv_layers: List[Tuple[int, int, int]], + mode: Wav2VecConvExtractorMode = Wav2VecConvExtractorMode.default, + conv_bias: bool = False, + ): + super().__init__() + + def block( + n_in, n_out, k, stride, is_layer_norm=False, is_group_norm=False, conv_bias=False, + ): + def make_conv(): + conv = nn.Conv1d(n_in, n_out, k, stride=stride, bias=conv_bias) + nn.init.kaiming_normal_(conv.weight) + return conv + + assert (is_layer_norm and is_group_norm) is False, "layer norm and group norm are exclusive" + + if is_layer_norm: + return nn.Sequential( + make_conv(), + nn.Sequential(TransposeLast(), nn.LayerNorm(dim, elementwise_affine=True), TransposeLast()), + nn.GELU(), + ) + elif is_group_norm: + return nn.Sequential(make_conv(), nn.GroupNorm(dim, dim, affine=True), nn.GELU(),) + else: + return nn.Sequential(make_conv(), nn.GELU()) + + in_d = 1 + self.conv_layers = nn.ModuleList() + for i, cl in enumerate(conv_layers): + assert len(cl) == 3, "invalid conv definition: " + str(cl) + (dim, k, stride) = cl + + self.conv_layers.append( + block( + in_d, + dim, + k, + stride, + is_layer_norm=mode is Wav2VecConvExtractorMode.layer_norm, + is_group_norm=mode is Wav2VecConvExtractorMode.default and i == 0, + conv_bias=conv_bias, + ) + ) + in_d = dim + + def forward(self, x): + # BxT -> BxCxT + x = x.unsqueeze(1) + for conv in self.conv_layers: + x = conv(x) + return x + + +class Wav2VecTransformerEncoder(nn.Module): + def __init__(self, cfg: Wav2VecTransformerConfig): + super().__init__() + + conv_cfg = cfg.conv + + self.dropout = cfg.dropout + self.embedding_dim = cfg.encoder.embedding_dim + self.layer_norm_first = cfg.encoder.layer_norm_first + + # positional convolutional embeddings + self.pos_conv = nn.Conv1d( + self.embedding_dim, + self.embedding_dim, + kernel_size=conv_cfg.conv_pos, + padding=conv_cfg.conv_pos // 2, + groups=conv_cfg.conv_pos_groups, + ) + + self.feature_dropout = nn.Dropout(self.dropout) + + dropout = 0 + std = math.sqrt((4 * (1.0 - dropout)) / (conv_cfg.conv_pos * self.embedding_dim)) + nn.init.normal_(self.pos_conv.weight, mean=0, std=std) + nn.init.constant_(self.pos_conv.bias, 0) + + self.pos_conv = nn.utils.weight_norm(self.pos_conv, name="weight", dim=2) + self.pos_conv = nn.Sequential(self.pos_conv, SamePad(conv_cfg.conv_pos), nn.GELU()) + + encoder_cfg = cfg.encoder + self.transformer_encoder = nn.TransformerEncoder( + encoder_layer=nn.TransformerEncoderLayer( + d_model=self.embedding_dim, + nhead=encoder_cfg.num_attention_heads, + dim_feedforward=encoder_cfg.ffn_embedding_dim, + dropout=self.dropout, + activation=encoder_cfg.activation_fn.value, + ), + num_layers=encoder_cfg.encoder_layers, + ) + self.layer_norm = nn.LayerNorm(self.embedding_dim) + self.apply(init_bert_params) + + def forward(self, x, padding_mask=None): + x = self.extract_features(x, padding_mask) + + if self.layer_norm_first: + x = self.layer_norm(x) + + return x + + def extract_features(self, x, padding_mask=None): + + if padding_mask is not None: + x[padding_mask] = 0 + + x_conv = self.pos_conv(x.transpose(1, 2)) + x_conv = x_conv.transpose(1, 2) + x += x_conv + + if not self.layer_norm_first: + x = self.layer_norm(x) + + x = self.feature_dropout(x) + + # B x T x C -> T x B x C + x = x.transpose(0, 1) + + x = self.transformer_encoder(x, src_key_padding_mask=padding_mask) + + # T x B x C -> B x T x C + x = x.transpose(0, 1) + + return x + + +class GradMultiply(torch.autograd.Function): + @staticmethod + def forward(ctx, x, scale): + ctx.scale = scale + res = x.new(x) + return res + + @staticmethod + def backward(ctx, grad): + return grad * ctx.scale, None + + +def init_bert_params(module): + """ + Initialize the weights specific to the BERT Model. + This overrides the default initializations depending on the specified arguments. + 1. If normal_init_linear_weights is set then weights of linear + layer will be initialized using the normal distribution and + bias will be set to the specified value. + 2. If normal_init_embed_weights is set then weights of embedding + layer will be initialized using the normal distribution. + 3. If normal_init_proj_weights is set then weights of + in_project_weight for MultiHeadAttention initialized using + the normal distribution (to be validated). + """ + + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=0.02) + if module.bias is not None: + module.bias.data.zero_() + if isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=0.02) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + if isinstance(module, nn.TransformerEncoderLayer): + module.self_attn.in_proj_weight.data.normal_(mean=0.0, std=0.02) diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/__init__.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/activations.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/activations.py new file mode 100644 index 0000000000..55ebaca7e1 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/activations.py @@ -0,0 +1,24 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch.nn as nn + +__all__ = ['Swish'] + + +class Swish(nn.SiLU): + """ + Swish activation function introduced in 'https://arxiv.org/abs/1710.05941' + Mathematically identical to SiLU. See note in nn.SiLU for references. + """ diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/asr_module_utils.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/asr_module_utils.py new file mode 100644 index 0000000000..e077d7948c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/asr_module_utils.py @@ -0,0 +1,82 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from omegaconf import DictConfig, open_dict + +from nemo.collections.asr.modules import conv_asr +from nemo.collections.asr.parts.submodules import jasper +from nemo.utils import logging + + +def change_conv_asr_se_context_window(model: 'ASRModel', context_window: int, update_config: bool = True): + """ + Update the context window of the SqueezeExcitation module if the provided model contains an + `encoder` which is an instance of `ConvASREncoder`. + + Args: + model: A subclass of `ASRModel`, itself a subclass of `ModelPT`. + context_window: An integer representing the number of input timeframes that will be used + to compute the context. Each timeframe corresponds to a single window stride of the + STFT features. + + Say the window_stride = 0.01s, then a context window of 128 represents 128 * 0.01 s + of context to compute the Squeeze step. + update_config: Whether to update the config or not with the new context window. + """ + if update_config and not hasattr(model.cfg, 'encoder'): + logging.info( + "Could not change the context window in SqueezeExcite module " + "since the model provided does not contain an `encoder` module in its config." + ) + return + + if not isinstance(model.encoder, conv_asr.ConvASREncoder): + logging.info( + f"Could not change the context window in SqueezeExcite module " + f"since the `encoder` module is not an instance of `ConvASREncoder`.\n" + f"Provided encoder class = {model.encoder.__class__.__name__}" + ) + return + + enc_cfg = model.cfg.encoder if update_config else None + + if enc_cfg is not None: + with open_dict(enc_cfg): + _update_se_context_window(model, context_window, cfg=enc_cfg) + else: + _update_se_context_window(model, context_window) + + # Update model config + if update_config: + model.cfg.encoder = enc_cfg + + +def _update_se_context_window(model: 'ASRModel', context_window: int, cfg: Optional[DictConfig] = None): + jasper_block_counter = -1 + for name, m in model.named_modules(): + if type(m) == jasper.JasperBlock: + jasper_block_counter += 1 + + if type(m) == jasper.MaskedConv1d: + if m.conv.stride[0] > 1 and 'mconv' in name: + context_window = context_window // m.conv.stride[0] + + if type(m) == jasper.SqueezeExcite: + m.change_context_window(context_window=context_window) + + # update config + if cfg is not None: + cfg.jasper[jasper_block_counter].se_context_size = context_window diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/nmse_clustering.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/nmse_clustering.py new file mode 100644 index 0000000000..0f65931fc7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/nmse_clustering.py @@ -0,0 +1,271 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) 2007-2020 The scikit-learn developers. + +# BSD 3-Clause License + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This file is part of https://github.com/scikit-learn/scikit-learn/blob/114616d9f6ce9eba7c1aacd3d4a254f868010e25/sklearn/manifold/_spectral_embedding.py and +# https://github.com/tango4j/Auto-Tuning-Spectral-Clustering. + + +import numpy as np +import scipy +from scipy import sparse +from sklearn.cluster import SpectralClustering as sklearn_SpectralClustering +from sklearn.metrics.pairwise import cosine_similarity +from sklearn.preprocessing import MinMaxScaler + +scaler = MinMaxScaler(feature_range=(0, 1)) + + +def get_kneighbors_conn(X_dist, p_neighbors): + X_dist_out = np.zeros_like(X_dist) + for i, line in enumerate(X_dist): + sorted_idx = np.argsort(line) + sorted_idx = sorted_idx[::-1] + indices = sorted_idx[:p_neighbors] + X_dist_out[indices, i] = 1 + return X_dist_out + + +def get_X_conn_from_dist(X_dist_raw, p_neighbors): + X_r = get_kneighbors_conn(X_dist_raw, p_neighbors) + X_conn_from_dist = 0.5 * (X_r + X_r.T) + return X_conn_from_dist + + +def isFullyConnected(X_conn_from_dist): + """ Return whether the graph is connected (True) or Not (False). + Parameters + ---------- + graph : {array-like, sparse matrix} of shape (n_samples, n_samples) + Adjacency matrix of the graph, non-zero weight means an edge + between the nodes. + Returns + ------- + is_connected : bool + True means the graph is fully connected and False means not. + """ + gC = _graph_connected_component(X_conn_from_dist, 0).sum() == X_conn_from_dist.shape[0] + return gC + + +def gc_thres_min_gc(mat, max_n, n_list): + p_neighbors, index = 1, 0 + X_conn_from_dist = get_X_conn_from_dist(mat, p_neighbors) + fully_connected = isFullyConnected(X_conn_from_dist) + for i, p_neighbors in enumerate(n_list): + fully_connected = isFullyConnected(X_conn_from_dist) + X_conn_from_dist = get_X_conn_from_dist(mat, p_neighbors) + if fully_connected or p_neighbors > max_n: + break + + return X_conn_from_dist, p_neighbors + + +def _graph_connected_component(graph, node_id): + """Find the largest graph connected components that contains one + given node. + Parameters + ---------- + graph : array-like of shape (n_samples, n_samples) + Adjacency matrix of the graph, non-zero weight means an edge + between the nodes. + node_id : int + The index of the query node of the graph. + Returns + ------- + connected_components_matrix : array-like of shape (n_samples,) + An array of bool value indicating the indexes of the nodes + belonging to the largest connected components of the given query + node. + """ + n_node = graph.shape[0] + if sparse.issparse(graph): + graph = graph.tocsr() + connected_nodes = np.zeros(n_node, dtype=np.bool) + nodes_to_explore = np.zeros(n_node, dtype=np.bool) + nodes_to_explore[node_id] = True + for _ in range(n_node): + last_num_component = connected_nodes.sum() + np.logical_or(connected_nodes, nodes_to_explore, out=connected_nodes) + if last_num_component >= connected_nodes.sum(): + break + indices = np.where(nodes_to_explore)[0] + nodes_to_explore.fill(False) + for i in indices: + if sparse.issparse(graph): + neighbors = graph[i].toarray().ravel() + else: + neighbors = graph[i] + np.logical_or(nodes_to_explore, neighbors, out=nodes_to_explore) + return connected_nodes + + +def getLaplacian(X): + """ + returns laplacian matrix of X + """ + X[np.diag_indices(X.shape[0])] = 0 + A = X + D = np.sum(np.abs(A), axis=1) + D = np.diag(D) + L = D - A + return L + + +def eig_decompose(L, k): + try: + lambdas, eig_vecs = scipy.linalg.eigh(L) + except: + try: + lambdas = scipy.linalg.eigvals(L) + eig_vecs = None + except: + lambdas, eig_vecs = scipy.sparse.linalg.eigsh(L) ### Inaccurate results + return lambdas, eig_vecs + + +def getLamdaGaplist(lambdas): + lambda_gap_list = [] + for i in range(len(lambdas) - 1): + lambda_gap_list.append(float(lambdas[i + 1]) - float(lambdas[i])) + return lambda_gap_list + + +def estimate_num_of_spkrs(X_conn, SPK_MAX): + """ + Estimate number of speakers using eigen decompose on Laplacian Matrix + """ + L = getLaplacian(X_conn) + lambdas, eig_vals = eig_decompose(L, k=X_conn.shape[0]) + lambdas = np.sort(lambdas) + lambda_gap_list = getLamdaGaplist(lambdas) + num_of_spk = np.argmax(lambda_gap_list[: min(SPK_MAX, len(lambda_gap_list))]) + 1 + return num_of_spk, lambdas, lambda_gap_list + + +def NMEanalysis(mat, SPK_MAX, max_rp_threshold=0.250, sparse_search=True, search_p_volume=20, fixed_thres=None): + """ + Normalized Eigengap analysis to get p_neighbours and estimated num of speakers + Reference: https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=8937487&tag=1 + """ + eps = 1e-10 + eig_ratio_list = [] + if fixed_thres: + p_neighbors_list = [int(mat.shape[0] * fixed_thres)] + max_N = p_neighbors_list[0] + else: + max_N = int(mat.shape[0] * max_rp_threshold) + if sparse_search: + N = min(max_N, search_p_volume) + p_neighbors_list = list(np.linspace(1, max_N, N, endpoint=True).astype(int)) + else: + p_neighbors_list = list(range(1, max_N)) + + est_spk_n_dict = {} + for p_neighbors in p_neighbors_list: + X_conn_from_dist = get_X_conn_from_dist(mat, p_neighbors) + est_num_of_spk, lambdas, lambda_gap_list = estimate_num_of_spkrs(X_conn_from_dist, SPK_MAX) + est_spk_n_dict[p_neighbors] = (est_num_of_spk, lambdas) + arg_sorted_idx = np.argsort(lambda_gap_list[:SPK_MAX])[::-1] + max_key = arg_sorted_idx[0] + max_eig_gap = lambda_gap_list[max_key] / (max(lambdas) + eps) + eig_ratio_value = (p_neighbors / mat.shape[0]) / (max_eig_gap + eps) + eig_ratio_list.append(eig_ratio_value) + + index_nn = np.argmin(eig_ratio_list) + rp_p_neighbors = p_neighbors_list[index_nn] + X_conn_from_dist = get_X_conn_from_dist(mat, rp_p_neighbors) + if not isFullyConnected(X_conn_from_dist): + X_conn_from_dist, rp_p_neighbors = gc_thres_min_gc(mat, max_N, p_neighbors_list) + + return ( + X_conn_from_dist, + float(rp_p_neighbors / mat.shape[0]), + est_spk_n_dict[rp_p_neighbors][0], + est_spk_n_dict[rp_p_neighbors][1], + rp_p_neighbors, + ) + + +def get_eigen_matrix(emb): + """ + input + emb: embedding + + output + similarity matrix + """ + sim_d = cosine_similarity(emb) + scaler.fit(sim_d) + sim_d = scaler.transform(sim_d) + + return sim_d + + +def COSclustering(key, emb, oracle_num_speakers=None, max_num_speaker=8, MIN_SAMPLES=6): + """ + input: + key (str): speaker uniq name + emb (np array): speaker embedding + oracle_num_speaker (int or None): oracle number of speakers if known else None + max_num_speakers (int): maximum number of clusters to consider for each session + MIN_SAMPLES (int): minimum number of samples required for NME clustering, this avoids + zero p_neighbour_lists. Default of 6 is selected since (1/rp_threshold) >= 4. + output: + Y (List[int]): speaker labels + """ + est_num_spks_out_list = [] + mat = get_eigen_matrix(emb) + if oracle_num_speakers: + max_num_speaker = oracle_num_speakers + + X_conn_spkcount, rp_thres_spkcount, est_num_of_spk, lambdas, p_neigh = NMEanalysis(mat, max_num_speaker) + + if emb.shape[0] > MIN_SAMPLES: + X_conn_from_dist = get_X_conn_from_dist(mat, p_neigh) + else: + X_conn_from_dist = mat + + if oracle_num_speakers: + est_num_of_spk = oracle_num_speakers + + est_num_spks_out_list.append([key, str(est_num_of_spk)]) + + # Perform spectral clustering + spectral_model = sklearn_SpectralClustering( + affinity='precomputed', + eigen_solver='amg', + random_state=0, + n_jobs=-1, + n_clusters=est_num_of_spk, + eigen_tol=1e-10, + ) + + Y = spectral_model.fit_predict(X_conn_from_dist) + + return Y diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/numba_utils.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/numba_utils.py new file mode 100644 index 0000000000..8a5048aaf7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/numba_utils.py @@ -0,0 +1,88 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np +from numba import jit + + +def phase_vocoder(D: np.ndarray, rate: float, phi_advance: np.ndarray, scale_buffer: np.ndarray): + """ + Optimized implementation of phase vocoder from Librosa. + Reference implementation: + - https://librosa.github.io/librosa/generated/librosa.core.phase_vocoder.html + Args: + D: Complex spectograms of shape [d, t, complex=2]. + rate: Speed rate, must be float greater than 0. + phi_advance: Precomputed phase advance buffer array of length [n_fft + 1] + scale_buffer: Precomputed numpy buffer array of length [n_fft + 1] + Returns: + Complex64 ndarray of shape [d, t / rate, complex=2] + """ + time_steps = np.arange(0, D.shape[1], rate, dtype=np.float) + + # Create an empty output array + d_stretch = np.zeros((D.shape[0], len(time_steps)), D.dtype, order='F') + + # Phase accumulator; initialize to the first sample + phase_acc = np.angle(D[:, 0]) + + # Pad 0 columns to simplify boundary logic + D = np.pad(D, [(0, 0), (0, 2)], mode='constant') + + d_stretch = _phase_vocoder_kernel(D, time_steps, phi_advance, d_stretch, phase_acc, scale_buffer) + + return d_stretch + + +@jit(nopython=True, nogil=True) +def _phase_vocoder_kernel(D, time_steps, phi_advance, d_stretch, phase_acc, scale_buffer): + """ + Numba optimized kernel to compute the phase vocoder step. + Args: + D: Complex spectograms of shape [d, t, complex=2]. + rate: Speed rate, must be float greater than 0. + time_steps: Numpy ndarray of linearly spaced time steps, shape = [t] + phi_advance: Precomputed phase advance buffer array of length [n_fft + 1] + d_stretch: Output complex matrix of shape [d, t / rate, complex=2] + phase_acc: Phase accumulator initialized to first sample of shape [d, complex=2] + scale_buffer: Precomputed numpy buffer array of length [n_fft + 1] + Returns: + Complex64 ndarray of shape [d, t / rate, complex=2] + """ + two_pi = 2.0 * np.pi + + for (t, step) in enumerate(time_steps): + columns = D[:, int(step) : int(step + 2)] + columns_0 = columns[:, 0] + columns_1 = columns[:, 1] + + # Weighting for linear magnitude interpolation + alpha = np.mod(step, 1.0) + mag = (1.0 - alpha) * np.abs(columns_0) + alpha * np.abs(columns_1) + + # Store to output array + d_stretch[:, t] = mag * np.exp(1.0j * phase_acc) + + # Compute phase advance + dphase = np.angle(columns_1) - np.angle(columns_0) - phi_advance + + # Wrap to -pi:pi range + scale = dphase / two_pi + np.round(scale, 0, scale_buffer) + + dphase = dphase - two_pi * scale_buffer + + # Accumulate phase + phase_acc += phi_advance + dphase + + return d_stretch diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/rnnt_utils.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/rnnt_utils.py new file mode 100644 index 0000000000..7deba94a2e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/rnnt_utils.py @@ -0,0 +1,89 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2017 Johns Hopkins University (Shinji Watanabe) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Union + +import torch + + +@dataclass +class Hypothesis: + """Hypothesis class for beam search algorithms. + + score: A float score obtained from an AbstractRNNTDecoder module's score_hypothesis method. + + y_sequence: Either a sequence of integer ids pointing to some vocabulary, or a packed torch.Tensor + behaving in the same manner. dtype must be torch.Long in the latter case. + + dec_state: A list (or list of list) of LSTM-RNN decoder states. Can be None. + + text: (Optional) A decoded string after processing via CTC / RNN-T decoding (removing the CTC/RNNT + `blank` tokens, and optionally merging word-pieces). Should be used as decoded string for + Word Error Rate calculation. + + timestep: (Optional) A list of integer indices representing at which index in the decoding + process did the token appear. Should be of same length as the number of non-blank tokens. + + alignments: (Optional) Represents the CTC / RNNT token alignments as integer tokens along an axis of + time T (for CTC) or Time x Target (TxU). + For CTC, represented as a single list of integer indices. + For RNNT, represented as a dangling list of list of integer indices. + Outer list represents Time dimension (T), inner list represents Target dimension (U). + The set of valid indices **includes** the CTC / RNNT blank token in order to represent alignments. + + length: Represents the length of the sequence (the original length without padding), otherwise + defaults to 0. + + y: (Unused) A list of torch.Tensors representing the list of hypotheses. + + lm_state: (Unused) A dictionary state cache used by an external Language Model. + + lm_scores: (Unused) Score of the external Language Model. + + tokens: (Optional) A list of decoded tokens (can be characters or word-pieces. + """ + + score: float + y_sequence: Union[List[int], torch.Tensor] + dec_state: Optional[Union[List[List[torch.Tensor]], List[torch.Tensor]]] = None + text: Optional[str] = None + timestep: Union[List[int], torch.Tensor] = field(default_factory=list) + alignments: Optional[Union[List[int], List[List[int]]]] = None + length: int = 0 + y: List[torch.tensor] = None + lm_state: Union[Dict[str, Any], List[Any]] = None + lm_scores: torch.Tensor = None + tokens: Optional[Union[List[int], torch.Tensor]] = None + + +@dataclass +class NBestHypotheses: + """List of N best hypotheses""" + + n_best_hypotheses: Optional[List[Hypothesis]] diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/speaker_utils.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/speaker_utils.py new file mode 100644 index 0000000000..09a32b9b89 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/speaker_utils.py @@ -0,0 +1,400 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import math +import os +import pickle as pkl +from copy import deepcopy + +import numpy as np +from omegaconf import ListConfig +from pyannote.core import Annotation, Segment +from pyannote.metrics.diarization import DiarizationErrorRate +from tqdm import tqdm + +from nemo.collections.asr.parts.utils.nmse_clustering import COSclustering +from nemo.utils import logging + + +""" +This file contains all the utility functions required for speaker embeddings part in diarization scripts +""" + + +def audio_rttm_map(audio_file_list, rttm_file_list=None): + """ + Returns a AUDIO_TO_RTTM dictionary thats maps all the unique file names with + audio file paths and corresponding ground truth rttm files calculated from audio + file list and rttm file list + + Args: + audio_file_list(list,str): either list of audio file paths or file containing paths to audio files (required) + rttm_file_list(lisr,str): either list of rttm file paths or file containing paths to rttm files (optional) + [Required if DER needs to be calculated] + Returns: + AUDIO_RTTM_MAP (dict): dictionary thats maps all the unique file names with + audio file paths and corresponding ground truth rttm files + """ + rttm_notfound = False + if type(audio_file_list) in [list, ListConfig]: + audio_files = audio_file_list + else: + audio_pointer = open(audio_file_list, 'r') + audio_files = audio_pointer.read().splitlines() + + if rttm_file_list: + if type(rttm_file_list) in [list, ListConfig]: + rttm_files = rttm_file_list + else: + rttm_pointer = open(rttm_file_list, 'r') + rttm_files = rttm_pointer.read().splitlines() + else: + rttm_notfound = True + rttm_files = ['-'] * len(audio_files) + + assert len(audio_files) == len(rttm_files) + + AUDIO_RTTM_MAP = {} + rttm_dict = {} + audio_dict = {} + for audio_file, rttm_file in zip(audio_files, rttm_files): + uniq_audio_name = audio_file.split('/')[-1].rsplit('.', 1)[0] + uniq_rttm_name = rttm_file.split('/')[-1].rsplit('.', 1)[0] + + if rttm_notfound: + uniq_rttm_name = uniq_audio_name + + audio_dict[uniq_audio_name] = audio_file + rttm_dict[uniq_rttm_name] = rttm_file + + for key, value in audio_dict.items(): + + AUDIO_RTTM_MAP[key] = {'audio_path': audio_dict[key], 'rttm_path': rttm_dict[key]} + + assert len(rttm_dict.items()) == len(audio_dict.items()) + + return AUDIO_RTTM_MAP + + +def get_contiguous_stamps(stamps): + """ + Return contiguous time stamps + """ + lines = deepcopy(stamps) + contiguous_stamps = [] + for i in range(len(lines) - 1): + start, end, speaker = lines[i].split() + next_start, next_end, next_speaker = lines[i + 1].split() + if float(end) > float(next_start): + avg = str((float(next_start) + float(end)) / 2.0) + lines[i + 1] = ' '.join([avg, next_end, next_speaker]) + contiguous_stamps.append(start + " " + avg + " " + speaker) + else: + contiguous_stamps.append(start + " " + end + " " + speaker) + start, end, speaker = lines[-1].split() + contiguous_stamps.append(start + " " + end + " " + speaker) + return contiguous_stamps + + +def merge_stamps(lines): + """ + merge time stamps of same speaker + """ + stamps = deepcopy(lines) + overlap_stamps = [] + for i in range(len(stamps) - 1): + start, end, speaker = stamps[i].split() + next_start, next_end, next_speaker = stamps[i + 1].split() + if float(end) == float(next_start) and speaker == next_speaker: + stamps[i + 1] = ' '.join([start, next_end, next_speaker]) + else: + overlap_stamps.append(start + " " + end + " " + speaker) + + start, end, speaker = stamps[-1].split() + overlap_stamps.append(start + " " + end + " " + speaker) + + return overlap_stamps + + +def labels_to_pyannote_object(labels): + """ + converts labels to pyannote object to calculate DER and for visualization + """ + annotation = Annotation() + for label in labels: + start, end, speaker = label.strip().split() + start, end = float(start), float(end) + annotation[Segment(start, end)] = speaker + + return annotation + + +def labels_to_rttmfile(labels, uniq_id, out_rttm_dir): + """ + write rttm file with uniq_id name in out_rttm_dir with time_stamps in labels + """ + filename = os.path.join(out_rttm_dir, uniq_id + '.rttm') + with open(filename, 'w') as f: + for line in labels: + line = line.strip() + start, end, speaker = line.split() + duration = float(end) - float(start) + start = float(start) + log = 'SPEAKER {} 1 {:.3f} {:.3f} {} \n'.format(uniq_id, start, duration, speaker) + f.write(log) + + +def rttm_to_labels(rttm_filename): + """ + prepares time stamps label list from rttm file + """ + labels = [] + with open(rttm_filename, 'r') as f: + for line in f.readlines(): + rttm = line.strip().split() + start, end, speaker = float(rttm[3]), float(rttm[4]) + float(rttm[3]), rttm[7] + labels.append('{} {} {}'.format(start, end, speaker)) + return labels + + +def get_time_stamps(embeddings_file, reco2num, manifest_path, sample_rate, window, shift): + """ + Loads embedding file and generates time stamps based on window and shift for speaker embeddings + clustering. + + Args: + embeddings_file (str): Path to embeddings pickle file + reco2num (int,str): common integer number of speakers for every recording or path to + file that states unique_file id with number of speakers + manifest_path (str): path to manifest for time stamps matching + sample_rate (int): sample rate + window (float): window length in sec for speaker embeddings extraction + shift (float): shift length in sec for speaker embeddings extraction window shift + + Returns: + embeddings (dict): Embeddings with key as unique_id + time_stamps (dict): time stamps list for each audio recording + speakers (dict): number of speaker for each audio recording + """ + embeddings = pkl.load(open(embeddings_file, 'rb')) + all_uniq_files = list(embeddings.keys()) + num_files = len(all_uniq_files) + logging.info("Number of files to diarize: {}".format(num_files)) + sample = all_uniq_files[0] + logging.info("sample '{}' embeddings shape is {}\n".format(sample, embeddings[sample][0].shape)) + + speakers = {} + if isinstance(reco2num, int) or reco2num is None: + for key in embeddings.keys(): + speakers[key] = reco2num + elif isinstance(reco2num, str) and os.path.exists(reco2num): + for key in open(reco2num).readlines(): + key = key.strip() + wav_id, num = key.split() + speakers[wav_id] = int(num) + else: + raise TypeError("reco2num must be None, int or path to file containing number of speakers for each uniq id") + + with open(manifest_path, 'r') as manifest: + time_stamps = {} + for line in manifest.readlines(): + line = line.strip() + line = json.loads(line) + audio, offset, duration = line['audio_filepath'], line['offset'], line['duration'] + audio = os.path.basename(audio).rsplit('.', 1)[0] + if audio not in time_stamps: + time_stamps[audio] = [] + start = offset + slice_end = start + duration + base = math.ceil((duration - window) / shift) + slices = 1 if base < 0 else base + 1 + for slice_id in range(slices): + end = start + window + if end > slice_end: + end = slice_end + stamp = '{:.3f} {:.3f} '.format(start, end) + time_stamps[audio].append(stamp) + start = offset + (slice_id + 1) * shift + + return embeddings, time_stamps, speakers + + +def perform_clustering(embeddings, time_stamps, speakers, audio_rttm_map, out_rttm_dir, max_num_speakers=8): + """ + performs spectral clustering on embeddings with time stamps generated from VAD output + Args: + + embeddings (dict): Embeddings with key as unique_id + time_stamps (dict): time stamps list for each audio recording + speakers (dict): number of speaker for each audio recording + audio_rttm_map (dict): AUDIO_RTTM_MAP for mapping unique id with audio file path and rttm path + out_rttm_dir (str): Path to write predicted rttms + max_num_speakers (int): maximum number of speakers to consider for spectral clustering. Will be ignored if speakers['key'] is not None + + Returns: + all_reference (list[Annotation]): reference annotations for score calculation + all_hypothesis (list[Annotation]): hypothesis annotations for score calculation + + """ + all_hypothesis = [] + all_reference = [] + no_references = False + + for uniq_key in tqdm(embeddings.keys()): + NUM_speakers = speakers[uniq_key] + emb = embeddings[uniq_key] + emb = np.asarray(emb) + + cluster_labels = COSclustering( + uniq_key, emb, oracle_num_speakers=NUM_speakers, max_num_speaker=max_num_speakers + ) + + lines = time_stamps[uniq_key] + assert len(cluster_labels) == len(lines) + for idx, label in enumerate(cluster_labels): + tag = 'speaker_' + str(label) + lines[idx] += tag + + a = get_contiguous_stamps(lines) + labels = merge_stamps(a) + if out_rttm_dir: + labels_to_rttmfile(labels, uniq_key, out_rttm_dir) + hypothesis = labels_to_pyannote_object(labels) + all_hypothesis.append(hypothesis) + + rttm_file = audio_rttm_map[uniq_key]['rttm_path'] + if os.path.exists(rttm_file) and not no_references: + ref_labels = rttm_to_labels(rttm_file) + reference = labels_to_pyannote_object(ref_labels) + all_reference.append(reference) + else: + no_references = True + all_reference = [] + + return all_reference, all_hypothesis + + +def get_DER(all_reference, all_hypothesis): + """ + calculates DER, CER, FA and MISS + + Args: + all_reference (list[Annotation]): reference annotations for score calculation + all_hypothesis (list[Annotation]): hypothesis annotations for score calculation + + Returns: + DER (float): Diarization Error Rate + CER (float): Confusion Error Rate + FA (float): False Alarm + Miss (float): Miss Detection + + """ + metric = DiarizationErrorRate(collar=0.25, skip_overlap=True) + DER = 0 + + for reference, hypothesis in zip(all_reference, all_hypothesis): + metric(reference, hypothesis, detailed=True) + + DER = abs(metric) + CER = metric['confusion'] / metric['total'] + FA = metric['false alarm'] / metric['total'] + MISS = metric['missed detection'] / metric['total'] + + metric.reset() + + return DER, CER, FA, MISS + + +def perform_diarization( + embeddings_file=None, + reco2num=2, + manifest_path=None, + sample_rate=16000, + window=1.5, + shift=0.75, + audio_rttm_map=None, + out_rttm_dir=None, + max_num_speakers=8, +): + """ + Performs diarization with embeddings generated based on VAD time stamps with recording 2 num of speakers (reco2num) + for spectral clustering + """ + embeddings, time_stamps, speakers = get_time_stamps( + embeddings_file, reco2num, manifest_path, sample_rate, window, shift + ) + logging.info("Performing Clustering") + all_reference, all_hypothesis = perform_clustering( + embeddings, time_stamps, speakers, audio_rttm_map, out_rttm_dir, max_num_speakers + ) + + if len(all_reference) and len(all_hypothesis): + DER, CER, FA, MISS = get_DER(all_reference, all_hypothesis) + logging.info( + "Cumulative results of all the files: \n FA: {:.3f}\t MISS {:.3f}\t \ + Diarization ER: {:.3f}\t, Confusion ER:{:.3f}".format( + FA, MISS, DER, CER + ) + ) + else: + logging.warning("Please check if each ground truth RTTMs was present in provided path2groundtruth_rttm_files") + logging.warning("Skipping calculation of Diariazation Error Rate") + + +def write_rttm2manifest(paths2audio_files, paths2rttm_files, manifest_file): + """ + writes manifest file based on rttm files (or vad table out files). This manifest file would be used by + speaker diarizer to compute embeddings and cluster them. This function also takes care of overlap time stamps + + Args: + audio_file_list(list,str): either list of audio file paths or file containing paths to audio files (required) + rttm_file_list(lisr,str): either list of rttm file paths or file containing paths to rttm files (optional) + manifest (str): path to write manifest file + + Returns: + manifest (str): path to write manifest file + """ + AUDIO_RTTM_MAP = audio_rttm_map(paths2audio_files, paths2rttm_files) + + with open(manifest_file, 'w') as outfile: + for key in AUDIO_RTTM_MAP: + f = open(AUDIO_RTTM_MAP[key]['rttm_path'], 'r') + audio_path = AUDIO_RTTM_MAP[key]['audio_path'] + lines = f.readlines() + time_tup = (-1, -1) + for line in lines: + vad_out = line.strip().split() + if len(vad_out) > 3: + start, dur, activity = float(vad_out[3]), float(vad_out[4]), vad_out[7] + else: + start, dur, activity = float(vad_out[0]), float(vad_out[1]), vad_out[2] + start, dur = float("{:.3f}".format(start)), float("{:.3f}".format(dur)) + + if time_tup[0] >= 0 and start > time_tup[1]: + dur2 = float("{:.3f}".format(time_tup[1] - time_tup[0])) + meta = {"audio_filepath": audio_path, "offset": time_tup[0], "duration": dur2, "label": 'UNK'} + json.dump(meta, outfile) + outfile.write("\n") + time_tup = (start, start + dur) + else: + if time_tup[0] == -1: + time_tup = (start, start + dur) + else: + time_tup = (min(time_tup[0], start), max(time_tup[1], start + dur)) + dur2 = float("{:.3f}".format(time_tup[1] - time_tup[0])) + meta = {"audio_filepath": audio_path, "offset": time_tup[0], "duration": dur2, "label": 'UNK'} + json.dump(meta, outfile) + outfile.write("\n") + f.close() + return manifest_file diff --git a/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/vad_utils.py b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/vad_utils.py new file mode 100644 index 0000000000..5675a1e0a7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/asr/parts/utils/vad_utils.py @@ -0,0 +1,544 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import glob +import json +import os +from itertools import repeat +from multiprocessing import Pool + +import librosa +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from pyannote.core import Annotation, Segment +from pyannote.metrics import detection + +from nemo.utils import logging + + +""" +This file contains all the utility functions required for speaker embeddings part in diarization scripts +""" + + +def prepare_manifest(config): + """ + Perform VAD on long audio snippet might cause CUDA out of memory issue. + Automatically split manifest entry by split_duration to avoid the potential memory issue. + """ + manifest_vad_input = config.get('manifest_vad_input', "manifest_vad_input.json") + input_audios = [] + with open(config['manifest_filepath'], 'r') as manifest: + for line in manifest.readlines(): + input_audios.append(json.loads(line.strip())) + + p = Pool(processes=config['num_workers']) + args_func = { + 'label': 'infer', + 'split_duration': config['split_duration'], + 'time_length': config['time_length'], + } + results = p.starmap(write_vad_infer_manifest, zip(input_audios, repeat(args_func))) + p.close() + + if os.path.exists(manifest_vad_input): + logging.info("The prepared manifest file exists. Overwriting!") + os.remove(manifest_vad_input) + + with open(manifest_vad_input, 'a') as fout: + for res in results: + for r in res: + json.dump(r, fout) + fout.write('\n') + fout.flush() + + return manifest_vad_input + + +def write_vad_infer_manifest(file, args_func): + """ + Used by prepare_manifest. + Given a list of files, write them to manifest for dataloader with restrictions. + Args: + files (json) : file to be processed + label (str): label for audio snippet. + split_duration (float): max duration of each audio clip (each line in json) + time_length (float) : length of window for generating the frame. Used for taking care of joint. + Returns: + res (list) : list of generated metadata line of json for file + """ + res = [] + label = args_func['label'] + split_duration = args_func['split_duration'] + time_length = args_func['time_length'] + + filepath = file['audio_filepath'] + in_duration = file['duration'] + in_offset = file['offset'] + + try: + sr = 16000 + x, _sr = librosa.load(filepath, sr=sr, offset=in_offset, duration=in_duration) + duration = librosa.get_duration(x, sr=sr) + left = duration + current_offset = in_offset + + status = 'single' + while left > 0: + if left <= split_duration: + status = 'end' + write_duration = left + time_length + current_offset -= time_length + offset_inc = left + left = 0 + else: + if status == 'start' or status == 'next': + status = 'next' + else: + status = 'start' + + if status == 'start': + write_duration = split_duration + offset_inc = split_duration + else: + write_duration = split_duration + time_length + current_offset -= time_length + offset_inc = split_duration + time_length + + left -= split_duration + + metadata = { + 'audio_filepath': filepath, + 'duration': write_duration, + 'label': label, + 'text': '_', + 'offset': current_offset, + } + res.append(metadata) + + current_offset += offset_inc + + except Exception as e: + err_file = "error.log" + with open(err_file, 'w') as fout: + fout.write(file + ":" + str(e)) + return res + + +def get_vad_stream_status(data): + """ + Generate a list of status for each snippet in manifest. A snippet should be in single, start, next or end status. + Used for concatenate to full audio file. + Args: + data (list): list of filepath of audio snippet + Returns: + status (list): list of status of each snippet. + """ + if len(data) == 1: + return ['single'] + + status = [None] * len(data) + for i in range(len(data)): + if i == 0: + status[i] = 'start' if data[i] == data[i + 1] else 'single' + elif i == len(data) - 1: + status[i] = 'end' if data[i] == data[i - 1] else 'single' + else: + if data[i] != data[i - 1] and data[i] == data[i + 1]: + status[i] = 'start' + elif data[i] == data[i - 1] and data[i] == data[i + 1]: + status[i] = 'next' + elif data[i] == data[i - 1] and data[i] != data[i + 1]: + status[i] = 'end' + else: + status[i] = 'single' + return status + + +def generate_overlap_vad_seq(frame_pred_dir, smoothing_method, overlap, seg_len, shift_len, num_workers): + """ + Gnerate predictions with overlapping input windows/segments. Then a smoothing filter is applied to decide the label for a frame spanned by multiple windows. + Two common smoothing filters are supported: majority vote (median) and average (mean). + Args: + frame_pred_dir (str): Directory of frame prediction file to be processed. + smoothing_method (str): median or mean smoothing filter. + overlap (float): amounts of overlap of adjacent windows. + seg_len (float): length of window for generating the frame. + shift_len (float): amount of shift of window for generating the frame. + out_dir (str): directory of generated predictions. + num_workers(float): number of process for multiprocessing + Returns: + overlap_out_dir(str): directory of generate predictions. + """ + + p = Pool(processes=num_workers) + frame_filepathlist = glob.glob(frame_pred_dir + "/*.frame") + + overlap_out_dir = frame_pred_dir + "/overlap_smoothing_output" + "_" + smoothing_method + "_" + str(overlap) + + if not os.path.exists(overlap_out_dir): + os.mkdir(overlap_out_dir) + + per_args = { + "out_dir": overlap_out_dir, + "smoothing_method": smoothing_method, + "overlap": overlap, + "seg_len": seg_len, + "shift_len": shift_len, + } + p.starmap(generate_overlap_vad_seq_per_file, zip(frame_filepathlist, repeat(per_args))) + p.close() + p.join() + + return overlap_out_dir + + +def generate_overlap_vad_seq_per_file(frame_filepath, per_args): + """ + Use generated frame prediction (generated by shifting window of shift_len (10ms)) to generate prediction with overlapping input window/segments + See discription in generate_overlap_vad_seq. + """ + try: + smoothing_method = per_args['smoothing_method'] + overlap = per_args['overlap'] + seg_len = per_args['seg_len'] + shift_len = per_args['shift_len'] + out_dir = per_args['out_dir'] + + frame = np.loadtxt(frame_filepath) + name = os.path.basename(frame_filepath).split(".frame")[0] + "." + smoothing_method + overlap_filepath = os.path.join(out_dir, name) + + shift = int(shift_len / 0.01) # number of units of shift + seg = int((seg_len / 0.01 + 1)) # number of units of each window/segment + + jump_on_target = int(seg * (1 - overlap)) # jump on target generated sequence + jump_on_frame = int(jump_on_target / shift) # jump on input frame sequence + + if jump_on_frame < 1: + raise ValueError( + f"Note we jump over frame sequence to generate overlapping input segments. \n \ + Your input makes jump_on_fram={jump_on_frame} < 1 which is invalid because it cannot jump and will stuck.\n \ + Please try different seg_len, shift_len and overlap choices. \n \ + jump_on_target = int(seg * (1 - overlap)) \n \ + jump_on_frame = int(jump_on_frame/shift) " + ) + + target_len = int(len(frame) * shift) + + if smoothing_method == 'mean': + preds = np.zeros(target_len) + pred_count = np.zeros(target_len) + + for i, og_pred in enumerate(frame): + if i % jump_on_frame != 0: + continue + start = i * shift + end = start + seg + preds[start:end] = preds[start:end] + og_pred + pred_count[start:end] = pred_count[start:end] + 1 + + preds = preds / pred_count + last_non_zero_pred = preds[pred_count != 0][-1] + preds[pred_count == 0] = last_non_zero_pred + + elif smoothing_method == 'median': + preds = [[] for _ in range(target_len)] + for i, og_pred in enumerate(frame): + if i % jump_on_frame != 0: + continue + + start = i * shift + end = start + seg + for j in range(start, end): + if j <= target_len - 1: + preds[j].append(og_pred) + + preds = np.array([np.median(l) for l in preds]) + nan_idx = np.isnan(preds) + last_non_nan_pred = preds[~nan_idx][-1] + preds[nan_idx] = last_non_nan_pred + + else: + raise ValueError("smoothing_method should be either mean or median") + + round_final = np.round(preds, 4) + np.savetxt(overlap_filepath, round_final, delimiter='\n') + + except Exception as e: + raise (e) + + +def generate_vad_segment_table(vad_pred_dir, threshold, shift_len, num_workers): + """ + Convert frame level prediction to speech segment in start and end times format. + And save to csv file in rttm-like format + 0, 10, speech + 17,18, speech + Args: + vad_pred_dir (str): directory of prediction files to be processed. + threshold (float): threshold for prediction score (from 0 to 1). + shift_len (float): amount of shift of window for generating the frame. + out_dir (str): output dir of generated table/csv file. + num_workers(float): number of process for multiprocessing + Returns: + table_out_dir(str): directory of generate table. + """ + + p = Pool(processes=num_workers) + suffixes = ("frame", "mean", "median") + vad_pred_filepath_list = [os.path.join(vad_pred_dir, x) for x in os.listdir(vad_pred_dir) if x.endswith(suffixes)] + + table_out_dir = os.path.join(vad_pred_dir, "table_output_" + str(threshold)) + if not os.path.exists(table_out_dir): + os.mkdir(table_out_dir) + + per_args = { + "threshold": threshold, + "shift_len": shift_len, + "out_dir": table_out_dir, + } + + p.starmap(generate_vad_segment_table_per_file, zip(vad_pred_filepath_list, repeat(per_args))) + p.close() + p.join() + + return table_out_dir + + +def generate_vad_segment_table_per_file(pred_filepath, per_args): + """ + See discription in generate_overlap_vad_seq. + """ + threshold = per_args['threshold'] + shift_len = per_args['shift_len'] + out_dir = per_args['out_dir'] + + name = pred_filepath.split("/")[-1].rsplit(".", 1)[0] + + sequence = np.loadtxt(pred_filepath) + start = 0 + start_list = [0] + dur_list = [] + state_list = [] + current_state = "non-speech" + for i in range(len(sequence) - 1): + current_state = "non-speech" if sequence[i] <= threshold else "speech" + next_state = "non-speech" if sequence[i + 1] <= threshold else "speech" + if next_state != current_state: + dur = i * shift_len + shift_len - start # shift_len for handling joint + state_list.append(current_state) + dur_list.append(dur) + + start = (i + 1) * shift_len + start_list.append(start) + + dur_list.append((i + 1) * shift_len + shift_len - start) + state_list.append(current_state) + + seg_table = pd.DataFrame({'start': start_list, 'dur': dur_list, 'vad': state_list}) + seg_speech_table = seg_table[seg_table['vad'] == 'speech'] + + save_name = name + ".txt" + save_path = os.path.join(out_dir, save_name) + seg_speech_table.to_csv(save_path, sep='\t', index=False, header=False) + return save_path + + +def vad_construct_pyannote_object_per_file(vad_table_filepath, groundtruth_RTTM_file): + """ + Construct pyannote object for evaluation. + Args: + vad_table_filepath(str) : path of vad rttm-like table. + groundtruth_RTTM_file(str): path of groundtruth rttm file. + Returns: + reference(pyannote.Annotation): groundtruth + hypothesise(pyannote.Annotation): prediction + """ + + pred = pd.read_csv(vad_table_filepath, sep="\t", header=None) + label = pd.read_csv(groundtruth_RTTM_file, sep=" ", delimiter=None, header=None) + label = label.rename(columns={3: "start", 4: "dur", 7: "speaker"}) + + # construct reference + reference = Annotation() + for index, row in label.iterrows(): + reference[Segment(row['start'], row['start'] + row['dur'])] = row['speaker'] + + # construct hypothsis + hypothesis = Annotation() + for index, row in pred.iterrows(): + hypothesis[Segment(row[0], row[0] + row[1])] = 'Speech' + return reference, hypothesis + + +def vad_tune_threshold_on_dev(thresholds, vad_pred, groundtruth_RTTM, vad_pred_method="frame", focus_metric="DetER"): + """ + Tune threshold on dev set. Return best threshold which gives the lowest detection error rate (DetER) in thresholds. + Args: + thresholds (list): list of thresholds. + vad_pred_method (str): suffix of prediction file. Use to locate file. Should be either in "frame", "mean" or "median". + vad_pred_dir (str): directory of vad predictions or a file contains the paths of them + groundtruth_RTTM_dir (str): directory of groundtruch rttm files or a file contains the paths of them. + focus_metric (str): metrics we care most when tuning threshold. Should be either in "DetER", "FA", "MISS" + Returns: + best_threhsold (float): threshold that gives lowest DetER. + """ + threshold_perf = {} + best_threhsold = thresholds[0] + min_score = 100 + + try: + thresholds[0] >= 0 and thresholds[-1] <= 1 + except: + raise ValueError("Invalid threshold! Should be in [0, 1]") + + for threshold in thresholds: + metric = detection.DetectionErrorRate() + paired_filenames, groundtruth_RTTM_dict, vad_pred_dict = pred_rttm_map( + vad_pred, groundtruth_RTTM, vad_pred_method + ) + print(paired_filenames) + for filename in paired_filenames: + vad_pred_filepath = vad_pred_dict[filename] + groundtruth_RTTM_file = groundtruth_RTTM_dict[filename] + + if os.path.isdir(vad_pred): + table_out_dir = os.path.join(vad_pred, "table_output_" + str(threshold)) + else: + table_out_dir = os.path.join("tmp_table_outputs", "table_output_" + str(threshold)) + + if not os.path.exists(table_out_dir): + os.makedirs(table_out_dir) + + per_args = {"threshold": threshold, "shift_len": 0.01, "out_dir": table_out_dir} + vad_table_filepath = generate_vad_segment_table_per_file(vad_pred_filepath, per_args) + + reference, hypothesis = vad_construct_pyannote_object_per_file(vad_table_filepath, groundtruth_RTTM_file) + metric(reference, hypothesis) # accumulation + + report = metric.report(display=False) + DetER = report.iloc[[-1]][('detection error rate', '%')].item() + FA = report.iloc[[-1]][('false alarm', '%')].item() + MISS = report.iloc[[-1]][('miss', '%')].item() + + if focus_metric == "DetER": + score = DetER + elif focus_metric == "FA": + score = FA + elif focus_metric == "MISS": + score = MISS + else: + raise ValueError("Metric we care most should be only in 'DetER', 'FA'or 'MISS'!") + + threshold_perf[threshold] = {'DetER (%)': DetER, 'FA (%)': FA, 'MISS (%)': MISS} + logging.info(f"threshold {threshold}, {threshold_perf[threshold]}") + del report + metric.reset() # reset internal accumulator + if score < min_score: + min_score = score + best_threhsold = threshold + return best_threhsold + + +def pred_rttm_map(vad_pred, groundtruth_RTTM, vad_pred_method="frame"): + """ + Find paired files in vad_pred and groundtruth_RTTM + """ + groundtruth_RTTM_dict = {} + if os.path.isfile(groundtruth_RTTM): + with open(groundtruth_RTTM, "r") as fp: + groundtruth_RTTM_files = fp.read().splitlines() + elif os.path.isdir(groundtruth_RTTM): + groundtruth_RTTM_files = glob.glob(os.path.join(groundtruth_RTTM, "*.rttm")) + else: + raise ValueError( + "groundtruth_RTTM should either be a directory contains rttm files or a file contains paths to them!" + ) + for f in groundtruth_RTTM_files: + filename = os.path.basename(f).rsplit(".", 1)[0] + groundtruth_RTTM_dict[filename] = f + + vad_pred_dict = {} + if os.path.isfile(vad_pred): + with open(vad_pred, "r") as fp: + vad_pred_files = fp.read().splitlines() + elif os.path.isdir(vad_pred): + vad_pred_files = glob.glob(os.path.join(vad_pred, "*." + vad_pred_method)) + else: + raise ValueError( + "vad_pred should either be a directory contains vad pred files or a file contains paths to them!" + ) + for f in vad_pred_files: + filename = os.path.basename(f).rsplit(".", 1)[0] + vad_pred_dict[filename] = f + + paired_filenames = groundtruth_RTTM_dict.keys() & vad_pred_dict.keys() + return paired_filenames, groundtruth_RTTM_dict, vad_pred_dict + + +def plot(path2audio_file, path2_vad_pred, path2ground_truth_label=None, threshold=0.85): + """ + Plot VAD outputs for demonstration in tutorial + Args: + path2audio_file (str): path to audio file. + path2_vad_pred (str): path to vad prediction file, + path2ground_truth_label(str): path to groundtruth label file. + threshold (float): threshold for prediction score (from 0 to 1). + """ + plt.figure(figsize=[20, 2]) + FRAME_LEN = 0.01 + audio, sample_rate = librosa.load(path=path2audio_file, sr=16000, mono=True) + dur = librosa.get_duration(audio, sr=sample_rate) + time = np.arange(0, dur, FRAME_LEN) + frame = np.loadtxt(path2_vad_pred) + len_pred = len(frame) + ax1 = plt.subplot() + ax1.plot(np.arange(audio.size) / sample_rate, audio, 'gray') + ax1.set_xlim([0, int(dur) + 1]) + ax1.tick_params(axis='y', labelcolor='b') + ax1.set_ylabel('Signal') + ax1.set_ylim([-1, 1]) + ax2 = ax1.twinx() + prob = frame + pred = np.where(prob >= threshold, 1, 0) + if path2ground_truth_label: + label = extract_labels(path2ground_truth_label, time) + ax2.plot(np.arange(len_pred) * FRAME_LEN, label, 'r', label='label') + ax2.plot(np.arange(len_pred) * FRAME_LEN, pred, 'b', label='pred') + ax2.plot(np.arange(len_pred) * FRAME_LEN, prob, 'g--', label='speech prob') + ax2.tick_params(axis='y', labelcolor='r') + ax2.legend(loc='lower right', shadow=True) + ax2.set_ylabel('Preds and Probas') + ax2.set_ylim([-0.1, 1.1]) + return None + + +def extract_labels(path2ground_truth_label, time): + """ + Extract groundtruth label for given time period. + path2ground_truth_label (str): path of groundtruth label file + time (list) : a list of array represent time period. + """ + + data = pd.read_csv(path2ground_truth_label, sep=" ", delimiter=None, header=None) + data = data.rename(columns={3: "start", 4: "dur", 7: "speaker"}) + labels = [] + for pos in time: + line = data[(data["start"] <= pos) & (data["start"] + data["dur"] > pos)] + if len(line) >= 1: + labels.append(1) + else: + labels.append(0) + return labels diff --git a/models/tacotron2/NeMo/nemo/collections/common/__init__.py b/models/tacotron2/NeMo/nemo/collections/common/__init__.py new file mode 100644 index 0000000000..9c20362210 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nemo.collections.common.callbacks +from nemo.collections.common import data, losses, parts, tokenizers +from nemo.package_info import __version__ + +# Set collection version equal to NeMo version. +__version = __version__ + +# Authorship. +__author__ = "NVIDIA Corporation" + +# Set collection name. +__description__ = "Common collection" diff --git a/models/tacotron2/NeMo/nemo/collections/common/callbacks/__init__.py b/models/tacotron2/NeMo/nemo/collections/common/callbacks/__init__.py new file mode 100644 index 0000000000..9ad5c9c85a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/callbacks/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.common.callbacks.callbacks import LogEpochTimeCallback diff --git a/models/tacotron2/NeMo/nemo/collections/common/callbacks/callbacks.py b/models/tacotron2/NeMo/nemo/collections/common/callbacks/callbacks.py new file mode 100644 index 0000000000..3d7882e49c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/callbacks/callbacks.py @@ -0,0 +1,97 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import time + +import numpy as np +from pytorch_lightning.callbacks.base import Callback +from pytorch_lightning.utilities import rank_zero_only + +# from sacrebleu import corpus_bleu + + +class LogEpochTimeCallback(Callback): + """Simple callback that logs how long each epoch takes, in seconds, to a pytorch lightning log + """ + + @rank_zero_only + def on_train_epoch_start(self, trainer, pl_module): + self.epoch_start = time.time() + + @rank_zero_only + def on_train_epoch_end(self, trainer, pl_module, outputs): + curr_time = time.time() + duration = curr_time - self.epoch_start + trainer.logger.log_metrics({"epoch_time": duration}, step=trainer.global_step) + + +# class MachineTranslationLogEvalCallback(Callback): +# def _on_eval_end(self, trainer, pl_module, mode): +# counts = np.array(self._non_pad_tokens) +# eval_loss = np.sum(np.array(self._losses) * counts) / np.sum(counts) +# sacre_bleu = corpus_bleu(self._translations, [self._ground_truths], tokenize="13a") +# print(f"{mode} results for process with global rank {pl_module.global_rank}".upper()) +# for i in range(pl_module.num_examples[mode]): +# print('\u0332'.join(f"EXAMPLE {i}:")) # Underline output +# sent_id = np.random.randint(len(self._translations)) +# print(f"Ground truth: {self._ground_truths[sent_id]}\n") +# print(f"Translation: {self._translations[sent_id]}\n") +# print() +# print("-" * 50) +# print(f"loss: {eval_loss:.3f}") +# print(f"SacreBLEU: {sacre_bleu}") +# print("-" * 50) + +# @rank_zero_only +# def on_test_end(self, trainer, pl_module): +# self._on_eval_end(trainer, pl_module, "test") + +# @rank_zero_only +# def on_validation_end(self, trainer, pl_module): +# self._on_eval_end(trainer, pl_module, "val") + +# @rank_zero_only +# def on_sanity_check_end(self, trainer, pl_module): +# self._on_eval_end(trainer, pl_module, "val") + +# def _on_eval_batch_end(self, trainer, pl_module, outputs, batch, batch_idx, dataloader_idx, mode): +# self._translations.extend(outputs['translations']) +# self._ground_truths.extend(outputs['ground_truths']) +# self._non_pad_tokens.append(outputs['num_non_pad_tokens']) +# self._losses.append(outputs[f'{mode}_loss']) + +# @rank_zero_only +# def on_test_batch_end(self, trainer, pl_module, batch, outputs, batch_idx, dataloader_idx): +# self._on_eval_batch_end(trainer, pl_module, batch, outputs, batch_idx, dataloader_idx, 'test') + +# @rank_zero_only +# def on_validation_batch_end(self, trainer, pl_module, batch, outputs, batch_idx, dataloader_idx): +# self._on_eval_batch_end(trainer, pl_module, batch, outputs, batch_idx, dataloader_idx, 'val') + +# def _on_eval_start(self, trainer, pl_module): +# self._translations = [] +# self._ground_truths = [] +# self._losses = [] +# self._non_pad_tokens = [] + +# @rank_zero_only +# def on_test_start(self, trainer, pl_module): +# self._on_eval_start(trainer, pl_module) + +# @rank_zero_only +# def on_validation_start(self, trainer, pl_module): +# self._on_eval_start(trainer, pl_module) + +# @rank_zero_only +# def on_sanity_check_start(self, trainer, pl_module): +# self._on_eval_start(trainer, pl_module) diff --git a/models/tacotron2/NeMo/nemo/collections/common/data/__init__.py b/models/tacotron2/NeMo/nemo/collections/common/data/__init__.py new file mode 100644 index 0000000000..4eb798b78a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/data/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.common.data.dataset import ConcatDataset diff --git a/models/tacotron2/NeMo/nemo/collections/common/data/dataset.py b/models/tacotron2/NeMo/nemo/collections/common/data/dataset.py new file mode 100644 index 0000000000..3e087e2eb8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/data/dataset.py @@ -0,0 +1,178 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, List + +import numpy as np +import torch.utils.data as pt_data +from torch.utils.data import IterableDataset + +__all__ = ['ConcatDataset'] + + +class ConcatDataset(IterableDataset): + """ + A dataset that accepts as argument multiple datasets and then samples from them based on the specified + sampling technique. + Args: + datasets (list): A list of datasets to sample from. + shuffle (bool): Whether to shuffle individual datasets. Only works with non-iterable datasets. + Defaults to True. + sampling_technique (str): Sampling technique to choose which dataset to draw a sample from. + Defaults to 'temperature'. Currently supports 'temperature', 'random' and 'round-robin'. + sampling_temperature (int): Temperature value for sampling. Only used when sampling_technique = 'temperature'. + Defaults to 5. + sampling_probabilities (list): Probability values for sampling. Only used when sampling_technique = 'random'. + global_rank (int): Worker rank, used for partitioning map style datasets. Defaults to 0. + world_size (int): Total number of processes, used for partitioning map style datasets. Defaults to 1. + """ + + def __init__( + self, + datasets: List[Any], + shuffle: bool = True, + sampling_technique: str = 'temperature', + sampling_temperature: int = 5, + sampling_probabilities: List[float] = None, + global_rank: int = 0, + world_size: int = 1, + ): + super().__init__() + + supported_sampling_techniques = ['temperature', 'random', 'round-robin'] + self.datasets = datasets + self.iterables = [None] * len(datasets) + self.shuffle = shuffle + self.global_rank = global_rank + self.world_size = world_size + self.sampling_kwargs = {} + if sampling_technique == 'temperature': + self.index_generator = ConcatDataset.temperature_generator + self.sampling_kwargs['temperature'] = sampling_temperature + elif sampling_technique == 'random': + self.index_generator = ConcatDataset.random_generator + self.sampling_kwargs['p'] = sampling_probabilities + elif sampling_technique == 'round-robin': + self.index_generator = ConcatDataset.round_robin_generator + else: + raise ValueError(f"Currently we only support sampling techniques in {supported_sampling_techniques}.") + self.length = 0 + + if isinstance(datasets[0], IterableDataset): + self.kind = 'iterable' + else: + self.kind = 'map' + + for idx, dataset in enumerate(datasets): + isiterable = isinstance(dataset, IterableDataset) + if (isiterable and not self.kind == 'iterable') or (not isiterable and self.kind == 'iterable'): + raise ValueError("All datasets in ConcatDataset must be of the same kind (Iterable or Map).") + + if self.kind == 'map': + self.length += len(dataset) // world_size + else: + self.length += len(dataset) + + def get_iterable(self, dataset): + if isinstance(dataset, IterableDataset): + return dataset.__iter__() + else: + indices = np.arange(len(dataset)) + if self.shuffle: + np.random.shuffle(indices) + return iter(indices) + + def __iter__(self): + worker_info = pt_data.get_worker_info() + if worker_info is None: + max_elements = self.length + wid = 0 + wnum = 1 + else: + wid = worker_info.id + wnum = worker_info.num_workers + max_elements = len(range(wid, self.length, wnum)) + + if self.kind == 'map': + for idx in range(len(self.datasets)): + start_idx = (len(self.datasets[idx]) // self.world_size) * self.global_rank + end_idx = start_idx + (len(self.datasets[idx]) // self.world_size) + if self.global_rank == self.world_size - 1: + end_idx = len(self.datasets[idx]) + indices = range(start_idx + wid, end_idx, wnum) + self.datasets[idx] = pt_data.Subset(self.datasets[idx], indices) + + for idx, dataset in enumerate(self.datasets): + iterable = self.get_iterable(dataset) + self.iterables[idx] = iterable + + n = 0 + ind_gen = self.index_generator(self.datasets, **self.sampling_kwargs) + while n < max_elements: + n += 1 + try: + ind = next(ind_gen) + except StopIteration: + return + try: + val = next(self.iterables[ind]) + if self.kind == 'map': + val = self.datasets[ind][val] + yield val + except StopIteration: + self.iterables[ind] = self.get_iterable(self.datasets[ind]) + n -= 1 + + def __len__(self): + return self.length + + @staticmethod + def temperature_generator(datasets, **kwargs): + temp = kwargs.get('temperature') + if not temp: + raise ValueError("Temperature generator expects a 'temperature' keyowrd argument.") + + lengths = [] + num = len(datasets) + for dataset in datasets: + lengths.append(len(dataset)) + + p = np.array(lengths) / np.sum(lengths) + p = np.power(p, 1 / temp) + p = p / np.sum(p) + + while True: + ind = np.random.choice(np.arange(num), p=p) + yield ind + + @staticmethod + def round_robin_generator(datasets, **kwargs): + num = len(datasets) + while True: + for i in range(num): + yield i + + @staticmethod + def random_generator(datasets, **kwargs): + p = kwargs.get('p') + if not p: + raise ValueError("Random generator expects a 'p' keyowrd argument for sampling probabilities.") + + num = len(datasets) + if len(p) != num: + raise ValueError("Length of probabilities list must be equal to the number of datasets.") + + while True: + ind = np.random.choice(np.arange(num), p=p) + yield ind diff --git a/models/tacotron2/NeMo/nemo/collections/common/losses/__init__.py b/models/tacotron2/NeMo/nemo/collections/common/losses/__init__.py new file mode 100644 index 0000000000..cf9224d3e9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/losses/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.common.losses.aggregator import AggregatorLoss +from nemo.collections.common.losses.cross_entropy import CrossEntropyLoss, NLLLoss +from nemo.collections.common.losses.mse_loss import MSELoss +from nemo.collections.common.losses.multi_similarity_loss import MultiSimilarityLoss +from nemo.collections.common.losses.smoothed_cross_entropy import SmoothedCrossEntropyLoss +from nemo.collections.common.losses.spanning_loss import SpanningLoss diff --git a/models/tacotron2/NeMo/nemo/collections/common/losses/aggregator.py b/models/tacotron2/NeMo/nemo/collections/common/losses/aggregator.py new file mode 100644 index 0000000000..1987ddd22b --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/losses/aggregator.py @@ -0,0 +1,67 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import torch + +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types import LossType, NeuralType + +__all__ = ['AggregatorLoss'] + + +class AggregatorLoss(Loss): + """ + Sums several losses into one. + + Args: + num_inputs: number of input losses + weights: a list of coefficient for merging losses + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + input_types = {} + for i in range(self._num_losses): + input_types["loss_" + str(i + 1)] = NeuralType(elements_type=LossType()) + + return input_types + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return {"loss": NeuralType(elements_type=LossType())} + + def __init__(self, num_inputs: int = 2, weights: List[float] = None): + super().__init__() + self._num_losses = num_inputs + if weights is not None and len(weights) != num_inputs: + raise ValueError("Length of weights should be equal to the number of inputs (num_inputs)") + + self._weights = weights + + @typecheck() + def forward(self, **kwargs): + values = [kwargs[x] for x in sorted(kwargs.keys())] + loss = torch.zeros_like(values[0]) + for loss_idx, loss_value in enumerate(values): + if self._weights is not None: + loss = loss.add(loss_value, alpha=self._weights[loss_idx]) + else: + loss = loss.add(loss_value) + return loss diff --git a/models/tacotron2/NeMo/nemo/collections/common/losses/cross_entropy.py b/models/tacotron2/NeMo/nemo/collections/common/losses/cross_entropy.py new file mode 100644 index 0000000000..9a8dc187b7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/losses/cross_entropy.py @@ -0,0 +1,138 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torch import nn + +from nemo.core.classes import Serialization, Typing, typecheck +from nemo.core.neural_types import LabelsType, LogitsType, LogprobsType, LossType, MaskType, NeuralType + +__all__ = ['CrossEntropyLoss', 'NLLLoss'] + + +class CrossEntropyLoss(nn.CrossEntropyLoss, Serialization, Typing): + """ + CrossEntropyLoss + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "logits": NeuralType(['B'] + ['ANY'] * (self._logits_dim - 1), LogitsType()), + "labels": NeuralType(['B'] + ['ANY'] * (self._logits_dim - 2), LabelsType()), + "loss_mask": NeuralType(['B'] + ['ANY'] * (self._logits_dim - 2), MaskType(), optional=True), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return {"loss": NeuralType(elements_type=LossType())} + + def __init__(self, logits_ndim=2, weight=None, reduction='mean', ignore_index=-100): + """ + Args: + logits_ndim (int): number of dimensions (or rank) of the logits tensor + weight (list): list of rescaling weight given to each class + reduction (str): type of the reduction over the batch + """ + if weight is not None and not torch.is_tensor(weight): + weight = torch.FloatTensor(weight) + super().__init__(weight=weight, reduction=reduction, ignore_index=ignore_index) + self._logits_dim = logits_ndim + + @typecheck() + def forward(self, logits, labels, loss_mask=None): + """ + Args: + logits (float): output of the classifier + labels (long): ground truth labels + loss_mask (bool/float/int): tensor to specify the masking + """ + logits_flatten = torch.flatten(logits, start_dim=0, end_dim=-2) + labels_flatten = torch.flatten(labels, start_dim=0, end_dim=-1) + + if loss_mask is not None: + if loss_mask.dtype is not torch.bool: + loss_mask = loss_mask > 0.5 + loss_mask_flatten = torch.flatten(loss_mask, start_dim=0, end_dim=-1) + logits_flatten = logits_flatten[loss_mask_flatten] + labels_flatten = labels_flatten[loss_mask_flatten] + + if len(labels_flatten) == 0: + return super().forward(logits, torch.argmax(logits, dim=-1)) + + loss = super().forward(logits_flatten, labels_flatten) + return loss + + +class NLLLoss(nn.NLLLoss, Serialization, Typing): + """ + NLLLoss + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "log_probs": NeuralType(("B", "T", "D"), LogprobsType()), + "labels": NeuralType(("B", "T"), LabelsType()), + "output_mask": NeuralType(("B", "T"), MaskType(), optional=True), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return {"loss": NeuralType(elements_type=LossType())} + + def __init__(self, log_probs_ndim=2, weight=None, reduction='mean', ignore_index=-100): + """ + Args: + log_probs_ndim (int): number of dimensions (or rank) of the logprobs tensor + weight (list): list of rescaling weight given to each class + reduction (str): type of the reduction over the batch + ignore_index (int): mask out loss computation where labels = ignore_index + """ + if weight is not None and not torch.is_tensor(weight): + weight = torch.FloatTensor(weight) + super().__init__(weight=weight, reduction=reduction, ignore_index=ignore_index) + self._log_probs_dim = log_probs_ndim + + @typecheck() + def forward(self, log_probs, labels, loss_mask=None): + """ + Args: + log_probs (float): output log probability tensor + labels (long): ground truth labels + loss_mask (bool/float/int): tensor to specify the masking + """ + log_probs_flatten = torch.flatten(log_probs, start_dim=0, end_dim=-2) + labels_flatten = torch.flatten(labels, start_dim=0, end_dim=-1) + + if loss_mask is not None: + if loss_mask.dtype is not torch.bool: + loss_mask = loss_mask > 0.5 + loss_mask_flatten = torch.flatten(loss_mask, start_dim=0, end_dim=-1) + log_probs_flatten = log_probs_flatten[loss_mask_flatten] + labels_flatten = labels_flatten[loss_mask_flatten] + + if len(labels_flatten) == 0: + return super().forward(log_probs, torch.argmax(log_probs, dim=-1)) + + loss = super().forward(log_probs_flatten, labels_flatten) + return loss diff --git a/models/tacotron2/NeMo/nemo/collections/common/losses/mse_loss.py b/models/tacotron2/NeMo/nemo/collections/common/losses/mse_loss.py new file mode 100644 index 0000000000..802e8ca492 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/losses/mse_loss.py @@ -0,0 +1,57 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from torch import Tensor, nn + +from nemo.core.classes import Serialization, Typing, typecheck +from nemo.core.neural_types import LabelsType, LossType, NeuralType, RegressionValuesType + +__all__ = ['MSELoss'] + + +class MSELoss(nn.MSELoss, Serialization, Typing): + """ + MSELoss + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "preds": NeuralType(tuple('B'), RegressionValuesType()), + "labels": NeuralType(tuple('B'), LabelsType()), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return {"loss": NeuralType(elements_type=LossType())} + + def __init__(self, reduction: str = 'mean'): + """ + Args: + reduction: type of the reduction over the batch + """ + super().__init__(reduction=reduction) + + @typecheck() + def forward(self, preds: Tensor, labels: Tensor) -> Tensor: + """ + Args: + preds: output of the classifier + labels: ground truth labels + """ + return super().forward(preds, labels) diff --git a/models/tacotron2/NeMo/nemo/collections/common/losses/multi_similarity_loss.py b/models/tacotron2/NeMo/nemo/collections/common/losses/multi_similarity_loss.py new file mode 100644 index 0000000000..022f6d6de6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/losses/multi_similarity_loss.py @@ -0,0 +1,95 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch + +from nemo.core.classes import Loss +from nemo.core.classes.common import typecheck +from nemo.core.neural_types import LabelsType, LogitsType, LossType, NeuralType +from nemo.utils import logging + +__all__ = ['MultiSimilarityLoss'] + + +class MultiSimilarityLoss(Loss): + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return {"logits": NeuralType(('B', 'D'), LogitsType()), "labels": NeuralType(('B'), LabelsType())} + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return {"loss": NeuralType(elements_type=LossType())} + + def __init__( + self, + scale_pos: Optional[float] = 2.0, # Params found to work best in our experiments + scale_neg: Optional[float] = 40.0, + offset: Optional[float] = 0.5, + margin: Optional[float] = 0.1, + ): + super().__init__() + self._scale_pos = scale_pos + self._scale_neg = scale_neg + self._offset = offset + self._margin = margin + self._epsilon = 1e-5 + + @typecheck() + def forward(self, logits, labels): + cos_sim = torch.matmul(logits, torch.t(logits)) + losses = [] + + for i in range(logits.size(0)): + # mine hard pairs relative to anchor i + positive_sims = cos_sim[i][labels.eq(labels[i])] + positive_sims = positive_sims[positive_sims.lt(1 - self._epsilon)] # omit identical pairs + negative_sims = cos_sim[i][labels.ne(labels[i])] + + if len(negative_sims) == 0 or len(positive_sims) == 0: + continue + + # negatives that are more similar than the least-similar positive + hard_negatives = negative_sims[negative_sims.gt(min(positive_sims) - self._margin)] + + # positives that are less similar than the most-similar negative + hard_positives = positive_sims[positive_sims.lt(max(negative_sims) + self._margin)] + + if len(hard_negatives) == 0 or len(hard_positives) == 0: + continue + + pos_term = ( + 1.0 + / self._scale_pos + * torch.log(1 + torch.sum(torch.exp(-self._scale_pos * (hard_positives - self._offset)))) + ) + neg_term = ( + 1.0 + / self._scale_neg + * torch.log(1 + torch.sum(torch.exp(self._scale_neg * (hard_negatives - self._offset)))) + ) + losses.append(pos_term + neg_term) + + if len(losses) == 0: + loss = torch.zeros([], requires_grad=True).cuda() + logging.info(f'Encountered zero loss in multisimloss, loss = {loss}. No hard examples found in the batch') + else: + loss = torch.sum(torch.stack(losses)) / logits.size(0) + + return loss diff --git a/models/tacotron2/NeMo/nemo/collections/common/losses/smoothed_cross_entropy.py b/models/tacotron2/NeMo/nemo/collections/common/losses/smoothed_cross_entropy.py new file mode 100644 index 0000000000..8e37d29ba4 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/losses/smoothed_cross_entropy.py @@ -0,0 +1,103 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch + +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types import LabelsType, LogprobsType, LossType, MaskType, NeuralType + +__all__ = ["SmoothedCrossEntropyLoss"] + + +class SmoothedCrossEntropyLoss(Loss): + """ + Calculates Cross-entropy loss with label smoothing for a batch of sequences. + + SmoothedCrossEntropyLoss: + 1) excludes padding tokens from loss calculation + 2) allows to use label smoothing regularization + 3) allows to calculate loss for the desired number of last tokens + + Args: + label_smoothing (float): label smoothing regularization coefficient + predict_last_k (int): parameter which sets the number of last tokens to calculate the loss for, for example + 0: (default) calculate loss on the entire sequence (e.g., NMT) + 1: calculate loss on the last token only (e.g., LM evaluation) + Intermediate values allow to control the trade-off between eval + time (proportional to the number of batches) and eval performance + (proportional to the number of context tokens) + pad_id (int): padding id + eps (float): the small eps number to avoid division buy zero + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "log_probs": NeuralType(("B", "T", "D"), LogprobsType()), + "labels": NeuralType(("B", "T"), LabelsType()), + "output_mask": NeuralType(("B", "T"), MaskType(), optional=True), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return {"loss": NeuralType(elements_type=LossType())} + + def __init__( + self, + pad_id: Optional[int] = None, + label_smoothing: Optional[float] = 0.0, + predict_last_k: Optional[int] = 0, + eps: float = 1e-6, + ): + super().__init__() + self._pad_id = pad_id + self._eps = eps + self._predict_last_k = predict_last_k + self._label_smoothing = label_smoothing + + @typecheck() + def forward(self, log_probs, labels, output_mask=None): + """ + Args: + log_probs: float tensor of shape batch_size x seq_len x vocab_size, values should be log probabilities + labels: int tensor of shape batch_size x seq_len + output_mask: binary tensor of shape batch_size x seq_len + eps: epsilon param to avoid divide by zero in loss calculation + """ + if output_mask is None and self._pad_id is None: + raise ValueError("Both output_mask and pad_id are None") + if output_mask is None and self._pad_id is not None: + output_mask = (labels != self._pad_id).to(log_probs.dtype) + + if output_mask.dtype is not log_probs.dtype: + output_mask = output_mask.to(log_probs.dtype) + + batch_size, seq_len, vocab_size = log_probs.size() + smoothing = vocab_size * self._label_smoothing / (vocab_size - 1) + target_log_probs = log_probs.gather(2, labels.unsqueeze(2)).squeeze(2) + + smoothing_log_probs = log_probs.mean(dim=-1) + neg_log_likelihood = (1.0 - smoothing) * target_log_probs + smoothing * smoothing_log_probs + neg_log_likelihood = neg_log_likelihood[:, -self._predict_last_k :] + output_mask = output_mask[:, -self._predict_last_k :] + neg_log_likelihood = -torch.sum(neg_log_likelihood * output_mask) + neg_log_likelihood = neg_log_likelihood / (output_mask.sum() + self._eps) + + return neg_log_likelihood diff --git a/models/tacotron2/NeMo/nemo/collections/common/losses/spanning_loss.py b/models/tacotron2/NeMo/nemo/collections/common/losses/spanning_loss.py new file mode 100644 index 0000000000..a12dab64af --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/losses/spanning_loss.py @@ -0,0 +1,79 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from torch import nn + +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types import ChannelType, LogitsType, LossType, NeuralType + +__all__ = ['SpanningLoss'] + + +class SpanningLoss(Loss): + """ + implements start and end loss of a span e.g. for Question Answering. + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return { + "logits": NeuralType(('B', 'T', 'D'), LogitsType()), + "start_positions": NeuralType(tuple('B'), ChannelType()), + "end_positions": NeuralType(tuple('B'), ChannelType()), + } + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return { + "loss": NeuralType(elements_type=LossType()), + "start_logits": NeuralType(('B', 'T'), LogitsType()), + "end_logits": NeuralType(('B', 'T'), LogitsType()), + } + + def __init__(self,): + super().__init__() + + @typecheck() + def forward(self, logits, start_positions, end_positions): + """ + Args: + logits: Output of question answering head, which is a token classfier. + start_positions: Ground truth start positions of the answer w.r.t. + input sequence. If question is unanswerable, this will be + pointing to start token, e.g. [CLS], of the input sequence. + end_positions: Ground truth end positions of the answer w.r.t. + input sequence. If question is unanswerable, this will be + pointing to start token, e.g. [CLS], of the input sequence. + """ + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = nn.CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + return total_loss, start_logits, end_logits diff --git a/models/tacotron2/NeMo/nemo/collections/common/metrics/__init__.py b/models/tacotron2/NeMo/nemo/collections/common/metrics/__init__.py new file mode 100644 index 0000000000..10c4950c70 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/metrics/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.common.metrics.classification_accuracy import TopKClassificationAccuracy +from nemo.collections.common.metrics.global_average_loss_metric import GlobalAverageLossMetric +from nemo.collections.common.metrics.perplexity import Perplexity diff --git a/models/tacotron2/NeMo/nemo/collections/common/metrics/classification_accuracy.py b/models/tacotron2/NeMo/nemo/collections/common/metrics/classification_accuracy.py new file mode 100644 index 0000000000..3bcd323684 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/metrics/classification_accuracy.py @@ -0,0 +1,152 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import torch +from torchmetrics import Metric + +__all__ = ['TopKClassificationAccuracy'] + + +class TopKClassificationAccuracy(Metric): + """ + This metric computes numerator and denominator for Overall Accuracy between logits and labels. + When doing distributed training/evaluation the result of res=TopKClassificationAccuracy(logits, labels) calls + will be all-reduced between all workers using SUM operations. + Here contains two numbers res=[correctly_predicted, total_samples]. Accuracy=correctly_predicted/total_samples. + + If used with PytorchLightning LightningModule, include correct_count and total_count inside validation_step results. + Then aggregate (sum) then at the end of validation epoch to correctly compute validation WER. + + Example: + def validation_step(self, batch, batch_idx): + ... + correct_count, total_count = self._accuracy(logits, labels) + return {'val_loss': loss_value, 'val_correct_count': correct_count, 'val_total_count': total_count} + + def validation_epoch_end(self, outputs): + ... + val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean() + correct_counts = torch.stack([x['val_correct_counts'] for x in outputs]) + total_counts = torch.stack([x['val_total_counts'] for x in outputs]) + + topk_scores = compute_topk_accuracy(correct_counts, total_counts) + + tensorboard_log = {'val_loss': val_loss_mean} + for top_k, score in zip(self._accuracy.top_k, topk_scores): + tensorboard_log['val_epoch_top@{}'.format(top_k)] = score + + return {'log': tensorboard_log} + + Args: + top_k: Optional list of integers. Defaults to [1]. + + Returns: + res: a torch.Tensor object with two elements: [correct_count, total_count]. To correctly compute average + accuracy, compute acc=correct_count/total_count + """ + + def __init__(self, top_k=None, dist_sync_on_step=False): + super().__init__(dist_sync_on_step=dist_sync_on_step) + + if top_k is None: + top_k = [1] + + self.top_k = top_k + self.add_state( + "correct_counts_k", default=torch.zeros(len(self.top_k)), dist_reduce_fx='sum', persistent=False + ) + self.add_state("total_counts_k", default=torch.zeros(len(self.top_k)), dist_reduce_fx='sum', persistent=False) + + @torch.no_grad() + def top_k_predicted_labels(self, logits: torch.Tensor) -> torch.Tensor: + max_k = max(self.top_k) + _, predictions = logits.topk(max_k, dim=1, largest=True, sorted=True) + return predictions + + def update(self, logits: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: + with torch.no_grad(): + predictions = self.top_k_predicted_labels(logits) + predictions = predictions.t() + correct = predictions.eq(labels.view(1, -1)).expand_as(predictions) + + correct_counts_k = [] + total_counts_k = [] + + for k in self.top_k: + correct_k = correct[:k].reshape(-1).long().sum() + total_k = labels.shape[0] + + correct_counts_k.append(correct_k) + total_counts_k.append(total_k) + + self.correct_counts_k = torch.tensor(correct_counts_k, dtype=labels.dtype, device=labels.device) + self.total_counts_k = torch.tensor(total_counts_k, dtype=labels.dtype, device=labels.device) + + def compute(self): + """ + Computes the top-k accuracy. + + Returns: + A list of length `K`, such that k-th index corresponds to top-k accuracy + over all distributed processes. + """ + if not len(self.correct_counts_k) == len(self.top_k) == len(self.total_counts_k): + raise ValueError("length of counts must match to topk length") + + if self.top_k == [1]: + return [self.correct_counts_k.float() / self.total_counts_k] + + else: + top_k_scores = compute_topk_accuracy(self.correct_counts_k, self.total_counts_k) + + return top_k_scores + + @property + def top_k(self) -> List[int]: + return self._top_k + + @top_k.setter + def top_k(self, value: List[int]): + if value is None: + value = [1] + + if type(value) == int: + value = [value] + + if type(value) != list: + value = list(value) + + self._top_k = value + + +def compute_topk_accuracy(correct_counts_k, total_counts_k): + """ + Computes the top-k accuracy + Args: + correct_counts: Tensor of shape [K], K being the top-k parameter. + total_counts: Tensor of shape [K], and K being the top-k parameter. + Returns: + A list of length `K`, such that k-th index corresponds to top-k accuracy + over all distributed processes. + """ + top_k_scores = [] + + for ki in range(len(correct_counts_k)): + correct_count = correct_counts_k[ki].item() + total_count = total_counts_k[ki].item() + top_k_scores.append(correct_count / float(total_count)) + + return top_k_scores diff --git a/models/tacotron2/NeMo/nemo/collections/common/metrics/global_average_loss_metric.py b/models/tacotron2/NeMo/nemo/collections/common/metrics/global_average_loss_metric.py new file mode 100644 index 0000000000..5f6f00e974 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/metrics/global_average_loss_metric.py @@ -0,0 +1,73 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torchmetrics import Metric + +__all__ = ['GlobalAverageLossMetric'] + + +class GlobalAverageLossMetric(Metric): + """ + This class is for averaging loss across multiple processes if a distributed backend is used. True average is + computed not running average. It does not accumulate gradients so the averaged loss cannot be used for optimization. + If ``take_avg_loss`` is ``True``, the :meth:`update` method ``loss`` argument has to be a mean loss. If + ``take_avg_loss`` is ``False`` then the :meth:`update` method ``loss`` argument has to be a sum of losses. + + See :doc:`PyTorch Lightning Metrics` for the metric usage instruction. + + Args: + compute_on_step: + Forward only calls ``update()`` and returns ``None`` if this is set to ``False``. default: ``True`` + dist_sync_on_step: + Synchronize metric state across processes at each ``forward()`` + before returning the value at the step. + process_group: + Specify the process group on which synchronization is called. default: ``None`` (which selects the entire + world) + take_avg_loss: + If ``True`` values of :meth:`update` method ``loss`` argument has to be a mean loss. If ``False`` + values of :meth:`update` method ``loss`` argument has to be a sum of losses. default: ``True`` + """ + + def __init__(self, compute_on_step=True, dist_sync_on_step=False, process_group=None, take_avg_loss=True): + super().__init__( + compute_on_step=compute_on_step, dist_sync_on_step=dist_sync_on_step, process_group=process_group + ) + self.add_state("loss_sum", torch.tensor(0.0, dtype=torch.float64), dist_reduce_fx='sum') + self.add_state("num_measurements", torch.tensor(0, dtype=torch.int64), dist_reduce_fx='sum') + self.take_avg_loss = take_avg_loss + + def update(self, loss, num_measurements): + """ + Updates :attr:`loss_sum` and :attr:`num_measurements`. + + Args: + loss: A float zero dimensional ``torch.Tensor`` which is either sum or average of losses for processed + examples. See ``take_avg_loss`` parameter of :meth:`__init__`. + num_measurements: An integer zero dimensional ``torch.Tensor`` which contains a number of loss measurements. + The sum or mean of the results of these measurements are in the ``loss`` parameter. + """ + if self.take_avg_loss: + loss *= num_measurements + self.loss_sum += loss + self.num_measurements += num_measurements + + def compute(self): + """ + Returns mean loss. + """ + if self.num_measurements.eq(0): + return torch.tensor(float('nan')) + return self.loss_sum / self.num_measurements diff --git a/models/tacotron2/NeMo/nemo/collections/common/metrics/perplexity.py b/models/tacotron2/NeMo/nemo/collections/common/metrics/perplexity.py new file mode 100644 index 0000000000..cddcc9febd --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/metrics/perplexity.py @@ -0,0 +1,71 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torch.distributions.categorical import Categorical +from torchmetrics import Metric + +__all__ = ['Perplexity'] + + +class Perplexity(Metric): + """ + This class computes mean perplexity of distributions in the last dimension of inputs. It is a wrapper around + :doc:`torch.distributions.Categorical.perplexity` method. You have to provide either + ``probs`` or ``logits`` to the :meth:`update` method. The class computes perplexities for distributions passed to + :meth:`update` method in ``probs`` or ``logits`` arguments and averages the perplexities. Reducing results between + all workers is done via SUM operations. + See :doc:`PyTorch Lightning Metrics` for the metric usage instructions. + Args: + compute_on_step: + Forward only calls ``update()`` and returns ``None`` if this is set to ``False``. default: ``True`` + dist_sync_on_step: + Synchronize metric state across processes at each ``forward()`` + before returning the value at the step. + process_group: + Specify the process group on which synchronization is called. default: ``None`` (which selects the entire + world) + validate_args: + If ``True`` values of :meth:`update` method parameters are checked. ``logits`` has to not contain NaNs and + ``probs`` last dim has to be valid probability distribution. + """ + + def __init__(self, compute_on_step=True, dist_sync_on_step=False, process_group=None, validate_args=True): + super().__init__( + compute_on_step=compute_on_step, dist_sync_on_step=dist_sync_on_step, process_group=process_group + ) + self.validate_args = validate_args + self.add_state('perplexities_sum', torch.tensor(0.0, dtype=torch.float64), dist_reduce_fx='sum') + # Total number of distributions seen since last reset + self.add_state('num_distributions', torch.tensor(0, dtype=torch.int64), dist_reduce_fx='sum') + + def update(self, probs=None, logits=None): + """ + Updates :attr:`perplexities_sum` and :attr:`num_distributions`. + Args: + probs: A ``torch.Tensor`` which innermost dimension is valid probability distribution. + logits: A ``torch.Tensor`` without NaNs. + """ + d = Categorical(probs, logits, validate_args=self.validate_args) + ppl = d.perplexity() + self.num_distributions += ppl.numel() + self.perplexities_sum += ppl.sum() + + def compute(self): + """ + Returns perplexity across all workers and resets to 0 :attr:`perplexities_sum` and :attr:`num_distributions`. + """ + if self.num_distributions.eq(0): + return None + return self.perplexities_sum / self.num_distributions diff --git a/models/tacotron2/NeMo/nemo/collections/common/parts/__init__.py b/models/tacotron2/NeMo/nemo/collections/common/parts/__init__.py new file mode 100644 index 0000000000..7f3916e365 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/parts/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.common.parts.multi_layer_perceptron import MultiLayerPerceptron +from nemo.collections.common.parts.transformer_utils import * +from nemo.collections.common.parts.utils import * diff --git a/models/tacotron2/NeMo/nemo/collections/common/parts/multi_layer_perceptron.py b/models/tacotron2/NeMo/nemo/collections/common/parts/multi_layer_perceptron.py new file mode 100644 index 0000000000..76c06bf23e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/parts/multi_layer_perceptron.py @@ -0,0 +1,61 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + + +class MultiLayerPerceptron(torch.nn.Module): + """ + A simple MLP that can either be used independently or put on top + of pretrained models (such as BERT) and act as a classifier. + Args: + hidden_size (int): the size of each layer + num_classes (int): number of output classes + num_layers (int): number of layers + activation (str): type of activations for layers in between + log_softmax (bool): whether to add a log_softmax layer before output + """ + + def __init__( + self, + hidden_size: int, + num_classes: int, + num_layers: int = 2, + activation: str = 'relu', + log_softmax: bool = True, + ): + super().__init__() + self.layers = 0 + for _ in range(num_layers - 1): + layer = torch.nn.Linear(hidden_size, hidden_size) + setattr(self, f'layer{self.layers}', layer) + setattr(self, f'layer{self.layers + 1}', getattr(torch, activation)) + self.layers += 2 + layer = torch.nn.Linear(hidden_size, num_classes) + setattr(self, f'layer{self.layers}', layer) + self.layers += 1 + self.log_softmax = log_softmax + + @property + def last_linear_layer(self): + return getattr(self, f'layer{self.layers - 1}') + + def forward(self, hidden_states): + output_states = hidden_states[:] + for i in range(self.layers): + output_states = getattr(self, f'layer{i}')(output_states) + + if self.log_softmax: + output_states = torch.log_softmax(output_states, dim=-1) + return output_states diff --git a/models/tacotron2/NeMo/nemo/collections/common/parts/patch_utils.py b/models/tacotron2/NeMo/nemo/collections/common/parts/patch_utils.py new file mode 100644 index 0000000000..1946a639ef --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/parts/patch_utils.py @@ -0,0 +1,132 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch +from packaging import version + +from nemo.utils import logging + +# Library version globals +TORCH_VERSION = None +TORCH_VERSION_MIN = version.Version('1.7') + + +def stft_patch( + input: torch.Tensor, + n_fft: int, + hop_length: Optional[int] = None, + win_length: Optional[int] = None, + window: Optional[torch.Tensor] = None, + center: bool = True, + pad_mode: str = 'reflect', + normalized: bool = False, + onesided: Optional[bool] = None, + return_complex: Optional[bool] = None, +): + """ + Patch over torch.stft for PyTorch <= 1.6. + Arguments are same as torch.stft(). + + # TODO: Remove once PyTorch 1.7+ is a requirement. + """ + global TORCH_VERSION + if TORCH_VERSION is None: + TORCH_VERSION = version.parse(torch.__version__) + + logging.warning( + "torch.stft() signature has been updated for PyTorch 1.7+\n" + "Please update PyTorch to remain compatible with later versions of NeMo." + ) + + if TORCH_VERSION < TORCH_VERSION_MIN: + return torch.stft( + input=input, + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=window, + center=center, + pad_mode=pad_mode, + normalized=normalized, + onesided=True, + ) + else: + return torch.stft( + input=input, + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=window, + center=center, + pad_mode=pad_mode, + normalized=normalized, + onesided=onesided, + return_complex=return_complex, + ) + + +def istft_patch( + input: torch.Tensor, + n_fft: int, + hop_length: Optional[int] = None, + win_length: Optional[int] = None, + window: Optional[torch.Tensor] = None, + center: bool = True, + normalized: bool = False, + onesided: Optional[bool] = None, + length: int = None, + return_complex: Optional[bool] = False, +): + """ + Patch over torch.stft for PyTorch <= 1.6. + Arguments are same as torch.stft(). + + # TODO: Remove once PyTorch 1.7+ is a requirement. + """ + global TORCH_VERSION + if TORCH_VERSION is None: + TORCH_VERSION = version.parse(torch.__version__) + + logging.warning( + "torch.stft() signature has been updated for PyTorch 1.7+\n" + "Please update PyTorch to remain compatible with later versions of NeMo." + ) + + if TORCH_VERSION < TORCH_VERSION_MIN: + return torch.istft( + input=input, + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=window, + center=center, + length=length, + normalized=normalized, + onesided=True, + ) + else: + return torch.istft( + input=input, + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=window, + center=center, + length=length, + normalized=normalized, + onesided=onesided, + return_complex=return_complex, + ) diff --git a/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/__init__.py b/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/cleaners.py b/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/cleaners.py new file mode 100644 index 0000000000..20e9685cc8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/cleaners.py @@ -0,0 +1,259 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import inflect +from unidecode import unidecode + +from nemo.utils import logging + +NUM_CHECK = re.compile(r'([$]?)(^|\s)(\S*[0-9]\S*)(?=(\s|$)((\S*)(\s|$))?)') + +TIME_CHECK = re.compile(r'([0-9]{1,2}):([0-9]{2})(am|pm)?') +CURRENCY_CHECK = re.compile(r'\$') +ORD_CHECK = re.compile(r'([0-9]+)(st|nd|rd|th)') +THREE_CHECK = re.compile(r'([0-9]{3})([.,][0-9]{1,2})?([!.?])?$') +DECIMAL_CHECK = re.compile(r'([.,][0-9]{1,2})$') + +ABBREVIATIONS_COMMON = [ + (re.compile('\\b%s\\.' % x[0]), x[1]) + for x in [ + ("ms", "miss"), + ("mrs", "misess"), + ("mr", "mister"), + ("messrs", "messeurs"), + ("dr", "doctor"), + ("drs", "doctors"), + ("st", "saint"), + ("co", "company"), + ("jr", "junior"), + ("sr", "senior"), + ("rev", "reverend"), + ("hon", "honorable"), + ("sgt", "sergeant"), + ("capt", "captain"), + ("maj", "major"), + ("col", "colonel"), + ("lt", "lieutenant"), + ("gen", "general"), + ("prof", "professor"), + ("lb", "pounds"), + ("rep", "representative"), + ("st", "street"), + ("ave", "avenue"), + ("etc", "et cetera"), + ("jan", "january"), + ("feb", "february"), + ("mar", "march"), + ("apr", "april"), + ("jun", "june"), + ("jul", "july"), + ("aug", "august"), + ("sep", "september"), + ("oct", "october"), + ("nov", "november"), + ("dec", "december"), + ] +] + +ABBREVIATIONS_EXPANDED = [ + (re.compile('\\b%s\\.' % x[0]), x[1]) + for x in [ + ("ltd", "limited"), + ("fig", "figure"), + ("figs", "figures"), + ("gent", "gentlemen"), + ("ft", "fort"), + ("esq", "esquire"), + ("prep", "preperation"), + ("bros", "brothers"), + ("ind", "independent"), + ("mme", "madame"), + ("pro", "professional"), + ("vs", "versus"), + ("inc", "include"), + ] +] + +ABBREVIATIONS_TTS_FASTPITCH = [ + (re.compile('\\b%s\\.' % x[0]), x[1]) + for x in [ + ("ms", "miss"), + ("mrs", "misess"), + ("mr", "mister"), + ("dr", "doctor"), + ("drs", "doctors"), + ("st", "saint"), + ("co", "company"), + ("jr", "junior"), + ("sr", "senior"), + ("rev", "reverend"), + ("hon", "honorable"), + ("sgt", "sergeant"), + ("capt", "captain"), + ("maj", "major"), + ("col", "colonel"), + ("lt", "lieutenant"), + ("gen", "general"), + ("prof", "professor"), + ("lb", "pounds"), + ("rep", "representative"), + ("st", "street"), + ("ave", "avenue"), + ("jan", "january"), + ("feb", "february"), + ("mar", "march"), + ("apr", "april"), + ("jun", "june"), + ("jul", "july"), + ("aug", "august"), + ("sep", "september"), + ("oct", "october"), + ("nov", "november"), + ("dec", "december"), + ("ltd", "limited"), + ("fig", "figure"), + ("figs", "figures"), + ("gent", "gentlemen"), + ("ft", "fort"), + ("esq", "esquire"), + ("prep", "preperation"), + ("bros", "brothers"), + ("ind", "independent"), + ("mme", "madame"), + ("pro", "professional"), + ("vs", "versus"), + ] +] + + +inflect = inflect.engine() + + +def clean_text(string, table, punctuation_to_replace, abbreviation_version=None): + warn_common_chars(string) + string = unidecode(string) + string = string.lower() + string = re.sub(r'\s+', " ", string) + string = clean_numbers(string) + string = clean_abbreviations(string, version=abbreviation_version) + string = clean_punctuations(string, table, punctuation_to_replace) + string = re.sub(r'\s+', " ", string).strip() + return string + + +def warn_common_chars(string): + if re.search(r'[£€]', string): + logging.warning("Your transcript contains one of '£' or '€' which we do not currently handle") + + +def clean_numbers(string): + cleaner = NumberCleaner() + string = NUM_CHECK.sub(cleaner.clean, string) + return string + + +def clean_abbreviations(string, version=None): + abbbreviations = ABBREVIATIONS_COMMON + if version == "fastpitch": + abbbreviations = ABBREVIATIONS_TTS_FASTPITCH + elif version == "expanded": + abbbreviations.extend = ABBREVIATIONS_EXPANDED + for regex, replacement in abbbreviations: + string = re.sub(regex, replacement, string) + return string + + +def clean_punctuations(string, table, punctuation_to_replace): + for punc, replacement in punctuation_to_replace.items(): + string = re.sub('\\{}'.format(punc), " {} ".format(replacement), string) + if table: + string = string.translate(table) + return string + + +class NumberCleaner: + def __init__(self): + super().__init__() + self.reset() + + def reset(self): + self.curr_num = [] + self.currency = None + + def format_final_number(self, whole_num, decimal): + if self.currency: + return_string = inflect.number_to_words(whole_num) + return_string += " dollar" if whole_num == 1 else " dollars" + if decimal: + return_string += " and " + inflect.number_to_words(decimal) + return_string += " cent" if whole_num == decimal else " cents" + self.reset() + return return_string + + self.reset() + if decimal: + whole_num += "." + decimal + return inflect.number_to_words(whole_num) + else: + # Check if there are non-numbers + def convert_to_word(match): + return " " + inflect.number_to_words(match.group(0)) + " " + + return re.sub(r'[0-9,]+', convert_to_word, whole_num) + + def clean(self, match): + ws = match.group(2) + number = match.group(3) + _proceeding_symbol = match.group(7) + + time_match = TIME_CHECK.match(number) + if time_match: + string = ws + inflect.number_to_words(time_match.group(1)) + "{}{}" + mins = int(time_match.group(2)) + min_string = "" + if mins != 0: + min_string = " " + inflect.number_to_words(time_match.group(2)) + ampm_string = "" + if time_match.group(3): + ampm_string = " " + time_match.group(3) + return string.format(min_string, ampm_string) + + ord_match = ORD_CHECK.match(number) + if ORD_CHECK.match(number): + return ws + inflect.number_to_words(ord_match.group(0)) + + if self.currency is None: + # Check if it is a currency + self.currency = match.group(1) or CURRENCY_CHECK.match(number) + + # Check to see if next symbol is a number + # If it is a number and it has 3 digits, then it is probably a + # continuation + three_match = THREE_CHECK.match(match.group(6)) + if three_match: + self.curr_num.append(number) + return " " + # Else we can output + else: + # Check for decimals + whole_num = "".join(self.curr_num) + number + decimal = None + decimal_match = DECIMAL_CHECK.search(whole_num) + if decimal_match: + decimal = decimal_match.group(1)[1:] + whole_num = whole_num[: -len(decimal) - 1] + whole_num = re.sub(r'\.', '', whole_num) + return ws + self.format_final_number(whole_num, decimal) diff --git a/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/collections.py b/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/collections.py new file mode 100644 index 0000000000..b3003b586b --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/collections.py @@ -0,0 +1,468 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import json +import os +from typing import Any, Dict, List, Optional, Union + +import pandas as pd + +from nemo.collections.common.parts.preprocessing import manifest, parsers +from nemo.utils import logging + + +class _Collection(collections.UserList): + """List of parsed and preprocessed data.""" + + OUTPUT_TYPE = None # Single element output type. + + +class Text(_Collection): + """Simple list of preprocessed text entries, result in list of tokens.""" + + OUTPUT_TYPE = collections.namedtuple('TextEntity', 'tokens') + + def __init__(self, texts: List[str], parser: parsers.CharParser): + """Instantiates text manifest and do the preprocessing step. + + Args: + texts: List of raw texts strings. + parser: Instance of `CharParser` to convert string to tokens. + """ + + data, output_type = [], self.OUTPUT_TYPE + for text in texts: + tokens = parser(text) + + if tokens is None: + logging.warning("Fail to parse '%s' text line.", text) + continue + + data.append(output_type(tokens)) + + super().__init__(data) + + +class FromFileText(Text): + """Another form of texts manifest with reading from file.""" + + def __init__(self, file: str, parser: parsers.CharParser): + """Instantiates text manifest and do the preprocessing step. + + Args: + file: File path to read from. + parser: Instance of `CharParser` to convert string to tokens. + """ + + texts = self.__parse_texts(file) + + super().__init__(texts, parser) + + @staticmethod + def __parse_texts(file: str) -> List[str]: + if not os.path.exists(file): + raise ValueError('Provided texts file does not exists!') + + _, ext = os.path.splitext(file) + if ext == '.csv': + texts = pd.read_csv(file)['transcript'].tolist() + elif ext == '.json': # Not really a correct json. + texts = list(item['text'] for item in manifest.item_iter(file)) + else: + with open(file, 'r') as f: + texts = f.readlines() + + return texts + + +class AudioText(_Collection): + """List of audio-transcript text correspondence with preprocessing.""" + + OUTPUT_TYPE = collections.namedtuple( + typename='AudioTextEntity', field_names='id audio_file duration text_tokens offset text_raw speaker orig_sr', + ) + + def __init__( + self, + ids: List[int], + audio_files: List[str], + durations: List[float], + texts: List[str], + offsets: List[str], + speakers: List[Optional[int]], + orig_sampling_rates: List[Optional[int]], + parser: parsers.CharParser, + min_duration: Optional[float] = None, + max_duration: Optional[float] = None, + max_number: Optional[int] = None, + do_sort_by_duration: bool = False, + index_by_file_id: bool = False, + ): + """Instantiates audio-text manifest with filters and preprocessing. + + Args: + ids: List of examples positions. + audio_files: List of audio files. + durations: List of float durations. + texts: List of raw text transcripts. + offsets: List of duration offsets or None. + speakers: List of optional speakers ids. + orig_sampling_rates: List of original sampling rates of audio files. + parser: Instance of `CharParser` to convert string to tokens. + min_duration: Minimum duration to keep entry with (default: None). + max_duration: Maximum duration to keep entry with (default: None). + max_number: Maximum number of samples to collect. + do_sort_by_duration: True if sort samples list by duration. Not compatible with index_by_file_id. + index_by_file_id: If True, saves a mapping from filename base (ID) to index in data. + """ + + output_type = self.OUTPUT_TYPE + data, duration_filtered, num_filtered, total_duration = [], 0.0, 0, 0.0 + if index_by_file_id: + self.mapping = {} + + for id_, audio_file, duration, offset, text, speaker, orig_sr in zip( + ids, audio_files, durations, offsets, texts, speakers, orig_sampling_rates + ): + # Duration filters. + if min_duration is not None and duration < min_duration: + duration_filtered += duration + num_filtered += 1 + continue + + if max_duration is not None and duration > max_duration: + duration_filtered += duration + num_filtered += 1 + continue + + text_tokens = parser(text) + if text_tokens is None: + duration_filtered += duration + num_filtered += 1 + continue + + total_duration += duration + + data.append(output_type(id_, audio_file, duration, text_tokens, offset, text, speaker, orig_sr)) + if index_by_file_id: + file_id, _ = os.path.splitext(os.path.basename(audio_file)) + self.mapping[file_id] = len(data) - 1 + + # Max number of entities filter. + if len(data) == max_number: + break + + if do_sort_by_duration: + if index_by_file_id: + logging.warning("Tried to sort dataset by duration, but cannot since index_by_file_id is set.") + else: + data.sort(key=lambda entity: entity.duration) + + logging.info("Dataset loaded with %d files totalling %.2f hours", len(data), total_duration / 3600) + logging.info("%d files were filtered totalling %.2f hours", num_filtered, duration_filtered / 3600) + + super().__init__(data) + + +class ASRAudioText(AudioText): + """`AudioText` collector from asr structured json files.""" + + def __init__(self, manifests_files: Union[str, List[str]], *args, **kwargs): + """Parse lists of audio files, durations and transcripts texts. + + Args: + manifests_files: Either single string file or list of such - + manifests to yield items from. + *args: Args to pass to `AudioText` constructor. + **kwargs: Kwargs to pass to `AudioText` constructor. + """ + + ids, audio_files, durations, texts, offsets, speakers, orig_srs = [], [], [], [], [], [], [] + for item in manifest.item_iter(manifests_files): + ids.append(item['id']) + audio_files.append(item['audio_file']) + durations.append(item['duration']) + texts.append(item['text']) + offsets.append(item['offset']) + speakers.append(item['speaker']) + orig_srs.append(item['orig_sr']) + + super().__init__(ids, audio_files, durations, texts, offsets, speakers, orig_srs, *args, **kwargs) + + +class SpeechLabel(_Collection): + """List of audio-label correspondence with preprocessing.""" + + OUTPUT_TYPE = collections.namedtuple(typename='SpeechLabelEntity', field_names='audio_file duration label offset',) + + def __init__( + self, + audio_files: List[str], + durations: List[float], + labels: List[Union[int, str]], + offsets: List[Optional[float]], + min_duration: Optional[float] = None, + max_duration: Optional[float] = None, + max_number: Optional[int] = None, + do_sort_by_duration: bool = False, + index_by_file_id: bool = False, + ): + """Instantiates audio-label manifest with filters and preprocessing. + + Args: + audio_files: List of audio files. + durations: List of float durations. + labels: List of labels. + offsets: List of offsets or None. + min_duration: Minimum duration to keep entry with (default: None). + max_duration: Maximum duration to keep entry with (default: None). + max_number: Maximum number of samples to collect. + do_sort_by_duration: True if sort samples list by duration. + index_by_file_id: If True, saves a mapping from filename base (ID) to index in data. + """ + + if index_by_file_id: + self.mapping = {} + output_type = self.OUTPUT_TYPE + data, duration_filtered = [], 0.0 + for audio_file, duration, command, offset in zip(audio_files, durations, labels, offsets): + # Duration filters. + if min_duration is not None and duration < min_duration: + duration_filtered += duration + continue + + if max_duration is not None and duration > max_duration: + duration_filtered += duration + continue + + data.append(output_type(audio_file, duration, command, offset)) + + if index_by_file_id: + file_id, _ = os.path.splitext(os.path.basename(audio_file)) + self.mapping[file_id] = len(data) - 1 + + # Max number of entities filter. + if len(data) == max_number: + break + + if do_sort_by_duration: + if index_by_file_id: + logging.warning("Tried to sort dataset by duration, but cannot since index_by_file_id is set.") + else: + data.sort(key=lambda entity: entity.duration) + + logging.info( + "Filtered duration for loading collection is %f.", duration_filtered, + ) + self.uniq_labels = sorted(set(map(lambda x: x.label, data))) + logging.info("# {} files loaded accounting to # {} labels".format(len(data), len(self.uniq_labels))) + + super().__init__(data) + + +class ASRSpeechLabel(SpeechLabel): + """`SpeechLabel` collector from structured json files.""" + + def __init__(self, manifests_files: Union[str, List[str]], is_regression_task=False, *args, **kwargs): + """Parse lists of audio files, durations and transcripts texts. + + Args: + manifests_files: Either single string file or list of such - + manifests to yield items from. + is_regression_task: It's a regression task + *args: Args to pass to `SpeechLabel` constructor. + **kwargs: Kwargs to pass to `SpeechLabel` constructor. + """ + audio_files, durations, labels, offsets = [], [], [], [] + + for item in manifest.item_iter(manifests_files, parse_func=self.__parse_item): + audio_files.append(item['audio_file']) + durations.append(item['duration']) + if not is_regression_task: + labels.append(item['label']) + else: + labels.append(float(item['label'])) + + offsets.append(item['offset']) + + super().__init__(audio_files, durations, labels, offsets, *args, **kwargs) + + def __parse_item(self, line: str, manifest_file: str) -> Dict[str, Any]: + item = json.loads(line) + + # Audio file + if 'audio_filename' in item: + item['audio_file'] = item.pop('audio_filename') + elif 'audio_filepath' in item: + item['audio_file'] = item.pop('audio_filepath') + else: + raise ValueError( + f"Manifest file has invalid json line " f"structure: {line} without proper audio file key." + ) + item['audio_file'] = os.path.expanduser(item['audio_file']) + + # Duration. + if 'duration' not in item: + raise ValueError(f"Manifest file has invalid json line " f"structure: {line} without proper duration key.") + + # Label. + if 'command' in item: + item['label'] = item.pop('command') + elif 'target' in item: + item['label'] = item.pop('target') + elif 'label' in item: + pass + else: + raise ValueError(f"Manifest file has invalid json line " f"structure: {line} without proper label key.") + + item = dict( + audio_file=item['audio_file'], + duration=item['duration'], + label=item['label'], + offset=item.get('offset', None), + ) + + return item + + +class FeatureSequenceLabel(_Collection): + """List of feature sequence of label correspondence with preprocessing.""" + + OUTPUT_TYPE = collections.namedtuple(typename='FeatureSequenceLabelEntity', field_names='feature_file seq_label',) + + def __init__( + self, + feature_files: List[str], + seq_labels: List[str], + max_number: Optional[int] = None, + index_by_file_id: bool = False, + ): + """Instantiates feature-SequenceLabel manifest with filters and preprocessing. + + Args: + feature_files: List of feature files. + seq_labels: List of sequences of abels. + max_number: Maximum number of samples to collect. + index_by_file_id: If True, saves a mapping from filename base (ID) to index in data. + """ + + output_type = self.OUTPUT_TYPE + data, num_filtered = ( + [], + 0.0, + ) + self.uniq_labels = set() + + if index_by_file_id: + self.mapping = {} + + for feature_file, seq_label in zip(feature_files, seq_labels): + + label_tokens, uniq_labels_in_seq = self.relative_speaker_parser(seq_label) + + data.append(output_type(feature_file, label_tokens)) + self.uniq_labels |= uniq_labels_in_seq + + if label_tokens is None: + num_filtered += 1 + continue + + if index_by_file_id: + file_id, _ = os.path.splitext(os.path.basename(feature_file)) + self.mapping[feature_file] = len(data) - 1 + + # Max number of entities filter. + if len(data) == max_number: + break + + logging.info("# {} files loaded including # {} unique labels".format(len(data), len(self.uniq_labels))) + super().__init__(data) + + def relative_speaker_parser(self, seq_label): + """ Convert sequence of speaker labels to relative labels. + Convert sequence of absolute speaker to sequence of relative speaker [E A C A E E C] -> [0 1 2 1 0 0 2] + In this seq of label , if label do not appear before, assign new relative labels len(pos); else reuse previous assigned relative labels. + Args: + seq_label (str): A string of a sequence of labels. + + Return: + relative_seq_label (List) : A list of relative sequence of labels + unique_labels_in_seq (Set): A set of unique labels in the sequence + """ + seq = seq_label.split() + conversion_dict = dict() + relative_seq_label = [] + + for seg in seq: + if seg in conversion_dict: + converted = conversion_dict[seg] + else: + converted = len(conversion_dict) + conversion_dict[seg] = converted + + relative_seq_label.append(converted) + + unique_labels_in_seq = set(conversion_dict.keys()) + return relative_seq_label, unique_labels_in_seq + + +class ASRFeatureSequenceLabel(FeatureSequenceLabel): + """`FeatureSequenceLabel` collector from asr structured json files.""" + + def __init__( + self, manifests_files: Union[str, List[str]], max_number: Optional[int] = None, index_by_file_id: bool = False, + ): + + """Parse lists of feature files and sequences of labels. + + Args: + manifests_files: Either single string file or list of such - + manifests to yield items from. + max_number: Maximum number of samples to collect; pass to `FeatureSequenceLabel` constructor. + index_by_file_id: If True, saves a mapping from filename base (ID) to index in data; pass to `FeatureSequenceLabel` constructor. + """ + + feature_files, seq_labels = [], [] + for item in manifest.item_iter(manifests_files, parse_func=self._parse_item): + feature_files.append(item['feature_file']) + seq_labels.append(item['seq_label']) + + super().__init__(feature_files, seq_labels, max_number, index_by_file_id) + + def _parse_item(self, line: str, manifest_file: str) -> Dict[str, Any]: + item = json.loads(line) + + # Feature file + if 'feature_filename' in item: + item['feature_file'] = item.pop('feature_filename') + elif 'feature_filepath' in item: + item['feature_file'] = item.pop('feature_filepath') + else: + raise ValueError( + f"Manifest file has invalid json line " f"structure: {line} without proper feature file key." + ) + item['feature_file'] = os.path.expanduser(item['feature_file']) + + # Seq of Label. + if 'seq_label' in item: + item['seq_label'] = item.pop('seq_label') + else: + raise ValueError( + f"Manifest file has invalid json line " f"structure: {line} without proper seq_label key." + ) + + item = dict(feature_file=item['feature_file'], seq_label=item['seq_label'],) + + return item diff --git a/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/manifest.py b/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/manifest.py new file mode 100644 index 0000000000..8c84d8b56d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/manifest.py @@ -0,0 +1,120 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from os.path import expanduser +from typing import Any, Callable, Dict, Iterator, List, Optional, Union + + +class ManifestBase: + def __init__(self, *args, **kwargs): + raise ValueError( + "This class is deprecated, look at https://github.com/NVIDIA/NeMo/pull/284 for correct behaviour." + ) + + +class ManifestEN: + def __init__(self, *args, **kwargs): + raise ValueError( + "This class is deprecated, look at https://github.com/NVIDIA/NeMo/pull/284 for correct behaviour." + ) + + +def item_iter( + manifests_files: Union[str, List[str]], parse_func: Callable[[str, Optional[str]], Dict[str, Any]] = None +) -> Iterator[Dict[str, Any]]: + """Iterate through json lines of provided manifests. + + NeMo ASR pipelines often assume certain manifest files structure. In + particular, each manifest file should consist of line-per-sample files with + each line being correct json dict. Each such json dict should have a field + for audio file string, a field for duration float and a field for text + string. Offset also could be additional field and is set to None by + default. + + Args: + manifests_files: Either single string file or list of such - + manifests to yield items from. + + parse_func: A callable function which accepts as input a single line + of a manifest and optionally the manifest file itself, + and parses it, returning a dictionary mapping from str -> Any. + + Yields: + Parsed key to value item dicts. + + Raises: + ValueError: If met invalid json line structure. + """ + + if isinstance(manifests_files, str): + manifests_files = [manifests_files] + + if parse_func is None: + parse_func = __parse_item + + k = -1 + for manifest_file in manifests_files: + with open(expanduser(manifest_file), 'r') as f: + for line in f: + k += 1 + item = parse_func(line, manifest_file) + item['id'] = k + + yield item + + +def __parse_item(line: str, manifest_file: str) -> Dict[str, Any]: + item = json.loads(line) + + # Audio file + if 'audio_filename' in item: + item['audio_file'] = item.pop('audio_filename') + elif 'audio_filepath' in item: + item['audio_file'] = item.pop('audio_filepath') + else: + raise ValueError( + f"Manifest file {manifest_file} has invalid json line structure: {line} without proper audio file key." + ) + item['audio_file'] = expanduser(item['audio_file']) + + # Duration. + if 'duration' not in item: + raise ValueError( + f"Manifest file {manifest_file} has invalid json line structure: {line} without proper duration key." + ) + + # Text. + if 'text' in item: + pass + elif 'text_filepath' in item: + with open(item.pop('text_filepath'), 'r') as f: + item['text'] = f.read().replace('\n', '') + elif 'normalized_text' in item: + item['text'] = item['normalized_text'] + else: + raise ValueError( + f"Manifest file {manifest_file} has invalid json line structure: {line} without proper text key." + ) + + item = dict( + audio_file=item['audio_file'], + duration=item['duration'], + text=item['text'], + offset=item.get('offset', None), + speaker=item.get('speaker', None), + orig_sr=item.get('orig_sample_rate', None), + ) + + return item diff --git a/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/parsers.py b/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/parsers.py new file mode 100644 index 0000000000..749ca6c70a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/parts/preprocessing/parsers.py @@ -0,0 +1,185 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import string +from typing import List, Optional + +import frozendict + +from nemo.collections.common.parts.preprocessing import cleaners + + +class CharParser: + """Functor for parsing raw strings into list of int tokens. + + Examples: + >>> parser = CharParser(['a', 'b', 'c']) + >>> parser('abc') + [0, 1, 2] + """ + + def __init__( + self, + labels: List[str], + *, + unk_id: int = -1, + blank_id: int = -1, + do_normalize: bool = True, + do_lowercase: bool = True, + ): + """Creates simple mapping char parser. + + Args: + labels: List of labels to allocate indexes for. Essentially, + this is a id to str mapping. + unk_id: Index to choose for OOV words (default: -1). + blank_id: Index to filter out from final list of tokens + (default: -1). + do_normalize: True if apply normalization step before tokenizing + (default: True). + do_lowercase: True if apply lowercasing at normalizing step + (default: True). + """ + + self._labels = labels + self._unk_id = unk_id + self._blank_id = blank_id + self._do_normalize = do_normalize + self._do_lowercase = do_lowercase + + self._labels_map = {label: index for index, label in enumerate(labels)} + self._special_labels = set([label for label in labels if len(label) > 1]) + + def __call__(self, text: str) -> Optional[List[int]]: + if self._do_normalize: + text = self._normalize(text) + if text is None: + return None + + text_tokens = self._tokenize(text) + + return text_tokens + + def _normalize(self, text: str) -> Optional[str]: + text = text.strip() + + if self._do_lowercase: + text = text.lower() + + return text + + def _tokenize(self, text: str) -> List[int]: + tokens = [] + # Split by word for find special labels. + for word_id, word in enumerate(text.split(' ')): + if word_id != 0: # Not first word - so we insert space before. + tokens.append(self._labels_map.get(' ', self._unk_id)) + + if word in self._special_labels: + tokens.append(self._labels_map[word]) + continue + + for char in word: + tokens.append(self._labels_map.get(char, self._unk_id)) + + # If unk_id == blank_id, OOV tokens are removed. + tokens = [token for token in tokens if token != self._blank_id] + + return tokens + + +class ENCharParser(CharParser): + """Incorporates english-specific parsing logic.""" + + PUNCTUATION_TO_REPLACE = frozendict.frozendict({'+': 'plus', '&': 'and', '%': 'percent'}) + + def __init__(self, abbreviation_version=None, make_table=True, *args, **kwargs): + """Creates english-specific mapping char parser. + + This class overrides normalizing implementation. + + Args: + *args: Positional args to pass to `CharParser` constructor. + **kwargs: Key-value args to pass to `CharParser` constructor. + """ + + super().__init__(*args, **kwargs) + + self._table = None + if make_table: + self._table = self.__make_trans_table() + self.abbreviation_version = abbreviation_version + + def __make_trans_table(self): + punctuation = string.punctuation + + for char in self.PUNCTUATION_TO_REPLACE: + punctuation = punctuation.replace(char, '') + + for label in self._labels: + punctuation = punctuation.replace(label, '') + + table = str.maketrans(punctuation, ' ' * len(punctuation)) + + return table + + def _normalize(self, text: str) -> Optional[str]: + # noinspection PyBroadException + try: + text = cleaners.clean_text( + string=text, + table=self._table, + punctuation_to_replace=self.PUNCTUATION_TO_REPLACE, + abbreviation_version=self.abbreviation_version, + ) + except Exception: + return None + + return text + + +NAME_TO_PARSER = frozendict.frozendict({'base': CharParser, 'en': ENCharParser}) + + +def make_parser(labels: Optional[List[str]] = None, name: str = 'base', **kwargs,) -> CharParser: + """Creates parser from labels, set of arguments and concise parser name. + + Args: + labels: List of labels to allocate indexes for. If set to + None then labels would be ascii table list. Essentially, this is a + id to str mapping (default: None). + name: Concise name of parser to create (default: 'base'). + (default: -1). + **kwargs: Other set of kwargs to pass to parser constructor. + + Returns: + Instance of `CharParser`. + + Raises: + ValueError: For invalid parser name. + + Examples: + >>> type(make_parser(['a', 'b', 'c'], 'en')) + ENCharParser + """ + + if name not in NAME_TO_PARSER: + raise ValueError('Invalid parser name.') + + if labels is None: + labels = list(string.printable) + + parser_type = NAME_TO_PARSER[name] + parser = parser_type(labels=labels, **kwargs) + + return parser diff --git a/models/tacotron2/NeMo/nemo/collections/common/parts/rnn.py b/models/tacotron2/NeMo/nemo/collections/common/parts/rnn.py new file mode 100644 index 0000000000..81e30778d8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/parts/rnn.py @@ -0,0 +1,553 @@ +# Copyright (c) 2019, Myrtle Software Limited. All rights reserved. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from nemo.utils import logging + + +def rnn( + input_size: int, + hidden_size: int, + num_layers: int, + norm: Optional[str] = None, + forget_gate_bias: Optional[float] = 1.0, + dropout: Optional[float] = 0.0, + norm_first_rnn: Optional[bool] = None, + t_max: Optional[int] = None, + weights_init_scale: float = 1.0, + hidden_hidden_bias_scale: float = 0.0, +) -> torch.nn.Module: + """ + Utility function to provide unified interface to common LSTM RNN modules. + + Args: + input_size: Input dimension. + + hidden_size: Hidden dimension of the RNN. + + num_layers: Number of RNN layers. + + norm: Optional string representing type of normalization to apply to the RNN. + Supported values are None, batch and layer. + + forget_gate_bias: float, set by default to 1.0, which constructs a forget gate + initialized to 1.0. + Reference: + [An Empirical Exploration of Recurrent Network Architectures](http://proceedings.mlr.press/v37/jozefowicz15.pdf) + + dropout: Optional dropout to apply to end of multi-layered RNN. + + norm_first_rnn: Whether to normalize the first RNN layer. + + t_max: int value, set to None by default. If an int is specified, performs Chrono Initialization + of the LSTM network, based on the maximum number of timesteps `t_max` expected during the course + of training. + Reference: + [Can recurrent neural networks warp time?](https://openreview.net/forum?id=SJcKhk-Ab) + + weights_init_scale: Float scale of the weights after initialization. Setting to lower than one + sometimes helps reduce variance between runs. + + hidden_hidden_bias_scale: Float scale for the hidden-to-hidden bias scale. Set to 0.0 for + the default behaviour. + + Returns: + A RNN module + """ + if norm not in [None, "batch", "layer"]: + raise ValueError(f"unknown norm={norm}") + + if norm is None: + return LSTMDropout( + input_size=input_size, + hidden_size=hidden_size, + num_layers=num_layers, + dropout=dropout, + forget_gate_bias=forget_gate_bias, + t_max=t_max, + weights_init_scale=weights_init_scale, + hidden_hidden_bias_scale=hidden_hidden_bias_scale, + ) + + if norm == "batch": + return BNRNNSum( + input_size=input_size, + hidden_size=hidden_size, + rnn_layers=num_layers, + batch_norm=True, + dropout=dropout, + forget_gate_bias=forget_gate_bias, + t_max=t_max, + norm_first_rnn=norm_first_rnn, + weights_init_scale=weights_init_scale, + hidden_hidden_bias_scale=hidden_hidden_bias_scale, + ) + + if norm == "layer": + return torch.jit.script( + ln_lstm( # torch.jit.script( + input_size=input_size, + hidden_size=hidden_size, + num_layers=num_layers, + dropout=dropout, + forget_gate_bias=forget_gate_bias, + t_max=t_max, + weights_init_scale=weights_init_scale, + hidden_hidden_bias_scale=hidden_hidden_bias_scale, + ) + ) + + +class OverLastDim(torch.nn.Module): + """Collapses a tensor to 2D, applies a module, and (re-)expands the tensor. + An n-dimensional tensor of shape (s_1, s_2, ..., s_n) is first collapsed to + a tensor with shape (s_1*s_2*...*s_n-1, s_n). The module is called with + this as input producing (s_1*s_2*...*s_n-1, s_n') --- note that the final + dimension can change. This is expanded to (s_1, s_2, ..., s_n-1, s_n') and + returned. + Args: + module (torch.nn.Module): Module to apply. Must accept a 2D tensor as + input and produce a 2D tensor as output, optionally changing the + size of the last dimension. + """ + + def __init__(self, module: torch.nn.Module): + super().__init__() + self.module = module + + def forward(self, x: torch.Tensor) -> torch.Tensor: + *dims, _ = x.size() + + reduced_dims = 1 + for dim in dims: + reduced_dims *= dim + + x = x.view(reduced_dims, -1) + x = self.module(x) + x = x.view(*dims, -1) + return x + + +class LSTMDropout(torch.nn.Module): + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int, + dropout: Optional[float], + forget_gate_bias: Optional[float], + t_max: Optional[int] = None, + weights_init_scale: float = 1.0, + hidden_hidden_bias_scale: float = 0.0, + ): + """Returns an LSTM with forget gate bias init to `forget_gate_bias`. + Args: + input_size: See `torch.nn.LSTM`. + hidden_size: See `torch.nn.LSTM`. + num_layers: See `torch.nn.LSTM`. + dropout: See `torch.nn.LSTM`. + + forget_gate_bias: float, set by default to 1.0, which constructs a forget gate + initialized to 1.0. + Reference: + [An Empirical Exploration of Recurrent Network Architectures](http://proceedings.mlr.press/v37/jozefowicz15.pdf) + + t_max: int value, set to None by default. If an int is specified, performs Chrono Initialization + of the LSTM network, based on the maximum number of timesteps `t_max` expected during the course + of training. + Reference: + [Can recurrent neural networks warp time?](https://openreview.net/forum?id=SJcKhk-Ab) + + weights_init_scale: Float scale of the weights after initialization. Setting to lower than one + sometimes helps reduce variance between runs. + + hidden_hidden_bias_scale: Float scale for the hidden-to-hidden bias scale. Set to 0.0 for + the default behaviour. + + Returns: + A `torch.nn.LSTM`. + """ + super(LSTMDropout, self).__init__() + + self.lstm = torch.nn.LSTM( + input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, dropout=dropout, + ) + + if t_max is not None: + # apply chrono init + for name, v in self.lstm.named_parameters(): + if 'bias' in name: + p = getattr(self.lstm, name) + n = p.nelement() + hidden_size = n // 4 + p.data.fill_(0) + p.data[hidden_size : 2 * hidden_size] = torch.log( + torch.nn.init.uniform_(p.data[0:hidden_size], 1, t_max - 1) + ) + # forget gate biases = log(uniform(1, Tmax-1)) + p.data[0:hidden_size] = -p.data[hidden_size : 2 * hidden_size] + # input gate biases = -(forget gate biases) + + elif forget_gate_bias is not None: + for name, v in self.lstm.named_parameters(): + if "bias_ih" in name: + bias = getattr(self.lstm, name) + bias.data[hidden_size : 2 * hidden_size].fill_(forget_gate_bias) + if "bias_hh" in name: + bias = getattr(self.lstm, name) + bias.data[hidden_size : 2 * hidden_size] *= float(hidden_hidden_bias_scale) + + self.dropout = torch.nn.Dropout(dropout) if dropout else None + + for name, v in self.named_parameters(): + if 'weight' in name or 'bias' in name: + v.data *= float(weights_init_scale) + + def forward( + self, x: torch.Tensor, h: Optional[Tuple[torch.Tensor, torch.Tensor]] = None + ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + x, h = self.lstm(x, h) + + if self.dropout: + x = self.dropout(x) + + return x, h + + +class RNNLayer(torch.nn.Module): + """A single RNNLayer with optional batch norm.""" + + def __init__( + self, + input_size: int, + hidden_size: int, + rnn_type: torch.nn.Module = torch.nn.LSTM, + batch_norm: bool = True, + forget_gate_bias: Optional[float] = 1.0, + t_max: Optional[int] = None, + weights_init_scale: float = 1.0, + hidden_hidden_bias_scale: float = 0.0, + ): + super().__init__() + + if batch_norm: + self.bn = OverLastDim(torch.nn.BatchNorm1d(input_size)) + + if isinstance(rnn_type, torch.nn.LSTM) and not batch_norm: + # batch_norm will apply bias, no need to add a second to LSTM + self.rnn = LSTMDropout( + input_size=input_size, + hidden_size=hidden_size, + num_layers=1, + dropout=0.0, + forget_gate_bias=forget_gate_bias, + t_max=t_max, + weights_init_scale=weights_init_scale, + hidden_hidden_bias_scale=hidden_hidden_bias_scale, + ) + else: + self.rnn = rnn_type(input_size=input_size, hidden_size=hidden_size, bias=not batch_norm) + + def forward( + self, x: torch.Tensor, hx: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None + ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if hasattr(self, 'bn'): + x = x.contiguous() + x = self.bn(x) + x, h = self.rnn(x, hx=hx) + return x, h + + def _flatten_parameters(self): + self.rnn.flatten_parameters() + + +class BNRNNSum(torch.nn.Module): + """RNN wrapper with optional batch norm. + Instantiates an RNN. If it is an LSTM it initialises the forget gate + bias =`lstm_gate_bias`. Optionally applies a batch normalisation layer to + the input with the statistics computed over all time steps. If dropout > 0 + then it is applied to all layer outputs except the last. + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + rnn_type: torch.nn.Module = torch.nn.LSTM, + rnn_layers: int = 1, + batch_norm: bool = True, + dropout: Optional[float] = 0.0, + forget_gate_bias: Optional[float] = 1.0, + norm_first_rnn: bool = False, + t_max: Optional[int] = None, + weights_init_scale: float = 1.0, + hidden_hidden_bias_scale: float = 0.0, + ): + super().__init__() + self.rnn_layers = rnn_layers + + self.layers = torch.nn.ModuleList() + for i in range(rnn_layers): + final_layer = (rnn_layers - 1) == i + + self.layers.append( + RNNLayer( + input_size, + hidden_size, + rnn_type=rnn_type, + batch_norm=batch_norm and (norm_first_rnn or i > 0), + forget_gate_bias=forget_gate_bias, + t_max=t_max, + weights_init_scale=weights_init_scale, + hidden_hidden_bias_scale=hidden_hidden_bias_scale, + ) + ) + + if dropout is not None and dropout > 0.0 and not final_layer: + self.layers.append(torch.nn.Dropout(dropout)) + + input_size = hidden_size + + def forward( + self, x: torch.Tensor, hx: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None + ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + hx = self._parse_hidden_state(hx) + + hs = [] + cs = [] + rnn_idx = 0 + for layer in self.layers: + if isinstance(layer, torch.nn.Dropout): + x = layer(x) + else: + x, h_out = layer(x, hx=hx[rnn_idx]) + hs.append(h_out[0]) + cs.append(h_out[1]) + rnn_idx += 1 + del h_out + + h_0 = torch.stack(hs, dim=0) + c_0 = torch.stack(cs, dim=0) + return x, (h_0, c_0) + + def _parse_hidden_state( + self, hx: Optional[Tuple[torch.Tensor, torch.Tensor]] + ) -> Union[List[None], List[Tuple[torch.Tensor, torch.Tensor]]]: + """ + Dealing w. hidden state: + Typically in pytorch: (h_0, c_0) + h_0 = ``[num_layers * num_directions, batch, hidden_size]`` + c_0 = ``[num_layers * num_directions, batch, hidden_size]`` + """ + if hx is None: + return [None] * self.rnn_layers + else: + h_0, c_0 = hx + + if h_0.shape[0] != self.rnn_layers: + raise ValueError( + 'Provided initial state value `h_0` must be of shape : ' + '[num_layers * num_directions, batch, hidden_size]' + ) + + return [(h_0[i], c_0[i]) for i in range(h_0.shape[0])] + + def _flatten_parameters(self): + for layer in self.layers: + if isinstance(layer, (torch.nn.LSTM, torch.nn.GRU, torch.nn.RNN)): + layer._flatten_parameters() + + +class StackTime(torch.nn.Module): + """ + Stacks time within the feature dim, so as to behave as a downsampling operation. + """ + + def __init__(self, factor: int): + super().__init__() + self.factor = int(factor) + + def forward(self, x: List[Tuple[torch.Tensor]]) -> (torch.Tensor, torch.Tensor): + # T, B, U + x, x_lens = x + seq = [x] + for i in range(1, self.factor): + tmp = torch.zeros_like(x) + tmp[:-i, :, :] = x[i:, :, :] + seq.append(tmp) + x_lens = torch.ceil(x_lens.float() / self.factor).int() + return torch.cat(seq, dim=2)[:: self.factor, :, :], x_lens + + +def ln_lstm( + input_size: int, + hidden_size: int, + num_layers: int, + dropout: Optional[float], + forget_gate_bias: Optional[float], + t_max: Optional[int], + weights_init_scale: Optional[float] = None, # ignored + hidden_hidden_bias_scale: Optional[float] = None, # ignored +) -> torch.nn.Module: + """Returns a ScriptModule that mimics a PyTorch native LSTM.""" + # The following are not implemented. + if dropout is not None and dropout != 0.0: + raise ValueError('`dropout` not supported with LayerNormLSTM') + + if t_max is not None: + logging.warning("LayerNormLSTM does not support chrono init via `t_max`") + + if weights_init_scale is not None: + logging.warning("`weights_init_scale` is ignored for LayerNormLSTM") + + if hidden_hidden_bias_scale is not None: + logging.warning("`hidden_hidden_bias_scale` is ignored for LayerNormLSTM") + + return StackedLSTM( + num_layers, + LSTMLayer, + first_layer_args=[LayerNormLSTMCell, input_size, hidden_size, forget_gate_bias], + other_layer_args=[LayerNormLSTMCell, hidden_size, hidden_size, forget_gate_bias], + ) + + +class LSTMLayer(torch.nn.Module): + def __init__(self, cell, *cell_args): + super(LSTMLayer, self).__init__() + self.cell = cell(*cell_args) + + def forward( + self, input: torch.Tensor, state: Tuple[torch.Tensor, torch.Tensor] + ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + inputs = input.unbind(0) + outputs = [] + for i in range(len(inputs)): + out, state = self.cell(inputs[i], state) + outputs += [out] + return torch.stack(outputs), state + + +class LayerNormLSTMCell(torch.nn.Module): + def __init__(self, input_size, hidden_size, forget_gate_bias): + super().__init__() + self.input_size = input_size + self.hidden_size = hidden_size + self.weight_ih = torch.nn.Parameter(torch.randn(4 * hidden_size, input_size)) + self.weight_hh = torch.nn.Parameter(torch.randn(4 * hidden_size, hidden_size)) + + # LayerNorm provide learnable biases + self.layernorm_i = torch.nn.LayerNorm(4 * hidden_size) + self.layernorm_h = torch.nn.LayerNorm(4 * hidden_size) + self.layernorm_c = torch.nn.LayerNorm(hidden_size) + + self.reset_parameters() + + self.layernorm_i.bias.data[hidden_size : 2 * hidden_size].fill_(0.0) + self.layernorm_h.bias.data[hidden_size : 2 * hidden_size].fill_(forget_gate_bias) + + def reset_parameters(self): + stdv = 1.0 / math.sqrt(self.hidden_size) + for weight in self.parameters(): + torch.nn.init.uniform_(weight, -stdv, stdv) + + def forward( + self, input: torch.Tensor, state: Tuple[torch.Tensor, torch.Tensor] + ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + hx, cx = state + igates = self.layernorm_i(torch.mm(input, self.weight_ih.t())) + hgates = self.layernorm_h(torch.mm(hx, self.weight_hh.t())) + gates = igates + hgates + ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1) + + ingate = torch.sigmoid(ingate) + forgetgate = torch.sigmoid(forgetgate) + cellgate = torch.tanh(cellgate) + outgate = torch.sigmoid(outgate) + + cy = self.layernorm_c((forgetgate * cx) + (ingate * cellgate)) + hy = outgate * torch.tanh(cy) + + return hy, (hy, cy) + + +def init_stacked_lstm( + num_layers: int, layer: torch.nn.Module, first_layer_args: List, other_layer_args: List +) -> torch.nn.ModuleList: + layers = [layer(*first_layer_args)] + [layer(*other_layer_args) for _ in range(num_layers - 1)] + return torch.nn.ModuleList(layers) + + +class StackedLSTM(torch.nn.Module): + def __init__(self, num_layers: int, layer: torch.nn.Module, first_layer_args: List, other_layer_args: List): + super(StackedLSTM, self).__init__() + self.layers: torch.nn.ModuleList = init_stacked_lstm(num_layers, layer, first_layer_args, other_layer_args) + + def forward( + self, input: torch.Tensor, states: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] + ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: + if states is None: + temp_states: List[Tuple[torch.Tensor, torch.Tensor]] = [] + batch = input.size(1) + for layer in self.layers: + temp_states.append( + ( + torch.zeros(batch, layer.cell.hidden_size, dtype=input.dtype, device=input.device), + torch.zeros(batch, layer.cell.hidden_size, dtype=input.dtype, device=input.device), + ) + ) + + states = temp_states + + output_states: List[Tuple[torch.Tensor, torch.Tensor]] = [] + output = input + for i, rnn_layer in enumerate(self.layers): + state = states[i] + output, out_state = rnn_layer(output, state) + output_states.append(out_state) + i += 1 + return output, output_states + + +def label_collate(labels, device=None): + """Collates the label inputs for the rnn-t prediction network. + If `labels` is already in torch.Tensor form this is a no-op. + + Args: + labels: A torch.Tensor List of label indexes or a torch.Tensor. + device: Optional torch device to place the label on. + + Returns: + A padded torch.Tensor of shape (batch, max_seq_len). + """ + + if isinstance(labels, torch.Tensor): + return labels.type(torch.int64) + if not isinstance(labels, (list, tuple)): + raise ValueError(f"`labels` should be a list or tensor not {type(labels)}") + + batch_size = len(labels) + max_len = max(len(label) for label in labels) + + cat_labels = np.full((batch_size, max_len), fill_value=0.0, dtype=np.int32) + for e, l in enumerate(labels): + cat_labels[e, : len(l)] = l + labels = torch.tensor(cat_labels, dtype=torch.int64, device=device) + + return labels diff --git a/models/tacotron2/NeMo/nemo/collections/common/parts/transformer_utils.py b/models/tacotron2/NeMo/nemo/collections/common/parts/transformer_utils.py new file mode 100644 index 0000000000..a40e7eebc9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/parts/transformer_utils.py @@ -0,0 +1,79 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn + +__all__ = ['NEG_INF', 'form_attention_mask', 'transformer_weights_init', 'mask_padded_tokens'] + +NEG_INF = -10000.0 + + +def form_attention_mask(input_mask, diagonal=None): + """ + Build attention mask with optional masking of future tokens we forbid + to attend to (e.g. as it is in Transformer decoder). + + Args: + input_mask: binary mask of size B x L with 1s corresponding to valid + tokens and 0s corresponding to padding tokens + diagonal: diagonal where triangular future mask starts + None -- do not mask anything + 0 -- regular translation or language modeling future masking + 1 -- query stream masking as in XLNet architecture + Returns: + attention_mask: mask of size B x 1 x L x L with 0s corresponding to + tokens we plan to attend to and -10000 otherwise + """ + + if input_mask is None: + return None + attn_shape = (1, input_mask.shape[1], input_mask.shape[1]) + attn_mask = input_mask.to(dtype=bool).unsqueeze(1) + if diagonal is not None: + future_mask = torch.tril(torch.ones(attn_shape, dtype=torch.bool, device=input_mask.device), diagonal) + attn_mask = attn_mask & future_mask + attention_mask = (1 - attn_mask.to(torch.float)) * NEG_INF + return attention_mask.unsqueeze(1) + + +def transformer_weights_init(module, std_init_range=0.02, xavier=True): + """ + Initialize different weights in Transformer model. + + Args: + module: torch.nn.Module to be initialized + std_init_range: standard deviation of normal initializer + xavier: if True, xavier initializer will be used in Linear layers + as was proposed in AIAYN paper, otherwise normal initializer + will be used (like in BERT paper) + """ + + if isinstance(module, nn.Linear): + if xavier: + nn.init.xavier_uniform_(module.weight) + else: + nn.init.normal_(module.weight, mean=0.0, std=std_init_range) + if module.bias is not None: + nn.init.constant_(module.bias, 0.0) + elif isinstance(module, nn.Embedding): + nn.init.normal_(module.weight, mean=0.0, std=std_init_range) + elif isinstance(module, nn.LayerNorm): + nn.init.constant_(module.weight, 1.0) + nn.init.constant_(module.bias, 0.0) + + +def mask_padded_tokens(tokens, pad_id): + mask = tokens != pad_id + return mask diff --git a/models/tacotron2/NeMo/nemo/collections/common/parts/utils.py b/models/tacotron2/NeMo/nemo/collections/common/parts/utils.py new file mode 100644 index 0000000000..0542e5dce9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/parts/utils.py @@ -0,0 +1,57 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import os +from typing import List + +__all__ = ['if_exist', '_compute_softmax'] + + +def if_exist(outfold: str, files: List[str]): + """ + Returns true if all given files exist in the given folder + Args: + outfold: folder path + files: list of file names relative to outfold + """ + if not os.path.exists(outfold): + return False + for file in files: + if not os.path.exists(f'{outfold}/{file}'): + return False + return True + + +def _compute_softmax(scores): + """Compute softmax probability over raw logits.""" + if not scores: + return [] + + max_score = None + for score in scores: + if max_score is None or score > max_score: + max_score = score + + exp_scores = [] + total_sum = 0.0 + for score in scores: + x = math.exp(score - max_score) + exp_scores.append(x) + total_sum += x + + probs = [] + for score in exp_scores: + probs.append(score / total_sum) + return probs diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/__init__.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/__init__.py new file mode 100644 index 0000000000..ae6f0950d6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.common.tokenizers.char_tokenizer import CharTokenizer +from nemo.collections.common.tokenizers.huggingface.auto_tokenizer import AutoTokenizer +from nemo.collections.common.tokenizers.sentencepiece_tokenizer import SentencePieceTokenizer +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec +from nemo.collections.common.tokenizers.word_tokenizer import WordTokenizer diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/char_tokenizer.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/char_tokenizer.py new file mode 100644 index 0000000000..a9dc8b2ad2 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/char_tokenizer.py @@ -0,0 +1,142 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec + +__all__ = ['CharTokenizer'] + + +class CharTokenizer(TokenizerSpec): + "Tokenizes each character" + + def __init__( + self, + vocab_file: str, + mask_token: Optional[str] = None, + bos_token: Optional[str] = None, + eos_token: Optional[str] = None, + pad_token: Optional[str] = None, + sep_token: Optional[str] = None, + cls_token: Optional[str] = None, + unk_token: Optional[str] = None, + ): + """ + Args: + vocab_file: path to file with vocabulary which consists + of characters separated by \n + mask_token: mask token + bos_token: the beginning of sequence token + eos_token: the end of sequence token. Usually equal to sep_token + pad_token: token to use for padding + sep_token: token used for separating sequences + cls_token: class token. Usually equal to bos_token + unk_token: token to use for unknown tokens + """ + + vocab_list = open(vocab_file, "r").readlines() + self.vocab = {vocab_list[i].strip(): i for i in range(len(vocab_list))} + + special_tokens_dict = {} + if unk_token: + special_tokens_dict["unk_token"] = unk_token + if sep_token: + special_tokens_dict["sep_token"] = sep_token + if mask_token: + special_tokens_dict["mask_token"] = mask_token + if bos_token: + special_tokens_dict["bos_token"] = bos_token + if eos_token: + special_tokens_dict["eos_token"] = eos_token + if pad_token: + special_tokens_dict["pad_token"] = pad_token + if cls_token: + special_tokens_dict["cls_token"] = cls_token + + self.add_special_tokens(special_tokens_dict) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.vocab_size = len(self.vocab) + self.special_tokens = self.tokens_to_ids(special_tokens_dict.values()) + + def add_special_tokens(self, special_tokens_dict: dict) -> int: + """ + Adds a dictionary of special tokens (eos, pad, cls...). + If special tokens are NOT in the vocabulary, they are added + to it (indexed starting from the last index of the current vocabulary). + Args: + special_tokens_dict: dict of special tokens + """ + for token in special_tokens_dict: + token_str = special_tokens_dict[token] + if token_str not in self.vocab: + self.vocab[token_str] = len(self.vocab) + setattr(self, token, token_str) + + def text_to_tokens(self, text): + token_candidates = [char for char in text] + tokens = [] + for token in token_candidates: + if token in self.vocab: + tokens.append(token) + else: + tokens.append(self.unk_token) + return tokens + + def tokens_to_text(self, tokens): + return self.ids_to_text(self.tokens_to_ids(tokens)) + + def text_to_ids(self, text): + return [self.vocab[token] for token in self.text_to_tokens(text)] + + def ids_to_text(self, ids): + ids_ = [id_ for id_ in ids if id_ not in self.special_tokens] + return "".join(self.ids_to_tokens(ids_)) + + def tokens_to_ids(self, tokens): + return [self.vocab[token] for token in tokens] + + def token_to_id(self, token): + return self.vocab[token] + + def ids_to_tokens(self, ids): + return [self.inv_vocab[id] for id in ids] + + @property + def pad_id(self): + return self.vocab[self.pad_token] + + @property + def bos_id(self): + return self.vocab[self.bos_token] + + @property + def eos_id(self): + return self.vocab[self.eos_token] + + @property + def unk_id(self): + return self.vocab[self.unk_token] + + @property + def mask_id(self): + return self.vocab[self.mask_token] + + @property + def sep_id(self): + return self.vocab[self.sep_token] + + @property + def cls_id(self): + return self.vocab[self.cls_token] diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/chinese_tokenizers.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/chinese_tokenizers.py new file mode 100644 index 0000000000..a9923f0a4f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/chinese_tokenizers.py @@ -0,0 +1,63 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The MIT License (MIT) +# Copyright (c) 2016 The-Orizon +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + +# The detokenize function is based on : https://github.com/The-Orizon/nlputils/blob/master/detokenizer.py + +import re +from typing import List + +import jieba +import opencc +from pangu import spacing + + +class ChineseProcessor: + """ + Tokenizer, Detokenizer and Normalizer utilities for Chinese. + """ + + def __init__(self): + self.normalizer = opencc.OpenCC('t2s.json') + + def normalize(self, text: str) -> str: + return self.normalizer.convert(text) + + def detokenize(self, text: List[str]) -> str: + RE_WS_IN_FW = re.compile( + r'([\u2018\u2019\u201c\u201d\u2e80-\u312f\u3200-\u32ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff00-\uffef])\s+(?=[\u2018\u2019\u201c\u201d\u2e80-\u312f\u3200-\u32ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff00-\uffef])' + ) + + detokenize = lambda s: spacing(RE_WS_IN_FW.sub(r'\1', s)).strip() + return detokenize(' '.join(text)) + + def tokenize(self, text: str) -> str: + text = jieba.cut(text) + return ' '.join(text) diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/en_ja_tokenizers.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/en_ja_tokenizers.py new file mode 100644 index 0000000000..bf5287bab3 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/en_ja_tokenizers.py @@ -0,0 +1,57 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List + +from sacremoses import MosesDetokenizer, MosesPunctNormalizer, MosesTokenizer + + +class EnJaProcessor: + """ + Tokenizer, Detokenizer and Normalizer utilities for Japanese & English + Args: + lang_id: One of ['en', 'ja']. + """ + + def __init__(self, lang_id: str): + self.lang_id = lang_id + self.moses_tokenizer = MosesTokenizer(lang=lang_id) + self.moses_detokenizer = MosesDetokenizer(lang=lang_id) + self.normalizer = MosesPunctNormalizer( + lang=lang_id, pre_replace_unicode_punct=True, post_remove_control_chars=True + ) + + def detokenize(self, tokens: List[str]) -> str: + """ + Detokenizes a list of tokens + Args: + tokens: list of strings as tokens + Returns: + detokenized Japanese or English string + """ + return self.moses_detokenizer.detokenize(tokens) + + def tokenize(self, text) -> str: + """ + Tokenizes text using Moses. Returns a string of tokens. + """ + tokens = self.moses_tokenizer.tokenize(text) + return ' '.join(tokens) + + def normalize(self, text) -> str: + # Normalization doesn't handle Japanese periods correctly; + # '。'becomes '.'. + if self.lang_id == 'en': + return self.normalizer.normalize(text) + else: + return text diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/fairseq_tokenizer.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/fairseq_tokenizer.py new file mode 100644 index 0000000000..aa30bacd8f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/fairseq_tokenizer.py @@ -0,0 +1,126 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright (c) 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the LICENSE file in +# the root directory of this source tree. An additional grant of patent rights +# can be found in the PATENTS file in the same directory. + +""" Code from +https://github.com/NVIDIA/DeepLearningExamples/blob/ +master/PyTorch/Translation/Transformer/fairseq/tokenizer.py +""" + +import re +import sys +import unicodedata +from collections import defaultdict + +__all__ = ['get_unicode_categories', 'tokenize_en'] + + +def get_unicode_categories(): + cats = defaultdict(list) + for c in map(chr, range(sys.maxunicode + 1)): + cats[unicodedata.category(c)].append(c) + return cats + + +NUMERICS = ''.join(get_unicode_categories()['No']) + + +def tokenize_en(line): + line = line.strip() + line = ' ' + line + ' ' + # remove ASCII junk + line = re.sub(r'\s+', ' ', line) + line = re.sub(r'[\x00-\x1F]', '', line) + # fix whitespaces + line = re.sub(r'\ +', ' ', line) + line = re.sub('^ ', '', line) + line = re.sub(' $', '', line) + # separate other special characters + line = re.sub(r'([^\s\.\'\`\,\-\w]|[_' + NUMERICS + '])', r' \g<1> ', line) + line = re.sub(r'(\w)\-(?=\w)', r'\g<1> @-@ ', line) + + # multidots stay together + line = re.sub(r'\.([\.]+)', r' DOTMULTI\g<1>', line) + while re.search(r'DOTMULTI\.', line): + line = re.sub(r'DOTMULTI\.([^\.])', r'DOTDOTMULTI \g<1>', line) + line = re.sub(r'DOTMULTI\.', r'DOTDOTMULTI', line) + + # separate out "," except if within numbers (5,300) + line = re.sub(r'([\D])[,]', r'\g<1> , ', line) + line = re.sub(r'[,]([\D])', r' , \g<1>', line) + + # separate "," after a number if it's the end of sentence + line = re.sub(r'(\d)[,]$', r'\g<1> ,', line) + + # split contractions right + line = re.sub(r'([\W\d])[\']([\W\d])', r'\g<1> \' \g<2>', line) + line = re.sub(r'(\W)[\']([\w\D])', r'\g<1> \' \g<2>', line) + line = re.sub(r'([\w\D])[\']([\W\d])', r'\g<1> \' \g<2>', line) + line = re.sub(r'([\w\D])[\']([\w\D])', r'\g<1> \'\g<2>', line) + # special case for "1990's" + line = re.sub(r'([\W\d])[\']([s])', r'\g<1> \'\g<2>', line) + + # apply nonbreaking prefixes + words = line.split() + line = '' + for i in range(len(words)): + word = words[i] + match = re.search(r'^(\S+)\.$', word) + if match: + pre = match.group(1) + if i == len(words) - 1: + """split last words independently as they are unlikely + to be non-breaking prefixes""" + word = pre + ' .' + else: + word = pre + ' .' + + word += ' ' + line += word + + # clean up extraneous spaces + line = re.sub(' +', ' ', line) + line = re.sub('^ ', '', line) + line = re.sub(' $', '', line) + + # .' at end of sentence is missed + line = re.sub(r'\.\' ?$', ' . \' ', line) + + # restore multi-dots + while re.search('DOTDOTMULTI', line): + line = re.sub('DOTDOTMULTI', 'DOTMULTI.', line) + + line = re.sub('DOTMULTI', '.', line) + + # escape special characters + line = re.sub(r'\&', r'&', line) + line = re.sub(r'\|', r'|', line) + line = re.sub(r'\<', r'<', line) + line = re.sub(r'\>', r'>', line) + line = re.sub(r'\'', r''', line) + line = re.sub(r'\"', r'"', line) + line = re.sub(r'\[', r'[', line) + line = re.sub(r'\]', r']', line) + + # ensure final line breaks + # if line[-1] is not '\n': + # line += '\n' + + return line diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/huggingface/__init__.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/huggingface/__init__.py new file mode 100644 index 0000000000..7231c7949a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/huggingface/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.common.tokenizers.huggingface.auto_tokenizer import AutoTokenizer diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/huggingface/auto_tokenizer.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/huggingface/auto_tokenizer.py new file mode 100644 index 0000000000..51e195f88e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/huggingface/auto_tokenizer.py @@ -0,0 +1,260 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Optional + +from transformers import AutoTokenizer as AUTOTOKENIZER + +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec +from nemo.utils import logging + +__all__ = [ + 'AutoTokenizer', +] + + +def handle_quotes(text): + text_ = "" + quote = 0 + i = 0 + while i < len(text): + if text[i] == "\"": + if quote % 2: + text_ = text_[:-1] + "\"" + else: + text_ += "\"" + i += 1 + quote += 1 + else: + text_ += text[i] + i += 1 + return text_ + + +def remove_spaces(text): + text = text.replace("( ", "(") + text = text.replace(" )", ")") + text = text.replace("[ ", "[") + text = text.replace(" ]", "]") + text = text.replace(" / ", "/") + text = text.replace("„ ", "„") + text = text.replace(" - ", "-") + text = text.replace(" ' ", "'") + text = re.sub(r'([0-9])( )([\.,])', '\\1\\3', text) + text = re.sub(r'([\.,])( )([0-9])', '\\1\\3', text) + text = re.sub(r'([0-9])(:)( )([0-9])', '\\1\\2\\4', text) + text = text.replace(" %", "%") + text = text.replace("$ ", "$") + text = re.sub(r'([^0-9])(,)([0-9])', '\\1\\2 \\3', text) + return text + + +class AutoTokenizer(TokenizerSpec): + ''' + Wrapper of HuggingFace AutoTokenizer https://huggingface.co/transformers/model_doc/auto.html#autotokenizer. + ''' + + def __init__( + self, + pretrained_model_name: str, + vocab_file: Optional[str] = None, + mask_token: Optional[str] = None, + bos_token: Optional[str] = None, + eos_token: Optional[str] = None, + pad_token: Optional[str] = None, + sep_token: Optional[str] = None, + cls_token: Optional[str] = None, + unk_token: Optional[str] = None, + use_fast: Optional[bool] = False, + ): + + """ + Args: + pretrained_model_name: corresponds to HuggingFace-AutoTokenizer's 'pretrained_model_name_or_path' input argument. + For more details please refer to https://huggingface.co/transformers/_modules/transformers/tokenization_auto.html#AutoTokenizer.from_pretrained. + The list of all supported models can be found here: ALL_PRETRAINED_CONFIG_ARCHIVE_MAP + vocab_file: path to file with vocabulary which consists + of characters separated by '\n'. + mask_token: mask token + bos_token: the beginning of sequence token + eos_token: the end of sequence token. Usually equal to sep_token + pad_token: token to use for padding + sep_token: token used for separating sequences + cls_token: class token. Usually equal to bos_token + unk_token: token to use for unknown tokens + use_fast: whether to use fast HuggingFace tokenizer + """ + try: + if vocab_file is not None: + message = 'Using "slow" HuggingFace tokenizer' + if use_fast: + message += f'{vocab_file} is ignored in "fast" tokenizers, using a "slow" version' + self.tokenizer = AUTOTOKENIZER.from_pretrained( + pretrained_model_name_or_path=pretrained_model_name, vocab_file=vocab_file, use_fast=False + ) + else: + self.tokenizer = AUTOTOKENIZER.from_pretrained( + pretrained_model_name_or_path=pretrained_model_name, use_fast=use_fast + ) + except Exception as e: + raise ValueError(f'{pretrained_model_name} is not supported by HuggingFace. {e}') + + special_tokens_dict = {} + + # # setting special tokens, by default the default model's special tokens will be preserved + # # unless passes new values to the special tokens + if unk_token is not None: + special_tokens_dict["unk_token"] = unk_token + if mask_token is not None: + special_tokens_dict["mask_token"] = mask_token + if pad_token is not None: + special_tokens_dict["pad_token"] = pad_token + + if sep_token is not None: + special_tokens_dict["sep_token"] = sep_token + elif self.tokenizer.sep_token is None and self.tokenizer.eos_token: + special_tokens_dict["sep_token"] = self.tokenizer.eos_token + if bos_token is not None: + special_tokens_dict["bos_token"] = bos_token + elif self.tokenizer.bos_token is None and self.tokenizer.cls_token: + special_tokens_dict["bos_token"] = self.tokenizer.cls_token + if eos_token is not None: + special_tokens_dict["eos_token"] = eos_token + elif self.tokenizer.eos_token is None and self.tokenizer.sep_token: + special_tokens_dict["eos_token"] = self.tokenizer.sep_token + if cls_token is not None: + special_tokens_dict["cls_token"] = cls_token + elif self.tokenizer.cls_token is None and self.tokenizer.bos_token: + special_tokens_dict["cls_token"] = self.tokenizer.bos_token + + new_tokens_in_vocab = [] + for token in [mask_token, bos_token, eos_token, pad_token, sep_token, cls_token, unk_token]: + if token is not None and token not in self.tokenizer.get_vocab(): + new_tokens_in_vocab.append(token) + + if len(new_tokens_in_vocab) > 0: + """ + Special tokens that were not previously included in the tokenizer's vocabulary file will be added to + the vocabulary and, as a result, the model should be resized, for example: + + # define your model + pretrained_model_name = 'roberta-base' + model = nemo_nlp.modules.get_lm_model(pretrained_model_name=pretrained_model_name) + + # define pretrained tokenizer + tokenizer_default = nemo_nlp.modules.get_tokenizer(tokenizer_name=pretrained_model_name) + + special_tokens = {'bos_token': '', + 'cls_token': '', + 'additional_special_tokens': ['', '']} + tokenizer_default.add_special_tokens(special_tokens_dict=special_tokens) + + # resize your model so that the embeddings for newly added tokens are updated during training/finetuning + model.resize_token_embeddings(tokenizer_default.vocab_size) + + See NLP_Tokenizers.ipynb for more details. + """ + logging.warning( + f'{new_tokens_in_vocab} \n will be added to the vocabulary.\n' + f'Please resize your model accordingly, ' + f'see NLP_Tokenizers.ipynb for more details.' + ) + self.add_special_tokens(special_tokens_dict) + + @property + def vocab_size(self): + return len(self.tokenizer) + + def add_special_tokens(self, special_tokens_dict: dict) -> int: + """ + Adds a dictionary of special tokens (eos, pad, cls...). If special tokens are NOT in the vocabulary, they are added + to it (indexed starting from the last index of the current vocabulary). + Args: + special_tokens_dict: dict of string. Keys should be in the list of predefined special attributes: + [``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``, + ``additional_special_tokens``]. + Tokens are only added if they are not already in the vocabulary. + Returns: + Number of tokens added to the vocabulary. + """ + num_tokens_added = self.tokenizer.add_special_tokens(special_tokens_dict) + + if num_tokens_added > 0: + logging.info(f'{num_tokens_added} special tokens added, resize your model accordingly.') + for k in self.tokenizer.SPECIAL_TOKENS_ATTRIBUTES: + setattr(self, k, getattr(self.tokenizer, k, None)) + return num_tokens_added + + def text_to_tokens(self, text): + tokens = self.tokenizer.tokenize(text) + return tokens + + def tokens_to_text(self, tokens): + text = self.tokenizer.convert_tokens_to_string(tokens) + return remove_spaces(handle_quotes(text.strip())) + + def token_to_id(self, token): + return self.tokens_to_ids([token])[0] + + def tokens_to_ids(self, tokens): + ids = self.tokenizer.convert_tokens_to_ids(tokens) + return ids + + def ids_to_tokens(self, ids): + tokens = self.tokenizer.convert_ids_to_tokens(ids) + return tokens + + def text_to_ids(self, text): + tokens = self.text_to_tokens(text) + ids = self.tokens_to_ids(tokens) + return ids + + def ids_to_text(self, ids): + tokens = self.ids_to_tokens(ids) + tokens_clean = [t for t in tokens if t not in self.tokenizer.all_special_tokens] + text = self.tokens_to_text(tokens_clean) + return text + + @property + def pad_id(self): + return self.tokens_to_ids([getattr(self, 'pad_token')])[0] + + @property + def bos_id(self): + return self.tokens_to_ids([getattr(self, 'bos_token')])[0] + + @property + def eos_id(self): + return self.tokens_to_ids([getattr(self, 'eos_token')])[0] + + @property + def sep_id(self): + return self.tokens_to_ids([getattr(self, 'sep_token')])[0] + + @property + def cls_id(self): + return self.tokens_to_ids([getattr(self, 'cls_token')])[0] + + @property + def unk_id(self): + return self.tokens_to_ids([getattr(self, 'unk_token')])[0] + + @property + def mask_id(self): + return self.tokens_to_ids([getattr(self, 'mask_token')])[0] + + @property + def name(self): + return type(self.tokenizer).__name__ diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/moses_tokenizers.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/moses_tokenizers.py new file mode 100644 index 0000000000..27e91e6c52 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/moses_tokenizers.py @@ -0,0 +1,47 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +from sacremoses import MosesDetokenizer, MosesPunctNormalizer, MosesTokenizer + + +class MosesProcessor: + """ + Tokenizer, Detokenizer and Normalizer utilities in Moses + """ + + def __init__(self, lang_id: str): + self.moses_tokenizer = MosesTokenizer(lang=lang_id) + self.moses_detokenizer = MosesDetokenizer(lang=lang_id) + self.normalizer = MosesPunctNormalizer(lang=lang_id) + + def detokenize(self, tokens: List[str]) -> str: + """ + Detokenizes a list of tokens + Args: + tokens: list of strings as tokens + Returns: + detokenized string + """ + return self.moses_detokenizer.detokenize(tokens) + + def tokenize(self, text: str): + """ + Tokenizes text using Moses -> Sentencepiece. + """ + return self.moses_tokenizer.tokenize(text, escape=False, return_str=True) + + def normalize(self, text: str): + return self.normalizer.normalize(text) diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/sentencepiece_tokenizer.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/sentencepiece_tokenizer.py new file mode 100644 index 0000000000..9f6e4dc1f0 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/sentencepiece_tokenizer.py @@ -0,0 +1,353 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +from typing import Dict, List, Optional, Union + +import numpy as np +import sentencepiece + +from nemo.collections.common.parts.utils import if_exist +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec +from nemo.utils import logging + +__all__ = ['SentencePieceTokenizer', 'create_spt_model'] + + +class SentencePieceTokenizer(TokenizerSpec): + ''' + Sentencepiecetokenizer https://github.com/google/sentencepiece. + Args: + model_path: path to sentence piece tokenizer model. To create the model use create_spt_model() + special_tokens: either list of special tokens or dictionary of token name to token value + legacy: when set to True, the previous behavior of the SentecePiece wrapper will be restored, + including the possibility to add special tokens inside wrapper. + ''' + + def __init__( + self, model_path: str, special_tokens: Optional[Union[Dict[str, str], List[str]]] = None, legacy: bool = False + ): + if not model_path or not os.path.exists(model_path): + raise ValueError(f"model_path: {model_path} is invalid") + self.tokenizer = sentencepiece.SentencePieceProcessor() + self.tokenizer.Load(model_path) + self.original_vocab_size = self.tokenizer.get_piece_size() + self.vocab_size = self.tokenizer.get_piece_size() + self.legacy = legacy + self.special_token_to_id = {} + self.id_to_special_token = {} + if special_tokens: + if not self.legacy: + raise ValueError( + "Special tokens must be None when legacy is set to False. Provide special tokens at train time." + ) + self.add_special_tokens(special_tokens) + + def text_to_tokens(self, text): + if self.legacy: + tokens = [] + idx = 0 + last_idx = 0 + + while 1: + indices = {} + + for token in self.special_token_to_id: + try: + indices[token] = text[idx:].index(token) + except ValueError: + continue + + if len(indices) == 0: + break + + next_token = min(indices, key=indices.get) + next_idx = idx + indices[next_token] + + tokens.extend(self.tokenizer.encode_as_pieces(text[idx:next_idx])) + tokens.append(next_token) + idx = next_idx + len(next_token) + + tokens.extend(self.tokenizer.encode_as_pieces(text[idx:])) + return tokens + + return self.tokenizer.encode_as_pieces(text) + + def text_to_ids(self, text): + if self.legacy: + ids = [] + idx = 0 + last_idx = 0 + + while 1: + indices = {} + + for token in self.special_token_to_id: + try: + indices[token] = text[idx:].index(token) + except ValueError: + continue + + if len(indices) == 0: + break + + next_token = min(indices, key=indices.get) + next_idx = idx + indices[next_token] + + ids.extend(self.tokenizer.encode_as_ids(text[idx:next_idx])) + ids.append(self.special_token_to_id[next_token]) + idx = next_idx + len(next_token) + + ids.extend(self.tokenizer.encode_as_ids(text[idx:])) + return ids + + return self.tokenizer.encode_as_ids(text) + + def tokens_to_text(self, tokens): + if isinstance(tokens, np.ndarray): + tokens = tokens.tolist() + + return self.tokenizer.decode_pieces(tokens) + + def ids_to_text(self, ids): + if isinstance(ids, np.ndarray): + ids = ids.tolist() + + if self.legacy: + text = "" + last_i = 0 + + for i, id in enumerate(ids): + if id in self.id_to_special_token: + text += self.tokenizer.decode_ids(ids[last_i:i]) + " " + text += self.id_to_special_token[id] + " " + last_i = i + 1 + + text += self.tokenizer.decode_ids(ids[last_i:]) + return text.strip() + + return self.tokenizer.decode_ids(ids) + + def token_to_id(self, token): + if self.legacy and token in self.special_token_to_id: + return self.special_token_to_id[token] + + return self.tokenizer.piece_to_id(token) + + def ids_to_tokens(self, ids): + tokens = [] + for id in ids: + if id >= self.original_vocab_size: + tokens.append(self.id_to_special_token[id]) + else: + tokens.append(self.tokenizer.id_to_piece(id)) + return tokens + + def tokens_to_ids(self, tokens: Union[str, List[str]]) -> Union[int, List[int]]: + if isinstance(tokens, str): + tokens = [tokens] + ids = [] + for token in tokens: + ids.append(self.token_to_id(token)) + return ids + + def add_special_tokens(self, special_tokens): + if not self.legacy: + raise AttributeError("Special Token addition does not work when legacy is set to False.") + + if isinstance(special_tokens, list): + for token in special_tokens: + if ( + self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id() + and token not in self.special_token_to_id + ): + self.special_token_to_id[token] = self.vocab_size + self.id_to_special_token[self.vocab_size] = token + self.vocab_size += 1 + elif isinstance(special_tokens, dict): + for token_name, token in special_tokens.items(): + setattr(self, token_name, token) + if ( + self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id() + and token not in self.special_token_to_id + ): + self.special_token_to_id[token] = self.vocab_size + self.id_to_special_token[self.vocab_size] = token + self.vocab_size += 1 + + @property + def pad_id(self): + if self.legacy: + pad_id = self.tokens_to_ids([self.pad_token])[0] + else: + pad_id = self.tokenizer.pad_id() + return pad_id + + @property + def bos_id(self): + if self.legacy: + bos_id = self.tokens_to_ids([self.bos_token])[0] + else: + bos_id = self.tokenizer.bos_id() + return bos_id + + @property + def eos_id(self): + if self.legacy: + eos_id = self.tokens_to_ids([self.eos_token])[0] + else: + eos_id = self.tokenizer.eos_id() + return eos_id + + @property + def sep_id(self): + if self.legacy: + return self.tokens_to_ids([self.sep_token])[0] + else: + raise NameError("Use function token_to_id to retrieve special tokens other than unk, pad, bos, and eos.") + + @property + def cls_id(self): + if self.legacy: + return self.tokens_to_ids([self.cls_token])[0] + else: + raise NameError("Use function token_to_id to retrieve special tokens other than unk, pad, bos, and eos.") + + @property + def unk_id(self): + return self.tokenizer.unk_id() + + +def create_spt_model( + data_file: str, + vocab_size: int, + sample_size: int, + do_lower_case: bool, + tokenizer_type: str = 'unigram', + output_dir: Optional[str] = None, + character_coverage: float = 1.0, + train_extremely_large_corpus: bool = False, + max_sentencepiece_length: int = -1, + bos: bool = False, + eos: bool = False, + pad: bool = False, + control_symbols: List[str] = None, + user_defined_symbols: List[str] = None, +): + """ + Creates sentence piece tokenizer model from data file. + Args: + data_file: data file + vocab_size: vocabulary size + sample_size: maximum size of sentences the trainer loads + do_lower_case: if text should be lower cased before tokenizer model is created + character_coverage: float value between 0 and 1 (as a percentage). For languages with a vast charset, + can be < 1.0, but for all other languages, it should be set as 1.0 + output_dir: folder to save created tokenizer model. If not specified will store model at data_file/../spt folder + train_extremely_large_corpus: If training on huge datasets, pass this flag to allow SentencePiece + to build the tokenizer. + max_sentencepiece_length: Limits the maximum length of the SentencePiece subword that can be constructed. + By default, no limit is placed. + bos: when True, bos token "" is added to the vocabulary. + eos: when True, eos token "" is added to the vocabulary. + pad: when True, pad token "" is added to the vocabulary. + control_symbols: control symbols to add to tokenizer, as defined by sentencepiece. + These tokens get removed at decode time and are not encoded from the text - can only be added to the input programatically. + user_defined_symbols: user symbols to add to tokenizer, as defined by sentencepiece. + These tokens remain in the decoded text and are encoded automatically when present in the input text. + """ + + if not data_file or not os.path.exists(data_file): + raise ValueError(f"data_file must be valid file path, but got {data_file}") + data_dir = os.path.dirname(data_file) + vocab = [] + special_tokens = ["", "", "", ""] + if not output_dir: + output_dir = f'{data_dir}/spt' + if if_exist(output_dir, ['tokenizer.model']): + logging.info(f"tokenizer model {output_dir}/tokenizer.model already exists") + return f'{output_dir}/tokenizer.model', f'{output_dir}/vocab.txt' + logging.info(f'Processing {data_file} and store at {output_dir}') + os.makedirs(output_dir, exist_ok=True) + + cmd = ( + f"--input={data_file} --model_prefix={output_dir}/tokenizer " + f"--vocab_size={vocab_size} " + f"--shuffle_input_sentence=true --hard_vocab_limit=false " + f"--model_type={tokenizer_type} " + f"--character_coverage={character_coverage}" + ) + + pad_id = 3 + if not bos: + pad_id -= 1 + cmd += " --bos_id=-1" + + if not eos: + pad_id -= 1 + cmd += " --eos_id=-1" + + if pad: + cmd += f" --pad_id={pad_id}" + + if control_symbols: + control_string = (",").join(control_symbols) + cmd += f" --control_symbols={control_string}" + special_tokens += control_symbols + + if user_defined_symbols: + user_string = (",").join(user_defined_symbols) + cmd += f" --user_defined_symbols={user_string}" + special_tokens += user_defined_symbols + + if do_lower_case: + cmd += " --normalization_rule_name=nmt_nfkc_cf" + + if sample_size > 0: + cmd += f" --input_sentence_size={sample_size}" + + if train_extremely_large_corpus: + cmd += " --train_extremely_large_corpus=true" + + if max_sentencepiece_length >= 0: + cmd += f" --max_sentencepiece_length={max_sentencepiece_length}" + + sentencepiece.SentencePieceTrainer.Train(cmd) + + # Add BERT control symbols + tokens = [] + + with open(f"{output_dir}/tokenizer.vocab", "r") as f: + # Read tokens from each line and parse for vocab + for line in f: + piece = line.split("\t")[0] + if piece in special_tokens: + # skip special tokens + continue + token = piece[1:] if piece.startswith("▁") else f"##{piece}" + + if len(token) > 0: + tokens.append(token) + else: + tokens.append(piece[0]) + + vocab.extend(tokens) + + # Save vocabulary to output file + vocab_file = f'{output_dir}/vocab.txt' + with open(vocab_file, "w") as f: + for token in vocab: + f.write(f"{token}\n") + return f'{output_dir}/tokenizer.model', vocab_file diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/tokenizer_spec.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/tokenizer_spec.py new file mode 100644 index 0000000000..252571d76e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/tokenizer_spec.py @@ -0,0 +1,55 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import List + +__all__ = ['TokenizerSpec'] + + +class TokenizerSpec(ABC): + """ + Inherit this class to implement a new tokenizer. + """ + + @abstractmethod + def text_to_tokens(self, text): + pass + + @abstractmethod + def tokens_to_text(self, tokens): + pass + + @abstractmethod + def tokens_to_ids(self, tokens): + pass + + @abstractmethod + def ids_to_tokens(self, ids): + pass + + @abstractmethod + def text_to_ids(self, text): + pass + + @abstractmethod + def ids_to_text(self, ids): + pass + + def add_special_tokens(self, special_tokens: List[str]): + raise NotImplementedError("To be implemented") + + @property + def name(self): + return type(self).__name__ diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/word_tokenizer.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/word_tokenizer.py new file mode 100644 index 0000000000..f3431af9d7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/word_tokenizer.py @@ -0,0 +1,72 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from nemo.collections.common.tokenizers.char_tokenizer import CharTokenizer + +__all__ = ['WordTokenizer'] + + +class WordTokenizer(CharTokenizer): + "Tokenizes at word boundary" + + def __init__( + self, + vocab_file: str, + mask_token: Optional[str] = None, + bos_token: Optional[str] = None, + eos_token: Optional[str] = None, + pad_token: Optional[str] = None, + sep_token: Optional[str] = None, + cls_token: Optional[str] = None, + unk_token: Optional[str] = None, + ): + """ + Args: + vocab_file: path to file with vocabulary which consists + of characters separated by \n + mask_token: mask token + bos_token: the beginning of sequence token + eos_token: the end of sequence token. Usually equal to sep_token + pad_token: token to use for padding + sep_token: token used for separating sequences + cls_token: class token. Usually equal to bos_token + unk_token: token to use for unknown tokens + """ + + super().__init__( + vocab_file=vocab_file, + mask_token=mask_token, + bos_token=bos_token, + eos_token=eos_token, + pad_token=pad_token, + unk_token=unk_token, + sep_token=sep_token, + cls_token=cls_token, + ) + + def text_to_tokens(self, text): + token_candidates = text.strip().split() + tokens = [] + for token in token_candidates: + if token in self.vocab: + tokens.append(token) + else: + tokens.append(self.unk_token) + return tokens + + def ids_to_text(self, ids): + ids_ = [id_ for id_ in ids if id_ not in self.special_tokens] + return " ".join(self.ids_to_tokens(ids_)) diff --git a/models/tacotron2/NeMo/nemo/collections/common/tokenizers/youtokentome_tokenizer.py b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/youtokentome_tokenizer.py new file mode 100644 index 0000000000..79517169bf --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/common/tokenizers/youtokentome_tokenizer.py @@ -0,0 +1,70 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import youtokentome as yttm + +from nemo.collections.common.tokenizers import TokenizerSpec + +__all__ = ['YouTokenToMeTokenizer'] + + +class YouTokenToMeTokenizer(TokenizerSpec): + def __init__(self, model_path, bpe_dropout=0.0, legacy=False): + model_path = Path(model_path).expanduser() + self.tokenizer = yttm.BPE(model=str(model_path)) + self.vocab_size = len(self.tokenizer.vocab()) + self.special_tokens = self.tokens_to_ids(["", "", "", ""]) + self.bpe_dropout = bpe_dropout + self.legacy = legacy + + def text_to_tokens(self, text): + return self.tokenizer.encode(text, output_type=yttm.OutputType.SUBWORD, dropout_prob=self.bpe_dropout) + + def tokens_to_text(self, tokens): + return self.ids_to_text(self.tokens_to_ids(tokens)) + + def text_to_ids(self, text): + return self.tokenizer.encode(text, output_type=yttm.OutputType.ID, dropout_prob=self.bpe_dropout) + + def ids_to_text(self, ids): + ids_ = [id_ for id_ in ids if id_ not in self.special_tokens] + return self.tokenizer.decode([ids_])[0] + + def tokens_to_ids(self, tokens): + return [self.tokenizer.subword_to_id(token) for token in tokens] + + def ids_to_tokens(self, ids): + if self.legacy: + ids_ = [id_ for id_ in ids if id_ not in self.special_tokens] + else: + ids_ = ids + return [self.tokenizer.id_to_subword(id_) for id_ in ids_] + + @property + def pad_id(self): + return self.tokenizer.subword_to_id("") + + @property + def bos_id(self): + return self.tokenizer.subword_to_id("") + + @property + def eos_id(self): + return self.tokenizer.subword_to_id("") + + @property + def unk_id(self): + return self.tokenizer.subword_to_id("") diff --git a/models/tacotron2/NeMo/nemo/collections/cv/__init__.py b/models/tacotron2/NeMo/nemo/collections/cv/__init__.py new file mode 100644 index 0000000000..01d8fbbd40 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/cv/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.cv import datasets, losses, models, modules +from nemo.package_info import __version__ + +# Set collection version equal to NeMo version. +__version = __version__ + +# Authorship. +__author__ = "NVIDIA Corporation" + +# Set collection name. +__description__ = "Computer Vision collection" diff --git a/models/tacotron2/NeMo/nemo/collections/cv/datasets/__init__.py b/models/tacotron2/NeMo/nemo/collections/cv/datasets/__init__.py new file mode 100644 index 0000000000..31325313ce --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/cv/datasets/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.cv.datasets.mnist_dataset import MNISTDataset, MNISTDatasetConfig diff --git a/models/tacotron2/NeMo/nemo/collections/cv/datasets/mnist_dataset.py b/models/tacotron2/NeMo/nemo/collections/cv/datasets/mnist_dataset.py new file mode 100644 index 0000000000..5d6fdb9bf6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/cv/datasets/mnist_dataset.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from os.path import expanduser +from typing import Optional + +from torchvision.datasets import MNIST +from torchvision.transforms import Compose, Resize, ToTensor + +from nemo.core.classes import Dataset +from nemo.core.config import Config +from nemo.core.neural_types.axes import AxisKind, AxisType +from nemo.core.neural_types.elements import ClassificationTarget, Index, NormalizedImageValue, StringLabel +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils.decorators import add_port_docs, experimental + + +@dataclass +class MNISTDatasetConfig(Config): + """ + Structured config for MNISTDataset class. + + Args: + height: image height (DEFAULT: 28) + width: image width (DEFAULT: 28) + data_folder: path to the folder with data, can be relative to user (DEFAULT: "~/data/mnist") + train: use train or test splits (DEFAULT: True) + name: Name of the module (DEFAULT: None) + """ + + height: int = 28 + width: int = 28 + data_folder: str = "~/data/mnist" + train: bool = True + download: bool = True + + +class MNISTDataset(Dataset): + """ + A "thin wrapper" around the torchvision's MNIST dataset. + """ + + def __init__(self, cfg: MNISTDatasetConfig = MNISTDatasetConfig()): + """ + Initializes the MNIST dataset. + + Args: + cfg: Configuration object of type MNISTDatasetConfig. + + """ + # Call the base class constructor of Dataset. + Dataset.__init__(self) # , name=name) + + # Store height and width. + self._height = cfg.height + self._width = cfg.width + + # Create transformations: up-scale and transform to tensors. + mnist_transforms = Compose([Resize((self._height, self._width)), ToTensor()]) + + # Get absolute path. + abs_data_folder = expanduser(cfg.data_folder) + + # Create the MNIST dataset object. + self._dataset = MNIST(root=abs_data_folder, train=cfg.train, download=cfg.download, transform=mnist_transforms) + + # Class names. + labels = 'Zero One Two Three Four Five Six Seven Eight Nine'.split(' ') + word_to_ix = {labels[i]: i for i in range(10)} + + # Reverse mapping. + self._ix_to_word = {value: key for (key, value) in word_to_ix.items()} + + @property + def output_types(self): + """ + Creates definitions of output ports. + """ + return { + "indices": NeuralType(tuple('B'), elements_type=Index()), + "images": NeuralType( + axes=( + AxisType(kind=AxisKind.Batch), + AxisType(kind=AxisKind.Channel, size=1), + AxisType(kind=AxisKind.Height, size=self._height), + AxisType(kind=AxisKind.Width, size=self._width), + ), + elements_type=NormalizedImageValue(), # float, <0-1> + ), + "targets": NeuralType(tuple('B'), elements_type=ClassificationTarget()), # Target are ints! + "labels": NeuralType(tuple('B'), elements_type=StringLabel()), # Labels is string! + } + + def __len__(self): + """ + Returns: + Length of the dataset. + """ + return len(self._dataset) + + def __getitem__(self, index: int): + """ + Returns a single sample. + + Args: + index: index of the sample to return. + """ + # Get image and target. + img, target = self._dataset.__getitem__(index) + + # Return sample. + return index, img, target, self._ix_to_word[target] + + @property + def ix_to_word(self): + """ + Returns: + Dictionary with mapping of target indices (int) to labels (class names as strings) + that can we used by other modules. + """ + return self._ix_to_word diff --git a/models/tacotron2/NeMo/nemo/collections/cv/losses/__init__.py b/models/tacotron2/NeMo/nemo/collections/cv/losses/__init__.py new file mode 100644 index 0000000000..8f238c51e5 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/cv/losses/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.cv.losses.nll_loss import NLLLoss diff --git a/models/tacotron2/NeMo/nemo/collections/cv/losses/nll_loss.py b/models/tacotron2/NeMo/nemo/collections/cv/losses/nll_loss.py new file mode 100644 index 0000000000..f2e61d1724 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/cv/losses/nll_loss.py @@ -0,0 +1,54 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from torch.nn import NLLLoss as torch_NLLLoss + +from nemo.core.classes import Serialization, Typing, typecheck +from nemo.core.neural_types import ClassificationTarget, LogprobsType, LossType, NeuralType +from nemo.utils.decorators import experimental + + +@experimental +class NLLLoss(torch_NLLLoss, Serialization, Typing): + """ Class representing a simple NLL loss. """ + + def __init__(self, name: Optional[str] = None): + """ + Constructor. + + Args: + name: Name of the module (DEFAULT: None) + """ + # Call the base constructors. + # Serialization.__init__(self, name=name) + torch_NLLLoss.__init__(self) + + @property + def input_types(self): + """ Returns definitions of module input ports. """ + return { + "predictions": NeuralType(axes=('B', 'ANY'), elements_type=LogprobsType()), + "targets": NeuralType(axes=('B'), elements_type=ClassificationTarget()), + } + + @property + def output_types(self): + """ Returns definitions of module output ports. """ + return {"loss": NeuralType(elements_type=LossType())} + + @typecheck() + def forward(self, predictions, targets): + return torch_NLLLoss().forward(input=predictions, target=targets) diff --git a/models/tacotron2/NeMo/nemo/collections/cv/models/__init__.py b/models/tacotron2/NeMo/nemo/collections/cv/models/__init__.py new file mode 100644 index 0000000000..ca3e8e3d5c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/cv/models/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.cv.models.mnist_lenet5 import MNISTLeNet5, MNISTLeNet5Config diff --git a/models/tacotron2/NeMo/nemo/collections/cv/models/mnist_lenet5.py b/models/tacotron2/NeMo/nemo/collections/cv/models/mnist_lenet5.py new file mode 100644 index 0000000000..cd6cf2091e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/cv/models/mnist_lenet5.py @@ -0,0 +1,157 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict, Optional + +from torch.utils.data import DataLoader + +from nemo.collections.cv.datasets import MNISTDataset, MNISTDatasetConfig +from nemo.collections.cv.losses import NLLLoss +from nemo.collections.cv.modules import LeNet5 as LeNet5Module +from nemo.core.classes import ModelPT +from nemo.core.classes.common import typecheck +from nemo.core.config import Config, CosineAnnealingParams, DataLoaderConfig, NovogradParams +from nemo.core.neural_types import * + + +@dataclass +class NovogradScheduler(Config): + """ + Scheduler setup for novograd + """ + + # Mandatory parameters + name: str = "CosineAnnealing" + args: CosineAnnealingParams = CosineAnnealingParams() # can be a string to param, or "auto", just like in YAML + + # pytorch lightning parameters + monitor: str = "val_loss" + iters_per_batch: Optional[float] = None # computed at runtime + max_steps: Optional[int] = None # computed at runtime or explicitly set here + reduce_on_plateau: bool = False + + +@dataclass +class MNISTOptimizer(Config): + """ + Optimizer setup for novograd + """ + + # Mandatory arguments + name: str = "novograd" + lr: float = 0.01 + + args: NovogradParams = NovogradParams(betas=(0.8, 0.5)) + # sched: NovogradScheduler = NovogradScheduler() + + +@dataclass +class MNISTLeNet5Config(Config): + """ + Structured config for LeNet-5 model class - that also contains parameters of dataset and dataloader. + """ + + dataset: MNISTDatasetConfig = MNISTDatasetConfig(width=32, height=32) + dataloader: DataLoaderConfig = DataLoaderConfig(batch_size=64, shuffle=True) + module: Config = Config() + optim: MNISTOptimizer = MNISTOptimizer() + + +class MNISTLeNet5(ModelPT): + """ + The LeNet-5 convolutional model. + """ + + def __init__(self, cfg: MNISTLeNet5Config = MNISTLeNet5Config()): + super().__init__(cfg=cfg) + + # Initialize modules. + self.module = LeNet5Module(cfg.module) + self.loss = NLLLoss() + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + """ + Returns: + :class:`LeNet5Module` input types. + """ + return self.module.input_types + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """ + Returns: + :class:`LeNet5Module` output types. + """ + return self.module.output_types + + @typecheck() + def forward(self, images): + """ Propagates data by calling the module :class:`LeNet5Module` forward. """ + return self.module.forward(images=images) + + def setup_training_data(self, train_data_layer_config: Optional[Dict] = None): + """ Creates dataset, wrap it with dataloader and return the latter """ + # Instantiate dataset. + mnist_ds = MNISTDataset(self._cfg.dataset) + # Configure data loader. + train_dataloader = DataLoader(dataset=mnist_ds, **(self._cfg.dataloader)) + self._train_dl = train_dataloader + + def setup_validation_data(self, val_data_layer_config: Optional[Dict] = None): + """ Not implemented. """ + self._val_dl = None + + def setup_test_data(self, test_data_layer_params: Optional[Dict] = None): + """ Not implemented. """ + self._test_dl = None + + def training_step(self, batch, what_is_this_input): + """ Training step, calculate loss. """ + # "Unpack" the batch. + _, images, targets, _ = batch + + # Get predictions. + predictions = self(images=images) + + # Calculate loss. + loss = self.loss(predictions=predictions, targets=targets) + + # Return it. + return {"loss": loss} + # of course "return loss" doesn't work :] + + def train_dataloader(self): + """ Not implemented. """ + return self._train_dl + + def save_to(self, save_path: str): + """ Not implemented. """ + pass + + @classmethod + def restore_from(cls, restore_path: str): + """ Not implemented. """ + pass + + @classmethod + def list_available_models(cls) -> Optional[Dict[str, str]]: + """ Not implemented. """ + pass + + @classmethod + def from_pretrained(cls, name: str): + """ Not implemented. """ + pass diff --git a/models/tacotron2/NeMo/nemo/collections/cv/modules/__init__.py b/models/tacotron2/NeMo/nemo/collections/cv/modules/__init__.py new file mode 100644 index 0000000000..10848091f1 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/cv/modules/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.cv.modules.lenet5 import LeNet5 diff --git a/models/tacotron2/NeMo/nemo/collections/cv/modules/lenet5.py b/models/tacotron2/NeMo/nemo/collections/cv/modules/lenet5.py new file mode 100644 index 0000000000..bb930d15df --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/cv/modules/lenet5.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from torch import nn + +from nemo.core.classes import NeuralModule, typecheck +from nemo.core.config import Config +from nemo.core.neural_types import AxisKind, AxisType, ImageValue, LogprobsType, NeuralType + + +class LeNet5(NeuralModule): + """ + Classical LeNet-5 model for MNIST image classification. + """ + + def __init__(self, cfg: Config = Config()): + """ + Creates the LeNet-5 model. + + Args: + cfg: Default NeMo config containing name. + """ + # Call the base class constructor. + super().__init__() # name=name) + + # Create the LeNet-5 model. + self.model = nn.Sequential( + nn.Conv2d(1, 6, kernel_size=(5, 5)), + nn.ReLU(), + nn.MaxPool2d(kernel_size=(2, 2), stride=2), + nn.Conv2d(6, 16, kernel_size=(5, 5)), + nn.ReLU(), + nn.MaxPool2d(kernel_size=(2, 2), stride=2), + nn.Conv2d(16, 120, kernel_size=(5, 5)), + nn.ReLU(), + # reshape to [-1, 120] + nn.Flatten(), + nn.Linear(120, 84), + nn.ReLU(), + nn.Linear(84, 10), + nn.LogSoftmax(dim=1), + ) + + @property + def input_types(self): + """ Returns definitions of module input ports. """ + return { + "images": NeuralType( + axes=( + AxisType(kind=AxisKind.Batch), + AxisType(kind=AxisKind.Channel, size=1), + AxisType(kind=AxisKind.Height, size=32), + AxisType(kind=AxisKind.Width, size=32), + ), + elements_type=ImageValue(), + ) + } + + @property + def output_types(self): + """ Returns definitions of module output ports. """ + return { + "predictions": NeuralType( + axes=(AxisType(kind=AxisKind.Batch), AxisType(kind=AxisKind.Dimension)), elements_type=LogprobsType() + ) + } + + @typecheck() + def forward(self, images): + """ + Performs the forward step of the LeNet-5 model. + + Args: + images: Batch of images to be classified. + + Returns: + Batch of predictions. + """ + + predictions = self.model(images) + return predictions + + def save_to(self, save_path: str): + """Not implemented yet. + Serialize model. + + Args: + save_path (str): path to save serialization. + """ + pass + + @classmethod + def restore_from(cls, restore_path: str): + """ Not implemented yet. + Restore module from serialization. + + Args: + restore_path (str): path to serialization + """ + pass diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/__init__.py new file mode 100644 index 0000000000..f6e986d0e8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp import data, losses, models, modules +from nemo.package_info import __version__ + +# Set collection version equal to NeMo version. +__version = __version__ + +# Authorship. +__author__ = "NVIDIA Corporation" + +# Set collection name. +__description__ = "Natural Language Processing collection" diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/__init__.py new file mode 100644 index 0000000000..3780dc61ca --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/__init__.py @@ -0,0 +1,40 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.data_utils import * +from nemo.collections.nlp.data.entity_linking.entity_linking_dataset import EntityLinkingDataset +from nemo.collections.nlp.data.information_retrieval.information_retrieval_dataset import ( + BertInformationRetrievalDataset, +) +from nemo.collections.nlp.data.language_modeling.l2r_lm_dataset import ( + L2RLanguageModelingDataset, + TarredL2RLanguageModelingDataset, +) +from nemo.collections.nlp.data.language_modeling.lm_bert_dataset import ( + BertPretrainingDataset, + BertPretrainingPreprocessedDataloader, +) +from nemo.collections.nlp.data.language_modeling.sentence_dataset import SentenceDataset, TarredSentenceDataset +from nemo.collections.nlp.data.machine_translation.machine_translation_dataset import ( + TarredTranslationDataset, + TranslationDataset, +) +from nemo.collections.nlp.data.neural_machine_translation.neural_machine_translation_dataset import ( + NeuralMachineTranslationDataset, +) +from nemo.collections.nlp.data.question_answering_squad.qa_dataset import SquadDataset +from nemo.collections.nlp.data.token_classification.token_classification_dataset import ( + BertTokenClassificationDataset, + BertTokenClassificationInferDataset, +) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/data_utils/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/data_utils/__init__.py new file mode 100644 index 0000000000..f57d67bdea --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/data_utils/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.data_utils.data_preprocessing import * diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/data_utils/data_preprocessing.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/data_utils/data_preprocessing.py new file mode 100644 index 0000000000..7cc595c5c6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/data_utils/data_preprocessing.py @@ -0,0 +1,479 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import json +import os +import pickle +import random +import re +import string +from collections import Counter + +import numpy as np +import torch +from tqdm.auto import tqdm + +from nemo.utils import logging +from nemo.utils.env_var_parsing import get_envint + +__all__ = [ + 'DataProcessor', + 'get_label_stats', + 'partition_data', + 'write_files', + 'write_data', + 'create_dataset', + 'read_csv', + 'get_dataset', + 'partition', + 'map_entities', + 'get_entities', + 'get_data', + 'reverse_dict', + 'get_intent_labels', + 'get_stats', + 'DATABASE_EXISTS_TMP', + 'MODE_EXISTS_TMP', + 'is_whitespace', + 'write_vocab', + 'if_exist', + 'remove_punctuation_from_sentence', + 'dataset_to_ids', + 'get_freq_weights', + 'fill_class_weights', + 'normalize_answer', + 'get_labels_to_labels_id_mapping', + 'get_vocab', + 'find_newlines', + 'load_data_indices', +] + +DATABASE_EXISTS_TMP = '{} dataset has already been processed and stored at {}' +MODE_EXISTS_TMP = '{} mode of {} dataset has already been processed and stored at {}' + + +class DataProcessor(object): + """Base class for data converters for sequence classification data sets.""" + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + @classmethod + def _read_tsv(cls, input_file, quotechar=None): + """Reads a tab separated value file.""" + with open(input_file, "r", encoding="utf-8-sig") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + # if sys.version_info[0] == 2: + # line = list(unicode(cell, 'utf-8') for cell in line) + lines.append(line) + return lines + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + return re.sub(r'\b(a|an|the)\b', ' ', text) + + def white_space_fix(text): + return ' '.join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return ''.join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def get_label_stats(labels, outfile='stats.tsv', verbose=True): + ''' + + Args: + labels: list of all labels + outfile: path to the file where to save label stats + + Returns: + total (int): total number of labels + label_frequencies (list of tuples): each tuple represent (label, label frequency) + max id of the labels + ''' + labels = Counter(labels) + total = sum(labels.values()) + out = open(outfile, 'w') + i = 0 + freq_dict = {} + label_frequencies = labels.most_common() + for k, v in label_frequencies: + out.write(f'{k}\t\t{round(v/total,5)}\t\t{v}\n') + if verbose and i < 3: + logging.info(f'label: {k}, {v} out of {total} ({(v / total)*100.0:.2f}%).') + i += 1 + freq_dict[k] = v + + return total, freq_dict, max(labels.keys()) + + +def partition_data(intent_queries, slot_tags, split=0.1): + n = len(intent_queries) + n_dev = int(n * split) + dev_idx = set(random.sample(range(n), n_dev)) + dev_intents, dev_slots, train_intents, train_slots = [], [], [], [] + + dev_intents.append('sentence\tlabel\n') + train_intents.append('sentence\tlabel\n') + + for i, item in enumerate(intent_queries): + if i in dev_idx: + dev_intents.append(item) + dev_slots.append(slot_tags[i]) + else: + train_intents.append(item) + train_slots.append(slot_tags[i]) + return train_intents, train_slots, dev_intents, dev_slots + + +def write_files(data, outfile): + with open(outfile, 'w') as f: + for item in data: + item = f'{item.strip()}\n' + f.write(item) + + +def write_data(data, slot_dict, intent_dict, outfold, mode, uncased): + intent_file = open(f'{outfold}/{mode}.tsv', 'w') + intent_file.write('sentence\tlabel\n') + slot_file = open(f'{outfold}/{mode}_slots.tsv', 'w') + for tokens, slots, intent in data: + text = ' '.join(tokens) + if uncased: + text = text.lower() + intent_file.write(f'{text}\t{intent_dict[intent]}\n') + slots = [str(slot_dict[slot]) for slot in slots] + slot_file.write(' '.join(slots) + '\n') + intent_file.close() + slot_file.close() + + +def create_dataset(train, dev, slots, intents, uncased, outfold): + os.makedirs(outfold, exist_ok=True) + if 'O' in slots: + slots.remove('O') + slots = sorted(list(slots)) + ['O'] + intents = sorted(list(intents)) + slots = write_vocab(slots, f'{outfold}/dict.slots.csv') + intents = write_vocab(intents, f'{outfold}/dict.intents.csv') + write_data(train, slots, intents, outfold, 'train', uncased) + write_data(dev, slots, intents, outfold, 'test', uncased) + + +def read_csv(file_path): + rows = [] + with open(file_path, 'r') as csvfile: + read_csv = csv.reader(csvfile, delimiter=',') + for row in read_csv: + rows.append(row) + return rows + + +def get_dataset(files, dev_split=0.1): + # entity2value, value2entity = get_entities(files) + data, slots, intents = get_data(files) + if len(data) == 1: + train, dev = partition(data[0], split=dev_split) + else: + train, dev = data[0], data[1] + return train, dev, slots, intents + + +def partition(data, split=0.1): + n = len(data) + n_dev = int(n * split) + dev_idx = set(random.sample(range(n), n_dev)) + dev, train = [], [] + + for i, item in enumerate(data): + if i in dev_idx: + dev.append(item) + else: + train.append(item) + return train, dev + + +def map_entities(entity2value, entities): + for key in entities: + if 'data' in entities[key]: + if key not in entity2value: + entity2value[key] = set([]) + + values = [] + for value in entities[key]['data']: + values.append(value['value']) + values.extend(value['synonyms']) + entity2value[key] = entity2value[key] | set(values) + + return entity2value + + +def get_entities(files): + entity2value = {} + for file in files: + with open(file, 'r') as json_file: + data = json.load(json_file) + entity2value = map_entities(entity2value, data['entities']) + + value2entity = reverse_dict(entity2value) + return entity2value, value2entity + + +def get_data(files): + all_data, all_slots, all_intents = [], set(['O']), set() + for file in files: + file_data = [] + with open(file, 'r') as json_file: + data = json.load(json_file) + for intent in data['intents']: + all_intents.add(intent) + utterances = data['intents'][intent]['utterances'] + for utterance in utterances: + tokens, slots = [], [] + for frag in utterance['data']: + frag_tokens = frag['text'].strip().split() + tokens.extend(frag_tokens) + if 'slot_name' not in frag: + slot = 'O' + else: + slot = frag['slot_name'] + all_slots.add(slot) + slots.extend([slot] * len(frag_tokens)) + file_data.append((tokens, slots, intent)) + all_data.append(file_data) + return all_data, all_slots, all_intents + + +def reverse_dict(entity2value): + value2entity = {} + for entity in entity2value: + for value in entity2value[entity]: + value2entity[value] = entity + return value2entity + + +def get_intent_labels(intent_file): + labels = {} + label = 0 + with open(intent_file, 'r') as f: + for line in f: + intent = line.strip() + labels[intent] = label + label += 1 + return labels + + +def get_stats(lengths): + logging.info('Some stats of the lengths of the sequences:') + lengths = np.asarray(lengths) + logging.info( + f'Min: {np.min(lengths)} | \ + Max: {np.max(lengths)} | \ + Mean: {np.mean(lengths)} | \ + Median: {np.median(lengths)}' + ) + logging.info(f'75 percentile: {np.percentile(lengths, 75):.2f}') + logging.info(f'99 percentile: {np.percentile(lengths, 99):.2f}') + + +def is_whitespace(c): + if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: + return True + return False + + +def write_vocab(items, outfile): + vocab = {} + idx = 0 + with open(outfile, 'w') as f: + for item in items: + f.write(item + '\n') + vocab[item] = idx + idx += 1 + return vocab + + +def get_labels_to_labels_id_mapping(file): + ''' + Reads labels from the file and returns labels to id mapping dictionary + Args: + file: path to file + Returns: + labels to id mapping dictionary + ''' + lines = open(file, 'r').readlines() + lines = [line.strip() for line in lines if line.strip()] + label_ids = {lines[i]: i for i in range(len(lines))} + return label_ids + + +def if_exist(outfold, files): + if not os.path.exists(outfold): + return False + for file in files: + if not os.path.exists(f'{outfold}/{file}'): + return False + return True + + +def remove_punctuation_from_sentence(sentence): + sentence = re.sub('[' + string.punctuation + ']', '', sentence) + sentence = sentence.lower() + return sentence + + +def dataset_to_ids(dataset, tokenizer, cache_ids=False, add_bos_eos=True, cache_data_per_node=False, use_cache=False): + """ + Reads dataset from file line by line, tokenizes each line with tokenizer, + and returns list of lists which corresponds to ids of tokenized strings. + + Args: + dataset (str): path to dataset + tokenizer: tokenizer to convert text into ids + cache_ids (bool): if True, ids are saved to disk as pickle file + with similar name (e.g., data.txt --> data.txt.pkl) + add_bos_eos (bool): whether to add and symbols (e.g., for NMT) + cache_data_per_node (bool): Cache data on local_rank 0. Use when there is not a shared-filesystem. + use_cache (bool): Use cached ids if they exist. + Returns: + ids: list of ids which correspond to tokenized strings of the dataset + """ + + cached_ids_dataset = dataset + str(".pkl") + if use_cache and os.path.isfile(cached_ids_dataset): + logging.info("Loading cached tokenized dataset ...") + ids = pickle.load(open(cached_ids_dataset, "rb")) + else: + logging.info(f"Tokenizing dataset {dataset}...") + data = open(dataset, "rb").readlines() + ids = [] + for sentence in tqdm(data, desc='Tokenizing sentence'): + sent_ids = tokenizer.text_to_ids(sentence.decode("utf-8")) + if add_bos_eos: + sent_ids = [tokenizer.bos_id] + sent_ids + [tokenizer.eos_id] + ids.append(sent_ids) + if cache_ids and ( + not torch.distributed.is_initialized() or (cache_data_per_node and get_envint("LOCAL_RANK", 0) == 0) + ): + logging.info("Caching tokenized dataset ...") + pickle.dump(ids, open(cached_ids_dataset, "wb")) + return ids + + +def get_freq_weights(label_freq): + """ + Goal is to give more weight to the classes with less samples + so as to match the ones with the higher frequencies. We achieve this by + dividing the total frequency by the freq of each label to calculate its weight. + """ + total_size = 0 + for lf in label_freq.values(): + total_size += lf + weighted_slots = {label: (total_size / (len(label_freq) * freq)) for label, freq in label_freq.items()} + return weighted_slots + + +def fill_class_weights(weights, max_id=-1): + """ + Gets a dictionary of labels with their weights and creates a list with size of the labels filled with those weights. + Missing labels in the dictionary would get value 1. + + Args: + weights: dictionary of weights for labels, labels as keys and weights are their values + max_id: the largest label id in the dataset, default=-1 would consider the largest label in the weights dictionary as max_id + Returns: + weights_list: list of weights for labels + """ + if max_id < 0: + max_id = 0 + for l in weights.keys(): + max_id = max(max_id, l) + + all_weights = [1.0] * (max_id + 1) + for i in range(len(all_weights)): + if i in weights: + all_weights[i] = weights[i] + return all_weights + + +def get_vocab(file): + lines = open(file, 'r').readlines() + lines = [line.strip() for line in lines if line.strip()] + labels = {i: lines[i] for i in range(len(lines))} + return labels + + +def find_newlines(contents): + """ + Finds all of the newline positions in a text file. + """ + start = 0 + + while True: + try: + # index and split are much faster than Python for loops + new_start = contents.index(b"\n", start) + line = ( + contents[start:new_start] + .replace(b"\xc2\x99", b" ") + .replace(b"\xc2\xa0", b" ") + .decode("utf-8", errors="ignore") + ) + + if len(line.split()) > 0: + yield start + + start = new_start + 1 + + except ValueError: + break + + +def load_data_indices(idx_file: str, data_file: str, savename: str): + """ + Loads dataset index file if it exsits + """ + data_dir = data_file[: data_file.rfind('/')] + mode = data_file[data_file.rfind('/') + 1 : data_file.rfind('.')] + idx_file = f"{data_dir}/{mode}_{savename}.pkl" + + if os.path.isfile(idx_file): + # If the sentence indices file already exists, load from it + with open(idx_file, "rb") as f: + indices = pickle.load(f) + + return indices, idx_file, data_dir + + return None, idx_file, data_dir diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/__init__.py new file mode 100644 index 0000000000..6d62191d35 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.dialogue_state_tracking.sgd.data_processor import SGDDataProcessor +from nemo.collections.nlp.data.dialogue_state_tracking.sgd.dataset import SGDDataset +from nemo.collections.nlp.data.dialogue_state_tracking.sgd.schema import Schema diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/__init__.py new file mode 100644 index 0000000000..9e32500719 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/data_processor.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/data_processor.py new file mode 100644 index 0000000000..05e0bdfb70 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/data_processor.py @@ -0,0 +1,676 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file contains code artifacts adapted from the original implementation: +https://github.com/google-research/google-research/blob/master/schema_guided_dst/baseline/data_utils.py +""" + +import collections +import json +import os +import pickle +import re +from typing import Dict, List + +import numpy as np + +from nemo.collections.nlp.data.dialogue_state_tracking.sgd.input_example import InputExample +from nemo.utils import logging + +__all__ = ['SGDDataProcessor'] + +FILE_RANGES = { + "sgd_single_domain": {"train": range(1, 44), "dev": range(1, 8), "test": range(1, 12)}, + "sgd_multi_domain": {"train": range(44, 128), "dev": range(8, 21), "test": range(12, 35)}, + "sgd_all": {"train": range(1, 128), "dev": range(1, 21), "test": range(1, 35)}, + "sgd_all_single": {"train": range(1, 128), "dev": range(1, 8), "test": range(1, 12)}, + "multiwoz": {"train": range(1, 18), "dev": range(1, 3), "test": range(1, 3)}, + "debug_sample": {"train": range(1, 2), "dev": range(1, 2), "test": range(1, 2)}, +} + + +class SGDDataProcessor(object): + """Data generator for SGD dialogues.""" + + def __init__( + self, + task_name: str, + data_dir: str, + dialogues_example_dir: str, + tokenizer: object, + schemas: object, + schema_config: Dict[str, int], + subsample: bool = False, + ): + """ + Constructs SGDDataProcessor + Args: + task_name: task name, e.g. "sgd_single_domain" + data_dir: path to data directory + dialogues_example_dir: path to store processed dialogue examples + tokenizer: tokenizer object + schemas: schema object + schema_config: schema configuration + subsample: whether to balance positive and negative samples in dataset + """ + self.data_dir = data_dir + + self._task_name = task_name + self.schemas = schemas + self.schema_config = schema_config + + train_file_range = FILE_RANGES[task_name]["train"] + dev_file_range = FILE_RANGES[task_name]["dev"] + test_file_range = FILE_RANGES[task_name]["test"] + + self._file_ranges = { + "train": train_file_range, + "dev": dev_file_range, + "test": test_file_range, + } + + self._seen_services = { + "train": set(), + "dev": set(), + "test": set(), + } + + self._tokenizer = tokenizer + self._subsample = subsample + self._dialogues_example_dir = dialogues_example_dir + + self.dial_files = {} + + # slots_relation_list.np would contain the candidate list of slots for each (service, slot) which would be + # looked into when a switch between two services happens in the dialogue and we can not find any value for a slot in the current user utterance. + # This file would get generated from the dialogues in the training set. + self.slots_relation_file = os.path.join(dialogues_example_dir, f"{task_name}_train_slots_relation_list.np") + for dataset in ["train", "dev", "test"]: + # Process dialogue files + dial_file = f"{task_name}_{dataset}_examples.processed" + dial_file = os.path.join(dialogues_example_dir, dial_file) + self.dial_files[(task_name, dataset)] = dial_file + + dialog_paths = SGDDataProcessor.get_dialogue_files(data_dir, dataset, task_name) + dialogs = SGDDataProcessor.load_dialogues(dialog_paths) + for dialog in dialogs: + self._seen_services[dataset].update(set(dialog['services'])) + + def save_dialog_examples(self, overwrite_dial_files: bool): + """ + Preprocesses dialogues and saves to disk. + Args: + overwrite_dial_files: whether or not to overwrite saved file if already exists + """ + for dataset in ["train", "dev", "test"]: + dial_file = self.dial_files[(self._task_name, dataset)] + if not os.path.exists(dial_file) or overwrite_dial_files: + logging.info(f"Start generating the dialogue examples for {dataset} dataset.") + if not os.path.exists(self._dialogues_example_dir): + os.makedirs(self._dialogues_example_dir) + dial_examples, slots_relation_list = self._generate_dialog_examples( + dataset, self.schemas, self._subsample + ) + for ex in dial_examples: + del ex.service_schema + del ex._tokenizer + del ex.schema_config + del ex.user_utterance + del ex.categorical_slot_id + del ex.system_utterance + del ex.noncategorical_slot_id + del ex.categorical_slot_value_id + del ex.requested_slot_id + del ex.intent_id + with open(dial_file, "wb") as f: + np.save(f, dial_examples) + + if dataset == "train": + with open(self.slots_relation_file, "wb") as f: + pickle.dump(slots_relation_list, f) + logging.info(f"The slot carry-over list for train set is stored at {self.slots_relation_file}") + + logging.info(f"The dialogue examples for {dataset} dataset saved at {dial_file}") + logging.info(f"Finish generating the dialogue examples for {dataset} dataset.") + + def get_dialog_examples(self, dataset_split: str) -> List[object]: + """ + Loads preprocessed dialogue examples from disk. + Args: + dataset_split: dataset split + Returns: + dial_examples: list of InputExample's. + """ + if (self._task_name, dataset_split) not in self.dial_files or not os.path.exists( + self.dial_files[(self._task_name, dataset_split)] + ): + raise ValueError( + f"{dataset_split} dialogue examples were not processed for {self._task_name} task. Re-initialize SGDDataProcessor and add {dataset_split} dataset split to datasets arg." + ) + dial_file = self.dial_files[(self._task_name, dataset_split)] + logging.info(f"Loading dialogue examples from {dial_file}.") + + with open(dial_file, "rb") as f: + dial_examples = np.load(f, allow_pickle=True) + f.close() + + if not os.path.exists(self.slots_relation_file): + raise ValueError( + f"Slots relation file {self.slots_relation_file} does not exist. It is needed for the carry-over mechanism of state tracker for switches between services." + ) + if os.path.getsize(self.slots_relation_file) > 0: + with open(self.slots_relation_file, "rb") as f: + self.schemas._slots_relation_list = pickle.load(f) + logging.info( + f"Loaded the slot relation list for value carry-over between services from {self.slots_relation_file}." + ) + + return dial_examples + + def get_seen_services(self, dataset_split: str): + """ + Returns list of seen services, i.e. both in given and training split + Args: + dataset_split: data split + Returns: + seen_services: list of seen services + """ + seen_services = self._seen_services[dataset_split] + return seen_services + + def _generate_dialog_examples(self, dataset_split: str, schemas: object, subsample: bool): + """ + Returns a list of `InputExample`s of the data splits' dialogues. + Args: + dataset_split: data split, can be "train", "dev", or "test". + schemas: schema for all services of all datasets + subsample: whether to balance postive and negative samples in the dataset + Returns: + examples: a list of `InputExample`s. + """ + logging.info(f'Creating examples and slot relation list from the dialogues started...') + dialog_paths = [ + os.path.join(self.data_dir, dataset_split, "dialogues_{:03d}.json".format(i)) + for i in self._file_ranges[dataset_split] + ] + dialogs = SGDDataProcessor.load_dialogues(dialog_paths) + + examples = [] + slot_carryover_candlist = collections.defaultdict(int) + for dialog_idx, dialog in enumerate(dialogs): + if dialog_idx % 1000 == 0: + logging.info(f'Processed {dialog_idx} dialogues.') + examples.extend( + self._create_examples_from_dialog(dialog, schemas, dataset_split, slot_carryover_candlist, subsample) + ) + + slots_relation_list = collections.defaultdict(list) + for slots_relation, relation_size in slot_carryover_candlist.items(): + if relation_size > 0: + slots_relation_list[(slots_relation[0], slots_relation[1])].append( + (slots_relation[2], slots_relation[3], relation_size) + ) + slots_relation_list[(slots_relation[2], slots_relation[3])].append( + (slots_relation[0], slots_relation[1], relation_size) + ) + + return examples, slots_relation_list + + def _create_examples_from_dialog( + self, dialog: dict, schemas: object, dataset_split: str, slot_carryover_candlist: dict, subsample: bool + ): + """ + Create examples for every turn in the dialogue. + Args: + dialog: dialogue example + schemas: schema for all services of all datasets + dataset_split: data split + slot_carryover_candlist: a dictionary to keep and count the number of carry-over cases between two slots from two different services + subsample: whether to balance postive and negative samples in the dataset + Returns: + examples: a list of `InputExample`s. + """ + dialog_id = dialog["dialogue_id"] + prev_states = {} + examples = [] + for turn_idx, turn in enumerate(dialog["turns"]): + # Generate an example for every frame in every user turn. + if turn["speaker"] == "USER": + user_utterance = turn["utterance"] + user_frames = {f["service"]: f for f in turn["frames"]} + if turn_idx > 0: + system_turn = dialog["turns"][turn_idx - 1] + system_utterance = system_turn["utterance"] + system_frames = {f["service"]: f for f in system_turn["frames"]} + else: + system_utterance = "" + system_frames = {} + + turn_id = "{}-{}-{:02d}".format(dataset_split, dialog_id, turn_idx) + turn_examples, prev_states, slot_carryover_values = self._create_examples_from_turn( + turn_id, + system_utterance, + user_utterance, + system_frames, + user_frames, + prev_states, + schemas, + subsample, + ) + examples.extend(turn_examples) + + for value, slots_list in slot_carryover_values.items(): + if value in ["True", "False"]: + continue + if len(slots_list) > 1: + for service1, slot1 in slots_list: + for service2, slot2 in slots_list: + if service1 == service2: + continue + if service1 > service2: + service1, service2 = service2, service1 + slot1, slot2 = slot2, slot1 + slot_carryover_candlist[(service1, slot1, service2, slot2)] += 1 + return examples + + def _get_state_update(self, current_state: dict, prev_state: dict) -> dict: + """ + Updates dialogue state + Args: + current_state: slot values pairs for the current dialogue turn + prev_state: slot values pairs for the previous dialogue turns + Returns: + state_update: slot values pairs that are added/updated during the current dialogue turn + """ + state_update = dict(current_state) + for slot, values in current_state.items(): + if slot in prev_state and prev_state[slot][0] in values: + # Remove the slot from state if its value didn't change. + state_update.pop(slot) + return state_update + + def _create_examples_from_turn( + self, + turn_id: int, + system_utterance: str, + user_utterance: str, + system_frames: dict, + user_frames: dict, + prev_states: dict, + schemas: object, + subsample: bool, + ): + """ + Creates an example for each frame in the user turn. + Args: + turn_id: turn number + system_utterance: last system utterance + user_utterance: lst user utterance + system_frames: all system utterances and slot - slot value pairs + user_frames: all user utterances and slot - slot value pairs + prev_states: slot - slot value pairs from the previous turns + schemas: schema for all services of all datasets + subsample: whether to balance postive and negative samples in the dataset + Returns: + examples: a list of `InputExample`s. + prev_states: updated dialogue state e.g. {'Restaurants_1': {'city': ['San Jose'], 'cuisine': ['American']}} + """ + user_tokens, user_alignments, user_inv_alignments = self._tokenize(user_utterance) + system_tokens, system_alignments, system_inv_alignments = self._tokenize(system_utterance) + system_user_utterance = system_utterance + ' ' + user_utterance + system_user_tokens, system_user_alignments, system_user_inv_alignments = self._tokenize(system_user_utterance) + states = {} + + examples = [] + slot_carryover_values = collections.defaultdict(list) + for service, user_frame in user_frames.items(): + + base_example = InputExample(schema_config=self.schema_config, tokenizer=self._tokenizer,) + base_example.service_schema = schemas.get_service_schema(service) + base_example.service_id = schemas.get_service_schema(service).service_id + system_frame = system_frames.get(service, None) + state = user_frame["state"]["slot_values"] + state_update = self._get_state_update(state, prev_states.get(service, {})) + states[service] = state + dataset_split, dialog_id, turn_id_ = turn_id.split('-') + dialog_id_1, dialog_id_2 = dialog_id.split('_') + + base_example.example_id = f"{turn_id}-{service}" + base_example.example_id_num = [ + int(dialog_id_1), + int(dialog_id_2), + int(turn_id_), + schemas.get_service_id(service), + ] + + for model_task in range(self.schema_config["NUM_TASKS"]): + if model_task == 0: + for intent_id, intent in enumerate(schemas.get_service_schema(service).intents): + task_example = base_example.make_copy() + task_example.task_mask[model_task] = 1 + task_example.intent_id = intent_id + task_example.example_id += f"-{model_task}-{intent_id}-0" + task_example.example_id_num.extend([model_task, intent_id, 0]) + intent_description = ( + intent + " " + schemas.get_service_schema(service).intent_descriptions[intent] + ) + intent_tokens, intent_alignments, intent_inv_alignments = self._tokenize(intent_description) + task_example.add_utterance_features( + intent_tokens, + intent_inv_alignments, + system_user_tokens, + system_user_inv_alignments, + intent_description, + system_user_utterance, + ) + task_example.add_intents(user_frame) + examples.append(task_example) + + if model_task == 1: + for slot_id, slot in enumerate(schemas.get_service_schema(service).slots): + task_example = base_example.make_copy() + task_example.task_mask[model_task] = 1 + task_example.requested_slot_id = slot_id + task_example.example_id += f"-{model_task}-{slot_id}-0" + task_example.example_id_num.extend([model_task, slot_id, 0]) + slot_description = slot + " " + schemas.get_service_schema(service).slot_descriptions[slot] + slot_tokens, slot_alignments, slot_inv_alignments = self._tokenize(slot_description) + task_example.add_utterance_features( + slot_tokens, + slot_inv_alignments, + user_tokens, + user_inv_alignments, + slot_description, + user_utterance, + ) + task_example.add_requested_slots(user_frame) + examples.append(task_example) + if model_task == 2: + off_slots = [] + on_slots = [] + for slot_id, slot in enumerate(schemas.get_service_schema(service).categorical_slots): + task_example = base_example.make_copy() + task_example.task_mask[model_task] = 1 + + # assert task_example.task_mask == [0, 0, 1, 0, 0, 0] + task_example.categorical_slot_id = slot_id + task_example.example_id += f"-{model_task}-{slot_id}-0" + task_example.example_id_num.extend([model_task, slot_id, 0]) + slot_description = slot + " " + schemas.get_service_schema(service).slot_descriptions[slot] + slot_tokens, slot_alignments, slot_inv_alignments = self._tokenize(slot_description) + task_example.add_utterance_features( + slot_tokens, + slot_inv_alignments, + system_user_tokens, + system_user_inv_alignments, + slot_description, + system_user_utterance, + ) + task_example.add_categorical_slots(state_update) + if task_example.categorical_slot_status == 0: + off_slots.append(task_example) + else: + on_slots.append(task_example) + examples.append(task_example) + old_example = task_example + + for value_id, value in enumerate( + schemas.get_service_schema(service).get_categorical_slot_values(slot) + ): + if dataset_split != 'train' or task_example.categorical_slot_status == 1: + task_example = old_example.make_copy_of_categorical_features() + task_example.task_mask[3] = 1 + # assert task_example.task_mask == [0, 0, 0, 1, 0, 0] + task_example.categorical_slot_id = slot_id + task_example.categorical_slot_value_id = value_id + task_example.example_id = base_example.example_id + f"-3-{slot_id}-{value_id}" + task_example.example_id_num = base_example.example_id_num + [3, slot_id, value_id] + slot_description = slot + " " + value # add slot description + slot_tokens, slot_alignments, slot_inv_alignments = self._tokenize(slot_description) + task_example.add_utterance_features( + slot_tokens, + slot_inv_alignments, + system_user_tokens, + system_user_inv_alignments, + slot_description, + system_user_utterance, + ) + task_example.add_categorical_slots(state_update) + assert task_example.categorical_slot_status == old_example.categorical_slot_status + examples.append(task_example) + + if dataset_split == 'train' and subsample: + num_on_slots = len(on_slots) + examples.extend( + np.random.choice(off_slots, replace=False, size=min(max(num_on_slots, 1), len(off_slots))) + ) + else: + examples.extend(off_slots) + + if model_task == 4: # noncat slot status + off_slots = [] + on_slots = [] + for slot_id, slot in enumerate(schemas.get_service_schema(service).non_categorical_slots): + task_example = base_example.make_copy() + task_example.task_mask[model_task] = 1 + # assert task_example.task_mask == [0, 0, 0, 0, 1, 0] + task_example.noncategorical_slot_id = slot_id + task_example.example_id += f"-{model_task}-{slot_id}-0" + task_example.example_id_num.extend([model_task, slot_id, 0]) + slot_description = slot + " " + schemas.get_service_schema(service).slot_descriptions[slot] + slot_tokens, slot_alignments, slot_inv_alignments = self._tokenize(slot_description) + task_example.add_utterance_features( + slot_tokens, + slot_inv_alignments, + system_user_tokens, + system_user_inv_alignments, + slot_description, + system_user_utterance, + ) + + user_span_boundaries = self._find_subword_indices( + state_update, + user_utterance, + user_frame["slots"], + user_alignments, + user_tokens, + 2 + len(slot_tokens) + len(system_tokens), + ) + if system_frame is not None: + system_span_boundaries = self._find_subword_indices( + state_update, + system_utterance, + system_frame["slots"], + system_alignments, + system_tokens, + 2 + len(slot_tokens), + ) + else: + system_span_boundaries = {} + task_example.add_noncategorical_slots( + state_update, user_span_boundaries, system_span_boundaries + ) + if task_example.noncategorical_slot_status == 0: + off_slots.append(task_example) + else: + on_slots.append(task_example) + examples.append(task_example) + + if dataset_split != 'train' or task_example.noncategorical_slot_status == 1: + task_example = task_example.make_copy_of_non_categorical_features() + task_example.task_mask[5] = 1 + # assert task_example.task_mask == [0, 0, 0, 0, 0, 1] + task_example.example_id = base_example.example_id + f"-5-{slot_id}-0" + task_example.example_id_num = base_example.example_id_num + [5, slot_id, 0] + examples.append(task_example) + if dataset_split == 'train' and subsample: + num_on_slots = len(on_slots) + examples.extend( + np.random.choice(off_slots, replace=False, size=min(max(num_on_slots, 1), len(off_slots))) + ) + else: + examples.extend(off_slots) + + if service not in prev_states and int(turn_id_) > 0: + for slot_name, values in state_update.items(): + for value in values: + slot_carryover_values[value].append((service, slot_name)) + for prev_service, prev_slot_value_list in prev_states.items(): + if prev_service == service: + continue + if prev_service in state: + prev_slot_value_list = state[prev_service] + for prev_slot_name, prev_values in prev_slot_value_list.items(): + for prev_value in prev_values: + slot_carryover_values[prev_value].append((prev_service, prev_slot_name)) + + return examples, states, slot_carryover_values + + def _find_subword_indices( + self, + slot_values: dict, + utterance: str, + char_slot_spans: dict, + alignments: List[int], + subwords: List[str], + bias: int, + ) -> dict: + """ + Find indices for subwords corresponding to slot values. + Args: + slot_values: slot - slot value pairs + utterance: utterance + char_slot_spans: char - slot spans + alignments: alignments + subwords: subtokens mapping + bias: offset + Returns: + span_boundaries: span boundaries + """ + span_boundaries = {} + for slot, values in slot_values.items(): + # Get all values present in the utterance for the specified slot. + value_char_spans = {} + for slot_span in char_slot_spans: + if slot_span["slot"] == slot: + value = utterance[slot_span["start"] : slot_span["exclusive_end"]] + start_tok_idx = alignments[slot_span["start"]] + end_tok_idx = alignments[slot_span["exclusive_end"] - 1] + if 0 <= start_tok_idx < len(subwords): + end_tok_idx = min(end_tok_idx, len(subwords) - 1) + value_char_spans[value] = (start_tok_idx + bias, end_tok_idx + bias) + for v in values: + if v in value_char_spans: + span_boundaries[slot] = value_char_spans[v] + break + return span_boundaries + + def _tokenize(self, utterance: str): + """ + Tokenize the utterance + + Args: + utterance: A string containing the utterance to be tokenized. + + Returns: + bert_tokens: A list of tokens obtained by word-piece tokenization of the + utterance. + alignments: A dict mapping indices of characters corresponding to start + and end positions of words (not subwords) to corresponding indices in + bert_tokens list. + inverse_alignments: A list of size equal to bert_tokens. Each element is a + tuple containing the index of the starting and inclusive ending + character of the word corresponding to the subword. This list is used + during inference to map word-piece indices to spans in the original + utterance. + """ + # utterance = tokenization.convert_to_unicode(utterance) + + # After _naive_tokenize, spaces and punctuation marks are all retained, i.e. + # direct concatenation of all the tokens in the sequence will be the + # original string. + tokens = SGDDataProcessor._naive_tokenize(utterance) + # ['I', ' ', 'am', ' ', 'feeling', ' ', 'hungry', ' ', 'so', ' ', 'I', ' ', 'would', ' ', 'like', ' ', 'to', ' ', 'find', ' ', 'a', ' ', 'place', ' ', 'to', ' ', 'eat', '.'] + # Filter out empty tokens and obtain aligned character index for each token. + alignments = {} + char_index = 0 + bert_tokens = ( + [] + ) # ['I', 'am', 'feeling', 'hungry', 'so', 'I', 'would', 'like', 'to', 'find', 'a', 'place', 'to', 'eat', '.'] + # These lists store inverse alignments to be used during inference. + bert_tokens_start_chars = [] + bert_tokens_end_chars = [] + for token in tokens: + if token.strip(): + subwords = self._tokenizer.text_to_tokens(token) + # Store the alignment for the index of starting character and the + # inclusive ending character of the token. + alignments[char_index] = len(bert_tokens) + bert_tokens_start_chars.extend([char_index] * len(subwords)) + bert_tokens.extend(subwords) + # The inclusive ending character index corresponding to the word. + inclusive_char_end = char_index + len(token) - 1 + alignments[inclusive_char_end] = len(bert_tokens) - 1 + bert_tokens_end_chars.extend([inclusive_char_end] * len(subwords)) + char_index += len(token) + inverse_alignments = list(zip(bert_tokens_start_chars, bert_tokens_end_chars)) + return bert_tokens, alignments, inverse_alignments + + @classmethod + def _naive_tokenize(cls, s: str): + """ + Tokenizes a string, separating words, spaces and punctuations. + Args: + s: a string + Returns: + seq_tok: list of words, spaces and punctuations from the string + """ + # Spaces and punctuation marks are all retained, i.e. direct concatenation + # of all the tokens in the sequence will be the original string. + seq_tok = [tok for tok in re.split(r"([^a-zA-Z0-9])", s) if tok] + return seq_tok + + @classmethod + def load_dialogues(cls, dialog_json_filepaths: List[str]) -> List[dict]: + """ + Obtain the list of all dialogues from specified json files. + Args: + dialog_json_filepaths: list of json files + Returns: + dialogs: the list of all dialogues + """ + dialogs = [] + for dialog_json_filepath in sorted(dialog_json_filepaths): + with open(dialog_json_filepath, 'r') as f: + dialogs.extend(json.load(f)) + f.close() + return dialogs + + @classmethod + def get_dialogue_files(cls, data_dir: str, dataset_split: str, task_name: str): + """ + Obtain the list of all dialogue json files + Args: + data_dir: path to the data folder + dataset_split: data split + task_name: SGD task name, see keys of the FILE_RANGES + Returns: + dialog: the list of all dialogue json files paths + """ + return [ + os.path.join(data_dir, dataset_split, 'dialogues_{:03d}.json'.format(fid)) + for fid in FILE_RANGES[task_name][dataset_split] + ] diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/dataset.py new file mode 100644 index 0000000000..74d90a7a87 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/dataset.py @@ -0,0 +1,85 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file contains code artifacts adapted from the original implementation: +https://github.com/google-research/google-research/blob/master/schema_guided_dst +""" +from typing import Dict, Optional + +import numpy as np + +from nemo.core.classes import Dataset +from nemo.core.neural_types import ChannelType, LabelsType, NeuralType + +__all__ = ['SGDDataset'] + + +class SGDDataset(Dataset): + """Processes SGD dataset""" + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports.""" + return { + "example_id_num": NeuralType(('B', 'T'), ChannelType()), + "service_id": NeuralType(('B'), ChannelType()), + "utterance_ids": NeuralType(('B', 'T'), ChannelType()), + "token_type_ids": NeuralType(('B', 'T'), ChannelType()), # utterance segment + "attention_mask": NeuralType(('B', 'T'), ChannelType()), # utterance mask + "intent_status": NeuralType(('B'), LabelsType()), + "requested_slot_status": NeuralType(('B'), LabelsType()), + "categorical_slot_status": NeuralType(('B'), LabelsType()), + "categorical_slot_value_status": NeuralType(('B'), LabelsType()), + "noncategorical_slot_status": NeuralType(('B'), LabelsType()), + "noncategorical_slot_value_start": NeuralType(('B'), LabelsType()), + "noncategorical_slot_value_end": NeuralType(('B'), LabelsType()), + "start_char_idx": NeuralType(('B', 'T'), LabelsType()), + "end_char_idx": NeuralType(('B', 'T'), LabelsType()), + "task_mask": NeuralType(('B', 'T'), ChannelType()), + } + + def __init__(self, dataset_split: str, dialogues_processor: object): + """ Constructor + Args: + dataset_split: dataset split + dialogues_processor: Data generator for SGD dialogues + """ + self.features = dialogues_processor.get_dialog_examples(dataset_split) + + def __len__(self): + return len(self.features) + + def __getitem__(self, idx: int): + ex = self.features[idx] + service_id = ex.service_id + + return ( + np.array(ex.example_id_num), + np.array(service_id), + np.array(ex.utterance_ids), + np.array(ex.utterance_segment), + np.array(ex.utterance_mask, dtype=np.long), + np.array(ex.intent_status, dtype=np.float32), + np.array(ex.requested_slot_status, dtype=np.float32), + np.array(ex.categorical_slot_status), + np.array(ex.categorical_slot_value_status, dtype=np.float32), + np.array(ex.noncategorical_slot_status), + np.array(ex.noncategorical_slot_value_start), + np.array(ex.noncategorical_slot_value_end), + np.array(ex.start_char_idx), # noncat_alignment_start + np.array(ex.end_char_idx), # noncat_alignment_end + np.array(ex.task_mask), # noncat_alignment_end + ) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/evaluate.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/evaluate.py new file mode 100644 index 0000000000..6d189da786 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/evaluate.py @@ -0,0 +1,294 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Evaluate predictions JSON file, w.r.t. ground truth file. +This file contains code artifacts adapted from the original implementation: +https://github.com/google-research/google-research/blob/master/schema_guided_dst/evaluate.py +""" + +import collections +import glob +import json +import os + +import numpy as np + +from nemo.collections.nlp.data.dialogue_state_tracking.sgd.metrics import ( + ACTIVE_INTENT_ACCURACY, + JOINT_CAT_ACCURACY, + JOINT_GOAL_ACCURACY, + JOINT_NONCAT_ACCURACY, + NAN_VAL, + REQUESTED_SLOTS_F1, + REQUESTED_SLOTS_PRECISION, + REQUESTED_SLOTS_RECALL, + SLOT_TAGGING_F1, + SLOT_TAGGING_PRECISION, + SLOT_TAGGING_RECALL, + get_active_intent_accuracy, + get_average_and_joint_goal_accuracy, + get_requested_slots_f1, + get_slot_tagging_f1, +) +from nemo.utils import logging + +__all__ = ['get_in_domain_services'] + +ALL_SERVICES = "#ALL_SERVICES" +SEEN_SERVICES = "#SEEN_SERVICES" +UNSEEN_SERVICES = "#UNSEEN_SERVICES" + +# Name of the file containing all predictions and their corresponding frame metrics. +PER_FRAME_OUTPUT_FILENAME = "dialogues_and_metrics.json" + + +def get_service_set(schema_path: str) -> set: + """ + Get the set of all services present in a schema. + Args: + schema_path: schema file path + Returns: + service_set: set of services in file + """ + service_set = set() + with open(schema_path) as f: + schema = json.load(f) + for service in schema: + service_set.add(service["service_name"]) + f.close() + return service_set + + +def get_in_domain_services(schema_path: str, service_set: set) -> set: + """Get the set of common services between a schema and set of services. + Args: + schema_path: path to schema file + service_set: set of services + Returns: + joint_services: joint services between schema path file and service set + """ + joint_services = get_service_set(schema_path) & service_set + return joint_services + + +def get_dataset_as_dict(file_path_patterns) -> dict: + """Read the DSTC8/SGD json dialogue data as dictionary with dialog ID as keys. + Args: + file_path_patterns: list or directory of files + Returns: + dataset_dict: dataset dictionary with dialog ID as keys + """ + dataset_dict = {} + if isinstance(file_path_patterns, list): + list_fp = file_path_patterns + else: + list_fp = sorted(glob.glob(file_path_patterns)) + for fp in list_fp: + if PER_FRAME_OUTPUT_FILENAME in fp: + continue + logging.debug("Loading file: %s", fp) + with open(fp) as f: + data = json.load(f) + if isinstance(data, list): + for dial in data: + dataset_dict[dial["dialogue_id"]] = dial + elif isinstance(data, dict): + dataset_dict.update(data) + f.close() + return dataset_dict + + +def get_metrics( + dataset_ref: dict, + dataset_hyp: dict, + service_schemas: dict, + in_domain_services: set, + joint_acc_across_turn: bool, + use_fuzzy_match: bool, +): + """Calculate the DSTC8/SGD metrics. + Args: + dataset_ref: The ground truth dataset represented as a dict mapping dialogue id to the corresponding dialogue. + dataset_hyp: The predictions in the same format as `dataset_ref`. + service_schemas: A dict mapping service name to the schema for the service. + in_domain_services: The set of services which are present in the training set. + joint_acc_across_turn: Whether to compute joint accuracy across turn instead of across service. Should be set to True when conducting multiwoz style evaluation. + use_fuzzy_match: Whether to use fuzzy string matching when comparing non-categorical slot values. Should be set to False when conducting multiwoz style evaluation. + + Returns: + all_metric_aggregate: A dict mapping a metric collection name to a dict containing the values + for various metrics. Each metric collection aggregates the metrics across a specific set of frames in the dialogues. + per_frame_metric: metrics aggregated for each frame + """ + # Metrics can be aggregated in various ways, eg over all dialogues, only for + # dialogues containing unseen services or for dialogues corresponding to a + # single service. This aggregation is done through metric_collections, which + # is a dict mapping a collection name to a dict, which maps a metric to a list + # of values for that metric. Each value in this list is the value taken by + # the metric on a frame. + metric_collections = collections.defaultdict(lambda: collections.defaultdict(list)) + + # Ensure the dialogs in dataset_hyp also occur in dataset_ref. + assert set(dataset_hyp.keys()).issubset(set(dataset_ref.keys())) + logging.debug("len(dataset_hyp)=%d, len(dataset_ref)=%d", len(dataset_hyp), len(dataset_ref)) + + # Store metrics for every frame for debugging. + per_frame_metric = {} + + for dial_id, dial_hyp in dataset_hyp.items(): + dial_ref = dataset_ref[dial_id] + + if set(dial_ref["services"]) != set(dial_hyp["services"]): + raise ValueError( + "Set of services present in ground truth and predictions don't match " + "for dialogue with id {}".format(dial_id) + ) + + joint_metrics = [JOINT_GOAL_ACCURACY, JOINT_CAT_ACCURACY, JOINT_NONCAT_ACCURACY] + for turn_id, (turn_ref, turn_hyp) in enumerate(zip(dial_ref["turns"], dial_hyp["turns"])): + metric_collections_per_turn = collections.defaultdict(lambda: collections.defaultdict(lambda: 1.0)) + if turn_ref["speaker"] != turn_hyp["speaker"]: + raise ValueError("Speakers don't match in dialogue with id {}".format(dial_id)) + + # Skip system turns because metrics are only computed for user turns. + if turn_ref["speaker"] != "USER": + continue + + if turn_ref["utterance"] != turn_hyp["utterance"]: + logging.error("Ref utt: %s", turn_ref["utterance"]) + logging.error("Hyp utt: %s", turn_hyp["utterance"]) + raise ValueError("Utterances don't match for dialogue with id {}".format(dial_id)) + + hyp_frames_by_service = {frame["service"]: frame for frame in turn_hyp["frames"]} + + # Calculate metrics for each frame in each user turn. + for frame_ref in turn_ref["frames"]: + service_name = frame_ref["service"] + if service_name not in hyp_frames_by_service: + raise ValueError( + "Frame for service {} not found in dialogue with id {}".format(service_name, dial_id) + ) + service = service_schemas[service_name] + frame_hyp = hyp_frames_by_service[service_name] + + active_intent_acc = get_active_intent_accuracy(frame_ref, frame_hyp) + slot_tagging_f1_scores = get_slot_tagging_f1(frame_ref, frame_hyp, turn_ref["utterance"], service) + requested_slots_f1_scores = get_requested_slots_f1(frame_ref, frame_hyp) + goal_accuracy_dict = get_average_and_joint_goal_accuracy( + frame_ref, frame_hyp, service, use_fuzzy_match + ) + + frame_metric = { + ACTIVE_INTENT_ACCURACY: active_intent_acc, + REQUESTED_SLOTS_F1: requested_slots_f1_scores.f1, + REQUESTED_SLOTS_PRECISION: requested_slots_f1_scores.precision, + REQUESTED_SLOTS_RECALL: requested_slots_f1_scores.recall, + } + if slot_tagging_f1_scores is not None: + frame_metric[SLOT_TAGGING_F1] = slot_tagging_f1_scores.f1 + frame_metric[SLOT_TAGGING_PRECISION] = slot_tagging_f1_scores.precision + frame_metric[SLOT_TAGGING_RECALL] = slot_tagging_f1_scores.recall + frame_metric.update(goal_accuracy_dict) + + frame_id = "{:s}-{:03d}-{:s}".format(dial_id, turn_id, frame_hyp["service"]) + per_frame_metric[frame_id] = frame_metric + # Add the frame-level metric result back to dialogues. + frame_hyp["metrics"] = frame_metric + + # Get the domain name of the service. + domain_name = frame_hyp["service"].split("_")[0] + domain_keys = [ALL_SERVICES, frame_hyp["service"], domain_name] + if frame_hyp["service"] in in_domain_services: + domain_keys.append(SEEN_SERVICES) + + else: + domain_keys.append(UNSEEN_SERVICES) + for domain_key in domain_keys: + for metric_key, metric_value in frame_metric.items(): + if metric_value != NAN_VAL: + if joint_acc_across_turn and metric_key in joint_metrics: + metric_collections_per_turn[domain_key][metric_key] *= metric_value + else: + metric_collections[domain_key][metric_key].append(metric_value) + if joint_acc_across_turn: + # Conduct multiwoz style evaluation that computes joint goal accuracy + # across all the slot values of all the domains for each turn. + for domain_key in metric_collections_per_turn: + for metric_key, metric_value in metric_collections_per_turn[domain_key].items(): + metric_collections[domain_key][metric_key].append(metric_value) + + all_metric_aggregate = {} + for domain_key, domain_metric_vals in metric_collections.items(): + domain_metric_aggregate = {} + for metric_key, value_list in domain_metric_vals.items(): + if value_list: + # Metrics are macro-averaged across all frames. + domain_metric_aggregate[metric_key] = round(float(np.mean(value_list)) * 100.0, 2) + else: + domain_metric_aggregate[metric_key] = NAN_VAL + all_metric_aggregate[domain_key] = domain_metric_aggregate + return all_metric_aggregate, per_frame_metric + + +def evaluate( + prediction_dir: str, + data_dir: str, + eval_dataset: str, + in_domain_services: set, + joint_acc_across_turn: bool, + use_fuzzy_match: bool, +) -> dict: + """Calculate the DSTC8/SGD metrics for given data. + + Args: + prediction_dir: prediction location + data_dir: ground truth data location. + eval_dataset: evaluation data split + in_domain_services: The set of services which are present in the training set. + joint_acc_across_turn: Whether to compute joint goal accuracy across turn instead of across service. Should be set to True when conducting multiwoz style evaluation. + use_fuzzy_match: Whether to use fuzzy string matching when comparing non-categorical slot values. Should be set to False when conducting multiwoz style evaluation. + + Returns: + A dict mapping a metric collection name to a dict containing the values + for various metrics for all dialogues and all services + """ + + with open(os.path.join(data_dir, eval_dataset, "schema.json")) as f: + eval_services = {} + list_services = json.load(f) + for service in list_services: + eval_services[service["service_name"]] = service + f.close() + + dataset_ref = get_dataset_as_dict(os.path.join(data_dir, eval_dataset, "dialogues_*.json")) + dataset_hyp = get_dataset_as_dict(os.path.join(prediction_dir, "*.json")) + + # has ALLSERVICE, SEEN_SERVICES, UNSEEN_SERVICES, SERVICE, DOMAIN + all_metric_aggregate, _ = get_metrics( + dataset_ref, dataset_hyp, eval_services, in_domain_services, joint_acc_across_turn, use_fuzzy_match + ) + if SEEN_SERVICES in all_metric_aggregate: + logging.info(f'Dialog metrics for {SEEN_SERVICES} : {sorted(all_metric_aggregate[SEEN_SERVICES].items())}') + if UNSEEN_SERVICES in all_metric_aggregate: + logging.info(f'Dialog metrics for {UNSEEN_SERVICES}: {sorted(all_metric_aggregate[UNSEEN_SERVICES].items())}') + if ALL_SERVICES in all_metric_aggregate: + logging.info(f'Dialog metrics for {ALL_SERVICES} : {sorted(all_metric_aggregate[ALL_SERVICES].items())}') + + # Write the per-frame metrics values with the corrresponding dialogue frames. + with open(os.path.join(prediction_dir, PER_FRAME_OUTPUT_FILENAME), "w") as f: + json.dump(dataset_hyp, f, indent=2, separators=(",", ": ")) + f.close() + return all_metric_aggregate[ALL_SERVICES] diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/input_example.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/input_example.py new file mode 100644 index 0000000000..619ae1b147 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/input_example.py @@ -0,0 +1,399 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file contains code artifacts adapted from the original implementation: +https://github.com/google-research/google-research/blob/master/schema_guided_dst/baseline/data_utils.py +""" + +from typing import List + +from nemo.utils import logging + +__all__ = ['InputExample', 'STR_DONTCARE', 'STATUS_OFF', 'STATUS_ACTIVE', 'STATUS_DONTCARE'] + +STR_DONTCARE = "dontcare" + +# These are used to represent the status of slots (off, active, dontcare) and +# intents (off, active) in dialogue state tracking. +STATUS_OFF = 0 +STATUS_ACTIVE = 1 +STATUS_DONTCARE = 2 + + +class InputExample(object): + """An example for training/inference.""" + + def __init__( + self, + schema_config: dict, + tokenizer: object, + service_schema: object = None, + example_id: str = "NONE", + example_id_num: List[int] = [], + ): + """ + Constructs an InputExample. + Args: + schema_config: configuration + tokenizer: tokenizer object + service_schema: A ServiceSchema object wrapping the schema for the service + corresponding to this example. + example_id: Unique identifier for the example, like: 'train-1_00000-00-Restaurants_1' + example_id_num: dialogue_id and turn_id combined and service id combined into a list of ints, + like: [1, 0, 0, 18] + """ + self.schema_config = schema_config + self.service_schema = service_schema + self.service_id = None + if service_schema: + self.service_id = service_schema.service_id + self.example_id = example_id + self.example_id_num = example_id_num + self._max_seq_length = schema_config["MAX_SEQ_LENGTH"] + self._tokenizer = tokenizer + if self._tokenizer is None: + raise ValueError("Must specify tokenizer") + + self.user_utterance = '' + self.system_utterance = '' + # The id of each subword in the vocabulary for BERT. + self.utterance_ids = [0] * self._max_seq_length + # Denotes the identity of the sequence. Takes values 0 (schema description) and 1 (system and user utterance). + self.utterance_segment = [0] * self._max_seq_length + # Mask which takes the value 0 for padded tokens and 1 otherwise. + self.utterance_mask = [0] * self._max_seq_length + # Start and inclusive end character indices in the original utterance + # corresponding to the tokens. This is used to obtain the character indices + # from the predicted subword indices during inference. + # NOTE: A positive value indicates the character indices in the schema description + # whereas a negative value indicates the character indices in the + # utterance. The indices are offset by 1 to prevent ambiguity in the + # 0 index, which could be in either the schema description or utterance by the + # above convention. Now the 0 index corresponds to padded tokens. + self.start_char_idx = [0] * self._max_seq_length + self.end_char_idx = [0] * self._max_seq_length + + # Id of categorical slot present in the example or 0 if not present. + self.categorical_slot_id = 0 + # Id of non categorical slot present in the example or 0 if not present. + self.noncategorical_slot_id = 0 + # The status of categorical slot in the example. + self.categorical_slot_status = STATUS_OFF + # The status of non categorical slot in the example. + self.noncategorical_slot_status = STATUS_OFF + # Masks out tasks not represented by example + self.task_mask = [0] * schema_config["NUM_TASKS"] + + # The index of the starting subword corresponding to the slot span + # for a non-categorical slot value. + self.noncategorical_slot_value_start = 0 + # The index of the ending (inclusive) subword corresponding to the slot span + # for a non-categorical slot value. + self.noncategorical_slot_value_end = 0 + + # Id of categorical slot value present in the example or 0 if not present. + self.categorical_slot_value_id = 0 + # The status of categorical slot value in the example. + self.categorical_slot_value_status = STATUS_OFF + # Id of requested slot present in the example or 0 if not present. + self.requested_slot_id = 0 + # Takes value 1 if the corresponding slot is requested, 0 otherwise. + self.requested_slot_status = STATUS_OFF + + # ID of intent present in the example. + self.intent_id = 0 + # Takes value 1 if the intent is active, 0 otherwise. + self.intent_status = STATUS_OFF + + @property + def readable_summary(self): + """Get a readable dict that summarizes the attributes of an InputExample.""" + seq_length = sum(self.utterance_mask) + utt_toks = self._tokenizer.ids_to_tokens(self.utterance_ids[:seq_length]) + utt_tok_mask_pairs = list(zip(utt_toks, self.utterance_segment[:seq_length])) + active_intent = ( + self.service_schema.get_intent_from_id(self.intent_id) if self.intent_status == STATUS_ACTIVE else "" + ) + slot_values_in_state = {} + if self.categorical_slot_status == STATUS_ACTIVE: + slot_values_in_state[ + self.service_schema.get_categorical_slot_from_id(self.categorical_slot_id) + ] = self.service_schema.get_categorical_slot_value_from_id( + self.categorical_slot_id, self.categorical_slot_value_id + ) + elif self.categorical_slot_status == STATUS_DONTCARE: + slot_values_in_state[ + self.service_schema.get_categorical_slot_from_id(self.categorical_slot_id) + ] = STR_DONTCARE + if self.noncategorical_slot_status == STATUS_ACTIVE: + slot = self.service_schema.get_non_categorical_slot_from_id(self.noncategorical_slot_id) + start_id = self.noncategorical_slot_value_start[idx] + end_id = self.noncategorical_slot_value_end[idx] + # Token list is consisted of the subwords that may start with "##". We + # remove "##" to reconstruct the original value. Note that it's not a + # strict restoration of the original string. It's primarily used for + # debugging. + # ex. ["san", "j", "##ose"] --> "san jose" + readable_value = " ".join(utt_toks[start_id : end_id + 1]).replace(" ##", "") + slot_values_in_state[slot] = readable_value + elif self.noncategorical_slot_status == STATUS_DONTCARE: + slot = self.service_schema.get_non_categorical_slot_from_id(self.noncategorical_slot_id) + slot_values_in_state[slot] = STR_DONTCARE + + summary_dict = { + "utt_tok_mask_pairs": utt_tok_mask_pairs, + "utt_len": seq_length, + "categorical_slot_id": self.categorical_slot_id, + "noncategorical_slot_id": self.noncategorical_slot_id, + "intent_id": self.intent_id, + "service_name": self.service_schema.service_name, + "active_intent": active_intent, + "slot_values_in_state": slot_values_in_state, + } + return summary_dict + + def add_utterance_features( + self, system_tokens, system_inv_alignments, user_tokens, user_inv_alignments, system_utterance, user_utterance + ): + """Add utterance related features input to InputExample. + + Note: this method modifies the system tokens and user_tokens in place to + make their total length <= the maximum input length for BERT model. + + Args: + system_tokens: a list of strings which represents schema description. + system_inv_alignments: a list of tuples which denotes the start and end + charater of the tpken that a bert token originates from in the original + schema description. + user_tokens: a list of strings which represents utterance. + user_inv_alignments: a list of tuples which denotes the start and end + charater of the token that a bert token originates from in the original + system and user utterance. + """ + # Input sequence length for utterance BERT encoder + max_utt_len = self._max_seq_length + + # Modify lengths of schema description & utterance so that length of total utt + # (including cls_token, setp_token, sep_token) is no more than max_utt_len + is_too_long = truncate_seq_pair(system_tokens, user_tokens, max_utt_len - 3) + if is_too_long: + logging.debug( + f'Utterance sequence truncated in example id - {self.example_id} from {len(system_tokens) + len(user_tokens)}.' + ) + + # Construct the tokens, segment mask and valid token mask which will be + # input to BERT, using the tokens for schema description (sequence A) and + # system and user utterance (sequence B). + utt_subword = [] + utt_seg = [] + utt_mask = [] + start_char_idx = [] + end_char_idx = [] + + utt_subword.append(self._tokenizer.cls_token) + utt_seg.append(0) + utt_mask.append(1) + start_char_idx.append(0) + end_char_idx.append(0) + + for subword_idx, subword in enumerate(system_tokens): + utt_subword.append(subword) + utt_seg.append(0) + utt_mask.append(1) + st, en = system_inv_alignments[subword_idx] + start_char_idx.append(-(st + 1)) + end_char_idx.append(-(en + 1)) + + utt_subword.append(self._tokenizer.sep_token) + utt_seg.append(0) + utt_mask.append(1) + start_char_idx.append(0) + end_char_idx.append(0) + + for subword_idx, subword in enumerate(user_tokens): + utt_subword.append(subword) + utt_seg.append(1) + utt_mask.append(1) + st, en = user_inv_alignments[subword_idx] + start_char_idx.append(st + 1) + end_char_idx.append(en + 1) + + utt_subword.append(self._tokenizer.sep_token) + utt_seg.append(1) + utt_mask.append(1) + start_char_idx.append(0) + end_char_idx.append(0) + + utterance_ids = self._tokenizer.tokens_to_ids(utt_subword) + + # Zero-pad up to the BERT input sequence length. + while len(utterance_ids) < max_utt_len: + utterance_ids.append(0) + utt_seg.append(0) + utt_mask.append(0) + start_char_idx.append(0) + end_char_idx.append(0) + self.utterance_ids = utterance_ids + self.utterance_segment = utt_seg + self.utterance_mask = utt_mask + self.start_char_idx = start_char_idx + self.end_char_idx = end_char_idx + + self.user_utterance = user_utterance + self.system_utterance = system_utterance + + def make_copy(self): + """Make a copy of the current example with utterance features.""" + new_example = InputExample( + schema_config=self.schema_config, + service_schema=self.service_schema, + example_id=self.example_id, + example_id_num=self.example_id_num.copy(), + tokenizer=self._tokenizer, + ) + return new_example + + def make_copy_of_categorical_features(self): + """Make a copy of the current example with utterance and categorical features.""" + new_example = self.make_copy() + + new_example.categorical_slot_status = self.categorical_slot_status + return new_example + + def make_copy_of_non_categorical_features(self): + """Make a copy of the current example with utterance features and non categorical features.""" + new_example = self.make_copy() + new_example.noncategorical_slot_id = self.noncategorical_slot_id + new_example.noncategorical_slot_status = self.noncategorical_slot_status + new_example.utterance_ids = list(self.utterance_ids) + new_example.utterance_segment = list(self.utterance_segment) + new_example.utterance_mask = list(self.utterance_mask) + new_example.start_char_idx = list(self.start_char_idx) + new_example.end_char_idx = list(self.end_char_idx) + new_example.user_utterance = self.user_utterance + new_example.system_utterance = self.system_utterance + new_example.noncategorical_slot_status = self.noncategorical_slot_status + new_example.noncategorical_slot_value_start = self.noncategorical_slot_value_start + new_example.noncategorical_slot_value_end = self.noncategorical_slot_value_end + return new_example + + def add_categorical_slots(self, state_update: dict): + """Add features for categorical slots. + Args: + state_update: slot value pairs of the state update + """ + + categorical_slots = self.service_schema.categorical_slots + if not categorical_slots: + return + slot = categorical_slots[self.categorical_slot_id] + values = state_update.get(slot, []) + + if not values: + self.categorical_slot_status = STATUS_OFF + elif values[0] == STR_DONTCARE: + self.categorical_slot_status = STATUS_DONTCARE + else: + self.categorical_slot_status = STATUS_ACTIVE + self.categorical_slot_value_status = ( + self.categorical_slot_value_id == self.service_schema.get_categorical_slot_value_id(slot, values[0]) + ) + + def add_noncategorical_slots(self, state_update: dict, system_span_boundaries: dict, user_span_boundaries: dict): + """Add features for non-categorical slots. + Args: + state_update: slot value pairs of state update + system_span_boundaries: span boundaries of schema description + user_span_boundaries: span boundaries of utterance + """ + + noncategorical_slots = self.service_schema.non_categorical_slots + slot = noncategorical_slots[self.noncategorical_slot_id] + + values = state_update.get(slot, []) + if not values: + self.noncategorical_slot_status = STATUS_OFF + elif values[0] == STR_DONTCARE: + self.noncategorical_slot_status = STATUS_DONTCARE + else: + self.noncategorical_slot_status = STATUS_ACTIVE + # Add indices of the start and end tokens for the first encountered + # value. Spans in user utterance are prioritized over the system + # utterance. If a span is not found, the slot value is ignored. + if slot in user_span_boundaries: + start, end = user_span_boundaries[slot] + elif slot in system_span_boundaries: + start, end = system_span_boundaries[slot] + else: + # A span may not be found because the value was cropped out or because + # the value was mentioned earlier in the dialogue. Since this model + # only makes use of the last two utterances to predict state updates, + # it will fail in such cases. + logging.debug( + f'"Slot values {str(values)} not found in user or system utterance in example with id - {self.example_id}.' + ) + start = 0 + end = 0 + self.noncategorical_slot_value_start = start + self.noncategorical_slot_value_end = end + + def add_requested_slots(self, frame: dict): + """Add requested slots to InputExample + Args: + frame: frame object from which requested slots are extracted + """ + all_slots = self.service_schema.slots + slot = all_slots[self.requested_slot_id] + if slot in frame["state"]["requested_slots"]: + self.requested_slot_status = STATUS_ACTIVE + + def add_intents(self, frame): + """Add intents to InputExample + Args: + frame: frame object from which intents are extracted + """ + all_intents = self.service_schema.intents + intent = all_intents[self.intent_id] + if intent == frame["state"]["active_intent"]: + self.intent_status = STATUS_ACTIVE + + +# Modified from run_classifier._truncate_seq_pair in the public bert model repo. +# https://github.com/google-research/bert/blob/master/run_classifier.py. +def truncate_seq_pair(tokens_a: List[int], tokens_b: List[int], max_length: int) -> bool: + """Truncate a seq pair in place so that their total length <= max_length. + Args: + tokens_a: first token sequence + tokens_b: second token sequence + max_length: truncated sequence length + Returns: + is_too_long: whether combined sequences exceed maximum sequence length + """ + is_too_long = False + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + is_too_long = True + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + return is_too_long diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/metrics.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/metrics.py new file mode 100644 index 0000000000..d833fa0543 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/metrics.py @@ -0,0 +1,341 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluation metrics for Schema-guided dialogue. + +This library provides functions for calculating the evaluation metrics for a +single dialogue. The following metrics are defined: + +(1) Active intent accuracy: The fraction of user turns for which the active + intent has been correctly predicted. +(2) Slot tagging F1: The macro-averaged F1 score for tagging slot values for + non-categorical slots. This metric is optional to report in the final paper + if participants decide not to use slot tagging. +(3) Requested slots F1: The macro-averaged F1 score for requested slots over the + turns. For a turn, if there are no requested slots in both the ground truth + and the prediction, that turn is skipped. The reported number is the average + F1 score for all un-skipped user turns. This metric is optional to report in + the final paper. +(4) Average goal accuracy: For each turn, participants must predict a single + value for each slot present in the dialogue state. The slots which have a + non-empty assignment in the ground truth dialogue state are only considered. + This is the average accuracy of predicting the value of a slot correctly. A + fuzzy matching based score is used for non-categorical slots. +(5) Joint goal accuracy: This is the average accuracy of predicting all slot + assignments for a turn correctly. A fuzzy matching based score is used for + non-categorical slots. This is the primary evaluation metric used for ranking + submissions. More details to follow with the evaluation script. + +This file contains code artifacts adapted from the original implementation: +https://github.com/google-research/google-research/blob/master/schema_guided_dst/metrics.py +""" + +import collections + +import numpy as np +from rapidfuzz import fuzz + +F1Scores = collections.namedtuple("F1Scores", ["f1", "precision", "recall"]) + +# Evaluation and other relevant metrics for DSTC8/SGD Schema-guided DST. +# (1) Active intent accuracy. +ACTIVE_INTENT_ACCURACY = "active_intent_accuracy" +# (2) Slot tagging F1. +SLOT_TAGGING_F1 = "slot_tagging_f1" +SLOT_TAGGING_PRECISION = "slot_tagging_precision" +SLOT_TAGGING_RECALL = "slot_tagging_recall" +# (3) Requested slots F1. +REQUESTED_SLOTS_F1 = "requested_slots_f1" +REQUESTED_SLOTS_PRECISION = "requested_slots_precision" +REQUESTED_SLOTS_RECALL = "requested_slots_recall" +# (4) Average goal accuracy. +AVERAGE_GOAL_ACCURACY = "average_goal_accuracy" +AVERAGE_CAT_ACCURACY = "average_cat_accuracy" +AVERAGE_NONCAT_ACCURACY = "average_noncat_accuracy" +# (5) Joint goal accuracy. +JOINT_GOAL_ACCURACY = "joint_goal_accuracy" +JOINT_CAT_ACCURACY = "joint_cat_accuracy" +JOINT_NONCAT_ACCURACY = "joint_noncat_accuracy" + + +AVERAGE_CAT_STATUS_ACCURACY = "average_cat_status_accuracy" +AVERAGE_CAT_VALUE_ACCURACY = "average_cat_value_accuracy" +AVERAGE_NONCAT_STATUS_ACCURACY = "average_noncat_status_accuracy" +AVERAGE_NONCAT_VALUE_ACCURACY = "average_noncat_value_accuracy" + +JOINT_CAT_STATUS_ACCURACY = "joint_cat_status_accuracy" +JOINT_CAT_VALUE_ACCURACY = "joint_cat_value_accuracy" +JOINT_NONCAT_STATUS_ACCURACY = "joint_noncat_status_accuracy" +JOINT_NONCAT_VALUE_ACCURACY = "joint_noncat_value_accuracy" + + +NAN_VAL = "NA" + + +def compute_f1(list_ref, list_hyp): + """Compute F1 score from reference (grouth truth) list and hypothesis list. + Args: + list_ref: List of true elements. + list_hyp: List of postive (retrieved) elements. + Returns: + A F1Scores object containing F1, precision, and recall scores. + """ + + ref = collections.Counter(list_ref) + hyp = collections.Counter(list_hyp) + true = sum(ref.values()) + positive = sum(hyp.values()) + true_positive = sum((ref & hyp).values()) + precision = float(true_positive) / positive if positive else 1.0 + recall = float(true_positive) / true if true else 1.0 + if precision + recall > 0.0: + f1 = 2.0 * precision * recall / (precision + recall) + else: # The F1-score is defined to be 0 if both precision and recall are 0. + f1 = 0.0 + + return F1Scores(f1=f1, precision=precision, recall=recall) + + +def fuzzy_string_match(str_ref, str_hyp): + """Returns fuzzy string similarity score in range [0.0, 1.0]. + Args: + str_ref: reference string + str_hyp: hypothesis string + Returns: + fuzzy string similarity + """ + + # The higher the score, the higher the similarity between the two strings. + return fuzz.token_sort_ratio(str_ref, str_hyp) / 100.0 + + +def noncat_slot_value_match(str_ref_list, str_hyp, use_fuzzy_match): + """Calculate non-categorical slots correctness. + Args: + str_ref_list: a list of reference strings. + str_hyp: the hypothesis string. + use_fuzzy_match: whether to use fuzzy string matching. + Returns: + score: The highest fuzzy string match score of the references and hypotheis. + """ + score = 0.0 + for str_ref in str_ref_list: + if use_fuzzy_match: + match_score = fuzzy_string_match(str_ref, str_hyp) + else: + match_score = float(str_ref == str_hyp) + score = max(score, match_score) + return score + + +def compare_slot_values(slot_values_ref, slot_values_hyp, service, use_fuzzy_match): + """Compare and get correctness of goal state's slot_values. + + Args: + slot_values_ref: goal state slot_values from reference (ground truth). + slot_values_hyp: goal state slot_values from hypothesis (prediction). + service: a service data structure in the schema. We use it to obtain the + list of slots in the service and infer whether a slot is categorical. + use_fuzzy_match: whether to use fuzzy string matching for non-categorical + slot values + + Returns: + list_cor: list of corectness scores, each corresponding to one slot in the + service. The score is a float either 0.0 or 1.0 for categorical slot, + and in range [0.0, 1.0] for non-categorical slot. + slot_active: list indicating whether the element in list_cor corresponds to + an active ground-truth slot. + slot_cat: list indicating whether the element in list_cor corresponds to a + categorical slot. + list_cor_status: list of correct slot statuses + list_cor_value: list of correctness score only for active slots. Monactive slots are assigned -1. + """ + list_cor = [] + list_cor_status = [] + list_cor_value = [] + slot_active = [] + slot_cat = [] + + for slot in service["slots"]: + slot_name = slot["name"] + slot_cat.append(slot["is_categorical"]) + + if slot_name in slot_values_ref: # REF=active + slot_active.append(True) + if slot_name in slot_values_hyp: # HYP=active, apply matching + value_ref_list = slot_values_ref[slot_name] + value_hyp = slot_values_hyp[slot_name][0] + if slot["is_categorical"]: + cor = float(value_ref_list[0] == value_hyp) + else: + cor = noncat_slot_value_match(value_ref_list, value_hyp, use_fuzzy_match) + list_cor.append(cor) + list_cor_status.append(1.0) + list_cor_value.append(cor) + else: # HYP=off + list_cor.append(0.0) + list_cor_status.append(0.0) + list_cor_value.append(-1.0) + else: # REF=off + slot_active.append(False) + if slot_name in slot_values_hyp: # HYP=active + list_cor.append(0.0) + list_cor_status.append(0.0) + else: # HYP=off + list_cor.append(1.0) + list_cor_status.append(1.0) + list_cor_value.append(-1.0) + + assert len(list_cor) == len(service["slots"]) + assert len(slot_active) == len(service["slots"]) + assert len(slot_cat) == len(service["slots"]) + return list_cor, slot_active, slot_cat, list_cor_status, list_cor_value + + +def get_active_intent_accuracy(frame_ref, frame_hyp): + """Get active intent accuracy of a frame. + + Args: + frame_ref: single semantic frame from reference (ground truth) file. + frame_hyp: single semantic frame from hypothesis (prediction) file. + + Returns: + 1.0 if the intent prediction is correct, otherwise 0.0. + """ + return float(frame_ref["state"]["active_intent"] == frame_hyp["state"]["active_intent"]) + + +def get_slot_tagging_f1(frame_ref, frame_hyp, utt, service): + """Get slot tagging (non-categorical slots only) F1 scores of a frame. + + Args: + frame_ref: single semantic frame from reference (ground truth) file. + frame_hyp: single semantic frame from hypothesis (prediction) file. + utt: user utterance. Slot tagging annotations are the character positions in + the utterance. + service: a service data structure in the schema. We use it to infer whether + a slot is non-categorical. + + Returns: + A F1Scores object containing F1, precision, and recall scores. + """ + list_noncat_slots = [s["name"] for s in service["slots"] if not s["is_categorical"]] + if "slots" not in frame_hyp: + return None + else: + list_ref = [ + (s["slot"], utt[s["start"] : s["exclusive_end"]]) + for s in frame_ref["slots"] + if s["slot"] in list_noncat_slots + ] + list_hyp = [ + (s["slot"], utt[s["start"] : s["exclusive_end"]]) + for s in frame_hyp["slots"] + if s["slot"] in list_noncat_slots + ] + return compute_f1(list_ref, list_hyp) + + +def get_requested_slots_f1(frame_ref, frame_hyp): + """Get requested slots F1 scores of a frame. + + Args: + frame_ref: single semantic frame from reference (ground truth) file. + frame_hyp: single semantic frame from hypothesis (prediction) file. + + Returns: + A F1Scores object containing F1, precision, and recall scores. + """ + return compute_f1(frame_ref["state"]["requested_slots"], frame_hyp["state"]["requested_slots"]) + + +def get_average_and_joint_goal_accuracy(frame_ref, frame_hyp, service, use_fuzzy_match): + """Get average and joint goal accuracies of a frame. + + Args: + frame_ref: single semantic frame from reference (ground truth) file. + frame_hyp: single semantic frame from hypothesis (prediction) file. + service: a service data structure in the schema. We use it to obtain the + list of slots in the service and infer whether a slot is categorical. + use_fuzzy_match: whether to use fuzzy string matching for comparing + non-categorical slot values. + + Returns: + goal_acc: a dict whose values are average / joint + all-goal / categorical-goal / non-categorical-goal accuracies. + """ + goal_acc = {} + + list_acc, slot_active, slot_cat, list_status_acc, list_value_acc = compare_slot_values( + frame_ref["state"]["slot_values"], frame_hyp["state"]["slot_values"], service, use_fuzzy_match + ) + + # (4) Average goal accuracy. + active_acc = [acc for acc, active in zip(list_acc, slot_active) if active] + goal_acc[AVERAGE_GOAL_ACCURACY] = np.mean(active_acc) if active_acc else NAN_VAL + # (4-a) categorical. + active_cat_acc = [acc for acc, active, cat in zip(list_acc, slot_active, slot_cat) if active and cat] + goal_acc[AVERAGE_CAT_ACCURACY] = np.mean(active_cat_acc) if active_cat_acc else NAN_VAL + # (4-b) non-categorical. + active_noncat_acc = [acc for acc, active, cat in zip(list_acc, slot_active, slot_cat) if active and not cat] + goal_acc[AVERAGE_NONCAT_ACCURACY] = np.mean(active_noncat_acc) if active_noncat_acc else NAN_VAL + + # (5) Joint goal accuracy. + goal_acc[JOINT_GOAL_ACCURACY] = np.prod(list_acc) if list_acc else NAN_VAL + # (5-a) categorical. + cat_acc = [acc for acc, cat in zip(list_acc, slot_cat) if cat] + goal_acc[JOINT_CAT_ACCURACY] = np.prod(cat_acc) if cat_acc else NAN_VAL + # (5-b) non-categorical. + noncat_acc = [acc for acc, cat in zip(list_acc, slot_cat) if not cat] + goal_acc[JOINT_NONCAT_ACCURACY] = np.prod(noncat_acc) if noncat_acc else NAN_VAL + + # !!!!!!!!!!DEBUG!!!!!!!!!!!!! + # cat status acc for both active and non active + active_cat_status_acc = [acc for acc, active, cat in zip(list_status_acc, slot_active, slot_cat) if cat and active] + goal_acc[AVERAGE_CAT_STATUS_ACCURACY] = np.mean(active_cat_status_acc) if active_cat_status_acc else NAN_VAL + # joint cat status acc for both active and non active + cat_status_acc = [acc for acc, cat in zip(list_status_acc, slot_cat) if cat] + goal_acc[JOINT_CAT_STATUS_ACCURACY] = np.prod(cat_status_acc) if cat_status_acc else NAN_VAL + + # non cat status acc for both active and non active + active_noncat_status_acc = [ + acc for acc, active, cat in zip(list_status_acc, slot_active, slot_cat) if not cat and active + ] + goal_acc[AVERAGE_NONCAT_STATUS_ACCURACY] = ( + np.mean(active_noncat_status_acc) if active_noncat_status_acc else NAN_VAL + ) + # joint non cat status acc for both active and non active + noncat_status_acc = [acc for acc, cat in zip(list_status_acc, slot_cat) if not cat] + goal_acc[JOINT_NONCAT_STATUS_ACCURACY] = np.prod(noncat_status_acc) if noncat_status_acc else NAN_VAL + + # cat value acc for both active and non active + active_cat_val_acc = [ + acc for acc, active, cat in zip(list_value_acc, slot_active, slot_cat) if cat and acc > -0.5 and active + ] + goal_acc[AVERAGE_CAT_VALUE_ACCURACY] = np.mean(active_cat_val_acc) if active_cat_val_acc else NAN_VAL + # joint cat value acc for both active and non active + cat_val_acc = [acc for acc, cat in zip(list_value_acc, slot_cat) if cat and acc > -0.5] + goal_acc[JOINT_CAT_VALUE_ACCURACY] = np.prod(cat_val_acc) if cat_val_acc else NAN_VAL + + # cat non value acc for both active and non active + active_noncat_val_acc = [ + acc for acc, active, cat in zip(list_value_acc, slot_active, slot_cat) if not cat and acc > -0.5 and active + ] + goal_acc[AVERAGE_NONCAT_VALUE_ACCURACY] = np.mean(active_noncat_val_acc) if active_noncat_val_acc else NAN_VAL + # joint non cat value acc for both active and non active + noncat_val_acc = [acc for acc, cat in zip(list_value_acc, slot_cat) if not cat and acc > -0.5] + goal_acc[JOINT_NONCAT_VALUE_ACCURACY] = np.prod(noncat_val_acc) if noncat_val_acc else NAN_VAL + + return goal_acc diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/prediction_utils.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/prediction_utils.py new file mode 100644 index 0000000000..68fa645072 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/prediction_utils.py @@ -0,0 +1,251 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Prediction and evaluation-related utility functions. +This file contains code artifacts adapted from the original implementation: +https://github.com/google-research/google-research/blob/master/schema_guided_dst/baseline/pred_utils.py +""" + +import json +import os +from collections import OrderedDict, defaultdict +from typing import Dict, List, Optional + +from nemo.collections.nlp.data.dialogue_state_tracking.sgd.input_example import ( + STATUS_ACTIVE, + STATUS_DONTCARE, + STR_DONTCARE, +) +from nemo.utils import logging + +REQ_SLOT_THRESHOLD = 0.5 + + +__all__ = ['write_predictions_to_file'] + + +def set_cat_slot(predictions_status: dict, predictions_value: dict, cat_slot_values: Dict[str, List[str]]) -> dict: + """ + Extract predicted categorical slot information + Args: + predictions_status: predicted statuses + predictions_value: predicted slot values + cat_slot_values: possible categorical slots and their potential values for this service + Returns: + out_dict: predicted slot value pairs + """ + out_dict = {} + for slot_idx, slot in enumerate(cat_slot_values): + slot_status = predictions_status[slot_idx][0]["cat_slot_status"] + if slot_status == STATUS_DONTCARE: + out_dict[slot] = STR_DONTCARE + elif slot_status == STATUS_ACTIVE: + tmp = predictions_value[slot_idx] + value_idx = max(tmp, key=lambda k: tmp[k]['cat_slot_value_status'][0].item()) + out_dict[slot] = cat_slot_values[slot][value_idx] + return out_dict + + +def set_noncat_slot( + predictions_status: dict, + predictions_value: dict, + non_cat_slots: List[str], + user_utterance: str, + sys_slots_agg: Optional[dict] = None, +) -> dict: + """ + Extract predicted non categorical slot information + Args: + predictions_status: predicted statuses + predictions_value: predicted slot values + non_cat_slots: list of possible non categorical slots for this service + user_utterance: system and user utterance + sys_slots_agg: system retrieval lookup table. Contains for each slot the most recent value seen in the history + Returns: + out_dict: predicted slot value pairs + """ + out_dict = {} + for slot_idx, slot in enumerate(non_cat_slots): + slot_status = predictions_status[slot_idx][0]["noncat_slot_status"] + if slot_status == STATUS_DONTCARE: + out_dict[slot] = STR_DONTCARE + elif slot_status == STATUS_ACTIVE: + tok_start_idx = predictions_value[slot_idx][0]["noncat_slot_start"] + tok_end_idx = predictions_value[slot_idx][0]["noncat_slot_end"] + ch_start_idx = predictions_value[slot_idx][0]["noncat_alignment_start"][tok_start_idx] + ch_end_idx = predictions_value[slot_idx][0]["noncat_alignment_end"][tok_end_idx] + if ch_start_idx > 0 and ch_end_idx > 0: + # Add span from the utterance. + out_dict[slot] = user_utterance[ch_start_idx - 1 : ch_end_idx] + elif sys_slots_agg and slot in sys_slots_agg: + # system retrieval + out_dict[slot] = sys_slots_agg[slot] + return out_dict + + +def get_predicted_dialog(dialog: dict, all_predictions: dict, schemas: object, state_tracker: str) -> dict: + """Overwrite the labels in the turn with the predictions from the model. For test set, these labels are missing from the data and hence they are added. + Args: + dialog: ground truth dialog + all_predictions: predictions + schemas: schema object of all services of all datasets + state_tracker: state tracker option, e.g. nemotracker + Returns: + dialog: dialog overwritten with prediction information + """ + dialog_id = dialog["dialogue_id"] + if state_tracker == "baseline": + sys_slots_agg = {} + else: + sys_slots_agg = defaultdict(OrderedDict) + all_slot_values = defaultdict(dict) + for turn_idx, turn in enumerate(dialog["turns"]): + if turn["speaker"] == "SYSTEM" and state_tracker == 'nemotracker': + for frame in turn["frames"]: + if frame["service"] not in sys_slots_agg: + sys_slots_agg[frame["service"]] = OrderedDict() + for action in frame["actions"]: + if action["slot"] and len(action["values"]) > 0: + sys_slots_agg[frame["service"]][action["slot"]] = action["values"][0] + if turn["speaker"] == "USER": + user_utterance = turn["utterance"] + system_utterance = dialog["turns"][turn_idx - 1]["utterance"] if turn_idx else "" + system_user_utterance = system_utterance + ' ' + user_utterance + turn_id = "{:02d}".format(turn_idx) + for frame in turn["frames"]: + + predictions = all_predictions[(dialog_id, turn_id, frame["service"])] + slot_values = all_slot_values[frame["service"]] + service_schema = schemas.get_service_schema(frame["service"]) + # Remove the slot spans and state if present. + frame.pop("slots", None) + frame.pop("state", None) + + # The baseline model doesn't predict slot spans. Only state predictions + # are added. + state = {} + + # Add prediction for active intent. No Offset is subtracted since schema has now NONE intent at index 0 + state["active_intent"] = get_predicted_intent( + predictions=predictions[0], intents=service_schema.intents + ) + # Add prediction for requested slots. + state["requested_slots"] = get_requested_slot(predictions=predictions[1], slots=service_schema.slots) + + # Add prediction for user goal (slot values). + # Categorical slots. + cat_out_dict = set_cat_slot( + predictions_status=predictions[2], + predictions_value=predictions[3], + cat_slot_values=service_schema.categorical_slot_values, + ) + for k, v in cat_out_dict.items(): + slot_values[k] = v + + # Non-categorical slots. + noncat_out_dict = set_noncat_slot( + predictions_status=predictions[4], + predictions_value=predictions[5], + non_cat_slots=service_schema.non_categorical_slots, + user_utterance=system_user_utterance, + sys_slots_agg=sys_slots_agg.get(frame["service"], None), + ) + for k, v in noncat_out_dict.items(): + slot_values[k] = v + # Create a new dict to avoid overwriting the state in previous turns + # because of use of same objects. + state["slot_values"] = {s: [v] for s, v in slot_values.items()} + frame["state"] = state + return dialog + + +def get_predicted_intent(predictions: dict, intents: List[str]) -> str: + """ + Returns intent name with maximum score + Args: + predictions: predictions + intents: list of possible intents for this service + Returns: + intent: predicted intent + """ + assert len(predictions) == len(intents) + active_intent_id = max(predictions, key=lambda k: predictions[k][0]['intent_status']) + intent = intents[active_intent_id] + return intent + + +def get_requested_slot(predictions: dict, slots: List[str]) -> List[str]: + """ + Returns list of slots which are predicted to be requested + Args: + predictions: predictions + slots: list of possible slots + Returns: + requested_slots: list of requested slots + """ + active_indices = [k for k in predictions if predictions[k][0]["req_slot_status"] > REQ_SLOT_THRESHOLD] + requested_slots = list(map(lambda k: slots[k], active_indices)) + return requested_slots + + +def write_predictions_to_file( + predictions: List[dict], + input_json_files: List[str], + output_dir: str, + schemas: object, + state_tracker: str, + eval_debug: bool, + in_domain_services: set, +): + """Save predicted dialogues as json files. + + Args: + predictions: An iterator containing model predictions. This is the output of + the predict method in the estimator. + input_json_files: A list of json paths containing the dialogues to run + inference on. + output_dir: The directory where output json files will be created. + schemas: Schemas to all services in the dst dataset + state_tracker: state tracker option + eval_debug: output evaluation debugging information + in_domain_services: in domain services + """ + logging.info(f"Writing predictions to {output_dir} started.") + + # Index all predictions. + all_predictions = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) + for idx, prediction in enumerate(predictions): + eval_dataset, dialog_id, turn_id, service_name, model_task, slot_intent_id, value_id = prediction[ + 'example_id' + ].split('-') + all_predictions[(dialog_id, turn_id, service_name)][int(model_task)][int(slot_intent_id)][ + int(value_id) + ] = prediction + logging.info(f'Predictions for {idx} examples in {eval_dataset} dataset are getting processed.') + + # Read each input file and write its predictions. + for input_file_path in input_json_files: + with open(input_file_path) as f: + dialogs = json.load(f) + logging.debug(f'{input_file_path} file is loaded') + pred_dialogs = [] + for d in dialogs: + pred_dialog = get_predicted_dialog(d, all_predictions, schemas, state_tracker) + pred_dialogs.append(pred_dialog) + input_file_name = os.path.basename(input_file_path) + output_file_path = os.path.join(output_dir, input_file_name) + with open(output_file_path, "w") as f: + json.dump(pred_dialogs, f, indent=2, separators=(",", ": "), sort_keys=True) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/schema.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/schema.py new file mode 100644 index 0000000000..9aba267790 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/dialogue_state_tracking/sgd/schema.py @@ -0,0 +1,222 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Wrappers for schemas of different services. +This file contains code artifacts adapted from the original implementation: +https://github.com/google-research/google-research/blob/master/schema_guided_dst/schema.py +""" + +import json +from typing import List, Optional, Union + +from nemo.utils import logging + +__all__ = ['Schema'] + + +class ServiceSchema(object): + """A wrapper for schema for a service.""" + + def __init__(self, schema_json: dict, service_id: Optional[int] = None): + """ + Constructor for ServiceSchema. + Args: + schema_json: schema json dict + service_id: service ID + """ + self._service_name = schema_json["service_name"] + self._description = schema_json["description"] + self._schema_json = schema_json + self._service_id = service_id + + # Construct the vocabulary for intents, slots, categorical slots, + # non-categorical slots and categorical slot values. + self._intents = ["NONE"] + sorted(i["name"] for i in schema_json["intents"]) + self._intent_descriptions = {i["name"]: i["description"] for i in schema_json["intents"]} + self._intent_descriptions["NONE"] = "none" + self._slots = sorted(s["name"] for s in schema_json["slots"]) + self._slots_descriptions = {s["name"]: s["description"] for s in schema_json["slots"]} + self._categorical_slots = sorted( + s["name"] for s in schema_json["slots"] if s["is_categorical"] and s["name"] in self.state_slots + ) + self._non_categorical_slots = sorted( + s["name"] for s in schema_json["slots"] if not s["is_categorical"] and s["name"] in self.state_slots + ) + slot_schemas = {s["name"]: s for s in schema_json["slots"]} + categorical_slot_values = {} + categorical_slot_value_ids = {} + categorical_slot_ids = {} + non_categorical_slot_ids = {} + for slot_id, slot in enumerate(self._categorical_slots): + slot_schema = slot_schemas[slot] + values = sorted(slot_schema["possible_values"]) + categorical_slot_values[slot] = values + value_ids = {value: idx for idx, value in enumerate(values)} + categorical_slot_value_ids[slot] = value_ids + categorical_slot_ids[slot] = slot_id + + for slot_id, slot in enumerate(self._non_categorical_slots): + non_categorical_slot_ids[slot] = slot_id + + self._categorical_slot_values = categorical_slot_values + self._categorical_slot_value_ids = categorical_slot_value_ids + + self._categorical_slot_ids = categorical_slot_ids + self._non_categorical_slot_ids = non_categorical_slot_ids + + @property + def schema_json(self) -> dict: + """Returns schema json dictionary""" + return self._schema_json + + @property + def state_slots(self) -> set: + """Set of slots which are permitted to be in the dialogue state.""" + state_slots = set() + for intent in self._schema_json["intents"]: + state_slots.update(intent["required_slots"]) + state_slots.update(intent["optional_slots"]) + return state_slots + + @property + def service_name(self): + return self._service_name + + @property + def service_id(self): + return self._service_id + + @property + def description(self): + return self._description + + @property + def slots(self): + return self._slots + + @property + def intents(self): + return self._intents + + @property + def intent_descriptions(self): + return self._intent_descriptions + + @property + def slot_descriptions(self): + return self._slots_descriptions + + @property + def categorical_slots(self): + return self._categorical_slots + + @property + def non_categorical_slots(self): + return self._non_categorical_slots + + @property + def categorical_slot_values(self): + return self._categorical_slot_values + + def get_categorical_slot_values(self, slot): + return self._categorical_slot_values[slot] + + def get_slot_from_id(self, slot_id): + return self._slots[slot_id] + + def get_intent_from_id(self, intent_id): + return self._intents[intent_id] + + def get_categorical_slot_from_id(self, slot_id): + return self._categorical_slots[slot_id] + + def get_non_categorical_slot_from_id(self, slot_id): + return self._non_categorical_slots[slot_id] + + def get_categorical_slot_value_from_id(self, slot_id, value_id): + slot = self._categorical_slots[slot_id] + return self._categorical_slot_values[slot][value_id] + + def get_categorical_slot_value_id(self, slot, value): + return self._categorical_slot_value_ids[slot][value] + + def get_categorical_slot_id(self, slot): + return self._categorical_slot_ids[slot] + + def get_non_categorical_slot_id(self, slot): + return self._non_categorical_slot_ids[slot] + + +class Schema(object): + """Wrapper for schemas for all services in a dataset.""" + + def __init__(self, schema_json_paths: Union[str, List[str]]): + """ + schema_json_paths: list of .json path to schema files of a single str with path to the json file. + """ + # Load the schema from the json file. + if isinstance(schema_json_paths, str): + with open(schema_json_paths, "r") as f: + all_schemas = json.load(f) + f.close() + else: + # load multiple schemas from the list of the json files + all_schemas = [] + completed_services = [] + for schema_json_path in schema_json_paths: + with open(schema_json_path, "r") as f: + schemas = json.load(f) + f.close() + logging.debug("Num of services in %s: %s", schema_json_path, len(schemas)) + + for service in schemas: + if service['service_name'] not in completed_services: + completed_services.append(service['service_name']) + all_schemas.append(service) + + self._services = sorted(schema["service_name"] for schema in all_schemas) + self._services_vocab = {v: k for k, v in enumerate(self._services)} + self._services_id_to_vocab = {v: k for k, v in self._services_vocab.items()} + service_schemas = {} + for schema in all_schemas: + service = schema["service_name"] + service_schemas[service] = ServiceSchema(schema, service_id=self.get_service_id(service)) + + self._service_schemas = service_schemas + self._schemas = all_schemas + self._slots_relation_list = {} + + def get_service_id(self, service: str): + return self._services_vocab[service] + + def get_service_from_id(self, service_id: int): + return self._services[service_id] + + def get_service_schema(self, service: str): + return self._service_schemas[service] + + @property + def services(self): + return self._services + + def save_to_file(self, file_path): + """ + Saves schema object to file + Args: + file_path: path to store schema object at + """ + with open(file_path, "w") as f: + json.dump(self._schemas, f, indent=2) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/entity_linking/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/entity_linking/__init__.py new file mode 100644 index 0000000000..659718d71b --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/entity_linking/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.entity_linking.entity_linking_dataset import EntityLinkingDataset diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/entity_linking/entity_linking_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/entity_linking/entity_linking_dataset.py new file mode 100644 index 0000000000..3b1d97a354 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/entity_linking/entity_linking_dataset.py @@ -0,0 +1,135 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import array +import pickle as pkl +from typing import Optional + +import torch + +from nemo.collections.nlp.data.data_utils.data_preprocessing import find_newlines, load_data_indices +from nemo.core.classes import Dataset +from nemo.utils import logging + +__all__ = ['EntityLinkingDataset'] + + +class EntityLinkingDataset(Dataset): + """ + Parent class for entity linking encoder training and index + datasets + + Args: + tokenizer (obj): huggingface tokenizer, + data_file (str): path to tab separated column file where data + pairs apear in the format + concept_ID\tconcept_synonym1\tconcept_synonym2\n + newline_idx_file (str): path to pickle file containing location + of data_file newline characters + max_seq_length (int): maximum length of a concept in tokens + is_index_data (bool): Whether dataset will be used for building + a nearest neighbors index + """ + + def __init__( + self, + tokenizer: object, + data_file: str, + newline_idx_file: Optional[str] = None, + max_seq_length: Optional[int] = 512, + is_index_data: bool = False, + ): + + self.tokenizer = tokenizer + + # Try and load pair indices file if already exists + newline_indices, newline_idx_file, _ = load_data_indices(newline_idx_file, data_file, "newline_indices") + + # If pair indices file doesn't exists, generate and store them + if newline_indices is None: + logging.info("Getting datafile newline indices") + + with open(data_file, "rb") as f: + contents = f.read() + newline_indices = find_newlines(contents) + newline_indices = array.array("I", newline_indices) + + # Store data file indicies to avoid generating them again + with open(newline_idx_file, "wb") as f: + pkl.dump(newline_indices, f) + + self.newline_indices = newline_indices + self.data_file = data_file + self.num_lines = len(newline_indices) + self.max_seq_length = max_seq_length + self.is_index_data = is_index_data + + logging.info(f"Loaded dataset with {self.num_lines} examples") + + def __len__(self): + return self.num_lines + + def __getitem__(self, idx): + + concept_offset = self.newline_indices[idx] + + with open(self.data_file, "r", encoding='utf-8-sig') as f: + # Find data pair within datafile using byte offset + f.seek(concept_offset) + concept = f.readline()[:-1] + concept = concept.strip().split("\t") + + if self.is_index_data: + concept_id, concept = concept + return (int(concept_id), concept) + + else: + concept_id, concept1, concept2 = concept + return (int(concept_id), concept1, concept2) + + def _collate_fn(self, batch): + """collate batch of input_ids, segment_ids, input_mask, and label + + Args: + batch: A list of tuples of format (concept_ID, concept_synonym1, concept_synonym2). + """ + if self.is_index_data: + concept_ids, concepts = zip(*batch) + concept_ids = list(concept_ids) + concepts = list(concepts) + + else: + concept_ids, concepts1, concepts2 = zip(*batch) + concept_ids = list(concept_ids) + concept_ids.extend(concept_ids) # Need to double label list to match each concept + concepts = list(concepts1) + concepts.extend(concepts2) + + batch = self.tokenizer( + concepts, + add_special_tokens=True, + padding=True, + truncation=True, + max_length=self.max_seq_length, + return_token_type_ids=True, + return_attention_mask=True, + return_length=True, + ) + + return ( + torch.LongTensor(batch["input_ids"]), + torch.LongTensor(batch["token_type_ids"]), + torch.LongTensor(batch["attention_mask"]), + torch.LongTensor(concept_ids), + ) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/__init__.py new file mode 100644 index 0000000000..753411382b --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.glue_benchmark.glue_benchmark_dataset import GLUEDataset diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/data_processors.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/data_processors.py new file mode 100644 index 0000000000..19cda84834 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/data_processors.py @@ -0,0 +1,323 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from nemo.collections.nlp.data.data_utils.data_preprocessing import DataProcessor +from nemo.utils import logging + +__all__ = [ + 'ColaProcessor', + 'MnliProcessor', + 'MnliMismatchedProcessor', + 'MrpcProcessor', + 'Sst2Processor', + 'StsbProcessor', + 'QqpProcessor', + 'QnliProcessor', + 'RteProcessor', + 'WnliProcessor', +] + + +class MrpcProcessor(DataProcessor): + """Processor for the MRPC data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + logging.info(f'LOOKING AT {os.path.join(data_dir, "train.tsv")}') + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, i) + text_a = line[3] + text_b = line[4] + label = line[0] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class MnliProcessor(DataProcessor): + """Processor for the MultiNLI data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")), "dev_matched") + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[8] + text_b = line[9] + label = line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class MnliMismatchedProcessor(MnliProcessor): + """Processor for the MultiNLI Mismatched data set (GLUE version).""" + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev_mismatched.tsv")), "dev_matched") + + +class ColaProcessor(DataProcessor): + """Processor for the CoLA data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = "%s-%s" % (set_type, i) + text_a = line[3] + label = line[1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +class Sst2Processor(DataProcessor): + """Processor for the SST-2 data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, i) + text_a = line[0] + label = line[1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +class StsbProcessor(DataProcessor): + """Processor for the STS-B data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return [None] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[7] + text_b = line[8] + label = line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class QqpProcessor(DataProcessor): + """Processor for the QQP data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + try: + text_a = line[3] + text_b = line[4] + label = line[5] + except IndexError: + continue + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class QnliProcessor(DataProcessor): + """Processor for the QNLI data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["entailment", "not_entailment"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[1] + text_b = line[2] + label = line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class RteProcessor(DataProcessor): + """Processor for the RTE data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["entailment", "not_entailment"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[1] + text_b = line[2] + label = line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class WnliProcessor(DataProcessor): + """Processor for the WNLI data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[1] + text_b = line[2] + label = line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class InputExample(object): + """A single training/test example for simple sequence classification. + + Args: + guid: Unique id for the example. + text_a: The untokenized text of the first sequence. + For single sequence tasks, only this sequence must be specified. + text_b: The untokenized text of the second + sequence. Only must be specified for sequence pair tasks. + label:The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + + def __init__(self, guid: int, text_a: str, text_b: str = None, label: str = None): + """Constructs a InputExample.""" + self.guid = guid + self.text_a = text_a + self.text_b = text_b + self.label = label diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/glue_benchmark_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/glue_benchmark_dataset.py new file mode 100644 index 0000000000..0b3f6aee4f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/glue_benchmark/glue_benchmark_dataset.py @@ -0,0 +1,365 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Some code of this file was adapted from the HuggingFace library available at +# https://github.com/huggingface/transformers + +import os +import pickle +from typing import Dict, List, Optional, Union + +import numpy as np +import torch + +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec +from nemo.collections.nlp.data.glue_benchmark.data_processors import ( + ColaProcessor, + MnliMismatchedProcessor, + MnliProcessor, + MrpcProcessor, + QnliProcessor, + QqpProcessor, + RteProcessor, + Sst2Processor, + StsbProcessor, + WnliProcessor, +) +from nemo.core.classes import Dataset +from nemo.core.neural_types import CategoricalValuesType, ChannelType, MaskType, NeuralType, RegressionValuesType +from nemo.utils import logging + +__all__ = ['GLUEDataset'] + +processors = { + "cola": ColaProcessor, + "mnli": MnliProcessor, + "mnli-mm": MnliMismatchedProcessor, + "mrpc": MrpcProcessor, + "sst-2": Sst2Processor, + "sts-b": StsbProcessor, + "qqp": QqpProcessor, + "qnli": QnliProcessor, + "rte": RteProcessor, + "wnli": WnliProcessor, +} +output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", +} +GLUE_TASKS_NUM_LABELS = { + "cola": 2, + "mnli": 3, + "mrpc": 2, + "sst-2": 2, + "sts-b": 1, + "qqp": 2, + "qnli": 2, + "rte": 2, + "wnli": 2, +} + + +class GLUEDataset(Dataset): + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + 'input_ids': NeuralType(('B', 'T'), ChannelType()), + 'segment_ids': NeuralType(('B', 'T'), ChannelType()), + 'input_mask': NeuralType(('B', 'T'), MaskType()), + "labels": NeuralType( + tuple('B'), RegressionValuesType() if self.task_name == 'sts-b' else CategoricalValuesType() + ), + } + + def __init__( + self, file_name: str, task_name: str, tokenizer: TokenizerSpec, max_seq_length: str, use_cache: bool = True, + ): + """ + Processes GLUE datasets + Args: + file_name: path to file + task_name: GLUE task name + tokenizer: such as AutoTokenizer + max_seq_length: max sequence length minus 2 for [CLS] and [SEP] + use_cache: whether to use data cache + """ + logging.info(f'Processing {file_name}') + data_dir, file_name = os.path.split(file_name) + file_name = file_name[:-4] + self.tokenizer = tokenizer + evaluate = False if 'train' in file_name else True + + if task_name not in processors: + raise ValueError(f'{task_name} not supported. Choose from {processors.keys()}') + + if task_name == 'mnli' and 'dev_mismatched' in file_name: + self.task_name = 'mnli-mm' + else: + self.task_name = task_name + + processor = processors[self.task_name]() + output_mode = output_modes[self.task_name] + self.label_list = processor.get_labels() + + self.examples = processor.get_dev_examples(data_dir) if evaluate else processor.get_train_examples(data_dir) + processor_name = type(processor).__name__ + vocab_size = getattr(tokenizer, "vocab_size", 0) + cached_features_file = os.path.join( + data_dir, + "cached_{}_{}_{}_{}_{}".format( + processor_name, file_name, tokenizer.name, str(max_seq_length), str(vocab_size) + ), + ) + + if use_cache and os.path.exists(cached_features_file): + logging.info(f"loading from {cached_features_file}") + with open(cached_features_file, "rb") as reader: + self.features = pickle.load(reader) + else: + token_params = { + 'bos_token': None, + 'eos_token': tokenizer.eos_token, + 'pad_token': tokenizer.pad_token, + 'cls_token': tokenizer.cls_token, + 'sep_token_extra': tokenizer.eos_token if 'roberta' in tokenizer.name.lower() else None, + } + + self.features = self.convert_examples_to_features( + self.examples, self.label_list, max_seq_length, tokenizer, output_mode, **token_params + ) + master_device = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + if master_device: + logging.info(f'Saving train features into {cached_features_file}') + with open(cached_features_file, "wb") as writer: + pickle.dump(self.features, writer) + + def __len__(self): + return len(self.features) + + def __getitem__(self, idx): + feature = self.features[idx] + return ( + np.array(feature.input_ids), + np.array(feature.segment_ids), + np.array(feature.input_mask, dtype=np.long), + np.array(feature.label_id), + ) + + def convert_examples_to_features( + self, + examples: List[str], + label_list: List[int], + max_seq_length: int, + tokenizer: TokenizerSpec, + output_mode: str, + bos_token: str = None, + eos_token: str = '[SEP]', + pad_token: str = '[PAD]', + cls_token: str = '[CLS]', + sep_token_extra: str = None, + cls_token_at_end: bool = False, + cls_token_segment_id: int = 0, + pad_token_segment_id: int = 0, + pad_on_left: bool = False, + mask_padding_with_zero: bool = True, + sequence_a_segment_id: int = 0, + sequence_b_segment_id: int = 1, + ): + """ + Loads a data file into a list of `InputBatch`s. + The `cls_token_at_end` defines the location of the CLS token: + + * False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP] + * True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS] + + The `cls_token_segment_id` defines the segment id associated to the CLS token (0 for BERT, 2 for XLNet) + + The convention in BERT is: + + a. For sequence pairs: + * tokens: [CLS] is this jack ##ville ? [SEP] no it is not . [SEP] + * type_ids: 0 0 0 0 0 0 0 1 1 1 1 1 1 + b. For single sequences: + * tokens: [CLS] the dog is hairy . [SEP] + * type_ids: 0 0 0 0 0 0 0 + + Where "type_ids" are used to indicate whether this is the first + sequence or the second sequence. The embedding vectors for `type=0` + and `type=1` were learned during pre-training and are added to the + wordpiece embedding vector (and position vector). This is + not *strictly* necessarysince the [SEP] token unambiguously separates + the sequences, but it makes it easier for the model to learn + the concept of sequences. + For classification tasks, the first vector (corresponding to [CLS]) + is used as as the "sentence vector". Note that this only makes sense + because the entire model is fine-tuned. + + The convention for NMT is: + + a. For sequence pairs: + * tokens: is this jack ##ville ? no it is not . + * type_ids:0 0 0 0 0 0 0 1 1 1 1 1 1 1 + b. For single sequences: + * tokens: the dog is hairy . + * type_ids: 0 0 0 0 0 0 0 + + """ + label_map = {label: i for i, label in enumerate(label_list)} + + features = [] + for ex_index, example in enumerate(examples): + if ex_index % 10000 == 0: + logging.info("Writing example %d of %d" % (ex_index, len(examples))) + + tokens_a = tokenizer.text_to_tokens(example.text_a) + + tokens_b = None + if example.text_b: + tokens_b = tokenizer.text_to_tokens(example.text_b) + + special_tokens_count = 2 if eos_token else 0 + special_tokens_count += 1 if sep_token_extra else 0 + special_tokens_count += 2 if bos_token else 0 + special_tokens_count += 1 if cls_token else 0 + self._truncate_seq_pair(tokens_a, tokens_b, max_seq_length - special_tokens_count) + else: + special_tokens_count = 1 if eos_token else 0 + special_tokens_count += 1 if sep_token_extra else 0 + special_tokens_count += 1 if bos_token else 0 + if len(tokens_a) > max_seq_length - special_tokens_count: + tokens_a = tokens_a[: max_seq_length - special_tokens_count] + # Add special tokens to sequence_a + tokens = tokens_a + if bos_token: + tokens = [bos_token] + tokens + if eos_token: + tokens += [eos_token] + segment_ids = [sequence_a_segment_id] * len(tokens) + + # Add sequence separator between sequences + if tokens_b and sep_token_extra: + tokens += [sep_token_extra] + segment_ids += [sequence_a_segment_id] + + # Add special tokens to sequence_b + if tokens_b: + if bos_token: + tokens += [bos_token] + segment_ids += [sequence_b_segment_id] + tokens += tokens_b + segment_ids += [sequence_b_segment_id] * (len(tokens_b)) + if eos_token: + tokens += [eos_token] + segment_ids += [sequence_b_segment_id] + + # Add classification token - for BERT models + if cls_token: + if cls_token_at_end: + tokens += [cls_token] + segment_ids += [cls_token_segment_id] + else: + tokens = [cls_token] + tokens + segment_ids = [cls_token_segment_id] + segment_ids + input_ids = tokenizer.tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) + + # Zero-pad up to the sequence length. + padding_length = max_seq_length - len(input_ids) + pad_token_id = tokenizer.tokens_to_ids([pad_token])[0] + if pad_on_left: + input_ids = ([pad_token_id] * padding_length) + input_ids + input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask + segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids + else: + input_ids = input_ids + ([pad_token_id] * padding_length) + input_mask = input_mask + ([0 if mask_padding_with_zero else 1] * padding_length) + segment_ids = segment_ids + ([pad_token_segment_id] * padding_length) + if len(input_ids) != max_seq_length: + raise ValueError("input_ids must be of length max_seq_length") + if len(input_mask) != max_seq_length: + raise ValueError("input_mask must be of length max_seq_length") + if len(segment_ids) != max_seq_length: + raise ValueError("segment_ids must be of length max_seq_length") + if output_mode == "classification": + label_id = label_map[example.label] + elif output_mode == "regression": + label_id = np.float32(example.label) + else: + raise KeyError(output_mode) + + if ex_index < 5: + logging.info("*** Example ***") + logging.info("guid: %s" % (example.guid)) + logging.info("tokens: %s" % " ".join(list(map(str, tokens)))) + logging.info("input_ids: %s" % " ".join(list(map(str, input_ids)))) + logging.info("input_mask: %s" % " ".join(list(map(str, input_mask)))) + logging.info("segment_ids: %s" % " ".join(list(map(str, segment_ids)))) + logging.info("label: %s (id = %d)" % (example.label, label_id)) + + features.append( + InputFeatures(input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids, label_id=label_id) + ) + return features + + def _truncate_seq_pair(self, tokens_a: str, tokens_b: str, max_length: int): + """Truncates a sequence pair in place to the maximum length. + + This will always truncate the longer sequence one token at a time. + This makes more sense than truncating an equal percent + of tokens from each, since if one sequence is very short then each token + that's truncated likely contains more information than a longer sequence. + """ + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + + +class InputFeatures(object): + """A single set of features of data. + + Args: + input_ids: input/token ids + input_mask: masks out subword tokens + segment_ids: distinguish one sentence from the other one (if present) + label_ids: label for the current example + """ + + def __init__( + self, input_ids: List[int], input_mask: List[int], segment_ids: List[int], label_id: Union[float, int] + ): + """Initialized InputFeatures.""" + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.label_id = label_id diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/information_retrieval/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/information_retrieval/__init__.py new file mode 100644 index 0000000000..a32196ee7c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/information_retrieval/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.information_retrieval.information_retrieval_dataset import ( + BertInformationRetrievalDataset, +) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/information_retrieval/information_retrieval_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/information_retrieval/information_retrieval_dataset.py new file mode 100644 index 0000000000..61e0a3cffc --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/information_retrieval/information_retrieval_dataset.py @@ -0,0 +1,278 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing as mp +import os +import pickle +import random +from typing import Optional + +import numpy as np +from torch.utils.data import Dataset + +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec + +__all__ = ["BertInformationRetrievalDataset"] + + +class BaseInformationRetrievalDataset(Dataset): + """ + Base information retrieval dataset on which other datasets are built. + + Args: + tokenizer: tokenizer + max_query_length: maximum length of query in tokens + max_passage_length: maximum length of passage in tokens + """ + + def __init__( + self, tokenizer: TokenizerSpec, max_query_length: Optional[int] = 31, max_passage_length: Optional[int] = 190, + ): + self.tokenizer = tokenizer + self.max_query_length = max_query_length + self.max_passage_length = max_passage_length + + def parse_npz(self, file, max_seq_length): + """ + Function which parses passages (documents) in npz format. + After pre-processing and tokenization, the dataset will be saved + as numpy matrix, i_th entry of which corresponds to i_th passage (document) + and has the following form: + [n, token_1, ..., token_n, 0, ..., 0] + where n is the passage length (in tokens) and 0s correspond to pad tokens. + + Args: + file: str, path to file with passages (documents) + max_seq_length: maximum length of sequence in tokens + """ + cached_collection = file + ".npz" + if os.path.isfile(cached_collection): + dataset_npz = np.load(cached_collection)["data"] + else: + dataset_dict = self.tokenize_dataset(file, max_seq_length) + dataset_npz = np.zeros((len(dataset_dict), max_seq_length + 1)) + for key in dataset_dict: + dataset_npz[key][0] = len(dataset_dict[key]) + dataset_npz[key][1 : len(dataset_dict[key]) + 1] = dataset_dict[key] + np.savez(cached_collection, data=dataset_npz) + return dataset_npz + + def parse_pkl(self, file, max_seq_length): + """ + Function which parses passages (documents, queries) in pkl format. + After pre-processing and tokenization, the dataset will be saved + as pkl dict, i_th entry of which corresponds to i_th passage (document, query) + and has the following form: + {passage_id: [token_1, ..., token_n]} + where n is the passage length (in tokens). + + Args: + file: str, path to file with passages (documents) + max_seq_length: maximum length of sequence in tokens + """ + cached_collection = file + ".pkl" + if os.path.isfile(cached_collection): + dataset_dict = pickle.load(open(cached_collection, "rb")) + else: + dataset_dict = self.tokenize_dataset(file, max_seq_length) + pickle.dump(dataset_dict, open(cached_collection, "wb")) + return dataset_dict + + def tokenize_dataset(self, file, max_seq_length): + """ + Function which pre-tokenizes the dataset. + """ + lines = open(file, "r").readlines() + with mp.Pool() as pool: + dataset_dict = pool.map(self.preprocess_line, lines) + dataset_dict = {id_: tokens[:max_seq_length] for (id_, tokens) in dataset_dict} + return dataset_dict + + def preprocess_line(self, line): + """ + Parse a single entry (line) of tsv file. + """ + if "\t" not in line: + raise ValueError(f"Provided dataset does not have a form of tsv file") + id_, text = line.split("\t") + token_ids = self.tokenizer.text_to_ids(text.strip()) + return int(id_), token_ids + + def construct_input(self, token_ids1, max_seq_length, token_ids2=None): + """ + Function which constructs a valid input to BERT from tokens. + + If only one list of tokens (token_ids1) is passed, the input will be + [CLS] token_ids1 [SEP] + + if two lists of tokens are passed, the input will be + [CLS] token_ids1 [SEP] token_ids2 [SEP] + """ + + input_ids = [self.tokenizer.pad_id] * max_seq_length + bert_input = [self.tokenizer.cls_id] + token_ids1 + [self.tokenizer.sep_id] + sentence1_length = len(bert_input) + if token_ids2 is not None: + bert_input = bert_input + token_ids2 + [self.tokenizer.sep_id] + + bert_input = bert_input[:max_seq_length] + + num_nonpad_tokens = len(bert_input) + + input_ids[:num_nonpad_tokens] = bert_input + input_ids = np.array(input_ids, dtype=np.long) + input_mask = input_ids != self.tokenizer.pad_id + input_type_ids = np.ones_like(input_ids) + input_type_ids[:sentence1_length] = 0 + + return input_ids, input_mask, input_type_ids + + def preprocess_bert(self, query_id, psg_ids): + """ + Transforms query id (Q) and a list of passages ids (P1, ..., Pk) + into a tensor of size [k, max_length] with the following rows: + [CLS] Q_text [SEP] Pi_text [SEP], i = 1, ..., k + """ + + max_seq_length = self.max_query_length + self.max_passage_length + 3 + input_ids, input_mask, input_type_ids = [], [], [] + for psg_id in psg_ids: + inputs = self.construct_input(self.queries[query_id], max_seq_length, self._psgid2tokens(psg_id)) + input_ids.append(inputs[0]) + input_mask.append(inputs[1]) + input_type_ids.append(inputs[2]) + + input_ids = np.stack(input_ids) + input_mask = np.stack(input_mask) + input_type_ids = np.stack(input_type_ids) + + return input_ids, input_mask, input_type_ids + + def preprocess_dpr(self, query_id, psg_ids): + """ + Transforms query id (Q) and a list of passages ids (P1, ..., Pk) + into two tensors of sizes [1, max_q_length] and [k, max_p_length] + with the following rows: + 1) [CLS] Q_text [SEP] + 2) [CLS] Pi_text [SEP], i = 1, ..., k + """ + + q_input_ids, q_input_mask, q_type_ids = self.construct_input(self.queries[query_id], self.max_query_length + 2) + input_ids, input_mask, input_type_ids = [], [], [] + for psg_id in psg_ids: + inputs = self.construct_input(self._psgid2tokens(psg_id), self.max_passage_length + 2) + input_ids.append(inputs[0]) + input_mask.append(inputs[1]) + input_type_ids.append(inputs[2]) + input_ids = np.stack(input_ids) + input_mask = np.stack(input_mask) + input_type_ids = np.stack(input_type_ids) + return ( + q_input_ids[None, ...], + q_input_mask[None, ...], + q_type_ids[None, ...], + input_ids, + input_mask, + input_type_ids, + ) + + def _psgid2tokens(self, psg_id): + """ + Internal function which maps passage id to its tokens. + """ + pass + + def psgid2tokens_npz(self, psg_id): + """ + Mapping from passage id to its tokens in case of npz cache format. + """ + seq_len = self.passages[psg_id][0] + return self.passages[psg_id][1 : seq_len + 1].tolist() + + def psgid2tokens_pkl(self, psg_id): + """ + Mapping from passage id to its tokens in case of pkl cache format. + """ + return self.passages[psg_id] + + +class BertInformationRetrievalDataset(BaseInformationRetrievalDataset): + def __init__( + self, + tokenizer: TokenizerSpec, + passages: str, + queries: str, + query_to_passages: str, + max_query_length: Optional[int] = 31, + max_passage_length: Optional[int] = 190, + num_negatives: Optional[int] = 10, + preprocess_fn: Optional[str] = "preprocess_bert", + psg_cache_format: Optional[str] = "npz", + ): + """ + Dataset for training information retrieval models. + + Args: + tokenizer: tokenizer + passages: path to tsv with [psg_id, psg_text] entries + queries: path to tsv with [query_id, query_text] entries + query_to_passages: path to tsv with + [query_id, pos_psg_id, neg_psg_id_1, ..., neg_psg_id_k] entries + max_query_length: maximum length of query in tokens + max_passage_length: maximum length of passage in tokens + num_negatives: number of negative passages per positive to use for training + preprocess_fn: either preprocess_bert or preprocess_dpr + preprocess_bert: joint input: [CLS] query [SEP] passage [SEP] + preprocess_dpr: separate inputs: [CLS] query [SEP], [CLS] passage [SEP] + psg_cache_format: either pkl or npz + """ + + super().__init__(tokenizer, max_query_length, max_passage_length) + self.num_negatives = num_negatives + + self.passages = getattr(self, f"parse_{psg_cache_format}")(passages, max_passage_length) + self._psgid2tokens = getattr(self, f"psgid2tokens_{psg_cache_format}") + self.queries = self.parse_pkl(queries, max_query_length) + self.idx2psgs = self.parse_query_to_passages(query_to_passages) + self._preprocess_fn = getattr(self, preprocess_fn) + + def __getitem__(self, idx): + query_and_psgs = self.idx2psgs[idx] + query_id, psg_ids = query_and_psgs[0], query_and_psgs[1:] + inputs = self._preprocess_fn(query_id, psg_ids) + return [*inputs, query_id, np.array(psg_ids)] + + def __len__(self): + return len(self.idx2psgs) + + def parse_query_to_passages(self, file): + """ + Function which parses query to passages correspondence file. + """ + idx2psgs = {} + idx = 0 + for line in open(file, "r").readlines(): + if "\t" not in line: + raise ValueError(f"Provided dataset does not have a form of tsv file") + query_and_psgs = line.split("\t") + query_and_psgs_ids = [int(id_) for id_ in query_and_psgs] + query_and_rel_psg_ids, irrel_psgs_ids = query_and_psgs_ids[:2], query_and_psgs_ids[2:] + random.shuffle(irrel_psgs_ids) + num_samples = len(irrel_psgs_ids) // self.num_negatives + for j in range(num_samples): + left = self.num_negatives * j + right = self.num_negatives * (j + 1) + idx2psgs[idx] = query_and_rel_psg_ids + irrel_psgs_ids[left:right] + idx += 1 + return idx2psgs diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/__init__.py new file mode 100644 index 0000000000..f6336a7b1e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.collections.nlp.data.intent_slot_classification.intent_slot_classification_dataset import ( + IntentSlotClassificationDataset, + IntentSlotInferenceDataset, +) +from nemo.collections.nlp.data.intent_slot_classification.intent_slot_classification_descriptor import ( + IntentSlotDataDesc, +) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/intent_slot_classification_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/intent_slot_classification_dataset.py new file mode 100644 index 0000000000..66f09cbe20 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/intent_slot_classification_dataset.py @@ -0,0 +1,292 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, Optional + +import numpy as np + +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec +from nemo.collections.nlp.data.data_utils import get_stats +from nemo.core.classes import Dataset +from nemo.core.neural_types import ChannelType, LabelsType, MaskType, NeuralType +from nemo.utils import logging + +__all__ = ['IntentSlotClassificationDataset', 'IntentSlotInferenceDataset'] + + +def get_features( + queries, + max_seq_length, + tokenizer, + pad_label=128, + raw_slots=None, + ignore_extra_tokens=False, + ignore_start_end=False, +): + all_subtokens = [] + all_loss_mask = [] + all_subtokens_mask = [] + all_segment_ids = [] + all_input_ids = [] + all_input_mask = [] + sent_lengths = [] + all_slots = [] + + with_label = False + if raw_slots is not None: + with_label = True + + for i, query in enumerate(queries): + words = query.strip().split() + subtokens = [tokenizer.cls_token] + loss_mask = [1 - ignore_start_end] + subtokens_mask = [0] + if with_label: + slots = [pad_label] + + for j, word in enumerate(words): + word_tokens = tokenizer.text_to_tokens(word) + subtokens.extend(word_tokens) + + loss_mask.append(1) + loss_mask.extend([int(not ignore_extra_tokens)] * (len(word_tokens) - 1)) + + subtokens_mask.append(1) + subtokens_mask.extend([0] * (len(word_tokens) - 1)) + + if with_label: + slots.extend([raw_slots[i][j]] * len(word_tokens)) + + subtokens.append(tokenizer.sep_token) + loss_mask.append(1 - ignore_start_end) + subtokens_mask.append(0) + sent_lengths.append(len(subtokens)) + all_subtokens.append(subtokens) + all_loss_mask.append(loss_mask) + all_subtokens_mask.append(subtokens_mask) + all_input_mask.append([1] * len(subtokens)) + if with_label: + slots.append(pad_label) + all_slots.append(slots) + + max_seq_length_data = max(sent_lengths) + max_seq_length = min(max_seq_length, max_seq_length_data) if max_seq_length > 0 else max_seq_length_data + logging.info(f'Setting max length to: {max_seq_length}') + get_stats(sent_lengths) + too_long_count = 0 + + for i, subtokens in enumerate(all_subtokens): + if len(subtokens) > max_seq_length: + subtokens = [tokenizer.cls_token] + subtokens[-max_seq_length + 1 :] + all_input_mask[i] = [1] + all_input_mask[i][-max_seq_length + 1 :] + all_loss_mask[i] = [1 - ignore_start_end] + all_loss_mask[i][-max_seq_length + 1 :] + all_subtokens_mask[i] = [0] + all_subtokens_mask[i][-max_seq_length + 1 :] + + if with_label: + all_slots[i] = [pad_label] + all_slots[i][-max_seq_length + 1 :] + too_long_count += 1 + + all_input_ids.append([tokenizer.tokens_to_ids(t) for t in subtokens]) + + if len(subtokens) < max_seq_length: + extra = max_seq_length - len(subtokens) + all_input_ids[i] = all_input_ids[i] + [0] * extra + all_loss_mask[i] = all_loss_mask[i] + [0] * extra + all_subtokens_mask[i] = all_subtokens_mask[i] + [0] * extra + all_input_mask[i] = all_input_mask[i] + [0] * extra + + if with_label: + all_slots[i] = all_slots[i] + [pad_label] * extra + + all_segment_ids.append([0] * max_seq_length) + + logging.info(f'{too_long_count} are longer than {max_seq_length}') + + # May be useful for debugging + logging.debug("*** Some Examples of Processed Data ***") + for i in range(min(len(all_input_ids), 5)): + logging.debug("i: %s" % (i)) + logging.debug("subtokens: %s" % " ".join(list(map(str, all_subtokens[i])))) + logging.debug("loss_mask: %s" % " ".join(list(map(str, all_loss_mask[i])))) + logging.debug("input_mask: %s" % " ".join(list(map(str, all_input_mask[i])))) + logging.debug("subtokens_mask: %s" % " ".join(list(map(str, all_subtokens_mask[i])))) + if with_label: + logging.debug("slots_label: %s" % " ".join(list(map(str, all_slots[i])))) + + return (all_input_ids, all_segment_ids, all_input_mask, all_loss_mask, all_subtokens_mask, all_slots) + + +class IntentSlotClassificationDataset(Dataset): + """ + Creates dataset to use for the task of joint intent + and slot classification with pretrained model. + + Converts from raw data to an instance that can be used by + NMDataLayer. + + For dataset to use during inference without labels, see + IntentSlotDataset. + + Args: + input_file: file to sequence + label. the first line is header (sentence [tab] label) + each line should be [sentence][tab][label] + slot_file: file to slot labels, each line corresponding to slot labels for a sentence in input_file. No header. + max_seq_length: max sequence length minus 2 for [CLS] and [SEP] + tokenizer: such as NemoBertTokenizer + num_samples: number of samples you want to use for the dataset. If -1, use all dataset. Useful for testing. + pad_label: pad value use for slot labels. by default, it's the neutral label. + ignore_extra_tokens: whether to ignore extra tokens in the loss_mask. + ignore_start_end: whether to ignore bos and eos tokens in the loss_mask. + do_lower_case: convert query to lower case or not + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + 'input_ids': NeuralType(('B', 'T'), ChannelType()), + 'segment_ids': NeuralType(('B', 'T'), ChannelType()), + 'input_mask': NeuralType(('B', 'T'), MaskType()), + 'loss_mask': NeuralType(('B', 'T'), MaskType()), + 'subtokens_mask': NeuralType(('B', 'T'), MaskType()), + 'intent_labels': NeuralType(('B'), LabelsType()), + 'slot_labels': NeuralType(('B', 'T'), LabelsType()), + } + + def __init__( + self, + input_file: str, + slot_file: str, + max_seq_length: int, + tokenizer: TokenizerSpec, + num_samples: int = -1, + pad_label: int = 128, + ignore_extra_tokens: bool = False, + ignore_start_end: bool = False, + do_lower_case: bool = False, + ): + if num_samples == 0: + raise ValueError("num_samples has to be positive", num_samples) + + with open(slot_file, 'r') as f: + slot_lines = f.readlines() + + with open(input_file, 'r') as f: + input_lines = f.readlines()[1:] + + assert len(slot_lines) == len(input_lines) + + dataset = list(zip(slot_lines, input_lines)) + + if num_samples > 0: + dataset = dataset[:num_samples] + + raw_slots, queries, raw_intents = [], [], [] + for slot_line, input_line in dataset: + raw_slots.append([int(slot) for slot in slot_line.strip().split()]) + parts = input_line.strip().split() + raw_intents.append(int(parts[-1])) + query = ' '.join(parts[:-1]) + if do_lower_case: + query = query.lower() + queries.append(query) + + features = get_features( + queries, + max_seq_length, + tokenizer, + pad_label=pad_label, + raw_slots=raw_slots, + ignore_extra_tokens=ignore_extra_tokens, + ignore_start_end=ignore_start_end, + ) + self.all_input_ids = features[0] + self.all_segment_ids = features[1] + self.all_input_mask = features[2] + self.all_loss_mask = features[3] + self.all_subtokens_mask = features[4] + self.all_slots = features[5] + self.all_intents = raw_intents + + def __len__(self): + return len(self.all_input_ids) + + def __getitem__(self, idx): + return ( + np.array(self.all_input_ids[idx]), + np.array(self.all_segment_ids[idx]), + np.array(self.all_input_mask[idx], dtype=np.long), + np.array(self.all_loss_mask[idx]), + np.array(self.all_subtokens_mask[idx]), + self.all_intents[idx], + np.array(self.all_slots[idx]), + ) + + +class IntentSlotInferenceDataset(Dataset): + """ + Creates dataset to use for the task of joint intent + and slot classification with pretrained model. + This is to be used during inference only. + It uses list of queries as the input. + + Args: + queries (list): list of queries to run inference on + max_seq_length (int): max sequence length minus 2 for [CLS] and [SEP] + tokenizer (Tokenizer): such as NemoBertTokenizer + pad_label (int): pad value use for slot labels. + by default, it's the neutral label. + + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """ + Returns definitions of module output ports. + """ + return { + 'input_ids': NeuralType(('B', 'T'), ChannelType()), + 'segment_ids': NeuralType(('B', 'T'), ChannelType()), + 'input_mask': NeuralType(('B', 'T'), MaskType()), + 'loss_mask': NeuralType(('B', 'T'), MaskType()), + 'subtokens_mask': NeuralType(('B', 'T'), MaskType()), + } + + def __init__(self, queries, max_seq_length, tokenizer, do_lower_case): + if do_lower_case: + for idx, query in enumerate(queries): + queries[idx] = queries[idx].lower() + + features = get_features(queries, max_seq_length, tokenizer) + + self.all_input_ids = features[0] + self.all_segment_ids = features[1] + self.all_input_mask = features[2] + self.all_loss_mask = features[3] + self.all_subtokens_mask = features[4] + + def __len__(self): + return len(self.all_input_ids) + + def __getitem__(self, idx): + return ( + np.array(self.all_input_ids[idx]), + np.array(self.all_segment_ids[idx]), + np.array(self.all_input_mask[idx], dtype=np.long), + np.array(self.all_loss_mask[idx]), + np.array(self.all_subtokens_mask[idx]), + ) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/intent_slot_classification_descriptor.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/intent_slot_classification_descriptor.py new file mode 100644 index 0000000000..544b5e1db8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/intent_slot_classification/intent_slot_classification_descriptor.py @@ -0,0 +1,163 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +from typing import List + +from nemo.collections.nlp.data.data_utils.data_preprocessing import ( + fill_class_weights, + get_freq_weights, + get_label_stats, + if_exist, +) +from nemo.utils import logging + + +class IntentSlotDataDesc: + """ Convert the raw data to the standard format supported by + IntentSlotDataDesc. + + By default, the None label for slots is 'O'. + + IntentSlotDataDesc requires two files: + + input_file: file to sequence + label. + the first line is header (sentence [tab] label) + each line should be [sentence][tab][label] + + slot_file: file to slot labels, each line corresponding to + slot labels for a sentence in input_file. No header. + + To keep the mapping from label index to label consistent during + training and inferencing we require the following files: + dicts.intents.csv: each line is an intent. The first line + corresponding to the 0 intent label, the second line + corresponding to the 1 intent label, and so on. + + dicts.slots.csv: each line is a slot. The first line + corresponding to the 0 slot label, the second line + corresponding to the 1 slot label, and so on. + + Args: + data_dir: the directory of the dataset + modes: ['train', 'test', 'dev'], + none_slot_label: the label for slots that aren't identified defaulted to 'O' + pad_label: the int used for padding. If set to -1, it'll be set to the whatever the None label is. + """ + + def __init__( + self, + data_dir: str, + modes: List[str] = ['train', 'test', 'dev'], + none_slot_label: str = 'O', + pad_label: int = -1, + ): + if not if_exist(data_dir, ['dict.intents.csv', 'dict.slots.csv']): + raise FileNotFoundError( + "Make sure that your data follows the standard format " + "supported by JointIntentSlotDataset. Your data must " + "contain dict.intents.csv and dict.slots.csv." + ) + + self.data_dir = data_dir + self.intent_dict_file = self.data_dir + '/dict.intents.csv' + self.slot_dict_file = self.data_dir + '/dict.slots.csv' + + self.intents_label_ids = IntentSlotDataDesc.label2idx(self.intent_dict_file) + self.num_intents = len(self.intents_label_ids) + self.slots_label_ids = IntentSlotDataDesc.label2idx(self.slot_dict_file) + self.num_slots = len(self.slots_label_ids) + + infold = self.data_dir + for mode in modes: + if not if_exist(self.data_dir, [f'{mode}.tsv']): + logging.info(f' Stats calculation for {mode} mode' f' is skipped as {mode}.tsv was not found.') + continue + logging.info(f' Stats calculating for {mode} mode...') + slot_file = f'{self.data_dir}/{mode}_slots.tsv' + with open(slot_file, 'r') as f: + slot_lines = f.readlines() + + input_file = f'{self.data_dir}/{mode}.tsv' + with open(input_file, 'r') as f: + input_lines = f.readlines()[1:] # Skipping headers at index 0 + + if len(slot_lines) != len(input_lines): + raise ValueError( + "Make sure that the number of slot lines match the " + "number of intent lines. There should be a 1-1 " + "correspondence between every slot and intent lines." + ) + + dataset = list(zip(slot_lines, input_lines)) + + raw_slots, raw_intents = [], [] + for slot_line, input_line in dataset: + slot_list = [int(slot) for slot in slot_line.strip().split()] + raw_slots.append(slot_list) + parts = input_line.strip().split() + raw_intents.append(int(parts[-1])) + + logging.info(f'Three most popular intents in {mode} mode:') + total_intents, intent_label_freq, max_id = get_label_stats( + raw_intents, infold + f'/{mode}_intent_stats.tsv' + ) + + merged_slots = itertools.chain.from_iterable(raw_slots) + logging.info(f'Three most popular slots in {mode} mode:') + slots_total, slots_label_freq, max_id = get_label_stats(merged_slots, infold + f'/{mode}_slot_stats.tsv') + + logging.info(f'Total Number of Intents: {total_intents}') + logging.info(f'Intent Label Frequencies: {intent_label_freq}') + logging.info(f'Total Number of Slots: {slots_total}') + logging.info(f'Slots Label Frequencies: {slots_label_freq}') + + if mode == 'train': + intent_weights_dict = get_freq_weights(intent_label_freq) + logging.info(f'Intent Weights: {intent_weights_dict}') + slot_weights_dict = get_freq_weights(slots_label_freq) + logging.info(f'Slot Weights: {slot_weights_dict}') + + self.intent_weights = fill_class_weights(intent_weights_dict, self.num_intents - 1) + self.slot_weights = fill_class_weights(slot_weights_dict, self.num_slots - 1) + + if pad_label != -1: + self.pad_label = pad_label + else: + if none_slot_label not in self.slots_label_ids: + raise ValueError(f'none_slot_label {none_slot_label} not ' f'found in {self.slot_dict_file}.') + self.pad_label = self.slots_label_ids[none_slot_label] + + @staticmethod + def label2idx(file): + lines = open(file, 'r').readlines() + lines = [line.strip() for line in lines if line.strip()] + labels = {lines[i]: i for i in range(len(lines))} + return labels + + @staticmethod + def intent_slot_dicts(data_dir): + ''' + Return Intent and slot dictionaries + ''' + intent_dict_file = data_dir + '/dict.intents.csv' + slot_dict_file = data_dir + '/dict.slots.csv' + + intents_labels = open(intent_dict_file, 'r').readlines() + intents_labels = [line.strip() for line in intents_labels if line.strip()] + + slots_labels = open(slot_dict_file, 'r').readlines() + slots_labels = [line.strip() for line in slots_labels if line.strip()] + + return intents_labels, slots_labels diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/__init__.py new file mode 100644 index 0000000000..16831be857 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.language_modeling.l2r_lm_dataset import L2RLanguageModelingDataset +from nemo.collections.nlp.data.language_modeling.lm_bert_dataset import ( + BertPretrainingDataset, + BertPretrainingPreprocessedDataloader, +) +from nemo.collections.nlp.data.language_modeling.sentence_dataset import SentenceDataset, TarredSentenceDataset diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/l2r_lm_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/l2r_lm_dataset.py new file mode 100644 index 0000000000..eff0737c7f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/l2r_lm_dataset.py @@ -0,0 +1,240 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import json +from typing import Optional + +import braceexpand +import numpy as np +import webdataset as wd +from torch.utils.data import Dataset, IterableDataset + +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec +from nemo.collections.nlp.data.data_utils import dataset_to_ids +from nemo.utils import logging + +__all__ = ['L2RLanguageModelingDataset', 'TarredL2RLanguageModelingDataset'] + + +class L2RLanguageModelingDataset(Dataset): + """ + Dataset for training and evaluating left-to-right language models. + + Args: + tokenizer: tokenizer, such as WordTokenizer or CharTokenizer + dataset: path to data + max_seq_length: maximum sequence length (in tokens) of input tensors + batch_step: distance (in tokens) between two successive sequences of + the text. By default, it is equal to max_seq_length which corresponds + to splitting text into disjoint segments covering full dataset + use_cache: bool value, defaults to False. Determines whether the preprocessed, + tokenized dataset should be cached into a pickle file. If true, cache is saved + at the path provided in `dataset`. + """ + + def __init__( + self, + tokenizer: TokenizerSpec, + dataset: str, + max_seq_length: Optional[int] = 512, + batch_step: Optional[int] = None, + use_cache: bool = False, + ): + self.tokenizer = tokenizer + self.max_seq_length = max_seq_length + self.batch_step = batch_step or self.max_seq_length + ids = dataset_to_ids(dataset, tokenizer, cache_ids=use_cache, add_bos_eos=False) + self.ids = np.array([j for i in ids for j in i]) + + def __len__(self): + return (len(self.ids) - self.max_seq_length) // self.batch_step + + def __getitem__(self, idx): + left = idx * self.batch_step + right = left + self.max_seq_length + src_ids = self.ids[left:right] + labels = self.ids[left + 1 : right + 1] + src_mask = (src_ids != self.tokenizer.pad_id).astype(np.float32) + return src_ids, src_mask, labels + + +class TarredL2RLanguageModelingDataset(IterableDataset): + """ + A similar Dataset to the L2RLanguageModelingDataset, but which loads tarred tokenized numpy files. + Accepts a single JSON metadata manifest file as well as the path(s) to the tarball(s) containing the wav files. + The manifest should contain information such as the number of shards, the number of tokens in the corpus, + and the number of tokens contained within each shard of the tarfile(s). + + Valid formats for the text_tar_filepaths argument include: + (1) a single string that can be brace-expanded, e.g. 'path/to/text.tar' or 'path/to/text_{1..100}.tar.gz', or + (2) a list of file paths that will not be brace-expanded, e.g. ['text_1.tar', 'text_2.tar', ...]. + + Note: For brace expansion in (1), there may be cases where `{x..y}` syntax cannot be used due to shell interference. + This occurs most commonly inside SLURM scripts. Therefore we provide a few equivalent replacements. + Supported opening braces - { <=> (, [, < and the special tag _OP_. + Supported closing braces - } <=> ), ], > and the special tag _CL_. + For SLURM based tasks, we suggest the use of the special tags for ease of use. + See the WebDataset documentation for more information about accepted data and input formats. + + If using multiple processes the number of shards should be divisible by the number of workers to ensure an + even split among workers. If it is not divisible, logging will give a warning but training will proceed. + + Additionally, please note that the len() of this DataLayer is assumed to be the number of tokens + of the text data. An incorrect manifest length may lead to some DataLoader issues down the line. + + Args: + text_tar_filepaths: Either a list of tokenized text tarball filepaths, or a + string (can be brace-expandable). + metadata_path (str): Path to the metadata manifest. + tokenizer: tokenizer, such as WordTokenizer or CharTokenizer + dataset: path to data + max_seq_length: maximum sequence length (in tokens) of input tensors + batch_step: distance (in tokens) between two successive sequences of + the text. By default, it is equal to max_seq_length which corresponds + to splitting text into disjoint segments covering full dataset + shuffle_n (int): How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + Defaults to 0. + shard_strategy (str): Tarred dataset shard distribution strategy chosen as a str value during ddp. + - `scatter`: The default shard strategy applied by WebDataset, where each node gets + a unique set of shards, which are permanently pre-allocated and never changed at runtime. + - `replicate`: Optional shard strategy, where each node gets all of the set of shards + available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. + The benefit of replication is that it allows each node to sample data points from the entire + dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. + Note: Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. + global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. + world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. + """ + + def __init__( + self, + text_tar_filepaths: str, + metadata_path: str, + tokenizer, + max_seq_length: int = 512, + batch_step: int = None, + shuffle_n: int = 1, + shard_strategy: str = "scatter", + global_rank: int = 0, + world_size: int = 0, + ): + super(TarredL2RLanguageModelingDataset, self).__init__() + + self.tokenizer = tokenizer + self.max_seq_length = max_seq_length + self.batch_step = batch_step or self.max_seq_length + + valid_shard_strategies = ['scatter', 'replicate'] + if shard_strategy not in valid_shard_strategies: + raise ValueError(f"`shard_strategy` must be one of {valid_shard_strategies}") + + with open(metadata_path, 'r') as f: + metadata = json.load(f) + + self.metadata = metadata + + if isinstance(text_tar_filepaths, str): + # Replace '(', '[', '<' and '_OP_' with '{' + brace_keys_open = ['(', '[', '<', '_OP_'] + for bkey in brace_keys_open: + if bkey in text_tar_filepaths: + text_tar_filepaths = text_tar_filepaths.replace(bkey, "{") + + # Replace ')', ']', '>' and '_CL_' with '}' + brace_keys_close = [')', ']', '>', '_CL_'] + for bkey in brace_keys_close: + if bkey in text_tar_filepaths: + text_tar_filepaths = text_tar_filepaths.replace(bkey, "}") + + if isinstance(text_tar_filepaths, str): + # Brace expand + text_tar_filepaths = list(braceexpand.braceexpand(text_tar_filepaths)) + + if shard_strategy == 'scatter': + logging.info("All tarred dataset shards will be scattered evenly across all nodes.") + + if len(text_tar_filepaths) % world_size != 0: + logging.warning( + f"Number of shards in tarred dataset ({len(text_tar_filepaths)}) is not divisible " + f"by number of distributed workers ({world_size})." + ) + + begin_idx = (len(text_tar_filepaths) // world_size) * global_rank + end_idx = begin_idx + (len(text_tar_filepaths) // world_size) + text_tar_filepaths = text_tar_filepaths[begin_idx:end_idx] + logging.info( + "Partitioning tarred dataset: process (%d) taking shards [%d, %d)", global_rank, begin_idx, end_idx + ) + + elif shard_strategy == 'replicate': + logging.info("All tarred dataset shards will be replicated across all nodes.") + + else: + raise ValueError(f"Invalid shard strategy ! Allowed values are : {valid_shard_strategies}") + + self.tarpath = text_tar_filepaths + + # Put together WebDataset + self._dataset = wd.WebDataset(urls=text_tar_filepaths, nodesplitter=None) + + if shuffle_n > 0: + self._dataset = self._dataset.shuffle(shuffle_n) + else: + logging.info("WebDataset will not shuffle files within the tar files.") + + self._dataset = self._dataset.rename(npy='npy', key='__key__').to_tuple('npy', 'key').map(f=self._build_sample) + + def _build_sample(self, tup): + # Load file + npy, filepath = tup + npy = io.BytesIO(npy) + data = np.load(npy) # loads np.int64 vector + npy.close() + + # Select random contiguous subsegment + idx = np.random.randint(0, (len(data) - self.max_seq_length) // self.batch_step) + + # Slice of data chunk + left = idx * self.batch_step + right = left + self.max_seq_length + data = data[left : right + 1] + + # Create batch + src_ids = data[:-1] + labels = data[1:] + src_mask = (src_ids != self.tokenizer.pad_id).astype(np.float32) + return src_ids, src_mask, labels + + def __iter__(self): + # We need to wrap an infinite generator since the actual files + # within the tar files contains large chunks of contiguous data. + # This prevents PTL from early exiting the train loop after exhausting + # all of the files in one iteration (though the actual dataset is many + # times larger due to each file containing a large chunk of data). + dl_iter = iter(self._dataset) + while True: + try: + batch = next(dl_iter) + yield batch + except StopIteration: + dl_iter = iter(self._dataset) + continue + + def __len__(self): + return (self.metadata['num_text'] - self.max_seq_length) // self.batch_step diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/lm_bert_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/lm_bert_dataset.py new file mode 100644 index 0000000000..4196f9736d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/lm_bert_dataset.py @@ -0,0 +1,406 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import array +import os +import pickle +import random +from typing import Dict, List, Optional + +import h5py +import numpy as np +from torch.utils.data import DataLoader, DistributedSampler +from tqdm import tqdm + +from nemo.collections.nlp.data.data_utils.data_preprocessing import find_newlines, load_data_indices +from nemo.core.classes import Dataset + +__all__ = ['BertPretrainingDataset', 'BertPretrainingPreprocessedDataloader'] + + +def load_h5(input_file: str): + return h5py.File(input_file, "r") + + +class BertPretrainingDataset(Dataset): + """ + Dataset for bert pretraining when using data preprocessing including tokenization + """ + + def __init__( + self, + tokenizer: object, + data_file: str, + max_seq_length: Optional[int] = 128, + mask_prob: Optional[float] = 0.15, + short_seq_prob: Optional[float] = 0.1, + seq_a_ratio: Optional[float] = 0.6, + sentence_idx_file: Optional[str] = None, + ): + """ + Args: + tokenizer: tokenizer + data_file: path to data + max_seq_length: maximum sequence length of input tensors + mask_probability: proability to mask token + short_seq_prob: probability to create a sequence shorter than max_seq_length + seq_a_ratio: ratio between lengths of first and second sequence + sentence_idx_file: sentence indices file for caching + """ + self.tokenizer = tokenizer + + # Loading enormous datasets into RAM isn't always feasible -- for + # example, the pubmed corpus is 200+ GB, which doesn't fit into RAM on + # most computers. To get around this, we store the indices of newlines + # in each file so we can seek to and retrieve sentences immediately + # from main memory when needed during training. + + # Try and load sentence indices file if already exists + sentence_indices, sentence_idx_file, data_dir = load_data_indices( + sentence_idx_file, data_file, "sentence_indices" + ) + + # If sentence indices file doesn't exists, generate and store sentence indices + if sentence_indices is None: + sentence_indices = {} + filenames = [data_file] + + for filename in tqdm(filenames): + with open(filename, "rb") as f: + contents = f.read() + newline_indices = find_newlines(contents) + + if os.path.isdir(data_dir): + # Only keep the parts of the filepath that are invariant to + # the dataset's location on disk + filename = os.path.basename(filename) + + # In python, arrays are much more space-efficient than lists + sentence_indices[filename] = array.array("I", newline_indices) + + # Save sentence indices so we don't have to do this again + with open(sentence_idx_file, "wb") as f: + pickle.dump(sentence_indices, f) + + corpus_size = 0 + empty_files = [] + + # Find total number of newlines across entire corpus and remove files + # without any newlines + for filename in sentence_indices: + if len(sentence_indices[filename]) <= 1: + empty_files.append(filename) + else: + corpus_size += len(sentence_indices[filename]) + + for filename in empty_files: + del sentence_indices[filename] + + self.corpus_size = corpus_size + self.dataset = data_dir + self.filenames = list(sentence_indices.keys()) + self.mask_probability = mask_prob + self.max_seq_length = max_seq_length + self.sentence_indices = sentence_indices + self.vocab_size = self.tokenizer.vocab_size + self.short_seq_prob = short_seq_prob + self.seq_a_ratio = seq_a_ratio + + def __len__(self): + return self.corpus_size + + def __getitem__(self, idx: int, min_doc_length: Optional[int] = 16): + # Each sequence has three special tokens, as follows: + # tokenizer.cls_token tokenizer.sep_token tokenizer.eos_token + num_special_tokens = 3 + + max_num_tokens = self.max_seq_length - num_special_tokens + target_seq_length = max_num_tokens + if random.random() < self.short_seq_prob: + # TODO: maybe introduce an argument to control this. + target_seq_length = random.randint(2, max_num_tokens) + + # prefer the seq_a to be slightly longer than seq_b, 0.6 by default + target_seq_length_a = int(round(target_seq_length * self.seq_a_ratio)) + target_seq_length_b = target_seq_length - target_seq_length_a + + def get_document(filepath, offset): + # Retrieve a specific line from a file and return as a document + if os.path.isdir(self.dataset): + filepath = os.path.join(self.dataset, filepath) + + with open(filepath, "rb") as f: + f.seek(offset) + doc_text = f.readline()[:-1].decode("utf-8", errors="ignore") + document = self.tokenizer.text_to_ids(doc_text) + + return document + + def match_target_seq_length( + document: str, target_seq_length: int, filename: str, line_idx: int, sentence_indices: Dict[str, dict] + ): + # If document is shorter than target sequence length, + # append the next line or take a random line as replacement. + num_lines = len(sentence_indices[filename]) + + while len(document) < target_seq_length: + if line_idx < (num_lines - 1): + # append the next line + line_idx += 1 + else: + # current line is the last line, take a random one + line_idx = random.randrange(num_lines) + document = [] + + offset = sentence_indices[filename][line_idx] + document += get_document(filename, offset) + + return document, line_idx + + # Take sequence A from a random file and a random line + a_filename = random.choice(self.filenames) + a_line_idx = random.randrange(len(self.sentence_indices[a_filename])) + a_line_offset = self.sentence_indices[a_filename][a_line_idx] + a_document = get_document(a_filename, a_line_offset) + a_document, a_line_idx = match_target_seq_length( + a_document, target_seq_length_a, a_filename, a_line_idx, self.sentence_indices + ) + + is_last_line = a_line_idx >= (len(self.sentence_indices[a_filename]) - 1) + # About 50% of the time, B is a random sentence from the corpus + take_random_b = (random.random() < 0.5) or is_last_line + + if take_random_b: + # This should rarely go for more than one iteration for large + # corpora. However, just to be careful, we try to make sure that + # the random document is not the same as the document + # we're processing. + for _ in range(10): + b_filename = random.choice(self.filenames) + b_line_idx = random.choice(range(len(self.sentence_indices[b_filename]))) + if b_filename != a_filename: + break + else: + # Take another line from the same file + b_line_pos = self.sentence_indices[b_filename][b_line_idx] + a_line_pos = self.sentence_indices[a_filename][a_line_idx] + # TODO unclear about the following check + if abs(b_line_pos - a_line_pos) > max_num_tokens: + break + else: + pass + else: + b_filename = a_filename + b_line_idx = a_line_idx + 1 + + is_next = int(not take_random_b) + b_line_pos = self.sentence_indices[b_filename][b_line_idx] + b_document = get_document(b_filename, b_line_pos) + b_document, b_line_idx = match_target_seq_length( + b_document, target_seq_length_b, b_filename, b_line_idx, self.sentence_indices + ) + + def truncate_seq_pair(a, b, max_num_tokens): + # Truncates a pair of sequences to a maximum sequence length + while (len(a) + len(b)) > max_num_tokens: + # Truncate the longer sequence + if len(a) > len(b): + trunc_document = a + else: + trunc_document = b + + if len(trunc_document) <= 1: + raise ValueError( + "Input text corpora probably too small. " + "Failed to truncate sequence pair to " + "maximum sequence legnth." + ) + + # Randomly truncate from the front or the back + if random.random() < 0.5: + del trunc_document[0] + else: + trunc_document.pop() + + truncate_seq_pair(a_document, b_document, max_num_tokens) + + output_ids = ( + [self.tokenizer.cls_id] + a_document + [self.tokenizer.sep_id] + b_document + [self.tokenizer.eos_id] + ) + + input_ids, output_mask = self.mask_ids(output_ids) + + input_mask = np.zeros(self.max_seq_length, dtype=np.long) + input_mask[: len(input_ids)] = 1 + + input_type_ids = np.zeros(self.max_seq_length, dtype=np.int) + input_type_ids[len(a_document) + 2 : len(output_ids) + 1] = 1 + + padding_length = max(0, self.max_seq_length - len(input_ids)) + if padding_length > 0: + input_ids.extend([self.tokenizer.pad_id] * padding_length) + output_ids.extend([self.tokenizer.pad_id] * padding_length) + output_mask.extend([0] * padding_length) + + # TODO: wrap the return value with () for consistent style. + return ( + np.array(input_ids), + input_type_ids, + np.array(input_mask, dtype=np.long), + np.array(output_ids), + np.array(output_mask, dtype=np.float32), + is_next, + ) + + def mask_ids(self, ids: List[int]): + """ + Args: + ids: list of token ids representing a chunk of text + Returns: + masked_ids: list of input tokens with some of the entries masked + according to the following protocol from the original BERT paper: + each token is masked with a probability of 15% and is replaced with + 1) the [MASK] token 80% of the time, + 2) random token 10% of the time, + 3) the same token 10% of the time. + output_mask: list of binary variables which indicate what tokens has + been masked (to calculate the loss function for these tokens only) + """ + + # Whole-word masking by default, as it gives better performance. + cand_indexes = [[ids[0]]] + for tid in ids[1:]: + token = self.tokenizer.ids_to_tokens([tid])[0] + is_suffix = token.startswith('\u2581') + if is_suffix: + # group together with its previous token to form a whole-word + cand_indexes[-1].append(tid) + else: + cand_indexes.append([tid]) + + masked_ids, output_mask = [], [] + mask_id = self.tokenizer.token_to_id("[MASK]") + + for word_ids in cand_indexes: + is_special = (word_ids[0] == self.tokenizer.cls_id) or (word_ids[0] == self.tokenizer.sep_id) + if is_special or (random.random() > self.mask_probability): + output_mask.extend([0] * len(word_ids)) + masked_ids.extend(word_ids) + else: + output_mask.extend([1] * len(word_ids)) + p = random.random() + # for 80%, replace with mask + if p < 0.8: + masked_ids.extend([mask_id] * len(word_ids)) + # for 10%, replace by a random token + elif p < 0.9: + for _ in word_ids: + # randomly select a valid word + random_word = random.randrange(self.vocab_size) + while random_word in (self.tokenizer.cls_id, self.tokenizer.sep_id): + random_word = random.randrange(self.vocab_size) + masked_ids.append(random_word) + # for 10%, use same token + else: + masked_ids.extend(word_ids) + + return masked_ids, output_mask + + +class BertPretrainingPreprocessedDataset(Dataset): + """ + Dataset for already preprocessed data. + """ + + def __init__(self, input_file: str, max_predictions_per_seq: int): + """ + Args: + input_file: data file in hdf5 format with preprocessed data in array format + max_predictions_per_seq: maximum number of masked tokens per sequence. Need to be consistent with data in input file. + """ + self.input_file = input_file + self.max_predictions_per_seq = max_predictions_per_seq + f = load_h5(input_file) + keys = [ + 'input_ids', + 'input_mask', + 'segment_ids', + 'masked_lm_positions', + 'masked_lm_ids', + 'next_sentence_labels', + ] + self.inputs = [np.asarray(f[key][:]) for key in keys] + f.close() + + def __len__(self): + 'Denotes the total number of samples' + return len(self.inputs[0]) + + def __getitem__(self, index: int): + [input_ids, input_mask, segment_ids, masked_lm_positions, masked_lm_ids, next_sentence_labels] = [ + input[index].astype(np.int64) for input in self.inputs + ] + + output_mask = np.zeros_like(input_ids) + output_ids = input_ids.copy() + + index = self.max_predictions_per_seq + padded_mask_indices = (masked_lm_positions == 0).nonzero() + if len(padded_mask_indices[0]) != 0: + index = padded_mask_indices[0][0] + + output_mask[masked_lm_positions[:index]] = 1.0 + output_ids[masked_lm_positions[:index]] = masked_lm_ids[:index] + + # input_mask = np.asarray(input_mask, dtype=np.float32) + # output_mask = np.asarray(output_mask, dtype=np.float32) + return (input_ids, segment_ids, input_mask, output_ids, output_mask, next_sentence_labels) + + +class BertPretrainingPreprocessedDataloader(DataLoader): + """ + Dataloader for already preprocessed data in hdf5 files that is already in the format expected by BERT model. + """ + + def __init__(self, data_files: List[str], max_predictions_per_seq: int, batch_size: int, seed: Optional[int] = 42): + """ + Args: + data_files: list of data files in hdf5 format with preprocessed data in array format + max_predictions_per_seq: maximum number of masked tokens per sequence. Need to be consistent with data in input file. + batch_size: batch size per gpu per forward pass + seed: seed to ensure each gpu process opens the same data file in each iteration + """ + super().__init__(None, batch_size=batch_size) + self.random = random.Random(seed) + self.data_files = data_files + self.max_predictions_per_seq = max_predictions_per_seq + + # def __len__(self): + # return sum([len(load_h5(data_file)['input_ids']) for data_file in self.data_files])//(self.batch_size) + + def __iter__(self): + self.random.shuffle(self.data_files) + for data_file in self.data_files: + train_data = BertPretrainingPreprocessedDataset( + input_file=data_file, max_predictions_per_seq=self.max_predictions_per_seq + ) + train_sampler = DistributedSampler(train_data) + # print("---") + # print(os.getpid(), train_sampler.rank, train_sampler.num_replicas, train_sampler.num_samples) + # print("---") + train_dataloader = DataLoader( + dataset=train_data, sampler=train_sampler, batch_size=self.batch_size, shuffle=False, + ) + for x in train_dataloader: + yield x diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/sentence_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/sentence_dataset.py new file mode 100644 index 0000000000..ebc6c823cb --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/language_modeling/sentence_dataset.py @@ -0,0 +1,273 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pytorch Dataset with sentences packed into batches by length.""" +import io +import json +import logging +import pickle +from collections import OrderedDict +from typing import Any + +import braceexpand +import numpy as np +import webdataset as wd +from torch.utils.data import IterableDataset + +from nemo.collections.nlp.data.data_utils.data_preprocessing import dataset_to_ids +from nemo.core import Dataset + +__all__ = ['SentenceDataset', 'TarredSentenceDataset'] + + +class SentenceDataset(Dataset): + def __init__( + self, + tokenizer: Any, + dataset: Any, + tokens_in_batch: int = 1024, + clean: bool = False, + cache_ids: bool = False, + max_seq_length: int = 512, + min_seq_length: int = 1, + ): + + self.tokenizer = tokenizer + self.tokens_in_batch = tokens_in_batch + + ids = dataset_to_ids(dataset, tokenizer, cache_ids=cache_ids) + if clean: + ids = self.clean(ids, max_tokens=max_seq_length, min_tokens=min_seq_length) + self.batch_sent_ids, self.batch_elem_lengths = self.pack_data_into_batches(ids) + self.batches = self.pad_batches(ids) + + def __len__(self): + return len(self.batches) + + def __getitem__(self, idx): + ids = self.batches[idx] + mask = (ids != self.tokenizer.pad_id).astype(np.int32) + return ids, mask + + def pad_batches(self, ids): + """ + Augments source and target ids in the batches with padding symbol + to make the lengths of all sentences in the batches equal. + """ + + batches = [] + for batch_elem_len, batch_sent_ids in zip(self.batch_elem_lengths, self.batch_sent_ids): + batch = self.tokenizer.pad_id * np.ones((len(batch_sent_ids), batch_elem_len), dtype=np.int) + for i, sentence_idx in enumerate(batch_sent_ids): + batch[i][: len(ids[sentence_idx])] = ids[sentence_idx] + batches.append(batch) + return batches + + def pack_data_into_batches(self, ids): + """ + Takes two lists of source and target sentences, sorts them, and packs + into batches to minimize the use of padding tokens. Returns a list of + batches where each batch contains indices of sentences included into it + """ + + # create buckets sorted by the number of src tokens + # each bucket is also sorted by the number of tgt tokens + buckets = {} + for i, line_ids in enumerate(ids): + len_ = len(line_ids) + if len_ not in buckets: + buckets[len_] = [i] + else: + buckets[len_].append(i) + + for b_idx in buckets: + buckets[b_idx] = sorted(buckets[b_idx]) + + buckets = OrderedDict(sorted(buckets.items())) + + batches = [] + batch_elem_lengths = [] + curr_batch = [] + len_of_longest_sent = 0 + for sent_len, bucket in buckets.items(): + for sent_i in bucket: + if sent_len * (len(curr_batch) + 1) > self.tokens_in_batch: + if not curr_batch: + raise ValueError( + f"The limitation on number of tokens in batch {self.tokens_in_batch} is too strong." + f"Several sentences contain {sent_len} tokens." + ) + batches.append(curr_batch) + batch_elem_lengths.append(sent_len) + curr_batch = [] + curr_batch.append(sent_i) + len_of_longest_sent = sent_len + if curr_batch: + batches.append(curr_batch) + batch_elem_lengths.append(len_of_longest_sent) + return batches, batch_elem_lengths + + def clean(self, ids, max_tokens=None, min_tokens=None): + """ + Cleans source and target sentences to get rid of noisy data. + Specifically, a pair of sentences is removed if + -- either source or target is longer than *max_tokens* + -- either source or target is shorter than *min_tokens* + -- absolute difference between source and target is larger than + *max_tokens_diff* + -- one sentence is *max_tokens_ratio* times longer than the other + """ + + ids_ = [] + for i in range(len(ids)): + len_ = len(ids[i]) + if (max_tokens is not None and len_ > max_tokens) or (min_tokens is not None and len_ < min_tokens): + continue + ids_.append(ids[i]) + return ids_ + + +class TarredSentenceDataset(IterableDataset): + """ + A similar Dataset to the SentenceDataset, but which loads tarred tokenized pickle files. + Accepts a single JSON metadata file containing the total number of batches + as well as the path(s) to the tarball(s) containing the wav files. + Valid formats for the text_tar_filepaths argument include: + (1) a single string that can be brace-expanded, e.g. 'path/to/text.tar' or 'path/to/text_{1..100}.tar.gz', or + (2) a list of file paths that will not be brace-expanded, e.g. ['text_1.tar', 'text_2.tar', ...]. + Note: For brace expansion in (1), there may be cases where `{x..y}` syntax cannot be used due to shell interference. + This occurs most commonly inside SLURM scripts. Therefore we provide a few equivalent replacements. + Supported opening braces - { <=> (, [, < and the special tag _OP_. + Supported closing braces - } <=> ), ], > and the special tag _CL_. + For SLURM based tasks, we suggest the use of the special tags for ease of use. + See the WebDataset documentation for more information about accepted data and input formats. + If using multiple processes the number of shards should be divisible by the number of workers to ensure an + even split among workers. If it is not divisible, logging will give a warning but training will proceed. + Additionally, please note that the len() of this DataLayer is assumed to be the number of tokens + of the text data. An incorrect manifest length may lead to some DataLoader issues down the line. + Args: + text_tar_filepaths: Either a list of tokenized text tarball filepaths, or a + string (can be brace-expandable). + metadata_path (str): Path to the metadata manifest. + encoder_tokenizer: Autokenizer wrapped BPE tokenizer model, such as YTTM + decoder_tokenizer: Autokenizer wrapped BPE tokenizer model, such as YTTM + shuffle_n (int): How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + Defaults to 0. + shard_strategy (str): Tarred dataset shard distribution strategy chosen as a str value during ddp. + - `scatter`: The default shard strategy applied by WebDataset, where each node gets + a unique set of shards, which are permanently pre-allocated and never changed at runtime. + - `replicate`: Optional shard strategy, where each node gets all of the set of shards + available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. + The benefit of replication is that it allows each node to sample data points from the entire + dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. + Note: Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. + global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. + world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. + reverse_lang_direction (bool): When True, swaps the source and target directions when returning minibatches. + """ + + def __init__( + self, + text_tar_filepaths: str, + metadata_path: str, + tokenizer: str, + shuffle_n: int = 1, + shard_strategy: str = "scatter", + global_rank: int = 0, + world_size: int = 0, + ): + super(TarredSentenceDataset, self).__init__() + + self.tokenizer = tokenizer + self.pad_id = tokenizer.pad_id + + valid_shard_strategies = ['scatter', 'replicate'] + if shard_strategy not in valid_shard_strategies: + raise ValueError(f"`shard_strategy` must be one of {valid_shard_strategies}") + + with open(metadata_path, 'r') as f: + metadata = json.load(f) + + self.metadata = metadata + + if isinstance(text_tar_filepaths, str): + # Replace '(', '[', '<' and '_OP_' with '{' + brace_keys_open = ['(', '[', '<', '_OP_'] + for bkey in brace_keys_open: + if bkey in text_tar_filepaths: + text_tar_filepaths = text_tar_filepaths.replace(bkey, "{") + + # Replace ')', ']', '>' and '_CL_' with '}' + brace_keys_close = [')', ']', '>', '_CL_'] + for bkey in brace_keys_close: + if bkey in text_tar_filepaths: + text_tar_filepaths = text_tar_filepaths.replace(bkey, "}") + + if isinstance(text_tar_filepaths, str): + # Brace expand + text_tar_filepaths = list(braceexpand.braceexpand(text_tar_filepaths)) + + if shard_strategy == 'scatter': + logging.info("All tarred dataset shards will be scattered evenly across all nodes.") + if len(text_tar_filepaths) % world_size != 0: + logging.warning( + f"Number of shards in tarred dataset ({len(text_tar_filepaths)}) is not divisible " + f"by number of distributed workers ({world_size})." + ) + begin_idx = (len(text_tar_filepaths) // world_size) * global_rank + end_idx = begin_idx + (len(text_tar_filepaths) // world_size) + logging.info('Begin Index : %d' % (begin_idx)) + logging.info('End Index : %d' % (end_idx)) + text_tar_filepaths = text_tar_filepaths[begin_idx:end_idx] + logging.info( + "Partitioning tarred dataset: process (%d) taking shards [%d, %d)", global_rank, begin_idx, end_idx + ) + + elif shard_strategy == 'replicate': + logging.info("All tarred dataset shards will be replicated across all nodes.") + + else: + raise ValueError(f"Invalid shard strategy ! Allowed values are : {valid_shard_strategies}") + + self.tarpath = text_tar_filepaths + + # Put together WebDataset + self._dataset = wd.WebDataset(urls=text_tar_filepaths, nodesplitter=None) + + if shuffle_n > 0: + self._dataset = self._dataset.shuffle(shuffle_n) + else: + logging.info("WebDataset will not shuffle files within the tar files.") + + self._dataset = self._dataset.rename(pkl='pkl', key='__key__').to_tuple('pkl', 'key').map(f=self._build_sample) + + def _build_sample(self, fname): + # Load file + pkl_file, _ = fname + pkl_file = io.BytesIO(pkl_file) + data = pickle.load(pkl_file) # loads np.int64 vector + pkl_file.close() + ids = data["src"] + mask = (ids != self.pad_id).astype(np.int32) + return ids, mask + + def __iter__(self): + return self._dataset.__iter__() + + def __len__(self): + return self.metadata['num_batches'] diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/__init__.py new file mode 100644 index 0000000000..0cd89ca5ac --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.machine_translation.machine_translation_dataset import ( + TarredTranslationDataset, + TranslationDataset, +) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/machine_translation_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/machine_translation_dataset.py new file mode 100644 index 0000000000..00147341af --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/machine_translation_dataset.py @@ -0,0 +1,445 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pytorch Dataset for training Neural Machine Translation.""" + +import io +import json +import pickle +from collections import OrderedDict +from dataclasses import dataclass +from typing import Any, List, Optional + +import braceexpand +import numpy as np +import webdataset as wd +from torch.utils.data import IterableDataset + +from nemo.collections.nlp.data.data_utils.data_preprocessing import dataset_to_ids +from nemo.core import Dataset +from nemo.utils import logging + +__all__ = ['TranslationDataset', 'TarredTranslationDataset'] + + +@dataclass +class TranslationDataConfig: + src_file_name: Optional[Any] = None # Any = str or List[str] + tgt_file_name: Optional[Any] = None # Any = str or List[str] + use_tarred_dataset: bool = False + tar_files: Optional[Any] = None # Any = str or List[str] + metadata_file: Optional[Any] = None # Any = str or List[str] + lines_per_dataset_fragment: Optional[int] = 1000000 + num_batches_per_tarfile: Optional[int] = 1000 + shard_strategy: Optional[str] = 'scatter' + tokens_in_batch: int = 512 + clean: bool = False + max_seq_length: int = 512 + min_seq_length: int = 1 + cache_ids: bool = False + cache_data_per_node: bool = False + use_cache: bool = False + shuffle: bool = False + num_samples: int = -1 + drop_last: bool = False + pin_memory: bool = False + num_workers: int = 8 + reverse_lang_direction: bool = False + load_from_tarred_dataset: bool = False + metadata_path: Optional[str] = None + tar_shuffle_n: int = 100 + n_preproc_jobs: int = -2 + tar_file_prefix: str = 'parallel' + concat_sampling_technique: Optional[str] = 'temperature' + concat_sampling_temperature: Optional[int] = 5 + concat_sampling_probabilities: Optional[List[float]] = None + + +class TranslationDataset(Dataset): + def __init__( + self, + dataset_src: str, + dataset_tgt: str, + tokens_in_batch: int = 1024, + clean: bool = False, + max_seq_length: int = 512, + min_seq_length: int = 1, + max_seq_length_diff: int = 512, + max_seq_length_ratio: int = 512, + cache_ids: bool = False, + cache_data_per_node: bool = False, + use_cache: bool = False, + reverse_lang_direction: bool = False, + prepend_id: int = None, + ): + self.dataset_src = dataset_src + self.dataset_tgt = dataset_tgt + self.tokens_in_batch = tokens_in_batch + self.cache_ids = cache_ids + self.use_cache = use_cache + self.clean = clean + self.cache_data_per_node = cache_data_per_node + self.max_seq_length = max_seq_length + self.min_seq_length = min_seq_length + self.max_seq_length_diff = max_seq_length_diff + self.max_seq_length_ratio = max_seq_length_ratio + self.reverse_lang_direction = reverse_lang_direction + self.prepend_id = prepend_id + + # deprecation warnings for cache_ids, use_cache, and cache_data_per_node + if self.cache_ids is True or self.use_cache is True or self.cache_data_per_node is True: + logging.warning( + 'Deprecation warning. self.cache_ids, self.use_cache, and self.cache_data_per_node will be removed. Data caching to be done with tarred datasets moving forward.' + ) + + def batchify(self, tokenizer_src, tokenizer_tgt): + src_ids = dataset_to_ids( + self.dataset_src, + tokenizer_src, + cache_ids=self.cache_ids, + cache_data_per_node=self.cache_data_per_node, + use_cache=self.use_cache, + ) + tgt_ids = dataset_to_ids( + self.dataset_tgt, + tokenizer_tgt, + cache_ids=self.cache_ids, + cache_data_per_node=self.cache_data_per_node, + use_cache=self.use_cache, + ) + if self.clean: + src_ids, tgt_ids = self.clean_src_and_target( + src_ids, + tgt_ids, + max_tokens=self.max_seq_length, + min_tokens=self.min_seq_length, + max_tokens_diff=self.max_seq_length_diff, + max_tokens_ratio=self.max_seq_length_ratio, + ) + self.src_pad_id = tokenizer_src.pad_id + self.tgt_pad_id = tokenizer_tgt.pad_id + + self.batch_indices = self.pack_data_into_batches(src_ids, tgt_ids) + self.batches = self.pad_batches(src_ids, tgt_ids, self.batch_indices) + + def __len__(self): + return len(self.batches) + + def __getitem__(self, idx): + src_ids = self.batches[idx]["src"] + tgt = self.batches[idx]["tgt"] + if self.reverse_lang_direction: + src_ids, tgt = tgt, src_ids + labels = tgt[:, 1:] + tgt_ids = tgt[:, :-1] + if self.prepend_id: + src_ids = np.insert(src_ids, 0, self.prepend_id, axis=-1) + src_mask = (src_ids != self.src_pad_id).astype(np.int32) + tgt_mask = (tgt_ids != self.tgt_pad_id).astype(np.int32) + return src_ids, src_mask, tgt_ids, tgt_mask, labels + + def pad_batches(self, src_ids, tgt_ids, batch_indices): + """ + Augments source and target ids in the batches with padding symbol + to make the lengths of all sentences in the batches equal. + """ + + batches = {} + for batch_idx, b in enumerate(batch_indices): + src_len = max([len(src_ids[i]) for i in b]) + tgt_len = max([len(tgt_ids[i]) for i in b]) + src_ids_ = self.src_pad_id * np.ones((len(b), src_len), dtype=np.int) + tgt_ids_ = self.tgt_pad_id * np.ones((len(b), tgt_len), dtype=np.int) + for i, sentence_idx in enumerate(b): + src_ids_[i][: len(src_ids[sentence_idx])] = src_ids[sentence_idx] + tgt_ids_[i][: len(tgt_ids[sentence_idx])] = tgt_ids[sentence_idx] + batches[batch_idx] = {"src": src_ids_, "tgt": tgt_ids_} + return batches + + def pack_data_into_batches(self, src_ids, tgt_ids): + """ + Takes two lists of source and target sentences, sorts them, and packs + into batches to minimize the use of padding tokens. Returns a list of + batches where each batch contains indices of sentences included into it + """ + + # create buckets sorted by the number of src tokens + # each bucket is also sorted by the number of tgt tokens + buckets = {} + for i, src_id in enumerate(src_ids): + src_len, tgt_len = len(src_id), len(tgt_ids[i]) + if src_len not in buckets: + buckets[src_len] = [(tgt_len, i)] + else: + buckets[src_len].append((tgt_len, i)) + + for b_idx in buckets: + buckets[b_idx] = sorted(buckets[b_idx]) + + buckets = OrderedDict(sorted(buckets.items())) + indices = list(buckets.keys()) + + batches = [[]] + num_batches = 0 + batch_size = 0 + i = 0 + src_len = 0 + tgt_len = 0 + + while i < len(buckets): + while buckets[indices[i]]: + + i_src = max(src_len, indices[i]) + i_tgt = max(tgt_len, buckets[indices[i]][0][0]) + + try: + ip1_src = max(src_len, indices[i + 1]) + ip1_tgt = max(tgt_len, buckets[indices[i + 1]][0][0]) + except IndexError: + ip1_src = i_src + 1 + ip1_tgt = i_tgt + 1 + + if i_src + i_tgt <= ip1_src + ip1_tgt: + src_len = i_src + tgt_len = i_tgt + _, idx = buckets[indices[i]].pop(0) + else: + src_len = ip1_src + tgt_len = ip1_tgt + _, idx = buckets[indices[i + 1]].pop(0) + + batches[num_batches].append(idx) + batch_size += 1 + + if batch_size * (src_len + tgt_len) > self.tokens_in_batch: + + num_examples_to_split = len(batches[num_batches]) + batches_to_evict = 8 * ((num_examples_to_split - 1) // 8) + + if batches_to_evict == 0: + batches_to_evict = num_examples_to_split + + batches.append(batches[num_batches][batches_to_evict:]) + batches[num_batches] = batches[num_batches][:batches_to_evict] + batch_size = num_examples_to_split - batches_to_evict + + num_batches += 1 + if batch_size > 0: + src_len = max([len(src_ids[j]) for j in batches[num_batches]]) + tgt_len = max([len(tgt_ids[j]) for j in batches[num_batches]]) + else: + src_len = 0 + tgt_len = 0 + break + + if not buckets[indices[i]]: + i = i + 1 + + if not batches[-1]: + batches.pop(-1) + + return batches + + def clean_src_and_target( + self, + src_ids, + tgt_ids, + max_tokens=None, + min_tokens=None, + max_tokens_diff=None, + max_tokens_ratio=None, + filter_equal_src_and_dest=False, + ): + """ + Cleans source and target sentences to get rid of noisy data. + Specifically, a pair of sentences is removed if + -- either source or target is longer than *max_tokens* + -- either source or target is shorter than *min_tokens* + -- absolute difference between source and target is larger than + *max_tokens_diff* + -- one sentence is *max_tokens_ratio* times longer than the other + """ + + if len(src_ids) != len(tgt_ids): + raise ValueError("Source and target corpora have different lengths!") + src_ids_, tgt_ids_ = [], [] + for i in range(len(src_ids)): + src_len, tgt_len = len(src_ids[i]), len(tgt_ids[i]) + if ( + (max_tokens is not None and (src_len > max_tokens or tgt_len > max_tokens)) + or (min_tokens is not None and (src_len < min_tokens or tgt_len < min_tokens)) + or (filter_equal_src_and_dest and src_ids[i] == tgt_ids[i]) + or (max_tokens_diff is not None and np.abs(src_len - tgt_len) > max_tokens_diff) + ): + continue + if max_tokens_ratio is not None: + ratio = max(src_len - 2, 1) / max(tgt_len - 2, 1) + if ratio > max_tokens_ratio or ratio < (1 / max_tokens_ratio): + continue + src_ids_.append(src_ids[i]) + tgt_ids_.append(tgt_ids[i]) + return src_ids_, tgt_ids_ + + +class TarredTranslationDataset(IterableDataset): + """ + A similar Dataset to the TranslationDataset, but which loads tarred tokenized pickle files. + Accepts a single JSON metadata file containing the total number of batches + as well as the path(s) to the tarball(s) containing the pickled parallel dataset batch files. + Valid formats for the text_tar_filepaths argument include: + (1) a single string that can be brace-expanded, e.g. 'path/to/text.tar' or 'path/to/text_{1..100}.tar', or + (2) a list of file paths that will not be brace-expanded, e.g. ['text_1.tar', 'text_2.tar', ...]. + Note: For brace expansion in (1), there may be cases where `{x..y}` syntax cannot be used due to shell interference. + This occurs most commonly inside SLURM scripts. Therefore we provide a few equivalent replacements. + Supported opening braces - { <=> (, [, < and the special tag _OP_. + Supported closing braces - } <=> ), ], > and the special tag _CL_. + For SLURM based tasks, we suggest the use of the special tags for ease of use. + See the WebDataset documentation for more information about accepted data and input formats. + If using multiple processes the number of shards should be divisible by the number of workers to ensure an + even split among workers. If it is not divisible, logging will give a warning but training will proceed. + Additionally, please note that the len() of this DataLayer is assumed to be the number of tokens + of the text data. An incorrect manifest length may lead to some DataLoader issues down the line. + Args: + text_tar_filepaths: Either a list of tokenized text tarball filepaths, or a + string (can be brace-expandable). + metadata_path (str): Path to the metadata manifest. + encoder_tokenizer: Autokenizer wrapped BPE tokenizer model, such as YTTM + decoder_tokenizer: Autokenizer wrapped BPE tokenizer model, such as YTTM + shuffle_n (int): How many samples to look ahead and load to be shuffled. + See WebDataset documentation for more details. + Defaults to 0. + shard_strategy (str): Tarred dataset shard distribution strategy chosen as a str value during ddp. + - `scatter`: The default shard strategy applied by WebDataset, where each node gets + a unique set of shards, which are permanently pre-allocated and never changed at runtime. + - `replicate`: Optional shard strategy, where each node gets all of the set of shards + available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. + The benefit of replication is that it allows each node to sample data points from the entire + dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. + Note: Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. + global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. + world_size (int): Total number of processes, used for partitioning shards. Defaults to 1. + reverse_lang_direction (bool): When True, swaps the source and target directions when returning minibatches. + prepend_id (int): Prepends the specificed token id to the start of every source sentence. Defaults to None. + """ + + def __init__( + self, + text_tar_filepaths: str, + metadata_path: str, + encoder_tokenizer: str, + decoder_tokenizer: str, + shuffle_n: int = 1, + shard_strategy: str = "scatter", + global_rank: int = 0, + world_size: int = 1, + reverse_lang_direction: bool = False, + prepend_id: int = None, + ): + super(TarredTranslationDataset, self).__init__() + + self.encoder_tokenizer = encoder_tokenizer + self.decoder_tokenizer = decoder_tokenizer + self.reverse_lang_direction = reverse_lang_direction + self.src_pad_id = encoder_tokenizer.pad_id + self.tgt_pad_id = decoder_tokenizer.pad_id + self.prepend_id = prepend_id + + valid_shard_strategies = ['scatter', 'replicate'] + if shard_strategy not in valid_shard_strategies: + raise ValueError(f"`shard_strategy` must be one of {valid_shard_strategies}") + + with open(metadata_path, 'r') as f: + metadata = json.load(f) + + self.metadata = metadata + + if isinstance(text_tar_filepaths, str): + # Replace '(', '[', '<' and '_OP_' with '{' + brace_keys_open = ['(', '[', '<', '_OP_'] + for bkey in brace_keys_open: + if bkey in text_tar_filepaths: + text_tar_filepaths = text_tar_filepaths.replace(bkey, "{") + + # Replace ')', ']', '>' and '_CL_' with '}' + brace_keys_close = [')', ']', '>', '_CL_'] + for bkey in brace_keys_close: + if bkey in text_tar_filepaths: + text_tar_filepaths = text_tar_filepaths.replace(bkey, "}") + + if isinstance(text_tar_filepaths, str): + # Brace expand + text_tar_filepaths = list(braceexpand.braceexpand(text_tar_filepaths)) + + if shard_strategy == 'scatter': + logging.info("All tarred dataset shards will be scattered evenly across all nodes.") + if len(text_tar_filepaths) % world_size != 0: + logging.warning( + f"Number of shards in tarred dataset ({len(text_tar_filepaths)}) is not divisible " + f"by number of distributed workers ({world_size})." + ) + begin_idx = (len(text_tar_filepaths) // world_size) * global_rank + end_idx = begin_idx + (len(text_tar_filepaths) // world_size) + logging.info('Begin Index : %d' % (begin_idx)) + logging.info('End Index : %d' % (end_idx)) + text_tar_filepaths = text_tar_filepaths[begin_idx:end_idx] + logging.info( + "Partitioning tarred dataset: process (%d) taking shards [%d, %d)", global_rank, begin_idx, end_idx + ) + self.length = self.metadata['num_batches'] // world_size + + elif shard_strategy == 'replicate': + logging.info("All tarred dataset shards will be replicated across all nodes.") + self.length = self.metadata['num_batches'] + + else: + raise ValueError(f"Invalid shard strategy ! Allowed values are : {valid_shard_strategies}") + + self.tarpath = text_tar_filepaths + + # Put together WebDataset + self._dataset = wd.WebDataset(urls=text_tar_filepaths, nodesplitter=None) + + if shuffle_n > 0: + self._dataset = self._dataset.shuffle(shuffle_n) + else: + logging.info("WebDataset will not shuffle files within the tar files.") + + self._dataset = self._dataset.rename(pkl='pkl', key='__key__').to_tuple('pkl', 'key').map(f=self._build_sample) + + def _build_sample(self, fname): + # Load file + pkl_file, _ = fname + pkl_file = io.BytesIO(pkl_file) + data = pickle.load(pkl_file) # loads np.int64 vector + pkl_file.close() + src_ids = data["src"] + tgt = data["tgt"] + if self.reverse_lang_direction: + src_ids, tgt = tgt, src_ids + labels = tgt[:, 1:] + tgt_ids = tgt[:, :-1] + if self.prepend_id: + src_ids = np.insert(src_ids, 0, self.prepend_id, axis=-1) + src_mask = (src_ids != self.src_pad_id).astype(np.int32) + tgt_mask = (tgt_ids != self.tgt_pad_id).astype(np.int32) + return src_ids, src_mask, tgt_ids, tgt_mask, labels + + def __iter__(self): + return self._dataset.__iter__() + + def __len__(self): + return self.length diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/preproc_mt_data.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/preproc_mt_data.py new file mode 100644 index 0000000000..2ce1b0bc51 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/machine_translation/preproc_mt_data.py @@ -0,0 +1,1020 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import glob +import json +import os +import pickle +import tarfile +import tempfile + +import youtokentome as yttm +from joblib import Parallel, delayed +from omegaconf import ListConfig, OmegaConf +from pytorch_lightning import Trainer + +from nemo.collections.common.tokenizers.sentencepiece_tokenizer import create_spt_model +from nemo.collections.nlp.data.language_modeling.sentence_dataset import SentenceDataset +from nemo.collections.nlp.data.machine_translation.machine_translation_dataset import TranslationDataset +from nemo.collections.nlp.models.machine_translation.mt_enc_dec_config import MTEncDecModelConfig +from nemo.collections.nlp.modules.common.tokenizer_utils import get_nmt_tokenizer, get_tokenizer +from nemo.utils import logging + + +class MTDataPreproc: + """ Automatically trains tokenizers and preprocesses machine translation data based on the MTEncDecModelConfig. + For training NMT models with datasets larger than 5M sentence pairs, + it can be inefficient to train them without first creating a tarred dataset. + If the user wants to change the tokenizer, vocab size, or batch size, for example, + they must reprocess the data with the correct configuration. + With MTDataPreproc users can sweep through data configurations and the tarred dataset will + be automatically created according to the model configuration. + To train tokenizer model and create tarred dataset specify in configuration: + model.preproc_out_dir=/path/to/preproc_out + model.encoder_tokenizer.vocab_size=32000 + model.decoder_tokenizer.vocab_size=32000 + model.train_ds.use_tarred_dataset=True + model.train_ds.src_file_name=/path/to/src.txt + model.train_ds.tgt_file_name=/path/to/tgt.txt + model.train_ds.tokens_in_batch=16000 + Once a dataset has been constructed based on this configuration, MTDataPreproc will not process it again. + If a previously trained tokenizer model or tarred dataset is found, MTDataPreproc will not preprocess the data. + + Note: the only tokenizer currently supported is YouTokenToMe. + """ + + def __init__(self, cfg: MTEncDecModelConfig, trainer: Trainer = None) -> None: + self._cfg = cfg + self.global_rank = 0 + self.world_size = 1 + if trainer is not None: + self.global_rank = (trainer.node_rank * trainer.num_gpus) + trainer.local_rank + self.world_size = trainer.num_nodes * trainer.num_gpus + + if hasattr(cfg, 'train_ds'): + supported_tokenizers = ['yttm', 'huggingface', 'sentencepiece'] + supported_train_tokenizers = ['yttm', 'sentencepiece'] + + if ( + cfg.encoder_tokenizer.get('library') not in supported_tokenizers + or cfg.decoder_tokenizer.get('library') not in supported_tokenizers + ): + raise NotImplementedError(f"Currently we only support {supported_tokenizers}.") + + if cfg.get('shared_tokenizer') and cfg.encoder_tokenizer.get('library') != cfg.decoder_tokenizer.get( + 'library' + ): + raise ValueError("Shared tokenizers cannot be from different libraries.") + + # Prepare tokenizers + if ( + cfg.encoder_tokenizer.get('library') in supported_train_tokenizers + or cfg.decoder_tokenizer.get('library') in supported_train_tokenizers + ): + + # Train tokenizer models if using yttm or sentencepiece and they don't exist + if ( + cfg.encoder_tokenizer.get('library') in supported_train_tokenizers + and cfg.encoder_tokenizer.get('tokenizer_model') is None + ) or ( + cfg.decoder_tokenizer.get('library') in supported_train_tokenizers + and cfg.decoder_tokenizer.get('tokenizer_model') is None + ): + if cfg.get('preproc_out_dir') is None: + raise ValueError('Tokenizer model training required but cfg.preproc_out_dir is None.') + if cfg.train_ds.get('src_file_name') is None or cfg.train_ds.get('tgt_file_name') is None: + raise ValueError( + 'src_file_name and tgt_file_name needed to train tokenizers but could not be found.' + ) + + src_fname = cfg.train_ds.get('src_file_name') + tgt_fname = cfg.train_ds.get('tgt_file_name') + src_language = cfg.get('src_language') + tgt_language = cfg.get('tgt_language') + spt_symbols = None + tempdir = tempfile.TemporaryDirectory() + + if cfg.get('multilingual'): + spt_symbols = [] + if isinstance(src_fname, ListConfig): + fnames = (" ").join(src_fname) + src_fname = os.path.join(tempdir.name, 'src.txt') + os.system('cat %s > %s' % (fnames, src_fname)) + + if isinstance(tgt_fname, ListConfig): + fnames = (" ").join(tgt_fname) + tgt_fname = os.path.join(tempdir.name, 'tgt.txt') + os.system('cat %s > %s' % (fnames, tgt_fname)) + + if isinstance(src_language, ListConfig): + for lng in src_language: + spt_symbols.append("<" + lng + ">") + + if isinstance(tgt_language, ListConfig): + for lng in tgt_language: + spt_symbols.append("<" + lng + ">") + + # train tokenizer model on training data + self.encoder_tokenizer_model, self.decoder_tokenizer_model = MTDataPreproc.train_tokenizers( + out_dir=cfg.get('preproc_out_dir'), + src_fname=src_fname, + tgt_fname=tgt_fname, + shared_tokenizer=cfg.get('shared_tokenizer'), + encoder_tokenizer_vocab_size=cfg.encoder_tokenizer.get('vocab_size'), + decoder_tokenizer_vocab_size=cfg.decoder_tokenizer.get('vocab_size'), + encoder_tokenizer_name=cfg.encoder_tokenizer.get('library'), + decoder_tokenizer_name=cfg.decoder_tokenizer.get('library'), + encoder_tokenizer_coverage=cfg.encoder_tokenizer.get('coverage', 0.999), + decoder_tokenizer_coverage=cfg.decoder_tokenizer.get('coverage', 0.999), + global_rank=self.global_rank, + encoder_training_sample_size=cfg.encoder_tokenizer.get('training_sample_size', -1), + decoder_training_sample_size=cfg.decoder_tokenizer.get('training_sample_size', -1), + encoder_special_tokens=OmegaConf.to_container(cfg.encoder_tokenizer.special_tokens) + if cfg.encoder_tokenizer.special_tokens + else None, + decoder_special_tokens=OmegaConf.to_container(cfg.decoder_tokenizer.special_tokens) + if cfg.decoder_tokenizer.special_tokens + else None, + spt_symbols=spt_symbols, + multilingual=cfg.get('multilingual', False), + ) + # update config + self._cfg.encoder_tokenizer.tokenizer_model = self.encoder_tokenizer_model + self._cfg.decoder_tokenizer.tokenizer_model = self.decoder_tokenizer_model + + tempdir.cleanup() + else: + self.encoder_tokenizer_model = cfg.encoder_tokenizer.get('tokenizer_model') + self.decoder_tokenizer_model = cfg.decoder_tokenizer.get('tokenizer_model') + + self.encoder_tokenizer, self.decoder_tokenizer = self.get_enc_dec_tokenizers( + encoder_tokenizer_name=cfg.encoder_tokenizer.get('library'), + encoder_model_name=cfg.encoder.get('model_name'), + encoder_tokenizer_model=self.encoder_tokenizer_model, + encoder_bpe_dropout=cfg.encoder_tokenizer.get('bpe_dropout', 0.0), + decoder_tokenizer_name=cfg.decoder_tokenizer.get('library'), + decoder_model_name=cfg.decoder.get('model_name'), + decoder_tokenizer_model=self.decoder_tokenizer_model, + decoder_bpe_dropout=cfg.decoder_tokenizer.get('bpe_dropout', 0.0), + ) + + # If using tarred dataset for training, automatically create it if needed + if cfg.train_ds.get('use_tarred_dataset'): + if cfg.train_ds.get('tar_files') is None and cfg.train_ds.get('metadata_file') is None: + if cfg.get('preproc_out_dir') is None: + raise ValueError('Data preprocessing required but cfg.preproc_out_dir is None.') + if cfg.train_ds.get('src_file_name') is None or cfg.train_ds.get('tgt_file_name') is None: + raise ValueError( + 'src_file_name and tgt_file_name needed to create tarred dataset but could not be found.' + ) + # Preprocess data and cache for use during training + if self.global_rank == 0: + logging.info( + f"Using tarred dataset for src: {cfg.train_ds.get('src_file_name')} and tgt: {cfg.train_ds.get('tgt_file_name')}" + ) + + if not cfg.get('multilingual'): + src_file_list = [cfg.train_ds.get('src_file_name')] + tgt_file_list = [cfg.train_ds.get('tgt_file_name')] + outdir_list = [cfg.get('preproc_out_dir')] + else: + src_file_list = cfg.train_ds.get('src_file_name') + tgt_file_list = cfg.train_ds.get('tgt_file_name') + if isinstance(cfg.get('src_language'), ListConfig): + langs = cfg.get('src_language') + elif isinstance(cfg.get('tgt_language'), ListConfig): + langs = cfg.get('tgt_language') + else: + raise ValueError( + "Expect either cfg.src_language or cfg.tgt_language to be a list when multilingual=True." + ) + outdir_list = [] + for lang in langs: + outdir_list.append(os.path.join(cfg.get('preproc_out_dir'), lang)) + + if len(src_file_list) != len(tgt_file_list) or len(src_file_list) != len(outdir_list): + raise ValueError( + "Number of source files, target files, and multilingual language pairs must be the same." + ) + + # TODO: have to get tokenizers instide .preprocess_parallel because they can't be pickled + metadata_file_list = [] + for idx, src_file in enumerate(src_file_list): + self.train_tar_files, self.train_metadata_file = MTDataPreproc.preprocess_parallel_dataset( + clean=cfg.train_ds.clean, + src_fname=src_file, + tgt_fname=tgt_file_list[idx], + out_dir=outdir_list[idx], + encoder_tokenizer_name=cfg.encoder_tokenizer.get('library'), + encoder_model_name=cfg.encoder.get('model_name'), + encoder_tokenizer_model=self.encoder_tokenizer_model, + encoder_bpe_dropout=cfg.encoder_tokenizer.get('bpe_dropout', 0.0), + decoder_tokenizer_name=cfg.decoder_tokenizer.get('library'), + decoder_model_name=cfg.decoder.get('model_name'), + decoder_tokenizer_model=self.decoder_tokenizer_model, + decoder_bpe_dropout=cfg.decoder_tokenizer.get('bpe_dropout', 0.0), + max_seq_length=cfg.train_ds.get('max_seq_length', 512), + tokens_in_batch=cfg.train_ds.get('tokens_in_batch', 8192), + lines_per_dataset_fragment=cfg.train_ds.get('lines_per_dataset_fragment', 1000000), + num_batches_per_tarfile=cfg.train_ds.get('num_batches_per_tarfile', 1000), + min_seq_length=1, + global_rank=self.global_rank, + world_size=self.world_size, + n_jobs=cfg.train_ds.get('n_preproc_jobs', -2), + tar_file_prefix=cfg.train_ds.get('tar_file_prefix', 'parallel'), + ) + metadata_file_list.append(self.train_metadata_file) + # update config + # self._cfg.train_ds.tar_files = self.tar_files_to_string(self.train_tar_files) + # self._cfg.train_ds.tar_files = self.train_tar_files + if not cfg.get('multilingual'): + self._cfg.train_ds.metadata_file = metadata_file_list[0] + else: + self._cfg.train_ds.metadata_file = metadata_file_list + + logging.info( + f"Using tarred dataset created in folder(s) {outdir_list} and metadata created at {self._cfg.train_ds.metadata_file}" + ) + elif cfg.train_ds.get('tar_files') is not None and cfg.train_ds.get('metadata_file') is None: + raise ValueError('A metadata file is required for tarred dataset but cfg.metadata_file is None.') + elif cfg.train_ds.get('tar_files') is None and cfg.train_ds.get('metadata_file') is not None: + if isinstance(cfg.train_ds.get('metadata_file'), str): + metadata_file_list = [cfg.train_ds.get('metadata_file')] + else: + metadata_file_list = cfg.train_ds.get('metadata_file') + + for metadata_file in metadata_file_list: + with open(metadata_file) as metadata_reader: + metadata = json.load(metadata_reader) + if metadata['tar_files']: + logging.info(f"Using tarred dataset: {metadata['tar_files']}") + else: + raise ValueError(f'tar_files not provided and metadata does not have tar files') + else: + self.train_tar_files = cfg.train_ds.get('tar_files') + self.train_metadata_file = cfg.train_ds.get('metadata_file') + logging.info( + f"Using tarred dataset from config at {self.train_tar_files} and metadata from {self.train_metadata_file}" + ) + + def tar_files_to_string(self, tar_files): + """ Tar files are generated in the following format: basename.number.tar + Where number is an integer from 1 to the number of tar files. + We convert this list to a string that can be used in the model config to specify + tarred datasets: basename_OP_1..num_tar_files_CL_.tar + + Args: + tar_files (List[str]): List of tar files generated by preprocess_parallel_dataset + """ + num_tar_files = len(tar_files) + split_on_dot = tar_files[0].split('.') + basename = '.'.join(split_on_dot[0:-2]) + tar_file_string = f'{basename}._OP_1..{num_tar_files}_CL_.tar' + return tar_file_string + + @staticmethod + def get_enc_dec_tokenizers( + encoder_tokenizer_name=None, + encoder_tokenizer_model=None, + encoder_bpe_dropout=0.0, + encoder_model_name=None, + decoder_tokenizer_name=None, + decoder_tokenizer_model=None, + decoder_bpe_dropout=0.0, + decoder_model_name=None, + ): + + # if encoder_tokenizer_name != 'yttm' or decoder_tokenizer_name != 'yttm': + # raise NotImplementedError(f"Currently we only support yttm tokenizer.") + + if encoder_bpe_dropout is None: + encoder_bpe_dropout = 0.0 + + if decoder_bpe_dropout is None: + decoder_bpe_dropout = 0.0 + + encoder_tokenizer = get_nmt_tokenizer( + library=encoder_tokenizer_name, + model_name=encoder_model_name, + tokenizer_model=encoder_tokenizer_model, + bpe_dropout=encoder_bpe_dropout, + ) + decoder_tokenizer = get_nmt_tokenizer( + library=decoder_tokenizer_name, + model_name=decoder_model_name, + tokenizer_model=decoder_tokenizer_model, + bpe_dropout=decoder_bpe_dropout, + ) + + return encoder_tokenizer, decoder_tokenizer + + @staticmethod + def get_monolingual_tokenizer( + tokenizer_name=None, tokenizer_model=None, bpe_dropout=0.0, + ): + if tokenizer_name != 'yttm': + raise NotImplementedError(f"Currently we only support yttm tokenizer.") + + if bpe_dropout is None: + bpe_dropout = 0.0 + + tokenizer = get_tokenizer( + tokenizer_name=tokenizer_name, tokenizer_model=tokenizer_model, bpe_dropout=bpe_dropout, + ) + + return tokenizer + + # TODO: add local or global rank 0 decorator + @staticmethod + def preprocess_parallel_dataset( + clean, + src_fname, + tgt_fname, + out_dir, + encoder_tokenizer_name, + encoder_tokenizer_model, + encoder_bpe_dropout, + encoder_model_name, + decoder_tokenizer_name, + decoder_tokenizer_model, + decoder_bpe_dropout, + decoder_model_name, + max_seq_length, + min_seq_length, + tokens_in_batch, + lines_per_dataset_fragment, + num_batches_per_tarfile, + global_rank, + world_size, + n_jobs=-2, + tar_file_prefix='parallel', + ): + """Create tarred dataset from large paired translation data. + + Args: + clean (str): Cleans source and target sentences to get rid of noisy data. + src_fname (str): path to source text data + tgt_fname (str): path to target text data + out_dir (str): path to write tarred dataset + encoder_tokenizer (Any): tokenizer for encoder + decoder_tokenizer (Any): tokenizer for decoder + max_seq_length (int): maximum sequence length + min_seq_length (int): minimum sequence length + tokens_in_batch (int): tokens per batch per GPU, effectively batch size + lines_per_dataset_fragment (int): number of lines to consider for bucketing and padding + num_batches_per_tarfile (int): number of batches (pickle files) within each tarfile + tar_file_prefix (str) : add string prefix to tar files + n_jobs (int): number of processes to use for data processing (-2 to use all but 2) + """ + + os.makedirs(out_dir, exist_ok=True) + + metadata_path = os.path.join(out_dir, f'metadata.tokens.{tokens_in_batch}.json') + + if global_rank == 0: + tar_files_in_out_dir = glob.glob(f'{out_dir}/*.tar') + if tar_files_in_out_dir: + logging.info( + f'Tarred dataset detected: {tar_files_in_out_dir} and will be used. Remove if reprocessing.' + ) + else: + filenames = [src_fname, tgt_fname] + + # get number of lines so that we can create a partition of the lines of the text file + num_src_lines, num_tgt_lines = Parallel(n_jobs=2)( + delayed(MTDataPreproc._get_num_lines)(filename) for filename in filenames + ) + logging.info(f'Found {num_src_lines} source lines and {num_tgt_lines} target lines.') + assert num_src_lines == num_tgt_lines, 'Number of source lines should equal number of target lines.' + + # create a partition of lines that we can parallelize over + lines_partition = MTDataPreproc._get_lines_partition(num_src_lines, lines_per_dataset_fragment) + logging.info(f"Found {len(lines_partition)} fragments to parallelize over.") + + # create tarfiles for each fragment in parallel + results_list = Parallel(n_jobs=n_jobs)( + delayed(MTDataPreproc._process_fragment)( + src_filename=src_fname, + tgt_filename=tgt_fname, + lines_indices=lines_indices, + out_dir=out_dir, + num_batches_per_tarfile=num_batches_per_tarfile, + clean=clean, + max_seq_length=max_seq_length, + min_seq_length=min_seq_length, + tokens_in_batch=tokens_in_batch, + encoder_tokenizer_name=encoder_tokenizer_name, + encoder_tokenizer_model=encoder_tokenizer_model, + encoder_bpe_dropout=encoder_bpe_dropout, + encoder_model_name=encoder_model_name, + decoder_tokenizer_name=decoder_tokenizer_name, + decoder_tokenizer_model=decoder_tokenizer_model, + decoder_bpe_dropout=decoder_bpe_dropout, + decoder_model_name=decoder_model_name, + fragment_index=fragment_index, + ) + for fragment_index, lines_indices in enumerate(lines_partition) + ) + + # compute total batches so far + total_batches = sum([batch_count for batch_count, _ in results_list]) + + # save batches from tar files containing the left over batches (if there's enough batches) + remainder_tar_file_ctr = 0 + remainder_tar_file_path = os.path.join( + out_dir, f'remainder-batches.tokens.{tokens_in_batch}.tar_file_{remainder_tar_file_ctr}.tar' + ) + remainder_tar_file_ptr = tarfile.open(remainder_tar_file_path, 'w') + batch_in_tar_ctr = 0 + for _, tar_file_path in results_list: + tar_file_ptr = tarfile.open(tar_file_path, 'r') + for member in tar_file_ptr.getmembers(): + remainder_tar_file_ptr.addfile(member, tar_file_ptr.extractfile(member.name)) + batch_in_tar_ctr += 1 + if batch_in_tar_ctr == num_batches_per_tarfile: + remainder_tar_file_ctr += 1 + remainder_tar_file_ptr.close() + remainder_tar_file_path = os.path.join( + out_dir, + f'remainder-batches.tokens.{tokens_in_batch}.tar_file_{remainder_tar_file_ctr}.tar', + ) + remainder_tar_file_ptr = tarfile.open(remainder_tar_file_path, 'w',) + batch_in_tar_ctr = 0 + tar_file_ptr.close() + os.remove(tar_file_path) + + # log the number of batches remaining as they will be discarded + num_batches_discarded = len(remainder_tar_file_ptr.getmembers()) + total_batches -= num_batches_discarded + logging.info( + f'Number of batches discarded: {num_batches_discarded}, total batches kept: {total_batches}' + ) + remainder_tar_file_ptr.close() + os.remove(remainder_tar_file_path) + + # dump metadata to json + metadata = {} + metadata['num_batches'] = total_batches + + # rename tar files so they can be more easily used with CLI and YAML + tar_file_paths = glob.glob(f'{out_dir}/*.tar') + for index, path in enumerate(tar_file_paths): + os.rename( + path, os.path.join(out_dir, f'{tar_file_prefix}.batches.tokens.{tokens_in_batch}.{index}.tar') + ) + + # add tar files to manifest + tar_file_paths = glob.glob(f'{out_dir}/*.tar') + metadata['tar_files'] = tar_file_paths + json.dump(metadata, open(metadata_path, 'w')) + + tar_file_paths = glob.glob(f'{out_dir}/*.tar') + + num_tar_files = len(tar_file_paths) + if num_tar_files < world_size: + raise ValueError( + ( + f'Number of tar files found: {num_tar_files} is less than world size: {world_size}. ' + f'There should be at least one tar file per GPU (ideally many tar files per GPU). ' + f'This may be due to dataset size, it is advisable to use at least 5M sentence pairs for tarred datasets. ' + f'Decrease num_batches_per_tarfile or num_tokens_per_batch to increase the number of tarfiles. ' + f'Also using shard_strategy=replicate will use all available tarfiles for every GPU. ' + ) + ) + + return tar_file_paths, metadata_path + + @staticmethod + def _get_num_lines(filename): + with open(filename) as f: + for i, l in enumerate(f): + pass + return i + 1 + + @staticmethod + def _get_lines_partition(num_lines, lines_per_dataset_fragment): + # create partition based on fragment size + fragment_indices = [] + for i in range(0, num_lines, lines_per_dataset_fragment): + fragment_indices.append([i, i + lines_per_dataset_fragment]) + # modify last indices + last_indices = fragment_indices.pop() + last_indices[1] = -1 + fragment_indices.append(last_indices) + # if fragment_indices[-1][1] >= num_lines: + # fragment_indices.pop() + return fragment_indices + + @staticmethod + def _process_fragment( + src_filename, + tgt_filename, + lines_indices, + out_dir, + num_batches_per_tarfile, + clean, + max_seq_length, + min_seq_length, + tokens_in_batch, + encoder_tokenizer_name, + encoder_tokenizer_model, + encoder_bpe_dropout, + encoder_model_name, + decoder_tokenizer_name, + decoder_tokenizer_model, + decoder_bpe_dropout, + decoder_model_name, + fragment_index, + ): + start = lines_indices[0] + stop = lines_indices[1] + + # write lines in partition to temporary files to be consumed by write_parallel_batches_to_tarfiles + tmp_f_src = tempfile.NamedTemporaryFile(delete=False, mode='w') + tmp_f_tgt = tempfile.NamedTemporaryFile(delete=False, mode='w') + + with open(src_filename, 'r') as src_in, open(tgt_filename) as tgt_in: + for line_number, (src_line, tgt_line) in enumerate(zip(src_in, tgt_in)): + if start <= line_number and line_number < stop: + if src_line and tgt_line: + tmp_f_src.write(src_line) + tmp_f_tgt.write(tgt_line) + + tmp_f_src.close() + tmp_f_tgt.close() + + num_batches_from_fragment, remainder_tar_file_path = MTDataPreproc.write_parallel_batches_to_tarfiles( + out_dir=out_dir, + num_batches_per_tarfile=num_batches_per_tarfile, + clean=clean, + max_seq_length=max_seq_length, + min_seq_length=min_seq_length, + src_fname=tmp_f_src.name, + tgt_fname=tmp_f_tgt.name, + num_tokens=tokens_in_batch, + encoder_tokenizer_name=encoder_tokenizer_name, + encoder_tokenizer_model=encoder_tokenizer_model, + encoder_bpe_dropout=encoder_bpe_dropout, + encoder_model_name=encoder_model_name, + decoder_tokenizer_name=decoder_tokenizer_name, + decoder_tokenizer_model=decoder_tokenizer_model, + decoder_bpe_dropout=decoder_bpe_dropout, + decoder_model_name=decoder_model_name, + fragment_index=fragment_index, + ) + + os.remove(tmp_f_src.name) + os.remove(tmp_f_tgt.name) + + return num_batches_from_fragment, remainder_tar_file_path + + @staticmethod + def preprocess_monolingual_dataset( + clean, + fname, + out_dir, + tokenizer, + max_seq_length, + min_seq_length, + tokens_in_batch, + lines_per_dataset_fragment, + num_batches_per_tarfile, + pkl_file_prefix, + global_rank, + world_size, + ): + """Create tarred dataset from a large monolingual corpus. + + Args: + clean (str): Cleans sentences to get rid of very long or short sentences. + fname (str): Path to source text data + out_dir (str): Path to write tarred dataset + tokenizer (Any): Path to tokenizer model + max_seq_length (int): maximum sequence length + min_seq_length (int): minimum sequence length + tokens_in_batch (int): tokens per batch per GPU, effectively batch size + lines_per_dataset_fragment (int): number of lines to consider for bucketing and padding + num_batches_per_tarfile (int): number of batches (pickle files) within each tarfile + global_rank (int): if set to zero, data will be processed on this node + world_size (int): total number of processes being run (for training only, set to 1 when preproc only) + """ + os.makedirs(out_dir, exist_ok=True) + + tar_file_ctr = 1 + num_files_in_tar = 0 + num_lines = 0 + shard_num = 0 + global_batch_ctr = 0 + tmp_f = tempfile.NamedTemporaryFile(delete=False, mode='w') + tar_file_ptr = tarfile.open( + os.path.join(out_dir, '%s-batches.tokens.%d.%d.tar' % (pkl_file_prefix, tokens_in_batch, 1)), 'w' + ) + metadata_path = os.path.join(out_dir, f'metadata.tokens.{tokens_in_batch}.json') + with open(fname, 'r') as f: + for line in f: + tmp_f.write(line) + num_lines += 1 + + if num_lines == lines_per_dataset_fragment: + tmp_f.close() + ( + tar_file_ptr, + global_batch_ctr, + num_files_in_tar, + tar_file_ctr, + ) = MTDataPreproc.write_monolingual_batches_to_tarfiles( + out_dir, + num_batches_per_tarfile, + clean, + max_seq_length, + min_seq_length, + tmp_f.name, + tokens_in_batch, + tokenizer, + num_files_in_tar=num_files_in_tar, + tar_file_ptr=tar_file_ptr, + tar_file_ctr=tar_file_ctr, + global_batch_ctr=global_batch_ctr, + pkl_file_prefix=pkl_file_prefix, + ) + + num_lines = 0 + shard_num += 1 + os.remove(tmp_f.name) + tmp_f = tempfile.NamedTemporaryFile(delete=False, mode='w') + + tmp_f.close() + ( + tar_file_ptr, + global_batch_ctr, + num_files_in_tar, + tar_file_ctr, + ) = MTDataPreproc.write_monolingual_batches_to_tarfiles( + out_dir, + num_batches_per_tarfile, + clean, + max_seq_length, + min_seq_length, + tmp_f.name, + tokens_in_batch, + tokenizer, + num_files_in_tar=num_files_in_tar, + tar_file_ptr=tar_file_ptr, + tar_file_ctr=tar_file_ctr, + global_batch_ctr=global_batch_ctr, + pkl_file_prefix=pkl_file_prefix, + ) + tar_file_ptr.close() + os.remove(tmp_f.name) + + if num_files_in_tar != num_batches_per_tarfile: + os.remove( + os.path.join(out_dir, '%s-batches.tokens.%d.%d.tar' % (pkl_file_prefix, tokens_in_batch, tar_file_ctr)) + ) + global_batch_ctr -= num_files_in_tar + print('Dropping %d batches because of overflow' % (num_files_in_tar)) + + json.dump({'num_batches': global_batch_ctr}, open(os.path.join(out_dir, 'metadata.json'), 'w')) + + tar_file_paths = glob.glob(f'{out_dir}/{pkl_file_prefix}-batches.tokens.{tokens_in_batch}.*.tar') + + num_tar_files = len(tar_file_paths) + if num_tar_files < world_size: + raise ValueError( + ( + f'Number of tar files found: {num_tar_files} is less than world size: {world_size}. ' + f'There should be at least one tar file per GPU (ideally many tar files per GPU). ' + f'This may be due to dataset size, it is advisable to use at least 5M sentence pairs for tarred datasets. ' + f'Decrease num_batches_per_tarfile or num_tokens_per_batch to increase the number of tarfiles. ' + f'Also using shard_strategy=replicate will use all available tarfiles for every GPU. ' + ) + ) + + return tar_file_paths, metadata_path + + @staticmethod + def train_tokenizers( + out_dir, + src_fname, + tgt_fname, + shared_tokenizer, + encoder_tokenizer_name, + encoder_tokenizer_vocab_size, + encoder_tokenizer_coverage, + decoder_tokenizer_name, + decoder_tokenizer_vocab_size, + decoder_tokenizer_coverage, + global_rank, + encoder_training_sample_size=-1, + decoder_training_sample_size=-1, + encoder_special_tokens=None, + decoder_special_tokens=None, + spt_symbols=None, + multilingual=False, + ): + encoder_tokenizer_model = None + decoder_tokenizer_model = None + os.makedirs(out_dir, exist_ok=True) + + supported_train_tokenizers = ['yttm', 'sentencepiece'] + + if encoder_special_tokens: + if isinstance(encoder_special_tokens, dict): + encoder_special_tokens = list(encoder_special_tokens.values()) + print(encoder_special_tokens) + + if decoder_special_tokens: + if isinstance(decoder_special_tokens, dict): + decoder_special_tokens = list(decoder_special_tokens.values()) + + if multilingual and encoder_tokenizer_name != 'sentencepiece': + raise NotImplementedError( + f"Currently we only support training setencepiece tokenizer for multilingual model." + ) + + if shared_tokenizer: + if ( + encoder_tokenizer_name not in supported_train_tokenizers + or decoder_tokenizer_name not in supported_train_tokenizers + ): + raise NotImplementedError( + f"Currently we only support tokenizers in {supported_train_tokenizers} for shared tokenizer." + ) + + encoder_tokenizer_model = os.path.join( + out_dir, 'shared_tokenizer.%d.BPE.model' % (encoder_tokenizer_vocab_size) + ) + decoder_tokenizer_model = encoder_tokenizer_model + if global_rank == 0: + if os.path.isfile(encoder_tokenizer_model): + logging.info( + f'Shared tokenizer model {encoder_tokenizer_model} already exists. Remove file if training a new tokenizer model.' + ) + else: + logging.info( + f'Shared tokenizer model {encoder_tokenizer_model} not found. Training tokenizer model.' + ) + with tempfile.TemporaryDirectory() as tmp: + concat_data_path = os.path.join(tmp, 'concat_dataset.txt') + os.system('cat %s %s > %s' % (src_fname, tgt_fname, concat_data_path)) + if encoder_tokenizer_name == "yttm": + yttm.BPE.train( + data=concat_data_path, + vocab_size=encoder_tokenizer_vocab_size, + model=os.path.join(out_dir, encoder_tokenizer_model), + coverage=encoder_tokenizer_coverage, + n_threads=-1, + ) + else: + create_spt_model( + data_file=concat_data_path, + vocab_size=encoder_tokenizer_vocab_size, + sample_size=encoder_training_sample_size, + do_lower_case=False, + tokenizer_type='bpe', + character_coverage=encoder_tokenizer_coverage, + output_dir=out_dir, + bos=True, + eos=True, + pad=True, + control_symbols=spt_symbols, + user_defined_symbols=encoder_special_tokens, + ) + os.rename( + os.path.join(out_dir, 'tokenizer.model'), + os.path.join(out_dir, encoder_tokenizer_model), + ) + else: + if encoder_tokenizer_name in supported_train_tokenizers: + encoder_tokenizer_model = os.path.join( + out_dir, 'tokenizer.encoder.%d.BPE.model' % (encoder_tokenizer_vocab_size) + ) + if global_rank == 0: + if os.path.isfile(encoder_tokenizer_model): + logging.info( + f'Encoder tokenizer model {encoder_tokenizer_model} already exists. Remove file if training a new tokenizer model.' + ) + else: + logging.info( + f'Encoder tokenizer model {encoder_tokenizer_model} not found. Training tokenizer model.' + ) + if encoder_tokenizer_name == "yttm": + yttm.BPE.train( + data=src_fname, + vocab_size=encoder_tokenizer_vocab_size, + model=encoder_tokenizer_model, + coverage=encoder_tokenizer_coverage, + n_threads=-1, + ) + else: + dir_name = os.path.dirname(encoder_tokenizer_model) + create_spt_model( + data_file=src_fname, + vocab_size=encoder_tokenizer_vocab_size, + sample_size=encoder_training_sample_size, + do_lower_case=False, + tokenizer_type='bpe', + character_coverage=encoder_tokenizer_coverage, + output_dir=dir_name, + bos=True, + eos=True, + pad=True, + control_symbols=spt_symbols, + user_defined_symbols=encoder_special_tokens, + ) + os.rename(os.path.join(dir_name, 'tokenizer.model'), os.path.join(encoder_tokenizer_model)) + + if decoder_tokenizer_name in supported_train_tokenizers: + decoder_tokenizer_model = os.path.join( + out_dir, 'tokenizer.decoder.%d.BPE.model' % (decoder_tokenizer_vocab_size) + ) + if global_rank == 0: + if os.path.isfile(decoder_tokenizer_model): + logging.info( + f'Decoder tokenizer model {decoder_tokenizer_model} already exists. Remove file if training a new tokenizer model.' + ) + else: + logging.info( + f'Decoder tokenizer model {decoder_tokenizer_model} not found. Training tokenizer model.' + ) + if decoder_tokenizer_name == "yttm": + yttm.BPE.train( + data=tgt_fname, + vocab_size=decoder_tokenizer_vocab_size, + model=decoder_tokenizer_model, + coverage=decoder_tokenizer_coverage, + n_threads=-1, + ) + else: + dir_name = os.path.dirname(decoder_tokenizer_model) + create_spt_model( + data_file=tgt_fname, + vocab_size=decoder_tokenizer_vocab_size, + sample_size=decoder_training_sample_size, + do_lower_case=False, + tokenizer_type='bpe', + character_coverage=decoder_tokenizer_coverage, + output_dir=dir_name, + bos=True, + eos=True, + pad=True, + control_symbols=spt_symbols, + user_defined_symbols=decoder_special_tokens, + ) + os.rename(os.path.join(dir_name, 'tokenizer.model'), os.path.join(decoder_tokenizer_model)) + + return encoder_tokenizer_model, decoder_tokenizer_model + + @staticmethod + def write_parallel_batches_to_tarfiles( + out_dir, + num_batches_per_tarfile, + clean, + max_seq_length, + min_seq_length, + src_fname, + tgt_fname, + num_tokens, + encoder_tokenizer_name, + encoder_tokenizer_model, + encoder_bpe_dropout, + encoder_model_name, + decoder_tokenizer_name, + decoder_tokenizer_model, + decoder_bpe_dropout, + decoder_model_name, + fragment_index, + ): + """ + Writes current fragment of the overall parallel corpus to tarfiles by: + (1) Creating a minibatches using a TranslationDataset object. + (2) Writing each minibatch to a pickle file. + (3) Adding pickle files to a tarfile until it reaches num_batches_per_tarfile. + """ + + dataset = TranslationDataset( + dataset_src=src_fname, + dataset_tgt=tgt_fname, + tokens_in_batch=num_tokens, + clean=clean, + max_seq_length=max_seq_length, + min_seq_length=min_seq_length, + max_seq_length_diff=max_seq_length, + max_seq_length_ratio=max_seq_length, + cache_ids=False, + cache_data_per_node=False, + use_cache=False, + ) + encoder_tokenizer, decoder_tokenizer = MTDataPreproc.get_enc_dec_tokenizers( + encoder_tokenizer_name, + encoder_tokenizer_model, + encoder_bpe_dropout, + encoder_model_name, + decoder_tokenizer_name, + decoder_tokenizer_model, + decoder_bpe_dropout, + decoder_model_name, + ) + dataset.batchify(encoder_tokenizer, decoder_tokenizer) + + tar_file_ctr = 0 + tar_file_path = os.path.join( + out_dir, 'fragment-%s-batches.tokens.%d.%d.tar' % (fragment_index, num_tokens, tar_file_ctr) + ) + tar_file_ptr = tarfile.open(tar_file_path, 'w') + total_batch_ctr = 0 + batch_ctr = 0 + for _, batch in dataset.batches.items(): + total_batch_ctr += 1 + batch_ctr += 1 + pickle.dump( + batch, + open(os.path.join(out_dir, 'fragment-%s-batch-%d.pkl' % (fragment_index, total_batch_ctr)), 'wb'), + ) + tar_file_ptr.add(os.path.join(out_dir, 'fragment-%s-batch-%d.pkl' % (fragment_index, total_batch_ctr))) + os.remove(os.path.join(out_dir, 'fragment-%s-batch-%d.pkl' % (fragment_index, total_batch_ctr))) + + if batch_ctr == num_batches_per_tarfile: + tar_file_ctr += 1 + tar_file_ptr.close() + tar_file_path = os.path.join( + out_dir, 'fragment-%s-batches.tokens.%d.%d.tar' % (fragment_index, num_tokens, tar_file_ctr) + ) + tar_file_ptr = tarfile.open(tar_file_path, 'w',) + batch_ctr = 0 + + # return tar files paths that have batches remaining + remainder_tar_file_path = tar_file_ptr.name + tar_file_ptr.close() + + return total_batch_ctr, remainder_tar_file_path + + @staticmethod + def write_monolingual_batches_to_tarfiles( + out_dir, + num_batches_per_tarfile, + clean, + max_seq_length, + min_seq_length, + fname, + num_tokens, + tokenizer, + num_files_in_tar, + tar_file_ptr, + tar_file_ctr, + global_batch_ctr, + pkl_file_prefix, + ): + """ + Writes current fragment of the overall parallel corpus to tarfiles by: + (1) Creating a minibatches using a SentenceDataset object. + (2) Writing each minibatch to a pickle file. + (3) Adding pickle files to a tarfile until it reaches num_batches_per_tarfile. + """ + + dataset = SentenceDataset( + tokenizer=tokenizer, + dataset=fname, + tokens_in_batch=num_tokens, + clean=clean, + max_seq_length=max_seq_length, + min_seq_length=min_seq_length, + cache_ids=False, + ) + + for batch in dataset.batches: + global_batch_ctr += 1 + batch = {'src': batch} + pickle.dump( + batch, open(os.path.join(out_dir, '%s-batch-%d.pkl' % (pkl_file_prefix, global_batch_ctr)), 'wb') + ) + + if num_files_in_tar == num_batches_per_tarfile: + tar_file_ctr += 1 + tar_file_ptr.close() + tar_file_ptr = tarfile.open( + os.path.join(out_dir, '%s-batches.tokens.%d.%d.tar' % (pkl_file_prefix, num_tokens, tar_file_ctr)), + 'w', + ) + num_files_in_tar = 0 + + tar_file_ptr.add(os.path.join(out_dir, '%s-batch-%d.pkl' % (pkl_file_prefix, global_batch_ctr))) + num_files_in_tar += 1 + os.remove(os.path.join(out_dir, '%s-batch-%d.pkl' % (pkl_file_prefix, global_batch_ctr))) + return tar_file_ptr, global_batch_ctr, num_files_in_tar, tar_file_ctr + + @property + def cfg(self): + return self._cfg diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/neural_machine_translation/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/neural_machine_translation/__init__.py new file mode 100644 index 0000000000..42ee85aaa5 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/neural_machine_translation/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.collections.nlp.data.neural_machine_translation.neural_machine_translation_dataset import ( + NeuralMachineTranslationDataset, +) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/neural_machine_translation/neural_machine_translation_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/neural_machine_translation/neural_machine_translation_dataset.py new file mode 100644 index 0000000000..43dc4b26db --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/neural_machine_translation/neural_machine_translation_dataset.py @@ -0,0 +1,146 @@ +# Copyright (c) 2020, MeetKai Inc. All rights reserved. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from typing import Dict, List, Optional, Tuple + +import numpy as np + +from nemo.collections.common.tokenizers import AutoTokenizer +from nemo.core.classes import Dataset +from nemo.core.neural_types import ChannelType, MaskType, NeuralType + +__all__ = ["NeuralMachineTranslationDataset"] + + +class NeuralMachineTranslationDataset(Dataset): + """A dataset class that converts raw data to a dataset that can be used by NeuralMachineTranslationModel. + + Args: + filepath: .tsv file to sequence + label. + the first line is header (sentence [tab] label) + each line should be [sentence][tab][label] + encoder_tokenizer: encoder tokenizer object such as AutoTokenizer + decoder_tokenizer: decoder tokenizer object. If using BART or end to end model, set this to encoder_tokenizer + max_seq_length: max sequence length including bos and eos tokens + num_samples: number of samples you want to use for the dataset. If -1, use all dataset. Useful for testing. + convert_labels: if true, converts labels for masked lm and updates pad_id to -100 + for hf masked loss + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + "input_ids": NeuralType(("B", "T"), ChannelType()), + "attention_mask": NeuralType(("B", "T"), MaskType()), + "decoder_input_ids": NeuralType(("B", "T"), ChannelType()), + "lm_labels": NeuralType(("B", "T"), ChannelType()), + } + + def __init__( + self, + filepath: str, + encoder_tokenizer: AutoTokenizer, + decoder_tokenizer: AutoTokenizer, + encoder_add_special_tokens: bool, + decoder_add_special_tokens: bool, + max_seq_length: int, + num_samples: int = -1, + convert_labels: bool = False, + ): + self.filepath = filepath + self.encoder_tokenizer = encoder_tokenizer + self.decoder_tokenizer = decoder_tokenizer + self.encoder_add_special_tokens = encoder_add_special_tokens + self.decoder_add_special_tokens = decoder_add_special_tokens + self.max_seq_length = max_seq_length + self.num_samples = num_samples + self.convert_labels = convert_labels + + if num_samples == 0: + raise ValueError("num_samples has to be positive.", num_samples) + + if self.max_seq_length and self.max_seq_length <= 2: + self.max_seq_length = None + + if not os.path.exists(filepath): + raise FileNotFoundError( + f"{filepath} not found. The filepath must be set in train_ds.filepath and validation_ds.filepath." + ) + + with open(filepath) as f: + lines = f.readlines()[1:] + + if num_samples > 0: + lines = lines[:num_samples] + + input_ids, input_masks, label_ids = [], [], [] + for line in lines: + try: + sentence, label = line.split("\t") + except ValueError: + raise ValueError("Each line of input file should contain the format [sentence][tab][label].") + ids, mask = self.text_to_ids( + sentence, tokenizer=encoder_tokenizer, add_special_tokens=encoder_add_special_tokens + ) + input_ids.append(ids) + input_masks.append(mask) + label_ids.append( + self.text_to_ids(label, tokenizer=decoder_tokenizer, add_special_tokens=decoder_add_special_tokens)[0] + ) + + self.input_ids = np.asarray(input_ids) + self.input_masks = np.asarray(input_masks) + self.label_ids = np.asarray(label_ids) + + def text_to_ids( + self, text: str, tokenizer: AutoTokenizer, add_special_tokens=False + ) -> Tuple[List[int], List[int]]: + """Converts text to ids. Truncates and adds padding.""" + text_tokens = tokenizer.text_to_ids(text.strip()) + num_special_tokens = 2 if add_special_tokens else 0 + + if self.max_seq_length and self.max_seq_length > num_special_tokens: + text_tokens = text_tokens[: self.max_seq_length - num_special_tokens] + if add_special_tokens: + text_tokens = [tokenizer.bos_id] + text_tokens + [tokenizer.eos_id] + mask = [1] * len(text_tokens) + + if self.max_seq_length and self.max_seq_length > num_special_tokens: + pad_length = self.max_seq_length - len(text_tokens) + text_tokens += [tokenizer.pad_id] * pad_length + mask += [0] * pad_length + + return text_tokens, mask + + def __len__(self): + return len(self.input_ids) + + def convert_label_ids(self, label_ids: List[int]) -> Tuple[List[int], List[int]]: + decoder_input_ids = label_ids[:-1] + lm_labels = label_ids[1:].copy() + lm_labels[label_ids[1:] == self.decoder_tokenizer.pad_id] = -100 # for huggingface masked lm loss + return decoder_input_ids, lm_labels + + def __getitem__(self, idx): + if self.convert_labels: + decoder_input_ids, lm_labels = self.convert_label_ids(self.label_ids[idx]) + else: + decoder_input_ids = self.label_ids[idx] + lm_labels = self.label_ids[idx] + return self.input_ids[idx], self.input_masks[idx], decoder_input_ids, lm_labels diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/__init__.py new file mode 100644 index 0000000000..9e32500719 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/qa_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/qa_dataset.py new file mode 100644 index 0000000000..337af84688 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/qa_dataset.py @@ -0,0 +1,443 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import collections +import json +import os +import pickle +from typing import Dict, List, Optional + +import numpy as np +import torch + +from nemo.collections.common.parts.utils import _compute_softmax +from nemo.collections.nlp.data.question_answering_squad.qa_squad_processing import ( + EVALUATION_MODE, + INFERENCE_MODE, + TRAINING_MODE, + SquadProcessor, + apply_no_ans_threshold, + convert_examples_to_features, + exact_match_score, + f1_score, + find_all_best_thresh, + get_best_indexes, + get_final_text, + make_eval_dict, + merge_eval, + normalize_answer, +) +from nemo.core.classes import Dataset +from nemo.utils import logging + +__all__ = ['SquadDataset'] + + +class SquadDataset(Dataset): + """ + Creates SQuAD dataset for Question Answering. + Args: + data_file (str): train.*.json eval.*.json or test.*.json. + tokenizer (obj): Tokenizer object, e.g. AutoTokenizer. + version_2_with_negative (bool): True if training should allow + unanswerable questions. + doc_stride (int): When splitting up a long document into chunks, + how much stride to take between chunks. + max_query_length (iny): All training files which have a duration less + than min_duration are dropped. Can't be used if the `utt2dur` file + does not exist. Defaults to None. + max_seq_length (int): All training files which have a duration more + than max_duration are dropped. Can't be used if the `utt2dur` file + does not exist. Defaults to None. + num_samples: number of samples you want to use for the dataset. + If -1, use all dataset. Useful for testing. + mode (str): Use TRAINING_MODE/EVALUATION_MODE/INFERENCE_MODE to define between + training, evaluation and inference dataset. + use_cache (bool): Caches preprocessed data for future usage + """ + + def __init__( + self, + data_file: str, + tokenizer: object, + doc_stride: int, + max_query_length: int, + max_seq_length: int, + version_2_with_negative: bool, + num_samples: int, + mode: str, + use_cache: bool, + ): + self.tokenizer = tokenizer + self.version_2_with_negative = version_2_with_negative + self.processor = SquadProcessor(data_file=data_file, mode=mode) + self.mode = mode + if mode not in [TRAINING_MODE, EVALUATION_MODE, INFERENCE_MODE]: + raise ValueError( + f"mode should be either {TRAINING_MODE}, {EVALUATION_MODE}, {INFERENCE_MODE} but got {mode}" + ) + self.examples = self.processor.get_examples() + + vocab_size = getattr(tokenizer, "vocab_size", 0) + cached_features_file = ( + data_file + + '_cache' + + '_{}_{}_{}_{}_{}_{}_{}'.format( + mode, + tokenizer.name, + str(vocab_size), + str(max_seq_length), + str(doc_stride), + str(max_query_length), + str(num_samples), + ) + ) + + # check number of samples. Should be either -1 not to limit or positive number + if num_samples == 0: + raise ValueError( + f"num_samples has to be positive or -1 (to use the entire dataset), however got {num_samples}." + ) + elif num_samples > 0: + self.examples = self.examples[:num_samples] + + if use_cache and os.path.exists(cached_features_file): + logging.info(f"loading from {cached_features_file}") + with open(cached_features_file, "rb") as reader: + self.features = pickle.load(reader) + else: + logging.info(f"Preprocessing data.") + + self.features = convert_examples_to_features( + examples=self.examples, + tokenizer=tokenizer, + max_seq_length=max_seq_length, + doc_stride=doc_stride, + max_query_length=max_query_length, + has_groundtruth=mode != INFERENCE_MODE, + ) + + if use_cache: + master_device = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + if master_device: + logging.info(" Saving train features into cached file %s", cached_features_file) + with open(cached_features_file, "wb") as writer: + pickle.dump(self.features, writer) + + def __len__(self): + return len(self.features) + + def __getitem__(self, idx): + feature = self.features[idx] + if self.mode == INFERENCE_MODE: + return ( + np.array(feature.input_ids), + np.array(feature.segment_ids), + np.array(feature.input_mask), + np.array(feature.unique_id), + ) + else: + return ( + np.array(feature.input_ids), + np.array(feature.segment_ids), + np.array(feature.input_mask), + np.array(feature.unique_id), + np.array(feature.start_position), + np.array(feature.end_position), + ) + + def get_predictions( + self, + unique_ids: List[int], + start_logits: List[List[float]], + end_logits: List[List[float]], + n_best_size: int, + max_answer_length: int, + do_lower_case: bool, + version_2_with_negative: bool, + null_score_diff_threshold: float, + ): + example_index_to_features = collections.defaultdict(list) + + unique_id_to_pos = {} + for index, unique_id in enumerate(unique_ids): + unique_id_to_pos[unique_id] = index + + for feature in self.features: + example_index_to_features[feature.example_index].append(feature) + + _PrelimPrediction = collections.namedtuple( + "PrelimPrediction", ["feature_index", "start_index", "end_index", "start_logit", "end_logit"] + ) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + for (example_index, example) in enumerate(self.examples): + + # finish this loop if we went through all batch examples + if example_index >= len(unique_ids): + break + + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + # large and positive + score_null = 1000000 + # the paragraph slice with min null score + min_null_feature_index = 0 + # start logit at the slice with min null score + null_start_logit = 0 + # end logit at the slice with min null score + null_end_logit = 0 + for (feature_index, feature) in enumerate(features): + pos = unique_id_to_pos[feature.unique_id] + start_indexes = get_best_indexes(start_logits[pos], n_best_size) + end_indexes = get_best_indexes(end_logits[pos], n_best_size) + # if we could have irrelevant answers, + # get the min score of irrelevant + if version_2_with_negative: + feature_null_score = start_logits[pos][0] + end_logits[pos][0] + if feature_null_score < score_null: + score_null = feature_null_score + min_null_feature_index = feature_index + null_start_logit = start_logits[pos][0] + null_end_logit = end_logits[pos][0] + for start_index in start_indexes: + for end_index in end_indexes: + # We could hypothetically create invalid predictions, + # e.g., predict that the start of the span is in the + # question. We throw out all invalid predictions. + if start_index >= len(feature.tokens): + continue + if end_index >= len(feature.tokens): + continue + if start_index not in feature.token_to_orig_map: + continue + if end_index not in feature.token_to_orig_map: + continue + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index, + end_index=end_index, + start_logit=start_logits[pos][start_index], + end_logit=end_logits[pos][end_index], + ) + ) + + if version_2_with_negative: + prelim_predictions.append( + _PrelimPrediction( + feature_index=min_null_feature_index, + start_index=0, + end_index=0, + start_logit=null_start_logit, + end_logit=null_end_logit, + ) + ) + prelim_predictions = sorted(prelim_predictions, key=lambda x: (x.start_logit + x.end_logit), reverse=True) + + _NbestPrediction = collections.namedtuple("NbestPrediction", ["text", "start_logit", "end_logit"]) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + if pred.start_index > 0: # this is a non-null prediction + tok_tokens = feature.tokens[pred.start_index : (pred.end_index + 1)] + orig_doc_start = feature.token_to_orig_map[pred.start_index] + orig_doc_end = feature.token_to_orig_map[pred.end_index] + orig_tokens = example.doc_tokens[orig_doc_start : (orig_doc_end + 1)] + tok_text = " ".join(tok_tokens) + + # De-tokenize WordPieces that have been split off. + tok_text = tok_text.replace(" ##", "") + tok_text = tok_text.replace("##", "") + + # Clean whitespace + tok_text = tok_text.strip() + tok_text = " ".join(tok_text.split()) + orig_text = " ".join(orig_tokens) + + final_text = get_final_text(tok_text, orig_text, do_lower_case) + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + else: + final_text = "" + seen_predictions[final_text] = True + + nbest.append(_NbestPrediction(text=final_text, start_logit=pred.start_logit, end_logit=pred.end_logit)) + + # if we didn't include the empty option in the n-best, include it + if version_2_with_negative: + if "" not in seen_predictions: + nbest.append(_NbestPrediction(text="", start_logit=null_start_logit, end_logit=null_end_logit)) + + # In very rare edge cases we could only + # have single null pred. We just create a nonce prediction + # in this case to avoid failure. + if len(nbest) == 1: + nbest.insert(0, _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append(_NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + assert len(nbest) >= 1 + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_logit + entry.end_logit) + if not best_non_null_entry: + if entry.text: + best_non_null_entry = entry + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["question"] = example.question_text + output["text"] = entry.text + output["probability"] = probs[i] + output["start_logit"] = ( + entry.start_logit if isinstance(entry.start_logit, float) else list(entry.start_logit) + ) + output["end_logit"] = entry.end_logit if isinstance(entry.end_logit, float) else list(entry.end_logit) + nbest_json.append(output) + + assert len(nbest_json) >= 1 + if not version_2_with_negative: + all_predictions[example.qas_id] = nbest_json[0]["text"] + else: + # predict "" iff the null score - + # the score of best non-null > threshold + score_diff = score_null - best_non_null_entry.start_logit - (best_non_null_entry.end_logit) + scores_diff_json[example.qas_id] = score_diff + if score_diff > null_score_diff_threshold: + all_predictions[example.qas_id] = "" + else: + all_predictions[example.qas_id] = best_non_null_entry.text + all_nbest_json[example.qas_id] = nbest_json + + return all_predictions, all_nbest_json, scores_diff_json + + def evaluate_predictions( + self, + all_predictions: Dict[str, str], + no_answer_probs: Optional[float] = None, + no_answer_probability_threshold: float = 1.0, + ): + qas_id_to_has_answer = { + example.qas_id: bool(example.answers) for example in self.examples[: len(all_predictions)] + } + has_answer_qids = [qas_id for qas_id, has_answer in qas_id_to_has_answer.items() if has_answer] + no_answer_qids = [qas_id for qas_id, has_answer in qas_id_to_has_answer.items() if not has_answer] + if no_answer_probs is None: + no_answer_probs = {k: 0.0 for k in all_predictions} + + exact, f1 = self.get_raw_scores(all_predictions) + + exact_threshold = apply_no_ans_threshold( + exact, no_answer_probs, qas_id_to_has_answer, no_answer_probability_threshold + ) + f1_threshold = apply_no_ans_threshold( + f1, no_answer_probs, qas_id_to_has_answer, no_answer_probability_threshold + ) + + evaluation = make_eval_dict(exact_threshold, f1_threshold) + + if has_answer_qids: + has_ans_eval = make_eval_dict(exact_threshold, f1_threshold, qid_list=has_answer_qids) + merge_eval(evaluation, has_ans_eval, "HasAns") + + if no_answer_qids: + no_ans_eval = make_eval_dict(exact_threshold, f1_threshold, qid_list=no_answer_qids) + merge_eval(evaluation, no_ans_eval, "NoAns") + + if no_answer_probs: + find_all_best_thresh(evaluation, all_predictions, exact, f1, no_answer_probs, qas_id_to_has_answer) + + return evaluation["best_exact"], evaluation["best_f1"] + + def get_raw_scores(self, preds: Dict[str, str]): + """ + Computes the exact and f1 scores from the examples + and the model predictions + """ + exact_scores = {} + f1_scores = {} + + for example in self.examples: + qas_id = example.qas_id + gold_answers = [answer["text"] for answer in example.answers if normalize_answer(answer["text"])] + + if not gold_answers: + # For unanswerable questions, + # only correct answer is empty string + gold_answers = [""] + + if qas_id not in preds: + logging.warning("Missing prediction for %s" % qas_id) + continue + + prediction = preds[qas_id] + exact_scores[qas_id] = max(exact_match_score(a, prediction) for a in gold_answers) + f1_scores[qas_id] = max(f1_score(a, prediction) for a in gold_answers) + + return exact_scores, f1_scores + + def evaluate( + self, + unique_ids: List[str], + start_logits: List[List[float]], + end_logits: List[List[float]], + n_best_size: int, + max_answer_length: int, + do_lower_case: bool, + version_2_with_negative: bool, + null_score_diff_threshold: float, + ): + (all_predictions, all_nbest_json, scores_diff_json) = self.get_predictions( + unique_ids, + start_logits, + end_logits, + n_best_size, + max_answer_length, + do_lower_case, + version_2_with_negative, + null_score_diff_threshold, + ) + + exact_match, f1 = self.evaluate_predictions(all_predictions) + + return exact_match, f1, all_predictions, all_nbest_json diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/qa_squad_processing.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/qa_squad_processing.py new file mode 100644 index 0000000000..5847baa825 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/question_answering_squad/qa_squad_processing.py @@ -0,0 +1,644 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import json +from typing import Dict, List, Optional + +from tqdm import tqdm +from transformers.models.bert.tokenization_bert import BasicTokenizer + +from nemo.collections.nlp.data.data_utils import DataProcessor, is_whitespace, normalize_answer +from nemo.utils import logging + +""" +Utility functions for Question Answering NLP tasks +Some parts of this code were adapted from the HuggingFace library at +https://github.com/huggingface/transformers +""" + +TRAINING_MODE = "train" +EVALUATION_MODE = "eval" +INFERENCE_MODE = "infer" + + +def _get_tokens(s): + """get normalized tokens""" + if not s: + return [] + return normalize_answer(s).split() + + +def get_best_indexes(logits, n_best_size): + """Get the n-best logits from a list.""" + index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) + + best_indexes = [] + for i in range(len(index_and_score)): + if i >= n_best_size: + break + best_indexes.append(index_and_score[i][0]) + return best_indexes + + +def get_final_text(pred_text: str, orig_text: str, do_lower_case: bool, verbose_logging: bool = False): + """Project the tokenized prediction back to the original text. + When we created the data, we kept track of the alignment between original + (whitespace tokenized) tokens and our WordPiece tokenized tokens. So + now `orig_text` contains the span of our original text corresponding to + the span that we predicted. + + However, `orig_text` may contain extra characters that we don't want in + our prediction. + + For example, let's say: + pred_text = steve smith + orig_text = Steve Smith's + + We don't want to return `orig_text` because it contains the extra "'s". + + We don't want to return `pred_text` because it's already been normalized + (the SQuAD eval script also does punctuation stripping/lower casing but + our tokenizer does additional normalization like stripping accent + characters). + + What we really want to return is "Steve Smith". + + Therefore, we have to apply a semi-complicated alignment heuristic + between `pred_text` and `orig_text` to get a character-to-character + alignment. This can fail in certain cases in which case we just return + `orig_text`.""" + + def _strip_spaces(text): + ns_chars = [] + ns_to_s_map = collections.OrderedDict() + for (i, c) in enumerate(text): + if c == " ": + continue + ns_to_s_map[len(ns_chars)] = i + ns_chars.append(c) + ns_text = "".join(ns_chars) + return ns_text, ns_to_s_map + + # We first tokenize `orig_text`, strip whitespace from the result + # and `pred_text`, and check if they are the same length. If they are + # NOT the same length, the heuristic has failed. If they are the same + # length, we assume the characters are one-to-one aligned. + tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + + tok_text = " ".join(tokenizer.tokenize(orig_text)) + + start_position = tok_text.find(pred_text) + if start_position == -1: + if verbose_logging: + logging.warning("Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) + return orig_text + end_position = start_position + len(pred_text) - 1 + + (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) + (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) + + if len(orig_ns_text) != len(tok_ns_text): + if verbose_logging: + logging.warning( + "Length not equal after stripping spaces: '%s' vs '%s'", orig_ns_text, tok_ns_text, + ) + return orig_text + + # We then project the characters in `pred_text` back to `orig_text` using + # the character-to-character alignment. + tok_s_to_ns_map = {} + for (i, tok_index) in tok_ns_to_s_map.items(): + tok_s_to_ns_map[tok_index] = i + + orig_start_position = None + if start_position in tok_s_to_ns_map: + ns_start_position = tok_s_to_ns_map[start_position] + if ns_start_position in orig_ns_to_s_map: + orig_start_position = orig_ns_to_s_map[ns_start_position] + + if orig_start_position is None: + if verbose_logging: + logging.warning("Couldn't map start position") + return orig_text + + orig_end_position = None + if end_position in tok_s_to_ns_map: + ns_end_position = tok_s_to_ns_map[end_position] + if ns_end_position in orig_ns_to_s_map: + orig_end_position = orig_ns_to_s_map[ns_end_position] + + if orig_end_position is None: + if verbose_logging: + logging.warning("Couldn't map end position") + return orig_text + + output_text = orig_text[orig_start_position : (orig_end_position + 1)] + return output_text + + +def f1_score(prediction, ground_truth): + """computes f1 score between prediction and ground truth""" + prediction_tokens = _get_tokens(prediction) + ground_truth_tokens = _get_tokens(ground_truth) + common = collections.Counter(prediction_tokens) & collections.Counter(ground_truth_tokens) + num_same = sum(common.values()) + if len(ground_truth_tokens) == 0 or len(prediction_tokens) == 0: + # If either is no-answer, then F1 is 1 if they agree, 0 otherwise + return int(ground_truth_tokens == prediction_tokens) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(ground_truth_tokens) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def exact_match_score(prediction, ground_truth): + """computes exact match between prediction and ground truth""" + return int(normalize_answer(prediction) == normalize_answer(ground_truth)) + + +def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thresh): + """Applies no answer threshhold""" + new_scores = {} + for qid, s in scores.items(): + pred_na = na_probs[qid] > na_prob_thresh + if pred_na: + new_scores[qid] = float(not qid_to_has_ans[qid]) + else: + new_scores[qid] = s + return new_scores + + +def make_eval_dict(exact_scores, f1_scores, qid_list=None): + """returns dictionary with formatted evaluation scores""" + if not qid_list: + total = len(exact_scores) + return collections.OrderedDict( + [ + ("exact", 100.0 * sum(exact_scores.values()) / total), + ("f1", 100.0 * sum(f1_scores.values()) / total), + ("total", total), + ] + ) + else: + total = len(qid_list) + return collections.OrderedDict( + [ + ("exact", 100.0 * sum(exact_scores[k] for k in qid_list) / total), + ("f1", 100.0 * sum(f1_scores[k] for k in qid_list) / total), + ("total", total), + ] + ) + + +def merge_eval(main_eval, new_eval, prefix): + """merges 2 evaluation dictionaries into the first one by adding prefix as key for name collision handling""" + for k in new_eval: + main_eval["%s_%s" % (prefix, k)] = new_eval[k] + + +def find_all_best_thresh(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): + """ + Find best threshholds to maximize all evaluation metrics. + """ + best_exact, exact_thresh = _find_best_thresh(preds, exact_raw, na_probs, qid_to_has_ans) + best_f1, f1_thresh = _find_best_thresh(preds, f1_raw, na_probs, qid_to_has_ans) + + main_eval["best_exact"] = best_exact + main_eval["best_exact_thresh"] = exact_thresh + main_eval["best_f1"] = best_f1 + main_eval["best_f1_thresh"] = f1_thresh + + +def _find_best_thresh(preds, scores, na_probs, qid_to_has_ans): + """ + Find best threshhold to maximize evaluation metric + """ + num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) + cur_score = num_no_ans + best_score = cur_score + best_thresh = 0.0 + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + for _, qid in enumerate(qid_list): + if qid not in scores: + continue + if qid_to_has_ans[qid]: + diff = scores[qid] + else: + if preds[qid]: + diff = -1 + else: + diff = 0 + cur_score += diff + if cur_score > best_score: + best_score = cur_score + best_thresh = na_probs[qid] + return 100.0 * best_score / len(scores), best_thresh + + +class SquadProcessor(DataProcessor): + """ + Processor for the SQuAD data set. + used by the version 1.1 and version 2.0 of SQuAD, respectively. + + Args: + data_file: data file path + mode: TRAINING_MODE/EVALUATION_MODE/INFERENCE_MODE for creating training/evaluation/inference dataset + """ + + def __init__(self, data_file: str, mode: str): + self.data_file = data_file + self.mode = mode + + def get_examples(self): + if self.data_file is None: + raise ValueError(f"{self.mode} data file is None.") + + with open(self.data_file, "r", encoding="utf-8") as reader: + input_data = json.load(reader)["data"] + return self._create_examples(input_data, set_type=self.mode) + + def _create_examples(self, input_data, set_type): + examples = [] + for entry in tqdm(input_data): + title = entry["title"] + for paragraph in entry["paragraphs"]: + context_text = paragraph["context"] + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + start_position_character = None + answer_text = None + answers = [] + if "is_impossible" in qa: + is_impossible = qa["is_impossible"] + else: + is_impossible = False + + if not is_impossible: + if set_type in [TRAINING_MODE, EVALUATION_MODE]: + answer = qa["answers"][0] + answer_text = answer["text"] + start_position_character = answer["answer_start"] + if set_type == EVALUATION_MODE: + answers = qa["answers"] + + example = SquadExample( + qas_id=qas_id, + question_text=question_text, + context_text=context_text, + answer_text=answer_text, + start_position_character=start_position_character, + title=title, + is_impossible=is_impossible, + answers=answers, + ) + + examples.append(example) + return examples + + +class SquadExample(object): + """ + A single training/test example for the Squad dataset, as loaded from disk. + Args: + qas_id: The example's unique identifier + question_text: The question string + context_text: The context string + answer_text: The answer string + start_position_character: The character position of the start of + the answer, 0 indexed + title: The title of the example + answers: None by default, this is used during evaluation. + Holds answers as well as their start positions. + is_impossible: False by default, set to True if the example has + no possible answer. + """ + + def __init__( + self, + qas_id: str, + question_text: str, + context_text: str, + answer_text: str, + start_position_character: int, + title: str, + answers: List[str] = [], + is_impossible: bool = False, + ): + self.qas_id = qas_id + self.question_text = question_text + self.context_text = context_text + self.answer_text = answer_text + self.title = title + self.is_impossible = is_impossible + self.answers = answers + + self.start_position, self.end_position = 0, 0 + + doc_tokens = [] + char_to_word_offset = [] + prev_is_whitespace = True + + # Split on whitespace so that different tokens + # may be attributed to their original position. + # ex: context_text = ["hi yo"] + # char_to_word_offset = [0, 0, 0, 1, 1] + # doc_tokens = ["hi", "yo"] + for c in self.context_text: + if is_whitespace(c): + prev_is_whitespace = True + else: + if prev_is_whitespace: + doc_tokens.append(c) + else: + doc_tokens[-1] += c + prev_is_whitespace = False + char_to_word_offset.append(len(doc_tokens) - 1) + + self.doc_tokens = doc_tokens + self.char_to_word_offset = char_to_word_offset + + # Start end end positions only has a value during evaluation. + if start_position_character is not None and not is_impossible: + # start_position is index of word, end_position inclusive + self.start_position = char_to_word_offset[start_position_character] + self.end_position = char_to_word_offset[ + min(start_position_character + len(answer_text) - 1, len(char_to_word_offset) - 1) + ] + + +def convert_examples_to_features( + examples: List[object], + tokenizer: object, + max_seq_length: int, + doc_stride: int, + max_query_length: int, + has_groundtruth: bool, +): + """Loads a data file into a list of `InputBatch`s.""" + + unique_id = 1000000000 + + features = [] + for (example_index, example) in enumerate(examples): + query_tokens = tokenizer.text_to_tokens(example.question_text) + + if len(query_tokens) > max_query_length: + query_tokens = query_tokens[0:max_query_length] + + # context: index of token -> index of word + tok_to_orig_index = [] + # context: index of word -> index of first token in token list + orig_to_tok_index = [] + # context without white spaces after tokenization + all_doc_tokens = [] + # doc tokens is word separated context + for (i, token) in enumerate(example.doc_tokens): + orig_to_tok_index.append(len(all_doc_tokens)) + sub_tokens = tokenizer.text_to_tokens(token) + for sub_token in sub_tokens: + tok_to_orig_index.append(i) + all_doc_tokens.append(sub_token) + + # idx of query token start and end in context + tok_start_position = None + tok_end_position = None + if has_groundtruth and example.is_impossible: + tok_start_position = -1 + tok_end_position = -1 + if has_groundtruth and not example.is_impossible: + tok_start_position = orig_to_tok_index[example.start_position] + if example.end_position < len(example.doc_tokens) - 1: + tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 + else: + tok_end_position = len(all_doc_tokens) - 1 + + (tok_start_position, tok_end_position) = _improve_answer_span( + all_doc_tokens, tok_start_position, tok_end_position, tokenizer, example.answer_text + ) + # The -3 accounts for tokenizer.cls_token, tokenizer.sep_token and tokenizer.sep_token + # doc_spans contains all possible contexts options of given length + max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + _DocSpan = collections.namedtuple("DocSpan", ["start", "length"]) + doc_spans = [] + start_offset = 0 + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(_DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, doc_stride) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + tokens = [] + # maps context tokens idx in final input -> word idx in context + token_to_orig_map = {} + token_is_max_context = {} + segment_ids = [] + tokens.append(tokenizer.cls_token) + segment_ids.append(0) + for token in query_tokens: + tokens.append(token) + segment_ids.append(0) + tokens.append(tokenizer.sep_token) + segment_ids.append(0) + + for i in range(doc_span.length): + split_token_index = doc_span.start + i + token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] + + is_max_context = _check_is_max_context(doc_spans, doc_span_index, split_token_index) + token_is_max_context[len(tokens)] = is_max_context + tokens.append(all_doc_tokens[split_token_index]) + segment_ids.append(1) + tokens.append(tokenizer.sep_token) + segment_ids.append(1) + + input_ids = tokenizer.tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. + # Only real tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(tokenizer.pad_id) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + # calculate start and end position in final array + # of tokens in answer if no answer, + # 0 for both pointing to tokenizer.cls_token + start_position = None + end_position = None + if has_groundtruth and not example.is_impossible: + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + out_of_span = False + if not (tok_start_position >= doc_start and tok_end_position <= doc_end): + out_of_span = True + if out_of_span: + start_position = 0 + end_position = 0 + else: + doc_offset = len(query_tokens) + 2 + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + if has_groundtruth and example.is_impossible: + # if our document chunk does not contain + # an annotation we throw it out, since there is nothing + # to predict. + start_position = 0 + end_position = 0 + + if example_index < 1: + logging.info("*** Example ***") + logging.info("unique_id: %s" % (unique_id)) + logging.info("example_index: %s" % (example_index)) + logging.info("doc_span_index: %s" % (doc_span_index)) + logging.info("tokens: %s" % " ".join(tokens)) + logging.info( + "token_to_orig_map: %s" % " ".join(["%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()]) + ) + logging.info( + "token_is_max_context: %s" + % " ".join(["%d:%s" % (x, y) for (x, y) in token_is_max_context.items()]) + ) + logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) + logging.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) + if has_groundtruth and example.is_impossible: + logging.info("impossible example") + if has_groundtruth and not example.is_impossible: + answer_text = " ".join(tokens[start_position : (end_position + 1)]) + logging.info("start_position: %d" % (start_position)) + logging.info("end_position: %d" % (end_position)) + logging.info("answer: %s" % (answer_text)) + if example_index % 100 == 0: + logging.info(f"Finished processing: {example_index}") + features.append( + InputFeatures( + unique_id=unique_id, + example_index=example_index, + doc_span_index=doc_span_index, + tokens=tokens, + token_to_orig_map=token_to_orig_map, + token_is_max_context=token_is_max_context, + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + start_position=start_position, + end_position=end_position, + is_impossible=example.is_impossible, + ) + ) + unique_id += 1 + + return features + + +def _improve_answer_span( + doc_tokens: List[str], input_start: int, input_end: int, tokenizer: object, orig_answer_text: str +): + """Returns tokenized answer spans that + better match the annotated answer.""" + tok_answer_text = " ".join(tokenizer.text_to_tokens(orig_answer_text)) + + for new_start in range(input_start, input_end + 1): + for new_end in range(input_end, new_start - 1, -1): + text_span = " ".join(doc_tokens[new_start : (new_end + 1)]) + if text_span == tok_answer_text: + return (new_start, new_end) + + return (input_start, input_end) + + +def _check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token. + Because of the sliding window approach taken to scoring documents, + a single token can appear in multiple documents. + Example: + Doc: the man went to the store and bought a gallon of milk + Span A: the man went to the + Span B: to the store and bought + Span C: and bought a gallon of + ... + Now the word 'bought' will have two scores from spans B and C. We only + want to consider the score with "maximum context", which we define as + the *minimum* of its left and right context (the *sum* of left and + right context will always be the same, of course). + In the example the maximum context for 'bought' would be span C since + it has 1 left context and 3 right context, while span B has 4 left context + and 0 right context. + Code adapted from the code by the Google AI and HuggingFace. + """ + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__( + self, + unique_id: int, + example_index: int, + doc_span_index: int, + tokens: List[str], + token_to_orig_map: Dict[int, int], + token_is_max_context: Dict[int, bool], + input_ids: List[int], + input_mask: List[int], + segment_ids: List[int], + start_position: Optional[int] = None, + end_position: Optional[int] = None, + is_impossible: Optional[int] = None, + ): + self.unique_id = unique_id + self.example_index = example_index + self.doc_span_index = doc_span_index + self.tokens = tokens + self.token_to_orig_map = token_to_orig_map + self.token_is_max_context = token_is_max_context + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/text_classification/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/text_classification/__init__.py new file mode 100644 index 0000000000..b8bf784072 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/text_classification/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.collections.nlp.data.text_classification.text_classification_dataset import ( + TextClassificationDataset, + calc_class_weights, +) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/text_classification/text_classification_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/text_classification/text_classification_dataset.py new file mode 100644 index 0000000000..42ce06c096 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/text_classification/text_classification_dataset.py @@ -0,0 +1,297 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pickle +import random +from typing import Dict, List, Optional + +import numpy as np +import torch + +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec +from nemo.collections.nlp.data.data_utils.data_preprocessing import ( + fill_class_weights, + get_freq_weights, + get_label_stats, + get_stats, +) +from nemo.collections.nlp.parts.utils_funcs import list2str +from nemo.core.classes import Dataset +from nemo.core.neural_types import ChannelType, LabelsType, MaskType, NeuralType +from nemo.utils import logging +from nemo.utils.env_var_parsing import get_envint + +__all__ = ['TextClassificationDataset', 'calc_class_weights'] + + +class TextClassificationDataset(Dataset): + """A dataset class that converts from raw data to + a dataset that can be used by DataLayerNM. + Args: + input_file: file to sequence + label. + the first line is header (sentence [tab] label) + each line should be [sentence][tab][label] + tokenizer: tokenizer object such as AutoTokenizer + max_seq_length: max sequence length minus 2 for [CLS] and [SEP] + num_samples: number of samples you want to use for the dataset. + If -1, use all dataset. Useful for testing. + shuffle: Shuffles the dataset after loading. + use_cache: Enables caching to use pickle format to store and read data from + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + 'input_ids': NeuralType(('B', 'T'), ChannelType()), + 'segment_ids': NeuralType(('B', 'T'), ChannelType()), + 'input_mask': NeuralType(('B', 'T'), MaskType()), + 'label': NeuralType(('B',), LabelsType()), + } + + def __init__( + self, + tokenizer: TokenizerSpec, + input_file: str = None, + queries: List[str] = None, + max_seq_length: int = -1, + num_samples: int = -1, + shuffle: bool = False, + use_cache: bool = False, + ): + if not input_file and not queries: + raise ValueError("Either input_file or queries should be passed to the text classification dataset.") + + if input_file and not os.path.exists(input_file): + raise FileNotFoundError( + f'Data file `{input_file}` not found! Each line of the data file should contain text sequences, where ' + f'words are separated with spaces and the label separated by [TAB] following this format: ' + f'[WORD][SPACE][WORD][SPACE][WORD][TAB][LABEL]' + ) + + self.input_file = input_file + self.tokenizer = tokenizer + self.max_seq_length = max_seq_length + self.num_samples = num_samples + self.shuffle = shuffle + self.use_cache = use_cache + self.vocab_size = self.tokenizer.vocab_size + self.pad_id = tokenizer.pad_id + + self.features = None + labels, all_sents = [], [] + if input_file: + data_dir, filename = os.path.split(input_file) + vocab_size = getattr(tokenizer, "vocab_size", 0) + tokenizer_name = tokenizer.name + cached_features_file = os.path.join( + data_dir, + f"cached_{filename}_{tokenizer_name}_{max_seq_length}_{vocab_size}_{num_samples}_{self.pad_id}_{shuffle}.pkl", + ) + + if get_envint("LOCAL_RANK", 0) == 0: + if use_cache and os.path.exists(cached_features_file): + logging.warning( + f"Processing of {input_file} is skipped as caching is enabled and a cache file " + f"{cached_features_file} already exists." + ) + logging.warning( + f"You may need to delete the cache file if any of the processing parameters (eg. tokenizer) or " + f"the data are updated." + ) + else: + with open(input_file, "r") as f: + lines = f.readlines() + logging.info(f'Read {len(lines)} examples from {input_file}.') + if num_samples > 0: + lines = lines[:num_samples] + logging.warning( + f"Parameter 'num_samples' is set, so just the first {len(lines)} examples are kept." + ) + + if shuffle: + random.shuffle(lines) + + for index, line in enumerate(lines): + if index % 20000 == 0: + logging.debug(f"Processing line {index}/{len(lines)}") + line_splited = line.strip().split() + try: + label = int(line_splited[-1]) + except ValueError: + logging.debug(f"Skipping line {line}") + continue + labels.append(label) + sent_words = line_splited[:-1] + all_sents.append(sent_words) + verbose = True + + self.features = self.get_features( + all_sents=all_sents, + tokenizer=tokenizer, + max_seq_length=max_seq_length, + labels=labels, + verbose=verbose, + ) + with open(cached_features_file, 'wb') as out_file: + pickle.dump(self.features, out_file, protocol=pickle.HIGHEST_PROTOCOL) + else: + for query in queries: + all_sents.append(query.strip().split()) + labels = [-1] * len(all_sents) + verbose = False + self.features = self.get_features( + all_sents=all_sents, tokenizer=tokenizer, max_seq_length=max_seq_length, labels=labels, verbose=verbose + ) + + # wait until the master process writes to the processed data files + if torch.distributed.is_initialized(): + torch.distributed.barrier() + + if input_file: + with open(cached_features_file, "rb") as input_file: + self.features = pickle.load(input_file) + + def __len__(self): + return len(self.features) + + def __getitem__(self, idx): + return self.features[idx] + + def _collate_fn(self, batch): + """collate batch of input_ids, segment_ids, input_mask, and label + Args: + batch: A list of tuples of (input_ids, segment_ids, input_mask, label). + """ + max_length = 0 + for input_ids, segment_ids, input_mask, label in batch: + if len(input_ids) > max_length: + max_length = len(input_ids) + + padded_input_ids = [] + padded_segment_ids = [] + padded_input_mask = [] + labels = [] + for input_ids, segment_ids, input_mask, label in batch: + if len(input_ids) < max_length: + pad_width = max_length - len(input_ids) + padded_input_ids.append(np.pad(input_ids, pad_width=[0, pad_width], constant_values=self.pad_id)) + padded_segment_ids.append(np.pad(segment_ids, pad_width=[0, pad_width], constant_values=self.pad_id)) + padded_input_mask.append(np.pad(input_mask, pad_width=[0, pad_width], constant_values=self.pad_id)) + else: + padded_input_ids.append(input_ids) + padded_segment_ids.append(segment_ids) + padded_input_mask.append(input_mask) + labels.append(label) + + return ( + torch.LongTensor(padded_input_ids), + torch.LongTensor(padded_segment_ids), + torch.LongTensor(padded_input_mask), + torch.LongTensor(labels), + ) + + @staticmethod + def get_features(all_sents, tokenizer, max_seq_length, labels=None, verbose=True): + """Encode a list of sentences into a list of tuples of (input_ids, segment_ids, input_mask, label).""" + features = [] + sent_lengths = [] + too_long_count = 0 + for sent_id, sent in enumerate(all_sents): + if sent_id % 1000 == 0: + logging.debug(f"Encoding sentence {sent_id}/{len(all_sents)}") + sent_subtokens = [tokenizer.cls_token] + for word in sent: + word_tokens = tokenizer.text_to_tokens(word) + sent_subtokens.extend(word_tokens) + + if max_seq_length > 0 and len(sent_subtokens) + 1 > max_seq_length: + sent_subtokens = sent_subtokens[:max_seq_length] + too_long_count += 1 + + sent_subtokens.append(tokenizer.sep_token) + sent_lengths.append(len(sent_subtokens)) + + input_ids = [tokenizer.tokens_to_ids(t) for t in sent_subtokens] + + # The mask has 1 for real tokens and 0 for padding tokens. + # Only real tokens are attended to. + input_mask = [1] * len(input_ids) + segment_ids = [0] * len(input_ids) + + if verbose and sent_id < 2: + logging.info("*** Example ***") + logging.info(f"example {sent_id}: {sent}") + logging.info("subtokens: %s" % " ".join(sent_subtokens)) + logging.info("input_ids: %s" % list2str(input_ids)) + logging.info("segment_ids: %s" % list2str(segment_ids)) + logging.info("input_mask: %s" % list2str(input_mask)) + logging.info("label: %s" % labels[sent_id] if labels else "**Not Provided**") + + label = labels[sent_id] if labels else -1 + features.append([np.asarray(input_ids), np.asarray(segment_ids), np.asarray(input_mask), label]) + + if max_seq_length > -1 and too_long_count > 0: + logging.warning( + f'Found {too_long_count} out of {len(all_sents)} sentences with more than {max_seq_length} subtokens. ' + f'Truncated long sentences from the end.' + ) + if verbose: + get_stats(sent_lengths) + return features + + +def calc_class_weights(file_path: str, num_classes: int): + """ + iterates over a data file and calculate the weights of each class to be used for class_balancing + Args: + file_path: path to the data file + num_classes: number of classes in the dataset + """ + + if not os.path.exists(file_path): + raise FileNotFoundError(f"Could not find data file {file_path} to calculate the class weights!") + + with open(file_path, 'r') as f: + input_lines = f.readlines() + + labels = [] + for input_line in input_lines: + parts = input_line.strip().split() + try: + label = int(parts[-1]) + except ValueError: + raise ValueError( + f'No numerical labels found for {file_path}. Labels should be integers and separated by [TAB] at the end of each line.' + ) + labels.append(label) + + logging.info(f'Calculating stats of {file_path}...') + total_sents, sent_label_freq, max_id = get_label_stats(labels, f'{file_path}_sentence_stats.tsv', verbose=False) + if max_id >= num_classes: + raise ValueError(f'Found an invalid label in {file_path}! Labels should be from [0, num_classes-1].') + + class_weights_dict = get_freq_weights(sent_label_freq) + + logging.info(f'Total Sentence: {total_sents}') + logging.info(f'Sentence class frequencies: {sent_label_freq}') + + logging.info(f'Class Weights: {class_weights_dict}') + class_weights = fill_class_weights(weights=class_weights_dict, max_id=num_classes - 1) + + return class_weights diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/__init__.py new file mode 100644 index 0000000000..9e32500719 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py new file mode 100644 index 0000000000..3a4dfbc9cc --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py @@ -0,0 +1,466 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = ['BertPunctuationCapitalizationDataset', 'BertPunctuationCapitalizationInferDataset'] + +import itertools +import os +import pickle +from typing import Dict, List, Optional + +import numpy as np +import torch + +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec +from nemo.collections.nlp.data.data_utils.data_preprocessing import get_label_stats, get_stats +from nemo.core.classes import Dataset +from nemo.core.neural_types import ChannelType, LabelsType, MaskType, NeuralType +from nemo.utils import logging + + +def get_features( + queries: List[str], + max_seq_length: int, + tokenizer: TokenizerSpec, + punct_label_ids: dict = None, + capit_label_ids: dict = None, + pad_label: str = 'O', + punct_labels_lines=None, + capit_labels_lines=None, + ignore_extra_tokens=False, + ignore_start_end: Optional[bool] = False, +): + """ + Processes the data and returns features. + + Args: + queries: text sequences + max_seq_length: max sequence length minus 2 for [CLS] and [SEP] + tokenizer: such as AutoTokenizer + pad_label: pad value use for labels. By default, it's the neutral label. + punct_label_ids: dict to map punctuation labels to label ids. + Starts with pad_label->0 and then increases in alphabetical order. + Required for training and evaluation, not needed for inference. + capit_label_ids: dict to map labels to label ids. Starts + with pad_label->0 and then increases in alphabetical order. + Required for training and evaluation, not needed for inference. + punct_labels: list of labels for every word in a sequence (str) + capit_labels: list of labels for every word in a sequence (str) + ignore_extra_tokens: whether to ignore extra tokens in the loss_mask + ignore_start_end: whether to ignore bos and eos tokens in the loss_mask + + Returns: + all_input_ids: input ids for all tokens + all_segment_ids: token type ids + all_input_mask: attention mask to use for BERT model + all_subtokens_mask: masks out all subwords besides the first one + all_loss_mask: loss mask to mask out tokens during training + punct_all_labels: all labels for punctuation task (ints) + capit_all_labels: all labels for capitalization task (ints) + punct_label_ids: label (str) to id (int) map for punctuation task + capit_label_ids: label (str) to id (int) map for capitalization task + """ + all_subtokens = [] + all_loss_mask = [] + all_subtokens_mask = [] + all_segment_ids = [] + all_input_ids = [] + all_input_mask = [] + sent_lengths = [] + punct_all_labels = [] + capit_all_labels = [] + with_label = False + + if punct_labels_lines and capit_labels_lines: + with_label = True + + for i, query in enumerate(queries): + words = query.strip().split() + + # add bos token + subtokens = [tokenizer.cls_token] + loss_mask = [1 - ignore_start_end] + subtokens_mask = [0] + if with_label: + pad_id = punct_label_ids[pad_label] + punct_labels = [pad_id] + punct_query_labels = [punct_label_ids[lab] for lab in punct_labels_lines[i]] + + capit_labels = [pad_id] + capit_query_labels = [capit_label_ids[lab] for lab in capit_labels_lines[i]] + + for j, word in enumerate(words): + word_tokens = tokenizer.text_to_tokens(word) + subtokens.extend(word_tokens) + + loss_mask.append(1) + loss_mask.extend([int(not ignore_extra_tokens)] * (len(word_tokens) - 1)) + + subtokens_mask.append(1) + subtokens_mask.extend([0] * (len(word_tokens) - 1)) + + if with_label: + punct_labels.extend([punct_query_labels[j]] * len(word_tokens)) + capit_labels.extend([capit_query_labels[j]] * len(word_tokens)) + + # add eos token + subtokens.append(tokenizer.sep_token) + loss_mask.append(1 - ignore_start_end) + subtokens_mask.append(0) + sent_lengths.append(len(subtokens)) + all_subtokens.append(subtokens) + all_loss_mask.append(loss_mask) + all_subtokens_mask.append(subtokens_mask) + all_input_mask.append([1] * len(subtokens)) + + if with_label: + punct_labels.append(pad_id) + punct_all_labels.append(punct_labels) + capit_labels.append(pad_id) + capit_all_labels.append(capit_labels) + + max_seq_length = min(max_seq_length, max(sent_lengths)) + logging.info(f'Max length: {max_seq_length}') + get_stats(sent_lengths) + too_long_count = 0 + + for i, subtokens in enumerate(all_subtokens): + if len(subtokens) > max_seq_length: + subtokens = [tokenizer.cls_token] + subtokens[-max_seq_length + 1 :] + all_input_mask[i] = [1] + all_input_mask[i][-max_seq_length + 1 :] + all_loss_mask[i] = [int(not ignore_start_end)] + all_loss_mask[i][-max_seq_length + 1 :] + all_subtokens_mask[i] = [0] + all_subtokens_mask[i][-max_seq_length + 1 :] + + if with_label: + punct_all_labels[i] = [pad_id] + punct_all_labels[i][-max_seq_length + 1 :] + capit_all_labels[i] = [pad_id] + capit_all_labels[i][-max_seq_length + 1 :] + too_long_count += 1 + + all_input_ids.append(tokenizer.tokens_to_ids(subtokens)) + + if len(subtokens) < max_seq_length: + extra = max_seq_length - len(subtokens) + all_input_ids[i] = all_input_ids[i] + [0] * extra + all_loss_mask[i] = all_loss_mask[i] + [0] * extra + all_subtokens_mask[i] = all_subtokens_mask[i] + [0] * extra + all_input_mask[i] = all_input_mask[i] + [0] * extra + + if with_label: + punct_all_labels[i] = punct_all_labels[i] + [pad_id] * extra + capit_all_labels[i] = capit_all_labels[i] + [pad_id] * extra + + all_segment_ids.append([0] * max_seq_length) + + logging.info(f'{too_long_count} are longer than {max_seq_length}') + + for i in range(min(len(all_input_ids), 5)): + logging.info("*** Example ***") + logging.info("i: %s" % (i)) + logging.info("subtokens: %s" % " ".join(list(map(str, all_subtokens[i])))) + logging.info("loss_mask: %s" % " ".join(list(map(str, all_loss_mask[i])))) + logging.info("input_mask: %s" % " ".join(list(map(str, all_input_mask[i])))) + logging.info("subtokens_mask: %s" % " ".join(list(map(str, all_subtokens_mask[i])))) + if with_label: + logging.info("punct_labels: %s" % " ".join(list(map(str, punct_all_labels[i])))) + logging.info("capit_labels: %s" % " ".join(list(map(str, capit_all_labels[i])))) + + return ( + all_input_ids, + all_segment_ids, + all_input_mask, + all_subtokens_mask, + all_loss_mask, + punct_all_labels, + capit_all_labels, + punct_label_ids, + capit_label_ids, + ) + + +class BertPunctuationCapitalizationDataset(Dataset): + """ + Creates dataset to use during training for punctuaion and capitalization tasks with a pretrained model. + For dataset to use during inference without labels, see BertPunctuationCapitalizationInferDataset. + + Args: + text_file: file to sequences, each line should a sentence, no header. + label_file: file to labels, each line corresponds to word labels for a sentence in the text_file. No header. + max_seq_length: max sequence length minus 2 for [CLS] and [SEP] + tokenizer: such as AutoTokenizer + num_samples: number of samples you want to use for the dataset. + If -1, use all dataset. Useful for testing. + pad_label: pad value use for labels. + by default, it's the neutral label. + punct_label_ids and capit_label_ids (dict): + dict to map labels to label ids. + Starts with pad_label->0 and then increases in alphabetical order + For dev set use label_ids generated during training to support + cases when not all labels are present in the dev set. + For training set label_ids should be None or loaded from cache + ignore_extra_tokens: whether to ignore extra tokens in the loss_mask + ignore_start_end: whether to ignore bos and eos tokens in the loss_mask + use_cache: whether to use processed data cache or not + get_label_frequencies: whether to generate label frequencies + punct_label_ids_file and capit_label_ids_file: name of the files to save in .nemo + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. """ + return { + 'input_ids': NeuralType(('B', 'T'), ChannelType()), + 'segment_ids': NeuralType(('B', 'T'), ChannelType()), + 'input_mask': NeuralType(('B', 'T'), MaskType()), + 'subtokens_mask': NeuralType(('B', 'T'), MaskType()), + 'loss_mask': NeuralType(('B', 'T'), MaskType()), + 'punct_labels': NeuralType(('B', 'T'), LabelsType()), + 'capit_labels': NeuralType(('B', 'T'), LabelsType()), + } + + def __init__( + self, + text_file: str, + label_file: str, + max_seq_length: int, + tokenizer: TokenizerSpec, + num_samples: int = -1, + pad_label: str = 'O', + punct_label_ids: Dict[str, int] = None, + capit_label_ids: Dict[str, int] = None, + ignore_extra_tokens: bool = False, + ignore_start_end: bool = False, + use_cache: bool = True, + get_label_frequencies: bool = False, + punct_label_ids_file: str = 'punct_label_ids.csv', + capit_label_ids_file: str = 'capit_label_ids.csv', + ): + """ Initializes BertPunctuationCapitalizationDataset. """ + + if not (os.path.exists(text_file) and os.path.exists(label_file)): + raise FileNotFoundError( + f'{text_file} or {label_file} not found. The data should be splitted into 2 files: text.txt and \ + labels.txt. Each line of the text.txt file contains text sequences, where words are separated with \ + spaces. The labels.txt file contains corresponding labels for each word in text.txt, the labels are \ + separated with spaces. Each line of the files should follow the format: \ + [WORD] [SPACE] [WORD] [SPACE] [WORD] (for text.txt) and \ + [LABEL] [SPACE] [LABEL] [SPACE] [LABEL] (for labels.txt).' + ) + + # Cache features + data_dir = os.path.dirname(text_file) + filename = os.path.basename(text_file) + + if not filename.endswith('.txt'): + raise ValueError("{text_file} should have extension .txt") + + filename = filename[:-4] + vocab_size = getattr(tokenizer, "vocab_size", 0) + features_pkl = os.path.join( + data_dir, + "cached_{}_{}_{}_{}_{}".format( + filename, tokenizer.name, str(max_seq_length), str(vocab_size), str(num_samples) + ), + ) + + self.punct_label_ids_file = os.path.join(data_dir, punct_label_ids_file) + self.capit_label_ids_file = os.path.join(data_dir, capit_label_ids_file) + + master_device = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + cache_files_exist = ( + os.path.exists(features_pkl) + and os.path.exists(self.punct_label_ids_file) + and os.path.exists(self.capit_label_ids_file) + ) + features = None + if master_device and not (cache_files_exist and use_cache): + if num_samples == 0: + raise ValueError("num_samples has to be positive", num_samples) + logging.info(f'Processing {text_file}') + with open(text_file, 'r') as f: + text_lines = f.readlines() + + # Collect all possible labels + punct_unique_labels = set() + capit_unique_labels = set() + punct_labels_lines = [] + capit_labels_lines = [] + with open(label_file, 'r') as f: + for line in f: + line = line.strip().split() + + # extract punctuation and capitalization labels + punct_line, capit_line = zip(*line) + punct_labels_lines.append(punct_line) + capit_labels_lines.append(capit_line) + + punct_unique_labels.update(punct_line) + capit_unique_labels.update(capit_line) + + if len(punct_labels_lines) != len(text_lines): + raise ValueError("Labels file should contain labels for every word") + + dataset = list(zip(text_lines, punct_labels_lines, capit_labels_lines)) + + if num_samples > 0: + dataset = dataset[:num_samples] + + dataset = list(zip(*dataset)) + text_lines = dataset[0] + punct_labels_lines = dataset[1] + capit_labels_lines = dataset[2] + + # for dev/test sets use label mapping from training set + if punct_label_ids: + if len(punct_label_ids) != len(punct_unique_labels): + logging.info( + 'Not all labels from the specified' + + 'label_ids dictionary are present in the' + + 'current dataset. Using the provided' + + 'label_ids dictionary.' + ) + else: + logging.info('Using the provided label_ids dictionary.') + else: + logging.info( + 'Creating a new label to label_id dictionary.' + + ' It\'s recommended to use label_ids generated' + + ' during training for dev/test sets to avoid' + + ' errors if some labels are not' + + ' present in the dev/test sets.' + + ' For training set label_ids should be None.' + ) + + def create_label_ids(unique_labels, pad_label=pad_label): + label_ids = {pad_label: 0} + if pad_label in unique_labels: + unique_labels.remove(pad_label) + for label in sorted(unique_labels): + label_ids[label] = len(label_ids) + return label_ids + + punct_label_ids = create_label_ids(punct_unique_labels) + capit_label_ids = create_label_ids(capit_unique_labels) + + self._save_label_ids(punct_label_ids, self.punct_label_ids_file) + self._save_label_ids(capit_label_ids, self.capit_label_ids_file) + + features = get_features( + text_lines, + max_seq_length, + tokenizer, + pad_label=pad_label, + punct_labels_lines=punct_labels_lines, + capit_labels_lines=capit_labels_lines, + punct_label_ids=punct_label_ids, + capit_label_ids=capit_label_ids, + ignore_extra_tokens=ignore_extra_tokens, + ignore_start_end=ignore_start_end, + ) + + pickle.dump(features, open(features_pkl, "wb")) + logging.info(f'Features saved to {features_pkl}') + + # wait until the master process writes to the processed data files + if torch.distributed.is_initialized(): + torch.distributed.barrier() + + if features is None: + features = pickle.load(open(features_pkl, 'rb')) + logging.info(f'Features restored from {features_pkl}') + + self.all_input_ids = features[0] + self.all_segment_ids = features[1] + self.all_input_mask = features[2] + self.all_subtokens_mask = features[3] + self.all_loss_mask = features[4] + self.punct_all_labels = features[5] + self.capit_all_labels = features[6] + self.punct_label_ids = features[7] + self.capit_label_ids = features[8] + + if get_label_frequencies: + self.punct_label_frequencies = self._calculate_label_frequencies(self.punct_all_labels, data_dir, 'punct') + self.capit_label_frequencies = self._calculate_label_frequencies(self.capit_all_labels, data_dir, 'capit') + + def _calculate_label_frequencies(self, all_labels: List[int], data_dir: str, name: str) -> Dict[str, float]: + """ Calculates labels frequencies """ + merged_labels = itertools.chain.from_iterable(all_labels) + logging.info('Three most popular labels') + _, label_frequencies, _ = get_label_stats(merged_labels, data_dir + '/label_count_' + name + '.tsv') + return label_frequencies + + def _save_label_ids(self, label_ids: Dict[str, int], filename: str) -> None: + """ Saves label ids map to a file """ + with open(filename, 'w') as out: + labels, _ = zip(*sorted(label_ids.items(), key=lambda x: x[1])) + out.write('\n'.join(labels)) + logging.info(f'Labels: {label_ids}') + logging.info(f'Labels mapping saved to : {out.name}') + + def __len__(self): + return len(self.all_input_ids) + + def __getitem__(self, idx): + return ( + np.array(self.all_input_ids[idx]), + np.array(self.all_segment_ids[idx]), + np.array(self.all_input_mask[idx], dtype=np.long), + np.array(self.all_subtokens_mask[idx]), + np.array(self.all_loss_mask[idx]), + np.array(self.punct_all_labels[idx]), + np.array(self.capit_all_labels[idx]), + ) + + +class BertPunctuationCapitalizationInferDataset(Dataset): + """ + Creates dataset to use during inference for punctuation and capitalization tasks with a pretrained model. + For dataset to use during training with labels, see BertPunctuationCapitalizationDataset. + + Args: + queries file to sequences, each line should a sentence, no header. + max_seq_length: max sequence length minus 2 for [CLS] and [SEP] + tokenizer: such as AutoTokenizer + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + 'input_ids': NeuralType(('B', 'T'), ChannelType()), + 'segment_ids': NeuralType(('B', 'T'), ChannelType()), + 'input_mask': NeuralType(('B', 'T'), MaskType()), + 'subtokens_mask': NeuralType(('B', 'T'), MaskType()), + } + + def __init__(self, queries: List[str], max_seq_length: int, tokenizer: TokenizerSpec): + """ Initializes BertPunctuationCapitalizationInferDataset. """ + features = get_features(queries=queries, max_seq_length=max_seq_length, tokenizer=tokenizer) + + self.all_input_ids = features[0] + self.all_segment_ids = features[1] + self.all_input_mask = features[2] + self.all_subtokens_mask = features[3] + + def __len__(self): + return len(self.all_input_ids) + + def __getitem__(self, idx): + return ( + np.array(self.all_input_ids[idx]), + np.array(self.all_segment_ids[idx]), + np.array(self.all_input_mask[idx], dtype=np.float32), + np.array(self.all_subtokens_mask[idx]), + ) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/token_classification_dataset.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/token_classification_dataset.py new file mode 100644 index 0000000000..d102ff78e0 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/token_classification_dataset.py @@ -0,0 +1,338 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utility functions for Token Classification NLP tasks +Some parts of this code were adapted from the HuggingFace library at +https://github.com/huggingface/pytorch-pretrained-BERT +""" + +import os +import pickle +from typing import Dict, List, Optional + +import numpy as np +import torch + +from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec +from nemo.collections.nlp.data.data_utils.data_preprocessing import get_stats +from nemo.core.classes import Dataset +from nemo.core.neural_types import ChannelType, LabelsType, MaskType, NeuralType +from nemo.utils import logging + +__all__ = ['BertTokenClassificationDataset', 'BertTokenClassificationInferDataset'] + + +def get_features( + queries: List[str], + tokenizer: TokenizerSpec, + max_seq_length: int = -1, + label_ids: dict = None, + pad_label: str = 'O', + raw_labels: List[str] = None, + ignore_extra_tokens: bool = False, + ignore_start_end: bool = False, +): + """ + Processes the data and returns features. + Args: + queries: text sequences + tokenizer: such as AutoTokenizer + max_seq_length: max sequence length minus 2 for [CLS] and [SEP], when -1 - use the max len from the data + pad_label: pad value use for labels. By default, it's the neutral label. + raw_labels: list of labels for every word in a sequence + label_ids: dict to map labels to label ids. + Starts with pad_label->0 and then increases in alphabetical order. + Required for training and evaluation, not needed for inference. + ignore_extra_tokens: whether to ignore extra tokens in the loss_mask + ignore_start_end: whether to ignore bos and eos tokens in the loss_mask + """ + all_subtokens = [] + all_loss_mask = [] + all_subtokens_mask = [] + all_segment_ids = [] + all_input_ids = [] + all_input_mask = [] + sent_lengths = [] + all_labels = [] + with_label = False + + if raw_labels is not None: + with_label = True + + for i, query in enumerate(queries): + words = query.strip().split() + + # add bos token + subtokens = [tokenizer.cls_token] + loss_mask = [1 - ignore_start_end] + subtokens_mask = [0] + if with_label: + pad_id = label_ids[pad_label] + labels = [pad_id] + query_labels = [label_ids[lab] for lab in raw_labels[i]] + + for j, word in enumerate(words): + word_tokens = tokenizer.text_to_tokens(word) + subtokens.extend(word_tokens) + + loss_mask.append(1) + loss_mask.extend([int(not ignore_extra_tokens)] * (len(word_tokens) - 1)) + + subtokens_mask.append(1) + subtokens_mask.extend([0] * (len(word_tokens) - 1)) + + if with_label: + labels.extend([query_labels[j]] * len(word_tokens)) + # add eos token + subtokens.append(tokenizer.sep_token) + loss_mask.append(1 - ignore_start_end) + subtokens_mask.append(0) + sent_lengths.append(len(subtokens)) + all_subtokens.append(subtokens) + all_loss_mask.append(loss_mask) + all_subtokens_mask.append(subtokens_mask) + all_input_mask.append([1] * len(subtokens)) + + if with_label: + labels.append(pad_id) + all_labels.append(labels) + + max_seq_length_data = max(sent_lengths) + max_seq_length = min(max_seq_length, max_seq_length_data) if max_seq_length > 0 else max_seq_length_data + logging.info(f'Setting Max Seq length to: {max_seq_length}') + get_stats(sent_lengths) + too_long_count = 0 + + for i, subtokens in enumerate(all_subtokens): + if len(subtokens) > max_seq_length: + subtokens = [tokenizer.cls_token] + subtokens[-max_seq_length + 1 :] + all_input_mask[i] = [1] + all_input_mask[i][-max_seq_length + 1 :] + all_loss_mask[i] = [int(not ignore_start_end)] + all_loss_mask[i][-max_seq_length + 1 :] + all_subtokens_mask[i] = [0] + all_subtokens_mask[i][-max_seq_length + 1 :] + + if with_label: + all_labels[i] = [pad_id] + all_labels[i][-max_seq_length + 1 :] + too_long_count += 1 + + all_input_ids.append(tokenizer.tokens_to_ids(subtokens)) + + if len(subtokens) < max_seq_length: + extra = max_seq_length - len(subtokens) + all_input_ids[i] = all_input_ids[i] + [0] * extra + all_loss_mask[i] = all_loss_mask[i] + [0] * extra + all_subtokens_mask[i] = all_subtokens_mask[i] + [0] * extra + all_input_mask[i] = all_input_mask[i] + [0] * extra + + if with_label: + all_labels[i] = all_labels[i] + [pad_id] * extra + + all_segment_ids.append([0] * max_seq_length) + + logging.warning(f'{too_long_count} are longer than {max_seq_length}') + + for i in range(min(len(all_input_ids), 1)): + logging.info("*** Example ***") + logging.info("i: %s", i) + logging.info("subtokens: %s", " ".join(list(map(str, all_subtokens[i])))) + logging.info("loss_mask: %s", " ".join(list(map(str, all_loss_mask[i])))) + logging.info("input_mask: %s", " ".join(list(map(str, all_input_mask[i])))) + logging.info("subtokens_mask: %s", " ".join(list(map(str, all_subtokens_mask[i])))) + if with_label: + logging.info("labels: %s", " ".join(list(map(str, all_labels[i])))) + return (all_input_ids, all_segment_ids, all_input_mask, all_subtokens_mask, all_loss_mask, all_labels) + + +class BertTokenClassificationDataset(Dataset): + """ + Creates dataset to use during training for token classification tasks with a pretrained model. + + Converts from raw data to an instance that can be used by Dataloader. + For dataset to use during inference without labels, see BertTokenClassificationInferDataset. + + Args: + text_file: file to sequences, each line should a sentence, no header. + label_file: file to labels, each line corresponds to word labels for a sentence in the text_file. No header. + max_seq_length: max sequence length minus 2 for [CLS] and [SEP] + tokenizer: such as AutoTokenizer + num_samples: number of samples you want to use for the dataset. + If -1, use all dataset. Useful for testing. + pad_label: pad value use for labels. By default, it's the neutral label. + label_ids: label_ids (dict): dict to map labels to label ids. + Starts with pad_label->0 and then increases in alphabetical order + For dev set use label_ids generated during training to support + cases when not all labels are present in the dev set. + For training set label_ids should be None. + ignore_extra_tokens: whether to ignore extra tokens in the loss_mask + ignore_start_end: whether to ignore bos and eos tokens in the loss_mask + use_cache: whether to use processed data cache or not + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + 'input_ids': NeuralType(('B', 'T'), ChannelType()), + 'segment_ids': NeuralType(('B', 'T'), ChannelType()), + 'input_mask': NeuralType(('B', 'T'), MaskType()), + 'subtokens_mask': NeuralType(('B', 'T'), MaskType()), + 'loss_mask': NeuralType(('B', 'T'), MaskType()), + 'labels': NeuralType(('B', 'T'), LabelsType()), + } + + def __init__( + self, + text_file: str, + label_file: str, + max_seq_length: int, + tokenizer: TokenizerSpec, + num_samples: int = -1, + pad_label: str = 'O', + label_ids: Dict[str, int] = None, + ignore_extra_tokens: bool = False, + ignore_start_end: bool = False, + use_cache: bool = True, + ): + """ Initializes BertTokenClassificationDataset. """ + + data_dir = os.path.dirname(text_file) + filename = os.path.basename(text_file) + + if not filename.endswith('.txt'): + raise ValueError("{text_file} should have extension .txt") + + vocab_size = getattr(tokenizer, "vocab_size", 0) + features_pkl = os.path.join( + data_dir, + "cached_{}_{}_{}_{}_{}".format( + filename, tokenizer.name, str(max_seq_length), str(vocab_size), str(num_samples) + ), + ) + + master_device = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + features = None + if master_device and (not use_cache or not os.path.exists(features_pkl)): + if num_samples == 0: + raise ValueError("num_samples has to be positive", num_samples) + + with open(text_file, 'r') as f: + text_lines = f.readlines() + + labels_lines = [] + with open(label_file, 'r') as f: + for line in f: + line = line.strip().split() + labels_lines.append(line) + + if len(labels_lines) != len(text_lines): + raise ValueError("Labels file should contain labels for every word") + + if num_samples > 0: + dataset = list(zip(text_lines, labels_lines)) + dataset = dataset[:num_samples] + + dataset = list(zip(*dataset)) + text_lines = dataset[0] + labels_lines = dataset[1] + + features = get_features( + queries=text_lines, + max_seq_length=max_seq_length, + tokenizer=tokenizer, + pad_label=pad_label, + raw_labels=labels_lines, + label_ids=label_ids, + ignore_extra_tokens=ignore_extra_tokens, + ignore_start_end=ignore_start_end, + ) + + pickle.dump(features, open(features_pkl, "wb")) + logging.info(f'features saved to {features_pkl}') + + # wait until the master process writes to the processed data files + if torch.distributed.is_initialized(): + torch.distributed.barrier() + + if features is None: + features = pickle.load(open(features_pkl, 'rb')) + logging.info(f'features restored from {features_pkl}') + + self.all_input_ids = features[0] + self.all_segment_ids = features[1] + self.all_input_mask = features[2] + self.all_subtokens_mask = features[3] + self.all_loss_mask = features[4] + self.all_labels = features[5] + + def __len__(self): + return len(self.all_input_ids) + + def __getitem__(self, idx): + return ( + np.array(self.all_input_ids[idx]), + np.array(self.all_segment_ids[idx]), + np.array(self.all_input_mask[idx], dtype=np.long), + np.array(self.all_subtokens_mask[idx]), + np.array(self.all_loss_mask[idx]), + np.array(self.all_labels[idx]), + ) + + +class BertTokenClassificationInferDataset(Dataset): + """ + Creates dataset to use during inference for token classification tasks with a pretrained model. + For dataset to use during training with labels, see BertTokenClassificationDataset. + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + 'input_ids': NeuralType(('B', 'T'), ChannelType()), + 'segment_ids': NeuralType(('B', 'T'), ChannelType()), + 'input_mask': NeuralType(('B', 'T'), MaskType()), + 'subtokens_mask': NeuralType(('B', 'T'), MaskType()), + } + + def __init__( + self, queries: List[str], max_seq_length: int, tokenizer: TokenizerSpec, + ): + """ + Initializes BertTokenClassificationInferDataset + Args: + queries: text sequences + max_seq_length: max sequence length minus 2 for [CLS] and [SEP] + tokenizer: such as AutoTokenizer + """ + features = get_features(queries=queries, max_seq_length=max_seq_length, tokenizer=tokenizer) + + self.all_input_ids = features[0] + self.all_segment_ids = features[1] + self.all_input_mask = features[2] + self.all_subtokens_mask = features[3] + + def __len__(self): + return len(self.all_input_ids) + + def __getitem__(self, idx): + return ( + np.array(self.all_input_ids[idx]), + np.array(self.all_segment_ids[idx]), + np.array(self.all_input_mask[idx], dtype=np.long), + np.array(self.all_subtokens_mask[idx]), + ) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/token_classification_utils.py b/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/token_classification_utils.py new file mode 100644 index 0000000000..828ef1180e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/data/token_classification/token_classification_utils.py @@ -0,0 +1,118 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pickle +from typing import Dict + +from nemo.collections.nlp.data.data_utils.data_preprocessing import ( + fill_class_weights, + get_freq_weights, + get_label_stats, +) +from nemo.utils import logging + +__all__ = ['get_label_ids'] + + +def get_label_ids( + label_file: str, + is_training: bool = False, + pad_label: str = 'O', + label_ids_dict: Dict[str, int] = None, + get_weights: bool = True, + class_labels_file_artifact='label_ids.csv', +): + """ + Generates str to int labels mapping for training data or checks correctness of the label_ids_dict + file for non-training files or if label_ids_dict is specified + + Args: + label_file: the path of the label file to process + is_training: indicates whether the label_file is used for training + pad_label: token used for padding + label_ids_dict: str label name to int ids mapping. Required for non-training data. + If specified, the check that all labels from label_file are present in label_ids_dict will be performed. + For training data, if label_ids_dict is None, a new mapping will be generated from label_file. + get_weights: set to True to calculate class weights, required for Weighted Loss. + class_labels_file_artifact: name of the file to save in .nemo + """ + if not os.path.exists(label_file): + raise ValueError(f'File {label_file} was not found.') + + logging.info(f'Processing {label_file}') + if not is_training and label_ids_dict is None: + raise ValueError( + f'For non training data, label_ids_dict created during preprocessing of the training data ' + f'should be provided' + ) + + # collect all labels from the label_file + data_dir = os.path.dirname(label_file) + unique_labels = set(pad_label) + all_labels = [] + with open(label_file, 'r') as f: + for line in f: + line = line.strip().split() + all_labels.extend(line) + unique_labels.update(line) + + # check that all labels from label_file are present in the specified label_ids_dict + # or generate label_ids_dict from data (for training only) + if label_ids_dict: + logging.info(f'Using provided labels mapping {label_ids_dict}') + for name in unique_labels: + if name not in label_ids_dict: + raise ValueError(f'{name} class from {label_file} not found in the provided mapping: {label_ids_dict}') + else: + label_ids_dict = {pad_label: 0} + if pad_label in unique_labels: + unique_labels.remove(pad_label) + for label in sorted(unique_labels): + label_ids_dict[label] = len(label_ids_dict) + + label_ids_filename = os.path.join(data_dir, class_labels_file_artifact) + if is_training: + with open(label_ids_filename, 'w') as f: + labels, _ = zip(*sorted(label_ids_dict.items(), key=lambda x: x[1])) + f.write('\n'.join(labels)) + logging.info(f'Labels mapping {label_ids_dict} saved to : {label_ids_filename}') + + # calculate label statistics + base_name = os.path.splitext(os.path.basename(label_file))[0] + stats_file = os.path.join(data_dir, f'{base_name}_label_stats.tsv') + if os.path.exists(stats_file) and not is_training and not get_weights: + logging.info(f'{stats_file} found, skipping stats calculation.') + else: + all_labels = [label_ids_dict[label] for label in all_labels] + logging.info(f'Three most popular labels in {label_file}:') + total_labels, label_frequencies, max_id = get_label_stats(all_labels, stats_file) + logging.info(f'Total labels: {total_labels}. Label frequencies - {label_frequencies}') + + if get_weights: + class_weights_pkl = os.path.join(data_dir, f'{base_name}_weights.p') + if os.path.exists(class_weights_pkl): + class_weights = pickle.load(open(class_weights_pkl, 'rb')) + logging.info(f'Class weights restored from {class_weights_pkl}') + else: + class_weights_dict = get_freq_weights(label_frequencies) + logging.info(f'Class Weights: {class_weights_dict}') + class_weights = fill_class_weights(class_weights_dict, max_id) + + pickle.dump(class_weights, open(class_weights_pkl, "wb")) + logging.info(f'Class weights saved to {class_weights_pkl}') + else: + class_weights = None + + return label_ids_dict, label_ids_filename, class_weights diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/losses/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/losses/__init__.py new file mode 100644 index 0000000000..36eaed2511 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/losses/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.losses.sgd_loss import SGDDialogueStateLoss diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/losses/sgd_loss.py b/models/tacotron2/NeMo/nemo/collections/nlp/losses/sgd_loss.py new file mode 100644 index 0000000000..0a109a609d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/losses/sgd_loss.py @@ -0,0 +1,218 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +This file contains code artifacts adapted from the original implementation: +https://github.com/google-research/google-research/blob/master/schema_guided_dst/baseline/train_and_predict.py +''' + +import torch + +from nemo.core.classes import Loss, Typing, typecheck +from nemo.core.neural_types import ChannelType, LabelsType, LogitsType, LossType, NeuralType +from nemo.utils import logging + +__all__ = ['SGDDialogueStateLoss'] + + +class SGDDialogueStateLoss(Loss, Typing): + """ + Neural module which implements loss for SGD model. + """ + + @property + def input_types(self): + """Returns definitions of module input ports. + logit_intent_status: Output of SGD model + intent_status: intent label + logit_req_slot_status: Output of SGD model + requested_slot_status: Takes value 1 if the corresponding slot is requested, 0 otherwise + logit_cat_slot_status: Output of SGD model + categorical_slot_status: The status of each categorical slot in the service + logit_cat_slot_value_status: Output of SGD model + categorical_slot_value_status: Takes value 1 if the corresponding slot value is correct, 0 otherwise + logit_noncat_slot_status: Output of SGD model + noncategorical_slot_status: The status of each noncategorical slot in the service\ + logit_spans: Output of SGD model + noncategorical_slot_value_start: The index of the starting subword corresponding to the slot span for a non-categorical slot value + noncategorical_slot_value_end: The index of the ending (inclusive) subword corresponding to the slot span for a non-categorical slot value + task_mask: Mask contains 1 if its the current task, 0 otherwise + """ + return { + "logit_intent_status": NeuralType(('B', 'T'), LogitsType()), + "intent_status": NeuralType(('B'), LabelsType()), + "logit_req_slot_status": NeuralType(('B', 'T'), LogitsType()), + "requested_slot_status": NeuralType(('B'), LabelsType()), + "logit_cat_slot_status": NeuralType(('B', 'T'), LogitsType()), + "categorical_slot_status": NeuralType(('B'), LabelsType()), + "logit_cat_slot_value_status": NeuralType(('B', 'T'), LogitsType()), + "categorical_slot_value_status": NeuralType(('B'), LabelsType()), + "logit_noncat_slot_status": NeuralType(('B', 'T'), LogitsType()), + "noncategorical_slot_status": NeuralType(('B'), LabelsType()), + "logit_spans": NeuralType(('B', 'T', 'D'), LogitsType()), + "noncategorical_slot_value_start": NeuralType(('B'), LabelsType()), + "noncategorical_slot_value_end": NeuralType(('B'), LabelsType()), + "task_mask": NeuralType(('B', 'T'), ChannelType()), + } + + @property + def output_types(self): + """ + Returns definitions of module output ports. + loss: + NeuralType(None) + """ + return {"loss": NeuralType(elements_type=LossType())} + + def __init__(self, reduction: str = 'mean'): + """ + Args: + reduction: specifies the reduction to apply to the final loss, choose 'mean' or 'sum' + """ + super().__init__() + + if reduction not in ['mean', 'sum']: + logging.warning(f'{reduction} reduction is not supported. Setting reduction to "mean"') + reduction = 'mean' + + self.reduction = reduction + self._cross_entropy = torch.nn.CrossEntropyLoss(reduction=self.reduction) + self._cross_entropy_bin = torch.nn.BCEWithLogitsLoss(reduction=self.reduction) + + def _helper(self, logits, labels, loss_mask=None): + """ + flattens logits and labels according loss mask + Args: + logits: logits + labels: labels + loss_mask: loss mask + Returns: + logits_flatten: flattened logits where loss mask is true + labels_flatten: flattened labels where loss mask is true + """ + logits_flatten = torch.flatten(logits, start_dim=0, end_dim=-2) + labels_flatten = torch.flatten(labels, start_dim=0, end_dim=-1) + + if loss_mask is not None: + if loss_mask.dtype is not torch.bool: + loss_mask = loss_mask > 0.5 + loss_mask_flatten = torch.flatten(loss_mask, start_dim=0, end_dim=-1) + logits_flatten = logits_flatten[loss_mask_flatten] + labels_flatten = labels_flatten[loss_mask_flatten] + + return logits_flatten, labels_flatten + + @typecheck() + def forward( + self, + logit_intent_status, + intent_status, + logit_req_slot_status, + requested_slot_status, + logit_cat_slot_status, + categorical_slot_status, + logit_cat_slot_value_status, + categorical_slot_value_status, + logit_noncat_slot_status, + noncategorical_slot_status, + logit_spans, + noncategorical_slot_value_start, + noncategorical_slot_value_end, + task_mask, + ): + # Intent loss + + old_logit_intent_status = logit_intent_status + logit_intent_status, intent_status = self._helper(logit_intent_status, intent_status, task_mask[:, 0]) + if len(intent_status) == 0: + intent_loss = torch.clamp(torch.max(old_logit_intent_status.view(-1)), 0, 0) + else: + intent_loss = self._cross_entropy_bin(logit_intent_status.squeeze(dim=-1), intent_status) + + old_logit_req_slot_status = logit_req_slot_status + logit_req_slot_status, requested_slot_status = self._helper( + logit_req_slot_status, requested_slot_status, task_mask[:, 1] + ) + if len(requested_slot_status) == 0: + requested_slot_loss = torch.clamp(torch.max(old_logit_req_slot_status.view(-1)), 0, 0) + else: + requested_slot_loss = self._cross_entropy_bin(logit_req_slot_status.squeeze(dim=-1), requested_slot_status) + + old_logit_cat_slot_status = logit_cat_slot_status + logit_cat_slot_status, categorical_slot_status = self._helper( + logit_cat_slot_status, categorical_slot_status, task_mask[:, 2] + ) + if len(categorical_slot_status) == 0: + cat_slot_status_loss = torch.clamp(torch.max(old_logit_cat_slot_status.view(-1)), 0, 0) + else: + cat_slot_status_loss = self._cross_entropy(logit_cat_slot_status, categorical_slot_status,) + old_logit_cat_slot_value_status = logit_cat_slot_value_status + logit_cat_slot_value_status, categorical_slot_value_status = self._helper( + logit_cat_slot_value_status, categorical_slot_value_status, task_mask[:, 3] + ) + if len(categorical_slot_value_status) == 0: + cat_slot_value_status_loss = torch.clamp(torch.max(old_logit_cat_slot_value_status.view(-1)), 0, 0) + else: + cat_slot_value_status_loss = self._cross_entropy_bin( + logit_cat_slot_value_status.squeeze(dim=-1), categorical_slot_value_status + ) + + old_logit_noncat_slot_status = logit_noncat_slot_status + logit_noncat_slot_status, noncategorical_slot_status = self._helper( + logit_noncat_slot_status, noncategorical_slot_status, task_mask[:, 4] + ) + if len(noncategorical_slot_status) == 0: + noncat_slot_status_loss = torch.clamp(torch.max(old_logit_noncat_slot_status.view(-1)), 0, 0) + else: + noncat_slot_status_loss = self._cross_entropy(logit_noncat_slot_status, noncategorical_slot_status,) + + logit_noncat_slot_start, logit_noncat_slot_end = torch.unbind(logit_spans, dim=-1) + + _, max_num_tokens = logit_noncat_slot_start.size() + old_logit_noncat_slot_start = logit_noncat_slot_start + logit_noncat_slot_start, noncategorical_slot_value_start = self._helper( + logit_noncat_slot_start, noncategorical_slot_value_start, task_mask[:, 5] + ) + if len(noncategorical_slot_value_start) == 0: + span_start_loss = torch.clamp(torch.max(old_logit_noncat_slot_start.view(-1)), 0, 0) + else: + span_start_loss = self._cross_entropy(logit_noncat_slot_start, noncategorical_slot_value_start) + + old_logit_noncat_slot_end = logit_noncat_slot_end + logit_noncat_slot_end, noncategorical_slot_value_end = self._helper( + logit_noncat_slot_end, noncategorical_slot_value_end, task_mask[:, 5] + ) + if len(noncategorical_slot_value_end) == 0: + span_end_loss = torch.clamp(torch.max(old_logit_noncat_slot_end.view(-1)), 0, 0) + else: + span_end_loss = self._cross_entropy(logit_noncat_slot_end, noncategorical_slot_value_end) + + losses = { + "intent_loss": intent_loss, + "requested_slot_loss": requested_slot_loss, + "cat_slot_status_loss": cat_slot_status_loss, + "cat_slot_value_status_loss": cat_slot_value_status_loss, + "noncat_slot_status_loss": noncat_slot_status_loss, + "span_start_loss": span_start_loss, + "span_end_loss": span_end_loss, + } + + total_loss = sum(losses.values()) + if self.reduction == 'mean': + total_loss = total_loss / len(losses) + else: + batch_size = logit_intent_status.shape[0] + total_loss = total_loss / batch_size + return total_loss diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/metrics/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/metrics/__init__.py new file mode 100644 index 0000000000..dc01627ca1 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/metrics/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.metrics.classification_report import ClassificationReport +from nemo.collections.nlp.metrics.sequence_perplexity import SequencePerplexity diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/metrics/classification_report.py b/models/tacotron2/NeMo/nemo/collections/nlp/metrics/classification_report.py new file mode 100644 index 0000000000..5897677419 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/metrics/classification_report.py @@ -0,0 +1,173 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, Optional + +import torch +from torchmetrics import Metric +from torchmetrics.utilities.data import METRIC_EPS + +__all__ = ['ClassificationReport'] + + +class ClassificationReport(Metric): + """ + This metric computes the number of True Positive, False Negative, and False Positive examples per class. + When doing distributed training/evaluation the result of res=ClassificationReport(predictions, labels) calls + will be all-reduced between all workers using SUM operations. + + If used with PytorchLightning LightningModule, include TPs, FNs, and FPs inside validation_step results. + Then aggregate them at the end of validation epoch to correctly compute validation precision, recall, f1 + using get_precision_recall_f1(). + + Example: + def validation_step(self, batch, batch_idx): + ... + tp, fn, fp, _ = self.classification_report(preds, labels) + + return {'val_loss': val_loss, 'tp': tp, 'fn': fn, 'fp': fp} + + def validation_epoch_end(self, outputs): + ... + # calculate metrics and classification report + precision, recall, f1, report = self.classification_report.compute() + + logging.info(report) + + self.log('val_loss', avg_loss, prog_bar=True) + self.log('precision', precision) + self.log('f1', f1) + self.log('recall', recall) + + Args: + num_classes: number of classes in the dataset + label_ids (optional): label name to label id mapping + mode: how to compute the average + dist_sync_on_step: sync across ddp + process_group: which processes to sync across + Return: + aggregated precision, recall, f1, report + """ + + def __init__( + self, + num_classes: int, + label_ids: Dict[str, int] = None, + mode: str = 'macro', + dist_sync_on_step: bool = False, + process_group: Optional[Any] = None, + ): + super().__init__(dist_sync_on_step=dist_sync_on_step, process_group=process_group) + self.num_classes = num_classes + if label_ids: + self.ids_to_labels = {v: k for k, v in label_ids.items()} + else: + self.ids_to_labels = None + self.mode = mode + + self.add_state("tp", default=torch.zeros(num_classes), dist_reduce_fx='sum', persistent=False) + self.add_state("fn", default=torch.zeros(num_classes), dist_reduce_fx='sum', persistent=False) + self.add_state("fp", default=torch.zeros(num_classes), dist_reduce_fx='sum', persistent=False) + self.add_state( + "num_examples_per_class", default=torch.zeros(num_classes), dist_reduce_fx='sum', persistent=False + ) + + def update(self, predictions: torch.Tensor, labels: torch.Tensor): + TP = [] + FN = [] + FP = [] + for label_id in range(self.num_classes): + current_label = labels == label_id + label_predicted = predictions == label_id + + TP.append((label_predicted == current_label)[label_predicted].sum()) + FP.append((label_predicted != current_label)[label_predicted].sum()) + FN.append((label_predicted != current_label)[current_label].sum()) + + tp = torch.tensor(TP).to(predictions.device) + fn = torch.tensor(FN).to(predictions.device) + fp = torch.tensor(FP).to(predictions.device) + num_examples_per_class = tp + fn + + self.tp += tp + self.fn += fn + self.fp += fp + self.num_examples_per_class += num_examples_per_class + + def compute(self): + """ + Aggregates and then calculates logs classification report similar to sklearn.metrics.classification_report. + Typically used during epoch_end. + Return: + aggregated precision, recall, f1, report + """ + total_examples = torch.sum(self.num_examples_per_class) + num_non_empty_classes = torch.nonzero(self.num_examples_per_class).size(0) + + precision = torch.true_divide(self.tp * 100, (self.tp + self.fp + METRIC_EPS)) + recall = torch.true_divide(self.tp * 100, (self.tp + self.fn + METRIC_EPS)) + f1 = torch.true_divide(2 * precision * recall, (precision + recall + METRIC_EPS)) + + report = '\n{:50s} {:10s} {:10s} {:10s} {:10s}'.format('label', 'precision', 'recall', 'f1', 'support') + for i in range(len(self.tp)): + label = f'label_id: {i}' + if self.ids_to_labels and i in self.ids_to_labels: + label = f'{self.ids_to_labels[i]} ({label})' + + report += '\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format( + label, precision[i], recall[i], f1[i], self.num_examples_per_class[i] + ) + + micro_precision = torch.true_divide(torch.sum(self.tp) * 100, torch.sum(self.tp + self.fp) + METRIC_EPS) + micro_recall = torch.true_divide(torch.sum(self.tp) * 100, torch.sum(self.tp + self.fn) + METRIC_EPS) + micro_f1 = torch.true_divide(2 * micro_precision * micro_recall, (micro_precision + micro_recall + METRIC_EPS)) + + macro_precision = torch.sum(precision) / num_non_empty_classes + macro_recall = torch.sum(recall) / num_non_empty_classes + macro_f1 = torch.sum(f1) / num_non_empty_classes + weighted_precision = torch.sum(precision * self.num_examples_per_class) / total_examples + weighted_recall = torch.sum(recall * self.num_examples_per_class) / total_examples + weighted_f1 = torch.sum(f1 * self.num_examples_per_class) / total_examples + + report += "\n-------------------" + + report += '\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format( + 'micro avg', micro_precision, micro_recall, micro_f1, total_examples + ) + + report += '\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format( + 'macro avg', macro_precision, macro_recall, macro_f1, total_examples + ) + report += ( + '\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format( + 'weighted avg', weighted_precision, weighted_recall, weighted_f1, total_examples + ) + + '\n' + ) + + self.total_examples = total_examples + + if self.mode == 'macro': + return macro_precision, macro_recall, macro_f1, report + elif self.mode == 'weighted': + return weighted_precision, weighted_recall, weighted_f1, report + elif self.mode == 'micro': + return micro_precision, micro_recall, micro_f1, report + elif self.mode == 'all': + return precision, recall, f1, report + else: + raise ValueError( + f'{self.mode} mode is not supported. Choose "macro" to get aggregated numbers \ + or "all" to get values for each class.' + ) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/metrics/sequence_perplexity.py b/models/tacotron2/NeMo/nemo/collections/nlp/metrics/sequence_perplexity.py new file mode 100644 index 0000000000..688f9db87e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/metrics/sequence_perplexity.py @@ -0,0 +1,78 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torchmetrics import Metric + +__all__ = ['SequencePerplexity'] + + +class SequencePerplexity(Metric): + """ + This class computes mean perplexity across the batches of sequences. + + You have to provide ``log_probs`` (float tensor of shape [batch_size x seq_length x vocab_size]) and + ``labels`` (int tensor of shape [batch_size x seq_length] with values from the range [0, vocab_size-1]) + to the :meth:`update` method. If some of the sequences are shorter than seq_length, you can also provide + an optional argument ``mask`` (bool tensor of shape [batch_size x seq_length]) which masks out tokens + not participating in perplexity computation. + + See :doc:`PyTorch Lightning Metrics` for the metric usage instructions. + + Args: + compute_on_step: + Forward only calls ``update()`` and returns ``None`` if this is set to ``False``. default: ``True`` + dist_sync_on_step: + Synchronize metric state across processes at each ``forward()`` before returning the value at the step. + process_group: + Specify the process group on which synchronization is called. default: ``None`` (which selects the entire + world) + dist_sync_fn: + Callback that performs the allgather operation on the metric state. When ``None``, DDP will be used + to perform the allgather. + """ + + def __init__(self, compute_on_step=True, dist_sync_on_step=False, process_group=None, dist_sync_fn=None): + super().__init__( + compute_on_step=compute_on_step, + dist_sync_on_step=dist_sync_on_step, + process_group=process_group, + dist_sync_fn=dist_sync_fn, + ) + + # Total sum of exponentiated average negative log likelihoods + self.add_state('perplexities_sum', default=torch.tensor(0.0, dtype=torch.float64), dist_reduce_fx='sum') + # Total number of sequences in all batches + self.add_state('num_sequences', default=torch.tensor(0, dtype=torch.int64), dist_reduce_fx='sum') + + def update(self, log_probs: torch.Tensor, labels: torch.Tensor, mask=None): + + if mask is None: + mask = torch.ones_like(labels) + if mask.dtype is not log_probs.dtype: + mask = mask.to(log_probs.dtype) + + target_log_probs = log_probs.gather(2, labels.unsqueeze(2)).squeeze(2) + avg_neg_ll = -(target_log_probs * mask).sum(dim=-1) / mask.sum(dim=-1) + ppl = avg_neg_ll.exp() + self.num_sequences += ppl.numel() + self.perplexities_sum += ppl.sum() + + def compute(self): + """ + Returns perplexity across all workers and resets to 0 :attr:`perplexities_sum` and :attr:`num_sequences`. + """ + if self.num_sequences.eq(0): + return None + return self.perplexities_sum / self.num_sequences diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/__init__.py new file mode 100644 index 0000000000..42f20f128e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.models.dialogue_state_tracking.sgdqa_model import SGDQAModel +from nemo.collections.nlp.models.entity_linking.entity_linking_model import EntityLinkingModel +from nemo.collections.nlp.models.glue_benchmark.glue_benchmark_model import GLUEModel +from nemo.collections.nlp.models.information_retrieval import BertDPRModel, BertJointIRModel +from nemo.collections.nlp.models.intent_slot_classification import IntentSlotClassificationModel +from nemo.collections.nlp.models.language_modeling.bert_lm_model import BERTLMModel +from nemo.collections.nlp.models.language_modeling.transformer_lm_model import TransformerLMModel +from nemo.collections.nlp.models.machine_translation import MTEncDecModel +from nemo.collections.nlp.models.neural_machine_translation.neural_machine_translation_model import ( + NeuralMachineTranslationModel, +) +from nemo.collections.nlp.models.question_answering.qa_model import QAModel +from nemo.collections.nlp.models.text_classification import TextClassificationModel +from nemo.collections.nlp.models.token_classification import PunctuationCapitalizationModel, TokenClassificationModel diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/dialogue_state_tracking/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/dialogue_state_tracking/__init__.py new file mode 100644 index 0000000000..b7d20a2ee5 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/dialogue_state_tracking/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.models.dialogue_state_tracking.sgdqa_model import SGDQAModel diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/dialogue_state_tracking/sgdqa_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/dialogue_state_tracking/sgdqa_model.py new file mode 100644 index 0000000000..5232447d49 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/dialogue_state_tracking/sgdqa_model.py @@ -0,0 +1,580 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +This file contains code artifacts adapted from the original implementation: +https://github.com/google-research/google-research/blob/master/schema_guided_dst/baseline/train_and_predict.py +''' + +import os +from typing import Dict, List, Optional + +import torch +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer +from torch.utils.data import DataLoader + +from nemo.collections.nlp.data.dialogue_state_tracking import Schema, SGDDataProcessor, SGDDataset +from nemo.collections.nlp.data.dialogue_state_tracking.sgd.evaluate import evaluate, get_in_domain_services +from nemo.collections.nlp.data.dialogue_state_tracking.sgd.prediction_utils import write_predictions_to_file +from nemo.collections.nlp.losses import SGDDialogueStateLoss +from nemo.collections.nlp.models.nlp_model import NLPModel +from nemo.collections.nlp.modules import SGDDecoder, SGDEncoder +from nemo.collections.nlp.modules.common.lm_utils import get_lm_model +from nemo.collections.nlp.parts.utils_funcs import tensor2list +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types import NeuralType +from nemo.utils.get_rank import is_global_rank_zero + +__all__ = ['SGDQAModel'] + +NUM_TASKS = 6 # number of multi-head tasks + + +class SGDQAModel(NLPModel): + """Dialogue State Tracking Model SGD-QA""" + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return self.bert_model.input_types + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return self.decoder.output_types + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + + self.data_prepared = False + self.setup_tokenizer(cfg.tokenizer) + super().__init__(cfg=cfg, trainer=trainer) + self.bert_model = get_lm_model( + pretrained_model_name=cfg.language_model.pretrained_model_name, + config_file=self.register_artifact('language_model.config_file', cfg.language_model.config_file), + config_dict=OmegaConf.to_container(cfg.language_model.config) if cfg.language_model.config else None, + checkpoint_file=cfg.language_model.lm_checkpoint, + vocab_file=self.register_artifact('tokenizer.vocab_file', cfg.tokenizer.vocab_file), + ) + + self.encoder = SGDEncoder(hidden_size=self.bert_model.config.hidden_size, dropout=self._cfg.encoder.dropout) + self.decoder = SGDDecoder(embedding_dim=self.bert_model.config.hidden_size) + self.loss = SGDDialogueStateLoss(reduction="mean") + + @typecheck() + def forward(self, input_ids, token_type_ids, attention_mask): + token_embeddings = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask + ) + encoded_utterance, token_embeddings = self.encoder(hidden_states=token_embeddings) + ( + logit_intent_status, + logit_req_slot_status, + logit_cat_slot_status, + logit_cat_slot_value_status, + logit_noncat_slot_status, + logit_spans, + ) = self.decoder( + encoded_utterance=encoded_utterance, token_embeddings=token_embeddings, utterance_mask=attention_mask + ) + return ( + logit_intent_status, + logit_req_slot_status, + logit_cat_slot_status, + logit_cat_slot_value_status, + logit_noncat_slot_status, + logit_spans, + ) + + def training_step(self, batch, batch_idx): + ( + example_id_num, + service_id, + utterance_ids, + token_type_ids, + attention_mask, + intent_status, + requested_slot_status, + categorical_slot_status, + categorical_slot_value_status, + noncategorical_slot_status, + noncategorical_slot_value_start, + noncategorical_slot_value_end, + start_char_idx, + end_char_idx, + task_mask, + ) = batch + ( + logit_intent_status, + logit_req_slot_status, + logit_cat_slot_status, + logit_cat_slot_value_status, + logit_noncat_slot_status, + logit_spans, + ) = self(input_ids=utterance_ids, token_type_ids=token_type_ids, attention_mask=attention_mask) + loss = self.loss( + logit_intent_status=logit_intent_status, + intent_status=intent_status, + logit_req_slot_status=logit_req_slot_status, + requested_slot_status=requested_slot_status, + logit_cat_slot_status=logit_cat_slot_status, + categorical_slot_status=categorical_slot_status, + logit_cat_slot_value_status=logit_cat_slot_value_status, + categorical_slot_value_status=categorical_slot_value_status, + logit_noncat_slot_status=logit_noncat_slot_status, + noncategorical_slot_status=noncategorical_slot_status, + logit_spans=logit_spans, + noncategorical_slot_value_start=noncategorical_slot_value_start, + noncategorical_slot_value_end=noncategorical_slot_value_end, + task_mask=task_mask, + ) + lr = self._optimizer.param_groups[0]['lr'] + + self.log('train_loss', loss) + self.log('lr', lr, prog_bar=True) + + return { + 'loss': loss, + 'lr': lr, + } + + def validation_step(self, batch: List[torch.Tensor], batch_idx: int, dataloader_idx: int = 0) -> dict: + """ + Called at every validation step to aggregate and postprocess outputs on each GPU + Args: + batch: input batch at validation step + batch_idx: batch index + dataloader_idx: dataloader index + """ + loss, tensors = self.eval_step_helper(batch=batch) + self.log(f'val_loss', loss) + return {f'val_loss': loss, f'tensors': tensors} + + def test_step(self, batch: List[torch.Tensor], batch_idx: int, dataloader_idx: int = 0) -> dict: + """ + Called at every test step to aggregate and postprocess outputs on each GPU + Args: + batch: input batch at test step + batch_idx: batch index + dataloader_idx: dataloader index + """ + loss, tensors = self.eval_step_helper(batch=batch) + return {f'test_loss': loss, f'tensors': tensors} + + def eval_step_helper(self, batch: List[torch.Tensor]): + """ + Helper called at every validation/test step to aggregate and postprocess outputs on each GPU + Args: + batch: input batch at step + Returns: + loss: averaged batch loss + tensors: collection of aggregated output tensors across all GPU workers + """ + ( + example_id_num, + service_id, + utterance_ids, + token_type_ids, + attention_mask, + intent_status, + requested_slot_status, + categorical_slot_status, + categorical_slot_value_status, + noncategorical_slot_status, + noncategorical_slot_value_start, + noncategorical_slot_value_end, + start_char_idx, + end_char_idx, + task_mask, + ) = batch + ( + logit_intent_status, + logit_req_slot_status, + logit_cat_slot_status, + logit_cat_slot_value_status, + logit_noncat_slot_status, + logit_spans, + ) = self(input_ids=utterance_ids, token_type_ids=token_type_ids, attention_mask=attention_mask) + loss = self.loss( + logit_intent_status=logit_intent_status, + intent_status=intent_status, + logit_req_slot_status=logit_req_slot_status, + requested_slot_status=requested_slot_status, + logit_cat_slot_status=logit_cat_slot_status, + categorical_slot_status=categorical_slot_status, + logit_cat_slot_value_status=logit_cat_slot_value_status, + categorical_slot_value_status=categorical_slot_value_status, + logit_noncat_slot_status=logit_noncat_slot_status, + noncategorical_slot_status=noncategorical_slot_status, + logit_spans=logit_spans, + noncategorical_slot_value_start=noncategorical_slot_value_start, + noncategorical_slot_value_end=noncategorical_slot_value_end, + task_mask=task_mask, + ) + + all_example_id_num = [] + all_service_id = [] + all_logit_intent_status = [] + all_logit_req_slot_status = [] + all_logit_cat_slot_status = [] + all_logit_cat_slot_value_status = [] + all_logit_noncat_slot_status = [] + all_logit_spans = [] + all_start_char_idx = [] + all_end_char_idx = [] + + if self.trainer.gpus and self.trainer.world_size > 1: + world_size = self.trainer.world_size + for ind in range(world_size): + all_example_id_num.append(torch.empty_like(example_id_num)) + all_service_id.append(torch.empty_like(service_id)) + all_logit_intent_status.append(torch.empty_like(logit_intent_status)) + all_logit_req_slot_status.append(torch.empty_like(logit_req_slot_status)) + all_logit_cat_slot_status.append(torch.empty_like(logit_cat_slot_status)) + all_logit_cat_slot_value_status.append(torch.empty_like(logit_cat_slot_value_status)) + all_logit_noncat_slot_status.append(torch.empty_like(logit_noncat_slot_status)) + all_logit_spans.append(torch.empty_like(logit_spans)) + all_start_char_idx.append(torch.empty_like(start_char_idx)) + all_end_char_idx.append(torch.empty_like(end_char_idx)) + + torch.distributed.all_gather(all_example_id_num, example_id_num) + torch.distributed.all_gather(all_service_id, service_id) + torch.distributed.all_gather(all_logit_intent_status, logit_intent_status) + torch.distributed.all_gather(all_logit_req_slot_status, logit_req_slot_status) + torch.distributed.all_gather(all_logit_cat_slot_status, logit_cat_slot_status) + torch.distributed.all_gather(all_logit_cat_slot_value_status, logit_cat_slot_value_status) + torch.distributed.all_gather(all_logit_noncat_slot_status, logit_noncat_slot_status) + torch.distributed.all_gather(all_logit_spans, logit_spans) + torch.distributed.all_gather(all_start_char_idx, start_char_idx) + torch.distributed.all_gather(all_end_char_idx, end_char_idx) + else: + all_example_id_num.append(example_id_num) + all_service_id.append(service_id) + all_logit_intent_status.append(logit_intent_status) + all_logit_req_slot_status.append(logit_req_slot_status) + all_logit_cat_slot_status.append(logit_cat_slot_status) + all_logit_cat_slot_value_status.append(logit_cat_slot_value_status) + all_logit_noncat_slot_status.append(logit_noncat_slot_status) + all_logit_spans.append(logit_spans) + all_start_char_idx.append(start_char_idx) + all_end_char_idx.append(end_char_idx) + + # after this: all_x is list of tensors, of length world_size + example_id_num = torch.cat(all_example_id_num) + service_id = torch.cat(all_service_id) + logit_intent_status = torch.cat(all_logit_intent_status) + logit_req_slot_status = torch.cat(all_logit_req_slot_status) + logit_cat_slot_status = torch.cat(all_logit_cat_slot_status) + logit_cat_slot_value_status = torch.cat(all_logit_cat_slot_value_status) + logit_noncat_slot_status = torch.cat(all_logit_noncat_slot_status) + logit_spans = torch.cat(all_logit_spans) + start_char_idx = torch.cat(all_start_char_idx) + end_char_idx = torch.cat(all_end_char_idx) + + intent_status = torch.nn.Sigmoid()(logit_intent_status) + + # Scores are output for each requested slot. + req_slot_status = torch.nn.Sigmoid()(logit_req_slot_status) + + # For categorical slots, the status of each slot and the predicted value are output. + cat_slot_status_dist = torch.nn.Softmax(dim=-1)(logit_cat_slot_status) + + cat_slot_status = torch.argmax(logit_cat_slot_status, axis=-1) + cat_slot_status_p = torch.max(cat_slot_status_dist, axis=-1)[0] + cat_slot_value_status = torch.nn.Sigmoid()(logit_cat_slot_value_status) + + # For non-categorical slots, the status of each slot and the indices for spans are output. + noncat_slot_status_dist = torch.nn.Softmax(dim=-1)(logit_noncat_slot_status) + + noncat_slot_status = torch.argmax(logit_noncat_slot_status, axis=-1) + noncat_slot_status_p = torch.max(noncat_slot_status_dist, axis=-1)[0] + + softmax = torch.nn.Softmax(dim=1) + + scores = softmax(logit_spans) + start_scores, end_scores = torch.unbind(scores, dim=-1) + + batch_size, max_num_tokens = end_scores.size() + # Find the span with the maximum sum of scores for start and end indices. + total_scores = torch.unsqueeze(start_scores, axis=2) + torch.unsqueeze(end_scores, axis=1) + start_idx = torch.arange(max_num_tokens, device=total_scores.get_device()).view(1, -1, 1) + end_idx = torch.arange(max_num_tokens, device=total_scores.get_device()).view(1, 1, -1) + invalid_index_mask = (start_idx > end_idx).repeat(batch_size, 1, 1) + total_scores = torch.where( + invalid_index_mask, + torch.zeros(total_scores.size(), device=total_scores.get_device(), dtype=total_scores.dtype), + total_scores, + ) + max_span_index = torch.argmax(total_scores.view(-1, max_num_tokens ** 2), axis=-1) + max_span_p = torch.max(total_scores.view(-1, max_num_tokens ** 2), axis=-1)[0] + + span_start_index = torch.floor_divide(max_span_index, max_num_tokens) + span_end_index = torch.fmod(max_span_index, max_num_tokens) + + tensors = { + 'example_id_num': example_id_num, + 'service_id': service_id, + 'intent_status': intent_status, + 'req_slot_status': req_slot_status, + 'cat_slot_status': cat_slot_status, + 'cat_slot_status_p': cat_slot_status_p, + 'cat_slot_value_status': cat_slot_value_status, + 'noncat_slot_status': noncat_slot_status, + 'noncat_slot_status_p': noncat_slot_status_p, + 'noncat_slot_p': max_span_p, + 'noncat_slot_start': span_start_index, + 'noncat_slot_end': span_end_index, + 'noncat_alignment_start': start_char_idx, + 'noncat_alignment_end': end_char_idx, + } + return loss, tensors + + def multi_validation_epoch_end(self, outputs: List[dict], dataloader_idx: int = 0): + """ + Called at the end of validation to post process outputs into human readable format + Args: + outputs: list of individual outputs of each validation step + dataloader_idx: dataloader index + """ + avg_loss = torch.stack([x[f'val_loss'] for x in outputs]).mean() + split = self._validation_names[dataloader_idx][:-1] + dataloader = self._validation_dl[dataloader_idx] + metrics = self.multi_eval_epoch_end_helper(outputs=outputs, split=split, dataloader=dataloader) + + for k, v in metrics.items(): + self.log(f'{split}_{k}', v) + + self.log(f'val_loss', avg_loss, prog_bar=True) + + def multi_test_epoch_end(self, outputs: List[dict], dataloader_idx: int = 0): + """ + Called at the end of test to post process outputs into human readable format + Args: + outputs: list of individual outputs of each test step + dataloader_idx: dataloader index + """ + avg_loss = torch.stack([x[f'test_loss'] for x in outputs]).mean() + split = self._test_names[dataloader_idx][:-1] + dataloader = self._test_dl[dataloader_idx] + metrics = self.multi_eval_epoch_end_helper(outputs=outputs, split=split, dataloader=dataloader) + + for k, v in metrics.items(): + self.log(f'{split}_{k}', v) + + self.log(f'test_loss', avg_loss, prog_bar=True) + + def multi_eval_epoch_end_helper( + self, outputs: List[dict], split: str, dataloader: torch.utils.data.DataLoader + ) -> dict: + """ + Helper called at the end of evaluation to post process outputs into human readable format + Args: + outputs: list of individual outputs of each step + split: data split + dataloader: dataloader + Returns: + metrics: metrics collection + """ + + def get_str_example_id(split: str, ids_to_service_names_dict: dict, example_id_num: torch.Tensor) -> str: + """ + Constructs string representation of example ID + Args: + split: evaluation data split + ids_to_service_names_dict: id to service name mapping + example_id_num: tensor example id + """ + + def format_turn_id(ex_id_num): + dialog_id_1, dialog_id_2, turn_id, service_id, model_task_id, slot_intent_id, value_id = ex_id_num + return "{}-{}_{:05d}-{:02d}-{}-{}-{}-{}".format( + split, + dialog_id_1, + dialog_id_2, + turn_id, + ids_to_service_names_dict[service_id], + model_task_id, + slot_intent_id, + value_id, + ) + + return list(map(format_turn_id, tensor2list(example_id_num))) + + def combine_predictions_in_example(predictions: dict, batch_size: int): + ''' + Combines predicted values to a single example. + Args: + predictions: predictions ordered by keys then batch + batch_size: batch size + Returns: + examples_preds: predictions ordered by batch then key + ''' + examples_preds = [{} for _ in range(batch_size)] + for k, v in predictions.items(): + if k != 'example_id': + v = torch.chunk(v, batch_size) + + for i in range(batch_size): + if k == 'example_id': + examples_preds[i][k] = v[i] + else: + examples_preds[i][k] = v[i].view(-1) + return examples_preds + + example_id_num = torch.cat([x[f'tensors']['example_id_num'] for x in outputs]) + service_id = torch.cat([x[f'tensors']['service_id'] for x in outputs]) + intent_status = torch.cat([x[f'tensors']['intent_status'] for x in outputs]) + req_slot_status = torch.cat([x[f'tensors']['req_slot_status'] for x in outputs]) + cat_slot_status = torch.cat([x[f'tensors']['cat_slot_status'] for x in outputs]) + cat_slot_status_p = torch.cat([x[f'tensors']['cat_slot_status_p'] for x in outputs]) + cat_slot_value_status = torch.cat([x[f'tensors']['cat_slot_value_status'] for x in outputs]) + noncat_slot_status = torch.cat([x[f'tensors']['noncat_slot_status'] for x in outputs]) + noncat_slot_status_p = torch.cat([x[f'tensors']['noncat_slot_status_p'] for x in outputs]) + noncat_slot_p = torch.cat([x[f'tensors']['noncat_slot_p'] for x in outputs]) + noncat_slot_start = torch.cat([x[f'tensors']['noncat_slot_start'] for x in outputs]) + noncat_slot_end = torch.cat([x[f'tensors']['noncat_slot_end'] for x in outputs]) + noncat_alignment_start = torch.cat([x[f'tensors']['noncat_alignment_start'] for x in outputs]) + noncat_alignment_end = torch.cat([x[f'tensors']['noncat_alignment_end'] for x in outputs]) + + ids_to_service_names_dict = self.dialogues_processor.schemas._services_id_to_vocab + example_id = get_str_example_id(dataloader.dataset, ids_to_service_names_dict, example_id_num) + + metrics = {} + try: + prediction_dir = self.trainer.log_dir if self.trainer.log_dir is not None else "" + except: + prediction_dir = "" + + if self.trainer.global_rank == 0: + prediction_dir = os.path.join( + prediction_dir, 'predictions', 'pred_res_{}_{}'.format(split, self._cfg.dataset.task_name) + ) + os.makedirs(prediction_dir, exist_ok=True) + + input_json_files = SGDDataProcessor.get_dialogue_files( + self._cfg.dataset.data_dir, split, self._cfg.dataset.task_name + ) + + predictions = {} + predictions['example_id'] = example_id + predictions['service_id'] = service_id + predictions['intent_status'] = intent_status + predictions['req_slot_status'] = req_slot_status + predictions['cat_slot_status'] = cat_slot_status + predictions['cat_slot_status_p'] = cat_slot_status_p + predictions['cat_slot_value_status'] = cat_slot_value_status + predictions['noncat_slot_status'] = noncat_slot_status + predictions['noncat_slot_status_p'] = noncat_slot_status_p + predictions['noncat_slot_p'] = noncat_slot_p + predictions['noncat_slot_start'] = noncat_slot_start + predictions['noncat_slot_end'] = noncat_slot_end + predictions['noncat_alignment_start'] = noncat_alignment_start + predictions['noncat_alignment_end'] = noncat_alignment_end + + in_domain_services = get_in_domain_services( + os.path.join(self._cfg.dataset.data_dir, split, "schema.json"), + self.dialogues_processor.get_seen_services("train"), + ) + predictions = combine_predictions_in_example(predictions, service_id.shape[0]) + + # write predictions to file in Dstc8/SGD format + write_predictions_to_file( + predictions, + input_json_files, + output_dir=prediction_dir, + schemas=self.dialogues_processor.schemas, + state_tracker=self._cfg.dataset.state_tracker, + eval_debug=False, + in_domain_services=in_domain_services, + ) + metrics = evaluate( + prediction_dir, + self._cfg.dataset.data_dir, + split, + in_domain_services, + joint_acc_across_turn=self._cfg.dataset.joint_acc_across_turn, + use_fuzzy_match=self._cfg.dataset.use_fuzzy_match, + ) + + return metrics + + def prepare_data(self): + """ + Preprocessed schema and dialogues and caches this + """ + if self.data_prepared: + return + schema_config = { + "MAX_NUM_CAT_SLOT": self._cfg.dataset.max_num_cat_slot, + "MAX_NUM_NONCAT_SLOT": self._cfg.dataset.max_num_noncat_slot, + "MAX_NUM_VALUE_PER_CAT_SLOT": self._cfg.dataset.max_value_per_cat_slot, + "MAX_NUM_INTENT": self._cfg.dataset.max_num_intent, + "NUM_TASKS": NUM_TASKS, + "MAX_SEQ_LENGTH": self._cfg.dataset.max_seq_length, + } + all_schema_json_paths = [] + for dataset_split in ['train', 'test', 'dev']: + all_schema_json_paths.append(os.path.join(self._cfg.dataset.data_dir, dataset_split, "schema.json")) + schemas = Schema(all_schema_json_paths) + + self.dialogues_processor = SGDDataProcessor( + task_name=self._cfg.dataset.task_name, + data_dir=self._cfg.dataset.data_dir, + dialogues_example_dir=self._cfg.dataset.dialogues_example_dir, + tokenizer=self.tokenizer, + schemas=schemas, + schema_config=schema_config, + subsample=self._cfg.dataset.subsample, + ) + + if is_global_rank_zero(): + overwrite_dial_files = not self._cfg.dataset.use_cache + self.dialogues_processor.save_dialog_examples(overwrite_dial_files=overwrite_dial_files) + + self.data_prepared = True + + def setup_training_data(self, train_data_config: Optional[DictConfig] = None): + self.prepare_data() + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config, split=train_data_config.ds_item) + + def setup_validation_data(self, val_data_config: Optional[DictConfig] = None): + self.prepare_data() + self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config, split=val_data_config.ds_item) + + def setup_test_data(self, test_data_config: Optional[DictConfig] = None): + self.prepare_data() + self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config, split=test_data_config.ds_item) + + def _setup_dataloader_from_config(self, cfg: DictConfig, split: str) -> DataLoader: + dataset_cfg = self._cfg.dataset + data_dir = dataset_cfg.data_dir + + if not os.path.exists(data_dir): + raise FileNotFoundError(f"Data directory is not found at: {data_dir}.") + + dataset = SGDDataset(dataset_split=split, dialogues_processor=self.dialogues_processor) + + dl = torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg.batch_size, + collate_fn=dataset.collate_fn, + drop_last=cfg.drop_last, + shuffle=cfg.shuffle, + num_workers=cfg.num_workers, + pin_memory=cfg.pin_memory, + ) + return dl + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + pass diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/enc_dec_nlp_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/enc_dec_nlp_model.py new file mode 100644 index 0000000000..c08f026b40 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/enc_dec_nlp_model.py @@ -0,0 +1,85 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any + +from omegaconf.omegaconf import MISSING +from pytorch_lightning.trainer.trainer import Trainer + +from nemo.collections.nlp.models.nlp_model import NLPModel +from nemo.collections.nlp.modules.common.decoder_module import DecoderModule +from nemo.collections.nlp.modules.common.encoder_module import EncoderModule +from nemo.collections.nlp.modules.common.tokenizer_utils import TokenizerConfig +from nemo.core.config.modelPT import ModelConfig + + +@dataclass +class EncDecNLPModelConfig(ModelConfig): + encoder_tokenizer: TokenizerConfig = MISSING + decoder_tokenizer: TokenizerConfig = MISSING + encoder: Any = MISSING + decoder: Any = MISSING + head: Any = MISSING + + +class EncDecNLPModel(NLPModel): + """Base class for encoder-decoder NLP models. + """ + + def __init__(self, cfg: EncDecNLPModelConfig, trainer: Trainer = None): + super().__init__(cfg=cfg, trainer=trainer) + + @property + def encoder_vocab_size(self): + return self.encoder_tokenizer.vocab_size + + @property + def decoder_vocab_size(self): + return self.decoder_tokenizer.vocab_size + + @property + def encoder_tokenizer(self): + return self._encoder_tokenizer + + @encoder_tokenizer.setter + def encoder_tokenizer(self, tokenizer): + self._encoder_tokenizer = tokenizer + + @property + def decoder_tokenizer(self): + return self._decoder_tokenizer + + @decoder_tokenizer.setter + def decoder_tokenizer(self, tokenizer): + self._decoder_tokenizer = tokenizer + + @property + def encoder(self) -> EncoderModule: + return self._encoder + + @encoder.setter + def encoder(self, encoder): + self._encoder = encoder + + @property + def decoder(self) -> DecoderModule: + return self._decoder + + @decoder.setter + def decoder(self, decoder): + self._decoder = decoder + + def export(self, **kwargs): + raise NotImplementedError('For EncDecNLPModel, you must export encoder and decoder separately!') diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/entity_linking/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/entity_linking/__init__.py new file mode 100644 index 0000000000..925bfc18c7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/entity_linking/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.models.entity_linking.entity_linking_model import EntityLinkingModel diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/entity_linking/entity_linking_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/entity_linking/entity_linking_model.py new file mode 100644 index 0000000000..a35583af29 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/entity_linking/entity_linking_model.py @@ -0,0 +1,193 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional + +import torch +from omegaconf import DictConfig +from pytorch_lightning import Trainer +from transformers import AutoTokenizer + +from nemo.collections.common.losses import MultiSimilarityLoss +from nemo.collections.nlp.data import EntityLinkingDataset +from nemo.collections.nlp.models.nlp_model import NLPModel +from nemo.collections.nlp.modules.common.lm_utils import get_lm_model +from nemo.core.classes.common import typecheck +from nemo.core.classes.exportable import Exportable +from nemo.core.neural_types import ChannelType, LogitsType, MaskType, NeuralType +from nemo.utils import logging + +__all__ = ['EntityLinkingModel'] + + +class EntityLinkingModel(NLPModel, Exportable): + """ + Second stage pretraining of BERT based language model + for entity linking task. An implementation of Liu et. al's + NAACL 2021 paper Self-Alignment Pretraining for Biomedical Entity Representations. + """ + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "attention_mask": NeuralType(('B', 'T'), MaskType(), optional=True), + "token_type_ids": NeuralType(('B', 'T'), ChannelType(), optional=True), + } + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return {"logits": NeuralType(('B', 'D'), LogitsType())} + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + """Initializes the SAP-BERT model for entity linking.""" + + # tokenizer needed before super().__init__() so dataset and loader can process data + self._setup_tokenizer(cfg.tokenizer) + + super().__init__(cfg=cfg, trainer=trainer) + + self.model = get_lm_model( + pretrained_model_name=cfg.language_model.pretrained_model_name, + config_file=cfg.language_model.config_file, + config_dict=cfg.language_model.config, + checkpoint_file=cfg.language_model.lm_checkpoint, + ) + + # Token to use for the self-alignment loss, typically the first token, [CLS] + self._idx_conditioned_on = 0 + self.loss = MultiSimilarityLoss() + + def _setup_tokenizer(self, cfg: DictConfig): + tokenizer = AutoTokenizer.from_pretrained( + cfg.tokenizer_name, vocab_file=cfg.vocab_file, do_lower_case=cfg.do_lower_case + ) + + self.tokenizer = tokenizer + + @typecheck() + def forward(self, input_ids, token_type_ids, attention_mask): + hidden_states = self.model(input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask) + + # normalize to unit sphere + logits = torch.nn.functional.normalize(hidden_states[:, self._idx_conditioned_on], p=2, dim=1) + return logits + + def training_step(self, batch, batch_idx): + """ + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. + """ + input_ids, token_type_ids, attention_mask, concept_ids = batch + logits = self.forward(input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask) + train_loss = self.loss(logits=logits, labels=concept_ids) + + # No hard examples found in batch, + # shouldn't use this batch to update model weights + if train_loss == 0: + train_loss = None + lr = None + + else: + lr = self._optimizer.param_groups[0]["lr"] + self.log("train_loss", train_loss) + self.log("lr", lr, prog_bar=True) + + return {"loss": train_loss, "lr": lr} + + def validation_step(self, batch, batch_idx): + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. + """ + input_ids, input_type_ids, input_mask, concept_ids = batch + with torch.no_grad(): + logits = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + val_loss = self.loss(logits=logits, labels=concept_ids) + + # No hard examples found in batch, + # val loss not used to update model weights + if val_loss == 0: + val_loss = None + else: + self.log("val_loss", val_loss) + logging.info(f"val loss: {val_loss}") + + return {"val_loss": val_loss} + + def validation_epoch_end(self, outputs): + """ + Called at the end of validation to aggregate outputs. + + Args: + outputs: list of individual outputs of each validation step. + Returns: + + """ + if outputs: + avg_loss = torch.stack([x["val_loss"] for x in outputs if x["val_loss"] != None]).mean() + self.log(f"val_loss", avg_loss, prog_bar=True) + + return {"val_loss": avg_loss} + + def setup_training_data(self, train_data_config: Optional[DictConfig]): + if not train_data_config or not train_data_config.data_file: + logging.info( + f"Dataloader config or file_path or processed data path for the train dataset is missing, \ + so no data loader for train is created!" + ) + + self._train_dl = None + return + + self._train_dl = self.setup_dataloader(cfg=train_data_config) + + def setup_validation_data(self, val_data_config: Optional[DictConfig]): + if not val_data_config or not val_data_config.data_file: + logging.info( + f"Dataloader config or file_path or processed data path for the val dataset is missing, \ + so no data loader for validation is created!" + ) + + self._validation_dl = None + return + + self._validation_dl = self.setup_dataloader(cfg=val_data_config) + + def setup_dataloader(self, cfg: Dict, is_index_data: bool = False) -> 'torch.utils.data.DataLoader': + + dataset = EntityLinkingDataset( + tokenizer=self.tokenizer, + data_file=cfg.data_file, + max_seq_length=cfg.max_seq_length, + is_index_data=is_index_data, + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg.batch_size, + collate_fn=dataset.collate_fn, + shuffle=cfg.get("shuffle", True), + num_workers=cfg.get("num_wokers", 2), + pin_memory=cfg.get("pin_memory", False), + drop_last=cfg.get("drop_last", False), + ) + + @classmethod + def list_available_models(cls) -> Optional[Dict[str, str]]: + pass + + @classmethod + def from_pretrained(cls, name: str): + pass diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/__init__.py new file mode 100644 index 0000000000..eecc4db910 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.models.glue_benchmark.glue_benchmark_model import GLUEModel diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/glue_benchmark_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/glue_benchmark_model.py new file mode 100644 index 0000000000..81cc5bf573 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/glue_benchmark_model.py @@ -0,0 +1,279 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Dict, Optional, Union + +import numpy as np +import torch +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer + +from nemo.collections.common.losses import CrossEntropyLoss, MSELoss +from nemo.collections.nlp.data.glue_benchmark.glue_benchmark_dataset import GLUE_TASKS_NUM_LABELS, GLUEDataset +from nemo.collections.nlp.models.glue_benchmark.metrics_for_glue import compute_metrics +from nemo.collections.nlp.models.nlp_model import NLPModel +from nemo.collections.nlp.modules.common import SequenceClassifier, SequenceRegression +from nemo.collections.nlp.modules.common.lm_utils import get_lm_model +from nemo.collections.nlp.parts.utils_funcs import list2str, tensor2list +from nemo.core.classes import typecheck +from nemo.core.neural_types import NeuralType +from nemo.utils import logging + +__all__ = ['GLUEModel'] + +''' +Some transformer of this code were adapted from the HuggingFace library at +https://github.com/huggingface/transformers +Example of running a pretrained BERT model on the 9 GLUE tasks, read more +about GLUE benchmark here: https://gluebenchmark.com +Download the GLUE data by running the script: +https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e + +Some of these tasks have a small dataset and training can lead to high variance +in the results between different runs. Below is the median on 5 runs +(with different seeds) for each of the metrics on the dev set of the benchmark +with an uncased BERT base model (the checkpoint bert-base-uncased) +(source https://github.com/huggingface/transformers/tree/master/examples#glue). +Task Metric Result +CoLA Matthew's corr 48.87 +SST-2 Accuracy 91.74 +MRPC F1/Accuracy 90.70/86.27 +STS-B Person/Spearman corr. 91.39/91.04 +QQP Accuracy/F1 90.79/87.66 +MNLI Matched acc./Mismatched acc. 83.70/84.83 +QNLI Accuracy 89.31 +RTE Accuracy 71.43 +WNLI Accuracy 43.66 + +''' + + +class GLUEModel(NLPModel): + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return self.bert_model.input_types + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return self.pooler.output_types + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + """ + Initializes model to use BERT model for GLUE tasks. + """ + + if cfg.task_name not in cfg.supported_tasks: + raise ValueError(f'{cfg.task_name} not in supported task. Choose from {cfg.supported_tasks}') + self.task_name = cfg.task_name + + # needed to setup validation on multiple datasets + # MNLI task has two separate dev sets: matched and mismatched + if not self._is_model_being_restored(): + if self.task_name == "mnli": + cfg.validation_ds.ds_item = [ + os.path.join(cfg.dataset.data_dir, 'dev_matched.tsv'), + os.path.join(cfg.dataset.data_dir, 'dev_mismatched.tsv'), + ] + else: + cfg.validation_ds.ds_item = os.path.join(cfg.dataset.data_dir, cfg.validation_ds.ds_item) + cfg.train_ds.ds_item = os.path.join(cfg.dataset.data_dir, cfg.train_ds.ds_item) + logging.info(f'Using {cfg.validation_ds.ds_item} for model evaluation.') + + self.setup_tokenizer(cfg.tokenizer) + super().__init__(cfg=cfg, trainer=trainer) + + num_labels = GLUE_TASKS_NUM_LABELS[self.task_name] + + self.bert_model = get_lm_model( + pretrained_model_name=cfg.language_model.pretrained_model_name, + config_file=self.register_artifact('language_model.config_file', cfg.language_model.config_file), + config_dict=OmegaConf.to_container(cfg.language_model.config) if cfg.language_model.config else None, + checkpoint_file=cfg.language_model.lm_checkpoint, + vocab_file=self.register_artifact('tokenizer.vocab_file', cfg.tokenizer.vocab_file), + ) + + # uses [CLS] token for classification (the first token) + if self.task_name == "sts-b": + self.pooler = SequenceRegression(hidden_size=self.bert_model.config.hidden_size) + self.loss = MSELoss() + else: + self.pooler = SequenceClassifier( + hidden_size=self.bert_model.config.hidden_size, num_classes=num_labels, log_softmax=False + ) + self.loss = CrossEntropyLoss() + + def update_data_dir(self, data_dir: str) -> None: + """ + Update data directory and get data stats with Data Descriptor + Weights are later used to setup loss + + Args: + data_dir: path to data directory + """ + self._cfg.dataset.data_dir = data_dir + logging.info(f'Setting model.dataset.data_dir to {data_dir}.') + if self.task_name == "mnli": + self._cfg.validation_ds.ds_item = [ + os.path.join(data_dir, 'dev_matched.tsv'), + os.path.join(data_dir, 'dev_mismatched.tsv'), + ] + else: + self._cfg.validation_ds.ds_item = os.path.join(data_dir, 'dev.tsv') + + self._cfg.train_ds.ds_item = os.path.join(data_dir, 'train.tsv') + logging.info(f'Using {self._cfg.validation_ds.ds_item} for model evaluation.') + + @typecheck() + def forward(self, input_ids, token_type_ids, attention_mask): + hidden_states = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask + ) + output = self.pooler(hidden_states=hidden_states) + return output + + def training_step(self, batch, batch_idx): + input_ids, input_type_ids, input_mask, labels = batch + model_output = self(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + + if self.task_name == "sts-b": + loss = self.loss(preds=model_output, labels=labels) + else: + loss = self.loss(logits=model_output, labels=labels) + + lr = self._optimizer.param_groups[0]['lr'] + + self.log('train_loss', loss) + self.log('lr', lr, prog_bar=True) + + return { + 'loss': loss, + 'lr': lr, + } + + def validation_step(self, batch, batch_idx, dataloader_idx=0): + input_ids, input_type_ids, input_mask, labels = batch + model_output = self(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + + if self.task_name == "sts-b": + val_loss = self.loss(preds=model_output, labels=labels) + else: + val_loss = self.loss(logits=model_output, labels=labels) + + if self.task_name != 'sts-b': + model_output = torch.argmax(model_output, 1) + + eval_tensors = {'preds': model_output, 'labels': labels} + return {'val_loss': val_loss, 'eval_tensors': eval_tensors} + + def multi_validation_epoch_end(self, outputs, dataloader_idx: int = 0): + """ + Called at the end of validation to aggregate outputs. + outputs: list of individual outputs of each validation step. + """ + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() + preds = torch.cat([x['eval_tensors']['preds'] for x in outputs]) + labels = torch.cat([x['eval_tensors']['labels'] for x in outputs]) + + all_preds = [] + all_labels = [] + if torch.distributed.is_initialized(): + world_size = torch.distributed.get_world_size() + for ind in range(world_size): + all_preds.append(torch.empty_like(preds)) + all_labels.append(torch.empty_like(labels)) + torch.distributed.all_gather(all_preds, preds) + torch.distributed.all_gather(all_labels, labels) + else: + all_preds.append(preds) + all_labels.append(labels) + + tensorboard_logs = {} + if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + preds = [] + labels = [] + for p in all_preds: + preds.extend(tensor2list(p)) + for l in all_labels: + labels.extend(tensor2list(l)) + + tensorboard_logs = compute_metrics(self.task_name, np.array(preds), np.array(labels)) + val_name = self._validation_names[dataloader_idx].upper() + logging.info(f'{val_name} evaluation: {tensorboard_logs}') + + # writing labels and predictions to a file in output_dir is specified in the config + output_dir = self._cfg.output_dir + if output_dir: + os.makedirs(output_dir, exist_ok=True) + filename = os.path.join(output_dir, f'{self.task_name}_{val_name}.txt') + logging.info(f'Saving labels and predictions to {filename}') + with open(filename, 'w') as f: + f.write('labels\t' + list2str(labels) + '\n') + f.write('preds\t' + list2str(preds) + '\n') + + tensorboard_logs['val_loss'] = avg_loss + for key in tensorboard_logs: + self.log(f'{key}', tensorboard_logs[key], prog_bar=True) + + return {'val_loss': avg_loss, 'log': tensorboard_logs} + + def setup_training_data(self, train_data_config: Optional[DictConfig] = None): + if train_data_config is None: + train_data_config = self._cfg.train_ds + + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config) + + def setup_validation_data(self, val_data_config: Optional[DictConfig] = None): + if val_data_config is None: + val_data_config = self._cfg.validation_ds + + self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config) + + def setup_multiple_validation_data(self, val_data_config: Union[DictConfig, Dict] = None): + if val_data_config is None: + val_data_config = self._cfg.validation_ds + + return super().setup_multiple_validation_data(val_data_config) + + def _setup_dataloader_from_config(self, cfg: DictConfig): + file_name = cfg.ds_item + if not os.path.exists(file_name): + raise FileNotFoundError( + "GLUE datasets not found. For more details on how to get the data, see: " + "https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e" + ) + + dataset = GLUEDataset( + file_name=file_name, + task_name=self.task_name, + tokenizer=self.tokenizer, + max_seq_length=self._cfg.dataset.max_seq_length, + use_cache=self._cfg.dataset.use_cache, + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + collate_fn=dataset.collate_fn, + batch_size=cfg.batch_size, + shuffle=cfg.shuffle, + num_workers=self._cfg.dataset.num_workers, + pin_memory=self._cfg.dataset.pin_memory, + drop_last=self._cfg.dataset.drop_last, + ) + + @classmethod + def list_available_models(cls) -> Optional[Dict[str, str]]: + pass diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/metrics_for_glue.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/metrics_for_glue.py new file mode 100644 index 0000000000..dd4ecada1a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/glue_benchmark/metrics_for_glue.py @@ -0,0 +1,66 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List + +from scipy.stats import pearsonr, spearmanr +from sklearn.metrics import f1_score, matthews_corrcoef + +__all__ = ['compute_metrics'] + + +def accuracy(preds: List[int], labels: List[int]): + return {"acc": (preds == labels).mean()} + + +def acc_and_f1(preds: List[int], labels: List[int]): + accuracy = (preds == labels).mean() + f1 = f1_score(y_true=labels, y_pred=preds) + return {"acc": accuracy, "f1": f1} + + +def mcc(preds: List[int], labels: List[int]): + return {"mcc": matthews_corrcoef(labels, preds)} + + +def pearson_and_spearman(preds: List[int], labels: List[int]): + pearson_corr = pearsonr(preds, labels)[0] + spearman_corr = spearmanr(preds, labels)[0] + return {"pearson": pearson_corr, "spearmanr": spearman_corr, "pear+spear av": (pearson_corr + spearman_corr) / 2} + + +def compute_metrics(task_name: str, preds: List[int], labels: List[int]) -> Dict[str, float]: + """ + Computes metrics for GLUE tasks + Args: + task_name: GLUE task name + preds: model predictions + labels: golden labels + Returns: + metrics + """ + if len(preds) != len(labels): + raise ValueError("Predictions and labels must have the same length") + + metric_fn = accuracy + if task_name == 'cola': + metric_fn = mcc + elif task_name in ['mrpc', 'qqp']: + metric_fn = acc_and_f1 + elif task_name == 'sts-b': + metric_fn = pearson_and_spearman + + return metric_fn(preds, labels) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/__init__.py new file mode 100644 index 0000000000..f07a53c76c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.models.information_retrieval.bert_dpr_model import BertDPRModel +from nemo.collections.nlp.models.information_retrieval.bert_joint_ir_model import BertJointIRModel diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/base_ir_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/base_ir_model.py new file mode 100644 index 0000000000..2bf5c0210d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/base_ir_model.py @@ -0,0 +1,211 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Dict, Optional + +import numpy as np +import torch +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer + +from nemo.collections.nlp.data import BertInformationRetrievalDataset +from nemo.collections.nlp.models.nlp_model import NLPModel +from nemo.collections.nlp.modules.common.lm_utils import get_lm_model +from nemo.core.classes.common import typecheck + +__all__ = ['BaseIRModel'] + + +class BaseIRModel(NLPModel): + """ + Base class for information retrieval models. + """ + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + + self.setup_tokenizer(cfg.tokenizer) + + super().__init__(cfg=cfg, trainer=trainer) + + @typecheck() + def forward(self, *args): + pass + + def compute_scores_and_loss(self, inputs): + pass + + @staticmethod + def get_lm_model_with_padded_embedding(cfg: DictConfig): + """ + Function which ensures that vocabulary size is divisivble by 8 + for faster mixed precision training. + """ + model = get_lm_model( + pretrained_model_name=cfg.language_model.pretrained_model_name, + config_file=cfg.language_model.config_file, + config_dict=OmegaConf.to_container(cfg.language_model.config) if cfg.language_model.config else None, + checkpoint_file=cfg.language_model.lm_checkpoint, + vocab_file=cfg.tokenizer.vocab_file, + ) + vocab_size, hidden_size = model.config.vocab_size, model.config.hidden_size + tokens_to_add = 8 * math.ceil(vocab_size / 8) - vocab_size + zeros = torch.zeros((tokens_to_add, hidden_size)) + model.embeddings.word_embeddings.weight.data = torch.cat((model.embeddings.word_embeddings.weight.data, zeros)) + return model + + @staticmethod + def calculate_mean_reciprocal_rank(query2passages, query2rel): + """ + Helper function which calculates mean reciprocal rank. + Args: + query2passages: dict which contains passage ids and corresponding + scores for each query + query2rel: dict which contains ids of relevant passages for each query + """ + reciprocal_ranks = [] + + for query in query2passages: + indices = np.argsort(query2passages[query]["scores"])[::-1] + sorted_psgs = query2passages[query]["psg_ids"][indices] + + reciprocal_ranks.append(0) + for i, psg_id in enumerate(sorted_psgs): + if psg_id in query2rel[query]: + reciprocal_ranks[-1] = 1 / (i + 1) + break + return np.mean(reciprocal_ranks) + + def training_step(self, batch, batch_idx): + """ + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. + """ + scores, train_loss = self.compute_scores_and_loss(batch[:-2]) + tensorboard_logs = {"train_loss": train_loss, "lr": self._optimizer.param_groups[0]["lr"]} + return {"loss": train_loss, "log": tensorboard_logs} + + def validation_step(self, batch, batch_idx): + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. + """ + scores, val_loss = self.compute_scores_and_loss(batch[:-2]) + query_ids, passage_ids = batch[-2:] + data_for_val = { + "val_loss": val_loss, + "scores": scores, + "query_ids": query_ids, + "passage_ids": passage_ids, + } + return data_for_val + + def validation_epoch_end(self, outputs): + """ + Called at the end of validation to aggregate outputs. + :param outputs: list of individual outputs of each validation step. + """ + + query_ids = torch.cat([x["query_ids"] for x in outputs]) + passage_ids = torch.cat([x["passage_ids"] for x in outputs]) + scores = torch.cat([x["scores"] for x in outputs]) + + all_query_ids, all_passage_ids, all_scores = [], [], [] + if torch.distributed.is_initialized(): + world_size = torch.distributed.get_world_size() + for ind in range(world_size): + all_query_ids.append(torch.empty_like(query_ids)) + all_passage_ids.append(torch.empty_like(passage_ids)) + all_scores.append(torch.empty_like(scores)) + torch.distributed.all_gather(all_query_ids, query_ids) + torch.distributed.all_gather(all_passage_ids, passage_ids) + torch.distributed.all_gather(all_scores, scores) + else: + all_query_ids.append(query_ids) + all_passage_ids.append(passage_ids) + all_scores.append(scores) + + val_mrr = 0 + if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + query2passages, query2rels = {}, {} + processed_queries = set() + + for i in range(len(all_query_ids)): + + query_ids = all_query_ids[i].detach().cpu().numpy() + passage_ids = all_passage_ids[i].detach().cpu().numpy() + scores = all_scores[i].detach().cpu().numpy() + + for j, query_id in enumerate(query_ids): + + if query_id not in processed_queries: + processed_queries.add(query_id) + query2passages[query_id] = { + "psg_ids": passage_ids[j], + "scores": scores[j], + } + query2rels[query_id] = [passage_ids[j][0]] + else: + query2passages[query_id]["psg_ids"] = np.concatenate( + (query2passages[query_id]["psg_ids"], passage_ids[j][1:]) + ) + query2passages[query_id]["scores"] = np.concatenate( + (query2passages[query_id]["scores"], scores[j][1:]) + ) + + val_mrr = self.calculate_mean_reciprocal_rank(query2passages, query2rels) + + val_loss = torch.stack([x["val_loss"] for x in outputs]).mean() + + tensorboard_logs = { + "val_mrr": val_mrr, + "val_loss": val_loss, + } + + return {"log": tensorboard_logs} + + def setup_training_data(self, train_data_config: Optional[DictConfig]): + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config) + + def setup_validation_data(self, val_data_config: Optional[DictConfig]): + self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config) + + def setup_test_data(self, test_data_config: Optional[DictConfig]): + self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config) + + def _setup_dataloader_from_config(self, cfg: DictConfig): + + dataset = BertInformationRetrievalDataset( + tokenizer=self.tokenizer, + passages=cfg.passages, + queries=cfg.queries, + query_to_passages=cfg.query_to_passages, + num_negatives=cfg.num_negatives, + psg_cache_format=cfg.get("psg_cache_format", "pkl"), + max_query_length=cfg.get("max_query_length", 31), + max_passage_length=cfg.get("max_passage_length", 190), + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg.batch_size, + shuffle=cfg.shuffle, + num_workers=cfg.get("num_workers", 2), + pin_memory=cfg.get("pin_memory", False), + drop_last=cfg.get("drop_last", False), + ) + + @classmethod + def list_available_models(cls) -> Optional[Dict[str, str]]: + pass diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/bert_dpr_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/bert_dpr_model.py new file mode 100644 index 0000000000..03b62d9117 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/bert_dpr_model.py @@ -0,0 +1,131 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional + +import torch +from omegaconf import DictConfig +from pytorch_lightning import Trainer + +from nemo.collections.common.losses import SmoothedCrossEntropyLoss +from nemo.collections.nlp.data import BertInformationRetrievalDataset +from nemo.collections.nlp.models.information_retrieval.base_ir_model import BaseIRModel +from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer +from nemo.core.classes.common import typecheck +from nemo.core.neural_types import ChannelType, LogitsType, MaskType, NeuralType + +__all__ = ["BertDPRModel"] + + +class BertDPRModel(BaseIRModel): + """ + Information retrieval model which encodes query and passage separately + with two different BERT encoders and computes their similarity score + as a dot-product between corresponding [CLS] token representations. + """ + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "q_input_ids": NeuralType(("B", "T"), ChannelType()), + "q_attention_mask": NeuralType(("B", "T"), MaskType()), + "q_token_type_ids": NeuralType(("B", "T"), ChannelType()), + "p_input_ids": NeuralType(("B", "T"), ChannelType()), + "p_attention_mask": NeuralType(("B", "T"), MaskType()), + "p_token_type_ids": NeuralType(("B", "T"), ChannelType()), + } + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return {"logits": NeuralType(("B", "D"), LogitsType())} + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + + model_name = cfg.language_model.pretrained_model_name + self.tokenizer = get_tokenizer(tokenizer_name=model_name) + + super().__init__(cfg=cfg, trainer=trainer) + + self.q_encoder = self.get_lm_model_with_padded_embedding(cfg) + self.p_encoder = self.get_lm_model_with_padded_embedding(cfg) + self.loss = SmoothedCrossEntropyLoss(pad_id=self.tokenizer.pad_id) + + @typecheck() + def forward( + self, q_input_ids, q_token_type_ids, q_attention_mask, p_input_ids, p_token_type_ids, p_attention_mask, + ): + + q_vectors = self.q_encoder( + input_ids=q_input_ids, token_type_ids=q_token_type_ids, attention_mask=q_attention_mask, + ) + q_vectors = q_vectors[:, 0] + batch_size, hidden_size = q_vectors.size() + + p_vectors = self.p_encoder( + input_ids=p_input_ids, token_type_ids=p_token_type_ids, attention_mask=p_attention_mask, + ) + num_passages = p_vectors.shape[0] // batch_size + p_vectors = p_vectors[:, 0].view(-1, num_passages, hidden_size) + p_positives, p_negatives = p_vectors[:, 0], p_vectors[:, 1:] + scores = torch.cat( + (torch.matmul(q_vectors, p_positives.T), torch.einsum("ij,ipj->ip", q_vectors, p_negatives),), dim=1, + ) + + return scores + + def compute_scores_and_loss(self, inputs): + (q_input_ids, q_input_mask, q_input_type_ids, p_input_ids, p_input_mask, p_input_type_ids,) = inputs + batch_size, num_passages, p_seq_length = p_input_ids.size() + q_seq_length = q_input_ids.size()[-1] + + scores = self( + q_input_ids=q_input_ids.view(-1, q_seq_length), + q_token_type_ids=q_input_type_ids.view(-1, q_seq_length), + q_attention_mask=q_input_mask.view(-1, q_seq_length), + p_input_ids=p_input_ids.view(-1, p_seq_length), + p_token_type_ids=p_input_type_ids.view(-1, p_seq_length), + p_attention_mask=p_input_mask.view(-1, p_seq_length), + ).view(batch_size, 1, batch_size + num_passages - 1) + normalized_scores = torch.log_softmax(scores, dim=-1) + + labels = torch.arange(batch_size)[:, None].long().to(normalized_scores.device) + loss = self.loss(log_probs=normalized_scores, labels=labels, output_mask=torch.ones_like(labels),) + + scores = scores[:, 0] + scores = torch.cat((torch.diag(scores)[:, None], scores[:, batch_size:]), dim=1,) + + return scores, loss + + def _setup_dataloader_from_config(self, cfg: DictConfig): + + dataset = BertInformationRetrievalDataset( + tokenizer=self.tokenizer, + passages=cfg.passages, + queries=cfg.queries, + query_to_passages=cfg.query_to_passages, + num_negatives=cfg.num_negatives, + psg_cache_format=cfg.get("psg_cache_format", "pkl"), + max_query_length=cfg.get("max_query_length", 31), + max_passage_length=cfg.get("max_passage_length", 190), + preprocess_fn="preprocess_dpr", + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg.batch_size, + shuffle=cfg.shuffle, + num_workers=cfg.get("num_workers", 2), + pin_memory=cfg.get("pin_memory", False), + drop_last=cfg.get("drop_last", False), + ) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/bert_joint_ir_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/bert_joint_ir_model.py new file mode 100644 index 0000000000..e474e594d3 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/information_retrieval/bert_joint_ir_model.py @@ -0,0 +1,84 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional + +import torch +from omegaconf import DictConfig +from pytorch_lightning import Trainer + +from nemo.collections.common.losses import SmoothedCrossEntropyLoss +from nemo.collections.nlp.models.information_retrieval.base_ir_model import BaseIRModel +from nemo.collections.nlp.modules.common import SequenceRegression +from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer +from nemo.core.classes.common import typecheck +from nemo.core.neural_types import NeuralType + +__all__ = ["BertJointIRModel"] + + +class BertJointIRModel(BaseIRModel): + """ + Information retrieval model which jointly encodes both query and passage + and passes them to BERT encoder followed by a fully-connected layer for + similarity score prediction. + """ + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return self.bert_model.input_types + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return self.sim_score_regressor.output_types + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + + model_name = cfg.language_model.pretrained_model_name + self.tokenizer = get_tokenizer(tokenizer_name=model_name) + + super().__init__(cfg=cfg, trainer=trainer) + + self.bert_model = self.get_lm_model_with_padded_embedding(cfg) + hidden_size = self.bert_model.config.hidden_size + self.sim_score_regressor = SequenceRegression( + hidden_size=hidden_size, num_layers=1, dropout=cfg.language_model.sim_score_dropout, + ) + self.loss = SmoothedCrossEntropyLoss(pad_id=self.tokenizer.pad_id) + + @typecheck() + def forward(self, input_ids, token_type_ids, attention_mask): + + hidden_states = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask, + ) + scores = self.sim_score_regressor(hidden_states=hidden_states) + + return scores + + def compute_scores_and_loss(self, inputs): + input_ids, input_mask, input_type_ids = inputs + batch_size, num_passages, seq_length = input_ids.size() + + unnormalized_scores = self( + input_ids=input_ids.view(-1, seq_length), + token_type_ids=input_type_ids.view(-1, seq_length), + attention_mask=input_mask.view(-1, seq_length), + ).view(batch_size, 1, num_passages) + scores = torch.log_softmax(unnormalized_scores, dim=-1) + + labels = torch.zeros_like(input_ids[:, :1, 0]) + loss = self.loss(log_probs=scores, labels=labels, output_mask=torch.ones_like(labels)) + + return unnormalized_scores[:, 0], loss diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/intent_slot_classification/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/intent_slot_classification/__init__.py new file mode 100644 index 0000000000..081dd1cc49 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/intent_slot_classification/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.models.intent_slot_classification.intent_slot_classification_model import ( + IntentSlotClassificationModel, +) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/intent_slot_classification/intent_slot_classification_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/intent_slot_classification/intent_slot_classification_model.py new file mode 100644 index 0000000000..4a230f2c15 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/intent_slot_classification/intent_slot_classification_model.py @@ -0,0 +1,487 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Dict, List, Optional + +import onnx +import torch +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer +from torch.utils.data import DataLoader + +from nemo.collections.common.losses import AggregatorLoss, CrossEntropyLoss +from nemo.collections.nlp.data.intent_slot_classification import ( + IntentSlotClassificationDataset, + IntentSlotDataDesc, + IntentSlotInferenceDataset, +) +from nemo.collections.nlp.metrics.classification_report import ClassificationReport +from nemo.collections.nlp.models.nlp_model import NLPModel +from nemo.collections.nlp.modules.common import SequenceTokenClassifier +from nemo.collections.nlp.modules.common.lm_utils import get_lm_model +from nemo.collections.nlp.parts.utils_funcs import tensor2list +from nemo.core.classes import typecheck +from nemo.core.classes.common import PretrainedModelInfo +from nemo.core.neural_types import NeuralType +from nemo.utils import logging + + +class IntentSlotClassificationModel(NLPModel): + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return self.bert_model.input_types + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return self.classifier.output_types + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + """ Initializes BERT Joint Intent and Slot model. + """ + self.max_seq_length = cfg.language_model.max_seq_length + + # Setup tokenizer. + self.setup_tokenizer(cfg.tokenizer) + + # Check the presence of data_dir. + if not cfg.data_dir or not os.path.exists(cfg.data_dir): + # Disable setup methods. + IntentSlotClassificationModel._set_model_restore_state(is_being_restored=True) + # Set default values of data_desc. + self._set_defaults_data_desc(cfg) + else: + self.data_dir = cfg.data_dir + # Update configuration of data_desc. + self._set_data_desc_to_cfg(cfg, cfg.data_dir, cfg.train_ds, cfg.validation_ds) + + # init superclass + super().__init__(cfg=cfg, trainer=trainer) + + # Enable setup methods. + IntentSlotClassificationModel._set_model_restore_state(is_being_restored=False) + + # Initialize Bert model + self.bert_model = get_lm_model( + pretrained_model_name=self.cfg.language_model.pretrained_model_name, + config_file=self.register_artifact('language_model.config_file', cfg.language_model.config_file), + config_dict=OmegaConf.to_container(self.cfg.language_model.config) + if self.cfg.language_model.config + else None, + checkpoint_file=self.cfg.language_model.lm_checkpoint, + vocab_file=self.register_artifact('tokenizer.vocab_file', cfg.tokenizer.vocab_file), + ) + + # Initialize Classifier. + self._reconfigure_classifier() + + def _set_defaults_data_desc(self, cfg): + """ + Method makes sure that cfg.data_desc params are set. + If not, set's them to "dummy" defaults. + """ + if not hasattr(cfg, "data_desc"): + OmegaConf.set_struct(cfg, False) + cfg.data_desc = {} + # Intents. + cfg.data_desc.intent_labels = " " + cfg.data_desc.intent_label_ids = {" ": 0} + cfg.data_desc.intent_weights = [1] + # Slots. + cfg.data_desc.slot_labels = " " + cfg.data_desc.slot_label_ids = {" ": 0} + cfg.data_desc.slot_weights = [1] + + cfg.data_desc.pad_label = "O" + OmegaConf.set_struct(cfg, True) + + def _set_data_desc_to_cfg(self, cfg, data_dir, train_ds, validation_ds): + """ Method creates IntentSlotDataDesc and copies generated values to cfg.data_desc. """ + # Save data from data desc to config - so it can be reused later, e.g. in inference. + data_desc = IntentSlotDataDesc(data_dir=data_dir, modes=[train_ds.prefix, validation_ds.prefix]) + OmegaConf.set_struct(cfg, False) + if not hasattr(cfg, "data_desc") or cfg.data_desc is None: + cfg.data_desc = {} + # Intents. + cfg.data_desc.intent_labels = list(data_desc.intents_label_ids.keys()) + cfg.data_desc.intent_label_ids = data_desc.intents_label_ids + cfg.data_desc.intent_weights = data_desc.intent_weights + # Slots. + cfg.data_desc.slot_labels = list(data_desc.slots_label_ids.keys()) + cfg.data_desc.slot_label_ids = data_desc.slots_label_ids + cfg.data_desc.slot_weights = data_desc.slot_weights + + cfg.data_desc.pad_label = data_desc.pad_label + + # for older(pre - 1.0.0.b3) configs compatibility + if not hasattr(cfg, "class_labels") or cfg.class_labels is None: + cfg.class_labels = {} + cfg.class_labels = OmegaConf.create( + {'intent_labels_file': 'intent_labels.csv', 'slot_labels_file': 'slot_labels.csv'} + ) + + slot_labels_file = os.path.join(data_dir, cfg.class_labels.slot_labels_file) + intent_labels_file = os.path.join(data_dir, cfg.class_labels.intent_labels_file) + self._save_label_ids(data_desc.slots_label_ids, slot_labels_file) + self._save_label_ids(data_desc.intents_label_ids, intent_labels_file) + + self.register_artifact('class_labels.intent_labels_file', intent_labels_file) + self.register_artifact('class_labels.slot_labels_file', slot_labels_file) + OmegaConf.set_struct(cfg, True) + + def _save_label_ids(self, label_ids: Dict[str, int], filename: str) -> None: + """ Saves label ids map to a file """ + with open(filename, 'w') as out: + labels, _ = zip(*sorted(label_ids.items(), key=lambda x: x[1])) + out.write('\n'.join(labels)) + logging.info(f'Labels: {label_ids}') + logging.info(f'Labels mapping saved to : {out.name}') + + def _reconfigure_classifier(self): + """ Method reconfigures the classifier depending on the settings of model cfg.data_desc """ + + self.classifier = SequenceTokenClassifier( + hidden_size=self.bert_model.config.hidden_size, + num_intents=len(self.cfg.data_desc.intent_labels), + num_slots=len(self.cfg.data_desc.slot_labels), + dropout=self.cfg.head.fc_dropout, + num_layers=self.cfg.head.num_output_layers, + log_softmax=False, + ) + + # define losses + if self.cfg.class_balancing == 'weighted_loss': + # You may need to increase the number of epochs for convergence when using weighted_loss + self.intent_loss = CrossEntropyLoss(logits_ndim=2, weight=self.cfg.data_desc.intent_weights) + self.slot_loss = CrossEntropyLoss(logits_ndim=3, weight=self.cfg.data_desc.slot_weights) + else: + self.intent_loss = CrossEntropyLoss(logits_ndim=2) + self.slot_loss = CrossEntropyLoss(logits_ndim=3) + + self.total_loss = AggregatorLoss( + num_inputs=2, weights=[self.cfg.intent_loss_weight, 1.0 - self.cfg.intent_loss_weight] + ) + + # setup to track metrics + self.intent_classification_report = ClassificationReport( + num_classes=len(self.cfg.data_desc.intent_labels), + label_ids=self.cfg.data_desc.intent_label_ids, + dist_sync_on_step=True, + mode='micro', + ) + self.slot_classification_report = ClassificationReport( + num_classes=len(self.cfg.data_desc.slot_labels), + label_ids=self.cfg.data_desc.slot_label_ids, + dist_sync_on_step=True, + mode='micro', + ) + + def update_data_dir_for_training(self, data_dir: str, train_ds, validation_ds) -> None: + """ + Update data directory and get data stats with Data Descriptor. + Also, reconfigures the classifier - to cope with data with e.g. different number of slots. + + Args: + data_dir: path to data directory + """ + logging.info(f'Setting data_dir to {data_dir}.') + self.data_dir = data_dir + # Update configuration with new data. + self._set_data_desc_to_cfg(self.cfg, data_dir, train_ds, validation_ds) + # Reconfigure the classifier for different settings (number of intents, slots etc.). + self._reconfigure_classifier() + + def update_data_dir_for_testing(self, data_dir) -> None: + """ + Update data directory. + + Args: + data_dir: path to data directory + """ + logging.info(f'Setting data_dir to {data_dir}.') + self.data_dir = data_dir + + @typecheck() + def forward(self, input_ids, token_type_ids, attention_mask): + """ + No special modification required for Lightning, define it as you normally would + in the `nn.Module` in vanilla PyTorch. + """ + hidden_states = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask + ) + intent_logits, slot_logits = self.classifier(hidden_states=hidden_states) + return intent_logits, slot_logits + + def training_step(self, batch, batch_idx): + """ + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. + """ + # forward pass + input_ids, input_type_ids, input_mask, loss_mask, subtokens_mask, intent_labels, slot_labels = batch + intent_logits, slot_logits = self( + input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask + ) + + # calculate combined loss for intents and slots + intent_loss = self.intent_loss(logits=intent_logits, labels=intent_labels) + slot_loss = self.slot_loss(logits=slot_logits, labels=slot_labels, loss_mask=loss_mask) + train_loss = self.total_loss(loss_1=intent_loss, loss_2=slot_loss) + lr = self._optimizer.param_groups[0]['lr'] + + self.log('train_loss', train_loss) + self.log('lr', lr, prog_bar=True) + + return { + 'loss': train_loss, + 'lr': lr, + } + + def validation_step(self, batch, batch_idx): + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. + """ + input_ids, input_type_ids, input_mask, loss_mask, subtokens_mask, intent_labels, slot_labels = batch + intent_logits, slot_logits = self( + input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask + ) + + # calculate combined loss for intents and slots + intent_loss = self.intent_loss(logits=intent_logits, labels=intent_labels) + slot_loss = self.slot_loss(logits=slot_logits, labels=slot_labels, loss_mask=loss_mask) + val_loss = self.total_loss(loss_1=intent_loss, loss_2=slot_loss) + + # calculate accuracy metrics for intents and slot reporting + # intents + preds = torch.argmax(intent_logits, axis=-1) + self.intent_classification_report.update(preds, intent_labels) + # slots + subtokens_mask = subtokens_mask > 0.5 + preds = torch.argmax(slot_logits, axis=-1)[subtokens_mask] + slot_labels = slot_labels[subtokens_mask] + self.slot_classification_report.update(preds, slot_labels) + + return { + 'val_loss': val_loss, + 'intent_tp': self.intent_classification_report.tp, + 'intent_fn': self.intent_classification_report.fn, + 'intent_fp': self.intent_classification_report.fp, + 'slot_tp': self.slot_classification_report.tp, + 'slot_fn': self.slot_classification_report.fn, + 'slot_fp': self.slot_classification_report.fp, + } + + def validation_epoch_end(self, outputs): + """ + Called at the end of validation to aggregate outputs. + :param outputs: list of individual outputs of each validation step. + """ + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() + + # calculate metrics and log classification report (separately for intents and slots) + intent_precision, intent_recall, intent_f1, intent_report = self.intent_classification_report.compute() + logging.info(f'Intent report: {intent_report}') + + slot_precision, slot_recall, slot_f1, slot_report = self.slot_classification_report.compute() + logging.info(f'Slot report: {slot_report}') + + self.log('val_loss', avg_loss) + self.log('intent_precision', intent_precision) + self.log('intent_recall', intent_recall) + self.log('intent_f1', intent_f1) + self.log('slot_precision', slot_precision) + self.log('slot_recall', slot_recall) + self.log('slot_f1', slot_f1) + + self.intent_classification_report.reset() + self.slot_classification_report.reset() + + return { + 'val_loss': avg_loss, + 'intent_precision': intent_precision, + 'intent_recall': intent_recall, + 'intent_f1': intent_f1, + 'slot_precision': slot_precision, + 'slot_recall': slot_recall, + 'slot_f1': slot_f1, + } + + def test_step(self, batch, batch_idx): + """ + Lightning calls this inside the test loop with the data from the test dataloader + passed in as `batch`. + """ + return self.validation_step(batch, batch_idx) + + def test_epoch_end(self, outputs): + """ + Called at the end of test to aggregate outputs. + :param outputs: list of individual outputs of each test step. + """ + return self.validation_epoch_end(outputs) + + def setup_training_data(self, train_data_config: Optional[DictConfig]): + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config) + + def setup_validation_data(self, val_data_config: Optional[DictConfig]): + self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config) + + def setup_test_data(self, test_data_config: Optional[DictConfig]): + self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config) + + def _setup_dataloader_from_config(self, cfg: DictConfig): + input_file = f'{self.data_dir}/{cfg.prefix}.tsv' + slot_file = f'{self.data_dir}/{cfg.prefix}_slots.tsv' + + if not (os.path.exists(input_file) and os.path.exists(slot_file)): + raise FileNotFoundError( + f'{input_file} or {slot_file} not found. Please refer to the documentation for the right format \ + of Intents and Slots files.' + ) + + dataset = IntentSlotClassificationDataset( + input_file=input_file, + slot_file=slot_file, + tokenizer=self.tokenizer, + max_seq_length=self.max_seq_length, + num_samples=cfg.num_samples, + pad_label=self.cfg.data_desc.pad_label, + ignore_extra_tokens=self.cfg.ignore_extra_tokens, + ignore_start_end=self.cfg.ignore_start_end, + ) + + return DataLoader( + dataset=dataset, + batch_size=cfg.batch_size, + shuffle=cfg.shuffle, + num_workers=cfg.num_workers, + pin_memory=cfg.pin_memory, + drop_last=cfg.drop_last, + collate_fn=dataset.collate_fn, + ) + + def _setup_infer_dataloader(self, queries: List[str], test_ds) -> 'torch.utils.data.DataLoader': + """ + Setup function for a infer data loader. + Args: + queries: text + batch_size: batch size to use during inference + Returns: + A pytorch DataLoader. + """ + + dataset = IntentSlotInferenceDataset( + tokenizer=self.tokenizer, queries=queries, max_seq_length=-1, do_lower_case=False + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + collate_fn=dataset.collate_fn, + batch_size=test_ds.batch_size, + shuffle=test_ds.shuffle, + num_workers=test_ds.num_workers, + pin_memory=test_ds.pin_memory, + drop_last=test_ds.drop_last, + ) + + def predict_from_examples(self, queries: List[str], test_ds) -> List[List[str]]: + """ + Get prediction for the queries (intent and slots) + Args: + queries: text sequences + test_ds: Dataset configuration section. + Returns: + predicted_intents, predicted_slots: model intent and slot predictions + """ + predicted_intents = [] + predicted_slots = [] + mode = self.training + try: + device = 'cuda' if torch.cuda.is_available() else 'cpu' + + # Retrieve intent and slot vocabularies from configuration. + intent_labels = self.cfg.data_desc.intent_labels + slot_labels = self.cfg.data_desc.slot_labels + + # Initialize tokenizer. + # if not hasattr(self, "tokenizer"): + # self._setup_tokenizer(self.cfg.tokenizer) + # Initialize modules. + # self._reconfigure_classifier() + + # Switch model to evaluation mode + self.eval() + self.to(device) + + # Dataset. + infer_datalayer = self._setup_infer_dataloader(queries, test_ds) + + for batch in infer_datalayer: + input_ids, input_type_ids, input_mask, loss_mask, subtokens_mask = batch + + intent_logits, slot_logits = self.forward( + input_ids=input_ids.to(device), + token_type_ids=input_type_ids.to(device), + attention_mask=input_mask.to(device), + ) + + # predict intents and slots for these examples + # intents + intent_preds = tensor2list(torch.argmax(intent_logits, axis=-1)) + + # convert numerical outputs to Intent and Slot labels from the dictionaries + for intent_num in intent_preds: + if intent_num < len(intent_labels): + predicted_intents.append(intent_labels[int(intent_num)]) + else: + # should not happen + predicted_intents.append("Unknown Intent") + + # slots + slot_preds = torch.argmax(slot_logits, axis=-1) + + for slot_preds_query, mask_query in zip(slot_preds, subtokens_mask): + query_slots = '' + for slot, mask in zip(slot_preds_query, mask_query): + if mask == 1: + if slot < len(slot_labels): + query_slots += slot_labels[int(slot)] + ' ' + else: + query_slots += 'Unknown_slot ' + predicted_slots.append(query_slots.strip()) + + finally: + # set mode back to its original value + self.train(mode=mode) + + return predicted_intents, predicted_slots + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + result = [] + model = PretrainedModelInfo( + pretrained_model_name="Joint_Intent_Slot_Assistant", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemonlpmodels/versions/1.0.0a5/files/Joint_Intent_Slot_Assistant.nemo", + description="This models is trained on this https://github.com/xliuhw/NLU-Evaluation-Data dataset which includes 64 various intents and 55 slots. Final Intent accuracy is about 87%, Slot accuracy is about 89%.", + ) + result.append(model) + return result diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/__init__.py new file mode 100644 index 0000000000..408e9c858a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.models.language_modeling.bert_lm_model import BERTLMModel +from nemo.collections.nlp.models.language_modeling.transformer_lm_model import TransformerLMModel diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/bert_lm_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/bert_lm_model.py new file mode 100644 index 0000000000..f1e3ab1b0a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/bert_lm_model.py @@ -0,0 +1,272 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Dict, Optional + +import torch +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer +from torch.utils.data import DataLoader + +from nemo.collections.common.losses import AggregatorLoss, CrossEntropyLoss, SmoothedCrossEntropyLoss +from nemo.collections.common.metrics import Perplexity +from nemo.collections.nlp.data.language_modeling.lm_bert_dataset import ( + BertPretrainingDataset, + BertPretrainingPreprocessedDataloader, +) +from nemo.collections.nlp.modules.common import BertPretrainingTokenClassifier, SequenceClassifier +from nemo.collections.nlp.modules.common.lm_utils import get_lm_model +from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.classes.modelPT import ModelPT +from nemo.core.neural_types import NeuralType +from nemo.utils import logging + +__all__ = ["BERTLMModel"] + + +class BERTLMModel(ModelPT): + """ + BERT language model pretraining. + """ + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return self.bert_model.input_types + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + output_types_dict = {"mlm_log_probs": self.mlm_classifier.output_types["log_probs"]} + if not self.only_mlm_loss: + output_types_dict["nsp_logits"] = self.nsp_classifier.output_types["logits"] + return output_types_dict + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + + if cfg.tokenizer is not None: + self._setup_tokenizer(cfg.tokenizer) + else: + self.tokenizer = None + + super().__init__(cfg=cfg, trainer=trainer) + + self.bert_model = get_lm_model( + pretrained_model_name=cfg.language_model.pretrained_model_name, + config_file=self.register_artifact('language_model.config_file', cfg.language_model.config_file), + config_dict=OmegaConf.to_container(cfg.language_model.config) if cfg.language_model.config else None, + checkpoint_file=cfg.language_model.lm_checkpoint, + vocab_file=self.register_artifact('tokenizer.vocab_file', cfg.tokenizer.vocab_file) + if cfg.tokenizer + else None, + ) + + self.hidden_size = self.bert_model.config.hidden_size + self.vocab_size = self.bert_model.config.vocab_size + self.only_mlm_loss = cfg.only_mlm_loss + + self.mlm_classifier = BertPretrainingTokenClassifier( + hidden_size=self.hidden_size, + num_classes=self.vocab_size, + num_layers=cfg.num_tok_classification_layers, + activation="gelu", + log_softmax=True, + use_transformer_init=True, + ) + + self.mlm_loss = SmoothedCrossEntropyLoss() + + if not self.only_mlm_loss: + self.nsp_classifier = SequenceClassifier( + hidden_size=self.hidden_size, + num_classes=2, + num_layers=cfg.num_seq_classification_layers, + log_softmax=False, + activation="tanh", + use_transformer_init=True, + ) + + self.nsp_loss = CrossEntropyLoss() + self.agg_loss = AggregatorLoss(num_inputs=2) + + # # tie weights of MLM softmax layer and embedding layer of the encoder + if ( + self.mlm_classifier.mlp.last_linear_layer.weight.shape + != self.bert_model.embeddings.word_embeddings.weight.shape + ): + raise ValueError("Final classification layer does not match embedding layer.") + self.mlm_classifier.mlp.last_linear_layer.weight = self.bert_model.embeddings.word_embeddings.weight + # create extra bias + + # setup to track metrics + self.validation_perplexity = Perplexity(compute_on_step=False) + + self.setup_optimization(cfg.optim) + + @typecheck() + def forward(self, input_ids, token_type_ids, attention_mask): + """ + No special modification required for Lightning, define it as you normally would + in the `nn.Module` in vanilla PyTorch. + """ + hidden_states = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask, + ) + mlm_log_probs = self.mlm_classifier(hidden_states=hidden_states) + if self.only_mlm_loss: + return (mlm_log_probs,) + nsp_logits = self.nsp_classifier(hidden_states=hidden_states) + return mlm_log_probs, nsp_logits + + def _compute_losses(self, mlm_log_probs, nsp_logits, output_ids, output_mask, labels): + mlm_loss = self.mlm_loss(log_probs=mlm_log_probs, labels=output_ids, output_mask=output_mask) + if self.only_mlm_loss: + loss, nsp_loss = mlm_loss, None + else: + nsp_loss = self.nsp_loss(logits=nsp_logits, labels=labels) + loss = self.agg_loss(loss_1=mlm_loss, loss_2=nsp_loss) + return mlm_loss, nsp_loss, loss + + def _parse_forward_outputs(self, forward_outputs): + if self.only_mlm_loss: + return forward_outputs[0], None + else: + return forward_outputs + + def training_step(self, batch, batch_idx): + """ + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. + """ + input_ids, input_type_ids, input_mask, output_ids, output_mask, labels = batch + forward_outputs = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + mlm_log_probs, nsp_logits = self._parse_forward_outputs(forward_outputs) + _, _, loss = self._compute_losses(mlm_log_probs, nsp_logits, output_ids, output_mask, labels) + lr = self._optimizer.param_groups[0]['lr'] + self.log('train_loss', loss) + self.log('lr', lr, prog_bar=True) + return {"loss": loss, "lr": lr} + + def validation_step(self, batch, batch_idx): + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. + """ + input_ids, input_type_ids, input_mask, output_ids, output_mask, labels = batch + forward_outputs = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + mlm_log_probs, nsp_logits = self._parse_forward_outputs(forward_outputs) + _, _, loss = self._compute_losses(mlm_log_probs, nsp_logits, output_ids, output_mask, labels) + self.validation_perplexity(logits=mlm_log_probs) + return {'val_loss': loss} + + def validation_epoch_end(self, outputs): + """Called at the end of validation to aggregate outputs. + + Args: + outputs (list): The individual outputs of each validation step. + + Returns: + dict: Validation loss and tensorboard logs. + """ + if outputs: + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() + perplexity = self.validation_perplexity.compute() + logging.info(f"evaluation perplexity {perplexity.cpu().item()}") + self.log(f'val_loss', avg_loss) + + def setup_training_data(self, train_data_config: Optional[DictConfig]): + self._train_dl = ( + self._setup_preprocessed_dataloader(train_data_config) + if self.tokenizer is None + else self._setup_dataloader(train_data_config) + ) + + def setup_validation_data(self, val_data_config: Optional[DictConfig]): + self._validation_dl = ( + self._setup_preprocessed_dataloader(val_data_config) + if self.tokenizer is None + else self._setup_dataloader(val_data_config) + ) + + def setup_test_data(self, test_data_config: Optional[DictConfig]): + pass + + def _setup_preprocessed_dataloader(self, cfg: Optional[DictConfig]): + dataset = cfg.data_file + max_predictions_per_seq = cfg.max_predictions_per_seq + batch_size = cfg.batch_size + + if os.path.isdir(dataset): + files = [os.path.join(dataset, f) for f in os.listdir(dataset) if os.path.isfile(os.path.join(dataset, f))] + else: + files = [dataset] + files.sort() + dl = BertPretrainingPreprocessedDataloader( + data_files=files, max_predictions_per_seq=max_predictions_per_seq, batch_size=batch_size, + ) + return dl + + def _setup_tokenizer(self, cfg: DictConfig): + tokenizer = get_tokenizer( + tokenizer_name=cfg.tokenizer_name, + tokenizer_model=cfg.tokenizer_model, + special_tokens=OmegaConf.to_container(cfg.special_tokens) if cfg.special_tokens else None, + vocab_file=cfg.vocab_file, + ) + self.tokenizer = tokenizer + + def _setup_dataloader(self, cfg: DictConfig): + dataset = BertPretrainingDataset( + tokenizer=self.tokenizer, + data_file=cfg.data_file, + max_seq_length=cfg.max_seq_length, + mask_prob=cfg.mask_prob, + short_seq_prob=cfg.short_seq_prob, + ) + dl = torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg.batch_size, + collate_fn=dataset.collate_fn, + drop_last=cfg.get("drop_last", False), + shuffle=cfg.shuffle, + num_workers=cfg.get("num_workers", 0), + ) + return dl + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + result = [] + + result.append( + PretrainedModelInfo( + pretrained_model_name="bertbaseuncased", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/bertbaseuncased/versions/1.0.0rc1/files/bertbaseuncased.nemo", + description="The model was trained EN Wikipedia and BookCorpus on a sequence length of 512.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="bertlargeuncased", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/bertlargeuncased/versions/1.0.0rc1/files/bertlargeuncased.nemo", + description="The model was trained EN Wikipedia and BookCorpus on a sequence length of 512.", + ) + ) + return result diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/transformer_lm_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/transformer_lm_model.py new file mode 100644 index 0000000000..b10798560d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/language_modeling/transformer_lm_model.py @@ -0,0 +1,276 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import math +from typing import Dict, Optional + +import numpy as np +import torch +import torch.utils.data as pt_data +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer + +from nemo.collections.common.losses import SmoothedCrossEntropyLoss +from nemo.collections.common.metrics import GlobalAverageLossMetric +from nemo.collections.common.parts import transformer_weights_init +from nemo.collections.nlp.data import SentenceDataset, TarredSentenceDataset +from nemo.collections.nlp.metrics import SequencePerplexity +from nemo.collections.nlp.modules.common import TokenClassifier +from nemo.collections.nlp.modules.common.lm_utils import get_transformer +from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer +from nemo.core.classes.common import typecheck +from nemo.core.classes.modelPT import ModelPT +from nemo.utils import logging, model_utils + +__all__ = ["TransformerLMModel"] + + +class TransformerLMModel(ModelPT): + """ + Left-to-right Transformer language model. + """ + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + + # Get global rank and total number of GPU workers for IterableDataset partitioning, if applicable + self.world_size = 1 + if trainer is not None: + self.world_size = trainer.num_nodes * trainer.num_gpus + + cfg = model_utils.convert_model_config_to_dict_config(cfg) + cfg = model_utils.maybe_update_config_version(cfg) + + # Instantiates tokenizer and register to be saved with NeMo Model archive + # After this call, ther will be self.tokenizer which can convert between tokens and token_ids. + self.setup_tokenizer( + tokenizer_name=cfg.tokenizer.get("tokenizer_name", "yttm"), + tokenizer_model=cfg.tokenizer.get("tokenizer_model", None), + vocab_file=cfg.tokenizer.get("vocab_file", None), + bpe_dropout=cfg.tokenizer.get("bpe_dropout", 0.0), + ) + + # init superclass + super().__init__(cfg=cfg, trainer=trainer) + + # make vocabulary size divisible by 8 for fast fp16 training + vocab_size = 8 * math.ceil(self.tokenizer.vocab_size / 8) + + # encoder from NeMo, Megatron-LM, or HuggingFace + encoder_cfg_dict = OmegaConf.to_container(cfg.get('encoder')) + encoder_cfg_dict['vocab_size'] = vocab_size + library = encoder_cfg_dict.pop('library', 'nemo') + model_name = encoder_cfg_dict.pop('model_name', None) + pretrained = encoder_cfg_dict.pop('pretrained', False) + self.encoder = get_transformer( + library=library, model_name=model_name, pretrained=pretrained, config_dict=encoder_cfg_dict, encoder=True, + ) + + self.log_softmax = TokenClassifier( + hidden_size=self.encoder.hidden_size, + num_classes=vocab_size, + activation=cfg.head.activation, + log_softmax=cfg.head.log_softmax, + dropout=cfg.head.dropout, + use_transformer_init=cfg.head.use_transformer_init, + ) + + # tie weights of embedding and softmax matrices + self.log_softmax.mlp.layer0.weight = self.encoder.embedding.token_embedding.weight + + std_init_range = 1 / self.encoder.hidden_size ** 0.5 + + # initialize weights if not using pretrained encoder + if not self._cfg.encoder.get('pretrained', False): + self.encoder.apply(lambda module: transformer_weights_init(module, std_init_range)) + + self.log_softmax.apply(lambda module: transformer_weights_init(module, std_init_range)) + + self.loss_fn = SmoothedCrossEntropyLoss(pad_id=self.tokenizer.pad_id, label_smoothing=cfg.label_smoothing) + self.eval_loss_fn = SmoothedCrossEntropyLoss(pad_id=self.tokenizer.pad_id) + self.eval_loss = GlobalAverageLossMetric(dist_sync_on_step=False, take_avg_loss=True) + self.eval_ppl = SequencePerplexity() + + @typecheck() + def forward(self, input_ids, attention_mask): + """ + No special modification required for Lightning, define it as you normally would + in the `nn.Module` in vanilla PyTorch. + """ + + hidden_states = self.encoder(input_ids=input_ids, encoder_mask=attention_mask) + log_probs = self.log_softmax(hidden_states=hidden_states) + + return log_probs + + def training_step(self, batch, batch_idx): + """ + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. + """ + # forward pass + for i in range(len(batch)): + if batch[i].ndim == 3: + # Dataset returns already batched data and the first dimension of size 1 + # added by DataLoader is excess. + batch[i] = batch[i].squeeze(dim=0) + ids, mask = batch + input_ids, labels = ids[:, :-1], ids[:, 1:] + input_mask = mask[:, :-1] + log_probs = self(input_ids=input_ids, attention_mask=input_mask) + + train_loss = self.loss_fn(log_probs=log_probs, labels=labels) + + tensorboard_logs = { + "train_loss": train_loss, + "lr": self._optimizer.param_groups[0]["lr"], + } + return {"loss": train_loss, "log": tensorboard_logs} + + def eval_step(self, batch, batch_idx): + for i in range(len(batch)): + if batch[i].ndim == 3: + # Dataset returns already batched data and the first dimension of size 1 + # added by DataLoader is excess. + batch[i] = batch[i].squeeze(dim=0) + ids, mask = batch + input_ids, labels = ids[:, :-1], ids[:, 1:] + input_mask, output_mask = mask[:, :-1], mask[:, 1:] + log_probs = self(input_ids=input_ids, attention_mask=input_mask) + eval_loss = self.eval_loss_fn(log_probs=log_probs, labels=labels) + + self.eval_loss(loss=eval_loss, num_measurements=log_probs.shape[0] * log_probs.shape[1]) + self.eval_ppl(log_probs=log_probs, labels=labels, mask=output_mask) + return {} + + def test_step(self, batch, batch_idx): + return self.eval_step(batch, batch_idx) + + def validation_step(self, batch, batch_idx): + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. + """ + return self.eval_step(batch, batch_idx) + + def eval_epoch_end(self, outputs, mode): + eval_loss = self.eval_loss.compute() + eval_ppl = self.eval_ppl.compute() + ans = {f"{mode}_loss": eval_loss, f"{mode}_ppl": eval_ppl} + ans['log'] = dict(ans) + dataset_name = "Validation" if mode == 'val' else "Test" + logging.info(f"\n\n\n\n{dataset_name} PPL: {np.round(eval_ppl.item(), 2)}") + return ans + + def validation_epoch_end(self, outputs): + """ + Called at the end of validation to aggregate outputs. + :param outputs: list of individual outputs of each validation step. + """ + self.log_dict(self.eval_epoch_end(outputs, 'val'), sync_dist=True) + self.eval_loss.reset() + self.eval_ppl.reset() + + def test_epoch_end(self, outputs): + return self.eval_epoch_end(outputs, 'test') + + def setup_tokenizer( + self, tokenizer_name=None, tokenizer_model=None, vocab_file=None, bpe_dropout=0.0, + ): + + supported_tokenizers = ['yttm', 'huggingface', 'sentencepiece', 'word'] + if tokenizer_name not in supported_tokenizers: + raise NotImplementedError(f"Currently we only support tokenizers in {supported_tokenizers}.") + + self.tokenizer = get_tokenizer( + tokenizer_name=tokenizer_name, + tokenizer_model=self.register_artifact("cfg.tokenizer.tokenizer_model", tokenizer_model), + vocab_file=vocab_file, + bpe_dropout=bpe_dropout, + special_tokens=None, + use_fast=False, + ) + + def setup_training_data(self, train_data_config: Optional[DictConfig]): + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config) + + def setup_validation_data(self, val_data_config: Optional[DictConfig]): + self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config) + + def setup_test_data(self, test_data_config: Optional[DictConfig]): + self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config) + + def _setup_dataloader_from_config(self, cfg: DictConfig, predict_last_k=0): + + if cfg.get("use_tarred_dataset", False): + if cfg.get("metadata_file") is None: + raise FileNotFoundError("Trying to use tarred data set but could not find metadata path in config.") + else: + metadata_file = cfg.get('metadata_file') + with open(metadata_file) as metadata_reader: + metadata = json.load(metadata_reader) + if cfg.get('tar_files') is None: + tar_files = metadata.get('tar_files') + if tar_files is not None: + logging.info(f'Loading from tarred dataset {tar_files}') + else: + raise FileNotFoundError("Could not find tarred dataset in config or metadata.") + else: + tar_files = cfg.get('tar_files') + if metadata.get('tar_files') is not None: + raise ValueError( + 'Tar files specified in config and in metadata file. Tar files should only be specified once.' + ) + dataset = TarredSentenceDataset( + text_tar_filepaths=tar_files, + metadata_path=metadata_file, + tokenizer=self.tokenizer, + shuffle_n=cfg.get("tar_shuffle_n", 100), + shard_strategy=cfg.get("shard_strategy", "scatter"), + global_rank=self.global_rank, + world_size=self.world_size, + ) + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=1, + num_workers=cfg.get("num_workers", 2), + pin_memory=cfg.get("pin_memory", False), + drop_last=cfg.get("drop_last", False), + ) + else: + dataset = SentenceDataset( + tokenizer=self.tokenizer, + dataset=cfg.file_name, + tokens_in_batch=cfg.tokens_in_batch, + clean=cfg.get("clean", False), + max_seq_length=cfg.get("max_seq_length", 512), + min_seq_length=cfg.get("min_seq_length", 1), + cache_ids=cfg.get("cache_ids", False), + ) + if cfg.shuffle: + sampler = pt_data.RandomSampler(dataset) + else: + sampler = pt_data.SequentialSampler(dataset) + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=1, + sampler=sampler, + num_workers=cfg.get("num_workers", 2), + pin_memory=cfg.get("pin_memory", False), + drop_last=cfg.get("drop_last", False), + ) + + @classmethod + def list_available_models(cls) -> Optional[Dict[str, str]]: + pass diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/__init__.py new file mode 100644 index 0000000000..eeff22aa4d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.models.machine_translation.mt_enc_dec_model import MTEncDecModel diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/mt_enc_dec_config.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/mt_enc_dec_config.py new file mode 100644 index 0000000000..2022ca7e76 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/mt_enc_dec_config.py @@ -0,0 +1,134 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Optional, Tuple + +from omegaconf.omegaconf import MISSING + +from nemo.collections.nlp.data.machine_translation.machine_translation_dataset import TranslationDataConfig +from nemo.collections.nlp.models.enc_dec_nlp_model import EncDecNLPModelConfig +from nemo.collections.nlp.modules.common.token_classifier import TokenClassifierConfig +from nemo.collections.nlp.modules.common.tokenizer_utils import TokenizerConfig +from nemo.collections.nlp.modules.common.transformer.transformer import ( + NeMoTransformerConfig, + NeMoTransformerEncoderConfig, +) +from nemo.core.config.modelPT import ModelConfig, OptimConfig, SchedConfig + + +@dataclass +class MTSchedConfig(SchedConfig): + name: str = 'InverseSquareRootAnnealing' + warmup_ratio: Optional[float] = None + last_epoch: int = -1 + + +# TODO: Refactor this dataclass to to support more optimizers (it pins the optimizer to Adam-like optimizers). +@dataclass +class MTOptimConfig(OptimConfig): + name: str = 'adam' + lr: float = 1e-3 + betas: Tuple[float, float] = (0.9, 0.98) + weight_decay: float = 0.0 + sched: Optional[MTSchedConfig] = MTSchedConfig() + + +@dataclass +class MTEncDecModelConfig(EncDecNLPModelConfig): + # machine translation configurations + num_val_examples: int = 3 + num_test_examples: int = 3 + max_generation_delta: int = 10 + label_smoothing: Optional[float] = 0.0 + beam_size: int = 4 + len_pen: float = 0.0 + src_language: Any = 'en' # Any = str or List[str] + tgt_language: Any = 'en' # Any = str or List[str] + find_unused_parameters: Optional[bool] = True + shared_tokenizer: Optional[bool] = True + multilingual: Optional[bool] = False + preproc_out_dir: Optional[str] = None + + # network architecture configuration + encoder_tokenizer: Any = MISSING + encoder: Any = MISSING + + decoder_tokenizer: Any = MISSING + decoder: Any = MISSING + + head: TokenClassifierConfig = TokenClassifierConfig(log_softmax=True) + + # dataset configurations + train_ds: Optional[TranslationDataConfig] = TranslationDataConfig( + src_file_name=MISSING, + tgt_file_name=MISSING, + tokens_in_batch=512, + clean=True, + shuffle=True, + cache_ids=False, + use_cache=False, + ) + validation_ds: Optional[TranslationDataConfig] = TranslationDataConfig( + src_file_name=MISSING, + tgt_file_name=MISSING, + tokens_in_batch=512, + clean=False, + shuffle=False, + cache_ids=False, + use_cache=False, + ) + test_ds: Optional[TranslationDataConfig] = TranslationDataConfig( + src_file_name=MISSING, + tgt_file_name=MISSING, + tokens_in_batch=512, + clean=False, + shuffle=False, + cache_ids=False, + use_cache=False, + ) + optim: Optional[OptimConfig] = MTOptimConfig() + + +@dataclass +class AAYNBaseConfig(MTEncDecModelConfig): + + # Attention is All You Need Base Configuration + encoder_tokenizer: TokenizerConfig = TokenizerConfig(library='yttm') + decoder_tokenizer: TokenizerConfig = TokenizerConfig(library='yttm') + + encoder: NeMoTransformerEncoderConfig = NeMoTransformerEncoderConfig( + library='nemo', + model_name=None, + pretrained=False, + hidden_size=512, + inner_size=2048, + num_layers=6, + num_attention_heads=8, + ffn_dropout=0.1, + attn_score_dropout=0.1, + attn_layer_dropout=0.1, + ) + + decoder: NeMoTransformerConfig = NeMoTransformerConfig( + library='nemo', + model_name=None, + pretrained=False, + inner_size=2048, + num_layers=6, + num_attention_heads=8, + ffn_dropout=0.1, + attn_score_dropout=0.1, + attn_layer_dropout=0.1, + ) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/mt_enc_dec_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/mt_enc_dec_model.py new file mode 100644 index 0000000000..11552f9fe8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/machine_translation/mt_enc_dec_model.py @@ -0,0 +1,836 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import json +import random +from multiprocessing import Value +from pathlib import Path +from typing import Dict, List, Optional, Union + +import numpy as np +import torch +import torch.distributed as dist +import torch.utils.data as pt_data +from omegaconf import DictConfig, ListConfig, OmegaConf +from pytorch_lightning import Trainer +from pytorch_lightning.utilities import rank_zero_only +from sacrebleu import corpus_bleu + +from nemo.collections.common.data import ConcatDataset +from nemo.collections.common.losses import NLLLoss, SmoothedCrossEntropyLoss +from nemo.collections.common.metrics import GlobalAverageLossMetric +from nemo.collections.common.parts import transformer_weights_init +from nemo.collections.common.tokenizers.chinese_tokenizers import ChineseProcessor +from nemo.collections.common.tokenizers.en_ja_tokenizers import EnJaProcessor +from nemo.collections.common.tokenizers.moses_tokenizers import MosesProcessor +from nemo.collections.nlp.data import TarredTranslationDataset, TranslationDataset +from nemo.collections.nlp.models.enc_dec_nlp_model import EncDecNLPModel +from nemo.collections.nlp.models.machine_translation.mt_enc_dec_config import MTEncDecModelConfig +from nemo.collections.nlp.modules.common import TokenClassifier +from nemo.collections.nlp.modules.common.lm_utils import get_transformer +from nemo.collections.nlp.modules.common.tokenizer_utils import get_nmt_tokenizer +from nemo.collections.nlp.modules.common.transformer import BeamSearchSequenceGenerator, TopKSequenceGenerator +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.utils import logging, model_utils + +__all__ = ['MTEncDecModel'] + + +class MTEncDecModel(EncDecNLPModel): + """ + Encoder-decoder machine translation model. + """ + + def __init__(self, cfg: MTEncDecModelConfig, trainer: Trainer = None): + cfg = model_utils.convert_model_config_to_dict_config(cfg) + # Get global rank and total number of GPU workers for IterableDataset partitioning, if applicable + # Global_rank and local_rank is set by LightningModule in Lightning 1.2.0 + + self.world_size = 1 + if trainer is not None: + self.world_size = trainer.num_nodes * trainer.num_gpus + + cfg = model_utils.maybe_update_config_version(cfg) + + self.src_language = cfg.get("src_language", None) + self.tgt_language = cfg.get("tgt_language", None) + + self.multilingual = cfg.get("multilingual", False) + self.multilingual_ids = [] + + # Instantiates tokenizers and register to be saved with NeMo Model archive + # After this call, ther will be self.encoder_tokenizer and self.decoder_tokenizer + # Which can convert between tokens and token_ids for SRC and TGT languages correspondingly. + self.setup_enc_dec_tokenizers( + encoder_tokenizer_library=cfg.encoder_tokenizer.get('library', 'yttm'), + encoder_tokenizer_model=cfg.encoder_tokenizer.get('tokenizer_model'), + encoder_bpe_dropout=cfg.encoder_tokenizer.get('bpe_dropout', 0.0) + if cfg.encoder_tokenizer.get('bpe_dropout', 0.0) is not None + else 0.0, + encoder_model_name=cfg.encoder.get('model_name') if hasattr(cfg.encoder, 'model_name') else None, + decoder_tokenizer_library=cfg.decoder_tokenizer.get('library', 'yttm'), + decoder_tokenizer_model=cfg.decoder_tokenizer.tokenizer_model, + decoder_bpe_dropout=cfg.decoder_tokenizer.get('bpe_dropout', 0.0) + if cfg.decoder_tokenizer.get('bpe_dropout', 0.0) is not None + else 0.0, + decoder_model_name=cfg.decoder.get('model_name') if hasattr(cfg.decoder, 'model_name') else None, + ) + + if self.multilingual: + if isinstance(self.src_language, ListConfig) and isinstance(self.tgt_language, ListConfig): + raise ValueError( + "cfg.src_language and cfg.tgt_language cannot both be lists. We only support many-to-one or one-to-many multilingual models." + ) + elif isinstance(self.src_language, ListConfig): + for lng in self.src_language: + self.multilingual_ids.append(self.encoder_tokenizer.token_to_id("<" + lng + ">")) + elif isinstance(self.tgt_language, ListConfig): + for lng in self.tgt_language: + self.multilingual_ids.append(self.encoder_tokenizer.token_to_id("<" + lng + ">")) + else: + raise ValueError( + "Expect either cfg.src_language or cfg.tgt_language to be a list when multilingual=True." + ) + + if isinstance(self.src_language, ListConfig): + self.tgt_language = [self.tgt_language] * len(self.src_language) + else: + self.src_language = [self.src_language] * len(self.tgt_language) + + self.source_processor_list = [] + self.target_processor_list = [] + for src_lng, tgt_lng in zip(self.src_language, self.tgt_language): + src_prcsr, tgt_prscr = self.setup_pre_and_post_processing_utils( + source_lang=src_lng, target_lang=tgt_lng + ) + self.source_processor_list.append(src_prcsr) + self.target_processor_list.append(tgt_prscr) + + else: + # After this call, the model will have self.source_processor and self.target_processor objects + self.setup_pre_and_post_processing_utils(source_lang=self.src_language, target_lang=self.tgt_language) + self.multilingual_ids = [None] + + # TODO: Why is this base constructor call so late in the game? + super().__init__(cfg=cfg, trainer=trainer) + + # encoder from NeMo, Megatron-LM, or HuggingFace + encoder_cfg_dict = OmegaConf.to_container(cfg.get('encoder')) + encoder_cfg_dict['vocab_size'] = self.encoder_vocab_size + library = encoder_cfg_dict.pop('library', 'nemo') + model_name = encoder_cfg_dict.pop('model_name', None) + pretrained = encoder_cfg_dict.pop('pretrained', False) + self.encoder = get_transformer( + library=library, + model_name=model_name, + pretrained=pretrained, + config_dict=encoder_cfg_dict, + encoder=True, + pre_ln_final_layer_norm=encoder_cfg_dict.get('pre_ln_final_layer_norm', False), + ) + + # decoder from NeMo, Megatron-LM, or HuggingFace + decoder_cfg_dict = OmegaConf.to_container(cfg.get('decoder')) + decoder_cfg_dict['vocab_size'] = self.decoder_vocab_size + library = decoder_cfg_dict.pop('library', 'nemo') + model_name = decoder_cfg_dict.pop('model_name', None) + pretrained = decoder_cfg_dict.pop('pretrained', False) + decoder_cfg_dict['hidden_size'] = self.encoder.hidden_size + self.decoder = get_transformer( + library=library, + model_name=model_name, + pretrained=pretrained, + config_dict=decoder_cfg_dict, + encoder=False, + pre_ln_final_layer_norm=decoder_cfg_dict.get('pre_ln_final_layer_norm', False), + ) + + self.log_softmax = TokenClassifier( + hidden_size=self.decoder.hidden_size, + num_classes=self.decoder_vocab_size, + activation=cfg.head.activation, + log_softmax=cfg.head.log_softmax, + dropout=cfg.head.dropout, + use_transformer_init=cfg.head.use_transformer_init, + ) + + self.beam_search = BeamSearchSequenceGenerator( + embedding=self.decoder.embedding, + decoder=self.decoder.decoder, + log_softmax=self.log_softmax, + max_sequence_length=self.decoder.max_sequence_length, + beam_size=cfg.beam_size, + bos=self.decoder_tokenizer.bos_id, + pad=self.decoder_tokenizer.pad_id, + eos=self.decoder_tokenizer.eos_id, + len_pen=cfg.len_pen, + max_delta_length=cfg.max_generation_delta, + ) + + # tie weights of embedding and softmax matrices + self.log_softmax.mlp.layer0.weight = self.decoder.embedding.token_embedding.weight + + # TODO: encoder and decoder with different hidden size? + std_init_range = 1 / self.encoder.hidden_size ** 0.5 + + # initialize weights if not using pretrained encoder/decoder + if not self._cfg.encoder.get('pretrained', False): + self.encoder.apply(lambda module: transformer_weights_init(module, std_init_range)) + + if not self._cfg.decoder.get('pretrained', False): + self.decoder.apply(lambda module: transformer_weights_init(module, std_init_range)) + + self.log_softmax.apply(lambda module: transformer_weights_init(module, std_init_range)) + + self.loss_fn = SmoothedCrossEntropyLoss( + pad_id=self.decoder_tokenizer.pad_id, label_smoothing=cfg.label_smoothing + ) + self.eval_loss_fn = NLLLoss(ignore_index=self.decoder_tokenizer.pad_id) + + def filter_predicted_ids(self, ids): + ids[ids >= self.decoder_tokenizer.vocab_size] = self.decoder_tokenizer.unk_id + return ids + + @typecheck() + def forward(self, src, src_mask, tgt, tgt_mask): + src_hiddens = self.encoder(input_ids=src, encoder_mask=src_mask) + tgt_hiddens = self.decoder( + input_ids=tgt, decoder_mask=tgt_mask, encoder_embeddings=src_hiddens, encoder_mask=src_mask + ) + log_probs = self.log_softmax(hidden_states=tgt_hiddens) + return log_probs + + def training_step(self, batch, batch_idx): + """ + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. + """ + # forward pass + for i in range(len(batch)): + if batch[i].ndim == 3: + # Dataset returns already batched data and the first dimension of size 1 added by DataLoader + # is excess. + batch[i] = batch[i].squeeze(dim=0) + src_ids, src_mask, tgt_ids, tgt_mask, labels = batch + log_probs = self(src_ids, src_mask, tgt_ids, tgt_mask) + train_loss = self.loss_fn(log_probs=log_probs, labels=labels) + tensorboard_logs = { + 'train_loss': train_loss, + 'lr': self._optimizer.param_groups[0]['lr'], + } + return {'loss': train_loss, 'log': tensorboard_logs} + + def eval_step(self, batch, batch_idx, mode, dataloader_idx=0): + for i in range(len(batch)): + if batch[i].ndim == 3: + # Dataset returns already batched data and the first dimension of size 1 added by DataLoader + # is excess. + batch[i] = batch[i].squeeze(dim=0) + + if self.multilingual: + self.source_processor = self.source_processor_list[dataloader_idx] + self.target_processor = self.target_processor_list[dataloader_idx] + + src_ids, src_mask, tgt_ids, tgt_mask, labels = batch + log_probs = self(src_ids, src_mask, tgt_ids, tgt_mask) + eval_loss = self.eval_loss_fn(log_probs=log_probs, labels=labels) + # this will run encoder twice -- TODO: potentially fix + _, translations = self.batch_translate(src=src_ids, src_mask=src_mask) + if dataloader_idx == 0: + getattr(self, f'{mode}_loss')(loss=eval_loss, num_measurements=log_probs.shape[0] * log_probs.shape[1]) + else: + getattr(self, f'{mode}_loss_{dataloader_idx}')( + loss=eval_loss, num_measurements=log_probs.shape[0] * log_probs.shape[1] + ) + np_tgt = tgt_ids.detach().cpu().numpy() + ground_truths = [self.decoder_tokenizer.ids_to_text(tgt) for tgt in np_tgt] + ground_truths = [self.target_processor.detokenize(tgt.split(' ')) for tgt in ground_truths] + num_non_pad_tokens = np.not_equal(np_tgt, self.decoder_tokenizer.pad_id).sum().item() + return { + 'translations': translations, + 'ground_truths': ground_truths, + 'num_non_pad_tokens': num_non_pad_tokens, + } + + def test_step(self, batch, batch_idx, dataloader_idx=0): + return self.eval_step(batch, batch_idx, 'test', dataloader_idx) + + @rank_zero_only + def log_param_stats(self): + for name, p in self.named_parameters(): + if p.requires_grad: + self.trainer.logger.experiment.add_histogram(name + '_hist', p, global_step=self.global_step) + self.trainer.logger.experiment.add_scalars( + name, + {'mean': p.mean(), 'stddev': p.std(), 'max': p.max(), 'min': p.min()}, + global_step=self.global_step, + ) + + def validation_step(self, batch, batch_idx, dataloader_idx=0): + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. + """ + return self.eval_step(batch, batch_idx, 'val', dataloader_idx) + + def eval_epoch_end(self, outputs, mode): + # if user specifies one validation dataloader, then PTL reverts to giving a list of dictionary instead of a list of list of dictionary + if isinstance(outputs[0], dict): + outputs = [outputs] + + loss_list = [] + sb_score_list = [] + for dataloader_idx, output in enumerate(outputs): + if dataloader_idx == 0: + eval_loss = getattr(self, f'{mode}_loss').compute() + else: + eval_loss = getattr(self, f'{mode}_loss_{dataloader_idx}').compute() + + translations = list(itertools.chain(*[x['translations'] for x in output])) + ground_truths = list(itertools.chain(*[x['ground_truths'] for x in output])) + assert len(translations) == len(ground_truths) + + # Gather translations and ground truths from all workers + tr_and_gt = [None for _ in range(self.world_size)] + # we also need to drop pairs where ground truth is an empty string + dist.all_gather_object( + tr_and_gt, [(t, g) for (t, g) in zip(translations, ground_truths) if g.strip() != ''] + ) + if self.global_rank == 0: + _translations = [] + _ground_truths = [] + for rank in range(0, self.world_size): + _translations += [t for (t, g) in tr_and_gt[rank]] + _ground_truths += [g for (t, g) in tr_and_gt[rank]] + + if self.tgt_language in ['ja']: + sacre_bleu = corpus_bleu(_translations, [_ground_truths], tokenize="ja-mecab") + elif self.tgt_language in ['zh']: + sacre_bleu = corpus_bleu(_translations, [_ground_truths], tokenize="zh") + else: + sacre_bleu = corpus_bleu(_translations, [_ground_truths], tokenize="13a") + + # because the reduction op later is average (over word_size) + sb_score = sacre_bleu.score * self.world_size + + dataset_name = "Validation" if mode == 'val' else "Test" + logging.info( + f"Dataset name: {dataset_name}, Dataloader index: {dataloader_idx}, Set size: {len(translations)}" + ) + logging.info( + f"Dataset name: {dataset_name}, Dataloader index: {dataloader_idx}, Val Loss = {eval_loss}" + ) + logging.info( + f"Dataset name: {dataset_name}, Dataloader index: {dataloader_idx}, Sacre BLEU = {sb_score / self.world_size}" + ) + logging.info( + f"Dataset name: {dataset_name}, Dataloader index: {dataloader_idx}, Translation Examples:" + ) + for i in range(0, 3): + ind = random.randint(0, len(translations) - 1) + logging.info(" " + '\u0332'.join(f"Example {i}:")) + logging.info(f" Prediction: {translations[ind]}") + logging.info(f" Ground Truth: {ground_truths[ind]}") + else: + sb_score = 0.0 + + loss_list.append(eval_loss.cpu().numpy()) + sb_score_list.append(sb_score) + if dataloader_idx == 0: + self.log(f"{mode}_loss", eval_loss, sync_dist=True) + self.log(f"{mode}_sacreBLEU", sb_score, sync_dist=True) + getattr(self, f'{mode}_loss').reset() + else: + self.log(f"{mode}_loss_dl_index_{dataloader_idx}", eval_loss, sync_dist=True) + self.log(f"{mode}_sacreBLEU_dl_index_{dataloader_idx}", sb_score, sync_dist=True) + getattr(self, f'{mode}_loss_{dataloader_idx}').reset() + + if len(loss_list) > 1: + self.log(f"{mode}_loss_avg", np.mean(loss_list), sync_dist=True) + self.log(f"{mode}_sacreBLEU_avg", np.mean(sb_score_list), sync_dist=True) + + def validation_epoch_end(self, outputs): + """ + Called at the end of validation to aggregate outputs. + :param outputs: list of individual outputs of each validation step. + """ + self.eval_epoch_end(outputs, 'val') + + def test_epoch_end(self, outputs): + self.eval_epoch_end(outputs, 'test') + + def setup_enc_dec_tokenizers( + self, + encoder_tokenizer_library=None, + encoder_tokenizer_model=None, + encoder_bpe_dropout=0.0, + encoder_model_name=None, + decoder_tokenizer_library=None, + decoder_tokenizer_model=None, + decoder_bpe_dropout=0.0, + decoder_model_name=None, + ): + + supported_tokenizers = ['yttm', 'huggingface', 'sentencepiece'] + if ( + encoder_tokenizer_library not in supported_tokenizers + or decoder_tokenizer_library not in supported_tokenizers + ): + raise NotImplementedError(f"Currently we only support tokenizers in {supported_tokenizers}.") + + self.encoder_tokenizer = get_nmt_tokenizer( + library=encoder_tokenizer_library, + tokenizer_model=self.register_artifact("encoder_tokenizer.tokenizer_model", encoder_tokenizer_model), + bpe_dropout=encoder_bpe_dropout, + model_name=encoder_model_name, + vocab_file=None, + special_tokens=None, + use_fast=False, + ) + self.decoder_tokenizer = get_nmt_tokenizer( + library=decoder_tokenizer_library, + tokenizer_model=self.register_artifact("decoder_tokenizer.tokenizer_model", decoder_tokenizer_model), + bpe_dropout=decoder_bpe_dropout, + model_name=decoder_model_name, + vocab_file=None, + special_tokens=None, + use_fast=False, + ) + + def setup_training_data(self, train_data_config: Optional[DictConfig]): + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config) + + def setup_multiple_validation_data(self, val_data_config: Union[DictConfig, Dict]): + self.setup_validation_data(self._cfg.get('validation_ds')) + + def setup_multiple_test_data(self, test_data_config: Union[DictConfig, Dict]): + self.setup_test_data(self._cfg.get('test_ds')) + + def setup_validation_data(self, val_data_config: Optional[DictConfig]): + self._validation_dl = self._setup_eval_dataloader_from_config(cfg=val_data_config) + # instantiate Torchmetric for each val dataloader + if self._validation_dl is not None: + for dataloader_idx in range(len(self._validation_dl)): + if dataloader_idx == 0: + setattr( + self, f'val_loss', GlobalAverageLossMetric(dist_sync_on_step=False, take_avg_loss=True), + ) + else: + setattr( + self, + f'val_loss_{dataloader_idx}', + GlobalAverageLossMetric(dist_sync_on_step=False, take_avg_loss=True), + ) + + def setup_test_data(self, test_data_config: Optional[DictConfig]): + self._test_dl = self._setup_eval_dataloader_from_config(cfg=test_data_config) + # instantiate Torchmetric for each test dataloader + if self._test_dl is not None: + for dataloader_idx in range(len(self._test_dl)): + if dataloader_idx == 0: + setattr( + self, f'test_loss', GlobalAverageLossMetric(dist_sync_on_step=False, take_avg_loss=True), + ) + else: + setattr( + self, + f'test_loss_{dataloader_idx}', + GlobalAverageLossMetric(dist_sync_on_step=False, take_avg_loss=True), + ) + + def _setup_dataloader_from_config(self, cfg: DictConfig): + if cfg.get("use_tarred_dataset", False): + if cfg.get("metadata_file") is None: + raise FileNotFoundError("Trying to use tarred data set but could not find metadata path in config.") + else: + if not self.multilingual: + metadata_file_list = [cfg.get('metadata_file')] + else: + metadata_file_list = cfg.get('metadata_file') + + datasets = [] + for idx, metadata_file in enumerate(metadata_file_list): + with open(metadata_file) as metadata_reader: + metadata = json.load(metadata_reader) + if cfg.get('tar_files') is None: + tar_files = metadata.get('tar_files') + if tar_files is not None: + logging.info(f'Loading from tarred dataset {tar_files}') + else: + raise FileNotFoundError("Could not find tarred dataset in config or metadata.") + else: + tar_files = cfg.get('tar_files') + if self.multilingual: + tar_files = tar_files[idx] + if metadata.get('tar_files') is not None: + logging.info( + f'Tar file paths found in both cfg and metadata using one in cfg by default - {tar_files}' + ) + + dataset = TarredTranslationDataset( + text_tar_filepaths=tar_files, + metadata_path=metadata_file, + encoder_tokenizer=self.encoder_tokenizer, + decoder_tokenizer=self.decoder_tokenizer, + shuffle_n=cfg.get("tar_shuffle_n", 100), + shard_strategy=cfg.get("shard_strategy", "scatter"), + global_rank=self.global_rank, + world_size=self.world_size, + reverse_lang_direction=cfg.get("reverse_lang_direction", False), + prepend_id=self.multilingual_ids[idx], + ) + datasets.append(dataset) + + if self.multilingual: + dataset = ConcatDataset( + datasets=datasets, + sampling_technique=cfg.get('concat_sampling_technique'), + sampling_temperature=cfg.get('concat_sampling_temperature'), + sampling_probabilities=cfg.get('concat_sampling_probabilities'), + global_rank=self.global_rank, + world_size=self.world_size, + ) + else: + dataset = datasets[0] + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=1, + num_workers=cfg.get("num_workers", 2), + pin_memory=cfg.get("pin_memory", False), + drop_last=cfg.get("drop_last", False), + ) + else: + if not self.multilingual: + src_file_list = [cfg.src_file_name] + tgt_file_list = [cfg.tgt_file_name] + else: + src_file_list = cfg.src_file_name + tgt_file_list = cfg.tgt_file_name + + if len(src_file_list) != len(tgt_file_list): + raise ValueError( + 'The same number of filepaths must be passed in for source and target while training multilingual.' + ) + + datasets = [] + for idx, src_file in enumerate(src_file_list): + dataset = TranslationDataset( + dataset_src=str(Path(src_file).expanduser()), + dataset_tgt=str(Path(tgt_file_list[idx]).expanduser()), + tokens_in_batch=cfg.tokens_in_batch, + clean=cfg.get("clean", False), + max_seq_length=cfg.get("max_seq_length", 512), + min_seq_length=cfg.get("min_seq_length", 1), + max_seq_length_diff=cfg.get("max_seq_length_diff", 512), + max_seq_length_ratio=cfg.get("max_seq_length_ratio", 512), + cache_ids=cfg.get("cache_ids", False), + cache_data_per_node=cfg.get("cache_data_per_node", False), + use_cache=cfg.get("use_cache", False), + reverse_lang_direction=cfg.get("reverse_lang_direction", False), + prepend_id=self.multilingual_ids[idx], + ) + dataset.batchify(self.encoder_tokenizer, self.decoder_tokenizer) + datasets.append(dataset) + + if self.multilingual: + dataset = ConcatDataset( + datasets=datasets, + shuffle=cfg.get('shuffle'), + sampling_technique=cfg.get('concat_sampling_technique'), + sampling_temperature=cfg.get('concat_sampling_temperature'), + sampling_probabilities=cfg.get('concat_sampling_probabilities'), + global_rank=self.global_rank, + world_size=self.world_size, + ) + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=1, + num_workers=cfg.get("num_workers", 2), + pin_memory=cfg.get("pin_memory", False), + drop_last=cfg.get("drop_last", False), + ) + else: + dataset = datasets[0] + + if cfg.shuffle: + sampler = pt_data.RandomSampler(dataset) + else: + sampler = pt_data.SequentialSampler(dataset) + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=1, + sampler=sampler, + num_workers=cfg.get("num_workers", 2), + pin_memory=cfg.get("pin_memory", False), + drop_last=cfg.get("drop_last", False), + ) + + def replace_beam_with_sampling(self, topk=500): + self.beam_search = TopKSequenceGenerator( + embedding=self.decoder.embedding, + decoder=self.decoder.decoder, + log_softmax=self.log_softmax, + max_sequence_length=self.beam_search.max_seq_length, + beam_size=topk, + bos=self.decoder_tokenizer.bos_id, + pad=self.decoder_tokenizer.pad_id, + eos=self.decoder_tokenizer.eos_id, + ) + + def _setup_eval_dataloader_from_config(self, cfg: DictConfig): + src_file_name = cfg.get('src_file_name') + tgt_file_name = cfg.get('tgt_file_name') + + if src_file_name is None or tgt_file_name is None: + raise ValueError( + 'Validation dataloader needs both cfg.src_file_name and cfg.tgt_file_name to not be None.' + ) + else: + # convert src_file_name and tgt_file_name to list of strings + if isinstance(src_file_name, str): + src_file_list = [src_file_name] + elif isinstance(src_file_name, ListConfig): + src_file_list = src_file_name + else: + raise ValueError("cfg.src_file_name must be string or list of strings") + if isinstance(tgt_file_name, str): + tgt_file_list = [tgt_file_name] + elif isinstance(tgt_file_name, ListConfig): + tgt_file_list = tgt_file_name + else: + raise ValueError("cfg.tgt_file_name must be string or list of strings") + if len(src_file_list) != len(tgt_file_list): + raise ValueError('The same number of filepaths must be passed in for source and target validation.') + + dataloaders = [] + prepend_idx = 0 + for idx, src_file in enumerate(src_file_list): + if self.multilingual: + prepend_idx = idx + dataset = TranslationDataset( + dataset_src=str(Path(src_file).expanduser()), + dataset_tgt=str(Path(tgt_file_list[idx]).expanduser()), + tokens_in_batch=cfg.tokens_in_batch, + clean=cfg.get("clean", False), + max_seq_length=cfg.get("max_seq_length", 512), + min_seq_length=cfg.get("min_seq_length", 1), + max_seq_length_diff=cfg.get("max_seq_length_diff", 512), + max_seq_length_ratio=cfg.get("max_seq_length_ratio", 512), + cache_ids=cfg.get("cache_ids", False), + cache_data_per_node=cfg.get("cache_data_per_node", False), + use_cache=cfg.get("use_cache", False), + reverse_lang_direction=cfg.get("reverse_lang_direction", False), + prepend_id=self.multilingual_ids[prepend_idx], + ) + dataset.batchify(self.encoder_tokenizer, self.decoder_tokenizer) + + if cfg.shuffle: + sampler = pt_data.RandomSampler(dataset) + else: + sampler = pt_data.SequentialSampler(dataset) + + dataloader = torch.utils.data.DataLoader( + dataset=dataset, + batch_size=1, + sampler=sampler, + num_workers=cfg.get("num_workers", 2), + pin_memory=cfg.get("pin_memory", False), + drop_last=cfg.get("drop_last", False), + ) + dataloaders.append(dataloader) + + return dataloaders + + def setup_pre_and_post_processing_utils(self, source_lang, target_lang): + """ + Creates source and target processor objects for input and output pre/post-processing. + """ + self.source_processor, self.target_processor = None, None + if (source_lang == 'en' and target_lang == 'ja') or (source_lang == 'ja' and target_lang == 'en'): + self.source_processor = EnJaProcessor(source_lang) + self.target_processor = EnJaProcessor(target_lang) + else: + if source_lang == 'zh': + self.source_processor = ChineseProcessor() + if target_lang == 'zh': + self.target_processor = ChineseProcessor() + if source_lang is not None and source_lang not in ['ja', 'zh']: + self.source_processor = MosesProcessor(source_lang) + if target_lang is not None and target_lang not in ['ja', 'zh']: + self.target_processor = MosesProcessor(target_lang) + + return self.source_processor, self.target_processor + + @torch.no_grad() + def batch_translate( + self, src: torch.LongTensor, src_mask: torch.LongTensor, + ): + """ + Translates a minibatch of inputs from source language to target language. + Args: + src: minibatch of inputs in the src language (batch x seq_len) + src_mask: mask tensor indicating elements to be ignored (batch x seq_len) + Returns: + translations: a list strings containing detokenized translations + inputs: a list of string containing detokenized inputs + """ + mode = self.training + try: + self.eval() + src_hiddens = self.encoder(input_ids=src, encoder_mask=src_mask) + beam_results = self.beam_search(encoder_hidden_states=src_hiddens, encoder_input_mask=src_mask) + beam_results = self.filter_predicted_ids(beam_results) + + translations = [self.decoder_tokenizer.ids_to_text(tr) for tr in beam_results.cpu().numpy()] + inputs = [self.encoder_tokenizer.ids_to_text(inp) for inp in src.cpu().numpy()] + if self.target_processor is not None: + translations = [ + self.target_processor.detokenize(translation.split(' ')) for translation in translations + ] + + if self.source_processor is not None: + inputs = [self.source_processor.detokenize(item.split(' ')) for item in inputs] + finally: + self.train(mode=mode) + return inputs, translations + + # TODO: We should drop source/target_lang arguments in favor of using self.src/tgt_language + @torch.no_grad() + def translate(self, text: List[str], source_lang: str = None, target_lang: str = None) -> List[str]: + """ + Translates list of sentences from source language to target language. + Should be regular text, this method performs its own tokenization/de-tokenization + Args: + text: list of strings to translate + source_lang: if not None, corresponding MosesTokenizer and MosesPunctNormalizer will be run + target_lang: if not None, corresponding MosesDecokenizer will be run + Returns: + list of translated strings + """ + # __TODO__: This will reset both source and target processors even if you want to reset just one. + if source_lang is not None or target_lang is not None: + self.setup_pre_and_post_processing_utils(source_lang, target_lang) + + mode = self.training + prepend_ids = [] + if self.multilingual: + if source_lang is None or target_lang is None: + raise ValueError("Expect source_lang and target_lang to infer for multilingual model.") + src_symbol = self.encoder_tokenizer.token_to_id('<' + source_lang + '>') + tgt_symbol = self.encoder_tokenizer.token_to_id('<' + target_lang + '>') + prepend_ids = [src_symbol if src_symbol in self.multilingual_ids else tgt_symbol] + try: + self.eval() + inputs = [] + for txt in text: + if self.source_processor is not None: + txt = self.source_processor.normalize(txt) + txt = self.source_processor.tokenize(txt) + ids = self.encoder_tokenizer.text_to_ids(txt) + ids = prepend_ids + [self.encoder_tokenizer.bos_id] + ids + [self.encoder_tokenizer.eos_id] + inputs.append(ids) + max_len = max(len(txt) for txt in inputs) + src_ids_ = np.ones((len(inputs), max_len)) * self.encoder_tokenizer.pad_id + for i, txt in enumerate(inputs): + src_ids_[i][: len(txt)] = txt + + src_mask = torch.FloatTensor((src_ids_ != self.encoder_tokenizer.pad_id)).to(self.device) + src = torch.LongTensor(src_ids_).to(self.device) + _, translations = self.batch_translate(src, src_mask) + finally: + self.train(mode=mode) + return translations + + @classmethod + def list_available_models(cls) -> Optional[Dict[str, str]]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + result = [] + model = PretrainedModelInfo( + pretrained_model_name="nmt_en_de_transformer12x2", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/nmt_en_de_transformer12x2/versions/1.0.0rc1/files/nmt_en_de_transformer12x2.nemo", + description="En->De translation model. See details here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_de_transformer12x2", + ) + result.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="nmt_de_en_transformer12x2", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/nmt_de_en_transformer12x2/versions/1.0.0rc1/files/nmt_de_en_transformer12x2.nemo", + description="De->En translation model. See details here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_de_en_transformer12x2", + ) + result.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="nmt_en_es_transformer12x2", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/nmt_en_es_transformer12x2/versions/1.0.0rc1/files/nmt_en_es_transformer12x2.nemo", + description="En->Es translation model. See details here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_es_transformer12x2", + ) + result.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="nmt_es_en_transformer12x2", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/nmt_es_en_transformer12x2/versions/1.0.0rc1/files/nmt_es_en_transformer12x2.nemo", + description="Es->En translation model. See details here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_es_en_transformer12x2", + ) + result.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="nmt_en_fr_transformer12x2", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/nmt_en_fr_transformer12x2/versions/1.0.0rc1/files/nmt_en_fr_transformer12x2.nemo", + description="En->Fr translation model. See details here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_fr_transformer12x2", + ) + result.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="nmt_fr_en_transformer12x2", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/nmt_fr_en_transformer12x2/versions/1.0.0rc1/files/nmt_fr_en_transformer12x2.nemo", + description="Fr->En translation model. See details here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_fr_en_transformer12x2", + ) + result.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="nmt_en_ru_transformer6x6", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/nmt_en_ru_transformer6x6/versions/1.0.0rc1/files/nmt_en_ru_transformer6x6.nemo", + description="En->Ru translation model. See details here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_ru_transformer6x6", + ) + result.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="nmt_ru_en_transformer6x6", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/nmt_ru_en_transformer6x6/versions/1.0.0rc1/files/nmt_ru_en_transformer6x6.nemo", + description="Ru->En translation model. See details here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_ru_en_transformer6x6", + ) + result.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="nmt_zh_en_transformer6x6", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/nmt_zh_en_transformer6x6/versions/1.0.0rc1/files/nmt_zh_en_transformer6x6.nemo", + description="Zh->En translation model. See details here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_zh_en_transformer6x6", + ) + result.append(model) + + model = PretrainedModelInfo( + pretrained_model_name="nmt_en_zh_transformer6x6", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/nmt_en_zh_transformer6x6/versions/1.0.0rc1/files/nmt_en_zh_transformer6x6.nemo", + description="En->Zh translation model. See details here: https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_zh_transformer6x6", + ) + result.append(model) + + return result diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/neural_machine_translation/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/neural_machine_translation/__init__.py new file mode 100644 index 0000000000..4e61389d8c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/neural_machine_translation/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.models.neural_machine_translation.neural_machine_translation_model import ( + NeuralMachineTranslationModel, +) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/neural_machine_translation/neural_machine_translation_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/neural_machine_translation/neural_machine_translation_model.py new file mode 100644 index 0000000000..b329394655 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/neural_machine_translation/neural_machine_translation_model.py @@ -0,0 +1,245 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, MeetKai Inc. All rights reserved. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Dict, List, Optional, Tuple, Union + +import torch +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer +from transformers import AutoModel, BartForConditionalGeneration, EncoderDecoderModel + +from nemo.collections.common.metrics import Perplexity +from nemo.collections.nlp.data.neural_machine_translation import NeuralMachineTranslationDataset +from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer +from nemo.core.classes.common import typecheck +from nemo.core.classes.modelPT import ModelPT +from nemo.core.neural_types import ChannelType, LossType, MaskType, NeuralType +from nemo.utils import logging + +__all__ = ["NeuralMachineTranslationModel"] + + +class NeuralMachineTranslationModel(ModelPT): + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "attention_mask": NeuralType(('B', 'T'), MaskType(), optional=True), + "decoder_input_ids": NeuralType(('B', 'T'), ChannelType(), optional=True), + "labels": NeuralType(('B', 'T'), ChannelType(), optional=True), + } + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "loss": NeuralType((), LossType()), + "decoder_hidden_states": NeuralType(("B", "T", "D"), ChannelType(), optional=True), + "encoder_hidden_states": NeuralType(("B", "T", "D"), ChannelType(), optional=True), + } + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + + # must assign tokenizers before init + if cfg.language_model.pretrained_model_name: + if cfg.language_model.pretrained_encoder_model_name or cfg.language_model.pretrained_decoder_model_name: + raise ValueError( + "Must have either pretrained_model_name or both pretrained_encoder_model name and " + "pretrained_decoder_model_name." + ) + # setup tokenizer + self.encoder_tokenizer = self.setup_tokenizer(cfg.encoder_tokenizer) + self.encoder_add_special_tokens = cfg.encoder_tokenizer.add_special_tokens + + # set decoder to encoder + self.decoder_tokenizer = self.encoder_tokenizer + self.decoder_add_special_tokens = self.encoder_add_special_tokens + else: + if not ( + cfg.language_model.pretrained_encoder_model_name and cfg.language_model.pretrained_decoder_model_name + ): + raise ValueError("Both encoder and decoder must be specified") + + # setup tokenizers + self.encoder_tokenizer = self.setup_tokenizer(cfg.encoder_tokenizer) + self.encoder_add_special_tokens = cfg.encoder_tokenizer.add_special_tokens + + self.decoder_tokenizer = self.setup_tokenizer(cfg.decoder_tokenizer) + self.decoder_add_special_tokens = cfg.decoder_tokenizer.add_special_tokens + + if not self.encoder_tokenizer: + raise TypeError("encoder_tokenizer failed to initialize") + if not self.decoder_tokenizer: + raise TypeError("decoder_tokenizer failed to initialize") + + # init superclass + super().__init__(cfg=cfg, trainer=trainer) + + # must assign modules after init + if cfg.language_model.pretrained_model_name: + # Setup end-to-end model + if "bart" in cfg.language_model.pretrained_model_name: + self.model = BartForConditionalGeneration.from_pretrained(cfg.language_model.pretrained_model_name) + else: + self.model = AutoModel.from_pretrained(cfg.language_model.pretrained_model_name) + else: + if not ( + cfg.language_model.pretrained_encoder_model_name and cfg.language_model.pretrained_decoder_model_name + ): + raise ValueError("Both encoder and decoder must be specified") + + # Setup encoder/decoder model + self.model = EncoderDecoderModel.from_encoder_decoder_pretrained( + encoder=cfg.language_model.pretrained_encoder_model_name, + decoder=cfg.language_model.pretrained_decoder_model_name, + ) + + self.validation_perplexity = Perplexity(compute_on_step=False) + + self.setup_optimization(cfg.optim) + + @typecheck() + def forward( + self, + input_ids: torch.Tensor, + attention_mask: torch.Tensor = None, + decoder_input_ids: torch.Tensor = None, + labels: torch.Tensor = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: + """ + No special modification required for Lightning, define it as you normally would + in the `nn.Module` in vanilla PyTorch. + """ + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + labels=labels, + return_dict=False, + ) + return outputs + + @typecheck.disable_checks() + def generate(self, input_ids: torch.Tensor) -> torch.Tensor: + """Wraps huggingface EncoderDecoder.generate().""" + outputs = self.model.generate( + input_ids=input_ids, + pad_token_id=self.encoder_tokenizer.pad_id, + bos_token_id=self.encoder_tokenizer.bos_id, + eos_token_id=self.encoder_tokenizer.eos_id, + decoder_start_token_id=self.decoder_tokenizer.bos_id, + **self._cfg.generate, + ) + return outputs + + def training_step(self, batch: Tuple, batch_idx: int) -> Dict: + """ + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. Loss calculation from HuggingFace's BartForConditionalGeneration. + """ + input_ids, input_mask, decoder_input_ids, labels = batch + loss = self.forward( + input_ids=input_ids, attention_mask=input_mask, decoder_input_ids=decoder_input_ids, labels=labels, + )[0] + + tensorboard_logs = {"train_loss": loss, "lr": self._optimizer.param_groups[0]["lr"]} + + return {"loss": loss, "log": tensorboard_logs} + + def validation_step(self, batch: Tuple, batch_idx: int) -> Dict: + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. Loss calculation from HuggingFace's BartForConditionalGeneration. + """ + input_ids, input_mask, decoder_input_ids, labels = batch + loss, logits = self.forward( + input_ids=input_ids, attention_mask=input_mask, decoder_input_ids=decoder_input_ids, labels=labels, + )[:2] + + self.validation_perplexity(logits=logits) + + tensorboard_logs = {"val_loss": loss} + + return {"val_loss": loss, "log": tensorboard_logs} + + def validation_epoch_end(self, outputs: List[Dict]) -> Dict: + """ + Called at the end of validation to aggregate outputs. + :param outputs: list of individual outputs of each validation step. + """ + avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean() + perplexity = self.validation_perplexity.compute() + tensorboard_logs = {"val_loss": avg_loss, "perplexity": perplexity} + logging.info(f"evaluation perplexity {perplexity.item()}") + return {"val_loss": avg_loss, "log": tensorboard_logs} + + @typecheck.disable_checks() + def test_step(self, batch: Tuple, batch_idx: int) -> torch.Tensor: + """Lightning calls this inside the test loop with data from the test dataloader.""" + input_ids, input_mask, decoder_input_ids, labels = batch + sequences = self.generate(input_ids=input_ids) + return sequences + + @typecheck.disable_checks() + def test_epoch_end(self, outputs: List[torch.Tensor]) -> Dict[str, List[str]]: + """Called at the end of test to aggregate outputs and decode them.""" + texts = [self.encoder_tokenizer.ids_to_text(seq) for batch in outputs for seq in batch] + return {"texts": texts} + + def setup_tokenizer(self, cfg: DictConfig): + tokenizer = get_tokenizer( + tokenizer_name=cfg.tokenizer_name, + tokenizer_model=cfg.tokenizer_model, + special_tokens=OmegaConf.to_container(cfg.special_tokens) if cfg.special_tokens else None, + vocab_file=cfg.vocab_file, + ) + return tokenizer + + def setup_training_data(self, train_data_config: Optional[DictConfig]): + self._train_dl = self.setup_dataloader_from_config(cfg=train_data_config) + + def setup_validation_data(self, val_data_config: Optional[DictConfig]): + self._validation_dl = self.setup_dataloader_from_config(cfg=val_data_config) + + def setup_test_data(self, test_data_config: Optional[DictConfig]): + self._test_dl = self.setup_dataloader_from_config(cfg=test_data_config) + + def setup_dataloader_from_config(self, cfg: DictConfig): + dataset = NeuralMachineTranslationDataset( + filepath=cfg.filepath, + encoder_tokenizer=self.encoder_tokenizer, + decoder_tokenizer=self.decoder_tokenizer, + encoder_add_special_tokens=self.encoder_add_special_tokens, + decoder_add_special_tokens=self.decoder_add_special_tokens, + max_seq_length=self._cfg.max_seq_length, + num_samples=cfg.get("num_samples", -1), + convert_labels=self._cfg.convert_labels, + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=self._cfg.batch_size, + shuffle=cfg.shuffle, + num_workers=cfg.get("num_workers", 2), + pin_memory=cfg.get("pin_memory", False), + drop_last=cfg.get("drop_last", False), + collate_fn=dataset.collate_fn, + ) + + @classmethod + def list_available_models(cls) -> Optional[Dict[str, str]]: + pass diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/nlp_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/nlp_model.py new file mode 100644 index 0000000000..3711db2c8f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/nlp_model.py @@ -0,0 +1,463 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import hashlib +import json +import os +import shutil +import tarfile +import tempfile +from typing import Any, Dict, Optional, Union + +import torch +from megatron import mpu +from megatron.checkpointing import get_checkpoint_version, set_checkpoint_version +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer +from pytorch_lightning.accelerators.accelerator import Accelerator +from pytorch_lightning.utilities import rank_zero_only +from transformers import TRANSFORMERS_CACHE + +from nemo.collections.common.tokenizers.huggingface.auto_tokenizer import AutoTokenizer +from nemo.collections.nlp.modules import BertModule, MegatronBertEncoder +from nemo.collections.nlp.modules.common.megatron.megatron_utils import compute_model_parallel_rank +from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer +from nemo.collections.nlp.parts.nlp_overrides import NLPCheckpointConnector +from nemo.core.classes import ModelPT +from nemo.core.classes.exportable import Exportable +from nemo.utils import AppState, logging +from nemo.utils.exp_manager import configure_checkpointing +from nemo.utils.get_rank import is_global_rank_zero + +__all__ = ['NLPModel'] + +NEMO_NLP_TMP = os.path.join(os.path.dirname(str(TRANSFORMERS_CACHE)), "nemo_nlp_tmp") + +os.makedirs(NEMO_NLP_TMP, exist_ok=True) + + +class NLPModel(ModelPT, Exportable): + """Base class for NLP Models. + """ + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + super().__init__(cfg, trainer) + self.set_world_size(trainer) + + def register_artifact(self, config_path: str, src: str, verify_src_exists: bool = False): + """ Overrides ModelPT register_artifact default behavior. NLP models usually need artifacts that are optional.""" + return super().register_artifact(config_path, src, verify_src_exists=verify_src_exists) + + @rank_zero_only + def register_bert_model(self): + """Adds encoder config to .nemo archive for Jarvis. + """ + # check if there is an encoder, warn if not + if self.bert_model is None: + raise ValueError('Instantiate self.bert_model before registering it.') + else: + # get encoder config and create source for artifact + if isinstance(self.bert_model, MegatronBertEncoder): + pretrained_model_name = self.bert_model._model_name + encoder_config_path = pretrained_model_name + '_encoder_config' + encoder_config_src = os.path.join(NEMO_NLP_TMP, encoder_config_path + '.json') + config_for_json = OmegaConf.to_container(self.bert_model.config) + with open(encoder_config_src, 'w', encoding='utf-8') as f: + f.write(json.dumps(config_for_json, indent=2, sort_keys=True) + '\n') + self.register_artifact('language_model.config_file', encoder_config_src) # for .nemo + elif isinstance(self.bert_model, BertModule): + # HuggingFace Transformer Config + pretrained_model_name = self.bert_model.name_or_path + # Some HF names have "/" in them so we replace with _ + pretrained_model_name = pretrained_model_name.replace("/", "_") + encoder_config_path = pretrained_model_name + '_encoder_config' + encoder_config_src = os.path.join(NEMO_NLP_TMP, encoder_config_path + '.json') + self.bert_model.config.to_json_file(encoder_config_src) # name requested by jarvis team + self.register_artifact('language_model.config_file', encoder_config_src) # for .nemo + else: + logging.info( + f'Registering BERT model config for {self.bert_model} is not yet supported. Please override this method if needed.' + ) + + def setup_tokenizer(self, cfg: DictConfig): + """Instantiates tokenizer based on config and registers tokenizer artifacts. + + If model is being restored from .nemo file then the tokenizer.vocab_file will + be used (if it exists). + + Otherwise, we will use the vocab file provided in the config (if it exists). + + Finally, if no vocab file is given (this happens frequently when using HF), + we will attempt to extract the vocab from the tokenizer object and then register it. + + Args: + cfg (DictConfig): Tokenizer config + """ + vocab_file = None + if cfg.vocab_file: + vocab_file = self.register_artifact(config_path='tokenizer.vocab_file', src=cfg.vocab_file) + tokenizer = get_tokenizer( + tokenizer_name=cfg.tokenizer_name, + vocab_file=vocab_file, + special_tokens=OmegaConf.to_container(cfg.special_tokens) if cfg.special_tokens else None, + tokenizer_model=self.register_artifact(config_path='tokenizer.tokenizer_model', src=cfg.tokenizer_model), + ) + self.tokenizer = tokenizer + + if vocab_file is None: + # when there is no vocab file we try to get the vocab from the tokenizer and register it + self._register_vocab_from_tokenizer(vocab_file_config_path='tokenizer.vocab_file', cfg=cfg) + + @rank_zero_only + def _register_vocab_from_tokenizer( + self, + vocab_file_config_path: str = 'tokenizer.vocab_file', + vocab_dict_config_path: str = 'tokenizer_vocab_dict', + cfg: DictConfig = None, + ): + """Creates vocab file from tokenizer if vocab file is None. + + Args: + vocab_file_config_path: path to the vocab_file in the config + vocab_dict_config_path: path to the vocab_dict in the config + cfg: tokenizer config + """ + if self.tokenizer is None: + raise ValueError('Instantiate self.tokenizer before registering vocab from it.') + else: + if isinstance(self.tokenizer, AutoTokenizer): + # extract vocab from tokenizer + vocab_dict = self.tokenizer.tokenizer.get_vocab() + + # for fast and slow tokenizer vocabularies compatibility + vocab_dict = dict(sorted(vocab_dict.items(), key=lambda item: item[1])) + + # get hash of vocab_dict to create a unique directory to write vocab_dict and vocab_file + m = hashlib.md5() + if 'tokenizer_name' in cfg: + if cfg.tokenizer_name is not None: + # different pretrained models with the same vocab will have different hash + m.update(cfg.tokenizer_name.encode()) + # get string representation of vocab_dict + vocab_dict_str = json.dumps(vocab_dict, sort_keys=True).encode() + m.update(vocab_dict_str) + vocab_dict_hash = m.hexdigest() + + hash_path = os.path.join(NEMO_NLP_TMP, vocab_dict_hash) + os.makedirs(hash_path, exist_ok=True) + + vocab_json_src = os.path.join(hash_path, vocab_dict_config_path) + + with open(vocab_json_src, 'w', encoding='utf-8') as f: + f.write(json.dumps(vocab_dict, indent=2, sort_keys=True) + '\n') + self.register_artifact(config_path=vocab_dict_config_path, src=vocab_json_src) + # create vocab file + vocab_file_src = os.path.join(hash_path, vocab_file_config_path) + with open(vocab_file_src, 'w', encoding='utf-8') as f: + for key in vocab_dict: + f.write(key + '\n') + + cfg.vocab_file = vocab_file_src + self.register_artifact(config_path=vocab_file_config_path, src=vocab_file_src) + else: + logging.info( + f'Registering tokenizer vocab for {self.tokenizer} is not yet supported. Please override this method if needed.' + ) + + def _clip_gradients(self, optimizer, clip_val=None): + """ Override of PTL Gradient Clipping. + Enables model parallel gradient clipping from Megatron-LM. + + Args: + optimizer ([type]): [description] + clip_val ([type], optional): [description]. Defaults to None. + """ + app_state = AppState() + + # get clip_val from trainer if None is provided + if clip_val is None: + clip_val = float(self._trainer.gradient_clip_val) + + if app_state.model_parallel_size is not None: + model = self._trainer.get_model() + parameters = model.parameters() + if mpu.model_parallel_is_initialized(): + mpu.grads.clip_grad_norm(parameters=parameters, max_norm=clip_val) + else: + raise ValueError('Model parallel groups must be intialized to use model parallel gradient clipping.') + + else: + return Accelerator._clip_gradients(self, optimizer, clip_val) + + def setup(self, stage: str) -> None: + """ PTL hook that is called on all DDP processes. """ + + if stage == 'fit': + + # adds self.bert_model config to .nemo file + if hasattr(self, 'bert_model') and self.bert_model is not None: + self.register_bert_model() + + app_state = AppState() + + if app_state.model_parallel_size is not None: + + self._trainer.checkpoint_connector = NLPCheckpointConnector(self._trainer) + + # # Configure checkpointing for model parallel + # if app_state.create_checkpoint_callback: + # # global rank 0 is configured by exp_manager + # if not is_global_rank_zero() and app_state.data_parallel_rank == 0: + # configure_checkpointing( + # self._trainer, + # app_state.log_dir, + # app_state.checkpoint_name, + # app_state.checkpoint_callback_params, + # ) + + def on_save_checkpoint(self, checkpoint: Dict[str, Any]) -> None: + """ LightningModule hook that's used to save things in addition to model weights. """ + + if hasattr(self, "bert_model") and isinstance(self.bert_model, MegatronBertEncoder): + checkpoint['checkpoint_version'] = get_checkpoint_version() + return None + + def on_load_checkpoint(self, checkpoint: Dict[str, Any]) -> None: + """ LightningModule hook that's used to restore things saved with on_save_checkpoint.""" + + if hasattr(self, "bert_model") and isinstance(self.bert_model, MegatronBertEncoder): + if get_checkpoint_version(): + assert ( + checkpoint['checkpoint_version'] == get_checkpoint_version() + ), 'checkpoint version found on_load_checkpoint different than get_checkpoint_version' + else: + set_checkpoint_version(checkpoint['checkpoint_version']) + logging.info(f"Setting Megatron checkpoint version: {checkpoint['checkpoint_version']}") + return None + + # no rank check as model parallel models need to be saved on data parallel rank 0 + def save_to(self, save_path: str): + """ + Saves model instance (weights and configuration) into .nemo file + You can use "restore_from" method to fully restore instance from .nemo file. + + .nemo file is an archive (tar.gz) with the following: + model_config.yaml - model configuration in .yaml format. You can deserialize this into cfg argument for model's constructor + model_weights.ckpt - model checkpoint + + Args: + save_path: Path to .nemo file where model instance should be saved + """ + save_path = os.path.abspath(save_path) + app_state = AppState() + if app_state.model_parallel_size is not None: + self._default_save_to(save_path) + else: + # super.save_to only runs on global rank 0 + return super().save_to(save_path) + + def _default_save_to(self, save_path: str): + app_state = AppState() + if app_state.model_parallel_size is not None: + # each model parallel rank creates a .nemo file + # after all .nemo files are created, each rank + # will add their checkpoint to global rank 0 + + base_dir = os.path.dirname(save_path) # use the directory to merge mp_rank .nemo files into one + + # update save_path based on model parallel_rank + base_path = os.path.splitext(save_path)[0] # everything except the extension + + mp_save_path = f'{base_path}_mp_rank_{app_state.model_parallel_rank:02d}.nemo' + + if app_state.data_parallel_rank == 0: + super()._default_save_to(mp_save_path) + + # barrier so that all processes have finished writing their weights before creating .nemo file + torch.distributed.barrier() + + if is_global_rank_zero(): + # extract all tar files + for mp_rank in range(app_state.model_parallel_size): + mp_tar_path = f'{base_path}_mp_rank_{mp_rank:02d}.nemo' + mp_tar = tarfile.open(mp_tar_path, 'r:gz') + mp_tar.extractall(path=os.path.join(base_dir, f'mp_rank_{mp_rank:02d}')) + mp_tar.close() + os.remove(mp_tar_path) + + # move rank 0 .nemo extract to base_path + shutil.move(os.path.join(base_dir, 'mp_rank_00'), base_path) + + # move mp_rank_00 checkpoint to mp_rank_00 directory inside base_path + os.mkdir(os.path.join(base_path, 'mp_rank_00')) + shutil.move(os.path.join(base_path, 'model_weights.ckpt'), os.path.join(base_path, 'mp_rank_00')) + + # move other mp_rank checkpoints from base_dir to base_path + for mp_rank in range(1, app_state.model_parallel_size): + os.mkdir(os.path.join(base_path, f'mp_rank_{mp_rank:02d}')) + shutil.move( + os.path.join(base_dir, f'mp_rank_{mp_rank:02d}', 'model_weights.ckpt'), + os.path.join(base_path, f'mp_rank_{mp_rank:02d}'), + ) + # clean up leftover directory + shutil.rmtree(os.path.join(base_dir, f'mp_rank_{mp_rank:02d}')) + + # create tar file from base_path + self._make_nemo_file_from_folder(save_path, base_path) + + # clean up base_path + shutil.rmtree(base_path) + + elif is_global_rank_zero(): + return super()._default_save_to(save_path) + else: + return + + @classmethod + def restore_from( + cls, + restore_path: str, + override_config_path: Optional[Union[OmegaConf, str]] = None, + map_location: Optional[torch.device] = None, + strict: bool = True, + return_config: bool = False, + trainer: Trainer = None, + ): + """ + Restores model instance (weights and configuration) from .nemo file. + + Args: + restore_path: path to .nemo file from which model should be instantiated + override_config_path: path to a yaml config that will override the internal + config file or an OmegaConf / DictConfig object representing the model config. + map_location: Optional torch.device() to map the instantiated model to a device. + By default (None), it will select a GPU if available, falling back to CPU otherwise. + strict: Passed to load_state_dict. Set to True by default. + return_config: If set to true, will return just the underlying config of the restored + model as an OmegaConf DictConfig object without instantiating the model. + trainer: PyTorch Lightning trainer. Must be passed in order to use model parallel .nemo + + Example: + ``` + model = nemo.collections.nlp.models.TokenClassificationModel.restore_from('token_classification.nemo') + assert isinstance(model, nemo.collections.nlp.models.TokenClassificationModel) + ``` + + Returns: + An instance of type cls or its underlying config (if return_config is set). + """ + if not os.path.exists(restore_path): + raise FileNotFoundError(f"Can't find {restore_path}") + + app_state = AppState() + app_state.model_restore_path = os.path.abspath(os.path.expanduser(restore_path)) + + # detect if we have a model parallel .nemo file + with tempfile.TemporaryDirectory() as tmpdir: + cwd = os.getcwd() + os.chdir(tmpdir) + # detect if model parallel from tarfile + tar = tarfile.open(app_state.model_restore_path, "r:gz") + names = tar.getnames() + mp_ranks = [] + for name in names: + if 'mp_rank' in name: + mp_ranks.append(name) + if mp_ranks: + app_state.model_parallel_size = len(mp_ranks) // 2 # directory and file are included in getnames() + + # get checkpoint version + checkpoint_version_member = None + for member in tar.getmembers(): + if 'megatron_checkpoint_version.json' in member.name: + checkpoint_version_member = member + tar.extract(checkpoint_version_member, tmpdir) + with open(checkpoint_version_member.name, 'r') as f: + checkpoint_version = json.load(f).get('checkpoint_version', None) + logging.info( + ( + f'Detected model parallel .nemo file: {restore_path}. ' + f'Assuming megatron model parallelism with ' + f'model_parallel_size: {app_state.model_parallel_size} ' + f'and checkpoint version: {checkpoint_version}' + ) + ) + tar.close() + os.chdir(cwd) + + if app_state.model_parallel_size is not None: + if not isinstance(trainer, Trainer): + raise ValueError("trainer must be a PyTorch Lightning Trainer to restore model parallel .nemo files.") + + if checkpoint_version is None: + raise ValueError( + "Restoring from megatron model parallel .nemo but could not find megatron checkpoint version." + ) + else: + logging.info(f"Setting megatron checkpoint version: {checkpoint_version}") + set_checkpoint_version(checkpoint_version) + + app_state.world_size = trainer.num_gpus * trainer.num_nodes + + if trainer.local_rank is not None: + app_state.local_rank = trainer.local_rank + else: + raise ValueError("trainer.local_rank is None. local_rank needed to restore model parallel models.") + + model_parallel_rank = compute_model_parallel_rank(trainer.local_rank, app_state.model_parallel_size) + app_state.model_parallel_rank = model_parallel_rank + + restored_model = cls._default_restore_from( + restore_path, override_config_path, map_location, strict, return_config + ) + restored_model.set_trainer(trainer) + return restored_model + else: + return super().restore_from(restore_path, override_config_path, map_location, strict, return_config) + + @rank_zero_only + def register_megatron_checkpoint_version(self): + """ Adds checkpoint version to .nemo archive """ + if self.bert_model is None: + raise ValueError('Instantiate self.bert_model before registering megatron checkpoint version.') + else: + # get encoder config and create source for artifact + if isinstance(self.bert_model, MegatronBertEncoder): + checkpoint_version = get_checkpoint_version() + if checkpoint_version is None: + raise ValueError('Unable to get megatron checkpoint version.') + else: + checkpoint_version_dict = {'checkpoint_version': checkpoint_version} + checkpoint_version_path = 'megatron_checkpoint_version.json' + checkpoint_version_src = os.path.join(NEMO_NLP_TMP, checkpoint_version_path) + with open(checkpoint_version_src, 'w') as f: + f.write(json.dumps(checkpoint_version_dict)) + self.register_artifact(checkpoint_version_path, checkpoint_version_src) + + @staticmethod + def _unpack_nemo_file(path2file: str, out_folder: str) -> str: + return super(NLPModel, NLPModel)._unpack_nemo_file(path2file, out_folder) + + @staticmethod + def _make_nemo_file_from_folder(filename, source_dir): + return super(NLPModel, NLPModel)._make_nemo_file_from_folder(filename, source_dir) + + @property + def input_module(self): + return self.bert_model + + @property + def output_module(self): + return self.classifier diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/question_answering/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/question_answering/__init__.py new file mode 100644 index 0000000000..4ced15ed8a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/question_answering/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.models.question_answering.qa_model import QAModel diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/question_answering/qa_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/question_answering/qa_model.py new file mode 100644 index 0000000000..652074eaa6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/question_answering/qa_model.py @@ -0,0 +1,404 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import Dict, Optional + +import torch +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer +from torch.utils.data import DataLoader + +from nemo.collections.common.losses import SpanningLoss +from nemo.collections.nlp.data import SquadDataset +from nemo.collections.nlp.data.question_answering_squad.qa_squad_processing import ( + EVALUATION_MODE, + INFERENCE_MODE, + TRAINING_MODE, +) +from nemo.collections.nlp.models.nlp_model import NLPModel +from nemo.collections.nlp.modules.common import TokenClassifier +from nemo.collections.nlp.modules.common.lm_utils import get_lm_model +from nemo.collections.nlp.parts.utils_funcs import tensor2list +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types import NeuralType +from nemo.utils import logging + +__all__ = ['QAModel'] + + +class QAModel(NLPModel): + """ + BERT encoder with QA head training. + """ + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return self.bert_model.input_types + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return self.classifier.output_types + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + self.setup_tokenizer(cfg.tokenizer) + super().__init__(cfg=cfg, trainer=trainer) + self.bert_model = get_lm_model( + pretrained_model_name=cfg.language_model.pretrained_model_name, + config_file=self.register_artifact('language_model.config_file', cfg.language_model.config_file), + config_dict=OmegaConf.to_container(cfg.language_model.config) if cfg.language_model.config else None, + checkpoint_file=cfg.language_model.lm_checkpoint, + vocab_file=self.register_artifact('tokenizer.vocab_file', cfg.tokenizer.vocab_file) + if cfg.tokenizer is not None + else None, + ) + + self.classifier = TokenClassifier( + hidden_size=self.bert_model.config.hidden_size, + num_classes=cfg.token_classifier.num_classes, + num_layers=cfg.token_classifier.num_layers, + activation=cfg.token_classifier.activation, + log_softmax=cfg.token_classifier.log_softmax, + dropout=cfg.token_classifier.dropout, + use_transformer_init=cfg.token_classifier.use_transformer_init, + ) + + self.loss = SpanningLoss() + + @typecheck() + def forward(self, input_ids, token_type_ids, attention_mask): + hidden_states = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask + ) + logits = self.classifier(hidden_states=hidden_states) + return logits + + def training_step(self, batch, batch_idx): + input_ids, input_type_ids, input_mask, unique_ids, start_positions, end_positions = batch + logits = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + loss, _, _ = self.loss(logits=logits, start_positions=start_positions, end_positions=end_positions) + + lr = self._optimizer.param_groups[0]['lr'] + self.log('train_loss', loss) + self.log('lr', lr, prog_bar=True) + return {'loss': loss, 'lr': lr} + + def validation_step(self, batch, batch_idx): + if self.trainer.testing: + prefix = 'test' + else: + prefix = 'val' + + input_ids, input_type_ids, input_mask, unique_ids, start_positions, end_positions = batch + logits = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + loss, start_logits, end_logits = self.loss( + logits=logits, start_positions=start_positions, end_positions=end_positions + ) + + tensors = { + 'unique_ids': unique_ids, + 'start_logits': start_logits, + 'end_logits': end_logits, + } + return {f'{prefix}_loss': loss, f'{prefix}_tensors': tensors} + + def test_step(self, batch, batch_idx): + return self.validation_step(batch, batch_idx) + + def validation_epoch_end(self, outputs): + if self.trainer.testing: + prefix = 'test' + else: + prefix = 'val' + + avg_loss = torch.stack([x[f'{prefix}_loss'] for x in outputs]).mean() + + unique_ids = torch.cat([x[f'{prefix}_tensors']['unique_ids'] for x in outputs]) + start_logits = torch.cat([x[f'{prefix}_tensors']['start_logits'] for x in outputs]) + end_logits = torch.cat([x[f'{prefix}_tensors']['end_logits'] for x in outputs]) + + all_unique_ids = [] + all_start_logits = [] + all_end_logits = [] + if torch.distributed.is_initialized(): + world_size = torch.distributed.get_world_size() + for ind in range(world_size): + all_unique_ids.append(torch.empty_like(unique_ids)) + all_start_logits.append(torch.empty_like(start_logits)) + all_end_logits.append(torch.empty_like(end_logits)) + torch.distributed.all_gather(all_unique_ids, unique_ids) + torch.distributed.all_gather(all_start_logits, start_logits) + torch.distributed.all_gather(all_end_logits, end_logits) + else: + all_unique_ids.append(unique_ids) + all_start_logits.append(start_logits) + all_end_logits.append(end_logits) + + exact_match, f1, all_predictions, all_nbest = -1, -1, [], [] + if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + + unique_ids = [] + start_logits = [] + end_logits = [] + for u in all_unique_ids: + unique_ids.extend(tensor2list(u)) + for u in all_start_logits: + start_logits.extend(tensor2list(u)) + for u in all_end_logits: + end_logits.extend(tensor2list(u)) + + eval_dataset = self._test_dl.dataset if self.trainer.testing else self._validation_dl.dataset + exact_match, f1, all_predictions, all_nbest = eval_dataset.evaluate( + unique_ids=unique_ids, + start_logits=start_logits, + end_logits=end_logits, + n_best_size=self._cfg.dataset.n_best_size, + max_answer_length=self._cfg.dataset.max_answer_length, + version_2_with_negative=self._cfg.dataset.version_2_with_negative, + null_score_diff_threshold=self._cfg.dataset.null_score_diff_threshold, + do_lower_case=self._cfg.dataset.do_lower_case, + ) + + logging.info(f"{prefix} exact match {exact_match}") + logging.info(f"{prefix} f1 {f1}") + + self.log(f'{prefix}_loss', avg_loss) + self.log(f'{prefix}_exact_match', exact_match) + self.log(f'{prefix}_f1', f1) + + def test_epoch_end(self, outputs): + return self.validation_epoch_end(outputs) + + @torch.no_grad() + def inference( + self, + file: str, + batch_size: int = 1, + num_samples: int = -1, + output_nbest_file: Optional[str] = None, + output_prediction_file: Optional[str] = None, + ): + """ + Get prediction for unlabeled inference data + + Args: + file: inference data + batch_size: batch size to use during inference + num_samples: number of samples to use of inference data. Default: -1 if all data should be used. + output_nbest_file: optional output file for writing out nbest list + output_prediction_file: optional output file for writing out predictions + + Returns: + model predictions, model nbest list + """ + # store predictions for all queries in a single list + all_predictions = [] + all_nbest = [] + mode = self.training + device = 'cuda' if torch.cuda.is_available() else 'cpu' + try: + # Switch model to evaluation mode + self.eval() + self.to(device) + logging_level = logging.get_verbosity() + logging.set_verbosity(logging.WARNING) + dataloader_cfg = { + "batch_size": batch_size, + "file": file, + "shuffle": False, + "num_samples": num_samples, + 'num_workers': 2, + 'pin_memory': False, + 'drop_last': False, + } + dataloader_cfg = OmegaConf.create(dataloader_cfg) + infer_datalayer = self._setup_dataloader_from_config(cfg=dataloader_cfg, mode=INFERENCE_MODE) + + all_logits = [] + all_unique_ids = [] + for i, batch in enumerate(infer_datalayer): + input_ids, token_type_ids, attention_mask, unique_ids = batch + logits = self.forward( + input_ids=input_ids.to(device), + token_type_ids=token_type_ids.to(device), + attention_mask=attention_mask.to(device), + ) + all_logits.append(logits) + all_unique_ids.append(unique_ids) + logits = torch.cat(all_logits) + unique_ids = tensor2list(torch.cat(all_unique_ids)) + s, e = logits.split(dim=-1, split_size=1) + start_logits = tensor2list(s.squeeze()) + end_logits = tensor2list(e.squeeze()) + (all_predictions, all_nbest, scores_diff) = infer_datalayer.dataset.get_predictions( + unique_ids=unique_ids, + start_logits=start_logits, + end_logits=end_logits, + n_best_size=self._cfg.dataset.n_best_size, + max_answer_length=self._cfg.dataset.max_answer_length, + version_2_with_negative=self._cfg.dataset.version_2_with_negative, + null_score_diff_threshold=self._cfg.dataset.null_score_diff_threshold, + do_lower_case=self._cfg.dataset.do_lower_case, + ) + + with open(file, 'r') as test_file_fp: + test_data = json.load(test_file_fp)["data"] + id_to_question_mapping = {} + for title in test_data: + for par in title["paragraphs"]: + for question in par["qas"]: + id_to_question_mapping[question["id"]] = question["question"] + + for question_id in all_predictions: + all_predictions[question_id] = (id_to_question_mapping[question_id], all_predictions[question_id]) + + if output_nbest_file is not None: + with open(output_nbest_file, "w") as writer: + writer.write(json.dumps(all_nbest, indent=4) + "\n") + if output_prediction_file is not None: + with open(output_prediction_file, "w") as writer: + writer.write(json.dumps(all_predictions, indent=4) + "\n") + + finally: + # set mode back to its original value + self.train(mode=mode) + logging.set_verbosity(logging_level) + + return all_predictions, all_nbest + + def setup_training_data(self, train_data_config: Optional[DictConfig]): + if not train_data_config or not train_data_config.file: + logging.info( + f"Dataloader config or file_path for the train is missing, so no data loader for test is created!" + ) + self._test_dl = None + return + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config, mode=TRAINING_MODE) + + def setup_validation_data(self, val_data_config: Optional[DictConfig]): + if not val_data_config or not val_data_config.file: + logging.info( + f"Dataloader config or file_path for the validation is missing, so no data loader for test is created!" + ) + self._test_dl = None + return + self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config, mode=EVALUATION_MODE) + + def setup_test_data(self, test_data_config: Optional[DictConfig]): + if not test_data_config or test_data_config.file is None: + logging.info( + f"Dataloader config or file_path for the test is missing, so no data loader for test is created!" + ) + self._test_dl = None + return + self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config, mode=EVALUATION_MODE) + + def _setup_dataloader_from_config(self, cfg: DictConfig, mode: str): + dataset = SquadDataset( + tokenizer=self.tokenizer, + data_file=cfg.file, + doc_stride=self._cfg.dataset.doc_stride, + max_query_length=self._cfg.dataset.max_query_length, + max_seq_length=self._cfg.dataset.max_seq_length, + version_2_with_negative=self._cfg.dataset.version_2_with_negative, + num_samples=cfg.num_samples, + mode=mode, + use_cache=self._cfg.dataset.use_cache, + ) + + dl = torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg.batch_size, + collate_fn=dataset.collate_fn, + drop_last=cfg.drop_last, + shuffle=cfg.shuffle, + num_workers=cfg.num_workers, + pin_memory=cfg.pin_memory, + ) + return dl + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + result = [] + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv1.1_bertbase", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv1_1_bertbase/versions/1.0.0rc1/files/qa_squadv1.1_bertbase.nemo", + description="Question answering model finetuned from NeMo BERT Base Uncased on SQuAD v1.1 dataset which obtains an exact match (EM) score of 82.78% and an F1 score of 82.78%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv2.0_bertbase", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv2_0_bertbase/versions/1.0.0rc1/files/qa_squadv2.0_bertbase.nemo", + description="Question answering model finetuned from NeMo BERT Base Uncased on SQuAD v2.0 dataset which obtains an exact match (EM) score of 75.04% and an F1 score of 78.08%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv1_1_bertlarge", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv1_1_bertlarge/versions/1.0.0rc1/files/qa_squadv1.1_bertlarge.nemo", + description="Question answering model finetuned from NeMo BERT Large Uncased on SQuAD v1.1 dataset which obtains an exact match (EM) score of 85.44% and an F1 score of 92.06%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv2.0_bertlarge", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv2_0_bertlarge/versions/1.0.0rc1/files/qa_squadv2.0_bertlarge.nemo", + description="Question answering model finetuned from NeMo BERT Large Uncased on SQuAD v2.0 dataset which obtains an exact match (EM) score of 80.22% and an F1 score of 83.05%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv1_1_megatron_cased", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv1_1_megatron_cased/versions/1.0.0rc1/files/qa_squadv1.1_megatron_cased.nemo", + description="Question answering model finetuned from Megatron Cased on SQuAD v1.1 dataset which obtains an exact match (EM) score of 88.18% and an F1 score of 94.07%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv2.0_megatron_cased", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv2_0_megatron_cased/versions/1.0.0rc1/files/qa_squadv2.0_megatron_cased.nemo", + description="Question answering model finetuned from Megatron Cased on SQuAD v2.0 dataset which obtains an exact match (EM) score of 84.73% and an F1 score of 87.89%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv1.1_megatron_uncased", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv1_1_megatron_uncased/versions/1.0.0rc1/files/qa_squadv1.1_megatron_uncased.nemo", + description="Question answering model finetuned from Megatron Unased on SQuAD v1.1 dataset which obtains an exact match (EM) score of 87.61% and an F1 score of 94.00%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv2.0_megatron_uncased", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv2_0_megatron_uncased/versions/1.0.0rc1/files/qa_squadv2.0_megatron_uncased.nemo", + description="Question answering model finetuned from Megatron Uncased on SQuAD v2.0 dataset which obtains an exact match (EM) score of 84.48% and an F1 score of 87.65%.", + ) + ) + return result diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/text_classification/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/text_classification/__init__.py new file mode 100644 index 0000000000..10ef0f00b8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/text_classification/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.models.text_classification.text_classification_model import TextClassificationModel diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/text_classification/text_classification_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/text_classification/text_classification_model.py new file mode 100644 index 0000000000..ffb9b411ef --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/text_classification/text_classification_model.py @@ -0,0 +1,324 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from typing import Dict, List, Optional + +import torch +from omegaconf import DictConfig +from pytorch_lightning import Trainer + +from nemo.collections.common.losses import CrossEntropyLoss +from nemo.collections.nlp.data.text_classification import TextClassificationDataset, calc_class_weights +from nemo.collections.nlp.metrics.classification_report import ClassificationReport +from nemo.collections.nlp.models.nlp_model import NLPModel +from nemo.collections.nlp.modules.common import SequenceClassifier +from nemo.collections.nlp.modules.common.lm_utils import get_lm_model +from nemo.collections.nlp.parts.utils_funcs import tensor2list +from nemo.core.classes.common import typecheck +from nemo.core.classes.exportable import Exportable +from nemo.core.neural_types import NeuralType +from nemo.utils import logging + +__all__ = ['TextClassificationModel'] + + +class TextClassificationModel(NLPModel, Exportable): + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return self.bert_model.input_types + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return self.classifier.output_types + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + """Initializes the BERTTextClassifier model.""" + + # shared params for dataset and data loaders + self.dataset_cfg = cfg.dataset + # tokenizer needs to get initialized before the super.__init__() + # as dataloaders and datasets need it to process the data + self.setup_tokenizer(cfg.tokenizer) + + self.class_weights = None + + super().__init__(cfg=cfg, trainer=trainer) + + self.bert_model = get_lm_model( + pretrained_model_name=cfg.language_model.pretrained_model_name, + config_file=self.register_artifact('language_model.config_file', cfg.language_model.config_file), + config_dict=cfg.language_model.config, + checkpoint_file=cfg.language_model.lm_checkpoint, + vocab_file=self.register_artifact('tokenizer.vocab_file', cfg.tokenizer.vocab_file), + ) + + self.classifier = SequenceClassifier( + hidden_size=self.bert_model.config.hidden_size, + num_classes=cfg.dataset.num_classes, + num_layers=cfg.classifier_head.num_output_layers, + activation='relu', + log_softmax=False, + dropout=cfg.classifier_head.fc_dropout, + use_transformer_init=True, + idx_conditioned_on=0, + ) + + self.create_loss_module() + + # setup to track metrics + self.classification_report = ClassificationReport( + num_classes=cfg.dataset.num_classes, mode='micro', dist_sync_on_step=True + ) + + # register the file containing the labels into the artifacts to get stored in the '.nemo' file later + if 'class_labels' in cfg and 'class_labels_file' in cfg.class_labels and cfg.class_labels.class_labels_file: + self.register_artifact('class_labels', cfg.class_labels.class_labels_file) + + def create_loss_module(self): + # create the loss module if it is not yet created by the training data loader + if not hasattr(self, 'loss'): + if hasattr(self, 'class_weights') and self.class_weights: + # You may need to increase the number of epochs for convergence when using weighted_loss + self.loss = CrossEntropyLoss(weight=self.class_weights) + else: + self.loss = CrossEntropyLoss() + + @typecheck() + def forward(self, input_ids, token_type_ids, attention_mask): + """ + No special modification required for Lightning, define it as you normally would + in the `nn.Module` in vanilla PyTorch. + """ + hidden_states = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask + ) + logits = self.classifier(hidden_states=hidden_states) + return logits + + def training_step(self, batch, batch_idx): + """ + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. + """ + # forward pass + input_ids, input_type_ids, input_mask, labels = batch + logits = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + + train_loss = self.loss(logits=logits, labels=labels) + + lr = self._optimizer.param_groups[0]['lr'] + + self.log('train_loss', train_loss) + self.log('lr', lr, prog_bar=True) + + return { + 'loss': train_loss, + 'lr': lr, + } + + def validation_step(self, batch, batch_idx): + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. + """ + input_ids, input_type_ids, input_mask, labels = batch + logits = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + + val_loss = self.loss(logits=logits, labels=labels) + + preds = torch.argmax(logits, axis=-1) + + tp, fn, fp, _ = self.classification_report(preds, labels) + + return {'val_loss': val_loss, 'tp': tp, 'fn': fn, 'fp': fp} + + def validation_epoch_end(self, outputs): + """ + Called at the end of validation to aggregate outputs. + :param outputs: list of individual outputs of each validation step. + """ + if not outputs: + return {} + if self.trainer.testing: + prefix = 'test' + else: + prefix = 'val' + + avg_loss = torch.stack([x[f'val_loss'] for x in outputs]).mean() + + # calculate metrics and classification report + precision, recall, f1, report = self.classification_report.compute() + + logging.info(f'{prefix}_report: {report}') + + self.log(f'{prefix}_loss', avg_loss, prog_bar=True) + self.log(f'{prefix}_precision', precision) + self.log(f'{prefix}_f1', f1) + self.log(f'{prefix}_recall', recall) + + self.classification_report.reset() + + def test_step(self, batch, batch_idx): + """ + Lightning calls this inside the test loop with the data from the test dataloader + passed in as `batch`. + """ + return self.validation_step(batch, batch_idx) + + def test_epoch_end(self, outputs): + """ + Called at the end of test to aggregate outputs. + :param outputs: list of individual outputs of each test step. + """ + return self.validation_epoch_end(outputs) + + def setup_training_data(self, train_data_config: Optional[DictConfig]): + if not train_data_config or not train_data_config.file_path: + logging.info( + f"Dataloader config or file_path for the train is missing, so no data loader for test is created!" + ) + self._test_dl = None + return + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config) + + # calculate the class weights to be used in the loss function + if self.cfg.dataset.class_balancing == 'weighted_loss': + self.class_weights = calc_class_weights(train_data_config.file_path, self.cfg.dataset.num_classes) + else: + self.class_weights = None + # we need to create/update the loss module by using the weights calculated from the training data + self.create_loss_module() + + def setup_validation_data(self, val_data_config: Optional[DictConfig]): + if not val_data_config or not val_data_config.file_path: + logging.info( + f"Dataloader config or file_path for the validation is missing, so no data loader for test is created!" + ) + self._test_dl = None + return + self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config) + + def setup_test_data(self, test_data_config: Optional[DictConfig]): + if not test_data_config or not test_data_config.file_path: + logging.info( + f"Dataloader config or file_path for the test is missing, so no data loader for test is created!" + ) + self._test_dl = None + return + self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config) + + def _setup_dataloader_from_config(self, cfg: Dict) -> 'torch.utils.data.DataLoader': + input_file = cfg.file_path + if not os.path.exists(input_file): + raise FileNotFoundError( + f'{input_file} not found! The data should be be stored in TAB-separated files \n\ + "validation_ds.file_path" and "train_ds.file_path" for train and evaluation respectively. \n\ + Each line of the files contains text sequences, where words are separated with spaces. \n\ + The label of the example is separated with TAB at the end of each line. \n\ + Each line of the files should follow the format: \n\ + [WORD][SPACE][WORD][SPACE][WORD][...][TAB][LABEL]' + ) + + dataset = TextClassificationDataset( + tokenizer=self.tokenizer, + input_file=input_file, + max_seq_length=self.dataset_cfg.max_seq_length, + num_samples=cfg.get("num_samples", -1), + shuffle=cfg.shuffle, + use_cache=self.dataset_cfg.use_cache, + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg.batch_size, + shuffle=cfg.shuffle, + num_workers=cfg.get("num_workers", 0), + pin_memory=cfg.get("pin_memory", False), + drop_last=cfg.get("drop_last", False), + collate_fn=dataset.collate_fn, + ) + + @torch.no_grad() + def classifytext(self, queries: List[str], batch_size: int = 1, max_seq_length: int = -1) -> List[int]: + """ + Get prediction for the queries + Args: + queries: text sequences + batch_size: batch size to use during inference + max_seq_length: sequences longer than max_seq_length will get truncated. default -1 disables truncation. + Returns: + all_preds: model predictions + """ + # store predictions for all queries in a single list + all_preds = [] + mode = self.training + device = next(self.parameters()).device + try: + # Switch model to evaluation mode + self.eval() + logging_level = logging.get_verbosity() + logging.set_verbosity(logging.WARNING) + dataloader_cfg = {"batch_size": batch_size, "num_workers": 3, "pin_memory": False} + infer_datalayer = self._setup_infer_dataloader(dataloader_cfg, queries, max_seq_length) + + for i, batch in enumerate(infer_datalayer): + input_ids, input_type_ids, input_mask, subtokens_mask = batch + + logits = self.forward( + input_ids=input_ids.to(device), + token_type_ids=input_type_ids.to(device), + attention_mask=input_mask.to(device), + ) + + preds = tensor2list(torch.argmax(logits, axis=-1)) + all_preds.extend(preds) + finally: + # set mode back to its original value + self.train(mode=mode) + logging.set_verbosity(logging_level) + return all_preds + + def _setup_infer_dataloader( + self, cfg: Dict, queries: List[str], max_seq_length: int = -1 + ) -> 'torch.utils.data.DataLoader': + """ + Setup function for a infer data loader. + + Args: + cfg: config dictionary containing data loader params like batch_size, num_workers and pin_memory + queries: text + max_seq_length: maximum length of queries, default is -1 for no limit + Returns: + A pytorch DataLoader. + """ + dataset = TextClassificationDataset(tokenizer=self.tokenizer, queries=queries, max_seq_length=max_seq_length) + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg["batch_size"], + shuffle=False, + num_workers=cfg.get("num_workers", 0), + pin_memory=cfg.get("pin_memory", False), + drop_last=False, + collate_fn=dataset.collate_fn, + ) + + @classmethod + def list_available_models(cls) -> Optional[Dict[str, str]]: + pass + + @classmethod + def from_pretrained(cls, name: str): + pass diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/__init__.py new file mode 100644 index 0000000000..61b79282c2 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.collections.nlp.models.token_classification.punctuation_capitalization_model import ( + PunctuationCapitalizationModel, +) +from nemo.collections.nlp.models.token_classification.token_classification_model import TokenClassificationModel diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/punctuation_capitalization_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/punctuation_capitalization_model.py new file mode 100644 index 0000000000..93599197ca --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/punctuation_capitalization_model.py @@ -0,0 +1,490 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Dict, List, Optional + +import onnx +import torch +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer + +from nemo.collections.common.losses import AggregatorLoss, CrossEntropyLoss +from nemo.collections.nlp.data.token_classification.punctuation_capitalization_dataset import ( + BertPunctuationCapitalizationDataset, + BertPunctuationCapitalizationInferDataset, +) +from nemo.collections.nlp.metrics.classification_report import ClassificationReport +from nemo.collections.nlp.models.nlp_model import NLPModel +from nemo.collections.nlp.modules.common import TokenClassifier +from nemo.collections.nlp.modules.common.lm_utils import get_lm_model +from nemo.collections.nlp.parts.utils_funcs import tensor2list +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.classes.exportable import Exportable, ExportFormat +from nemo.core.neural_types import LogitsType, NeuralType +from nemo.utils import logging + +__all__ = ['PunctuationCapitalizationModel'] + + +class PunctuationCapitalizationModel(NLPModel, Exportable): + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return self.bert_model.input_types + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "punct_logits": NeuralType(('B', 'T', 'C'), LogitsType()), + "capit_logits": NeuralType(('B', 'T', 'C'), LogitsType()), + } + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + """ + Initializes BERT Punctuation and Capitalization model. + """ + self.setup_tokenizer(cfg.tokenizer) + + super().__init__(cfg=cfg, trainer=trainer) + + self.bert_model = get_lm_model( + pretrained_model_name=cfg.language_model.pretrained_model_name, + config_file=self.register_artifact('language_model.config_file', cfg.language_model.config_file), + config_dict=OmegaConf.to_container(cfg.language_model.config) if cfg.language_model.config else None, + checkpoint_file=cfg.language_model.lm_checkpoint, + vocab_file=self.register_artifact('tokenizer.vocab_file', cfg.tokenizer.vocab_file), + ) + + self.punct_classifier = TokenClassifier( + hidden_size=self.bert_model.config.hidden_size, + num_classes=len(self._cfg.punct_label_ids), + activation=cfg.punct_head.activation, + log_softmax=False, + dropout=cfg.punct_head.fc_dropout, + num_layers=cfg.punct_head.punct_num_fc_layers, + use_transformer_init=cfg.punct_head.use_transformer_init, + ) + + self.capit_classifier = TokenClassifier( + hidden_size=self.bert_model.config.hidden_size, + num_classes=len(self._cfg.capit_label_ids), + activation=cfg.capit_head.activation, + log_softmax=False, + dropout=cfg.capit_head.fc_dropout, + num_layers=cfg.capit_head.capit_num_fc_layers, + use_transformer_init=cfg.capit_head.use_transformer_init, + ) + + self.loss = CrossEntropyLoss(logits_ndim=3) + self.agg_loss = AggregatorLoss(num_inputs=2) + + # setup to track metrics + self.punct_class_report = ClassificationReport( + num_classes=len(self._cfg.punct_label_ids), + label_ids=self._cfg.punct_label_ids, + mode='macro', + dist_sync_on_step=True, + ) + self.capit_class_report = ClassificationReport( + num_classes=len(self._cfg.capit_label_ids), + label_ids=self._cfg.capit_label_ids, + mode='macro', + dist_sync_on_step=True, + ) + + @typecheck() + def forward(self, input_ids, attention_mask, token_type_ids=None): + """ + No special modification required for Lightning, define it as you normally would + in the `nn.Module` in vanilla PyTorch. + """ + hidden_states = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask + ) + punct_logits = self.punct_classifier(hidden_states=hidden_states) + capit_logits = self.capit_classifier(hidden_states=hidden_states) + return punct_logits, capit_logits + + def _make_step(self, batch): + input_ids, input_type_ids, input_mask, subtokens_mask, loss_mask, punct_labels, capit_labels = batch + punct_logits, capit_logits = self( + input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask + ) + + punct_loss = self.loss(logits=punct_logits, labels=punct_labels, loss_mask=loss_mask) + capit_loss = self.loss(logits=capit_logits, labels=capit_labels, loss_mask=loss_mask) + loss = self.agg_loss(loss_1=punct_loss, loss_2=capit_loss) + return loss, punct_logits, capit_logits + + def training_step(self, batch, batch_idx): + """ + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. + """ + loss, _, _ = self._make_step(batch) + lr = self._optimizer.param_groups[0]['lr'] + + self.log('lr', lr, prog_bar=True) + self.log('train_loss', loss) + + return {'loss': loss, 'lr': lr} + + def validation_step(self, batch, batch_idx, dataloader_idx=0): + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. + """ + _, _, _, subtokens_mask, _, punct_labels, capit_labels = batch + val_loss, punct_logits, capit_logits = self._make_step(batch) + + subtokens_mask = subtokens_mask > 0.5 + punct_preds = torch.argmax(punct_logits, axis=-1)[subtokens_mask] + punct_labels = punct_labels[subtokens_mask] + self.punct_class_report.update(punct_preds, punct_labels) + + capit_preds = torch.argmax(capit_logits, axis=-1)[subtokens_mask] + capit_labels = capit_labels[subtokens_mask] + self.capit_class_report.update(capit_preds, capit_labels) + + return { + 'val_loss': val_loss, + 'punct_tp': self.punct_class_report.tp, + 'punct_fn': self.punct_class_report.fn, + 'punct_fp': self.punct_class_report.fp, + 'capit_tp': self.capit_class_report.tp, + 'capit_fn': self.capit_class_report.fn, + 'capit_fp': self.capit_class_report.fp, + } + + def test_step(self, batch, batch_idx, dataloader_idx=0): + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. + """ + _, _, _, subtokens_mask, _, punct_labels, capit_labels = batch + test_loss, punct_logits, capit_logits = self._make_step(batch) + + subtokens_mask = subtokens_mask > 0.5 + punct_preds = torch.argmax(punct_logits, axis=-1)[subtokens_mask] + punct_labels = punct_labels[subtokens_mask] + self.punct_class_report.update(punct_preds, punct_labels) + + capit_preds = torch.argmax(capit_logits, axis=-1)[subtokens_mask] + capit_labels = capit_labels[subtokens_mask] + self.capit_class_report.update(capit_preds, capit_labels) + + return { + 'test_loss': test_loss, + 'punct_tp': self.punct_class_report.tp, + 'punct_fn': self.punct_class_report.fn, + 'punct_fp': self.punct_class_report.fp, + 'capit_tp': self.capit_class_report.tp, + 'capit_fn': self.capit_class_report.fn, + 'capit_fp': self.capit_class_report.fp, + } + + def multi_validation_epoch_end(self, outputs, dataloader_idx: int = 0): + """ + Called at the end of validation to aggregate outputs. + outputs: list of individual outputs of each validation step. + """ + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() + + # calculate metrics and log classification report for Punctuation task + punct_precision, punct_recall, punct_f1, punct_report = self.punct_class_report.compute() + logging.info(f'Punctuation report: {punct_report}') + + # calculate metrics and log classification report for Capitalization task + capit_precision, capit_recall, capit_f1, capit_report = self.capit_class_report.compute() + logging.info(f'Capitalization report: {capit_report}') + + self.log('val_loss', avg_loss, prog_bar=True) + self.log('punct_precision', punct_precision) + self.log('punct_f1', punct_f1) + self.log('punct_recall', punct_recall) + self.log('capit_precision', capit_precision) + self.log('capit_f1', capit_f1) + self.log('capit_recall', capit_recall) + + self.punct_class_report.reset() + self.capit_class_report.reset() + + def multi_test_epoch_end(self, outputs, dataloader_idx: int = 0): + """ + Called at the end of test to aggregate outputs. + outputs: list of individual outputs of each validation step. + """ + avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean() + + # calculate metrics and log classification report for Punctuation task + punct_precision, punct_recall, punct_f1, punct_report = self.punct_class_report.compute() + logging.info(f'Punctuation report: {punct_report}') + + # calculate metrics and log classification report for Capitalization task + capit_precision, capit_recall, capit_f1, capit_report = self.capit_class_report.compute() + logging.info(f'Capitalization report: {capit_report}') + + self.log('test_loss', avg_loss, prog_bar=True) + self.log('punct_precision', punct_precision) + self.log('punct_f1', punct_f1) + self.log('punct_recall', punct_recall) + self.log('capit_precision', capit_precision) + self.log('capit_f1', capit_f1) + self.log('capit_recall', capit_recall) + + def update_data_dir(self, data_dir: str) -> None: + """ + Update data directory + + Args: + data_dir: path to data directory + """ + if os.path.exists(data_dir): + logging.info(f'Setting model.dataset.data_dir to {data_dir}.') + self._cfg.dataset.data_dir = data_dir + else: + raise ValueError(f'{data_dir} not found') + + def setup_training_data(self, train_data_config: Optional[DictConfig] = None): + """Setup training data""" + if train_data_config is None: + train_data_config = self._cfg.train_ds + + # for older(pre - 1.0.0.b3) configs compatibility + if not hasattr(self._cfg, "class_labels") or self._cfg.class_labels is None: + OmegaConf.set_struct(self._cfg, False) + self._cfg.class_labels = {} + self._cfg.class_labels = OmegaConf.create( + {'punct_labels_file': 'punct_label_ids.csv', 'capit_labels_file': 'capit_label_ids.csv'} + ) + + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config) + + if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + self.register_artifact('class_labels.punct_labels_file', self._train_dl.dataset.punct_label_ids_file) + self.register_artifact('class_labels.capit_labels_file', self._train_dl.dataset.capit_label_ids_file) + + # save label maps to the config + self._cfg.punct_label_ids = OmegaConf.create(self._train_dl.dataset.punct_label_ids) + self._cfg.capit_label_ids = OmegaConf.create(self._train_dl.dataset.capit_label_ids) + + def setup_validation_data(self, val_data_config: Optional[Dict] = None): + """ + Setup validaton data + + val_data_config: validation data config + """ + if val_data_config is None: + val_data_config = self._cfg.validation_ds + + self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config) + + def setup_test_data(self, test_data_config: Optional[Dict] = None): + if test_data_config is None: + test_data_config = self._cfg.test_ds + self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config) + + def _setup_dataloader_from_config(self, cfg: DictConfig): + # use data_dir specified in the ds_item to run evaluation on multiple datasets + if 'ds_item' in cfg and cfg.ds_item is not None: + data_dir = cfg.ds_item + else: + data_dir = self._cfg.dataset.data_dir + + text_file = os.path.join(data_dir, cfg.text_file) + label_file = os.path.join(data_dir, cfg.labels_file) + + dataset = BertPunctuationCapitalizationDataset( + tokenizer=self.tokenizer, + text_file=text_file, + label_file=label_file, + pad_label=self._cfg.dataset.pad_label, + punct_label_ids=self._cfg.punct_label_ids, + capit_label_ids=self._cfg.capit_label_ids, + max_seq_length=self._cfg.dataset.max_seq_length, + ignore_extra_tokens=self._cfg.dataset.ignore_extra_tokens, + ignore_start_end=self._cfg.dataset.ignore_start_end, + use_cache=self._cfg.dataset.use_cache, + num_samples=cfg.num_samples, + punct_label_ids_file=self._cfg.class_labels.punct_labels_file + if 'class_labels' in self._cfg + else 'punct_label_ids.csv', + capit_label_ids_file=self._cfg.class_labels.capit_labels_file + if 'class_labels' in self._cfg + else 'capit_label_ids.csv', + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + collate_fn=dataset.collate_fn, + batch_size=cfg.batch_size, + shuffle=cfg.shuffle, + num_workers=self._cfg.dataset.num_workers, + pin_memory=self._cfg.dataset.pin_memory, + drop_last=self._cfg.dataset.drop_last, + ) + + def _setup_infer_dataloader( + self, queries: List[str], batch_size: int, max_seq_length: int = None + ) -> 'torch.utils.data.DataLoader': + """ + Setup function for a infer data loader. + + Args: + queries: lower cased text without punctuation + batch_size: batch size to use during inference + max_seq_length: maximum sequence length after tokenization + Returns: + A pytorch DataLoader. + """ + if max_seq_length is None: + max_seq_length = self._cfg.dataset.max_seq_length + + dataset = BertPunctuationCapitalizationInferDataset( + tokenizer=self.tokenizer, queries=queries, max_seq_length=max_seq_length + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + collate_fn=dataset.collate_fn, + batch_size=batch_size, + shuffle=False, + num_workers=self._cfg.dataset.num_workers, + pin_memory=self._cfg.dataset.pin_memory, + drop_last=False, + ) + + def add_punctuation_capitalization( + self, queries: List[str], batch_size: int = None, max_seq_length: int = 512 + ) -> List[str]: + """ + Adds punctuation and capitalization to the queries. Use this method for debugging and prototyping. + Args: + queries: lower cased text without punctuation + batch_size: batch size to use during inference + max_seq_length: maximum sequence length after tokenization + Returns: + result: text with added capitalization and punctuation + """ + if queries is None or len(queries) == 0: + return [] + if batch_size is None: + batch_size = len(queries) + logging.info(f'Using batch size {batch_size} for inference') + + # We will store the output here + result = [] + + # Model's mode and device + mode = self.training + device = 'cuda' if torch.cuda.is_available() else 'cpu' + try: + # Switch model to evaluation mode + self.eval() + self = self.to(device) + + infer_datalayer = self._setup_infer_dataloader(queries, batch_size, max_seq_length) + + # store predictions for all queries in a single list + all_punct_preds = [] + all_capit_preds = [] + + for batch in infer_datalayer: + input_ids, input_type_ids, input_mask, subtokens_mask = batch + + punct_logits, capit_logits = self.forward( + input_ids=input_ids.to(device), + token_type_ids=input_type_ids.to(device), + attention_mask=input_mask.to(device), + ) + + subtokens_mask = subtokens_mask > 0.5 + + punct_preds = [ + tensor2list(p_l[subtokens_mask[i]]) for i, p_l in enumerate(torch.argmax(punct_logits, axis=-1)) + ] + capit_preds = [ + tensor2list(c_l[subtokens_mask[i]]) for i, c_l in enumerate(torch.argmax(capit_logits, axis=-1)) + ] + + all_punct_preds.extend(punct_preds) + all_capit_preds.extend(capit_preds) + + punct_ids_to_labels = {v: k for k, v in self._cfg.punct_label_ids.items()} + capit_ids_to_labels = {v: k for k, v in self._cfg.capit_label_ids.items()} + + queries = [q.strip().split() for q in queries] + for i, query in enumerate(queries): + punct_preds = all_punct_preds[i] + capit_preds = all_capit_preds[i] + if len(query) != len(punct_preds): + logging.warning( + f'Max sequence length of query {query} is set to {max_seq_length}. Truncating the input.' + ) + + # removing the end of phrase punctuation of the truncated segment + punct_preds[-1] = 0 + max_len = len(punct_preds) + query = query[:max_len] + + query_with_punct_and_capit = '' + for j, word in enumerate(query): + punct_label = punct_ids_to_labels[punct_preds[j]] + capit_label = capit_ids_to_labels[capit_preds[j]] + + if capit_label != self._cfg.dataset.pad_label: + word = word.capitalize() + query_with_punct_and_capit += word + if punct_label != self._cfg.dataset.pad_label: + query_with_punct_and_capit += punct_label + query_with_punct_and_capit += ' ' + + result.append(query_with_punct_and_capit.strip()) + finally: + # set mode back to its original value + self.train(mode=mode) + return result + + @classmethod + def list_available_models(cls) -> Optional[Dict[str, str]]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + result = [] + result.append( + PretrainedModelInfo( + pretrained_model_name="punctuation_en_bert", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/punctuation_en_bert/versions/1.0.0rc1/files/punctuation_en_bert.nemo", + description="The model was trained with NeMo BERT base uncased checkpoint on a subset of data from the following sources: Tatoeba sentences, books from Project Gutenberg, Fisher transcripts.", + ) + ) + result.append( + PretrainedModelInfo( + pretrained_model_name="punctuation_en_distilbert", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/punctuation_en_distilbert/versions/1.0.0rc1/files/punctuation_en_distilbert.nemo", + description="The model was trained with DiltilBERT base uncased checkpoint from HuggingFace on a subset of data from the following sources: Tatoeba sentences, books from Project Gutenberg, Fisher transcripts.", + ) + ) + return result + + @property + def input_module(self): + return self.bert_model + + @property + def output_module(self): + return self diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/token_classification_model.py b/models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/token_classification_model.py new file mode 100644 index 0000000000..5781fbf418 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/models/token_classification/token_classification_model.py @@ -0,0 +1,522 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Dict, List, Optional, Union + +import onnx +import torch +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer +from torch.utils.data import DataLoader + +from nemo.collections.common.losses import CrossEntropyLoss +from nemo.collections.nlp.data.data_utils.data_preprocessing import get_labels_to_labels_id_mapping +from nemo.collections.nlp.data.token_classification.token_classification_dataset import ( + BertTokenClassificationDataset, + BertTokenClassificationInferDataset, +) +from nemo.collections.nlp.data.token_classification.token_classification_utils import get_label_ids +from nemo.collections.nlp.metrics.classification_report import ClassificationReport +from nemo.collections.nlp.models.nlp_model import NLPModel +from nemo.collections.nlp.modules.common import TokenClassifier +from nemo.collections.nlp.modules.common.lm_utils import get_lm_model +from nemo.collections.nlp.parts.utils_funcs import get_classification_report, plot_confusion_matrix, tensor2list +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types import NeuralType +from nemo.utils import logging + +__all__ = ['TokenClassificationModel'] + + +class TokenClassificationModel(NLPModel): + """Token Classification Model with BERT, applicable for tasks such as Named Entity Recognition""" + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return self.bert_model.input_types + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return self.classifier.output_types + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + """Initializes Token Classification Model.""" + # extract str to int labels mapping if a mapping file provided + if isinstance(cfg.label_ids, str): + if os.path.exists(cfg.label_ids): + logging.info(f'Reusing label_ids file found at {cfg.label_ids}.') + label_ids = get_labels_to_labels_id_mapping(cfg.label_ids) + # update the config to store name to id mapping + cfg.label_ids = OmegaConf.create(label_ids) + else: + raise ValueError(f'{cfg.label_ids} not found.') + + self.setup_tokenizer(cfg.tokenizer) + self.class_weights = None + super().__init__(cfg=cfg, trainer=trainer) + + self.bert_model = get_lm_model( + pretrained_model_name=cfg.language_model.pretrained_model_name, + config_file=self.register_artifact('language_model.config_file', cfg.language_model.config_file), + config_dict=OmegaConf.to_container(cfg.language_model.config) if cfg.language_model.config else None, + checkpoint_file=cfg.language_model.lm_checkpoint, + vocab_file=self.register_artifact('tokenizer.vocab_file', cfg.tokenizer.vocab_file), + ) + + self.classifier = TokenClassifier( + hidden_size=self.bert_model.config.hidden_size, + num_classes=len(self._cfg.label_ids), + num_layers=self._cfg.head.num_fc_layers, + activation=self._cfg.head.activation, + log_softmax=False, + dropout=self._cfg.head.fc_dropout, + use_transformer_init=self._cfg.head.use_transformer_init, + ) + + self.loss = self.setup_loss(class_balancing=self._cfg.dataset.class_balancing) + + # setup to track metrics + self.classification_report = ClassificationReport( + len(self._cfg.label_ids), label_ids=self._cfg.label_ids, dist_sync_on_step=True + ) + + def update_data_dir(self, data_dir: str) -> None: + """ + Update data directory and get data stats with Data Descriptor + Weights are later used to setup loss + + Args: + data_dir: path to data directory + """ + self._cfg.dataset.data_dir = data_dir + logging.info(f'Setting model.dataset.data_dir to {data_dir}.') + + def setup_loss(self, class_balancing: str = None): + """Setup loss + Setup or update loss. + + Args: + class_balancing: whether to use class weights during training + """ + if class_balancing not in ['weighted_loss', None]: + raise ValueError(f'Class balancing {class_balancing} is not supported. Choose from: [null, weighted_loss]') + if class_balancing == 'weighted_loss' and self.class_weights: + # you may need to increase the number of epochs for convergence when using weighted_loss + loss = CrossEntropyLoss(logits_ndim=3, weight=self.class_weights) + logging.debug(f'Using {class_balancing} class balancing.') + else: + loss = CrossEntropyLoss(logits_ndim=3) + logging.debug(f'Using CrossEntropyLoss class balancing.') + return loss + + @typecheck() + def forward(self, input_ids, token_type_ids, attention_mask): + hidden_states = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask + ) + logits = self.classifier(hidden_states=hidden_states) + return logits + + def training_step(self, batch, batch_idx): + """ + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. + """ + input_ids, input_type_ids, input_mask, subtokens_mask, loss_mask, labels = batch + logits = self(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + loss = self.loss(logits=logits, labels=labels, loss_mask=loss_mask) + lr = self._optimizer.param_groups[0]['lr'] + + self.log('train_loss', loss) + self.log('lr', lr, prog_bar=True) + + return { + 'loss': loss, + 'lr': lr, + } + + def validation_step(self, batch, batch_idx): + """ + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. + """ + input_ids, input_type_ids, input_mask, subtokens_mask, loss_mask, labels = batch + logits = self(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + val_loss = self.loss(logits=logits, labels=labels, loss_mask=loss_mask) + + subtokens_mask = subtokens_mask > 0.5 + + preds = torch.argmax(logits, axis=-1)[subtokens_mask] + labels = labels[subtokens_mask] + tp, fn, fp, _ = self.classification_report(preds, labels) + + return {'val_loss': val_loss, 'tp': tp, 'fn': fn, 'fp': fp} + + def validation_epoch_end(self, outputs): + """ + Called at the end of validation to aggregate outputs. + outputs: list of individual outputs of each validation step. + """ + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() + + # calculate metrics and classification report + precision, recall, f1, report = self.classification_report.compute() + + logging.info(report) + + self.log('val_loss', avg_loss, prog_bar=True) + self.log('precision', precision) + self.log('f1', f1) + self.log('recall', recall) + + self.classification_report.reset() + + def test_step(self, batch, batch_idx): + input_ids, input_type_ids, input_mask, subtokens_mask, loss_mask, labels = batch + logits = self(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + val_loss = self.loss(logits=logits, labels=labels, loss_mask=loss_mask) + + subtokens_mask = subtokens_mask > 0.5 + + preds = torch.argmax(logits, axis=-1)[subtokens_mask] + labels = labels[subtokens_mask] + tp, fn, fp, _ = self.classification_report(preds, labels) + + return {'test_loss': val_loss, 'tp': tp, 'fn': fn, 'fp': fp} + + def test_epoch_end(self, outputs): + avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean() + # calculate metrics and classification report + precision, recall, f1, report = self.classification_report.compute() + logging.info(report) + + self.log('test_loss', avg_loss, prog_bar=True) + self.log('precision', precision) + self.log('f1', f1) + self.log('recall', recall) + + def setup_training_data(self, train_data_config: Optional[DictConfig] = None): + if train_data_config is None: + train_data_config = self._cfg.train_ds + + labels_file = os.path.join(self._cfg.dataset.data_dir, train_data_config.labels_file) + + # for older(pre - 1.0.0.b3) configs compatibility + if not hasattr(self._cfg, "class_labels") or self._cfg.class_labels is None: + OmegaConf.set_struct(self._cfg, False) + self._cfg.class_labels = {} + self._cfg.class_labels = OmegaConf.create({'class_labels_file': 'label_ids.csv'}) + OmegaConf.set_struct(self._cfg, True) + + label_ids, label_ids_filename, self.class_weights = get_label_ids( + label_file=labels_file, + is_training=True, + pad_label=self._cfg.dataset.pad_label, + label_ids_dict=self._cfg.label_ids, + get_weights=True, + class_labels_file_artifact=self._cfg.class_labels.class_labels_file, + ) + # save label maps to the config + self._cfg.label_ids = OmegaConf.create(label_ids) + + self.register_artifact('class_labels.class_labels_file', label_ids_filename) + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config) + + def setup_validation_data(self, val_data_config: Optional[DictConfig] = None): + if val_data_config is None: + val_data_config = self._cfg.validation_ds + + labels_file = os.path.join(self._cfg.dataset.data_dir, val_data_config.labels_file) + get_label_ids( + label_file=labels_file, + is_training=False, + pad_label=self._cfg.dataset.pad_label, + label_ids_dict=self._cfg.label_ids, + get_weights=False, + ) + + self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config) + + def setup_test_data(self, test_data_config: Optional[DictConfig] = None): + if test_data_config is None: + test_data_config = self._cfg.test_ds + + labels_file = os.path.join(self._cfg.dataset.data_dir, test_data_config.labels_file) + get_label_ids( + label_file=labels_file, + is_training=False, + pad_label=self._cfg.dataset.pad_label, + label_ids_dict=self._cfg.label_ids, + get_weights=False, + ) + + self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config) + + def _setup_dataloader_from_config(self, cfg: DictConfig) -> DataLoader: + """ + Setup dataloader from config + Args: + cfg: config for the dataloader + Return: + Pytorch Dataloader + """ + dataset_cfg = self._cfg.dataset + data_dir = dataset_cfg.data_dir + + if not os.path.exists(data_dir): + raise FileNotFoundError(f"Data directory is not found at: {data_dir}.") + + text_file = os.path.join(data_dir, cfg.text_file) + labels_file = os.path.join(data_dir, cfg.labels_file) + + if not (os.path.exists(text_file) and os.path.exists(labels_file)): + raise FileNotFoundError( + f'{text_file} or {labels_file} not found. The data should be split into 2 files: text.txt and \ + labels.txt. Each line of the text.txt file contains text sequences, where words are separated with \ + spaces. The labels.txt file contains corresponding labels for each word in text.txt, the labels are \ + separated with spaces. Each line of the files should follow the format: \ + [WORD] [SPACE] [WORD] [SPACE] [WORD] (for text.txt) and \ + [LABEL] [SPACE] [LABEL] [SPACE] [LABEL] (for labels.txt).' + ) + dataset = BertTokenClassificationDataset( + text_file=text_file, + label_file=labels_file, + max_seq_length=dataset_cfg.max_seq_length, + tokenizer=self.tokenizer, + num_samples=cfg.num_samples, + pad_label=dataset_cfg.pad_label, + label_ids=self._cfg.label_ids, + ignore_extra_tokens=dataset_cfg.ignore_extra_tokens, + ignore_start_end=dataset_cfg.ignore_start_end, + use_cache=dataset_cfg.use_cache, + ) + return DataLoader( + dataset=dataset, + collate_fn=dataset.collate_fn, + batch_size=cfg.batch_size, + shuffle=cfg.shuffle, + num_workers=dataset_cfg.num_workers, + pin_memory=dataset_cfg.pin_memory, + drop_last=dataset_cfg.drop_last, + ) + + def _setup_infer_dataloader(self, queries: List[str], batch_size: int) -> 'torch.utils.data.DataLoader': + """ + Setup function for a infer data loader. + + Args: + queries: text + batch_size: batch size to use during inference + + Returns: + A pytorch DataLoader. + """ + + dataset = BertTokenClassificationInferDataset(tokenizer=self.tokenizer, queries=queries, max_seq_length=-1) + + return torch.utils.data.DataLoader( + dataset=dataset, + collate_fn=dataset.collate_fn, + batch_size=batch_size, + shuffle=False, + num_workers=self._cfg.dataset.num_workers, + pin_memory=self._cfg.dataset.pin_memory, + drop_last=False, + ) + + @torch.no_grad() + def _infer(self, queries: List[str], batch_size: int = None) -> List[int]: + """ + Get prediction for the queries + Args: + queries: text sequences + batch_size: batch size to use during inference. + Returns: + all_preds: model predictions + """ + # store predictions for all queries in a single list + all_preds = [] + mode = self.training + try: + device = 'cuda' if torch.cuda.is_available() else 'cpu' + # Switch model to evaluation mode + self.eval() + self.to(device) + infer_datalayer = self._setup_infer_dataloader(queries, batch_size) + + for batch in infer_datalayer: + input_ids, input_type_ids, input_mask, subtokens_mask = batch + + logits = self.forward( + input_ids=input_ids.to(device), + token_type_ids=input_type_ids.to(device), + attention_mask=input_mask.to(device), + ) + + subtokens_mask = subtokens_mask > 0.5 + preds = tensor2list(torch.argmax(logits, axis=-1)[subtokens_mask]) + all_preds.extend(preds) + finally: + # set mode back to its original value + self.train(mode=mode) + return all_preds + + def add_predictions( + self, queries: Union[List[str], str], batch_size: int = 32, output_file: Optional[str] = None + ) -> List[str]: + """ + Add predicted token labels to the queries. Use this method for debugging and prototyping. + Args: + queries: text + batch_size: batch size to use during inference. + output_file: file to save models predictions + Returns: + result: text with added entities + """ + if queries is None or len(queries) == 0: + return [] + + if isinstance(queries, str): + logging.info(f'Reading from {queries} file') + with open(queries, 'r') as f: + queries = f.readlines() + + result = [] + all_preds = self._infer(queries, batch_size) + + queries = [q.strip().split() for q in queries] + num_words = [len(q) for q in queries] + if sum(num_words) != len(all_preds): + raise ValueError('Pred and words must have the same length') + + ids_to_labels = {v: k for k, v in self._cfg.label_ids.items()} + start_idx = 0 + end_idx = 0 + for query in queries: + end_idx += len(query) + + # extract predictions for the current query from the list of all predictions + preds = all_preds[start_idx:end_idx] + start_idx = end_idx + + query_with_entities = '' + for j, word in enumerate(query): + # strip out the punctuation to attach the entity tag to the word not to a punctuation mark + # that follows the word + if word[-1].isalpha(): + punct = '' + else: + punct = word[-1] + word = word[:-1] + + query_with_entities += word + label = ids_to_labels[preds[j]] + + if label != self._cfg.dataset.pad_label: + query_with_entities += '[' + label + ']' + query_with_entities += punct + ' ' + result.append(query_with_entities.strip()) + + if output_file is not None: + with open(output_file, 'w') as f: + for r in result: + f.write(r + '\n') + logging.info(f'Predictions saved to {output_file}') + return result + + def evaluate_from_file( + self, + output_dir: str, + text_file: str, + labels_file: Optional[str] = None, + add_confusion_matrix: Optional[bool] = False, + normalize_confusion_matrix: Optional[bool] = True, + batch_size: int = 1, + ) -> None: + """ + Run inference on data from a file, plot confusion matrix and calculate classification report. + Use this method for final evaluation. + + Args: + output_dir: path to output directory to store model predictions, confusion matrix plot (if set to True) + text_file: path to file with text. Each line of the text.txt file contains text sequences, where words + are separated with spaces: [WORD] [SPACE] [WORD] [SPACE] [WORD] + labels_file (Optional): path to file with labels. Each line of the labels_file should contain + labels corresponding to each word in the text_file, the labels are separated with spaces: + [LABEL] [SPACE] [LABEL] [SPACE] [LABEL] (for labels.txt).' + add_confusion_matrix: whether to generate confusion matrix + normalize_confusion_matrix: whether to normalize confusion matrix + batch_size: batch size to use during inference. + """ + output_dir = os.path.abspath(output_dir) + + with open(text_file, 'r') as f: + queries = f.readlines() + + all_preds = self._infer(queries, batch_size) + with_labels = labels_file is not None + if with_labels: + with open(labels_file, 'r') as f: + all_labels_str = f.readlines() + all_labels_str = ' '.join([labels.strip() for labels in all_labels_str]) + + # writing labels and predictions to a file in output_dir is specified in the config + os.makedirs(output_dir, exist_ok=True) + filename = os.path.join(output_dir, 'infer_' + os.path.basename(text_file)) + try: + with open(filename, 'w') as f: + if with_labels: + f.write('labels\t' + all_labels_str + '\n') + logging.info(f'Labels save to {filename}') + + # convert labels from string label to ids + ids_to_labels = {v: k for k, v in self._cfg.label_ids.items()} + all_preds_str = [ids_to_labels[pred] for pred in all_preds] + f.write('preds\t' + ' '.join(all_preds_str) + '\n') + logging.info(f'Predictions saved to {filename}') + + if with_labels and add_confusion_matrix: + all_labels = all_labels_str.split() + # convert labels from string label to ids + label_ids = self._cfg.label_ids + all_labels = [label_ids[label] for label in all_labels] + + plot_confusion_matrix( + all_labels, all_preds, output_dir, label_ids=label_ids, normalize=normalize_confusion_matrix + ) + logging.info(get_classification_report(all_labels, all_preds, label_ids)) + except Exception: + logging.error( + f'When providing a file with labels, check that all labels in {labels_file} were' + f'seen during training.' + ) + raise + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + result = [] + model = PretrainedModelInfo( + pretrained_model_name="ner_en_bert", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/ner_en_bert/versions/1.0.0rc1/files/ner_en_bert.nemo", + description="The model was trained on GMB (Groningen Meaning Bank) corpus for entity recognition and achieves 74.61 F1 Macro score.", + ) + result.append(model) + return result diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/__init__.py new file mode 100644 index 0000000000..c46d0014d0 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.collections.nlp.modules.common import ( + AlbertEncoder, + BertEncoder, + BertModule, + DistilBertEncoder, + MegatronBertEncoder, + RobertaEncoder, + SequenceClassifier, + SequenceRegression, + SequenceTokenClassifier, + get_lm_model, + get_megatron_lm_models_list, + get_pretrained_lm_models_list, + get_tokenizer, + get_tokenizer_list, +) +from nemo.collections.nlp.modules.dialogue_state_tracking.sgd_decoder import SGDDecoder +from nemo.collections.nlp.modules.dialogue_state_tracking.sgd_encoder import SGDEncoder diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/__init__.py new file mode 100644 index 0000000000..fa30748e7b --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/__init__.py @@ -0,0 +1,34 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.modules.common.bert_module import BertModule +from nemo.collections.nlp.modules.common.huggingface import ( + AlbertEncoder, + BertEncoder, + DistilBertEncoder, + RobertaEncoder, +) +from nemo.collections.nlp.modules.common.lm_utils import ( + get_lm_model, + get_megatron_lm_models_list, + get_pretrained_lm_models_list, +) +from nemo.collections.nlp.modules.common.megatron import MegatronBertEncoder +from nemo.collections.nlp.modules.common.sequence_classifier import SequenceClassifier +from nemo.collections.nlp.modules.common.sequence_regression import SequenceRegression +from nemo.collections.nlp.modules.common.sequence_token_classifier import SequenceTokenClassifier +from nemo.collections.nlp.modules.common.token_classifier import BertPretrainingTokenClassifier, TokenClassifier +from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer, get_tokenizer_list diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/bert_module.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/bert_module.py new file mode 100644 index 0000000000..5cabc713b7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/bert_module.py @@ -0,0 +1,93 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +from typing import Dict, Optional + +import torch + +from nemo.core.classes import NeuralModule +from nemo.core.classes.exportable import Exportable +from nemo.core.neural_types import ChannelType, MaskType, NeuralType +from nemo.utils import logging + +__all__ = ['BertModule'] + + +class BertModule(NeuralModule, Exportable): + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "attention_mask": NeuralType(('B', 'T'), MaskType(), optional=True), + "token_type_ids": NeuralType(('B', 'T'), ChannelType(), optional=True), + } + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return {"last_hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} + + @classmethod + def restore_from(cls, restore_path: str): + """Restores module/model with weights""" + pass + + @classmethod + def save_to(self, save_path: str): + """Saves module/model with weights""" + pass + + def restore_weights(self, restore_path: str): + """Restores module/model's weights""" + logging.info(f"Restoring weights from {restore_path}") + + if not os.path.exists(restore_path): + logging.warning(f'Path {restore_path} not found') + return + + pretrained_dict = torch.load(restore_path) + + # backward compatibility with NeMo0.11 + if "state_dict" in pretrained_dict.keys(): + pretrained_dict = pretrained_dict["state_dict"] + + # remove prefix from pretrained dict + m = re.match("^bert.*?\.", list(pretrained_dict.keys())[0]) + if m: + prefix = m.group(0) + pretrained_dict = {k[len(prefix) :]: v for k, v in pretrained_dict.items()} + model_dict = self.state_dict() + pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict} + + # starting with transformers 3.1.0, embeddings.position_ids is added to the model's state dict and could be + # missing in checkpoints trained with older transformers version + if 'embeddings.position_ids' in model_dict and 'embeddings.position_ids' not in pretrained_dict: + pretrained_dict['embeddings.position_ids'] = model_dict['embeddings.position_ids'] + + assert len(pretrained_dict) == len(model_dict) + model_dict.update(pretrained_dict) + self.load_state_dict(model_dict) + logging.info(f"Weights for {type(self).__name__} restored from {restore_path}") + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + sample = next(self.parameters()) + input_ids = torch.randint(low=0, high=2048, size=(2, 16), device=sample.device) + attention_mask = torch.randint(low=0, high=1, size=(2, 16), device=sample.device) + return tuple([input_ids, attention_mask, attention_mask]) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/classifier.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/classifier.py new file mode 100644 index 0000000000..8d0a5f5938 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/classifier.py @@ -0,0 +1,87 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional + +import torch +from torch import nn as nn + +from nemo.collections.common.parts import transformer_weights_init +from nemo.core.classes import Exportable, NeuralModule, typecheck +from nemo.core.neural_types import ChannelType, NeuralType + +__all__ = ['Classifier'] + + +class Classifier(NeuralModule, Exportable): + """ + A baseclass for modules to perform various classification tasks. + """ + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + """ + Returns definitions of module input ports. + We implement it here since all NLP classifiers have the same inputs + """ + return {"hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} + + def __init__(self, hidden_size: int, dropout: float = 0.0,) -> None: + """ + Initializes the Classifier base module. + Args: + hidden_size: the size of the hidden dimension + dropout: dropout to apply to the input hidden states + """ + super().__init__() + self._hidden_size = hidden_size + self.dropout = nn.Dropout(dropout) + + def post_init(self, use_transformer_init: bool): + """ + Common post-processing to be called at the end of concrete Classifiers init methods + Args: + use_transformer_init : whether or not to apply transformer_weights_init + """ + if use_transformer_init: + self.apply(lambda module: transformer_weights_init(module, xavier=False)) + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + bs = 8 + seq = 64 + sample = next(self.parameters()) + input_example = torch.randn(bs, seq, self._hidden_size).to(sample.device).to(sample.dtype) + return tuple([input_example]) + + def save_to(self, save_path: str): + """ + Saves the module to the specified path. + Args: + save_path: Path to where to save the module. + """ + pass + + @classmethod + def restore_from(cls, restore_path: str): + """ + Restores the module from the specified path. + Args: + restore_path: Path to restore the module from. + """ + pass diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/decoder_module.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/decoder_module.py new file mode 100644 index 0000000000..32f95e6ab5 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/decoder_module.py @@ -0,0 +1,58 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC +from typing import Any, Dict, Optional + +from nemo.core.classes import NeuralModule +from nemo.core.neural_types import ChannelType, MaskType, NeuralType + +__all__ = ['DecoderModule'] + + +class DecoderModule(NeuralModule, ABC): + """ Base class for decoder neural module to be used in NLP models. """ + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "decoder_mask": NeuralType(('B', 'T'), MaskType(), optional=True), + "encoder_mask": NeuralType(('B', 'T'), MaskType(), optional=True), + "encoder_embeddings": NeuralType(('B', 'T', 'D'), ChannelType(), optional=True), + } + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return {"last_hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} + + @property + def hidden_size(self) -> Optional[int]: + raise NotImplementedError + + @property + def vocab_size(self) -> Optional[int]: + raise NotImplementedError + + @property + def embedding(self) -> Optional[Any]: + raise NotImplementedError + + @property + def decoder(self) -> Optional[Any]: + raise NotImplementedError + + @property + def max_sequence_length(self) -> Optional[int]: + raise NotImplementedError diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/encoder_module.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/encoder_module.py new file mode 100644 index 0000000000..384561f4ad --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/encoder_module.py @@ -0,0 +1,41 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC +from typing import Dict, Optional + +from nemo.core.classes import NeuralModule +from nemo.core.neural_types import ChannelType, MaskType, NeuralType +from nemo.utils import logging + +__all__ = ['EncoderModule'] + + +class EncoderModule(NeuralModule, ABC): + """ Base class for encoder neural module to be used in NLP models. """ + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "input_ids": NeuralType(('B', 'T'), ChannelType()), + "encoder_mask": NeuralType(('B', 'T'), MaskType()), + } + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return {"last_hidden_states": NeuralType(('B', 'T', 'D'), ChannelType())} + + @property + def hidden_size(self) -> Optional[int]: + raise NotImplementedError diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/__init__.py new file mode 100644 index 0000000000..d43cb437c0 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.modules.common.huggingface.albert import AlbertEncoder +from nemo.collections.nlp.modules.common.huggingface.bert import BertEncoder +from nemo.collections.nlp.modules.common.huggingface.distilbert import DistilBertEncoder +from nemo.collections.nlp.modules.common.huggingface.huggingface_utils import ( + get_huggingface_lm_model, + get_huggingface_pretrained_lm_models_list, +) +from nemo.collections.nlp.modules.common.huggingface.roberta import RobertaEncoder diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/albert.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/albert.py new file mode 100644 index 0000000000..eff926f029 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/albert.py @@ -0,0 +1,33 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from transformers import AlbertModel + +from nemo.collections.nlp.modules.common.bert_module import BertModule +from nemo.core.classes import typecheck + +__all__ = ['AlbertEncoder'] + + +class AlbertEncoder(AlbertModel, BertModule): + """ + Wraps around the Huggingface transformers implementation repository for easy use within NeMo. + """ + + @typecheck() + def forward(self, input_ids, attention_mask, token_type_ids): + res = super().forward(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)[0] + return res diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/bert.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/bert.py new file mode 100644 index 0000000000..92abf78580 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/bert.py @@ -0,0 +1,33 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from transformers import BertModel + +from nemo.collections.nlp.modules.common.bert_module import BertModule +from nemo.core.classes import typecheck + +__all__ = ['BertEncoder'] + + +class BertEncoder(BertModel, BertModule): + """ + Wraps around the Huggingface transformers implementation repository for easy use within NeMo. + """ + + @typecheck() + def forward(self, input_ids, attention_mask, token_type_ids): + res = super().forward(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)[0] + return res diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/distilbert.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/distilbert.py new file mode 100644 index 0000000000..21c3469b38 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/distilbert.py @@ -0,0 +1,34 @@ +# Copyright 2020 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from transformers import DistilBertModel + +from nemo.collections.nlp.modules.common.bert_module import BertModule +from nemo.core.classes import typecheck + +__all__ = ['DistilBertEncoder'] + + +class DistilBertEncoder(DistilBertModel, BertModule): + """ + Wraps around the Huggingface transformers implementation repository for easy use within NeMo. + """ + + @typecheck() + def forward(self, input_ids, attention_mask, token_type_ids=None): + # distilBert does not use token_type_ids as the most of the other Bert models + res = super().forward(input_ids=input_ids, attention_mask=attention_mask)[0] + return res diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_decoder.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_decoder.py new file mode 100644 index 0000000000..20049ae6d7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_decoder.py @@ -0,0 +1,79 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from hydra.utils import instantiate +from transformers import AutoConfig, AutoModel + +from nemo.collections.nlp.modules.common.decoder_module import DecoderModule +from nemo.collections.nlp.modules.common.huggingface.huggingface_utils import get_huggingface_pretrained_lm_models_list +from nemo.utils import logging + + +class HuggingFaceDecoderModule(DecoderModule): + """Gets HuggingFace based model to be used as an Decoder in NeMo NLP. + Use the model_name arg to get a named model architecture. + Available model names can be found with get_huggingface_pretrained_lm_models_list() or + by going to https://huggingface.co/models. + Use the pretrained arg to get the named model architecture with or without pretrained weights. + + If model_name is None, then we can pass in a custom configuration via the config_dict. + For example, to instantiate a HuggingFace BERT model with custom configuration we would do: + config_dict={ + '_target_': 'transformers.BertConfig', + 'hidden_size': 1536 + } + + + Args: + model_name (Optional[str]): Named model architecture from HuggingFace. Defaults to None. + pretrained (bool): Use True to get pretrained weights. + False will use the same architecture but with randomly initialized weights. + Defaults to False. + config_dict (Optional[dict], optional): Use for custom configuration of the HuggingFace model. Defaults to None. + checkpoint_file (Optional[str], optional): Provide weights for the transformer from a local checkpoint. Defaults to None. + """ + + def __init__( + self, + model_name: Optional[str] = None, + pretrained: bool = False, + config_dict: Optional[dict] = None, + checkpoint_file: Optional[str] = None, + ): + super().__init__() + model = None + if model_name is not None: + if model_name in get_huggingface_pretrained_lm_models_list(): + if pretrained: + model = AutoModel.from_pretrained(model_name) + else: + cfg = AutoConfig.from_pretrained(model_name) + model = AutoModel.from_config(cfg) + else: + logging.error(f'{model_name} not found in list of HuggingFace pretrained models') + else: + cfg = instantiate(config_dict) + model = AutoModel.from_config(cfg) + self._hidden_size = model.config.hidden_size + self._vocab_size = model.config.vocab_size + + @property + def hidden_size(self) -> Optional[int]: + return self._hidden_size + + @property + def vocab_size(self) -> Optional[int]: + return self._vocab_size diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_encoder.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_encoder.py new file mode 100644 index 0000000000..b21c3a752a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_encoder.py @@ -0,0 +1,99 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from hydra.utils import instantiate +from transformers import AutoConfig, AutoModel + +from nemo.collections.nlp.modules.common.encoder_module import EncoderModule +from nemo.collections.nlp.modules.common.huggingface.huggingface_utils import get_huggingface_pretrained_lm_models_list +from nemo.core.classes.common import typecheck +from nemo.utils import logging + + +class HuggingFaceEncoderModule(EncoderModule): + """ Class for using HuggingFace encoders in NeMo NLP.""" + + def __init__( + self, + model_name: Optional[str] = None, + pretrained: bool = False, + config_dict: Optional[dict] = None, + checkpoint_file: Optional[str] = None, + ): + """Gets HuggingFace based model to be used as an Encoder in NeMo NLP. + Use the model_name arg to get a named model architecture. + Available model names can be found with get_huggingface_pretrained_lm_models_list() or + by going to https://huggingface.co/models. + Use the pretrained arg to get the named model architecture with or without pretrained weights. + + If model_name is None, then we can pass in a custom configuration via the config_dict. + For example, to instantiate a HuggingFace BERT model with custom configuration we would do: + config_dict={ + '_target_': 'transformers.BertConfig', + 'hidden_size': 1536 + } + + + Args: + model_name (Optional[str]): Named model architecture from HuggingFace. Defaults to None. + pretrained (bool): Use True to get pretrained weights. + False will use the same architecture but with randomly initialized weights. + Defaults to False. + config_dict (Optional[dict], optional): Use for custom configuration of the HuggingFace model. Defaults to None. + checkpoint_file (Optional[str], optional): Provide weights for the transformer from a local checkpoint. Defaults to None. + """ + super().__init__() + + if checkpoint_file: + raise NotImplementedError('Restoring from checkpoint file not implemented yet.') + + model = None + if model_name is not None: + if model_name in get_huggingface_pretrained_lm_models_list(include_external=True): + if pretrained: + config_dict.pop('vocab_size') + if config_dict: + raise ValueError( + f'When using pretrained model, config_dict should be None or empty. Got: {config_dict}' + ) + model = AutoModel.from_pretrained(model_name) + else: + cfg = AutoConfig.from_pretrained(model_name) + model = AutoModel.from_config(cfg) + else: + logging.error(f'{model_name} not found in list of HuggingFace pretrained models') + else: + if pretrained: + raise ValueError(f'If not using model_name, then pretrained should be False. Got: {pretrained}.') + cfg = instantiate(config_dict) + model = AutoModel.from_config(cfg) + self._hidden_size = model.config.hidden_size + self._vocab_size = model.config.vocab_size + + self._encoder = model + + @typecheck() + def forward(self, input_ids, encoder_mask): + encoder_hidden_states = self._encoder.forward(input_ids=input_ids, attention_mask=encoder_mask)[0] + return encoder_hidden_states + + @property + def hidden_size(self) -> Optional[int]: + return self._hidden_size + + @property + def vocab_size(self) -> Optional[int]: + return self._vocab_size diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py new file mode 100644 index 0000000000..39bb2b03cb --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py @@ -0,0 +1,126 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import List, Optional + +from transformers import ( + ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST, + ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, + BERT_PRETRAINED_MODEL_ARCHIVE_LIST, + DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST, + ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST, + AlbertConfig, + AutoModel, + BertConfig, + DistilBertConfig, + RobertaConfig, +) + +from nemo.collections.nlp.modules.common.huggingface.albert import AlbertEncoder +from nemo.collections.nlp.modules.common.huggingface.bert import BertEncoder +from nemo.collections.nlp.modules.common.huggingface.distilbert import DistilBertEncoder +from nemo.collections.nlp.modules.common.huggingface.roberta import RobertaEncoder +from nemo.utils import logging + +__all__ = ["get_huggingface_lm_model", "get_huggingface_pretrained_lm_models_list"] + + +HUGGINGFACE_MODELS = { + "BertModel": { + "default": "bert-base-uncased", + "class": BertEncoder, + "config": BertConfig, + "pretrained_model_list": BERT_PRETRAINED_MODEL_ARCHIVE_LIST, + }, + "DistilBertModel": { + "default": "distilbert-base-uncased", + "class": DistilBertEncoder, + "config": DistilBertConfig, + "pretrained_model_list": DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST, + }, + "RobertaModel": { + "default": "roberta-base", + "class": RobertaEncoder, + "config": RobertaConfig, + "pretrained_model_list": ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST, + }, + "AlbertModel": { + "default": "albert-base-v2", + "class": AlbertEncoder, + "config": AlbertConfig, + "pretrained_model_list": ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST, + }, +} + + +def get_huggingface_lm_model( + pretrained_model_name: str, config_dict: Optional[dict] = None, config_file: Optional[str] = None, +): + """ + Returns lm model instantiated with Huggingface + + Args: + pretrained_mode_name: specify this to instantiate pretrained model from Huggingface, + e.g. bert-base-cased. For entire list, see get_huggingface_pretrained_lm_models_list(). + config_dict: model configuration dictionary used to instantiate Huggingface model from scratch + config_file: path to model configuration file used to instantiate Huggingface model from scratch + + Returns: + BertModule + """ + + try: + automodel = AutoModel.from_pretrained(pretrained_model_name) + except Exception as e: + raise ValueError(f"{pretrained_model_name} is not supported by HuggingFace. {e}") + + model_type = type(automodel).__name__ + if model_type in HUGGINGFACE_MODELS: + model_class = HUGGINGFACE_MODELS[model_type]["class"] + if config_file: + if not os.path.exists(config_file): + logging.warning( + f"Config file was not found at {config_file}. Will attempt to use config_dict or pretrained_model_name." + ) + else: + config_class = HUGGINGFACE_MODELS[model_type]["config"] + return model_class(config_class.from_json_file(config_file)) + if config_dict: + config_class = HUGGINGFACE_MODELS[model_type]["config"] + return model_class(config=config_class(**config_dict)) + else: + return model_class.from_pretrained(pretrained_model_name) + else: + raise ValueError(f"Use HuffingFace API directly in NeMo for {pretrained_model_name}") + + +def get_huggingface_pretrained_lm_models_list(include_external: bool = False,) -> List[str]: + """ + Returns the list of pretrained HuggingFace language models + + Args: + include_external if true includes all HuggingFace model names, not only those supported language models in NeMo. + + Returns the list of HuggingFace models + """ + + huggingface_models = [] + if include_external: + huggingface_models = list(ALL_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()) + else: + for model in HUGGINGFACE_MODELS: + model_names = HUGGINGFACE_MODELS[model]["pretrained_model_list"] + huggingface_models.extend(model_names) + return huggingface_models diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/roberta.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/roberta.py new file mode 100644 index 0000000000..f4c4f1f9e4 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/huggingface/roberta.py @@ -0,0 +1,34 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from transformers import RobertaModel + +from nemo.collections.nlp.modules.common.bert_module import BertModule +from nemo.core.classes import typecheck + +__all__ = ['RobertaEncoder'] + + +class RobertaEncoder(RobertaModel, BertModule): + """ + Wraps around the Huggingface transformers implementation repository for easy use within NeMo. + """ + + @typecheck() + def forward(self, input_ids, token_type_ids, attention_mask): + res = super().forward(input_ids=input_ids, attention_mask=attention_mask)[0] + return res diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/lm_utils.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/lm_utils.py new file mode 100644 index 0000000000..46ae9e9346 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/lm_utils.py @@ -0,0 +1,179 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from dataclasses import dataclass +from typing import List, Optional, Union + +from attr import asdict + +from nemo.collections.nlp.modules.common.bert_module import BertModule +from nemo.collections.nlp.modules.common.decoder_module import DecoderModule +from nemo.collections.nlp.modules.common.encoder_module import EncoderModule +from nemo.collections.nlp.modules.common.huggingface.huggingface_utils import ( + get_huggingface_lm_model, + get_huggingface_pretrained_lm_models_list, +) +from nemo.collections.nlp.modules.common.megatron.megatron_utils import ( + get_megatron_lm_model, + get_megatron_lm_models_list, +) +from nemo.collections.nlp.modules.common.transformer.transformer import NeMoTransformerConfig +from nemo.collections.nlp.modules.common.transformer.transformer_utils import ( + get_huggingface_transformer, + get_nemo_transformer, +) +from nemo.utils import logging + +__all__ = ['get_pretrained_lm_models_list', 'get_lm_model'] + + +def get_pretrained_lm_models_list(include_external: bool = False) -> List[str]: + """ + Returns the list of supported pretrained model names + + Args: + include_external if true includes all HuggingFace model names, not only those supported language models in NeMo. + + """ + return get_megatron_lm_models_list() + get_huggingface_pretrained_lm_models_list(include_external=include_external) + + +def get_lm_model( + pretrained_model_name: str, + config_dict: Optional[dict] = None, + config_file: Optional[str] = None, + checkpoint_file: Optional[str] = None, + vocab_file: Optional[str] = None, +) -> BertModule: + """ + Helper function to instantiate a language model encoder, either from scratch or a pretrained model. + If only pretrained_model_name are passed, a pretrained model is returned. + If a configuration is passed, whether as a file or dictionary, the model is initialized with random weights. + + Args: + pretrained_model_name: pretrained model name, for example, bert-base-uncased or megatron-bert-cased. + See get_pretrained_lm_models_list() for full list. + config_dict: path to the model configuration dictionary + config_file: path to the model configuration file + checkpoint_file: path to the pretrained model checkpoint + vocab_file: path to vocab_file to be used with Megatron-LM + + Returns: + Pretrained BertModule + """ + + # check valid model type + if not pretrained_model_name or pretrained_model_name not in get_pretrained_lm_models_list(include_external=False): + logging.warning( + f'{pretrained_model_name} is not in get_pretrained_lm_models_list(include_external=False), ' + f'will be using AutoModel from HuggingFace.' + ) + + # warning when user passes both configuration dict and file + if config_dict and config_file: + logging.warning( + f"Both config_dict and config_file were found, defaulting to use config_file: {config_file} will be used." + ) + + if "megatron" in pretrained_model_name: + model, checkpoint_file = get_megatron_lm_model( + config_dict=config_dict, + config_file=config_file, + pretrained_model_name=pretrained_model_name, + checkpoint_file=checkpoint_file, + vocab_file=vocab_file, + ) + else: + model = get_huggingface_lm_model( + config_dict=config_dict, config_file=config_file, pretrained_model_name=pretrained_model_name, + ) + + if checkpoint_file and os.path.exists(checkpoint_file): + model.restore_weights(restore_path=checkpoint_file) + + return model + + +# @dataclass +# class TransformerConfig: +# library: str = 'nemo' +# model_name: Optional[str] = None +# pretrained: bool = False +# config_dict: Optional[dict] = None +# checkpoint_file: Optional[str] = None +# encoder: bool = True + + +def get_transformer( + library: str = 'nemo', + model_name: Optional[str] = None, + pretrained: bool = False, + config_dict: Optional[dict] = None, + checkpoint_file: Optional[str] = None, + encoder: bool = True, + pre_ln_final_layer_norm=True, +) -> Union[EncoderModule, DecoderModule]: + """Gets Transformer based model to be used as an Encoder or Decoder in NeMo NLP. + First choose the library to get the transformer from. This can be huggingface, + megatron, or nemo. Use the model_name arg to get a named model architecture + and use the pretrained arg to get the named model architecture with pretrained weights. + + If model_name is None, then we can pass in a custom configuration via the config_dict. + For example, to instantiate a HuggingFace BERT model with custom configuration we would do: + encoder = get_transformer(library='huggingface', + config_dict={ + '_target_': 'transformers.BertConfig', + 'hidden_size': 1536 + }) + + + Args: + library (str, optional): Can be 'nemo', 'huggingface', or 'megatron'. Defaults to 'nemo'. + model_name (Optional[str], optional): Named model architecture from the chosen library. Defaults to None. + pretrained (bool, optional): Use True to get pretrained weights. + False will use the same architecture but with randomly initialized weights. + Defaults to False. + config_dict (Optional[dict], optional): Use for custom configuration of transformer. Defaults to None. + checkpoint_file (Optional[str], optional): Provide weights for the transformer from a local checkpoint. Defaults to None. + encoder (bool, optional): True returns an EncoderModule, False returns a DecoderModule. Defaults to True. + + Returns: + Union[EncoderModule, DecoderModule]: Ensures that Encoder/Decoder will work in EncDecNLPModel + """ + + model = None + + if library == 'nemo': + if isinstance(config_dict, NeMoTransformerConfig): + config_dict = asdict(config_dict) + model = get_nemo_transformer( + model_name=model_name, + pretrained=pretrained, + config_dict=config_dict, + encoder=encoder, + pre_ln_final_layer_norm=pre_ln_final_layer_norm, + ) + + if checkpoint_file is not None: + if os.path.isfile(checkpoint_file): + raise ValueError(f'Loading transformer weights from checkpoint file has not been implemented yet.') + + elif library == 'huggingface': + model = get_huggingface_transformer( + model_name=model_name, pretrained=pretrained, config_dict=config_dict, encoder=encoder + ) + + return model diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/__init__.py new file mode 100644 index 0000000000..7f603a7a72 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.modules.common.megatron.megatron_bert import MegatronBertEncoder +from nemo.collections.nlp.modules.common.megatron.megatron_utils import ( + get_megatron_checkpoint, + get_megatron_lm_models_list, +) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/megatron_bert.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/megatron_bert.py new file mode 100644 index 0000000000..0c9056dc6c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/megatron_bert.py @@ -0,0 +1,216 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +from megatron import get_args, initialize_megatron +from megatron.checkpointing import set_checkpoint_version +from megatron.model import get_language_model +from megatron.model.bert_model import bert_attention_mask_func, bert_extended_attention_mask, bert_position_ids +from megatron.mpu import ( + get_model_parallel_group, + model_parallel_is_initialized, + set_pipeline_model_parallel_rank, + set_pipeline_model_parallel_world_size, +) +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.modules.common.bert_module import BertModule +from nemo.core.classes import typecheck +from nemo.utils import logging +from nemo.utils.app_state import AppState + +__all__ = ['MegatronBertEncoder'] + + +def complete_lazy_init(self): + # finish megatron-lm initialization + if hasattr(self, "_lazy_init_fn") and self._lazy_init_fn is not None: + self._lazy_init_fn() + self._lazy_init_fn = None + + +class MegatronBertEncoder(BertModule): + """ + MegatronBERT wraps around the Megatron Language model + from https://github.com/NVIDIA/Megatron-LM + + Args: + config_file (str): path to model configuration file. + vocab_file (str): path to vocabulary file. + tokenizer_type (str): tokenizer type, currently only 'BertWordPieceLowerCase' supported. + """ + + def __init__(self, model_name, config, vocab_file, model_parallel_size=None, model_parallel_rank=None): + + super().__init__() + + self._model_parallel_size = model_parallel_size + self._model_parallel_rank = model_parallel_rank + self._restore_path = None + self._app_state = None + self._model_name = model_name + + if not os.path.exists(vocab_file): + raise ValueError(f'Vocab file not found at {vocab_file}') + + # convert config to dictionary + if isinstance(config, DictConfig): + config = OmegaConf.to_container(config) + config["vocab_file"] = vocab_file + config['tokenizer_type'] = 'BertWordPieceLowerCase' + config['lazy_mpu_init'] = True + config['onnx_safe'] = True + + # if 'model_parallel_size' in config: + if self._model_parallel_size is not None: + app_state = AppState() + self._app_state = app_state + + # must be set for model parallel megatron-lm + os.environ["WORLD_SIZE"] = str(app_state.world_size) + os.environ["RANK"] = str(self._model_parallel_rank) + + extra_args_provider = self._update_megatron_args(tensor_model_parallel_size=self._model_parallel_size) + + else: + extra_args_provider = self._update_megatron_args() + + # configure globals for megatron + set_pipeline_model_parallel_rank(0) # pipeline model parallelism not implemented in NeMo + set_pipeline_model_parallel_world_size(1) # pipeline model parallelism not implemented in NeMo + + # Initialize part of Megatron global state that is needed for its constructor. + # We set 'lazy_mpu_init' flag on to make Megatron do only the initialization that does not depend + # on ddp be initialized yet (and we don't want Megatron to initialize DDP itself either) + # and to return a hook for us to call after PTL has torch.distributed initialized. + # (or if no PTL in case of inference - then we'll initialize torch.distributed) + # We call and clear this hook on first call to forward() + self._lazy_init_fn = initialize_megatron( + extra_args_provider=extra_args_provider, args_defaults=config, ignore_unknown_args=True + ) + + # read Megatron arguments back + args = get_args() + logging.info(f'Megatron-lm argparse args: {args}') + + self.language_model, self._language_model_key = get_language_model( + attention_mask_func=bert_attention_mask_func, num_tokentypes=2, add_pooler=False + ) + + self.config = OmegaConf.create(config) + # key used for checkpoints + self._hidden_size = self.language_model.hidden_size + + def _update_megatron_args( + self, + micro_batch_size=1, + tensor_model_parallel_size=1, + scaled_masked_softmax_fusion=False, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + ): + def extra_args_provider(parser): + parser.set_defaults(micro_batch_size=micro_batch_size) + parser.set_defaults(tensor_model_parallel_size=tensor_model_parallel_size) + parser.set_defaults(scaled_masked_softmax_fusion=scaled_masked_softmax_fusion) + parser.set_defaults(bias_gelu_fusion=bias_gelu_fusion) + parser.set_defaults(bias_dropout_fusion=bias_dropout_fusion) + + return parser + + return extra_args_provider + + def complete_lazy_init(self): + # finish megatron-lm initialization + if hasattr(self, "_lazy_init_fn") and self._lazy_init_fn is not None: + self._lazy_init_fn() + self._lazy_init_fn = None + + @property + def hidden_size(self): + """ + Property returning hidden size. + + Returns: + Hidden size. + """ + return self._hidden_size + + @typecheck() + def forward(self, input_ids, attention_mask, token_type_ids): + app_state = AppState() + if app_state.model_parallel_size is None: + self.complete_lazy_init() + + extended_attention_mask = bert_extended_attention_mask(attention_mask) + position_ids = bert_position_ids(input_ids) + + sequence_output = self.language_model( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=extended_attention_mask, + tokentype_ids=token_type_ids, + ) + return sequence_output + + def _load_checkpoint(self, filename): + """Helper function for loading megatron checkpoints. + + Args: + filename (str): Path to megatron checkpoint. + """ + state_dict = torch.load(filename, map_location='cpu') + if 'checkpoint_version' in state_dict: + if state_dict['checkpoint_version'] is not None: + set_checkpoint_version(state_dict['checkpoint_version']) + logging.info( + f"Megatron-lm checkpoint version found. Setting checkpoint_version to {state_dict['checkpoint_version']}." + ) + else: + logging.warning('Megatron-lm checkpoint version not found. Setting checkpoint_version to 0.') + set_checkpoint_version(0) + # to load from Megatron pretrained checkpoint + if 'model' in state_dict: + self.language_model.load_state_dict(state_dict['model'][self._language_model_key]) + else: + self.load_state_dict(state_dict) + + logging.info(f"Checkpoint loaded from from {filename}") + + def restore_weights(self, restore_path: str): + """Restores module/model's weights. + For model parallel checkpoints the directory structure + should be restore_path/mp_rank_0X/model_optim_rng.pt + + Args: + restore_path (str): restore_path should a file or a directory if using model parallel + """ + self._restore_path = restore_path + + if os.path.isfile(restore_path): + self._load_checkpoint(restore_path) + elif os.path.isdir(restore_path): + # need model parallel groups to restore model parallel checkpoints + if model_parallel_is_initialized(): + model_parallel_rank = torch.distributed.get_rank(group=get_model_parallel_group()) + mp_restore_path = f'{restore_path}/mp_rank_{model_parallel_rank:02d}/model_optim_rng.pt' + self._load_checkpoint(mp_restore_path) + else: + logging.info(f'torch.distributed not initialized yet. Will not restore model parallel checkpoint') + else: + logging.error(f'restore_path: {restore_path} must be a file or directory.') diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/megatron_utils.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/megatron_utils.py new file mode 100644 index 0000000000..4f2056fded --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/megatron/megatron_utils.py @@ -0,0 +1,307 @@ +# Copyright 2020 The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import json +import os +from typing import Dict, List, Optional, Tuple + +import torch +import wget +from torch.hub import _get_torch_home + +from nemo.collections.nlp.modules.common.megatron.megatron_bert import MegatronBertEncoder +from nemo.utils import AppState, logging + +__all__ = [ + "get_megatron_lm_model", + "get_megatron_lm_models_list", + "get_megatron_checkpoint", + "is_lower_cased_megatron", + "get_megatron_tokenizer", +] + + +torch_home = _get_torch_home() + +if not isinstance(torch_home, str): + logging.info("Torch home not found, caching megatron in cwd") + torch_home = os.getcwd() + +MEGATRON_CACHE = os.path.join(torch_home, "megatron") + + +CONFIGS = {"345m": {"hidden_size": 1024, "num_attention_heads": 16, "num_layers": 24, "max_position_embeddings": 512}} + +MEGATRON_CONFIG_MAP = { + "megatron-bert-345m-uncased": { + "config": CONFIGS["345m"], + "checkpoint": "https://api.ngc.nvidia.com/v2/models/nvidia/megatron_bert_345m/versions/v0.0/files/release/mp_rank_00/model_optim_rng.pt", + "vocab": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt", + "do_lower_case": True, + "tokenizer_name": "bert-large-uncased", + }, + "megatron-bert-345m-cased": { + "config": CONFIGS["345m"], + "checkpoint": "https://api.ngc.nvidia.com/v2/models/nvidia/megatron_bert_345m/versions/v0.1_cased/files/release/mp_rank_00/model_optim_rng.pt", + "vocab": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-vocab.txt", + "do_lower_case": False, + "tokenizer_name": "bert-large-cased", + }, + "megatron-bert-uncased": { + "config": None, + "checkpoint": None, + "vocab": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt", + "do_lower_case": True, + "tokenizer_name": "bert-large-uncased", + }, + "megatron-bert-cased": { + "config": None, + "checkpoint": None, + "vocab": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-vocab.txt", + "do_lower_case": False, + "tokenizer_name": "bert-large-cased", + }, + "biomegatron-bert-345m-uncased": { + "config": CONFIGS["345m"], + "checkpoint": "https://api.ngc.nvidia.com/v2/models/nvidia/biomegatron345muncased/versions/0/files/MegatronBERT.pt", + "vocab": "https://api.ngc.nvidia.com/v2/models/nvidia/biomegatron345muncased/versions/0/files/vocab.txt", + "do_lower_case": True, + "tokenizer_name": "bert-large-uncased", + }, + "biomegatron-bert-345m-cased": { + "config": CONFIGS["345m"], + "checkpoint": "https://api.ngc.nvidia.com/v2/models/nvidia/biomegatron345mcased/versions/0/files/MegatronBERT.pt", + "vocab": "https://api.ngc.nvidia.com/v2/models/nvidia/biomegatron345mcased/versions/0/files/vocab.txt", + "do_lower_case": False, + "tokenizer_name": "bert-large-cased", + }, +} + + +def get_megatron_lm_model( + pretrained_model_name: str, + config_dict: Optional[dict] = None, + config_file: Optional[str] = None, + checkpoint_file: Optional[str] = None, + vocab_file: Optional[str] = None, +) -> Tuple[MegatronBertEncoder, str]: + """ + Returns MegatronBertEncoder and a default or user specified path to the checkpoint file + + Args: + pretrained_mode_name: model name from MEGATRON_CONFIG_MAP + for example: megatron-bert-cased + config_dict: model configuration parameters + config_file: path to model configuration file. Takes precedence over config_dict if both supplied. + checkpoint_file: path to checkpoint file or directory if using model parallel. + vocab_file: path to vocab file + + Returns: + model: MegatronBertEncoder + checkpoint_file: path to checkpoint file or directory + """ + config = None + # get default config and checkpoint + if config_file: + with open(config_file) as f: + config = json.load(f) + # replace dashes with underscores in config keys + fixed_config = {} + for key in config.keys(): + fixed_key = key.replace("-", "_") + if fixed_key == 'max_seq_length': + fixed_key = 'max_position_embeddings' + fixed_config[fixed_key] = config[key] + # 'vocab_size" no longer used. + if 'vocab_size' in fixed_config: + fixed_config.pop('vocab_size') + config = fixed_config + elif config_dict: + config = config_dict + elif pretrained_model_name in get_megatron_lm_models_list(): + config = get_megatron_config(pretrained_model_name) + else: + raise ValueError(f"{pretrained_model_name} is not supported") + + if config is None: + raise ValueError(f"config_file or config_dict is required for {pretrained_model_name}") + + if not checkpoint_file: + checkpoint_file = get_megatron_checkpoint(pretrained_model_name) + + if not vocab_file: + vocab_file = get_megatron_vocab_file(pretrained_model_name) + + app_state = AppState() + if app_state.model_parallel_size is not None and app_state.model_parallel_rank is not None: + # model parallel already known from .nemo restore + model_parallel_size = app_state.model_parallel_size + model_parallel_rank = app_state.model_parallel_rank + elif os.path.isdir(checkpoint_file): + # starting training from megatron-lm checkpoint + mp_ranks = glob.glob(os.path.join(checkpoint_file, 'mp_rank*')) + model_parallel_size = len(mp_ranks) + app_state.model_parallel_size = model_parallel_size + logging.info( + ( + f'restore_path: {checkpoint_file} is a directory. ' + f'Assuming megatron model parallelism with ' + f'model_parallel_size: {model_parallel_size}' + ) + ) + # try to get local rank from global + local_rank = None + try: + local_rank = int(os.environ['LOCAL_RANK']) + except: + logging.info('Global variable LOCAL_RANK not yet specified') + if local_rank is not None: + app_state.local_rank = local_rank + else: + # if local is None then we are on the main process + local_rank = 0 + model_parallel_rank = compute_model_parallel_rank(local_rank, model_parallel_size) + app_state.model_parallel_rank = model_parallel_rank + else: + model_parallel_size = None + model_parallel_rank = None + + model = MegatronBertEncoder( + model_name=pretrained_model_name, + config=config, + vocab_file=vocab_file, + model_parallel_size=model_parallel_size, + model_parallel_rank=model_parallel_rank, + ) + + return model, checkpoint_file + + +def compute_model_parallel_rank(local_rank, model_parallel_size): + return local_rank % model_parallel_size + + +def get_megatron_lm_models_list() -> List[str]: + """ + Returns the list of supported Megatron-LM models + """ + return list(MEGATRON_CONFIG_MAP.keys()) + + +def get_megatron_config(pretrained_model_name: str) -> Dict[str, int]: + """ + Returns Megatron-LM model config file + + Args: + pretrained_model_name (str): pretrained model name + + Returns: + config (dict): contains model configuration: number of hidden layers, number of attention heads, etc + """ + _check_megatron_name(pretrained_model_name) + return MEGATRON_CONFIG_MAP[pretrained_model_name]["config"] + + +def _check_megatron_name(pretrained_model_name: str) -> None: + megatron_model_list = get_megatron_lm_models_list() + if pretrained_model_name not in megatron_model_list: + raise ValueError(f'For Megatron-LM models, choose from the following list: {megatron_model_list}') + + +def get_megatron_vocab_file(pretrained_model_name: str) -> str: + """ + Gets vocabulary file from cache or downloads it + + Args: + pretrained_model_name: pretrained model name + + Returns: + path: path to the vocab file + """ + _check_megatron_name(pretrained_model_name) + url = MEGATRON_CONFIG_MAP[pretrained_model_name]["vocab"] + + path = os.path.join(MEGATRON_CACHE, pretrained_model_name + "_vocab") + path = _download(path, url) + return path + + +def get_megatron_checkpoint(pretrained_model_name: str) -> str: + """ + Gets checkpoint file from cache or downloads it + Args: + pretrained_model_name: pretrained model name + Returns: + path: path to model checkpoint + """ + _check_megatron_name(pretrained_model_name) + url = MEGATRON_CONFIG_MAP[pretrained_model_name]["checkpoint"] + path = os.path.join(MEGATRON_CACHE, pretrained_model_name) + return _download(path, url) + + +def _download(path: str, url: str): + """ + Gets a file from cache or downloads it + + Args: + path: path to the file in cache + url: url to the file + Returns: + path: path to the file in cache + """ + if url is None: + return None + + if not os.path.exists(path): + master_device = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + if not os.path.exists(path): + if master_device: + os.makedirs(MEGATRON_CACHE, exist_ok=True) + logging.info(f"Downloading from {url}") + wget.download(url, path) + # wait until the master process downloads the file and writes it to the cache dir + if torch.distributed.is_initialized(): + torch.distributed.barrier() + + return path + + +def is_lower_cased_megatron(pretrained_model_name): + """ + Returns if the megatron is cased or uncased + + Args: + pretrained_model_name (str): pretrained model name + Returns: + do_lower_cased (bool): whether the model uses lower cased data + """ + _check_megatron_name(pretrained_model_name) + return MEGATRON_CONFIG_MAP[pretrained_model_name]["do_lower_case"] + + +def get_megatron_tokenizer(pretrained_model_name: str): + """ + Takes a pretrained_model_name for megatron such as "megatron-bert-cased" and returns the according + tokenizer name for tokenizer instantiating. + + Args: + pretrained_model_name: pretrained_model_name for megatron such as "megatron-bert-cased" + Returns: + tokenizer name for tokenizer instantiating + """ + _check_megatron_name(pretrained_model_name) + return MEGATRON_CONFIG_MAP[pretrained_model_name]["tokenizer_name"] diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_classifier.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_classifier.py new file mode 100644 index 0000000000..aac2a2707b --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_classifier.py @@ -0,0 +1,72 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional + +from nemo.collections.common.parts import MultiLayerPerceptron +from nemo.collections.nlp.modules.common.classifier import Classifier +from nemo.core.classes import typecheck +from nemo.core.neural_types import LogitsType, LogprobsType, NeuralType + +__all__ = ['SequenceClassifier'] + + +class SequenceClassifier(Classifier): + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + if not self.log_softmax: + return {"logits": NeuralType(('B', 'D'), LogitsType())} + else: + return {"log_probs": NeuralType(('B', 'D'), LogprobsType())} + + def __init__( + self, + hidden_size: int, + num_classes: int, + num_layers: int = 2, + activation: str = 'relu', + log_softmax: bool = True, + dropout: float = 0.0, + use_transformer_init: bool = True, + idx_conditioned_on: int = 0, + ): + """ + Initializes the SequenceClassifier module. + Args: + hidden_size: the hidden size of the mlp head on the top of the encoder + num_classes: number of the classes to predict + num_layers: number of the linear layers of the mlp head on the top of the encoder + activation: type of activations between layers of the mlp head + log_softmax: applies the log softmax on the output + dropout: the dropout used for the mlp head + use_transformer_init: initializes the weights with the same approach used in Transformer + idx_conditioned_on: index of the token to use as the sequence representation for the classification task, default is the first token + """ + super().__init__(hidden_size=hidden_size, dropout=dropout) + self.log_softmax = log_softmax + self._idx_conditioned_on = idx_conditioned_on + self.mlp = MultiLayerPerceptron( + hidden_size=hidden_size, + num_classes=num_classes, + num_layers=num_layers, + activation=activation, + log_softmax=log_softmax, + ) + self.post_init(use_transformer_init=use_transformer_init) + + @typecheck() + def forward(self, hidden_states): + hidden_states = self.dropout(hidden_states) + logits = self.mlp(hidden_states[:, self._idx_conditioned_on]) + return logits diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_regression.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_regression.py new file mode 100644 index 0000000000..f3150727ce --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_regression.py @@ -0,0 +1,69 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional + +from torch import Tensor + +from nemo.collections.common.parts import MultiLayerPerceptron +from nemo.collections.nlp.modules.common.classifier import Classifier +from nemo.core.classes import typecheck +from nemo.core.neural_types import NeuralType, RegressionValuesType + +__all__ = ['SequenceRegression'] + + +class SequenceRegression(Classifier): + """ + Args: + hidden_size: the hidden size of the mlp head on the top of the encoder + num_layers: number of the linear layers of the mlp head on the top of the encoder + activation: type of activations between layers of the mlp head + dropout: the dropout used for the mlp head + use_transformer_init: initializes the weights with the same approach used in Transformer + idx_conditioned_on: index of the token to use as the sequence representation for the classification task, + default is the first token + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return {"preds": NeuralType(tuple('B'), RegressionValuesType())} + + def __init__( + self, + hidden_size: int, + num_layers: int = 2, + activation: str = 'relu', + dropout: float = 0.0, + use_transformer_init: bool = True, + idx_conditioned_on: int = 0, + ): + """ Initializes the SequenceRegression module. """ + super().__init__(hidden_size=hidden_size, dropout=dropout) + self._idx_conditioned_on = idx_conditioned_on + self.mlp = MultiLayerPerceptron( + hidden_size, num_classes=1, num_layers=num_layers, activation=activation, log_softmax=False, + ) + self.post_init(use_transformer_init=use_transformer_init) + + @typecheck() + def forward(self, hidden_states: Tensor) -> Tensor: + """ Forward pass through the module. + + Args: + hidden_states: hidden states for each token in a sequence, for example, BERT module output + """ + hidden_states = self.dropout(hidden_states) + preds = self.mlp(hidden_states[:, self._idx_conditioned_on]) + return preds.view(-1) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_token_classifier.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_token_classifier.py new file mode 100644 index 0000000000..018fc74a40 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/sequence_token_classifier.py @@ -0,0 +1,80 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional + +from nemo.collections.common.parts import MultiLayerPerceptron +from nemo.collections.nlp.modules.common.classifier import Classifier +from nemo.core.classes import typecheck +from nemo.core.neural_types import LogitsType, NeuralType + +__all__ = ['SequenceTokenClassifier'] + + +class SequenceTokenClassifier(Classifier): + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + return { + "intent_logits": NeuralType(('B', 'D'), LogitsType()), + "slot_logits": NeuralType(('B', 'T', 'D'), LogitsType()), + } + + def __init__( + self, + hidden_size: int, + num_intents: int, + num_slots: int, + num_layers: int = 2, + activation: str = 'relu', + log_softmax: bool = False, + dropout: float = 0.0, + use_transformer_init: bool = True, + ): + """ + Initializes the SequenceTokenClassifier module, could be used for tasks that train sequence and + token classifiers jointly, for example, for intent detection and slot tagging task. + Args: + hidden_size: hidden size of the mlp head on the top of the encoder + num_intents: number of the intents to predict + num_slots: number of the slots to predict + num_layers: number of the linear layers of the mlp head on the top of the encoder + activation: type of activations between layers of the mlp head + log_softmax: applies the log softmax on the output + dropout: the dropout used for the mlp head + use_transformer_init: initializes the weights with the same approach used in Transformer + """ + super().__init__(hidden_size=hidden_size, dropout=dropout) + self.intent_mlp = MultiLayerPerceptron( + hidden_size=hidden_size, + num_classes=num_intents, + num_layers=num_layers, + activation=activation, + log_softmax=log_softmax, + ) + self.slot_mlp = MultiLayerPerceptron( + hidden_size=hidden_size, + num_classes=num_slots, + num_layers=num_layers, + activation=activation, + log_softmax=log_softmax, + ) + self.post_init(use_transformer_init=use_transformer_init) + + @typecheck() + def forward(self, hidden_states): + hidden_states = self.dropout(hidden_states) + # intent is classified by first hidden position + intent_logits = self.intent_mlp(hidden_states[:, 0]) + slot_logits = self.slot_mlp(hidden_states) + return intent_logits, slot_logits diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/token_classifier.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/token_classifier.py new file mode 100644 index 0000000000..a46d4ef50b --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/token_classifier.py @@ -0,0 +1,165 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Dict, Optional + +from omegaconf.omegaconf import MISSING +from torch import nn as nn + +from nemo.collections.common.parts import MultiLayerPerceptron +from nemo.collections.nlp.modules.common.classifier import Classifier +from nemo.core.classes import typecheck +from nemo.core.neural_types import LogitsType, LogprobsType, NeuralType + +__all__ = ['BertPretrainingTokenClassifier', 'TokenClassifier'] + +ACT2FN = {"gelu": nn.functional.gelu, "relu": nn.functional.relu} + + +@dataclass +class TokenClassifierConfig: + num_layers: int = 1 + activation: str = 'relu' + log_softmax: bool = True + dropout: float = 0.0 + use_transformer_init: bool = True + + +class TokenClassifier(Classifier): + """ + A module to perform token level classification tasks such as Named entity recognition. + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """ + Returns definitions of module output ports. + """ + if not self.log_softmax: + return {"logits": NeuralType(('B', 'T', 'C'), LogitsType())} + else: + return {"log_probs": NeuralType(('B', 'T', 'C'), LogprobsType())} + + def __init__( + self, + hidden_size: int, + num_classes: int, + num_layers: int = 1, + activation: str = 'relu', + log_softmax: bool = True, + dropout: float = 0.0, + use_transformer_init: bool = True, + ) -> None: + + """ + Initializes the Token Classifier module. + + Args: + hidden_size: the size of the hidden dimension + num_classes: number of classes + num_layers: number of fully connected layers in the multilayer perceptron (MLP) + activation: activation to usee between fully connected layers in the MLP + log_softmax: whether to apply softmax to the output of the MLP + dropout: dropout to apply to the input hidden states + use_transformer_init: whether to initialize the weights of the classifier head with the same approach used in Transformer + """ + super().__init__(hidden_size=hidden_size, dropout=dropout) + self.log_softmax = log_softmax + self.mlp = MultiLayerPerceptron( + hidden_size, num_classes, num_layers=num_layers, activation=activation, log_softmax=log_softmax + ) + self.post_init(use_transformer_init=use_transformer_init) + + @typecheck() + def forward(self, hidden_states): + """ + Performs the forward step of the module. + Args: + hidden_states: batch of hidden states (for example, from the BERT encoder module) + [BATCH_SIZE x SEQ_LENGTH x HIDDEN_SIZE] + Returns: logits value for each class [BATCH_SIZE x SEQ_LENGTH x NUM_CLASSES] + """ + hidden_states = self.dropout(hidden_states) + logits = self.mlp(hidden_states) + return logits + + +class BertPretrainingTokenClassifier(Classifier): + """ + A module to perform token level classification tasks for Bert pretraining. + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """ + Returns definitions of module output ports. + """ + if not self.log_softmax: + return {"logits": NeuralType(('B', 'T', 'C'), LogitsType())} + else: + return {"log_probs": NeuralType(('B', 'T', 'C'), LogprobsType())} + + def __init__( + self, + hidden_size: int, + num_classes: int, + num_layers: int = 1, + activation: str = 'relu', + log_softmax: bool = True, + dropout: float = 0.0, + use_transformer_init: bool = True, + ) -> None: + + """ + Initializes the Token Classifier module. + + Args: + hidden_size: the size of the hidden dimension + num_classes: number of classes + num_layers: number of fully connected layers in the multilayer perceptron (MLP) + activation: activation to usee between fully connected layers in the MLP + log_softmax: whether to apply softmax to the output of the MLP + dropout: dropout to apply to the input hidden states + use_transformer_init: whether to initialize the weights of the classifier head with the same approach used in Transformer + """ + super().__init__(hidden_size=hidden_size, dropout=dropout) + + self.log_softmax = log_softmax + + if activation not in ACT2FN: + raise ValueError(f'activation "{activation}" not found') + self.dense = nn.Linear(hidden_size, hidden_size) + self.act = ACT2FN[activation] + self.norm = nn.LayerNorm(hidden_size, eps=1e-12) + self.mlp = MultiLayerPerceptron( + hidden_size, num_classes, num_layers=num_layers, activation=activation, log_softmax=log_softmax + ) + self.post_init(use_transformer_init=use_transformer_init) + + @typecheck() + def forward(self, hidden_states): + """ + Performs the forward step of the module. + Args: + hidden_states: batch of hidden states (for example, from the BERT encoder module) + [BATCH_SIZE x SEQ_LENGTH x HIDDEN_SIZE] + Returns: logits value for each class [BATCH_SIZE x SEQ_LENGTH x NUM_CLASSES] + """ + hidden_states = self.dropout(hidden_states) + hidden_states = self.dense(hidden_states) + hidden_states = self.act(hidden_states) + transform = self.norm(hidden_states) + logits = self.mlp(transform) + return logits diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/tokenizer_utils.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/tokenizer_utils.py new file mode 100644 index 0000000000..bb8022febf --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/tokenizer_utils.py @@ -0,0 +1,144 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os.path +from dataclasses import MISSING, dataclass +from os import path +from typing import Dict, List, Optional + +import nemo +from nemo.collections.common.tokenizers.char_tokenizer import CharTokenizer +from nemo.collections.common.tokenizers.huggingface.auto_tokenizer import AutoTokenizer +from nemo.collections.common.tokenizers.word_tokenizer import WordTokenizer +from nemo.collections.common.tokenizers.youtokentome_tokenizer import YouTokenToMeTokenizer +from nemo.collections.nlp.modules.common.huggingface.huggingface_utils import get_huggingface_pretrained_lm_models_list +from nemo.collections.nlp.modules.common.lm_utils import get_pretrained_lm_models_list +from nemo.collections.nlp.modules.common.megatron.megatron_utils import get_megatron_tokenizer +from nemo.utils import logging + +__all__ = ['get_tokenizer', 'get_tokenizer_list'] + + +def get_tokenizer_list() -> List[str]: + """ + Returns all all supported tokenizer names + """ + s = set(get_pretrained_lm_models_list()) + s.update(set(get_huggingface_pretrained_lm_models_list(include_external=True))) + return ["sentencepiece", "char", "word"] + list(s) + + +@dataclass +class TokenizerConfig: + library: str = MISSING + tokenizer_model: Optional[str] = None + vocab_size: Optional[int] = None + vocab_file: Optional[str] = None + special_tokens: Optional[Dict[str, str]] = None + bpe_dropout: Optional[float] = 0.0 + coverage: Optional[float] = 0.999 + training_sample_size: Optional[int] = None + + +def get_tokenizer( + tokenizer_name: str, + tokenizer_model: Optional[str] = None, + vocab_file: Optional[str] = None, + special_tokens: Optional[Dict[str, str]] = None, + use_fast: Optional[bool] = False, + bpe_dropout: Optional[float] = 0.0, +): + """ + Args: + tokenizer_name: sentencepiece or pretrained model from the hugging face list, + for example: bert-base-cased + To see the list of all HuggingFace pretrained models, use: nemo_nlp.modules.common.get_huggingface_pretrained_lm_models_list() + tokenizer_model: tokenizer model file of sentencepiece or youtokentome + special_tokens: dict of special tokens + vocab_file: path to vocab file + use_fast: (only for HuggingFace AutoTokenizer) set to True to use fast HuggingFace tokenizer + bpe_dropout: (only supported by YTTM tokenizer) BPE dropout tries to corrupt the standard segmentation procedure of BPE to help + model better learn word compositionality and become robust to segmentation errors. + It has emperically been shown to improve inference time BLEU scores. + """ + if special_tokens is None: + special_tokens_dict = {} + else: + special_tokens_dict = special_tokens + + if 'megatron' in tokenizer_name: + if vocab_file is None: + vocab_file = nemo.collections.nlp.modules.common.megatron.megatron_utils.get_megatron_vocab_file( + tokenizer_name + ) + tokenizer_name = get_megatron_tokenizer(tokenizer_name) + + if tokenizer_name == 'sentencepiece': + return nemo.collections.common.tokenizers.sentencepiece_tokenizer.SentencePieceTokenizer( + model_path=tokenizer_model, special_tokens=special_tokens, legacy=True + ) + elif tokenizer_name == 'yttm': + return YouTokenToMeTokenizer(model_path=tokenizer_model, bpe_dropout=bpe_dropout) + elif tokenizer_name == 'word': + return WordTokenizer(vocab_file=vocab_file, **special_tokens_dict) + elif tokenizer_name == 'char': + return CharTokenizer(vocab_file=vocab_file, **special_tokens_dict) + + return AutoTokenizer( + pretrained_model_name=tokenizer_name, vocab_file=vocab_file, **special_tokens_dict, use_fast=use_fast + ) + + +def get_nmt_tokenizer( + library: str = 'yttm', + model_name: Optional[str] = None, + tokenizer_model: Optional[str] = None, + vocab_file: Optional[str] = None, + special_tokens: Optional[Dict[str, str]] = None, + use_fast: Optional[bool] = False, + bpe_dropout: Optional[float] = 0.0, +): + """ + Args: + model_name: if using a pretrained model from NeMo or HuggingFace + tokenizer_model: tokenizer model file of sentencepiece or youtokentome + special_tokens: dict of special tokens + vocab_file: path to vocab file + use_fast: (only for HuggingFace AutoTokenizer) set to True to use fast HuggingFace tokenizer + bpe_dropout: (only supported by YTTM tokenizer) BPE dropout tries to corrupt the standard segmentation procedure of BPE to help + model better learn word compositionality and become robust to segmentation errors. + It has emperically been shown to improve inference time BLEU scores. + """ + if special_tokens is None: + special_tokens_dict = {} + else: + special_tokens_dict = special_tokens + + if library == 'yttm': + logging.info(f'Getting YouTokenToMeTokenizer with model: {tokenizer_model}.') + return YouTokenToMeTokenizer(model_path=tokenizer_model, bpe_dropout=bpe_dropout) + elif library == 'huggingface': + logging.info(f'Getting HuggingFace AutoTokenizer with pretrained_model_name: {model_name}') + return AutoTokenizer( + pretrained_model_name=model_name, vocab_file=vocab_file, **special_tokens_dict, use_fast=use_fast + ) + elif library == 'sentencepiece': + logging.info(f'Getting SentencePiece with model: {model_name}') + return nemo.collections.common.tokenizers.sentencepiece_tokenizer.SentencePieceTokenizer( + model_path=tokenizer_model, special_tokens=special_tokens_dict + ) + else: + raise NotImplementedError( + 'Currently we only support "yttm", "huggingface", and "sentencepiece" tokenizer library.' + ) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/__init__.py new file mode 100644 index 0000000000..4a0c9eec4c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.modules.common.transformer.transformer_decoders import * +from nemo.collections.nlp.modules.common.transformer.transformer_encoders import * +from nemo.collections.nlp.modules.common.transformer.transformer_generators import * +from nemo.collections.nlp.modules.common.transformer.transformer_modules import * diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer.py new file mode 100644 index 0000000000..ea04c2efcf --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer.py @@ -0,0 +1,248 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional + +import torch +from omegaconf.omegaconf import MISSING + +from nemo.collections.nlp.modules.common.decoder_module import DecoderModule +from nemo.collections.nlp.modules.common.encoder_module import EncoderModule +from nemo.collections.nlp.modules.common.transformer.transformer_decoders import TransformerDecoder +from nemo.collections.nlp.modules.common.transformer.transformer_encoders import TransformerEncoder +from nemo.collections.nlp.modules.common.transformer.transformer_modules import TransformerEmbedding +from nemo.core.classes.common import typecheck +from nemo.core.classes.exportable import Exportable + +# @dataclass +# class TransformerConfig: +# # named model arguments +# library: str = 'nemo' +# model_name: Optional[str] = None +# pretrained: bool = False + + +@dataclass +class NeMoTransformerConfig: + # must be configured by the user + hidden_size: int = MISSING + num_layers: int = MISSING + inner_size: int = MISSING + num_attention_heads: int = MISSING + + # embedding + max_sequence_length: int = 512 + num_token_types: int = 2 + embedding_dropout: float = 0.0 + learn_positional_encodings: bool = False + + # transformer + ffn_dropout: float = 0.0 + attn_score_dropout: float = 0.0 + attn_layer_dropout: float = 0.0 + hidden_act: str = 'relu' + pre_ln: bool = False + pre_ln_final_layer_norm: bool = True + + # named model arguments + library: str = 'nemo' + model_name: Optional[str] = None + pretrained: bool = False + + +@dataclass +class NeMoTransformerEncoderConfig(NeMoTransformerConfig): + mask_future: bool = False + + +class TransformerEncoderNM(EncoderModule, Exportable): + def __init__( + self, + vocab_size: int, + hidden_size: int, + num_layers: int, + inner_size: int, + num_attention_heads: int, + max_sequence_length: int = 512, + num_token_types: int = 2, + embedding_dropout: float = 0.0, + learn_positional_encodings: bool = False, + ffn_dropout: float = 0.0, + attn_score_dropout: float = 0.0, + attn_layer_dropout: float = 0.0, + hidden_act: str = 'relu', + mask_future: bool = False, + pre_ln: bool = False, + pre_ln_final_layer_norm: bool = True, + ): + super().__init__() + + self._vocab_size = vocab_size + self._hidden_size = hidden_size + self._max_sequence_length = max_sequence_length + + self._embedding = TransformerEmbedding( + vocab_size=self._vocab_size, + hidden_size=self._hidden_size, + max_sequence_length=max_sequence_length, + num_token_types=num_token_types, + embedding_dropout=embedding_dropout, + learn_positional_encodings=learn_positional_encodings, + ) + + self._encoder = TransformerEncoder( + hidden_size=self._hidden_size, + num_layers=num_layers, + inner_size=inner_size, + num_attention_heads=num_attention_heads, + ffn_dropout=ffn_dropout, + attn_score_dropout=attn_score_dropout, + attn_layer_dropout=attn_layer_dropout, + hidden_act=hidden_act, + mask_future=mask_future, + pre_ln=pre_ln, + pre_ln_final_layer_norm=pre_ln_final_layer_norm, + ) + + @typecheck() + def forward(self, input_ids, encoder_mask): + embeddings = self._embedding(input_ids=input_ids) + encoder_hidden_states = self._encoder(encoder_states=embeddings, encoder_mask=encoder_mask) + return encoder_hidden_states + + @property + def hidden_size(self): + return self._hidden_size + + @property + def vocab_size(self): + return self._vocab_size + + @property + def max_sequence_length(self): + return self._max_sequence_length + + @property + def embedding(self): + return self._embedding + + @property + def encoder(self): + return self._encoder + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + sample = next(self.parameters()) + input_ids = torch.randint(low=0, high=2048, size=(2, 16), device=sample.device) + encoder_mask = torch.randint(low=0, high=1, size=(2, 16), device=sample.device) + return tuple([input_ids, encoder_mask]) + + +class TransformerDecoderNM(DecoderModule, Exportable): + def __init__( + self, + vocab_size: int, + hidden_size: int, + num_layers: int, + inner_size: int, + num_attention_heads: int, + max_sequence_length: int = 512, + num_token_types: int = 2, + embedding_dropout: float = 0.0, + learn_positional_encodings: bool = False, + ffn_dropout: float = 0.0, + attn_score_dropout: float = 0.0, + attn_layer_dropout: float = 0.0, + hidden_act: str = 'relu', + pre_ln: bool = False, + pre_ln_final_layer_norm: bool = True, + ): + super().__init__() + + self._vocab_size = vocab_size + self._hidden_size = hidden_size + self._max_sequence_length = max_sequence_length + + self._embedding = TransformerEmbedding( + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + max_sequence_length=max_sequence_length, + num_token_types=num_token_types, + embedding_dropout=embedding_dropout, + learn_positional_encodings=learn_positional_encodings, + ) + + self._decoder = TransformerDecoder( + hidden_size=self.hidden_size, + num_layers=num_layers, + inner_size=inner_size, + num_attention_heads=num_attention_heads, + ffn_dropout=ffn_dropout, + attn_score_dropout=attn_score_dropout, + attn_layer_dropout=attn_layer_dropout, + hidden_act=hidden_act, + pre_ln=pre_ln, + pre_ln_final_layer_norm=pre_ln_final_layer_norm, + ) + + @typecheck() + def forward(self, input_ids, decoder_mask, encoder_embeddings, encoder_mask): + decoder_embeddings = self._embedding(input_ids=input_ids) + decoder_hidden_states = self._decoder( + decoder_states=decoder_embeddings, + decoder_mask=decoder_mask, + encoder_states=encoder_embeddings, + encoder_mask=encoder_mask, + ) + return decoder_hidden_states + + @property + def hidden_size(self): + return self._hidden_size + + @property + def vocab_size(self): + return self._vocab_size + + @property + def max_sequence_length(self): + return self._max_sequence_length + + @property + def embedding(self): + return self._embedding + + @property + def decoder(self): + return self._decoder + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + sample = next(self.parameters()) + input_ids = torch.randint(low=0, high=2048, size=(2, 16), device=sample.device) + encoder_mask = torch.randint(low=0, high=1, size=(2, 16), device=sample.device) + return tuple([input_ids, encoder_mask, self._embedding(input_ids), encoder_mask]) + + def _prepare_for_export(self, **kwargs): + self._decoder.diagonal = None + super()._prepare_for_export(**kwargs) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_decoders.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_decoders.py new file mode 100644 index 0000000000..d1dc2086cd --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_decoders.py @@ -0,0 +1,200 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +from dataclasses import dataclass + +import torch +import torch.nn as nn +from omegaconf.omegaconf import MISSING + +from nemo.collections.common.parts import form_attention_mask +from nemo.collections.nlp.modules.common.transformer.transformer_modules import MultiHeadAttention, PositionWiseFF + +__all__ = ["TransformerDecoder"] + + +class TransformerDecoderBlock(nn.Module): + """ + Building block of Transformer decoder. + + Args: + hidden_size: size of the embeddings in the model, also known as d_model + inner_size: number of neurons in the intermediate part of feed-forward + net, usually is (4-8 x hidden_size) in the papers + num_attention_heads: number of heads in multi-head attention + attn_score_dropout: probability of dropout applied to attention scores + attn_layer_dropout: probability of dropout applied to the output of the + attention layers, but before layer normalization + ffn_dropout: probability of dropout applied to FFN output + hidden_act: activation function used between two linear layers in FFN + """ + + def __init__( + self, + hidden_size: int, + inner_size: int, + num_attention_heads: int = 1, + attn_score_dropout: float = 0.0, + attn_layer_dropout: float = 0.0, + ffn_dropout: float = 0.0, + hidden_act: str = "relu", + pre_ln: bool = False, + ): + super().__init__() + self.pre_ln = pre_ln + self.layer_norm_1 = nn.LayerNorm(hidden_size, eps=1e-5) + self.first_sub_layer = MultiHeadAttention( + hidden_size, num_attention_heads, attn_score_dropout, attn_layer_dropout + ) + self.layer_norm_2 = nn.LayerNorm(hidden_size, eps=1e-5) + self.second_sub_layer = MultiHeadAttention( + hidden_size, num_attention_heads, attn_score_dropout, attn_layer_dropout + ) + self.layer_norm_3 = nn.LayerNorm(hidden_size, eps=1e-5) + self.third_sub_layer = PositionWiseFF(hidden_size, inner_size, ffn_dropout, hidden_act) + + def forward_preln(self, decoder_query, decoder_mask, decoder_keys, encoder_states, encoder_mask): + """ + Pre-LayerNorm block + Order of operations: LN -> Self-Attn -> Residual -> LN -> Cross-Attn -> Residual -> LN -> FFN + """ + residual = decoder_query + decoder_query = self.layer_norm_1(decoder_query) + decoder_keys = self.layer_norm_1(decoder_keys) + self_attn_output = self.first_sub_layer(decoder_query, decoder_keys, decoder_keys, decoder_mask) + self_attn_output += residual + + residual = self_attn_output + self_attn_output = self.layer_norm_2(self_attn_output) + enc_dec_attn_output = self.second_sub_layer(self_attn_output, encoder_states, encoder_states, encoder_mask) + enc_dec_attn_output += residual + + residual = enc_dec_attn_output + enc_dec_attn_output = self.layer_norm_3(enc_dec_attn_output) + output_states = self.third_sub_layer(enc_dec_attn_output) + output_states += residual + + return output_states + + def forward_postln(self, decoder_query, decoder_mask, decoder_keys, encoder_states, encoder_mask): + """ + Post-LayerNorm block + Order of operations: Self-Attn -> Residual -> LN -> Cross-Attn -> Residual -> LN -> FFN -> Residual -> LN + """ + self_attn_output = self.first_sub_layer(decoder_query, decoder_keys, decoder_keys, decoder_mask) + self_attn_output += decoder_query + self_attn_output = self.layer_norm_1(self_attn_output) + + enc_dec_attn_output = self.second_sub_layer(self_attn_output, encoder_states, encoder_states, encoder_mask) + enc_dec_attn_output += self_attn_output + enc_dec_attn_output = self.layer_norm_2(enc_dec_attn_output) + + output_states = self.third_sub_layer(enc_dec_attn_output) + output_states += enc_dec_attn_output + return self.layer_norm_3(output_states) + + def forward(self, decoder_query, decoder_mask, decoder_keys, encoder_states, encoder_mask): + if self.pre_ln: + return self.forward_preln(decoder_query, decoder_mask, decoder_keys, encoder_states, encoder_mask) + else: + return self.forward_postln(decoder_query, decoder_mask, decoder_keys, encoder_states, encoder_mask) + + +class TransformerDecoder(nn.Module): + def __init__( + self, + num_layers: int, + hidden_size: int, + inner_size: int, + num_attention_heads: int = 1, + attn_score_dropout: float = 0.0, + attn_layer_dropout: float = 0.0, + ffn_dropout: float = 0.0, + hidden_act: str = "relu", + pre_ln: bool = False, + pre_ln_final_layer_norm: bool = True, + ): + super().__init__() + + if pre_ln and pre_ln_final_layer_norm: + self.final_layer_norm = nn.LayerNorm(hidden_size, eps=1e-5) + else: + self.final_layer_norm = None + + layer = TransformerDecoderBlock( + hidden_size, + inner_size, + num_attention_heads, + attn_score_dropout, + attn_layer_dropout, + ffn_dropout, + hidden_act, + pre_ln, + ) + self.layers = nn.ModuleList([copy.deepcopy(layer) for _ in range(num_layers)]) + self.diagonal = 0 + + def _get_memory_states(self, decoder_states, decoder_mems_list=None, i=0): + if decoder_mems_list is not None: + memory_states = torch.cat((decoder_mems_list[i], decoder_states), dim=1) + else: + memory_states = decoder_states + return memory_states + + def forward( + self, decoder_states, decoder_mask, encoder_states, encoder_mask, decoder_mems_list=None, return_mems=False + ): + """ + Args: + decoder_states: output of the embedding layer (B x L_dec x H) + decoder_mask: decoder inputs mask (B x L_dec) + encoder_states: output of the encoder (B x L_enc x H) + encoder_mask: encoder inputs mask (B x L_enc) + decoder_mems_list: list of the cached decoder hidden states + for fast autoregressive generation which will be used instead + of decoder_states as keys and values if not None + return_mems: bool, whether to return outputs of all decoder layers + or the last layer only + """ + decoder_attn_mask = form_attention_mask(decoder_mask, diagonal=self.diagonal) + encoder_attn_mask = form_attention_mask(encoder_mask) + memory_states = self._get_memory_states(decoder_states, decoder_mems_list, 0) + cached_mems_list = [memory_states] + + for i, layer in enumerate(self.layers): + decoder_states = layer(decoder_states, decoder_attn_mask, memory_states, encoder_states, encoder_attn_mask) + memory_states = self._get_memory_states(decoder_states, decoder_mems_list, i + 1) + cached_mems_list.append(memory_states) + + if self.final_layer_norm is not None: + decoder_states = self.final_layer_norm(decoder_states) + memory_states = self._get_memory_states(decoder_states, decoder_mems_list, i + 1) + cached_mems_list.append(memory_states) + + if return_mems: + return cached_mems_list + else: + return cached_mems_list[-1] + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + sample = next(self.parameters()) + input_ids = torch.randint(low=0, high=2048, size=(2, 16, 1024), device=sample.device) + encoder_mask = torch.randint(low=0, high=1, size=(2, 16), device=sample.device) + return tuple([input_ids, encoder_mask, input_ids, encoder_mask]) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_encoders.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_encoders.py new file mode 100644 index 0000000000..2cecef79c8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_encoders.py @@ -0,0 +1,176 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +from dataclasses import dataclass + +import torch +import torch.nn as nn +from omegaconf.omegaconf import MISSING + +from nemo.collections.common.parts import form_attention_mask +from nemo.collections.nlp.modules.common.transformer.transformer_modules import MultiHeadAttention, PositionWiseFF + +__all__ = ["TransformerEncoder"] + + +class TransformerEncoderBlock(nn.Module): + """ + Building block of Transformer encoder. + + Args: + hidden_size: size of the embeddings in the model, also known as d_model + inner_size: number of neurons in the intermediate part of feed-forward + net, usually is (4-8 x hidden_size) in the papers + num_attention_heads: number of heads in multi-head attention + attn_score_dropout: probability of dropout applied to attention scores + attn_layer_dropout: probability of dropout applied to the output of the + attention layers, but before layer normalization + ffn_dropout: probability of dropout applied to FFN output + hidden_act: activation function used between two linear layers in FFN + """ + + def __init__( + self, + hidden_size: int, + inner_size: int, + num_attention_heads: int = 1, + attn_score_dropout: float = 0.0, + attn_layer_dropout: float = 0.0, + ffn_dropout: float = 0.0, + hidden_act: str = "relu", + pre_ln: bool = False, + ): + super().__init__() + self.pre_ln = pre_ln + self.layer_norm_1 = nn.LayerNorm(hidden_size, eps=1e-5) + self.first_sub_layer = MultiHeadAttention( + hidden_size, num_attention_heads, attn_score_dropout, attn_layer_dropout + ) + self.layer_norm_2 = nn.LayerNorm(hidden_size, eps=1e-5) + self.second_sub_layer = PositionWiseFF(hidden_size, inner_size, ffn_dropout, hidden_act) + + def forward_preln(self, encoder_query, encoder_mask, encoder_keys): + """ + Pre-LayerNorm block + Order of operations: LN -> Self-Attn -> Residual -> LN -> Cross-Attn -> Residual -> LN -> FFN + """ + residual = encoder_query + encoder_query = self.layer_norm_1(encoder_query) + encoder_keys = self.layer_norm_1(encoder_keys) + self_attn_output = self.first_sub_layer(encoder_query, encoder_keys, encoder_keys, encoder_mask) + self_attn_output += residual + + residual = self_attn_output + self_attn_output = self.layer_norm_2(self_attn_output) + output_states = self.second_sub_layer(self_attn_output) + output_states += residual + + return output_states + + def forward_postln(self, encoder_query, encoder_mask, encoder_keys): + """ + Post-LayerNorm block + Order of operations: Self-Attn -> Residual -> LN -> Cross-Attn -> Residual -> LN -> FFN -> Residual -> LN + """ + self_attn_output = self.first_sub_layer(encoder_query, encoder_keys, encoder_keys, encoder_mask) + self_attn_output += encoder_query + self_attn_output = self.layer_norm_1(self_attn_output) + + output_states = self.second_sub_layer(self_attn_output) + output_states += self_attn_output + output_states = self.layer_norm_2(output_states) + + return output_states + + def forward(self, encoder_query, encoder_mask, encoder_keys): + if self.pre_ln: + return self.forward_preln(encoder_query, encoder_mask, encoder_keys) + else: + return self.forward_postln(encoder_query, encoder_mask, encoder_keys) + + +class TransformerEncoder(nn.Module): + def __init__( + self, + num_layers: int, + hidden_size: int, + inner_size: int, + mask_future: bool = False, + num_attention_heads: int = 1, + attn_score_dropout: float = 0.0, + attn_layer_dropout: float = 0.0, + ffn_dropout: float = 0.0, + hidden_act: str = "relu", + pre_ln: bool = False, + pre_ln_final_layer_norm: bool = True, + ): + super().__init__() + + if pre_ln and pre_ln_final_layer_norm: + self.final_layer_norm = nn.LayerNorm(hidden_size, eps=1e-5) + else: + self.final_layer_norm = None + + layer = TransformerEncoderBlock( + hidden_size, + inner_size, + num_attention_heads, + attn_score_dropout, + attn_layer_dropout, + ffn_dropout, + hidden_act, + pre_ln, + ) + self.layers = nn.ModuleList([copy.deepcopy(layer) for _ in range(num_layers)]) + self.diag = 0 if mask_future else None + + def _get_memory_states(self, encoder_states, encoder_mems_list=None, i=0): + if encoder_mems_list is not None: + memory_states = torch.cat((encoder_mems_list[i], encoder_states), dim=1) + else: + memory_states = encoder_states + return memory_states + + def forward(self, encoder_states, encoder_mask, encoder_mems_list=None, return_mems=False): + """ + Args: + encoder_states: output of the embedding_layer (B x L_enc x H) + encoder_mask: encoder inputs mask (B x L_enc) + encoder_mems_list: list of the cached encoder hidden states + for fast autoregressive generation which will be used instead + of encoder_states as keys and values if not None + return_mems: bool, whether to return outputs of all encoder layers + or the last layer only + """ + + encoder_attn_mask = form_attention_mask(encoder_mask, self.diag) + + memory_states = self._get_memory_states(encoder_states, encoder_mems_list, 0) + cached_mems_list = [memory_states] + + for i, layer in enumerate(self.layers): + encoder_states = layer(encoder_states, encoder_attn_mask, memory_states) + memory_states = self._get_memory_states(encoder_states, encoder_mems_list, i + 1) + cached_mems_list.append(memory_states) + + if self.final_layer_norm is not None: + encoder_states = self.final_layer_norm(encoder_states) + memory_states = self._get_memory_states(encoder_states, encoder_mems_list, i + 1) + cached_mems_list.append(memory_states) + + if return_mems: + return cached_mems_list + else: + return cached_mems_list[-1] diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_generators.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_generators.py new file mode 100644 index 0000000000..59fe403941 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_generators.py @@ -0,0 +1,541 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import contextmanager + +import torch + +from nemo.collections.common.parts import NEG_INF, mask_padded_tokens + +__all__ = [ + "GreedySequenceGenerator", + "TopKSequenceGenerator", + "BeamSearchSequenceGenerator", + "BeamSearchSequenceGeneratorWithLanguageModel", +] + + +class GreedySequenceGenerator: + """ + Greedy sequence generator based on the decoder followed by log_softmax. + + Args: + embedding: nn.Module, transforms input_ids into vector embeddings + decoder: nn.Module, takes embeddings and produces hidden_states + log_softmax: nn.Module, takes hidden_states and produces log_probs + which correspond to probability distribution of tokens (ids) + pad: index of padding token in the vocabulary + bos: index of beginning of sequence token in the vocabulary + eos: index of end of sequence token in the vocabulary + max_sequence_length: maximum allowed length for generated sequences + max_delta_length: in case of encoder-decoder generation (e.g. NMT), + forbids generated sequences to be longer than the length of + source sequences plus max_delta_length + batch_size: size of the batch of generated sequences if neither + source nor target starting sequences are provided + """ + + def __init__( + self, + embedding, + decoder, + log_softmax, + pad=0, + bos=1, + eos=2, + max_sequence_length=512, + max_delta_length=20, + batch_size=1, + ): + super().__init__() + self.embedding = embedding + self.decoder = decoder + self.log_softmax = log_softmax + self.pad, self.bos, self.eos = pad, bos, eos + self.max_seq_length = max_sequence_length + self.max_delta_len = max_delta_length + self.batch_size = batch_size + + def _one_step_forward( + self, + decoder_input_ids=None, + encoder_hidden_states=None, + encoder_input_mask=None, + decoder_mems_list=None, + pos=0, + ): + """ + One step of autoregressive output generation. + + Args: + decoder_input_ids: starting sequence of tokens to generate from; + if None, generation will start from a batch of tokens + encoder_hidden_states: output of the encoder for conditional + sequence generation; if None, generator will use unconditional + mode (e.g., language modeling) + encoder_input_mask: input mask used in the encoder + decoder_mems_list: list of size num_layers with cached activations + of sequence (x[1], ..., x[k-1]) for fast generation of x[k] + pos: starting position in positional encoding + """ + + decoder_hidden_states = self.embedding.forward(decoder_input_ids, start_pos=pos) + decoder_input_mask = mask_padded_tokens(decoder_input_ids, self.pad).float() + + if encoder_hidden_states is not None: + decoder_mems_list = self.decoder.forward( + decoder_hidden_states, + decoder_input_mask, + encoder_hidden_states, + encoder_input_mask, + decoder_mems_list, + return_mems=True, + ) + else: + decoder_mems_list = self.decoder.forward( + decoder_hidden_states, decoder_input_mask, decoder_mems_list, return_mems=True + ) + log_probs = self.log_softmax.forward(hidden_states=decoder_mems_list[-1][:, -1:]) + return log_probs, decoder_mems_list + + def _prepare_for_search(self, decoder_input_ids=None, encoder_hidden_states=None): + """ + Helper function which defines starting sequence to begin generating + with and maximum allowed number of tokens to be generated. + """ + + decoder_parameter = next(self.decoder.parameters()) + batch_size = self.batch_size + + # for encoder-decoder generation, maximum length of generated sequence + # is min(max_sequence_length, src_len + max_delta_length) + if encoder_hidden_states is not None: + batch_size, src_len, _ = encoder_hidden_states.size() + max_seq_length = min(self.max_seq_length, src_len + self.max_delta_len) + else: + max_seq_length = self.max_seq_length + + # if no input is provided, start with the batch of tokens + if decoder_input_ids is not None: + tgt = decoder_input_ids + batch_size, tgt_len = decoder_input_ids.size() + else: + tgt = torch.zeros(batch_size, 1).long().fill_(self.bos).to(decoder_parameter.device) + tgt_len = 1 + max_generation_length = max_seq_length - tgt_len + + return tgt, batch_size, max_generation_length + + def _forward(self, decoder_input_ids=None, encoder_hidden_states=None, encoder_input_mask=None): + + tgt, batch_size, max_generation_length = self._prepare_for_search(decoder_input_ids, encoder_hidden_states) + + # pad profile tracks sequences ending with token to replace + # everything after with token + decoder_parameter = next(self.decoder.parameters()) + pad_profile = torch.zeros(batch_size, 1).long().to(decoder_parameter.device) + + decoder_mems_list = None + for i in range(max_generation_length): + + log_probs, decoder_mems_list = self._one_step_forward( + tgt[:, -1:], encoder_hidden_states, encoder_input_mask, decoder_mems_list, i + ) + + next_tokens = torch.argmax(log_probs[:, -1], dim=-1, keepdim=True) + next_tokens = self.pad * pad_profile + next_tokens * (1 - pad_profile) + pad_profile = torch.max(pad_profile, (next_tokens == self.eos).long()) + tgt = torch.cat((tgt, next_tokens), dim=-1) + + # abort generation if all sequences end with + if pad_profile.sum() == batch_size: + break + + return tgt + + def __call__(self, decoder_input_ids=None, encoder_hidden_states=None, encoder_input_mask=None): + with self.as_frozen(): + return self._forward(decoder_input_ids, encoder_hidden_states, encoder_input_mask) + + def freeze(self) -> None: + """Freeze weights of embedding, decoder, and classification layers to prevent memory leak. + """ + for param in self.embedding.parameters(): + param.requires_grad = False + self.embedding.eval() + for param in self.decoder.parameters(): + param.requires_grad = False + self.decoder.eval() + for param in self.log_softmax.parameters(): + param.require_grad = False + self.log_softmax.eval() + + def unfreeze(self) -> None: + """Unfreeze weights of embedding, decoder, and classification layers. + """ + for param in self.embedding.parameters(): + param.requires_grad = True + self.embedding.train() + for param in self.decoder.parameters(): + param.requires_grad = True + self.decoder.train() + for param in self.log_softmax.parameters(): + param.require_grad = True + self.log_softmax.train() + + @contextmanager + def as_frozen(self): + """ + Context manager which temporarily freezes embedding, decoder, and log_softmax modules, + yields control and finally unfreezes the modules. + """ + self.freeze() + + try: + yield + finally: + self.unfreeze() + + +class TopKSequenceGenerator(GreedySequenceGenerator): + """ + Top-k sequence generator based on the decoder followed by log_softmax. + + Args: + *all args of GreedySequenceGenerator class + beam_size: size of the beam (parameter k in top-k) + temperature: temperature of top-k sampling, all logits are divided + by temperature before rescaling. High temperature leads to + uniform distribution, low leads to delta-like distribution. + Kwargs: + all remaining parameters of GreedySequenceGenerator class + """ + + def __init__(self, embedding, decoder, log_softmax, beam_size=1, temperature=1.0, **kwargs): + super().__init__(embedding, decoder, log_softmax, **kwargs) + self.beam_size = beam_size + self.temp = temperature + + # @torch.no_grad() + def _one_step_forward( + self, + decoder_input_ids=None, + encoder_hidden_states=None, + encoder_input_mask=None, + decoder_mems_list=None, + pos=0, + ): + log_probs, decoder_mems_list = super()._one_step_forward( + decoder_input_ids, encoder_hidden_states, encoder_input_mask, decoder_mems_list, pos + ) + + batch_size, seq_len, vocab_size = log_probs.size() + scores, indices = torch.topk(log_probs, self.beam_size, dim=-1) + + rescaled_logexp = torch.zeros_like(log_probs).scatter(-1, indices, scores.div(self.temp).exp()) + probs = rescaled_logexp / rescaled_logexp.norm(1, -1, keepdim=True) + + # We randomly sample next tokens from rescaled probability distribution + # over top-k candidates and return a binary tensor which indicates + # candidates that have been selected. We call this object + # `pseudo_log_probs` as genuine log_probs should have -infs instead of + # 0s and 0s instead of 1s. + ids = torch.multinomial(probs.view(-1, vocab_size), 1).view(-1, seq_len, 1) + pseudo_log_probs = torch.zeros_like(log_probs).scatter(-1, ids, 1.0) + + return pseudo_log_probs, decoder_mems_list + + +class BeamSearchSequenceGenerator(GreedySequenceGenerator): + def __init__(self, embedding, decoder, log_softmax, beam_size=1, len_pen=0, **kwargs): + """ + Beam Search sequence generator based on the decoder followed by + log_softmax. + + Args: + *all args of GreedySequenceGenerator class + beam_size: size of the beam + len_pen: length penalty parameter + Kwargs: + all remaining parameters of GreedySequenceGenerator class + """ + + super().__init__(embedding, decoder, log_softmax, **kwargs) + self.beam_size = beam_size + self.len_pen = len_pen + + @staticmethod + def compute_len_penalty(lengths, alpha): + """Returns length penalty according to https://arxiv.org/pdf/1609.08144.pdf""" + return ((5 + lengths) / 6).pow(alpha) + + def _forward(self, decoder_input_ids=None, encoder_hidden_states=None, encoder_input_mask=None): + tgt, batch_size, max_generation_length = self._prepare_for_search(decoder_input_ids, encoder_hidden_states) + + # generate initial buffer of beam_size prefixes-hypotheses + log_probs, decoder_mems_list = self._one_step_forward(tgt, encoder_hidden_states, encoder_input_mask, None, 0) + scores, prefixes = torch.topk(log_probs.permute(0, 2, 1), self.beam_size, dim=1) + scores, prefixes = scores.view(-1, 1), prefixes.view(-1, 1) + + # repeat init target prefixes and cached memory states beam_size times + prefixes = torch.cat((tgt.repeat(1, self.beam_size).view(-1, 1), prefixes), dim=1) + for j in range(len(decoder_mems_list)): + decoder_mems_list[j] = decoder_mems_list[j].repeat(self.beam_size, 1, 1) + + # repeat source sequence beam_size times for beam search + if encoder_hidden_states is not None: + _, src_length, hidden_size = encoder_hidden_states.size() + encoder_input_mask = encoder_input_mask.repeat(1, self.beam_size).view(-1, src_length) + encoder_hidden_states = encoder_hidden_states.repeat(1, self.beam_size, 1).view( + -1, src_length, hidden_size + ) + else: + hidden_size = decoder_mems_list[0].size(2) + + # pad_profile tracks finished hypotheses to generate only tokens + # if or has been generated + pad_profile = torch.zeros_like(scores).long() + + # prefixes_len tracks lengths of generated hypotheses to perform + # length penalty correction + prefixes_len = torch.zeros_like(scores).fill_(prefixes.size(1) + 1) + + for i in range(max_generation_length): + + # mask all finished hypotheses to exclude them from beam + pad_mask = pad_profile.repeat(1, self.beam_size) + + # generate and score candidates for prefixes continuation + log_probs, decoder_mems_list = self._one_step_forward( + prefixes[:, -1:], encoder_hidden_states, encoder_input_mask, decoder_mems_list, i + 1 + ) + scores_i, prefixes_i = torch.topk(log_probs[:, -1, :], self.beam_size, dim=-1) + + # for all prefixes ending with or replace generated + # continuations with + prefixes_i = self.pad * pad_mask + prefixes_i * (1 - pad_mask) + + # force all hypotheses but one generated from already finished + # hypotheses to have extremely low score, so they will not be + # considered during beam re-ranking + pad_mask[:, 1:] = pad_mask[:, 1:] * NEG_INF + scores = scores + scores_i * (1 - pad_mask).to(scores.dtype) + + # choose top-k hypotheses with length penalty applied + len_penalties = self.compute_len_penalty(prefixes_len, self.len_pen) + scores = scores / len_penalties + scores, indices_i = torch.topk(scores.view(-1, self.beam_size ** 2), self.beam_size, dim=1) + scores = scores.view(-1, 1) * len_penalties + + # select prefixes which correspond to the chosen hypotheses + prefixes = prefixes.unsqueeze(1).repeat(1, self.beam_size, 1) + prefixes = torch.cat((prefixes, prefixes_i.unsqueeze(2)), dim=2) + prefixes = prefixes.view(batch_size, self.beam_size ** 2, -1) + p_len = prefixes.size(2) + prefixes_ids = indices_i.unsqueeze(2).repeat(1, 1, p_len) + prefixes = prefixes.gather(1, prefixes_ids).view(-1, p_len) + + # reshuffle cached decoder memory states to restore the order + # of hypotheses broken after top-k selection + mems_ids = indices_i.unsqueeze(2).unsqueeze(3).repeat(1, 1, p_len - 1, hidden_size) // self.beam_size + for j in range(len(decoder_mems_list)): + decoder_mems_list[j] = ( + decoder_mems_list[j] + .view(-1, self.beam_size, p_len - 1, hidden_size) + .gather(1, mems_ids) + .view(-1, p_len - 1, hidden_size) + ) + + # update prefixes_len and pad_profile + not_eos_pad = prefixes.ne(self.eos) & prefixes.ne(self.pad) + prefixes_len = 1 + not_eos_pad.sum(dim=1, keepdim=True).to(scores.dtype) + pad_profile = (~not_eos_pad[:, -1:]).long() + + # if all hypotheses end with or , interrupt search + if pad_profile.sum() == batch_size * self.beam_size: + break + + # select best performing hypotheses in each element of the batch + len_penalties = self.compute_len_penalty(prefixes_len, self.len_pen) + scores = scores / len_penalties + best_guesses = ( + torch.argmax(scores.view(-1, self.beam_size), dim=1, keepdim=True).repeat(1, prefixes.size(1)).unsqueeze(1) + ) + tgt = prefixes.view(batch_size, self.beam_size, -1).gather(1, best_guesses) + + return tgt.squeeze(1) + + +class BeamSearchSequenceGeneratorWithLanguageModel(GreedySequenceGenerator): + def __init__( + self, embedding, decoder, log_softmax, language_model, beam_size=1, len_pen=0, fusion_coef=0.0, **kwargs + ): + """ + Beam Search sequence generator based on the decoder followed by log_softmax + with external language model fusion. + + Args: + *all args of BeamSearchSequenceGenerator class + language_model: nemo TransformerLMModel + fusion_coef: coefficient before language model score, the resulting score is + score = log P_NMT(y|x) + fusion_coef * log P_LM(y|x) + Kwargs: + all remaining parameters of GreedySequenceGenerator class + """ + + super().__init__(embedding, decoder, log_softmax, **kwargs) + self.language_model = language_model + self.beam_size = beam_size + self.len_pen = len_pen + self.fusion_coef = fusion_coef + + def _one_step_forward( + self, + decoder_input_ids=None, + encoder_hidden_states=None, + encoder_input_mask=None, + decoder_mems_list=None, + lm_mems_list=None, + pos=0, + ): + + nmt_log_probs, decoder_mems_list = super()._one_step_forward( + decoder_input_ids, encoder_hidden_states, encoder_input_mask, decoder_mems_list, pos, + ) + input_mask = mask_padded_tokens(decoder_input_ids, self.pad).float() + lm_hidden_states = self.language_model.encoder.embedding.forward(decoder_input_ids, start_pos=pos) + + lm_mems_list = self.language_model.encoder.encoder.forward( + lm_hidden_states, input_mask, lm_mems_list, return_mems=True, + ) + lm_log_probs = self.language_model.log_softmax.forward(hidden_states=lm_mems_list[-1][:, -1:]) + + log_probs = nmt_log_probs + self.fusion_coef * lm_log_probs + + return log_probs, decoder_mems_list, lm_mems_list + + @staticmethod + def compute_len_penalty(lengths, alpha): + """Returns length penalty according to https://arxiv.org/pdf/1609.08144.pdf""" + return ((5 + lengths) / 6).pow(alpha) + + def _forward(self, decoder_input_ids=None, encoder_hidden_states=None, encoder_input_mask=None): + + tgt, batch_size, max_generation_length = self._prepare_for_search(decoder_input_ids, encoder_hidden_states) + + # generate initial buffer of beam_size prefixes-hypotheses + log_probs, decoder_mems_list, lm_mems_list = self._one_step_forward( + tgt, encoder_hidden_states, encoder_input_mask, None, None, 0 + ) + scores, prefixes = torch.topk(log_probs.permute(0, 2, 1), self.beam_size, dim=1) + scores, prefixes = scores.view(-1, 1), prefixes.view(-1, 1) + + # repeat init target prefixes and cached memory states beam_size times + prefixes = torch.cat((tgt.repeat(1, self.beam_size).view(-1, 1), prefixes), dim=1) + for j in range(len(decoder_mems_list)): + decoder_mems_list[j] = decoder_mems_list[j].repeat(self.beam_size, 1, 1) + for j in range(len(lm_mems_list)): + lm_mems_list[j] = lm_mems_list[j].repeat(self.beam_size, 1, 1) + + # repeat source sequence beam_size times for beam search + if encoder_hidden_states is not None: + _, src_length, hidden_size = encoder_hidden_states.size() + encoder_input_mask = encoder_input_mask.repeat(1, self.beam_size).view(-1, src_length) + encoder_hidden_states = encoder_hidden_states.repeat(1, self.beam_size, 1).view( + -1, src_length, hidden_size + ) + else: + hidden_size = decoder_mems_list[0].size(2) + lm_hidden_size = lm_mems_list[0].size(2) + + # pad_profile tracks finished hypotheses to generate only tokens + # if or has been generated + pad_profile = torch.zeros_like(scores).long() + + # prefixes_len tracks lengths of generated hypotheses to perform + # length penalty correction + prefixes_len = torch.zeros_like(scores).fill_(prefixes.size(1) + 1) + + for i in range(max_generation_length): + + # mask all finished hypotheses to exclude them from beam + pad_mask = pad_profile.repeat(1, self.beam_size) + + # generate and score candidates for prefixes continuation + log_probs, decoder_mems_list, lm_mems_list = self._one_step_forward( + prefixes[:, -1:], encoder_hidden_states, encoder_input_mask, decoder_mems_list, lm_mems_list, i + 1 + ) + scores_i, prefixes_i = torch.topk(log_probs[:, -1, :], self.beam_size, dim=-1) + + # for all prefixes ending with or replace generated + # continuations with + prefixes_i = self.pad * pad_mask + prefixes_i * (1 - pad_mask) + + # force all hypotheses but one generated from already finished + # hypotheses to have extremely low score, so they will not be + # considered during beam re-ranking + pad_mask[:, 1:] = pad_mask[:, 1:] * NEG_INF + scores = scores + scores_i * (1 - pad_mask).to(scores.dtype) + + # choose top-k hypotheses with length penalty applied + len_penalties = self.compute_len_penalty(prefixes_len, self.len_pen) + scores = scores / len_penalties + scores, indices_i = torch.topk(scores.view(-1, self.beam_size ** 2), self.beam_size, dim=1) + scores = scores.view(-1, 1) * len_penalties + + # select prefixes which correspond to the chosen hypotheses + prefixes = prefixes.unsqueeze(1).repeat(1, self.beam_size, 1) + prefixes = torch.cat((prefixes, prefixes_i.unsqueeze(2)), dim=2) + prefixes = prefixes.view(batch_size, self.beam_size ** 2, -1) + p_len = prefixes.size(2) + prefixes_ids = indices_i.unsqueeze(2).repeat(1, 1, p_len) + prefixes = prefixes.gather(1, prefixes_ids).view(-1, p_len) + + # reshuffle cached decoder memory states to restore the order + # of hypotheses broken after top-k selection + mems_ids = indices_i.unsqueeze(2).unsqueeze(3).repeat(1, 1, p_len - 1, hidden_size) // self.beam_size + for j in range(len(decoder_mems_list)): + decoder_mems_list[j] = ( + decoder_mems_list[j] + .view(-1, self.beam_size, p_len - 1, hidden_size) + .gather(1, mems_ids) + .view(-1, p_len - 1, hidden_size) + ) + lm_mems_ids = indices_i.unsqueeze(2).unsqueeze(3).repeat(1, 1, p_len - 1, lm_hidden_size) // self.beam_size + for j in range(len(lm_mems_list)): + lm_mems_list[j] = ( + lm_mems_list[j] + .view(-1, self.beam_size, p_len - 1, lm_hidden_size) + .gather(1, lm_mems_ids) + .view(-1, p_len - 1, lm_hidden_size) + ) + + # update prefixes_len and pad_profile + not_eos_pad = prefixes.ne(self.eos) & prefixes.ne(self.pad) + prefixes_len = 1 + not_eos_pad.sum(dim=1, keepdim=True).to(scores.dtype) + pad_profile = (~not_eos_pad[:, -1:]).long() + + # if all hypotheses end with or , interrupt search + if pad_profile.sum() == batch_size * self.beam_size: + break + + # select best performing hypotheses in each element of the batch + len_penalties = self.compute_len_penalty(prefixes_len, self.len_pen) + scores = scores / len_penalties + best_guesses = ( + torch.argmax(scores.view(-1, self.beam_size), dim=1, keepdim=True).repeat(1, prefixes.size(1)).unsqueeze(1) + ) + tgt = prefixes.view(batch_size, self.beam_size, -1).gather(1, best_guesses) + + return tgt.squeeze(1) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_modules.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_modules.py new file mode 100644 index 0000000000..4f5fe87fcc --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_modules.py @@ -0,0 +1,210 @@ +# Copyright 2018 The Google AI Language Team Authors and +# The HuggingFace Inc. team. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from dataclasses import dataclass + +import torch +from omegaconf.omegaconf import MISSING +from torch import nn +from torch.nn.functional import gelu + +__all__ = ["TransformerEmbedding"] + + +class FixedPositionalEncoding(nn.Module): + """ + Fixed positional encoding (embedding layer) from sine and cosine functions + of different frequencies according to https://arxiv.org/abs/1706.03762 + + Args: + hidden_size: size of the embeddings in the model, also known as d_model + max_sequence_length: maximum allowed length of the input sequence + """ + + def __init__(self, hidden_size, max_sequence_length=512): + super().__init__() + + pos_enc = torch.zeros(max_sequence_length, hidden_size) + position = torch.arange(0.0, max_sequence_length).unsqueeze(1) + coef = -math.log(10000.0) / hidden_size + div_term = torch.exp(coef * torch.arange(0.0, hidden_size, 2)) + pos_enc[:, 0::2] = torch.sin(position * div_term) + pos_enc[:, 1::2] = torch.cos(position * div_term) + pos_enc.div_(math.sqrt(hidden_size)) + self.register_buffer('pos_enc', pos_enc) + + def forward(self, position_ids): + return torch.embedding(self.pos_enc, position_ids) + + +class TransformerEmbedding(nn.Module): + """ + Embedding from token and position embeddings. + Optionally add token_type embedding (e.g. type of the sentence in BERT). + + Args: + vocab_size: size of the vocabulary + hidden_size: size of the embeddings in the model, also known as d_model + max_sequence_length: maximum allowed length of the input sequence + num_token_types: number of different token types + (e.g. tokens of sentence A and tokens of sentence B in BERT) + embedding_dropout: probability of dropout applied to embeddings + learn_positional_encodings: whether to learn positional encodings or + use fixed (sine-cosine) ones + """ + + def __init__( + self, + vocab_size, + hidden_size, + max_sequence_length=512, + num_token_types=2, + embedding_dropout=0.0, + learn_positional_encodings=False, + ): + super().__init__() + + self.max_sequence_length = max_sequence_length + self.token_embedding = nn.Embedding(vocab_size, hidden_size, padding_idx=0) + if learn_positional_encodings: + self.position_embedding = nn.Embedding(max_sequence_length, hidden_size) + else: + self.position_embedding = FixedPositionalEncoding(hidden_size, max_sequence_length) + if num_token_types > 0: + self.token_type_embedding = nn.Embedding(num_token_types, hidden_size) + self.layer_norm = nn.LayerNorm(hidden_size, eps=1e-5) + self.dropout = nn.Dropout(embedding_dropout) + + def forward(self, input_ids, token_type_ids=None, start_pos=0): + seq_length = input_ids.size(1) + if seq_length > self.max_sequence_length: + raise ValueError( + f"Input sequence is longer than maximum allowed sequence length for positional encoding. " + f"Got {seq_length} and {self.max_sequence_length}" + ) + position_ids = torch.arange( + start=start_pos, end=start_pos + seq_length, dtype=torch.long, device=input_ids.device + ) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + + token_embeddings = self.token_embedding(input_ids) + position_embeddings = self.position_embedding(position_ids) + embeddings = token_embeddings + position_embeddings + + if token_type_ids is not None: + token_type_embeddings = self.token_type_embedding(token_type_ids) + embeddings = embeddings + token_type_embeddings + + embeddings = self.layer_norm(embeddings) + embeddings = self.dropout(embeddings) + + return embeddings + + +class MultiHeadAttention(nn.Module): + """ + Multi-head scaled dot-product attention layer. + + Args: + hidden_size: size of the embeddings in the model, also known as d_model + num_attention_heads: number of heads in multi-head attention + attn_score_dropout: probability of dropout applied to attention scores + attn_layer_dropout: probability of dropout applied to the output of the + whole layer, but before layer normalization + """ + + def __init__(self, hidden_size, num_attention_heads, attn_score_dropout=0.0, attn_layer_dropout=0.0): + super().__init__() + if hidden_size % num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number " + "of attention heads (%d)" % (hidden_size, num_attention_heads) + ) + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.attn_head_size = int(hidden_size / num_attention_heads) + self.attn_scale = math.sqrt(math.sqrt(self.attn_head_size)) + + self.query_net = nn.Linear(hidden_size, hidden_size) + self.key_net = nn.Linear(hidden_size, hidden_size) + self.value_net = nn.Linear(hidden_size, hidden_size) + self.out_projection = nn.Linear(hidden_size, hidden_size) + + self.attn_dropout = nn.Dropout(attn_score_dropout) + self.layer_dropout = nn.Dropout(attn_layer_dropout) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attn_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, queries, keys, values, attention_mask): + + # attention_mask is needed to hide the tokens which correspond to [PAD] + # in the case of BERT, or to hide the future tokens in the case of + # vanilla language modeling and translation + query = self.query_net(queries) + key = self.key_net(keys) + value = self.value_net(values) + query = self.transpose_for_scores(query) / self.attn_scale + key = self.transpose_for_scores(key) / self.attn_scale + value = self.transpose_for_scores(value) + + # for numerical stability we pre-divide query and key by sqrt(sqrt(d)) + attention_scores = torch.matmul(query, key.transpose(-1, -2)) + if attention_mask is not None: + attention_scores = attention_scores + attention_mask.to(attention_scores.dtype) + attention_probs = torch.softmax(attention_scores, dim=-1) + attention_probs = self.attn_dropout(attention_probs) + + context = torch.matmul(attention_probs, value) + context = context.permute(0, 2, 1, 3).contiguous() + new_context_shape = context.size()[:-2] + (self.hidden_size,) + context = context.view(*new_context_shape) + + # output projection + output_states = self.out_projection(context) + output_states = self.layer_dropout(output_states) + return output_states + + +class PositionWiseFF(nn.Module): + """ + Position-wise feed-forward network of Transformer block. + + Args: + hidden_size: size of the embeddings in the model, also known as d_model + inner_size: number of neurons in the intermediate part of feed-forward + net, usually is (4-8 x hidden_size) in the papers + ffn_dropout: probability of dropout applied to net output + hidden_act: activation function used between two linear layers + """ + + def __init__(self, hidden_size, inner_size, ffn_dropout=0.0, hidden_act="relu"): + super().__init__() + self.dense_in = nn.Linear(hidden_size, inner_size) + self.dense_out = nn.Linear(inner_size, hidden_size) + self.layer_dropout = nn.Dropout(ffn_dropout) + ACT2FN = {"gelu": gelu, "relu": torch.relu} + self.act_fn = ACT2FN[hidden_act] + + def forward(self, hidden_states): + output_states = self.dense_in(hidden_states) + output_states = self.act_fn(output_states) + output_states = self.dense_out(output_states) + output_states = self.layer_dropout(output_states) + return output_states diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_utils.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_utils.py new file mode 100644 index 0000000000..83fcaed8c6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/common/transformer/transformer_utils.py @@ -0,0 +1,118 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Optional, Union + +from omegaconf.dictconfig import DictConfig + +from nemo.collections.nlp.modules.common.huggingface.huggingface_decoder import HuggingFaceDecoderModule +from nemo.collections.nlp.modules.common.huggingface.huggingface_encoder import HuggingFaceEncoderModule +from nemo.collections.nlp.modules.common.transformer.transformer import TransformerDecoderNM, TransformerEncoderNM + + +def get_nemo_transformer( + model_name: Optional[str] = None, + pretrained: bool = False, + config_dict: Optional[Union[dict, DictConfig]] = None, + encoder: bool = True, + pre_ln_final_layer_norm: bool = True, +) -> Union[TransformerEncoderNM, TransformerDecoderNM]: + """Returns NeMo transformer. + The following configurations are mandatory: + vocab_size: int + hidden_size: int + num_layers: int + inner_size: int + and must be specified if using config_dict. + + Args: + model_name (Optional[str]): model name to download from NGC + pretrained: (bool): False will instantiate the named model architecture with random weights. + config_dict (Optional[dict], optional): model configuration parameters. Defaults to None. + config_file (Optional[str], optional): path to json file containing model configuration. Defaults to None. + checkpoint_file (Optional[str], optional): load weights from path to local checkpoint. Defaults to None. + encoder (bool, optional): True will use EncoderTransformerNM, False will use DecoderTransformerNM. Defaults to True. + """ + if model_name is not None: + raise ValueError(f'NeMo transformers cannot be loaded from NGC yet. model_name should be None') + + if pretrained: + raise ValueError(f'NeMo transformers cannot be loaded from NGC yet. pretrained should be False') + + cfg = None + + if not pretrained: + assert ( + config_dict.get('vocab_size') is not None + and config_dict.get('hidden_size') is not None + and config_dict.get('num_layers') is not None + and config_dict.get('inner_size') is not None + ), f'Using config_dict: {config_dict}. vocab_size, hidden_size, num_layers, and inner_size must are mandatory arguments' + + cfg = config_dict + + if encoder: + model = TransformerEncoderNM( + vocab_size=cfg.get('vocab_size'), + hidden_size=cfg.get('hidden_size'), + num_layers=cfg.get('num_layers'), + inner_size=cfg.get('inner_size'), + max_sequence_length=cfg.get('max_sequence_length', 512), + embedding_dropout=cfg.get('embedding_dropout', 0.0), + learn_positional_encodings=cfg.get('learn_positional_encodings', False), + num_attention_heads=cfg.get('num_attention_heads'), + ffn_dropout=cfg.get('ffn_dropout', 0.0), + attn_score_dropout=cfg.get('attn_score_dropout', 0.0), + attn_layer_dropout=cfg.get('attn_layer_dropout', 0.0), + hidden_act=cfg.get('hidden_act', 'relu'), + mask_future=cfg.get('mask_future', False), + pre_ln=cfg.get('pre_ln', False), + pre_ln_final_layer_norm=pre_ln_final_layer_norm, + num_token_types=cfg.get('num_token_types', 2), + ) + else: + model = TransformerDecoderNM( + vocab_size=cfg.get('vocab_size'), + hidden_size=cfg.get('hidden_size'), + num_layers=cfg.get('num_layers'), + inner_size=cfg.get('inner_size'), + max_sequence_length=cfg.get('max_sequence_length', 512), + embedding_dropout=cfg.get('embedding_dropout', 0.0), + learn_positional_encodings=cfg.get('learn_positional_encodings', False), + num_attention_heads=cfg.get('num_attention_heads'), + ffn_dropout=cfg.get('ffn_dropout', 0.0), + attn_score_dropout=cfg.get('attn_score_dropout', 0.0), + attn_layer_dropout=cfg.get('attn_layer_dropout', 0.0), + hidden_act=cfg.get('hidden_act', 'relu'), + pre_ln=cfg.get('pre_ln', False), + pre_ln_final_layer_norm=pre_ln_final_layer_norm, + num_token_types=cfg.get('num_token_types', 2), + ) + + return model + + +def get_huggingface_transformer( + model_name: Optional[str] = None, + pretrained: bool = False, + config_dict: Optional[Union[dict, DictConfig]] = None, + encoder: bool = True, +) -> Union[HuggingFaceEncoderModule, HuggingFaceDecoderModule]: + if encoder: + model = HuggingFaceEncoderModule(model_name, pretrained, config_dict) + else: + model = HuggingFaceDecoderModule(model_name, pretrained, config_dict) + + return model diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/__init__.py new file mode 100644 index 0000000000..9e32500719 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/sgd_decoder.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/sgd_decoder.py new file mode 100644 index 0000000000..2ffe533018 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/sgd_decoder.py @@ -0,0 +1,195 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional + +import torch +from torch import nn as nn + +from nemo.core.classes import NeuralModule, typecheck +from nemo.core.neural_types import LogitsType, NeuralType + +__all__ = ['SGDDecoder'] + + +class LogitsQA(nn.Module): + def __init__(self, num_classes: int, embedding_dim: int): + """Get logits for elements by conditioning on input embedding. + Args: + num_classes: An int containing the number of classes for which logits are to be generated. + embedding_dim: hidden size of the BERT + + Returns: + A tensor of shape (batch_size, num_classes) containing the logits. + """ + super().__init__() + self.num_classes = num_classes + self.utterance_proj = nn.Linear(embedding_dim, embedding_dim) + self.activation = nn.functional.gelu + + self.layer1 = nn.Linear(embedding_dim, num_classes) + + def forward(self, encoded_utterance): + """ + Args: + encoded_utterance: [CLS] token hidden state from BERT encoding of the utterance + """ + + # Project the utterance embeddings. + utterance_embedding = self.utterance_proj(encoded_utterance) + utterance_embedding = self.activation(utterance_embedding) + + logits = self.layer1(utterance_embedding) + return logits + + +class SGDDecoder(NeuralModule): + """ + SGDDecoder + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """ + Returns definitions of module output ports. + """ + return { + "logit_intent_status": NeuralType(('B', 'T'), LogitsType()), #'B' + "logit_req_slot_status": NeuralType(('B', 'T'), LogitsType()), #'B' + "logit_cat_slot_status": NeuralType(('B', 'T'), LogitsType()), + "logit_cat_slot_value_status": NeuralType(('B', 'T'), LogitsType()), #'B' + "logit_noncat_slot_status": NeuralType(('B', 'T'), LogitsType()), + "logit_spans": NeuralType(('B', 'T', 'D'), LogitsType()), + } + + def __init__(self, embedding_dim: int) -> None: + + """Get logits for elements by conditioning on utterance embedding. + + Args: + embedding_dim: hidden size of the BERT + """ + super().__init__() + + projection_module = LogitsQA + + self.intent_layer = projection_module(1, embedding_dim) + self.requested_slots_layer = projection_module(1, embedding_dim) + + self.cat_slot_value_layer = projection_module(1, embedding_dim) + + # Slot status values: none, dontcare, active. + self.slot_status_layer = projection_module(3, embedding_dim) + + # dim 2 for non_categorical slot - to represent start and end position + self.noncat_layer1 = nn.Linear(embedding_dim, embedding_dim) + self.noncat_activation = nn.functional.gelu + self.noncat_layer2 = nn.Linear(embedding_dim, 2) + + @typecheck() + def forward(self, encoded_utterance, token_embeddings, utterance_mask): + """ + Args: + encoded_utterance: [CLS] token hidden state from BERT encoding of the utterance + token_embeddings: token embeddings from BERT encoding of the utterance + utterance_mask: utterance mask wiht 0 for padding + """ + _, _ = encoded_utterance.size() + logit_intent_status = self._get_intents(encoded_utterance) + + logit_req_slot_status = self._get_requested_slots(encoded_utterance) + + logit_cat_slot_status, logit_cat_slot_value_status = self._get_categorical_slot_goals(encoded_utterance) + + (logit_noncat_slot_status, logit_spans) = self._get_noncategorical_slot_goals( + encoded_utterance=encoded_utterance, utterance_mask=utterance_mask, token_embeddings=token_embeddings + ) + + return ( + logit_intent_status, + logit_req_slot_status, + logit_cat_slot_status, + logit_cat_slot_value_status, + logit_noncat_slot_status, + logit_spans, + ) + + def _get_intents(self, encoded_utterance): + """Obtain logits for intents. + Args: + encoded_utterance: representation of utterance + """ + logits = self.intent_layer(encoded_utterance=encoded_utterance,) + return logits + + def _get_requested_slots(self, encoded_utterance): + """Obtain logits for requested slots. + Args: + encoded_utterance: representation of utterance + """ + + logits = self.requested_slots_layer(encoded_utterance=encoded_utterance) + return logits + + def _get_categorical_slot_goals(self, encoded_utterance): + """ + Obtain logits for status and values for categorical slots + Slot status values: none, dontcare, active + Args: + encoded_utterance: representation of utterance + """ + + # Predict the status of all categorical slots. + status_logits = self.slot_status_layer(encoded_utterance=encoded_utterance) + + value_status_logits = self.cat_slot_value_layer(encoded_utterance=encoded_utterance) + return status_logits, value_status_logits + + def _get_noncategorical_slot_goals(self, encoded_utterance, utterance_mask, token_embeddings): + """ + Obtain logits for status and slot spans for non-categorical slots. + Slot status values: none, dontcare, active + Args: + encoded_utterance: [CLS] token hidden state from BERT encoding of the utterance + utterance_mask: utterance mask wiht 0 for padding + token_embeddings: token embeddings from BERT encoding of the utterance + """ + status_logits = self.slot_status_layer(encoded_utterance=encoded_utterance) + + # Project the combined embeddings to obtain logits, Shape: (batch_size, max_num_slots, max_num_tokens, 2) + span_logits = self.noncat_layer1(token_embeddings) + span_logits = self.noncat_activation(span_logits) + span_logits = self.noncat_layer2(span_logits) + + # Mask out invalid logits for padded tokens. + utterance_mask = utterance_mask.to(bool) # Shape: (batch_size, max_num_tokens). + repeated_utterance_mask = utterance_mask.unsqueeze(-1) + negative_logits = (torch.finfo(span_logits.dtype).max * -0.7) * torch.ones( + span_logits.size(), device=span_logits.get_device(), dtype=span_logits.dtype + ) + + span_logits = torch.where(repeated_utterance_mask, span_logits, negative_logits) + + return status_logits, span_logits + + def _get_negative_logits(self, logits): + """Returns tensor with negative logits that will be used to mask out unused values for a particular service + Args: + logits: logits whose shape and type will be used to create negative tensor + """ + negative_logits = (torch.finfo(logits.dtype).max * -0.7) * torch.ones( + logits.size(), dtype=logits.dtype, device=logits.get_device() + ) + return negative_logits diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/sgd_encoder.py b/models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/sgd_encoder.py new file mode 100644 index 0000000000..948a806ad3 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/modules/dialogue_state_tracking/sgd_encoder.py @@ -0,0 +1,77 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional + +from torch import nn as nn + +from nemo.collections.nlp.modules.common.classifier import Classifier +from nemo.core.classes import typecheck +from nemo.core.neural_types import ChannelType, LogitsType, NeuralType + +__all__ = ['SGDEncoder'] + +ACT2FN = {"tanh": nn.functional.tanh, "relu": nn.functional.relu} + + +class SGDEncoder(Classifier): + """ + Neural module which encodes BERT hidden states + """ + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """ + Returns definitions of module output ports. + """ + + return { + "logits": NeuralType(('B', 'T'), LogitsType()), + 'hidden_states': NeuralType(('B', 'T', 'C'), ChannelType()), + } + + def __init__( + self, hidden_size: int, activation: str = 'tanh', dropout: float = 0.0, use_transformer_init: bool = True, + ) -> None: + + """ + Args: + hidden_size: hidden size of the BERT model + activation: activation function applied + dropout: dropout ratio + use_transformer_init: use transformer initialization + """ + super().__init__(hidden_size=hidden_size, dropout=dropout) + self.fc = nn.Linear(hidden_size, hidden_size) + + if activation not in ACT2FN: + raise ValueError(f'{activation} is not in supported ' + '{ACTIVATIONS_F.keys()}') + + self.activation = ACT2FN[activation] + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + self.post_init(use_transformer_init=use_transformer_init) + + @typecheck() + def forward(self, hidden_states): + """ + Args: + hidden_states: bert output hidden states + """ + first_token_hidden_states = hidden_states[:, 0] + logits = self.fc(first_token_hidden_states) + logits = self.activation(logits) + logits = self.dropout1(logits) + return logits, self.dropout2(hidden_states) diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/parts/__init__.py b/models/tacotron2/NeMo/nemo/collections/nlp/parts/__init__.py new file mode 100644 index 0000000000..e60fccbe20 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/parts/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.collections.nlp.parts.utils_funcs import list2str, tensor2list diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/parts/nlp_overrides.py b/models/tacotron2/NeMo/nemo/collections/nlp/parts/nlp_overrides.py new file mode 100644 index 0000000000..4c268ce7d8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/parts/nlp_overrides.py @@ -0,0 +1,233 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Any, Dict, List, Optional, Union + +import torch +from megatron import mpu +from megatron.checkpointing import get_checkpoint_version, set_checkpoint_version +from megatron.initialize import _set_random_seed +from pytorch_lightning.core.lightning import LightningModule +from pytorch_lightning.overrides import LightningDistributedModule +from pytorch_lightning.plugins.environments.cluster_environment import ClusterEnvironment +from pytorch_lightning.plugins.training_type.ddp import DDPPlugin +from pytorch_lightning.trainer.connectors.checkpoint_connector import CheckpointConnector +from pytorch_lightning.utilities import rank_zero_warn +from pytorch_lightning.utilities.cloud_io import atomic_save +from torch.nn.parallel import DistributedDataParallel + +from nemo.collections.nlp.modules.common.megatron.megatron_bert import MegatronBertEncoder +from nemo.utils import AppState, logging + + +class NLPDDPPlugin(DDPPlugin): + """ DDP plugin for Pytorch Lightning. Needed to customize DDP for model parallel models. + """ + + distributed_backend = "ddp" + + def __init__( + self, + parallel_devices: Optional[List[torch.device]] = None, + num_nodes: int = 1, + cluster_environment: ClusterEnvironment = None, + sync_batchnorm: bool = False, + **kwargs: Union[Any, Dict[str, Any]], + ) -> None: + super().__init__(parallel_devices, num_nodes, cluster_environment, sync_batchnorm, **kwargs) + + def init_ddp_connection(self, global_rank: int = None, world_size: int = None) -> None: + # call PTL init ddp + super().init_ddp_connection() + + # init model parallel + app_state = AppState() + + if app_state.model_parallel_size is not None: + + if isinstance(self.lightning_module.bert_model, MegatronBertEncoder): + + if app_state.model_parallel_group is None: + self.init_model_parallel(app_state.global_rank, app_state.world_size) + + def start_training(self, trainer: 'Trainer') -> None: + """ PTL Hook that is called after DPP is initialized. """ + + if isinstance(self.lightning_module.bert_model, MegatronBertEncoder): + app_state = AppState() + if app_state.model_parallel_size is not None: + # mpu grad clipping needs parameters to have the attribute model_parallel + parameters = self.lightning_module.parameters() + for p in parameters: + if not hasattr(p, 'model_parallel'): + p.model_parallel = False + + # TODO: figure out how to override clip gradients again + # Update PTL trainer to use our _clip_gradients + # self._trainer.accelerator_backend._clip_gradients = self._clip_gradients + + if get_checkpoint_version(): + # Restored from .nemo, checkpoint_version will already be set + pass + elif trainer.resume_from_checkpoint is not None: + # PTL auto-resuming, need to update checkpoint name + # update path based on model parallel rank + filepath = trainer.resume_from_checkpoint + dirname = os.path.dirname(os.path.dirname(filepath)) + basename = os.path.basename(filepath) + filepath = f'{dirname}/mp_rank_{app_state.model_parallel_rank:02d}/{basename}' + trainer.resume_from_checkpoint = filepath + logging.info(f'Resuming training from checkpoint {trainer.resume_from_checkpoint}') + # need to set checkpoint version for megatron-lm + checkpoint_version = torch.load(trainer.resume_from_checkpoint).get('checkpoint_version', None) + if checkpoint_version is not None: + set_checkpoint_version(checkpoint_version) + else: + logging.warning('Megatron-lm checkpoint version not found. Setting checkpoint_version to 0.') + set_checkpoint_version(0) + else: + logging.info( + f"Restoring from pretrained model parallel checkpoint: {self.lightning_module.bert_model._restore_path}" + ) + self.lightning_module.bert_model.restore_weights(self.lightning_module.bert_model._restore_path) + + self.lightning_module.register_megatron_checkpoint_version() + + return super().start_training(trainer) + + def start_testing(self, trainer: 'Trainer') -> None: + """ PTL Hook that is called after DPP is initialized. """ + app_state = AppState() + + if app_state.model_parallel_size is not None: + + if isinstance(self.lightning_module.bert_model, MegatronBertEncoder): + # check megatron checkpoint version + checkpoint_version = get_checkpoint_version() + if checkpoint_version is None: + raise ValueError("Unable to find megatron checkpoint version.") + + return super().start_testing(trainer) + + def configure_ddp(self): + """ Override LightningModule ddp if using model parallel. + Sets find_unused_parameters to True. + """ + + app_state = AppState() + + if app_state.model_parallel_size is not None: + logging.info(f"Configuring DDP for model parallelism.") + + # With model parallelism, multiple GPUs form a large "logical GPU" + # this means that data parallel groups span multiple GPUs + # and are non-trivial + device_ids = self.determine_ddp_device_ids() + self._model = DistributedDataParallel( + LightningDistributedModule(self.model), + device_ids=device_ids, + output_device=device_ids[0], + process_group=app_state.data_parallel_group, + **self._ddp_kwargs, + ) + + else: + super().configure_ddp() + + def init_model_parallel(self, global_rank: int, world_size: int) -> None: + """ Initializes Megatron-LM model parallel if using model parallelism. + + Args: + global_rank (int): the global process index. + world_size (int): the total number of GPUs, num_nodes * num_gpus + is_slurm_managing_tasks (bool, optional): is the cluster managed by SLURM. + """ + app_state = AppState() + + # we initialize megatron-lm model parallel and data parallel groups + # after initializing DDP with PTL. + if app_state.model_parallel_size is not None: + if torch.distributed.is_initialized(): + mpu.initialize_model_parallel(app_state.model_parallel_size) + app_state.model_parallel_group = mpu.get_model_parallel_group() + app_state.data_parallel_group = mpu.get_data_parallel_group() + app_state.model_parallel_rank = mpu.get_tensor_model_parallel_rank() + app_state.data_parallel_rank = mpu.get_data_parallel_rank() + app_state.data_parallel_size = mpu.get_data_parallel_world_size() + logging.info(f'mp_rank: {app_state.model_parallel_rank}') + logging.info(f'dp_rank: {app_state.data_parallel_rank}') + # TODO: get random seed from PTL + seed = os.environ.get("PL_GLOBAL_SEED", 1234) + # random seed must be set for megatron model parallel init + _set_random_seed(seed) + + @property + def distributed_sampler_kwargs(self): + app_state = AppState() + if app_state.model_parallel_size is not None: + # When using model parallel, data parallel groups are non-trivial and they + # correspond to the logical GPUs. This means that the GPUs that form a + # single logical GPU all need to get the same batch of data. + distributed_sampler_kwargs = dict( + num_replicas=app_state.data_parallel_size, rank=app_state.data_parallel_rank + ) + return distributed_sampler_kwargs + + else: + return super().distributed_sampler_kwargs + + +class NLPCheckpointConnector(CheckpointConnector): + """ Override PTL CheckpointConnector to support model parallel checkpoints from Megatron-LM. + """ + + def __init__(self, trainer): + super().__init__(trainer) + + def save_checkpoint(self, filepath, weights_only: bool): + """Slightly modified version of PyTorch Lightning's save_checkpoint. + + Args: + filepath ([str]): [description] + weights_only (bool): [description] + + Returns: + [type]: [description] + """ + app_state = AppState() + if app_state.model_parallel_size is not None: + # filepath needs to be updated to include mp_rank + dirname = os.path.dirname(filepath) + basename = os.path.basename(filepath) + filepath = f'{dirname}/mp_rank_{app_state.model_parallel_rank:02d}/{basename}' + + # dump states as a checkpoint dictionary object + checkpoint = self.dump_checkpoint(weights_only) + + # each model parallel rank needs to save a copy of its model + if app_state.data_parallel_rank == 0: + # write the checkpoint dictionary on the file + if self.trainer.accelerator_backend: + checkpoint = self.trainer.accelerator_backend.on_save(checkpoint) + try: + atomic_save(checkpoint, filepath) + except AttributeError as err: + if LightningModule.CHECKPOINT_HYPER_PARAMS_KEY in checkpoint: + del checkpoint[LightningModule.CHECKPOINT_HYPER_PARAMS_KEY] + rank_zero_warn( + 'Warning, `hyper_parameters` dropped from checkpoint.' f' An attribute is not picklable {err}' + ) + atomic_save(checkpoint, filepath) + return None diff --git a/models/tacotron2/NeMo/nemo/collections/nlp/parts/utils_funcs.py b/models/tacotron2/NeMo/nemo/collections/nlp/parts/utils_funcs.py new file mode 100644 index 0000000000..cc0091eee9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/nlp/parts/utils_funcs.py @@ -0,0 +1,130 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = ['list2str', 'tensor2list', 'plot_confusion_matrix', 'get_classification_report'] + +import os +import time +from typing import Dict, List, Union + +import numpy as np +from matplotlib import pyplot as plt +from sklearn.metrics import classification_report, confusion_matrix +from torch import Tensor + +from nemo.utils import logging + + +def list2str(l: List[int]) -> str: + """ Converts list to a string""" + return ' '.join([str(x) for x in l]) + + +def tensor2list(tensor: Tensor) -> List[Union[int, float]]: + """ Converts tensor to a list """ + return tensor.detach().cpu().tolist() + + +def plot_confusion_matrix( + labels: List[int], + preds: List[int], + graph_fold: str, + label_ids: Dict[str, int] = None, + normalize: bool = False, + prefix: str = '', +): + ''' + Plot confusion matrix. + Args: + labels: ground truth labels + preds: model predictions + graph_fold: path to a folder to store confusion matrix plot + label_ids: str label to id map, for example: {'O': 0, 'LOC': 1} + normalize: whether to normalize confusion matrix + prefix: prefix for the plot name + ''' + if label_ids is None: + _plot_confusion_matrix(labels, preds, graph_fold) + + else: + # remove labels from label_ids that don't appear in the dev set + used_labels = set(labels) | set(preds) + label_ids = {k: label_ids[k] for k, v in label_ids.items() if v in used_labels} + + ids_to_labels = {label_ids[k]: k for k in label_ids} + classes = [ids_to_labels[id] for id in sorted(label_ids.values())] + + title = 'Confusion_matrix' + cm = confusion_matrix(labels, preds) + if normalize: + sums = cm.sum(axis=1)[:, np.newaxis] + sums = np.where(sums == 0, 1, sums) + cm = cm.astype('float') / sums + title = 'Normalized_' + title + + fig = plt.figure() + ax = fig.add_subplot(111) + + cax = ax.matshow(cm) + + ax.set_xticks(np.arange(-1, len(classes))) + ax.set_yticks(np.arange(-1, len(classes))) + ax.set_xticklabels([''] + classes, rotation=90) + ax.set_yticklabels([''] + classes) + ax.set_ylabel('True') + ax.set_xlabel('Predicted') + + os.makedirs(graph_fold, exist_ok=True) + fig.colorbar(cax) + + title = (prefix + title).strip() + fig_name = os.path.join(graph_fold, title + '_' + time.strftime('%Y%m%d-%H%M%S')) + plt.savefig(fig_name) + logging.info(f'Confusion matrix saved to {fig_name}') + + +def _plot_confusion_matrix(labels: List[int], preds: List[int], graph_fold: str): + """ + Plot confusion matrix + Args: + labels: ground truth labels + preds: model predictions + graph_fold: path to a folder to store confusion matrix plot + """ + cm = confusion_matrix(labels, preds) + logging.info(f'Confusion matrix:\n{cm}') + fig = plt.figure() + ax = fig.add_subplot(111) + cax = ax.matshow(cm) + plt.title('Confusion matrix of the classifier') + fig.colorbar(cax) + plt.xlabel('Predicted') + plt.ylabel('True') + os.makedirs(graph_fold, exist_ok=True) + plt.savefig(os.path.join(graph_fold, time.strftime('%Y%m%d-%H%M%S'))) + + +def get_classification_report(labels, preds, label_ids, output_dict=False): + """ + Returns classification report + """ + # remove labels from label_ids that don't appear in predictions or ground truths + used_labels = set(labels) | set(preds) + labels_names = [ + k + ' (label id: ' + str(v) + ')' + for k, v in sorted(label_ids.items(), key=lambda item: item[1]) + if v in used_labels + ] + + return classification_report(labels, preds, target_names=labels_names, digits=4, output_dict=output_dict) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/__init__.py b/models/tacotron2/NeMo/nemo/collections/tts/__init__.py new file mode 100644 index 0000000000..6d71a975fd --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nemo.collections.tts.data +import nemo.collections.tts.helpers +import nemo.collections.tts.models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/data/__init__.py b/models/tacotron2/NeMo/nemo/collections/tts/data/__init__.py new file mode 100644 index 0000000000..b03b6c3c8f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/data/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nemo.collections.tts.data.datalayers diff --git a/models/tacotron2/NeMo/nemo/collections/tts/data/datalayers.py b/models/tacotron2/NeMo/nemo/collections/tts/data/datalayers.py new file mode 100644 index 0000000000..cd5da7925b --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/data/datalayers.py @@ -0,0 +1,768 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MIT License + +# Copyright (c) 2019 Jeongmin Liu + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import collections as py_collections +import json +import logging +import math +import os +import pickle +import random +import shutil +import sys +from os.path import expanduser +from pathlib import Path +from typing import Any, Dict, Optional, Union + +import librosa +import numpy as np +import soundfile as sf +import torch +from torch.nn.utils.rnn import pad_sequence +from tqdm import tqdm + +from nemo.collections.asr.parts.preprocessing.features import WaveformFeaturizer +from nemo.collections.asr.parts.preprocessing.segment import AudioSegment +from nemo.collections.common.parts.preprocessing import collections, parsers +from nemo.core.classes import Dataset +from nemo.core.neural_types.elements import * +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + +DataDict = Dict[str, Any] + + +class AudioDataset(Dataset): + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + "audio_signal": NeuralType(("B", "T"), AudioSignal()), + "a_sig_length": NeuralType(tuple("B"), LengthsType()), + } + + def __init__( + self, + manifest_filepath: Union[str, "pathlib.Path"], + n_segments: int, + max_duration: Optional[float] = None, + min_duration: Optional[float] = None, + trim: Optional[bool] = False, + truncate_to: Optional[int] = 1, + ): + """ + Mostly compliant with nemo.collections.asr.data.datalayers.AudioToTextDataset except it only returns Audio + without text. Dataset that loads tensors via a json file containing paths to audio files, transcripts, and + durations (in seconds). Each new line is a different sample. Note that text is required, but is ignored for + AudioDataset. Example below: + {"audio_filepath": "/path/to/audio.wav", "text_filepath": + "/path/to/audio.txt", "duration": 23.147} + ... + {"audio_filepath": "/path/to/audio.wav", "text": "the + transcription", "offset": 301.75, "duration": 0.82, "utt": + "utterance_id", "ctm_utt": "en_4156", "side": "A"} + Args: + manifest_filepath (str, Path): Path to manifest json as described above. Can be comma-separated paths + such as "train_1.json,train_2.json" which is treated as two separate json files. + n_segments (int): The length of audio in samples to load. For example, given a sample rate of 16kHz, and + n_segments=16000, a random 1 second section of audio from the clip will be loaded. The section will + be randomly sampled everytime the audio is batched. Can be set to -1 to load the entire audio. + max_duration (float): If audio exceeds this length in seconds, it is filtered from the dataset. + Defaults to None, which does not filter any audio. + min_duration(float): If audio is less than this length in seconds, it is filtered from the dataset. + Defaults to None, which does not filter any audio. + trim (bool): Whether to use librosa.effects.trim on the audio clip + truncate_to (int): Ensures that the audio segment returned is a multiple of truncate_to. + Defaults to 1, which does no truncating. + """ + + self.collection = collections.ASRAudioText( + manifests_files=manifest_filepath.split(","), + parser=parsers.make_parser(), + min_duration=min_duration, + max_duration=max_duration, + ) + self.trim = trim + self.n_segments = n_segments + self.truncate_to = truncate_to + + def _collate_fn(self, batch): + """ + Takes a batch: a lists of length batch_size, defined in the dataloader. Returns 2 padded and batched + tensors corresponding to the audio and audio_length. + """ + + def find_max_len(seq, index): + max_len = -1 + for item in seq: + if item[index].size(0) > max_len: + max_len = item[index].size(0) + return max_len + + batch_size = len(batch) + + audio_signal, audio_lengths = None, None + if batch[0][0] is not None: + if self.n_segments > 0: + max_audio_len = self.n_segments + else: + max_audio_len = find_max_len(batch, 0) + + audio_signal = torch.zeros(batch_size, max_audio_len, dtype=torch.float) + audio_lengths = [] + for i, sample in enumerate(batch): + audio_signal[i].narrow(0, 0, sample[0].size(0)).copy_(sample[0]) + audio_lengths.append(sample[1]) + audio_lengths = torch.tensor(audio_lengths, dtype=torch.long) + + return audio_signal, audio_lengths + + def __getitem__(self, index): + """ + Given a index, returns audio and audio_length of the corresponding element. Audio clips of n_segments are + randomly chosen if the audio is longer than n_segments. + """ + example = self.collection[index] + features = AudioSegment.segment_from_file(example.audio_file, n_segments=self.n_segments, trim=self.trim,) + features = torch.tensor(features.samples) + audio, audio_length = features, torch.tensor(features.shape[0]).long() + + truncate = audio_length % self.truncate_to + if truncate != 0: + audio_length -= truncate.long() + audio = audio[:audio_length] + + return audio, audio_length + + def __len__(self): + return len(self.collection) + + +class MelAudioDataset(Dataset): + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + "audio_signal": NeuralType(("B", "T"), AudioSignal()), + "audio_length": NeuralType(tuple("B"), LengthsType()), + "melspec": NeuralType(("B", "D", "T"), MelSpectrogramType()), + } + + def __init__( + self, + manifest_filepath: Union[str, "pathlib.Path"], + n_segments: int, + max_duration: Optional[float] = float("inf"), + min_duration: Optional[float] = 0, + mel_hop_size=256, + mel_load_func=np.load, + ): + """ + `manifest_filepath` should point to a file with the following structure: + {"audio_filepath": "/path/to/audio.wav", "mel_filepath": "/path/to/mel.npy", "duration": 23.147} + where `mel.npy` is a pre-computed tensor of a melspectrogram. + this dataset class is used for fine-tuning vocoders with pre-computed mel-spectrograms. + """ + self.collection = [json.loads(line) for line in open(manifest_filepath, "r")] + self.collection = list(filter(lambda item: min_duration <= item["duration"] <= max_duration, self.collection)) + self.n_segments = n_segments + self.mel_hop_size = mel_hop_size + self.mel_load_func = mel_load_func + + def __getitem__(self, index): + """ + Given a index, returns audio and audio_length of the corresponding element. Audio clips of n_segments are + randomly chosen if the audio is longer than n_segments. + """ + example = self.collection[index] + audio_file = example["audio_filepath"] + mel_file = example["mel_filepath"] + + audio, sr = sf.read(audio_file) + audio = torch.FloatTensor(audio).unsqueeze(0) + mel = self.mel_load_func(mel_file) + + frames = math.ceil(self.n_segments / self.mel_hop_size) + if audio.shape[1] > self.n_segments: + start = random.randint(0, mel.shape[1] - frames - 1) + mel = mel[:, start : start + frames] + audio = audio[:, start * self.mel_hop_size : (start + frames) * self.mel_hop_size] + else: + mel = np.pad(mel, ((0, 0), (0, frames - mel.shape[1]))) + audio = torch.nn.functional.pad(audio, (0, self.n_segments - audio.shape[1])) + + return audio.squeeze(0), audio.shape[1], mel + + def __len__(self): + return len(self.collection) + + +class SplicedAudioDataset(Dataset): + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + 'audio_signal': NeuralType(('B', 'T'), AudioSignal()), + 'a_sig_length': NeuralType(tuple('B'), LengthsType()), + } + + def __init__( + self, + manifest_filepath: Union[str, 'pathlib.Path'], + n_segments: int, + max_duration: Optional[float] = None, + min_duration: Optional[float] = None, + trim: Optional[bool] = False, + truncate_to: Optional[int] = 1, + ): + """ + See above AudioDataset for details on dataset and manifest formats. + + Unlike the regular AudioDataset, which samples random segments from each audio array as an example, + SplicedAudioDataset concatenates all audio arrays together and indexes segments as examples. This way, + the model sees more data (about 9x for LJSpeech) per epoch. + + Note: this class is not recommended to be used in validation. + + Args: + manifest_filepath (str, Path): Path to manifest json as described above. Can be comma-separated paths + such as "train_1.json,train_2.json" which is treated as two separate json files. + n_segments (int): The length of audio in samples to load. For example, given a sample rate of 16kHz, and + n_segments=16000, a random 1 second section of audio from the clip will be loaded. The section will + be randomly sampled everytime the audio is batched. Can be set to -1 to load the entire audio. + max_duration (float): If audio exceeds this length in seconds, it is filtered from the dataset. + Defaults to None, which does not filter any audio. + min_duration(float): If audio is less than this length in seconds, it is filtered from the dataset. + Defaults to None, which does not filter any audio. + trim (bool): Whether to use librosa.effects.trim on the audio clip + truncate_to (int): Ensures that the audio segment returned is a multiple of truncate_to. + Defaults to 1, which does no truncating. + """ + assert n_segments > 0 + + collection = collections.ASRAudioText( + manifests_files=manifest_filepath.split(','), + parser=parsers.make_parser(), + min_duration=min_duration, + max_duration=max_duration, + ) + self.trim = trim + self.n_segments = n_segments + self.truncate_to = truncate_to + + self.samples = [] + for index in range(len(collection)): + example = collection[index] + with sf.SoundFile(example.audio_file, 'r') as f: + samples = f.read(dtype='float32').transpose() + self.samples.append(samples) + self.samples = np.concatenate(self.samples, axis=0) + self.samples = self.samples[: self.samples.shape[0] - (self.samples.shape[0] % self.n_segments), ...] + + def __getitem__(self, index): + """ + Given a index, returns audio and audio_length of the corresponding element. Audio clips of n_segments are + randomly chosen if the audio is longer than n_segments. + """ + audio_index = index * self.n_segments + audio = self.samples[audio_index : audio_index + self.n_segments] + + return audio, self.n_segments + + def __len__(self): + return self.samples.shape[0] // self.n_segments + + +class NoisySpecsDataset(Dataset): + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports. + """ + return { + 'x': NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + 'mag': NeuralType(('B', 'any', 'D', 'T'), SpectrogramType()), + 'max_length': NeuralType(None, LengthsType()), + 'y': NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + 'T_ys': NeuralType(tuple('B'), LengthsType()), + 'length': NeuralType(tuple('B'), LengthsType()), + 'path_speech': NeuralType(tuple('B'), StringType()), + } + + def __init__( + self, destination: Union[str, 'pathlib.Path'], subdir: str, n_fft: int, hop_length: int, num_snr: int, + ): + self.tar_dir = Path("%s/degli_data_%d_%dx%d/%s/" % (destination, n_fft, hop_length, num_snr, subdir)) + """ + A modified dataset for training deep-griffin-lim iteration. Contains MSTFT (mag), STFT (y) , and noisy STFT which is + used for initial phase. By using different levels of noise, the Degli model can learn to improve any phase, and thus + it can be used iteratively. + + Args: + destination (str, Path): Path to a directory containing the main data set folder, Similar to the directory + provided to the preprocessor script, which generates this dataset. + subdir (str): Either 'train', or 'valid', when using the standard script for generation. + n_fft (int): STFT parameter. Also detrmines the STFT filter length. + hop_length (int): STFT parameter. + num_snr (int): number of noisy samples per clean audio in the original dataset. + """ + + self._all_files = [f for f in os.listdir(self.tar_dir) if 'npz' in f] + + def __getitem__(self, index): + + file = Path(self.tar_dir / self._all_files[index]) + sample = dict() + + with np.load(file, mmap_mode='r') as npz_data: + for k, v in npz_data.items(): + if k in ['x', 'y', 'y_mag']: + sample[k] = torch.from_numpy(v) + elif k == "path_speech": + sample[k] = str(v) + elif k in ['T_x', 'T_y', 'length']: + sample[k] = int(v) + else: + sample[k] = v + return sample + + def __len__(self): + return len(self._all_files) + + @torch.no_grad() + def _collate_fn(self, batch): + """ return data with zero-padding + + Important data like x, y are all converted to Tensor(cpu). + :param batch: + :return: DataDict + Values can be an Tensor(cpu), list of str, ndarray of int. + """ + + result = dict() + T_xs = np.array([item.pop('T_x') for item in batch]) + idxs_sorted = np.argsort(T_xs) + T_xs = T_xs[idxs_sorted].tolist() + T_ys = [batch[idx].pop('T_y') for idx in idxs_sorted] + length = [batch[idx].pop('length') for idx in idxs_sorted] + + result['T_xs'], result['T_ys'], result['length'] = T_xs, T_ys, length + + for key, value in batch[0].items(): + if type(value) == str: + list_data = [batch[idx][key] for idx in idxs_sorted] + set_data = set(list_data) + if len(set_data) == 1: + result[key] = set_data.pop() + else: + result[key] = list_data + else: + if len(batch) > 1: + # B, T, F, C + data = [batch[idx][key].permute(1, 0, 2) for idx in idxs_sorted] + data = pad_sequence(data, batch_first=True) + # B, C, F, T + data = data.permute(0, 3, 2, 1) + else: # B, C, F, T + data = batch[0][key].unsqueeze(0).permute(0, 3, 1, 2) + + result[key] = data.contiguous() + + x = result['x'] + mag = result['y_mag'] + max_length = max(result['length']) + y = result['y'] + T_ys = result['T_ys'] + length = result['length'] + path_speech = result['path_speech'] + return x, mag, max_length, y, T_ys, length, path_speech + + @staticmethod + @torch.no_grad() + def decollate_padded(batch: DataDict, idx: int) -> DataDict: + """ select the `idx`-th data, get rid of padded zeros and return it. + + Important data like x, y are all converted to ndarray. + :param batch: + :param idx: + :return: DataDict + Values can be an str or ndarray. + """ + result = dict() + for key, value in batch.items(): + if type(value) == str: + result[key] = value + elif type(value) == list: + result[key] = value[idx] + elif not key.startswith('T_'): + T_xy = 'T_xs' if 'x' in key else 'T_ys' + value = value[idx, :, :, : batch[T_xy][idx]] # C, F, T + value = value.permute(1, 2, 0).contiguous() # F, T, C + value = value.numpy() + if value.shape[-1] == 2: + value = value.view(dtype=np.complex64) # F, T, 1 + result[key] = value + + return result + + +def setup_noise_augmented_dataset(files_list, num_snr, kwargs_stft, dest, desc): + + os.makedirs(dest) + with open(files_list, 'r') as list_file: + all_lines = [line for line in list_file] + list_file_pbar = tqdm(all_lines, desc=desc, dynamic_ncols=True) + + i_speech = 0 + for line in list_file_pbar: + audio_file = line.split('|')[0] + speech = sf.read(audio_file)[0].astype(np.float32) + spec_clean = np.ascontiguousarray(librosa.stft(speech, **kwargs_stft)) + mag_clean = np.ascontiguousarray(np.abs(spec_clean)[..., np.newaxis]) + signal_power = np.mean(np.abs(speech) ** 2) + + y = spec_clean.view(dtype=np.float32).reshape((*spec_clean.shape, 2)) + ##y = torch.from_numpy(y) + T_y = spec_clean.shape[1] + ##mag_clean = torch.from_numpy(mag_clean) + for k in range(num_snr): + snr_db = -6 * np.random.rand() + snr = librosa.db_to_power(snr_db) + noise_power = signal_power / snr + noisy = speech + np.sqrt(noise_power) * np.random.randn(len(speech)) + spec_noisy = librosa.stft(noisy, **kwargs_stft) + spec_noisy = np.ascontiguousarray(spec_noisy) + T_x = spec_noisy.shape[1] + x = spec_noisy.view(dtype=np.float32).reshape((*spec_noisy.shape, 2)) + ##x = torch.from_numpy(x) + mdict = dict(x=x, y=y, y_mag=mag_clean, path_speech=audio_file, length=len(speech), T_x=T_x, T_y=T_y) + np.savez( + f"{dest}/audio_{i_speech}_{k}.npz", **mdict, + ) + i_speech = i_speech + 1 + + return i_speech + + +def preprocess_linear_specs_dataset(valid_filelist, train_filelist, n_fft, hop_length, num_snr, destination): + kwargs_stft = dict(hop_length=hop_length, window='hann', center=True, n_fft=n_fft, dtype=np.complex64) + + tar_dir = "%s/degli_data_%d_%dx%d/" % (destination, n_fft, hop_length, num_snr) + if not os.path.isdir(tar_dir): + + if valid_filelist == "none" or train_filelist == "none": + logging.error(f"Director {tar_dir} does not exist. Filelists for validation and train must be provided.") + raise NameError("Missing Argument") + else: + logging.info( + f"Director {tar_dir} does not exist. Preprocessing audio files listed in {valid_filelist}, {train_filelist} to create new dataset." + ) + os.makedirs(tar_dir) + n_train = 0 + n_valid = 0 + try: + n_train = setup_noise_augmented_dataset( + train_filelist, num_snr, kwargs_stft, tar_dir + "train/", desc="Initializing Train Dataset" + ) + n_valid = setup_noise_augmented_dataset( + valid_filelist, num_snr, kwargs_stft, tar_dir + "valid/", desc="Initializing Validation Dataset" + ) + except FileNotFoundError as err: + shutil.rmtree(tar_dir) + raise err + except: + e = sys.exc_info()[0] + shutil.rmtree(tar_dir) + raise e + + if n_train == 0: + shutil.rmtree(tar_dir) + raise EOFError("Dataset initialization failed. No files to preprocess train dataset") + + if n_valid == 0: + shutil.rmtree(tar_dir) + raise EOFError("Dataset initialization failed. No files to preprocess validation dataset") + + return tar_dir + + +class FastSpeech2Dataset(Dataset): + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Returns definitions of module output ports.""" + return { + 'audio_signal': NeuralType(('B', 'T'), AudioSignal()), + 'a_sig_length': NeuralType(('B'), LengthsType()), + 'transcripts': NeuralType(('B', 'T'), TokenIndex()), + 'transcript_length': NeuralType(('B'), LengthsType()), + 'durations': NeuralType(('B', 'T'), TokenDurationType()), + 'pitches': NeuralType(('B', 'T'), RegressionValuesType()), + 'energies': NeuralType(('B', 'T'), RegressionValuesType()), + } + + def __init__( + self, + manifest_filepath: str, + mappings_filepath: str, + sample_rate: int, + max_duration: Optional[float] = None, + min_duration: Optional[float] = None, + ignore_file: Optional[str] = None, + trim: bool = False, + load_supplementary_values=True, # Set to False for validation + ): + """ + Dataset that loads audio, phonemes and their durations, pitches per frame, and energies per frame + for FastSpeech 2 from paths described in a JSON manifest (see the AudioDataset documentation for details + on the manifest format), as well as a mappings file for word to phones and phones to indices. + The text in the manifest is ignored; instead, the phoneme indices for prediction come from the + duration files. + + For each sample, paths for duration, energy, and pitch files are inferred from the manifest's audio + filepaths by replacing '/wavs' with '/phoneme_durations', '/pitches', and '/energies', and swapping out + the file extension to '.pt', '.npy', and '.npy' respectively. + For example, given manifest audio path `/data/LJSpeech/wavs/LJ001-0001.wav`, the inferred duration and + phonemes file path would be `/data/LJSpeech/phoneme_durations/LJ001-0001.pt`. + + Note that validation datasets only need the audio files and phoneme & duration files, set + `load_supplementary_values` to False for validation sets. + + Args: + manifest_filepath (str): Path to the JSON manifest file that lists audio files. + mappings_filepath (str): Path to a JSON mappings file that contains mappings "word2phones" and + "phone2idx". The latter is used to determine the padding index. + sample_rate (int): Target sample rate of the audio. + max_duration (float): If audio exceeds this length in seconds, it is filtered from the dataset. + Defaults to None, which does not filter any audio. + min_duration (float): If audio is shorter than this length in seconds, it is filtered from the dataset. + Defaults to None, which does not filter any audio. + ignore_file (str): Optional pickled file which contains a list of files to ignore (e.g. files that + contain OOV words). + Defaults to None. + trim (bool): Whether to use librosa.effects.trim on the audio clip. + Defaults to False. + load_supplementary_values (bool): Whether or not to load pitch and energy files. Set this to False for + validation datasets. + Defaults to True. + """ + super().__init__() + + # Retrieve mappings from file + with open(mappings_filepath, 'r') as f: + mappings = json.load(f) + self.word2phones = mappings['word2phones'] + self.phone2idx = mappings['phone2idx'] + + # Load data from manifests + audio_files = [] + total_duration = 0 + if isinstance(manifest_filepath, str): + manifest_filepath = [manifest_filepath] + for manifest_file in manifest_filepath: + with open(expanduser(manifest_file), 'r') as f: + logging.info(f"Loading dataset from {manifest_file}.") + for line in f: + item = json.loads(line) + audio_files.append({"audio_filepath": item["audio_filepath"], "duration": item["duration"]}) + total_duration += item["duration"] + + total_dataset_len = len(audio_files) + logging.info(f"Loaded dataset with {total_dataset_len} files totalling {total_duration/3600:.2f} hours.") + self.data = [] + if load_supplementary_values: + dataitem = py_collections.namedtuple( + typename='AudioTextEntity', field_names='audio_file duration text_tokens pitches energies' + ) + else: + dataitem = py_collections.namedtuple( + typename='AudioTextEntity', field_names='audio_file duration text_tokens' + ) + + if ignore_file: + logging.info(f"using {ignore_file} to prune dataset.") + with open(ignore_file, "rb") as f: + wavs_to_ignore = set(pickle.load(f)) + + pruned_duration = 0 + pruned_items = 0 + for item in audio_files: + audio_path = item['audio_filepath'] + LJ_id = os.path.splitext(os.path.basename(audio_path))[0] + + # Prune data according to min/max_duration & the ignore file + if (min_duration and item["duration"] < min_duration) or ( + max_duration and item["duration"] > max_duration + ): + pruned_duration += item["duration"] + pruned_items += 1 + continue + if ignore_file and (LJ_id in wavs_to_ignore): + pruned_items += 1 + pruned_duration += item["duration"] + wavs_to_ignore.remove(LJ_id) + continue + + # Else not pruned, load additional info + + # Phoneme durations and text token indices from durations file + dur_path = audio_path.replace('/wavs/', '/phoneme_durations/').replace('.wav', '.pt') + duration_info = torch.load(dur_path) + durs = duration_info['token_duration'] + text_tokens = duration_info['text_encoded'] + + if load_supplementary_values: + # Load pitch file (F0s) + pitch_path = audio_path.replace('/wavs/', '/pitches/').replace('.wav', '.npy') + pitches = torch.from_numpy(np.load(pitch_path).astype(dtype='float32')) + + # Load energy file (L2-norm of the amplitude of each STFT frame of an utterance) + energies_path = audio_path.replace('/wavs/', '/energies/').replace('.wav', '.npy') + energies = torch.from_numpy(np.load(energies_path)) + + self.data.append( + dataitem( + audio_file=item['audio_filepath'], + duration=durs, + pitches=torch.clamp(pitches, min=1e-5), + energies=energies, + text_tokens=text_tokens, + ) + ) + else: + self.data.append(dataitem(audio_file=item['audio_filepath'], duration=durs, text_tokens=text_tokens,)) + + logging.info(f"Pruned {pruned_items} files and {pruned_duration/3600:.2f} hours.") + logging.info( + f"Final dataset contains {len(self.data)} files and {(total_duration-pruned_duration)/3600:.2f} hours." + ) + + self.featurizer = WaveformFeaturizer(sample_rate=sample_rate) + self.trim = trim + self.load_supplementary_values = load_supplementary_values + + def __getitem__(self, index): + sample = self.data[index] + + features = self.featurizer.process(sample.audio_file, trim=self.trim) + f, fl = features, torch.tensor(features.shape[0]).long() + t, tl = sample.text_tokens.long(), torch.tensor(len(sample.text_tokens)).long() + + if self.load_supplementary_values: + return f, fl, t, tl, sample.duration, sample.pitches, sample.energies + else: + return f, fl, t, tl, sample.duration, None, None + + def __len__(self): + return len(self.data) + + def _collate_fn(self, batch): + pad_id = len(self.phone2idx) + if self.load_supplementary_values: + _, audio_lengths, _, tokens_lengths, duration, pitches, energies = zip(*batch) + else: + _, audio_lengths, _, tokens_lengths, duration, _, _ = zip(*batch) + max_audio_len = 0 + max_audio_len = max(audio_lengths).item() + max_tokens_len = max(tokens_lengths).item() + max_durations_len = max([len(i) for i in duration]) + max_duration_sum = max([sum(i) for i in duration]) + if self.load_supplementary_values: + max_pitches_len = max([len(i) for i in pitches]) + max_energies_len = max([len(i) for i in energies]) + if max_pitches_len != max_energies_len or max_pitches_len != max_duration_sum: + logging.warning( + f"max_pitches_len: {max_pitches_len} != max_energies_len: {max_energies_len} != " + f"max_duration_sum:{max_duration_sum}. Your training run will error out!" + ) + + # Add padding where necessary + audio_signal, tokens, duration_batched, pitches_batched, energies_batched = [], [], [], [], [] + for sample_tuple in batch: + if self.load_supplementary_values: + sig, sig_len, tokens_i, tokens_i_len, duration, pitch, energy = sample_tuple + else: + sig, sig_len, tokens_i, tokens_i_len, duration, _, _ = sample_tuple + sig_len = sig_len.item() + if sig_len < max_audio_len: + pad = (0, max_audio_len - sig_len) + sig = torch.nn.functional.pad(sig, pad) + audio_signal.append(sig) + tokens_i_len = tokens_i_len.item() + if tokens_i_len < max_tokens_len: + pad = (0, max_tokens_len - tokens_i_len) + tokens_i = torch.nn.functional.pad(tokens_i, pad, value=pad_id) + tokens.append(tokens_i) + if len(duration) < max_durations_len: + pad = (0, max_durations_len - len(duration)) + duration = torch.nn.functional.pad(duration, pad) + duration_batched.append(duration) + + if self.load_supplementary_values: + pitch = pitch.squeeze(0) + if len(pitch) < max_pitches_len: + pad = (0, max_pitches_len - len(pitch)) + pitch = torch.nn.functional.pad(pitch.squeeze(0), pad) + pitches_batched.append(pitch) + + if len(energy) < max_energies_len: + pad = (0, max_energies_len - len(energy)) + energy = torch.nn.functional.pad(energy, pad) + energies_batched.append(energy) + + audio_signal = torch.stack(audio_signal) + audio_lengths = torch.stack(audio_lengths) + tokens = torch.stack(tokens) + tokens_lengths = torch.stack(tokens_lengths) + duration_batched = torch.stack(duration_batched) + + if self.load_supplementary_values: + pitches_batched = torch.stack(pitches_batched) + energies_batched = torch.stack(energies_batched) + assert pitches_batched.shape == energies_batched.shape + + return ( + audio_signal, + audio_lengths, + tokens, + tokens_lengths, + duration_batched, + pitches_batched, + energies_batched, + ) + return (audio_signal, audio_lengths, tokens, tokens_lengths, duration_batched, None, None) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/helpers/__init__.py b/models/tacotron2/NeMo/nemo/collections/tts/helpers/__init__.py new file mode 100644 index 0000000000..3da88782dd --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/helpers/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nemo.collections.tts.helpers.helpers diff --git a/models/tacotron2/NeMo/nemo/collections/tts/helpers/helpers.py b/models/tacotron2/NeMo/nemo/collections/tts/helpers/helpers.py new file mode 100644 index 0000000000..d175699556 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/helpers/helpers.py @@ -0,0 +1,362 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from typing import Dict, Sequence + +import librosa +import matplotlib.pylab as plt +import numpy as np +import torch +from numba import jit, prange +from numpy import ndarray +from pesq import pesq +from pystoi import stoi +from pytorch_lightning.utilities import rank_zero_only + +from nemo.utils import logging + + +class OperationMode(Enum): + """Training or Inference (Evaluation) mode""" + + training = 0 + validation = 1 + infer = 2 + + +def binarize_attention(attn, in_len, out_len): + """Convert soft attention matrix to hard attention matrix. + + Args: + attn (torch.Tensor): B x 1 x max_mel_len x max_text_len. Soft attention matrix. + in_len (torch.Tensor): B. Lengths of texts. + out_len (torch.Tensor): B. Lengths of spectrograms. + + Output: + attn_out (torch.Tensor): B x 1 x max_mel_len x max_text_len. Hard attention matrix, final dim max_text_len should sum to 1. + """ + b_size = attn.shape[0] + with torch.no_grad(): + attn_cpu = attn.data.cpu().numpy() + attn_out = torch.zeros_like(attn) + for ind in range(b_size): + hard_attn = mas(attn_cpu[ind, 0, : out_len[ind], : in_len[ind]]) + attn_out[ind, 0, : out_len[ind], : in_len[ind]] = torch.tensor(hard_attn, device=attn.device) + return attn_out + + +def binarize_attention_parallel(attn, in_lens, out_lens): + """For training purposes only. Binarizes attention with MAS. + These will no longer recieve a gradient. + + Args: + attn: B x 1 x max_mel_len x max_text_len + """ + with torch.no_grad(): + attn_cpu = attn.data.cpu().numpy() + attn_out = b_mas(attn_cpu, in_lens.cpu().numpy(), out_lens.cpu().numpy(), width=1) + return torch.from_numpy(attn_out).to(attn.get_device()) + + +def get_mask_from_lengths(lengths, max_len=None): + if not max_len: + max_len = torch.max(lengths).item() + ids = torch.arange(0, max_len, device=lengths.device, dtype=torch.long) + mask = (ids < lengths.unsqueeze(1)).bool() + return mask + + +@jit(nopython=True) +def mas(attn_map, width=1): + # assumes mel x text + opt = np.zeros_like(attn_map) + attn_map = np.log(attn_map) + attn_map[0, 1:] = -np.inf + log_p = np.zeros_like(attn_map) + log_p[0, :] = attn_map[0, :] + prev_ind = np.zeros_like(attn_map, dtype=np.int64) + for i in range(1, attn_map.shape[0]): + for j in range(attn_map.shape[1]): # for each text dim + prev_j = np.arange(max(0, j - width), j + 1) + prev_log = np.array([log_p[i - 1, prev_idx] for prev_idx in prev_j]) + + ind = np.argmax(prev_log) + log_p[i, j] = attn_map[i, j] + prev_log[ind] + prev_ind[i, j] = prev_j[ind] + + # now backtrack + curr_text_idx = attn_map.shape[1] - 1 + for i in range(attn_map.shape[0] - 1, -1, -1): + opt[i, curr_text_idx] = 1 + curr_text_idx = prev_ind[i, curr_text_idx] + opt[0, curr_text_idx] = 1 + + assert opt.sum(0).all() + assert opt.sum(1).all() + + return opt + + +@jit(nopython=True) +def mas_width1(attn_map): + """mas with hardcoded width=1""" + # assumes mel x text + opt = np.zeros_like(attn_map) + attn_map = np.log(attn_map) + attn_map[0, 1:] = -np.inf + log_p = np.zeros_like(attn_map) + log_p[0, :] = attn_map[0, :] + prev_ind = np.zeros_like(attn_map, dtype=np.int64) + for i in range(1, attn_map.shape[0]): + for j in range(attn_map.shape[1]): # for each text dim + prev_log = log_p[i - 1, j] + prev_j = j + + if j - 1 >= 0 and log_p[i - 1, j - 1] >= log_p[i - 1, j]: + prev_log = log_p[i - 1, j - 1] + prev_j = j - 1 + + log_p[i, j] = attn_map[i, j] + prev_log + prev_ind[i, j] = prev_j + + # now backtrack + curr_text_idx = attn_map.shape[1] - 1 + for i in range(attn_map.shape[0] - 1, -1, -1): + opt[i, curr_text_idx] = 1 + curr_text_idx = prev_ind[i, curr_text_idx] + opt[0, curr_text_idx] = 1 + return opt + + +@jit(nopython=True, parallel=True) +def b_mas(b_attn_map, in_lens, out_lens, width=1): + assert width == 1 + attn_out = np.zeros_like(b_attn_map) + + for b in prange(b_attn_map.shape[0]): + out = mas_width1(b_attn_map[b, 0, : out_lens[b], : in_lens[b]]) + attn_out[b, 0, : out_lens[b], : in_lens[b]] = out + return attn_out + + +def griffin_lim(magnitudes, n_iters=50, n_fft=1024): + """ + Griffin-Lim algorithm to convert magnitude spectrograms to audio signals + """ + phase = np.exp(2j * np.pi * np.random.rand(*magnitudes.shape)) + complex_spec = magnitudes * phase + signal = librosa.istft(complex_spec) + if not np.isfinite(signal).all(): + logging.warning("audio was not finite, skipping audio saving") + return np.array([0]) + + for _ in range(n_iters): + _, phase = librosa.magphase(librosa.stft(signal, n_fft=n_fft)) + complex_spec = magnitudes * phase + signal = librosa.istft(complex_spec) + return signal + + +@rank_zero_only +def log_audio_to_tb( + swriter, + spect, + name, + step, + griffin_lim_mag_scale=1024, + griffin_lim_power=1.2, + sr=22050, + n_fft=1024, + n_mels=80, + fmax=8000, +): + filterbank = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels, fmax=fmax) + log_mel = spect.data.cpu().numpy().T + mel = np.exp(log_mel) + magnitude = np.dot(mel, filterbank) * griffin_lim_mag_scale + audio = griffin_lim(magnitude.T ** griffin_lim_power) + swriter.add_audio(name, audio / max(np.abs(audio)), step, sample_rate=sr) + + +@rank_zero_only +def tacotron2_log_to_tb_func( + swriter, + tensors, + step, + tag="train", + log_images=False, + log_images_freq=1, + add_audio=True, + griffin_lim_mag_scale=1024, + griffin_lim_power=1.2, + sr=22050, + n_fft=1024, + n_mels=80, + fmax=8000, +): + _, spec_target, mel_postnet, gate, gate_target, alignments = tensors + if log_images and step % log_images_freq == 0: + swriter.add_image( + f"{tag}_alignment", plot_alignment_to_numpy(alignments[0].data.cpu().numpy().T), step, dataformats="HWC", + ) + swriter.add_image( + f"{tag}_mel_target", plot_spectrogram_to_numpy(spec_target[0].data.cpu().numpy()), step, dataformats="HWC", + ) + swriter.add_image( + f"{tag}_mel_predicted", + plot_spectrogram_to_numpy(mel_postnet[0].data.cpu().numpy()), + step, + dataformats="HWC", + ) + swriter.add_image( + f"{tag}_gate", + plot_gate_outputs_to_numpy(gate_target[0].data.cpu().numpy(), torch.sigmoid(gate[0]).data.cpu().numpy(),), + step, + dataformats="HWC", + ) + if add_audio: + filterbank = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels, fmax=fmax) + log_mel = mel_postnet[0].data.cpu().numpy().T + mel = np.exp(log_mel) + magnitude = np.dot(mel, filterbank) * griffin_lim_mag_scale + audio = griffin_lim(magnitude.T ** griffin_lim_power) + swriter.add_audio(f"audio/{tag}_predicted", audio / max(np.abs(audio)), step, sample_rate=sr) + + log_mel = spec_target[0].data.cpu().numpy().T + mel = np.exp(log_mel) + magnitude = np.dot(mel, filterbank) * griffin_lim_mag_scale + audio = griffin_lim(magnitude.T ** griffin_lim_power) + swriter.add_audio(f"audio/{tag}_target", audio / max(np.abs(audio)), step, sample_rate=sr) + + +def plot_alignment_to_numpy(alignment, info=None): + fig, ax = plt.subplots(figsize=(6, 4)) + im = ax.imshow(alignment, aspect='auto', origin='lower', interpolation='none') + fig.colorbar(im, ax=ax) + xlabel = 'Decoder timestep' + if info is not None: + xlabel += '\n\n' + info + plt.xlabel(xlabel) + plt.ylabel('Encoder timestep') + plt.tight_layout() + + fig.canvas.draw() + data = save_figure_to_numpy(fig) + plt.close() + return data + + +def plot_spectrogram_to_numpy(spectrogram): + spectrogram = spectrogram.astype(np.float32) + fig, ax = plt.subplots(figsize=(12, 3)) + im = ax.imshow(spectrogram, aspect="auto", origin="lower", interpolation='none') + plt.colorbar(im, ax=ax) + plt.xlabel("Frames") + plt.ylabel("Channels") + plt.tight_layout() + + fig.canvas.draw() + data = save_figure_to_numpy(fig) + plt.close() + return data + + +def plot_gate_outputs_to_numpy(gate_targets, gate_outputs): + fig, ax = plt.subplots(figsize=(12, 3)) + ax.scatter( + range(len(gate_targets)), gate_targets, alpha=0.5, color='green', marker='+', s=1, label='target', + ) + ax.scatter( + range(len(gate_outputs)), gate_outputs, alpha=0.5, color='red', marker='.', s=1, label='predicted', + ) + + plt.xlabel("Frames (Green target, Red predicted)") + plt.ylabel("Gate State") + plt.tight_layout() + + fig.canvas.draw() + data = save_figure_to_numpy(fig) + plt.close() + return data + + +def save_figure_to_numpy(fig): + # save it to a numpy array. + data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') + data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) + return data + + +@rank_zero_only +def waveglow_log_to_tb_func( + swriter, tensors, step, tag="train", n_fft=1024, hop_length=256, window="hann", mel_fb=None, +): + _, audio_pred, spec_target, mel_length = tensors + mel_length = mel_length[0] + spec_target = spec_target[0].data.cpu().numpy()[:, :mel_length] + swriter.add_image( + f"{tag}_mel_target", plot_spectrogram_to_numpy(spec_target), step, dataformats="HWC", + ) + if mel_fb is not None: + mag, _ = librosa.core.magphase( + librosa.core.stft( + np.nan_to_num(audio_pred[0].cpu().detach().numpy()), n_fft=n_fft, hop_length=hop_length, window=window, + ) + ) + mel_pred = np.matmul(mel_fb.cpu().numpy(), mag).squeeze() + log_mel_pred = np.log(np.clip(mel_pred, a_min=1e-5, a_max=None)) + swriter.add_image( + f"{tag}_mel_predicted", plot_spectrogram_to_numpy(log_mel_pred[:, :mel_length]), step, dataformats="HWC", + ) + + +def remove(conv_list): + new_conv_list = torch.nn.ModuleList() + for old_conv in conv_list: + old_conv = torch.nn.utils.remove_weight_norm(old_conv) + new_conv_list.append(old_conv) + return new_conv_list + + +def eval_tts_scores( + y_clean: ndarray, y_est: ndarray, T_ys: Sequence[int] = (0,), sampling_rate=22050 +) -> Dict[str, float]: + """ + calculate metric using EvalModule. y can be a batch. + Args: + y_clean: real audio + y_est: estimated audio + T_ys: length of the non-zero parts of the histograms + sampling_rate: The used Sampling rate. + + Returns: + A dictionary mapping scoring systems (string) to numerical scores. + 1st entry: 'STOI' + 2nd entry: 'PESQ' + """ + + if y_clean.ndim == 1: + y_clean = y_clean[np.newaxis, ...] + y_est = y_est[np.newaxis, ...] + if T_ys == (0,): + T_ys = (y_clean.shape[1],) * y_clean.shape[0] + + clean = y_clean[0, : T_ys[0]] + estimated = y_est[0, : T_ys[0]] + stoi_score = stoi(clean, estimated, sampling_rate, extended=False) + pesq_score = pesq(16000, np.asarray(clean), estimated, 'wb') + ## fs was set 16,000, as pesq lib doesnt currently support felxible fs. + + return {'STOI': stoi_score, 'PESQ': pesq_score} diff --git a/models/tacotron2/NeMo/nemo/collections/tts/losses/__init__.py b/models/tacotron2/NeMo/nemo/collections/tts/losses/__init__.py new file mode 100644 index 0000000000..c9f20c1fe2 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/losses/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nemo.collections.tts.losses.tacotron2loss +import nemo.collections.tts.losses.waveglowloss diff --git a/models/tacotron2/NeMo/nemo/collections/tts/losses/aligner_loss.py b/models/tacotron2/NeMo/nemo/collections/tts/losses/aligner_loss.py new file mode 100644 index 0000000000..5ad6ee96e3 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/losses/aligner_loss.py @@ -0,0 +1,89 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import torch +import torch.nn.functional as F + +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types.elements import LengthsType, LogprobsType, LossType, ProbsType +from nemo.core.neural_types.neural_type import NeuralType + + +class ForwardSumLoss(Loss): + def __init__(self, blank_logprob=-1): + super().__init__() + self.log_softmax = torch.nn.LogSoftmax(dim=3) + self.ctc_loss = torch.nn.CTCLoss(zero_infinity=True) + self.blank_logprob = blank_logprob + + @property + def input_types(self): + return { + "attn_logprob": NeuralType(('B', 'S', 'T', 'D'), LogprobsType()), + "in_lens": NeuralType(tuple('B'), LengthsType()), + "out_lens": NeuralType(tuple('B'), LengthsType()), + } + + @property + def output_types(self): + return { + "forward_sum_loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, attn_logprob, in_lens, out_lens): + key_lens = in_lens + query_lens = out_lens + attn_logprob_padded = F.pad(input=attn_logprob, pad=(1, 0), value=self.blank_logprob) + + total_loss = 0.0 + for bid in range(attn_logprob.shape[0]): + target_seq = torch.arange(1, key_lens[bid] + 1).unsqueeze(0) + curr_logprob = attn_logprob_padded[bid].permute(1, 0, 2)[: query_lens[bid], :, : key_lens[bid] + 1] + + curr_logprob = self.log_softmax(curr_logprob[None])[0] + loss = self.ctc_loss( + curr_logprob, + target_seq, + input_lengths=query_lens[bid : bid + 1], + target_lengths=key_lens[bid : bid + 1], + ) + total_loss += loss + + total_loss /= attn_logprob.shape[0] + return total_loss + + +class BinLoss(Loss): + def __init__(self): + super().__init__() + + @property + def input_types(self): + return { + "hard_attention": NeuralType(('B', 'S', 'T', 'D'), ProbsType()), + "soft_attention": NeuralType(('B', 'S', 'T', 'D'), ProbsType()), + } + + @property + def output_types(self): + return { + "bin_loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, hard_attention, soft_attention): + log_sum = torch.log(torch.clamp(soft_attention[hard_attention == 1], min=1e-12)).sum() + return -log_sum / hard_attention.sum() diff --git a/models/tacotron2/NeMo/nemo/collections/tts/losses/fastpitchloss.py b/models/tacotron2/NeMo/nemo/collections/tts/losses/fastpitchloss.py new file mode 100644 index 0000000000..f9e9b3926e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/losses/fastpitchloss.py @@ -0,0 +1,150 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# BSD 3-Clause License +# +# Copyright (c) 2021, NVIDIA Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import torch +import torch.nn.functional as F + +from nemo.collections.tts.modules.transformer import mask_from_lens +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types.elements import ( + LengthsType, + LossType, + MelSpectrogramType, + RegressionValuesType, + TokenDurationType, + TokenLogDurationType, +) +from nemo.core.neural_types.neural_type import NeuralType + + +class DurationLoss(Loss): + def __init__(self, loss_scale=0.1): + super().__init__() + self.loss_scale = loss_scale + + @property + def input_types(self): + return { + "log_durs_predicted": NeuralType(('B', 'T'), TokenLogDurationType()), + "durs_tgt": NeuralType(('B', 'T'), TokenDurationType()), + "len": NeuralType(('B'), LengthsType()), + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, log_durs_predicted, durs_tgt, len): + dur_mask = mask_from_lens(len, max_len=durs_tgt.size(1)) + log_durs_tgt = torch.log(durs_tgt.float() + 1) + loss_fn = F.mse_loss + dur_loss = loss_fn(log_durs_predicted, log_durs_tgt, reduction='none') + dur_loss = (dur_loss * dur_mask).sum() / dur_mask.sum() + dur_loss *= self.loss_scale + + return dur_loss + + +class PitchLoss(Loss): + def __init__(self, loss_scale=0.1): + super().__init__() + self.loss_scale = loss_scale + + @property + def input_types(self): + return { + "pitch_predicted": NeuralType(('B', 'T'), RegressionValuesType()), + "pitch_tgt": NeuralType(('B', 'T'), RegressionValuesType()), + "len": NeuralType(('B'), LengthsType()), + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, pitch_predicted, pitch_tgt, len): + dur_mask = mask_from_lens(len, max_len=pitch_tgt.size(1)) + ldiff = pitch_tgt.size(1) - pitch_predicted.size(1) + pitch_predicted = F.pad(pitch_predicted, (0, ldiff, 0, 0), value=0.0) + pitch_loss = F.mse_loss(pitch_tgt, pitch_predicted, reduction='none') + pitch_loss = (pitch_loss * dur_mask).sum() / dur_mask.sum() + pitch_loss *= self.loss_scale + + return pitch_loss + + +class MelLoss(Loss): + @property + def input_types(self): + return { + "spect_predicted": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "spect_tgt": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, spect_predicted, spect_tgt): + spect_tgt.requires_grad = False + spect_tgt = spect_tgt.transpose(1, 2) # (B, T, H) + spect_predicted = spect_predicted.transpose(1, 2) # (B, T, H) + + ldiff = spect_tgt.size(1) - spect_predicted.size(1) + spect_predicted = F.pad(spect_predicted, (0, 0, 0, ldiff, 0, 0), value=0.0) + mel_mask = spect_tgt.ne(0).float() + loss_fn = F.mse_loss + mel_loss = loss_fn(spect_predicted, spect_tgt, reduction='none') + mel_loss = (mel_loss * mel_mask).sum() / mel_mask.sum() + + return mel_loss diff --git a/models/tacotron2/NeMo/nemo/collections/tts/losses/fastspeech2loss.py b/models/tacotron2/NeMo/nemo/collections/tts/losses/fastspeech2loss.py new file mode 100644 index 0000000000..fb8146aacf --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/losses/fastspeech2loss.py @@ -0,0 +1,127 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from nemo.collections.tts.helpers.helpers import get_mask_from_lengths +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types.elements import LengthsType, LossType, MaskType, MelSpectrogramType, TokenDurationType +from nemo.core.neural_types.neural_type import NeuralType + + +class DurationLoss(Loss): + """A Loss module that computes the log duration prediction loss for FastSpeech 2.""" + + @property + def input_types(self): + return { + "log_duration_pred": NeuralType(('B', 'T'), TokenDurationType()), + "duration_target": NeuralType(('B', 'T'), TokenDurationType()), + "mask": NeuralType(('B', 'T', 'D'), MaskType()), + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, *, log_duration_pred, duration_target, mask): + log_duration_target = torch.log(duration_target + 1) + return torch.nn.functional.mse_loss(log_duration_pred, log_duration_target) + + +class L1MelLoss(Loss): + """A Loss module that computes L1 mel loss for FastSpeech 2.""" + + @property + def input_types(self): + return { + "spec_pred": NeuralType(('B', 'C', 'T'), MelSpectrogramType()), + "spec_target": NeuralType(('B', 'C', 'T'), MelSpectrogramType()), + "spec_target_len": NeuralType(('B'), LengthsType()), + "pad_value": NeuralType(), + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, *, spec_pred, spec_target, spec_target_len, pad_value): + spec_target.requires_grad = False + max_len = spec_target.shape[2] + + if max_len < spec_pred.shape[2]: + # Predicted len is larger than reference + # Need to slice + spec_pred = spec_pred.narrow(2, 0, max_len) + elif max_len > spec_pred.shape[2]: + # Need to do padding + pad_amount = max_len - spec_pred.shape[2] + spec_pred = torch.nn.functional.pad(spec_pred, (0, pad_amount), value=pad_value) + max_len = spec_pred.shape[2] + + mask = ~get_mask_from_lengths(spec_target_len, max_len=max_len) + mask = mask.expand(spec_target.shape[1], mask.size(0), mask.size(1)) + mask = mask.permute(1, 0, 2) + spec_pred.masked_fill_(mask, pad_value) + + mel_loss = torch.nn.functional.l1_loss(spec_pred, spec_target) + return mel_loss + + +class L2MelLoss(Loss): + """A Loss module that computes L2 mel loss for FastSpeech 2""" + + @property + def input_types(self): + return { + "spec_pred": NeuralType(('B', 'C', 'T'), MelSpectrogramType()), # Transposed from target! + "spec_target": NeuralType(('B', 'C', 'T'), MelSpectrogramType()), + "spec_target_len": NeuralType(('B'), LengthsType()), + "pad_value": NeuralType(), + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, *, spec_pred, spec_target, spec_target_len, pad_value): + spec_target.requires_grad = False + max_len = spec_target.shape[2] + + if max_len < spec_pred.shape[2]: + # Predicted len is larger than reference + # Need to slice + spec_pred = spec_pred.narrow(2, 0, max_len) + elif max_len > spec_pred.shape[2]: + # Need to do padding + pad_amount = max_len - spec_pred.shape[2] + spec_pred = torch.nn.functional.pad(spec_pred, (0, pad_amount), value=pad_value) + max_len = spec_pred.shape[2] + + mask = ~get_mask_from_lengths(spec_target_len, max_len=max_len) + mask = mask.expand(spec_target.shape[1], mask.size(0), mask.size(1)) + mask = mask.permute(1, 0, 2) + spec_pred.masked_fill_(mask, pad_value) + + mel_loss = torch.nn.functional.mse_loss(spec_pred, spec_target) + return mel_loss diff --git a/models/tacotron2/NeMo/nemo/collections/tts/losses/glow_tts_loss.py b/models/tacotron2/NeMo/nemo/collections/tts/losses/glow_tts_loss.py new file mode 100644 index 0000000000..54b8ac191f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/losses/glow_tts_loss.py @@ -0,0 +1,81 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MIT License +# +# Copyright (c) 2020 Jaehyeon Kim +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import math + +import torch + +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types.elements import * +from nemo.core.neural_types.neural_type import NeuralType + + +class GlowTTSLoss(Loss): + """ + Loss for the GlowTTS model + """ + + @property + def input_types(self): + return { + "z": NeuralType(('B', 'D', 'T'), NormalDistributionSamplesType()), + "y_m": NeuralType(('B', 'D', 'T'), NormalDistributionMeanType()), + "y_logs": NeuralType(('B', 'D', 'T'), NormalDistributionLogVarianceType()), + "logdet": NeuralType(('B',), LogDeterminantType()), + "logw": NeuralType(('B', 'T'), TokenLogDurationType()), + "logw_": NeuralType(('B', 'T'), TokenLogDurationType()), + "x_lengths": NeuralType(('B',), LengthsType()), + "y_lengths": NeuralType(('B',), LengthsType()), + } + + @property + def output_types(self): + return { + "l_mle": NeuralType(elements_type=LossType()), + "l_length": NeuralType(elements_type=LossType()), + "logdet": NeuralType(elements_type=VoidType()), + } + + @typecheck() + def forward(self, z, y_m, y_logs, logdet, logw, logw_, x_lengths, y_lengths): + + logdet = torch.sum(logdet) + l_mle = 0.5 * math.log(2 * math.pi) + ( + torch.sum(y_logs) + 0.5 * torch.sum(torch.exp(-2 * y_logs) * (z - y_m) ** 2) - logdet + ) / (torch.sum(y_lengths) * z.shape[1]) + + l_length = torch.sum((logw - logw_) ** 2) / torch.sum(x_lengths) + return l_mle, l_length, logdet diff --git a/models/tacotron2/NeMo/nemo/collections/tts/losses/hifigan_losses.py b/models/tacotron2/NeMo/nemo/collections/tts/losses/hifigan_losses.py new file mode 100644 index 0000000000..1386606b3f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/losses/hifigan_losses.py @@ -0,0 +1,132 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# MIT License +# +# Copyright (c) 2020 Jungil Kong +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# The forward functions onf the following classes are based on code from https://github.com/jik876/hifi-gan: +# FeatureMatchingLoss, DiscriminatorLoss, GeneratorLoss + +import torch + +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types.elements import LossType, VoidType +from nemo.core.neural_types.neural_type import NeuralType + + +class FeatureMatchingLoss(Loss): + """Feature Matching Loss module""" + + @property + def input_types(self): + return { + "fmap_r": [[NeuralType(elements_type=VoidType())]], + "fmap_g": [[NeuralType(elements_type=VoidType())]], + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, fmap_r, fmap_g): + loss = 0 + for dr, dg in zip(fmap_r, fmap_g): + for rl, gl in zip(dr, dg): + loss += torch.mean(torch.abs(rl - gl)) + + return loss * 2 + + +class DiscriminatorLoss(Loss): + """Discriminator Loss module""" + + @property + def input_types(self): + return { + "disc_real_outputs": [NeuralType(('B', 'T'), VoidType())], + "disc_generated_outputs": [NeuralType(('B', 'T'), VoidType())], + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + "real_losses": [NeuralType(elements_type=LossType())], + "fake_losses": [NeuralType(elements_type=LossType())], + } + + @typecheck() + def forward(self, disc_real_outputs, disc_generated_outputs): + loss = 0 + r_losses = [] + g_losses = [] + for dr, dg in zip(disc_real_outputs, disc_generated_outputs): + r_loss = torch.mean((1 - dr) ** 2) + g_loss = torch.mean(dg ** 2) + loss += r_loss + g_loss + r_losses.append(r_loss.item()) + g_losses.append(g_loss.item()) + + return loss, r_losses, g_losses + + +class GeneratorLoss(Loss): + """Generator Loss module""" + + @property + def input_types(self): + return { + "disc_outputs": [NeuralType(('B', 'T'), VoidType())], + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + "fake_losses": [NeuralType(elements_type=LossType())], + } + + @typecheck() + def forward(self, disc_outputs): + loss = 0 + gen_losses = [] + for dg in disc_outputs: + l = torch.mean((1 - dg) ** 2) + gen_losses.append(l) + loss += l + + return loss, gen_losses diff --git a/models/tacotron2/NeMo/nemo/collections/tts/losses/stftlosses.py b/models/tacotron2/NeMo/nemo/collections/tts/losses/stftlosses.py new file mode 100644 index 0000000000..1c664c3fcc --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/losses/stftlosses.py @@ -0,0 +1,248 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The MIT License +# +# Copyright (c) 2019 Tomoki Hayashi +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +# The following functions/classes were based on: +# https://github.com/kan-bayashi/ParallelWaveGAN/blob/master/parallel_wavegan/losses/stft_loss.py +# stft +# SpectralConvergenceLoss +# LogSTFTMagnitudeLoss +# STFTLoss +# MultiResolutionSTFTLoss + +import torch +import torch.nn.functional as F + +from nemo.collections.common.parts.patch_utils import stft_patch +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types.elements import AudioSignal, LengthsType, LossType, SpectrogramType +from nemo.core.neural_types.neural_type import NeuralType + + +def stft(x, fft_size, hop_size, win_length, window): + """Perform STFT and convert to magnitude spectrogram. + Args: + x (Tensor): Input signal tensor (B, T). + fft_size (int): FFT size. + hop_size (int): Hop size. + win_length (int): Window length. + window (str): Window function type. + Returns: + Tensor: Magnitude spectrogram (B, #frames, fft_size // 2 + 1). + """ + x_stft = stft_patch(x, fft_size, hop_size, win_length, window, return_complex=False) + real = x_stft[..., 0] + imag = x_stft[..., 1] + + # NOTE(kan-bayashi): clamp is needed to avoid nan or inf + return torch.sqrt(torch.clamp(real ** 2 + imag ** 2, min=1e-7)).transpose(2, 1) + + +class SpectralConvergenceLoss(Loss): + """Spectral convergence loss module.""" + + @property + def input_types(self): + return { + "x_mag": NeuralType(('B', 'T', 'D'), SpectrogramType()), + "y_mag": NeuralType(('B', 'T', 'D'), SpectrogramType()), + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, *, x_mag, y_mag): + """Calculate forward propagation. It is assumed that x_mag and y_mag were padded to fit the maximum batch + sequence length with silence, hence it is assumed that the norm of these extra padded values are 0. Therefore, + input_lengths is not a argument unlike in LogSTFTMagnitudeLoss. + Args: + x_mag (Tensor): Magnitude spectrogram of predicted signal (B, #frames, #freq_bins). + y_mag (Tensor): Magnitude spectrogram of groundtruth signal (B, #frames, #freq_bins). + Returns: + Tensor: Spectral convergence loss value. + """ + # Mean across time and freq_bins first + loss = torch.norm(y_mag - x_mag, p="fro", dim=(1, 2)) / torch.norm(y_mag, p="fro", dim=(1, 2)) + # Mean across batches + loss = torch.mean(loss) + return loss + + +class LogSTFTMagnitudeLoss(Loss): + """Log STFT magnitude loss module.""" + + @property + def input_types(self): + return { + "x_mag": NeuralType(('B', 'T', 'D'), SpectrogramType()), + "y_mag": NeuralType(('B', 'T', 'D'), SpectrogramType()), + "input_lengths": NeuralType(('B'), LengthsType(), optional=True), + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, *, x_mag, y_mag, input_lengths=None): + """Calculate forward propagation. + Args: + x_mag (Tensor): Magnitude spectrogram of predicted signal (B, #frames, #freq_bins). + y_mag (Tensor): Magnitude spectrogram of groundtruth signal (B, #frames, #freq_bins). + input_lengths (Tensor): Length of groundtruth sample in samples (B). + Returns: + Tensor: Log STFT magnitude loss value. + """ + if input_lengths is None: + # During training, we used fixed sequence length, so just average across all dimensions + return F.l1_loss(torch.log(y_mag), torch.log(x_mag)) + loss = F.l1_loss(torch.log(y_mag), torch.log(x_mag), reduction='none') + # First sum and average across time and freq bins + loss = loss / loss.shape[2] + loss = torch.sum(loss, dim=[1, 2]) + loss = loss / input_lengths + # Last average across batch + return torch.sum(loss) / loss.shape[0] + + +class STFTLoss(Loss): + """STFT loss module.""" + + def __init__(self, fft_size=1024, shift_size=120, win_length=600, window="hann_window"): + """Initialize STFT loss module.""" + super(STFTLoss, self).__init__() + self.fft_size = fft_size + self.shift_size = shift_size + self.win_length = win_length + self.window = getattr(torch, window)(win_length) + self.spectral_convergence_loss = SpectralConvergenceLoss() + self.log_stft_magnitude_loss = LogSTFTMagnitudeLoss() + + @property + def input_types(self): + return { + "x": NeuralType(('B', 'T'), AudioSignal()), + "y": NeuralType(('B', 'T'), AudioSignal()), + "input_lengths": NeuralType(('B'), LengthsType(), optional=True), + } + + @property + def output_types(self): + return { + "sc_loss": NeuralType(elements_type=LossType()), + "mag_loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, *, x, y, input_lengths=None): + """Calculate forward propagation. + Args: + x (Tensor): Predicted signal (B, T). + y (Tensor): Groundtruth signal (B, T). + input_lengths (Tensor): Length of groundtruth sample in samples (B). + Returns: + Tensor: Spectral convergence loss value. + Tensor: Log STFT magnitude loss value. + """ + if self.window.device != x.device: + self.window = self.window.to(x.device) + x_mag = stft(x, self.fft_size, self.shift_size, self.win_length, self.window) + y_mag = stft(y, self.fft_size, self.shift_size, self.win_length, self.window) + sc_loss = self.spectral_convergence_loss(x_mag=x_mag, y_mag=y_mag) + if input_lengths is not None: + input_lengths = torch.floor(input_lengths / float(self.shift_size)) + 1 + assert max(input_lengths) == x_mag.shape[1], f"{max(input_lengths)} != {x_mag.shape[1]}" + mag_loss = self.log_stft_magnitude_loss(x_mag=x_mag, y_mag=y_mag, input_lengths=input_lengths) + + return sc_loss, mag_loss + + +class MultiResolutionSTFTLoss(Loss): + """Multi resolution STFT loss module.""" + + def __init__(self, fft_sizes, hop_sizes, win_lengths, window="hann_window"): + """Initialize Multi resolution STFT loss module. + Args: + fft_sizes (list): List of FFT sizes. + hop_sizes (list): List of hop sizes. + win_lengths (list): List of window lengths. + window (str): Window function type. + """ + super(MultiResolutionSTFTLoss, self).__init__() + assert len(fft_sizes) == len(hop_sizes) == len(win_lengths) + self.stft_losses = torch.nn.ModuleList() + for fs, ss, wl in zip(fft_sizes, hop_sizes, win_lengths): + self.stft_losses += [STFTLoss(fs, ss, wl, window)] + + @property + def input_types(self): + return { + "x": NeuralType(('B', 'T'), AudioSignal()), + "y": NeuralType(('B', 'T'), AudioSignal()), + "input_lengths": NeuralType(('B'), LengthsType(), optional=True), + } + + @property + def output_types(self): + return { + "sc_loss": [NeuralType(elements_type=LossType())], + "mag_loss": [NeuralType(elements_type=LossType())], + } + + @typecheck() + def forward(self, *, x, y, input_lengths=None): + """Calculate forward propagation. + Args: + x (Tensor): Predicted signal (B, T). + y (Tensor): Groundtruth signal (B, T). + input_lengths (Tensor): Length of groundtruth sample in samples (B). + Returns: + List[Tensor]: Multi resolution spectral convergence loss value. + List[Tensor]: Multi resolution log STFT magnitude loss value. + """ + sc_loss = [0.0] * len(self.stft_losses) + mag_loss = [0.0] * len(self.stft_losses) + for i, f in enumerate(self.stft_losses): + sc_l, mag_l = f(x=x, y=y, input_lengths=input_lengths) + sc_loss[i] = sc_l + mag_loss[i] = mag_l + + return sc_loss, mag_loss diff --git a/models/tacotron2/NeMo/nemo/collections/tts/losses/tacotron2loss.py b/models/tacotron2/NeMo/nemo/collections/tts/losses/tacotron2loss.py new file mode 100644 index 0000000000..2645aa7cb8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/losses/tacotron2loss.py @@ -0,0 +1,84 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from nemo.collections.tts.helpers.helpers import get_mask_from_lengths +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types.elements import LengthsType, LogitsType, LossType, MelSpectrogramType +from nemo.core.neural_types.neural_type import NeuralType + + +class Tacotron2Loss(Loss): + """A Loss module that computes loss for Tacotron2""" + + @property + def input_types(self): + return { + "spec_pred_dec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "spec_pred_postnet": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "gate_pred": NeuralType(('B', 'T'), LogitsType()), + "spec_target": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "spec_target_len": NeuralType(('B'), LengthsType()), + "pad_value": NeuralType(), + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + "gate_target": NeuralType(('B', 'T'), LogitsType()), # Used for evaluation + } + + @typecheck() + def forward(self, *, spec_pred_dec, spec_pred_postnet, gate_pred, spec_target, spec_target_len, pad_value): + # Make the gate target + max_len = spec_target.shape[2] + gate_target = torch.zeros(spec_target_len.shape[0], max_len) + gate_target = gate_target.type_as(gate_pred) + for i, length in enumerate(spec_target_len): + gate_target[i, length.data - 1 :] = 1 + + spec_target.requires_grad = False + gate_target.requires_grad = False + gate_target = gate_target.view(-1, 1) + + max_len = spec_target.shape[2] + + if max_len < spec_pred_dec.shape[2]: + # Predicted len is larger than reference + # Need to slice + spec_pred_dec = spec_pred_dec.narrow(2, 0, max_len) + spec_pred_postnet = spec_pred_postnet.narrow(2, 0, max_len) + gate_pred = gate_pred.narrow(1, 0, max_len).contiguous() + elif max_len > spec_pred_dec.shape[2]: + # Need to do padding + pad_amount = max_len - spec_pred_dec.shape[2] + spec_pred_dec = torch.nn.functional.pad(spec_pred_dec, (0, pad_amount), value=pad_value) + spec_pred_postnet = torch.nn.functional.pad(spec_pred_postnet, (0, pad_amount), value=pad_value) + gate_pred = torch.nn.functional.pad(gate_pred, (0, pad_amount), value=1e3) + max_len = spec_pred_dec.shape[2] + + mask = ~get_mask_from_lengths(spec_target_len, max_len=max_len) + mask = mask.expand(spec_target.shape[1], mask.size(0), mask.size(1)) + mask = mask.permute(1, 0, 2) + spec_pred_dec.data.masked_fill_(mask, pad_value) + spec_pred_postnet.data.masked_fill_(mask, pad_value) + gate_pred.data.masked_fill_(mask[:, 0, :], 1e3) + + gate_pred = gate_pred.view(-1, 1) + rnn_mel_loss = torch.nn.functional.mse_loss(spec_pred_dec, spec_target) + postnet_mel_loss = torch.nn.functional.mse_loss(spec_pred_postnet, spec_target) + gate_loss = torch.nn.functional.binary_cross_entropy_with_logits(gate_pred, gate_target) + return rnn_mel_loss + postnet_mel_loss + gate_loss, gate_target diff --git a/models/tacotron2/NeMo/nemo/collections/tts/losses/uniglowloss.py b/models/tacotron2/NeMo/nemo/collections/tts/losses/uniglowloss.py new file mode 100644 index 0000000000..af3ed66b1f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/losses/uniglowloss.py @@ -0,0 +1,58 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from nemo.collections.tts.losses.stftlosses import MultiResolutionSTFTLoss +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types.elements import AudioSignal, LossType, NormalDistributionSamplesType, VoidType +from nemo.core.neural_types.neural_type import NeuralType + + +class UniGlowLoss(Loss): + """A Loss module that computes loss for UniGlow""" + + def __init__(self, stft_loss_coef): + super(UniGlowLoss, self).__init__() + self.stft_loss = MultiResolutionSTFTLoss( + fft_sizes=[1024, 2048, 512], hop_sizes=[120, 240, 50], win_lengths=[600, 1200, 240], window="hann_window" + ) + self.stft_loss_coef = stft_loss_coef + + @property + def input_types(self): + return { + "z": NeuralType(('B', 'flowgroup', 'T'), NormalDistributionSamplesType()), + "logdet": NeuralType(elements_type=VoidType()), + "gt_audio": NeuralType(('B', 'T'), AudioSignal()), + "predicted_audio": NeuralType(('B', 'T'), AudioSignal()), + "sigma": NeuralType(optional=True), + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, *, z, logdet, gt_audio, predicted_audio, sigma=1.0): + nll_loss = torch.sum(z * z) / (2 * sigma * sigma) - logdet + nll_loss = nll_loss / (z.size(0) * z.size(1) * z.size(2)) + sc_loss, mag_loss = self.stft_loss(predicted_audio, gt_audio) + sc_loss = sum(sc_loss) / len(sc_loss) + mag_loss = sum(mag_loss) / len(mag_loss) + stft_loss = sc_loss + mag_loss + loss = nll_loss + self.stft_loss_coef * stft_loss + return loss diff --git a/models/tacotron2/NeMo/nemo/collections/tts/losses/waveglowloss.py b/models/tacotron2/NeMo/nemo/collections/tts/losses/waveglowloss.py new file mode 100644 index 0000000000..74e45f5076 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/losses/waveglowloss.py @@ -0,0 +1,51 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +from nemo.core.classes import Loss, typecheck +from nemo.core.neural_types.elements import LossType, NormalDistributionSamplesType, VoidType +from nemo.core.neural_types.neural_type import NeuralType + + +class WaveGlowLoss(Loss): + """ A Loss module that computes loss for WaveGlow + """ + + @property + def input_types(self): + return { + "z": NeuralType(('B', 'flowgroup', 'T'), NormalDistributionSamplesType()), + "log_s_list": [NeuralType(('B', 'flowgroup', 'T'), VoidType())], # TODO: Figure out a good typing + "log_det_W_list": [NeuralType(elements_type=VoidType())], # TODO: Figure out a good typing + "sigma": NeuralType(optional=True), + } + + @property + def output_types(self): + return { + "loss": NeuralType(elements_type=LossType()), + } + + @typecheck() + def forward(self, *, z, log_s_list, log_det_W_list, sigma=1.0): + for i, log_s in enumerate(log_s_list): + if i == 0: + log_s_total = torch.sum(log_s) + log_det_W_total = log_det_W_list[i] + else: + log_s_total = log_s_total + torch.sum(log_s) + log_det_W_total += log_det_W_list[i] + + loss = torch.sum(z * z) / (2 * sigma * sigma) - log_s_total - log_det_W_total + return loss / (z.size(0) * z.size(1) * z.size(2)) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/__init__.py b/models/tacotron2/NeMo/nemo/collections/tts/models/__init__.py new file mode 100644 index 0000000000..dd9ac60599 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/__init__.py @@ -0,0 +1,53 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.tts.models.aligner import AlignerModel +from nemo.collections.tts.models.degli import DegliModel +from nemo.collections.tts.models.ed_mel2spec import EDMel2SpecModel +from nemo.collections.tts.models.fastpitch import FastPitchModel +from nemo.collections.tts.models.fastpitch_hifigan_e2e import FastPitchHifiGanE2EModel +from nemo.collections.tts.models.fastspeech2 import FastSpeech2Model +from nemo.collections.tts.models.fastspeech2_hifigan_e2e import FastSpeech2HifiGanE2EModel +from nemo.collections.tts.models.glow_tts import GlowTTSModel +from nemo.collections.tts.models.hifigan import HifiGanModel +from nemo.collections.tts.models.melgan import MelGanModel +from nemo.collections.tts.models.squeezewave import SqueezeWaveModel +from nemo.collections.tts.models.tacotron2 import Tacotron2Model +from nemo.collections.tts.models.talknet import TalkNetDursModel, TalkNetPitchModel, TalkNetSpectModel +from nemo.collections.tts.models.two_stages import GriffinLimModel, MelPsuedoInverseModel, TwoStagesModel +from nemo.collections.tts.models.uniglow import UniGlowModel +from nemo.collections.tts.models.waveglow import WaveGlowModel + +__all__ = [ + "GlowTTSModel", + "SqueezeWaveModel", + "Tacotron2Model", + "WaveGlowModel", + "DegliModel", + "EDMel2SpecModel", + "TwoStagesModel", + "GriffinLimModel", + "MelPsuedoInverseModel", + "TalkNetDursModel", + "TalkNetPitchModel", + "TalkNetSpectModel", + "UniGlowModel", + "MelGanModel", + "HifiGanModel", + "FastPitchModel", + "FastSpeech2Model", + "FastPitchHifiGanE2EModel", + "FastSpeech2HifiGanE2EModel", + "AlignerModel", +] diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/aligner.py b/models/tacotron2/NeMo/nemo/collections/tts/models/aligner.py new file mode 100644 index 0000000000..efa0b95735 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/aligner.py @@ -0,0 +1,157 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import torch +from hydra.utils import instantiate +from omegaconf import DictConfig +from pytorch_lightning import Trainer +from pytorch_lightning.loggers import WandbLogger +from torch import nn + +from nemo.collections.asr.data.audio_to_text import AudioToCharWithDursF0Dataset +from nemo.collections.tts.helpers.helpers import binarize_attention, get_mask_from_lengths, plot_alignment_to_numpy +from nemo.collections.tts.losses.aligner_loss import BinLoss, ForwardSumLoss +from nemo.core.classes import ModelPT +from nemo.core.classes.common import typecheck +from nemo.utils import logging + +HAVE_WANDB = True +try: + import wandb +except ModuleNotFoundError: + HAVE_WANDB = False + + +class AlignerModel(ModelPT): + """Speech-to-text alignment pipeline.""" + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + super().__init__(cfg=cfg, trainer=trainer) + typecheck.set_typecheck_enabled(enabled=False) + + cfg = self._cfg + self.vocab = AudioToCharWithDursF0Dataset.make_vocab(**cfg.train_ds.dataset.vocab) + self.embed = nn.Embedding(len(self.vocab.labels), cfg.d_char) + self.preprocessor = instantiate(cfg.preprocessor) + self.alignment_encoder = instantiate(cfg.alignment_encoder) + + self.forward_sum_loss = ForwardSumLoss() + self.bin_loss = BinLoss() + + self.bin_start_ratio = cfg.bin_start_ratio + self.add_bin_loss = False + + def forward(self, *, spec, spec_len, text, text_len, attn_prior=None): + with torch.cuda.amp.autocast(enabled=False): + attn_soft, attn_logprob = self.alignment_encoder( + queries=spec, + keys=self.embed(text).transpose(1, 2), + mask=get_mask_from_lengths(text_len).unsqueeze(-1) == 0, + attn_prior=attn_prior, + ) + + return attn_soft, attn_logprob + + def _metrics(self, attn_soft, attn_logprob, spec_len, text_len): + loss, bin_loss, attn_hard = 0.0, 0.0, None + + forward_sum_loss = self.forward_sum_loss(attn_logprob, text_len, spec_len) + loss += forward_sum_loss + + if self.add_bin_loss: + attn_hard = binarize_attention(attn_soft, text_len, spec_len) + bin_loss = self.bin_loss(attn_hard, attn_soft) + loss += bin_loss + + return loss, forward_sum_loss, bin_loss, attn_hard + + def on_train_epoch_start(self): + # Add bin loss when current_epoch >= bin_start_ratio * max_epochs + if not self.add_bin_loss and self.current_epoch >= np.ceil(self.bin_start_ratio * self._trainer.max_epochs): + logging.info(f"Using hard attentions after epoch: {self.current_epoch}") + self.add_bin_loss = True + + def training_step(self, batch, batch_idx): + audio, audio_len, text, text_len, attn_prior = batch + spec, spec_len = self.preprocessor(audio, audio_len) + attn_soft, attn_logprob = self( + spec=spec, spec_len=spec_len, text=text, text_len=text_len, attn_prior=attn_prior + ) + + loss, forward_sum_loss, bin_loss, _ = self._metrics(attn_soft, attn_logprob, spec_len, text_len) + + train_log = {'train_forward_sum_loss': forward_sum_loss, 'train_bin_loss': bin_loss} + return {'loss': loss, 'progress_bar': train_log, 'log': train_log} + + def validation_step(self, batch, batch_idx): + audio, audio_len, text, text_len, attn_prior = batch + spec, spec_len = self.preprocessor(audio, audio_len) + attn_soft, attn_logprob = self( + spec=spec, spec_len=spec_len, text=text, text_len=text_len, attn_prior=attn_prior + ) + + loss, forward_sum_loss, bin_loss, attn_hard = self._metrics(attn_soft, attn_logprob, spec_len, text_len) + + # plot once per epoch + if batch_idx == 0 and isinstance(self.logger, WandbLogger) and HAVE_WANDB: + if attn_hard is None: + attn_hard = binarize_attention(attn_soft, text_len, spec_len) + + attn_matrices = [] + for i in range(min(5, audio.shape[0])): + attn_matrices.append( + wandb.Image( + plot_alignment_to_numpy( + np.fliplr(np.rot90(attn_soft[i, 0, : spec_len[i], : text_len[i]].data.cpu().numpy())) + ), + caption=f"attn soft", + ), + ) + + attn_matrices.append( + wandb.Image( + plot_alignment_to_numpy( + np.fliplr(np.rot90(attn_hard[i, 0, : spec_len[i], : text_len[i]].data.cpu().numpy())) + ), + caption=f"attn hard", + ) + ) + + self.logger.experiment.log({"attn_matrices": attn_matrices}) + + val_log = {'val_loss': loss, 'val_forward_sum_loss': forward_sum_loss, 'val_bin_loss': bin_loss} + self.log_dict(val_log, prog_bar=False, on_epoch=True, logger=True, sync_dist=True) + + @staticmethod + def _loader(cfg): + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader( # noqa + dataset=dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params, + ) + + def setup_training_data(self, cfg): + self._train_dl = self._loader(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self._loader(cfg) + + def setup_test_data(self, cfg): + """Omitted.""" + pass + + @classmethod + def list_available_models(cls): + """Empty.""" + pass diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/base.py b/models/tacotron2/NeMo/nemo/collections/tts/models/base.py new file mode 100644 index 0000000000..ea96ad4125 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/base.py @@ -0,0 +1,305 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import ABC, abstractmethod +from contextlib import ExitStack, contextmanager + +import torch +from torch_stft import STFT + +from nemo.collections.common.parts.patch_utils import istft_patch, stft_patch +from nemo.collections.tts.helpers.helpers import OperationMode +from nemo.collections.tts.models import * # Avoid circular imports +from nemo.core.classes import ModelPT +from nemo.core.classes.common import typecheck +from nemo.core.neural_types.elements import AudioSignal +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +class SpectrogramGenerator(ModelPT, ABC): + """ Base class for all TTS models that turn text into a spectrogram """ + + @abstractmethod + def parse(self, str_input: str, **kwargs) -> 'torch.tensor': + """ + A helper function that accepts raw python strings and turns them into a tensor. The tensor should have 2 + dimensions. The first is the batch, which should be of size 1. The second should represent time. The tensor + should represent either tokenized or embedded text, depending on the model. + """ + + @abstractmethod + def generate_spectrogram(self, tokens: 'torch.tensor', **kwargs) -> 'torch.tensor': + """ + Accepts a batch of text or text_tokens and returns a batch of spectrograms + + Args: + tokens: A torch tensor representing the text to be generated + + Returns: + spectrograms + """ + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + for subclass in cls.__subclasses__(): + subclass_models = subclass.list_available_models() + if subclass_models is not None and len(subclass_models) > 0: + list_of_models.extend(subclass_models) + return list_of_models + + +class Vocoder(ModelPT, ABC): + """ Base class for all TTS models that generate audio conditioned a on spectrogram """ + + @abstractmethod + def convert_spectrogram_to_audio(self, spec: 'torch.tensor', **kwargs) -> 'torch.tensor': + """ + Accepts a batch of spectrograms and returns a batch of audio + + Args: + spec: A torch tensor representing the spectrograms to be vocoded + + Returns: + audio + """ + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + for subclass in cls.__subclasses__(): + subclass_models = subclass.list_available_models() + if subclass_models is not None and len(subclass_models) > 0: + list_of_models.extend(subclass_models) + return list_of_models + + +class GlowVocoder(Vocoder): + """ Base class for all Vocoders that use a Glow or reversible Flow-based setup. All child class are expected + to have a parameter called audio_to_melspec_precessor that is an instance of + nemo.collections.asr.parts.FilterbankFeatures""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._mode = OperationMode.infer + self.stft = None + self.istft = None + self.n_mel = None + self.bias_spect = None + + @property + def mode(self): + return self._mode + + @contextmanager + def temp_mode(self, mode): + old_mode = self.mode + self.mode = mode + try: + yield + finally: + self.mode = old_mode + + @contextmanager + def nemo_infer(self): # Prepend with nemo to avoid any .infer() clashes with lightning or pytorch + with ExitStack() as stack: + stack.enter_context(self.temp_mode(OperationMode.infer)) + stack.enter_context(torch.no_grad()) + yield + + def check_children_attributes(self): + if self.stft is None: + if isinstance(self.audio_to_melspec_precessor.stft, STFT): + logging.warning( + "torch_stft is deprecated. Please change your model to use torch.stft and torch.istft instead." + ) + self.stft = self.audio_to_melspec_precessor.stft.transform + self.istft = self.audio_to_melspec_precessor.stft.inverse + else: + try: + n_fft = self.audio_to_melspec_precessor.n_fft + hop_length = self.audio_to_melspec_precessor.hop_length + win_length = self.audio_to_melspec_precessor.win_length + window = self.audio_to_melspec_precessor.window.to(self.device) + except AttributeError as e: + raise AttributeError( + f"{self} could not find a valid audio_to_melspec_precessor. GlowVocoder requires child class " + "to have audio_to_melspec_precessor defined to obtain stft parameters. " + "audio_to_melspec_precessor requires n_fft, hop_length, win_length, window, and nfilt to be " + "defined." + ) from e + + def yet_another_patch(audio, n_fft, hop_length, win_length, window): + spec = stft_patch(audio, n_fft=n_fft, hop_length=hop_length, win_length=win_length, window=window) + if spec.dtype in [torch.cfloat, torch.cdouble]: + spec = torch.view_as_real(spec) + return torch.sqrt(spec.pow(2).sum(-1)), torch.atan2(spec[..., -1], spec[..., 0]) + + self.stft = lambda x: yet_another_patch( + x, n_fft=n_fft, hop_length=hop_length, win_length=win_length, window=window, + ) + self.istft = lambda x, y: istft_patch( + torch.complex(x * torch.cos(y), x * torch.sin(y)), + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=window, + ) + + if self.n_mel is None: + try: + self.n_mel = self.audio_to_melspec_precessor.nfilt + except AttributeError as e: + raise AttributeError( + f"{self} could not find a valid audio_to_melspec_precessor. GlowVocoder requires child class to " + "have audio_to_melspec_precessor defined to obtain stft parameters. audio_to_melspec_precessor " + "requires nfilt to be defined." + ) from e + + def update_bias_spect(self): + self.check_children_attributes() # Ensure stft parameters are defined + + with self.nemo_infer(): + spect = torch.zeros((1, self.n_mel, 88)).to(self.device) + bias_audio = self.convert_spectrogram_to_audio(spec=spect, sigma=0.0, denoise=False) + bias_spect, _ = self.stft(bias_audio) + self.bias_spect = bias_spect[..., 0][..., None] + + @typecheck( + input_types={"audio": NeuralType(('B', 'T'), AudioSignal()), "strength": NeuralType(optional=True)}, + output_types={"audio": NeuralType(('B', 'T'), AudioSignal())}, + ) + def denoise(self, audio: 'torch.tensor', strength: float = 0.01): + self.check_children_attributes() # Ensure self.n_mel and self.stft are defined + + if self.bias_spect is None: + self.update_bias_spect() + audio_spect, audio_angles = self.stft(audio) + audio_spect_denoised = audio_spect - self.bias_spect.to(audio.device) * strength + audio_spect_denoised = torch.clamp(audio_spect_denoised, 0.0) + audio_denoised = self.istft(audio_spect_denoised, audio_angles) + return audio_denoised + + +class LinVocoder(ModelPT, ABC): + """ + A base class for models that convert from the linear (magnitude) spectrogram to audio. Note: The `Vocoder` class + differs from this class as the `Vocoder` class takes as input mel spectrograms. + """ + + @abstractmethod + def convert_linear_spectrogram_to_audio(self, spec: 'torch.tensor', **kwargs) -> 'torch.tensor': + """ + Accepts a batch of linear spectrograms and returns a batch of audio + + Args: + spec: A torch tensor representing the linear spectrograms to be vocoded ['B', 'n_freqs', 'T'] + + Returns: + audio + """ + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + for subclass in cls.__subclasses__(): + subclass_models = subclass.list_available_models() + if subclass_models is not None and len(subclass_models) > 0: + list_of_models.extend(subclass_models) + return list_of_models + + +class MelToSpec(ModelPT, ABC): + """ + A base class for models that convert mel spectrograms to linear (magnitude) spectrograms + """ + + @abstractmethod + def convert_mel_spectrogram_to_linear(self, mel: 'torch.tensor', **kwargs) -> 'torch.tensor': + """ + Accepts a batch of spectrograms and returns a batch of linear spectrograms + + Args: + mel: A torch tensor representing the mel spectrograms ['B', 'mel_freqs', 'T'] + + Returns: + spec: A torch tensor representing the linear spectrograms ['B', 'n_freqs', 'T'] + """ + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + for subclass in cls.__subclasses__(): + subclass_models = subclass.list_available_models() + if subclass_models is not None and len(subclass_models) > 0: + list_of_models.extend(subclass_models) + return list_of_models + + +class TextToWaveform(ModelPT, ABC): + """ Base class for all end-to-end TTS models that generate a waveform from text """ + + @abstractmethod + def parse(self, str_input: str, **kwargs) -> 'torch.tensor': + """ + A helper function that accepts raw python strings and turns them into a tensor. The tensor should have 2 + dimensions. The first is the batch, which should be of size 1. The second should represent time. The tensor + should represent either tokenized or embedded text, depending on the model. + """ + + @abstractmethod + def convert_text_to_waveform(self, *, tokens: 'torch.tensor', **kwargs) -> 'List[torch.tensor]': + """ + Accepts a batch of text and returns a list containing a batch of audio + + Args: + tokens: A torch tensor representing the text to be converted to speech + + Returns: + audio: A list of length batch_size containing torch tensors representing the waveform output + """ + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + for subclass in cls.__subclasses__(): + subclass_models = subclass.list_available_models() + if subclass_models is not None and len(subclass_models) > 0: + list_of_models.extend(subclass_models) + return list_of_models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/degli.py b/models/tacotron2/NeMo/nemo/collections/tts/models/degli.py new file mode 100755 index 0000000000..7b871405ad --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/degli.py @@ -0,0 +1,330 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MIT License + +# Copyright (c) 2019 Jeongmin Liu + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import warnings +from dataclasses import dataclass +from typing import Any, Dict, Optional, Sequence + +import librosa +import numpy as np +import soundfile as sf +import torch +from hydra.utils import instantiate +from numpy import ndarray +from omegaconf import MISSING, DictConfig, OmegaConf, open_dict +from torch import Tensor, nn + +from nemo.collections.tts.helpers.helpers import eval_tts_scores +from nemo.collections.tts.models.base import LinVocoder +from nemo.collections.tts.modules.degli import OperationMode +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import IntType, LengthsType, SpectrogramType +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +@dataclass +class DegliConfig: + degli: Dict[Any, Any] = MISSING + train_ds: Optional[Dict[Any, Any]] = None + validation_ds: Optional[Dict[Any, Any]] = None + train_params: Optional[Dict[Any, Any]] = None + sched: Optional[Dict[Any, Any]] = None + + +def reconstruct_wave(*args: ndarray, kwargs_istft, n_sample=-1) -> ndarray: + """ + construct time-domain wave from complex spectrogram + Args: + *args: the complex spectrogram. + kwargs_istft: arguments of Inverse STFT. + n_sample: expected audio length. + Returns: + audio (numpy) + """ + + if len(args) == 1: + spec = args[0].squeeze() + mag = None + phase = None + assert np.iscomplexobj(spec) + elif len(args) == 2: + spec = None + mag = args[0].squeeze() + phase = args[1].squeeze() + assert np.isrealobj(mag) and np.isrealobj(phase) + else: + raise ValueError + + kwarg_len = dict(length=n_sample) if n_sample != -1 else dict() + if spec is None: + spec = mag * np.exp(1j * phase) + + wave = librosa.istft(spec, **kwargs_istft, **kwarg_len) + return wave + + +class DegliModel(LinVocoder): + """Deep Griffin Lim model used to convert between spectrograms and audio""" + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + super().__init__(cfg=cfg, trainer=trainer) + + schema = OmegaConf.structured(DegliConfig) + # ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + # Ensure passed cfg is compliant with schema + OmegaConf.merge(cfg, schema) + + self.degli = instantiate(self._cfg.degli) + self.mode = OperationMode.infer + self.criterion = nn.L1Loss(reduction='none') # maybe should be loss subclass? + self.l_hop = self._cfg.degli.hop_length + self.n_fft = self._cfg.degli.n_fft + self.kwargs_stft = dict(hop_length=self.l_hop, window='hann', center=True, n_fft=self.n_fft, dtype=np.float32) + self.kwargs_istft = dict(hop_length=self.l_hop, window='hann', center=True, dtype=np.float32) + + len_weight = self._cfg.train_params.repeat_training + self.loss_weight = nn.Parameter(torch.tensor([1.0 / i for i in range(len_weight, 0, -1)]), requires_grad=False) + self.loss_weight /= self.loss_weight.sum() + + @property + def mode(self): + return self._mode + + @mode.setter + def mode(self, new_mode): + if new_mode == OperationMode.training: + self.train() + else: + self.eval() + self._mode = new_mode + self.degli.mode = new_mode + + @property + def input_types(self): + return { + "x": NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + "mag": NeuralType(('B', 'any', 'D', 'T'), SpectrogramType()), + "max_length": NeuralType(None, LengthsType()), + "repeats": NeuralType(None, IntType()), + } + + @property + def output_types(self): + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + return { + "out_repeats": NeuralType(('B', 'any', 'C', 'D', 'T'), SpectrogramType()), + "final_out": NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + "residual": NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + } + else: + return { + "final_out": NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + } + + @typecheck() + def forward(self, *, x, mag, max_length, repeats): + if self.mode != self.degli.mode: + raise ValueError(f"Degli's mode {self.mode} does not match DegliModule's mode {self.degli.mode}") + + tensors = self.degli(x=x, mag=mag, max_length=max_length, repeat=repeats) + return tensors # audio_pred + + def convert_linear_spectrogram_to_audio(self, spec: torch.Tensor, Ts=None, repeats: int = 32) -> torch.Tensor: + self.mode = OperationMode.infer + + batch_size = spec.shape[0] + if len(spec.shape) == 3: + spec = spec.unsqueeze(1) + + x = torch.normal(0, 1, [batch_size, 2, spec.shape[2], spec.shape[3]]).to(self.device) + length = (spec.shape[3] - 1) * self.l_hop + with torch.no_grad(): + y = self.degli(x=x, mag=spec, max_length=length, repeat=repeats) + + if Ts is None: + Ts = [y.shape[3]] * batch_size + + max_size = (max(Ts) - 1) * self.l_hop + + audios = torch.zeros(batch_size, max_size) + + for i in range(batch_size): + y_i = self.postprocess(y, Ts, i) + my_len = (Ts[i] - 1) * self.l_hop + audio = reconstruct_wave(y_i, kwargs_istft=self.kwargs_istft, n_sample=my_len) + audios[i, 0:my_len] = torch.from_numpy(audio) + + return audios + + def calc_loss(self, out_blocks: Tensor, y: Tensor, T_ys: Sequence[int]) -> Tensor: + """ + calculate L1 loss (criterion) between the real spectrogram and the outputs. + + Args: + out_blocks: output for the Degli model, may include several concatrated outputs. + y: desired output. + T_ys: lengths (time domain) of non-zero parts of the histrograms, for each sample in the batch. + Returns: + a loss score. + """ + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + loss_no_red = self.criterion(out_blocks, y.unsqueeze(1)) + loss_blocks = torch.zeros(out_blocks.shape[1], device=y.device) + for T, loss_batch in zip(T_ys, loss_no_red): + loss_blocks += torch.mean(loss_batch[..., :T], dim=(1, 2, 3)) + + if len(loss_blocks) == 1: + loss = loss_blocks.squeeze() + else: + loss = loss_blocks @ self.loss_weight + return loss + + def training_step(self, batch, batch_idx): + self.mode = OperationMode.infer + + x, mag, max_length, y, T_ys, _, _ = batch + output_loss, _, _ = self(x=x, mag=mag, max_length=max_length, repeats=self._cfg.train_params.repeat_training) + loss = self.calc_loss(output_loss, y, T_ys) + + output = { + 'loss': loss, + 'progress_bar': {'training_loss': loss}, + 'log': {'loss': loss}, + } + return output + + @torch.no_grad() + def postprocess(self, output: Tensor, Ts: ndarray, idx: int): + one = output[idx, :, :, : Ts[idx]] + one = one.permute(1, 2, 0).contiguous() # F, T, 2 + one = one.cpu().numpy().view(dtype=np.complex64) # F, T, 1 + return one + + def validation_step(self, batch, batch_idx): + """ + A validation step that also calculates the STOI/PESQ scores, + and the scored for high repetition count (repeat_validation argument) + """ + self.mode = OperationMode.infer + val_repeats = self._cfg.train_params.repeat_validation + + x, mag, max_length, y, T_ys, length, path_speech = batch + output_loss, output, _ = self(x=x, mag=mag, max_length=max_length, repeats=1) + _, output_x, _ = self(x=x, mag=mag, max_length=max_length, repeats=val_repeats) + + loss = self.calc_loss(output_loss, y, T_ys) + cnt = x.shape[0] + + stoi, pesq, stoi_x, pesq_x = (0.0, 0.0, 0.0, 0.0) + for p in range(cnt): + y_wav_path = path_speech[p] + y_wav = sf.read(y_wav_path)[0].astype(np.float32) + + n_sample = length[p] + out = self.postprocess(output, T_ys, p) + out_wav = reconstruct_wave(out, kwargs_istft=self.kwargs_istft, n_sample=n_sample) + measure = eval_tts_scores(y_wav, out_wav) + stoi += torch.tensor(measure['STOI']) + pesq += torch.tensor(measure['PESQ']) + + out = self.postprocess(output_x, T_ys, p) + out_wav = reconstruct_wave(out, kwargs_istft=self.kwargs_istft, n_sample=n_sample) + measure = eval_tts_scores(y_wav, out_wav) + stoi_x += torch.tensor(measure['STOI']) + pesq_x += torch.tensor(measure['PESQ']) + + return { + "val_loss": loss, + "stoi": stoi / cnt, + "pesq": pesq / cnt, + "stoi_x%d" % val_repeats: stoi_x / cnt, + "pesq_x%d" % val_repeats: pesq_x / cnt, + } + + def validation_epoch_end(self, outputs): + tensorboard_logs = {} + + for k in outputs[0].keys(): + tensorboard_logs[k] = torch.stack([x[k] for x in outputs]).mean() + + return {'val_loss': tensorboard_logs['val_loss'], 'log': tensorboard_logs} + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + if "dataset" not in cfg or not isinstance(cfg.dataset, DictConfig): + raise ValueError(f"No dataset for {name}") # TODO + if "dataloader_params" not in cfg or not isinstance(cfg.dataloader_params, DictConfig): + raise ValueError(f"No dataloder_params for {name}") # TODO + if shuffle_should_be: + if 'shuffle' not in cfg.dataloader_params: + logging.warning( + f"Shuffle should be set to True for {self}'s {name} dataloader but was not found in its " + "config. Manually setting to True" + ) + with open_dict(cfg["dataloader_params"]): + cfg.dataloader_params.shuffle = True + elif not cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to False!!!") + elif not shuffle_should_be and cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to True!!!") + + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader(dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params) + + def setup_training_data(self, cfg): + self._train_dl = self.__setup_dataloader_from_config(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self.__setup_dataloader_from_config(cfg, shuffle_should_be=False, name="validation") + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + + return list_of_models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/ed_mel2spec.py b/models/tacotron2/NeMo/nemo/collections/tts/models/ed_mel2spec.py new file mode 100755 index 0000000000..2c57814ecc --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/ed_mel2spec.py @@ -0,0 +1,321 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from dataclasses import dataclass +from typing import Any, Dict, Optional, Sequence + +import numpy as np +import soundfile as sf +import torch +import torch.nn.functional as F +from hydra.utils import instantiate +from omegaconf import MISSING, DictConfig, OmegaConf, open_dict +from torch import Tensor, nn + +from nemo.collections.tts.helpers.helpers import eval_tts_scores, griffin_lim +from nemo.collections.tts.models.base import MelToSpec +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import MelSpectrogramType, SpectrogramType +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +@dataclass +class EDMel2SpecConfig: + mel2spec: Dict[Any, Any] = MISSING + train_ds: Optional[Dict[Any, Any]] = None + validation_ds: Optional[Dict[Any, Any]] = None + train_params: Optional[Dict[Any, Any]] = None + sched: Optional[Dict[Any, Any]] = None + + +def gen_filter(k): + K = torch.ones(1, 1, k, 1) + K.requires_grad = False + return K + + +class EDMel2SpecModel(MelToSpec): + """ + A model that convert mel spectrograms to linear spectrograms, using an encoder- decoder like model + The module relies on convolutions (encoders) and transposed convolutions (decoders), + which does not affect the time-dimension length, and thus applicable to any input length. + """ + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + super().__init__(cfg=cfg, trainer=trainer) + + schema = OmegaConf.structured(EDMel2SpecConfig) + # ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + # Ensure passed cfg is compliant with schema + OmegaConf.merge(cfg, schema) + + self.ed_mel2spec = instantiate(self._cfg.mel2spec) + + self.criterion = nn.L1Loss(reduction='none') # maybe should be loss subclass? + loss_mode = self._cfg.train_params.loss_mode + self.lreg_factor = self._cfg.train_params.lreg_factor + + self.f_specs = { + 0: [(5, 2), (15, 5)], + 1: [(5, 2)], + 2: [(3, 1)], + 3: [(3, 1), (5, 2)], + 4: [(3, 1), (5, 2), (7, 3)], + 5: [(15, 5)], + 6: [(3, 1), (5, 2), (7, 3), (15, 5), (25, 10)], + 7: [(1, 1)], + 8: [(1, 1), (3, 1), (5, 2), (15, 5), (7, 3), (25, 10), (9, 4), (20, 5), (5, 3)], + 9: [(6, 2), (10, 4)], + }[loss_mode] + + self.filters = [gen_filter(k) for k, s in self.f_specs] + + @property + def input_types(self): + return { + "mel": NeuralType(('B', 'C', 'D', 'T'), MelSpectrogramType()), + } + + @property + def output_types(self): + return { + "spec": NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + } + + @typecheck() + def forward(self, *, mel): + if self.mode != self.ed_mel2spec.mode: + raise ValueError( + f"Encoder-Decoder Mel-to-Spec's mode {self.mode} does not match EDMel2SpecModule's mode {self.ed_mel2spec.mode}" + ) + + spec = self.ed_mel2spec(mel=mel) + return spec # audio_pred + + def convert_mel_spectrogram_to_linear(self, mel: torch.Tensor) -> torch.Tensor: + self.eval() + if len(mel.shape) == 3: + mel = mel.unsqueeze(1) + + spec = self(mel=mel) + spec = torch.clamp(spec, min=1e-5) + return spec.squeeze(1) + + def calc_loss(self, x: Tensor, y: Tensor, T_ys: Sequence[int], crit) -> Tensor: + """ + x: B, C, F, T + y: B, C, F, T + """ + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + loss_no_red = crit(x, y) + + loss_blocks = torch.zeros(x.shape[1], device=y.device) + + tot = 0 + for T, loss_batch in zip(T_ys, loss_no_red): + tot += T + loss_blocks += torch.sum(loss_batch[..., :T]) + loss_blocks = loss_blocks / tot + + loss = loss_blocks.squeeze() + + return loss + + def calc_loss_smooth( + self, _x: Tensor, _y: Tensor, T_ys: Sequence[int], kern: int, stride: int, pad: int = 0 + ) -> Tensor: + """ + out_blocks: B, depth, C, F, T + y: B, C, F, T + """ + + crit = self.criterion + + x = F.max_pool2d(_x, (kern, 1), stride=stride) + y = F.max_pool2d(_y, (kern, 1), stride=stride) + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + loss_no_red = crit(x, y) + + loss_blocks = torch.zeros(x.shape[1], device=y.device) + + tot = 0 + for T, loss_batch in zip(T_ys, loss_no_red): + tot += T + loss_blocks += torch.sum(loss_batch[..., :T]) + loss_blocks = loss_blocks / tot + + loss1 = loss_blocks.squeeze() + + x = F.max_pool2d(-1 * _x, (kern, 1), stride=stride) + y = F.max_pool2d(-1 * _y, (kern, 1), stride=stride) + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + loss_no_red = crit(x, y) + + loss_blocks = torch.zeros(x.shape[1], device=y.device) + + tot = 0 + for T, loss_batch in zip(T_ys, loss_no_red): + tot += T + loss_blocks += torch.sum(loss_batch[..., :T]) + loss_blocks = loss_blocks / tot + + loss2 = loss_blocks.squeeze() + + loss = loss1 + loss2 + return loss + + def training_step(self, batch, batch_idx): + _, y_spec, _, _, T_ys, _, _ = batch + + x_mel = self.ed_mel2spec.spec_to_mel(y_spec) + + x_spec = self(mel=x_mel) + z_mel = self.ed_mel2spec.spec_to_mel(x_spec) + + loss_L1 = self.calc_loss(x_spec, y_spec, T_ys, self.criterion) + loss_reg = self.calc_loss(x_mel, z_mel, T_ys, self.criterion) + + loss = loss_L1 + self.lreg_factor * loss_reg + + for (k, s) in self.f_specs: + loss = loss + self.calc_loss_smooth(x_spec, y_spec, T_ys, k, s) + + output = { + 'loss': loss, + 'progress_bar': {'training_loss': loss}, + 'log': {'loss': loss}, + } + return output + + def validation_step(self, batch, batch_idx): + _, y_spec, _, _, T_ys, _, path_speech = batch + + x_mel = self.ed_mel2spec.spec_to_mel(y_spec) + + x_spec = self(mel=x_mel) + z_mel = self.ed_mel2spec.spec_to_mel(x_spec) + + loss_L1 = self.calc_loss(x_spec, y_spec, T_ys, self.criterion) + loss_reg = self.calc_loss(x_mel, z_mel, T_ys, self.criterion) + + loss = loss_L1 + self.lreg_factor * loss_reg + + output = { + 'val_loss': loss, + 'loss_L1': loss_L1, + 'loss_reg': loss_reg, + } + + if self._cfg.train_params.validate_scores: + ''' + For validaiton, estimate the wave using standard griffin lim, + comparing the real wave with the griffin lim counterpart. + ''' + + cnt = x_spec.shape[0] + np_x = x_spec.to('cpu').numpy() + np_y = y_spec.to('cpu').numpy() + stoi_real, pesq_real, stoi_est, pesq_est = (0.0, 0.0, 0.0, 0.0) + + for p in range(cnt): + y_wav_path = path_speech[p] + wav = sf.read(y_wav_path)[0].astype(np.float32) + + y_est_wav = griffin_lim(np_y[p, 0, :, :]) + x_est_wav = griffin_lim(np_x[p, 0, :, :]) + + min_size = min(wav.shape[0], x_est_wav.shape[0], y_est_wav.shape[0]) + wav = wav[0:min_size, ...] + y_est_wav = y_est_wav[0:min_size, ...] + x_est_wav = x_est_wav[0:min_size, ...] + + measure = eval_tts_scores(x_est_wav, wav) + stoi_real += torch.tensor(measure['STOI']) + pesq_real += torch.tensor(measure['PESQ']) + + measure = eval_tts_scores(x_est_wav, y_est_wav) + stoi_est += torch.tensor(measure['STOI']) + pesq_est += torch.tensor(measure['PESQ']) + + output['stoi_real'] = stoi_real / cnt + output['pesq_real'] = pesq_real / cnt + output['stoi_est'] = stoi_est / cnt + output['pesq_est'] = pesq_est / cnt + + for (k, s) in self.f_specs: + new_loss = self.calc_loss_smooth(x_spec, y_spec, T_ys, k, s) + output[f'loss_{k}_{s}'] = new_loss + loss = loss + new_loss + + output['val_loss'] = loss + + return output + + def validation_epoch_end(self, outputs): + tensorboard_logs = {} + + for k in outputs[0].keys(): + tensorboard_logs[k] = torch.stack([x[k] for x in outputs]).mean() + + return {'val_loss': tensorboard_logs['val_loss'], 'log': tensorboard_logs} + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + if "dataset" not in cfg or not isinstance(cfg.dataset, DictConfig): + raise ValueError(f"No dataset for {name}") # TODO + if "dataloader_params" not in cfg or not isinstance(cfg.dataloader_params, DictConfig): + raise ValueError(f"No dataloder_params for {name}") # TODO + if shuffle_should_be: + if 'shuffle' not in cfg.dataloader_params: + logging.warning( + f"Shuffle should be set to True for {self}'s {name} dataloader but was not found in its " + "config. Manually setting to True" + ) + with open_dict(cfg["dataloader_params"]): + cfg.dataloader_params.shuffle = True + elif not cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to False!!!") + elif not shuffle_should_be and cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to True!!!") + + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader(dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params) + + def setup_training_data(self, cfg): + self._train_dl = self.__setup_dataloader_from_config(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self.__setup_dataloader_from_config(cfg, shuffle_should_be=False, name="validation") + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + return list_of_models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/fastpitch.py b/models/tacotron2/NeMo/nemo/collections/tts/models/fastpitch.py new file mode 100644 index 0000000000..49e811787a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/fastpitch.py @@ -0,0 +1,414 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict + +import torch +from hydra.utils import instantiate +from omegaconf import MISSING, DictConfig, OmegaConf, open_dict +from pytorch_lightning import Trainer +from pytorch_lightning.loggers import LoggerCollection, TensorBoardLogger + +from nemo.collections.asr.data.audio_to_text import AudioToCharWithDursF0Dataset +from nemo.collections.common.parts.preprocessing import parsers +from nemo.collections.tts.helpers.helpers import plot_alignment_to_numpy, plot_spectrogram_to_numpy +from nemo.collections.tts.losses.aligner_loss import BinLoss, ForwardSumLoss +from nemo.collections.tts.losses.fastpitchloss import DurationLoss, MelLoss, PitchLoss +from nemo.collections.tts.models.base import SpectrogramGenerator +from nemo.collections.tts.modules.fastpitch import FastPitchModule +from nemo.core.classes import Exportable +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import ( + Index, + LengthsType, + MelSpectrogramType, + ProbsType, + RegressionValuesType, + TokenDurationType, + TokenIndex, +) +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +@dataclass +class FastPitchConfig: + parser: Dict[Any, Any] = MISSING + preprocessor: Dict[Any, Any] = MISSING + input_fft: Dict[Any, Any] = MISSING + output_fft: Dict[Any, Any] = MISSING + duration_predictor: Dict[Any, Any] = MISSING + pitch_predictor: Dict[Any, Any] = MISSING + + +class FastPitchModel(SpectrogramGenerator, Exportable): + """FastPitch Model that is used to generate mel spectrograms from text""" + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + + self.learn_alignment = False + if "learn_alignment" in cfg: + self.learn_alignment = cfg.learn_alignment + self._parser = None + self._tb_logger = None + super().__init__(cfg=cfg, trainer=trainer) + + schema = OmegaConf.structured(FastPitchConfig) + # ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + # Ensure passed cfg is compliant with schema + OmegaConf.merge(cfg, schema) + + self.bin_loss_warmup_epochs = 100 + self.aligner = None + self.log_train_images = False + self.mel_loss = MelLoss() + loss_scale = 0.1 if self.learn_alignment else 1.0 + self.pitch_loss = PitchLoss(loss_scale=loss_scale) + self.duration_loss = DurationLoss(loss_scale=loss_scale) + input_fft_kwargs = {} + if self.learn_alignment: + self.aligner = instantiate(self._cfg.alignment_module) + self.forward_sum_loss = ForwardSumLoss() + self.bin_loss = BinLoss() + self.vocab = AudioToCharWithDursF0Dataset.make_vocab(**self._cfg.train_ds.dataset.vocab) + input_fft_kwargs["n_embed"] = len(self.vocab.labels) + input_fft_kwargs["padding_idx"] = self.vocab.pad + + self.preprocessor = instantiate(self._cfg.preprocessor) + + input_fft = instantiate(self._cfg.input_fft, **input_fft_kwargs) + output_fft = instantiate(self._cfg.output_fft) + duration_predictor = instantiate(self._cfg.duration_predictor) + pitch_predictor = instantiate(self._cfg.pitch_predictor) + + self.fastpitch = FastPitchModule( + input_fft, + output_fft, + duration_predictor, + pitch_predictor, + self.aligner, + cfg.n_speakers, + cfg.symbols_embedding_dim, + cfg.pitch_embedding_kernel_size, + cfg.n_mel_channels, + ) + + @property + def tb_logger(self): + if self._tb_logger is None: + if self.logger is None and self.logger.experiment is None: + return None + tb_logger = self.logger.experiment + if isinstance(self.logger, LoggerCollection): + for logger in self.logger: + if isinstance(logger, TensorBoardLogger): + tb_logger = logger.experiment + break + self._tb_logger = tb_logger + return self._tb_logger + + @property + def parser(self): + if self._parser is not None: + return self._parser + + if self.learn_alignment: + vocab = AudioToCharWithDursF0Dataset.make_vocab(**self._cfg.train_ds.dataset.vocab) + self._parser = vocab.encode + else: + self._parser = parsers.make_parser( + labels=self._cfg.labels, + name='en', + unk_id=-1, + blank_id=-1, + do_normalize=True, + abbreviation_version="fastpitch", + make_table=False, + ) + return self._parser + + def parse(self, str_input: str) -> torch.tensor: + if str_input[-1] not in [".", "!", "?"]: + str_input = str_input + "." + + tokens = self.parser(str_input) + + x = torch.tensor(tokens).unsqueeze_(0).long().to(self.device) + return x + + @typecheck( + input_types={ + "text": NeuralType(('B', 'T'), TokenIndex()), + "durs": NeuralType(('B', 'T'), TokenDurationType()), + "pitch": NeuralType(('B', 'T'), RegressionValuesType()), + "speaker": NeuralType(('B'), Index()), + "pace": NeuralType(optional=True), + "spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType(), optional=True), + "attn_prior": NeuralType(('B', 'T', 'T'), ProbsType(), optional=True), + "mel_lens": NeuralType(('B'), LengthsType(), optional=True), + "input_lens": NeuralType(('B'), LengthsType(), optional=True), + } + ) + def forward( + self, + *, + text, + durs=None, + pitch=None, + speaker=0, + pace=1.0, + spec=None, + attn_prior=None, + mel_lens=None, + input_lens=None, + ): + return self.fastpitch( + text=text, + durs=durs, + pitch=pitch, + speaker=speaker, + pace=pace, + spec=spec, + attn_prior=attn_prior, + mel_lens=mel_lens, + input_lens=input_lens, + ) + + @typecheck(output_types={"spect": NeuralType(('B', 'C', 'T'), MelSpectrogramType())}) + def generate_spectrogram(self, tokens: 'torch.tensor', speaker: int = 0, pace: float = 1.0) -> torch.tensor: + # FIXME: return masks as well? + self.eval() + spect, *_ = self(text=tokens, durs=None, pitch=None, speaker=speaker, pace=pace) + return spect + + def training_step(self, batch, batch_idx): + attn_prior, durs, speakers = None, None, None + if self.learn_alignment: + audio, audio_lens, text, text_lens, attn_prior, pitch = batch + else: + audio, audio_lens, text, text_lens, durs, pitch, speakers = batch + mels, spec_len = self.preprocessor(input_signal=audio, length=audio_lens) + + mels_pred, _, _, log_durs_pred, pitch_pred, attn_soft, attn_logprob, attn_hard, attn_hard_dur, pitch = self( + text=text, + durs=durs, + pitch=pitch, + speaker=speakers, + pace=1.0, + spec=mels if self.learn_alignment else None, + attn_prior=attn_prior, + mel_lens=spec_len, + input_lens=text_lens, + ) + if durs is None: + durs = attn_hard_dur + + mel_loss = self.mel_loss(spect_predicted=mels_pred, spect_tgt=mels) + dur_loss = self.duration_loss(log_durs_predicted=log_durs_pred, durs_tgt=durs, len=text_lens) + loss = mel_loss + dur_loss + if self.learn_alignment: + ctc_loss = self.forward_sum_loss(attn_logprob=attn_logprob, in_lens=text_lens, out_lens=spec_len) + bin_loss_weight = min(self.current_epoch / self.bin_loss_warmup_epochs, 1.0) * 1.0 + bin_loss = self.bin_loss(hard_attention=attn_hard, soft_attention=attn_soft) * bin_loss_weight + loss += ctc_loss + bin_loss + + pitch_loss = self.pitch_loss(pitch_predicted=pitch_pred, pitch_tgt=pitch, len=text_lens) + loss += pitch_loss + + self.log("t_loss", loss) + self.log("t_mel_loss", mel_loss) + self.log("t_dur_loss", dur_loss) + self.log("t_pitch_loss", pitch_loss) + if self.learn_alignment: + self.log("t_ctc_loss", ctc_loss) + self.log("t_bin_loss", bin_loss) + + # Log images to tensorboard + if self.log_train_images: + self.log_train_images = False + + self.tb_logger.add_image( + "train_mel_target", + plot_spectrogram_to_numpy(mels[0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + spec_predict = mels_pred[0].data.cpu().numpy().T + self.tb_logger.add_image( + "train_mel_predicted", plot_spectrogram_to_numpy(spec_predict), self.global_step, dataformats="HWC", + ) + if self.learn_alignment: + attn = attn_hard[0].data.cpu().numpy().squeeze() + self.tb_logger.add_image( + "train_attn", plot_alignment_to_numpy(attn.T), self.global_step, dataformats="HWC", + ) + soft_attn = attn_soft[0].data.cpu().numpy().squeeze() + self.tb_logger.add_image( + "train_soft_attn", plot_alignment_to_numpy(soft_attn.T), self.global_step, dataformats="HWC", + ) + + return loss + + def validation_step(self, batch, batch_idx): + attn_prior, durs, speakers = None, None, None + if self.learn_alignment: + audio, audio_lens, text, text_lens, attn_prior, pitch = batch + else: + audio, audio_lens, text, text_lens, durs, pitch, speakers = batch + mels, mel_lens = self.preprocessor(input_signal=audio, length=audio_lens) + + # Calculate val loss on ground truth durations to better align L2 loss in time + mels_pred, _, _, log_durs_pred, pitch_pred, _, _, _, attn_hard_dur, pitch = self( + text=text, + durs=durs, + pitch=pitch, + speaker=speakers, + pace=1.0, + spec=mels if self.learn_alignment else None, + attn_prior=attn_prior, + mel_lens=mel_lens, + input_lens=text_lens, + ) + if durs is None: + durs = attn_hard_dur + + mel_loss = self.mel_loss(spect_predicted=mels_pred, spect_tgt=mels) + dur_loss = self.duration_loss(log_durs_predicted=log_durs_pred, durs_tgt=durs, len=text_lens) + pitch_loss = self.pitch_loss(pitch_predicted=pitch_pred, pitch_tgt=pitch, len=text_lens) + loss = mel_loss + dur_loss + pitch_loss + + return { + "val_loss": loss, + "mel_loss": mel_loss, + "dur_loss": dur_loss, + "pitch_loss": pitch_loss, + "mel_target": mels if batch_idx == 0 else None, + "mel_pred": mels_pred if batch_idx == 0 else None, + } + + def validation_epoch_end(self, outputs): + collect = lambda key: torch.stack([x[key] for x in outputs]).mean() + val_loss = collect("val_loss") + mel_loss = collect("mel_loss") + dur_loss = collect("dur_loss") + pitch_loss = collect("pitch_loss") + self.log("v_loss", val_loss) + self.log("v_mel_loss", mel_loss) + self.log("v_dur_loss", dur_loss) + self.log("v_pitch_loss", pitch_loss) + + """_, _, _, _, spec_target, spec_predict = outputs[0].values() + self.tb_logger.add_image( + "val_mel_target", + plot_spectrogram_to_numpy(spec_target[0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + spec_predict = spec_predict[0].data.cpu().numpy() + self.tb_logger.add_image( + "val_mel_predicted", plot_spectrogram_to_numpy(spec_predict.T), self.global_step, dataformats="HWC", + ) + self.log_train_images = True""" + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + if "dataset" not in cfg or not isinstance(cfg.dataset, DictConfig): + raise ValueError(f"No dataset for {name}") + if "dataloader_params" not in cfg or not isinstance(cfg.dataloader_params, DictConfig): + raise ValueError(f"No dataloder_params for {name}") + if shuffle_should_be: + if 'shuffle' not in cfg.dataloader_params: + logging.warning( + f"Shuffle should be set to True for {self}'s {name} dataloader but was not found in its " + "config. Manually setting to True" + ) + with open_dict(cfg.dataloader_params): + cfg.dataloader_params.shuffle = True + elif not cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to False!!!") + elif not shuffle_should_be and cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to True!!!") + + kwargs_dict = {} + if cfg.dataset._target_ == "nemo.collections.asr.data.audio_to_text.FastPitchDataset": + kwargs_dict["parser"] = self.parser + dataset = instantiate(cfg.dataset, **kwargs_dict) + return torch.utils.data.DataLoader(dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params) + + def setup_training_data(self, cfg): + self._train_dl = self.__setup_dataloader_from_config(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self.__setup_dataloader_from_config(cfg, shuffle_should_be=False, name="val") + + def setup_test_data(self, cfg): + """Omitted.""" + pass + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_en_fastpitch", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_fastpitch/versions/1.0.0/files/tts_en_fastpitch.nemo", + description="This model is trained on LJSpeech sampled at 22050Hz with and can be used to generate female English voices with an American accent.", + class_=cls, + ) + list_of_models.append(model) + + return list_of_models + + @property + def input_module(self): + return self.fastpitch + + @property + def output_module(self): + return self.fastpitch + + def forward_for_export(self, text): + ( + spect, + num_frames, + durs_predicted, + log_durs_predicted, + pitch_predicted, + attn_soft, + attn_logprob, + attn_hard, + attn_hard_dur, + pitch, + ) = self.fastpitch(text=text) + return spect, num_frames, durs_predicted, log_durs_predicted, pitch_predicted + + @property + def disabled_deployment_input_names(self): + """Implement this method to return a set of input names disabled for export""" + return set(["durs", "pitch", "speaker", "pace", "spec", "attn_prior", "mel_lens", "input_lens"]) + + @property + def disabled_deployment_output_names(self): + """Implement this method to return a set of input names disabled for export""" + return set(["attn_soft", "pitch", "attn_logprob", "attn_hard", "attn_hard_dur",]) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/fastpitch_hifigan_e2e.py b/models/tacotron2/NeMo/nemo/collections/tts/models/fastpitch_hifigan_e2e.py new file mode 100644 index 0000000000..6695f5df8d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/fastpitch_hifigan_e2e.py @@ -0,0 +1,432 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from itertools import chain +from typing import Any, Dict + +import numpy as np +import torch +from hydra.utils import instantiate +from omegaconf import MISSING, DictConfig, OmegaConf +from pytorch_lightning import Trainer +from pytorch_lightning.loggers import LoggerCollection, TensorBoardLogger + +from nemo.collections.asr.data.audio_to_text import FastPitchDataset +from nemo.collections.common.parts.preprocessing import parsers +from nemo.collections.tts.helpers.helpers import plot_spectrogram_to_numpy +from nemo.collections.tts.losses.fastpitchloss import DurationLoss, PitchLoss +from nemo.collections.tts.losses.fastspeech2loss import L1MelLoss +from nemo.collections.tts.losses.hifigan_losses import DiscriminatorLoss, FeatureMatchingLoss, GeneratorLoss +from nemo.collections.tts.models.base import TextToWaveform +from nemo.collections.tts.modules.fastpitch import regulate_len +from nemo.collections.tts.modules.hifigan_modules import MultiPeriodDiscriminator, MultiScaleDiscriminator +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import ( + MelSpectrogramType, + RegressionValuesType, + TokenDurationType, + TokenIndex, + TokenLogDurationType, +) +from nemo.core.neural_types.neural_type import NeuralType +from nemo.core.optim.lr_scheduler import NoamAnnealing +from nemo.utils import logging + + +@dataclass +class FastPitchHifiGanE2EConfig: + parser: Dict[Any, Any] = MISSING + preprocessor: Dict[Any, Any] = MISSING + input_fft: Dict[Any, Any] = MISSING + output_fft: Dict[Any, Any] = MISSING + duration_predictor: Dict[Any, Any] = MISSING + pitch_predictor: Dict[Any, Any] = MISSING + + +class FastPitchHifiGanE2EModel(TextToWaveform): + """An end-to-end speech synthesis model based on FastPitch and HiFiGan that converts strings to audio without using + the intermediate mel spectrogram representation. + """ + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + + self._parser = parsers.make_parser( + labels=cfg.labels, + name='en', + unk_id=-1, + blank_id=-1, + do_normalize=True, + abbreviation_version="fastpitch", + make_table=False, + ) + + super().__init__(cfg=cfg, trainer=trainer) + + schema = OmegaConf.structured(FastPitchHifiGanE2EConfig) + # ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + # Ensure passed cfg is compliant with schema + OmegaConf.merge(cfg, schema) + + self.preprocessor = instantiate(cfg.preprocessor) + self.melspec_fn = instantiate(cfg.preprocessor, highfreq=None, use_grads=True) + + self.encoder = instantiate(cfg.input_fft) + self.duration_predictor = instantiate(cfg.duration_predictor) + self.pitch_predictor = instantiate(cfg.pitch_predictor) + + self.generator = instantiate(cfg.generator) + self.multiperioddisc = MultiPeriodDiscriminator() + self.multiscaledisc = MultiScaleDiscriminator() + self.mel_val_loss = L1MelLoss() + self.feat_matching_loss = FeatureMatchingLoss() + self.disc_loss = DiscriminatorLoss() + self.gen_loss = GeneratorLoss() + + self.max_token_duration = cfg.max_token_duration + + self.pitch_emb = torch.nn.Conv1d( + 1, + cfg.symbols_embedding_dim, + kernel_size=cfg.pitch_embedding_kernel_size, + padding=int((cfg.pitch_embedding_kernel_size - 1) / 2), + ) + + # Store values precomputed from training data for convenience + self.register_buffer('pitch_mean', torch.zeros(1)) + self.register_buffer('pitch_std', torch.zeros(1)) + + self.pitchloss = PitchLoss() + self.durationloss = DurationLoss() + + self.mel_loss_coeff = cfg.mel_loss_coeff + + self.log_train_images = False + self.logged_real_samples = False + self._tb_logger = None + self.hann_window = None + self.splice_length = cfg.splice_length + self.sample_rate = cfg.sample_rate + self.hop_size = cfg.hop_size + + @property + def tb_logger(self): + if self._tb_logger is None: + if self.logger is None and self.logger.experiment is None: + return None + tb_logger = self.logger.experiment + if isinstance(self.logger, LoggerCollection): + for logger in self.logger: + if isinstance(logger, TensorBoardLogger): + tb_logger = logger.experiment + break + self._tb_logger = tb_logger + return self._tb_logger + + @property + def parser(self): + if self._parser is not None: + return self._parser + + self._parser = parsers.make_parser( + labels=self._cfg.labels, + name='en', + unk_id=-1, + blank_id=-1, + do_normalize=True, + abbreviation_version="fastpitch", + make_table=False, + ) + return self._parser + + def parse(self, str_input: str) -> torch.tensor: + if str_input[-1] not in [".", "!", "?"]: + str_input = str_input + "." + + tokens = self.parser(str_input) + + x = torch.tensor(tokens).unsqueeze_(0).long().to(self.device) + return x + + def configure_optimizers(self): + gen_params = chain( + self.pitch_emb.parameters(), + self.encoder.parameters(), + self.duration_predictor.parameters(), + self.pitch_predictor.parameters(), + self.generator.parameters(), + ) + disc_params = chain(self.multiscaledisc.parameters(), self.multiperioddisc.parameters()) + opt1 = torch.optim.AdamW(disc_params, lr=self._cfg.lr) + opt2 = torch.optim.AdamW(gen_params, lr=self._cfg.lr) + num_procs = self._trainer.num_gpus * self._trainer.num_nodes + num_samples = len(self._train_dl.dataset) + batch_size = self._train_dl.batch_size + iter_per_epoch = np.ceil(num_samples / (num_procs * batch_size)) + max_steps = iter_per_epoch * self._trainer.max_epochs + logging.info(f"MAX STEPS: {max_steps}") + sch1 = NoamAnnealing(opt1, d_model=1, warmup_steps=1000, max_steps=max_steps, last_epoch=-1) + sch1_dict = { + 'scheduler': sch1, + 'interval': 'step', + } + sch2 = NoamAnnealing(opt2, d_model=1, warmup_steps=1000, max_steps=max_steps, last_epoch=-1) + sch2_dict = { + 'scheduler': sch2, + 'interval': 'step', + } + return [opt1, opt2], [sch1_dict, sch2_dict] + + @typecheck( + input_types={ + "text": NeuralType(('B', 'T'), TokenIndex()), + "durs": NeuralType(('B', 'T'), TokenDurationType(), optional=True), + "pitch": NeuralType(('B', 'T'), RegressionValuesType(), optional=True), + "pace": NeuralType(optional=True), + "splice": NeuralType(optional=True), + }, + output_types={ + "audio": NeuralType(('B', 'S', 'T'), MelSpectrogramType()), + "splices": NeuralType(), + "log_dur_preds": NeuralType(('B', 'T'), TokenLogDurationType()), + "pitch_preds": NeuralType(('B', 'T'), RegressionValuesType()), + }, + ) + def forward(self, *, text, durs=None, pitch=None, pace=1.0, splice=True): + if self.training: + assert durs is not None + assert pitch is not None + + # Input FFT + enc_out, enc_mask = self.encoder(input=text, conditioning=0) + + # Embedded for predictors + pred_enc_out, pred_enc_mask = enc_out, enc_mask + + # Predict durations + log_durs_predicted = self.duration_predictor(pred_enc_out, pred_enc_mask) + durs_predicted = torch.clamp(torch.exp(log_durs_predicted) - 1, 0, self.max_token_duration) + + # Predict pitch + pitch_predicted = self.pitch_predictor(enc_out, enc_mask) + if pitch is None: + pitch_emb = self.pitch_emb(pitch_predicted.unsqueeze(1)) + else: + pitch_emb = self.pitch_emb(pitch.unsqueeze(1)) + enc_out = enc_out + pitch_emb.transpose(1, 2) + + if durs is None: + len_regulated, dec_lens = regulate_len(durs_predicted, enc_out, pace) + else: + len_regulated, dec_lens = regulate_len(durs, enc_out, pace) + + gen_in = len_regulated + splices = None + if splice: + output = [] + splices = [] + for i, sample in enumerate(len_regulated): + start = np.random.randint(low=0, high=min(int(sample.size(0)), int(dec_lens[i])) - self.splice_length) + # Splice generated spec + output.append(sample[start : start + self.splice_length, :]) + splices.append(start) + gen_in = torch.stack(output) + + output = self.generator(x=gen_in.transpose(1, 2)) + + return output, splices, log_durs_predicted, pitch_predicted + + def training_step(self, batch, batch_idx, optimizer_idx): + audio, _, text, text_lens, durs, pitch, _ = batch + + # train discriminator + if optimizer_idx == 0: + with torch.no_grad(): + audio_pred, splices, _, _ = self(text=text, durs=durs, pitch=pitch) + real_audio = [] + for i, splice in enumerate(splices): + real_audio.append(audio[i, splice * self.hop_size : (splice + self.splice_length) * self.hop_size]) + real_audio = torch.stack(real_audio).unsqueeze(1) + + real_score_mp, gen_score_mp, _, _ = self.multiperioddisc(real_audio, audio_pred) + real_score_ms, gen_score_ms, _, _ = self.multiscaledisc(real_audio, audio_pred) + + loss_mp, loss_mp_real, _ = self.disc_loss(real_score_mp, gen_score_mp) + loss_ms, loss_ms_real, _ = self.disc_loss(real_score_ms, gen_score_ms) + loss_mp /= len(loss_mp_real) + loss_ms /= len(loss_ms_real) + loss_disc = loss_mp + loss_ms + + self.log("loss_discriminator", loss_disc, prog_bar=True) + self.log("loss_discriminator_ms", loss_ms) + self.log("loss_discriminator_mp", loss_mp) + return loss_disc + + # train generator + elif optimizer_idx == 1: + audio_pred, splices, log_dur_preds, pitch_preds = self(text=text, durs=durs, pitch=pitch) + real_audio = [] + for i, splice in enumerate(splices): + real_audio.append(audio[i, splice * self.hop_size : (splice + self.splice_length) * self.hop_size]) + real_audio = torch.stack(real_audio).unsqueeze(1) + + dur_loss = self.durationloss(log_durs_predicted=log_dur_preds, durs_tgt=durs, len=text_lens) + pitch_loss = self.pitchloss(pitch_predicted=pitch_preds, pitch_tgt=pitch,) + + # Do HiFiGAN generator loss + audio_length = torch.tensor([self.splice_length * self.hop_size for _ in range(real_audio.shape[0])]).to( + real_audio.device + ) + real_spliced_spec, _ = self.melspec_fn(real_audio.squeeze(), audio_length) + pred_spliced_spec, _ = self.melspec_fn(audio_pred.squeeze(), audio_length) + loss_mel = torch.nn.functional.l1_loss(real_spliced_spec, pred_spliced_spec) + loss_mel *= self.mel_loss_coeff + _, gen_score_mp, _, _ = self.multiperioddisc(real_audio, audio_pred) + _, gen_score_ms, _, _ = self.multiscaledisc(real_audio, audio_pred) + loss_gen_mp, list_loss_gen_mp = self.gen_loss(gen_score_mp) + loss_gen_ms, list_loss_gen_ms = self.gen_loss(gen_score_ms) + loss_gen_mp /= len(list_loss_gen_mp) + loss_gen_ms /= len(list_loss_gen_ms) + total_loss = loss_gen_mp + loss_gen_ms + loss_mel + total_loss += dur_loss + total_loss += pitch_loss + + self.log(name="loss_gen_mel", value=loss_mel) + self.log(name="loss_gen_disc", value=loss_gen_mp + loss_gen_ms) + self.log(name="loss_gen_disc_mp", value=loss_gen_mp) + self.log(name="loss_gen_disc_ms", value=loss_gen_ms) + self.log(name="loss_gen_duration", value=dur_loss) + self.log(name="loss_gen_pitch", value=pitch_loss) + + # Log images to tensorboard + if self.log_train_images: + self.log_train_images = False + + if self.logger is not None and self.logger.experiment is not None: + self.tb_logger.add_image( + "train_mel_target", + plot_spectrogram_to_numpy(real_spliced_spec[0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + spec_predict = pred_spliced_spec[0].data.cpu().numpy() + self.tb_logger.add_image( + "train_mel_predicted", + plot_spectrogram_to_numpy(spec_predict), + self.global_step, + dataformats="HWC", + ) + self.log(name="loss_gen", prog_bar=True, value=total_loss) + return total_loss + + def validation_step(self, batch, batch_idx): + audio, audio_lens, text, _, _, _, _ = batch + mels, mel_lens = self.preprocessor(audio, audio_lens) + + audio_pred, _, log_durs_predicted, _ = self(text=text, durs=None, pitch=None, splice=False) + audio_length = torch.sum(torch.clamp(torch.exp(log_durs_predicted - 1), 0), axis=1) + audio_pred.squeeze_() + pred_spec, _ = self.melspec_fn(audio_pred, audio_length) + loss = self.mel_val_loss( + spec_pred=pred_spec, spec_target=mels, spec_target_len=mel_lens, pad_value=-11.52, transpose=False + ) + + return { + "val_loss": loss, + "audio_target": audio if batch_idx == 0 else None, + "audio_pred": audio_pred.squeeze() if batch_idx == 0 else None, + } + + def validation_epoch_end(self, outputs): + if self.tb_logger is not None: + _, audio_target, audio_predict = outputs[0].values() + if not self.logged_real_samples: + self.tb_logger.add_audio("val_target", audio_target[0].data.cpu(), self.global_step, self.sample_rate) + self.logged_real_samples = True + audio_predict = audio_predict[0].data.cpu() + self.tb_logger.add_audio("val_pred", audio_predict, self.global_step, self.sample_rate) + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() # This reduces across batches, not workers! + self.log('val_loss', avg_loss, sync_dist=True) + + self.log_train_images = True + + def _loader(self, cfg): + dataset = FastPitchDataset( + manifest_filepath=cfg['manifest_filepath'], + parser=self.parser, + sample_rate=cfg['sample_rate'], + int_values=cfg.get('int_values', False), + max_duration=cfg.get('max_duration', None), + min_duration=cfg.get('min_duration', None), + max_utts=cfg.get('max_utts', 0), + trim=cfg.get('trim_silence', True), + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg['batch_size'], + collate_fn=dataset.collate_fn, + drop_last=cfg.get('drop_last', True), + shuffle=cfg['shuffle'], + num_workers=cfg.get('num_workers', 16), + ) + + def setup_training_data(self, cfg): + self._train_dl = self._loader(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self._loader(cfg) + + def setup_test_data(self, cfg): + """Omitted.""" + pass + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_en_e2e_fastpitchhifigan", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_e2e_fastpitchhifigan/versions/1.0.0/files/tts_en_e2e_fastpitchhifigan.nemo", + description="This model is trained on LJSpeech sampled at 22050Hz with and can be used to generate female English voices with an American accent.", + class_=cls, + ) + list_of_models.append(model) + + return list_of_models + + def convert_text_to_waveform(self, *, tokens): + """ + Accepts tokens returned from self.parse() and returns a list of tensors. Note: The tensors in the list can have + different lengths. + """ + self.eval() + audio, _, log_dur_pred, _ = self(text=tokens, splice=False) + audio = audio.squeeze(1) + durations = torch.sum(torch.clamp(torch.exp(log_dur_pred) - 1, 0, self.max_token_duration), 1).to(torch.int) + audio_list = [] + for i, sample in enumerate(audio): + audio_list.append(sample[: durations[i] * self.hop_size]) + + return audio_list diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/fastspeech2.py b/models/tacotron2/NeMo/nemo/collections/tts/models/fastspeech2.py new file mode 100644 index 0000000000..d93d3b2563 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/fastspeech2.py @@ -0,0 +1,297 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import torch +from hydra.utils import instantiate +from omegaconf import MISSING, DictConfig, OmegaConf, open_dict +from pytorch_lightning.loggers import LoggerCollection, TensorBoardLogger + +from nemo.collections.common.parts.preprocessing import parsers +from nemo.collections.tts.helpers.helpers import plot_spectrogram_to_numpy +from nemo.collections.tts.losses.fastspeech2loss import DurationLoss, L2MelLoss +from nemo.collections.tts.models.base import SpectrogramGenerator +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import * +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +@dataclass +class FastSpeech2Config: + preprocessor: Dict[Any, Any] = MISSING + encoder: Dict[Any, Any] = MISSING + decoder: Dict[Any, Any] = MISSING + variance_adaptor: Dict[Any, Any] = MISSING + train_ds: Optional[Dict[Any, Any]] = None + validation_ds: Optional[Dict[Any, Any]] = None + + +class FastSpeech2Model(SpectrogramGenerator): + """FastSpeech 2 model used to convert from text (phonemes) to mel-spectrograms.""" + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + super().__init__(cfg=cfg, trainer=trainer) + + schema = OmegaConf.structured(FastSpeech2Config) + # ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + # Ensure passed cfg is compliant with schema + OmegaConf.merge(cfg, schema) + + self.pitch = cfg.add_pitch_predictor + self.energy = cfg.add_energy_predictor + self.duration_coeff = cfg.duration_coeff + + self.audio_to_melspec_preprocessor = instantiate(self._cfg.preprocessor) + self.encoder = instantiate(self._cfg.encoder) + self.mel_decoder = instantiate(self._cfg.decoder) + self.variance_adapter = instantiate(self._cfg.variance_adaptor) + self.loss = L2MelLoss() + self.mseloss = torch.nn.MSELoss() + self.durationloss = DurationLoss() + + self.log_train_images = False + + # Parser and mappings are used for inference only. + self.parser = parsers.make_parser(name='en') + if 'mappings_filepath' in cfg: + mappings_filepath = cfg.get('mappings_filepath') + else: + logging.error( + "ERROR: You must specify a mappings.json file in the config file under model.mappings_filepath." + ) + mappings_filepath = self.register_artifact('mappings_filepath', mappings_filepath) + with open(mappings_filepath, 'r') as f: + mappings = json.load(f) + self.word2phones = mappings['word2phones'] + self.phone2idx = mappings['phone2idx'] + + @typecheck( + input_types={ + "text": NeuralType(('B', 'T'), TokenIndex()), + "text_length": NeuralType(('B'), LengthsType()), + "spec_len": NeuralType(('B'), LengthsType(), optional=True), + "durations": NeuralType(('B', 'T'), TokenDurationType(), optional=True), + "pitch": NeuralType(('B', 'T'), RegressionValuesType(), optional=True), + "energies": NeuralType(('B', 'T'), RegressionValuesType(), optional=True), + }, + output_types={ + "mel_spec": NeuralType(('B', 'T', 'C'), MelSpectrogramType()), + "log_dur_preds": NeuralType(('B', 'T'), TokenDurationType(), optional=True), + "pitch_preds": NeuralType(('B', 'T'), RegressionValuesType(), optional=True), + "energy_preds": NeuralType(('B', 'T'), RegressionValuesType(), optional=True), + "encoded_text_mask": NeuralType(('B', 'T', 'D'), MaskType()), + }, + ) + def forward(self, *, text, text_length, spec_len=None, durations=None, pitch=None, energies=None): + encoded_text, encoded_text_mask = self.encoder(text=text, text_length=text_length) + aligned_text, log_dur_preds, pitch_preds, energy_preds, spec_len = self.variance_adapter( + x=encoded_text, + x_len=text_length, + dur_target=durations, + pitch_target=pitch, + energy_target=energies, + spec_len=spec_len, + ) + mel = self.mel_decoder(decoder_input=aligned_text, lengths=spec_len) + return mel, log_dur_preds, pitch_preds, energy_preds, encoded_text_mask + + def training_step(self, batch, batch_idx): + f, fl, t, tl, durations, pitch, energies = batch + spec, spec_len = self.audio_to_melspec_preprocessor(f, fl) + mel, log_dur_preds, pitch_preds, energy_preds, encoded_text_mask = self( + text=t, text_length=tl, spec_len=spec_len, durations=durations, pitch=pitch, energies=energies + ) + total_loss = self.loss( + spec_pred=mel.transpose(1, 2), spec_target=spec, spec_target_len=spec_len, pad_value=-11.52 + ) + self.log(name="train_mel_loss", value=total_loss.clone().detach()) + + # Duration prediction loss + dur_loss = self.durationloss( + log_duration_pred=log_dur_preds, duration_target=durations.float(), mask=encoded_text_mask + ) + dur_loss *= self.duration_coeff + self.log(name="train_dur_loss", value=dur_loss) + total_loss += dur_loss + + # Pitch prediction loss + if self.pitch: + pitch_loss = self.mseloss(pitch_preds, pitch) + total_loss += pitch_loss + self.log(name="train_pitch_loss", value=pitch_loss) + + # Energy prediction loss + if self.energy: + energy_loss = self.mseloss(energy_preds, energies) + total_loss += energy_loss + self.log(name="train_energy_loss", value=energy_loss) + self.log(name="train_loss", value=total_loss) + return {"loss": total_loss, "outputs": [spec, mel]} + + def training_epoch_end(self, training_step_outputs): + if self.log_train_images and self.logger is not None and self.logger.experiment is not None: + tb_logger = self.logger.experiment + if isinstance(self.logger, LoggerCollection): + for logger in self.logger: + if isinstance(logger, TensorBoardLogger): + tb_logger = logger.experiment + break + spec_target, spec_predict = training_step_outputs[0]["outputs"] + tb_logger.add_image( + "train_mel_target", + plot_spectrogram_to_numpy(spec_target[0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + spec_predict = spec_predict[0].data.cpu().numpy() + tb_logger.add_image( + "train_mel_predicted", plot_spectrogram_to_numpy(spec_predict.T), self.global_step, dataformats="HWC", + ) + self.log_train_images = False + + def validation_step(self, batch, batch_idx): + f, fl, t, tl, _, _, _ = batch + spec, spec_len = self.audio_to_melspec_preprocessor(f, fl) + mel, _, _, _, _ = self(text=t, text_length=tl, spec_len=spec_len) + loss = self.loss(spec_pred=mel.transpose(1, 2), spec_target=spec, spec_target_len=spec_len, pad_value=-11.52) + return { + "val_loss": loss, + "mel_target": spec, + "mel_pred": mel, + } + + def validation_epoch_end(self, outputs): + if self.logger is not None and self.logger.experiment is not None: + tb_logger = self.logger.experiment + if isinstance(self.logger, LoggerCollection): + for logger in self.logger: + if isinstance(logger, TensorBoardLogger): + tb_logger = logger.experiment + break + _, spec_target, spec_predict = outputs[0].values() + tb_logger.add_image( + "val_mel_target", + plot_spectrogram_to_numpy(spec_target[0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + spec_predict = spec_predict[0].data.cpu().numpy() + tb_logger.add_image( + "val_mel_predicted", plot_spectrogram_to_numpy(spec_predict.T), self.global_step, dataformats="HWC", + ) + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() # This reduces across batches, not workers! + self.log('val_loss', avg_loss, sync_dist=True) + + self.log_train_images = True + + def parse(self, str_input: str, additional_word2phones=None) -> torch.tensor: + """ + Parses text input and converts them to phoneme indices. + + str_input (str): The input text to be converted. + additional_word2phones (dict): Optional dictionary mapping words to phonemes for updating the model's + word2phones. This will not overwrite the existing dictionary, just update it with OOV or new mappings. + Defaults to None, which will keep the existing mapping. + """ + # Update model's word2phones if applicable + if additional_word2phones is not None: + self.word2phones.update(additional_word2phones) + + # Convert text -> normalized text -> list of phones per word -> indices + if str_input[-1] not in [".", "!", "?"]: + str_input = str_input + "." + norm_text = re.findall(r"""[\w']+|[.,!?;"]""", self.parser._normalize(str_input)) + + try: + phones = [self.word2phones[t] for t in norm_text] + except KeyError as error: + logging.error( + f"ERROR: The following word in the input is not in the model's dictionary and could not be converted" + f" to phonemes: ({error}).\n" + f"You can pass in an `additional_word2phones` dictionary with a conversion for" + f" this word, e.g. {{'{error}': \['phone1', 'phone2', ...\]}} to update the model's mapping." + ) + raise + + tokens = [] + for phone_list in phones: + inds = [self.phone2idx[p] for p in phone_list] + tokens += inds + + x = torch.tensor(tokens).unsqueeze_(0).long().to(self.device) + return x + + @typecheck(output_types={"spect": NeuralType(('B', 'C', 'T'), MelSpectrogramType())}) + def generate_spectrogram(self, tokens: torch.Tensor) -> torch.Tensor: + self.eval() + token_len = torch.tensor([tokens.shape[1]]).to(self.device) + spect, *_ = self(text=tokens, text_length=token_len) + + return spect.transpose(1, 2) + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + if "dataset" not in cfg or not isinstance(cfg.dataset, DictConfig): + raise ValueError(f"No dataset for {name}") + if "dataloader_params" not in cfg or not isinstance(cfg.dataloader_params, DictConfig): + raise ValueError(f"No dataloder_params for {name}") + if shuffle_should_be: + if 'shuffle' not in cfg.dataloader_params: + logging.warning( + f"Shuffle should be set to True for {self}'s {name} dataloader but was not found in its " + "config. Manually setting to True" + ) + with open_dict(cfg.dataloader_params): + cfg.dataloader_params.shuffle = True + elif not cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to False!!!") + elif not shuffle_should_be and cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to True!!!") + + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader(dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params) + + def setup_training_data(self, cfg): + self._train_dl = self.__setup_dataloader_from_config(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self.__setup_dataloader_from_config(cfg, shuffle_should_be=False, name="validation") + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained models which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_en_fastspeech2", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_fastspeech_2/versions/1.0.0/files/tts_en_fastspeech2.nemo", + description="This model is trained on LJSpeech sampled at 22050Hz, and can be used to generate female English voices with an American accent.", + class_=cls, + aliases=["FastSpeech2-22050Hz"], + ) + list_of_models.append(model) + return list_of_models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/fastspeech2_hifigan_e2e.py b/models/tacotron2/NeMo/nemo/collections/tts/models/fastspeech2_hifigan_e2e.py new file mode 100644 index 0000000000..1ca2f76bb9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/fastspeech2_hifigan_e2e.py @@ -0,0 +1,430 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re +from itertools import chain + +import numpy as np +import torch +from hydra.utils import instantiate +from omegaconf import DictConfig, OmegaConf, open_dict +from pytorch_lightning.loggers import LoggerCollection, TensorBoardLogger + +from nemo.collections.common.parts.preprocessing import parsers +from nemo.collections.tts.helpers.helpers import plot_spectrogram_to_numpy +from nemo.collections.tts.losses.fastspeech2loss import DurationLoss, L1MelLoss +from nemo.collections.tts.losses.hifigan_losses import DiscriminatorLoss, FeatureMatchingLoss, GeneratorLoss +from nemo.collections.tts.models.base import TextToWaveform +from nemo.collections.tts.modules.hifigan_modules import MultiPeriodDiscriminator, MultiScaleDiscriminator +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import ( + LengthsType, + MaskType, + MelSpectrogramType, + RegressionValuesType, + TokenDurationType, + TokenIndex, + TokenLogDurationType, +) +from nemo.core.neural_types.neural_type import NeuralType +from nemo.core.optim.lr_scheduler import NoamAnnealing +from nemo.utils import logging + + +class FastSpeech2HifiGanE2EModel(TextToWaveform): + """An end-to-end speech synthesis model based on FastSpeech2 and HiFiGan that converts strings to audio without + using the intermediate mel spectrogram representation.""" + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + super().__init__(cfg=cfg, trainer=trainer) + + self.audio_to_melspec_precessor = instantiate(cfg.preprocessor) + self.encoder = instantiate(cfg.encoder) + self.variance_adapter = instantiate(cfg.variance_adaptor) + + self.generator = instantiate(cfg.generator) + self.multiperioddisc = MultiPeriodDiscriminator() + self.multiscaledisc = MultiScaleDiscriminator() + + self.melspec_fn = instantiate(cfg.preprocessor, highfreq=None, use_grads=True) + self.mel_val_loss = L1MelLoss() + self.durationloss = DurationLoss() + self.feat_matching_loss = FeatureMatchingLoss() + self.disc_loss = DiscriminatorLoss() + self.gen_loss = GeneratorLoss() + self.mseloss = torch.nn.MSELoss() + + self.energy = cfg.add_energy_predictor + self.pitch = cfg.add_pitch_predictor + self.mel_loss_coeff = cfg.mel_loss_coeff + self.pitch_loss_coeff = cfg.pitch_loss_coeff + self.energy_loss_coeff = cfg.energy_loss_coeff + self.splice_length = cfg.splice_length + + self.use_energy_pred = False + self.use_pitch_pred = False + self.log_train_images = False + self.logged_real_samples = False + self._tb_logger = None + self.sample_rate = cfg.sample_rate + self.hop_size = cfg.hop_size + + # Parser and mappings are used for inference only. + self.parser = parsers.make_parser(name='en') + if 'mappings_filepath' in cfg: + mappings_filepath = cfg.get('mappings_filepath') + else: + logging.error( + "ERROR: You must specify a mappings.json file in the config file under model.mappings_filepath." + ) + mappings_filepath = self.register_artifact('mappings_filepath', mappings_filepath) + with open(mappings_filepath, 'r') as f: + mappings = json.load(f) + self.word2phones = mappings['word2phones'] + self.phone2idx = mappings['phone2idx'] + + @property + def tb_logger(self): + if self._tb_logger is None: + if self.logger is None and self.logger.experiment is None: + return None + tb_logger = self.logger.experiment + if isinstance(self.logger, LoggerCollection): + for logger in self.logger: + if isinstance(logger, TensorBoardLogger): + tb_logger = logger.experiment + break + self._tb_logger = tb_logger + return self._tb_logger + + def configure_optimizers(self): + gen_params = chain(self.encoder.parameters(), self.generator.parameters(), self.variance_adapter.parameters(),) + disc_params = chain(self.multiscaledisc.parameters(), self.multiperioddisc.parameters()) + opt1 = torch.optim.AdamW(disc_params, lr=self._cfg.lr) + opt2 = torch.optim.AdamW(gen_params, lr=self._cfg.lr) + num_procs = self._trainer.num_gpus * self._trainer.num_nodes + num_samples = len(self._train_dl.dataset) + batch_size = self._train_dl.batch_size + iter_per_epoch = np.ceil(num_samples / (num_procs * batch_size)) + max_steps = iter_per_epoch * self._trainer.max_epochs + logging.info(f"MAX STEPS: {max_steps}") + sch1 = NoamAnnealing(opt1, d_model=256, warmup_steps=3000, max_steps=max_steps, min_lr=1e-5) + sch1_dict = { + 'scheduler': sch1, + 'interval': 'step', + } + sch2 = NoamAnnealing(opt2, d_model=256, warmup_steps=3000, max_steps=max_steps, min_lr=1e-5) + sch2_dict = { + 'scheduler': sch2, + 'interval': 'step', + } + return [opt1, opt2], [sch1_dict, sch2_dict] + + @typecheck( + input_types={ + "text": NeuralType(('B', 'T'), TokenIndex()), + "text_length": NeuralType(('B'), LengthsType()), + "splice": NeuralType(optional=True), + "spec_len": NeuralType(('B'), LengthsType(), optional=True), + "durations": NeuralType(('B', 'T'), TokenDurationType(), optional=True), + "pitch": NeuralType(('B', 'T'), RegressionValuesType(), optional=True), + "energies": NeuralType(('B', 'T'), RegressionValuesType(), optional=True), + }, + output_types={ + "audio": NeuralType(('B', 'S', 'T'), MelSpectrogramType()), + "splices": NeuralType(), + "log_dur_preds": NeuralType(('B', 'T'), TokenLogDurationType()), + "pitch_preds": NeuralType(('B', 'T'), RegressionValuesType()), + "energy_preds": NeuralType(('B', 'T'), RegressionValuesType()), + "encoded_text_mask": NeuralType(('B', 'T', 'D'), MaskType()), + }, + ) + def forward(self, *, text, text_length, splice=True, durations=None, pitch=None, energies=None, spec_len=None): + encoded_text, encoded_text_mask = self.encoder(text=text, text_length=text_length) + + context, log_dur_preds, pitch_preds, energy_preds, spec_len = self.variance_adapter( + x=encoded_text, + x_len=text_length, + dur_target=durations, + pitch_target=pitch, + energy_target=energies, + spec_len=spec_len, + ) + + gen_in = context + splices = None + if splice: + # Splice generated spec + output = [] + splices = [] + for i, sample in enumerate(context): + start = np.random.randint(low=0, high=min(int(sample.size(0)), int(spec_len[i])) - self.splice_length) + output.append(sample[start : start + self.splice_length, :]) + splices.append(start) + gen_in = torch.stack(output) + + output = self.generator(x=gen_in.transpose(1, 2)) + + return output, splices, log_dur_preds, pitch_preds, energy_preds, encoded_text_mask + + def training_step(self, batch, batch_idx, optimizer_idx): + f, fl, t, tl, durations, pitch, energies = batch + spec, spec_len = self.audio_to_melspec_precessor(f, fl) + + # train discriminator + if optimizer_idx == 0: + with torch.no_grad(): + audio_pred, splices, _, _, _, _ = self( + spec=spec, + spec_len=spec_len, + text=t, + text_length=tl, + durations=durations, + pitch=pitch if not self.use_pitch_pred else None, + energies=energies if not self.use_energy_pred else None, + ) + real_audio = [] + for i, splice in enumerate(splices): + real_audio.append(f[i, splice * self.hop_size : (splice + self.splice_length) * self.hop_size]) + real_audio = torch.stack(real_audio).unsqueeze(1) + + real_score_mp, gen_score_mp, _, _ = self.multiperioddisc(real_audio, audio_pred) + real_score_ms, gen_score_ms, _, _ = self.multiscaledisc(real_audio, audio_pred) + + loss_mp, loss_mp_real, _ = self.disc_loss(real_score_mp, gen_score_mp) + loss_ms, loss_ms_real, _ = self.disc_loss(real_score_ms, gen_score_ms) + loss_mp /= len(loss_mp_real) + loss_ms /= len(loss_ms_real) + loss_disc = loss_mp + loss_ms + + self.log("loss_discriminator", loss_disc, prog_bar=True) + self.log("loss_discriminator_ms", loss_ms) + self.log("loss_discriminator_mp", loss_mp) + return loss_disc + + # train generator + elif optimizer_idx == 1: + audio_pred, splices, log_dur_preds, pitch_preds, energy_preds, encoded_text_mask = self( + spec=spec, + spec_len=spec_len, + text=t, + text_length=tl, + durations=durations, + pitch=pitch if not self.use_pitch_pred else None, + energies=energies if not self.use_energy_pred else None, + ) + real_audio = [] + for i, splice in enumerate(splices): + real_audio.append(f[i, splice * self.hop_size : (splice + self.splice_length) * self.hop_size]) + real_audio = torch.stack(real_audio).unsqueeze(1) + + # Do HiFiGAN generator loss + audio_length = torch.tensor([self.splice_length * self.hop_size for _ in range(real_audio.shape[0])]).to( + real_audio.device + ) + real_spliced_spec, _ = self.melspec_fn(real_audio.squeeze(), seq_len=audio_length) + pred_spliced_spec, _ = self.melspec_fn(audio_pred.squeeze(), seq_len=audio_length) + loss_mel = torch.nn.functional.l1_loss(real_spliced_spec, pred_spliced_spec) + loss_mel *= self.mel_loss_coeff + _, gen_score_mp, real_feat_mp, gen_feat_mp = self.multiperioddisc(real_audio, audio_pred) + _, gen_score_ms, real_feat_ms, gen_feat_ms = self.multiscaledisc(real_audio, audio_pred) + loss_gen_mp, list_loss_gen_mp = self.gen_loss(gen_score_mp) + loss_gen_ms, list_loss_gen_ms = self.gen_loss(gen_score_ms) + loss_gen_mp /= len(list_loss_gen_mp) + loss_gen_ms /= len(list_loss_gen_ms) + total_loss = loss_gen_mp + loss_gen_ms + loss_mel + loss_feat_mp = self.feat_matching_loss(real_feat_mp, gen_feat_mp) + loss_feat_ms = self.feat_matching_loss(real_feat_ms, gen_feat_ms) + total_loss += loss_feat_mp + loss_feat_ms + self.log(name="loss_gen_disc_feat", value=loss_feat_mp + loss_feat_ms) + self.log(name="loss_gen_disc_feat_ms", value=loss_feat_ms) + self.log(name="loss_gen_disc_feat_mp", value=loss_feat_mp) + + self.log(name="loss_gen_mel", value=loss_mel) + self.log(name="loss_gen_disc", value=loss_gen_mp + loss_gen_ms) + self.log(name="loss_gen_disc_mp", value=loss_gen_mp) + self.log(name="loss_gen_disc_ms", value=loss_gen_ms) + + dur_loss = self.durationloss( + log_duration_pred=log_dur_preds, duration_target=durations.float(), mask=encoded_text_mask + ) + self.log(name="loss_gen_duration", value=dur_loss) + total_loss += dur_loss + if self.pitch: + pitch_loss = self.mseloss(pitch_preds, pitch.float()) * self.pitch_loss_coeff + total_loss += pitch_loss + self.log(name="loss_gen_pitch", value=pitch_loss) + if self.energy: + energy_loss = self.mseloss(energy_preds, energies) * self.energy_loss_coeff + total_loss += energy_loss + self.log(name="loss_gen_energy", value=energy_loss) + + # Log images to tensorboard + if self.log_train_images: + self.log_train_images = False + if self.logger is not None and self.logger.experiment is not None: + self.tb_logger.add_image( + "train_mel_target", + plot_spectrogram_to_numpy(real_spliced_spec[0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + spec_predict = pred_spliced_spec[0].data.cpu().numpy() + self.tb_logger.add_image( + "train_mel_predicted", + plot_spectrogram_to_numpy(spec_predict), + self.global_step, + dataformats="HWC", + ) + self.log(name="loss_gen", prog_bar=True, value=total_loss) + return total_loss + + def validation_step(self, batch, batch_idx): + f, fl, t, tl, _, _, _ = batch + spec, spec_len = self.audio_to_melspec_precessor(f, fl) + audio_pred, _, _, _, _, _ = self(spec=spec, spec_len=spec_len, text=t, text_length=tl, splice=False) + audio_pred.squeeze_() + pred_spec, _ = self.melspec_fn(audio_pred, seq_len=spec_len) + loss = self.mel_val_loss(spec_pred=pred_spec, spec_target=spec, spec_target_len=spec_len, pad_value=-11.52) + + return { + "val_loss": loss, + "audio_target": f.squeeze() if batch_idx == 0 else None, + "audio_pred": audio_pred if batch_idx == 0 else None, + } + + def on_train_epoch_start(self): + # Switch to using energy predictions after 50% of training + if not self.use_energy_pred and self.current_epoch >= np.ceil(0.5 * self._trainer.max_epochs): + logging.info(f"Using energy predictions after epoch: {self.current_epoch}") + self.use_energy_pred = True + + # Switch to using pitch predictions after 62.5% of training + if not self.use_pitch_pred and self.current_epoch >= np.ceil(0.625 * self._trainer.max_epochs): + logging.info(f"Using pitch predictions after epoch: {self.current_epoch}") + self.use_pitch_pred = True + + def validation_epoch_end(self, outputs): + if self.tb_logger is not None: + _, audio_target, audio_predict = outputs[0].values() + if not self.logged_real_samples: + self.tb_logger.add_audio("val_target", audio_target[0].data.cpu(), self.global_step, self.sample_rate) + self.logged_real_samples = True + audio_predict = audio_predict[0].data.cpu() + self.tb_logger.add_audio("val_pred", audio_predict, self.global_step, self.sample_rate) + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() # This reduces across batches, not workers! + self.log('val_loss', avg_loss, sync_dist=True) + + self.log_train_images = True + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + if "dataset" not in cfg or not isinstance(cfg.dataset, DictConfig): + raise ValueError(f"No dataset for {name}") + if "dataloader_params" not in cfg or not isinstance(cfg.dataloader_params, DictConfig): + raise ValueError(f"No dataloder_params for {name}") + if shuffle_should_be: + if 'shuffle' not in cfg.dataloader_params: + logging.warning( + f"Shuffle should be set to True for {self}'s {name} dataloader but was not found in its " + "config. Manually setting to True" + ) + with open_dict(cfg.dataloader_params): + cfg.dataloader_params.shuffle = True + elif not cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to False!!!") + elif not shuffle_should_be and cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to True!!!") + + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader(dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params) + + def setup_training_data(self, cfg): + self._train_dl = self.__setup_dataloader_from_config(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self.__setup_dataloader_from_config(cfg, shuffle_should_be=False, name="validation") + + def parse(self, str_input: str, additional_word2phones=None) -> torch.tensor: + """ + Parses text input and converts them to phoneme indices. + + str_input (str): The input text to be converted. + additional_word2phones (dict): Optional dictionary mapping words to phonemes for updating the model's + word2phones. This will not overwrite the existing dictionary, just update it with OOV or new mappings. + Defaults to None, which will keep the existing mapping. + """ + # Update model's word2phones if applicable + if additional_word2phones is not None: + self.word2phones.update(additional_word2phones) + + # Convert text -> normalized text -> list of phones per word -> indices + if str_input[-1] not in [".", "!", "?"]: + str_input = str_input + "." + norm_text = re.findall(r"""[\w']+|[.,!?;"]""", self.parser._normalize(str_input)) + + try: + phones = [self.word2phones[t] for t in norm_text] + except KeyError as error: + logging.error( + f"ERROR: The following word in the input is not in the model's dictionary and could not be converted" + f" to phonemes: ({error}).\n" + f"You can pass in an `additional_word2phones` dictionary with a conversion for" + f" this word, e.g. {{'{error}': \['phone1', 'phone2', ...\]}} to update the model's mapping." + ) + raise + + tokens = [] + for phone_list in phones: + inds = [self.phone2idx[p] for p in phone_list] + tokens += inds + + x = torch.tensor(tokens).unsqueeze_(0).long().to(self.device) + return x + + def convert_text_to_waveform(self, *, tokens): + """ + Accepts tokens returned from self.parse() and returns a list of tensors. Note: The tensors in the list can have + different lengths. + """ + self.eval() + token_len = torch.tensor([len(i) for i in tokens]).to(self.device) + audio, _, log_dur_pred, _, _, _ = self(text=tokens, text_length=token_len, splice=False) + audio = audio.squeeze(1) + durations = torch.sum(torch.exp(log_dur_pred) - 1, 1).to(torch.int) + audio_list = [] + for i, sample in enumerate(audio): + audio_list.append(sample[: durations[i] * self.hop_size]) + + return audio_list + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_en_e2e_fastspeech2hifigan", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_e2e_fastspeech2hifigan/versions/1.0.0/files/tts_en_e2e_fastspeech2hifigan.nemo", + description="This model is trained on LJSpeech sampled at 22050Hz with and can be used to generate female English voices with an American accent.", + class_=cls, + ) + list_of_models.append(model) + + return list_of_models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/glow_tts.py b/models/tacotron2/NeMo/nemo/collections/tts/models/glow_tts.py new file mode 100644 index 0000000000..a5ceb18e9d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/glow_tts.py @@ -0,0 +1,279 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import torch +import torch.utils.data +from hydra.utils import instantiate +from omegaconf import MISSING, DictConfig, OmegaConf +from pytorch_lightning import Trainer +from pytorch_lightning.loggers import LoggerCollection, TensorBoardLogger + +from nemo.collections.asr.data.audio_to_text import _AudioTextDataset +from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations +from nemo.collections.tts.helpers.helpers import log_audio_to_tb, plot_alignment_to_numpy, plot_spectrogram_to_numpy +from nemo.collections.tts.losses.glow_tts_loss import GlowTTSLoss +from nemo.collections.tts.models.base import SpectrogramGenerator +from nemo.collections.tts.modules.glow_tts import GlowTTSModule +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import LengthsType, MelSpectrogramType, TokenIndex +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +@dataclass +class GlowTTSConfig: + encoder: Dict[Any, Any] = MISSING + decoder: Dict[Any, Any] = MISSING + parser: Dict[Any, Any] = MISSING + preprocessor: Dict[Any, Any] = MISSING + train_ds: Optional[Dict[Any, Any]] = None + validation_ds: Optional[Dict[Any, Any]] = None + test_ds: Optional[Dict[Any, Any]] = None + + +class GlowTTSModel(SpectrogramGenerator): + """ + GlowTTS model used to generate spectrograms from text + Consists of a text encoder and an invertible spectrogram decoder + """ + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + + self.parser = instantiate(cfg.parser) + super().__init__(cfg=cfg, trainer=trainer) + + schema = OmegaConf.structured(GlowTTSConfig) + # ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + # Ensure passed cfg is compliant with schema + OmegaConf.merge(cfg, schema) + + self.preprocessor = instantiate(self._cfg.preprocessor) + + encoder = instantiate(self._cfg.encoder) + decoder = instantiate(self._cfg.decoder) + + self.glow_tts = GlowTTSModule(encoder, decoder, n_speakers=cfg.n_speakers, gin_channels=cfg.gin_channels) + self.loss = GlowTTSLoss() + + def parse(self, str_input: str) -> torch.tensor: + if str_input[-1] not in [".", "!", "?"]: + str_input = str_input + "." + + tokens = self.parser(str_input) + + x = torch.tensor(tokens).unsqueeze_(0).long().to(self.device) + return x + + @typecheck( + input_types={ + "x": NeuralType(('B', 'T'), TokenIndex()), + "x_lengths": NeuralType(('B'), LengthsType()), + "y": NeuralType(('B', 'D', 'T'), MelSpectrogramType(), optional=True), + "y_lengths": NeuralType(('B'), LengthsType(), optional=True), + "gen": NeuralType(optional=True), + "noise_scale": NeuralType(optional=True), + "length_scale": NeuralType(optional=True), + } + ) + def forward(self, *, x, x_lengths, y=None, y_lengths=None, gen=False, noise_scale=0.3, length_scale=1.0): + if gen: + return self.glow_tts.generate_spect( + text=x, text_lengths=x_lengths, noise_scale=noise_scale, length_scale=length_scale + ) + else: + return self.glow_tts(text=x, text_lengths=x_lengths, spect=y, spect_lengths=y_lengths) + + def step(self, y, y_lengths, x, x_lengths): + z, y_m, y_logs, logdet, logw, logw_, y_lengths, attn = self( + x=x, x_lengths=x_lengths, y=y, y_lengths=y_lengths, gen=False + ) + + l_mle, l_length, logdet = self.loss( + z=z, + y_m=y_m, + y_logs=y_logs, + logdet=logdet, + logw=logw, + logw_=logw_, + x_lengths=x_lengths, + y_lengths=y_lengths, + ) + + loss = sum([l_mle, l_length]) + + return l_mle, l_length, logdet, loss, attn + + def training_step(self, batch, batch_idx): + y, y_lengths, x, x_lengths = batch + + y, y_lengths = self.preprocessor(input_signal=y, length=y_lengths) + + l_mle, l_length, logdet, loss, _ = self.step(y, y_lengths, x, x_lengths) + + output = { + "loss": loss, # required + "progress_bar": {"l_mle": l_mle, "l_length": l_length, "logdet": logdet}, + "log": {"loss": loss, "l_mle": l_mle, "l_length": l_length, "logdet": logdet}, + } + + return output + + def validation_step(self, batch, batch_idx): + y, y_lengths, x, x_lengths = batch + + y, y_lengths = self.preprocessor(input_signal=y, length=y_lengths) + + l_mle, l_length, logdet, loss, attn = self.step(y, y_lengths, x, x_lengths) + + y_gen, attn_gen = self(x=x, x_lengths=x_lengths, gen=True) + + return { + "loss": loss, + "l_mle": l_mle, + "l_length": l_length, + "logdet": logdet, + "y": y, + "y_gen": y_gen, + "x": x, + "attn": attn, + "attn_gen": attn_gen, + "progress_bar": {"l_mle": l_mle, "l_length": l_length, "logdet": logdet}, + } + + def validation_epoch_end(self, outputs): + avg_loss = torch.stack([x['loss'] for x in outputs]).mean() + avg_mle = torch.stack([x['l_mle'] for x in outputs]).mean() + avg_length_loss = torch.stack([x['l_length'] for x in outputs]).mean() + avg_logdet = torch.stack([x['logdet'] for x in outputs]).mean() + tensorboard_logs = { + 'val_loss': avg_loss, + 'val_mle': avg_mle, + 'val_length_loss': avg_length_loss, + 'val_logdet': avg_logdet, + } + if self.logger is not None and self.logger.experiment is not None: + tb_logger = self.logger.experiment + if isinstance(self.logger, LoggerCollection): + for logger in self.logger: + if isinstance(logger, TensorBoardLogger): + tb_logger = logger.experiment + break + separated_phonemes = "|".join([self.parser.symbols[c] for c in outputs[0]['x'][0]]) + tb_logger.add_text("separated phonemes", separated_phonemes, self.global_step) + tb_logger.add_image( + "real_spectrogram", + plot_spectrogram_to_numpy(outputs[0]['y'][0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + tb_logger.add_image( + "generated_spectrogram", + plot_spectrogram_to_numpy(outputs[0]['y_gen'][0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + tb_logger.add_image( + "alignment_for_real_sp", + plot_alignment_to_numpy(outputs[0]['attn'][0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + tb_logger.add_image( + "alignment_for_generated_sp", + plot_alignment_to_numpy(outputs[0]['attn_gen'][0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + log_audio_to_tb(tb_logger, outputs[0]['y'][0], "true_audio_gf", self.global_step) + log_audio_to_tb(tb_logger, outputs[0]['y_gen'][0], "generated_audio_gf", self.global_step) + return {'val_loss': avg_loss, 'log': tensorboard_logs} + + def _setup_dataloader_from_config(self, cfg: DictConfig): + + if 'manifest_filepath' in cfg and cfg['manifest_filepath'] is None: + logging.warning(f"Could not load dataset as `manifest_filepath` was None. Provided config : {cfg}") + return None + + if 'augmentor' in cfg: + augmentor = process_augmentations(cfg['augmentor']) + else: + augmentor = None + + dataset = _AudioTextDataset( + manifest_filepath=cfg['manifest_filepath'], + parser=self.parser, + sample_rate=cfg['sample_rate'], + int_values=cfg.get('int_values', False), + augmentor=augmentor, + max_duration=cfg.get('max_duration', None), + min_duration=cfg.get('min_duration', None), + max_utts=cfg.get('max_utts', 0), + trim=cfg.get('trim_silence', True), + ) + + return torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg['batch_size'], + collate_fn=dataset.collate_fn, + drop_last=cfg.get('drop_last', False), + shuffle=cfg['shuffle'], + num_workers=cfg.get('num_workers', 0), + ) + + def setup_training_data(self, train_data_config: Optional[DictConfig]): + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config) + + def setup_validation_data(self, val_data_config: Optional[DictConfig]): + self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config) + + def setup_test_data(self, test_data_config: Optional[DictConfig]): + self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config) + + def generate_spectrogram( + self, tokens: 'torch.tensor', noise_scale: float = 0.0, length_scale: float = 1.0 + ) -> torch.tensor: + + self.eval() + + token_len = torch.tensor([tokens.shape[1]]).to(self.device) + spect, _ = self(x=tokens, x_lengths=token_len, gen=True, noise_scale=noise_scale, length_scale=length_scale) + + return spect + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_en_glowtts", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_glowtts/versions/1.0.0rc1/files/tts_en_glowtts.nemo", + description="This model is trained on LJSpeech sampled at 22050Hz, and can be used to generate female English voices with an American accent.", + class_=cls, + aliases=["GlowTTS-22050Hz"], + ) + list_of_models.append(model) + return list_of_models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/hifigan.py b/models/tacotron2/NeMo/nemo/collections/tts/models/hifigan.py new file mode 100644 index 0000000000..4611657e79 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/hifigan.py @@ -0,0 +1,357 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools + +import torch +import torch.nn.functional as F +from hydra.utils import instantiate +from omegaconf import DictConfig, OmegaConf, open_dict +from pytorch_lightning.loggers.wandb import WandbLogger + +from nemo.collections.common.parts.patch_utils import stft_patch +from nemo.collections.tts.data.datalayers import MelAudioDataset +from nemo.collections.tts.helpers.helpers import plot_spectrogram_to_numpy +from nemo.collections.tts.losses.hifigan_losses import DiscriminatorLoss, FeatureMatchingLoss, GeneratorLoss +from nemo.collections.tts.models.base import Vocoder +from nemo.collections.tts.modules.hifigan_modules import MultiPeriodDiscriminator, MultiScaleDiscriminator +from nemo.core.classes import Exportable +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import AudioSignal, MelSpectrogramType +from nemo.core.neural_types.neural_type import NeuralType +from nemo.core.optim.lr_scheduler import CosineAnnealing +from nemo.utils import logging + +HAVE_WANDB = True +try: + import wandb +except ModuleNotFoundError: + HAVE_WANDB = False + + +class HifiGanModel(Vocoder, Exportable): + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + super().__init__(cfg=cfg, trainer=trainer) + + self.audio_to_melspec_precessor = instantiate(cfg.preprocessor) + # use a different melspec extractor because: + # 1. we need to pass grads + # 2. we need remove fmax limitation + self.trg_melspec_fn = instantiate(cfg.preprocessor, highfreq=None, use_grads=True) + self.generator = instantiate(cfg.generator) + self.mpd = MultiPeriodDiscriminator() + self.msd = MultiScaleDiscriminator() + self.feature_loss = FeatureMatchingLoss() + self.discriminator_loss = DiscriminatorLoss() + self.generator_loss = GeneratorLoss() + + self.l1_factor = cfg.get("l1_loss_factor", 45) + + self.sample_rate = self._cfg.preprocessor.sample_rate + self.stft_bias = None + + if self._train_dl and isinstance(self._train_dl.dataset, MelAudioDataset): + self.input_as_mel = True + else: + self.input_as_mel = False + + self.automatic_optimization = False + + def configure_optimizers(self): + self.optim_g = instantiate(self._cfg.optim, params=self.generator.parameters(),) + self.optim_d = instantiate( + self._cfg.optim, params=itertools.chain(self.msd.parameters(), self.mpd.parameters()), + ) + + self.scheduler_g = CosineAnnealing( + optimizer=self.optim_g, + max_steps=self._cfg.max_steps, + min_lr=self._cfg.sched.min_lr, + warmup_steps=self._cfg.sched.warmup_ratio * self._cfg.max_steps, + ) # Use warmup to delay start + sch1_dict = { + 'scheduler': self.scheduler_g, + 'interval': 'step', + } + + self.scheduler_d = CosineAnnealing( + optimizer=self.optim_d, max_steps=self._cfg.max_steps, min_lr=self._cfg.sched.min_lr, + ) + sch2_dict = { + 'scheduler': self.scheduler_d, + 'interval': 'step', + } + + return [self.optim_g, self.optim_d], [sch1_dict, sch2_dict] + + @property + def input_types(self): + return { + "spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + } + + @property + def output_types(self): + return { + "audio": NeuralType(('B', 'S', 'T'), AudioSignal(self.sample_rate)), + } + + @typecheck() + def forward(self, *, spec): + """ + Runs the generator, for inputs and outputs see input_types, and output_types + """ + return self.generator(x=spec) + + @typecheck( + input_types={"spec": NeuralType(('B', 'C', 'T'), MelSpectrogramType())}, + output_types={"audio": NeuralType(('B', 'T'), AudioSignal())}, + ) + def convert_spectrogram_to_audio(self, spec: 'torch.tensor') -> 'torch.tensor': + return self(spec=spec).squeeze(1) + + def training_step(self, batch, batch_idx, optimizer_idx): + # if in finetune mode the mels are pre-computed using a + # spectrogram generator + if self.input_as_mel: + audio, audio_len, audio_mel = batch + # else, we compute the mel using the ground truth audio + else: + audio, audio_len = batch + # mel as input for generator + audio_mel, _ = self.audio_to_melspec_precessor(audio, audio_len) + + # mel as input for L1 mel loss + audio_trg_mel, _ = self.trg_melspec_fn(audio, audio_len) + audio = audio.unsqueeze(1) + + audio_pred = self.generator(x=audio_mel) + audio_pred_mel, _ = self.trg_melspec_fn(audio_pred.squeeze(1), audio_len) + + # train discriminator + self.optim_d.zero_grad() + mpd_score_real, mpd_score_gen, _, _ = self.mpd(y=audio, y_hat=audio_pred.detach()) + loss_disc_mpd, _, _ = self.discriminator_loss( + disc_real_outputs=mpd_score_real, disc_generated_outputs=mpd_score_gen + ) + msd_score_real, msd_score_gen, _, _ = self.msd(y=audio, y_hat=audio_pred.detach()) + loss_disc_msd, _, _ = self.discriminator_loss( + disc_real_outputs=msd_score_real, disc_generated_outputs=msd_score_gen + ) + loss_d = loss_disc_msd + loss_disc_mpd + self.manual_backward(loss_d) + self.optim_d.step() + + # train generator + self.optim_g.zero_grad() + loss_mel = F.l1_loss(audio_pred_mel, audio_trg_mel) + _, mpd_score_gen, fmap_mpd_real, fmap_mpd_gen = self.mpd(y=audio, y_hat=audio_pred) + _, msd_score_gen, fmap_msd_real, fmap_msd_gen = self.msd(y=audio, y_hat=audio_pred) + loss_fm_mpd = self.feature_loss(fmap_r=fmap_mpd_real, fmap_g=fmap_mpd_gen) + loss_fm_msd = self.feature_loss(fmap_r=fmap_msd_real, fmap_g=fmap_msd_gen) + loss_gen_mpd, _ = self.generator_loss(disc_outputs=mpd_score_gen) + loss_gen_msd, _ = self.generator_loss(disc_outputs=msd_score_gen) + loss_g = loss_gen_msd + loss_gen_mpd + loss_fm_msd + loss_fm_mpd + loss_mel * self.l1_factor + self.manual_backward(loss_g) + self.optim_g.step() + + metrics = { + "g_l1_loss": loss_mel, + "g_loss_fm_mpd": loss_fm_mpd, + "g_loss_fm_msd": loss_fm_msd, + "g_loss_gen_mpd": loss_gen_mpd, + "g_loss_gen_msd": loss_gen_msd, + "g_loss": loss_g, + "d_loss_mpd": loss_disc_mpd, + "d_loss_msd": loss_disc_msd, + "d_loss": loss_d, + "global_step": self.global_step, + "lr": self.optim_g.param_groups[0]['lr'], + } + self.log_dict(metrics, on_step=True, sync_dist=True) + self.log("g_l1_loss", loss_mel, prog_bar=True, logger=False, sync_dist=True) + + def validation_step(self, batch, batch_idx): + if self.input_as_mel: + audio, audio_len, audio_mel = batch + audio_mel_len = [audio_mel.shape[1]] * audio_mel.shape[0] + else: + audio, audio_len = batch + audio_mel, audio_mel_len = self.audio_to_melspec_precessor(audio, audio_len) + audio_pred = self(spec=audio_mel) + + # perform bias denoising + pred_denoised = self._bias_denoise(audio_pred, audio_mel).squeeze(1) + pred_denoised_mel, _ = self.audio_to_melspec_precessor(pred_denoised, audio_len) + + if self.input_as_mel: + gt_mel, gt_mel_len = self.audio_to_melspec_precessor(audio, audio_len) + audio_pred_mel, _ = self.audio_to_melspec_precessor(audio_pred.squeeze(1), audio_len) + loss_mel = F.l1_loss(audio_mel, audio_pred_mel) + + self.log_dict({"val_loss": loss_mel}, on_epoch=True, sync_dist=True) + + # plot audio once per epoch + if batch_idx == 0 and isinstance(self.logger, WandbLogger) and HAVE_WANDB: + clips = [] + specs = [] + for i in range(min(5, audio.shape[0])): + clips += [ + wandb.Audio( + audio[i, : audio_len[i]].data.cpu().numpy(), + caption=f"real audio {i}", + sample_rate=self.sample_rate, + ), + wandb.Audio( + audio_pred[i, 0, : audio_len[i]].data.cpu().numpy().astype('float32'), + caption=f"generated audio {i}", + sample_rate=self.sample_rate, + ), + wandb.Audio( + pred_denoised[i, : audio_len[i]].data.cpu().numpy(), + caption=f"denoised audio {i}", + sample_rate=self.sample_rate, + ), + ] + specs += [ + wandb.Image( + plot_spectrogram_to_numpy(audio_mel[i, :, : audio_mel_len[i]].data.cpu().numpy()), + caption=f"input mel {i}", + ), + wandb.Image( + plot_spectrogram_to_numpy(audio_pred_mel[i, :, : audio_mel_len[i]].data.cpu().numpy()), + caption=f"output mel {i}", + ), + wandb.Image( + plot_spectrogram_to_numpy(pred_denoised_mel[i, :, : audio_mel_len[i]].data.cpu().numpy()), + caption=f"denoised mel {i}", + ), + ] + if self.input_as_mel: + specs += [ + wandb.Image( + plot_spectrogram_to_numpy(gt_mel[i, :, : audio_mel_len[i]].data.cpu().numpy()), + caption=f"gt mel {i}", + ), + ] + + self.logger.experiment.log({"audio": clips, "specs": specs}) + + def _bias_denoise(self, audio, mel): + def stft(x): + comp = stft_patch(x.squeeze(1), n_fft=1024, hop_length=256, win_length=1024) + real, imag = comp[..., 0], comp[..., 1] + mags = torch.sqrt(real ** 2 + imag ** 2) + phase = torch.atan2(imag, real) + return mags, phase + + def istft(mags, phase): + comp = torch.stack([mags * torch.cos(phase), mags * torch.sin(phase)], dim=-1) + x = torch.istft(comp, n_fft=1024, hop_length=256, win_length=1024) + return x + + # create bias tensor + if self.stft_bias is None or self.stft_bias.shape[0] != audio.shape[0]: + audio_bias = self(spec=torch.zeros_like(mel, device=mel.device)) + self.stft_bias, _ = stft(audio_bias) + self.stft_bias = self.stft_bias[:, :, 0][:, :, None] + + audio_mags, audio_phase = stft(audio) + audio_mags = audio_mags - self.cfg.get("denoise_strength", 0.0025) * self.stft_bias + audio_mags = torch.clamp(audio_mags, 0.0) + audio_denoised = istft(audio_mags, audio_phase).unsqueeze(1) + + return audio_denoised + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + if "dataset" not in cfg or not isinstance(cfg.dataset, DictConfig): + raise ValueError(f"No dataset for {name}") + if "dataloader_params" not in cfg or not isinstance(cfg.dataloader_params, DictConfig): + raise ValueError(f"No dataloder_params for {name}") + if shuffle_should_be: + if 'shuffle' not in cfg.dataloader_params: + logging.warning( + f"Shuffle should be set to True for {self}'s {name} dataloader but was not found in its " + "config. Manually setting to True" + ) + with open_dict(cfg["dataloader_params"]): + cfg.dataloader_params.shuffle = True + elif not cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to False!!!") + elif not shuffle_should_be and cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to True!!!") + + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader(dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params) + + def setup_training_data(self, cfg): + self._train_dl = self.__setup_dataloader_from_config(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self.__setup_dataloader_from_config(cfg, shuffle_should_be=False, name="validation") + + @classmethod + def list_available_models(cls) -> 'Optional[Dict[str, str]]': + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_hifigan", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_hifigan/versions/1.0.0rc1/files/tts_hifigan.nemo", + description="This model is trained on LJSpeech audio sampled at 22050Hz and mel spectrograms generated from Tacotron2, TalkNet, and FastPitch. This model has been tested on generating female English voices with an American accent.", + class_=cls, + ) + list_of_models.append(model) + return list_of_models + + def load_state_dict(self, state_dict, strict=True): + # override load_state_dict to give us some flexibility to be backward-compatible + # with old checkpoints + new_state_dict = {} + num_resblocks = len(self.cfg['generator']['resblock_kernel_sizes']) + for k, v in state_dict.items(): + new_k = k + if 'resblocks' in k: + parts = k.split(".") + # only do this is the checkpoint type is older + if len(parts) == 6: + layer = int(parts[2]) + new_layer = f"{layer//num_resblocks}.{layer%num_resblocks}" + new_k = f"generator.resblocks.{new_layer}.{'.'.join(parts[3:])}" + new_state_dict[new_k] = v + super().load_state_dict(new_state_dict, strict=strict) + + def _prepare_for_export(self, **kwargs): + """ + Override this method to prepare module for export. This is in-place operation. + Base version does common necessary module replacements (Apex etc) + """ + if self.generator is not None: + self.generator.remove_weight_norm() + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + par = next(self.parameters()) + mel = torch.randn((1, self.cfg['preprocessor']['nfilt'], 96), device=par.device, dtype=par.dtype) + return ({'spec': mel},) + + def forward_for_export(self, spec): + """ + Runs the generator, for inputs and outputs see input_types, and output_types + """ + return self.generator(x=spec) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/melgan.py b/models/tacotron2/NeMo/nemo/collections/tts/models/melgan.py new file mode 100644 index 0000000000..6eac17c6ef --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/melgan.py @@ -0,0 +1,338 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import numpy as np +import torch +from hydra.utils import instantiate +from omegaconf import DictConfig, OmegaConf, open_dict + +from nemo.collections.tts.helpers.helpers import get_mask_from_lengths, plot_spectrogram_to_numpy +from nemo.collections.tts.models.base import Vocoder +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import AudioSignal, MelSpectrogramType +from nemo.core.neural_types.neural_type import NeuralType +from nemo.core.optim.lr_scheduler import CosineAnnealing +from nemo.utils import logging + + +class MelGanModel(Vocoder): + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + """NeMo Model that implement Full-band MelGAN as described in https://arxiv.org/abs/2005.05106 + """ + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + super().__init__(cfg=cfg, trainer=trainer) + + self.audio_to_melspec_precessor = instantiate(self._cfg.preprocessor) + self.generator = instantiate(self._cfg.generator) + if "discriminator" in self._cfg: + self.discriminator = instantiate(self._cfg.discriminator) + + self.loss = instantiate(self._cfg.loss) + self.mse_loss = torch.nn.MSELoss() # Used for LSE GAN loss + + self.start_training_disc = False + self.logged_real_samples = False + self.sample_rate = self._cfg.preprocessor.sample_rate + + def configure_optimizers(self): + opt1 = torch.optim.Adam(self.discriminator.parameters(), lr=1e-3, eps=1e-07, amsgrad=True) + opt2 = torch.optim.Adam(self.generator.parameters(), lr=1e-3, eps=1e-07, amsgrad=True) + num_procs = self._trainer.num_gpus * self._trainer.num_nodes + num_samples = len(self._train_dl.dataset) + batch_size = self._train_dl.batch_size + iter_per_epoch = np.ceil(num_samples / (num_procs * batch_size)) + max_steps = iter_per_epoch * self._trainer.max_epochs + logging.info(f"MAX STEPS: {max_steps}") + sch1 = CosineAnnealing( + opt1, max_steps=max_steps, min_lr=1e-5, warmup_steps=np.ceil(0.2 * max_steps) + ) # Use warmup to delay start + sch1_dict = { + 'scheduler': sch1, + 'interval': 'step', + } + sch2 = CosineAnnealing(opt2, max_steps=max_steps, min_lr=1e-5) + sch2_dict = { + 'scheduler': sch2, + 'interval': 'step', + } + return [opt1, opt2], [sch1_dict, sch2_dict] + + @property + def input_types(self): + return { + "spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + } + + @property + def output_types(self): + return { + "audio": NeuralType(('B', 'S', 'T'), AudioSignal(self.sample_rate)), + } + + @typecheck() + def forward(self, *, spec): + """ + Runs the generator, for inputs and outputs see input_types, and output_types + """ + return self.generator(spec=spec) + + @typecheck(output_types={"audio": NeuralType(('B', 'T'), AudioSignal())}) + def convert_spectrogram_to_audio(self, spec: 'torch.tensor') -> 'torch.tensor': + return self(spec=spec).squeeze(1) + + def training_step(self, batch, batch_idx, optimizer_idx): + audio, audio_len = batch + spec, _ = self.audio_to_melspec_precessor(audio, audio_len) + audio_pred = self(spec=spec) + + # TODO: Lightning has a bug in 1.1.0, just always log something as a workaround + # https://github.com/PyTorchLightning/pytorch-lightning/issues/5063 + self.log("Dummy", 0.0, logger=False) + + # train discriminator + if optimizer_idx == 0 and self.start_training_disc: + fake_score = self.discriminator(x=audio_pred.detach())[0] + real_score = self.discriminator(x=audio.unsqueeze(1))[0] + + loss_disc_real = [0] * len(fake_score) + loss_disc_fake = [0] * len(fake_score) + for i, _ in enumerate(fake_score): + loss_disc_real[i] += self.mse_loss(real_score[i], real_score[i].new_ones(real_score[i].size())) + loss_disc_fake[i] += torch.mean(fake_score[i] ** 2) + sum_loss_dis = sum(loss_disc_real) + sum(loss_disc_fake) + sum_loss_dis /= len(fake_score) + + self.log("loss_discriminator", sum_loss_dis, prog_bar=True, sync_dist=True) + for i, _ in enumerate(fake_score): + self.log(f"loss_discriminator_real_{i}", loss_disc_real[i] / len(fake_score), sync_dist=True) + self.log(f"loss_discriminator_fake_{i}", loss_disc_fake[i] / len(fake_score), sync_dist=True) + return sum_loss_dis + + # train generator + elif optimizer_idx == 1: + loss = 0 + + # full-band loss + sc_loss, mag_loss = self.loss(x=audio_pred.squeeze(1), y=audio) + loss_feat = sum(sc_loss) + sum(mag_loss) + loss_feat /= len(sc_loss) + loss += loss_feat + + if self.start_training_disc: + fake_score = self.discriminator(x=audio_pred)[0] + + loss_gan = [0] * len(fake_score) + for i, scale in enumerate(fake_score): + loss_gan[i] += self.mse_loss(scale, scale.new_ones(scale.size())) + + sum_loss_gan = sum(loss_gan) / len(fake_score) + + loss += sum_loss_gan + + self.log("loss_generator", loss, sync_dist=True, prog_bar=True) + if self.start_training_disc: + self.log("loss_generator_gan_loss", sum_loss_gan, sync_dist=True) + for i, _ in enumerate(fake_score): + self.log( + f"loss_generator_gan_loss_{i}", loss_gan[i] / len(fake_score), sync_dist=True, + ) + self.log("loss_generator_feat_loss", loss_feat, sync_dist=True) + self.log("loss_generator_feat_loss_fb_sc", sum(sc_loss) / len(sc_loss), sync_dist=True) + self.log("loss_generator_feat_loss_fb_mag", sum(mag_loss) / len(sc_loss), sync_dist=True) + for i, _ in enumerate(sc_loss): + self.log(f"loss_generator_feat_loss_fb_sc_{i}", sc_loss[i] / len(sc_loss), sync_dist=True) + self.log(f"loss_generator_feat_loss_fb_mag_{i}", mag_loss[i] / len(sc_loss), sync_dist=True) + + return loss + return None + + def validation_step(self, batch, batch_idx): + audio, audio_len = batch + with torch.no_grad(): + spec, _ = self.audio_to_melspec_precessor(audio, audio_len) + audio_pred = self(spec=spec) + + loss = 0 + loss_dict = {} + spec_pred, _ = self.audio_to_melspec_precessor(audio_pred.squeeze(1), audio_len) + + # Ensure that audio len is consistent between audio_pred and audio + # For SC Norm loss, we can just zero out + # For Mag L1 loss, we need to mask + if audio_pred.shape[-1] < audio.shape[-1]: + # prediction audio is less than audio, pad predicted audio to real audio + pad_amount = audio.shape[-1] - audio_pred.shape[-1] + audio_pred = torch.nn.functional.pad(audio_pred, (0, pad_amount), value=0.0) + else: + # prediction audio is larger than audio, slice predicted audio to real audio + audio_pred = audio_pred[:, :, : audio.shape[1]] + + mask = ~get_mask_from_lengths(audio_len, max_len=torch.max(audio_len)) + mask = mask.unsqueeze(1) + audio_pred.data.masked_fill_(mask, 0.0) + + # full-band loss + sc_loss, mag_loss = self.loss(x=audio_pred.squeeze(1), y=audio, input_lengths=audio_len) + loss_feat = (sum(sc_loss) + sum(mag_loss)) / len(sc_loss) + loss_dict["sc_loss"] = sc_loss + loss_dict["mag_loss"] = mag_loss + + loss += loss_feat + loss_dict["loss_feat"] = loss_feat + + if self.start_training_disc: + fake_score = self.discriminator(x=audio_pred)[0] + + loss_gen = [0] * len(fake_score) + for i, scale in enumerate(fake_score): + loss_gen[i] += self.mse_loss(scale, scale.new_ones(scale.size())) + + loss_dict["gan_loss"] = loss_gen + loss += sum(loss_gen) / len(fake_score) + + if not self.logged_real_samples: + loss_dict["spec"] = spec + loss_dict["audio"] = audio + loss_dict["audio_pred"] = audio_pred + loss_dict["spec_pred"] = spec_pred + loss_dict["loss"] = loss + return loss_dict + + def validation_epoch_end(self, outputs): + # Los images and audio manually + if self.logger is not None and self.logger.experiment is not None: + if not self.logged_real_samples: + self.logger.experiment.add_image( + "val_mel_target", + plot_spectrogram_to_numpy(outputs[0]["spec"][0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + self.logger.experiment.add_audio( + "val_wav_target", + outputs[0]["audio"][0].data.cpu().numpy(), + self.global_step, + sample_rate=self.sample_rate, + ) + self.logged_real_samples = True + self.logger.experiment.add_image( + "val_mel_predicted", + plot_spectrogram_to_numpy(outputs[0]["spec_pred"][0].data.cpu().numpy()), + self.global_step, + dataformats="HWC", + ) + self.logger.experiment.add_audio( + "val_wav_predicted", + outputs[0]["audio_pred"][0].data.cpu().numpy(), + self.global_step, + sample_rate=self.sample_rate, + ) + + def get_stack(list_of_dict, key): + """ + Helper function to take a list of losses and reduce across all validation batches + """ + return_list = [[]] * len(list_of_dict[0][key]) + for dict_ in list_of_dict: + list_of_losses = dict_[key] + for i, loss in enumerate(list_of_losses): + return_list[i].append(loss) + for i, loss in enumerate(return_list): + return_list[i] = torch.mean(torch.stack(loss)) + return return_list + + loss = torch.stack([x['loss'] for x in outputs]).mean() + self.log("val_loss", loss, sync_dist=True) + + if self.start_training_disc: + gan_loss = get_stack(outputs, "gan_loss") + self.log("val_loss_gan_loss", sum(gan_loss) / len(gan_loss), sync_dist=True) + for i, _ in enumerate(gan_loss): + self.log( + f"val_loss_gan_loss_{i}", gan_loss[i] / len(gan_loss), sync_dist=True, + ) + + sc_loss = get_stack(outputs, "sc_loss") + mag_loss = get_stack(outputs, "mag_loss") + self.log("val_loss_feat_loss", torch.stack([x['loss_feat'] for x in outputs]).mean(), sync_dist=True) + self.log("val_loss_feat_loss_fb_sc", sum(sc_loss) / len(sc_loss), sync_dist=True) + self.log("val_loss_feat_loss_fb_mag", sum(mag_loss) / len(sc_loss), sync_dist=True) + for i, _ in enumerate(sc_loss): + self.log(f"val_loss_feat_loss_fb_sc_{i}", sc_loss[i] / len(sc_loss), sync_dist=True) + self.log(f"val_loss_feat_loss_fb_mag_{i}", mag_loss[i] / len(sc_loss), sync_dist=True) + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + if "dataset" not in cfg or not isinstance(cfg.dataset, DictConfig): + raise ValueError(f"No dataset for {name}") + if "dataloader_params" not in cfg or not isinstance(cfg.dataloader_params, DictConfig): + raise ValueError(f"No dataloder_params for {name}") + if shuffle_should_be: + if 'shuffle' not in cfg.dataloader_params: + logging.warning( + f"Shuffle should be set to True for {self}'s {name} dataloader but was not found in its " + "config. Manually setting to True" + ) + with open_dict(cfg["dataloader_params"]): + cfg.dataloader_params.shuffle = True + elif not cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to False!!!") + elif not shuffle_should_be and cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to True!!!") + + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader(dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params) + + def setup_training_data(self, cfg): + self._train_dl = self.__setup_dataloader_from_config(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self.__setup_dataloader_from_config(cfg, shuffle_should_be=False, name="validation") + + def training_epoch_end(self, outputs): + # Do manual logginging of learning rate and epoch + if self.current_epoch % 100 == 0: + lrs = [] + for scheduler in self._trainer.lr_schedulers: + param_groups = scheduler['scheduler'].optimizer.param_groups + lrs.append(param_groups[0]['lr']) + self.logger.experiment.add_scalar("lr-Adam", lrs[0], self.global_step) + self.logger.experiment.add_scalar("lr-Adam-1", lrs[1], self.global_step) + self.logger.experiment.add_scalar("epoch", self.current_epoch, self.global_step) + + # Start training discriminator after 20% of training + if self.current_epoch >= np.ceil(0.2 * self._trainer.max_epochs): + logging.info(f"MelGAN discriminator was enabled at epoch: {self.current_epoch}") + self.start_training_disc = True + + return super().training_epoch_end(outputs) + + @classmethod + def list_available_models(cls) -> 'Optional[Dict[str, str]]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_melgan", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_melgan/versions/1.0.0/files/tts_melgan.nemo", + description="This model is trained on LJSpeech sampled at 22050Hz, and has been tested on generating female English voices with an American accent.", + class_=cls, + ) + list_of_models.append(model) + return list_of_models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/squeezewave.py b/models/tacotron2/NeMo/nemo/collections/tts/models/squeezewave.py new file mode 100644 index 0000000000..56212d14a6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/squeezewave.py @@ -0,0 +1,227 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import torch +from hydra.utils import instantiate +from omegaconf import MISSING, DictConfig, OmegaConf, open_dict +from pytorch_lightning.loggers import LoggerCollection, TensorBoardLogger + +from nemo.collections.tts.helpers.helpers import OperationMode, waveglow_log_to_tb_func +from nemo.collections.tts.losses.waveglowloss import WaveGlowLoss +from nemo.collections.tts.models.base import GlowVocoder +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import ( + AudioSignal, + LengthsType, + LogDeterminantType, + MelSpectrogramType, + NormalDistributionSamplesType, + VoidType, +) +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +@dataclass +class SqueezeWaveConfig: + squeezewave: Dict[Any, Any] = MISSING + preprocessor: Dict[Any, Any] = MISSING + sigma: float = MISSING + train_ds: Optional[Dict[Any, Any]] = None + validation_ds: Optional[Dict[Any, Any]] = None + + +class SqueezeWaveModel(GlowVocoder): + """ SqueezeWave model that generates audio conditioned on mel-spectrogram + """ + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + super().__init__(cfg=cfg, trainer=trainer) + + schema = OmegaConf.structured(SqueezeWaveConfig) + # ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + # Ensure passed cfg is compliant with schema + OmegaConf.merge(cfg, schema) + + self.sigma = self._cfg.sigma + self.audio_to_melspec_precessor = instantiate(self._cfg.preprocessor) + self.squeezewave = instantiate(self._cfg.squeezewave) + self.loss = WaveGlowLoss() # Same loss as WaveGlow + + @GlowVocoder.mode.setter + def mode(self, new_mode): + if new_mode == OperationMode.training: + self.train() + else: + self.eval() + self._mode = new_mode + self.squeezewave.mode = new_mode + + @property + def input_types(self): + return { + "audio": NeuralType(('B', 'T'), AudioSignal()), + "audio_len": NeuralType(('B'), LengthsType()), + "run_inverse": NeuralType(optional=True), + } + + @property + def output_types(self): + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + output_dict = { + "pred_normal_dist": NeuralType(('B', 'flowgroup', 'T'), NormalDistributionSamplesType()), + "log_s_list": [NeuralType(('B', 'flowgroup', 'T'), VoidType())], # TODO: Figure out a good typing + "log_det_W_list": [NeuralType(elements_type=LogDeterminantType())], + } + if self.mode == OperationMode.validation: + output_dict["audio_pred"] = NeuralType(('B', 'T'), AudioSignal()) + output_dict["spec"] = NeuralType(('B', 'T', 'D'), MelSpectrogramType()) + output_dict["spec_len"] = NeuralType(('B'), LengthsType()) + return output_dict + return { + "audio_pred": NeuralType(('B', 'T'), AudioSignal()), + } + + @typecheck() + def forward(self, *, audio, audio_len, run_inverse=True): + if self.mode != self.squeezewave.mode: + raise ValueError( + f"SqueezeWaveModel's mode {self.mode} does not match SqueezeWaveModule's mode {self.squeezewave.mode}" + ) + spec, spec_len = self.audio_to_melspec_precessor(audio, audio_len) + tensors = self.squeezewave(spec=spec, audio=audio, run_inverse=run_inverse) + if self.mode == OperationMode.training: + return tensors[:-1] # z, log_s_list, log_det_W_list + elif self.mode == OperationMode.validation: + z, log_s_list, log_det_W_list, audio_pred = tensors + return z, log_s_list, log_det_W_list, audio_pred, spec, spec_len + return tensors # audio_pred + + @typecheck( + input_types={ + "spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "sigma": NeuralType(optional=True), + "denoise": NeuralType(optional=True), + "denoiser_strength": NeuralType(optional=True), + }, + output_types={"audio": NeuralType(('B', 'T'), AudioSignal())}, + ) + def convert_spectrogram_to_audio( + self, spec: torch.Tensor, sigma: bool = 1.0, denoise: bool = True, denoiser_strength: float = 0.01 + ) -> torch.Tensor: + with self.nemo_infer(): + audio = self.squeezewave(spec=spec, run_inverse=True, audio=None, sigma=sigma) + if denoise: + audio = self.denoise(audio=audio, strength=denoiser_strength) + + return audio + + def training_step(self, batch, batch_idx): + self.mode = OperationMode.training + + audio, audio_len = batch + z, log_s_list, log_det_W_list = self.forward(audio=audio, audio_len=audio_len, run_inverse=False) + + loss = self.loss(z=z, log_s_list=log_s_list, log_det_W_list=log_det_W_list, sigma=self.sigma) + return { + 'loss': loss, + 'progress_bar': {'training_loss': loss}, + 'log': {'loss': loss}, + } + + def validation_step(self, batch, batch_idx): + self.mode = OperationMode.validation + + audio, audio_len = batch + z, log_s_list, log_det_W_list, audio_pred, spec, spec_len = self.forward( + audio=audio, audio_len=audio_len, run_inverse=(batch_idx == 0) + ) + loss = self.loss(z=z, log_s_list=log_s_list, log_det_W_list=log_det_W_list, sigma=self.sigma) + return { + "val_loss": loss, + "audio_pred": audio_pred, + "mel_target": spec, + "mel_len": spec_len, + } + + def validation_epoch_end(self, outputs): + if self.logger is not None and self.logger.experiment is not None: + tb_logger = self.logger.experiment + if isinstance(self.logger, LoggerCollection): + for logger in self.logger: + if isinstance(logger, TensorBoardLogger): + tb_logger = logger.experiment + break + waveglow_log_to_tb_func( + tb_logger, + outputs[0].values(), + self.global_step, + tag="eval", + mel_fb=self.audio_to_melspec_precessor.fb, + ) + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() + self.log('val_loss', avg_loss) + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + if "dataset" not in cfg or not isinstance(cfg.dataset, DictConfig): + raise ValueError(f"No dataset for {name}") + if "dataloader_params" not in cfg or not isinstance(cfg.dataloader_params, DictConfig): + raise ValueError(f"No dataloder_params for {name}") + if shuffle_should_be: + if 'shuffle' not in cfg.dataloader_params: + logging.warning( + f"Shuffle should be set to True for {self}'s {name} dataloader but was not found in its " + "config. Manually setting to True" + ) + with open_dict(cfg["dataloader_params"]): + cfg.dataloader_params.shuffle = True + elif not cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to False!!!") + elif not shuffle_should_be and cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to True!!!") + + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader(dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params) + + def setup_training_data(self, cfg): + self._train_dl = self.__setup_dataloader_from_config(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self.__setup_dataloader_from_config(cfg, shuffle_should_be=False, name="validation") + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_squeezewave", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_squeezewave/versions/1.0.0rc1/files/tts_squeezewave.nemo", + description="This model is trained on LJSpeech sampled at 22050Hz, and has been tested on generating female English voices with an American accent.", + class_=cls, + ) + list_of_models.append(model) + return list_of_models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/tacotron2.py b/models/tacotron2/NeMo/nemo/collections/tts/models/tacotron2.py new file mode 100644 index 0000000000..f608ac206d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/tacotron2.py @@ -0,0 +1,321 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +import torch +from hydra.utils import instantiate +from omegaconf import MISSING, DictConfig, OmegaConf, open_dict +from omegaconf.errors import ConfigAttributeError +from pytorch_lightning.loggers import LoggerCollection, TensorBoardLogger +from torch import nn + +from nemo.collections.common.parts.preprocessing import parsers +from nemo.collections.tts.helpers.helpers import get_mask_from_lengths, tacotron2_log_to_tb_func +from nemo.collections.tts.losses.tacotron2loss import Tacotron2Loss +from nemo.collections.tts.models.base import SpectrogramGenerator +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import ( + AudioSignal, + EmbeddedTextType, + LengthsType, + LogitsType, + MelSpectrogramType, + SequenceToSequenceAlignmentType, +) +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +@dataclass +class Preprocessor: + _target_: str = MISSING + pad_value: float = MISSING + + +@dataclass +class Tacotron2Config: + preprocessor: Preprocessor = Preprocessor() + encoder: Dict[Any, Any] = MISSING + decoder: Dict[Any, Any] = MISSING + postnet: Dict[Any, Any] = MISSING + labels: List = MISSING + train_ds: Optional[Dict[Any, Any]] = None + validation_ds: Optional[Dict[Any, Any]] = None + + +class Tacotron2Model(SpectrogramGenerator): + """Tacotron 2 Model that is used to generate mel spectrograms from text""" + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + super().__init__(cfg=cfg, trainer=trainer) + + schema = OmegaConf.structured(Tacotron2Config) + # ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + # Ensure passed cfg is compliant with schema + try: + OmegaConf.merge(cfg, schema) + self.pad_value = self._cfg.preprocessor.pad_value + except ConfigAttributeError: + self.pad_value = self._cfg.preprocessor.params.pad_value + logging.warning( + "Your config is using an old NeMo yaml configuration. Please ensure that the yaml matches the " + "current version in the main branch for future compatibility." + ) + + self._parser = None + self.audio_to_melspec_precessor = instantiate(self._cfg.preprocessor) + self.text_embedding = nn.Embedding(len(cfg.labels) + 3, 512) + self.encoder = instantiate(self._cfg.encoder) + self.decoder = instantiate(self._cfg.decoder) + self.postnet = instantiate(self._cfg.postnet) + self.loss = Tacotron2Loss() + self.calculate_loss = True + + @property + def parser(self): + if self._parser is not None: + return self._parser + if self._validation_dl is not None: + return self._validation_dl.dataset.parser + if self._test_dl is not None: + return self._test_dl.dataset.parser + if self._train_dl is not None: + return self._train_dl.dataset.parser + + # Else construct a parser + # Try to get params from validation, test, and then train + params = {} + try: + params = self._cfg.validation_ds.dataset + except ConfigAttributeError: + pass + if params == {}: + try: + params = self._cfg.test_ds.dataset + except ConfigAttributeError: + pass + if params == {}: + try: + params = self._cfg.train_ds.dataset + except ConfigAttributeError: + pass + + name = params.get('parser', None) or 'en' + unk_id = params.get('unk_index', None) or -1 + blank_id = params.get('blank_index', None) or -1 + do_normalize = params.get('normalize', None) or False + self._parser = parsers.make_parser( + labels=self._cfg.labels, name=name, unk_id=unk_id, blank_id=blank_id, do_normalize=do_normalize, + ) + return self._parser + + def parse(self, str_input: str) -> torch.tensor: + tokens = self.parser(str_input) + # Parser doesn't add bos and eos ids, so maunally add it + tokens = [len(self._cfg.labels)] + tokens + [len(self._cfg.labels) + 1] + tokens_tensor = torch.tensor(tokens).unsqueeze_(0).to(self.device) + + return tokens_tensor + + @property + def input_types(self): + if self.training: + return { + "tokens": NeuralType(('B', 'T'), EmbeddedTextType()), + "token_len": NeuralType(('B'), LengthsType()), + "audio": NeuralType(('B', 'T'), AudioSignal()), + "audio_len": NeuralType(('B'), LengthsType()), + } + else: + return { + "tokens": NeuralType(('B', 'T'), EmbeddedTextType()), + "token_len": NeuralType(('B'), LengthsType()), + "audio": NeuralType(('B', 'T'), AudioSignal(), optional=True), + "audio_len": NeuralType(('B'), LengthsType(), optional=True), + } + + @property + def output_types(self): + if not self.calculate_loss and not self.training: + return { + "spec_pred_dec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "spec_pred_postnet": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "gate_pred": NeuralType(('B', 'T'), LogitsType()), + "alignments": NeuralType(('B', 'T', 'T'), SequenceToSequenceAlignmentType()), + "pred_length": NeuralType(('B'), LengthsType()), + } + return { + "spec_pred_dec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "spec_pred_postnet": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "gate_pred": NeuralType(('B', 'T'), LogitsType()), + "spec_target": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "spec_target_len": NeuralType(('B'), LengthsType()), + "alignments": NeuralType(('B', 'T', 'T'), SequenceToSequenceAlignmentType()), + } + + @typecheck() + def forward(self, *, tokens, token_len, audio=None, audio_len=None): + if audio is not None and audio_len is not None: + spec_target, spec_target_len = self.audio_to_melspec_precessor(audio, audio_len) + token_embedding = self.text_embedding(tokens).transpose(1, 2) + encoder_embedding = self.encoder(token_embedding=token_embedding, token_len=token_len) + if self.training: + spec_pred_dec, gate_pred, alignments = self.decoder( + memory=encoder_embedding, decoder_inputs=spec_target, memory_lengths=token_len + ) + else: + spec_pred_dec, gate_pred, alignments, pred_length = self.decoder( + memory=encoder_embedding, memory_lengths=token_len + ) + spec_pred_postnet = self.postnet(mel_spec=spec_pred_dec) + + if not self.calculate_loss: + return spec_pred_dec, spec_pred_postnet, gate_pred, alignments, pred_length + return spec_pred_dec, spec_pred_postnet, gate_pred, spec_target, spec_target_len, alignments + + @typecheck( + input_types={"tokens": NeuralType(('B', 'T'), EmbeddedTextType())}, + output_types={"spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType())}, + ) + def generate_spectrogram(self, *, tokens): + self.eval() + self.calculate_loss = False + token_len = torch.tensor([len(i) for i in tokens]).to(self.device) + tensors = self(tokens=tokens, token_len=token_len) + spectrogram_pred = tensors[1] + + if spectrogram_pred.shape[0] > 1: + # Silence all frames past the predicted end + mask = ~get_mask_from_lengths(tensors[-1]) + mask = mask.expand(spectrogram_pred.shape[1], mask.size(0), mask.size(1)) + mask = mask.permute(1, 0, 2) + spectrogram_pred.data.masked_fill_(mask, self.pad_value) + + return spectrogram_pred + + def training_step(self, batch, batch_idx): + audio, audio_len, tokens, token_len = batch + spec_pred_dec, spec_pred_postnet, gate_pred, spec_target, spec_target_len, _ = self.forward( + audio=audio, audio_len=audio_len, tokens=tokens, token_len=token_len + ) + + loss, _ = self.loss( + spec_pred_dec=spec_pred_dec, + spec_pred_postnet=spec_pred_postnet, + gate_pred=gate_pred, + spec_target=spec_target, + spec_target_len=spec_target_len, + pad_value=self.pad_value, + ) + + output = { + 'loss': loss, + 'progress_bar': {'training_loss': loss}, + 'log': {'loss': loss}, + } + return output + + def validation_step(self, batch, batch_idx): + audio, audio_len, tokens, token_len = batch + spec_pred_dec, spec_pred_postnet, gate_pred, spec_target, spec_target_len, alignments = self.forward( + audio=audio, audio_len=audio_len, tokens=tokens, token_len=token_len + ) + + loss, gate_target = self.loss( + spec_pred_dec=spec_pred_dec, + spec_pred_postnet=spec_pred_postnet, + gate_pred=gate_pred, + spec_target=spec_target, + spec_target_len=spec_target_len, + pad_value=self.pad_value, + ) + return { + "val_loss": loss, + "mel_target": spec_target, + "mel_postnet": spec_pred_postnet, + "gate": gate_pred, + "gate_target": gate_target, + "alignments": alignments, + } + + def validation_epoch_end(self, outputs): + if self.logger is not None and self.logger.experiment is not None: + tb_logger = self.logger.experiment + if isinstance(self.logger, LoggerCollection): + for logger in self.logger: + if isinstance(logger, TensorBoardLogger): + tb_logger = logger.experiment + break + tacotron2_log_to_tb_func( + tb_logger, outputs[0].values(), self.global_step, tag="val", log_images=True, add_audio=False, + ) + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() # This reduces across batches, not workers! + self.log('val_loss', avg_loss) + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + if "dataset" not in cfg or not isinstance(cfg.dataset, DictConfig): + raise ValueError(f"No dataset for {name}") + if "dataloader_params" not in cfg or not isinstance(cfg.dataloader_params, DictConfig): + raise ValueError(f"No dataloder_params for {name}") + if shuffle_should_be: + if 'shuffle' not in cfg.dataloader_params: + logging.warning( + f"Shuffle should be set to True for {self}'s {name} dataloader but was not found in its " + "config. Manually setting to True" + ) + with open_dict(cfg.dataloader_params): + cfg.dataloader_params.shuffle = True + elif not cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to False!!!") + elif not shuffle_should_be and cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to True!!!") + + labels = self._cfg.labels + + dataset = instantiate( + cfg.dataset, labels=labels, bos_id=len(labels), eos_id=len(labels) + 1, pad_id=len(labels) + 2 + ) + return torch.utils.data.DataLoader(dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params) + + def setup_training_data(self, cfg): + self._train_dl = self.__setup_dataloader_from_config(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self.__setup_dataloader_from_config(cfg, shuffle_should_be=False, name="validation") + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_en_tacotron2", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_tacotron2/versions/1.0.0/files/tts_en_tacotron2.nemo", + description="This model is trained on LJSpeech sampled at 22050Hz, and can be used to generate female English voices with an American accent.", + class_=cls, + aliases=["Tacotron2-22050Hz"], + ) + list_of_models.append(model) + return list_of_models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/talknet.py b/models/tacotron2/NeMo/nemo/collections/tts/models/talknet.py new file mode 100644 index 0000000000..57f628d007 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/talknet.py @@ -0,0 +1,413 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import OrderedDict +from typing import List + +import torch +from hydra.utils import instantiate +from omegaconf import DictConfig +from pytorch_lightning import Trainer +from torch import nn +from torch.nn import functional as F + +from nemo.collections.asr.data.audio_to_text import AudioToCharWithDursF0Dataset +from nemo.collections.tts.helpers.helpers import get_mask_from_lengths +from nemo.collections.tts.models.base import SpectrogramGenerator +from nemo.collections.tts.modules.talknet import GaussianEmbedding, MaskedInstanceNorm1d, StyleResidual +from nemo.core import Exportable +from nemo.core.classes import ModelPT, PretrainedModelInfo, typecheck +from nemo.core.neural_types import MelSpectrogramType, NeuralType + + +class TalkNetDursModel(ModelPT): + """TalkNet's durations prediction pipeline.""" + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + super().__init__(cfg=cfg, trainer=trainer) + typecheck.set_typecheck_enabled(enabled=False) + + cfg = self._cfg + self.vocab = AudioToCharWithDursF0Dataset.make_vocab(**cfg.train_ds.dataset.vocab) + self.embed = nn.Embedding(len(self.vocab.labels), cfg.d_char) + self.model = instantiate(cfg.model) + d_out = cfg.model.jasper[-1].filters + self.proj = nn.Conv1d(d_out, 1, kernel_size=1) + + def forward(self, text, text_len): + x, x_len = self.embed(text).transpose(1, 2), text_len + y, _ = self.model(x, x_len) + durs = self.proj(y).squeeze(1) + return durs + + @staticmethod + def _metrics(true_durs, true_text_len, pred_durs): + loss = F.mse_loss(pred_durs, (true_durs + 1).float().log(), reduction='none') + mask = get_mask_from_lengths(true_text_len) + loss *= mask.float() + loss = loss.sum() / mask.sum() + + durs_pred = pred_durs.exp() - 1 + durs_pred[durs_pred < 0.0] = 0.0 + durs_pred = durs_pred.round().long() + + acc = ((true_durs == durs_pred) * mask).sum().float() / mask.sum() * 100 + acc_dist_1 = (((true_durs - durs_pred).abs() <= 1) * mask).sum().float() / mask.sum() * 100 + acc_dist_3 = (((true_durs - durs_pred).abs() <= 3) * mask).sum().float() / mask.sum() * 100 + + return loss, acc, acc_dist_1, acc_dist_3 + + def training_step(self, batch, batch_idx): + _, _, text, text_len, durs, *_ = batch + pred_durs = self(text=text, text_len=text_len) + loss, acc, acc_dist_1, acc_dist_3 = self._metrics(true_durs=durs, true_text_len=text_len, pred_durs=pred_durs,) + train_log = { + 'train_loss': loss, + 'train_acc': acc, + 'train_acc_dist_1': acc_dist_1, + 'train_acc_dist_3': acc_dist_3, + } + return {'loss': loss, 'progress_bar': train_log, 'log': train_log} + + def validation_step(self, batch, batch_idx): + _, _, text, text_len, durs, *_ = batch + pred_durs = self(text=text, text_len=text_len) + loss, acc, acc_dist_1, acc_dist_3 = self._metrics(true_durs=durs, true_text_len=text_len, pred_durs=pred_durs,) + val_log = {'val_loss': loss, 'val_acc': acc, 'val_acc_dist_1': acc_dist_1, 'val_acc_dist_3': acc_dist_3} + self.log_dict(val_log, prog_bar=False, on_epoch=True, logger=True, sync_dist=True) + + @staticmethod + def _loader(cfg): + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader( # noqa + dataset=dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params, + ) + + def setup_training_data(self, cfg): + self._train_dl = self._loader(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self._loader(cfg) + + def setup_test_data(self, cfg): + """Omitted.""" + pass + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_en_talknet", + location=( + "https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_talknet/versions/1.0.0rc1/files" + "/talknet_durs.nemo" + ), + description=( + "This model is trained on LJSpeech sampled at 22050Hz, and can be used to generate durations " + "values for English voice with an American accent." + ), + class_=cls, # noqa + aliases=["TalkNet-22050Hz"], + ) + list_of_models.append(model) + return list_of_models + + +class TalkNetPitchModel(ModelPT): + """TalkNet's pitch prediction pipeline.""" + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + super().__init__(cfg=cfg, trainer=trainer) + typecheck.set_typecheck_enabled(enabled=False) + + cfg = self._cfg + self.vocab = AudioToCharWithDursF0Dataset.make_vocab(**cfg.train_ds.dataset.vocab) + self.embed = GaussianEmbedding(self.vocab, cfg.d_char) + self.model = instantiate(cfg.model) + d_out = cfg.model.jasper[-1].filters + self.sil_proj = nn.Conv1d(d_out, 1, kernel_size=1) + self.body_proj = nn.Conv1d(d_out, 1, kernel_size=1) + self.f0_mean, self.f0_std = cfg.f0_mean, cfg.f0_std + + def forward(self, text, text_len, durs): + x, x_len = self.embed(text, durs).transpose(1, 2), durs.sum(-1) + y, _ = self.model(x, x_len) + f0_sil = self.sil_proj(y).squeeze(1) + f0_body = self.body_proj(y).squeeze(1) + return f0_sil, f0_body + + def _metrics(self, true_f0, true_f0_mask, pred_f0_sil, pred_f0_body): + sil_mask = true_f0 < 1e-5 + sil_gt = sil_mask.long() + sil_loss = F.binary_cross_entropy_with_logits(input=pred_f0_sil, target=sil_gt.float(), reduction='none',) + sil_loss *= true_f0_mask.type_as(sil_loss) + sil_loss = sil_loss.sum() / true_f0_mask.sum() + sil_acc = ((torch.sigmoid(pred_f0_sil) > 0.5).long() == sil_gt).float() # noqa + sil_acc *= true_f0_mask.type_as(sil_acc) + sil_acc = sil_acc.sum() / true_f0_mask.sum() + + body_mse = F.mse_loss(pred_f0_body, (true_f0 - self.f0_mean) / self.f0_std, reduction='none') + body_mask = ~sil_mask + body_mse *= body_mask.type_as(body_mse) # noqa + body_mse = body_mse.sum() / body_mask.sum() # noqa + body_mae = ((pred_f0_body * self.f0_std + self.f0_mean) - true_f0).abs() + body_mae *= body_mask.type_as(body_mae) # noqa + body_mae = body_mae.sum() / body_mask.sum() # noqa + + loss = sil_loss + body_mse + + return loss, sil_acc, body_mae + + def training_step(self, batch, batch_idx): + _, audio_len, text, text_len, durs, f0, f0_mask = batch + pred_f0_sil, pred_f0_body = self(text=text, text_len=text_len, durs=durs) + loss, sil_acc, body_mae = self._metrics( + true_f0=f0, true_f0_mask=f0_mask, pred_f0_sil=pred_f0_sil, pred_f0_body=pred_f0_body, + ) + train_log = {'train_loss': loss, 'train_sil_acc': sil_acc, 'train_body_mae': body_mae} + return {'loss': loss, 'progress_bar': train_log, 'log': train_log} + + def validation_step(self, batch, batch_idx): + _, _, text, text_len, durs, f0, f0_mask = batch + pred_f0_sil, pred_f0_body = self(text=text, text_len=text_len, durs=durs) + loss, sil_acc, body_mae = self._metrics( + true_f0=f0, true_f0_mask=f0_mask, pred_f0_sil=pred_f0_sil, pred_f0_body=pred_f0_body, + ) + + val_log = {'val_loss': loss, 'val_sil_acc': sil_acc, 'val_body_mae': body_mae} + self.log_dict(val_log, prog_bar=False, on_epoch=True, logger=True, sync_dist=True) + + @staticmethod + def _loader(cfg): + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader( # noqa + dataset=dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params, + ) + + def setup_training_data(self, cfg): + self._train_dl = self._loader(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self._loader(cfg) + + def setup_test_data(self, cfg): + """Omitted.""" + pass + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_en_talknet", + location=( + "https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_talknet/versions/1.0.0rc1/files" + "/talknet_pitch.nemo" + ), + description=( + "This model is trained on LJSpeech sampled at 22050Hz, and can be used to generate pitch " + "values for English voice with an American accent." + ), + class_=cls, # noqa + aliases=["TalkNet-22050Hz"], + ) + list_of_models.append(model) + return list_of_models + + +class TalkNetSpectModel(SpectrogramGenerator, Exportable): + """TalkNet's mel spectrogram prediction pipeline.""" + + @property + def output_types(self): + return OrderedDict({"mel-spectrogram": NeuralType(('B', 'D', 'T'), MelSpectrogramType())}) + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + super().__init__(cfg=cfg, trainer=trainer) + typecheck.set_typecheck_enabled(enabled=False) + + cfg = self._cfg + self.vocab = AudioToCharWithDursF0Dataset.make_vocab(**cfg.train_ds.dataset.vocab) + self.blanking = cfg.train_ds.dataset.blanking + self.preprocessor = instantiate(cfg.preprocessor) + self.embed = GaussianEmbedding(self.vocab, cfg.d_char) + self.norm_f0 = MaskedInstanceNorm1d(1) + self.res_f0 = StyleResidual(cfg.d_char, 1, kernel_size=3) + self.model = instantiate(cfg.model) + d_out = cfg.model.jasper[-1].filters + self.proj = nn.Conv1d(d_out, cfg.n_mels, kernel_size=1) + + def forward(self, text, text_len, durs, f0): + x, x_len = self.embed(text, durs).transpose(1, 2), durs.sum(-1) + f0, f0_mask = f0.clone(), f0 > 0.0 + f0 = self.norm_f0(f0.unsqueeze(1), f0_mask) + f0[~f0_mask.unsqueeze(1)] = 0.0 + x = self.res_f0(x, f0) + y, _ = self.model(x, x_len) + mel = self.proj(y) + return mel + + @staticmethod + def _metrics(true_mel, true_mel_len, pred_mel): + loss = F.mse_loss(pred_mel, true_mel, reduction='none').mean(dim=-2) + mask = get_mask_from_lengths(true_mel_len) + loss *= mask.float() + loss = loss.sum() / mask.sum() + return loss + + def training_step(self, batch, batch_idx): + audio, audio_len, text, text_len, durs, f0, f0_mask = batch + mel, mel_len = self.preprocessor(audio, audio_len) + pred_mel = self(text=text, text_len=text_len, durs=durs, f0=f0) + loss = self._metrics(true_mel=mel, true_mel_len=mel_len, pred_mel=pred_mel) + train_log = {'train_loss': loss} + return {'loss': loss, 'progress_bar': train_log, 'log': train_log} + + def validation_step(self, batch, batch_idx): + audio, audio_len, text, text_len, durs, f0, f0_mask = batch + mel, mel_len = self.preprocessor(audio, audio_len) + pred_mel = self(text=text, text_len=text_len, durs=durs, f0=f0) + loss = self._metrics(true_mel=mel, true_mel_len=mel_len, pred_mel=pred_mel) + val_log = {'val_loss': loss} + self.log_dict(val_log, prog_bar=False, on_epoch=True, logger=True, sync_dist=True) + + @staticmethod + def _loader(cfg): + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader( # noqa + dataset=dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params, + ) + + def setup_training_data(self, cfg): + self._train_dl = self._loader(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self._loader(cfg) + + def setup_test_data(self, cfg): + """Omitted.""" + pass + + def parse(self, text: str, **kwargs) -> torch.Tensor: + return torch.tensor(self.vocab.encode(text)).long().unsqueeze(0).to(self.device) + + def generate_spectrogram(self, tokens: torch.Tensor, **kwargs) -> torch.Tensor: + assert hasattr(self, '_durs_model') and hasattr(self, '_pitch_model') + + if self.blanking: + tokens = [ + AudioToCharWithDursF0Dataset.interleave( + x=torch.empty(len(t) + 1, dtype=torch.long, device=t.device).fill_(self.vocab.blank), y=t, + ) + for t in tokens + ] + tokens = AudioToCharWithDursF0Dataset.merge(tokens, value=self.vocab.pad, dtype=torch.long) + + text_len = torch.tensor(tokens.shape[-1], dtype=torch.long).unsqueeze(0) + durs = self._durs_model(tokens, text_len) + durs = durs.exp() - 1 + durs[durs < 0.0] = 0.0 + durs = durs.round().long() + + # Pitch + f0_sil, f0_body = self._pitch_model(tokens, text_len, durs) + sil_mask = f0_sil.sigmoid() > 0.5 + f0 = f0_body * self._pitch_model.f0_std + self._pitch_model.f0_mean + f0 = (~sil_mask * f0).float() + + # Spect + mel = self(tokens, text_len, durs, f0) + + return mel + + def force_spectrogram( + self, tokens: torch.Tensor, durs: torch.Tensor, f0: torch.Tensor, **kwargs + ) -> torch.Tensor: + if self.blanking: + tokens = [ + AudioToCharWithDursF0Dataset.interleave( + x=torch.empty(len(t) + 1, dtype=torch.long, device=t.device).fill_( + self.vocab.blank + ), + y=t, + ) + for t in tokens + ] + tokens = AudioToCharWithDursF0Dataset.merge( + tokens, value=self.vocab.pad, dtype=torch.long + ) + + text_len = torch.tensor(tokens.shape[-1], dtype=torch.long).unsqueeze(0) + durs_len = torch.tensor(durs.shape[-1], dtype=torch.long).unsqueeze(0) + assert text_len == durs_len + + # Spect + mel = self(tokens, text_len, durs, f0) + return mel + + def forward_for_export(self, tokens: torch.Tensor, text_len: torch.Tensor): + durs = self._durs_model(tokens, text_len) + durs = durs.exp() - 1 + durs[durs < 0.0] = 0.0 + durs = durs.round().long() + + # Pitch + f0_sil, f0_body = self._pitch_model(tokens, text_len, durs) + sil_mask = f0_sil.sigmoid() > 0.5 + f0 = f0_body * self._pitch_model.f0_std + self._pitch_model.f0_mean + f0 = (~sil_mask * f0).float() + + # Spect + x, x_len = self.embed(tokens, durs).transpose(1, 2), durs.sum(-1) + f0, f0_mask = f0.clone(), f0 > 0.0 + f0 = self.norm_f0(f0.unsqueeze(1), f0_mask) + f0[~f0_mask.unsqueeze(1)] = 0.0 + x = self.res_f0(x, f0) + y, _ = self.model(x, x_len) + mel = self.proj(y) + + return mel + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_en_talknet", + location=( + "https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_talknet/versions/1.0.0rc1/files" + "/talknet_spect.nemo" + ), + description=( + "This model is trained on LJSpeech sampled at 22050Hz, and can be used to generate female " + "English voices with an American accent." + ), + class_=cls, # noqa + aliases=["TalkNet-22050Hz"], + ) + list_of_models.append(model) + return list_of_models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/two_stages.py b/models/tacotron2/NeMo/nemo/collections/tts/models/two_stages.py new file mode 100644 index 0000000000..bab2d2986a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/two_stages.py @@ -0,0 +1,205 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict + +import librosa +import torch +from hydra.utils import instantiate +from omegaconf import MISSING, DictConfig, OmegaConf + +from nemo.collections.tts.helpers.helpers import OperationMode, griffin_lim +from nemo.collections.tts.models.base import LinVocoder, MelToSpec, Vocoder +from nemo.core.classes.common import PretrainedModelInfo +from nemo.core.neural_types.elements import AudioSignal, MelSpectrogramType +from nemo.core.neural_types.neural_type import NeuralType + + +class MelPsuedoInverseModel(MelToSpec): + def __init__(self, cfg: DictConfig): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + + super().__init__(cfg=cfg) + + sampling_rate = self._cfg['sampling_rate'] + n_fft = self._cfg['n_fft'] + mel_fmin = self._cfg['mel_fmin'] + mel_fmax = self._cfg['mel_fmax'] + mel_freq = self._cfg['mel_freq'] + + melinv = librosa.filters.mel(sr=sampling_rate, n_fft=n_fft, fmin=mel_fmin, fmax=mel_fmax, n_mels=mel_freq) + self.mel_pseudo_inverse = torch.tensor(melinv, dtype=torch.float) + + def convert_mel_spectrogram_to_linear(self, mel): + lin_spec = torch.tensordot(mel, self.mel_pseudo_inverse, dims=[[1], [0]]) + lin_spec = lin_spec.permute(0, 2, 1) + return lin_spec + + def setup_training_data(self, cfg): + pass + + def setup_validation_data(self, cfg): + pass + + def cuda(self, *args, **kwargs): + self.mel_pseudo_inverse = self.mel_pseudo_inverse.cuda(*args, **kwargs) + return self + + +class GriffinLimModel(LinVocoder): + def __init__(self, cfg: DictConfig): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + + super().__init__(cfg=cfg) + + self.n_iters = self._cfg['n_iters'] + self.n_fft = self._cfg['n_fft'] + self.l_hop = self._cfg['l_hop'] + + def convert_linear_spectrogram_to_audio(self, spec, Ts=None): + batch_size = spec.shape[0] + + T_max = spec.shape[2] + if Ts is None: + Ts = [T_max] * batch_size + + max_size = (max(Ts) - 1) * self.l_hop + audios = torch.zeros(batch_size, max_size) + # Lazy GL implementation. Could be improved by moving to pytorch. + for i in range(batch_size): + audio = griffin_lim(spec[i, :, 0 : Ts[i]].cpu().numpy(), n_iters=self.n_iters, n_fft=self.n_fft) + my_len = audio.shape[0] + audios[i, 0:my_len] = torch.from_numpy(audio) + + return audios + + def setup_training_data(self, cfg): + pass + + def setup_validation_data(self, cfg): + pass + + def cuda(self, *args, **kwargs): + return self + + +@dataclass +class TwoStagesConfig: + mel2spec: Dict[Any, Any] = MISSING + linvocoder: Dict[Any, Any] = MISSING + + +class TwoStagesModel(Vocoder): + """Two Stages model used to convert mel spectrograms, to linear spectrograms, and then to audio""" + + def __init__(self, cfg: DictConfig): + + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + + super().__init__(cfg=cfg) + + schema = OmegaConf.structured(TwoStagesConfig) + # ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + # Ensure passed cfg is compliant with schema + OmegaConf.merge(cfg, schema) + + if '_target_' in self._cfg.mel2spec: + self.mel2spec = instantiate(self._cfg.mel2spec) + else: + self.mel2spec = None + + if '_target_' in self._cfg.linvocoder: + self.linvocoder = instantiate(self._cfg.linvocoder) + else: + self.linvocoder = None + + def set_mel_to_spec_model(self, mel2spec: MelToSpec): + self.mel2spec = mel2spec + + def set_linear_vocoder(self, linvocoder: LinVocoder): + self.linvocoder = linvocoder + + def cuda(self, *args, **kwargs): + self.mel2spec.cuda(*args, **kwargs) + self.linvocoder.cuda(*args, **kwargs) + return super().cuda(*args, **kwargs) + + @property + def input_types(self): + return { + "mel": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + } + + @property + def output_types(self): + return { + "wave": NeuralType(('B', 'T'), AudioSignal()), + } + + def forward(self, *, mel): + pass + + def convert_spectrogram_to_audio(self, spec: torch.Tensor, **kwargs) -> torch.Tensor: + self.eval() + try: + self.mel2spec.mode = OperationMode.infer + except AttributeError: + pass + try: + self.linvocoder.mode = OperationMode.infer + except AttributeError: + pass + + with torch.no_grad(): + exp_spec = torch.exp(spec) + linear_spec = self.mel2spec.convert_mel_spectrogram_to_linear(exp_spec) + audio = self.linvocoder.convert_linear_spectrogram_to_audio(linear_spec, **kwargs) + + return audio + + def training_step(self, batch, batch_idx): + pass + + def validation_step(self, batch, batch_idx): + pass + + def validation_epoch_end(self, outputs): + pass + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + pass + + def setup_training_data(self, cfg): + pass + + def setup_validation_data(self, cfg): + pass + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + return list_of_models diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/uniglow.py b/models/tacotron2/NeMo/nemo/collections/tts/models/uniglow.py new file mode 100644 index 0000000000..563170efe1 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/uniglow.py @@ -0,0 +1,264 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import torch +from hydra.utils import instantiate +from omegaconf import MISSING, DictConfig, OmegaConf, open_dict +from pystoi import stoi +from pytorch_lightning.loggers import LoggerCollection, TensorBoardLogger + +from nemo.collections.tts.helpers.helpers import OperationMode, waveglow_log_to_tb_func +from nemo.collections.tts.losses.uniglowloss import UniGlowLoss +from nemo.collections.tts.models.base import GlowVocoder +from nemo.collections.tts.modules.uniglow import UniGlowModule +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import ( + AudioSignal, + LengthsType, + LogDeterminantType, + MelSpectrogramType, + NormalDistributionSamplesType, +) +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +@dataclass +class WaveglowConfig: + waveglow: Dict[Any, Any] = MISSING + preprocessor: Dict[Any, Any] = MISSING + sigma: float = MISSING + train_ds: Optional[Dict[Any, Any]] = None + validation_ds: Optional[Dict[Any, Any]] = None + + +class UniGlowModel(GlowVocoder): + """Waveglow model used to convert betweeen spectrograms and audio""" + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + super().__init__(cfg=cfg, trainer=trainer) + + schema = OmegaConf.structured(WaveglowConfig) + # ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + # Ensure passed cfg is compliant with schema + OmegaConf.merge(cfg, schema) + + self.sigma = self._cfg.sigma + self.audio_to_melspec_precessor = instantiate(self._cfg.preprocessor) + self.model = UniGlowModule( + self._cfg.uniglow.n_mel_channels, + self._cfg.uniglow.n_flows, + self._cfg.uniglow.n_group, + self._cfg.uniglow.n_wn_channels, + self._cfg.uniglow.n_wn_layers, + self._cfg.uniglow.wn_kernel_size, + self.get_upsample_factor(), + ) + self.mode = OperationMode.infer + self.loss = UniGlowLoss(self._cfg.uniglow.stft_loss_coef) + self.removed_weightnorm = False + + @property + def mode(self): + return self._mode + + @mode.setter + def mode(self, new_mode): + if new_mode == OperationMode.training: + self.train() + else: + self.eval() + self._mode = new_mode + self.model.mode = new_mode + + @property + def input_types(self): + return { + "audio": NeuralType(('B', 'T'), AudioSignal()), + "audio_len": NeuralType(('B'), LengthsType()), + } + + @property + def output_types(self): + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + output_dict = { + "pred_normal_dist": NeuralType(('B', 'flowgroup', 'T'), NormalDistributionSamplesType()), + "logdet": NeuralType(elements_type=LogDeterminantType()), + "predicted_audio": NeuralType(('B', 'T'), AudioSignal()), + } + if self.mode == OperationMode.validation: + output_dict["spec"] = NeuralType(('B', 'T', 'D'), MelSpectrogramType()) + output_dict["spec_len"] = NeuralType(('B'), LengthsType()) + return output_dict + return { + "audio_pred": NeuralType(('B', 'T'), AudioSignal()), + } + + @typecheck() + def forward(self, *, audio, audio_len): + if self.mode != self.model.mode: + raise ValueError(f"UniGlowModel's mode {self.mode} does not match UniGlowModule's mode {self.model.mode}") + spec, spec_len = self.audio_to_melspec_precessor(audio, audio_len) + tensors = self.model(spec=spec, audio=audio, sigma=self.sigma) + if self.mode == OperationMode.training: + return tensors # z, logdet, audio_pred + elif self.mode == OperationMode.validation: + z, logdet, audio_pred = tensors + return z, logdet, audio_pred, spec, spec_len + return tensors + + @typecheck( + input_types={ + "spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "sigma": NeuralType(optional=True), + "denoise": NeuralType(optional=True), + "denoiser_strength": NeuralType(optional=True), + }, + output_types={"audio": NeuralType(('B', 'T'), AudioSignal())}, + ) + def convert_spectrogram_to_audio( + self, spec: torch.Tensor, sigma: bool = 1.0, denoise: bool = True, denoiser_strength: float = 0.01 + ) -> torch.Tensor: + if not self.removed_weightnorm: + self.model.remove_weightnorm() + self.removed_weightnorm = True + self.mode = OperationMode.infer + + with torch.no_grad(): + audio = self.model(spec=spec, audio=None, sigma=sigma) + if denoise: + audio = self.denoise(audio=audio, strength=denoiser_strength) + + return audio + + def training_step(self, batch, batch_idx): + self.mode = OperationMode.training + audio, audio_len = batch + z, logdet, predicted_audio = self(audio=audio, audio_len=audio_len) + loss = self.loss(z=z, logdet=logdet, gt_audio=audio, predicted_audio=predicted_audio, sigma=self.sigma) + output = { + 'loss': loss, + 'progress_bar': {'training_loss': loss}, + 'log': {'loss': loss}, + } + return output + + def validation_step(self, batch, batch_idx): + self.mode = OperationMode.validation + audio, audio_len = batch + z, logdet, predicted_audio, spec, spec_len = self(audio=audio, audio_len=audio_len) + loss = self.loss(z=z, logdet=logdet, gt_audio=audio, predicted_audio=predicted_audio, sigma=self.sigma) + + # compute average stoi score for batch + stoi_score = 0 + sr = self._cfg.preprocessor.sample_rate + for audio_i, audio_recon_i in zip(audio.cpu(), predicted_audio.cpu()): + stoi_score += stoi(audio_i, audio_recon_i, sr) + stoi_score /= audio.shape[0] + + return { + "val_loss": loss, + "predicted_audio": predicted_audio, + "mel_target": spec, + "mel_len": spec_len, + "stoi": stoi_score, + } + + def validation_epoch_end(self, outputs): + if self.logger is not None and self.logger.experiment is not None: + tb_logger = self.logger.experiment + if isinstance(self.logger, LoggerCollection): + for logger in self.logger: + if isinstance(logger, TensorBoardLogger): + tb_logger = logger.experiment + break + waveglow_log_to_tb_func( + tb_logger, + tuple(outputs[0].values())[:-1], + self.global_step, + tag="eval", + mel_fb=self.audio_to_melspec_precessor.fb, + ) + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() + avg_stoi = torch.FloatTensor([x['stoi'] for x in outputs]).mean() + tensorboard_logs = {'val_loss': avg_loss, 'stoi': avg_stoi} + logging.info(f"Validation summary | Epoch {self.current_epoch} | NLL {avg_loss:.2f} | STOI: {avg_stoi:.2f}") + return {'val_loss': avg_loss, 'log': tensorboard_logs} + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + if "dataset" not in cfg or not isinstance(cfg.dataset, DictConfig): + raise ValueError(f"No dataset for {name}") + if "dataloader_params" not in cfg or not isinstance(cfg.dataloader_params, DictConfig): + raise ValueError(f"No dataloder_params for {name}") + if shuffle_should_be: + if 'shuffle' not in cfg.dataloader_params: + logging.warning( + f"Shuffle should be set to True for {self}'s {name} dataloader but was not found in its " + "config. Manually setting to True" + ) + with open_dict(cfg["dataloader_params"]): + cfg.dataloader_params.shuffle = True + elif not cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to False!!!") + elif not shuffle_should_be and cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to True!!!") + + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader(dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params) + + def setup_training_data(self, cfg): + self._train_dl = self.__setup_dataloader_from_config(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self.__setup_dataloader_from_config(cfg, shuffle_should_be=False, name="validation") + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_uniglow", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_uniglow/versions/1.0.0rc1/files/tts_uniglow.nemo", + description="This model is trained on LJSpeech sampled at 22050Hz, and has been tested on generating female English voices with an American accent.", + class_=cls, + ) + list_of_models.append(model) + return list_of_models + + def get_upsample_factor(self) -> int: + """ + As the MelSpectrogram upsampling is done using interpolation, the upsampling factor is determined + by the ratio of the MelSpectrogram length and the waveform length + Returns: + An integer representing the upsampling factor + """ + audio = torch.ones(1, self._cfg.train_ds.dataset.n_segments) + spec, spec_len = self.audio_to_melspec_precessor(audio, torch.FloatTensor([len(audio)])) + spec = spec[:, :, :-1] + audio = audio.unfold(1, self._cfg.uniglow.n_group, self._cfg.uniglow.n_group).permute(0, 2, 1) + upsample_factor = audio.shape[2] // spec.shape[2] + return upsample_factor diff --git a/models/tacotron2/NeMo/nemo/collections/tts/models/waveglow.py b/models/tacotron2/NeMo/nemo/collections/tts/models/waveglow.py new file mode 100644 index 0000000000..8f20d8a519 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/models/waveglow.py @@ -0,0 +1,260 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import torch +from hydra.utils import instantiate +from omegaconf import MISSING, DictConfig, OmegaConf, open_dict +from pytorch_lightning.loggers import LoggerCollection, TensorBoardLogger + +from nemo.collections.tts.helpers.helpers import OperationMode, waveglow_log_to_tb_func +from nemo.collections.tts.losses.waveglowloss import WaveGlowLoss +from nemo.collections.tts.models.base import GlowVocoder +from nemo.core.classes import Exportable +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.neural_types.elements import ( + AudioSignal, + LengthsType, + LogDeterminantType, + MelSpectrogramType, + NormalDistributionSamplesType, + VoidType, +) +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +@dataclass +class WaveglowConfig: + waveglow: Dict[Any, Any] = MISSING + preprocessor: Dict[Any, Any] = MISSING + sigma: float = MISSING + train_ds: Optional[Dict[Any, Any]] = None + validation_ds: Optional[Dict[Any, Any]] = None + + +class WaveGlowModel(GlowVocoder, Exportable): + """Waveglow model used to convert betweeen spectrograms and audio""" + + def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None): + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + super().__init__(cfg=cfg, trainer=trainer) + + schema = OmegaConf.structured(WaveglowConfig) + # ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + # Ensure passed cfg is compliant with schema + OmegaConf.merge(cfg, schema) + + self.sigma = self._cfg.sigma + self.audio_to_melspec_precessor = instantiate(self._cfg.preprocessor) + self.waveglow = instantiate(self._cfg.waveglow) + self.loss = WaveGlowLoss() + + @GlowVocoder.mode.setter + def mode(self, new_mode): + if new_mode == OperationMode.training: + self.train() + else: + self.eval() + self._mode = new_mode + self.waveglow.mode = new_mode + + @property + def input_types(self): + return { + "audio": NeuralType(('B', 'T'), AudioSignal()), + "audio_len": NeuralType(('B'), LengthsType()), + "run_inverse": NeuralType(optional=True), + } + + @property + def output_types(self): + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + output_dict = { + "pred_normal_dist": NeuralType(('B', 'flowgroup', 'T'), NormalDistributionSamplesType()), + "log_s_list": [NeuralType(('B', 'flowgroup', 'T'), VoidType())], # TODO: Figure out a good typing + "log_det_W_list": [NeuralType(elements_type=LogDeterminantType())], + } + if self.mode == OperationMode.validation: + output_dict["audio_pred"] = NeuralType(('B', 'T'), AudioSignal()) + output_dict["spec"] = NeuralType(('B', 'T', 'D'), MelSpectrogramType()) + output_dict["spec_len"] = NeuralType(('B'), LengthsType()) + return output_dict + return { + "audio_pred": NeuralType(('B', 'T'), AudioSignal()), + } + + @typecheck() + def forward(self, *, audio, audio_len, run_inverse=True): + if self.mode != self.waveglow.mode: + raise ValueError( + f"WaveGlowModel's mode {self.mode} does not match WaveGlowModule's mode {self.waveglow.mode}" + ) + spec, spec_len = self.audio_to_melspec_precessor(audio, audio_len) + tensors = self.waveglow(spec=spec, audio=audio, run_inverse=run_inverse, sigma=self.sigma) + if self.mode == OperationMode.training: + return tensors[:-1] # z, log_s_list, log_det_W_list + elif self.mode == OperationMode.validation: + z, log_s_list, log_det_W_list, audio_pred = tensors + return z, log_s_list, log_det_W_list, audio_pred, spec, spec_len + return tensors # audio_pred + + @typecheck( + input_types={ + "spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "sigma": NeuralType(optional=True), + "denoise": NeuralType(optional=True), + "denoiser_strength": NeuralType(optional=True), + }, + output_types={"audio": NeuralType(('B', 'T'), AudioSignal())}, + ) + def convert_spectrogram_to_audio( + self, spec: torch.Tensor, sigma: float = 1.0, denoise: bool = True, denoiser_strength: float = 0.01 + ) -> torch.Tensor: + with self.nemo_infer(): + self.waveglow.remove_weightnorm() + audio = self.waveglow( + spec=spec.to(self.waveglow.upsample.weight.dtype), run_inverse=True, audio=None, sigma=sigma + ) + if denoise: + audio = self.denoise(audio=audio, strength=denoiser_strength) + + return audio + + def training_step(self, batch, batch_idx): + self.mode = OperationMode.training + audio, audio_len = batch + z, log_s_list, log_det_W_list = self(audio=audio, audio_len=audio_len, run_inverse=False) + + loss = self.loss(z=z, log_s_list=log_s_list, log_det_W_list=log_det_W_list, sigma=self.sigma) + output = { + 'loss': loss, + 'progress_bar': {'training_loss': loss}, + 'log': {'loss': loss}, + } + return output + + def validation_step(self, batch, batch_idx): + self.mode = OperationMode.validation + audio, audio_len = batch + z, log_s_list, log_det_W_list, audio_pred, spec, spec_len = self( + audio=audio, audio_len=audio_len, run_inverse=(batch_idx == 0) + ) + loss = self.loss(z=z, log_s_list=log_s_list, log_det_W_list=log_det_W_list, sigma=self.sigma) + return { + "val_loss": loss, + "audio_pred": audio_pred, + "mel_target": spec, + "mel_len": spec_len, + } + + def validation_epoch_end(self, outputs): + if self.logger is not None and self.logger.experiment is not None: + tb_logger = self.logger.experiment + if isinstance(self.logger, LoggerCollection): + for logger in self.logger: + if isinstance(logger, TensorBoardLogger): + tb_logger = logger.experiment + break + waveglow_log_to_tb_func( + tb_logger, + outputs[0].values(), + self.global_step, + tag="eval", + mel_fb=self.audio_to_melspec_precessor.fb, + ) + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() + self.log('val_loss', avg_loss) + + def __setup_dataloader_from_config(self, cfg, shuffle_should_be: bool = True, name: str = "train"): + if "dataset" not in cfg or not isinstance(cfg.dataset, DictConfig): + raise ValueError(f"No dataset for {name}") + if "dataloader_params" not in cfg or not isinstance(cfg.dataloader_params, DictConfig): + raise ValueError(f"No dataloder_params for {name}") + if shuffle_should_be: + if 'shuffle' not in cfg.dataloader_params: + logging.warning( + f"Shuffle should be set to True for {self}'s {name} dataloader but was not found in its " + "config. Manually setting to True" + ) + with open_dict(cfg["dataloader_params"]): + cfg.dataloader_params.shuffle = True + elif not cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to False!!!") + elif not shuffle_should_be and cfg.dataloader_params.shuffle: + logging.error(f"The {name} dataloader for {self} has shuffle set to True!!!") + + dataset = instantiate(cfg.dataset) + return torch.utils.data.DataLoader(dataset, collate_fn=dataset.collate_fn, **cfg.dataloader_params) + + def setup_training_data(self, cfg): + self._train_dl = self.__setup_dataloader_from_config(cfg) + + def setup_validation_data(self, cfg): + self._validation_dl = self.__setup_dataloader_from_config(cfg, shuffle_should_be=False, name="validation") + + @classmethod + def list_available_models(cls) -> 'List[PretrainedModelInfo]': + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + Returns: + List of available pre-trained models. + """ + list_of_models = [] + model = PretrainedModelInfo( + pretrained_model_name="tts_waveglow_268m", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_waveglow_268m/versions/1.0.0rc1/files/tts_waveglow_268m.nemo", + description="This model is trained on LJSpeech sampled at 22050Hz, and has been tested on generating female English voices with an American accent and Mandarin voices.", + class_=cls, + ) + list_of_models.append(model) + model = PretrainedModelInfo( + pretrained_model_name="tts_waveglow_88m", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_waveglow_88m/versions/1.0.0/files/tts_waveglow.nemo", + description="This model is trained on LJSpeech sampled at 22050Hz, and has been tested on generating female English voices with an American accent and Mandarin voices.", + class_=cls, + aliases=["WaveGlow-22050Hz", "tts_waveglow"], + ) + list_of_models.append(model) + return list_of_models + + @property + def input_module(self): + return self.waveglow + + @property + def output_module(self): + return self.waveglow + + def _prepare_for_export(self, **kwargs): + self.update_bias_spect() + self.waveglow._prepare_for_export(**kwargs) + + def forward_for_export(self, spec, z=None): + return self.waveglow(spec, z) + + def load_state_dict(self, state_dict, strict=True): + # Remove convinv.inv_conv weights since they are not initialized until forward is called during training + # and can be computed from convinv.conv.weight + # Ideally, we should remove this during saving instead of ignoring during loading + for i in range(self._cfg.waveglow.n_flows): + del state_dict[f"waveglow.convinv.{i}.inv_conv.weight"] + super().load_state_dict(state_dict, strict=strict) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/__init__.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/__init__.py new file mode 100644 index 0000000000..63477dd964 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.tts.modules.degli import DegliModule +from nemo.collections.tts.modules.ed_mel2spec import EDMel2SpecModule +from nemo.collections.tts.modules.glow_tts import GlowTTSModule +from nemo.collections.tts.modules.melgan_modules import ( + MelGANDiscriminator, + MelGANGenerator, + MelGANMultiScaleDiscriminator, +) +from nemo.collections.tts.modules.squeezewave import SqueezeWaveModule +from nemo.collections.tts.modules.tacotron2 import Decoder as Taco2Decoder +from nemo.collections.tts.modules.tacotron2 import Encoder as Taco2Encoder +from nemo.collections.tts.modules.tacotron2 import Postnet as Taco2Postnet +from nemo.collections.tts.modules.waveglow import WaveGlowModule diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/aligner.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/aligner.py new file mode 100644 index 0000000000..9daa03d743 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/aligner.py @@ -0,0 +1,106 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torch import nn + + +class ConvNorm(torch.nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=None, + dilation=1, + bias=True, + w_init_gain='linear', + ): + super().__init__() + if padding is None: + assert kernel_size % 2 == 1 + padding = int(dilation * (kernel_size - 1) / 2) + + self.conv = torch.nn.Conv1d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + ) + + torch.nn.init.xavier_uniform_(self.conv.weight, gain=torch.nn.init.calculate_gain(w_init_gain)) + + def forward(self, signal): + conv_signal = self.conv(signal) + return conv_signal + + +class AlignmentEncoder(torch.nn.Module): + """Module for alignment text and mel spectrogram. """ + + def __init__( + self, n_mel_channels=80, n_text_channels=512, n_att_channels=80, temperature=0.0005, + ): + super().__init__() + self.temperature = temperature + self.softmax = torch.nn.Softmax(dim=3) + self.log_softmax = torch.nn.LogSoftmax(dim=3) + + self.key_proj = nn.Sequential( + ConvNorm(n_text_channels, n_text_channels * 2, kernel_size=3, bias=True, w_init_gain='relu'), + torch.nn.ReLU(), + ConvNorm(n_text_channels * 2, n_att_channels, kernel_size=1, bias=True), + ) + + self.query_proj = nn.Sequential( + ConvNorm(n_mel_channels, n_mel_channels * 2, kernel_size=3, bias=True, w_init_gain='relu'), + torch.nn.ReLU(), + ConvNorm(n_mel_channels * 2, n_mel_channels, kernel_size=1, bias=True), + torch.nn.ReLU(), + ConvNorm(n_mel_channels, n_att_channels, kernel_size=1, bias=True), + ) + + def forward(self, queries, keys, mask=None, attn_prior=None): + """Forward pass of the aligner encoder. + + Args: + queries (torch.tensor): B x C x T1 tensor (probably going to be mel data). + keys (torch.tensor): B x C2 x T2 tensor (text data). + mask (torch.tensor): uint8 binary mask for variable length entries (should be in the T2 domain). + attn_prior (torch.tensor): prior for attention matrix. + Output: + attn (torch.tensor): B x 1 x T1 x T2 attention mask. Final dim T2 should sum to 1. + attn_logprob (torch.tensor): B x 1 x T1 x T2 log-prob attention mask. + """ + keys_enc = self.key_proj(keys) # B x n_attn_dims x T2 + queries_enc = self.query_proj(queries) + + # Simplistic Gaussian Isotopic Attention + attn = (queries_enc[:, :, :, None] - keys_enc[:, :, None]) ** 2 # B x n_attn_dims x T1 x T2 + attn = -self.temperature * attn.sum(1, keepdim=True) + + if attn_prior is not None: + attn = self.log_softmax(attn) + torch.log(attn_prior[:, None] + 1e-8) + + attn_logprob = attn.clone() + + if mask is not None: + attn.data.masked_fill_(mask.permute(0, 2, 1).unsqueeze(2), -float("inf")) + + attn = self.softmax(attn) # softmax along T2 + return attn, attn_logprob diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/degli.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/degli.py new file mode 100755 index 0000000000..2fa365afbb --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/degli.py @@ -0,0 +1,617 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MIT License + +# Copyright (c) 2019 Jeongmin Liu + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import math +from collections import OrderedDict + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nemo.collections.common.parts.patch_utils import stft_patch +from nemo.collections.tts.helpers.helpers import OperationMode +from nemo.core.classes import Exportable, NeuralModule, typecheck +from nemo.core.neural_types.elements import IntType, LengthsType, SpectrogramType +from nemo.core.neural_types.neural_type import NeuralType + + +def str2act(txt): + """Translates text to neural network activation""" + return { + "sigmoid": nn.Sigmoid(), + "relu": nn.ReLU(), + "none": nn.Sequential(), + "lrelu": nn.LeakyReLU(0.1), + "selu": nn.SELU(), + }[txt.lower()] + + +def replace_magnitude(x, mag): + """ + Extract the phase from x and apply it on mag + x [B,2,F,T] : A tensor, where [:,0,:,:] is real and [:,1,:,:] is imaginary + mag [B,1,F,T] : A tensor containing the absolute magnitude. + """ + + phase = torch.atan2(x[:, 1:], x[:, :1]) # imag, real + return torch.cat([mag * torch.cos(phase), mag * torch.sin(phase)], dim=1) + + +def overlap_add(x, hop_length, eye=None): + """ + component which is used for STFT + x: B, W, T + eye: identity matrix of size (W, W) + return: B, W + hop_length * (T - 1) + """ + n_batch, W, _ = x.shape + if eye is None: + eye = torch.eye(W, device=x.device) + + x = F.conv_transpose1d(x, eye, stride=hop_length, padding=0) + x = x.view(n_batch, -1) + return x + + +class InverseSTFT(nn.Module): + """ + An STFT implementation for pytorch. + """ + + def __init__(self, n_fft, hop_length=None, win_length=None, window=None): + super().__init__() + + self.n_fft = n_fft + + # Set the default hop if it's not already specified + if hop_length is None: + self.hop_length = int(win_length // 4) + else: + self.hop_length = hop_length + + if win_length is None: + win_length = n_fft + + # kernel for overlap_add + eye = torch.eye(n_fft) + self.eye = nn.Parameter(eye.unsqueeze(1), requires_grad=False) # n_fft, 1, n_fft + + # default window is a rectangular window (convention of torch.stft) + if window is None: + window = torch.ones(win_length) + else: + assert win_length == len(window) + + # pad window so that its length is n_fft + diff = n_fft - win_length + window = F.pad(window.unsqueeze(0), [diff // 2, math.ceil(diff / 2)]) + window.unsqueeze_(2) # 1, n_fft, 1 + + # square of window for calculating the numerical error occured during stft & istft + self.win_sq = nn.Parameter(window ** 2, requires_grad=False) # 1, n_fft, 1 + self.win_sq_sum = None + + # ifft basis * window + # The reason why this basis is used instead of torch.ifft is + # because torch.ifft / torch.irfft randomly cause segfault + # when the model is in nn.DataParallel + # of PyTorch 1.2.0 (py3.7_cuda10.0.130_cudnn7.6.2_01.2) + eye_realimag = torch.stack((eye, torch.zeros(n_fft, n_fft)), dim=-1) + basis = torch.fft.ifft(eye_realimag, signal_ndim=1) # n_fft, n_fft, 2 + basis[..., 1] *= -1 # because (a+b*1j)*(c+d*1j) == a*c - b*d + basis *= window + self.basis = nn.Parameter(basis, requires_grad=False) # n_fft, n_fft, 2 + + def forward(self, stft_matrix, center=True, normalized=False, onesided=True, length=None): + """stft_matrix: (n_batch (B), n_freq, n_frames (T), 2)) + if `onesided == True`, `n_freq == n_fft` should be satisfied. + else, `n_freq == n_fft // 2+ 1` should be satisfied. + + """ + n_batch, n_freq, n_frames, _ = stft_matrix.shape + + assert (not onesided) and (n_freq == self.n_fft) or onesided and (n_freq == self.n_fft // 2 + 1) + + if length: + padded_length = length + if center: + padded_length += self.n_fft + n_frames = min(n_frames, math.ceil(padded_length / self.hop_length)) + + stft_matrix = stft_matrix[:, :, :n_frames] + + if onesided: + flipped = stft_matrix[:, 1:-1].flip(1) + flipped[..., 1] *= -1 + stft_matrix = torch.cat((stft_matrix, flipped), dim=1) + # now stft_matrix is (B, n_fft, T, 2) + + # The reason why this basis is used instead of torch.ifft is + # because torch.ifft / torch.irfft randomly cause segfault + # when the model is in nn.DataParallel + # of PyTorch 1.2.0 (py3.7_cuda10.0.130_cudnn7.6.2_01.2) + ytmp = torch.einsum('bftc,fwc->bwt', stft_matrix, self.basis) + y = overlap_add(ytmp, self.hop_length, self.eye) + # now y is (B, n_fft + hop_length * (n_frames - 1)) + + # compensate numerical errors of window function + if self.win_sq_sum is None or self.win_sq_sum.shape[1] != y.shape[1]: + win_sq = self.win_sq.expand(1, -1, n_frames) # 1, n_fft, n_frames + win_sq_sum = overlap_add(win_sq, self.hop_length, self.eye) + win_sq_sum[win_sq_sum <= torch.finfo(torch.float32).tiny] = 1.0 + # now win_sq_sum is (1, y.shape[1]) + self.win_sq_sum = win_sq_sum + + y /= self.win_sq_sum + + if center: + y = y[:, self.n_fft // 2 :] + if length is not None: + if length < y.shape[1]: + y = y[:, :length] + else: + y = F.pad(y, [0, length - y.shape[1]]) + # now y is (B, length) + + if normalized: + y *= self.n_fft ** 0.5 + + return y + + +class ConvGLU(nn.Module): + """ + A convGlu operation, used by the Degli paper's model. + """ + + def __init__(self, in_ch, out_ch, kernel_size=(7, 7), padding=None, batchnorm=False, act="sigmoid", stride=None): + super().__init__() + if not padding: + padding = (kernel_size[0] // 2, kernel_size[1] // 2) + if stride is None: + self.conv = nn.Conv2d(in_ch, out_ch * 2, kernel_size, padding=padding) + else: + self.conv = nn.Conv2d(in_ch, out_ch * 2, kernel_size, padding=padding, stride=stride) + self.weight = self.conv.weight + self.bias = self.conv.bias + + if batchnorm: + self.conv = nn.Sequential(self.conv, nn.BatchNorm2d(out_ch * 2)) + self.sigmoid = str2act(act) + + def forward(self, x): + x = self.conv(x) + ch = x.shape[1] + x = x[:, : ch // 2, ...] * self.sigmoid(x[:, ch // 2 :, ...]) + return x + + +class DegliDNN(nn.Module): + """ + The tiny model, which was used by the paper. + Very efficient in terms of memory (~400KB), + but utilizes 11x11 /7x7 convolutions which results slow execution. + Inferece costs about the same at 5-32 ED model, with lower quality. + """ + + def __init__(self): + super().__init__() + + k_x1, k_y1, k_x2, k_y2 = self.parse() + ch_hidden = self.ch_hidden + self.convglu_first = ConvGLU(6, ch_hidden, kernel_size=(k_y1, k_x1), batchnorm=True, act=self.act) + self.two_convglus = nn.Sequential( + ConvGLU(ch_hidden, ch_hidden, batchnorm=True, act=self.act, kernel_size=(k_y2, k_x2)), + ConvGLU(ch_hidden, ch_hidden, act=self.act, kernel_size=(k_y2, k_x2)), + ) + self.convglu_last = ConvGLU(ch_hidden, ch_hidden, act=self.act) + + self.conv = nn.Conv2d(ch_hidden, 2, kernel_size=(k_y2, k_x2), padding=((k_y2 - 1) // 2, (k_x2 - 1) // 2)) + + def forward(self, x, mag_replaced, consistent, train_step=-1): + x = torch.cat([x, mag_replaced, consistent], dim=1) + x = self.convglu_first(x) + residual = x + x = self.two_convglus(x) + x += residual + x = self.convglu_last(x) + x = self.conv(x) + return x + + def parse( + self, k_x1: int = 11, k_y1: int = 11, k_x2: int = 7, k_y2: int = 7, num_channel: int = 16, act="sigmoid" + ): + self.ch_hidden = num_channel + + self.act = act.lower() + return (k_x1, k_y1, k_x2, k_y2) + + +class DegliED(nn.Module): + + """ + This is the default model. + ED - Encoder-Decoder + + Arguments: + layers: Number of encoder/deooders layers, + k_x: conv kernels size in time domain, + k_y: conv kernels size in frequency domain, + s_x: conv stride in time domain , + s_y: conv stride in frequency domain domain, + widening: a factor for expanding the number of channels during encoding phase, + use_bn: use batch norm after encoders/decoders, + linear_finalizer: apply fully connected after the final decoder. + convGlu: use GLU after conv (recommended with sigmoid), + act: string mapped by str2act() to the post encoder activation, + act2: string mapped by str2act() to the post decoder activation, + glu_bn: use batchnorm with the GLU operation, if enabled, + use_weight_norm: use weigh-normalization, + """ + + def __init__(self, n_freq, config): + super().__init__() + + self.parse(**config) + + layer_specs = [ + 6, + self.widening, + self.widening * 2, + self.widening * 4, + self.widening * 8, + self.widening * 8, + self.widening * 8, + self.widening * 8, + self.widening * 8, + ] + + layer_specs = layer_specs[0 : self.n_layers + 1] + + self.encoders = nn.ModuleList() + + conv, pad = self._gen_conv( + layer_specs[0], + layer_specs[1], + convGlu=self.convGlu, + rounding_needed=True, + use_weight_norm=self.use_weight_norm, + ) + + # Encoder 0 + self.encoders.append(nn.Sequential(pad, conv)) + + last_ch = layer_specs[1] + self.lamb = 0 + # Rest of the encoders + for i, ch_out in enumerate(layer_specs[2:]): + d = OrderedDict() + d['act'] = str2act(self.act) + gain = math.sqrt(2.0 / (1.0 + self.lamb ** 2)) + gain = gain / math.sqrt(2) # for naive signal propagation with residual w/o bn + + conv, pad = self._gen_conv( + last_ch, + ch_out, + gain=gain, + convGlu=self.convGlu, + kernel_size=self.k_xy, + use_weight_norm=self.use_weight_norm, + ) + if not pad is None: + d['pad'] = pad + d['conv'] = conv + + if self.use_batchnorm: + d['bn'] = nn.BatchNorm2d(ch_out) + + encoder_block = nn.Sequential(d) + self.encoders.append(encoder_block) + last_ch = ch_out + + layer_specs.reverse() + self.decoders = nn.ModuleList() + kernel_size = 4 # needed to match size of encoders inputs/outputs. + # Decoders + for i, ch_out in enumerate(layer_specs[1:]): + + d = OrderedDict() + d['act'] = str2act(self.act2) + gain = math.sqrt(2.0 / (1.0 + self.lamb ** 2)) + gain = gain / math.sqrt(2) + + if i == len(layer_specs) - 2: + kernel_size = 5 + ch_out = 2 + conv = self._gen_deconv(last_ch, ch_out, gain=gain, k=kernel_size, use_weight_norm=self.use_weight_norm) + d['conv'] = conv + + if self.use_batchnorm and i < self.n_layers - 1: + d['bn'] = nn.BatchNorm2d(ch_out) + + decoder_block = nn.Sequential(d) + self.decoders.append(decoder_block) + last_ch = ch_out * 2 + + if self.use_linear_finalizer: + init_alpha = 0.001 + self.linear_finalizer = nn.Parameter(torch.ones(n_freq) * init_alpha, requires_grad=True) + + def parse( + self, + layers: int, + k_x: int, + k_y: int, + s_x: int, + s_y: int, + widening: int, + use_bn: bool, + linear_finalizer: bool, + convGlu: bool, + act: str, + act2: str, + glu_bn: bool, + use_weight_norm: bool, + ): + + self.n_layers = layers + self.k_xy = (k_y, k_x) + self.s_xy = (s_y, s_x) + self.widening = widening + self.use_batchnorm = use_bn + self.use_linear_finalizer = linear_finalizer + self.convGlu = convGlu + self.act = act + self.act2 = act2 + self.glu_bn = glu_bn + self.use_weight_norm = use_weight_norm + + def forward(self, x, mag_replaced, consistent, train_step=-1): + x = torch.cat([x, mag_replaced, consistent], dim=1) + + encoders_output = [] + + for i, encoder in enumerate(self.encoders): + x = encoder(x) + encoders_output.append(x) + + for i, decoder in enumerate(self.decoders[:-1]): + x = decoder(x) + x = torch.cat([x, encoders_output[-(i + 2)]], dim=1) + + x = self.decoders[-1](x) + + if self.use_linear_finalizer: + x_perm = x.permute(0, 1, 3, 2) + x = torch.mul(x_perm, self.linear_finalizer) + x = x.permute(0, 1, 3, 2) + + return x + + def _gen_conv( + self, + in_ch, + out_ch, + strides=(2, 1), + kernel_size=(5, 3), + gain=math.sqrt(2), + convGlu=False, + rounding_needed=False, + use_weight_norm=False, + ): + # [batch, in_height, in_width, in_channels] => [batch, out_height, out_width, out_channels] + ky, kx = kernel_size + p1x = (kx - 1) // 2 + p2x = kx - 1 - p1x + p1y = (ky - 1) // 2 + p2y = ky - 1 - p1y + + if rounding_needed: + pad_counts = (p1x, p2x, p1y - 1, p2y) + pad = torch.nn.ReplicationPad2d(pad_counts) + else: + pad = None + + if convGlu: + conv = ConvGLU( + in_ch, + out_ch, + kernel_size=kernel_size, + stride=strides, + batchnorm=self.glu_bn, + padding=(0, 0), + act="sigmoid", + ) + else: + if pad is None: + conv = nn.Conv2d(in_ch, out_ch, kernel_size=kernel_size, stride=strides, padding=(p1y, p1x)) + else: + conv = nn.Conv2d(in_ch, out_ch, kernel_size=kernel_size, stride=strides, padding=0) + + if use_weight_norm: + conv = torch.nn.utils.weight_norm(conv, name='weight') + + w = conv.weight + k = w.size(1) * w.size(2) * w.size(3) + conv.weight.data.normal_(0.0, gain / math.sqrt(k)) + nn.init.constant_(conv.bias, 0.01) + return conv, pad + + def _gen_deconv(self, in_ch, out_ch, strides=(2, 1), k=4, gain=math.sqrt(2), p=1, use_weight_norm=False): + # [batch, in_height, in_width, in_channels] => [batch, out_height, out_width, out_channels] + + conv = nn.ConvTranspose2d( + in_ch, out_ch, kernel_size=(k, 3), stride=strides, padding_mode='zeros', padding=(p, 1), dilation=1 + ) + + if use_weight_norm: + conv = torch.nn.utils.weight_norm(conv, name='weight') + + w = conv.weight + k = w.size(1) * w.size(2) * w.size(3) + conv.weight.data.normal_(0.0, gain / math.sqrt(k)) + nn.init.constant_(conv.bias, 0.01) + + return conv + + +class DegliModule(NeuralModule, Exportable): + def __init__(self, n_fft: int, hop_length: int, depth: int, out_all_block: bool, tiny: bool, **kwargs): + + """ + Degli module + + Args: + n_fft (int): STFT argument. + hop_length (int): STFT argument. + depth (int): depth > 1 will cause the NN to repeat, which new trainable weights. + out_all_block (bool) output all blocks, not just the final output. + tiny (bool) override all model hyperparameters and use the paper's small model. + kwargs: to be passed for the dnn. + """ + super().__init__() + n_freq = n_fft // 2 + 1 + self.out_all_block = out_all_block + self.n_fft = n_fft + self.hop_length = hop_length + self.window = nn.Parameter(torch.hann_window(n_fft), requires_grad=False) + self.istft = InverseSTFT(n_fft, hop_length=hop_length, window=self.window.data) + + if tiny: + self.dnns = nn.ModuleList([DegliDNN() for _ in range(depth)]) + else: + self.dnns = nn.ModuleList([DegliED(n_freq, kwargs) for _ in range(depth)]) + + self.mode = OperationMode.infer + + def stft(self, x): + return stft_patch(x, n_fft=self.n_fft, hop_length=self.hop_length, window=self.window, return_complex=False) + + @typecheck() + def forward(self, x, mag, max_length=None, repeat=2): + + """ + + Args: + x: the tensor containing the phase [B,2,F,T]. Could be random for first iteration + mag: the tensor containing the magnitude [B,1,F,T] + max_length: maximum length of the audio in the batch + repeat: how many iterations to apply using this network + Returns: + training/ validation: + out_repeats: output of all blocks: [B, repeats, 2, F, T] + final_out: output of the final block: [B, 2, F, T] + residual: output of the final dnn: [B, 2, F, T] + inference: the output of the final block + final_out: output of the final block: [B, 2, F, T] + """ + + if self.training and self.mode != OperationMode.training: + raise ValueError(f"{self} has self.training set to True but self.OperationMode was not set to training") + if not self.training and self.mode == OperationMode.training: + raise ValueError(f"{self} has self.training set to False but self.OperationMode was set to training") + + if isinstance(max_length, torch.Tensor): + max_length = max_length.item() + + out_repeats = [] + for _ in range(repeat): + for dnn in self.dnns: + # B, 2, F, T + mag_replaced = replace_magnitude(x, mag) + + # B, F, T, 2 + waves = self.istft(mag_replaced.permute(0, 2, 3, 1), length=max_length) + consistent = self.stft(waves) + + # B, 2, F, T + consistent = consistent.permute(0, 3, 1, 2) + + residual = dnn(x, mag_replaced, consistent) + + x = consistent - residual + + if self.out_all_block: + out_repeats.append(x) + + final_out = replace_magnitude(x, mag) + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + if self.out_all_block: + out_repeats = torch.stack(out_repeats, dim=1) + else: + out_repeats = x.unsqueeze(1) + + return out_repeats, final_out, residual + else: + return final_out + + @property + def input_types(self): + return { + "x": NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + "mag": NeuralType(('B', 'any', 'D', 'T'), SpectrogramType()), + "max_length": NeuralType(None, LengthsType()), + "repeat": NeuralType(None, IntType()), + } + + @property + def output_types(self): + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + return { + "out_repeats": NeuralType(('B', 'any', 'C', 'D', 'T'), SpectrogramType()), + "final_out": NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + "residual": NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + } + else: + return { + "final_out": NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + } + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + par = next(self.parameters()) + n_freq = self.n_fft // 2 + 1 + T = 500 + x = torch.randn((1, 2, n_freq, T), device=par.device, dtype=par.dtype) + mag = torch.randn((1, 1, n_freq, T), device=par.device, dtype=par.dtype) + max_length = torch.tensor((T - 1) * self.hop_length) + repeat = torch.tensor(8) + out = tuple([x, mag, max_length, repeat]) + return out diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/ed_mel2spec.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/ed_mel2spec.py new file mode 100755 index 0000000000..52f19cde49 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/ed_mel2spec.py @@ -0,0 +1,261 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from collections import OrderedDict + +import librosa +import torch +import torch.nn as nn +from numpy.linalg import pinv + +from nemo.core.classes import Exportable, NeuralModule, typecheck +from nemo.core.neural_types.elements import SpectrogramType +from nemo.core.neural_types.neural_type import NeuralType + + +def str2act(txt): + """Translates text to neural network activation""" + return { + "sigmoid": nn.Sigmoid(), + "relu": nn.ReLU(), + "none": nn.Sequential(), + "lrelu": nn.LeakyReLU(0.2), + "selu": nn.SELU(), + }[txt.lower()] + + +def create_mel_filterbank(*args, **kwargs): + return librosa.filters.mel(*args, **kwargs) + + +class EDMel2SpecModule(NeuralModule, Exportable): + def __init__( + self, + n_fft: int, + hop_length: int, + mel_fmin: int, + mel_fmax: int, + mel_freq: int, + layers: int, + sampling_rate: int, + widening: int, + use_batchnorm: bool, + droprate: float, + num_dropout: int, + pre_final_lin: bool, + act1: str, + act2: str, + use_weight_norm: bool, + ): + """ + Encoder Decoder module trained to transfrom Mel encoded spectrograms to normal spectrograms. + The module relies on convolutions (encoders) and transposed convolutions (decoders), + which does not affect the time-dimension length, and thus applicable to any input length. + + Args: + + n_fft: STFT parameter. + hop_length: STFT parameter. + mel_fmin: Mel minimum frequency. + mel_fmax: Mel maximum frequency. + mel_freq: Number of Mel frequencies. + layers: Number of layers in the model. + sampling_rate: audio property. + widening: factor for extending the number of encoders' channels. + use_batchnorm: should use batchnorm after encoders/decoders. + droprate: droprate used on decoders ouytput. + num_dropout: number of decoders to use dropout on. + pre_final_lin: adds a fully connected layer after the second to last decoder. + act1: string describing the activation function after the encoders. + act2: string describing the activation function after the decoders. + use_weight_norm: use weight norm. + + """ + + super().__init__() + + n_freq = n_fft // 2 + 1 + self.hop_length = hop_length + + self.n_layers = layers + self.sampling_rate = sampling_rate + self.widening = widening + self.use_batchnorm = use_batchnorm + self.droprate = droprate + self.num_dropout = num_dropout + self.pre_final_lin = pre_final_lin + self.act1 = act1 + self.act2 = act2 + self.use_weight_norm = use_weight_norm + + meltrans = librosa.filters.mel(sr=sampling_rate, n_fft=n_fft, fmin=mel_fmin, fmax=mel_fmax, n_mels=mel_freq) + + self.meltrans = nn.Parameter( + torch.transpose(torch.tensor(meltrans, dtype=torch.float), 0, 1), requires_grad=False + ) + self.meltrans_inv = nn.Parameter( + torch.transpose(torch.tensor(pinv(meltrans), dtype=torch.float), 0, 1), requires_grad=False + ) + + layer_specs = [ + 1, + self.widening, + self.widening * 2, + self.widening * 4, + self.widening * 8, + self.widening * 8, + self.widening * 8, + self.widening * 8, + self.widening * 8, + self.widening * 8, + ] + + layer_specs = layer_specs[0 : self.n_layers + 1] + self.encoders = nn.ModuleList() + + conv, pad = self._gen_conv(layer_specs[0], layer_specs[1], use_weight_norm=self.use_weight_norm) + self.encoders.append(nn.Sequential(pad, conv)) + + last_ch = layer_specs[1] + + for i, ch_out in enumerate(layer_specs[2:]): + d = OrderedDict() + d['act'] = str2act(self.act1) + gain = math.sqrt(2.0 / (1.0)) + gain = gain / math.sqrt(2) + + conv, pad = self._gen_conv(last_ch, ch_out, gain=gain, use_weight_norm=self.use_weight_norm) + + d['pad'] = pad + d['conv'] = conv + + if self.use_batchnorm: + d['bn'] = nn.BatchNorm2d(ch_out) + + encoder_block = nn.Sequential(d) + self.encoders.append(encoder_block) + last_ch = ch_out + + layer_specs.reverse() + self.decoders = nn.ModuleList() + for i, ch_out in enumerate(layer_specs[1:]): + + d = OrderedDict() + d['act'] = str2act(self.act2) + gain = math.sqrt(2.0 / (1.0)) + gain = gain / math.sqrt(2) + + kernel_size = 4 if i < len(layer_specs) - 2 else 5 + conv = self._gen_deconv(last_ch, ch_out, gain=gain, k=kernel_size, use_weight_norm=self.use_weight_norm) + d['conv'] = conv + + if i < self.num_dropout and self.droprate > 0.0: + d['dropout'] = nn.Dropout(self.droprate) + + if self.use_batchnorm and i < self.n_layers - 1: + d['bn'] = nn.BatchNorm2d(ch_out) + + decoder_block = nn.Sequential(d) + self.decoders.append(decoder_block) + last_ch = ch_out * 2 + + init_alpha = 0.001 + self.linear_finalizer = nn.Parameter(torch.ones(n_freq) * init_alpha, requires_grad=True) + + if self.pre_final_lin: + self.linear_pre_final = nn.Parameter(torch.ones(self.widening * 2, n_freq // 2), requires_grad=True) + + def mel_pseudo_inverse(self, x): + return torch.tensordot(x, self.meltrans_inv, dims=[[2], [0]]).permute(0, 1, 3, 2) + + def spec_to_mel(self, x): + return torch.tensordot(x, self.meltrans, dims=[[2], [0]]).permute(0, 1, 3, 2) + + @typecheck() + def forward(self, mel): + x = self.mel_pseudo_inverse(mel) + x_in = x + + encoders_output = [] + + for i, encoder in enumerate(self.encoders): + x = encoder(x) + encoders_output.append(x) + + for i, decoder in enumerate(self.decoders[:-1]): + x = decoder(x) + x = torch.cat([x, encoders_output[-(i + 2)]], dim=1) + + if self.pre_final_lin: + x_perm = x.permute(0, 3, 1, 2) + x = torch.mul(x_perm, self.linear_pre_final).permute(0, 2, 3, 1) + + x = self.decoders[-1](x) + x_perm = x.permute(0, 1, 3, 2) + x = torch.mul(x_perm, self.linear_finalizer) + x = x.permute(0, 1, 3, 2) + + x = x + x_in + return x + + def _gen_conv( + self, + in_ch, + out_ch, + strides=(2, 1), + kernel_size=(5, 3), + gain=math.sqrt(2), + pad=(1, 1, 1, 2), + use_weight_norm=False, + ): + pad = torch.nn.ReplicationPad2d(pad) + conv = nn.Conv2d(in_ch, out_ch, kernel_size=kernel_size, stride=strides, padding=0) + + if use_weight_norm: + conv = torch.nn.utils.weight_norm(conv, name='weight') + + w = conv.weight + k = w.size(1) * w.size(2) * w.size(3) + conv.weight.data.normal_(0.0, gain / math.sqrt(k)) + nn.init.constant_(conv.bias, 0.01) + return conv, pad + + def _gen_deconv(self, in_ch, out_ch, strides=(2, 1), k=4, gain=math.sqrt(2), p=1, use_weight_norm=False): + conv = nn.ConvTranspose2d( + in_ch, out_ch, kernel_size=(k, 3), stride=strides, padding_mode='zeros', padding=(p, 1), dilation=1 + ) + + if use_weight_norm: + conv = torch.nn.utils.weight_norm(conv, name='weight') + + w = conv.weight + k = w.size(1) * w.size(2) * w.size(3) + conv.weight.data.normal_(0.0, gain / math.sqrt(k)) + nn.init.constant_(conv.bias, 0.01) + return conv + + @property + def input_types(self): + return { + "mel": NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + } + + @property + def output_types(self): + return { + "spec": NeuralType(('B', 'C', 'D', 'T'), SpectrogramType()), + } + + def input_example(self): + pass diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/fastpitch.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/fastpitch.py new file mode 100644 index 0000000000..ba1bf8a14b --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/fastpitch.py @@ -0,0 +1,309 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# BSD 3-Clause License +# +# Copyright (c) 2021, NVIDIA Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import torch + +from nemo.collections.tts.helpers.helpers import binarize_attention_parallel +from nemo.core.classes import NeuralModule, typecheck +from nemo.core.neural_types.elements import ( + EncodedRepresentation, + Index, + LengthsType, + LogprobsType, + MelSpectrogramType, + ProbsType, + RegressionValuesType, + TokenDurationType, + TokenIndex, + TokenLogDurationType, +) +from nemo.core.neural_types.neural_type import NeuralType + + +def regulate_len(durations, enc_out, pace=1.0, mel_max_len=None): + """If target=None, then predicted durations are applied""" + dtype = enc_out.dtype + reps = durations.float() / pace + reps = (reps + 0.5).long() + dec_lens = reps.sum(dim=1) + + max_len = dec_lens.max() + reps_cumsum = torch.cumsum(torch.nn.functional.pad(reps, (1, 0, 0, 0), value=0.0), dim=1)[:, None, :] + reps_cumsum = reps_cumsum.to(dtype) + + range_ = torch.arange(max_len).to(enc_out.device)[None, :, None] + mult = (reps_cumsum[:, :, :-1] <= range_) & (reps_cumsum[:, :, 1:] > range_) + mult = mult.to(dtype) + enc_rep = torch.matmul(mult, enc_out) + + if mel_max_len: + enc_rep = enc_rep[:, :mel_max_len] + dec_lens = torch.clamp_max(dec_lens, mel_max_len) + return enc_rep, dec_lens + + +def average_pitch(pitch, durs): + durs_cums_ends = torch.cumsum(durs, dim=1).long() + durs_cums_starts = torch.nn.functional.pad(durs_cums_ends[:, :-1], (1, 0)) + pitch_nonzero_cums = torch.nn.functional.pad(torch.cumsum(pitch != 0.0, dim=2), (1, 0)) + pitch_cums = torch.nn.functional.pad(torch.cumsum(pitch, dim=2), (1, 0)) + + bs, l = durs_cums_ends.size() + n_formants = pitch.size(1) + dcs = durs_cums_starts[:, None, :].expand(bs, n_formants, l) + dce = durs_cums_ends[:, None, :].expand(bs, n_formants, l) + + pitch_sums = (torch.gather(pitch_cums, 2, dce) - torch.gather(pitch_cums, 2, dcs)).float() + pitch_nelems = (torch.gather(pitch_nonzero_cums, 2, dce) - torch.gather(pitch_nonzero_cums, 2, dcs)).float() + + pitch_avg = torch.where(pitch_nelems == 0.0, pitch_nelems, pitch_sums / pitch_nelems) + return pitch_avg + + +class ConvReLUNorm(torch.nn.Module): + def __init__(self, in_channels, out_channels, kernel_size=1, dropout=0.0): + super(ConvReLUNorm, self).__init__() + self.conv = torch.nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size, padding=(kernel_size // 2)) + self.norm = torch.nn.LayerNorm(out_channels) + self.dropout = torch.nn.Dropout(dropout) + + def forward(self, signal): + out = torch.nn.functional.relu(self.conv(signal)) + out = self.norm(out.transpose(1, 2)).transpose(1, 2) + return self.dropout(out) + + +class TemporalPredictor(NeuralModule): + """Predicts a single float per each temporal location""" + + def __init__(self, input_size, filter_size, kernel_size, dropout, n_layers=2): + super(TemporalPredictor, self).__init__() + + self.layers = torch.nn.Sequential( + *[ + ConvReLUNorm( + input_size if i == 0 else filter_size, filter_size, kernel_size=kernel_size, dropout=dropout + ) + for i in range(n_layers) + ] + ) + self.fc = torch.nn.Linear(filter_size, 1, bias=True) + + @property + def input_types(self): + return { + "enc": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "enc_mask": NeuralType(('B', 'T', 1), TokenDurationType()), + } + + @property + def output_types(self): + return { + "out": NeuralType(('B', 'T'), EncodedRepresentation()), + } + + def forward(self, enc, enc_mask): + out = enc * enc_mask + out = self.layers(out.transpose(1, 2)).transpose(1, 2) + out = self.fc(out) * enc_mask + return out.squeeze(-1) + + +class FastPitchModule(NeuralModule): + def __init__( + self, + encoder_module: NeuralModule, + decoder_module: NeuralModule, + duration_predictor: NeuralModule, + pitch_predictor: NeuralModule, + aligner: NeuralModule, + n_speakers: int, + symbols_embedding_dim: int, + pitch_embedding_kernel_size: int, + n_mel_channels: int = 80, + max_token_duration: int = 75, + ): + super().__init__() + + self.encoder = encoder_module + self.decoder = decoder_module + self.duration_predictor = duration_predictor + self.pitch_predictor = pitch_predictor + self.aligner = aligner + self.learn_alignment = aligner is not None + self.use_duration_predictor = True + self.binarize = False + + if n_speakers > 1: + self.speaker_emb = torch.nn.Embedding(n_speakers, symbols_embedding_dim) + else: + self.speaker_emb = None + + self.max_token_duration = max_token_duration + + self.pitch_emb = torch.nn.Conv1d( + 1, + symbols_embedding_dim, + kernel_size=pitch_embedding_kernel_size, + padding=int((pitch_embedding_kernel_size - 1) / 2), + ) + + # Store values precomputed from training data for convenience + self.register_buffer('pitch_mean', torch.zeros(1)) + self.register_buffer('pitch_std', torch.zeros(1)) + + self.proj = torch.nn.Linear(self.decoder.d_model, n_mel_channels, bias=True) + + @property + def input_types(self): + return { + "text": NeuralType(('B', 'T'), TokenIndex()), + "durs": NeuralType(('B', 'T'), TokenDurationType()), + "pitch": NeuralType(('B', 'T'), RegressionValuesType()), + "speaker": NeuralType(('B'), Index()), + "pace": NeuralType(optional=True), + "spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType(), optional=True), + "attn_prior": NeuralType(('B', 'T', 'T'), ProbsType(), optional=True), + "mel_lens": NeuralType(('B'), LengthsType(), optional=True), + "input_lens": NeuralType(('B'), LengthsType(), optional=True), + } + + @property + def output_types(self): + return { + "spect": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "num_frames": NeuralType(('B'), TokenDurationType()), + "durs_predicted": NeuralType(('B', 'T'), TokenDurationType()), + "log_durs_predicted": NeuralType(('B', 'T'), TokenLogDurationType()), + "pitch_predicted": NeuralType(('B', 'T'), RegressionValuesType()), + "attn_soft": NeuralType(('B', 'S', 'T', 'D'), ProbsType()), + "attn_logprob": NeuralType(('B', 'S', 'T', 'D'), LogprobsType()), + "attn_hard": NeuralType(('B', 'S', 'T', 'D'), ProbsType()), + "attn_hard_dur": NeuralType(('B', 'T'), TokenDurationType()), + "pitch": NeuralType(('B', 'T'), RegressionValuesType()), + } + + @typecheck() + def forward( + self, + *, + text, + durs=None, + pitch=None, + speaker=None, + pace=1.0, + spec=None, + attn_prior=None, + mel_lens=None, + input_lens=None, + ): + + if not self.learn_alignment and self.training: + assert durs is not None + assert pitch is not None + + # Calculate speaker embedding + if self.speaker_emb is None or speaker is None: + spk_emb = 0 + else: + spk_emb = self.speaker_emb(speaker).unsqueeze(1) + + # Input FFT + enc_out, enc_mask = self.encoder(input=text, conditioning=spk_emb) + + log_durs_predicted = self.duration_predictor(enc_out, enc_mask) + durs_predicted = torch.clamp(torch.exp(log_durs_predicted) - 1, 0, self.max_token_duration) + + attn_soft, attn_hard, attn_hard_dur, attn_logprob = None, None, None, None + if self.learn_alignment and spec is not None: + text_emb = self.encoder.word_emb(text) + attn_soft, attn_logprob = self.aligner(spec, text_emb.permute(0, 2, 1), enc_mask == 0, attn_prior) + attn_hard = binarize_attention_parallel(attn_soft, input_lens, mel_lens) + attn_hard_dur = attn_hard.sum(2)[:, 0, :] + + # Predict pitch + pitch_predicted = self.pitch_predictor(enc_out, enc_mask) + if pitch is not None: + if self.learn_alignment: + pitch = average_pitch(pitch.unsqueeze(1), attn_hard_dur).squeeze(1) + pitch_emb = self.pitch_emb(pitch.unsqueeze(1)) + else: + pitch_emb = self.pitch_emb(pitch_predicted.unsqueeze(1)) + + enc_out = enc_out + pitch_emb.transpose(1, 2) + + if self.learn_alignment and spec is not None: + len_regulated, dec_lens = regulate_len(attn_hard_dur, enc_out, pace) + elif spec is None and durs is not None: + len_regulated, dec_lens = regulate_len(durs, enc_out, pace) + # Use predictions during inference + elif spec is None: + len_regulated, dec_lens = regulate_len(durs_predicted, enc_out, pace) + + # Output FFT + dec_out, _ = self.decoder(input=len_regulated, seq_lens=dec_lens) + spect = self.proj(dec_out).transpose(1, 2) + return ( + spect, + dec_lens, + durs_predicted, + log_durs_predicted, + pitch_predicted, + attn_soft, + attn_logprob, + attn_hard, + attn_hard_dur, + pitch, + ) + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + par = next(self.parameters()) + inp = torch.randint(0, self.encoder.word_emb.num_embeddings, (1, 44), device=par.device, dtype=torch.int64) + + return ({'text': inp},) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/fastspeech2.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/fastspeech2.py new file mode 100644 index 0000000000..da0eb1dcba --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/fastspeech2.py @@ -0,0 +1,314 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np +import torch +import torch.nn as nn + +from nemo.collections.tts.helpers.helpers import get_mask_from_lengths +from nemo.collections.tts.modules.fastspeech2_submodules import FFTransformer, LengthRegulator, VariancePredictor +from nemo.core.classes import NeuralModule, typecheck +from nemo.core.neural_types.elements import ( + EncodedRepresentation, + LengthsType, + MaskType, + MelSpectrogramType, + RegressionValuesType, + TokenDurationType, + TokenIndex, +) +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +class FastSpeech2Encoder(NeuralModule): + def __init__( + self, + d_model=256, + n_layers=4, + n_attn_heads=2, + d_attn_head=256, + d_inner=1024, + kernel_size=9, + dropout=0.1, + attn_dropout=0.1, + n_embed=84, + padding_idx=83, + ): + """ + FastSpeech 2 encoder. Converts phoneme sequence to the phoneme hidden sequence. + Consists of a phoneme embedding lookup, positional encoding, and feed-forward + Transformer blocks (4 by default). + + Args: + d_model: Model input (embedding) dimension. Defaults to 256. + n_layers: Number of feed-forward Transformer layers in the encoder. Defaults to 4. + n_attn_heads: Number of attention heads for the feed-forward Transformer in the encoder. + Defaults to 2. + d_attn_head: Dimensionality of the attention heads. Defaults to 256. + d_inner: Encoder hidden dimension. Defaults to 1024. + kernel_size: Encoder Conv1d kernel size (kernel_size, 1). Defaults to 9. + dropout: Encoder feed-forward Transformer dropout. Defaults to 0.1. + attn_dropout: Encoder attention dropout. Defaults to 0.1. + n_embed: Embedding input dim, should match number of tokens. Defaults to 84. + padding_idx: Padding token index. Deafaults to 83. + """ + super().__init__() + + self.encoder = FFTransformer( + n_layer=n_layers, + n_head=n_attn_heads, + d_model=d_model, + d_head=d_attn_head, + d_inner=d_inner, + kernel_size=(kernel_size, 1), + dropout=dropout, + dropatt=attn_dropout, # Not in the FS2 paper + embed_input=True, # For the encoder, need to do embedding lookup + n_embed=n_embed, + padding_idx=padding_idx, + ) + + @property + def input_types(self): # phonemes + return {"text": NeuralType(('B', 'T'), TokenIndex()), "text_length": NeuralType(('B'), LengthsType())} + + @property + def output_types(self): + return { + "out": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "mask": NeuralType(('B', 'T', 'D'), MaskType()), + } + + @typecheck() + def forward(self, *, text, text_length): + return self.encoder(text, seq_lens=text_length) + + +class VarianceAdaptor(NeuralModule): + def __init__( + self, + d_model=256, + dropout=0.2, + dur_d_hidden=256, + dur_kernel_size=3, + pitch=True, + log_pitch=True, + n_f0_bins=256, + pitch_kernel_size=3, + pitch_min=80.0, + pitch_max=800.0, + energy=True, + n_energy_bins=256, + energy_kernel_size=3, + energy_min=0.0, + energy_max=600.0, + ): + """ + FastSpeech 2 variance adaptor, which adds information like duration, pitch, etc. to the phoneme encoding. + Sets of conv1D blocks with ReLU and dropout. + + Args: + d_model: Input and hidden dimension. Defaults to 256 (default encoder output dim). + dropout: Variance adaptor dropout. Defaults to 0.2. + dur_d_hidden: Hidden dim of the duration predictor. Defaults to 256. + dur_kernel_size: Kernel size for the duration predictor. Defaults to 3. + pitch (bool): Whether or not to use the pitch predictor. + log_pitch (bool): If True, uses log pitch. Defaults to True. + n_f0_bins: Number of F0 bins for the pitch predictor. Defaults to 256. + pitch_kernel_size: Kernel size for the pitch predictor. Defaults to 3. + pitch_min: Defaults to 80.0. + pitch_max: Defaults to 800.0. + pitch_d_hidden: Hidden dim of the pitch predictor. + energy (bool): Whether or not to use the energy predictor. + n_energy_bins: Number of energy bins. Defaults to 256. + energy_kernel_size: Kernel size for the energy predictor. Defaults to 3. + energy_min: Defaults to 0.0. + energy_max: Defaults to 600.0. + """ + super().__init__() + + # -- Duration Setup -- + self.duration_predictor = VariancePredictor( + d_model=d_model, d_inner=dur_d_hidden, kernel_size=dur_kernel_size, dropout=dropout + ) + self.length_regulator = LengthRegulator() + + self.pitch = pitch + self.energy = energy + + # -- Pitch Setup -- + # NOTE: Pitch is clamped to 1e-5 which gets mapped to bin 1. But it is padded with 0s that get mapped to bin 0. + if self.pitch: + if log_pitch: + pitch_min = np.log(pitch_min) + pitch_max = np.log(pitch_max) + pitch_operator = torch.exp if log_pitch else lambda x: x + pitch_bins = pitch_operator(torch.linspace(start=pitch_min, end=pitch_max, steps=n_f0_bins - 1)) + # Prepend 0 for unvoiced frames + pitch_bins = torch.cat((torch.tensor([0.0]), pitch_bins)) + + self.register_buffer("pitch_bins", pitch_bins) + self.pitch_predictor = VariancePredictor( + d_model=d_model, d_inner=n_f0_bins, kernel_size=pitch_kernel_size, dropout=dropout + ) + # Predictor outputs values directly rather than one-hot vectors, therefore Embedding + self.pitch_lookup = nn.Embedding(n_f0_bins, d_model) + + # -- Energy Setup -- + if self.energy: + self.register_buffer( # Linear scale bins + "energy_bins", torch.linspace(start=energy_min, end=energy_max, steps=n_energy_bins - 1) + ) + self.energy_predictor = VariancePredictor( + d_model=d_model, d_inner=n_energy_bins, kernel_size=energy_kernel_size, dropout=dropout, + ) + self.energy_lookup = nn.Embedding(n_energy_bins, d_model) + + @property + def input_types(self): + return { + "x": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "x_len": NeuralType(('B'), LengthsType()), + "dur_target": NeuralType(('B', 'T'), TokenDurationType(), optional=True), + "pitch_target": NeuralType(('B', 'T'), RegressionValuesType(), optional=True), + "energy_target": NeuralType(('B', 'T'), RegressionValuesType(), optional=True), + "spec_len": NeuralType(('B'), LengthsType(), optional=True), + } + + @property + def output_types(self): + return { + "out": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "log_dur_preds": NeuralType(('B', 'T'), TokenDurationType()), + "pitch_preds": NeuralType(('B', 'T'), RegressionValuesType()), + "energy_preds": NeuralType(('B', 'T'), RegressionValuesType()), + "spec_len": NeuralType(('B'), LengthsType()), + } + + @typecheck() + def forward(self, *, x, x_len, dur_target=None, pitch_target=None, energy_target=None, spec_len=None): + """ + Args: + x: Input from the encoder. + x_len: Length of the input. + dur_target: Duration targets for the duration predictor. Needs to be passed in during training. + pitch_target: Pitch targets for the pitch predictor. Needs to be passed in during training. + energy_target: Energy targets for the energy predictor. Needs to be passed in during training. + spec_len: Target spectrogram length. Needs to be passed in during training. + """ + # Duration predictions (or ground truth) fed into Length Regulator to + # expand the hidden states of the encoder embedding + log_dur_preds = self.duration_predictor(x) + log_dur_preds.masked_fill_(~get_mask_from_lengths(x_len), 0) + # Output is Batch, Time + if dur_target is not None: + dur_out = self.length_regulator(x, dur_target) + else: + dur_preds = torch.clamp_min(torch.round(torch.exp(log_dur_preds)) - 1, 0).long() + if not torch.sum(dur_preds, dim=1).bool().all(): + logging.error("Duration prediction failed on this batch. Settings to 1s") + dur_preds += 1 + dur_out = self.length_regulator(x, dur_preds) + spec_len = torch.sum(dur_preds, dim=1) + out = dur_out + out *= get_mask_from_lengths(spec_len).unsqueeze(-1) + + # Pitch + pitch_preds = None + if self.pitch: + # Possible future work: + # Add pitch spectrogram prediction & conversion back to pitch contour using iCWT + # (see Appendix C of the FastSpeech 2/2s paper). + pitch_preds = self.pitch_predictor(dur_out) + pitch_preds.masked_fill_(~get_mask_from_lengths(spec_len), 0) + if pitch_target is not None: + pitch_out = self.pitch_lookup(torch.bucketize(pitch_target, self.pitch_bins)) + else: + pitch_out = self.pitch_lookup(torch.bucketize(pitch_preds.detach(), self.pitch_bins)) + out += pitch_out + out *= get_mask_from_lengths(spec_len).unsqueeze(-1) + + # Energy + energy_preds = None + if self.energy: + energy_preds = self.energy_predictor(dur_out) + if energy_target is not None: + energy_out = self.energy_lookup(torch.bucketize(energy_target, self.energy_bins)) + else: + energy_out = self.energy_lookup(torch.bucketize(energy_preds.detach(), self.energy_bins)) + out += energy_out + out *= get_mask_from_lengths(spec_len).unsqueeze(-1) + + return out, log_dur_preds, pitch_preds, energy_preds, spec_len + + +class MelSpecDecoder(NeuralModule): + def __init__( + self, + d_model=256, + d_out=80, + n_layers=4, + n_attn_heads=2, + d_attn_head=256, + d_inner=1024, + kernel_size=9, + dropout=0.1, + attn_dropout=0.1, + ): + """ + FastSpeech 2 mel-spectrogram decoder. Converts adapted hidden sequence to a mel-spectrogram sequence. + Consists of four feed-forward Transformer blocks by default. + + Args: + d_model: Input dimension. Defaults to 256. + d_out: Dimensionality of output. Defaults to 80. + n_layers: Number of feed-forward Transformer layers in the mel-spec decoder. Defaults to 4. + n_attn_heads: Number of attention heads for the feed-forward Transformer. Defaults to 2. + d_attn_head: Dimensionality of the attention heads. Defaults to 256. + d_inner: Mel-spec decoder hidden dimension. Defaults to 1024. + kernel_size: Mel-spec decoder Conv1d kernel size (kernel_size, 1). Defaults to 9. + dropout: Mel-spec decoder feed-forward Transformer dropout. Defaults to 0.1. + attn_dropout: Mel-spec decoder attention dropout. Defaults to 0.1. + """ + super().__init__() + + self.decoder = FFTransformer( + n_layer=n_layers, + n_head=n_attn_heads, + d_model=d_model, + d_head=d_attn_head, + d_inner=d_inner, + kernel_size=(kernel_size, 1), + dropout=dropout, + dropatt=attn_dropout, + embed_input=False, + ) + self.linear = nn.Linear(d_model, d_out) + + @property + def input_types(self): + return { + "decoder_input": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "lengths": NeuralType(('B'), LengthsType()), + } + + @property + def output_types(self): + return {"mel_spec": NeuralType(('B', 'T', 'C'), MelSpectrogramType())} + + @typecheck() + def forward(self, *, decoder_input, lengths): + decoder_out, _ = self.decoder(decoder_input, lengths) + mel_out = self.linear(decoder_out) + return mel_out diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/fastspeech2_submodules.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/fastspeech2_submodules.py new file mode 100644 index 0000000000..2c25978f2a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/fastspeech2_submodules.py @@ -0,0 +1,160 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nemo.collections.tts.helpers.helpers import get_mask_from_lengths +from nemo.collections.tts.modules.transformer import PositionalEmbedding, TransformerLayer + + +class FFTransformer(nn.Module): + def __init__( + self, + n_layer, + n_head, + d_model, + d_head, + d_inner, + kernel_size, + dropout, + dropatt, + dropemb=0.0, + pre_lnorm=False, + embed_input=True, + n_embed=84, + padding_idx=83, + ): + """ + Feed-Forward Transformer submodule for FastSpeech 2 that consists of multiple TransformerLayers. + """ + super(FFTransformer, self).__init__() + self.d_model = d_model + self.n_head = n_head + self.d_head = d_head + + self.padding_idx = padding_idx + + if embed_input: + self.word_emb = nn.Embedding(n_embed, d_model, padding_idx=self.padding_idx) + else: + self.word_emb = None + + self.pos_emb = PositionalEmbedding(self.d_model) + self.drop = nn.Dropout(dropemb) + self.layers = nn.ModuleList() + + for _ in range(n_layer): + self.layers.append( + TransformerLayer( + n_head, d_model, d_head, d_inner, kernel_size, dropout, dropatt=dropatt, pre_lnorm=pre_lnorm + ) + ) + + def forward(self, dec_inp, seq_lens=None): + if self.word_emb is None: + inp = dec_inp + mask = get_mask_from_lengths(seq_lens).unsqueeze(2) + else: + inp = self.word_emb(dec_inp) + # [bsz x L x 1] + mask = (dec_inp != self.padding_idx).unsqueeze(2) + + pos_seq = torch.arange(inp.size(1), device=inp.device, dtype=inp.dtype) + pos_emb = self.pos_emb(pos_seq) * mask + out = self.drop(inp + pos_emb) + + for layer in self.layers: + out = layer(out, mask=mask) + + # out = self.drop(out) + return out, mask + + +class Transpose(nn.Module): + def forward(self, x): + return x.transpose(1, 2) + + +class VariancePredictor(nn.Module): + def __init__(self, d_model, d_inner, kernel_size, dropout): + """ + Variance predictor submodule for FastSpeech 2/, used for pitch and energy prediction. + + Args: + d_model: Input dimension. + d_inner: Hidden dimension of the variance predictor. + kernel_size: Conv1d kernel size. + dropout: Dropout value for the variance predictor. + """ + super().__init__() + + self.d_model = d_model + self.d_inner = d_inner + self.kernel_size = kernel_size + + self.layers = nn.Sequential( + Transpose(), + nn.Conv1d(d_model, d_inner, kernel_size, stride=1, padding=(kernel_size // 2)), + nn.ReLU(), + Transpose(), + nn.LayerNorm(d_inner), + Transpose(), + nn.Dropout(dropout), + nn.Conv1d(d_inner, d_inner, kernel_size, stride=1, padding=(kernel_size // 2)), + nn.ReLU(), + Transpose(), + nn.LayerNorm(d_inner), + nn.Dropout(dropout), + nn.Linear(d_inner, 1), + ) + + def forward(self, vp_input): + return self.layers(vp_input).squeeze(-1) + + +class LengthRegulator(nn.Module): + def forward(self, x, dur): + """ + Expands the hidden states according to the duration target/prediction (depends on train vs inference). + For frame i of a single batch element x, repeats each the frame dur[i] times. + + Args: + x: Hidden states of dimension (batch, time, emb_dim) + dur: Timings for each frame of the hiddens, dimension (batch, time) + """ + output = [] + for x_i, dur_i in zip(x, dur): + expanded = self.expand(x_i, dur_i) + output.append(expanded) + output = self.pad(output) + return output + + def expand(self, x, dur): + output = [] + for i, frame in enumerate(x): + expanded_len = int(dur[i] + 0.5) + expanded = frame.expand(expanded_len, -1) + output.append(expanded) + output = torch.cat(output, 0) + return output + + def pad(self, x): + output = [] + max_len = max([x[i].size(0) for i in range(len(x))]) + for i, seq in enumerate(x): + padded = F.pad(seq, [0, 0, 0, max_len - seq.size(0)], 'constant', 0.0) + output.append(padded) + output = torch.stack(output) + return output diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts.py new file mode 100644 index 0000000000..4d31af2c36 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts.py @@ -0,0 +1,441 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MIT License +# +# Copyright (c) 2020 Jaehyeon Kim +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import math + +import torch +import torch.nn.functional as F +from torch import nn + +from nemo.collections.tts.modules import glow_tts_submodules +from nemo.core.classes import NeuralModule, typecheck +from nemo.core.neural_types.elements import ( + AcousticEncodedRepresentation, + IntType, + LengthsType, + LogDeterminantType, + MaskType, + MelSpectrogramType, + NormalDistributionLogVarianceType, + NormalDistributionMeanType, + NormalDistributionSamplesType, + SequenceToSequenceAlignmentType, + TokenIndex, + TokenLogDurationType, +) +from nemo.core.neural_types.neural_type import NeuralType + + +class TextEncoder(NeuralModule): + def __init__( + self, + n_vocab: int, + out_channels: int, + hidden_channels: int, + filter_channels: int, + filter_channels_dp: int, + n_heads: int, + n_layers: int, + kernel_size: int, + p_dropout: float, + window_size: int, + mean_only: bool = False, + prenet: bool = False, + gin_channels: int = 0, + ): + """ + GlowTTS text encoder. Takes in the input text tokens and produces prior distribution statistics for the latent + representation corresponding to each token, as well as the log durations (the Duration Predictor is also part of + this module). + Architecture is similar to Transformer TTS with slight modifications. + Args: + n_vocab (int): Number of tokens in the vocabulary + out_channels (int): Latent representation channels + hidden_channels (int): Number of channels in the intermediate representations + filter_channels (int): Number of channels for the representations in the feed-forward layer + filter_channels_dp (int): Number of channels for the representations in the duration predictor + n_heads (int): Number of attention heads + n_layers (int): Number of transformer layers + kernel_size (int): Kernels size for the feed-forward layer + p_dropout (float): Dropout probability + mean_only (bool): Return zeros for logs if true + prenet (bool): Use an additional network before the transformer modules + gin_channels (int): Number of channels in speaker embeddings + """ + super().__init__() + + self.n_layers = n_layers + self.hidden_channels = hidden_channels + self.prenet = prenet + self.mean_only = mean_only + + self.emb = nn.Embedding(n_vocab, hidden_channels) + nn.init.normal_(self.emb.weight, 0.0, hidden_channels ** -0.5) + + if prenet: + self.pre = glow_tts_submodules.ConvReluNorm( + hidden_channels, hidden_channels, hidden_channels, kernel_size=5, n_layers=3, p_dropout=0.1, + ) + + self.drop = nn.Dropout(p_dropout) + self.attn_layers = nn.ModuleList() + self.norm_layers_1 = nn.ModuleList() + self.ffn_layers = nn.ModuleList() + self.norm_layers_2 = nn.ModuleList() + + for _ in range(self.n_layers): + self.attn_layers.append( + glow_tts_submodules.AttentionBlock( + hidden_channels, hidden_channels, n_heads, window_size=window_size, p_dropout=p_dropout, + ) + ) + self.norm_layers_1.append(glow_tts_submodules.LayerNorm(hidden_channels)) + self.ffn_layers.append( + glow_tts_submodules.FeedForwardNetwork( + hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout + ) + ) + self.norm_layers_2.append(glow_tts_submodules.LayerNorm(hidden_channels)) + + self.proj_m = nn.Conv1d(hidden_channels, out_channels, 1) + if not mean_only: + self.proj_s = nn.Conv1d(hidden_channels, out_channels, 1) + self.proj_w = glow_tts_submodules.DurationPredictor( + hidden_channels + gin_channels, filter_channels_dp, kernel_size, p_dropout + ) + + @property + def input_types(self): + return { + "text": NeuralType(('B', 'T'), TokenIndex()), + "text_lengths": NeuralType(('B',), LengthsType()), + "speaker_embeddings": NeuralType(('B', 'D'), AcousticEncodedRepresentation(), optional=True), + } + + @property + def output_types(self): + return { + "x_m": NeuralType(('B', 'D', 'T'), NormalDistributionMeanType()), + "x_logs": NeuralType(('B', 'D', 'T'), NormalDistributionLogVarianceType()), + "logw": NeuralType(('B', 'T'), TokenLogDurationType()), + "x_mask": NeuralType(('B', 'D', 'T'), MaskType()), + } + + @typecheck() + def forward(self, *, text, text_lengths, speaker_embeddings=None): + + x = self.emb(text) * math.sqrt(self.hidden_channels) # [b, t, h] + + x = torch.transpose(x, 1, -1) # [b, h, t] + x_mask = torch.unsqueeze(glow_tts_submodules.sequence_mask(text_lengths, x.size(2)), 1).to(x.dtype) + + if self.prenet: + x = self.pre(x, x_mask) + + attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1) + for i in range(self.n_layers): + x = x * x_mask + y = self.attn_layers[i](x, x, attn_mask) + y = self.drop(y) + x = self.norm_layers_1[i](x + y) + + y = self.ffn_layers[i](x, x_mask) + y = self.drop(y) + x = self.norm_layers_2[i](x + y) + x = x * x_mask + + if speaker_embeddings is not None: + g_exp = speaker_embeddings.expand(-1, -1, x.size(-1)) + x_dp = torch.cat([torch.detach(x), g_exp], 1) + else: + x_dp = torch.detach(x) + + x_m = self.proj_m(x) * x_mask + if not self.mean_only: + x_logs = self.proj_s(x) * x_mask + else: + x_logs = torch.zeros_like(x_m) + + logw = self.proj_w(spect=x_dp, mask=x_mask) + + return x_m, x_logs, logw, x_mask + + +class FlowSpecDecoder(NeuralModule): + def __init__( + self, + in_channels: int, + hidden_channels: int, + kernel_size: int, + dilation_rate: int, + n_blocks: int, + n_layers: int, + p_dropout: float = 0.0, + n_split: int = 4, + n_sqz: int = 2, + sigmoid_scale: bool = False, + gin_channels: int = 0, + ): + """ + Flow-based invertible decoder for GlowTTS. Converts spectrograms to latent representations and back. + Args: + in_channels (int): Number of channels in the input spectrogram + hidden_channels (int): Number of channels in the intermediate representations + kernel_size (int): Kernel size in the coupling blocks + dilation_rate (int): Dilation rate in the WaveNet-like blocks + n_blocks (int): Number of flow blocks + n_layers (int): Number of layers within each coupling block + p_dropout (float): Dropout probability + n_split (int): Group size for the invertible convolution + n_sqz (int): The rate by which the spectrograms are squeezed before applying the flows + sigmoid_scale (bool): Apply sigmoid to logs in the coupling blocks + """ + super().__init__() + + self.n_sqz = n_sqz + + self.flows = nn.ModuleList() + for _ in range(n_blocks): + self.flows.append(glow_tts_submodules.ActNorm(channels=in_channels * n_sqz)) + self.flows.append(glow_tts_submodules.InvConvNear(channels=in_channels * n_sqz, n_split=n_split)) + self.flows.append( + glow_tts_submodules.CouplingBlock( + in_channels * n_sqz, + hidden_channels, + kernel_size=kernel_size, + dilation_rate=dilation_rate, + n_layers=n_layers, + p_dropout=p_dropout, + sigmoid_scale=sigmoid_scale, + gin_channels=gin_channels, + ) + ) + + @property + def input_types(self): + return { + "spect": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "spect_mask": NeuralType(('B', 'D', 'T'), MaskType()), + "speaker_embeddings": NeuralType(('B', 'D'), AcousticEncodedRepresentation(), optional=True), + "reverse": NeuralType(elements_type=IntType(), optional=True), + } + + @property + def output_types(self): + return { + "z": NeuralType(('B', 'D', 'T'), NormalDistributionSamplesType()), + "logdet_tot": NeuralType(('B',), LogDeterminantType()), + } + + @typecheck() + def forward(self, *, spect, spect_mask, speaker_embeddings=None, reverse=False): + if not reverse: + flows = self.flows + logdet_tot = 0 + else: + flows = reversed(self.flows) + logdet_tot = None + + x = spect + x_mask = spect_mask + + if self.n_sqz > 1: + x, x_mask = self.squeeze(x, x_mask, self.n_sqz) + for f in flows: + if not reverse: + x, logdet = f(x, x_mask, g=speaker_embeddings, reverse=reverse) + logdet_tot += logdet + else: + x, logdet = f(x, x_mask, g=speaker_embeddings, reverse=reverse) + if self.n_sqz > 1: + x, x_mask = self.unsqueeze(x, x_mask, self.n_sqz) + return x, logdet_tot + + def squeeze(self, x, x_mask=None, n_sqz=2): + b, c, t = x.size() + + t = (t // n_sqz) * n_sqz + x = x[:, :, :t] + x_sqz = x.view(b, c, t // n_sqz, n_sqz) + x_sqz = x_sqz.permute(0, 3, 1, 2).contiguous().view(b, c * n_sqz, t // n_sqz) + + if x_mask is not None: + x_mask = x_mask[:, :, n_sqz - 1 :: n_sqz] + else: + x_mask = torch.ones(b, 1, t // n_sqz).to(device=x.device, dtype=x.dtype) + return x_sqz * x_mask, x_mask + + def unsqueeze(self, x, x_mask=None, n_sqz=2): + b, c, t = x.size() + + x_unsqz = x.view(b, n_sqz, c // n_sqz, t) + x_unsqz = x_unsqz.permute(0, 2, 3, 1).contiguous().view(b, c // n_sqz, t * n_sqz) + + if x_mask is not None: + x_mask = x_mask.unsqueeze(-1).repeat(1, 1, 1, n_sqz).view(b, 1, t * n_sqz) + else: + x_mask = torch.ones(b, 1, t * n_sqz).to(device=x.device, dtype=x.dtype) + return x_unsqz * x_mask, x_mask + + def store_inverse(self): + for f in self.flows: + f.store_inverse() + + +class GlowTTSModule(NeuralModule): + def __init__( + self, encoder_module: NeuralModule, decoder_module: NeuralModule, n_speakers: int = 1, gin_channels: int = 0 + ): + """ + Main GlowTTS module. Contains the encoder and decoder. + Args: + encoder_module (NeuralModule): Text encoder for predicting latent distribution statistics + decoder_module (NeuralModule): Invertible spectrogram decoder + n_speakers (int): Number of speakers + gin_channels (int): Channels in speaker embeddings + """ + super().__init__() + + self.encoder = encoder_module + self.decoder = decoder_module + + if n_speakers > 1: + self.emb_g = nn.Embedding(n_speakers, gin_channels) + nn.init.uniform_(self.emb_g.weight, -0.1, 0.1) + + @property + def input_types(self): + return { + "text": NeuralType(('B', 'T'), TokenIndex()), + "text_lengths": NeuralType(('B'), LengthsType()), + "spect": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "spect_lengths": NeuralType(('B'), LengthsType()), + "speaker": NeuralType(('B'), IntType(), optional=True), + } + + @property + def output_types(self): + return { + "z": NeuralType(('B', 'D', 'T'), NormalDistributionSamplesType()), + "y_m": NeuralType(('B', 'D', 'T'), NormalDistributionMeanType()), + "y_logs": NeuralType(('B', 'D', 'T'), NormalDistributionLogVarianceType()), + "logdet": NeuralType(('B'), LogDeterminantType()), + "log_durs_predicted": NeuralType(('B', 'T'), TokenLogDurationType()), + "log_durs_extracted": NeuralType(('B', 'T'), TokenLogDurationType()), + "spect_lengths": NeuralType(('B'), LengthsType()), + "attn": NeuralType(('B', 'T', 'T'), SequenceToSequenceAlignmentType()), + } + + @typecheck() + def forward(self, *, text, text_lengths, spect, spect_lengths, speaker=None): + + if speaker is not None: + speaker = F.normalize(self.emb_g(speaker)).unsqueeze(-1) # [b, h] + + x_m, x_logs, log_durs_predicted, x_mask = self.encoder( + text=text, text_lengths=text_lengths, speaker_embeddings=speaker + ) + + y_max_length = spect.size(2) + y_max_length = (y_max_length // self.decoder.n_sqz) * self.decoder.n_sqz + spect = spect[:, :, :y_max_length] + + spect_lengths = (spect_lengths // self.decoder.n_sqz) * self.decoder.n_sqz + + y_mask = torch.unsqueeze(glow_tts_submodules.sequence_mask(spect_lengths, y_max_length), 1).to(x_mask.dtype) + attn_mask = torch.unsqueeze(x_mask, -1) * torch.unsqueeze(y_mask, 2) + + z, logdet = self.decoder(spect=spect, spect_mask=y_mask, speaker_embeddings=speaker, reverse=False) + + with torch.no_grad(): + x_s_sq_r = torch.exp(-2 * x_logs) + logp1 = torch.sum(-0.5 * math.log(2 * math.pi) - x_logs, [1]).unsqueeze(-1) # [b, t, 1] + logp2 = torch.matmul(x_s_sq_r.transpose(1, 2), -0.5 * (z ** 2)) # [b, t, d] x [b, d, t'] = [b, t, t'] + logp3 = torch.matmul((x_m * x_s_sq_r).transpose(1, 2), z) # [b, t, d] x [b, d, t'] = [b, t, t'] + logp4 = torch.sum(-0.5 * (x_m ** 2) * x_s_sq_r, [1]).unsqueeze(-1) # [b, t, 1] + logp = logp1 + logp2 + logp3 + logp4 # [b, t, t'] + + attn = (glow_tts_submodules.maximum_path(logp, attn_mask.squeeze(1)).unsqueeze(1).detach()).squeeze(1) + + y_m = torch.matmul(x_m, attn) + y_logs = torch.matmul(x_logs, attn) + + log_durs_extracted = torch.log(1e-8 + torch.sum(attn, -1)) * x_mask.squeeze() + + return z, y_m, y_logs, logdet, log_durs_predicted, log_durs_extracted, spect_lengths, attn + + @typecheck( + input_types={ + "text": NeuralType(('B', 'T'), TokenIndex()), + "text_lengths": NeuralType(('B',), LengthsType()), + "speaker": NeuralType(('B'), IntType(), optional=True), + "noise_scale": NeuralType(optional=True), + "length_scale": NeuralType(optional=True), + }, + output_types={ + "y": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "attn": NeuralType(('B', 'T', 'T'), SequenceToSequenceAlignmentType()), + }, + ) + def generate_spect(self, *, text, text_lengths, speaker=None, noise_scale=0.3, length_scale=1.0): + + if speaker is not None: + speaker = F.normalize(self.emb_g(speaker)).unsqueeze(-1) # [b, h] + + x_m, x_logs, log_durs_predicted, x_mask = self.encoder( + text=text, text_lengths=text_lengths, speaker_embeddings=speaker + ) + + w = torch.exp(log_durs_predicted) * x_mask.squeeze() * length_scale + w_ceil = torch.ceil(w) + spect_lengths = torch.clamp_min(torch.sum(w_ceil, [1]), 1).long() + y_max_length = None + + spect_lengths = (spect_lengths // self.decoder.n_sqz) * self.decoder.n_sqz + + y_mask = torch.unsqueeze(glow_tts_submodules.sequence_mask(spect_lengths, y_max_length), 1).to(x_mask.dtype) + attn_mask = torch.unsqueeze(x_mask, -1) * torch.unsqueeze(y_mask, 2) + + attn = glow_tts_submodules.generate_path(w_ceil.squeeze(1), attn_mask.squeeze(1)) + + y_m = torch.matmul(x_m, attn) + y_logs = torch.matmul(x_logs, attn) + + z = (y_m + torch.exp(y_logs) * torch.randn_like(y_m) * noise_scale) * y_mask + y, _ = self.decoder(spect=z, spect_mask=y_mask, speaker_embeddings=speaker, reverse=True) + + return y, attn diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts_parser.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts_parser.py new file mode 100644 index 0000000000..6c8411bdb9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts_parser.py @@ -0,0 +1,364 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MIT License +# +# Copyright (c) 2020 Jaehyeon Kim +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import re +from typing import List, Optional + +import inflect +from unidecode import unidecode + + +class CMUDict: + def __init__(self, file_or_path, keep_ambiguous=True): + """ + Thin wrapper around CMUDict data. http://www.speech.cs.cmu.edu/cgi-bin/cmudict + Args: + file_or_path: file or path to cmu dictionary + keep_ambiguous: keep entries with multiple possible pronunciations + """ + + # fmt: off + self.valid_symbols = [ + 'AA', 'AA0', 'AA1', 'AA2', 'AE', 'AE0', 'AE1', 'AE2', 'AH', 'AH0', 'AH1', 'AH2', + 'AO', 'AO0', 'AO1', 'AO2', 'AW', 'AW0', 'AW1', 'AW2', 'AY', 'AY0', 'AY1', 'AY2', + 'B', 'CH', 'D', 'DH', 'EH', 'EH0', 'EH1', 'EH2', 'ER', 'ER0', 'ER1', 'ER2', 'EY', + 'EY0', 'EY1', 'EY2', 'F', 'G', 'HH', 'IH', 'IH0', 'IH1', 'IH2', 'IY', 'IY0', 'IY1', + 'IY2', 'JH', 'K', 'L', 'M', 'N', 'NG', 'OW', 'OW0', 'OW1', 'OW2', 'OY', 'OY0', + 'OY1', 'OY2', 'P', 'R', 'S', 'SH', 'T', 'TH', 'UH', 'UH0', 'UH1', 'UH2', 'UW', + 'UW0', 'UW1', 'UW2', 'V', 'W', 'Y', 'Z', 'ZH' + ] + # fmt: on + + self._valid_symbol_set = set(self.valid_symbols) + + if isinstance(file_or_path, str): + with open(file_or_path, encoding="latin-1") as f: + entries = self._parse_cmudict(f) + else: + entries = self._parse_cmudict(file_or_path) + if not keep_ambiguous: + entries = {word: pron for word, pron in entries.items() if len(pron) == 1} + self._entries = entries + + def __len__(self): + return len(self._entries) + + def lookup(self, word): + """Returns list of ARPAbet pronunciations of the given word.""" + return self._entries.get(word.upper()) + + def _get_pronunciation(self, s): + parts = s.strip().split(" ") + for part in parts: + if part not in self._valid_symbol_set: + return None + return " ".join(parts) + + def _parse_cmudict(self, file): + _alt_re = re.compile(r"\([0-9]+\)") + + cmudict = {} + for line in file: + if len(line) and (line[0] >= "A" and line[0] <= "Z" or line[0] == "'"): + parts = line.split(" ") + word = re.sub(_alt_re, "", parts[0]) + pronunciation = self._get_pronunciation(parts[1]) + if pronunciation: + if word in cmudict: + cmudict[word].append(pronunciation) + else: + cmudict[word] = [pronunciation] + return cmudict + + +class GlowTTSParser: + def __init__(self, cmu_dict_path=None): + """ + Parser for the glow tts model. + Converts characters to phonemes where possible. + Args: + cmu_dict_path (str): Path to cmu dictionary + """ + + self.cmu_dict = None + + if cmu_dict_path: + self.cmu_dict = CMUDict(cmu_dict_path) + + _pad = "_" + _punctuation = "!'(),.:;? " + _special = "-" + _letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + self.symbols = [_pad] + list(_special) + list(_punctuation) + list(_letters) + + if self.cmu_dict: + # Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters): + _arpabet = ["@" + s for s in self.cmu_dict.valid_symbols] + + self.symbols = self.symbols + _arpabet + + # Mappings from symbol to numeric ID and vice versa: + self._symbol_to_id = {s: i for i, s in enumerate(self.symbols)} + self._id_to_symbol = {i: s for i, s in enumerate(self.symbols)} + + # Regular expression matching text enclosed in curly braces: + self._curly_re = re.compile(r"(.*?)\{(.+?)\}(.*)") + + # Regular expression matching whitespace: + self._whitespace_re = re.compile(r"\s+") + + # List of (regular expression, replacement) pairs for abbreviations: + self._abbreviations = [ + (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1]) + for x in [ + ("mrs", "missus"), + ("mr", "mister"), + ("dr", "doctor"), + ("st", "saint"), + ("co", "company"), + ("jr", "junior"), + ("maj", "major"), + ("gen", "general"), + ("drs", "doctors"), + ("rev", "reverend"), + ("lt", "lieutenant"), + ("hon", "honorable"), + ("sgt", "sergeant"), + ("capt", "captain"), + ("esq", "esquire"), + ("ltd", "limited"), + ("col", "colonel"), + ("ft", "fort"), + ] + ] + + # List of other replacement pairs: + self._replace = [ + (re.compile("\\b%s\\b" % x[0], re.IGNORECASE), x[1]) + for x in [ + ("_", "underscore"), + ("src", "source"), + ("dll", "d l l"), + ("btw", "by the way"), + ("http", "h t t p"), + ("www", "w w w"), + ("c\+\+", "c plus plus"), + ] + ] + + self._inflect = inflect.engine() + self._comma_number_re = re.compile(r"([0-9][0-9,]+[0-9])") + self._decimal_number_re = re.compile(r"([0-9]+\.[0-9]+)") + self._pounds_re = re.compile(r"£([0-9,]*[0-9]+)") + self._dollars_re = re.compile(r"\$([0-9.,]*[0-9]+)") + self._ordinal_re = re.compile(r"[0-9]+(st|nd|rd|th)") + self._number_re = re.compile(r"[0-9]+") + + def __call__(self, text: str) -> Optional[List[int]]: + return self.text_to_sequence(text, ["english_cleaners"], self.cmu_dict) + + def _expand_ordinal(self, m): + return self._inflect.number_to_words(m.group(0)) + + def _expand_number(self, m): + num = int(m.group(0)) + if num > 1000 and num < 3000: + if num == 2000: + return "two thousand" + elif num > 2000 and num < 2010: + return "two thousand " + self._inflect.number_to_words(num % 100) + elif num % 100 == 0: + return self._inflect.number_to_words(num // 100) + " hundred" + else: + return self._inflect.number_to_words(num, andword="", zero="oh", group=2).replace(", ", " ") + else: + return self._inflect.number_to_words(num, andword="") + + def expand_numbers(self, text): + text = re.sub(self._comma_number_re, self._remove_commas, text) + text = re.sub(self._pounds_re, r"\1 pounds", text) + text = re.sub(self._dollars_re, self._expand_dollars, text) + text = re.sub(self._decimal_number_re, self._expand_decimal_point, text) + text = re.sub(self._ordinal_re, self._expand_ordinal, text) + text = " ".join(re.split('(\d+)', text)) + text = re.sub(self._number_re, self._expand_number, text) + return text + + def replace(self, text): + for regex, replacement in self._abbreviations: + text = re.sub(regex, replacement, text) + for regex, replacement in self._replace: + text = re.sub(regex, replacement, text) + return text + + def lowercase(self, text): + return text.lower() + + def collapse_whitespace(self, text): + return re.sub(self._whitespace_re, " ", text) + + def convert_to_ascii(self, text): + return unidecode(text) + + def basic_cleaners(self, text): + """Basic pipeline that lowercases and collapses whitespace without transliteration.""" + text = self.lowercase(text) + text = self.collapse_whitespace(text) + return text + + def transliteration_cleaners(self, text): + """Pipeline for non-English text that transliterates to ASCII.""" + text = self.convert_to_ascii(text) + text = self.lowercase(text) + text = self.collapse_whitespace(text) + return text + + def english_cleaners(self, text): + """Pipeline for English text, including number and abbreviation expansion.""" + text = self.convert_to_ascii(text) + text = self.lowercase(text) + text = self.expand_numbers(text) + text = self.replace(text) + text = self.collapse_whitespace(text) + return text + + def get_arpabet(self, word, dictionary): + word_arpabet = dictionary.lookup(word) + if word_arpabet is not None: + return "{" + word_arpabet[0] + "}" + else: + return word + + def text_to_sequence(self, text, cleaner_names, dictionary=None): + """Converts a string of text to a sequence of IDs corresponding to the symbols in the text. + + The text can optionally have ARPAbet sequences enclosed in curly braces embedded + in it. For example, "Turn left on {HH AW1 S S T AH0 N} Street." + + Args: + text: string to convert to a sequence + cleaner_names: names of the cleaner functions to run the text through + dictionary: arpabet class with arpabet dictionary + + Returns: + List of integers corresponding to the symbols in the text + """ + + sequence = [] + + space = self._symbols_to_sequence(" ") + # Check for curly braces and treat their contents as ARPAbet: + while len(text): + m = self._curly_re.match(text) + if not m: + clean_text = self._clean_text(text, cleaner_names) + if dictionary is not None: + clean_text = [self.get_arpabet(w, dictionary) for w in re.findall(r"[\w']+|[.,!?;]", clean_text)] + for i in range(len(clean_text)): + t = clean_text[i] + if t.startswith("{"): + sequence += self._arpabet_to_sequence(t[1:-1]) + else: + sequence += self._symbols_to_sequence(t) + sequence += space + else: + sequence += self._symbols_to_sequence(clean_text) + break + sequence += self._symbols_to_sequence(self._clean_text(m.group(1), cleaner_names)) + sequence += self._arpabet_to_sequence(m.group(2)) + text = m.group(3) + + # remove trailing space + if dictionary is not None: + sequence = sequence[:-1] if sequence[-1] == space[0] else sequence + + return sequence + + def sequence_to_text(self, sequence): + """Converts a sequence of IDs back to a string""" + result = "" + for symbol_id in sequence: + if symbol_id in self._id_to_symbol: + s = self._id_to_symbol[symbol_id] + # Enclose ARPAbet back in curly braces: + if len(s) > 1 and s[0] == "@": + s = "{%s}" % s[1:] + result += s + return result.replace("}{", " ") + + def _clean_text(self, text, cleaner_names): + for name in cleaner_names: + cleaner = getattr(self, name, "english_cleaners") + if not cleaner: + raise Exception("Unknown cleaner: %s" % name) + text = cleaner(text) + return text + + def _symbols_to_sequence(self, symbols): + return [self._symbol_to_id[s] for s in symbols if self._should_keep_symbol(s)] + + def _arpabet_to_sequence(self, text): + return self._symbols_to_sequence(["@" + s for s in text.split()]) + + def _should_keep_symbol(self, s): + return s in self._symbol_to_id and s is not "_" and s is not "~" + + def _remove_commas(self, m): + return m.group(1).replace(",", "") + + def _expand_decimal_point(self, m): + return m.group(1).replace(".", " point ") + + def _expand_dollars(self, m): + match = m.group(1) + parts = match.split(".") + if len(parts) > 2: + return match + " dollars" # Unexpected format + dollars = int(parts[0]) if parts[0] else 0 + cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0 + if dollars and cents: + dollar_unit = "dollar" if dollars == 1 else "dollars" + cent_unit = "cent" if cents == 1 else "cents" + return "%s %s, %s %s" % (dollars, dollar_unit, cents, cent_unit) + elif dollars: + dollar_unit = "dollar" if dollars == 1 else "dollars" + return "%s %s" % (dollars, dollar_unit) + elif cents: + cent_unit = "cent" if cents == 1 else "cents" + return "%s %s" % (cents, cent_unit) + else: + return "zero dollars" diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts_submodules.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts_submodules.py new file mode 100644 index 0000000000..a981722f1a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/glow_tts_submodules.py @@ -0,0 +1,645 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MIT License +# +# Copyright (c) 2020 Jaehyeon Kim +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import math + +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + + +def convert_pad_shape(pad_shape): + """ + Used to get arguments for F.pad + """ + l = pad_shape[::-1] + pad_shape = [item for sublist in l for item in sublist] + return pad_shape + + +def sequence_mask(length, max_length=None): + """ + Get masks for given lengths + """ + if max_length is None: + max_length = length.max() + x = torch.arange(max_length, dtype=length.dtype, device=length.device) + return x.unsqueeze(0) < length.unsqueeze(1) + + +def maximum_path(value, mask, max_neg_val=None): + """ + Monotonic alignment search algorithm + Numpy-friendly version. It's about 4 times faster than torch version. + value: [b, t_x, t_y] + mask: [b, t_x, t_y] + """ + if max_neg_val is None: + max_neg_val = -np.inf # Patch for Sphinx complaint + value = value * mask + + device = value.device + dtype = value.dtype + value = value.cpu().detach().numpy() + mask = mask.cpu().detach().numpy().astype(np.bool) + + b, t_x, t_y = value.shape + direction = np.zeros(value.shape, dtype=np.int64) + v = np.zeros((b, t_x), dtype=np.float32) + x_range = np.arange(t_x, dtype=np.float32).reshape(1, -1) + for j in range(t_y): + v0 = np.pad(v, [[0, 0], [1, 0]], mode="constant", constant_values=max_neg_val)[:, :-1] + v1 = v + max_mask = v1 >= v0 + v_max = np.where(max_mask, v1, v0) + direction[:, :, j] = max_mask + + index_mask = x_range <= j + v = np.where(index_mask, v_max + value[:, :, j], max_neg_val) + direction = np.where(mask, direction, 1) + + path = np.zeros(value.shape, dtype=np.float32) + index = mask[:, :, 0].sum(1).astype(np.int64) - 1 + index_range = np.arange(b) + for j in reversed(range(t_y)): + path[index_range, index, j] = 1 + index = index + direction[index_range, index, j] - 1 + path = path * mask.astype(np.float32) + path = torch.from_numpy(path).to(device=device, dtype=dtype) + return path + + +def generate_path(duration, mask): + """ + Generate alignment based on predicted durations + duration: [b, t_x] + mask: [b, t_x, t_y] + """ + + b, t_x, t_y = mask.shape + cum_duration = torch.cumsum(duration, 1) + + cum_duration_flat = cum_duration.view(b * t_x) + path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype) + path = path.view(b, t_x, t_y) + path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1] + path = path * mask + return path + + +@torch.jit.script +def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): + n_channels_int = n_channels[0] + in_act = input_a + input_b + t_act = torch.tanh(in_act[:, :n_channels_int, :]) + s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) + acts = t_act * s_act + return acts + + +class LayerNorm(nn.Module): + def __init__(self, channels, eps=1e-4): + super().__init__() + self.channels = channels + self.eps = eps + + self.gamma = nn.Parameter(torch.ones(channels)) + self.beta = nn.Parameter(torch.zeros(channels)) + + def forward(self, x): + n_dims = len(x.shape) + mean = torch.mean(x, 1, keepdim=True) + variance = torch.mean((x - mean) ** 2, 1, keepdim=True) + + x = (x - mean) * torch.rsqrt(variance + self.eps) + + shape = [1, -1] + [1] * (n_dims - 2) + x = x * self.gamma.view(*shape) + self.beta.view(*shape) + return x + + +class ConvReluNorm(nn.Module): + def __init__( + self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout, + ): + super().__init__() + self.in_channels = in_channels + self.hidden_channels = hidden_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.n_layers = n_layers + self.p_dropout = p_dropout + assert n_layers > 1, "Number of layers should be larger than 0." + + self.conv_layers = nn.ModuleList() + self.norm_layers = nn.ModuleList() + self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size // 2)) + self.norm_layers.append(LayerNorm(hidden_channels)) + self.relu_drop = nn.Sequential(nn.ReLU(), nn.Dropout(p_dropout)) + for _ in range(n_layers - 1): + self.conv_layers.append( + nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size // 2,) + ) + self.norm_layers.append(LayerNorm(hidden_channels)) + self.proj = nn.Conv1d(hidden_channels, out_channels, 1) + self.proj.weight.data.zero_() + self.proj.bias.data.zero_() + + def forward(self, x, x_mask): + x_org = x + for i in range(self.n_layers): + x = self.conv_layers[i](x * x_mask) + x = self.norm_layers[i](x) + x = self.relu_drop(x) + x = x_org + self.proj(x) + return x * x_mask + + +class WN(torch.nn.Module): + def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=0, gin_channels=0): + super(WN, self).__init__() + assert kernel_size % 2 == 1 + assert hidden_channels % 2 == 0 + + self.hidden_channels = hidden_channels + self.n_layers = n_layers + + self.in_layers = torch.nn.ModuleList() + self.res_skip_layers = torch.nn.ModuleList() + self.drop = nn.Dropout(p_dropout) + + if gin_channels != 0: + cond_layer = torch.nn.Conv1d(gin_channels, 2 * hidden_channels * n_layers, 1) + self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') + + for i in range(n_layers): + dilation = dilation_rate ** i + padding = int((kernel_size * dilation - dilation) / 2) + in_layer = torch.nn.Conv1d( + hidden_channels, 2 * hidden_channels, kernel_size, dilation=dilation, padding=padding, + ) + in_layer = torch.nn.utils.weight_norm(in_layer, name="weight") + self.in_layers.append(in_layer) + + # last one is not necessary + if i < n_layers - 1: + res_skip_channels = 2 * hidden_channels + else: + res_skip_channels = hidden_channels + + res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1) + res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name="weight") + self.res_skip_layers.append(res_skip_layer) + + def forward(self, x, x_mask=None, g=None, **kwargs): + output = torch.zeros_like(x) + n_channels_tensor = torch.IntTensor([self.hidden_channels]) + + if g is not None: + g = self.cond_layer(g) + + for i in range(self.n_layers): + x_in = self.in_layers[i](x) + x_in = self.drop(x_in) + if g is not None: + cond_offset = i * 2 * self.hidden_channels + g_l = g[:, cond_offset : cond_offset + 2 * self.hidden_channels, :] + else: + g_l = torch.zeros_like(x_in) + + acts = fused_add_tanh_sigmoid_multiply(x_in, g_l, n_channels_tensor) + + res_skip_acts = self.res_skip_layers[i](acts) + if i < self.n_layers - 1: + x = (x + res_skip_acts[:, : self.hidden_channels, :]) * x_mask + output = output + res_skip_acts[:, self.hidden_channels :, :] + else: + output = output + res_skip_acts + return output * x_mask + + def remove_weight_norm(self): + for l in self.in_layers: + torch.nn.utils.remove_weight_norm(l) + for l in self.res_skip_layers: + torch.nn.utils.remove_weight_norm(l) + + +class ActNorm(nn.Module): + def __init__(self, channels, ddi=False, **kwargs): + super().__init__() + self.channels = channels + self.initialized = not ddi + + self.logs = nn.Parameter(torch.zeros(1, channels, 1)) + self.bias = nn.Parameter(torch.zeros(1, channels, 1)) + + def forward(self, x, x_mask=None, reverse=False, **kwargs): + if x_mask is None: + x_mask = torch.ones(x.size(0), 1, x.size(2)).to(device=x.device, dtype=x.dtype) + x_len = torch.sum(x_mask, [1, 2]) + if not self.initialized: + self.initialize(x, x_mask) + self.initialized = True + + if reverse: + z = (x - self.bias) * torch.exp(-self.logs) * x_mask + logdet = None + else: + z = (self.bias + torch.exp(self.logs) * x) * x_mask + logdet = torch.sum(self.logs) * x_len # [b] + + return z, logdet + + def store_inverse(self): + pass + + def set_ddi(self, ddi): + self.initialized = not ddi + + def initialize(self, x, x_mask): + with torch.no_grad(): + denom = torch.sum(x_mask, [0, 2]) + m = torch.sum(x * x_mask, [0, 2]) / denom + m_sq = torch.sum(x * x * x_mask, [0, 2]) / denom + v = m_sq - (m ** 2) + logs = 0.5 * torch.log(torch.clamp_min(v, 1e-6)) + + bias_init = (-m * torch.exp(-logs)).view(*self.bias.shape).to(dtype=self.bias.dtype) + logs_init = (-logs).view(*self.logs.shape).to(dtype=self.logs.dtype) + + self.bias.data.copy_(bias_init) + self.logs.data.copy_(logs_init) + + +class InvConvNear(nn.Module): + def __init__(self, channels, n_split=4, no_jacobian=False, **kwargs): + super().__init__() + assert n_split % 2 == 0 + self.channels = channels + self.n_split = n_split + self.no_jacobian = no_jacobian + + w_init = torch.qr(torch.FloatTensor(self.n_split, self.n_split).normal_())[0] + if torch.det(w_init) < 0: + w_init[:, 0] = -1 * w_init[:, 0] + self.weight = nn.Parameter(w_init) + + def forward(self, x, x_mask=None, reverse=False, **kwargs): + b, c, t = x.size() + assert c % self.n_split == 0 + if x_mask is None: + x_mask = 1 + x_len = torch.ones((b,), dtype=x.dtype, device=x.device) * t + else: + x_len = torch.sum(x_mask, [1, 2]) + + x = x.view(b, 2, c // self.n_split, self.n_split // 2, t) + x = x.permute(0, 1, 3, 2, 4).contiguous().view(b, self.n_split, c // self.n_split, t) + + if reverse: + if hasattr(self, "weight_inv"): + weight = self.weight_inv + else: + weight = torch.inverse(self.weight.float()).to(dtype=self.weight.dtype) + logdet = None + else: + weight = self.weight + if self.no_jacobian: + logdet = 0 + else: + logdet = torch.logdet(self.weight) * (c / self.n_split) * x_len # [b] + + weight = weight.view(self.n_split, self.n_split, 1, 1) + z = F.conv2d(x, weight) + + z = z.view(b, 2, self.n_split // 2, c // self.n_split, t) + z = z.permute(0, 1, 3, 2, 4).contiguous().view(b, c, t) * x_mask + return z, logdet + + def store_inverse(self): + self.weight_inv = torch.inverse(self.weight.float()).to(dtype=self.weight.dtype) + + +class CouplingBlock(nn.Module): + def __init__( + self, + in_channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=0, + p_dropout=0, + sigmoid_scale=False, + ): + super().__init__() + self.in_channels = in_channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.gin_channels = gin_channels + self.p_dropout = p_dropout + self.sigmoid_scale = sigmoid_scale + + start = torch.nn.Conv1d(in_channels // 2, hidden_channels, 1) + start = torch.nn.utils.weight_norm(start) + self.start = start + # Initializing last layer to 0 makes the affine coupling layers + # do nothing at first. This helps with training stability + end = torch.nn.Conv1d(hidden_channels, in_channels, 1) + end.weight.data.zero_() + end.bias.data.zero_() + self.end = end + + self.wn = WN(hidden_channels, kernel_size, dilation_rate, n_layers) + + def forward(self, x, x_mask=None, reverse=False, g=None, **kwargs): + if x_mask is None: + x_mask = 1 + x_0, x_1 = x[:, : self.in_channels // 2], x[:, self.in_channels // 2 :] + + x = self.start(x_0) * x_mask + x = self.wn(x, x_mask, g) + out = self.end(x) + + z_0 = x_0 + m = out[:, : self.in_channels // 2, :] + logs = out[:, self.in_channels // 2 :, :] + if self.sigmoid_scale: + logs = torch.log(1e-6 + torch.sigmoid(logs + 2)) + + if reverse: + z_1 = (x_1 - m) * torch.exp(-logs) * x_mask + logdet = None + else: + z_1 = (m + torch.exp(logs) * x_1) * x_mask + logdet = torch.sum(logs * x_mask, [1, 2]) + + z = torch.cat([z_0, z_1], 1) + return z, logdet + + def store_inverse(self): + self.wn.remove_weight_norm() + + +class AttentionBlock(nn.Module): + def __init__( + self, + channels, + out_channels, + n_heads, + window_size=None, + heads_share=True, + p_dropout=0.0, + block_length=None, + proximal_bias=False, + proximal_init=False, + ): + super().__init__() + assert channels % n_heads == 0 + + self.channels = channels + self.out_channels = out_channels + self.n_heads = n_heads + self.window_size = window_size + self.heads_share = heads_share + self.block_length = block_length + self.proximal_bias = proximal_bias + self.p_dropout = p_dropout + self.attn = None + + self.k_channels = channels // n_heads + self.conv_q = nn.Conv1d(channels, channels, 1) + self.conv_k = nn.Conv1d(channels, channels, 1) + self.conv_v = nn.Conv1d(channels, channels, 1) + if window_size is not None: + n_heads_rel = 1 if heads_share else n_heads + rel_stddev = self.k_channels ** -0.5 + self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev) + self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev) + self.conv_o = nn.Conv1d(channels, out_channels, 1) + self.drop = nn.Dropout(p_dropout) + + nn.init.xavier_uniform_(self.conv_q.weight) + nn.init.xavier_uniform_(self.conv_k.weight) + if proximal_init: + self.conv_k.weight.data.copy_(self.conv_q.weight.data) + self.conv_k.bias.data.copy_(self.conv_q.bias.data) + nn.init.xavier_uniform_(self.conv_v.weight) + + def forward(self, x, c, attn_mask=None): + q = self.conv_q(x) + k = self.conv_k(c) + v = self.conv_v(c) + + x, self.attn = self.attention(q, k, v, mask=attn_mask) + + x = self.conv_o(x) + return x + + def attention(self, query, key, value, mask=None): + # reshape [b, d, t] -> [b, n_h, t, d_k] + b, d, t_s, t_t = key.size(0), key.size(1), key.size(2), query.size(2) + query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3) + key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) + value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) + + scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(self.k_channels) + if self.window_size is not None: + assert t_s == t_t, "Relative attention is only available for self-attention." + key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s) + rel_logits = self._matmul_with_relative_keys(query, key_relative_embeddings) + rel_logits = self._relative_position_to_absolute_position(rel_logits) + scores_local = rel_logits / math.sqrt(self.k_channels) + scores = scores + scores_local + if self.proximal_bias: + assert t_s == t_t, "Proximal bias is only available for self-attention." + scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype) + if mask is not None: + scores = scores.masked_fill(mask == 0, -1e4) + if self.block_length is not None: + block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length) + scores = scores * block_mask + -1e4 * (1 - block_mask) + p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s] + p_attn = self.drop(p_attn) + output = torch.matmul(p_attn, value) + if self.window_size is not None: + relative_weights = self._absolute_position_to_relative_position(p_attn) + value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s) + output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings) + output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t] + return output, p_attn + + def _matmul_with_relative_values(self, x, y): + """ + x: [b, h, l, m] + y: [h or 1, m, d] + ret: [b, h, l, d] + """ + ret = torch.matmul(x, y.unsqueeze(0)) + return ret + + def _matmul_with_relative_keys(self, x, y): + """ + x: [b, h, l, d] + y: [h or 1, m, d] + ret: [b, h, l, m] + """ + ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1)) + return ret + + def _get_relative_embeddings(self, relative_embeddings, length): + # Pad first before slice to avoid using cond ops. + pad_length = max(length - (self.window_size + 1), 0) + slice_start_position = max((self.window_size + 1) - length, 0) + slice_end_position = slice_start_position + 2 * length - 1 + if pad_length > 0: + padded_relative_embeddings = F.pad( + relative_embeddings, convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]), + ) + else: + padded_relative_embeddings = relative_embeddings + used_relative_embeddings = padded_relative_embeddings[:, slice_start_position:slice_end_position] + return used_relative_embeddings + + def _relative_position_to_absolute_position(self, x): + """ + x: [b, h, l, 2*l-1] + ret: [b, h, l, l] + """ + batch, heads, length, _ = x.size() + # Concat columns of pad to shift from relative to absolute indexing. + x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]])) + + # Concat extra elements so to add up to shape (len+1, 2*len-1). + x_flat = x.view([batch, heads, length * 2 * length]) + x_flat = F.pad(x_flat, convert_pad_shape([[0, 0], [0, 0], [0, length - 1]])) + + # Reshape and slice out the padded elements. + x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[:, :, :length, length - 1 :] + return x_final + + def _absolute_position_to_relative_position(self, x): + """ + x: [b, h, l, l] + ret: [b, h, l, 2*l-1] + """ + batch, heads, length, _ = x.size() + # padd along column + x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]])) + x_flat = x.view([batch, heads, length ** 2 + length * (length - 1)]) + # add 0's in the beginning that will skew the elements after reshape + x_flat = F.pad(x_flat, convert_pad_shape([[0, 0], [0, 0], [length, 0]])) + x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:] + return x_final + + def _attention_bias_proximal(self, length): + """Bias for self-attention to encourage attention to close positions. + Args: + length: an integer scalar. + Returns: + a Tensor with shape [1, 1, length, length] + """ + r = torch.arange(length, dtype=torch.float32) + diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1) + return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0) + + +class FeedForwardNetwork(nn.Module): + def __init__( + self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0.0, activation=None, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.filter_channels = filter_channels + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.activation = activation + + self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size, padding=kernel_size // 2) + self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size, padding=kernel_size // 2) + self.drop = nn.Dropout(p_dropout) + + def forward(self, x, x_mask): + x = self.conv_1(x * x_mask) + if self.activation == "gelu": + x = x * torch.sigmoid(1.702 * x) + else: + x = torch.relu(x) + x = self.drop(x) + x = self.conv_2(x * x_mask) + return x * x_mask + + +class DurationPredictor(nn.Module): + def __init__(self, in_channels, filter_channels, kernel_size, p_dropout): + """ + Token duration predictor for the GlowTTS model. + Takes in embeddings of the input tokens and predicts how many frames of + mel-spectrogram are aligned to each text token. + Architecture is the same as the duration predictor in FastSpeech. + + Args: + in_channels: Number of channels for the token embeddings + filter_channels: Number of channels in the intermediate layers + kernel_size: Kernels size for the convolution layers + p_dropout: Dropout probability + """ + + super().__init__() + + self.drop = nn.Dropout(p_dropout) + self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size, padding=kernel_size // 2) + self.norm_1 = LayerNorm(filter_channels) + self.conv_2 = nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size // 2) + self.norm_2 = LayerNorm(filter_channels) + self.proj = nn.Conv1d(filter_channels, 1, 1) + + def forward(self, spect, mask): + x = self.conv_1(spect * mask) + x = torch.relu(x) + x = self.norm_1(x) + x = self.drop(x) + x = self.conv_2(x * mask) + x = torch.relu(x) + x = self.norm_2(x) + x = self.drop(x) + x = self.proj(x * mask) + durs = x * mask + return durs.squeeze(1) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/hifigan_modules.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/hifigan_modules.py new file mode 100644 index 0000000000..11874e4582 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/hifigan_modules.py @@ -0,0 +1,447 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MIT License +# +# Copyright (c) 2020 Jungil Kong +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# The following functions/classes were based on code from https://github.com/jik876/hifi-gan: +# ResBlock1, ResBlock2, Generator, DiscriminatorP, DiscriminatorS, MultiScaleDiscriminator, +# MultiPeriodDiscriminator, init_weights, get_padding + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import AvgPool1d, Conv1d, Conv2d, ConvTranspose1d +from torch.nn.utils import remove_weight_norm, spectral_norm, weight_norm + +from nemo.core.classes import NeuralModule +from nemo.core.classes.common import typecheck +from nemo.core.neural_types.elements import AudioSignal, MelSpectrogramType, VoidType +from nemo.core.neural_types.neural_type import NeuralType + +LRELU_SLOPE = 0.1 + + +def init_weights(m, mean=0.0, std=0.01): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + m.weight.data.normal_(mean, std) + + +def get_padding(kernel_size, dilation=1): + return int((kernel_size * dilation - dilation) / 2) + + +class ResBlock1(torch.nn.Module): + __constants__ = ['lrelu_slope'] + + def __init__(self, channels, kernel_size, dilation): + super().__init__() + self.lrelu_slope = LRELU_SLOPE + self.convs1 = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[2], + padding=get_padding(kernel_size, dilation[2]), + ) + ), + ] + ) + self.convs1.apply(init_weights) + + self.convs2 = nn.ModuleList( + [ + weight_norm( + Conv1d(channels, channels, kernel_size, 1, dilation=1, padding=get_padding(kernel_size, 1)) + ), + weight_norm( + Conv1d(channels, channels, kernel_size, 1, dilation=1, padding=get_padding(kernel_size, 1)) + ), + weight_norm( + Conv1d(channels, channels, kernel_size, 1, dilation=1, padding=get_padding(kernel_size, 1)) + ), + ] + ) + self.convs2.apply(init_weights) + + def forward(self, x): + for c1, c2 in zip(self.convs1, self.convs2): + xt = F.leaky_relu(x, self.lrelu_slope) + xt = c1(xt) + xt = F.leaky_relu(xt, self.lrelu_slope) + xt = c2(xt) + x = xt + x + return x + + def remove_weight_norm(self): + for l in self.convs1: + remove_weight_norm(l) + for l in self.convs2: + remove_weight_norm(l) + + +class ResBlock2(torch.nn.Module): + __constants__ = ['lrelu_slope'] + + def __init__(self, channels, kernel_size, dilation): + super().__init__() + self.convs = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]), + ) + ), + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]), + ) + ), + ] + ) + self.convs.apply(init_weights) + self.lrelu_slope = LRELU_SLOPE + + def forward(self, x): + for c in self.convs: + xt = F.leaky_relu(x, self.lrelu_slope) + xt = c(xt) + x = xt + x + return x + + def remove_weight_norm(self): + for l in self.convs: + remove_weight_norm(l) + + +class Generator(NeuralModule): + __constants__ = ['lrelu_slope', 'num_kernels', 'num_upsamples'] + + def __init__( + self, + resblock, + upsample_rates, + upsample_kernel_sizes, + upsample_initial_channel, + resblock_kernel_sizes, + resblock_dilation_sizes, + initial_input_size=80, + apply_weight_init_conv_pre=False, + ): + super().__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + self.conv_pre = weight_norm(Conv1d(initial_input_size, upsample_initial_channel, 7, 1, padding=3)) + self.lrelu_slope = LRELU_SLOPE + resblock = ResBlock1 if resblock == 1 else ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + self.ups.append( + weight_norm( + ConvTranspose1d( + upsample_initial_channel // (2 ** i), + upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ) + ) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + resblock_list = nn.ModuleList() + ch = upsample_initial_channel // (2 ** (i + 1)) + for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)): + resblock_list.append(resblock(ch, k, d)) + self.resblocks.append(resblock_list) + + self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3)) + self.ups.apply(init_weights) + self.conv_post.apply(init_weights) + if apply_weight_init_conv_pre: + self.conv_pre.apply(init_weights) + + @property + def input_types(self): + return { + "x": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + } + + @property + def output_types(self): + return { + "audio": NeuralType(('B', 'S', 'T'), AudioSignal()), + } + + @typecheck() + def forward(self, x): + x = self.conv_pre(x) + for upsample_layer, resblock_group in zip(self.ups, self.resblocks): + x = F.leaky_relu(x, self.lrelu_slope) + x = upsample_layer(x) + xs = torch.zeros(x.shape, dtype=x.dtype, device=x.device) + for resblock in resblock_group: + xs += resblock(x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + + return x + + def remove_weight_norm(self): + print('Removing weight norm...') + for l in self.ups: + remove_weight_norm(l) + for group in self.resblocks: + for block in group: + block.remove_weight_norm() + remove_weight_norm(self.conv_pre) + remove_weight_norm(self.conv_post) + + +class DiscriminatorP(NeuralModule): + def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False): + super().__init__() + self.period = period + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))), + norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(2, 0))), + ] + ) + self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0))) + + @property + def input_types(self): + return { + "x": NeuralType(('B', 'S', 'T'), AudioSignal()), + } + + @property + def output_types(self): + return { + "decision": NeuralType(('B', 'T'), VoidType()), + "feature_maps": [NeuralType(("B", "C", "H", "W"), VoidType())], + } + + @typecheck() + def forward(self, x): + fmap = [] + + # 1d to 2d + b, c, t = x.shape + if t % self.period != 0: # pad first + n_pad = self.period - (t % self.period) + x = F.pad(x, (0, n_pad), "reflect") + t = t + n_pad + x = x.view(b, c, t // self.period, self.period) + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class MultiPeriodDiscriminator(NeuralModule): + def __init__(self): + super().__init__() + self.discriminators = nn.ModuleList( + [DiscriminatorP(2), DiscriminatorP(3), DiscriminatorP(5), DiscriminatorP(7), DiscriminatorP(11),] + ) + + @property + def output_types(self): + return { + "y": NeuralType(('B', 'S', 'T'), AudioSignal()), + "y_hat": NeuralType(('B', 'S', 'T'), AudioSignal()), + } + + @property + def output_types(self): + return { + "real_scores": [NeuralType(('B', 'T'), VoidType())], + "fake_scores": [NeuralType(('B', 'T'), VoidType())], + "real_feature_maps": [[NeuralType(("B", "C", "H", "W"), VoidType())]], + "fake_feature_maps": [[NeuralType(("B", "C", "H", "W"), VoidType())]], + } + + @typecheck() + def forward(self, y, y_hat): + y_d_rs = [] + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(x=y) + y_d_g, fmap_g = d(x=y_hat) + y_d_rs.append(y_d_r) + fmap_rs.append(fmap_r) + y_d_gs.append(y_d_g) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class DiscriminatorS(NeuralModule): + def __init__(self, use_spectral_norm=False): + super().__init__() + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList( + [ + norm_f(Conv1d(1, 128, 15, 1, padding=7)), + norm_f(Conv1d(128, 128, 41, 2, groups=4, padding=20)), + norm_f(Conv1d(128, 256, 41, 2, groups=16, padding=20)), + norm_f(Conv1d(256, 512, 41, 4, groups=16, padding=20)), + norm_f(Conv1d(512, 1024, 41, 4, groups=16, padding=20)), + norm_f(Conv1d(1024, 1024, 41, 1, groups=16, padding=20)), + norm_f(Conv1d(1024, 1024, 5, 1, padding=2)), + ] + ) + self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1)) + + @property + def input_types(self): + return { + "x": NeuralType(('B', 'S', 'T'), AudioSignal()), + } + + @property + def output_types(self): + return { + "decision": NeuralType(('B', 'T'), VoidType()), + "feature_maps": [NeuralType(("B", "C", "T"), VoidType())], + } + + @typecheck() + def forward(self, x): + fmap = [] + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class MultiScaleDiscriminator(NeuralModule): + def __init__(self): + super().__init__() + self.discriminators = nn.ModuleList( + [DiscriminatorS(use_spectral_norm=True), DiscriminatorS(), DiscriminatorS(),] + ) + self.meanpools = nn.ModuleList([AvgPool1d(4, 2, padding=2), AvgPool1d(4, 2, padding=2)]) + + @property + def output_types(self): + return { + "y": NeuralType(('B', 'S', 'T'), AudioSignal()), + "y_hat": NeuralType(('B', 'S', 'T'), AudioSignal()), + } + + @property + def output_types(self): + return { + "real_scores": [NeuralType(('B', 'T'), VoidType())], + "fake_scores": [NeuralType(('B', 'T'), VoidType())], + "real_feature_maps": [[NeuralType(("B", "C", "T"), VoidType())]], + "fake_feature_maps": [[NeuralType(("B", "C", "T"), VoidType())]], + } + + @typecheck() + def forward(self, y, y_hat): + y_d_rs = [] + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + if i != 0: + y = self.meanpools[i - 1](y) + y_hat = self.meanpools[i - 1](y_hat) + y_d_r, fmap_r = d(x=y) + y_d_g, fmap_g = d(x=y_hat) + y_d_rs.append(y_d_r) + fmap_rs.append(fmap_r) + y_d_gs.append(y_d_g) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/melgan_modules.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/melgan_modules.py new file mode 100644 index 0000000000..6bed81c8bb --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/melgan_modules.py @@ -0,0 +1,508 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The MIT License +# +# Copyright (c) 2019 Tomoki Hayashi +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +# The following functions/classes were based on code from https://github.com/kan-bayashi/ParallelWaveGAN/: +# ResidualStack +# MelGANGenerator +# MelGANDiscriminator +# MelGANMultiScaleDiscriminator + +from typing import List + +import numpy as np +import torch + +from nemo.core.classes import NeuralModule +from nemo.core.classes.common import typecheck +from nemo.core.neural_types.elements import AudioSignal, MelSpectrogramType, VoidType +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + +__all__ = ['MelGANGenerator', 'MelGANDiscriminator', 'MelGANMultiScaleDiscriminator'] + + +class ResidualStack(torch.nn.Module): + """Residual stack module introduced in MelGAN.""" + + def __init__( + self, + nonlinear_activation: torch.nn.Module, + kernel_size: int, + channels: int, + dilation: int = 1, + bias: bool = True, + ): + """Initialize ResidualStack module. + Args: + kernel_size (int): Kernel size of dilation convolution layer. + channels (int): Number of channels of convolution layers. + dilation (int): Dilation factor. + bias (bool): Whether to add bias parameter in convolution layers. + nonlinear_activation (torch.nn.Module): Activation function. + """ + super().__init__() + + # defile residual stack part + assert (kernel_size - 1) % 2 == 0, "Not support even number kernel size." + self.stack = torch.nn.Sequential( + nonlinear_activation, + torch.nn.ReflectionPad1d((kernel_size - 1) // 2 * dilation), + torch.nn.Conv1d(channels, channels, kernel_size, dilation=dilation, bias=bias), + nonlinear_activation, + torch.nn.Conv1d(channels, channels, 1, bias=bias), + ) + + # defile extra layer for skip connection + self.skip_layer = torch.nn.Conv1d(channels, channels, 1, bias=bias) + + def forward(self, x): + """Calculate forward propagation. + Args: + c (Tensor): Input tensor (B, channels, T). + Returns: + Tensor: Output tensor (B, chennels, T). + """ + return self.stack(x) + self.skip_layer(x) + + +class MelGANGenerator(NeuralModule): + """MelGAN generator module.""" + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + channels: int, + upsample_scales: List[int], + stack_kernel_size: int, + stacks: int, + nonlinear_activation: torch.nn.Module = None, + bias: bool = True, + use_final_nonlinear_activation: bool = True, + use_weight_norm: bool = True, + ): + """Initialize MelGANGenerator module. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + kernel_size (int): Kernel size of initial and final conv layer. + channels (int): Initial number of channels for conv layer. + bias (bool): Whether to add bias parameter in convolution layers. + upsample_scales (list): List of upsampling scales. + stack_kernel_size (int): Kernel size of dilated conv layers in residual stack. + stacks (int): Number of stacks in a single residual stack. + nonlinear_activation (torch.nn.Module): Activation function. + use_final_nonlinear_activation (bool): Whether to add an activation function for the final layer. + use_weight_norm (bool): Whether to use weight norm. + If set to true, it will be applied to all of the conv layers. + """ + super().__init__() + + # check hyper parameters is valid + assert channels >= np.prod(upsample_scales) + assert channels % (2 ** len(upsample_scales)) == 0 + assert (kernel_size - 1) % 2 == 0, "Not support even number kernel size." + + if nonlinear_activation is None: + nonlinear_activation = torch.nn.LeakyReLU(negative_slope=0.2) + + # add initial layer + layers = [] + layers += [ + torch.nn.ReflectionPad1d((kernel_size - 1) // 2), + torch.nn.Conv1d(in_channels, channels, kernel_size, bias=bias), + ] + + for i, upsample_scale in enumerate(upsample_scales): + # add upsampling layer + layers += [nonlinear_activation] + layers += [ + torch.nn.ConvTranspose1d( + channels // (2 ** i), + channels // (2 ** (i + 1)), + upsample_scale * 2, + stride=upsample_scale, + padding=upsample_scale // 2 + upsample_scale % 2, + output_padding=upsample_scale % 2, + bias=bias, + ) + ] + + # add residual stack + for j in range(stacks): + layers += [ + ResidualStack( + kernel_size=stack_kernel_size, + channels=channels // (2 ** (i + 1)), + dilation=stack_kernel_size ** j, + bias=bias, + nonlinear_activation=nonlinear_activation, + ) + ] + + # add final layer + layers += [nonlinear_activation] + layers += [ + torch.nn.ReflectionPad1d((kernel_size - 1) // 2), + torch.nn.Conv1d(channels // (2 ** len(upsample_scales)), out_channels, kernel_size, bias=bias), + ] + + if use_final_nonlinear_activation: + layers += [torch.nn.Tanh()] + + # define the model as a single function + self.melgan = torch.nn.Sequential(*layers) + + # apply weight norm + if use_weight_norm: + self.apply_weight_norm() + + # reset parameters + self.reset_parameters() + + @property + def input_types(self): + return { + "spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + } + + @property + def output_types(self): + return { + "audio": NeuralType(('B', 'S', 'T'), AudioSignal()), + } + + @typecheck() + def forward(self, *, spec): + """Calculate forward propagation. + + Args: + c (Tensor): Input tensor (B, channels, T). + + Returns: + Tensor: Output tensor (B, 1, T ** prod(upsample_scales)). + + """ + return self.melgan(spec) + + def remove_weight_norm(self): + """Remove weight normalization module from all of the layers.""" + + def _remove_weight_norm(module): + try: + logging.debug(f"Weight norm is removed from {module}.") + torch.nn.utils.remove_weight_norm(module) + except ValueError: # this module didn't have weight norm + return + + self.apply(_remove_weight_norm) + + def apply_weight_norm(self): + """Apply weight normalization module from all of the layers.""" + + def _apply_weight_norm(module): + if isinstance(module, torch.nn.Conv1d) or isinstance(module, torch.nn.ConvTranspose1d): + torch.nn.utils.weight_norm(module) + logging.debug(f"Weight norm is applied to {module}.") + + self.apply(_apply_weight_norm) + + def reset_parameters(self): + """Reset parameters. + + This initialization follows official implementation manner. + https://github.com/descriptinc/melgan-neurips/blob/master/mel2wav/modules.py + + """ + + def _reset_parameters(module): + if isinstance(module, torch.nn.Conv1d) or isinstance(module, torch.nn.ConvTranspose1d): + module.weight.data.normal_(0.0, 0.02) + logging.debug(f"Reset parameters in {module}.") + + self.apply(_reset_parameters) + + +class MelGANDiscriminator(NeuralModule): + """MelGAN discriminator module.""" + + def __init__( + self, + kernel_sizes: int, + channels: int, + max_downsample_channels: int, + downsample_scales: List[int], + nonlinear_activation: torch.nn.Module, + in_channels: int = 1, + out_channels: int = 1, + bias: bool = True, + ): + """Initilize MelGAN discriminator module. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + kernel_sizes (list): List of two kernel sizes. The prod will be used for the first conv layer, + and the first and the second kernel sizes will be used for the last two layers. + For example if kernel_sizes = [5, 3], the first layer kernel size will be 5 * 3 = 15, + the last two layers' kernel size will be 5 and 3, respectively. + channels (int): Initial number of channels for conv layer. + max_downsample_channels (int): Maximum number of channels for downsampling layers. + bias (bool): Whether to add bias parameter in convolution layers. + downsample_scales (list): List of downsampling scales. + nonlinear_activation (torch.nn.Module): Activation function. + """ + super().__init__() + self.layers = torch.nn.ModuleList() + + # check kernel size is valid + assert len(kernel_sizes) == 2 + assert kernel_sizes[0] % 2 == 1 + assert kernel_sizes[1] % 2 == 1 + + # add first layer + self.layers += [ + torch.nn.Sequential( + torch.nn.ReflectionPad1d((np.prod(kernel_sizes) - 1) // 2), + torch.nn.Conv1d(in_channels, channels, np.prod(kernel_sizes), bias=bias), + nonlinear_activation, + ) + ] + + # add downsample layers + in_chs = channels + for downsample_scale in downsample_scales: + out_chs = min(in_chs * downsample_scale, max_downsample_channels) + self.layers += [ + torch.nn.Sequential( + torch.nn.Conv1d( + in_chs, + out_chs, + kernel_size=downsample_scale * 10 + 1, + stride=downsample_scale, + padding=downsample_scale * 5, + groups=in_chs // 4, + bias=bias, + ), + nonlinear_activation, + ) + ] + in_chs = out_chs + + # add final layers + out_chs = min(in_chs * 2, max_downsample_channels) + self.layers += [ + torch.nn.Sequential( + torch.nn.Conv1d(in_chs, out_chs, kernel_sizes[0], padding=(kernel_sizes[0] - 1) // 2, bias=bias,), + nonlinear_activation, + ) + ] + self.layers += [ + torch.nn.Conv1d(out_chs, out_channels, kernel_sizes[1], padding=(kernel_sizes[1] - 1) // 2, bias=bias,), + ] + + @property + def input_types(self): + return { + "x": NeuralType(('B', 'S', 'T'), AudioSignal()), + } + + @property + def output_types(self): + return { + "decision": [NeuralType(('B', 'S', 'T'), VoidType())], + } + + @typecheck() + def forward(self, *, x): + """Calculate forward propagation. + + Args: + x (Tensor): Input noise signal (B, 1, T). + + Returns: + List: List of output tensors of each layer. + + """ + output = x + for layer in self.layers: + output = layer(output) + + return output + + +class MelGANMultiScaleDiscriminator(NeuralModule): + """MelGAN multi-scale discriminator module.""" + + def __init__( + self, + scales: int, + downsample_scales: List[int], + kernel_sizes: List[int], + channels: int, + nonlinear_activation: torch.nn.Module = None, + in_channels: int = 1, + out_channels: int = 1, + max_downsample_channels: int = 1024, + bias: bool = True, + use_weight_norm: bool = True, + use_spectral_norm: bool = False, + ): + """Initilize MelGAN multi-scale discriminator module. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + kernel_sizes (list): List of two kernel sizes. The sum will be used for the first conv layer, + and the first and the second kernel sizes will be used for the last two layers. + channels (int): Initial number of channels for conv layer. + max_downsample_channels (int): Maximum number of channels for downsampling layers. + bias (bool): Whether to add bias parameter in convolution layers. + downsample_scales (list): List of downsampling scales. + nonlinear_activation (torch.nn.Module): Activation function. If None, defaults to leaky ReLU + """ + super().__init__() + + if nonlinear_activation is None: + nonlinear_activation = torch.nn.LeakyReLU(negative_slope=0.2) + self.discriminators = torch.nn.ModuleList() + + # add discriminators + for _ in range(scales): + self.discriminators += [ + MelGANDiscriminator( + in_channels=in_channels, + out_channels=out_channels, + kernel_sizes=kernel_sizes, + channels=channels, + max_downsample_channels=max_downsample_channels, + bias=bias, + downsample_scales=downsample_scales, + nonlinear_activation=nonlinear_activation, + ) + ] + self.pooling = torch.nn.AvgPool1d(kernel_size=4, stride=2, padding=1, count_include_pad=False) + + if use_spectral_norm and use_weight_norm: + raise NotImplementedError + + # apply weight norm + if use_weight_norm: + self.apply_weight_norm() + # reset parameters + self.reset_parameters() + + if use_spectral_norm: + self.apply_spectral_norm() + + @property + def input_types(self): + return { + "x": NeuralType(('B', 'S', 'T'), AudioSignal()), + } + + @property + def output_types(self): + return { + "decision": [NeuralType((('B', 'S', 'T')), VoidType())], + } + + @typecheck() + def forward(self, *, x): + """Calculate forward propagation. + + Args: + x (Tensor): Input noise signal (B, 1, T). + + Returns: + List: List of list of each discriminator outputs, which consists of each layer output tensors. + + """ + outs = [] + for disc in self.discriminators: + outs += [disc(x=x)] + x = self.pooling(x) + + return (outs,) + + def remove_weight_norm(self): + """Remove weight normalization module from all of the layers.""" + + def _remove_weight_norm(module): + try: + logging.debug(f"Weight norm is removed from {module}.") + torch.nn.utils.remove_weight_norm(module) + except ValueError: # this module didn't have weight norm + return + + self.apply(_remove_weight_norm) + + def apply_weight_norm(self): + """Apply weight normalization module from all of the layers.""" + + def _apply_weight_norm(module): + if isinstance(module, torch.nn.Conv1d) or isinstance(module, torch.nn.ConvTranspose1d): + torch.nn.utils.weight_norm(module) + logging.debug(f"Weight norm is applied to {module}.") + + self.apply(_apply_weight_norm) + + def apply_spectral_norm(self): + """Apply weight normalization module from all of the layers.""" + + def _apply_spectral_norm(module): + if isinstance(module, torch.nn.Conv1d) or isinstance(module, torch.nn.ConvTranspose1d): + torch.nn.utils.spectral_norm(module) + logging.debug(f"Weight norm is applied to {module}.") + + self.apply(_apply_spectral_norm) + + def reset_parameters(self): + """Reset parameters. + + This initialization follows official implementation manner. + https://github.com/descriptinc/melgan-neurips/blob/master/mel2wav/modules.py + + """ + + def _reset_parameters(module): + if isinstance(module, torch.nn.Conv1d) or isinstance(module, torch.nn.ConvTranspose1d): + module.weight.data.normal_(0.0, 0.02) + logging.debug(f"Reset parameters in {module}.") + + self.apply(_reset_parameters) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/squeezewave.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/squeezewave.py new file mode 100644 index 0000000000..4321710e2d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/squeezewave.py @@ -0,0 +1,212 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MIT License +# +# Copyright (c) 2020 Tianren Gao, Bohan Zhai, Flora Xue, +# Daniel Rothchild, Bichen Wu, Joseph E. Gonzalez, Kurt Keutzer +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import torch + +from nemo.collections.tts.helpers.helpers import OperationMode +from nemo.collections.tts.modules.squeezewave_submodules import SqueezeWaveNet +from nemo.collections.tts.modules.submodules import Invertible1x1Conv +from nemo.core.classes import NeuralModule, typecheck +from nemo.core.neural_types.elements import ( + AudioSignal, + IntType, + MelSpectrogramType, + NormalDistributionSamplesType, + VoidType, +) +from nemo.core.neural_types.neural_type import NeuralType + + +class SqueezeWaveModule(NeuralModule): + def __init__( + self, + n_mel_channels: int, + n_flows: int, + n_group: int, + n_early_every: int, + n_early_size: int, + n_wn_channels: int, + n_wn_layers: int, + wn_kernel_size: int, + ): + """ + SqueezeWave module + + Args: + n_mel_channels (int): Number of mel channels to output. + n_flows (int): Number of flow layers + n_group (int): Number of groups to respace the inputs + n_early_every (int): Every n_early_every layers, n_early_size gets skip connected to the output + n_early_size (int): The size of the chunk to be skip connected + n_wn_channels (int): Number of channels for the non-invertible wavenet transformation + n_wn_layers (int): Number of layers for the non-invertible wavenet transformation + wn_kernel_size (int): Kernel size for the non-invertible wavenet transformation + """ + super().__init__() + + assert n_group % 2 == 0 + self.n_flows = n_flows + self.n_group = n_group + self.n_early_every = n_early_every + self.n_early_size = n_early_size + self.wavenet = torch.nn.ModuleList() + self.convinv = torch.nn.ModuleList() + self.mode = OperationMode.infer + + n_half = int(n_group / 2) + + # Set up layers with the right sizes based on how many dimensions + # have been output already + n_remaining_channels = n_group + for k in range(n_flows): + if k % self.n_early_every == 0 and k > 0: + n_half = n_half - int(self.n_early_size / 2) + n_remaining_channels = n_remaining_channels - self.n_early_size + self.convinv.append(Invertible1x1Conv(n_remaining_channels)) + self.wavenet.append( + SqueezeWaveNet( + n_half, n_mel_channels, n_layers=n_wn_layers, n_channels=n_wn_channels, kernel_size=wn_kernel_size, + ) + ) + self.n_remaining_channels = n_remaining_channels + + @typecheck() + def forward(self, *, spec, audio=None, run_inverse=True, sigma=1.0): + """ TODO + """ + if self.training and self.mode != OperationMode.training: + raise ValueError(f"{self} has self.training set to True but self.OperationMode was not set to training") + if not self.training and self.mode == OperationMode.training: + raise ValueError(f"{self} has self.training set to False but self.OperationMode was set to training") + + audio_pred = torch.zeros((1, 1)) + if audio is not None and self.mode != OperationMode.infer: + # audio_to_normal_dist is used to calculate loss so only run this in train or val model + z, log_s_list, log_det_W_list = self.audio_to_normal_dist(spec=spec, audio=audio) + if run_inverse: + # norm_dist_to_audio is used to predict audio from spectrogram so only used in val or infer mode + # Could also log train audio but currently not done + audio_pred = self.norm_dist_to_audio(spec=spec, sigma=sigma) + + # Return the necessary tensors + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + return z, log_s_list, log_det_W_list, audio_pred + return audio_pred + + @property + def input_types(self): + return { + "spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "audio": NeuralType(('B', 'T'), AudioSignal(), optional=True), + "run_inverse": NeuralType(elements_type=IntType(), optional=True), + "sigma": NeuralType(optional=True), + } + + @property + def output_types(self): + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + return { + "pred_normal_dist": NeuralType(('B', 'flowgroup', 'T'), NormalDistributionSamplesType()), + "log_s_list": [NeuralType(('B', 'flowgroup', 'T'), VoidType())], # TODO: Figure out a good typing + "log_det_W_list": [NeuralType(elements_type=VoidType())], # TODO: Figure out a good typing + "audio_pred": NeuralType(('B', 'T'), AudioSignal()), + } + else: + return { + "audio": NeuralType(('B', 'T'), AudioSignal()), + } + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + par = next(self.parameters()) + mel = torch.randn((1, self.n_mel_channels, 96), device=par.device, dtype=par.dtype) + return tuple([mel]) + + def audio_to_normal_dist(self, *, spec: torch.Tensor, audio: torch.Tensor) -> (torch.Tensor, list, list): + audio = audio.unfold(1, self.n_group, self.n_group).permute(0, 2, 1) + output_audio = [] + log_s_list = [] + log_det_W_list = [] + + for k in range(self.n_flows): + if k % self.n_early_every == 0 and k > 0: + output_audio.append(audio[:, : self.n_early_size, :]) + audio = audio[:, self.n_early_size :, :] + + audio, log_det_W = self.convinv[k](audio) + log_det_W_list.append(log_det_W) + + n_half = int(audio.size(1) / 2) + audio_0 = audio[:, :n_half, :] + audio_1 = audio[:, n_half:, :] + + output = self.wavenet[k]((audio_0, spec)) + log_s = output[:, n_half:, :] + b = output[:, :n_half, :] + audio_1 = torch.exp(log_s) * audio_1 + b + log_s_list.append(log_s) + + audio = torch.cat([audio_0, audio_1], 1) + + output_audio.append(audio) + return torch.cat(output_audio, 1), log_s_list, log_det_W_list + + def norm_dist_to_audio(self, *, spec, sigma: float = 1.0): + # Note: hard-coded 256 is hop_length for computing mel-spectrogram + l = 256 * spec.size(2) // self.n_group + audio = sigma * torch.randn(spec.size(0), self.n_remaining_channels, l, device=spec.device).to(spec.dtype) + + for k in reversed(range(self.n_flows)): + n_half = int(audio.size(1) / 2) + audio_0 = audio[:, :n_half, :] + audio_1 = audio[:, n_half:, :] + + output = self.wavenet[k]((audio_0, spec)) + s = output[:, n_half:, :] + b = output[:, :n_half, :] + audio_1 = (audio_1 - b) / torch.exp(s) + audio = torch.cat((audio_0, audio_1), 1) + + audio = self.convinv[k](audio, reverse=True) + if k % self.n_early_every == 0 and k > 0: + z = sigma * torch.randn(spec.size(0), self.n_early_size, l, device=spec.device).to(spec.dtype) + audio = torch.cat((z, audio), 1) + return audio.permute(0, 2, 1).contiguous().view(audio.size(0), -1) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/squeezewave_submodules.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/squeezewave_submodules.py new file mode 100644 index 0000000000..76e321e715 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/squeezewave_submodules.py @@ -0,0 +1,151 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MIT License +# +# Copyright (c) 2020 Tianren Gao, Bohan Zhai, Flora Xue, +# Daniel Rothchild, Bichen Wu, Joseph E. Gonzalez, Kurt Keutzer +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import torch + +from nemo.collections.tts.helpers.helpers import remove +from nemo.collections.tts.modules.submodules import fused_add_tanh_sigmoid_multiply + + +def fuse_conv_and_bn(conv, bn): + fusedconv = torch.nn.Conv1d( + conv.in_channels, + conv.out_channels, + kernel_size=conv.kernel_size, + padding=conv.padding, + bias=True, + groups=conv.groups, + ) + w_conv = conv.weight.clone().view(conv.out_channels, -1) + w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var))) + w_bn = w_bn.clone() + fusedconv.weight.data = torch.mm(w_bn, w_conv).view(fusedconv.weight.size()) + if conv.bias is not None: + b_conv = conv.bias + else: + b_conv = torch.zeros(conv.weight.size(0)) + b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps)) + b_bn = torch.unsqueeze(b_bn, 1) + bn_3 = b_bn.expand(-1, 3) + b = torch.matmul(w_conv, torch.transpose(bn_3, 0, 1))[range(b_bn.size()[0]), range(b_bn.size()[0])] + fusedconv.bias.data = b_conv + b + return fusedconv + + +def remove_batchnorm(conv_list): + new_conv_list = torch.nn.ModuleList() + for old_conv in conv_list: + depthwise = fuse_conv_and_bn(old_conv[1], old_conv[0]) + pointwise = old_conv[2] + new_conv_list.append(torch.nn.Sequential(depthwise, pointwise)) + return new_conv_list + + +def remove_weightnorm(model): + squeezewave = model + for wavenet in squeezewave.wavenet: + wavenet.start = torch.nn.utils.remove_weight_norm(wavenet.start) + wavenet.in_layers = remove_batchnorm(wavenet.in_layers) + wavenet.cond_layer = torch.nn.utils.remove_weight_norm(wavenet.cond_layer) + wavenet.res_skip_layers = remove(wavenet.res_skip_layers) + return squeezewave + + +class SqueezeWaveNet(torch.nn.Module): + """ + This is the WaveNet like layer for the affine coupling. The primary + difference from WaveNet is the convolutions need not be causal. There is + also no dilation size reset. The dilation only doubles on each layer + """ + + def __init__(self, n_in_channels, n_mel_channels, n_layers, n_channels, kernel_size): + super().__init__() + assert kernel_size % 2 == 1 + assert n_channels % 2 == 0 + self.n_layers = n_layers + self.n_channels = n_channels + self.in_layers = torch.nn.ModuleList() + self.res_skip_layers = torch.nn.ModuleList() + self.upsample = torch.nn.Upsample(scale_factor=2, mode='nearest') + + start = torch.nn.Conv1d(n_in_channels, n_channels, 1) + start = torch.nn.utils.weight_norm(start, name='weight') + self.start = start + + # Initializing last layer to 0 makes the affine coupling layers + # do nothing at first. This helps with training stability + end = torch.nn.Conv1d(n_channels, 2 * n_in_channels, 1) + end.weight.data.zero_() + end.bias.data.zero_() + self.end = end + + cond_layer = torch.nn.Conv1d(n_mel_channels, 2 * n_channels * n_layers, 1) + self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') + + padding = (kernel_size - 1) // 2 + for _ in range(n_layers): + self.in_layers.append( + torch.nn.Sequential( + torch.nn.BatchNorm1d(n_channels), + torch.nn.Conv1d(n_channels, n_channels, kernel_size, padding=padding, groups=n_channels), + torch.nn.Conv1d(n_channels, 2 * n_channels, 1), + ) + ) + + res_skip_layer = torch.nn.Conv1d(n_channels, n_channels, 1) + res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight') + self.res_skip_layers.append(res_skip_layer) + + def forward(self, forward_input): + audio, spect = forward_input + audio = self.start(audio) + n_channels_tensor = torch.IntTensor([self.n_channels]) + + spect = self.cond_layer(spect) + + for i in range(self.n_layers): + spect_offset = i * 2 * self.n_channels + cond = spect[:, spect_offset : spect_offset + 2 * self.n_channels, :] + if cond.size(2) < audio.size(2): + cond = self.upsample(cond) + + acts = fused_add_tanh_sigmoid_multiply(self.in_layers[i](audio), cond, n_channels_tensor) + + res_skip_acts = self.res_skip_layers[i](acts) + audio = audio + res_skip_acts + + return self.end(audio) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/submodules.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/submodules.py new file mode 100644 index 0000000000..715da500c2 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/submodules.py @@ -0,0 +1,297 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Tuple + +import torch +from torch.autograd import Variable +from torch.nn import functional as F + + +class LinearNorm(torch.nn.Module): + def __init__(self, in_dim, out_dim, bias=True, w_init_gain='linear'): + super().__init__() + self.linear_layer = torch.nn.Linear(in_dim, out_dim, bias=bias) + + torch.nn.init.xavier_uniform_(self.linear_layer.weight, gain=torch.nn.init.calculate_gain(w_init_gain)) + + def forward(self, x): + return self.linear_layer(x) + + +class ConvNorm(torch.nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=None, + dilation=1, + bias=True, + w_init_gain='linear', + ): + super().__init__() + if padding is None: + assert kernel_size % 2 == 1 + padding = int(dilation * (kernel_size - 1) / 2) + + self.conv = torch.nn.Conv1d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + ) + + torch.nn.init.xavier_uniform_(self.conv.weight, gain=torch.nn.init.calculate_gain(w_init_gain)) + + def forward(self, signal): + conv_signal = self.conv(signal) + return conv_signal + + +class LocationLayer(torch.nn.Module): + def __init__(self, attention_n_filters, attention_kernel_size, attention_dim): + super().__init__() + padding = int((attention_kernel_size - 1) / 2) + self.location_conv = ConvNorm( + 2, + attention_n_filters, + kernel_size=attention_kernel_size, + padding=padding, + bias=False, + stride=1, + dilation=1, + ) + self.location_dense = LinearNorm(attention_n_filters, attention_dim, bias=False, w_init_gain='tanh') + + def forward(self, attention_weights_cat): + processed_attention = self.location_conv(attention_weights_cat) + processed_attention = processed_attention.transpose(1, 2) + processed_attention = self.location_dense(processed_attention) + return processed_attention + + +class Attention(torch.nn.Module): + def __init__( + self, + attention_rnn_dim, + embedding_dim, + attention_dim, + attention_location_n_filters, + attention_location_kernel_size, + ): + super().__init__() + self.query_layer = LinearNorm(attention_rnn_dim, attention_dim, bias=False, w_init_gain='tanh') + self.memory_layer = LinearNorm(embedding_dim, attention_dim, bias=False, w_init_gain='tanh') + self.v = LinearNorm(attention_dim, 1, bias=False) + self.location_layer = LocationLayer( + attention_location_n_filters, attention_location_kernel_size, attention_dim, + ) + self.score_mask_value = -float("inf") + + def get_alignment_energies(self, query, processed_memory, attention_weights_cat): + """ + PARAMS + ------ + query: decoder output (batch, n_mel_channels * n_frames_per_step) + processed_memory: processed encoder outputs (B, T_in, attention_dim) + attention_weights_cat: cumulative and prev. att weights (B, 2, max_time) + RETURNS + ------- + alignment (batch, max_time) + """ + + processed_query = self.query_layer(query.unsqueeze(1)) + processed_attention_weights = self.location_layer(attention_weights_cat) + energies = self.v(torch.tanh(processed_query + processed_attention_weights + processed_memory)) + + energies = energies.squeeze(-1) + return energies + + def forward( + self, attention_hidden_state, memory, processed_memory, attention_weights_cat, mask, + ): + """ + PARAMS + ------ + attention_hidden_state: attention rnn last output + memory: encoder outputs + processed_memory: processed encoder outputs + attention_weights_cat: previous and cummulative attention weights + mask: binary mask for padded data + """ + alignment = self.get_alignment_energies(attention_hidden_state, processed_memory, attention_weights_cat) + + if mask is not None: + alignment.data.masked_fill_(mask, self.score_mask_value) + + attention_weights = F.softmax(alignment, dim=1) + attention_context = torch.bmm(attention_weights.unsqueeze(1), memory) + attention_context = attention_context.squeeze(1) + + return attention_context, attention_weights + + +class Prenet(torch.nn.Module): + def __init__(self, in_dim, sizes, p_dropout=0.5): + super().__init__() + in_sizes = [in_dim] + sizes[:-1] + self.p_dropout = p_dropout + self.layers = torch.nn.ModuleList( + [LinearNorm(in_size, out_size, bias=False) for (in_size, out_size) in zip(in_sizes, sizes)] + ) + + def forward(self, x, inference=False): + if inference: + for linear in self.layers: + x = F.relu(linear(x)) + x0 = x[0].unsqueeze(0) + mask = torch.autograd.Variable(torch.bernoulli(x0.data.new(x0.data.size()).fill_(1 - self.p_dropout))) + mask = mask.expand(x.size(0), x.size(1)) + x = x * mask * 1 / (1 - self.p_dropout) + else: + for linear in self.layers: + x = F.dropout(F.relu(linear(x)), p=self.p_dropout, training=True) + return x + + +def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels_int): + in_act = input_a + input_b + t_act = torch.tanh(in_act[:, :n_channels_int, :]) + s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) + acts = t_act * s_act + return acts + + +class Invertible1x1Conv(torch.nn.Module): + """ + The layer outputs both the convolution, and the log determinant + of its weight matrix. If reverse=True it does convolution with + inverse + """ + + def __init__(self, c): + super().__init__() + self.conv = torch.nn.Conv1d(c, c, kernel_size=1, stride=1, padding=0, bias=False) + + # Sample a random orthonormal matrix to initialize weights + W = torch.qr(torch.FloatTensor(c, c).normal_())[0] + + # Ensure determinant is 1.0 not -1.0 + if torch.det(W) < 0: + W[:, 0] = -1 * W[:, 0] + W = W.view(c, c, 1) + self.conv.weight.data = W + self.inv_conv = None + + def forward(self, z, reverse: bool = False): + if reverse: + if self.inv_conv is None: + # Inverse convolution - initialized here only for backwards + # compatibility with weights from existing checkpoints. + # Should be moved to init() with next incompatible change. + self.inv_conv = torch.nn.Conv1d( + self.conv.in_channels, self.conv.out_channels, kernel_size=1, stride=1, padding=0, bias=False + ) + W_inverse = self.conv.weight.squeeze().data.float().inverse() + W_inverse = Variable(W_inverse[..., None]) + self.inv_conv.weight.data = W_inverse + self.inv_conv.to(device=self.conv.weight.device, dtype=self.conv.weight.dtype) + return self.inv_conv(z) + else: + # Forward computation + # shape + W = self.conv.weight.squeeze() + batch_size, group_size, n_of_groups = z.size() + log_det_W = batch_size * n_of_groups * torch.logdet(W.float()) + z = self.conv(z) + return ( + z, + log_det_W, + ) + + +class WaveNet(torch.nn.Module): + """ + This is the WaveNet like layer for the affine coupling. The primary + difference from WaveNet is the convolutions need not be causal. There is + also no dilation size reset. The dilation only doubles on each layer + """ + + def __init__(self, n_in_channels, n_mel_channels, n_layers, n_channels, kernel_size): + super().__init__() + assert kernel_size % 2 == 1 + assert n_channels % 2 == 0 + self.n_layers = n_layers + self.n_channels = n_channels + self.in_layers = torch.nn.ModuleList() + self.res_skip_layers = torch.nn.ModuleList() + + start = torch.nn.Conv1d(n_in_channels, n_channels, 1) + start = torch.nn.utils.weight_norm(start, name='weight') + self.start = start + + # Initializing last layer to 0 makes the affine coupling layers + # do nothing at first. This helps with training stability + end = torch.nn.Conv1d(n_channels, 2 * n_in_channels, 1) + end.weight.data.zero_() + end.bias.data.zero_() + self.end = end + + cond_layer = torch.nn.Conv1d(n_mel_channels, 2 * n_channels * n_layers, 1) + self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') + + for i in range(n_layers): + dilation = 2 ** i + padding = int((kernel_size * dilation - dilation) / 2) + in_layer = torch.nn.Conv1d(n_channels, 2 * n_channels, kernel_size, dilation=dilation, padding=padding,) + in_layer = torch.nn.utils.weight_norm(in_layer, name='weight') + self.in_layers.append(in_layer) + + # last one is not necessary + if i < n_layers - 1: + res_skip_channels = 2 * n_channels + else: + res_skip_channels = n_channels + res_skip_layer = torch.nn.Conv1d(n_channels, res_skip_channels, 1) + res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight') + self.res_skip_layers.append(res_skip_layer) + + def forward(self, forward_input: Tuple[torch.Tensor, torch.Tensor]): + audio, spect = forward_input[0], forward_input[1] + audio = self.start(audio) + output = torch.zeros_like(audio) + + spect = self.cond_layer(spect) + + for i in range(self.n_layers): + spect_offset = i * 2 * self.n_channels + acts = fused_add_tanh_sigmoid_multiply( + self.in_layers[i](audio), + spect[:, spect_offset : spect_offset + 2 * self.n_channels, :], + self.n_channels, + ) + + res_skip_acts = self.res_skip_layers[i](acts) + if i < self.n_layers - 1: + audio = audio + res_skip_acts[:, : self.n_channels, :] + output = output + res_skip_acts[:, self.n_channels :, :] + else: + output = output + res_skip_acts + + return self.end(output) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/tacotron2.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/tacotron2.py new file mode 100644 index 0000000000..de62aaeccf --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/tacotron2.py @@ -0,0 +1,442 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torch.autograd import Variable +from torch.nn import functional as F + +from nemo.collections.tts.helpers.helpers import get_mask_from_lengths +from nemo.collections.tts.modules.submodules import Attention, ConvNorm, LinearNorm, Prenet +from nemo.core.classes import NeuralModule, typecheck +from nemo.core.neural_types.elements import ( + EmbeddedTextType, + LengthsType, + LogitsType, + MelSpectrogramType, + SequenceToSequenceAlignmentType, +) +from nemo.core.neural_types.neural_type import NeuralType +from nemo.utils import logging + + +class Encoder(NeuralModule): + def __init__( + self, encoder_n_convolutions: int, encoder_embedding_dim: int, encoder_kernel_size: int, + ): + """ + Tacotron 2 Encoder. A number of convolution layers that feed into a LSTM + + Args: + encoder_n_convolutions (int): Number of convolution layers. + encoder_embedding_dim (int): Final output embedding size. Also used to create the convolution and LSTM layers. + encoder_kernel_size (int): Kernel of the convolution front-end. + """ + super().__init__() + + convolutions = [] + for _ in range(encoder_n_convolutions): + conv_layer = torch.nn.Sequential( + ConvNorm( + encoder_embedding_dim, + encoder_embedding_dim, + kernel_size=encoder_kernel_size, + stride=1, + padding=int((encoder_kernel_size - 1) / 2), + dilation=1, + w_init_gain='relu', + ), + torch.nn.BatchNorm1d(encoder_embedding_dim), + ) + convolutions.append(conv_layer) + self.convolutions = torch.nn.ModuleList(convolutions) + + self.lstm = torch.nn.LSTM( + encoder_embedding_dim, int(encoder_embedding_dim / 2), 1, batch_first=True, bidirectional=True, + ) + + @property + def input_types(self): + return { + "token_embedding": NeuralType(('B', 'D', 'T'), EmbeddedTextType()), + "token_len": NeuralType(('B'), LengthsType()), + } + + @property + def output_types(self): + return { + "encoder_embedding": NeuralType(('B', 'T', 'D'), EmbeddedTextType()), + } + + @typecheck() + def forward(self, *, token_embedding, token_len): + for conv in self.convolutions: + token_embedding = F.dropout(F.relu(conv(token_embedding)), 0.5, self.training) + + token_embedding = token_embedding.transpose(1, 2) + + # pytorch tensor are not reversible, hence the conversion + input_lengths = token_len.cpu().numpy() + + token_embedding = torch.nn.utils.rnn.pack_padded_sequence( + token_embedding, input_lengths, batch_first=True, enforce_sorted=False + ) + + self.lstm.flatten_parameters() + outputs, _ = self.lstm(token_embedding) + + outputs, _ = torch.nn.utils.rnn.pad_packed_sequence(outputs, batch_first=True) + + return outputs + + +class Decoder(NeuralModule): + def __init__( + self, + n_mel_channels: int, + n_frames_per_step: int, + encoder_embedding_dim: int, + attention_dim: int, + attention_location_n_filters: int, + attention_location_kernel_size: int, + attention_rnn_dim: int, + decoder_rnn_dim: int, + prenet_dim: int, + max_decoder_steps: int, + gate_threshold: float, + p_attention_dropout: float, + p_decoder_dropout: float, + early_stopping: bool, + prenet_p_dropout: float = 0.5, + ): + """ + Tacotron 2 Decoder. Consists of a 2 layer LSTM, one of which interfaces with the attention mechanism while the + other is used as a regular LSTM. Includes the prenet and attention modules as well. + + Args: + n_mel_channels (int): Number of mel channels to output + n_frames_per_step (int): Number of spectrogram frames to predict per decoder step. + encoder_embedding_dim (int): The size of the output from the encoder. + attention_dim (int): The output dimension of the attention layer. + attention_location_n_filters (int): Channel size for the convolution used the attention mechanism. + attention_location_kernel_size (int): Kernel size for the convolution used the attention mechanism. + attention_rnn_dim (int): The output dimension of the attention LSTM layer. + decoder_rnn_dim (int): The output dimension of the second LSTM layer. + prenet_dim (int): The output dimension of the prenet. + max_decoder_steps (int): For evaluation, the max number of steps to predict. + gate_threshold (float): At each step, tacotron 2 predicts a probability of stopping. Rather than sampling, + this module checks if predicted probability is above the gate_threshold. Only in evaluation. + p_attention_dropout (float): Dropout probability on the attention LSTM. + p_decoder_dropout (float): Dropout probability on the second LSTM. + early_stopping (bool): In evaluation mode, whether to stop when all batches hit the gate_threshold or to + continue until max_decoder_steps. + prenet_p_dropout (float): Dropout probability for prenet. Note, dropout is on even in eval() mode. + Defaults to 0.5. + """ + super().__init__() + self.n_mel_channels = n_mel_channels + self.n_frames_per_step = n_frames_per_step + self.encoder_embedding_dim = encoder_embedding_dim + self.attention_rnn_dim = attention_rnn_dim + self.decoder_rnn_dim = decoder_rnn_dim + self.prenet_dim = prenet_dim + self.max_decoder_steps = max_decoder_steps + self.gate_threshold = gate_threshold + self.p_attention_dropout = p_attention_dropout + self.p_decoder_dropout = p_decoder_dropout + self.early_stopping = early_stopping + + self.prenet = Prenet(n_mel_channels * n_frames_per_step, [prenet_dim, prenet_dim], prenet_p_dropout) + + self.attention_rnn = torch.nn.LSTMCell(prenet_dim + encoder_embedding_dim, attention_rnn_dim) + + self.attention_layer = Attention( + attention_rnn_dim, + encoder_embedding_dim, + attention_dim, + attention_location_n_filters, + attention_location_kernel_size, + ) + + self.decoder_rnn = torch.nn.LSTMCell(attention_rnn_dim + encoder_embedding_dim, decoder_rnn_dim, 1) + + self.linear_projection = LinearNorm( + decoder_rnn_dim + encoder_embedding_dim, n_mel_channels * n_frames_per_step + ) + + self.gate_layer = LinearNorm(decoder_rnn_dim + encoder_embedding_dim, 1, bias=True, w_init_gain='sigmoid') + + @property + def input_types(self): + input_dict = { + "memory": NeuralType(('B', 'T', 'D'), EmbeddedTextType()), + "memory_lengths": NeuralType(('B'), LengthsType()), + } + if self.training: + input_dict["decoder_inputs"] = NeuralType(('B', 'D', 'T'), MelSpectrogramType()) + return input_dict + + @property + def output_types(self): + output_dict = { + "mel_outputs": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "gate_outputs": NeuralType(('B', 'T'), LogitsType()), + "alignments": NeuralType(('B', 'T', 'T'), SequenceToSequenceAlignmentType()), + } + if not self.training: + output_dict["mel_lengths"] = NeuralType(('B'), LengthsType()) + return output_dict + + @typecheck() + def forward(self, *args, **kwargs): + if self.training: + return self.train_forward(**kwargs) + return self.infer(**kwargs) + + def get_go_frame(self, memory): + B = memory.size(0) + decoder_input = Variable(memory.data.new(B, self.n_mel_channels * self.n_frames_per_step).zero_()) + return decoder_input + + def initialize_decoder_states(self, memory, mask): + B = memory.size(0) + MAX_TIME = memory.size(1) + + self.attention_hidden = Variable(memory.data.new(B, self.attention_rnn_dim).zero_()) + self.attention_cell = Variable(memory.data.new(B, self.attention_rnn_dim).zero_()) + + self.decoder_hidden = Variable(memory.data.new(B, self.decoder_rnn_dim).zero_()) + self.decoder_cell = Variable(memory.data.new(B, self.decoder_rnn_dim).zero_()) + + self.attention_weights = Variable(memory.data.new(B, MAX_TIME).zero_()) + self.attention_weights_cum = Variable(memory.data.new(B, MAX_TIME).zero_()) + self.attention_context = Variable(memory.data.new(B, self.encoder_embedding_dim).zero_()) + + self.memory = memory + self.processed_memory = self.attention_layer.memory_layer(memory) + self.mask = mask + + def parse_decoder_inputs(self, decoder_inputs): + # (B, n_mel_channels, T_out) -> (B, T_out, n_mel_channels) + decoder_inputs = decoder_inputs.transpose(1, 2) + decoder_inputs = decoder_inputs.view( + decoder_inputs.size(0), int(decoder_inputs.size(1) / self.n_frames_per_step), -1, + ) + # (B, T_out, n_mel_channels) -> (T_out, B, n_mel_channels) + decoder_inputs = decoder_inputs.transpose(0, 1) + return decoder_inputs + + def parse_decoder_outputs(self, mel_outputs, gate_outputs, alignments): + # (T_out, B) -> (B, T_out) + alignments = torch.stack(alignments).transpose(0, 1) + # (T_out, B) -> (B, T_out) + # Add a -1 to prevent squeezing the batch dimension in case + # batch is 1 + gate_outputs = torch.stack(gate_outputs).squeeze(-1).transpose(0, 1) + gate_outputs = gate_outputs.contiguous() + # (T_out, B, n_mel_channels) -> (B, T_out, n_mel_channels) + mel_outputs = torch.stack(mel_outputs).transpose(0, 1).contiguous() + # decouple frames per step + mel_outputs = mel_outputs.view(mel_outputs.size(0), -1, self.n_mel_channels) + # (B, T_out, n_mel_channels) -> (B, n_mel_channels, T_out) + mel_outputs = mel_outputs.transpose(1, 2) + + return mel_outputs, gate_outputs, alignments + + def decode(self, decoder_input): + cell_input = torch.cat((decoder_input, self.attention_context), -1) + + self.attention_hidden, self.attention_cell = self.attention_rnn( + cell_input, (self.attention_hidden, self.attention_cell) + ) + self.attention_hidden = F.dropout(self.attention_hidden, self.p_attention_dropout, self.training) + + attention_weights_cat = torch.cat( + (self.attention_weights.unsqueeze(1), self.attention_weights_cum.unsqueeze(1)), dim=1, + ) + self.attention_context, self.attention_weights = self.attention_layer( + self.attention_hidden, self.memory, self.processed_memory, attention_weights_cat, self.mask, + ) + + self.attention_weights_cum += self.attention_weights + decoder_input = torch.cat((self.attention_hidden, self.attention_context), -1) + + self.decoder_hidden, self.decoder_cell = self.decoder_rnn( + decoder_input, (self.decoder_hidden, self.decoder_cell) + ) + self.decoder_hidden = F.dropout(self.decoder_hidden, self.p_decoder_dropout, self.training) + + decoder_hidden_attention_context = torch.cat((self.decoder_hidden, self.attention_context), dim=1) + decoder_output = self.linear_projection(decoder_hidden_attention_context) + + gate_prediction = self.gate_layer(decoder_hidden_attention_context) + return decoder_output, gate_prediction, self.attention_weights + + def train_forward(self, *, memory, decoder_inputs, memory_lengths): + decoder_input = self.get_go_frame(memory).unsqueeze(0) + decoder_inputs = self.parse_decoder_inputs(decoder_inputs) + decoder_inputs = torch.cat((decoder_input, decoder_inputs), dim=0) + decoder_inputs = self.prenet(decoder_inputs) + + self.initialize_decoder_states(memory, mask=~get_mask_from_lengths(memory_lengths)) + + mel_outputs, gate_outputs, alignments = [], [], [] + while len(mel_outputs) < decoder_inputs.size(0) - 1: + decoder_input = decoder_inputs[len(mel_outputs)] + mel_output, gate_output, attention_weights = self.decode(decoder_input) + + mel_outputs += [mel_output.squeeze(1)] + gate_outputs += [gate_output] + alignments += [attention_weights] + + mel_outputs, gate_outputs, alignments = self.parse_decoder_outputs(mel_outputs, gate_outputs, alignments) + return mel_outputs, gate_outputs, alignments + + def infer(self, *, memory, memory_lengths): + decoder_input = self.get_go_frame(memory) + + if memory.size(0) > 1: + mask = ~get_mask_from_lengths(memory_lengths) + else: + mask = None + + self.initialize_decoder_states(memory, mask=mask) + + mel_lengths = torch.zeros([memory.size(0)], dtype=torch.int32) + not_finished = torch.ones([memory.size(0)], dtype=torch.int32) + if torch.cuda.is_available(): + mel_lengths = mel_lengths.cuda() + not_finished = not_finished.cuda() + + mel_outputs, gate_outputs, alignments = [], [], [] + stepped = False + while True: + decoder_input = self.prenet(decoder_input, inference=True) + mel_output, gate_output, alignment = self.decode(decoder_input) + + dec = torch.le(torch.sigmoid(gate_output.data), self.gate_threshold).to(torch.int32).squeeze(1) + + not_finished = not_finished * dec + mel_lengths += not_finished + + if self.early_stopping and torch.sum(not_finished) == 0 and stepped: + break + stepped = True + + mel_outputs += [mel_output.squeeze(1)] + gate_outputs += [gate_output] + alignments += [alignment] + + if len(mel_outputs) == self.max_decoder_steps: + logging.warning("Reached max decoder steps %d.", self.max_decoder_steps) + break + + decoder_input = mel_output + + mel_outputs, gate_outputs, alignments = self.parse_decoder_outputs(mel_outputs, gate_outputs, alignments) + + return mel_outputs, gate_outputs, alignments, mel_lengths + + +class Postnet(NeuralModule): + def __init__( + self, + n_mel_channels: int, + postnet_embedding_dim: int, + postnet_kernel_size: int, + postnet_n_convolutions: int, + p_dropout: float = 0.5, + ): + """ + Tacotron 2 Postnet. A convolutional network with postnet_n_convolutions number of layers. Each layer has a + kernel of postnet_kernel_size. Each layer apart from the last outputs postnet_embedding_dim channels, the last + outputs n_mel_channels channels. After each layer is a dropout layer with p_dropout% drop. The last linear has + no activation, all intermediate layers have tanh activation. + + Args: + n_mel_channels (int): Number of mel channels to output from Posnet. + postnet_embedding_dim (int): Number of channels to output from the intermediate layers. + postnet_kernel_size (int): The kernel size for the convolution layers. + postnet_n_convolutions (int): The number of convolutions layers. + p_dropout (float): Dropout probability. Defaults to 0.5. + """ + super().__init__() + self.convolutions = torch.nn.ModuleList() + + self.convolutions.append( + torch.nn.Sequential( + ConvNorm( + n_mel_channels, + postnet_embedding_dim, + kernel_size=postnet_kernel_size, + stride=1, + padding=int((postnet_kernel_size - 1) / 2), + dilation=1, + w_init_gain='tanh', + ), + torch.nn.BatchNorm1d(postnet_embedding_dim), + ) + ) + + for _ in range(1, postnet_n_convolutions - 1): + self.convolutions.append( + torch.nn.Sequential( + ConvNorm( + postnet_embedding_dim, + postnet_embedding_dim, + kernel_size=postnet_kernel_size, + stride=1, + padding=int((postnet_kernel_size - 1) / 2), + dilation=1, + w_init_gain='tanh', + ), + torch.nn.BatchNorm1d(postnet_embedding_dim), + ) + ) + + self.convolutions.append( + torch.nn.Sequential( + ConvNorm( + postnet_embedding_dim, + n_mel_channels, + kernel_size=postnet_kernel_size, + stride=1, + padding=int((postnet_kernel_size - 1) / 2), + dilation=1, + w_init_gain='linear', + ), + torch.nn.BatchNorm1d(n_mel_channels), + ) + ) + self.p_dropout = p_dropout + + @property + def input_types(self): + return { + "mel_spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + } + + @property + def output_types(self): + return { + "mel_spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + } + + @typecheck() + def forward(self, *, mel_spec): + mel_spec_out = mel_spec + for i in range(len(self.convolutions) - 1): + mel_spec_out = F.dropout(torch.tanh(self.convolutions[i](mel_spec_out)), self.p_dropout, self.training) + mel_spec_out = F.dropout(self.convolutions[-1](mel_spec_out), self.p_dropout, self.training) + + return mel_spec + mel_spec_out diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/talknet.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/talknet.py new file mode 100644 index 0000000000..626faf6ce5 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/talknet.py @@ -0,0 +1,136 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torch import nn +from torch.nn import functional as F + +from nemo.collections.asr.data.audio_to_text import AudioToCharWithDursF0Dataset + + +class GaussianEmbedding(nn.Module): + """Gaussian embedding layer..""" + + EPS = 1e-6 + + def __init__( + self, vocab, d_emb, sigma_c=2.0, merge_blanks=False, + ): + super().__init__() + + self.embed = nn.Embedding(len(vocab.labels), d_emb) + self.pad = vocab.pad + self.sigma_c = sigma_c + self.merge_blanks = merge_blanks + + def forward(self, text, durs): + """See base class.""" + # Fake padding + text = F.pad(text, [0, 2, 0, 0], value=self.pad) + durs = F.pad(durs, [0, 2, 0, 0], value=0) + + repeats = AudioToCharWithDursF0Dataset.repeat_merge(text, durs, self.pad) + total_time = repeats.shape[-1] + + # Centroids: [B,T,N] + c = (durs / 2.0) + F.pad(torch.cumsum(durs, dim=-1)[:, :-1], [1, 0, 0, 0], value=0) + c = c.unsqueeze(1).repeat(1, total_time, 1) + + # Sigmas: [B,T,N] + sigmas = durs + sigmas = sigmas.float() / self.sigma_c + sigmas = sigmas.unsqueeze(1).repeat(1, total_time, 1) + self.EPS + assert c.shape == sigmas.shape + + # Times at indexes + t = torch.arange(total_time, device=c.device).view(1, -1, 1).repeat(durs.shape[0], 1, durs.shape[-1]).float() + t = t + 0.5 + + ns = slice(None) + if self.merge_blanks: + ns = slice(1, None, 2) + + # Weights: [B,T,N] + d = torch.distributions.normal.Normal(c, sigmas) + w = d.log_prob(t).exp()[:, :, ns] # [B,T,N] + pad_mask = (text == self.pad)[:, ns].unsqueeze(1).repeat(1, total_time, 1) + w.masked_fill_(pad_mask, 0.0) # noqa + w = w / (w.sum(-1, keepdim=True) + self.EPS) + pad_mask = (repeats == self.pad).unsqueeze(-1).repeat(1, 1, text[:, ns].size(1)) # noqa + w.masked_fill_(pad_mask, 0.0) # noqa + pad_mask[:, :, :-1] = False + w.masked_fill_(pad_mask, 1.0) # noqa + + # Embeds + u = torch.bmm(w, self.embed(text)[:, ns, :]) # [B,T,E] + + return u + + +class MaskedInstanceNorm1d(nn.Module): + """Instance norm + masking.""" + + MAX_CNT = 1e5 + + def __init__(self, d_channel: int, unbiased: bool = True, affine: bool = False): + super().__init__() + + self.d_channel = d_channel + self.unbiased = unbiased + + self.affine = affine + if self.affine: + gamma = torch.ones(d_channel, dtype=torch.float) + beta = torch.zeros_like(gamma) + self.register_parameter('gamma', nn.Parameter(gamma)) + self.register_parameter('beta', nn.Parameter(beta)) + + def forward(self, x: torch.Tensor, x_mask: torch.Tensor) -> torch.Tensor: # noqa + """`x`: [B,C,T], `x_mask`: [B,T] => [B,C,T].""" + x_mask = x_mask.unsqueeze(1).type_as(x) # [B,1,T] + cnt = x_mask.sum(dim=-1, keepdim=True) # [B,1,1] + + # Mean: [B,C,1] + cnt_for_mu = cnt.clamp(1.0, self.MAX_CNT) + mu = (x * x_mask).sum(dim=-1, keepdim=True) / cnt_for_mu + + # Variance: [B,C,1] + sigma = (x - mu) ** 2 + cnt_fot_sigma = (cnt - int(self.unbiased)).clamp(1.0, self.MAX_CNT) + sigma = (sigma * x_mask).sum(dim=-1, keepdim=True) / cnt_fot_sigma + sigma = (sigma + 1e-8).sqrt() + + y = (x - mu) / sigma + + if self.affine: + gamma = self.gamma.unsqueeze(0).unsqueeze(-1) + beta = self.beta.unsqueeze(0).unsqueeze(-1) + y = y * gamma + beta + + return y + + +class StyleResidual(nn.Module): + """Styling.""" + + def __init__(self, d_channel: int, d_style: int, kernel_size: int = 1): + super().__init__() + + self.rs = nn.Conv1d( + in_channels=d_style, out_channels=d_channel, kernel_size=kernel_size, stride=1, padding=kernel_size // 2, + ) + + def forward(self, x: torch.Tensor, s: torch.Tensor) -> torch.Tensor: + """`x`: [B,C,T], `s`: [B,S,T] => [B,C,T].""" + return x + self.rs(s) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/transformer.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/transformer.py new file mode 100644 index 0000000000..c8f12813f8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/transformer.py @@ -0,0 +1,262 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nemo.core.classes import NeuralModule, typecheck +from nemo.core.neural_types.elements import EncodedRepresentation, LengthsType, MaskType, TokenIndex +from nemo.core.neural_types.neural_type import NeuralType + + +def mask_from_lens(lens, max_len: Optional[int] = None): + if max_len is None: + max_len = lens.max() + ids = torch.arange(0, max_len, device=lens.device, dtype=lens.dtype) + mask = torch.lt(ids, lens.unsqueeze(1)) + return mask + + +class PositionalEmbedding(nn.Module): + def __init__(self, demb): + super(PositionalEmbedding, self).__init__() + self.demb = demb + inv_freq = 1 / (10000 ** (torch.arange(0.0, demb, 2.0) / demb)) + self.register_buffer('inv_freq', inv_freq) + + def forward(self, pos_seq, bsz=None): + # sinusoid_inp = torch.ger(pos_seq, self.inv_freq) + sinusoid_inp = torch.matmul(torch.unsqueeze(pos_seq, -1), torch.unsqueeze(self.inv_freq, 0)) + + pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=1) + if bsz is not None: + return pos_emb[None, :, :].expand(bsz, -1, -1) + else: + return pos_emb[None, :, :] + + +class PositionwiseConvFF(nn.Module): + def __init__(self, d_model, d_inner, kernel_size, dropout, pre_lnorm=False): + super(PositionwiseConvFF, self).__init__() + + self.d_model = d_model + self.d_inner = d_inner + self.dropout = dropout + + if type(kernel_size) is not tuple: + kernel_size = (kernel_size, kernel_size) + + self.CoreNet = nn.Sequential( + nn.Conv1d(d_model, d_inner, kernel_size[0], 1, (kernel_size[0] // 2)), + nn.ReLU(), + # nn.Dropout(dropout), # worse convergence + nn.Conv1d(d_inner, d_model, kernel_size[1], 1, (kernel_size[1] // 2)), + nn.Dropout(dropout), + ) + self.layer_norm = nn.LayerNorm(d_model) + self.pre_lnorm = pre_lnorm + + def forward(self, inp): + return self._forward(inp) + + def _forward(self, inp): + if self.pre_lnorm: + # layer normalization + positionwise feed-forward + core_out = inp.transpose(1, 2) + core_out = self.CoreNet(self.layer_norm(core_out).to(inp.dtype)) + core_out = core_out.transpose(1, 2) + + # residual connection + output = core_out + inp + else: + # positionwise feed-forward + core_out = inp.transpose(1, 2) + core_out = self.CoreNet(core_out) + core_out = core_out.transpose(1, 2) + + # residual connection + layer normalization + output = self.layer_norm(inp + core_out).to(inp.dtype) + + return output + + +class MultiHeadAttn(nn.Module): + def __init__(self, n_head, d_model, d_head, dropout, dropatt=0.1, pre_lnorm=False): + super(MultiHeadAttn, self).__init__() + + self.n_head = n_head + self.d_model = d_model + self.d_head = d_head + self.scale = 1 / (d_head ** 0.5) + self.pre_lnorm = pre_lnorm + + self.qkv_net = nn.Linear(d_model, 3 * n_head * d_head) + self.drop = nn.Dropout(dropout) + self.dropatt = nn.Dropout(dropatt) + self.o_net = nn.Linear(n_head * d_head, d_model, bias=False) + self.layer_norm = nn.LayerNorm(d_model) + + def forward(self, inp, attn_mask=None): + return self._forward(inp, attn_mask) + + def _forward(self, inp, attn_mask=None): + residual = inp + + if self.pre_lnorm: + # layer normalization + inp = self.layer_norm(inp) + + n_head, d_head = self.n_head, self.d_head + + head_q, head_k, head_v = torch.chunk(self.qkv_net(inp), 3, dim=2) + + head_q = head_q.view(inp.size(0), inp.size(1), n_head, d_head) + head_k = head_k.view(inp.size(0), inp.size(1), n_head, d_head) + head_v = head_v.view(inp.size(0), inp.size(1), n_head, d_head) + + q = head_q.permute(0, 2, 1, 3).reshape(-1, inp.size(1), d_head) + k = head_k.permute(0, 2, 1, 3).reshape(-1, inp.size(1), d_head) + v = head_v.permute(0, 2, 1, 3).reshape(-1, inp.size(1), d_head) + + attn_score = torch.bmm(q, k.transpose(1, 2)) + attn_score.mul_(self.scale) + + if attn_mask is not None: + attn_mask = attn_mask.unsqueeze(1).to(attn_score.dtype) + attn_mask = attn_mask.repeat(n_head, attn_mask.size(2), 1) + attn_score.masked_fill_(attn_mask.to(torch.bool), -float('inf')) + + attn_prob = F.softmax(attn_score, dim=2) + attn_prob = self.dropatt(attn_prob) + attn_vec = torch.bmm(attn_prob, v) + + attn_vec = attn_vec.view(n_head, inp.size(0), inp.size(1), d_head) + attn_vec = attn_vec.permute(1, 2, 0, 3).contiguous().view(inp.size(0), inp.size(1), n_head * d_head) + + # linear projection + attn_out = self.o_net(attn_vec) + attn_out = self.drop(attn_out) + + if self.pre_lnorm: + # residual connection + output = residual + attn_out + else: + # residual connection + layer normalization + output = self.layer_norm(residual + attn_out) + + return output + + +class TransformerLayer(nn.Module): + def __init__(self, n_head, d_model, d_head, d_inner, kernel_size, dropout, **kwargs): + super(TransformerLayer, self).__init__() + + self.dec_attn = MultiHeadAttn(n_head, d_model, d_head, dropout, **kwargs) + self.pos_ff = PositionwiseConvFF(d_model, d_inner, kernel_size, dropout, pre_lnorm=kwargs.get('pre_lnorm')) + + def forward(self, dec_inp, mask=None): + output = self.dec_attn(dec_inp, attn_mask=~mask.squeeze(2)) + output *= mask + output = self.pos_ff(output) + output *= mask + return output + + +class FFTransformerDecoder(NeuralModule): + def __init__( + self, n_layer, n_head, d_model, d_head, d_inner, kernel_size, dropout, dropatt, dropemb=0.0, pre_lnorm=False + ): + super(FFTransformerDecoder, self).__init__() + self.d_model = d_model + self.n_head = n_head + self.d_head = d_head + + self.pos_emb = PositionalEmbedding(self.d_model) + self.drop = nn.Dropout(dropemb) + self.layers = nn.ModuleList() + + for _ in range(n_layer): + self.layers.append( + TransformerLayer( + n_head, d_model, d_head, d_inner, kernel_size, dropout, dropatt=dropatt, pre_lnorm=pre_lnorm + ) + ) + + @property + def input_types(self): + return { + "input": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "seq_lens": NeuralType(('B'), LengthsType()), + "conditioning": NeuralType(('B', 'T', 'D'), EncodedRepresentation(), optional=True), + } + + @property + def output_types(self): + return { + "out": NeuralType(('B', 'T', 'D'), EncodedRepresentation()), + "mask": NeuralType(('B', 'T', 'D'), MaskType()), + } + + @typecheck() + def forward(self, input, seq_lens, conditioning=0): + return self._forward(input, mask_from_lens(seq_lens).unsqueeze(2), conditioning) + + def _forward(self, inp, mask, conditioning): + pos_seq = torch.arange(inp.size(1), device=inp.device).to(inp.dtype) + pos_emb = self.pos_emb(pos_seq) * mask + out = self.drop(inp + pos_emb + conditioning) + + for layer in self.layers: + out = layer(out, mask=mask) + + # out = self.drop(out) + return out, mask + + +class FFTransformerEncoder(FFTransformerDecoder): + def __init__( + self, + n_layer, + n_head, + d_model, + d_head, + d_inner, + kernel_size, + dropout, + dropatt, + dropemb=0.0, + pre_lnorm=False, + n_embed=None, + d_embed=None, + padding_idx=0, + ): + super(FFTransformerEncoder, self).__init__( + n_layer, n_head, d_model, d_head, d_inner, kernel_size, dropout, dropatt, dropemb, pre_lnorm + ) + + self.padding_idx = padding_idx + self.word_emb = nn.Embedding(n_embed, d_embed or d_model, padding_idx=self.padding_idx) + + @property + def input_types(self): + return { + "input": NeuralType(('B', 'T'), TokenIndex()), + "conditioning": NeuralType(('B', 'T', 'D'), EncodedRepresentation(), optional=True), + } + + def forward(self, input, conditioning=0): + + return self._forward(self.word_emb(input), (input != self.padding_idx).unsqueeze(2), conditioning) # (B, L, 1) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/uniglow.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/uniglow.py new file mode 100644 index 0000000000..5e6b7556e8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/uniglow.py @@ -0,0 +1,177 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn.functional as F + +from nemo.collections.tts.helpers.helpers import OperationMode, remove +from nemo.collections.tts.modules.submodules import Invertible1x1Conv, WaveNet +from nemo.core.classes import Exportable, NeuralModule, typecheck +from nemo.core.neural_types.elements import AudioSignal, MelSpectrogramType, NormalDistributionSamplesType, VoidType +from nemo.core.neural_types.neural_type import NeuralType + + +class UniGlowModule(NeuralModule, Exportable): + def __init__( + self, + n_mel_channels: int, + n_flows: int, + n_group: int, + n_wn_channels: int, + n_wn_layers: int, + wn_kernel_size: int, + upsample_factor: int, + ): + """ + UniGlow module + Args: + n_mel_channels (int): Number of mel channels to output. + n_flows (int): Number of flow layers + n_group (int): Number of groups to respace the inputs + n_early_every (int): Every n_early_every layers, n_early_size gets skip connected to the output + n_early_size (int): The size of the chunk to be skip connected + n_wn_channels (int): Number of channels for the non-invertible wavenet transformation + n_wn_layers (int): Number of layers for the non-invertible wavenet transformation + wn_kernel_size (int): Kernel size for the non-invertible wavenet transformation + """ + super().__init__() + + assert n_group % 2 == 0 + self.n_flows = n_flows + self.n_group = n_group + n_half = int(n_group / 2) + + self.conv = Invertible1x1Conv(n_group) + self.wn = WaveNet(n_half, n_mel_channels, n_wn_layers, n_wn_channels, wn_kernel_size) + self.upsample_factor = upsample_factor + self.mode = OperationMode.infer + + @typecheck() + def forward(self, spec, audio=None, sigma=1.0): + """ + Args: + spec: the melspectrogram tensor [B,F,Tfd] + audio: the time-domain audio tensor [B,Ttd] + sigma: the standard deviation of the latent variable z + Returns: + training/validation: + z: latent variable + lodget: the sum of log-determinants + audio_pred: time-domain signal predicted from (spec,z') where + z' is a latent variable sampled from a N(0,sigma) + inference: + audio_pred: time-domain signal predicted from (spec,z') where + z' is a latent variable sampled from a N(0,sigma) + """ + if self.training and self.mode != OperationMode.training: + raise ValueError(f"{self} has self.training set to True but self.OperationMode was not set to training") + if not self.training and self.mode == OperationMode.training: + raise ValueError(f"{self} has self.training set to False but self.OperationMode was set to training") + + if audio is not None and self.mode != OperationMode.infer: + # audio_to_normal_dist is used to calculate loss so only run this in train or val model + z, logdet = self.audio_to_normal_dist(spec=spec, audio=audio) + # audio_pred is used to calculate the stft-loss and when running inference + audio_pred = self.norm_dist_to_audio(spec=spec, sigma=sigma) + + # Return the necessary tensors + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + return z, logdet, audio_pred + return audio_pred + + @property + def input_types(self): + return { + "spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "audio": NeuralType(('B', 'T'), AudioSignal(), optional=True), + "sigma": NeuralType(optional=True), + } + + @property + def output_types(self): + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + return { + "pred_normal_dist": NeuralType(('B', 'flowgroup', 'T'), NormalDistributionSamplesType()), + "logdet": NeuralType(elements_type=VoidType()), + "audio_pred": NeuralType(('B', 'T'), AudioSignal()), + } + else: + return { + "audio": NeuralType(('B', 'T'), AudioSignal()), + } + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + par = next(self.parameters()) + mel = torch.randn((1, self.n_mel_channels, 96), device=par.device, dtype=par.dtype) + return tuple([mel]) + + def audio_to_normal_dist(self, *, spec: torch.Tensor, audio: torch.Tensor) -> (torch.Tensor, float): + logdet = 0 + + spec = spec[:, :, :-1] + audio = audio.unfold(1, self.n_group, self.n_group).permute(0, 2, 1) + if spec.size(2) != audio.size(2): + spec = F.interpolate(spec, size=audio.size(2)) + + for _ in range(self.n_flows): + audio, log_det_W = self.conv(audio) + logdet += log_det_W + + n_half = int(audio.size(1) / 2) + audio_0 = audio[:, :n_half, :] + audio_1 = audio[:, n_half:, :] + + output = self.wn((audio_0, spec)) + log_s = output[:, n_half:, :] + b = output[:, :n_half, :] + audio_1 = torch.exp(log_s) * audio_1 + b + logdet += torch.sum(log_s) + + audio = torch.cat([audio_0, audio_1], 1) + + return audio, logdet + + def norm_dist_to_audio(self, *, spec, sigma: float = 1.0): + spec = spec[:, :, :-1] + audio_len = spec.shape[2] * self.upsample_factor + spec = F.interpolate(spec, size=audio_len) + audio = sigma * torch.randn(spec.size(0), self.n_group, audio_len, device=spec.device).to(spec.dtype) + + for _ in reversed(range(self.n_flows)): + n_half = int(audio.size(1) / 2) + audio_0 = audio[:, :n_half, :] + audio_1 = audio[:, n_half:, :] + + output = self.wn((audio_0, spec)) + + s = output[:, n_half:, :] + b = output[:, :n_half, :] + audio_1 = (audio_1 - b) / torch.exp(s) + audio = torch.cat([audio_0, audio_1], 1) + + audio = self.conv(audio, reverse=True) + + audio = audio.permute(0, 2, 1).contiguous().view(audio.size(0), -1) + return audio + + def remove_weightnorm(self): + self.wn.start = torch.nn.utils.remove_weight_norm(self.wn.start) + self.wn.in_layers = remove(self.wn.in_layers) + self.wn.cond_layer = torch.nn.utils.remove_weight_norm(self.wn.cond_layer) + self.wn.res_skip_layers = remove(self.wn.res_skip_layers) diff --git a/models/tacotron2/NeMo/nemo/collections/tts/modules/waveglow.py b/models/tacotron2/NeMo/nemo/collections/tts/modules/waveglow.py new file mode 100644 index 0000000000..9439407d40 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/collections/tts/modules/waveglow.py @@ -0,0 +1,263 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +from nemo.collections.tts.helpers.helpers import OperationMode, remove +from nemo.collections.tts.modules.submodules import Invertible1x1Conv, WaveNet +from nemo.core.classes import Exportable, NeuralModule, typecheck +from nemo.core.neural_types.elements import ( + AudioSignal, + IntType, + MelSpectrogramType, + NormalDistributionSamplesType, + VoidType, +) +from nemo.core.neural_types.neural_type import NeuralType + + +class WaveGlowModule(NeuralModule, Exportable): + def __init__( + self, + n_mel_channels: int, + n_flows: int, + n_group: int, + n_early_every: int, + n_early_size: int, + n_wn_channels: int, + n_wn_layers: int, + wn_kernel_size: int, + ): + """ + WaveGlow module + + Args: + n_mel_channels (int): Number of mel channels to output. + n_flows (int): Number of flow layers + n_group (int): Number of groups to respace the inputs + n_early_every (int): Every n_early_every layers, n_early_size gets skip connected to the output + n_early_size (int): The size of the chunk to be skip connected + n_wn_channels (int): Number of channels for the non-invertible wavenet transformation + n_wn_layers (int): Number of layers for the non-invertible wavenet transformation + wn_kernel_size (int): Kernel size for the non-invertible wavenet transformation + """ + super().__init__() + + self.upsample = torch.nn.ConvTranspose1d(n_mel_channels, n_mel_channels, 1024, stride=256) + self.n_mel_channels = n_mel_channels + assert n_group % 2 == 0 + self.n_flows = n_flows + self.n_group = n_group + self.n_early_every = n_early_every + self.n_early_size = n_early_size + self.wavenet = torch.nn.ModuleList() + self.convinv = torch.nn.ModuleList() + self.mode = OperationMode.infer + + n_half = n_group // 2 + + # Set up layers with the right sizes based on how many dimensions + # have been output already + n_remaining_channels = n_group + for k in range(n_flows): + if k % self.n_early_every == 0 and k > 0: + n_half = n_half - int(self.n_early_size / 2) + n_remaining_channels = n_remaining_channels - self.n_early_size + self.convinv.append(Invertible1x1Conv(n_remaining_channels)) + self.wavenet.append( + WaveNet( + n_half, + n_mel_channels * n_group, + n_layers=n_wn_layers, + n_channels=n_wn_channels, + kernel_size=wn_kernel_size, + ) + ) + self.n_remaining_channels = n_remaining_channels + self.time_cutoff = self.upsample.stride[0] - self.upsample.kernel_size[0] + + # Pre-calculating the sizes of noise to use so it's not dynamic + n_halves = [] + n_half = self.n_remaining_channels // 2 + for k in reversed(range(self.n_flows)): + n_halves.append(n_half) + if k % self.n_early_every == 0 and k > 0: + n_half = n_half + int(self.n_early_size / 2) + n_halves.reverse() + self.n_halves = n_halves + + self.removed_weightnorm = False + self.converted_to_2D = False + + def _prepare_for_export(self, **kwargs): + """ + Override this method to prepare module for export. This is in-place operation. + Base version does common necessary module replacements (Apex etc) + """ + self.remove_weightnorm() + if not self.converted_to_2D: + super()._prepare_for_export(replace_1D_2D=True, **kwargs) + self.converted_to_2D = True + + @typecheck() + def forward(self, spec, z=None, audio=None, run_inverse=True, sigma=1.0): + """ TODO + """ + if self.training and self.mode != OperationMode.training: + raise ValueError(f"{self} has self.training set to True but self.OperationMode was not set to training") + if not self.training and self.mode == OperationMode.training: + raise ValueError(f"{self} has self.training set to False but self.OperationMode was set to training") + + audio_pred = torch.zeros((1, 1)) + if audio is not None and self.mode != OperationMode.infer: + # audio_to_normal_dist is used to calculate loss so only run this in train or val model + z1, log_s_list, log_det_W_list = self.audio_to_normal_dist(spec=spec, audio=audio) + if run_inverse: + # norm_dist_to_audio is used to predict audio from spectrogram so only used in val or infer mode + # Could also log train audio but currently not done + audio_pred = self.norm_dist_to_audio(spec=spec, sigma=sigma, z=z) + + # Return the necessary tensors + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + return z1, log_s_list, log_det_W_list, audio_pred + return audio_pred + + @property + def input_types(self): + return { + "spec": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "z": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), + "audio": NeuralType(('B', 'T'), AudioSignal(), optional=True), + "run_inverse": NeuralType(elements_type=IntType(), optional=True), + "sigma": NeuralType(optional=True), + } + + @property + def output_types(self): + if self.mode == OperationMode.training or self.mode == OperationMode.validation: + return { + "pred_normal_dist": NeuralType(('B', 'flowgroup', 'T'), NormalDistributionSamplesType()), + "log_s_list": [NeuralType(('B', 'flowgroup', 'T'), VoidType())], # TODO: Figure out a good typing + "log_det_W_list": [NeuralType(elements_type=VoidType())], # TODO: Figure out a good typing + "audio_pred": NeuralType(('B', 'T'), AudioSignal()), + } + else: + return { + "audio": NeuralType(('B', 'T'), AudioSignal()), + } + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + par = next(self.parameters()) + mel = torch.randn((1, self.n_mel_channels, 96), device=par.device, dtype=par.dtype) + z = torch.randn( + (1, self.n_mel_channels, 96 * self.upsample.stride[0] // self.n_group), device=par.device, dtype=par.dtype + ) + return {"spec": mel, "z": z} + + def audio_to_normal_dist(self, *, spec: torch.Tensor, audio: torch.Tensor) -> (torch.Tensor, list, list): + # Upsample spectrogram to size of audio + spec = self.upsample(spec) + assert spec.size(2) >= audio.size(1) + if spec.size(2) > audio.size(1): + spec = spec[:, :, : audio.size(1)] + + spec = spec.unfold(2, self.n_group, self.n_group).permute(0, 2, 1, 3) + spec = spec.contiguous().view(spec.size(0), spec.size(1), -1) + spec = spec.permute(0, 2, 1) + + audio = audio.unfold(1, self.n_group, self.n_group).permute(0, 2, 1) + output_audio = [] + log_s_list = [] + log_det_W_list = [] + + for k in range(self.n_flows): + if k % self.n_early_every == 0 and k > 0: + output_audio.append(audio[:, : self.n_early_size, :]) + audio = audio[:, self.n_early_size :, :] + + audio, log_det_W = self.convinv[k](audio) + log_det_W_list.append(log_det_W) + + n_half = int(audio.size(1) / 2) + audio_0 = audio[:, :n_half, :] + audio_1 = audio[:, n_half:, :] + + output = self.wavenet[k]((audio_0, spec)) + log_s = output[:, n_half:, :] + b = output[:, :n_half, :] + audio_1 = torch.exp(log_s) * audio_1 + b + log_s_list.append(log_s) + + audio = torch.cat([audio_0, audio_1], 1) + + output_audio.append(audio) + return torch.cat(output_audio, 1), log_s_list, log_det_W_list + + def norm_dist_to_audio(self, *, spec, z=None, sigma: float = 1.0): + if self.converted_to_2D: + spec = torch.unsqueeze(spec, 3) + spec = self.upsample(spec) + spec = spec.contiguous().view(spec.size(0), spec.size(1), -1) + # trim conv artifacts. maybe pad spec to kernel multiple + if self.time_cutoff != 0: + spec = spec[:, :, : self.time_cutoff] + + spec = spec.unfold(2, self.n_group, self.n_group).permute(0, 2, 1, 3) + spec = spec.contiguous().view(spec.size(0), spec.size(1), -1) + spec = spec.permute(0, 2, 1) + + z_size = torch.Size([spec.size(0), self.n_group, spec.size(2)]) + if z is None: + z = sigma * torch.randn(z_size, device=spec.device).to(spec.dtype) + + if self.converted_to_2D: + z = torch.unsqueeze(z, 3) + spec = torch.unsqueeze(spec, 3) + + audio, z = torch.split(z, [self.n_remaining_channels, z.size(1) - self.n_remaining_channels], 1) + + for k in reversed(range(self.n_flows)): + n_half = self.n_halves[k] + audio_0, audio_1 = torch.split(audio, [n_half, audio.size(1) - n_half], 1) + + output = self.wavenet[k]((audio_0, spec)) + + b, s = torch.split(output, [n_half, output.size(1) - n_half], 1) + + audio_1 = audio_1 - b + audio_1 = audio_1 / torch.exp(s) + audio = torch.cat((audio_0, audio_1), 1) + + audio = self.convinv[k](audio, reverse=True) + if k % self.n_early_every == 0 and k > 0: + z1, z = torch.split(z, [self.n_early_size, z.size(1) - self.n_early_size], 1) + audio = torch.cat((z1, audio), 1) + + if self.converted_to_2D: + audio = audio.view(audio.size(0), audio.size(1), -1) + return audio.permute(0, 2, 1).contiguous().view(audio.size(0), -1) + + def remove_weightnorm(self): + if self.removed_weightnorm: + return + for wavenet in self.wavenet: + wavenet.start = torch.nn.utils.remove_weight_norm(wavenet.start) + wavenet.in_layers = remove(wavenet.in_layers) + wavenet.cond_layer = torch.nn.utils.remove_weight_norm(wavenet.cond_layer) + wavenet.res_skip_layers = remove(wavenet.res_skip_layers) + self.removed_weightnorm = True diff --git a/models/tacotron2/NeMo/nemo/constants.py b/models/tacotron2/NeMo/nemo/constants.py new file mode 100644 index 0000000000..f678ea13a2 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/constants.py @@ -0,0 +1,18 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +NEMO_ENV_VARNAME_ENABLE_COLORING = "NEMO_ENABLE_COLORING" +NEMO_ENV_VARNAME_REDIRECT_LOGS_TO_STDERR = "NEMO_REDIRECT_LOGS_TO_STDERR" +NEMO_ENV_VARNAME_TESTING = "NEMO_TESTING" # Set to True to enable nemo.util.logging's debug mode +NEMO_ENV_VARNAME_VERSION = "NEMO_EXPM_VERSION" # Used for nemo.utils.exp_manager versioning diff --git a/models/tacotron2/NeMo/nemo/core/__init__.py b/models/tacotron2/NeMo/nemo/core/__init__.py new file mode 100644 index 0000000000..44d13c9efd --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nemo.core.neural_types +from nemo.core.classes import * diff --git a/models/tacotron2/NeMo/nemo/core/classes/__init__.py b/models/tacotron2/NeMo/nemo/core/classes/__init__.py new file mode 100644 index 0000000000..f3ad63b38c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/classes/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.core.classes.common import ( + FileIO, + Model, + PretrainedModelInfo, + Serialization, + Typing, + is_typecheck_enabled, + typecheck, +) +from nemo.core.classes.dataset import Dataset, IterableDataset +from nemo.core.classes.exportable import Exportable, ExportFormat +from nemo.core.classes.loss import Loss +from nemo.core.classes.modelPT import ModelPT +from nemo.core.classes.module import NeuralModule diff --git a/models/tacotron2/NeMo/nemo/core/classes/common.py b/models/tacotron2/NeMo/nemo/core/classes/common.py new file mode 100644 index 0000000000..15e36b0507 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/classes/common.py @@ -0,0 +1,812 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Interfaces common to all Neural Modules and Models.""" +import hashlib +import traceback +from abc import ABC, abstractmethod +from contextlib import contextmanager +from dataclasses import dataclass, field +from enum import Enum +from functools import total_ordering +from pathlib import Path +from typing import Dict, List, Optional, Union + +import hydra +import wrapt +from omegaconf import DictConfig, OmegaConf + +import nemo +from nemo.core.neural_types import NeuralType, NeuralTypeComparisonResult +from nemo.utils import logging +from nemo.utils.cloud import maybe_download_from_cloud +from nemo.utils.model_utils import import_class_by_path, maybe_update_config_version + +__all__ = ['Typing', 'FileIO', 'Model', 'Serialization', 'typecheck'] + + +_TYPECHECK_ENABLED = True + + +def is_typecheck_enabled(): + """ + Getter method for typechecking state. + """ + return _TYPECHECK_ENABLED + + +@dataclass +class TypecheckMetadata: + """ + Metadata class for input/output neural types. + + # Primary attributes + original_types: Preserve the dictionary of type information provided. + + ignore_collections: For backward compatibility, container support can be disabled explicitly + using this flag. When set to True, all nesting is ignored and nest-depth checks are skipped. + + # Derived attributed + mandatory_types: Sub-dictionary of `original_types` which contains only those types which + are mandatory to include when calling the function. + + base_types: Dictionary of flattened `str: NeuralType` definitions, disregarding the nest level + details into appropriate arguments. + + container_depth: Dictionary mapping `str: int` - such that the valid depth of the nest of this + neural type is recorded. + + has_container_types: Bool flag declaring if any of the neural types declares a container nest + in its signature. + + is_singular_container_type: Bool flag declaring if this is a single Neural Type with a container + nest in its signature. Required for supporting python list expansion in return statement. + + """ + + original_types: Dict[str, NeuralType] + ignore_collections: bool + + mandatory_types: Dict[str, NeuralType] = field(init=False) + base_types: Dict[str, NeuralType] = field(init=False) + + container_depth: Dict[str, int] = field(init=False) + has_container_types: bool = field(init=False) + is_singular_container_type: bool = field(init=False) + + def __post_init__(self): + # If even one NeuralType declares a container nest, set to True + has_container_types = False + for type_val in self.original_types.values(): + if isinstance(type_val, (list, tuple)): + has_container_types = True + break + self.has_container_types = has_container_types + + # If only one NeuralType is declared, and it declares a container nest, set to True + if self.has_container_types and len(self.original_types) == 1: + self.is_singular_container_type = True + else: + self.is_singular_container_type = False + + # If container nests are declared, flatten the nest into `base_types` + # Also compute the nest depth for each of the NeuralTypes + if self.has_container_types: + self.base_types = {} + self.container_depth = {} + + for type_key, type_val in self.original_types.items(): + depth = 0 + while isinstance(type_val, (list, tuple)): + if len(type_val) > 1: + raise TypeError( + f"Neural Type `{type_key}`: {type_val} definition contains more than one element when" + "declaring the nested container structure.\n" + "Please ensure that you have only 1 NeuralType inside of the entire nested structure " + "definition." + ) + + type_val = type_val[0] + depth += 1 + + self.base_types[type_key] = type_val + self.container_depth[type_key] = depth + else: + # Otherwise, simply preserve the original_types and set depth of nest to 0. + self.base_types = self.original_types + self.container_depth = {type_key: 0 for type_key in self.base_types.keys()} + + # Compute subset of original_types which are mandatory in the call argspec + self.mandatory_types = { + type_key: type_val for type_key, type_val in self.base_types.items() if not type_val.optional + } + + +class Typing(ABC): + """ + An interface which endows module with neural types + """ + + @property + def input_types(self) -> Optional[Dict[str, NeuralType]]: + """Define these to enable input neural type checks""" + return None + + @property + def output_types(self) -> Optional[Dict[str, NeuralType]]: + """Define these to enable output neural type checks""" + return None + + def _validate_input_types(self, input_types=None, ignore_collections=False, **kwargs): + """ + This function does a few things. + 1) It ensures that len(self.input_types ) <= len(kwargs) <= len(self.input_types). + 2) For each (keyword name, keyword value) passed as input to the wrapped function: + - Check if the keyword name exists in the list of valid self.input_types names. + - Check if keyword value has the `neural_type` property. + - If it does, then perform a comparative check and assert that neural types + are compatible (SAME or GREATER). + - Check if keyword value is a container type (list or tuple). If yes, + then perform the elementwise test of neural type above on each element + of the nested structure, recursively. + + Args: + input_types: Either the `input_types` defined at class level, or the local function + overridden type definition. + ignore_collections: For backward compatibility, container support can be disabled explicitly + using this flag. When set to True, all nesting is ignored and nest-depth checks are skipped. + kwargs: Dictionary of argument_name:argument_value pairs passed to the wrapped + function upon call. + """ + # TODO: Properly implement this + if input_types is not None: + # Precompute metadata + metadata = TypecheckMetadata(original_types=input_types, ignore_collections=ignore_collections) + + total_input_types = len(input_types) + mandatory_input_types = len(metadata.mandatory_types) + + # Allow number of input arguments to be <= total input neural types. + if len(kwargs) < mandatory_input_types or len(kwargs) > total_input_types: + raise TypeError( + f"Number of input arguments provided ({len(kwargs)}) is not as expected. Function has " + f"{total_input_types} total inputs with {mandatory_input_types} mandatory inputs." + ) + + for key, value in kwargs.items(): + # Check if keys exists in the defined input types + if key not in input_types: + raise TypeError( + f"Input argument {key} has no corresponding input_type match. " + f"Existing input_types = {input_types.keys()}" + ) + + # Perform neural type check + if hasattr(value, 'neural_type') and not metadata.base_types[key].compare(value.neural_type) in ( + NeuralTypeComparisonResult.SAME, + NeuralTypeComparisonResult.GREATER, + ): + error_msg = [ + f"{input_types[key].compare(value.neural_type)} :", + f"Input type expected : {input_types[key]}", + f"Input type found : {value.neural_type}", + f"Argument: {key}", + ] + for i, dict_tuple in enumerate(metadata.base_types[key].elements_type.type_parameters.items()): + error_msg.insert(i + 2, f' input param_{i} : {dict_tuple[0]}: {dict_tuple[1]}') + for i, dict_tuple in enumerate(value.neural_type.elements_type.type_parameters.items()): + error_msg.append(f' input param_{i} : {dict_tuple[0]}: {dict_tuple[1]}') + raise TypeError("\n".join(error_msg)) + + # Perform input ndim check + if hasattr(value, 'shape'): + value_shape = value.shape + type_shape = metadata.base_types[key].axes + name = key + + if type_shape is not None and len(value_shape) != len(type_shape): + raise TypeError( + f"Input shape mismatch occured for {name} in module {self.__class__.__name__} : \n" + f"Input shape expected = {metadata.base_types[key].axes} | \n" + f"Input shape found : {value_shape}" + ) + + # Perform recursive neural type check for homogeneous elements + elif isinstance(value, list) or isinstance(value, tuple): + for ind, val in enumerate(value): + """ + This initiates a DFS, tracking the depth count as it goes along the nested structure. + Initial depth is 1 as we consider the current loop to be the 1st step inside the nest. + """ + self.__check_neural_type(val, metadata, depth=1, name=key) + + def _attach_and_validate_output_types(self, out_objects, ignore_collections=False, output_types=None): + """ + This function does a few things. + 1) It ensures that len(out_object) == len(self.output_types). + 2) If the output is a tensor (or list/tuple of list/tuple ... of tensors), it + attaches a neural_type to it. For objects without the neural_type attribute, + such as python objects (dictionaries and lists, primitive data types, structs), + no neural_type is attached. + + Note: tensor.neural_type is only checked during _validate_input_types which is + called prior to forward(). + + Args: + output_types: Either the `output_types` defined at class level, or the local function + overridden type definition. + ignore_collections: For backward compatibility, container support can be disabled explicitly + using this flag. When set to True, all nesting is ignored and nest-depth checks are skipped. + out_objects: The outputs of the wrapped function. + """ + # TODO: Properly implement this + if output_types is not None: + # Precompute metadata + metadata = TypecheckMetadata(original_types=output_types, ignore_collections=ignore_collections) + out_types_list = list(metadata.base_types.items()) + + # First convert all outputs to list/tuple format to check correct number of outputs + if type(out_objects) in (list, tuple): + out_container = out_objects # can be any rank nested structure + else: + out_container = [out_objects] + + # If this neural type has a *single output*, with *support for nested outputs*, + # then *do not* perform any check on the number of output items against the number + # of neural types (in this case, 1). + # This is done as python will *not* wrap a single returned list into a tuple of length 1, + # instead opting to keep the list intact. Therefore len(out_container) in such a case + # is the length of all the elements of that list - each of which has the same corresponding + # neural type (defined as the singular container type). + if metadata.is_singular_container_type: + pass + + # In all other cases, python will wrap multiple outputs into an outer tuple. + # As such, now the number of elements in this outer tuple should exactly match + # the number of output types defined. + elif len(out_types_list) != len(out_container): + raise TypeError( + "Number of output arguments provided ({}) is not as expected ({}).\n" + "This can be either because insufficient number of output NeuralTypes were provided," + "or the provided NeuralTypes {} should enable container support " + "(add '[]' to the NeuralType definition)".format( + len(out_container), len(output_types), output_types + ) + ) + + # Attach types recursively, if possible + if not isinstance(out_objects, tuple) and not isinstance(out_objects, list): + # Here, out_objects is a single object which can potentially be attached with a NeuralType + try: + out_objects.neural_type = out_types_list[0][1] + except Exception: + pass + + # Perform output ndim check + if hasattr(out_objects, 'shape'): + value_shape = out_objects.shape + type_shape = out_types_list[0][1].axes + name = out_types_list[0][0] + + if type_shape is not None and len(value_shape) != len(type_shape): + raise TypeError( + f"Output shape mismatch occured for {name} in module {self.__class__.__name__} : \n" + f"Output shape expected = {type_shape} | \n" + f"Output shape found : {value_shape}" + ) + + elif metadata.is_singular_container_type: + # If only a single neural type is provided, and it defines a container nest, + # then all elements of the returned list/tuple are assumed to belong to that + # singular neural type. + # As such, the "current" depth inside the DFS loop is counted as 1, + # and subsequent nesting will increase this count. + + # NOTE: + # As the flag `is_singular_container_type` will activate only for + # the case where there is 1 output type defined with container nesting, + # this is a safe assumption to make. + depth = 1 + + # NOTE: + # A user may chose to explicitly wrap the single output list within an explicit tuple + # In such a case we reduce the "current" depth to 0 - to acknowledge the fact that + # the actual nest exists within a wrapper tuple. + if len(out_objects) == 1 and type(out_objects) == tuple: + depth = 0 + + for ind, res in enumerate(out_objects): + self.__attach_neural_type(res, metadata, depth=depth, name=out_types_list[0][0]) + else: + # If more then one item is returned in a return statement, python will wrap + # the output with an outer tuple. Therefore there must be a 1:1 correspondence + # of the output_neural type (with or without nested structure) to the actual output + # (whether it is a single object or a nested structure of objects). + # Therefore in such a case, we "start" the DFS at depth 0 - since the recursion is + # being applied on 1 neural type : 1 output struct (single or nested output). + # Since we are guarenteed that the outer tuple will be built by python, + # assuming initial depth of 0 is appropriate. + for ind, res in enumerate(out_objects): + self.__attach_neural_type(res, metadata, depth=0, name=out_types_list[ind][0]) + + def __check_neural_type(self, obj, metadata, depth, name=None): + if isinstance(obj, tuple) or isinstance(obj, list): + for elem in obj: + self.__check_neural_type(elem, metadata, depth + 1, name=name) + return # after processing nest, return to avoid testing nest itself + + type_val = metadata.base_types[name] + + # If nest depth doesnt match neural type structure depth, raise an error + if not metadata.ignore_collections and depth != metadata.container_depth[name]: + raise TypeError( + "While checking input neural types,\n" + "Nested depth of value did not match container specification:\n" + f"Current nested depth of NeuralType '{name}' ({type_val}): {depth}\n" + f"Expected nested depth : {metadata.container_depth[name]}" + ) + + if hasattr(obj, 'neural_type') and not type_val.compare(obj.neural_type) in ( + NeuralTypeComparisonResult.SAME, + NeuralTypeComparisonResult.GREATER, + ): + raise TypeError( + f"{type_val.compare(obj.neural_type)} : \n" + f"Input type expected = {type_val} | \n" + f"Input type found : {obj.neural_type}" + ) + + # Perform input ndim check + if hasattr(obj, 'shape'): + value_shape = obj.shape + type_shape = type_val.axes + + if type_shape is not None and len(value_shape) != len(type_shape): + raise TypeError( + f"Input shape mismatch occured for {name} in module {self.__class__.__name__} : \n" + f"Input shape expected = {type_shape} | \n" + f"Input shape found : {value_shape}" + ) + + def __attach_neural_type(self, obj, metadata, depth, name=None): + if isinstance(obj, tuple) or isinstance(obj, list): + for elem in obj: + self.__attach_neural_type(elem, metadata, depth=depth + 1, name=name) + return # after processing nest, return to avoid argument insertion into nest itself + + type_val = metadata.base_types[name] + + # If nest depth doesnt match neural type structure depth, raise an error + if not metadata.ignore_collections and depth != metadata.container_depth[name]: + raise TypeError( + "While attaching output neural types,\n" + "Nested depth of value did not match container specification:\n" + f"Current nested depth of NeuralType '{name}' ({type_val}): {depth}\n" + f"Expected nested depth : {metadata.container_depth[name]}" + ) + + try: + obj.neural_type = type_val + except Exception: + pass + + # Perform output ndim check + if hasattr(obj, 'shape'): + value_shape = obj.shape + type_shape = type_val.axes + + if type_shape is not None and len(value_shape) != len(type_shape): + raise TypeError( + f"Output shape mismatch occured for {name} in module {self.__class__.__name__} : \n" + f"Output shape expected = {type_shape} | \n" + f"Output shape found : {value_shape}" + ) + + +class Serialization(ABC): + @classmethod + def from_config_dict(cls, config: DictConfig): + """Instantiates object using DictConfig-based configuration""" + # Resolve the config dict + if isinstance(config, DictConfig): + config = OmegaConf.to_container(config, resolve=True) + config = OmegaConf.create(config) + OmegaConf.set_struct(config, True) + + config = maybe_update_config_version(config) + + # Hydra 0.x API + if ('cls' in config or 'target' in config) and 'params' in config: + # regular hydra-based instantiation + instance = hydra.utils.instantiate(config=config) + # Hydra 1.x API + elif '_target_' in config: + # regular hydra-based instantiation + instance = hydra.utils.instantiate(config=config) + else: + instance = None + imported_cls_tb = None + # Attempt class path resolution from config `target` class (if it exists) + if 'target' in config: + target_cls = config.target + imported_cls = None + try: + # try to import the target class + imported_cls = import_class_by_path(target_cls) + except Exception: + imported_cls_tb = traceback.format_exc() + + # try instantiating model with target class + if imported_cls is not None: + # if calling class (cls) is subclass of imported class, + # use subclass instead + if issubclass(cls, imported_cls): + imported_cls = cls + + try: + instance = imported_cls(cfg=config) + except Exception: + imported_cls_tb = traceback.format_exc() + instance = None + + # target class resolution was unsuccessful, fall back to current `cls` + if instance is None: + if imported_cls_tb is not None: + logging.debug( + f"Model instantiation from target class {target_cls} failed with following error.\n" + f"Falling back to `cls`.\n" + f"{imported_cls_tb}" + ) + instance = cls(cfg=config) + + if not hasattr(instance, '_cfg'): + instance._cfg = config + return instance + + def to_config_dict(self) -> DictConfig: + """Returns object's configuration to config dictionary""" + if hasattr(self, '_cfg') and self._cfg is not None and isinstance(self._cfg, DictConfig): + # Resolve the config dict + config = OmegaConf.to_container(self._cfg, resolve=True) + config = OmegaConf.create(config) + OmegaConf.set_struct(config, True) + + config = maybe_update_config_version(config) + + self._cfg = config + + return self._cfg + else: + raise NotImplementedError( + 'to_config_dict() can currently only return object._cfg but current object does not have it.' + ) + + +class FileIO(ABC): + def save_to(self, save_path: str): + """Saves module/model with weights""" + raise NotImplementedError() + + @classmethod + def restore_from( + cls, + restore_path: str, + override_config_path: Optional[str] = None, + map_location: Optional['torch.device'] = None, + strict: bool = True, + return_config: bool = False, + ): + """Restores module/model with weights""" + raise NotImplementedError() + + @classmethod + def from_config_file(cls, path2yaml_file: str): + """ + Instantiates an instance of NeMo Model from YAML config file. + Weights will be initialized randomly. + Args: + path2yaml_file: path to yaml file with model configuration + + Returns: + + """ + if issubclass(cls, Serialization): + conf = OmegaConf.load(path2yaml_file) + return cls.from_config_dict(config=conf) + else: + raise NotImplementedError() + + def to_config_file(self, path2yaml_file: str): + """ + Saves current instance's configuration to YAML config file. Weights will not be saved. + Args: + path2yaml_file: path2yaml_file: path to yaml file where model model configuration will be saved + + Returns: + """ + if hasattr(self, '_cfg'): + self._cfg = maybe_update_config_version(self._cfg) + with open(path2yaml_file, 'w') as fout: + OmegaConf.save(config=self._cfg, f=fout, resolve=True) + else: + raise NotImplementedError() + + +@total_ordering +@dataclass +class PretrainedModelInfo: + pretrained_model_name: str + description: str + location: str + class_: 'Model' = None + aliases: List[str] = None + + def __repr__(self): + base = self.__class__.__name__ + extras = ( + "pretrained_model_name={pretrained_model_name},\n\t" + "description={description},\n\t" + "location={location}".format(**self.__dict__) + ) + + if self.class_ is not None: + extras = "{extras},\n\t" "class_={class_}".format(extras=extras, **self.__dict__) + + representation = f"{base}(\n\t{extras}\n)" + return representation + + def __hash__(self): + # assumes that locations are unique urls, and therefore their hashes + # should ideally also be unique + location_hash = hash(self.location) + return location_hash + + def __eq__(self, other): + # another object is equal to self, iff + # if it's hash is equal to hash(self) + return hash(self) == hash(other) or self.pretrained_model_name == other.pretrained_model_name + + def __lt__(self, other): + return self.pretrained_model_name < other.pretrained_model_name + + +class Model(Typing, Serialization, FileIO): + """ + Abstract class offering interface which should be implemented by all NeMo models. + """ + + @classmethod + @abstractmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + Should list all pre-trained models available via NVIDIA NGC cloud. + Note: There is no check that requires model names and aliases to be unique. In the case of a collIsion, whatever + model (or alias) is listed first in the this returned list will be instantiated. + + Returns: + A list of PretrainedModelInfo entries + """ + pass + + @classmethod + def get_available_model_names(cls) -> List[str]: + """ + Returns the list of model names available via NVIDIA NGC cloud, + to get the complete model description use list_available_models() + Returns: + A list of model names + """ + model_names = [] + if cls.list_available_models() is not None: + model_names = [model.pretrained_model_name for model in cls.list_available_models()] + return model_names + + @classmethod + def from_pretrained( + cls, + model_name: str, + refresh_cache: bool = False, + override_config_path: Optional[str] = None, + map_location: Optional['torch.device'] = None, + strict: bool = True, + return_config: bool = False, + ): + """ + Instantiates an instance of NeMo from NVIDIA NGC cloud + Use restore_from() to instantiate from a local .nemo file. + Args: + model_name: string key which will be used to find the module. + refresh_cache: If set to True, then when fetching from cloud, this will re-fetch the file + from cloud even if it is already found in a cache locally. + override_config_path: path to a yaml config that will override the internal + config file + map_location: Optional torch.device() to map the instantiated model to a device. + By default (None), it will select a GPU if available, falling back to CPU otherwise. + strict: Passed to torch.load_state_dict. By default true. + return_config: If set to true, will return just the underlying config of the restored + model as an OmegaConf DictConfig object without instantiating the model. + + Returns: + A model instance of a particular model class or its underlying config (if return_config is set). + """ + location_in_the_cloud = None + description = None + models = cls.list_available_models() + if models is not None: + for pretrained_model_info in cls.list_available_models(): + found = False + if pretrained_model_info.pretrained_model_name == model_name: + found = True + elif pretrained_model_info.aliases is not None: + for alias in pretrained_model_info.aliases: + if alias == model_name: + found = True + break + if found: + location_in_the_cloud = pretrained_model_info.location + description = pretrained_model_info.description + class_ = pretrained_model_info.class_ + break + + if location_in_the_cloud is None: + raise FileNotFoundError( + f"Model {model_name} was not found. Check cls.list_available_models() for the list of all available models." + ) + filename = location_in_the_cloud.split("/")[-1] + url = location_in_the_cloud.replace(filename, "") + cache_dir = Path.joinpath(Path.home(), f'.cache/torch/NeMo/NeMo_{nemo.__version__}/{filename[:-5]}') + # If either description and location in the cloud changes, this will force re-download + cache_subfolder = hashlib.md5((location_in_the_cloud + description).encode('utf-8')).hexdigest() + # if file exists on cache_folder/subfolder, it will be re-used, unless refresh_cache is True + nemo_model_file_in_cache = maybe_download_from_cloud( + url=url, filename=filename, cache_dir=cache_dir, subfolder=cache_subfolder, refresh_cache=refresh_cache + ) + logging.info("Instantiating model from pre-trained checkpoint") + if class_ is None: + class_ = cls + instance = class_.restore_from( + restore_path=nemo_model_file_in_cache, + override_config_path=override_config_path, + map_location=map_location, + strict=strict, + return_config=return_config, + ) + return instance + + +class typecheck: + class TypeState(Enum): + """ + Placeholder to denote the default value of type information provided. + If the constructor of this decorator is used to override the class level type definition, + this enum value indicate that types will be overridden. + """ + + UNINITIALIZED = 0 + + def __init__( + self, + input_types: Union[TypeState, Dict[str, NeuralType]] = TypeState.UNINITIALIZED, + output_types: Union[TypeState, Dict[str, NeuralType]] = TypeState.UNINITIALIZED, + ignore_collections: bool = False, + ): + """ + A decorator which performs input-output neural type checks, and attaches + neural types to the output of the function that it wraps. + + Requires that the class inherit from `nemo.core.Typing` in order to perform + type checking, and will raise an error if that is not the case. + + # Usage (Class level type support) + @typecheck() + def fn(self, arg1, arg2, ...): + ... + + # Usage (Function level type support) + @typecheck(input_types=..., output_types=...) + def fn(self, arg1, arg2, ...): + ... + + Points to be noted: + 1) The brackets () in `@typecheck()` are necessary. + + You will encounter a TypeError: __init__() takes 1 positional argument but X + were given without those brackets. + + 2) The function can take any number of positional arguments during definition. + + When you call this function, all arguments must be passed using kwargs only. + + """ + self.input_types = input_types + self.output_types = output_types + + if input_types == self.TypeState.UNINITIALIZED: + self.input_override = False + else: + self.input_override = True + + if output_types == self.TypeState.UNINITIALIZED: + self.output_override = False + else: + self.output_override = True + + self.ignore_collections = ignore_collections + + @wrapt.decorator(enabled=is_typecheck_enabled) + def __call__(self, wrapped, instance: Typing, args, kwargs): + if instance is None: + raise RuntimeError("Only classes which inherit nemo.core.Typing can use this decorator !") + + if not isinstance(instance, Typing): + raise RuntimeError("Only classes which inherit nemo.core.Typing can use this decorator !") + + if hasattr(instance, 'input_ports') or hasattr(instance, 'output_ports'): + raise RuntimeError( + "Typing requires override of `input_types()` and `output_types()`, " + "not `input_ports() and `output_ports()`" + ) + + # Preserve type information + if self.input_types is typecheck.TypeState.UNINITIALIZED: + self.input_types = instance.input_types + + if self.output_types is typecheck.TypeState.UNINITIALIZED: + self.output_types = instance.output_types + + # Resolve global type or local overridden type + if self.input_override: + input_types = self.input_types + else: + input_types = instance.input_types + + if self.output_override: + output_types = self.output_types + else: + output_types = instance.output_types + + # If types are not defined, skip type checks and just call the wrapped method + if input_types is None and output_types is None: + return wrapped(*args, **kwargs) + + # Check that all arguments are kwargs + if input_types is not None and len(args) > 0: + raise TypeError("All arguments must be passed by kwargs only for typed methods") + + # Perform rudimentary input checks here + instance._validate_input_types(input_types=input_types, ignore_collections=self.ignore_collections, **kwargs) + + # Call the method - this can be forward, or any other callable method + outputs = wrapped(*args, **kwargs) + + instance._attach_and_validate_output_types( + output_types=output_types, ignore_collections=self.ignore_collections, out_objects=outputs + ) + + return outputs + + @staticmethod + def set_typecheck_enabled(enabled: bool = True): + global _TYPECHECK_ENABLED + _TYPECHECK_ENABLED = enabled + + @staticmethod + @contextmanager + def disable_checks(): + typecheck.set_typecheck_enabled(enabled=False) + try: + yield + finally: + typecheck.set_typecheck_enabled(enabled=True) diff --git a/models/tacotron2/NeMo/nemo/core/classes/dataset.py b/models/tacotron2/NeMo/nemo/core/classes/dataset.py new file mode 100644 index 0000000000..51bd46ef79 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/classes/dataset.py @@ -0,0 +1,109 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Optional + +from torch.utils import data + +from nemo.core.classes import Serialization, Typing, typecheck + +__all__ = ['Dataset', 'IterableDataset'] + + +class Dataset(data.Dataset, Typing, Serialization): + """Dataset with output ports + + Please Note: Subclasses of IterableDataset should *not* implement input_types. + """ + + def _collate_fn(self, batch): + """ + A default implementation of a collation function. + Users should override this method to define custom data loaders. + """ + return data.dataloader.default_collate(batch) + + @typecheck() + def collate_fn(self, batch): + """ + This is the method that user pass as functor to DataLoader. + The method optionally performs neural type checking and add types to the outputs. + + Please note, subclasses of Dataset should not implement `input_types`. + + # Usage: + dataloader = torch.utils.data.DataLoader( + ...., + collate_fn=dataset.collate_fn, + .... + ) + + Returns: + Collated batch, with or without types. + """ + if self.input_types is not None: + raise TypeError("Datasets should not implement `input_types` as they are not checked") + + # Simply forward the inner `_collate_fn` + return self._collate_fn(batch) + + +class IterableDataset(data.IterableDataset, Typing, Serialization): + """Iterable Dataset with output ports + + Please Note: Subclasses of IterableDataset should *not* implement input_types. + """ + + def _collate_fn(self, batch): + """ + A default implementation of a collation function. + Users should override this method to define custom data loaders. + """ + return data.dataloader.default_collate(batch) + + @typecheck() + def collate_fn(self, batch): + """ + This is the method that user pass as functor to DataLoader. + The method optionally performs neural type checking and add types to the outputs. + + # Usage: + dataloader = torch.utils.data.DataLoader( + ...., + collate_fn=dataset.collate_fn, + .... + ) + + Returns: + Collated batch, with or without types. + """ + if self.input_types is not None: + raise TypeError("Datasets should not implement `input_types` as they are not checked") + + # Simply forward the inner `_collate_fn` + return self._collate_fn(batch) + + +@dataclass +class DatasetConfig: + """ + + """ + + # ... + batch_size: int = 32 + drop_last: bool = False + shuffle: bool = False + num_workers: Optional[int] = None + pin_memory: bool = True diff --git a/models/tacotron2/NeMo/nemo/core/classes/exportable.py b/models/tacotron2/NeMo/nemo/core/classes/exportable.py new file mode 100644 index 0000000000..36cab8b406 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/classes/exportable.py @@ -0,0 +1,308 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from abc import ABC +from collections import defaultdict +from enum import Enum +from typing import Dict + +import onnx +import torch + +from nemo.core.classes import typecheck +from nemo.core.neural_types import AxisKind, NeuralType +from nemo.utils import logging +from nemo.utils.export_utils import replace_for_export + +__all__ = ['ExportFormat', 'Exportable'] + + +class ExportFormat(Enum): + """Which format to use when exporting a Neural Module for deployment""" + + ONNX = (1,) + TORCHSCRIPT = (2,) + + +_EXT_DICT = { + ".pt": ExportFormat.TORCHSCRIPT, + ".ts": ExportFormat.TORCHSCRIPT, + ".onnx": ExportFormat.ONNX, +} + + +def get_input_names(self): + if not (hasattr(self, 'input_types')): + raise NotImplementedError('For export to work you must define input_types') + input_names = list(self.input_types.keys()) + return input_names + + +def get_output_names(self): + if not (hasattr(self, 'output_types')): + raise NotImplementedError('For export to work you must define output_types') + output_names = list(self.output_types.keys()) + return output_names + + +def get_input_dynamic_axes(self, input_names): + dynamic_axes = defaultdict(list) + for name in input_names: + dynamic_axes = { + **dynamic_axes, + **Exportable._extract_dynamic_axes(name, self.input_types[name]), + } + return dynamic_axes + + +def get_output_dynamic_axes(self, output_names): + dynamic_axes = defaultdict(list) + for name in output_names: + dynamic_axes = { + **dynamic_axes, + **Exportable._extract_dynamic_axes(name, self.output_types[name]), + } + return dynamic_axes + + +def to_onnxrt_input(input_names, input_list, input_dict): + odict = {} + for k, v in input_dict.items(): + odict[k] = v.cpu().numpy() + for i, input in enumerate(input_list): + odict[input_names[i]] = input.cpu().numpy() + return odict + + +class Exportable(ABC): + """ + This Interface should be implemented by particular classes derived from nemo.core.NeuralModule or nemo.core.ModelPT. + It gives these entities ability to be exported for deployment to formats such as ONNX. + """ + + @staticmethod + def get_format(filename: str): + _, ext = os.path.splitext(filename) + try: + return _EXT_DICT[ext] + except KeyError: + raise ValueError(f"Export file {filename} extension does not correspond to any export format!") + + @property + def input_module(self): + return self + + @property + def output_module(self): + return self + + def export( + self, + output: str, + input_example=None, + output_example=None, + verbose=False, + export_params=True, + do_constant_folding=True, + keep_initializers_as_inputs=False, + onnx_opset_version: int = 13, + try_script: bool = False, + set_eval: bool = True, + check_trace: bool = False, + use_dynamic_axes: bool = True, + dynamic_axes=None, + check_tolerance=0.01, + ): + my_args = locals() + del my_args['self'] + + qual_name = self.__module__ + '.' + self.__class__.__qualname__ + output_descr = qual_name + ' exported to ONNX' + + try: + # Disable typechecks + typecheck.set_typecheck_enabled(enabled=False) + + # Allow user to completely override forward method to export + if hasattr(type(self), "forward_for_export"): + forward_method = type(self).forward_for_export + old_forward_method = type(self).forward + type(self).forward = forward_method + else: + forward_method = None + + # Set module to eval mode + if set_eval: + self.freeze() + self.input_module.freeze() + self.output_module.freeze() + + format = self.get_format(output) + + if input_example is None: + input_example = self.input_module.input_example() + + my_args['input_example'] = input_example + + # Run (posibly overridden) prepare method before calling forward() + self._prepare_for_export(**my_args) + + input_list = list(input_example) + input_dict = {} + # process possible kwargs + if isinstance(input_list[-1], dict): + input_dict = input_list[-1] + input_list = input_list[:-1] + + input_names = get_input_names(self.input_module) + # remove unnecessary inputs for input_ports + for name in self.disabled_deployment_input_names: + input_names.remove(name) + output_names = get_output_names(self.output_module) + # remove unnecessary inputs for input_ports + for name in self.disabled_deployment_output_names: + output_names.remove(name) + + output_example = self.forward(*input_list, **input_dict) + + with torch.jit.optimized_execution(True), torch.no_grad(): + jitted_model = None + if try_script: + try: + jitted_model = torch.jit.script(self) + except Exception as e: + print("jit.script() failed!", e) + + if format == ExportFormat.TORCHSCRIPT: + if jitted_model is None: + jitted_model = torch.jit.trace_module( + self, + {"forward": tuple(input_list) + tuple(input_dict.values())}, + strict=False, + optimize=True, + check_trace=check_trace, + check_tolerance=check_tolerance, + ) + if verbose: + print(jitted_model.code) + jitted_model.save(output) + assert os.path.exists(output) + + elif format == ExportFormat.ONNX: + if jitted_model is None: + jitted_model = self + + # dynamic axis is a mapping from input/output_name => list of "dynamic" indices + if dynamic_axes is None and use_dynamic_axes: + dynamic_axes = get_input_dynamic_axes(self.input_module, input_names) + dynamic_axes = {**dynamic_axes, **get_output_dynamic_axes(self.output_module, output_names)} + + torch.onnx.export( + jitted_model, + input_example, + output, + input_names=input_names, + output_names=output_names, + verbose=verbose, + export_params=export_params, + do_constant_folding=do_constant_folding, + keep_initializers_as_inputs=keep_initializers_as_inputs, + dynamic_axes=dynamic_axes, + opset_version=onnx_opset_version, + example_outputs=output_example, + ) + + # Verify the model can be read, and is valid + onnx_model = onnx.load(output) + onnx.checker.check_model(onnx_model, full_check=True) + test_runtime = check_trace + if test_runtime: + try: + import onnxruntime + except (ImportError, ModuleNotFoundError): + test_runtime = False + logging.warning( + f"ONNX generated at {output}, not verified - please install onnxruntime.\n" + ) + + if test_runtime: + sess = onnxruntime.InferenceSession(onnx_model.SerializeToString()) + + ort_out = sess.run(None, to_onnxrt_input(input_names, input_list, input_dict)) + all_good = True + for out_name, out in enumerate(ort_out): + expected = output_example[out_name].cpu() + if not torch.allclose( + torch.from_numpy(out), expected, rtol=check_tolerance, atol=100 * check_tolerance + ): + all_good = False + logging.info( + f"onnxruntime results mismatch! PyTorch(expected):\n{expected}\nONNXruntime:\n{out}" + ) + status = "SUCCESS" if all_good else "FAIL" + logging.info(f"ONNX generated at {output} verified with onnxruntime : " + status) + else: + raise ValueError(f'Encountered unknown export format {format}.') + finally: + typecheck.set_typecheck_enabled(enabled=True) + if forward_method: + type(self).forward = old_forward_method + return ([output], [output_descr]) + + @property + def disabled_deployment_input_names(self): + """Implement this method to return a set of input names disabled for export""" + return set() + + @property + def disabled_deployment_output_names(self): + """Implement this method to return a set of output names disabled for export""" + return set() + + @property + def supported_export_formats(self): + """Implement this method to return a set of export formats supported. Default is all types.""" + return set([ExportFormat.ONNX, ExportFormat.TORCHSCRIPT]) + + @staticmethod + def _extract_dynamic_axes(name: str, ntype: NeuralType): + """ + Implement this method to provide dynamic axes id for ONNX export. + By default, this method will extract BATCH and TIME dimension ids from each provided input/output name argument. + + For example, if module/model accepts argument named "input_signal" with type corresponding to [Batch, Time, Dim] + shape, then the returned result should contain "input_signal" -> [0, 1] because Batch and Time are dynamic axes + as they can change from call to call during inference. + + Args: + name: Name of input or output parameter + ntype: Corresponding Neural Type + + Returns: + + """ + dynamic_axes = defaultdict(list) + if ntype.axes: + for ind, axis in enumerate(ntype.axes): + if axis.kind in [AxisKind.Batch, AxisKind.Time, AxisKind.Width, AxisKind.Height]: + dynamic_axes[name].append(ind) + return dynamic_axes + + def _prepare_for_export(self, **kwargs): + """ + Override this method to prepare module for export. This is in-place operation. + Base version does common necessary module replacements (Apex etc) + """ + replace_1D_2D = kwargs.get('replace_1D_2D', False) + replace_for_export(self, replace_1D_2D) diff --git a/models/tacotron2/NeMo/nemo/core/classes/loss.py b/models/tacotron2/NeMo/nemo/core/classes/loss.py new file mode 100644 index 0000000000..d6ede4049c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/classes/loss.py @@ -0,0 +1,26 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from nemo.core.classes.common import Serialization, Typing + +__all__ = ['Loss'] + + +class Loss(torch.nn.modules.loss._Loss, Typing, Serialization): + """Inherit this class to implement custom loss.""" + + def __init__(self, **kwargs): + super(Loss, self).__init__(**kwargs) diff --git a/models/tacotron2/NeMo/nemo/core/classes/modelPT.py b/models/tacotron2/NeMo/nemo/core/classes/modelPT.py new file mode 100644 index 0000000000..7b8162a035 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/classes/modelPT.py @@ -0,0 +1,1419 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import hashlib +import inspect +import os +import shutil +import tarfile +import tempfile +import uuid +from abc import abstractmethod +from os import path +from os.path import expanduser +from typing import Callable, Dict, List, Optional, Union + +import hydra +import torch +from omegaconf import DictConfig, OmegaConf, open_dict +from pytorch_lightning import LightningModule, Trainer +from pytorch_lightning.utilities import rank_zero_only + +import nemo +from nemo.core import optim +from nemo.core.classes.common import Model +from nemo.core.optim import prepare_lr_scheduler +from nemo.utils import logging, model_utils +from nemo.utils.app_state import AppState +from nemo.utils.get_rank import is_global_rank_zero + +# Need to set them before EFF import as it is using them. +_MODEL_CONFIG_YAML = "model_config.yaml" +_MODEL_WEIGHTS = "model_weights.ckpt" + +__all__ = ['ModelPT'] + +""" +Internal global flags that determine core functionality of ModelPT. + +_MODEL_IS_RESTORED: + This flag determines the context of the model - whether the model is currently being + restored or not. + - When set, it can be assumed that the model's will disable all automatic methods - + setup_training_data(), setup_validation/test_data() and their multi equivalents. + - If a model is being restored from a archive file (tarfile), it can be assumed that + under this context, the cwd is *inside* the tarfile itself. + +_MODEL_RESTORE_PATH: + A string path to a a file from which the model is being restored. + This file can either be a PyTorch Lightning Checkpoint, or a archive (tarfile) that contains + artifact objects. + If it is an archive file, during restoration, the cwd will be temporarily moved to inside the + archive itself. +""" +_MODEL_IS_RESTORED = False +_NEMO_FILE_FOLDER = None + + +class ModelPT(LightningModule, Model): + """ + Interface for Pytorch-lightning based NeMo models + """ + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + """ + Base class from which all NeMo models should inherit + + Args: + cfg (DictConfig): configuration object. + The cfg object should have (optionally) the following sub-configs: + + * train_ds - to instantiate training dataset + * validation_ds - to instantiate validation dataset + * test_ds - to instantiate testing dataset + * optim - to instantiate optimizer with learning rate scheduler + + trainer (Optional): Pytorch Lightning Trainer instance + """ + if trainer is not None and not isinstance(trainer, Trainer): + raise ValueError( + f"trainer constructor argument must be either None or pytroch_lightning.Trainer. But got {type(trainer)} instead." + ) + super().__init__() + + # Convert config to a DictConfig + cfg = model_utils.convert_model_config_to_dict_config(cfg) + + # Convert config to support Hydra 1.0+ instantiation + cfg = model_utils.maybe_update_config_version(cfg) + + if 'target' not in cfg: + # This is for Jarvis service. + OmegaConf.set_struct(cfg, False) + cfg.target = "{0}.{1}".format(self.__class__.__module__, self.__class__.__name__) + OmegaConf.set_struct(cfg, True) + + self._cfg = cfg + + self.save_hyperparameters(self._cfg) + self._train_dl = None + self._validation_dl = None + self._test_dl = None + self._optimizer = None + self._scheduler = None + self.trainer = trainer # reference required for self.*_rank + self._trainer = self.trainer # alias for backward compatibility + + self._set_model_guid() + + # Set device_id in AppState + if torch.cuda.is_available() and torch.cuda.current_device() is not None: + app_state = AppState() + app_state.device_id = torch.cuda.current_device() + + if self._cfg is not None and not self._is_model_being_restored(): + if 'train_ds' in self._cfg and self._cfg.train_ds is not None: + self.setup_training_data(self._cfg.train_ds) + + if 'validation_ds' in self._cfg and self._cfg.validation_ds is not None: + self.setup_multiple_validation_data(val_data_config=None) + + if 'test_ds' in self._cfg and self._cfg.test_ds is not None: + self.setup_multiple_test_data(test_data_config=None) + + else: + if 'train_ds' in self._cfg and self._cfg.train_ds is not None: + logging.warning( + f"If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method " + f"and provide a valid configuration file to setup the train data loader.\n" + f"Train config : \n{OmegaConf.to_yaml(self._cfg.train_ds)}" + ) + + if 'validation_ds' in self._cfg and self._cfg.validation_ds is not None: + logging.warning( + f"If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method " + f"and provide a valid configuration file to setup the validation data loader(s). \n" + f"Validation config : \n{OmegaConf.to_yaml(self._cfg.validation_ds)}" + ) + + if 'test_ds' in self._cfg and self._cfg.test_ds is not None: + logging.warning( + f"Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method " + f"and provide a valid configuration file to setup the test data loader(s).\n" + f"Test config : \n{OmegaConf.to_yaml(self._cfg.test_ds)}" + ) + + # ModelPT wrappers over subclass implementations + self.training_step = model_utils.wrap_training_step(self.training_step) + + def register_artifact(self, config_path: str, src: str, verify_src_exists: bool = True): + """ Register model artifacts with this function. These artifacts (files) will be included inside .nemo file + when model.save_to("mymodel.nemo") is called. + + How it works: + 1. It always returns existing absolute path which can be used during Model constructor call + EXCEPTION: src is None or "" in which case nothing will be done and src will be returned + 2. It will add (config_path, model_utils.ArtifactItem()) pair to self.artifacts + + If "src" is local existing path, then it will be returned in absolute path form. + elif "src" starts with "nemo_file:unique_artifact_name": + .nemo will be untarred to a temporary folder location and an actual existing path will be returned + else an error will be raised. + + WARNING: use .register_artifact calls in your models' constructors. + The returned path is not guaranteed to exist after you have exited your model's constuctor. + + Args: + config_path (str): Artifact key. Usually corresponds to the model config. + src (str): Path to artifact. + verify_src_exists (bool): If set to False, then the artifact is optional and register_artifact will return None even if + src is not found. Defaults to True. + + Returns: + str: If src is not None or empty it always returns absolute path which is guaranteed to exists during model instnce life + """ + + app_state = AppState() + + if src is None or src == "": + return src + + if not hasattr(self, 'artifacts'): + self.artifacts = {} + + if self.artifacts is None: + self.artifacts = {} + + if config_path in self.artifacts.keys(): + logging.warning( + f"You tried to register an artifact under config key={config_path} but an artifact for " + f"it has already been registered." + ) + + artifact_item = model_utils.ArtifactItem() + + # This is for backward compatibility, if the src objects exists simply inside of the tarfile + # without its key having been overriden, this pathway will be used. + src_obj_name = os.path.basename(src) + if _NEMO_FILE_FOLDER is not None: + src_obj_path = os.path.abspath(os.path.join(_NEMO_FILE_FOLDER, src_obj_name)) + else: + src_obj_path = src_obj_name + + # src is a local existing path - register artifact and return exact same path for usage by the model + if os.path.exists(os.path.abspath(src)): + return_path = os.path.abspath(src) + artifact_item.path_type = model_utils.ArtifactPathType.LOCAL_PATH + + # this is the case when artifact must be retried from the nemo file + # we are assuming that the location of the right nemo file is available from _MODEL_RESTORE_PATH + elif src.startswith("nemo:"): + return_path = os.path.abspath(os.path.join(_NEMO_FILE_FOLDER, src[5:])) + artifact_item.path_type = model_utils.ArtifactPathType.TAR_PATH + + # backward compatibility implementation + elif os.path.exists(src_obj_path): + return_path = src_obj_path + artifact_item.path_type = model_utils.ArtifactPathType.TAR_PATH + else: + if verify_src_exists: + raise FileNotFoundError( + f"src path does not exist or it is not a path in nemo file. src value I got was: {src}. Absolute: {os.path.abspath(src)}" + ) + else: + # artifact is optional and we simply return None + return None + + assert os.path.exists(return_path) + + artifact_item.path = os.path.abspath(src) + self.artifacts[config_path] = artifact_item + # we were called by ModelPT + if hasattr(self, "cfg"): + with open_dict(self._cfg): + OmegaConf.update(self.cfg, config_path, return_path) + return return_path + + def _handle_artifacts(self, nemo_file_folder): + tarfile_artifacts = [] + app_state = AppState() + for conf_path, artiitem in self.artifacts.items(): + if artiitem.path_type == model_utils.ArtifactPathType.LOCAL_PATH: + if not os.path.exists(artiitem.path): + raise FileNotFoundError(f"Artifact {conf_path} not found at location: {artiitem.path}") + + # Generate new uniq artifact name and copy it to nemo_file_folder + # Note uuid.uuid4().hex is guaranteed to be 32 character long + artifact_base_name = os.path.basename(artiitem.path) + artifact_uniq_name = f"{uuid.uuid4().hex}_{artifact_base_name}" + shutil.copy2(artiitem.path, os.path.join(nemo_file_folder, artifact_uniq_name)) + + # Update artifacts registry + artiitem.hashed_path = "nemo:" + artifact_uniq_name + self.artifacts[conf_path] = artiitem + + elif artiitem.path_type == model_utils.ArtifactPathType.TAR_PATH: + # process all tarfile artifacts in one go, so preserve key-value pair + tarfile_artifacts.append((conf_path, artiitem)) + + else: + raise ValueError(f"Directly referencing artifacts from other nemo files isn't supported yet") + + # Process current tarfile artifacts by unpacking the previous tarfile and extract the artifacts + # that are currently required. + model_metadata = app_state.get_model_metadata_from_guid(self.model_guid) + if len(tarfile_artifacts) > 0 and model_metadata.restoration_path is not None: + # Need to step into nemo archive to extract file + # Get path where the command is executed - the artifacts will be "retrieved" there + # (original .nemo behavior) + cwd = os.getcwd() + try: + # Step into the nemo archive to try and find the file + with tempfile.TemporaryDirectory() as archive_dir: + self._unpack_nemo_file(path2file=model_metadata.restoration_path, out_folder=archive_dir) + os.chdir(archive_dir) + for conf_path, artiitem in tarfile_artifacts: + # Get basename and copy it to nemo_file_folder + if 'nemo:' in artiitem.path: + artifact_base_name = artiitem.path.split('nemo:')[1] + else: + artifact_base_name = os.path.basename(artiitem.path) + # no need to hash here as we are in tarfile_artifacts which are already hashed + artifact_uniq_name = artifact_base_name + shutil.copy2(artifact_base_name, os.path.join(nemo_file_folder, artifact_uniq_name)) + + # Update artifacts registry + new_artiitem = model_utils.ArtifactItem() + new_artiitem.path = "nemo:" + artifact_uniq_name + new_artiitem.path_type = model_utils.ArtifactPathType.TAR_PATH + self.artifacts[conf_path] = new_artiitem + finally: + # change back working directory + os.chdir(cwd) + + def _update_artifact_paths(self, path2yaml_file): + if self.artifacts is not None and len(self.artifacts) > 0: + conf = OmegaConf.load(path2yaml_file) + for conf_path, item in self.artifacts.items(): + if item.hashed_path is None: + OmegaConf.update(conf, conf_path, item.path) + else: + OmegaConf.update(conf, conf_path, item.hashed_path) + with open(path2yaml_file, 'w') as fout: + OmegaConf.save(config=conf, f=fout, resolve=True) + + def _default_save_to(self, save_path: str): + """ + Saves model instance (weights and configuration) into .nemo file. + You can use "restore_from" method to fully restore instance from .nemo file. + + .nemo file is an archive (tar.gz) with the following: + model_config.yaml - model configuration in .yaml format. You can deserialize this into cfg argument for model's constructor + model_wights.chpt - model checkpoint + + Args: + save_path: Path to .nemo file where model instance should be saved + + """ + with tempfile.TemporaryDirectory() as tmpdir: + config_yaml = path.join(tmpdir, _MODEL_CONFIG_YAML) + model_weights = path.join(tmpdir, _MODEL_WEIGHTS) + self.to_config_file(path2yaml_file=config_yaml) + if hasattr(self, 'artifacts') and self.artifacts is not None: + self._handle_artifacts(nemo_file_folder=tmpdir) + # We should not update self._cfg here - the model can still be in use + self._update_artifact_paths(path2yaml_file=config_yaml) + torch.save(self.state_dict(), model_weights) + self._make_nemo_file_from_folder(filename=save_path, source_dir=tmpdir) + + @rank_zero_only + def save_to(self, save_path: str): + """ + Saves model instance (weights and configuration) into .nemo file + You can use "restore_from" method to fully restore instance from .nemo file. + + .nemo file is an archive (tar.gz) with the following: + model_config.yaml - model configuration in .yaml format. You can deserialize this into cfg argument for model's constructor + model_wights.chpt - model checkpoint + + Args: + save_path: Path to .nemo file where model instance should be saved + """ + + # Add NeMo rank check as well + if not is_global_rank_zero(): + return + else: + self._default_save_to(save_path) + + @classmethod + def _default_restore_from( + cls, + restore_path: str, + override_config_path: Optional[Union[OmegaConf, str]] = None, + map_location: Optional[torch.device] = None, + strict: bool = True, + return_config: bool = False, + ): + """ + Restores model instance (weights and configuration) into .nemo file + Args: + restore_path: path to .nemo file from which model should be instantiated + override_config_path: path to a yaml config that will override the internal + config file or an OmegaConf / DictConfig object representing the model config. + map_location: Optional torch.device() to map the instantiated model to a device. + By default (None), it will select a GPU if available, falling back to CPU otherwise. + strict: Passed to load_state_dict. By default True + return_config: If set to true, will return just the underlying config of the restored + model as an OmegaConf DictConfig object without instantiating the model. + + Example: + ``` + model = nemo.collections.asr.models.EncDecCTCModel.restore_from('asr.nemo') + assert isinstance(model, nemo.collections.asr.models.EncDecCTCModel) + ``` + + Returns: + An instance of type cls or its underlying config (if return_config is set). + """ + # Get path where the command is executed - the artifacts will be "retrieved" there + # (original .nemo behavior) + cwd = os.getcwd() + + if map_location is None: + if torch.cuda.is_available(): + map_location = torch.device('cuda') + else: + map_location = torch.device('cpu') + + with tempfile.TemporaryDirectory() as tmpdir: + try: + cls._set_model_restore_state(is_being_restored=True, folder=tmpdir) + cls._unpack_nemo_file(path2file=restore_path, out_folder=tmpdir) + os.chdir(tmpdir) + if override_config_path is None: + config_yaml = path.join(tmpdir, _MODEL_CONFIG_YAML) + else: + # can be str path or OmegaConf / DictConfig object + config_yaml = override_config_path + if not isinstance(config_yaml, (OmegaConf, DictConfig)): + conf = OmegaConf.load(config_yaml) + else: + conf = config_yaml + if override_config_path is not None: + # Resolve the override config + conf = OmegaConf.to_container(conf, resolve=True) + conf = OmegaConf.create(conf) + # If override is top level config, extract just `model` from it + if 'model' in conf: + conf = conf.model + + if return_config: + instance = conf + else: + app_state = AppState() + if app_state.model_parallel_rank is not None: + model_weights = path.join( + tmpdir, f'mp_rank_{app_state.model_parallel_rank:02}', _MODEL_WEIGHTS + ) + else: + model_weights = path.join(tmpdir, _MODEL_WEIGHTS) + OmegaConf.set_struct(conf, True) + os.chdir(cwd) + instance = cls.from_config_dict(config=conf) + instance = instance.to(map_location) + instance.load_state_dict(torch.load(model_weights, map_location=map_location), strict=strict) + + logging.info(f'Model {instance.__class__.__name__} was successfully restored from {restore_path}.') + finally: + cls._set_model_restore_state(is_being_restored=False) + os.chdir(cwd) + + return instance + + @classmethod + def restore_from( + cls, + restore_path: str, + override_config_path: Optional[Union[OmegaConf, str]] = None, + map_location: Optional[torch.device] = None, + strict: bool = True, + return_config: bool = False, + ): + """ + Restores model instance (weights and configuration) from .nemo file. + + Args: + restore_path: path to .nemo file from which model should be instantiated + override_config_path: path to a yaml config that will override the internal + config file or an OmegaConf / DictConfig object representing the model config. + map_location: Optional torch.device() to map the instantiated model to a device. + By default (None), it will select a GPU if available, falling back to CPU otherwise. + strict: Passed to load_state_dict. By default True. + return_config: If set to true, will return just the underlying config of the restored + model as an OmegaConf DictConfig object without instantiating the model. + + Example: + ``` + model = nemo.collections.asr.models.EncDecCTCModel.restore_from('asr.nemo') + assert isinstance(model, nemo.collections.asr.models.EncDecCTCModel) + ``` + + Returns: + An instance of type cls or its underlying config (if return_config is set). + """ + app_state = AppState() + if not path.exists(restore_path): + raise FileNotFoundError(f"Can't find {restore_path}") + + app_state.model_restore_path = os.path.abspath(os.path.expanduser(restore_path)) + return cls._default_restore_from(restore_path, override_config_path, map_location, strict, return_config) + + @classmethod + def load_from_checkpoint( + cls, + checkpoint_path: str, + *args, + map_location: Optional[Union[Dict[str, str], str, torch.device, int, Callable]] = None, + hparams_file: Optional[str] = None, + strict: bool = True, + **kwargs, + ): + """ + Loads ModelPT from checkpoint, with some maintenance of restoration. + For documentation, please refer to LightningModule.load_from_checkpoin() documentation. + """ + checkpoint = None + try: + cls._set_model_restore_state(is_being_restored=True) + + checkpoint = super().load_from_checkpoint( + checkpoint_path=checkpoint_path, + *args, + map_location=map_location, + hparams_file=hparams_file, + strict=strict, + **kwargs, + ) + + finally: + cls._set_model_restore_state(is_being_restored=False) + return checkpoint + + @abstractmethod + def setup_training_data(self, train_data_config: Union[DictConfig, Dict]): + """ + Setups data loader to be used in training + + Args: + train_data_layer_config: training data layer parameters. + Returns: + + """ + pass + + @abstractmethod + def setup_validation_data(self, val_data_config: Union[DictConfig, Dict]): + """ + Setups data loader to be used in validation + Args: + + val_data_layer_config: validation data layer parameters. + Returns: + + """ + pass + + def setup_test_data(self, test_data_config: Union[DictConfig, Dict]): + """ + (Optionally) Setups data loader to be used in test + + Args: + test_data_layer_config: test data layer parameters. + Returns: + + """ + raise NotImplementedError() + + def setup_multiple_validation_data(self, val_data_config: Union[DictConfig, Dict]): + """ + (Optionally) Setups data loader to be used in validation, with support for multiple data loaders. + + Args: + val_data_layer_config: validation data layer parameters. + """ + # Set some placeholder overriden by helper method + self._val_dl_idx = 0 + self._validation_names = None + self._validation_dl = None # type: torch.utils.data.DataLoader + + # preserve config + self._update_dataset_config(dataset_name='validation', config=val_data_config) + + try: + self._multi_dataset_mode = True + model_utils.resolve_validation_dataloaders(model=self) + finally: + self._multi_dataset_mode = False + + if self._validation_names is None: + if self._validation_dl is not None and type(self._validation_dl) in [list, tuple]: + self._validation_names = ['val_{}_'.format(idx) for idx in range(len(self._validation_dl))] + + def setup_multiple_test_data(self, test_data_config: Union[DictConfig, Dict]): + """ + (Optionally) Setups data loader to be used in test, with support for multiple data loaders. + + Args: + test_data_layer_config: test data layer parameters. + """ + # Set some placeholder overriden by helper method + self._test_dl_idx = 0 + self._test_names = None + self._test_dl = None # type: torch.utils.data.DataLoader + + # preserve config + self._update_dataset_config(dataset_name='test', config=test_data_config) + + try: + self._multi_dataset_mode = True + model_utils.resolve_test_dataloaders(model=self) + finally: + self._multi_dataset_mode = False + + if self._test_names is None: + if self._test_dl is not None and type(self._test_dl) in [list, tuple]: + self._test_names = ['test_{}_'.format(idx) for idx in range(len(self._test_dl))] + + def setup_optimization(self, optim_config: Optional[Union[DictConfig, Dict]] = None): + """ + Prepares an optimizer from a string name and its optional config parameters. + + Args: + optim_config: A dictionary containing the following keys: + + * "lr": mandatory key for learning rate. Will raise ValueError if not provided. + * "optimizer": string name pointing to one of the available optimizers in the registry. \ + If not provided, defaults to "adam". + * "opt_args": Optional list of strings, in the format "arg_name=arg_value". \ + The list of "arg_value" will be parsed and a dictionary of optimizer kwargs \ + will be built and supplied to instantiate the optimizer. + """ + # If config was not explicitly passed to us + if optim_config is None: + # See if internal config has `optim` namespace + if self._cfg is not None and hasattr(self._cfg, 'optim'): + optim_config = self._cfg.optim + + # If config is still None, or internal config has no Optim, return without instantiation + if optim_config is None: + logging.info('No optimizer config provided, therefore no optimizer was created') + return + + else: + # Preserve the configuration + if not isinstance(optim_config, DictConfig): + optim_config = OmegaConf.create(optim_config) + + # See if internal config has `optim` namespace before preservation + if self._cfg is not None and hasattr(self._cfg, 'optim'): + if self._cfg.optim is None: + self._cfg.optim = copy.deepcopy(optim_config) + else: + with open_dict(self._cfg.optim): + self._cfg.optim = copy.deepcopy(optim_config) + + # Setup optimizer and scheduler + if optim_config is not None and isinstance(optim_config, DictConfig): + optim_config = OmegaConf.to_container(optim_config, resolve=True) + + if self._trainer is None: + logging.warning(f"Trainer wasn't specified in model constructor. Make sure that you really wanted it.") + + if 'sched' in optim_config and self._trainer is not None: + if not isinstance(self._trainer.accumulate_grad_batches, int): + raise ValueError("We do not currently support gradient acculumation that is not an integer.") + if self._trainer.max_steps is None: + # Store information needed to calculate max_steps + optim_config['sched']['t_max_epochs'] = self._trainer.max_epochs + optim_config['sched']['t_accumulate_grad_batches'] = self._trainer.accumulate_grad_batches + optim_config['sched']['t_limit_train_batches'] = self._trainer.limit_train_batches + if self._trainer.distributed_backend is None: + optim_config['sched']['t_num_workers'] = self._trainer.num_gpus or 1 + elif self._trainer.distributed_backend == "ddp_cpu": + optim_config['sched']['t_num_workers'] = self._trainer.num_processes * self._trainer.num_nodes + elif self._trainer.distributed_backend == "ddp": + optim_config['sched']['t_num_workers'] = self._trainer.num_gpus * self._trainer.num_nodes + else: + logging.warning( + f"The lightning trainer received accelerator: {self._trainer.distributed_backend}. We " + "recommend to use 'ddp' instead." + ) + optim_config['sched']['t_num_workers'] = self._trainer.num_gpus * self._trainer.num_nodes + else: + optim_config['sched']['max_steps'] = self._trainer.max_steps + + # Force into DictConfig from nested structure + optim_config = OmegaConf.create(optim_config) + # Get back nested dict so we its mutable + optim_config = OmegaConf.to_container(optim_config, resolve=True) + + # Extract scheduler config if inside optimizer config + if 'sched' in optim_config: + scheduler_config = optim_config.pop('sched') + else: + scheduler_config = None + + # Check if caller provided optimizer name, default to Adam otherwise + optimizer_cls = optim_config.get('_target_', None) + + if optimizer_cls is None: + # Try to get optimizer name for dynamic resolution, defaulting to Adam + optimizer_name = optim_config.get('name', 'adam') + else: + if inspect.isclass(optimizer_cls): + optimizer_name = optimizer_cls.__name__.lower() + else: + # resolve the class name (lowercase) from the class path if not provided + optimizer_name = optimizer_cls.split(".")[-1].lower() + + # We are guarenteed to have lr since it is required by the argparser + # But maybe user forgot to pass it to this function + lr = optim_config.get('lr', None) + + # Check if caller has optimizer kwargs, default to empty dictionary + if 'args' in optim_config: + optimizer_args = optim_config.pop('args') + optimizer_args = optim.parse_optimizer_args(optimizer_name, optimizer_args) + else: + optimizer_args = copy.deepcopy(optim_config) + + # Remove extra parameters from optimizer_args nest + # Assume all other parameters are to be passed into optimizer constructor + optimizer_args.pop('name', None) + optimizer_args.pop('cls', None) + optimizer_args.pop('lr', None) + + # Adaptive schedulers don't need `lr` + if lr is not None: + optimizer_args['lr'] = lr + + # Actually instantiate the optimizer + if optimizer_cls is not None: + if inspect.isclass(optimizer_cls): + optimizer = optimizer_cls(self.parameters(), **optimizer_args) + logging.info("Optimizer config = %s", str(optimizer)) + + self._optimizer = optimizer + + else: + # Attempt class path resolution + try: + optimizer_cls = OmegaConf.create({'_target_': optimizer_cls}) + if lr is not None: + optimizer_config = {'lr': lr} + else: + optimizer_config = {} + optimizer_config.update(optimizer_args) + + optimizer_instance = hydra.utils.instantiate( + optimizer_cls, self.parameters(), **optimizer_config + ) # type: DictConfig + + logging.info("Optimizer config = %s", str(optimizer_instance)) + + self._optimizer = optimizer_instance + + except Exception as e: + logging.error( + "Could not instantiate class path - {} with kwargs {}".format( + optimizer_cls, str(optimizer_config) + ) + ) + raise e + + else: + optimizer = optim.get_optimizer(optimizer_name) + optimizer = optimizer(self.parameters(), **optimizer_args) + + logging.info("Optimizer config = %s", str(optimizer)) + + self._optimizer = optimizer + + # Try to instantiate scheduler for optimizer + self._scheduler = prepare_lr_scheduler( + optimizer=self._optimizer, scheduler_config=scheduler_config, train_dataloader=self._train_dl + ) + + # Return the optimizer with/without scheduler + # This return allows multiple optimizers or schedulers to be created + return self._optimizer, self._scheduler + + def configure_optimizers(self): + self.setup_optimization() + + if self._scheduler is None: + return self._optimizer + else: + return [self._optimizer], [self._scheduler] + + def train_dataloader(self): + if self._train_dl is not None: + return self._train_dl + + def val_dataloader(self): + if self._validation_dl is not None: + return self._validation_dl + + def test_dataloader(self): + if self._test_dl is not None: + return self._test_dl + + def validation_epoch_end( + self, outputs: Union[List[Dict[str, torch.Tensor]], List[List[Dict[str, torch.Tensor]]]] + ) -> Optional[Dict[str, Dict[str, torch.Tensor]]]: + """ + Default DataLoader for Validation set which automatically supports multiple data loaders + via `multi_validation_epoch_end`. + + If multi dataset support is not required, override this method entirely in base class. + In such a case, there is no need to implement `multi_validation_epoch_end` either. + + .. note:: + If more than one data loader exists, and they all provide `val_loss`, + only the `val_loss` of the first data loader will be used by default. + This default can be changed by passing the special key `val_dl_idx: int` + inside the `validation_ds` config. + + Args: + outputs: Single or nested list of tensor outputs from one or more data loaders. + + Returns: + A dictionary containing the union of all items from individual data_loaders, + along with merged logs from all data loaders. + """ + # Case where we dont provide data loaders + if outputs is not None and len(outputs) == 0: + return {} + + # Case where we provide exactly 1 data loader + if type(outputs[0]) == dict: + output_dict = self.multi_validation_epoch_end(outputs, dataloader_idx=0) + + if output_dict is not None and 'log' in output_dict: + self.log_dict(output_dict.pop('log'), on_epoch=True) + + return output_dict + + else: # Case where we provide more than 1 data loader + output_dict = {'log': {}} + + # The output is a list of list of dicts, outer list corresponds to dataloader idx + for dataloader_idx, val_outputs in enumerate(outputs): + # Get prefix and dispatch call to multi epoch end + dataloader_prefix = self.get_validation_dataloader_prefix(dataloader_idx) + dataloader_logs = self.multi_validation_epoch_end(val_outputs, dataloader_idx=dataloader_idx) + + # If result was not provided, generate empty dict + dataloader_logs = dataloader_logs or {} + + # Perform `val_loss` resolution first (if provided outside logs) + if 'val_loss' in dataloader_logs: + if 'val_loss' not in output_dict and dataloader_idx == self._val_dl_idx: + output_dict['val_loss'] = dataloader_logs['val_loss'] + + # For every item in the result dictionary + for k, v in dataloader_logs.items(): + # If the key is `log` + if k == 'log': + # Parse every element of the log, and attach the prefix name of the data loader + log_dict = {} + + for k_log, v_log in v.items(): + # If we are logging the metric, but dont provide it at result level, + # store it twice - once in log and once in result level. + # Also mark log with prefix name to avoid log level clash with other data loaders + if k_log not in output_dict['log'] and dataloader_idx == self._val_dl_idx: + new_k_log = k_log + + # Also insert duplicate key with prefix for ease of comparison / avoid name clash + log_dict[dataloader_prefix + k_log] = v_log + + else: + # Simply prepend prefix to key and save + new_k_log = dataloader_prefix + k_log + + # Store log value + log_dict[new_k_log] = v_log + + # Update log storage of individual data loader + output_logs = output_dict['log'] + output_logs.update(log_dict) + + # Update global log storage + output_dict['log'] = output_logs + + else: + # If any values are stored outside 'log', simply prefix name and store + new_k = dataloader_prefix + k + output_dict[new_k] = v + + if 'log' in output_dict: + self.log_dict(output_dict.pop('log'), on_epoch=True) + + # return everything else + return output_dict + + def test_epoch_end( + self, outputs: Union[List[Dict[str, torch.Tensor]], List[List[Dict[str, torch.Tensor]]]] + ) -> Optional[Dict[str, Dict[str, torch.Tensor]]]: + """ + Default DataLoader for Test set which automatically supports multiple data loaders + via `multi_test_epoch_end`. + + If multi dataset support is not required, override this method entirely in base class. + In such a case, there is no need to implement `multi_test_epoch_end` either. + + .. note:: + If more than one data loader exists, and they all provide `test_loss`, + only the `test_loss` of the first data loader will be used by default. + This default can be changed by passing the special key `test_dl_idx: int` + inside the `test_ds` config. + + Args: + outputs: Single or nested list of tensor outputs from one or more data loaders. + + Returns: + A dictionary containing the union of all items from individual data_loaders, + along with merged logs from all data loaders. + """ + # Case where we dont provide data loaders + if outputs is not None and len(outputs) == 0: + return {} + + # Case where we provide exactly 1 data loader + if type(outputs[0]) == dict: + output_dict = self.multi_test_epoch_end(outputs, dataloader_idx=0) + + if output_dict is not None and 'log' in output_dict: + self.log_dict(output_dict.pop('log'), on_epoch=True) + + return output_dict + + else: # Case where we provide more than 1 data loader + output_dict = {'log': {}} + + # The output is a list of list of dicts, outer list corresponds to dataloader idx + for dataloader_idx, test_outputs in enumerate(outputs): + # Get prefix and dispatch call to multi epoch end + dataloader_prefix = self.get_test_dataloader_prefix(dataloader_idx) + dataloader_logs = self.multi_test_epoch_end(test_outputs, dataloader_idx=dataloader_idx) + + # If result was not provided, generate empty dict + dataloader_logs = dataloader_logs or {} + + # Perform `test_loss` resolution first (if provided outside logs) + if 'test_loss' in dataloader_logs: + if 'test_loss' not in output_dict and dataloader_idx == self._test_dl_idx: + output_dict['test_loss'] = dataloader_logs['test_loss'] + + # For every item in the result dictionary + for k, v in dataloader_logs.items(): + # If the key is `log` + if k == 'log': + # Parse every element of the log, and attach the prefix name of the data loader + log_dict = {} + for k_log, v_log in v.items(): + # If we are logging the loss, but dont provide it at result level, + # store it twice - once in log and once in result level. + # Also mark log with prefix name to avoid log level clash with other data loaders + if k_log not in output_dict['log'] and dataloader_idx == self._test_dl_idx: + new_k_log = k_log + + # Also insert duplicate key with prefix for ease of comparison / avoid name clash + log_dict[dataloader_prefix + k_log] = v_log + + else: + # Simply prepend prefix to key and save + new_k_log = dataloader_prefix + k_log + + log_dict[new_k_log] = v_log + + # Update log storage of individual data loader + output_logs = output_dict.get('log', {}) + output_logs.update(log_dict) + + # Update global log storage + output_dict['log'] = output_logs + + else: + # If any values are stored outside 'log', simply prefix name and store + new_k = dataloader_prefix + k + output_dict[new_k] = v + + if 'log' in output_dict: + self.log_dict(output_dict.pop('log'), on_epoch=True) + + # return everything else + return output_dict + + def multi_validation_epoch_end( + self, outputs: List[Dict[str, torch.Tensor]], dataloader_idx: int = 0 + ) -> Optional[Dict[str, Dict[str, torch.Tensor]]]: + """ + Adds support for multiple validation datasets. Should be overriden by subclass, + so as to obtain appropriate logs for each of the dataloaders. + + Args: + outputs: Same as that provided by LightningModule.validation_epoch_end() + for a single dataloader. + dataloader_idx: int representing the index of the dataloader. + + Returns: + A dictionary of values, optionally containing a sub-dict `log`, + such that the values in the log will be pre-pended by the dataloader prefix. + """ + logging.warning( + "Multi data loader support has been enabled, but " + "`multi_validation_epoch_end(outputs, dataloader_idx) has not been implemented.\n" + "If you require multi data loader support for validation sets, please override this method.\n" + "If you do not require multi data loader support, please instead override " + "`validation_epoch_end(outputs)." + ) + + def multi_test_epoch_end( + self, outputs: List[Dict[str, torch.Tensor]], dataloader_idx: int = 0 + ) -> Optional[Dict[str, Dict[str, torch.Tensor]]]: + """ + Adds support for multiple test datasets. Should be overriden by subclass, + so as to obtain appropriate logs for each of the dataloaders. + + Args: + outputs: Same as that provided by LightningModule.validation_epoch_end() + for a single dataloader. + dataloader_idx: int representing the index of the dataloader. + + Returns: + A dictionary of values, optionally containing a sub-dict `log`, + such that the values in the log will be pre-pended by the dataloader prefix. + """ + logging.warning( + "Multi data loader support has been enabled, but " + "`multi_test_epoch_end(outputs, dataloader_idx) has not been implemented.\n" + "If you require multi data loader support for validation sets, please override this method.\n" + "If you do not require multi data loader support, please instead override " + "`test_epoch_end(outputs)." + ) + + def get_validation_dataloader_prefix(self, dataloader_idx: int = 0) -> str: + """ + Get the name of one or more data loaders, which will be prepended to all logs. + + Args: + dataloader_idx: Index of the data loader. + + Returns: + str name of the data loader at index provided. + """ + return self._validation_names[dataloader_idx] + + def get_test_dataloader_prefix(self, dataloader_idx: int = 0) -> str: + """ + Get the name of one or more data loaders, which will be prepended to all logs. + + Args: + dataloader_idx: Index of the data loader. + + Returns: + str name of the data loader at index provided. + """ + return self._test_names[dataloader_idx] + + @rank_zero_only + def maybe_init_from_pretrained_checkpoint(self, cfg: OmegaConf, map_location: str = 'cpu'): + """ + Initializes a given model with the parameters obtained via specific config arguments. + The state dict of the provided model will be updated with `strict=False` setting so as to prevent + requirement of exact model parameters matching. + + Initializations: + init_from_nemo_model: Str path to a .nemo model, which will be instantiated in order + to extract the state dict. + + init_from_pretrained_model: Str name of a pretrained model checkpoint (obtained via cloud). + The model will be downloaded (or a cached copy will be used), instantiated and then + its state dict will be extracted. + + init_from_ptl_ckpt: Str name of a Pytorch Lightning checkpoint file. It will be loaded and + the state dict will extracted. + + Args: + cfg: The config used to instantiate the model. It need only contain one of the above keys. + map_location: str or torch.device() which represents where the intermediate state dict + (from the pretrained model or checkpoint) will be loaded. + + """ + args = ['init_from_nemo_model', 'init_from_pretrained_model', 'init_from_ptl_ckpt'] + arg_matches = [(1 if arg in cfg and arg is not None else 0) for arg in args] + + if sum(arg_matches) == 0: + # model weights do not need to be restored + return + + if sum(arg_matches) > 1: + raise ValueError( + f"Cannot pass more than one model initialization arguments to config!\n" + f"Found : {[args[idx] for idx, arg_present in enumerate(arg_matches) if arg_present]}" + ) + + if 'init_from_nemo_model' in cfg and cfg.init_from_nemo_model is not None: + with open_dict(cfg): + # Restore model + model_path = cfg.pop('init_from_nemo_model') + restored_model = self.restore_from(model_path, map_location=map_location, strict=True) + + # Restore checkpoint into current model + self.load_state_dict(restored_model.state_dict(), strict=False) + logging.info(f'Model checkpoint restored from nemo file with path : `{model_path}`') + + del restored_model + + if 'init_from_pretrained_model' in cfg and cfg.init_from_pretrained_model is not None: + with open_dict(cfg): + # Restore model + model_name = cfg.pop('init_from_pretrained_model') + + # Check if model is being resumed or not - only works if `Trainer` is attached to model + if hasattr(self, 'trainer') and self.trainer is not None: + trainer = self.trainer + if hasattr(trainer, 'resume_from_checkpoint') and trainer.resume_from_checkpoint is not None: + logging.info( + "Model training is being resumed via Pytorch Lightning.\n" + "Initialization from pretrained model (via cloud) will be skipped." + ) + return + + restored_model = self.from_pretrained(model_name, map_location=map_location, strict=True) + + # Restore checkpoint into current model + self.load_state_dict(restored_model.state_dict(), strict=False) + logging.info(f'Model checkpoint restored from pretrained chackpoint with name : `{model_name}`') + + del restored_model + + if 'init_from_ptl_ckpt' in cfg and cfg.init_from_ptl_ckpt is not None: + with open_dict(cfg): + # Restore checkpoint + ckpt_path = cfg.pop('init_from_ptl_ckpt') + ckpt = torch.load(ckpt_path, map_location=map_location) + + # Restore checkpoint into current model + self.load_state_dict(ckpt['state_dict'], strict=False) + logging.info(f'Model checkpoint restored from pytorch lightning chackpoint with path : `{ckpt_path}`') + + del ckpt + + def teardown(self, stage: str): + """ + Called at the end of fit and test. + + Args: + stage: either 'fit' or 'test' + """ + if stage == 'fit': + # Update env variable to bypass multi gpu issue after training + # This fix affects usage of trainer.test() after trainer.train() + # If trainer.train() was done on multiple GPUs, then trainer.test() + # will try to do ddp, even if its a new Trainer object with just 1 GPU. + # Temporary patch to fix that + if 'PL_TRAINER_GPUS' in os.environ: + os.environ.pop('PL_TRAINER_GPUS') + + super().teardown(stage) + + @classmethod + def extract_state_dict_from(cls, restore_path: str, save_dir: str, split_by_module: bool = False): + """ + Extract the state dict(s) from a provided .nemo tarfile and save it to a directory. + + Args: + restore_path: path to .nemo file from which state dict(s) should be extracted + save_dir: directory in which the saved state dict(s) should be stored + split_by_module: bool flag, which determins whether the output checkpoint should + be for the entire Model, or the individual module's that comprise the Model + + Example: + To convert the .nemo tarfile into a single Model level PyTorch checkpoint + :: + state_dict = nemo.collections.asr.models.EncDecCTCModel.extract_state_dict_from('asr.nemo', './asr_ckpts') + + + To restore a model from a Model level checkpoint + :: + model = nemo.collections.asr.models.EncDecCTCModel(cfg) # or any other method of restoration + model.load_state_dict(torch.load("./asr_ckpts/model_weights.ckpt")) + + + To convert the .nemo tarfile into multiple Module level PyTorch checkpoints + :: + state_dict = nemo.collections.asr.models.EncDecCTCModel.extract_state_dict_from('asr.nemo', './asr_ckpts', split_by_module=True) + + + To restore a module from a Module level checkpoint + :: + model = nemo.collections.asr.models.EncDecCTCModel(cfg) # or any other method of restoration + + # load the individual components + model.preprocessor.load_state_dict(torch.load("./asr_ckpts/preprocessor.ckpt")) + model.encoder.load_state_dict(torch.load("./asr_ckpts/encoder.ckpt")) + model.decoder.load_state_dict(torch.load("./asr_ckpts/decoder.ckpt")) + + + Returns: + The state dict that was loaded from the original .nemo checkpoint + """ + if not path.exists(restore_path): + raise FileExistsError(f"Can't find {restore_path}") + + cwd = os.getcwd() + + save_dir = os.path.abspath(save_dir) + if not os.path.exists(save_dir): + os.makedirs(save_dir, exist_ok=True) + + with tempfile.TemporaryDirectory() as tmpdir: + try: + cls._unpack_nemo_file(path2file=restore_path, out_folder=tmpdir) + os.chdir(tmpdir) + model_weights = path.join(tmpdir, _MODEL_WEIGHTS) + state_dict = torch.load(model_weights) + + if not split_by_module: + filepath = os.path.join(save_dir, _MODEL_WEIGHTS) + torch.save(state_dict, filepath) + + else: + key_set = set([key.split(".")[0] for key in state_dict.keys()]) + for primary_key in key_set: + inner_keys = [key for key in state_dict.keys() if key.split(".")[0] == primary_key] + state_dict_subset = { + ".".join(inner_key.split(".")[1:]): state_dict[inner_key] for inner_key in inner_keys + } + filepath = os.path.join(save_dir, f"{primary_key}.ckpt") + torch.save(state_dict_subset, filepath) + + logging.info(f'Checkpoints from {restore_path} were successfully extracted into {save_dir}.') + finally: + os.chdir(cwd) + + return state_dict + + def prepare_test(self, trainer: 'Trainer') -> bool: + """ + Helper method to check whether the model can safely be tested + on a dataset after training (or loading a checkpoint). + + :: + + trainer = Trainer() + if model.prepare_test(trainer): + trainer.test(model) + + Returns: + bool which declares the model safe to test. Provides warnings if it has to + return False to guide the user. + """ + if not hasattr(self._cfg, 'test_ds'): + logging.info("No `test_ds` config found within the manifest.") + return False + + # Replace ddp multi-gpu until PTL has a fix + DDP_WARN = """\n\nDuring testing, it is currently advisable to construct a new Trainer " + "with single GPU and no DDP to obtain accurate results. + "Following pattern should be used: " + "gpu = 1 if cfg.trainer.gpus != 0 else 0" + "trainer = Trainer(gpus=gpu)" + "if model.prepare_test(trainer):" + " trainer.test(model)\n\n""" + + if trainer is not None: + if trainer.num_gpus > 1: + logging.warning(DDP_WARN) + return False + + # Assign trainer to the model + self.set_trainer(trainer) + return True + + def set_trainer(self, trainer: Trainer): + """ + Set an instance of Trainer object. + + Args: + trainer: PyTorch Lightning Trainer object. + """ + self.trainer = trainer + self._trainer = trainer + self.set_world_size(self._trainer) + + def set_world_size(self, trainer: Trainer): + """ + Determines the world size from the PyTorch Lightning Trainer. + And then updates AppState. + + Args: + trainer (Trainer): PyTorch Lightning Trainer object + """ + # Update AppState with world information from trainer + if isinstance(trainer, Trainer): + app_state = AppState() + if self._trainer.num_gpus and self._trainer.num_nodes: + app_state.world_size = self._trainer.num_gpus * self._trainer.num_nodes + else: + logging.warning(f'World size can only be set by PyTorch Lightning Trainer.') + + def _update_dataset_config(self, dataset_name: str, config: Optional[Union[DictConfig, Dict]]): + """ + Update the config (if not None) of the dataset by given name. + Preserves said config after updating. + + Args: + dataset_name: str name of the dataset whose config is being updated. + Can be one of `train`, `validation` and `test`. + config: Optional DictConfig or dict. If None is passed, this method simply returns. + If dict is passed, it is cast into a DictConfig. + The internal config is updated with the passed config. + """ + if hasattr(self, '_multi_dataset_mode') and self._multi_dataset_mode is True: + return + + if config is not None: + if not isinstance(config, DictConfig): + config = OmegaConf.create(config) + + if dataset_name in ['train', 'validation', 'test']: + OmegaConf.set_struct(self.cfg, False) + + key_name = dataset_name + "_ds" + self.cfg[key_name] = config + + OmegaConf.set_struct(self.cfg, True) + + # Update hyper parameters by calling property setter + self.cfg = self._cfg + else: + raise ValueError("`dataset_name` when updating config must be one of [train, validation, test]") + + @property + def num_weights(self): + """ + Utility property that returns the total number of parameters of the Model. + """ + num: int = 0 + for p in self.parameters(): + if p.requires_grad: + num += p.numel() + return num + + @property + def cfg(self): + """ + Property that holds the finalized internal config of the model. + + Note: + Changes to this config are not reflected in the state of the model. + Please create a new model using an updated config to properly update the model. + """ + return self._cfg + + @cfg.setter + def cfg(self, cfg): + """ + Property that holds the finalized internal config of the model. + + Note: + Changes to this config are not reflected in the state of the model. + Please create a new model using an updated config to properly update the model. + """ + self._cfg = cfg + self._set_hparams(self._cfg) + self._hparams_initial = copy.deepcopy(self._hparams) + + @staticmethod + def _make_nemo_file_from_folder(filename, source_dir): + with tarfile.open(filename, "w:gz") as tar: + tar.add(source_dir, arcname=".") + + @staticmethod + def _unpack_nemo_file(path2file: str, out_folder: str) -> str: + if not path.exists(path2file): + raise FileNotFoundError(f"{path2file} does not exist") + tar = tarfile.open(path2file, "r:gz") + tar.extractall(path=out_folder) + tar.close() + return out_folder + + @staticmethod + def _is_model_being_restored() -> bool: + global _MODEL_IS_RESTORED + return _MODEL_IS_RESTORED + + @staticmethod + def _set_model_restore_state(is_being_restored: bool, folder: str = None): + global _MODEL_IS_RESTORED + global _NEMO_FILE_FOLDER + _MODEL_IS_RESTORED = is_being_restored + _NEMO_FILE_FOLDER = folder + + def _set_model_guid(self): + if not hasattr(self, 'model_guid'): + appstate = AppState() + + # Generate a unique uuid for the instance + # also determine if the model is being restored or not, and preserve the path + self.model_guid = str(uuid.uuid4()) + if self._is_model_being_restored(): + restore_path = appstate.model_restore_path + else: + restore_path = None + + appstate.register_model_guid(self.model_guid, restoration_path=restore_path) + + @staticmethod + def _is_restore_type_tarfile() -> bool: + """ + Utility method that checks if the restore path of the underlying Model + is a tarfile (can be any valid archive)._MODEL_EFF_SAVE + """ + app_state = AppState() + + if app_state.model_restore_path is None: + return False + else: + if app_state.model_restore_path: + if tarfile.is_tarfile(app_state.model_restore_path): + return True + else: + return False + + @staticmethod + def set_eff_save(use_eff_save: bool): + global _MODEL_EFF_SAVE + _MODEL_EFF_SAVE = use_eff_save + + @staticmethod + def use_eff_save() -> bool: + global _MODEL_EFF_SAVE + return _MODEL_EFF_SAVE diff --git a/models/tacotron2/NeMo/nemo/core/classes/module.py b/models/tacotron2/NeMo/nemo/core/classes/module.py new file mode 100644 index 0000000000..c640afa4c6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/classes/module.py @@ -0,0 +1,77 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import contextmanager + +from torch.nn import Module + +from nemo.core.classes.common import FileIO, Serialization, Typing + +__all__ = ['NeuralModule'] + + +class NeuralModule(Module, Typing, Serialization, FileIO): + """ + Abstract class offering interface shared between all PyTorch Neural Modules. + """ + + @property + def num_weights(self): + """ + Utility property that returns the total number of parameters of NeuralModule. + """ + num: int = 0 + for p in self.parameters(): + if p.requires_grad: + num += p.numel() + return num + + def input_example(self): + """ + Override this method if random inputs won't work + Returns: + A tuple sample of valid input data. + """ + + return + + def freeze(self) -> None: + r""" + Freeze all params for inference. + """ + for param in self.parameters(): + param.requires_grad = False + + self.eval() + + def unfreeze(self) -> None: + """ + Unfreeze all parameters for training. + """ + for param in self.parameters(): + param.requires_grad = True + + self.train() + + @contextmanager + def as_frozen(self): + """ + Context manager which temporarily freezes a module, yields control and finally unfreezes the module. + """ + self.freeze() + + try: + yield + finally: + self.unfreeze() diff --git a/models/tacotron2/NeMo/nemo/core/config/__init__.py b/models/tacotron2/NeMo/nemo/core/config/__init__.py new file mode 100644 index 0000000000..6fd644b941 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/config/__init__.py @@ -0,0 +1,47 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.core.config.base_config import Config +from nemo.core.config.hydra_runner import hydra_runner +from nemo.core.config.optimizers import ( + AdadeltaParams, + AdagradParams, + AdamaxParams, + AdamParams, + AdamWParams, + NovogradParams, + OptimizerParams, + RMSpropParams, + RpropParams, + SGDParams, + get_optimizer_config, + register_optimizer_params, +) +from nemo.core.config.pytorch import DataLoaderConfig +from nemo.core.config.pytorch_lightning import TrainerConfig +from nemo.core.config.schedulers import ( + CosineAnnealingParams, + InverseSquareRootAnnealingParams, + NoamAnnealingParams, + PolynomialDecayAnnealingParams, + PolynomialHoldDecayAnnealingParams, + SchedulerParams, + SquareAnnealingParams, + SquareRootAnnealingParams, + WarmupAnnealingParams, + WarmupHoldSchedulerParams, + WarmupSchedulerParams, + get_scheduler_config, + register_scheduler_params, +) diff --git a/models/tacotron2/NeMo/nemo/core/config/base_config.py b/models/tacotron2/NeMo/nemo/core/config/base_config.py new file mode 100644 index 0000000000..44649ef7b4 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/config/base_config.py @@ -0,0 +1,30 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional + +__all__ = ['Config'] + + +@dataclass +class Config: + """ + Abstract NeMo Configuration class. + + Args: + name: name of the module/dataset/loss/model object (used in serialization, DEFAULT: None) + """ + + name: Optional[str] = None diff --git a/models/tacotron2/NeMo/nemo/core/config/hydra_runner.py b/models/tacotron2/NeMo/nemo/core/config/hydra_runner.py new file mode 100644 index 0000000000..dfeff788a2 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/config/hydra_runner.py @@ -0,0 +1,110 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import os +import sys +from typing import Any, Callable, Optional + +from hydra._internal.utils import _run_hydra, get_args_parser +from hydra.core.config_store import ConfigStore +from hydra.types import TaskFunction +from omegaconf import DictConfig + + +def hydra_runner( + config_path: Optional[str] = ".", config_name: Optional[str] = None, schema: Optional[Any] = None +) -> Callable[[TaskFunction], Any]: + """ + Decorator used for passing the Config paths to main function. + Optionally registers a schema used for validation/providing default values. + + Args: + config_path: Optional path that will be added to config search directory. + NOTE: The default value of `config_path` has changed between Hydra 1.0 and Hydra 1.1+. + Please refer to https://hydra.cc/docs/next/upgrades/1.0_to_1.1/changes_to_hydra_main_config_path/ + for details. + config_name: Pathname of the config file. + schema: Structured config type representing the schema used for validation/providing default values. + """ + + def decorator(task_function: TaskFunction) -> Callable[[], None]: + @functools.wraps(task_function) + def wrapper(cfg_passthrough: Optional[DictConfig] = None) -> Any: + # Check it config was passed. + if cfg_passthrough is not None: + return task_function(cfg_passthrough) + else: + args = get_args_parser() + + # Parse arguments in order to retrieve overrides + parsed_args = args.parse_args() # type: argparse.Namespace + + # Get overriding args in dot string format + overrides = parsed_args.overrides # type: list + + # Disable the creation of .hydra subdir + # https://hydra.cc/docs/tutorials/basic/running_your_app/working_directory + overrides.append("hydra.output_subdir=null") + # Hydra logging outputs only to stdout (no log file). + # https://hydra.cc/docs/configure_hydra/logging + overrides.append("hydra/job_logging=stdout") + + # Set run.dir ONLY for ExpManager "compatibility" - to be removed. + overrides.append("hydra.run.dir=.") + + # Check if user set the schema. + if schema is not None: + # Create config store. + cs = ConfigStore.instance() + + # Get the correct ConfigStore "path name" to "inject" the schema. + if parsed_args.config_name is not None: + path, name = os.path.split(parsed_args.config_name) + # Make sure the path is not set - as this will disable validation scheme. + if path != '': + sys.stderr.write( + f"ERROR Cannot set config file path using `--config-name` when " + "using schema. Please set path using `--config-path` and file name using " + "`--config-name` separately.\n" + ) + sys.exit(1) + else: + name = config_name + + # Register the configuration as a node under the name in the group. + cs.store(name=name, node=schema) # group=group, + + # Wrap a callable object with name `parse_args` + # This is to mimic the ArgParser.parse_args() API. + class _argparse_wrapper: + def __init__(self, arg_parser): + self.arg_parser = arg_parser + self._actions = arg_parser._actions + + def parse_args(self, args=None, namespace=None): + return parsed_args + + # no return value from run_hydra() as it may sometime actually run the task_function + # multiple times (--multirun) + _run_hydra( + args_parser=_argparse_wrapper(args), + task_function=task_function, + config_path=config_path, + config_name=config_name, + ) + + return wrapper + + return decorator diff --git a/models/tacotron2/NeMo/nemo/core/config/modelPT.py b/models/tacotron2/NeMo/nemo/core/config/modelPT.py new file mode 100644 index 0000000000..3f5b16dde2 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/config/modelPT.py @@ -0,0 +1,181 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + +from omegaconf import MISSING + +from nemo.core import config +from nemo.core.classes.dataset import DatasetConfig +from nemo.utils import exp_manager + + +@dataclass +class SchedConfig: + name: str = MISSING + min_lr: float = 0.0 + last_epoch: int = -1 + + +@dataclass +class OptimConfig: + name: str = MISSING + sched: Optional[SchedConfig] = None + + +@dataclass +class ModelConfig: + """ + Model component inside ModelPT + """ + + # ... + train_ds: Optional[DatasetConfig] = None + validation_ds: Optional[DatasetConfig] = None + test_ds: Optional[DatasetConfig] = None + optim: Optional[OptimConfig] = None + + +@dataclass +class HydraConfig: + run: Dict[str, Any] = field(default_factory=lambda: {"dir": "."}) + job_logging: Dict[str, Any] = field(default_factory=lambda: {"root": {"handlers": None}}) + + +@dataclass +class NemoConfig: + name: str = MISSING + model: ModelConfig = MISSING + trainer: config.TrainerConfig = config.TrainerConfig( + accelerator="ddp", checkpoint_callback=False, logger=False, log_every_n_steps=1 + ) + exp_manager: Optional[Any] = exp_manager.ExpManagerConfig() + hydra: HydraConfig = HydraConfig() + + +class ModelConfigBuilder: + def __init__(self, model_cfg: ModelConfig): + """ + Base class for any Model Config Builder. + + A Model Config Builder is a utility class that accepts a ModelConfig dataclass, + and via a set of utility methods (that are implemented by the subclassed ModelConfigBuilder), + builds a finalized ModelConfig that can be supplied to a NemoModel dataclass as + the `model` component. + + Subclasses *must* implement the private method `_finalize_cfg`. + Inside this method, they must update `self.model_cfg` with all interdependent config + options that need to be set (either updated by user explicitly or with their default value). + + The updated model config must then be preserved in `self.model_cfg`. + + Example: + # Create the config builder + config_builder = ModelConfigBuilder() + + # Update the components of the config that are modifiable + config_builder.set_X(X) + config_builder.set_Y(Y) + + # Create a "finalized" config dataclass that will contain all the updates + # that were specified by the builder + model_config = config_builder.build() + + # Use model config as is (or further update values), then create a new Model + model = nemo..models.Model(cfg=model_config, trainer=Trainer()) + + Supported build methods: + - set_train_ds: All model configs can accept a subclass of `DatasetConfig` as their + training config. Subclasses can override this method to enable auto-complete + by replacing `Optional[DatasetConfig]` with `Optional[]`. + + - set_validation_ds: All model configs can accept a subclass of `DatasetConfig` as their + validation config. Subclasses can override this method to enable auto-complete + by replacing `Optional[DatasetConfig]` with `Optional[]`. + + - set_test_ds: All model configs can accept a subclass of `DatasetConfig` as their + test config. Subclasses can override this method to enable auto-complete + by replacing `Optional[DatasetConfig]` with `Optional[]`. + + - set_optim: A build method that supports changes to the Optimizer (and optionally, + the Scheduler) used for training the model. The function accepts two inputs - + + `cfg`: A subclass of `OptimizerParams` - any OptimizerParams subclass can be used, + in order to select an appropriate Optimizer. Examples: AdamParams. + + `sched_cfg`: A subclass of `SchedulerParams` - any SchedulerParams subclass can be used, + in order to select an appropriate Scheduler. Examples: CosineAnnealingParams. + Note that this argument is optional. + + - build(): The method which should return a "finalized" ModelConfig dataclass. + Subclasses *should* always override this method, and update the signature + of this method with the return type of the Dataclass, so that it enables + autocomplete for the user. + + Example: + def build(self) -> EncDecCTCConfig: + return super().build() + + Any additional build methods must be added by subclasses of ModelConfigBuilder. + + Args: + model_cfg: + """ + self.model_cfg = model_cfg + self.train_ds_cfg = None + self.validation_ds_cfg = None + self.test_ds_cfg = None + self.optim_cfg = None + + def set_train_ds(self, cfg: Optional[DatasetConfig] = None): + self.model_cfg.train_ds = cfg + + def set_validation_ds(self, cfg: Optional[DatasetConfig] = None): + self.model_cfg.validation_ds = cfg + + def set_test_ds(self, cfg: Optional[DatasetConfig] = None): + self.model_cfg.test_ds = cfg + + def set_optim(self, cfg: config.OptimizerParams, sched_cfg: Optional[config.SchedulerParams] = None): + @dataclass + class WrappedOptimConfig(OptimConfig, cfg.__class__): + pass + + # Setup optim + optim_name = cfg.__class__.__name__.replace("Params", "").lower() + wrapped_cfg = WrappedOptimConfig(name=optim_name, sched=None, **vars(cfg)) + + if sched_cfg is not None: + + @dataclass + class WrappedSchedConfig(SchedConfig, sched_cfg.__class__): + pass + + # Setup scheduler + sched_name = sched_cfg.__class__.__name__.replace("Params", "") + wrapped_sched_cfg = WrappedSchedConfig(name=sched_name, **vars(sched_cfg)) + + wrapped_cfg.sched = wrapped_sched_cfg + + self.model_cfg.optim = wrapped_cfg + + def _finalize_cfg(self): + raise NotImplementedError() + + def build(self) -> ModelConfig: + # validate config + self._finalize_cfg() + + return self.model_cfg diff --git a/models/tacotron2/NeMo/nemo/core/config/optimizers.py b/models/tacotron2/NeMo/nemo/core/config/optimizers.py new file mode 100644 index 0000000000..385e1a2bb3 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/config/optimizers.py @@ -0,0 +1,265 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from functools import partial +from typing import Any, Dict, Optional, Tuple + +from omegaconf import MISSING, OmegaConf + +__all__ = [ + 'OptimizerParams', + 'AdamParams', + 'NovogradParams', + 'SGDParams', + 'AdadeltaParams', + 'AdamaxParams', + 'AdagradParams', + 'AdamWParams', + 'RMSpropParams', + 'RpropParams', +] + + +@dataclass +class OptimizerParams: + """ + Base Optimizer params with no values. User can chose it to explicitly override via + command line arguments + """ + + lr: Optional[float] = MISSING + + +@dataclass +class SGDParams(OptimizerParams): + """ + Default configuration for Adam optimizer. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + + ..note: + For the details on the function/meanings of the arguments, please refer to: + https://pytorch.org/docs/stable/optim.html?highlight=sgd#torch.optim.SGD + """ + + momentum: float = 0 + dampening: float = 0 + weight_decay: float = 0 + nesterov: bool = False + + +@dataclass +class AdamParams(OptimizerParams): + """ + Default configuration for Adam optimizer. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + + ..note: + For the details on the function/meanings of the arguments, please refer to: + https://pytorch.org/docs/stable/optim.html?highlight=adam#torch.optim.Adam + """ + + # betas: Tuple[float, float] = (0.9, 0.999) + eps: float = 1e-08 + weight_decay: float = 0 + amsgrad: bool = False + + +@dataclass +class AdamWParams(OptimizerParams): + """ + Default configuration for AdamW optimizer. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + + ..note: + For the details on the function/meanings of the arguments, please refer to: + https://pytorch.org/docs/stable/optim.html#torch.optim.AdamW + """ + + betas: Tuple[float, float] = (0.9, 0.999) + eps: float = 1e-08 + weight_decay: float = 0 + amsgrad: bool = False + + +@dataclass +class AdadeltaParams(OptimizerParams): + """ + Default configuration for Adadelta optimizer. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + + ..note: + For the details on the function/meanings of the arguments, please refer to: + https://pytorch.org/docs/stable/optim.html#torch.optim.Adadelta + """ + + rho: float = 0.9 + eps: float = 1e-6 + weight_decay: float = 0 + + +@dataclass +class AdamaxParams(OptimizerParams): + """ + Default configuration for Adamax optimizer. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + + ..note: + For the details on the function/meanings of the arguments, please refer to: + https://pytorch.org/docs/stable/optim.html#torch.optim.Adamax + """ + + betas: Tuple[float, float] = (0.9, 0.999) + eps: float = 1e-8 + weight_decay: float = 0 + + +@dataclass +class AdagradParams(OptimizerParams): + """ + Default configuration for Adagrad optimizer. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + + ..note: + For the details on the function/meanings of the arguments, please refer to: + https://pytorch.org/docs/stable/optim.html#torch.optim.Adagrad + """ + + lr_decay: float = 0 + weight_decay: float = 0 + initial_accumulator_value: float = 0 + eps: float = 1e-10 + + +@dataclass +class RMSpropParams(OptimizerParams): + """ + Default configuration for RMSprop optimizer. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + + ..note: + For the details on the function/meanings of the arguments, please refer to: + https://pytorch.org/docs/stable/optim.html#torch.optim.RMSprop + """ + + alpha: float = 0.99 + eps: float = 1e-8 + weight_decay: float = 0 + momentum: float = 0 + centered: bool = False + + +@dataclass +class RpropParams(OptimizerParams): + """ + Default configuration for RpropParams optimizer. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + + ..note: + For the details on the function/meanings of the arguments, please refer to: + https://pytorch.org/docs/stable/optim.html#torch.optim.Rprop + """ + + etas: Tuple[float, float] = (0.5, 1.2) + step_sizes: Tuple[float, float] = (1e-6, 50) + + +@dataclass +class NovogradParams(OptimizerParams): + """ + Configuration of the Novograd optimizer. + + It has been proposed in "Stochastic Gradient Methods with Layer-wise + Adaptive Moments for Training of Deep Networks" + (https://arxiv.org/abs/1905.11286) + + Args: + lr (float, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + amsgrad (boolean, optional): whether to use the AMSGrad variant of this + algorithm from the paper "On the Convergence of Adam and Beyond" + """ + + betas: Tuple[float, float] = (0.95, 0.98) + eps: float = 1e-8 + weight_decay: float = 0 + grad_averaging: bool = False + amsgrad: bool = False + luc: bool = False + luc_trust: float = 1e-3 + luc_eps: float = 1e-8 + + +def register_optimizer_params(name: str, optimizer_params: OptimizerParams): + """ + Checks if the optimizer param name exists in the registry, and if it doesnt, adds it. + + This allows custom optimizer params to be added and called by name during instantiation. + + Args: + name: Name of the optimizer. Will be used as key to retrieve the optimizer. + optimizer_params: Optimizer class + """ + if name in AVAILABLE_OPTIMIZER_PARAMS: + raise ValueError(f"Cannot override pre-existing optimizers. Conflicting optimizer name = {name}") + + AVAILABLE_OPTIMIZER_PARAMS[name] = optimizer_params + + +def get_optimizer_config(name: str, **kwargs: Optional[Dict[str, Any]]) -> OptimizerParams: + """ + Convenience method to obtain a OptimizerParams class and partially instantiate it with optimizer kwargs. + + Args: + name: Name of the OptimizerParams in the registry. + kwargs: Optional kwargs of the optimizer used during instantiation. + + Returns: + a partially instantiated OptimizerParams + """ + if name is None: + return kwargs + + if name not in AVAILABLE_OPTIMIZER_PARAMS: + raise ValueError( + f"Cannot resolve optimizer parameters '{name}'. Available optimizer parameters are : " + f"{AVAILABLE_OPTIMIZER_PARAMS.keys()}" + ) + + scheduler_params = AVAILABLE_OPTIMIZER_PARAMS[name] + + if kwargs is not None and len(kwargs) != 0: + kwargs = OmegaConf.create(kwargs) + OmegaConf.merge(scheduler_params(), kwargs) + + scheduler_params = partial(scheduler_params, **kwargs) + return scheduler_params + + +AVAILABLE_OPTIMIZER_PARAMS = { + 'optim_params': OptimizerParams, + 'adam_params': AdamParams, + 'novograd_params': NovogradParams, + 'sgd_params': SGDParams, + 'adadelta_params': AdadeltaParams, + 'adamax_params': AdamaxParams, + 'adagrad_params': AdagradParams, + 'adamw_params': AdamWParams, + 'rmsprop_params': RMSpropParams, + 'rprop_params': RpropParams, +} diff --git a/models/tacotron2/NeMo/nemo/core/config/pytorch.py b/models/tacotron2/NeMo/nemo/core/config/pytorch.py new file mode 100644 index 0000000000..944df02ed8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/config/pytorch.py @@ -0,0 +1,45 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Optional + +from omegaconf import MISSING + +__all__ = ['DataLoaderConfig'] + + +@dataclass +class DataLoaderConfig: + """ + Configuration of PyTorch DataLoader. + + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + + ..note: + For the details on the function/meanings of the arguments, please refer to: + https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader + """ + + batch_size: int = MISSING + shuffle: bool = False + sampler: Optional[Any] = None + batch_sampler: Optional[Any] = None + num_workers: int = 0 + collate_fn: Optional[Any] = None + pin_memory: bool = False + drop_last: bool = False + timeout: int = 0 + worker_init_fn: Optional[Any] = None + multiprocessing_context: Optional[Any] = None diff --git a/models/tacotron2/NeMo/nemo/core/config/pytorch_lightning.py b/models/tacotron2/NeMo/nemo/core/config/pytorch_lightning.py new file mode 100644 index 0000000000..f997cc2a57 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/config/pytorch_lightning.py @@ -0,0 +1,97 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict, Optional + +from hydra.core.config_store import ConfigStore + +__all__ = ['TrainerConfig'] + + +cs = ConfigStore.instance() + + +@dataclass +class TrainerConfig: + """ + Configuration of PyTorch Lightning Trainer. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + ..warning: + Picked just few params of the PTL trainer for now. This needs to be discussed. + ..note: + For the details on the function/meanings of the arguments, please refer to: + https://pytorch-lightning.readthedocs.io/en/latest/trainer.html# + """ + + logger: Any = True + checkpoint_callback: Any = True + callbacks: Optional[Any] = None + default_root_dir: Optional[str] = None + gradient_clip_val: float = 0 + process_position: int = 0 + num_nodes: int = 1 + num_processes: int = 1 + gpus: Optional[Any] = None + auto_select_gpus: bool = False + tpu_cores: Optional[Any] = None + log_gpu_memory: Optional[str] = None + progress_bar_refresh_rate: int = 1 + overfit_batches: Any = 0.0 + track_grad_norm: Any = -1 + check_val_every_n_epoch: int = 1 + fast_dev_run: bool = False + accumulate_grad_batches: Any = 1 + max_epochs: int = 1000 + min_epochs: int = 1 + max_steps: Optional[int] = None + min_steps: Optional[int] = None + limit_train_batches: Any = 1.0 + limit_val_batches: Any = 1.0 + limit_test_batches: Any = 1.0 + val_check_interval: Any = 1.0 + flush_logs_every_n_steps: int = 100 + log_every_n_steps: int = 50 + accelerator: Optional[str] = None + sync_batchnorm: bool = False + precision: int = 32 + weights_summary: Optional[str] = "full" # ModelSummary.MODE_DEFAULT + weights_save_path: Optional[str] = None + num_sanity_val_steps: int = 2 + truncated_bptt_steps: Optional[int] = None + resume_from_checkpoint: Optional[str] = None + profiler: Optional[Any] = None + benchmark: bool = False + deterministic: bool = False + reload_dataloaders_every_epoch: bool = False + auto_lr_find: Any = False + replace_sampler_ddp: bool = True + terminate_on_nan: bool = False + auto_scale_batch_size: Any = False + prepare_data_per_node: bool = True + amp_backend: str = 'native' + amp_level: str = 'O2' # backward compatible, todo: remove in v1.0.0 + plugins: Optional[Any] = None # Optional[Union[str, list]] + move_metrics_to_cpu: bool = False + multiple_trainloader_mode: str = 'max_size_cycle' + limit_predict_batches: float = 1.0 + stochastic_weight_avg: bool = False + gradient_clip_algorithm: str = 'norm' + max_time: Optional[Any] = None # can be one of Union[str, timedelta, Dict[str, int], None] + + +# Register the trainer config. +cs.store( + group="trainer", name="trainer", node=TrainerConfig, +) diff --git a/models/tacotron2/NeMo/nemo/core/config/schedulers.py b/models/tacotron2/NeMo/nemo/core/config/schedulers.py new file mode 100644 index 0000000000..846b785d64 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/config/schedulers.py @@ -0,0 +1,251 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from functools import partial +from typing import Any, Dict, Optional + + +@dataclass +class SchedulerParams: + """ + Base configuration for all schedulers. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + last_epoch: int = -1 + + +@dataclass +class WarmupSchedulerParams(SchedulerParams): + """ + Base configuration for all schedulers. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + max_steps: int = 0 + warmup_steps: Optional[float] = None + warmup_ratio: Optional[float] = None + + +@dataclass +class WarmupHoldSchedulerParams(WarmupSchedulerParams): + """ + Base configuration for all schedulers. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + hold_steps: Optional[float] = None + hold_ratio: Optional[float] = None + min_lr: float = 0.0 + + +@dataclass +class SquareAnnealingParams(WarmupSchedulerParams): + """ + Square Annealing parameter config + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + min_lr: float = 1e-5 + + +@dataclass +class SquareRootAnnealingParams(WarmupSchedulerParams): + """ + Square Root Annealing parameter config + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + min_lr: float = 0.0 + + +@dataclass +class CosineAnnealingParams(WarmupSchedulerParams): + """ + Cosine Annealing parameter config + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + min_lr: float = 0.0 + + +@dataclass +class NoamAnnealingParams(WarmupSchedulerParams): + """ + Cosine Annealing parameter config + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + min_lr: float = 0.0 + + +@dataclass +class WarmupAnnealingParams(WarmupSchedulerParams): + """ + Warmup Annealing parameter config + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + warmup_ratio: 0.0 + + +@dataclass +class InverseSquareRootAnnealingParams(WarmupSchedulerParams): + """ + Inverse Square Root Annealing parameter config + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + +@dataclass +class PolynomialDecayAnnealingParams(WarmupSchedulerParams): + """ + Polynomial Decay Annealing parameter config + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + power: float = 1.0 + cycle: bool = False + + +@dataclass +class PolynomialHoldDecayAnnealingParams(WarmupSchedulerParams): + """ + Polynomial Hold Decay Annealing parameter config + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + power: float = 1.0 + cycle: bool = False + + +""" +Pytorch Optimizers +""" + + +@dataclass +class StepLRParams(SchedulerParams): + """ + Config for StepLR. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + step_size: float = 0.1 + gamma: float = 0.1 + + +@dataclass +class ExponentialLRParams(SchedulerParams): + """ + Config for ExponentialLR. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + gamma: float = 0.9 + + +@dataclass +class ReduceLROnPlateauParams: + """ + Config for ReduceLROnPlateau. + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + mode: str = 'min' + factor: float = 0.1 + patience: int = 10 + verbose: bool = False + threshold: float = 1e-4 + threshold_mode: str = 'rel' + cooldown: int = 0 + min_lr: float = 0 + eps: float = 1e-8 + + +@dataclass +class CyclicLRParams(SchedulerParams): + """ + Config for CyclicLR. + NOTE: + # `scale_fn` is not supported + + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + base_lr: float = 0.001 + max_lr: float = 0.1 + step_size_up: int = 2000 + step_size_down: Optional[int] = None + mode: str = 'triangular' + gamma: float = 1.0 + scale_mode: str = 'cycle' + # scale_fn is not supported + cycle_momentum: bool = True + base_momentum: float = 0.8 + max_momentum: float = 0.9 + + +def register_scheduler_params(name: str, scheduler_params: SchedulerParams): + """ + Checks if the schduler config name exists in the registry, and if it doesnt, adds it. + + This allows custom schedulers to be added and called by name during instantiation. + + Args: + name: Name of the optimizer. Will be used as key to retrieve the optimizer. + scheduler_params: SchedulerParams class + """ + if name in AVAILABLE_SCHEDULER_PARAMS: + raise ValueError(f"Cannot override pre-existing optimizers. Conflicting optimizer name = {name}") + + AVAILABLE_SCHEDULER_PARAMS[name] = scheduler_params + + +def get_scheduler_config(name: str, **kwargs: Optional[Dict[str, Any]]) -> SchedulerParams: + """ + Convenience method to obtain a SchedulerParams class and partially instantiate it with optimizer kwargs. + + Args: + name: Name of the SchedulerParams in the registry. + kwargs: Optional kwargs of the optimizer used during instantiation. + + Returns: + a partially instantiated SchedulerParams + """ + if name not in AVAILABLE_SCHEDULER_PARAMS: + raise ValueError( + f"Cannot resolve scheduler parameters '{name}'. Available scheduler parameters are : " + f"{AVAILABLE_SCHEDULER_PARAMS.keys()}" + ) + + scheduler_params = AVAILABLE_SCHEDULER_PARAMS[name] + scheduler_params = partial(scheduler_params, **kwargs) + return scheduler_params + + +AVAILABLE_SCHEDULER_PARAMS = { + 'SchedulerParams': SchedulerParams, + 'WarmupPolicyParams': WarmupSchedulerParams, + 'WarmupHoldPolicyParams': WarmupHoldSchedulerParams, + 'SquareAnnealingParams': SquareAnnealingParams, + 'SquareRootAnnealingParams': SquareRootAnnealingParams, + 'InverseSquareRootAnnealingParams': InverseSquareRootAnnealingParams, + 'CosineAnnealingParams': CosineAnnealingParams, + 'NoamAnnealingParams': NoamAnnealingParams, + 'WarmupAnnealingParams': WarmupAnnealingParams, + 'PolynomialDecayAnnealingParams': PolynomialDecayAnnealingParams, + 'PolynomialHoldDecayAnnealingParams': PolynomialHoldDecayAnnealingParams, +} diff --git a/models/tacotron2/NeMo/nemo/core/neural_types/__init__.py b/models/tacotron2/NeMo/nemo/core/neural_types/__init__.py new file mode 100644 index 0000000000..8598bbe67f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/neural_types/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.core.neural_types.axes import * +from nemo.core.neural_types.comparison import * +from nemo.core.neural_types.elements import * +from nemo.core.neural_types.neural_type import * diff --git a/models/tacotron2/NeMo/nemo/core/neural_types/axes.py b/models/tacotron2/NeMo/nemo/core/neural_types/axes.py new file mode 100644 index 0000000000..cda781f021 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/neural_types/axes.py @@ -0,0 +1,101 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from typing import Optional + +__all__ = ['AxisKindAbstract', 'AxisKind', 'AxisType'] + + +class AxisKindAbstract(Enum): + """This is an abstract Enum to represents what does varying axis dimension mean. + In practice, you will almost always use AxisKind Enum. This Enum should be inherited by + your OWN Enum if you aren't satisfied with AxisKind. Then your own Enum can be used + instead of AxisKind.""" + + pass + + +class AxisKind(AxisKindAbstract): + """This Enum represents what does varying axis dimension mean. + For example, does this dimension correspond to width, batch, time, etc. + The "Dimension" and "Channel" kinds are the same and used to represent + a general axis. "Any" axis will accept any axis kind fed to it. + """ + + Batch = 0 + Time = 1 + Dimension = 2 + Channel = 2 + Width = 3 + Height = 4 + Any = 5 + Sequence = 6 + FlowGroup = 7 + Singleton = 8 # Used to represent a axis that has size 1 + + def __repr__(self): + return self.__str__() + + def __str__(self): + return str(self.name).lower() + + @staticmethod + def from_str(label): + """Returns AxisKind instance based on short string representation""" + _label = label.lower().strip() + if _label == "b" or _label == "n" or _label == "batch": + return AxisKind.Batch + elif _label == "t" or _label == "time": + return AxisKind.Time + elif _label == "d" or _label == "c" or _label == "channel": + return AxisKind.Dimension + elif _label == "w" or _label == "width": + return AxisKind.Width + elif _label == "h" or _label == "height": + return AxisKind.Height + elif _label == "s" or _label == "singleton": + return AxisKind.Singleton + elif _label == "flowgroup": + return AxisKind.FlowGroup + elif _label == "any": + return AxisKind.Any + else: + raise ValueError(f"Can't create AxisKind from {label}") + + +class AxisType(object): + """This class represents axis semantics and (optionally) it's dimensionality + Args: + kind (AxisKindAbstract): what kind of axis it is? For example Batch, Height, etc. + size (int, optional): specify if the axis should have a fixed size. By default it is set to None and you + typically do not want to set it for Batch and Time + is_list (bool, default=False): whether this is a list or a tensor axis + """ + + def __init__(self, kind: AxisKindAbstract, size: Optional[int] = None, is_list=False): + if size is not None and is_list: + raise ValueError("The axis can't be list and have a fixed size") + self.kind = kind + self.size = size + self.is_list = is_list + + def __repr__(self): + if self.size is None: + representation = str(self.kind) + else: + representation = f"{str(self.kind)}:{self.size}" + if self.is_list: + representation += "_listdim" + return representation diff --git a/models/tacotron2/NeMo/nemo/core/neural_types/comparison.py b/models/tacotron2/NeMo/nemo/core/neural_types/comparison.py new file mode 100644 index 0000000000..391096221d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/neural_types/comparison.py @@ -0,0 +1,32 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum + +__all__ = ['NeuralTypeComparisonResult'] + + +class NeuralTypeComparisonResult(Enum): + """The result of comparing two neural type objects for compatibility. + When comparing A.compare_to(B):""" + + SAME = 0 + LESS = 1 # A is B + GREATER = 2 # B is A + DIM_INCOMPATIBLE = 3 # Resize connector might fix incompatibility + TRANSPOSE_SAME = 4 # A transpose and/or converting between lists and tensors will make them same + CONTAINER_SIZE_MISMATCH = 5 # A and B contain different number of elements + INCOMPATIBLE = 6 # A and B are incompatible + SAME_TYPE_INCOMPATIBLE_PARAMS = 7 # A and B are of the same type but parametrized differently + UNCHECKED = 8 # type comparison wasn't done diff --git a/models/tacotron2/NeMo/nemo/core/neural_types/elements.py b/models/tacotron2/NeMo/nemo/core/neural_types/elements.py new file mode 100644 index 0000000000..8a4ffb4cb4 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/neural_types/elements.py @@ -0,0 +1,324 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +from abc import ABC +from typing import Dict, Optional, Tuple + +from nemo.core.neural_types.comparison import NeuralTypeComparisonResult + +__all__ = [ + 'ElementType', + 'VoidType', + 'ChannelType', + 'AcousticEncodedRepresentation', + 'AudioSignal', + 'SpectrogramType', + 'MelSpectrogramType', + 'MFCCSpectrogramType', + 'LogitsType', + 'LabelsType', + 'HypothesisType', + 'LossType', + 'RegressionValuesType', + 'CategoricalValuesType', + 'PredictionsType', + 'LogprobsType', + 'LengthsType', + 'EmbeddedTextType', + 'EncodedRepresentation', + 'MaskType', + 'Target', + 'ClassificationTarget', + 'ImageFeatureValue', + 'Index', + 'ImageValue', + 'NormalizedImageValue', + 'StringLabel', + 'StringType', + 'TokenIndex', + 'Length', + 'IntType', + 'FloatType', + 'NormalDistributionSamplesType', + 'NormalDistributionMeanType', + 'NormalDistributionLogVarianceType', + 'TokenDurationType', + 'TokenLogDurationType', + 'LogDeterminantType', + 'SequenceToSequenceAlignmentType', +] + + +class ElementType(ABC): + """Abstract class defining semantics of the tensor elements. + We are relying on Python for inheritance checking""" + + def __str__(self): + return self.__doc__ + + def __repr__(self): + return self.__class__.__name__ + + @property + def type_parameters(self) -> Dict: + """Override this property to parametrize your type. For example, you can specify 'storage' type such as + float, int, bool with 'dtype' keyword. Another example, is if you want to represent a signal with a + particular property (say, sample frequency), then you can put sample_freq->value in there. + When two types are compared their type_parameters must match.""" + return {} + + @property + def fields(self) -> Optional[Tuple]: + """This should be used to logically represent tuples/structures. For example, if you want to represent a + bounding box (x, y, width, height) you can put a tuple with names ('x', y', 'w', 'h') in here. + Under the hood this should be converted to the last tesnor dimension of fixed size = len(fields). + When two types are compared their fields must match.""" + return None + + def compare(self, second) -> NeuralTypeComparisonResult: + # First, check general compatibility + first_t = type(self) + second_t = type(second) + + if first_t == second_t: + result = NeuralTypeComparisonResult.SAME + elif issubclass(first_t, second_t): + result = NeuralTypeComparisonResult.LESS + elif issubclass(second_t, first_t): + result = NeuralTypeComparisonResult.GREATER + else: + result = NeuralTypeComparisonResult.INCOMPATIBLE + + if result != NeuralTypeComparisonResult.SAME: + return result + else: + # now check that all parameters match + check_params = set(self.type_parameters.keys()) == set(second.type_parameters.keys()) + if check_params is False: + return NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS + else: + for k1, v1 in self.type_parameters.items(): + if v1 is None or second.type_parameters[k1] is None: + # Treat None as Void + continue + if v1 != second.type_parameters[k1]: + return NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS + # check that all fields match + if self.fields == second.fields: + return NeuralTypeComparisonResult.SAME + else: + return NeuralTypeComparisonResult.INCOMPATIBLE + + +class VoidType(ElementType): + """Void-like type which is compatible with everything. + It is a good practice to use this type only as necessary. + For example, when you need template-like functionality. + """ + + def compare(cls, second: abc.ABCMeta) -> NeuralTypeComparisonResult: + return NeuralTypeComparisonResult.SAME + + +# TODO: Consider moving these files elsewhere +class ChannelType(ElementType): + """Element to represent convolutional input/output channel. + """ + + +class EmbeddedTextType(ChannelType): + """Element to represent output on word/text embedding layers + """ + + +class LogitsType(ElementType): + """Element type to represent logits""" + + +class ProbsType(ElementType): + """Element type to represent probabilities. For example, outputs of softmax layers.""" + + +class LogprobsType(ElementType): + """Element type to represent log-probabilities. For example, outputs of log softmax layers.""" + + +class LabelsType(ElementType): + """Element type to represent some sort of labels. This is often used as a base class to create + a more concrete types such as RegressionValuesType, etc.""" + + +class HypothesisType(LabelsType): + """Element type to represent some decoded hypothesis, which may further be processed to obtain + a concrete label.""" + + +class LengthsType(ElementType): + """Element type representing lengths of something""" + + +class LossType(ElementType): + """Element type to represent outputs of Loss modules""" + + +class EncodedRepresentation(ChannelType): + """Element type to represent encoded representation, for example, encoder's output""" + + +class AcousticEncodedRepresentation(EncodedRepresentation): + """Element type to represent encoded representation returned by the acoustic encoder model""" + + +class AudioSignal(ElementType): + """Element type to represent encoded representation returned by the acoustic encoder model + Args: + freq (int): sampling frequency of a signal. Note that two signals will only be the same if their + freq is the same. + """ + + def __init__(self, freq: int = None): + self._params = {} + self._params['freq'] = freq + + @property + def type_parameters(self): + return self._params + + +class SpectrogramType(ChannelType): + """Element type to represent generic spectrogram signal""" + + +class MelSpectrogramType(SpectrogramType): + """Element type to represent mel spectrogram signal""" + + +class MFCCSpectrogramType(SpectrogramType): + """Element type to represent MFCC spectrogram signal""" + + +class PredictionsType(LabelsType): + """Element type to represent some sort of predictions returned by model""" + + +class RegressionValuesType(PredictionsType): + """Element type to represent labels for regression task""" + + +class CategoricalValuesType(PredictionsType): + """Element type to represent labels for categorical classification task""" + + +class MaskType(PredictionsType): + """Element type to represent a boolean mask""" + + +class Index(ElementType): + """Type representing an element being an index of the sample.""" + + +class Target(ElementType): + """ + Type representing an element being a target value. + """ + + +class ClassificationTarget(Target): + """ + Type representing an element being target value in the classification task, i.e. identifier of a desired class. + """ + + +class ImageValue(ElementType): + """ + Type representing an element/value of a single image channel, + e.g. a single element (R) of RGB image. + """ + + +class NormalizedImageValue(ImageValue): + """ + Type representing an element/value of a single image channel normalized to <0-1> range, + e.g. a single element (R) of normalized RGB image. + """ + + +class ImageFeatureValue(ImageValue): + """Type representing an element (single value) of a (image) feature maps.""" + + +class StringType(ElementType): + """Element type representing a single string""" + + +class StringLabel(StringType): + """ + Type representing an label being a string with class name (e.g. the "hamster" class in CIFAR100). + """ + + +class BoolType(ElementType): + """Element type representing a single integer""" + + +class IntType(ElementType): + """Element type representing a single integer""" + + +class FloatType(ElementType): + """Element type representing a single float""" + + +class TokenIndex(IntType): + """Type representing an element being index of a token in some kind of a vocabulary.""" + + +class Length(IntType): + """Type representing an element storing a "length" (e.g. length of a list).""" + + +class ProbabilityDistributionSamplesType(ElementType): + """Element to represent tensors that meant to be sampled from a valid probability distribution + """ + + +class NormalDistributionSamplesType(ProbabilityDistributionSamplesType): + """Element to represent tensors that meant to be sampled from a valid normal distribution + """ + + +class SequenceToSequenceAlignmentType(ElementType): + """Class to represent the alignment from seq-to-seq attention outputs. Generally a mapping from endcoder time steps + to decoder time steps.""" + + +class NormalDistributionMeanType(ElementType): + """Element to represent the mean of a normal distribution""" + + +class NormalDistributionLogVarianceType(ElementType): + """Element to represent the log variance of a normal distribution""" + + +class TokenDurationType(ElementType): + """Element for representing the duration of a token""" + + +class TokenLogDurationType(ElementType): + """Element for representing the log-duration of a token""" + + +class LogDeterminantType(ElementType): + """Element for representing log determinants usually used in flow models""" diff --git a/models/tacotron2/NeMo/nemo/core/neural_types/neural_type.py b/models/tacotron2/NeMo/nemo/core/neural_types/neural_type.py new file mode 100644 index 0000000000..26d64da9b4 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/neural_types/neural_type.py @@ -0,0 +1,243 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Tuple + +from nemo.core.neural_types.axes import AxisKind, AxisType +from nemo.core.neural_types.comparison import NeuralTypeComparisonResult +from nemo.core.neural_types.elements import ElementType, VoidType + +__all__ = [ + 'NeuralType', + 'NeuralTypeError', + 'NeuralPortNameMismatchError', + 'NeuralPortNmTensorMismatchError', +] + + +class NeuralType(object): + """This is the main class which would represent neural type concept. + It is used to represent *the types* of inputs and outputs. + + Args: + axes (Optional[Tuple]): a tuple of AxisTypes objects representing the semantics of what varying each axis means + You can use a short, string-based form here. For example: ('B', 'C', 'H', 'W') would correspond to an NCHW + format frequently used in computer vision. ('B', 'T', 'D') is frequently used for signal processing and + means [batch, time, dimension/channel]. + elements_type (ElementType): an instance of ElementType class representing the semantics of what is stored + inside the tensor. For example: logits (LogitsType), log probabilities (LogprobType), etc. + optional (bool): By default, this is false. If set to True, it would means that input to the port of this + type can be optional. + """ + + def __str__(self): + + if self.axes is not None: + return f"axes: {self.axes}; elements_type: {self.elements_type.__class__.__name__}" + else: + return f"axes: None; elements_type: {self.elements_type.__class__.__name__}" + + def __init__(self, axes: Optional[Tuple] = None, elements_type: ElementType = VoidType(), optional=False): + if not isinstance(elements_type, ElementType): + raise ValueError( + "elements_type of NeuralType must be an instance of a class derived from ElementType. " + "Did you pass a class instead?" + ) + self.elements_type = elements_type + if axes is not None: + NeuralType.__check_sanity(axes) + axes_list = [] + for axis in axes: + if isinstance(axis, str): + axes_list.append(AxisType(AxisKind.from_str(axis), None)) + elif isinstance(axis, AxisType): + axes_list.append(axis) + else: + raise ValueError("axis type must be either str or AxisType instance") + self.axes = tuple(axes_list) + else: + self.axes = None + self.optional = optional + + def compare(self, second) -> NeuralTypeComparisonResult: + """Performs neural type comparison of self with second. When you chain two modules' inputs/outputs via + __call__ method, this comparison will be called to ensure neural type compatibility.""" + # First, handle dimensionality + axes_a = self.axes + axes_b = second.axes + + # "Big void" type + if isinstance(self.elements_type, VoidType) and self.axes is None: + return NeuralTypeComparisonResult.SAME + + if self.axes is None: + if second.axes is None: + return self.elements_type.compare(second.elements_type) + else: + return NeuralTypeComparisonResult.INCOMPATIBLE + + dimensions_pass = NeuralType.__compare_axes(axes_a, axes_b) + element_comparison_result = self.elements_type.compare(second.elements_type) + + # SAME DIMS + if dimensions_pass == 0: + return element_comparison_result + # TRANSPOSE_SAME DIMS + elif dimensions_pass == 1: + if element_comparison_result == NeuralTypeComparisonResult.SAME: + return NeuralTypeComparisonResult.TRANSPOSE_SAME + else: + return NeuralTypeComparisonResult.INCOMPATIBLE + # DIM_INCOMPATIBLE DIMS + elif dimensions_pass == 2: + if element_comparison_result == NeuralTypeComparisonResult.SAME: + return NeuralTypeComparisonResult.DIM_INCOMPATIBLE + else: + return NeuralTypeComparisonResult.INCOMPATIBLE + else: + return NeuralTypeComparisonResult.INCOMPATIBLE + + def compare_and_raise_error(self, parent_type_name, port_name, second_object): + """ Method compares definition of one type with another and raises an error if not compatible. """ + type_comatibility = self.compare(second_object) + if ( + type_comatibility != NeuralTypeComparisonResult.SAME + and type_comatibility != NeuralTypeComparisonResult.GREATER + ): + raise NeuralPortNmTensorMismatchError( + parent_type_name, port_name, str(self), str(second_object.ntype), type_comatibility + ) + + def __eq__(self, other): + if isinstance(other, NeuralType): + return self.compare(other) + + return False + + @staticmethod + def __check_sanity(axes): + # check that list come before any tensor dimension + are_strings = True + for axis in axes: + if not isinstance(axis, str): + are_strings = False + if isinstance(axis, str) and not are_strings: + raise ValueError("Either use full class names or all strings") + if are_strings: + return + checks_passed = True + saw_tensor_dim = False + for axis in axes: + if not axis.is_list: + saw_tensor_dim = True + else: # current axis is a list + if saw_tensor_dim: # which is preceded by tensor dim + checks_passed = False + if not checks_passed: + raise ValueError( + "You have list dimension after Tensor dimension. All list dimensions must preceed Tensor dimensions" + ) + + @staticmethod + def __compare_axes(axes_a, axes_b) -> int: + """ + Compares axes_a and axes_b + Args: + axes_a: first axes tuple + axes_b: second axes tuple + + Returns: + 0 - if they are exactly the same + 1 - if they are "TRANSPOSE_SAME" + 2 - if the are "DIM_INCOMPATIBLE" + 3 - if they are different + """ + if axes_a is None and axes_b is None: + return 0 + elif axes_a is None and axes_b is not None: + return 3 + elif axes_a is not None and axes_b is None: + return 3 + elif len(axes_a) != len(axes_b): + return 3 + # After these ifs we know that len(axes_a) == len(axes_b) + + same = True + kinds_a = dict() + kinds_b = dict() + for axis_a, axis_b in zip(axes_a, axes_b): + kinds_a[axis_a.kind] = axis_a.size + kinds_b[axis_b.kind] = axis_b.size + if axis_a.kind == AxisKind.Any: + same = True + elif ( + axis_a.kind != axis_b.kind + or axis_a.is_list != axis_b.is_list + or (axis_a.size != axis_b.size and axis_a.size is not None) + ): + same = False + if same: + return 0 + else: + # can be TRANSPOSE_SAME, DIM_INCOMPATIBLE + if kinds_a.keys() == kinds_b.keys(): + for key, value in kinds_a.items(): + if kinds_b[key] != value: + return 2 + return 1 + else: + return 3 + + def __repr__(self): + if self.axes is not None: + axes = str(self.axes) + else: + axes = "None" + + if self.elements_type is not None: + element_type = repr(self.elements_type) + else: + element_type = "None" + + data = f"axis={axes}, element_type={element_type}" + + if self.optional: + data = f"{data}, optional={self.optional}" + + final = f"{self.__class__.__name__}({data})" + return final + + +class NeuralTypeError(Exception): + """Base class for neural type related exceptions.""" + + +class NeuralPortNameMismatchError(NeuralTypeError): + """Exception raised when neural module is called with incorrect port + names.""" + + def __init__(self, input_port_name): + super().__init__() + self.message = "Wrong input port name: {0}".format(input_port_name) + + +class NeuralPortNmTensorMismatchError(NeuralTypeError): + """Exception raised when a port is fed with a NmTensor of incompatible + type.""" + + def __init__(self, class_name, port_name, first_type, second_type, type_comatibility): + super().__init__() + self.message = "\nIn {}. \nPort: {} and a NmTensor it was fed are \n".format(class_name, port_name) + self.message += "of incompatible neural types:\n\n{} \n\n and \n\n{}".format(first_type, second_type) + self.message += "\n\nType comparison result: {}".format(type_comatibility) diff --git a/models/tacotron2/NeMo/nemo/core/optim/__init__.py b/models/tacotron2/NeMo/nemo/core/optim/__init__.py new file mode 100644 index 0000000000..b421582c00 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/optim/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.core.optim.lr_scheduler import ( + CosineAnnealing, + InverseSquareRootAnnealing, + NoamAnnealing, + PolynomialDecayAnnealing, + PolynomialHoldDecayAnnealing, + SquareAnnealing, + SquareRootAnnealing, + WarmupAnnealing, + WarmupHoldPolicy, + WarmupPolicy, + prepare_lr_scheduler, +) +from nemo.core.optim.novograd import Novograd +from nemo.core.optim.optimizers import get_optimizer, parse_optimizer_args, register_optimizer diff --git a/models/tacotron2/NeMo/nemo/core/optim/lr_scheduler.py b/models/tacotron2/NeMo/nemo/core/optim/lr_scheduler.py new file mode 100644 index 0000000000..f775ff56bb --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/optim/lr_scheduler.py @@ -0,0 +1,686 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import dataclasses +import math +import warnings +from functools import partial +from typing import Any, Dict, Optional, Union + +import hydra +import torch.optim as optim +import torch.optim.lr_scheduler as pt_scheduler +import torch.utils.data.dataloader as dataloader +from omegaconf import DictConfig, OmegaConf +from torch.optim.lr_scheduler import _LRScheduler + +from nemo.core.config import SchedulerParams, get_scheduler_config, register_scheduler_params +from nemo.utils import logging +from nemo.utils.model_utils import maybe_update_config_version + + +class WarmupPolicy(_LRScheduler): + """Adds warmup kwargs and warmup logic to lr policy. + All arguments should be passed as kwargs for clarity, + Args: + warmup_steps: Number of training steps in warmup stage + warmup_ratio: Ratio of warmup steps to total steps + max_steps: Total number of steps while training or `None` for + infinite training + """ + + def __init__(self, optimizer, *, warmup_steps=None, warmup_ratio=None, max_steps=None, min_lr=0.0, last_epoch=-1): + assert not ( + warmup_steps is not None and warmup_ratio is not None + ), "Either use particular number of step or ratio" + assert warmup_ratio is None or max_steps is not None, "If there is a ratio, there should be a total steps" + + # It is necessary to assign all attributes *before* __init__, + # as class is wrapped by an inner class. + self.max_steps = max_steps + if warmup_steps is not None: + self.warmup_steps = warmup_steps + elif warmup_ratio is not None: + self.warmup_steps = int(warmup_ratio * max_steps) + else: + self.warmup_steps = 0 + + self.min_lr = min_lr + super().__init__(optimizer, last_epoch) + + def get_lr(self): + if not self._get_lr_called_within_step: + warnings.warn( + "To get the last learning rate computed by the scheduler, please use `get_last_lr()`.", UserWarning + ) + + step = self.last_epoch + + if step <= self.warmup_steps: + lr_val = (step + 1) / (self.warmup_steps + 1) + return [initial_lr * lr_val for initial_lr in self.base_lrs] + + if step > self.max_steps: + return [self.min_lr for _ in self.base_lrs] + + return self._get_lr(step) + + def _get_lr(self, step): + """Simple const lr policy""" + return self.base_lrs + + +class WarmupHoldPolicy(WarmupPolicy): + """Variant of WarmupPolicy which maintains high learning rate for a defined number of steps. + All arguments should be passed as kwargs for clarity, + Args: + warmup_steps: Number of training steps in warmup stage + warmup_ratio: Ratio of warmup steps to total steps + hold_steps: Number of training steps to hold the learning rate after warm up + hold_ratio: Ratio of hold steps to total steps + max_steps: Total number of steps while training or `None` for + infinite training + """ + + def __init__( + self, + optimizer, + *, + warmup_steps=None, + warmup_ratio=None, + hold_steps=None, + hold_ratio=None, + max_steps=None, + min_lr=0.0, + last_epoch=-1, + ): + assert not (hold_steps is not None and hold_ratio is not None), "Either use particular number of step or ratio" + assert hold_ratio is None or max_steps is not None, "If there is a ratio, there should be a total steps" + + self.min_lr = min_lr + self._last_warmup_lr = 0.0 + + # Necessary to duplicate as class attributes are hidden in inner class + self.max_steps = max_steps + if warmup_steps is not None: + self.warmup_steps = warmup_steps + elif warmup_ratio is not None: + self.warmup_steps = int(warmup_ratio * max_steps) + else: + self.warmup_steps = 0 + + if hold_steps is not None: + self.hold_steps = hold_steps + self.warmup_steps + elif hold_ratio is not None: + self.hold_steps = int(hold_ratio * max_steps) + self.warmup_steps + else: + self.hold_steps = 0 + + super().__init__( + optimizer, + warmup_steps=warmup_steps, + warmup_ratio=warmup_ratio, + max_steps=max_steps, + last_epoch=last_epoch, + min_lr=min_lr, + ) + + def get_lr(self): + if not self._get_lr_called_within_step: + warnings.warn( + "To get the last learning rate computed by the scheduler, " "please use `get_last_lr()`.", UserWarning + ) + + step = self.last_epoch + + # Warmup phase + if step <= self.warmup_steps: + lr_val = (step + 1) / (self.warmup_steps + 1) + return [initial_lr * lr_val for initial_lr in self.base_lrs] + + # Hold phase + if (step >= self.warmup_steps) and (step < self.hold_steps): + return self.base_lrs + + if step > self.max_steps: + return [self.min_lr for _ in self.base_lrs] + + return self._get_lr(step) + + +def _squareroot_annealing(initial_lr, step, max_steps, min_lr): + mult = ((max_steps - step) / max_steps) ** 0.5 + out_lr = initial_lr * mult + out_lr = max(out_lr, min_lr) + return out_lr + + +def _square_annealing(initial_lr, step, max_steps, min_lr): + mult = ((max_steps - step) / max_steps) ** 2 + out_lr = initial_lr * mult + out_lr = max(out_lr, min_lr) + return out_lr + + +def _cosine_annealing(initial_lr, step, max_steps, min_lr): + mult = 0.5 * (1 + math.cos(math.pi * step / max_steps)) + out_lr = (initial_lr - min_lr) * mult + min_lr + return out_lr + + +def _poly_decay(initial_lr, step, decay_steps, power, min_lr, cycle): + if cycle: + multiplier = 1.0 if step == 0 else math.ceil(step / decay_steps) + decay_steps *= multiplier + else: + step = min(step, decay_steps) + p = step / decay_steps + lr = (initial_lr - min_lr) * math.pow(1.0 - p, power) + lr += min_lr + return lr + + +class SquareAnnealing(WarmupPolicy): + def __init__(self, optimizer, *, max_steps, min_lr=1e-5, last_epoch=-1, **kwargs): + super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) + + def _get_lr(self, step): + new_lrs = [ + _square_annealing( + initial_lr=initial_lr, + step=step - self.warmup_steps, + max_steps=self.max_steps - self.warmup_steps, + min_lr=self.min_lr, + ) + for initial_lr in self.base_lrs + ] + return new_lrs + + +class SquareRootAnnealing(WarmupPolicy): + def __init__(self, optimizer, *, max_steps, min_lr=0, last_epoch=-1, **kwargs): + super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) + + def _get_lr(self, step): + new_lrs = [ + _squareroot_annealing(initial_lr=initial_lr, step=step, max_steps=self.max_steps, min_lr=self.min_lr) + for initial_lr in self.base_lrs + ] + return new_lrs + + +class CosineAnnealing(WarmupPolicy): + def __init__(self, optimizer, *, max_steps, min_lr=0, last_epoch=-1, **kwargs): + super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) + + def _get_lr(self, step): + for initial_lr in self.base_lrs: + if initial_lr < self.min_lr: + raise ValueError( + f"{self} received an initial learning rate that was lower than the minimum learning rate." + ) + + new_lrs = [ + _cosine_annealing( + initial_lr=initial_lr, + step=step - self.warmup_steps, + max_steps=self.max_steps - self.warmup_steps, + min_lr=self.min_lr, + ) + for initial_lr in self.base_lrs + ] + return new_lrs + + +class NoamAnnealing(_LRScheduler): + def __init__( + self, optimizer, *, d_model, warmup_steps=None, warmup_ratio=None, max_steps=None, min_lr=0.0, last_epoch=-1 + ): + self._normalize = d_model ** (-0.5) + assert not ( + warmup_steps is not None and warmup_ratio is not None + ), "Either use particular number of step or ratio" + assert warmup_ratio is None or max_steps is not None, "If there is a ratio, there should be a total steps" + + # It is necessary to assign all attributes *before* __init__, + # as class is wrapped by an inner class. + self.max_steps = max_steps + if warmup_steps is not None: + self.warmup_steps = warmup_steps + elif warmup_ratio is not None: + self.warmup_steps = int(warmup_ratio * max_steps) + else: + self.warmup_steps = 0 + + self.min_lr = min_lr + super().__init__(optimizer, last_epoch) + + def get_lr(self): + if not self._get_lr_called_within_step: + warnings.warn( + "To get the last learning rate computed by the scheduler, please use `get_last_lr()`.", UserWarning + ) + + step = max(1, self.last_epoch) + + if step > self.max_steps: + return [self.min_lr for _ in self.base_lrs] + + for initial_lr in self.base_lrs: + if initial_lr < self.min_lr: + raise ValueError( + f"{self} received an initial learning rate that was lower than the minimum learning rate." + ) + + new_lrs = [self._noam_annealing(initial_lr=initial_lr, step=step) for initial_lr in self.base_lrs] + return new_lrs + + def _noam_annealing(self, initial_lr, step): + mult = self._normalize * min(step ** (-0.5), step * (self.warmup_steps ** (-1.5))) + out_lr = initial_lr * mult + if step > self.warmup_steps: + out_lr = max(out_lr, self.min_lr) + return out_lr + + +class WarmupAnnealing(WarmupPolicy): + def __init__(self, optimizer, *, max_steps, last_epoch=-1, min_lr=0.0, **kwargs): + super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) + + def _get_lr(self, step): + progress = float(step / self.max_steps) + warmup_ratio = float(self.warmup_steps / self.max_steps) + + mult = max((progress - 1.0) / (warmup_ratio - 1.0), 0.0) + out_lr = [initial_lr * mult for initial_lr in self.base_lrs] + + return out_lr + + +class InverseSquareRootAnnealing(WarmupPolicy): + def __init__(self, optimizer, *, max_steps, last_epoch=-1, min_lr=0.0, **kwargs): + super().__init__(optimizer=optimizer, max_steps=max_steps, **kwargs, last_epoch=last_epoch, min_lr=min_lr) + + def _get_lr(self, step): + denom = ((step + 1) / (self.warmup_steps + 1)) ** 0.5 + out_lr = [initial_lr / denom for initial_lr in self.base_lrs] + return out_lr + + +class PolynomialDecayAnnealing(WarmupPolicy): + def __init__(self, optimizer, *, max_steps, min_lr=0.0, power=1.0, cycle=False, last_epoch=-1, **kwargs): + self.power = power + self.cycle = cycle + + super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) + + def _get_lr(self, step): + new_lrs = [ + _poly_decay( + initial_lr, + step=step - self.warmup_steps, + decay_steps=self.max_steps - self.warmup_steps, + power=self.power, + min_lr=self.min_lr, + cycle=self.cycle, + ) + for initial_lr in self.base_lrs + ] + return new_lrs + + +class PolynomialHoldDecayAnnealing(WarmupHoldPolicy): + def __init__(self, optimizer, *, max_steps, min_lr=0.0, power=1.0, cycle=False, last_epoch=-1, **kwargs): + self.power = power + self.cycle = cycle + + super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) + + def _get_lr(self, step): + new_lrs = [ + _poly_decay( + initial_lr, + step=step - self.hold_steps, + decay_steps=self.max_steps - max(self.warmup_steps, self.hold_steps), + power=self.power, + min_lr=self.min_lr, + cycle=self.cycle, + ) + for initial_lr in self.base_lrs + ] + return new_lrs + + +def register_scheduler(name: str, scheduler: _LRScheduler, scheduler_params: SchedulerParams): + """ + Checks if the scheduler name exists in the registry, and if it doesnt, adds it. + + This allows custom schedulers to be added and called by name during instantiation. + + Args: + name: Name of the optimizer. Will be used as key to retrieve the optimizer. + scheduler: Scheduler class (inherits from _LRScheduler) + scheduler_params: The parameters as a dataclass of the scheduler + """ + if name in AVAILABLE_SCHEDULERS: + raise ValueError(f"Cannot override pre-existing schedulers. Conflicting scheduler name = {name}") + + AVAILABLE_SCHEDULERS[name] = scheduler + + sched_name = "{}_params".format(scheduler.__name__) + register_scheduler_params(name=sched_name, scheduler_params=scheduler_params) + + +def get_scheduler(name: str, **kwargs: Optional[Dict[str, Any]]) -> _LRScheduler: + """ + Convenience method to obtain an _LRScheduler class and partially instantiate it with optimizer kwargs. + + Args: + name: Name of the scheduler in the registry. + kwargs: Optional kwargs of the scheduler used during instantiation. + + Returns: + a partially instantiated _LRScheduler + """ + if name not in AVAILABLE_SCHEDULERS: + raise ValueError( + f"Cannot resolve scheduler{name}'. Available optimizers are : " f"{AVAILABLE_SCHEDULERS.keys()}" + ) + + scheduler_cls = AVAILABLE_SCHEDULERS[name] + scheduler = partial(scheduler_cls, **kwargs) + return scheduler + + +def prepare_lr_scheduler( + optimizer: optim.Optimizer, + scheduler_config: Union[Dict[str, Any], DictConfig], + train_dataloader: Optional[dataloader.DataLoader] = None, +) -> Optional[Dict[str, Any]]: + """ + Constructs an LR Scheduler (optionally) for a given optimizer, based on a config with the following schema + + optim: + name: + lr: + + # + args: + name: auto # special keyword, resolves to correct optimizer config for given optimizer name + # cls: nemo.core.config.optimizers.NovogradParams # explicit instantiation by class path + params: # optional override parameters for the optimizer config + betas: [0.8, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: + iters_per_batch: null # computed at runtime; mandatory to have + max_steps: null # computed at runtime or explicitly set here; mandatory to have + + # pytorch lightning args + monitor: val_loss + reduce_on_plateau: false + + # + args: + name: auto # special keyword, resolves to correct optimizer config for given optimizer name + # cls: nemo.core.config.schedulers.CosineAnnealingParams # explicit instantiation by class path + params: # optional override parameters for the optimizer config + warmup_steps: null + warmup_ratio: null + min_lr: 0.0 + last_epoch: -1 + + Args: + optimizer: An instantiated Optimizer. + scheduler_config: A dictionary / config dict which follows the above schema. + train_dataloader: Optional requirement, must be passed if "iters_per_batch" is defined + instead of "max_steps". Used to compute effective "max_steps". + + Returns: + A dictionary containing the LR Scheduler implementation if the config was successfully parsed + along with other parameters required by Pytorch Lightning, otherwise None. + """ + if scheduler_config is not None: + scheduler_config = maybe_update_config_version(scheduler_config) + + # Build nested dictionary for convenience out of structured objects + if isinstance(scheduler_config, DictConfig): + scheduler_config = OmegaConf.to_container(scheduler_config, resolve=True) + + elif dataclasses.is_dataclass(scheduler_config): + # Recursively transform data classes to basic dictionaries + scheduler_config = OmegaConf.create(scheduler_config) + scheduler_config = OmegaConf.to_container(scheduler_config, resolve=True) + + # Test to see if config follows above schema + + add_max_args_flag = True + interval = 'step' + if scheduler_config is not None: + if 'args' in scheduler_config: + scheduler_args = scheduler_config.pop('args') + else: + scheduler_args = copy.deepcopy(scheduler_config) + + # Remove extra parameters from scheduler_args nest + # Assume all other parameters are to be passed into scheduler constructor + + if 'name' in scheduler_args and scheduler_args['name'] == 'ReduceLROnPlateau': + add_max_args_flag = False + interval = 'epoch' + + scheduler_args.pop('name', None) + scheduler_args.pop('t_max_epochs', None) + scheduler_args.pop('t_accumulate_grad_batches', None) + scheduler_args.pop('t_limit_train_batches', None) + scheduler_args.pop('t_num_workers', None) + scheduler_args.pop('monitor', None) + scheduler_args.pop('reduce_on_plateau', None) + + else: + # Return gracefully in case `sched` was not supplied; inform user + logging.info('Scheduler not initialized as no `sched` config supplied to setup_optimizer()') + return None + + # Try instantiation of scheduler params from config class path + if '_target_' in scheduler_args: + scheduler_args_cfg = OmegaConf.create(scheduler_args) + scheduler_conf = hydra.utils.instantiate(scheduler_args_cfg) + scheduler_args = vars(scheduler_conf) + + # Get name of the scheduler + scheduler_name = scheduler_conf.__class__.__name__ + + if 'Params' in scheduler_name: + scheduler_name = scheduler_name.replace('Params', '') + + else: + # Class path instantiation failed; try resolving "name" component + + # Get name of the scheduler + if 'name' in scheduler_config: + scheduler_name = scheduler_config['name'] + else: + logging.warning( + "Could not resolve classpath for Scheduler Config, and `name` " + "was not provided either. \n" + "Scheduler cannot be instantiated !" + ) + return None + + # If class path was not provided, perhaps `name` is provided for resolution + if 'name' in scheduler_args: + # If `auto` is passed as name for resolution of optimizer name, + # then lookup optimizer name and resolve its parameter config + if scheduler_args['name'] == 'auto': + scheduler_params_name = "{}Params".format(scheduler_name) + else: + scheduler_params_name = scheduler_args['name'] + + # Get override arguments provided in the config yaml file / Dict Config + scheduler_params_override = scheduler_args.get('params', {}) + + # If params is itself a dict config object provided explicitly in Dict Config + # Resolve to dictionary for convenience + if isinstance(scheduler_params_override, DictConfig): + scheduler_params_override = OmegaConf.to_container(scheduler_params_override, resolve=True) + + # Get and instantiate the Config dataclass for this scheduler + scheduler_params_cls = get_scheduler_config(scheduler_params_name, **scheduler_params_override) + scheduler_params = scheduler_params_cls() # instantiate the parameters object + scheduler_args = vars(scheduler_params) # extract just the dictionary from the Config object + + else: + # assume the input dictionary is schedular args (from dataclasses / omegaconf) + pass + + # Extract value to monitor in losses, if provided. + if 'monitor' in scheduler_config: + monitor = scheduler_config.get('monitor') + else: + # Default to train loss + monitor = 'loss' + + # Store exact max_steps if it is provided + if 'max_steps' in scheduler_config and scheduler_config['max_steps'] is not None: + max_steps = scheduler_config['max_steps'] + + elif 't_max_epochs' in scheduler_config: + # Compute effective max_steps if t_max_epochs is provided + if train_dataloader is None: + logging.warning( + 'As `t_max_epochs` is provided/computed, it is required to pass the train dataloader in order\n' + 'to compute effective maximum number of steps.\n' + 'Scheduler will not be instantiated !' + ) + return None + + # Raise exception if neither `max_steps` nor `t_max_epochs` is provided + if scheduler_config.get('t_max_epochs', None) is None: + logging.warning( + "`t_max_epochs` cannot be None when `max_steps` is not not provided.\n" + "This can occur when `train dataloader` is not available to correctly " + "prepare the scheduler.\n" + "Scheduler will not be instantiated !" + ) + return None + + # Get iters_per_batch + max_epochs = scheduler_config.get('t_max_epochs') + accumulate_grad_batches = scheduler_config.get('t_accumulate_grad_batches') + limit_train_batches = scheduler_config.get('t_limit_train_batches') + num_workers = scheduler_config.get('t_num_workers') + + # Compute effective num max_steps + num_samples = len(train_dataloader.dataset) + batch_size = train_dataloader.batch_size + drop_last = train_dataloader.drop_last + + max_steps = compute_max_steps( + max_epochs=max_epochs, + accumulate_grad_batches=accumulate_grad_batches, + limit_train_batches=limit_train_batches, + num_workers=num_workers, + num_samples=num_samples, + batch_size=batch_size, + drop_last=drop_last, + ) + + else: + logging.warning( + "Neither `max_steps` nor `iters_per_batch` were provided to `optim.sched`, " + "cannot compute effective `max_steps` !\n" + "Scheduler will not be instantiated !" + ) + return None + + # Inject max_steps (effective or provided) into the scheduler config + if add_max_args_flag: + scheduler_args['max_steps'] = max_steps + + # Get the scheduler class from the config + scheduler_cls = get_scheduler(scheduler_name, **scheduler_args) + + # Instantiate the LR schedule + schedule = scheduler_cls(optimizer, **scheduler_args) + + logging.info( + 'Scheduler "%s" \nwill be used during training (effective maximum steps = %d) - \nParameters : \n(%s)', + str(schedule), + max_steps, + OmegaConf.to_yaml(OmegaConf.create(scheduler_args)), + ) + + # Wrap the schedule in PTL arguments to perform stepwise computation + # Rather than epoch level computation + if isinstance(schedule, optim.lr_scheduler.ReduceLROnPlateau): + reduce_lr_on_plateau = True + else: + reduce_lr_on_plateau = False + + schedule_dict = { + 'scheduler': schedule, + 'interval': interval, + 'frequency': 1, + 'monitor': monitor, + 'reduce_on_plateau': reduce_lr_on_plateau, + } + return schedule_dict + + +def compute_max_steps( + max_epochs, accumulate_grad_batches, limit_train_batches, num_workers, num_samples, batch_size, drop_last +): + _round = math.floor if drop_last else math.ceil + + sampler_num_samples = math.ceil(num_samples / num_workers) + + if drop_last and num_workers > 1: + logging.warning( + "Please note that drop_last is broken in pytorch 1.6.0. We will fix when pytorch 1.7.0 is released" + ) + # TODO: Master verion, not in pytorch 1.6.0 + # sampler_num_samples = math.ceil((num_samples - num_workers)/ num_workers) + + steps_per_epoch = _round(sampler_num_samples / batch_size) + if isinstance(limit_train_batches, int) or limit_train_batches == 0.0: + steps_per_epoch = min(steps_per_epoch, int(limit_train_batches)) + elif steps_per_epoch != float('inf'): + # limit_train_batches is a percentage of batches per epoch + steps_per_epoch = int(steps_per_epoch * limit_train_batches) + if accumulate_grad_batches == 1: + steps_per_epoch = max(steps_per_epoch, 1) + + return math.ceil(steps_per_epoch / accumulate_grad_batches) * max_epochs + + +AVAILABLE_SCHEDULERS = { + 'WarmupPolicy': WarmupPolicy, + 'WarmupHoldPolicy': WarmupHoldPolicy, + 'SquareAnnealing': SquareAnnealing, + 'CosineAnnealing': CosineAnnealing, + 'NoamAnnealing': NoamAnnealing, + 'WarmupAnnealing': WarmupAnnealing, + 'InverseSquareRootAnnealing': InverseSquareRootAnnealing, + 'SquareRootAnnealing': SquareRootAnnealing, + 'PolynomialDecayAnnealing': PolynomialDecayAnnealing, + 'PolynomialHoldDecayAnnealing': PolynomialHoldDecayAnnealing, + 'StepLR': pt_scheduler.StepLR, + 'ExponentialLR': pt_scheduler.ExponentialLR, + 'ReduceLROnPlateau': pt_scheduler.ReduceLROnPlateau, + 'CyclicLR': pt_scheduler.CyclicLR, +} diff --git a/models/tacotron2/NeMo/nemo/core/optim/novograd.py b/models/tacotron2/NeMo/nemo/core/optim/novograd.py new file mode 100644 index 0000000000..8e39ab9719 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/optim/novograd.py @@ -0,0 +1,145 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torch.optim.optimizer import Optimizer + +__all__ = ['Novograd'] + + +def _check_valid_opt_params(lr, eps, betas): + if lr < 0: + raise ValueError(f"Invalid learning rate: {lr}") + if eps < 0: + raise ValueError(f"Invalid epsilon value: {eps}") + if not (0.0 <= betas[0] < 1.0 and 0.0 <= betas[1] < 1.0): + raise ValueError(f"Betas have to be between 0 and 1: {betas}") + + +class Novograd(Optimizer): + """Implements Novograd algorithm. + It has been proposed in "Stochastic Gradient Methods with Layer-wise + Adaptive Moments for Training of Deep Networks" + (https://arxiv.org/abs/1905.11286) + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + amsgrad (boolean, optional): whether to use the AMSGrad variant of this + algorithm from the paper "On the Convergence of Adam and Beyond" + """ + + def __init__( + self, + params, + lr=1e-3, + betas=(0.95, 0.98), + eps=1e-8, + weight_decay=0, + grad_averaging=False, + amsgrad=False, + luc=False, + luc_trust=1e-3, + luc_eps=1e-8, + ): + _check_valid_opt_params(lr, eps, betas) + defaults = dict( + lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, grad_averaging=grad_averaging, amsgrad=amsgrad, + ) + self.luc = luc + self.luc_trust = luc_trust + self.luc_eps = luc_eps + super(Novograd, self).__init__(params, defaults) + + def __setstate__(self, state): + super(Novograd, self).__setstate__(state) + for group in self.param_groups: + group.setdefault("amsgrad", False) + + def step(self, closure=None): + """Performs a single optimization step. + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group["params"]: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError("Sparse gradients are not supported.") + amsgrad = group["amsgrad"] + state = self.state[p] + + # State initialization + if not state: + state["step"] = 0 + # Exponential moving average of gradient values + state["exp_avg"] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state["exp_avg_sq"] = torch.zeros([]).to(state["exp_avg"].device) + if amsgrad: + # Maintains max of all exp moving avg of squared grad + state["max_exp_avg_sq"] = torch.zeros([]).to(state["exp_avg"].device) + + exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"] + if amsgrad: + max_exp_avg_sq = state["max_exp_avg_sq"] + beta1, beta2 = group["betas"] + + state["step"] += 1 + + norm = grad.norm().pow(2) + + if exp_avg_sq == 0: + exp_avg_sq.copy_(norm) + else: + exp_avg_sq.mul_(beta2).add_(norm, alpha=1.0 - beta2) + + if amsgrad: + # Maintains max of all 2nd moment running avg till now + torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq) + # Use the max for normalizing running avg. of gradient + denom = max_exp_avg_sq.sqrt().add_(group["eps"]) + else: + denom = exp_avg_sq.sqrt().add_(group["eps"]) + + grad.div_(denom) + if group["weight_decay"] != 0: + grad.add_(p.data, alpha=group["weight_decay"]) + if group["grad_averaging"]: + grad.mul_(1 - beta1) + exp_avg.mul_(beta1).add_(grad) + + if self.luc: + # Clip update so that updates are less than eta*weights + data_norm = torch.norm(p.data) + grad_norm = torch.norm(exp_avg.data) + luc_factor = self.luc_trust * data_norm / (grad_norm + self.luc_eps) + luc_factor = min(luc_factor, group["lr"]) + p.data.add_(exp_avg, alpha=-luc_factor) + else: + p.data.add_(exp_avg, alpha=-group["lr"]) + + return loss diff --git a/models/tacotron2/NeMo/nemo/core/optim/optimizers.py b/models/tacotron2/NeMo/nemo/core/optim/optimizers.py new file mode 100644 index 0000000000..127e44beed --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/optim/optimizers.py @@ -0,0 +1,171 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +from functools import partial +from typing import Any, Dict, List, Optional, Union + +import hydra +import torch.optim as optim +from omegaconf import DictConfig, OmegaConf +from torch.optim import adadelta, adagrad, adamax, rmsprop, rprop +from torch.optim.optimizer import Optimizer + +from nemo.core.config import OptimizerParams, get_optimizer_config, register_optimizer_params +from nemo.core.optim.novograd import Novograd +from nemo.utils import logging +from nemo.utils.model_utils import maybe_update_config_version + +AVAILABLE_OPTIMIZERS = { + 'sgd': optim.SGD, + 'adam': optim.Adam, + 'adamw': optim.AdamW, + 'adadelta': adadelta.Adadelta, + 'adamax': adamax.Adamax, + 'adagrad': adagrad.Adagrad, + 'rmsprop': rmsprop.RMSprop, + 'rprop': rprop.Rprop, + 'novograd': Novograd, +} + +try: + from apex.optimizers import FusedLAMB + + AVAILABLE_OPTIMIZERS['lamb'] = FusedLAMB +except ModuleNotFoundError: + logging.warning("Apex was not found. Using the lamb optimizer will error out.") + +__all__ = ['get_optimizer', 'register_optimizer', 'parse_optimizer_args'] + + +def parse_optimizer_args( + optimizer_name: str, optimizer_kwargs: Union[DictConfig, Dict[str, Any]] +) -> Union[Dict[str, Any], DictConfig]: + """ + Parses a list of strings, of the format "key=value" or "key2=val1,val2,..." + into a dictionary of type {key=value, key2=[val1, val2], ...} + + This dictionary is then used to instantiate the chosen Optimizer. + + Args: + optimizer_name: string name of the optimizer, used for auto resolution of params + optimizer_kwargs: Either a list of strings in a specified format, + or a dictionary. If a dictionary is provided, it is assumed the dictionary + is the final parsed value, and simply returned. + If a list of strings is provided, each item in the list is parsed into a + new dictionary. + + Returns: + A dictionary + """ + kwargs = {} + + if optimizer_kwargs is None: + return kwargs + + optimizer_kwargs = copy.deepcopy(optimizer_kwargs) + optimizer_kwargs = maybe_update_config_version(optimizer_kwargs) + + if isinstance(optimizer_kwargs, DictConfig): + optimizer_kwargs = OmegaConf.to_container(optimizer_kwargs, resolve=True) + + # If it is a dictionary, perform stepwise resolution + if hasattr(optimizer_kwargs, 'keys'): + # Attempt class path resolution + if '_target_' in optimizer_kwargs: # captures (target, _target_) + optimizer_kwargs_config = OmegaConf.create(optimizer_kwargs) + optimizer_instance = hydra.utils.instantiate(optimizer_kwargs_config) # type: DictConfig + optimizer_instance = vars(optimizer_instance) + return optimizer_instance + + # If class path was not provided, perhaps `name` is provided for resolution + if 'name' in optimizer_kwargs: + # If `auto` is passed as name for resolution of optimizer name, + # then lookup optimizer name and resolve its parameter config + if optimizer_kwargs['name'] == 'auto': + optimizer_params_name = "{}_params".format(optimizer_name) + optimizer_kwargs.pop('name') + else: + optimizer_params_name = optimizer_kwargs.pop('name') + + # Override arguments provided in the config yaml file + if 'params' in optimizer_kwargs: + # If optimizer kwarg overrides are wrapped in yaml `params` + optimizer_params_override = optimizer_kwargs.get('params') + else: + # If the kwargs themselves are a DictConfig + optimizer_params_override = optimizer_kwargs + + if isinstance(optimizer_params_override, DictConfig): + optimizer_params_override = OmegaConf.to_container(optimizer_params_override, resolve=True) + + optimizer_params_cls = get_optimizer_config(optimizer_params_name, **optimizer_params_override) + + # If we are provided just a Config object, simply return the dictionary of that object + if optimizer_params_name is None: + optimizer_params = vars(optimizer_params_cls) + return optimizer_params + + else: + # If we are provided a partial class instantiation of a Config, + # Instantiate it and retrieve its vars as a dictionary + optimizer_params = optimizer_params_cls() # instantiate the parameters object + optimizer_params = vars(optimizer_params) + return optimizer_params + + # simply return the dictionary that was provided + return optimizer_kwargs + + return kwargs + + +def register_optimizer(name: str, optimizer: Optimizer, optimizer_params: OptimizerParams): + """ + Checks if the optimizer name exists in the registry, and if it doesnt, adds it. + + This allows custom optimizers to be added and called by name during instantiation. + + Args: + name: Name of the optimizer. Will be used as key to retrieve the optimizer. + optimizer: Optimizer class + optimizer_params: The parameters as a dataclass of the optimizer + """ + if name in AVAILABLE_OPTIMIZERS: + raise ValueError(f"Cannot override pre-existing optimizers. Conflicting optimizer name = {name}") + + AVAILABLE_OPTIMIZERS[name] = optimizer + + optim_name = "{}_params".format(optimizer.__name__) + register_optimizer_params(name=optim_name, optimizer_params=optimizer_params) + + +def get_optimizer(name: str, **kwargs: Optional[Dict[str, Any]]) -> Optimizer: + """ + Convenience method to obtain an Optimizer class and partially instantiate it with optimizer kwargs. + + Args: + name: Name of the Optimizer in the registry. + kwargs: Optional kwargs of the optimizer used during instantiation. + + Returns: + a partially instantiated Optimizer + """ + if name not in AVAILABLE_OPTIMIZERS: + raise ValueError( + f"Cannot resolve optimizer '{name}'. Available optimizers are : " f"{AVAILABLE_OPTIMIZERS.keys()}" + ) + + optimizer = AVAILABLE_OPTIMIZERS[name] + optimizer = partial(optimizer, **kwargs) + return optimizer diff --git a/models/tacotron2/NeMo/nemo/core/utils/__init__.py b/models/tacotron2/NeMo/nemo/core/utils/__init__.py new file mode 100644 index 0000000000..cf01eed5ab --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/utils/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.core.utils import numba_utils diff --git a/models/tacotron2/NeMo/nemo/core/utils/numba_utils.py b/models/tacotron2/NeMo/nemo/core/utils/numba_utils.py new file mode 100644 index 0000000000..1f53dead53 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/core/utils/numba_utils.py @@ -0,0 +1,142 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import logging as pylogger +import operator +import os + +from nemo.utils import model_utils + +# Prevent Numba CUDA logs from showing at info level +cuda_logger = pylogger.getLogger('numba.cuda.cudadrv.driver') +cuda_logger.setLevel(pylogger.ERROR) # only show error + +__NUMBA_MINIMUM_VERSION__ = "0.53.0" + + +NUMBA_INSTALLATION_MESSAGE = ( + "Could not import `numba`.\n" + "Please install numba in one of the following ways." + "1) If using conda, simply install it with conda using `conda install -c numba numba`\n" + "2) If using pip (not recommended), `pip install --upgrade numba`\n" + "followed by `export NUMBAPRO_LIBDEVICE='/usr/local/cuda/nvvm/libdevice/'` and \n" + "`export NUMBAPRO_NVVM='/usr/local/cuda/nvvm/lib64/libnvvm.so'`.\n" + "It is advised to always install numba using conda only, " + "as pip installations might interfere with other libraries such as llvmlite.\n" + "If pip install does not work, you can also try adding `--ignore-installed` to the pip command,\n" + "but this is not advised." +) + +STRICT_NUMBA_COMPAT_CHECK = True + +# Get environment key if available +if 'STRICT_NUMBA_COMPAT_CHECK' in os.environ: + check_str = os.environ.get('STRICT_NUMBA_COMPAT_CHECK') + check_bool = str(check_str).lower() in ("yes", "true", "t", "1") + STRICT_NUMBA_COMPAT_CHECK = check_bool + + +def is_numba_compat_strict() -> bool: + """ + Returns strictness level of numba cuda compatibility checks. + + If value is true, numba cuda compatibility matrix must be satisfied. + If value is false, only cuda availability is checked, not compatibility. + Numba Cuda may still compile and run without issues in such a case, or it may fail. + """ + return STRICT_NUMBA_COMPAT_CHECK + + +def set_numba_compat_strictness(strict: bool): + """ + Sets the strictness level of numba cuda compatibility checks. + + If value is true, numba cuda compatibility matrix must be satisfied. + If value is false, only cuda availability is checked, not compatibility. + Numba Cuda may still compile and run without issues in such a case, or it may fail. + + Args: + strict: bool value, whether to enforce strict compatibility checks or relax them. + """ + global STRICT_NUMBA_COMPAT_CHECK + STRICT_NUMBA_COMPAT_CHECK = strict + + +@contextlib.contextmanager +def with_numba_compat_strictness(strict: bool): + initial_strictness = is_numba_compat_strict() + set_numba_compat_strictness(strict=strict) + yield + set_numba_compat_strictness(strict=initial_strictness) + + +def numba_cuda_is_supported(min_version: str) -> bool: + """ + Tests if an appropriate version of numba is installed, and if it is, + if cuda is supported properly within it. + + Args: + min_version: The minimum version of numba that is required. + + Returns: + bool, whether cuda is supported with this current installation or not. + """ + module_available, msg = model_utils.check_lib_version('numba', checked_version=min_version, operator=operator.ge) + + # If numba is not installed + if module_available is None: + return False + + # If numba version is installed and available + if module_available is True: + from numba import cuda + + # this method first arrived in 0.53, and that's the minimum version required + if hasattr(cuda, 'is_supported_version'): + try: + cuda_available = cuda.is_available() + if cuda_available: + cuda_compatible = cuda.is_supported_version() + else: + cuda_compatible = False + + if is_numba_compat_strict(): + return cuda_available and cuda_compatible + else: + return cuda_available + + except OSError: + # dlopen(libcudart.dylib) might fail if CUDA was never installed in the first place. + return False + else: + # assume cuda is supported, but it may fail due to CUDA incompatibility + return False + + else: + return False + + +def skip_numba_cuda_test_if_unsupported(min_version: str): + """ + Helper method to skip pytest test case if numba cuda is not supported. + + Args: + min_version: The minimum version of numba that is required. + """ + numba_cuda_support = numba_cuda_is_supported(min_version) + if not numba_cuda_support: + import pytest + + pytest.skip(f"Numba cuda test is being skipped. Minimum version required : {min_version}") diff --git a/models/tacotron2/NeMo/nemo/package_info.py b/models/tacotron2/NeMo/nemo/package_info.py new file mode 100644 index 0000000000..cb39b87bf6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/package_info.py @@ -0,0 +1,35 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +MAJOR = 1 +MINOR = 0 +PATCH = 2 +PRE_RELEASE = '' + +# Use the following formatting: (major, minor, patch, pre-release) +VERSION = (MAJOR, MINOR, PATCH, PRE_RELEASE) + +__shortversion__ = '.'.join(map(str, VERSION[:3])) +__version__ = '.'.join(map(str, VERSION[:3])) + ''.join(VERSION[3:]) + +__package_name__ = 'nemo_toolkit' +__contact_names__ = 'NVIDIA' +__contact_emails__ = 'nemo-toolkit@nvidia.com' +__homepage__ = 'https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/' +__repository_url__ = 'https://github.com/nvidia/nemo' +__download_url__ = 'https://github.com/NVIDIA/NeMo/releases' +__description__ = 'NeMo - a toolkit for Conversational AI' +__license__ = 'Apache2' +__keywords__ = 'deep learning, machine learning, gpu, NLP, NeMo, nvidia, pytorch, torch, tts, speech, language' diff --git a/models/tacotron2/NeMo/nemo/utils/__init__.py b/models/tacotron2/NeMo/nemo/utils/__init__.py new file mode 100644 index 0000000000..139dbd8423 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.utils.app_state import AppState +from nemo.utils.nemo_logging import Logger as _Logger +from nemo.utils.nemo_logging import LogMode as logging_mode +from nemo.utils.lightning_logger_patch import add_memory_handlers_to_pl_logger + +logging = _Logger() +add_memory_handlers_to_pl_logger() diff --git a/models/tacotron2/NeMo/nemo/utils/app_state.py b/models/tacotron2/NeMo/nemo/utils/app_state.py new file mode 100644 index 0000000000..f2c7104431 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/app_state.py @@ -0,0 +1,386 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from threading import Lock +from typing import Dict, Optional + +from nemo.utils.metaclasses import Singleton + + +@dataclass() +class ModelMetadataRegistry: + guid: str + gidx: int + restoration_path: Optional[str] = None + + +class AppState(metaclass=Singleton): + def __init__(self): + # method call lock + self.__lock = Lock() + + # TODO: should we store global config in hydra_runner? + self._app_cfg = None + + # World info + self._device_id = None + self._local_rank = None + self._global_rank = None + self._model_parallel_rank = None + self._data_parallel_rank = None + + self._world_size = None + self._model_parallel_size = None + self._model_parallel_group = None + self._data_parallel_size = None + self._data_parallel_group = None + + self._random_seed = None + + # Logging info + self._log_dir = None + self._exp_dir = None + self._name = None + self._checkpoint_name = None + self._version = None + self._create_checkpoint_callback = None + self._checkpoint_callback_params = None + + # Save and Restore (.nemo) + self._tmpdir_name = None + self._model_config_yaml = "model_config.yaml" + self._model_weights_ckpt = "model_weights.ckpt" + self._model_restore_path = None + self._all_model_restore_paths = [] + self._model_guid_map = {} # type: Dict[str, ModelMetadataRegistry] + + @property + def device_id(self): + """ Property returns the device_id + Returns: + device_id + """ + return self._device_id + + @device_id.setter + def device_id(self, id): + """ Property sets the device_id. + Args: + size (int): The device id. + """ + self._device_id = id + + @property + def world_size(self): + """ Property returns the total number of GPUs. + Returns: + Total number of GPUs. + """ + return self._world_size + + @world_size.setter + def world_size(self, size): + """ Property sets the total number of GPUs. + Args: + size (int): Total number of GPUs. + """ + self._world_size = size + + @property + def model_parallel_size(self): + """ Property returns the number of GPUs in each model parallel group. + Returns: + Number of GPUs in each model parallel group. + """ + return self._model_parallel_size + + @model_parallel_size.setter + def model_parallel_size(self, size): + """ Property sets the number of GPUs in each model parallel group. + Args: + size (int): Number of GPUs in each model parallel group. + """ + self._model_parallel_size = size + + @property + def data_parallel_size(self): + """ Property returns the number of GPUs in each data parallel group. + Returns: + Number of GPUs in each data parallel group. + """ + return self._data_parallel_size + + @data_parallel_size.setter + def data_parallel_size(self, size): + """ Property sets the number of GPUs in each data parallel group. + Args: + size (int): Number of GPUs in each data parallel group. + """ + self._data_parallel_size = size + + @property + def local_rank(self): + """ Property returns the local rank. + Returns: + Local rank. + """ + return self._local_rank + + @local_rank.setter + def local_rank(self, rank): + """ Property sets the local rank. + Args: + rank (int): Local rank. + """ + self._local_rank = rank + + @property + def global_rank(self): + """ Property returns the global rank. + Returns: + Global rank. + """ + return self._global_rank + + @global_rank.setter + def global_rank(self, rank): + """ Property sets the global rank. + Args: + rank (int): Global rank. + """ + self._global_rank = rank + + @property + def model_parallel_rank(self): + """ Property returns the model parallel rank. + Returns: + Model parallel rank. + """ + return self._model_parallel_rank + + @model_parallel_rank.setter + def model_parallel_rank(self, rank): + """ Property sets the model parallel rank. + Args: + rank (int): Model parallel rank. + """ + self._model_parallel_rank = rank + + @property + def model_parallel_group(self): + """ Property returns the model parallel group. + Returns: + Model parallel group. + """ + return self._model_parallel_group + + @model_parallel_group.setter + def model_parallel_group(self, group): + """ Property sets the model parallel group. + Args: + group: Model parallel group. + """ + self._model_parallel_group = group + + @property + def data_parallel_rank(self): + """ Property returns the data parallel rank. + Returns: + Data parallel rank. + """ + return self._data_parallel_rank + + @data_parallel_rank.setter + def data_parallel_rank(self, rank): + """ Property sets the data parallel rank. + Args: + rank (int): Data parallel rank. + """ + self._data_parallel_rank = rank + + @property + def data_parallel_group(self): + """ Property returns the data parallel group. + Returns: + Data parallel group. + """ + return self._data_parallel_group + + @data_parallel_group.setter + def data_parallel_group(self, group): + """ Property sets the data parallel group. + Args: + group: Data parallel group. + """ + self._data_parallel_group = group + + @property + def random_seed(self): + """ Property returns the random seed. + Returns: + Random seed. + """ + return self._random_seed + + @random_seed.setter + def random_seed(self, seed): + """ Property sets the random seed. + Args: + seed (int): Random seed. + """ + self._random_seed = seed + + @property + def log_dir(self): + """Returns the log_dir set by exp_manager. + """ + return self._log_dir + + @log_dir.setter + def log_dir(self, dir): + """Sets the log_dir property. + + Args: + dir (str): Log_dir set by exp_manager. + """ + self._log_dir = dir + + @property + def exp_dir(self): + """Returns the exp_dir set by exp_manager. + """ + return self._exp_dir + + @exp_dir.setter + def exp_dir(self, dir): + """Sets the log_dir property. + + Args: + dir (str): Log_dir set by exp_manager. + """ + self._exp_dir = dir + + @property + def name(self): + """Returns the name set by exp_manager. + """ + return self._name + + @name.setter + def name(self, name): + """Sets the name property. + + Args: + dir (str): name set by exp_manager. + """ + self._name = name + + @property + def checkpoint_name(self): + """Returns the name set by exp_manager. + """ + return self._checkpoint_name + + @checkpoint_name.setter + def checkpoint_name(self, name): + """Sets the name property. + + Args: + dir (str): name set by exp_manager. + """ + self._checkpoint_name = name + + @property + def version(self): + """Returns the version set by exp_manager. + """ + return self._version + + @version.setter + def version(self, version): + """Sets the version property. + + Args: + dir (str): version set by exp_manager. + """ + self._version = version + + @property + def create_checkpoint_callback(self): + """Returns the create_checkpoint_callback set by exp_manager. + """ + return self._create_checkpoint_callback + + @create_checkpoint_callback.setter + def create_checkpoint_callback(self, create_checkpoint_callback): + """Sets the create_checkpoint_callback property. + + Args: + dir (bool): create_checkpoint_callback set by exp_manager. + """ + self._create_checkpoint_callback = create_checkpoint_callback + + @property + def checkpoint_callback_params(self): + """Returns the version set by exp_manager. + """ + return self._checkpoint_callback_params + + @checkpoint_callback_params.setter + def checkpoint_callback_params(self, params): + """Sets the name property. + + Args: + params (dict): checkpoint_callback_params set by exp_manager. + """ + self._checkpoint_callback_params = params + + @property + def model_config_yaml(self): + return self._model_config_yaml + + @property + def model_weights_ckpt(self): + return self._model_weights_ckpt + + @property + def model_restore_path(self): + restore_path = self._all_model_restore_paths[-1] if len(self._all_model_restore_paths) > 0 else None + return restore_path + + @model_restore_path.setter + def model_restore_path(self, path): + with self.__lock: + self._model_restore_path = path + self._all_model_restore_paths.append(path) + + def register_model_guid(self, guid: str, restoration_path: Optional[str] = None): + # Maps a guid to its restore path (None or last absolute path) + with self.__lock: + if guid in self._model_guid_map: + idx = self._model_guid_map[guid].gidx + else: + idx = len(self._model_guid_map) + self._model_guid_map[guid] = ModelMetadataRegistry(guid, idx, restoration_path=restoration_path) + + def reset_model_guid_registry(self): + # Reset the guid mapping + with self.__lock: + self._model_guid_map.clear() + + def get_model_metadata_from_guid(self, guid) -> ModelMetadataRegistry: + # Returns the global model idx and restoration path + metadata = self._model_guid_map[guid] + return metadata diff --git a/models/tacotron2/NeMo/nemo/utils/arguments.py b/models/tacotron2/NeMo/nemo/utils/arguments.py new file mode 100644 index 0000000000..12d825728e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/arguments.py @@ -0,0 +1,132 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser +from typing import Any, Dict, List, Optional, Union + + +def add_optimizer_args( + parent_parser: ArgumentParser, + optimizer: str = 'adam', + default_lr: float = None, + default_opt_args: Optional[Union[Dict[str, Any], List[str]]] = None, +) -> ArgumentParser: + """Extends existing argparse with support for optimizers. + + # Example of adding optimizer args to command line : + python train_script.py ... --optimizer "novograd" --lr 0.01 \ + --opt_args betas=0.95,0.5 weight_decay=0.001 + + Args: + parent_parser (ArgumentParser): Custom CLI parser that will be extended. + optimizer (str): Default optimizer required. + default_lr (float): Default learning rate that should be overriden during training. + default_opt_args (list(str)): List of overriding arguments for the instantiated optimizer. + + Returns: + ArgumentParser: Parser extended by Optimizers arguments. + """ + if default_opt_args is None: + default_opt_args = [] + + parser = ArgumentParser(parents=[parent_parser], add_help=True, conflict_handler='resolve') + + parser.add_argument('--optimizer', type=str, default=optimizer, help='Name of the optimizer. Defaults to Adam.') + parser.add_argument('--lr', type=float, default=default_lr, help='Learning rate of the optimizer.') + parser.add_argument( + '--opt_args', + default=default_opt_args, + nargs='+', + type=str, + help='Overriding arguments for the optimizer. \n Must follow the pattern : \n name=value separated by spaces.' + 'Example: --opt_args weight_decay=0.001 eps=1e-8 betas=0.9,0.999', + ) + + return parser + + +def add_scheduler_args(parent_parser: ArgumentParser) -> ArgumentParser: + """Extends existing argparse with default LR scheduler args. + + Args: + parent_parser (ArgumentParser): Custom CLI parser that will be extended. + + Returns: + ArgumentParser: Parser extended by LR Scheduler arguments. + """ + parser = ArgumentParser(parents=[parent_parser], add_help=False, conflict_handler='resolve') + parser.add_argument("--warmup_steps", type=int, required=False, default=None, help="Number of warmup steps") + parser.add_argument( + "--warmup_ratio", + type=float, + required=False, + default=None, + help="Number of warmup steps as a percentage of total training steps", + ) + parser.add_argument("--hold_steps", type=int, required=False, default=None, help="Number of hold LR steps") + parser.add_argument( + "--hold_ratio", + type=float, + required=False, + default=None, + help="Number of hold LR steps as a percentage of total training steps", + ) + parser.add_argument("--min_lr", type=float, required=False, default=0.0, help="Minimum learning rate") + parser.add_argument( + "--last_epoch", type=int, required=False, default=-1, help="Last epoch id. -1 indicates training from scratch" + ) + return parser + + +def add_asr_args(parent_parser: ArgumentParser) -> ArgumentParser: + """Extends existing argparse with default ASR collection args. + + Args: + parent_parser (ArgumentParser): Custom CLI parser that will be extended. + + Returns: + ArgumentParser: Parser extended by NeMo ASR Collection arguments. + """ + parser = ArgumentParser(parents=[parent_parser], add_help=False, conflict_handler='resolve') + parser.add_argument("--asr_model", type=str, required=True, default="bad_quartznet15x5.yaml", help="") + parser.add_argument("--train_dataset", type=str, required=True, default=None, help="training dataset path") + parser.add_argument("--eval_dataset", type=str, required=True, help="evaluation dataset path") + return parser + + +def add_nlp_args(parent_parser: ArgumentParser) -> ArgumentParser: + """Extends existing argparse with default NLP collection args. + + Args: + parent_parser (ArgumentParser): Custom CLI parser that will be extended. + + Returns: + ArgumentParser: Parser extended by NeMo NLP Collection arguments. + """ + parser = ArgumentParser(parents=[parent_parser], add_help=False, conflict_handler='resolve') + parser.add_argument( + "--data_dir", type=str, required=False, help="data directory to training or/and evaluation dataset" + ) + parser.add_argument( + "--config_file", type=str, required=False, default=None, help="Huggingface model configuration file" + ) + parser.add_argument( + "--pretrained_model_name", default='bert-base-uncased', type=str, required=False, help="pretrained model name" + ) + parser.add_argument( + "--tokenizer_name", default='nemobert', type=str, choices=['sentencepiece', 'nemobert'], help="Tokenizer type" + ) + parser.add_argument("--tokenizer_model", default=None, type=str, help="Tokenizer file for sentence piece") + parser.add_argument("--do_lower_case", action='store_true', required=False, help="lower case data") + return parser diff --git a/models/tacotron2/NeMo/nemo/utils/cloud.py b/models/tacotron2/NeMo/nemo/utils/cloud.py new file mode 100644 index 0000000000..8b74963f12 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/cloud.py @@ -0,0 +1,82 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path +from time import sleep + +import wget + +from nemo.utils import logging + + +def maybe_download_from_cloud(url, filename, subfolder=None, cache_dir=None, refresh_cache=False) -> str: + """ + Helper function to download pre-trained weights from the cloud + Args: + url: (str) URL of storage + filename: (str) what to download. The request will be issued to url/filename + subfolder: (str) subfolder within cache_dir. The file will be stored in cache_dir/subfolder. Subfolder can + be empty + cache_dir: (str) a cache directory where to download. If not present, this function will attempt to create it. + If None (default), then it will be $HOME/.cache/torch/NeMo + refresh_cache: (bool) if True and cached file is present, it will delete it and re-fetch + + Returns: + If successful - absolute local path to the downloaded file + else - empty string + """ + # try: + if cache_dir is None: + cache_location = Path.joinpath(Path.home(), ".cache/torch/NeMo") + else: + cache_location = cache_dir + if subfolder is not None: + destination = Path.joinpath(cache_location, subfolder) + else: + destination = cache_location + + if not os.path.exists(destination): + os.makedirs(destination, exist_ok=True) + + destination_file = Path.joinpath(destination, filename) + + if os.path.exists(destination_file): + logging.info(f"Found existing object {destination_file}.") + if refresh_cache: + logging.info("Asked to refresh the cache.") + logging.info(f"Deleting file: {destination_file}") + os.remove(destination_file) + else: + logging.info(f"Re-using file from: {destination_file}") + return str(destination_file) + # download file + wget_uri = url + filename + logging.info(f"Downloading from: {wget_uri} to {str(destination_file)}") + # NGC links do not work everytime so we try and wait + i = 0 + max_attempts = 3 + while i < max_attempts: + i += 1 + try: + wget.download(wget_uri, str(destination_file)) + if os.path.exists(destination_file): + return destination_file + else: + return "" + except: + logging.info(f"Download from cloud failed. Attempt {i} of {max_attempts}") + sleep(0.05) + continue + raise ValueError("Not able to download url right now, please try again.") diff --git a/models/tacotron2/NeMo/nemo/utils/config_utils.py b/models/tacotron2/NeMo/nemo/utils/config_utils.py new file mode 100644 index 0000000000..952e20305e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/config_utils.py @@ -0,0 +1,258 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import inspect +from dataclasses import is_dataclass +from typing import Dict, List, Optional + +from omegaconf import DictConfig, OmegaConf, open_dict + +from nemo.core.config.modelPT import NemoConfig +from nemo.utils import logging + + +def update_model_config(model_cls: NemoConfig, update_cfg: DictConfig, drop_missing_subconfigs: bool = True): + """ + Helper class that updates the default values of a ModelPT config class with the values + in a DictConfig that mirrors the structure of the config class. + + Assumes the `update_cfg` is a DictConfig (either generated manually, via hydra or instantiated via yaml/model.cfg). + This update_cfg is then used to override the default values preset inside the ModelPT config class. + + If `drop_missing_subconfigs` is set, the certain sub-configs of the ModelPT config class will be removed, iff + they are not found in the mirrored `update_cfg`. The following sub-configs are subject to potential removal: + - `train_ds` + - `validation_ds` + - `test_ds` + - `optim` + nested `sched`. + + Args: + model_cls: A subclass of NemoConfig, that details in entirety all of the parameters that constitute + the NeMo Model. + + update_cfg: A DictConfig that mirrors the structure of the NemoConfig data class. Used to update the + default values of the config class. + + drop_missing_subconfigs: Bool which determins whether to drop certain sub-configs from the NemoConfig + class, if the corresponding sub-config is missing from `update_cfg`. + + Returns: + A DictConfig with updated values that can be used to instantiate the NeMo Model along with supporting + infrastructure. + """ + if not (is_dataclass(model_cls) or isinstance(model_cls, DictConfig)): + raise ValueError("`model_cfg` must be a dataclass or a structured OmegaConf object") + + if not isinstance(update_cfg, DictConfig): + update_cfg = OmegaConf.create(update_cfg) + + if is_dataclass(model_cls): + model_cls = OmegaConf.structured(model_cls) + + # Update optional configs + model_cls = _update_subconfig( + model_cls, update_cfg, subconfig_key='train_ds', drop_missing_subconfigs=drop_missing_subconfigs + ) + model_cls = _update_subconfig( + model_cls, update_cfg, subconfig_key='validation_ds', drop_missing_subconfigs=drop_missing_subconfigs + ) + model_cls = _update_subconfig( + model_cls, update_cfg, subconfig_key='test_ds', drop_missing_subconfigs=drop_missing_subconfigs + ) + model_cls = _update_subconfig( + model_cls, update_cfg, subconfig_key='optim', drop_missing_subconfigs=drop_missing_subconfigs + ) + + # Add optim and sched additional keys to model cls + model_cls = _add_subconfig_keys(model_cls, update_cfg, subconfig_key='optim') + + # Perform full merge of model config class and update config + # Remove ModelPT artifact `target` + if 'target' in update_cfg.model: + # Assume artifact from ModelPT and pop + if 'target' not in model_cls.model: + with open_dict(update_cfg.model): + update_cfg.model.pop('target') + + model_cfg = OmegaConf.merge(model_cls, update_cfg) + + return model_cfg + + +def _update_subconfig( + model_cfg: DictConfig, update_cfg: DictConfig, subconfig_key: str, drop_missing_subconfigs: bool +): + """ + Updates the NemoConfig DictConfig such that: + 1) If the sub-config key exists in the `update_cfg`, but does not exist in ModelPT config: + - Add the sub-config from update_cfg to ModelPT config + + 2) If the sub-config key does not exist in `update_cfg`, but exists in ModelPT config: + - Remove the sub-config from the ModelPT config; iff the `drop_missing_subconfigs` flag is set. + + Args: + model_cfg: A DictConfig instantiated from the NemoConfig subclass. + update_cfg: A DictConfig that mirrors the structure of `model_cfg`, used to update its default values. + subconfig_key: A str key used to check and update the sub-config. + drop_missing_subconfigs: A bool flag, whether to allow deletion of the NemoConfig sub-config, + if its mirror sub-config does not exist in the `update_cfg`. + + Returns: + The updated DictConfig for the NemoConfig + """ + with open_dict(model_cfg.model): + # If update config has the key, but model cfg doesnt have the key + # Add the update cfg subconfig to the model cfg + if subconfig_key in update_cfg.model and subconfig_key not in model_cfg.model: + model_cfg.model[subconfig_key] = update_cfg.model[subconfig_key] + + # If update config does not the key, but model cfg has the key + # Remove the model cfg subconfig in order to match layout of update cfg + if subconfig_key not in update_cfg.model and subconfig_key in model_cfg.model: + if drop_missing_subconfigs: + model_cfg.model.pop(subconfig_key) + + return model_cfg + + +def _add_subconfig_keys(model_cfg: DictConfig, update_cfg: DictConfig, subconfig_key: str): + """ + For certain sub-configs, the default values specified by the NemoConfig class is insufficient. + In order to support every potential value in the merge between the `update_cfg`, it would require + explicit definition of all possible cases. + + An example of such a case is Optimizers, and their equivalent Schedulers. All optimizers share a few basic + details - such as name and lr, but almost all require additional parameters - such as weight decay. + It is impractical to create a config for every single optimizer + every single scheduler combination. + + In such a case, we perform a dual merge. The Optim and Sched Dataclass contain the bare minimum essential + components. The extra values are provided via update_cfg. + + In order to enable the merge, we first need to update the update sub-config to incorporate the keys, + with dummy temporary values (merge update config with model config). This is done on a copy of the + update sub-config, as the actual override values might be overriden by the NemoConfig defaults. + + Then we perform a merge of this temporary sub-config with the actual override config in a later step + (merge model_cfg with original update_cfg, done outside this function). + + Args: + model_cfg: A DictConfig instantiated from the NemoConfig subclass. + update_cfg: A DictConfig that mirrors the structure of `model_cfg`, used to update its default values. + subconfig_key: A str key used to check and update the sub-config. + + Returns: + A ModelPT DictConfig with additional keys added to the sub-config. + """ + with open_dict(model_cfg.model): + # Create copy of original model sub config + if subconfig_key in update_cfg.model: + if subconfig_key not in model_cfg.model: + # create the key as a placeholder + model_cfg.model[subconfig_key] = None + + subconfig = copy.deepcopy(model_cfg.model[subconfig_key]) + update_subconfig = copy.deepcopy(update_cfg.model[subconfig_key]) + + # Add the keys and update temporary values, will be updated during full merge + subconfig = OmegaConf.merge(update_subconfig, subconfig) + # Update sub config + model_cfg.model[subconfig_key] = subconfig + + return model_cfg + + +def assert_dataclass_signature_match( + cls: 'class_type', + datacls: 'dataclass', + ignore_args: Optional[List[str]] = None, + remap_args: Optional[Dict[str, str]] = None, +): + """ + Analyses the signature of a provided class and its respective data class, + asserting that the dataclass signature matches the class __init__ signature. + + Note: + This is not a value based check. This function only checks if all argument + names exist on both class and dataclass and logs mismatches. + + Args: + cls: Any class type - but not an instance of a class. Pass type(x) where x is an instance + if class type is not easily available. + datacls: A corresponding dataclass for the above class. + ignore_args: (Optional) A list of string argument names which are forcibly ignored, + even if mismatched in the signature. Useful when a dataclass is a superset of the + arguments of a class. + remap_args: (Optional) A dictionary, mapping an argument name that exists (in either the + class or its dataclass), to another name. Useful when argument names are mismatched between + a class and its dataclass due to indirect instantiation via a helper method. + + Returns: + A tuple containing information about the analysis: + 1) A bool value which is True if the signatures matched exactly / after ignoring values. + False otherwise. + 2) A set of arguments names that exist in the class, but *do not* exist in the dataclass. + If exact signature match occurs, this will be None instead. + 3) A set of argument names that exist in the data class, but *do not* exist in the class itself. + If exact signature match occurs, this will be None instead. + """ + class_sig = inspect.signature(cls.__init__) + + class_params = dict(**class_sig.parameters) + class_params.pop('self') + + dataclass_sig = inspect.signature(datacls) + + dataclass_params = dict(**dataclass_sig.parameters) + dataclass_params.pop("_target_", None) + + class_params = set(class_params.keys()) + dataclass_params = set(dataclass_params.keys()) + + if remap_args is not None: + for original_arg, new_arg in remap_args.items(): + if original_arg in class_params: + class_params.remove(original_arg) + class_params.add(new_arg) + logging.info(f"Remapped {original_arg} -> {new_arg} in {cls.__name__}") + + if original_arg in dataclass_params: + dataclass_params.remove(original_arg) + dataclass_params.add(new_arg) + logging.info(f"Remapped {original_arg} -> {new_arg} in {datacls.__name__}") + + if ignore_args is not None: + ignore_args = set(ignore_args) + + class_params = class_params - ignore_args + dataclass_params = dataclass_params - ignore_args + logging.info(f"Removing ignored arguments - {ignore_args}") + + intersection = set.intersection(class_params, dataclass_params) + subset_cls = class_params - intersection + subset_datacls = dataclass_params - intersection + + if (len(class_params) != len(dataclass_params)) or len(subset_cls) > 0 or len(subset_datacls) > 0: + logging.error(f"Class {cls.__name__} arguments do not match " f"Dataclass {datacls.__name__}!") + + if len(subset_cls) > 0: + logging.error(f"Class {cls.__name__} has additional arguments :\n" f"{subset_cls}") + + if len(subset_datacls): + logging.error(f"Dataclass {datacls.__name__} has additional arguments :\n{subset_datacls}") + + return False, subset_cls, subset_datacls + + else: + return True, None, None diff --git a/models/tacotron2/NeMo/nemo/utils/decorators/__init__.py b/models/tacotron2/NeMo/nemo/utils/decorators/__init__.py new file mode 100644 index 0000000000..4468a3bc09 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/decorators/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.utils.decorators.deprecated import deprecated +from nemo.utils.decorators.experimental import experimental +from nemo.utils.decorators.port_docs import add_port_docs diff --git a/models/tacotron2/NeMo/nemo/utils/decorators/deprecated.py b/models/tacotron2/NeMo/nemo/utils/decorators/deprecated.py new file mode 100644 index 0000000000..f409a0f69c --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/decorators/deprecated.py @@ -0,0 +1,72 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +__all__ = [ + 'deprecated', +] + +import functools + +import wrapt + +from nemo.utils import logging + +# Remember which deprecation warnings have been printed already. +_PRINTED_WARNING = {} + + +def deprecated(wrapped=None, version=None, explanation=None): + """ Decorator class used for indicating that a function is deprecated and going to be removed. + Tracks down which functions printed the warning and will print it only once per function. + """ + + if wrapped is None: + return functools.partial(deprecated, version=version, explanation=explanation) + + @wrapt.decorator + def wrapper(wrapped, instance, args, kwargs): + """ + Method prints the adequate warning (only once per function) when + required and calls the function func, passing the original arguments, + i.e. version and explanation. + + Args: + version: Version in which the function will be removed (optional) + explanation: Additional explanation (optional), e.g. use method ``blabla instead``. + """ + + # Check if we already warned about that function. + if wrapped.__name__ not in _PRINTED_WARNING.keys(): + # Add to list so we won't print it again. + _PRINTED_WARNING[wrapped.__name__] = True + + # Prepare the warning message. + msg = "Function ``{}`` is deprecated.".format(wrapped.__name__) + + # Optionally, add version and alternative. + if version is not None: + msg = msg + " It is going to be removed in " + msg = msg + "the {} version.".format(version) + + if explanation is not None: + msg = msg + " " + explanation + + # Display the deprecated warning. + logging.warning(msg) + + # Call the function. + return wrapped(*args, **kwargs) + + return wrapper(wrapped) diff --git a/models/tacotron2/NeMo/nemo/utils/decorators/experimental.py b/models/tacotron2/NeMo/nemo/utils/decorators/experimental.py new file mode 100644 index 0000000000..35b26fb869 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/decorators/experimental.py @@ -0,0 +1,33 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +__all__ = ['experimental'] + +from nemo.utils import logging + + +def experimental(cls): + """ Decorator which indicates that module is experimental. + Use it to mark experimental or research modules. + """ + + def wrapped(cls): + logging.warning( + f'Module {cls} is experimental, not ready for production and is not fully supported. Use at your own risk.' + ) + + return cls + + return wrapped(cls=cls) diff --git a/models/tacotron2/NeMo/nemo/utils/decorators/port_docs.py b/models/tacotron2/NeMo/nemo/utils/decorators/port_docs.py new file mode 100644 index 0000000000..e47a72d8f7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/decorators/port_docs.py @@ -0,0 +1,90 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# The "add_port_docs" decorator is needed to nicely generate neural types in Sphynx for input and output ports + +__all__ = [ + 'add_port_docs', +] + +import functools +import sys + +import wrapt + + +def _normalize_docstring(docstring): + """Normalizes the docstring. + Replaces tabs with spaces, removes leading and trailing blanks lines, and + removes any indentation. + Copied from PEP-257: + https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation + Args: + docstring: the docstring to normalize + Returns: + The normalized docstring + """ + if not docstring: + return '' + # Convert tabs to spaces (following the normal Python rules) + # and split into a list of lines: + lines = docstring.expandtabs().splitlines() + # Determine minimum indentation (first line doesn't count): + # (we use sys.maxsize because sys.maxint doesn't exist in Python 3) + indent = sys.maxsize + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < sys.maxsize: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + # Strip off trailing and leading blank lines: + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + # Return a single string: + return '\n'.join(trimmed) + + +def add_port_docs(wrapped=None, instance=None, value=''): + if wrapped is None: + return functools.partial(add_port_docs, value=value) + + @wrapt.decorator + def wrapper(wrapped, instance=None, args=None, kwargs=None): + return wrapped(*args, **kwargs) + + decorated = wrapper(wrapped) + try: + port_2_ntype = decorated(instance) + except: + port_2_ntype = None + + port_description = "" + if port_2_ntype is not None: + for port, ntype in port_2_ntype.items(): + port_description += "* *" + port + "* : " + str(ntype) + port_description += "\n\n" + + __doc__ = _normalize_docstring(wrapped.__doc__) + '\n\n' + str(port_description) + __doc__ = _normalize_docstring(__doc__) + + wrapt.FunctionWrapper.__setattr__(decorated, "__doc__", __doc__) + + return decorated diff --git a/models/tacotron2/NeMo/nemo/utils/env_var_parsing.py b/models/tacotron2/NeMo/nemo/utils/env_var_parsing.py new file mode 100644 index 0000000000..bd37f2b18e --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/env_var_parsing.py @@ -0,0 +1,207 @@ +# The MIT Licence (MIT) +# +# Copyright (c) 2016 YunoJuno Ltd +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Vendored dependency from : https://github.com/yunojuno/python-env-utils/blob/master/env_utils/utils.py +# +# +# Modified by NVIDIA +# +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import decimal +import json +import os + +from dateutil import parser + +__all__ = [ + "get_env", + "get_envbool", + "get_envint", + "get_envfloat", + "get_envdecimal", + "get_envdate", + "get_envdatetime", + "get_envlist", + "get_envdict", + "CoercionError", + "RequiredSettingMissingError", +] + + +class CoercionError(Exception): + """Custom error raised when a value cannot be coerced.""" + + def __init__(self, key, value, func): + msg = "Unable to coerce '{}={}' using {}.".format(key, value, func.__name__) + super(CoercionError, self).__init__(msg) + + +class RequiredSettingMissingError(Exception): + """Custom error raised when a required env var is missing.""" + + def __init__(self, key): + msg = "Required env var '{}' is missing.".format(key) + super(RequiredSettingMissingError, self).__init__(msg) + + +def _get_env(key, default=None, coerce=lambda x: x, required=False): + """ + Return env var coerced into a type other than string. + This function extends the standard os.getenv function to enable + the coercion of values into data types other than string (all env + vars are strings by default). + Args: + key: string, the name of the env var to look up + Kwargs: + default: the default value to return if the env var does not exist. NB the + default value is **not** coerced, and is assumed to be of the correct type. + coerce: a function that is used to coerce the value returned into + another type + required: bool, if True, then a RequiredSettingMissingError error is raised + if the env var does not exist. + Returns the env var, passed through the coerce function + """ + try: + value = os.environ[key] + except KeyError: + if required is True: + raise RequiredSettingMissingError(key) + else: + return default + + try: + return coerce(value) + except Exception: + raise CoercionError(key, value, coerce) + + +# standard type coercion functions +def _bool(value): + if isinstance(value, bool): + return value + + return not (value is None or value.lower() in ("false", "0", "no", "n", "f", "none")) + + +def _int(value): + return int(value) + + +def _float(value): + return float(value) + + +def _decimal(value): + return decimal.Decimal(value) + + +def _dict(value): + return json.loads(value) + + +def _datetime(value): + return parser.parse(value) + + +def _date(value): + return parser.parse(value).date() + + +def get_env(key, *default, **kwargs): + """ + Return env var. + This is the parent function of all other get_foo functions, + and is responsible for unpacking args/kwargs into the values + that _get_env expects (it is the root function that actually + interacts with environ). + Args: + key: string, the env var name to look up. + default: (optional) the value to use if the env var does not + exist. If this value is not supplied, then the env var is + considered to be required, and a RequiredSettingMissingError + error will be raised if it does not exist. + Kwargs: + coerce: a func that may be supplied to coerce the value into + something else. This is used by the default get_foo functions + to cast strings to builtin types, but could be a function that + returns a custom class. + Returns the env var, coerced if required, and a default if supplied. + """ + assert len(default) in (0, 1), "Too many args supplied." + func = kwargs.get('coerce', lambda x: x) + required = len(default) == 0 + default = default[0] if not required else None + return _get_env(key, default=default, coerce=func, required=required) + + +def get_envbool(key, *default): + """Return env var cast as boolean.""" + return get_env(key, *default, coerce=_bool) + + +def get_envint(key, *default): + """Return env var cast as integer.""" + return get_env(key, *default, coerce=_int) + + +def get_envfloat(key, *default): + """Return env var cast as float.""" + return get_env(key, *default, coerce=_float) + + +def get_envdecimal(key, *default): + """Return env var cast as Decimal.""" + return get_env(key, *default, coerce=_decimal) + + +def get_envdate(key, *default): + """Return env var as a date.""" + return get_env(key, *default, coerce=_date) + + +def get_envdatetime(key, *default): + """Return env var as a datetime.""" + return get_env(key, *default, coerce=_datetime) + + +def get_envlist(key, *default, **kwargs): + """Return env var as a list.""" + separator = kwargs.get('separator', ' ') + return get_env(key, *default, coerce=lambda x: x.split(separator)) + + +def get_envdict(key, *default): + """Return env var as a dict.""" + return get_env(key, *default, coerce=_dict) diff --git a/models/tacotron2/NeMo/nemo/utils/exceptions.py b/models/tacotron2/NeMo/nemo/utils/exceptions.py new file mode 100644 index 0000000000..baf7808e87 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/exceptions.py @@ -0,0 +1,19 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class NeMoBaseException(Exception): + """ NeMo Base Exception. All exceptions created in NeMo should inherit from this class""" + + pass diff --git a/models/tacotron2/NeMo/nemo/utils/exp_manager.py b/models/tacotron2/NeMo/nemo/utils/exp_manager.py new file mode 100644 index 0000000000..8e9400075d --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/exp_manager.py @@ -0,0 +1,715 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys +import time +from copy import deepcopy +from dataclasses import dataclass +from pathlib import Path +from shutil import copy, move +from typing import Any, Dict, List, Optional, Union + +import torch +from hydra.core.hydra_config import HydraConfig +from hydra.utils import get_original_cwd +from omegaconf import DictConfig, OmegaConf, open_dict +from pytorch_lightning.callbacks import ModelCheckpoint +from pytorch_lightning.loggers import LoggerCollection as _LoggerCollection +from pytorch_lightning.loggers import TensorBoardLogger, WandbLogger +from pytorch_lightning.utilities import rank_zero_only + +from nemo.constants import NEMO_ENV_VARNAME_VERSION +from nemo.utils import app_state, logging +from nemo.utils.app_state import AppState +from nemo.utils.exceptions import NeMoBaseException +from nemo.utils.get_rank import is_global_rank_zero +from nemo.utils.lightning_logger_patch import add_filehandlers_to_pl_logger + + +class NotFoundError(NeMoBaseException): + """ Raised when a file or folder is not found""" + + +class LoggerMisconfigurationError(NeMoBaseException): + """ Raised when a mismatch between trainer.logger and exp_manager occurs""" + + def __init__(self, message): + message = ( + message + + " You can disable lighning's trainer from creating a logger by passing logger=False to its constructor." + ) + super().__init__(message) + + +class CheckpointMisconfigurationError(NeMoBaseException): + """ Raised when a mismatch between trainer.callbacks and exp_manager occurs""" + + +@dataclass +class CallbackParams: + filepath: Optional[str] = None # Deprecated + dirpath: Optional[str] = None # If None, exp_manager will attempt to handle the filepath + filename: Optional[str] = None # If None, exp_manager will attempt to handle the filepath + monitor: Optional[str] = "val_loss" + verbose: Optional[bool] = True + save_last: Optional[bool] = True + save_top_k: Optional[int] = 3 + save_weights_only: Optional[bool] = False + mode: Optional[str] = "min" + period: Optional[int] = 1 + prefix: Optional[str] = None # If None, exp_manager will attempt to handle the filepath + postfix: str = ".nemo" + save_best_model: bool = False + always_save_nemo: bool = False + + +@dataclass +class ExpManagerConfig: + # Log dir creation parameters + explicit_log_dir: Optional[str] = None + exp_dir: Optional[str] = None + name: Optional[str] = None + version: Optional[str] = None + use_datetime_version: Optional[bool] = True + resume_if_exists: Optional[bool] = False + resume_past_end: Optional[bool] = False + resume_ignore_no_checkpoint: Optional[bool] = False + # Logging parameters + create_tensorboard_logger: Optional[bool] = True + summary_writer_kwargs: Optional[Dict[Any, Any]] = None + create_wandb_logger: Optional[bool] = False + wandb_logger_kwargs: Optional[Dict[Any, Any]] = None + # Checkpointing parameters + create_checkpoint_callback: Optional[bool] = True + checkpoint_callback_params: Optional[CallbackParams] = CallbackParams() + # Additional exp_manager arguments + files_to_copy: Optional[List[str]] = None + + +def exp_manager(trainer: 'pytorch_lightning.Trainer', cfg: Optional[Union[DictConfig, Dict]] = None) -> Path: + """ + exp_manager is a helper function used to manage folders for experiments. It follows the pytorch lightning paradigm + of exp_dir/model_or_experiment_name/version. If the lightning trainer has a logger, exp_manager will get exp_dir, + name, and version from the logger. Otherwise it will use the exp_dir and name arguments to create the logging + directory. exp_manager also allows for explicit folder creation via explicit_log_dir. + + The version can be a datetime string or an integer. Datestime version can be disabled if use_datetime_version is set + to False. It optionally creates TensorBoardLogger, WandBLogger, ModelCheckpoint objects from pytorch lightning. + It copies sys.argv, and git information if available to the logging directory. It creates a log file for each + process to log their output into. + + exp_manager additionally has a resume feature (resume_if_exists) which can be used to continuing training from + the constructed log_dir. When you need to continue the training repeatedly (like on a cluster which you need + multiple consecutive jobs), you need to avoid creating the version folders. Therefore from v1.0.0, when + resume_if_exists is set to True, creating the version folders is ignored. + + Args: + trainer (pytorch_lightning.Trainer): The lightning trainer. + cfg (DictConfig, dict): Can have the following keys: + - explicit_log_dir (str, Path): Can be used to override exp_dir/name/version folder creation. Defaults to + None, which will use exp_dir, name, and version to construct the logging directory. + - exp_dir (str, Path): The base directory to create the logging directory. Defaults to None, which logs to + ./nemo_experiments. + - name (str): The name of the experiment. Defaults to None which turns into "default" via name = name or + "default". + - version (str): The version of the experiment. Defaults to None which uses either a datetime string or + lightning's TensorboardLogger system of using version_{int}. + - use_datetime_version (bool): Whether to use a datetime string for version. Defaults to True. + - resume_if_exists (bool): Whether this experiment is resuming from a previous run. If True, it sets + trainer.resume_from_checkpoint so that the trainer should auto-resume. exp_manager will move files + under log_dir to log_dir/run_{int}. Defaults to False. From v1.0.0, when resume_if_exists is True, + we would not create version folders to make it easier to find the log folder for next runs. + - resume_past_end (bool): exp_manager errors out if resume_if_exists is True and a checkpoint matching + *end.ckpt indicating a previous training run fully completed. This behaviour can be disabled, in which + case the *end.ckpt will be loaded by setting resume_past_end to True. Defaults to False. + - resume_ignore_no_checkpoint (bool): exp_manager errors out if resume_if_exists is True and no checkpoint + could be found. This behaviour can be disabled, in which case exp_manager will print a message and + continue without restoring, by setting resume_ignore_no_checkpoint to True. Defaults to False. + - create_tensorboard_logger (bool): Whether to create a tensorboard logger and attach it to the pytorch + lightning trainer. Defaults to True. + - summary_writer_kwargs (dict): A dictionary of kwargs that can be passed to lightning's TensorboardLogger + class. Note that log_dir is passed by exp_manager and cannot exist in this dict. Defaults to None. + - create_wandb_logger (bool): Whether to create a Weights and Baises logger and attach it to the pytorch + lightning trainer. Defaults to False. + - wandb_logger_kwargs (dict): A dictionary of kwargs that can be passed to lightning's WandBLogger + class. Note that name and project are required parameters if create_wandb_logger is True. + Defaults to None. + - create_checkpoint_callback (bool): Whether to create a ModelCheckpoint callback and attach it to the + pytorch lightning trainer. The ModelCheckpoint saves the top 3 models with the best "val_loss", the most + recent checkpoint under *last.ckpt, and the final checkpoint after training completes under *end.ckpt. + Defaults to True. + - files_to_copy (list): A list of files to copy to the experiment logging directory. Defaults to None which + copies no files. + + returns: + log_dir (Path): The final logging directory where logging files are saved. Usually the concatenation of + exp_dir, name, and version. + """ + # Add rank information to logger + # Note: trainer.global_rank and trainer.is_global_zero are not set until trainer.fit, so have to hack around it + global_rank = trainer.node_rank * trainer.num_gpus + int(os.environ.get("LOCAL_RANK", 0)) + logging.rank = global_rank + + if cfg is None: + logging.error("exp_manager did not receive a cfg argument. It will be disabled.") + return + if trainer.fast_dev_run: + logging.info("Trainer was called with fast_dev_run. exp_manager will return without any functionality.") + return + + # Ensure passed cfg is compliant with ExpManagerConfig + schema = OmegaConf.structured(ExpManagerConfig) + if isinstance(cfg, dict): + cfg = OmegaConf.create(cfg) + elif not isinstance(cfg, DictConfig): + raise ValueError(f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig") + cfg = OmegaConf.create(OmegaConf.to_container(cfg, resolve=True)) + cfg = OmegaConf.merge(schema, cfg) + + error_checks(trainer, cfg) # Ensures that trainer options are compliant with NeMo and exp_manager arguments + + log_dir, exp_dir, name, version = get_log_dir( + trainer=trainer, + exp_dir=cfg.exp_dir, + name=cfg.name, + version=cfg.version, + explicit_log_dir=cfg.explicit_log_dir, + use_datetime_version=cfg.use_datetime_version, + resume_if_exists=cfg.resume_if_exists, + ) + + if cfg.resume_if_exists: + check_resume(trainer, log_dir, cfg.resume_past_end, cfg.resume_ignore_no_checkpoint) + + checkpoint_name = name + # If name returned from get_log_dir is "", use cfg.name for checkpointing + if checkpoint_name is None or checkpoint_name == '': + checkpoint_name = cfg.name or "default" + cfg.name = name # Used for configure_loggers so that the log_dir is properly set even if name is "" + cfg.version = version + + # update app_state with log_dir, exp_dir, etc + app_state = AppState() + app_state.log_dir = log_dir + app_state.exp_dir = exp_dir + app_state.name = name + app_state.version = version + app_state.checkpoint_name = checkpoint_name + app_state.create_checkpoint_callback = cfg.create_checkpoint_callback + app_state.checkpoint_callback_params = cfg.checkpoint_callback_params + + # Create the logging directory if it does not exist + os.makedirs(log_dir, exist_ok=True) # Cannot limit creation to global zero as all ranks write to own log file + logging.info(f'Experiments will be logged at {log_dir}') + trainer._default_root_dir = log_dir + + # Handle Loggers by creating file and handle DEBUG statements + log_file = log_dir / f'nemo_log_globalrank-{global_rank}_localrank-{int(os.environ.get("LOCAL_RANK", 0))}.txt' + logging.add_file_handler(log_file) + + # For some reason, LearningRateLogger requires trainer to have a logger. Safer to create logger on all ranks + # not just global rank 0. + if cfg.create_tensorboard_logger or cfg.create_wandb_logger: + configure_loggers( + trainer, + exp_dir, + cfg.name, + cfg.version, + cfg.create_tensorboard_logger, + cfg.summary_writer_kwargs, + cfg.create_wandb_logger, + cfg.wandb_logger_kwargs, + ) + + if cfg.create_checkpoint_callback: + configure_checkpointing(trainer, log_dir, checkpoint_name, cfg.checkpoint_callback_params) + + if is_global_rank_zero(): + # Move files_to_copy to folder and add git information if present + if cfg.files_to_copy: + for _file in cfg.files_to_copy: + copy(Path(_file), log_dir) + + # Create files for cmd args and git info + with open(log_dir / 'cmd-args.log', 'w') as _file: + _file.write(" ".join(sys.argv)) + + # Try to get git hash + git_repo, git_hash = get_git_hash() + if git_repo: + with open(log_dir / 'git-info.log', 'w') as _file: + _file.write(f'commit hash: {git_hash}') + _file.write(get_git_diff()) + + # Add err_file logging to global_rank zero + logging.add_err_file_handler(log_dir / 'nemo_error_log.txt') + + # Add lightning file logging to global_rank zero + add_filehandlers_to_pl_logger(log_dir / 'lightning_logs.txt', log_dir / 'nemo_error_log.txt') + + return log_dir + + +def error_checks(trainer: 'pytorch_lightning.Trainer', cfg: Optional[Union[DictConfig, Dict]] = None): + """ + Checks that the passed trainer is compliant with NeMo and exp_manager's passed configuration. Checks that: + - Throws error when hydra has changed the working directory. This causes issues with lightning's DDP + - Throws error when trainer has loggers defined but create_tensorboard_logger or create_WandB_logger is True + - Prints error messages when 1) run on multi-node and not Slurm, and 2) run on multi-gpu without DDP + """ + if HydraConfig.initialized() and get_original_cwd() != os.getcwd(): + raise ValueError( + "Hydra changed the working directory. This interferes with ExpManger's functionality. Please pass " + "hydra.run.dir=. to your python script." + ) + if trainer.logger is not None and (cfg.create_tensorboard_logger or cfg.create_wandb_logger): + raise LoggerMisconfigurationError( + "The pytorch lightning trainer that was passed to exp_manager contained a logger, and either " + f"create_tensorboard_logger: {cfg.create_tensorboard_logger} or create_wandb_logger: " + f"{cfg.create_wandb_logger} was set to True. These can only be used if trainer does not already have a" + " logger." + ) + if trainer.num_nodes > 1 and not check_slurm(trainer): + logging.error( + "You are running multi-node training without SLURM handling the processes." + " Please note that this is not tested in NeMo and could result in errors." + ) + if trainer.num_gpus > 1 and not trainer.use_ddp: + logging.error( + "You are running multi-gpu without ddp.Please note that this is not tested in NeMo and could result in " + "errors." + ) + + +def check_resume( + trainer: 'pytorch_lightning.Trainer', + log_dir: str, + resume_past_end: bool = False, + resume_ignore_no_checkpoint: bool = False, +): + """Checks that resume=True was used correctly with the arguments pass to exp_manager. Sets + trainer.resume_from_checkpoint as necessary. + + Returns: + log_dir (Path): the log_dir + exp_dir (str): the base exp_dir without name nor version + name (str): The name of the experiment + version (str): The version of the experiment + + Raises: + NotFoundError: If resume is True, resume_ignore_no_checkpoint is False, and checkpoints could not be found. + ValueError: If resume is True, and there were more than 1 checkpoint could found. + """ + if not log_dir: + raise ValueError(f"Resuming requires the log_dir {log_dir} to be passed to exp_manager") + + checkpoint_dir = Path(Path(log_dir) / "checkpoints") + + checkpoint = None + end_checkpoints = list(checkpoint_dir.rglob("*end.ckpt")) + last_checkpoints = list(checkpoint_dir.rglob("*last.ckpt")) + if not checkpoint_dir.exists(): + if resume_ignore_no_checkpoint: + logging.warning( + f"There was no checkpoint folder at checkpoint_dir :{checkpoint_dir}. Training from scratch." + ) + return + else: + raise NotFoundError(f"There was no checkpoint folder at checkpoint_dir :{checkpoint_dir}. Cannot resume.") + elif len(end_checkpoints) > 0: + if resume_past_end: + if len(end_checkpoints) > 1: + if 'mp_rank' in str(end_checkpoints[0]): + checkpoint = end_checkpoints[0] + else: + raise ValueError(f"Multiple checkpoints {end_checkpoints} that matches *end.ckpt.") + logging.info(f"Resuming from {end_checkpoints[0]}") + else: + raise ValueError( + f"Found {end_checkpoints[0]} indicating that the last training run has already completed." + ) + elif not len(last_checkpoints) > 0: + if resume_ignore_no_checkpoint: + logging.warning(f"There were no checkpoints found in {checkpoint_dir}. Training from scratch.") + return + else: + raise NotFoundError(f"There were no checkpoints found in {checkpoint_dir}. Cannot resume.") + elif len(last_checkpoints) > 1: + if 'mp_rank' in str(last_checkpoints[0]): + checkpoint = last_checkpoints[0] + else: + raise ValueError(f"Multiple checkpoints {last_checkpoints} that matches *last.ckpt.") + else: + logging.info(f"Resuming from {last_checkpoints[0]}") + checkpoint = last_checkpoints[0] + + trainer.resume_from_checkpoint = str(checkpoint) + + if is_global_rank_zero(): + # Check to see if any files exist that need to be moved + files_to_move = [] + for child in Path(log_dir).iterdir(): + if child.is_file(): + files_to_move.append(child) + + if len(files_to_move) > 0: + # Move old files to a new folder + other_run_dirs = Path(log_dir).glob("run_*") + run_count = 0 + for fold in other_run_dirs: + if fold.is_dir(): + run_count += 1 + new_run_dir = Path(Path(log_dir) / f"run_{run_count}") + new_run_dir.mkdir() + for _file in files_to_move: + move(str(_file), str(new_run_dir)) + + +def check_explicit_log_dir( + trainer: 'pytorch_lightning.Trainer', explicit_log_dir: [Path, str], exp_dir: str, name: str, version: str +) -> (Path, str, str, str): + """ Checks that the passed arguments are compatible with explicit_log_dir. + + Returns: + log_dir (Path): the log_dir + exp_dir (str): the base exp_dir without name nor version + name (str): The name of the experiment + version (str): The version of the experiment + + Raise: + LoggerMisconfigurationError + """ + if trainer.logger is not None: + raise LoggerMisconfigurationError( + "The pytorch lightning trainer that was passed to exp_manager contained a logger and explicit_log_dir: " + f"{explicit_log_dir} was pass to exp_manager. Please remove the logger from the lightning trainer." + ) + # Checking only (explicit_log_dir) vs (exp_dir and version). + # The `name` will be used as the actual name of checkpoint/archive. + if exp_dir or version: + logging.error( + f"exp_manager received explicit_log_dir: {explicit_log_dir} and at least one of exp_dir: {exp_dir}, " + f"or version: {version}. Please note that exp_dir, name, and version will be ignored." + ) + if is_global_rank_zero() and Path(explicit_log_dir).exists(): + logging.warning(f"Exp_manager is logging to {explicit_log_dir}, but it already exists.") + return Path(explicit_log_dir), str(explicit_log_dir), "", "" + + +def get_log_dir( + trainer: 'pytorch_lightning.Trainer', + exp_dir: str = None, + name: str = None, + version: str = None, + explicit_log_dir: str = None, + use_datetime_version: bool = True, + resume_if_exists: bool = False, +) -> (Path, str, str, str): + """ + Obtains the log_dir used for exp_manager. + + Returns: + log_dir (Path): the log_dir + exp_dir (str): the base exp_dir without name nor version + name (str): The name of the experiment + version (str): The version of the experiment + explicit_log_dir (str): The explicit path to the log folder. Defaults to False. + use_datetime_version (bool): Uses date and time as the version of the log folder. Defaults to True. + resume_if_exists (bool): if resume_if_exists of the exp_manager's config is enabled or not. When enabled, the + version folders would not get created. + + Raise: + LoggerMisconfigurationError: If trainer is incompatible with arguments + NotFoundError: If resume is True, resume_ignore_no_checkpoint is False, and checkpoints could not be found. + ValueError: If resume is True, and there were more than 1 checkpoint could found. + """ + if explicit_log_dir: # If explicit log_dir was passed, short circuit + return check_explicit_log_dir(trainer, explicit_log_dir, exp_dir, name, version) + + # Default exp_dir to ./nemo_experiments if None was passed + _exp_dir = exp_dir + if exp_dir is None: + _exp_dir = str(Path.cwd() / 'nemo_experiments') + + # If the user has already defined a logger for the trainer, use the logger defaults for logging directory + if trainer.logger is not None: + if trainer.logger.save_dir: + if exp_dir: + raise LoggerMisconfigurationError( + "The pytorch lightning trainer that was passed to exp_manager contained a logger, the logger's " + f"save_dir was not None, and exp_dir ({exp_dir}) was not None. If trainer.logger.save_dir " + "exists, exp_manager will use trainer.logger.save_dir as the logging directory and exp_dir " + "must be None." + ) + _exp_dir = trainer.logger.save_dir + if name: + raise LoggerMisconfigurationError( + "The pytorch lightning trainer that was passed to exp_manager contained a logger, and name: " + f"{name} was also passed to exp_manager. If the trainer contains a " + "logger, exp_manager will use trainer.logger.name, and name passed to exp_manager must be None." + ) + name = trainer.logger.name + version = f"version_{trainer.logger.version}" + # Use user-defined exp_dir, project_name, exp_name, and versioning options + else: + name = name or "default" + version = version or os.environ.get(NEMO_ENV_VARNAME_VERSION, None) + + if not version: + if resume_if_exists: + logging.warning( + "No version folders would be created under the log folder as 'resume_if_exists' is enabled." + ) + version = None + elif is_global_rank_zero(): + if use_datetime_version: + version = time.strftime('%Y-%m-%d_%H-%M-%S') + else: + tensorboard_logger = TensorBoardLogger(save_dir=Path(_exp_dir), name=name, version=version) + version = f"version_{tensorboard_logger.version}" + os.environ[NEMO_ENV_VARNAME_VERSION] = "" if version is None else version + + log_dir = Path(_exp_dir) / Path(str(name)) / Path("" if version is None else str(version)) + return log_dir, str(_exp_dir), name, version + + +def get_git_hash(): + """ + Helper function that tries to get the commit hash if running inside a git folder + + returns: + Bool: Whether the git subprocess ran without error + str: git subprocess output or error message + """ + try: + return ( + True, + subprocess.check_output(['git', 'rev-parse', 'HEAD'], stderr=subprocess.STDOUT).decode(), + ) + except subprocess.CalledProcessError as err: + return False, "{}\n".format(err.output.decode("utf-8")) + + +def get_git_diff(): + """ + Helper function that tries to get the git diff if running inside a git folder + + returns: + Bool: Whether the git subprocess ran without error + str: git subprocess output or error message + """ + try: + return subprocess.check_output(['git', 'diff'], stderr=subprocess.STDOUT).decode() + except subprocess.CalledProcessError as err: + return "{}\n".format(err.output.decode("utf-8")) + + +class LoggerList(_LoggerCollection): + """ A thin wrapper on Lightning's LoggerCollection such that name and version are better aligned with exp_manager + """ + + def __init__(self, _logger_iterable, nemo_name=None, nemo_version=""): + super().__init__(_logger_iterable) + self._nemo_name = nemo_name + self._nemo_version = nemo_version + + @property + def name(self) -> str: + return self._nemo_name + + @property + def version(self) -> str: + return self._nemo_version + + +def configure_loggers( + trainer: 'pytorch_lightning.Trainer', + exp_dir: [Path, str], + name: str, + version: str, + create_tensorboard_logger: bool, + summary_writer_kwargs: dict, + create_wandb_logger: bool, + wandb_kwargs: dict, +): + """ Creates TensorboardLogger and/or WandBLogger and attach them to trainer. Raises ValueError if + summary_writer_kwargs or wandb_kwargs are misconfigured. + """ + # Potentially create tensorboard logger and/or WandBLogger + logger_list = [] + if create_tensorboard_logger: + if summary_writer_kwargs is None: + summary_writer_kwargs = {} + elif "log_dir" in summary_writer_kwargs: + raise ValueError( + "You cannot pass `log_dir` as part of `summary_writer_kwargs`. `log_dir` is handled by lightning's " + "TensorBoardLogger logger." + ) + tensorboard_logger = TensorBoardLogger(save_dir=exp_dir, name=name, version=version, **summary_writer_kwargs) + logger_list.append(tensorboard_logger) + logging.info("TensorboardLogger has been set up") + + if create_wandb_logger: + if wandb_kwargs is None: + wandb_kwargs = {} + if "name" not in wandb_kwargs and "project" not in wandb_kwargs: + raise ValueError("name and project are required for wandb_logger") + wandb_logger = WandbLogger(save_dir=exp_dir, version=version, **wandb_kwargs) + + logger_list.append(wandb_logger) + logging.info("WandBLogger has been set up") + + logger_list = ( + LoggerList(logger_list, nemo_name=name, nemo_version=version) if len(logger_list) > 1 else logger_list[0] + ) + trainer.logger_connector.configure_logger(logger_list) + + +class NeMoModelCheckpoint(ModelCheckpoint): + """ Light wrapper around Lightning's ModelCheckpoint to force a saved checkpoint on train_end + """ + + def __init__(self, always_save_nemo=False, save_best_model=False, postfix=".nemo", **kwargs): + # Parse and store "extended" parameters: save_best model and postfix. + self.always_save_nemo = always_save_nemo + self.save_best_model = save_best_model + self.postfix = postfix + self.previous_best_path = "" + + # `prefix` is deprecated + if 'prefix' in kwargs: + self.prefix = kwargs.pop('prefix') + else: + self.prefix = "" + + # Call the parent class constructor with the remaining kwargs. + super().__init__(**kwargs) + + @rank_zero_only + def on_save_checkpoint(self, trainer, pl_module, checkpoint): + output = super().on_save_checkpoint(trainer, pl_module, checkpoint) + + if not self.always_save_nemo: + return output + + # Load the best model and then re-save it + app_state = AppState() + # since we are creating tarfile artifacts we need to update .nemo path + app_state.model_restore_path = os.path.abspath( + os.path.expanduser(os.path.join(self.dirpath, self.prefix + self.postfix)) + ) + if self.save_best_model: + if not os.path.exists(self.best_model_path): + return output + + if self.best_model_path == self.previous_best_path: + return output + + self.previous_model_path = self.best_model_path + old_state_dict = deepcopy(pl_module.state_dict()) + checkpoint = torch.load(self.best_model_path, map_location='cpu') + if 'state_dict' in checkpoint: + checkpoint = checkpoint['state_dict'] + # get a new instanace of the model + pl_module.load_state_dict(checkpoint, strict=True) + pl_module.save_to(save_path=app_state.model_restore_path) + pl_module.load_state_dict(old_state_dict, strict=True) + else: + pl_module.save_to(save_path=app_state.model_restore_path) + return output + + @rank_zero_only + def on_train_end(self, trainer, pl_module): + if trainer.fast_dev_run: + return None + app_state = AppState() + if app_state.model_parallel_size is not None: + return None + + # TODO: make this work for model parallel, need to call on data parallel rank 0 and update best_model_path + # Load the best model and then re-save it + if self.save_best_model: + trainer.checkpoint_connector.restore(self.best_model_path, on_gpu=trainer.on_gpu) + pl_module.save_to(save_path=os.path.join(self.dirpath, self.prefix + self.postfix)) + + +def configure_checkpointing(trainer: 'pytorch_lightning.Trainer', log_dir: Path, name: str, params: 'DictConfig'): + """ Adds ModelCheckpoint to trainer. Raises CheckpointMisconfigurationError if trainer already has a ModelCheckpoint + callback or if trainer.weights_save_path was passed to Trainer. + """ + for callback in trainer.callbacks: + if isinstance(callback, ModelCheckpoint): + raise CheckpointMisconfigurationError( + "The pytorch lightning trainer that was passed to exp_manager contained a ModelCheckpoint " + "and create_checkpoint_callback was set to True. Please either set create_checkpoint_callback " + "to False, or remove ModelCheckpoint from the lightning trainer" + ) + if Path(trainer.weights_save_path) != Path.cwd(): + raise CheckpointMisconfigurationError( + "The pytorch lightning was passed weights_save_path. This variable is ignored by exp_manager" + ) + + # Create the callback and attach it to trainer + if "filepath" in params: + if params.filepath is not None: + logging.warning("filepath is deprecated. Please switch to dirpath and filename instead") + if params.dirpath is None: + params.dirpath = Path(params.filepath).parent + if params.filename is None: + params.filename = Path(params.filepath).name + with open_dict(params): + del params["filepath"] + if params.dirpath is None: + params.dirpath = Path(log_dir / 'checkpoints') + if params.filename is None: + params.filename = f'{name}--{{{params.monitor}:.2f}}-{{epoch}}' + if params.prefix is None: + params.prefix = name + NeMoModelCheckpoint.CHECKPOINT_NAME_LAST = params.filename + '-last' + + logging.debug(params.dirpath) + logging.debug(params.filename) + logging.debug(params.prefix) + + if "val" in params.monitor: + if ( + trainer.max_epochs is not None + and trainer.max_epochs != -1 + and trainer.max_epochs < trainer.check_val_every_n_epoch + ): + logging.error( + "The checkpoint callback was told to monitor a validation value but trainer.max_epochs(" + f"{trainer.max_epochs}) was less than trainer.check_val_every_n_epoch({trainer.check_val_every_n_epoch}" + f"). It is very likely this run will fail with ModelCheckpoint(monitor='{params.monitor}') not found " + "in the returned metrics. Please ensure that validation is run within trainer.max_epochs." + ) + elif trainer.max_steps is not None: + logging.warning( + "The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to " + f"{trainer.max_steps}. Please ensure that max_steps will run for at least " + f"{trainer.check_val_every_n_epoch} epochs to ensure that checkpointing will not error out." + ) + + checkpoint_callback = NeMoModelCheckpoint(**params) + checkpoint_callback.last_model_path = trainer.resume_from_checkpoint or "" + trainer.callbacks.append(checkpoint_callback) + + +def check_slurm(trainer): + try: + return trainer.accelerator_connector.is_slurm_managing_tasks + except AttributeError: + return False diff --git a/models/tacotron2/NeMo/nemo/utils/export_utils.py b/models/tacotron2/NeMo/nemo/utils/export_utils.py new file mode 100644 index 0000000000..20227988e8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/export_utils.py @@ -0,0 +1,225 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Dict, Optional, Type + +import onnx +import torch +import torch.nn as nn + +from nemo.utils import logging + +apex_available = True + +try: + from apex.normalization.fused_layer_norm import FusedLayerNorm + + def replace_FusedLayerNorm(n: nn.Module) -> Optional[nn.BatchNorm2d]: + """ + Replaces Apex's FusedLayerNorm with nn.LayerNorm. This is required for ONNX export. + Args: + n: the FusedLayerNorm pytorch module to replace + Returns: + Equivalent LayerNorm module + """ + if not apex_available or not isinstance(n, FusedLayerNorm): + return None + + dev = next(n.parameters()).device + mod = nn.LayerNorm(n.normalized_shape, eps=n.eps, elementwise_affine=n.elementwise_affine,).to(dev) + + n_state = n.state_dict() + mod.load_state_dict(n_state) + return mod + + default_Apex_replacements = {"FusedLayerNorm": replace_FusedLayerNorm} + +except Exception as e: + default_Apex_replacements = {} + apex_available = False + + +def expand_Conv1D(conv1d: nn.Module) -> Optional[nn.Conv2d]: + """ + Expands a Conv1D into a Conv2D. This is required for many (closed source) commercial tools with poor support for 1D Convolutions in Onnx. + Args: + conv1d: the Conv1D pytorch module to expand + Returns: + conv2d: Conv2D module with identical weights and params + """ + if not isinstance(conv1d, nn.Conv1d): + return None + conv2d = nn.Conv2d( + conv1d.in_channels, + conv1d.out_channels, + kernel_size=(conv1d.kernel_size[0], 1), + stride=(conv1d.stride[0], 1), + padding=(conv1d.padding[0], 0), + dilation=(conv1d.dilation[0], 1), + groups=conv1d.groups, + padding_mode=conv1d.padding_mode, + ).to(device=conv1d.weight.device, dtype=conv1d.weight.dtype) + conv2d.bias = conv1d.bias + conv2d.weight = nn.Parameter(conv1d.weight.unsqueeze(-1)) + # check that expansion is valid + for _ in range(2): + sample_input = torch.rand(1, conv1d.in_channels, 256).to( + device=conv1d.weight.device, dtype=conv1d.weight.dtype + ) + close = conv1d(sample_input).mean() - conv2d(sample_input.unsqueeze(-1)).squeeze().mean() + if close.abs() > 1.0: + raise ValueError("Unable to expand Conv1D to Conv2D") + return conv2d + + +def expand_BatchNorm1d(bn1d: nn.Module) -> Optional[nn.BatchNorm2d]: + """ + Expands a BatchNorm1d into a BatchNorm2d. This is required for many (closed source) commercial tools with poor support for BatchNorm1d in Onnx. + Args: + bn1d: the BatchNorm1d pytorch module to expand + Returns: + bn2d: BatchNorm2d module with identical weights and params + """ + if not isinstance(bn1d, nn.BatchNorm1d): + return None + mod = torch.nn.BatchNorm2d( + bn1d.num_features, + eps=bn1d.eps, + momentum=bn1d.momentum, + affine=bn1d.affine, + track_running_stats=bn1d.track_running_stats, + ).to(device=conv1d.weight.device, dtype=conv1d.weight.dtype) + bn_state = bn1d.state_dict() + mod.load_state_dict(bn_state) + return mod + + +def expand_ConvTranspose1D(conv1d: nn.Module) -> Optional[nn.ConvTranspose2d]: + """ + Expands a Conv1D into a Conv2D. This is required for many (closed source) commercial tools with poor support for 1D Convolutions in Onnx. + Args: + conv1d: the Conv1D pytorch module to expand + Returns: + conv2d: Conv2D module with identical weights and params + """ + if not isinstance(conv1d, nn.ConvTranspose1d): + return None + conv2d = nn.ConvTranspose2d( + conv1d.in_channels, + conv1d.out_channels, + kernel_size=(conv1d.kernel_size[0], 1), + stride=(conv1d.stride[0], 1), + padding=(conv1d.padding[0], 0), + dilation=(conv1d.dilation[0], 1), + groups=conv1d.groups, + padding_mode=conv1d.padding_mode, + ).to(device=conv1d.weight.device, dtype=conv1d.weight.dtype) + conv2d.bias = conv1d.bias + conv2d.weight = nn.Parameter(conv1d.weight.unsqueeze(-1)) + # check that expansion is valid + for _ in range(2): + sample_input = torch.rand(1, conv1d.in_channels, 256).to( + device=conv1d.weight.device, dtype=conv1d.weight.dtype + ) + close = conv1d(sample_input).mean() - conv2d(sample_input.unsqueeze(-1)).squeeze().mean() + if close.abs() > 1.0: + raise ValueError("Unable to expand Conv1D to Conv2D") + return conv2d + + +def simple_replace(BaseT: Type[nn.Module], DestT: Type[nn.Module]) -> Callable[[nn.Module], Optional[nn.Module]]: + """ + Generic function generator to replace BaseT module with DestT. BaseT and DestT should have same atrributes. No weights are copied. + Args: + BaseT : module type to replace + DestT : destination module type + Returns: + swap function to replace BaseT module with DestT + """ + + def expansion_fn(mod: nn.Module) -> Optional[nn.Module]: + if not isinstance(mod, BaseT): + return None + args = [getattr(mod, name, None) for name in mod.__constants__] + out = DestT(*args) + return out + + return expansion_fn + + +def swap_modules(model: nn.Module, mapping: Dict[str, nn.Module]): + """ + This function swaps nested modules as specified by "dot paths" in mod with a desired replacement. This allows + for swapping nested modules through arbitrary levels if children + + NOTE: This occurs in place, if you want to preserve model then make sure to copy it first. + + """ + for path, new_mod in mapping.items(): + expanded_path = path.split(".") + parent_mod = model + for sub_path in expanded_path[:-1]: + parent_mod = parent_mod._modules[sub_path] # noqa + parent_mod._modules[expanded_path[-1]] = new_mod # noqa + + return model + + +def replace_modules( + model: nn.Module, expansions: Dict[str, Callable[[nn.Module], Optional[nn.Module]]] = None +) -> nn.Module: + """ + Top-level function to replace modules in model, specified by class name with a desired replacement. + NOTE: This occurs in place, if you want to preserve model then make sure to copy it first. + Args: + model : top level module + expansions : replacement dictionary: module class name -> replacement function generator + Returns: + model, possibly modified in-place + """ + mapping: Dict[str, nn.Module] = {} + for name, m in model.named_modules(): + m_type = type(m).__name__ + if m_type in expansions: + swapped = expansions[m_type](m) + if swapped: + mapping[name] = swapped + logging.warning(f"Swapped {len(mapping)} modules") + swap_modules(model, mapping) + return model + + +default_1D_2D_replacements = { + "Conv1d": expand_Conv1D, + "ConvTranspose1d": expand_ConvTranspose1D, + "BatchNorm1d": expand_BatchNorm1d, + "AdaptiveAvgPool1d": simple_replace(nn.AdaptiveAvgPool1d, nn.AdaptiveAvgPool2d), + "AvgPool1d": simple_replace(nn.AvgPool1d, nn.AvgPool2d), +} + + +def replace_for_export(model: nn.Module, replace_1D_2D: bool = False) -> nn.Module: + """ + Top-level function to replace default set of modules in model + NOTE: This occurs in place, if you want to preserve model then make sure to copy it first. + Args: + model : top level module + replace_1D_2D : include 1D -> 2D replacements + Returns: + model, possibly modified in-place + """ + replace_modules(model, default_Apex_replacements) + if replace_1D_2D: + # TODO: add squeeze/unsqueeze + replace_modules(model, default_1D_2D_replacements) diff --git a/models/tacotron2/NeMo/nemo/utils/formatters/__init__.py b/models/tacotron2/NeMo/nemo/utils/formatters/__init__.py new file mode 100644 index 0000000000..9e32500719 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/formatters/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo/utils/formatters/base.py b/models/tacotron2/NeMo/nemo/utils/formatters/base.py new file mode 100644 index 0000000000..d4b6fef881 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/formatters/base.py @@ -0,0 +1,135 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging + +from nemo.utils.formatters.colors import Fore as ForegroundColors +from nemo.utils.formatters.utils import check_color_support, to_unicode + +__all__ = ["BaseNeMoFormatter"] + + +class BaseFormatter(logging.Formatter): + """ + Log formatter used in Tornado. Key features of this formatter are: + * Color support when logging to a terminal that supports it. + * Timestamps on every log line. + * Robust against str/bytes encoding problems. + """ + + DEFAULT_FORMAT = "%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s" + + DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" + + DEFAULT_COLORS = { + logging.DEBUG: ForegroundColors.CYAN, + logging.INFO: ForegroundColors.GREEN, + logging.WARNING: ForegroundColors.YELLOW, + logging.ERROR: ForegroundColors.MAGENTA, + logging.CRITICAL: ForegroundColors.RED, + } + + def __init__(self, color=True, fmt=None, datefmt=None, colors=None): + r""" + :arg bool color: Enables color support. + :arg string fmt: Log message format. + It will be applied to the attributes dict of log records. The + text between ``%(color)s`` and ``%(end_color)s`` will be colored + depending on the level if color support is on. + :arg dict colors: color mappings from logging level to terminal color + code + :arg string datefmt: Datetime format. + Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. + .. versionchanged:: 3.2 + Added ``fmt`` and ``datefmt`` arguments. + """ + + if fmt is None: + fmt = self.DEFAULT_FORMAT + + if datefmt is None: + datefmt = self.DEFAULT_DATE_FORMAT + + if colors is None: + colors = self.DEFAULT_COLORS + + logging.Formatter.__init__(self, datefmt=datefmt) + + self._fmt = fmt + self._colors = {} + self._normal = "" + + if color and check_color_support(): + self._colors = colors + self._normal = ForegroundColors.RESET + + def format(self, record): + try: + message = record.getMessage() + assert isinstance(message, str) # guaranteed by logging + # Encoding notes: The logging module prefers to work with character + # strings, but only enforces that log messages are instances of + # basestring. In python 2, non-ascii bytestrings will make + # their way through the logging framework until they blow up with + # an unhelpful decoding error (with this formatter it happens + # when we attach the prefix, but there are other opportunities for + # exceptions further along in the framework). + # + # If a byte string makes it this far, convert it to unicode to + # ensure it will make it out to the logs. Use repr() as a fallback + # to ensure that all byte strings can be converted successfully, + # but don't do it by default so we don't add extra quotes to ascii + # bytestrings. This is a bit of a hacky place to do this, but + # it's worth it since the encoding errors that would otherwise + # result are so useless (and tornado is fond of using utf8-encoded + # byte strings wherever possible). + record.message = to_unicode(message) + + except Exception as e: + record.message = "Bad message (%r): %r" % (e, record.__dict__) + + record.asctime = self.formatTime(record, self.datefmt) + + if record.levelno in self._colors: + record.color = self._colors[record.levelno] + record.end_color = self._normal + else: + record.color = record.end_color = "" + + formatted = self._fmt % record.__dict__ + + if record.exc_info: + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + + if record.exc_text: + # exc_text contains multiple lines. We need to _safe_unicode + # each line separately so that non-utf8 bytes don't cause + # all the newlines to turn into '\n'. + lines = [formatted.rstrip()] + lines.extend(to_unicode(ln) for ln in record.exc_text.split("\n")) + + formatted = "\n".join(lines) + return formatted.replace("\n", "\n ") + + +class BaseNeMoFormatter(BaseFormatter): + DEFAULT_FORMAT = "%(color)s[NeMo %(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s" + + +class DebugNeMoFormatter(BaseFormatter): + DEFAULT_FORMAT = ( + "%(color)s[NeMo %(levelname)1.1s %(asctime)s %(module)s:%(lineno)d rank:%(rank)s]%(end_color)s %(message)s" + ) diff --git a/models/tacotron2/NeMo/nemo/utils/formatters/colors.py b/models/tacotron2/NeMo/nemo/utils/formatters/colors.py new file mode 100644 index 0000000000..f8eb616b33 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/formatters/colors.py @@ -0,0 +1,121 @@ +# Source: https://github.com/tartley/colorama/blob/master/colorama/ansi.py +# Copyright: Jonathan Hartley 2013. BSD 3-Clause license. +# +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CSI = "\033[" +OSC = "\033]" +BEL = "\007" + + +def code_to_chars(code): + return CSI + str(code) + "m" + + +def set_title(title): + return OSC + "2;" + title + BEL + + +def clear_screen(mode=2): + return CSI + str(mode) + "J" + + +def clear_line(mode=2): + return CSI + str(mode) + "K" + + +class AnsiCodes(object): + def __init__(self): + # the subclasses declare class attributes which are numbers. + # Upon instantiation we define instance attributes, which are the same + # as the class attributes but wrapped with the ANSI escape sequence + for name in dir(self): + if not name.startswith("_"): + value = getattr(self, name) + setattr(self, name, code_to_chars(value)) + + +class AnsiCursor(object): + def UP(self, n=1): + return CSI + str(n) + "A" + + def DOWN(self, n=1): + return CSI + str(n) + "B" + + def FORWARD(self, n=1): + return CSI + str(n) + "C" + + def BACK(self, n=1): + return CSI + str(n) + "D" + + def POS(self, x=1, y=1): + return CSI + str(y) + ";" + str(x) + "H" + + +class AnsiFore(AnsiCodes): + BLACK = 30 + RED = 31 + GREEN = 32 + YELLOW = 33 + BLUE = 34 + MAGENTA = 35 + CYAN = 36 + WHITE = 37 + RESET = 39 + + # These are fairly well supported, but not part of the standard. + LIGHTBLACK_EX = 90 + LIGHTRED_EX = 91 + LIGHTGREEN_EX = 92 + LIGHTYELLOW_EX = 93 + LIGHTBLUE_EX = 94 + LIGHTMAGENTA_EX = 95 + LIGHTCYAN_EX = 96 + LIGHTWHITE_EX = 97 + + +class AnsiBack(AnsiCodes): + BLACK = 40 + RED = 41 + GREEN = 42 + YELLOW = 43 + BLUE = 44 + MAGENTA = 45 + CYAN = 46 + WHITE = 47 + RESET = 49 + + # These are fairly well supported, but not part of the standard. + LIGHTBLACK_EX = 100 + LIGHTRED_EX = 101 + LIGHTGREEN_EX = 102 + LIGHTYELLOW_EX = 103 + LIGHTBLUE_EX = 104 + LIGHTMAGENTA_EX = 105 + LIGHTCYAN_EX = 106 + LIGHTWHITE_EX = 107 + + +class AnsiStyle(AnsiCodes): + BRIGHT = 1 + DIM = 2 + NORMAL = 22 + RESET_ALL = 0 + + +Fore = AnsiFore() +Back = AnsiBack() +Style = AnsiStyle() +Cursor = AnsiCursor() diff --git a/models/tacotron2/NeMo/nemo/utils/formatters/utils.py b/models/tacotron2/NeMo/nemo/utils/formatters/utils.py new file mode 100644 index 0000000000..570f75a740 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/formatters/utils.py @@ -0,0 +1,46 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys + +from nemo.constants import NEMO_ENV_VARNAME_ENABLE_COLORING +from nemo.utils.env_var_parsing import get_envbool + +__all__ = ["check_color_support", "to_unicode"] + + +def check_color_support(): + # Colors can be forced with an env variable + if not sys.platform.lower().startswith("win") and get_envbool(NEMO_ENV_VARNAME_ENABLE_COLORING, False): + return True + + +def to_unicode(value): + """ + Converts a string argument to a unicode string. + If the argument is already a unicode string or None, it is returned + unchanged. Otherwise it must be a byte string and is decoded as utf8. + """ + try: + if isinstance(value, (str, type(None))): + return value + + if not isinstance(value, bytes): + raise TypeError("Expected bytes, unicode, or None; got %r" % type(value)) + + return value.decode("utf-8") + + except UnicodeDecodeError: + return repr(value) diff --git a/models/tacotron2/NeMo/nemo/utils/get_rank.py b/models/tacotron2/NeMo/nemo/utils/get_rank.py new file mode 100644 index 0000000000..8d3fd0e5c7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/get_rank.py @@ -0,0 +1,48 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.utils.env_var_parsing import get_envint + + +def is_global_rank_zero(): + """ Helper function to determine if the current process is global_rank 0 (the main process) + """ + # Try to get the pytorch RANK env var + # RANK is set by torch.distributed.launch + rank = get_envint("RANK", None) + if rank: + return rank == 0 + + # If not set by pytorch, we need to determine node_rank + def get_node_rank(): + # Use an equivalent of pytorch lightning's determine_ddp_node_rank() + node_rank = 0 + # First check if running on a slurm cluster + # TODO: This check could probably be better + num_slurm_tasks = get_envint("SLURM_NTASKS", 0) + if num_slurm_tasks > 0: + node_rank = get_envint("SLURM_NODEID", 0) + else: + node_rank_env = get_envint("NODE_RANK", None) + group_rank = get_envint("GROUP_RANK", None) + if group_rank: + node_rank = group_rank + # Take from NODE_RANK whenever available + if node_rank_env: + node_rank = node_rank_env + return node_rank + + node_rank = get_node_rank() + local_rank = get_envint("LOCAL_RANK", 0) + return node_rank == 0 and local_rank == 0 diff --git a/models/tacotron2/NeMo/nemo/utils/lightning_logger_patch.py b/models/tacotron2/NeMo/nemo/utils/lightning_logger_patch.py new file mode 100644 index 0000000000..1b21ce3b1a --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/lightning_logger_patch.py @@ -0,0 +1,58 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging as _logging +from logging.handlers import MemoryHandler + +import pytorch_lightning as pl + +HANDLERS = {} +PATCHED = False + + +def add_memory_handlers_to_pl_logger(): + """ + Adds two MemoryHandlers to pytorch_lightning's logger. These two handlers are essentially message buffers. This + function is called in nemo.utils.__init__.py. These handlers are used in add_filehandlers_to_pl_logger to flush + buffered messages to files. + """ + if not HANDLERS: + HANDLERS["memory_err"] = MemoryHandler(-1) + HANDLERS["memory_err"].addFilter(lambda record: record.levelno > _logging.INFO) + HANDLERS["memory_all"] = MemoryHandler(-1) + pl._logger.addHandler(HANDLERS["memory_err"]) + pl._logger.addHandler(HANDLERS["memory_all"]) + + +def add_filehandlers_to_pl_logger(all_log_file, err_log_file): + """ + Adds two filehandlers to pytorch_lightning's logger. Called in nemo.utils.exp_manager(). The first filehandler + logs all messages to all_log_file while the second filehandler logs all WARNING and higher messages to err_log_file. + If "memory_err" and "memory_all" exist in HANDLERS, then those buffers are flushed to err_log_file and all_log_file + respectively, and then closed. + """ + HANDLERS["file"] = _logging.FileHandler(all_log_file) + pl._logger.addHandler(HANDLERS["file"]) + HANDLERS["file_err"] = _logging.FileHandler(err_log_file) + HANDLERS["file_err"].addFilter(lambda record: record.levelno > _logging.INFO) + pl._logger.addHandler(HANDLERS["file_err"]) + + if HANDLERS.get("memory_all", None): + HANDLERS["memory_all"].setTarget(HANDLERS["file"]) + HANDLERS["memory_all"].close() + del HANDLERS["memory_all"] + if HANDLERS.get("memory_err", None): + HANDLERS["memory_err"].setTarget(HANDLERS["file_err"]) + HANDLERS["memory_err"].close() + del HANDLERS["memory_err"] diff --git a/models/tacotron2/NeMo/nemo/utils/metaclasses.py b/models/tacotron2/NeMo/nemo/utils/metaclasses.py new file mode 100644 index 0000000000..5fad7cb150 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/metaclasses.py @@ -0,0 +1,39 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import threading + + +class Singleton(type): + """ Implementation of a generic, tread-safe singleton meta-class. + Can be used as meta-class, i.e. will create + """ + + # List of instances - one per class. + __instances = {} + # Lock used for accessing the instance. + __lock = threading.Lock() + + def __call__(cls, *args, **kwargs): + """ Returns singleton instance. A thread safe implementation. """ + if cls not in cls.__instances: + # Enter critical section. + with cls.__lock: + # Check once again. + if cls not in cls.__instances: + # Create a new object instance - one per class. + cls.__instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + # Return the instance. + return cls.__instances[cls] diff --git a/models/tacotron2/NeMo/nemo/utils/model_utils.py b/models/tacotron2/NeMo/nemo/utils/model_utils.py new file mode 100644 index 0000000000..c226cbcdb8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/model_utils.py @@ -0,0 +1,528 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import os +from dataclasses import dataclass, is_dataclass +from enum import Enum +from pathlib import Path +from typing import List, Optional, Union + +import pytorch_lightning as pl +import wrapt +from omegaconf import DictConfig, ListConfig, OmegaConf +from omegaconf import errors as omegaconf_errors +from packaging import version + +from nemo.utils import logging + +_VAL_TEST_FASTPATH_KEY = 'ds_item' + + +class ArtifactPathType(Enum): + """ + ArtifactPathType refers to the type of the path that the artifact is located at. + + LOCAL_PATH: A user local filepath that exists on the file system. + TAR_PATH: A (generally flattened) filepath that exists inside of an archive (that may have its own full path). + """ + + LOCAL_PATH = 0 + TAR_PATH = 1 + + +@dataclass(init=False) +class ArtifactItem: + path: str + path_type: ArtifactPathType + hashed_path: Optional[str] = None + + +def resolve_dataset_name_from_cfg(cfg: DictConfig) -> str: + """ + Parses items of the provided sub-config to find the first potential key that + resolves to an existing file or directory. + + # Fast-path Resolution + In order to handle cases where we need to resolve items that are not paths, a fastpath + key can be provided as defined in the global `_VAL_TEST_FASTPATH_KEY`. + + This key can be used in two ways : + + ## _VAL_TEST_FASTPATH_KEY points to another key in the config + + If this _VAL_TEST_FASTPATH_KEY points to another key in this config itself, + then we assume we want to loop through the values of that key. + + This allows for any key in the config to become a fastpath key. + + Example: + validation_ds: + splits: "val" + ... + <_VAL_TEST_FASTPATH_KEY>: "splits" <-- this points to the key name "splits" + + Then we can write the following when overriding in hydra: + ```python + python train_file.py ... \ + model.validation_ds.splits=[val1, val2, dev1, dev2] ... + ``` + + ## _VAL_TEST_FASTPATH_KEY itself acts as the resolved key + + If this _VAL_TEST_FASTPATH_KEY does not point to another key in the config, then + it is assumed that the items of this key itself are used for resolution. + + Example: + validation_ds: + ... + <_VAL_TEST_FASTPATH_KEY>: "val" <-- this points to the key name "splits" + + Then we can write the following when overriding in hydra: + ```python + python train_file.py ... \ + model.validation_ds.<_VAL_TEST_FASTPATH_KEY>=[val1, val2, dev1, dev2] ... + ``` + + # IMPORTANT NOTE: + It potentially mismatch if there exist more than 2 valid paths, and the + first path does *not* resolve the the path of the data file (but does resolve to + some other valid path). + + To avoid this side-effect, place the data path as the first item on the config file. + + Args: + cfg: DictConfig (Sub-config) that should be parsed. + + Returns: + A str representing the `key` of the config which hosts the filepath(s), + or None in case path could not be resolved. + """ + if _VAL_TEST_FASTPATH_KEY in cfg and cfg[_VAL_TEST_FASTPATH_KEY] is not None: + fastpath_key = cfg[_VAL_TEST_FASTPATH_KEY] + + if isinstance(fastpath_key, str) and fastpath_key in cfg: + return cfg[fastpath_key] + else: + return _VAL_TEST_FASTPATH_KEY + + for key, value in cfg.items(): + if type(value) in [list, tuple, ListConfig]: + # Count the number of valid paths in the list + values_are_paths = 0 + for val_i in value: + val_i = str(val_i) + + if os.path.exists(val_i) or os.path.isdir(val_i): + values_are_paths += 1 + else: + # reset counter and break inner loop + break + + if values_are_paths == len(value): + return key + + else: + if os.path.exists(str(value)) or os.path.isdir(str(value)): + return key + + return None + + +def parse_dataset_as_name(name: str) -> str: + """ + Constructs a valid prefix-name from a provided file path. + + Args: + name: str path to some valid data/manifest file or a python object that + will be used as a name for the data loader (via str() cast). + + Returns: + str prefix used to identify uniquely this data/manifest file. + """ + if os.path.exists(str(name)) or os.path.isdir(str(name)): + name = Path(name).stem + else: + name = str(name) + + # cleanup name + name = name.replace('-', '_') + + if 'manifest' in name: + name = name.replace('manifest', '') + + if 'dataset' in name: + name = name.replace('dataset', '') + + if '_' != name[-1]: + name = name + '_' + + return name + + +def unique_names_check(name_list: Optional[List[str]]): + """ + Performs a uniqueness check on the name list resolved, so that it can warn users + about non-unique keys. + + Args: + name_list: List of strings resolved for data loaders. + """ + if name_list is None: + return + + # Name uniqueness checks + names = set() + for name in name_list: + if name in names: + logging.warning( + "Name resolution has found more than one data loader having the same name !\n" + "In such cases, logs will nor be properly generated. " + "Please rename the item to have unique names.\n" + f"Resolved name : {name}" + ) + else: + names.add(name) # we need just hash key check, value is just a placeholder + + +def resolve_validation_dataloaders(model: 'ModelPT'): + """ + Helper method that operates on the ModelPT class to automatically support + multiple dataloaders for the validation set. + + It does so by first resolving the path to one/more data files via `resolve_dataset_name_from_cfg()`. + If this resolution fails, it assumes the data loader is prepared to manually support / not support + multiple data loaders and simply calls the appropriate setup method. + + If resolution succeeds: + Checks if provided path is to a single file or a list of files. + If a single file is provided, simply tags that file as such and loads it via the setup method. + If multiple files are provided: + Inject a new manifest path at index "i" into the resolved key. + Calls the appropriate setup method to set the data loader. + Collects the initialized data loader in a list and preserves it. + Once all data loaders are processed, assigns the list of loaded loaders to the ModelPT. + Finally assigns a list of unique names resolved from the file paths to the ModelPT. + + Args: + model: ModelPT subclass, which requires >=1 Validation Dataloaders to be setup. + """ + cfg = copy.deepcopy(model._cfg) + dataloaders = [] + + # process val_loss_idx + if 'val_dl_idx' in cfg.validation_ds: + cfg = OmegaConf.to_container(cfg) + val_dl_idx = cfg['validation_ds'].pop('val_dl_idx') + cfg = OmegaConf.create(cfg) + else: + val_dl_idx = 0 + + # Set val_loss_idx + model._val_dl_idx = val_dl_idx + + ds_key = resolve_dataset_name_from_cfg(cfg.validation_ds) + + if ds_key is None: + logging.debug( + "Could not resolve file path from provided config - {}. " + "Disabling support for multi-dataloaders.".format(cfg.validation_ds) + ) + + model.setup_validation_data(cfg.validation_ds) + return + + ds_values = cfg.validation_ds[ds_key] + + if isinstance(ds_values, (list, tuple, ListConfig)): + + for ds_value in ds_values: + cfg.validation_ds[ds_key] = ds_value + model.setup_validation_data(cfg.validation_ds) + dataloaders.append(model._validation_dl) + + model._validation_dl = dataloaders + model._validation_names = [parse_dataset_as_name(ds) for ds in ds_values] + + unique_names_check(name_list=model._validation_names) + return + + else: + model.setup_validation_data(cfg.validation_ds) + model._validation_names = [parse_dataset_as_name(ds_values)] + + unique_names_check(name_list=model._validation_names) + + +def resolve_test_dataloaders(model: 'ModelPT'): + """ + Helper method that operates on the ModelPT class to automatically support + multiple dataloaders for the test set. + + It does so by first resolving the path to one/more data files via `resolve_dataset_name_from_cfg()`. + If this resolution fails, it assumes the data loader is prepared to manually support / not support + multiple data loaders and simply calls the appropriate setup method. + + If resolution succeeds: + Checks if provided path is to a single file or a list of files. + If a single file is provided, simply tags that file as such and loads it via the setup method. + If multiple files are provided: + Inject a new manifest path at index "i" into the resolved key. + Calls the appropriate setup method to set the data loader. + Collects the initialized data loader in a list and preserves it. + Once all data loaders are processed, assigns the list of loaded loaders to the ModelPT. + Finally assigns a list of unique names resolved from the file paths to the ModelPT. + + Args: + model: ModelPT subclass, which requires >=1 Test Dataloaders to be setup. + """ + cfg = copy.deepcopy(model._cfg) + dataloaders = [] + + # process test_loss_idx + if 'test_dl_idx' in cfg.test_ds: + cfg = OmegaConf.to_container(cfg) + test_dl_idx = cfg['test_ds'].pop('test_dl_idx') + cfg = OmegaConf.create(cfg) + else: + test_dl_idx = 0 + + # Set val_loss_idx + model._test_dl_idx = test_dl_idx + + ds_key = resolve_dataset_name_from_cfg(cfg.test_ds) + + if ds_key is None: + logging.debug( + "Could not resolve file path from provided config - {}. " + "Disabling support for multi-dataloaders.".format(cfg.test_ds) + ) + + model.setup_test_data(cfg.test_ds) + return + + ds_values = cfg.test_ds[ds_key] + + if isinstance(ds_values, (list, tuple, ListConfig)): + + for ds_value in ds_values: + cfg.test_ds[ds_key] = ds_value + model.setup_test_data(cfg.test_ds) + dataloaders.append(model._test_dl) + + model._test_dl = dataloaders + model._test_names = [parse_dataset_as_name(ds) for ds in ds_values] + + unique_names_check(name_list=model._test_names) + return + + else: + model.setup_test_data(cfg.test_ds) + model._test_names = [parse_dataset_as_name(ds_values)] + + unique_names_check(name_list=model._test_names) + + +@wrapt.decorator +def wrap_training_step(wrapped, instance: pl.LightningModule, args, kwargs): + output_dict = wrapped(*args, **kwargs) + + if isinstance(output_dict, dict) and output_dict is not None and 'log' in output_dict: + log_dict = output_dict.pop('log') + instance.log_dict(log_dict, on_step=True) + + return output_dict + + +def convert_model_config_to_dict_config(cfg: Union[DictConfig, 'NemoConfig']) -> DictConfig: + """ + Converts its input into a standard DictConfig. + Possible input values are: + - DictConfig + - A dataclass which is a subclass of NemoConfig + + Args: + cfg: A dict-like object. + + Returns: + The equivalent DictConfig + """ + if not isinstance(cfg, (OmegaConf, DictConfig)) and is_dataclass(cfg): + cfg = OmegaConf.structured(cfg) + + if not isinstance(cfg, DictConfig): + raise ValueError(f"cfg constructor argument must be of type DictConfig/dict but got {type(cfg)} instead.") + + config = OmegaConf.to_container(cfg, resolve=True) + config = OmegaConf.create(config) + return config + + +def _convert_config(cfg: OmegaConf): + """ Recursive function convertint the configuration from old hydra format to the new one. """ + + # Get rid of cls -> _target_. + if 'cls' in cfg and '_target_' not in cfg: + cfg._target_ = cfg.pop('cls') + + # Get rid of params. + if 'params' in cfg: + params = cfg.pop('params') + for param_key, param_val in params.items(): + cfg[param_key] = param_val + + # Recursion. + try: + for _, sub_cfg in cfg.items(): + if isinstance(sub_cfg, DictConfig): + _convert_config(sub_cfg) + except omegaconf_errors.OmegaConfBaseException as e: + logging.warning(f"Skipped conversion for config/subconfig:\n{cfg}\n Reason: {e}.") + + +def maybe_update_config_version(cfg: DictConfig): + """ + Recursively convert Hydra 0.x configs to Hydra 1.x configs. + + Changes include: + - `cls` -> `_target_`. + - `params` -> drop params and shift all arguments to parent. + - `target` -> `_target_` cannot be performed due to ModelPT injecting `target` inside class. + + Args: + cfg: Any Hydra compatible DictConfig + + Returns: + An updated DictConfig that conforms to Hydra 1.x format. + """ + if cfg is not None and not isinstance(cfg, DictConfig): + try: + temp_cfg = OmegaConf.create(cfg) + cfg = temp_cfg + except omegaconf_errors.OmegaConfBaseException: + # Cannot be cast to DictConfig, skip updating. + return cfg + + # Make a copy of model config. + cfg = copy.deepcopy(cfg) + OmegaConf.set_struct(cfg, False) + + # Convert config. + _convert_config(cfg) + + # Update model config. + OmegaConf.set_struct(cfg, True) + + return cfg + + +def import_class_by_path(path: str): + """ + Recursive import of class by path string. + """ + paths = path.split('.') + path = ".".join(paths[:-1]) + class_name = paths[-1] + mod = __import__(path, fromlist=[class_name]) + mod = getattr(mod, class_name) + return mod + + +def resolve_subclass_pretrained_model_info(base_class) -> List['PretrainedModelInfo']: + """ + Recursively traverses the inheritance graph of subclasses to extract all pretrained model info. + First constructs a set of unique pretrained model info by performing DFS over the inheritance graph. + All model info belonging to the same class is added together. + + Args: + base_class: The root class, whose subclass graph will be traversed. + + Returns: + A list of unique pretrained model infos belonging to all of the inherited subclasses of + this baseclass. + """ + list_of_models = set() + + def recursive_subclass_walk(cls): + for subclass in cls.__subclasses__(): + # step into its immediate subclass + recursive_subclass_walk(subclass) + + subclass_models = subclass.list_available_models() + + if subclass_models is not None and len(subclass_models) > 0: + # Inject subclass info into pretrained model info + # if not already overriden by subclass + for model_info in subclass_models: + # If subclass manually injects class_, dont override. + if model_info.class_ is None: + model_info.class_ = subclass + + for model_info in subclass_models: + list_of_models.add(model_info) + + recursive_subclass_walk(base_class) + + list_of_models = list(sorted(list_of_models)) + return list_of_models + + +def check_lib_version(lib_name: str, checked_version: str, operator) -> (Optional[bool], str): + """ + Checks if a library is installed, and if it is, checks the operator(lib.__version__, checked_version) as a result. + This bool result along with a string analysis of result is returned. + + If the library is not installed at all, then returns None instead, along with a string explaining + that the library is not installed + + Args: + lib_name: lower case str name of the library that must be imported. + checked_version: semver string that is compared against lib.__version__. + operator: binary callable function func(a, b) -> bool; that compares lib.__version__ against version in + some manner. Must return a boolean. + + Returns: + A tuple of results: + - Bool or None. Bool if the library could be imported, and the result of + operator(lib.__version__, checked_version) or False if __version__ is not implemented in lib. + None is passed if the library is not installed at all. + - A string analysis of the check. + """ + try: + mod = __import__(lib_name) + + if hasattr(mod, '__version__'): + lib_ver = version.Version(mod.__version__) + match_ver = version.Version(checked_version) + + if operator(lib_ver, match_ver): + msg = f"Lib {lib_name} version is satisfied !" + return True, msg + else: + msg = ( + f"Lib {lib_name} version ({lib_ver}) is not {operator.__name__} than required version {checked_version}.\n" + f"Please upgrade the lib using either pip or conda to the latest version." + ) + return False, msg + else: + msg = ( + f"Lib {lib_name} does not implement __version__ in its init file. " + f"Could not check version compatibility." + ) + return False, msg + except (ImportError, ModuleNotFoundError): + pass + + msg = f"Lib {lib_name} has not been installed. Please use pip or conda to install this package." + return None, msg diff --git a/models/tacotron2/NeMo/nemo/utils/nemo_logging.py b/models/tacotron2/NeMo/nemo/utils/nemo_logging.py new file mode 100644 index 0000000000..95e17e5c5f --- /dev/null +++ b/models/tacotron2/NeMo/nemo/utils/nemo_logging.py @@ -0,0 +1,421 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import enum +import logging as _logging +import sys +import threading +import warnings +from contextlib import contextmanager +from logging.handlers import MemoryHandler + +from nemo.constants import NEMO_ENV_VARNAME_REDIRECT_LOGS_TO_STDERR, NEMO_ENV_VARNAME_TESTING +from nemo.utils.env_var_parsing import get_envbool +from nemo.utils.formatters.base import BaseNeMoFormatter, DebugNeMoFormatter +from nemo.utils.get_rank import is_global_rank_zero +from nemo.utils.metaclasses import Singleton + +__all__ = ["Logger", "LogMode"] + + +class LogMode(enum.IntEnum): + EACH = 0 # Log the message each time + ONCE = 1 # Log the message only once. The same message will not be logged again. + + +class Logger(metaclass=Singleton): + + # Level 0 + NOTSET = _logging.NOTSET + + # Level 10 + DEBUG = _logging.DEBUG + + # Level 20 + INFO = _logging.INFO + + # Level 30 + WARNING = _logging.WARNING + + # Level 40 + ERROR = _logging.ERROR + + # Level 50 + CRITICAL = _logging.CRITICAL + + _level_names = { + 0: "NOTSET", + 10: "DEBUG", + 20: "INFO", + 30: "WARNING", + 40: "ERROR", + 50: "CRITICAL", + } + + def __init__(self, capture_warnings=True): + + self._logger = None + # Multi-GPU runs run in separate processes, thread locks shouldn't be needed + self._logger_lock = threading.Lock() + self._handlers = dict() + self.old_warnings_showwarning = None + self._define_logger(capture_warnings) + self.once_logged = set() + self.rank = 0 if is_global_rank_zero() else "UNK" + + def _define_logger(self, capture_warnings=True): + """ Creates the logger if not already created. Called in init""" + + # Use double-checked locking to avoid taking lock unnecessarily. + if self._logger is not None: + return self._logger + + with self._logger_lock: + try: + self._logger = _logging.getLogger("nemo_logger") + # By default, silence all loggers except the logger for rank 0 + self.remove_stream_handlers() + # If NEMO_TESTING is set, add a streamhandler to all ranks + if get_envbool(NEMO_ENV_VARNAME_TESTING, False): + old_factory = _logging.getLogRecordFactory() + + def record_factory(*args, **kwargs): + record = old_factory(*args, **kwargs) + record.rank = self.rank + return record + + _logging.setLogRecordFactory(record_factory) + self.add_stream_handlers(formatter=DebugNeMoFormatter) + elif is_global_rank_zero(): + self.add_stream_handlers() + + # Add memoryhandlers, essentially buffers. They are used to save messages that we will flush to file + # once the appropriate file handlers are added. + if is_global_rank_zero(): + # Add a memoryhandler for error messages. Only logged on rank 0 + self._handlers["memory_err"] = MemoryHandler(-1) + self._handlers["memory_err"].addFilter(lambda record: record.levelno > _logging.INFO) + formatter = BaseNeMoFormatter + self._handlers["memory_err"].setFormatter(formatter()) + self._logger.addHandler(self._handlers["memory_err"]) + # Add a memoryhandler for all messages on all ranks + self._handlers["memory_all"] = MemoryHandler(-1) + formatter = BaseNeMoFormatter + self._handlers["memory_all"].setFormatter(formatter()) + self._logger.addHandler(self._handlers["memory_all"]) + + finally: + level = Logger.INFO + if get_envbool(NEMO_ENV_VARNAME_TESTING, False): + level = Logger.DEBUG + self.set_verbosity(verbosity_level=level) + self.captureWarnings(capture_warnings) + + self._logger.propagate = False + + def remove_stream_handlers(self): + """ Removes StreamHandler that log to stdout and stderr from the logger.""" + if self._logger is None: + raise RuntimeError("Impossible to set handlers if the Logger is not predefined") + + # ======== Remove Handler if already existing ======== + + try: + self._logger.removeHandler(self._handlers["stream_stdout"]) + del self._handlers["stream_stdout"] + except KeyError: + pass + + try: + self._logger.removeHandler(self._handlers["stream_stderr"]) + del self._handlers["stream_stderr"] + except KeyError: + pass + + def add_stream_handlers(self, formatter=BaseNeMoFormatter): + """Add StreamHandler that log to stdout and stderr to the logger. INFO and lower logs are streamed to stdout + while WARNING and higher are streamed to stderr. If the NEMO_ENV_VARNAME_REDIRECT_LOGS_TO_STDERR environment + variable is set, all logs are sent to stderr instead. + """ + if self._logger is None: + raise RuntimeError("Impossible to set handlers if the Logger is not predefined") + + # Add the output handler. + if get_envbool(NEMO_ENV_VARNAME_REDIRECT_LOGS_TO_STDERR, False): + self._handlers["stream_stdout"] = _logging.StreamHandler(sys.stderr) + + else: + self._handlers["stream_stdout"] = _logging.StreamHandler(sys.stdout) + self._handlers["stream_stdout"].addFilter(lambda record: record.levelno <= _logging.INFO) + + self._handlers["stream_stderr"] = _logging.StreamHandler(sys.stderr) + self._handlers["stream_stderr"].addFilter(lambda record: record.levelno > _logging.INFO) + + self._handlers["stream_stdout"].setFormatter(formatter()) + self._logger.addHandler(self._handlers["stream_stdout"]) + + try: + self._handlers["stream_stderr"].setFormatter(formatter()) + self._logger.addHandler(self._handlers["stream_stderr"]) + except KeyError: + pass + + def reset_stream_handler(self, formatter=BaseNeMoFormatter): + """Removes then adds stream handlers.""" + self.remove_stream_handlers() + self.add_stream_handlers(formatter=formatter) + + def add_file_handler(self, log_file): + """Add a FileHandler to logger that logs all messages to a file. If the logger had a MemoryHandler at + self._handlers["memory_all"], those buffered messages are flushed to the new file, and the MemoryHandler is + closed.""" + if self._logger is None: + raise RuntimeError("Impossible to set handlers if the Logger is not predefined") + + self._handlers["file"] = _logging.FileHandler(log_file) + formatter = BaseNeMoFormatter + self._handlers["file"].setFormatter(formatter()) + self._logger.addHandler(self._handlers["file"]) + + if self._handlers.get("memory_all", None): + self._handlers["memory_all"].setTarget(self._handlers["file"]) + self._handlers["memory_all"].close() # flush and remove + del self._handlers["memory_all"] + + def add_err_file_handler(self, log_file): + """Add a FileHandler to logger that logs all WARNING and higher messages to a file. If the logger had a + MemoryHandler at self._handlers["memory_err"], those buffered messages are flushed to the new file, and the + MemoryHandler is closed.""" + if self._logger is None: + raise RuntimeError("Impossible to set handlers if the Logger is not predefined") + + self._handlers["file_err"] = _logging.FileHandler(log_file) + self._handlers["file_err"].addFilter(lambda record: record.levelno > _logging.INFO) + + formatter = BaseNeMoFormatter + self._handlers["file_err"].setFormatter(formatter()) + self._logger.addHandler(self._handlers["file_err"]) + + if self._handlers.get("memory_err", None): + self._handlers["memory_err"].setTarget(self._handlers["file_err"]) + self._handlers["memory_err"].close() # flush and remove + del self._handlers["memory_err"] + + def getEffectiveLevel(self): + """Return how much logging output will be produced.""" + if self._logger is not None: + return self._logger.getEffectiveLevel() + + def get_verbosity(self): + """See getEffectiveLevel""" + return self.getEffectiveLevel() + + def setLevel(self, verbosity_level): + """Sets the threshold for what messages will be logged.""" + if self._logger is not None: + self._logger.setLevel(verbosity_level) + + for handler in self._logger.handlers: + handler.setLevel(verbosity_level) + + def set_verbosity(self, verbosity_level): + """See setLevel""" + self.setLevel(verbosity_level) + + @contextmanager + def patch_stderr_handler(self, stream): + """ Sends messages that should log to stderr to stream instead. Useful for unittests """ + if self._logger is not None: + try: + old_stream = self._handlers["stream_stderr"].stream + if old_stream is None: + raise ValueError + + # Port backwards set_stream() from python 3.7 + self._handlers["stream_stderr"].acquire() + try: + self._handlers["stream_stderr"].flush() + self._handlers["stream_stderr"].stream = stream + finally: + self._handlers["stream_stderr"].release() + + yield stream + except (KeyError, ValueError): + raise RuntimeError("Impossible to patch logging handlers if handler does not exist") + finally: + # Port backwards set_stream() from python 3.7 + self._handlers["stream_stderr"].acquire() + try: + self._handlers["stream_stderr"].flush() + self._handlers["stream_stderr"].stream = old_stream + finally: + self._handlers["stream_stderr"].release() + + else: + raise RuntimeError("Impossible to patch logging handlers if handler does not exist") + + @contextmanager + def patch_stdout_handler(self, stream): + """ Sends messages that should log to stdout to stream instead. Useful for unittests """ + if self._logger is not None: + try: + old_stream = self._handlers["stream_stdout"].stream + if old_stream is None: + raise ValueError + + # Port backwards set_stream() from python 3.7 + self._handlers["stream_stdout"].acquire() + try: + self._handlers["stream_stdout"].flush() + self._handlers["stream_stdout"].stream = stream + finally: + self._handlers["stream_stdout"].release() + + yield stream + except (KeyError, ValueError): + raise RuntimeError("Impossible to patch logging handlers if handler does not exist") + finally: + # Port backwards set_stream() from python 3.7 + self._handlers["stream_stdout"].acquire() + try: + self._handlers["stream_stdout"].flush() + self._handlers["stream_stdout"].stream = old_stream + finally: + self._handlers["stream_stdout"].release() + + else: + raise RuntimeError("Impossible to patch logging handlers if handler does not exist") + + @contextmanager + def temp_verbosity(self, verbosity_level): + """Sets the a temporary threshold for what messages will be logged.""" + + if self._logger is not None: + + old_verbosity = self.get_verbosity() + + try: + self.set_verbosity(verbosity_level) + yield + + finally: + self.set_verbosity(old_verbosity) + + else: + try: + yield + + finally: + pass + + def captureWarnings(self, capture): + """ + If capture is true, redirect all warnings to the logging package. + If capture is False, ensure that warnings are not redirected to logging + but to their original destinations. + """ + + if self._logger is not None: + + if capture and self.old_warnings_showwarning is None: + # Backup Method + self.old_warnings_showwarning = warnings.showwarning + warnings.showwarning = self._showwarning + + elif not capture and self.old_warnings_showwarning is not None: + # Restore Method + warnings.showwarning = self.old_warnings_showwarning + self.old_warnings_showwarning = None + + def _showwarning(self, message, category, filename, lineno, file=None, line=None): + """ + Implementation of showwarnings which redirects to logging. + It will call warnings.formatwarning and will log the resulting string + with level logging.WARNING. + """ + s = warnings.formatwarning(message, category, filename, lineno, line) + self.warning("%s", s) + + def _logged_once(self, msg, mode): + PREFIX_LEN = 12 + if mode == LogMode.ONCE: + if msg[PREFIX_LEN:] in self.once_logged: + return True + self.once_logged.add(msg[PREFIX_LEN:]) + return False + + def debug(self, msg, *args, mode=LogMode.EACH, **kwargs): + """ + Log 'msg % args' with severity 'DEBUG'. + + To pass exception information, use the keyword argument exc_info with + a true value, e.g. + + logger.debug("Houston, we have a %s", "thorny problem", exc_info=1) + """ + if self._logger is not None and self._logger.isEnabledFor(Logger.DEBUG) and not self._logged_once(msg, mode): + self._logger._log(Logger.DEBUG, msg, args, **kwargs) + + def info(self, msg, *args, mode=LogMode.EACH, **kwargs): + """ + Log 'msg % args' with severity 'INFO'. + + To pass exception information, use the keyword argument exc_info with + a true value, e.g. + + logger.info("Houston, we have a %s", "interesting problem", exc_info=1) + """ + if self._logger is not None and self._logger.isEnabledFor(Logger.INFO) and not self._logged_once(msg, mode): + self._logger._log(Logger.INFO, msg, args, **kwargs) + + def warning(self, msg, *args, mode=LogMode.EACH, **kwargs): + """ + Log 'msg % args' with severity 'WARNING'. + + To pass exception information, use the keyword argument exc_info with + a true value, e.g. + + logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1) + """ + if self._logger is not None and self._logger.isEnabledFor(Logger.WARNING) and not self._logged_once(msg, mode): + self._logger._log(Logger.WARNING, msg, args, **kwargs) + + def error(self, msg, *args, mode=LogMode.EACH, **kwargs): + """ + Log 'msg % args' with severity 'ERROR'. + + To pass exception information, use the keyword argument exc_info with + a true value, e.g. + + logger.error("Houston, we have a %s", "major problem", exc_info=1) + """ + if self._logger is not None and self._logger.isEnabledFor(Logger.ERROR) and not self._logged_once(msg, mode): + self._logger._log(Logger.ERROR, msg, args, **kwargs) + + def critical(self, msg, *args, mode=LogMode.EACH, **kwargs): + """ + Log 'msg % args' with severity 'CRITICAL'. + + To pass exception information, use the keyword argument exc_info with + a true value, e.g. + + logger.critical("Houston, we have a %s", "major disaster", exc_info=1) + """ + if ( + self._logger is not None + and self._logger.isEnabledFor(Logger.CRITICAL) + and not self._logged_once(msg, mode) + ): + self._logger._log(Logger.CRITICAL, msg, args, **kwargs) diff --git a/models/tacotron2/NeMo/nemo_text_processing/README.md b/models/tacotron2/NeMo/nemo_text_processing/README.md new file mode 100644 index 0000000000..fd0f0197c0 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/README.md @@ -0,0 +1,8 @@ +**nemo_text_processing** +========================== + +Introduction +------------ + +NeMo's `nemo_text_processing` is a Python package that is installed with the `nemo_toolkit`. +See [documentation](https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/) for details. \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/README.md b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/README.md new file mode 100644 index 0000000000..30d2f22309 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/README.md @@ -0,0 +1,12 @@ +Inverse Text Normalization system for english, e.g. `one hundred twenty three kilograms` -> `123 kg` +Offers prediction and evaluation on text normalization data, e.g. [Google text normalization dataset](https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish). + + +Install dependencies: +bash setup.sh + +Example prediction run: +python run_predict.py --input=`INPUT_FILE` --output=`OUTPUT_FILE` [--verbose] + +Example evaluation run: +python run_evaluate.py --input=./en_with_types/output-00001-of-00100 [--cat CATEGORY] [--filter] diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/__init__.py new file mode 100644 index 0000000000..51e4804c62 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/__init__.py @@ -0,0 +1,32 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.verbalizers.verbalize_final import VerbalizeFinalFst + +from nemo.utils import logging + +try: + import pynini + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + logging.warning( + "`pynini` is not installed ! \n" + "Please run the `nemo_text_processing/setup.sh` script" + "prior to usage of this toolkit." + ) + + PYNINI_AVAILABLE = False diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/clean_eval_data.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/clean_eval_data.py new file mode 100644 index 0000000000..d9bc2fccb6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/clean_eval_data.py @@ -0,0 +1,342 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser +from typing import List + +import regex as re +from nemo_text_processing.text_normalization.data_loader_utils import ( + EOS_TYPE, + Instance, + load_files, + training_data_to_sentences, +) + + +""" +This file is for evaluation purposes. +filter_loaded_data() cleans data (list of instances) for inverse text normalization. Filters and cleaners can be specified for each semiotic class individually. +For example, normalized text should only include characters and whitespace characters but no punctuation. + Cardinal unnormalized instances should contain at least one integer and all other characters are removed. +""" + + +class Filter: + """ + Filter class + + Args: + class_type: semiotic class used in dataset + process_func: function to transform text + filter_func: function to filter text + + """ + + def __init__(self, class_type: str, process_func: object, filter_func: object): + self.class_type = class_type + self.process_func = process_func + self.filter_func = filter_func + + def filter(self, instance: Instance) -> bool: + """ + filter function + + Args: + filters given instance with filter function + + Returns: True if given instance fulfills criteria or does not belong to class type + """ + if instance.token_type != self.class_type: + return True + return self.filter_func(instance) + + def process(self, instance: Instance) -> Instance: + """ + process function + + Args: + processes given instance with process function + + Returns: processed instance if instance belongs to expected class type or original instance + """ + if instance.token_type != self.class_type: + return instance + return self.process_func(instance) + + +def filter_cardinal_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_cardinal_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r"[^0-9]", "", un_normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_ordinal_1(instance: Instance) -> bool: + ok = re.search(r"(st|nd|rd|th)\s*$", instance.un_normalized) + return ok + + +def process_ordinal_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r"[,\s]", "", un_normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_decimal_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_decimal_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + un_normalized = re.sub(r",", "", un_normalized) + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_measure_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_measure_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r",", "", un_normalized) + un_normalized = re.sub(r"m2", "m²", un_normalized) + un_normalized = re.sub(r"(\d)([^\d.\s])", r"\1 \2", un_normalized) + normalized = re.sub(r"[^a-z\s]", "", normalized) + normalized = re.sub(r"per ([a-z\s]*)s$", r"per \1", normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_money_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_money_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r",", "", un_normalized) + un_normalized = re.sub(r"a\$", r"$", un_normalized) + un_normalized = re.sub(r"us\$", r"$", un_normalized) + un_normalized = re.sub(r"(\d)m\s*$", r"\1 million", un_normalized) + un_normalized = re.sub(r"(\d)bn?\s*$", r"\1 billion", un_normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_time_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_time_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + un_normalized = re.sub(r": ", ":", un_normalized) + un_normalized = re.sub(r"(\d)\s?a\s?m\s?", r"\1 a.m.", un_normalized) + un_normalized = re.sub(r"(\d)\s?p\s?m\s?", r"\1 p.m.", un_normalized) + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_plain_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_plain_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_punct_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_punct_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_date_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_date_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + un_normalized = re.sub(r",", "", un_normalized) + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_letters_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_letters_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_verbatim_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_verbatim_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_digit_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_digit_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_telephone_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_telephone_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_electronic_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_electronic_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_fraction_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_fraction_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_address_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_address_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +filters = [] +filters.append(Filter(class_type="CARDINAL", process_func=process_cardinal_1, filter_func=filter_cardinal_1)) +filters.append(Filter(class_type="ORDINAL", process_func=process_ordinal_1, filter_func=filter_ordinal_1)) +filters.append(Filter(class_type="DECIMAL", process_func=process_decimal_1, filter_func=filter_decimal_1)) +filters.append(Filter(class_type="MEASURE", process_func=process_measure_1, filter_func=filter_measure_1)) +filters.append(Filter(class_type="MONEY", process_func=process_money_1, filter_func=filter_money_1)) +filters.append(Filter(class_type="TIME", process_func=process_time_1, filter_func=filter_time_1)) + +filters.append(Filter(class_type="DATE", process_func=process_date_1, filter_func=filter_date_1)) +filters.append(Filter(class_type="PLAIN", process_func=process_plain_1, filter_func=filter_plain_1)) +filters.append(Filter(class_type="PUNCT", process_func=process_punct_1, filter_func=filter_punct_1)) +filters.append(Filter(class_type="LETTERS", process_func=process_letters_1, filter_func=filter_letters_1)) +filters.append(Filter(class_type="VERBATIM", process_func=process_verbatim_1, filter_func=filter_verbatim_1)) +filters.append(Filter(class_type="DIGIT", process_func=process_digit_1, filter_func=filter_digit_1)) +filters.append(Filter(class_type="TELEPHONE", process_func=process_telephone_1, filter_func=filter_telephone_1)) +filters.append(Filter(class_type="ELECTRONIC", process_func=process_electronic_1, filter_func=filter_electronic_1)) +filters.append(Filter(class_type="FRACTION", process_func=process_fraction_1, filter_func=filter_fraction_1)) +filters.append(Filter(class_type="ADDRESS", process_func=process_address_1, filter_func=filter_address_1)) +filters.append(Filter(class_type=EOS_TYPE, process_func=lambda x: x, filter_func=lambda x: True)) + + +def filter_loaded_data(data: List[Instance], verbose: bool = False) -> List[Instance]: + """ + Filters list of instances + + Args: + data: list of instances + + Returns: filtered and transformed list of instances + """ + updates_instances = [] + for instance in data: + updated_instance = False + for fil in filters: + if fil.class_type == instance.token_type and fil.filter(instance): + instance = fil.process(instance) + updated_instance = True + if updated_instance: + if verbose: + print(instance) + updates_instances.append(instance) + return updates_instances + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("--input", help="input file path", type=str, default='./en_with_types/output-00001-of-00100') + parser.add_argument("--verbose", help="print filtered instances", action='store_true') + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + file_path = args.input + + print("Loading training data: " + file_path) + instance_list = load_files([file_path]) # List of instances + filtered_instance_list = filter_loaded_data(instance_list, args.verbose) + training_data_to_sentences(filtered_instance_list) diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/currency.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/currency.tsv new file mode 100644 index 0000000000..1c300bf774 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/currency.tsv @@ -0,0 +1,35 @@ +$ dollar +$ us dollar +$ united states dollar +£ pound +€ euro +₩ won +nzd new zealand dollar +rs rupee +chf swiss franc +dkk danish kroner +fim finnish markka +aed arab emirates dirham +¥ yen +czk czech koruna +mro mauritanian ouguiya +pkr pakistani rupee +crc costa rican colon +hk$ hong kong dollar +npr nepalese rupee +awg aruban florin +nok norwegian kroner +tzs tanzanian shilling +sek swedish kronor +cyp cypriot pound +r real +sar saudi riyal +cve cape verde escudo +rsd serbian dinar +dm german mark +shp saint helena pounds +php philippine peso +cad canadian dollar +ssp south sudanese pound +scr seychelles rupee +mvr maldivian rufiyaa \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/domain.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/domain.tsv new file mode 100644 index 0000000000..cb4bc3f8f4 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/domain.tsv @@ -0,0 +1 @@ +com com \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/server_name.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/server_name.tsv new file mode 100644 index 0000000000..c447cf4682 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/electronic/server_name.tsv @@ -0,0 +1,2 @@ +gmail g mail +nvidia n vidia \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/magnitudes.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/magnitudes.tsv new file mode 100644 index 0000000000..3bfa4849e8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/magnitudes.tsv @@ -0,0 +1,4 @@ +k thousand +m million +b billion +t trillion \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/measurements.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/measurements.tsv new file mode 100644 index 0000000000..db22e2aee2 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/measurements.tsv @@ -0,0 +1,110 @@ +f fahrenheit +c celsius +km kilometer +m meter +cm centimeter +mm millimeter +ha hectare +mi mile +m² square meter +km² square kilometer +ft foot +% percent +hz hertz +kw kilowatt +hp horsepower +mg milligram +kg kilogram +ghz gigahertz +khz kilohertz +mhz megahertz +lb pound +v volt +h hour +mc mega coulomb +s second +nm nanometer +rpm revolution per minute +min minute +mA milli ampere +% per cent +kwh kilo watt hour +m³ cubic meter +mph mile per hour +tw tera watt +mv milli volt +mw megawatt +μm micrometer +" inch +tb terabyte +cc c c +g gram +da dalton +atm atmosphere +ω ohm +db decibel +ps peta second +oz ounce +hl hecto liter +μg microgram +pg petagram +gb gigabyte +kb kilobit +ev electron volt +mb megabyte +kb kilobyte +kbps kilobit per second +mbps megabit per second +st stone +kl kilo liter +tj tera joule +kv kilo volt +mv mega volt +kn kilonewton +mm megameter +au astronomical unit +yd yard +rad radian +lm lumen +hs hecto second +mol mole +gpa giga pascal +ml milliliter +gw gigawatt +ma mega ampere +kt knot +kgf kilogram force +ng nano gram +ns nanosecond +ms mega siemens +bar bar +gl giga liter +μs microsecond +da deci ampere +pa pascal +ds deci second +ms milli second +dm deci meter +dm³ cubic deci meter +amu atomic mass unit +mb megabit +mf mega farad +bq becquerel +pb petabit +mm² square millimeter +cm² square centimeter +sq mi square mile +sq ft square foot +kpa kilopascal +cd candela +tl tera liter +ms mega second +mpa megapascal +pm peta meter +pb peta byte +gwh giga watt hour +kcal kilo calory +gy gray +sv sievert +cwt hundredweight +cc c c diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/months.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/months.tsv new file mode 100644 index 0000000000..6e7903c0ba --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/months.tsv @@ -0,0 +1,12 @@ +january +february +march +april +may +june +july +august +september +october +november +december \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/digit.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/digit.tsv new file mode 100644 index 0000000000..fa329799db --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +one 1 +two 2 +three 3 +four 4 +five 5 +six 6 +seven 7 +eight 8 +nine 9 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/hundred.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/hundred.tsv new file mode 100644 index 0000000000..10e2b9645a --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/hundred.tsv @@ -0,0 +1 @@ +hundred \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/teen.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/teen.tsv new file mode 100644 index 0000000000..8e60fa1d3c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +ten 10 +eleven 11 +twelve 12 +thirteen 13 +fourteen 14 +fifteen 15 +sixteen 16 +seventeen 17 +eighteen 18 +nineteen 19 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/thousands.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/thousands.tsv new file mode 100644 index 0000000000..bf30542bb6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/thousands.tsv @@ -0,0 +1,22 @@ +thousand +million +billion +trillion +quadrillion +quintillion +sextillion +septillion +octillion +nonillion +decillion +undecillion +duodecillion +tredecillion +quattuordecillion +quindecillion +sexdecillion +septendecillion +octodecillion +novemdecillion +vigintillion +centillion \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/ties.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/ties.tsv new file mode 100644 index 0000000000..d7ec82e114 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/ties.tsv @@ -0,0 +1,9 @@ +twenty 2 +thirty 3 +forty 4 +fourty 4 +fifty 5 +sixty 6 +seventy 7 +eighty 8 +ninety 9 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/zero.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/zero.tsv new file mode 100644 index 0000000000..c479272d40 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/numbers/zero.tsv @@ -0,0 +1 @@ +zero 0 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/digit.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/digit.tsv new file mode 100644 index 0000000000..4b7fc24f16 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/digit.tsv @@ -0,0 +1,9 @@ +first one +second two +third three +fourth four +fifth five +sixth sixth +seventh seven +eighth eight +ninth nine \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/teen.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/teen.tsv new file mode 100644 index 0000000000..496fefc086 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/ordinals/teen.tsv @@ -0,0 +1 @@ +twelfth twelve \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_suffix.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_suffix.tsv new file mode 100644 index 0000000000..b39a9ef9a9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_suffix.tsv @@ -0,0 +1,8 @@ +p m p.m. +pm p.m. +p.m. +p.m p.m. +am a.m. +a.m. +a.m a.m. +a m a.m. \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_to.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_to.tsv new file mode 100644 index 0000000000..1b006ea3ea --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_to.tsv @@ -0,0 +1,12 @@ +one 12 +two 1 +three 2 +four 3 +five 4 +six 5 +seven 6 +eigh 7 +nine 8 +ten 9 +eleven 10 +twelve 11 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_zone.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_zone.tsv new file mode 100644 index 0000000000..3e0ade467a --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/time/time_zone.tsv @@ -0,0 +1,7 @@ +cst c s t +cet c e t +pst p s t +est e s t +pt p t +et e t +gmt g m t diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/whitelist.tsv b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/whitelist.tsv new file mode 100644 index 0000000000..92f3877d7e --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/data/whitelist.tsv @@ -0,0 +1,10 @@ +e.g. for example +i.e. that is +dr. doctor +mr. mister +mrs. misses +ms. miss +st. saint +7-eleven seven eleven +es3 e s three +s&p s and p diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/inverse_normalize.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/inverse_normalize.py new file mode 100644 index 0000000000..44f4d5ece9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/inverse_normalize.py @@ -0,0 +1,71 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser +from typing import List + +from nemo_text_processing.inverse_text_normalization.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.verbalizers.verbalize_final import VerbalizeFinalFst +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.token_parser import TokenParser + + +class InverseNormalizer(Normalizer): + """ + Inverse normalizer that converts text from spoken to written form. Useful for ASR postprocessing. + Input is expected to have no punctuation and be lower cased. + """ + + def __init__(self): + self.tagger = ClassifyFst() + self.verbalizer = VerbalizeFinalFst() + self.parser = TokenParser() + + def inverse_normalize_list(self, texts: List[str], verbose=False) -> List[str]: + """ + NeMo inverse text normalizer + + Args: + texts: list of input strings + verbose: whether to print intermediate meta information + + Returns converted list of input strings + """ + return self.normalize_list(texts=texts, verbose=verbose) + + def inverse_normalize(self, text: str, verbose: bool) -> str: + """ + Main function. Inverse normalizes tokens from spoken to written form + e.g. twelve kilograms -> 12 kg + + Args: + text: string that may include semiotic classes + verbose: whether to print intermediate meta information + + Returns: written form + """ + return self.normalize(text=text, verbose=verbose) + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("input_string", help="input string", type=str) + parser.add_argument("--verbose", help="print info for debugging", action='store_true') + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + inverse_normalizer = InverseNormalizer() + print(inverse_normalizer.inverse_normalize(args.input_string, verbose=args.verbose)) diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/run_evaluate.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/run_evaluate.py new file mode 100644 index 0000000000..845486f322 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/run_evaluate.py @@ -0,0 +1,109 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser + +from nemo_text_processing.inverse_text_normalization.clean_eval_data import filter_loaded_data +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.data_loader_utils import ( + evaluate, + known_types, + load_files, + training_data_to_sentences, + training_data_to_tokens, +) + + +''' +Runs Evaluation on data in the format of : \t\t<`self` if trivial class or normalized text> +like the Google text normalization data https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish +''' + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("--input", help="input file path", type=str) + parser.add_argument( + "--cat", + dest="category", + help="focus on class only (" + ", ".join(known_types) + ")", + type=str, + default=None, + choices=known_types, + ) + parser.add_argument("--filter", action='store_true', help="clean data for inverse normalization purposes") + return parser.parse_args() + + +if __name__ == "__main__": + # Example usage: + # python run_evaluate.py --input= --cat= --filter + args = parse_args() + file_path = args.input + inverse_normalizer = InverseNormalizer() + + print("Loading training data: " + file_path) + training_data = load_files([file_path]) + + if args.filter: + training_data = filter_loaded_data(training_data) + + if args.category is None: + print("Sentence level evaluation...") + sentences_un_normalized, sentences_normalized, _ = training_data_to_sentences(training_data) + print("- Data: " + str(len(sentences_normalized)) + " sentences") + sentences_prediction = inverse_normalizer.inverse_normalize_list(sentences_normalized) + print("- Denormalized. Evaluating...") + sentences_accuracy = evaluate( + preds=sentences_prediction, labels=sentences_un_normalized, input=sentences_normalized + ) + print("- Accuracy: " + str(sentences_accuracy)) + + print("Token level evaluation...") + tokens_per_type = training_data_to_tokens(training_data, category=args.category) + token_accuracy = {} + for token_type in tokens_per_type: + print("- Token type: " + token_type) + tokens_un_normalized, tokens_normalized = tokens_per_type[token_type] + print(" - Data: " + str(len(tokens_normalized)) + " tokens") + tokens_prediction = inverse_normalizer.inverse_normalize_list(tokens_normalized) + print(" - Denormalized. Evaluating...") + token_accuracy[token_type] = evaluate(tokens_prediction, tokens_un_normalized, input=tokens_normalized) + print(" - Accuracy: " + str(token_accuracy[token_type])) + token_count_per_type = {token_type: len(tokens_per_type[token_type][0]) for token_type in tokens_per_type} + token_weighted_accuracy = [ + token_count_per_type[token_type] * accuracy for token_type, accuracy in token_accuracy.items() + ] + print("- Accuracy: " + str(sum(token_weighted_accuracy) / sum(token_count_per_type.values()))) + + print(" - Total: " + str(sum(token_count_per_type.values())), '\n') + + for token_type in token_accuracy: + if token_type not in known_types: + raise ValueError("Unexpected token type: " + token_type) + + if args.category is None: + c1 = ['Class', 'sent level'] + known_types + c2 = ['Num Tokens', len(sentences_normalized)] + [ + token_count_per_type[known_type] if known_type in tokens_per_type else '0' for known_type in known_types + ] + c3 = ["Denormalization", sentences_accuracy] + [ + token_accuracy[known_type] if known_type in token_accuracy else '0' for known_type in known_types + ] + + for i in range(len(c1)): + print(f'{str(c1[i]):10s} | {str(c2[i]):10s} | {str(c3[i]):5s}') + else: + print(f'numbers\t{token_count_per_type[args.category]}') + print(f'Denormalization\t{token_accuracy[args.category]}') diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/run_predict.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/run_predict.py new file mode 100644 index 0000000000..39ce2ca40d --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/run_predict.py @@ -0,0 +1,75 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser +from typing import List + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + + +''' +Runs inverse text normalization on text data +''' + + +def load_file(file_path: str) -> List[str]: + """ + Load given text file into list of string. + + Args: + file_path: file path + + Returns: flat list of string + """ + res = [] + with open(file_path, 'r') as fp: + for line in fp: + res.append(line) + return res + + +def write_file(file_path: str, data: List[str]): + """ + Writes out list of string to file. + + Args: + file_path: file path + data: list of string + + """ + with open(file_path, 'w') as fp: + for line in data: + fp.write(line + '\n') + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("--input", help="input file path", required=True, type=str) + parser.add_argument("--output", help="output file path", required=True, type=str) + parser.add_argument("--verbose", help="print denormalization info. For debugging", action='store_true') + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + file_path = args.input + inverse_normalizer = InverseNormalizer() + + print("Loading data: " + file_path) + data = load_file(file_path) + + print("- Data: " + str(len(data)) + " sentences") + inverse_normalizer_prediction = inverse_normalizer.inverse_normalize_list(data, verbose=args.verbose) + write_file(args.output, inverse_normalizer_prediction) + print(f"- Denormalized. Writing out to {args.output}") diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/cardinal.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/cardinal.py new file mode 100644 index 0000000000..20c78af2d8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/cardinal.py @@ -0,0 +1,134 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.inverse_text_normalization.utils import get_abs_path, num_to_word +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, +) + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. minus twenty three -> cardinal { integer: "23" negative: "-" } } + Numbers below thirteen are not converted. + """ + + def __init__(self): + super().__init__(name="cardinal", kind="classify") + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + + graph_hundred = pynini.cross("hundred", "") + + graph_hundred_component = pynini.union(graph_digit + delete_space + graph_hundred, pynutil.insert("0")) + graph_hundred_component += delete_space + graph_hundred_component += pynini.union( + graph_teen | pynutil.insert("00"), + (graph_ties | pynutil.insert("0")) + delete_space + (graph_digit | pynutil.insert("0")), + ) + + graph_hundred_component_at_least_one_none_zero_digit = graph_hundred_component @ ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT) + ) + self.graph_hundred_component_at_least_one_none_zero_digit = ( + graph_hundred_component_at_least_one_none_zero_digit + ) + + graph_thousands = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("thousand"), + pynutil.insert("000", weight=0.1), + ) + + graph_million = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("million"), + pynutil.insert("000", weight=0.1), + ) + graph_billion = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("billion"), + pynutil.insert("000", weight=0.1), + ) + graph_trillion = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("trillion"), + pynutil.insert("000", weight=0.1), + ) + graph_quadrillion = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("quadrillion"), + pynutil.insert("000", weight=0.1), + ) + graph_quintillion = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("quintillion"), + pynutil.insert("000", weight=0.1), + ) + graph_sextillion = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("sextillion"), + pynutil.insert("000", weight=0.1), + ) + + graph = pynini.union( + graph_sextillion + + delete_space + + graph_quintillion + + delete_space + + graph_quadrillion + + delete_space + + graph_trillion + + delete_space + + graph_billion + + delete_space + + graph_million + + delete_space + + graph_thousands + + delete_space + + graph_hundred_component, + graph_zero, + ) + + graph = graph @ pynini.union( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT), "0" + ) + + labels_exception = [num_to_word(x) for x in range(0, 13)] + graph_exception = pynini.union(*labels_exception) + + graph = pynini.cdrewrite(pynutil.delete("and"), NEMO_SPACE, NEMO_SPACE, NEMO_SIGMA) @ graph + + self.graph_no_exception = graph + + self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + + optional_minus_graph = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("minus", "\"-\"") + NEMO_SPACE, 0, 1 + ) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/date.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/date.py new file mode 100644 index 0000000000..60088db97d --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/date.py @@ -0,0 +1,163 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + GraphFst, + delete_extra_space, + delete_space, +) + +try: + import pynini + from pynini.lib import pynutil + + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")).optimize() + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize() + ties_graph = pynini.string_file(get_abs_path("data/numbers/ties.tsv")).optimize() + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + graph_teen = None + graph_digit = None + ties_graph = None + + PYNINI_AVAILABLE = True + + +def _get_month_graph(): + """ + Transducer for month, e.g. march -> march + """ + month_graph = pynini.string_file(get_abs_path("data/months.tsv")) + return month_graph + + +def _get_ties_graph(): + """ + Transducer for 20-99 e.g + twenty three -> 23 + """ + graph = ties_graph + (delete_space + graph_digit | pynutil.insert("0")) + return graph + + +def _get_range_graph(): + """ + Transducer for decades (1**0s, 2**0s), centuries (2*00s, 1*00s), millennia (2000s) + """ + graph_ties = _get_ties_graph() + graph = (graph_ties | graph_teen) + delete_space + pynini.cross("hundreds", "00s") + graph |= pynini.cross("two", "2") + delete_space + pynini.cross("thousands", "000s") + graph |= ( + (graph_ties | graph_teen) + + delete_space + + (pynini.closure(NEMO_ALPHA, 1) + (pynini.cross("ies", "y") | pynutil.delete("s"))) + @ (graph_ties | pynini.cross("ten", "10")) + + pynutil.insert("s") + ) + graph @= pynini.union("1", "2") + NEMO_DIGIT + NEMO_DIGIT + NEMO_DIGIT + "s" + return graph + + +def _get_year_graph(): + """ + Transducer for year, e.g. twenty twenty -> 2020 + """ + + def _get_digits_graph(): + zero = pynini.cross((pynini.accep("oh") | pynini.accep("o")), "0") + graph = zero + delete_space + graph_digit + graph.optimize() + return graph + + def _get_thousands_graph(): + graph_ties = _get_ties_graph() + graph_hundred_component = (graph_digit + delete_space + pynutil.delete("hundred")) | pynutil.insert("0") + graph = ( + graph_digit + + delete_space + + pynutil.delete("thousand") + + delete_space + + graph_hundred_component + + delete_space + + (graph_teen | graph_ties) + ) + return graph + + graph_ties = _get_ties_graph() + graph_digits = _get_digits_graph() + graph_thousands = _get_thousands_graph() + year_graph = ( + # 20 19, 40 12, 2012 - assuming no limit on the year + (graph_teen + delete_space + (graph_ties | graph_digits | graph_teen)) + | (graph_ties + delete_space + (graph_ties | graph_digits | graph_teen)) + | graph_thousands + ) + year_graph.optimize() + return year_graph + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, + e.g. january fifth twenty twelve -> date { month: "january" day: "5" year: "2012" preserve_order: true } + e.g. the fifth of january twenty twelve -> date { day: "5" month: "january" year: "2012" preserve_order: true } + e.g. twenty twenty -> date { year: "2012" preserve_order: true } + + Args: + ordinal: OrdinalFst + """ + + def __init__(self, ordinal: GraphFst): + super().__init__(name="date", kind="classify") + + ordinal_graph = ordinal.graph + year_graph = _get_year_graph() + YEAR_WEIGHT = 0.001 + year_graph = pynutil.add_weight(year_graph, YEAR_WEIGHT) + month_graph = _get_month_graph() + + month_graph = pynutil.insert("month: \"") + month_graph + pynutil.insert("\"") + + day_graph = pynutil.insert("day: \"") + pynutil.add_weight(ordinal_graph, -0.7) + pynutil.insert("\"") + optional_day_graph = pynini.closure(delete_extra_space + day_graph, 0, 1) + optional_graph_year = pynini.closure( + delete_extra_space + + pynutil.insert("year: \"") + + pynutil.add_weight(year_graph, -YEAR_WEIGHT) + + pynutil.insert("\""), + 0, + 1, + ) + graph_mdy = month_graph + optional_day_graph + optional_graph_year + graph_dmy = ( + pynutil.delete("the") + + delete_space + + day_graph + + delete_space + + pynutil.delete("of") + + delete_extra_space + + month_graph + + optional_graph_year + ) + graph_year = pynutil.insert("year: \"") + (year_graph | _get_range_graph()) + pynutil.insert("\"") + + final_graph = graph_mdy | graph_dmy | graph_year + final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/decimal.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/decimal.py new file mode 100644 index 0000000000..47017f4d36 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/decimal.py @@ -0,0 +1,95 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import NEMO_DIGIT, GraphFst, delete_extra_space, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. one million -> integer_part: "1" quantity: "million" + e.g. one point five million -> integer_part: "1" fractional_part: "5" quantity: "million" + + Args: + decimal: decimal FST + cardinal_up_to_hundred: cardinal FST + """ + numbers = cardinal_up_to_hundred @ ( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT) + ) + suffix = pynini.union("million", "billion", "trillion", "quadrillion", "quintillion", "sextillion") + res = ( + pynutil.insert("integer_part: \"") + + numbers + + pynutil.insert("\"") + + delete_extra_space + + pynutil.insert("quantity: \"") + + suffix + + pynutil.insert("\"") + ) + res |= decimal + delete_extra_space + pynutil.insert("quantity: \"") + (suffix | "thousand") + pynutil.insert("\"") + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal + e.g. minus twelve point five o o six billion -> decimal { negative: "true" integer_part: "12" fractional_part: "5006" quantity: "billion" } + e.g. one billion -> decimal { integer_part: "1" quantity: "billion" } + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="decimal", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + + graph_decimal = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_decimal |= pynini.string_file(get_abs_path("data/numbers/zero.tsv")) | pynini.cross("o", "0") + + graph_decimal = pynini.closure(graph_decimal + delete_space) + graph_decimal + self.graph = graph_decimal + + point = pynutil.delete("point") + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("minus", "\"true\"") + delete_extra_space, 0, 1 + ) + + graph_fractional = pynutil.insert("fractional_part: \"") + graph_decimal + pynutil.insert("\"") + graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + final_graph_wo_sign = ( + pynini.closure(graph_integer + delete_extra_space, 0, 1) + point + delete_extra_space + graph_fractional + ) + final_graph = optional_graph_negative + final_graph_wo_sign + + self.final_graph_wo_negative = final_graph_wo_sign | get_quantity( + final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit + ) + final_graph |= optional_graph_negative + get_quantity( + final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit + ) + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/electronic.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/electronic.py new file mode 100644 index 0000000000..018efd6cf7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/electronic.py @@ -0,0 +1,66 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import NEMO_ALPHA, GraphFst, insert_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic: as URLs, email addresses, etc. + e.g. c d f one at a b c dot e d u -> tokens { electronic { username: "cdf1" domain: "abc.edu" } } + """ + + def __init__(self): + super().__init__(name="electronic", kind="classify") + + delete_extra_space = pynutil.delete(" ") + alpha_num = ( + NEMO_ALPHA + | pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + | pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + ) + username = ( + pynutil.insert("username: \"") + + pynini.closure( + alpha_num + delete_extra_space + pynini.closure(pynini.cross("dot", '.') + delete_extra_space, 0, 1) + ) + + alpha_num + + pynutil.insert("\"") + ) + single_alphanum = pynini.closure(alpha_num + delete_extra_space) + alpha_num + server = single_alphanum | pynini.string_file(get_abs_path("data/electronic/server_name.tsv")).invert() + domain = single_alphanum | pynini.string_file(get_abs_path("data/electronic/domain.tsv")).invert() + domain_graph = ( + pynutil.insert("domain: \"") + + server + + delete_extra_space + + pynini.cross("dot", ".") + + delete_extra_space + + domain + + pynutil.insert("\"") + ) + graph = username + delete_extra_space + pynutil.delete("at") + insert_space + delete_extra_space + domain_graph + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/fraction.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/fraction.py new file mode 100644 index 0000000000..6b2cd74948 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/fraction.py @@ -0,0 +1,26 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import GraphFst + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + """ + + def __init__(self): + super().__init__(name="fraction", kind="classify") + # integer_part # numerator # denominator diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/measure.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/measure.py new file mode 100644 index 0000000000..9cb6755b05 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/measure.py @@ -0,0 +1,103 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_SIGMA, + GraphFst, + convert_space, + delete_extra_space, + delete_space, + get_singulars, +) + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure + e.g. minus twelve kilograms -> measure { negative: "true" cardinal { integer: "12" } units: "kg" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + super().__init__(name="measure", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + + graph_unit = pynini.string_file(get_abs_path("data/measurements.tsv")) + graph_unit_singular = pynini.invert(graph_unit) # singular -> abbr + graph_unit_plural = get_singulars(graph_unit_singular) # plural -> abbr + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("minus", "\"true\"") + delete_extra_space, 0, 1 + ) + + unit_singular = convert_space(graph_unit_singular) + unit_plural = convert_space(graph_unit_plural) + unit_misc = pynutil.insert("/") + pynutil.delete("per") + delete_space + convert_space(graph_unit_singular) + + unit_singular = ( + pynutil.insert("units: \"") + + (unit_singular | unit_misc | pynutil.add_weight(unit_singular + delete_space + unit_misc, 0.01)) + + pynutil.insert("\"") + ) + unit_plural = ( + pynutil.insert("units: \"") + + (unit_plural | unit_misc | pynutil.add_weight(unit_plural + delete_space + unit_misc, 0.01)) + + pynutil.insert("\"") + ) + + subgraph_decimal = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal.final_graph_wo_negative + + pynutil.insert(" }") + + delete_extra_space + + unit_plural + ) + subgraph_cardinal = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + ((NEMO_SIGMA - "one") @ cardinal_graph) + + pynutil.insert("\"") + + pynutil.insert(" }") + + delete_extra_space + + unit_plural + ) + subgraph_cardinal |= ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + pynini.cross("one", "1") + + pynutil.insert("\"") + + pynutil.insert(" }") + + delete_extra_space + + unit_singular + ) + final_graph = subgraph_decimal | subgraph_cardinal + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/money.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/money.py new file mode 100644 index 0000000000..8787419fdb --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/money.py @@ -0,0 +1,112 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + convert_space, + delete_extra_space, + delete_space, + get_singulars, + insert_space, +) + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money + e.g. twelve dollars and five cents -> money { integer_part: "12" fractional_part: 05 currency: "$" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + super().__init__(name="money", kind="classify") + # quantity, integer_part, fractional_part, currency + + cardinal_graph = cardinal.graph_no_exception + graph_decimal_final = decimal.final_graph_wo_negative + + unit = pynini.string_file(get_abs_path("data/currency.tsv")) + unit_singular = pynini.invert(unit) + unit_plural = get_singulars(unit_singular) + + graph_unit_singular = pynutil.insert("currency: \"") + convert_space(unit_singular) + pynutil.insert("\"") + graph_unit_plural = pynutil.insert("currency: \"") + convert_space(unit_plural) + pynutil.insert("\"") + + add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) + # twelve dollars (and) fifty cents, zero cents + cents_standalone = ( + pynutil.insert("fractional_part: \"") + + pynini.union( + pynutil.add_weight(((NEMO_SIGMA - "one") @ cardinal_graph), -0.7) @ add_leading_zero_to_double_digit + + delete_space + + pynutil.delete("cents"), + pynini.cross("one", "01") + delete_space + pynutil.delete("cent"), + ) + + pynutil.insert("\"") + ) + + optional_cents_standalone = pynini.closure( + delete_space + + pynini.closure(pynutil.delete("and") + delete_space, 0, 1) + + insert_space + + cents_standalone, + 0, + 1, + ) + # twelve dollars fifty, only after integer + optional_cents_suffix = pynini.closure( + delete_extra_space + + pynutil.insert("fractional_part: \"") + + pynutil.add_weight(cardinal_graph @ add_leading_zero_to_double_digit, -0.7) + + pynutil.insert("\""), + 0, + 1, + ) + + graph_integer = ( + pynutil.insert("integer_part: \"") + + ((NEMO_SIGMA - "one") @ cardinal_graph) + + pynutil.insert("\"") + + delete_extra_space + + graph_unit_plural + + (optional_cents_standalone | optional_cents_suffix) + ) + graph_integer |= ( + pynutil.insert("integer_part: \"") + + pynini.cross("one", "1") + + pynutil.insert("\"") + + delete_extra_space + + graph_unit_singular + + (optional_cents_standalone | optional_cents_suffix) + ) + graph_decimal = graph_decimal_final + delete_extra_space + graph_unit_plural + graph_decimal |= pynutil.insert("currency: \"$\" integer_part: \"0\" ") + cents_standalone + final_graph = graph_integer | graph_decimal + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/ordinal.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/ordinal.py new file mode 100644 index 0000000000..54b48ae5dc --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/ordinal.py @@ -0,0 +1,50 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import NEMO_CHAR, GraphFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ImportError, ModuleNotFoundError): + PYNINI_AVAILABLE = False + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal + e.g. thirteenth -> ordinal { integer: "13" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="ordinal", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv")) + graph_teens = pynini.string_file(get_abs_path("data/ordinals/teen.tsv")) + graph = pynini.closure(NEMO_CHAR) + pynini.union( + graph_digit, graph_teens, pynini.cross("tieth", "ty"), pynini.cross("th", "") + ) + + self.graph = graph @ cardinal_graph + final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/punctuation.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/punctuation.py new file mode 100644 index 0000000000..bd5a426b30 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/punctuation.py @@ -0,0 +1,41 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import GraphFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = False +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + """ + + def __init__(self): + super().__init__(name="punctuation", kind="classify") + + s = "!#$%&\'()*+,-./:;<=>?@^_`{|}~" + punct = pynini.union(*s) + + graph = pynutil.insert("name: \"") + punct + pynutil.insert("\"") + + self.fst = graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/telephone.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/telephone.py new file mode 100644 index 0000000000..8705f0e752 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/telephone.py @@ -0,0 +1,67 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import GraphFst, delete_space, insert_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying telephone numbers, e.g. + one two three one two three five six seven eight -> { number_part: "123-123-5678" } + """ + + def __init__(self): + super().__init__(name="telephone", kind="classify") + delete_space = pynutil.delete(' ') + # country code, number_part, extension + add_separator = pynutil.insert(" ") # between components + digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))).optimize() | pynini.cross( + "0", pynini.union("o", "oh", "zero") + ) + + number_part = ( + ( + (pynini.closure(digit + insert_space, 2, 2) + digit + pynutil.delete("-")) + | ( + pynutil.delete("(") + + pynini.closure(digit + insert_space, 2, 2) + + digit + + pynutil.delete(")") + + pynini.closure(pynutil.delete("-"), 0, 1) + + delete_space + ) + ) + + add_separator + + pynini.closure(digit + insert_space, 2, 2) + + digit + + pynutil.delete("-") + + add_separator + + pynini.closure(digit + insert_space, 3, 3) + + digit + ) + number_part = pynutil.insert("number_part: \"") + pynini.invert(number_part) + pynutil.insert("\"") + + graph = number_part + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/time.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/time.py new file mode 100644 index 0000000000..1132ee9a13 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/time.py @@ -0,0 +1,124 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.inverse_text_normalization.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.utils import get_abs_path, num_to_word +from nemo_text_processing.text_normalization.graph_utils import ( + GraphFst, + convert_space, + delete_extra_space, + delete_space, + insert_space, +) + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time + e.g. twelve thirty -> time { hours: "12" minutes: "30" } + e.g. twelve past one -> time { minutes: "12" hours: "1" } + e.g. two o clock a m -> time { hours: "2" suffix: "a.m." } + e.g. quarter to two -> time { hours: "1" minutes: "45" } + e.g. quarter past two -> time { hours: "2" minutes: "15" } + e.g. half past two -> time { hours: "2" minutes: "30" } + """ + + def __init__(self): + super().__init__(name="time", kind="classify") + # hours, minutes, seconds, suffix, zone, style, speak_period + + suffix_graph = pynini.string_file(get_abs_path("data/time/time_suffix.tsv")) + time_zone_graph = pynini.invert(pynini.string_file(get_abs_path("data/time/time_zone.tsv"))) + time_to_graph = pynini.string_file(get_abs_path("data/time/time_to.tsv")) + + # only used for < 1000 thousand -> 0 weight + cardinal = pynutil.add_weight(CardinalFst().graph_no_exception, weight=-0.7) + + labels_hour = [num_to_word(x) for x in range(0, 24)] + labels_minute_single = [num_to_word(x) for x in range(1, 10)] + labels_minute_double = [num_to_word(x) for x in range(10, 60)] + + graph_hour = pynini.union(*labels_hour) @ cardinal + + graph_minute_single = pynini.union(*labels_minute_single) @ cardinal + graph_minute_double = pynini.union(*labels_minute_double) @ cardinal + graph_minute_verbose = pynini.cross("half", "30") | pynini.cross("quarter", "15") + oclock = pynini.cross(pynini.union("o' clock", "o clock", "o'clock", "oclock"), "") + + final_graph_hour = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"") + final_graph_minute = ( + pynutil.insert("minutes: \"") + + ( + pynutil.insert("00") + | oclock + pynutil.insert("00") + | pynutil.delete("o") + delete_space + graph_minute_single + | graph_minute_double + ) + + pynutil.insert("\"") + ) + final_suffix = pynutil.insert("suffix: \"") + convert_space(suffix_graph) + pynutil.insert("\"") + final_suffix_optional = pynini.closure(delete_space + insert_space + final_suffix, 0, 1) + final_time_zone_optional = pynini.closure( + delete_space + + insert_space + + pynutil.insert("zone: \"") + + convert_space(time_zone_graph) + + pynutil.insert("\""), + 0, + 1, + ) + + # five o' clock + # two o eight, two thiry five (am/pm) + # two pm/am + graph_hm = final_graph_hour + delete_extra_space + final_graph_minute + # 10 past four, quarter past four, half past four + graph_mh = ( + pynutil.insert("minutes: \"") + + pynini.union(graph_minute_single, graph_minute_double, graph_minute_verbose) + + pynutil.insert("\"") + + delete_space + + pynutil.delete("past") + + delete_extra_space + + final_graph_hour + ) + + graph_quarter_time = ( + pynutil.insert("minutes: \"") + + pynini.cross("quarter", "45") + + pynutil.insert("\"") + + delete_space + + pynutil.delete(pynini.union("to", "till")) + + delete_extra_space + + pynutil.insert("hours: \"") + + time_to_graph + + pynutil.insert("\"") + ) + final_graph = ( + (graph_hm | graph_mh | graph_quarter_time) + final_suffix_optional + final_time_zone_optional + ).optimize() + + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/tokenize_and_classify.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000..0c7718c499 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/tokenize_and_classify.py @@ -0,0 +1,91 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.taggers.date import DateFst +from nemo_text_processing.inverse_text_normalization.taggers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.taggers.electronic import ElectronicFst +from nemo_text_processing.inverse_text_normalization.taggers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.taggers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.taggers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.taggers.punctuation import PunctuationFst +from nemo_text_processing.inverse_text_normalization.taggers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.taggers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.taggers.whitelist import WhiteListFst +from nemo_text_processing.inverse_text_normalization.taggers.word import WordFst +from nemo_text_processing.text_normalization.graph_utils import GraphFst, delete_extra_space, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + """ + + def __init__(self): + super().__init__(name="tokenize_and_classify", kind="classify") + + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal) + ordinal_graph = ordinal.fst + + decimal = DecimalFst(cardinal) + decimal_graph = decimal.fst + + measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal).fst + date_graph = DateFst(ordinal=ordinal).fst + word_graph = WordFst().fst + time_graph = TimeFst().fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst + whitelist_graph = WhiteListFst().fst + punct_graph = PunctuationFst().fst + electronic_graph = ElectronicFst().fst + telephone_graph = TelephoneFst().fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(date_graph, 1.09) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(telephone_graph, 1.1) + | pynutil.add_weight(electronic_graph, 1.1) + | pynutil.add_weight(word_graph, 100) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/whitelist.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/whitelist.py new file mode 100644 index 0000000000..3ac3b9eb04 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/whitelist.py @@ -0,0 +1,40 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import GraphFst, convert_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelisted tokens + e.g. misses -> tokens { name: "mrs." } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + """ + + def __init__(self): + super().__init__(name="whitelist", kind="classify") + + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")).invert() + graph = pynutil.insert("name: \"") + convert_space(whitelist) + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/word.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/word.py new file mode 100644 index 0000000000..6be7360b78 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/taggers/word.py @@ -0,0 +1,36 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_SPACE, GraphFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. sleep -> tokens { name: "sleep" } + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/utils.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/utils.py new file mode 100644 index 0000000000..6a8bd48cda --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/utils.py @@ -0,0 +1,47 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Union + +import inflect + +_inflect = inflect.engine() + + +def num_to_word(x: Union[str, int]): + """ + converts integer to spoken representation + + Args + x: integer + + Returns: spoken representation + """ + if isinstance(x, int): + x = str(x) + x = _inflect.number_to_words(str(x)).replace("-", " ").replace(",", "") + return x + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/cardinal.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/cardinal.py new file mode 100644 index 0000000000..905da49895 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/cardinal.py @@ -0,0 +1,55 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal + e.g. cardinal { integer: "23" negative: "-" } -> -23 + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space, + 0, + 1, + ) + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + self.numbers = graph + graph = optional_sign + graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/date.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/date.py new file mode 100644 index 0000000000..00ee9c5806 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/date.py @@ -0,0 +1,87 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_space, +) + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { month: "january" day: "5" year: "2012" preserve_order: true } -> february 5 2012 + date { day: "5" month: "january" year: "2012" preserve_order: true } -> 5 february 2012 + """ + + def __init__(self): + super().__init__(name="date", kind="verbalize") + month = ( + pynutil.delete("month:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + day = ( + pynutil.delete("day:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + year = ( + pynutil.delete("year:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + delete_space + + pynutil.delete("\"") + ) + + # month (day) year + graph_mdy = ( + month + pynini.closure(delete_extra_space + day, 0, 1) + pynini.closure(delete_extra_space + year, 0, 1) + ) + + # (day) month year + graph_dmy = ( + pynini.closure(day + delete_extra_space, 0, 1) + month + pynini.closure(delete_extra_space + year, 0, 1) + ) + + optional_preserve_order = pynini.closure( + pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + | pynutil.delete("field_order:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space + ) + + final_graph = (graph_mdy | year | graph_dmy) + delete_space + optional_preserve_order + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/decimal.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/decimal.py new file mode 100644 index 0000000000..82645afc90 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/decimal.py @@ -0,0 +1,65 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, e.g. + decimal { negative: "true" integer_part: "12" fractional_part: "5006" quantity: "billion" } -> -12.5006 billion + """ + + def __init__(self): + super().__init__(name="decimal", kind="verbalize") + optionl_sign = pynini.closure(pynini.cross("negative: \"true\"", "-") + delete_space, 0, 1) + integer = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_integer = pynini.closure(integer + delete_space, 0, 1) + fractional = ( + pynutil.insert(".") + + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_fractional = pynini.closure(fractional + delete_space, 0, 1) + quantity = ( + pynutil.delete("quantity:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1) + graph = optional_integer + optional_fractional + optional_quantity + self.numbers = graph + graph = optionl_sign + graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/electronic.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/electronic.py new file mode 100644 index 0000000000..8a8cdf8e55 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/electronic.py @@ -0,0 +1,52 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic + e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> cdf1@abc.edu + """ + + def __init__(self): + super().__init__(name="electronic", kind="verbalize") + user_name = ( + pynutil.delete("username:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + domain = ( + pynutil.delete("domain:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + graph = user_name + delete_space + pynutil.insert("@") + domain + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/fraction.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/fraction.py new file mode 100644 index 0000000000..45a6bd03b5 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/fraction.py @@ -0,0 +1,25 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import GraphFst + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction, + """ + + def __init__(self): + super().__init__(name="fraction", kind="verbalize") diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/measure.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/measure.py new file mode 100644 index 0000000000..734beffa24 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/measure.py @@ -0,0 +1,69 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_CHAR, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + + PYNINI_AVAILABLE = False + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { negative: "true" cardinal { integer: "12" } units: "kg" } -> -12 kg + + Args: + decimal: DecimalFst + cardinal: CardinalFst + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst): + super().__init__(name="measure", kind="verbalize") + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "-"), 0, 1) + unit = ( + pynutil.delete("units:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + + delete_space + ) + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + optional_sign + + delete_space + + decimal.numbers + + delete_space + + pynutil.delete("}") + ) + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + delete_space + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + graph = (graph_cardinal | graph_decimal) + delete_space + pynutil.insert(" ") + unit + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/money.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/money.py new file mode 100644 index 0000000000..30dec90c76 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/money.py @@ -0,0 +1,48 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_CHAR, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + + PYNINI_AVAILABLE = False + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { integer_part: "12" fractional_part: "05" currency: "$" } -> $12.05 + + Args: + decimal: DecimalFst + """ + + def __init__(self, decimal: GraphFst): + super().__init__(name="money", kind="verbalize") + unit = ( + pynutil.delete("currency:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = unit + delete_space + decimal.numbers + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/ordinal.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/ordinal.py new file mode 100644 index 0000000000..8896067c1b --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/ordinal.py @@ -0,0 +1,64 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal, e.g. + ordinal { integer: "13" } -> 13th + """ + + def __init__(self): + super().__init__(name="ordinal", kind="verbalize") + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + convert_eleven = pynini.cross("11", "11th") + convert_twelve = pynini.cross("12", "12th") + convert_thirteen = pynini.cross("13", "13th") + convert_one = pynini.cross("1", "1st") + convert_two = pynini.cross("2", "2nd") + convert_three = pynini.cross("3", "3rd") + convert_rest = pynutil.insert("th", weight=0.01) + + suffix = pynini.cdrewrite( + convert_eleven + | convert_twelve + | convert_thirteen + | convert_one + | convert_two + | convert_three + | convert_rest, + "", + "[EOS]", + NEMO_SIGMA, + ) + graph = graph @ suffix + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/telephone.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/telephone.py new file mode 100644 index 0000000000..5e6329e13c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/telephone.py @@ -0,0 +1,39 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, GraphFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone, e.g. + telephone { number_part: "123-123-5678" } + -> 123-123-5678 + """ + + def __init__(self): + super().__init__(name="telephone", kind="verbalize") + + number_part = pynutil.delete("number_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + delete_tokens = self.delete_tokens(number_part) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/time.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/time.py new file mode 100644 index 0000000000..0319738f0e --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/time.py @@ -0,0 +1,87 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + GraphFst, + delete_space, + insert_space, +) + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + time { hours: "12" minutes: "30" } -> 12:30 + time { hours: "1" minutes: "12" } -> 01:12 + time { hours: "2" suffix: "a.m." } -> 02:00 a.m. + """ + + def __init__(self): + super().__init__(name="time", kind="verbalize") + add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete("\"") + ) + minute = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete("\"") + ) + suffix = ( + delete_space + + insert_space + + pynutil.delete("suffix:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + optional_suffix = pynini.closure(suffix, 0, 1) + zone = ( + delete_space + + insert_space + + pynutil.delete("zone:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + optional_zone = pynini.closure(zone, 0, 1) + graph = ( + hour @ add_leading_zero_to_double_digit + + delete_space + + pynutil.insert(":") + + (minute @ add_leading_zero_to_double_digit) + + optional_suffix + + optional_zone + ) + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/verbalize.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/verbalize.py new file mode 100644 index 0000000000..684a239d79 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/verbalize.py @@ -0,0 +1,62 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.verbalizers.date import DateFst +from nemo_text_processing.inverse_text_normalization.verbalizers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.verbalizers.electronic import ElectronicFst +from nemo_text_processing.inverse_text_normalization.verbalizers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.verbalizers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.verbalizers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.verbalizers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.verbalizers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.graph_utils import GraphFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + ordinal_graph = OrdinalFst().fst + decimal = DecimalFst() + decimal_graph = decimal.fst + measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst + money_graph = MoneyFst(decimal=decimal).fst + time_graph = TimeFst().fst + date_graph = DateFst().fst + whitelist_graph = WhiteListFst().fst + telephone_graph = TelephoneFst().fst + electronic_graph = ElectronicFst().fst + graph = ( + time_graph + | date_graph + | money_graph + | measure_graph + | ordinal_graph + | decimal_graph + | cardinal_graph + | whitelist_graph + | telephone_graph + | electronic_graph + ) + self.fst = graph diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/verbalize_final.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/verbalize_final.py new file mode 100644 index 0000000000..e6f66103d7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/verbalize_final.py @@ -0,0 +1,50 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.graph_utils import GraphFst, delete_extra_space, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + verbalize = VerbalizeFst().fst + word = WordFst().fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/whitelist.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/whitelist.py new file mode 100644 index 0000000000..4bea2e50ab --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/whitelist.py @@ -0,0 +1,44 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.text_normalization.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist + e.g. tokens { name: "mrs." } -> mrs. + """ + + def __init__(self): + super().__init__(name="whitelist", kind="verbalize") + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/word.py b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/word.py new file mode 100644 index 0000000000..24565800ee --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/inverse_text_normalization/verbalizers/word.py @@ -0,0 +1,39 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain tokens + e.g. tokens { name: "sleep" } -> sleep + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/setup.sh b/models/tacotron2/NeMo/nemo_text_processing/setup.sh new file mode 100644 index 0000000000..e410b40fd7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/setup.sh @@ -0,0 +1,2 @@ +#!/bin/bash +conda install -c conda-forge -y pynini=2.1.3 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/README.md b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/README.md new file mode 100644 index 0000000000..f4405dd6c1 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/README.md @@ -0,0 +1,11 @@ +Text Normalization system for english, e.g. `123 kg` -> `one hundred twenty three kilograms` +Offers prediction and evaluation on text normalization data, e.g. [Google text normalization dataset](https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish). + + +Install dependencies: +bash ../setup.sh + +Example prediction run: +python run_predict.py --input=`INPUT_FILE` --output=`OUTPUT_FILE` [--verbose] +Example evaluation run: +python run_evaluate.py --input=./en_with_types/output-00001-of-00100 [--cat CATEGORY] diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/__init__.py new file mode 100644 index 0000000000..97aabf71f3 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import GraphFst +from nemo_text_processing.text_normalization.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.text_normalization.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.verbalizers.verbalize_final import VerbalizeFinalFst + +from nemo.utils import logging + +try: + import pynini + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + logging.warning( + "`pynini` is not installed ! \n" + "Please run the `nemo_text_processing/setup.sh` script" + "prior to usage of this toolkit." + ) + + PYNINI_AVAILABLE = False diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/clean_eval_data.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/clean_eval_data.py new file mode 100644 index 0000000000..8c33c4fa62 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/clean_eval_data.py @@ -0,0 +1,342 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser +from typing import List + +import regex as re +from nemo_text_processing.text_normalization.data_loader_utils import ( + EOS_TYPE, + Instance, + load_files, + training_data_to_sentences, +) + + +""" +This file is for evaluation purposes. +filter_loaded_data() cleans data (list of instances) for text normalization. Filters and cleaners can be specified for each semiotic class individually. +For example, normalized text should only include characters and whitespace characters but no punctuation. + Cardinal unnormalized instances should contain at least one integer and all other characters are removed. +""" + + +class Filter: + """ + Filter class + + Args: + class_type: semiotic class used in dataset + process_func: function to transform text + filter_func: function to filter text + + """ + + def __init__(self, class_type: str, process_func: object, filter_func: object): + self.class_type = class_type + self.process_func = process_func + self.filter_func = filter_func + + def filter(self, instance: Instance) -> bool: + """ + filter function + + Args: + filters given instance with filter function + + Returns: True if given instance fulfills criteria or does not belong to class type + """ + if instance.token_type != self.class_type: + return True + return self.filter_func(instance) + + def process(self, instance: Instance) -> Instance: + """ + process function + + Args: + processes given instance with process function + + Returns: processed instance if instance belongs to expected class type or original instance + """ + if instance.token_type != self.class_type: + return instance + return self.process_func(instance) + + +def filter_cardinal_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_cardinal_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r"[^0-9]", "", un_normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_ordinal_1(instance: Instance) -> bool: + ok = re.search(r"(st|nd|rd|th)\s*$", instance.un_normalized) + return ok + + +def process_ordinal_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r"[,\s]", "", un_normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_decimal_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_decimal_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + un_normalized = re.sub(r",", "", un_normalized) + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_measure_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_measure_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r",", "", un_normalized) + un_normalized = re.sub(r"m2", "m²", un_normalized) + un_normalized = re.sub(r"(\d)([^\d.\s])", r"\1 \2", un_normalized) + normalized = re.sub(r"[^a-z\s]", "", normalized) + normalized = re.sub(r"per ([a-z\s]*)s$", r"per \1", normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_money_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_money_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + un_normalized = re.sub(r",", "", un_normalized) + un_normalized = re.sub(r"a\$", r"$", un_normalized) + un_normalized = re.sub(r"us\$", r"$", un_normalized) + un_normalized = re.sub(r"(\d)m\s*$", r"\1 million", un_normalized) + un_normalized = re.sub(r"(\d)bn?\s*$", r"\1 billion", un_normalized) + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_time_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_time_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + un_normalized = re.sub(r": ", ":", un_normalized) + un_normalized = re.sub(r"(\d)\s?a\s?m\s?", r"\1 a.m.", un_normalized) + un_normalized = re.sub(r"(\d)\s?p\s?m\s?", r"\1 p.m.", un_normalized) + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_plain_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_plain_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_punct_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_punct_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_date_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_date_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + un_normalized = re.sub(r",", "", un_normalized) + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_letters_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_letters_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_verbatim_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_verbatim_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_digit_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_digit_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_telephone_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_telephone_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_electronic_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_electronic_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_fraction_1(instance: Instance) -> bool: + ok = re.search(r"[0-9]", instance.un_normalized) + return ok + + +def process_fraction_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +def filter_address_1(instance: Instance) -> bool: + ok = True + return ok + + +def process_address_1(instance: Instance) -> Instance: + un_normalized = instance.un_normalized + normalized = instance.normalized + normalized = re.sub(r"[^a-z ]", "", normalized) + return Instance(token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized) + + +filters = [] +filters.append(Filter(class_type="CARDINAL", process_func=process_cardinal_1, filter_func=filter_cardinal_1)) +filters.append(Filter(class_type="ORDINAL", process_func=process_ordinal_1, filter_func=filter_ordinal_1)) +filters.append(Filter(class_type="DECIMAL", process_func=process_decimal_1, filter_func=filter_decimal_1)) +filters.append(Filter(class_type="MEASURE", process_func=process_measure_1, filter_func=filter_measure_1)) +filters.append(Filter(class_type="MONEY", process_func=process_money_1, filter_func=filter_money_1)) +filters.append(Filter(class_type="TIME", process_func=process_time_1, filter_func=filter_time_1)) + +filters.append(Filter(class_type="DATE", process_func=process_date_1, filter_func=filter_date_1)) +filters.append(Filter(class_type="PLAIN", process_func=process_plain_1, filter_func=filter_plain_1)) +filters.append(Filter(class_type="PUNCT", process_func=process_punct_1, filter_func=filter_punct_1)) +filters.append(Filter(class_type="LETTERS", process_func=process_letters_1, filter_func=filter_letters_1)) +filters.append(Filter(class_type="VERBATIM", process_func=process_verbatim_1, filter_func=filter_verbatim_1)) +filters.append(Filter(class_type="DIGIT", process_func=process_digit_1, filter_func=filter_digit_1)) +filters.append(Filter(class_type="TELEPHONE", process_func=process_telephone_1, filter_func=filter_telephone_1)) +filters.append(Filter(class_type="ELECTRONIC", process_func=process_electronic_1, filter_func=filter_electronic_1)) +filters.append(Filter(class_type="FRACTION", process_func=process_fraction_1, filter_func=filter_fraction_1)) +filters.append(Filter(class_type="ADDRESS", process_func=process_address_1, filter_func=filter_address_1)) +filters.append(Filter(class_type=EOS_TYPE, process_func=lambda x: x, filter_func=lambda x: True)) + + +def filter_loaded_data(data: List[Instance], verbose: bool = False) -> List[Instance]: + """ + Filters list of instances + + Args: + data: list of instances + + Returns: filtered and transformed list of instances + """ + updates_instances = [] + for instance in data: + updated_instance = False + for fil in filters: + if fil.class_type == instance.token_type and fil.filter(instance): + instance = fil.process(instance) + updated_instance = True + if updated_instance: + if verbose: + print(instance) + updates_instances.append(instance) + return updates_instances + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("--input", help="input file path", type=str, default='./en_with_types/output-00001-of-00100') + parser.add_argument("--verbose", help="print filtered instances", action='store_true') + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + file_path = args.input + + print("Loading training data: " + file_path) + instance_list = load_files([file_path]) # List of instances + filtered_instance_list = filter_loaded_data(instance_list, args.verbose) + training_data_to_sentences(filtered_instance_list) diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency.tsv new file mode 100644 index 0000000000..1c300bf774 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency.tsv @@ -0,0 +1,35 @@ +$ dollar +$ us dollar +$ united states dollar +£ pound +€ euro +₩ won +nzd new zealand dollar +rs rupee +chf swiss franc +dkk danish kroner +fim finnish markka +aed arab emirates dirham +¥ yen +czk czech koruna +mro mauritanian ouguiya +pkr pakistani rupee +crc costa rican colon +hk$ hong kong dollar +npr nepalese rupee +awg aruban florin +nok norwegian kroner +tzs tanzanian shilling +sek swedish kronor +cyp cypriot pound +r real +sar saudi riyal +cve cape verde escudo +rsd serbian dinar +dm german mark +shp saint helena pounds +php philippine peso +cad canadian dollar +ssp south sudanese pound +scr seychelles rupee +mvr maldivian rufiyaa \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency_minor.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency_minor.tsv new file mode 100644 index 0000000000..61597b9203 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency_minor.tsv @@ -0,0 +1,2 @@ +cents +pence \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency_minor_one.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency_minor_one.tsv new file mode 100644 index 0000000000..d6ab85fdcf --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/currency/currency_minor_one.tsv @@ -0,0 +1,2 @@ +cent +penny \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/domain.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/domain.tsv new file mode 100644 index 0000000000..cb4bc3f8f4 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/domain.tsv @@ -0,0 +1 @@ +com com \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/server_name.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/server_name.tsv new file mode 100644 index 0000000000..c447cf4682 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/server_name.tsv @@ -0,0 +1,2 @@ +gmail g mail +nvidia n vidia \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/symbols.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/symbols.tsv new file mode 100644 index 0000000000..c14f465c39 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/electronic/symbols.tsv @@ -0,0 +1,20 @@ +. dot +- dash +_ underscore +! exclamation mark +# number sign +$ dollar sign +% percent sign +& ampersand +' quote +* asterisk ++ plus +/ slash += equal sign +? question mark +^ circumflex +` right single quote +{ left brace +| vertical bar +} right brace +~ tilde \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/letter_pronunciation.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/letter_pronunciation.tsv new file mode 100644 index 0000000000..3ec49acb9f --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/letter_pronunciation.tsv @@ -0,0 +1,2 @@ +k kay +j jay \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/magnitudes.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/magnitudes.tsv new file mode 100644 index 0000000000..3bfa4849e8 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/magnitudes.tsv @@ -0,0 +1,4 @@ +k thousand +m million +b billion +t trillion \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/measurements.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/measurements.tsv new file mode 100644 index 0000000000..b86459a1f6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/measurements.tsv @@ -0,0 +1,115 @@ +f fahrenheit +c celsius +km kilometer +m meter +cm centimeter +mm millimeter +ha hectare +mi mile +m² square meter +m2 square meter +km² square kilometer +km2 square kilometer +ft foot +% percent +hz hertz +kw kilowatt +hp horsepower +mg milligram +kg kilogram +ghz gigahertz +khz kilohertz +mhz megahertz +lb pound +lbs pound +lbs. pound +v volt +h hour +mc mega coulomb +s second +nm nanometer +rpm revolution per minute +min minute +mA milli ampere +kwh kilo watt hour +m³ cubic meter +m3 cubic meter +mph mile per hour +tw tera watt +mv milli volt +mw megawatt +μm micrometer +" inch +tb terabyte +cc c c +g gram +da dalton +atm atmosphere +ω ohm +db decibel +ps peta second +oz ounce +hl hecto liter +μg microgram +pg petagram +gb gigabyte +kb kilobit +ev electron volt +mb megabyte +kb kilobyte +kbps kilobit per second +mbps megabit per second +kl kilo liter +tj tera joule +kv kilo volt +mv mega volt +kn kilonewton +mm megameter +au astronomical unit +yd yard +rad radian +lm lumen +hs hecto second +mol mole +gpa giga pascal +ml milliliter +gw gigawatt +ma mega ampere +kt knot +kgf kilogram force +ng nano gram +ns nanosecond +ms mega siemens +bar bar +gl giga liter +μs microsecond +da deci ampere +pa pascal +ds deci second +ms milli second +dm deci meter +dm³ cubic deci meter +dm3 cubic deci meter +amu atomic mass unit +mb megabit +mf mega farad +bq becquerel +pb petabit +mm² square millimeter +mm2 square millimeter +cm² square centimeter +cm2 square centimeter +sq mi square mile +sq ft square foot +kpa kilopascal +cd candela +tl tera liter +ms mega second +mpa megapascal +pb peta byte +gwh giga watt hour +kcal kilo calory +gy gray +sv sievert +cwt hundredweight +cc c c diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/abbr.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/abbr.tsv new file mode 100644 index 0000000000..5609e211d6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/abbr.tsv @@ -0,0 +1,12 @@ +jan january +feb february +mar march +apr april +jun june +jul july +aug august +sep september +sept september +oct october +nov november +dec december \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/names.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/names.tsv new file mode 100644 index 0000000000..1c05ac02a3 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/names.tsv @@ -0,0 +1,12 @@ +january +february +march +april +may +june +july +august +september +october +november +december diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/numbers.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/numbers.tsv new file mode 100644 index 0000000000..f0cacd5d1e --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/months/numbers.tsv @@ -0,0 +1,24 @@ +1 january +2 february +3 march +4 april +5 may +6 june +7 july +8 august +9 september +10 october +11 november +12 december +01 january +02 february +03 march +04 april +05 may +06 june +07 july +08 august +09 september +10 october +11 november +12 december \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/cardinal_number_name.far b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/cardinal_number_name.far new file mode 100755 index 0000000000000000000000000000000000000000..2ec682541ca72e575663537b1555059536f6876f GIT binary patch literal 325078 zcmbTf1-M+*756|bWh%Aftd@3)`l&di$s^0oFp`<#2uy-$37#okl3TCH71Zn^Wg9sAm+ zop#=0?1(LP95K0l_Ay20ZQ|>N^H2O(KJ2^C4CT|p0h-J{FKIu!m+xHPQuZ^l(5~0% ztqMJ5p~n`wPoc*Z+P!rCoeDj5p?5B{`#!{ep!?b9r2{>UeI9B**qhcqFB9nL?DH`D zLI3pjdD%esx6jMj5B?9Z&&%5n{&sS(?>^*h5yKcW+J{9U?0sPKdEb3JM-24N)O^v^ z<1vpZOoQy*eHdNnnVa`6ug6EeX0gv6AH1^`UTnJWV6!F6N#dHlV8rD)LT(QG>^Z{s zISVg6Xc?@x0=(c^% zyyR$JL!1}dbCP+9bv{F!7rHYIdVK&}An`6}bk<8=zF&Bu7ZSYaUAXW<53cZf|3!a0 z{nzceb|c$)`9brhvBeVReGMo@kRC!%GgB|EwD>3hxh#o`{jxA5}0VhrJ&QIWXpZv`0^^ zEabp`A~5>qrxk3Kf(>ZiTP>e|jGx&D??>wT^FouGUlbZUzbrI$U$xL2Z+=y1j$gkv z+It%MH}={45&F0G+1GpQt!AISC!trj&)!?;|DApIwHkU2`|SOM|7+T3?;-qM%RYOr zso2*xB*wl@p|QPgq0zNop^1I{LgUK@g(kKQjgGNzWc1SZ1HQ3+j3cCgQ$JL2lpbKiZ+;jG!0fW2;fL+odQ{rK=T!RI=8 z?$o<$$9do#Z=d~G3w^eO!6&ECfp*u3YddbnQg%|o%qqAOO-L3FK?_S~EBk}Gj zc*)CNg%^5n!Hdm(lHPqQy!$2I{R=N^K|XJZamSw9eX8D`+r#L{8+&d~!(z|vRd}%( zd#*VhP3r;c&jH0()pPrp&SecJU;7pts^|7IEcV>~M#r8zpvYp!&C+8B+UMAF2MS(# z?4ZI6eX!uA#||#M(1!?K>U~Jzg+8ppduZZ4LhzEa!wN6-QGyqnhbO&9S9p&|yvG(^ z>TmGGx*h36sz3e=FW8jLd*`L|U8{9W`Hoy4RcK%DeEgyFHgOMebVKX4jbkpo{UwGquf%go7k=HXTn3BWZSwar%Y=O~J=TxwB1xDYT zSHaE~7AUTXYts_|7qFLk`S!Mf+te;2&e23~T<@hjGmT3uW8L=3s> z1V+xU7Z`cHv4XMQ#N09eZ#FqU#zEgw^h92AcZoUdywG#2T@7!{2J1Qof9_fsH6UtkK6cbnjY3jqVf} zajEr<7!|g2v3r-$NlZ!(?4LreU3A_nFwP_HE11f~-G;TxkF(w5FX=c<-<=^ad{b)~ zxlnTCV%dI^+Cz@@JJuJu`-?B}Sl({FYqdNkAMm=3w$c9ML&ZoO4>TOTy#V%Cfzb;O zRxlMmI@MYZrsU|$e^u;VEHHAaF!J<}ki!=x2lj}NYd7DuT1PYm>dhy8`DjJ%VS&+y z*9Z(>4iFge{j}0+S5)L4E7;`Cd*`KHK6m`=_L{PRZ;v-Jy}6)&o~U58c-8R*drFQR zK3VL=n82Pg%-3t^rv)$PGxOTybYAFZ1n=nPyH;!d!VCRu;kBF<`7~D$gsm z8>JI-^+AVh5%Uy%I$z+!@Ft*co*l*aiyPKnjC_13F!FJkz|g7YBEEbq}p80-IkJ&h@kyEfmoTFM@Ba|xYW z7rD6w#yX+wfqk#Y#rlKIW0=jT=RIp!GuvH5d z`vIN5Hq7$`{Tso{yjjckV9pD@ny?9WOMj21cMSx)&9JCv^+r#t75!_wxQ*W#*0y*5 z@v%n3+q+(oS2ZV?*J=(Tw|Qf@S0~5HHHAI)QfpPPwJX><0%MIiawd?t=A7`?Hv!07Q!1jZcOw1RC`!G2%Cwh$Qg*s_9+s9+-n zM!vVIU|S1}e2x|v`P@ce>7L&CgxvNOxg8QV zxY5&jVU9E`<$I2EJ0&@X$=S{oYBOiwgIgU9;2#o#rh5}>XuIwRqq>y9Zew4uIN$gp=CJxrW z#rk)?BBtp~kM}zG?Sncr_;KQz7}y_#4`4@Euww*9?;Kmfj;mnD3yhjx)`Z+UH}LZ$ zAxBOBEHLV(bW)p>g&ei{Q25erzH7DqBINM%uMPHpw}1Vc(8KXz;eKwcHy2>12|4^e zy@H)l!OpB;XIHRu3KrKFd|Aw5i0kvYf|s1mKCr`EQrtSP;qB%TKF3GcUesy-@o|39 ztLD-F>H5|MChI;@zl(~l7!T{%ZN}%k&=*&DFDZKAy)^N<9_I38g%=+#FTBlLvUx=X zyE0)WI`KU7stR^>1^au!R6g?g;p3RodZU89S;5|_U~gBjcPiMs1#6a%;vas_H?a5E{(gmbiJ5wN=V%n}yN~LrB@K%` zwNjz!sig`{PYo$FJ+*S7>8Yg)O-~IiG(ELUq3Nk%M#r97*67$%%NZSeYI&n$Ppx2d z?5P!vjy?55qhn9~$mrNpKQ`KXs(C9p$M< zUhi0=(9Dt5g_=U9N4-QY`qG0SLyMX zQ+zqPNoKDvU{8x{?d0q-4m_@-EsAXO)^K!r|6&SV7qX|6 z{6${$47V?x7kW#fmpu5h)Xoe2XQ7u^{f}QbFZ4*E7i_Dw!ktJR_H zUzD+P;*C8x-mut%yA<9&Mb>LZZYLNPdqbUj#~$6)_+yXmR&+OQQuNc4yBpRn-}&2~ z#TNJ;LY5xgE6MsnJ^Hz~kfmq$O|ttJS@8XYEVVo!$sSZ>!4Is+9-L$kO|pjwS?X|D zl0CD`L-50eEIu8PWREgB&dVb!vVTaj$0XUKi>%sj4zzu2+;7frzyEL=pZ_#Vhi1Qd zQlZ&zE>QSc=l@)A>U4_Pc75zOPcArX|5VfU?l=Ex{BgheH=}(nV*ga5<9_orqlX3k zryK3_276~19rv4O8XfnWXBq9so__vj!+pi~|62XxUifU|jlP{@boA|9qoZ%<86AB) z-{|Pu1x82TE;Ks&c9GHUo4Ouwv0?5jdAX#})Zx-XlZVR+P2MjzI>vT|(J{6wjgGNh zWps?~YNKOpe>XbDc8$?7wrh=!v0Z2McjbN1l)b)PZ@kgB8;p*=-Dq_5?Ixq6Z#Nqq zefyu)HTrgo;nBBSjs7m*=1#u-!+4`_w;3IMyWQyM+Z{$n-|jRz`gWJm(YL#ej=tSv z^mp~dKMjjLac`mNiTesoPuyQylLU3p(V<^2icjlMl;boA{hqoZ$68y$Un#^~tV_e_8E?ODU4Z_gS1UB3M^ z`S!f=M&DjAI{Nma(b2b;jE=s&Y;^SP6{DkXuNobFd(CL~t$Fj0HrmarJzrq!yB|jw zZx&wmF8=+50=_LUj^*zNjCJx|fpJ`WuY$cVFpfJP2n?McR*1TCK| z3yfUMD=>7(=u;6l+6jDhrSphFmkc9z{tf=f#LTu6>ONm$i=b( zL+5f8Z25wbi@0|G&H5m&-KVC$i0kng3G@6B?`n3wk^2eXRxg;czlLGa{+fw5_V8MU z#U5U}@Y2K4*L9Mfc4k#)ie?!6t&29DHUc6wV91so*6in-#s# zn^$;$pLn+@yu>}&9WLXAY5YTXUM~V*?<>vT!Bic7YdsR{uv+SwSclaOi*@*2;l-a= zhc%KO_kex#nufLgi!#6$8y0J}PT}oSWIgZHb6vw?&Ca)TX{UEAyc-zq`*G&@ zdPXm8G&Ntp=x*MM47?lKm+kVMzin)^>jvLQ$f9?XB)eIW1>aQ2(o35s*)57J`0s@* zy)`_^jwrI=TUKO8CfTi%>{ddSI*dxP=a+pL_-G-EPunEfF-Ciy!MCl*ZkJ@oCE2k> zmVS+Tx36G36pZ=f^&jsqHuC-HJ0?BOGsa(R>@4(vjW^8wf!?LU`=wp`kG#89c=NsM zyA|G;18nY5!S<|RdnL?cZTpLjy$cp|$2ySj{n}6Pwi_+>OjJ7W+P~q|zYjkk`Qtj! zb6~-mEql?;Jp4c12bQ06JXrWkpWbG=V}Bh|;m!Af9h&@iH{1SV21Fgj|onV@E03@n$KUm#|x*Z%TdBU*dHp`{?mQ8&Zorht5^>vn`}}! z?!P=n*kkS;D=@C#9VakiI9_18x(hax+%6rcb%KyX=j8shVHE`*vUdCu`4++B}ZQT_iouwuu}v^ZU0)q{w6STeQE_et-cyyoq6`MB*J?v|5Y_RU05W?}yxHccip_r%J@m0xVOM{#ahtFKmTio#*tlKTAdWkP4X`^6Yd4;BUbam({9D$D zyM-RGdy<~*lAhRW|15f9KA2%Z~wR;3|rP{;+nkhHtOtqpFd!~ox1m$(Gs!LK8|*Ma*A56Y3FKg7rKwoOKqPodZEu0ddcThrYEjnQwzQDu3LP7 z?kjl7%ld^Ex}V@h&xT3wG!@>B6YsQzml}J{m>bgzjQsT%82K6?Fnpb%g3VaLW~yL* z|3%zO#u#Q17%|K$Fk+ZZV8k$c1)D=))NoFLp>wVZ=J#;KzK-*o`vA~0PvRYFlElL}qTdsony)@r7zgH9(G5ko{k}TTjI~yNl70H_ppl4U6+~x5Vp`ZGW-xOS9?og|VvekGOs%F!HD{ z&YOQNB@FN9ulvUkx7y^qifwr}FyPwp0g)vW#6LQRx;{`^)FIAZ*${unj z2%XH669vXRQTE9H$wH1}^y6aP;h3T@{Cq;lk?VEDdPP3h6&U$kPhi+vUtrk#P+-(n zVfgZqkRx^_hc6!sIcoHYz>qsr)B(BAg&ej0LSW=<{(j#1z1OOP`?bIAKClU{+xM;l`%fqP5qn11M=t+u$>)R~=Im+qJKKCb0n2}P@_Ats-j@uE zYtzfgX4}n_F~_w2p&N6|`722->U_0=y;iVh=`!tpt(^7kb;F!yPQFoS`sK|+Q@^(g zy|c;qc-!b9_744y(S9u(`dy=k8jb$)J^vR@b$Wxp&mvY!{(=p%K=fAeQ>laSunnJYgv1+lNc5_*}-`S1GtyTY6Q zcF%tbubLC3_2 zeQR{=yO#A%oD-879s6$b(pzdyjJ4cH-&)t3bk|L%+Wul= zV9AG)+izy)&~gU~jB6C?p4}Z?l?{c_8>m<3s9g*g>J+;4e z=UI$ly(Aa0^%Le=@MQymwVlDnh6#&0H%gfEGnY0_Sd`nOf^C{G*U3EV+_&0|-Lba6 z*jTqB=l7S!oW*=@F62f!yAAG(jq4pfQPa7F&`D1W7Z^IZ$2r8h0gKrtc`^S_-nUc+fJ4-A<4A??0we?wp#_;Jz=hLWYK#-!s2|~BVp|# z<8OObu)QkS-W6=03bt>;+z0l*#1Q*+q%+t!(B#_9r_Rd{Qto5C2MHU@!GjAHZHz8@ z4@ou_uh=+L*q|>C6E?7ac(So%#l{iB2FFjo57~P!#=$;0+gPe%<49qHYeK3XVA;ly zijAX$4dVDi!D1X>M;q2Iweh#5D>fDr>j~G3juAG%j!ia(R%{$s^u#rUW0$H2*zw86 zu!@Znl8ty>;~)Jz3*Gf5Vz(vCIY;`7jT4hj|8AVW*iaa~`bQy0kN-(v)a}!>?z7;BdD6YNYOM+|3Gu(Jh53<@I#?xBwJgIu2{bdt041vau1dh1ky(Yp$xcP|if z)b>Jw5yNz%Ux;1VL+&D>lh`j77_lpRBTMX;2svU`7_na}>omIq`PP1T7fYi6vlkG zO~}zNw+oEeml8clK9xP>?hrc3%bfxvc4cp5C-l}`LXOxKM(lSBIby#@V8pOOC3a;G zxqk|s#D1^9h+WyE=ie6ng1v`?PVBuSk40}gYpHr8m8{urABKD zd+2^9!4 z=$xri_umVh*t@P$_YLMPaN1V;V-TfzP(FlzL51^Y%|6G|OgQ~8J9`9PmdCNPfilM9R* zO;N$76xhh(%e%r)boLQ)U{e)r#-eB8^6qba4U4~Xo3+r`>{sZmip(^H=H7+b3QeA; zEi`$UuF&jNrY|(>Vt=EDm~Q9+M#sI-3`P$%e5+!AMx*0iXrR%<3`74+M#sI-Afw}6 zXlA41UT7AhbA8+Xm5td2Mqbpt&8%C;R@SXKgiiWt&I0A7W2o_^B{zrS7Fh zr^4`4Vc1g`a{a_H347|^W_n_i^!F{jHrU%jV94zz?1617WkCOP*UU;TdM7-N)M zO2|>en%oc}M|?{QjGWeV4oz}{-R&}NP4$g23`=qmTQ*^?4quiNSi9I;K4DSk3JG%& z&IeXZSd{x=1^ZFL{8~Nxm|v#Y<8}NWCpq_p^`LXlaPJz^b_N^EROI~L-qs?Z5m%l%yyKe_)q=9gO6bl#nG#(w=x@-t$;O_;~G zg}>N%O5BGAHfR>-=ZwrxX4<`eZJ&sqr^MZ zc(JjuVZKj-cf>= zyo@fq(Ax-J^lqECdGLjje}Gm&+5&pJ(okZOvJGV|&E-V{G%7z8G8E=os6h z#vfyw*YFtITBbk7wzkn88}rVo3I1YZ@zQS*qi>fG7=5s$z(#gdww4kYaziTE6BbvT zQ*oXzJ$vswA8K^ub!wEq*cfH;#$&)R!HYl385Y;R)Z@WxtCa7S!BTXG*U^7)*=@b!6t;p+<( z>_vg$=Su=Z=gSrBm4tchjJMkUNzZ{5L{Ad?iUPxz9}0{Zek3sDeq6y`FEP+3@z{H) z)h`}<4@-luJ_?);jn(EQ$Fxl4j2@pUVKIh-g)hhrN^)+G zHFM^KMV+%G%;iS=i;Yvl^d z%=z2?Vq-|s>EBU%e@-g)Ow7x-LMOH7d7`-fa9y&dvt`$wqCKaOJ0SVu82z$hiXr+r zS<)G?$t&0t33GeI_v2(QV*adOj)8f$vXBF-o>FEzP7Lay!Z+4!08h4|DnNz~y) z5g&3t7dnaW7ZprBTZBD@QTJa8o%o{Uz*ZG&fd_iXoQ_lerpL*^Ho&G;CzRSlf0}^!PN6L*i#twTT|%77bOR_mXK>VS!uP_78t#|Or>|#^EJev zFk*O2oJ)bNBYYu-*JgHyd;1gY`aY9l)Is@zTn$strVyWc76zUEKOC~3__ChB+Rkoc zeSx7e&?JHV``TWkZ2chtXnhht6#T)@~xO`>Msh51mg|>b|YeiM_W(-Pyk! z*53{H_5|lFn~C^{eRF|v4)J?|jV|$RQNe}_j2N~Q7&=E(Fn=z`e&}+rZ8gKys|c^=Tz9YFCl8pxt)6E0zVaop9&-P@s-%sa~0^U zVbjm%S?}#T?Cnyqr=B0cp29fapCEK%Pukjsrz^IY(6HNJuP9+Ccllxfs z!uqB#a<-R zUfHPD`d&}@k#c?A`5I?amPx z^Hn`#!kVfu@~8f0Jv!C1BJ35WsT_k(ox#2CJwBte_FM-T>w)Kt*zvEaq#)k*KA72dxm-pdOw^T%_B&1;M|)@87hWvrR5 zpG9n~3buB_Tpjk7NPDNKbDbimYI|+tXZM=t^%cL@P4=UW^%Ca3F`x4DFpe?rYxdN$ zN$e+XsQ9@-@+JDYVZwY4jB$r-+*HxIQId-~H%XZ5Vhy^rqI1)V+&?OEn2-m zn8(37vw5@F^&U63b_N?y2szFTo)p+t9kJF^0weaP1;%-Ul4E^%M#!P_FcAasJuBo! z7e7@DU@uhUUKALypVsK^%_rDPLXOy#9GH?LFE0z7V6O;l>k|8`73?*Ek?Yqh*c%07 z-g$qr$2hKUXAW?#a=g)bZnga@8~J~4zS-#2*B{1t6w!ok`2A~i=KnSEe$f+o@h$&9i4Q8gFWEIYj~Uqyg@4S23mQLrdvojNyH@K$ zfic%F5*R&wvB2n+O9VzwD>?f1QXz-V%LGPzmkW%%s2GsDvLg4f$R|2^_9N~W>F;ZV zPI7j21yge5Ov!;MIr905hyje}Vqy&R#HT_I>;@47HTq1*f!$Q(RNpUWPQ+gRyy%I% z%%{8N=AwOaj{gaaerlDqE#{OuDV@~f>x#~A1V$ecL(CJt zeA~!%_dEC&dC@a@<9GLZ(i9>Nc-1o}^nRbBha68;^hBHF7+!L~b2~A2@J=oKA+LQa zn3BU@KOqOk^E%O&t(*RBwWbwvVABbVzL~y)^%ofD^D_u+RPkj-fw8|GC@^X`Qw19& zFnVfcfuVDj3O1|2$jioadoaDnrcounaY7Dz8wol5d~x8UI+5F6=p@&32n=856d1Xl zOJLZWyMldBV8k$wz|h&QVE$}W++W3O;&aa4drf>Uqa$y;J~g*t%NUBy?-gF=#PR`~ zC+Qg$u(n~#8ckgD7GADJ`??P{pJ79crXKSfy|mGz?8nCfMK-P{VBa^)bwMvAc&XRU z7xPha3G}HQds$ zSi>R7pXE&wnXQ#PwQ=EB47x%*L{2i(dF?@s%97AJp~@ z(-UjGUGXRK64%p(7y4M?JIAY0{oG1#O*kej)^}2jHB4dX+_9o_X`z$j+V0k)*%xxN zwveN*6vpv+9U+IG>k5q6pAj|1Ph}6e^@L82mFo+Pd@6hRxl<*d%U5(NjGpjk>T<4$ zVGW@ZUnUdxxFWZw&`JG92%XeVVbpJ=ki(a)1coo~RO+YfA-A>AN&Q9%jQU|O&PQT@ ztvNP#*TL5ty7yWeb$O%k68{dxX4Jctm(IqH&3a+~Q?Em|cvdT}QS|JN!gu6$5*RV> zEU*d1-X>x{LL66?{71iu3BRf1w!#K^-7aDAoM{(hg6*0x4+ndfH3#Y5^^iKw zIkj{4V&rKzp_BM_7Z^3%Ltyx_XTf3)(Ycplo(t%`g&zDK+bHPuo4!!?>5BuB{doM@ zr`V^a`zHIbPWx4`{S)RH#_w7!4is|qp2|0Cjmiab>bV@^J4D!PyR$Y9Em*8myFoht zkIO$pA1?Gz!y^PnpByGI>NK_JOX{oSz*H{Ksh*btJE{`zx_vr<@78&1%_Mwd&ip~d z1a`E*m>0)XFqMl@#g}7+9DY7l_Fk-=v41~KYeK|6Ntl~reLO+f!Ok<15$+3Gtv1{(9C09+mnPGzi+L{ zot)&NFQ+6d&an%pbI-cII7QvhOR+mf3@4@-9K%nAF;~?8enF>thGw`cw(*nn`%M3i z8l5lv1XIt0U{7JhpfK`sfv|^8^~?u46-Hj}5cx!>dOl-2Pk@aJg)hW@ZAGWT@Ka&L zevz<;pXzx6bSjM4cOT@D^!6aw6P5mad%$ z*=hgr@h_p5o_?t4g?_m3vR8BYw!hd|d`5rmtrK#OBt4H-Y(AcNpAa_5-;)(=f|zs6 z51vCHKR!QLGiS5&XQx@4mYty!SB#l8^eJHT3-}xSNx4;^sf*0T4O}y_3Ugmtx_4|dFoW#A^he8iFKWeb< zI!^vn-+W@cG43Psb+5&IRpI@2 zvUy&`=6{49dK{bdjX#Hx*FAW%%?m0v|66R5ch^cE|4;Y>mTg>AvGKLAK^)%*8(`lW z)^>-=xFp%|uUT6r>*rMOaR6*`!`cp)aaqz6du@uMC+36sH>I!v)+gDxs$yfRWW(nd z*D1WCLJ5?>^7z= z_M;pzPG7CQ>%5#Iu0ON(3@CDHt$BX-jy!%V-qFR*i-niDK7-H;He-Wzb3Rb; zGAH4sw|U-y{J44M)}W#%V#v)bF#3EJfzh9{Ry}5M{}B7?5nwi9%^vTK#!ug zSLma;D{|N;-uV36JV`G0SUX{Eof>Cf9HVY$rM`8n?JqXwD>`E?+6`iVLv*SRAN2F( z?*ikuvr(t((!6nu`YPX$J6qJE?ZP&eG5bE)>*f3P9itb{NioKG`~8%+h%J;b*Ezyp zYz!6{{k2F1Q`bU}JF7`vw`bfA*rFAkKPXtN6LO0g<}pDpF7&kBkd150@1Ucec7wE9 z*OhxH3fWqs(WB3AMtrZgH-H9gIp4r_!{)Jfe7P7J3xW0C_?a>Tcs@RJyx6LS>od4Z8@PSbN&H=gvaKlGKtsMkl8{;0`;#e9(CQRbl6 z19}w^BiM~I^z#0DUJs;q{|oOeiFdNuI=nF-9DjZ$az$^e`+Lci!kB|U7dpwKk^}pt zkRzs51x8If_ucl^wCy3=_(erd-AhY+N+*7*`+AX6I?2nggfGao`ntEhF(CKrA{Tq8 z?d&!>@pZ2wDI56x8(|-8GGU)S_^prwQ}Lp+uh0orlUq&LV_i@fIb2=HjqjwX^*ezv z2L=c~*}Exw$gLrCvL9PhVDzi9ho2`4IdZnPz{rcj$jdrH4twhgjQD1+!ZyAT71XIH{lbqA=vRpF8@ATn*c?62qIq7wm0r z*Ov19WBvY%h!0=J6npAC;lxI+yI!AFSyN6DHpuUv6XxqMd!OCYy^@Y`{&u>M13ROF zotZGt48H6k@=FdC#+*A#=p=_qjyZRBk&F2SJI64O7y8^}!!eGf=L>AK2WH~}fsv;R zE7(N|b6v!Ap1{yK>(tKKyVetfy0@78DhzwvquZD@?QtLRgS3yh*qGuz;u52Wnhg7h zOO1~Eh|7$Q`-sbpj{AsdXX@QYTw!?JM_g%ij+ef;s$k7q^80H0a)^Bg@86A%eqUpB z^!r+)quMH|@PslmE)?e^kpr1$z8Jqs`U_8udvTi*U+f`z`ccbg%3JgQd9vtFr3)GI=c_+As(=#m%ZC)k@6xwiyHd~a8$i$@M1!qt8AS7;D#O0vla)eqO=85Ewo2 zyui@;Wd-}HU^5o`3zv6)`;TG1#~NO~>u2YUPGj@G1*cE`S7?4`I8EUv&tDguJbY8= zor?Z%3(a1rW%5G|gPzRjxHp>I=(snU!sum;AAhDadYIAZ?_+e_8%<^Oa)uMn)JDg> zQD39;v4U8pEm&Mf!KO3J*HP%{D|-4@^bAOPW)QsOa^A9bL+>QkT>P0)=m8s;Y|d18 z;~EUFy4SbuDYN1Kn?2`_IwGfj?>w{F^B$vzW+^nXgM>e<4GSbJ-jCO^-irQ2Y{4Yw zoXlgt2lu<=zAxlBKGozF5^}`1u)yfgn$E#VZm_#u#*1m67-L^F$wll333GM$vY5cy z#opoxi#nG`n2T`ExMaej+)@>6NW$X2>B0VPp}XI640}r_Ip?IeJNLLmhF6lkh`m~{xF)te zNE@A;b=UIjTh#Mfvf=XNX~pDQ#Qa|HSnH_sb)l0v{v)9itd{#XgiieAp68ffu$s;{ zlg`+$DdAHlofgg-(1sqvWDVm&y5l0$)C#ws%iC!t}&F(Z{ zWidtGvkUKx1z#8?_P(Sk@V*&dzMFrl-gBROOxAJwWWhlldb`5EK;b9X-!C|Q_|GEG zxyix>-?_-&YxofR41J%`u}|+edT6l!fYGr}A2d4l>A#GQefp5m@!aQOqhp^wVs!4) zE&MARk0vbU^sx%|c)}LxsB2!INSJ>~e;--u-@FwW@~ZAF-mUPw*SvSv7AL{@d)TT^ z^8XNfviJtC$IHA_|6eBNt?fq3cq+x{dRPaa7TD+__e=$QHu>fv^jQ8o?#~Hc@|gct z`|}mv{I}X)NdCDGlJL*={osQy{+zZ#LqW`&UT6YO@Wa|b&ueV zMNZw5h}>I3CpmpvV932AF#LQ^V93qh*PZH}yZE_wsdvm7a_%?-8c3bCTRIqx>ryQwj_}`v`1wN31nf1)Ex6aH;4jxWbs_HynEb?9&Q+;8a1%X%7l$^9&a7y2r( z_7bD&3t}8piBa9FMvP$5H+0U{=8@}cfe!|zuHdUkr(IONnlX!tg}h>7{r zXV9cN@%!Dfz7x|>VASbqKdhF-id)yB_tz%Dk-&voeOOA@a^6M5&e+<48Ay;ks| zXa2$qeZAmC&y1x%pl>X^>{;9 zDAzWu?O&8}!~p-G`y=1)<}r;gax?btm5aHUH~AK^`6}4_33Cg~;{^pq&wXED^xQ%M zqn-;_u)!5<(F*4Gu;qL*7xTSXZD%cG=cZWQI^$ z?dBD^uHW-j^eivz)5|Le4EySSFyyWh`+wGJg`qQJo+tAAL*WbS-I;wo@NVBZh1}mO zzNmY!h(XEWXU3wR52ZeczWk=d5Nn2?zcnoCSxxW~?}D}n#~%9Djw9a{|LQ^yHh-6F zu2Fd78V~Q96+LSuJ!=a+j|CIU&(>3 z-^l6f+#3j;)KuN4LtfN5!a&&42$d0g^AbouqIq$nCAd`Xt^(kTskkg zytME_UncZ2=Pxh3(2EPb%$F+)FZ7i{FW6Oy_iDk*9R7RZg}z4cqW9XQ_qq!2^@;a} z!b=T3XV|AFYk zt*4W3G2Uk?*t5y+czjyG{+4(=J}qdp`$mlM`1F0l;_+#r!pqtgk53CH%(c=(&n182 z@oBKh#pBZ=MGyYC9*#%Pn;y>r^x}e-HSC4L3%#V^<#_aB;e}pG@Q!vzY`j!>p_i`k zzMOcM5xlI`uM}SBWd$!bUrlcvlp>ja`JPelL+1Vr z>Gye#A@`o}oBsVkVC45hfzh8I35?$OxPpCB!9J^Ce*b7(zntP&mG9qdJ6OhAm22M{ zSFp`0*cKIRLnLF~_OL=m^@og$QhxKa*(6<7Dqf5Z)kMX>oHL8jSTY`nCBZ8Ue?NZEZ!vP@mP`D)G&{gx^0$t z4Vrj--Ehj@tV$d_A&)9aX`Ou3*O$jN@zU z;q5HG*u!H}ysl+ZzMLR<$>S`g9?-iAUh;Z<>dV~(FL~bG zu-FrORCxDHyn6{=`f%3bFZAAm7awLzy|zzz{-i*hv-aVp)U}; zqHjr;E$l}M=N!HJ6qo3~yS?V?_ z$@+P(%YwfzWU0^SB*vi;_G2N-oY^+Xo?qr4_@_b^pT;CvKPQiN zKd;Dcmt_5%J<5JrWG5@N^*8$LrgZO>%Q429-nbq|Y+S+yBV_Ln&e|cc*EZ6Bd~BcO z;(ZT0BrL|Tqrm8eohsPQ1&ezKVqdIZCr{Ah1#i3YdeZ;rvP;9O?;$$Z>YnE{b-vS* z662bXe0NFw;W@GB`*>%tv75=c9_ZbRp2l>;(OnNtYW~+ycRkp%$TCm&C^YNEv9q{> z-gOE8pDFt;=;I47wRTDR9seWNb_N@J7k^`H?FQ-oAA;Qqt)8XB_kD_NJ`#%j&{?7eo4FG`O1 zrW1Q2d{NJ0p;KYRr~ZCAI@PmS#HXHd!k)s2VPGW&^=uS66-Er|xhY~$&p#1E-}1YN z*t^U@_529-4itUHTBtB|9wg*gKMoeyE}eF1O)he_Yr&K~^kLNC`{k14$5hv$-F z{mIYCLJx8NAjNrt;HBReE4p%%#6pu)ihDvqL^ktzf5B zu+tOfzKn6jHqH)bLz^(eDeS8o4j5Bg!6f=Lwz6 zlk){euPJ*Y+^CI@NQ}9y-Mjg&gn2T_pbg}T0IeCe|sKccK z!`@{T>~euo_bUX3&MPa}RRSZ|>RA(xomUGveBtlt#&v}n{oUl+CGY-rXZk&}e}{dZ zONn}Rat0eq_3PF1@eKZLZ++48Ny40yH6}lc(srTH+q{q5Nb_Cf`Fxq`h` z!QQN3?^Lk&E7(T~^SG#Q&WmHz@LExOu9p_8n{C z^M?5v4*i1A!#sGgf~_moP5PDRT$=KjEx+D0+rv3QgU9Bd*iZ>n{m^ zz+Sca6not%;>gcDIL2{9$*~5$TqVrY5+ODXK*90%egEtcI>w=ehyp?$06uj)+ z-buV~3tnpSZsL8f@N%s2Fo^Bn30ssP?frdWgX3lP=L5mZ@$)0Y+ClG!f*0REPQ0H8 zUVQ&F@qQ+FIj-bfe_nXWiO&;wzZ80~`Bj5;_h{r#_04~bH^$wX*^|&+_hYXAD{SJ= z=gspt-9G+bh4<@ZbMlJKZ-gG^%C`;HT{~LkzR(yayptIgR=NJinciNY`edhikVS|0L!ti@$AvfL?+n7aQ^v0p3|6`r#O=S81}{pIbuJdk~5_fx$T5bayC|AMyELt!o} zIb#27<@l#`65kv%d$iqJ$3EOB`5E)MaRu8XVQzsrF`{6x55bmCYqH{HO+=5-}{db_4n(^^UlI1*!T)IAz>Z|W4DC) z_w>u|0%MLT3}5yTa^!SRfl<@)>N&=+*#FyAc*iE*ae|k;EL!}9-d^zH z!(wR;?oi=fBJu86c&TyB?|q`@$lv_}BYzJF3|}9tVE?LM4_B~93Pz8_{@=;si~YZI ziZ}N3go3F)-QBR*r+XM3`*hDDiw`l652ZfcOYoA{RZ^esEqKZE+o@0Y5xnGmU&CUb z?pNX6Kk*(Qc)fss#z zaZJBc$c--s?-CgM&t*m5at@>HA$PaX$-ea-fwA|(9(@?+!s{j%=K_Dz2i~C;54pJ4 zc;m6)zGO4b8~#=gdg8Ib|K3hK7Ccz=!0RdGc)Rt0-u3)p!OJ{(#4xWpG=Hxq>Lr$u zMKAPYLN7h~c+m^Z-=&Fq!JbUK{GFP}OW!?Rc%h%E`0#Af%ip7kdhy};#M}A%G_4lA zZBK9+XSRFKC*LYQ#I>KE{+rFeyx*obe_HmhQI7so7%|^5^`tuK>Dl^Cis8%b0^^!N z{<{~h1z+Y6a?Cv?M|`J=*wM+~REV(?-}i*Q(e9896$5hfR^&btHA3g6Vr^l)%Kv}N zeIRG2i}=8l963{RU`mdBZY2Dq?hA<6$@PK)BWGWiywK+!AK3pIth<&eUUIluxtEx{ zeJk{WP1fJ7bguzT-q7Fi`}e|24n~R`kh`ghp4d~!^%WSon?_(`J1(}SOISQU_D@*5 z$oSiUg0Tnk3Kn~yUF7orpqshE zoBs#hEP@wb*EeJCAM~urKR4g@7aOw)Y-|T=%`R+!&0$!~!JL8@`}4JX{n>stH=wVR z%vJQnJ{wc~ncwudTr=hDlh+*^e;m-+`|j*4Hs&!szW0T08|`}s){uFP_Pqo2d`1s5 zdMEqwF~8BiCxBkS=;e%N{w`>=?=QyMkB{#g9b@v`aBNtpf}PRs$i@CZZef#)dIl#w zF4>6N`~EXK)tggdSI>bUx46(rjOv*U)*1E8269UXo$P^@ELesx-$V3b$ni`_>~;LySlGi) zLPp+|RbZ z*cd4=Vo=XD(3k2t3giaPIXHJ$5VfW3&5*-aHZS4#`IReNW$% zd)I>8&Pgs}<15%M0;88t7k+}BAuwvatI$cU6-ErZ2{~%LyTGW~O_f?Jd&un}bW-a* z1vb``YD3v0hP{Lw*xnUvpMu5yA=fXZ>r-b{a=)*zLGBfXZ~F;3a=*X8$l<+}+$(#? z9Uyd)`vV0wwv+mnvPTRD2|2KXE7&0dqn97(zG0#`v;+yeEcn8ixwUJ*0-En6Gy~OO>)l9vCp5Sh;c-@(}W!LuF0J)mKwsD8Rh`si#!k|iCHm{t6+*z@AR|UJfg54u9=7)M7f%xtda$xsY zum=(r>-Vn;_D}_TIALBJj)yM_jJ{U)-4naQSQ}mvI*DD$fxRl^+U~54*968I{d&RF zn)0$?ZIkyOAFl{KV>?i5sY>6ed+f=Dk|P(>if7J|@DqDVZfqxIt#^uC z>>;ps4fA;j{a)dvZjQ0P|4?A`=|>53)A;s*z>xdrw9eVxXT;;+mIHgS=*!1K=U8XA zv8u=~zHqO8zYc59wf3LP(0i@@6SJ|j@sa;ejUH+=*VaEXdYM3fZuBstxwig=(eYaQ z0J|=@oZotsVdmi6|Aqows6HZwoQ8p|E}#XHjYod<=^2;?YV(t!~82#B_=p=R}2R4I{V{I5H zu(msEV}QVjeYA)Hd+NCcVi-_4o{bPXk(;Ub64%#u<0XD}=drSZZ-Yck)Me%hR*P3X ztARZwM-FEZF@eo$SnR{ul5Y{4UFaECax-?vPf$Jsr z3XJ1De@DsMcH+R7`-B{^D>-7nI{l4VSBy^ew}Xi90by@+@k+&j+=CUl6GZ)}d;Xu{ zZi>7pjMx=M?EY*`t~(gd*~DDa6FZBZ0DDNpM?S|3Ik1O?964Jw&7-!vUB(Y8*y0sz z$qKeq!rT=3+%^5Kqh0*)w;@7~eC}3}TUyAG&)q9>Lz7(eWmv-8KXR?E_j4cWlhZk8 z?^?iG_msfKIlGOAMQxc6PYXG)XDZlp73};nSDLml`%BpmzU(|h?|H;pmWy~Ev9{4e zji33mj?wWvVqK$$8P0jcdPc|di1m$L&T#s31Eb@4#D+$D4%L3-CwAXPJeF-#c;h@H z-&a@G1BJ04+F0nsS0x9wiI5xPq1aIWyM}(cUigVGn+l!8r{wTOVaRPI?%`yeR2VuH zMttgjpb+0S!XCaT40{S|H`#2pwh(zij{DeSPY|EFZyvefLMMK1DKOTU_CnI>|qfDbsr_5z)JM3T&J+ z*tkbv)GuG#a(=LTg&h5IpTLlNpn|cn5~7~U>SX<>is?Q%yzEj$1H4aV>xx2 zKH0X=+ZFj)3O{)sQ*iRIU79up@RoqH5t=Ca&mjJ(V(FxG7KzjesZO#LR+Nk1tJdkSm2DjV|% zUy!Rk`_UG1_%g4+h(SGb!1|*wbSex#6~ynR;PORh4`W{>?y46s%-2f zdV<&|>+c-B_S*iHjXjdwqMfgs*Of#*@g+Zp5am`Da`flk$(|b`cAl?bO^Z5zTG4rA z%5{|cS&>uw$4>0sd7OQVdVVhKlZ$;)Oi}KaLXJMzw<7l|A&0%GQaz)+UnjY^Ki)6d zi}xSyUoi6Ia`^eaVNO%0ZKn0A_3p`8E6*F$y?*3etBAkJL|-0|{E5CEn6S1hE90Q# ztAE`t*!%-~&j}8$@aF$5cZlG{*Qf1zs{03hXyJ{&_Zwb(SYhVQ2aY$dHrE2XefCO2 z_s+)=MfR|wmpR~HgB>pX13S{NcG2N)M+iNv^+#2(KUA=r#WiL29LEYd`sX--jVrz$ zFEI88tBT_e*aO z#Wf^stkmDN_QnME8-a~;b{lsHoz&$*;R`y|vm>nGV8rWtEn+y`dc^lGI z4ByT)%=bm`o|XE`_dH-{C(NLU`M;>dfeqhVA$N|+`QD1&IJf9w-g<4o&NFO?(e(8B zM*F^wd|gmvv18Cg-iyr#*M;nbf)_oP6kh0y1ur?ewD3-(1uwB)R(PQ=ukc=;c&`+^ zZg;%Y8H_hPL?(KipzMBO`A1~c^QaNfgRAAI(8G#Yw zunM+Z1^bDZr^Il*<>I^IyGCHtV%+?b$}vyx5OUbNRpbop9|>F1OmGe3pmaQU>*RXR zev|q_uJ;lcx!zk~6UZOXC>U4AR$LmBb?i5&~*k0~RSR26J?@pLQ*t@5K z%_NSw$o*5u;pZlz*Txlp?oDzrzWXZJjm<@a-h2-)Iv;4{y7_*v(B%96q}Rp3YR_CP zJezZN>onNOGJah4%dBf2$GC#6BrtNb@J!Cs)#+5*Uu--q{A5kwc_r*QKYRCW#k@f7 z5uuYleKcV)zBT7{3%!1#GuJoDoi|^voacq{P_pOWqw|Rh_GATns$lW>J+2sB%g&o( zT^<(urgq_du41G1{E&KHiI`R^YecLWxqrT5?}ZBXVg-9iV8r)|!0@y7jE;IH2s!`z zT)9Vx@70RE*DBcS73>XxZBgp?^sJNm!u(Lr=&+8xBXokjTfyEdSifT5_siqT=dbKb z-*d9Y+}*eLTH-xM4>f$N!v9aBea{SiuhH>Z;(bOhYdGiQ_ZuCrB|c!Z@2gp39yHqP z!kq9qFv`EOk_y29Hx@%Agc)X@FK z<}?W#>|hzYl$;|(A@59UI{*9mAT2EVOuKx@uG;@6V zN?f}q|Dq3jB+Sjz(>zZW_n6tg|3Ch0+U!SrkL_IJo}m)Qp2^o3$6g6r(&FIy!N7{n zy^~zjxo^TeH@o?ZjhQPt_p8XwQjt3#$wgldN|^gVo$|9pQSJ~SN6*Yz@#U~27j+(y zuqDlx(e99qxs%R_eJ^3o&#~^I%3gox3Z|YHA}{^Y{@Lw;&DQ*1>)sv&o2DZ7dO25( z>oIb3m`;xudRn1}`O;rt)J4Tap5AYJ6uo|fy(-S5!PN6g_&IErNp<3jl0)Y_m3lr^ z_5iGPqOQM08?4xL}y^`Yg0o^R5%rJ;s# z%>34{xL0VU9*TQ~$r5I2Cicbr$)C7anB3&zUSW!&2j5)}dxHf`&ys=`5#7Z^pUhL9pNqA?dhνgPV%4i|l}uMM5kC0o}Q`d80$5z|OvlU(t4JYtSn z3q}b&xY3D=@xqNUu5wpXo1V>E&nda#-VA&lC;BAD{NAVGW@pdT$6A$t)69)?tRrJH zo6cc#e8#yI=I|1NqZXDF9CLCh!7(3}E^x~fxaA7m@`9sg^5?564=}X@vc0>4u&+lH zxakG%*aCM#fjhatonGM1%s3B$vCU(5jylV^;ZCNuLYB`8!oIGUaZXY9>f8dS&Mj~o zig}HnibGGGclaD2|J%spvY#7xyyR(R5fkgfu=@90Hs_Vc3pZysN6pU?95Fvut!*~* z-8roRah;)m^^6tqD?QfA)r8;F{pQ&@*nOiv^;{ACts!iZ?==hDS_N+H0=JIfSbx?N z9Qj>eaI8NY2#)nz>aH@9Z)oNB@IFOw`@tX}-icP`N27$1#bC_^Pq_@e`X~5vVzcKeoPR# zX8aUKUbe~d5`9s6__?jnYAA?QxISpgIzlZ^w>UkCBfa1_o&q6TA6^G5g315hP zTQMJrLD@tPj_btu`Oe?1zxesiKT`k5jMq|n8WqcUB;c5_|$=|Y#BAD!v?wVmt2A0u=bgJUyYzxH!o z_~Qz?$7i~JZ5VY=5W0-fiJ9(;og2FFCkb7AIyuwz>(1f9=Ti#0r)Ii-T^e=&m2~6T zkM%)a`X6m=VV;G5B}W5#T-_t&&%y>9qpV~%eJ*KhAxNf z^Hyi=?OVNU!H9OwPkmy?HCZPvC~y~MTs#YRtBvXN%SD-<_s1FK;sSR`fxA?2%%hnq z@1fNf){JR$%*YY@6~ZQ;?_DXlX>Q7zvbjTpYP(A4QKN4b?#sVg=y7d&ZGpS4z+Epm zd|4pXPpk>n_XP#VJYPs~+{xM1f};l8vd`Sz3fwh9 zkNBR+o_%pW;(Jzb=)EgA^xn(3;bs%PwYqu_&G?+7KJO4?%R2pF;^G=jYVVOXkaPvjcuJ`4(%s@{X+W_a-YMx90mYF~6|0a2Ho>)`=sy^Toacr|z>QrdP#WfP1aLy)L+|-63o53XVJ}U+{BS zA-+Wl9QWB$YwiOvt{Qg$x!ChP!7j@mtFXM|aV9yU7*j$9K%oCE0HROhTYX80Cm zv2#n3r7mVK^r`N{MQ_epuel#~O?G0RW*Qj@p>7 z!2K%Yya)QjZ*myVDF^m;$yQ(JZ;-Ibx|7G?R;a!CgS--Bnb zziIkr)Gzs6Nc2I?%#p2au1Ae3ztI~~@QuGQz?rd+=!4o@IP=ZNki0EY;15p}BiLvJHt zlRDj4aMWqupXZbIXtwspePI(}lQZ=*VlBev?=!vF-!p~&HcNW3*2(7<#sobb8vD2gQqNyy z@p`SXuIwiI$L~A4c#o~;6RralM{VsFji|pbh@a}dAbe4u2VnDW zU1y9Bn=4v=^VqSeuGO)r{;nK0)x9d%RM*wm{Da6R^Fwje&m^J8{Fp2_>U`I%UOW@b z57i%fQ-n?C$DV>CFZ9Q{HNyJiI)1w8jW^6WfaegA9c>)B@aGW61lj!A!wG@o`9kcB z4V*s{ILfN z!Y(y^T(S#3KFP*4k@yt9smxL_-oSezNB4j!LpOVSG zCS;jEr)IKmCfOJ_v7KSEc`VHBywA5qpUm}tWqrOQWbv<0pVoZ7FJ$rWw9M`YLY6wp z^ZB8W#qQ~K+|9K@$c8Qd>ENs|D9EkUDISMPk(A>3t86gb84=6COo$w zd%kfMM(yi)LYBJ1F5~F;W99XLY;O00WH;6owR5596E3%Nk+6e57YjRZt7iGTMA%_o zUMlRsU2a_EVW)PPutV)$A?(0inc2BY$TIh5@6vk4zq%lMjd8wDpnGkSjd`b5zt8SJ z_mC=ov38y4qpBHx4{7t6?fhOMBw-vbC3)~%oWB%V&;O;JP_Y}B$1;;s{Tjde8p2>+%-A}`MqVCbz(M?%< zSoDY9fI%+N+8`$llAMQ@4uh~E1mhON?gzABz+qz>K?df5C{ z_yYG`f%~9f^TPu7k>Dn!7`Tro)&ViRX?k&9f0Eg8jydtSEEmqvJ{5X!pJm)o2>beZ z;?l=QZ6oYG^7MsqjydCfnJ~V8mGIwWzF*|1-?<6m1E=Wh0%w;_!$_3H-qI>P>=`;0B>j*0czgt7Hp z!qoHg3FFHP2@~r~)_08kCF4h#efXCR$Jk#n9AkggaE$#m!!h>P4aeBuFdSol({P-p zu~s^yywoBl6}g!ON6Z}s#~#;7aMbNA1#Z@i8|H4OHmR$B(_Cw!pOXt*_spj2aX#*n zaoz*mBV8Qnd&b|apZU}r_esV1OTupVG~a2VRL}co0J?@ z9JyFg_(^|CkFj4V>BSm?TiH0Dhu~GRKAdBHUQKY+>FNb;jg0e9@q1OlVY7dCmu%Ko z^kq$xTmgfeB4tvFdXA`#Mo~rxJfQxP2Fchyy|!OQHLr|aGQ(%u$k+{`q?7Wi`d{lB+I2l%U=d)pSqc)J3(ea3kY?A1F6j#```=JO<1wx&47 zX-8p`8diF6I}1I=a~Hu?wI}v-LBX*mtsr8cKXpG9F|1h|Z;pd^H7WJCw6KZZuE`hb z*;T5f*YLAh&#Di6+fBs8xa?lwO7X5J{H8ypM-C^7nBev>F0M6y$b5_3Bw>f~&FAlw z%#NoJU;k9#{+w|`Q||rCof+rvVe>B;=YF$4{WasF%|V50_m17Xht?XZ{4HzyW;Xpj ze97-;a&D5pSX1{;q1U&+OSJY!&Fq);7h@Ql#o$&df3fzMxF3VO{Mg4On*DjsXn!m8 z_xFr*ohm8z5ILjP{*mz%9?#13oTHA<&bT-?&Pg2C zOs>Zmyfd(oi>cnNwRa7BE|_!g8P0vJ{4Hy{RQ}qWJB})Uv35b`kH1Hs+$S@WznR*k z^i|!LgWiS0CVpNdIDSsrPwW|crvCa1j=Cyc7tJN~rX`z7kF!c8^oVbu;C3qbiQeFX zp5K3yj}bQa6MBq8eol8QGMj<=?LSfldBTuI_bzHPoA$%JbAb#Vb=6(6J`y+&hSXH55C@T zT+?qb9M|+44aYV8Cc|+}zu9nH({C{x*YsNr$2I*n!}&ZTmfI7j_WV1H8*ZA&s`Fjt zrmgwkoy_}S9^57Rr@l@RH9N^oSyLQ2yj$30Je3~Yy+V&O^8JD%_H%@vRqdty?WuyE z-FeI zd1=;t?pw68gy^4KJXq*|DWOMgJyg(Jrl6OPnTLZf+=IoqL@w8h@hvOzL~X5NT%2c* zWPW=e$Ud65m~XhpjEi<2PqMcB)op~mvSvMD+(^UJ&69@Xn)H<6xF$VqIIc<07>;Yw zvxeiEw4SX8aZP&8__!uLZ#d42g&n5$LdJPav==k3g0QbI)ttV^bzA$rA};Iyy>Typ z%wttEGyV>~ms6igc7@K3oF~<`6%EI@95FtxXTCef^8@b|xc3U&`+}ptjrw~Jt?LWp z|6id8_d$XCu)uv(;64@{e(pYKMnB>9C~)^y?du&kxP5Mtu*p17_i!=yKNm5;eNo`P zEO4)8YqHlm<8ZS0ndep>mbKgK1*mmwsimtbmuThi)Bhf9dmQ&ab##%`woYGvsAFiJ z3%^g8Gt!R` zX4GR0o)vkHay?Q5I{59$q*(G{6_Dpn_H0<*)%3`OF+3|kCeT7}}&@ah?#|yi}+CRyH z2MD`xb7it~3t93qPm%@CD`c@dUuO4L1=%W-9hhX**i5xV#<7`ZcvQ&Y-gTmz>s_qv zLCG$)6S;wX8sikXJ^d|fgM}WoJb!^(pujDdaYGwj)vr%=^DnJ2u)Z#o>A581p?)V1 z+^OR126sgLVQ;fPw@FQ|F4iCP&JdsP<7dvrT<3o$H^x3Ri!Yv?|CoNqeiwJpT0HaL zJxA-Azq{d4CNrg$Y-{Ubc(i@b=O8@|`?><|WjLPsdmHxkVsg@7#pau@S;XGQ_;}{; zYuLw3`SwYwS$z4(iY)r}vEk_3Cx)YMpBj$7eP%fN_POEc+ZTqTZ(kaA-|Atu?>ukB z@s)9o$-~zPGmhUROdh^Xm{=C;)tdM3jE}K>Z#c&GgW(w4kA`DxKN*g({cJeK)@Hek zvCWj`yUP0>SuYB>5ftKsNdXT$B)M4!yJ zE+!j&o6T_aZFa-aw>b<)-{v$Ned}sC`qs^G^sT$$_G;n~yJn9y(ZggNQxk_KezKde zc38sX`C(fZV%`roKE`&0;TYSIhGT3;8IG|{HymR-+Hj2R7{f8PV-2^L_rY1-ds=MK zw_b*$Z@mpi-})GizV$U6ed}j9`qtlY^lgA)_pSco-&)(X@ulq>J6rlX3H~;nHPyWB zXX{NIqy7mqMgtONjOI$1F`C(9w+&?U zZ63qXw|Nam-{vzMefyQ+=v!qt=53(i=-VK}?fLeM?bR`Fm)bmtzFlTG`gXbD=-U;B zqikY@e z-C#KScBA3;e7nuK=-cgvqi=T@j=tS#IQn*%;pp4lhNExmTI|ucdyJ31-D}u=v(l=Y z&+Dzz_sd$`j`_@>z5CkK8ZpfJjcdeP$rfwGB8i`z`g%L@QxaY@8?(g>$Gj|VIOb)z z;h2{ZhGSkv8jg7xWjN+#wBeYSF^1b4v-2(BIA#|Zj=o)JIQn*x;pp4NhNEwn7>>S8 z?%nF!rN&3!E;H=Sjwf}i?A8B!dt980xt<@{%OzUR+HkL?c(Lh{aBtKHoTgsoWNJOS z`!`K4?jfVH{+!z>>5VRMV-ja2Q8%xnUDNlER*l?X=P|?HKeC>8u2Ua3&hw7!6NaO& zo>RCdGtMQc)v=jBK6dCmWqLk#oN1pn9Q|=L)nBZQGdrFG@bi_w+w|QL*FNKuEcnIB z*;ZNNoRDO}F9})vT_VYXUn$5gnaRE;WQlvJBny5+$kON1nccSvvdd($?yvR>LHIx&@w^9Xt((t% z_+R^U-M#$9+KQsjy&F{9tUdgj=KOSoKk9jeDK21bWnq(C^1B%07_e5XBJ|*XojB&I z_XD@8ah1Q3+Le`mqxU85Gay&oM}Y5Z2p@S!IoF~8Rp9CK^E0=IsF+pxg-a}F^tj;OEID@VY}{+{Xi+r-_m{yjC#{vyY70L1JZ`S?d|vw412X6!V38Jwo`ySRE-i*1Mww$9gxtz#U!SjxBJ<368NizQElh zVn^@9OfSxvpVPh*^UL0LO0Czd*^@InUTxG+eviGMOW^j;^y0i+rMp|vKU=%P^eW^0 zzc+ri#{Bm8i1Es-kH}qB;I0-NHFKTdsF~{pN6p+IIBKS6H}|Y{j>6qo(7UO?-7Glr za!Y}Gr%*HMK6-54Dr_>==ceb);uyhUlUnl{VeZ~$HvO8IT-=`eKz6io)m@pLc>Q#@apTMmYyCY*mi6506z*Q*;vRON;Zep@5BDcs{59&Az4HCut=C@v zHJtBv*!hpy@qWP%2)pFr!6XZQSJ)+=4<%Xf!@@4yBbn@@LYBNdmSn+?3t8+wk=cE+ zAp2A%`*f0xbwxdvo;Uc74Q3p--x?kj^1D$z*v`SxP^H_flvSs>Q zE7k)0@SyX{d+R4P z)Y=q>&F6(L)Xv2{-9T$Dh(Xyz4~{&=b>#)?FRm*urv8x~Z9Ou7UNSDOD=%lh#&zYD z%#M3bZf5P%+_U_Bu0`lGu0gMwowx?Qmh5(NMQc9B#QC~$j>*Y~R)dbo`$vXjt-oRN zqk{aK$u{eoQNOHFZy8tl8>#sUqeMd-xa##{Jl)qud!Sg{(Yg# zdh*{)*RR1`7yg5S?uVJKU!z6ckAyDc@NuU5rd|8EF8n7#7oR@Obp84-_WM~u_w!8G zuMeZ{7fDyGkzW?LuL|4=f+Nna3mne}#9E_XzNz(^*GAta%o?$_T?6Fn0CS7y0GLZI z$$9mEvBP!LcfxnBqrNY2{ySOQ^G)4HPE0=to9tiD+BoJm;eOY>eeUOi&9>eRy_hF# z&SYHWbZSqfpSMO?^8KOtRv~C#*H8Oplx2@V{x zliju;`&lNtLqYb7Om^3T?3bDB#3akS92)ZZcy>>z&u7}qX$;X@7{b1GtT}!Dc-_{c zyq>80^}Ec-P4Ty^om%;8>;B7JKThb?;-)bu^V6+xuAfbC)YhZ{F4@{&339j-^tyQ(n=DJqDS2HuNGjXjP zocR*h%K0iBxeL~m9_#iPp@+@0#XKXv zu|jXBMt5y02DpCWYyrnT*z8Mgj(WLC%un)C;wA_`$&1p1Q+njg?@7=BwnBkhG2^^n*3FeO zu5vQ9YwNlL`Pb$e8n3~*hZtXeo!N}n)vG6tT#PlV=&fN~yjET_$<`%ja-Iw7el6oh z8s==ew&78RIe)HG(0#pAtL``EOw|3hc23_{Jij^Pu~YL4ButGgm@wnJP{LD8$J^FD z8-0U^81H?9hZ-Jj82bww9ux2|!=5wjFJjnp242+gc*EprF~gpJ{9D{`97jjgSN`*s zZ__&7#f7YS+??5evUSz-kKaEVj&ZbE?Zh}{G92URU^vDxv*8%WfpfOT(b4!AM<>Ji z+^_sCYya%+ug$&C>xLRTNN~h-aDh7{B#L<@5{}+DOr5`W0Ga9JyzI- zJFe!M=a%DzEH#8IYvb5n?qM@Oj#zI`Om^bBiQY+qW8FMiaMawX1&(`2V(!?3PcuCq zJMihrPLw5gcbHrp=cnqTZN?bwoFV*SEb%ACUzu%hv#0CmdI~k(YOB)MC2d}N+q?0l zZ6(WbW#9XMZ%fsD(;t72u{=xo19x_TJ2&HE{O1?A3k1iy>7O~q+Hjk+i!wcbpS)cx zIOdh&i0=}i$9P{VIBIdNLJh0_(7R07q=qjS95tNx7xQvOrWenqS7uzy+2Mt=>6Nzr z#c^;%Uak@xbNy<;5yLftW8N!0a(1oI!)9;6Q4`k*j`^x$K=1m3-nL@gsihl)9(mcQ zA@*Ia<;YblRCRm_(?u*N}O8bUbPtF9KWtmTPGH3>t@kEwRMZ&$kX;B zhxmP~&?8Suk38*Au(_k)sFB-5f7F(W0lnJ`dV7m8qqcSyzL1xl3Y^j-FG>$i>5;Rk z!cS`J4iO(?bZ6q&zr2>ni$5zl(lF=gBMpx-%)W3`X4iG$rgt_F&J%HMyg94UxHjIJ zxLE7d;qAt`9q=8Q9q$p}ZxdYQ0@fA~pM6&~V`svijc?eq|0(jy_}?Y^fV;cE-6Ock zss9i9_&54YaG%g4-unf|UaL6#{EyHhzYhqGdOKX?g*sLJq4%J$Nu53@4@q>x|qq{qivPGxxcG_(;Syr>yKsqJIA{7c*ezAe(rB?Ke$!JJYp@{Tj`bEJpM#?6+eo5$&&(*rmn6t{? zvi5SOSJ`|1-`lGh=kHOUpBK(puL(WkdtGqU=Np0}hBpOAUX&iS`{92weoEy`<{>`@84yz9|&3MCAa&Lkflbx z&+L9IWbxs{0j=ZonUE#FKV){l5VFj*JZE1DS?vC3cD=E7|5W4FxwFzNnop!VX++XEtF6e`XhU;QD6yn?u-P?#(Iez;!dO3NdsQcBtL%!VX-I%uY{X zhwYYB~xK^o;0ZA|R56ACPW!+Tw zoe}@WX&;VuxV}EYo>6jN!E*~=u`^GBo43HtC*p`_JYw>&#|`$^=K0foqg7cSLm}+z zz>IUM@)v89(sNiU{%&^8it%5WenuK)S#P>0S@1Ol*?yVqE!odr-7jk3R&kbP-mPWx z&ht?Zp?xAgp@v5Vys+WXhOs})@R)!X zF+A2V_7^oAf6lg;;qk^3=i-JZ7{7Td!X7cgxaiwR!_l` zd>vyr`Zm^Z^lhBs=-YV1(YFbP-M4Q3#oCa@m-=-BdyV7Ju~X|fbV`^wXGxgYW=)v9 zbxxRZ=wdj=KAYhf`|O5e>~k0%V?Gi4oQ7lUT@A7>?t6hMmRZ_#SI~ z9N*&%=Xs;{dWq{!@^-$^gS()>T`0J%+y!ghdirbY7_t{lPV2T>`{2E${_9%FE$;W!328IEIcv*9=fw-}CNaI0aD zTh+(+16qCCtbeO-n;VY4ZPc~Zw~Y-)-?lLM=-U}qf6=!sjgP)#hs` zq0lA1M>Ac&m)&*YKQ8D#mg)Mv?NRqrp-Ubg&vehX&s|&>{&S&=PfujJexH2o_sfFr zlbNpHHy?GsPP#LhFEqwrRBwN$bq$)7gcQeIc*^?B^@vmH!96YX*f)lV^_TjdQ1DZ6 z`1y?J4_~gfJhd0YIME+DQ}A~fCK7O=kvl!yoJ|{Tl$nyp6g^U|UXZE#+ z*q5k@7c)IqBZfQE=OD4pxb_{_%_UpwY)^m7+IYdS*Gv!`HL--?rZr5qDZRZCx1`X+ z=CrOZ)aobkEhY4rzbXdwmM-YMB=U*P4MYtx4$BCeO$!;_A3iLZ0_6DJ<@Y&6`@C-R1D)3;Ruy{WgQoc&q&NxiHq;$w`~OPsof+2 zhQ}MvIPPIMuCsqI9M_hi4pW;cdnVA=+=+?%$o!6a&Jpv+1@4oKb1RI`r-I`=c!U@` zYD{tD^D|+STqr%bFN7ZF>n{aYc_7w47aVyxLBv3R>YjIE_(oiJ5c|=>CVF2bU*aBG z)n4Lfv+h(M`1ZAkiE;U+z?I@XQTR=NN{<|VD`JBC&bU~I-)Fu>?gwFq+8$^J?^x?U z30Z0({~OqCeLcZ?ukKvnx@26Ohri7@?}1#+Cb+F!z}i+_{k7>!jA`~vFLHBaT=eD3 z^fP*{OAknsmcQfezv=IV zP$MzFeMAgZt<~1nS8(LMU*ck(;QAZqzJUj1c47|yI?y|7`t68#2WEPa8&u#1CocNd z-Wc9#W7=N-^VfEo_tCgy(HX>Drn37h_m5>(8xF z4@(uer3>6L1y0@9NNuWnC8@tFdXY|O~YP)UHpr+__@o*-CKQ|t6QsYa~qDn z&0{$FHm~8aW)t7$GaP*z*1y%aUl|{Ls|?4v;l7c#wUXb|l}j>SZ}fK5x~?+Lio@pG z!Y1{o^x)PJdd#;^3!i_hdn)l$arn8e=nr3R&gPqE0`47=7jmZV{e)A#U{mSADLvxb zLF_$n>x=l{HV_Mzk*9A(e(_szjOQl8 zCOK4kaGMG}>TWZ^5qsmlz1I0rrTBhV(DQrm;=GDKU$~@eE9VG5xi65p8ucy}dgNtu z;S2fXo=au(@`BARgiZV`*}O8@jI~vHkk%SGYpzYX-!a~+gm3h}W#(JdyGH0ycUu+o zt}E!}-^k^9(a-Bef7skQ^E2AqMsSr2Si7&MzqZ!v?*5jwZIfOc7r5<= zi*tT^AgYdrhnZieH& zxPYCt;-0;`@o`_AXgJRYIi4-;nK2&|UEJEBs`VOf`XGO|HLuwDL!pmJ1#Yt7w(@SQ zEtp*+SM|G%mnpSg^Lez$ooT=^@2Tg}rWx<;ggN}RHKx7%Eo&zxj@&%;W-(F=oqfMy?q4np58`t+Ia- zvgCexCfoP{WUF0re{?3hzpzUlR_)U2!#@hL$7FU76td+08~gmeA_?~O%ychCt&-ly zhhsClhY7pL9+$};S&;p+&2g_^bf>3rvykf+uWt^r`i<-OnI;?8@v{uab^L6@aUDO$ z@L03TI)1L>T(YdHFLpW(5=xBCr8-~MAb`u2d~=-Y#a z^EICN^^D+{PtOW&qMNezoZzUT=L_5mf@8d26dX2RDsV3+j(HRJp2Y{X?mfc|doGY2 zYluEZ8s~Y%pHZ2tYm%E+tk1aaj~BA!>SNwq8Szx8aVaIKer!S22oPt*uvc$1Y zk_E3TWbtRcBnw`@AiF^(yJ3=5<2qA6_rCR<#kh759OF8(;P9`~?r;kn3 zSR?By#Ji<&ac$U2$P)ioNfx}dki~~>k}P=Jg6wvg?Dk1k)#m&nC&aRV;P7=p!QtCN z1#XDo$m>wSVRPXEH!N`-(%8i{XA$G#nzLvo8~2OFjEmPQiziv~7T2fYnVqrLH@=QA zZk%D_8kxzun^XP8TCV}EYuQ*K%Xo}4F4op~Ap(&uuS-Q^3iD`c`OCRz0}huiGuEnW|NzHocOT#ww5Fz3=c6P{9!PFveu zhU2++HR~&$-|jX(p5N{<9M5m}8ur@2{(XkW1-!c1i|4mB411m7-CLi-Y zx5>xY)-m~*_j!zud7syCjP3Y7t+CB#e2nb`*h=JZIJ0k-v%3wzRhnq`nG`K z=-YyZqi+itj(HzqIQll!u=~cE>FX73uFAjD*Ko^sZS{Kv!_n^*4M)GvvD%A%pKCb! zy^_gCzqhh6h<>kZeDr%2!_jX?yZMW?bxpTDzu(I8JN53C69Q|I@ z>RzX*eJ6ZT&54zqPp9<2EXA z8)ux?KA)Lxl5yU{RDZFyso>c2e=oR+4Yjt-1jl~*LJt>e&O=9Vn-}!9C~*Gls@vq8 ze6FKumy8TL9MKZh4|{n`4cdv2z? zWA?iNe2r(kZnob)<@JM)6H`CL?zPD{>|vbOCbEA>vaEq#cW{$3&ZuAZ`pH=zUf<|V zF>ah;^1Ek}RXID;es&P!-pgcT&ZZ_^VsRg)`ir$`#(CX>x3#g#$DG{non*oL7|vzM z(I1m6cynQwSpSq{!G9KZ;r^1z{#D45vwf2+ct0VF-TgDWe=ErTJ(K-Ml4bloE37RC z6u9fVHS(wQ{+a3d813%j)`s@-*VefX=g-B)F^M&`i`_>y&RFuXYr3yY)zq=pf2^t9 zOg7fk?nzhG)N!U8YibW6OAQ^LWWke!EH!jOk_F#sIL7LT@jNlfg7*}5;ZDkAPZoB` z*(pgDe5#Pe?!Pj-rxj#R&t&%z{!aA(txZnrDfm2LmwcX|?1C>4vgGsao{ez>UnpdW z?V@BCd~recl1%neAxl0lOS0h0g)Dtuk=ebnAbV9Ndv%g!?s>hEqicj6=GUoqfl>J& zr1nJmISzBk-=3PdYt2rTzV|OLm{6P-|Ee!mntu!3W$@v;O!wua3xAr>W!$dMbYDxl z@TUu1a(Y9i`)1OGKck@gZ{sQtBDHri-7|$Q<9t@8`+m}eKU?VH(>a;$hgrYp7IbgS z;{7DkJum4}Cl$fj*PBe&8*5-yqsGgb1|0i;Z%>N*)rsx4+KxVF9X6<;&)6wh@^E;P zozzfiyF=K8yDRy=XOg{J$ZnrFWcNy#J%*h4+tU(vPqGs^^zIeh4oUAm!7=ClBRFz* zp8ZU-y}eL9m%n%FFaKLeJ}+tCrn3q$I7j{;%zSYUKOZV^4`-bBMGPYPHGwQ*iXXRWsKkIItzVBz(Vvcu9xPQV^5+2vxziIV{_SvsfRAOFjR`n)0&nojHC|HjlkWY^vwl@a0uulk@y*1@83% z_eOzxM{ummEA)4-TlWUI6^-*T2J_5UoPS*3-eh&(RbUdaMopK6C2pnrqIXZxW_%zsmY_UAX-DyvoVcz7=}Rp(*LP<5(xeIG25%5@S}+ z!C>c&?EXgY8aqSk8rSEj>6WX?Wc|Olr;?rM5Bu2vvZgS0KZ=;)ekyQ33vOz|wYCmv zu5xbhx0wqw3y!(aQE<$KU3$8e*7}3%RM4BHz|AT+V((nwMil14{k>=O6Pt4+n`+Fu z)?Bkr=M?=@!}Nij`1xHo(;IItkelww4zi<-Bi}uY8xv%ECRxU9Lg0F3cE$#-w{hbP zvmf?Jvdmqd8*qJ%i~D*%!}0T!{z(^qP4$;Ez<+EXb6eEe1BUZGfjS#tc3c)bm#|A7 z=1#KU`-ENcIZu)W&nxW0&6mmkO30FzD#?Ne3R&z9%Ipp<$j+b1E|6pyW6vP<*(t3l zYTOR8^)HUw!G`0w9g@~ZWp`u`@1;3+D~G9_ntg`nW5zzdO*ZG9W8Yq|jzh1No$NEC z%4Ge&w~pyEBz&vtcN;Ii?(SckzBwWvLkqcDy`ZG0ux5JIKbl zyQpz-?k<+~8RzcenVmRyhZ`5??uaC-YG9;su?9vNjx{hk>Ef^FlDYbNHh13;bBwt< z#_af5gU1TH%+*(tUGS^IF8Lgn?1IM&yKoaS*(HQ5b9Koi3tmddVt47x?lJ}0Wi#33 zk}P!`bF_TMdG7J&&)H`;&T-~Y&n)ik0@m`sWf7m@d7__7w9Xs)+o<5n#)4zMtz6(% zDR7@9U%I$U)_hJ;4~N>?<2B3tJIt`xEckH4KCia7|K5%;?DGnIq+y>glby77lwqGQ z;OU0P8z%On4NovkevdKSUY>dmY@g#9!j3MI+ScjoiuHQdoFBigbJIKzua+<|t}1ee ze>^+LeD^N$emGbDFLt>eU0w7^U9C~z{BJkp`NCKAycsdBDQr?#YZbT?#Q31MPC;+o z#KpA^z4eT%JdD()i)(uJu{FfcH;8wF?J*Tb?CSyI8ie(AiA?qgAlMAxTWwNIiWS7rm&rGtcnVwF@$*%XgXVIYO4T^!rTqJRwWWKV-5O3fXDN$K3A4LY8&u$IR{}LKYvUS>9tFE*G-o z^QX-2l|q(zlIQFyA&cFaI=A|;ji@2)`m=UkLvc*uI%amZ6S9+1K4;BjcMyFtrrQfU zaJiiwg`H`sPh|%#x3iP5!}$7hjCl^=X3OH*McCOP`LnCA12;!zXE$Mo8roghf$N&t znJ8o#zj?bh&cxt92wC=&`KyL3cv3-jzGN5K$w@Y@m+S?<6@Qx!e>N&`8z+vs9AP=8 zzhBSZ`gz5}*-z%3aea#QeW7u&zAs9$s=hDI?8N%M#JE`BmnPY|%&m8?QDVN# zxOh!}x#4(Ce?`*m;BKeJ_*`XNtoN&hEaNrT;D#*t8X-%ruT6Hr*A-;1&tz{%vgFKN zrq+%VV@NE=3l3jT5FEapSl~_)9C<%kaM(Ph!1*)3v3_E$-Da_mHx@g$XL*aYc877X z*6z&m7T40dGCQ%>?lvyg+C52D)!Mzr#ag@1aICfalWqqOF*U~L0pnt=Jt$-uuU}=g z_K=VzM-L~v;71Cwk7lxuC0SK#7l<(=mJ0=k?-vOU-!3k2mk5qpyHs%4ysW@op12Na zEMl!aZLyEH_p$R#$_28q)}A#k*4lGPmb}GUdp@%hYwZQ&Vy(THWL2%bWL&JZmkq~S zdnM_1pwR3q^n)8oVnYK@3mqK@lSD_ z^RE+nd!~F`FSxxD_jF(XruABNTH;iH=-nV}a;CmfaGaA>f8@}gGtYhD`r}Sv6F(J) zpLYvA{QN}pho7GcjxoBXn$b_jNO6qOeZm*U=zhTw!&`+hQvIR#A7PW&9}pbzJt#Qh zQylR{yKQK*5dy7y6KI#_o$IKk{x8@{`jVGV}k5kNp>bTYAx=MZ)bMK8c!eZ z7&p!^`F}T)bz4*Y#o7+pb?pa2mi_QUP_Wz2{>gjJI!7&cA3tt!q#W4-Qo^uB+&puk;J;4Ud}mlwDz3*1!& z?%D!(eSy27!2KkC9?p8;zdxeAoV9grpPM=3T#_?D&w|ZPLT{QYTPx|!D)gu&|J@qx z#n46QQD?midb10?-Mt%YeF}PW)_VFroinS(&o=B!-Ojgbspz*O`tawCa*lbtY~i!# z~W>DvbEO=2NOMMMavf#xEvh!!Mix*@UwmkbD zjosl%R$Uv9F>Yjtbvw&Lyf)rmcN&iyA z;{v|S@OZ=2$>oM87^cRrFdS>VGUWfgeJJ)u_NrA_;gyI;ZZ^YS;*uNDVv42MwISz~0W@6xlzyVrJG_bc++v8xYSD~HXALJyALRiSEe z5243c-7nUC#^Vn{kDPxhuHmuync&FbPNIgXw^w`47(01U9I?M9^zieI0{6M_lR4wR zQz9P+IDWrG%=IJ}xAu8ZKBu$^JNF}H`+Sl=_}_vmhW{iu&UA+eo0ThDJ6Le|sn!|#Q)?G_Q+mYELF|?2 zsWrK(y~NMvb)2$;Z-a zX>xJwjxTU0WL%uzCnhe&#QJiwaoz{`l+2F%O}^Fh|CI|^yGrD(s+s2B+}%6jSH#}V zd^}b35BIMEcUt0NjMV0t#(9k3sa@TXW{w@fy_|jC8@X2mhi_*K-^klJf}7}uttk$h z=L$W>={&))54C6TyrgQ74*9HZC#5l z&g{f_aY=!@TyU(*R|t+exU#@qm2q*dU7fgkYqWTVcu=Vgx8n3Ql)AWYwgzTDU!Zpb zlh*|krdIAx@_a5hB=PKX6MM9N?)ZSo`{%Ln4;mhAnEoCzJSO0W4g1=R{YMP@+6;cw zu&?>}|Cr$khVl1t!*O24F}lTkjoiWxQ@b|Hr@u{&Tvy<(FK{;qj;H=lKOc+1WMpx%rZ8EQV)&x+b-_ zZ1;wpdbs>sYvk>A5kF&bhv1kWcM6Vt-z7N4Q0cM8-YxX7xnUu`dxRb}t73qgyT50m zb$sFY{RA;*%)77CxI~V;l(_qapX5d9!6`j*=D)8Yj{)uh!7=U+7PyB5$2dG(;2z1i zxK=!tIO=|cdA*yvZf*Wq8f!ai@p-fQyRGB0t-odM$*d25uksgbPt}~>Gke&kQ<;qa z_x51rudRJBUw&JdFVBennJ>=@j=FeGaOCuP!7=tqkMVs$=wWla!hCsA=u!771~~sc z0C~>f_?-Z1PW0$9qaJyAP1q!Mr3a_<$cz6TgxpWKHw4F=c(cI0B{*{Zc7b~*<9trA zN4}eJ?ltF$r^R_1?wJC&L3Z8eHd(j+EBb@`AaTr3*Q2gKGS2-5e=KC#vre#U9hU{K zklh>L95wTa@P|40q2Mah_4cXgzp9z$-+iNuaOI6Yveb=4(sqkwZYbW zz`bA4+ap_-hI(+ReJk{sx7V3(dH(lKdR@h5Sj6&cdM|;_}?Zt@_l=OyQ9E; zFXF>bp3PITFIO^v2LXW*w=`jyC6MF2Qn+uLP)A*dfb-los#%J}d9OnbY z;iuy8)1N8MbB3QY3yyi%QE=qCliy^zw-!=e6Bxm#xWwFiX)$Wg)ii@pWuk$utGjnf9UlWHi>=zPR9sZ@A!yPjSRILg*3SNWtOf`GxpYf9Q=8Hi>Vv z;22xgA2yd2de~e*)C4vcOk5XVO04tj^Fs~a9|n7w8V!(#$o z)9_fsoXOWR96$42+wgef8NYQ5PcV#c>l*ewD4rcEf6LkjgBp8$1GTlSmt;Br`TJEO z*B2aps^>PT3-z2RdK(CvoVhkEaJPt>p+Ciu``-wg_@eaSek=4iZ@t&o6V!TcAoq6( zKk;R#s3*9MM1T15k&Rj2A30O~p;zM6GoJ7;?SHZ`xFyJW7Y=RDEdMcCXi^|!0w z*mI7nT&Ou$9AWeL%26xFS~sZQj2wA+U)=LYf5XK7M}J=y_7!ClJ;l|on~NvZ-?YYn zUg^2g-xqv2QP`xv-9@hHFMpQP-23J8@!8X7GI5PWZ=$ft*zO@X#-XRMNi8XkG5UkB zi7(xSO?*){(Ni2}W$w|8vEwKAWJZp4Lg}#<^bns1G2bT@;v1H&pFT$DZCoGpTWg7H zolOMCI`})mF-DsTj`)-w@%>)tVY63n_qx?j#$hv|x08!oQ!${oc|mWA$R{?3h5(&~2dDJN=K{h{#(ity3%TAVaUB{dtY29p@3FkZ&r+vmd8$m>|9ex< z#*&LwMNJZu;)rQAp@-kQikNn4xY9OGM2gVKXjdc?Q3@RKoH zL-tuRWirc^JU5MfL!X|zy4nH>&dcVvo!4bn2f+L141&7UpMGSC<6u3h(&TE%7 z`ml`iTwzl^he^GtXFGRvWow5Eo9L;UpugQk?ZTm_=Hsp1Tj%57($8Sy-olt2DSX54 z4YT_3SnykM?5jr!oA|Bt;Eoo0m3M0G%Is&{{vPYkF+z{}X*}!Kx<^#5Y;Af$Z>vJ= z$|fxfvPsU46~3Uy^OJG>&^s>a#X6|!=!mcR*_`SFzmFIF!)+>ZL5-Xs^x#yy z*i_F>!l75+W?`|6u>Wv#*m>14>+LBvM;tTWb0(h86HiS%`Px7Gyz)fzKYm_$lHt+8 z{>g^p=ar`z9&7wB_6^6+EB|G9yz%&Zn&An-ztau-nWdiE_MQ9A+OwtA_(;R}`c{ut z-~MjDV?X-#55v*7w@p6!_KxA`+j$my^zB{aqi^pS&d*cibvIE1)TiQz;fxfos?WxA zIL&yGjde~Q|DEhajygM2aOmwW{KmJl3VLt$oG};lH%Zu}znt^Eg|qW{R-1W#$suQa zm8Y|7Kbv{l($@Ff4*hQ>IQ-r^l{1N%#PPNIoLbdof&2OIyHTNuEkk} z_xS{qIk>F8sbim-sHobp7v9xGwzu1>FlX zUH|(OQTOjcmpopS>H6o5t_y#F(8Z^VGhP2%7qQ<13%ZwNy8gE>qVB;-mzpw{e(~v8 zs~evi9sEmbjL)TJ*BkTyr(TX)&jy@}&gecPx3|A#jc2Z7|J1E|W{&O&@CC!8gZ&o`$FtT;hT~c5Wy8J) z(cdeE<5}xf!@l?7-)n~RdHYLF5?2qCNeyw))V}M|h#T4O6DAKoBuqa)Cd}CXlrVn& zoG^KK!}^YSnaQ51ih1c^IOb($!!a)%4adB6G92?Vi{Y4;Sq;a$bT%CG64#Ko3VF$& zZ5v?|Ki0>{gvraOgvrb3gvra8gvra9wzkE*j59vwWxU~-mkEYrUY0N%^RlGjn3tst z$Gj|UIOb&;!=4va6Ftp0kCmD@zHe*nCnQYlCniknxA$qt6Z=VtC-!~qIwQt@hsnp- z?=&1^zsv9#^M{(e+wfS!*uTecjQw82G4}fm$Jp;T++Iz*oy{Y!7vg$HaGYu1EpYE; z+)(d6wNEPlrn%on?&AXYL1xpfFkT-PxQ`0l#|2J3k2lFXwWgl&W6k{}*;H%pr^dxO z^;yEy)#nK_&R--<9ersy*3nmnV;y~MIM&fOhR2w{)X}$wV;y~GIM&hkhGQN5U^v#% zkA{8hY{{#e$3m?f*Q?dHJ#9`!-}W*beVb}H`ZmpQ^lfj$(YM)b4n^PgF+TeCN5k%0 zJtZ<0v)K&Cc{aP@IM3!V9Ov1b zhTB^MZnn6iZwK~l_3a?T(YJ#QN8b)H9DO^~aP)2dYz;Wf_~_f=hGQK#V!YHdij!Qx zn&QZ3*A#DDE2vSW2iHyLF|RiiwZr(U=Lz9n6LXLllpdVYBfcKOPhxmU^auB{;K;S| z1y1>bO{E8y>kYSY+p``$`~Ao%20pKH4!vRN?8Q7@#IVm7{8`j+UNe=yWo_rmUt8B- z?Di_;t+(K6Q)cb=eP^`E`1VP9ta&a8*Vj0Y3*1l0QeQh*A1({-ovsnpzCFO~j5Jy5 zVJ^dQJ(=5ZoU`*7j&pWi!*R~eXE@H;&#Kn-h9Uo#`T*rqd%=j#v@MI6q+OUMl^CE_0-WN3-^S+qjnD@mE$Gi_W z9P>WHaLoHi!*LxSWjN-2v|-PCH~(U7NaIWWnqLc!G1(aF*o297T*BmWe8R+jNuP#3 zv0rL9#y-L1W9&;9j=-s=W)*3h?w|CNeuV;g!e@Emhf2MZD+Slf})@7O3ms$UDp06Zi zcX9=5Tbe&E3tqV(dwP048rfBn>`b+##>=Giy?PFhHL~$L_FD79x~raxLr-y4>Th-7 z3wrzYccIq)@KZgDhTa;&CVsA2;N}q55%i}x>VGX^6JL}b+}c8qS~yC?PA&Ene&Wk2 z;yz2bbwz*pGC=5&Gu0n@B~Cs2Mn2WEZ`ky|U73$NzN{x=s9fCI`hvrzdTtC~<`Vft zZv$ZyUp6do1BFfcQygRT8(|Y)lpfq~g&wtZcA=IQ5`N;#s-h;~HWK~e%X~tQoT>iM zQ(RS#Y+Kv4V*N$$3=to>9wy>L&;M>^dv&(4h!0;j5gazv^Je%myx`04giUg`so?fX zdHKEI&>JuM!xzOdwwno?#HaM&HWzx-;3~ow*0W^j{ltTX!iOQApYoC^IZ zj{fcwbxnWsh z`Rr1SYi)^mMI=MKTJ8qZUohkZ+<9P-8bUkY1tXi-6bJJJr^J~-J zwZ(N4ajNGMIG3I+{9(O6r@(DAz`bkDIsK{o-^uH_$!44v%$b+0w&NVST*#7-D~xkl z@RdTAIdjAu4O#G2LYDfyI@twZQ;@wjlf6#JGPc(zS?~=)mOgLH?A}z6y*ZP;CCSD) z#CYB+IL54WFZpdk51V~NturRK*LwfkXSt75je3f8L9Xshb{IdN8Huvw?ye*|$<0~Y zEY)e0h5KPpLx$_O(%;oWmU*L|h2eVcZjmqa?hzcP{eL|xlb8SNY<#Zu6=OTkaE$GI!!foC49D0mG#q2Q$Z(A9 zV#6`EOANOc+d?)8<8{E0gc-A;2@~7G2~$H`^=uuto2;)G+c1-lu`Oaa#gc z6DJRD{h8lWnH@L5S?}or_e_C%R&aYJ->&K9-!x-##I?>1S-g?EQE*dTz}j=7KlaS$ z1;>1UL2&GQU-og!t$xD2SkQZ^z`ZOua{WqyTS?>$J?@vclKrJIZ{9GQUNhh~3w^wk zIBM99QN!=nT(gG1smG$ZhJPoev6j_s5@N z1MD?EIobTb?vFp!aj!bvaNMiTFzoeAfB!Zd_u(@Q$9?!L!*N~n zx}zOxdqd+TsCt)mHk2QTJ!7;}>6u6lar^fK?Uaezzj$zLebL3pZ?fLpcKj-xR z*U9>=QNe7T>#9tKm2>mm1jGk0cBc94zh;T*=r z^>EIt&$u3T&FsYWu$ysSKg{LsNmh+X598vz>S;KRNw1`fzn&A;zY*DbI8v-zjBRhT z6W7B&!Y=FIV#zLeabcHy_Dy!d{e)e({+a9mAxqBYO0wX&g)DaG$?VQskex4+{Z*2U zbC8@>f}_uYq7UZh+csA`*Wg`6T@%+vJ$yu4&qSOp)P4VZySTOdZv}aO=nbxYYd%9* z8R!4Kec07sTd%FDrzNv`T1>=9ZWcE#*3)nyOZ_a7)zb(eOI}ANyWmj;+0mKo7$HlZ z$0k|uI3Y`)<1@Px3bIRNvP&je>dCWBpF0-DTHOoISih6@)@Tzh{~I`!cbM8z!WY)s z6$;#!z1=`-O`^9_L2u>6#kC5(Rg9~WU;gFF8npJod|6f4p|;nuxm2sV5o_C~+{8Yq z=e0B0-GnUlyiO*&hmfV7*Ue-n3t4Juy-aqmg6#U4?A`_04Kvxl6l8ys$?lhA)qd2W zTkC%Gh<#u+%JNK}9<4vXYF$4VxBPE9Y|!6FsCA5~hqVfI^u77pp5EF*kJ{Kc^%rXa z?!KzAUV+!mYAl`^)-x_%L#?05#xuhP#>F$khDnyW8qW;B$?U{4!*7inXHm0OY?Nfx z7;S7^JTq)!IF8Zpl5Wfcb7Gxrjo(zrGS(Bm_nqXoBD@^c%JmuZPp{h@b^u*uJEj};ubra#u0xThUwdT~!XKG{KbwE0CY zPB1R+X(wiV#`DlgnVonZI@!2!W|#b*lF521ruvJuiP@RvbRo+apJ7}))BIb=63cGc zb?cczmNojUWEXsPLH3+X_FN%L+~*}(@cBZPJ}=1ZURaR5D3iUomerr{<ISP+26AE zVt0St*KZl9+mQ6tzpU4{@!2c$=bv`1=+`Oi5et}I_ZPgNVP8AA4`Ct0zIK3z81^-R zzK0t2H37V^VPBV+8^a9y`ofqjVmQW8IZW+lF_t?d?pDE3&$kHt{B+GVKLhr+ z;hxXtzH@voGqjhZ=DFC{CfYNiKm6og%Q$9?=d-5gcEHaGJJcU`;&@icfftfq>;rCk zeNMJ}r;@C?FO)uJvvoAa=!kKeU2yn4hv2BSL(>``{U$ea3O(vj=}}LI37f=lxZs$V z%1`w6>N{iX==nXUd0ybSA2r6u{GBfJ$jgxhPU(>sr3a_<$eG`to5ujhy}B_5#{FfX z2RBvZjIn)1=)vtR^eT7H+G~Yc&hK&Lp4eFT)Z}?RTj$pKhCTi$Tlrho&d%mY{&%Wf z2HzX^k~Y@?=NNA|)d%yCJ^o{nA8PRv!EsLiRB-62dsp%MGm%%g&yDkS2>eCXM_lWE zsyTfgKC^19rS_$69h04z6LvfJx`nN$BMoz{vZdishKX~lwB9P+DVgq`neJXmxBf`p zZ2C1D{Vr$o&N1_F`Gkpmg@h-YjJFjNX50qaS-JzG7_O8;-G^ zVD@8dKNufl`_XWW?Q!cb#&%*B+ewB!@6@CFzO}z)ZSbs(Saxf`w*Te#s3p<1HoMM= z>+-_C5OzjMT(|UIyr#&1m&|>3;SuY4f!?k8T`*y4Wub)0_mG6Aq`oHhXwCOf<72)T zHayy7sq0~eW4;$L9P_=X;h68m499#gZaCJ`aKkY#j#y_qWI2o6%muEa;K;{G1Kf&U z*Jl-ad%C!_&VrlPpxTs8_RcOskGb%&)ne{Tm2A!?^x%5;bfMN5;QAD}z6EZC)q&fr z%iq7XR@JX%Ip~R%R0J& ztsi-A7>jPgZ^pN~;Hdc?f@2JO3XZx^dV9Hh)_Ms&Y+fwp1p7p9p+{a+3~+vrXPy^0 z?)PNOTym1XSo=BcLy;pdC9a?Fle{QBIHgC<{QlQG1~~48jeauja|u1Txdq2K%v0dz z&A7Op|0;2@mRWyCbZO29k7lpz+9=j}mGu$1fr(RlM(0@?djd7HvkApLW0!=9bJv6? zxOBkPbPk2g$ZhOMk(2@2%cn{;_p7966qm8FdCK--<#$>~B&zNF3?iqU;9&h^i zzn9^-XG}F5>tL9})E*b>dzHBP>zp;uwEi|+-@c9S!J7jP-(7{Lt}Hi=#7!6`lR;`f{8e!_9@Y4npdW|Yu_8!b3;J*L2o%{X5hCc1mp#tClM z2Guq`Jhv1adD=>F zjHl9L%(fPK*xWpwhvIk=-!?*zdQ~x?w_QQc?-9*w3!5Xd>vWf7f8I&hBxgGmIHgC< zlpdVYBcFc%Y#syLVj>1=Vkw~qx0{H8F0f5nV*6>4Io0=II3TP1Po+>-xZ&HUWr{S(t_q7S&$ zGcLxoM#e>(`=@hK^bMO|S`MR~HHDqZjagf(=9+7qzgPK-wS9ZGu5oJ@?7UdtAJnu{ z)fO5r`C9pMx|b;WPHlW{pHD>H^E9nex0OSE$d0tRwY zpL)QAS%G^2bH*GnXGIJsppsEh%sJl`+Vm3cg@u1)@k-_UH|`H zFmw0ps_Lrh>OMW^oS7l_O=4$0VTZZAU+j^XAH^}|8A6YBH&bw|rC+=TV|`HX{%S2Z z?h*eG^-9?Xy^m;zeLU@ZMAqG3!VYECJ+ah#*!|&p)y(r4`(z#|%lz0|Cy{c4xpT+kY-CNHY2m9P1$&K~N_aQrq+_1lv)f&_N&E;II z6Z-OP$J#!Ccn|MC%5vQ7yA#bH;=vi&$BVM;f6-66Ua^k%gdMX>JS541X%_Z?c2f^~ zEFbHsw^w3k?~I-489VzVcJ|HKx!>*7nOojQ`}5L0?T_-9kW=@R7Opo`^vAm6J(saR zt`BF3_Y2h9PuOJb?4NM-!`%5OIhPr^k2Bn-8Sb+T_hp9rI>UXF;l9go-)FcVGF+vL zWv4v0oWphxXygim-kK$P1qnR2OgN*sMvutYY%TQ0yWO1L+(M6QyWJn#R1EV9J+Au) zX7t($J+37OW%N2Gdhy)lFKudmw)1#<`LjKxzq`0n+wwWx-j3-*AIDq^`#K)wHtQX8 z4mSYXF@U%qupI-4`vdm{<`@nDt`B%+;09pYUj?{dz^elH2j(1J4R`?X2>iFUI&d5# zL*zz!KGixt3S3;rM<=C=Vt7jm)JR9$g+Mua#`XFT$XEhQH`kfaG`UmjxPac*CJR4XJFjpwTUww)Bm#^ zv){8Fv(C~GC1t079Jm6Tz=L5$&z5qDZ@rA&#jxPd^b$l^!tm8|7(>mrl zbZL?^ljLzr-RH_z z=Pt|WsryYCyRyj`)V;6NQ#P5iD}^uAB0`@%AL+t59_dRx#c?c zuz!y+YwIq^#@yTu9CLFIaLmoUz%e)X0mt0j4;*td132bpCUDG+A;zfgJ!QSTm8s91 zp5oZw@r%}cQF(d`e)a~6J7fri>N!qby-tzY4+y^IhFf%=z}B&>Lc=&|b=LFFVJ%VDi*^6lBKq)VVt1IZOIcfA3mV1q0WQlLvif3s8?u)BHvU39p1lkI z;yV2va9pR~2afCX2f%Ti{t!5>(;orb+BM4i`WQH_)1Lsxb^24_Sl`j_cinHsw3OSl zr}q-hUSrPR&u|}PxDN%#TKci0^-$g;lY6Y{V40&Y>RwUqS=Ifgtn*LYmuh;@bKKv3 z3eGU!)BndY=iX9Q=GyQDa2)e5f#aBe1suoxYv4HM-vGxk{}woo`G0}qn12Tx z$NYc5P1WPK-O9ezbuIhW3^@AM960*c0=NOz8GB3M=-ZpvzeV5X0v~;A1#G^>^{=p( z&{pVFTeoYdQu*HFs7^h+Xk#+i$sfTP9`5)j#~gznTsQW^{mGK3Cz}VvE27`D_E;w! z+z#i6NwU5a$Nqm6daPNcNA5SFM?dX5qf$Rvlih?K{Zt(N{9WkLm!EO`PWvO*Md&eS z>ONL-$`{&HdgQ3r&Rjz4ReP#FruTKWnX5d1=XT6o&EuHs%e;RDZx7rTcyITq18|%(9f2Fb(S9f3IA=No$2rplIL?`_z;PVnI{im74p!HU>(31L zSHeY`=k~1i$9ZQqISvPdvs{cb5wt^qEf*Z$LtR&`t1q@KkIPGrIW8|dW}aSk%sk%? zpW`@Gu-3$Jr~{7U&Q4rw*^o$(YFPG zqi_ELj=n7f+yLv0ePQ6}+aHN56~M7Z3~@|)C1Y(I*9OHg-)%hJxc0FQlpeXc zg&xhwE2=4;&H8Q!kIktsqSTD&J;&q24rH->7`s;Z`un#8Q-}1&FgZ#jJ220F7dss zT5HC%FRwLY9W(xMjyd1PJ7#Tkgimp;nE*b{=YxUcd_Dv?&gVmc8{i}DPXvzh`7q$P z)*KET=kp|Bi@TXUdE>g?v#9M~h$k*;0%cjwx9&JEvLaaGP9oPF+b$WdYliD4xIN4z zv{@bOu{?g{x(hwVG=GL$z&Vwh1;JUow6kzVHvQhSMO;?ZQvoi%C+Ojr_0-cb$9qx7 ztf$3*V?8Yn9P4Qb;8;&f0>^sl1sv;XDd1R7O9RJxS_U}Q)3U%e4(;p-tyk@-`k3yY zmV<1Jb$Q2(bp^-F@rsTa`$~=(dvD+vdmrE!dtcxfdp&TBy#YAJ-VZp&-XA!|J^(nz zzA|uHgRJ@AeGQ4nIIh>ck1ul6BNuD1sAdy;B2O~oUn|+anHcTtAbey0I|`0^gTy+_ zHM359Hyqc@(b%V?x!|}ej(R(Z{>Garw3eOiu{_7=r{Y*QIj*g*X)*WgPjS3AVG-dA zKLc1CCH zOw8ChEU|OAu*2FsUT};{-J`_0pg8KK_pw-RxhGK^ZJr=}Vf_sgb;}r(P3kF*=V?zA z{gFEcXw78hvkdonhWj$ZjS;zKUQQ9Q zlRGuTohCTu>=)q+bM~v?n6uvm$DFN(xt)#!bN0K?Blky!`zymu%H-^H5j%5shI4U! zB6la&xx&ZUp3cnJIV;1Ro#Dg}B2PR)$Vc_Mb^{(R@uxZInJOYZtkYg|u`$A$~sE=*KgFMqbD3&GibjcZr8j@n9Z;JxQv5$GV&BsaQCH;<&jbSJ5Yd3Msylm>2dD+Y{{odR$^YYpJ<-BYG zKIUah;Fy;|z%egd0mr;-4IJ~b4RFlMw!kqj+X2VCEMw5yf2#>;{Js{;8S7(>PBp*B zx$LiyHNQCze{A-g)MBgYKqS zp6?tdb^q2_?`N{`Z-R4|X1IsN{Tu90-6uppr?|~H-4VgIdN?C&3%*I+fb^tA=X#nv;9OH1Hb|8x0qA&%*AxARYayPC&>dlKKx1i5=f z4#?dHuJHcsNgcm=KIbs^mH9Dw#x+y)&-!{maIB{X1;?>GTfDa<_mI$I&bP!`na*eG z%@TU#?0&FRkMsF%u}(4e*TnIgvD-ZksZGXi_cWv&eX)BOn&Rx9g_L9Lc8@~J(Wc## z&=hC)AQ%_xkT%ErdP%=|e`b_r9}`@b@1^*CtWlQSqt$#jo?p&==PG5n_Nx0%`JVkT zF^<%GTyT6p|AgSUE+RMUsuA#4FxOcmmc4J(IsQ0SSV?N%< zaJFAhZ8Bcohr;=0x#9ZF`z1J6-|{{v+ZVE|jkm$s{*kipB>oHt+`EY#+hkZ6x^+Vuzto+Dz=`TcQ%RY!>li8x~L*lr~HTh%MF)n33(2$aWU8%+Ydf%WJ|cQ7828X~A)P)x8R= zp_SUtS(oI}@AN`R&2C-Y?>N_(k8TNPO0>C=_ZR2PD7ShoVkrD?d+Jx+x0R1mjQ4Q= z-a@rz@6xfnW*^$2yk<`Xwz(L6;5^;X&pQ!s#5n~OY=OFvV;nD-HLO4-MYTFnzLn#jhz!{Xrj zVlz1Ec4LYWr_O;S?05^%M#aTWE&FMenOUWx4+904-m5K zbLGVDDjC^T6WP@g*;qTPgNwDZMj{()XH9UicGhxPRXb}Z9ENcAarT5{?TdA_4)kJO zt?PDFUH#OlTvzKsHrCbpuB+;5Ah=jp8wgp}(S|NdypfP)9c}Eg#G43Nj_IZ@OT1Y| zcJoAb3n9xo+R|l-2MJmBxm99!>x}F+iR`wCY^ITi|%CiFLIdxL8-)yR53K z9TF~%;ckNCnC&4rj?rMjF+W2x+}?s?&i4@o%3EOS2GWr=SPvdq^AmnFU_BRe#ay+z1!jP`R`;@gBQ`#d1A zJ2fMFU?O{m%f`B7?$!NCqkRkxf}J>?h6;N?8z{byn{lZ7lgMo#^f+D{3XXc}z9w?K zUns`Mb9lT5Db`C-ji6GwuSc!699Bkqyh?WaM0QM)n|Mxfg{pz_ewX{?|8}TL?;l^ZgcDRV0Ih&N>ju0Gk zrhd`wI})+mF^c#okqe$zI9hOA ztB%QV$7Z_&(r3u?X7t1T?x9j zw<7Kh+!vVl?(_k!54bO|y-#E8^}ziCZUF8N%wuLhVB4Q@j`jzRaTp@UdoE%=SnKB^ zSF!fb6M1Eyv=hg_u#V6!$mG2C-8+rv=G6Tm>|<7YvsOO7@*3L1f@2(y2#&daRB+7A zV}cuRE}$vBu{EmlxX`1`Cj`g%o)jF{Jrx7>p2_IheJW|rXp{G>#4+PMc~00Q_k!S< zPo+mr=`o*5kDStDuI>JwG(K{?*C)oudbvpGk$XkN$ML&Z=#hI(=rL!P2t9I>Gu)+u z<2vwcW^DK9F=t-59;x5)q%W6={y2{=7aYe(*v2s4X&IE_IFvP z?$mt;jPJH;fBM?p%{Z5CT~@orb9(J=X}rdEyZhbFUx>%T^m9#z&R@R&bE0F`!(onD ztA{&g%};X7v0W7VfZphr_z3Xv*m@*zU+`RmjslLy)}w*rvGo|>cx*iuxIgrn>*Ih2 z0Mp;&fzui->?O3RNq+6M!XD7>09WAI{#*O1r#+U(x3HJcj_P8MnmaDTosi)=wVcx? zWBj_?oE+!!bRX+F6Y)02pYIeL^M04$n8UjT$8|;Naeco>=+WlYVq6&Cy+V)2L=^+M z85zBqf@AFSiM=to2ZSDDS9;`>9`o{`uu1MA!EtP7Ww?h0$6P;>;U2AWjlXqWpjPa^ zwfB+FpAJ1@4B%Jlbwd44ty;Jx>Gxr*U4#)ne#gwGhfpkGuAsDb6tEIefLI8#CL;_>*PJaah<#u zxE}JX)BAwqI(a{ETqkD$$8~Zha9k%J0FJr0@nn0ZniqW>x(LVl0)qD6TJE=<8pJ$f zZTA!0n3~B-f59>C0fOTgtSmUrPo>8>xQfuD%?CQ0Q{@`vy0xm%V{NGzsJD7X?>RAF zXmf7Cjr5$YEo?GpYi2m5$DAoWa!QZ+d|voTu9e_e6E9@A^+gOEqZfr9xeZ(|?n}9D zU4@As*Rh{kmB%{nsa{I@h->4^31?OcdqBHKoaZX4Or^3&G6(Htaz&NX>-3V{O|o2* zTt}9vE@owpHeadw**LGw3c0pjYkw2Jnu+PP4EK75`*(($+QYnQ^sT6(+C#eLyyJG_ zm{R@?aB(erSIBZ6uF<|Re(UM2*7jIFcCa=d#Qvo)+S}Vgmh0C;iR`;VmUDAfBKy9O zWi3CP$bKYbIX_dop9opb$wv~qp9)#}ur=mW!6e}Mb0N$8KAPD5Qpj>`OLO*>kfq(n z61!K3wSjhfi}5G-cw*-&VTbGM)nXkX_e5gnT49Idt?ZCX?OZSHaD0^=a;cpgg&mIl zdm=yNo=W1nS=izHcwg8d_e^5vR$+(r^?|TM?zzOyG+~D|a)IFJ&xL~H+`UL}9G{B? zN8c_H9LGcHF~^gI9%HyvaI9VW8P`DSeOsM7jeGaYgdNIW?p(CPJ{Ig=o9L|bD}^1( zs(X8=_mlg>9A6{sQ1&|E5A#nu%pLEYstfB8WoajJ9FGsZ#uTUY*xyG&j~wrriZ)qq z9}7Klw~1V^ralpR?quw_{kMVsgIF6C(kKBHKe2ewMy!;|;a-IzpdW_*$p~o10 zb1u$v_Lu(N8~cy4>_7cYx9^2rjuU^2jbrnJkmY=(EdAwuN-;mo`Hvz_=JjXMN8#%O zzk_Ik;|E&VALYIBzRpd;HNuqR9y@oRi~TNE+<*R&@yCACCCaWV&hs&se~OsN{gvS= z#hh_)EO$l!X^iA{6CC?%CTx;xp5a;uj{V(+y?E*iZMGD8A;?tRRzrE0-FC8*m#|+mg!*$MZA7uROlF{p$ z;ksqG?ip_W4EKfm*{)J+z5Lmp(sO{zb}XL*T+T7;WO>INs}&sc9N>zMxsLp+D4zpd z34A;U*c-Sn{NNhd2e>}qzQ7H@v|kS#&jB_7_Xp1!=m$IinEv($PUjrCg^L=e=Eox7 z;`}IFw$4<%F$X;gdqICc>@0A-!7?9<1IPH60FLo52^{0^1svmF3OL5UG;oZ68Q>WI zvcPHl%+*t3J>s!JT^pxfj$1C%-|`u51;H`DHwuovq}OsQXd~?bOQ!=vlJr7a#3YV>`G&bHZ*taTJ9y4;cigS?U z>V+Q1Grd;5RAaZ52G^USn!pgvLU04M; zt_!OI$8}*f;J7ZV4&2n5^$zMdu33*Ex6!x9funCv07u`R1dhHv1sr{Q9(_mOo(3O% zdj{Bi<7Wm84YmigHHzxf?C%(8-m-={o~=<&>AdDVQrC64Rw#}(*DU6=$vRMafQc{ZZFo$*F6wj6vy%-`03|n&Z%{D zb8xnJ9 zdG@yha9pc)1deOfPQVs-J9|RwReP#FrhDt1i*l^HIA*N7I%d7?=9s?h?wGOd0UTo= z3>;(M6F9~`1USaN7jTSyZ{QgFKEN^deSu@_LxC-JHLk_sn6Vz_^`Y2OoW#1l&~K2X!gu{RqgK@67vF&NJ^vI?ucpJ0fub%A4S>j5{F_q}kuh;wm$$VT4=0!QCA0FJ(G2poOe2srw- zF>v&46X59Eroc`4Hf-*)Z^MD3ZzF)CZwCNJ-wp(hz8wS{eOtC$*|(A4qi>^t&9`>; zgcjHT^m|CxYFVz4wH>oYHb?O4H^NnW}_G;t)z|dayfHtLrJ(jtB?Iko`1FoyV z>Lc~Yss11BSW{rG8ho6TowEy*)%~9%IQFluvC;3py3c8Iq`icuINDSkV;Ju5i((A4 zsjgiW)!y(u!PP&?@v%R3{f0Kx^&;wBBy2MFiv>5%f^vGSG;cm!qH)Xh+Gu$nL z8&avY+cMnk&c)nQZyLBNU8%LHRiO8j+h9z4LyU8MTj02sZwK5LJogvd1J?(<18`i+ zcLa`W`MLOPMO@2w0w34%oq^M_XB>AFO>-l9HqA{JdYsRX3Xc69n6cRtdy%I6oYJFd z?ygLK(~+B|Y(AFJJ9p8h_0)RI_(o>pdpzkc&fS&ful1+)8tG?V{^@V-ah|@?wzl`+ zUgPh6gYPxS$NGF7IM(OCfn$BX0UYb|P2gCcZvn^pd>c5{=R3eQM(X^^Ysgt|u;WYw z?e(JO8`tuG14ms$gY5y0-%mcg1}p#M`^Y(t_Y|g7UN6br>s&Jv@kZ{xLaz()7{#^y ze&1s$?p(nQ^S&mCc|z`B!7+C;;9D9K^Egv*te*!2H`Gj_U9pfomh(jJ!D3F1$AB4P zUQ+KNp+|0(;F$Y|Go1Q9VCp?0^te{5>rB);!{=-qzpAN9rShAv1F=TP@$-ZeAZM+2 zVUOh;R$s!aPx>Fq&O|I0M!QJm=|q;VL!|O7fMNS@?YY7p%lit}$)P=J`wHSWge>dk zO>nl35x*s5Sx1MsUE;TeEXVmBw@dtPM)ti#_I)ACy8FOoi9Zyw?DHd+CH^=g`$;1E zX-0Nwtap|(+C6J7fMopX-*uwKMwmF7x<0|0TDOx)lxvDy`dPNZM7@1h*dL90(a+B_ z+!qOFR_N#Rtqe8VjNEAFVtrHZ!v$-3BL2ECP5qgIfyo{x9$UXHOsm{Kb8(#KDfU0g z#L?34?%Xul6BJdc+QWaVpm9Fgdws6S-+{B^AIJNDiL4!4IVZjcXUA5`{@}88c+J`X z{3rKgVXrjyX~$r4KY{B9%sToR*ksi)_7`yR82c-5A4qdf{{|fUF~qgum}IT^y)d@X zCquOJM^RoY{sc~CnTNkzmbij`Qd!1Y=ROcO%g8oQWLpSX=B%a563-=M*=MW7ZtIL} zn?!c*L^jsWJm6yO%$vx@+L;ertev(lTW6_5i?!1(;f&`zYt_Y2;~2)eY7f0wR~_7r zs;iFRVqJ9tj&;@9b>rM)U3CFxbC|fRkY!zUb6MiBxg$6O09bj*FxB<$a8 zY>6)dZ(~b*F|gGp-_u?KY_&-|8MpzM_Adptx+0zeY<15ZT?TCROaCqhwi<0?PiQu0 zIHqgOQ}%6b;ON^rz|psLfunEh0XM)NeOVtk`Zf?a`nCaZ^ld|6^R0SX5`XnF=q>#& zpQ$+Z$2|T59DSVz9DO|kd5^vx2^@XB1M<<=>A=y~JAtFGzXHd(9@idTt70uKk<^~~ z#hfn*&itZmFPE(|^JsDHSt_w({!?#haOOY9eHoWk>&sj{%j?UskhR=#9azqF>+qVj zxSpINIO;7gIL3LF;OP4b8E!?vF~*eyN1MGfT%Uxq7{}QITD{;HUxVNnUq8Xom;M=U zK!#f-!`b!LSbwpm*2eh8npy`q%Er1`4_vI9^x8oyIreLad^5f^1;_Z-5*&S5JHxFbIL5cG;AnHb3}@Ga<5(Li z>;WzP+`)#z4v!P+I?m{tnaV~&kAADq1kq;hI?l$zCi_$84LL5G2tD?vuGP?{;%3Lt z*#d3c#~EUOn^pU3JYGN0r6yZvPu|$Sx+cQ@H+R3AQ3=n~+Z*SwY~B%Xk;E9eEfa2; zV6T-QqoQv-2h+}DRNRR;Hd_3O8(ay+^! zjeT*4^1l8c_#F53BZ2!umi9*h+g^uwG;rM4j{&y*4f`7l9QXC(faAV?Jh0Vu8+$@C z-&spPU~T9POkaNlj=uc_9DVy4IQsSraP;k0;ON_Lz|ptgfunDK0Gn^^>;Y|=WUY^D=u_3Y);UY>Mg0KzjXC?!F?05jW9IB*$IRI$ zj+wIwsHK>*Pr=8W{RcSa>@(nc#KW9@4jgm#1#rySm%uS+UjfIQeGQ!EjC-7uJI~2+ z4Le0}V@w?FRKd~T(*(!doi4cXHIo&kH`ci`gdT13w*>24)}9#OnL>~IXB7kW&d%uB zb(l1twE1>1C&%N=`NAf1c5a4Kdd!*9Bd7G3PrF8y#z5{35d&*tqR=CEv50|VbePa1 zH@T{(uS;Bqy-4(PX@lM-B{pq8%sF{PhC4Fh%v@m)Xx%#6qdul~caD1|lPv53?QZNr zt$((os(t8~?yK%s^)WX6JP-c?QukZSif6!YV^%~`e#hIkEe_kmZ_?=IjR{OS@eXyA^S~qg}hkla495u8Eyy!VcG^=FY`BBG)ak(^A;s zcq=>PQai1M9geTELoT&5x3I&pKSs)p#3a3j?XWGqi??oj^m;9nB(7s9&LU#zj;(%XBpq`LXUOL z7-Fqb@8{~gYCHz~DeO@8FXy5i_VEMu#MU?m>cm)3R$c$1-e2wy^WQ?)q3m4FMSnOp zw8Ome`eI$p&FUj%X(w_VtK+@?6sPpq-w8sG9Iq=zo2MOjxIpM}{4Nw6W9Tb5#;*FK-bF%>v0p4W#;*FK zpKo=y4C*!SN@0_EQ5^GfmC$24uOz4h@CR^xz<&Za0FQFpe*wq$ z)*qnn{@_@@m3HO#)^)(vxNa9!RC`GG^4EEOC~HzhO|-(#{uUVwqf+oyOmg;rDFCSa_cOj`oc? zFr7zm%_E&7l${RQlq>8dw9Tu>?DE_ys!Z*-=bF+xEqQ-sD$K`>Os-}MZn*dNgmdb= zXq#^3b4znO9^tz4IA%T0>v**D17X|d3h{j4W4*Tpj`iLSIM#c6;8^b+fcwE7`|AiC z>%9|jJeSlN*mCJ*Upn`g-e-SNmvXEZJ7%nxIOaMx*)e^&)G=e70vuz%3^>MqIdF{q z3gCKp%ej9gaE$#b;28VWz%lk~fMe{}0$c2AT*rWm=M2X>W~}2JGq&-L8QW&4`xx5< z@G-W7fn#ik0N01u4h4>}O$3gy9R?g@I~+L1HVHVLBb)~>iCQQ3a)x_Fa2)GviaGlu z_i9G(+Kk>)9p+u@k=-aix(YNb>qi@#(N8fG$ zj=qh;F)R9ZBlzgsO~B^cKe-2Ny>4Y+SHqefeO(kqpt&jqpxc~KKi;QaP)O8 z;OOfH&^KS>@s0D|>aH-_+YoVFALrbRqAWXH$g(bHCb9<#S^3$d`(RO)ogid6|LpTB5TD)D`AGWk5CrjWb~{YirR*#S;@|9cgpg%^PeaYw9)|ey z9;Q|vH|F;Xa3)LqrI2NQPj$P*UkO?I`?cF8{w5>)Z6f<$A+H!J=ZU(G!Sz{jjPqdkC)#9xIlYv# zu^!_--Py_hX!LU_f8CybP5LuE#_(H)``x*?Migcc?K$jS%?{fiiJjR0UkPVA%*7v_ z4K@00h`Hc7)jE688}-O3-*)U;6CUk4x3;xE%6-sI1@j=~*ngeisHe_LQBR#mVO=PW zem2YWr}XGcbD=lP!b0mWa?i0pTX6JKarCpL=#RdfDQq$?$|m&`S5zaYR9a;E>nr+W z4C-77`?LFqn;Q34f@ADEi~F`{a~Hu;Z#Tg)zSg2Y+We(j%er<43O(j}_e_6ngdY3j zIgdKey*)9X>KqI8<`y>T%RJ7tbDO<_?ezWEypWCWNiJ?v!>rl)oab??t>aN{^AeY5 z-jBpO)f*gfJIKdGlrWTGe@@o$DA#IzGBX91s`*E z8*t3o?Z6F)mG-9s$DAz)dogGK0**Ob$YaA$R-47b@wVS{kG|~>9DN%G9DN%O9DN%B z9DO?gIQn*7;@g4Xqi+WR+juJ9-a(wvx7E9peOm)K`nD!;^ldHR=-b-B(YJMgW8T&U zj=rr2Y`&?Qco6Gktcmp@YnV0hknPDkHn10C+ZH&+wjFR&d7p$ai8XNqaP;j+;ON^?z|ps( zfunE707u`}Olsm-@X@#9fSdBIb+@u_+ar$X+YZ3dw;h3_Z#w};-*yI$zU=}W^R_E+ z^ldlbrfTB8uH|*&e#cxVW;h;YCeUU&9$kZ#2Y_SVdmv9Swg1dhI)1sr|bCaIUR!AIZD0dC5-Ws;g0j<}<5BY>lC2LMOk4g`+A9RwVG z8wniqHVQcUHX698ns}ybc@25iG1rji9FH;+XwN$yU4xYufMed5K%Qc3FM^M;y#!nj zUDm|Qzzx8({|a!7ZOJ6>y?|qEO93~P_eq$;(YGUjqi;t7N8gSDj=miY9DO?mIQsT# z;@h#{qi@Fnn{RdYuP{V0)L zJtO-D&MC*()^J&MZgOXER%^`JF2Gi6d|$n*=!e{o?h`RTNOu5Vt(c| z$}%rMC9!TQe4y;li7Y=88|^Z-7W0?uV>e-!s?rO^&A~0^tdkGA@sN=@_Ju0GwqGJ`vW$uridqcA5oTdaG1*yA1-8B2M_x> zUgAkYmU%tG?GhiEkv%GrJzB_eY>#nS;$wv@`#dhOdwfRrghckljBGk*PZqMgpXg`2 z`7Ss>c)q@cfyTAf&=`9_`z4XRQONR`c0X#fsA|<7ZWgkvqhAxdw+Y$tZY#ArRmhHY z*%^rs(}XO2u-{=RT+g2F6tW!CnTg%Ig)GN7&DlLdmUbUZ?EVR#(s@a{c0DW|CvvkA zJC(L`%8s;`&{}{is(;lUn!9XWc#UKFFzlv%l1uHh6m~d<$_}~IPAg%D<7?L%Q-8=E zm|P2OBkXVvoG$E;dm_o@Ji^Wha|7)RVTasPiJkd`oe4EFm3Kv+Ikt0Qj~$Q4XNf+! zk4(QCY;1CE(Co0C?RGfs1qnQ#Bl;l!m+ucKer{D(ec3I+kge?2KBC&g4M)t}?c7~80 zQcG{;YRDG;w>`b$@5MQv?KR3?1KBvfhUoiu9cx@Od-BFH?1XD{aSW%rohVB?Cw8yN z65lIinUj-TmiWGm?8%Aj{TbPb_-sgAD`&W@dhgN~Tzv0R4; zeaKa6lY9UB1;^NC2#$Wv6x?|022JUWtx=T+gdS~9FHER>>|}fo3O&}1ih+6$W%LG! zd~*N%u+U>(6gS@EQygRGxu!UN9HU1?f6U8c&NcHs?1|^a7H(Tx3yHrIF)F;9~-Hm3-V@x3hiV~wa7sP}3{kLRG`n6XAK z6}~Vpiev1GW9;9H@guj9;8;Jeix|lLTW}nsH!|Ftf@98h66bN*-=~6OE&WGu%$fRb zGy3_N&>LSXCHcEI%vmM#enWjH8sn=IdaNZC1NE9^^nMiMN1KBM$2$95#Lk>G7yXe_ zdd!*9Bd7G3Po58r^MT{uQuxC0o6EUoK4$jBV{Xr)wuTY^B60zdYWuayHNMxh7mF%W zsVsy)>~H&T?W>HPn|qiJ+-(`|_6&D#hTGA(m>bs9mdIOC^{V#pR%?4M zpT}eV2PLv^3t7&?trFRHg)Hl6>qPc_AzCXxL}$g<8-yPpVI*4?&=-A{!qeX!q6 zC~E1dK7KA_ncwXb*)N4G=V+R7|&g{Ot|>{J#$q#eLnJ?>Nyh4 z4TG<#Z`GIZD(}~yU?ON;+S_B}oQ>xkTDfmAAIxL=ezUcZWv-^-{h!$-Zj;1p9&qeB z3668Ev*0*?x`_U1^V{kg)_5#tA2BYjMGJN}ZyN8zZYS)r27hw9jH|tn<$6O|`pff; zu|_$EJBTvzV^^h*rD%TGh8>pv5$ql?xWw#^I6s0HP(`O!Iv9Z?ubt3i-KZF-*FK}SfXFp%P8aJK z$FHNX$((h_a7vFkQ+ni-9`ng_sj((F?wy4%%yk#%;ymPWVzHw9o_`_F6JQ3bE@>1(?x|IZL04xW_*hYJ=T+ofqIK)^cE5M zq|IkCHM*p*$(${b;glY8ru4`uJ?4|=rd5qDC46D7mv)YIXGO;vP4_ZAMV^@7deH~x zNQ2;5cl`v%9QGF+{T?7V=1J)>Pb&*O+T1#`Hm)M{SR*P1=4I84-eMx3wE1DC?p7Bz znX}b0oYG^?lpZ;y$9(d-fvUSTg)hwYTF%9~<9@PdQ9hTjr01!6%LVFlCH!Z8hv9vf zVa7k)G5Z*pA2T>)>3caSWA6?t(Mx@6PnEv&VzliuE#nb z3LJgg4>zWv%oq%Ez8ng)DtN!Ow#bujaBbKV#j;4cpou<@a42%aiAw zljFLeIF7U83NwYamhgpo>b*Jp}=KE!Z&Qo7zj#H^tF6#nCr%(YL0Ka~r#F%`IVQmQ&if1Zz=m;BiKvO$N4O z0e!estjApAM)Wj|@;OoN*Qen6p7le#iO3J_oa9xCuHgC_AX1jgOKIA zHo;|ycNDVpeJ7VC-Z>+?OCr0gkY(<7b6Miug)IBrBe6R;BfDoJJ0v5U-gmZFMmD{l zYVV9}I{y2*EZ5OxtaQAMuWrD~byPTaOqbgDQ+8-#hjPa6SLGV7qeku`aEAFB@7awt z$Kzu0fgJNHM&+8^9Q zJ%F60$5tMbd5ny){NLD7$2azWkmrD7ZQd{kBQxA6!Eue?Z;wU4IX8CiQC@3CXYBBI z!<3!0X2)dg@b|!?9ljSx&&%^(_9)AB=a~*Q--*X1ea3k*2(syV%X3OIu}&98d&~W1 z!vl$I?l&7AOk{Jv+3-*z`=#&WV%+21ztnE-HydUpc5}bkV4vkGm;^lMezW0`L^k)E z4QbADzuEA3Vz(mZBWu@=L+O~3dn&QR-)xBaruOA(z_8{Vs#D z!|_e+n{98!wscGTtMR=?yf%GAqGyt<)8qYK zuGl}hALcKw%}0qGG_`JSD*9x-A1yeJQCramkJqbVuVFD0A0uQ73l6RJKCSV4s~`36 zl4R}*$k=~tKO;_S3;7dATOQ}rGO|7W{Tkyu zJtLdu;0)m(_j{KLj(fV^g(a#yFNS*V(mmZ-!XL_}`?j;Ivj1%DT)28)P1y(LAdT}p z5ewryKf_%he6NO%Sf}-T6*yp>2?i_spL5YI&X)_7Yk;2d}R7 zsn?Icn}cSLYr-`mKZOaPrS*8d_ZjQDs7k`?j`l}meqycm^!d=vWqbLvJ^jCa|L3V~ zYPHK+`n&J{JPq=(R_DSR6l?W#@Ud1~fseI18GNkOOMzppP63X!+8X*zjcGdm`yjvn E2SNrX6aWAK literal 0 HcmV?d00001 diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/digit.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/digit.tsv new file mode 100644 index 0000000000..fa329799db --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +one 1 +two 2 +three 3 +four 4 +five 5 +six 6 +seven 7 +eight 8 +nine 9 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/hundred.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/hundred.tsv new file mode 100644 index 0000000000..10e2b9645a --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/hundred.tsv @@ -0,0 +1 @@ +hundred \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/teen.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/teen.tsv new file mode 100644 index 0000000000..8e60fa1d3c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +ten 10 +eleven 11 +twelve 12 +thirteen 13 +fourteen 14 +fifteen 15 +sixteen 16 +seventeen 17 +eighteen 18 +nineteen 19 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/thousands.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/thousands.tsv new file mode 100644 index 0000000000..bf30542bb6 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/thousands.tsv @@ -0,0 +1,22 @@ +thousand +million +billion +trillion +quadrillion +quintillion +sextillion +septillion +octillion +nonillion +decillion +undecillion +duodecillion +tredecillion +quattuordecillion +quindecillion +sexdecillion +septendecillion +octodecillion +novemdecillion +vigintillion +centillion \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/ties.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/ties.tsv new file mode 100644 index 0000000000..65f9839fda --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/ties.tsv @@ -0,0 +1,8 @@ +twenty 2 +thirty 3 +forty 4 +fifty 5 +sixty 6 +seventy 7 +eighty 8 +ninety 9 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/zero.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/zero.tsv new file mode 100644 index 0000000000..c479272d40 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/numbers/zero.tsv @@ -0,0 +1 @@ +zero 0 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/digit.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/digit.tsv new file mode 100644 index 0000000000..4b7fc24f16 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/digit.tsv @@ -0,0 +1,9 @@ +first one +second two +third three +fourth four +fifth five +sixth sixth +seventh seven +eighth eight +ninth nine \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/teen.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/teen.tsv new file mode 100644 index 0000000000..496fefc086 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/ordinals/teen.tsv @@ -0,0 +1 @@ +twelfth twelve \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/digit_teen.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/digit_teen.tsv new file mode 100644 index 0000000000..cd0991331f --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/digit_teen.tsv @@ -0,0 +1,49 @@ +i 1 +ii 2 +iii 3 +iv 4 +v 5 +vi 6 +vii 7 +viii 8 +ix 9 +x 10 +xi 11 +xii 12 +xiii 13 +xiv 14 +xv 15 +xvi 16 +xvii 17 +xviii 18 +xix 19 +xx 20 +xxi 21 +xxii 22 +xxiii 23 +xxiv 24 +xxv 25 +xxvi 26 +xxvii 27 +xxviii 28 +xxix 29 +xxx 30 +xxxi 31 +xxxii 32 +xxxiii 33 +xxxiv 34 +xxxv 35 +xxxvi 36 +xxxvii 37 +xxxviii 38 +xxxix 39 +xl 40 +xli 41 +xlii 42 +xliii 43 +xliv 44 +xlv 45 +xlvi 46 +xlvii 47 +xlviii 48 +xlix 49 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/hundreds.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/hundreds.tsv new file mode 100644 index 0000000000..0aafad3049 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/hundreds.tsv @@ -0,0 +1,9 @@ +c 100 +cc 200 +ccc 300 +cd 400 +d 500 +dc 600 +dcc 700 +dccc 800 +cm 900 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/ties.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/ties.tsv new file mode 100644 index 0000000000..5516676f4b --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/roman/ties.tsv @@ -0,0 +1,5 @@ +l 50 +lx 60 +lxx 70 +lxxx 80 +xc 90 \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/suppletive.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/suppletive.tsv new file mode 100644 index 0000000000..dea620f37a --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/suppletive.tsv @@ -0,0 +1,37 @@ +deer +fish +sheep +foot feet +goose geese +man men +mouse mice +tooth teeth +woman women +won +child children +ox oxen +wife wives +wolf wolves +analysis analyses +criterion criteria +focus foci +percent +hertz +kroner krone +inch inches +calory calories +yen +megahertz +gigahertz +kilohertz +hertz +c c +horsepower +hundredweight +kilogram force kilograms force +mega siemens +revolution per minute revolutions per minute +mile per hour miles per hour +megabit per second megabits per second +square foot square feet +kilobit per second kilobits per second \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/time_suffix.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/time_suffix.tsv new file mode 100644 index 0000000000..98664c5a32 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/time_suffix.tsv @@ -0,0 +1,6 @@ +p.m. p m +p.m. p m +pm p m +a.m. a m +a.m a m +am a m \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/time_zone.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/time_zone.tsv new file mode 100644 index 0000000000..1d39472c84 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/time_zone.tsv @@ -0,0 +1,14 @@ +cst c s t +c.s.t c s t +cet c e t +c.e.t c e t +pst p s t +p.s.t p s t +est e s t +e.s.t e s t +pt p t +p.t p t +et e t +e.t e t +gmt g m t +g.m.t g m t diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/whitelist.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/whitelist.tsv new file mode 100644 index 0000000000..8539931904 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/whitelist.tsv @@ -0,0 +1,15924 @@ +Ph.D. p h d +Hon. honorable +& and +Mt. Mount +Maj. Major +Rev. Reverend +# hash +Gov. governor +7-eleven seven eleven +A. A. a a +A.A. a a +A.A.A. a a a +A&A a and a +AAAI a a a i +AAAM a a a m +AAAs a a a's +AAAS a a a s +AAAW a a a w +AABA a a b a +AaB a a b +A. A. B. a a b +AAB- a a b +AAB a a b +AABC a a b c +Aabo a a b o +AABP a a b p +AABW a a b w +aac a a c +AAC a a c +AACAR a a c a r +AACC a a c c +AACCUP a a c c u p +AACMI a a c m i +AACNo a a c n o +AACR a a c r +AACS a a c s +AACSB a a c s b +AACTA a a c t a +AACUPR a a c u p r +A.A.D. a a d +AADT a a d t +AADTs a a d t's +Aadu a a d u +AAEA a a e a +AAE a a e +AAERT a a e r t +AAF a a f +AAFCA a a f c a +AAFC a a f c +AAFC's a a f c's +AAFld a a f l d +AAFPOA a a f p o a +AAGPBL a a g p b l +AAHHE a a h h e +AAI a a i +AAIB a a i b +AAIP a a i p +AAK a a k +Aap a a p +AAP a a p +AAPB a a p b +AAPC a a p c +AAPEP a a p e p +AAPG a a p g +AAPM a a p m +AAPT a a p t +A.A.R.M. a a r m +A.B.A. a b a +A. B. a b +A.B. a b +A.B.A. J. a b a j +A.B.C. a b c +A. B. G. a b g +ABG a b g +ABH a b h +ABHD a b h d +Abhi a b h i +ABK a b k +abl a b l +Abl a b l +ABL a b l +ABLV a b l v +ABM a b m +ABMC a b m c +ABN a b n +ABN's a b n's +Abp a b p +ABP- a b p +ABP a b p +ABPD a b p d +ABPI a b p i +ABPN a b p n +ABPP a b p p +ABPRS a b p r s +ABPW a b p w +Abr a b r +ABR a b r +abv a b v +ABV- a b v +ABV a b v +ABW a b w +ABX a b x +Abz a b z +A.C.A. a c a +ACA a c a +A&C a and c +A. C. a c +A.C. a c +acac a c a c +Acad a c a d +ACAD a c a d +A.C.A.P. a c a p +ACAP a c a p +ACAS a c a s +ACAZ a c a z +ACBA a c b a +ACB a c b +ACBL a c b l +ACBS a c b s +ACBSP a c b s p +ACCA a c c a +ACC- a c c +ACC a c c +ACCC a c c c +ACCJ a c c j +ACC&S a c c and s +ACC's a c c's +ACCS a c c s +ACD a c d +ACDC a c d c +A.C. D.F.C. A.F.C. a c d f c a f c +A.C.E. a c e +ACF a c f +ACFE a c f e +A.C.G. a c g +ACG a c g +ACGIH's a c g i h's +A.C.H. a c h +ACH a c h +ACHP a c h p +ACHR a c h r +ACHRE a c h r e +Achs a c h's +ACHS a c h s +ACIA a c i a +ACI a c i +ACICS a c i c s +ACIGA a c i g a +ACIP a c i p +Acis a c i's +ACIS a c i s +ACLA a c l a +A.C.L. a c l +ACL a c l +A.C.L.N. a c l n +acls a c l s +ACL's a c l's +ACLS a c l s +ACLU a c l u +ACM- a c m +ACM a c m +ACMI a c m i +ACMs a c m's +ACN a c n +ACO's a c o's +AcpA a c p a +ACP- a c p +ACP a c p +AcpB a c p b +ACPI a c p i +ACPO a c p o +ACPP a c p p +ACPSEM a c p s e m +acq a c q +A&CR a and c r +ACR a c r +ACRPS a c r p s +ACSA a c s a +ACSAC a c s a c +ACSBR a c s b r +ACSCN a c s c n +ACSEL a c s e l +ACSIA a c s i a +ACSI a c s i +AcSOC a c s o c +ACSR a c s r +ACSS a c s s +ACSS's a c s s's +A.C.T. a c t +A.C.T.A.F.L. a c t a f l +acu a c u +ACU a c u +ACU's a c u's +ACUS a c u s +ACUV a c u v +ACV a c v +ACWA a c w a +AC&W a c and w +ACW a c w +ACWM a c w m +ACWS a c w s +Acy a c y +ADAA a d a a +A&D a and d +Adab a d a b +ADAC- a d a c +ADAC a d a c +A. D. a d +A.D. a d +A.D.A.M. a d a m +AdaSL a d a s l +ADATA a d a t a +ADBAC a d b a c +ADB- a d b +ADB a d b +ADBGR a d b g r +ADBICA a d b i c a +ADCA a d c a +AdC a d c +A.D.C. a d c +ADC a d c +ADCAF a d c a f +ADCC a d c c +ADC's a d c's +ADCs a d c's +ADCY a d c y +ADDA a d d a +AD&D a d and d +ADF a d f +ADF's a d f's +ADFS a d f s +ADG a d g +ADGB a d g b +ADH a d h +ADHD a d h d +ADHM a d h m +ADHS a d h s +ADIAC a d i a c +AdK a d k +ADK a d k +Adl a d l +ADL a d l +ADLL a d l l +ADLs a d l's +A.D.M.A.C. a d m a c +adm a d m +Adm a d m +ADM- a d m +ADM a d m +ADMK a d m k +admn a d m n +ADMN a d m n +ADNAC a d n a c +ADNEC a d n e c +ADNs a d n's +AdP a d p +A.D.P. a d p +ADP- a d p +ADP a d p +ADPCM a d p c m +ADPF a d p f +ADQ a d q +ADQ's a d q's +ADR a d r +ADRC a d r c +ADRP a d r p +ADRs a d r's +ADSB a d s b +ADSL a d s l +ADSP- a d s p +ADSR a d s r +ADSRs a d s r's +ADSs a d s's +ADT a d t +ADTG a d t g +Adwa a d w a +ADWC a d w c +ADX a d x +AEA a e a +A&E a and e +A. E. a e +A.E. a e +AEBN a e b n +A.E.C. a e c +AEC a e c +AECL a e c l +AECR a e c r +AEC's a e c's +Aed a e d +AED a e d +AEE a e e +AEF a e f +AEG a e g +A.E.G.I.S. a e g i s +aegte a e g t e +AEHL a e h l +AEIA a e i a +AEI a e i +AEIO- a e i o +AEIOU a e i o u +Aeka a e k a +AEK a e k +ael a e l +AEL a e l +Aema a e m a +AEM a e m +aen a e n +AEO a e o +AEPA a e p a +AEP a e p +aere a e r e +AEre a e r e +AErn a e r n +AERN a e r n +Aert a e r t +AESA a e s a +Aes a e's +AES- a e s +AES a e s +aet a e t +AET a e t +AETC a e t c +AEU a e u +AEV a e v +AEW a e w +AEX a e x +Afa a f a +AFA a f a +af a f +Af a f +A. F. a f +A.F. a f +AF- a f +AF a f +Afaf a f a f +AFAP a f a p +AFB a f b +AFCA- a f c a +AFCA a f c a +afc a f c +A.F.C. a f c +AFC a f c +AFCEC a f c e c +AFCO a f c o +AFC's a f c's +AFCS a f c s +AFCSThe a f c s t h e +Afd a f d +AfD a f d +AfDB a f d b +AFDD a f d d +Afe a f e +AFE a f e +Aff a f f +AFF a f f +AFFDL- a f f d l +AFG a f g +AFGM a f g m +AFH a f h +AFIA a f i a +Afi a f i +AFI a f i +AFIP a f i p +Afiq a f i q +AFI's a f i's +AFJ a f j +A.F.L. a f l +AFL- a f l +AFL a f l +AFLEG a f l e g +AFLPA a f l p a +AFLP a f l p +AFL's a f l's +AFLUA a f l u a +A. F.M.A. a f m a +AFM- a f m +AFM a f m +AFMC a f m c +AFMs a f m's +AFN a f n +AFNET a f n e t +AFNOR a f n o r +AfNS a f n s +Afo a f o +AFO a f o +AFOP a f o p +AFP a f p +AFRA a f r a +Afr a f r +AFR a f r +AFRC a f r c +AFRL a f r l +AFROTC a f r o t c +AFRTS a f r t s +AFS a f s +AFSC a f s c +AFSPC a f s p c +A. F. T. a f t +AFTRA a f t r a +AFTRA's a f t r a's +AFTRS a f t r s +AFV a f v +AFW a f w +AFWS a f w s +AFX a f x +A. G. a g +A.G. a g +AGB a g b +AGC a g c +Agco a g c o +AGCO a g c o +Agda a g d a +AGD a g d +A.G.E. a g e +AGF- a g f +AGF a g f +AG&G a g and g +agg a g g +Agi a g i +AGI a g i +AGID a g i d +AGIR a g i r +Agis a g i's +A.G.J. a g j +A. G. K. a g k +AGL a g l +agli a g l i +AGM- a g m +AGM a g m +AGMR- a g m r +Agn a g n +AgN a g n +AGN a g n +AGNs a g n's +A.G.P. a g p +AGP a g p +AGPL a g p l +AGPS a g p s +AGPW a g p w +AGRs a g r's +AGS a g s +AGSM a g s m +AGSS- a g s s +Agsu a g s u +agt a g t +AGT a g t +Agte a g t e +Agu a g u +AGU a g u +AGV a g v +A.H.A. a h a +A. H. a h +A.H. a h +A. H. C. a h c +AHC a h c +AHD a h d +Ahdhra a h d h r a +AHDR a h d r +ahe a h e +Ahe a h e +AHEC a h e c +AHF a h f +Ahirs a h i r's +AHJD- a h j d +Ahk a h k +AHL's a h l's +AHP a h p +AHRC a h r c +AHRS a h r s +AHSAA a h s a a +Ahsa'i a h s a i +Ahta a h t a +aht a h t +AHT a h t +AHTD a h t d +Ahu a h u +Ahva a h v a +AHV a h v +AIAA a i a a +A.I.A. a i a +A&I a and i +A. I. a i +A.I. a i +AI- a i +AI a i +AIAP a i a p +Aias a i a's +AIAS a i a s +AIATSIS a i a t s i s +AIAW a i a w +AIAWU a i a w u +A.I.B.A. a i b a +AIB a i b +AIBO's a i b o's +AIBS a i b s +AIC a i c +AICAR a i c a r +AICC a i c c +AICCCR a i c c c r +aici a i c i +AICL- a i c l +Aicme a i c m e +AICN a i c n +AICPA a i c p a +AICP a i c p +AICPS a i c p s +AICs a i c's +AICTE a i c t e +AICUF a i c u f +AIDA a i d a +AIDN a i d n +AIEC a i e c +AIEE- a i e e +AIEE a i e e +Aiel a i e l +AIEP a i e p +AIFA a i f a +AIF- a i f +AIF a i f +AIFB a i f b +Aife a i f e +AIFF a i f f +AIFM a i f m +AIFMD a i f m d +AiG a i g +AIG a i g +Aigis a i g i's +AIHA a i h a +AIH a i h +AIHL a i h l +AIIMS a i i m s +Aija a i j a +Aik a i k +AIK- a i k +AIK a i k +AIKR a i k r +A.I.M. a i m +AIP a i p +A.I.R. a i r +AIs a i's +AISC a i s c +AISD a i s d +AISRI a i s r i +Aist a i s t +AIST a i s t +AIT's a i t's +Aitu a i t u +Aiud a i u d +aius a i u s +AIVC a i v c +AIW a i w +AIX a i x +A. J. a j +A.J. a j +Ajba a j b a +A.J.B. a j b +AJC a j c +A.J.E. a j e +A.J.G.C. a j g c +A. J. J. a j j +A.J.K. a j k +AJK a j k +AJKF a j k f +AJL a j l +A. J. M. a j m +Ajmi a j m i +AJN a j n +AJO a j o +Ajok a j o k +A. J. P. a j p +A.J.P. a j p +AJPW a j p w +A.J.R. a j r +AJR- a j r +AJR a j r +Ajsa a j s a +A.J.S. a j s +AJS- a j s +AJS a j s +A. J. T. a j t +A.J.T. a j t +AJT a j t +Aju a j u +AJUFE a j u f e +AJUSCO a j u s c o +AJV a j v +AJW a j w +AJWRC a j w r c +Akaa a k a a +a.k.a. a k a +aka. a k a +aka a k a +Aka. a k a +Aka a k a +AKA- a k a +AKA a k a +A&K a and k +ak- a k +A. K. a k +A.K. a k +AK- a k +AK a k +AKAP a k a p +AKAPs a k a p's +akas a k a's +AKAs a k a's +AKB a k b +AKC a k c +AKCR a k c r +AKD a k d +AKG a k g +AKG's a k g's +akh a k h +AKH a k h +AKHS a k h s +AKK- a k k +AKM a k m +AKN a k n +Akpa a k p a +AKP a k p +AKPD a k p d +AKQA a k q a +AKR- a k r +AKR a k r +A.K.S. a k s +AKS a k s +Aku a k u +AKU a k u +AKW a k w +A&L a and l +A. L. a l +A.L. a l +ALDF a l d f +ALDH a l d h +ALDS a l d s +ALDT a l d t +ALFASID a l f a s i d +ALFIDI a l f i d i +ALFTP a l f t p +Alh a l h +ALH a l h +A. L. M. a l m +A.L.P. a l p +ALP- a l p +ALPG a l p g +ALQ- a l q +ALR a l r +ALSC a l s c +ALSF a l s f +ALSP a l s p +ALUs a l u's +A.L.V. a l v +ALVF a l v f +ALW a l w +ALWS a l w s +AMAA a m a a +A'ma a m a +A&M a and m +A. M. a m +A.M. a m +AMAP a m a p +AMARC a m a r c +AMAs a m a's +amb a m b +Amb a m b +AMB a m b +AmBX a m b x +AMc- a m c +A.M.C. a m c +AMC- a m c +AMC a m c +AMC's a m c's +AMCs a m c's +Amda a m d a +Amd a m d +AMD a m d +AMDH a m d h +AMDISA a m d i s a +AMD's a m d's +A.M.E. a m e +AMF a m f +AM&FM a m and f m +A. M. G. a m g +AMG- a m g +AMG a m g +AMGTV a m g t v +Amha a m h a +AMH a m h +A.M.H.S. a m h s +A. M. J. a m j +AMJ a m j +A. M. K. a m k +A.M.K. a m k +AMK a m k +aml a m l +AML- a m l +AML a m l +Amli a m l i +AMLS a m l s +Amlwch a m l w c h +Amm a m m +A. M. M. a m m +AMM a m m +AMMAYI a m m a y i +AMMB a m m b +AMMK a m m k +AMN a m n +AMNH a m n h +AMNRL a m n r l +AM&O a m and o +AMPK- a m p k +AMPK a m p k +AMPL a m p l +AMP's a m p's +Ampt a m p t +AMPTP a m p t p +amr a m r +A. M. R. a m r +AMREF a m r e f +AMSA a m s a +A&M's a and m's +Ams' a m's +AMS- a m s +amsl a m s l +AMSO's a m s o's +AMSRS a m s r s +AMS's a m s's +AMSS a m s s +AMSU- a m s u +AMSU a m s u +Amta a m t a +amt a m t +Amt a m t +A.M.T. a m t +AMT- a m t +AMT a m t +AMTB a m t b +AMTPAT a m t p a t +Amts a m t's +AMTV a m t v +Amu a m u +AMU a m u +AMU's a m u's +A.M.V.M. a m v m +AMW a m w +AMX- a m x +AMX a m x +AMYF a m y f +A. N. a n +A.N. a n +ANB a n b +ANBO a n b o +ANC a n c +ANCC a n c c +ANC's a n c's +andFHL a n d f h l +ANDOC a n d o c +ANDP a n d p +ANDPOP a n d p o p +andPPP a n d p p p +Anej a n e j +ANELFA a n e l f a +ANF a n f +ANFIS a n f i s +ANFP's a n f p's +ANGB a n g b +ANGPTL a n g p t l +ANGRAU a n g r a u +Angti a n g t i +Angu a n g u +Angul- a n g u l +Anhe a n h e +ANI a n i +ANL a n l +ANM a n m +ANP a n p +ANPP a n p p +ANPR a n p r +Anra a n r a +Anr a n r +ANR a n r +ANRC a n r c +ANREF's a n r e f's +Anrep a n r e p +ANRS a n r s +ANRW a n r w +Ans a n's +A.N.S. a n s +ANS a n s +ANSF a n s f +ANTB a n t b +ANTM a n t m +ANTV a n t v +ANUGA a n u g a +ANWB a n w b +ANX a n x +ANZ a n z +A&O a and o +AOA a o a +AOAC a o a c +A. O. a o +A.O. a o +AOB a o b +AOC a o c +AOCB a o c b +AOCCs a o c c's +AOCE a o c e +AOD a o d +Aodh a o d h +Aodla a o d l a +AODV a o d v +AOE a o e +AOGCC a o g c c +Aogo a o g o +AOH a o h +Aoke a o k e +Aoko a o k o +AOKP a o k p +Aola a o l a +Aol a o l +AOL- a o l +AOL a o l +AOL's a o l's +AOLTV a o l t v +AOM a o m +AOMC a o m c +AONB a o n b +Aone a o n e +Aoni a o n i +Aoos a o o's +AOPA a o p a +AOP a o p +aor a o r +AOR a o r +AORs a o r's +AORS a o r s +aos a o s +AoS a o s +AOS a o s +Aotus a o t u's +AOTW a o t w +aov a o v +aovf a o v f +AOWC a o w c +A&P a and p +A. P. a p +A.P. a p +APBA a p b a +APB a p b +APBL a p b l +APCA a p c a +Apc a p c +APC- a p c +APC a p c +APCCR a p c c r +APCh a p c h +APCR a p c r +APCRDA a p c r d a +APC's a p c's +APCs a p c's +APDA a p d a +APD- a p d +APD a p d +APDM a p d m +APEGBC a p e g b c +APFA a p f a +APF a p f +APFCT a p f c t +APFOL a p f o l +APFSDS a p f s d s +APFUTU a p f u t u +apg a p g +APG a p g +APGAW a p g a w +Aph a p h +A.P.H. a p h +APH a p h +APHEDA a p h e d a +APHL a p h l +Apiao a p i a o +Api a p i +APi a p i +API- a p i +API a p i +Apic a p i c +APICv a p i c v +API's a p i's +A.P.J. a p j +APJ a p j +Apl a p l +APL a p l +APLP a p l p +APLS a p l s +Aplu a p l u +APM a p m +APML a p m l +APMR a p m r +APMSO a p m s o +APN- a p n +APN a p n +A.P.N.C. a p n c +Apphttp a p p h t t p +Appl a p p l +APPO's a p p o's +appr a p p r +Appts a p p t's +appu a p p u +Appu a p p u +APRA's a p r a's +APRC a p r c +APR's a p r's +APRS a p r s +APRST a p r s t +A&P's a and p's +Aps a p's +APs a p's +APS- a p s +APS a p s +APSF a p s f +APSL a p s l +APSU- a p s u +APTA- a p t a +APTA a p t a +AP&T a p and t +APW a p w +APX a p x +APXS a p x s +Aqa a q a +AQA a q a +AQC a q c +AQI a q i +Aql a q l +AQP a q p +Aqr a q r +AQS- a q s +A&R a and r +ARA a r a +A. R. a r +A.R. a r +A.R.B. a r b +ARD a r d +ARDF a r d f +A.R.E. a r e +A.R.F. a r f +ARF a r f +A.R.G. a r g +ArgR a r g r +ARGs a r g's +ARGT a r g t +ArH a r h +ARJ a r j +ARL a r l +ARLFC a r l f c +A. R. M. a r m +ARMv a r m v +Arnd a r n d +ARNG a r n g +ARNT a r n t +A.R.P. a r p +ARPP- a r p p +ARQ- a r q +ARQ a r q +ARTL a r t l +A.R.U. a r u +arv a r v +Arv a r v +ARV a r v +ARVD a r v d +ARVN a r v n +ARW a r w +Arwi a r w i +ARWU a r w u +A.S.A. a s a +ASA a s a +As'ad a s a d +Asai a s a i +Asao a s a o +A.s a's +A. S. a s +A.S. a s +ASAS a s a s +Asasp a s a s p +ASAU a s a u +A.S.B. a s b +ASB a s b +ASBDA a s b d a +asbl a s b l +ASBM a s b m +ASBMH a s b m h +ASBO a s b o +A.S.C. a s c +ASC a s c +ASCB a s c b +ASCE a s c e +asci a s c i +Asci a s c i +ASCP a s c p +ASCW a s c w +A.S.D. a s d +ASD a s d +ASDIC a s d i c +Asdis a s d i's +ASE a s e +ASEC a s e c +ASEE a s e e +ASEF a s e f +ASFA a s f a +ASF a s f +Asfi a s f i +ASGA a s g a +ASG a s g +ASGC a s g c +ASGE a s g e +ASGS a s g s +A. S. H. a s h +A.S.I. a s i +ASL a s l +A.S.M. a s m +ASM a s m +ASMD a s m d +ASME a s m e +ASMIK a s m i k +ASML a s m l +ASMPH a s m p h +ASMSU a s m s u +Asn a s n +ASN- a s n +ASN a s n +Aso a s o +ASO a s o +ASP a s p +ASPCA a s p c a +Aspe a s p e +ASPTS a s p t s +ASQ- a s q +Asr a s r +ASR a s r +Assn a s s n +assoc a s s o c +Assoc a s s o c +ASSPs a s s p's +ASSR a s s r +Asst a s s t +ASSU a s s u +A. S. T. a s t +ASTAT a s t a t +ASTCL a s t c l +Aste a s t e +ASTE a s t e +ASTM a s t m +ASTR a s t r +Astt a s t t +A.S.U. a s u +ASU- a s u +ASU a s u +A.S.V. a s v +ASV a s v +Aswa a s w a +ASW- a s w +ASW a s w +ASX a s x +A&T a and t +atac a t a c +Atac a t a c +ATAC- a t a c +Atad a t a d +Ata's a t a's +Atas a t a's +ATA's a t a's +A. T. a t +A.T. a t +ATBF a t b f +ATBs a t b's +atc a t c +A.T.C. a t c +ATC a t c +atcc a t c c +ATCC- a t c c +ATCC a t c c +ATCDE a t c d e +ATCL a t c l +ATDC a t d c +ATF- a t f +ATF a t f +Atg a t g +ATG a t g +ATGM a t g m +ath a t h +Ath a t h +ATH a t h +ATHN a t h n +ATIA a t i a +Atia's a t i a's +Ati a t i +ATi a t i +ATI a t i +Atid a t i d +Atiiq a t i i q +Atil a t i l +ATINC a t i n c +ATIP a t i p +Atiq a t i q +atis a t i s +Ativ a t i v +ATJ a t j +Atka a t k a +ATK a t k +ATKN a t k n +ATK's a t k's +Atl a t l +ATL a t l +Atli a t l i +ATLY a t l y +A.T.M.A. a t m a +atm a t m +Atm a t m +ATM a t m +ATMs a t m's +ATN a t n +ATO- a t o +ATO a t o +Atos a t o's +ATO's a t o's +ATOs a t o's +ATOS a t o s +ATP- a t p +ATP a t p +atpB a t p b +A.T.Q. a t q +ATR- a t r +ATR a t r +atri a t r i +Atri a t r i +ATRP a t r p +A&T's a and t's +A.T.s a t's +A.T.S. a t s +ATS a t s +ATSC a t s c +ATSDR a t s d r +AT&SF a t and s f +ATSF a t s f +ATSIC a t s i c +AT&T- a t and t +AT&T a t and t +attd a t t d +ATTESA a t t e s a +ATTF a t t f +AT&T's a t and t's +ATTS a t t s +ATTWI a t t w i +ATTWX's a t t w x's +ATU's a t u's +atv a t v +ATV- a t v +ATV a t v +ATV's a t v's +ATVs a t v's +ATW a t w +AtxA a t x a +ATX a t x +ATXN a t x n +A. U. a u +A.U. a u +Aub a u b +Aubl a u b l +AUC a u c +AUVs a u v's +Auw a u w +avab a v a b +AVAC a v a c +A. V. a v +A.V. a v +AVCA a v c a +AVC- a v c +AVC a v c +AVCHD a v c h d +AVCs a v c's +AVCS a v c s +AVD a v d +avg a v g +AVG a v g +AVK a v k +AVL a v l +A. V. M. a v m +AVM a v m +Avn a v n +AVN a v n +Avo a v o +AvP a v p +AVP- a v p +AVP a v p +AVR a v r +AVRs a v r's +AVSA a v s a +Avs' a v's +Avs a v's +AVS a v s +AVSM a v s m +AVU a v u +avvo a v v o +AVX a v x +Awa a w a +A. W. A. a w a +AWA a w a +A. W. A. M. a w a m +AWAs a w a's +A. W. a w +A.W. a w +A. W. B. a w b +A.W.B. a w b +AWB a w b +AWB's a w b's +A.W.C. a w c +AWC a w c +AWD a w d +A. W. F. a w f +AWG a w g +AWGIE a w g i e +AWGN a w g n +A. W. H. a w h +AWHL a w h l +AWI a w i +AWM a w m +Awo a w o +AWP a w p +AWP's a w p's +AWR a w r +AWSA a w s a +AWU a w u +Awwa a w w a +AXAF a x a f +A. X. a x +AXS a x s +AXV a x v +A. Y. a y +A.Y. a y +AYF a y f +AYK a y k +AYP a y p +A. Z. a z +A.Z. a z +AZD- a z d +AZE a z e +Azg a z g +Azi a z i +AZI a z i +AZL a z l +azm a z m +AZN a z n +AZS a z s +AZSTA's a z s t a's +AZT a z t +AzTV a z t v +Baad b a a d +BAAG b a a g +Ba'al b a a l +Baal b a a l +BAAL b a a l +Baam b a a m +Baap b a a p +ba'as b a a's +Baat b a a t +Ba'ath b a a t h +B. A. b a +B.A. b a +B&A b and a +Bac b a c +B.A.C. b a c +BAC b a c +bae b a e +Bae b a e +BAe b a e +BAE b a e +BAFA b a f a +BAF b a f +BAMF b a m f +BAPED- b a p e d +BAPH- b a p h +B.A.R. b a r +BARV b a r v +BASCA b a s c a +BASEC b a s e c +BASF b a s f +B.A.S.P. b a s p +BATVG b a t v g +Bauw b a u w +B'Av b a v +BAV b a v +BBA b b a +BBAG b b a g +B&B b and b +bb b b +B. B. b b +B.B. b b +BB- b b +BB b b +BBB b b b +bbc b b c +B.B.C. b b c +BBC- b b c +BBC b b c +BBCBBC b b c b b c +BBCH- b b c h +BB&CI b b and c i +BBC&PJR b b c and p j r +BBC's b b c's +B.B.D. b b d +BBDO b b d o +BBE b b e +BBF b b f +BBFC b b f c +BBFF b b f f +BBG b b g +BBH b b h +BBI b b i +BBK b b k +BBKL b b k l +BBLB b b l b +BBL- b b l +BBL b b l +BBM b b m +BBMP b b m p +BBNG b b n g +BBN's b b n's +BBO b b o +BBP b b p +BBQ'er b b q e r +B&Bs b and b's +BB&S b b and s +BBS b b s +BBSes b b s e's +BBS's b b s's +BB&T b b and t +BBT b b t +BBTV b b t v +BBU b b u +BBVA b b v a +BbvCI b b v c i +BBV's b b v's +BBWAA b b w a a +BBWA b b w a +BBWR's b b w r's +BBYA b b y a +BBYO b b y o +BCA- b c a +BCA b c a +BCAD b c a d +BCAM b c a m +BCATP b c a t p +BCBA b c b a +BCB b c b +bc b c +B. C. b c +B.C. b c +BC- b c +BC, b c +BC b c +BCCA b c c a +BCC b c c +BCCCA b c c c a +Bcci b c c i +BCCI b c c i +BCCP b c c p +BCC's b c c's +BCDA b c d a +BCD b c d +BCEA b c e a +BCE- b c e +BCE, b c e +BCE b c e +BCE's b c e's +BCF b c f +BCG b c g +BCG's b c g's +BCHL b c h l +BCHR's b c h r's +BCHS b c h s +BCI b c i +B. C. J. b c j +bcl- b c l +Bcl- b c l +BCL b c l +BCLR b c l r +BCMA b c m a +BCM b c m +BCMG b c m g +BCMHS b c m h s +BCMS b c m s +BCN b c n +BCN's b c n's +BCP b c p +BCPM b c p m +BCR- b c r +BCR b c r +BCRF's b c r f's +B.C.'s b c's +BC's b c's +BCs b c's +BCS- b c s +BCS b c s +BCSC b c s c +BCSic b c s i c +BCSN b c s n +BCT b c t +BCTC- b c t c +BCTC b c t c +BCU b c u +BCYP b c y p +BdA b d a +BDA b d a +bd b d +B. D. b d +B.D. b d +BD- b d +BD b d +BDBL b d b l +BDC b d c +BDC's b d c's +BDD b d d +Bde b d e +B. D. E. b d e +BDE b d e +BDF b d f +BDI b d i +BDJ b d j +BDK b d k +BDK's b d k's +BDM- b d m +BDMI b d m i +BDN b d n +BDNF b d n f +BDO b d o +BDO's b d o's +BDOS b d o s +BDP b d p +BDR b d r +bds b d s +BDS b d s +BDSM b d s m +BdU b d u +BDU b d u +BDV b d v +B. D. W. b d w +B.E.A. b e a +B. E. b e +B.E. b e +Bedw b e d w +B.E.E. b e e +bef b e f +B.E.F. b e f +BEF b e f +BEF's b e f's +bei b e i +Bei b e i +Beih b e i h +BEIR b e i r +Beis b e i's +bej b e j +B.E.M. b e m +BEMs b e m's +BEP- b e p +BEP b e p +Ber b e r +BER b e r +Bes b e's +B.E.S. b e s +BES b e s +B.F.A. b f a +BFA b f a +bf b f +Bf- b f +B. F. b f +B.F. b f +BF b f +BFCA b f c a +BFC b f c +BFC's b f c's +BFDG b f d g +BFES b f e s +BFF b f f +BFFs b f f's +B. F. G. b f g +BFG b f g +BFG's b f g's +BFI b f i +BFI's b f i's +BFJA b f j a +BFKL b f k l +BFL b f l +BFM b f m +BFO b f o +BFRA b f r a +BFR b f r +BFSA b f s a +BFS b f s +BFT b f t +BFU b f u +bfy b f y +BGAB b g a b +BGA b g a +BGB b g b +B. G. b g +B.G. b g +BGC b g c +BGCI b g c i +BGD b g d +BGEA b g e a +B. G. E. b g e +BGH b g h +B. G. J. S. b g j s +BGL b g l +BglII b g l i i +BGM- b g m +BGM b g m +BGP- b g p +BGP b g p +BGRA b g r a +BGR b g r +BGRC b g r c +BGRSO b g r s o +bgs b g s +B.G.S. b g s +BGS b g s +BGSU b g s u +BGT b g t +Bgy b g y +Bha b h a +B&H b and h +B. H. b h +B.H. b h +BHCC b h c c +BHDP b h d p +BH&E b h and e +Bhe b h e +BHL b h l +BHMA b h m a +BHMO b h m o +BHMs b h m's +BHMT b h m t +BHO b h o +B. H. P. b h p +BHP b h p +BHRT b h r t +BHS b h s +BHSN b h s n +BHSU b h s u +bhttp b h t t p +BHU b h u +BHUSD b h u s d +BHVS b h v s +B. I. b i +B.I. b i +Bie b i e +BIE b i e +biedt b i e d t +Biem b i e m +B.I.G. b i g +B.I.G.'s b i g's +BiH b i h +BIH b i h +B.I.O.L.A. b i o l a +B. J. b j +B.J. b j +BJC- b j c +BJCC b j c c +BJCP b j c p +BJD b j d +B.J.F. b j f +B. J. I. b j i +BJJ b j j +BJP b j p +BJP's b j p's +BJPs b j p's +BJPS b j p s +B.J.'s b j's +B.J.T. b j t +BJT b j t +BJU b j u +BJY b j y +B.K.A. b k a +B&K b and k +BKB- b k b +B. K. b k +B.K. b k +BKCa b k c a +BKC b k c +BKF b k f +BKI b k i +BKN's b k n's +BKO b k o +B.K.R. b k r +BKR b k r +BK's b k's +BKS b k s +BKT b k t +BKTV b k t v +BKV b k v +B. L. b l +B.L. b l +BL- b l +BL b l +BLC b l c +BLCC b l c c +BLCN b l c n +ble b l e +bleg b l e g +blev b l e v +BLG- b l g +BLG b l g +BLH- b l h +bli b l i +BLI b l i +Blla b l l a +BLL- b l l +BLL b l l +BLM- b l m +BLM b l m +BLMC b l m c +BLOU's b l o u's +BLPP b l p p +BLR b l r +BLRC b l r c +BLR&D b l r and d +B.L.S. b l s +BLS b l s +BLT b l t +BLTF b l t f +BLTs b l t's +BLTTA- b l t t a +BLV b l v +BMAA b m a a +B.M.A. b m a +BMA b m a +BMAL b m a l +BMARC b m a r c +BMA's b m a's +B&M b and m +BMB b m b +B. M. b m +B.M. b m +BMCA b m c a +B.M.C. b m c +BMC b m c +bmd b m d +BMD b m d +BME b m e +BMF b m f +BMG b m g +BMHS b m h s +bmi b m i +BMI b m i +BMIC b m i c +BMIR b m i r +BMI's b m i's +BMIT b m i t +BMJ b m j +BMK b m k +BMKG b m k g +BMML- b m m l +BMMO b m m o +BMNA b m n a +BMNH b m n h +BMNP b m n p +BMO b m o +BMP- b m p +BMP b m p +BMPs b m p's +BMR- b m r +BMR b m r +BMRs b m r's +BMSA b m s a +B.M.S. b m s +BMS- b m s +BMS b m s +bmt b m t +B. M. T. b m t +B.M.T. b m t +BMT b m t +BMT's b m t's +BMU b m u +B.M.V. b m v +BMV b m v +B. M. W. b m w +BMW b m w +BMW M b m w +BMW's b m w's +BMX- b m x +BMX b m x +BNA b n a +B&N b and n +bnb b n b +BNB b n b +B. N. b n +B.N. b n +BNC b n c +BNCI b n c i +BNCM b n c m +BNCT b n c t +BND b n d +BNDY b n d y +BNET b n e t +BNF b n f +BNFL b n f l +BNI b n i +BNIC b n i c +BNL b n l +BNO b n o +BNP b n p +BNPH- b n p h +BNR b n r +BNRC b n r c +BNS b n s +BNSF b n s f +BNST b n s t +BNT b n t +BNU b n u +BNY b n y +BNZ b n z +Boac b o a c +BOAC- b o a c +BOAC b o a c +B&O b and o +B. O. b o +B.O. b o +Boc b o c +BoC b o c +BOC- b o c +BOC b o c +BOCs b o c's +BOCS b o c s +boj b o j +Boj b o j +BOJ b o j +B.O.M.B. b o m b +B&O's b and o's +B.O.S. b o s +BOYZZ b o y z z +BPA b p a +BPB b p b +bp b p +B. P. b p +B.P. b p +BP b p +BPC b p c +BPD b p d +BPER b p e r +B.P.H. b p h +BPH- b p h +BPI b p i +BPJ b p j +BPK b p k +bpl b p l +BPL b p l +B. P. M. b p m +BPM b p m +BPMN b p m n +B.P.N. b p n +BPN b p n +B.P.O. b p o +BPO b p o +BPPA b p p a +BPP b p p +BPR b p r +B.P.R.D. b p r d +BPRD b p r d +BPSA b p s a +BP's b p's +BPS b p s +BPSK b p s k +BPUP b p u p +BPV b p v +BPY b p y +B&Q b and q +BQB b q b +B. Q. b q +BQM- b q m +BRAC's b r a c's +BRAM b r a m +BRBDP b r b d p +Brbic b r b i c +B. R. b r +B.R. b r +BRBR b r b r +BRBs b r b's +BRCA b r c a +BRC b r c +BRCC b r c c +BRD b r d +BRF b r f +BRGF b r g f +BRGM b r g m +BRHS b r h s +B.R.I.C.K. b r i c k +BRK b r k +BRL b r l +BRM b r m +BRMS b r m s +Brn- b r n +BRN b r n +BRNC b r n c +Brne b r n e +Brno b r n o +BRNV b r n v +BRP b r p +BRPS b r p s +BRRA b r r a +BRS b r s +BRSCC's b r s c c's +BRT b r t +BRTs b r t's +BRTS b r t s +BRVM b r v m +BRW b r w +BRWK b r w k +Brza b r z a +BRZ b r z +brzu b r z u +brzy b r z y +Brzyk b r z y k +BSAA b s a a +B.S.A. b s a +BSA- b s a +BSA b s a +BSAC b s a c +BsaL b s a l +BSBA b s b a +BSB b s b +BSBI b s b i +B. s b's +B.'s b's +Bs b's +B. S. b s +B.S. b s +BS- b s +BS b s +B.S.C. b s c +BSC b s c +BSCL b s c l +bsd b s d +BSD- b s d +BSD b s d +BSDE b s d e +BSE b s e +BSET b s e t +BSFA b s f a +BSF b s f +BSG b s g +BSh- b s h +BSH b s h +BSIB b s i b +BSI b s i +BSK b s k +BSL- b s l +BSL b s l +BSM b s m +BSME b s m e +BSN b s n +BSNL b s n l +BSOG b s o g +BSPA b s p a +BSP b s p +BSPP b s p p +BSR b s r +BSRN b s r n +BSS b s s +BST- b s t +BSU b s u +BSV b s v +BSX b s x +BTA b t a +BTAF b t a f +BTB- b t b +BTB b t b +BTBD b t b d +bt b t +B. T. b t +B.T. b t +BT- b t +BT b t +BTCA b t c a +BTC- b t c +BTC b t c +BTCC b t c c +BTD b t d +BTF b t f +BTG b t g +BTH b t h +BTK b t k +BTL b t l +BTM b t m +BTN b t n +BTNK b t n k +BTOB b t o b +BTO b t o +BTP b t p +BTR- b t r +BTRC b t r c +BTRDA b t r d a +Btry b t r y +btsan b t s a n +BT's b t's +BTS b t s +BTT b t t +BTU b t u +btus b t u s +BTUs b t u's +BTV b t v +B.U. b u +B.U.M. b u m +BUV b u v +BVA b v a +BVB b v b +BVB's b v b's +B. V. b v +B.V. b v +BVC b v c +BVFB b v f b +BVI b v i +BVM b v m +BVO b v o +BVR b v r +BVRC b v r c +BVT b v t +BVVL b v v l +Bwa b w a +BWAF b w a f +B&W b and w +BWB b w b +BWBR b w b r +B. W. b w +B.W. b w +B.W.C. b w c +BWC b w c +BWF b w f +BWFC b w f c +BWHBC b w h b c +B. W. I. b w i +BWI b w i +BWIR b w i r +BWO b w o +BWR b w r +BWS b w s +BWT b w t +BWTs b w t's +BWV b w v +BWW b w w +bwwtv b w w t v +Bxa b x a +BX b x +Bxe b x e +BXML b x m l +BxPC- b x p c +BYA's b y a's +BYB b y b +B. Y. b y +byc b y c +BYC b y c +BYD b y d +BYFC b y f c +BYG b y g +bygd b y g d +Byk b y k +byn b y n +BYOB b y o b +Byo b y o +BYO b y o +Byou b y o u +BYR b y r +Byrl b y r l +bySLC b y s l c +BYST b y s t +byt b y t +B.Y.U. b y u +BYU b y u +BYU's b y u's +BYX b y x +BYZ b y z +B.Z. b z +BZ b z +BZD b z d +bzhed b z h e d +Bzik b z i k +BZK b z k +BZP b z p +Bzyb b z y b +CAA c a a +CAAC c a a c +CAAC's c a a c's +CAAHEP c a a h e p +CAAM c a a m +CAAS c a a s +CAASE c a a s e +Caat c a a t +CAAT c a a t +C. A. c a +C.A. c a +CA- c a +CA c a +C&A c and a +CAC c a c +CA&CC c a and c c +CACC c a c c +CACM c a c m +Cadw c a d w +CADW c a d w +caeca c a e c a +Cae c a e +CAE c a e +C.A.F.B. c a f b +CAFTT c a f t t +C. A. G. c a g +CAG c a g +CAGM c a g m +C. A. I. c a i +C. A. J. c a j +CALUX c a l u x +CAMC c a m c +CAMLG c a m l g +CAMPCO c a m p c o +C.A.M.'s c a m's +C. A. N. c a n +Capt. captain +C. A. R. c a r +cas c a s +Cas c a's +C.A.S. c a s +CAS c a s +Casc c a s c +CASC c a s c +CASD c a s d +CASF c a s f +CASG c a s g +CASQ c a s q +C. A. T. c a t +C.A.T. c a t +CATV c a t v +C.A.W. c a w +CBA c b a +CBBB c b b b +CBB c b b +CBBC c b b c +CBBS c b b s +CBCA c b c a +cb c b +C. B. c b +C.B. c b +CB- c b +CB c b +cbc c b c +CBC- c b c +CBC c b c +CB&CNS c b and c n s +CBC's c b c's +CBCS c b c s +CBCV- c b c v +CBDA c b d a +CBD c b d +CBDs c b d's +C.B.E. c b e +CBE c b e +CBEF c b e f +CBERS- c b e r s +CBE's c b e's +CBF- c b f +CBF c b f +CBFT- c b f t +CBFX- c b f x +CBGB c b g b +CBGB's c b g b's +CBGBs c b g b's +CBG c b g +CBH c b h +CBHG c b h g +CBI c b i +CBI's c b i's +CbiXS c b i x s +CBKB c b k b +CBK c b k +CBKMT c b k m t +CBKRT c b k r t +CBL c b l +CBL's c b l's +CBM- c b m +CBM c b m +CBME- c b m e +CBMS c b m s +CBMT- c b m t +CBN c b n +CBN's c b n's +CBNT c b n t +CBO- c b o +CBO c b o +CBON- c b o n +CBOT c b o t +CBP- c b p +CBP c b p +CBPE c b p e +CB&Q c b and q +CBR c b r +CBRE c b r e +CBRN c b r n +CBRNE c b r n e +CBSA- c b s a +CBSA c b s a +CBs c b's +CBS- c b s +CBS c b s +C.B.S.E. c b s e +CBSE c b s e +CBS's c b s's +CBT c b t +CBTC c b t c +CBTU c b t u +CBU c b u +CBUT c b u t +CBV- c b v +CBV c b v +CBWT c b w t +CBX c b x +CBY- c b y +CBYT- c b y t +CBYT c b y t +CBZH c b z h +'c c +CcaA c c a a +CCAA c c a a +CCAAT c c a a t +CCAB- c c a b +cca c c a +Cca c c a +CCA c c a +CCAF c c a f +CCAP c c a p +CCAR c c a r +CCAS c c a s +CCB c c b +CCBCC c c b c c +CCBE c c b e +CCCA c c c a +C&C c and c +C. C. c c +C.C. c c +CCCE c c c e +CCCF c c c f +CCCP- c c c p +CCCP c c c p +CCC's c c c's +CCDB c c d b +CCDC c c d c +ccd c c d +CCD c c d +CCDev c c d e v +CCDI c c d i +CCDI's c c d i's +CCDM c c d m +CCDR c c d r +CCD's c c d's +CCDs c c d's +CCEd c c e d +C.C.E.D. c c e d +CCF c c f +CCFO c c f o +CCG c c g +C.C.G.S. c c g s +CCGS c c g s +CCHA c c h a +CCHC c c h c +C.C.H. c c h +CCH c c h +CCHD c c h d +CCHS c c h s +C.C.I.A.A. c c i a a +CCIAA c c i a a +CCi c c i +CCI c c i +CCID c c i d +CCIE c c i e +CCIH c c i h +CCITT c c i t t +CCJ c c j +CCK c c k +CCLC c c l c +CCL c c l +CCMA c c m a +ccm c c m +CCM c c m +CCMM c c m m +CCMP c c m p +CCMS c c m s +CCN c c n +CCNY c c n y +CCO c c o +CCOKC c c o k c +CCOO c c o o +CCOP c c o p +C.C.O.W.E. E.T.F. c c o w e e t f +CCP c c p +CCPD c c p d +CCPL c c p l +CCPN c c p n +CCP's c c p's +CCQ c c q +CCRC c c r c +CCR c c r +CCRCs c c r c's +CCRH c c r h +CCRS c c r s +CCSA c c s a +CCSC c c s c +CC's c c's +CCS c c s +CCSD c c s d +C.C.S.D.N.Y. c c s d n y +C.C.S.M. c c s m +CCSN c c s n +CCSR c c s r +CCS's c c s's +cct c c t +CCT c c t +CCTF c c t f +CCTRN c c t r n +CCTT c c t t +CCTV- c c t v +CCTV c c t v +CCUA c c u a +CCU c c u +ccus c c u s +CCV c c v +CCVG c c v g +C. C. W. c c w +CCWD c c w d +CDAC- c d a c +CDA c d a +CDAT c d a t +CDB c d b +CDBS c d b s +CDCA c d c a +C&D c and d +cdc c d c +CDC- c d c +CDC c d c +cd c d +C. D. c d +C.D. c d +CD- c d +CD c d +CDC's c d c's +CDDB c d d b +CD&DR c d and d r +CDE c d e +CDF c d f +CDFI c d f i +CDFW c d f w +CDG- c d g +CDH c d h +CDi c d i +CDI c d i +CDISC c d i s c +CDI's c d i's +CDK c d k +CDKN c d k n +CDL c d l +CDLI c d l i +CDLS c d l s +CDMA- c d m a +CDMA c d m a +CDM- c d m +CDM c d m +cDNA c d n a +cdnas c d n a s +cDNAs c d n a's +CDN c d n +CDN's c d n's +CDO c d o +CDOs c d o's +CDPC c d p c +CDP c d p +CDP's c d p's +CDPs c d p's +CD&R c d and r +CDR c d r +cds c d s +CD's c d's +CDs, c d's +CDs c d's +CDS- c d s +CDS c d s +CDSG c d s g +CDSP c d s p +CDSPCo c d s p c o +CdtA c d t a +CDT c d t +CdTe c d t e +CDTi c d t i +CDTI c d t i +CDT's c d t's +CDTV c d t v +CDTV's c d t v's +CDU- c d u +CDU c d u +CDU's c d u's +CD&V c d and v +CDV c d v +CDW c d w +CDWP c d w p +Cec c e c +CEC c e c +c'e c e +ce c e +C'e c e +Ce c e +C. E. c e +C.E. c e +CE, c e +CE c e +CECP c e c p +CEDA c e d a +Ced- c e d +CED- c e d +CEDO c e d o +CEDR c e d r +CEDS c e d s +CEEBA c e e b a +CEEB c e e b +CEEH c e e h +Ceel c e e l +CEEOL c e e o l +CEESA c e e s a +CEFC c e f c +CEF c e f +Cefn c e f n +CEFP c e f p +CEFR c e f r +CEGB c e g b +C.E.G. c e g +C. E. H. c e h +CEI c e i +C&EI's c and e i's +C. E. M. c e m +C&EN c and e n +cen c e n +Cen c e n +CEN c e n +ceo c e o +C.E.O. c e o +CEO c e o +Ceol c e o l +CEOP c e o p +Ceorl c e o r l +Ceos c e o's +CEO's c e o's +CEOs' c e o's +CEOs c e o's +C.E.P. c e p +cer c e r +Cer c e r +CER c e r +CERP c e r p +CERPER c e r p e r +CERS c e r s +ces c e s +Ces c e's +C.E.S. c e s +CES c e s +CESL c e s l +CESO c e s o +CETB c e t b +C. E. T. c e t +CEV c e v +CEVCP c e v c p +CEVG c e v g +CEW c e w +Cex c e x +CeX c e x +CEX c e x +CEZ c e z +CEZMS c e z m s +CFAB c f a b +Cfa c f a +CFA c f a +CFAS c f a s +C. F. B. c f b +CFB c f b +CFBG c f b g +CFBISD c f b i s d +CFB's c f b's +C&F c and f +CFC- c f c +CFC c f c +cf c f +C. F. c f +C.F. c f +CF- c f +CF c f +CFCF- c f c f +CFC's c f c's +CFDA c f d a +CFD c f d +C.F.E. c f e +CFE c f e +CFFB- c f f b +CFFC c f f c +C. F. F. c f f +CFF- c f f +CFF c f f +C. F. H. c f h +CFH c f h +CFHS c f h s +CFI c f i +CFIT c f i t +CFJC- c f j c +CFJL c f j l +CFJP- c f j p +CFJR c f j r +CFL c f l +CFL's c f l's +CFM c f m +CFMEU c f m e u +CFMI c f m i +CFMM- c f m m +CFND c f n d +CFNY- c f n y +CFNY c f n y +C.F.O.A. c f o a +CFOA- c f o a +CFO c f o +CFOP c f o p +CFOs c f o's +CFPI c f p i +CFPL c f p l +CFQ c f q +CFRB c f r b +C.F.R. c f r +CFR- c f r +CFR c f r +CFRP c f r p +CFS c f s +CFSCI c f s c i +CFSP c f s p +CFTA c f t a +CFT c f t +CFTK- c f t k +CFTO- c f t o +CFTO c f t o +CFTR c f t r +CFTs c f t's +CFTU- c f t u +CFTXOP c f t x o p +CFU c f u +CFX- c f x +CFYN c f y n +CFZ c f z +CGA c g a +CGAP c g a p +CGB c g b +C&G c and g +CGCFAD c g c f a d +C. G. c g +C.G. c g +CGCRI c g c r i +CGCS c g c s +CGD c g d +CGDK c g d k +CGE c g e +CGF c g f +CGFP c g f p +CGG c g g +CGI- c g i +CGI c g i +CGIL c g i l +CGKD c g k d +cgl c g l +CGL c g l +CGP c g p +CGPM c g p m +CGPME c g p m e +CGPW c g p w +CGR c g r +CGRP c g r p +CGS c g s +CGSS c g s s +CGT c g t +CGTG c g t g +CGTIC c g t i c +CGU c g u +CGW c g w +CGW's c g w's +C.H.B. c h b +CHBM c h b m +CHBX c h b x +C&H c and h +C.H.C. c h c +C. H. c h +C.H. c h +CHCH- c h c h +CHCO c h c o +CHD c h d +CHDK c h d k +CHDS c h d s +CHF c h f +CHFI c h f i +C.H.G. c h g +CHGO c h g o +CHH c h h +CHHOTO c h h o t o +CHHS c h h s +Chhu c h h u +Chirs c h i r's +CHISMS c h i s m s +CHISZ c h i s z +CHJX- c h j x +CHK c h k +CHL c h l +CHLR c h l r +CHL's c h l's +C. H. M. c h m +CHM c h m +CHN c h n +CHP- c h p +CHP c h p +CHP'den c h p d e n +CHPs c h p's +CHPS c h p s +CHR c h r +CHRDI c h r d i +CHRGD c h r g d +C.H.S. c h s +CHS c h s +CHSE c h s e +CHT c h t +CHV c h v +CHX c h x +Chy c h y +CHYR c h y r +CIAC c i a c +Cia c i a +C.I.A. c i a +CIA- c i a +CIA, c i a +CIA c i a +CIC- c i c +CIC c i c +ci c i +Ci c i +C. I. c i +C.I. c i +CI c i +Cicic c i c i c +CICL c i c l +C.I.C.M. c i c m +CICO c i c o +CIC's c i c's +CICT- c i c t +C.I.D. c i d +CIFF c i f f +CIFL c i f l +CIHE c i h e +CIHR c i h r +cii c i i +CII c i i +CIID c i i d +CIJ c i j +CIK- c i k +CIKM c i k m +C.I.L. c i l +CIL c i l +cim c i m +Cim c i m +CIM c i m +Cio c i o +C.I.O. c i o +CIO c i o +CIOs c i o's +CIOS- c i o s +CIOT- c i o t +CIOT c i o t +CIPCA c i p c a +Cip c i p +C.I.P. c i p +CIP c i p +CIPD c i p d +CIPFA c i p f a +CIPM c i p m +CISA c i s a +CISC c i s c +cis- c i s +cis c i s +CIs c i's +CIS- c i s +CIS c i s +CITB c i t b +cit c i t +CIT- c i t +CIT c i t +CiTD c i t d +CITM c i t m +CITN c i t n +CITP c i t p +citS c i t s +CITV c i t v +CITWF c i t w f +CitX c i t x +Ciuc c i u c +Ciu c i u +CiU c i u +CIU c i u +CIUT c i u t +CIVD c i v d +CIW c i w +CIWS c i w s +CIX c i x +Cixi c i x i +CIY c i y +CIZN c i z n +CJA c j a +CJAD- c j a d +CJBHL c j b h l +CJCA c j c a +CJCB c j c b +CJC c j c +C. J. c j +C.J. c j +CJD c j d +C. J. E. c j e +CJFL c j f l +CJFT c j f t +CJFX c j f x +CJGC c j g c +CJGO c j g o +CJHL c j h l +CJIC- c j i c +CJK c j k +CJL c j l +CJLS c j l s +CJM c j m +CJNT c j n t +CJOC c j o c +CJOT- c j o t +CJP c j p +C. J. R. c j r +C.J.R. c j r +CJR c j r +CJSC c j s c +C. J. S. c j s +CJS c j s +CJT c j t +CJTF- c j t f +CJUF c j u f +CJWC c j w c +CKAC c k a c +CKC c k c +C. K. c k +C.K. c k +CK c k +CKD c k d +CKE c k e +CKLM c k l m +CKLN- c k l n +CKLQ c k l q +CKLW c k l w +CKND- c k n d +CKRY- c k r y +CKS c k s +CKSO- c k s o +CKSO c k s o +CKUE- c k u e +CKVL c k v l +C. K. W. c k w +CKWK c k w k +ckx c k x +CKXX c k x x +CKY c k y +CLAC c l a c +CLA c l a +CLAS c l a s +CLB- c l b +CLBs c l b's +clc c l c +CLC- c l c +CLC c l c +cl c l +C. L. c l +C.L. c l +CL- c l +CL c l +CLCN c l c n +CLD c l d +Cle c l e +CLE c l e +CLF c l f +C.L.G. c l g +CLG c l g +CLHIA c l h i a +CLHS c l h s +CLI c l i +clk- c l k +CLK c l k +cllr c l l r +CLM c l m +CLMD c l m d +CLMP c l m p +CL&N c l and n +Clo c l o +CL&P c l and p +CLP c l p +CLRC c l r c +C. L. R. c l r +C.L.R. c l r +CLR c l r +CLRK c l r k +CLSA c l s a +CLSC c l s c +CLs c l's +CLS- c l s +CLS c l s +CLT c l t +CLTD c l t d +CLTPA c l t p a +CLTs c l t's +Clu c l u +CLU c l u +CLUK c l u k +CLV c l v +Clwyd c l w y d +CLX c l x +C&MA c and m a +CMAC c m a c +C.M.A. c m a +CMA c m a +CMAL c m a l +CMAP's c m a p's +CMAS- c m a s +CMAS c m a s +CMAT c m a t +C.M.B. c m b +CMB c m b +CMCC c m c c +CMCCDI c m c c d i +C.M.C. c m c +CMC- c m c +CMC c m c +cm- c m +cm c m +C. M. c m +C.M. c m +CM- c m +CM c m +CMCs c m c's +CMCS c m c s +CMD c m d +Cmde c m d e +CMDISE c m d i s e +cmdr c m d r +CMEC c m e c +CME c m e +CMF c m f +CMFS c m f s +C. M. G. c m g +C.M.G. c m g +CMG c m g +CMHC c m h c +CMH c m h +CMHR c m h r +CMI c m i +CMJ c m j +CMLA c m l a +cml c m l +C.M.L. c m l +CML c m l +CMLDP c m l d p +CMLL c m l l +CMLL's c m l l's +CMM c m m +CMMI c m m i +CMMs c m m's +cmn c m n +CMO c m o +CMO's c m o's +CMPC c m p c +CMP c m p +CMPS c m p s +CMPSO c m p s o +CMPV c m p v +CMQ- c m q +CMQ c m q +CMQT c m q t +C. M. R. c m r +CMR c m r +CMSAF c m s a f +C.M.S. c m s +CMS c m s +C.M.S.L. c m s l +CMT c m t +Cmte c m t e +CMT's c m t's +CMTU c m t u +CMUCL c m u c l +CMU c m u +CMU's c m u's +CMV c m v +cmvIL- c m v i l +CMVM c m v m +CMVSS c m v s s +CMWF c m w f +CMXXII c m x x i i +CMYK c m y k +CNA c n a +CNBC- c n b c +CNBC c n b c +CNB c n b +CNC c n c +cn c n +C. N. c n +C.N. c n +CN c n +CNC's c n c's +CND c n d +CNDL c n d l +CNEB c n e b +CNE c n e +CNF c n f +CNFE c n f e +CNFK c n f k +CNG- c n g +CNG c n g +CNI c n i +CNJ c n j +CNJF c n j f +CNK c n k +C. N. L. c n l +C.N.L. c n l +CNL c n l +CNMs c n m's +CNN- c n n +CNN c n n +CNO c n o +CNPC c n p c +CNP c n p +cnr c n r +CNR c n r +CNRMA c n r m a +CNRS c n r s +CNSA c n s a +CNs c n's +CNS c n s +CNSS c n s s +CNSW c n s w +CNTA c n t a +CNT c n t +CNTE c n t e +CNTI c n t i +CNTK c n t k +CNTN c n t n +CNTs c n t's +CNTS c n t s +CNTV c n t v +CNU c n u +Cnut's c n u t's +CNV c n v +CNVs c n v's +CNW c n w +CNWS c n w s +CNZ c n z +COA c o a +C&O c and o +C. O. c o +C.O. c o +CO- c o +Co. company +C.O.D. c o d +Col. colonel +C.O.M.L. c o m l +Cov c o v +COX c o x +CPAC c p a c +CPA c p a +CPAP- c p a p +CPA's c p a's +CPB c p b +CPBL c p b l +C&P c and p +CPC- c p c +CPC c p c +cp c p +C. P. c p +C.P. c p +CP- c p +CP c p +CPC's c p c's +CPCs c p c's +CPCS c p c s +CPD c p d +CPDM c p d m +CPD's c p d's +CPEB c p e b +CPEC c p e c +C.P.E. c p e +CPE c p e +CPEO c p e o +CPF c p f +CPFO c p f o +CPG c p g +CPHC c p h c +CPH c p h +CPI- c p i +CPI c p i +CPIFL c p i f l +CPIM c p i m +CPI's c p i's +CPJ c p j +CPL c p l +CPLP c p l p +CPM c p m +CPN c p n +CPNI c p n i +cpo c p o +C.P.O. c p o +CPO c p o +CPOE c p o e +CPPA c p p a +CPPCC c p p c c +CPP c p p +CPPE c p p e +CPPIB c p p i b +CPPM c p p m +CPQ c p q +CPR c p r +CPRS c p r s +CPSA c p s a +CPSC c p s c +CPS- c p s +CPS c p s +CPSF c p s f +CPSK c p s k +CPSL c p s l +CPSU c p s u +CPTA c p t a +CPT c p t +CPTM c p t m +CPUC c p u c +CPU- c p u +CPU c p u +CPUID c p u i d +CPUSA c p u s a +CPUs c p u's +CPUT c p u t +CPVA c p v a +CPV c p v +CPW c p w +CQAC c q a c +CQBR c q b r +CQC c q c +cq c q +CQ c q +CQD c q d +CQDs c q d's +CQM c q m +CQO c q o +CQU c q u +CQW- c q w +CRABP- c r a b p +C. R. A. c r a +CRA c r a +C.R.A.Z.Y. c r a z y +CRBC c r b c +CRC c r c +cr- c r +cr c r +Cr- c r +C. R. c r +C.R. c r +CR- c r +CR c r +Cre c r e +CRE c r e +CRF- c r f +CRF c r f +CRG c r g +CRHK c r h k +CRH's c r h's +CRHS c r h s +CRJ c r j +crk c r k +Crkva c r k v a +crkve c r k v e +CRL- c r l +CRL c r l +CRLs c r l's +crm c r m +CRM c r m +CRNA c r n a +CRN- c r n +CRN c r n +Crne c r n e +Crni c r n i +Crno c r n o +CRP c r p +CRPF c r p f +CRRF c r r f +CRs c r's +CRS- c r s +CRS c r s +CRSP c r s p +CRST c r s t +CRTC c r t c +CRT c r t +CRTC's c r t c's +CRUA c r u a +CRWRC c r w r c +CRW's c r w's +CRX c r x +CSAC c s a c +csa c s a +Csa c s a +C.S.A. c s a +CSA c s a +CSAH c s a h +Csak c s a k +Csaky c s a k y +CSAR c s a r +CSAT c s a t +CSAV c s a v +CSB c s b +C&S c and s +C.S.C. c s c +CSC c s c +CSCD c s c d +CSCE c s c e +CSCR c s c r +cs c s +C.s c's +Cs c's +C. S. c s +C.S. c s +CS- c s +CS c s +CSC's c s c's +CSDA c s d a +CSDB c s d b +CSD c s d +CSDL c s d l +CSDN c s d n +CSDP c s d p +CSD's c s d's +CSDS c s d s +CSE c s e +C&SF c and s f +CSF c s f +CSFL c s f l +CSFSO c s f s o +CSGA c s g a +csg c s g +CSG c s g +CSG's c s g's +CSH c s h +CSHL c s h l +CSIC c s i c +CsI c s i +CSI c s i +CSIDC c s i d c +CSILUS c s i l u s +CSIO c s i o +CSIR c s i r +CSIRO c s i r o +CSis c s i's +CSIs c s i's +CSIS- c s i s +CSIS c s i s +CSIU c s i u +CSKA- c s k a +CSKA c s k a +CSKA's c s k a's +CSK c s k +CSK's c s k's +CSL- c s l +CSL c s l +CSLI c s l i +CSM c s m +CSNB c s n b +CSNBX c s n b x +CSN c s n +CSNJ- c s n j +CSN's c s n's +CSOB c s o b +CSO c s o +CSO's c s o's +CSOs c s o's +cspA c s p a +csp c s p +CSP c s p +CSPH c s p h +CSPI c s p i +CSPRA c s p r a +CSPs c s p's +CSRA c s r a +CSR c s r +CSRS c s r s +CSRT c s r t +CSRTT c s r t t +CSSA c s s a +CSS c s s +CSSD c s s d +CSSE c s s e +CSSL c s s l +CSSNCs c s s n c's +CSSR c s s r +CSSS c s s s +CSST c s s t +CSTB c s t b +CST c s t +CSTF c s t f +CSTI c s t i +CSTS c s t s +CSUB c s u b +CSU c s u +CSU's c s u's +Csuz c s u z +CSV c s v +CSW c s w +CSWF c s w f +CSWIP c s w i p +CSWS c s w s +CSX c s x +CSX's c s x's +CSXT c s x t +C.T.A. c t a +CTA c t a +CTAF c t a f +CTARL c t a r l +CTA's c t a's +CTAs c t a's +CTBT c t b t +CTBTO c t b t o +C&T c and t +CTC c t c +ct c t +C. T. c t +C.T. c t +CT- c t +CT c t +CTD c t d +cte c t e +CTE c t e +CTEQ c t e q +CTF c t f +CTGF c t g f +CTI c t i +CTLA- c t l a +CTLA c t l a +CTL c t l +CTLL- c t l l +CTLLS c t l l s +CTLs c t l's +CTM c t m +CTMP- c t m p +CTN c t n +CTO c t o +CTP c t p +ctr c t r +CTRG c t r g +Ctrip c t r i p +CTRL c t r l +CTRP c t r p +CTR's c t r's +CTS- c t s +CTS c t s +CTSD c t s d +CTSF c t s f +CTSG c t s g +CTUC c t u c +CTU c t u +CTUG c t u g +CTU's c t u's +CTVA- c t v a +CTVA c t v a +ctv c t v +CTV c t v +CTV's c t v's +CTX c t x +CTZ c t z +cu c u +C. U. c u +CU- c u +CU c u +C.U.L. c u l +C. U. S. c u s +C. U. T. c u t +CUW c u w +Cuyp c u y p +CVA- c v a +CVA c v a +CVAK c v a k +CVARG c v a r g +CVB c v b +C&V c and v +CVC c v c +cvcp c v c p +'cv c v +cv- c v +cv c v +C. V. c v +C.V. c v +CV- c v +CV c v +CVD c v d +CVEs c v e's +CVG c v g +CVI c v i +CVJM c v j m +CVM c v m +C.V.N. c v n +CVN- c v n +CVN c v n +C.V.O. c v o +CVO c v o +CVODE c v o d e +CVOs c v o's +CVP c v p +CVR c v r +CVs c v's +CVS c v s +CVSNT c v s n t +CVT c v t +CVTs c v t's +CVV c v v +CVVHDF c v v h d f +CVW- c v w +CVW c v w +CVZ c v z +CWA c w a +CWBI c w b i +CWBL c w b l +CWC c w c +C. W. c w +C.W. c w +CWD c w d +CWDS c w d s +C. W. E. c w e +CWE c w e +Cwej c w e j +Cwele c w e l e +CWF c w f +CWGC c w g c +CWG c w g +CWI c w i +C. W. K. c w k +cwm c w m +CWM c w m +CWO c w o +CWPA c w p a +CWR c w r +cwrt c w r t +CWRU c w r u +CWSAC c w s a c +CW's c w's +CWS c w s +CWTS c w t s +CWUR c w u r +C. W. Y. c w y +CWYFL c w y f l +CXAM- c x a m +CXC- c x c +CXC c x c +CXCL c x c l +CXCR c x c r +C.X. c x +C. Y. c y +C.Y. c y +cyl c y l +Cyn c y n +CYOG c y o g +CyP c y p +CYP c y p +Cyrl c y r l +Cys- c y s +Cys c y's +CYS c y s +CYSK c y s k +CYSY c y s y +czci c z c i +cz c z +C.Z. c z +CZ- c z +CZ c z +CZE c z e +CZM c z m +CZW c z w +czy c z y +DAAD d a a d +Daai d a a i +D. A. d a +D.A. d a +D&AD d and a d +D. A. H. d a h +D.A.M. d a m +D.A.N.C.E. d a n c e +D. A. T. d a t +Dav d a v +D.A.V. d a v +DAV d a v +DAV's d a v's +dBa d b a +DBA d b a +DBC d b c +DBCS d b c s +db d b +D. B. d b +D.B. d b +DB- d b +DB, d b +DB d b +DBE d b e +DBI d b i +DBL d b l +DBLE d b l e +DBLP d b l p +DBM d b m +DBMS d b m s +DBR d b r +DB's d b's +D. B. S. d b s +DBS d b s +DBU d b u +DBYC d b y c +DBZ d b z +DCA- d c a +DCA d c a +DCAF d c a f +DCAM d c a m +DCB d c b +DCCC d c c c +DCCCD d c c c d +DCC d c c +DCCG d c c g +DCCs d c c's +D&C d and c +dc d c +D. C. d c +D.C. d c +DC- d c +DC d c +DCE d c e +DCFC d c f c +DCF d c f +DCG- d c g +DCG d c g +DCH d c h +DCI d c i +DCK d c k +D.C.L. d c l +DCL d c l +DCMA d c m a +DCM d c m +DCMF d c m f +DCMI d c m i +DCMS d c m s +DCO- d c o +DCO d c o +DCom d c o m +DCOM d c o m +DCP d c p +DCR- d c r +DCR d c r +D.C.'s d c's +DC's d c's +DCs d c's +DCS d c s +DCSD d c s d +DCSO d c s o +DCSPER d c s p e r +DCSS d c s s +DCTA d c t a +DCT d c t +DCU d c u +DCUO d c u o +DCUs d c u's +DDA d d a +DDB d d b +D.D.C. d d c +DDC- d d c +DDC d d c +D&D d and d +D. D. d d +D.D. d d +DDF d d f +DDG- d d g +DDG d d g +DDHH d d h h +DDi d d i +DDI d d i +DDK d d k +DDLJ d d l j +DDM d d m +DDMS d d m s +DDN d d n +DDNOS d d n o s +Ddoc d d o c +DDO d d o +DDP d d p +DDR- d d r +DDR d d r +DDRMAX d d r m a x +DDSB d d s b +D.D.S. d d s +DDS d d s +DDT d d t +DDTV d d t v +Ddu d d u +DDU- d d u +DDWFTTW d d w f t t w +DDX d d x +DEAE- d e a e +D.E.A.F. d e a f +Deah d e a h +DEA's d e a's +DECC d e c c +DEC d e c +DECS d e c s +DEDD d e d d +D. E. d e +D.E. d e +deg d e g +D. E. G. d e g +DEG d e g +DEH d e h +DEHP d e h p +dei d e i +Dei d e i +DeI d e i +DEI d e i +Deijm d e i j m +Deip d e i p +D. E. J. d e j +D.E.R. d e r +dez d e z +Dez d e z +Dfa d f a +DFA d f a +DFB- d f b +DFB d f b +D.F.C. d f c +DFC d f c +DFCs d f c's +D. F. d f +D.F. d f +DFD's d f d's +DfE d f e +D.F.E. d f e +DFE d f e +DfES d f e s +DFF d f f +DFG d f g +DFJ d f j +DFL d f l +DFM d f m +DFOA d f o a +DFO d f o +DFP d f p +DFPs d f p's +DFRC d f r c +DFR d f r +DFs d f's +DFS d f s +DFT d f t +DFW d f w +dfx d f x +DFX d f x +DGAC d g a c +DGA d g a +DGALS d g a l s +DGAP d g a p +DGB d g b +DGCA d g c a +DGC d g c +DGCR d g c r +D&G d and g +D. G. d g +D.G. d g +D. G. E. d g e +DGE d g e +DGERT d g e r t +DGF d g f +DGFI d g f i +DGH d g h +DGK d g k +D. G. L. R. d g l r +DGM d g m +DGMO d g m o +DGP d g p +DGPS d g p s +DGR d g r +Dha d h a +DHA d h a +DHB d h b +DHBs d h b's +DHCP d h c p +D&H d and h +dh d h +D. H. d h +D.H. d h +DH- d h +DH d h +DHEA d h e a +d'HEC d h e c +dhe d h e +DHE d h e +Dheku d h e k u +DHFS d h f s +DHHC d h h c +DHH d h h +DHL d h l +DHMO d h m o +DHNC d h n c +D.H.N. d h n +DHNS d h n s +Dhod d h o d +dhol d h o l +Dhol d h o l +Dhoo d h o o +Dhor d h o r +Dhou d h o u +DHP d h p +DHPH d h p h +DHQ d h q +dhr d h r +D.H.R. d h r +DHRS d h r s +DHS d h s +DHSs d h s's +DHSS d h s s +DHT d h t +DHV d h v +DHW d h w +DHX d h x +D.I. d i +DIFC d i f c +Dif d i f +DIF d i f +Dir d i r +DIR d i r +Dirs d i r's +DISD d i s d +div d i v +Div d i v +DIV d i v +DIW d i w +DIX d i x +diy d i y +DiY d i y +D.I.Y. d i y +DIY d i y +Diyn d i y n +DIYs d i y's +D. J. A. d j a +Djam d j a m +dj d j +D. J. d j +D.J. d j +DJ- d j +DJ d j +DJ'ed d j e d +Djem d j e m +D.J.H. d j h +Dji d j i +DJI d j i +D.J.N. d j n +Djo d j o +D'Jok d j o k +djr d j r +DJR d j r +D.J.s d j's +DJ's d j's +DJs d j's +D. J. S. d j s +D.J.T. d j t +DjVu d j v u +DJVU d j v u +DjVus d j v u's +D. J. Y. d j y +DKC d k c +D. K. d k +D.K. d k +DKI d k i +DKIM d k i m +DKK d k k +DKNY d k n y +DKPP d k p p +D. K. R. d k r +DKW d k w +dla d l a +DLA d l a +DLC d l c +DLD d l d +dl d l +D. L. d l +D.L. d l +DL- d l +DL d l +DLE d l e +DLEG- d l e g +D.L.F. d l f +DLF- d l f +DLF d l f +DLFN d l f n +DLG d l g +D. L. H. d l h +DLHS- d l h s +dlia d l i a +DLI d l i +DLJ d l j +DLJ's d l j's +DLK d l k +DLL d l l +DL&LR d l and l r +DLL's d l l's +DLLs d l l's +DLM d l m +DLMPST d l m p s t +DLNA- d l n a +dlo d l o +DLP- d l p +DLP d l p +DLPFC d l p f c +DLP's d l p's +DLR d l r +DLS d l s +DLSS d l s s +DLSU d l s u +DLSU's d l s u's +DLT- d l t +DLT d l t +DL&W d l and w +DLZ d l z +DMAA d m a a +DMA d m a +DMBA- d m b a +DMB d m b +DMBT d m b t +DMCA d m c a +DMCC d m c c +DMC- d m c +DMC d m c +DMCM d m c m +D.M.C.'s d m c's +D&M d and m +DMD d m d +DMDK d m d k +D. M. d m +D.M. d m +DM- d m +DM d m +DMDS d m d s +DMed d m e d +DM&E d m and e +DME d m e +DMF d m f +DMG d m g +DMGT d m g t +DMHS d m h s +DMI d m i +DMK d m k +DML d m l +DMN d m n +DMO d m o +DMOFD d m o f d +DMOZ d m o z +DMP d m p +DMRC d m r c +DMR d m r +DMRG d m r g +DMs d m's +DMS- d m s +DMS d m s +DMSII d m s i i +DMT- d m t +DMT d m t +DMTF d m t f +DMU d m u +DMUs d m u's +DMV d m v +DMX d m x +DMZ d m z +Dna d n a +DnA d n a +DNA- d n a +DNA d n a +DNADTC d n a d t c +DNAITV d n a i t v +DNAJA d n a j a +DNAJC d n a j c +DnaJ d n a j +DNAN d n a n +DNB d n b +DNBY d n b y +DNC d n c +DNCG d n c g +DND d n d +D. N. d n +D.N. d n +Dnepr d n e p r +dnes d n e s +Dnes d n e's +DNES d n e s +DNF d n f +DNF's d n f's +DNFs d n f's +DNG d n g +DNJO d n j o +DNKA d n k a +DNK d n k +DNM d n m +DNMT d n m t +DNP d n p +DNQ d n q +DNR d n r +DNSBLs d n s b l's +dns d n s +D.N.S. d n s +DNS d n s +DNSS d n s s +DNT d n t +DNU d n u +DNV- d n v +DNVP d n v p +DNX- d n x +D.O.A. d o a +D.O.A.'s d o a's +DOAS d o a s +D.O.B. d o b +D.O.C. d o c +DOCG d o c g +dod d o d +Dod d o d +DoD d o d +DOD d o d +D. O. d o +D.O. d o +DOD's d o d's +D.O.E. d o e +DOGNY d o g n y +DOHC d o h c +D'oh d o h +DOJ d o j +DOJ's d o j's +dok d o k +Dok- d o k +Dok d o k +domt d o m t +Domt d o m t +Dop d o p +DOP d o p +DOPR d o p r +DOSAAF d o s a a f +DOTC d o t c +dotCMS d o t c m s +DOTD d o t d +DOTP d o t p +DotSVN d o t s v n +Douw d o u w +DOW d o w +DOXO d o x o +dozd d o z d +d'OZ d o z +Dozhd d o z h d +DPAA's d p a a's +DPA d p a +DPB d p b +DPD d p d +dp- d p +dp d p +D. P. d p +D.P. d p +DP- d p +DP d p +DPF- d p f +DPG d p g +DPJ- d p j +DPJ d p j +DPMA d p m a +DPM d p m +DPN d p n +DPNM d p n m +DPO d p o +dpon d p o n +DPP d p p +DPP's d p p's +DPPX d p p x +DPR d p r +DPRK- d p r k +DPRK d p r k +DPRK's d p r k's +DProf d p r o f +DPRP d p r p +DPRT d p r t +DPs d p's +DPS d p s +DPSS d p s s +DPSSL d p s s l +DPT d p t +DPW d p w +DPW's d p w's +DPX d p x +DPYS d p y s +D.Q. d q +DQOY d q o y +DQV d q v +DRB d r b +DRBG d r b g +DRB's d r b's +DRCA d r c a +drc d r c +DRC- d r c +DRC d r c +DRDA d r d a +DRDC d r d c +DRD d r d +Dr. doctor +D. R. d r +D.R. d r +DRG d r g +dri d r i +Dri d r i +DRI d r i +DRIs d r i's +DRITTES d r i t t e s +Driu d r i u +Driv d r i v +DRJTBC d r j t b c +DRK d r k +DRL d r l +DRLR d r l r +DRMC d r m c +D.R.M. d r m +DRM- d r m +DRM d r m +Drnis d r n i's +Dro d r o +DRO d r o +DRP d r p +Drs. doctors +DR's d r's +DRS d r s +Drska d r s k a +DRT d r t +druj d r u j +DSA d s a +DsbA d s b a +DS&BB d s and b b +DSB- d s b +DSB d s b +DSBL d s b l +D.S.C. d s c +DSC- d s c +DSC d s c +D.S.C.H. d s c h +DSCM d s c m +DSCR d s c r +DSCS d s c s +DSDB d s d b +DSD d s d +DSDM d s d m +dsDNA d s d n a +DSDP d s d p +ds d s +Ds d's +D. S. d s +D.S. d s +DS- d s +DS d s +DSDS- d s d s +DSDS d s d s +DSG d s g +DSGi d s g i +DSIC d s i c +dsi d s i +DSi d s i +DSI d s i +DSIF d s i f +DSIR d s i r +DSJ d s j +dsl d s l +D. S. L. d s l +DSL- d s l +DSL d s l +DSLR d s l r +DSMB d s m b +D.S.M. d s m +DSM- d s m +DSM d s m +DSMs d s m's +dsn d s n +D&SNGRR d and s n g r r +DSNP d s n p +D.S.O. d s o +DSO d s o +DSPA d s p a +DSP d s p +DSR- d s r +DSR d s r +DS's d s's +DSS d s s +DST d s t +dsu' d s u +DSU d s u +DSV d s v +DSW d s w +DSX d s x +DSZG d s z g +D.T.A. d t a +DTA- d t a +DTA d t a +DTC d t c +DTC's d t c's +DTCs d t c's +DTDC d t d c +DTD d t d +DTDs d t d's +D. T. d t +D.T. d t +DT&E d t and e +DTE d t e +DTEK d t e k +DTES d t e s +DTG d t g +DTH d t h +DTIC d t i c +DT&I d t and i +DTI d t i +DTLA d t l a +dtl d t l +DTL d t l +DTLLS d t l l s +DTLS d t l s +DTM d t m +DTO d t o +DTPA d t p a +DTPC d t p c +DTRA d t r a +DTR d t r +DTRS d t r s +DTs d t's +DTS- d t s +DTS d t s +DTT d t t +D.T.U. d t u +DTU d t u +DTV d t v +dty d t y +D.U. d u +dva d v a +Dva d v a +DVA d v a +DVB- d v b +DVB d v b +DVBE d v b e +DVBIC d v b i c +DVC d v c +DVCs d v c's +DVCS d v c s +dvd d v d +DVD- d v d +DVD, d v d +DVD d v d +DVD's d v d's +DVDs d v d's +dv d v +D. V. d v +D.V. d v +DV- d v +DV d v +dve d v e +dvfb d v f b +DVFS d v f s +DVGS d v g s +DVHS d v h s +dvi d v i +DVI d v i +dvije d v i j e +Dvin d v i n +Dvir d v i r +DVLA d v l a +DVLA's d v l a's +DVL- d v l +DVL d v l +DVM d v m +DVP d v p +DVR d v r +DVRJRBC d v r j r b c +DVR's d v r's +DVSC d v s c +DVS d v s +DVT d v t +DVV d v v +DVVT d v v t +DVWK d v w k +Dwa d w a +DWA d w a +D&W d and w +DWDM- d w d m +D. W. d w +D.W. d w +Dwedw d w e d w +DWET d w e t +DWF d w f +DWGD- d w g d +DWG d w g +D.W.H. d w h +Dwi d w i +DWI d w i +DWM d w m +DWN d w n +DWNU d w n u +DWP d w p +dwr d w r +DWR d w r +DWRFC d w r f c +DWS d w s +DWT d w t +DWTS d w t s +dwur d w u r +DWWX- d w w x +dxa d x a +DXA d x a +dx d x +D.X. d x +DX- d x +DX d x +DXED- d x e d +DXED d x e d +DXers d x e r's +DXII d x i i +DXK d x k +DXP- d x p +DXR d x r +DXZ d x z +Dydd d y d d +dy d y +Dy d y +D. Y. d y +DY d y +DYFI d y f i +DYIS d y i s +DYNC d y n c +dypl d y p l +dyr d y r +DYSP d y s p +DZBB d z b b +DZB d z b +dz d z +DZEC d z e c +DZEM d z e m +dzis d z i s +DZNE d z n e +Dzog d z o g +Dzor d z o r +DZRH d z r h +DZRH's d z r h's +DZS d z s +Dzus d z u's +DZXL d z x l +DZZ d z z +EAAB e a a b +EAA e a a +EAAP e a a p +E. A. C. e a c +EAC e a c +EACEF e a c e f +EACS e a c s +EAD e a d +E. A. e a +E.A. e a +EAF e a f +EAGE e a g e +E.A.H. e a h +EAH e a h +EAI e a i +EAIE e a i e +E. A. J. e a j +EAMs e a m's +EAP- e a p +EAP e a p +E.A.R.L. e a r l +E.A.R.T.H. e a r t h +EASA e a s a +EASB e a s b +eas e a s +Eas e a's +EA's e a's +EAS e a s +Eav e a v +EAX e a x +E. B. e b +E.B. e b +EBL e b l +Ebn e b n +EBNF e b n f +EBOV e b o v +E&BR e and b r +EBR- e b r +EBR e b r +Ebru e b r u +EBS e b s +EBSL e b s l +EBT e b t +EBU e b u +EBV e b v +E.B.W. e b w +EBWs e b w's +EBZR e b z r +ECAC- e c a c +ECAC e c a c +ECA e c a +ECAR e c a r +ECB, e c b +ECB e c b +ECBJ e c b j +ECBL e c b l +ECCA e c c a +ECCC e c c c +ecc e c c +ECC e c c +ECCI e c c i +Eccl e c c l +ECCRFA e c c r f a +ECCW e c c w +ECD e c d +ec e c +Ec e c +E. C. e c +E.C. e c +EC- e c +EC e c +ece e c e +Ece e c e +ECE e c e +Ecem e c e m +ECFA e c f a +ECF e c f +ECFR e c f r +ECFU e c f u +ecg e c g +ECG e c g +ECH e c h +ECHL e c h l +ECHR e c h r +ECI e c i +ECJ e c j +ECLAC e c l a c +ECLA e c l a +ECLC e c l c +ECL e c l +ECLI e c l i +ECLSS e c l s s +ecma e c m a +ECMA e c m a +ECMC e c m c +ECMDB e c m d b +E.C.M. e c m +ECM e c m +ECMI e c m i +ECML e c m l +ECMLRUS e c m l r u s +ECMWF e c m w f +ECN e c n +ECNP e c n p +ECNs e c n's +ECNS e c n s +ECNZ e c n z +ECP&DA's e c p and d a's +ECP e c p +ECPR e c p r +ECQ e c q +ECR e c r +eCRM e c r m +ECRR e c r r +ECR's e c r's +ECSA e c s a +EC's e c's +E. C. S. e c s +E.C.S. e c s +ECS e c s +ect e c t +E. C. T. e c t +ECT e c t +ECTLO e c t l o +ECTN e c t n +Ecto- e c t o +ECTS e c t s +ECV e c v +ECVs e c v's +ECWA e c w a +ECW e c w +edb e d b +EDB e d b +EDC e d c +E. D. e d +E.D. e d +EDF e d f +Edh e d h +EDH e d h +edhe e d h e +EDHF e d h f +EDIB e d i b +EDI e d i +EDK e d k +EDL e d l +EDMA e d m a +EDM- e d m +EDM e d m +EDMs e d m's +EDMW e d m w +Edmx e d m x +edn e d n +Edn e d n +EDN e d n +EDP e d p +EDQH e d q h +E.D.R. e d r +EDRP e d r p +EDSA e d s a +EDSC e d s c +eds e d s +Eds e d's +E. D. S. e d s +EDS- e d s +EDS e d s +EDSP e d s p +EDTA e d t a +EDT e d t +EDVAC e d v a c +EDWDO e d w d o +Edw e d w +EDXRD e d x r d +EEA e e a +EEAF e e a f +EEAS e e a s +EEBC e e b c +Eeb e e b +EEC e e c +EECS e e c s +E. E. e e +E.E. e e +Eef e e f +EEF e e f +Eega e e g a +EEI e e i +EEI's e e i's +EEJF e e j f +EELC e e l c +eene e e n e +EEOC e e o c +EEPCO e e p c o +eep e e p +EEP e e p +EERC e e r c +EERI e e r i +Ees e e's +EES e e s +EEST e e s t +EEUU e e u u +eeuw e e u w +Eeuw e e u w +EEZs e e z's +EFA e f a +EFAS e f a s +EFCC e f c c +efc e f c +EFC e f c +EFCJ e f c j +EFD e f d +ef e f +Ef e f +E. F. e f +E.F. e f +EF- e f +EF e f +E. F. J. e f j +E. F. K. e f k +EFL e f l +EFM e f m +EFNB e f n b +EFPA e f p a +EFP e f p +EFQUEL e f q u e l +EFR e f r +EFRU e f r u +EFSA e f s a +eFS e f s +Efs e f's +EFSET e f s e t +EFTPOS e f t p o s +EFUA e f u a +Efu e f u +EFV e f v +EFVR e f v r +EFVS e f v s +E. F. W. e f w +EFX e f x +EFY e f y +E. G. A. e g a +EGA e g a +Egba e g b a +EGBA e g b a +EGBT e g b t +EGC e g c +EGD e g d +EGF- e g f +EGF e g f +egfl e g f l +EGFL e g f l +e.g. for example +e. g. for example +EGFR e g f r +EG&G e g and g +E. G. H. e g h +Egi e g i +EGL e g l +EGM e g m +E&GR e and g r +EGR e g r +EGREM e g r e m +EGSC e g s c +EGX e g x +egy e g y +EGY e g y +Egyl e g y l +EHAAT e h a a t +EHAs e h a's +EHC e h c +EHCI e h c i +EHD e h d +E. H. e h +E.H. e h +ehf e h f +EHF e h f +E. H. M. e h m +EHP e h p +EHRC e h r c +EHR e h r +EHS e h s +EHSH e h s h +Ehttp e h t t p +EHV- e h v +EIA- e i a +EIA e i a +EIC e i c +EiCs e i c's +EIC's e i c's +E. I. e i +E.I. e i +EIES e i e s +eIF e i f +EIF e i f +EIFF e i f f +EIF's e i f's +EIT e i t +EITI e i t i +Eitr e i t r +EITs e i t's +EIU e i u +E. J. A. e j a +EJB e j b +EJBs e j b's +EJC e j c +ej e j +E. J. e j +E.J. e j +EJ e j +E. J. H. e j h +E.J.H. e j h +EJH e j h +EJHL e j h l +EJMA e j m a +E. J. M. e j m +EJML e j m l +E. J. R. e j r +E.J.S. e j s +EJSM e j s m +EJV e j v +EKB e k b +EKBO e k b o +EKCO's e k c o's +EKD e k d +E. K. e k +E.K. e k +EKF e k f +EKG e k g +eki e k i +EKIZ e k i z +EKL e k l +EKLR e k l r +Ekma e k m a +EKNU e k n u +Ekow e k o w +eks e k s +Eks e k's +EKS- e k s +EKS e k s +E. K. U. e k u +EKU e k u +E. L. C. e l c +E. L. e l +E.L. e l +ELH e l h +ELTs e l t's +ELUs e l u's +ELV e l v +EMA e m a +EMAS e m a s +Embd e m b d +EMB- e m b +EMB e m b +eMC e m c +EMC e m c +EMD e m d +EMD's e m d's +EMDs e m d's +EMDS e m d s +Emea e m e a +EMEA e m e a +E&M e and m +E. M. e m +E.M. e m +EMF e m f +Emge e m g e +EMG- e m g +EMG e m g +EML- e m l +EMLL e m l l +E.M.M. e m m +EMNLP e m n l p +EMP- e m p +EMP e m p +emra e m r a +EMR e m r +EMSC e m s c +E.M.S. e m s +EMTE e m t e +EMT e m t +EMV e m v +E&NA e and n a +Enas e n a's +ENAS e n a s +EnBW e n b w +ENC e n c +EncFS e n c f s +Encyc e n c y c +E.N.D. e n d +ENEC e n e c +E. N. e n +E.N. e n +Enes e n e's +engl e n g l +Engl e n g l +ENGM e n g m +Engr e n g r +Eni e n i +ENI e n i +enn e n n +Enn e n n +ENP e n p +ENR e n r +ens e n s +Ens e n's +ENS e n s +Enso e n s o +ENSO e n s o +ENTJs e n t j's +ENTP e n t p +ENTV e n t v +Env e n v +ENV e n v +EOC e o c +EOC's e o c's +EoD e o d +EOD e o d +E. O. e o +E.O. e o +EOFFTV e o f f t v +EOG e o g +Eois e o i's +EOKA e o k a +eok e o k +Eole e o l e +Eol e o l +EOL e o l +Eolss e o l s's +Eom e o m +EOOW e o o w +eop e o p +eoptta e o p t t a +Eora e o r a +EOR e o r +Eorl e o r l +Eors e o r's +EOS- e o s +EOSHD e o s h d +EOTC e o t c +EOTDC e o t d c +EOT- e o t +EOU e o u +E.O.W. e o w +E&P e and p +Epe e p e +E. P. E. e p e +EPEG e p e g +ep e p +Ep e p +E. P. e p +E.P. e p +EP- e p +EP, e p +EP e p +EPF e p f +EPFL e p f l +EPG e p g +EPHAR e p h a r +EPHB e p h b +Eph e p h +EPH e p h +Ephs e p h's +EPIA e p i a +epi e p i +Epi e p i +EPI e p i +E.P.J. e p j +EPL e p l +EPLF e p l f +EPM- e p m +EPM e p m +EPOC e p o c +ePO e p o +EPO e p o +epos' e p o's +epos e p o s +Epos e p o's +EPRDF e p r d f +EPR e p r +EPRI e p r i +EPRO e p r o +EPSC e p s c +E.P.s e p's +EPT e p t +EPV e p v +EPZA e p z a +EPZ e p z +EQAO e q a o +eq e q +Eq e q +EQ- e q +EQ e q +EQG e q g +EQN e q n +EQP e q p +EQS e q s +EQT e q t +E.R.A. e r a +ERAF e r a f +ERAU e r a u +ERCB e r c b +ERC e r c +ERCES e r c e s +ERCIM e r c i m +ERCS e r c s +E. R. D. e r d +E. R. e r +E.R. e r +ERF e r f +ERH e r h +ERJs e r j's +ERK e r k +ERP e r p +ERPM e r p m +ERPMRP e r p m r p +ERRC e r r c +E. R. R. e r r +E.R.R. e r r +ERSAR e r s a r +ERT e r t +Eru e r u +ERU e r u +ERUUF e r u u f +Eruv e r u v +Erv e r v +ERW e r w +ES3 e s three +Esa e s a +E.S.A. e s a +ESA- e s a +ESA e s a +Esam e s a m +Esa's e s a's +ESA's e s a's +Esat e s a t +ESB e s b +ESC e s c +ESCHL e s c h l +ESCs e s c's +Esd e s d +ESD e s d +ESDI e s d i +E.S.E. e s e +E. S. e s +E.S. e s +ESF e s f +ESFG e s f g +E.S.G. e s g +ESG e s g +ESGR e s g r +esi e s i +ESI e s i +ESL e s l +ESM e s m +ESMLC e s m l c +ESMT e s m t +ESNAGI e s n a g i +ESN e s n +ESPA e s p a +ESPC e s p c +ESPD e s p d +ESP e s p +espn e s p n +ESPN, e s p n +ESPN e s p n +ESPNhttp e s p n h t t p +ESPN's e s p n's +ESPNU e s p n u +ESPNW's e s p n w's +esq e s q +Esq e s q +Esq. esquire +Esraa e s r a a +Esra e s r a +ESRB e s r b +ESRC e s r c +Esref e s r e f +ESR- e s r +ESR e s r +ESRF e s r f +Esri e s r i +ESRI- e s r i +ESRI e s r i +ESSR e s s r +estd e s t d +Estd e s t d +EST e s t +ESTP e s t p +esu e s u +ESU e s u +ESU's e s u's +Esva e s v a +ESV e s v +ESWAT e s w a t +ESW e s w +Etad e t a d +ETA e t a +ETAF e t a f +ETA's e t a's +ETASU e t a s u +ETB e t b +ETBs e t b's +E.T.C. e t c +ETCO's e t c o's +ETCS e t c s +ETCSL e t c s l +ETD e t d +E. T. e t +E.T. e t +ETF e t f +ETFs e t f's +ETFS e t f s +Et'hem e t h e m +ETH e t h +Eti e t i +ETI e t i +ETIM e t i m +ETIM's e t i m's +etj e t j +ETJ e t j +ETL e t l +ETNZ e t n z +Eto e t o +ETO e t o +ETRM e t r m +ETRS e t r s +ETSEIB e t s e i b +ets e t s +Ets e t's +ET's e t's +ETs e t's +ETS- e t s +ETS e t s +Etsi e t s i +ETSI e t s i +ett e t t +Ett e t t +ETT e t t +Ettre e t t r e +Etts e t t's +ETU e t u +ETV e t v +Etz e t z +EUA e u a +EUB e u b +EUBs e u b's +EUBS e u b s +EUCAP e u c a p +EUCE e u c e +EUC- e u c +EUC e u c +EUCOM's e u c o m's +EUDEC e u d e c +EUD e u d +E. U. e u +E.U. e u +EU- e u +EU e u +eup e u p +EUP e u p +Eurwg e u r w g +EUSA e u s a +Euse e u s e +EU's e u's +EUS e u s +EUSEW e u s e w +Eusi e u s i +EUV e u v +EUX e u x +EVAAP e v a a p +EvaGT e v a g t +EVDC e v d c +EVD e v d +EVDO e v d o +E. V. e v +E.V. e v +EVEX e v e x +EVIT e v i t +evl e v l +EVL e v l +EVM e v m +Evna e v n a +EVN e v n +EVNS e v n s +evnt e v n t +E.V.O. e v o +E.V.O.L. e v o l +EVP e v p +EVR e v r +EVs e v's +EVS- e v s +EVS e v s +evv e v v +EVV e v v +Ewa's e w a's +EWD e w d +EWEB e w e b +E. W. e w +E.W. e w +EWF- e w f +EWF e w f +EWF's e w f's +Ewha e w h a +EWH e w h +E.W.K. e w k +EWLA e w l a +EWL e w l +EWO e w o +EWOs e w o's +EWP e w p +EW's e w's +EWS- e w s +EWS e w s +EWTN e w t n +Ewu e w u +EWU e w u +excl e x c l +exd e x d +E.X. e x +exhb e x h b +exh e x h +EXI e x i +EXIF e x i f +EXIN e x i n +EXL e x l +EXLM e x l m +E. X. M. e x m +Exptl e x p t l +EXR e x r +exsul e x s u l +EXT e x t +EYA e y a +EYCN e y c n +EYD e y d +E&Y e and y +E. Y. e y +EY e y +Eyk e y k +eyu e y u +Ezaa e z a a +ez e z +Ez e z +EZ- e z +EZ e z +Ezh e z h +EZH e z h +EZO e z o +EZR e z r +EZTV e z t v +Ezu e z u +EZW e z w +Ezy e z y +Faaa f a a a +faa f a a +Faa f a a +FAA- f a a +FAA f a a +faama f a a m a +FAANP f a a n p +FabH f a b h +FABM f a b m +FabR f a b r +F.A.B.'s f a b's +F.A.C.E. f a c e +FACR f a c r +F.A.C.S. f a c s +FADD f a d d +Fadl f a d l +FADS f a d s +FAEM f a e m +fa f a +Fa f a +F. A. f a +F.A. f a +FA- f a +FA f a +Faf f a f +FAFG f a f g +Fafhrd f a f h r d +FAFL f a f l +F. A. G. f a g +F.A.H. f a h +fai f a i +Fai f a i +FAI- f a i +FAI f a i +FAIM f a i m +FAIP f a i p +FAIPs f a i p's +Faiq f a i q +fajn f a j n +FAK f a k +Fal f a l +FAL f a l +F.A.M.E. f a m e +F.A.M. f a m +FAQ f a q +FARC's f a r c's +FARDC f a r d c +F.A.R. f a r +farw f a r w +FASA f a s a +FASB f a s b +fasc f a s c +Fasc f a s c +fas f a s +Fas f a's +FAs f a's +F.A.S. f a s +FAS f a s +FASG f a s g +FasL f a s l +FASRE f a s r e +FATF f a t f +FAUP f a u p +FAVC f a v c +FBA f b a +FBD f b d +FBFA f b f a +F. B. f b +F.B. f b +FB- f b +FB f b +F.B.G.s f b g's +F.B.I. f b i +FBI- f b i +FBI f b i +FBI's f b i's +FBK f b k +FBLA- f b l a +FBL f b l +FBMW f b m w +FBN f b n +FBO f b o +FBOs f b o's +FBPA f b p a +FBR f b r +FBS f b s +FBSN f b s n +FBSU f b s u +FBT f b t +FCA f c a +fcb f c b +FCB f c b +FCBF f c b f +F.C.B.I. f c b i +FCBL f c b l +FCBs f c b's +F.C.C. f c c +FCC f c c +FCD f c d +F&C f and c +fc f c +F. C. f c +F.C. f c +FCF f c f +FCGR f c g r +F. C. H. f c h +F.C.H.L. f c h l +FCIAC f c i a c +FCIC f c i c +FCI f c i +FCIP f c i p +FCK f c k +FCL f c l +FCN- f c n +FCNM f c n m +FCNZ f c n z +FCO f c o +FCP f c p +F.C.P.R.E.M.I.X. f c p r e m i x +FCPS f c p s +FCRLA f c r l a +F.C.'s f c's +FC's f c's +FCS f c s +FCT f c t +FCU f c u +FCUL f c u l +FCV- f c v +FCV f c v +F. C. W. f c w +FCW f c w +FCW's f c w's +FDA f d a +FDCC f d c c +FD&C f d and c +FDCPA f d c p a +FDCPAs f d c p a's +FDD f d d +FDDI f d d i +FDE f d e +F. D. f d +F.D. f d +FDFL f d f l +FDGB- f d g b +FDGB f d g b +FDG f d g +FDICs f d i c's +FdI f d i +FDI f d i +FDIO f d i o +FDJ f d j +FDL f d l +FDLT f d l t +FDM f d m +FDMNES f d m n e s +FDNY f d n y +FDP f d p +FDP's f d p's +FDQ, f d q +FDR f d r +FDU f d u +F. E. f e +F.E. f e +FEFK f e f k +FEFSI f e f s i +FEFY f e f y +FEG f e g +FEGs f e g's +FEP f e p +F.E.R.S. f e r s +Fes f e's +FES f e s +FFA f f a +FFAK f f a k +ffc f f c +FFC f f c +FFC's f f c's +FFDO f f d o +ff f f +F. F. f f +F.F. f f +FF- f f +FF f f +FFF f f f +FFG f f g +FFH f f h +FFI f f i +FFK- f f k +FFK f f k +FFL f f l +FFL's f f l's +FFME f f m e +ffm f f m +FFOM f f o m +FFP f f p +FFPIR f f p i r +FFR f f r +FFSA f f s a +FF's f f's +FFs f f's +FFS f f s +FFU f f u +FFV f f v +FFWD f f w d +FGCU f g c u +FGFA f g f a +FGF f g f +F. G. f g +F.G. f g +FGFR f g f r +FGID f g i d +fgk f g k +FGL f g l +FGM f g m +FGR f g r +FGs f g's +FGS f g s +FGU f g u +FGV f g v +F. H. A. f h a +FHA- f h a +FHA f h a +FHAR f h a r +F. H. C. f h c +FHFA f h f a +F. H. f h +F.H. f h +FHI's f h i's +FHL f h l +fhm f h m +FHM f h m +FHM's f h m's +FHQ f h q +FHSAA f h s a a +FHS f h s +FHWA f h w a +F. H. W. f h w +FIAC f i a c +fiadh f i a d h +fia f i a +FIA f i a +FIAPO f i a p o +FIA's f i a's +FIAS f i a s +FIBCO f i b c o +F. I. B. f i b +FiBL f i b l +FIBT f i b t +FIC f i c +Fi'd f i d +FIDH f i d h +F.I f +F&I f and i +F. I. f i +F.I. f i +FIFTA f i f t a +FIGC f i g c +Figl f i g l +FIH f i h +FIH's f i h's +fija f i j a +FIJL f i j l +FIK f i k +F.I.M. f i m +FIM f i m +FIMI- f i m i +FIMI f i m i +Fio f i o +FiO f i o +FIO f i o +F.I.R. f i r +F.I.T.T. f i t t +FIU f i u +FIVB f i v b +F. J. A. f j a +FJE f j e +F. J. f j +F.J. f j +F.J.G. f j g +FJK f j k +F. J. M. f j m +FKA f k a +FKBP f k b p +F. K. f k +F.K. f k +FKK f k k +FKL's f k l's +FKS f k s +FKY f k y +Fla f l a +FLB f l b +FLCL f l c l +FLCS f l c s +FLD f l d +FLDS f l d s +FLEC f l e c +FLE f l e +Fles f l e's +FLEST f l e s t +FLETC f l e t c +fl f l +F. L. f l +F.L. f l +FL- f l +FL f l +FLH f l h +fli f l i +FLI f l i +FLI's f l i's +FLL f l l +F.L.M. f l m +FLM f l m +FLNB f l n b +FLN f l n +FLP f l p +FLPP f l p p +FLQ f l q +FLRA f l r a +FLR f l r +FLRL f l r l +FLS f l s +FLSW f l s w +FLT f l t +FLTK f l t k +FLTR f l t r +FLTS f l t s +FLV f l v +FLW f l w +flyr f l y r +FMA f m a +FMARS f m a r s +FMC f m c +FMCG f m c g +FMCSA f m c s a +FMCS f m c s +FMF f m f +fm f m +F. M. f m +F.M. f m +FM- f m +FM f m +FMH f m h +FMHS f m h s +FMI f m i +FMJD f m j d +FMLN f m l n +FMM f m m +FMN f m n +FMNJ f m n j +FMNP f m n p +FMPA's f m p a's +FMP f m p +FMQB f m q b +FMR f m r +fMRI f m r i +FMRP f m r p +FMRU f m r u +FM's f m's +FMs f m's +FMS- f m s +FMS f m s +FMSF's f m s f's +FMSO f m s o +FMV f m v +FMW f m w +FMX f m x +FMY f m y +FNAC f n a c +F. N. A. f n a +FNA f n a +FNB f n b +FNC f n c +F. N. D. f n d +FNE f n e +F. N. f n +F.N. f n +FNIC f n i c +FNI f n i +FNLA f n l a +FNLC f n l c +FNLC's f n l c's +FNL f n l +FNO f n o +FNP f n p +FNPI f n p i +FNSEA f n s e a +FNU f n u d s a +FOF f o f +F. O. f o +FPA- f p a +FPA f p a +FPB f p b +FPC f p c +FPCP f p c p +F.P.D. f p d +FPD- f p d +FPDs f p d's +fp f p +F. P. f p +F.P. f p +FP- f p +FP f p +FPGA f p g a +FPGAs f p g a's +FPI f p i +FPJ f p j +FPK f p k +F. P. L. f p l +FPL f p l +FPMR f p m r +FPM&SA f p m and s a +FPO f p o +FPP f p p +FPPS f p p s +F.P.R. f p r +FPR f p r +FPSC f p s c +fps f p s +FPS- f p s +FPS f p s +FPSO f p s o +FPSs f p s's +FPTP- f p t p +FPTP f p t p +FPU f p u +FQAD f q a d +F.Q. f q +FRA f r a +FRA's f r a's +FRAXA f r a x a +FRB f r b +FRBNF f r b n f +FRCC f r c c +F. R. C. f r c +FRC f r c +FRCNA f r c n a +FRCP f r c p +FRCS f r c s +FRDF f r d f +F.R.D. f r d +fr f r +F. R. f r +F.R. f r +FR- f r +FR f r +FRFS f r f s +FRFT f r f t +FRG f r g +F.R.G.S. f r g s +F.R.H.S. f r h s +F.R.I.B.A. f r i b a +FRMN f r m n +FRMR f r m r +FRMS f r m s +F. R. P. f r p +FRP f r p +FRR f r r +F.R.S.A. f r s a +FRSA f r s a +FRSC f r s c +FRSE f r s e +frs f r s +F. R. S. f r s +F.R.S. f r s +FRS f r s +FRSL f r s l +Fru f r u +FRU f r u +FRV f r v +FRWT f r w t +Frwydr f r w y d r +FSA f s a +FSB f s b +FSC- f s c +FSC f s c +FSCS f s c s +FSE f s e +fsf f s f +FSF f s f +FSFH f s f h +FSFLA f s f l a +fs f s +F. s f's +Fs f's +F. S. f s +F.S. f s +FS f s +FSFS f s f s +FSG f s g +FSGS f s g s +FSH f s h +FSIC f s i c +FSI f s i +FSIL f s i l +FSK f s k +FSKN f s k n +FSLF f s l f +FSL f s l +FSLN f s l n +FSM f s m +FSM's f s m's +fsn f s n +FSN f s n +FSO f s o +FSP f s p +FSPS f s p s +FSR f s r +FSRU f s r u +FSSB f s s b +FSS f s s +FSSM f s s m +FSSRU f s s r u +FSSs f s s's +FST f s t +FSU f s u +FSU's f s u's +FSV f s v +FTA f t a +FTAs f t a's +F.T.B. f t b +FTC f t c +FTD f t d +FTDI f t d i +FTE f t e +FTF f t f +Ft. Fort +F. T. f t +F.T. f t +FTI f t i +FTII f t i i +FTJ f t j +FTK f t k +FTL f t l +FTM f t m +FTN f t n +FTO f t o +ftp f t p +FTP f t p +FTPI f t p i +FTPL f t p l +FTRA f t r a +FTR f t r +FtsA f t s a +FT's f t's +FTS f t s +FTT f t t +FTTH f t t h +FTUC f t u c +Ftuh f t u h +FTU's f t u's +FTV f t v +FVA- f v a +F. V. B. f v b +F. V. f v +F.V. f v +FVGM f v g m +FVHS f v h s +FVJC f v j c +FVL f v l +FVP f v p +FVW f v w +FVWM f v w m +FWAA f w a a +FWA f w a +FWCD f w c d +FWD f w d +FWE f w e +F&W f and w +F. W. f w +F.W. f w +FWH f w h +FWI f w i +FWMW f w m w +FWO f w o +fwr f w r +F.W.R. f w r +F. W. S. f w s +F.W.S. f w s +FWS f w s +FWX f w x +FXCM's f x c m's +FXE f x e +FXFL f x f l +fx f x +FX f x +FXI f x i +F.X.R. f x r +FX's f x's +FXS f x s +FXX f x x +FXX's f x x's +FXXT f x x t +Fyb f y b +FYCO f y c o +F.Y. f y +F.Y.I. f y i +FYI f y i +Fyn f y n +FYN f y n +FYP f y p +fyr f y r +F.Y.R. f y r +FYR f y r +FYT f y t +FYU f y u +Fyw f y w +FZCI f z c i +FZD f z d +F.Z. f z +FZG f z g +FZJD f z j d +FZKA f z k a +Gaac g a a c +GAAF g a a f +G.A.A. g a a +G. A. g a +G.A. g a +G.A.R. g a r +GASL g a s l +GATA g a t a +GATV g a t v +GATX g a t x +G. A. W. g a w +GBA g b a +GBAG g b a g +GBASE g b a s e +GBA's g b a's +GBB g b b +GBBI g b b i +GBCA g b c a +GBC g b c +GBCM g b c m +GBD g b d +Gbe g b e +GbE g b e +GBE g b e +G. B. g b +G.B. g b +GBGC g b g c +GBGM g b g m +GBH g b h +GBIB g b i b +GBIF g b i f +GBI g b i +GBK g b k +GBLA- g b l a +GBL g b l +GBMA g b m a +G. B. M. g b m +GB&NDR g b and n d r +GBP g b p +GBR g b r +GB's g b's +GBS g b s +GBV- g b v +GBV g b v +GBX g b x +GBYLA g b y l a +GCAC g c a c +GCAF g c a f +GCA g c a +G. C. B. g c b +GCB g c b +GCC g c c +GCCN g c c n +GCC's g c c's +GCCS g c c s +GCD g c d +G.C.E. g c e +GCEP g c e p +GCF g c f +G&C g and c +G. C. g c +G.C. g c +GCHQ g c h q +GCI g c i +GCIP g c i p +GCIT's g c i t's +GCIV g c i v +GCJ g c j +GCL g c l +GCMC g c m c +G.C.M. g c m +GCM g c m +GCMG g c m g +GCN g c n +GCNT g c n t +GCOEC g c o e c +GCOS g c o s +GCPD g c p d +GCPH g c p h +GCR- g c r +GCR g c r +GCRIO g c r i o +GCSE g c s e +GCSEs g c s e's +GC&SF g c and s f +GCS g c s +GCSI g c s i +GCT- g c t +GCU g c u +GCV g c v +GCVO g c v o +GCVS g c v s +GDA g d a +Gdal g d a l +GDAL g d a l +GDAP g d a p +GDB g d b +GDC g d c +GDD g d d +GDDM g d d m +GDDR g d d r +Gde g d e +G. D. F. g d f +GDF g d f +gd g d +G. D. g d +G.D. g d +GD g d +GDI g d i +G.D.L. g d l +GDNFR- g d n f r +GDN g d n +G.D.O.D. g d o d +Gdow g d o w +G.D.P. g d p +GDP- g d p +GDP g d p +GDPs g d p's +gdr g d r +GDR g d r +GDR's g d r's +GDS g d s +GDST g d s t +GDT g d t +GDV g d v +Gebr g e b r +Gebt g e b t +GEC g e c +Ged g e d +GED g e d +GEF g e f +GEGB g e g b +ge g e +G. E. g e +G.E. g e +GE- g e +GE g e +GEHA g e h a +GEHX g e h x +Gek g e k +G.E.M. g e m +Gen. general +G.E.O. g e o +G. E. P. g e p +geq g e q +Geu g e u +GEU g e u +Gev g e v +GeV g e v +GEV g e v +Gfa g f a +GFA g f a +GF&A g f and a +GFC g f c +GFD g f d +GFDL g f d l +GFE g f e +GFF g f f +G. F. g f +G.F. g f +GFM g f m +GFP- g f p +GFP g f p +G. F. R. g f r +GFR g f r +GFS g f s +GFSN g f s n +GFSR g f s r +GFT g f t +GFWC g f w c +GFW g f w +GFY g f y +G. G. g g +G.G. g g +G. G. J. g g j +GGL g g l +GGM g g m +GGMRC g g m r c +GGS g g s +G.H.A. g h a +GHB g h b +GHC g h c +GHD g h d +Ghe g h e +G. H. g h +G.H. g h +GHGs g h g's +Ghir g h i r +GHK g h k +GHMC g h m c +G.H.N. g h n +G.H.Q. g h q +GHQ g h q +ghra g h r a +GHR g h r +GH&SA g h and s a +GHSA g h s a +GHSFHA g h s f h a +GHS g h s +GHSR g h s r +GHSU g h s u +GHU g h u +G.H.W. g h w +GICL g i c l +Giei g i e i +GIFV g i f v +G. I. g i +G.I. g i +G.J.B. g j b +G. J. g j +G.J. g j +G. K. g k +G.K. g k +Gla g l a +GLA g l a +GLB g l b +GLBT g l b t +GLBTQ g l b t q +GLC- g l c +GLC g l c +G&L g and l +G. L. g l +G.L. g l +gli g l i +Gli g l i +GLI g l i +Glis g l i's +GLK g l k +GLL g l l +GLLI g l l i +GLM g l m +glnA g l n a +G.L.O.R.Y. g l o r y +Glos g l o's +GLP- g l p +GLRS g l r s +GLS g l s +GLT g l t +GLVC g l v c +GLX- g l x +GLX g l x +GLYX- g l y x +GMAC g m a c +gma g m a +GMA- g m a +GMA g m a +GMAM g m a m +GMA's g m a's +GMAT g m a t +GMB g m b +GmbH g m b h +GMBH g m b h +GMB's g m b's +GMCC g m c c +GMC g m c +GMCL g m c l +GMC's g m c's +GMDSS g m d s s +GME g m e +Gmel g m e l +GMES g m e s +GMFF g m f f +GMF g m f +GMG g m g +gm g m +G. M. g m +G.M. g m +GM- g m +GM g m +GMG's g m g's +gmin g m i n +GMM g m m +GMO g m o +GMOs g m o's +GMPC g m p c +GMPE g m p e +GMP g m p +GMPTE g m p t e +GMR g m r +GMRL g m r l +GM's g m's +GMs g m's +GMS g m s +gmt g m t +GMT g m t +GMTV g m t v +Gmul g m u l +GNAA g n a a +GNA g n a +gnb g n b +GNB g n b +GNCC g n c c +G. N. C. g n c +GNC g n c +G. N. g n +G.N. g n +GNL g n l +GNP&BR g n p and b r +GNPDA g n p d a +gnp g n p +GNP g n p +GNPs g n p's +GNRD g n r d +GNR g n r +GNR's g n r's +GNS g n s +GNSS g n s s +GNT g n t +G.O. g o +G.O.O.D. g o o d +GOPB g o p b +G.O.P. g o p +GOP- g o p +GOP g o p +GOPIO g o p i o +GOP's g o p's +GOQ g o q +GPA- g p a +GPA g p a +GPB g p b +G.P.C. g p c +GPC g p c +GPCR g p c r +GPCRs g p c r's +GPD g p d +GPdI g p d i +GPDSC g p d s c +GPF g p f +GPFS g p f s +G&P g and p +GPG g p g +gp g p +G. P. g p +G.P. g p +GP- g p +GP g p +GPGPU g p g p u +GPI g p i +GpIIb g p i i b +GPIO g p i o +GPL g p l +GPMGs g p m g's +G.P.O. g p o +GPO g p o +GPOPS- g p o p s +GPP g p p +GPRA g p r a +GPRF g p r f +GPR g p r +GPRs g p r's +GPRS g p r s +GPSC g p s c +GPS- g p s +GPS g p s +GPT g p t +GPU g p u +GPUs g p u's +GPX g p x +G. Q. g q +GQ g q +GQNYGK g q n y g k +GQ's g q's +GQT's g q t's +G.R.A.B. g r a b +grac g r a c +gra g r a +Gra g r a +GRA g r a +Grbac g r b a c +GRB g r b +GRBs g r b's +grc g r c +GRC g r c +GRCs g r c's +GRDA g r d a +GRDC g r d c +GRDF- g r d f +G.R.D. g r d +GRD g r d +gre g r e +Gre g r e +GRE g r e +GRF g r f +GRG g r g +gr g r +G. R. g r +G.R. g r +GR- g r +GR g r +GRK g r k +G.R.L. g r l +GRL g r l +GRMG g r m g +GRM- g r m +GRM g r m +GRO g r o +GRP g r p +grrl g r r l +G. R. S. g r s +GRS g r s +GRT g r t +GRU g r u +GRX g r x +Gryf g r y f +gry g r y +Gry g r y +GRY g r y +grz g r z +GSA g s a +GSAPP g s a p p +GSAS g s a s +GSB g s b +GSC g s c +GSDF g s d f +GSD g s d +G.S.E. g s e +GSE g s e +GSFC g s f c +GSF g s f +G&S g and s +GSG- g s g +GSG g s g +Gs g's +G. S. g s +G.S. g s +GS- g s +GS g s +GSHD g s h d +GSh- g s h +GSH g s h +GSI g s i +GSIs g s i's +GSIS g s i s +GSK- g s k +GSK g s k +GSL g s l +GSLV g s l v +GSMA g s m a +GSM- g s m +GSM g s m +GSNAS g s n a s +G. S. N. g s n +GSN g s n +GSNM g s n m +GSN's g s n's +GSOC- g s o c +GSO g s o +GSOM's g s o m's +GSOp g s o p +GSPC g s p c +G. S. P. g s p +GSP g s p +GSR g s r +GSSE g s s e +GSS- g s s +GSS g s s +GSSP g s s p +GSTA g s t a +G.S.T. g s t +GST- g s t +GST g s t +GSU g s u +GSUSA g s u s a +GSVD g s v d +GSVS g s v s +GSX g s x +GTAA g t a a +GTA g t a +GTAs g t a's +GTB g t b +GTBK g t b k +GTBP g t b p +GTC g t c +GTCR g t c r +GTD g t d +GTEC g t e c +GTE g t e +GTF g t f +GTFT g t f t +GTFU g t f u +gt g t +G. T. g t +G.T. g t +GT- g t +GT g t +G. T. H. g t h +GTi g t i +GTI g t i +GTK g t k +GTL g t l +GTMO g t m o +G. T. O. g t o +GTO g t o +GTPBP g t p b p +GTP- g t p +GTP g t p +GTPR g t p r +GTRA g t r a +GTR g t r +GTRI g t r i +GTs g t's +GTS g t s +GTSV g t s v +GTT g t t +GTTP g t t p +GTU g t u +GTV- g t v +GTV g t v +GTW- g t w +GTW g t w +GTW's g t w's +GTX g t x +G. U. g u +G.U. g u +GUI's g u i's +GUIs g u i's +G.U.Y. g u y +GVA g v a +GVAV g v a v +GVBA g v b a +GVC g v c +G. V. g v +G.V. g v +GVHSS g v h s s +GVJZ g v j z +GVK g v k +GvpA g v p a +GVP- g v p +GVP g v p +GVRD g v r d +GVSU g v s u +GVT g v t +GVU g v u +GVVV g v v v +GVW g v w +GVWR g v w r +GWA g w a +GWB g w b +G. W. C. g w c +GWe g w e +Gwet g w e t +G.W.F. g w f +GWF g w f +gw g w +G. W. g w +G.W. g w +GW g w +G. W. H. g w h +GWHS g w h s +Gwi g w i +GWK g w k +G.W.L. g w l +GWO g w o +GWOT g w o t +gwr g w r +GWR g w r +GWR's g w r's +GW's g w's +G. W. S. g w s +GWS g w s +GWU g w u +GXG g x g +GX- g x +GX g x +GXP g x p +GYAN's g y a n's +GYC g y c +GYF g y f +Gy g y +G. Y. g y +GY- g y +GY g y +Gyn g y n +GYN g y n +Gyps g y p's +gyu g y u +Gyu g y u +GZA g z a +GZ g z +GZR g z r +GZSZ- g z s z +HAFV h a f v +H. A. h a +H.A. h a +H&A h and a +HBA h b a +HBA's h b a's +HBBO h b b o +HBC h b c +HBCo h b c o +HBC's h b c's +HBCU h b c u +HBES h b e s +HBF h b f +HBGAs h b g a's +H. B. h b +H.B. h b +HB h b +HBI h b i +HBK- h b k +HBK h b k +HBLX h b l x +HBMP h b m p +HBO h b o +HBO's h b o's +HBPU h b p u +H&BR h and b r +HBR h b r +HBRN h b r n +HBRY h b r y +HB's h b's +HBS h b s +HBSS h b s s +HBU's h b u's +HBV h b v +HBW h b w +HCA h c a +HCAW h c a w +HCBE h c b e +HCB h c b +HCBS h c b s +HCCA h c c a +HCCB h c c b +HCCC h c c c +HCC h c c +HCDCH h c d c h +HCD- h c d +HCE h c e +HCEs h c e's +HCF h c f +H. C. G. h c g +HCGVN h c g v n +H. C. h c +H.C. h c +HC h c +HCH h c h +HCHS h c h s +HCI h c i +HCJB h c j b +hcl h c l +HCL h c l +HClO h c l o +HCMC's h c m c's +HCME h c m e +HCM h c m +HCN h c n +HCO h c o +HCoV h c o v +HCPA h c p a +H. C. P. h c p +HCP h c p +HCPs h c p's +HCR h c r +HCS h c s +HCT h c t +HCTV's h c t v's +HCUP h c u p +HCV h c v +HCVHDB- h c v h d b +Hcy h c y +HDAC h d a c +Hdad h d a d +hDAF h d a f +HDA h d a +HDB h d b +HDBK- h d b k +HDCAM h d c a m +HDC h d c +HDCP h d c p +HDD h d d +HDDs h d d's +HDFC h d f c +H.D.F. h d f +HDFM h d f m +H.D.G. h d g +HDG h d g +hd h d +H. D. h d +H.D. h d +HD- h d +HD h d +hdh h d h +HDi h d i +HDI h d i +HDLC h d l c +HDL- h d l +HDL h d l +HDLS h d l s +HDM h d m +HDMI h d m i +HDPE h d p e +HDP h d p +HDR h d r +HDRO h d r o +HDRR h d r r +H. E. h e +H.E. h e +hezb h e z b +Hezb h e z b +HFA h f a +HFB h f b +HFC h f c +HFCS h f c s +HFDF h f d f +HFF h f f +HFF's h f f's +H&F h and f +H. F. h f +H.F. h f +HFI h f i +H. F. J. h f j +HFNB h f n b +HFPA h f p a +HFP h f p +HFPPV h f p p v +HFRS h f r s +HFS h f s +HFTP h f t p +H.G.B. h g b +hgcA h g c a +HGC h g c +HGF h g f +hg h g +H. G. h g +H.G. h g +HG- h g +HG h g +HGH h g h +HGNC h g n c +HgO h g o +H. G. O. h g o +HGPIN h g p i n +HGPs h g p's +HGTV h g t v +HGTV's h g t v's +HgU h g u +HGVs h g v's +HGY h g y +HHA h h a +H. H. B. h h b +HHC h h c +HHD h h d +HHDP h h d p +HHG h h g +H&H h and h +HHHDB h h h d b +H. H. h h +H.H. h h +HHPC h h p c +HHSAA h h s a a +HHS h h s +H.H.T. h h t +H. I. h i +H.I. h i +H.I.V.E. h i v e +HIV- h i v +HIV h i v +HIY h i y +Hizb h i z b +hja h j a +HJC h j c +H. J. h j +H.J. h j +HJ- h j +HJ h j +HJK h j k +H.J.M. h j m +HJN h j n +H. J. R. h j r +H.J.R. h j r +HJT- h j t +H. J. W. h j w +HJW h j w +HKAA h k a a +Hkam h k a m +HKB h k b +HKCEC h k c e c +HKCEE h k c e e +HKDF h k d f +HKD h k d +HKDSE h k d s e +HKFC h k f c +H&K h and k +H. K. h k +H.K. h k +HK- h k +HK h k +H. K. J. h k j +HKK h k k +HKMA h k m a +HKN h k n +H.K.P. h k p +HKP h k p +HKSAR h k s a r +HKSCS- h k s c s +HK's h k's +HKTB- h k t b +HKT h k t +HKTWU h k t w u +HKU h k u +Hla h l a +HLA- h l a +HLA h l a +HLBN h l b n +HLD- h l d +HLE h l e +HLH h l h +H. L. h l +H.L. h l +HLL h l l +HLLS h l l s +HLN h l n +HLN's h l n's +HLS h l s +HLT h l t +HLTV h l t v +Hluk h l u k +hlutr h l u t r +HLV h l v +Hly h l y +HMAC h m a c +HMA- h m a +HMA h m a +Hman h m a n +H.M.A.S. h m a s +HMAS h m a s +HMB h m b +HMC h m c +HMCS h m c s +HMDA h m d a +HMD h m d +HMDI h m d i +HMDP h m d p +HME h m e +HMEL h m e l +HMF h m f +HMGA h m g a +HMG- h m g +HMG h m g +HMGIC h m g i c +HMGN h m g n +H&M h and m +HMH- h m h +HMH h m h +hm h m +H. M. h m +H.M. h m +HM- h m +HM h m +HMI h m i +HMIS h m i s +HMMC h m m c +HMM- h m m +HMM h m m +HMMWV h m m w v +HMN h m n +HMNZS h m n z s +HMNZT h m n z t +HMO h m o +HMOs h m o's +HMPC h m p c +HMP h m p +HMRC h m r c +hMRE h m r e +H&M's h and m's +H. M. S. h m s +H.M.S. h m s +HMS h m s +HMSN h m s n +H.M.S.O. h m s o +HMSO, h m s o +HMSO h m s o +HNF- h n f +H. N. h n +H.N. h n +HNI h n i +HNK h n k +HNL- h n l +HNL h n l +HNLMS h n l m s +HNMT h n m t +HNN h n n +HNO h n o +HNS h n s +H. O. h o +H.O. h o +HPA- h p a +HPA h p a +H.P.B. h p b +hpc h p c +HPC h p c +HPCT h p c t +HPDD- h p d d +H. P. D. h p d +HPD h p d +HPE h p e +HPH h p h +hp h p +H. P. h p +H.P. h p +HP- h p +HP h p +HPI h p i +HPLC h p l c +HPL h p l +HPP h p p +HPRA h p r a +HPR h p r +HPRT h p r t +HP's h p's +HPS h p s +H. P. T. h p t +HPV h p v +HQAA h q a a +HQDA h q d a +H.Q. h q +HQ- h q +HQ h q +HRG h r g +H&R h and r +H.R.H. h r h +HRH h r h +H. R. h r +H.R. h r +HR- h r +HR h r +HRK h r k +HRL h r l +HRM h r m +hroa h r o a +HRo h r o +HRO h r o +HRP h r p +HRSA h r s a +HRSA's h r s a's +HRSG h r s g +hrs h r s +HRs h r's +HRST h r s t +HRT h r t +HRV h r v +HRW h r w +HRX h r x +HSA h s a +HSAN h s a n +HSBC h s b c +HSB h s b +HSCA h s c a +HSC h s c +hscy h s c y +HSD h s d +HSDPA h s d p a +Hsee h s e e +HSE h s e +HSE's h s e's +HSF h s f +HSG h s g +HSHB h s h b +H.S.H. h s h +hs h s +Hs h's +H. S. h s +H.S. h s +HS- h s +HS h s +HSHS h s h s +hsi h s i +HSJ h s j +HSK h s k +HSL- h s l +HSL h s l +hSlo h s l o +H.S.M. h s m +HSM h s m +HSMR h s m r +HSN h s n +HSNY's h s n y's +HSPA h s p a +HSPA's h s p a's +HSPG h s p g +HSPH h s p h +HSP h s p +HSP's h s p's +HSPs h s p's +HSRC h s r c +HSR h s r +HSSCCT h s s c c t +HSSF h s s f +HSS h s s +HST h s t +HST's h s t's +HSTs h s t's +HSTS h s t s +hsv h s v +HSV- h s v +HSV h s v +HSX h s x +HSY h s y +HSYK's h s y k's +HSZ h s z +HTA- h t a +HTA h t a +hTAS h t a s +HTAs h t a's +htc h t c +HTC- h t c +HTC h t c +HTC's h t c's +HTGL h t g l +HTHC h t h c +ht h t +H. T. h t +H.T. h t +HT- h t +HT h t +HTIB h t i b +hti h t i +Htin h t i n +HTK h t k +HTLA h t l a +HTLV- h t l v +HTLV h t l v +HTML h t m l +HTM's h t m's +Htoo h t o o +HTPB h t p b +HTPC h t p c +HTP h t p +HtrA h t r a +HTR h t r +HTRs h t r's +HTS h t s +http h t t p +HTTP- h t t p +HTTP h t t p +https h t t p s +HTTPS h t t p s +HTV h t v +HTV's h t v's +Htwa h t w a +HTW h t w +HTX h t x +H. U. h u +HVC h v c +HVCs h v c's +hvcv h v c v +HVDC- h v d c +HVDC h v d c +hvem h v e m +Hvem h v e m +HVF h v f +HVHS h v h s +H. V. h v +H.V. h v +H.W.A. h w a +H. W. F. h w f +H. W. h w +H.W. h w +H.W.L. h w l +HWL h w l +HWM h w m +HWRT h w r t +HWT h w t +HWV h w v +hwy h w y +Hwy h w y +hxt h x t +H. Y. h y +H.Y. h y +H. Z. h z +HZ&PC h z and p c +iaaf i a a f +IAAF i a a f +IAAF's i a a f's +IAA i a a +IAAPA i a a p a +IAAP i a a p +IaaS i a a s +IAAS i a a s +IABSE i a b s e +Iaca i a c a +IACC i a c c +IACD i a c d +IAC i a c +Iacob i a c o b +IACR i a c r +IAC's i a c's +IACTs i a c t's +Iacub i a c u b +IACUC i a c u c +IACW i a c w +IAD i a d +IAD's i a d's +IAEA i a e a +IAEG i a e g +I.A.E. i a e +IAE i a e +IAF i a f +IAF's i a f's +IAG i a g +IAG's i a g's +iagt i a g t +IAHP i a h p +IAHRC i a h r c +I. A. i a +I.A. i a +I.A.M. i a m +IAO i a o +Iapa i a p a +IAPHC i a p h c +IAP- i a p +IAP i a p +iar i a r +Iar i a r +IAR- i a r +IAR i a r +IAST i a s t +IATA i a t a +I.A.T. i a t +IAT i a t +IATR i a t r +IAUC i a u c +IAUCs i a u c's +IAU i a u +IAV i a v +IAWTV i a w t v +Iax i a x +IAX i a x +Ibac i b a c +Ibaes i b a e's +IBA's i b a's +IBB i b b +IBC i b c +IBCs i b c's +IBCT i b c t +IBDA'A i b d a a +Ibda i b d a +ibdal i b d a l +IBDB i b d b +IBDE i b d e +IBD i b d +IBE i b e +IBF i b f +IBGE i b g e +IBH i b h +I. B. i b +I.B. i b +IBJJF i b j j f +IBL i b l +IBLT i b l t +IBMA's i b m a's +IBMC i b m c +Ibm i b m +I.B.M. i b m +IBM- i b m +IBM i b m +IBM's i b m's +IBMXF i b m x f +Ibne i b n e +ibn i b n +IBO i b o +IBPA i b p a +IBP i b p +Ibra i b r a +IBRA i b r a +IBRC i b r c +IBR i b r +IBRO i b r o +Ibsa i b s a +IBSA i b s a +IBSF i b s f +Ibs i b's +IBS i b s +IBSL i b s l +IBTC i b t c +iBT i b t +IBTS i b t s +Ibu i b u +IBU i b u +IBV i b v +IBWFF i b w f f +iby i b y +ICBA i c b a +ICBC i c b c +ICB i c b +ICBL i c b l +ICBM i c b m +ICBMs i c b m's +ICBP i c b p +ICBS i c b s +ICBTT i c b t t +ICCC i c c c +ICCC's i c c c's +ICCEC i c c e c +ICCF i c c f +Icche i c c h e +I.C.C. i c c +ICC- i c c +ICC i c c +ICCID i c c i d +ICCI i c c i +ICCR i c c r +ICC's i c c's +ICCS i c c s +ICCW i c c w +ICDC i c d c +ICD- i c d +ICD i c d +ICDS- i c d s +ICDS i c d s +I.C.E. i c e +ICFF i c f f +icf i c f +ICF- i c f +ICF i c f +ICFR i c f r +ICG i c g +ICHC i c h c +I&C i and c +ic i c +Ic i c +I. C. i c +I.C. i c +IC i c +Ici i c i +ICI i c i +ICJ i c j +ICL i c l +ICL's i c l's +ICMA i c m a +ICME i c m e +I.C.M. i c m +ICM i c m +iCN i c n +ICN i c n +Ico i c o +ICO i c o +ICOP i c o p +ICPC i c p c +ICPD i c p d +ICP i c p +ICP's i c p's +ICQ i c q +ICRC i c r c +ICRF i c r f +ICR i c r +IcRn i c r n +ICRP i c r p +ICRS i c r s +ICRW i c r w +ICSA i c s a +ICSB i c s b +ICSC i c s c +ICSD- i c s d +ICSE i c s e +ICSG i c s g +ics i c s +ICs i c's +ICS i c s +ICSS i c s s +ICSSR i c s s r +ICTA i c t a +ICT i c t +ICTM i c t m +ICTP i c t p +ICTSF i c t s f +ICTs i c t's +ICTS i c t s +ICTU i c t u +ICTUR i c t u r +ICTVdB i c t v d b +ICTV's i c t v's +ICTY i c t y +ICU i c u +ICUN i c u n +ICU's i c u's +ICv i c v +ICV i c v +ICVS i c v s +ICWA i c w a +ICWC i c w c +ICW i c w +ICW's i c w's +iCyt i c y t +ICZ i c z +ICZN i c z n +IDAF i d a f +IDBF i d b f +IDB i d b +IDBI i d b i +IDC i d c +Iddaa i d d a a +Idd i d d +ID'd i d d +IDEF i d e f +IDEG- i d e g +IDE i d e +IDFA i d f a +IDFG i d f g +IDF i d f +IDFL i d f l +IDF's i d f's +IDG i d g +I. D. H. i d h +IDH- i d h +IDH i d h +id i d +I. D. i d +I.D. i d +ID- i d +ID i d +Idi i d i +Idir i d i r +IDIS i d i s +IDLHs i d l h's +IDL i d l +Idm i d m +IDM i d m +IDMR i d m r +IDMS i d m s +IDNA i d n a +IDN i d n +IDNR i d n r +IDNR's i d n r's +IDPs i d p's +IDRC i d r c +IDRO i d r o +IDSA i d s a +IDSA's i d s a's +IDSF i d s f +ids i d s +ID's i d's +IDs i d's +IDV i d v +IDW i d w +IDWO i d w o +IDW's i d w's +IDx i d x +IDX i d x +IEA- i e a +IEA i e a +I.E.M. i e m +IEM i e m +I.E.S. i e s +IES i e s +i.e. that is +Ifa i f a +IFA- i f a +IFA i f a +IFBB i f b b +IFBC i f b c +I.F.B. i f b +IFCC- i f c c +IFCC i f c c +ifc i f c +IFC i f c +IFC's i f c's +IfF i f f +I. F. F. i f f +IFF i f f +IFFI i f f i +IFFK i f f k +Ifft i f f t +IFHA i f h a +IFHP i f h p +ifi i f i +Ifi i f i +IFI i f i +IFIP i f i p +IFIs i f i's +IFK i f k +IFLA i f l a +IFL i f l +IfM i f m +IFM i f m +IFN- i f n +IFN i f n +Ifni i f n i +IFNs i f n's +IFO i f o +Ifop i f o p +IFP i f p +IFPI i f p i +IFPRI i f p r i +IFRC i f r c +IFR i f r +IFRS i f r s +IFRSs i f r s's +IFSB i f s b +IFSC i f s c +IFSTA i f s t a +IFTA i f t a +IFTF i f t f +IFT i f t +Iga i g a +IgA i g a +IGA i g a +IGA's i g a's +IGBT i g b t +IGC i g c +IGCP i g c p +IGCSE i g c s e +IGD i g d +IGF- i g f +IGF i g f +IGFI i g f i +IGFs i g f's +IGIB i g i b +'ig i g +Ig i g +I.G. i g +IG i g +igi i g i +IGI i g i +IgM i g m +IGM i g m +IGMP i g m p +ign i g n +Ign i g n +IGN i g n +IHF i h f +IHGS i h g s +IHH i h h +Ih i h +I. H. i h +IH i h +ihi i h i +IHI i h i +IHL i h l +IHL's i h l's +IHMC i h m c +IHME i h m e +ihm i h m +ihn i h n +Ihn i h n +IHOM i h o m +Ihor i h o r +IHPK i h p k +IHRC i h r c +IHSAA- i h s a a +IHSAA i h s a a +IHSA i h s a +IHS i h s +Iht i h t +IIA i i a +IIb i i b +IIB- i i b +IIB i i b +IIc i i c +IIDC i i d c +IID i i d +IIDI i i d i +IIDS i i d s +IIDX i i d x +IIED i i e d +IIFA i i f a +IIF i i f +IIfx i i f x +Iiga i i g a +IIga i i g a +IIG i i g +IIgs i i g's +IIGs i i g's +IIGS i i g s +IIGS's i i g s's +IIHF i i h f +IIHR i i h r +IIHS i i h s +IIIA i i i a +IIIb i i i b +IIIB i i i b +IIIBy i i i b y +IIIc i i i c +IIId i i i d +IIIF i i i f +I. I. i i +I.I. i i +ija i j a +Ija i j a +IJA i j a +IJF i j f +IJGL i j g l +IJHCD i j h c d +IJHL i j h l +ij i j +I. J. i j +IJ- i j +IJ i j +IJN i j n +Ijok i j o k +IJSB i j s b +IJSE i j s e +IJSEM i j s e m +IJS i j s +Ijui i j u i +IJVM i j v m +IKBN i k b n +IKCO i k c o +IKF i k f +Ikh i k h +I.K. i k +IKK- i k k +IKK i k k +IKL i k l +IKr i k r +Iksa i k s a +IKTA i k t a +Iku i k u +Ilbe i l b e +ILB i l b +ILC i l c +ILCS i l c s +ILD i l d +ILDIS i l d i s +ILFC i l f c +ILF i l f +IL&FS i l and f s +ili i l i +Ili i l i +ILI i l i +I. L. i l +I.L. i l +Ilm i l m +ILM i l m +ILM's i l m's +ILN i l n +ILP i l p +ILR i l r +ILRS i l r s +ilu i l u +Ilu i l u +ILU i l u +IMbd i m b d +IMBD i m b d +IMB i m b +IMCB i m c b +IMCI i m c i +imc i m c +IMC i m c +imdb i m d b +IMDb. i m d b +IMDb i m d b +IMDB- i m d b +IMDB. i m d b +IMDB i m d b +IMD i m d +IMD's i m d's +ime i m e +IME i m e +IMF, i m f +IMF i m f +IMG i m g +I&M i and m +IMI i m i +I. M. i m +I.M. i m +IMIS i m i s +IMK i m k +Iml i m l +IMMEX i m m e x +Imm i m m +Imms' i m m's +IMMU- i m m u +imoa i m o a +IMO's i m o's +IMPA i m p a +IMPDH i m p d h +impr i m p r +Impr i m p r +IMPSA i m p s a +IMR i m r +IMSLP i m s l p +iMSNs i m s n's +Imst i m s t +IMTA i m t a +IMT- i m t +IMT i m t +IMTV i m t v +IMUK i m u k +IMU's i m u's +IMWF i m w f +INAC i n a c +Inba i n b a +INBA i n b a +INBF i n b f +inb i n b +incl i n c l +INCT- i n c t +INDC i n d c +ind i n d +Ind i n d +IND i n d +INFJ i n f j +I.N.F.O. i n f o +Infs i n f's +ING i n g +INGR i n g r +inHg i n h g +INH i n h +INI i n i +I. N. i n +I.N. i n +InlB i n l b +INL i n l +INM i n m +INPD i n p d +INPE i n p e +InP i n p +INP i n p +INPP i n p p +INRA- i n r a +Inre i n r e +I.N.R.I. i n r i +INR i n r +INSB i n s b +INSDC i n s d c +INSD i n s d +Inspx i n s p x +INSS i n s s +INTF i n t f +Ints i n t's +INTs i n t's +Intu i n t u +INVA i n v a +inv i n v +INXS i n x s +INXS's i n x s's +Ioba i o b a +IOCG i o c g +IOC- i o c +IOC i o c +IOCP i o c p +IODP i o d p +IODs i o d's +IOE i o e +IOFB i o f b +IOF i o f +io. i o +io i o +.I.o. i o +Io. i o +Io i o +.IO i o +I/O i o +IO- i o +IO i o +Ioka i o k a +Iok i o k +IOK- i o k +IOL i o l +IoM i o m +IOM i o m +IOPI i o p i +IOP i o p +IOPs i o p's +IORG i o r g +IOR- i o r +IOR i o r +IORM i o r m +IOSA i o s a +IOSB i o s b +iOS- i o s +iOS i o s +Io's i o's +I/Os i o's +I.O.'s i o's +IOS i o s +IOSR i o s r +Iosu i o s u +IoT i o t +IOT i o t +I.O.U. i o u +IOU i o u +IoW i o w +IOW i o w +IPA i p a +IPA's i p a's +IPB i p b +IPCC i p c c +IPCC's i p c c's +IPC i p c +IPCRI i p c r i +IPCs i p c's +IPDA i p d a +IPDI i p d i +IPD i p d +IPE i p e +IPFA i p f a +IPFI i p f i +IPF i p f +IPFP i p f p +IPFW i p f w +IPG i p g +IPH i p h +ip i p +Ip i p +I.P. i p +IP- i p +IP i p +IPL i p l +IPMG i p m g +IPM i p m +IPNI i p n i +IPN i p n +Ipo i p o +I.P.O. i p o +IPO i p o +IPP i p p +IPPL i p p l +IPPs i p p's +IPPT i p p t +Ippu i p p u +iPPV i p p v +IPR i p r +iPSC i p s c +IPSC i p s c +iPSCs i p s c's +ipse i p s e +IPSF i p s f +iPS i p s +IPs i p's +I.P.S. i p s +IPS i p s +IPTA i p t a +IPTC i p t c +IPTG i p t g +IPT i p t +IPTL i p t l +IPTV i p t v +IPU i p u +IPUM i p u m +IPv i p v +IPWA i p w a +IPW i p w +IPX i p x +iQ i q +I.Q. i q +IQ i q +IRAC i r a c +I.R.A. i r a +IRB i r b +IRBM i r b m +IRCA i r c a +IRC i r c +IRCM i r c m +IRCTC i r c t c +IRCT i r c t +IRD i r d +IRDS i r d s +IREC i r e c +I.R.E. i r e +IRE i r e +IRFCA i r f c a +IRFC i r f c +IRF i r f +IRFU i r f u +IRGC i r g c +Irgm i r g m +IRHG i r h g +IRHT i r h t +IRIB i r i b +Irig i r i g +IRI i r i +Iril i r i l +IRIN i r i n +i'r i r +ir i r +Ir i r +I. R. i r +I.R. i r +IR- i r +IR i r +IRIX i r i x +Irla i r l a +IRL i r l +IRM i r m +Irmis i r m i's +IRM's i r m's +IRNE i r n e +IRN- i r n +IRN i r n +iro i r o +Iro i r o +IRO i r o +IRP i r p +IRP's i r p's +IRPS i r p s +IRQ i r q +Irra i r r a +IRRI i r r i +IRR i r r +IRs i r's +I.R.S. i r s +IRS- i r s +IRS i r s +IRSN i r s n +I.R.T. i r t +IRT i r t +iru i r u +Iru i r u +ISA i s a +ISAv i s a v +Isba i s b a +ISBA i s b a +ISBER i s b e r +ISBHF i s b h f +ISB i s b +isbn i s b n +ISBN i s b n +ISBNs i s b n's +ISBO i s b o +ISCB i s c b +ISCC- i s c c +ISCE i s c e +isCf i s c f +ISCI i s c i +iSC i s c +Isc i s c +ISC- i s c +ISC i s c +ISCM i s c m +ISCM's i s c m's +ISCP i s c p +ISDA i s d a +ISDB- i s d b +Isd i s d +ISD i s d +isdn i s d n +ISDN i s d n +ISDS i s d s +ISDT i s d t +ISFI i s f i +ISF i s f +Isgec i s g e c +ISG i s g +ISHC i s h c +ishq i s h q +Ishq i s h q +ISHS i s h s +I. S. i s +I.S. i s +ISKB i s k b +ISK i s k +Isl i s l +ISL i s l +ISMF i s m f +ISMIR i s m i r +ISMN- i s m n +ISMNs i s m n's +isn i s n +ISN i s n +ISNTUC i s n t u c +ISOGG i s o g g +ISO- i s o +ISO i s o +ISO's i s o's +I.S.P.C.K. i s p c k +ISPF i s p f +ISPI i s p i +ISP i s p +ISPL i s p l +ISPR i s p r +ISPs i s p's +Isra'il i s r a i l +Isra i s r a +ISRA i s r a +ISR i s r +ISRN i s r n +ISRO i s r o +ISRP i s r p +ISSCC i s s c c +ISSF i s s f +iss i s s +Iss i s's +ISS i s s +ISSN- i s s n +ISSN i s s n +ISSNs i s s n's +ISSSSP i s s s s p +ISSUU i s s u u +ISTAF i s t a f +ISTD i s t d +Isu i s u +ISU i s u +ISU's i s u's +ISWC i s w c +ISWI i s w i +ISW i s w +I.T.A. i t a +ITA i t a +ITB i t b +ITC i t c +Ite i t e +ITE i t e +ITF i t f +ITG i t g +ITHF i t h f +I.T.H.M. i t h m +ITHs i t h's +iti i t i +Iti i t i +ITI i t i +I.T. i t +itk i t k +Itk i t k +ITK i t k +ITM i t m +ITMO i t m o +ITN i t n +ITNNs i t n n's +ITN's i t n's +Itoi i t o i +ITPF i t p f +iTP i t p +ITP i t p +ITRD i t r d +Itse i t s e +ITSF i t s f +ITTF i t t f +itt i t t +ITT i t t +ITUC i t u c +Itu i t u +ITU- i t u +ITU i t u +ITV i t v +ITV's i t v's +ITWF i t w f +ITW i t w +Itz i t z +IUB i u b +I.U.C.N.B. i u c n b +Iucn i u c n +IUCN- i u c n +IUCN i u c n +IUCN's i u c n's +IUD i u d +IUDs i u d's +IU i u +IUT i u t +IUU i u u +Iuz i u z +IVAA i v a a +IVAW i v a w +IVB i v b +I. V. i v +I.V. i v +IVL i v l +IVLP i v l p +IVM's i v m's +IVP i v p +IVPP i v p p +IVPTC i v p t c +IVRCL i v r c l +IVs i v's +Ivu i v u +IVWA i v w a +IWA i w a +Iwas i w a's +Iwaz i w a z +IWBs i w b's +IWC i w c +IWD i w d +IWF i w f +IWFL i w f l +IWF's i w f's +IWFS i w f s +IWG i w g +IWGP i w g p +iwi i w i +iw i w +I. W. i w +IW- i w +IW i w +IWL i w l +IWMF i w m f +IWM i w m +IWPR i w p r +iwrg i w r g +IWRG i w r g +IWR i w r +IWRS i w r s +IWSA i w s a +iwspy i w s p y +Iwuh i w u h +IWU's i w u's +I.W.W. i w w +IWW i w w +IXA i x a +IXb i x b +IXBs i x b's +IXC i x c +IXCs i x c's +IXe i x e +IX i x +iXL i x l +IXO i x o +ixtle i x t l e +IXV i x v +Iya i y a +IYA i y a +Iyar i y a r +IYC i y c +iyem i y e m +IYHF i y h f +Iyi i y i +iy i y +I. Y. i y +I.Y. i y +IY i y +J.A.A. j a a +J. A. B. j a b +Jaf j a f +J.A.G. j a g +JAG j a g +JAIR j a i r +J. A. j a +J.A.K.E. j a k e +J. A. N. j a n +J.A.P. j a p +J.A.S. j a s +JBA j b a +JBBA j b b a +JBB j b b +JBC j b c +JBG j b g +J&B j and b +jb j b +J. B. j b +J.B. j b +JB j b +J.B.L.D. j b l d +JBL j b l +J.B.M. j b m +JBM j b m +JB's j b's +J. B. S. j b s +J.B.S. j b s +JBT j b t +JBZ j b z +JCA j c a +JCB j c b +JCC j c c +JCF j c f +JCFS j c f s +J.C.G. j c g +JCG j c g +JCGM j c g m +JCIB j c i b +JCI j c i +jcis j c i s +jc j c +J. C. j c +J.C. j c +JC j c +JCJC j c j c +J. C. L. j c l +JCPA j c p a +J. C. P. j c p +JCP j c p +JCPV j c p v +JCRB j c r b +JCR j c r +JCRs j c r's +J.C.S. j c s +JCS j c s +JCSR j c s r +JCVI j c v i +J. C. W. j c w +JCW j c w +J. D. A. j d a +JDA j d a +JDBC j d b c +JDC j d c +JdeBP j d e b p +J.D.E. j d e +JDE j d e +JDENET j d e n e t +J.D.F. j d f +JDF j d f +jdi j d i +JDI j d i +jd j d +J. D. j d +J.D. j d +JD- j d +JD j d +jdk j d k +JDK j d k +J.D.M. j d m +JDM j d m +JDO j d o +JDRF j d r f +J&D's j and d's +J.D.s j d's +JDT j d t +JDU j d u +JDV j d v +J. E. A. j e a +J.E.A.L. j e a l +J.E.B. j e b +J. E. C. j e c +J. E. E. j e e +J. E. j e +J.E. j e +J.F.G. j f g +JFH j f h +jf j f +J. F. j f +J.F. j f +JF- j f +JF j f +JFK j f k +JFN j f n +JFQA j f q a +JFS j f s +J. G. A. j g a +J.G.B. j g b +JGI j g i +J. G. j g +J.G. j g +J. G. L. j g l +JGLS j g l s +J. G. M. j g m +J.G.M. j g m +JGO j g o +JGP j g p +JGR j g r +JGR's j g r's +JGSDF j g s d f +JGTC j g t c +JGU j g u +J.H.B. j h b +JHB j h b +J. H. C. j h c +JHC j h c +JHED j h e d +J.H.F. j h f +J.H.H. j h h +jh j h +J. H. j h +J.H. j h +JH- j h +JH j h +JHL j h l +J. H. M. j h m +J.H.R. j h r +JHs j h's +JHS j h s +JHSVs j h s v's +JHud j h u d +JHU- j h u +JHU j h u +J. I. j i +J.I. j i +J.J.A. j j a +JJB j j b +J. J. C. j j c +J.J.G. j j g +J&J j and j +J. J. j j +J.J. j j +JJ j j +JJK j j k +J. J. L. j j l +J.J.M. j j m +J.J.N. j j n +JJN j j n +Jka j k a +JKA j k a +JKC j k c +jkd j k d +JKH j k h +J&K j and k +J.K. J. j k j +J. K. j k +J.K. j k +JK j k +JKNPP j k n p p +J. K. S. j k s +JKT j k t +jkx j k x +JKX j k x +JLA j l a +J.L.B. j l b +JLB j l b +JLC j l c +J. L. E. j l e +J.L.E. j l e +JLG j l g +JLI j l i +J. L. j l +J.L. j l +J.L.L. j l l +JLM j l m +JLO's j l o's +JLPGA j l p g a +JLP j l p +JLS j l s +JLT j l t +JMA j m a +JMBG j m b g +J.M.C. j m c +JMC j m c +J. M. D. j m d +JMD j m d +J. M. E. j m e +JMI j m i +J. M. J. j m j +J.M.J. j m j +J. M. j m +J.M. j m +JMKR j m k r +J. M. L. j m l +JML j m l +JMM j m m +JMP j m p +JMS j m s +JMU j m u +JMV j m v +J. M. W. j m w +J.M.W. j m w +JMX j m x +JNA j n a +jnb j n b +JNF j n f +JNG- j n g +JNG j n g +JNIH- j n i h +JNI j n i +J. N. j n +J.N. j n +JN j n +JNK j n k +J. N. L. j n l +Jno j n o +JNPT j n p t +jnr j n r +JNR j n r +JNS's j n s's +JOJ j o j +J. O. j o +J.O. j o +J.P.D.B. j p d b +J. P. F. j p f +JPF j p f +jpg j p g +JPG j p g +JPH j p h +J&P j and p +jp j p +J.P. j p +JP- j p +JP j p +J.P.R. j p r +J.Q. j q +JQ j q +J.R.A. j r a +JRA j r a +J.R.C. j r c +JRC j r c +J.R.E. j r e +JRG j r g +JRHU j r h u +J. R. I. j r i +J&R j and r +J. R. j r +J.R. j r +JR j r +JRJ's j r j's +Jr. junior +JRPG j r p g +J. R. P. j r p +J. R. R. j r r +J.R.R. j r r +J.R.'s j r's +JRSP j r s p +J. R. T. j r t +JRU j r u +JRX j r x +JSA j s a +JSAP j s a p +JSA's j s a's +jsb j s b +JSCA j s c a +JSC j s c +J.S.D. j s d +JSD j s d +JSE j s e +JSF j s f +JSH j s h +Js j's +J. S. j s +J.S. j s +JS j s +J.S.K. j s k +J.S.L. j s l +JSL j s l +J. S. M. j s m +JSM j s m +JSNN j s n n +JSOC j s o c +JSO j s o +JSP j s p +JSPS j s p s +JSR j s r +JSS j s s +JSW j s w +JSW's j s w's +JSX j s x +JTA j t a +J.T.B. j t b +JTB j t b +JTC j t c +JTF- j t f +JTF j t f +JTG j t g +JTI j t i +J&T j and t +J. T. j t +J.T. j t +JT j t +JT&KW j t and k w +JTL j t l +JTM- j t m +JTO j t o +JTR j t r +JTRO j t r o +JTS j t s +JTTF j t t f +JTT j t t +JTV j t v +J. U. j u +J.U.L.I.A. j u l i a +J.V.B. j v b +JVB j v b +JVC j v c +J.V.E. j v e +JVG j v g +JVH j v h +J&V j and v +J. V. j v +J.V. j v +JV- j v +JV j v +JVM j v m +JVNW j v n w +JVP j v p +JVRA j v r a +JVS j v s +Jwa j w a +J. W. A. j w a +J. W. B. j w b +JWB j w b +J.W.C. U.S. j w c u s +JWHA j w h a +JWH- j w h +J.W.J. j w j +J. W. j w +J.W. j w +JWM j w m +J. W. P. j w p +JWP j w p +JWRC j w r c +J. W. S. j w s +J.W.S. j w s +JWS j w s +JWST j w s t +JWT j w t +jx j x +JX- j x +JX j x +J.X.W.P. j x w p +JYJ j y j +JYJ's j y j's +J. Y. j y +J.Y. j y +JY j y +Jym j y m +Jymn j y m n +JYP j y p +J. Y. S. j y s +J. Z. j z +J.Z. j z +JZ- j z +JZ j z +K.A.A. k a a +K. A. k a +K.A. k a +K.A.N. k a n +K. A. R. k a r +K.A.S. k a s +KBBI k b b i +KBB k b b +KBCI k b c i +KBC k b c +KBCO k b c o +KBD k b d +KBEH- k b e h +KBE k b e +KBEV- k b e v +KBFC k b f c +KBFL k b f l +KBFR k b f r +KBGD k b g d +KBG k b g +KBIG k b i g +kbi k b i +K&B k and b +K. B. k b +K.B. k b +KBKR k b k r +KBKS k b k s +KBKW k b k w +KBL k b l +KBMT- k b m t +KBMT k b m t +KBND k b n d +KBNZ- k b n z +KBNZ k b n z +KBO k b o +KBRC k b r c +KBR k b r +KBSC- k b s c +KBS k b s +KBSN k b s n +KCB k c b +KCBS- k c b s +KCBS k c b s +KCCC k c c c +KCCI k c c i +KCC k c c +KCCO k c c o +KCDC k c d c +K. C. D. k c d +KCD k c d +KCED k c e d +KCET k c e t +KCG k c g +KCGW- k c g w +KCHK- k c h k +KCHS k c h s +K.C.I.E. k c i e +K. C. k c +K.C. k c +KCK k c k +KCKM k c k m +KCLU k c l u +K.C.M.G. k c m g +KCMG k c m g +KCMO- k c m o +KCMU- k c m u +KCNA k c n a +KCNH k c n h +KCNJ k c n j +KCN k c n +KCNN k c n n +KCNQ k c n q +KCNV k c n v +KCPD k c p d +KCPI k c p i +KCPM k c p m +KCPQ k c p q +KCPQ's k c p q's +KCQL k c q l +KCRA k c r a +KCRC k c r c +KCRH k c r h +KCR k c r +KCRW k c r w +KCSD- k c s d +KCSD k c s d +KCSG k c s g +K.C.S.I. k c s i +KCSI k c s i +K.C.'s k c's +KCS k c s +KCSP k c s p +KCTD k c t d +KCT k c t +KCTL k c t l +KCTV k c t v +KCUB k c u b +KCU k c u +KCVO k c v o +KCYX k c y x +kDa k d a +KDA k d a +kdal k d a l +KDAL k d a l +KDB k d b +KDCE k d c e +KDCO k d c o +KDDG k d d g +KDDI k d d i +KDD k d d +KDE- k d e +KDE k d e +KDEX- k d e x +KDFW k d f w +KDFX k d f x +K. D. G. k d g +KDGS k d g s +KDHS k d h s +KDIS- k d i s +KDKA- k d k a +KDKA k d k a +K&D k and d +kd k d +K. D. k d +K.D. k d +KD- k d +KD k d +KDP k d p +KDPM k d p m +KDR k d r +K. D.'s k d's +K.D.'s k d's +KDS k d s +ke k e +Ke- k e +Ke k e +K. E. k e +K.E. k e +KE k e +KFAC k f a c +KFAED k f a e d +KFAI k f a i +KFA k f a +KFBK k f b k +KFBT k f b t +KFCC k f c c +KFCD k f c d +kfc k f c +K.F.C. k f c +KFC k f c +KFC's k f c's +KFDI- k f d i +KFD k f d +KFDM- k f d m +KFDM k f d m +KFEM k f e m +KFFL k f f l +KFH k f h +KFI k f i +KFJB k f j b +K.F. k f +KFLX k f l x +KFMB- k f m b +KFMB's k f m b's +KFMG- k f m g +KFMJ k f m j +K. F. M. k f m +K.F.M. k f m +KFOR k f o r +KFQC k f q c +KFQ k f q +KFRC k f r c +KFRC's k f r c's +KFRD k f r d +KFRG k f r g +KFRH k f r h +KFRO k f r o +KFSD k f s d +KFSN- k f s n +KFSN k f s n +KFST k f s t +KFTA- k f t a +KFT k f t +KFTL- k f t l +KFTL k f t l +KFUK- k f u k +KFUM- k f u m +KFUM's k f u m's +KFVD k f v d +KFV k f v +KFWB k f w b +KFXO k f x o +KFYO- k f y o +KFYO's k f y o's +KFYR- k f y r +KGA k g a +K.G.B. k g b +KGB- k g b +KGB k g b +KGBX- k g b x +KGC k g c +KGPH k g p h +KGP k g p +KGRI k g r i +kgr k g r +KGRK k g r k +KGS- k g s +KGS k g s +kgt k g t +KGT k g t +KGTV k g t v +KgU k g u +KGU k g u +KGUN k g u n +kgv k g v +K.G.V. k g v +KGWB k g w b +KGW k g w +KHHZ k h h z +KHI k h i +Khizr k h i z r +KHJ- k h j +KHJ k h j +Kh- k h +K. H. k h +K.H. k h +KH- k h +KH k h +KHK k h k +KHKV k h k v +Khlav k h l a v +KHL k h l +KHL's k h l's +Khmu k h m u +KHMY k h m y +KHOJ k h o j +KHOW k h o w +KHQA k h q a +KHQ k h q +Khri k h r i +Khru k h r u +KHRW k h r w +KHSAA k h s a a +KHSC- k h s c +KHS k h s +KHTL k h t l +KHTZ k h t z +KHUI k h u i +KHUT k h u t +KHWI k h w i +K.I.D. k i d +K. I. k i +K.I. k i +K.I.M. k i m +KIP k i p +K.I.S.S. k i s s +KJCE k j c e +KJCT k j c t +KJEF k j e f +Kjer k j e r +KJHK k j h k +KJI k j i +KJIN k j i n +kj k j +K. J. k j +K.J. k j +KJ k j +KJKP k j k p +KJMM k j m m +KJNE k j n e +KJNW k j n w +Kjop k j o p +KJo's k j o's +KJQY k j q y +KJV k j v +KJZI k j z i +KKBL k k b l +KKBQ k k b q +KKCR k k c r +KKDA k k d a +KKDS- k k d s +K.K.E. k k e +KKE k k e +K. K. k k +K.K. k k +KKMC k k m c +KKMI k k m i +KKM k k m +KKMX k k m x +KKRG k k r g +KKR k k r +KKSF k k s f +KK's k k's +KKSY k k s y +KKT k k t +KKWB- k k w b +KLA k l a +KLBK k l b k +KLBM k l b m +KLCI k l c i +KLC k l c +KLDS k l d s +Kle k l e +KLE k l e +KLF k l f +KLGH k l g h +KLH k l h +KLK k l k +kl k l +K. L. k l +K.L. k l +KL k l +KLKS k l k s +KLLP k l l p +KLLV k l l v +KLM k l m +KLN's k l n's +KLQQ k l q q +KLQT k l q t +KLRY k l r y +KLSC k l s c +KLSD k l s d +KLS- k l s +KLS k l s +KLSR- k l s r +KLSX k l s x +KLT k l t +KLX k l x +KMB k m b +KMCI- k m c i +KMC k m c +KMCX- k m c x +KMD k m d +KMGO k m g o +KMGT k m g t +KMIH k m i h +KMIR k m i r +KMITL k m i t l +KMJC- k m j c +KMJ k m j +K&M k and m +KMK k m k +km k m +K. M. k m +K.M. k m +KM- k m +KM k m +KMLA k m l a +KML k m l +KMME k m m e +KMML k m m l +KMRL k m r l +KMS k m s +KMSP k m s p +KMT k m t +KMTP k m t p +KMTV k m t v +KMVA k m v a +KMVQ- k m v q +KMXW k m x w +KMZG- k m z g +KMZ k m z +KNFM k n f m +KNHP k n h p +K&N k and n +kn k n +K. N. k n +K.N. k n +KN k n +KNKT k n k t +KNLA k n l a +KNMI k n m i +KNNV k n n v +KNPN k n p n +KNPR k n p r +KNPU k n p u +KNRE k n r e +KNRS k n r s +KNS k n s +KNSM k n s m +KNST- k n s t +KNST k n s t +KNTH k n t h +K.N.T. k n t +KNTN k n t n +KNTU k n t u +KNVA's k n v a's +KNVB k n v b +KNVN k n v n +KNWA- k n w a +KNX- k n x +KNX k n x +KNXT k n x t +KNXV- k n x v +K.O.D. k o d +k'o k o +K. O. k o +K.O. k o +KOTC k o t c +KOTD k o t d +K.O.T. k o t +KOWL k o w l +K. P. A. C. k p a c +KPAC- k p a c +KPAC k p a c +KPA k p a +KPAN k p a n +KPCC k p c c +KPC k p c +KPD k p d +KPDR- k p d r +KPE k p e +KPFT- k p f t +KPIF k p i f +KPI- k p i +KPI k p i +KPIs k p i's +K&P k and p +K. P. k p +K.P. k p +KP k p +KPLC k p l c +KPL k p l +KPLZ k p l z +KPMG k p m g +KPM k p m +KPNA k p n a +KPNB k p n b +KPO k p o +KPPC- k p p c +Kppen k p p e n +KPPV k p p v +KPRC- k p r c +KPS k p s +KPSL- k p s l +kptm k p t m +KPTV- k p t v +KPTV k p t v +KPU k p u +KPVD k p v d +KPVX k p v x +KPWR k p w r +KPXE k p x e +KPXO- k p x o +KPZK- k p z k +KQA k q a +KQBU- k q b u +KQCD- k q c d +KQED k q e d +KQEG- k q e g +KQHN k q h n +KQKK k q k k +KQKS k q k s +KQLT k q l t +KQLZ k q l z +KQMO k q m o +KQRS- k q r s +KQV k q v +KQXR k q x r +KRBM k r b m +krc k r c +KRCR k r c r +KRCs k r c's +KRF k r f +KRH k r h +K.R.I.T. k r i t +K.R.I.T.'s k r i t's +Krka k r k a +KRKC k r k c +KRK k r k +kr k r +K. R. k r +K.R. k r +KR k r +KRMT k r m t +Krne k r n e +KRNS k r n s +KRNV- k r n v +KRNV k r n v +KROQ- k r o q +KROQ k r o q +kroz k r o z +Kroz k r o z +KRPS k r p s +KRRT k r r t +K. R. S. k r s +KRS- k r s +KRS k r s +KRT k r t +Kru k r u +KRU k r u +KRWC k r w c +KRX k r x +K.R.Y. k r y +KRZA k r z a +ksa k s a +KSA k s a +K. S. B. k s b +KSBW- k s b w +KSBW k s b w +KSBY k s b y +KSCB k s c b +KSC k s c +KSCO k s c o +KSC's k s c's +KSCS k s c s +KSDB k s d b +KSD- k s d +KSDO k s d o +KSEB k s e b +KSEG k s e g +KSE k s e +KSEQ k s e q +kset k s e t +KSET k s e t +Kseur k s e u r +Ksevt k s e v t +KSFO k s f o +KSG k s g +KSHB k s h b +KSI's k s i's +KSJR k s j r +KSKJ k s k j +KSKK k s k k +K.S.K. k s k +KSK k s k +ks k s +K. S. k s +K.S. k s +KS k s +KSLA- k s l a +KSLC k s l c +KSL k s l +KSLM k s l m +KSLU k s l u +KSMB k s m b +KSNB- k s n b +KSNR k s n r +KSNV k s n v +KSP k s p +KSPS k s p s +KSTE k s t e +KSTH k s t h +KST k s t +KSTP- k s t p +KSTP k s t p +KSTV- k s t v +KSU- k s u +KSU k s u +K.S.V. k s v +KSWD k s w d +KSWT k s w t +KSWW- k s w w +KSWW k s w w +KSYL k s y l +KSZE- k s z e +KSZR k s z r +KTAC- k t a c +Ktav k t a v +KTAV k t a v +KTBS k t b s +KTCC k t c c +KTC k t c +KTCZ k t c z +KTDA k t d a +KTDD k t d d +KTE k t e +KTFA- k t f a +KTH k t h +KTHV- k t h v +KT&K k t and k +kt k t +K. T. k t +K.T. k t +KT- k t +KT k t +KTKZ k t k z +KTLA k t l a +K. T. M. k t m +KTM k t m +KTMS- k t m s +KTNE- k t n e +KTN k t n +KTOK k t o k +KTO k t o +KTRE k t r e +KTRH k t r h +KTRK k t r k +KTR k t r +KT's k t's +KTSM k t s m +KTT k t t +KTTN k t t n +KTTV k t t v +KTU k t u +KTUL- k t u l +KTUM k t u m +KTUU- k t u u +KTVA k t v a +KTVK k t v k +KTVS k t v s +KTVT k t v t +KTVU k t v u +KTVZ k t v z +KTWD's k t w d's +KTWN- k t w n +KTWO k t w o +KTXC- k t x c +KTX k t x +KTXL k t x l +KTXS- k t x s +KTZR- k t z r +Kuaa k u a a +Kud k u d +KUFM k u f m +KUGB k u g b +kuih k u i h +K. U. k u +K.U. k u +Kutb k u t b +KUTV k u t v +KUUU k u u u +KUWL k u w l +KUYI k u y i +KVAB k v a b +KVBC k v b c +KVB k v b +K.V.G.K. k v g k +KVHV k v h v +Kvik k v i k +KVI k v i +Kvit k v i t +KVIT k v i t +KVK k v k +K. V. k v +K.V. k v +KVLO k v l o +KVLY- k v l y +KVLY k v l y +KVM k v m +KVMRT k v m r t +KVMX k v m x +KVNG k v n g +KVN k v n +KVP k v p +KVTV k v t v +KVVS's k v v s's +KVVV k v v v +Kvyat k v y a t +KVZ k v z +kwa k w a +KWBM k w b m +KWC k w c +KWD k w d +KWEI k w e i +kwe k w e +KWG k w g +KWJJ k w j j +KWKH k w k h +KWKW k w k w +KWMR k w m r +KWMT k w m t +KWNA k w n a +KWNK k w n k +KWOA k w o a +KWP k w p +KWPT k w p t +KWQC k w q c +KWRE k w r e +KWRU k w r u +KWSB k w s b +KWSX k w s x +KWU k w u +KWVE- k w v e +KWWL's k w w l's +KXAL- k x a l +KXAN k x a n +KXAS k x a s +KXHD- k x h d +KXI k x i +KXJB- k x j b +KXJB k x j b +KXKS k x k s +KXKW- k x k w +KX k x +KXLX k x l x +KXLY k x l y +KXMC k x m c +KXNA k x n a +KXOL's k x o l's +KXTA k x t a +KXTN k x t n +KXTX k x t x +KYAY k y a y +KYBE k y b e +KYEZ k y e z +KYLX- k y l x +KYNM k y n m +Kyse k y s e +KYSL k y s l +KYTC k y t c +kyt k y t +Kyt k y t +KYW- k y w +KYW k y w +kz k z +K. Z. k z +KZ k z +KZLZ k z l z +KZM- k z m +KZMP k z m p +KZMU k z m u +KZQX k z q x +KZZQ k z z q +L. A. G. l a g +L. A. l a +L.A.'s l a's +Lay's l a y's +L.B.A. l b a +LBA l b a +LBCC l b c c +LBC l b c +LBC's l b c's +LBE- l b e +LBE l b e +LBF l b f +LBi l b i +LBJ l b j +LBJ's l b j's +L. B. l b +L.B. l b +LB l b +LBL l b l +LBM l b m +LBN l b n +LBNL l b n l +LBP l b p +LBPs l b p's +LBR l b r +LBS l b s +LBV l b v +LBW l b w +LCAC l c a c +LCA l c a +LCAs l c a's +LCAS l c a s +L.C.B. l c b +LCBL l c b l +LCBM l c b m +lcc l c c +LCC l c c +LCCN l c c n +LCCs l c c's +LCCS l c c s +LCDB l c d b +LCD- l c d +LCD l c d +LCDP l c d p +LC&DR l c and d r +LCDR l c d r +LCDs l c d's +L.C.E. l c e +LCFC l c f c +LCF l c f +LCH l c h +LCI l c i +LCIs l c i's +L. C. l c +L.C. l c +LC- l c +LC l c +LCL l c l +LCMC l c m c +LCM l c m +LCMR l c m r +LCMV l c m v +lcn l c n +LCO l c o +LCP l c p +LCQ l c q +LCR l c r +LCSB l c s b +LCSC l c s c +LC's l c's +LCS- l c s +LCS l c s +LCSs l c s's +LCST l c s t +LCT- l c t +LCT l c t +LCTs l c t's +LCU l c u +LCV l c v +LCVP l c v p +LCVPs l c v p's +LCZ l c z +Lda l d a +LDA l d a +LDAP l d a p +LDB l d b +LDBV l d b v +L.D.C. l d c +LDC l d c +LDCM l d c m +LDCs l d c's +LDD l d d +LDDP l d d p +LDF l d f +LDH l d h +L.D.I. l d i +LDK l d k +L. D. l d +L.D. l d +LDL- l d l +LDL l d l +LDLR l d l r +LDM l d m +LDMR l d m r +LDN l d n +LDP l d p +LDS l d s +LDU l d u +LDV l d v +LEB l e b +L. E. l e +L.E. l e +LFA- l f a +LFA l f a +L.F.C. l f c +LFC l f c +L.F.D. l f d +LFE l f e +LFFCs l f f c's +LFF l f f +LFG l f g +LFHC l f h c +LFH l f h +L. F. l f +L.F. l f +LFL l f l +LFM l f m +LFOC l f o c +LFO l f o +LFOM l f o m +LFPB l f p b +LFP l f p +LFQD l f q d +LFRJ l f r j +LFR l f r +LFs l f's +LFS l f s +LFSRs l f s r's +LFTR l f t r +LFXA l f x a +LGA l g a +LGB l g b +LGBTI l g b t i +LGBT- l g b t +LGBT l g b t +LGBTQ l g b t q +LGC l g c +LGD- l g d +LGD l g d +LGI l g i +L. G. l g +L.G. l g +LG- l g +LG l g +LGMB l g m b +LGM- l g m +LGMs l g m's +LGN l g n +LGPL l g p l +LGPZ l g p z +LG&RDD l g and r d d +LGR l g r +LGs l g's +LGS l g s +LGTB l g t b +LGT's l g t's +LGU l g u +LGUs l g u's +LGV l g v +LGVs l g v's +Lha l h a +LHB l h b +L. H. C. l h c +LHC l h c +LHD l h d +LHFP l h f p +LHICE l h i c e +LHI l h i +L. H. l h +L.H. l h +LHMC l h m c +L.H.M. l h m +LHO l h o +L. H. P. l h p +LHP l h p +LHS l h s +LHW l h w +LHX l h x +LHY l h y +Li- l i +L. I. l i +L.I. l i +LJBL l j b l +Lje l j e +LJJ l j j +L. J. K. l j k +L. J. l j +L.J. l j +L.J.V. l j v +LKB l k b +LKG l k g +LKK l k k +L. K. l k +LK l k +LKL l k l +LKML l k m l +LKPR l k p r +LKRN l k r n +lks l k s +LKS l k s +LKTI l k t i +Llapi l l a p i +LLAW l l a w +L.L.B. l l b +LLB l l b +LLCC l l c c +L.L.C. l l c +LLC- l l c +LLC l l c +LLD l l d +lle l l e +LLE l l e +LLHS l l h s +lli l l i +LLI l l i +L&L l and l +LLLE l l l e +L. L. l l +L.L. l l +LL l l +LLL- l l l +LLL l l l +LLLP l l l p +LLM l l m +LLMNR l l m n r +LLPFX l l p f x +llp l l p +LLP l l p +LLPX l l p x +LLRW l l r w +LLT l l t +LLTV l l t v +llu l l u +LLVM l l v m +LLWS l l w s +LMA l m a +LMC l m c +LMCT l m c t +LMDB l m d b +LMDC l m d c +LMD l m d +LME l m e +LMFAO's l m f a o's +LMFF l m f f +LMG l m g +LMGs l m g's +LMGTE l m g t e +LMH l m h +LMHS l m h s +lm l m +L. M. l m +L.M. l m +LM- l m +LM l m +LMM l m m +LMP l m p +LMPs l m p's +LMQ l m q +LMQs l m q's +L.M.S. l m s +LMS l m s +LMT l m t +LMTP l m t p +LMU l m u +LMWH l m w h +LMX l m x +LnAIB l n a i b +LNA l n a +LNAV l n a v +LNBF l n b f +LNB l n b +L.N.C. l n c +LNCS l n c s +L.N.E.R. l n e r +LNER l n e r +LNFS l n f s +LNG l n g +LNH l n h +L&N l and n +L. N. l n +L.N. l n +L.O.C.'s l o c's +Lokk l o k k +l'OL l o l +L.O.L. l o l +L. O. l o +L.O. l o +L.O.V.E. l o v e +L.O.V. l o v +LOXL l o x l +LPA l p a +L.P.A.M. l p a m +LPAM l p a m +LPARs l p a r's +LPAVS l p a v s +LPBG l p b g +L.P.B. l p b +LPB l p b +LPC l p c +LPCM l p c m +LPDA l p d a +LPDR l p d r +LPE l p e +LPF l p f +LPFM l p f m +LPGA l p g a +LPG l p g +L.P.H. l p h +LPH- l p h +LPH l p h +LPI l p i +LPK- l p k +LPLA l p l a +L&P l and p +LPL- l p l +LPL l p l +lp l p +L. P. l p +L.P. l p +LP- l p +LP, l p +LP l p +L&PM l and p m +LPMN l p m n +LPMud l p m u d +LPMUD l p m u d +LPN l p n +LPO l p o +LPRP l p r p +LPSC l p s c +LP's l p's +LPs l p's +LPS l p s +LPSN l p s n +LPTB l p t b +LPThe l p t h e +LPTV l p t v +LQ l q +LRAD l r a d +LRA l r a +LRCD l r c d +LRC l r c +L.R.C.P. l r c p +LRCP l r c p +LRDG l r d g +LRG- l r g +LRG l r g +lr l r +L. R. l r +L.R. l r +LR l r +LRO l r o +L.R.P.C. l r p c +LRP l r p +LRPPRC l r p p r c +LRRI l r r i +LRRP l r r p +LRS l r s +LRTA l r t a +LRT- l r t +LRT l r t +LRTR l r t r +LRTs l r t's +LRVs l r v's +LRY l r y +LSAC l s a c +LSA l s a +LSBC l s b c +LSB l s b +LSC- l s c +LSC l s c +LSCS l s c s +LSDHH l s d h h +L.S.D. l s d +LSD- l s d +LSD l s d +LSDP l s d p +LSDs l s d's +LSE l s e +LSF l s f +LSG l s g +LSH l s h +LSi l s i +LSI- l s i +LSI l s i +LSJ l s j +LSK l s k +L.S.L. l s l +ls l s +Ls l's +L. S. l s +L.S. l s +LSM l s m +LSO l s o +LSP l s p +LSPN l s p n +LSQC l s q c +LSRI l s r i +L. S. R. l s r +LSS l s s +LSSP l s s p +LSSR l s s r +LST- l s t +LST l s t +LSTM l s t m +LSTs l s t's +LSUA l s u a +LSU l s u +LSU's l s u's +LSVCCs l s v c c's +LSV l s v +LSWR l s w r +LTAF l t a f +L. T. B. l t b +LTB l t b +LTBP- l t b p +LTCF l t c f +LTCI l t c i +L. T. C. l t c +L.T.C. l t c +LTC l t c +Ltda l t d a +LTDA l t d a +Ltd. limited +LTE- l t e +LTE l t e +LTFA l t f a +LTHS l t h s +LTI l t i +LTK l t k +L&T l and t +Lt. lieutenant +LTL l t l +lt l t +L. T. l t +L.T. l t +L.T.M. l t m +LTM l t m +LTMPS l t m p s +LTN l t n +ltoh l t o h +LTP l t p +LTRPC l t r p c +LTTE l t t e +LTTR l t t r +LTU l t u +LTV l t v +L. U. l u +lv l v +L. V. l v +L.V. l v +LV- l v +LV l v +LVMH l v m h +LWDB l w d b +LWD l w d +LWDS l w d s +LWE l w e +LWF l w f +LWH l w h +L. W. l w +L.W. l w +L.Y. l y +L&YR l and y r +M.A.D.E. m a d e +M.A.D. m a d +M.A.J. m a j +M.A.K. m a k +M. A. m a +M.A. m a +M&A m and a +M. A. O. m a o +M.A.O. m a o +M.A.R. m a r +M.A.S.K. m a s k +M.A.S. m a s +M.A.S.S. m a s s +M.B.A. m b a +M.B.B.S. m b b s +MBBS m b b s +MBC- m b c +MBC m b c +MBC's m b c's +MBDA m b d a +MBD m b d +M.B. D.P.M. m b d p m +M.B.E. m b e +MBE m b e +Mbewu m b e w u +MBF m b f +MBFW m b f w +MBGN m b g n +MBH- m b h +MBH m b h +mbi m b i +MBI m b i +M. B. J. m b j +MBJ m b j +MBK m b k +MBL m b l +M. B. m b +M.B. m b +MBM m b m +MBNA m b n a +mBo m b o +MBO m b o +MBPJ m b p j +MBP m b p +Mbre m b r e +mbr m b r +MBR m b r +MBSE m b s e +MBSI m b s i +MBS- m b s +MBS m b s +MBTI m b t i +MBT- m b t +MBT m b t +MBU's m b u's +MBX m b x +MCAF m c a f +MCAL m c a l +M.C.A. m c a +MCA- m c a +MCA m c a +MCA's m c a's +MCAS m c a s +MCB- m c b +MCB m c b +M. C. C. m c c +M.C.C. m c c +MCC m c c +MCC's m c c's +MCCs m c c's +MCCU m c c u +MCDA m c d a +MCDC m c d c +MCD m c d +MCDM m c d m +MCE m c e +MCFA m c f a +M. C. F. m c f +MCF m c f +MCFM m c f m +MCGJCW m c g j c w +MCG m c g +MCI m c i +MCJ m c j +MCLA m c l a +Mcl- m c l +MCL m c l +M. C. m c +M.C. m c +MCMC m c m c +M. C. M. m c m +MCM m c m +MCN m c n +MCO m c o +MCOT's m c o t's +MCPC m c p c +MCPI m c p i +MCP- m c p +MCP m c p +MCPON m c p o n +MCPP m c p p +MCQ m c q +MCRD m c r d +MCR m c r +MCRP m c r p +MCSBA m c s b a +MCs m c's +M.C. S. m c s +MCS- m c s +MCS m c s +MCST m c s t +MCTFS m c t f s +MCT m c t +MCTU m c t u +MCTV m c t v +MCU m c u +MCV m c v +MCVTS m c v t s +MCW m c w +mcyG m c y g +MCYO m c y o +MCZ m c z +MDAA m d a a +MDAH m d a h +mda- m d a +MDA m d a +mdb m d b +MDB m d b +MDC- m d c +MDC m d c +MDC's m d c's +MDCs m d c's +MDCT m d c t +mdDA m d d a +MD&DI m d and d i +Mde m d e +MDE m d e +MDF m d f +MDG m d g +MDGs m d g's +MDH m d h +MDHUs m d h u's +mDia m d i a +MDIB m d i b +MDic m d i c +MDI m d i +MDJT's m d j t's +MDK m d k +mdla m d l a +MDL m d l +MDMA- m d m a +MDMA m d m a +md m d +M. D. m d +M.D. m d +MD- m d +MD m d +MDM m d m +MDNA m d n a +M.D.N. m d n +MDPD's m d p d's +MDP m d p +MDPPP m d p p p +MDPS m d p s +MDPV m d p v +MDQ m d q +MDRC m d r c +mdr m d r +MDR- m d r +MDR m d r +MDT m d t +MDV m d v +MDX m d x +M. E. m e +M.E. m e +M.E.N. m e n +M. E. P. m e p +M.F.A. m f a +MFA m f a +MFB m f b +MFCC m f c c +MFC m f c +MFCS m f c s +MFD m f d +MFDs m f d's +MFe m f e +MFF m f f +MFG m f g +MFI m f i +MFJ m f j +MFK m f k +M. F. m f +M.F. m f +MFMF m f m f +MFM m f m +MFN m f n +MFNW m f n w +MFP m f p +MFR m f r +MFS m f s +MFS's m f s's +MFTBC m f t b c +MFT m f t +MFTs m f t's +Mgadla m g a d l a +Mgal m g a l +mga m g a +Mga m g a +MGA m g a +MGB m g b +Mgbo m g b o +MGCCC m g c c c +MGC m g c +MGE m g e +MGen m g e n +MGG m g g +MGH m g h +MGIMO m g i m o +MGIT m g i t +MGJH m g j h +MGK m g k +M.G.L. m g l +M&G m and g +M. G. m g +M.G. m g +M. G. M. m g m +MGM- m g m +MGM m g m +MGM's m g m's +MGMT m g m t +M&GN m and g n +MGN m g n +MGO m g o +MGP m g p +M&GR's m and g r's +Mha m h a +M.H.A. m h a +MHA m h a +MHB's m h b's +MHC- m h c +MHC m h c +MHCs m h c's +MHD m h d +MHI m h i +MHK m h k +MH&L m h and l +MHL m h l +M&H m and h +M. H. m h +M.H. m h +Mhor m h o r +Mhow m h o w +MHPL m h p l +M.H.P. m h p +MHP m h p +MHP's m h p's +MHRA m h r a +M. H. R. m h r +MHSA m h s a +MHSC m h s c +MHS m h s +MHT m h t +M.I.A. m i a +M.I.A.'s m i a's +M.I.C.M. m i c m +M.I.H. m i h +M&I m and i +M. I. m i +M.I. m i +M.I.N. m i n +M.I.R.V. m i r v +M.I.S. m i s +M.I.T. m i t +MJAHL's m j a h l's +M.J.A. m j a +MJA m j a +MJBHA m j b h a +MJB m j b +MJC m j c +M.J.F. m j f +MJF m j f +MJG m j g +MJHL m j h l +MJHL's m j h l's +M&J m and j +M. J. m j +M.J. m j +M. J. Y. m j y +MKBHD m k b h d +MKB m k b +MKDE m k d e +MKE m k e +MKF- m k f +MKFM m k f m +MKG m k g +MKK m k k +MKMF m k m f +M. K. m k +M.K. m k +MKNG m k n g +MKP m k p +MKRN m k r n +MKs m k's +MKS- m k s +MKS m k s +MKT m k t +MKTO m k t o +MKTV m k t v +MKZ's m k z's +M.L.A. m l a +MLA m l a +MLANA m l a n a +MLAs m l a's +MLB- m l b +MLB m l b +MLBPA m l b p a +MLCA m l c a +mlc m l c +M.L.C. m l c +MLC m l c +MLCs m l c's +MLD m l d +MLE m l e +MLF m l f +mlg m l g +MLG m l g +MLH m l h +MLIA m l i a +mli m l i +MLI m l i +MLIM m l i m +MLK m l k +MLK's m l k's +Mlle m l l e +MLL m l l +MLLT m l l t +M&L m and l +M. L. m l +M.L. m l +MLM m l m +MLND m l n d +M.L.N. m l n +MLN m l n +MLPH m l p h +MLP m l p +MLPs m l p's +M. L. R. m l r +MLR m l r +MLSE m l s e +M. L. S. m l s +M.L.S. m l s +MLS m l s +MLT m l t +MLW- m l w +MLW m l w +MLWS m l w s +MM&A m m and a +M. M. J. m m j +MMK m m k +MML m m l +M&M m and m +M. M. m m +M.M. m m +MMSA m m s a +M&M's m and m's +M. M. S. m m s +MMS- m m s +MMS m m s +MMST m m s t +MMTB's m m t b's +MMTS m m t s +MMTV m m t v +MMWR m m w r +MMX m m x +MNA m n a +MNBA m n b a +MNC m n c +MNCPPC m n c p p c +MNCs m n c's +MNDM m n d m +MND- m n d +MNDNR m n d n r +MNDO m n d o +MNDR m n d r +M&NF m and n f +MNF- m n f +MNF m n f +MNG m n g +MNI- m n i +MNI m n i +MNLA m n l a +MNLA's m n l a's +MNLF m n l f +MNM m n m +M. N. m n +M.N. m n +MNNA m n n a +MNN m n n +MNP m n p +MNPP m n p p +MNRG m n r g +MNSD m n s d +MNS m n s +MNTC m n t c +mnt m n t +MNT m n t +MNZM m n z m +MNZ m n z +MOBKL m o b k l +MOBK m o b k +M.O.B. m o b +M.O.D. m o d +M.O.D.O.K. m o d o k +M.O.G.U.E.R.A.'s m o g u e r a's +M. O. H. m o h +M. O. m o +M.O. m o +M.O.P. m o p +M.O.R. m o r +M.O.T. m o t +MPAA m p a a +MpA m p a +MPA m p a +MPAs m p a's +MPB m p b +MPBN m p b n +MPBu m p b u +MPCA m p c a +MPCI m p c i +MPC- m p c +MPC m p c +MPD m p d +MPE m p e +MPF m p f +MPG m p g +M.P.H. m p h +MPH m p h +MP&I m p and i +MPi m p i +MPI m p i +MPIO m p i o +MPKAB m p k a b +MPLAD m p l a d +MPLA m p l a +MPL m p l +MPLMs m p l m's +MPLR m p l r +MPLS m p l s +M&P m and p +MPM m p m +M. P. m p +M.P. m p +MPO m p o +MPPJ m p p j +M.P.P. m p p +MPP m p p +MPPSC m p p s c +MPPs m p p's +MPQC m p q c +MPR m p r +MPRO m p r o +MPRP m p r p +MPRS m p r s +MPSE m p s e +MPSF m p s f +M.P.'s m p's +MP's m p's +MPs m p's +MPS- m p s +MPS m p s +MPThe m p t h e +MPT m p t +MPTP- m p t p +Mpu m p u +MPU m p u +MPV m p v +MPW m p w +MPZ m p z +MQM m q m +M. Q. m q +Mra m r a +MRA m r a +M. R. B. m r b +MRB m r b +MRBs m r b's +MRCB m r c b +MRCK m r c k +MRCM m r c m +MRC- m r c +MRC m r c +M.R.C.S. m r c s +MRCS m r c s +MRDC m r d c +MRD m r d +mre m r e +MRF m r f +MR&LE m r and l e +MRL m r l +Mr. mister +MRM m r m +M. R. m r +M.R. m r +mRNA m r n a +mRNAs m r n a's +MRPGM m r p g m +MRP m r p +MRPS m r p s +MRR m r r +MRSA m r s a +M.R.S.C. m r s c +Mrs. misses +MRSM m r s m +MRT m r t +MRTs m r t's +MRTS m r t s +MRTT m r t t +MRU- m r u +MRU m r u +MSAA m s a a +MSAC m s a c +MSAD m s a d +MSA m s a +M.S.A.S. m s a s +MSAW m s a w +MSB m s b +MSBO m s b o +MSBSD m s b s d +MSBS- m s b s +MSCC m s c c +M.S.C. m s c +MSC- m s c +MSC m s c +msd m s d +MSD m s d +MSDN m s d n +MSDS m s d s +Mse m s e +MSFC m s f c +M.S.F. m s f +MSF m s f +MSFT m s f t +MSG m s g +MSI m s i +MSJ m s j +MSK m s k +MSL m s l +M&S m and s +Ms. miss +MSML m s m l +M.S.M. m s m +MSM m s m +M. S. m s +M.S. m s +MSNBC m s n b c +MSNBC's m s n b c's +MSN m s n +MSOB m s o b +MSOM m s o m +M.S.P. m s p +MSP m s p +MSPs m s p's +MSR m s r +MSRP m s r p +MSRTC m s r t c +MSSH m s s h +MSSK m s s k +MSSMLP m s s m l p +MSSM m s s m +MSS m s s +MSTA m s t a +mst m s t +MST m s t +MSTS m s t s +MSU's m s u's +MSVCC m s v c c +MSV m s v +MSVU m s v u +MSX m s x +MSY m s y +MSZ- m s z +MSZP m s z p +MTAC m t a c +Mta m t a +M.T.A. m t a +MTA- m t a +MTA m t a +MTA's m t a's +MTAs m t a's +MTBE m t b e +MTBI m t b i +MTB- m t b +MTB m t b +MTC m t c +MTDB m t d b +M.T.D. m t d +MTD m t d +MTD's m t d's +MTG m t g +MTHFR m t h f r +MTHL m t h l +MTIC m t i c +MTI m t i +MTIP- m t i p +MTJ m t j +MTK m t k +MTKO m t k o +M&T m and t +MTM m t m +MTMR m t m r +M. T. m t +M.T. m t +Mtor m t o r +MTOSI m t o s i +M.T.O.W. m t o w +MTPA m t p a +MTPC m t p c +MTP m t p +MTQ m t q +MTRCB m t r c b +MTRC m t r c +MTRJ m t r j +MTR- m t r +MTR m t r +M. T. S. m t s +MTS- m t s +MTS m t s +MTSU m t s u +MTTM m t t m +MTT m t t +MTU m t u +MTV- m t v +MTV m t v +MTVR m t v r +MTV's m t v's +mtvU m t v u +MTVu m t v u +MTVU m t v u +MTY m t y +MTZ- m t z +MTZ m t z +MUKW m u k w +M. U. m u +MUSL m u s l +M.U.s m u's +M. U.S. m u s +MVA m v a +M. V. C. m v c +M.V.C. m v c +MVC m v c +M. V. D. m v d +MVD m v d +MVH m v h +MVK m v k +MVL- m v l +M.V.M. m v m +MVM m v m +MVMs m v m's +M. V. m v +M.V. m v +M.V.O. m v o +MVO m v o +MVP- m v p +MVP m v p +MVPs m v p's +MVPS m v p s +M.V. P.T. m v p t +MVs m v's +MVS m v s +MVSN m v s n +MVSR m v s r +MVT m v t +MVV m v v +M.W.A.M. m w a m +MWA m w a +MWC m w c +MWe m w e +M. W. E. m w e +MWF m w f +MWHL m w h l +MWIFF m w i f f +MWI m w i +MWJCHL m w j c h l +MWJHL m w j h l +M. W. J. m w j +M. W. m w +M.W. m w +M. X. m x +myb m y b +myc m y c +Myc m y c +MYC m y c +MYCN- m y c n +MYCN m y c n +MyDD m y d d +M. Y. m y +M.Y. m y +MySQL m y s q l +MYSQL m y s q l +MZC m z c +MZH m z h +MZM m z m +M. Z. m z +MZP m z p +MZT m z t +NAACCR n a a c c r +NAACL n a a c l +NAAC n a a c +N.A.A.C.P. n a a c p +NAACP n a a c p +NAACP's n a a c p's +N.A.C.L. n a c l +NAFBL n a f b l +NAFC n a f c +NAFC's n a f c's +NAFH n a f h +NAFI n a f i +N. A. n a +N.A. n a +N.A.S.A. n a s a +NBADL n b a d l +N.B.A. n b a +NBA- n b a +NBA n b a +N.B.A.'s n b a's +NBA's n b a's +NBAs n b a's +NBCC n b c c +NBC- n b c +NBC n b c +NBC's n b c's +NBCSN n b c s n +NBDL n b d l +NBD n b d +NBFA's n b f a's +NBF n b f +NBG n b g +NBI n b i +NBK n b k +NBL n b l +N. B. n b +N.B. n b +N.B.N. n b n +NBN n b n +NBP n b p +NBQ n b q +NBR n b r +NBR's n b r's +NBSK n b s k +nbs n b s +NBS n b s +NBSP n b s p +NBTE n b t e +NBT n b t +NBTwo n b t w o +NBTY n b t y +NBW n b w +NCAAs n c a a's +NCAM n c a m +NCA- n c a +NCA n c a +NCAP n c a p +NCBC n c b c +NCBI- n c b i +NCBI n c b i +NCCC n c c c +NCCF n c c f +NCCMH n c c m h +NCC n c c +NCCP n c c p +NCCs n c c's +NCCS n c c s +NCCU n c c u +NCDC n c d c +NCDD's n c d d's +NCD n c d +NCES n c e s +NCFA n c f a +N.C.F. n c f +NCF n c f +NCG n c g +NCGS n c g s +NCHC n c h c +NCH n c h +NCHU n c h u +NCID n c i d +NCI n c i +NCIS n c i s +NCIS's n c i s's +NCKU n c k u +NCLB n c l b +NCLC n c l c +NCL n c l +NCLR n c l r +NCLT n c l t +NCMA n c m a +NCMC n c m c +NCMM n c m m +NCM n c m +NCMP n c m p +NCNB n c n b +N. C. n c +N.C. n c +N.C.O. n c o +NCO n c o +NCO's n c o's +NCOs n c o's +NCPA n c p a +NCPC n c p c +NCP n c p +NCPO n c p o +NCRC n c r c +NCR n c r +NCSU n c s u +NCSY n c s y +NCTA n c t a +NCTC n c t c +NCTE n c t e +NCT n c t +NCUA n c u a +NCVO n c v o +NCVS n c v s +NCVT n c v t +NCWM n c w m +NCW n c w +NDA n d a +NDE n d e +NDEP n d e p +NDFB n d f b +NDFT n d f t +N.D.G. n d g +N. D. n d +N.D. n d +N.D.N.Y. n d n y +NDPH n d p h +NDPK n d p k +N.D.P. n d p +NDP- n d p +NDP n d p +NDP's n d p's +NDRC n d r c +NDRE n d r e +NDRF n d r f +N.E.A.R. n e a r +NECW n e c w +N. E. n e +N.E. n e +N.E.R.D. n e r d +NERFU n e r f u +ner n e r +N.E.R. n e r +NER n e r +N.E.W.S.T. n e w s t +NFA n f a +NFATc n f a t c +NFAT n f a t +NFB n f b +NFB's n f b's +NFC n f c +NFC's n f c's +NFD n f d +NFEA n f e a +nfed n f e d +NFFC n f f c +NFF n f f +NFH n f h +NFHS n f h s +NFIB n f i b +NFI n f i +NFISD n f i s d +N.F.L. n f l +NFL n f l +NFLPA n f l p a +NFL's n f l's +N. F. n f +N.F. n f +NFO n f o +NFPA n f p a +NFPF n f p f +NFP n f p +NFPW n f p w +NFRS n f r s +NFS n f s +NGC n g c +Nge n g e +NGF- n g f +NGF n g f +N.G.L. n g l +N. G. n g +N.G. n g +NGO n g o +NGO's n g o's +NGOs n g o's +NGRC n g r c +N.G.R. n g r +NGR n g r +NGRR n g r r +NGS- n g s +NGS n g s +NGSS n g s s +NGST n g s t +NGTC n g t c +Nha n h a +NHA n h a +NHCEs n h c e's +NHC n h c +NHCP n h c p +NHCs n h c's +NHD n h d +NHHD n h h d +NHHI n h h i +nhi n h i +NHI n h i +NHK n h k +NHK's n h k's +N.H.L. n h l +NHM&W n h m and w +N. H. n h +N.H. n h +NHP n h p +NHPs n h p's +NHPS n h p s +NHRA n h r a +NHRC n h r c +NHSCT n h s c t +NHS- n h s +NHS n h s +NHST n h s t +NHW n h w +N.I.B.B.L.E. n i b b l e +NIBC n i b c +NIBR n i b r +N. I. n i +N.I. n i +N'I n i +N.J.A.C. n j a c +N. J. A. n j a +NJA n j a +N. J. C. n j c +NJC n j c +NJIT n j i t +N. J. M. n j m +NJM n j m +N. J. n j +N.J. n j +NJN n j n +Njoo n j o o +NJPS n j p s +NJPW n j p w +N.J.S.A. n j s a +N.J.'s n j's +NKL n k l +N. K. n k +N.K. n k +N. K. N. n k n +NKP n k p +NKP's n k p's +NKR n k r +NKT n k t +NKU n k u +nkvd n k v d +NKVD n k v d +NKVM n k v m +nkv n k v +NKX n k x +NLAES n l a e s +NLAI n l a i +NLA n l a +NLB n l b +NLCF n l c f +NLC n l c +NLCS n l c s +NLDC n l d c +NLD n l d +NLD's n l d's +NLDS n l d s +N. L. n l +N.L. n l +NLP n l p +NLRB n l r b +NLRP n l r p +NLRs n l r's +NLW n l w +NLX n l x +NMAA n m a a +NMA n m a +NMBS n m b s +NMBU n m b u +NMCA n m c a +NMCB n m c b +NMC n m c +N. M. n m +N.M. n m +NMP n m p +NMR n m r +NMRW n m r w +NMT n m t +NMU n m u +NMV n m v +NMW n m w +NNCL n n c l +NNFL n n f l +NNF n n f +N. N. n n +N.N. n n +NNSA n n s a +NNSS n n s s +NNSU n n s u +NNSW n n s w +NNW- n n w +No. number +N.O.R. n o r +NPA- n p a +NPA n p a +NPBD n p b d +NPBL n p b l +NPB n p b +NPC n p c +NPCs n p c's +NPDC n p d c +NPD n p d +NPF- n p f +NPF n p f +NPF's n p f's +NPGL n p g l +NPG n p g +NPGS n p g s +NPH n p h +NPHS n p h s +NPIM n p i m +NPI n p i +NPJ n p j +NPK n p k +NPL n p l +NPMA n p m a +NPM n p m +NPMs n p m's +N. P. n p +N.P. n p +NP- n p +NP n p +NPO n p o +NPP n p p +NPP's n p p's +NPRL n p r l +NPRN n p r n +NPR- n p r +NPR n p r +NPRR n p r r +NPR's n p r's +NPRSO n p r s o +NPSC n p s c +NPSG n p s g +NPSL n p s l +NP's n p's +NPs n p's +NPS n p s +NPTI n p t i +NPT n p t +NPV n p v +NPWE n p w e +NPWS n p w s +NPY n p y +NQAI n q a i +NQEA n q e a +NQHS n q h s +N. Q. n q +nri n r i +Nri n r i +NRI n r i +NRIs n r i's +NRJ n r j +NRJs n r j's +NRK n r k +NRLA n r l a +NRL n r l +NRL's n r l's +NRMA n r m a +NRM n r m +NRN n r n +nr n r +N. R. n r +N.R. n r +NSA- n s a +NSA n s a +NSAP n s a p +NSA's n s a's +NSB- n s b +NSB n s b +NSC- n s c +NSC n s c +NSD n s d +NSE n s e +NSFE n s f e +NSF n s f +NSFW n s f w +NSG n s g +NSI n s i +NSL n s l +NSM n s m +NSN n s n +N. S. n s +N.S. n s +NSO n s o +NSOs n s o's +NSPCC n s p c c +NSPC n s p c +NSP n s p +NSRI n s r i +NSRL n s r l +NSR n s r +NSSCD n s s c d +NSSDC n s s d c +NSS- n s s +NSS n s s +NSTA n s t a +NST n s t +NSU n s u +NSV n s v +NSVT n s v t +NSWBLF n s w b l f +NSWC- n s w c +NSWC n s w c +NSWCV n s w c v +NSWEC n s w e c +NSWGR n s w g r +N.S.W. n s w +NSW- n s w +NSW n s w +Nta n t a +NTA n t a +NTA's n t a's +ntb n t b +NTB n t b +NTC n t c +NTD n t d +NTFL n t f l +NTF n t f +NTI n t i +NTKF n t k f +NTL n t l +N. T. n t +N.T. n t +NTRK- n t r k +NTR n t r +NTR's n t r's +NTSB n t s b +NTSB's n t s b's +NTSC n t s c +NTS n t s +NTTF n t t f +NTT n t t +NTTR n t t r +NTUC n t u c +NTU n t u +NTVB n t v b +NTV n t v +ntw n t w +N.U. n u +NVA n v a +NVAO n v a o +NVC n v c +NVDA n v d a +NvDA's n v d a's +NVFC n v f c +NVGOP n v g o p +NVHJ's n v h j's +NVH n v h +NVIDIA's n v i d i a's +NVIDIA n v i d i a +NVI n v i +NVL n v l +N. V. n v +N.V. n v +NWAFU n w a f u +nwa n w a +N.W.A. n w a +NWA- n w a +NWA n w a +NWA's n w a's +N.W.F.P. n w f p +NWFP n w f p +NWMP n w m p +N&W n and w +NWN n w n +N. W. n w +N.W. n w +NWPD n w p d +N.W.P. n w p +NWP n w p +NWS n w s +N.W.T. n w t +NWT n w t +nyc n y c +N.Y.C. n y c +NYC- n y c +NYC n y c +NYC's n y c's +N.Y. G.O.P. n y g o p +N.Y.L. n y l +N. Y. n y +N.Y. n y +N.Y.P.D. n y p d +NYPD n y p d +NYPD's n y p d's +NYPL n y p l +N.Y.P. n y p +NY's n y's +N.Y.S. n y s +N.Y.S.V. n y s v +N.Y.U. n y u +NYU- n y u +NYU n y u +NYU's n y u's +NYW&B- n y w and b +NYYC n y y c +NZAID n z a i d +NZAOD n z a o d +NZCA n z c a +NZDT n z d t +NZEF n z e f +NZETC n z e t c +NZFC n z f c +NZF n z f +N'Zif n z i f +NZiK n z i k +N'Zi n z i +NZIV n z i v +NZL n z l +NZLP n z l p +N.Z. n z +NZPA n z p a +NZR- n z r +NZSAS n z s a s +NZ's n z's +NZS n z s +NZTA n z t a +N.Z.W.P.W. n z w p w +O.A.C. o a c +O. A. o a +O.A. o a +O.A.R. o a r +O.A.S. o a s +O.B.E. o b e +obl o b l +Oblt o b l t +O.B. o b +O. C. o c +O.C. o c +Octl o c t l +O.C.T. o c t +O. D. o d +O.E. o e +O.F.M. o f m +O.F. o f +ofr o f r +O.F.R. o f r +OFR o f r +O.F.T.B. o f t b +OGL o g l +O&G o and g +O. G. o g +O.G. o g +O.G.S. o g s +OGS o g s +O.H.A. o h a +O. H. o h +O.H. o h +OHV o h v +O. I. o i +O.I. o i +O. J. o j +O.J. o j +OJ o j +OJSC o j s c +OKBM o k b m +OKB- o k b +OKB o k b +OKC o k c +OK'd o k d +OKd o k d +OKD o k d +oke o k e +Oke o k e +OKH o k h +O.K.I. o k i +OKK- o k k +OKK o k k +OKM o k m +O&K o and k +O. K. o k +O.K. o k +O.L.F.A.L. o l f a l +O. L. K. o l k +O. L. o l +O.L. o l +Olo o l o +Olov o l o v +OLPC o l p c +OLPH o l p h +ols o l s +OLs o l's +OLS o l s +OLSR o l s r +OLTL o l t l +OLVT o l v t +olvwm o l v w m +olwm o l w m +OMB o m b +OMC o m c +OMCS o m c s +OMD o m d +O&MFL o and m f l +OMF o m f +omg o m g +OMG o m g +OMGs o m g's +O. M. o m +O.M. o m +OmOm o m o m +Om's o m's +OMTP o m t p +OMVG o m v g +OMW o m w +OMX o m x +ONCHR o n c h r +ONC o n c +oncu o n c u +ONDH o n d h +ond o n d +onf o n f +ONF o n f +ONGC o n g c +ONIR o n i r +ONJSC o n j s c +ONM o n m +ONMR o n m r +O. N. o n +O.N. o n +O&O o and o +OOOA o o o a +O. O. o o +O.O. o o +OO o o +OOO's o o o's +Oop o o p +OOP o o p +OOPSLA o o p s l a +oor o o r +oose o o s e +Oo's o o's +OOTP o o t p +Ootw o o t w +OOUR o o u r +OPAC o p a c +OPAG o p a g +O.P.A. o p a +OPBF o p b f +OPB o p b +OPC o p c +OPCS o p c s +OPCW o p c w +OPD- o p d +OPD o p d +O.P.I. o p i +OPIRG o p i r g +OPJHL o p j h l +OPLC o p l c +OPL o p l +OPMB o p m b +OPM o p m +OPMs o p m's +OPN o p n +O. P. o p +O.P. o p +opr o p r +Opr o p r +OPR o p r +O.R.C. o r c +ORECA o r e c a +ORFs o r f's +ORMO o r m o +ORNL o r n l +OR&N o r and n +O. R. o r +osaa o s a a +OSAA o s a a +osa o s a +Osa o s a +O.S.A. o s a +OSA o s a +O.S.B. o s b +OSB o s b +OSBs o s b's +O.S.N. o s n +OSN o s n +O. S. o s +O.S. o s +OSP o s p +OSR o s r +OSTM o s t m +O.S.T. o s t +OSTP o s t p +OSTs o s t's +osv o s v +OSV- o s v +OSV o s v +OTB o t b +OTC o t c +OTJ o t j +OTL o t l +OTMH o t m h +OTO o t o +O.T. o t +OTP o t p +OTR o t r +OTs o t's +OTS- o t s +OTS o t s +Otu o t u +OTU o t u +otv o t v +O.U. o u +OU- o u +OU o u +OUP o u p +OUSA o u s a +OVA's o v a's +ovca o v c a +Ovca o v c a +OVC o v c +OVC's o v c's +Ovda o v d a +OVF o v f +OVM o v m +Ovo o v o +OVO o v o +ov o v +Ov o v +O. V. o v +O.V. o v +OV- o v +OV o v +OVP o v p +OVS o v s +OVT o v t +OVW o v w +O.W.A. o w a +OWBT o w b t +OWCs o w c's +OWGR o w g r +OWHA's o w h a's +OWIU o w i u +OWM o w m +O. W. o w +O.W. o w +P. A. C. p a c +P. A. J. p a j +P.A.J. p a j +P. A. M. p a m +P.A.M. p a m +pa p a +P. A. p a +P.A. p a +PA- p a +PA p a +PAQ p a q +P.A.R. p a r +P. A. S. p a s +PBAA p b a a +PBA- p b a +PBA p b a +PBA's p b a's +PBB p b b +PBCC p b c c +pBCE p b c e +pbc p b c +P.B.C. p b c +PBF p b f +PBG p b g +PBIL p b i l +PBI p b i +PBK p b k +PBL p b l +PBM p b m +PBMR p b m r +PBO p b o +pb p b +pB- p b +P. B. p b +P.B. p b +PB- p b +PB p b +PBP p b p +PBR p b r +PBS p b s +PBT p b t +PBX p b x +PBY- p b y +PBY p b y +P. C. A. p c a +PCA p c a +PCB p c b +PCB's p c b's +PCBs p c b's +PCBS p c b s +PCC- p c c +PCC p c c +PCCS p c c s +PCCW p c c w +PCCW's p c c w's +PCDDs p c d d's +PCDHB p c d h b +PCDH p c d h +PCD p c d +PCE p c e +PCeU p c e u +PCFCL p c f c l +PCFL p c f l +PCF p c f +PCFS p c f s +PCG p c g +PCGS p c g s +PCHA p c h a +PCHA's p c h a's +P.C.H. p c h +PCH p c h +PCHR p c h r +PCIe p c i e +pci p c i +PCI- p c i +PCI p c i +PCJHL p c j h l +P. C. J. p c j +PCJ p c j +PCK p c k +PCLM p c l m +pcl p c l +PCL p c l +PCMCIA p c m c i a +PCMNO p c m n o +PCM p c m +PCMS p c m s +PCNA- p c n a +PCNA p c n a +PCN p c n +PcoA p c o a +PCOE p c o e +PCO p c o +pc p c +P. C. p c +P.C. p c +PC- p c +PC p c +PCPFL p c p f l +PCP p c p +PCPV p c p v +PCRC p c r c +PCRE p c r e +PCRev p c r e v +PCRM p c r m +PCR- p c r +PCR p c r +PCSK p c s k +PCSOM p c s o m +pcs p c s +P.C.s p c's +PC's p c's +PCTL p c t l +PCT p c t +PCTV p c t v +PCU p c u +PCU's p c u's +PCW p c w +PCX p c x +pDAB p d a b +PDAB p d a b +PDA p d a +PDAs p d a's +PDBML p d b m l +PDB p d b +pdbp p d b p +PDBsum p d b s u m +PDCI p d c i +PDC- p d c +PDC p d c +PDCPD p d c p d +PDC's p d c's +PDCs p d c's +PD&D p d and d +PDD- p d d +PDD p d d +PDE p d e +PDEs p d e's +PDES p d e s +pdf p d f +PDF p d f +PDFs p d f's +PDFT p d f t +PDGF- p d g f +PDGF p d g f +PDG p d g +PDHJ p d h j +PDH p d h +PDI- p d i +PDI p d i +PDJ p d j +PDK p d k +PDL p d l +PDN p d n +PDO p d o +P&D p and d +P. D. p d +P.D. p d +PD- p d +PD p d +PDP- p d p +PDP p d p +PDP's p d p's +PDPs p d p's +P. D. Q. p d q +PDR p d r +PDSA p d s a +PDSI p d s i +PD's p d's +PDs p d's +P.D.S. p d s +PDS p d s +PDSP p d s p +PDT p d t +P.E.I. p e i +P.E.I.'s p e i's +P.E.N. p e n +P. E. p e +P.E. p e +P.F.L. p f l +PF's p f's +PFs p f's +PFS p f s +PFU p f u +PFV p f v +PFW&C p f w and c +P.G.A. p g a +PGA p g a +PGM p g m +PGMs p g m's +PG&N p g and n +P&G p and g +P. G. p g +P.G. p g +P.G.T. p g t +PHD p h d +P. H. G. p h g +PHH p h h +PHHS p h h s +PHLF p h l f +Phlo p h l o +PHL p h l +PHN p h n +P&H p and h +P. H. p h +P.H. p h +php p h p +PHP- p h p +PHP p h p +PHPs p h p's +P.I.D.E. p i d e +P&I p and i +P. I. p i +P.I. p i +P. I. W. p i w +Piz p i z +PJB p j b +PJC p j c +PJD p j d +P. J. F. p j f +P.J.K. p j k +pj p j +P. J. p j +P.J. p j +PJ- p j +PJ p j +PJ's p j's +PJs p j's +P.J.T. p j t +P. K. p k +P.K. p k +PK- p k +PK p k +PLCC p l c c +plc p l c +PLC- p l c +PLC p l c +PLC's p l c's +PLCs p l c's +P.L.O. p l o +P. L. p l +P.L. p l +PLP p l p +PLX- p l x +PLX p l x +P. M. A. p m a +PMC- p m c +PMC p m c +PMG p m g +PML- p m l +PML p m l +PMLP p m l p +PMMA p m m a +PMMoV p m m o v +PMM p m m +PMMS p m m s +PMMT p m m t +PMOI p m o i +PMO p m o +PMPC p m p c +PMPF p m p f +pm p m +P. M. p m +P.M. p m +PM- p m +PM p m +PMP p m p +PMP's p m p's +PMRN p m r n +PMRO p m r o +PMR p m r +PMRR p m r r +PMSE p m s e +PMSL p m s l +PMSM p m s m +PM's p m's +PMs p m's +P. M. S. p m s +PMS p m s +PMSSY p m s s y +PMT p m t +PMVY p m v y +PMWA p m w a +PNaCl p n a c l +pna p n a +PNA p n a +Pnau p n a u +pnb p n b +PNB p n b +PNC p n c +PNE p n e +PNE's p n e's +PNEs p n e's +PNETs p n e t's +pneus p n e u s +PNFA p n f a +PNF p n f +PNGIA p n g i a +png p n g +PNG p n g +PNH p n h +PNI p n i +PNL p n l +PNL's p n l's +PNMT p n m t +PNNL p n n l +PNNs p n n's +PNoy p n o y +pn p n +P. N. p n +P.N. p n +PN- p n +PN p n +PNP p n p +P.N.R.A. p n r a +P.O.D. p o d +P.O.D.'s p o d's +P&O p and o +P. O. p o +P.O. p o +P.O.S. p o s +P.O.V. p o v +P.O.W. p o w +P.O.W.'s p o w's +PPACA p p a c a +PPA p p a +PPBS p p b s +PPC p p c +PPD- p p d +PPD p p d +PPE p p e +PPF p p f +PPG p p g +P&PH p and p h +PPH p p h +PPi p p i +P.P.I. p p i +PPI p p i +PPJ p p j +PPKM p p k m +PPK p p k +PPL p p l +PPM p p m +PPNB p p n b +PPN p p n +PPO p p o +P. P. p p +P.P. p p +PPRP p p r p +PPR p p r +PP's p p's +PPS p p s +PPT p p t +PPTV p p t v +PPV p p v +PPVs p p v's +PPy p p y +PQBP- p q b p +PQDT p q d t +PQM- p q m +PQ- p q +PQ p q +PQQ p q q +PQ's p q's +PQS p q s +PRB- p r b +PRB p r b +PRCA p r c a +PRC- p r c +PRC p r c +PRCS p r c s +PRDM p r d m +PRD p r d +PRD's p r d's +P.R.I.M.E. p r i m e +pr p r +P. R. p r +P.R. p r +PR- p r +PR p r +P.S.A. p s a +P.S.C. p s c +P.S.K. p s k +PSP- p s p +PSP p s p +P. s p's +P.'s p's +Ps- p s +Ps p's +P. S. p s +P.S. p s +PSQL p s q l +PSR p s r +PSRU p s r u +PSSA- p s s a +PSSA p s s a +PSSA's p s s a's +PSSAs p s s a's +PSSI p s s i +PSS p s s +PSS's p s s's +PST p s t +PSU p s u +PSV p s v +PSW p s w +pTA p t a +PTA p t a +PTA's p t a's +PTAs p t a's +PTB p t b +PTBT p t b t +PTCH p t c h +PTC p t c +Pte p t e +PTEs p t e's +PTFE p t f e +ptf p t f +PTF p t f +PTH p t h +Ptie p t i e +PTI- p t i +PTI p t i +PTI's p t i's +PTK p t k +PTLLS p t l l s +PTL p t l +PTLs p t l's +PTNNT p t n n t +Ptol p t o l +PTO p t o +PTPN p t p n +PTP p t p +pt p t +P. T. p t +P.T. p t +PT- p t +PT p t +PTSD p t s d +PTSE p t s e +PTS- p t s +PTS p t s +PTTGC p t t g c +PTTG p t t g +PTTOW p t t o w +PTT p t t +PTTs p t t's +PTUN p t u n +PTV p t v +PTV's p t v's +P'Twa p t w a +PTY p t y +P.U.F. p u f +Puiu p u i u +PUKKE p u k k e +PUK p u k +Pul p u l +P. U. M. p u m +PVA p v a +PVAs p v a's +PVB p v b +P.V.C. p v c +PVC p v c +PvdA p v d a +PVEM p v e m +pve p v e +PVH p v h +P.V.L. p v l +PVL p v l +PVN p v n +PVO p v o +PVP p v p +Pv- p v +P. V. p v +P.V. p v +PV- p v +PV p v +pvr p v r +PVR p v r +PVRS p v r s +PVs p v's +PVS p v s +PVU p v u +PWA p w a +PWB p w b +PWC p w c +PWD p w d +PWF p w f +P. W. G. p w g +P.W.G. p w g +PWG p w g +PWI p w i +pwll p w l l +PWM p w m +P.W.O. p w o +P&W p and w +P. W. p w +P.W. p w +PW- p w +PW p w +Pyi p y i +PYI p y i +Pyk p y k +PYK p y k +Pyl p y l +PyL p y l +PYP p y p +P. Y. p y +Pyu p y u +Pyw p y w +Pyx p y x +PZB p z b +PZL- p z l +PZL p z l +PZP p z p +P. Z. p z +Q. A. P. q a p +QAP q a p +Q. A. q a +QA q a +Q&A q and a +Q&A's q and a's +Q&As q and a's +QbA q b a +QBE q b e +QBH q b h +QBL q b l +Q.B. q b +QB q b +Q.C.B. q c b +QCD q c d +Q.C. q c +QD q d +QDS q d s +QEA q e a +QEC q e c +Q.E.D. q e d +QED q e d +Q.E.H. q e h +Q. E. q e +Q.H.C. q h c +QHP q h p +Q.H. q h +Q.I. q i +Q. J. q j +QJ q j +Q. J. R. q j r +Q. N. q n +QPF q p f +QPFS q p f s +QPM q p m +QPO q p o +QPOs q p o's +QPP q p p +QP q p +Q.P.R. q p r +QPR q p r +QQ q q +QRL q r l +QRL's q r l's +QRNA q r n a +QROPS q r o p s +QRP q r p +QR q r +QRS q r s +QRT q r t +QRV q r v +QSAR q s a r +QSES q s e s +QSI q s i +QSK- q s k +QSL q s l +QSM q s m +QSO q s o +QST q s t +QSY q s y +QSZ- q s z +QTE- q t e +QVC q v c +QVD q v d +QVM q v m +Q.V. q v +R.A.B. r a b +R. A. E. r a e +R.A.F. r a f +R.A.J. r a j +R. A. r a +R.A. r a +R&A r and a +RAV r a v +R&AW r and a w +RAZR r a z r +RBAC r b a c +RbAg r b a g +RBA r b a +RBBP r b b p +RBC r b c +RBCs r b c's +RBD r b d +RBE r b e +RBGE r b g e +RBG r b g +RBI r b i +RBIs r b i's +R. B. J. r b j +R.B.J. r b j +RBK- r b k +RBK r b k +R.B.L. r b l +RBL r b l +RBMG r b m g +RBMG's r b m g's +RBM r b m +RBMY r b m y +RBO r b o +RBP r b p +R&B- r and b +R&B r and b +R. B. r b +R.B. r b +RBR r b r +RBS r b s +RBTCO's r b t c o's +RBTH r b t h +RBWH r b w h +RBW r b w +RBX r b x +RBZ r b z +RCAC r c a c +RCAF r c a f +rca r c a +R.C.A. r c a +RCA- r c a +RCA r c a +RCA's r c a's +RCBC r c b c +RCB r c b +RCCA r c c a +RCCD r c c d +RCCL r c c l +RCCM r c c m +RCC r c c +RCI r c i +R. C. K. r c k +RCL r c l +R.C.M. r c m +RCM r c m +RCN r c n +RCPO r c p o +RCP r c p +RCPT r c p t +R&C r and c +rc r c +R. C. r c +R.C. r c +RCRD r c r d +RCR- r c r +RCR r c r +RCSB r c s b +RCS r c s +RCT r c t +RCTs r c t's +RCTS r c t s +RCTV r c t v +RCV r c v +RCW r c w +RDB r d b +RDC- r d c +RDC r d c +RDD r d d +RdE r d e +RDFC r d f c +RDF r d f +R.D.G. r d g +R&D- r and d +R&D r and d +R. D. r d +R.D. r d +R.E.A. r e a +R. E. B. r e b +R.E.D. r e d +R.E.L. r e l +R.E.M. r e m +R.E.M.'s r e m's +R.E.O. r e o +REPL r e p l +REPLs r e p l's +R.E.P. r e p +R. E. r e +R.E. r e +R.E.R. r e r +R.F.A. r f a +R.F.C. r f c +R. F. r f +R.F. r f +R. F.S. r f s +R.F.W. r f w +RFW r f w +RFX r f x +RGB r g b +RGD r g d +RGF r g f +RGG r g g +RGI r g i +RGK r g k +RGMA r g m a +RGNL r g n l +RGPH r g p h +R&G r and g +R. G. r g +R.G. r g +RGR r g r +RGS- r g s +RGS r g s +RGS's r g s's +RGTP r g t p +RGU r g u +RGV r g v +RGX r g x +R. H. C. r h c +RHC r h c +RHD r h d +RHHF r h h f +RHH r h h +RHHS r h h s +RHIBs r h i b's +RHIC r h i c +RHI r h i +Rhiw r h i w +RHK r h k +rhl r h l +RHM r h m +RhoG r h o g +Rho's r h o's +RHP r h p +R. H. r h +R.H. r h +R.H.S.J. r h s j +R. I. C. r i c +R.I.C. r i c +R. I. P. r i p +R.I.P. r i p +R&I r and i +R. I. r i +R.I. r i +riu r i u +Riu r i u +rivs r i v s +Rivu r i v u +RIXS r i x s +Rixt r i x t +RJB r j b +RJD- r j d +RJD r j d +RJE r j e +rjf r j f +RJHS r j h s +R. J. J. r j j +RJJ r j j +RJL r j l +RJN r j n +R. J. r j +R.J. r j +RJ r j +RJR r j r +RJ's r j's +RJTD r j t d +R. K. B. r k b +RKD r k d +RKG- r k g +RKI r k i +RKKA r k k a +RKL r k l +R.K.O. r k o +RKO- r k o +RKO r k o +RKO's r k o's +R. K. r k +R.K. r k +RLM r l m +RLM's r l m's +R. L. r l +R.L. r l +RL r l +RLS- r l s +RLV r l v +rly r l y +RMAF r m a f +RMAG r m a g +RMA r m a +RMB r m b +RMCH r m c h +RMCL r m c l +R.M.C. r m c +RMC r m c +RMDs r m d's +R.M.E.S. r m e s +RMFL r m f l +R.M.F. r m f +RMF r m f +RMG r m g +R. M. H. r m h +RMI r m i +RMIT r m i t +RMIT's r m i t's +RMJM r m j m +R.M.K. r m k +RMK r m k +R. M. L. r m l +RML r m l +RMLs r m l's +RMM r m m +RMP r m p +rm r m +R. M. r m +R.M. r m +RM r m +RMR r m r +RMRS r m r s +RMSD r m s d +RMSDs r m s d's +RMSE r m s e +RMs r m's +RMS r m s +R. M. W. r m w +R.M.W. r m w +RMW r m w +RMX r m x +RNAO r n a o +rna r n a +RNA- r n a +RNA r n a +RNA's r n a's +RNAs r n a's +RNAS r n a s +RNC r n c +RND r n d +RNE r n e +Rnet r n e t +RNF r n f +RNG r n g +Rnic r n i c +RNJD r n j d +RNK r n k +RNLAF r n l a f +RNLI r n l i +RNLI's r n l i's +RNNs r n n's +RNOH r n o h +RNP r n p +RNPs r n p's +RNPS r n p s +R. N. r n +R.N. r n +RN- r n +RN r n +R.N.R. r n r +RNZ r n z +R.O.C.K. r o c k +R.O.C. r o c +ROKMC r o k m c +ROKN r o k n +rOmpB r o m p b +Rooi r o o i +ROP r o p +R. O. r o +R.O. r o +ROVs r o v's +Roxb r o x b +Roxx r o x x +RPA r p a +RPAYC r p a y c +RPB r p b +RP&C r p and c +rpc r p c +RPC r p c +Rpe r p e +RPE r p e +RPF r p f +RPGA r p g a +rpg r p g +RPG- r p g +RPG r p g +RPG's r p g's +RPGs r p g's +RPI r p i +RPI's r p i's +RPK r p k +RPL r p l +rpm r p m +RPM r p m +RPMS r p m s +RPO r p o +RPP r p p +R. P. r p +R.P. r p +RP- r p +RP r p +RPs r p's +R.P.S. r p s +RPS r p s +RPT r p t +RPVE r p v e +RPV r p v +R. Q. r q +R.Q. r q +RQ- r q +RQW r q w +rra r r a +RRA r r a +RRC r r c +RRDE r r d e +rrd r r d +Rreli r r e l i +rre r r e +RRE r r e +RRG r r g +R. R. H. r r h +RRH r r h +RRKM r r k m +R.R.K. r r k +rRNA r r n a +rRNAs r r n a's +RRN r r n +RRP r r p +R&R r and r +rr r r +R. R. r r +R.R. r r +RR- r r +RR r r +RRR r r r +RRSR r r s r +RRS r r s +RSAF r s a f +R. S. A. r s a +RSA r s a +RSCG r s c g +RSCJ r s c j +R.S.C. r s c +RSC r s c +RSCS r s c s +RSD r s d +RSE r s e +RSF r s f +RSFSR r s f s r +RSG- r s g +RSG r s g +RSHA r s h a +rsh r s h +RSHS r s h s +RSICC r s i c c +RSID r s i d +RSI r s i +Rsis r s i's +RSI's r s i's +RSIS r s i s +RSK r s k +RSL r s l +rsly r s l y +RsmA r s m a +RSMC r s m c +RSMI r s m i +RSML r s m l +RSM r s m +RSM's r s m's +RSNO r s n o +RSN r s n +RSNZ r s n z +R. S. O. r s o +RSO r s o +RSPB r s p b +RSPCA r s p c a +RSpec r s p e c +RSPK r s p k +RSPO r s p o +R. S. P. r s p +RSP r s p +RSR r s r +rs r s +Rs r's +R. S. r s +R.S. r s +RS- r s +RS r s +RSSI r s s i +R. S. S. r s s +RSS- r s s +RSS r s s +RSSSF r s s s f +RSTC r s t c +RST r s t +RSu r s u +R.S.U. r s u +RSU r s u +RSU's r s u's +RSVP r s v p +RSV r s v +RSX r s x +RTAFB r t a f b +RTA r t a +RTBF r t b f +rtb r t b +RTB r t b +RTCG's r t c g's +RTC r t c +RTCs r t c's +RTD r t d +RTEC r t e c +RTEjr r t e j r +RTeOR r t e o r +RTE r t e +RTE's r t e's +RTFB r t f b +RTFM r t f m +RTF r t f +RTF's r t f's +RTFS r t f s +RTHK r t h k +RTHK's r t h k's +RTHL r t h l +RTIP r t i p +RTI r t i +RTKL r t k l +RTK r t k +RTLM r t l m +RTL r t l +RTL's r t l's +RTML r t m l +RTMP r t m p +RTM r t m +RTNDA- r t n d a +RTN r t n +RTO r t o +RTOs r t o's +RTOS r t o s +rtPA r t p a +RTP r t p +RTR r t r +rt r t +R. T. r t +R.T. r t +RT- r t +RT r t +rts r t s +RTS- r t s +RTS r t s +RTS's r t s's +RTTOV r t t o v +RTT r t t +RTTY r t t y +RTUK r t u k +RTU r t u +RTVC r t v c +RTVE r t v e +RTVFBiH r t v f b i h +RTV r t v +RTVV r t v v +RTW r t w +RTXC r t x c +RTX r t x +RTZ r t z +Ruao r u a o +RUC r u c +RUC's r u c's +RUFC r u f c +Rukn r u k n +Ruk r u k +RUMC r u m c +RUNX r u n x +Rupf r u p f +Rupr r u p r +rup r u p +Rup r u p +RUP r u p +R.U.R. r u r +RUR r u r +R. U. r u +R.U. r u +RU r u +RUSD r u s d +RutB r u t b +Ruu r u u +RUV r u v +Ruwa r u w a +Ruy r u y +Ruyt r u y t +RVAH- r v a h +RVAR r v a r +Rvat r v a t +RVC r v c +RVCT r v c t +RVD r v d +R. V. E. r v e +RVE r v e +RVGK r v g k +R.V.G. r v g +R. V. J. r v j +R.V.J. r v j +RVM r v m +RVNG r v n g +Rvo r v o +RVO r v o +RVR r v r +R. V. r v +R.V. r v +RV r v +rvs r v s +RVTD's r v t d's +RVU r v u +RVV- r v v +rwa r w a +Rwa r w a +RWA r w a +rwb r w b +RWB r w b +RWC r w c +RWD r w d +RWDSU r w d s u +RWE r w e +RWEs r w e's +RWFC r w f c +RWIS r w i s +rwjf r w j f +R.W.R.J. r w r j +R. W. r w +R.W. r w +RW r w +RWSL r w s l +R. W. S. r w s +RWS r w s +Rxa r x a +RXL r x l +RX- r x +RX r x +RZR r z r +rz r z +R. Z. r z +RZ r z +RZS r z s +S.A.B. s a b +S.A.D. s a d +SAIC- s a i c +SAIC s a i c +Sa'id s a i d +S.A.I. s a i +SAIT's s a i t's +SAKEC s a k e c +S. A. L. s a l +S.A.P.A. s a p a +S.A.P.I. s a p i +S. A. R. s a r +S.A.R. s a r +S. A. s a +S.A. s a +SA s a +S.A.S. s a s +SAS s a s +S.A.V.A. s a v a +Saxl s a x l +Sa'yo s a y o +SAZU s a z u +SBAC's s b a c's +Sbai s b a i +S.B.A. s b a +SBA s b a +SBBK s b b k +SBB- s b b +SBB s b b +SBCM s b c m +SBCMT s b c m t +sbc s b c +SBC s b c +SBCs s b c's +SBDE s b d e +SBD s b d +SBE s b e +SBF s b f +SBI s b i +SBKP s b k p +sbk s b k +SBK s b k +SBL s b l +SBML s b m l +SBMNH s b m n h +SBM s b m +SBN s b n +SBOE s b o e +SBOL s b o l +SBOP s b o p +Sborz s b o r z +SBP- s b p +SBP s b p +SBRJ s b r j +sb s b +S. B. s b +S.B. s b +SB- s b +SB s b +SBSE s b s e +SBS s b s +SBS's s b s's +SBTDC s b t d c +SBT s b t +SBTU s b t u +Sbu s b u +SBU s b u +SBY s b y +SBZ- s b z +SBZ s b z +S.C.A. s c a +SCA s c a +SCBA s c b a +SCB s c b +SCBWI s c b w i +SCCA s c c a +SCCA's s c c a's +SCCC s c c c +SCCI s c c i +SCCP s c c p +S.C.C. s c c +SCC s c c +scr s c r +SCR s c r +SCRs s c r's +SCRS s c r s +SCRTC s c r t c +scry s c r y +sc s c +S. C. s c +S.C. s c +SC- s c +SC s c +S.C.S.C. s c s c +SC's s c's +SCS s c s +SCTC s c t c +SCTE s c t e +SCTO s c t o +SCTP s c t p +SCT s c t +SCTS s c t s +SCTV s c t v +SCUAA s c u a a +Scuf s c u f +SCU's s c u's +SCW s c w +Scymn s c y m n +SD&AE s d and a e +SDAP s d a p +SDAPS s d a p s +SDA s d a +SDAS s d a s +SDASS s d a s s +SDAT s d a t +SDAX s d a x +SDB s d b +SDCC s d c c +SDCG s d c g +SDC s d c +SDDI s d d i +SDDOT s d d o t +SDD s d d +SDDS s d d s +Sdei s d e i +sde s d e +Sde s d e +SDF s d f +SDG&E s d g and e +SDHA- s d h a +SDHA s d h a +SDHC s d h c +SDHD s d h d +SDHHD s d h h d +SDH s d h +SDI, s d i +SDI s d i +SDJ s d j +sdk s d k +SDK s d k +SDK's s d k's +SDLC s d l c +SDLP s d l p +SDLP's s d l p's +S. D. L. s d l +S.D.N.Y. s d n y +S. D. s d +S.D. s d +SDSM&T's s d s m and t's +S. D. S. s d s +S.E.C. s e c +Sejms s e j m's +sejr s e j r +Sekl s e k l +Sek s e k +SEK s e k +S. E. s e +S.E. s e +SESL s e s l +ses s e s +Ses s e's +SEs s e's +S.E.S. s e s +SES- s e s +SES s e s +S.E.S.'s s e s's +S.E.X. s e x +Seyh s e y h +SFACS- s f a c s +sfadb s f a d b +SFA s f a +SFA's s f a's +Sfax s f a x +S.F.B.J. s f b j +SFB s f b +SFCC s f c c +SFCH s f c h +SFC s f c +SFDR s f d r +SFD s f d +SFE s f e +SFES s f e s +SFFAS s f f a s +SFFCo s f f c o +SFFH s f f h +SFF s f f +SFG s f g +SFI s f i +S. F. L. s f l +SFL s f l +sfn s f n +SFN s f n +SFOD- s f o d +SFOR s f o r +SFO s f o +SFPA s f p a +SFPD s f p d +SFP s f p +SFRA s f r a +SFRJ s f r j +sFRP- s f r p +SFR s f r +'sf s f +sf s f +S. F. s f +S.F. s f +SF- s f +SF s f +SFSG s f s g +SFSR s f s r +sfs s f s +S.F.'s s f's +SFS- s f s +SFS s f s +SFSS s f s s +SFSU s f s u +SFU s f u +SFWA- s f w a +SFWA s f w a +SFX s f x +SFX's s f x's +SGAE s g a e +SGAP s g a p +SGA s g a +SGA's s g a's +sgb s g b +SGB s g b +S.G.C. s g c +SGC s g c +SGD s g d +SGE s g e +S. G. F. s g f +SGF s g f +SGH- s g h +SGH s g h +Sgip s g i p +SGI s g i +SGL s g l +sgml s g m l +SGML- s g m l +SGML s g m l +SGP s g p +SGRAM s g r a m +sgra s g r a +SGR s g r +S. G. s g +S.G. s g +SGSN s g s n +SGSNs s g s n's +SGS s g s +SGSY s g s y +Sgt. sergeant +sgt s g t +SGT s g t +SGU s g u +SGV s g v +SGX s g x +SHBG s h b g +SHB s h b +SHBT s h b t +SHCA s h c a +Shch- s h c h +SHC s h c +SHG s h g +SHH s h h +S.H.I.E.L.D.'s s h i e l d's +S.H.I.E.L. s h i e l +SHL- s h l +SHL s h l +Shma s h m a +SHMD s h m d +SHM s h m +SHN s h n +S. H. s h +S.H. s h +S.I.D. s i d +SIF s i f +S.I.R. s i r +S. I. s i +S.I. s i +S.I.T. s i t +SJAM s j a m +SJCH s j c h +SJC s j c +SJDA s j d a +S.J.D. s j d +sjef s j e f +SJEM s j e m +S. J. J. F. s j j f +SJK s j k +S.J.L. s j l +SJL s j l +SJM s j m +Sjon s j o n +SJPCD s j p c d +SJP s j p +SJPT s j p t +SJR s j r +SJR's s j r's +S. J. s j +S.J. s j +SJ s j +SKB- s k b +SKB s k b +SKC s k c +SKD s k d +S. K. F. s k f +SKF- s k f +SKF s k f +SKG s k g +SKH s k h +SKJ s k j +SKPC s k p c +SKP s k p +SKR s k r +SKSD s k s d +S. K. s k +S.K. s k +SK- s k +SK s k +S. K. S. s k s +S.L.A.A.'s s l a a's +SLAF s l a f +sla s l a +SLA s l a +SLAs s l a's +SLCO s l c o +SLC- s l c +SLC s l c +SLDL s l d l +SLD- s l d +SLD s l d +S.L.E. s l e +SLF s l f +SLG s l g +SLHS s l h s +S.L.I.F.E.R. s l i f e r +SLL s l l +SLMC s l m c +S. L. M. s l m +SLM s l m +SLN s l n +SLPIM s l p i m +SLP s l p +SLRC s l r c +SLR s l r +S&L s and l +SLS&E s l s and e +SLSF s l s f +SLSK s l s k +S. L. s l +S.L. s l +SL- s l +SL s l +SLS s l s +SLVR s l v r +SLV s l v +S.M.A.R.T. s m a r t +S.M.A.S.H. s m a s h +SMA s m a +SMB s m b +SMe s m e +SME s m e +SME's s m e's +SMEs s m e's +SMF s m f +SMG- s m g +SMG s m g +SMHI s m h i +SMH s m h +SmI s m i +SMI s m i +SMJR s m j r +SMK s m k +SMLS s m l s +SMMT s m m t +SMN s m n +SMNS s m n s +SMP s m p +SMP's s m p's +SMPSs s m p s's +SMPTE s m p t e +SMRJ s m r j +smr s m r +SMR- s m r +SMR s m r +S&M s and m +sm s m +S. M. s m +S.M. s m +SM s m +S.M.'s s m's +SMs s m s +S.M.S. s m s +SMS. s m s +SMS s m s +SMS's s m s's +SMSU s m s u +SMTP s m t p +S.M.T. s m t +SMT s m t +SMTV s m t v +SMU s m u +SMW s m w +SMX s m x +Smyl s m y l +SMYS s m y s +SNA s n a +SNBA s n b a +SNB s n b +SNCB s n c b +SNCC s n c c +SNCF s n c f +SNC- s n c +SNC s n c +SNDC s n d c +SND s n d +SNEP- s n e p +SNEP s n e p +SNESjr s n e s j r +SNES s n e s +SNET s n e t +SNF s n f +SNFU s n f u +sngle s n g l e +SNGM- s n g m +SNG s n g +SNJ- s n j +SNK s n k +SNLA s n l a +SNL s n l +SNLS s n l s +SNMCMG s n m c m g +SNMC s n m c +SNMMA s n m m a +SNMP s n m p +SNNPR s n n p r +SNNR s n n r +SNPJ s n p j +SNP s n p +SNPs s n p's +SNRI s n r i +SNR s n r +SNSD's s n s d's +SnSe s n s e +sn- s n +S. N. s n +S.N. s n +SN- s n +SN s n +SNS s n s +S.O.E. s o e +S. O. s o +S.O. s o +Sos s o's +S.O.S. s o s +SOS s o s +Sovn s o v n +Sov s o v +SOX s o x +S. O. Y. s o y +S&P 500 s and p five hundred +S.P.A.L. s p a l +SPB s p b +SPCAs s p c a's +SPCG s p c g +SPCK s p c k +S.P.C. s p c +SPC s p c +SPDI s p d i +SPD- s p d +SPD s p d +Spe s p e +SPE s p e +SPES s p e s +SPFH s p f h +SPFL s p f l +SPF s p f +SPG s p g +SPHL s p h l +SPH- s p h +SPH s p h +SPINE's s p i n e's +SPIN's s p i n's +SPIR s p i r +SPI s p i +S.P.I.T. s p i t +SPJA s p j a +SPK s p k +SPLA- s p l a +SPLA s p l a +SPLC s p l c +SPLM- s p l m +SPL s p l +SPME s p m e +SPML s p m l +SPMRL s p m r l +SPM s p m +SPNJ s p n j +SPNM s p n m +SPNN s p n n +SPN s p n +Spoa s p o a +SPOC s p o c +SPOF s p o f +SPO s p o +SPOU s p o u +SpPIn s p p i n +SPP s p p +SPP's s p p's +SPQA s p q a +SPRL s p r l +SPRM s p r m +SPR s p r +SPRU s p r u +S&P s and p +SPSA s p s a +SPSL s p s l +S. P. s p +S.P. s p +SP- s p +SP s p +SP's s p's +SPs s p's +SPS- s p s +SPS s p s +SPSS s p s s +SPTA s p t a +SPT s p t +SPUC s p u c +SPUP s p u p +SPU s p u +SPUs s p u's +SPV s p v +SPX- s p x +SQA s q a +SQBB s q b b +SQI- s q i +SQI s q i +sql s q l +SQL- s q l +SQL s q l +SQM s q m +SQQ- s q q +sq s q +S. Q. s q +SQ s q +Sra s r a +SRA s r a +SRAs s r a's +Srba s r b a +SRBIJA s r b i j a +Srbi s r b i +SRBOC s r b o c +SRBP s r b p +SRB s r b +SRBs s r b's +SRBY s r b y +SRCC s r c c +SRCL s r c l +SRC s r c +SRC's s r c's +SRCS s r c s +S.R.E. s r e +SRE s r e +SRFC s r f c +S.R.F. s r f +SRF s r f +S.R.G. s r g +SRG s r g +SRGs s r g's +Srhir s r h i r +SRH s r h +S. R. J. s r j +SRK- s r k +SRK s r k +SRK's s r k's +SRLGs s r l g's +S.R.L. s r l +SRL s r l +SRMC s r m c +S.R.M. s r m +SRM s r m +SRMs s r m's +Srni s r n i +SRN s r n +srp s r p +SRPT s r p t +S. R. R. s r r +SRR s r r +sr s r +S. R. s r +S.R. s r +SR- s r +SR s r +SRS s r s +SRT- s r t +SRT s r t +SRT's s r t's +Srul s r u l +SRU s r u +SRU's s r u's +SRV s r v +srx s r x +SRY s r y +SRZ- s r z +SSAA s s a a +SSAB s s a b +SSAC s s a c +SSAR s s a r +SSA s s a +SSAs s s a's +SSBSE s s b s e +S.S.B. s s b +SSB s s b +SSCCC s s c c c +S.S.C. s s c +SSC- s s c +SSC s s c +SSCV s s c v +SSDB s s d b +SSDL s s d l +SSDP s s d p +S.S.D. s s d +SSD s s d +SSDs s s d's +SSE s s e +SSFL s s f l +SSF s s f +SSGRC s s g r c +SSG- s s g +SSG s s g +SSHSA's s s h s a's +SSH s s h +SSI s s i +SSIS s s i s +SSJA s s j a +SSKI's s s k i's +SSK s s k +SSLAM s s l a m +ssl s s l +SSL s s l +SSLT s s l t +SSME s s m e +SSM- s s m +SSM s s m +SSN- s s n +SSN s s n +SSOD s s o d +SSOF s s o f +SSO s s o +SSoSV s s o s v +SSPA s s p a +SSPH s s p h +SSPR s s p r +SSP s s p +SSPX s s p x +SSQ s s q +SSRAA s s r a a +SSRF s s r f +SSRI s s r i +SSRIs s s r i's +SSRN s s r n +SSRP s s r p +SSR s s r +S&S s and s +SSSC s s s c +SSSI s s s i +SSSR s s s r +ss s s +S. s s's +Ss s's +S. S. s s +S.S. s s +SS- s s +SSS s s s +SSTC- s s t c +SSTH s s t h +SSTL- s s t l +SSTL s s t l +SSTR s s t r +SST s s t +Ssu s s u +SSU s s u +SSVC s s v c +SSV s s v +SSWAHS s s w a h s +SSWC s s w c +ssw s s w +SSW s s w +SSX s s x +Ssy s s y +ST&AJ s t and a j +S.T.A.L.K.E.R. s t a l k e r +S.T.A.T.U.S. s t a t u s +STB s t b +STCC- s t c c +STCC s t c c +S.T.C. s t c +STC s t c +STCW s t c w +STDP s t d p +STDs s t d's +STD- s t d +STD s t d +STF s t f +stfv s t f v +STGs s t g's +STG s t g +STIs s t i's +STIS s t i s +STi s t i +STI- s t i +STI s t i +STK s t k +stl s t l +S.T.L. s t l +STL s t l +STMB s t m b +STMIK s t m i k +STM's s t m's +STM s t m +STNS s t n s +STN- s t n +STN s t n +STPI s t p i +STPNS s t p n s +STPRI s t p r i +STP s t p +S.T.R.I.D.E. s t r i d e +STRV s t r v +Stryj s t r y j +STRZ s t r z +St. saint +S&T s and t +STSAT- s t s a t +STScI s t s c i +ST's s t's +STS- s t s +STS s t s +S. T. s t +S.T. s t +ST- s t +STT s t t +STVL s t v l +STVS s t v s +STV s t v +STXBP s t x b p +STX s t x +S.U.C. s u c +SUDV s u d v +suo s u o +Suo s u o +S. U. s u +SUSV s u s v +Susz s u s z +SUTs s u t's +SUT s u t +Suu s u u +SUVF s u v f +SUVs s u v's +SUV- s u v +SUV s u v +SUWN s u w n +SUW s u w +Suy s u y +Svac s v a c +svar s v a r +SVA s v a +SVB s v b +SVC s v c +SVD s v d +sve s v e +Sve s v e +SVGA s v g a +SVG s v g +SVGT s v g t +SVIA s v i a +SVM's s v m's +SVMs s v m's +SVM s v m +svn s v n +svom s v o m +SVOPC s v o p c +SVO s v o +SVP s v p +SVPW s v p w +SVSCEP s v s c e p +SVSC s v s c +SVS s v s +sv s v +S. V. s v +S.V. s v +SV- s v +SV s v +SVTs s v t's +SVT s v t +S.V.U. s v u +SVU s v u +SVV s v v +SVW s v w +SVYASA s v y a s a +SVZ s v z +SWABC s w a b c +SWAC s w a c +S.W.A. s w a +SWC s w c +SWD- s w d +SWD s w d +S.W.E.A.T. s w e a t +Swe s w e +SWE s w e +SWF s w f +SWG s w g +SWHL s w h l +SWHS s w h s +SWH s w h +SWJN s w j n +S. W. K. s w k +SWMRS s w m r s +SWM s w m +SWNH s w n h +SWOC s w o c +S.W.O.R.D. s w o r d +SWPA s w p a +SWPL s w p l +SWP s w p +SwRI s w r i +SWR- s w r +SWR s w r +S&W s and w +SWS s w s +sw s w +S. W. s w +S.W. s w +SW- s w +SW s w +SWTPC s w t p c +SWT s w t +SWWTP s w w t p +SXCT s x c t +SXN s x n +SXSW s x s w +sx s x +S.X. s x +SX- s x +SX s x +SXY s x y +SXZ s x z +SysML s y s m l +SysRq s y s r q +Sys s y's +SYS s y s +SysV s y s v +S. Y. s y +S.Y. s y +SyT s y t +syv s y v +Syxx s y x x +SZA s z a +Szasz s z a s z +SZD- s z d +SZDSZ s z d s z +Szer s z e r +szkic s z k i c +Szklo s z k l o +Szlak s z l a k +SzMME s z m m e +Szpir s z p i r +SZSE s z s e +sz s z +S. Z. s z +SZ s z +Szu s z u +Szyk s z y k +SZZ s z z +taf t a f +Taf t a f +TAF t a f +T.A.P. t a p +T.A.S. t a s +T. A. t a +T.A. t a +T&A t and a +TAZ t a z +TBA t b a +TBCA t b c a +TBCs t b c's +TBCS t b c s +TBD t b d +TBE t b e +TBF t b f +TBHS t b h s +TBI t b i +TBK- t b k +TBK t b k +TBL t b l +TBMA t b m a +TBMM t b m m +TBMs t b m's +T.B.M. t b m +TBM t b m +TBNJ t b n j +TBN t b n +TBP- t b p +TBP t b p +TBSC t b s c +TBS t b s +T. B. t b +T.B. t b +TB- t b +TB t b +TBTG t b t g +TBT t b t +TBV t b v +TBWA t b w a +TBX t b x +TCAR t c a r +TCA t c a +TCC t c c +TCDB t c d b +TCDC t c d c +TCDD t c d d +TCDOG t c d o g +TCDSB t c d s b +TCEB t c e b +T.C.E. t c e +TCE t c e +TCFAQ t c f a q +TCF t c f +TCGA t c g a +TCG- t c g +TCG t c g +TCHC t c h c +TCHRD t c h r d +TCH t c h +TCiAP t c i a p +TCI t c i +TCKs t c k's +TCLP t c l p +TCL t c l +TCM t c m +TCO t c o +TCPL t c p l +TCPMP t c p m p +TCP's t c p's +TCP- t c p +TCP t c p +TCR t c r +T. C. S. t c s +TCS t c s +tc t c +Tc- t c +T. C. t c +T.C. t c +TC- t c +TC t c +TCT t c t +TCU's t c u's +TCUs t c u's +tcu t c u +TCU t c u +TCV t c v +TCWC t c w c +TCWP t c w p +TCW t c w +TDA t d a +tdb t d b +TDCC t d c c +TDCi t d c i +TDCJ t d c j +TDC's t d c's +TDC t d c +TDD t d d +TDE t d e +TDF t d f +TDH t d h +TdIF t d i f +TDI t d i +TDK t d k +TDLAS t d l a s +TDMA- t d m a +TDMA t d m a +TDME t d m e +TDM- t d m +TDM t d m +TDO t d o +TDP t d p +TDRS t d r s +TDR t d r +TDSB's t d s b's +TD's t d's +TDs t d's +T. D. t d +T.D. t d +TDT t d t +Teatr t e a t r +TEBD t e b d +T. E. t e +T.E. t e +TEVT t e v t +TFAP t f a p +TFAS t f a s +TFA t f a +TFC t f c +TFCU t f c u +TFF t f f +TFG t f g +T.F.H. t f h +TFH t f h +TFIH t f i h +TFIID t f i i d +TFK t f k +TFL t f l +TFM t f m +TFN t f n +TFO t f o +TFP t f p +TFR t f r +TFSA t f s a +TFSI t f s i +TFSP t f s p +TFS t f s +T&F t and f +TFTC t f t c +tf t f +T. F. t f +T.F. t f +TF- t f +TF t f +TFTR t f t r +TFT t f t +TFVC t f v c +TFWA t f w a +TFW t f w +TGA t g a +T. G. C. t g c +TGC t g c +tge t g e +T.G.E. t g e +TGE t g e +tgf t g f +TGF- t g f +TGF t g f +TGG t g g +TGIF t g i f +TGL t g l +TGM's t g m's +TGM t g m +TGP t g p +TGS t g s +tg t g +T. G. t g +T.G. t g +TG t g +TGVs t g v's +tgv t g v +TGV- t g v +TGV t g v +THB t h b +THCD- t h c d +THCF t h c f +THC- t h c +THC t h c +T.H.E.M. t h e m +T.H.F. t h f +THF t h f +THG t h g +THHK t h h k +T.H.I.N.K. B.I.G. t h i n k b i g +THJ- t h j +THLL t h l l +THL t h l +THMP t h m p +THM t h m +THNN t h n n +Thok t h o k +T.H.P. t h p +THQ's t h q's +THQ t h q +THR's t h r's +THR t h r +ThSe t h s e +THSR t h s r +T. H. S. t h s +T. H. t h +T.H. t h +T.H.U.N.D.E.R. t h u n d e r +THW t h w +THX t h x +TIAA- t i a a +TIAA t i a a +T. I. A. t i a +TIFR t i f r +TIF t i f +T.I.'s t i's +T.I. t i +Tiu t i u +tiv t i v +Tiv t i v +TIYM t i y m +Tiy's t i y's +TJD- t j d +TJFR t j f r +T.J.F. t j f +T. J. J. t j j +TJJ t j j +TJP t j p +TJRC t j r c +T. J. S. t j s +T. J. t j +T.J. t j +TJ t j +t'ju t j u +TJX t j x +tjz t j z +TKAG t k a g +TKA's t k a's +TKA t k a +T. K. B. t k b +TKB- t k b +TKCA- t k c a +TKD t k d +TKE t k e +TKI t k i +tko t k o +Tko t k o +TKO- t k o +TKO t k o +TKP t k p +TK's t k's +TKS t k s +tk t k +T. K. t k +T.K. t k +TK- t k +TK t k +TKT t k t +Tlas t l a's +TLA t l a +TLC's t l c's +TLC t l c +TlCu t l c u +TLDs t l d's +TLDS t l d s +TLD t l d +TLE t l e +TLF t l f +TLG t l g +TLH t l h +tli t l i +TLI t l i +TLM t l m +TLN t l n +tlp t l p +TLP t l p +TLRs t l r's +TLR t l r +TLs t l's +TLS t l s +TLTB t l t b +tl t l +T. L. t l +T.L. t l +TL- t l +TL t l +Tluk t l u k +TLU's t l u's +Tluszcz t l u s z c z +TLV- t l v +TLV t l v +TMA- t m a +TMA t m a +TMBG t m b g +TMB t m b +tmc t m c +TMC t m c +TMDL t m d l +TMD t m d +TMEM t m e m +TMF t m f +TMGs t m g's +TMG t m g +TMI t m i +TMJ t m j +T. M. M. t m m +TMMTX t m m t x +TMNT t m n t +TMOK t m o k +tmos t m o s +TMO's t m o's +TMOS t m o s +TMO t m o +TMPRSS t m p r s s +TMP t m p +tmRNA t m r n a +TMRS t m r s +TMR t m r +TMSI t m s i +TMSs t m s's +T.M.s t m's +TM's t m's +TMS t m s +tm t m +T. M. t m +T.M. t m +TM- t m +TM t m +T.M.T. t m t +TMT t m t +TMU t m u +TMX t m x +TMZ t m z +TNA's t n a's +TNAS t n a s +tna t n a +TNA t n a +tnbc t n b c +TNCA t n c a +TNCC t n c c +TNCDSB t n c d s b +TNC- t n c +TNC t n c +TNFA t n f a +TNFRSF t n f r s f +TNFSF t n f s f +TNF- t n f +TNF t n f +TNGHT t n g h t +TNG t n g +TNIP t n i p +TNI t n i +TNK- t n k +TNL t n l +TNMM t n m m +TNMS t n m s +TNM t n m +TNNC t n n c +TNN t n n +TNO t n o +TNPL t n p l +TNP t n p +TNQ t n q +TNRIS t n r i s +TNRP t n r p +TNR t n r +TNSDL t n s d l +TNSTC- t n s t c +TNSTC t n s t c +TNS- t n s +TNS t n s +TNTAs t n t a's +tn t n +T. N. t n +T.N. t n +TN- t n +TN t n +TNT's t n t's +TNT t n t +TNX- t n x +T.O.'s t o's +T. O. t o +T.O. t o +TOTP t o t p +tou t o u +Tou t o u +ToU t o u +TOU t o u +TPAC t p a c +TPAO t p a o +tPA t p a +TPA- t p a +TPA t p a +TPB t p b +TPC- t p c +TPC t p c +TPEC t p e c +TPF t p f +TPG- t p g +TPG t p g +TPH t p h +TPI t p i +TPLF t p l f +TPMS t p m s +TPM t p m +TPO t p o +TPP t p p +tpr t p r +TPR t p r +TPS- t p s +TPS t p s +tp t p +T. P. t p +T.P. t p +TP- t p +TP t p +TPU t p u +TPVs t p v's +TPWS t p w s +TPW t p w +TQR t q r +TQS t q s +T.Q. t q +TQ t q +tra t r a +Tra t r a +TRA t r a +TRAXX t r a x x +Trbic t r b i c +T. R. B. t r b +TRB t r b +Trcek t r c e k +TRC t r c +TRDB- t r d b +TRD t r d +TRFA t r f a +TRF t r f +TRG t r g +TRHS t r h s +TRH t r h +T.R.I.B.E. t r i b e +TRIZ t r i z +TrkA t r k a +TRL- t r l +TRL t r l +Trmcic t r m c i c +TRMM t r m m +T.R.M.P.A. t r m p a +T. R. M. t r m +TRM- t r m +tRNA t r n a +TRNC t r n c +TRPC t r p c +TRPM t r p m +TRPP t r p p +TRP's t r p's +TRPs t r p's +TRP t r p +TRPV t r p v +TRPY t r p y +TRSC t r s c +TRSI t r s i +trs t r s +T.R.S. t r s +TRS t r s +TRSV t r s v +tr t r +T. R. t r +T.R. t r +TR- t r +TR t r +TRT t r t +T.R.U. t r u +truTV t r u t v +TruTV t r u t v +Trve t r v e +TRVL t r v l +TRV t r v +TRW t r w +TRX t r x +TRZ t r z +Tsa t s a +TSA t s a +TSB t s b +TSCC t s c c +TSC- t s c +TSC t s c +TSF t s f +TSG- t s g +TSG t s g +TSH t s h +TSIO- t s i o +TSI t s i +tsit t s i t +Tsiv t s i v +TSJDF t s j d f +TSLA t s l a +TSLC t s l c +TSL t s l +TSN t s n +T.S.O.L. t s o l +tso t s o +Tso t s o +T.S.O. t s o +TSO t s o +TSP t s p +TSQ- t s q +TSRA t s r a +TSRTC t s r t c +tsr t s r +TSR t s r +TSSA t s s a +TSS t s s +TSTA t s t a +TSTC t s t c +Ts t's +T. S. t s +T.S. t s +TS- t s +TS t s +TST t s t +tsus t s u s +Tsvi t s v i +TSV t s v +TSW t s w +TSX t s x +TTAB t t a b +Tta t t a +TTA t t a +TTBB t t b b +TTCP t t c p +TTC's t t c's +TTC t t c +TTDC t t d c +TTD t t d +tteok t t e o k +tte t t e +TTFF t t f f +TTHS t t h s +TTH t t h +TTI t t i +TTKG t t k g +TTKST t t k s t +TTL- t t l +TTL t t l +T.T.N. t t n +ttp t t p +TTP t t p +TTR t t r +TTS t t s +T&T t and t +tt- t t +T. T. t t +T.T. t t +TT- t t +TT t t +TTU t t u +TTV t t v +TTW t t w +TUAW t u a w +TUBB t u b b +TUDCA t u d c a +TUHUS t u h u s +TUKP's t u k p's +T'uqu t u q u +TUSCC t u s c c +TUSC t u s c +T.U. t u +TU- t u +TU t u +Tuzk t u z k +tvaan t v a a n +Tvam t v a m +TVARK t v a r k +tva t v a +TVA t v a +TVB's t v b's +TVB t v b +TVCM t v c m +TVC t v c +TVD t v d +TVETs t v e t's +TVE t v e +TVF t v f +TVGN t v g n +TVGOS t v g o s +TVii t v i i +TVIn t v i n +TVi t v i +TVI t v i +TVL t v l +TVMK t v m k +TVM t v m +TVN t v n +TVNZ's t v n z's +TVNZ t v n z +tvo t v o +Tvo t v o +TVO t v o +TVP t v p +TVRi t v r i +tvr t v r +TVR t v r +TVSpy t v s p y +tvs t v s +TV's t v's +TVs t v's +TVS t v s +tVTA t v t a +TVT- t v t +TVT t v t +tv t v +T. V. t v +T.V. t v +TV- t v +TV t v +TVXQ's t v x q's +TVXQ t v x q +TVX t v x +TWAP t w a p +TWA's t w a's +Twa t w a +TWA t w a +T. W. B. t w b +TWCS t w c s +TWC t w c +TWF t w f +Twi t w i +TWI t w i +TWK t w k +T.W.P. t w p +TWP t w p +TWR- t w r +TWR t w r +TWTs t w t's +TWT t w t +T. W. t w +T.W. t w +TW t w +T.W.U. t w u +TWU t w u +TXC t x c +TxDOT t x d o t +TXII t x i i +TXJS t x j s +TXK t x k +TXMM t x m m +TXNDC t x n d c +TXP t x p +TXTC t x t c +TXT t x t +T.X. t x +TX- t x +TX t x +TXU t x u +tya t y a +TYC t y c +TYIB t y i b +TYIN t y i n +Tyk t y k +Tza'ar t z a a r +Tze t z e +TZL t z l +Tzrif t z r i f +Tzuh t z u h +TZUM t z u m +Tzvi t z v i +UAAP's u a a p's +UAAP u a a p +UAB's u a b's +UAB u a b +UAC- u a c +UAC u a c +UADY u a d y +UAE's u a e's +U.A.E. u a e +UAE- u a e +UAE u a e +UAF u a f +UAHC u a h c +UAIC u a i c +UAI u a i +UALR u a l r +UAL u a l +UALVP u a l v p +UAM u a m +UANL u a n l +UAN u a n +UAOC u a o c +UAP u a p +UAR's u a r's +UARS u a r s +UAR u a r +UASLP u a s l p +UASL u a s l +Ua's u a's +UAs u a's +UAS u a s +uat u a t +U.A. u a +UA u a +UAVs u a v's +UAV u a v +U.A.W. u a w +UAW- u a w +UAW u a w +Uba u b a +UBA u b a +UBBC u b b c +UBB u b b +UbcM u b c m +UBC's u b c's +UBC u b c +UBD u b d +Ube u b e +UBE u b e +ubi u b i +Ubi u b i +U.B.I. u b i +UBI u b i +UBM's u b m's +UBM u b m +UBOS u b o s +UBO u b o +UBPR u b p r +UBP u b p +UBRE u b r e +UBRS u b r s +UB's u b's +UBS u b s +UB- u b +UB u b +UBU u b u +UBV u b v +UBX u b x +UCAC u c a c +Ucar u c a r +UCAR u c a r +Uca u c a +UCA u c a +UCAV u c a v +UCBH u c b h +UCBTLA u c b t l a +UCB u c b +UCCE u c c e +UCCJ u c c j +UCCS u c c s +UCC- u c c +UCC u c c +UCDA u c d a +UCDP u c d p +UCD u c d +UCF's u c f's +UCF u c f +UCG u c g +UCIEP u c i e p +UCISA- u c i s a +UCI's u c i's +UCI u c i +UCLA's u c l a's +UCLA- u c l a +UCLA u c l a +UCLH u c l h +UCL u c l +Ucmak u c m a k +UCMJ u c m j +UCMMA u c m m a +UCMP u c m p +UCM u c m +Ucn u c n +UCOS u c o s +UCO u c o +UCP u c p +UCREF u c r e f +UCR u c r +UCSB u c s b +UCSC u c s c +UCSD's u c s d's +UCSD u c s d +UCSF u c s f +UCSN u c s n +uCs u c's +UCS u c s +UCTI u c t i +UCT u c t +Uc u c +U.C. u c +UC- u c +UC u c +Ucuncu u c u n c u +UCU u c u +UCVTS u c v t s +UCW u c w +Uczta u c z t a +UDAR u d a r +UDA's u d a's +Uda u d a +UDA u d a +UDC u d c +UDF u d f +UDG u d g +UDHR u d h r +UDICHI u d i c h i +Udit u d i t +UDK u d k +UDN u d n +Udny u d n y +UDP- u d p +UDP u d p +UDRP u d r p +UDR u d r +UDS u d s +UDTs u d t's +UDT u d t +ud u d +Ud u d +U. D. u d +UD u d +udu u d u +Udu u d u +UDYCO u d y c o +UEA u e a +UEBT u e b t +UEC u e c +Ueda's u e d a's +Uéda u e acute d a +UEFA's u e f a's +UEFA u e f a +UEFI u e f i +UEFS u e f s +UEF u e f +UEGCL u e g c l +UEI u e i +UET u e t +U. E. u e +UE u e +UFABC u f a b c +UFA's u f a's +UFCO u f c o +UFC's u f c's +UFC u f c +UFCW u f c w +UFD u f d +UFE u f e +UFF u f f +UFIA u f i a +UFIP u f i p +UFI u f i +UFJ u f j +UFL's u f l's +UFL u f l +UFM u f m +UFOFU u f o f u +UFORE u f o r e +UFO's u f o's +UFOs u f o's +Ufot u f o t +ufo u f o +Ufo u f o +UFO- u f o +UFO u f o +UFPE u f p e +UFPLS u f p l s +UFPR u f p r +UFRGS u f r g s +UFRJ u f r j +UFSB u f s b +UFSM u f s m +UF's u f's +UFS u f s +UFTP u f t p +UFT u f t +U. F. u f +UF- u f +UF u f +UFU u f u +UFW u f w +Uga u g a +UGA u g a +UGCCWA u g c c w a +UGC- u g c +UGC u g c +UGEED- u g e e d +Uge u g e +UGF- u g f +UGK u g k +UGME u g m e +Ugni u g n i +UGP u g p +Ugra u g r a +Ugrszke u g r s z k e +UGS u g s +UGT u g t +Ug u g +UG u g +UHD u h d +UHF u h f +Uhha u h h a +UHMW u h m w +uhn u h n +UHOA u h o a +UHPT u h p t +UHP- u h p +UHP u h p +UHRA u h r a +UHRF u h r f +UHSAA u h s a a +UHTCs u h t c's +UHTI u h t i +Uhud u h u d +U.H. u h +uhur u h u r +UIAA u i a a +Uiam u i a m +UIA u i a +UIAW u i a w +Uibh u i b h +Uible u i b l e +UIC u i c +UID u i d +UIFL u i f l +Uig u i g +UIL u i l +UIMC u i m c +UIM u i m +UIN u i n +UIP u i p +uisae u i s a e +UISD u i s d +UiS u i s +UI's u i's +UIS u i s +UiTM u i t m +uit u i t +Uit u i t +UIT u i t +UI- u i +UI u i +UIV u i v +Uiy u i y +UJA u j a +UJC u j c +Ujed u j e d +UJEP u j e p +Ujsag u j s a g +UJS u j s +uj u j +U.J. u j +UJ u j +Uka u k a +UKA u k a +UKCC u k c c +UKC u k c +Ukhra u k h r a +Ukic u k i c +UKIP's u k i p's +Uki u k i +UKM u k m +UKNC u k n c +Ukoh u k o h +Uko u k o +UKPC u k p c +UKRDA's u k r d a's +UKRDA u k r d a +ukr u k r +Ukr u k r +UKR u k r +UK's u k's +UKTV's u k t v's +UKTV u k t v +Ukui u k u i +u'k u k +uk u k +Uk u k +U.K. u k +UK- u k +UK u k +UKUP u k u p +UKVRN u k v r n +UKW u k w +Ukwu u k w u +ULC- u l c +ULC u l c +ULDB u l d b +ULEB u l e b +ULEV u l e v +ULFA u l f a +Ulic u l i c +Ull u l l +Ulms u l m's +UL's u l's +ULS u l s +UlSU u l s u +Uluj u l u j +U.L. u l +UL u l +ulus u l u s +Ulus u l u's +ULV u l v +Ulwa u l w a +Ulwe u l w e +UMBC u m b c +Umbr u m b r +UMCC u m c c +UMC u m c +UMD u m d +umelcu u m e l c u +UMe u m e +UMF u m f +UMG u m g +UMHB u m h b +UMH u m h +UMKC's u m k c's +UMKC u m k c +UMK u m k +Umla u m l a +UML u m l +Umme u m m e +UMMSM u m m s m +UMNO's u m n o's +Umno u m n o +UMNO u m n o +UMN u m n +UMO u m o +UMPP u m p p +UMP's u m p's +Umptee- u m p t e e +Umri u m r i +UM's u m's +UMTS u m t s +UMTV u m t v +UMUC u m u c +U. M. u m +U.M. u m +UMWA's u m w a's +UMWA u m w a +UMW's u m w's +UNAB u n a b +UNAFF u n a f f +UNAH u n a h +UNAMIR u n a m i r +UNAPACK's u n a p a c k's +UNB u n b +UNCAC u n c a c +UNCAF u n c a f +UNCG's u n c g's +UNCG u n c g +UNCHE u n c h e +UNCHS u n c h s +UNCMAC u n c m a c +UNCRO u n c r o +UNC's u n c's +UNCTAD u n c t a d +UNC- u n c +UNC u n c +UNCW u n c w +UNFCCC u n f c c c +UNFICYP u n f i c y p +UNFPA u n f p a +UNFP u n f p +UNF- u n f +UNF u n f +UNFWP u n f w p +UNGC u n g c +UNHCR u n h c r +UNHRC's u n h r c's +UNH u n h +UNJLC u n j l c +Unli u n l i +UNL u n l +Unlu u n l u +UNLV u n l v +UNMCK u n m c k +UNMC- u n m c +UNMC u n m c +UNMEE's u n m e e's +UNMEE u n m e e +UNMIH u n m i h +UNMIK u n m i k +UNMIL u n m i l +UNMOVIC u n m o v i c +unm u n m +UNM u n m +UNNC u n n c +unnd u n n d +UNODC u n o d c +UNOMIG u n o m i g +UNPA u n p a +UNPD u n p d +UNPO u n p o +UNP u n p +UNRA u n r a +UNRSGN u n r s g n +UNRWA u n r w a +UNSCOM u n s c o m +UNSCR u n s c r +UNSC's u n s c's +UNSC u n s c +UNSh u n s h +Unst u n s t +Uns u n's +U.N.'s u n's +UNSW u n s w +UNTSO u n t s o +UNTS u n t s +Unt u n t +UNT u n t +U.N. u n +UNU u n u +UNV u n v +UNWE u n w e +UNWTO u n w t o +Unz u n z +UOCl u o c l +UODA u o d a +UofM u o f m +UOHI u o h i +Uoho u o h o +UOJCA u o j c a +UOL u o l +UOMINI u o m i n i +UoMs u o m's +UOP u o p +uORF u o r f +UOR u o r +UO's u o's +UO u o +UoW u o w +UPABA u p a b a +UPBs u p b's +UPCI u p c i +UPC u p c +UPDRS u p d r s +UPDS u p d s +Upd u p d +UPD u p d +UPEC u p e c +UPFA- u p f a +UPFA u p f a +UPF's u p f's +Upf u p f +UPF u p f +UPGMA u p g m a +UPHSD u p h s d +UPIN u p i n +Upir u p i r +UPI's u p i's +UPI u p i +UPLA u p l a +UPLB u p l b +UPL u p l +UPMC u p m c +UPMSS u p m s s +UPND u p n d +UPNE u p n e +UPnP u p n p +UPN u p n +UPOV u p o v +UPPP u p p p +Uppu u p p u +UPRR u p r r +UPR u p r +UPSC u p s c +U.P.S.E.B. u p s e b +UPSIDC u p s i d c +UPSID u p s i d +UPSI u p s i +UPSMF u p s m f +UPTI u p t i +UPTU u p t u +U.P. u p +UPW u p w +UPX u p x +UPyD u p y d +UQAIB u q a i b +UQAM's u q a m's +UQC u q c +U.Q.P. u q p +UQP u q p +uq u q +UQ u q +URAA u r a a +URAC u r a c +URAP- u r a p +URAP u r a p +URBACT u r b a c t +URBED u r b e d +Urbz u r b z +URCL u r c l +URCSA u r c s a +URCs u r c's +URC u r c +Urdd u r d d +Urei u r e i +urf u r f +Urla u r l a +URLhttp u r l h t t p +URLs u r l's +URL u r l +URNU u r n u +UROC u r o c +URRACA u r r a c a +URSB u r s b +URSEC u r s e c +URSS u r s s +URW's u r w's +URW u r w +URZ u r z +USAACE u s a a c e +USAAC u s a a c +U.S.A.A.F. u s a a f +USAAF u s a a f +USAB u s a b +USACE u s a c e +USAC u s a c +USAFE u s a f e +USAF u s a f +USAHA u s a h a +USAID's u s a i d's +USAID- u s a i d +USAID u s a i d +Usal u s a l +usan u s a n +Usan u s a n +USAN u s a n +usao u s a o +USAO u s a o +USAPA u s a p a +USARPS u s a r p s +USASA u s a s a +USA's u s a's +usata u s a t a +USATF u s a t f +USAT u s a t +usa u s a +.USA u s a +U.S.A.. u s a +U.S.A. u s a +U.S.A u s a +USA- u s a +USA. u s a +USA u s a +USAya u s a y a +USBA u s b a +USBC's u s b c's +USBC u s b c +USBs u s b's +USB- u s b +USB u s b +USBWA u s b w a +USCCB u s c c b +USCF u s c f +USCGAUX u s c g a u x +USCGC u s c g c +USCG u s c g +USCHS u s c h s +Uscie u s c i e +USCIRF u s c i r f +USCIS u s c i s +USCITA u s c i t a +USCOB u s c o b +USCRP u s c r p +U.S.C.'s u s c's +USC's u s c's +USCT u s c t +U.S.C. u s c +USC- u s c +USC u s c +USDA's u s d a's +U.S.D.A. u s d a +USDA- u s d a +USDA u s d a +USDoE u s d o e +USDOJ u s d o j +USDP u s d p +USDTV u s d t v +usd u s d +USD u s d +USEER u s e e r +USENIX u s e n i x +USFbA u s f b a +USFE u s f e +USFF u s f f +USFLPA u s f l p a +USFL u s f l +USFS u s f s +USF u s f +USF&WS u s f and w s +USFWS u s f w s +USFW u s f w +USGA u s g a +U.S.G.S. u s g s +USGS- u s g s +USGS u s g s +U.S. H.R. u s h r +UShs u s h's +Usia u s i a +USIA u s i a +usih u s i h +USIP's u s i p's +USIP u s i p +USISL u s i s l +USIS u s i s +USIU u s i u +USJA u s j a +USJ u s j +USLES u s l e s +USLHE u s l h e +USLMRA u s l m r a +USLS- u s l s +USLTA u s l t a +USL- u s l +USL u s l +Uslu u s l u +Usmar u s m a r +USMA u s m a +USMC M u s m c +USMCR u s m c r +USMC's u s m c's +USMC u s m c +USMLE u s m l e +USML u s m l +USMMA u s m m a +USM u s m +Usna u s n a +USNA u s n a +USNG u s n g +Usnic u s n i c +USNM u s n m +USNO u s n o +USNR u s n r +USNSCS u s n s c s +USNS u s n s +USNTDP u s n t d p +U.S.N. u s n +USN u s n +USOC's u s o c's +USOC u s o c +Usoi u s o i +Usos u s o's +Uso u s o +USO u s o +USPHL u s p h l +USPO u s p o +USPPD u s p p d +U.S.P.Q. u s p q +USPSA u s p s a +USPs u s p's +USPS- u s p s +USPS u s p s +USPTA u s p t a +USPTO u s p t o +U.S.P. u s p +USP u s p +USRA u s r a +USRC u s r c +USSA u s s a +USSBA u s s b a +USSB u s s b +USSD u s s d +USSF's u s s f's +USSF u s s f +USSR's u s s r's +USSR u s s r +U.S.'s u s's +US's u s's +U.S.S. u s s +USS u s s +Usti u s t i +Ustka u s t k a +USTR's u s t r's +USTR u s t r +UST's u s t's +Usui u s u i +usum u s u m +U. S. u s +U.S. u s +Usut u s u t +USVC u s v c +Usvit u s v i t +USVI u s v i +U.S.V. u s v +USWA u s w a +USWNT's u s w n t's +USWNT u s w n t +U.S.W. u s w +USW u s w +Uta's u t a's +uta u t a +Uta u t a +UTA u t a +UTB u t b +Utca u t c a +UTC- u t c +UTC u t c +Utd's u t d's +Utd u t d +utea u t e a +UTEP u t e p +Utes u t e's +UTET u t e t +UTF- u t f +UTF u t f +Uth u t h +uti u t i +Uti u t i +UTJ u t j +UTMB u t m b +UTMF u t m f +UTM u t m +Utne u t n e +Utnur u t n u r +Uto u t o +UTO u t o +UTPB u t p b +UTP u t p +utrci u t r c i +UTRGV u t r g v +UTR u t r +utsav u t s a v +Utsav u t s a v +UTSC u t s c +UTSI u t s i +UT's u t's +UTS u t s +Utsu u t s u +UTTR u t t r +U.T. u t +Utu u t u +Utva u t v a +UTVA u t v a +UTV u t v +UUA u u a +UUCP u u c p +Uuh u u h +UUKKY u u k k y +Uul u u l +Uulu u u l u +UUP u u p +Uusi u u s i +UUs u u's +Uuto u u t o +UUT u u t +Uvac u v a c +UVB u v b +UVC u v c +Uvea u v e a +UVF u v f +UVIMB u v i m b +UVI u v i +uvnitr u v n i t r +Uvo u v o +UVT u v t +UVU u v u +U.V. u v +UV u v +UWAP u w a p +Uwasa u w a s a +UWASA u w a s a +U.W.A. u w a +UWA u w a +UWB u w b +UWC u w c +UWFi u w f i +UWF u w f +UWG u w g +UWH u w h +UWI u w i +UWM's u w m's +UWP u w p +UWRF u w r f +UWSP's u w s p's +UWSU u w s u +UW's u w's +UWS u w s +UW- u w +UW u w +UWW u w w +VASP v a s p +VA's v a's +V. A. v a +V.A. v a +VA- v a +VA v a +V&A v and a +Vav v a v +VAV v a v +VAW- v a w +Vay v a y +VBA v b a +VBBS v b b s +VBE v b e +VBF- v b f +VBI v b i +VBM v b m +VBR v b r +vb v b +V. B. v b +V.B. v b +VB- v b +VB v b +VCA's v c a's +VCA v c a +VCCS v c c s +VCDs v c d's +VCD v c d +VCE v c e +VCH v c h +VCIT v c i t +VCI v c i +VCJ v c j +V. C. M. v c m +VCO v c o +VCP v c p +VCQ v c q +VCRs v c r's +VCR v c r +VCSELs v c s e l's +VCSEL v c s e l +VCs v c's +VCS v c s +VCTs v c t's +VCT v c t +VCU v c u +vc v c +V. C. v c +V.C. v c +VC- v c +VC v c +VCXIV v c x i v +VCX- v c x +VCY v c y +VDAC v d a c +vda v d a +Vda v d a +VDCs v d c's +VDC v d c +vdiq v d i q +VDJ v d j +VDL's v d l's +VDL- v d l +VDL v d l +vdm v d m +VDM v d m +VDP v d p +VDR v d r +VDSL v d s l +VDUP v d u p +VDU v d u +V. D. v d +V.D. v d +VD- v d +VD v d +VDV v d v +V. D. W. v d w +ve v e +Ve v e +V. E. v e +VE v e +vez v e z +Vez v e z +VEZ v e z +V.F.A. v f a +VFA- v f a +VFA v f a +VFC v f c +VFDS v f d s +V.F.D. v f d +VFD v f d +VFIL v f i l +VFLI v f l i +VFL's v f l's +VFL v f l +VFMC v f m c +VFM v f m +vfp v f p +VFP- v f p +vfr v f r +VFR v f r +VF's v f's +VFS v f s +VFTS v f t s +VFTX v f t x +VFU v f u +V. F. v f +V.F. v f +VF- v f +VF v f +VFW- v f w +VFW v f w +VFX v f x +VGA v g a +VGC- v g c +V'Ger v g e r +VGIK v g i k +VGKO v g k o +VGMA v g m a +vgmdb v g m d b +VGMDB v g m d b +VGM v g m +VGo v g o +VGP v g p +Vgr- v g r +VGSCs v g s c's +VGSoM v g s o m +VGTU v g t u +VGU v g u +V. G. v g +V.G. v g +VG- v g +VG v g +VGV v g v +VGX v g x +VHB v h b +VHC v h c +VHD v h d +VHF v h f +VHL v h l +VHP v h p +VHR v h r +VHSL v h s l +VHS v h s +V. H. v h +VH- v h +VH v h +V.I.C.'s v i c's +Vict v i c t +VICT v i c t +viita v i i t a +vijf v i j f +vij v i j +Vij v i j +V.I.P.s v i p's +VIPs v i p's +V.I.P. v i p +VIP- v i p +VIP v i p +V. I. v i +V.I. v i +V. J. J. v j j +VJL v j l +VJs v j's +VJTF v j t f +V. J. v j +V.J. v j +VJ- v j +VJ v j +VKG v k g +VKh- v k h +VKIBC's v k i b c's +V. K. N. v k n +VKOS v k o s +VKS v k s +V. K. v k +V.K. v k +VK v k +Vlah v l a h +VLA v l a +VLBA v l b a +VLBI v l b i +VLB v l b +VLCCs v l c c's +VLCS v l c s +VLC v l c +VLDB v l d b +VLDL v l d l +vlei v l e i +VLE v l e +VLEX v l e x +VLF- v l f +VLF v l f +Vlijt v l i j t +VLN v l n +VLQ v l q +VLR v l r +VLSI v l s i +VLS v l s +VLT v l t +V. L. v l +V.L. v l +VL v l +VMAs v m a's +VMAT- v m a t +VMAT v m a t +VMA- v m a +VMA v m a +VMBO- v m b o +VMCAS v m c a s +VMCM v m c m +VME v m e +VMFM v m f m +VMF- v m f +VMF v m f +VMG v m g +VMI's v m i's +VMI v m i +VMK v m k +VMM- v m m +VMM v m m +VMO- v m o +VMPS v m p s +VMS v m s +VMT v m t +vm v m +V. M. v m +V.M. v m +VM- v m +VM v m +VMX- v m x +VNAF v n a f +VNAV v n a v +VNCH v n c h +VNC v n c +vner v n e r +V.N.E. v n e +VNI v n i +VNO- v n o +VNO v n o +VNPT v n p t +VNQDD v n q d d +VNR v n r +VNSA v n s a +VNS v n s +VNTH v n t h +VNTR v n t r +VNU v n u +V. N. v n +V.N. v n +VN- v n +VN v n +V.O. v o +VO v o +Vov v o v +VOX's v o x's +Voz v o z +VPAL v p a l +VPB- v p b +VPD v p d +VPH v p h +vpis v p i s +V. P. I. v p i +VPI v p i +VPLS v p l s +VPL v p l +VPM v p m +VPNs v p n's +VPN v p n +VPO's v p o's +VPP v p p +VPRO's v p r o's +VPRO v p r o +VPS v p s +vpu v p u +V. P. v p +V.P. v p +VP- v p +VP v p +VQT v q t +vq v q +VQ v q +vrak v r a k +VRA v r a +Vrba v r b a +Vrbuv v r b u v +VRB- v r b +VRD v r d +Vrej v r e j +vrem v r e m +VRE v r e +VRF v r f +Vrin v r i n +VRI v r i +vrj v r j +VRLA v r l a +vrn v r n +vroee v r o e e +vrou v r o u +vrouw v r o u w +Vrouw v r o u w +VRO v r o +VRR v r r +Vrsac v r s a c +VRSA v r s a +VR's v r's +VRS v r s +Vrtis v r t i's +VRT's v r t's +VRT v r t +V&R v and r +vr v r +V. R. v r +V.R. v r +VR- v r +VR v r +VRV v r v +VSANs v s a n's +VSAN v s a n +VSAP's v s a p's +VSATs v s a t's +VSA- v s a +VSB's v s b's +VSB v s b +VSCP v s c p +VSC v s c +VSD v s d +Vsekh v s e k h +vse v s e +VSE v s e +VSFV v s f v +VSG v s g +VSI v s i +VSM v s m +VSNL v s n l +vso v s o +VSO v s o +VSPs v s p's +VSP v s p +VSS v s s +VSTM v s t m +VST v s t +VSU v s u +vs. versus +_vs._ versus +VSV v s v +VSW v s w +VSX v s x +vsyo v s y o +VSZ v s z +VTA v t a +VTB v t b +VTCO v t c o +VTD v t d +VTEC v t e c +VTE v t eL +Vtic v t i c +VTi v t i +VTL v t l +VTM- v t m +VTM v t m +VTR v t r +VTSK v t s k +V&T's v and t's +VTS v t s +VTU v t u +V&T v and t +VTVL v t v l +V.T. v t +VT- v t +VT v t +VTV v t v +Vuur v u u r +VUU v u u +V.V.B. v v b +VVCS v v c s +VVD v v d +VVIPs v v i p's +VVI v v i +VVMC v v m c +VVN- v v n +VVO v v o +VVS v v s +VVT v v t +V. V. v v +V.V. v v +VV v v +VWI- v w i +VWM v w m +VWR v w r +VW's v w's +VWs v w's +VWS v w s +VW- v w +VW v w +VXL v x l +VXR v x r +VX- v x +VX v x +Vyg v y g +vyr v y r +vy v y +VY v y +vz v z +V. Z. v z +WAAAAAGH w a a a a a g h +WAAAF w a a a f +WAAC w a a c +WAAF w a a f +WAAI w a a i +WAAV w a a v +Waay w a a y +WABB w a b b +WABC- w a b c +WABC w a b c +WABD w a b d +WABG's w a b g's +WABG w a b g +W. A. C. w a c +W.A.C. w a c +Wa'il w a i l +WAIO w a i o +W.A.J. w a j +Wakf w a k f +W.A.K.O. w a k o +WAKR- w a k r +wa'l w a l +WANFL w a n f l +WAOK w a o k +waqf w a q f +waqt w a q t +Waqt w a q t +WAQZ w a q z +WARH w a r h +W.A.R. w a r +WASC w a s c +Wasl w a s l +WASL w a s l +W. A. S. w a s +W.A.S. w a s +Watfa w a t f a +WATL w a t l +WATW w a t w +wau w a u +Wau w a u +WAU w a u +WAVF w a v f +WAVM w a v m +W. A. w a +W.A. w a +WA w a +WAWLI w a w l i +waw w a w +WAWZ w a w z +WAYH w a y h +waza w a z a +Waza w a z a +WBAK- w b a k +WBAL's w b a l's +WBAL- w b a l +WBAM w b a m +WBA w b a +WBBL w b b l +WBBM- w b b m +WBBM w b b m +WBBP w b b p +WBCA w b c a +WBCN- w b c n +WBCN w b c n +W.B.C. w b c +WBC w b c +WBCY w b c y +WBEV w b e v +WBEZ w b e z +WBFF- w b f f +WBFJ w b f j +WBFO w b f o +WBF w b f +WBGU- w b g u +WBG w b g +WBHY w b h y +WBIC w b i c +WBIL w b i l +WBIR- w b i r +WBJEE w b j e e +WBKI- w b k i +WBKO- w b k o +WBKO w b k o +WBK w b k +WBLQ w b l q +WBME w b m e +WBNI w b n i +WBNX w b n x +WBON w b o n +WBO w b o +WBQC w b q c +WBRC w b r c +WBRE- w b r e +WBRE w b r e +WBRO w b r o +WBRZ w b r z +WBs w b's +WBS w b s +WBTI w b t i +WBT's w b t's +WBT w b t +WBTW w b t w +WBTZ w b t z +WBUR- w b u r +WBUR w b u r +WBVE w b v e +WBV w b v +W. B. w b +W.B. w b +WB- w b +WB w b +WBWT- w b w t +WBXH w b x h +WBZB w b z b +WBZT- w b z t +WBZ- w b z +WBZ w b z +WCAA w c a a +WCAC- w c a c +WCAG w c a g +WCAU- w c a u +WCAU w c a u +WCA w c a +WCBA w c b a +WCBI- w c b i +WCBN w c b n +W&CBR w and c b r +WCBR w c b r +WCBS- w c b s +WCBS w c b s +WCB w c b +WCCES w c c e s +WCCG w c c g +WCCO- w c c o +WCCO w c c o +WCCP- w c c p +WCC's w c c's +WCC w c c +WCDA w c d a +WCEO w c e o +WCFL w c f l +WCF w c f +WCG w c g +WCHA w c h a +WCHL's w c h l's +WCHL w c h l +WCHS w c h s +WCHV w c h v +WCIU w c i u +WCKD w c k d +WCKY w c k y +WCLN w c l n +WCLU w c l u +WCLV w c l v +WCL w c l +WCMA w c m a +WCMC w c m c +WCMFA w c m f a +WCMH w c m h +WCMR w c m r +WCM w c m +WCNB w c n b +WCNG- w c n g +WCOZ w c o z +WCPO w c p o +WCPW w c p w +WCRC w c r c +WCRE w c r e +WCRL w c r l +WCSA w c s a +WCSM- w c s m +WCSM w c s m +WCS w c s +WCSY- w c s y +WCTD w c t d +WCTG w c t g +WCTI- w c t i +W.C.T.U. w c t u +WCTU w c t u +WCTV w c t v +WCT w c t +WCTX w c t x +WCVB's w c v b's +WCVB w c v b +WCVE w c v e +WCVP- w c v p +WCWA w c w a +wc w c +W. C. w c +W.C. w c +WC- w c +WC w c +WCWC w c w c +WCW w c w +WCWW- w c w w +WCXR w c x r +WCYB w c y b +WCZE w c z e +WCZU- w c z u +WDAF- w d a f +WDAF w d a f +WDA w d a +WDCN- w d c n +WDC w d c +WDDM w d d m +WDFN w d f n +WDF w d f +WDGY w d g y +WDH w d h +WDIA w d i a +WDIF w d i f +WDIG w d i g +WDIS w d i s +WDIV w d i v +WDJM- w d j m +WDJT w d j t +WDLA w d l a +WDL w d l +WDM- w d m +WDM w d m +WDNN w d n n +WDNS w d n s +WDNT w d n t +WDOD- w d o d +WDOK- w d o k +WDOK w d o k +WDO w d o +WDRE w d r e +WDR w d r +WDSE w d s e +WDSN w d s n +WDSO w d s o +WDSU w d s u +WDS- w d s +WDS w d s +WDTM w d t m +WDUQ w d u q +WDVE w d v e +WDVR w d v r +W. D. w d +W.D. w d +WD- w d +WDWS w d w s +WDW w d w +WDXZ w d x z +WDYZ w d y z +W. E. B. w e b +W.E.B. w e b +W. E. H. w e h +W.E.H. w e h +WEVD- w e v d +WEVD w e v d +WEVV w e v v +W. E. w e +W.E. w e +WEZQ w e z q +WFAA- w f a a +WFAA w f a a +WFAL w f a l +WFAN w f a n +WFBC- w f b c +WFB w f b +WFCA w f c a +WFCB- w f c b +WFC w f c +WFDC w f d c +WFDD w f d d +WFDF w f d f +WFDU w f d u +WFD w f d +WFFF- w f f f +WFGF w f g f +WFHE w f h e +W.F.H. w f h +WFH w f h +WFIKKN w f i k k n +WFIL w f i l +WFIRST w f i r s t +WFLA- w f l a +WFLN- w f l n +WFL's w f l's +WFL w f l +WFLX w f l x +WFLZ- w f l z +WFMT w f m t +WFMU w f m u +W. F. M. w f m +WFM w f m +WFMZ- w f m z +WF&NW w f and n w +WFNZ w f n z +WFOY w f o y +WFPC w f p c +WFPF w f p f +WFPMA w f p m a +W.F.P. w f p +WFP w f p +WFRX- w f r x +WFSBP w f s b p +WFS w f s +WFTDA w f t d a +WFTM w f t m +WFTV w f t v +WFT w f t +WFVA w f v a +W&F w and f +WFWA w f w a +W. F. w f +W.F. w f +WF w f +WFWM w f w m +WFXR w f x r +WFXW- w f x w +wga w g a +WGA w g a +WGBH- w g b h +WGBH w g b h +WGBI w g b i +W. G. B. w g b +WGB w g b +WGCB w g c b +W.G.C. w g c +WGC- w g c +WGC w g c +WGEM- w g e m +WGEM w g e m +WGFT w g f t +WGGH w g g h +WGHP w g h p +WGH w g h +WGHW w g h w +WGI w g i +WGN's w g n's +WGNT w g n t +wgn w g n +WGN- w g n +WGN w g n +WGOK w g o k +WGO w g o +WGP w g p +WGPX w g p x +WGRM- w g r m +WGRP w g r p +WGRT w g r t +WGRV w g r v +WGRZ- w g r z +WGSA w g s a +WGSS w g s s +WGS- w g s +WGS w g s +WGTN- w g t n +WGUN w g u n +WGVU- w g v u +wg w g +W. G. w g +W.G. w g +WG w g +WGWG- w g w g +WGWW w g w w +WGZR w g z r +W.H.B. w h b +WHCA w h c a +WHCB w h c b +WHCC w h c c +W. H. C. w h c +W.H.C. w h c +WHC w h c +WHDG w h d g +WHDH- w h d h +WHDI w h d i +W.H.D. w h d +WHEC- w h e c +WHEC w h e c +WHFA w h f a +WHFM w h f m +WHFS- w h f s +WHFS w h f s +WHHS w h h s +W.H.H. w h h +WHJA w h j a +W.H.J. w h j +WHJY w h j y +WHKT w h k t +WHL's w h l's +WHL w h l +WHNE- w h n e +WHNO w h n o +WHP- w h p +WHP w h p +WHRC w h r c +WHSG- w h s g +WHSN w h s n +WHSS w h s s +W.H.S. w h s +WHS w h s +W. H. T. w h t +WHTZ w h t z +W&H w and h +W. H. w h +W.H. w h +WH w h +Whyld w h y l d +WHYN- w h y n +WHYP- w h y p +WHYT w h y t +W.I.B.C. w i b c +Wica w i c a +WICA- w i c a +wici w i c i +WIC's w i c's +WICU w i c u +Wif w i f +WIF w i f +WIHB- w i h b +WIJD w i j d +Wijk w i j k +WIKQ- w i k q +WIP w i p +W.I.S.E. w i s e +W.I.T.C.H. w i t c h +Wiwa w i w a +W. I. w i +WIXY's w i x y's +WJAC- w j a c +WJAL w j a l +WJAR w j a r +wjaz w j a z +WJBB- w j b b +WJBC w j b c +WJBF w j b f +WJBK- w j b k +WJBK w j b k +WJBR w j b r +WJCI w j c i +WJC w j c +WJEC w j e c +WJEF w j e f +WJER w j e r +WJET w j e t +W. J. J. O. w j j o +W.J.J.O. w j j o +WJJZ w j j z +WJKA's w j k a's +WJKA w j k a +WJKS w j k s +WJLA- w j l a +WJLA w j l a +WJMK w j m k +WJMN- w j m n +WJM w j m +WJOI w j o i +WJRD w j r d +WJR w j r +WJSC w j s c +W&J's w and j's +WJTD w j t d +WJUX w j u x +W&J w and j +W. J. w j +W.J. w j +WJ w j +WJXX w j x x +WJYL w j y l +WJYM w j y m +WJYS- w j y s +WJZA w j z a +WJZJ w j z j +WJZ w j z +WKAG w k a g +WKAQ- w k a q +WKAQ w k a q +WKAR- w k a r +WKAR w k a r +WKA w k a +WKBD w k b d +WKBF's w k b f's +WKB w k b +WKBW w k b w +W. K. C. w k c +WKDM w k d m +WKDN w k d n +WKDZ w k d z +WKFS w k f s +WKHG- w k h g +W.K.H. w k h +WKH w k h +WKIK w k i k +WKIP w k i p +WKKB w k k b +WKKD w k k d +WKKZ w k k z +WKLH w k l h +WKLK- w k l k +WKNA- w k n a +WKNDS w k n d s +WKNR- w k n r +WKN w k n +WKQI w k q i +WKQX- w k q x +WKRC- w k r c +WKRK- w k r k +WKRR w k r r +WKSE w k s e +WKSM w k s m +WKSU w k s u +WKTU's w k t u's +WKTU w k t u +W.K.V. w k v +W. K. w k +W.K. w k +WKXJ w k x j +WKX w k x +WKYS's w k y s's +WKYT w k y t +WKY w k y +W.L.A.K. w l a k +WLB w l b +WLCH w l c h +WLDM- w l d m +W&LE w and l e +Wley w l e y +WLFM- w l f m +WLGH w l g h +WLH w l h +WLHZ- w l h z +WLIE- w l i e +WLII w l i i +WLIR w l i r +WLIX- w l i x +WLJY w l j y +WLKT w l k t +WLKY w l k y +WLLM w l l m +WLNE- w l n e +WLNP w l n p +WLNQ w l n q +WLNS w l n s +WLPA w l p a +WLPR w l p r +WLPWR w l p w r +wlrs w l r s +WLSG w l s g +WLS- w l s +WLS w l s +WLTM w l t m +WLTQ- w l t q +WLUK's w l u k's +WLUK w l u k +WLUP- w l u p +WLWC w l w c +W. L. w l +W.L. w l +WL- w l +WL w l +WLW- w l w +WLW w l w +WLYH's w l y h's +WLYN w l y n +WLY w l y +WLZK w l z k +WMAC w m a c +WMAG w m a g +WMAN- w m a n +WMAP w m a p +WMAQ- w m a q +WMAQ w m a q +WMAR- w m a r +WMATA's w m a t a's +WMATA w m a t a +W.M.A. w m a +WMA w m a +WMAZ w m a z +WMBD- w m b d +WMBG w m b g +WMCAT w m c a t +WMCA w m c a +WMCT- w m c t +WMDB w m d b +WMDI w m d i +WMD w m d +WMEC- w m e c +WMEE w m e e +WMEN w m e n +WMET w m e t +WME w m e +WMEX w m e x +WMFC w m f c +WMFM w m f m +WMFP w m f p +W.M.F. w m f +WMF w m f +WMGF w m g f +WMG w m g +WMGY w m g y +WMHG w m h g +WMHT w m h t +WMIX w m i x +WMJR w m j r +WMJZ w m j z +WMKG- w m k g +WMKQ w m k q +WMKW w m k w +wml w m l +WMLW w m l w +WMLY w m l y +WMLZ- w m l z +WMMS w m m s +W. M. M. w m m +WMNF w m n f +WMNI- w m n i +WMN w m n +WMOB w m o b +WMO w m o +WMRA w m r a +WMRO w m r o +WMSG w m s g +W.M.S. w m s +WMS w m s +WMTL- w m t l +WMTS w m t s +WMT w m t +WMTW w m t w +WMUB w m u b +WMU w m u +WMVM w m v m +WMVP- w m v p +WMV w m v +W&M w and m +W. M. w m +W.M. w m +WM- w m +WM w m +WMWM w m w m +WMXB w m x b +WMX w m x +WMYE w m y e +WMYL w m y l +WNAK w n a k +W.N.B.A. w n b a +WNBA w n b a +WNBC w n b c +WNBL w n b l +WNBR w n b r +WNB w n b +WNCN w n c n +WNCT- w n c t +WNC w n c +WNCX w n c x +WND w n d +WNED w n e d +WNEG- w n e g +WNEL w n e l +WNEM w n e m +WNEP w n e p +WNET w n e t +WNEW's w n e w's +WNEW w n e w +WNFM w n f m +WNGL w n g l +WNHC- w n h c +WNIC- w n i c +WNIT w n i t +WNJN w n j n +WNJO w n j o +WNJY w n j y +WNK w n k +WNLO w n l o +WNLU w n l u +WNL w n l +WNMX w n m x +WNNS- w n n s +WNO w n o +WNPT w n p t +WNRN w n r n +WNR's w n r's +WNRV w n r v +WNSH w n s h +WNSR w n s r +WNTM w n t m +Wnt- w n t +WNT w n t +WNUA w n u a +WNUF w n u f +WNUSP w n u s p +WNWBL w n w b l +W. N. w n +W.N. w n +WN w n +WNWO w n w o +WNWS w n w s +WNW w n w +WNYA w n y a +WNYC's w n y c's +WNYC w n y c +WNYM w n y m +WNYO- w n y o +WNY w n y +WNYW- w n y w +WNYZ- w n y z +WNYZ w n y z +wnzaa w n z a a +WOAA w o a a +WOAK w o a k +WOA w o a +WOBC- w o b c +WOC w o c +W&OD w and o d +W.O.G. w o g +WOGX's w o g x's +WOJG w o j g +WOKZ- w o k z +WOLH w o l h +WOOM w o o m +WOPC w o p c +Worh w o r h +WOSM w o s m +WOTM- w o t m +W. O. w o +W.O. w o +WOWOW w o w o w +WOXY- w o x y +WOXY w o x y +WOZN w o z n +WPAC w p a c +WPA- w p a +WPA w p a +WPBC w p b c +WPBF- w p b f +WPBSA w p b s a +WPBS w p b s +WPBT w p b t +WPCH- w p c h +WPCQ- w p c q +W. P. C. w p c +WPC w p c +WPDH w p d h +WPD w p d +WPFF w p f f +WPFL w p f l +WPF w p f +WPGC w p g c +WPGG- w p g g +WPHG- w p h g +WPHL w p h l +WPH w p h +WPIAL w p i a l +WPI w p i +WPIX- w p i x +WPIX w p i x +WPLA w p l a +WPLB- w p l b +WPLG w p l g +WPLS w p l s +WPMW w p m w +WPO w p o +WPPT w p p t +WPP- w p p +WPP w p p +WPRI w p r i +WPRO w p r o +WPRP w p r p +WPSC w p s c +WPSL w p s l +WPS w p s +WPTB w p t b +WPTD's w p t d's +wpt w p t +WPT w p t +WPTY w p t y +W. P. U. w p u +wp w p +W. P. w p +W.P. w p +WP- w p +WP w p +WPXN w p x n +WPYR w p y r +WQDE w q d e +WQDR- w q d r +WQHT w q h t +WQHU- w q h u +WQIV w q i v +WQLR w q l r +WQLT- w q l t +WQLZ w q l z +WQMS w q m s +WQOK w q o k +WQSJ- w q s j +WQSO w q s o +WQSX w q s x +WQVC- w q v c +WQ w q +WQXI w q x i +WQXR w q x r +W. R. A. S. w r a s +WRAT w r a t +WRBU w r b u +WRCB w r c b +WRCN w r c n +WRC- w r c +WRC w r c +WRDL w r d l +WRDW- w r d w +Wrec w r e c +WRFD w r f d +WRFK w r f k +WRFM w r f m +WRFX w r f x +WRGO w r g o +WRGP w r g p +WRGT- w r g t +W. R. G. w r g +WRHA w r h a +WRHC w r h c +WRHI w r h i +W.R.I. w r i +WRJK- w r j k +WRJN- w r j n +WRKA w r k a +WRKO w r k o +WRKT w r k t +W.R.K. w r k +WRKY w r k y +WRNS w r n s +WRN w r n +WRNY w r n y +WRNZ w r n z +WROJ w r o j +WROK w r o k +WROO w r o o +WRPT w r p t +WRQQ w r q q +WRRK- w r r k +WRSW- w r s w +WRTA w r t a +WRTC w r t c +WRUV w r u v +WRU w r u +WRVL w r v l +WRVS w r v s +WRWB w r w b +W. R. w r +W.R. w r +WR- w r +WR w r +W.R.W. w r w +WRXT w r x t +WSAR w s a r +WSAV w s a v +W.S.A. w s a +WSA w s a +WSAZ w s a z +WSBG w s b g +WSBM w s b m +WSBT w s b t +WSB- w s b +WSB w s b +WSCA's w s c a's +WSCR w s c r +W. S. C. w s c +WSC w s c +WSDD w s d d +WSDG w s d g +WSDOT w s d o t +WSDP w s d p +WSDR w s d r +WSEE- w s e e +WSEG w s e g +WSEK w s e k +WSER- w s e r +WSE w s e +WSFA w s f a +WSFL w s f l +WSFM w s f m +WSFS w s f s +WSFX w s f x +WSGE w s g e +WSGW w s g w +WSHL w s h l +WSHU w s h u +WSH w s h +WSIPC w s i p c +WSIS w s i s +WSJM w s j m +WSJO w s j o +WSJ w s j +WSKG- w s k g +WSK- w s k +WSLU w s l u +WSL w s l +WSMG- w s m g +WSMK w s m k +W.S.M. w s m +WSM- w s m +WSM w s m +WSNJ w s n j +WSNS w s n s +WSNT w s n t +WSN w s n +WSNX- w s n x +WSOF w s o f +WSOPE w s o p e +WSOP's w s o p's +WSOP, w s o p +WSOP w s o p +WSORR w s o r r +WSO w s o +WSOY w s o y +WSPA- w s p a +WSPA w s p a +WSPC w s p c +WSPD w s p d +WSPN w s p n +WSPS w s p s +WSPU w s p u +WSP w s p +WSRA w s r a +WSRE's w s r e's +WSRF w s r f +WSSA w s s a +WSSCC w s s c c +WSSSA- w s s s a +WSSSA w s s s a +WSS w s s +WSTE w s t e +WSTR- w s t r +WSTR w s t r +WSUA w s u a +WSUD w s u d +WSUE w s u e +WSUS w s u s +WSU w s u +WSVF- w s v f +WSV w s v +WSVZ w s v z +WSWI w s w i +ws w s +W. S. w s +W.S. w s +WS- w s +WS w s +WSW w s w +WSYN w s y n +WSYX w s y x +WSYY- w s y y +WSZE- w s z e +WTAA w t a a +WTAB- w t a b +WTAE- w t a e +WTAE w t a e +WTAF w t a f +WTAG w t a g +WTAM's w t a m's +W. T. A. w t a +WTA w t a +WTAW w t a w +WTCC w t c c +W. T. C. w t c +W.T.C. w t c +WTC w t c +WTDY w t d y +WTEL w t e l +WTEV's w t e v's +WTEV w t e v +WTF w t f +WTGB- w t g b +WTGB w t g b +WTHR's w t h r's +WTHS w t h s +WTIU w t i u +WTI w t i +WTIX w t i x +WTKG w t k g +WTKO w t k o +WTKR w t k r +WTKS- w t k s +WTKS w t k s +WTLS w t l s +WTMA w t m a +WTNH w t n h +WTNO- w t n o +WTN w t n +WTOK- w t o k +WTOP- w t o p +WTOP w t o p +WTO's w t o's +WTOS w t o s +WTO w t o +WTPA w t p a +WTPL- w t p l +WTPM- w t p m +WTPQ w t p q +WTRF- w t r f +WTR w t r +WTSP- w t s p +WTSQ- w t s q +WTs w t's +WTS w t s +WTTF w t t f +WTTM w t t m +WTTs w t t's +WTT w t t +WTTW w t t w +WTTZ w t t z +WTVA- w t v a +WTVA w t v a +WTVC w t v c +WTVD w t v d +WTVF- w t v f +WTVI w t v i +WTVM w t v m +WTVU w t v u +WTVW w t v w +WTVX w t v x +WTWF w t w f +W. T. w t +W.T. w t +WT- w t +WT w t +WTX w t x +WUAB w u a b +WUAP w u a p +WUAs w u a's +WUC w u c +Wudl w u d l +Wuhr w u h r +Wuhu w u h u +WUKB w u k b +WUPG w u p g +WUPN w u p n +WUPZ w u p z +Wu's w u's +WUSW w u s w +WUTK w u t k +WUVC w u v c +wuv w u v +W. U. w u +Wuz w u z +WVAN- w v a n +WVa w v a +WVBX w v b x +WVCA w v c a +WVFJ- w v f j +WVIR- w v i r +WVIT w v i t +WVIZ- w v i z +WVJC w v j c +WVJP w v j p +WVLZ w v l z +WVNI w v n i +WVNS- w v n s +WVNY w v n y +W. V. O. w v o +WVO w v o +WVPA w v p a +WVPO w v p o +WVPX- w v p x +WVSSAC w v s s a c +WVST- w v s t +WVSU w v s u +WVTF- w v t f +WVTU's w v t u's +WVTU w v t u +WVUE w v u e +WVU w v u +WVUW w v u w +W. V. w v +W.V. w v +WV- w v +WV w v +W- w +WWAA w w a a +WWAC- w w a c +WWAMI w w a m i +WWASP w w a s p +WWA w w a +WWAX w w a x +WWBZ w w b z +WWCD w w c d +WWC w w c +WWDB's w w d b's +WWDC- w w d c +WWDC w w d c +WWDR w w d r +WWD w w d +WWEN- w w e n +WWE's w w e's +Wwe w w e +WWE w w e +WWF's w w f's +WWF- w w f +WWF w w f +WWGC w w g c +WWHT w w h t +WWHW w w h w +WWIL- w w i l +WWIS w w i s +WWJP w w j p +WWJ- w w j +WWKB w w k b +WWKL w w k l +WWML w w m l +WWMM w w m m +WWMX w w m x +WWMY w w m y +WWN w w n +WWNW w w n w +WWNY w w n y +WWOX w w o x +WWPG w w p g +WWP w w p +WWPW w w p w +WWRFC w w r f c +WWRV w w r v +WWSS w w s s +WWS w w s +WWTI's w w t i's +WWTI w w t i +WWTN w w t n +WWU w w u +WWVA w w v a +WWVR w w v r +WWV w w v +WWWF w w w f +WWWQ's w w w q's +W. W. w w +W.W. w w +WW- w w +WW w w +WWW w w w +WWWW w w w w +WWYD w w y d +WXB w x b +WXCN w x c n +WXCY- w x c y +WXDU w x d u +WXEL- w x e l +WXEL w x e l +WXII- w x i i +WXIL w x i l +WXIN's w x i n's +WXIN w x i n +WXJM's w x j m's +WXJ w x j +WXL- w x l +WXN w x n +WXQW w x q w +WXRK- w x r k +WXSR- w x s r +WXTG w x t g +WXTH- w x t h +WXW w x w +wx w x +WX w x +W.Y. w y +WY w y +wyzc w y z c +WZBC- w z b c +WZBC w z b c +WZFG w z f g +WZFT w z f t +WZGC w z g c +WZJQ w z j q +WZKC w z k c +WZLR w z l r +WZLX- w z l x +WZLX w z l x +WZRB w z r b +WZRD w z r d +WZST w z s t +WZT- w z t +W. Z. w z +WZ w z +WZX w z x +WZZM- w z z m +WZZM w z z m +WZZX w z z x +Xav x a v +XAV x a v +xa x a +Xa x a +XA- x a +XA x a +xbg x b g +XBMC x b m c +XBM x b m +Xbra x b r a +XBRL x b r l +XBV x b v +XB- x b +XB x b +XCB x c b +XCE x c e +XCMS x c m s +XCOM x c o m +XCOR x c o r +xCo x c o +XCR x c r +XCT x c t +XCVI x c v i +XCV x c v +X. C. x c +XC- x c +XC x c +XCX x c x +xda x d a +XDDM x d d m +XDE- x d e +XDH x d h +XDK x d k +XDRs x d r's +XDR- x d r +XDR x d r +XD's x d's +XDS x d s +X. D. x d +XD x d +Xfce x f c e +XFC x f c +XFDL x f d l +XFD x f d +XFL x f l +XFM x f m +XFR- x f r +XFS x f s +XFU x f u +xf x f +X.F. x f +XF- x f +XF x f +XHVFC x h v f c +xh x h +X. H. x h +XH x h +xix x i x +XIX- x i x +XJD x j d +XJ- x j +XJ x j +XKE x k e +XKL x k l +xk x k +XK x k +XL x l +XMB x m b +xmc x m c +xml x m l +XML- x m l +XML x m l +XMMS x m m s +XMM- x m m +XMPP x m p p +XMP x m p +XMRV x m r v +XMSN x m s n +XM's x m's +XMU x m u +X.M. x m +XM x m +XN x n +xO x o +X.O. x o +XO- x o +XO x o +XPA x p a +XPD x p d +XPe x p e +XPL x p l +XPN x p n +XP's x p's +XPS x p s +XPT x p t +XPW x p w +XP- x p +XP x p +X.Q. x q +XQ x q +XRCC x r c c +XRCO x r c o +XRDS x r d s +XRD x r d +XRE x r e +XR x r +XSD x s d +xSi- x s i +XSi x s i +XSI x s i +XSLT x s l t +XSL- x s l +xsr x s r +XS x s +XTB x t b +XTC x t c +XTG x t g +XTwas x t w a's +xt x t +XT x t +XVAS x v a +XV- x v +XV x v +XWA x w a +XWB x w b +XWF x w f +XWIS x w i s +X. W. x w +XYNU x y n u +XYP x y p +xy x y +XY x y +xyz x y z +xyZ x y z +XYZ x y z +XZ x z +Y. A. y a +Y.A. y a +YA- y a +YBA y b a +YBL y b l +YBNL y b n l +YBN y b n +YBR y b r +YBT- y b t +YBX y b x +Y. B. y b +Y.B. y b +YB- y b +YB y b +YBYS y b y s +YCC y c c +YCD y c d +ycia y c i a +ycie y c i e +YCL y c l +YCO y c o +YCU y c u +YCV y c v +yc y c +Y. C. y c +Y.C. y c +YC- y c +YC y c +YCYW y c y w +YDA y d a +Ydby y d b y +Yde y d e +YDG y d g +YdiB y d i b +YDK y d k +YDM- y d m +YDNC y d n c +yd y d +Y. D. y d +Y.D. y d +YD y d +Y.E. y e +YFCMD y f c m d +YFCs y f c's +YFCU y f c u +YFC- y f c +YFC y f c +YFD- y f d +yfle y f l e +YF&R y f and r +YF- y f +YF y f +YGB y g b +YGEX y g e x +YGLA y g l a +YGL y g l +YG's y g's +yg y g +Y. G. y g +Y.G. y g +YG- y g +YG y g +YHA y h a +YHWH y h w h +Y. H. y h +YH- y h +YH y h +YIF y i f +YIG y i g +YITP y i t p +Y. I. y i +Y.I. y i +YJO y j o +yj y j +Y. J. y j +YJ- y j +YJ y j +Ykt y k t +Y. K. y k +Y.K. y k +YK y k +YKY y k y +Yle y l e +YLE y l e +YLF y l f +yl- y l +yl y l +Y. L. y l +YL- y l +YL y l +YMCA's y m c a's +Y.M.C.A. y m c a +YMCA. y m c a +YMCA y m c a +YMF y m f +Ymke y m k e +Y.M.M. y m m +YMM y m m +YMO y m o +YMS- y m s +ym y m +Y. M. y m +YM y m +yndi y n d i +Yndi y n d i +yne y n e +Yngve y n g v e +Ynis y n i's +YNK y n k +YNU y n u +yn y n +Yn y n +Y. N. y n +Y.N. y n +YN- y n +YN y n +Y.O. y o +YO y o +ypa y p a +YPC y p c +YPF y p f +YPG y p g +YPM y p m +YPO y p o +Y.P.P. y p p +YPP y p p +Y.P.S. y p s +YPT- y p t +YPT y p t +YPU y p u +Y.P. y p +YP- y p +YP y p +Y. Q. y q +YQ y q +yra y r a +Yra y r a +YRC y r c +YRF y r f +YRK y r k +YRM y r m +YRT y r t +Y&R y and r +YSD- y s d +YSK'da y s k d a +YSK'ya y s k y a +YSK y s k +YSL y s l +YSRCP y s r c p +YSR y s r +YSU's y s u's +YSU y s u +Ys y's +Y.S. y s +YS- y s +YS y s +YSY y s y +yta y t a +YTB y t b +Ytre y t r e +YTS y t s +YTV's y t v's +YTV y t v +Y&T y and t +Y. T. y t +Y.T. y t +YT y t +Y. V. y v +YWCA- y w c a +YWCA. y w c a +YWCA y w c a +YWC y w c +Y. W. y w +Y.Y. y y +Z. A. B. z a b +Z. A. z a +Z.A. z a +ZBC z b c +ZBS z b s +ZB z b +ZCE z c e +ZCGWM z c g w m +ZCMI z c m i +ZCTA z c t a +ZCY z c y +ZC z c +Zdar z d a r +zda z d a +ZDF z d f +ZDR z d r +Z.E.R.O. z e r o +ZFC z f c +ZFK z f k +ZFM z f m +ZFP z f p +ZFS z f s +ZFX z f x +ZF z f +ZGE z g e +ZGI z g i +zg z g +Z.G. z g +ZG z g +zh z h +Z. H. z h +Z.H. z h +ZH z h +zijn z i j n +zij z i j +Zij z i j +Z. I. z i +Z. J. z j +ZKK z k k +ZKM z k m +Z.K. z k +Z.L. z l +Z.M. z m +ZNA z n a +ZNF z n f +ZNK z n k +ZnO z n o +ZNW z n w +ZPAP z p a p +Zpav z p a v +ZPAV z p a v +ZPA z p a +ZPG- z p g +ZPK z p k +ZPU- z p u +Z.P. z p +ZP- z p +ZP z p +ZQ z q +ZRC z r c +ZrI z r i +ZRK z r k +ZSC z s c +ZSE z s e +ZSF z s f +ZSG's z s g's +ZSIS z s i s +ZSL z s l +ZSO z s o +ZSSS z s s s +Zsuzsa z s u z s a +ZSU- z s u +ZTE z t e +ZWYCI z w y c i +Z.W. z w +Z.X.V. z x v +Z.X. z x +ZYF z y f +Z. Y. z y +Z.Y. z y +Z. Z. z z +Z.Z. z z diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/whitelist_alternatives.tsv b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/whitelist_alternatives.tsv new file mode 100644 index 0000000000..8cea774c58 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data/whitelist_alternatives.tsv @@ -0,0 +1,17 @@ +Hon. Honorable +Hon. Honourable +St. Street +St Street +St. Saint +St Saint +Dr. Drive +Dr. Doctor +Mr. Mister +Mrs. Misses +Ms. Miss +Mr Mister +Mrs Misses +Ms Miss +&Co. and Co. +§ section += equals \ No newline at end of file diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data_loader_utils.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data_loader_utils.py new file mode 100644 index 0000000000..1aaac7a762 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/data_loader_utils.py @@ -0,0 +1,269 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import csv +import json +import os +import re +from collections import defaultdict, namedtuple +from typing import Dict, List, Optional, Set, Tuple + +EOS_TYPE = "EOS" +PUNCT_TYPE = "PUNCT" +PLAIN_TYPE = "PLAIN" +Instance = namedtuple('Instance', 'token_type un_normalized normalized') +known_types = [ + "PLAIN", + "DATE", + "CARDINAL", + "LETTERS", + "VERBATIM", + "MEASURE", + "DECIMAL", + "ORDINAL", + "DIGIT", + "MONEY", + "TELEPHONE", + "ELECTRONIC", + "FRACTION", + "TIME", + "ADDRESS", +] + + +def load_kaggle_text_norm_file(file_path: str) -> List[Instance]: + """ + https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish + Loads text file in the Kaggle Google text normalization file format: \t\t<`self` if trivial class or normalized text> + E.g. + PLAIN Brillantaisia + PLAIN is + PLAIN a + PLAIN genus + PLAIN of + PLAIN plant + PLAIN in + PLAIN family + PLAIN Acanthaceae + PUNCT . sil + + + Args: + file_path: file path to text file + + Returns: flat list of instances + """ + res = [] + with open(file_path, 'r') as fp: + for line in fp: + parts = line.strip().split("\t") + if parts[0] == "": + res.append(Instance(token_type=EOS_TYPE, un_normalized="", normalized="")) + else: + l_type, l_token, l_normalized = parts + l_token = l_token.lower() + l_normalized = l_normalized.lower() + + if l_type == PLAIN_TYPE: + res.append(Instance(token_type=l_type, un_normalized=l_token, normalized=l_token)) + elif l_type != PUNCT_TYPE: + res.append(Instance(token_type=l_type, un_normalized=l_token, normalized=l_normalized)) + return res + + +def load_files(file_paths: List[str], load_func=load_kaggle_text_norm_file) -> List[Instance]: + """ + Load given list of text files using the `load_func` function. + + Args: + file_paths: list of file paths + load_func: loading function + + Returns: flat list of instances + """ + res = [] + for file_path in file_paths: + res.extend(load_func(file_path=file_path)) + return res + + +def clean_generic(text: str) -> str: + """ + Cleans text without affecting semiotic classes. + + Args: + text: string + + Returns: cleaned string + """ + text = text.strip() + text = text.lower() + return text + + +def evaluate(preds: List[str], labels: List[str], input: Optional[List[str]] = None, verbose: bool = True) -> float: + """ + Evaluates accuracy given predictions and labels. + + Args: + preds: predictions + labels: labels + input: optional, only needed for verbosity + verbose: if true prints [input], golden labels and predictions + + Returns accuracy + """ + acc = 0 + nums = len(preds) + for i in range(nums): + pred_norm = clean_generic(preds[i]) + label_norm = clean_generic(labels[i]) + if pred_norm == label_norm: + acc = acc + 1 + else: + if input: + print(f"inpu: {json.dumps(input[i])}") + print(f"gold: {json.dumps(label_norm)}") + print(f"pred: {json.dumps(pred_norm)}") + return acc / nums + + +def training_data_to_tokens( + data: List[Instance], category: Optional[str] = None +) -> Dict[str, Tuple[List[str], List[str]]]: + """ + Filters the instance list by category if provided and converts it into a map from token type to list of un_normalized and normalized strings + + Args: + data: list of instances + category: optional semiotic class category name + + Returns Dict: token type -> (list of un_normalized strings, list of normalized strings) + """ + result = defaultdict(lambda: ([], [])) + for instance in data: + if instance.token_type != EOS_TYPE: + if category is None or instance.token_type == category: + result[instance.token_type][0].append(instance.un_normalized) + result[instance.token_type][1].append(instance.normalized) + return result + + +def training_data_to_sentences(data: List[Instance]) -> Tuple[List[str], List[str], List[Set[str]]]: + """ + Takes instance list, creates list of sentences split by EOS_Token + Args: + data: list of instances + Returns (list of unnormalized sentences, list of normalized sentences, list of sets of categories in a sentence) + """ + # split data at EOS boundaries + sentences = [] + sentence = [] + categories = [] + sentence_categories = set() + + for instance in data: + if instance.token_type == EOS_TYPE: + sentences.append(sentence) + sentence = [] + categories.append(sentence_categories) + sentence_categories = set() + else: + sentence.append(instance) + sentence_categories.update([instance.token_type]) + un_normalized = [" ".join([instance.un_normalized for instance in sentence]) for sentence in sentences] + normalized = [" ".join([instance.normalized for instance in sentence]) for sentence in sentences] + return un_normalized, normalized, categories + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + label_tsv = open(abs_path) + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def post_process_punctuation(text: str) -> str: + """ + Normalized quotes and spaces + + Args: + text: text + + Returns: text with normalized spaces and quotes + """ + text = ( + text.replace('( ', '(') + .replace(' )', ')') + .replace('{ ', '{') + .replace(' }', '}') + .replace('[ ', '[') + .replace(' ]', ']') + .replace(' ', ' ') + .replace('”', '"') + .replace("’", "'") + .replace("»", '"') + .replace("«", '"') + .replace("\\", "") + .replace("„", '"') + .replace("´", "'") + .replace("’", "'") + .replace('“', '"') + .replace("‘", "'") + .replace('`', "'") + .replace('- -', "--") + ) + + for punct in "!,.:;?": + text = text.replace(f' {punct}', punct) + return text.strip() + + +def pre_process(text: str) -> str: + """ + Adds space around punctuation marks + + Args: + text: string that may include semiotic classes + + Returns: text with spaces around punctuation marks + """ + space_both = '*<=>^[]{}' + for punct in space_both: + text = text.replace(punct, ' ' + punct + ' ') + + text = text.replace('--', ' ' + '--' + ' ') + # remove extra space + text = re.sub(r' +', ' ', text) + return text diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/graph_utils.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/graph_utils.py new file mode 100644 index 0000000000..71cdc62917 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/graph_utils.py @@ -0,0 +1,216 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path + +from nemo_text_processing.text_normalization.data_loader_utils import get_abs_path + +try: + import pynini + from pynini import Far + from pynini.examples import plurals + from pynini.lib import byte, pynutil, utf8 + + NEMO_CHAR = utf8.VALID_UTF8_CHAR + + NEMO_DIGIT = byte.DIGIT + NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() + NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() + NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() + NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() + NEMO_HEX = pynini.union(*string.hexdigits).optimize() + NEMO_NON_BREAKING_SPACE = u"\u00A0" + NEMO_SPACE = " " + NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00A0").optimize() + NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() + NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + + NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() + NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + + NEMO_SIGMA = pynini.closure(NEMO_CHAR) + + delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) + insert_space = pynutil.insert(" ") + delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") + + suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) + # _v = pynini.union("a", "e", "i", "o", "u") + _c = pynini.union( + "b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "n", "p", "q", "r", "s", "t", "v", "w", "x", "y", "z" + ) + _ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") + _es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") + _s = NEMO_SIGMA + pynutil.insert("s") + + graph_plural = plurals._priority_union( + suppletive, plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), NEMO_SIGMA + ).optimize() + + SINGULAR_TO_PLURAL = graph_plural + PLURAL_TO_SINGULAR = pynini.invert(graph_plural) + TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) + TO_UPPER = pynini.invert(TO_LOWER) + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + # Create placeholders + NEMO_CHAR = None + + NEMO_DIGIT = None + NEMO_LOWER = None + NEMO_UPPER = None + NEMO_ALPHA = None + NEMO_ALNUM = None + NEMO_HEX = None + NEMO_NON_BREAKING_SPACE = u"\u00A0" + NEMO_SPACE = " " + NEMO_WHITE_SPACE = None + NEMO_NOT_SPACE = None + NEMO_NOT_QUOTE = None + + NEMO_PUNCT = None + NEMO_GRAPH = None + + NEMO_SIGMA = None + + delete_space = None + insert_space = None + delete_extra_space = None + + suppletive = None + # _v = pynini.union("a", "e", "i", "o", "u") + _c = None + _ies = None + _es = None + _s = None + + graph_plural = None + + SINGULAR_TO_PLURAL = None + PLURAL_TO_SINGULAR = None + TO_LOWER = None + TO_UPPER = None + + PYNINI_AVAILABLE = False + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = str + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/normalize.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/normalize.py new file mode 100644 index 0000000000..353025a3a9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/normalize.py @@ -0,0 +1,246 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +from argparse import ArgumentParser +from collections import OrderedDict +from typing import List + +from nemo_text_processing.text_normalization.data_loader_utils import post_process_punctuation, pre_process +from nemo_text_processing.text_normalization.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.text_normalization.token_parser import PRESERVE_ORDER_KEY, TokenParser +from nemo_text_processing.text_normalization.verbalizers.verbalize_final import VerbalizeFinalFst +from tqdm import tqdm + +try: + import pynini + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class Normalizer: + """ + Normalizer class that converts text from written to spoken form. + Useful for TTS preprocessing. + + Args: + input_case: expected input capitalization + """ + + def __init__(self, input_case: str): + assert input_case in ["lower_cased", "cased"] + + self.tagger = ClassifyFst(input_case=input_case, deterministic=True) + self.verbalizer = VerbalizeFinalFst(deterministic=True) + self.parser = TokenParser() + + def normalize_list(self, texts: List[str], verbose=False) -> List[str]: + """ + NeMo text normalizer + + Args: + texts: list of input strings + verbose: whether to print intermediate meta information + + Returns converted list input strings + """ + res = [] + for input in tqdm(texts): + try: + text = self.normalize(input, verbose=verbose) + except: + print(input) + raise Exception + res.append(text) + return res + + def normalize( + self, text: str, verbose: bool, punct_pre_process: bool = False, punct_post_process: bool = False + ) -> str: + """ + Main function. Normalizes tokens from written to spoken form + e.g. 12 kg -> twelve kilograms + + Args: + text: string that may include semiotic classes + verbose: whether to print intermediate meta information + punct_pre_process: whether to perform punctuation pre-processing, for example, [25] -> [ 25 ] + punct_post_process: whether to normalize punctuation + + Returns: spoken form + """ + if punct_pre_process: + text = pre_process(text) + text = text.strip() + if not text: + if verbose: + print(text) + return text + text = pynini.escape(text) + tagged_lattice = self.find_tags(text) + tagged_text = self.select_tag(tagged_lattice) + if verbose: + print(tagged_text) + self.parser(tagged_text) + tokens = self.parser.parse() + tags_reordered = self.generate_permutations(tokens) + for tagged_text in tags_reordered: + tagged_text = pynini.escape(tagged_text) + verbalizer_lattice = self.find_verbalizer(tagged_text) + if verbalizer_lattice.num_states() == 0: + continue + output = self.select_verbalizer(verbalizer_lattice) + if punct_post_process: + output = post_process_punctuation(output) + return output + raise ValueError() + + def _permute(self, d: OrderedDict) -> List[str]: + """ + Creates reorderings of dictionary elements and serializes as strings + + Args: + d: (nested) dictionary of key value pairs + + Return permutations of different string serializations of key value pairs + """ + l = [] + if PRESERVE_ORDER_KEY in d.keys(): + d_permutations = [d.items()] + else: + d_permutations = itertools.permutations(d.items()) + for perm in d_permutations: + subl = [""] + for k, v in perm: + if isinstance(v, str): + subl = ["".join(x) for x in itertools.product(subl, [f"{k}: \"{v}\" "])] + elif isinstance(v, OrderedDict): + rec = self._permute(v) + subl = ["".join(x) for x in itertools.product(subl, [f" {k} {{ "], rec, [f" }} "])] + elif isinstance(v, bool): + subl = ["".join(x) for x in itertools.product(subl, [f"{k}: true "])] + else: + raise ValueError() + l.extend(subl) + return l + + def generate_permutations(self, tokens: List[dict]): + """ + Generates permutations of string serializations of list of dictionaries + + Args: + tokens: list of dictionaries + + Returns string serialization of list of dictionaries + """ + + def _helper(prefix: str, tokens: List[dict], idx: int): + """ + Generates permutations of string serializations of given dictionary + + Args: + tokens: list of dictionaries + prefix: prefix string + idx: index of next dictionary + + Returns string serialization of dictionary + """ + if idx == len(tokens): + yield prefix + return + token_options = self._permute(tokens[idx]) + for token_option in token_options: + yield from _helper(prefix + token_option, tokens, idx + 1) + + return _helper("", tokens, 0) + + def find_tags(self, text: str) -> 'pynini.FstLike': + """ + Given text use tagger Fst to tag text + + Args: + text: sentence + + Returns: tagged lattice + """ + lattice = text @ self.tagger.fst + return lattice + + def select_tag(self, lattice: 'pynini.FstLike') -> str: + """ + Given tagged lattice return shortest path + + Args: + tagged_text: tagged text + + Returns: shortest path + """ + tagged_text = pynini.shortestpath(lattice, nshortest=1, unique=True).string() + return tagged_text + + def find_verbalizer(self, tagged_text: str) -> 'pynini.FstLike': + """ + Given tagged text creates verbalization lattice + This is context-independent. + + Args: + tagged_text: input text + + Returns: verbalized lattice + """ + lattice = tagged_text @ self.verbalizer.fst + return lattice + + def select_verbalizer(self, lattice: 'pynini.FstLike') -> str: + """ + Given verbalized lattice return shortest path + + Args: + lattice: verbalization lattice + + Returns: shortest path + """ + output = pynini.shortestpath(lattice, nshortest=1, unique=True).string() + return output + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("input_string", help="input string", type=str) + parser.add_argument( + "--input_case", help="input capitalization", choices=["lower_cased", "cased"], default="cased", type=str + ) + parser.add_argument("--verbose", help="print info for debugging", action='store_true') + parser.add_argument( + "--punct_post_process", help="set to True to enable punctuation post processing", action="store_true" + ) + parser.add_argument( + "--punct_pre_process", help="set to True to enable punctuation pre processing", action="store_true" + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + normalizer = Normalizer(input_case=args.input_case) + print( + normalizer.normalize( + args.input_string, + verbose=args.verbose, + punct_pre_process=args.punct_pre_process, + punct_post_process=args.punct_post_process, + ) + ) diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/normalize_with_audio.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/normalize_with_audio.py new file mode 100644 index 0000000000..4df248f2d4 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/normalize_with_audio.py @@ -0,0 +1,327 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import time +from argparse import ArgumentParser +from typing import List, Tuple + +from joblib import Parallel, delayed +from nemo_text_processing.text_normalization.data_loader_utils import post_process_punctuation, pre_process +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.text_normalization.verbalizers.verbalize_final import VerbalizeFinalFst + +from nemo.collections.asr.metrics.wer import word_error_rate +from nemo.collections.asr.models import ASRModel + +try: + import pynini + from pynini.lib import rewrite + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + +""" +The script provides multiple normalization options and chooses the best one that minimizes CER of the ASR output +(most of the semiotic classes use deterministic=False flag). + +To run this script with a .json manifest file: + python normalize_with_audio.py \ + --audio_data PATH/TO/MANIFEST.JSON \ + --model QuartzNet15x5Base-En \ + --verbose + + The manifest file should contain the following fields: + "audio_filepath" - path to the audio file + "text" - raw text + "transcript" - ASR model prediction (optional) + + +To run with a single audio file, specify path to audio and text with: + python normalize_with_audio.py \ + --audio_data PATH/TO/AUDIO.WAV \ + --text raw text OR PATH/TO/.TXT/FILE + --model QuartzNet15x5Base-En \ + --verbose + +To see possible normalization options for a text input without an audio file (could be used for debugging), run: + python python normalize_with_audio.py --text "RAW TEXT" +""" + + +class NormalizerWithAudio(Normalizer): + """ + Normalizer class that converts text from written to spoken form. + Useful for TTS preprocessing. + + Args: + input_case: expected input capitalization + """ + + def __init__(self, input_case: str): + super().__init__(input_case) + + self.tagger = ClassifyFst(input_case=input_case, deterministic=False) + self.verbalizer = VerbalizeFinalFst(deterministic=False) + + def normalize( + self, + text: str, + n_tagged: int, + punct_pre_process: bool = True, + punct_post_process: bool = True, + verbose: bool = False, + ) -> str: + """ + Main function. Normalizes tokens from written to spoken form + e.g. 12 kg -> twelve kilograms + + Args: + text: string that may include semiotic classes + n_tagged: number of tagged options to consider, -1 - to get all possible tagged options + punct_pre_process: whether to perform punctuation pre-processing, for example, [25] -> [ 25 ] + punct_post_process: whether to normalize punctuation + verbose: whether to print intermediate meta information + + Returns: + normalized text options (usually there are multiple ways of normalizing a given semiotic class) + """ + if punct_pre_process: + text = pre_process(text) + text = text.strip() + if not text: + if verbose: + print(text) + return text + + text = pynini.escape(text) + if n_tagged == -1: + tagged_texts = rewrite.rewrites(text, self.tagger.fst) + else: + tagged_texts = rewrite.top_rewrites(text, self.tagger.fst, nshortest=n_tagged) + + normalized_texts = [] + for tagged_text in tagged_texts: + self._verbalize(tagged_text, normalized_texts) + if len(normalized_texts) == 0: + raise ValueError() + if punct_post_process: + normalized_texts = [post_process_punctuation(t) for t in normalized_texts] + normalized_texts = set(normalized_texts) + return normalized_texts + + def _verbalize(self, tagged_text: str, normalized_texts: List[str]): + """ + Verbalizes tagged text + + Args: + tagged_text: text with tags + normalized_texts: list of possible normalization options + """ + + def get_verbalized_text(tagged_text): + tagged_text = pynini.escape(tagged_text) + return rewrite.rewrites(tagged_text, self.verbalizer.fst) + + try: + normalized_texts.extend(get_verbalized_text(tagged_text)) + except pynini.lib.rewrite.Error: + self.parser(tagged_text) + tokens = self.parser.parse() + tags_reordered = self.generate_permutations(tokens) + for tagged_text_reordered in tags_reordered: + try: + normalized_texts.extend(get_verbalized_text(tagged_text_reordered)) + except pynini.lib.rewrite.Error: + continue + + def select_best_match( + self, normalized_texts: List[str], transcript: str, verbose: bool = False, remove_punct: bool = False + ): + """ + Selects the best normalization option based on the lowest CER + + Args: + normalized_texts: normalized text options + transcript: ASR model transcript of the audio file corresponding to the normalized text + verbose: whether to print intermediate meta information + remove_punct: whether to remove punctuation before calculating CER + + Returns: + normalized text with the lowest CER and CER value + """ + normalized_texts = calculate_cer(normalized_texts, transcript, remove_punct) + normalized_texts = sorted(normalized_texts, key=lambda x: x[1]) + normalized_text, cer = normalized_texts[0] + + if verbose: + print('-' * 30) + for option in normalized_texts: + print(option) + print('-' * 30) + return normalized_text, cer + + +def calculate_cer(normalized_texts: List[str], transcript: str, remove_punct=False) -> List[Tuple[str, float]]: + """ + Calculates character error rate (CER) + + Args: + normalized_texts: normalized text options + transcript: ASR model output + + Returns: normalized options with corresponding CER + """ + normalized_options = [] + for text in normalized_texts: + text_clean = text.replace('-', ' ').lower() + if remove_punct: + for punct in "!?:;,.-()*+-/<=>@^_": + text_clean = text_clean.replace(punct, "") + cer = round(word_error_rate([transcript], [text_clean], use_cer=True) * 100, 2) + normalized_options.append((text, cer)) + return normalized_options + + +def get_asr_model(asr_model: ASRModel): + """ + Returns ASR Model + + Args: + asr_model: NeMo ASR model + """ + if os.path.exists(args.model): + asr_model = ASRModel.restore_from(asr_model) + elif args.model in ASRModel.get_available_model_names(): + asr_model = ASRModel.from_pretrained(asr_model) + else: + raise ValueError( + f'Provide path to the pretrained checkpoint or choose from {ASRModel.get_available_model_names()}' + ) + return asr_model + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("--text", help="input string or path to a .txt file", default=None, type=str) + parser.add_argument( + "--input_case", help="input capitalization", choices=["lower_cased", "cased"], default="cased", type=str + ) + parser.add_argument("--audio_data", help="path to an audio file or .json manifest") + parser.add_argument( + '--model', type=str, default='QuartzNet15x5Base-En', help='Pre-trained model name or path to model checkpoint' + ) + parser.add_argument( + "--n_tagged", + type=int, + default=1000, + help="number of tagged options to consider, -1 - return all possible tagged options", + ) + parser.add_argument("--verbose", help="print info for debugging", action="store_true") + parser.add_argument("--remove_punct", help="remove punctuation before calculating cer", action="store_true") + parser.add_argument( + "--no_punct_pre_process", help="set to True to disable punctuation pre processing", action="store_true" + ) + parser.add_argument( + "--no_punct_post_process", help="set to True to disable punctuation post processing", action="store_true" + ) + return parser.parse_args() + + +def _normalize_line(normalizer: NormalizerWithAudio, line: str, asr_model: ASRModel = None): + line = json.loads(line) + audio = line['audio_filepath'] + if 'transcript' in line: + transcript = line['transcript'] + else: + transcript = asr_model.transcribe([audio])[0] + + normalized_texts = normalizer.normalize( + text=line['text'], + verbose=args.verbose, + n_tagged=args.n_tagged, + punct_pre_process=not args.no_punct_pre_process, + punct_post_process=not args.no_punct_post_process, + ) + normalized_text, cer = normalizer.select_best_match(normalized_texts, transcript, args.verbose, args.remove_punct) + line['nemo_normalized'] = normalized_text + line['CER_nemo_normalized'] = cer + return line + + +def normalize_manifest(args): + """ + Args: + args.audio_data: path to .json manifest file. + """ + normalizer = NormalizerWithAudio(input_case=args.input_case) + manifest_out = args.audio_data.replace('.json', '_normalized.json') + asr_model = None + with open(args.audio_data, 'r') as f: + with open(manifest_out, 'w') as f_out: + lines = f.readlines() + first_line = json.loads(lines[0]) + if 'transcript' not in first_line: + asr_model = get_asr_model(args.model) + normalized_lines = Parallel(n_jobs=-1)( + delayed(_normalize_line)(normalizer, line, asr_model) for line in lines + ) + + for line in normalized_lines: + f_out.write(json.dumps(line, ensure_ascii=False) + '\n') + print(f'Normalized version saved at {manifest_out}') + + +if __name__ == "__main__": + args = parse_args() + + start = time.time() + if args.text: + normalizer = NormalizerWithAudio(input_case=args.input_case) + if os.path.exists(args.text): + with open(args.text, 'r') as f: + args.text = f.read().strip() + normalized_texts = normalizer.normalize( + text=args.text, + verbose=args.verbose, + n_tagged=args.n_tagged, + punct_pre_process=not args.no_punct_pre_process, + punct_post_process=not args.no_punct_post_process, + ) + if args.audio_data: + asr_model = get_asr_model(args.model) + transcript = asr_model.transcribe([args.audio_data])[0] + normalized_text, cer = normalizer.select_best_match( + normalized_texts, transcript, args.verbose, args.remove_punct + ) + print(f'Transcript: {transcript}') + print(f'Normalized: {normalized_text}') + else: + print('Normalization options:') + for norm_text in normalized_texts: + print(norm_text) + elif not os.path.exists(args.audio_data): + raise ValueError(f'{args.audio_data} not found.') + elif args.audio_data.endswith('.json'): + normalize_manifest(args) + else: + raise ValueError( + "Provide either path to .json manifest in '--audio_data' OR " + + "'--audio_data' path to audio file and '--text' path to a text file OR" + "'--text' string text (for debugging without audio)" + ) + print(f'Execution time: {round((time.time() - start)/60, 2)} min.') diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/run_evaluate.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/run_evaluate.py new file mode 100644 index 0000000000..3065a09005 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/run_evaluate.py @@ -0,0 +1,115 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser + +from nemo_text_processing.text_normalization.clean_eval_data import filter_loaded_data +from nemo_text_processing.text_normalization.data_loader_utils import ( + evaluate, + known_types, + load_files, + training_data_to_sentences, + training_data_to_tokens, +) +from nemo_text_processing.text_normalization.normalize import Normalizer + + +''' +Runs Evaluation on data in the format of : \t\t<`self` if trivial class or normalized text> +like the Google text normalization data https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish +''' + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("--input", help="input file path", type=str) + parser.add_argument( + "--input_case", help="input capitalization", choices=["lower_cased", "cased"], default="cased", type=str + ) + parser.add_argument( + "--cat", + dest="category", + help="focus on class only (" + ", ".join(known_types) + ")", + type=str, + default=None, + choices=known_types, + ) + parser.add_argument("--filter", action='store_true', help="clean data for normalization purposes") + return parser.parse_args() + + +if __name__ == "__main__": + # Example usage: + # python run_evaluate.py --input= --cat= --filter + args = parse_args() + file_path = args.input + normalizer = Normalizer(input_case=args.input_case) + + print("Loading training data: " + file_path) + training_data = load_files([file_path]) + + if args.filter: + training_data = filter_loaded_data(training_data) + + if args.category is None: + print("Sentence level evaluation...") + sentences_un_normalized, sentences_normalized, _ = training_data_to_sentences(training_data) + print("- Data: " + str(len(sentences_normalized)) + " sentences") + sentences_prediction = normalizer.normalize_list(sentences_un_normalized) + print("- Normalized. Evaluating...") + sentences_accuracy = evaluate( + preds=sentences_prediction, labels=sentences_normalized, input=sentences_un_normalized + ) + print("- Accuracy: " + str(sentences_accuracy)) + + print("Token level evaluation...") + tokens_per_type = training_data_to_tokens(training_data, category=args.category) + token_accuracy = {} + for token_type in tokens_per_type: + print("- Token type: " + token_type) + tokens_un_normalized, tokens_normalized = tokens_per_type[token_type] + print(" - Data: " + str(len(tokens_normalized)) + " tokens") + tokens_prediction = normalizer.normalize_list(tokens_un_normalized) + print(" - Denormalized. Evaluating...") + token_accuracy[token_type] = evaluate( + preds=tokens_prediction, labels=tokens_normalized, input=tokens_un_normalized + ) + print(" - Accuracy: " + str(token_accuracy[token_type])) + token_count_per_type = {token_type: len(tokens_per_type[token_type][0]) for token_type in tokens_per_type} + token_weighted_accuracy = [ + token_count_per_type[token_type] * accuracy for token_type, accuracy in token_accuracy.items() + ] + print("- Accuracy: " + str(sum(token_weighted_accuracy) / sum(token_count_per_type.values()))) + print(" - Total: " + str(sum(token_count_per_type.values())), '\n') + + print(" - Total: " + str(sum(token_count_per_type.values())), '\n') + + for token_type in token_accuracy: + if token_type not in known_types: + raise ValueError("Unexpected token type: " + token_type) + + if args.category is None: + c1 = ['Class', 'sent level'] + known_types + c2 = ['Num Tokens', len(sentences_normalized)] + [ + token_count_per_type[known_type] if known_type in tokens_per_type else '0' for known_type in known_types + ] + c3 = ['Normalization', sentences_accuracy] + [ + token_accuracy[known_type] if known_type in token_accuracy else '0' for known_type in known_types + ] + + for i in range(len(c1)): + print(f'{str(c1[i]):10s} | {str(c2[i]):10s} | {str(c3[i]):5s}') + else: + print(f'numbers\t{token_count_per_type[args.category]}') + print(f'Normalization\t{token_accuracy[args.category]}') diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/run_predict.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/run_predict.py new file mode 100644 index 0000000000..3059dfe07f --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/run_predict.py @@ -0,0 +1,78 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser +from typing import List + +from nemo_text_processing.text_normalization.normalize import Normalizer + + +''' +Runs normalization prediction on text data +''' + + +def load_file(file_path: str) -> List[str]: + """ + Load given text file into list of string. + + Args: + file_path: file path + + Returns: flat list of string + """ + res = [] + with open(file_path, 'r') as fp: + for line in fp: + res.append(line) + return res + + +def write_file(file_path: str, data: List[str]): + """ + Writes out list of string to file. + + Args: + file_path: file path + data: list of string + + """ + with open(file_path, 'w') as fp: + for line in data: + fp.write(line + '\n') + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("--input", help="input file path", required=True, type=str) + parser.add_argument("--output", help="output file path", required=True, type=str) + parser.add_argument( + "--input_case", help="input capitalization", choices=["lower_cased", "cased"], default="cased", type=str + ) + parser.add_argument("--verbose", help="print meta info for debugging", action='store_true') + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + file_path = args.input + normalizer = Normalizer(input_case=args.input_case) + + print("Loading data: " + file_path) + data = load_file(file_path) + + print("- Data: " + str(len(data)) + " sentences") + normalizer_prediction = normalizer.normalize_list(data, verbose=args.verbose) + write_file(args.output, normalizer_prediction) + print(f"- Normalized. Writing out to {args.output}") diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/cardinal.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/cardinal.py new file mode 100644 index 0000000000..2d01a49357 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/cardinal.py @@ -0,0 +1,115 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.text_normalization.data_loader_utils import get_abs_path, load_labels +from nemo_text_processing.text_normalization.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, insert_space +from nemo_text_processing.text_normalization.taggers.date import get_hundreds_graph + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + -23 -> cardinal { negative: "true" integer: "twenty three" } } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + graph = pynini.Far(get_abs_path("data/numbers/cardinal_number_name.far")).get_fst() + self.graph_hundred_component_at_least_one_none_zero_digit = ( + pynini.closure(NEMO_DIGIT, 2, 3) | pynini.difference(NEMO_DIGIT, pynini.accep("0")) + ) @ graph + self.graph = ( + pynini.closure(NEMO_DIGIT, 1, 3) + + pynini.closure(pynini.closure(pynutil.delete(","), 0, 1) + NEMO_DIGIT + NEMO_DIGIT + NEMO_DIGIT) + ) @ graph + + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + single_digits_graph = pynutil.add_weight(pynini.invert(graph_digit | graph_zero), 1.2) | pynutil.add_weight( + pynini.cross("0", "oh"), 1.1 + ) + self.single_digits_graph = single_digits_graph + pynini.closure(pynutil.insert(" ") + single_digits_graph) + + if not deterministic: + single_digits_graph_with_commas = pynini.closure( + self.single_digits_graph + pynutil.insert(" "), 1, 3 + ) + pynini.closure( + pynutil.delete(",") + + single_digits_graph + + pynutil.insert(" ") + + single_digits_graph + + pynutil.insert(" ") + + single_digits_graph, + 1, + ) + self.graph |= self.single_digits_graph | get_hundreds_graph() | single_digits_graph_with_commas + self.range_graph = ( + pynini.closure(pynutil.insert("from "), 0, 1) + + self.graph + + (pynini.cross("-", " to ") | pynini.cross("-", " ")) + + self.graph + ) + + self.range_graph |= self.graph + (pynini.cross("x", " by ") | pynini.cross(" x ", " by ")) + self.graph + self.range_graph = self.range_graph.optimize() + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + final_graph = self.graph | pynutil.add_weight(self.get_serial_graph(), 1.2) + + if not deterministic: + final_graph |= self.range_graph + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + final_graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() + + def get_serial_graph(self): + """ + Finite state transducer for classifying serial. + The serial is a combination of digits, letters and dashes, e.g.: + c325-b -> tokens { serial { value: "c three two five b" } } + """ + alpha = NEMO_ALPHA + + if self.deterministic: + num_graph = self.single_digits_graph + else: + num_graph = self.graph + letter_pronunciation = pynini.string_map(load_labels(get_abs_path("data/letter_pronunciation.tsv"))) + alpha |= letter_pronunciation + + delimiter = insert_space | pynini.cross("-", " ") | pynini.cross("/", " ") + letter_num = pynini.closure(alpha + delimiter, 1) + num_graph + num_letter = pynini.closure(num_graph + delimiter, 1) + alpha + next_alpha_or_num = pynini.closure(delimiter + (alpha | num_graph)) + serial_graph = (letter_num | num_letter) + next_alpha_or_num + + if not self.deterministic: + serial_graph += pynini.closure(pynini.accep("s") | pynini.cross("s", "es"), 0, 1) + return serial_graph diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/date.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/date.py new file mode 100644 index 0000000000..8c7c6a713a --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/date.py @@ -0,0 +1,201 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.data_loader_utils import get_abs_path, load_labels +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_SIGMA, + TO_LOWER, + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) + +try: + import pynini + from pynini.lib import pynutil + + graph_teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))).optimize() + graph_digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))).optimize() + ties_graph = pynini.invert(pynini.string_file(get_abs_path("data/numbers/ties.tsv"))).optimize() + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + # Add placeholders for global variables + graph_teen = None + graph_digit = None + ties_graph = None + + PYNINI_AVAILABLE = True + + +def get_ties_graph(deterministic: bool = True): + """ + Returns two digit transducer, e.g. + 03 -> o three + 12 -> thirteen + 20 -> twenty + """ + graph = graph_teen | ties_graph + pynutil.delete("0") | ties_graph + insert_space + graph_digit + + if deterministic: + graph = graph | pynini.cross("0", "o") + insert_space + graph_digit + else: + graph = ( + graph + | (pynini.cross("0", "oh") | pynini.cross("0", "o") | pynini.cross("0", "zero")) + + insert_space + + graph_digit + ) + + return graph.optimize() + + +def get_hundreds_graph(deterministic: bool = True): + """ + Returns a four digit transducer which is combination of ties/teen or digits + (using hundred instead of thousand format), e.g. + 1219 -> twelve nineteen + 3900 -> thirty nine hundred + """ + graph_ties = get_ties_graph(deterministic) + graph = ( + graph_ties + insert_space + graph_ties + | graph_teen + insert_space + pynini.cross("00", "hundred") + | (graph_teen + insert_space + (ties_graph | pynini.cross("1", "ten")) + pynutil.delete("0s")) + @ pynini.cdrewrite(pynini.cross("y", "ies") | pynutil.insert("s"), "", "[EOS]", NEMO_SIGMA) + | pynutil.add_weight( + graph_digit + + insert_space + + pynini.cross("00", "thousand") + + (pynutil.delete("0") | insert_space + graph_digit), + weight=-0.001, + ) + ) + return graph + + +def _get_year_graph(deterministic: bool = True): + """ + Transducer for year, only from 1000 - 2999 e.g. + 1290 -> twelve nineteen + 2000 - 2009 will be verbalized as two thousand. + """ + graph = get_hundreds_graph(deterministic) + graph = ( + pynini.union("1", "2") + + NEMO_DIGIT + + NEMO_DIGIT + + NEMO_DIGIT + + pynini.closure(pynini.cross(" s", "s") | "s", 0, 1) + ) @ graph + return graph + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, e.g. + jan. 5, 2012 -> date { month: "january" day: "five" year: "twenty twelve" preserve_order: true } + jan. 5 -> date { month: "january" day: "five" preserve_order: true } + 5 january 2012 -> date { day: "five" month: "january" year: "twenty twelve" preserve_order: true } + 2012-01-05 -> date { year: "twenty twelve" month: "january" day: "five" } + 2012.01.05 -> date { year: "twenty twelve" month: "january" day: "five" } + 2012/01/05 -> date { year: "twenty twelve" month: "january" day: "five" } + 2012 -> date { year: "twenty twelve" } + + Args: + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + month_graph = pynini.string_file(get_abs_path("data/months/names.tsv")).optimize() + month_graph |= (TO_LOWER + pynini.closure(NEMO_CHAR)) @ month_graph + month_abbr_graph = pynini.string_file(get_abs_path("data/months/abbr.tsv")).optimize() + month_abbr_graph = ( + month_abbr_graph | (TO_LOWER + pynini.closure(NEMO_CHAR)) @ month_abbr_graph + ) + pynini.closure(pynutil.delete("."), 0, 1) + month_graph |= month_abbr_graph + + # to support all caps names + names_all_caps = [[x[0].upper()] for x in load_labels(get_abs_path("data/months/names.tsv"))] + abbr_all_caps = [(x.upper(), y) for x, y in load_labels(get_abs_path("data/months/abbr.tsv"))] + month_graph |= pynini.string_map(names_all_caps) | ( + pynini.string_map(abbr_all_caps) + pynini.closure(pynutil.delete("."), 0, 1) + ) + + month_numbers_graph = pynini.string_file(get_abs_path("data/months/numbers.tsv")).optimize() + cardinal_graph = cardinal.graph_hundred_component_at_least_one_none_zero_digit + + year_graph = _get_year_graph(deterministic) + + YEAR_WEIGHT = 0.001 + year_graph_standalone = ( + pynutil.insert("year: \"") + pynutil.add_weight(year_graph, YEAR_WEIGHT) + pynutil.insert("\"") + ) + + month_graph = pynutil.insert("month: \"") + month_graph + pynutil.insert("\"") + month_numbers_graph = pynutil.insert("month: \"") + month_numbers_graph + pynutil.insert("\"") + + day_graph = ( + pynutil.insert("day: \"") + + ((pynini.union("1", "2", "3") + NEMO_DIGIT) | NEMO_DIGIT) @ cardinal_graph + + pynutil.insert("\"") + ) + optional_day_graph = pynini.closure(delete_extra_space + day_graph, 0, 1) + + year_graph = pynutil.insert("year: \"") + year_graph + pynutil.insert("\"") + optional_graph_year = pynini.closure(delete_extra_space + year_graph, 0, 1,) + graph_mdy = ( + month_graph + + optional_day_graph + + delete_space + + pynini.closure(pynutil.delete(","), 0, 1) + + optional_graph_year + ) + + delete_sep = pynutil.delete(pynini.union("-", "/", ".")) + graph_mdy |= ( + month_numbers_graph + + delete_sep + + insert_space + + pynini.closure(pynutil.delete("0"), 0, 1) + + day_graph + + delete_sep + + insert_space + + year_graph + ) + + graph_dmy = day_graph + delete_extra_space + month_graph + optional_graph_year + graph_ymd = ( + year_graph + + delete_sep + + insert_space + + month_numbers_graph + + delete_sep + + insert_space + + pynini.closure(pynutil.delete("0"), 0, 1) + + day_graph + ) + + final_graph = (graph_mdy | graph_dmy) + pynutil.insert(" preserve_order: true") + final_graph |= graph_ymd | year_graph_standalone + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/decimal.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/decimal.py new file mode 100644 index 0000000000..c4eba3fe65 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/decimal.py @@ -0,0 +1,101 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.data_loader_utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import GraphFst, delete_extra_space, delete_space + +try: + import pynini + from pynini.lib import pynutil + + delete_space = pynutil.delete(" ") + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. 1 million -> integer_part: "one" quantity: "million" + e.g. 1.5 million -> integer_part: "one" fractional_part: "five" quantity: "million" + + Args: + decimal: decimal FST + cardinal_up_to_hundred: cardinal FST + """ + numbers = cardinal_up_to_hundred + suffix = pynini.union("million", "billion", "trillion", "quadrillion", "quintillion", "sextillion") + res = ( + pynutil.insert("integer_part: \"") + + numbers + + pynutil.insert("\"") + + delete_extra_space + + pynutil.insert("quantity: \"") + + suffix + + pynutil.insert("\"") + ) + res |= decimal + delete_extra_space + pynutil.insert("quantity: \"") + (suffix | "thousand") + pynutil.insert("\"") + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -12.5006 billion -> decimal { negative: "true" integer_part: "12" fractional_part: "five o o six" quantity: "billion" } + 1 billion -> decimal { integer_part: "one" quantity: "billion" } + + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + cardinal_graph_hundred_component_at_least_one_none_zero_digit = ( + cardinal.graph_hundred_component_at_least_one_none_zero_digit + ) + + graph_decimal = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_decimal |= pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + graph_decimal = ( + pynini.cross("zero", "0") + | graph_decimal + | (graph_decimal | pynini.cross("o", "0")) + + pynini.closure(delete_space + (graph_decimal | pynini.cross("o", "0")), 1) + ) + self.graph = pynini.invert(graph_decimal).optimize() + if not deterministic: + self.graph = self.graph | cardinal_graph + + point = pynutil.delete(".") + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + graph_fractional = pynutil.insert("fractional_part: \"") + self.graph + pynutil.insert("\"") + graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + final_graph_wo_sign = ( + pynini.closure(graph_integer + pynutil.insert(" "), 0, 1) + point + pynutil.insert(" ") + graph_fractional + ) + + self.final_graph_wo_negative = final_graph_wo_sign | get_quantity( + final_graph_wo_sign, cardinal_graph_hundred_component_at_least_one_none_zero_digit + ) + + final_graph = optional_graph_negative + self.final_graph_wo_negative + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/electronic.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/electronic.py new file mode 100644 index 0000000000..097a8cd918 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/electronic.py @@ -0,0 +1,63 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.text_normalization.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, get_abs_path + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic: as URLs, email addresses, etc. + e.g. cdf1@abc.edu -> tokens { electronic { username: "cdf1" domain: "abc.edu" } } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + accepted_symbols = [] + with open(get_abs_path("data/electronic/symbols.tsv"), 'r') as f: + for line in f: + symbol, _ = line.split('\t') + accepted_symbols.append(pynini.accep(symbol)) + + username = ( + pynutil.insert("username: \"") + + NEMO_ALPHA + + pynini.closure(NEMO_ALPHA | NEMO_DIGIT | pynini.union(*accepted_symbols)) + + pynutil.insert("\"") + + pynini.cross('@', ' ') + ) + domain_graph = ( + NEMO_ALPHA + + (pynini.closure(NEMO_ALPHA | NEMO_DIGIT | pynini.accep('-') | pynini.accep('.'))) + + (NEMO_ALPHA | NEMO_DIGIT) + ) + domain_graph = pynutil.insert("domain: \"") + domain_graph + pynutil.insert("\"") + graph = username + domain_graph + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/fraction.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/fraction.py new file mode 100644 index 0000000000..0feb5ce1dc --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/fraction.py @@ -0,0 +1,50 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import GraphFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + "23 4/5" -> + tokens { fraction { integer: "twenty three" numerator: "four" denominator: "five" } } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + integer = pynutil.insert("integer: \"") + cardinal_graph + pynutil.insert("\"") + pynini.accep(" ") + numerator = ( + pynutil.insert("numerator: \"") + cardinal_graph + (pynini.cross("/", "\" ") | pynini.cross(" / ", "\" ")) + ) + denominator = pynutil.insert("denominator: \"") + cardinal_graph + pynutil.insert("\"") + + self.graph = pynini.closure(integer, 0, 1) + numerator + denominator + final_graph = self.add_tokens(self.graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/measure.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/measure.py new file mode 100644 index 0000000000..a0475c5b10 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/measure.py @@ -0,0 +1,162 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.data_loader_utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_ALPHA, + NEMO_NON_BREAKING_SPACE, + NEMO_SIGMA, + SINGULAR_TO_PLURAL, + GraphFst, + convert_space, + delete_space, +) + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure, suppletive aware, e.g. + -12kg -> measure { negative: "true" cardinal { integer: "twelve" } units: "kilograms" } + 1kg -> measure { cardinal { integer: "one" } units: "kilogram" } + .5kg -> measure { decimal { fractional_part: "five" } units: "kilograms" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + if not deterministic: + cardinal_graph |= cardinal.range_graph + + graph_unit = pynini.string_file(get_abs_path("data/measurements.tsv")) + graph_unit_plural = convert_space(graph_unit @ SINGULAR_TO_PLURAL) + graph_unit = convert_space(graph_unit) + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + graph_unit2 = pynini.cross("/", "per") + delete_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit + + optional_graph_unit2 = pynini.closure( + delete_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit2, 0, 1, + ) + + unit_plural = ( + pynutil.insert("units: \"") + + (graph_unit_plural + optional_graph_unit2 | graph_unit2) + + pynutil.insert("\"") + ) + + unit_singular = ( + pynutil.insert("units: \"") + (graph_unit + optional_graph_unit2 | graph_unit2) + pynutil.insert("\"") + ) + + subgraph_decimal = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal.final_graph_wo_negative + + delete_space + + pynutil.insert(" } ") + + unit_plural + ) + + subgraph_cardinal = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + ((NEMO_SIGMA - "1") @ cardinal_graph) + + delete_space + + pynutil.insert("\"") + + pynutil.insert(" } ") + + unit_plural + ) + + subgraph_cardinal |= ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + pynini.cross("1", "one") + + delete_space + + pynutil.insert("\"") + + pynutil.insert(" } ") + + unit_singular + ) + + cardinal_dash_alpha = ( + pynutil.insert("cardinal { integer: \"") + + cardinal_graph + + pynini.cross('-', '') + + pynutil.insert("\" } units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + alpha_dash_cardinal = ( + pynutil.insert("units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynini.cross('-', '') + + pynutil.insert("\"") + + pynutil.insert(" cardinal { integer: \"") + + cardinal_graph + + pynutil.insert("\" } preserve_order: true") + ) + + decimal_dash_alpha = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynini.cross('-', '') + + pynutil.insert(" } units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynutil.insert("\"") + ) + + alpha_dash_decimal = ( + pynutil.insert("units: \"") + + pynini.closure(NEMO_ALPHA, 1) + + pynini.cross('-', '') + + pynutil.insert("\"") + + pynutil.insert(" decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" } preserve_order: true") + ) + + subgraph_fraction = ( + pynutil.insert("fraction { ") + fraction.graph + delete_space + pynutil.insert(" } ") + unit_plural + ) + + final_graph = ( + subgraph_decimal + | subgraph_cardinal + | cardinal_dash_alpha + | alpha_dash_cardinal + | decimal_dash_alpha + | alpha_dash_decimal + | subgraph_fraction + ) + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/money.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/money.py new file mode 100644 index 0000000000..5965934e3b --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/money.py @@ -0,0 +1,86 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.data_loader_utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_SIGMA, + SINGULAR_TO_PLURAL, + GraphFst, + convert_space, + insert_space, +) +from nemo_text_processing.text_normalization.taggers.date import get_hundreds_graph + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, suppletive aware, e.g. + $12.05 -> money { currency: "dollars" integer_part: "twelve" fractional_part: "o five" } + $1 -> money { currency: "dollar" integer_part: "one" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + graph_decimal_final = decimal.final_graph_wo_negative + + unit_singular = pynini.string_file(get_abs_path("data/currency/currency.tsv")) + unit_plural = convert_space(unit_singular @ SINGULAR_TO_PLURAL) + unit_singular = convert_space(unit_singular) + + graph_unit_singular = pynutil.insert("currency: \"") + unit_singular + pynutil.insert("\"") + graph_unit_plural = pynutil.insert("currency: \"") + unit_plural + pynutil.insert("\"") + + singular_graph = ( + graph_unit_singular + pynutil.insert(" integer_part: \"") + pynini.cross("1", "one") + pynutil.insert("\"") + ) + + graph_decimal = graph_unit_plural + insert_space + graph_decimal_final + + if deterministic: + graph_integer = ( + graph_unit_plural + + pynutil.insert(" integer_part: \"") + + ((NEMO_SIGMA - "1") @ cardinal_graph) + + pynutil.insert("\"") + ) + else: + graph_integer = ( + graph_unit_plural + + pynutil.insert(" integer_part: \"") + + ((NEMO_SIGMA - "1") @ (get_hundreds_graph(deterministic) | cardinal_graph)) + + pynutil.insert("\"") + ) + graph_decimal |= singular_graph + insert_space + graph_decimal_final + + graph_integer |= singular_graph + + final_graph = graph_integer | graph_decimal + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/ordinal.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/ordinal.py new file mode 100644 index 0000000000..49e452729d --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/ordinal.py @@ -0,0 +1,49 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_DIGIT, GraphFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ImportError, ModuleNotFoundError): + PYNINI_AVAILABLE = False + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal, e.g. + 13th -> ordinal { integer: "thirteen" } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + endings = ["rd", "th", "st", "nd"] + endings += [x.upper() for x in endings] + self.graph = ( + (pynini.closure(NEMO_DIGIT | pynini.accep(",")) + pynutil.delete(pynini.union(*endings))) @ cardinal_graph + ).optimize() + final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/punctuation.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/punctuation.py new file mode 100644 index 0000000000..6a42e2d7f4 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/punctuation.py @@ -0,0 +1,46 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import GraphFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = False +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + + s = "!#$%&\'()*+,-./:;<=>?@^_`{|}~\"" + punct = pynini.union(*s) + + graph = pynutil.insert("name: \"") + punct + pynutil.insert("\"") + + self.fst = graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/roman.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/roman.py new file mode 100644 index 0000000000..136d0fb520 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/roman.py @@ -0,0 +1,61 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.text_normalization.data_loader_utils import load_labels +from nemo_text_processing.text_normalization.graph_utils import GraphFst, get_abs_path, insert_space +from nemo_text_processing.text_normalization.taggers.cardinal import CardinalFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class RomanFst(GraphFst): + """ + Finite state transducer for classifying electronic: as URLs, email addresses, etc. + e.g. cdf1@abc.edu -> tokens { electronic { username: "cdf1" domain: "abc.edu" } } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="roman", kind="classify", deterministic=deterministic) + + def _load_roman(file: str): + roman = load_labels(get_abs_path(file)) + roman_numerals = [(x, y) for x, y in roman] + [(x.upper(), y) for x, y in roman] + return pynini.string_map(roman_numerals) + + cardinal_graph = CardinalFst(deterministic=True).graph + digit_teen = _load_roman("data/roman/digit_teen.tsv") @ cardinal_graph + ties = _load_roman("data/roman/ties.tsv") @ cardinal_graph + hundreds = _load_roman("data/roman/hundreds.tsv") @ cardinal_graph + + graph = ( + (ties | digit_teen | hundreds) + | (ties + insert_space + digit_teen) + | (hundreds + pynini.closure(insert_space + ties, 0, 1) + pynini.closure(insert_space + digit_teen, 0, 1)) + ).optimize() + + graph = pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + graph = self.add_tokens(graph) + self.fst = graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/telephone.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/telephone.py new file mode 100644 index 0000000000..fa2ec7848b --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/telephone.py @@ -0,0 +1,91 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.data_loader_utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import GraphFst, delete_space, insert_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying telephone, which includes country code, number part and extension + country code optional: +*** + number part: ***-***-****, or (***) ***-**** + extension optional: 1-9999 + E.g + +1 123-123-5678-1 -> telephone { country_code: "one" number_part: "one two three, one two three, five six seven eight" extension: "one" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + + add_separator = pynutil.insert(", ") # between components + digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))).optimize() | pynini.cross( + "0", "o" + ) + + country_code = ( + pynutil.insert("country_code: \"") + + pynutil.delete("+") + + pynini.closure(digit + insert_space, 0, 2) + + digit + + pynutil.insert("\"") + ) + optional_country_code = pynini.closure( + country_code + pynini.closure(pynutil.delete("-"), 0, 1) + delete_space + insert_space, 0, 1 + ) + number_part = ( + ( + (pynini.closure(digit + insert_space, 2, 2) + digit + pynutil.delete("-")) + | ( + pynutil.delete("(") + + pynini.closure(digit + insert_space, 2, 2) + + digit + + pynutil.delete(")") + + pynini.closure(pynutil.delete("-"), 0, 1) + + delete_space + ) + ) + + add_separator + + pynini.closure(digit + insert_space, 2, 2) + + digit + + pynutil.delete("-") + + add_separator + + pynini.closure(digit + insert_space, 3, 3) + + digit + ) + number_part = pynutil.insert("number_part: \"") + number_part + pynutil.insert("\"") + extension = ( + pynutil.insert("extension : \"") + + pynini.closure(digit + insert_space, 0, 3) + + digit + + pynutil.insert("\"") + ) + optional_extension = pynini.closure(insert_space + pynutil.delete("-") + extension, 0, 1) + + graph = optional_country_code + number_part + optional_extension + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/time.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/time.py new file mode 100644 index 0000000000..7da3f56763 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/time.py @@ -0,0 +1,115 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.text_normalization.data_loader_utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_DIGIT, + GraphFst, + convert_space, + delete_space, + insert_space, +) + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 12:30 a.m. est -> time { hours: "twelve" minutes: "thirty" suffix: "a m" zone: "e s t" } + 2.30 a.m. -> time { hours: "two" minutes: "thirty" suffix: "a m" } + 02.30 a.m. -> time { hours: "two" minutes: "thirty" suffix: "a m" } + 2.00 a.m. -> time { hours: "two" suffix: "a m" } + 2 a.m. -> time { hours: "two" suffix: "a m" } + 02:00 -> time { hours: "two" } + 2:00 -> time { hours: "two" } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + suffix_graph = pynini.string_file(get_abs_path("data/time_suffix.tsv")) + time_zone_graph = pynini.string_file(get_abs_path("data/time_zone.tsv")) + + # only used for < 1000 thousand -> 0 weight + cardinal = cardinal.graph + + labels_hour = [str(x) for x in range(0, 24)] + labels_minute_single = [str(x) for x in range(1, 10)] + labels_minute_double = [str(x) for x in range(10, 60)] + + delete_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | ( + pynini.closure(pynutil.delete("0"), 0, 1) + NEMO_DIGIT + ) + + graph_hour = delete_leading_zero_to_double_digit @ pynini.union(*labels_hour) @ cardinal + + graph_minute_single = pynini.union(*labels_minute_single) @ cardinal + graph_minute_double = pynini.union(*labels_minute_double) @ cardinal + + final_graph_hour = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"") + final_graph_minute = ( + pynutil.insert("minutes: \"") + + (pynini.cross("0", "o") + insert_space + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_suffix = pynutil.insert("suffix: \"") + convert_space(suffix_graph) + pynutil.insert("\"") + final_suffix_optional = pynini.closure(delete_space + insert_space + final_suffix, 0, 1) + final_time_zone_optional = pynini.closure( + delete_space + + insert_space + + pynutil.insert("zone: \"") + + convert_space(time_zone_graph) + + pynutil.insert("\""), + 0, + 1, + ) + + # 2:30 pm, 02:30, 2:00 + graph_hm = ( + final_graph_hour + + pynutil.delete(":") + + (pynutil.delete("00") | insert_space + final_graph_minute) + + final_suffix_optional + + final_time_zone_optional + ) + + # 2.xx pm/am + graph_hm2 = ( + final_graph_hour + + pynutil.delete(".") + + (pynutil.delete("00") | insert_space + final_graph_minute) + + delete_space + + insert_space + + final_suffix + + final_time_zone_optional + ) + # 2 pm est + graph_h = final_graph_hour + delete_space + insert_space + final_suffix + final_time_zone_optional + final_graph = (graph_hm | graph_h | graph_hm2).optimize() + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/tokenize_and_classify.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/tokenize_and_classify.py new file mode 100644 index 0000000000..1bef666a11 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/tokenize_and_classify.py @@ -0,0 +1,107 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.text_normalization.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.taggers.date import DateFst +from nemo_text_processing.text_normalization.taggers.decimal import DecimalFst +from nemo_text_processing.text_normalization.taggers.electronic import ElectronicFst +from nemo_text_processing.text_normalization.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.taggers.money import MoneyFst +from nemo_text_processing.text_normalization.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.taggers.roman import RomanFst +from nemo_text_processing.text_normalization.taggers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.taggers.time import TimeFst +from nemo_text_processing.text_normalization.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.taggers.word import WordFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, input_case: str, deterministic: bool = True): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + ordinal_graph = ordinal.fst + + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + fraction = FractionFst(deterministic=deterministic, cardinal=cardinal) + fraction_graph = fraction.fst + + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + date_graph = DateFst(cardinal=cardinal, deterministic=deterministic).fst + word_graph = WordFst(deterministic=deterministic).fst + time_graph = TimeFst(cardinal=cardinal, deterministic=deterministic).fst + telephone_graph = TelephoneFst(deterministic=deterministic).fst + electonic_graph = ElectronicFst(deterministic=deterministic).fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic).fst + whitelist_graph = WhiteListFst(input_case=input_case, deterministic=deterministic).fst + punct_graph = PunctuationFst(deterministic=deterministic).fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(date_graph, 1.09) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(telephone_graph, 1.1) + | pynutil.add_weight(electonic_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.1) + | pynutil.add_weight(word_graph, 100) + ) + + if not deterministic: + roman_graph = RomanFst(deterministic=deterministic).fst + # the weight matches the word_graph weight for "I" cases in long sentences with multiple semiotic tokens + classify |= pynutil.add_weight(roman_graph, 100) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/whitelist.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/whitelist.py new file mode 100644 index 0000000000..50c883f803 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/whitelist.py @@ -0,0 +1,68 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.data_loader_utils import get_abs_path, load_labels +from nemo_text_processing.text_normalization.graph_utils import GraphFst, convert_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + misses -> tokens { name: "mrs" } + for non-deterministic case: "Dr. Abc" -> + tokens { name: "drive" } tokens { name: "Abc" } + tokens { name: "doctor" } tokens { name: "Abc" } + tokens { name: "Dr." } tokens { name: "Abc" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, input_case: str, deterministic: bool = True): + super().__init__(name="whitelist", kind="classify") + + def _get_whitelist_graph(input_case, file="data/whitelist.tsv"): + whitelist = load_labels(get_abs_path(file)) + if input_case == "lower_cased": + whitelist = [(x.lower(), y) for x, y in whitelist] + else: + whitelist = [(x, y) for x, y in whitelist] + graph = pynini.string_map(whitelist) + return graph + + def _get_whitelist_non_deterministic_graph(file="data/whitelist_alternatives.tsv"): + whitelist = load_labels(get_abs_path(file)) + whitelist_lower = [(x.lower(), y.lower()) for x, y in whitelist] + whitelist_cased = [(x, y) for x, y in whitelist] + graph = pynini.string_map(whitelist_lower + whitelist_cased) + return graph + + graph = _get_whitelist_graph(input_case) + if not deterministic: + graph |= _get_whitelist_graph("lower_cased") | _get_whitelist_non_deterministic_graph() + + graph = pynutil.insert("name: \"") + convert_space(graph) + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/word.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/word.py new file mode 100644 index 0000000000..6ff35ee1f7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/taggers/word.py @@ -0,0 +1,41 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_SPACE, GraphFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. Considers sentence boundary exceptions. + e.g. sleep -> tokens { name: "sleep" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/token_parser.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/token_parser.py new file mode 100644 index 0000000000..d3f7fd95ad --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/token_parser.py @@ -0,0 +1,192 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import string +from collections import OrderedDict +from typing import Dict, List, Union + +PRESERVE_ORDER_KEY = "preserve_order" +EOS = "" + + +class TokenParser: + """ + Parses tokenized/classified text, e.g. 'tokens { money { integer: "20" currency: "$" } } tokens { name: "left"}' + + Args + text: tokenized text + """ + + def __call__(self, text): + """ + Setup function + + Args: + text: text to be parsed + + """ + self.text = text + self.len_text = len(text) + self.char = text[0] # cannot handle empty string + self.index = 0 + + def parse(self) -> List[dict]: + """ + Main function. Implements grammar: + A -> space F space F space F ... space + + Returns list of dictionaries + """ + l = list() + while self.parse_ws(): + token = self.parse_token() + if not token: + break + l.append(token) + return l + + def parse_token(self) -> Dict[str, Union[str, dict]]: + """ + Implements grammar: + F-> no_space KG no_space + + Returns: K, G as dictionary values + """ + d = OrderedDict() + key = self.parse_string_key() + if key is None: + return None + self.parse_ws() + if key == PRESERVE_ORDER_KEY: + self.parse_char(":") + self.parse_ws() + value = self.parse_chars("true") + else: + value = self.parse_token_value() + + d[key] = value + return d + + def parse_token_value(self) -> Union[str, dict]: + """ + Implements grammar: + G-> no_space :"VALUE" no_space | no_space {A} no_space + + Returns: string or dictionary + """ + if self.char == ":": + self.parse_char(":") + self.parse_ws() + self.parse_char("\"") + value_string = self.parse_string_value() + self.parse_char("\"") + return value_string + elif self.char == "{": + d = OrderedDict() + self.parse_char("{") + list_token_dicts = self.parse() + # flatten tokens + for tok_dict in list_token_dicts: + for k, v in tok_dict.items(): + d[k] = v + self.parse_char("}") + return d + else: + raise ValueError() + + def parse_char(self, exp) -> bool: + """ + Parses character + + Args: + exp: character to read in + + Returns true if successful + """ + assert self.char == exp + self.read() + return True + + def parse_chars(self, exp) -> bool: + """ + Parses characters + + Args: + exp: characters to read in + + Returns true if successful + """ + ok = False + for x in exp: + ok |= self.parse_char(x) + return ok + + def parse_string_key(self) -> str: + """ + Parses string key, can only contain ascii and '_' characters + + Returns parsed string key + """ + assert self.char not in string.whitespace and self.char != EOS + incl_criterium = string.ascii_letters + "_" + l = [] + while self.char in incl_criterium: + l.append(self.char) + if not self.read(): + raise ValueError() + + if not l: + return None + return "".join(l) + + def parse_string_value(self) -> str: + """ + Parses string value, ends with quote followed by space + + Returns parsed string value + """ + assert self.char not in string.whitespace and self.char != EOS + l = [] + while self.char != "\"" or self.text[self.index + 1] != " ": + l.append(self.char) + if not self.read(): + raise ValueError() + + if not l: + return None + return "".join(l) + + def parse_ws(self): + """ + Deletes whitespaces. + + Returns true if not EOS after parsing + """ + not_eos = self.char != EOS + while not_eos and self.char == " ": + not_eos = self.read() + return not_eos + + def read(self): + """ + Reads in next char. + + Returns true if not EOS + """ + if self.index < self.len_text - 1: # should be unique + self.index += 1 + self.char = self.text[self.index] + return True + self.char = EOS + return False diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/__init__.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/__init__.py new file mode 100644 index 0000000000..bc443be41c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/cardinal.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/cardinal.py new file mode 100644 index 0000000000..bdf7a2ab38 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/cardinal.py @@ -0,0 +1,56 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal, e.g. + cardinal { negative: "true" integer: "23" } -> minus twenty three + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + self.optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "minus ") + delete_space, 0, 1) + + if deterministic: + integer = pynini.closure(NEMO_NOT_QUOTE, 1) + else: + integer = ( + pynini.closure(NEMO_NOT_QUOTE) + + pynini.closure(pynini.cross("hundred ", "") | pynini.cross("hundred ", "hundred and "), 0, 1) + + pynini.closure(NEMO_NOT_QUOTE) + ) + + self.integer = delete_space + pynutil.delete("\"") + integer + pynutil.delete("\"") + integer = pynutil.delete("integer:") + self.integer + + self.numbers = self.optional_sign + integer + delete_tokens = self.delete_tokens(self.numbers) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/date.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/date.py new file mode 100644 index 0000000000..08195927df --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/date.py @@ -0,0 +1,100 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_space, +) + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { month: "february" day: "five" year: "twenty twelve" preserve_order: true } -> february fifth twenty twelve + date { day: "five" month: "february" year: "twenty twelve" preserve_order: true } -> the fifth of february twenty twelve + + Args: + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + month = pynini.closure(NEMO_NOT_QUOTE, 1) + day_cardinal = ( + pynutil.delete("day:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + day = day_cardinal @ ordinal.suffix + + if not deterministic: + month |= pynutil.insert(" of ") + month + day |= day_cardinal + + month = pynutil.delete("month:") + delete_space + pynutil.delete("\"") + month + pynutil.delete("\"") + + year = ( + pynutil.delete("year:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + delete_space + + pynutil.delete("\"") + ) + + # month (day) year + graph_mdy = ( + month + pynini.closure(delete_extra_space + day, 0, 1) + pynini.closure(delete_extra_space + year, 0, 1) + ) + + # day month year + graph_dmy = ( + pynutil.insert("the ") + + day + + delete_extra_space + + pynutil.insert("of ") + + month + + pynini.closure(delete_extra_space + year, 0, 1) + ) + + optional_preserve_order = pynini.closure( + pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + | pynutil.delete("field_order:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space + ) + + final_graph = (graph_mdy | year | graph_dmy) + delete_space + optional_preserve_order + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/decimal.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/decimal.py new file mode 100644 index 0000000000..9ebefebce9 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/decimal.py @@ -0,0 +1,74 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, e.g. + decimal { negative: "true" integer_part: "twelve" fractional_part: "five o o six" quantity: "billion" } -> minus twelve point five o o six billion + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal, deterministic: bool = True): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + + self.optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "minus ") + delete_space, 0, 1) + self.integer = pynutil.delete("integer_part:") + cardinal.integer + self.optional_integer = pynini.closure(self.integer + delete_space + insert_space, 0, 1) + self.fractional_default = ( + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + if deterministic: + self.fractional = pynutil.insert("point ") + self.fractional_default + else: + self.fractional = pynini.closure(pynutil.insert("point "), 0, 1) + self.fractional_default + + self.quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + self.optional_quantity = pynini.closure(self.quantity, 0, 1) + + graph = self.optional_sign + ( + self.integer + | (self.integer + self.quantity) + | (self.optional_integer + self.fractional + self.optional_quantity) + ) + self.numbers = graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/electronic.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/electronic.py new file mode 100644 index 0000000000..c51f84a2ac --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/electronic.py @@ -0,0 +1,91 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.data_loader_utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_ALPHA, + NEMO_NOT_QUOTE, + GraphFst, + delete_space, + insert_space, +) + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic + e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> c d f one at a b c dot e d u + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + graph_digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))).optimize() + graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).optimize() + user_name = ( + pynutil.delete("username:") + + delete_space + + pynutil.delete("\"") + + ( + pynini.closure( + pynutil.add_weight(graph_digit + insert_space, 1.09) + | pynutil.add_weight(pynini.closure(graph_symbols + pynutil.insert(" ")), 1.09) + | pynutil.add_weight(NEMO_NOT_QUOTE + insert_space, 1.1) + ) + ) + + pynutil.delete("\"") + ) + + domain_default = ( + pynini.closure(NEMO_NOT_QUOTE + insert_space) + + pynini.cross(".", "dot ") + + NEMO_NOT_QUOTE + + pynini.closure(insert_space + NEMO_NOT_QUOTE) + ) + + server_default = ( + pynini.closure((graph_digit | NEMO_ALPHA) + insert_space, 1) + + pynini.closure(graph_symbols + insert_space) + + pynini.closure((graph_digit | NEMO_ALPHA) + insert_space, 1) + ) + server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) + insert_space + + domain_common = pynini.cross(".", "dot ") + pynini.string_file(get_abs_path("data/electronic/domain.tsv")) + + domain = ( + pynutil.delete("domain:") + + delete_space + + pynutil.delete("\"") + + (pynutil.add_weight(server_common, 1.09) | pynutil.add_weight(server_default, 1.1)) + + (pynutil.add_weight(domain_common, 1.09) | pynutil.add_weight(domain_default, 1.1)) + + delete_space + + pynutil.delete("\"") + ) + + graph = user_name + delete_space + pynutil.insert("at ") + delete_space + domain + delete_space + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/fraction.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/fraction.py new file mode 100644 index 0000000000..d49bd4bf5f --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/fraction.py @@ -0,0 +1,72 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, GraphFst, insert_space +from nemo_text_processing.text_normalization.verbalizers.ordinal import OrdinalFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. tokens { fraction { integer: "twenty three" numerator: "four" denominator: "five" } } -> + twenty three four fifth + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + suffix = OrdinalFst().suffix + + integer = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\" ") + numerator = pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\" ") + numerator_one = pynutil.delete("numerator: \"") + pynini.accep("one") + pynutil.delete("\" ") + denominator = pynutil.delete("denominator: \"") + ( + pynini.closure(NEMO_NOT_QUOTE) @ suffix | pynini.cross('four', 'quarter') + ) + conjunction = pynutil.insert("and ") + if not deterministic: + conjunction = pynini.closure(conjunction, 0, 1) + + integer = pynini.closure(integer + insert_space + conjunction, 0, 1) + + denominator_half = pynini.cross("numerator: \"one\" denominator: \"two\"", "a half") + denominator_one_two = pynini.cross("denominator: \"one\"", "over one") | pynini.cross( + "denominator: \"two\"", "halves" + ) + fraction_default = pynutil.add_weight( + numerator + insert_space + denominator + pynutil.insert("s") + pynutil.delete("\""), 0.001 + ) + fraction_with_one = pynutil.add_weight( + numerator_one + insert_space + denominator + pynutil.delete("\""), 0.0001 + ) + + graph = integer + denominator_half | (fraction_with_one | fraction_default) + graph |= pynini.cross("numerator: \"one\" denominator: \"two\"", "one half") + graph |= (numerator | numerator_one) + insert_space + denominator_one_two + + self.graph = graph + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/measure.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/measure.py new file mode 100644 index 0000000000..3937efe26c --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/measure.py @@ -0,0 +1,76 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_CHAR, GraphFst, delete_space, insert_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { negative: "true" cardinal { integer: "twelve" } units: "kilograms" } -> minus twelve kilograms + measure { decimal { integer_part: "twelve" fractional_part: "five" } units: "kilograms" } -> twelve point five kilograms + tokens { measure { units: "covid" decimal { integer_part: "nineteen" fractional_part: "five" } } } -> covid nineteen point five + + Args: + decimal: DecimalFst + cardinal: CardinalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + optional_sign = cardinal.optional_sign + unit = pynutil.delete("units: \"") + pynini.closure(NEMO_CHAR - " ", 1) + pynutil.delete("\"") + delete_space + + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + optional_sign + + delete_space + + decimal.numbers + + delete_space + + pynutil.delete("}") + ) + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + delete_space + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + + graph_fraction = ( + pynutil.delete("fraction {") + delete_space + fraction.graph + delete_space + pynutil.delete("}") + ) + + graph = (graph_cardinal | graph_decimal | graph_fraction) + delete_space + insert_space + unit + + # SH adds "preserve_order: true" by default + preserve_order = pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + graph |= unit + insert_space + (graph_cardinal | graph_decimal) + delete_space + pynini.closure(preserve_order) + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/money.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/money.py new file mode 100644 index 0000000000..97b95080bf --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/money.py @@ -0,0 +1,117 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_space, + get_abs_path, + insert_space, +) + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + + PYNINI_AVAILABLE = False + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { integer_part: "twelve" fractional_part: "o five" currency: "dollars" } -> twelve o five dollars + + Args: + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + def _get_minor_currencies(file): + minor_currencies = [] + with open(get_abs_path(file), 'r') as f: + for line in f: + min_cur = line.strip() + minor_currencies.append(pynutil.insert(min_cur)) + return minor_currencies + + unit = ( + pynutil.delete("currency:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + graph = decimal.numbers + delete_space + pynutil.insert(" ") + unit + + if not deterministic: + minor_currencies_singular = _get_minor_currencies("data/currency/currency_minor_one.tsv") + minor_currencies_singular = pynini.union(*minor_currencies_singular) + minor_currencies_singular = ( + pynini.closure(NEMO_NOT_QUOTE) + + ( + pynini.accep("one") + | pynini.cross("zero one", "one") + | pynini.cross("oh one", "one") + | pynini.cross(" o one", " one") + ) + + insert_space + + minor_currencies_singular + ) + + minor_currencies_plural = _get_minor_currencies("data/currency/currency_minor.tsv") + minor_currencies_plural = insert_space + pynini.union(*minor_currencies_plural) + + fractional_default = ( + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete("\"") + + ((pynini.closure(NEMO_NOT_QUOTE, 1) + minor_currencies_plural) | minor_currencies_singular) + + pynutil.delete("\"") + ) + + # $2.00 {two zero zero dollars} -> two dollars + fractional_with_zeros = ( + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete("\"") + + pynini.cross('zero', '') + + pynini.closure(pynini.cross(' zero', '')) + + delete_space + + pynutil.delete("\"") + + delete_space + ) + + fractional = fractional_with_zeros | fractional_default + + graph |= ( + decimal.integer + + delete_space + + insert_space + + unit + + delete_space + + insert_space + + pynini.closure(pynutil.insert("and "), 0, 1) + + fractional + ) + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/ordinal.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/ordinal.py new file mode 100644 index 0000000000..20ec399a29 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/ordinal.py @@ -0,0 +1,63 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.text_normalization.data_loader_utils import get_abs_path +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal, e.g. + ordinal { integer: "thirteen" } } -> thirteenth + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv")).invert() + graph_teens = pynini.string_file(get_abs_path("data/ordinals/teen.tsv")).invert() + + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + convert_rest = pynutil.insert("th", weight=0.01) + + suffix = pynini.cdrewrite( + graph_digit | graph_teens | pynutil.add_weight(pynini.cross("ty", "tieth"), weight=0.001) | convert_rest, + "", + "[EOS]", + NEMO_SIGMA, + ).optimize() + graph = graph @ suffix + self.suffix = suffix + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/roman.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/roman.py new file mode 100644 index 0000000000..bb42f3c522 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/roman.py @@ -0,0 +1,46 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, GraphFst +from nemo_text_processing.text_normalization.verbalizers.ordinal import OrdinalFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class RomanFst(GraphFst): + """ + Finite state transducer for verbalizing roman numerals + e.g. tokens { roman { integer: "one" } } -> one + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="roman", kind="verbalize", deterministic=deterministic) + suffix = OrdinalFst().suffix + + integer = pynini.closure(NEMO_NOT_QUOTE) + integer |= pynini.closure(pynutil.insert("the "), 0, 1) + integer @ suffix + graph = pynutil.delete("integer: \"") + integer + pynutil.delete("\"") + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/telephone.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/telephone.py new file mode 100644 index 0000000000..3920819adb --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/telephone.py @@ -0,0 +1,69 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone numbers, e.g. + telephone { country_code: "one" number_part: "one two three, one two three, five six seven eight" extension: "one" } + -> one, one two three, one two three, five six seven eight, one + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + add_separator = pynutil.insert(",") # between components + + optional_country_code = pynini.closure( + pynutil.delete("country_code: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + add_separator + + insert_space, + 0, + 1, + ) + + number_part = pynutil.delete("number_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + optional_extension = pynini.closure( + delete_space + + add_separator + + insert_space + + pynutil.delete("extension: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\""), + 0, + 1, + ) + + graph = optional_country_code + number_part + optional_extension + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/time.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/time.py new file mode 100644 index 0000000000..c5881c3ee7 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/time.py @@ -0,0 +1,74 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + time { hours: "twelve" minutes: "thirty" suffix: "a m" zone: "e s t" } -> twelve thirty a m e s t + time { hours: "twelve" } -> twelve o'clock + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + minute = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + suffix = ( + pynutil.delete("suffix:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_suffix = pynini.closure(delete_space + insert_space + suffix, 0, 1) + zone = ( + pynutil.delete("zone:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_zone = pynini.closure(delete_space + insert_space + zone, 0, 1) + graph = hour + delete_space + insert_space + minute + optional_suffix + optional_zone + graph |= hour + insert_space + pynutil.insert("o'clock") + optional_zone + graph |= hour + delete_space + insert_space + suffix + optional_zone + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/verbalize.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/verbalize.py new file mode 100644 index 0000000000..b14468a8f0 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/verbalize.py @@ -0,0 +1,79 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import GraphFst +from nemo_text_processing.text_normalization.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.verbalizers.date import DateFst +from nemo_text_processing.text_normalization.verbalizers.decimal import DecimalFst +from nemo_text_processing.text_normalization.verbalizers.electronic import ElectronicFst +from nemo_text_processing.text_normalization.verbalizers.fraction import FractionFst +from nemo_text_processing.text_normalization.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.verbalizers.roman import RomanFst +from nemo_text_processing.text_normalization.verbalizers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.verbalizers.time import TimeFst +from nemo_text_processing.text_normalization.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + fraction = FractionFst(deterministic=deterministic) + fraction_graph = fraction.fst + telephone_graph = TelephoneFst(deterministic=deterministic).fst + electronic_graph = ElectronicFst(deterministic=deterministic).fst + measure = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + time_graph = TimeFst(deterministic=deterministic).fst + date_graph = DateFst(ordinal=ordinal, deterministic=deterministic).fst + money_graph = MoneyFst(decimal=decimal, deterministic=deterministic).fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + + graph = ( + time_graph + | date_graph + | money_graph + | measure_graph + | ordinal_graph + | decimal_graph + | cardinal_graph + | telephone_graph + | electronic_graph + | fraction_graph + | whitelist_graph + ) + + if not deterministic: + roman_graph = RomanFst(deterministic=deterministic).fst + graph |= roman_graph + + self.fst = graph diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/verbalize_final.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/verbalize_final.py new file mode 100644 index 0000000000..c2b2ac8647 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/verbalize_final.py @@ -0,0 +1,54 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.text_normalization.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.verbalizers.word import WordFst + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "twelve" minutes: "thirty" } } tokens { name: "now" } tokens { name: "." } -> its twelve thirty now . + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/whitelist.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/whitelist.py new file mode 100644 index 0000000000..5d1889d9b1 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/whitelist.py @@ -0,0 +1,48 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.text_normalization.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist + e.g. tokens { name: "misses" } } -> misses + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/word.py b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/word.py new file mode 100644 index 0000000000..8b1fcac449 --- /dev/null +++ b/models/tacotron2/NeMo/nemo_text_processing/text_normalization/verbalizers/word.py @@ -0,0 +1,43 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + +try: + import pynini + from pynini.lib import pynutil + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + PYNINI_AVAILABLE = False + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing word + e.g. tokens { name: "sleep" } -> sleep + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/models/tacotron2/NeMo/reinstall.sh b/models/tacotron2/NeMo/reinstall.sh new file mode 100755 index 0000000000..8e2eb10792 --- /dev/null +++ b/models/tacotron2/NeMo/reinstall.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -e + +INSTALL_OPTION=${1:-"dev"} + + +PIP=pip + +echo 'Uninstalling stuff' +${PIP} uninstall -y nemo_toolkit +${PIP} uninstall -y sacrebleu + +# Kept for legacy purposes +${PIP} uninstall -y nemo_asr +${PIP} uninstall -y nemo_nlp +${PIP} uninstall -y nemo_tts +${PIP} uninstall -y nemo_simple_gan +${PIP} uninstall -y nemo_cv + +${PIP} install -U setuptools + +echo 'Installing nemo and nemo_text_processing' +if [[ "$INSTALL_OPTION" == "dev" ]]; then + ${PIP} install --editable ".[all]" +else + rm -rf dist/ + python setup.py bdist_wheel + DIST_FILE=$(find ./dist -name "*.whl" | head -n 1) + ${PIP} install "${DIST_FILE}[all]" +fi + +echo 'Installing additional nemo_text_processing conda dependency' +bash nemo_text_processing/setup.sh > /dev/null 2>&1 && echo "nemo_text_processing installed!" || echo "nemo_text_processing could not be installed!" + +if [ -x "$(command -v conda)" ]; then + echo 'Attempting update to numba installation via conda' + conda update -c numba numba -y > /dev/null 2>&1 && echo "Numba updated!" || echo "Numba could not be updated!" +fi + +echo 'All done!' diff --git a/models/tacotron2/NeMo/requirements/requirements.txt b/models/tacotron2/NeMo/requirements/requirements.txt new file mode 100644 index 0000000000..ac7d1a3dd4 --- /dev/null +++ b/models/tacotron2/NeMo/requirements/requirements.txt @@ -0,0 +1,19 @@ +numpy>=1.18.2 +onnx>=1.7.0 +pytorch-lightning>=1.3.0 +python-dateutil +torch +wget +wrapt +ruamel.yaml +scikit-learn +omegaconf>=2.1.0 +hydra-core>=1.1.0 +transformers>=4.0.1 +sentencepiece<1.0.0 +webdataset>=0.1.48,<=0.1.62 +tqdm>=4.41.0 +opencc +pangu +jieba +numba diff --git a/models/tacotron2/NeMo/requirements/requirements_asr.txt b/models/tacotron2/NeMo/requirements/requirements_asr.txt new file mode 100644 index 0000000000..1f8b8419db --- /dev/null +++ b/models/tacotron2/NeMo/requirements/requirements_asr.txt @@ -0,0 +1,24 @@ +braceexpand +editdistance +frozendict +inflect +kaldi-io +librosa +marshmallow +packaging +ruamel.yaml +soundfile +sox +unidecode +kaldi-python-io +kaldiio +scipy +pandas +g2p_en +pydub +pyannote.core +pyannote.metrics +pyamg +torch-stft +ipywidgets +matplotlib diff --git a/models/tacotron2/NeMo/requirements/requirements_cv.txt b/models/tacotron2/NeMo/requirements/requirements_cv.txt new file mode 100644 index 0000000000..f0eabe59f4 --- /dev/null +++ b/models/tacotron2/NeMo/requirements/requirements_cv.txt @@ -0,0 +1,2 @@ +pillow +torchvision diff --git a/models/tacotron2/NeMo/requirements/requirements_docs.txt b/models/tacotron2/NeMo/requirements/requirements_docs.txt new file mode 100644 index 0000000000..8a380b7020 --- /dev/null +++ b/models/tacotron2/NeMo/requirements/requirements_docs.txt @@ -0,0 +1,6 @@ +latexcodec +sphinx>=3.0 +sphinx_rtd_theme +sphinxcontrib-bibtex +wrapt +numpy diff --git a/models/tacotron2/NeMo/requirements/requirements_nlp.txt b/models/tacotron2/NeMo/requirements/requirements_nlp.txt new file mode 100644 index 0000000000..b2a0d4b4f7 --- /dev/null +++ b/models/tacotron2/NeMo/requirements/requirements_nlp.txt @@ -0,0 +1,13 @@ +boto3 +h5py +matplotlib>=3.3.2 +sentencepiece +unidecode +youtokentome>=1.0.5 +numpy +rapidfuzz +gdown +megatron-lm==2.2.0 +inflect +sacrebleu[ja] +sacremoses>=0.0.43 diff --git a/models/tacotron2/NeMo/requirements/requirements_test.txt b/models/tacotron2/NeMo/requirements/requirements_test.txt new file mode 100644 index 0000000000..7c629176cb --- /dev/null +++ b/models/tacotron2/NeMo/requirements/requirements_test.txt @@ -0,0 +1,11 @@ +black==19.10b0 +isort[requirements] < 5 +parameterized +pytest +pytest-runner +ruamel.yaml +sphinx +sphinxcontrib-bibtex +wrapt +wget +wandb diff --git a/models/tacotron2/NeMo/requirements/requirements_text_processing.txt b/models/tacotron2/NeMo/requirements/requirements_text_processing.txt new file mode 100644 index 0000000000..9b7085f3ed --- /dev/null +++ b/models/tacotron2/NeMo/requirements/requirements_text_processing.txt @@ -0,0 +1,2 @@ +inflect +regex diff --git a/models/tacotron2/NeMo/requirements/requirements_tts.txt b/models/tacotron2/NeMo/requirements/requirements_tts.txt new file mode 100644 index 0000000000..f8721050e4 --- /dev/null +++ b/models/tacotron2/NeMo/requirements/requirements_tts.txt @@ -0,0 +1,7 @@ +matplotlib +pypinyin +attrdict +pystoi +pesq +g2p_en +nltk diff --git a/models/tacotron2/NeMo/scripts/asr_language_modeling/neural_rescorer/create_tarred_transformer_lm_dataset.py b/models/tacotron2/NeMo/scripts/asr_language_modeling/neural_rescorer/create_tarred_transformer_lm_dataset.py new file mode 100644 index 0000000000..6c9a836914 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/asr_language_modeling/neural_rescorer/create_tarred_transformer_lm_dataset.py @@ -0,0 +1,318 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Execution : + +python create_tarred_tokenized_text_lm_dataset.py \ + --text_path= \ + --data_root= \ + --tokenizer_name="bert-base-cased" \ + --tokenizer_vocab_file= \ + --num_shards=64 \ + --chunk_size=8192 \ + --chunk_write_buffer=512 \ + --lower_case \ + --log +""" + + +import argparse +import glob +import json +import logging +import os +import tarfile + +import joblib +import numpy as np +from tqdm import tqdm + +from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer + +parser = argparse.ArgumentParser(description='Tarred Tokenized dataset for text language modelling') + +# Data path arguments +parser.add_argument('--text_path', required=True, default=None, type=str, help='Text paths, seperated by commas') +parser.add_argument('--data_root', required=True, default=None, type=str, help='Output directory') + +# General arguments +parser.add_argument( + '--chunk_write_buffer', + default=128, + type=int, + help='Number of chunks of `chunk_size` to buffer for parallel tokenization and serial write to disk', +) +parser.add_argument('--lower_case', action='store_true', help='Whether to lower case the corpus') +parser.add_argument('--log', action='store_true', help='Whether to print logs to terminal') + +# Tokenizer arguments +parser.add_argument('--tokenizer_name', required=False, default=None, type=str, help='Tokenizer name for resolution') +parser.add_argument( + '--tokenizer_model', required=False, default=None, type=str, help='Path to tokenizer model for sentencepiece' +) +parser.add_argument('--tokenizer_vocab_file', required=False, type=str, default=None, help='Path to a vocab file') +parser.add_argument( + '--tokenizer_special_tokens', default=None, type=str, nargs='+', help='List of special tokens for the tokenizer' +) + +# Tarred dataset arguments +parser.add_argument('--num_shards', default=1, type=int, help='Number of shards for the tarfile') +parser.add_argument('--chunk_size', default=8192, type=int, help='Number of rows of data concatenated into a vector') + +parser.set_defaults(log=False, lower_case=False) +args = parser.parse_args() + + +def __build_dataset_from_text(texts: str, lower_case: bool, chunk_size: int): + if ',' in texts: + texts = texts.split(',') + else: + texts = [texts] + + num_lines = 0 + text_dataset = [] + + for text in texts: + with open(text, 'r') as in_reader: + reader = tqdm(iter(lambda: in_reader.readline(), ''), desc="Read 0 lines", unit=' lines') + + for i, line in enumerate(reader): + # Clean text line + line = line.replace("\n", "").strip() + + if lower_case: + line = line.lower() + + if line: + text_dataset.append(line) + + num_lines += 1 + if num_lines % 100000 == 0: + reader.set_description(f"Read {num_lines} lines") + + if num_lines % chunk_size == 0: + yield text_dataset, num_lines + + # Empty cache + text_dataset = [] + + logging.info(f"Finished extracting manifest : {text}") + + logging.info("Finished extracting all manifests ! Number of sentences : {}".format(num_lines)) + + if len(text_dataset) != 0: + yield text_dataset, num_lines + + +def __tokenize_str(texts, tokenizer): + tokenized_text = [] + for text in texts: + tok_text = tokenizer.text_to_ids(text) + tokenized_text.extend(tok_text) + return tokenized_text + + +def __tokenize_text( + text_paths, tokenizer, tokenized_cachedir, lower_case: bool = False, chunk_size=8192, write_buffer: int = -1 +): + if write_buffer < 1: + write_buffer = max(os.cpu_count() - write_buffer, 1) + + logging.info(f"Using write chunk buffer of size {write_buffer}") + + if not os.path.exists(tokenized_cachedir): + os.makedirs(tokenized_cachedir) + + # global parameters + global_chunk_idx = 0 + chunk_paths = [] + chunk_lens = [] + + # buffer parameters + data_cache = [] + chunk_idx = 0 + + text_generator = iter(__build_dataset_from_text(text_paths, lower_case=lower_case, chunk_size=chunk_size)) + global_num_lines = 0 + last_batch = False + + with joblib.Parallel(n_jobs=-2, verbose=10) as parallel: + + while True: + try: + data, num_lines = next(text_generator) + data_cache.append(data) + + global_num_lines = num_lines + except StopIteration: + last_batch = True + + # Update counters + chunk_idx += 1 + + if (chunk_idx == write_buffer) or last_batch: + # write the chunks into disk after parallel tokenization + tokenized_data_list = parallel( + joblib.delayed(__tokenize_str)(chunk, tokenizer) for chunk in data_cache + ) + + # Sequential write cache + for chunk in tokenized_data_list: + fp = os.path.join(tokenized_cachedir, f"chunk_{global_chunk_idx}.npy") + chunk = np.asarray(chunk, dtype=np.int64) + np.save(fp, chunk, allow_pickle=False) + + chunk_paths.append(fp) + chunk_lens.append(len(chunk)) + + global_chunk_idx += 1 + + logging.info(f"Wrote a total of {global_chunk_idx} chunks to file...") + + # reset buffers + data_cache.clear() + del data_cache + + data_cache = [] + chunk_idx = 0 + + if last_batch: + logging.info("Finished tokenizing last chunk") + break + + logging.info( + f"Chunking {global_num_lines} rows into {global_num_lines // chunk_size} tasks (each chunk contains {chunk_size} elements)" + ) + return chunk_paths, chunk_lens + + +def __create_chunk(data_root, chunk_path, shard_id, compute_metrics=False): + """Creates a tarball containing the tokenized text chunks. + """ + tar = tarfile.open(os.path.join(data_root, f'text_{shard_id}.tar'), mode='a') + + # We squash the filename since we do not preserve directory structure of tokenized text in the tarball. + base, ext = os.path.splitext(chunk_path) + base = base.replace(os.pathsep, '_') + # Need the following replacement as long as WebDataset splits on first period + base = base.replace('.', '_') + squashed_filename = f'{base}{ext}' + tar.add(chunk_path, arcname=squashed_filename) + + tar.close() + + if compute_metrics: + data = np.load(chunk_path, allow_pickle=False) + chunk_len = len(data) + return (chunk_len,) + else: + return None + + +def __write_tarred_tokenized_text_dataset(data_root, num_shards, chunk_paths, chunk_lens): + num_chunks = len(chunk_paths) + + if chunk_lens is not None: + num_text = sum(chunk_lens) + shard_counts = {chunk_id: chunk_len for chunk_id, chunk_len in enumerate(chunk_lens)} + compute_metrics = False + + else: + num_text = 0 + shard_counts = {} + compute_metrics = True + + for chunk_id, chunk_path in enumerate(tqdm(chunk_paths, desc='Writing chunk ', unit=' chunks')): + shard_id = chunk_id % num_shards + metrics = __create_chunk(data_root, chunk_path, shard_id, compute_metrics=compute_metrics) + + if metrics is not None: + num_text += metrics[0] + shard_counts[chunk_id] = metrics[0] + + # write metadata + metadata_path = os.path.join(data_root, 'metadata.json') + with open(metadata_path, 'w') as f: + metadata = {'num_chunks': num_chunks, 'num_text': num_text, 'shard_count': shard_counts} + json.dump(metadata, f, indent=4) + logging.info("Metadata writen..") + + +def main(): + text_path = args.text_path + data_root = args.data_root + + if args.log: + logging.basicConfig(level=logging.INFO) + + tokenized_cachedir = os.path.join(data_root, '_tokenized_dataset_cachedir') + + if os.path.exists(tokenized_cachedir): + logging.warning( + f'Tokenized cache directory {tokenized_cachedir} already potentially contains files.' + f'In such a case, please be aware that the tarfiles will be **appended** instead of overridden!' + ) + + if not os.path.exists(data_root): + os.makedirs(data_root) + + chunk_paths = None + chunk_lens = None + + if os.path.exists(tokenized_cachedir): + paths = glob.glob(os.path.join(tokenized_cachedir, "*.npy")) + if len(paths) > 0: + logging.info("Cached tokenized numpy files found, skipping re-tokenization of dataset") + + chunk_paths = paths + chunk_lens = None + + if chunk_paths is None: + if args.tokenizer_name is None: + raise ValueError("`tokenizer_name` name is required when tokenizing the dataset for the first time.") + + if args.tokenizer_vocab_file is None: + raise ValueError("`tokenizer_vocab_file` is required when constructing the tokenized dataset") + + tokenizer = get_tokenizer( + tokenizer_name=args.tokenizer_name, + tokenizer_model=args.tokenizer_model, + vocab_file=args.tokenizer_vocab_file, + special_tokens=args.tokenizer_special_tokens, + ) + + logging.info("Built tokenizer") + + # tokenize text data into sub-words + chunk_paths, chunk_lens = __tokenize_text( + text_paths=text_path, + tokenizer=tokenizer, + tokenized_cachedir=tokenized_cachedir, + lower_case=args.lower_case, + chunk_size=args.chunk_size, + write_buffer=args.chunk_write_buffer, + ) + logging.info(f"Tokenized dataset into sub-words and serialized cache at {tokenized_cachedir}") + + # Write tarred dataset + __write_tarred_tokenized_text_dataset( + data_root, num_shards=args.num_shards, chunk_paths=chunk_paths, chunk_lens=chunk_lens + ) + + logging.info('Done preparing tokenized dataset!') + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/scripts/asr_language_modeling/neural_rescorer/eval_neural_rescorer.py b/models/tacotron2/NeMo/scripts/asr_language_modeling/neural_rescorer/eval_neural_rescorer.py new file mode 100644 index 0000000000..f713af6049 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/asr_language_modeling/neural_rescorer/eval_neural_rescorer.py @@ -0,0 +1,288 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script would evaluate a neural language model (Transformer) trained with +`examples/nlp/language_modeling/transformer_lm.py' as a rescorer for ASR systems. +Given a trained TransformerLMModel `.nemo` file, this script can be used to re-score the beams obtained from a beam +search decoder of an ASR model. + +USAGE: +1. Obtain `.tsv` file with beams and their corresponding scores. Scores can be from a regular beam search decoder or + in fusion with an N-gram LM scores. For a given beam size `beam_size` and a number of examples + for evaluation `num_eval_examples`, it should contain (`beam_size` x `num_eval_examples`) lines of + form `beam_candidate_text \t score`. This file can be generated by `scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py`. + +2. Rescore the candidates: + python eval_neural_rescorer.py + --lm_model=[path to .nemo file of the LM] + --beams_file=[path to beams .tsv file] + --beam_size=[size of the beams] + --eval_manifest=[path to eval manifest .json file] + --batch_size=[batch size used for inference on the LM model] + --alpha=[the value for the parameter rescorer_alpha] + --beta=[the value for the parameter rescorer_beta] + +You may find more info on how to use this script at: +https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/asr_language_modeling.html + +""" + +import contextlib +import json +from argparse import ArgumentParser + +import editdistance +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import torch +import tqdm + +from nemo.collections.nlp.models.language_modeling import TransformerLMModel +from nemo.utils import logging + + +class BeamScoresDataset(torch.utils.data.Dataset): + """ + Dataset to read the score file containing the beams and their score + + Args: + data_path: path to the beams file + tokenizer: tokenizer of the LM model + manifest_path: manifest `.json` file which contains the ground truths transcripts + beam_size: the number of beams per sample + max_seq_length: the maximum length of sequences + """ + + def __init__(self, data_path, tokenizer, manifest_path, beam_size=128, max_seq_length=256): + self.data = pd.read_csv(data_path, delimiter="\t", header=None) + self.tokenizer = tokenizer + self.ground_truths = [] + with open(manifest_path, 'r') as f_orig: + for line in f_orig: + item = json.loads(line) + self.ground_truths.append(item['text']) + self.beam_size = beam_size + self.max_seq_length = max_seq_length + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + text = str(self.data[0][idx]) + tokens = [self.tokenizer.bos_id] + self.tokenizer.text_to_ids(text) + [self.tokenizer.eos_id] + input_ids = [self.tokenizer.pad_id] * self.max_seq_length + input_ids[: len(tokens)] = tokens + input_ids = np.array(input_ids) + input_mask = (input_ids != self.tokenizer.pad_id).astype(np.float32) + acoustic_score = self.data[1][idx] + dist = editdistance.eval(text.split(), self.ground_truths[idx // self.beam_size].split()) + ref_len = len(self.ground_truths[idx // self.beam_size].split()) + len_in_chars = len(str(self.data[0][idx])) + return input_ids, input_mask, acoustic_score, dist, ref_len, len_in_chars, idx + + +def linear_search_wer( + dists, scores1, scores2, total_len, coef_range=[0, 10], coef_steps=10000, param_name='parameter' +): + """ + performs linear search to find the best coefficient when two set of scores are getting linearly fused. + + Args: + dists: Tesnor of the distances between the ground truth and the candidates with shape of [number of samples, beam size] + scores1: Tensor of the first set of scores with shape of [number of samples, beam size] + scores2: Tensor of the second set of scores with shape of [number of samples, beam size] + total_len: The total length of all samples + coef_range: the search range for the coefficient + coef_steps: the number of steps that the search range would get divided into + param_name: the name of the parameter to be used in the figure + + Output: + (best coefficient found, best WER achived) + """ + scale = scores1.mean().abs().item() / scores2.mean().abs().item() + left = coef_range[0] * scale + right = coef_range[1] * scale + coefs = np.linspace(left, right, coef_steps) + + best_wer = 10000 + best_coef = left + wers = [] + for coef in coefs: + scores = scores1 + coef * scores2 + wer = compute_wer(dists, scores, total_len) + wers.append(wer) + if wer < best_wer: + best_wer = wer + best_coef = coef + + plt.plot(coefs, wers) + plt.title(f'WER% after rescoring with different values of {param_name}') + plt.ylabel('WER%') + plt.xlabel(param_name) + plt.show() + return best_coef, best_wer + + +def compute_wer(dists, scores, total_len): + """ + Sorts the candidates based on the scores and calculates the WER with the new top candidates. + + Args: + dists: Tensor of the distances between the ground truth and the candidates with shape of [number of samples, beam size] + scores: Tensor of the scores for candidates with shape of [number of samples, beam size] + total_len: The total length of all samples + + Output: + WER with the new scores + """ + indices = scores.max(dim=1, keepdim=True)[1] + wer = dists.gather(dim=1, index=indices).sum() / total_len + wer = wer.item() + return wer + + +def main(): + parser = ArgumentParser() + parser.add_argument("--lm_model_file", type=str, required=True, help="path to LM model .nemo file") + parser.add_argument("--beams_file", type=str, required=True, help="path to beams .tsv file") + parser.add_argument( + "--eval_manifest", type=str, required=True, help="path to the evaluation `.json` manifest file" + ) + parser.add_argument("--beam_size", type=int, required=True, help="number of beams per candidate") + parser.add_argument("--batch_size", type=int, default=256, help="inference batch size") + parser.add_argument("--alpha", type=float, default=None, help="parameter alpha of the fusion") + parser.add_argument("--beta", type=float, default=None, help="parameter beta of the fusion") + parser.add_argument( + "--scores_output_file", default=None, type=str, help="The optional path to store the rescored beams" + ) + parser.add_argument( + "--device", default="cuda", type=str, help="The device to load the model onto to calculate the scores" + ) + parser.add_argument( + "--use_amp", action="store_true", help="Whether to use AMP if available to calculate the scores" + ) + args = parser.parse_args() + + device = args.device + if device.startswith("cuda") and not torch.cuda.is_available(): + logging.info(f"cuda is not available! switched to cpu.") + device = "cpu" + + if args.lm_model_file.endswith(".nemo"): + logging.info("Attempting to initialize from .nemo file") + model = TransformerLMModel.restore_from( + restore_path=args.lm_model_file, map_location=torch.device(device) + ).eval() + else: + raise NotImplementedError(f"Only supports .nemo files, but got: {args.model}") + + max_seq_length = model.encoder._embedding.position_embedding.pos_enc.shape[0] + dataset = BeamScoresDataset(args.beams_file, model.tokenizer, args.eval_manifest, args.beam_size, max_seq_length) + data_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=args.batch_size) + + if args.use_amp: + if torch.cuda.is_available() and hasattr(torch.cuda, 'amp') and hasattr(torch.cuda.amp, 'autocast'): + logging.info("AMP is enabled!\n") + autocast = torch.cuda.amp.autocast + else: + + @contextlib.contextmanager + def autocast(): + yield + + logging.info(f"Rescoring with beam_size: {args.beam_size}") + logging.info("Calculating the scores...") + with autocast(): + with torch.no_grad(): + am_scores, lm_scores, dists, ref_lens, lens_in_chars = [], [], [], [], [] + for batch in tqdm.tqdm(data_loader): + input_ids, input_mask, acoustic_score, dist, ref_len, len_in_chars, idx = batch + + max_len_in_batch = input_mask.sum(dim=0).argmin().item() + input_ids, input_mask = input_ids[:, :max_len_in_batch], input_mask[:, :max_len_in_batch] + if torch.cuda.is_available(): + input_ids, input_mask = input_ids.to(device), input_mask.to(device) + dist, acoustic_score, len_in_chars = ( + dist.to(device), + acoustic_score.to(device), + len_in_chars.to(device), + ) + + log_probs = model.forward(input_ids[:, :-1], input_mask[:, :-1]) + target_log_probs = log_probs.gather(2, input_ids[:, 1:].unsqueeze(2)).squeeze(2) + neural_lm_score = torch.sum(target_log_probs * input_mask[:, 1:], dim=-1) + + am_scores.append(acoustic_score) + lm_scores.append(neural_lm_score) + dists.append(dist) + ref_lens.append(ref_len) + lens_in_chars.append(len_in_chars) + + am_scores = torch.cat(am_scores).view(-1, args.beam_size) + lm_scores = torch.cat(lm_scores).view(-1, args.beam_size) + dists = torch.cat(dists).view(-1, args.beam_size) + ref_lens = torch.cat(ref_lens).view(-1, args.beam_size) + lens_in_chars = torch.cat(lens_in_chars).view(-1, args.beam_size).to(am_scores.dtype) + + total_len = ref_lens[:, 0].sum() + model_wer = dists[:, 0].sum() / total_len + ideal_wer = dists.min(dim=1)[0].sum() / total_len + + if args.alpha is None: + logging.info("Linear search for alpha...") + coef1, _ = linear_search_wer( + dists=dists, scores1=am_scores, scores2=lm_scores, total_len=total_len, param_name='alpha' + ) + coef1 = np.round(coef1, 3) + logging.info(f"alpha={coef1} achieved the best WER.") + logging.info(f"------------------------------------------------") + else: + coef1 = args.alpha + + scores = am_scores + coef1 * lm_scores + + if args.beta is None: + logging.info("Linear search for beta...") + coef2, _ = linear_search_wer( + dists=dists, scores1=scores, scores2=lens_in_chars, total_len=total_len, param_name='beta' + ) + coef2 = np.round(coef2, 3) + logging.info(f"beta={coef2} achieved the best WER.") + logging.info(f"------------------------------------------------") + else: + coef2 = args.beta + + new_scores = am_scores + coef1 * lm_scores + coef2 * lens_in_chars + rescored_wer = compute_wer(dists, new_scores, total_len) + + logging.info(f"Input beams WER: {np.round(model_wer.item() * 100, 2)}%") + logging.info(f"------------------------------------------------") + logging.info(f" +LM rescoring WER: {np.round(rescored_wer * 100, 2)}%") + logging.info(f" with alpha={coef1}, beta={coef2}") + logging.info(f"------------------------------------------------") + logging.info(f"Best possible WER: {np.round(ideal_wer.item() * 100, 2)}%") + logging.info(f"------------------------------------------------") + + new_scores_flatten = new_scores.flatten() + if args.scores_output_file is not None: + logging.info(f'Saving the candidates with their new scores at `{args.scores_output_file}`...') + with open(args.scores_output_file, "w") as fout: + for sample_id in range(len(dataset)): + fout.write(f"{dataset.data[0][sample_id]}\t{new_scores_flatten[sample_id]}\n") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py b/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py new file mode 100644 index 0000000000..6f5b224bef --- /dev/null +++ b/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py @@ -0,0 +1,345 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script would evaluate an N-gram language model trained with KenLM library (https://github.com/kpu/kenlm) in +# fusion with beam search decoders on top of a trained ASR model. NeMo's beam search decoders are capable of using the +# KenLM's N-gram models to find the best candidates. This script supports both character level and BPE level +# encodings and models which is detected automatically from the type of the model. +# You may train the LM model with 'scripts/ngram_lm/train_kenlm.py'. +# +# USAGE: python eval_beamsearch_ngram.py --nemo_model_file \ +# --input_manifest \ +# --beam_width \ +# --beam_alpha \ +# --beam_width \ +# --preds_output_folder \ +# --decoding_mode beam_search_ngram +# ... +# +# You may find more info on how to use this script at: +# https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/asr_language_modeling.html + + +# Please check train_kenlm.py to find out why we need TOKEN_OFFSET for BPE-based models +TOKEN_OFFSET = 100 + +import argparse +import contextlib +import json +import os +import pickle + +import editdistance +import kenlm_utils +import numpy as np +import torch +from sklearn.model_selection import ParameterGrid +from tqdm.auto import tqdm + +import nemo +import nemo.collections.asr as nemo_asr +from nemo.utils import logging + + +def beam_search_eval( + all_probs, + target_transcripts, + vocab, + ids_to_text_func=None, + preds_output_file=None, + lm_path=None, + beam_alpha=1.0, + beam_beta=0.0, + beam_width=128, + beam_batch_size=128, + progress_bar=True, +): + # creating the beam search decoder + beam_search_lm = nemo_asr.modules.BeamSearchDecoderWithLM( + vocab=vocab, + beam_width=beam_width, + alpha=beam_alpha, + beta=beam_beta, + lm_path=lm_path, + num_cpus=max(os.cpu_count(), 1), + input_tensor=False, + ) + + wer_dist_first = cer_dist_first = 0 + wer_dist_best = cer_dist_best = 0 + words_count = 0 + chars_count = 0 + sample_idx = 0 + if preds_output_file: + out_file = open(preds_output_file, 'w') + + if progress_bar: + it = tqdm( + range(int(np.ceil(len(all_probs) / beam_batch_size))), + desc=f"Beam search decoding with width={beam_width}, alpha={beam_alpha}, beta={beam_beta}", + ncols=120, + ) + else: + it = range(int(np.ceil(len(all_probs) / beam_batch_size))) + for batch_idx in it: + # disabling type checking + with nemo.core.typecheck.disable_checks(): + probs_batch = all_probs[batch_idx * beam_batch_size : (batch_idx + 1) * beam_batch_size] + beams_batch = beam_search_lm.forward(log_probs=probs_batch, log_probs_length=None,) + + for beams_idx, beams in enumerate(beams_batch): + target = target_transcripts[sample_idx + beams_idx] + target_split_w = target.split() + target_split_c = list(target) + words_count += len(target_split_w) + chars_count += len(target_split_c) + wer_dist_min = cer_dist_min = 10000 + for candidate_idx, candidate in enumerate(beams): + if ids_to_text_func is not None: + # For BPE encodings, need to shift by TOKEN_OFFSET to retrieve the original sub-word ids + pred_text = ids_to_text_func([ord(c) - TOKEN_OFFSET for c in candidate[1]]) + else: + pred_text = candidate[1] + pred_split_w = pred_text.split() + wer_dist = editdistance.eval(target_split_w, pred_split_w) + pred_split_c = list(pred_text) + cer_dist = editdistance.eval(target_split_c, pred_split_c) + + wer_dist_min = min(wer_dist_min, wer_dist) + cer_dist_min = min(cer_dist_min, cer_dist) + + if candidate_idx == 0: + # first candidate + wer_dist_first += wer_dist + cer_dist_first += cer_dist + + score = candidate[0] + if preds_output_file: + out_file.write('{}\t{}\n'.format(pred_text, score)) + wer_dist_best += wer_dist_min + cer_dist_best += cer_dist_min + sample_idx += len(probs_batch) + + if preds_output_file: + out_file.close() + logging.info(f"Stored the predictions of beam search decoding at '{preds_output_file}'.") + + if lm_path: + logging.info( + 'WER/CER with beam search decoding and N-gram model = {:.2%}/{:.2%}'.format( + wer_dist_first / words_count, cer_dist_first / chars_count + ) + ) + else: + logging.info( + 'WER/CER with beam search decoding = {:.2%}/{:.2%}'.format( + wer_dist_first / words_count, cer_dist_first / chars_count + ) + ) + logging.info( + 'Best WER/CER in candidates = {:.2%}/{:.2%}'.format(wer_dist_best / words_count, cer_dist_best / chars_count) + ) + logging.info(f"=================================================================================") + + +def main(): + parser = argparse.ArgumentParser( + description='Evaluate an ASR model with beam search decoding and n-gram KenLM language model.' + ) + parser.add_argument( + "--nemo_model_file", required=True, type=str, help="The path of the '.nemo' file of the ASR model" + ) + parser.add_argument( + "--kenlm_model_file", required=False, default=None, type=str, help="The path of the KenLM binary model file" + ) + parser.add_argument("--input_manifest", required=True, type=str, help="The manifest file of the evaluation set") + parser.add_argument( + "--preds_output_folder", default=None, type=str, help="The optional folder where the predictions are stored" + ) + parser.add_argument( + "--probs_cache_file", default=None, type=str, help="The cache file for storing the outputs of the model" + ) + parser.add_argument( + "--acoustic_batch_size", default=16, type=int, help="The batch size to calculate log probabilities" + ) + parser.add_argument( + "--device", default="cuda", type=str, help="The device to load the model onto to calculate log probabilities" + ) + parser.add_argument( + "--use_amp", action="store_true", help="Whether to use AMP if available to calculate log probabilities" + ) + parser.add_argument( + "--decoding_mode", + choices=["greedy", "beamsearch", "beamsearch_ngram"], + default="beamsearch_ngram", + type=str, + help="The decoding scheme to be used for evaluation.", + ) + parser.add_argument( + "--beam_width", + required=True, + type=int, + nargs="+", + help="The width or list of the widths for the beam search decoding", + ) + parser.add_argument( + "--beam_alpha", + required=True, + type=float, + nargs="+", + help="The alpha parameter or list of the alphas for the beam search decoding", + ) + parser.add_argument( + "--beam_beta", + required=True, + type=float, + nargs="+", + help="The beta parameter or list of the betas for the beam search decoding", + ) + parser.add_argument( + "--beam_batch_size", default=128, type=int, help="The batch size to be used for beam search decoding" + ) + args = parser.parse_args() + + asr_model = nemo_asr.models.EncDecCTCModelBPE.restore_from( + args.nemo_model_file, map_location=torch.device(args.device) + ) + + target_transcripts = [] + with open(args.input_manifest, 'r') as manifest_file: + audio_file_paths = [] + for line in tqdm(manifest_file, desc=f"Reading Manifest {args.input_manifest} ...", ncols=120): + data = json.loads(line) + target_transcripts.append(data['text']) + audio_file_paths.append(data['audio_filepath']) + + if args.probs_cache_file and os.path.exists(args.probs_cache_file): + logging.info(f"Found a pickle file of probabilities at '{args.probs_cache_file}'.") + logging.info(f"Loading the cached pickle file of probabilities from '{args.probs_cache_file}' ...") + with open(args.probs_cache_file, 'rb') as probs_file: + all_probs = pickle.load(probs_file) + + if len(all_probs) != len(audio_file_paths): + raise ValueError( + f"The number of samples in the probabilities file '{args.probs_cache_file}' does not " + f"match the manifest file. You may need to delete the probabilities cached file." + ) + else: + if args.use_amp: + if torch.cuda.is_available() and hasattr(torch.cuda, 'amp') and hasattr(torch.cuda.amp, 'autocast'): + logging.info("AMP is enabled!\n") + autocast = torch.cuda.amp.autocast + else: + + @contextlib.contextmanager + def autocast(): + yield + + with autocast(): + with torch.no_grad(): + all_logits = asr_model.transcribe(audio_file_paths, batch_size=args.acoustic_batch_size, logprobs=True) + all_probs = [kenlm_utils.softmax(logits) for logits in all_logits] + if args.probs_cache_file: + logging.info(f"Writing pickle files of probabilities at '{args.probs_cache_file}'...") + with open(args.probs_cache_file, 'wb') as f_dump: + pickle.dump(all_probs, f_dump) + + wer_dist_greedy = 0 + cer_dist_greedy = 0 + words_count = 0 + chars_count = 0 + for batch_idx, probs in enumerate(all_probs): + preds = np.argmax(probs, axis=1) + preds_tensor = torch.tensor(preds, device='cpu').unsqueeze(0) + pred_text = asr_model._wer.ctc_decoder_predictions_tensor(preds_tensor)[0] + + pred_split_w = pred_text.split() + target_split_w = target_transcripts[batch_idx].split() + pred_split_c = list(pred_text) + target_split_c = list(target_transcripts[batch_idx]) + + wer_dist = editdistance.eval(target_split_w, pred_split_w) + cer_dist = editdistance.eval(target_split_c, pred_split_c) + + wer_dist_greedy += wer_dist + cer_dist_greedy += cer_dist + words_count += len(target_split_w) + chars_count += len(target_split_c) + + logging.info('Greedy WER/CER = {:.2%}/{:.2%}'.format(wer_dist_greedy / words_count, cer_dist_greedy / chars_count)) + + encoding_level = kenlm_utils.SUPPORTED_MODELS.get(type(asr_model).__name__, None) + if not encoding_level: + logging.warning( + f"Model type '{type(asr_model).__name__}' may not be supported. Would try to train a char-level LM." + ) + encoding_level = 'char' + + vocab = asr_model.decoder.vocabulary + ids_to_text_func = None + if encoding_level == "subword": + vocab = [chr(idx + TOKEN_OFFSET) for idx in range(len(vocab))] + ids_to_text_func = asr_model.tokenizer.ids_to_text + # delete the model to free the memory + del asr_model + + if args.decoding_mode == "beamsearch_ngram": + if not os.path.exists(args.kenlm_model_file): + raise FileNotFoundError(f"Could not find the KenLM model file '{args.kenlm_model_file}'.") + lm_path = args.kenlm_model_file + else: + lm_path = None + + # 'greedy' decoding_mode would skip the beam search decoding + if args.decoding_mode in ["beamsearch_ngram", "beamsearch"]: + + params = {'beam_width': args.beam_width, 'beam_alpha': args.beam_alpha, 'beam_beta': args.beam_beta} + hp_grid = ParameterGrid(params) + hp_grid = list(hp_grid) + + logging.info(f"==============================Starting the beam search decoding===============================") + logging.info(f"Grid search size: {len(hp_grid)}") + logging.info(f"It may take some time...") + logging.info(f"==============================================================================================") + + if args.preds_output_folder and not os.path.exists(args.preds_output_folder): + os.mkdir(args.preds_output_folder) + for hp in hp_grid: + if args.preds_output_folder: + preds_output_file = os.path.join( + args.preds_output_folder, + f"preds_out_width{hp['beam_width']}_alpha{hp['beam_alpha']}_beta{hp['beam_beta']}.tsv", + ) + else: + preds_output_file = None + + beam_search_eval( + all_probs=all_probs, + target_transcripts=target_transcripts, + vocab=vocab, + ids_to_text_func=ids_to_text_func, + preds_output_file=preds_output_file, + lm_path=lm_path, + beam_width=hp["beam_width"], + beam_alpha=hp["beam_alpha"], + beam_beta=hp["beam_beta"], + beam_batch_size=args.beam_batch_size, + progress_bar=True, + ) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/install_beamsearch_decoders.sh b/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/install_beamsearch_decoders.sh new file mode 100644 index 0000000000..44f5fa82fa --- /dev/null +++ b/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/install_beamsearch_decoders.sh @@ -0,0 +1,6 @@ +git clone https://github.com/NVIDIA/OpenSeq2Seq -b ctc-decoders +mv OpenSeq2Seq/decoders . +rm -rf OpenSeq2Seq +cd decoders +./setup.sh +cd .. diff --git a/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/kenlm_utils.py b/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/kenlm_utils.py new file mode 100644 index 0000000000..ed967ced33 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/kenlm_utils.py @@ -0,0 +1,112 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Utility methods to be used for training N-gram LM with KenLM in train_kenlm.py +""" + +import json +import os + +import numpy as np +from joblib import Parallel, delayed +from tqdm.auto import tqdm + +# List of the supported models to be used with N-gram LM and beam search decoding +SUPPORTED_MODELS = { + 'EncDecCTCModelBPE': 'subword', + 'EncDecCTCModel': 'char', +} + + +def softmax(x): + e = np.exp(x - np.max(x)) + return e / e.sum(axis=-1).reshape([x.shape[0], 1]) + + +def read_train_file(path, lowercase: bool = False): + lines_read = 0 + text_dataset = [] + + with open(path, 'r') as f: + reader = tqdm(iter(lambda: f.readline(), ''), desc="Read 0 lines", unit=' lines') + for i, line in enumerate(reader): + if path.endswith('.json'): + line = json.loads(line)['text'] + + line = line.replace("\n", "").strip() + if lowercase: + line = line.lower() + + if line: + text_dataset.append(line) + + lines_read += 1 + if lines_read % 100000 == 0: + reader.set_description(f"Read {lines_read} lines") + + return text_dataset + + +def tokenize_str(texts, tokenizer, offset): + tokenized_text = [] + for text in texts: + tok_text = tokenizer.text_to_ids(text) + tok_text = [chr(token + offset) for token in tok_text] + tokenized_text.append(tok_text) + return tokenized_text + + +def tokenize_text(data, tokenizer, path, chunk_size=8192, buffer_size=32, token_offset=100): + dataset_len = len(data) + print( + f"Chunking {dataset_len} rows into {dataset_len / float(chunk_size):0.4f} tasks (each chunk contains {chunk_size} elements)" + ) + + current_step = 0 + if os.path.exists(path): + print(f"Deleting previous file : {path}") + os.remove(path) + + with Parallel(n_jobs=-2, verbose=10) as parallel: + while True: + start = current_step * chunk_size + end = min((current_step + buffer_size) * chunk_size, dataset_len) + + tokenized_data = parallel( + delayed(tokenize_str)(data[start : start + chunk_size], tokenizer, token_offset) + for start in range(start, end, chunk_size) + ) + + # Write dataset + write_dataset(tokenized_data, path) + current_step += len(tokenized_data) + print(f"Finished writing {len(tokenized_data)} chunks to {path}. Current chunk index = {current_step}") + del tokenized_data + if end >= dataset_len: + break + + +def write_dataset(chunks, path): + basedir = os.path.dirname(path) + + if not os.path.exists(basedir): + os.makedirs(basedir, exist_ok=True) + + with open(path, 'a+', encoding='utf-8') as f: + for chunk_idx in tqdm(range(len(chunks)), desc='Chunk ', total=len(chunks), unit=' chunks'): + for text in chunks[chunk_idx]: + line = ' '.join(text) + f.write(f"{line}\n") diff --git a/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/train_kenlm.py b/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/train_kenlm.py new file mode 100644 index 0000000000..a28ecfe4b2 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/asr_language_modeling/ngram_lm/train_kenlm.py @@ -0,0 +1,154 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +# This script would train an N-gram language model with KenLM library (https://github.com/kpu/kenlm) which can be used +# with the beam search decoders on top of the ASR models. This script supports both character level and BPE level +# encodings and models which is detected automatically from the type of the model. +# After the N-gram model is trained, and stored in the binary format, you may use +# 'scripts/ngram_lm/eval_beamsearch_ngram.py' to evaluate it on an ASR model. +# +# You need to install the KenLM library and also the beam search decoders to use this feature. Please refer +# to 'scripts/ngram_lm/install_beamsearch_decoders.sh' on how to install them. +# +# USAGE: python train_kenlm.py --nemo_model_file \ +# --train_file \ +# --kenlm_model_file \ +# --ngram_length +# +# After training is done, the binary LM model is stored at the path specified by '--kenlm_model_file'. +# You may find more info on how to use this script at: +# https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/asr_language_modeling.html + +import argparse +import logging +import os +import subprocess +import sys + +import kenlm_utils +import torch + +import nemo.collections.asr as nemo_asr +from nemo.utils import logging + +""" +NeMo's beam search decoders only support char-level encodings. In order to make it work with BPE-level encodings, we +use a trick to encode the sub-word tokens of the training data as unicode characters and train a char-level KenLM. +TOKEN_OFFSET is the offset in the unicode table to be used to encode the BPE sub-words. This encoding scheme reduces +the required memory significantly, and the LM and its binary blob format require less storage space. +""" +TOKEN_OFFSET = 100 + +CHUNK_SIZE = 8192 +CHUNK_BUFFER_SIZE = 512 + + +def main(): + parser = argparse.ArgumentParser( + description='Train an N-gram language model with KenLM to be used with beam search decoder of ASR models.' + ) + parser.add_argument( + "--train_file", + required=True, + type=str, + help="Path to the training file, it can be a text file or JSON manifest", + ) + parser.add_argument( + "--nemo_model_file", required=True, type=str, help="The path of the '.nemo' file of the ASR model" + ) + parser.add_argument( + "--kenlm_model_file", required=True, type=str, help="The path to store the KenLM binary model file" + ) + parser.add_argument("--ngram_length", required=True, type=int, help="The order of N-gram LM") + parser.add_argument("--kenlm_bin_path", required=True, type=str, help="The path to the bin folder of KenLM") + parser.add_argument( + "--do_lowercase", action='store_true', help="Whether to apply lower case conversion on the training text" + ) + args = parser.parse_args() + + """ TOKENIZER SETUP """ + logging.info(f"Loading nemo model '{args.nemo_model_file}' ...") + model = nemo_asr.models.ASRModel.restore_from(args.nemo_model_file, map_location=torch.device('cpu')) + + encoding_level = kenlm_utils.SUPPORTED_MODELS.get(type(model).__name__, None) + if not encoding_level: + logging.warning( + f"Model type '{type(model).__name__}' may not be supported. Would try to train a char-level LM." + ) + encoding_level = 'char' + + """ DATASET SETUP """ + logging.info(f"Encoding the train file '{args.train_file}' ...") + dataset = kenlm_utils.read_train_file(args.train_file, lowercase=args.do_lowercase) + encoded_train_file = f"{args.kenlm_model_file}.tmp.txt" + if encoding_level == "subword": + kenlm_utils.tokenize_text( + dataset, + model.tokenizer, + path=encoded_train_file, + chunk_size=CHUNK_SIZE, + buffer_size=CHUNK_BUFFER_SIZE, + token_offset=TOKEN_OFFSET, + ) + # --discount_fallback is needed for training KenLM for BPE-based models + discount_arg = "--discount_fallback" + else: + with open(encoded_train_file, 'w', encoding='utf-8') as f: + for line in dataset: + f.write(f"{line}\n") + + discount_arg = "" + + del model + + arpa_file = f"{args.kenlm_model_file}.tmp.arpa" + """ LMPLZ ARGUMENT SETUP """ + kenlm_args = [ + os.path.join(args.kenlm_bin_path, 'lmplz'), + "-o", + f"{args.ngram_length}", + "--text", + encoded_train_file, + "--arpa", + arpa_file, + discount_arg, + ] + + ret = subprocess.run(kenlm_args, capture_output=False, text=True, stdout=sys.stdout, stderr=sys.stderr) + if ret.returncode != 0: + raise RuntimeError("Training KenLM was not successful!") + """ BINARY BUILD """ + logging.info(f"Running binary_build command \n\n{' '.join(kenlm_args)}\n\n") + kenlm_args = [ + os.path.join(args.kenlm_bin_path, "build_binary"), + "trie", + arpa_file, + args.kenlm_model_file, + ] + ret = subprocess.run(kenlm_args, capture_output=False, text=True, stdout=sys.stdout, stderr=sys.stderr) + + if ret.returncode != 0: + raise RuntimeError("Training KenLM was not successful!") + + os.remove(encoded_train_file) + logging.info(f"Deleted the temporary encoded training file '{encoded_train_file}'.") + os.remove(arpa_file) + logging.info(f"Deleted the arpa file '{arpa_file}'.") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/scripts/average_model_checkpoints.py b/models/tacotron2/NeMo/scripts/average_model_checkpoints.py new file mode 100644 index 0000000000..06c522f1e1 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/average_model_checkpoints.py @@ -0,0 +1,171 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2017 Johns Hopkins University (Shinji Watanabe) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +# Changes to script +Change the script to import the NeMo model class you would like to load a checkpoint for, +then update the model constructor to use this model class. This can be found by the line: + +<<< Change model class here ! >>> + +By default, this script imports and creates the `EncDecCTCModelBPE` class but it can be +changed to any NeMo Model. + +# Run the script + +## Saving a .nemo model file (loaded with ModelPT.restore_from(...)) +HYDRA_FULL_ERROR=1 python average_model_checkpoints.py \ + --config-path="" \ + --config-name="" \ + name= \ + +checkpoint_dir= \ + +checkpoint_paths=\"[/path/to/ptl_1.ckpt,/path/to/ptl_2.ckpt,/path/to/ptl_3.ckpt,...]\" + + +## Saving an averaged pytorch checkpoint (loaded with torch.load(...)) +HYDRA_FULL_ERROR=1 python average_model_checkpoints.py \ + --config-path="" \ + --config-name="" \ + name= \ + +checkpoint_dir= \ + +checkpoint_paths=\"[/path/to/ptl_1.ckpt,/path/to/ptl_2.ckpt,/path/to/ptl_3.ckpt,...]\" \ + +save_ckpt_only=true + +""" + +import os + +import pytorch_lightning as pl +import torch +from omegaconf import OmegaConf, open_dict + +# Change this import to the model you would like to average +from nemo.collections.asr.models import EncDecCTCModelBPE +from nemo.core.config import hydra_runner +from nemo.utils import logging + + +def process_config(cfg: OmegaConf): + if 'name' not in cfg or cfg.name is None: + raise ValueError("`cfg.name` must be provided to save a model checkpoint") + + if 'checkpoint_paths' not in cfg or cfg.checkpoint_paths is None: + raise ValueError( + "`cfg.checkpoint_paths` must be provided as a list of one or more str paths to " + "pytorch lightning checkpoints" + ) + + save_ckpt_only = False + + with open_dict(cfg): + name_prefix = cfg.name + checkpoint_paths = cfg.pop('checkpoint_paths') + + if 'checkpoint_dir' in cfg: + checkpoint_dir = cfg.pop('checkpoint_dir') + else: + checkpoint_dir = None + + if 'save_ckpt_only' in cfg: + save_ckpt_only = cfg.pop('save_ckpt_only') + + if type(checkpoint_paths) not in (list, tuple): + checkpoint_paths = str(checkpoint_paths).replace("[", "").replace("]", "") + checkpoint_paths = checkpoint_paths.split(",") + checkpoint_paths = [ckpt_path.strip() for ckpt_path in checkpoint_paths] + + if checkpoint_dir is not None: + checkpoint_paths = [os.path.join(checkpoint_dir, path) for path in checkpoint_paths] + + return name_prefix, checkpoint_paths, save_ckpt_only + + +@hydra_runner(config_path=None, config_name=None) +def main(cfg): + name_prefix, checkpoint_paths, save_ckpt_only = process_config(cfg) + + if not save_ckpt_only: + trainer = pl.Trainer(**cfg.trainer) + + # <<< Change model class here ! >>> + # Model architecture which will contain the averaged checkpoints + # Change the model constructor to the one you would like (if needed) + model = EncDecCTCModelBPE(cfg=cfg.model, trainer=trainer) + + """ < Checkpoint Averaging Logic > """ + # load state dicts + n = len(checkpoint_paths) + avg_state = None + + logging.info(f"Averaging {n} checkpoints ...") + + for ix, path in enumerate(checkpoint_paths): + checkpoint = torch.load(path, map_location='cpu') + + if 'state_dict' in checkpoint: + checkpoint = checkpoint['state_dict'] + + if ix == 0: + # Initial state + avg_state = checkpoint + + logging.info(f"Initialized average state dict with checkpoint : {path}") + else: + # Accumulated state + for k in avg_state: + avg_state[k] = avg_state[k] + checkpoint[k] + + logging.info(f"Updated average state dict with state from checkpoint : {path}") + + for k in avg_state: + if str(avg_state[k].dtype).startswith("torch.int"): + # For int type, not averaged, but only accumulated. + # e.g. BatchNorm.num_batches_tracked + pass + else: + avg_state[k] = avg_state[k] / n + + # Save model + if save_ckpt_only: + ckpt_name = name_prefix + '-averaged.ckpt' + torch.save(avg_state, ckpt_name) + + logging.info(f"Averaged pytorch checkpoint saved as : {ckpt_name}") + else: + # Set model state + logging.info("Loading averaged state dict in provided model") + model.load_state_dict(avg_state, strict=True) + + ckpt_name = name_prefix + '-averaged.nemo' + model.save_to(ckpt_name) + + logging.info(f"Averaged model saved as : {ckpt_name}") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/get_commonvoice_data.py b/models/tacotron2/NeMo/scripts/dataset_processing/get_commonvoice_data.py new file mode 100644 index 0000000000..b53415603b --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/get_commonvoice_data.py @@ -0,0 +1,178 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Copyright (c) 2020, SeanNaren. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# To convert mp3 files to wav using sox, you must have installed sox with mp3 support +# For example sudo apt-get install libsox-fmt-mp3 +import argparse +import csv +import json +import logging +import multiprocessing +import os +import subprocess +import sys +import tarfile +from multiprocessing.pool import ThreadPool +from pathlib import Path +from typing import List + +import sox +from sox import Transformer +from tqdm import tqdm + +parser = argparse.ArgumentParser(description='Downloads and processes Mozilla Common Voice dataset.') +parser.add_argument("--data_root", default='CommonVoice_dataset/', type=str, help="Directory to store the dataset.") +parser.add_argument('--manifest_dir', default='./', type=str, help='Output directory for manifests') +parser.add_argument("--num_workers", default=multiprocessing.cpu_count(), type=int, help="Workers to process dataset.") +parser.add_argument('--sample_rate', default=16000, type=int, help='Sample rate') +parser.add_argument('--n_channels', default=1, type=int, help='Number of channels for output wav files') +parser.add_argument( + '--files_to_process', + nargs='+', + default=['test.tsv', 'dev.tsv', 'train.tsv'], + type=str, + help='list of *.csv file names to process', +) +parser.add_argument( + '--version', + default='cv-corpus-5.1-2020-06-22', + type=str, + help='Version of the dataset (obtainable via https://commonvoice.mozilla.org/en/datasets', +) +parser.add_argument( + '--language', + default='en', + type=str, + help='Which language to download.(default english,' + 'check https://commonvoice.mozilla.org/en/datasets for more language codes', +) +args = parser.parse_args() +COMMON_VOICE_URL = ( + f"https://voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com/" + "{}/{}.tar.gz".format(args.version, args.language) +) + + +def create_manifest(data: List[tuple], output_name: str, manifest_path: str): + output_file = Path(manifest_path) / output_name + output_file.parent.mkdir(exist_ok=True, parents=True) + + with output_file.open(mode='w') as f: + for wav_path, duration, text in tqdm(data, total=len(data)): + f.write( + json.dumps({'audio_filepath': os.path.abspath(wav_path), "duration": duration, 'text': text}) + '\n' + ) + + +def process_files(csv_file, data_root, num_workers): + """ Read *.csv file description, convert mp3 to wav, process text. + Save results to data_root. + + Args: + csv_file: str, path to *.csv file with data description, usually start from 'cv-' + data_root: str, path to dir to save results; wav/ dir will be created + """ + wav_dir = os.path.join(data_root, 'wav/') + os.makedirs(wav_dir, exist_ok=True) + audio_clips_path = os.path.dirname(csv_file) + '/clips/' + + def process(x): + file_path, text = x + file_name = os.path.splitext(os.path.basename(file_path))[0] + text = text.lower().strip() + audio_path = os.path.join(audio_clips_path, file_path) + output_wav_path = os.path.join(wav_dir, file_name + '.wav') + + tfm = Transformer() + tfm.rate(samplerate=args.sample_rate) + tfm.channels(n_channels=args.n_channels) + tfm.build(input_filepath=audio_path, output_filepath=output_wav_path) + duration = sox.file_info.duration(output_wav_path) + return output_wav_path, duration, text + + logging.info('Converting mp3 to wav for {}.'.format(csv_file)) + with open(csv_file) as csvfile: + reader = csv.DictReader(csvfile, delimiter='\t') + next(reader, None) # skip the headers + data = [(row['path'], row['sentence']) for row in reader] + with ThreadPool(num_workers) as pool: + data = list(tqdm(pool.imap(process, data), total=len(data))) + return data + + +def main(): + data_root = args.data_root + os.makedirs(data_root, exist_ok=True) + + target_unpacked_dir = os.path.join(data_root, "CV_unpacked") + + if os.path.exists(target_unpacked_dir): + logging.info('Find existing folder {}'.format(target_unpacked_dir)) + else: + logging.info("Could not find Common Voice, Downloading corpus...") + + commands = [ + 'wget', + '--user-agent', + '"Mozilla/5.0 (Windows NT 10.0; WOW64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"', + '-P', + data_root, + f'{COMMON_VOICE_URL}', + ] + commands = " ".join(commands) + subprocess.run(commands, shell=True, stderr=sys.stderr, stdout=sys.stdout, capture_output=False) + filename = f"{args.language}.tar.gz" + target_file = os.path.join(data_root, os.path.basename(filename)) + + os.makedirs(target_unpacked_dir, exist_ok=True) + logging.info("Unpacking corpus to {} ...".format(target_unpacked_dir)) + tar = tarfile.open(target_file) + tar.extractall(target_unpacked_dir) + tar.close() + + folder_path = os.path.join(target_unpacked_dir, args.version + f'/{args.language}/') + + for csv_file in args.files_to_process: + data = process_files( + csv_file=os.path.join(folder_path, csv_file), + data_root=os.path.join(data_root, os.path.splitext(csv_file)[0]), + num_workers=args.num_workers, + ) + logging.info('Creating manifests...') + create_manifest( + data=data, + output_name=f'commonvoice_{os.path.splitext(csv_file)[0]}_manifest.json', + manifest_path=args.manifest_dir, + ) + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/get_hi-mia_data.py b/models/tacotron2/NeMo/scripts/dataset_processing/get_hi-mia_data.py new file mode 100644 index 0000000000..59dd629c04 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/get_hi-mia_data.py @@ -0,0 +1,185 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# USAGE: python get_aishell_data.py --data_root= + +import argparse +import json +import logging as _logging +import os +import tarfile +import urllib.request +from glob import glob + +import librosa as l +from sklearn.model_selection import StratifiedShuffleSplit +from tqdm import tqdm + +parser = argparse.ArgumentParser(description='HI-MIA Data download') +parser.add_argument("--data_root", required=True, default=None, type=str) +parser.add_argument("--log_level", default=20, type=int) +args = parser.parse_args() +logging = _logging.getLogger(__name__) +logging.addHandler(_logging.StreamHandler()) +logging.setLevel(args.log_level) + +URL = { + 'dev': "http://www.openslr.org/resources/85/dev.tar.gz", + 'test': "http://www.openslr.org/resources/85/test.tar.gz", + 'train': "http://www.openslr.org/resources/85/train.tar.gz", +} + + +def __maybe_download_file(destination: str, source: str): + """ + Downloads source to destination if it doesn't exist. + If exists, skips download + Args: + destination: local filepath + source: url of resource + + Returns: + + """ + source = URL[source] + if not os.path.exists(destination) and not os.path.exists(os.path.splitext(destination)[0]): + logging.info("{0} does not exist. Downloading ...".format(destination)) + urllib.request.urlretrieve(source, filename=destination + '.tmp') + os.rename(destination + '.tmp', destination) + logging.info("Downloaded {0}.".format(destination)) + elif os.path.exists(destination): + logging.info("Destination {0} exists. Skipping.".format(destination)) + elif os.path.exists(os.path.splitext(destination)[0]): + logging.warning( + "Assuming extracted folder %s contains the extracted files from %s. Will not download.", + os.path.basename(destination), + destination, + ) + return destination + + +def __extract_all_files(filepath: str, data_root: str, data_dir: str): + if not os.path.exists(data_dir): + extract_file(filepath, data_root) + audio_dir = os.path.join(data_dir, 'wav') + for subfolder, _, filelist in os.walk(audio_dir): + for ftar in filelist: + extract_file(os.path.join(subfolder, ftar), subfolder) + else: + logging.info('Skipping extracting. Data already there %s' % data_dir) + + +def extract_file(filepath: str, data_dir: str): + try: + tar = tarfile.open(filepath) + tar.extractall(data_dir) + tar.close() + except Exception: + logging.info('Not extracting. Maybe already there?') + + +def __remove_tarred_files(filepath: str, data_dir: str): + if os.path.exists(data_dir) and os.path.isfile(filepath): + logging.info('Deleting %s' % filepath) + os.remove(filepath) + + +def write_file(name, lines, idx): + with open(name, 'w') as fout: + for i in idx: + dic = lines[i] + json.dump(dic, fout) + fout.write('\n') + logging.info("wrote %s", name) + + +def __process_data(data_folder: str, data_set: str): + """ + To generate manifest + Args: + data_folder: source with wav files + Returns: + + """ + fullpath = os.path.abspath(data_folder) + scp = glob(fullpath + '/**/*.wav', recursive=True) + out = os.path.join(fullpath, data_set + '_all.json') + utt2spk = os.path.join(fullpath, 'utt2spk') + utt2spk_file = open(utt2spk, 'w') + id = -2 # speaker id + + if os.path.exists(out): + logging.warning( + "%s already exists and is assumed to be processed. If not, please delete %s and rerun this script", + out, + out, + ) + return + + speakers = [] + lines = [] + with open(out, 'w') as outfile: + for line in tqdm(scp): + line = line.strip() + y, sr = l.load(line, sr=None) + if sr != 16000: + y, sr = l.load(line, sr=16000) + l.output.write_wav(line, y, sr) + dur = l.get_duration(y=y, sr=sr) + if data_set == 'test': + speaker = line.split('/')[-1].split('.')[0].split('_')[0] + else: + speaker = line.split('/')[id] + speaker = list(speaker) + speaker = ''.join(speaker) + speakers.append(speaker) + meta = {"audio_filepath": line, "duration": float(dur), "label": speaker} + lines.append(meta) + json.dump(meta, outfile) + outfile.write("\n") + utt2spk_file.write(line.split('/')[-1] + "\t" + speaker + "\n") + + utt2spk_file.close() + + if data_set != 'test': + sss = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=42) + for train_idx, test_idx in sss.split(speakers, speakers): + print(len(train_idx)) + + out = os.path.join(fullpath, 'train.json') + write_file(out, lines, train_idx) + out = os.path.join(fullpath, 'dev.json') + write_file(out, lines, test_idx) + + +def main(): + data_root = args.data_root + for data_set in URL.keys(): + + # data_set = 'data_aishell' + logging.info("\n\nWorking on: {0}".format(data_set)) + file_path = os.path.join(data_root, data_set + ".tgz") + logging.info("Getting {0}".format(data_set)) + __maybe_download_file(file_path, data_set) + logging.info("Extracting {0}".format(data_set)) + data_folder = os.path.join(data_root, data_set) + __extract_all_files(file_path, data_root, data_folder) + __remove_tarred_files(file_path, data_folder) + logging.info("Processing {0}".format(data_set)) + __process_data(data_folder, data_set) + logging.info('Done!') + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/get_librispeech_data.py b/models/tacotron2/NeMo/scripts/dataset_processing/get_librispeech_data.py new file mode 100644 index 0000000000..e009a2322f --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/get_librispeech_data.py @@ -0,0 +1,171 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# USAGE: python get_librispeech_data.py --data_root= +# --data_set= --num_workers= +# where can be: dev_clean, dev_other, test_clean, +# test_other, train_clean_100, train_clean_360, train_other_500 or ALL +# You can also put more than one data_set comma-separated: +# --data_set=dev_clean,train_clean_100 +import argparse +import fnmatch +import functools +import json +import logging +import multiprocessing +import os +import subprocess +import tarfile +import urllib.request + +from sox import Transformer +from tqdm import tqdm + +parser = argparse.ArgumentParser(description='LibriSpeech Data download') +parser.add_argument("--data_root", required=True, default=None, type=str) +parser.add_argument("--data_sets", default="dev_clean", type=str) +parser.add_argument("--num_workers", default=4, type=int) +args = parser.parse_args() + +URLS = { + 'TRAIN_CLEAN_100': ("http://www.openslr.org/resources/12/train-clean-100.tar.gz"), + 'TRAIN_CLEAN_360': ("http://www.openslr.org/resources/12/train-clean-360.tar.gz"), + 'TRAIN_OTHER_500': ("http://www.openslr.org/resources/12/train-other-500.tar.gz"), + 'DEV_CLEAN': "http://www.openslr.org/resources/12/dev-clean.tar.gz", + 'DEV_OTHER': "http://www.openslr.org/resources/12/dev-other.tar.gz", + 'TEST_CLEAN': "http://www.openslr.org/resources/12/test-clean.tar.gz", + 'TEST_OTHER': "http://www.openslr.org/resources/12/test-other.tar.gz", +} + + +def __maybe_download_file(destination: str, source: str): + """ + Downloads source to destination if it doesn't exist. + If exists, skips download + Args: + destination: local filepath + source: url of resource + Returns: + """ + source = URLS[source] + if not os.path.exists(destination): + logging.info("{0} does not exist. Downloading ...".format(destination)) + urllib.request.urlretrieve(source, filename=destination + '.tmp') + os.rename(destination + '.tmp', destination) + logging.info("Downloaded {0}.".format(destination)) + else: + logging.info("Destination {0} exists. Skipping.".format(destination)) + return destination + + +def __extract_file(filepath: str, data_dir: str): + try: + tar = tarfile.open(filepath) + tar.extractall(data_dir) + tar.close() + except Exception: + logging.info('Not extracting. Maybe already there?') + + +def __process_transcript(file_path: str, dst_folder: str): + """ + Converts flac files to wav from a given transcript, capturing the metadata. + Args: + file_path: path to a source transcript with flac sources + dst_folder: path where wav files will be stored + Returns: + a list of metadata entries for processed files. + """ + entries = [] + root = os.path.dirname(file_path) + with open(file_path, encoding="utf-8") as fin: + for line in fin: + id, text = line[: line.index(" ")], line[line.index(" ") + 1 :] + transcript_text = text.lower().strip() + + # Convert FLAC file to WAV + flac_file = os.path.join(root, id + ".flac") + wav_file = os.path.join(dst_folder, id + ".wav") + if not os.path.exists(wav_file): + Transformer().build(flac_file, wav_file) + # check duration + duration = subprocess.check_output("soxi -D {0}".format(wav_file), shell=True) + + entry = {} + entry['audio_filepath'] = os.path.abspath(wav_file) + entry['duration'] = float(duration) + entry['text'] = transcript_text + entries.append(entry) + return entries + + +def __process_data(data_folder: str, dst_folder: str, manifest_file: str, num_workers: int): + """ + Converts flac to wav and build manifests's json + Args: + data_folder: source with flac files + dst_folder: where wav files will be stored + manifest_file: where to store manifest + num_workers: number of parallel workers processing files + Returns: + """ + + if not os.path.exists(dst_folder): + os.makedirs(dst_folder) + + files = [] + entries = [] + + for root, dirnames, filenames in os.walk(data_folder): + for filename in fnmatch.filter(filenames, '*.trans.txt'): + files.append(os.path.join(root, filename)) + + with multiprocessing.Pool(num_workers) as p: + processing_func = functools.partial(__process_transcript, dst_folder=dst_folder) + results = p.imap(processing_func, files) + for result in tqdm(results, total=len(files)): + entries.extend(result) + + with open(manifest_file, 'w') as fout: + for m in entries: + fout.write(json.dumps(m) + '\n') + + +def main(): + data_root = args.data_root + data_sets = args.data_sets + num_workers = args.num_workers + + if data_sets == "ALL": + data_sets = "dev_clean,dev_other,train_clean_100,train_clean_360,train_other_500,test_clean,test_other" + + for data_set in data_sets.split(','): + logging.info("\n\nWorking on: {0}".format(data_set)) + filepath = os.path.join(data_root, data_set + ".tar.gz") + logging.info("Getting {0}".format(data_set)) + __maybe_download_file(filepath, data_set.upper()) + logging.info("Extracting {0}".format(data_set)) + __extract_file(filepath, data_root) + logging.info("Processing {0}".format(data_set)) + __process_data( + os.path.join(os.path.join(data_root, "LibriSpeech"), data_set.replace("_", "-"),), + os.path.join(os.path.join(data_root, "LibriSpeech"), data_set.replace("_", "-"),) + "-processed", + os.path.join(data_root, data_set + ".json"), + num_workers=num_workers, + ) + logging.info('Done!') + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/get_openslr_rir_data.py b/models/tacotron2/NeMo/scripts/dataset_processing/get_openslr_rir_data.py new file mode 100644 index 0000000000..f1663061ba --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/get_openslr_rir_data.py @@ -0,0 +1,140 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# USAGE: python get_openslr_rir_data.py --data_root= +# Data is downloaded from OpenSLR's "Room Impulse Response and Noise Database" +# RIRs in multichannel files are separated into single channel files and +# a json file that can be used as in input to NeMo is created + +import argparse +import glob +import json +import logging +import os +import subprocess +import urllib.request +from shutil import copy, move +from zipfile import ZipFile + +parser = argparse.ArgumentParser(description='OpenSLR RIR Data download and process') +parser.add_argument("--data_root", required=True, default=None, type=str) +args = parser.parse_args() + +URLS = { + 'SLR28': ("http://www.openslr.org/resources/28/rirs_noises.zip"), +} + + +def __maybe_download_file(destination: str, source: str): + """ + Downloads source to destination if it doesn't exist. + If exists, skips download + Args: + destination: local filepath + source: url of resource + Returns: + """ + source = URLS[source] + if not os.path.exists(destination): + logging.info("{0} does not exist. Downloading ...".format(destination)) + urllib.request.urlretrieve(source, filename=destination + '.tmp') + os.rename(destination + '.tmp', destination) + logging.info("Downloaded {0}.".format(destination)) + else: + logging.info("Destination {0} exists. Skipping.".format(destination)) + return destination + + +def __extract_file(filepath: str, data_dir: str): + try: + with ZipFile(filepath, 'r') as zipObj: + zipObj.extractall(data_dir) + except Exception: + logging.info('Not extracting. Maybe already there?') + + +def __process_data(data_folder: str, dst_folder: str, manifest_file: str): + """ + Converts flac to wav and build manifests's json + Args: + data_folder: source with flac files + dst_folder: where wav files will be stored + manifest_file: where to store manifest + Returns: + """ + if not os.path.exists(dst_folder): + os.makedirs(dst_folder) + + real_rir_list = os.path.join(data_folder, "RIRS_NOISES", "real_rirs_isotropic_noises", "rir_list") + rirfiles = [] + with open(real_rir_list, "r") as rir_f: + for line in rir_f: + rirfiles.append(os.path.join(data_folder, line.rstrip().split(" ")[4])) + + real_rir_folder = os.path.join(dst_folder, "real_rirs") + if not os.path.exists(real_rir_folder): + os.makedirs(real_rir_folder) + # split multi-channel rir files to single channel + for rir_f in rirfiles: + n_chans = int(subprocess.check_output("soxi -c {0}".format(rir_f), shell=True)) + if n_chans == 1: + copy(rir_f, real_rir_folder) + else: + for chan in range(1, n_chans + 1): + chan_file_name = os.path.join( + real_rir_folder, os.path.splitext(os.path.basename(rir_f))[0] + "-" + str(chan) + ".wav" + ) + _ = subprocess.check_output(f"sox {rir_f} {chan_file_name} remix {chan}", shell=True) + + # move simulated rirs to processed + if not os.path.exists(os.path.join(dst_folder, "simulated_rirs")): + move(os.path.join(data_folder, "RIRS_NOISES", "simulated_rirs"), dst_folder) + + os.chdir(dst_folder) + all_rirs = glob.glob("**/*.wav", recursive=True) + with open(manifest_file, 'w') as man_f: + entry = {} + for rir in all_rirs: + rir_file = os.path.join(dst_folder, rir) + duration = subprocess.check_output("soxi -D {0}".format(rir_file), shell=True) + entry['audio_filepath'] = rir_file + entry['duration'] = float(duration) + entry['offset'] = 0 + entry['text'] = '_' + man_f.write(json.dumps(entry) + "\n") + + print('Done!') + + +def main(): + data_root = os.path.abspath(args.data_root) + data_set = "slr28" + logging.getLogger().setLevel(logging.INFO) + logging.info("\n\nWorking on: {0}".format(data_set)) + filepath = os.path.join(data_root, data_set + ".zip") + logging.info("Getting {0}".format(data_set)) + __maybe_download_file(filepath, data_set.upper()) + logging.info("Extracting {0}".format(data_set)) + __extract_file(filepath, data_root) + logging.info("Processing {0}".format(data_set)) + __process_data( + data_root, + os.path.join(os.path.join(data_root, "processed")), + os.path.join(os.path.join(data_root, "processed", "rir.json")), + ) + logging.info('Done!') + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/README b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/README new file mode 100644 index 0000000000..d16fdac55a --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/README @@ -0,0 +1,14 @@ +These scripts are used for extracting features from LJSpeech. + +You will need to run the following scripts (note that these will call the other +scripts as necessary): + +- `create_manifests_and_textfiles.py`: Creates the manifest and text files that + are used by the Montreal Forced Aligner (MFA) library. +- `extract_ljspeech_phonemes_and_durs.sh`: Extracts phonemes and alignments via + MFA and calculates phoneme durations. + Needs to be run with Conda active since it creates a Conda environment. + Also note that you may need to install OpenBlas manually (e.g. sudo apt-get + install libopenblas-dev), sometimes MFA can't find it if installed via Conda. + (This script calls `create_token2idx_dict.py` and `calculate_durs.py`.) +- `extract_ljspeech_energy_pitch.py` diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/calculate_durs.py b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/calculate_durs.py new file mode 100644 index 0000000000..f7a37c2a2e --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/calculate_durs.py @@ -0,0 +1,126 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Calculates durations for LJSpeech based on MFA TextGrid alignments. +""" +import argparse +import glob +import json +import os +import pickle +from math import ceil + +import numpy as np +import tgt +import torch +from tqdm import tqdm + +parser = argparse.ArgumentParser(description="Calculates phoneme durations for LJSpeech from TextGrids.") +parser.add_argument('--ljspeech_dir', required=True, default=None, type=str) +parser.add_argument( + '--mappings', + required=False, + default=None, + type=str, + help='JSON file of mappings created with create_token2idx_dict.py', +) +parser.add_argument('--sr', required=False, default=22050, type=int) +parser.add_argument('--hop_length', required=False, default=256, type=int) +args = parser.parse_args() + + +def calculate_durations(textgrid, phone2idx): + tokens = [] + durs = [] + + frames_per_second = args.sr / args.hop_length + tg = tgt.read_textgrid(textgrid, include_empty_intervals=True) + data_tier = tg.get_tier_by_name("phones") + + # Get total frames + total_frames = ceil((data_tier.end_time - data_tier.start_time) * frames_per_second) + + # Find start and end frames of each token + se_in_frames = np.array([(frames_per_second * d.start_time, frames_per_second * d.end_time) for d in data_tier]) + se_in_frames = np.round(se_in_frames) + durs = (se_in_frames[:, 1] - se_in_frames[:, 0]).astype(int) + blank_set = ('sil', 'sp', 'spn', '', '') + blank_token = " " + + # merge repeated blank tokens + tokens, durations = [], [] + for i in range(len(data_tier)): + x = data_tier[i].text + if x == 'spn': + return None, None, None + x = blank_token if x in blank_set else x + + if len(tokens) and tokens[-1] == blank_token and x == blank_token: + durations[-1] += durs[i] + else: + tokens.append(x) + durations.append(durs[i]) + + tokens_enc = [phone2idx[token] for token in tokens] + tokens_enc, durations = torch.LongTensor(tokens_enc), torch.LongTensor(durations) + + # Add rounding error to final token + durations[-1] += total_frames - durations.sum() + + return tokens, tokens_enc, durations + + +def main(): + textgrid_list = glob.glob(os.path.join(args.ljspeech_dir, 'alignments/wavs/*.TextGrid')) + + # Create target_dir if necessary + target_dir = os.path.join(args.ljspeech_dir, 'phoneme_durations/') + print(f"Calculating phoneme durations, files will be in: {target_dir}") + + if not os.path.exists(target_dir): + print(f"Creating {target_dir}") + os.mkdir(target_dir) + + # Read phoneme to idx mappings + phone2idx = None + if args.mappings: + with open(args.mappings, 'r') as f: + mappings = json.load(f) + phone2idx = mappings['phone2idx'] + + oov_samples = [] + + # Iterate through all TextGrid files + for textgrid in tqdm(textgrid_list): + basename = os.path.splitext(os.path.basename(textgrid))[0][5:] # Chop off 'wavs_' prefix + + phones_mfa, tokens_mfa, durs = calculate_durations(textgrid, phone2idx) + + if phones_mfa is None: + oov_samples.append(basename) + continue + + # Save to file + target_path = os.path.join(target_dir, f'{basename}.pt') + torch.save({'text_encoded': tokens_mfa, 'token_duration': durs}, target_path) + + print(f"Getting rid of {len(oov_samples)} samples with OOV words.") + oov_target = os.path.join(args.ljspeech_dir, 'wavs_to_ignore.pkl') + with open(oov_target, 'wb') as f: + pickle.dump(oov_samples, f) + print(f"List of OOV samples written to: {oov_target}") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/create_manifests_and_textfiles.py b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/create_manifests_and_textfiles.py new file mode 100644 index 0000000000..aede9f828d --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/create_manifests_and_textfiles.py @@ -0,0 +1,71 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Creates the manifest and .txt transcript files for an LJSpeech split. +The manifest will be used for training, and the .txt files are for the MFA library to find. +""" + +import argparse +import json +import os + +import sox +import wget + +from nemo.collections.common.parts.preprocessing import parsers + +parser = argparse.ArgumentParser() +parser.add_argument('--ljspeech_base', required=True, default=None, type=str) +args = parser.parse_args() + + +def main(): + filelist_base = 'https://raw.githubusercontent.com/NVIDIA/tacotron2/master/filelists/' + filelists = ['train', 'val', 'test'] + + # NeMo parser for text normalization + text_parser = parsers.make_parser(name='en') + + for split in filelists: + # Download file list if necessary + filelist_path = os.path.join(args.ljspeech_base, f"ljs_audio_text_{split}_filelist.txt") + if not os.path.exists(filelist_path): + wget.download(f"{filelist_base}/ljs_audio_text_{split}_filelist.txt", out=args.ljspeech_base) + + manifest_target = os.path.join(args.ljspeech_base, f"ljspeech_{split}.json") + with open(manifest_target, 'w') as f_out: + with open(filelist_path, 'r') as filelist: + print(f"\nCreating {manifest_target}...") + for line in filelist: + basename = line[6:16] + text = text_parser._normalize(line[21:].strip()) + + # Make sure corresponding wavfile exists and write .txt transcript + wav_path = os.path.join(args.ljspeech_base, 'wavs/', basename + '.wav') + assert os.path.exists(wav_path) + txt_path = os.path.join(args.ljspeech_base, 'wavs/', basename + '.txt') + with open(txt_path, 'w') as f_txt: + f_txt.write(text) + + # Write manifest entry + entry = { + 'audio_filepath': wav_path, + 'duration': sox.file_info.duration(wav_path), + 'text': text, + } + f_out.write(json.dumps(entry) + '\n') + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/create_token2idx_dict.py b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/create_token2idx_dict.py new file mode 100644 index 0000000000..2d3e73a38a --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/create_token2idx_dict.py @@ -0,0 +1,68 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Creates a dictionary from token to index based on dictionary .txt given. +""" +import argparse +import json +import os + +parser = argparse.ArgumentParser() +parser.add_argument('--dictionary', required=True, default=None, type=str) +parser.add_argument('--dict_out', required=True, default=None, type=str) +args = parser.parse_args() + + +def main(): + if not os.path.exists(args.dictionary): + raise FileNotFoundError(f"Could not find dictionary file {args.dictionary}") + + phonemes = set() + word2phones = {} + with open(args.dictionary, 'r') as f: + for line in f: + line = line.split() + word = line[0] + tokens = line[1:] + + word2phones[word] = tokens + phonemes.update(tokens) + + # Small list of additional punctuation + word2phones[','] = [' '] + word2phones[';'] = [' '] + word2phones['.'] = [' '] + word2phones['!'] = [' '] + word2phones['?'] = [' '] + word2phones['"'] = [' '] + word2phones['-'] = [' '] + + phone2idx = {k: i for i, k in enumerate(phonemes)} + phone2idx[' '] = len(phone2idx) + phone2idx['sil'] = phone2idx[' '] # Silence + phone2idx['sp'] = phone2idx[' '] # Space + phone2idx['spn'] = phone2idx[' '] # OOV/unk + + dicts = { + 'phone2idx': phone2idx, + 'word2phones': word2phones, + } + with open(args.dict_out, 'w') as f: + json.dump(dicts, f) + + print(f"Total number of phone indices: {len(phone2idx)}") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/extract_ljspeech_energy_pitch.py b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/extract_ljspeech_energy_pitch.py new file mode 100644 index 0000000000..a76ce6dd9f --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/extract_ljspeech_energy_pitch.py @@ -0,0 +1,88 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Extracts energy and pitch data from LJSpeech wav files and writes them to a directory as LJxxx-xxxx.npy files. +Assuming that the wavs are located at `/wavs`, this script will write the +energy files to `/energies/` and the pitch files to `/pitches`, +creating the relevant directories if necessary. + +USAGE: python extract_ljspeech_energy_pitch.py --ljspeech_dir= +""" +import argparse +from pathlib import Path + +import librosa +import numpy as np + +tqdm = None + +try: + from tqdm import tqdm +except ModuleNotFoundError: + pass + +parser = argparse.ArgumentParser( + description="Extracts energies (L2-norm of STFT frame amplitudes) and pitches (F0) from LJSpeech data." +) +parser.add_argument("--ljspeech_dir", required=True, default=None, type=str) +args = parser.parse_args() + + +def main(): + wavfile_dir = Path(args.ljspeech_dir) / "wavs" + wavfile_list = list(wavfile_dir.glob('*.wav')) + + target_dir = Path(args.ljspeech_dir) + # Create target dir /energies and /pitches if necessary + if not Path(target_dir / "energies").exists(): + print(f"Creating target directory: {target_dir/'energies'}") + Path(target_dir / "energies").mkdir() + if not Path(target_dir / "pitches").exists(): + print(f"Creating target directory: {target_dir/'pitches'}") + Path(target_dir / "pitches").mkdir() + + if tqdm is not None: + wavfile_list = tqdm(wavfile_list) + for count, file_ in enumerate(wavfile_list): + basename = Path(file_).stem + pitch_path = target_dir / "pitches" / f"{basename}.npy" + energy_path = target_dir / "energies" / f"{basename}.npy" + if pitch_path.exists() and energy_path.exists(): + continue + audio, sr = librosa.load(file_, sr=22050) + + # Calculate f0 + # Please note that fmin and fmax are good approximates for the speaker in LJSpeech and may not generalize to + # other speakers + f0, _, _ = librosa.pyin(audio, fmin=80, fmax=800, frame_length=1024, sr=sr, fill_na=0.0) + + # Save to new file + np.save(pitch_path, f0) + + # Calculate energy + stft_amplitude = np.abs(librosa.stft(audio, n_fft=1024, hop_length=256, win_length=1024)) + energy = np.linalg.norm(stft_amplitude, axis=0) # axis=0 since librosa.stft -> (freq bins, frames) + + # Save to new file + np.save(energy_path, energy) + + assert energy.shape == f0.shape + if tqdm is None and count % 1000 == 0: + print(f"Finished processing {count} wav files...") + + print(f"Finished energy extraction for a total of {len(wavfile_list)} wav files.") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/extract_ljspeech_phonemes_and_durs.sh b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/extract_ljspeech_phonemes_and_durs.sh new file mode 100755 index 0000000000..d5ee2778a5 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/ljspeech/extract_ljspeech_phonemes_and_durs.sh @@ -0,0 +1,150 @@ +#!/bin/bash + +# Extracts the phonemes and alignments for the LJSpeech dataset via the +# Montreal Forced Aligner (MFA) library, and computes durations per phoneme. +# Assumes you have downloaded and expanded the LJSpeech dataset, and that they +# are located at the directory specified. +# +# This script will create: +# - /mappings.json: Contains word->phone and phone->idx mappings +# - /wavs_to_ignore.pkl: A pickled list of wavs to ignore +# because of OOV words in their transcripts +# - /phoneme_durations/LJ*.npz: Numpy files for each wavfile +# with fields 'tokens' (token indices) and 'durs' (durations per token) +# +# It will also create intermediate files: +# - /alignments/wavs/LJ*.TextGrid: MFA output +# +# Note: This script will create a conda environment to set up MFA, so please +# make sure that conda is active before running this script. +# +# Note: You may need to install OpenBlas yourself if the MFA script can't find +# it. (e.g. sudo apt-get install libopenblas-dev) +# +# Example Usage: +# ./extract_ljspeech_phonemes_and_durs.sh /data/LJSpeech-1.1 + +ENV_NAME='aligner' + +SAMPLE_RATE=22050 +WINDOW_STRIDE=256 + +CMUDICT_URL='https://raw.githubusercontent.com/cmusphinx/cmudict/master/cmudict.dict' +SPLITS_BASE_URL='https://raw.githubusercontent.com/NVIDIA/tacotron2/master/filelists/' + +# Usage info +show_help() { +cat << EOF +Usage: $(basename "$0") [-h] \ + [--skip_env_setup] \ + [--g2p_dict=] \ + +Extracts phonemes and their respective durations for the LJSpeech dataset using +the Montreal Forced Aligner (MFA). + + -h Help message + --skip_env_setup (Optional) Skips setting up the MFA conda environment + "aligner". Use only if you already have this set up. + --g2p_dict (Optional) Path to the grapheme to phoneme dictionary + text file, if already generated. If set, skips the G2P + step. +EOF +} + +SKIP_ENV_SETUP=false +G2P_DICT='' + +while :; do + case $1 in + -h|-\?|--help) + show_help + exit + ;; + --g2p_dict=?*) + G2P_DICT=${1#*=} + ;; + --skip_env_setup) + SKIP_ENV_SETUP=true + ;; + *) + break + esac + shift +done + +if [[ -z $1 ]]; then + echo "Must specify a LJSpeech base directory." + exit 1 +fi +LJSPEECH_BASE=$1 + +# Check for conda +read -r -d '' CONDA_MESSAGE << EOM +This script requires either Anaconda or Miniconda to be active, as it installs the Montreal Forced Aligner via a conda environment. +See their documentation (https://montreal-forced-aligner.readthedocs.io/en/latest/installation.html) for more details. +EOM +if [ -z "$(which conda)" ]; then + echo $CONDA_MESSAGE + exit +fi + +CONDA_PREFIX=$(conda info --base) +source $CONDA_PREFIX/etc/profile.d/conda.sh + +# Set up env for MFA (env name "aligner") and install +if $SKIP_ENV_SETUP; then + echo "Skipping environment setup. Assuming env name "aligner" exists." +else + echo "Setting up conda environment for MFA (env name \"aligner\")..." + conda create -n $ENV_NAME -c conda-forge openblas python=3.8 openfst pynini ngram baumwelch + conda activate $ENV_NAME + pip install montreal-forced-aligner tgt torch + mfa thirdparty download + echo "Conda environment \"$ENV_NAME\" set up." +fi + +if ! conda activate $ENV_NAME; then + echo "Could not activate environment, see Conda output above." + exit +fi + +# Download CMU word-to-phoneme dict and clean out comments so they're not mistaken for tokens +if [ -z $G2P_DICT ]; then + if [ ! -f $LJSPEECH_BASE/cmudict.dict ]; then + echo "Downloading CMU dict." + wget -P $LJSPEECH_BASE $CMUDICT_URL + fi + if [ ! -f $LJSPEECH_BASE/uncommented_cmudict.dict ]; then + echo "Creating uncommented version of CMUdict." + sed 's/\ \#.*//' $LJSPEECH_BASE/cmudict.dict > $LJSPEECH_BASE/uncommented_cmudict.dict + fi + G2P_DICT=$LJSPEECH_BASE/uncommented_cmudict.dict +fi + +# Run alignment +echo "Starting MFA with dictionary at: $G2P_DICT" + +read -r -d '' MFA_ERROR_MSG << EOM +Could not run MFA. If it could not find OpenBlas, you may need to install it manually. +(e.g. sudo apt-get install libopenblas-dev) +EOM + +if ! mfa download acoustic english; then + echo $MFA_ERROR_MSG +fi +mfa align --clean $LJSPEECH_BASE $G2P_DICT english $LJSPEECH_BASE/alignments + +# Create JSON mappings from word to phonemes and phonemes to indices +echo "Creating word->phone and phone->idx mappings at $LJSPEECH_BASE/mappings.json..." +python create_token2idx_dict.py \ + --dictionary=$G2P_DICT \ + --dict_out=$LJSPEECH_BASE/mappings.json + +# Calculate phoneme durations +echo "Calculating phoneme durations..." +python calculate_durs.py \ + --ljspeech_dir=$LJSPEECH_BASE \ + --mappings=$LJSPEECH_BASE/mappings.json \ + --sr=$SAMPLE_RATE \ + --hop_length=$WINDOW_STRIDE +echo "Phoneme durations and tokens written." diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/process_an4_data.py b/models/tacotron2/NeMo/scripts/dataset_processing/process_an4_data.py new file mode 100644 index 0000000000..5eaace2a7c --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/process_an4_data.py @@ -0,0 +1,88 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import glob +import json +import logging +import os +import subprocess +import tarfile + +import librosa +import wget + +parser = argparse.ArgumentParser(description="AN4 dataset download and processing") +parser.add_argument("--data_root", required=True, default=None, type=str) +args = parser.parse_args() + + +def build_manifest(data_root, transcripts_path, manifest_path, wav_path): + with open(transcripts_path, 'r') as fin: + with open(manifest_path, 'w') as fout: + for line in fin: + # Lines look like this: + # transcript (fileID) + transcript = line[: line.find('(') - 1].lower() + transcript = transcript.replace('', '').replace('', '') + transcript = transcript.strip() + + file_id = line[line.find('(') + 1 : -2] # e.g. "cen4-fash-b" + audio_path = os.path.join( + data_root, wav_path, file_id[file_id.find('-') + 1 : file_id.rfind('-')], file_id + '.wav', + ) + + duration = librosa.core.get_duration(filename=audio_path) + + # Write the metadata to the manifest + metadata = { + "audio_filepath": audio_path, + "duration": duration, + "text": transcript, + } + json.dump(metadata, fout) + fout.write('\n') + + +def main(): + data_root = os.path.abspath(args.data_root) + + # Convert from .sph to .wav + logging.info("Converting audio files to .wav...") + sph_list = glob.glob(os.path.join(data_root, 'an4/**/*.sph'), recursive=True) + for sph_path in sph_list: + wav_path = sph_path[:-4] + '.wav' + cmd = ['sox', sph_path, wav_path] + subprocess.run(cmd) + logging.info("Finished conversion.") + + # Build manifests + logging.info("Building training manifest...") + train_transcripts = os.path.join(data_root, 'an4/etc/an4_train.transcription') + train_manifest = os.path.join(data_root, 'an4/train_manifest.json') + train_wavs = os.path.join(data_root, 'an4/wav/an4_clstk') + build_manifest(data_root, train_transcripts, train_manifest, train_wavs) + logging.info("Training manifests created.") + + logging.info("Building test manifest...") + test_transcripts = os.path.join(data_root, 'an4/etc/an4_test.transcription') + test_manifest = os.path.join(data_root, 'an4/test_manifest.json') + test_wavs = os.path.join(data_root, 'an4/wav/an4test_clstk') + build_manifest(data_root, test_transcripts, test_manifest, test_wavs) + logging.info("Test manifest created.") + + logging.info("Done with AN4 processing!") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/process_degli_data.py b/models/tacotron2/NeMo/scripts/dataset_processing/process_degli_data.py new file mode 100644 index 0000000000..ba1b26a3b4 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/process_degli_data.py @@ -0,0 +1,70 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +from nemo.collections.tts.data.datalayers import preprocess_linear_specs_dataset + +sys.path.insert(0, '../') + + +def main(): + parser = argparse.ArgumentParser( + description='Create dataset fitted for training and validating deep griffin iteration from wavefiles' + ) + parser.add_argument( + "-v", + "--valid_filelist", + help="Filelist for validation set, with all validation audio files listed", + required=True, + default=None, + type=str, + ) + parser.add_argument( + "-t", + "--train_filelist", + help="Filelist for train set, with all train audio files listed", + required=True, + default=None, + type=str, + ) + parser.add_argument( + "-n", + "--n_fft", + help="Value for the n_fft parameter, and the filter length for the STFT", + default=512, + type=int, + ) + parser.add_argument("--hop_length", help="STFT parameter", default=256, type=int) + parser.add_argument( + "-d", "--destination", help="Destination to save the preprocessed data set to", default="/tmp", type=str + ) + parser.add_argument( + "-s", + "--num_snr", + help="Number of distinctive noisy samples to generate for each clear sample at the file list", + default=1, + type=int, + ) + + args = parser.parse_args() + + preprocess_linear_specs_dataset( + args.valid_filelist, args.train_filelist, args.n_fft, args.hop_length, args.num_snr, args.destination + ) + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/process_hub5_data.py b/models/tacotron2/NeMo/scripts/dataset_processing/process_hub5_data.py new file mode 100644 index 0000000000..e67a0d04a0 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/process_hub5_data.py @@ -0,0 +1,256 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is heavily derived from the Patter HUB5 processing script written +# by Ryan Leary + +import argparse +import glob +import json +import os +import re +import subprocess +import sys +from collections import namedtuple +from math import ceil, floor +from operator import attrgetter + +import numpy as np +import scipy.io.wavfile as wavfile +from tqdm import tqdm + +parser = argparse.ArgumentParser(description="Prepare HUB5 data for training/eval") +parser.add_argument( + "--data_root", default=None, type=str, required=True, help="The path to the root LDC HUB5 dataset directory.", +) +parser.add_argument( + "--dest_root", + default=None, + type=str, + required=True, + help="Path to the destination root directory for processed files.", +) + +# Optional arguments +parser.add_argument( + "--min_slice_duration", default=10.0, type=float, help="Minimum audio slice duration after processing.", +) + +args = parser.parse_args() + +StmUtterance = namedtuple( + 'StmUtterance', ['filename', 'channel', 'speaker_id', 'begin', 'end', 'label', 'transcript',], +) +STM_LINE_FMT = re.compile(r"^(\w+)\s+(\w+)\s+(\w+)\s+([0-9.]+)\s+([0-9.]+)\s+(<.*>)?\s+(.+)$") + +# Transcription errors and their fixes +TRANSCRIPT_BUGS = {"en_4622-B-12079-12187": "KIND OF WEIRD BUT"} + + +def get_utt_id(segment): + """ + Gives utterance IDs in a form like: en_4156-a-36558-37113 + """ + return "{}-{}-{}-{}".format(segment.filename, segment.channel, int(segment.begin * 100), int(segment.end * 100),) + + +def convert_utterances(sph_path, wav_path): + """ + Converts a sphere audio file to wav. + """ + cmd = ["sph2pipe", "-f", "wav", "-p", sph_path, wav_path] + subprocess.run(cmd) + + +def create_wavs(data_root, dest_root): + """ + Converts the English sph files to wav using sph2pipe. + """ + sph_root = os.path.join(data_root, "hub5e_00", "english") + sph_list = glob.glob(os.path.join(sph_root, "*.sph")) + + # Iterate over each sphere file and conver to wav + for sph_path in tqdm(sph_list, desc="Converting to wav", unit="file"): + sph_name, _ = os.path.splitext(os.path.basename(sph_path)) + wav_path = os.path.join(dest_root, 'full_audio_wav', sph_name + ".wav") + cmd = ["sph2pipe", "-f", "wav", "-p", sph_path, wav_path] + subprocess.run(cmd) + + +def process_transcripts(dataset_root): + """ + Reads in transcripts for each audio segment and processes them. + """ + stm_path = os.path.join(dataset_root, "2000_hub5_eng_eval_tr", "reference", "hub5e00.english.000405.stm",) + results = [] + chars = set() + + with open(stm_path, "r") as fh: + for line in fh: + # lines with ';;' are comments + if line.startswith(";;"): + continue + + if "IGNORE_TIME_SEGMENT_" in line: + continue + line = line.replace("", "").replace("", "") + line = line.replace("(%HESITATION)", "UH") + line = line.replace("-", "") + line = line.replace("(%UH)", "UH") + line = line.replace("(%AH)", "UH") + line = line.replace("(", "").replace(")", "") + + line = line.lower() + + m = STM_LINE_FMT.search(line.strip()) + utt = StmUtterance(*m.groups()) + + # Convert begin/end times to float + utt = utt._replace(begin=float(utt.begin)) + utt = utt._replace(end=float(utt.end)) + + # Check for utterance in dict of transcript mistakes + transcript_update = TRANSCRIPT_BUGS.get(get_utt_id(utt)) + if transcript_update is not None: + utt = utt._replace(transcript=transcript_update) + + results.append(utt) + chars.update(list(utt.transcript)) + return results, chars + + +def write_one_segment(dest_root, speaker_id, count, audio, sr, duration, transcript): + """ + Writes out one segment of audio, and writes its corresponding transcript + in the manifest. + + Args: + dest_root: the path to the output directory root + speaker_id: ID of the speaker, used in file naming + count: number of segments from this speaker so far + audio: the segment's audio data + sr: sample rate of the audio + duration: duration of the audio + transcript: the corresponding transcript + """ + audio_path = os.path.join(dest_root, "audio", f"{speaker_id}_{count:03}.wav") + + manifest_path = os.path.join(dest_root, "manifest_hub5.json") + + # Write audio + wavfile.write(audio_path, sr, audio) + + # Write transcript + transcript = { + "audio_filepath": audio_path, + "duration": duration, + "text": transcript, + } + with open(manifest_path, 'a') as f: + json.dump(transcript, f) + f.write('\n') + + +def segment_audio(info_list, dest_root, min_slice_duration): + """ + Combines audio into >= min_slice_duration segments of the same speaker, + and writes the combined transcripts into a manifest. + + Args: + info_list: list of StmUtterance objects with transcript information. + dest_root: path to output destination + min_slice_duration: min number of seconds per output audio slice + """ + info_list = sorted(info_list, key=attrgetter('speaker_id', 'begin')) + + prev_id = None # For checking audio concatenation + id_count = 0 + + sample_rate, audio_data = None, None + transcript_buffer = '' + audio_buffer = [] + buffer_duration = 0.0 + + # Iterate through utterances to build segments + for info in info_list: + if info.speaker_id != prev_id: + # Scrap the remainder in the buffers and start next segment + prev_id = info.speaker_id + id_count = 0 + + sample_rate, audio_data = wavfile.read(os.path.join(dest_root, 'full_audio_wav', info.filename + '.wav')) + transcript_buffer = '' + audio_buffer = [] + buffer_duration = 0.0 + + # Append utterance info to buffers + transcript_buffer += info.transcript + channel = 0 if info.channel.lower() == 'a' else 1 + audio_buffer.append( + audio_data[floor(info.begin * sample_rate) : ceil(info.end * sample_rate), channel,] + ) + buffer_duration += info.end - info.begin + + if buffer_duration < min_slice_duration: + transcript_buffer += ' ' + else: + # Write out segment and transcript + id_count += 1 + write_one_segment( + dest_root, + info.speaker_id, + id_count, + np.concatenate(audio_buffer, axis=0), + sample_rate, + buffer_duration, + transcript_buffer, + ) + + transcript_buffer = '' + audio_buffer = [] + buffer_duration = 0.0 + + +def main(): + data_root = args.data_root + dest_root = args.dest_root + + min_slice_duration = args.min_slice_duration + + if not os.path.exists(os.path.join(dest_root, 'full_audio_wav')): + os.makedirs(os.path.join(dest_root, 'full_audio_wav')) + if not os.path.exists(os.path.join(dest_root, 'audio')): + os.makedirs(os.path.join(dest_root, 'audio')) + + # Create/wipe manifest contents + open(os.path.join(dest_root, "manifest_hub5.json"), 'w').close() + + # Convert full audio files from .sph to .wav + create_wavs(data_root, dest_root) + + # Get each audio transcript from transcript file + info_list, chars = process_transcripts(data_root) + + print("Writing out vocab file", file=sys.stderr) + with open(os.path.join(dest_root, "vocab.txt"), 'w') as fh: + for x in sorted(list(chars)): + fh.write(x + "\n") + + # Segment the audio data + print("Segmenting audio and writing manifest") + segment_audio(info_list, dest_root, min_slice_duration) + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/process_speech_commands_data.py b/models/tacotron2/NeMo/scripts/dataset_processing/process_speech_commands_data.py new file mode 100644 index 0000000000..9080b334fa --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/process_speech_commands_data.py @@ -0,0 +1,402 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Usage: + +python process_speech_commands_data.py \ + --data_root= \ + --data_version= \ + --class_split= \ + --rebalance \ + --log +""" + +import argparse +import glob +import json +import logging +import os +import re +import tarfile +import urllib.request + +import librosa +import numpy as np + +URL_v1 = "http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz" +URL_v2 = "http://download.tensorflow.org/data/speech_commands_v0.02.tar.gz" + + +def __maybe_download_file(destination: str, source: str): + """ + Downloads source to destination if it doesn't exist. + If exists, skips download + Args: + destination: local filepath + source: url of resource + + Returns: + + """ + if not os.path.exists(destination): + logging.info(f"{destination} does not exist. Downloading ...") + urllib.request.urlretrieve(source, filename=destination + '.tmp') + os.rename(destination + '.tmp', destination) + logging.info(f"Downloaded {destination}.") + else: + logging.info(f"Destination {destination} exists. Skipping.") + return destination + + +def __extract_all_files(filepath: str, data_root: str, data_dir: str): + if not os.path.exists(data_dir): + extract_file(filepath, data_dir) + else: + logging.info(f'Skipping extracting. Data already there {data_dir}') + + +def extract_file(filepath: str, data_dir: str): + try: + tar = tarfile.open(filepath) + tar.extractall(data_dir) + tar.close() + except Exception: + logging.info('Not extracting. Maybe already there?') + + +def __process_data( + data_folder: str, dst_folder: str, rebalance: bool = False, class_split: str = "all", skip_duration: bool = False +): + """ + To generate manifest + + Args: + data_folder: source with wav files and validation / test lists + dst_folder: where manifest files will be stored + rebalance: rebalance the classes to have same number of samples. + class_split: whether to use all classes as distinct labels, or to use + 10 classes subset and rest of the classes as noise or background. + skip_duration: Bool whether to skip duration computation. Use this only for + colab notebooks where knowing duration is not necessary for demonstration. + + Returns: + + """ + + if not os.path.exists(dst_folder): + os.makedirs(dst_folder) + + # Used for 10 classes + silence + unknown class setup + class_subset = ["yes", "no", "up", "down", "left", "right", "on", "off", "stop", "go"] + + pattern = re.compile(r"(.+\/)?(\w+)\/([^_]+)_.+wav") + all_files = glob.glob(os.path.join(data_folder, '*/*wav')) + + with open(os.path.join(data_folder, 'validation_list.txt'), 'r') as fin: + validation_files = fin.readlines() + + valset = set() + for entry in validation_files: + r = re.match(pattern, entry) + if r: + valset.add(r.group(3)) + + with open(os.path.join(data_folder, 'testing_list.txt'), 'r') as fin: + testing_files = fin.readlines() + + testset = set() + for entry in testing_files: + r = re.match(pattern, entry) + if r: + testset.add(r.group(3)) + + logging.info("Validation and test set lists extracted") + + label_count = {} + label_filepaths = {} + unknown_val_filepaths = [] + unknown_test_filepaths = [] + + train, val, test = [], [], [] + for entry in all_files: + r = re.match(pattern, entry) + if r: + label, uid = r.group(2), r.group(3) + + if label == '_background_noise_' or label == 'silence': + continue + + if class_split == "sub": + if label not in class_subset: + label = "unknown" # replace label + + if uid in valset: + unknown_val_filepaths.append((label, entry)) + elif uid in testset: + unknown_test_filepaths.append((label, entry)) + + sample = (label, entry) + + if uid not in valset and uid not in testset: + if label in label_count: + label_count[label] += 1 + else: + label_count[label] = 1 + + if label in label_filepaths: + label_filepaths[label] += [sample] + else: + label_filepaths[label] = [sample] + + if label == 'unknown': + continue + + if uid in valset: + val.append(sample) + elif uid in testset: + test.append(sample) + else: + train.append(sample) + + logging.info("Prepared filepaths for dataset") + + # Add silence and unknown class label samples + if class_split == "sub": + logging.info("Perforiming 10+2 class subsplit") + + silence_path = os.path.join(data_folder, "silence") + if not os.path.exists(silence_path): + os.mkdir(silence_path) + + silence_stride = 1000 # 0.0625 second stride + sampling_rate = 16000 + folder = os.path.join(data_folder, "_background_noise_") + + silence_files = [] + rng = np.random.RandomState(0) + + for file in os.listdir(folder): + if ".wav" in file: + load_path = os.path.join(folder, file) + y, sr = librosa.load(load_path, sr=sampling_rate) + + for i in range(0, len(y) - sampling_rate, silence_stride): + file_path = "silence/{}_{}.wav".format(file[:-4], i) + y_slice = y[i : i + sampling_rate] + magnitude = rng.uniform(0.0, 1.0) + y_slice *= magnitude + out_file_path = os.path.join(data_folder, file_path) + librosa.output.write_wav(out_file_path, y_slice, sr) + + silence_files.append(('silence', out_file_path)) + + rng = np.random.RandomState(0) + rng.shuffle(silence_files) + logging.info(f"Constructed silence set of {len(silence_files)}") + + # Create the splits + rng = np.random.RandomState(0) + silence_split = 0.1 + unknown_split = 0.1 + + # train split + num_total_samples = sum([label_count[cls] for cls in class_subset]) + num_silence_samples = int(np.ceil(silence_split * num_total_samples)) + + # initialize sample + label_count['silence'] = 0 + + for silence_id in range(num_silence_samples): + label_count['silence'] += 1 + + if 'silence' in label_filepaths: + label_filepaths['silence'] += [silence_files[silence_id]] + else: + label_filepaths['silence'] = [silence_files[silence_id]] + + train.extend(label_filepaths['silence']) + + # Update train unknown set + unknown_train_samples = label_filepaths['unknown'] + + rng.shuffle(unknown_train_samples) + unknown_size = int(np.ceil(unknown_split * num_total_samples)) + + label_count['unknown'] = unknown_size + label_filepaths['unknown'] = unknown_train_samples[:unknown_size] + + train.extend(label_filepaths['unknown']) + + logging.info("Train set prepared") + + # val set silence + num_val_samples = len(val) + num_silence_samples = int(np.ceil(silence_split * num_val_samples)) + + val_idx = label_count['silence'] + 1 + for silence_id in range(num_silence_samples): + val.append(silence_files[val_idx + silence_id]) + + # Update val unknown set + rng.shuffle(unknown_val_filepaths) + unknown_size = int(np.ceil(unknown_split * num_val_samples)) + + val.extend(unknown_val_filepaths[:unknown_size]) + + logging.info("Validation set prepared") + + # test set silence + num_test_samples = len(test) + num_silence_samples = int(np.ceil(silence_split * num_test_samples)) + + test_idx = val_idx + num_silence_samples + 1 + for silence_id in range(num_silence_samples): + test.append(silence_files[test_idx + silence_id]) + + # Update test unknown set + rng.shuffle(unknown_test_filepaths) + unknown_size = int(np.ceil(unknown_split * num_test_samples)) + + test.extend(unknown_test_filepaths[:unknown_size]) + + logging.info("Test set prepared") + + max_command = None + max_count = -1 + for command, count in label_count.items(): + if command == 'unknown': + continue + + if count > max_count: + max_count = count + max_command = command + + if rebalance: + logging.info(f"Command with maximum number of samples = {max_command} with {max_count} samples") + logging.info(f"Rebalancing dataset by duplicating classes with less than {max_count} samples...") + + for command, samples in label_filepaths.items(): + filepaths = [sample[1] for sample in samples] + + rng = np.random.RandomState(0) + filepaths = np.asarray(filepaths) + num_samples = len(filepaths) + + if num_samples < max_count: + difference = max_count - num_samples + duplication_ids = rng.choice(num_samples, difference, replace=True) + + filepaths = np.append(filepaths, filepaths[duplication_ids], axis=0) + + logging.info(f"Extended class label {command} from {num_samples} samples to {len(filepaths)} samples") + + label_filepaths[command] = [(command, filepath) for filepath in filepaths] + + del train + train = [] + for label, samples in label_filepaths.items(): + train.extend(samples) + + manifests = [ + ('train_manifest.json', train), + ('validation_manifest.json', val), + ('test_manifest.json', test), + ] + + for manifest_filename, dataset in manifests: + with open(os.path.join(dst_folder, manifest_filename), 'w') as fout: + num_files = len(dataset) + pct_file = num_files // 100 + file_count = 0 + + logging.info(f"Preparing manifest : {manifest_filename} with #{num_files} files") + + for label, audio_path in dataset: + if not skip_duration: + duration = librosa.core.get_duration(filename=audio_path) + else: + duration = 0.0 + + # Write the metadata to the manifest + metadata = { + "audio_filepath": audio_path, + "duration": duration, + "command": label, + } + json.dump(metadata, fout) + fout.write('\n') + fout.flush() + + file_count += 1 + if file_count % pct_file == 0: + if not skip_duration: + logging.info(f"Finished serializing {file_count} / {num_files} into {manifest_filename}") + + logging.info(f"Finished construction of manifest : {manifest_filename}") + + if skip_duration: + logging.info( + f"\n<> Duration computation was skipped for demonstration purposes on Colaboratory.\n" + f"In order to replicate paper results and properly perform data augmentation, \n" + f"please recompute the manifest file without the `--skip_duration` flag !\n" + ) + + +def main(): + parser = argparse.ArgumentParser(description='Google Speech Command Data download') + parser.add_argument("--data_root", required=True, default=None, type=str) + parser.add_argument('--data_version', required=True, default=1, type=int, choices=[1, 2]) + parser.add_argument('--class_split', required=False, default='all', type=str, choices=['all', 'sub']) + parser.add_argument('--rebalance', required=False, action='store_true') + parser.add_argument('--skip_duration', required=False, action='store_true') + parser.add_argument('--log', required=False, action='store_true') + parser.set_defaults(log=False, rebalance=False, skip_duration=False) + args = parser.parse_args() + + if args.log: + logging.basicConfig(level=logging.DEBUG) + + data_root = args.data_root + data_set = "google_speech_recognition_v{0}".format(args.data_version) + data_folder = os.path.join(data_root, data_set) + + logging.info(f"Working on: {data_set}") + + if args.data_version == 1: + URL = URL_v1 + else: + URL = URL_v2 + + # Download and extract + if not os.path.exists(data_folder): + file_path = os.path.join(data_root, data_set + ".tar.bz2") + logging.info(f"Getting {data_set}") + __maybe_download_file(file_path, URL) + logging.info(f"Extracting {data_set}") + __extract_all_files(file_path, data_root, data_folder) + + logging.info(f"Processing {data_set}") + __process_data( + data_folder, + data_folder, + rebalance=args.rebalance, + class_split=args.class_split, + skip_duration=args.skip_duration, + ) + logging.info('Done!') + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/scripts/dataset_processing/process_vad_data.py b/models/tacotron2/NeMo/scripts/dataset_processing/process_vad_data.py new file mode 100644 index 0000000000..ba98113d9b --- /dev/null +++ b/models/tacotron2/NeMo/scripts/dataset_processing/process_vad_data.py @@ -0,0 +1,486 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Usage: + +python process_vad_data.py \ + --out_dir= \ + --speech_data_root= \ + --background_data_root= \ + --rebalance_method=<'under' or 'over' or 'fixed'> \ + --log + (Optional --demo (for demonstration in tutorial). If you want to use your own background noise data, make sure to delete --demo) +""" +import argparse +import glob +import json +import logging +import os +import tarfile +import urllib.request + +import librosa +import numpy as np +import soundfile as sf +from sklearn.model_selection import train_test_split + +sr = 16000 + +# google speech command v2 +URL = "http://download.tensorflow.org/data/speech_commands_v0.02.tar.gz" + + +def __maybe_download_file(destination: str, source: str): + """ + Downloads source to destination if it doesn't exist. + If exists, skips download + Args: + destination: local filepath + source: url of resource + + Returns: + + """ + if not os.path.exists(destination): + logging.info(f"{destination} does not exist. Downloading ...") + urllib.request.urlretrieve(source, filename=destination + '.tmp') + os.rename(destination + '.tmp', destination) + logging.info(f"Downloaded {destination}.") + else: + logging.info(f"Destination {destination} exists. Skipping.") + return destination + + +def extract_file(filepath: str, data_dir: str): + try: + tar = tarfile.open(filepath) + tar.extractall(data_dir) + tar.close() + except Exception: + logging.info('Not extracting. Maybe already there?') + + +def __extract_all_files(filepath: str, data_root: str, data_dir: str): + if not os.path.exists(data_dir): + extract_file(filepath, data_dir) + else: + logging.info(f'Skipping extracting. Data already there {data_dir}') + + +def split_train_val_test(data_dir, file_type, test_size=0.1, val_size=0.1, demo=False): + X = [] + if file_type == "speech": + for o in os.listdir(data_dir): + if os.path.isdir(os.path.join(data_dir, o)) and o.split("/")[-1] != "_background_noise_": + X.extend(glob.glob(os.path.join(data_dir, o) + '/*.wav')) + + if demo: + logging.info( + f"For Demonstration, we use {int(len(X)/100)}/{len(X)} speech data. Make sure to remove --demo flag when you actually train your model!" + ) + X = np.random.choice(X, int(len(X) / 100), replace=False) + + else: + for o in os.listdir(data_dir): + if os.path.isdir(os.path.join(data_dir, o)): + X.extend(glob.glob(os.path.join(data_dir, o) + '/*.wav')) + else: # for using "_background_noise_" from google speech commands as background data + if o.endswith(".wav"): + X.append(os.path.join(data_dir, o)) + + X_train, X_test = train_test_split(X, test_size=test_size, random_state=1) + val_size_tmp = val_size / (1 - test_size) + X_train, X_val = train_test_split(X_train, test_size=val_size_tmp, random_state=1) + + with open(os.path.join(data_dir, file_type + "_training_list.txt"), "w") as outfile: + outfile.write("\n".join(X_train)) + with open(os.path.join(data_dir, file_type + "_testing_list.txt"), "w") as outfile: + outfile.write("\n".join(X_test)) + with open(os.path.join(data_dir, file_type + "_validation_list.txt"), "w") as outfile: + outfile.write("\n".join(X_val)) + + logging.info(f'Overall: {len(X)}, Train: {len(X_train)}, Validatoin: {len(X_val)}, Test: {len(X_test)}') + logging.info(f"Finish spliting train, val and test for {file_type}. Write to files!") + + +def process_google_speech_train(data_dir): + X = [] + for o in os.listdir(data_dir): + if os.path.isdir(os.path.join(data_dir, o)) and o.split("/")[-1] != "_background_noise_": + X.extend(glob.glob(os.path.join(data_dir, o) + '/*.wav')) + + short_files = [i.split(data_dir)[1] for i in files] + + with open(os.path.join(data_dir, 'testing_list.txt'), 'r') as allfile: + testing_list = allfile.read().splitlines() + + with open(os.path.join(data_dir, 'validation_list.txt'), 'r') as allfile: + validation_list = allfile.read().splitlines() + + exist_set = set(testing_list).copy() + exist_set.update(set(validation_list)) + + training_list = [i for i in short_files if i not in exist_set] + + with open(os.path.join(data_dir, "training_list.txt"), "w") as outfile: + outfile.write("\n".join(training_list)) + + logging.info( + f'Overall: {len(files)}, Train: {len(training_list)}, Validatoin: {len(validation_list)}, Test: {len(testing_list)}' + ) + + +def write_manifest( + out_dir, + files, + prefix, + manifest_name, + start=0.0, + end=None, + duration_stride=1.0, + duration_max=None, + duration_limit=100.0, + filter_long=False, +): + """ + Given a list of files, segment each file and write them to manifest with restrictions. + Args: + out_dir: directory of generated manifest + files: list of files to be processed + prefix: label of samples + manifest_name: name of generated manifest + start: beginning of audio of generating segment + end: end of audio of generating segment + duration_stride: stride for segmenting audio samples + duration_max: duration for each segment + duration_limit: duration threshold for filtering out long audio samples + filter_long: boolean to determine whether to filter out long audio samples + Returns: + """ + seg_num = 0 + skip_num = 0 + if duration_max is None: + duration_max = 1e9 + + if not os.path.exists(out_dir): + logging.info(f'Outdir {out_dir} does not exist. Creat directory.') + os.mkdir(out_dir) + + output_path = os.path.join(out_dir, manifest_name + '.json') + with open(output_path, 'w') as fout: + for file in files: + label = prefix + + try: + x, _sr = librosa.load(file, sr=sr) + duration = librosa.get_duration(x, sr=sr) + + except Exception: + continue + + if filter_long and duration > duration_limit: + skip_num += 1 + continue + + offsets = [] + durations = [] + + if duration > duration_max: + current_offset = start + + while current_offset < duration: + if end is not None and current_offset > end: + break + + difference = duration - current_offset + + if difference < duration_max: + break + + offsets.append(current_offset) + durations.append(duration_max) + + current_offset += duration_stride + + else: + # Duration is not long enough! Skip + skip_num += 1 + + for duration, offset in zip(durations, offsets): + metadata = { + 'audio_filepath': file, + 'duration': duration, + 'label': label, + 'text': '_', # for compatibility with ASRAudioText + 'offset': offset, + } + json.dump(metadata, fout) + fout.write('\n') + fout.flush() + seg_num += 1 + return skip_num, seg_num, output_path + + +def load_list_write_manifest( + data_dir, + out_dir, + filename, + prefix, + start, + end, + duration_stride=1.0, + duration_max=1.0, + duration_limit=100.0, + filter_long=True, +): + + filename = prefix + '_' + filename + file_path = os.path.join(data_dir, filename) + + with open(file_path, 'r') as allfile: + files = allfile.read().splitlines() + + manifest_name = filename.split('_list.txt')[0] + '_manifest' + skip_num, seg_num, output_path = write_manifest( + out_dir, + files, + prefix, + manifest_name, + start, + end, + duration_stride, + duration_max, + duration_limit, + filter_long=True, + ) + return skip_num, seg_num, output_path + + +def rebalance_json(data_dir, data_json, num, prefix): + data = [] + seg = 0 + for line in open(data_json, 'r'): + data.append(json.loads(line)) + filename = data_json.split('/')[-1] + fout_path = os.path.join(data_dir, prefix + "_" + filename) + + if len(data) >= num: + selected_sample = np.random.choice(data, num, replace=False) + else: + selected_sample = np.random.choice(data, num, replace=True) + + with open(fout_path, 'a') as fout: + for i in selected_sample: + seg += 1 + json.dump(i, fout) + fout.write('\n') + fout.flush() + + logging.info(f'Get {seg}/{num} to {fout_path} from {data_json}') + return fout_path + + +def generate_variety_noise(data_dir, filename, prefix): + + curr_dir = data_dir.split("_background_noise_")[0] + silence_path = os.path.join(curr_dir, "_background_noise_more") + + if not os.path.exists(silence_path): + os.mkdir(silence_path) + + silence_stride = 1000 # stride = 1/16 seconds + sampling_rate = 16000 + + silence_files = [] + rng = np.random.RandomState(0) + + filename = prefix + '_' + filename + file_path = os.path.join(data_dir, filename) + + with open(file_path, 'r') as allfile: + files = allfile.read().splitlines() + + for file in files: + y, sr = librosa.load(file, sr=sampling_rate) + + for i in range( + 0, len(y) - sampling_rate, silence_stride * 100 + ): # stride * 100 to generate less samples for demo + file_name = "{}_{}.wav".format(file.split("/")[-1], i) + y_slice = y[i : i + sampling_rate] + magnitude = rng.uniform(0.0, 1.0) + y_slice *= magnitude + out_file_path = os.path.join(silence_path, file_name) + sf.write(out_file_path, y_slice, sr) + + silence_files.append(out_file_path) + + new_list_file = os.path.join(silence_path, filename) + with open(new_list_file, "w") as outfile: + outfile.write("\n".join(silence_files)) + + logging.info(f"Generate {len(out_file_path)} background files for {file_path}. => {new_list_file} !") + return len(silence_files) + + +def main(): + parser = argparse.ArgumentParser(description='Speech and backgound data download and preprocess') + parser.add_argument("--out_dir", required=False, default='./manifest/', type=str) + parser.add_argument("--speech_data_root", required=True, default=None, type=str) + parser.add_argument("--background_data_root", required=True, default=None, type=str) + parser.add_argument('--test_size', required=False, default=0.1, type=float) + parser.add_argument('--val_size', required=False, default=0.1, type=float) + parser.add_argument('--seg_len', required=False, default=0.63, type=float) + parser.add_argument('--log', required=False, action='store_true') + parser.add_argument('--rebalance_method', required=False, default=None, type=str) + parser.add_argument('--demo', required=False, action='store_true') + parser.set_defaults(log=False, generate=False) + args = parser.parse_args() + + if not args.rebalance_method: + rebalance = False + else: + if args.rebalance_method != 'over' and args.rebalance_method != 'under' and args.rebalance_method != 'fixed': + raise NameError("Please select a valid sampling method: over/under/fixed.") + else: + rebalance = True + + if args.log: + logging.basicConfig(level=logging.DEBUG) + + # Download speech data + speech_data_root = args.speech_data_root + data_set = "google_speech_recognition_v2" + speech_data_folder = os.path.join(speech_data_root, data_set) + + background_data_folder = args.background_data_root + logging.info(f"Working on: {data_set}") + + # Download and extract speech data + if not os.path.exists(speech_data_folder): + file_path = os.path.join(speech_data_root, data_set + ".tar.bz2") + logging.info(f"Getting {data_set}") + __maybe_download_file(file_path, URL) + logging.info(f"Extracting {data_set}") + __extract_all_files(file_path, speech_data_root, speech_data_folder) + + logging.info(f"Split speech data!") + # dataset provide testing.txt and validation.txt feel free to split data using that with process_google_speech_train + split_train_val_test(speech_data_folder, "speech", args.test_size, args.val_size, args.demo) + + logging.info(f"Split background data!") + split_train_val_test(background_data_folder, "background", args.test_size, args.val_size) + + out_dir = args.out_dir + + # Process Speech manifest + logging.info(f"=== Write speech data to manifest!") + skip_num_val, speech_seg_num_val, speech_val = load_list_write_manifest( + speech_data_folder, out_dir, 'validation_list.txt', 'speech', 0.2, 0.8, args.seg_len, args.seg_len + ) + skip_num_test, speech_seg_num_test, speech_test = load_list_write_manifest( + speech_data_folder, out_dir, 'testing_list.txt', 'speech', 0.2, 0.8, 0.01, args.seg_len + ) + skip_num_train, speech_seg_num_train, speech_train = load_list_write_manifest( + speech_data_folder, out_dir, 'training_list.txt', 'speech', 0.2, 0.8, args.seg_len, args.seg_len + ) + + logging.info(f'Val: Skip {skip_num_val} samples. Get {speech_seg_num_val} segments! => {speech_val} ') + logging.info(f'Test: Skip {skip_num_test} samples. Get {speech_seg_num_test} segments! => {speech_test}') + logging.info(f'Train: Skip {skip_num_train} samples. Get {speech_seg_num_train} segments!=> {speech_train}') + + # Process background manifest + # if we select to generate more background noise data + if args.demo: + logging.info("Start generating more background noise data") + generate_variety_noise(background_data_folder, 'validation_list.txt', 'background') + generate_variety_noise(background_data_folder, 'training_list.txt', 'background') + generate_variety_noise(background_data_folder, 'testing_list.txt', 'background') + background_data_folder = os.path.join( + background_data_folder.split("_background_noise_")[0], "_background_noise_more" + ) + + logging.info(f"=== Write background data to manifest!") + skip_num_val, background_seg_num_val, background_val = load_list_write_manifest( + background_data_folder, out_dir, 'validation_list.txt', 'background', 0, None, 0.15, args.seg_len + ) + skip_num_test, background_seg_num_test, background_test = load_list_write_manifest( + background_data_folder, out_dir, 'testing_list.txt', 'background', 0, None, 0.01, args.seg_len + ) + skip_num_train, background_seg_num_train, background_train = load_list_write_manifest( + background_data_folder, out_dir, 'training_list.txt', 'background', 0, None, 0.15, args.seg_len + ) + + logging.info(f'Val: Skip {skip_num_val} samples. Get {background_seg_num_val} segments! => {background_val}') + logging.info(f'Test: Skip {skip_num_test} samples. Get {background_seg_num_test} segments! => {background_test}') + logging.info( + f'Train: Skip {skip_num_train} samples. Get {background_seg_num_train} segments! => {background_train}' + ) + min_val, max_val = min(speech_seg_num_val, background_seg_num_val), max(speech_seg_num_val, background_seg_num_val) + min_test, max_test = ( + min(speech_seg_num_test, background_seg_num_test), + max(speech_seg_num_test, background_seg_num_test), + ) + min_train, max_train = ( + min(speech_seg_num_train, background_seg_num_train), + max(speech_seg_num_train, background_seg_num_train), + ) + + logging.info('Finish generating manifest!') + + if rebalance: + # Random Oversampling: Randomly duplicate examples in the minority class. + # Random Undersampling: Randomly delete examples in the majority class. + if args.rebalance_method == 'under': + logging.info(f"Rebalancing number of samples in classes using {args.rebalance_method} sampling.") + logging.info(f'Val: {min_val} Test: {min_test} Train: {min_train}!') + + rebalance_json(out_dir, background_val, min_val, 'balanced') + rebalance_json(out_dir, background_test, min_test, 'balanced') + rebalance_json(out_dir, background_train, min_train, 'balanced') + + rebalance_json(out_dir, speech_val, min_val, 'balanced') + rebalance_json(out_dir, speech_test, min_test, 'balanced') + rebalance_json(out_dir, speech_train, min_train, 'balanced') + + if args.rebalance_method == 'over': + logging.info(f"Rebalancing number of samples in classes using {args.rebalance_method} sampling.") + logging.info(f'Val: {max_val} Test: {max_test} Train: {max_train}!') + + rebalance_json(out_dir, background_val, max_val, 'balanced') + rebalance_json(out_dir, background_test, max_test, 'balanced') + rebalance_json(out_dir, background_train, max_train, 'balanced') + + rebalance_json(out_dir, speech_val, max_val, 'balanced') + rebalance_json(out_dir, speech_test, max_test, 'balanced') + rebalance_json(out_dir, speech_train, max_train, 'balanced') + + if args.rebalance_method == 'fixed': + fixed_test, fixed_val, fixed_train = 200, 100, 500 + logging.info(f"Rebalancing number of samples in classes using {args.rebalance_method} sampling.") + logging.info(f'Val: {fixed_val} Test: {fixed_test} Train: {fixed_train}!') + + rebalance_json(out_dir, background_val, fixed_val, 'balanced') + rebalance_json(out_dir, background_test, fixed_test, 'balanced') + rebalance_json(out_dir, background_train, fixed_train, 'balanced') + + rebalance_json(out_dir, speech_val, fixed_val, 'balanced') + rebalance_json(out_dir, speech_test, fixed_test, 'balanced') + rebalance_json(out_dir, speech_train, fixed_train, 'balanced') + else: + logging.info("Don't rebalance number of samples in classes.") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/scripts/export/convasr_to_enemo.py b/models/tacotron2/NeMo/scripts/export/convasr_to_enemo.py new file mode 100644 index 0000000000..908236f418 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/export/convasr_to_enemo.py @@ -0,0 +1,76 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import os +import tarfile +from shutil import copyfile + +from nemo.collections.asr.models import EncDecClassificationModel, EncDecCTCModel, EncDecSpeakerLabelModel +from nemo.utils import logging + + +def get_parser(): + parser = argparse.ArgumentParser(description="Convert .nemo file to onnx file (encoder and decoder fused)") + parser.add_argument( + "--nemo_file", default=None, type=str, required=True, help="Path to .nemo file", + ) + parser.add_argument( + "--onnx_file", default=None, type=str, required=True, help="Path to the onnx output", + ) + parser.add_argument( + "--enemo_file", default=None, type=str, required=True, help="Path to the enemo output, consiting of onnx file", + ) + parser.add_argument( + "--model_type", + default="asr", + type=str, + choices=["asr", "speech_label", "speaker"], + help="Type of decoder used by the model.", + ) + return parser + + +def main( + nemo_file, enemo_file, onnx_file, model_type="asr", +): + if model_type == "asr": + logging.info("Preparing ASR model") + model = EncDecCTCModel.restore_from(nemo_file) + elif model_type == "speech_label": + logging.info("Preparing Speech Label Classification model") + model = EncDecClassificationModel.restore_from(nemo_file) + elif model_type == "speaker": + logging.info("Preparing Speaker Recognition model") + model = EncDecSpeakerLabelModel.restore_from(nemo_file) + else: + raise NameError("Available model names are asr, speech_label and speaker") + + logging.info("Writing onnx file") + model.export(onnx_file, onnx_opset_version=12) + logging.info("succesfully ported onnx file") + + with tarfile.open(nemo_file, "r") as archive: + archive.extract("./model_config.yaml") + with tarfile.open(enemo_file, "w") as enemo_archive: + enemo_archive.add("./model_config.yaml") + copyfile(onnx_file, "model_graph.onnx") + enemo_archive.add("model_graph.onnx") + os.remove("model_graph.onnx") # cleanup extra file + + +if __name__ == "__main__": + args = get_parser().parse_args() + main( + args.nemo_file, args.enemo_file, args.onnx_file, model_type=args.model_type, + ) diff --git a/models/tacotron2/NeMo/scripts/export/convasr_to_onnx.py b/models/tacotron2/NeMo/scripts/export/convasr_to_onnx.py new file mode 100644 index 0000000000..2b92e98835 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/export/convasr_to_onnx.py @@ -0,0 +1,66 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse + +from nemo.collections.asr.models import EncDecClassificationModel, EncDecCTCModel, EncDecSpeakerLabelModel +from nemo.utils import logging + + +def get_parser(): + parser = argparse.ArgumentParser(description="Convert .nemo file to encoder decoder onnx files") + parser.add_argument( + "--nemo_file", default=None, type=str, required=True, help="Path to .nemo file", + ) + parser.add_argument( + "--onnx_encoder", default=None, type=str, required=True, help="Path to the onnx encoder output.", + ) + parser.add_argument( + "--onnx_decoder", default=None, type=str, required=True, help="Path to the onnx decoder output.", + ) + parser.add_argument( + "--model_type", + default='asr', + type=str, + choices=['asr', 'speech_label', 'speaker'], + help="Type of decoder used by the model.", + ) + return parser + + +def main( + nemo_file, onnx_encoder, onnx_decoder, model_type='asr', +): + if model_type == 'asr': + logging.info("Preparing encoder decoder for ASR model") + model = EncDecCTCModel.restore_from(nemo_file) + elif model_type == 'speech_label': + logging.info("Preparing encoder decoder for Speech Label Classification model") + model = EncDecClassificationModel.restore_from(nemo_file) + elif model_type == 'speaker': + logging.info("Preparing encoder decoder for Speaker Recognition model") + model = EncDecSpeakerLabelModel.restore_from(nemo_file) + else: + raise NameError("Available model names are asr, speech_label and speaker ") + + logging.info("Writing onnx encoder and decoder onnx files") + model.encoder.export(onnx_encoder) + model.decoder.export(onnx_decoder, onnx_opset_version=10) + logging.info("succesfully ported onnx files") + + +if __name__ == "__main__": + args = get_parser().parse_args() + main( + args.nemo_file, args.onnx_encoder, args.onnx_decoder, model_type=args.model_type, + ) diff --git a/models/tacotron2/NeMo/scripts/export/convasr_to_single_onnx.py b/models/tacotron2/NeMo/scripts/export/convasr_to_single_onnx.py new file mode 100644 index 0000000000..7c7321f723 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/export/convasr_to_single_onnx.py @@ -0,0 +1,62 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse + +from nemo.collections.asr.models import EncDecClassificationModel, EncDecCTCModel, EncDecSpeakerLabelModel +from nemo.utils import logging + + +def get_parser(): + parser = argparse.ArgumentParser(description="Convert .nemo file to onnx file (encoder and decoder fused)") + parser.add_argument( + "--nemo_file", default=None, type=str, required=True, help="Path to .nemo file", + ) + parser.add_argument( + "--onnx_file", default=None, type=str, required=True, help="Path to the onnx output", + ) + parser.add_argument( + "--model_type", + default='asr', + type=str, + choices=['asr', 'speech_label', 'speaker'], + help="Type of decoder used by the model.", + ) + return parser + + +def main( + nemo_file, onnx_file, model_type='asr', +): + if model_type == 'asr': + logging.info("Preparing ASR model") + model = EncDecCTCModel.restore_from(nemo_file) + elif model_type == 'speech_label': + logging.info("Preparing Speech Label Classification model") + model = EncDecClassificationModel.restore_from(nemo_file) + elif model_type == 'speaker': + logging.info("Preparing Speaker Recognition model") + model = EncDecSpeakerLabelModel.restore_from(nemo_file) + else: + raise NameError("Available model names are asr, speech_label and speaker") + + logging.info("Writing onnx file") + model.export(onnx_file, onnx_opset_version=12) + logging.info("succesfully ported onnx file") + + +if __name__ == "__main__": + args = get_parser().parse_args() + main( + args.nemo_file, args.onnx_file, model_type=args.model_type, + ) diff --git a/models/tacotron2/NeMo/scripts/freesound_download_resample/download_resample_freesound.sh b/models/tacotron2/NeMo/scripts/freesound_download_resample/download_resample_freesound.sh new file mode 100644 index 0000000000..af0a07f44d --- /dev/null +++ b/models/tacotron2/NeMo/scripts/freesound_download_resample/download_resample_freesound.sh @@ -0,0 +1,115 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#!/bin/bash + +# This is bash script actually run the downloading and resampling script. +# See instructions in freesound_download.py + +# Change this arguments if you want +page_size=100 # Number of sounds per page +max_samples=200 # Maximum number of sound samples +min_filesize=0 # Minimum filesize allowed (in MB) +max_filesize=100 # Maximum filesize allowed (in MB) + +if [[ $# -ne 3 ]]; then + echo "Require number of all files | data directory | resample data directory as arguments to the script" + exit 2 +fi + + +NUM_ALL_FILES=$1 +DATADIR=$2 +RESAMPLE_DATADIR=$3 + + +if [ ! -d "$DATADIR" ]; then + echo "Creating dir $DATADIR" + mkdir -p "$DATADIR" +fi + +if [ ! -d "$RESAMPLE_DATADIR" ]; then + echo "Creating dir $RESAMPLE_DATADIR" + mkdir -p "$RESAMPLE_DATADIR" +fi + +# we just need background categories for constructing dataset, feel free to include other (speech) categories for testing and training your VAD model +# background +categories=( + "Air brake" + "Static" + "Acoustic environment" + "Distortion" + "Tape hiss" + "Hubbub" + "Vibration" + "Cacophony" + "Throbbing" + "Reverberation" + "Inside, public space" + "Inside, small room" + "Echo" + "Outside, rural" + "Outside, natural" + "Outside, urban" + "Outside, manmade" + "Car" + "Bus" + "Traffic noise" + "Roadway noise" + "Truck" + "Emergency vehicle" + "Motorcycle" + "Aircraft engine" + "Aircraft" + "Helicopter" + "Bicycle" + "Skateboard" + "Subway, metro, underground" + "Railroad car" + "Train wagon" + "Train" + "Sailboat" + "Rowboat" + "Ship" +) + + +WAV_FILECOUNT="$(find $DATADIR -name '*.wav' -type f | wc -l)" +FLAC_FILECOUNT="$(find $DATADIR -name '*.flac' -type f | wc -l)" +FILECOUNT="$((WAV_FILECOUNT + FLAC_FILECOUNT))" +echo "File count: " $FILECOUNT + + +while((FILECOUNT <= NUM_ALL_FILES)) +do + for category in "${categories[@]}" + do + python freesound_download.py --data_dir "${DATADIR}" --category "${category}" --page_size "${page_size}" --max_samples "${max_samples}" --min_filesize "${min_filesize}" --max_filesize "${max_filesize}" + ret=$? + if [ $ret -ne 0 ]; then + exit 1 + fi + done + + WAV_FILECOUNT="$(find $DATADIR -name '*.wav' -type f | wc -l)" + FLAC_FILECOUNT="$(find $DATADIR -name '*.flac' -type f | wc -l)" + FILECOUNT="$((WAV_FILECOUNT + FLAC_FILECOUNT))" + echo "Current file count is: " $FILECOUNT +done + +# RESAMPLE +echo "Got enough data. Start resample!" +python freesound_resample.py --data_dir="${DATADIR}" --resampled_dir="${RESAMPLE_DATADIR}" + +echo "Done resample data!" diff --git a/models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_download.py b/models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_download.py new file mode 100644 index 0000000000..076c69bfa6 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_download.py @@ -0,0 +1,585 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import os +import pickle +import time + +try: + import librosa + import requests + import requests_oauthlib + from joblib import Parallel, delayed + from oauthlib.oauth2 import TokenExpiredError +except (ModuleNotFoundError, ImportError) as e: + raise e + +try: + import freesound +except ModuleNotFoundError as e: + raise ModuleNotFoundError( + "freesound is not installed. Execute `pip install --no-cache-dir git+https://github.com/MTG/freesound-python.git` in terminal" + ) + + +""" +Instructions +1. We will need some requirements including freesound, requests, requests_oauthlib, joblib, librosa and sox. If they are not installed, please run `pip install -r freesound_requirements.txt` +2. Create an API key for freesound.org at https://freesound.org/help/developers/ +3. Create a python file called `freesound_private_apikey.py` and add lined `api_key = ` and `client_id = ` +4. Authorize by run `python freesound_download.py --authorize` and visit website, and paste response code +5. Feel free to change any arguments in download_resample_freesound.sh such as max_samples and max_filesize +6. Run `bash download_resample_freesound.sh ` +""" + +# Import the API Key +try: + from freesound_private_apikey import api_key, client_id + + print("API Key found !") +except ImportError: + raise ImportError( + "Create a python file called `freesound_private_apikey.py` and add lined `api_key = ` and `client_id = `" + ) + +auth_url = 'https://freesound.org/apiv2/oauth2/authorize/' +redirect_url = 'https://freesound.org/home/app_permissions/permission_granted/' +token_url = 'https://freesound.org/apiv2/oauth2/access_token/' +scope = ["read", "write"] + +BACKGROUND_CLASSES = [ + "Air brake", + "Static", + "Acoustic environment", + "Distortion", + "Tape hiss", + "Hubbub", + "Vibration", + "Cacophony", + "Throbbing", + "Reverberation", + "Inside, public space", + "Inside, small room", + "Echo", + "Outside, rural", + "Outside, natural", + "Outside, urban", + "Outside, manmade", + "Car", + "Bus", + "Traffic noise", + "Roadway noise", + "Truck", + "Emergency vehicle", + "Motorcycle", + "Aircraft engine", + "Aircraft", + "Helicopter", + "Bicycle", + "Skateboard", + "Subway, metro, underground", + "Railroad car", + "Train wagon", + "Train", + "Sailboat", + "Rowboat", + "Ship", +] + +SPEECH_CLASSES = [ + "Male speech", + "Female speech", + "Speech synthesizer", + "Babbling", + "Conversation", + "Child speech", + "Narration", + "Laughter", + "Yawn", + "Whispering", + "Whimper", + "Baby cry", + "Sigh", + "Groan", + "Humming", + "Male singing", + "Female singing", + "Child singing", + "Children shouting", +] + + +def initialize_oauth(): + # If token already exists, then just load it + if os.path.exists('_token.pkl'): + token = unpickle_object('_token') + oauth = requests_oauthlib.OAuth2Session(client_id, redirect_uri=redirect_url, scope=scope, token=token) + + else: + # Construct a new token after OAuth2 flow + # Initialize a OAuth2 session + oauth = requests_oauthlib.OAuth2Session(client_id, redirect_uri=redirect_url, scope=scope) + + authorization_url, state = oauth.authorization_url(auth_url) + print(f"Visit below website and paste access token below : \n\n{authorization_url}\n") + + authorization_response = input("Paste authorization response code here :\n") + + token = oauth.fetch_token( + token_url, + authorization_response=authorization_response, + code=authorization_response, + client_secret=api_key, + ) + + # Save the token generated + pickle_object(token, '_token') + + return oauth, token + + +def instantiate_session(): + # Reconstruct session in process, and force singular execution thread to reduce session + # connections to server + token = unpickle_object('_token') + session = requests_oauthlib.OAuth2Session(client_id, redirect_uri=redirect_url, scope=scope, token=token) + adapter = requests.adapters.HTTPAdapter(pool_connections=1, pool_maxsize=1) + session.mount('http://', adapter) + return session + + +def refresh_token(session): + print("Refreshing tokens...") + # Token expired, perform token refresh + extras = {'client_id': client_id, 'client_secret': api_key} + token = session.refresh_token(token_url, **extras) + print("Token refresh performed...") + # Save the refreshed token + pickle_object(token, '_token') + return session + + +def pickle_object(token, name): + with open(name + '.pkl', 'wb') as f: + pickle.dump(token, f) + + +def unpickle_object(name): + fp = name + '.pkl' + if os.path.exists(fp): + with open(fp, 'rb') as f: + token = pickle.load(f) + + return token + else: + raise FileNotFoundError('Token not found!') + + +def is_resource_limited(e: freesound.FreesoundException): + """ + Test if the reason for a freesound exception was either rate limit + or daily limit. + + If it was for either reason, sleep for an appropriate delay and return + to try again. + + Args: + e: Freesound Exception object + + Returns: + A boolean which describes whether the error was due to some + api limit issue, or if it was some other reason. + + If false is returned, then the user should carefully check the cause + and log it. + """ + detail = e.detail['detail'] + + if '2000' in detail: + # This is the request limit, hold off for 1 hour and try again + print(f"Hit daily limit, sleeping for 20 minutes.") + time.sleep(60 * 20) + return True + + elif '60' in detail: + # This is the request limit per minute, hold off for 1 minute and try again + print(f"Hit rate limit, sleeping for 1 minute.") + time.sleep(60) + return True + + else: + return False + + +def prepare_client(client: freesound.FreesoundClient, token) -> freesound.FreesoundClient: + # Initialize the client with token auth + client.set_token(token['access_token'], auth_type='oauth') + print("Client ready !") + return client + + +def get_text_query_with_resource_limit_checks(client, query: str, filters: list, fields: str, page_size: int): + """ + Performs a text query, checks for rate / api limits, and retries. + + Args: + client: FreesoundAPI client + query: query string (either exact or inexact) + filters: list of string filters + fields: String of values to recover + page_size: samples per page returned + + Returns: + + """ + pages = None + attempts = 20 + + while pages is None: + try: + pages = client.text_search(query=query, filter=" ".join(filters), fields=fields, page_size=str(page_size),) + + except freesound.FreesoundException as e: + # Most probably a rate limit or a request limit + # Check if that was the case, and wait appropriate ammount of time + # for retry + was_resource_limited = is_resource_limited(e) + + # If result of test False, it means that failure was due to some other reason. + # Log it, then break loop + if not was_resource_limited: + print(e.with_traceback(None)) + break + + attempts -= 1 + + # Attempt to refresh tokens if it fails multiple times + if attempts % 5 == 0 and attempts > 0: + session = instantiate_session() + refresh_token(session) + session.close() + token = unpickle_object('_token') + client = prepare_client(client, token) + + if attempts <= 0: + print(f"Failed to query pages for '{query}' after 10 attempts, skipping query") + break + + if pages is None: + print(f"Query attempts remaining = {attempts}") + + return client, pages + + +def get_resource_with_auto_refresh(session, download_url): + """ + Attempts download of audio with a token refresh if necessary. + """ + try: + result = session.get(download_url) + + except TokenExpiredError as e: + session = refresh_token(session) + result = session.get(download_url) + + except Exception as e: + result = None + + print(f"Skipping file {download_url} due to exception below\n\n") + print(e) + + return result.content + + +def download_song(basepath, id, name, download_url): + # Cleanup name + name = name.replace(":", "-") + name = name.replace(",", "-") + name = name.replace("/", "-") + name = name.replace("\\", "-") + name = name.replace(".", "-") + name = name.replace(" ", "") + + # Correct last `.` for filetype + name = name[:-4] + '.wav' + + # Add file id to filename + name = f"id_{id} " + name + + fp = os.path.join(basepath, name) + + # Check if file, if exists already, can be loaded by librosa + # If it cannot be loaded, possibly corrupted file. + # Delete and then re-download + if os.path.exists(fp): + try: + _ = librosa.load(fp) + except Exception: + # File is currupted, delete and re-download. + os.remove(fp) + + print(f"Pre-existing file {fp} was corrupt and was deleted, will be re-downloaded.") + + if not os.path.exists(fp): + print("Downloading file :", name) + + session = instantiate_session() + + data = None + attempts = 10 + + try: + while data is None: + + try: + # Get the sound data + data = get_resource_with_auto_refresh(session, download_url) + + except freesound.FreesoundException as e: + # Most probably a rate limit or a request limit + # Check if that was the case, and wait appropriate amount of time + # for retry + was_resource_limited = is_resource_limited(e) + + # If result of test False, it means that failure was due to some other reason. + # Log it, then break loop + if not was_resource_limited: + print(e) + break + + attempts -= 1 + + if attempts <= 0: + print(f"Failed to download file {fp} after 10 attempts, skipping file") + break + + if data is None: + print(f"Download attempts remaining = {attempts}") + + finally: + session.close() + + # Write the data to file + if data is not None: + print("Downloaded file :", name) + + with open(fp, 'wb') as f: + f.write(data) + + # If file size is less than 89, then this probably is a text format and not an actual audio file. + if os.path.getsize(fp) > 89: + print(f"File written : {fp}") + + else: + os.remove(fp) + print(f"File corrupted and has been deleted: {fp}") + + else: + print(f"File [{fp}] corrupted or faced some issue when downloading, skipped.") + + # Sleep to avoid hitting rate limits + time.sleep(5) + + else: + print(f"File [{fp}] already exists in dataset, skipping re-download.") + + +def get_songs_by_category( + client: freesound.FreesoundClient, + category: str, + data_dir: str, + max_num_samples=100, + page_size=100, + min_filesize_in_mb=0, + max_filesize_in_mb=10, + n_jobs=None, +): + """ + Download songs of a category with restrictions + + Args: + client: FreesoundAPI client + category: category to be downloaded + data_dir: directory of downloaded songs + max_num_samples: maximum number of samples of this category + page_size: samples per page returned + min_filesize_in_mb: minimum filesize of the song in MB + max_filesize_in_mb: maximum filesize of the song in MB + n_jobs: number of jobs for parallel processing + + Returns: + + """ + # quote string to force exact match + query = f'"{category}"' + print(f"Query : {query}") + + page_size = min(page_size, 150) + max_filesize = int(max_filesize_in_mb * (2 ** 20)) + + if min_filesize_in_mb == 0: + min_filesize_in_mb = 1 + else: + min_filesize_in_mb = int(min_filesize_in_mb * (2 ** 20)) + + if max_num_samples < 0: + max_num_samples = int(1e6) + + filters = [ + 'type:(wav OR flac)', + 'license:("Attribution" OR "Creative Commons 0")', + f'filesize:[{min_filesize_in_mb} TO {max_filesize}]', + ] + + fields = "id,name,download,license" + + client, pages = get_text_query_with_resource_limit_checks( + client, query=query, filters=filters, fields=fields, page_size=page_size + ) + + if pages is None: + print(f"Number of attempts exceeded limit, skipping query {query}") + return + + num_pages = pages.count + + # Check if returned empty result; if so, fallback to inexact category search + if num_pages == 0: + print(f"Found 0 samples of results for query '{query}'") + print(f"Trying less restricted query : {category}") + + client, pages = get_text_query_with_resource_limit_checks( + client, query=category, filters=filters, fields=fields, page_size=page_size + ) + + if pages is None: + print(f"Number of attempts exceeded limit, skipping query {query}") + return + + num_pages = pages.count + + print(f"Found {num_pages} samples of results for query '{query}'") + + category = category.replace(' ', '_') + basepath = os.path.join(data_dir, category) + + if not os.path.exists(basepath): + os.makedirs(basepath) + + sounds = [] + sample_count = 0 + + # Retrieve sound license information + with open(os.path.join(basepath, 'licenses.txt'), 'w') as f: + f.write("ID,LICENSE\n") + f.flush() + + while True: + for sound in pages: + if sample_count >= max_num_samples: + print( + f"Collected {sample_count} samples, which is >= max number of samples requested " + f"{max_num_samples}. Stopping for this category : {category}" + ) + break + + sounds.append(sound) + sample_count += 1 + + f.write(f"{sound.id},{sound.license}\n") + f.flush() + + if sample_count >= max_num_samples: + break + + try: + pages = pages.next_page() + except ValueError: + break + + if n_jobs is None: + n_jobs = max(1, len(sounds)) + + # Parallel download all songs + with Parallel(n_jobs=n_jobs, verbose=10) as parallel: + _ = parallel(delayed(download_song)(basepath, sound.id, sound.name, sound.download) for sound in sounds) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description="Freesound download script") + + parser.add_argument( + '--authorize', action='store_true', dest='auth', help='Flag to only perform OAuth2 authorization step' + ) + + parser.add_argument('-c', '--category', default='', type=str, help='Category required to download') + + parser.add_argument('-d', '--data_dir', default='', type=str, help='Destination folder to store data') + + parser.add_argument('--page_size', default=100, type=int, help='Number of sounds per page') + + parser.add_argument('--max_samples', default=100, type=int, help='Maximum number of sound samples') + + parser.add_argument('--min_filesize', default=0, type=int, help='Maximum filesize allowed (in MB)') + + parser.add_argument('--max_filesize', default=20, type=int, help='Maximum filesize allowed (in MB)') + + parser.set_defaults(auth=False) + + args = parser.parse_args() + + if args.auth: + """ Initialize oauth token to be used by all """ + oauth, token = initialize_oauth() + oauth.close() + + print("Authentication suceeded ! Token stored in `_token.pkl`") + exit(0) + + if not os.path.exists('_token.pkl'): + raise FileNotFoundError( + "Please authorize the application first using " "`python freesound_download.py --authorize`" + ) + if args.data_dir == '': + raise ValueError("Data dir must be passed as an argument using `--data_dir`") + + data_dir = args.data_dir + + page_size = args.page_size + max_num_samples = args.max_samples + min_filesize_in_mb = args.min_filesize + max_filesize_in_mb = args.max_filesize + + # Initialize and authenticate client + token = unpickle_object('_token') + freesound_client = freesound.FreesoundClient() + client = prepare_client(freesound_client, token) + + category = args.category + + if category == '': + raise ValueError("Cannot pass empty string as it will select all of FreeSound data !") + + print(f"Downloading category : {category}") + get_songs_by_category( + client, + category, + data_dir=data_dir, + max_num_samples=max_num_samples, + page_size=page_size, + min_filesize_in_mb=min_filesize_in_mb, + max_filesize_in_mb=max_filesize_in_mb, + n_jobs=30, + ) diff --git a/models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_requirements.txt b/models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_requirements.txt new file mode 100644 index 0000000000..127e50992a --- /dev/null +++ b/models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_requirements.txt @@ -0,0 +1,6 @@ +git+git://github.com/MTG/freesound-python.git +requests +requests_oauthlib +joblib +librosa +sox \ No newline at end of file diff --git a/models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_resample.py b/models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_resample.py new file mode 100644 index 0000000000..79a01c0946 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/freesound_download_resample/freesound_resample.py @@ -0,0 +1,119 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import glob +import os +import time + +import librosa +import sox +from joblib import Parallel, delayed + + +def resample_file(resampled_dir, filepath, ext, sample_rate): + """ + Resample an audio file to 16kHZ and transform to monochannel + Remove incompatible files. + + Args: + resampled_dir: Directory of transformed files. + filepath: Filepath of Audio + ext: File type e.g. "wav", "flac" + + Returns: + + """ + head, filename = os.path.split(filepath) + _, clsname = os.path.split(head) + + filename, _ = os.path.splitext(filename) + + new_dir = os.path.join(resampled_dir, clsname) + if not os.path.exists(new_dir): + os.makedirs(new_dir) + + new_path = os.path.join(new_dir, filename + f'.{ext}') + + # check if the resampled data exists. + if os.path.exists(new_path): + print(f"Resampled file {filepath} exists. Skip it.") + return None + + transform = sox.Transformer() + transform.set_output_format(file_type='wav') + transform.convert(samplerate=sample_rate, n_channels=1) + + try: + transform.build(filepath, new_path) + print(f"Finished converting file {filepath}.") + + return None + + except sox.core.SoxError as e: + + try: + # Check if the file is readable + librosa.load(filepath) + + # if it is, force input format and try again + transform.set_input_format(file_type=ext) + transform.build(filepath, new_path) + return None + + except Exception: + return filepath + + +def main(): + start = time.time() + parser = argparse.ArgumentParser(description='Freesound data resample') + parser.add_argument("--data_dir", required=True, default=None, type=str) + parser.add_argument('--resampled_dir', required=True, default=None, type=str) + parser.add_argument('--sample_rate', default=16000, type=int) + args = parser.parse_args() + + data_dir = args.data_dir + resampled_dir = args.resampled_dir + sample_rate = args.sample_rate + + wav_files = sorted(glob.glob(os.path.join(data_dir, '*/*.wav'))) + flac_files = sorted(glob.glob(os.path.join(data_dir, '*/*.flac'))) + + with Parallel(n_jobs=-1, verbose=10) as parallel: + wav_files_failed = parallel( + delayed(resample_file)(resampled_dir, filepath, ext='wav', sample_rate=sample_rate) + for filepath in wav_files + ) + + flac_files_failed = parallel( + delayed(resample_file)(resampled_dir, filepath, ext='flac', sample_rate=sample_rate) + for filepath in flac_files + ) + + with open('dataset_conversion_logs.txt', 'w') as f: + for file in wav_files_failed: + if file is not None: + f.write(f"{file}\n") + + for file in flac_files_failed: + if file is not None: + f.write(f"{file}\n") + + end = time.time() + print(f'Resample data in {data_dir} and save to {resampled_dir} takes {end-start} seconds.') + + +if __name__ == '__main__': + + main() diff --git a/models/tacotron2/NeMo/scripts/nemo_legacy_import/asr_checkpoint_port.py b/models/tacotron2/NeMo/scripts/nemo_legacy_import/asr_checkpoint_port.py new file mode 100644 index 0000000000..f9b7387ffb --- /dev/null +++ b/models/tacotron2/NeMo/scripts/nemo_legacy_import/asr_checkpoint_port.py @@ -0,0 +1,72 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This script converts old Jasper/QuartzNet models from NeMo 0.11.* to NeMo v1.0.0* +""" + +import argparse + +import torch +from omegaconf import DictConfig +from ruamel.yaml import YAML + +import nemo.collections.asr as nemo_asr +from nemo.utils import logging + + +def get_parser(): + parser = argparse.ArgumentParser(description="Converts old Jasper/QuartzNet models to NeMo v1.0beta") + parser.add_argument("--config_path", default=None, required=True, help="Path to model config (NeMo v1.0beta)") + parser.add_argument("--encoder_ckpt", default=None, required=True, help="Encoder checkpoint path") + parser.add_argument("--decoder_ckpt", default=None, required=True, help="Decoder checkpoint path") + parser.add_argument("--output_path", default=None, required=True, help="Output checkpoint path (should be .nemo)") + parser.add_argument( + "--model_type", + default='asr', + type=str, + choices=['asr', 'speech_label', 'speaker'], + help="Type of decoder used by the model.", + ) + + return parser + + +def main(config_path, encoder_ckpt, decoder_ckpt, output_path, model_type): + + yaml = YAML(typ='safe') + with open(config_path) as f: + params = yaml.load(f) + + model = None + if model_type == 'asr': + logging.info("Creating ASR NeMo 1.0 model") + model = nemo_asr.models.EncDecCTCModel(cfg=DictConfig(params['model'])) + elif model_type == 'speech_label': + logging.info("Creating speech label NeMo 1.0 model") + model = nemo_asr.models.EncDecClassificationModel(cfg=DictConfig(params['model'])) + else: + logging.info("Creating Speaker Recognition NeMo 1.0 model") + model = nemo_asr.models.EncDecSpeakerLabelModel(cfg=DictConfig(params['model'])) + + model.encoder.load_state_dict(torch.load(encoder_ckpt)) + model.decoder.load_state_dict(torch.load(decoder_ckpt)) + logging.info("Succesfully ported old checkpoint") + + model.save_to(output_path) + logging.info("new model saved at {}".format(output_path)) + + +if __name__ == "__main__": + args = get_parser().parse_args() + main(args.config_path, args.encoder_ckpt, args.decoder_ckpt, args.output_path, args.model_type) diff --git a/models/tacotron2/NeMo/scripts/neural_machine_translation/filter_langs_nmt.py b/models/tacotron2/NeMo/scripts/neural_machine_translation/filter_langs_nmt.py new file mode 100644 index 0000000000..73634063d2 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/neural_machine_translation/filter_langs_nmt.py @@ -0,0 +1,403 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import multiprocessing as mp +import re +import shutil +import warnings +from pathlib import Path +from time import sleep + +import fasttext +from tqdm import tqdm + +""" +Usage: +python filter_by_language.py --input-src train.en \ + --input-tgt train.de \ + --output-src train_lang_filtered.en \ + --output-tgt train_lang_filtered.de \ + --source-lang en \ + --target-lang de \ + --removed-src train_garbage.en \ + --removed-tgt train_garbage.de \ + --fasttext-model lid.176.bin +""" + + +logging.basicConfig(level=logging.INFO) + + +def get_args(): + parser = argparse.ArgumentParser( + description="It is a script for verifying language in machine translation data sets. If the script is used on " + "a parallel corpus, it verifies both a source and a target language. If number of jobs `--num-jobs` is bigger " + "than 1 than lines in an input file (or files if parallel corpus is checked) split equally between workers. " + "If `num_jobs > 1` is used, the best performance is achieved if dataset is shuffled and lines with different " + "lengths are distributed evenly in the input file. Filtered data is stored into `output_src`[, `--output-tgt`]" + " and removed lines are put into `removed_src`[, `--removed-tgt`] files. If language cannot be detected " + "(e.g. date), the line is removed. Working time on en-de wikimatrix (6.23M pairs: 700 MB German and 625 MB " + "English) from wmt20 on machine with 20 CPU cores: less than 1 minute." + ) + parser.add_argument( + "--input-src", + "-s", + help="Path to the input file which has to contain text in language `source_lang`.", + required=True, + type=Path, + ) + parser.add_argument( + "--input-tgt", + "-t", + help="Path to the input file which has to contain text in language `target_lang`. If not provided, data is " + "processed as monolingual.", + type=Path, + ) + parser.add_argument( + "--output-src", + "-S", + help="Path to the file where filtered `input_src` will be saved.", + required=True, + type=Path, + ) + parser.add_argument( + "--output-tgt", "-T", help="Path to the output target file", type=Path, + ) + parser.add_argument( + "--source-lang", + "-l", + required=True, + help="Input language. For options see https://fasttext.cc/docs/en/language-identification.html.", + ) + parser.add_argument( + "--target-lang", + "-L", + help="Output language. For options see https://fasttext.cc/docs/en/language-identification.html.", + ) + parser.add_argument( + "--removed-src", "-r", required=True, help="Path to file where removed source lines will be saved", type=Path, + ) + parser.add_argument( + "--removed-tgt", "-R", help="Path to file where removed target lines will be saved", type=Path, + ) + parser.add_argument( + "--num-jobs", + "-j", + type=int, + help="Number of jobs. By default, the number of jobs is equal to the number of CPU cores.", + ) + parser.add_argument( + "--fasttext-model", + "-m", + help="Path to fasttext model. The description and download links are here " + "https://fasttext.cc/docs/en/language-identification.html", + type=Path, + ) + args = parser.parse_args() + if not ( + args.output_tgt is None + and args.input_tgt is None + and args.target_lang is None + and args.removed_tgt is None + or args.output_tgt is not None + and args.input_tgt is not None + and args.target_lang is not None + and args.removed_tgt is not None + ): + raise ValueError( + f"Arguments `input_tgt`, `output_tgt`, `target_lang`, `removed_tgt` have to be either `None` " + f"simultaneously or not `None` simultaneously. Given " + f"input_tgt={args.input_tgt}, output_tgt={args.output_tgt}, target_lang={args.target_lang}, " + f"removed_tgt={args.removed_tgt}" + ) + args.input_src = args.input_src.expanduser() + if args.input_tgt is not None: + args.input_tgt = args.input_tgt.expanduser() + args.output_src = args.output_src.expanduser() + if args.output_tgt is not None: + args.output_tgt = args.output_tgt.expanduser() + args.removed_src = args.removed_src.expanduser() + if args.removed_tgt is not None: + args.removed_tgt = args.removed_tgt.expanduser() + args.fasttext_model = args.fasttext_model.expanduser() + return args + + +def get_lang(line, fasttext_model): + labels, _ = fasttext_model.predict(line, k=1) + lang = labels[0].split('__')[-1] + return lang + + +def get_edges_in_1_file(fn, num_parts): + num_lines = 0 + edges = [0] + with open(fn) as f: + i = 0 + for l in f: + i += len(l.encode('utf-8')) + edges.append(i) + num_lines += 1 + return [edges[int(i * num_lines / num_parts)] for i in range(num_parts)] + [edges[-1]], num_lines + + +def get_edges_and_num_lines(src_fn, tgt_fn, num_parts): + src_edges, src_num_lines = get_edges_in_1_file(src_fn, num_parts) + assert num_parts + 1 == len(src_edges) + src_edges = [(src_edges[i], src_edges[i + 1]) for i in range(len(src_edges) - 1)] + if tgt_fn is not None: + tgt_edges, tgt_num_lines = get_edges_in_1_file(tgt_fn, num_parts) + tgt_edges = [(tgt_edges[i], tgt_edges[i + 1]) for i in range(len(tgt_edges) - 1)] + if tgt_num_lines != src_num_lines: + raise ValueError( + f"Source {repr(src_fn)} and target {repr(tgt_fn)} files have different number of lines " + f"{src_num_lines} and {tgt_num_lines} correspondingly." + ) + + else: + tgt_edges = [None] * num_parts + assert len(src_edges) == num_parts + return src_edges, tgt_edges, src_num_lines + + +def filter_pairs( + src_edges, + tgt_edges, + input_src, + input_tgt, + filtered_dir_src, + filtered_dir_tgt, + removed_dir_src, + removed_dir_tgt, + source_lang, + target_lang, + fasttext_model, + rank, +): + global counter + fasttext_model = fasttext.load_model(str(fasttext_model)) + output_src = filtered_dir_src / Path(f"rank{rank}") + output_src_removed = removed_dir_src / Path(f"rank{rank}") + output_tgt = filtered_dir_tgt / Path(f"rank{rank}") + output_tgt_removed = removed_dir_tgt / Path(f"rank{rank}") + with open(input_src) as in_src, open(input_tgt) as in_tgt, open(output_src, 'w') as out_src, open( + output_tgt, 'w' + ) as out_tgt, open(output_src_removed, 'w') as out_r_src, open(output_tgt_removed, 'w') as out_r_tgt: + in_src.seek(src_edges[0]) + in_tgt.seek(tgt_edges[0]) + src_l, tgt_l, i = in_src.readline(), in_tgt.readline(), 0 + if in_src.tell() > src_edges[1] or in_tgt.tell() > tgt_edges[1]: + return + while src_l and tgt_l: + with counter.get_lock(): + counter.value += 1 + src_l = src_l.strip() + tgt_l = tgt_l.strip() + src_lang = get_lang(src_l, fasttext_model) + if src_lang is not None: + tgt_lang = get_lang(tgt_l, fasttext_model) + if src_lang is None or tgt_lang is None or src_lang != source_lang or tgt_lang != target_lang: + out_r_src.write(src_l + '\n') + out_r_tgt.write(tgt_l + '\n') + else: + out_src.write(src_l + '\n') + out_tgt.write(tgt_l + '\n') + if in_src.tell() >= src_edges[1]: + if in_tgt.tell() < tgt_edges[1]: + raise ValueError( + f"Edges of target and source has to be reached simultaneously, whereas " + f"in_src.tell()={in_src.tell()}, in_tgt.tell()={in_tgt.tell()}, " + f"src_edges[1]={src_edges[1]}, tgt_edges[1]={tgt_edges[1]}." + ) + break + if in_tgt.tell() >= tgt_edges[1]: + raise ValueError( + f"Edges of target and source has to be reached simultaneously, whereas " + f"in_src.tell()={in_src.tell()}, in_tgt.tell()={in_tgt.tell()}, " + f"src_edges[1]={src_edges[1]}, tgt_edges[1]={tgt_edges[1]}." + ) + src_l, tgt_l, i = in_src.readline(), in_tgt.readline(), i + 1 + with counter.get_lock(): + counter.value += 1 + + +def filter_singles( + src_edges, input_src, filtered_dir_src, removed_dir_src, source_lang, fasttext_model, rank, +): + logging.debug("filter singles") + global counter + fasttext_model = fasttext.load_model(str(fasttext_model)) + output_src = filtered_dir_src / Path(f"rank{rank}") + output_src_removed = removed_dir_src / Path(f"rank{rank}") + with open(input_src) as in_f, open(output_src, 'w') as out_f, open(output_src_removed, 'w') as out_r_f: + in_f.seek(src_edges[0]) + i, line = 0, in_f.readline() + if in_f.tell() > src_edges[1]: + return + while line: + with counter.get_lock(): + counter.value += 1 + line = line.strip() + in_lang = get_lang(line, fasttext_model) + if in_lang is None or in_lang != source_lang: + out_r_f.write(line + '\n') + else: + out_f.write(line + '\n') + if in_f.tell() >= src_edges[1]: + break + i, line = i + 1, in_f.readline() + with counter.get_lock(): + counter.value += 1 + + +def filter_by_lang(args): + ( + src_edges, + tgt_edges, + input_src, + input_tgt, + filtered_dir_src, + filtered_dir_tgt, + removed_dir_src, + removed_dir_tgt, + source_lang, + target_lang, + fasttext_model, + rank, + ) = args + logging.debug(f"filter by lang input_tgt: {input_tgt}") + if input_tgt is None: + if tgt_edges is not None: + warnings.warn("If input target is not provided `tgt_edges` argument is expected to be `None`") + filter_singles( + src_edges, input_src, filtered_dir_src, removed_dir_src, source_lang, fasttext_model, rank, + ) + else: + filter_pairs( + src_edges, + tgt_edges, + input_src, + input_tgt, + filtered_dir_src, + filtered_dir_tgt, + removed_dir_src, + removed_dir_tgt, + source_lang, + target_lang, + fasttext_model, + rank, + ) + + +def _cat_results(out_file, tmp_dir): + file_name_pattern = re.compile(r"/rank([1-9][0-9]*)|0$") + with out_file.open('w') as out_f: + for f in sorted(tmp_dir.iterdir()): + if not f.is_file(): + warnings.warn(f"Unexpected not file {f}") + elif not file_name_pattern.search(str(f)): + warnings.warn(f"Unexpected file {f}") + else: + with f.open('r') as in_f: + for l in in_f: + out_f.write(l) + + +def cat_results(out_files, tmp_dirs): + for o_f, t_d in zip(out_files, tmp_dirs): + if o_f is None or t_d is None: + if o_f is not None or t_d is not None: + warnings.warn( + f"Output file and tmp directory are expected to be `None` simultaneously whereas tmp directory " + f"is {t_d} and output file is {o_f}." + ) + else: + _cat_results(o_f, t_d) + + +counter = None + + +def init(args): + global counter + counter = args + + +def main(): + args = get_args() + tmp_dir = Path("tmp") + i = 0 + while tmp_dir.exists(): + tmp_dir = Path("tmp" + str(i)) + i += 1 + tmp_filtered = tmp_dir / Path("filtered") + tmp_filtered_src = tmp_filtered / Path("src") + tmp_filtered_src.mkdir(parents=True, exist_ok=True) + if args.input_tgt is None: + tmp_filtered_tgt = None + else: + tmp_filtered_tgt = tmp_filtered / Path("tgt") + tmp_filtered_tgt.mkdir(parents=True, exist_ok=True) + tmp_removed = tmp_dir / Path("removed") + tmp_removed_src = tmp_removed / Path("src") + tmp_removed_src.mkdir(parents=True, exist_ok=True) + if args.input_tgt is None: + tmp_removed_tgt = None + else: + tmp_removed_tgt = tmp_removed / Path("tgt") + tmp_removed_tgt.mkdir(parents=True, exist_ok=True) + num_jobs = mp.cpu_count() if args.num_jobs is None else args.num_jobs + src_edges, tgt_edges, num_lines = get_edges_and_num_lines(args.input_src, args.input_tgt, num_jobs) + global counter + counter = mp.Value('i', 0) + t = tqdm(total=num_lines, desc="processed lines / total number of lines") + with mp.Pool(num_jobs, initializer=init, initargs=(counter,)) as pool: + async_result = pool.map_async( + filter_by_lang, + [ + ( + se, + te, + args.input_src, + args.input_tgt, + tmp_filtered_src, + tmp_filtered_tgt, + tmp_removed_src, + tmp_removed_tgt, + args.source_lang, + args.target_lang, + args.fasttext_model, + rank, + ) + for rank, (se, te) in enumerate(zip(src_edges, tgt_edges)) + ], + ) + while not async_result.ready(): + t.update(counter.value) + with counter.get_lock(): + counter.value = 0 + sleep(0.1) + t.update(counter.value) + + cat_results( + [args.output_src, args.output_tgt, args.removed_src, args.removed_tgt], + [tmp_filtered_src, tmp_filtered_tgt, tmp_removed_src, tmp_removed_tgt], + ) + shutil.rmtree(tmp_dir) + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/scripts/speaker_recognition/rttm_to_manifest.py b/models/tacotron2/NeMo/scripts/speaker_recognition/rttm_to_manifest.py new file mode 100644 index 0000000000..3d7b772b05 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/speaker_recognition/rttm_to_manifest.py @@ -0,0 +1,40 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from nemo.collections.asr.parts.utils.speaker_utils import write_rttm2manifest +from nemo.utils import logging + +""" +This file converts vad outputs to manifest file for speaker diarization purposes +present in vad output directory. +every vad line consists of start_time, end_time , speech/non-speech +""" + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--paths2rttm_files", help="path to vad output rttm-like files. Could be a list or a text file", required=True + ) + parser.add_argument( + "--paths2audio_files", + help="path to audio files that vad was computed. Could be a list or a text file", + required=True, + ) + parser.add_argument("--manifest_file", help="output manifest file name", type=str, required=True) + args = parser.parse_args() + + write_rttm2manifest(args.paths2audio_files, args.paths2rttm_files, args.manifest_file) + logging.info("wrote {} file from vad output files present in {}".format(args.manifest_file, args.paths2rttm_files)) diff --git a/models/tacotron2/NeMo/scripts/speaker_recognition/scp_to_manifest.py b/models/tacotron2/NeMo/scripts/speaker_recognition/scp_to_manifest.py new file mode 100644 index 0000000000..1261db9fd5 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/speaker_recognition/scp_to_manifest.py @@ -0,0 +1,96 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import logging +import os + +import librosa as l +from sklearn.model_selection import StratifiedShuffleSplit +from tqdm import tqdm + + +""" +This scipt converts a scp file where each line contains + +to a manifest json file. +Args: +--scp: scp file name +--id: index of speaker label in filename present in scp file that is separated by '/' +--out: output manifest file name +--split: True / False if you would want to split the manifest file for training purposes + you may not need this for test set. output file names is _.json + Defaults to False +""" + + +def write_file(name, lines, idx): + with open(name, 'w') as fout: + for i in idx: + dic = lines[i] + json.dump(dic, fout) + fout.write('\n') + logging.info("wrote", name) + + +def main(scp, id, out, split=False): + if os.path.exists(out): + os.remove(out) + scp_file = open(scp, 'r').readlines() + + lines = [] + speakers = [] + with open(out, 'w') as outfile: + for line in tqdm(scp_file): + line = line.strip() + y, sr = l.load(line, sr=None) + dur = l.get_duration(y=y, sr=sr) + speaker = line.split('/')[id] + speaker = list(speaker) + speaker = ''.join(speaker) + speakers.append(speaker) + meta = {"audio_filepath": line, "offset": 0, "duration": float(dur), "label": speaker} + lines.append(meta) + json.dump(meta, outfile) + outfile.write("\n") + + path = os.path.dirname(out) + if split: + sss = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=42) + for train_idx, test_idx in sss.split(speakers, speakers): + logging.info(len(train_idx)) + + out = os.path.join(path, 'train.json') + write_file(out, lines, train_idx) + out = os.path.join(path, 'dev.json') + write_file(out, lines, test_idx) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--scp", help="scp file name", type=str, required=True) + parser.add_argument( + "--id", help="field num seperated by '/' to be considered as speaker label", type=int, required=True + ) + parser.add_argument("--out", help="manifest_file name", type=str, required=True) + parser.add_argument( + "--split", + help="bool if you would want to split the manifest file for training purposes", + required=False, + action='store_true', + ) + args = parser.parse_args() + + main(args.scp, args.id, args.out, args.split) diff --git a/models/tacotron2/NeMo/scripts/speech_recognition/convert_to_tarred_audio_dataset.py b/models/tacotron2/NeMo/scripts/speech_recognition/convert_to_tarred_audio_dataset.py new file mode 100644 index 0000000000..4e13c0536c --- /dev/null +++ b/models/tacotron2/NeMo/scripts/speech_recognition/convert_to_tarred_audio_dataset.py @@ -0,0 +1,601 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +# This script converts an existing audio dataset with a manifest to +# a tarred and sharded audio dataset that can be read by the +# TarredAudioToTextDataLayer. + +# Please make sure your audio_filepath DOES NOT CONTAIN '-sub'! +# Because we will use it to handle files which have duplicate filenames but with different offsets +# (see function create_shard for details) + +# Usage: +1) Creating a new tarfile dataset + +python convert_to_tarred_audio_dataset.py \ + --manifest_path= \ + --target_dir= \ + --num_shards= + --max_duration= \ + --min_duration= \ + --shuffle --shuffle_seed=1 + + +2) Concatenating more tarfiles to a pre-existing tarred dataset + +python convert_to_tarred_audio_dataset.py \ + --manifest_path= \ + --metadata_path= \ + --target_dir= \ + --max_duration= \ + --min_duration= \ + --shuffle --shuffle_seed=1 \ + --concat_manifest_paths \ + + +3) Writing an empty metadata file + +python convert_to_tarred_audio_dataset.py \ + --target_dir= \ + # any other optional argument + --num_shards=8 \ + --max_duration=16.7 \ + --min_duration=0.01 \ + --shuffle \ + --shuffle_seed=1 \ + --write_metadata + +""" +import argparse +import copy +import json +import os +import random +import tarfile +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, List, Optional + +from joblib import Parallel, delayed +from omegaconf import DictConfig, OmegaConf, open_dict + +parser = argparse.ArgumentParser( + description="Convert an existing ASR dataset to tarballs compatible with TarredAudioToTextDataLayer." +) +parser.add_argument( + "--manifest_path", default=None, type=str, required=False, help="Path to the existing dataset's manifest." +) + +parser.add_argument( + '--concat_manifest_paths', + nargs='+', + default=None, + type=str, + required=False, + help="Path to the additional dataset's manifests that will be concatenated with base dataset.", +) + +# Optional arguments +parser.add_argument( + "--target_dir", + default='./tarred', + type=str, + help="Target directory for resulting tarballs and manifest. Defaults to `./tarred`. Creates the path if necessary.", +) + +parser.add_argument( + "--metadata_path", required=False, default=None, type=str, help="Path to metadata file for the dataset.", +) + +parser.add_argument( + "--num_shards", + default=-1, + type=int, + help="Number of shards (tarballs) to create. Used for partitioning data among workers.", +) +parser.add_argument( + '--max_duration', + default=None, + type=float, + help='Maximum duration of audio clip in the dataset. By default, it is None and will not filter files.', +) +parser.add_argument( + '--min_duration', + default=None, + type=float, + help='Minimum duration of audio clip in the dataset. By default, it is None and will not filter files.', +) +parser.add_argument( + "--shuffle", + action='store_true', + help="Whether or not to randomly shuffle the samples in the manifest before tarring/sharding.", +) +parser.add_argument("--shuffle_seed", type=int, help="Random seed for use if shuffling is enabled.") +parser.add_argument( + '--write_metadata', + action='store_true', + help=( + "Flag to write a blank metadata with the current call config. " + "Note that the metadata will not contain the number of shards, " + "and it must be filled out by the user." + ), +) +parser.add_argument('--workers', type=int, default=1, help='Number of worker processes') +args = parser.parse_args() + + +@dataclass +class ASRTarredDatasetConfig: + num_shards: int = -1 + shuffle: bool = False + max_duration: Optional[float] = None + min_duration: Optional[float] = None + shuffle_seed: Optional[int] = None + + +@dataclass +class ASRTarredDatasetMetadata: + created_datetime: Optional[str] = None + version: int = 0 + num_samples_per_shard: Optional[int] = None + is_concatenated_manifest: bool = False + + dataset_config: Optional[ASRTarredDatasetConfig] = ASRTarredDatasetConfig() + history: Optional[List[Any]] = field(default_factory=lambda: []) + + def __post_init__(self): + self.created_datetime = self.get_current_datetime() + + def get_current_datetime(self): + return datetime.now().strftime("%m-%d-%Y %H-%M-%S") + + @classmethod + def from_config(cls, config: DictConfig): + obj = cls() + obj.__dict__.update(**config) + return obj + + @classmethod + def from_file(cls, filepath: str): + config = OmegaConf.load(filepath) + return ASRTarredDatasetMetadata.from_config(config=config) + + +class ASRTarredDatasetBuilder: + """ + Helper class that constructs a tarred dataset from scratch, or concatenates tarred datasets + together and constructs manifests for them. + """ + + def __init__(self): + self.config = None + + def configure(self, config: ASRTarredDatasetConfig): + """ + Sets the config generated from command line overrides. + + Args: + config: ASRTarredDatasetConfig dataclass object. + """ + self.config = config # type: ASRTarredDatasetConfig + + if self.config.num_shards < 0: + raise ValueError("`num_shards` must be > 0. Please fill in the metadata information correctly.") + + def create_new_dataset(self, manifest_path: str, target_dir: str = "./tarred/", num_workers: int = 0): + """ + Creates a new tarred dataset from a given manifest file. + + Args: + manifest_path: Path to the original ASR manifest. + target_dir: Output directory. + num_workers: Integer denoting number of parallel worker processes which will write tarfiles. + Defaults to 1 - which denotes sequential worker process. + + Output: + Writes tarfiles, along with the tarred dataset compatible manifest file. + Also preserves a record of the metadata used to construct this tarred dataset. + """ + if self.config is None: + raise ValueError("Config has not been set. Please call `configure(config: ASRTarredDatasetConfig)`") + + if manifest_path is None: + raise FileNotFoundError("Manifest filepath cannot be None !") + + config = self.config # type: ASRTarredDatasetConfig + + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + # Read the existing manifest + entries, filtered_entries, filtered_duration = self._read_manifest(manifest_path, config) + + if len(filtered_entries) > 0: + print(f"Filtered {len(filtered_entries)} files which amounts to {filtered_duration} seconds of audio.") + + if config.shuffle: + random.seed(config.shuffle_seed) + print("Shuffling...") + random.shuffle(entries) + + # Create shards and updated manifest entries + print(f"Number of samples added : {len(entries)}") + print(f"Remainder: {len(entries) % config.num_shards}") + + start_indices = [] + end_indices = [] + # Build indices + for i in range(config.num_shards): + start_idx = (len(entries) // config.num_shards) * i + end_idx = start_idx + (len(entries) // config.num_shards) + print(f"Shard {i} has entries {start_idx} ~ {end_idx}") + if i == config.num_shards - 1: + # We discard in order to have the same number of entries per shard. + print(f"Have {len(entries) - end_idx} entries left over that will be discarded.") + + start_indices.append(start_idx) + end_indices.append(end_idx) + + with Parallel(n_jobs=num_workers, verbose=config.num_shards) as parallel: + # Call parallel tarfile construction + new_entries_list = parallel( + delayed(self._create_shard)(entries[start_idx:end_idx], target_dir, i) + for i, (start_idx, end_idx) in enumerate(zip(start_indices, end_indices)) + ) + + # Flatten the list of list of entries to a list of entries + new_entries = [sample for manifest in new_entries_list for sample in manifest] + del new_entries_list + + print("Total number of files in manifest :", len(new_entries)) + + # Write manifest + new_manifest_path = os.path.join(target_dir, 'tarred_audio_manifest.json') + with open(new_manifest_path, 'w') as m2: + for entry in new_entries: + json.dump(entry, m2) + m2.write('\n') + + # Write metadata (default metadata for new datasets) + new_metadata_path = os.path.join(target_dir, 'metadata.yaml') + metadata = ASRTarredDatasetMetadata() + + # Update metadata + metadata.dataset_config = config + metadata.num_samples_per_shard = len(new_entries) // config.num_shards + + # Write metadata + metadata_yaml = OmegaConf.structured(metadata) + OmegaConf.save(metadata_yaml, new_metadata_path, resolve=True) + + def create_concatenated_dataset( + self, + base_manifest_path: str, + manifest_paths: List[str], + metadata: ASRTarredDatasetMetadata, + target_dir: str = "./tarred_concatenated/", + num_workers: int = 1, + ): + """ + Creates new tarfiles in order to create a concatenated dataset, whose manifest contains the data for + both the original dataset as well as the new data submitted in manifest paths. + + Args: + base_manifest_path: Path to the manifest file which contains the information for the original + tarred dataset (with flattened paths). + manifest_paths: List of one or more paths to manifest files that will be concatenated with above + base tarred dataset. + metadata: ASRTarredDatasetMetadata dataclass instance with overrides from command line. + target_dir: Output directory + + Output: + Writes tarfiles which with indices mapping to a "concatenated" tarred dataset, + along with the tarred dataset compatible manifest file which includes information + about all the datasets that comprise the concatenated dataset. + + Also preserves a record of the metadata used to construct this tarred dataset. + """ + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + if base_manifest_path is None: + raise FileNotFoundError("Base manifest filepath cannot be None !") + + if manifest_paths is None or len(manifest_paths) == 0: + raise FileNotFoundError("List of additional manifest filepaths cannot be None !") + + config = ASRTarredDatasetConfig(**(metadata.dataset_config)) + + # Read the existing manifest (no filtering here) + base_entries, _, _ = self._read_manifest(base_manifest_path, config) + print(f"Read base manifest containing {len(base_entries)} samples.") + + # Precompute number of samples per shard + if metadata.num_samples_per_shard is None: + num_samples_per_shard = len(base_entries) // config.num_shards + else: + num_samples_per_shard = metadata.num_samples_per_shard + + print("Number of samples per shard :", num_samples_per_shard) + + # Compute min and max duration and update config (if no metadata passed) + print(f"Selected max duration : {config.max_duration}") + print(f"Selected min duration : {config.min_duration}") + + entries = [] + for new_manifest_idx in range(len(manifest_paths)): + new_entries, filtered_new_entries, filtered_duration = self._read_manifest( + manifest_paths[new_manifest_idx], config + ) + + if len(filtered_new_entries) > 0: + print( + f"Filtered {len(filtered_new_entries)} files which amounts to {filtered_duration:0.2f}" + f" seconds of audio from manifest {manifest_paths[new_manifest_idx]}." + ) + + entries.extend(new_entries) + + if config.shuffle: + random.seed(config.shuffle_seed) + print("Shuffling...") + random.shuffle(entries) + + # Drop last section of samples that cannot be added onto a chunk + drop_count = len(entries) % num_samples_per_shard + total_new_entries = len(entries) + entries = entries[:-drop_count] + + print( + f"Dropping {drop_count} samples from total new samples {total_new_entries} since they cannot " + f"be added into a uniformly sized chunk." + ) + + # Create shards and updated manifest entries + num_added_shards = len(entries) // num_samples_per_shard + + print(f"Number of samples in base dataset : {len(base_entries)}") + print(f"Number of samples in additional datasets : {len(entries)}") + print(f"Number of added shards : {num_added_shards}") + print(f"Remainder: {len(entries) % num_samples_per_shard}") + + start_indices = [] + end_indices = [] + shard_indices = [] + for i in range(num_added_shards): + start_idx = (len(entries) // num_added_shards) * i + end_idx = start_idx + (len(entries) // num_added_shards) + shard_idx = i + config.num_shards + print(f"Shard {shard_idx} has entries {start_idx + len(base_entries)} ~ {end_idx + len(base_entries)}") + + start_indices.append(start_idx) + end_indices.append(end_idx) + shard_indices.append(shard_idx) + + with Parallel(n_jobs=num_workers, verbose=num_added_shards) as parallel: + # Call parallel tarfile construction + new_entries_list = parallel( + delayed(self._create_shard)(entries[start_idx:end_idx], target_dir, shard_idx) + for i, (start_idx, end_idx, shard_idx) in enumerate(zip(start_indices, end_indices, shard_indices)) + ) + + # Flatten the list of list of entries to a list of entries + new_entries = [sample for manifest in new_entries_list for sample in manifest] + del new_entries_list + + # Write manifest + if metadata is None: + new_version = 1 # start with `1`, where `0` indicates the base manifest + dataset + else: + new_version = metadata.version + 1 + + print("Total number of files in manifest :", len(base_entries) + len(new_entries)) + + new_manifest_path = os.path.join(target_dir, f'tarred_audio_manifest_version_{new_version}.json') + with open(new_manifest_path, 'w') as m2: + # First write all the entries of base manifest + for entry in base_entries: + json.dump(entry, m2) + m2.write('\n') + + # Finally write the new entries + for entry in new_entries: + json.dump(entry, m2) + m2.write('\n') + + # Preserve historical metadata + base_metadata = metadata + + # Write metadata (updated metadata for concatenated datasets) + new_metadata_path = os.path.join(target_dir, f'metadata_version_{new_version}.yaml') + metadata = ASRTarredDatasetMetadata() + + # Update config + config.num_shards = config.num_shards + num_added_shards + + # Update metadata + metadata.version = new_version + metadata.dataset_config = config + metadata.num_samples_per_shard = num_samples_per_shard + metadata.is_concatenated_manifest = True + metadata.created_datetime = metadata.get_current_datetime() + + # Attach history + current_metadata = OmegaConf.structured(base_metadata.history) + metadata.history = current_metadata + + # Write metadata + metadata_yaml = OmegaConf.structured(metadata) + OmegaConf.save(metadata_yaml, new_metadata_path, resolve=True) + + def _read_manifest(self, manifest_path: str, config: ASRTarredDatasetConfig): + """Read and filters data from the manifest""" + # Read the existing manifest + entries = [] + filtered_entries = [] + filtered_duration = 0.0 + with open(manifest_path, 'r') as m: + for line in m: + entry = json.loads(line) + if (config.max_duration is None or entry['duration'] < config.max_duration) and ( + config.min_duration is None or entry['duration'] > config.min_duration + ): + entries.append(entry) + else: + filtered_entries.append(entry) + filtered_duration += entry['duration'] + + return entries, filtered_entries, filtered_duration + + def _create_shard(self, entries, target_dir, shard_id): + """Creates a tarball containing the audio files from `entries`. + """ + new_entries = [] + tar = tarfile.open(os.path.join(target_dir, f'audio_{shard_id}.tar'), mode='w', dereference=True) + + count = dict() + for entry in entries: + # We squash the filename since we do not preserve directory structure of audio files in the tarball. + base, ext = os.path.splitext(entry['audio_filepath']) + base = base.replace('/', '_') + # Need the following replacement as long as WebDataset splits on first period + base = base.replace('.', '_') + squashed_filename = f'{base}{ext}' + if squashed_filename not in count: + tar.add(entry['audio_filepath'], arcname=squashed_filename) + + if 'label' in entry: + base, ext = os.path.splitext(squashed_filename) + # no suffix if it's single sample or starting sub parts, -sub1 for the second subpart -sub2 -sub3 ,etc. + if squashed_filename not in count: + to_write = squashed_filename + count[squashed_filename] = 1 + else: + to_write = base + "-sub" + str(count[squashed_filename]) + ext + count[squashed_filename] += 1 + + new_entry = { + 'audio_filepath': to_write, + 'duration': entry['duration'], + 'text': entry['text'], + 'label': entry['label'], + 'offset': entry['offset'], + 'shard_id': shard_id, # Keep shard ID for recordkeeping + } + else: + count[squashed_filename] = 1 + new_entry = { + 'audio_filepath': squashed_filename, + 'duration': entry['duration'], + 'text': entry['text'], + 'shard_id': shard_id, # Keep shard ID for recordkeeping + } + + new_entries.append(new_entry) + + tar.close() + return new_entries + + @classmethod + def setup_history(cls, base_metadata: ASRTarredDatasetMetadata, history: List[Any]): + if 'history' in base_metadata.keys(): + for history_val in base_metadata.history: + cls.setup_history(history_val, history) + + if base_metadata is not None: + metadata_copy = copy.deepcopy(base_metadata) + with open_dict(metadata_copy): + metadata_copy.pop('history', None) + history.append(metadata_copy) + + +def main(): + manifest_path = args.manifest_path + concat_manifest_paths = args.concat_manifest_paths + target_dir = args.target_dir + metadata_path = args.metadata_path + num_shards = args.num_shards + max_duration = args.max_duration + min_duration = args.min_duration + shuffle = args.shuffle + seed = args.shuffle_seed if args.shuffle_seed else None + write_metadata = args.write_metadata + num_workers = args.workers + + builder = ASRTarredDatasetBuilder() + + if write_metadata: + metadata = ASRTarredDatasetMetadata() + dataset_cfg = ASRTarredDatasetConfig( + num_shards=num_shards, + shuffle=shuffle, + max_duration=max_duration, + min_duration=min_duration, + shuffle_seed=seed, + ) + metadata.dataset_config = dataset_cfg + + output_path = os.path.join(target_dir, 'default_metadata.yaml') + OmegaConf.save(metadata, output_path, resolve=True) + print(f"Default metadata written to {output_path}") + exit(0) + + if concat_manifest_paths is None or len(concat_manifest_paths) == 0: + print("Creating new tarred dataset ...") + + # Create a tarred dataset from scratch + config = ASRTarredDatasetConfig( + num_shards=num_shards, + shuffle=shuffle, + max_duration=max_duration, + min_duration=min_duration, + shuffle_seed=seed, + ) + builder.configure(config) + builder.create_new_dataset(manifest_path=manifest_path, target_dir=target_dir, num_workers=num_workers) + + else: + print("Concatenating multiple tarred datasets ...") + + # Implicitly update config from base details + if metadata_path is not None: + metadata = ASRTarredDatasetMetadata.from_file(metadata_path) + else: + raise ValueError("`metadata` yaml file path must be provided!") + + # Preserve history + history = [] + builder.setup_history(OmegaConf.structured(metadata), history) + metadata.history = history + + # Add command line overrides (everything other than num_shards) + metadata.dataset_config.max_duration = max_duration + metadata.dataset_config.min_duration = min_duration + metadata.dataset_config.shuffle = shuffle + metadata.dataset_config.shuffle_seed = seed + + builder.configure(metadata.dataset_config) + + # Concatenate a tarred dataset onto a previous one + builder.create_concatenated_dataset( + base_manifest_path=manifest_path, + manifest_paths=concat_manifest_paths, + metadata=metadata, + target_dir=target_dir, + num_workers=num_workers, + ) + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/scripts/tokenizers/process_asr_text_tokenizer.py b/models/tacotron2/NeMo/scripts/tokenizers/process_asr_text_tokenizer.py new file mode 100644 index 0000000000..8c64507174 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/tokenizers/process_asr_text_tokenizer.py @@ -0,0 +1,282 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# USAGE: python process_asr_text_tokenizer.py --manifest= \ +# --data_root="" \ +# --vocab_size= \ +# --tokenizer=<"spe" or "wpe"> \ +# --log +# where can be: train_clean_100, train_clean_360, train_other_500 +# You can also put more than one data_set comma-separated: +# --manifest="train_clean_100,train_clean_360,train_other_500" +# or +# python process_asr_text_tokenizer.py --data_file= \ +# --data_root="" \ +# --vocab_size= \ +# --tokenizer=<"bpe" or "wpe"> \ +# --log +# where can be: train_clean_100, train_clean_360, train_other_500 +# You can also put more than one data_set comma-separated: +# --manifest="train_clean_100,train_clean_360,train_other_500" +# +# Args: +# --manifest or --data_file: If your text data lies inside of an ASR manifest file, +# then use the --manifest path. If instead the text data is inside a file with separate lines +# corresponding to different text lines, then use --data_file. +# In either case, you can add commas to concatenate different manifests or different data files. +# +# --data_root: The output directory (whose subdirectories will be created if not present) where +# the tokenizers will be placed. +# +# --vocab_size: The size of the tokenizer vocabulary. Larger vocabularies can accommodate almost entire, +# words but the decoder size of any model will grow proportionally. +# +# --tokenizer: Can be either spe or wpe . spe refers to the Google sentencepiece library tokenizer. +# wpe refers to the HuggingFace BERT Word Piece tokenizer. +# +# --no_lower_case: When this flag is passed, it will force the tokenizer to create seperate tokens for +# upper and lower case characters. By default, the script will turn all the text to lower case +# before tokenization (and if upper case characters are passed during training/inference, the +# tokenizer will emit a token equivalent to Out-Of-Vocabulary). Used primarily for the +# English language. +# +# --spe_type: The sentencepiece library has a few implementations of the tokenization technique, and +# spe_type refers to these implementations. Currently supported types are unigram, bpe, char, word. +# Defaults to bpe. +# +# --spe_character_coverage: The sentencepiece library considers how much of the original vocabulary it +# should cover in its "base set" of tokens (akin to the lower and upper case characters of the +# English language). For almost all languages with small base token sets (<1000 tokens), this +# should be kept at its default of 1.0. For languages with larger vocabularies (say Japanese, +# Mandarin, Korean etc), the suggested value is 0.9995. +# +# --spe_sample_size: If the dataset is too large, consider using a sampled dataset indicated by a +# positive integer. By default, any negative value (default = -1) will use the entire dataset. +# +# --spe_train_extremely_large_corpus: When training a sentencepiece tokenizer on very large amounts of text, +# sometimes the tokenizer will run out of memory or wont be able to process so much data on RAM. +# At some point you might receive the following error - "Input corpus too large, try with +# train_extremely_large_corpus=true". If your machine has large amounts of RAM, it might still be possible +# to build the tokenizer using the above flag. Will silently fail if it runs out of RAM. +# +# --spe_max_sentencepiece_length: Limits the maximum length that any any SentencePiece subword can be. +# Using this will change the subword tokens generated. +# +# --log: Whether the script should display log messages + + +import argparse +import json +import logging +import os + +import tokenizers + +from nemo.collections.common.tokenizers.sentencepiece_tokenizer import create_spt_model + +parser = argparse.ArgumentParser(description='Create tokenizer') +group = parser.add_mutually_exclusive_group(required=True) +group.add_argument("--manifest", default=None, type=str, help='Comma separated list of manifest files') +group.add_argument("--data_file", default=None, help='data file from which to create tokenizer model') +parser.add_argument("--data_root", required=True, default=None, type=str, help='Output directory') +parser.add_argument("--vocab_size", default=1024, type=int, help='Vocabulary size') +parser.add_argument("--tokenizer", default="wpe", choices=["spe", "wpe"], help='Type of tokenization to perform') +parser.add_argument( + "--spe_type", + default="bpe", + choices=['bpe', 'unigram', 'char', 'word'], + help='Type of the SentencePiece model. Can be `bpe`, `unigram`, `char` or `word`.' + 'Used only if --tokenizer == `spe`', +) +parser.add_argument( + '--spe_character_coverage', + type=float, + default=1.0, + help="Character coverage percentage for SentencePiece tokenization. For languages " + "with large vocabulary, should be close to 0.9995, otherwise kept as 1.0", +) +parser.add_argument( + '--spe_sample_size', + type=int, + default=-1, + help="Samples the dataset by `sample_size` if positive integer, otherwise uses whole dataset", +) +parser.add_argument('--spe_train_extremely_large_corpus', action='store_true', help='') +parser.add_argument( + '--spe_max_sentencepiece_length', + type=int, + default=-1, + help='Limit the maximum number of tokens in each SentencePiece subword. ' + 'Must be a positive integer > 0. By default places no limit on subword length.', +) +parser.add_argument('--no_lower_case', dest='lower_case', action='store_false') +parser.add_argument("--log", action='store_true') +parser.set_defaults(log=False, lower_case=True, spe_train_extremely_large_corpus=False) +args = parser.parse_args() + + +def __build_document_from_manifests( + data_root: str, manifests: str, +): + if ',' in manifests: + manifests = manifests.split(',') + else: + manifests = [manifests] + + document_dir = os.path.join(data_root, 'text_corpus') + if not os.path.exists(document_dir): + os.makedirs(document_dir) + + document_path = os.path.join(document_dir, 'document.txt') + + if os.path.exists(document_path): + logging.info('Corpus already exists at path : %s', document_path) + return document_path + + num_lines = 0 + with open(document_path, 'w') as out_writer: + for manifest in manifests: + with open(manifest, 'r') as in_reader: + for line in in_reader: + item = json.loads(line) + text = item['text'] + + out_writer.write(text + '\n') + out_writer.flush() + + num_lines += 1 + + logging.info(f"Finished extracting manifest : {manifest}") + + logging.info("Finished extracting all manifests ! Number of sentences : {}".format(num_lines)) + return document_path + + +def __process_data( + text_path: str, + dst_folder: str, + vocab_size: int, + tokenizer_type: str, + spe_type: str, + spe_character_coverage: float, + spe_train_extremely_large_corpus: bool, + spe_sample_size: int, + spe_max_sentencepiece_length: int, + lower_case: bool, +): + """ + Converts flac to wav and build manifests's json + Args: + text_path: source with text lines + dst_folder: where wav files will be stored + vocab_size: vocabular size used in encoding the text + tokenizer_type: type of tokenization to perform - wpe or spe + spe_type: type of tokenization model used for spe. + spe_character_coverage: float value between 0 and 1 (as a percentage). For languages with a vast charset, + can be < 1.0, but for all other languages, it should be set as 1.0 + spe_sample_size: int, default of -1. If positive integer is used, samples the dataset + by given sample size. + spe_train_extremely_large_corpus: bool. If dataset is too large, and user has sufficient RAM, + this flag can be set to try to trained the tokenizer. Will silently fail if it runs out of RAM. + spe_max_sentencepiece_length: Limits the maximum length of the SentencePiece subword that can be constructed. + By default, no limit is placed. + lower_case: whether to tokenize with lower case character set only (for english) + + Returns: + """ + if tokenizer_type == 'spe': + if spe_max_sentencepiece_length > 0: + tokenizer_dir = os.path.join(dst_folder, 'tokenizer_{}_{}_v{}_max{}').format( + tokenizer_type, spe_type, vocab_size, spe_max_sentencepiece_length + ) + else: + tokenizer_dir = os.path.join(dst_folder, 'tokenizer_{}_{}_v{}').format( + tokenizer_type, spe_type, vocab_size + ) + + if not os.path.exists(tokenizer_dir): + os.makedirs(tokenizer_dir) + + if os.path.exists(os.path.join(tokenizer_dir, 'tokenizer.model')): + logging.warning("Model file already exists, overriding old model file !") + os.remove(os.path.join(tokenizer_dir, 'tokenizer.model')) + + tokenizer_path, vocab_path = create_spt_model( + data_file=text_path, + vocab_size=vocab_size, + sample_size=spe_sample_size, + do_lower_case=lower_case, + output_dir=tokenizer_dir, + tokenizer_type=spe_type, + character_coverage=spe_character_coverage, + train_extremely_large_corpus=spe_train_extremely_large_corpus, + max_sentencepiece_length=spe_max_sentencepiece_length, + ) + + else: + tokenizer_dir = os.path.join(dst_folder, 'tokenizer_{}_v{}').format(tokenizer_type, vocab_size) + + if not os.path.exists(tokenizer_dir): + os.makedirs(tokenizer_dir) + + tokenizer = tokenizers.BertWordPieceTokenizer(lowercase=lower_case) + + tokenizer.train(text_path, vocab_size=vocab_size) + tokenizer.save_model(tokenizer_dir) + + return tokenizer_dir + + +def main(): + data_root = args.data_root + manifests = args.manifest + data_file = args.data_file + vocab_size = args.vocab_size + tokenizer = args.tokenizer + spe_type = args.spe_type + spe_character_coverage = args.spe_character_coverage + spe_sample_size = args.spe_sample_size + spe_train_extremely_large_corpus = args.spe_train_extremely_large_corpus + spe_max_sentencepiece_length = args.spe_max_sentencepiece_length + lower_case = args.lower_case + + if not os.path.exists(data_root): + os.makedirs(data_root) + + if args.log: + logging.basicConfig(level=logging.INFO) + + if manifests: + text_corpus_path = __build_document_from_manifests(data_root, manifests) + else: + text_corpus_path = data_file + tokenizer_path = __process_data( + text_corpus_path, + data_root, + vocab_size, + tokenizer, + spe_type, + lower_case=lower_case, + spe_character_coverage=spe_character_coverage, + spe_sample_size=spe_sample_size, + spe_train_extremely_large_corpus=spe_train_extremely_large_corpus, + spe_max_sentencepiece_length=spe_max_sentencepiece_length, + ) + + print("Serialized tokenizer at location :", tokenizer_path) + logging.info('Done!') + + +if __name__ == "__main__": + main() diff --git a/models/tacotron2/NeMo/scripts/voice_activity_detection/vad_overlap_posterior.py b/models/tacotron2/NeMo/scripts/voice_activity_detection/vad_overlap_posterior.py new file mode 100644 index 0000000000..0849a6515f --- /dev/null +++ b/models/tacotron2/NeMo/scripts/voice_activity_detection/vad_overlap_posterior.py @@ -0,0 +1,259 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import glob +import os +import time +from argparse import ArgumentParser +from itertools import repeat +from multiprocessing import Pool + +import numpy as np +import pandas as pd + + +""" +This script serves two purposes: + + 1) gen_overlap_seq: + Generate predictions with overlapping input segments by using the frame level prediction from NeMo/examples/asr/vad_infer.py. + Then a smoothing filter is applied to decide the label for a frame spanned by multiple segments. + + 2）gen_seg_table: + Converting frame level prediction to speech/no-speech segment in start and end times format. + +Usage: + +python vad_overlap_posterior.py --gen_overlap_seq --gen_seg_table --frame_folder= --method='median' --overlap=0.875 --num_workers=20 --threshold=0.8 + +""" + + +def gen_overlap_seq(frame_filepath, per_args): + """ + Given a frame level prediction, generate predictions with overlapping input segments by using it + Args: + frame_filepath : frame prediction file to be processed. + per_args: + method : Median or mean smoothing filter. + overlap : Amounts of overlap. + seg_len : Length of window for generating the frame. + shift_len : Amount of shift of window for generating the frame. + out_dir : Output dir of generated prediction. + """ + + try: + method = per_args['method'] + overlap = per_args['overlap'] + seg_len = per_args['seg_len'] + shift_len = per_args['shift_len'] + out_dir = per_args['out_dir'] + + frame = np.loadtxt(frame_filepath) + name = os.path.basename(frame_filepath).split(".frame")[0] + "." + method + overlap_filepath = os.path.join(out_dir, name) + + shift = int(shift_len / 0.01) # number of units of shift + seg = int((seg_len / 0.01 + 1)) # number of units of each window/segment + + jump_on_target = int(seg * (1 - overlap)) # jump on target generated sequence + jump_on_frame = int(jump_on_target / shift) # jump on input frame sequence + + if jump_on_frame < 1: + raise ValueError( + f"Note we jump over frame sequence to generate overlapping input segments. \n \ + Your input makes jump_on_fram={jump_on_frame} < 1 which is invalid because it cannot jump and will stuck.\n \ + Please try different seg_len, shift_len and overlap choices. \n \ + jump_on_target = int(seg * (1 - overlap)) \n \ + jump_on_frame = int(jump_on_frame/shift) " + ) + + target_len = int(len(frame) * shift) + + if method == 'mean': + preds = np.zeros(target_len) + pred_count = np.zeros(target_len) + + for i, og_pred in enumerate(frame): + if i % jump_on_frame != 0: + continue + start = i * shift + end = start + seg + preds[start:end] = preds[start:end] + og_pred + pred_count[start:end] = pred_count[start:end] + 1 + + preds = preds / pred_count + last_non_zero_pred = preds[pred_count != 0][-1] + preds[pred_count == 0] = last_non_zero_pred + + elif method == 'median': + preds = [[] for _ in range(target_len)] + for i, og_pred in enumerate(frame): + if i % jump_on_frame != 0: + continue + + start = i * shift + end = start + seg + for j in range(start, end): + if j <= target_len - 1: + preds[j].append(og_pred) + + preds = np.array([np.median(l) for l in preds]) + nan_idx = np.isnan(preds) + last_non_nan_pred = preds[~nan_idx][-1] + preds[nan_idx] = last_non_nan_pred + + else: + raise ValueError("method should be either mean or median") + + round_final = np.round(preds, 4) + np.savetxt(overlap_filepath, round_final, delimiter='\n') + print(f"Finished! {overlap_filepath}!") + + except Exception as e: + raise (e) + + +def gen_seg_table(frame_filepath, per_args): + + """ + Convert frame level prediction to speech/no-speech segment in start and end times format. + And save to csv file in rttm-like format + 0, 10, speech + 10,12, no-speech + Args: + frame_filepath : frame prediction file to be processed. + per_args : + threshold : threshold for prediction score (from 0 to 1). + shift_len : Amount of shift of window for generating the frame. + out_dir : Output dir of generated table/csv file. + """ + threshold = per_args['threshold'] + shift_len = per_args['shift_len'] + out_dir = per_args['out_dir'] + + print(f"process {frame_filepath}") + name = frame_filepath.split("/")[-1].rsplit(".", 1)[0] + + sequence = np.loadtxt(frame_filepath) + start = 0 + end = 0 + start_list = [0] + end_list = [] + state_list = [] + + for i in range(len(sequence) - 1): + current_sate = "non-speech" if sequence[i] <= threshold else "speech" + next_state = "non-speech" if sequence[i + 1] <= threshold else "speech" + if next_state != current_sate: + end = i * shift_len + shift_len # shift_len for handling joint + state_list.append(current_sate) + end_list.append(end) + + start = (i + 1) * shift_len + start_list.append(start) + + end_list.append((i + 1) * shift_len + shift_len) + state_list.append(current_sate) + + seg_table = pd.DataFrame({'start': start_list, 'end': end_list, 'vad': state_list}) + + save_name = name + ".txt" + save_path = os.path.join(out_dir, save_name) + seg_table.to_csv(save_path, sep='\t', index=False, header=False) + + +if __name__ == '__main__': + start = time.time() + parser = ArgumentParser() + parser.add_argument("--gen_overlap_seq", default=False, action='store_true') + parser.add_argument("--gen_seg_table", default=False, action='store_true') + parser.add_argument("--frame_folder", type=str, required=True) + parser.add_argument( + "--method", + type=str, + required=True, + help="Use mean/median for overlapped prediction. Use frame for gen_seg_table of frame prediction", + ) + parser.add_argument("--overlap_out_dir", type=str) + parser.add_argument("--table_out_dir", type=str) + parser.add_argument("--overlap", type=float, default=0.875, help="Overlap percentatge. Default is 0.875") + parser.add_argument("--seg_len", type=float, default=0.63) + parser.add_argument("--shift_len", type=float, default=0.01) + parser.add_argument("--threshold", type=float, default=0.5) + parser.add_argument("--num_workers", type=int, default=4) + args = parser.parse_args() + + if args.gen_overlap_seq: + p = Pool(processes=args.num_workers) + print("Generating predictions with overlapping input segments") + frame_filepathlist = glob.glob(args.frame_folder + "/*.frame") + + if not args.overlap_out_dir: + overlap_out_dir = "./overlap_smoothing_output" + "_" + args.method + "_" + str(args.overlap) + else: + overlap_out_dir = args.overlap_out_dir + if not os.path.exists(overlap_out_dir): + print(f"Creating output dir {overlap_out_dir}") + os.mkdir(overlap_out_dir) + + per_args = { + "method": args.method, + "overlap": args.overlap, + "seg_len": args.seg_len, + "shift_len": args.shift_len, + "out_dir": overlap_out_dir, + } + + p.starmap(gen_overlap_seq, zip(frame_filepathlist, repeat(per_args))) + p.close() + p.join() + + end = time.time() + print(f"Generate overlapped prediction takes {end-start} seconds!\n Save to {overlap_out_dir}") + + if args.gen_seg_table: + p = Pool(processes=args.num_workers) + print("Converting frame level prediction to speech/no-speech segment in start and end times format.") + + if args.gen_overlap_seq: + print("Use overlap prediction. Change if you want to use basic frame level prediction") + frame_filepath = overlap_out_dir + shift_len = 0.01 + else: + print("Use basic frame level prediction") + frame_filepath = args.frame_folder + shift_len = args.shift_len + + frame_filepathlist = glob.glob(frame_filepath + "/*." + args.method) + + if not args.table_out_dir: + table_out_dir = "table_output_" + str(args.threshold) + else: + table_out_dir = args.table_out_dir + if not os.path.exists(table_out_dir): + print(f"Creating rttm-like table output dir {table_out_dir}") + os.mkdir(table_out_dir) + + per_args = { + "threshold": args.threshold, + "shift_len": shift_len, + "out_dir": table_out_dir, + } + + p.starmap(gen_seg_table, zip(frame_filepathlist, repeat(per_args))) + p.close() + p.join() + + end = time.time() + print(f"Generate rttm-like table for {frame_filepath} takes {end-start} seconds!\n Save to {table_out_dir}") diff --git a/models/tacotron2/NeMo/scripts/voice_activity_detection/vad_tune_threshold.py b/models/tacotron2/NeMo/scripts/voice_activity_detection/vad_tune_threshold.py new file mode 100644 index 0000000000..1486ae730c --- /dev/null +++ b/models/tacotron2/NeMo/scripts/voice_activity_detection/vad_tune_threshold.py @@ -0,0 +1,58 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +import numpy as np + +from nemo.collections.asr.parts.utils.vad_utils import vad_tune_threshold_on_dev +from nemo.utils import logging + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--threshold_range", help="range of threshold in list 'START,END,STEP' to be tuned on", required=True + ) + parser.add_argument( + "--vad_pred", help="Directory of vad predictions or a file contains the paths of them.", required=True + ) + parser.add_argument( + "--groundtruth_RTTM", + help="Directory of groundtruch rttm files or a file contains the paths of them", + type=str, + required=True, + ) + parser.add_argument( + "--vad_pred_method", + help="suffix of prediction file. Should be either in 'frame', 'mean' or 'median'", + required=True, + ) + parser.add_argument( + "--focus_metric", + help="metrics we care most when tuning threshold. Should be either in 'DetER', 'FA', 'MISS' ", + type=str, + default='DetER', + ) + args = parser.parse_args() + + try: + start, stop, step = [float(i) for i in args.threshold_range.split(",")] + thresholds = np.arange(start, stop, step) + except: + raise ValueError("Theshold input is invalid! Please enter it as a 'START,STOP,STEP' ") + + best_threhsold = vad_tune_threshold_on_dev( + thresholds, args.vad_pred, args.groundtruth_RTTM, args.vad_pred_method, args.focus_metric + ) + logging.info(f"Best threshold selected from {thresholds} is {best_threhsold}!") diff --git a/models/tacotron2/NeMo/scripts/voice_activity_detection/write_long_audio_manifest.py b/models/tacotron2/NeMo/scripts/voice_activity_detection/write_long_audio_manifest.py new file mode 100644 index 0000000000..f4501224d4 --- /dev/null +++ b/models/tacotron2/NeMo/scripts/voice_activity_detection/write_long_audio_manifest.py @@ -0,0 +1,173 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import logging +import multiprocessing +import os +from argparse import ArgumentParser +from itertools import repeat + +import librosa +import numpy as np + + +""" +This script is designed for inference of frame level Voice Activity Detection (VAD) + +This script serves three goals: + (1) Write audio files to manifest + (2) Split audio file for avoiding CUDA memory issue + (3) Take care of joint of seperate json line for an audio file + +Usage: +python write_long_audio_manifest.py --inp_dir= --manifest_name= --split_duration=300 --time_length=0.63 --num_worker=10 + +""" + + +def write_manifest(file, args_func): + """ + Given a list of files, write them to manifest with restrictions. + Args: + files : file to be processed + label : label for audio snippet. + split_duration : Max duration of each audio clip (each line in json) + time_length : Used for taking care of joint. + Length of window for generating the frame. + Returns: + res : list of generated metadata line of json for file + """ + + label = args_func['label'] + split_duration = args_func['split_duration'] + time_length = args_func['time_length'] + + res = [] + try: + sr = 16000 + x, _sr = librosa.load(file, sr=sr) + duration = librosa.get_duration(x, sr=sr) + + left = duration + current_offset = 0 + status = 'single' + + while left > 0: + if left <= split_duration: + if status == 'single': + write_duration = left + current_offset = 0 + else: + status = 'end' + write_duration = left + time_length + current_offset -= time_length + offset_inc = left + left = 0 + else: + if status == 'start' or status == 'next': + status = 'next' + else: + status = 'start' + + if status == 'start': + write_duration = split_duration + offset_inc = split_duration + else: + write_duration = split_duration + time_length + current_offset -= time_length + offset_inc = split_duration + time_length + + left -= split_duration + + metadata = { + 'audio_filepath': file, + 'duration': write_duration, + 'label': label, + 'text': '_', + 'offset': current_offset, + } + res.append(metadata) + + current_offset += offset_inc + + except Exception as e: + err_file = "error.log" + with open(err_file, 'w') as fout: + fout.write(file + ":" + str(e)) + + return res + + +def main(): + parser = ArgumentParser() + parser.add_argument("--inp_dir", type=str, required=True, help="(full path) folder of files to be processed") + parser.add_argument( + "--inp_list", type=str, help="(full path) a file contains NAME of files inside inp_dir to be processed" + ) + parser.add_argument( + "--out_dir", type=str, default=".", help="[Optional](full path) location to store generated json file" + ) + parser.add_argument("--manifest_name", type=str, required=True, help="name of generated json file") + parser.add_argument("--split_duration", type=int, required=True, help="max duration of each audio clip/line") + parser.add_argument( + "--time_length", type=float, default=0.63, help="[Optional] time length of segment, default is 0.63s" + ) + parser.add_argument("--num_workers", type=int, default=4, help="[Optional] number of workers for multiprocessing") + args = parser.parse_args() + + if not args.inp_list: + input_audios = [] + for root, dirs, files in os.walk(args.inp_dir): + for basename in files: + if basename.endswith('.wav'): + filename = os.path.join(root, basename) + input_audios.append(filename) + else: + name_list = np.loadtxt(args.inp_list, dtype='str') + input_audios = [os.path.join(args.inp_dir, name + ".wav") for name in name_list] + print(f"Number of wav files to be processed: {len(input_audios)}") + + output_path = os.path.join(args.out_dir, args.manifest_name + '.json') + print(f"Save generate manifest to {output_path}!") + if not os.path.exists(args.out_dir): + logging.info(f'Outdir {args.out_dir} does not exist. Creat directory.') + os.mkdir(args.out_dir) + if os.path.exists(output_path): + print(f"Manifest {output_path} exists! Overwriting") + os.remove(output_path) + + p = multiprocessing.Pool(processes=args.num_workers) + + args_func = { + 'label': 'infer', + 'split_duration': args.split_duration, + 'time_length': args.time_length, + } + + results = p.starmap(write_manifest, zip(input_audios, repeat(args_func))) + + with open(output_path, 'a') as fout: + for res in results: + for r in res: + json.dump(r, fout) + fout.write('\n') + fout.flush() + + p.close() + + print(f"Done! Save to {output_path}") + + +if __name__ == '__main__': + main() diff --git a/models/tacotron2/NeMo/setup.cfg b/models/tacotron2/NeMo/setup.cfg new file mode 100644 index 0000000000..694e91590d --- /dev/null +++ b/models/tacotron2/NeMo/setup.cfg @@ -0,0 +1,36 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[aliases] +test=pytest + +# durations=0 will display all tests execution time, sorted in ascending order starting from from the slowest one. +# -vv will also display tests with durration = 0.00s +[tool:pytest] +addopts = --verbose --pyargs --durations=0 +markers = + unit: marks unit test, i.e. testing a single, well isolated functionality (deselect with '-m "not unit"') + integration: marks test checking the elements when integrated into subsystems (deselect with '-m "not integration"') + system: marks test working at the highest integration level (deselect with '-m "not system"') + acceptance: marks test checking whether the developed product/model passes the user defined acceptance criteria (deselect with '-m "not acceptance"') + docs: mark tests related to documentation (deselect with '-m "not docs"') + skipduringci: marks tests that are skipped ci as they are addressed by Jenkins jobs but should be run to test user setups + pleasefixme: marks tests that are broken and need fixing + +[isort] +known_localfolder = nemo,tests +sections = FUTURE,STDLIB,THIRDPARTY,LOCALFOLDER +default_section = THIRDPARTY +#TODO tests/unit/core/test_deploy_export.py gets screwed by isort +skip = setup.py, docs/source/conf.py, nemo/utils/__init__.py, tests/unit/core/test_deploy_export.py diff --git a/models/tacotron2/NeMo/setup.py b/models/tacotron2/NeMo/setup.py new file mode 100644 index 0000000000..77c8a56bd8 --- /dev/null +++ b/models/tacotron2/NeMo/setup.py @@ -0,0 +1,238 @@ +# ! /usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Setup for pip package.""" + +import codecs +import os +import subprocess +from distutils import cmd as distutils_cmd +from distutils import log as distutils_log +from itertools import chain +import imp + +import setuptools + + +package_info = imp.load_source('package_info', 'nemo/package_info.py') +from package_info import ( + __contact_emails__, + __contact_names__, + __description__, + __download_url__, + __homepage__, + __keywords__, + __license__, + __package_name__, + __repository_url__, + __version__, +) + +if os.path.exists('nemo/README.md'): + with open("nemo/README.md", "r") as fh: + long_description = fh.read() + long_description_content_type = "text/markdown" + +elif os.path.exists('README.rst'): + # codec is used for consistent encoding + long_description = codecs.open( + os.path.join(os.path.abspath(os.path.dirname(__file__)), 'README.rst'), 'r', 'utf-8', + ).read() + long_description_content_type = "text/x-rst" + +else: + long_description = 'See ' + __homepage__ + + +############################################################################### +# Dependency Loading # +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # + + +def req_file(filename, folder="requirements"): + with open(os.path.join(folder, filename)) as f: + content = f.readlines() + # you may also want to remove whitespace characters + # Example: `\n` at the end of each line + return [x.strip() for x in content] + + +install_requires = req_file("requirements.txt") + +extras_require = { + # User packages + 'test': req_file("requirements_test.txt"), + # Collections Packages + 'asr': req_file("requirements_asr.txt"), + 'cv': req_file("requirements_cv.txt"), + 'nlp': req_file("requirements_nlp.txt"), + 'tts': req_file("requirements_tts.txt"), + 'text_processing': req_file("requirements_text_processing.txt"), +} + +extras_require['all'] = list(chain(extras_require.values())) + +# TTS depends on ASR +extras_require['tts'] = list(chain([extras_require['tts'], extras_require['asr']])) + +tests_requirements = extras_require["test"] + + +############################################################################### +# Code style checkers # +# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # + + +class StyleCommand(distutils_cmd.Command): + __LINE_WIDTH = 119 + __ISORT_BASE = ( + 'isort ' + # These two lines makes isort compatible with black. + '--multi-line=3 --trailing-comma --force-grid-wrap=0 ' + f'--use-parentheses --line-width={__LINE_WIDTH} -rc -ws' + ) + __BLACK_BASE = f'black --skip-string-normalization --line-length={__LINE_WIDTH}' + description = 'Checks overall project code style.' + user_options = [ + ('scope=', None, 'Folder of file to operate within.'), + ('fix', None, 'True if tries to fix issues in-place.'), + ] + + def __call_checker(self, base_command, scope, check): + command = list(base_command) + + command.append(scope) + + if check: + command.extend(['--check', '--diff']) + + self.announce( + msg='Running command: %s' % str(' '.join(command)), level=distutils_log.INFO, + ) + + return_code = subprocess.call(command) + + return return_code + + def _isort(self, scope, check): + return self.__call_checker(base_command=self.__ISORT_BASE.split(), scope=scope, check=check,) + + def _black(self, scope, check): + return self.__call_checker(base_command=self.__BLACK_BASE.split(), scope=scope, check=check,) + + def _pass(self): + self.announce(msg='\033[32mPASS\x1b[0m', level=distutils_log.INFO) + + def _fail(self): + self.announce(msg='\033[31mFAIL\x1b[0m', level=distutils_log.INFO) + + # noinspection PyAttributeOutsideInit + def initialize_options(self): + self.scope = '.' + self.fix = '' + + def run(self): + scope, check = self.scope, not self.fix + isort_return = self._isort(scope=scope, check=check) + black_return = self._black(scope=scope, check=check) + + if isort_return == 0 and black_return == 0: + self._pass() + else: + self._fail() + exit(isort_return if isort_return != 0 else black_return) + + def finalize_options(self): + pass + + +############################################################################### + +setuptools.setup( + name=__package_name__, + # Versions should comply with PEP440. For a discussion on single-sourcing + # the version across setup.py and the project code, see + # https://packaging.python.org/en/latest/single_source_version.html + version=__version__, + description=__description__, + long_description=long_description, + long_description_content_type=long_description_content_type, + # The project's main homepage. + url=__repository_url__, + download_url=__download_url__, + # Author details + author=__contact_names__, + author_email=__contact_emails__, + # maintainer Details + maintainer=__contact_names__, + maintainer_email=__contact_emails__, + # The licence under which the project is released + license=__license__, + classifiers=[ + # How mature is this project? Common values are + # 1 - Planning + # 2 - Pre-Alpha + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + # 6 - Mature + # 7 - Inactive + 'Development Status :: 4 - Beta', + # Indicate who your project is intended for + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'Intended Audience :: Information Technology', + # Indicate what your project relates to + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Scientific/Engineering :: Image Recognition', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Utilities', + # Pick your license as you wish (should match "license" above) + 'License :: OSI Approved :: Apache Software License', + # Supported python versions + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + # Additional Setting + 'Environment :: Console', + 'Natural Language :: English', + 'Operating System :: OS Independent', + ], + packages=setuptools.find_packages(), + install_requires=install_requires, + setup_requires=['pytest-runner'], + tests_require=tests_requirements, + # List additional groups of dependencies here (e.g. development + # dependencies). You can install these using the following syntax, + # $ pip install -e ".[all]" + # $ pip install nemo_toolkit[all] + extras_require=extras_require, + # Add in any packaged data. + include_package_data=True, + exclude=['tools', 'tests'], + package_data={'': ['*.tsv', '*.txt', '*.far']}, + zip_safe=False, + # PyPI package information. + keywords=__keywords__, + # Custom commands. + cmdclass={'style': StyleCommand}, +) diff --git a/models/tacotron2/NeMo/tests/__init__.py b/models/tacotron2/NeMo/tests/__init__.py new file mode 100644 index 0000000000..9e32500719 --- /dev/null +++ b/models/tacotron2/NeMo/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/test_rnnt_pytorch.py b/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/test_rnnt_pytorch.py new file mode 100644 index 0000000000..c8f55021b3 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/test_rnnt_pytorch.py @@ -0,0 +1,256 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pytest +import torch + +from nemo.collections.asr.parts.numba.rnnt_loss.rnnt_numpy import RNNTLoss as RNNTLoss_Numpy +from nemo.collections.asr.parts.numba.rnnt_loss.rnnt_pytorch import RNNTLossNumba +from nemo.core.utils import numba_utils +from nemo.core.utils.numba_utils import __NUMBA_MINIMUM_VERSION__ + +DEVICES = ['cpu'] + +if torch.cuda.is_available(): + DEVICES.append('cuda') + + +def wrap_and_call(fn, acts, labels, device): + acts = torch.FloatTensor(acts) + if 'cuda' in device: + acts = acts.cuda() + acts.requires_grad = True + + lengths = [acts.shape[1]] * acts.shape[0] + label_lengths = [len(l) for l in labels] + labels = torch.IntTensor(labels) + lengths = torch.IntTensor(lengths) + label_lengths = torch.IntTensor(label_lengths) + if 'cuda' in device: + labels = labels.cuda() + lengths = lengths.cuda() + label_lengths = label_lengths.cuda() + + costs = fn(acts, labels, lengths, label_lengths) + cost = torch.sum(costs) + cost.backward() + return costs.data.cpu().numpy(), acts.grad.data.cpu().numpy() + + +class TestRNNTLossPytorch: + @pytest.mark.unit + @pytest.mark.parametrize('device', DEVICES) + def test_case_small(self, device): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + acts = np.array( + [ + [ + [[0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1], [0.1, 0.1, 0.2, 0.8, 0.1]], + [[0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.2, 0.1, 0.1], [0.7, 0.1, 0.2, 0.1, 0.1]], + ] + ] + ) + labels = [[1, 2]] + + fn_pt = RNNTLossNumba(blank=0, reduction='sum') + pt_cost, pt_grads = wrap_and_call(fn_pt, acts, labels, device) + + fn_np = RNNTLoss_Numpy() + np_cost, np_grads = wrap_and_call(fn_np, acts, labels, device) + + expected_cost = 4.495666 + expected_grads = np.array( + [ + [ + [ + [-0.13116688, -0.3999269, 0.17703125, 0.17703125, 0.17703125], + [-0.18572757, 0.12247056, -0.18168412, 0.12247056, 0.12247056], + [-0.32091254, 0.06269141, 0.06928472, 0.12624499, 0.06269141], + ], + [ + [0.05456069, -0.21824276, 0.05456069, 0.05456069, 0.05456069], + [0.12073959, 0.12073959, -0.48295835, 0.12073959, 0.12073959], + [-0.6925882, 0.16871116, 0.18645467, 0.16871116, 0.16871116], + ], + ] + ] + ) + assert np.allclose(pt_cost, expected_cost, rtol=1e-6), "small_test costs mismatch." + assert np.allclose(pt_grads, expected_grads), "small_test gradient mismatch." + + assert np.allclose(pt_cost, np_cost, rtol=1e-6), "small_test costs mismatch." + assert np.allclose(pt_grads, np_grads), "small_test gradient mismatch." + + @pytest.mark.unit + @pytest.mark.parametrize('device', DEVICES) + def test_case_small_random(self, device): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + rng = np.random.RandomState(0) + acts = rng.randn(1, 4, 3, 3) + labels = [[1, 2]] + + fn_pt = RNNTLossNumba(blank=0, reduction='sum') + pt_cost, pt_grads = wrap_and_call(fn_pt, acts, labels, device) + + fn_np = RNNTLoss_Numpy() + np_cost, np_grads = wrap_and_call(fn_np, acts, labels, device) + + assert np.allclose(pt_cost, np_cost, rtol=1e-6), "small_random_test costs mismatch." + assert np.allclose(pt_grads, np_grads), "small_random_test gradient mismatch." + + @pytest.mark.unit + @pytest.mark.parametrize('device', DEVICES) + def big_test(self, device): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + # minibatch x T x U x alphabet_size + activations = [ + [ + [ + [0.06535690384862791, 0.7875301411923206, 0.08159176605666074], + [0.5297155426466327, 0.7506749639230854, 0.7541348379087998], + [0.6097641124736383, 0.8681404965673826, 0.6225318186056529], + ], + [ + [0.6685222872103057, 0.8580392805336061, 0.16453892311765583], + [0.989779515236694, 0.944298460961015, 0.6031678586829663], + [0.9467833543605416, 0.666202507295747, 0.28688179752461884], + ], + [ + [0.09418426230195986, 0.3666735970751962, 0.736168049462793], + [0.1666804425271342, 0.7141542198635192, 0.3993997272216727], + [0.5359823524146038, 0.29182076440286386, 0.6126422611507932], + ], + [ + [0.3242405528768486, 0.8007644367291621, 0.5241057606558068], + [0.779194617063042, 0.18331417220174862, 0.113745182072432], + [0.24022162381327106, 0.3394695622533106, 0.1341595066017014], + ], + ], + [ + [ + [0.5055615569388828, 0.051597282072282646, 0.6402903936686337], + [0.43073311517251, 0.8294731834714112, 0.1774668847323424], + [0.3207001991262245, 0.04288308912457006, 0.30280282975568984], + ], + [ + [0.6751777088333762, 0.569537369330242, 0.5584738347504452], + [0.08313242153985256, 0.06016544344162322, 0.10795752845152584], + [0.7486153608562472, 0.943918041459349, 0.4863558118797222], + ], + [ + [0.4181986264486809, 0.6524078485043804, 0.024242983423721887], + [0.13458171554507403, 0.3663418070512402, 0.2958297395361563], + [0.9236695822497084, 0.6899291482654177, 0.7418981733448822], + ], + [ + [0.25000547599982104, 0.6034295486281007, 0.9872887878887768], + [0.5926057265215715, 0.8846724004467684, 0.5434495396894328], + [0.6607698886038497, 0.3771277082495921, 0.3580209022231813], + ], + ], + ] + + expected_costs = [4.2806528590890736, 3.9384369822503591] + expected_grads = [ + [ + [ + [-1.86843902e-01, -6.25548810e-02, 2.49398798e-01], + [-2.03376666e-01, 2.02399328e-01, 9.77333169e-04], + [-1.41016081e-01, 7.91234672e-02, 6.18926100e-02], + ], + [ + [-1.15517676e-02, -8.12802389e-02, 9.28319991e-02], + [-1.54257029e-01, 2.29432687e-01, -7.51756504e-02], + [-2.46593088e-01, 1.46404594e-01, 1.00188486e-01], + ], + [ + [-1.29182907e-02, -6.15932420e-02, 7.45115355e-02], + [-5.59857301e-02, 2.19830811e-01, -1.63845062e-01], + [-4.97626871e-01, 2.09239945e-01, 2.88386941e-01], + ], + [ + [1.36048580e-02, -3.02196294e-02, 1.66147724e-02], + [1.13924511e-01, 6.27811998e-02, -1.76705718e-01], + [-6.67078257e-01, 3.67658824e-01, 2.99419403e-01], + ], + ], + [ + [ + [-3.56343776e-01, -5.53474613e-02, 4.11691219e-01], + [-9.69219357e-02, 2.94591039e-02, 6.74628317e-02], + [-6.35175705e-02, 2.76544970e-02, 3.58630717e-02], + ], + [ + [-1.54499024e-01, -7.39420280e-02, 2.28441030e-01], + [-1.66789949e-01, -8.78955179e-05, 1.66877866e-01], + [-1.72369644e-01, 1.05565332e-01, 6.68043196e-02], + ], + [ + [2.38748826e-02, -1.18255816e-01, 9.43809375e-02], + [-1.04707085e-01, -1.08934477e-01, 2.13641584e-01], + [-3.69844258e-01, 1.80118099e-01, 1.89726159e-01], + ], + [ + [2.57137045e-02, -7.94617534e-02, 5.37480488e-02], + [1.22328237e-01, -2.38788679e-01, 1.16460443e-01], + [-5.98686993e-01, 3.02203178e-01, 2.96483815e-01], + ], + ], + ] + + activations = np.array(activations) + labels = [[1, 2], [1, 1]] + + fn_pt = RNNTLossNumba(blank=0, reduction='sum') + pt_costs, pt_grads = wrap_and_call(fn_pt, activations, labels, device) + + fn_np = RNNTLoss_Numpy() + np_costs, np_grads = wrap_and_call(fn_np, activations, labels, device) + + assert np.allclose(pt_costs, sum(expected_costs)), "big_test average costs mismatch." + assert np.allclose(pt_grads, expected_grads, rtol=1e-3), "big_test grads for average cost mismatch." + + assert np.allclose(pt_costs, np_costs), "big_test average costs mismatch." + assert np.allclose(pt_grads, np_grads, rtol=1e-3), "big_test grads for average cost mismatch." + + @pytest.mark.unit + @pytest.mark.parametrize('device', DEVICES) + def test_case_large_random(self, device): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + rng = np.random.RandomState(0) + acts = rng.randn(4, 8, 11, 5) + labels = [ + [1, 2, 4, 3, 2, 2, 1, 1, 1, 1], + [3, 2, 2, 3, 4, 1, 1, 1, 1, 1], + [4, 4, 1, 2, 1, 3, 4, 3, 1, 2], + [1, 1, 2, 1, 2, 3, 3, 1, 1, 1], + ] + + fn_pt = RNNTLossNumba(blank=0, reduction='sum') + pt_cost, pt_grads = wrap_and_call(fn_pt, acts, labels, device) + + fn_np = RNNTLoss_Numpy() + np_cost, np_grads = wrap_and_call(fn_np, acts, labels, device) + + assert np.allclose(pt_cost, np_cost, atol=1e-5, rtol=1e-3), "large_random_test costs mismatch." + assert np.allclose(pt_grads, np_grads, atol=1e-5, rtol=1e-3), "large_random_test gradient mismatch." + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_gpu_rnnt_kernel.py b/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_gpu_rnnt_kernel.py new file mode 100644 index 0000000000..a0cb84cbb7 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_gpu_rnnt_kernel.py @@ -0,0 +1,169 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pytest +import torch +from numba import cuda + +from nemo.collections.asr.parts.numba.rnnt_loss import rnnt_numpy +from nemo.collections.asr.parts.numba.rnnt_loss.rnnt_pytorch import certify_inputs +from nemo.collections.asr.parts.numba.rnnt_loss.utils.cuda_utils import gpu_rnnt_kernel, reduce +from nemo.core.utils import numba_utils +from nemo.core.utils.numba_utils import __NUMBA_MINIMUM_VERSION__ + + +def log_softmax(x, axis=-1): + x = torch.from_numpy(x) # zero-copy + x = torch.log_softmax(x, axis) + x = x.numpy() + return x + + +class TestRNNTCUDAKernels: + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Reductions can only be run when CUDA is available") + @pytest.mark.unit + def test_compute_alphas_kernel(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + random = np.random.RandomState(0) + original_shape = [1, 5, 11, 3] + B, T, U, V = original_shape + + # Numpy kernel + x = random.randn(*original_shape) + labels = np.array([[1, 1, 1, 2, 2, 2, 1, 2, 2, 1]]) # [1, 10] + label_len = len(labels[0]) + 1 + blank_idx = 0 + + x_np = log_softmax(x, axis=-1) + ground_alphas, ground_log_likelihood = rnnt_numpy.forward_pass( + x_np[0, :, :label_len, :], labels[0, : label_len - 1], blank_idx + ) + + # Pytorch kernel + device = torch.device('cuda') + if hasattr(cuda, 'external_stream'): + stream = cuda.external_stream(torch.cuda.current_stream(device).cuda_stream) + else: + stream = cuda.default_stream() + + x_c = torch.tensor(x, device=device, dtype=torch.float32) + labels_c = torch.tensor(labels, device=device, dtype=torch.int32) + + # Allocate workspace memory + denom = torch.zeros(B * T * U, device=device, dtype=x_c.dtype) + alphas = torch.zeros(B * T * U, device=device, dtype=x_c.dtype) + llForward = torch.zeros(B, device=device, dtype=x_c.dtype) + input_lengths = torch.tensor([T], dtype=torch.int32, device=device) + label_lengths = torch.tensor([len(labels[0])], dtype=torch.int32, device=device) + + # certify input data + certify_inputs(x_c, labels_c, input_lengths, label_lengths) + + # flatten activation tensor (for pointer based indexing) + x_c = x_c.view([-1]) + + # call kernel + # log softmax reduction + reduce.reduce_max(x_c, denom, rows=V, cols=B * T * U, minus=False, stream=stream) + reduce.reduce_exp(x_c, denom, rows=V, cols=B * T * U, minus=True, stream=stream) + + # alpha kernel + gpu_rnnt_kernel.compute_alphas_kernel[B, U, stream, 0]( + x_c, denom, alphas, llForward, input_lengths, label_lengths, labels_c, B, T, U, V, blank_idx, + ) + + # sync kernel + stream.synchronize() + + # reshape alphas + alphas = alphas.view([B, T, U]) + diff = ground_alphas - alphas[0].cpu().numpy() + + assert np.abs(diff).mean() <= 1e-5 + assert np.square(diff).mean() <= 1e-10 + + ll_diff = ground_log_likelihood - llForward[0].cpu().numpy() + + assert np.abs(ll_diff).mean() <= 1e-5 + assert np.square(ll_diff).mean() <= 1e-10 + + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Reductions can only be run when CUDA is available") + @pytest.mark.unit + def test_compute_betas_kernel(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + random = np.random.RandomState(0) + original_shape = [1, 5, 11, 3] + B, T, U, V = original_shape + + # Numpy kernel + x = random.randn(*original_shape) + labels = np.array([[1, 1, 1, 2, 2, 2, 1, 2, 2, 1]]) # [1, 10] + label_len = len(labels[0]) + 1 + blank_idx = 0 + + x_np = log_softmax(x, axis=-1) + ground_alphas, ground_log_likelihood = rnnt_numpy.backward_pass( + x_np[0, :, :label_len, :], labels[0, : label_len - 1], blank_idx + ) + + # Pytorch kernel + device = torch.device('cuda') + if hasattr(cuda, 'external_stream'): + stream = cuda.external_stream(torch.cuda.current_stream(device).cuda_stream) + else: + stream = cuda.default_stream() + + x_c = torch.tensor(x, device=device, dtype=torch.float32) + labels_c = torch.tensor(labels, device=device, dtype=torch.int32) + + # Allocate workspace memory + denom = torch.zeros(B * T * U, device=device, dtype=x_c.dtype) + betas = torch.zeros(B * T * U, device=device, dtype=x_c.dtype) + llBackward = torch.zeros(B, device=device, dtype=x_c.dtype) + input_lengths = torch.tensor([T], dtype=torch.int32, device=device) + label_lengths = torch.tensor([len(labels[0])], dtype=torch.int32, device=device) + + # certify input data + certify_inputs(x_c, labels_c, input_lengths, label_lengths) + + # flatten activation tensor (for pointer based indexing) + x_c = x_c.view([-1]) + + # call kernel + # log softmax reduction + reduce.reduce_max(x_c, denom, rows=V, cols=B * T * U, minus=False, stream=stream) + reduce.reduce_exp(x_c, denom, rows=V, cols=B * T * U, minus=True, stream=stream) + + # beta kernel + gpu_rnnt_kernel.compute_betas_kernel[B, U, stream, 0]( + x_c, denom, betas, llBackward, input_lengths, label_lengths, labels_c, B, T, U, V, blank_idx, + ) + + # sync kernel + stream.synchronize() + + # reshape alphas + betas = betas.view([B, T, U]) + diff = ground_alphas - betas[0].cpu().numpy() + + assert np.abs(diff).mean() <= 1e-5 + assert np.square(diff).mean() <= 1e-10 + + ll_diff = ground_log_likelihood - llBackward[0].cpu().numpy() + + assert np.abs(ll_diff).mean() <= 1e-5 + assert np.square(ll_diff).mean() <= 1e-10 diff --git a/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_reduce.py b/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_reduce.py new file mode 100644 index 0000000000..7c2ba6a412 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_reduce.py @@ -0,0 +1,86 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pytest +from numba import cuda + +from nemo.collections.asr.parts.numba.rnnt_loss.utils.cuda_utils import reduce +from nemo.core.utils import numba_utils +from nemo.core.utils.numba_utils import __NUMBA_MINIMUM_VERSION__ + + +class TestRNNTCUDAReductions: + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Reductions can only be run when CUDA is available") + @pytest.mark.unit + def test_reduce_max(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + random = np.random.RandomState(0) + original_shape = [1, 5, 4, 3] + x = random.randn(*original_shape).reshape([-1]) + dx = random.randn(*x.shape) + + stream = cuda.stream() + x_c = cuda.to_device(x, stream=stream) + dx_c = cuda.to_device(dx, stream=stream) + + # call kernel + cols = np.prod(original_shape[:3]) + reduce.reduce_max(x_c, dx_c, rows=original_shape[-1], cols=cols, minus=False, stream=stream) + + # sync kernel + stream.synchronize() + + dx_result = dx_c.copy_to_host(stream=stream) + del x_c, dx_c + + # collect results in first [B * T * U] values; for all V + assert np.abs(dx_result[cols:] - dx[cols:]).sum() <= 1e-7 + # make sure dx_result updates the [B * T * U] values + assert np.abs(dx_result[:cols] - dx[:cols]).sum() > 0 + + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Reductions can only be run when CUDA is available") + @pytest.mark.unit + def test_reduce_exp(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + random = np.random.RandomState(0) + original_shape = [1, 5, 4, 2] + x = random.randn(*original_shape).reshape([-1]) + dx = np.zeros_like(x) + + stream = cuda.stream() + x_c = cuda.to_device(x, stream=stream) + dx_c = cuda.to_device(dx, stream=stream) + + # call kernel + cols = np.prod(original_shape[:3]) + reduce.reduce_exp(x_c, dx_c, rows=original_shape[-1], cols=cols, minus=False, stream=stream) + + # sync kernel + stream.synchronize() + + dx_result = dx_c.copy_to_host(stream=stream) + del x_c, dx_c + + # collect results in first [B * T * U] values; for all V + assert (dx_result[cols:] - dx[cols:]).sum() <= 1e-7 + + # make sure dx_result updates the [B * T * U] values + assert np.abs(dx_result[:cols] - dx[:cols]).sum() > 0 + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_rnnt_helper.py b/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_rnnt_helper.py new file mode 100644 index 0000000000..d054bae761 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/numba/rnnt_loss/utils/test_rnnt_helper.py @@ -0,0 +1,319 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pytest +from numba import cuda + +from nemo.collections.asr.parts.numba.rnnt_loss.utils import global_constants, rnnt_helper +from nemo.core.utils import numba_utils +from nemo.core.utils.numba_utils import __NUMBA_MINIMUM_VERSION__ + + +class TestRNNTHelper: + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Helpers can only be run when CUDA is available") + @pytest.mark.unit + def test_log_sum_exp(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + # wrapper kernel for device function that is tested + @cuda.jit + def _kernel(x, y): + x_pos = cuda.grid(1) + if x_pos < x.shape[0] and x_pos < y.shape[0]: + x[x_pos] = rnnt_helper.log_sum_exp(x[x_pos], y[x_pos]) + + x = np.zeros([8]) # np.random.rand(8192) + y = np.ones([8]) # np.random.rand(8192) + + stream = cuda.stream() + x_c = cuda.to_device(x, stream=stream) + y_c = cuda.to_device(y, stream=stream) + + # call kernel + threads_per_block = global_constants.threads_per_block() + blocks_per_grid = (x.shape[0] + threads_per_block - 1) // threads_per_block + _kernel[blocks_per_grid, threads_per_block, stream](x_c, y_c) + + # sync kernel + stream.synchronize() + + x_new = x_c.copy_to_host(stream=stream) + del x_c, y_c + + assert (x_new.sum() - 10.506093500145782) <= 1e-5 + + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Helpers can only be run when CUDA is available") + @pytest.mark.unit + def test_log_sum_exp_neg_inf(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + # wrapper kernel for device function that is tested + @cuda.jit + def _kernel(x, y): + x_pos = cuda.grid(1) + if x_pos < x.shape[0] and x_pos < y.shape[0]: + x[x_pos] = rnnt_helper.log_sum_exp(x[x_pos], y[x_pos]) + + x = np.asarray([global_constants.FP32_NEG_INF] * 8) + y = np.ones([len(x)]) + + stream = cuda.stream() + x_c = cuda.to_device(x, stream=stream) + y_c = cuda.to_device(y, stream=stream) + + # call kernel + threads_per_block = global_constants.threads_per_block() + blocks_per_grid = (x.shape[0] + threads_per_block - 1) // threads_per_block + _kernel[blocks_per_grid, threads_per_block, stream](x_c, y_c) + + # sync kernel + stream.synchronize() + + x_new = x_c.copy_to_host(stream=stream) + del x_c, y_c + + assert np.allclose(x_new, np.ones_like(x_new), atol=1e-5) + + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Helpers can only be run when CUDA is available") + @pytest.mark.unit + def test_div_up(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + # wrapper kernel for device function that is tested + @cuda.jit + def _kernel(x, y): + x_pos = cuda.grid(1) + if x_pos < x.shape[0] and x_pos < y.shape[0]: + x[x_pos] = rnnt_helper.div_up(x[x_pos], y[x_pos]) + + x = np.full([8], fill_value=10) # np.random.rand(8192) + y = np.full([8], fill_value=2) # np.random.rand(8192) + + stream = cuda.stream() + x_c = cuda.to_device(x, stream=stream) + y_c = cuda.to_device(y, stream=stream) + + # call kernel + threads_per_block = global_constants.threads_per_block() + blocks_per_grid = (x.shape[0] + threads_per_block - 1) // threads_per_block + _kernel[blocks_per_grid, threads_per_block, stream](x_c, y_c) + + # sync kernel + stream.synchronize() + + x_new = x_c.copy_to_host(stream=stream) + del x_c, y_c + + for i in range(len(x_new)): + assert x_new[i] == ((10 + 2 - 1) // 2) + + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Helpers can only be run when CUDA is available") + @pytest.mark.unit + def test_add(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + # wrapper kernel for device function that is tested + @cuda.jit + def _kernel(x, y): + x_pos = cuda.grid(1) + if x_pos < x.shape[0] and x_pos < y.shape[0]: + x[x_pos] = rnnt_helper.add(x[x_pos], y[x_pos]) + + x = np.full([8], fill_value=10) # np.random.rand(8192) + y = np.full([8], fill_value=2) # np.random.rand(8192) + + stream = cuda.stream() + x_c = cuda.to_device(x, stream=stream) + y_c = cuda.to_device(y, stream=stream) + + # call kernel + threads_per_block = global_constants.threads_per_block() + blocks_per_grid = (x.shape[0] + threads_per_block - 1) // threads_per_block + _kernel[blocks_per_grid, threads_per_block, stream](x_c, y_c) + + # sync kernel + stream.synchronize() + + x_new = x_c.copy_to_host(stream=stream) + del x_c, y_c + + for i in range(len(x_new)): + assert x_new[i] == 12 + + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Helpers can only be run when CUDA is available") + @pytest.mark.unit + def test_maximum(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + # wrapper kernel for device function that is tested + @cuda.jit + def _kernel(x, y): + x_pos = cuda.grid(1) + if x_pos < x.shape[0] and x_pos < y.shape[0]: + x[x_pos] = rnnt_helper.maximum(x[x_pos], y[x_pos]) + + x = np.full([8], fill_value=10) # np.random.rand(8192) + y = np.full([8], fill_value=2) # np.random.rand(8192) + + stream = cuda.stream() + x_c = cuda.to_device(x, stream=stream) + y_c = cuda.to_device(y, stream=stream) + + # call kernel + threads_per_block = global_constants.threads_per_block() + blocks_per_grid = (x.shape[0] + threads_per_block - 1) // threads_per_block + _kernel[blocks_per_grid, threads_per_block, stream](x_c, y_c) + + # sync kernel + stream.synchronize() + + x_new = x_c.copy_to_host(stream=stream) + del x_c, y_c + + for i in range(len(x_new)): + assert x_new[i] == 10 + + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Helpers can only be run when CUDA is available") + @pytest.mark.unit + def test_identity(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + # wrapper kernel for device function that is tested + @cuda.jit + def _kernel(x): + x_pos = cuda.grid(1) + if x_pos < x.shape[0]: + x[x_pos] = rnnt_helper.identity(x[x_pos]) + + x = np.full([8], fill_value=10) # np.random.rand(8192) + + stream = cuda.stream() + x_c = cuda.to_device(x, stream=stream) + + # call kernel + threads_per_block = global_constants.threads_per_block() + blocks_per_grid = (x.shape[0] + threads_per_block - 1) // threads_per_block + _kernel[blocks_per_grid, threads_per_block, stream](x_c) + + # sync kernel + stream.synchronize() + + x_new = x_c.copy_to_host(stream=stream) + del x_c + + for i in range(len(x_new)): + assert x_new[i] == x[i] + + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Helpers can only be run when CUDA is available") + @pytest.mark.unit + def test_negate(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + # wrapper kernel for device function that is tested + @cuda.jit + def _kernel(x): + x_pos = cuda.grid(1) + if x_pos < x.shape[0]: + x[x_pos] = rnnt_helper.negate(x[x_pos]) + + x = np.full([8], fill_value=10) # np.random.rand(8192) + + stream = cuda.stream() + x_c = cuda.to_device(x, stream=stream) + + # call kernel + threads_per_block = global_constants.threads_per_block() + blocks_per_grid = (x.shape[0] + threads_per_block - 1) // threads_per_block + _kernel[blocks_per_grid, threads_per_block, stream](x_c) + + # sync kernel + stream.synchronize() + + x_new = x_c.copy_to_host(stream=stream) + del x_c + + for i in range(len(x_new)): + assert x_new[i] == -x[i] + + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Helpers can only be run when CUDA is available") + @pytest.mark.unit + def test_exponential(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + # wrapper kernel for device function that is tested + @cuda.jit + def _kernel(x): + x_pos = cuda.grid(1) + if x_pos < x.shape[0]: + x[x_pos] = rnnt_helper.exponential(x[x_pos]) + + x = np.random.rand(8) + + stream = cuda.stream() + x_c = cuda.to_device(x, stream=stream) + + # call kernel + threads_per_block = global_constants.threads_per_block() + blocks_per_grid = (x.shape[0] + threads_per_block - 1) // threads_per_block + _kernel[blocks_per_grid, threads_per_block, stream](x_c) + + # sync kernel + stream.synchronize() + + x_new = x_c.copy_to_host(stream=stream) + del x_c + + y = np.exp(x) + for i in range(len(x_new)): + assert (x_new[i] - y[i]) < 1e-4 + + @pytest.mark.skipif(not cuda.is_available(), reason="CUDA Helpers can only be run when CUDA is available") + @pytest.mark.unit + def test_log_plus(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + # wrapper kernel for device function that is tested + @cuda.jit + def _kernel(x, y): + x_pos = cuda.grid(1) + if x_pos < x.shape[0] and x_pos < y.shape[0]: + x[x_pos] = rnnt_helper.log_plus(x[x_pos], y[x_pos]) + + x = np.full([8], fill_value=10.0) # np.random.rand(8192) + y = np.full([8], fill_value=2.0) # np.random.rand(8192) + + stream = cuda.stream() + x_c = cuda.to_device(x, stream=stream) + y_c = cuda.to_device(y, stream=stream) + + # call kernel + threads_per_block = global_constants.threads_per_block() + blocks_per_grid = (x.shape[0] + threads_per_block - 1) // threads_per_block + _kernel[blocks_per_grid, threads_per_block, stream](x_c, y_c) + + # sync kernel + stream.synchronize() + + x_new = x_c.copy_to_host(stream=stream) + del x_c, y_c + + z = np.log1p(np.exp(-np.fabs(x - y))) + np.maximum(x, y) + + for i in range(len(x_new)): + assert x_new[i] == z[i] + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/models/tacotron2/NeMo/tests/collections/asr/numba/spec_augment/test_spec_aug_numba.py b/models/tacotron2/NeMo/tests/collections/asr/numba/spec_augment/test_spec_aug_numba.py new file mode 100644 index 0000000000..17b6d6c9cc --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/numba/spec_augment/test_spec_aug_numba.py @@ -0,0 +1,252 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import torch +from omegaconf import OmegaConf + +from nemo.collections.asr.parts.numba.spec_augment import spec_aug_numba +from nemo.core.utils import numba_utils +from nemo.core.utils.numba_utils import __NUMBA_MINIMUM_VERSION__ + + +def get_cfg(seed=0, dtype='float32'): + # fmt: off + cfg = OmegaConf.create( + dict(b=2, f=80, t=20, device='cuda', + freq_masks=2, time_masks=2, freq_width=27, time_width=0.05, mask_value=0.0, + seed=seed, dtype=dtype) + ) + # fmt: on + return cfg + + +# fmt: off +def prepare_data(b, f, t, device='cuda', freq_masks=0, time_masks=0, freq_width=10, time_width=0.1, + seed=0, dtype='float32', + **kwargs): + torch.manual_seed(seed) + + if dtype == 'float16': + dtype = torch.float16 + else: + dtype = torch.float + + x = torch.randn([b, f, t], dtype=dtype, device=device) + x_len = torch.randint(t, size=[b], device=x.device) + + sh = x.shape + bs = sh[0] + + if isinstance(time_width, int): + adaptive_temporal_width = False + else: + if time_width > 1.0 or time_width < 0.0: + raise ValueError('If `time_width` is a float value, must be in range [0, 1]') + + adaptive_temporal_width = True + + if adaptive_temporal_width: + time_width = max(1, int(sh[2] * time_width)) + else: + time_width = time_width + + # Construct the freq and time masks as well as start positions + if freq_masks > 0: + freq_starts = torch.randint(0, sh[1] - freq_width + 1, size=[bs, freq_masks], device=x.device) + freq_lengths = torch.randint(0, freq_width + 1, size=[bs, freq_masks], device=x.device) + else: + freq_starts = torch.zeros([bs, 1], dtype=torch.int64, device=x.device) + freq_lengths = torch.zeros([bs, 1], dtype=torch.int64, device=x.device) + + if time_masks > 0: + time_starts = torch.randint(0, sh[2] - time_width + 1, size=[bs, time_masks], device=x.device) + time_lengths = torch.randint(0, time_width + 1, size=[bs, time_masks], device=x.device) + else: + time_starts = torch.zeros([bs, 1], dtype=torch.int64, device=x.device) + time_lengths = torch.zeros([bs, 1], dtype=torch.int64, device=x.device) + + output = dict( + x=x, + x_len=x_len, + freq_starts=freq_starts, + freq_lengths=freq_lengths, + time_starts=time_starts, + time_lengths=time_lengths, + sh=sh, + ) + return output +# fmt: on + + +def launch_kernel(data, cfg): + # Launch CUDA kernel + # fmt: off + data['x'] = spec_aug_numba.launch_spec_augment_kernel( + x=data['x'], x_len=data['x_len'], + freq_starts=data['freq_starts'], freq_lengths=data['freq_lengths'], + time_starts=data['time_starts'], time_lengths=data['time_lengths'], + freq_masks=cfg.freq_masks, time_masks=cfg.time_masks, mask_value=cfg.mask_value + ) + # fmt: on + + +def freq_mask_check(x, x_len, f_start, f_len, mask_value, bidx): + check_result = True + for fidx in range(f_start, f_start + f_len): + if not (x[bidx, fidx, :] == mask_value).all(): + check_result = False + break + assert check_result + + +def time_mask_check(x, x_len, t_start, t_len, mask_value, bidx): + check_result = True + for tidx in range(t_start, t_start + t_len): + if tidx >= x_len[bidx]: + # this sample has smaller length than the time index of mask, ignore + continue + + if not (x[bidx, :, tidx] == mask_value).all(): + check_result = False + break + assert check_result + + +class TestSpecAugmentNumba: + @pytest.mark.unit + @pytest.mark.run_only_on('GPU') + @pytest.mark.parametrize('dtype', ['float16', 'float32']) + def test_spec_aug_kernel(self, dtype): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + cfg = get_cfg(seed=0, dtype=dtype) + cfg.freq_masks = 2 + cfg.time_masks = 10 + + data = prepare_data(**cfg) + + launch_kernel(data, cfg) + x, x_len, sh = data['x'], data['x_len'], data['sh'] + + # Assert freq masks are correct + for bidx in range(sh[0]): + for f_start, f_len in zip(data['freq_starts'][bidx], data['freq_lengths'][bidx]): + freq_mask_check(x, x_len, f_start, f_len, mask_value=cfg.mask_value, bidx=bidx) + + # Assert time masks are correct + for bidx in range(sh[0]): + for t_start, t_len in zip(data['time_starts'][bidx], data['time_lengths'][bidx]): + time_mask_check(x, x_len, t_start, t_len, mask_value=cfg.mask_value, bidx=bidx) + + @pytest.mark.unit + @pytest.mark.run_only_on('GPU') + def test_spec_aug_kernel_mask_value(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + cfg = get_cfg(seed=0) + cfg.freq_masks = 2 + cfg.time_masks = 10 + cfg.mask_value = -1.0 + + data = prepare_data(**cfg) + + launch_kernel(data, cfg) + x, x_len, sh = data['x'], data['x_len'], data['sh'] + + # Assert freq masks are correct + for bidx in range(sh[0]): + for f_start, f_len in zip(data['freq_starts'][bidx], data['freq_lengths'][bidx]): + freq_mask_check(x, x_len, f_start, f_len, mask_value=cfg.mask_value, bidx=bidx) + + # Assert time masks are correct + for bidx in range(sh[0]): + for t_start, t_len in zip(data['time_starts'][bidx], data['time_lengths'][bidx]): + time_mask_check(x, x_len, t_start, t_len, mask_value=cfg.mask_value, bidx=bidx) + + @pytest.mark.unit + @pytest.mark.run_only_on('GPU') + def test_spec_aug_kernel_grad(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + cfg = get_cfg(seed=0) + cfg.freq_masks = 2 + cfg.time_masks = 10 + + data = prepare_data(**cfg) + + launch_kernel(data, cfg) + + result = data['x'] # inplace modification via kernel + y = torch.ones_like(result, requires_grad=True) + z = y + result + z.mean().backward() + + assert y.grad is not None + + @pytest.mark.unit + @pytest.mark.run_only_on('GPU') + def test_spec_aug_kernel_no_freq_mask(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + cfg = get_cfg(seed=0) + cfg.freq_masks = 0 + cfg.time_masks = 10 + + data = prepare_data(**cfg) + + launch_kernel(data, cfg) + x, x_len, sh = data['x'], data['x_len'], data['sh'] + + # Assert time masks are correct + for bidx in range(sh[0]): + for t_start, t_len in zip(data['time_starts'][bidx], data['time_lengths'][bidx]): + time_mask_check(x, x_len, t_start, t_len, mask_value=cfg.mask_value, bidx=bidx) + + @pytest.mark.unit + @pytest.mark.run_only_on('GPU') + def test_spec_aug_kernel_no_time_mask(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + cfg = get_cfg(seed=0) + cfg.freq_masks = 2 + cfg.time_masks = 0 + + data = prepare_data(**cfg) + + launch_kernel(data, cfg) + x, x_len, sh = data['x'], data['x_len'], data['sh'] + + # Assert freq masks are correct + for bidx in range(sh[0]): + for f_start, f_len in zip(data['freq_starts'][bidx], data['freq_lengths'][bidx]): + freq_mask_check(x, x_len, f_start, f_len, mask_value=cfg.mask_value, bidx=bidx) + + @pytest.mark.unit + @pytest.mark.run_only_on('GPU') + def test_spec_aug_kernel_no_freq_time_mask(self): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + cfg = get_cfg(seed=0) + cfg.freq_masks = 0 + cfg.time_masks = 0 + + data = prepare_data(**cfg) + + x, x_len, sh = data['x'], data['x_len'], data['sh'] + x_copy = x.clone() + launch_kernel(data, cfg) + + # Assert no data edits occured + assert (data['x'] - x_copy).abs().mean() <= 1e-9 diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_asr_classification_model.py b/models/tacotron2/NeMo/tests/collections/asr/test_asr_classification_model.py new file mode 100644 index 0000000000..24c88d9beb --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_asr_classification_model.py @@ -0,0 +1,208 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import os + +import pytest +import torch +from omegaconf import DictConfig, ListConfig + +from nemo.collections.asr.data import audio_to_label +from nemo.collections.asr.models import EncDecClassificationModel, configs +from nemo.utils.config_utils import assert_dataclass_signature_match + + +@pytest.fixture() +def speech_classification_model(): + preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} + encoder = { + 'cls': 'nemo.collections.asr.modules.ConvASREncoder', + 'params': { + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 32, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], + }, + } + + decoder = { + 'cls': 'nemo.collections.asr.modules.ConvASRDecoderClassification', + 'params': {'feat_in': 32, 'num_classes': 30,}, + } + + modelConfig = DictConfig( + { + 'preprocessor': DictConfig(preprocessor), + 'encoder': DictConfig(encoder), + 'decoder': DictConfig(decoder), + 'labels': ListConfig(["dummy_cls_{}".format(i + 1) for i in range(30)]), + } + ) + model = EncDecClassificationModel(cfg=modelConfig) + return model + + +class TestEncDecClassificationModel: + @pytest.mark.unit + def test_constructor(self, speech_classification_model): + asr_model = speech_classification_model.train() + + conv_cnt = (64 * 32 * 1 + 32) + (64 * 1 * 1 + 32) # separable kernel + bias + pointwise kernel + bias + bn_cnt = (4 * 32) * 2 # 2 * moving averages + dec_cnt = 32 * 30 + 30 # fc + bias + + param_count = conv_cnt + bn_cnt + dec_cnt + assert asr_model.num_weights == param_count + + # Check to/from config_dict: + confdict = asr_model.to_config_dict() + instance2 = EncDecClassificationModel.from_config_dict(confdict) + + assert isinstance(instance2, EncDecClassificationModel) + + @pytest.mark.unit + def test_forward(self, speech_classification_model): + asr_model = speech_classification_model.eval() + + asr_model.preprocessor.featurizer.dither = 0.0 + asr_model.preprocessor.featurizer.pad_to = 0 + + input_signal = torch.randn(size=(4, 512)) + length = torch.randint(low=161, high=500, size=[4]) + + with torch.no_grad(): + # batch size 1 + logprobs_instance = [] + for i in range(input_signal.size(0)): + logprobs_ins = asr_model.forward( + input_signal=input_signal[i : i + 1], input_signal_length=length[i : i + 1] + ) + logprobs_instance.append(logprobs_ins) + logprobs_instance = torch.cat(logprobs_instance, 0) + + # batch size 4 + logprobs_batch = asr_model.forward(input_signal=input_signal, input_signal_length=length) + + assert logprobs_instance.shape == logprobs_batch.shape + diff = torch.mean(torch.abs(logprobs_instance - logprobs_batch)) + assert diff <= 1e-6 + diff = torch.max(torch.abs(logprobs_instance - logprobs_batch)) + assert diff <= 1e-6 + + @pytest.mark.unit + def test_vocab_change(self, speech_classification_model): + asr_model = speech_classification_model.train() + + old_labels = copy.deepcopy(asr_model._cfg.labels) + nw1 = asr_model.num_weights + asr_model.change_labels(new_labels=old_labels) + # No change + assert nw1 == asr_model.num_weights + new_labels = copy.deepcopy(old_labels) + new_labels.append('dummy_cls_31') + new_labels.append('dummy_cls_32') + new_labels.append('dummy_cls_33') + asr_model.change_labels(new_labels=new_labels) + # fully connected + bias + assert asr_model.num_weights == nw1 + 3 * (asr_model.decoder._feat_in + 1) + + @pytest.mark.unit + def test_transcription(self, speech_classification_model, test_data_dir): + # Ground truth labels = ["yes", "no"] + audio_filenames = ['an22-flrp-b.wav', 'an90-fbbh-b.wav'] + audio_paths = [os.path.join(test_data_dir, "asr", "train", "an4", "wav", fp) for fp in audio_filenames] + + model = speech_classification_model.eval() + + # Test Top 1 classification transcription + results = model.transcribe(audio_paths, batch_size=2) + assert len(results) == 2 + assert results[0].shape == torch.Size([1]) + + # Test Top 5 classification transcription + model._accuracy.top_k = [5] # set top k to 5 for accuracy calculation + results = model.transcribe(audio_paths, batch_size=2) + assert len(results) == 2 + assert results[0].shape == torch.Size([5]) + + # Test Top 1 and Top 5 classification transcription + model._accuracy.top_k = [1, 5] + results = model.transcribe(audio_paths, batch_size=2) + assert len(results) == 2 + assert results[0].shape == torch.Size([2, 1]) + assert results[1].shape == torch.Size([2, 5]) + assert model._accuracy.top_k == [1, 5] + + # Test log probs extraction + model._accuracy.top_k = [1] + results = model.transcribe(audio_paths, batch_size=2, logprobs=True) + assert len(results) == 2 + assert results[0].shape == torch.Size([len(model.cfg.labels)]) + + # Test log probs extraction remains same for any top_k + model._accuracy.top_k = [5] + results = model.transcribe(audio_paths, batch_size=2, logprobs=True) + assert len(results) == 2 + assert results[0].shape == torch.Size([len(model.cfg.labels)]) + + @pytest.mark.unit + def test_EncDecClassificationDatasetConfig_for_AudioToSpeechLabelDataset(self): + # ignore some additional arguments as dataclass is generic + IGNORE_ARGS = [ + 'is_tarred', + 'num_workers', + 'batch_size', + 'tarred_audio_filepaths', + 'shuffle', + 'pin_memory', + 'drop_last', + 'tarred_shard_strategy', + 'shuffle_n', + # `featurizer` is supplied at runtime + 'featurizer', + # additional ignored arguments + 'vad_stream', + 'int_values', + 'sample_rate', + 'normalize_audio', + 'augmentor', + ] + + REMAP_ARGS = {'trim_silence': 'trim'} + + result = assert_dataclass_signature_match( + audio_to_label.AudioToSpeechLabelDataset, + configs.EncDecClassificationDatasetConfig, + ignore_args=IGNORE_ARGS, + remap_args=REMAP_ARGS, + ) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py b/models/tacotron2/NeMo/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py new file mode 100644 index 0000000000..b42a06d9a0 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py @@ -0,0 +1,313 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import os +import shutil +import tempfile + +import pytest +import torch +from omegaconf import DictConfig + +from nemo.collections.asr.data import audio_to_text +from nemo.collections.asr.models import configs +from nemo.collections.asr.models.ctc_bpe_models import EncDecCTCModelBPE +from nemo.collections.common import tokenizers +from nemo.utils.config_utils import assert_dataclass_signature_match + + +@pytest.fixture() +def asr_model(test_data_dir): + preprocessor = {'_target_': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor'} + encoder = { + '_target_': 'nemo.collections.asr.modules.ConvASREncoder', + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 1024, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], + } + + decoder = { + '_target_': 'nemo.collections.asr.modules.ConvASRDecoder', + 'feat_in': 1024, + 'num_classes': -1, + 'vocabulary': None, + } + + tokenizer = {'dir': os.path.join(test_data_dir, "asr", "tokenizers", "an4_wpe_128"), 'type': 'wpe'} + + modelConfig = DictConfig( + { + 'preprocessor': DictConfig(preprocessor), + 'encoder': DictConfig(encoder), + 'decoder': DictConfig(decoder), + 'tokenizer': DictConfig(tokenizer), + } + ) + + model_instance = EncDecCTCModelBPE(cfg=modelConfig) + return model_instance + + +class TestEncDecCTCModel: + @pytest.mark.unit + def test_constructor(self, asr_model): + asr_model.train() + # TODO: make proper config and assert correct number of weights + # Check to/from config_dict: + confdict = asr_model.to_config_dict() + instance2 = EncDecCTCModelBPE.from_config_dict(confdict) + assert isinstance(instance2, EncDecCTCModelBPE) + + @pytest.mark.unit + def test_forward(self, asr_model): + asr_model = asr_model.eval() + + asr_model.preprocessor.featurizer.dither = 0.0 + asr_model.preprocessor.featurizer.pad_to = 0 + + input_signal = torch.randn(size=(4, 512)) + length = torch.randint(low=161, high=500, size=[4]) + + with torch.no_grad(): + # batch size 1 + logprobs_instance = [] + for i in range(input_signal.size(0)): + logprobs_ins, _, _ = asr_model.forward( + input_signal=input_signal[i : i + 1], input_signal_length=length[i : i + 1] + ) + logprobs_instance.append(logprobs_ins) + print(len(logprobs_ins)) + logprobs_instance = torch.cat(logprobs_instance, 0) + + # batch size 4 + logprobs_batch, _, _ = asr_model.forward(input_signal=input_signal, input_signal_length=length) + + assert logprobs_instance.shape == logprobs_batch.shape + diff = torch.mean(torch.abs(logprobs_instance - logprobs_batch)) + assert diff <= 1e-6 + diff = torch.max(torch.abs(logprobs_instance - logprobs_batch)) + assert diff <= 1e-6 + + @pytest.mark.unit + def test_save_restore_artifact(self, asr_model): + with tempfile.TemporaryDirectory() as tmpdir: + save_path = os.path.join(tmpdir, 'ctc_bpe.nemo') + asr_model.train() + asr_model.save_to(save_path) + + new_model = EncDecCTCModelBPE.restore_from(save_path) + assert isinstance(new_model, type(asr_model)) + assert new_model.vocab_path.endswith('_vocab.txt') + + assert len(new_model.tokenizer.tokenizer.get_vocab()) == 128 + + @pytest.mark.unit + def test_save_restore_artifact_spe(self, asr_model, test_data_dir): + with tempfile.TemporaryDirectory() as tmpdir: + tokenizer_dir = os.path.join(test_data_dir, "asr", "tokenizers", "an4_spe_128") + asr_model.change_vocabulary(new_tokenizer_dir=tokenizer_dir, new_tokenizer_type='bpe') + + save_path = os.path.join(tmpdir, 'ctc_bpe.nemo') + asr_model.train() + asr_model.save_to(save_path) + + new_model = EncDecCTCModelBPE.restore_from(save_path) + assert isinstance(new_model, type(asr_model)) + assert isinstance(new_model.tokenizer, tokenizers.SentencePieceTokenizer) + assert new_model.model_path.endswith('_tokenizer.model') + assert new_model.vocab_path.endswith('_vocab.txt') + assert new_model.spe_vocab_path.endswith('_tokenizer.vocab') + + assert len(new_model.tokenizer.tokenizer.get_vocab()) == 128 + + @pytest.mark.unit + def test_vocab_change(self, test_data_dir, asr_model): + old_vocab = copy.deepcopy(asr_model.decoder.vocabulary) + + with tempfile.TemporaryDirectory() as save_dir: + save_path = os.path.join(save_dir, 'temp.nemo') + + with tempfile.TemporaryDirectory() as tmpdir: + old_tmpdir_path = tmpdir + + old_tokenizer_dir = os.path.join(test_data_dir, "asr", "tokenizers", "an4_wpe_128", 'vocab.txt') + new_tokenizer_dir = os.path.join(tmpdir, 'tokenizer') + + os.makedirs(new_tokenizer_dir, exist_ok=True) + shutil.copy2(old_tokenizer_dir, new_tokenizer_dir) + + nw1 = asr_model.num_weights + asr_model.change_vocabulary(new_tokenizer_dir=new_tokenizer_dir, new_tokenizer_type='wpe') + # No change + assert nw1 == asr_model.num_weights + + with open(os.path.join(new_tokenizer_dir, 'vocab.txt'), 'a+') as f: + f.write("!\n") + f.write('$\n') + f.write('@\n') + + asr_model.change_vocabulary(new_tokenizer_dir=new_tokenizer_dir, new_tokenizer_type='wpe') + # fully connected + bias + assert asr_model.num_weights == nw1 + 3 * (asr_model.decoder._feat_in + 1) + + new_vocab = copy.deepcopy(asr_model.decoder.vocabulary) + assert len(old_vocab) != len(new_vocab) + + # save the model (after change of vocabulary) + asr_model.save_to(save_path) + assert os.path.exists(save_path) + # delete copied version of the vocabulary from nested tmpdir (by scope) + + # assert copied vocab no longer exists + assert not os.path.exists(os.path.join(old_tmpdir_path, 'tokenizer', 'vocab.txt')) + + # make a copy of the tokenizer before renaming + try: + os.rename(old_tokenizer_dir, old_tokenizer_dir + '.bkp') + assert not os.path.exists(old_tokenizer_dir) + + # restore model from .nemo + asr_model2 = EncDecCTCModelBPE.restore_from(save_path) + assert isinstance(asr_model2, EncDecCTCModelBPE) + + # Check if vocabulary size is same + assert asr_model.tokenizer.tokenizer.vocab_size == asr_model2.tokenizer.tokenizer.vocab_size + + # Make a copy of the tokenizer + new_tokenizer_dir = os.path.join(save_dir, 'tokenizer') + + os.makedirs(new_tokenizer_dir, exist_ok=True) + new_tokenizer_path = os.path.join(new_tokenizer_dir, 'vocab.txt') + with open(new_tokenizer_path, 'w') as f: + for v in asr_model2.tokenizer.tokenizer.get_vocab(): + f.write(f"{v}\n") + + # Add some new tokens too + f.write("^\n") + f.write("^^\n") + f.write("^^^\n") + + assert os.path.exists(new_tokenizer_path) + + # change vocabulary + asr_model2.change_vocabulary(new_tokenizer_dir, new_tokenizer_type='wpe') + assert asr_model.tokenizer.vocab_size != asr_model2.tokenizer.vocab_size + + new_save_path = os.path.join(save_dir, 'temp2.nemo') + asr_model2.save_to(new_save_path) + + asr_model3 = EncDecCTCModelBPE.restore_from(new_save_path) + assert isinstance(asr_model3, EncDecCTCModelBPE) + + # Check if vocabulary size is same + assert asr_model2.tokenizer.tokenizer.vocab_size == asr_model3.tokenizer.tokenizer.vocab_size + assert asr_model2.vocab_path != asr_model3.vocab_path + + # Model PT level checks + assert len(asr_model2.artifacts) == 1 + + finally: + os.rename(old_tokenizer_dir + '.bkp', old_tokenizer_dir) + + @pytest.mark.unit + def test_EncDecCTCDatasetConfig_for_AudioToBPEDataset(self): + # ignore some additional arguments as dataclass is generic + IGNORE_ARGS = [ + 'is_tarred', + 'num_workers', + 'batch_size', + 'tarred_audio_filepaths', + 'shuffle', + 'pin_memory', + 'drop_last', + 'tarred_shard_strategy', + 'shuffle_n', + 'parser', + 'normalize', + 'unk_index', + 'pad_id', + 'bos_id', + 'eos_id', + 'blank_index', + ] + + REMAP_ARGS = {'trim_silence': 'trim', 'labels': 'tokenizer'} + + result = assert_dataclass_signature_match( + audio_to_text.AudioToBPEDataset, + configs.EncDecCTCDatasetConfig, + ignore_args=IGNORE_ARGS, + remap_args=REMAP_ARGS, + ) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None + + @pytest.mark.unit + def test_EncDecCTCDatasetConfig_for_TarredAudioToBPEDataset(self): + # ignore some additional arguments as dataclass is generic + IGNORE_ARGS = [ + 'is_tarred', + 'num_workers', + 'batch_size', + 'shuffle', + 'pin_memory', + 'drop_last', + 'parser', + 'normalize', + 'unk_index', + 'pad_id', + 'bos_id', + 'eos_id', + 'blank_index', + 'global_rank', + 'world_size', + ] + + REMAP_ARGS = { + 'trim_silence': 'trim', + 'tarred_audio_filepaths': 'audio_tar_filepaths', + 'tarred_shard_strategy': 'shard_strategy', + 'shuffle_n': 'shuffle', + 'labels': 'tokenizer', + } + + result = assert_dataclass_signature_match( + audio_to_text.TarredAudioToBPEDataset, + configs.EncDecCTCDatasetConfig, + ignore_args=IGNORE_ARGS, + remap_args=REMAP_ARGS, + ) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_asr_ctcencdec_model.py b/models/tacotron2/NeMo/tests/collections/asr/test_asr_ctcencdec_model.py new file mode 100644 index 0000000000..be9c502f23 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_asr_ctcencdec_model.py @@ -0,0 +1,305 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy + +import pytest +import torch +from omegaconf import DictConfig, OmegaConf, open_dict + +import nemo.collections.asr as nemo_asr +from nemo.collections.asr.data import audio_to_text +from nemo.collections.asr.models import EncDecCTCModel, configs +from nemo.utils.config_utils import assert_dataclass_signature_match, update_model_config + + +@pytest.fixture() +def asr_model(): + preprocessor = {'_target_': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor'} + encoder = { + '_target_': 'nemo.collections.asr.modules.ConvASREncoder', + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 1024, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], + } + + decoder = { + '_target_': 'nemo.collections.asr.modules.ConvASRDecoder', + 'feat_in': 1024, + 'num_classes': 28, + 'vocabulary': [ + ' ', + 'a', + 'b', + 'c', + 'd', + 'e', + 'f', + 'g', + 'h', + 'i', + 'j', + 'k', + 'l', + 'm', + 'n', + 'o', + 'p', + 'q', + 'r', + 's', + 't', + 'u', + 'v', + 'w', + 'x', + 'y', + 'z', + "'", + ], + } + modelConfig = DictConfig( + {'preprocessor': DictConfig(preprocessor), 'encoder': DictConfig(encoder), 'decoder': DictConfig(decoder)} + ) + + model_instance = EncDecCTCModel(cfg=modelConfig) + return model_instance + + +class TestEncDecCTCModel: + @pytest.mark.unit + def test_constructor(self, asr_model): + asr_model.train() + # TODO: make proper config and assert correct number of weights + # Check to/from config_dict: + confdict = asr_model.to_config_dict() + instance2 = EncDecCTCModel.from_config_dict(confdict) + assert isinstance(instance2, EncDecCTCModel) + + @pytest.mark.unit + def test_forward(self, asr_model): + asr_model = asr_model.eval() + + asr_model.preprocessor.featurizer.dither = 0.0 + asr_model.preprocessor.featurizer.pad_to = 0 + + input_signal = torch.randn(size=(4, 512)) + length = torch.randint(low=161, high=500, size=[4]) + + with torch.no_grad(): + # batch size 1 + logprobs_instance = [] + for i in range(input_signal.size(0)): + logprobs_ins, _, _ = asr_model.forward( + input_signal=input_signal[i : i + 1], input_signal_length=length[i : i + 1] + ) + logprobs_instance.append(logprobs_ins) + print(len(logprobs_ins)) + logprobs_instance = torch.cat(logprobs_instance, 0) + + # batch size 4 + logprobs_batch, _, _ = asr_model.forward(input_signal=input_signal, input_signal_length=length) + + assert logprobs_instance.shape == logprobs_batch.shape + diff = torch.mean(torch.abs(logprobs_instance - logprobs_batch)) + assert diff <= 1e-6 + diff = torch.max(torch.abs(logprobs_instance - logprobs_batch)) + assert diff <= 1e-6 + + @pytest.mark.unit + def test_vocab_change(self, asr_model): + old_vocab = copy.deepcopy(asr_model.decoder.vocabulary) + nw1 = asr_model.num_weights + asr_model.change_vocabulary(new_vocabulary=old_vocab) + # No change + assert nw1 == asr_model.num_weights + new_vocab = copy.deepcopy(old_vocab) + new_vocab.append('!') + new_vocab.append('$') + new_vocab.append('@') + asr_model.change_vocabulary(new_vocabulary=new_vocab) + # fully connected + bias + assert asr_model.num_weights == nw1 + 3 * (asr_model.decoder._feat_in + 1) + + @pytest.mark.unit + def test_change_conv_asr_se_context_window(self, asr_model): + old_cfg = copy.deepcopy(asr_model.cfg) + asr_model.change_conv_asr_se_context_window(context_window=32) # 32 * 0.01s context + new_config = asr_model.cfg + + assert old_cfg.encoder.jasper[0].se_context_size == -1 + assert new_config.encoder.jasper[0].se_context_size == 32 + + for name, m in asr_model.encoder.named_modules(): + if type(m).__class__.__name__ == 'SqueezeExcite': + assert m.context_window == 32 + + @pytest.mark.unit + def test_change_conv_asr_se_context_window_no_config_update(self, asr_model): + old_cfg = copy.deepcopy(asr_model.cfg) + asr_model.change_conv_asr_se_context_window(context_window=32, update_config=False) # 32 * 0.01s context + new_config = asr_model.cfg + + assert old_cfg.encoder.jasper[0].se_context_size == -1 + assert new_config.encoder.jasper[0].se_context_size == -1 # no change + + for name, m in asr_model.encoder.named_modules(): + if type(m).__class__.__name__ == 'SqueezeExcite': + assert m.context_window == 32 + + @pytest.mark.unit + def test_dataclass_instantiation(self, asr_model): + model_cfg = configs.EncDecCTCModelConfig() + + # Update mandatory values + vocabulary = asr_model.decoder.vocabulary + model_cfg.model.labels = vocabulary + + # Update encoder + model_cfg.model.encoder.activation = 'relu' + model_cfg.model.encoder.feat_in = 64 + model_cfg.model.encoder.jasper = [ + nemo_asr.modules.conv_asr.JasperEncoderConfig( + filters=1024, + repeat=1, + kernel=[1], + stride=[1], + dilation=[1], + dropout=0.0, + residual=False, + se=True, + se_context_size=-1, + ) + ] + + # Update decoder + model_cfg.model.decoder.feat_in = 1024 + model_cfg.model.decoder.num_classes = 28 + model_cfg.model.decoder.vocabulary = vocabulary + + # Construct the model + asr_cfg = OmegaConf.create({'model': asr_model.cfg}) + model_cfg_v1 = update_model_config(model_cfg, asr_cfg) + new_model = EncDecCTCModel(cfg=model_cfg_v1.model) + + assert new_model.num_weights == asr_model.num_weights + # trainer and exp manager should be there + # assert 'trainer' in model_cfg_v1 + # assert 'exp_manager' in model_cfg_v1 + # datasets and optim/sched should not be there after ModelPT.update_model_dataclass() + assert 'train_ds' not in model_cfg_v1.model + assert 'validation_ds' not in model_cfg_v1.model + assert 'test_ds' not in model_cfg_v1.model + assert 'optim' not in model_cfg_v1.model + + # Construct the model, without dropping additional keys + asr_cfg = OmegaConf.create({'model': asr_model.cfg}) + model_cfg_v2 = update_model_config(model_cfg, asr_cfg, drop_missing_subconfigs=False) + + # Assert all components are in config + # assert 'trainer' in model_cfg_v2 + # assert 'exp_manager' in model_cfg_v2 + assert 'train_ds' in model_cfg_v2.model + assert 'validation_ds' in model_cfg_v2.model + assert 'test_ds' in model_cfg_v2.model + assert 'optim' in model_cfg_v2.model + + # Remove extra components (optim and sched can be kept without issue) + with open_dict(model_cfg_v2.model): + model_cfg_v2.model.pop('train_ds') + model_cfg_v2.model.pop('validation_ds') + model_cfg_v2.model.pop('test_ds') + + new_model = EncDecCTCModel(cfg=model_cfg_v2.model) + + assert new_model.num_weights == asr_model.num_weights + # trainer and exp manager should be there + + @pytest.mark.unit + def test_EncDecCTCDatasetConfig_for_AudioToCharDataset(self): + # ignore some additional arguments as dataclass is generic + IGNORE_ARGS = [ + 'is_tarred', + 'num_workers', + 'batch_size', + 'tarred_audio_filepaths', + 'shuffle', + 'pin_memory', + 'drop_last', + 'tarred_shard_strategy', + 'shuffle_n', + 'use_start_end_token', + 'use_start_end_token', + ] + + REMAP_ARGS = {'trim_silence': 'trim'} + + result = assert_dataclass_signature_match( + audio_to_text.AudioToCharDataset, + configs.EncDecCTCDatasetConfig, + ignore_args=IGNORE_ARGS, + remap_args=REMAP_ARGS, + ) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None + + @pytest.mark.unit + def test_EncDecCTCDatasetConfig_for_TarredAudioToCharDataset(self): + # ignore some additional arguments as dataclass is generic + IGNORE_ARGS = [ + 'is_tarred', + 'num_workers', + 'batch_size', + 'shuffle', + 'pin_memory', + 'drop_last', + 'global_rank', + 'world_size', + 'use_start_end_token', + ] + + REMAP_ARGS = { + 'trim_silence': 'trim', + 'tarred_audio_filepaths': 'audio_tar_filepaths', + 'tarred_shard_strategy': 'shard_strategy', + 'shuffle_n': 'shuffle', + } + + result = assert_dataclass_signature_match( + audio_to_text.TarredAudioToCharDataset, + configs.EncDecCTCDatasetConfig, + ignore_args=IGNORE_ARGS, + remap_args=REMAP_ARGS, + ) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_asr_datasets.py b/models/tacotron2/NeMo/tests/collections/asr/test_asr_datasets.py new file mode 100644 index 0000000000..b2a3ccfd09 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_asr_datasets.py @@ -0,0 +1,127 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import os + +import pytest +from omegaconf import OmegaConf + +from nemo.collections.asr.data.audio_to_text import TarredAudioToBPEDataset, TarredAudioToCharDataset +from nemo.collections.asr.data.audio_to_text_dataset import inject_dataloader_value_from_model_config +from nemo.collections.common import tokenizers +from nemo.utils import logging + + +class TestASRDatasets: + labels = [ + " ", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "'", + ] + + @pytest.mark.unit + def test_tarred_dataset(self, test_data_dir): + manifest_path = os.path.abspath(os.path.join(test_data_dir, 'asr/tarred_an4/tarred_audio_manifest.json')) + + # Test braceexpand loading + tarpath = os.path.abspath(os.path.join(test_data_dir, 'asr/tarred_an4/audio_{0..1}.tar')) + ds_braceexpand = TarredAudioToCharDataset( + audio_tar_filepaths=tarpath, manifest_filepath=manifest_path, labels=self.labels, sample_rate=16000 + ) + assert len(ds_braceexpand) == 32 + count = 0 + for _ in ds_braceexpand: + count += 1 + assert count == 32 + + # Test loading via list + tarpath = [os.path.abspath(os.path.join(test_data_dir, f'asr/tarred_an4/audio_{i}.tar')) for i in range(2)] + ds_list_load = TarredAudioToCharDataset( + audio_tar_filepaths=tarpath, manifest_filepath=manifest_path, labels=self.labels, sample_rate=16000 + ) + count = 0 + for _ in ds_list_load: + count += 1 + assert count == 32 + + @pytest.mark.unit + def test_mismatch_in_model_dataloader_config(self, caplog): + logging._logger.propagate = True + caplog.set_level(logging.WARNING) + + model_cfg = OmegaConf.create(dict(labels=OmegaConf.create(["a", "b", "c"]))) + dataloader_cfg = OmegaConf.create(dict(labels=copy.deepcopy(self.labels))) + + inject_dataloader_value_from_model_config(model_cfg, dataloader_cfg, key='labels') + + assert ( + """`labels` is explicitly provided to the data loader, and is different from the `labels` provided at the model level config.""" + in caplog.text + ) + + logging._logger.propagate = False + + @pytest.mark.unit + def test_tarred_bpe_dataset(self, test_data_dir): + manifest_path = os.path.abspath(os.path.join(test_data_dir, 'asr/tarred_an4/tarred_audio_manifest.json')) + + tokenizer_path = os.path.join(test_data_dir, "asr", "tokenizers", "an4_wpe_128", 'vocab.txt') + tokenizer = tokenizers.AutoTokenizer(pretrained_model_name='bert-base-cased', vocab_file=tokenizer_path) + + # Test braceexpand loading + tarpath = os.path.abspath(os.path.join(test_data_dir, 'asr/tarred_an4/audio_{0..1}.tar')) + ds_braceexpand = TarredAudioToBPEDataset( + audio_tar_filepaths=tarpath, manifest_filepath=manifest_path, tokenizer=tokenizer, sample_rate=16000 + ) + assert len(ds_braceexpand) == 32 + count = 0 + for _ in ds_braceexpand: + count += 1 + assert count == 32 + + # Test loading via list + tarpath = [os.path.abspath(os.path.join(test_data_dir, f'asr/tarred_an4/audio_{i}.tar')) for i in range(2)] + ds_list_load = TarredAudioToBPEDataset( + audio_tar_filepaths=tarpath, manifest_filepath=manifest_path, tokenizer=tokenizer, sample_rate=16000 + ) + count = 0 + for _ in ds_list_load: + count += 1 + assert count == 32 diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_asr_exportables.py b/models/tacotron2/NeMo/tests/collections/asr/test_asr_exportables.py new file mode 100644 index 0000000000..0ebd23345a --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_asr_exportables.py @@ -0,0 +1,301 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import tempfile + +import onnx +import pytest +from omegaconf import DictConfig, ListConfig + +from nemo.collections.asr.models import EncDecClassificationModel, EncDecCTCModel, EncDecSpeakerLabelModel +from nemo.collections.asr.modules import ConvASRDecoder, ConvASREncoder + + +class TestExportable: + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_EncDecCTCModel_export_to_onnx(self): + model_config = DictConfig( + { + 'preprocessor': DictConfig(self.preprocessor), + 'encoder': DictConfig(self.encoder_dict), + 'decoder': DictConfig(self.decoder_dict), + } + ) + model = EncDecCTCModel(cfg=model_config) + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, 'qn.onnx') + model.export(output=filename) + onnx_model = onnx.load(filename) + onnx.checker.check_model(onnx_model, full_check=True) # throws when failed + assert onnx_model.graph.input[0].name == 'audio_signal' + assert onnx_model.graph.output[0].name == 'logprobs' + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_EncDecClassificationModel_export_to_onnx(self, speech_classification_model): + model = speech_classification_model.train() + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, 'edc.onnx') + model.export(output=filename) + onnx_model = onnx.load(filename) + onnx.checker.check_model(onnx_model, full_check=True) # throws when failed + assert onnx_model.graph.input[0].name == 'audio_signal' + assert onnx_model.graph.output[0].name == 'logits' + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_EncDecSpeakerLabelModel_export_to_onnx(self, speaker_label_model): + model = speaker_label_model.train() + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, 'sl.onnx') + model.export(output=filename) + onnx_model = onnx.load(filename) + onnx.checker.check_model(onnx_model, full_check=True) # throws when failed + assert onnx_model.graph.input[0].name == 'audio_signal' + assert onnx_model.graph.output[0].name == 'logits' + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_EncDecCitrinetModel_export_to_onnx(self, citrinet_model): + model = citrinet_model.train() + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join('.', 'citri.onnx') + model.export(output=filename) + onnx_model = onnx.load(filename) + onnx.checker.check_model(onnx_model, full_check=True) # throws when failed + assert onnx_model.graph.input[0].name == 'audio_signal' + assert onnx_model.graph.output[0].name == 'logprobs' + + def setup_method(self): + self.preprocessor = { + 'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', + 'params': dict({}), + } + + self.encoder_dict = { + 'cls': 'nemo.collections.asr.modules.ConvASREncoder', + 'params': { + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 1024, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], + }, + } + + self.decoder_dict = { + 'cls': 'nemo.collections.asr.modules.ConvASRDecoder', + 'params': { + 'feat_in': 1024, + 'num_classes': 28, + 'vocabulary': [ + ' ', + 'a', + 'b', + 'c', + 'd', + 'e', + 'f', + 'g', + 'h', + 'i', + 'j', + 'k', + 'l', + 'm', + 'n', + 'o', + 'p', + 'q', + 'r', + 's', + 't', + 'u', + 'v', + 'w', + 'x', + 'y', + 'z', + "'", + ], + }, + } + + +@pytest.fixture() +def speech_classification_model(): + preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} + encoder = { + 'cls': 'nemo.collections.asr.modules.ConvASREncoder', + 'params': { + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 32, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], + }, + } + + decoder = { + 'cls': 'nemo.collections.asr.modules.ConvASRDecoderClassification', + 'params': {'feat_in': 32, 'num_classes': 30,}, + } + + modelConfig = DictConfig( + { + 'preprocessor': DictConfig(preprocessor), + 'encoder': DictConfig(encoder), + 'decoder': DictConfig(decoder), + 'labels': ListConfig(["dummy_cls_{}".format(i + 1) for i in range(30)]), + } + ) + model = EncDecClassificationModel(cfg=modelConfig) + return model + + +@pytest.fixture() +def speaker_label_model(): + preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} + encoder = { + 'cls': 'nemo.collections.asr.modules.ConvASREncoder', + 'params': { + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 512, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': False, + } + ], + }, + } + + decoder = { + 'cls': 'nemo.collections.asr.modules.SpeakerDecoder', + 'params': {'feat_in': 512, 'num_classes': 2, 'pool_mode': 'xvector', 'emb_sizes': [1024]}, + } + + modelConfig = DictConfig( + {'preprocessor': DictConfig(preprocessor), 'encoder': DictConfig(encoder), 'decoder': DictConfig(decoder)} + ) + speaker_model = EncDecSpeakerLabelModel(cfg=modelConfig) + return speaker_model + + +@pytest.fixture() +def citrinet_model(): + preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} + encoder = { + 'cls': 'nemo.collections.asr.modules.ConvASREncoder', + 'params': { + 'feat_in': 80, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 512, + 'repeat': 1, + 'kernel': [5], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + }, + { + 'filters': 512, + 'repeat': 5, + 'kernel': [11], + 'stride': [2], + 'dilation': [1], + 'dropout': 0.1, + 'residual': True, + 'separable': True, + 'se': True, + 'se_context_size': -1, + 'stride_last': True, + 'residual_mode': 'stride_add', + }, + { + 'filters': 512, + 'repeat': 5, + 'kernel': [13], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.1, + 'residual': True, + 'separable': True, + 'se': True, + 'se_context_size': -1, + }, + { + 'filters': 640, + 'repeat': 1, + 'kernel': [41], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': True, + 'separable': True, + 'se': True, + 'se_context_size': -1, + }, + ], + }, + } + + decoder = { + 'cls': 'nemo.collections.asr.modules.ConvASRDecoder', + 'params': {'feat_in': 640, 'num_classes': 1024, 'vocabulary': list(chr(i % 28) for i in range(0, 1024))}, + } + + modelConfig = DictConfig( + {'preprocessor': DictConfig(preprocessor), 'encoder': DictConfig(encoder), 'decoder': DictConfig(decoder)} + ) + citri_model = EncDecSpeakerLabelModel(cfg=modelConfig) + return citri_model diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_asr_filterbankfeatures_seq_len.py b/models/tacotron2/NeMo/tests/collections/asr/test_asr_filterbankfeatures_seq_len.py new file mode 100644 index 0000000000..24ba584985 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_asr_filterbankfeatures_seq_len.py @@ -0,0 +1,99 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import librosa +import numpy as np +import pytest +import torch + +from nemo.collections.asr.parts.preprocessing.features import FilterbankFeatures + + +class TestFilterbankFeatures: + @pytest.mark.unit + def test_seq_len(self): + fb_module = FilterbankFeatures(exact_pad=False, pad_to=1) + test_1 = torch.randn(1, 800) + test_1_len = torch.tensor([800]) + fb_spec, fb_len = fb_module(test_1, test_1_len) + assert fb_spec.shape[2] == fb_len[0], f"{fb_spec.shape} != {fb_len}" + librosa_spec = librosa.stft(test_1.cpu().detach().numpy().squeeze(), n_fft=512, hop_length=160, win_length=320) + + assert librosa_spec.shape[1] == fb_spec.shape[2], f"{librosa_spec.shape} != {fb_spec.shape}" + + @pytest.mark.unit + def test_random_stft_sizes(self): + for _ in range(5): + nfft = 2 ** np.random.randint(7, 12) + window_size = np.random.randint(100, nfft) + hop_size = np.random.randint(64, window_size) + fb_module = FilterbankFeatures( + exact_pad=False, + pad_to=1, + n_fft=nfft, + n_window_size=window_size, + n_window_stride=hop_size, + normalize=False, + ) + audio_length = np.random.randint(nfft, 2 ** 16) + test_1 = torch.randn(1, audio_length) + test_1_len = torch.tensor([audio_length]) + fb_spec, fb_len = fb_module(test_1, test_1_len) + assert ( + fb_spec.shape[2] == fb_len[0] + ), f"{fb_spec.shape} != {fb_len}: {nfft}, {window_size}, {hop_size}, {audio_length}" + + librosa_spec = librosa.stft( + test_1.cpu().detach().numpy().squeeze(), n_fft=nfft, hop_length=hop_size, win_length=window_size + ) + + assert ( + librosa_spec.shape[1] == fb_spec.shape[2] + ), f"{librosa_spec.shape} != {fb_spec.shape}: {nfft}, {window_size}, {hop_size}, {audio_length}" + + @pytest.mark.unit + def test_random_stft_sizes_exact_pad(self): + for _ in range(5): + nfft = 2 ** np.random.randint(7, 12) + window_size = np.random.randint(100, nfft) + hop_size = np.random.randint(64, window_size) + if hop_size % 2 == 1: + hop_size = hop_size - 1 + fb_module = FilterbankFeatures( + exact_pad=True, + pad_to=1, + n_fft=nfft, + n_window_size=window_size, + n_window_stride=hop_size, + normalize=False, + ) + audio_length = np.random.randint(nfft, 2 ** 16) + test_1 = torch.randn(1, audio_length) + test_1_len = torch.tensor([audio_length]) + fb_spec, fb_len = fb_module(test_1, test_1_len) + assert ( + fb_spec.shape[2] == fb_len[0] + ), f"{fb_spec.shape} != {fb_len}: {nfft}, {window_size}, {hop_size}, {audio_length}" + + test_2 = test_1.cpu().detach().numpy().squeeze() + test_2 = np.pad(test_2, int((nfft - hop_size) // 2), mode="reflect") + librosa_spec = librosa.stft(test_2, n_fft=nfft, hop_length=hop_size, win_length=window_size, center=False,) + + assert ( + fb_spec.shape[2] == librosa_spec.shape[1] + ), f"{fb_spec.shape} != {librosa_spec.shape}: {nfft}, {window_size}, {hop_size}, {audio_length}" + + assert ( + fb_spec.shape[2] == audio_length // hop_size + ), f"{fb_spec.shape}, {nfft}, {window_size}, {hop_size}, {audio_length}, {audio_length // hop_size}" diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_asr_metrics.py b/models/tacotron2/NeMo/tests/collections/asr/test_asr_metrics.py new file mode 100644 index 0000000000..7f78c1d720 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_asr_metrics.py @@ -0,0 +1,172 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import random + +import pytest +import torch + +from nemo.collections.asr.metrics.wer import WER, word_error_rate +from nemo.collections.asr.parts.utils.rnnt_utils import Hypothesis + + +class TestWordErrorRate: + + vocabulary = [ + " ", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "'", + ] + + def __string_to_ctc_tensor(self, txt: str) -> torch.Tensor: + # This function emulates how CTC output could like for txt + blank_id = len(self.vocabulary) + char_to_ind = dict([(self.vocabulary[i], i) for i in range(len(self.vocabulary))]) + string_in_id_form = [char_to_ind[c] for c in txt] + ctc_list = [] + prev_id = -1 + for c in string_in_id_form: + # when character is repeated we need to insert CTC blank symbol + if c != prev_id: + ctc_list.append(c) + else: + ctc_list.append(blank_id) + ctc_list.append(c) + prev_id = c + return torch.Tensor(ctc_list).unsqueeze(0) + + def __reference_string_to_tensor(self, txt: str) -> torch.Tensor: + # Reference tensors aren't produced by CTC logic + char_to_ind = dict([(self.vocabulary[i], i) for i in range(len(self.vocabulary))]) + string_in_id_form = [char_to_ind[c] for c in txt] + return torch.Tensor(string_in_id_form).unsqueeze(0) + + def get_wer(self, wer, prediction: str, reference: str): + wer( + predictions=self.__string_to_ctc_tensor(prediction), + targets=self.__reference_string_to_tensor(reference), + target_lengths=torch.tensor([len(reference)]), + ) + res, _, _ = wer.compute() + res = res.detach().cpu() + # return res[0] / res[1] + return res.item() + + @pytest.mark.unit + def test_wer_function(self): + assert word_error_rate(hypotheses=['cat'], references=['cot']) == 1.0 + assert word_error_rate(hypotheses=['GPU'], references=['G P U']) == 1.0 + assert word_error_rate(hypotheses=['G P U'], references=['GPU']) == 3.0 + assert word_error_rate(hypotheses=['ducati motorcycle'], references=['motorcycle']) == 1.0 + assert word_error_rate(hypotheses=['ducati motorcycle'], references=['ducuti motorcycle']) == 0.5 + assert word_error_rate(hypotheses=['a B c'], references=['a b c']) == 1.0 / 3.0 + + @pytest.mark.unit + def test_wer_metric_simple(self): + wer = WER(vocabulary=self.vocabulary, batch_dim_index=0, use_cer=False, ctc_decode=True) + + assert self.get_wer(wer, 'cat', 'cot') == 1.0 + assert self.get_wer(wer, 'gpu', 'g p u') == 1.0 + assert self.get_wer(wer, 'g p u', 'gpu') == 3.0 + assert self.get_wer(wer, 'ducati motorcycle', 'motorcycle') == 1.0 + assert self.get_wer(wer, 'ducati motorcycle', 'ducuti motorcycle') == 0.5 + assert abs(self.get_wer(wer, 'a f c', 'a b c') - 1.0 / 3.0) < 1e-6 + + @pytest.mark.unit + def test_wer_metric_randomized(self): + """This test relies on correctness of word_error_rate function.""" + + def __randomString(N): + return ''.join(random.choice(''.join(self.vocabulary)) for i in range(N)) + + wer = WER(vocabulary=self.vocabulary, batch_dim_index=0, use_cer=False, ctc_decode=True) + + for test_id in range(256): + n1 = random.randint(1, 512) + n2 = random.randint(1, 512) + s1 = __randomString(n1) + s2 = __randomString(n2) + # skip empty strings as reference + if s2.strip(): + assert ( + abs( + self.get_wer(wer, prediction=s1, reference=s2) + - word_error_rate(hypotheses=[s1], references=[s2]) + ) + < 1e-6 + ) + + @pytest.mark.unit + def test_wer_metric_decode(self): + wer = WER(vocabulary=self.vocabulary, batch_dim_index=0, use_cer=False, ctc_decode=True) + + tokens = self.__string_to_ctc_tensor('cat')[0].int().numpy().tolist() + assert tokens == [3, 1, 20] + + tokens_decoded = wer.decode_ids_to_tokens(tokens) + assert tokens_decoded == ['c', 'a', 't'] + + str_decoded = wer.decode_tokens_to_str(tokens) + assert str_decoded == 'cat' + + @pytest.mark.unit + def test_wer_metric_return_hypothesis(self): + wer = WER(vocabulary=self.vocabulary, batch_dim_index=0, use_cer=False, ctc_decode=True) + + tensor = self.__string_to_ctc_tensor('cat').int() + + # pass batchsize 1 tensor, get back list of length 1 Hypothesis + hyp = wer.ctc_decoder_predictions_tensor(tensor, return_hypotheses=True) + hyp = hyp[0] + assert isinstance(hyp, Hypothesis) + + assert hyp.y_sequence is None + assert hyp.score == -1.0 + assert hyp.text == 'cat' + assert hyp.alignments == [3, 1, 20] + assert hyp.length == 0 + + length = torch.tensor([tensor.shape[-1]], dtype=torch.long) + + # pass batchsize 1 tensor, get back list of length 1 Hypothesis [add length info] + hyp = wer.ctc_decoder_predictions_tensor(tensor, predictions_len=length, return_hypotheses=True) + hyp = hyp[0] + assert isinstance(hyp, Hypothesis) + assert hyp.length == 3 diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_asr_modules.py b/models/tacotron2/NeMo/tests/collections/asr/test_asr_modules.py new file mode 100644 index 0000000000..b4dd12b3c7 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_asr_modules.py @@ -0,0 +1,325 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import torch +from omegaconf import OmegaConf + +from nemo.collections.asr import modules +from nemo.collections.asr.parts.utils.rnnt_utils import Hypothesis +from nemo.core.utils import numba_utils +from nemo.core.utils.numba_utils import __NUMBA_MINIMUM_VERSION__ +from nemo.utils import config_utils, logging + + +class TestASRModulesBasicTests: + @pytest.mark.unit + def test_AudioToMelSpectrogramPreprocessor1(self): + # Test 1 that should test the pure stft implementation as much as possible + instance1 = modules.AudioToMelSpectrogramPreprocessor( + dither=0, stft_conv=False, mag_power=1.0, normalize=False, preemph=0.0, log=False, pad_to=0 + ) + instance2 = modules.AudioToMelSpectrogramPreprocessor( + dither=0, stft_conv=True, mag_power=1.0, normalize=False, preemph=0.0, log=False, pad_to=0 + ) + + # Ensure that the two functions behave similarily + for _ in range(10): + input_signal = torch.randn(size=(4, 512)) + length = torch.randint(low=161, high=500, size=[4]) + res1, length1 = instance1(input_signal=input_signal, length=length) + res2, length2 = instance2(input_signal=input_signal, length=length) + for len1, len2 in zip(length1, length2): + assert len1 == len2 + assert res1.shape == res2.shape + diff = torch.mean(torch.abs(res1 - res2)) + assert diff <= 1e-3 + diff = torch.max(torch.abs(res1 - res2)) + assert diff <= 1e-2 + + @pytest.mark.unit + def test_AudioToMelSpectrogramPreprocessor_config(self): + # Test that dataclass matches signature of module + result = config_utils.assert_dataclass_signature_match( + modules.AudioToMelSpectrogramPreprocessor, + modules.audio_preprocessing.AudioToMelSpectrogramPreprocessorConfig, + ) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None + + @pytest.mark.unit + def test_AudioToMelSpectrogramPreprocessor_batch(self): + # Test 1 that should test the pure stft implementation as much as possible + instance1 = modules.AudioToMelSpectrogramPreprocessor(normalize="per_feature", dither=0, pad_to=0) + + # Ensure that the two functions behave similarily + for _ in range(10): + input_signal = torch.randn(size=(4, 512)) + length = torch.randint(low=161, high=500, size=[4]) + + with torch.no_grad(): + # batch size 1 + res_instance, length_instance = [], [] + for i in range(input_signal.size(0)): + res_ins, length_ins = instance1(input_signal=input_signal[i : i + 1], length=length[i : i + 1]) + res_instance.append(res_ins) + length_instance.append(length_ins) + + res_instance = torch.cat(res_instance, 0) + length_instance = torch.cat(length_instance, 0) + + # batch size 4 + res_batch, length_batch = instance1(input_signal=input_signal, length=length) + + assert res_instance.shape == res_batch.shape + assert length_instance.shape == length_batch.shape + diff = torch.mean(torch.abs(res_instance - res_batch)) + assert diff <= 1e-3 + diff = torch.max(torch.abs(res_instance - res_batch)) + assert diff <= 1e-3 + + @pytest.mark.unit + def test_AudioToMelSpectrogramPreprocessor2(self): + # Test 2 that should test the stft implementation as used in ASR models + instance1 = modules.AudioToMelSpectrogramPreprocessor(dither=0, stft_conv=False) + instance2 = modules.AudioToMelSpectrogramPreprocessor(dither=0, stft_conv=True) + + # Ensure that the two functions behave similarily + for _ in range(5): + input_signal = torch.randn(size=(4, 512)) + length = torch.randint(low=161, high=500, size=[4]) + res1, length1 = instance1(input_signal=input_signal, length=length) + res2, length2 = instance2(input_signal=input_signal, length=length) + for len1, len2 in zip(length1, length2): + assert len1 == len2 + assert res1.shape == res2.shape + diff = torch.mean(torch.abs(res1 - res2)) + assert diff <= 3e-3 + diff = torch.max(torch.abs(res1 - res2)) + assert diff <= 3 + + @pytest.mark.unit + def test_SpectrogramAugmentationr(self): + # Make sure constructor works + instance1 = modules.SpectrogramAugmentation( + freq_masks=10, time_masks=3, rect_masks=3, use_numba_spec_augment=False + ) + assert isinstance(instance1, modules.SpectrogramAugmentation) + + # Make sure forward doesn't throw with expected input + instance0 = modules.AudioToMelSpectrogramPreprocessor(dither=0) + input_signal = torch.randn(size=(4, 512)) + length = torch.randint(low=161, high=500, size=[4]) + res0 = instance0(input_signal=input_signal, length=length) + res = instance1(input_spec=res0[0]) + + assert res.shape == res0[0].shape + + @pytest.mark.unit + @pytest.mark.run_only_on('GPU') + def test_SpectrogramAugmentationr_numba_kernel(self, caplog): + numba_utils.skip_numba_cuda_test_if_unsupported(__NUMBA_MINIMUM_VERSION__) + + logging._logger.propagate = True + original_verbosity = logging.get_verbosity() + logging.set_verbosity(logging.DEBUG) + caplog.set_level(logging.DEBUG) + + # Make sure constructor works + instance1 = modules.SpectrogramAugmentation( + freq_masks=10, time_masks=3, rect_masks=3, use_numba_spec_augment=True + ) + assert isinstance(instance1, modules.SpectrogramAugmentation) + + # Make sure forward doesn't throw with expected input + instance0 = modules.AudioToMelSpectrogramPreprocessor(dither=0) + input_signal = torch.randn(size=(8, 512)) + length = torch.randint(low=161, high=500, size=[8]) + res0 = instance0(input_signal=input_signal, length=length) + res = instance1(input_spec=res0[0], length=length) + + assert res.shape == res0[0].shape + + # check tha numba kernel debug message indicates that it is available for use + assert """Numba SpecAugment kernel is available""" in caplog.text + + logging._logger.propagate = False + logging.set_verbosity(original_verbosity) + + @pytest.mark.unit + def test_SpectrogramAugmentationr_config(self): + # Test that dataclass matches signature of module + result = config_utils.assert_dataclass_signature_match( + modules.SpectrogramAugmentation, modules.audio_preprocessing.SpectrogramAugmentationConfig, + ) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None + + @pytest.mark.unit + def test_CropOrPadSpectrogramAugmentation(self): + # Make sure constructor works + audio_length = 128 + instance1 = modules.CropOrPadSpectrogramAugmentation(audio_length=audio_length) + assert isinstance(instance1, modules.CropOrPadSpectrogramAugmentation) + + # Make sure forward doesn't throw with expected input + instance0 = modules.AudioToMelSpectrogramPreprocessor(dither=0) + input_signal = torch.randn(size=(4, 512)) + length = torch.randint(low=161, high=500, size=[4]) + res0 = instance0(input_signal=input_signal, length=length) + res, new_length = instance1(input_signal=res0[0], length=length) + + assert res.shape == torch.Size([4, 64, audio_length]) + assert all(new_length == torch.tensor([128] * 4)) + + @pytest.mark.unit + def test_CropOrPadSpectrogramAugmentation_config(self): + # Test that dataclass matches signature of module + result = config_utils.assert_dataclass_signature_match( + modules.CropOrPadSpectrogramAugmentation, + modules.audio_preprocessing.CropOrPadSpectrogramAugmentationConfig, + ) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None + + @pytest.mark.unit + def test_RNNTDecoder(self): + vocab = list(range(10)) + vocab = [str(x) for x in vocab] + vocab_size = len(vocab) + + pred_config = OmegaConf.create( + { + '_target_': 'nemo.collections.asr.modules.RNNTDecoder', + 'prednet': {'pred_hidden': 32, 'pred_rnn_layers': 1,}, + 'vocab_size': vocab_size, + 'blank_as_pad': True, + } + ) + + prednet = modules.RNNTDecoder.from_config_dict(pred_config) + + # num params + pred_hidden = pred_config.prednet.pred_hidden + embed = (vocab_size + 1) * pred_hidden # embedding with blank + rnn = ( + 2 * 4 * (pred_hidden * pred_hidden + pred_hidden) + ) # (ih + hh) * (ifco gates) * (indim * hiddendim + bias) + assert prednet.num_weights == (embed + rnn) + + # State initialization + x_ = torch.zeros(4, dtype=torch.float32) + states = prednet.initialize_state(x_) + + for state_i in states: + assert state_i.dtype == x_.dtype + assert state_i.device == x_.device + assert state_i.shape[1] == len(x_) + + # Blank hypotheses test + blank = vocab_size + hyp = Hypothesis(score=0.0, y_sequence=[blank]) + cache = {} + pred, states, _ = prednet.score_hypothesis(hyp, cache) + + assert pred.shape == torch.Size([1, 1, pred_hidden]) + assert len(states) == 2 + for state_i in states: + assert state_i.dtype == pred.dtype + assert state_i.device == pred.device + assert state_i.shape[1] == len(pred) + + # Blank stateless predict + g, states = prednet.predict(y=None, state=None, add_sos=False, batch_size=1) + + assert g.shape == torch.Size([1, 1, pred_hidden]) + assert len(states) == 2 + for state_i in states: + assert state_i.dtype == g.dtype + assert state_i.device == g.device + assert state_i.shape[1] == len(g) + + # Blank stateful predict + g, states2 = prednet.predict(y=None, state=states, add_sos=False, batch_size=1) + + assert g.shape == torch.Size([1, 1, pred_hidden]) + assert len(states2) == 2 + for state_i, state_j in zip(states, states2): + assert (state_i - state_j).square().sum().sqrt() > 0.0 + + # Predict with token and state + token = torch.full([1, 1], fill_value=0, dtype=torch.long) + g, states = prednet.predict(y=token, state=states2, add_sos=False, batch_size=None) + + assert g.shape == torch.Size([1, 1, pred_hidden]) + assert len(states) == 2 + + # Predict with blank token and no state + token = torch.full([1, 1], fill_value=blank, dtype=torch.long) + g, states = prednet.predict(y=token, state=None, add_sos=False, batch_size=None) + + assert g.shape == torch.Size([1, 1, pred_hidden]) + assert len(states) == 2 + + @pytest.mark.unit + def test_RNNTJoint(self): + vocab = list(range(10)) + vocab = [str(x) for x in vocab] + vocab_size = len(vocab) + + batchsize = 4 + encoder_hidden = 64 + pred_hidden = 32 + joint_hidden = 16 + + joint_cfg = OmegaConf.create( + { + '_target_': 'nemo.collections.asr.modules.RNNTJoint', + 'num_classes': vocab_size, + 'vocabulary': vocab, + 'jointnet': { + 'encoder_hidden': encoder_hidden, + 'pred_hidden': pred_hidden, + 'joint_hidden': joint_hidden, + 'activation': 'relu', + }, + } + ) + + jointnet = modules.RNNTJoint.from_config_dict(joint_cfg) + + enc = torch.zeros(batchsize, encoder_hidden, 48) # [B, D1, T] + dec = torch.zeros(batchsize, pred_hidden, 24) # [B, D2, U] + + # forward call test + out = jointnet(encoder_outputs=enc, decoder_outputs=dec) + assert out.shape == torch.Size([batchsize, 48, 24, vocab_size + 1]) # [B, T, U, V + 1] + + # joint() step test + enc2 = enc.transpose(1, 2) # [B, T, D1] + dec2 = dec.transpose(1, 2) # [B, U, D2] + out2 = jointnet.joint(enc2, dec2) # [B, T, U, V + 1] + assert (out - out2).abs().sum() <= 1e-5 + + # assert vocab size + assert jointnet.num_classes_with_blank == vocab_size + 1 diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_asr_regression_model.py b/models/tacotron2/NeMo/tests/collections/asr/test_asr_regression_model.py new file mode 100644 index 0000000000..eacce0f543 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_asr_regression_model.py @@ -0,0 +1,95 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest +from omegaconf import DictConfig + +from nemo.collections.asr.models.classification_models import EncDecRegressionModel + + +@pytest.fixture() +def speech_regression_model(): + preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} + encoder = { + 'cls': 'nemo.collections.asr.modules.ConvASREncoder', + 'params': { + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 32, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], + }, + } + + decoder = { + 'cls': 'nemo.collections.asr.modules.conv_asr.ConvASRDecoderClassification', + 'params': {'feat_in': 32, 'return_logits': True, 'num_classes': 1}, + } + + modelConfig = DictConfig( + { + 'preprocessor': DictConfig(preprocessor), + 'encoder': DictConfig(encoder), + 'decoder': DictConfig(decoder), + 'labels': None, + 'is_regression_task': True, + } + ) + model = EncDecRegressionModel(cfg=modelConfig) + return model + + +class TestEncDecRegressionModel: + @pytest.mark.unit + def test_constructor(self, speech_regression_model): + asr_model = speech_regression_model.train() + + conv_cnt = (64 * 32 * 1 + 32) + (64 * 1 * 1 + 32) # separable kernel + bias + pointwise kernel + bias + bn_cnt = (4 * 32) * 2 # 2 * moving averages + dec_cnt = 32 * 1 + 1 # fc + bias + + param_count = conv_cnt + bn_cnt + dec_cnt + assert asr_model.num_weights == param_count + + # Check to/from config_dict: + confdict = asr_model.to_config_dict() + instance2 = EncDecRegressionModel.from_config_dict(confdict) + + assert isinstance(instance2, EncDecRegressionModel) + + @pytest.mark.unit + def test_transcription(self, speech_regression_model, test_data_dir): + + audio_filenames = ['an22-flrp-b.wav', 'an90-fbbh-b.wav'] + audio_paths = [os.path.join(test_data_dir, "asr", "train", "an4", "wav", fp) for fp in audio_filenames] + + model = speech_regression_model.eval() + + # Test Top 1 classification transcription + results = model.transcribe(audio_paths, batch_size=2) + assert len(results) == 2 diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_asr_rnnt_encdec_model.py b/models/tacotron2/NeMo/tests/collections/asr/test_asr_rnnt_encdec_model.py new file mode 100644 index 0000000000..15124157a6 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_asr_rnnt_encdec_model.py @@ -0,0 +1,276 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy + +import pytest +import torch +from omegaconf import DictConfig, ListConfig + +from nemo.collections.asr.models import EncDecRNNTModel +from nemo.collections.asr.parts.submodules import rnnt_beam_decoding as beam_decode +from nemo.collections.asr.parts.submodules import rnnt_greedy_decoding as greedy_decode +from nemo.core.utils import numba_utils +from nemo.core.utils.numba_utils import __NUMBA_MINIMUM_VERSION__ +from nemo.utils.config_utils import assert_dataclass_signature_match + +NUMBA_RNNT_LOSS_AVAILABLE = numba_utils.numba_cuda_is_supported(__NUMBA_MINIMUM_VERSION__) + + +@pytest.fixture() +def asr_model(): + preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} + + # fmt: off + labels = [' ', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', + 'x', 'y', 'z', "'", + ] + # fmt: on + + model_defaults = {'enc_hidden': 1024, 'pred_hidden': 64} + + encoder = { + 'cls': 'nemo.collections.asr.modules.ConvASREncoder', + 'params': { + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': model_defaults['enc_hidden'], + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], + }, + } + + decoder = { + '_target_': 'nemo.collections.asr.modules.RNNTDecoder', + 'prednet': {'pred_hidden': model_defaults['pred_hidden'], 'pred_rnn_layers': 1}, + } + + joint = { + '_target_': 'nemo.collections.asr.modules.RNNTJoint', + 'jointnet': {'joint_hidden': 32, 'activation': 'relu'}, + } + + decoding = {'strategy': 'greedy_batch', 'greedy': {'max_symbols': 30}} + + modelConfig = DictConfig( + { + 'labels': ListConfig(labels), + 'preprocessor': DictConfig(preprocessor), + 'model_defaults': DictConfig(model_defaults), + 'encoder': DictConfig(encoder), + 'decoder': DictConfig(decoder), + 'joint': DictConfig(joint), + 'decoding': DictConfig(decoding), + } + ) + + model_instance = EncDecRNNTModel(cfg=modelConfig) + return model_instance + + +class TestEncDecRNNTModel: + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_constructor(self, asr_model): + asr_model.train() + # TODO: make proper config and assert correct number of weights + # Check to/from config_dict: + confdict = asr_model.to_config_dict() + instance2 = EncDecRNNTModel.from_config_dict(confdict) + assert isinstance(instance2, EncDecRNNTModel) + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_forward(self, asr_model): + asr_model = asr_model.eval() + + asr_model.preprocessor.featurizer.dither = 0.0 + asr_model.preprocessor.featurizer.pad_to = 0 + + asr_model.compute_eval_loss = False + + input_signal = torch.randn(size=(4, 512)) + length = torch.randint(low=161, high=500, size=[4]) + + with torch.no_grad(): + # batch size 1 + logprobs_instance = [] + for i in range(input_signal.size(0)): + logprobs_ins, _ = asr_model.forward( + input_signal=input_signal[i : i + 1], input_signal_length=length[i : i + 1] + ) + logprobs_instance.append(logprobs_ins) + logprobs_instance = torch.cat(logprobs_instance, 0) + + # batch size 4 + logprobs_batch, _ = asr_model.forward(input_signal=input_signal, input_signal_length=length) + + assert logprobs_instance.shape == logprobs_batch.shape + diff = torch.mean(torch.abs(logprobs_instance - logprobs_batch)) + assert diff <= 1e-6 + diff = torch.max(torch.abs(logprobs_instance - logprobs_batch)) + assert diff <= 1e-6 + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_vocab_change(self, asr_model): + old_vocab = copy.deepcopy(asr_model.joint.vocabulary) + nw1 = asr_model.num_weights + asr_model.change_vocabulary(new_vocabulary=old_vocab) + # No change + assert nw1 == asr_model.num_weights + new_vocab = copy.deepcopy(old_vocab) + new_vocab.append('!') + new_vocab.append('$') + new_vocab.append('@') + asr_model.change_vocabulary(new_vocabulary=new_vocab) + # fully connected + bias + # rnn embedding + joint + bias + pred_embedding = 3 * (asr_model.decoder.pred_hidden) + joint_joint = 3 * (asr_model.joint.joint_hidden + 1) + assert asr_model.num_weights == (nw1 + (pred_embedding + joint_joint)) + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_change_conv_asr_se_context_window(self, asr_model): + old_cfg = copy.deepcopy(asr_model.cfg) + asr_model.change_conv_asr_se_context_window(context_window=32) # 32 * 0.01s context + new_config = asr_model.cfg + + assert old_cfg.encoder.jasper[0].se_context_size == -1 + assert new_config.encoder.jasper[0].se_context_size == 32 + + for name, m in asr_model.encoder.named_modules(): + if type(m).__class__.__name__ == 'SqueezeExcite': + assert m.context_window == 32 + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_change_conv_asr_se_context_window_no_config_update(self, asr_model): + old_cfg = copy.deepcopy(asr_model.cfg) + asr_model.change_conv_asr_se_context_window(context_window=32, update_config=False) # 32 * 0.01s context + new_config = asr_model.cfg + + assert old_cfg.encoder.jasper[0].se_context_size == -1 + assert new_config.encoder.jasper[0].se_context_size == -1 # no change + + for name, m in asr_model.encoder.named_modules(): + if type(m).__class__.__name__ == 'SqueezeExcite': + assert m.context_window == 32 + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_decoding_change(self, asr_model): + assert isinstance(asr_model.decoding.decoding, greedy_decode.GreedyBatchedRNNTInfer) + + new_strategy = DictConfig({}) + new_strategy.strategy = 'greedy' + new_strategy.greedy = DictConfig({'max_symbols': 10}) + asr_model.change_decoding_strategy(decoding_cfg=new_strategy) + assert isinstance(asr_model.decoding.decoding, greedy_decode.GreedyRNNTInfer) + + new_strategy = DictConfig({}) + new_strategy.strategy = 'beam' + new_strategy.beam = DictConfig({'beam_size': 1}) + asr_model.change_decoding_strategy(decoding_cfg=new_strategy) + assert isinstance(asr_model.decoding.decoding, beam_decode.BeamRNNTInfer) + assert asr_model.decoding.decoding.search_type == "default" + + new_strategy = DictConfig({}) + new_strategy.strategy = 'beam' + new_strategy.beam = DictConfig({'beam_size': 2}) + asr_model.change_decoding_strategy(decoding_cfg=new_strategy) + assert isinstance(asr_model.decoding.decoding, beam_decode.BeamRNNTInfer) + assert asr_model.decoding.decoding.search_type == "default" + + new_strategy = DictConfig({}) + new_strategy.strategy = 'tsd' + new_strategy.beam = DictConfig({'beam_size': 2}) + asr_model.change_decoding_strategy(decoding_cfg=new_strategy) + assert isinstance(asr_model.decoding.decoding, beam_decode.BeamRNNTInfer) + assert asr_model.decoding.decoding.search_type == "tsd" + + new_strategy = DictConfig({}) + new_strategy.strategy = 'alsd' + new_strategy.beam = DictConfig({'beam_size': 2}) + asr_model.change_decoding_strategy(decoding_cfg=new_strategy) + assert isinstance(asr_model.decoding.decoding, beam_decode.BeamRNNTInfer) + assert asr_model.decoding.decoding.search_type == "alsd" + + @pytest.mark.unit + def test_GreedyRNNTInferConfig(self): + IGNORE_ARGS = ['decoder_model', 'joint_model', 'blank_index'] + + result = assert_dataclass_signature_match( + greedy_decode.GreedyRNNTInfer, greedy_decode.GreedyRNNTInferConfig, ignore_args=IGNORE_ARGS + ) + + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None + + @pytest.mark.unit + def test_GreedyBatchedRNNTInferConfig(self): + IGNORE_ARGS = ['decoder_model', 'joint_model', 'blank_index'] + + result = assert_dataclass_signature_match( + greedy_decode.GreedyBatchedRNNTInfer, greedy_decode.GreedyBatchedRNNTInferConfig, ignore_args=IGNORE_ARGS + ) + + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None + + @pytest.mark.unit + def test_BeamRNNTInferConfig(self): + IGNORE_ARGS = ['decoder_model', 'joint_model', 'blank_index'] + + result = assert_dataclass_signature_match( + beam_decode.BeamRNNTInfer, beam_decode.BeamRNNTInferConfig, ignore_args=IGNORE_ARGS + ) + + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_asr_rnnt_encoder_model_bpe.py b/models/tacotron2/NeMo/tests/collections/asr/test_asr_rnnt_encoder_model_bpe.py new file mode 100644 index 0000000000..294b03dd7d --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_asr_rnnt_encoder_model_bpe.py @@ -0,0 +1,246 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tempfile + +import pytest +import torch +from omegaconf import DictConfig + +from nemo.collections.asr.models.rnnt_bpe_models import EncDecRNNTBPEModel +from nemo.collections.asr.parts.submodules import rnnt_beam_decoding as beam_decode +from nemo.collections.asr.parts.submodules import rnnt_greedy_decoding as greedy_decode +from nemo.collections.common import tokenizers +from nemo.core.utils import numba_utils +from nemo.core.utils.numba_utils import __NUMBA_MINIMUM_VERSION__ + +NUMBA_RNNT_LOSS_AVAILABLE = numba_utils.numba_cuda_is_supported(__NUMBA_MINIMUM_VERSION__) + + +@pytest.fixture() +def asr_model(test_data_dir): + preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} + + model_defaults = {'enc_hidden': 1024, 'pred_hidden': 64} + + encoder = { + 'cls': 'nemo.collections.asr.modules.ConvASREncoder', + 'params': { + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': model_defaults['enc_hidden'], + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], + }, + } + + decoder = { + '_target_': 'nemo.collections.asr.modules.RNNTDecoder', + 'prednet': {'pred_hidden': model_defaults['pred_hidden'], 'pred_rnn_layers': 1,}, + } + + joint = { + '_target_': 'nemo.collections.asr.modules.RNNTJoint', + 'jointnet': {'joint_hidden': 32, 'activation': 'relu',}, + } + + decoding = {'strategy': 'greedy_batch', 'greedy': {'max_symbols': 30}} + + tokenizer = {'dir': os.path.join(test_data_dir, "asr", "tokenizers", "an4_wpe_128"), 'type': 'wpe'} + + modelConfig = DictConfig( + { + 'preprocessor': DictConfig(preprocessor), + 'model_defaults': DictConfig(model_defaults), + 'encoder': DictConfig(encoder), + 'decoder': DictConfig(decoder), + 'joint': DictConfig(joint), + 'tokenizer': DictConfig(tokenizer), + 'decoding': DictConfig(decoding), + } + ) + + model_instance = EncDecRNNTBPEModel(cfg=modelConfig) + return model_instance + + +class TestEncDecRNNTBPEModel: + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_constructor(self, asr_model): + asr_model.train() + # TODO: make proper config and assert correct number of weights + # Check to/from config_dict: + confdict = asr_model.to_config_dict() + instance2 = EncDecRNNTBPEModel.from_config_dict(confdict) + assert isinstance(instance2, EncDecRNNTBPEModel) + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_forward(self, asr_model): + asr_model = asr_model.eval() + + asr_model.preprocessor.featurizer.dither = 0.0 + asr_model.preprocessor.featurizer.pad_to = 0 + + asr_model.compute_eval_loss = False + + input_signal = torch.randn(size=(4, 512)) + length = torch.randint(low=161, high=500, size=[4]) + + with torch.no_grad(): + # batch size 1 + logprobs_instance = [] + for i in range(input_signal.size(0)): + logprobs_ins, _ = asr_model.forward( + input_signal=input_signal[i : i + 1], input_signal_length=length[i : i + 1] + ) + logprobs_instance.append(logprobs_ins) + logits_instance = torch.cat(logprobs_instance, 0) + + # batch size 4 + logprobs_batch, _ = asr_model.forward(input_signal=input_signal, input_signal_length=length) + + assert logits_instance.shape == logprobs_batch.shape + diff = torch.mean(torch.abs(logits_instance - logprobs_batch)) + assert diff <= 1e-6 + diff = torch.max(torch.abs(logits_instance - logprobs_batch)) + assert diff <= 1e-6 + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_save_restore_artifact(self, asr_model): + asr_model.train() + + with tempfile.TemporaryDirectory() as tmp_dir: + path = os.path.join(tmp_dir, 'rnnt_bpe.nemo') + asr_model.save_to(path) + + new_model = EncDecRNNTBPEModel.restore_from(path) + assert isinstance(new_model, type(asr_model)) + assert new_model.vocab_path.endswith('_vocab.txt') + + assert len(new_model.tokenizer.tokenizer.get_vocab()) == 128 + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_save_restore_artifact_spe(self, asr_model, test_data_dir): + asr_model.train() + + with tempfile.TemporaryDirectory() as tmpdir: + tokenizer_dir = os.path.join(test_data_dir, "asr", "tokenizers", "an4_spe_128") + asr_model.change_vocabulary(new_tokenizer_dir=tokenizer_dir, new_tokenizer_type='bpe') + + save_path = os.path.join(tmpdir, 'ctc_bpe.nemo') + asr_model.train() + asr_model.save_to(save_path) + + new_model = EncDecRNNTBPEModel.restore_from(save_path) + assert isinstance(new_model, type(asr_model)) + assert isinstance(new_model.tokenizer, tokenizers.SentencePieceTokenizer) + assert new_model.model_path.endswith('_tokenizer.model') + assert new_model.vocab_path.endswith('_vocab.txt') + assert new_model.spe_vocab_path.endswith('_tokenizer.vocab') + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_vocab_change(self, test_data_dir, asr_model): + with tempfile.TemporaryDirectory() as tmpdir: + old_tokenizer_dir = os.path.join(test_data_dir, "asr", "tokenizers", "an4_wpe_128", 'vocab.txt') + new_tokenizer_dir = os.path.join(tmpdir, 'tokenizer') + + os.makedirs(new_tokenizer_dir, exist_ok=True) + shutil.copy2(old_tokenizer_dir, new_tokenizer_dir) + + nw1 = asr_model.num_weights + asr_model.change_vocabulary(new_tokenizer_dir=new_tokenizer_dir, new_tokenizer_type='wpe') + # No change + assert nw1 == asr_model.num_weights + + with open(os.path.join(new_tokenizer_dir, 'vocab.txt'), 'a+') as f: + f.write("!\n") + f.write('$\n') + f.write('@\n') + + asr_model.change_vocabulary(new_tokenizer_dir=new_tokenizer_dir, new_tokenizer_type='wpe') + + # rnn embedding + joint + bias + pred_embedding = 3 * (asr_model.decoder.pred_hidden) + joint_joint = 3 * (asr_model.joint.joint_hidden + 1) + assert asr_model.num_weights == (nw1 + (pred_embedding + joint_joint)) + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + def test_decoding_change(self, asr_model): + assert isinstance(asr_model.decoding.decoding, greedy_decode.GreedyBatchedRNNTInfer) + + new_strategy = DictConfig({}) + new_strategy.strategy = 'greedy' + new_strategy.greedy = DictConfig({'max_symbols': 10}) + asr_model.change_decoding_strategy(decoding_cfg=new_strategy) + assert isinstance(asr_model.decoding.decoding, greedy_decode.GreedyRNNTInfer) + + new_strategy = DictConfig({}) + new_strategy.strategy = 'beam' + new_strategy.beam = DictConfig({'beam_size': 1}) + asr_model.change_decoding_strategy(decoding_cfg=new_strategy) + assert isinstance(asr_model.decoding.decoding, beam_decode.BeamRNNTInfer) + assert asr_model.decoding.decoding.search_type == "default" + + new_strategy = DictConfig({}) + new_strategy.strategy = 'beam' + new_strategy.beam = DictConfig({'beam_size': 2}) + asr_model.change_decoding_strategy(decoding_cfg=new_strategy) + assert isinstance(asr_model.decoding.decoding, beam_decode.BeamRNNTInfer) + assert asr_model.decoding.decoding.search_type == "default" + + new_strategy = DictConfig({}) + new_strategy.strategy = 'tsd' + new_strategy.beam = DictConfig({'beam_size': 2}) + asr_model.change_decoding_strategy(decoding_cfg=new_strategy) + assert isinstance(asr_model.decoding.decoding, beam_decode.BeamRNNTInfer) + assert asr_model.decoding.decoding.search_type == "tsd" + + new_strategy = DictConfig({}) + new_strategy.strategy = 'alsd' + new_strategy.beam = DictConfig({'beam_size': 2}) + asr_model.change_decoding_strategy(decoding_cfg=new_strategy) + assert isinstance(asr_model.decoding.decoding, beam_decode.BeamRNNTInfer) + assert asr_model.decoding.decoding.search_type == "alsd" diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_jasper_block.py b/models/tacotron2/NeMo/tests/collections/asr/test_jasper_block.py new file mode 100644 index 0000000000..2b6cfb41eb --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_jasper_block.py @@ -0,0 +1,295 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import torch + +from nemo.collections.asr.parts.submodules import jasper + + +class TestJasperBlock: + def jasper_base_config(self, **kwargs): + base = dict( + inplanes=16, + planes=8, + kernel_size=[11], + repeat=1, + stride=[1], + dilation=[1], + activation="relu", + conv_mask=True, + separable=False, + se=False, + ) + base.update(kwargs) + return base + + def check_module_exists(self, module, cls): + global _MODULE_EXISTS + _MODULE_EXISTS = 0 + + def _traverse(m): + if isinstance(m, cls): + global _MODULE_EXISTS + _MODULE_EXISTS += 1 + + module.apply(_traverse) + assert _MODULE_EXISTS > 0 + + @pytest.mark.unit + def test_basic_block(self): + config = self.jasper_base_config(residual=False) + act = jasper.jasper_activations.get(config.pop('activation'))() + + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 131]) + assert ylen[0] == 131 + + @pytest.mark.unit + def test_residual_block(self): + config = self.jasper_base_config(residual=True) + act = jasper.jasper_activations.get(config.pop('activation'))() + + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 131]) + assert ylen[0] == 131 + + @pytest.mark.unit + def test_basic_block_repeat(self): + config = self.jasper_base_config(residual=False, repeat=3) + act = jasper.jasper_activations.get(config.pop('activation'))() + + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 131]) + assert ylen[0] == 131 + assert len(block.mconv) == 3 * 3 + 1 # (3 repeats x {1 conv + 1 norm + 1 dropout} + final conv) + + @pytest.mark.unit + def test_basic_block_repeat_stride(self): + config = self.jasper_base_config(residual=False, repeat=3, stride=[2]) + act = jasper.jasper_activations.get(config.pop('activation'))() + + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 17]) # 131 // (stride ^ repeats) + assert ylen[0] == 17 # 131 // (stride ^ repeats) + assert len(block.mconv) == 3 * 3 + 1 # (3 repeats x {1 conv + 1 norm + 1 dropout} + final conv) + + @pytest.mark.unit + def test_basic_block_repeat_stride_last(self): + config = self.jasper_base_config(residual=False, repeat=3, stride=[2], stride_last=True) + act = jasper.jasper_activations.get(config.pop('activation'))() + + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 66]) # 131 // stride + assert ylen[0] == 66 # 131 // stride + assert len(block.mconv) == 3 * 3 + 1 # (3 repeats x {1 conv + 1 norm + 1 dropout} + final conv) + + @pytest.mark.unit + def test_basic_block_repeat_separable(self): + config = self.jasper_base_config(residual=False, repeat=3, separable=True) + act = jasper.jasper_activations.get(config.pop('activation'))() + + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 131]) + assert ylen[0] == 131 + assert len(block.mconv) == 3 * 4 + 1 # (3 repeats x {1 dconv + 1 pconv + 1 norm + 1 dropout} + final conv) + + @pytest.mark.unit + def test_basic_block_stride(self): + config = self.jasper_base_config(stride=[2], residual=False) + act = jasper.jasper_activations.get(config.pop('activation'))() + + print(config) + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 66]) + assert ylen[0] == 66 + + @pytest.mark.unit + def test_residual_block_stride(self): + config = self.jasper_base_config(stride=[2], residual=True, residual_mode='stride_add') + act = jasper.jasper_activations.get(config.pop('activation'))() + + print(config) + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 66]) + assert ylen[0] == 66 + + @pytest.mark.unit + def test_residual_block_activations(self): + for activation in jasper.jasper_activations.keys(): + config = self.jasper_base_config(activation=activation) + act = jasper.jasper_activations.get(config.pop('activation'))() + + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + self.check_module_exists(block, act.__class__) + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 131]) + assert ylen[0] == 131 + + @pytest.mark.unit + def test_residual_block_normalizations(self): + NORMALIZATIONS = ["batch", "layer", "group"] + for normalization in NORMALIZATIONS: + config = self.jasper_base_config(normalization=normalization) + act = jasper.jasper_activations.get(config.pop('activation'))() + + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 131]) + assert ylen[0] == 131 + + @pytest.mark.unit + def test_residual_block_se(self): + config = self.jasper_base_config(se=True, se_reduction_ratio=8) + act = jasper.jasper_activations.get(config.pop('activation'))() + + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + self.check_module_exists(block, jasper.SqueezeExcite) + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 131]) + assert ylen[0] == 131 + + @pytest.mark.unit + def test_residual_block_asymmetric_pad_future_contexts(self): + # test future contexts at various values + # 0 = no future context + # 2 = limited future context + # 5 = symmetric context + # 8 = excess future context (more future context than present or past context) + future_contexts = [0, 2, 5, 8] + for future_context in future_contexts: + print(future_context) + config = self.jasper_base_config(future_context=future_context) + act = jasper.jasper_activations.get(config.pop('activation'))() + + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + self.check_module_exists(block, torch.nn.ConstantPad1d) + self.check_module_exists(block, jasper.MaskedConv1d) + + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 131]) + assert ylen[0] == 131 + assert block.mconv[0].pad_layer is not None + assert block.mconv[0]._padding == (config['kernel_size'][0] - 1 - future_context, future_context) + + @pytest.mark.unit + def test_residual_block_asymmetric_pad_future_context_fallback(self): + # test future contexts at various values + # 15 = K < FC; fall back to symmetric context + future_context = 15 + print(future_context) + config = self.jasper_base_config(future_context=future_context) + act = jasper.jasper_activations.get(config.pop('activation'))() + + block = jasper.JasperBlock(**config, activation=act) + + x = torch.randn(1, 16, 131) + xlen = torch.tensor([131]) + y, ylen = block(([x], xlen)) + + self.check_module_exists(block, jasper.MaskedConv1d) + + assert isinstance(block, jasper.JasperBlock) + assert y[0].shape == torch.Size([1, config['planes'], 131]) + assert ylen[0] == 131 + assert block.mconv[0].pad_layer is None + assert block.mconv[0]._padding == config['kernel_size'][0] // 2 + + @pytest.mark.unit + def test_padding_size_conv1d(self): + input_channels = 1 + output_channels = 1 + kernel_sizes = [3, 7, 11] + dilation_sizes = [2, 3, 4] + stride = 1 + inp = torch.rand(2, 1, 40) + + for kernel_size in kernel_sizes: + for dilation_size in dilation_sizes: + padding = jasper.get_same_padding(kernel_size, stride, dilation_size) + + conv = torch.nn.Conv1d( + input_channels, output_channels, kernel_size=kernel_size, dilation=dilation_size, padding=padding + ) + + out = conv(inp) + assert out.shape == inp.shape diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_label_datasets.py b/models/tacotron2/NeMo/tests/collections/asr/test_label_datasets.py new file mode 100644 index 0000000000..fbe9025284 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_label_datasets.py @@ -0,0 +1,106 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pytest +import torch + +from nemo.collections.asr.data.audio_to_label import TarredAudioToClassificationLabelDataset +from nemo.collections.asr.data.feature_to_label import FeatureToSeqSpeakerLabelDataset +from nemo.collections.asr.parts.preprocessing.feature_loader import ExternalFeatureLoader +from nemo.collections.asr.parts.preprocessing.features import WaveformFeaturizer + + +class TestASRDatasets: + labels = ["fash", "fbbh", "fclc"] + unique_labels_in_seq = ['0', '1', '2', '3', "zero", "one", "two", "three"] + + @pytest.mark.unit + def test_tarred_dataset(self, test_data_dir): + manifest_path = os.path.abspath(os.path.join(test_data_dir, 'asr/tarred_an4/tarred_audio_manifest.json')) + + # Test braceexpand loading + tarpath = os.path.abspath(os.path.join(test_data_dir, 'asr/tarred_an4/audio_{0..1}.tar')) + featurizer = WaveformFeaturizer(sample_rate=16000, int_values=False, augmentor=None) + ds_braceexpand = TarredAudioToClassificationLabelDataset( + audio_tar_filepaths=tarpath, manifest_filepath=manifest_path, labels=self.labels, featurizer=featurizer + ) + + assert len(ds_braceexpand) == 32 + count = 0 + for _ in ds_braceexpand: + count += 1 + assert count == 32 + + # Test loading via list + tarpath = [os.path.abspath(os.path.join(test_data_dir, f'asr/tarred_an4/audio_{i}.tar')) for i in range(2)] + ds_list_load = TarredAudioToClassificationLabelDataset( + audio_tar_filepaths=tarpath, manifest_filepath=manifest_path, labels=self.labels, featurizer=featurizer + ) + count = 0 + for _ in ds_list_load: + count += 1 + assert count == 32 + + @pytest.mark.unit + def test_tarred_dataset_duplicate_name(self, test_data_dir): + manifest_path = os.path.abspath( + os.path.join(test_data_dir, 'asr/tarred_an4/tarred_duplicate_audio_manifest.json') + ) + + # Test braceexpand loading + tarpath = os.path.abspath(os.path.join(test_data_dir, 'asr/tarred_an4/audio_{0..1}.tar')) + featurizer = WaveformFeaturizer(sample_rate=16000, int_values=False, augmentor=None) + ds_braceexpand = TarredAudioToClassificationLabelDataset( + audio_tar_filepaths=tarpath, manifest_filepath=manifest_path, labels=self.labels, featurizer=featurizer + ) + + assert len(ds_braceexpand) == 6 + count = 0 + for _ in ds_braceexpand: + count += 1 + assert count == 6 + + # Test loading via list + tarpath = [os.path.abspath(os.path.join(test_data_dir, f'asr/tarred_an4/audio_{i}.tar')) for i in range(2)] + ds_list_load = TarredAudioToClassificationLabelDataset( + audio_tar_filepaths=tarpath, manifest_filepath=manifest_path, labels=self.labels, featurizer=featurizer + ) + count = 0 + for _ in ds_list_load: + count += 1 + assert count == 6 + + @pytest.mark.unit + def test_feat_seqlabel_dataset(self, test_data_dir): + manifest_path = os.path.abspath(os.path.join(test_data_dir, 'asr/feat/emb.json')) + feature_loader = ExternalFeatureLoader(file_path=manifest_path, sample_rate=16000, augmentor=None) + ds_braceexpand = FeatureToSeqSpeakerLabelDataset( + manifest_filepath=manifest_path, labels=self.unique_labels_in_seq, feature_loader=feature_loader + ) + # fmt: off + correct_label = torch.tensor( + [0.0, 1.0, 2.0, 2.0, 1.0, 2.0, 1.0, 2.0, 2.0, 1.0, 2.0, 2.0, 3.0, 1.0, 2.0, 2.0, 2.0, 0.0, 2.0, 1.0, 1.0, 2.0, 2.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 2.0, 0.0, 2.0, 2.0, 2.0, 1.0, 2.0, 1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 2.0, 1.0, 2.0, 1.0,] + ) + # fmt: on + correct_label_length = torch.tensor(50) + + assert ds_braceexpand[0][0].shape == (50, 32) + assert torch.equal(ds_braceexpand[0][2], correct_label) + assert torch.equal(ds_braceexpand[0][3], correct_label_length) + + count = 0 + for _ in ds_braceexpand: + count += 1 + assert count == 2 diff --git a/models/tacotron2/NeMo/tests/collections/asr/test_speaker_label_models.py b/models/tacotron2/NeMo/tests/collections/asr/test_speaker_label_models.py new file mode 100644 index 0000000000..f11f880969 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/asr/test_speaker_label_models.py @@ -0,0 +1,63 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +import pytest +from omegaconf import DictConfig + +from nemo.collections.asr.models import EncDecSpeakerLabelModel + + +class EncDecSpeechLabelModelTest(TestCase): + @pytest.mark.unit + def test_constructor(self): + preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} + encoder = { + 'cls': 'nemo.collections.asr.modules.ConvASREncoder', + 'params': { + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 512, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': False, + } + ], + }, + } + + decoder = { + 'cls': 'nemo.collections.asr.modules.SpeakerDecoder', + 'params': {'feat_in': 512, 'num_classes': 2, 'pool_mode': 'xvector', 'emb_sizes': [1024]}, + } + + modelConfig = DictConfig( + {'preprocessor': DictConfig(preprocessor), 'encoder': DictConfig(encoder), 'decoder': DictConfig(decoder)} + ) + speaker_model = EncDecSpeakerLabelModel(cfg=modelConfig) + speaker_model.train() + # TODO: make proper config and assert correct number of weights + + # Check to/from config_dict: + confdict = speaker_model.to_config_dict() + instance2 = EncDecSpeakerLabelModel.from_config_dict(confdict) + self.assertTrue(isinstance(instance2, EncDecSpeakerLabelModel)) diff --git a/models/tacotron2/NeMo/tests/collections/common/__init__.py b/models/tacotron2/NeMo/tests/collections/common/__init__.py new file mode 100644 index 0000000000..9e32500719 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/common/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/tacotron2/NeMo/tests/collections/common/loss_inputs.py b/models/tacotron2/NeMo/tests/collections/common/loss_inputs.py new file mode 100644 index 0000000000..cc3c384efe --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/common/loss_inputs.py @@ -0,0 +1,69 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2018-2020 William Falcon +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass + +import torch + +from .pl_utils import BATCH_SIZE, NUM_BATCHES, NUM_CLASSES + + +@dataclass(frozen=True) +class LossInput: + """ + The input for ``nemo.collections.common.metrics.GlobalAverageLossMetric`` metric tests. + + Args: + loss_sum_or_avg: a one dimensional float tensor which contains losses for averaging. Each element is either a + sum or mean of several losses depending on the parameter ``take_avg_loss`` of the + ``nemo.collections.common.metrics.GlobalAverageLossMetric`` class. + num_measurements: a one dimensional integer tensor which contains number of measurements which sums or average + values are in ``loss_sum_or_avg``. + """ + + loss_sum_or_avg: torch.Tensor + num_measurements: torch.Tensor + + +NO_ZERO_NUM_MEASUREMENTS = LossInput( + loss_sum_or_avg=torch.rand(NUM_BATCHES) * 2.0 - 1.0, num_measurements=torch.randint(1, 100, (NUM_BATCHES,)), +) + +SOME_NUM_MEASUREMENTS_ARE_ZERO = LossInput( + loss_sum_or_avg=torch.rand(NUM_BATCHES) * 2.0 - 1.0, + num_measurements=torch.cat( + ( + torch.randint(1, 100, (NUM_BATCHES // 2,), dtype=torch.int32), + torch.zeros(NUM_BATCHES - NUM_BATCHES // 2, dtype=torch.int32), + ) + ), +) + +ALL_NUM_MEASUREMENTS_ARE_ZERO = LossInput( + loss_sum_or_avg=torch.rand(NUM_BATCHES) * 2.0 - 1.0, num_measurements=torch.zeros(NUM_BATCHES, dtype=torch.int32), +) diff --git a/models/tacotron2/NeMo/tests/collections/common/perplexity_inputs.py b/models/tacotron2/NeMo/tests/collections/common/perplexity_inputs.py new file mode 100644 index 0000000000..6e785ba7a4 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/common/perplexity_inputs.py @@ -0,0 +1,49 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2018-2020 William Falcon +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple + +import torch + +from .pl_utils import BATCH_SIZE, NUM_BATCHES, NUM_CLASSES + +Input = namedtuple('Input', ["probs", "logits"]) + + +ONLY_PROBS = Input(probs=torch.rand(NUM_BATCHES, BATCH_SIZE, NUM_CLASSES), logits=None) + +ONLY_LOGITS1 = Input(probs=None, logits=torch.rand(NUM_BATCHES, BATCH_SIZE, NUM_CLASSES)) + +ONLY_LOGITS100 = Input(probs=None, logits=torch.rand(NUM_BATCHES, BATCH_SIZE, NUM_CLASSES) * 200 - 100) + +PROBS_AND_LOGITS = Input( + probs=torch.rand(NUM_BATCHES, BATCH_SIZE, NUM_CLASSES), + logits=torch.rand(NUM_BATCHES, BATCH_SIZE, NUM_CLASSES) * 200 - 100, +) + +NO_PROBS_NO_LOGITS = Input(probs=None, logits=None) diff --git a/models/tacotron2/NeMo/tests/collections/common/pl_utils.py b/models/tacotron2/NeMo/tests/collections/common/pl_utils.py new file mode 100644 index 0000000000..b902e408d4 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/common/pl_utils.py @@ -0,0 +1,557 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2018-2020 William Falcon +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pickle +import sys +from functools import partial +from typing import Callable, Optional + +import numpy as np +import pytest +import torch +from scipy.stats import entropy +from torch.distributions.utils import logits_to_probs +from torch.multiprocessing import Pool, set_start_method +from torchmetrics import Metric + +from nemo.collections.common.metrics import GlobalAverageLossMetric, Perplexity + +NUM_PROCESSES = 2 +NUM_BATCHES = 10 +BATCH_SIZE = 16 +NUM_CLASSES = 5 +EXTRA_DIM = 3 +THRESHOLD = 0.5 + + +def setup_ddp(rank, world_size): + """ Setup ddp enviroment """ + os.environ["MASTER_ADDR"] = 'localhost' + os.environ['MASTER_PORT'] = '8088' + + if torch.distributed.is_available() and sys.platform not in ['win32', 'cygwin']: + torch.distributed.init_process_group("gloo", rank=rank, world_size=world_size) + + +def _class_test( + rank: int, + worldsize: int, + preds: torch.Tensor, + target: torch.Tensor, + metric_class: Metric, + sk_metric: Callable, + dist_sync_on_step: bool, + metric_args: dict = {}, + check_dist_sync_on_step: bool = True, + check_batch: bool = True, + atol: float = 1e-8, +): + """ Utility function doing the actual comparison between lightning class metric + and reference metric. + Args: + rank: rank of current process + worldsize: number of processes + preds: torch tensor with predictions + target: torch tensor with targets + metric_class: lightning metric class that should be tested + sk_metric: callable function that is used for comparison + dist_sync_on_step: bool, if true will synchronize metric state across + processes at each ``forward()`` + metric_args: dict with additional arguments used for class initialization + check_dist_sync_on_step: bool, if true will check if the metric is also correctly + calculated per batch per device (and not just at the end) + check_batch: bool, if true will check if the metric is also correctly + calculated across devices for each batch (and not just at the end) + """ + # Instanciate lightning metric + metric = metric_class(compute_on_step=True, dist_sync_on_step=dist_sync_on_step, **metric_args) + + # verify metrics work after being loaded from pickled state + pickled_metric = pickle.dumps(metric) + metric = pickle.loads(pickled_metric) + + for i in range(rank, NUM_BATCHES, worldsize): + batch_result = metric(preds[i], target[i]) + + if metric.dist_sync_on_step: + if rank == 0: + ddp_preds = torch.stack([preds[i + r] for r in range(worldsize)]) + ddp_target = torch.stack([target[i + r] for r in range(worldsize)]) + sk_batch_result = sk_metric(ddp_preds, ddp_target) + # assert for dist_sync_on_step + if check_dist_sync_on_step: + assert np.allclose(batch_result.numpy(), sk_batch_result, atol=atol) + else: + sk_batch_result = sk_metric(preds[i], target[i]) + # assert for batch + if check_batch: + assert np.allclose(batch_result.numpy(), sk_batch_result, atol=atol) + + # check on all batches on all ranks + result = metric.compute() + assert isinstance(result, torch.Tensor) + + total_preds = torch.stack([preds[i] for i in range(NUM_BATCHES)]) + total_target = torch.stack([target[i] for i in range(NUM_BATCHES)]) + sk_result = sk_metric(total_preds, total_target) + + # assert after aggregation + assert np.allclose(result.numpy(), sk_result, atol=atol) + + +def _functional_test( + preds: torch.Tensor, + target: torch.Tensor, + metric_functional: Callable, + sk_metric: Callable, + metric_args: dict = {}, + atol: float = 1e-8, +): + """ Utility function doing the actual comparison between lightning functional metric + and reference metric. + Args: + preds: torch tensor with predictions + target: torch tensor with targets + metric_functional: lightning metric functional that should be tested + sk_metric: callable function that is used for comparison + metric_args: dict with additional arguments used for class initialization + """ + metric = partial(metric_functional, **metric_args) + + for i in range(NUM_BATCHES): + lightning_result = metric(preds[i], target[i]) + sk_result = sk_metric(preds[i], target[i]) + + # assert its the same + assert np.allclose(lightning_result.numpy(), sk_result, atol=atol) + + +class MetricTester: + """ Class used for efficiently run alot of parametrized tests in ddp mode. + Makes sure that ddp is only setup once and that pool of processes are + used for all tests. + All tests should subclass from this and implement a new method called + `test_metric_name` + where the method `self.run_metric_test` is called inside. + """ + + atol = 1e-8 + + def setup_class(self): + """ Setup the metric class. This will spawn the pool of workers that are + used for metric testing and setup_ddp + """ + try: + set_start_method('spawn') + except RuntimeError: + pass + self.poolSize = NUM_PROCESSES + self.pool = Pool(processes=self.poolSize) + self.pool.starmap(setup_ddp, [(rank, self.poolSize) for rank in range(self.poolSize)]) + + def teardown_class(self): + """ Close pool of workers """ + self.pool.close() + self.pool.join() + + def run_functional_metric_test( + self, + preds: torch.Tensor, + target: torch.Tensor, + metric_functional: Callable, + sk_metric: Callable, + metric_args: dict = {}, + ): + """ Main method that should be used for testing functions. Call this inside + testing method + Args: + preds: torch tensor with predictions + target: torch tensor with targets + metric_functional: lightning metric class that should be tested + sk_metric: callable function that is used for comparison + metric_args: dict with additional arguments used for class initialization + """ + _functional_test( + preds=preds, + target=target, + metric_functional=metric_functional, + sk_metric=sk_metric, + metric_args=metric_args, + atol=self.atol, + ) + + def run_class_metric_test( + self, + ddp: bool, + preds: torch.Tensor, + target: torch.Tensor, + metric_class: Metric, + sk_metric: Callable, + dist_sync_on_step: bool, + metric_args: dict = {}, + check_dist_sync_on_step: bool = True, + check_batch: bool = True, + ): + """ Main method that should be used for testing class. Call this inside testing + methods. + Args: + ddp: bool, if running in ddp mode or not + preds: torch tensor with predictions + target: torch tensor with targets + metric_class: lightning metric class that should be tested + sk_metric: callable function that is used for comparison + dist_sync_on_step: bool, if true will synchronize metric state across + processes at each ``forward()`` + metric_args: dict with additional arguments used for class initialization + check_dist_sync_on_step: bool, if true will check if the metric is also correctly + calculated per batch per device (and not just at the end) + check_batch: bool, if true will check if the metric is also correctly + calculated across devices for each batch (and not just at the end) + """ + if ddp: + if sys.platform == "win32": + pytest.skip("DDP not supported on windows") + + self.pool.starmap( + partial( + _class_test, + preds=preds, + target=target, + metric_class=metric_class, + sk_metric=sk_metric, + dist_sync_on_step=dist_sync_on_step, + metric_args=metric_args, + check_dist_sync_on_step=check_dist_sync_on_step, + check_batch=check_batch, + atol=self.atol, + ), + [(rank, self.poolSize) for rank in range(self.poolSize)], + ) + else: + _class_test( + 0, + 1, + preds=preds, + target=target, + metric_class=metric_class, + sk_metric=sk_metric, + dist_sync_on_step=dist_sync_on_step, + metric_args=metric_args, + check_dist_sync_on_step=check_dist_sync_on_step, + check_batch=check_batch, + atol=self.atol, + ) + + +def reference_perplexity_func(probs): + ent = entropy(probs, axis=-1) + ppl = np.exp(ent) + return ppl.mean() + + +def _perplexity_class_test( + rank: int, + worldsize: int, + probs: Optional[torch.Tensor], + logits: Optional[torch.Tensor], + dist_sync_on_step: bool, + metric_args: dict = {}, + check_dist_sync_on_step: bool = True, + check_batch: bool = True, + atol: float = 1e-8, +): + """ Utility function doing the actual comparison between lightning class metric + and reference metric. + Args: + rank: rank of current process + worldsize: number of processes + probs: torch tensor with probabilities + logits: torch tensor with logits. The function checks ``probs`` and ``logits are mutually exclusive for + ``Perplexity`` metric. + dist_sync_on_step: bool, if true will synchronize metric state across + processes at each ``forward()`` + metric_args: dict with additional arguments used for class initialization + check_dist_sync_on_step: bool, if true will check if the metric is also correctly + calculated per batch per device (and not just at the end) + check_batch: bool, if true will check if the metric is also correctly + calculated across devices for each batch (and not just at the end) + """ + # Instanciate lightning metric + perplexity = Perplexity(compute_on_step=True, dist_sync_on_step=dist_sync_on_step, **metric_args) + if (probs is None) == (logits is None): + with pytest.raises(ValueError): + perplexity(probs, logits) + return + + # verify perplexity works after being loaded from pickled state + pickled_metric = pickle.dumps(perplexity) + perplexity = pickle.loads(pickled_metric) + + for i in range(rank, NUM_BATCHES, worldsize): + batch_result = perplexity(None if probs is None else probs[i], None if logits is None else logits[i]) + + if perplexity.dist_sync_on_step: + if rank == 0: + if probs is not None: + ddp_probs = torch.stack([probs[i + r] for r in range(worldsize)]) + else: + ddp_logits = torch.stack([logits[i + r] for r in range(worldsize)]) + ddp_probs = logits_to_probs(ddp_logits, is_binary=False) + sk_batch_result = reference_perplexity_func(ddp_probs) + # assert for dist_sync_on_step + if check_dist_sync_on_step: + assert np.allclose(batch_result.numpy(), sk_batch_result, atol=atol) + else: + if probs is None: + p = logits_to_probs(logits[i], is_binary=False) + else: + p = probs[i] + sk_batch_result = reference_perplexity_func(p) + # assert for batch + if check_batch: + assert np.allclose(batch_result.numpy(), sk_batch_result, atol=atol) + + assert (probs is None) != (logits is None) + # check on all batches on all ranks + result = perplexity.compute() + assert isinstance(result, torch.Tensor) + + if probs is None: + probs = logits_to_probs(logits, is_binary=False) + sk_result = reference_perplexity_func(probs) + + # assert after aggregation + assert np.allclose(result.numpy(), sk_result, atol=atol) + + +class PerplexityTester(MetricTester): + def run_class_perplexity_test( + self, + ddp: bool, + probs: Optional[torch.Tensor], + logits: Optional[torch.Tensor], + dist_sync_on_step: bool, + metric_args: dict = {}, + check_dist_sync_on_step: bool = True, + check_batch: bool = True, + ): + """ Main method that should be used for testing class. Call this inside testing + methods. + Args: + ddp: bool, if running in ddp mode or not + probs: torch tensor with probabilities. + logits: torch tensor with logits. This test checks that probs and logits are mutually exclusive for + ``Perplexity`` metric. + dist_sync_on_step: bool, if true will synchronize metric state across + processes at each ``forward()`` + metric_args: dict with additional arguments used for class initialization + check_dist_sync_on_step: bool, if true will check if the metric is also correctly + calculated per batch per device (and not just at the end) + check_batch: bool, if true will check if the metric is also correctly + calculated across devices for each batch (and not just at the end) + """ + if ddp: + if sys.platform == "win32": + pytest.skip("DDP not supported on windows") + + self.pool.starmap( + partial( + _perplexity_class_test, + probs=probs, + logits=logits, + dist_sync_on_step=dist_sync_on_step, + metric_args=metric_args, + check_dist_sync_on_step=check_dist_sync_on_step, + check_batch=check_batch, + atol=self.atol, + ), + [(rank, self.poolSize) for rank in range(self.poolSize)], + ) + else: + _perplexity_class_test( + 0, + 1, + probs=probs, + logits=logits, + dist_sync_on_step=dist_sync_on_step, + metric_args=metric_args, + check_dist_sync_on_step=check_dist_sync_on_step, + check_batch=check_batch, + atol=self.atol, + ) + + +def reference_loss_func(loss_sum_or_avg: torch.Tensor, num_measurements: torch.Tensor, take_avg_loss: bool): + """ + Returns average loss for data from``loss_sum_or_avg``. This function sums all losses from ``loss_sum_or_avg`` and + divides the sum by the sum of ``num_measurements`` elements. + + If ``take_avg_loss`` is ``True`` then ``loss_sum_or_avg[i]`` elements are mean values of ``num_measurements[i]`` + losses. In that case before computing sum of losses each element of ``loss_sum_or_avg`` is multiplied by + corresponding element of ``num_measurements``. + + If ``num_measurements`` sum is zero then the function returns NaN tensor. + + The function is used for testing ``nemo.collections.common.metrics.GlobalAverageLossMetric`` class. + + Args: + loss_sum_or_avg: a one dimensional float ``torch.Tensor``. Sums or mean values of loss. + num_measurements: a one dimensional integer ``torch.Tensor``. Number of values on which sums of means in + ``loss_sum_or_avg`` are calculated. + take_avg_loss: if ``True`` then ``loss_sum_or_avg`` contains mean losses else ``loss_sum_or_avg`` contains + sums of losses. + """ + if take_avg_loss: + for i in range(loss_sum_or_avg.shape[0]): + loss_sum_or_avg[i] *= num_measurements[i] + nm_sum = num_measurements.sum() + if nm_sum.eq(0): + return torch.tensor(float('nan')) + return loss_sum_or_avg.sum() / nm_sum + + +def _loss_class_test( + rank: int, + worldsize: int, + loss_sum_or_avg: Optional[torch.Tensor], + num_measurements: Optional[torch.Tensor], + dist_sync_on_step: bool, + take_avg_loss: bool, + check_dist_sync_on_step: bool = True, + check_batch: bool = True, + atol: float = 1e-8, +): + """ Utility function doing the actual comparison between lightning class metric + and reference metric. + Args: + rank: rank of current process + worldsize: number of processes + loss_sum_or_avg: a one dimensional float torch tensor with loss sums or means. + num_measurements: a one dimensional integer torch tensor with number of values on which sums or means from + ``loss_sum_or_avg`` were computed. + dist_sync_on_step: bool, if true will synchronize metric state across + processes at each ``forward()`` + take_avg_loss: dict with additional arguments used for class initialization + check_dist_sync_on_step: bool, if true will check if the metric is also correctly + calculated per batch per device (and not just at the end) + check_batch: bool, if true will check if the metric is also correctly + calculated across devices for each batch (and not just at the end) + """ + # Instantiate lightning metric + loss_metric = GlobalAverageLossMetric( + compute_on_step=True, dist_sync_on_step=dist_sync_on_step, take_avg_loss=take_avg_loss + ) + + # verify perplexity works after being loaded from pickled state + pickled_metric = pickle.dumps(loss_metric) + loss_metric = pickle.loads(pickled_metric) + + for i in range(rank, NUM_BATCHES, worldsize): + batch_result = loss_metric(loss_sum_or_avg[i], num_measurements[i]) + if loss_metric.dist_sync_on_step: + if rank == 0: + ddp_loss_sum_or_avg = torch.stack([loss_sum_or_avg[i + r] for r in range(worldsize)]) + ddp_num_measurements = torch.stack([num_measurements[i + r] for r in range(worldsize)]) + sk_batch_result = reference_loss_func(ddp_loss_sum_or_avg, ddp_num_measurements, take_avg_loss) + # assert for dist_sync_on_step + if check_dist_sync_on_step: + if sk_batch_result.isnan(): + assert batch_result.isnan() + else: + assert ( + np.allclose(batch_result.numpy(), sk_batch_result, atol=atol), + f"batch_result = {batch_result.numpy()}, sk_batch_result = {sk_batch_result}", + ) + else: + ls = loss_sum_or_avg[i : i + 1] + nm = num_measurements[i : i + 1] + sk_batch_result = reference_loss_func(ls, nm, take_avg_loss) + # assert for batch + if check_batch: + if sk_batch_result.isnan(): + assert batch_result.isnan() + else: + assert ( + np.allclose(batch_result.numpy(), sk_batch_result, atol=atol), + f"batch_result = {batch_result.numpy()}, sk_batch_result = {sk_batch_result}", + ) + # check on all batches on all ranks + result = loss_metric.compute() + assert isinstance(result, torch.Tensor) + sk_result = reference_loss_func(loss_sum_or_avg, num_measurements, take_avg_loss) + + # assert after aggregation + if sk_result.isnan(): + assert result.isnan() + else: + assert ( + np.allclose(result.numpy(), sk_result, atol=atol), + f"result = {result.numpy()}, sk_result = {sk_result}", + ) + + +class LossTester(MetricTester): + def run_class_loss_test( + self, + ddp: bool, + loss_sum_or_avg: torch.Tensor, + num_measurements: torch.Tensor, + dist_sync_on_step: bool, + take_avg_loss: bool, + check_dist_sync_on_step: bool = True, + check_batch: bool = True, + ): + if ddp: + if sys.platform == "win32": + pytest.skip("DDP not supported on windows") + self.pool.starmap( + partial( + _loss_class_test, + loss_sum_or_avg=loss_sum_or_avg, + num_measurements=num_measurements, + dist_sync_on_step=dist_sync_on_step, + take_avg_loss=take_avg_loss, + check_dist_sync_on_step=check_dist_sync_on_step, + check_batch=check_batch, + atol=self.atol, + ), + [(rank, self.poolSize) for rank in range(self.poolSize)], + ) + else: + _loss_class_test( + 0, + 1, + loss_sum_or_avg=loss_sum_or_avg, + num_measurements=num_measurements, + dist_sync_on_step=dist_sync_on_step, + take_avg_loss=take_avg_loss, + check_dist_sync_on_step=check_dist_sync_on_step, + check_batch=check_batch, + atol=self.atol, + ) diff --git a/models/tacotron2/NeMo/tests/collections/common/test_metrics.py b/models/tacotron2/NeMo/tests/collections/common/test_metrics.py new file mode 100644 index 0000000000..6729ae3339 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/common/test_metrics.py @@ -0,0 +1,150 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import torch + +from .loss_inputs import ALL_NUM_MEASUREMENTS_ARE_ZERO, NO_ZERO_NUM_MEASUREMENTS, SOME_NUM_MEASUREMENTS_ARE_ZERO +from .perplexity_inputs import NO_PROBS_NO_LOGITS, ONLY_LOGITS1, ONLY_LOGITS100, ONLY_PROBS, PROBS_AND_LOGITS +from .pl_utils import LossTester, PerplexityTester +from nemo.collections.common.metrics.classification_accuracy import TopKClassificationAccuracy + + +class TestCommonMetrics: + top_k_logits = torch.tensor([[0.1, 0.3, 0.2, 0.0], [0.9, 0.6, 0.2, 0.3], [0.2, 0.1, 0.4, 0.3]],) # 1 # 0 # 2 + + @pytest.mark.unit + def test_top_1_accuracy(self): + labels = torch.tensor([0, 0, 2], dtype=torch.long) + + accuracy = TopKClassificationAccuracy(top_k=None) + acc = accuracy(logits=self.top_k_logits, labels=labels) + + assert accuracy.correct_counts_k.shape == torch.Size([1]) + assert accuracy.total_counts_k.shape == torch.Size([1]) + assert abs(acc[0] - 0.667) < 1e-3 + + @pytest.mark.unit + def test_top_1_2_accuracy(self): + labels = torch.tensor([0, 1, 0], dtype=torch.long) + + accuracy = TopKClassificationAccuracy(top_k=[1, 2]) + top1_acc, top2_acc = accuracy(logits=self.top_k_logits, labels=labels) + + assert accuracy.correct_counts_k.shape == torch.Size([2]) + assert accuracy.total_counts_k.shape == torch.Size([2]) + + assert abs(top1_acc - 0.0) < 1e-3 + assert abs(top2_acc - 0.333) < 1e-3 + + @pytest.mark.unit + def test_top_1_accuracy_distributed(self): + # Simulate test on 2 process DDP execution + labels = torch.tensor([[0, 0, 2], [2, 0, 0]], dtype=torch.long) + + accuracy = TopKClassificationAccuracy(top_k=None) + proc1_acc = accuracy(logits=self.top_k_logits, labels=labels[0]) + correct1, total1 = accuracy.correct_counts_k, accuracy.total_counts_k + + accuracy.reset() + proc2_acc = accuracy(logits=torch.flip(self.top_k_logits, dims=[1]), labels=labels[1]) # reverse logits + correct2, total2 = accuracy.correct_counts_k, accuracy.total_counts_k + + correct = torch.stack([correct1, correct2]) + total = torch.stack([total1, total2]) + + assert correct.shape == torch.Size([2, 1]) + assert total.shape == torch.Size([2, 1]) + + assert abs(proc1_acc[0] - 0.667) < 1e-3 # 2/3 + assert abs(proc2_acc[0] - 0.333) < 1e-3 # 1/3 + + accuracy.reset() + accuracy.correct_counts_k = torch.tensor([correct.sum()]) + accuracy.total_counts_k = torch.tensor([total.sum()]) + acc_topk = accuracy.compute() + acc_top1 = acc_topk[0] + + assert abs(acc_top1 - 0.5) < 1e-3 # 3/6 + + @pytest.mark.unit + def test_top_1_accuracy_distributed_uneven_batch(self): + # Simulate test on 2 process DDP execution + accuracy = TopKClassificationAccuracy(top_k=None) + + proc1_acc = accuracy(logits=self.top_k_logits, labels=torch.tensor([0, 0, 2])) + correct1, total1 = accuracy.correct_counts_k, accuracy.total_counts_k + + proc2_acc = accuracy( + logits=torch.flip(self.top_k_logits, dims=[1])[:2, :], # reverse logits, select first 2 samples + labels=torch.tensor([2, 0]), + ) # reduce number of labels + correct2, total2 = accuracy.correct_counts_k, accuracy.total_counts_k + + correct = torch.stack([correct1, correct2]) + total = torch.stack([total1, total2]) + + assert correct.shape == torch.Size([2, 1]) + assert total.shape == torch.Size([2, 1]) + + assert abs(proc1_acc[0] - 0.667) < 1e-3 # 2/3 + assert abs(proc2_acc[0] - 0.500) < 1e-3 # 1/2 + + accuracy.correct_counts_k = torch.tensor([correct.sum()]) + accuracy.total_counts_k = torch.tensor([total.sum()]) + acc_topk = accuracy.compute() + acc_top1 = acc_topk[0] + + assert abs(acc_top1 - 0.6) < 1e-3 # 3/5 + + +@pytest.mark.parametrize("ddp", [True, False]) +@pytest.mark.parametrize("dist_sync_on_step", [True, False]) +@pytest.mark.parametrize( + "probs, logits", + [ + (ONLY_PROBS.probs, ONLY_PROBS.logits), + (ONLY_LOGITS1.probs, ONLY_LOGITS1.logits), + (ONLY_LOGITS100.probs, ONLY_LOGITS100.logits), + (PROBS_AND_LOGITS.probs, PROBS_AND_LOGITS.logits), + (NO_PROBS_NO_LOGITS.probs, NO_PROBS_NO_LOGITS.logits), + ], +) +class TestPerplexity(PerplexityTester): + def test_perplexity(self, ddp, dist_sync_on_step, probs, logits): + self.run_class_perplexity_test( + ddp=ddp, probs=probs, logits=logits, dist_sync_on_step=dist_sync_on_step, + ) + + +@pytest.mark.parametrize("ddp", [True, False]) +@pytest.mark.parametrize("dist_sync_on_step", [True, False]) +@pytest.mark.parametrize("take_avg_loss", [True, False]) +@pytest.mark.parametrize( + "loss_sum_or_avg, num_measurements", + [ + (NO_ZERO_NUM_MEASUREMENTS.loss_sum_or_avg, NO_ZERO_NUM_MEASUREMENTS.num_measurements), + (SOME_NUM_MEASUREMENTS_ARE_ZERO.loss_sum_or_avg, SOME_NUM_MEASUREMENTS_ARE_ZERO.num_measurements), + (ALL_NUM_MEASUREMENTS_ARE_ZERO.loss_sum_or_avg, ALL_NUM_MEASUREMENTS_ARE_ZERO.num_measurements), + ], +) +class TestLoss(LossTester): + def test_loss(self, ddp, dist_sync_on_step, loss_sum_or_avg, num_measurements, take_avg_loss): + self.run_class_loss_test( + ddp=ddp, + loss_sum_or_avg=loss_sum_or_avg, + num_measurements=num_measurements, + dist_sync_on_step=dist_sync_on_step, + take_avg_loss=take_avg_loss, + ) diff --git a/models/tacotron2/NeMo/tests/collections/common/test_spc_tokenizer.py b/models/tacotron2/NeMo/tests/collections/common/test_spc_tokenizer.py new file mode 100644 index 0000000000..da7b079234 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/common/test_spc_tokenizer.py @@ -0,0 +1,267 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from nemo.collections.common.tokenizers.sentencepiece_tokenizer import SentencePieceTokenizer +from nemo.collections.common.tokenizers.youtokentome_tokenizer import YouTokenToMeTokenizer + +MODEL_SPECIAL_TOKENS = { + 'unk_token': '[UNK]', + 'sep_token': '[SEP]', + 'pad_token': '[PAD]', + 'bos_token': '[CLS]', + 'mask_token': '[MASK]', + 'eos_token': '[SEP]', + 'cls_token': '[CLS]', +} + + +class TestSentencePieceTokenizerLegacy: + model_name = "/m_common.model" + + @pytest.mark.unit + def test_add_special_tokens(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True) + special_tokens = MODEL_SPECIAL_TOKENS + tokenizer.add_special_tokens(special_tokens) + assert tokenizer.vocab_size == tokenizer.original_vocab_size + len(set(special_tokens.values())) + + @pytest.mark.unit + def test_text_to_tokens(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True) + special_tokens = MODEL_SPECIAL_TOKENS + tokenizer.add_special_tokens(special_tokens) + + text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" + tokens = tokenizer.text_to_tokens(text) + + assert len(tokens) == len(text.split()) + assert tokens.count("[CLS]") == 1 + assert tokens.count("[MASK]") == 1 + assert tokens.count("[SEP]") == 2 + + @pytest.mark.unit + def test_tokens_to_text(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True) + + text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" + tokens = tokenizer.text_to_tokens(text) + result = tokenizer.tokens_to_text(tokens) + + assert text == result + + @pytest.mark.unit + def test_text_to_ids(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True) + special_tokens = MODEL_SPECIAL_TOKENS + tokenizer.add_special_tokens(special_tokens) + + text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" + ids = tokenizer.text_to_ids(text) + + assert len(ids) == len(text.split()) + assert ids.count(tokenizer.token_to_id("[CLS]")) == 1 + assert ids.count(tokenizer.token_to_id("[MASK]")) == 1 + assert ids.count(tokenizer.token_to_id("[SEP]")) == 2 + + @pytest.mark.unit + def test_ids_to_text(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True) + special_tokens = MODEL_SPECIAL_TOKENS + tokenizer.add_special_tokens(special_tokens) + + text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" + ids = tokenizer.text_to_ids(text) + result = tokenizer.ids_to_text(ids) + + assert text == result + + @pytest.mark.unit + def test_tokens_to_ids(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True) + special_tokens = MODEL_SPECIAL_TOKENS + tokenizer.add_special_tokens(special_tokens) + + text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" + tokens = tokenizer.text_to_tokens(text) + ids = tokenizer.tokens_to_ids(tokens) + + assert len(ids) == len(tokens) + assert ids.count(tokenizer.token_to_id("[CLS]")) == 1 + assert ids.count(tokenizer.token_to_id("[MASK]")) == 1 + assert ids.count(tokenizer.token_to_id("[SEP]")) == 2 + + @pytest.mark.unit + def test_ids_to_tokens(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True) + special_tokens = MODEL_SPECIAL_TOKENS + tokenizer.add_special_tokens(special_tokens) + + text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]" + tokens = tokenizer.text_to_tokens(text) + ids = tokenizer.tokens_to_ids(tokens) + result = tokenizer.ids_to_tokens(ids) + + assert len(result) == len(tokens) + + for i in range(len(result)): + assert result[i] == tokens[i] + + +class TestSentencePieceTokenizer: + model_name = "/m_new.model" + + @pytest.mark.unit + def test_text_to_tokens(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name) + + # is user_defined_symbol in the test tokenizer model + # , , , and are control symbols + text = " a b c e f g h i
+
+
+ + NeMo Automatic Speech Recognition Demo + +
+ + +
+
+
+ ASR Transcription + +
+
+
+ + +
+ +
+ +
Model Name : + + +
+ +
+ + + + + + + +  
+ NeMo File : + + + + +
+ + + + +
+ +
+
+ + +
+ + + + + +
+
+
+ + + + +
+ + + + +
+
+ +
+ +
+
+
" + tokens = tokenizer.text_to_tokens(text) + + assert tokens.count("") == 1 + assert tokens.count("") == 0 + assert tokens.count("") == 0 + + @pytest.mark.unit + def test_tokens_to_text(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name) + + # is user_defined_symbol in the test tokenizer model + text = " a b c e f g h i" + tokens = tokenizer.text_to_tokens(text) + result = tokenizer.tokens_to_text(tokens) + + assert text == result + + @pytest.mark.unit + def test_text_to_ids(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name) + + # is user_defined_symbol in the test tokenizer model + # , , , and are control symbols + text = " a b c e f g h i " + tokens = tokenizer.text_to_ids(text) + + assert tokens.count(tokenizer.token_to_id("")) == 1 + assert tokens.count(tokenizer.token_to_id("")) == 0 + assert tokens.count(tokenizer.token_to_id("")) == 0 + + @pytest.mark.unit + def test_ids_to_text(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name) + + text = " a b c e f g h i " + ids = tokenizer.text_to_ids(text) + result = tokenizer.ids_to_text(ids) + + assert text == result + + @pytest.mark.unit + def test_tokens_to_ids(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name) + + tokens = ["", "a", "b", "c", "", "e", "f", "", "g", "h", "i", ""] + ids = tokenizer.tokens_to_ids(tokens) + + assert len(ids) == len(tokens) + assert ids.count(tokenizer.token_to_id("")) == 1 + assert ids.count(tokenizer.token_to_id("")) == 1 + assert ids.count(tokenizer.token_to_id("")) == 2 + + @pytest.mark.unit + def test_ids_to_tokens(self, test_data_dir): + tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name) + + tokens = ["", "a", "b", "c", "", "e", "f", "", "g", "h", "i", ""] + ids = tokenizer.tokens_to_ids(tokens) + result = tokenizer.ids_to_tokens(ids) + + assert len(result) == len(tokens) + + for i in range(len(result)): + assert result[i] == tokens[i] + + +class TestYouTokenToMeTokenizer: + model_name = "/yttm.4096.en-de.model" + + @pytest.mark.unit + def test_text_to_tokens(self, test_data_dir): + tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name) + + text = " a b c e f g h i " + tokens = tokenizer.text_to_tokens(text) + + assert tokens.count("") == 0 + assert tokens.count("") == 0 + assert tokens.count("") == 0 + + @pytest.mark.unit + def test_tokens_to_text(self, test_data_dir): + tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name) + + text = "a b c e f g h i" + tokens = tokenizer.text_to_tokens(text) + result = tokenizer.tokens_to_text(tokens) + + assert text == result + + @pytest.mark.unit + def test_text_to_ids(self, test_data_dir): + tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name) + + text = " a b c e f g h i " + tokens = tokenizer.text_to_ids(text) + + assert tokens.count(tokenizer.bos_id) == 0 + assert tokens.count(tokenizer.unk_id) == 0 + assert tokens.count(tokenizer.eos_id) == 0 + + @pytest.mark.unit + def test_ids_to_text(self, test_data_dir): + tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name) + + text = "a b c e f g h i" + ids = tokenizer.text_to_ids(text) + result = tokenizer.ids_to_text(ids) + + assert text == result + + @pytest.mark.unit + def test_tokens_to_ids(self, test_data_dir): + tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name) + + tokens = ["", "a", "b", "c", "", "e", "f", "", "g", "h", "i", ""] + ids = tokenizer.tokens_to_ids(tokens) + + assert len(ids) == len(tokens) + assert ids.count(tokenizer.bos_id) == 1 + assert ids.count(tokenizer.eos_id) == 1 + assert ids.count(tokenizer.unk_id) == 2 + + @pytest.mark.unit + def test_ids_to_tokens(self, test_data_dir): + tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name) + + tokens = ["", "a", "b", "c", "", "e", "f", "", "g", "h", "i", ""] + ids = tokenizer.tokens_to_ids(tokens) + result = tokenizer.ids_to_tokens(ids) + + assert len(result) == len(tokens) + + for i in range(len(result)): + assert result[i] == tokens[i] diff --git a/models/tacotron2/NeMo/tests/collections/nlp/test_classification_report.py b/models/tacotron2/NeMo/tests/collections/nlp/test_classification_report.py new file mode 100644 index 0000000000..de2c4f7a71 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/nlp/test_classification_report.py @@ -0,0 +1,48 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +import torch +from sklearn.metrics import precision_recall_fscore_support + +from nemo.collections.nlp.metrics.classification_report import ClassificationReport + + +class ClassificationReportTests: + num_classes = 3 + label_ids = {'a': 0, 'b': 1, 'c': 2} + + @pytest.mark.unit + def test_classification_report(self): + + preds = torch.Tensor([0, 1, 1, 1, 2, 2, 0]) + labels = torch.Tensor([1, 0, 0, 1, 2, 1, 0]) + + def __convert_to_tensor(sklearn_metric): + return torch.Tensor([round(sklearn_metric * 100)])[0] + + for mode in ['macro', 'micro', 'weighted']: + + classification_report_nemo = ClassificationReport( + num_classes=self.num_classes, label_ids=self.label_ids, mode=mode + ) + # pytest.set_trace() + precision, recall, f1, _ = classification_report_nemo(preds, labels) + tp, fp, fn = classification_report_nemo.tp, classification_report_nemo.fp, classification_report_nemo.fn + pr_sklearn, recall_sklearn, f1_sklearn, _ = precision_recall_fscore_support(labels, preds, average=mode) + + self.assertEqual(torch.round(precision), __convert_to_tensor(pr_sklearn), f'wrong precision for {mode}') + self.assertEqual(torch.round(recall), __convert_to_tensor(recall_sklearn), f'wrong recall for {mode}') + self.assertEqual(torch.round(f1), __convert_to_tensor(f1_sklearn), f'wrong f1 for {mode}') diff --git a/models/tacotron2/NeMo/tests/collections/nlp/test_entity_linking_model.py b/models/tacotron2/NeMo/tests/collections/nlp/test_entity_linking_model.py new file mode 100644 index 0000000000..0e96b33f9b --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/nlp/test_entity_linking_model.py @@ -0,0 +1,83 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tempfile + +import pytest +import wget +from omegaconf import OmegaConf + +from nemo.collections.nlp.models import EntityLinkingModel + + +def get_cfg(): + + language_model = OmegaConf.create( + {"pretrained_model_name": "bert-base-uncased", "config_file": None, "config": None, "lm_checkpoint": None} + ) + + tokenizer = OmegaConf.create( + {"tokenizer_name": "bert-base-uncased", "vocab_file": None, "tokenizer_model": None, "do_lower_case": True} + ) + + model = OmegaConf.create( + { + "nemo_path": "sap_entity_linking.nemo", + "max_seq_length": 128, + "language_model": language_model, + "tokenizer": tokenizer, + "train_ds": None, + "validation_ds": None, + } + ) + + cfg = OmegaConf.create({"model": model}) + + return cfg + + +class TestEntityLinkingModel: + @pytest.mark.unit + def test_creation_saving_restoring(self): + # Create a new temporary directory + with tempfile.TemporaryDirectory() as restore_dir: + with tempfile.TemporaryDirectory() as save_dir: + model = EntityLinkingModel(cfg=get_cfg().model) + assert isinstance(model, EntityLinkingModel) + + save_dir_path = save_dir + + # Where model will be saved + model_save_path = os.path.join(save_dir, f"{model.__class__.__name__}.nemo") + model.save_to(save_path=model_save_path) + + # Where model will be restored from + model_restore_path = os.path.join(restore_dir, f"{model.__class__.__name__}.nemo") + shutil.copy(model_save_path, model_restore_path) + + # at this point save_dir should not exist + assert save_dir_path is not None and not os.path.exists(save_dir_path) + assert not os.path.exists(model_save_path) + assert os.path.exists(model_restore_path) + + # attempt to restore + model_copy = model.__class__.restore_from(restore_path=model_restore_path) + assert model.num_weights == model_copy.num_weights + + +if __name__ == "__main__": + t = TestEntityLinkingModel() + t.test_creation_saving_restoring() diff --git a/models/tacotron2/NeMo/tests/collections/nlp/test_huggingface.py b/models/tacotron2/NeMo/tests/collections/nlp/test_huggingface.py new file mode 100644 index 0000000000..a179c6db95 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/nlp/test_huggingface.py @@ -0,0 +1,86 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile + +import pytest +import torch + +import nemo.collections.nlp as nemo_nlp +from nemo.collections.common.tokenizers.huggingface.auto_tokenizer import AutoTokenizer +from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer + + +def do_export(model, name: str): + with tempfile.TemporaryDirectory() as tmpdir: + # Generate filename in the temporary directory. + tmp_file_name = os.path.join(tmpdir, name + '.onnx') + # Test export. + if torch.cuda.is_available(): + model = model.cuda() + model.export(tmp_file_name) + + +class TestHuggingFace: + @pytest.mark.unit + def test_list_pretrained_models(self): + pretrained_lm_models = nemo_nlp.modules.get_pretrained_lm_models_list() + assert len(pretrained_lm_models) > 0 + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_get_pretrained_bert_model(self): + model = nemo_nlp.modules.get_lm_model(pretrained_model_name='bert-base-uncased') + assert isinstance(model, nemo_nlp.modules.BertEncoder) + do_export(model, "bert-base-uncased") + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_get_pretrained_distilbert_model(self): + model = nemo_nlp.modules.get_lm_model(pretrained_model_name='distilbert-base-uncased') + assert isinstance(model, nemo_nlp.modules.DistilBertEncoder) + do_export(model, "distilbert-base-uncased") + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_get_pretrained_roberta_model(self): + model = nemo_nlp.modules.get_lm_model(pretrained_model_name='roberta-base') + assert isinstance(model, nemo_nlp.modules.RobertaEncoder) + do_export(model, "roberta-base-uncased") + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_get_pretrained_albert_model(self): + model = nemo_nlp.modules.get_lm_model(pretrained_model_name='albert-base-v1') + assert isinstance(model, nemo_nlp.modules.AlbertEncoder) + do_export(model, "albert-base-v1") + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_get_pretrained_chinese_bert_wwm_model(self): + model_name = 'hfl/chinese-bert-wwm' + model = nemo_nlp.modules.get_lm_model(pretrained_model_name=model_name) + assert isinstance(model, nemo_nlp.modules.BertModule) + tokenizer = get_tokenizer(tokenizer_name=model_name) + assert isinstance(tokenizer, AutoTokenizer) + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_get_pretrained_arabic_model(self): + model_name = 'asafaya/bert-base-arabic' + model = nemo_nlp.modules.get_lm_model(pretrained_model_name=model_name) + assert isinstance(model, nemo_nlp.modules.BertModule) + tokenizer = get_tokenizer(tokenizer_name=model_name) + assert isinstance(tokenizer, AutoTokenizer) diff --git a/models/tacotron2/NeMo/tests/collections/nlp/test_megatron.py b/models/tacotron2/NeMo/tests/collections/nlp/test_megatron.py new file mode 100644 index 0000000000..4e7d448a78 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/nlp/test_megatron.py @@ -0,0 +1,76 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + import apex + + apex_available = True +except Exception: + apex_available = False + +import os +import tempfile + +import onnx +import pytest +import torch + +import nemo.collections.nlp as nemo_nlp +from nemo.core.classes import typecheck + + +def get_pretrained_bert_345m_uncased_model(): + model_name = "megatron-bert-345m-uncased" + model = nemo_nlp.modules.get_lm_model(pretrained_model_name=model_name) + if torch.cuda.is_available(): + model = model.cuda() + return model + + +class TestMegatron: + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_list_pretrained_models(self): + pretrained_lm_models = nemo_nlp.modules.get_pretrained_lm_models_list() + assert len(pretrained_lm_models) > 0 + + @pytest.mark.with_downloads() + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + @pytest.mark.skip("Only one Megatron model is allowed") + def test_get_model(self): + model = get_pretrained_bert_345m_uncased_model() + assert isinstance(model, nemo_nlp.modules.MegatronBertEncoder) + + typecheck.set_typecheck_enabled(enabled=False) + inp = model.input_example() + out = model.forward(*inp) + typecheck.set_typecheck_enabled(enabled=True) + + @pytest.mark.skipif(not os.path.exists('/home/TestData/nlp'), reason='Not a Jenkins machine') + @pytest.mark.with_downloads() + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_onnx_export(self): + model = get_pretrained_bert_345m_uncased_model() + assert model + with tempfile.TemporaryDirectory() as tmpdir: + # Generate filename in the temporary directory. + # Test export. + model.export(os.path.join(".", "megatron.onnx")) + + +if __name__ == "__main__": + t = TestMegatron() + t.test_onnx_export() diff --git a/models/tacotron2/NeMo/tests/collections/nlp/test_nlp_exportables.py b/models/tacotron2/NeMo/tests/collections/nlp/test_nlp_exportables.py new file mode 100644 index 0000000000..ac5d86026a --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/nlp/test_nlp_exportables.py @@ -0,0 +1,176 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import tempfile + +import onnx +import pytest +import pytorch_lightning as pl +import torch +import wget +from omegaconf import OmegaConf + +from nemo.collections import nlp as nemo_nlp +from nemo.collections.nlp.models import IntentSlotClassificationModel +from nemo.collections.nlp.modules.common import ( + SequenceClassifier, + SequenceRegression, + SequenceTokenClassifier, + TokenClassifier, +) + + +def classifier_export(obj): + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, obj.__class__.__name__ + '.onnx') + obj = obj.cuda() + obj.export(output=filename) + + +class TestExportableClassifiers: + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_token_classifier_export_to_onnx(self): + for num_layers in [1, 2, 4]: + classifier_export(TokenClassifier(hidden_size=256, num_layers=num_layers, num_classes=16)) + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_bert_pretraining_export_to_onnx(self): + for num_layers in [1, 2, 4]: + classifier_export(TokenClassifier(hidden_size=256, num_layers=num_layers, num_classes=16)) + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_sequence_token_classifier_export_to_onnx(self): + for num_layers in [1, 2, 4]: + classifier_export( + SequenceTokenClassifier(hidden_size=256, num_slots=8, num_intents=8, num_layers=num_layers) + ) + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_sequence_classifier_export_to_onnx(self): + for num_layers in [1, 2, 4]: + classifier_export(SequenceClassifier(hidden_size=256, num_classes=16, num_layers=num_layers)) + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_sequence_regression_export_to_onnx(self): + for num_layers in [1, 2, 4]: + classifier_export(SequenceRegression(hidden_size=256, num_layers=num_layers)) + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_IntentSlotClassificationModel_export_to_onnx(self, dummy_data): + with tempfile.TemporaryDirectory() as tmpdir: + wget.download( + 'https://raw.githubusercontent.com/NVIDIA/NeMo/main/examples/' + 'nlp/intent_slot_classification/conf/intent_slot_classification_config.yaml', + tmpdir, + ) + config_file = os.path.join(tmpdir, 'intent_slot_classification_config.yaml') + config = OmegaConf.load(config_file) + config = OmegaConf.create(OmegaConf.to_container(config, resolve=True)) + config.model.data_dir = dummy_data + config.trainer.gpus = 1 + config.trainer.precision = 32 + config.trainer.accelerator = None + trainer = pl.Trainer(**config.trainer) + model = IntentSlotClassificationModel(config.model, trainer=trainer) + filename = os.path.join(tmpdir, 'isc.onnx') + model.export(output=filename) + onnx_model = onnx.load(filename) + onnx.checker.check_model(onnx_model, full_check=True) # throws when failed + assert onnx_model.graph.input[0].name == 'input_ids' + assert onnx_model.graph.input[1].name == 'attention_mask' + assert onnx_model.graph.input[2].name == 'token_type_ids' + assert onnx_model.graph.output[0].name == 'intent_logits' + assert onnx_model.graph.output[1].name == 'slot_logits' + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + @pytest.mark.skip('Are we exporting to torchscript?') + def test_IntentSlotClassificationModel_export_to_torchscript(self, dummy_data): + with tempfile.TemporaryDirectory() as tmpdir: + wget.download( + 'https://raw.githubusercontent.com/NVIDIA/NeMo/main/examples/' + 'nlp/intent_slot_classification/conf/intent_slot_classification_config.yaml', + tmpdir, + ) + config_file = os.path.join(tmpdir, 'intent_slot_classification_config.yaml') + config = OmegaConf.load(config_file) + config = OmegaConf.create(OmegaConf.to_container(config, resolve=True)) + config.model.data_dir = dummy_data + config.trainer.gpus = 1 + config.trainer.precision = 32 + config.trainer.accelerator = None + trainer = pl.Trainer(**config.trainer) + model = IntentSlotClassificationModel(config.model, trainer=trainer) + filename = os.path.join(tmpdir, 'isc.pt') + model.export(output=filename, check_trace=True) + torchscript_model = torch.jit.load(filename) + inputs = list(torchscript_model.graph.inputs()) + assert inputs[1].debugName() == 'input_ids.1' + assert inputs[2].debugName() == 'input.1' + assert inputs[3].debugName() == 'attention_mask.1' + modules = list(torchscript_model.modules()) + assert modules[2].original_name == 'BertEmbeddings' + assert modules[46].original_name == 'BertSelfAttention' + assert modules[217].original_name == 'SequenceTokenClassifier' + assert modules[219].original_name == 'MultiLayerPerceptron' + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_TokenClassificationModel_export_to_onnx(self): + model = nemo_nlp.models.TokenClassificationModel.from_pretrained(model_name="ner_en_bert") + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, 'ner.onnx') + model.export(output=filename) + onnx_model = onnx.load(filename) + onnx.checker.check_model(onnx_model, full_check=True) # throws when failed + assert onnx_model.graph.input[0].name == 'input_ids' + assert onnx_model.graph.output[0].name == 'logits' + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_PunctuationCapitalizationModel_export_to_onnx(self): + model = nemo_nlp.models.PunctuationCapitalizationModel.from_pretrained(model_name="punctuation_en_distilbert") + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, 'puncap.onnx') + model.export(output=filename) + onnx_model = onnx.load(filename) + onnx.checker.check_model(onnx_model, full_check=True) # throws when failed + assert onnx_model.graph.input[0].name == 'input_ids' + # assert onnx_model.graph.input[2].name == 'token_type_ids' + assert onnx_model.graph.output[0].name == 'punct_logits' + assert onnx_model.graph.output[1].name == 'capit_logits' + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_QAModel_export_to_onnx(self): + model = nemo_nlp.models.QAModel.from_pretrained(model_name="qa_squadv2.0_bertbase") + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, 'qa.onnx') + model.export(output=filename) + onnx_model = onnx.load(filename) + onnx.checker.check_model(onnx_model, full_check=True) # throws when failed + assert onnx_model.graph.input[0].name == 'input_ids' + assert onnx_model.graph.input[2].name == 'token_type_ids' + assert onnx_model.graph.output[0].name == 'logits' + + +@pytest.fixture() +def dummy_data(test_data_dir): + return os.path.join(test_data_dir, 'nlp', 'dummy_data') diff --git a/models/tacotron2/NeMo/tests/collections/nlp/test_nmt_model.py b/models/tacotron2/NeMo/tests/collections/nlp/test_nmt_model.py new file mode 100644 index 0000000000..9f0277f463 --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/nlp/test_nmt_model.py @@ -0,0 +1,144 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import shutil +import tempfile + +import pytest +import torch +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models import MTEncDecModel +from nemo.collections.nlp.models.machine_translation.mt_enc_dec_config import AAYNBaseConfig + + +def export_test(model, suffix, try_script=False): + with tempfile.TemporaryDirectory() as restore_folder: + enc_filename = os.path.join(restore_folder, 'nmt_e' + suffix) + dec_filename = os.path.join(restore_folder, 'nmt_d' + suffix) + model.encoder.export(output=enc_filename, try_script=try_script) + model.decoder.export(output=dec_filename, try_script=try_script) + assert os.path.exists(enc_filename) + assert os.path.exists(dec_filename) + + +def get_cfg(): + cfg = AAYNBaseConfig() + cfg.encoder_tokenizer.tokenizer_name = 'yttm' + cfg.encoder_tokenizer.tokenizer_model = 'tests/.data/yttm.4096.en-de.model' + cfg.decoder_tokenizer.tokenizer_name = 'yttm' + cfg.decoder_tokenizer.tokenizer_model = 'tests/.data/yttm.4096.en-de.model' + cfg.train_ds = None + cfg.validation_ds = None + cfg.test_ds = None + return cfg + + +class TestMTEncDecModel: + @pytest.mark.unit + def test_creation_saving_restoring(self): + model = MTEncDecModel(cfg=get_cfg()) + assert isinstance(model, MTEncDecModel) + # Create a new temporary directory + with tempfile.TemporaryDirectory() as restore_folder: + with tempfile.TemporaryDirectory() as save_folder: + save_folder_path = save_folder + # Where model will be saved + model_save_path = os.path.join(save_folder, f"{model.__class__.__name__}.nemo") + model.save_to(save_path=model_save_path) + # Where model will be restored from + model_restore_path = os.path.join(restore_folder, f"{model.__class__.__name__}.nemo") + shutil.copy(model_save_path, model_restore_path) + # at this point save_folder should not exist + assert save_folder_path is not None and not os.path.exists(save_folder_path) + assert not os.path.exists(model_save_path) + assert os.path.exists(model_restore_path) + # attempt to restore + model_copy = model.__class__.restore_from(restore_path=model_restore_path) + assert model.num_weights == model_copy.num_weights + + @pytest.mark.unit + def test_no_artifact_name_collision(self): + model = MTEncDecModel(cfg=get_cfg()) + assert isinstance(model, MTEncDecModel) + with tempfile.TemporaryDirectory() as tmpdir1: + model.save_to("nmt_model.nemo") + with tempfile.TemporaryDirectory() as tmpdir: + model._unpack_nemo_file(path2file="nmt_model.nemo", out_folder=tmpdir) + conf = OmegaConf.load(os.path.join(tmpdir, "model_config.yaml")) + # Make sure names now differ in saved config + assert conf.encoder_tokenizer.tokenizer_model != conf.decoder_tokenizer.tokenizer_model + # Make sure names in config start with "nemo:" prefix + assert conf.encoder_tokenizer.tokenizer_model.startswith("nemo:") + assert conf.decoder_tokenizer.tokenizer_model.startswith("nemo:") + # Check if both tokenizers were included + assert os.path.exists(os.path.join(tmpdir, conf.encoder_tokenizer.tokenizer_model[5:])) + assert os.path.exists(os.path.join(tmpdir, conf.decoder_tokenizer.tokenizer_model[5:])) + + @pytest.mark.unit + def test_train_eval_loss(self): + cfg = get_cfg() + cfg.label_smoothing = 0.5 + model = MTEncDecModel(cfg=cfg) + assert isinstance(model, MTEncDecModel) + batch_size = 10 + time = 32 + vocab_size = 32000 + torch.manual_seed(42) + tgt_ids = torch.LongTensor(batch_size, time).random_(1, model.decoder_tokenizer.vocab_size) + logits = torch.FloatTensor(batch_size, time, vocab_size).random_(-1, 1) + log_probs = torch.nn.functional.log_softmax(logits, dim=-1) + train_loss = model.loss_fn(log_probs=log_probs, labels=tgt_ids) + eval_loss = model.eval_loss_fn(log_probs=log_probs, labels=tgt_ids) + assert not torch.allclose(train_loss, eval_loss) # , (train_loss, eval_loss) + + cfg.label_smoothing = 0 + model = MTEncDecModel(cfg=cfg) + # Train loss == val loss when label smoothing = 0 + train_loss = model.loss_fn(log_probs=log_probs, labels=tgt_ids) + eval_loss = model.eval_loss_fn(log_probs=log_probs, labels=tgt_ids) + assert torch.allclose(train_loss, eval_loss) + + @pytest.mark.unit + def test_cpu_export_onnx(self): + model = MTEncDecModel(cfg=get_cfg()) + assert isinstance(model, MTEncDecModel) + export_test(model, ".ts") + + @pytest.mark.unit + def test_cpu_export_ts(self): + model = MTEncDecModel(cfg=get_cfg()) + assert isinstance(model, MTEncDecModel) + export_test(model, ".onnx") + + @pytest.mark.skipif(not os.path.exists('/home/TestData/nlp'), reason='Not a Jenkins machine') + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_gpu_export_ts(self): + model = MTEncDecModel(cfg=get_cfg()).cuda() + assert isinstance(model, MTEncDecModel) + export_test(model, ".ts") + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_gpu_export_onnx(self): + model = MTEncDecModel(cfg=get_cfg()).cuda() + assert isinstance(model, MTEncDecModel) + export_test(model, ".onnx") + + +if __name__ == "__main__": + t = TestMTEncDecModel() + # t.test_gpu_export_ts() + t.test_train_eval_loss() diff --git a/models/tacotron2/NeMo/tests/collections/nlp/test_pretrained_models_performance.py b/models/tacotron2/NeMo/tests/collections/nlp/test_pretrained_models_performance.py new file mode 100644 index 0000000000..47f4f7640b --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/nlp/test_pretrained_models_performance.py @@ -0,0 +1,110 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path +from shutil import rmtree +from unittest import TestCase + +import pytest +import pytorch_lightning as pl +from omegaconf import OmegaConf + +import nemo.collections.nlp.models as models + + +def get_metrics(data_dir, model): + trainer = pl.Trainer(gpus=[0]) + + model.set_trainer(trainer) + model.update_data_dir(data_dir) + + test_ds = OmegaConf.create( + { + 'text_file': 'text_dev.txt', + 'labels_file': 'labels_dev.txt', + 'shuffle': False, + 'num_samples': -1, + 'batch_size': 8, + } + ) + + model._cfg.dataset.use_cache = False + trainer.test(model) + model.setup_test_data(test_data_config=test_ds) + metrics = trainer.test(model)[0] + + return metrics + + +def data_exists(data_dir): + return os.path.exists(data_dir) + + +class TestPretrainedModelPerformance: + @pytest.mark.with_downloads() + @pytest.mark.unit + @pytest.mark.run_only_on('GPU') + @pytest.mark.skipif( + not data_exists('/home/TestData/nlp/token_classification_punctuation/fisher'), reason='Not a Jenkins machine' + ) + def test_punct_capit_with_bert(self): + data_dir = '/home/TestData/nlp/token_classification_punctuation/fisher' + model = models.PunctuationCapitalizationModel.from_pretrained("punctuation_en_bert") + metrics = get_metrics(data_dir, model) + + assert abs(metrics['punct_precision'] - 52.3024) < 0.001 + assert abs(metrics['punct_recall'] - 58.9220) < 0.001 + assert abs(metrics['punct_f1'] - 53.2976) < 0.001 + assert abs(metrics['capit_precision'] - 87.0707) < 0.001 + assert abs(metrics['capit_recall'] - 87.0707) < 0.001 + assert abs(metrics['capit_f1'] - 87.0707) < 0.001 + assert int(model.punct_class_report.total_examples) == 128 + + preds_512 = model.add_punctuation_capitalization(['what can i do for you today'], max_seq_length=512)[0] + assert preds_512 == 'What can I do for you today?' + preds_5 = model.add_punctuation_capitalization(['what can i do for you today'], max_seq_length=5)[0] + assert preds_5 == 'What can i' + + @pytest.mark.with_downloads() + @pytest.mark.unit + @pytest.mark.run_only_on('GPU') + @pytest.mark.skipif( + not data_exists('/home/TestData/nlp/token_classification_punctuation/fisher'), reason='Not a Jenkins machine' + ) + def test_punct_capit_with_distilbert(self): + data_dir = '/home/TestData/nlp/token_classification_punctuation/fisher' + model = models.PunctuationCapitalizationModel.from_pretrained("punctuation_en_distilbert") + metrics = get_metrics(data_dir, model) + + assert abs(metrics['punct_precision'] - 53.0826) < 0.001 + assert abs(metrics['punct_recall'] - 56.2905) < 0.001 + assert abs(metrics['punct_f1'] - 52.4225) < 0.001 + assert int(model.punct_class_report.total_examples) == 128 + + @pytest.mark.with_downloads() + @pytest.mark.unit + @pytest.mark.run_only_on('GPU') + @pytest.mark.skipif( + not data_exists('/home/TestData/nlp/token_classification_punctuation/gmb'), reason='Not a Jenkins machine' + ) + def test_ner_model(self): + data_dir = '/home/TestData/nlp/token_classification_punctuation/gmb' + model = models.TokenClassificationModel.from_pretrained("ner_en_bert") + metrics = get_metrics(data_dir, model) + + assert abs(metrics['precision'] - 96.0937) < 0.001 + assert abs(metrics['recall'] - 96.0146) < 0.001 + assert abs(metrics['f1'] - 95.6076) < 0.001 + assert int(model.classification_report.total_examples) == 202 diff --git a/models/tacotron2/NeMo/tests/collections/tts/test_tts_datasets.py b/models/tacotron2/NeMo/tests/collections/tts/test_tts_datasets.py new file mode 100644 index 0000000000..04d557ef4b --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/tts/test_tts_datasets.py @@ -0,0 +1,43 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest +import torch + +from nemo.collections.tts.data.datalayers import FastSpeech2Dataset + + +class TestTTSDatasets: + @pytest.mark.unit + def test_fs2_dataset(self, test_data_dir): + manifest_path = os.path.join(test_data_dir, 'tts/mini_ljspeech/manifest.json') + mappings_file = os.path.join(test_data_dir, 'tts/mini_ljspeech/mappings.json') + ignore_file = os.path.join(test_data_dir, 'tts/mini_ljspeech/wavs_to_ignore.pkl') + + # Test loading data (including supplementary data) with ignore file + ds = FastSpeech2Dataset( + manifest_filepath=manifest_path, + mappings_filepath=mappings_file, + sample_rate=22050, + ignore_file=ignore_file, + load_supplementary_values=True, + ) + + assert len(ds) == 4 + count = 0 + for _ in ds: + count += 1 + assert count == 4 diff --git a/models/tacotron2/NeMo/tests/collections/tts/test_tts_exportables.py b/models/tacotron2/NeMo/tests/collections/tts/test_tts_exportables.py new file mode 100644 index 0000000000..59075d9e8f --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/tts/test_tts_exportables.py @@ -0,0 +1,62 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import tempfile + +import onnx +import pytest +from omegaconf import DictConfig, ListConfig, OmegaConf + +from nemo.collections.tts.models import FastPitchModel, HifiGanModel, TalkNetSpectModel, WaveGlowModel + + +@pytest.fixture() +def fastpitch_model(): + test_root = os.path.dirname(os.path.abspath(__file__)) + conf = OmegaConf.load(os.path.join(test_root, '../../../examples/tts/conf/fastpitch.yaml')) + conf.train_dataset = conf.validation_datasets = '.' + conf.model.train_ds = conf.model.test_ds = conf.model.validation_ds = None + model = FastPitchModel(cfg=conf.model) + return model + + +@pytest.fixture() +def hifigan_model(): + test_root = os.path.dirname(os.path.abspath(__file__)) + model = HifiGanModel.from_pretrained(model_name="tts_hifigan") + return model + + +class TestExportable: + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_FastPitchModel_export_to_onnx(self, fastpitch_model): + model = fastpitch_model.cuda() + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, 'fp.onnx') + model.export(output=filename, verbose=True, check_trace=True) + + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_HifiGanModel_export_to_onnx(self, hifigan_model): + model = hifigan_model.cuda() + assert hifigan_model.generator is not None + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, 'hfg.pt') + model.export(output=filename, verbose=True, check_trace=True) + + +if __name__ == "__main__": + t = TestExportable() + t.test_FastPitchModel_export_to_onnx(fastpitch_model()) diff --git a/models/tacotron2/NeMo/tests/collections/tts/test_waveglow.py b/models/tacotron2/NeMo/tests/collections/tts/test_waveglow.py new file mode 100644 index 0000000000..7bfd3e5b0e --- /dev/null +++ b/models/tacotron2/NeMo/tests/collections/tts/test_waveglow.py @@ -0,0 +1,110 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +from unittest import TestCase + +import onnx +import pytest +import torch +from omegaconf import DictConfig + +from nemo.collections.tts.models import WaveGlowModel +from nemo.collections.tts.modules import WaveGlowModule +from nemo.core.classes import typecheck + +mcfg = DictConfig( + { + "_target_": "nemo.collections.tts.modules.waveglow.WaveGlowModule", + "n_flows": 12, + "n_group": 8, + "n_mel_channels": 80, + "n_early_every": 4, + "n_early_size": 2, + "n_wn_channels": 512, + "n_wn_layers": 8, + "wn_kernel_size": 3, + } +) + +pcfg = DictConfig( + { + "_target_": "nemo.collections.asr.parts.preprocessing.features.FilterbankFeatures", + "dither": 0.0, + "nfilt": 80, + "stft_conv": True, + } +) + +wcfg = DictConfig({"waveglow": mcfg, "sigma": 1.0, "preprocessor": pcfg,}) + + +def input_example(sz): + mel = torch.randn(1, 1, 80, sz).cuda().half() + z = torch.randn(1, 8, sz * 256 // 8, 1).cuda().half() + return ( + mel, + z, + ) + + +def taco2wg(spec, z): + spec = spec.permute(0, 3, 2, 1).contiguous() + return spec.view(spec.size(0), spec.size(1), -1), z.view(z.size(0), z.size(1), -1) + + +# Wrapper method to convert Jasper's Taco2 output to WG input and call inference +def forward_wrapper(self, spec, z=None): + spec, z = taco2wg(spec, z) + audio = self.waveglow.norm_dist_to_audio(spec=spec, sigma=1.0, z=z) + return audio + + +class TestWaveGlow: + @pytest.mark.run_only_on('GPU') + @pytest.mark.unit + def test_export_to_onnx(self): + model = WaveGlowModel(wcfg) + model = model.cuda().half() + typecheck.set_typecheck_enabled(enabled=False) + with tempfile.TemporaryDirectory() as tmpdir, model.nemo_infer(): + # Generate filename in the temporary directory. + # TODO: Change `waveglow.ts` to `waveglow.onnx` for > 21.05 + tmp_file_name = os.path.join("waveglow.ts") + + n_mels = 80 + # Test export. + inp = input_example(n_mels) + inp1 = taco2wg(*inp) + inp2 = inp1 + res1 = model.waveglow(*inp1) + res2 = model.waveglow(*inp2) + assert torch.allclose(res1, res2, rtol=0.01, atol=0.1) + WaveGlowModel.forward_for_export = forward_wrapper + model.export( + tmp_file_name, + verbose=True, + input_example=inp, + output_example=res1, + try_script=False, + check_trace=False, + do_constant_folding=True, + dynamic_axes={"spec": [0], "z": [0], "audio": [0]}, + ) + + +if __name__ == "__main__": + t = TestWaveGlow() + t.test_export_to_onnx() diff --git a/models/tacotron2/NeMo/tests/conftest.py b/models/tacotron2/NeMo/tests/conftest.py new file mode 100644 index 0000000000..4a947a2a04 --- /dev/null +++ b/models/tacotron2/NeMo/tests/conftest.py @@ -0,0 +1,188 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tarfile +import urllib.request +from os import mkdir +from os.path import dirname, exists, getsize, join +from pathlib import Path +from shutil import rmtree + +import pytest + +# Those variables probably should go to main NeMo configuration file (config.yaml). +__TEST_DATA_FILENAME = "test_data.tar.gz" +__TEST_DATA_URL = "https://github.com/NVIDIA/NeMo/releases/download/v1.0.0rc1/" +__TEST_DATA_SUBDIR = ".data" + + +def pytest_addoption(parser): + """ + Additional command-line arguments passed to pytest. + For now: + --cpu: use CPU during testing (DEFAULT: GPU) + --use_local_test_data: use local test data/skip downloading from URL/GitHub (DEFAULT: False) + """ + parser.addoption( + '--cpu', action='store_true', help="pass that argument to use CPU during testing (DEFAULT: False = GPU)" + ) + parser.addoption( + '--use_local_test_data', + action='store_true', + help="pass that argument to use local test data/skip downloading from URL/GitHub (DEFAULT: False)", + ) + parser.addoption( + '--with_downloads', + action='store_true', + help="pass this argument to active tests which download models from the cloud.", + ) + parser.addoption( + '--relax_numba_compat', + action='store_false', + help="numba compatibility checks will be relaxed to just availability of cuda, " + "without cuda compatibility matrix check", + ) + + +@pytest.fixture +def device(request): + """ Simple fixture returning string denoting the device [CPU | GPU] """ + if request.config.getoption("--cpu"): + return "CPU" + else: + return "GPU" + + +@pytest.fixture(autouse=True) +def run_only_on_device_fixture(request, device): + if request.node.get_closest_marker('run_only_on'): + if request.node.get_closest_marker('run_only_on').args[0] != device: + pytest.skip('skipped on this device: {}'.format(device)) + + +@pytest.fixture(autouse=True) +def downloads_weights(request, device): + if request.node.get_closest_marker('with_downloads'): + if not request.config.getoption("--with_downloads"): + pytest.skip( + 'To run this test, pass --with_downloads option. It will download (and cache) models from cloud.' + ) + + +@pytest.fixture(autouse=True) +def cleanup_local_folder(): + # Asserts in fixture are not recommended, but I'd rather stop users from deleting expensive training runs + assert not Path("./lightning_logs").exists() + assert not Path("./NeMo_experiments").exists() + assert not Path("./nemo_experiments").exists() + + yield + + if Path("./lightning_logs").exists(): + rmtree('./lightning_logs', ignore_errors=True) + if Path("./NeMo_experiments").exists(): + rmtree('./NeMo_experiments', ignore_errors=True) + if Path("./nemo_experiments").exists(): + rmtree('./nemo_experiments', ignore_errors=True) + + +@pytest.fixture +def test_data_dir(): + """ Fixture returns test_data_dir. """ + # Test dir. + test_data_dir_ = join(dirname(__file__), __TEST_DATA_SUBDIR) + return test_data_dir_ + + +def pytest_configure(config): + """ + Initial configuration of conftest. + The function checks if test_data.tar.gz is present in tests/.data. + If so, compares its size with github's test_data.tar.gz. + If file absent or sizes not equal, function downloads the archive from github and unpacks it. + """ + config.addinivalue_line( + "markers", "run_only_on(device): runs the test only on a given device [CPU | GPU]", + ) + # Test dir and archive filepath. + test_dir = join(dirname(__file__), __TEST_DATA_SUBDIR) + test_data_archive = join(dirname(__file__), __TEST_DATA_SUBDIR, __TEST_DATA_FILENAME) + + # Get size of local test_data archive. + try: + test_data_local_size = getsize(test_data_archive) + except: + # File does not exist. + test_data_local_size = -1 + + if config.option.use_local_test_data: + if test_data_local_size == -1: + pytest.exit("Test data `{}` is not present in the system".format(test_data_archive)) + else: + print( + "Using the local `{}` test archive ({}B) found in the `{}` folder.".format( + __TEST_DATA_FILENAME, test_data_local_size, test_dir + ) + ) + return + + # Get size of remote test_data archive. + try: + url = __TEST_DATA_URL + __TEST_DATA_FILENAME + u = urllib.request.urlopen(url) + except: + # Couldn't access remote archive. + if test_data_local_size == -1: + pytest.exit("Test data not present in the system and cannot access the '{}' URL".format(url)) + else: + print( + "Cannot access the '{}' URL, using the test data ({}B) found in the `{}` folder.".format( + url, test_data_local_size, test_dir + ) + ) + return + + # Get metadata. + meta = u.info() + test_data_remote_size = int(meta["Content-Length"]) + + # Compare sizes. + if test_data_local_size != test_data_remote_size: + print( + "Downloading the `{}` test archive from `{}`, please wait...".format(__TEST_DATA_FILENAME, __TEST_DATA_URL) + ) + # Remove .data folder. + if exists(test_dir): + rmtree(test_dir) + # Create one .data folder. + mkdir(test_dir) + # Download + urllib.request.urlretrieve(url, test_data_archive) + # Extract tar + print("Extracting the `{}` test archive, please wait...".format(test_data_archive)) + tar = tarfile.open(test_data_archive) + tar.extractall(path=test_dir) + tar.close() + else: + print( + "A valid `{}` test archive ({}B) found in the `{}` folder.".format( + __TEST_DATA_FILENAME, test_data_local_size, test_dir + ) + ) + + if config.option.relax_numba_compat is not None: + from nemo.core.utils import numba_utils + + print("Setting numba compat :", config.option.relax_numba_compat) + numba_utils.set_numba_compat_strictness(strict=config.option.relax_numba_compat) diff --git a/models/tacotron2/NeMo/tests/core/test_config_utils.py b/models/tacotron2/NeMo/tests/core/test_config_utils.py new file mode 100644 index 0000000000..4b0d8bb67a --- /dev/null +++ b/models/tacotron2/NeMo/tests/core/test_config_utils.py @@ -0,0 +1,125 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any + +import pytest +import pytorch_lightning as ptl + +from nemo.core.config.pytorch_lightning import TrainerConfig +from nemo.utils import config_utils + + +@pytest.fixture() +def cls(): + class DummyClass: + def __init__(self, a, b=5, c: int = 0, d: 'ABC' = None): + pass + + return DummyClass + + +class TestConfigUtils: + @pytest.mark.unit + def test_all_args_exist(self, cls): + @dataclass + class DummyDataClass: + a: int = -1 + b: int = 5 + c: int = 0 + d: Any = None + + result = config_utils.assert_dataclass_signature_match(cls, DummyDataClass) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None + + @pytest.mark.unit + def test_all_args_dont_exist(self, cls): + @dataclass + class DummyDataClass: + a: int = -1 + b: int = 5 + c: int = 0 + + result = config_utils.assert_dataclass_signature_match(cls, DummyDataClass) + signatures_match, cls_subset, dataclass_subset = result + + assert not signatures_match + assert len(cls_subset) > 0 + assert len(dataclass_subset) == 0 + + @pytest.mark.unit + def test_extra_args_exist(self, cls): + @dataclass + class DummyDataClass: + a: int = -1 + b: int = 5 + c: int = 0 + d: Any = None + e: float = 0.0 + + result = config_utils.assert_dataclass_signature_match(cls, DummyDataClass) + signatures_match, cls_subset, dataclass_subset = result + + assert not signatures_match + assert len(cls_subset) == 0 + assert len(dataclass_subset) > 0 + + @pytest.mark.unit + def test_extra_args_exist_but_is_ignored(self, cls): + @dataclass + class DummyDataClass: + a: int = -1 + b: int = 5 + c: int = 0 + d: Any = None + e: float = 0.0 # Assume ignored + + result = config_utils.assert_dataclass_signature_match(cls, DummyDataClass, ignore_args=['e']) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None + + @pytest.mark.unit + def test_args_exist_but_is_remapped(self, cls): + @dataclass + class DummyDataClass: + a: int = -1 + b: int = 5 + c: int = 0 + e: Any = None # Assume remapped + + result = config_utils.assert_dataclass_signature_match(cls, DummyDataClass, remap_args={'e': 'd'}) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None + + @pytest.mark.unit + def test_ptl_config(self): + PTL_DEPRECATED = ['distributed_backend', 'automatic_optimization'] + + result = config_utils.assert_dataclass_signature_match(ptl.Trainer, TrainerConfig, ignore_args=PTL_DEPRECATED) + signatures_match, cls_subset, dataclass_subset = result + + assert signatures_match + assert cls_subset is None + assert dataclass_subset is None diff --git a/models/tacotron2/NeMo/tests/core/test_exp_manager.py b/models/tacotron2/NeMo/tests/core/test_exp_manager.py new file mode 100644 index 0000000000..e1fa6da95c --- /dev/null +++ b/models/tacotron2/NeMo/tests/core/test_exp_manager.py @@ -0,0 +1,357 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +import re +from pathlib import Path + +import pytest +import pytorch_lightning as pl +import torch +from omegaconf import OmegaConf +from omegaconf.errors import OmegaConfBaseException + +from nemo.constants import NEMO_ENV_VARNAME_VERSION +from nemo.core.classes import ModelPT +from nemo.utils.exp_manager import ( + CheckpointMisconfigurationError, + LoggerMisconfigurationError, + NotFoundError, + exp_manager, +) + + +class MyTestOptimizer(torch.optim.Optimizer): + def __init__(self, params): + self._step = 0 + super().__init__(params, {}) + + def step(self, *args, **kwargs): + for group in self.param_groups: + for p in group['params']: + if self._step == 0: + p.data = 0.1 * torch.ones(p.shape) + elif self._step == 1: + p.data = 0.0 * torch.ones(p.shape) + else: + p.data = 0.01 * torch.ones(p.shape) + self._step += 1 + return None + + +class OnesDataset(torch.utils.data.Dataset): + def __init__(self, dataset_len): + super().__init__() + self.__dataset_len = dataset_len + + def __getitem__(self, *args): + return torch.ones(2) + + def __len__(self): + return self.__dataset_len + + +class ExampleModel(ModelPT): + def __init__(self, *args, **kwargs): + cfg = OmegaConf.structured({}) + super().__init__(cfg) + self.l1 = torch.nn.modules.Linear(in_features=2, out_features=1) + + def train_dataloader(self): + dataset = OnesDataset(2) + return torch.utils.data.DataLoader(dataset, batch_size=2) + + def val_dataloader(self): + dataset = OnesDataset(10) + return torch.utils.data.DataLoader(dataset, batch_size=2) + + def forward(self, batch): + output = self.l1(batch) + output = torch.nn.functional.l1_loss(output, torch.zeros(output.size()).to(output.device)) + return output + + def validation_step(self, batch, batch_idx): + return self(batch) + + def training_step(self, batch, batch_idx): + return self(batch) + + def configure_optimizers(self): + return MyTestOptimizer(self.parameters()) + # return torch.optim.Adam(self.parameters(), lr=0.1) + + def list_available_models(self): + pass + + def setup_training_data(self): + pass + + def setup_validation_data(self): + pass + + def validation_epoch_end(self, loss): + self.log("val_loss", torch.stack(loss).mean()) + + +class TestExpManager: + @pytest.mark.unit + def test_omegaconf(self): + """Ensure omegaconf raises an error when an unexcepted argument is passed""" + with pytest.raises(OmegaConfBaseException): + exp_manager(pl.Trainer(), {"unused": 1}) + + @pytest.mark.unit + def test_trainer_loggers(self, tmp_path): + """ Test that a trainer with logger errors out with a number of arguments. Test that it works with + create_tensorboard_logger set to False + """ + test_trainer = pl.Trainer() # Should create logger and modelcheckpoint + + with pytest.raises(LoggerMisconfigurationError): # Fails because exp_manager defaults to trainer + exp_manager(test_trainer, {"exp_dir": str(tmp_path)}) + with pytest.raises(LoggerMisconfigurationError): # Fails because exp_manager defaults to trainer + exp_manager(test_trainer, {"explicit_log_dir": str(tmp_path)}) + with pytest.raises(LoggerMisconfigurationError): # Fails because exp_manager defaults to trainer + exp_manager(test_trainer, {"resume_if_exists": True}) + + # Check that exp_manager uses trainer.logger, it's exp_dir, name, and version + log_dir = exp_manager(test_trainer, {"create_tensorboard_logger": False, "create_checkpoint_callback": False}) + assert log_dir.resolve() == Path("./lightning_logs/version_0").resolve() + assert Path("./lightning_logs").exists() + assert Path("./lightning_logs/version_0").exists() + + # Check that a trainer without a logger gets a logger attached to it + test_trainer = pl.Trainer(logger=False) + log_dir = exp_manager( + test_trainer, + {"create_tensorboard_logger": True, "create_checkpoint_callback": False, "exp_dir": str(tmp_path)}, + ) + assert isinstance(test_trainer.logger, pl.loggers.TensorBoardLogger) + + test_trainer = pl.Trainer(logger=False) + # Check that a create_wandb_logger=True errors out unless wandb_logger_kwargs is passed. + with pytest.raises(ValueError): + log_dir = exp_manager( + test_trainer, + { + "create_tensorboard_logger": False, + "create_checkpoint_callback": False, + "exp_dir": str(tmp_path), + "create_wandb_logger": True, + }, + ) + # Check that a WandbLogger is attached to logger if create_wandb_logger=True and wandb_logger_kwargs has name + # and project + log_dir = exp_manager( + test_trainer, + { + "create_tensorboard_logger": False, + "create_checkpoint_callback": False, + "exp_dir": str(tmp_path), + "create_wandb_logger": True, + "wandb_logger_kwargs": {"name": "", "project": ""}, + }, + ) + assert isinstance(test_trainer.logger, pl.loggers.WandbLogger) + + @pytest.mark.unit + def test_checkpoint_configurations(self): + """ Test that trainer creating modelcheckpoint and asking exp_manager to do it too results in errors, but + is error free if only one is asked to do so. + """ + disable_tb_logger = {"create_tensorboard_logger": False} + test_trainer = pl.Trainer() # Should create logger and modelcheckpoint + with pytest.raises(CheckpointMisconfigurationError): # Fails because both try to create modelcheckpoint + exp_manager(test_trainer, disable_tb_logger) + + # Should succeed without error + exp_manager(test_trainer, {"create_checkpoint_callback": False, "create_tensorboard_logger": False}) + + test_trainer_2 = pl.Trainer(checkpoint_callback=False) + exp_manager(test_trainer_2, disable_tb_logger) # Should succeed without error + + @pytest.mark.unit + def test_default_log_dir(self): + """Check the default of ./nemo_experiments/default/datetime works as intended""" + test_trainer = pl.Trainer(checkpoint_callback=False, logger=False) + + log_dir = exp_manager(test_trainer, {"create_tensorboard_logger": False, "create_checkpoint_callback": False}) + assert (log_dir / "..").resolve() == Path("./nemo_experiments/default/").resolve() + assert Path("./nemo_experiments").exists() + assert Path("./nemo_experiments/default/").exists() + sub_dirs = [x for x in Path("./nemo_experiments/default/").iterdir() if x.is_dir()] + assert len(sub_dirs) == 1 + assert re.match(r"[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}", sub_dirs[0].name) + + @pytest.mark.unit + def test_log_dir_overrides(self, monkeypatch, tmp_path): + """Check a variety of trainer options with exp_manager""" + # Checks that explicit_log_dir ignores exp_dir, name, and version + test_trainer = pl.Trainer(checkpoint_callback=False, logger=False) + log_dir = exp_manager(test_trainer, {"explicit_log_dir": str(tmp_path / "test_log_dir_overrides")}) + assert log_dir.resolve() == (tmp_path / "test_log_dir_overrides").resolve() + assert Path(tmp_path).exists() + assert Path(tmp_path / "test_log_dir_overrides").exists() + + # Checks that exp_manager uses exp_dir, default name, and explicit version + test_trainer = pl.Trainer(checkpoint_callback=False, logger=False) + log_dir = exp_manager(test_trainer, {"exp_dir": str(tmp_path / "test_no_name"), "version": 957}) + assert log_dir.resolve() == (tmp_path / "test_no_name" / "default" / "957").resolve() + assert Path(tmp_path).exists() + assert Path(tmp_path / "test_no_name" / "default" / "957").exists() + + monkeypatch.delenv(NEMO_ENV_VARNAME_VERSION) + # Checks that use_datetime_version False toggle works + test_trainer = pl.Trainer(checkpoint_callback=False, logger=False) + log_dir = exp_manager(test_trainer, {"exp_dir": str(tmp_path / "test_no_name"), "use_datetime_version": False}) + assert log_dir.resolve() == (tmp_path / "test_no_name" / "default" / "version_0").resolve() + assert Path(tmp_path).exists() + assert Path(tmp_path / "test_no_name" / "default" / "version_0").exists() + + monkeypatch.delenv(NEMO_ENV_VARNAME_VERSION) + # Checks that use_datetime_version False toggle works and version increments + test_trainer = pl.Trainer(checkpoint_callback=False, logger=False) + log_dir = exp_manager(test_trainer, {"exp_dir": str(tmp_path / "test_no_name"), "use_datetime_version": False}) + assert log_dir.resolve() == (tmp_path / "test_no_name" / "default" / "version_1").resolve() + assert Path(tmp_path).exists() + assert Path(tmp_path / "test_no_name" / "default" / "version_1").exists() + + @pytest.mark.unit + def test_resume(self, tmp_path): + """ Tests the resume capabilities of exp_manager""" + test_trainer = pl.Trainer(checkpoint_callback=False, logger=False) + + # Error because explicit_log_dir does not exist + with pytest.raises(NotFoundError): + exp_manager( + test_trainer, + { + "exp_dir": str(tmp_path / "test_resume"), + "resume_if_exists": True, + "explicit_log_dir": "Does_not_exist", + }, + ) + + # Error because checkpoints folder does not exist + with pytest.raises(NotFoundError): + exp_manager(test_trainer, {"resume_if_exists": True, "exp_dir": str(tmp_path / "test_resume")}) + + # No error because we tell exp_manager to ignore notfounderror + exp_manager( + test_trainer, + { + "resume_if_exists": True, + "exp_dir": str(tmp_path / "test_resume_2"), + "resume_ignore_no_checkpoint": True, + }, + ) + + test_trainer = pl.Trainer(checkpoint_callback=False, logger=False) + Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints").mkdir(parents=True) + # Error because checkpoints do not exist in folder + with pytest.raises(NotFoundError): + exp_manager( + test_trainer, + { + "resume_if_exists": True, + "explicit_log_dir": str(tmp_path / "test_resume" / "default" / "version_0"), + }, + ) + + Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel--end.ckpt").touch() + # Error because *end.ckpt is in folder indicating that training has already finished + with pytest.raises(ValueError): + exp_manager( + test_trainer, + { + "resume_if_exists": True, + "explicit_log_dir": str(tmp_path / "test_resume" / "default" / "version_0"), + }, + ) + + Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel--end.ckpt").unlink() + Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel--last.ckpt").touch() + Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel2--last.ckpt").touch() + # Error because multiple *last.ckpt is in folder. If more than one, don't know which to restore + with pytest.raises(ValueError): + exp_manager( + test_trainer, + { + "resume_if_exists": True, + "explicit_log_dir": str(tmp_path / "test_resume" / "default" / "version_0"), + }, + ) + + # Finally succeed + Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel2--last.ckpt").unlink() + log_dir = exp_manager( + test_trainer, + {"resume_if_exists": True, "explicit_log_dir": str(tmp_path / "test_resume" / "default" / "version_0")}, + ) + checkpoint = Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel--last.ckpt") + assert Path(test_trainer.resume_from_checkpoint).resolve() == checkpoint.resolve() + + # Succeed again and make sure that run_0 exists and previous log files were moved + test_trainer = pl.Trainer(checkpoint_callback=False, logger=False) + exp_manager(test_trainer, {"resume_if_exists": True, "explicit_log_dir": str(log_dir)}) + checkpoint = Path(tmp_path / "test_resume" / "default" / "version_0" / "checkpoints" / "mymodel--last.ckpt") + assert Path(test_trainer.resume_from_checkpoint).resolve() == checkpoint.resolve() + prev_run_dir = Path(tmp_path / "test_resume" / "default" / "version_0" / "run_0") + assert prev_run_dir.exists() + prev_log = Path(tmp_path / "test_resume" / "default" / "version_0" / "run_0" / "lightning_logs.txt") + assert prev_log.exists() + + @pytest.mark.unit + def test_nemo_checkpoint_save_best_model_1(self, tmp_path): + test_trainer = pl.Trainer(checkpoint_callback=False, logger=False, max_epochs=4) + log_dir = exp_manager( + test_trainer, + {"checkpoint_callback_params": {"save_best_model": True}, "explicit_log_dir": str(tmp_path / "test")}, + ) + model = ExampleModel() + test_trainer.fit(model) + + assert Path(str(tmp_path / "test" / "checkpoints" / "default.nemo")).exists() + + model = ExampleModel.restore_from(str(tmp_path / "test" / "checkpoints" / "default.nemo")) + assert float(model(torch.tensor([1.0, 1.0], device=model.device))) == 0.0 + + @pytest.mark.unit + def test_nemo_checkpoint_save_best_model_2(self, tmp_path): + test_trainer = pl.Trainer(checkpoint_callback=False, logger=False, max_epochs=4) + log_dir = exp_manager(test_trainer, {"explicit_log_dir": str(tmp_path / "test")},) + model = ExampleModel() + test_trainer.fit(model) + + assert Path(str(tmp_path / "test" / "checkpoints" / "default.nemo")).exists() + + model = ExampleModel.restore_from(str(tmp_path / "test" / "checkpoints" / "default.nemo")) + assert math.fabs(float(model(torch.tensor([1.0, 1.0], device=model.device))) - 0.03) < 1e-5 + + @pytest.mark.unit + def test_nemo_checkpoint_always_save_nemo(self, tmp_path): + test_trainer = pl.Trainer(checkpoint_callback=False, logger=False, max_epochs=4) + log_dir = exp_manager( + test_trainer, + { + "checkpoint_callback_params": {"save_best_model": True, "always_save_nemo": True}, + "explicit_log_dir": str(tmp_path / "test"), + }, + ) + model = ExampleModel() + test_trainer.fit(model) + + assert Path(str(tmp_path / "test" / "checkpoints" / "default.nemo")).exists() + + model = ExampleModel.restore_from(str(tmp_path / "test" / "checkpoints" / "default.nemo")) + assert float(model(torch.tensor([1.0, 1.0], device=model.device))) == 0.0 diff --git a/models/tacotron2/NeMo/tests/core/test_fileio.py b/models/tacotron2/NeMo/tests/core/test_fileio.py new file mode 100644 index 0000000000..f737b23216 --- /dev/null +++ b/models/tacotron2/NeMo/tests/core/test_fileio.py @@ -0,0 +1,299 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile + +import numpy as np +import pytest +import torch +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.asr.models import EncDecCTCModel + +try: + from eff.cookbooks import NeMoCookbook + + _EFF_PRESENT_ = True +except ImportError: + _EFF_PRESENT_ = False + +# A decorator marking the EFF requirement. +requires_eff = pytest.mark.skipif(not _EFF_PRESENT_, reason="Export File Format library required to run test") + + +@pytest.fixture() +def asr_model(): + preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} + encoder = { + 'cls': 'nemo.collections.asr.modules.ConvASREncoder', + 'params': { + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 1024, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], + }, + } + + decoder = { + 'cls': 'nemo.collections.asr.modules.ConvASRDecoder', + 'params': { + 'feat_in': 1024, + 'num_classes': 28, + 'vocabulary': [ + ' ', + 'a', + 'b', + 'c', + 'd', + 'e', + 'f', + 'g', + 'h', + 'i', + 'j', + 'k', + 'l', + 'm', + 'n', + 'o', + 'p', + 'q', + 'r', + 's', + 't', + 'u', + 'v', + 'w', + 'x', + 'y', + 'z', + "'", + ], + }, + } + modelConfig = DictConfig( + {'preprocessor': DictConfig(preprocessor), 'encoder': DictConfig(encoder), 'decoder': DictConfig(decoder)} + ) + + model_instance = EncDecCTCModel(cfg=modelConfig) + return model_instance + + +class TestFileIO: + @pytest.mark.unit + def test_to_from_config_file(self, asr_model): + """" Test makes sure that the second instance created with the same configuration (BUT NOT checkpoint) + has different weights. """ + + with tempfile.NamedTemporaryFile() as fp: + yaml_filename = fp.name + asr_model.to_config_file(path2yaml_file=yaml_filename) + next_instance = EncDecCTCModel.from_config_file(path2yaml_file=yaml_filename) + + assert isinstance(next_instance, EncDecCTCModel) + + assert len(next_instance.decoder.vocabulary) == 28 + assert asr_model.num_weights == next_instance.num_weights + + w1 = asr_model.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + w2 = next_instance.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + + assert not np.array_equal(w1, w2) + + @pytest.mark.unit + def test_save_restore_from_nemo_file(self, asr_model): + """" Test makes sure that the second instance created from the same configuration AND checkpoint + has the same weights. """ + + with tempfile.NamedTemporaryFile() as fp: + filename = fp.name + + # Save model (with random artifact). + with tempfile.NamedTemporaryFile() as artifact: + asr_model.register_artifact(config_path="abc", src=artifact.name) + asr_model.save_to(save_path=filename) + + # Restore the model. + asr_model2 = EncDecCTCModel.restore_from(restore_path=filename) + + assert len(asr_model.decoder.vocabulary) == len(asr_model2.decoder.vocabulary) + assert asr_model.num_weights == asr_model2.num_weights + + w1 = asr_model.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + w2 = asr_model2.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + + assert np.array_equal(w1, w2) + + @requires_eff + @pytest.mark.unit + def test_eff_save_restore_from_nemo_file_encrypted(self, asr_model): + """" Test makes sure that after encrypted save-restore the model has the same weights. """ + + with tempfile.NamedTemporaryFile() as fp: + filename = fp.name + + # Set key - use checkpoint encryption. + NeMoCookbook.set_encryption_key("test_key") + + # Save model (with random artifact). + with tempfile.NamedTemporaryFile() as artifact: + asr_model.register_artifact(config_path="abc", src=artifact.name) + asr_model.save_to(save_path=filename) + + # Try to restore the encrypted archive (weights) without the encryption key. + NeMoCookbook.set_encryption_key(None) + with pytest.raises(PermissionError): + # Restore the model. + asr_model2 = EncDecCTCModel.restore_from(restore_path=filename) + + # Restore the model. + NeMoCookbook.set_encryption_key("test_key") + asr_model3 = EncDecCTCModel.restore_from(restore_path=filename) + # Reset encryption so it won't mess up with other save/restore. + NeMoCookbook.set_encryption_key(None) + + assert asr_model.num_weights == asr_model3.num_weights + + @pytest.mark.unit + def test_save_restore_from_nemo_file_with_override(self, asr_model, tmpdir): + """" Test makes sure that the second instance created from the same configuration AND checkpoint + has the same weights. + + Args: + tmpdir: fixture providing a temporary directory unique to the test invocation. + """ + # Name of the archive in tmp folder. + filename = os.path.join(tmpdir, "eff.nemo") + + # Get path where the command is executed - the artifacts will be "retrieved" there. + # (original .nemo behavior) + cwd = os.getcwd() + + with tempfile.NamedTemporaryFile(mode='a+') as conf_fp: + + # Create a "random artifact". + with tempfile.NamedTemporaryFile(mode="w", delete=False) as artifact: + artifact.write("magic content 42") + # Remember the filename of the artifact. + _, artifact_filename = os.path.split(artifact.name) + # Add artifact to model. + asr_model.register_artifact(config_path="abc", src=artifact.name) + # Save model (with "random artifact"). + asr_model.save_to(save_path=filename) + + # Modify config slightly + cfg = asr_model.cfg + cfg.encoder.activation = 'swish' + yaml_cfg = OmegaConf.to_yaml(cfg) + conf_fp.write(yaml_cfg) + conf_fp.seek(0) + + # Restore the model. + asr_model2 = EncDecCTCModel.restore_from(restore_path=filename, override_config_path=conf_fp.name) + + assert len(asr_model.decoder.vocabulary) == len(asr_model2.decoder.vocabulary) + assert asr_model.num_weights == asr_model2.num_weights + + w1 = asr_model.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + w2 = asr_model2.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + + assert np.array_equal(w1, w2) + + assert asr_model2.cfg.encoder.activation == 'swish' + + @pytest.mark.unit + def test_save_model_level_pt_ckpt(self, asr_model): + with tempfile.TemporaryDirectory() as ckpt_dir: + nemo_file = os.path.join(ckpt_dir, 'asr.nemo') + asr_model.save_to(nemo_file) + + # Save model level PT checkpoint + asr_model.extract_state_dict_from(nemo_file, ckpt_dir) + ckpt_path = os.path.join(ckpt_dir, 'model_weights.ckpt') + + assert os.path.exists(ckpt_path) + + # Restore the model. + asr_model2 = EncDecCTCModel.restore_from(restore_path=nemo_file) + + assert len(asr_model.decoder.vocabulary) == len(asr_model2.decoder.vocabulary) + assert asr_model.num_weights == asr_model2.num_weights + + # Change weights values + asr_model2.encoder.encoder[0].mconv[0].conv.weight.data += 1.0 + + w1 = asr_model.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + w2 = asr_model2.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + + assert not np.array_equal(w1, w2) + + # Restore from checkpoint + asr_model2.load_state_dict(torch.load(ckpt_path)) + + w1 = asr_model.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + w2 = asr_model2.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + + assert np.array_equal(w1, w2) + + @pytest.mark.unit + def test_save_module_level_pt_ckpt(self, asr_model): + with tempfile.TemporaryDirectory() as ckpt_dir: + nemo_file = os.path.join(ckpt_dir, 'asr.nemo') + asr_model.save_to(nemo_file) + + # Save model level PT checkpoint + asr_model.extract_state_dict_from(nemo_file, ckpt_dir, split_by_module=True) + encoder_path = os.path.join(ckpt_dir, 'encoder.ckpt') + decoder_path = os.path.join(ckpt_dir, 'decoder.ckpt') + preprocessor_path = os.path.join(ckpt_dir, 'preprocessor.ckpt') + + assert os.path.exists(encoder_path) + assert os.path.exists(decoder_path) + assert os.path.exists(preprocessor_path) + + # Restore the model. + asr_model2 = EncDecCTCModel.restore_from(restore_path=nemo_file) + + assert len(asr_model.decoder.vocabulary) == len(asr_model2.decoder.vocabulary) + assert asr_model.num_weights == asr_model2.num_weights + + # Change weights values + asr_model2.encoder.encoder[0].mconv[0].conv.weight.data += 1.0 + + w1 = asr_model.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + w2 = asr_model2.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + + assert not np.array_equal(w1, w2) + + # Restore from checkpoint + asr_model2.encoder.load_state_dict(torch.load(encoder_path)) + + w1 = asr_model.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + w2 = asr_model2.encoder.encoder[0].mconv[0].conv.weight.data.detach().cpu().numpy() + + assert np.array_equal(w1, w2) diff --git a/models/tacotron2/NeMo/tests/core/test_neural_types.py b/models/tacotron2/NeMo/tests/core/test_neural_types.py new file mode 100644 index 0000000000..e40ddd93b9 --- /dev/null +++ b/models/tacotron2/NeMo/tests/core/test_neural_types.py @@ -0,0 +1,224 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest + +from nemo.core.neural_types import ( + AcousticEncodedRepresentation, + AudioSignal, + AxisKind, + AxisKindAbstract, + AxisType, + ChannelType, + ElementType, + MelSpectrogramType, + MFCCSpectrogramType, + NeuralType, + NeuralTypeComparisonResult, + SpectrogramType, + VoidType, +) + + +class TestNeuralTypeSystem: + @pytest.mark.unit + def test_short_vs_long_version(self): + long_version = NeuralType( + axes=(AxisType(AxisKind.Batch, None), AxisType(AxisKind.Dimension, None), AxisType(AxisKind.Time, None)), + elements_type=AcousticEncodedRepresentation(), + ) + short_version = NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()) + assert long_version.compare(short_version) == NeuralTypeComparisonResult.SAME + assert short_version.compare(long_version) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_parameterized_type_audio_sampling_frequency(self): + audio16K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(16000)) + audio8K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(8000)) + another16K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(16000)) + + assert audio8K.compare(audio16K) == NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS + assert audio16K.compare(audio8K) == NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS + assert another16K.compare(audio16K) == NeuralTypeComparisonResult.SAME + assert audio16K.compare(another16K) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_transpose_same_1(self): + type1 = NeuralType(axes=('B', 'T', 'C')) + type2 = NeuralType(axes=('T', 'B', 'C')) + + assert type1.compare(type2) == NeuralTypeComparisonResult.TRANSPOSE_SAME + assert type2.compare(type1) == NeuralTypeComparisonResult.TRANSPOSE_SAME + + @pytest.mark.unit + def test_transpose_same_2(self): + audio16K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(16000)) + audio16K_t = NeuralType(axes=('T', 'B'), elements_type=AudioSignal(16000)) + + assert audio16K.compare(audio16K_t) == NeuralTypeComparisonResult.TRANSPOSE_SAME + + @pytest.mark.unit + def test_inheritance_spec_augment_example(self): + input = NeuralType(('B', 'D', 'T'), SpectrogramType()) + out1 = NeuralType(('B', 'D', 'T'), MelSpectrogramType()) + out2 = NeuralType(('B', 'D', 'T'), MFCCSpectrogramType()) + + assert out1.compare(out2) == NeuralTypeComparisonResult.INCOMPATIBLE + assert out2.compare(out1) == NeuralTypeComparisonResult.INCOMPATIBLE + assert input.compare(out1) == NeuralTypeComparisonResult.GREATER + assert input.compare(out2) == NeuralTypeComparisonResult.GREATER + assert out1.compare(input) == NeuralTypeComparisonResult.LESS + assert out2.compare(input) == NeuralTypeComparisonResult.LESS + + @pytest.mark.unit + def test_singletone(self): + loss_output1 = NeuralType(axes=None) + loss_output2 = NeuralType(axes=None) + + assert loss_output1.compare(loss_output2) == NeuralTypeComparisonResult.SAME + assert loss_output2.compare(loss_output1) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_list_of_lists(self): + T1 = NeuralType( + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=True), + AxisType(kind=AxisKind.Time, size=None, is_list=True), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + elements_type=ChannelType(), + ) + T2 = NeuralType( + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=False), + AxisType(kind=AxisKind.Time, size=None, is_list=False), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + elements_type=ChannelType(), + ) + # TODO: should this be incompatible instead??? + assert T1.compare(T2), NeuralTypeComparisonResult.TRANSPOSE_SAME + + @pytest.mark.unit + def test_void(self): + btc_spctr = NeuralType(('B', 'T', 'C'), SpectrogramType()) + btc_spct_bad = NeuralType(('B', 'T'), SpectrogramType()) + btc_void = NeuralType(('B', 'T', 'C'), VoidType()) + + assert btc_void.compare(btc_spctr) == NeuralTypeComparisonResult.SAME + assert btc_spctr.compare(btc_void) == NeuralTypeComparisonResult.INCOMPATIBLE + assert btc_void.compare(btc_spct_bad) == NeuralTypeComparisonResult.INCOMPATIBLE + + @pytest.mark.unit + def test_big_void(self): + big_void_1 = NeuralType(elements_type=VoidType()) + big_void_2 = NeuralType() + + btc_spctr = NeuralType(('B', 'T', 'C'), SpectrogramType()) + btc_spct_bad = NeuralType(('B', 'T'), SpectrogramType()) + t1 = NeuralType( + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=True), + AxisType(kind=AxisKind.Time, size=None, is_list=True), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + elements_type=ChannelType(), + ) + t2 = NeuralType( + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=False), + AxisType(kind=AxisKind.Time, size=None, is_list=False), + AxisType(kind=AxisKind.Dimension, size=32, is_list=False), + AxisType(kind=AxisKind.Dimension, size=128, is_list=False), + AxisType(kind=AxisKind.Dimension, size=256, is_list=False), + ), + elements_type=ChannelType(), + ) + + assert big_void_1.compare(btc_spctr) == NeuralTypeComparisonResult.SAME + assert big_void_1.compare(btc_spct_bad) == NeuralTypeComparisonResult.SAME + assert big_void_1.compare(t1) == NeuralTypeComparisonResult.SAME + assert big_void_1.compare(t2) == NeuralTypeComparisonResult.SAME + + assert big_void_2.compare(btc_spctr) == NeuralTypeComparisonResult.SAME + assert big_void_2.compare(btc_spct_bad) == NeuralTypeComparisonResult.SAME + assert big_void_2.compare(t1) == NeuralTypeComparisonResult.SAME + assert big_void_2.compare(t2) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_unspecified_dimensions(self): + t0 = NeuralType( + (AxisType(AxisKind.Batch, 64), AxisType(AxisKind.Time, 10), AxisType(AxisKind.Dimension, 128)), + SpectrogramType(), + ) + t1 = NeuralType(('B', 'T', 'C'), SpectrogramType()) + + assert t1.compare(t0), NeuralTypeComparisonResult.SAME + assert t0.compare(t1), NeuralTypeComparisonResult.DIM_INCOMPATIBLE + + @pytest.mark.unit + def test_any_axis(self): + t0 = NeuralType(('B', 'Any', 'Any'), VoidType()) + t1 = NeuralType(('B', 'Any', 'Any'), SpectrogramType()) + t2 = NeuralType(('B', 'T', 'C'), SpectrogramType()) + + assert t0.compare(t1) == NeuralTypeComparisonResult.SAME + assert t0.compare(t2) == NeuralTypeComparisonResult.SAME + assert t1.compare(t2) == NeuralTypeComparisonResult.SAME + assert t2.compare(t1) == NeuralTypeComparisonResult.INCOMPATIBLE + assert t1.compare(t0) == NeuralTypeComparisonResult.INCOMPATIBLE + + @pytest.mark.unit + def test_struct(self): + class BoundingBox(ElementType): + def __str__(self): + return "bounding box from detection model" + + def fields(self): + return ("X", "Y", "W", "H") + + # ALSO ADD new, user-defined, axis kind + class AxisKind2(AxisKindAbstract): + Image = 0 + + T1 = NeuralType( + elements_type=BoundingBox(), + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=True), + AxisType(kind=AxisKind2.Image, size=None, is_list=True), + ), + ) + + class BadBoundingBox(ElementType): + def __str__(self): + return "bad bounding box from detection model" + + def fields(self): + return ("X", "Y", "H") + + T2 = NeuralType( + elements_type=BadBoundingBox(), + axes=( + AxisType(kind=AxisKind.Batch, size=None, is_list=True), + AxisType(kind=AxisKind2.Image, size=None, is_list=True), + ), + ) + assert T2.compare(T1) == NeuralTypeComparisonResult.INCOMPATIBLE diff --git a/models/tacotron2/NeMo/tests/core/test_optimizers_schedulers.py b/models/tacotron2/NeMo/tests/core/test_optimizers_schedulers.py new file mode 100644 index 0000000000..84f7d14a7c --- /dev/null +++ b/models/tacotron2/NeMo/tests/core/test_optimizers_schedulers.py @@ -0,0 +1,809 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import random + +import omegaconf +import pytest +import pytorch_lightning as pl +import torch +import torch.optim + +from nemo.core import config, optim +from nemo.core.optim.lr_scheduler import AVAILABLE_SCHEDULERS +from nemo.core.optim.optimizers import AVAILABLE_OPTIMIZERS +from nemo.utils import logging + + +class TempModel(torch.nn.Module): + def __init__(self): + super(TempModel, self).__init__() + self.layer = torch.nn.Linear(5, 1) + + def forward(self, x): + x = self.layer(x) + return x + + +class OptCounter(torch.optim.SGD): + def __init__(self, *args, **kwargs): + self.count = 0 + super().__init__(*args, **kwargs) + + def step(self, closure=None): + try: + self.count += 1 + except AttributeError: + self.count = 1 + super().step(closure) + + +class RandomDataset(torch.utils.data.Dataset): + def __init__(self, dataset_len): + super().__init__() + self.__dataset_len = dataset_len + + def __getitem__(self, *args): + return torch.randn(2) + + def __len__(self): + return self.__dataset_len + + +class ExampleModel(pl.LightningModule): + def __init__(self, batch_size, dataset_len, drop_last, max_steps): + super().__init__() + self.l1 = torch.nn.modules.Linear(in_features=2, out_features=1) + self.batch_size = batch_size + self.dataset_len = dataset_len + self.drop_last = drop_last + self.max_steps = max_steps + + def train_dataloader(self): + dataset = RandomDataset(self.dataset_len) + return torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, drop_last=self.drop_last) + + def training_step(self, batch, batch_idx): + output = self.l1(batch) + output = torch.nn.functional.l1_loss(output, torch.ones(output.size()).to(output.device)) + return {"loss": output} + + def configure_optimizers(self): + self.my_opt = OptCounter(self.parameters(), lr=0.02) + return self.my_opt + + +class Callback(pl.callbacks.Callback): + @pl.utilities.distributed.rank_zero_only + def on_train_end(self, trainer, module): + if trainer.global_step != module.my_opt.count or trainer.global_step != module.max_steps: + logging.debug(f"max_epochs: {trainer.max_epochs}") + logging.debug(f"accumulate_grad_batches: {trainer.accumulate_grad_batches}") + logging.debug(f"limit_train_batches: {trainer.limit_train_batches}") + logging.debug(f"num_processes: {trainer.num_processes}") + logging.debug(f"batch_size: {module.batch_size}") + logging.debug(f"dataset_len: {module.dataset_len}") + logging.debug(f"drop_last: {module.drop_last}") + logging.debug(f"{len(trainer.train_dataloader)}") + logging.debug(f"{trainer.num_training_batches }") + assert ( + trainer.global_step == module.my_opt.count + ), f"{trainer.global_step} != {module.my_opt.count} != {module.max_steps}" + assert ( + trainer.global_step == module.max_steps + ), f"{trainer.global_step} != {module.my_opt.count} != {module.max_steps}" + + +class TestOptimizersSchedulers: + INITIAL_LR = 0.1 + MIN_LR = 1e-3 + MAX_STEPS = 10 + + @pytest.mark.unit + def test_get_optimizer(self): + model = TempModel() + + for opt_name in AVAILABLE_OPTIMIZERS.keys(): + opt_cls = optim.get_optimizer(opt_name) + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + assert isinstance(opt, AVAILABLE_OPTIMIZERS[opt_name]) + + @pytest.mark.unit + def test_register_optimizer(self): + class TempOpt(torch.optim.SGD): + pass + + class TempOptParams(config.optimizers.SGDParams): + pass + + optim.register_optimizer('TempOpt', TempOpt, TempOptParams) + + model = TempModel() + opt_cls = optim.get_optimizer('TempOpt') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + assert isinstance(opt, TempOpt) + + @pytest.mark.unit + def test_optim_config_parse_bypass(self): + basic_optim_config = {'weight_decay': 0.001, 'betas': [0.8, 0.5]} + parsed_params = optim.parse_optimizer_args('novograd', basic_optim_config) + assert parsed_params['weight_decay'] == basic_optim_config['weight_decay'] + assert parsed_params['betas'][0] == basic_optim_config['betas'][0] + assert parsed_params['betas'][1] == basic_optim_config['betas'][1] + + dict_config = omegaconf.OmegaConf.create(basic_optim_config) + parsed_params = optim.parse_optimizer_args('novograd', dict_config) + assert parsed_params['weight_decay'] == dict_config['weight_decay'] + assert parsed_params['betas'][0] == dict_config['betas'][0] + assert parsed_params['betas'][1] == dict_config['betas'][1] + + @pytest.mark.unit + def test_optim_config_parse_arg_by_name(self): + basic_optim_config = {'name': 'auto', 'weight_decay': 0.001, 'betas': [0.8, 0.5]} + parsed_params = optim.parse_optimizer_args('novograd', basic_optim_config) + assert parsed_params['weight_decay'] == basic_optim_config['weight_decay'] + assert parsed_params['betas'][0] == basic_optim_config['betas'][0] + assert parsed_params['betas'][1] == basic_optim_config['betas'][1] + + dict_config = omegaconf.OmegaConf.create(basic_optim_config) + parsed_params = optim.parse_optimizer_args('novograd', dict_config) + assert parsed_params['weight_decay'] == dict_config['weight_decay'] + assert parsed_params['betas'][0] == dict_config['betas'][0] + assert parsed_params['betas'][1] == dict_config['betas'][1] + + with pytest.raises(omegaconf.errors.ConfigKeyError): + optim.parse_optimizer_args('sgd', dict_config) + + @pytest.mark.unit + def test_optim_config_parse_arg_by_target(self): + basic_optim_config = { + '_target_': 'nemo.core.config.NovogradParams', + 'params': {'weight_decay': 0.001, 'betas': [0.8, 0.5]}, + } + basic_optim_config = omegaconf.OmegaConf.create(basic_optim_config) + parsed_params = optim.parse_optimizer_args('novograd', basic_optim_config) + assert parsed_params['weight_decay'] == basic_optim_config['params']['weight_decay'] + assert parsed_params['betas'][0] == basic_optim_config['params']['betas'][0] + assert parsed_params['betas'][1] == basic_optim_config['params']['betas'][1] + + dict_config = omegaconf.OmegaConf.create(basic_optim_config) + parsed_params = optim.parse_optimizer_args('novograd', dict_config) + assert parsed_params['weight_decay'] == dict_config['params']['weight_decay'] + assert parsed_params['betas'][0] == dict_config['params']['betas'][0] + assert parsed_params['betas'][1] == dict_config['params']['betas'][1] + + # Names are ignored when passing class path + # This will be captured during optimizer instantiation + output_config = optim.parse_optimizer_args('sgd', dict_config) + sgd_config = vars(config.SGDParams()) + novograd_config = vars(config.NovogradParams()) + + assert set(output_config.keys()) != set(sgd_config.keys()) + assert set(output_config.keys()) == set(novograd_config) + + @pytest.mark.unit + def test_get_scheduler(self): + model = TempModel() + optimizer = optim.Novograd(model.parameters(), lr=self.INITIAL_LR) + + for sched_name in AVAILABLE_SCHEDULERS.keys(): + sched_cls = optim.lr_scheduler.get_scheduler(sched_name) + + try: + sched = sched_cls(optimizer) + assert isinstance(sched, AVAILABLE_SCHEDULERS[sched_name]) + continue + except Exception: + pass + + try: + sched = sched_cls(optimizer, max_steps=self.MAX_STEPS) + assert isinstance(sched, AVAILABLE_SCHEDULERS[sched_name]) + continue + except Exception: + pass + + @pytest.mark.unit + def test_register_scheduler(self): + class TempSched(optim.lr_scheduler.CosineAnnealing): + pass + + class TempSchedParams(config.schedulers.CosineAnnealingParams): + pass + + optim.lr_scheduler.register_scheduler('TempSched', TempSched, TempSchedParams) + + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + sched_cls = optim.lr_scheduler.get_scheduler('TempSched') + sched = sched_cls(opt, max_steps=self.MAX_STEPS) + + assert isinstance(sched, TempSched) + + @pytest.mark.unit + def test_sched_config_parse_simple(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + basic_sched_config = {'name': 'CosineAnnealing', 'max_steps': 10} + scheduler_setup = optim.lr_scheduler.prepare_lr_scheduler(opt, basic_sched_config) + assert isinstance(scheduler_setup['scheduler'], optim.lr_scheduler.CosineAnnealing) + + dict_config = omegaconf.OmegaConf.create(basic_sched_config) + scheduler_setup = optim.lr_scheduler.prepare_lr_scheduler(opt, dict_config) + assert isinstance(scheduler_setup['scheduler'], optim.lr_scheduler.CosineAnnealing) + + @pytest.mark.unit + def test_sched_config_parse_from_cls(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + basic_sched_config = { + '_target_': 'nemo.core.config.CosineAnnealingParams', + 'params': {'min_lr': 0.1}, + 'max_steps': self.MAX_STEPS, + } + scheduler_setup = optim.lr_scheduler.prepare_lr_scheduler(opt, basic_sched_config) + assert isinstance(scheduler_setup['scheduler'], optim.lr_scheduler.CosineAnnealing) + + dict_config = omegaconf.OmegaConf.create(basic_sched_config) + scheduler_setup = optim.lr_scheduler.prepare_lr_scheduler(opt, dict_config) + assert isinstance(scheduler_setup['scheduler'], optim.lr_scheduler.CosineAnnealing) + + @pytest.mark.unit + def test_WarmupPolicy(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + # No warmup case + policy = optim.lr_scheduler.WarmupPolicy(opt, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr == self.INITIAL_LR + + for i in range(self.MAX_STEPS): + assert policy.get_last_lr()[0] == self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup steps available + policy = optim.lr_scheduler.WarmupPolicy(opt, warmup_steps=5, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 4: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + else: + assert policy.get_last_lr()[0] == self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + @pytest.mark.unit + def test_WarmupHoldPolicy(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + # No warmup case + policy = optim.lr_scheduler.WarmupHoldPolicy(opt, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr == self.INITIAL_LR + + for i in range(self.MAX_STEPS): + assert policy.get_last_lr()[0] == self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup steps available + policy = optim.lr_scheduler.WarmupHoldPolicy(opt, warmup_steps=5, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 4: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + else: + assert policy.get_last_lr()[0] == self.INITIAL_LR + + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup + Hold steps available + policy = optim.lr_scheduler.WarmupHoldPolicy( + opt, warmup_steps=5, hold_steps=3, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR + ) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 4: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + else: + assert policy.get_last_lr()[0] == self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + @pytest.mark.unit + def test_WarmupAnnealing(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + # No warmup case + policy = optim.lr_scheduler.WarmupAnnealing(opt, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr == self.INITIAL_LR + + for i in range(self.MAX_STEPS): + assert policy.get_last_lr()[0] <= self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup steps available + policy = optim.lr_scheduler.WarmupAnnealing(opt, warmup_steps=5, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 5: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + else: + assert policy.get_last_lr()[0] < self.INITIAL_LR + + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup + Hold steps available + policy = optim.lr_scheduler.WarmupHoldPolicy( + opt, warmup_steps=5, hold_steps=3, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR + ) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 4: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + else: + assert policy.get_last_lr()[0] == self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + @pytest.mark.unit + def test_SquareAnnealing(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + # No warmup case + policy = optim.lr_scheduler.SquareAnnealing(opt, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr == self.INITIAL_LR + + for i in range(self.MAX_STEPS): + assert policy.get_last_lr()[0] <= self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup steps available + policy = optim.lr_scheduler.SquareAnnealing(opt, warmup_steps=5, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 5: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + else: + assert policy.get_last_lr()[0] < self.INITIAL_LR + + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + @pytest.mark.unit + def test_SquareRootAnnealing(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + # No warmup case + policy = optim.lr_scheduler.SquareRootAnnealing(opt, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr == self.INITIAL_LR + + for i in range(self.MAX_STEPS): + assert policy.get_last_lr()[0] <= self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup steps available + policy = optim.lr_scheduler.SquareRootAnnealing( + opt, warmup_steps=5, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR + ) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 5: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + else: + assert policy.get_last_lr()[0] < self.INITIAL_LR + + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + @pytest.mark.unit + def test_CosineAnnealing(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + # No warmup case + policy = optim.lr_scheduler.CosineAnnealing(opt, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr == self.INITIAL_LR + + for i in range(self.MAX_STEPS): + assert policy.get_last_lr()[0] <= self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup steps available + policy = optim.lr_scheduler.CosineAnnealing(opt, warmup_steps=5, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 5: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + else: + assert policy.get_last_lr()[0] < self.INITIAL_LR + + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + @pytest.mark.unit + def test_PolynomialDecayAnnealing(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + # No warmup case + policy = optim.lr_scheduler.PolynomialDecayAnnealing( + opt, power=2, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR + ) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr == self.INITIAL_LR + + for i in range(self.MAX_STEPS): + assert policy.get_last_lr()[0] <= self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup steps available + policy = optim.lr_scheduler.PolynomialDecayAnnealing( + opt, warmup_steps=5, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR + ) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 5: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + else: + assert policy.get_last_lr()[0] < self.INITIAL_LR + + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + @pytest.mark.unit + def test_PolynomialHoldDecayAnnealing(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + # No warmup case + policy = optim.lr_scheduler.PolynomialHoldDecayAnnealing( + opt, power=2, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR + ) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr == self.INITIAL_LR + + for i in range(self.MAX_STEPS): + assert policy.get_last_lr()[0] <= self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup steps available + policy = optim.lr_scheduler.PolynomialHoldDecayAnnealing( + opt, power=2, warmup_steps=5, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR + ) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 5: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + else: + assert policy.get_last_lr()[0] < self.INITIAL_LR + + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup + Hold steps available + policy = optim.lr_scheduler.PolynomialHoldDecayAnnealing( + opt, warmup_steps=5, hold_steps=3, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR, power=2 + ) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 4: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + elif i <= 8: + assert policy.get_last_lr()[0] == self.INITIAL_LR + else: + assert policy.get_last_lr()[0] < self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + @pytest.mark.unit + def test_InverseSquareRootAnnealing(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + # No warmup case + policy = optim.lr_scheduler.InverseSquareRootAnnealing(opt, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr == self.INITIAL_LR + + for i in range(self.MAX_STEPS): + assert policy.get_last_lr()[0] <= self.INITIAL_LR + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + # Warmup steps available + policy = optim.lr_scheduler.InverseSquareRootAnnealing( + opt, warmup_steps=5, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR + ) + initial_lr = policy.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS): + if i <= 5: + assert policy.get_last_lr()[0] <= self.INITIAL_LR + else: + assert policy.get_last_lr()[0] < self.INITIAL_LR + + opt.step() + policy.step() + + policy.step() + final_lr = policy.get_last_lr()[0] + + assert final_lr == self.MIN_LR + + @pytest.mark.unit + @pytest.mark.run_only_on('CPU') + def test_max_step_computation(self): + def train( + max_epochs, accumulate_grad_batches, limit_train_batches, num_processes, batch_size, dataset_len, drop_last + ): + trainer = pl.Trainer( + max_epochs=max_epochs, + accelerator="ddp_cpu", + num_processes=num_processes, + accumulate_grad_batches=accumulate_grad_batches, + limit_train_batches=limit_train_batches, + checkpoint_callback=False, + progress_bar_refresh_rate=0, + weights_summary=None, + ) + max_steps = optim.lr_scheduler.compute_max_steps( + max_epochs, + accumulate_grad_batches, + limit_train_batches, + num_processes, + dataset_len, + batch_size, + drop_last, + ) + model = ExampleModel(batch_size, dataset_len, drop_last, max_steps) + trainer.callbacks.append(Callback()) + trainer.fit(model) + + # This test will break once we and lightning upgrade to pytorch 1.7.0 due to a bug fix in pytorch 1.7.0 + train( + 31, + accumulate_grad_batches=1, + limit_train_batches=1.0, + num_processes=9, + batch_size=60, + dataset_len=1613, + drop_last=True, + ) + train( + 5, + accumulate_grad_batches=1, + limit_train_batches=0.17382691901706027, + num_processes=4, + batch_size=97, + dataset_len=498, + drop_last=False, + ) + train( + 5, + accumulate_grad_batches=8, + limit_train_batches=0.1663306588594945, + num_processes=4, + batch_size=54, + dataset_len=629, + drop_last=True, + ) + train( + 5, + accumulate_grad_batches=1, + limit_train_batches=0.2121376533631948, + num_processes=1, + batch_size=68, + dataset_len=488, + drop_last=False, + ) + for _ in range(5): + drop_last = bool(random.randint(0, 1)) + accumulate_grad_batches = random.randint(1, 10) + + limit_train_batches_int = random.randint(1, 10) + limit_train_batches_float = random.uniform(0, 1) + limit_train_batches = random.choice([limit_train_batches_int, limit_train_batches_float]) + max_epochs = random.randint(4, 20) + num_processes = random.randint(1, 5) + dataset_len = random.randint(20, num_processes * 500) + batch_size = random.randint(math.ceil(5.0 / num_processes), min(dataset_len // num_processes, 128)) + train( + max_epochs, + accumulate_grad_batches, + limit_train_batches, + num_processes, + batch_size, + dataset_len, + drop_last, + ) diff --git a/models/tacotron2/NeMo/tests/core/test_save_restore.py b/models/tacotron2/NeMo/tests/core/test_save_restore.py new file mode 100644 index 0000000000..4c59fa0cdb --- /dev/null +++ b/models/tacotron2/NeMo/tests/core/test_save_restore.py @@ -0,0 +1,443 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import filecmp +import os +import shutil +import tempfile +from typing import Dict, Optional, Set, Union + +import pytest +import torch +from omegaconf import DictConfig, OmegaConf, open_dict + +from nemo.collections.asr.models import EncDecCTCModel, EncDecCTCModelBPE +from nemo.collections.nlp.models import PunctuationCapitalizationModel, TransformerLMModel +from nemo.core.classes import ModelPT +from nemo.utils.app_state import AppState + + +def getattr2(object, attr): + if not '.' in attr: + return getattr(object, attr) + else: + arr = attr.split('.') + return getattr2(getattr(object, arr[0]), '.'.join(arr[1:])) + + +class MockModel(ModelPT): + def __init__(self, cfg, trainer=None): + super(MockModel, self).__init__(cfg=cfg, trainer=trainer) + self.w = torch.nn.Linear(10, 1) + # mock temp file + if 'temp_file' in self.cfg and self.cfg.temp_file is not None: + self.temp_file = self.register_artifact('temp_file', self.cfg.temp_file) + with open(self.temp_file, 'r') as f: + self.temp_data = f.readlines() + else: + self.temp_file = None + self.temp_data = None + + def forward(self, x): + y = self.w(x) + return y, self.cfg.temp_file + + def setup_training_data(self, train_data_config: Union[DictConfig, Dict]): + self._train_dl = None + + def setup_validation_data(self, val_data_config: Union[DictConfig, Dict]): + self._validation_dl = None + + def setup_test_data(self, test_data_config: Union[DictConfig, Dict]): + self._test_dl = None + + def list_available_models(cls): + return [] + + +def _mock_model_config(): + conf = {'temp_file': None} + conf = OmegaConf.create({'model': conf}) + OmegaConf.set_struct(conf, True) + return conf + + +class TestSaveRestore: + def __test_restore_elsewhere( + self, + model: ModelPT, + attr_for_eq_check: Set[str] = None, + override_config_path: Optional[Union[str, DictConfig]] = None, + map_location: Optional[torch.device] = None, + strict: bool = False, + return_config: bool = False, + ): + """Test's logic: + 1. Save model into temporary folder (save_folder) + 2. Copy .nemo file from save_folder to restore_folder + 3. Delete save_folder + 4. Attempt to restore from .nemo file in restore_folder and compare to original instance + """ + # Create a new temporary directory + with tempfile.TemporaryDirectory() as restore_folder: + with tempfile.TemporaryDirectory() as save_folder: + save_folder_path = save_folder + # Where model will be saved + model_save_path = os.path.join(save_folder, f"{model.__class__.__name__}.nemo") + model.save_to(save_path=model_save_path) + # Where model will be restored from + model_restore_path = os.path.join(restore_folder, f"{model.__class__.__name__}.nemo") + shutil.copy(model_save_path, model_restore_path) + # at this point save_folder should not exist + assert save_folder_path is not None and not os.path.exists(save_folder_path) + assert not os.path.exists(model_save_path) + assert os.path.exists(model_restore_path) + # attempt to restore + model_copy = model.__class__.restore_from( + restore_path=model_restore_path, + map_location=map_location, + strict=strict, + return_config=return_config, + override_config_path=override_config_path, + ) + + if return_config: + return model_copy + + assert model.num_weights == model_copy.num_weights + if attr_for_eq_check is not None and len(attr_for_eq_check) > 0: + for attr in attr_for_eq_check: + assert getattr2(model, attr) == getattr2(model_copy, attr) + + return model_copy + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_EncDecCTCModel(self): + # TODO: Switch to using named configs because here we don't really care about weights + qn = EncDecCTCModel.from_pretrained(model_name="QuartzNet15x5Base-En") + self.__test_restore_elsewhere(model=qn, attr_for_eq_check=set(["decoder._feat_in", "decoder._num_classes"])) + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_EncDecCTCModelBPE(self): + # TODO: Switch to using named configs because here we don't really care about weights + cn = EncDecCTCModelBPE.from_pretrained(model_name="stt_en_citrinet_256") + self.__test_restore_elsewhere(model=cn, attr_for_eq_check=set(["decoder._feat_in", "decoder._num_classes"])) + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_EncDecCTCModelBPE_v2(self): + # TODO: Switch to using named configs because here we don't really care about weights + cn = EncDecCTCModelBPE.from_pretrained(model_name="stt_en_conformer_ctc_small") + self.__test_restore_elsewhere(model=cn, attr_for_eq_check=set(["decoder._feat_in", "decoder._num_classes"])) + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_PunctuationCapitalization(self): + # TODO: Switch to using named configs because here we don't really care about weights + pn = PunctuationCapitalizationModel.from_pretrained(model_name='punctuation_en_distilbert') + self.__test_restore_elsewhere( + model=pn, attr_for_eq_check=set(["punct_classifier.log_softmax", "punct_classifier.log_softmax"]) + ) + + @pytest.mark.unit + def test_mock_save_to_restore_from(self): + with tempfile.NamedTemporaryFile('w') as empty_file: + # Write some data + empty_file.writelines(["*****\n"]) + empty_file.flush() + + # Update config + cfg = _mock_model_config() + cfg.model.temp_file = empty_file.name + + # Create model + model = MockModel(cfg=cfg.model, trainer=None) + model = model.to('cpu') + + assert model.temp_file == empty_file.name + + # Save test + model_copy = self.__test_restore_elsewhere(model, map_location='cpu') + + # Restore test + diff = model.w.weight - model_copy.w.weight + # because of caching - cache gets prepended + assert os.path.basename(model_copy.temp_file).endswith(os.path.basename(model.temp_file)) + assert diff.mean() <= 1e-9 + # assert os.path.basename(model.temp_file) == model_copy.temp_file + assert model_copy.temp_data == ["*****\n"] + + @pytest.mark.unit + def test_mock_restore_from_config_only(self): + with tempfile.NamedTemporaryFile('w') as empty_file: + # Write some data + empty_file.writelines(["*****\n"]) + empty_file.flush() + + # Update config + cfg = _mock_model_config() + cfg.model.temp_file = os.path.abspath(empty_file.name) + + # Inject arbitrary config arguments (after creating model) + with open_dict(cfg.model): + cfg.model.xyz = "abc" + + # Create model + model = MockModel(cfg=cfg.model, trainer=None) + model = model.to('cpu') + + assert model.temp_file == empty_file.name + model_copy = self.__test_restore_elsewhere(model, map_location='cpu', return_config=False) + # because of caching - cache gets prepended + assert os.path.basename(model_copy.temp_file).endswith(os.path.basename(model.temp_file)) + # assert filecmp.cmp(model.temp_file, model_copy._cfg.temp_file) + assert model.cfg.xyz == model_copy.cfg.xyz + + @pytest.mark.unit + def test_mock_restore_from_config_override_with_OmegaConf(self): + with tempfile.NamedTemporaryFile('w') as empty_file: + # Write some data + empty_file.writelines(["*****\n"]) + empty_file.flush() + + # Update config + cfg = _mock_model_config() + cfg.model.temp_file = empty_file.name + + # Create model + model = MockModel(cfg=cfg.model, trainer=None) + model = model.to('cpu') + + assert model.temp_file == empty_file.name + + # Inject arbitrary config arguments (after creating model) + with open_dict(cfg.model): + cfg.model.xyz = "abc" + + # Save test (with overriden config as OmegaConf object) + model_copy = self.__test_restore_elsewhere(model, map_location='cpu', override_config_path=cfg) + + # Restore test + diff = model.w.weight - model_copy.w.weight + assert diff.mean() <= 1e-9 + assert model_copy.temp_data == ["*****\n"] + + # Test that new config has arbitrary content + assert model_copy.cfg.xyz == "abc" + + @pytest.mark.unit + def test_mock_restore_from_config_override_with_yaml(self): + with tempfile.NamedTemporaryFile('w') as empty_file, tempfile.NamedTemporaryFile('w') as config_file: + # Write some data + empty_file.writelines(["*****\n"]) + empty_file.flush() + + # Update config + cfg = _mock_model_config() + cfg.model.temp_file = empty_file.name + + # Create model + model = MockModel(cfg=cfg.model, trainer=None) + model = model.to('cpu') + + assert model.temp_file == empty_file.name + + # Inject arbitrary config arguments (after creating model) + with open_dict(cfg.model): + cfg.model.xyz = "abc" + + # Write new config into file + OmegaConf.save(cfg, config_file) + + # Save test (with overriden config as OmegaConf object) + model_copy = self.__test_restore_elsewhere( + model, map_location='cpu', override_config_path=config_file.name + ) + + # Restore test + diff = model.w.weight - model_copy.w.weight + assert diff.mean() <= 1e-9 + assert filecmp.cmp(model.temp_file, model_copy.temp_file) + assert model_copy.temp_data == ["*****\n"] + + # Test that new config has arbitrary content + assert model_copy.cfg.xyz == "abc" + + @pytest.mark.unit + def test_mock_save_to_restore_from_with_target_class(self): + with tempfile.NamedTemporaryFile('w') as empty_file: + # Write some data + empty_file.writelines(["*****\n"]) + empty_file.flush() + + # Update config + cfg = _mock_model_config() + cfg.model.temp_file = empty_file.name + + # Create model + model = MockModel(cfg=cfg.model, trainer=None) + model = model.to('cpu') # type: MockModel + + assert model.temp_file == empty_file.name + + # Save file using MockModel + with tempfile.TemporaryDirectory() as save_folder: + save_path = os.path.join(save_folder, "temp.nemo") + model.save_to(save_path) + + # Restore test (using ModelPT as restorer) + # This forces the target class = MockModel to be used as resolver + model_copy = ModelPT.restore_from(save_path, map_location='cpu') + # because of caching - cache gets prepended + assert os.path.basename(model_copy.temp_file).endswith(os.path.basename(model.temp_file)) + # assert filecmp.cmp(model.temp_file, model_copy.temp_file) + # Restore test + diff = model.w.weight - model_copy.w.weight + assert diff.mean() <= 1e-9 + assert isinstance(model_copy, MockModel) + assert model_copy.temp_data == ["*****\n"] + + @pytest.mark.unit + def test_mock_save_to_restore_from_multiple_models(self): + with tempfile.NamedTemporaryFile('w') as empty_file, tempfile.NamedTemporaryFile('w') as empty_file2: + # Write some data + empty_file.writelines(["*****\n"]) + empty_file.flush() + empty_file2.writelines(["+++++\n"]) + empty_file2.flush() + + # Update config + create ,pde;s + cfg = _mock_model_config() + cfg.model.temp_file = empty_file.name + cfg2 = _mock_model_config() + cfg2.model.temp_file = empty_file2.name + + # Create models + model = MockModel(cfg=cfg.model, trainer=None) + model = model.to('cpu') + model2 = MockModel(cfg=cfg2.model, trainer=None) + model2 = model2.to('cpu') + + assert model.temp_file == empty_file.name + assert model2.temp_file == empty_file2.name + + # Save test + model_copy = self.__test_restore_elsewhere(model, map_location='cpu') + model2_copy = self.__test_restore_elsewhere(model2, map_location='cpu') + + # Restore test + assert model_copy.temp_data == ["*****\n"] + assert model2_copy.temp_data == ["+++++\n"] + + @pytest.mark.unit + def test_mock_save_to_restore_from_multiple_models_inverted_order(self): + with tempfile.NamedTemporaryFile('w') as empty_file, tempfile.NamedTemporaryFile('w') as empty_file2: + # Write some data + empty_file.writelines(["*****\n"]) + empty_file.flush() + empty_file2.writelines(["+++++\n"]) + empty_file2.flush() + + # Update config + create ,pde;s + cfg = _mock_model_config() + cfg.model.temp_file = empty_file.name + cfg2 = _mock_model_config() + cfg2.model.temp_file = empty_file2.name + + # Create models + model = MockModel(cfg=cfg.model, trainer=None) + model = model.to('cpu') + model2 = MockModel(cfg=cfg2.model, trainer=None) + model2 = model2.to('cpu') + + assert model.temp_file == empty_file.name + assert model2.temp_file == empty_file2.name + + # Save test (inverted order) + model2_copy = self.__test_restore_elsewhere(model2, map_location='cpu') + model_copy = self.__test_restore_elsewhere(model, map_location='cpu') + + # Restore test + assert model_copy.temp_data == ["*****\n"] + assert model2_copy.temp_data == ["+++++\n"] + + @pytest.mark.unit + def test_mock_save_to_restore_chained(self): + with tempfile.NamedTemporaryFile('w') as empty_file, tempfile.NamedTemporaryFile('w') as empty_file2: + # Write some data + empty_file.writelines(["*****\n"]) + empty_file.flush() + + # Update config + create ,pde;s + cfg = _mock_model_config() + cfg.model.temp_file = empty_file.name + + # Create models + model = MockModel(cfg=cfg.model, trainer=None) + model = model.to('cpu') + + assert model.temp_file == empty_file.name + + def save_copy(model, save_folder, restore_folder): + # Where model will be saved + model_save_path = os.path.join(save_folder, f"{model.__class__.__name__}.nemo") + model.save_to(save_path=model_save_path) + # Where model will be restored from + model_restore_path = os.path.join(restore_folder, f"{model.__class__.__name__}.nemo") + shutil.copy(model_save_path, model_restore_path) + return model_restore_path + + # Save test + with tempfile.TemporaryDirectory() as level4: + with tempfile.TemporaryDirectory() as level3: + with tempfile.TemporaryDirectory() as level2: + with tempfile.TemporaryDirectory() as level1: + path = save_copy(model, level1, level2) + model_copy2 = model.__class__.restore_from(path) + path = save_copy(model_copy2, level2, level3) + model_copy3 = model.__class__.restore_from(path) + path = save_copy(model_copy3, level3, level4) + model_copy = model.__class__.restore_from(path) + + # Restore test + assert model_copy.temp_data == ["*****\n"] + + # AppState test + appstate = AppState() + metadata = appstate.get_model_metadata_from_guid(model_copy.model_guid) + assert metadata.guid != model.model_guid + assert metadata.restoration_path == path + + @pytest.mark.unit + def test_mock_save_to_multiple_times(self): + with tempfile.NamedTemporaryFile('w') as empty_file, tempfile.TemporaryDirectory() as tmpdir: + # Write some data + empty_file.writelines(["*****\n"]) + empty_file.flush() + + # Update config + cfg = _mock_model_config() + cfg.model.temp_file = empty_file.name + + # Create model + model = MockModel(cfg=cfg.model, trainer=None) # type: MockModel + model = model.to('cpu') + + assert model.temp_file == empty_file.name + + # Save test + model.save_to(os.path.join(tmpdir, 'save_0.nemo')) + model.save_to(os.path.join(tmpdir, 'save_1.nemo')) + model.save_to(os.path.join(tmpdir, 'save_2.nemo')) diff --git a/models/tacotron2/NeMo/tests/core/test_serialization.py b/models/tacotron2/NeMo/tests/core/test_serialization.py new file mode 100644 index 0000000000..42b1e22b71 --- /dev/null +++ b/models/tacotron2/NeMo/tests/core/test_serialization.py @@ -0,0 +1,165 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from omegaconf import DictConfig + +from nemo.collections.asr.models import EncDecCTCModel +from nemo.collections.asr.modules import SpectrogramAugmentation +from nemo.core.classes.common import Serialization + + +def get_class_path(cls): + return f"{cls.__module__}.{cls.__name__}" + + +class MockSerializationImpl(Serialization): + def __init__(self, cfg: DictConfig): + self.cfg = cfg + self.value = self.__class__.__name__ + + +class MockSerializationImplV2(MockSerializationImpl): + pass + + +class TestSerialization: + @pytest.mark.unit + def test_from_config_dict_with_cls(self): + """Here we test that instantiation works for configs with cls class path in them. + Note that just Serialization.from_config_dict can be used to create an object""" + config = DictConfig( + { + 'cls': 'nemo.collections.asr.modules.SpectrogramAugmentation', + 'params': {'rect_freq': 50, 'rect_masks': 5, 'rect_time': 120,}, + } + ) + obj = Serialization.from_config_dict(config=config) + assert isinstance(obj, SpectrogramAugmentation) + + @pytest.mark.unit + def test_from_config_dict_without_cls(self): + """Here we test that instantiation works for configs without cls class path in them. + IMPORTANT: in this case, correct class type should call from_config_dict. This should work for Models.""" + preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} + encoder = { + 'cls': 'nemo.collections.asr.modules.ConvASREncoder', + 'params': { + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 1024, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], + }, + } + + decoder = { + 'cls': 'nemo.collections.asr.modules.ConvASRDecoder', + 'params': { + 'feat_in': 1024, + 'num_classes': 28, + 'vocabulary': [ + ' ', + 'a', + 'b', + 'c', + 'd', + 'e', + 'f', + 'g', + 'h', + 'i', + 'j', + 'k', + 'l', + 'm', + 'n', + 'o', + 'p', + 'q', + 'r', + 's', + 't', + 'u', + 'v', + 'w', + 'x', + 'y', + 'z', + "'", + ], + }, + } + modelConfig = DictConfig( + {'preprocessor': DictConfig(preprocessor), 'encoder': DictConfig(encoder), 'decoder': DictConfig(decoder)} + ) + obj = EncDecCTCModel.from_config_dict(config=modelConfig) + assert isinstance(obj, EncDecCTCModel) + + @pytest.mark.unit + def test_config_updated(self): + config = DictConfig( + { + 'cls': 'nemo.collections.asr.modules.SpectrogramAugmentation', + 'params': {'rect_freq': 50, 'rect_masks': 5, 'rect_time': 120,}, + } + ) + obj = Serialization.from_config_dict(config=config) + new_config = obj.to_config_dict() + assert config != new_config + assert 'params' not in new_config + assert 'cls' not in new_config + assert '_target_' in new_config + + @pytest.mark.unit + def test_base_class_instantiation(self): + # Target class is V2 impl, calling class is Serialization (base class) + config = DictConfig({'target': get_class_path(MockSerializationImplV2)}) + obj = Serialization.from_config_dict(config=config) + new_config = obj.to_config_dict() + assert config == new_config + assert isinstance(obj, MockSerializationImplV2) + assert obj.value == "MockSerializationImplV2" + + @pytest.mark.unit + def test_self_class_instantiation(self): + # Target class is V1 impl, calling class is V1 (same class) + config = DictConfig({'target': get_class_path(MockSerializationImpl)}) + obj = MockSerializationImpl.from_config_dict(config=config) # Serialization is base class + new_config = obj.to_config_dict() + assert config == new_config + assert isinstance(obj, MockSerializationImpl) + assert obj.value == "MockSerializationImpl" + + @pytest.mark.unit + def test_sub_class_instantiation(self): + # Target class is V1 impl, calling class is V2 (sub class) + config = DictConfig({'target': get_class_path(MockSerializationImpl)}) + obj = MockSerializationImplV2.from_config_dict(config=config) # Serialization is base class + new_config = obj.to_config_dict() + assert config == new_config + assert isinstance(obj, MockSerializationImplV2) + assert obj.value == "MockSerializationImplV2" diff --git a/models/tacotron2/NeMo/tests/core/test_typecheck.py b/models/tacotron2/NeMo/tests/core/test_typecheck.py new file mode 100644 index 0000000000..c547ab334a --- /dev/null +++ b/models/tacotron2/NeMo/tests/core/test_typecheck.py @@ -0,0 +1,1075 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import torch + +from nemo.core import Typing, typecheck +from nemo.core.neural_types import * + + +# Perform recursive shape assert +def recursive_assert_shape(x, shape): + if isinstance(x, list) or isinstance(x, tuple): + for xi in x: + recursive_assert_shape(xi, shape) + return + + assert x.shape == shape + + +# Perform recursive type assert +def recursive_assert_homogeneous_type(x, type_val): + if isinstance(x, list) or isinstance(x, tuple): + for xi in x: + recursive_assert_homogeneous_type(xi, type_val) + return + + assert x.neural_type.compare(type_val) == NeuralTypeComparisonResult.SAME + + +class TestNeuralTypeCheckSystem: + @pytest.mark.unit + def test_no_types_passthrough(self): + class NoTypes(Typing): + @typecheck() + def __call__(self, x): + return torch.tensor(1.0) + + obj = NoTypes() + result = obj(torch.tensor(1.0)) + + assert result == torch.tensor(1.0) + assert not hasattr(result, 'neural_type') + + @pytest.mark.unit + def test_input_output_types(self): + class InputOutputTypes(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType())} + + @property + def output_types(self): + return {"y": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x): + x += 1 + return x + + obj = InputOutputTypes() + result = obj(x=torch.zeros(10)) + + assert result.sum() == torch.tensor(10.0) + assert result.neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + + # Test passing wrong key for input + with pytest.raises(TypeError): + _ = obj(a=torch.zeros(10)) + + # Test using positional args + with pytest.raises(TypeError): + _ = obj(torch.zeros(10)) + + @pytest.mark.unit + def test_input_types_only(self): + class InputTypes(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x): + x += 1 + return x + + obj = InputTypes() + result = obj(x=torch.zeros(10)) + + assert result.sum() == torch.tensor(10.0) + assert hasattr(result, 'neural_type') is False + + @pytest.mark.unit + def test_multiple_input_types_only(self): + class InputTypes(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType()), "y": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x, y): + x += y + return x + + obj = InputTypes() + result = obj(x=torch.zeros(10), y=torch.ones(10)) + + assert result.sum() == torch.tensor(10.0) + assert hasattr(result, 'neural_type') is False + + @pytest.mark.unit + def test_output_types_only(self): + class OutputTypes(Typing): + @property + def output_types(self): + return {"y": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x): + x += 1 + return x + + obj = OutputTypes() + result = obj(x=torch.zeros(10)) + + assert result.sum() == torch.tensor(10.0) + assert result.neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + + # Test passing positional args + # Positional args allowed if input types is not set ! + result = obj(torch.zeros(10)) + assert result.sum() == torch.tensor(10.0) + + @pytest.mark.unit + def test_multiple_output_types_only(self): + class MultipleOutputTypes(Typing): + @property + def output_types(self): + return {"y": NeuralType(('B',), ElementType()), "z": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x): + y = x + 1 + z = x + 2 + return y, z + + obj = MultipleOutputTypes() + result_y, result_z = obj(x=torch.zeros(10)) + + assert result_y.sum() == torch.tensor(10.0) + assert result_y.neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + + assert result_z.sum() == torch.tensor(20.0) + assert result_z.neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_multiple_mixed_output_types_only(self): + class MultipleMixedOutputTypes(Typing): + @property + def output_types(self): + return {"y": NeuralType(('B',), ElementType()), "z": [NeuralType(('B',), ElementType())]} + + @typecheck() + def __call__(self, x): + y = x + 1 + z = x + 2 + return y, [z, z] + + obj = MultipleMixedOutputTypes() + result_y, result_z = obj(x=torch.zeros(10)) + + assert result_y.sum() == torch.tensor(10.0) + assert result_y.neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + + assert result_z[0].sum() == torch.tensor(20.0) + assert result_z[0].neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + + assert result_z[1].sum() == torch.tensor(20.0) + assert result_z[1].neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_multiple_mixed_output_types_only_mismatched(self): + class MultipleMixedOutputTypes(Typing): + @property + def output_types(self): + return {"y": NeuralType(('B',), ElementType()), "z": [NeuralType(('B',), ElementType())]} + + @typecheck() + def __call__(self, x): + # Use list of y, single z, contrary to signature + y = x + 1 + z = x + 2 + return [y, y], z + + obj = MultipleMixedOutputTypes() + with pytest.raises(TypeError): + result_y, result_z = obj(x=torch.zeros(10)) + + @pytest.mark.unit + def test_incorrect_inheritance(self): + class IncorrectInheritance(object): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType())} + + @property + def output_types(self): + return {"y": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x): + x += 1 + return x + + obj = IncorrectInheritance() + + with pytest.raises(RuntimeError): + _ = obj(x=torch.zeros(10)) + + @pytest.mark.unit + def test_port_definition_rejection(self): + class InputPortDefinitionRejection(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType())} + + @property + def output_types(self): + return {"w": NeuralType(('B',), ElementType()), "u": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x, y): + x += 1 + y -= 1 + return x, y + + # Test input port mismatch + obj = InputPortDefinitionRejection() + + with pytest.raises(TypeError): + _ = obj(x=torch.zeros(10), y=torch.zeros(10)) + + class OutputPortDefinitionRejection(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType())} + + @property + def output_types(self): + return { + "w": NeuralType(('B',), ElementType()), + } + + @typecheck() + def __call__(self, x): + return x + 1, x - 1 + + obj = OutputPortDefinitionRejection() + + with pytest.raises(TypeError): + _ = obj(x=torch.zeros(10)) + + @pytest.mark.unit + def test_port_shape_rejection(self): + class InputPortShapeRejection(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B', 'T'), ElementType())} # expect rank 2 matrix + + @property + def output_types(self): + return {"w": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x): + x += 1 + return x + + # Test input port mismatch + obj = InputPortShapeRejection() + + with pytest.raises(TypeError): + _ = obj(x=torch.zeros(10)) + + class OutputPortShapeRejection(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType())} + + @property + def output_types(self): + return { + "w": NeuralType(('B', 'T', 'D'), ElementType()), # expect rank 3 matrix + } + + @typecheck() + def __call__(self, x): + return x + 1 + + obj = OutputPortShapeRejection() + + with pytest.raises(TypeError): + _ = obj(x=torch.zeros(10)) + + @pytest.mark.unit + def test_positional_args(self): + # Test positional check on input type + class InputPositional(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x): + x += 1 + return x + + obj = InputPositional() + + with pytest.raises(TypeError): + _ = obj(torch.zeros(10)) + + # Test positional pass-through for only output ports defined + # NOTE: This is required behaviour to support type checking of NeMo Dataset class + # during collate_fn() call. + class OutputPositionalPassthrough(Typing): + @property + def output_types(self): + return {"y": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x): + x += 1 + return x + + obj = OutputPositionalPassthrough() + result = obj(torch.zeros(10)) + + assert result.sum() == torch.tensor(10.0) + + @pytest.mark.unit + def test_optional_types(self): + class InputOptionalTypes(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType()), "y": NeuralType(('B',), ElementType(), optional=True)} + + @typecheck() + def __call__(self, x, y=None): + if y is None: + x += 1 + else: + x += y + return x + + obj = InputOptionalTypes() + result = obj(x=torch.zeros(10)) + + assert result.sum() == torch.tensor(10.0) + assert hasattr(result, 'neural_type') is False + + result2 = obj(x=torch.zeros(10), y=torch.full([10], fill_value=5, dtype=torch.int32)) + + assert result2.sum() == torch.tensor(10 * 5) + assert hasattr(result, 'neural_type') is False + + @pytest.mark.unit + def test_input_output_neural_types(self): + class NodeA(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType())} + + @property + def output_types(self): + return {"y": NeuralType(('B', 'D'), LogitsType())} + + @typecheck() + def __call__(self, x): + y = torch.randn(x.shape[0], 4) + return y + + class NodeB(Typing): + @property + def input_types(self): + return {"w": NeuralType(('B', 'D'), LogitsType())} + + @property + def output_types(self): + return {"u": NeuralType(('B',), LabelsType())} + + @typecheck() + def __call__(self, w): + _, u = w.max(-1) + return u + + nodeA = NodeA() + nodeB = NodeB() + + outA = nodeA(x=torch.zeros(10)) + outB = nodeB(w=outA) + + assert outB.shape == torch.Size([10]) + assert outB.neural_type.compare(NeuralType(('B',), LabelsType())) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_nested_input_output_neural_types(self): + class NestedNodeA(Typing): + @property + def input_types(self): + return {"x": [[NeuralType(('B',), ElementType())]]} + + @property + def output_types(self): + return { + "y0": [NeuralType(('B', 'D'), LogitsType())], + "y1": [NeuralType(('B', 'D'), LogitsType())], + } + + @typecheck(ignore_collections=False) + def __call__(self, x): + # input x = [[x1, x2], [x3]] + x0 = x[0][0] + y = torch.randn(x0.shape[0], 4) + # Output is same as + # 1) return ([y, y], [y]) + # 2) return [y, y], [y] + return [[y, y], [y]] + + # Non-homogeneous output types + class NestedNodeB(Typing): + @property + def input_types(self): + return {"w": [[NeuralType(('B', 'D'), LogitsType())]]} + + @property + def output_types(self): + return { + "u0": [NeuralType(('B',), LogprobsType())], # check non homogeneous type + "u1": [NeuralType(('B',), LabelsType())], + } + + @typecheck(ignore_collections=False) + def __call__(self, w): + # input x = [[x1, x2], [x3]] + _, u00 = w[0][0].max(-1) + _, u01 = w[0][1].max(-1) + _, u10 = w[1][0].max(-1) + # Output is same as + # 1) return ([u00, u01], [u10]) + # 2) return [u00, u01], [u10] + return [[u00, u01], [u10]] + + nodeA = NestedNodeA() + nodeB = NestedNodeB() + + input_nest = [[torch.zeros(10), torch.zeros(10)], [torch.zeros(10)]] + outA = nodeA(x=input_nest) + outB = nodeB(w=outA) + + # Perform recursive shape assert + recursive_assert_shape(outB, torch.Size([10])) + + # Assert non-homogeneous type assertions + assert outB[0][0].neural_type.compare(NeuralType(('B',), LogprobsType())) == NeuralTypeComparisonResult.SAME + assert outB[0][1].neural_type.compare(NeuralType(('B',), LogprobsType())) == NeuralTypeComparisonResult.SAME + assert outB[1][0].neural_type.compare(NeuralType(('B',), LabelsType())) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_nested_input_output_neural_types_ignore_collections(self): + class NestedNodeA(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType())} + + @property + def output_types(self): + return { + "y0": NeuralType(('B', 'D'), LogitsType()), + "y1": NeuralType(('B', 'D'), LogitsType()), + } + + @typecheck(ignore_collections=True) + def __call__(self, x): + # input x = [[x1, x2], [x3]] + x0 = x[0][0] + y = torch.randn(x0.shape[0], 4) + return [[y, y], [y]] + + # Non-homogeneous output types + class NestedNodeB(Typing): + @property + def input_types(self): + return {"w": NeuralType(('B', 'D'), LogitsType())} + + @property + def output_types(self): + return { + "u0": NeuralType(('B',), LogprobsType()), # check non homogeneous type + "u1": NeuralType(('B',), LabelsType()), + } + + @typecheck(ignore_collections=True) + def __call__(self, w): + # input x = [[x1, x2], [x3]] + _, u00 = w[0][0].max(-1) + _, u01 = w[0][1].max(-1) + _, u10 = w[1][0].max(-1) + return [[u00, u01], [u10]] + + nodeA = NestedNodeA() + nodeB = NestedNodeB() + + input_nest = [[torch.zeros(10), torch.zeros(10)], [torch.zeros(10)]] + outA = nodeA(x=input_nest) + outB = nodeB(w=outA) + + # Perform recursive shape assert + recursive_assert_shape(outB, torch.Size([10])) + + # Assert non-homogeneous type assertions + assert outB[0][0].neural_type.compare(NeuralType(('B',), LogprobsType())) == NeuralTypeComparisonResult.SAME + assert outB[0][1].neural_type.compare(NeuralType(('B',), LogprobsType())) == NeuralTypeComparisonResult.SAME + assert outB[1][0].neural_type.compare(NeuralType(('B',), LabelsType())) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_nested_mixed_input_output_neural_types(self): + class NestedMixedNodeA(Typing): + @property + def input_types(self): + return {"x1": NeuralType(('B',), ElementType()), "x2": [[NeuralType(('B',), ElementType())]]} + + @property + def output_types(self): + return { + "y0": NeuralType(('B', 'D'), LogprobsType()), + "y1": [[NeuralType(('B', 'D'), LogitsType())]], + } + + @typecheck(ignore_collections=False) + def __call__(self, x1, x2): + # input x = [[x1, x2], [x3]] + x0 = x2[0][0] + y = torch.randn(x0.shape[0], 4) + return y, [[y, y], [y]] + + # Non-homogeneous output types + class NestedMixedNodeB(Typing): + @property + def input_types(self): + return {"w": [[NeuralType(('B', 'D'), LogitsType())]]} + + @property + def output_types(self): + return { + "u0": [NeuralType(('B',), LogprobsType())], # check non homogeneous type + "u1": NeuralType(('B',), LabelsType()), + } + + @typecheck(ignore_collections=False) + def __call__(self, w): + # input x = [[x1, x2], [x3]] + _, u00 = w[0][0].max(-1) + _, u01 = w[0][1].max(-1) + _, u10 = w[1][0].max(-1) + return [u00, u01], u10 + + nodeA = NestedMixedNodeA() + nodeB = NestedMixedNodeB() + + input_nest = [[torch.zeros(10), torch.zeros(10)], [torch.zeros(10)]] + out_y, outA = nodeA(x1=torch.zeros(10), x2=input_nest) + outB, out_u = nodeB(w=outA) + + # Perform recursive shape assert + assert out_y.neural_type.compare(NeuralType(('B', 'D'), LogprobsType())) + recursive_assert_shape(outB, torch.Size([10])) + + # Assert non-homogeneous type assertions + assert outB[0].neural_type.compare(NeuralType(('B',), LogprobsType())) == NeuralTypeComparisonResult.SAME + assert outB[1].neural_type.compare(NeuralType(('B',), LogprobsType())) == NeuralTypeComparisonResult.SAME + assert out_u.neural_type.compare(NeuralType(('B',), LabelsType())) == NeuralTypeComparisonResult.SAME + + @pytest.mark.unit + def test_multi_forward_type(self): + class AdaptiveTypeCheck(Typing): + @property + def input_types(self): + if self.mode == 'train': + return {"x": NeuralType(('B',), ElementType())} + + elif self.mode == 'infer': + return {"y": NeuralType(('B',), ChannelType())} + + elif self.mode == 'eval': + return {"x": NeuralType(('B',), ElementType()), "y": NeuralType(('B',), ChannelType())} + else: + raise ValueError("Wrong mode of operation") + + @property + def output_types(self): + if self.mode == 'train': + return {"u": NeuralType(('B',), ElementType())} + + elif self.mode == 'infer': + return {"v": NeuralType(('B',), ChannelType())} + + elif self.mode == 'eval': + return {"u": NeuralType(('B',), ElementType()), "v": NeuralType(('B',), ChannelType())} + else: + raise ValueError("Wrong mode of operation") + + def __init__(self): + self.mode = 'train' + + def __call__(self, **kwargs): + # Call should call and forward appropriate method in its own mode + if self.mode == 'train': + return self.train_forward(x=kwargs['x']) + + elif self.mode == 'eval': + return self.eval_forward(x=kwargs['x'], y=kwargs['y']) + + elif self.mode == 'infer': + return self.infer_forward(y=kwargs['y']) + + @typecheck() + def train_forward(self, x): + return x + 10 + + @typecheck() + def eval_forward(self, x, y): + return x - 1, y - 1 + + @typecheck() + def infer_forward(self, y): + return y - 10 + + @property + def mode(self): + return self._mode + + @mode.setter + def mode(self, val): + if val not in ['train', 'infer', 'eval']: + raise ValueError('mode must be either train infer or eval') + self._mode = val + + obj = AdaptiveTypeCheck() + + x = torch.zeros(10) + y = torch.full([10], fill_value=5, dtype=torch.int32) + + obj.mode = 'train' + x = obj(x=x) + + assert torch.all(x == 10) + assert x.neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + + obj.mode = 'eval' + x, y = obj(x=x, y=y) + + assert torch.all(x == 9) + assert torch.all(y == 4) + assert x.neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + assert y.neural_type.compare(NeuralType(('B',), ChannelType())) == NeuralTypeComparisonResult.SAME + + obj.mode = 'infer' + y = obj(y=y) + + assert torch.all(y == -6) + assert y.neural_type.compare(NeuralType(('B',), ChannelType())) == NeuralTypeComparisonResult.SAME + + # Now perform assertions of wrong mode with wrong input combinations + obj.mode = 'train' + + # In train mode, call infer + with pytest.raises(TypeError): + _ = obj.eval_forward(x=x, y=y) + + with pytest.raises(TypeError): + # wrong input + wrong mode + _ = obj.infer_forward(y=x) + + @pytest.mark.unit + def test_input_type_override(self): + class InputTypesOverride(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x): + x += 1 + return x + + @typecheck(input_types={"y": NeuralType(('B',), CategoricalValuesType())}) + def forward(self, y): + y -= 1 + return y + + obj = InputTypesOverride() + result = obj(x=torch.zeros(10)) + + assert result.sum() == torch.tensor(10.0) + assert hasattr(result, 'neural_type') is False + + # Test override + result2 = obj.forward(y=torch.zeros(10)) + + assert result2.sum() == torch.tensor(-10.0) + assert hasattr(result2, 'neural_type') is False + + @pytest.mark.unit + def test_output_type_override(self): + class OutputTypes(Typing): + @property + def output_types(self): + return {"y": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x): + x += 1 + return x + + @typecheck(output_types={"z": NeuralType(('B',), CategoricalValuesType())}) + def forward(self, z): + z -= 1 + return z + + obj = OutputTypes() + result = obj(x=torch.zeros(10)) + + assert result.sum() == torch.tensor(10.0) + assert result.neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + + # Test passing positional args + # Positional args allowed if input types is not set ! + result = obj(torch.zeros(10)) + assert result.sum() == torch.tensor(10.0) + + # Test override + result2 = obj.forward(z=torch.zeros(10)) + + assert result2.sum() == torch.tensor(-10.0) + assert hasattr(result2, 'neural_type') + assert ( + result2.neural_type.compare(NeuralType(('B',), CategoricalValuesType())) == NeuralTypeComparisonResult.SAME + ) + + @pytest.mark.unit + def test_multi_type_override(self): + class AdaptiveTypeCheck(Typing): + @property + def input_types(self): + # __call__ assumed to be for inference only, + # therefore infer types checked at class scope + return {"y": NeuralType(('B',), ChannelType())} + + @property + def output_types(self): + # __call__ assumed to be for inference only, + # therefore infer types checked at class scope + return {"v": NeuralType(('B',), ChannelType())} + + def __call__(self, **kwargs): + # Call should call and forward appropriate method in its own mode + # Let default "forward" call be the infer mode (this is upto developer) + # Therefore default class level types == infer types + return self.infer_forward(y=kwargs['y']) + + @typecheck( + input_types={"x": NeuralType(('B',), ElementType())}, + output_types={"u": NeuralType(('B',), ElementType())}, + ) + def train_forward(self, x): + return x + 10 + + @typecheck( + input_types={"x": NeuralType(('B',), ElementType()), "y": NeuralType(('B',), ChannelType())}, + output_types={"u": NeuralType(('B',), ElementType()), "v": NeuralType(('B',), ChannelType())}, + ) + def eval_forward(self, x, y): + return x - 1, y - 1 + + @typecheck( + input_types={"y": NeuralType(('B',), ChannelType())}, + output_types={"v": NeuralType(('B',), ChannelType())}, + ) + def infer_forward(self, y): + return y - 10 + + obj = AdaptiveTypeCheck() + + x = torch.zeros(10) + y = torch.full([10], fill_value=5, dtype=torch.int32) + + # infer mode + y = obj(y=y) + + assert torch.all(y == -5) + assert y.neural_type.compare(NeuralType(('B',), ChannelType())) == NeuralTypeComparisonResult.SAME + + x, y = obj.eval_forward(x=x, y=y) + + assert torch.all(x == -1) + assert torch.all(y == -6) + assert x.neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + assert y.neural_type.compare(NeuralType(('B',), ChannelType())) == NeuralTypeComparisonResult.SAME + + x = obj.train_forward(x=x) + + assert torch.all(x == 9) + assert x.neural_type.compare(NeuralType(('B',), ElementType())) == NeuralTypeComparisonResult.SAME + + # In train func, call eval signature + with pytest.raises(TypeError): + _ = obj.train_forward(x=x, y=y) + + with pytest.raises(TypeError): + # wrong input + wrong mode + _ = obj.infer_forward(x=x) + + @pytest.mark.unit + def test_disable_typecheck(self): + class InputOutputTypes(Typing): + @property + def input_types(self): + return {"x": NeuralType(('B',), ElementType())} + + @property + def output_types(self): + return {"y": NeuralType(('B',), ElementType())} + + @typecheck() + def __call__(self, x, **kwargs): + x += 1 + return x + + # Disable typecheck tests + with typecheck.disable_checks(): + obj = InputOutputTypes() + + # Execute function without kwarg + result = obj(torch.zeros(10)) + + assert result.sum() == torch.tensor(10.0) + assert hasattr(result, 'neural_type') is False + + # Test passing wrong key for input + _ = obj(a=torch.zeros(10), x=torch.zeros(5)) + + @pytest.mark.unit + def test_nested_shape_mismatch(self): + class NestedShapeMismatch(Typing): + @property + def input_types(self): + return {"x": [[NeuralType(('D',), ElementType())]]} # Each element of nest will have 4 values + + @property + def output_types(self): + return {"y": [[NeuralType(('D',), ElementType())]]} # Each element of nest will have 4 values + + @typecheck() + def __call__(self, x): + # v-- this is to satisfy 1 output constraint, python will otherwise interpret x as a 3 output value + return x + + def bb(dim=4): + return torch.zeros(dim) + + obj = NestedShapeMismatch() + + # Arbitrary nest 1 (should pass) + data = [[bb(), bb(), bb()], [bb()], [bb(), bb()]] + result = obj(x=data) + + recursive_assert_shape(result, torch.Size([4])) + recursive_assert_homogeneous_type(result, NeuralType(('D',), ElementType())) + + # Arbitrary nest 2 (should pass) + def bb(dim=4): + return torch.zeros(dim, dim) + + data = [[bb(), bb(), bb()], [bb()], [bb(), bb()]] + # Fails since input shape is incorrect + with pytest.raises(TypeError): + _ = obj(x=data) + + # Arbitrary nest 3 + def bb(dim=4): + return torch.zeros(dim) + + data = [[[bb(), bb(), bb()]], [[bb()], [bb(), bb()]]] + # Check should fail since nest level is 3! + with pytest.raises(TypeError): + result = obj(x=data) + + @pytest.mark.unit + def test_nested_mixed_shape_mismatch(self): + class NestedMixedShapeMismatch(Typing): + @property + def input_types(self): + return {"x": [[NeuralType(('D',), ElementType())]]} # Each element of nest will have 4 values + + @property + def output_types(self): + return {"y": [NeuralType(('D',), ElementType())]} # Each element of nest will have 4 values + + @typecheck() + def __call__(self, x): + # v-- this is to satisfy 1 output constraint, python will otherwise interpret x as a 3 output value + x = x[0] + return x + + def bb(dim=4): + return torch.zeros(dim) + + obj = NestedMixedShapeMismatch() + + # Arbitrary nest 1 (should pass) + data = [[bb(), bb(), bb()], [bb()], [bb(), bb()]] + result = obj(x=data) + + recursive_assert_shape(result, torch.Size([4])) + recursive_assert_homogeneous_type(result, NeuralType(('D',), ElementType())) + + # Arbitrary nest 2 (should pass) + def bb(dim=4): + return torch.zeros(dim, dim) + + data = [[bb(), bb(), bb()], [bb()], [bb(), bb()]] + # Fails since input shape is incorrect + with pytest.raises(TypeError): + _ = obj(x=data) + + # Arbitrary nest 3 + def bb(dim=4): + return torch.zeros(dim) + + data = [[[bb(), bb(), bb()]], [[bb()], [bb(), bb()]]] + # Check should fail since nest level is 3! + with pytest.raises(TypeError): + result = obj(x=data) + + @pytest.mark.unit + def test_input_container_neural_types(self): + class NodeA(Typing): + @property + def input_types(self): + return {"x": [NeuralType(('B',), ElementType())]} + + @property + def output_types(self): + return {"y": NeuralType(('B', 'D'), LogitsType())} + + @typecheck() + def __call__(self, x: list()): + x1, x2, x3 = x # unpack x + y = torch.randn(x1.shape[0], 4) + return y + + nodeA = NodeA() + outA = nodeA(x=[torch.zeros(10), torch.zeros(10), torch.zeros(10)]) + + assert outA.neural_type.compare(NeuralType(('B', 'D'), LogitsType())) + + @pytest.mark.unit + def test_input_container_neural_types_incorrect(self): + class NodeA(Typing): + @property + def input_types(self): + # Nest depth level of 2 + return {"x": [[NeuralType(('B',), ElementType())]]} + + @property + def output_types(self): + return {"y": NeuralType(('B', 'D'), LogitsType())} + + @typecheck() + def __call__(self, x: list()): + x1, x2, x3 = x # unpack x + y = torch.randn(x1.shape[0], 4) + return y + + nodeA = NodeA() + # Input nest level of 1 + with pytest.raises(TypeError): + outA = nodeA(x=[torch.zeros(10), torch.zeros(10), torch.zeros(10)]) + + @pytest.mark.unit + def test_output_container_neural_types_incorrect(self): + class NodeA(Typing): + @property + def input_types(self): + # Nest depth level of 2 + return {"x": NeuralType(('B',), ElementType())} + + @property + def output_types(self): + return {"y": [[NeuralType(('B', 'D'), LogitsType())]]} + + @typecheck() + def __call__(self, x): + y = torch.randn(x.shape[0], 4) + return y, y, y + + nodeA = NodeA() + # Input nest level of 1 + with pytest.raises(TypeError): + outA = nodeA(x=torch.zeros(10)) + + @pytest.mark.unit + def test_output_container_neural_types_no_tuple_wrap(self): + class NodeA(Typing): + @property + def input_types(self): + # Nest depth level of 2 + return {"x": NeuralType(('B',), ElementType())} + + @property + def output_types(self): + return {"y": [NeuralType(('B', 'D'), LogitsType())]} + + @typecheck() + def __call__(self, x): + y = torch.randn(x.shape[0], 4) + y = [y, y, y] + return y + + nodeA = NodeA() + # Input nest level of 1 + outA = nodeA(x=torch.zeros(10)) + + assert len(outA) == 3 + for i in range(len(outA)): + assert outA[i].neural_type.compare(NeuralType(('B', 'D'), LogitsType())) + + @pytest.mark.unit + def test_output_container_neural_types_explicit_tuple_wrap(self): + class NodeA(Typing): + @property + def input_types(self): + # Nest depth level of 2 + return {"x": NeuralType(('B',), ElementType())} + + @property + def output_types(self): + return {"y": [NeuralType(('B', 'D'), LogitsType())]} + + @typecheck() + def __call__(self, x): + y = torch.randn(x.shape[0], 4) + y = [y, y, y] + return (y,) + + nodeA = NodeA() + # Input nest level of 1 + outA = nodeA(x=torch.zeros(10)) + + assert len(outA) == 1 + assert len(outA[0]) == 3 + for i in range(len(outA)): + assert outA[0][i].neural_type.compare(NeuralType(('B', 'D'), LogitsType())) diff --git a/models/tacotron2/NeMo/tests/core_ptl/check_for_ranks.py b/models/tacotron2/NeMo/tests/core_ptl/check_for_ranks.py new file mode 100644 index 0000000000..b936e2da1d --- /dev/null +++ b/models/tacotron2/NeMo/tests/core_ptl/check_for_ranks.py @@ -0,0 +1,155 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil + +import torch +from omegaconf import OmegaConf +from pytorch_lightning import Trainer +from pytorch_lightning.utilities.distributed import rank_zero_only + +from nemo.core import ModelPT +from nemo.utils import logging +from nemo.utils.exp_manager import ExpManagerConfig, exp_manager + + +class OnesDataset(torch.utils.data.Dataset): + def __init__(self, dataset_len): + super().__init__() + self.__dataset_len = dataset_len + + def __getitem__(self, *args): + return torch.ones(2) + + def __len__(self): + return self.__dataset_len + + +class ExampleModel(ModelPT): + def __init__(self, *args, **kwargs): + cfg = OmegaConf.structured({}) + super().__init__(cfg, trainer=kwargs.get('trainer', None)) + # dummy parameter in order to allow DDP to execute + self.l1 = torch.nn.modules.Linear(in_features=2, out_features=1) + + def train_dataloader(self): + return None + + def val_dataloader(self): + return None + + def predict_dataloader(self): + dataset = OnesDataset(2) + return torch.utils.data.DataLoader(dataset, batch_size=2) + + def forward(self, batch): + return batch.mean() + + def validation_step(self, batch, batch_idx): + return self(batch) + + def training_step(self, batch, batch_idx): + return self(batch) + + def list_available_models(self): + pass + + def setup_training_data(self): + pass + + def setup_validation_data(self): + pass + + def validation_epoch_end(self, loss): + self.log("val_loss", torch.stack(loss).mean()) + + +def instantiate_multinode_ddp_if_possible(): + num_gpus = torch.cuda.device_count() + trainer = Trainer(gpus=num_gpus, accelerator='ddp', logger=None, checkpoint_callback=None) + + exp_manager_cfg = ExpManagerConfig(exp_dir='./ddp_check/', use_datetime_version=False, version="") + exp_manager(trainer, cfg=OmegaConf.structured(exp_manager_cfg)) + return trainer + + +def setup_model(trainer: Trainer): + model = ExampleModel(trainer=trainer) + + logging.info(f"M.Global Rank:{model.global_rank}") + logging.info(f"M.Local Rank:{model.local_rank}") + logging.info(f"M.World Size:{model.trainer.world_size}") + + trainer.predict(model) + return model + + +def get_rank_info(texts: list, rank_key: str) -> int: + for line in texts: + if rank_key in line: + rank_value = line.split(":")[-1] + rank_value = int(rank_value) + return rank_value + + print("Could not find the correct rank key !") + exit(1) + + +@rank_zero_only +def check_model_ranks(model: ExampleModel): + basedir = os.path.join('./ddp_check/', 'default', 'version_0') + file_template = "nemo_log_globalrank-{rank}_localrank-{rank}.txt" + + world_size = torch.cuda.device_count() + for rank in range(world_size): + filename = file_template.format(rank=rank) + filepath = os.path.join(basedir, filename) + + with open(filepath, 'r') as f: + texts = f.readlines() + texts = [t.replace("\n", "") for t in texts] + + log_global_rank = get_rank_info(texts, rank_key='M.Global Rank') + log_world_size = get_rank_info(texts, rank_key='M.World Size') + + if log_global_rank != rank: + print("Logged global rank is not equal to trainer.global_rank !") + exit(1) + + if log_world_size != world_size: + print("Logged world size if not equal to trainer.world_size !") + exit(1) + + +@rank_zero_only +def cleanup(): + if os.path.exists('./ddp_check'): + shutil.rmtree('./ddp_check', ignore_errors=True) + + +def run_checks(): + cleanup() + + trainer = instantiate_multinode_ddp_if_possible() + model = setup_model(trainer) + check_model_ranks(model) + + print("DDP checks passed !") + + cleanup() + + +if __name__ == '__main__': + run_checks() diff --git a/models/tacotron2/NeMo/tests/hydra/config1.yaml b/models/tacotron2/NeMo/tests/hydra/config1.yaml new file mode 100644 index 0000000000..71425c0e8c --- /dev/null +++ b/models/tacotron2/NeMo/tests/hydra/config1.yaml @@ -0,0 +1 @@ +dataset_name: dark diff --git a/models/tacotron2/NeMo/tests/hydra/config1_invalid.yaml b/models/tacotron2/NeMo/tests/hydra/config1_invalid.yaml new file mode 100644 index 0000000000..44ea0bdf04 --- /dev/null +++ b/models/tacotron2/NeMo/tests/hydra/config1_invalid.yaml @@ -0,0 +1,2 @@ +dataset_name: magic +password: secret \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/hydra/config_subdir/config2.yaml b/models/tacotron2/NeMo/tests/hydra/config_subdir/config2.yaml new file mode 100644 index 0000000000..e773e40209 --- /dev/null +++ b/models/tacotron2/NeMo/tests/hydra/config_subdir/config2.yaml @@ -0,0 +1 @@ +dataset_name: ala_config2 diff --git a/models/tacotron2/NeMo/tests/hydra/config_subdir/config2_invalid.yaml b/models/tacotron2/NeMo/tests/hydra/config_subdir/config2_invalid.yaml new file mode 100644 index 0000000000..f06f3eee20 --- /dev/null +++ b/models/tacotron2/NeMo/tests/hydra/config_subdir/config2_invalid.yaml @@ -0,0 +1,2 @@ +dataset_name: ala_config2 +password: ala_secret2 \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/hydra/my_app.py b/models/tacotron2/NeMo/tests/hydra/my_app.py new file mode 100644 index 0000000000..dca19dc0bd --- /dev/null +++ b/models/tacotron2/NeMo/tests/hydra/my_app.py @@ -0,0 +1,44 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass + +import hydra +from omegaconf import MISSING, OmegaConf + +from nemo.core.config import hydra_runner + + +@dataclass +class DefaultConfig: + """ + This is structured config for this application. + + It provides the schema used for validation of user-written spec file + as well as default values of the selected parameters. + """ + + # Dataset. Available options: [imdb, sst2] + dataset_name: str = MISSING + + +@hydra_runner(config_name="DefaultConfig", schema=DefaultConfig) +def my_app(cfg): + print(OmegaConf.to_yaml(cfg)) + # Get dataset_name. + dataset_name = cfg.dataset_name + + +if __name__ == "__main__": + my_app() diff --git a/models/tacotron2/NeMo/tests/hydra/test_hydra_runner.py b/models/tacotron2/NeMo/tests/hydra/test_hydra_runner.py new file mode 100644 index 0000000000..1da0a914cf --- /dev/null +++ b/models/tacotron2/NeMo/tests/hydra/test_hydra_runner.py @@ -0,0 +1,95 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import sys +from os import path + +import pytest + + +class TestHydraRunner: + @pytest.mark.integration + def test_no_config(self): + """"Test app without config - fields missing causes error. + """ + # Create system call. + call = "python tests/hydra/my_app.py" + + with pytest.raises(subprocess.CalledProcessError): + # Run the call as subprocess. + subprocess.check_call(call, shell=True, stdout=sys.stdout, stderr=sys.stdout) + + @pytest.mark.integration + def test_config1(self): + """"Test injection of valid config1. + """ + # Create system call. + call = "python tests/hydra/my_app.py --config-name config1.yaml" + + # Run the call as subprocess. + subprocess.check_call(call, shell=True, stdout=sys.stdout, stderr=sys.stdout) + + # Make sure that .hydra dir is not present. + assert not path.exists(f".hydra") + # Make sure that default hydra log file is not present. + assert not path.exists(f"my_app.log") + + @pytest.mark.integration + def test_config1_invalid(self): + """"Test injection of invalid config1. + """ + # Create system call. + call = "python tests/hydra/my_app.py --config-name config1_invalid.yaml" + + with pytest.raises(subprocess.CalledProcessError): + # Run the call as subprocess. + subprocess.check_call(call, shell=True, stdout=sys.stdout, stderr=sys.stdout) + + @pytest.mark.integration + def test_config2(self): + """"Test injection of valid config2 from a different folder. + """ + # Create system call. + call = "python tests/hydra/my_app.py --config-path config_subdir --config-name config2.yaml" + + # Run the call as subprocess. + subprocess.check_call(call, shell=True, stdout=sys.stdout, stderr=sys.stdout) + + # Make sure that .hydra dir is not present. + assert not path.exists(f".hydra") + # Make sure that default hydra log file is not present. + assert not path.exists(f"my_app.log") + + @pytest.mark.integration + def test_config2_invalid(self): + """"Test injection of invalid config2 from a different folder. + """ + # Create system call. + call = "python tests/hydra/my_app.py --config-path config_subdir --config-name config2_invalid.yaml" + + with pytest.raises(subprocess.CalledProcessError): + # Run the call as subprocess. + subprocess.check_call(call, shell=True, stdout=sys.stdout, stderr=sys.stdout) + + @pytest.mark.integration + def test_config2_filepath_schema(self): + """"Test injection of valid config2 - using namepath with schema is prohibited. + """ + # Create system call. + call = "python tests/hydra/my_app.py --config-name config_subdir/config2_invalid.yaml" + + with pytest.raises(subprocess.CalledProcessError): + # Run the call as subprocess. + subprocess.check_call(call, shell=True, stdout=sys.stdout, stderr=sys.stdout) diff --git a/models/tacotron2/NeMo/tests/manualtest_model_downloads.py b/models/tacotron2/NeMo/tests/manualtest_model_downloads.py new file mode 100644 index 0000000000..a3ed1734a8 --- /dev/null +++ b/models/tacotron2/NeMo/tests/manualtest_model_downloads.py @@ -0,0 +1,92 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +WARNING: Running this test will download ALL pre-trained NeMo models. +This is bandwidth and disk space consuming. +""" + +import nemo.collections.asr as nemo_asr +import nemo.collections.nlp as nemo_nlp +import nemo.collections.tts as nemo_tts + + +def testclass_downloads(cls, refresh_cache, model_names=None): + for model_info in cls.list_available_models(): + model = cls.from_pretrained(model_name=model_info.pretrained_model_name, refresh_cache=refresh_cache) + assert isinstance(model, cls) + if model_names is not None: + assert set(model_names) == set([m.pretrained_model_name for m in cls.list_available_models()]) + + +for refresh_cache in [True, False]: + # Test ASR collection + testclass_downloads( + nemo_asr.models.EncDecCTCModel, + refresh_cache, + [ + 'QuartzNet15x5Base-En', + 'QuartzNet15x5Base-Zh', + 'QuartzNet5x5LS-En', + 'QuartzNet15x5NR-En', + 'Jasper10x5Dr-En', + ], + ) + testclass_downloads(nemo_asr.models.EncDecCTCModelBPE, refresh_cache, ['ContextNet-192-WPE-1024-8x-Stride']) + testclass_downloads( + nemo_asr.models.EncDecClassificationModel, + refresh_cache, + [ + 'MatchboxNet-3x1x64-v1', + 'MatchboxNet-3x2x64-v1', + 'MatchboxNet-3x1x64-v2', + 'MatchboxNet-3x1x64-v2', + 'MatchboxNet-3x1x64-v2-subset-task', + 'MatchboxNet-3x2x64-v2-subset-task', + 'MatchboxNet-VAD-3x2', + ], + ) + testclass_downloads( + nemo_asr.models.EncDecSpeakerLabelModel, + refresh_cache, + ['speakerrecognition_speakernet', 'speakerverification_speakernet', 'speakerdiarization_speakernet'], + ) + + # Test NLP collection + testclass_downloads(nemo_nlp.models.TokenClassificationModel, refresh_cache, ['NERModel']) + testclass_downloads( + nemo_nlp.models.PunctuationCapitalizationModel, + refresh_cache, + ['Punctuation_Capitalization_with_BERT', 'Punctuation_Capitalization_with_DistilBERT'], + ) + testclass_downloads( + nemo_nlp.models.QAModel, + refresh_cache, + [ + 'BERTBaseUncasedSQuADv1.1', + 'BERTBaseUncasedSQuADv2.0', + 'BERTLargeUncasedSQuADv1.1', + 'BERTLargeUncasedSQuADv2.0', + ], + ) + # testclass_downloads(nemo_nlp.models.IntentSlotClassificationModel, refresh_cache, ['Joint_Intent_Slot_Assistant']) + + # Test TTS collection + testclass_downloads(nemo_tts.models.Tacotron2Model, refresh_cache, ['Tacotron2-22050Hz']) + testclass_downloads(nemo_tts.models.WaveGlowModel, refresh_cache, ['WaveGlow-22050Hz']) + testclass_downloads(nemo_tts.models.SqueezeWaveModel, refresh_cache, ['SqueezeWave-22050Hz']) + testclass_downloads(nemo_tts.models.GlowTTSModel, refresh_cache, ['GlowTTS-22050Hz']) + + +print("############ THAT'S ALL FOLKS! ############") diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_cardinal.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 0000000000..fb668c03ac --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,24 @@ +nine trillion seven hundred eighty nine billion three hundred eighty two million five hundred thirty six thousand one hundred thirty~9789382536130 +two hundred and fifty four~254 +one hundred forty seven thousand four hundred fifty one~147451 +one million one hundred fifty six thousand one hundred seventy three~1156173 +one billion five hundred ninety three million seventy two thousand nine hundred sixty one~1593072961 +ninety seven quadrillion eight hundred eight trillion two hundred sixty four billion seven hundred seventy two million seven hundred ninety two thousand five~97808264772792005 +seventeen sextillion eight hundred fifty five quintillion thirty six quadrillion six hundred fifty seven trillion seven billion five hundred ninety six million one hundred ten thousand nine hundred forty nine~17855036657007596110949 +ten quadrillion ten trillion ten million one hundred thousand ten~10010000010100010 +minus twenty five thousand thirty seven~-25037 +one quadrillion two hundred sixty four trillion three hundred one billion nine hundred thirty eight million one hundred four~1264301938000104 +minus sixty~-60 +forty six thousand six hundred sixty four~46664 +sixty~60 +zero~zero +two million three~2000003 +one thousand thirteen~1013 +one thousand one~1001 +one thousand one hundred~1100 +one thousand twenty six~1026 +one thousand one hundred twenty six~1126 +eighteen million four hundred fifty thousand nine hundred ninety~18450990 +eighteen million nine hundred forty thousand seven hundred twenty two~18940722 +eighteen million six hundred ninety thousand nine hundred sixteen~18690916 +eighteen thousand eight hundred eighty~18880 diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_date.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_date.txt new file mode 100644 index 0000000000..3ec4268144 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_date.txt @@ -0,0 +1,34 @@ +july twenty fifth two thousand twelve~july 25 2012 +nineteen eighties~1980s +two thousand and twenty~2020 +two thousand and nine~2009 +the twenty fifth of july twenty twelve~25 july 2012 +the twenty fifth of july two thousand twelve~25 july 2012 +the twenty second of july twenty twelve~22 july 2012 +the fifteenth of january~15 january +the seventeenth of may twenty ten~17 may 2010 +january first~january 1 +july twenty second two thousand eight~july 22 2008 +june thirty~june 30 +july twenty fifth twenty twelve~july 25 2012 +nineteen seventeen~1917 +twenty twelve~2012 +march sixteen sixty five~march 1665 +sixteen sixty five~1665 +july two thousand twelve~july 2012 +october nineteen oh five~october 1905 +july fifteen o six~july 1506 +the twenty fifth of july twenty twelve~25 july 2012 +july twenty fifth twenty twelve~july 25 2012 +july twenty fifth two thousand twelve~july 25 2012 +july one thousand eight hundred seventy six~july 1876 +february twenty fifth twenty sixteen~february 25 2016 +november twenty fourth twenty fourteen~november 24 2014 +nineteen ninety four~1994 +two thousand three~2003 +one thousand eight~1008 +nineteen seventy six~1976 +june twentieth twenty fourteen~june 20 2014 +nineteen seventy three~1973 +nineteen seventy five~1975 +eleven fifty five~1155 \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_decimal.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_decimal.txt new file mode 100644 index 0000000000..2cd2272837 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_decimal.txt @@ -0,0 +1,63 @@ +five point two million~5.2 million +one hundred sixty four point five eight thousand~164.58 thousand +four hundred million~400 million +fifty billion~50 billion +four hundred five billion~405 billion +four point eight five billion~4.85 billion +one hundred billion~100 billion +one hundred ten billion~110 billion +one hundred thirty two billion~132 billion +one point eight four billion~1.84 billion +one point eight one billion~1.81 billion +one point five nine billion~1.59 billion +one point four five three billion~1.453 billion +one point seven two billion~1.72 billion +one point two five billion~1.25 billion +thirteen billion~13 billion +thirty billion~30 billion +two thousand eight hundred five point eight seven three billion~2805.873 billion +seventy trillion~70 trillion +thirteen million~13 million +eighteen billion~18 billion +four hundred fifty million~450 million +one hundred thirty million~130 million +ten million~10 million +four hundred million~400 million +five million~5 million +five hundred million~500 million +twelve million~12 million +thirteen million~13 million +four million~4 million +forty five million~45 million +fifteen million~15 million +fifteen trillion~15 trillion +fifteen billion~15 billion +two million~2 million +eight million~8 million +point one two o five~.1205 +minus sixty point two four zero zero~-60.2400 +zero point two six~0.26 +point zero two~.02 +sixty point two~60.2 +eighteen~18 +eighteen point eight five~18.85 +eighteen point five o~18.50 +eighteen point five six~18.56 +eighteen point nine~18.9 +eighteen point o five~18.05 +eighteen point one two~18.12 +eighteen point o one~18.01 +eighteen point o o o~18.000 +eighteen point six~18.6 +eighteen point three o o~18.300 +eighteen point three six~18.36 +eighteen point two five~18.25 +eighteen point two two~18.22 +eight hundred eighteen point three o three~818.303 +eight hundred eight point eight~808.8 +eight hundred eight point zero~808.0 +eight hundred eighty eight point one~888.1 +eight hundred eighty four point three~884.3 +eight hundred eighty two point eight~882.8 +eight hundred eighty two point zero~882.0 +eight hundred forty five point nine four~845.94 \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_electronic.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_electronic.txt new file mode 100644 index 0000000000..6267f87ab4 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_electronic.txt @@ -0,0 +1,7 @@ +a dot b c at g mail dot com~a.bc@gmail.com +c d f at a b c dot e d u~cdf@abc.edu +a b c at g mail dot a b c~abc@gmail.abc +a b c at a b c dot com~abc@abc.com +a s d f one two three at a b c dot com~asdf123@abc.com +a one b two at a b c dot com~a1b2@abc.com +a b three dot s d d dot three at g mail dot com~ab3.sdd.3@gmail.com \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_measure.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_measure.txt new file mode 100644 index 0000000000..4f6f540e6d --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_measure.txt @@ -0,0 +1,112 @@ +two hundred meters~200 m +fifty six point three per square kilometer~56.3 /km² +two hundred kilometers per hour~200 km/h +twenty eight kilograms force per square centimeter~28 kgf/cm² +forty two thousand two hundred fifty nine per square meter~42259 /m² +minus two thousand twelve kilo liters~-2012 kl +minus sixty six kilograms~-66 kg +two kilo watt hours~2 kwh +one point o o o o two eight cubic deci meters~1.000028 dm³ +seven point five peta bytes~7.5 pb +three hours~3 h +one milli volt~1 mv +two cubic meters~2 m³ +ninety grams~90 g +one hundred twenty four point three lumens~124.3 lm +four hundred forty milliliters~440 ml +thirty one thousand four hundred eighty square feet~31480 sq ft +one thousand six hundred hours~1600 h +thirty one thousand four hundred eighty square feet~31480 sq ft +two square miles~2 sq mi +zero point one nine square miles~0.19 sq mi +one thousand five hundred thirty one c c~1531 cc +three hundred micrometers~300 μm +sixty five thousand square kilometers~65000 km² +two miles per hour~2 mph +two hundred forty five miles per hour~245 mph +one hundred fifty c c~150 cc +sixty point two four zero zero kilograms~60.2400 kg +zero feet~0 ft +zero foot~zero foot +two feet~2 ft +twenty foot~20 foot +point two meters~.2 m +two square meters~2 m² +eighteen feet~18 ft +eighteen mega siemens~18 ms +eighteen ounces~18 oz +eighteen point five kilometers~18.5 km +eighteen point five two square kilometers~18.52 km² +eighteen point nine one square kilometers~18.91 km² +eighteen point one four percent~18.14 % +eighteen point one six percent~18.16 % +eighteen point one square kilometers~18.1 km² +eighteen point six percent~18.6 % +eighteen point two two kilometers~18.22 km +eighteen point zero kilometers~18.0 km +eighteen point zero percent~18.0 % +eighteen square kilometers~18 km² +eighteen thousand eight hundred giga watt hours~18800 gwh +eighteen thousand seven hundred hectares~18700 ha +eight hectares~8 ha +eight hundred eighty five astronomical units~885 au +eight hundred eighty hectares~880 ha +eight hundred eighty kilobytes~880 kb +eight hundred eighty kilometers~880 km +eight hundred eighty nine feet~889 ft +eight hundred eighty six kilometers~886 km +eight hundred eighty two megawatts~882 mw +eight hundred feet~800 ft +eight hundred fifty five square kilometers~855 km² +eight hundred fifty megahertz~850 mhz +eight hundred fifty meters~850 m +eight hundred fifty nanometers~850 nm +eight hundred fifty one meters~851 m +eight hundred fifty seven square kilometers~857 km² +eight hundred fifty three meters~853 m +eight hundred fifty three point six meters~853.6 m +eight hundred five point four six square kilometers~805.46 km² +eight hundred forty two point nine meters~842.9 m +eight hundred forty two square kilometers~842 km² +eight hundred gigabytes~800 gb +eight hundred horsepower~800 hp +eight hundred kilograms~800 kg +eight hundred kilo watt hours~800 kwh +eight hundred kilowatts~800 kw +eight hundred megahertz~800 mhz +eight hundred ninety four c c~894 cc +eight hundred ninety kilowatts~890 kw +eight hundred ninety millimeters~890 mm +eight hundred ninety two square kilometers~892 km² +eight hundred seventy horsepower~870 hp +eight hundred seventy meters~870 m +eight hundred sixty kilograms~860 kg +eight hundred sixty kilometers~860 km +eight hundred sixty miles~860 mi +eight hundred sixty six feet~866 ft +eight hundred ten hectares~810 ha +eight hundred ten kilohertz~810 khz +eight hundred thirty eight point two millimeters~838.2 mm +eight hundred thirty five kilometers~835 km +eight hundred thirty kilohertz~830 khz +eight hundred thirty megawatts~830 mw +eight hundred thirty nine kilometers~839 km +eight hundred thirty six meters~836 m +eight hundred twenty feet~820 ft +eight hundred twenty kilometers~820 km +eight hundred twenty meters~820 m +eight hundred twenty one point zero feet~821.0 ft +eight hundred two point eight nine kilometers~802.89 km +eight hundred volts~800 v +eight kilobits~8 kb +eight kilograms~8 kg +eight million two hundred thousand feet~8200000 ft +eight point eight kilometers~8.8 km +eight point eight meters~8.8 m +eight point eight miles~8.8 mi +eight point five centimeters~8.5 cm +eight point five five percent~8.55 % +eight point five megawatts~8.5 mw +eight point five meters~8.5 m +eight point five two percent~8.52 % +eight point four four percent~8.44 % diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_money.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_money.txt new file mode 100644 index 0000000000..24bc64b025 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_money.txt @@ -0,0 +1,48 @@ +two dollars~$2 +one cent~$0.01 +four united states dollars and sixty nine cents~$4.69 +seventy five dollars sixty three~$75.63 +twenty nine dollars fifty cents~$29.50 +eleven dollars and fifty one cents~$11.51 +nine hundred ninety three dollars and ninety two cents~$993.92 +four hundred sixty billion won~₩460 billion +thirty billion yen~¥30 billion +two point five billion dollars~$2.5 billion +forty five billion dollars~$45 billion +fifty million dollars~$50 million +fifty billion dollars~$50 billion +zero point two million dollars~$0.2 million +fifteen point two billion dollars~$15.2 billion +one point six nine billion yuan~1.69 billion yuan +one point four three six billion yuan~1.436 billion yuan +four million yuan~4 million yuan +one dollar~$1 +fifteen thousand dollars~$15000 +one dollars~one dollars +twenty dollar~20 dollar +twenty point five o six dollars~$20.506 +point five o six dollars~$.506 +eighteen dollars~$18 +eighteen million nine hundred twenty five thousand dollars~$18925000 +eighteen thousand eight hundred fifty four dollars~$18854 +eighteen thousand eight hundred one dollars~$18801 +eighteen thousand eight hundred seventy five dollars~$18875 +eighteen thousand eighty one dollars~$18081 +eighteen thousand fifty two dollars~$18052 +eighteen thousand five hundred forty two dollars~$18542 +eighteen thousand five hundred nineteen dollars~$18519 +eighteen thousand five hundred seventy dollars~$18570 +eighteen thousand five hundred seventy eight dollars~$18578 +eighteen thousand five hundred sixteen dollars~$18516 +eighteen thousand four hundred eighty two dollars~$18482 +eighteen thousand four hundred seventy eight dollars~$18478 +eighteen thousand four hundred sixty eight dollars~$18468 +eighteen thousand nine hundred three dollars~$18903 +eighteen thousand nine hundred twenty nine dollars~$18929 +eighteen thousand ninety five dollars~$18095 +eighteen thousand one hundred seventeen dollars~$18117 +eighteen thousand one hundred twenty eight dollars~$18128 +eighteen thousand one hundred twenty five dollars~$18125 +eighteen thousand one hundred twenty four dollars~$18124 +eighteen thousand one hundred twenty nine dollars~$18129 +eighteen thousand pounds~£18000 \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_ordinal.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_ordinal.txt new file mode 100644 index 0000000000..1ceaaf0acf --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,33 @@ +one hundredth~100th +twenty five thousand one hundred eleventh~25111th +second~2nd +zeroth~0th +first~1st +second~2nd +third~3rd +fourth~4th +eleventh~11th +twelfth~12th +thirteenth~13th +twenty first~21st +twenty third~23rd +one hundred eleventh~111th +one thousandth~1000th +one hundred twenty first~121st +second~2nd +tenth~10th +sixth~6th +third~3rd +nineteenth~19th +third~3rd +twelfth~12th +forty eighth~48th +seventy first~71st +third~3rd +forty second~42nd +seventeenth~17th +twentieth~20th +twenty first~21st +seventh~7th +second~2nd +fifth~5th \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_telephone.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_telephone.txt new file mode 100644 index 0000000000..c689d9e200 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_telephone.txt @@ -0,0 +1,4 @@ +one two three one two three five six seven eight~123-123-5678 +zero two three one two three five six seven eight~023-123-5678 +o two three one two three five six seven eight~023-123-5678 +oh two three one two three five six seven eight~023-123-5678 \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_time.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_time.txt new file mode 100644 index 0000000000..ecdb1d0d1c --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_time.txt @@ -0,0 +1,27 @@ +eight oclock g m t~08:00 gmt +seven a m e s t~07:00 a.m. est +two p m~02:00 p.m. +two thirty~02:30 +three o'clock~03:00 +quarter past one~01:15 +half past three~03:30 +eight fifty one~08:51 +eight fifty two~08:52 +eight forty~08:40 +eight nineteen~08:19 +eight o six~08:06 +eight thirty eight~08:38 +eight thirty two~08:32 +eight twenty nine~08:29 +eleven fifty five p m~11:55 p.m. +eleven fifty three p m~11:53 p.m. +eleven forty a m~11:40 a.m. +eleven forty five a m~11:45 a.m. +eleven forty p m~11:40 p.m. +eleven forty six a m~11:46 a.m. +eleven o six p m~11:06 p.m. +eleven thirteen a m~11:13 a.m. +half past twelve~12:30 +quarter past one~01:15 +quarter to one~12:45 +quarter to twelve~11:45 \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_whitelist.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_whitelist.txt new file mode 100644 index 0000000000..4f69671fd3 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,8 @@ +doctor dao~dr. dao +miss smith~ms. smith +misses smith~mrs. smith +mister dao~mr. dao +saint george~st. george +i like for example ice cream~i like e.g. ice cream +s and p five hundred~s&p 500 +seven eleven stores~7-eleven stores diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_word.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_word.txt new file mode 100644 index 0000000000..4d12a42be8 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_inverse_text_normalization/test_cases_word.txt @@ -0,0 +1,50 @@ +~ +e s three~es3 +yahoo!~yahoo! +twenty!~20 ! +x ~x +—~— +aaa~aaa +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aagnes~aagnes +aagomoni~aagomoni +aagon~aagon +aagoo~aagoo +aagot~aagot +aahar~aahar +aahh~aahh +aahperd~aahperd +aaibinterstate~aaibinterstate +aajab~aajab +aakasa~aakasa +aakervik~aakervik +aakirkeby~aakirkeby +aalam~aalam +aalbaek~aalbaek +aaldiu~aaldiu +aalem~aalem +a'ali~a'ali +aalilaassamthey~aalilaassamthey +aalin~aalin +aaliyan~aaliyan +aaliyan's~aaliyan's +aamadu~aamadu +aamara~aamara +aambala~aambala +aamera~aamera +aamer's~aamer's +aamina~aamina +aaminah~aaminah +aamjiwnaang~aamjiwnaang diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_boundary.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_boundary.txt new file mode 100644 index 0000000000..3b9d66f214 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_boundary.txt @@ -0,0 +1,7 @@ + 1~one +1~one +1, ~one , +1!!!!~one ! ! ! ! +(1)Hello~(1)Hello +!1~! one +1!hello~1!hello \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_cardinal.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 0000000000..3d505b2ce4 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,18 @@ +1~one +2~two +-2~minus two +3~three +123~one hundred twenty three +13,000~thirteen thousand +13,000~thirteen thousand +123,123,000~one hundred twenty three million one hundred twenty three thousand +123,000,012~one hundred twenty three million twelve +t-0t25d12-f~t oh t two five d one two f +133-ABC~one hundred thirty three ABC +B2A23C~B two A two three C +25d08A~two five d oh eight A +C24~C two four +W2s~W two s +1-4-a-b-1-5~one four a b one five +b-c-1-5-b-s-b~b c one five b s b +1/f-4s~one f four s diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_date.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_date.txt new file mode 100644 index 0000000000..b5f95f4669 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_date.txt @@ -0,0 +1,32 @@ +july 25 2012~july twenty fifth twenty twelve +jul 25 2012~july twenty fifth twenty twelve +1980s~nineteen eighties +1980 s~nineteen eighties +25 july 2012~the twenty fifth of july twenty twelve +25 jul 2012~the twenty fifth of july twenty twelve +22 july 2012~the twenty second of july twenty twelve +15 january~the fifteenth of january +17 may 2010~the seventeenth of may twenty ten +january 1~january first +june 30~june thirtieth +2012~twenty twelve +july 1506~july fifteen o six +july 1876~july eighteen seventy six +1994~nineteen ninety four +1976~nineteen seventy six +june 20 2014~june twentieth twenty fourteen +1973~nineteen seventy three +1975~nineteen seventy five +1155~eleven fifty five +2006-08-05~august fifth two thousand six +august 23, 2002~august twenty third two thousand two +20 january~the twentieth of january +2016-11-03~november third twenty sixteen +2016/07/03~july third twenty sixteen +1980s~nineteen eighties +01.10.2010~january tenth twenty ten +10/06/2005~october sixth two thousand five +1910s~nineteen tens +25 sept.~the twenty fifth of september +1000~one thousand +SEPT. 15TH~september fifteenth diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_decimal.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_decimal.txt new file mode 100644 index 0000000000..1b839abc20 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_decimal.txt @@ -0,0 +1,8 @@ +.1665~point one six six five +2.0~two point zero +2.00~two point o o +2.050~two point o five o +100 million~one hundred million +0.1 billion~zero point one billion +.1 trillion~point one trillion +-0.1~minus zero point one diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_electronic.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_electronic.txt new file mode 100644 index 0000000000..1c5f5a71b1 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_electronic.txt @@ -0,0 +1,10 @@ +a.bc@gmail.com~a dot b c at g mail dot com +cdf@abc.edu~c d f at a b c dot e d u +abc@gmail.abc~a b c at g mail dot a b c +abc@abc.com~a b c at a b c dot com +asdf123@abc.com~a s d f one two three at a b c dot com +a1b2@abc.com~a one b two at a b c dot com +ab3.sdd.3@gmail.com~a b three dot s d d dot three at g mail dot com +email is abc1@gmail.com~email is a b c one at g mail dot com +test .b@gmail.com~test . b at g mail dot com +email is a-1.b3_c&-d@gma4i-l.com test~email is a dash one dot b three underscore c ampersand dash d at g m a four i dash l dot com test \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_fraction.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_fraction.txt new file mode 100644 index 0000000000..15d32caa9b --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_fraction.txt @@ -0,0 +1,11 @@ +1/2007~one two thousand seventh +12639/12640~twelve thousand six hundred thirty nine twelve thousand six hundred fortieths +2/4~two quarters +1/4~one quarter +31/32~thirty one thirty seconds +22/3~twenty two thirds +1/3~one third +142/1~one hundred forty two over one +1/2~one half +2 1/2~two and a half +1795 / 1805~one thousand seven hundred ninety five one thousand eight hundred fifths \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_measure.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_measure.txt new file mode 100644 index 0000000000..3279e6863c --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,15 @@ +12kg~twelve kilograms +12/kg~twelve per kilogram +12kg/kg~twelve kilograms per kilogram +1 mbps~one megabit per second +3 mbps~three megabits per second +3 cc/s~three c c per second +100 million kg~one hundred million kilograms +a 4-kilogram bag~a four kilogram bag +55-millimeters long~fifty five millimeters long +a 123.2-millimeters long~a one hundred twenty three point two millimeters long +covid-19.5~covid nineteen point five +covid-19~covid nineteen +a 4-kilogram bag~a four kilogram bag +7.2-millimeter bullet~seven point two millimeter bullet +4 1/2 lbs~four and a half pounds diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_money.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_money.txt new file mode 100644 index 0000000000..22f0b58acb --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_money.txt @@ -0,0 +1,38 @@ +$2~two dollars +₩460 billion~four hundred sixty billion won +¥30 billion~thirty billion yen +$45 billion~forty five billion dollars +$0.2 million~zero point two million dollars +$15.2 billion~fifteen point two billion dollars +1.436 billion yuan~one point four three six billion yuan +4 million yuan~four million yuan +$15000~fifteen thousand dollars +$1~one dollar +$20~twenty dollars +$20.506~twenty point five o six dollars +$.506~point five o six dollars +$18~eighteen dollars +$18925000~eighteen million nine hundred twenty five thousand dollars +$18,925,000~eighteen million nine hundred twenty five thousand dollars +$18854~eighteen thousand eight hundred fifty four dollars +$18801~eighteen thousand eight hundred one dollars +$18875~eighteen thousand eight hundred seventy five dollars +$18081~eighteen thousand eighty one dollars +$18052~eighteen thousand fifty two dollars +$18542~eighteen thousand five hundred forty two dollars +$18519~eighteen thousand five hundred nineteen dollars +$18570~eighteen thousand five hundred seventy dollars +$18578~eighteen thousand five hundred seventy eight dollars +$18516~eighteen thousand five hundred sixteen dollars +$18482~eighteen thousand four hundred eighty two dollars +$18478~eighteen thousand four hundred seventy eight dollars +$18468~eighteen thousand four hundred sixty eight dollars +$18903~eighteen thousand nine hundred three dollars +$18929~eighteen thousand nine hundred twenty nine dollars +$18095~eighteen thousand ninety five dollars +$18117~eighteen thousand one hundred seventeen dollars +$18128~eighteen thousand one hundred twenty eight dollars +$18125~eighteen thousand one hundred twenty five dollars +$18124~eighteen thousand one hundred twenty four dollars +$18129~eighteen thousand one hundred twenty nine dollars +£18000~eighteen thousand pounds \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_normalize_with_audio.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_normalize_with_audio.txt new file mode 100644 index 0000000000..436bff1920 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_normalize_with_audio.txt @@ -0,0 +1,107 @@ +~123 +one twenty three +one two three +one hundred twenty three +one hundred and twenty three +~$123 +one twenty three dollars +one two three dollars +one hundred twenty three dollars +one hundred and twenty three dollars +~$123.2 +one twenty three dollars and two cents +one two three dollars two cents +one hundred twenty three dollars and two cents +one hundred and twenty three point two dollars +~1.24 +one point two four +one two four +one point twenty four +one twenty four +~$1.21 +one dollar and twenty one cents +one dollar twenty one cents +one point two one dollars +one twenty one dollars +~£1.00 +one pound +one point o o pounds +~t-0t25d12-f +t zero t twenty five d one two f +t zero t twenty five d twelve f +t zero t two five d one two f +t zero t two five d twelve f +t oh t two five d one two f +t oh t twenty five d one two f +t oh t twenty five d twelve f +t oh t two five d twelve f +~133-A +one hundred thirty three A +~B2A23C +B two A twenty three C +~25d08A +twenty five d zero eight A +~C24 +C twenty four +~It seemed to her that the jacket Oswald wore was darker than Commission Exhibit No. 162. +It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one sixty two. +It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one hundred sixty two. +It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one hundred and sixty two. +It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one six two. +~"Father, let this cup pass." He prayed--was heard. What cup was it that passed away from him? Sure not the death-cup, now filled to the brim! There was no quailing in the awful word; He still was king of kings, of lords the lord:-- He feared lest, in the suffering waste and grim, His faith might grow too faint and sickly dim." +"Father, let this cup pass." He prayed -- was heard. What cup was it that passed away from him? Sure not the death-cup, now filled to the brim! There was no quailing in the awful word; He still was king of kings, of lords the lord: -- He feared lest, in the suffering waste and grim, His faith might grow too faint and sickly dim." +~1970-2010 +nineteen seventy to twenty ten +one thousand nine seventy to two thousand ten +one thousand nine hundred and seventy to two thousand ten +one thousand nine hundred and seventy to twenty ten +nineteen seventy twenty ten +one thousand nine seventy two thousand ten +one thousand nine hundred and seventy two thousand ten +one thousand nine hundred and seventy twenty ten +from nineteen seventy to twenty ten +from one thousand nine seventy to two thousand ten +from one thousand nine hundred and seventy to two thousand ten +from one thousand nine hundred and seventy to twenty ten +from nineteen seventy twenty ten +from one thousand nine seventy two thousand ten +from one thousand nine hundred and seventy two thousand ten +from one thousand nine hundred and seventy twenty ten +~W26s +W twenty sixes +W twenty six s +~5-3-Ws +five three W s +five three Ws +~401-ks +four oh one kays +four oh one k s +four hundred one k s +~The box was 25 x 7 m. +The box was twenty five by seven meters. +The box was two five by seven meters. +The box was two five by seven m. +The box was twenty five by seven m. +~4567 +four thousand five hundred sixty seven +four five six seven +four thousand five sixty seven +forty five sixty seven +four thousand five hundred and sixty seven +~This example number 15,000 can be a very long one, and can fail to produce valid normalization for a such an easy number like 10,125 or dollar value $5349.01, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, 452. +This example number fifteen thousand can be a very long one, and can fail to produce valid normalization for a such an easy number like ten thousand one hundred twenty five or dollar value five thousand three hundred forty nine dollars and one cent, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, four five two. +~$1.01 +one dollar and one cent +one dollar one cent +one dollar and zero one cents +~$17.31 +seventeen dollars and thirty one cent +seventeen dollars and thirty one cents +seventeen point three one dollars +~25.] +two five.] +twenty five.] +~Francis I--test +Francis the first -- test +Francis one -- test +Francis first -- test \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_ordinal.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 0000000000..dce8934082 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,6 @@ +0th~zeroth +11th~eleventh +23rd~twenty third +1000th~one thousandth +1,000th~one thousandth +1000000th~one millionth \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_telephone.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_telephone.txt new file mode 100644 index 0000000000..2e586f60fe --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_telephone.txt @@ -0,0 +1,9 @@ ++1 123-123-5678~one, one two three, one two three, five six seven eight +123-123-5678~one two three, one two three, five six seven eight ++1-123-123-5678~one, one two three, one two three, five six seven eight ++1 (123)-123-5678~one, one two three, one two three, five six seven eight ++1(123) 123-5678~one, one two three, one two three, five six seven eight +(123)-123-5678~one two three, one two three, five six seven eight +(123) 123-5678~one two three, one two three, five six seven eight +123-123-5678-1111~one two three, one two three, five six seven eight, one one one one +123-123-5678-1~one two three, one two three, five six seven eight, one \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_time.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_time.txt new file mode 100644 index 0000000000..5388be9d4d --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_time.txt @@ -0,0 +1,14 @@ +01:00~one o'clock +01:00 am~one a m +01:00 a.m.~one a m +01:00 a.m~one a m +1 a.m.~one a m +23:00~twenty three o'clock +1:00 a.m.~one a m +1:00a.m.~one a m +1.59 p.m.~one fifty nine p m +01:59 p.m.~one fifty nine p m +1:59 p.m.~one fifty nine p m +1:59 p.m. est~one fifty nine p m e s t +1:59 p.m.est~one fifty nine p m e s t +1:59 p.m. e.s.t~one fifty nine p m e s t \ No newline at end of file diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_whitelist.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 0000000000..7eaa6376a1 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,3 @@ +dr. Evil~doctor Evil +mrs. Norris~misses Norris +Dr. Evil~Dr. Evil diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_word.txt b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_word.txt new file mode 100644 index 0000000000..e10df68195 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/data_text_normalization/test_cases_word.txt @@ -0,0 +1,23 @@ +~ +es3~e s three +x ~x +X!~X! +2!~two ! +—~— +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aalem~aalem +a'ali~a'ali +aaliyan's~aaliyan's +mother-in-law~mother-in-law diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_boundary.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_boundary.py new file mode 100644 index 0000000000..9074230050 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_boundary.py @@ -0,0 +1,39 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestBoundary: + + normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_boundary.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, n_tagged=100, punct_pre_process=False, punct_post_process=False + ) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_cardinal.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_cardinal.py new file mode 100644 index 0000000000..7c19ea3d42 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_cardinal.py @@ -0,0 +1,49 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestCardinal: + inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_date.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_date.py new file mode 100644 index 0000000000..42fce25b4d --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_date.py @@ -0,0 +1,64 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestDate: + inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_date.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_uncased(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic + + normalizer_uppercased = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None + cases_uppercased = {"Aug. 8": "august eighth", "8 Aug.": "the eighth of august", "aug. 8": "august eighth"} + + @parameterized.expand(cases_uppercased.items()) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_cased(self, test_input, expected): + pred = self.normalizer_uppercased.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_decimal.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_decimal.py new file mode 100644 index 0000000000..267fcceb5e --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_decimal.py @@ -0,0 +1,49 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestDecimal: + inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_decimal.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_decimal.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_electronic.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_electronic.py new file mode 100644 index 0000000000..a96ecc696a --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_electronic.py @@ -0,0 +1,51 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestElectronic: + inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_electronic.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_electronic.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, n_tagged=100, punct_post_process=False + ) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_fraction.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_fraction.py new file mode 100644 index 0000000000..24006c43cc --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_fraction.py @@ -0,0 +1,38 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestFraction: + normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_fraction.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_measure.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_measure.py new file mode 100644 index 0000000000..12f4137f67 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_measure.py @@ -0,0 +1,50 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestMeasure: + inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_measure.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_measure.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_money.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_money.py new file mode 100644 index 0000000000..e132866b6d --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_money.py @@ -0,0 +1,50 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestMoney: + inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_money.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_money.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_normalization_with_audio.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_normalization_with_audio.py new file mode 100644 index 0000000000..00345837b3 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_normalization_with_audio.py @@ -0,0 +1,33 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, get_test_cases_multiple + + +class TestNormalizeWithAudio: + + normalizer = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(get_test_cases_multiple('data_text_normalization/test_cases_normalize_with_audio.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, n_tagged=700) + assert len(set(pred).intersection(set(expected))) == len(expected), f'pred: {pred}' diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_ordinal.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_ordinal.py new file mode 100644 index 0000000000..47ae51e830 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_ordinal.py @@ -0,0 +1,50 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestOrdinal: + inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_sparrowhawk_inverse_text_normalization.sh b/models/tacotron2/NeMo/tests/nemo_text_processing/test_sparrowhawk_inverse_text_normalization.sh new file mode 100644 index 0000000000..268f88b057 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,64 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while read testcase; do + IFS='~' read spoken written <<< $testcase + denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} + +testITNCardinal() { + input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testITNDate() { + input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_date.txt + runtest $input +} + +testITNDecimal() { + input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_decimal.txt + runtest $input +} + +testITNOrdinal() { + input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testITNTime() { + input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_time.txt + runtest $input +} + +testITNMeasure() { + input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_measure.txt + runtest $input +} + +testITNMoney() { + input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_money.txt + runtest $input +} + +testITNWhitelist() { + input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_whitelist.txt + runtest $input +} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_sparrowhawk_normalization.sh b/models/tacotron2/NeMo/tests/nemo_text_processing/test_sparrowhawk_normalization.sh new file mode 100644 index 0000000000..c06dc05aff --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_sparrowhawk_normalization.sh @@ -0,0 +1,64 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while read testcase; do + IFS='~' read written spoken <<< $testcase + denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + +testTNCardinal() { + input=$PROJECT_DIR/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testTNDate() { + input=$PROJECT_DIR/data_text_normalization/test_cases_date.txt + runtest $input +} + +testTNDecimal() { + input=$PROJECT_DIR/data_text_normalization/test_cases_decimal.txt + runtest $input +} + +testTNOrdinal() { + input=$PROJECT_DIR/data_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testTNTime() { + input=$PROJECT_DIR/data_text_normalization/test_cases_time.txt + runtest $input +} + +testTNMeasure() { + input=$PROJECT_DIR/data_text_normalization/test_cases_measure.txt + runtest $input +} + +testTNMoney() { + input=$PROJECT_DIR/data_text_normalization/test_cases_money.txt + runtest $input +} + +testTNWhitelist() { + input=$PROJECT_DIR/data_text_normalization/test_cases_whitelist.txt + runtest $input +} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_telephone.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_telephone.py new file mode 100644 index 0000000000..5ea68d6f0c --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_telephone.py @@ -0,0 +1,50 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestTelephone: + inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_telephone.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_telephone.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_time.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_time.py new file mode 100644 index 0000000000..84e05a83b7 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_time.py @@ -0,0 +1,49 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestTime: + inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_time.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_whitelist.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_whitelist.py new file mode 100644 index 0000000000..8b3daf2b67 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_whitelist.py @@ -0,0 +1,65 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestWhitelist: + inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(input_case='lower_cased') if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic + + normalizer_uppercased = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None + cases_uppercased = {"Dr. Evil": "doctor Evil", "No. 4": "number four", "dr. Evil": "dr. Evil", "no. 4": "no. four"} + + @parameterized.expand(cases_uppercased.items()) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_cased(self, test_input, expected): + pred = self.normalizer_uppercased.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/test_word.py b/models/tacotron2/NeMo/tests/nemo_text_processing/test_word.py new file mode 100644 index 0000000000..ae60100c41 --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/test_word.py @@ -0,0 +1,52 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from parameterized import parameterized +from utils import PYNINI_AVAILABLE, parse_test_case_file + + +class TestWord: + inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_word.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None + normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None + + @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_word.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, n_tagged=100, punct_post_process=False + ) + assert expected in pred_non_deterministic diff --git a/models/tacotron2/NeMo/tests/nemo_text_processing/utils.py b/models/tacotron2/NeMo/tests/nemo_text_processing/utils.py new file mode 100644 index 0000000000..d2bb3554ef --- /dev/null +++ b/models/tacotron2/NeMo/tests/nemo_text_processing/utils.py @@ -0,0 +1,54 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +try: + import pynini + + PYNINI_AVAILABLE = True +except (ImportError, ModuleNotFoundError): + PYNINI_AVAILABLE = False + + +def parse_test_case_file(file_name: str): + """ + Prepares tests pairs for ITN and TN tests + """ + test_pairs = [] + with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r') as f: + for line in f: + spoken, written = line.split('~') + test_pairs.append((spoken, written.strip("\n"))) + return test_pairs + + +def get_test_cases_multiple(file_name: str = 'data_text_normalization/test_cases_normalize_with_audio.txt'): + """ + Prepares tests pairs for audio based TN tests + """ + test_pairs = [] + with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r') as f: + written = None + normalized_options = [] + for line in f: + if line.startswith('~'): + if written: + test_pairs.append((written, normalized_options)) + normalized_options = [] + written = line.strip().replace('~', '') + else: + normalized_options.append(line.strip()) + test_pairs.append((written, normalized_options)) + return test_pairs diff --git a/models/tacotron2/NeMo/tests/test_data_dir.py b/models/tacotron2/NeMo/tests/test_data_dir.py new file mode 100644 index 0000000000..931e02b73c --- /dev/null +++ b/models/tacotron2/NeMo/tests/test_data_dir.py @@ -0,0 +1,27 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from os.path import exists, join + +import pytest + + +class TestDataDir: + @pytest.mark.unit + def test_test_data_dir(self, test_data_dir): + """" Just a dummy tests showing how to use the test_data_dir fixture. """ + # test_data_dir contains the absolute path to nemo -> tests/.data + assert exists(test_data_dir) + assert exists(join(test_data_dir, "test_data.tar.gz")) diff --git a/models/tacotron2/NeMo/tools/ctc_segmentation/README.md b/models/tacotron2/NeMo/tools/ctc_segmentation/README.md new file mode 100644 index 0000000000..bcfe5e7bd9 --- /dev/null +++ b/models/tacotron2/NeMo/tools/ctc_segmentation/README.md @@ -0,0 +1,31 @@ +Dataset creation tool based on CTC-Segmentation +----------------------------------------------- + +This tool provides functionality to align long audio files and the corresponding transcripts into shorter fragments +that are suitable for an Automatic Speech Recognition (ASR) model training. + +More details could be found in [this tutorial](https://github.com/NVIDIA/NeMo/blob/main/tutorials/tools/CTC_Segmentation_Tutorial.ipynb). + +The tool is based on the [CTC Segmentation](https://github.com/lumaku/ctc-segmentation): +**CTC-Segmentation of Large Corpora for German End-to-end Speech Recognition** +https://doi.org/10.1007/978-3-030-60276-5_27 or pre-print https://arxiv.org/abs/2007.09127 + +``` +@InProceedings{ctcsegmentation, +author="K{\"u}rzinger, Ludwig +and Winkelbauer, Dominik +and Li, Lujun +and Watzel, Tobias +and Rigoll, Gerhard", +editor="Karpov, Alexey +and Potapova, Rodmonga", +title="CTC-Segmentation of Large Corpora for German End-to-End Speech Recognition", +booktitle="Speech and Computer", +year="2020", +publisher="Springer International Publishing", +address="Cham", +pages="267--278", +abstract="Recent end-to-end Automatic Speech Recognition (ASR) systems demonstrated the ability to outperform conventional hybrid DNN/HMM ASR. Aside from architectural improvements in those systems, those models grew in terms of depth, parameters and model capacity. However, these models also require more training data to achieve comparable performance.", +isbn="978-3-030-60276-5" +} +``` diff --git a/models/tacotron2/NeMo/tools/ctc_segmentation/requirements.txt b/models/tacotron2/NeMo/tools/ctc_segmentation/requirements.txt new file mode 100644 index 0000000000..b5ef916816 --- /dev/null +++ b/models/tacotron2/NeMo/tools/ctc_segmentation/requirements.txt @@ -0,0 +1,2 @@ +ctc_segmentation==1.1.0 +num2words \ No newline at end of file diff --git a/models/tacotron2/NeMo/tools/ctc_segmentation/run_sample.sh b/models/tacotron2/NeMo/tools/ctc_segmentation/run_sample.sh new file mode 100644 index 0000000000..d865a57d0c --- /dev/null +++ b/models/tacotron2/NeMo/tools/ctc_segmentation/run_sample.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +# default values for optional arguments +MIN_SCORE=-5 +CUT_PREFIX=0 +SCRIPTS_DIR="scripts" +OFFSET=0 +LANGUAGE='eng' # 'eng', 'ru', 'other' +MIN_SEGMENT_LEN=20 +MAX_SEGMENT_LEN=100 +ADDITIONAL_SPLIT_SYMBOLS='' +AUDIO_FORMAT='.mp3' +USE_NEMO_NORMALIZATION='True' + +for ARG in "$@" +do + key=$(echo $ARG | cut -f1 -d=) + value=$(echo $ARG | cut -f2 -d=) + + if [[ $key == *"--"* ]]; then + v="${key/--/}" + declare $v="${value}" + fi +done + +echo "MODEL_NAME_OR_PATH = $MODEL_NAME_OR_PATH" +echo "DATA_DIR = $DATA_DIR" +echo "OUTPUT_DIR = $OUTPUT_DIR" +echo "MIN_SCORE = $MIN_SCORE" +echo "CUT_PREFIX = $CUT_PREFIX" +echo "SCRIPTS_DIR = $SCRIPTS_DIR" +echo "OFFSET = $OFFSET" +echo "LANGUAGE = $LANGUAGE" +echo "MIN_SEGMENT_LEN = $MIN_SEGMENT_LEN" +echo "MAX_SEGMENT_LEN = $MAX_SEGMENT_LEN" +echo "ADDITIONAL_SPLIT_SYMBOLS = $ADDITIONAL_SPLIT_SYMBOLS" +echo "AUDIO_FORMAT = $AUDIO_FORMAT" +echo "USE_NEMO_NORMALIZATION = $USE_NEMO_NORMALIZATION" + +if [[ -z $MODEL_NAME_OR_PATH ]] || [[ -z $DATA_DIR ]] || [[ -z $OUTPUT_DIR ]]; then + echo "Usage: $(basename "$0") + --MODEL_NAME_OR_PATH=[model_name_or_path] + --DATA_DIR=[data_dir] + --OUTPUT_DIR=[output_dir] + --LANGUAGE=[language (Optional)] + --OFFSET=[offset value (Optional)] + --CUT_PREFIX=[cut prefix in sec (Optional)] + --SCRIPTS_DIR=[scripts_dir_path (Optional)] + --MIN_SEGMENT_LEN=[min number of characters of the text segment for alignment (Optional)] + --MAX_SEGMENT_LEN=[max number of characters of the text segment for alignment (Optional)] + --ADDITIONAL_SPLIT_SYMBOLS=[Additional symbols to use for + sentence split if eos sentence split resulted in sequence longer than --max_length. + Use '|' as a separator between symbols, for example: ';|:|' (Optional)] + --AUDIO_FORMAT=[choose from ['.mp3', '.wav'], input audio files format + --USE_NEMO_NORMALIZATION Set to 'True' to use NeMo Normalization tool to convert + numbers from written to spoken format. By default num2words package will be used. (Optional)" + exit 1 +fi + +NEMO_NORMALIZATION="" + if [[ ${USE_NEMO_NORMALIZATION,,} == "true" ]]; then + NEMO_NORMALIZATION="--use_nemo_normalization " + fi + +# STEP #1 +# Prepare text and audio data for segmentation +python $SCRIPTS_DIR/prepare_data.py \ +--in_text=$DATA_DIR/text \ +--audio_dir=$DATA_DIR/audio \ +--audio_format=$AUDIO_FORMAT \ +--output_dir=$OUTPUT_DIR/processed/ \ +--language=$LANGUAGE \ +--cut_prefix=$CUT_PREFIX \ +--model=$MODEL_NAME_OR_PATH \ +--min_length=$MIN_SEGMENT_LEN \ +--max_length=$MAX_SEGMENT_LEN \ +--additional_split_symbols=$ADDITIONAL_SPLIT_SYMBOLS $NEMO_NORMALIZATION || exit + +# STEP #2 +# Run CTC-segmenatation +# one might want to perform alignment with various window sizes +# note if the alignment with the initial window size isn't found, the window size will be double to re-attempt +# alignment +for WINDOW in 8000 12000 +do + python $SCRIPTS_DIR/run_ctc_segmentation.py \ + --output_dir=$OUTPUT_DIR \ + --data=$OUTPUT_DIR/processed/ \ + --model=$MODEL_NAME_OR_PATH \ + --window_len $WINDOW || exit +done + +# STEP #3 (Optional) +# Verify aligned segments only if multiple WINDOWs used in the Step #2) +python $SCRIPTS_DIR/verify_segments.py \ +--base_dir=$OUTPUT_DIR || exit + +# STEP #4 +# Cut the original audio files based on the alignments +# (use --alignment=$OUTPUT_DIR/segments if only 1 WINDOW size was used in the Step #2) +# Three manifests and corresponding clips folders will be created: +# - high scored clips +# - low scored clips +# - deleted segments +python $SCRIPTS_DIR/cut_audio.py \ +--output_dir=$OUTPUT_DIR \ +--model=$MODEL_NAME_OR_PATH \ +--alignment=$OUTPUT_DIR/verified_segments \ +--threshold=$MIN_SCORE \ +--offset=$OFFSET || exit + +# STEP #5 (Optional) +# If multiple audio files were segmented in the step #2, this step will aggregate manifests with high scored segments +# for all audio files into all_manifest.json +# Also a separate manifest with samples from across all high scored segments will be credated if --num_samples > 0 +# --num_samples samples will be taken from the beginning, end and the middle of the each audio file high score manifest +# and will be stored at sample_manifest.json +python $SCRIPTS_DIR/process_manifests.py \ +--output_dir=$OUTPUT_DIR \ +--manifests_dir=$OUTPUT_DIR/manifests/ \ +--num_samples 0 + +exit + diff --git a/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/cut_audio.py b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/cut_audio.py new file mode 100644 index 0000000000..50bd931be9 --- /dev/null +++ b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/cut_audio.py @@ -0,0 +1,313 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +import time +from pathlib import Path +from typing import List + +import torch +from scipy.io import wavfile + +from nemo.collections import asr as nemo_asr +from nemo.collections.asr.metrics.wer import WER, word_error_rate + +parser = argparse.ArgumentParser(description="Cut audio on the segments based on segments") +parser.add_argument("--output_dir", default='output', type=str, help='Path to output directory') +parser.add_argument( + "--alignment", + type=str, + required=True, + help='Path to a data directory with alignments or a single .txt file with timestamps - result of the ctc-segmentation', +) +parser.add_argument("--threshold", type=float, default=-5, help='Minimum score value accepted') +parser.add_argument( + '--model', + type=str, + default='QuartzNet15x5Base-En', + help='Path to model checkpoint or pre-trained CTC-based ASR model name', +) +parser.add_argument('--offset', type=int, default=0, help='Offset in seconds') +parser.add_argument("--batch_size", type=int, default=64, help='Batch size for inference') + + +def add_transcript_to_manifest( + manifest_original: str, manifest_updated: str, asr_model: nemo_asr.models.EncDecCTCModel, batch_size: int +) -> None: + """ + Adds transcripts generated by the asr_model to the manifest_original. + + Args: + manifest_original: path to the manifest + manifest_updated: path to the updated manifest with transcript included + asr_model: CTC-based ASR model, for example, QuartzNet15x5Base-En + batch_size: Batch size for asr_model inference + """ + transcripts = get_transcript(manifest_original, asr_model, batch_size) + with open(manifest_original, 'r', encoding='utf8') as f: + with open(manifest_updated, 'w', encoding='utf8') as f_updated: + for i, line in enumerate(f): + info = json.loads(line) + info['transcript'] = transcripts[i].strip() + info['WER'] = round(word_error_rate([info['transcript']], [info['text']]) * 100, 2) + info['CER'] = round(word_error_rate([info['transcript']], [info['text']], use_cer=True) * 100, 2) + json.dump(info, f_updated, ensure_ascii=False) + f_updated.write('\n') + + +def get_transcript(manifest_path: str, asr_model: nemo_asr.models.EncDecCTCModel, batch_size: int) -> List[str]: + """ + Returns transcripts for audio segments in the batch + + Args: + manifest_path: path to the manifest for inference + asr_model: CTC-based ASR model, for example, QuartzNet15x5Base-En + batch_size: batch size + + Returns: hypotheses: transcripts for the audio segments + """ + # batch inference + try: + from torch.cuda.amp import autocast + except ImportError: + from contextlib import contextmanager + + @contextmanager + def autocast(enabled=None): + yield + + torch.set_grad_enabled(False) + asr_model.setup_test_data( + test_data_config={ + 'sample_rate': 16000, + 'manifest_filepath': manifest_path, + 'labels': asr_model.decoder.vocabulary, + 'batch_size': batch_size, + 'normalize_transcripts': False, + } + ) + asr_model.eval() + wer = WER(vocabulary=asr_model.decoder.vocabulary) + hypotheses = [] + for test_batch in asr_model.test_dataloader(): + if torch.cuda.is_available(): + test_batch = [x.cuda() for x in test_batch] + with autocast(): + log_probs, encoded_len, greedy_predictions = asr_model( + input_signal=test_batch[0], input_signal_length=test_batch[1] + ) + hypotheses += wer.ctc_decoder_predictions_tensor(greedy_predictions) + del test_batch + torch.cuda.empty_cache() + return hypotheses + + +def process_alignment(alignment_file: str, args): + """ Cut original audio file into audio segments based on alignment_file + + Args: + alignment_file: path to the file with segmented text and corresponding time stamps. + The first line of the file contains the path to the original audio file + args: main script args + """ + if not os.path.exists(alignment_file): + raise ValueError(f'{alignment_file} not found') + + # read the segments, note the first line contains the path to the original audio + segments = [] + ref_text_processed = [] + ref_text_no_preprocessing = [] + with open(alignment_file, 'r') as f: + for line in f: + line = line.split('|') + # read audio file name from the first line + if len(line) == 1: + audio_file = line[0].strip() + continue + ref_text_processed.append(line[1].strip()) + ref_text_no_preprocessing.append(line[2].strip()) + line = line[0].split() + segments.append((float(line[0]) + args.offset / 1000, float(line[1]) + args.offset / 1000, float(line[2]))) + + # cut the audio into segments + # create manifest in /tmp directory first, then populate transcript values with the batch inference results + # and save the final manifests at output_dir + sampling_rate, signal = wavfile.read(audio_file) + original_duration = len(signal) / sampling_rate + print(f'Cutting {audio_file} based on {alignment_file}') + print(f'Original duration: {round(original_duration)}s or ~{round(original_duration / 60)}min') + + # create directories to store high score .wav audio fragments, low scored once, and deleted + args.output_dir = os.path.abspath(args.output_dir) + os.makedirs(args.output_dir, exist_ok=True) + fragments_dir = os.path.join(args.output_dir, "high_score_clips") + del_fragments = os.path.join(args.output_dir, 'deleted_clips') + low_score_segments_dir = os.path.join(args.output_dir, "low_score_clips") + + os.makedirs(fragments_dir, exist_ok=True) + os.makedirs(low_score_segments_dir, exist_ok=True) + os.makedirs(del_fragments, exist_ok=True) + + base_name = os.path.basename(alignment_file).replace('_segments.txt', '') + high_score_manifest = f'{base_name}_high_score_manifest.json' + low_score_manifest = f'{base_name}_low_score_manifest.json' + del_manifest = f'{base_name}_del_manifest.json' + manifests_dir = os.path.join(args.output_dir, 'manifests') + os.makedirs(manifests_dir, exist_ok=True) + tmp_dir = '/tmp' + + low_score_dur = 0 + high_score_dur = 0 + with open(os.path.join(tmp_dir, high_score_manifest), 'w', encoding='utf8') as f: + with open(os.path.join(tmp_dir, low_score_manifest), 'w', encoding='utf8') as low_score_f: + for i, (st, end, score) in enumerate(segments): + segment = signal[round(st * sampling_rate) : round(end * sampling_rate)] + duration = len(segment) / sampling_rate + if duration > 0: + text_processed = ref_text_processed[i].strip() + text_no_preprocessing = ref_text_no_preprocessing[i].strip() + if score > args.threshold: + high_score_dur += duration + audio_filepath = os.path.join(fragments_dir, f'{base_name}_{i:04}.wav') + file_to_write = f + else: + low_score_dur += duration + audio_filepath = os.path.join(low_score_segments_dir, f'{base_name}_{i:04}.wav') + file_to_write = low_score_f + + wavfile.write(audio_filepath, sampling_rate, segment) + + transcript = 'n/a' # asr_model.transcribe(paths2audio_files=[audio_filepath], batch_size=1)[0] + info = { + 'audio_filepath': audio_filepath, + 'duration': duration, + 'text': text_processed, + 'text_no_preprocessing': text_no_preprocessing, + 'score': round(score, 2), + 'transcript': transcript.strip(), + } + json.dump(info, file_to_write, ensure_ascii=False) + file_to_write.write('\n') + + add_transcript_to_manifest( + os.path.join(tmp_dir, high_score_manifest), + os.path.join(manifests_dir, high_score_manifest), + asr_model, + args.batch_size, + ) + add_transcript_to_manifest( + os.path.join(tmp_dir, low_score_manifest), + os.path.join(manifests_dir, low_score_manifest), + asr_model, + args.batch_size, + ) + print(f'High score files duration: {round(high_score_dur)}s or ~{round(high_score_dur/60)}min at {manifests_dir}') + print( + f'Low score files duration: {round(low_score_dur)}s or ~{round(low_score_dur/60)}min saved at {manifests_dir}' + ) + + # save deleted segments along with manifest + deleted = [] + del_duration = 0 + begin = 0 + i = 0 + with open(os.path.join(manifests_dir, del_manifest), 'w', encoding='utf8') as f: + for i, (st, end, _) in enumerate(segments): + if st - begin > 0.01: + segment = signal[int(begin * sampling_rate) : int(st * sampling_rate)] + audio_filepath = os.path.join(del_fragments, f'del_{base_name}_{i:04}.wav') + wavfile.write(audio_filepath, sampling_rate, segment) + duration = len(segment) / sampling_rate + del_duration += duration + deleted.append((begin, st)) + transcript = 'n/a' # asr_model.transcribe(paths2audio_files=[audio_filepath], batch_size=1)[0] + info = {'audio_filepath': audio_filepath, 'duration': duration, 'text': transcript} + json.dump(info, f, ensure_ascii=False) + f.write('\n') + begin = end + + segment = signal[int(begin * sampling_rate) :] + audio_filepath = os.path.join(del_fragments, f'del_{base_name}_{i+1:04}.wav') + wavfile.write(audio_filepath, sampling_rate, segment) + duration = len(segment) / sampling_rate + del_duration += duration + deleted.append((begin, original_duration)) + + info = {'audio_filepath': audio_filepath, 'duration': duration, 'text': 'n/a'} + json.dump(info, f) + f.write('\n') + + print(f'Saved DEL files duration: {round(del_duration)}s or ~ {round(del_duration/60)}min at {del_fragments}') + missing_audio = original_duration - high_score_dur - del_duration - low_score_dur + if missing_audio > 15: + raise ValueError(f'{round(missing_audio)}s or ~ {round(missing_audio/60)}min is missing. Check the args') + + stats = ( + args.output_dir, + base_name, + round(original_duration), + round(high_score_dur), + round(low_score_dur), + round(del_duration), + ) + return stats + + +if __name__ == '__main__': + start_time = time.time() + args = parser.parse_args() + + os.makedirs(args.output_dir, exist_ok=True) + + if os.path.exists(args.model): + asr_model = nemo_asr.models.EncDecCTCModel.restore_from(args.model, strict=False) + elif args.model in nemo_asr.models.EncDecCTCModel.get_available_model_names(): + asr_model = nemo_asr.models.EncDecCTCModel.from_pretrained(args.model, strict=False) + else: + raise ValueError( + f'Provide path to the pretrained checkpoint or choose from {nemo_asr.models.EncDecCTCModel.list_available_models()}' + ) + + alignment_files = Path(args.alignment) + if os.path.isdir(args.alignment): + alignment_files = alignment_files.glob("*.txt") + else: + alignment_files = [Path(alignment_files)] + + stats_file = os.path.join(args.output_dir, 'stats.tsv') + with open(stats_file, 'w') as f: + f.write('Folder\tSegment\tOriginal dur (s)\tHigh quality dur (s)\tLow quality dur (s)\tDeleted dur (s)\n') + + high_score_dur = 0 + low_score_dur = 0 + del_duration = 0 + + for alignment_file in alignment_files: + stats = process_alignment(alignment_file, args) + high_score_dur += stats[-3] + low_score_dur += stats[-2] + del_duration += stats[-1] + stats = '\t'.join([str(t) for t in stats]) + '\n' + f.write(stats) + + f.write(f'Total\t\t{round(high_score_dur)}\t{round(low_score_dur)}\t{del_duration}') + + total_time = time.time() - start_time + print(f'High score segments duration: {round(high_score_dur)}') + print(f'Low score segments duration: {round(low_score_dur)}') + print(f'Deleted segments duration: {round(del_duration)}') + print(f'Stats saved at {stats_file}') + print(f'Total execution time: ~{round(total_time / 60)}min') diff --git a/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/normalization_helpers.py b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/normalization_helpers.py new file mode 100644 index 0000000000..7e971e7c80 --- /dev/null +++ b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/normalization_helpers.py @@ -0,0 +1,69 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = ['LATIN_TO_RU', 'RU_ABBREVIATIONS', 'NUMBERS_TO_ENG', 'NUMBERS_TO_RU'] + +LATIN_TO_RU = { + 'a': 'а', + 'b': 'б', + 'c': 'к', + 'd': 'д', + 'e': 'е', + 'f': 'ф', + 'g': 'г', + 'h': 'х', + 'i': 'и', + 'j': 'ж', + 'k': 'к', + 'l': 'л', + 'm': 'м', + 'n': 'н', + 'o': 'о', + 'p': 'п', + 'q': 'к', + 'r': 'р', + 's': 'с', + 't': 'т', + 'u': 'у', + 'v': 'в', + 'w': 'в', + 'x': 'к', + 'y': 'у', + 'z': 'з', + 'à': 'а', + 'è': 'е', + 'é': 'е', + 'ß': 'в', + 'ä': 'а', + 'ö': 'о', + 'ü': 'у', + 'є': 'е', + 'ç': 'с', + 'ê': 'е', + 'ó': 'о', +} + +RU_ABBREVIATIONS = { + ' р.': ' рублей', + ' к.': ' копеек', + ' коп.': ' копеек', + ' копек.': ' копеек', + ' т.д.': ' так далее', + ' т. д.': ' так далее', + ' т.п.': ' тому подобное', + ' т. п.': ' тому подобное', + ' т.е.': ' то есть', + ' т. е.': ' то есть', + ' стр. ': ' страница ', +} diff --git a/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/prepare_data.py b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/prepare_data.py new file mode 100644 index 0000000000..47373fb357 --- /dev/null +++ b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/prepare_data.py @@ -0,0 +1,412 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import multiprocessing +import os +import re +from pathlib import Path +from typing import List + +import regex +import scipy.io.wavfile as wav +from normalization_helpers import LATIN_TO_RU, RU_ABBREVIATIONS +from num2words import num2words + +from nemo.collections import asr as nemo_asr + +try: + from nemo_text_processing.text_normalization.normalize import Normalizer + + NEMO_NORMALIZATION_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + NEMO_NORMALIZATION_AVAILABLE = False + + +parser = argparse.ArgumentParser(description="Prepares text and audio files for segmentation") +parser.add_argument("--in_text", type=str, default=None, help='Path to a text file or a directory with .txt files') +parser.add_argument("--output_dir", type=str, required=True, help='Path to output directory') +parser.add_argument("--audio_dir", type=str, help='Path to folder with .mp3 or .wav audio files') +parser.add_argument( + "--audio_format", type=str, default='.mp3', choices=['.mp3', '.wav'], help='Audio files format in --audio_dir' +) +parser.add_argument('--sample_rate', type=int, default=16000, help='Sampling rate used during ASR model training') +parser.add_argument( + '--language', type=str, default='eng', choices=['eng', 'ru', 'add other languages supported by num2words.'] +) +parser.add_argument( + '--cut_prefix', type=int, default=0, help='Number of seconds to cut from the beginning of the audio files.', +) +parser.add_argument( + '--model', type=str, default='QuartzNet15x5Base-En', help='Pre-trained model name or path to model checkpoint' +) +parser.add_argument('--min_length', type=int, default=0, help='Min number of chars of the text segment for alignment.') +parser.add_argument( + '--max_length', type=int, default=100, help='Max number of chars of the text segment for alignment.' +) +parser.add_argument( + '--additional_split_symbols', + type=str, + default='', + help='Additional symbols to use for \ + sentence split if eos sentence split resulted in sequence longer than --max_length. ' + 'Use "|" as a separator between symbols, for example: ";|:|" ', +) +parser.add_argument( + '--use_nemo_normalization', + action='store_true', + help='Set to True to use NeMo Normalization tool to convert numbers from written to spoken format.', +) + + +def convert_audio(in_file: str, wav_file: str = None, sample_rate: int = 16000) -> str: + """ + Convert .mp3 to .wav and/or change sample rate if needed + + Args: + in_file: Path to .mp3 or .wav file + sample_rate: Desired sample rate + + Returns: + path to .wav file + """ + print(f"Converting {in_file} to .wav format with sample rate {sample_rate}") + if not os.path.exists(in_file): + raise ValueError(f'{in_file} not found') + if wav_file is None: + wav_file = in_file.replace(os.path.splitext(in_file)[-1], f"_{sample_rate}.wav") + + os.system( + f'ffmpeg -i {in_file} -acodec pcm_s16le -ac 1 -af aresample=resampler=soxr -ar {sample_rate} {wav_file} -y' + ) + return wav_file + + +def process_audio(in_file: str, wav_file: str = None, cut_prefix: int = 0, sample_rate: int = 16000): + """Process audio file: .mp3 to .wav conversion and cut a few seconds from the beginning of the audio + + Args: + in_file: path to the .mp3 or .wav file for processing + wav_file: path to the output .wav file + cut_prefix: number of seconds to cut from the beginning of the audio file + sample_rate: target sampling rate + """ + wav_audio = convert_audio(str(in_file), wav_file, sample_rate) + + if cut_prefix > 0: + # cut a few seconds of audio from the beginning + sample_rate, signal = wav.read(wav_audio) + wav.write(wav_audio, data=signal[cut_prefix * sample_rate :], rate=sample_rate) + + +def split_text( + in_file: str, + out_file: str, + vocabulary: List[str] = None, + language='eng', + remove_brackets=True, + do_lower_case=True, + min_length=0, + max_length=100, + additional_split_symbols=None, + use_nemo_normalization=False, +): + """ + Breaks down the in_file roughly into sentences. Each sentence will be on a separate line. + Written form of the numbers will be converted to its spoken equivalent, OOV punctuation will be removed. + + Args: + in_file: path to original transcript + out_file: path to the output file + vocabulary: ASR model vocabulary + language: text language + remove_brackets: Set to True if square [] and curly {} brackets should be removed from text. + Text in square/curly brackets often contains inaudible fragments like notes or translations + do_lower_case: flag that determines whether to apply lower case to the in_file text + min_length: Min number of chars of the text segment for alignment. Short segments will be combined to be + at least min_length (not recommended for multi speaker data). + max_length: Max number of chars of the text segment for alignment + additional_split_symbols: Additional symbols to use for sentence split if eos sentence split resulted in + segments longer than --max_length + use_nemo_normalization: Set to True to use NeMo normalization tool to convert numbers from written to spoken + format. Normalization using num2words will be applied afterwards to make sure there are no numbers present + in the text, otherwise they will be replaced with a space and that could deteriorate segmentation results. + """ + + print(f'Splitting text in {in_file} into sentences.') + with open(in_file, "r") as f: + transcript = f.read() + + # remove some symbols for better split into sentences + transcript = ( + transcript.replace("\n", " ") + .replace("\t", " ") + .replace("…", "...") + .replace("\\", " ") + .replace("--", " -- ") + .replace(". . .", "...") + .replace("‘", "’") + ) + # remove extra space + transcript = re.sub(r' +', ' ', transcript) + transcript = re.sub(r'(\.+)', '. ', transcript) + + if remove_brackets: + transcript = re.sub(r'(\[.*?\])', ' ', transcript) + # remove text in curly brackets + transcript = re.sub(r'(\{.*?\})', ' ', transcript) + + lower_case_unicode = '' + upper_case_unicode = '' + if language == 'ru': + lower_case_unicode = '\u0430-\u04FF' + upper_case_unicode = '\u0410-\u042F' + elif language not in ['ru', 'eng']: + print(f'Consider using {language} unicode letters for better sentence split.') + + # remove space in the middle of the lower case abbreviation to avoid splitting into separate sentences + matches = re.findall(r'[a-z' + lower_case_unicode + ']\.\s[a-z' + lower_case_unicode + ']\.', transcript) + for match in matches: + transcript = transcript.replace(match, match.replace('. ', '.')) + + # find phrases in quotes + with_quotes = re.finditer(r'“[A-Za-z ?]+.*?”', transcript) + sentences = [] + last_idx = 0 + for m in with_quotes: + match = m.group() + match_idx = m.start() + if last_idx < match_idx: + sentences.append(transcript[last_idx:match_idx]) + sentences.append(match) + last_idx = m.end() + sentences.append(transcript[last_idx:]) + sentences = [s.strip() for s in sentences if s.strip()] + + # Read and split transcript by utterance (roughly, sentences) + split_pattern = f"(? 0] + + if min_length > 0: + sentences_comb = [] + sentences_comb.append(sentences[0]) + # combines short sentence + for i in range(1, len(sentences)): + if len(sentences_comb[-1]) < min_length or len(sentences[i]) < min_length: + sentences_comb[-1] += ' ' + sentences[i].strip() + else: + sentences_comb.append(sentences[i].strip()) + sentences = sentences_comb + + sentences = [s.strip() for s in sentences if s.strip()] + + # save split text with original punctuation and case + out_dir, out_file_name = os.path.split(out_file) + with open(os.path.join(out_dir, out_file_name[:-4] + '_with_punct.txt'), "w") as f: + f.write("\n".join(sentences)) + + # substitute common abbreviations before applying lower case + if language == 'ru': + for k, v in RU_ABBREVIATIONS.items(): + sentences = [s.replace(k, v) for s in sentences] + + if language == 'ru': + # replace Latin characters with Russian + for k, v in LATIN_TO_RU.items(): + sentences = [s.replace(k, v) for s in sentences] + + if language == 'eng' and use_nemo_normalization: + if not NEMO_NORMALIZATION_AVAILABLE: + raise ValueError(f'NeMo normalization tool is not installed.') + + print('Using NeMo normalization tool...') + normalizer = Normalizer(input_case='cased') + sentences_norm = normalizer.normalize_list(sentences, verbose=False) + if len(sentences_norm) != len(sentences): + raise ValueError(f'Normalization failed, number of sentences does not match.') + + sentences = '\n'.join(sentences) + + # replace numbers with num2words + try: + p = re.compile("\d+") + new_text = '' + match_end = 0 + for i, m in enumerate(p.finditer(sentences)): + match = m.group() + match_start = m.start() + if i == 0: + new_text = sentences[:match_start] + else: + new_text += sentences[match_end:match_start] + match_end = m.end() + new_text += sentences[match_start:match_end].replace(match, num2words(match, lang=language)) + new_text += sentences[match_end:] + sentences = new_text + except NotImplementedError: + print( + f'{language} might be missing in "num2words" package. Add required language to the choices for the' + f'--language argument.' + ) + raise + + sentences = ( + sentences.replace("’", "'") + .replace("»", '"') + .replace("«", '"') + .replace("\\", "") + .replace("”", '"') + .replace("„", '"') + .replace("´", "'") + .replace("-- --", "--") + .replace("--", " -- ") + .replace("’", "'") + .replace('“', '"') + .replace('“', '"') + .replace("‘", "'") + .replace('—', '-') + .replace("- -", "--") + .replace('`', "'") + .replace(' !', '!') + .replace(' ?', '?') + .replace(' ,', ',') + .replace(' .', '.') + .replace(' ;', ';') + .replace(' :', ':') + .replace('!!', '!') + .replace('--', '-') + .replace('“', '"') + .replace(', , ', ', ') + .replace('=', '') + ) + + allowed_punct = [',', '.', '?', '!', ':', ';', '-', '"', '(', ')'] + # clean up normalized text and keep only allowed_punct and ASR vocabulary (lower and upper case) + symbols_to_remove = ''.join( + set(sentences).difference(set(vocabulary + [s.upper() for s in vocabulary] + ['\n'] + allowed_punct)) + ) + sentences_norm = sentences.translate(''.maketrans(symbols_to_remove, len(symbols_to_remove) * ' ')) + + with open(os.path.join(out_dir, out_file_name[:-4] + '_with_punct_normalized.txt'), "w") as f: + f.write(sentences_norm) + + if do_lower_case: + sentences = sentences.lower() + + # remove all OOV symbols + symbols_to_remove = ''.join(set(sentences).difference(set(vocabulary + ['\n']))) + sentences = sentences.translate(''.maketrans(symbols_to_remove, len(symbols_to_remove) * ' ')) + + # remove extra space + sentences = re.sub(r' +', ' ', sentences) + with open(out_file, "w") as f: + f.write(sentences) + + +if __name__ == '__main__': + args = parser.parse_args() + + os.makedirs(args.output_dir, exist_ok=True) + + text_files = [] + if args.in_text: + vocabulary = None + if args.model is None: + print(f"No model provided, vocabulary won't be used") + elif os.path.exists(args.model): + asr_model = nemo_asr.models.EncDecCTCModel.restore_from(args.model) + vocabulary = asr_model.cfg.decoder.vocabulary + elif args.model in nemo_asr.models.EncDecCTCModel.get_available_model_names(): + asr_model = nemo_asr.models.EncDecCTCModel.from_pretrained(args.model) + vocabulary = asr_model.cfg.decoder.vocabulary + else: + raise ValueError( + f'Provide path to the pretrained checkpoint or choose from {nemo_asr.models.EncDecCTCModel.get_available_model_names()}' + ) + + if os.path.isdir(args.in_text): + text_files = Path(args.in_text).glob(("*.txt")) + else: + text_files.append(Path(args.in_text)) + for text in text_files: + base_name = os.path.basename(text)[:-4] + out_text_file = os.path.join(args.output_dir, base_name + '.txt') + + split_text( + text, + out_text_file, + vocabulary=vocabulary, + language=args.language, + min_length=args.min_length, + max_length=args.max_length, + additional_split_symbols=args.additional_split_symbols, + use_nemo_normalization=args.use_nemo_normalization, + ) + print(f'Processed text saved at {args.output_dir}') + + if args.audio_dir: + if not os.path.exists(args.audio_dir): + raise ValueError(f'{args.audio_dir} not found. "--audio_dir" should contain .mp3 or .wav files.') + + audio_paths = list(Path(args.audio_dir).glob(f"*{args.audio_format}")) + + workers = [] + for i in range(len(audio_paths)): + wav_file = os.path.join(args.output_dir, audio_paths[i].name.replace(args.audio_format, ".wav")) + worker = multiprocessing.Process( + target=process_audio, args=(audio_paths[i], wav_file, args.cut_prefix, args.sample_rate), + ) + workers.append(worker) + worker.start() + for w in workers: + w.join() + + print('Done.') diff --git a/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/process_manifests.py b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/process_manifests.py new file mode 100644 index 0000000000..d3ef9870fc --- /dev/null +++ b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/process_manifests.py @@ -0,0 +1,111 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +from pathlib import Path +from random import randint +from typing import List + +parser = argparse.ArgumentParser(description="Combine manifests and extract samples") +parser.add_argument( + "--manifests_dir", + default='_high_score_manifest.json', + type=str, + required=True, + help='Path to directory with *_high_score_manifest.json files', +) +parser.add_argument( + "--output_dir", default='output', required=True, help='Path to the directory to store final manifest' +) +parser.add_argument( + "--num_samples", + default=0, + type=int, + help='The number of samples to get from the beginning, end and randomly from the middle of highly scored samples', +) + + +def get_samples(manifest: str, num_samples: int) -> List[str]: + """ + Samples num_samples from the beginning, end and randomly from the middle of the manifest + + Args: + manifest: path to manifest + num_samples: number of samples to extract from the beginning, end and randomly from the middle of the manifest + + Returns: + num_samples: manifest lines + """ + samples = [] + with open(manifest, 'r', encoding='utf8') as f: + lines = f.readlines() + + # first n samples + samples.extend(lines[:num_samples]) + # random in the middle + for i in range(num_samples): + samples.append(lines[randint(num_samples, len(lines) - num_samples)]) + # last n samples + samples.extend(lines[-num_samples:]) + return samples + + +if __name__ == '__main__': + args = parser.parse_args() + manifest_dir = Path(args.manifests_dir) + manifests = sorted(list(manifest_dir.glob('*_high_score_manifest.json'))) + sample_manifest = os.path.join(args.output_dir, 'sample_manifest.json') + all_manifest = os.path.join(args.output_dir, 'all_manifest.json') + + if args.num_samples > 0: + with open(sample_manifest, 'w', encoding='utf8') as f: + for manifest in manifests: + samples = get_samples(manifest, args.num_samples) + for sample in samples: + f.write(sample) + + print(f'Sample manifest is saved at {sample_manifest}') + + total_duration = 0 + with open(all_manifest, 'w') as out_f: + for manifest in manifests: + with open(manifest, 'r', encoding='utf8') as f: + for line in f: + info = json.loads(line) + total_duration += info['duration'] + out_f.write(line) + + print(f'Aggregated manifest is saved at {all_manifest}') + print(f'Total files duration: ~{round(total_duration/60)} min or ~{round(total_duration/60/60)} hr') + + sets = {'_high_score_manifest.json': 0, '_low_score_manifest.json': 0, '_del_manifest.json': 0} + + for k in sets: + files = manifest_dir.glob(f'*{k}') + for manifest_f in files: + with open(manifest_f, 'r') as f: + for line in f: + line = json.loads(line) + sets[k] += line['duration'] + + total = 0 + for k in sets: + sets[k] = round(sets[k] / 3600, 1) + total += sets[k] + name = k.split('_')[1].upper() + print(f'Total {name} score duration: ~{sets[k]} hr') + print(f'Total duration: ~{total} hr') + print('Done.') diff --git a/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/run_ctc_segmentation.py b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/run_ctc_segmentation.py new file mode 100644 index 0000000000..06b4f3669b --- /dev/null +++ b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/run_ctc_segmentation.py @@ -0,0 +1,180 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import multiprocessing +import os +import sys +import time +from pathlib import Path + +import numpy as np +import scipy.io.wavfile as wav +import torch +from utils import get_segments, listener_configurer, listener_process, worker_configurer, worker_process + +import nemo.collections.asr as nemo_asr + +parser = argparse.ArgumentParser(description="CTC Segmentation") +parser.add_argument("--output_dir", default='output', type=str, help='Path to output directory') +parser.add_argument( + "--data", + type=str, + required=True, + help='Path to directory with audio files and associated transcripts (same respective names only formats are ' + 'different or path to wav file (transcript should have the same base name and be located in the same folder' + 'as the wav file.', +) +parser.add_argument('--window_len', type=int, default=8000, help='Window size for ctc segmentation algorithm') +parser.add_argument('--no_parallel', action='store_true', help='Flag to disable parallel segmentation') +parser.add_argument('--sample_rate', type=int, default=16000, help='Sampling rate') +parser.add_argument( + '--model', type=str, default='QuartzNet15x5Base-En', help='Path to model checkpoint or pre-trained model name', +) +parser.add_argument('--debug', action='store_true', help='Flag to enable debugging messages') + +logger = logging.getLogger('ctc_segmentation') # use module name + +if __name__ == '__main__': + + args = parser.parse_args() + + # setup logger + log_dir = os.path.join(args.output_dir, 'logs') + os.makedirs(log_dir, exist_ok=True) + log_file = os.path.join(log_dir, f'ctc_segmentation_{args.window_len}.log') + level = 'DEBUG' if args.debug else 'INFO' + + logger = logging.getLogger('CTC') + file_handler = logging.FileHandler(filename=log_file) + stdout_handler = logging.StreamHandler(sys.stdout) + handlers = [file_handler, stdout_handler] + logging.basicConfig(handlers=handlers, level=level) + + if os.path.exists(args.model): + asr_model = nemo_asr.models.EncDecCTCModel.restore_from(args.model) + elif args.model in nemo_asr.models.EncDecCTCModel.get_available_model_names(): + asr_model = nemo_asr.models.EncDecCTCModel.from_pretrained(args.model, strict=False) + else: + raise ValueError( + f'{args.model} not a valid model name or path. Provide path to the pre-trained checkpoint ' + f'or choose from {nemo_asr.models.EncDecCTCModel.list_available_models()}' + ) + + # extract ASR vocabulary and add blank symbol + vocabulary = asr_model.cfg.decoder.vocabulary + odim = len(vocabulary) + 1 + logging.debug(f'ASR Model vocabulary: {vocabulary}') + + # add blank to vocab + vocabulary = ["ε"] + list(vocabulary) + data = Path(args.data) + output_dir = Path(args.output_dir) + + if os.path.isdir(data): + audio_paths = data.glob("*.wav") + data_dir = data + else: + audio_paths = [Path(data)] + data_dir = Path(os.path.dirname(data)) + + all_log_probs = [] + all_transcript_file = [] + all_segment_file = [] + all_wav_paths = [] + segments_dir = os.path.join(args.output_dir, 'segments') + os.makedirs(segments_dir, exist_ok=True) + for path_audio in audio_paths: + transcript_file = os.path.join(data_dir, path_audio.name.replace(".wav", ".txt")) + segment_file = os.path.join( + segments_dir, f"{args.window_len}_" + path_audio.name.replace(".wav", "_segments.txt") + ) + try: + sample_rate, signal = wav.read(path_audio) + if sample_rate != args.sample_rate: + raise ValueError( + f'Sampling rate of the audio file {path_audio} doesn\'t match --sample_rate={args.sample_rate}' + ) + except ValueError: + logging.error( + f"{path_audio} should be a .wav mono file with the sampling rate used for the ASR model training" + f"specified with {args.sample_rate}." + ) + raise + + original_duration = len(signal) / sample_rate + logging.debug(f'Duration: {original_duration}s, file_name: {path_audio}') + log_probs = asr_model.transcribe(paths2audio_files=[str(path_audio)], batch_size=1, logprobs=True)[0] + + # move blank values to the first column + blank_col = log_probs[:, -1].reshape((log_probs.shape[0], 1)) + log_probs = np.concatenate((blank_col, log_probs[:, :-1]), axis=1) + + all_log_probs.append(log_probs) + all_segment_file.append(str(segment_file)) + all_transcript_file.append(str(transcript_file)) + all_wav_paths.append(path_audio) + + del asr_model + torch.cuda.empty_cache() + + if len(all_log_probs) == 0: + raise ValueError(f'No valid audio files found at {args.data}') + start_time = time.time() + if args.no_parallel: + for i in range(len(all_log_probs)): + get_segments( + all_log_probs[i], + all_wav_paths[i], + all_transcript_file[i], + all_segment_file[i], + vocabulary, + args.window_len, + ) + else: + queue = multiprocessing.Queue(-1) + + listener = multiprocessing.Process(target=listener_process, args=(queue, listener_configurer, log_file, level)) + listener.start() + workers = [] + for i in range(len(all_log_probs)): + worker = multiprocessing.Process( + target=worker_process, + args=( + queue, + worker_configurer, + level, + all_log_probs[i], + all_wav_paths[i], + all_transcript_file[i], + all_segment_file[i], + vocabulary, + args.window_len, + ), + ) + workers.append(worker) + worker.start() + for w in workers: + w.join() + queue.put_nowait(None) + listener.join() + + total_time = time.time() - start_time + logger.info(f'Total execution time: ~{round(total_time/60)}min') + logger.info(f'Saving logs to {log_file}') + + if os.path.exists(log_file): + with open(log_file, 'r') as f: + lines = f.readlines() diff --git a/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/utils.py b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/utils.py new file mode 100644 index 0000000000..713f8d4c5f --- /dev/null +++ b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/utils.py @@ -0,0 +1,173 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import logging.handlers +import multiprocessing +import os +from pathlib import PosixPath +from typing import List, Tuple, Union + +import ctc_segmentation as cs +import numpy as np + + +def get_segments( + log_probs: np.ndarray, + path_wav: Union[PosixPath, str], + transcript_file: Union[PosixPath, str], + output_file: str, + vocabulary: List[str], + window_size: int = 8000, + frame_duration_ms: int = 20, +) -> None: + """ + Segments the audio into segments and saves segments timings to a file + + Args: + log_probs: Log probabilities for the original audio from an ASR model, shape T * |vocabulary|. + values for blank should be at position 0 + path_wav: path to the audio .wav file + transcript_file: path to + output_file: path to the file to save timings for segments + vocabulary: vocabulary used to train the ASR model, note blank is at position 0 + window_size: the length of each utterance (in terms of frames of the CTC outputs) fits into that window. + frame_duration_ms: frame duration in ms + """ + config = cs.CtcSegmentationParameters() + config.char_list = vocabulary + config.min_window_size = window_size + config.frame_duration_ms = frame_duration_ms + config.blank = config.space + config.subsampling_factor = 2 + + with open(transcript_file, "r") as f: + text = f.readlines() + text = [t.strip() for t in text if t.strip()] + + # add corresponding original text without pre-processing + transcript_file_no_preprocessing = transcript_file.replace('.txt', '_with_punct.txt') + if not os.path.exists(transcript_file_no_preprocessing): + raise ValueError(f'{transcript_file_no_preprocessing} not found.') + + with open(transcript_file_no_preprocessing, "r") as f: + text_no_preprocessing = f.readlines() + text_no_preprocessing = [t.strip() for t in text_no_preprocessing if t.strip()] + + # add corresponding normalized original text + transcript_file_normalized = transcript_file.replace('.txt', '_with_punct_normalized.txt') + if not os.path.exists(transcript_file_normalized): + raise ValueError(f'{transcript_file_normalized} not found.') + + with open(transcript_file_normalized, "r") as f: + text_normalized = f.readlines() + text_normalized = [t.strip() for t in text_normalized if t.strip()] + + if len(text_no_preprocessing) != len(text): + raise ValueError(f'{transcript_file} and {transcript_file_no_preprocessing} do not match') + + if len(text_normalized) != len(text): + raise ValueError(f'{transcript_file} and {transcript_file_normalized} do not match') + + ground_truth_mat, utt_begin_indices = cs.prepare_text(config, text) + logging.debug(f"Syncing {transcript_file}") + logging.debug( + f"Audio length {os.path.basename(path_wav)}: {log_probs.shape[0]}. " + f"Text length {os.path.basename(transcript_file)}: {len(ground_truth_mat)}" + ) + + timings, char_probs, char_list = cs.ctc_segmentation(config, log_probs, ground_truth_mat) + segments = cs.determine_utterance_segments(config, utt_begin_indices, char_probs, timings, text) + write_output(output_file, path_wav, segments, text, text_no_preprocessing, text_normalized) + + +def write_output( + out_path: str, + path_wav: str, + segments: List[Tuple[float]], + text: str, + text_no_preprocessing: str, + text_normalized: str, + stride: int = 2, +): + """ + Write the segmentation output to a file + + out_path: Path to output file + path_wav: Path to the original audio file + segments: Segments include start, end and alignment score + text: Text used for alignment + text_no_preprocessing: Reference txt without any pre-processing + text_normalized: Reference text normalized + stride: Stride applied to an ASR input + """ + # Uses char-wise alignments to get utterance-wise alignments and writes them into the given file + with open(str(out_path), "w") as outfile: + outfile.write(str(path_wav) + "\n") + + for i, (start, end, score) in enumerate(segments): + outfile.write( + f'{start/stride} {end/stride} {score} | {text[i]} | {text_no_preprocessing[i]} | {text_normalized[i]}\n' + ) + + +##################### +# logging utils +##################### +def listener_configurer(log_file, level): + root = logging.getLogger() + h = logging.handlers.RotatingFileHandler(log_file, 'w') + f = logging.Formatter('%(asctime)s %(processName)-10s %(name)s %(levelname)-8s %(message)s') + h.setFormatter(f) + ch = logging.StreamHandler() + root.addHandler(h) + root.setLevel(level) + root.addHandler(ch) + + +def listener_process(queue, configurer, log_file, level): + configurer(log_file, level) + while True: + try: + record = queue.get() + if record is None: # We send this as a sentinel to tell the listener to quit. + break + logger = logging.getLogger(record.name) + logger.setLevel(logging.INFO) + logger.handle(record) # No level or filter logic applied - just do it! + + except Exception: + import sys + import traceback + + print('Problem:', file=sys.stderr) + traceback.print_exc(file=sys.stderr) + + +def worker_configurer(queue, level): + h = logging.handlers.QueueHandler(queue) # Just the one handler needed + root = logging.getLogger() + root.addHandler(h) + root.setLevel(level) + + +def worker_process( + queue, configurer, level, log_probs, path_wav, transcript_file, output_file, vocabulary, window_len +): + configurer(queue, level) + name = multiprocessing.current_process().name + innerlogger = logging.getLogger('worker') + innerlogger.info(f'{name} is processing {path_wav}, window_len={window_len}') + get_segments(log_probs, path_wav, transcript_file, output_file, vocabulary, window_len) + innerlogger.info(f'{name} completed segmentation of {path_wav}, segments saved to {output_file}') diff --git a/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/verify_segments.py b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/verify_segments.py new file mode 100644 index 0000000000..9d648df912 --- /dev/null +++ b/models/tacotron2/NeMo/tools/ctc_segmentation/scripts/verify_segments.py @@ -0,0 +1,143 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import re +from pathlib import Path + +import pandas as pd + +parser = argparse.ArgumentParser(description="Compare alignment segments generated with different window sizes") +parser.add_argument( + "--base_dir", + default='output', + type=str, + required=True, + help='Path to directory with "logs" and "segments" folders generated during the segmentation step', +) + + +def _process_log_file(log_file: str): + processes = {} + with open(log_file, 'r') as f: + for line in f: + if line[0].isdigit() and 'Process-' in line: + process_id = re.search(r'Process-\d+', line)[0] + window_size = re.findall(r'\d+', line)[-1] + path = re.search(r'\/.*?\.[\w:]+', line) + if 'is processing' in line: + if process_id not in processes: + processes[process_id] = {} + processes[process_id]['Window Increased'] = False + processes[process_id]['Start Window'] = window_size + processes[process_id]['Final Window'] = window_size + base_dir, base_name = os.path.split(path[0]) + processes[process_id]['Audio Dir'] = base_dir + processes[process_id]['Audio File'] = base_name + elif 'Increasing' in line: + processes[process_id]['Final Window'] = window_size + processes[process_id]['Window Increased'] = True + elif 'completed' in line: + processes[process_id]['Completed'] = True + + df = pd.DataFrame.from_dict( + processes, + columns=['Audio Dir', 'Audio File', 'Start Window', 'Final Window', 'Window Increased', 'Completed'], + orient='index', + ).reset_index() + return df + + +if __name__ == '__main__': + args = parser.parse_args() + log_dir = os.path.join(args.base_dir, 'logs') + if not os.path.exists(log_dir): + raise ValueError(f'"log" directory was not found at {args.base_dir}.') + + log_files = Path(log_dir).glob('*.log') + dfs = [] + for log in log_files: + dfs.append(_process_log_file(log)) + + dfs = pd.concat(dfs) + summary_file = os.path.join(log_dir, 'log_summary.csv') + dfs.to_csv(summary_file, index=False) + print(f'Log summary saved to {summary_file}') + + segments_dir = os.path.join(args.base_dir, 'segments') + if not os.path.exists(segments_dir): + raise ValueError(f'"segments" directory was not found at {args.base_dir}.') + + all_files = Path(segments_dir).glob('*_segments.txt') + all_alignment_files = {} + for file in all_files: + base_name = re.sub(r'^\d+_', '', file.name) + if base_name not in all_alignment_files: + all_alignment_files[base_name] = [] + all_alignment_files[base_name].append(file) + + verified_dir = os.path.join(args.base_dir, 'verified_segments') + os.makedirs(verified_dir, exist_ok=True) + + def readlines(file): + with open(file, 'r') as f: + lines = f.readlines() + return lines + + stats = {} + for part, alignment_files in all_alignment_files.items(): + stats[part] = {} + num_alignment_files = len(alignment_files) + all_alignments = [] + for alignment in alignment_files: + all_alignments.append(readlines(alignment)) + + with open(os.path.join(verified_dir, part), 'w') as f: + num_segments = len(all_alignments[0]) + stats[part]['Original number of segments'] = num_segments + stats[part]['Verified segments'] = 0 + stats[part]['Original Duration, min'] = 0 + stats[part]['Verified Duration, min'] = 0 + + for i in range(num_segments): + line = all_alignments[0][i] + valid_line = True + if i == 0: + duration = 0 + else: + info = line.split('|')[0].split() + duration = (float(info[1]) - float(info[0])) / 60 + stats[part]['Original Duration, min'] += duration + for alignment in all_alignments: + if line != alignment[i]: + valid_line = False + if valid_line: + f.write(line) + stats[part]['Verified segments'] += 1 + stats[part]['Verified Duration, min'] += duration + + stats = pd.DataFrame.from_dict(stats, orient='index').reset_index() + stats['Number dropped'] = stats['Original number of segments'] - stats['Verified segments'] + stats['Duration of dropped, min'] = round(stats['Original Duration, min'] - stats['Verified Duration, min']) + stats['% dropped, min'] = round(stats['Duration of dropped, min'] / stats['Original number of segments'] * 100) + stats['Misalignment present'] = stats['Number dropped'] > 0 + stats['Original Duration, min'] = round(stats['Original Duration, min']) + stats['Verified Duration, min'] = round(stats['Verified Duration, min']) + stats.loc["Total"] = stats.sum() + + stats_file = os.path.join(args.base_dir, 'alignment_summary.csv') + stats.to_csv(stats_file, index=False) + print(stats) + print(f'Alignment summary saved to {stats_file}') diff --git a/models/tacotron2/NeMo/tools/speech_data_explorer/README.md b/models/tacotron2/NeMo/tools/speech_data_explorer/README.md new file mode 100644 index 0000000000..9d431f20fd --- /dev/null +++ b/models/tacotron2/NeMo/tools/speech_data_explorer/README.md @@ -0,0 +1,26 @@ +Speech Data Explorer +-------------------- + +[Dash](https://plotly.com/dash/)-based tool for interactive exploration of ASR/TTS datasets. + +Features: +- dataset's statistics (alphabet, vocabulary, duration-based histograms) +- navigation across dataset (sorting, filtering) +- inspection of individual utterances (waveform, spectrogram, audio player) +- errors' analysis (Word Error Rate, Character Error Rate, Word Match Rate, Mean Word Accuracy, diff) + +Please make sure that requirements are installed. Then run: +``` +python data_explorer.py path_to_manifest.json +``` + +JSON manifest file should contain the following fields: +- "audio_filepath" (path to audio file) +- "duration" (duration of the audio file in seconds) +- "text" (reference transcript) + +Errors' analysis requires "pred_text" (ASR transcript) for all utterances. + +Any additional field will be parsed and displayed in 'Samples' tab. + +![Speech Data Explorer](screenshot.png) diff --git a/models/tacotron2/NeMo/tools/speech_data_explorer/data_explorer.py b/models/tacotron2/NeMo/tools/speech_data_explorer/data_explorer.py new file mode 100755 index 0000000000..e760915347 --- /dev/null +++ b/models/tacotron2/NeMo/tools/speech_data_explorer/data_explorer.py @@ -0,0 +1,717 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import base64 +import datetime +import difflib +import io +import json +import math +import operator +import os +import pickle +from collections import defaultdict + +import dash +import dash_bootstrap_components as dbc +import dash_core_components as dcc +import dash_html_components as html +import dash_table +import diff_match_patch +import editdistance +import librosa +import numpy as np +import soundfile as sf +import tqdm +from dash.dependencies import Input, Output +from dash.exceptions import PreventUpdate +from plotly import express as px +from plotly import graph_objects as go +from plotly.subplots import make_subplots + +# number of items in a table per page +DATA_PAGE_SIZE = 10 + +# operators for filtering items +filter_operators = [ + ['ge ', '>='], + ['le ', '<='], + ['lt ', '<'], + ['gt ', '>'], + ['ne ', '!='], + ['eq ', '='], + ['contains '], +] + +# parse table filter queries +def split_filter_part(filter_part): + for operator_type in filter_operators: + for op in operator_type: + if op in filter_part: + name_part, value_part = filter_part.split(op, 1) + name = name_part[name_part.find('{') + 1 : name_part.rfind('}')] + value_part = value_part.strip() + v0 = value_part[0] + if v0 == value_part[-1] and v0 in ("'", '"', '`'): + value = value_part[1:-1].replace('\\' + v0, v0) + else: + try: + value = float(value_part) + except ValueError: + value = value_part + return name, operator_type[0].strip(), value + return [None] * 3 + + +# standard command-line arguments parser +def parse_args(): + parser = argparse.ArgumentParser(description='Speech Data Explorer') + parser.add_argument( + 'manifest', help='path to JSON manifest file', + ) + parser.add_argument('--vocab', help='optional vocabulary to highlight OOV words') + parser.add_argument('--port', default='8050', help='serving port for establishing connection') + parser.add_argument( + '--disable-caching-metrics', action='store_true', help='disable caching metrics for errors analysis' + ) + parser.add_argument('--debug', '-d', action='store_true', help='enable debug mode') + args = parser.parse_args() + print(args) + return args + + +# load data from JSON manifest file +def load_data(data_filename, disable_caching, vocab=None): + if not disable_caching: + pickle_filename = data_filename.split('.json')[0] + json_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(data_filename)) + timestamp = json_mtime.strftime('%Y%m%d_%H%M') + pickle_filename += '_' + timestamp + '.pkl' + if os.path.exists(pickle_filename): + with open(pickle_filename, 'rb') as f: + data, wer, cer, wmr, mwa, num_hours, vocabulary_data, alphabet, metrics_available = pickle.load(f) + return data, wer, cer, wmr, mwa, num_hours, vocabulary_data, alphabet, metrics_available + + if vocab is not None: + # load external vocab + vocabulary_ext = {} + with open(vocab, 'r') as f: + for line in f: + if '\t' in line: + # parse word from TSV file + word = line.split('\t')[0] + else: + # assume each line contains just a single word + word = line.strip() + vocabulary_ext[word] = 1 + + data = [] + wer_dist = 0.0 + wer_count = 0 + cer_dist = 0.0 + cer_count = 0 + wmr_count = 0 + wer = 0 + cer = 0 + wmr = 0 + mwa = 0 + num_hours = 0 + vocabulary = defaultdict(lambda: 0) + alphabet = set() + match_vocab = defaultdict(lambda: 0) + + sm = difflib.SequenceMatcher() + metrics_available = False + with open(data_filename, 'r', encoding='utf8') as f: + for line in tqdm.tqdm(f): + item = json.loads(line) + num_words = len(item['text'].split()) + num_chars = len(item['text']) + orig = item['text'].split() + for word in orig: + vocabulary[word] += 1 + for char in item['text']: + alphabet.add(char) + num_hours += item['duration'] + + if 'pred_text' in item: + metrics_available = True + pred = item['pred_text'].split() + word_dist = editdistance.eval(orig, pred) + char_dist = editdistance.eval(item['text'], item['pred_text']) + wer_dist += word_dist + cer_dist += char_dist + wer_count += num_words + cer_count += num_chars + + sm.set_seqs(orig, pred) + num_matches = 0 + for m in sm.get_matching_blocks(): + for word_idx in range(m[0], m[0] + m[2]): + match_vocab[orig[word_idx]] += 1 + num_matches += 1 + wmr_count += num_matches + + data.append( + { + 'audio_filepath': item['audio_filepath'], + 'duration': round(item['duration'], 2), + 'num_words': num_words, + 'num_chars': num_chars, + 'word_rate': round(num_words / item['duration'], 2), + 'char_rate': round(num_chars / item['duration'], 2), + 'text': item['text'], + } + ) + if metrics_available: + data[-1]['pred_text'] = item['pred_text'] + data[-1]['WER'] = round(word_dist / num_words * 100.0, 2) + data[-1]['CER'] = round(char_dist / num_chars * 100.0, 2) + data[-1]['WMR'] = round(num_matches / len(orig) * 100.0, 2) + for k in item: + if k not in data[-1]: + data[-1][k] = item[k] + + vocabulary_data = [{'word': word, 'count': vocabulary[word]} for word in vocabulary] + if vocab is not None: + for item in vocabulary_data: + item['OOV'] = item['word'] not in vocabulary_ext + + if metrics_available: + wer = wer_dist / wer_count * 100.0 + cer = cer_dist / cer_count * 100.0 + wmr = wmr_count / wer_count * 100.0 + + acc_sum = 0 + for item in vocabulary_data: + w = item['word'] + word_accuracy = match_vocab[w] / vocabulary[w] * 100.0 + acc_sum += word_accuracy + item['accuracy'] = round(word_accuracy, 1) + mwa = acc_sum / len(vocabulary_data) + + num_hours /= 3600.0 + + if not disable_caching: + with open(pickle_filename, 'wb') as f: + pickle.dump( + [data, wer, cer, wmr, mwa, num_hours, vocabulary_data, alphabet, metrics_available], + f, + pickle.HIGHEST_PROTOCOL, + ) + + return data, wer, cer, wmr, mwa, num_hours, vocabulary_data, alphabet, metrics_available + + +# plot histogram of specified field in data list +def plot_histogram(data, key, label): + fig = px.histogram( + data_frame=[item[key] for item in data], + nbins=50, + log_y=True, + labels={'value': label}, + opacity=0.5, + color_discrete_sequence=['green'], + height=200, + ) + fig.update_layout(showlegend=False, margin=dict(l=0, r=0, t=0, b=0, pad=0)) + return fig + + +def plot_word_accuracy(vocabulary_data): + labels = ['Unrecognized', 'Sometimes recognized', 'Always recognized'] + counts = [0, 0, 0] + for word in vocabulary_data: + if word['accuracy'] == 0: + counts[0] += 1 + elif word['accuracy'] < 100: + counts[1] += 1 + else: + counts[2] += 1 + colors = ['red', 'orange', 'green'] + + fig = go.Figure( + data=[ + go.Bar( + x=labels, + y=counts, + marker_color=colors, + text=['{:.2%}'.format(count / sum(counts)) for count in counts], + textposition='auto', + ) + ] + ) + fig.update_layout( + showlegend=False, margin=dict(l=0, r=0, t=0, b=0, pad=0), height=200, yaxis={'title_text': '#words'} + ) + + return fig + + +args = parse_args() +print('Loading data...') +data, wer, cer, wmr, mwa, num_hours, vocabulary, alphabet, metrics_available = load_data( + args.manifest, args.disable_caching_metrics, args.vocab +) +print('Starting server...') +app = dash.Dash( + __name__, + suppress_callback_exceptions=True, + external_stylesheets=[dbc.themes.BOOTSTRAP], + title=os.path.basename(args.manifest), +) + + +figure_duration = plot_histogram(data, 'duration', 'Duration (sec)') +figure_num_words = plot_histogram(data, 'num_words', '#words') +figure_num_chars = plot_histogram(data, 'num_chars', '#chars') +figure_word_rate = plot_histogram(data, 'word_rate', '#words/sec') +figure_char_rate = plot_histogram(data, 'char_rate', '#chars/sec') + +if metrics_available: + figure_wer = plot_histogram(data, 'WER', 'WER, %') + figure_cer = plot_histogram(data, 'CER', 'CER, %') + figure_wmr = plot_histogram(data, 'WMR', 'WMR, %') + figure_word_acc = plot_word_accuracy(vocabulary) + +stats_layout = [ + dbc.Row(dbc.Col(html.H5(children='Global Statistics'), className='text-secondary'), className='mt-3'), + dbc.Row( + [ + dbc.Col(html.Div('Number of hours', className='text-secondary'), width=3, className='border-right'), + dbc.Col(html.Div('Number of utterances', className='text-secondary'), width=3, className='border-right'), + dbc.Col(html.Div('Vocabulary size', className='text-secondary'), width=3, className='border-right'), + dbc.Col(html.Div('Alphabet size', className='text-secondary'), width=3), + ], + className='bg-light mt-2 rounded-top border-top border-left border-right', + ), + dbc.Row( + [ + dbc.Col( + html.H5( + '{:.2f} hours'.format(num_hours), + className='text-center p-1', + style={'color': 'green', 'opacity': 0.7}, + ), + width=3, + className='border-right', + ), + dbc.Col( + html.H5(len(data), className='text-center p-1', style={'color': 'green', 'opacity': 0.7}), + width=3, + className='border-right', + ), + dbc.Col( + html.H5( + '{} words'.format(len(vocabulary)), + className='text-center p-1', + style={'color': 'green', 'opacity': 0.7}, + ), + width=3, + className='border-right', + ), + dbc.Col( + html.H5( + '{} chars'.format(len(alphabet)), + className='text-center p-1', + style={'color': 'green', 'opacity': 0.7}, + ), + width=3, + ), + ], + className='bg-light rounded-bottom border-bottom border-left border-right', + ), +] +if metrics_available: + stats_layout += [ + dbc.Row( + [ + dbc.Col( + html.Div('Word Error Rate (WER), %', className='text-secondary'), width=3, className='border-right' + ), + dbc.Col( + html.Div('Character Error Rate (CER), %', className='text-secondary'), + width=3, + className='border-right', + ), + dbc.Col( + html.Div('Word Match Rate (WMR), %', className='text-secondary'), + width=3, + className='border-right', + ), + dbc.Col(html.Div('Mean Word Accuracy, %', className='text-secondary'), width=3), + ], + className='bg-light mt-2 rounded-top border-top border-left border-right', + ), + dbc.Row( + [ + dbc.Col( + html.H5( + '{:.2f}'.format(wer), className='text-center p-1', style={'color': 'green', 'opacity': 0.7}, + ), + width=3, + className='border-right', + ), + dbc.Col( + html.H5( + '{:.2f}'.format(cer), className='text-center p-1', style={'color': 'green', 'opacity': 0.7} + ), + width=3, + className='border-right', + ), + dbc.Col( + html.H5( + '{:.2f}'.format(wmr), className='text-center p-1', style={'color': 'green', 'opacity': 0.7}, + ), + width=3, + className='border-right', + ), + dbc.Col( + html.H5( + '{:.2f}'.format(mwa), className='text-center p-1', style={'color': 'green', 'opacity': 0.7}, + ), + width=3, + ), + ], + className='bg-light rounded-bottom border-bottom border-left border-right', + ), + ] +stats_layout += [ + dbc.Row(dbc.Col(html.H5(children='Alphabet'), className='text-secondary'), className='mt-3'), + dbc.Row( + dbc.Col(html.Div('{}'.format(sorted(alphabet))),), className='mt-2 bg-light text-monospace rounded border' + ), + dbc.Row(dbc.Col(html.H5('Duration (per utterance)'), className='text-secondary'), className='mt-3'), + dbc.Row(dbc.Col(dcc.Graph(id='duration-graph', figure=figure_duration),),), + dbc.Row(dbc.Col(html.H5('Number of words (per utterance)'), className='text-secondary'), className='mt-3'), + dbc.Row(dbc.Col(dcc.Graph(id='num-words-graph', figure=figure_num_words),),), + dbc.Row(dbc.Col(html.H5('Number of characters (per utterance)'), className='text-secondary'), className='mt-3'), + dbc.Row(dbc.Col(dcc.Graph(id='num-chars-graph', figure=figure_num_chars),),), + dbc.Row(dbc.Col(html.H5('Word rate (per utterance)'), className='text-secondary'), className='mt-3'), + dbc.Row(dbc.Col(dcc.Graph(id='word-rate-graph', figure=figure_word_rate),),), + dbc.Row(dbc.Col(html.H5('Character rate (per utterance)'), className='text-secondary'), className='mt-3'), + dbc.Row(dbc.Col(dcc.Graph(id='char-rate-graph', figure=figure_char_rate),),), +] + +if metrics_available: + stats_layout += [ + dbc.Row(dbc.Col(html.H5('WER (per utterance)'), className='text-secondary'), className='mt-3'), + dbc.Row(dbc.Col(dcc.Graph(id='wer-graph', figure=figure_wer),),), + dbc.Row(dbc.Col(html.H5('CER (per utterance)'), className='text-secondary'), className='mt-3'), + dbc.Row(dbc.Col(dcc.Graph(id='cer-graph', figure=figure_cer),),), + dbc.Row(dbc.Col(html.H5('WMR (per utterance)'), className='text-secondary'), className='mt-3'), + dbc.Row(dbc.Col(dcc.Graph(id='wmr-graph', figure=figure_wmr),),), + dbc.Row(dbc.Col(html.H5('Word accuracy distribution'), className='text-secondary'), className='mt-3'), + dbc.Row(dbc.Col(dcc.Graph(id='word-acc-graph', figure=figure_word_acc),),), + ] + +wordstable_columns = [{'name': 'Word', 'id': 'word'}, {'name': 'Count', 'id': 'count'}] +if args.vocab is not None: + wordstable_columns.append({'name': 'OOV', 'id': 'OOV'}) +if metrics_available: + wordstable_columns.append({'name': 'Accuracy, %', 'id': 'accuracy'}) + +stats_layout += [ + dbc.Row(dbc.Col(html.H5('Vocabulary'), className='text-secondary'), className='mt-3'), + dbc.Row( + dbc.Col( + dash_table.DataTable( + id='wordstable', + columns=wordstable_columns, + filter_action='custom', + filter_query='', + sort_action='custom', + sort_mode='single', + page_action='custom', + page_current=0, + page_size=DATA_PAGE_SIZE, + cell_selectable=False, + page_count=math.ceil(len(vocabulary) / DATA_PAGE_SIZE), + sort_by=[{'column_id': 'word', 'direction': 'asc'}], + style_cell={'maxWidth': 0, 'textAlign': 'left'}, + style_header={'color': 'text-primary'}, + ), + ), + className='m-2', + ), +] + + +@app.callback( + [Output('wordstable', 'data'), Output('wordstable', 'page_count')], + [Input('wordstable', 'page_current'), Input('wordstable', 'sort_by'), Input('wordstable', 'filter_query')], +) +def update_wordstable(page_current, sort_by, filter_query): + vocabulary_view = vocabulary + filtering_expressions = filter_query.split(' && ') + for filter_part in filtering_expressions: + col_name, op, filter_value = split_filter_part(filter_part) + + if op in ('eq', 'ne', 'lt', 'le', 'gt', 'ge'): + vocabulary_view = [x for x in vocabulary_view if getattr(operator, op)(x[col_name], filter_value)] + print(len(vocabulary_view), len(vocabulary)) + elif op == 'contains': + vocabulary_view = [x for x in vocabulary_view if filter_value in str(x[col_name])] + print(len(vocabulary_view), len(vocabulary)) + + if len(sort_by): + col = sort_by[0]['column_id'] + descending = sort_by[0]['direction'] == 'desc' + vocabulary_view = sorted(vocabulary_view, key=lambda x: x[col], reverse=descending) + if page_current * DATA_PAGE_SIZE >= len(vocabulary_view): + page_current = len(vocabulary_view) // DATA_PAGE_SIZE + return [ + vocabulary_view[page_current * DATA_PAGE_SIZE : (page_current + 1) * DATA_PAGE_SIZE], + math.ceil(len(vocabulary_view) / DATA_PAGE_SIZE), + ] + + +samples_layout = [ + dbc.Row(dbc.Col(html.H5('Data'), className='text-secondary'), className='mt-3'), + dbc.Row( + dbc.Col( + dash_table.DataTable( + id='datatable', + columns=[{'name': k.replace('_', ' '), 'id': k, 'hideable': True} for k in data[0]], + filter_action='custom', + filter_query='', + sort_action='custom', + sort_mode='single', + sort_by=[], + row_selectable='single', + selected_rows=[0], + page_action='custom', + page_current=0, + page_size=DATA_PAGE_SIZE, + page_count=math.ceil(len(data) / DATA_PAGE_SIZE), + style_cell={'overflow': 'hidden', 'textOverflow': 'ellipsis', 'maxWidth': 0, 'textAlign': 'center'}, + style_header={ + 'color': 'text-primary', + 'text_align': 'center', + 'height': 'auto', + 'whiteSpace': 'normal', + }, + css=[{'selector': '.dash-spreadsheet-menu', 'rule': 'position:absolute; bottom:-36px'}], + ), + ) + ), +] + [ + dbc.Row( + [ + dbc.Col( + html.Div(children=k.replace('_', ' ')), + width=2, + className='mt-1 bg-light text-monospace text-break small rounded border', + ), + dbc.Col(html.Div(id='_' + k), className='mt-1 bg-light text-monospace text-break small rounded border'), + ] + ) + for k in data[0] +] + +if metrics_available: + samples_layout += [ + dbc.Row( + [ + dbc.Col( + html.Div(children='diff'), + width=2, + className='mt-1 bg-light text-monospace text-break small rounded border', + ), + dbc.Col( + html.Iframe( + id='_diff', + sandbox='', + srcDoc='', + style={'border': 'none', 'width': '100%', 'height': '100%'}, + className='bg-light text-monospace text-break small', + ), + className='mt-1 bg-light text-monospace text-break small rounded border', + ), + ] + ) + ] +samples_layout += [ + dbc.Row(dbc.Col(html.Audio(id='player', controls=True),), className='mt-3 '), + dbc.Row(dbc.Col(dcc.Graph(id='signal-graph')), className='mt-3'), +] + + +@app.callback( + [Output('datatable', 'data'), Output('datatable', 'page_count')], + [Input('datatable', 'page_current'), Input('datatable', 'sort_by'), Input('datatable', 'filter_query')], +) +def update_datatable(page_current, sort_by, filter_query): + data_view = data + filtering_expressions = filter_query.split(' && ') + for filter_part in filtering_expressions: + col_name, op, filter_value = split_filter_part(filter_part) + + if op in ('eq', 'ne', 'lt', 'le', 'gt', 'ge'): + data_view = [x for x in data_view if getattr(operator, op)(x[col_name], filter_value)] + elif op == 'contains': + data_view = [x for x in data_view if filter_value in str(x[col_name])] + + if len(sort_by): + col = sort_by[0]['column_id'] + descending = sort_by[0]['direction'] == 'desc' + data_view = sorted(data_view, key=lambda x: x[col], reverse=descending) + if page_current * DATA_PAGE_SIZE >= len(data_view): + page_current = len(data_view) // DATA_PAGE_SIZE + return [ + data_view[page_current * DATA_PAGE_SIZE : (page_current + 1) * DATA_PAGE_SIZE], + math.ceil(len(data_view) / DATA_PAGE_SIZE), + ] + + +app.layout = html.Div( + [ + dcc.Location(id='url', refresh=False), + dbc.NavbarSimple( + children=[ + dbc.NavItem(dbc.NavLink('Statistics', id='stats_link', href='/', active=True)), + dbc.NavItem(dbc.NavLink('Samples', id='samples_link', href='/samples')), + ], + brand='Speech Data Explorer', + sticky='top', + color='green', + dark=True, + ), + dbc.Container(id='page-content'), + ] +) + + +@app.callback( + [Output('page-content', 'children'), Output('stats_link', 'active'), Output('samples_link', 'active')], + [Input('url', 'pathname')], +) +def nav_click(url): + if url == '/samples': + return [samples_layout, False, True] + else: + return [stats_layout, True, False] + + +@app.callback( + [Output('_' + k, 'children') for k in data[0]], [Input('datatable', 'selected_rows'), Input('datatable', 'data')] +) +def show_item(idx, data): + if len(idx) == 0: + raise PreventUpdate + return [data[idx[0]][k] for k in data[0]] + + +@app.callback(Output('_diff', 'srcDoc'), [Input('datatable', 'selected_rows'), Input('datatable', 'data')]) +def show_diff(idx, data): + if len(idx) == 0: + raise PreventUpdate + + orig_words = data[idx[0]]['text'] + orig_words = '\n'.join(orig_words.split()) + '\n' + + pred_words = data[idx[0]]['pred_text'] + pred_words = '\n'.join(pred_words.split()) + '\n' + + diff = diff_match_patch.diff_match_patch() + diff.Diff_Timeout = 0 + orig_enc, pred_enc, enc = diff.diff_linesToChars(orig_words, pred_words) + diffs = diff.diff_main(orig_enc, pred_enc, False) + diff.diff_charsToLines(diffs, enc) + diffs_post = [] + for d in diffs: + diffs_post.append((d[0], d[1].replace('\n', ' '))) + + diff_html = diff.diff_prettyHtml(diffs_post) + + return diff_html + + +@app.callback(Output('signal-graph', 'figure'), [Input('datatable', 'selected_rows'), Input('datatable', 'data')]) +def plot_signal(idx, data): + if len(idx) == 0: + raise PreventUpdate + figs = make_subplots(rows=2, cols=1, subplot_titles=('Waveform', 'Spectrogram')) + try: + filename = data[idx[0]]['audio_filepath'] + audio, fs = librosa.load(filename, sr=None) + if 'offset' in data[idx[0]]: + audio = audio[ + int(data[idx[0]]['offset'] * fs) : int((data[idx[0]]['offset'] + data[idx[0]]['duration']) * fs) + ] + time_stride = 0.01 + hop_length = int(fs * time_stride) + n_fft = 512 + # linear scale spectrogram + s = librosa.stft(y=audio, n_fft=n_fft, hop_length=hop_length) + s_db = librosa.power_to_db(np.abs(s) ** 2, ref=np.max, top_db=100) + figs.add_trace( + go.Scatter( + x=np.arange(audio.shape[0]) / fs, + y=audio, + line={'color': 'green'}, + name='Waveform', + hovertemplate='Time: %{x:.2f} s
Amplitude: %{y:.2f}
', + ), + row=1, + col=1, + ) + figs.add_trace( + go.Heatmap( + z=s_db, + colorscale=[[0, 'rgb(30,62,62)'], [0.5, 'rgb(30,128,128)'], [1, 'rgb(30,255,30)'],], + colorbar=dict(yanchor='middle', lenmode='fraction', y=0.2, len=0.5, ticksuffix=' dB'), + dx=time_stride, + dy=fs / n_fft / 1000, + name='Spectrogram', + hovertemplate='Time: %{x:.2f} s
Frequency: %{y:.2f} kHz
Magnitude: %{z:.2f} dB', + ), + row=2, + col=1, + ) + figs.update_layout({'margin': dict(l=0, r=0, t=20, b=0, pad=0), 'height': 500}) + figs.update_xaxes(title_text='Time, s', row=1, col=1) + figs.update_yaxes(title_text='Amplitude', row=1, col=1) + figs.update_xaxes(title_text='Time, s', row=2, col=1) + figs.update_yaxes(title_text='Frequency, kHz', row=2, col=1) + except Exception: + pass + + return figs + + +@app.callback(Output('player', 'src'), [Input('datatable', 'selected_rows'), Input('datatable', 'data')]) +def update_player(idx, data): + if len(idx) == 0: + raise PreventUpdate + try: + filename = data[idx[0]]['audio_filepath'] + signal, sr = librosa.load(filename, sr=None) + if 'offset' in data[idx[0]]: + signal = signal[ + int(data[idx[0]]['offset'] * sr) : int((data[idx[0]]['offset'] + data[idx[0]]['duration']) * sr) + ] + with io.BytesIO() as buf: + # convert to PCM .wav + sf.write(buf, signal, sr, format='WAV') + buf.seek(0) + encoded = base64.b64encode(buf.read()) + return 'data:audio/wav;base64,{}'.format(encoded.decode()) + except Exception: + return '' + + +if __name__ == '__main__': + app.run_server(host='0.0.0.0', port=args.port, debug=args.debug) diff --git a/models/tacotron2/NeMo/tools/speech_data_explorer/requirements.txt b/models/tacotron2/NeMo/tools/speech_data_explorer/requirements.txt new file mode 100644 index 0000000000..54d15352c5 --- /dev/null +++ b/models/tacotron2/NeMo/tools/speech_data_explorer/requirements.txt @@ -0,0 +1,10 @@ +librosa +dash +dash_html_components +dash_bootstrap_components +plotly +dash_core_components +dash_table +numpy +diff_match_patch +editdistance diff --git a/models/tacotron2/NeMo/tools/speech_data_explorer/screenshot.png b/models/tacotron2/NeMo/tools/speech_data_explorer/screenshot.png new file mode 100755 index 0000000000000000000000000000000000000000..c892347bc307b8520a67f428fa470faedb35cc56 GIT binary patch literal 1135618 zcmeEuX;e~c-*$%h>va~tz5TdI@y zZ9dp3`79ND(*?3+%b#<<{i@504HFJk>kHvA>sDq?7(Taun_M2yY4xIN< zEVaBQeLwMv^xYGYi66kxhvW3zuc@axxE9`1A1mHz^T_-u_6wCR3` zs1e35DPeUz-Yw%}H+GCsvL*8k+k*#C%7RragzFN$kNoHR_woNV@!68i1y+30wP9b2 zZ!gYMr%`YpK-+Ky`~fYX)(0{-9h^?)kW2E=<#Ks>b}K^Pc%ZMe5rh>1Gyg6235N26; zMd-x?t~Q7_`yzJr|7kyOY^3W}JyhNQ`so=#h4q`fyb3e?iUsKFXpciyE}9W_Jx>KZ zU@?r?{U7%ecl)z*KE8}*<0@p4_|AO7D#i_65BPgtqa#pqFiT=q(~dBx;evpPNjsTcBZdmNmBL zfc|rXJ#g#dTi&v7j>vbWFwFgF?a{wZyxTrvPcU_C@`UIbqZ*ZxWKzpq#|mtAgJgU} zy@^|tXkjDI)_;qfG}7`B!YMTU-~Qtkd@P;k>P=O(4Kl5oyI?Lm z@erJcOO#z}>Z3l~atU!QAeuj9p1c6sj`ohfCCsT8@g(zEx>#FN9-pD#^LX~%l*1ES z6)yUyc(L(zj_HhXiy>rtbXjArwz>rm$ub4`av;zWjINm%9#+C8C*3m2E&{1&lqPccQ>s87cio6^^)HVLx5lH$lv;?+`_v^ z{<8*!8e#fMnQ4wwIdVGB*-31k0731(>~rl=bmTBabzYz!1q8b`grqt=m8YAmRVmXp z(>2BDG>lM-%KS7c?_|>3>LOZxA zPHEZ>{rDI`HHOu2wr?po5VCb)j#J2OV-iYaz=oHs9>rUxvE8dYI0)YbewktLVl+n% z=)IBcJUjRFy8hgn*?1LKK*nr7YU0@EM^Nor&3Pv1ysGYIOSoo>y&`?I^bIgO4CVUu zsJvP0eEeG9&c0g?%+ol-cnz(8EXwHG&W?7Sji(0zisPn@AC?VpW-b0{<_dqEcac@K zA)g10W*iuS9UZFxtEgbJO_ABu!lHMk#(bCL=Pj&)+Ym-ePz%;_>dhtzOZlw}PS?4@6MObSy3RYIbx zSl@231|H?$^BX*q$zXKrPa%G@@i2RQ`mC%xh9#%2fr2OOJ==^2u^*af1=(cn+m_U+%2O$84qJ~d4>`I`aj3zxmBSb zx+90tJNG^b-FTbhk%9F>==m|?7Mx z0i;44k^X&El*}`NT>f!yd0^*S;pTf@uU}H)uN`)X@)DN%KUXI?D;NY)P-Sh};ElSQ z7`x7?jy0f$m(JNqM@JIgTIXHxtH zv}8l#mj1YB`|Qv^N7ecriX`t2bR$g@K6{1PLP0ur6)O?v^K$D|ys2j)+8idK#p9H{ zRoG9ot&<7w^GUVUHCZ~@zCCxClv4%E9S`8pKokeJ8TJm1G_dU0Tju; zxw=eIsizfQ$XtNF43vivwO*qO&?t7^ugbBy@p1q|9cZ0c$vaBPRaqQTu@{#<1|FH) zbPDeE?>%DNg);Nb3nvGyrHq2Kq8K}i)~qX|Licf3*Ge2+m)g<>918H&w8X9K!V8q7 zq3*vlSoNA&PP4@1zOecBBI?BlFO~=^D2BctkIbW&^8uXnK~N0mjb$;q3B43C5*D!0 zGC8dHHs!S=3D{kd<}Ad|YZVhRg#o#qYX~bZ{`BN-SwY)yO~wt_g=!RJcd>8H2$|mD)q( zSrgMi_SUvgo`&Ji!M_v)pIXc!od=R4>&?4(lH%`FIbht|8Xv(@3q)UGd$l@4E7Gf( zVmue!ukFVe4I~!NHty{(D8zF{SSEoDz=68w8%@3{L={(l<9}58V>LPd1}UGW8%(Os z=-glGhT48g)67J=gTO}l+@!|vkox(#?gcJI4Xq*L+N0b8;Py>$O-e;c@pAI9?B z?Zgb-tetjsZFC@dWP4*z{*n84u2fUK5jLl`4yF?I^D&Csja~hms+N7~v2Qu$g7x>G z)gP06zd5#Vi##_yosi5GG6>iiqckHc^^60hQek=Yg(!sR{-}A;%zNLfE4z$JeNeEk zJJY_IVfwym-Bt&|lTiR_i!gW|Dk!|9ULxWc}$STg>qd^~~4MT|BVVahK6*6NB^lIQA!76T| z-jMY|l-V#e2WyXkxEfe7y|>K}##XKRBD|7n|6H~HcIn@kYHbr#Vp^O0TrBJoQ#pwb zu6%!k*DG5PqC)ulgd#4TiD$5D8O)#4S?CIJ+YFji4|PLK1W?A#LAY;P^S*JEp-M*d zrX{29qv>T!N-YuyW4gI|RT#;{>iSQg1>IsTDA;d*fMO5*uwb#6---)s>jI1Efo65% zD6!Te@}rAu12xG}?ji8}qsQa&LRt}idIOIgFPYz7jPVbnMAGUc70XG!l8p3cW8l{1 z7#nm1Q;T(=wL~j?2W{yeufKyr@z?Ew@;BNgayg#ELk-9884l2WTf-TUAEgO0r298ZV zR+Ra)Cs`UGzmYp3X{UTYoC;Z>YPy15o4O;=A$Mo;(iXm~m{2$Jpz7-*E-2C?FIc7D zN~Ds(+p-ODsP|c+9IAq5SYMWH;8Fq2N-eKCx4K-%JR}cu3}cf0SP6#duya~w@6iOp zoelQk9=EljQxZ%Cn6OxpcNNS;`4iTKsL|(F4ee@NF!a&sv*6>ua~vYOjzNRHlkviQ zL(Lp!?~^MkHaAgrp3OXadV1BueH|W!{fP zG-%)U#IH0eFPhw3VHuSsLR#PEogSVY=WxGgZcsg%N-f89YgfYwQVLn z>iydlT_TrNdYH=FMToT9yd=1HLftyZnW>Cu4s{#GD!wJdVQ;GXlXby)xw@=fn(ld>4V59471_+}3r8CPx&t2VhZXN-muX(ocDg*2(4*;UkJD8k= zIu^a>EgofhNYH+4O5KjZ>Z#q1P*#1WCt)7K1v1dU>eE0RFI5^?j0XtFigDW1Jh}&+ zFez~5vK$?cH(XBt+hd~9*k;b?`1ky3seNXKb3^&m!ZMGp^I%YUK@#3O>;u*-8`0mi zH6A1_cUhO_3=F59f3W$Nb8;Pqw5t}K`G&;~YPedjTs-iL>VT>YAtz&`s4!ppseIFb zbOv`}q^D}n60q-*yLG;};S9ksPZJqK6|1@nY9E``3NBl%W;vq^_07VG8JI=!Rjh(* z<&`=!IS>7=bffbEDgx^Z69s^(P!eW9GDSs(E&gUlZ){|B@aNB&kyP*-yFs_Y)3coF z$7-+7KcC+kUn0#>?!6~u1c^r{`k!6|w*-H=aw_cho>G8uZ+cDmw}(PL0aa;Ls3koI zr_}{|@`Qw!BCA-9s4RR?px#ee+ka>@gTY2zyxKSax|Z8#oB@-)I@K?Nh~GY|_*=4R zZ^Ku-77orR=IXgN_!TAr6DZNSa;x8Q!b9QIWBV&uN^s(%MCOol$dD4Uo!Hq_Nm4C; zquJ7kozjA(=176DB(Rjd&ZsXXiLvptM4DK|(t2i3S3YAIni>Ihc)0mU*vLLFJ@;cV zO#wyl49l)!ElQEZ+n1MJW9dzeuiE$+Wk!bR5XD9EI03N(>bBxdf*FJ1f|Wtn-lrtE-2#W z`?-}6GD3mZs2X)@Lf;r%x0J7$wtqOzsJ<-hVQQxX?t0(dT?}73G9t2-eT;j7-Om0y z5Q4|y(@~c5M(*u7UVADfI77@bLlb*~2m#KAxaS{e+k?Zo4bJ+P0uG;+yOfh-Oq2O^ zq~vzq@MF*2i|XHsGfT=|ZSo5!ta4(Ys!rON$*7F#u1|90U)4Hl8@xEPqHJ61&&rfF zW(Gw!#ZyR|-mNeCAoGP>@@rwI*r5?!K^WFavEFLy1b!GXAGt{y{&KKFj*`6S$8{++-TRk87zK@M-4y_x6&O!#S zziE%;e#gKRBdF!e!3E>;W4$88qYo!_pZ1CQSb`JlD(73|eT)QMuv6L?9LT_QM$UlG zfCY(tTD!nkPI>LTfOD_of{}Yb`BManz6i808tYpWBlr`Zy!g~vx{AncpSWp|VUi(? z_!pqn)z*#c1K>bU3Lqc^|Vzs^>wU4-3RH`+~DK-?KP5HKtZdofIUhXTt zQj^sr^}qpp=fS>;tPgmprK~N47J8=WerJ(9uk9X5-qvIyXbwx*Q$(dd4)?5sooy!Ia!9*t{{OzuIDiW}KCBw{H0ZsXy7WcHtlO|Vf7KO3^5Rl&$m@1L1L zF#{^s#HQCg`2+mGnWj_Q~8K31Weyh9j51?&gC( z2I&SdTcfYW;x?9`bke2~!+Ga*o72=dDFl2Ty!-oJy4FZCaI@Oun;=FA(U-f33|RJ5 zP&OM=yYx%_m{W@|tJdfxcedu1Jq_4K$^H926048=5N333sjAmCs_verQ*znihg=^C zpj%pkGyu5Ygos&Oy$Ut3!NLaXl@8PS$b8@|O&kD^P9ro;>+`8aKuje2Y1xmcp4V;)<8Z&952-IKldNE~?^~yg z$1{-_ZOWzMy@KJ9q6I1*Q~Q32Yo@z+#^_PtrUo(@fO=xf_R-mU z;p-jo+!o4$3_Q}Kt3B=-La4cXpq%l?OTR-!=XTf6@vvgNtIG2E=}$4o_bnc4Z85`! zL!OMAUF~y-?Z4gU6cAQn67&g6h@5fe)6tt)6P3Lh7F8n+kx0yfFkNI~0Iw0|Zf*o6 zOLb_CS3Ah%a_R|L;%d)rtKNjnZsIJwFHv2>s^51`Q;(XdK8xyc4qPeQAJ|F1<*zpS zn3=hrTsC_tYjg9F(KYw1%ud9>C9lpnsz*=2bZ<~^orZH(bb|!|JU5B1m}L)8W`ND^ zK+$j-OPJlFMp=3>`&1did47ccuJ9c<2Sk|Oym5+n!@D(XcZSvNw{t6f$uwjKT|xmU zat&q$>$g`N1)Ib-c~mwhEeqv4Ai$K_P}l%$(As7(lJk*fK_|ZhzjzBsuT1VSs~@Mk6qFIfoqM-ANKM;?;-D%0H92K-X*&=LO=7dW7&6nc1P1?`e>9g}M}{TmqJi_fhs zVFv#KbgVyfjNeV2G{SAPB~qv5dGl1S^ruK1wmDL7b)<#*if~UcXVfodB1xnG=YSoz zcK;lP^1UW!yED|4pfrjq2903c0v>f<`6$0SHj$%V>3+A!GakP{u>;CkoGRi%c`j#o=h2qw^8?LQ=K_>Fj!xcuU zXQnZL@@*eFf>ber1?m3mED$mUqS{NF-PBVzOT{!9koE8^0Ax&8@-0e77AXBL`Jv!Z zoXd){?CXelHaidVl3-doqv9wUaVS=X2<$TjIP-}QJb9Wf8U-wJHtCunz~{q)Y;h|O z4#tKSTNbAC)cf81Jbxw7mb>y*6v!`<8I znVYFfGW(0x7!BCHwi0Ygj6Sv9Pg<0j&<(G|_sDcNVQ?P76FXtw`{ZDlTeBamjne02 z9+P>IwR-(Ev0p5Dk_{zFov`!NOUP`we>tV;=adHPK%J2>_{y^v4d=*Q0s&8^T(ma2 z4KQ;wsf{EnWJ6m@S=V9V)ByUbG!ZJMFJUk22VW@=bJE1sg4bD6$w`foaoUv7be>zt z#bv*7)zki;-cU>;a>iwFPf8Na+E&zz7JWa7m1pmre=rcNZ9aj<^K7LN-s!tU5ANq6 zKP+0wP3Yq;&QOiS#?32)cLHC^7`ev2lXM# zD!XamIptUz#bUK$ioq^{_M|VdzWBuYOLVW@Lbrot64E^x62gf&oU4~I(1#C=FX_Wu z_uRrj3F+0#ZfH>|OnGAdY_(|CXWWUt#AQV{P~wUOKr)b#4M|ek#6vc6_}4{RixfT2 z$>C2$Yl(;AO8((RO52zB8ObxTyJefomtFf=rH$VnH0%2&sJf?|4wCv1e8zeHFZXj} zIzTJ2qlq#2Xr$QSToldmLJP?&fjw!hpP_gV{#eb+4@@lRaCN;JPCaeXEO9gZ_SweU zTa1(aUuhU4?^?)U$)6}rPO`R&d58VoRxJDA@g?Udum#scxnvNcZV8w9iP(9ZHJV4e z(oVclH7qYPyee*GBlg!lr=&{+Uf!T)rKmGKU*m3?3hiAJ2J1w*eo|-L%orS1XHHB@ zB-Q*5n)scUvVZ4Yr{TB~sIjxGW*bq>7<`bhk- zn3Ow*a!t$6Wm%L8?uvO{8$L~>8^TJxpWY{1qM$W3n;Q4&=S$U%!JnpIq^VL4s25r$3D#3>-G`k8d)uKL5?3DA3pX8Rr-9 zZL(bQ8xf(_c`DScX!xZD&Ej)59l!>x$EqVpJ<9riDJY)^&FY~gqwje%3B0l?V*5Umo}0$!p2N*;lf)l=N!99q1P^`|X zlP{T%wKWY+jcke?5sN+&eo-cdnlwnAmzhxfF>W|D0Ql}?-qDcR_xt(@mnrbJi40Ck z>Z@XG*m{CCC`ZpWmFDQN9oa?@F|nqk0-I-Khm%~21J?5j3GNY5Vrxg+p}Ae!M=wrF zhAf}E7Cz5P3;PiVr>CE9l+t)-Xg~B#>5A}fmKN!V$i%*0i8?E1nkWHw8k&Jkf-s3v zkhgbI^@rIk^@8J}=Y8lduCm1NQ70V!yjUnN;>6@`GTy93o$tNGm#CB5WYn@4!j(ib zeqBSE4xL?t>UW>s-{O~USkD!m;SQCk_zu-y-x`$H^#*G)4@vU5xsyA7s)bc;c0l#= zQq@vW0F`j&JMv-wk)>q>fblXj{X<*RttZPm|FeLOWm}rvD=Bhg_;Zb}8~$vn9V~RT z=m6+lvFFOitOpT5%!3!hIpGE*d-z^ zeAUGHx^Is_AnzZU^YQyU722Co7v`hFwkB1sn-Q5V*y+Ns89YN83V(LwpdF6oSEX2m zcT{ST;Jf~M<4Elgzj@!dUfR1}ZO&TjhfW3j*bYMZh*!VmMzzg(rIy^|fanIw@YVY3Sj zvq1l_^+q1zC^d(2o8lgtsLdcrRzmp{tmW@4ZRaB+dX9Oq(wBF2$5{YJ#g-pZ;5JX_ z@Fe_nuj9o@O9!}wbMrUf^86^PD)y}UrU>!6U@cKbS@mELA7tf~lu{9M*2s8qsKFm- zb2~f07PLCBEJewzo&{-%do%X+V|IH~He9J85X20&p~Gxj9?@Z{s!{=Y-unDzb3bKt zx~MC4yW8UZ;JS7d%eVP`w-&^Fz7VY<8)P*{=)q&Lf1Z zOM|MGAUAB*(5>_;cbWfL376MX{6LcWhW7I4Ghg6n*^4o3Qk?PwO%z4%_DT+x6s>W? z=QB=D;ue05T%MiJG(BdI9PkNvR5AfYmPH9(*y|KIqdRo7fXTz zQXPu>sk$^=b*6|PMlCT!3XVCZy6hhtdM)j`WKHQ-OGi4iB*+^PaTSDM>R@rZZt@WMQ4r&B9VI71iT8dVBu=vOhc_B}hd zjU;XFl9^i~OjP=#;E75{F=z5={4sJfPEh&F$DcEzAiG=?F3I(CWhZxb6C&9~Lum_7 zt&sjLSsThqh)a+ZSS#V=_rcr;6TFW#vHk8d2-xoQ#NtS$LfbS`(@DqLycj9J-AqZj zsuLn9l!H4yDAns4r$o8^F1JJXqoclDxJ*$)G9K()&-Fog(7Z>Tuu4%jUWX>8>8pLk z-}<#~qYK)K5UlV1`N{82UN@z=D#|H%aM-$Q{(peo8gLjpw6F$C6N8D(KmCuJnrxKi zPKd%{)n%Qt%Lke99`IiZNg}dHAgPuGrBfmxP^i^X(-uFEC~gkFnn8#1=r5=q@8kNV z`rXHq9FNwv&pf`!gsLQ~j)3UxNh55$+!JTT^MOQcwRkUFO`MB$LxmDq<^5>6giECH2w1>{N+|78g?fjc$< zHoSYNKA}E(JVthotJK0x8D7-r4;EX-Ty}2WMVnssf=++V$npxM^$5qKjC4+ zhGTDZaKSeUR)Sad-EB#JOb$^%b$QQlu|^MRYAa`EY&1zF+PD2_fS$U=BnzF%=1GMx z-_|LkFuN>{hW3?7=c-WpO~A9a3$GfA?+cTT?!{B<^GsRphn9`@J5Si!GQ+kccM+z# zvv4o$y~?b*@Df6yczr@~SFcDZy96oAx;rsw@%H!bgy~Ybt$t>z`GWCm_Cj$<;Ba?? z6ZNqxT6NBdO_=qSp64Rz?P4I0uyuv3ec(mhsrRPo?{Enz}#SKYbQWrM$g0sy;{h@8^{rWLAwRK7%K>F|Bd##kN_OYqMGiJE!|KmPV2T z`G3{e*W$Rjl~jb-BgIC8$c`7`T^t#D`rF3F8G(Ja89~meXr>@&zA)Zw_^&dHm~yhB zdNKPPy~$l_-|yQ{`$g0D+tw!h`xO@S7%2v@%e3iVv4{%u+L1O-IAX#Osv*va5{44B zSR3~8yL|Ub#&&3yhNQLP?Jz_M5q>jz~%~`$;(hNtKj(Ns^HhJf2Q3et8OBRuJ zGp+ok@t|bAHXYI`X7O^Q1lhkxk4m3XK`bO7(@zheT+aJNv)r?Bx6oCO7%Ey`&?FW&&u6w?P29?#1Uoek-Xjp1vQ#SZ!#z{hL zudC?l-97s3SZsgay(qzfgA;XY@n{8g9v1HR$*h6tL+bl26}F-<0#D7&3wJGgBZ<1O zP5WgJXznQWf1ny)hoOD;)#Ph>T1$J;L^Dl|HSW)2pYrQb`MQu-*lv&GoRaCa7;QO%9 z6M1EYth~1rn|XQmmrt$q1~&Jt&XYe?0`y^}<07Zl)K-nB$-x10nxWsX>20k|DzZuO zZ9m$uopAAQKSFwz#V<2;asH`KZ#ATDvvlAO|MGr;^EcU4be=X_&~9rMHX8#{pUpZ} z*CmPFfSjQ+aZSIe4dx?OH-=0ddD7ZWWSysE?s?A|M7E5>)xZ<~;CRPY*L(nz55d9C zf0{PDsahmcGJN9#r*$-!_Tu2D7~MT|Qf~WBs$HFHkH>(+tW`#4ikTH*%n&GEBfA-S zTAFCV8|G%_Y+{Mn)x_zVpaFiYW!*E4UlyBTQ^zI zA&(AMQ4|pB9+f;RF8QK=6Fh(5&AYGSHk%^3TAkSwpyOAPwm?nlmB0BpM<5@ypHx4m z1xi>COzM?oX=Y7wca>wSR z&%qG;n{%)4$Nx0xwoMf{Xsm(wqF>r9VOU(`t8Z(6g|S0$DcE^mkfyST-){IeU=4ytFO3t-qLRC3k*Jg}$k= zxSKgVXNBxMqy1EGR;YX9Jl#0>>Y{Waik&!=u88~}q_JhsIs3oD3ljiPEt*9a(-qYG zU`pqNUca048V8Pxpco57nGertbdx&Ujc_Hs8!okwo3ypF#7*amxM0REu!hOIQ237b zQ0G%?*2HI;{ZaPoo9{8MVQ#pl4;caPhYPcCd~TO6=6+c&qU2`0dX(Q?%ydVrPSwD% z92Ty$=oihnc*p>9Bdnt_l`4R+u|xV|pA2e$pyH;dgxs(z-evL*YamBS3zurKOTBIt(x>n%YyVQo*_y;1 zAmxJV(T0~H9vOvsKZKvF-Y$5@6w+1q?ZDNyUNGuUP#HbNtwSxd0FmshSKg(Eo_|f6bjnW{zQ=tzdgy!XDPJ_MgUu) z*Jo99{0GlMed$G=!3|QsqknFw!MHQv_=6GbFzi#+@XOB?m2sItj#9g-C#ivxPYAm!o2&vRls)DMxi9C{_n=j1k(w{ZKjb< zyozirt3M#|#OVu}LmkjRvoAToTSle^K~-BzZJZNBRIdiLgzAqM{@73=A?FLbO=gkf zdP<#hfBhDlZ4meS)`eqd*BRSP+JwGH(Gz`lvS@QnVR>e2zx#T6Fve($t38e)Yr63P zM6*6LNmvxU5UqErD01&Lz;kLlf0bp`&v1SnPwxzImpraW8Cg2MLNW^LJ4KNoKkMa< zAn`_)Hp8pjdO(1UXs+TwO@%{oMZhuMJJjl@?q|M?yfK zHuF$RLgJ_M^f)_W`jK8wQkCOc^FMajXFl?EfANf{R?r_xpc*IS$nj0!2t8OLcD46+ zr3eZ%9?hPRe149=QKUX6<;Nj zdK)g%OpyPr@rJXPDP$^)tDpTf(q=IK54O=cv5ytYF`xPOrtqO7bY*QtVQ4%7{UZH9))1%N&f1#$dFid%=k7;~9)K;cL9(F$OY5C>Bl^%5^yH%pSn3W)K0IA2!V z6o;YF|wEDNf`TR%AiT_x^K*sv{!k0mZG<@^&etMPTB}k;2G#~ zo(a=tWM_8v-d8r;6v(om|3um~`v<)yKG&|glqF<;&BT!-mKgfOhgK*HHM=^FUujX| zjygqtR{ZHuXV_`@2I_+hJY9ZW9`oJPc_+JmMd6_Ql=?_uUKOY8=EUsgo-ePV1pAeV zF9}J?SizoB;`Jku?-6fEYO=9<3vjOe-uH(gW2`s2rRC$@IOISW$nd84TgAKb(nr~M z@fR@9)myd5PhS`I*A)S>RgK>4QQ9_QezCwx?idUbufu$}EebfldA!7RTHP?qm4 z;^K0PB->1g;OaX*0LY>6%*+-Ye&*0x)Zf;P-{M}zdX;+sVUu3{?nOcsHoRCNa3CJx z5K{grqgJHzEWs(NBwT`MO*|vZ!TxNZp7$n_Hw~AF;6GJ(&46!x8UKg|=kH|dIlT$` zqsSc4nvv8DB}K=HOt;Ps*W`qDKs7d^jz`uRjvN9mM+B-t+up7H9n@uYW3K+TdEDdJ z(WJ{zyv@l-cT8l%R4e%>$^wyLDQoz~^fo`hGmK{1?=XFk&}Prh4fU?5Y+S6-No>|x zKfsN~+&F!Kno;JMO>O=?c_y!oZH+r{Ro|b!Uh;M&Wy{G%`NW|bqo*y~T4vPMbD7pg ztx!ACaqm~Emxv_p(8;SM{&x58bj@mbSLE`^Kbo)h{wvN(wr~vM`c!#?!-`|q@>5y7 zT^Y2noh>FZxG%>JME2@s^{lSza(R!FmfROsrgk|F-WcCwr+-{dS@*%m z`x(*x$uA-=LCEQF{tE@nW);1`R$fev*y!sg!xySUqX;q^XEN%Gr?BqTJ5u$`8;;+T+tI^LR2EztTe;s~+ueg8FRn)Ox&{sYEbo2>Ot=`x4|MlZM0E<{3J`z9{c&|A-!A zj^S%}b?gjfOPay{@6qSte;s%LyVc|u<0zS%h`VFiwCUObUzVo97Vn!d5~`aU#5<%H z=Jp;yc~npm5^|2D#l}0haxWnK0c5X=-~BP7sK4)7lz+nCrBuo3RJmSm!tgyVn9EIJ z9ZMj#3-6g5(7oP9+n^rKfRNEWHhABdrvTIwoEHhox@gVo^q7^l%L{il%DDxUP@u}| zC1m+E(zjhZaxfcVvAM{WZ*E%!V@U~wpB^?#x;K&j3Hf!25+SG4H+1SA7k>DN5*KeXQ<4S`IRf6f$D5uFE_QncF*?2UQz|lQ%oNEl1W=9F&h%LxEqvRp z^%gtK=>D3Sl34rkTMOxiU!qm?@0RvqI|f!k!sJi1Y<`pG5cNpV-t( zU<+F<{iBdQ*JE{#N8n&4b+#^)4$M_)sqren-M1GWrYRMpe{^RCKKgs2YY?0{*4M&* zF|qHg;OX}R`7*Dis*z=VX%eNQSRpLmde6WP=hzF2CK1^$0Da8Jrmv zKeW~W-*lqD@MM>}z7oV0xtP+G zZ2;8&JaON7g}c5v!XT;7kUMvTN(?J{$uRWNhV!EE`y1KpGJb|<>s3YO!^yJ`sG^S| z*@a;rcFIuwRhkarK3)l>Z|Ne;f zYF>2hhX^GS&a?IRmgK?G3H!wp(Yal3Ckk~=ukD4J(B14^)MU%<^X`(e(_Ks6c{!glXG;^b24D+BmtxRZeH)4L* zp{6Tc(0$%UF~sIRuLT;iVyUeIXNt(lXl?)jpx{E_4piJFo$>6G!RSyg9ELdTDFv@EHdme| z`dJ^KnzCcwx`=og%Kip8BD|R0#LxJiD`nRbTM~p?_O2v_*(@)wgSv7I%7{C2R)vaq zdVWF;RFYVBY4gX~>i&@A_J3JZHhk)zAgLH#NfQ&7$KC|cp9!9h=PDOla;CFXWE
@%>MO5_D86 z@j?d7gg0Gwx;P3pzM@TiRR(esx{AAN%ACNRx%UBdZX^P@dqI-!Wz>Sb6?^-RyUFUT zl^5Mz${l<4(q2pyprbmG5Mpm0TsWGsuMLWzjCK{qmyL;79`wLL(|=&l{Ly995r|#4 z*=X4qMP8jExIRJ(h-$eU?>>1Whv-`AW*|%W<4Z339ObEOa?2-tDg-b9pVH|vo2Vc? z$>K)L1YC|8O^VJooo~rm;%;DxG8tMF2(<3a#AtrZMf!RL0-0FN0CI4R;mMXlbyy39 z%~5bd2NN3Jh=lEJwqpd4&}lwiv2KK_U)2BlaJypc-ZA-K!Go$Bn*<|W-bqjf%>t6L zN!g$-r6mMHP+8saCG)orGe_l@Mob=GZ_lh8CQOv=hDT`RI=9(!mJO68<|$_nKuGBv z$z4;Qh$`Vz2K)V3kx1qwY&CW!Q=F+i{qSOl=_Yv+rD8$P!b%N{5li<{O95A(&Ky3s z_UWE*Q_+mVq$5n`EZ_6!;e=n2CM9pqrO>Hoxndrt-(BdML$stXaL@g_!^sb%?kN6A z2J#(}gg-d8b$%3eP*tvfcaNiuYapS^!!}=zI#0`{V3$>tb~dQ7o(zUF2&mkjm2928^vq^gR5k zy-<>igrqt&6oki`YVw@a^^Q+(1uExjXyJ!l?rPWOsjlVtS*XG70-?jerjbTK$nsR_qM7evH?a{j{ zlZlZ_Tx;P^97z%$-*^DFr1(B~P*tCze5c9ieI);VcaK%rNJgcDB(|VJnKGd{Y<^u? z7%;n20Y)5NSYa{ix?GO_onL=)%kDJal_W_5eHoB~3dE%WzkI86^v%GvA=3ONil1}~ z^YJ%Bs>yA5ho<-h=x5hF+SdUq$0-#V>EQx+s;`IWiV5%bUlZrFD7VBPmgS3;CP=xl z%(^hz-6n$Ah^Naz7-{gHqY38;&LP*>$`_CiQ)gLYr-O!zcDg`td{b=z2M!)wlTCoG z)`q4CRJH-{^_Uu!Ps!GwZuo9~Y9I)+WTjGJbXmX-;{tS?l9wjlP08Vns#`aaV7joh z({vPYQsm+W{+asz9!ER|aK@q>2Jk$5QZsG9ucfdnq3%HzDUd9%@lc_D3t7o>G7^_d znv$ZN+$=XeXqQ|z_--O?psXgVDoLWXZSu17ntCNgmRi5jJq{e*9hEK&N;Id@J=~j; zuFt!A*as^g4@tVH7#U>Sv-FiT@JYpQKEAHQLG&kWyme}Ui$Ck`$ZsTOiR-(U@441}B9nK0a zkU~YK(}4tdTcDW8>VKV?u{-AO+=Rk0+y7nBJ0iTT?MztIJYrZ^N-&g)vIizcyV z?uX8s+=Kl+0bb#a{9UproT+Ah(d6<0S|H7^F(l=SMnAD~w80RftNp-APPEC!h-iUK z7{gH<7|rXRAhv6nR{Tl|Ng6hsUb+|DX}ckT4FaPeU~Z=v8kH#c;DA8q)3?J^HdcR> zz({@r(tEtcHYik$(kpCVLzO7?@P3!?_VfH1xCa@YE4-bEW&D+!9!QT0{um*D$?Cm? zJ5hw_79d(k0p^_YXy7#hY{x`#Q$`)M$6$^Rt+|q8@4K?r(adN$`Y%|_8So|gmzTUZ^RHG5zY3o~6kCF=eo_3|_o2psX3sKJV}?oU=07-U9e-?hHx&5U}uKjV04e>OrPa?ed^LCTp;4Nwy(O$LZ-h++LFO}-RmNS&wLuZKlbfgNRgjXwL7=V#h|PoqYC6$Sm$(2scJexF-TY4_j1M?rTd zQIAD`Pm3ILI&@d(;{+PHt|i3$0K%dNci0FdjZDrtu=pp5`Z9!q7ALyb?l2~2F5iOn zy&O#faukY(7n6AK(z@24NrjRG8|9EZpx{@C!G!hO&Gbe|o@U2mn^)wxZ$etf4ijF= z5>bK|se*wzmg5EIGp4F|_4DpD|GFPra3ur@j#V2!eji;dnZQK9f}iVYq!KF? z(5)F*mglOZxvx4|1=5@Ty;ka(ri8Vjk~Z3;r}gIi(@=f6yz?LDOjKCa$)||0Viga> zS6l+~4-ijrAOu`4%oI%z9o*{2q2{4PWyr-H{_wAXr!*}kgiUy2L%w@1?GCAdxz8ud zvv&u~1{#{&yp1C2N|IX3?4}DfnIJZVgGx$OaUA;VovYwUe91Ud!8Y;fQ>4xLDA><{ zmYAlj2te()x1y{PwpDA0sMdRFM&`C+G242B_SR4Xr&zGZPEq#>Au*FmUZpF`bM4&Y zFF~90?HvW=2D;RL{7Odt=!wgT=M57=@9?DJ@}~<^A~(->BdRbNH>K(Qz~7RmB&MWoU_>wyp5LR$xbRN4@qLL zi{*qs)kmO-tOzAe>Kk z9&rpMF{muE19&4~Y95O-S=eivWruJwPYrYlwfZ3Go`}o0>F}r>80%$n2^DA(9xpMw zVbBMXwnNcHp=r8ZbG!k*m)$H#b7A}nO7he=uEoh25k!V@FWY2eK#eG&iokurg>NpAa@ zuI-#-&}n@r6z-OFVC9#YsqL(kSYq4QPV`o6AP@j_H%?n*JF(!AhrjDYogazp@8_nyH2w#o z5P7zdMmdV+CSw0DhyQhQz*o|NYJzMJ6Gl+-xFH=~5>ONo)BA>e`)+s5Tfc((%h~*? zw*W7$qy=Kc#pGMN!xCU))>25x6KRy@V?rdcO^R$N`o8mNU7B*~uir{o%5_haxO*D{ zPEHZ-*HdAEsY1z4wfa#_<&qT>bS0y>ndQG52uZ!BS{pPW>2XK7ExQ4DKcH(U|C{$o z4iEJvnU|A}bF%4pzFVT8=Mw1FGOB4l&2_D_COS#Jp`^jAQbi<;-^#U;+k+0Fzu4*= z9@KB0&zJ0iRL4aV>-!UR7DxdC^)U(@0e|r;oYKbWHiC*1c}7+X-fIoMSBa^Q9ftJ= z^pU=EKMNiN!8sy(=VB`Dur=`iu=k!(O?F$`C>AUrs2~=q4G<740U{uXq9DD75}Jw- zAkv$38z2a%G$HgBAR(cKKtd4_P&$MpQj(}B0YX5e6Uuk{?AP~u_a5iR8RySAXN)hu zleO-9W!7t6b1wI(-hKHGZm}5~i}+dgN{dgv0&Gb+3ZTm$ce3s#MNbrFu zXV&@;Rh7cFhgp3Nh~evHZ8HmZzEt>a%~#;}#PS`4i@1n6ZuMFP%p*eCTsgtH+h?vX zhQ!=ZDL3SRM{W|))!3oz?)lm=SrY(K>iEO+*@7YqW+#KdR-y8wG=P#jv#eDCUqOx# zLbl0)Pr!C$kIZv|CzW>vkmwcSVUR_~Os`c&_YUJ{pf@3Z>+^)prOi+L%R2zgY9{+N z!xXUEOzG774;KkYC@wfL*we|-4m8N`nYk5S{@ogKdUO`|Bg9^Jecja4`@)!W zajMWp1Xc!g)g(p+HqHd~MjAe$mM{Jxl>o5v_;;YYsLy}DEhB9M9hb%Y1&En z#iz&pmI&W?TXQ=wfHiNwqfYD!J#8R@s1x0Jw-7jnP!7a5jO?FOdF>I=BW^4r{KM-2 zh?4HXs;PYahrBe)zFff`(P&$81R>KsB7vp?k${?;rH|{zt{dbkcV_LZcRj*0!<;)N zY-_y8IAe#gsYRiG`y;aw`b?2>D$#E?{;MHlOz1wJ;QN25|NcB&@&J&!;HSR191{?M zUSzL7cix|xmWt|L=@Bnh% zO=OXd!~O?(=U@8Y=Wg+TrilID9CCJzS=*k$kDka41BNT5p}*Jk2mBtEfyw-vCFSCJ zQe)4u#2gFQ4tA&3C5$=_P;5e6UZ`t&JR=oLygQ2c&!KJiV#?GaWH#ct%i4tVwkwy? z6UBBuX8fn+eWG`wu0``iP~$)bT=hP$2`8q+7N=@;qSgbgY|2Cjuq^w4xq1Koff19X zjmsLpgTjALm#7BJ@WQ4u4w}YDYKCtn8wdO-EbPk~#(IbZ)K@lv{%wK-u*_*QG4((@ zX$|?H$s(f@0VW*YQB}WhLD)sc8`-V;o{Y2C|I=K)?|%5ou;WWxwz?if&sym`Ti)SA4H9r{qWYpe>^Q_L$N>Ak9Ux7=r`&*wd5%lTX z@!wnl?XX-mbF<9S#q9q)bgo~%umswvAKtvcS}6l7t~5d_=+9`tfBQJhl~<`f7xe@l z??2C1)9=pzYyRl#A=Cvqa~qo|(1`}u{dt5qGgkezZ#}m!kN(dzl}oeqXdeHEr>Smn zEmEx!OF8y=Chh7$Js7)^t@=wkl6}4}w7+sNCVtmB`H7ub9(^Q{iQ$`=-g=uR4z2uS zM66!VC-PJCzW&{21?1IW{4_o5_$XYM#_yVe(BHufP)1@QV?NZb=Y79Hv$yjn6AUco z&$H_98Vvs`|8Tk=-0Wm?x1$`CKgfQPR@Sh@a8cDbeJfjXLbdN|eDeQ3W~lP+;isxN zTX?R5zsmz9H|v6eEEPqkUjLTV>tNf%O(v5)y3UsjNIlH6msE|_zQ)V;l72hq-1V&P zpdSYq=r(FOS*6@W^)X0(0k@#irQL15wR`$Xf!(+?Yg(@z{O_IeqRwfG<;6V{mH%~& ztaq36#-VR%WYQvvzcICfY(r9vnA!sk|6)VG#Z6?WF3*%b-qGn#e-nc}Jlh?FY*e0m z)rj%{RxaAZvUF_%i75rv?hEpFasRgHv5VS6=7$Blrn=5YUcZ0-H4i&nddTzQ0WGyG zSrT1d1$uRH7oS8%IUT|o3OQfwzE)~Vh;&Iy`|$%^8kNf#c-)$J7$RHVOq93kHoT$V z$3wa{xO;#<)pdVX`!iwb!^|m%)nM>@ozo06v#LnE^5jE!Uu`V8Z$4H_$89ZT1e!y% zmAo0zZCFnkd)O5SXOG9~`FjKvqNusDgNa!e)fIhGk_6OQ1{On~VnZ&->8RcEVOtQ4 zZQ*Wyi>|-$lo#-({A1V^kAtk`O_AZGhLw$I=}@_`PDj6*=r7bMx!M)XT~6qMzelME zkf?XbkTxK4KVp6P#6wF6DkNC;hl_mTsn}^=UmR`?1iP;|7bJ2ipry80f?uscS2{D)h65V!y|W z@+LL|QFGP!4pAS~$5%HLE*k>JQ!4C=R6s*Yo5nfKPhSqdT*yBUHrEy5$7 zq^P#K!HMEfvwK^2#ve;^zfz32S3QTQx(mn8T7Evr+;53anN=7V<<20;t-B;RnhfM= z2HW^FA1{(d2brNds6&%{N3qM%(E3q)ad}W@g`nIHpSx#WlLlkaIb&s%I#shgZ~^X^ z*fTu3)%W`EQF7gS!WD3OYVNsc6mnN72!8; zF{h4xsQ@HGrgzhBtZ?`Esjnsp(}q zYpS1}wAJCtdkXz=E%*W6(%6vPq;iH|Ee77y`g?Cl@~snD%KSfySQKJSEsoAAz3S16 zC`~DAP(9*$a@8Uu@S>ji(13q$sLYGjNpB4J&cLf>UJfs3TS*UYTvKUHp0C7^?KNF>mMUBPp_1rZD>nC_f_- zt%s&RBtz%#}!TPF#D1%Rl}fOxg{)P z+(gYCA#R%iw|}>8kG6Te!Fm2iEFAub1;*-xKlQ8WDMYF>b`b2QlV78;_&7Wn*JqG0 z&m?@_xZR92w$I7yanPSI4^bt#QoeX`xwqkYZbxsaY>;<* z-76d`z1URmTNkb|MH!s8Ot3VKn>$%39-gfDU{i6+CG6Sap4*}w`?*SO{e-1CM#H1Q zBt;D+-Gz5La-(7BgcudPV@}@DQ$Q`W=RDOA5d}?kG|mJ4Igx@aH$+2qrR+6Kpt_QV zoH6$+shKqE7FPDXtWk{Qjb4fa%+kjVnnb*tyf4ry9)`2D^=GL;YpShWJ2dCv>~sJb zS+=;K@g_Nc$L70R#fA3QJ;OD@!O&|?wb4s)IBd6Y9s3*9>r)gu+;%iXzDfG$6c{w5 z_#m%A`?aNo9CN?6MJ=w+!qC3btj3OIaX(S{xEks2;cz8rJ|T!Z2g}uN-u~%7{KY?^ zzR_2>SA@f-=`l{84uushV0OdGlBi$^8zy68K9qcHg8w0x(!DUQCj{I_pX#wne?#B! zw|7kMSAo694*(Bx`Mdeh@L1087g1L3Gw(aBEEGb&ts<<1OxReN*UnSO(dyr90=y^Q zfUlGX!E~deXnp>C{7hFDh~&ftzVy0V{ST3{%~dv=|65sh!G+LAR}K9kLr9}AhinCXj*+K{%%arBBb<;pL=dvNwK?2enu=gCA9%D+`X;q%|A~Wedk*{ zHzGFb&}b`}bI*&JGZSIxwS2)QAGZRxPP&@9^AP3k(O@p5T1hlfqhB1ribahQDsJzm zk1|o$gbLT{_b#8xL0!R1mg(rgf|Vqp;f~cvrdkc3OY<|8w)KI!eP_n#r(d|Y1dH|! zHhs<*o$5*KR~Q~{?laKjQDPz#<$WgI1;<0eR~b(-f&pAb!>*JZS+_B)&pp~q{J71f zqN-quKa^V=3v4%&uNlRc*88$saTS+B!&RC& z-5Lr4r>3bA|JG4l7QTW~HemE7rrx zy|}An1C|-P<#)DtFd=Tqar|TI2I?V_?(gg8r<=z@WeHWgz6h=(R4&sRTF=_N{&^$+ z%&tT=wMnTdEQ;T*|I3LBEepji&Eb_1&7BBNx~yCu6>;OIE97a(4r1cF3tyAk`;`E% zik)@Tqg-Vyv`Dgl5)Q9q@oTBQaE=^tmpEt0!ES`tQPL|m@Eor{0dGldOoiJQHkU%8qDGl2j$eA# zM9_=GC5rr(7v-gw6xo~Zx)FE)XnGmV+-Xc>{&3}Fw)ZW-7i#C(WrSYUX!woJ7@`YNX^+pYSzv6uCmIu(2qGY6q-8;X3tZWzhx)vjXB zvMUp-?A3nbw(iBwSGkPhVf|d#e%ac9z@&~#i<>Uq%=V-#f63lon+xyqPBn}O3(guM z`I-`ctMteG&Jw-Q(h{zvJPDje>jlz=ieJDZ>uBwzC3XD9+~U3MzTcsaWj3SV%sq%M zcX-VduXnE>FI7#HSzRpH*GG9}BXh>B22jAIe|f8$zd50;Os`Lu$5f?TRU>3eh=w+0 z%Yx5$qHDx#VhQl4porJiqu$!f%HAM1J#vtm1P$|pA zw9?G)dh6s1MZSZ|FG3&WTEY^|=6{%mxK8q5BD;moS6hhEF$LGH9upiSq1id5t%Xuh zmTi$FoX6sa;1OKbg{rilZtFe%e7XdiiFe0Pvnh);Qb8_t_(2Y)SQUa4)0dLHqGCuo z@xvW0mwx|NJDh1`G4y_aEoJ=>$Xb%yhJbbiLYcRiWhD$JGXu(a9}sifCO^%f(I!?) z+Oj!vr`g5B=d&j(eJlwr=?B6gQO_*HSEgScJfE#vilZ!qO|UIF3ZECF<1X^KJFaeQ zuDlDkvmR@>r;XSroiMaCep(ptpi1Q$mMO_)-*jb!H3I6=m<=CD(c#x^J>II@9!}63 zY<$wGtPk1AF3pg`tY9|s>J36MR~~#GHr2Z-p-Ai)h`v_o@)oVS<;ut3ZYU!E6qvgh zjkK!sg6t>(-8p0UTcMbUD@Bf|xor4y1A?L+@itDlVP2RC5<{yVxm9*sr)LZ!966-a znZ-hFcn>0h7HP4L?+c-6PSG0j4edv7RA9$-;deKn_8Eu0KH9jp#)?ugn$x$$Gk!MHa*d9M3;Q0#is>lSt z^14jhdpJ!oy~U*gnjcQ->pi_H{uB$>`D%Y!$Z7$hEwWvO5IJ;KC_DE3cI>U*Q^`mb zTVdg;vhJH=p^>?1#=2y!WwHLJb@Q<_=w+Y=iyVR(x7!}h)tjaGS^X0uV-M%S1+mM* zCH&C-7Oghej~5zW9%9tqY};0azTJkoV;fh6xu=IcG2x@hGl}G)`@6!{7~k7xJ|sU~ zb|f_iIse!zUKhHRw>&Q3k);mRDvzk5+Snz>dDsx-KoxTfG9L*6#=gjSAw;rJqTl_D zM0Pi|hpQv^$=K?>93-cr-y$U-aUva6p@mO@QiYu`Oi$jg*-`MO=zw03mqlC$jO@qn z{?h5`GNbwg(#^~XD}i9G9jV)XQy2=!?F^02v#RC^b3;nsmImHK?ojdKKiq9CP-G%; z;+v>M{A5V;%jqKVz9RAg)uAVud%FNrXX-dtq@VS9-=bYErtGJCiB)w6(UYIH@F*gi zZPOm6^^Da~*ZNtPBJ|)5^ml;8owtDvyPs+a2t)Q$Y!zTqJcmh52UOrqB8OCjr=tZ4$fw||fyHX25H z@3D33vEP3~WxYps&5X`QI&?dDBgYab=|4_uC|<)EcS{=%C!r3<7aY6ci^%y*dl(r| zv#w{f_~mUb8peF+>YcS%t69tPwDQln0(+Z-qgYiI=edWksN#RjWSlCC3*r`AQAkIV z$*A6st{;=>40OoD5|ta^9_lUXFVS2bb;lObYBryV(~9kv^R6hCx|RCxt4i;KwVlMQ z)5=2MsYtvGFc?lL7ya2X*J!k@*Q*y0{x#^bEoe7?2E>&*bSNP`+i`sph_*ji6O1{TQGqt{n2rg)N{MfmF(2avTbUnmq0C z8CSwh(_Ea-%hq*VX*xK1#-Piga^p2~qoXvRF;}@U`(f#})j;j7JG@fvM$J3HH8Nvo zKK}6n`(K;nT)HKrs!c=n39gTdd{uY$+|PgfnmMJp{yMV5#Mgqkmru8QpuGGMXQTGL zx-w$bQR~LoHNUrY8$lr3Aik41Ukz;OR~d&U45;~2?T>_1NPJr&Wi}2v+(oz4s&fW> zVMadF1Pvmc7kuGB{WIuWXV*76YXOcKR%`! z9!daA-+#FS@CE5|R41Hp5lF8tj|P<8PcZfWl`e#mI!$ zW(JI73H_gjKRjZIW;kv!K)^zD|EebJX43=32cC%$pmX+4W&%Q9m`kdoL2U&EPUKJvWRA_7$&G=_zuZCj)=Fm`f(TKPc7lP)PC*UtjdcE{eKIxR87 z<)#Pem9b58BM7r)cBK$>`+>Z~*C_ovPbrrco4=nJE2$XEglshd%B0ecG+~_#L#Cqj@-ES)+U=CcA&ZK;HTA-nZ+mI?hF9cq6^iH z)t{Dra+sPR3J)t4MYA>DvO*tOFoF!j6lT)PWk^YOH)%XV&Lwhg_Yf}Fi^IFT-g&;@ z0lu5XbB8l$zrfqO_~PW4q#GqpTCuspF>6S*kFab=@+w~upFb7K;xV=cLaf3IYUaGx zS#?WyvoXt;RKi<@?%_LQ`e zoNC(QlJ89|l!$Ww!kI^$-Vyn!Y}NXJC^y% zI)F&es1;Qam|Zi?cPgeIufIYeUGqGMNP36qL?YcjzTeB)P#jfPhF;&A9m&A@QDTLl zj@^c_G*1zBsQr?eI`vLTAnPZD?*DkWG-cmr!m%!&tF}S*$$4uioq{Co#Gy4eKiv5e z%}|lY=L`&G^#b7mE{N^U>Bz~^uT|WFLi3`R_=K>_$|oe}AA21#DdAm~gbj6;J%_3R z@JP>+W&HeG1a={|FTQvxe8&lOJCwI6+($ErnJz;sNswdCr>DkX@K&0-91NlN&YCu5fY?61IsH975farU zNXEPZO}k`EEY`r_Giw%nQ}FC_YL+{z@#*#F4Wak8#O5~eF>%hpA-E3Go$}~J!$73` ze!nH(aPZj{06K@nn5A@#iyvfs+52CL{L&u??SMQ92rxC2?hdS_oYu&yiU?Z2KEHZy zHT|kBC<&qABS2maMfejzPBN+|@7);$oj?CaI|Glu6izS0Eo!4<951v{WG&ClWlr&^ z{%~g{zCU}Zu3k~zCa3JoJGDhmOvp^yTkQK!NSgbHmqq@yAEh^K{gg^Ncnvw?!!z9U zU$LhokOVYoqlTg;MH6}^+cT(bWiklFDmVbWayDLz?%A%qR}d0YpTT;>Nc=3U(v?vs zdSSdIPv@nJO?LD2(#c#5nUr+Q+`O{H{@EO{f9u-@FgL?Cvh-=weA@7kw5pVwYaGZ! zM)o3>2cj6|S!M%O1C8L%y1M13^&61Bvb-qvq-rgJoP)ZY4Z9n+-9Gh<2%DN8??0z% zx@x$9#Z!LW{-f=aGoOR$M;S{|+!-=6$G>ygWV6TQY$og>Ce(Si_oqr#5)a>}dF{79 zvpqNCugqPz$NygMS@{Z{tAGwZJG6|Q-kf+zcu*^z{f!4NUd|E$Bed@(zVcqF5mszj zyb~W1M$V{BNfGlyHpU$y0r3-VMoV=1u2p-{g;OENc`9RETCoVL;qa$0(U?l)nA<&C?pkw>f?n2u9RB&ZBkjU+ zP7(%oRoeXt$M~0EXJe$+m-3dem21gq{fWojfxXhwQ&6EW7dShOChG%HP>`czmgQh9 zdOjHbDPEL~$w&DV1G)b7f93k*lpk&cmbHk6EF|_|te$Xc8F>YqeVc`hfAq8W;1b9-3Vcc(fuw|#w@W8$-g}*M#)$M=@HSeNV6xhOtBAzf{8VrQ@aL*dw}HFHqs(s0b;y%}TD07>8J4GohJL z%sWQ;w@(MCA@UlIRP2R>41@#hQ~Avq55|S|z@VlQ^VZeMyBH&{FZmP+DQk?i8sjh( zK7W`zOq?$uPy15k-YG4t-J4F^c)|1g_<95lhcQs2^x)b+P01wO*Wz$KVYOT2W+Coi zx}P=CR5%pA^{sa5qpi~W3}qn)JoWM{fYWq54Uns)XDEXe40}sqS-@>JV_-`f)<;TS zh;Qm%uTR(kW(0_7=rtwf$Z|jl|2X$i!y}9$aym{p^SH^cD^kt{9TwR;vchW~2l9`i zCeD^`^*Dmm9sOAj%c-KH8;Lt#8jZ&smqaBfI7^(<&wt_$*(#r>*9sx=stpQdXs_2g zwfl+^SLSvE<#0%fW@J>I2L6-COa00XqMV&XDN;;Y&Z1V&OJo z$;gV2^V%TN#m|K(OVV&EUl*i4|$7GCX6y|ABWMY5W>2EY#bSKlIQcB-EM#@6%hJ|P!wQ3~*%+?sgV)g5Ay!0rc!XFml&s`T~99`~;a4Ktce z36=DP2L4|Eumg;U8oReeN3r2J9~V@#aW94P94G1Cch$LlbZ&!(a}-Ps-*Y?F@}ThL zp^Q35M|k7;kp&cC(DD==;S2R+mh_SCVMX&n|R2ykPoXdxuc}cqbw8Id$t#mQt&uP zLZ)wUtLE0RN9y-w1xny|hkAuuiZ1(uZKDAx9DX`(knksX$~h8jhZ48NnVIj|>c9wl zJ&QLB(~C~~aojrmjB!8K&g}*TNqCWwv+}w5EO4ixubOOuwS4pq;SRt$f#% z;FgYtF2{;uR$tJz){Z~FJ1`KjYux^ClKtN12^X#f?~D7Gygx3x^lfbG3+`_Jd#_o~ zlX9wBBQ?cJ0bu!0NxP<6(JzhzH)?u49qE%ZCi)^YR0PL3d#6^l;W=)Xd- zuV}dgZ}aJW-;qZ#t>#F#Cvx7Vb&>#dNU&MZb^BAbwtb~84PuRA9v+ETNj+t|u5D9T@+!&?KTaP6b z`zVRyw_h7zd?VT|F}qy=zb!iKKph}qyf#nK@OVw!04L@gW+{*r|`xJ|B^KiVKno_>ry=D8)b`iY`pVbI>;fUdccn+CYnC6; z&n5&h;*)RK>bqmG?Bj(;ja9=t7cX2ixtT+zQ`WbUs7yq6ZGb0YZX-bc>7x6GD*BNX zMrGwT(kBv>;)A*^!i*n!Q>HS?x)|3Sj-X(BhTjl2+KR6qYH>?@0cdLM?wug9$tN6~QgPdnc=>GKxvFAH{U}sB-<(Ko~w(|+LLi>Z@ zPDXW;y&1>_a-R^z$u|vO5D*?hXpuqR>h@D6d!GxBatQZ{3flbc(tTXXLjO?KO= zT;lVct}eiyM+ywe{?gQbZ=OtksKE0}zC2b#m&MwrjHF?gRj|!pbWv8I&R(YGVZo-44@MZPv0@ptQ`!iSV!A{)Psbg~yl(QfGf71}WvNi3>naD4< zFkL~AvT#O7BB<gyYssM3<_!jC>+U}G9@~2z$i3~r@{{g{ z%{w_Vqz)yyEVu4b3qJmhvA2lFS@A~0!wdtzrV1YUpvbYCm&+%V2!27dhuQP*8dFN5 zoSK5JvhA;$k{n&{8S_&^GWl9~R&;oAVKd+WuN6<&EMViHX}-*O%h&FZpWXf+xA2xk z-rfr=@|h0izD1EvBvq_lX;gBmnUj=#syp_&2W`S26gb@bM*$X#8;kD^bKk4@JF=g; zhtg3491Z!5lxa)^ZLdP?f;j8WJTY-`iyJx-W_uIN3cD}YGuZLHOLyY^&3XG6_3}j{ zkFG$b>*SYAGH1n<5Y%BO|EZ5m5$+QLNGMq=hcy8e_r)sNGfY)eGR&+h9IT8ibjtgN%YV7MpD6*c5;lpdz`rz1z{lg*3)#RmdPZi7-k zVKDbF-_;q3wd@On9g+aSM8&j?X&pW!Cj4RK)yz_rnOk zEp6N(Bc~+x8Dy2V_W8x-$d#h%p+4p1GxaYCjcJ)Z{F81Q>86P0YLhds$qqS1`B;PT zjm9tr^+GQdVFw#k{m?m%I2rbM!5}`_Csug0J3~3{uyXTgDEcH{x~i?Y9mJZt{j+_N zNC#acyuuG*eJ9;rAQu{m+SXK^ej`IOg9PwJiNUu)S-RbZjw{3lS=6pwQ2nK4UE!|= znI=0c>JjpPDiON>R3gBx*~y9%@22UMo3VD7kh1B-x+w*0*Wk`V1n-^CtjyZB9KE3; z-Oqh~fpD@*KOa~~)e;&|XM$Hy(n&tt_@<1g?cyGs-+xzJfVNRw?`iOvM38T;FiqQO zq^OeI!O-XHnSXE4JaY4Z2~pGos5Dm!a7ZNg|E>)*MN07*{^G5*D7~O&k2m_v z=T)EIykgj85y97av+4CAD>I}N;HP|0#qG>eBZi6T+m(!9wCi$C{oR&RKSL&>GM7+9 zcZUPyvF3>yxP}oc_@V(uI$Z#}4ApQ7JCJ*M4oq4NJ!C*~8VMdXeFFg32Bh?-+xKU# zVsbN3M{S}k(b+Nw5zw~K^5vtSRC-;S3X}UDa&^(R8fiAlCebyyQFa$NTIB@$c?0*{ z#A^^0ePyOwp?Yebt$?{d$s|-eM9!9#{bATya=)DPMQ{izapS_^jLl)!0Eo~Kz~RV3 zO2W%0FqP5lFpzOgQdY^n^Ojnj1lg(PH|PNDqUC4(21w*NrHh+k$d1pvLIH0 zIL%guUaIT~P?i+JLGJV$xa-K3fQX8H-`p8#b@YAHsX$Ceel*S$i)q8^mwWn4;VN$H z%*F~dH7YQ2%s%uIu1K1mlVw^ww6_%+F?Uw|ouhtV*8;E5S2Xrbw72I|8R4c5JH-8C z{2gefm5I}iY*)8`)XpC7^=#o#BFMw0r6ashCzb*Sh zhq{QBtti%Q6E?~34)bOjxr-|tEFH2AKAq~_t?V`;Fdw63DpQk6yjK9QULgK{o8EQP}0kUf2tH@``#t4SL+**nVFEtny5#s#`ASZ}dK z7JuG`b^YFG9eAO95s;u`7^8Qp$sH_Cqr^6Lk+!Z2YPG}A=X+`&W)oRbFf-W$->+2+G@qTf`USt&==BTRuA;S7K2n7}(K2HI zrTTUzX6z95_{a!*2d8+7UL*L@L;8SAME5Q~eNNvVC`5Llh<>&2l~z{o_ETyy3IojT z;tOsayParcL_P@{&B9wY{g@uukMm4jZ6_ue@Mvs?b#&;mlxWf^N9ao` zUOh!b{9i&79cx+w3aD?`v%3HFwA14J!(`HNKXPHQ`LL%~_O_@!h0EQXuzWIvs)hndWDGexdF4~dUk*0J2A=5tO z4-x$p;|SJv$x&y9RhzwXrI45u-^$&Ae5Vw5Az*QMab4J^rDiUB%ILX z1QN$_v$vZAqo#hg;T!}nB(Ue8tF{dxIPNIGz(~BHMbTwbXUc4`Drm2Y(rjaIDcOg+ z@S3A|+aPMfkxN4E?~Dsdz5$@EpXYpF0?4g=q=U1HM_$vkir}^>2;K5W ztfGHIfXfW>ccAN1)MDayd$5Ik|LXWltJgfN>n$A_bxY07B@D1#?R8jVu9D!K$l!|j z#VYJH&r-G85!e=3c3~2g_xmYiUA7u~?saO2aL_Q(^%o_ z6p_j?pz!#!*ORjduo2+WuAA)EmX%W?(p%UvqNKl7DOPX0rZd$Q3|>)9P5?mEoGU&} zKl{2n85sri<>oRTkMR4&;5S*`H6d1GK38VmeuertW7k|{7j+#6h*{f9F9sJFX|GUL%oSAMu$1El6vJ<|oV%bO@Kxs#*)ie`M^ zaSKsWm0(}NM?^=B0DE+jXRD+Lpy+Jd%gJ7`aYA7ud2q6kx#|J;^s#|o%#Sa6x!=DA z`J0{(WF5_`OLf~q2|*<04gvgGxr+i>bL#dC;v?C)Ix-4VXhsZsmv84ee* z+{6~ILTY`~`G2)0>ktx- zn{cGg6Ht5SC|NgthTgF7j0b9CLlYae;UE0`(xN?S_wNL3oVIYCsG(O(S=~bIt4Akk zzbV)~PqoT)&&)4NIbjV*mc|H#<`tD=2RY^LS`I9*_x8qlByi2Apq%2Rhb;b@%ZZ1t zIIhS{qnl!~Kc9~BYT=fi**MP^h)i&j-mSLh1xoBg0mPW^?(*D1TlN-5iJ z^tm#O7u~ibb&jO*Q{QB7wtu(tQ|Ov$%QTRrfwEzRX}HucZ9C# zPc#LtzuWWf9KeEn?Tz!&xApoZ3pmLJkVgk{414(dgRK|K_*rmr05!e=T6qtPERlqN z>e+nKtb{cye}IlI+%5LF#o!WCB~tAR!_q`$nqPJdJT5A~uq}23md%^>85)pSwkj3B zhnt&c+y8Cp+n~@@CsXXQxW9==?jrqeM@!jXVL5wrm%G!>B6Q^$h~`<_aelu|dxs<{ zs37_pY3Qc> zdoq;7J)+YayiPSH&IuFLb-__zYX!VH9W+!G*2$xD_E2w_%$KgmN`FAh!96bHds}A#BKfJ85*c$%PbWX} z&~D%oBTrmz9p=D7wK!F71Y%uFZ)mY=*fWranEqn*%=&8d7G>EcZT0<>x+J6tz7hQf zfU*t#1!cEh81rcIj9%ThQJQjy+x$ypXR;vf<9sG!1sXZF9ismgHfaPoIKv}0wp!p2 zxUJr`=)qlAx|eTdywv%_CIz>EPW6kM7iVSZ%v54E@8ED4rOEO>(c&|ta=$k<7fvov zg3I02c>Qn-lB;;3^Q^jv1cvcnRJ&GjZ>jv9h7rI#0fktcPwNnXIa0(PhRK*GU-h%F zn7Me($Hex8frZ)3Y}YS;;`!b#1N5eiJ$8!Cmv*2bee)JCjHA3wma}E;+OE z6|LUi)D=$pp>D|%umkwcZRg-{#)ai(H7zYvvnu|U2<)|)@2H}Duv@NP*QQ3Jx~Yo< zT+N|tFe!t=u`?O7-v`82M+=Xdmz3c_FiMk?10qSATE#;#Dix81nvbY+<>w1JSo`E*pVH*j@|cc<5OEtg1RM?Wk7nXPdw@ zggdf7WuO<_ns=3s+0A2&&cI_XPp2YJ6*j8lzZZ`P`*Lo}UuG&b(Iu zUhfJzhmvOwYo6+D68oT{QG5PMcB4n7+uP$mER8x!&TqsYHVsc!DnOy8creSg{qNMK z1zi5Tac3?C{;`Jlf*LS9;Bov>29aUhEWyG+11bTbvix(sp)lP*S#7zP||G(toA6 zy`pG!8At5sH?o?C2HBNs8~EC*UE85oF6S`7RMK-lTyp@y`bw_}eD zQEXrH^fS_c#*)vop#Ezk3c4WV`TahAFL60L!^2d$S-qjBg!3I^#>xuTMYt6UV?IOw z9pW47xH43{}(2=NdZsNpWrG<^El*sq_-#yk6`Y9txfh~%d!QKaGegIoF+r;^q%MTa0 zffL#;BTlVwTtb+PBzC|m1}vOqG~|iCre?_BxdsqJd8^tBkTxdw#+_`M(pe=-v|pBc za6ma8@m%GmMW+!7F~*A6WsotiIm=Tqcgv2kRv8&Jr{{Ak66mVY-L8EzesA!XhUl!1XdZN| zb@k**oaJkWDpiI-+A;K0<-E?k>7Ej|<*dOOQ=yL)t@l}*Zyse!%w_i7A9&h8hM)A=L;Jg)_sP*ELXyb0)@2rn<N^#D@MNtmsrc;KB!tsZ2J;r+xxfJ%1 zkakH+pII!^+~?<_)wfJxWck%7vQ`M51&06`FFrHIvg9yqR0B-l zC5b~?m-ecv-nN;zm8hgJOyw7{%suI`e6S@_Zt9Eu!aIaqS-<~ROn9@CS?5Fby6*@1 z%KXRxH6i^EHDM?OvAz1Nt^jubjmnm(BXtn)AHy7{vI- zVQNUPUr2oK&@1B-@CG4YMzMYE2!I;%P9iCSo4f{VE4E(?1hrXJ5Q@*yk4osHT|j&# z$m5t)*r~lz3oVZr+$vu<1Frv7J2*SmIwZO=k|Bilq^c#0H<<-E;cgg?L+&(r#$(us ztI+45J;a5)>PTl+%$7UKNcc&jE`c2gBj}Y&ZCSnf=GJ|xDY1H6Tl@yB`1kSQ6FK~8 z0hpYme92&-DNO}P#(Hl9{a_i3C**aFYeW3h%frDxr^mTIkhD`mVC4ffry^MLG^=2- zpMt*FO*PiYxaO_Czd|({vd1|aAM@~{*|y^|K4Xz9!eK!OuTLYcV5n2_^R@j+3RQ%` zjHQC1_seGDOrpBLei&9S1X2)uhG$X+tA zAbMXU;jj37EkE*Y*UWQhO^= zHILSIJh2snw!8o85&Z-54h`IH4_b*hEht7cf39Ola*|uDb%G$Z?FQ?O356lWZn3zn z{+^zJ^dC7h-}o)kd5pHq6eIo+KB;Dyt^;E$(LOD>lY_#S+uVuN#aiXiIu`?BOl#BT ziaGIQ`>Gx#{S;8yad?{3YoIRI_Vx;1BM(-6Rf+8Od@X0Y)z@!4c*7R!VE;~Nsp`zO_of3~cX;dm>|^gs5P-dvoQWGI zP(3>Kk)%##K&z!HCTF1EEdS7aB>Y_j6g7%VKw`Q8Pp{fgR`wIQX7WqrD#WKx! zG)6Q4Jo`sONxHFdS5nFS!T`;RXr@1DKq?C9h+g|Kh;SeMRvh*dW;kqk)AFto6STVC zOZl{tV04-JbvFjzmus#gX~hWSSnChJsXf#SeV7g7$WQs3zl2{atLHu-w&s=Vxo{@$89FG_JFHL z@XsZ)CbSk$6pA|rNQo)!`+o6Hl=EReq@wd3%XPn6sh52P%y5yD*|l_u>O*Y|IaRPAV^WYPNXML>sYt@dpQU}51_;Acusuy}x>2@SOpmy6qf<<(0nm01t zv{}ZV{;MTV)&7}$5c@sou!{Lmakj|%Q{DO|+}-Co&eKma5)G*}4T9?fd6M^9&dlo7 zZZtnM?61f|FUrjfmam0LF0gALm%;5q2(DYL0O0f-7;2Mc6MvY?1YenuEZt?{0$gC1 zo_F@|W?WBR{-%Eufa%2%%h4bqhog2+3Ab!>Z7oiHz?t|I7vF2yRmw-6-ZdP(u#0Qv zc!y85*f+QT{xMgt@sE9_`ir$;KsSAIc@@6>Z;R!kB{uDtDgg2*T^(FS$L_j)`A-j} z$i(^o{_T?kf>pa_{=FT1puCE~H5@;)1oUXGad8DWW&I_^0W+wA|D)mWB#eg(nk0Z} zzGOpMlw!r_8+FEUbOsr06hYv1kfM}`3P@KOL_k0~ z2!w$MNDGGEaYjW+AdDgesnUBd2_cYCNFp`V1PEnFs0l?#XbI&!IPbi#zwfN`&spoN zb=La6`~@pdlKXk?eeZkkYhU}ilM1DK$G4a6m@Kop|MQE7f1DVLcxjw*`SMC_Wv^AJ zzRxkQ|M52P&+C`}L!e^&U}-|&skK|C?!LgutFK(VKVX0K{PW={p~qAA9F)~|>n=84 zY|LG;z!=ksbJ36jImI8$oKd$M~8`T@o8@AH2$^&GypVnTy^`CynBS+pQM87-} z)@12(umf7RXg8ES*=79I;1jfYjzM6!$VS>(wuR^U-@XzbkrT_5crO8zN>D%lbfLKC z>re5+>xhK&GZrk)k)YZ4zlB|P_3LTH{r9OK>3e_E+cD#|QiG|$0|9mSCt!H!;w*}N z=D)vu>lwWq2K&BgNJIP99vqTbB6+8|ar4*RRRq4L^S_rJDY*g3a8I!G+`QTIuaIQm z>xz&uvwgU8HS@DAcZU9Z;VqdSp`!DQ-|0&*+l3Ih*C@^5DsZSWcz4-&$Lbe-{ezpE zsSwvEv+qT&zK1^lYX07@YCd_dx=C3=puo{h(KCxfu&SG?;HX{LjulnT{;9_BjbFZc zo_{cbZND`C8|ILobGSNqyJzL*1BZ0-Kgba1!d+=W^g(H3Q|3eec zP485To_Iv=QhEfDp;+6({JLwN6V;%gO-{~!G1-cQo&{)T&- z1trKG9CEqY_+}!)<7tCk!m8yt+ zDtFuVr{%_f1sNV4!iP^2Lf6osSRG@}VpQ%+*ul*+^&hsJh_@cNGO4Pw0H={5s1ID3k{ zkD_jf`sS-?zdf>@BR>tmCW9RsKKgk~X$x__A^4%F~69w>1g4PlaLMYk! zS3Qd;-`MzsrY>7TgU3yWz*M$=i%!&TROOY*tXesVZrK+6;mH|)){(8}3tpL5sVjq$ zQ%)DBTc33(6Cy}k7Lu04Y}%a!gkKg}HbYpJ4)oWWwuGbPe98B~=Lo9NHK$bX54X?F z2GCPT>xNIk?Q18Wst!Zp4nIyW4rM9UM6T41JiheBLp_x+eu_P>SvE(C7b_K={H!F~ zpi;vw*{E?J{g@iH_R_T{U?)<;08}zH%^(pmh3J~L`0DRDV3pyTjhY4+jV=HTD1|_b zRCP0oD7G93zn8K%Sm=DCJg$O#BNB4%eaI87XCl{@K5w0PLrZ0_d6z*2UIHyGnbv-Y z3X>`0$vW-El>WtBnUZG--hPP$Dp)89<=Z*$Z*?K&&~o4Be^;D728(bEY2y|jga&?k zp4wb~h2l4IErB{uP*lHhRYAs7G!w;lm-Mcv zsam&>b~EckHJ`BfzQk?+=Cz5PqjK_xqi@a3yn3%+4V+HFH?%4XP z;dmp|*eYUl;_|=8Uk{9b1Q%@GzYvkcAeLIgmYeb*BY#BNl;&QGmC#inW+qWv_2Vq{wchRWcErvMF}xlOf9|@A8(bo>flTlJvJjik9rPhHje7|?)Zu^c}Ih=n{(2-IF7gHLl`9AzI z>H3i|Nc&sNH05Qjya*H%C%V^-kX>d;)lHQqY0z zTDm^c`E{7UIrhb!7me(*oo5POC7Ft(pu*$kOXORe|IJd9n!n;B>Gy^y?Y$=6 zkRCM(@u(RT*+UDV_DEh+Aey9}^UK!IF3eQBOgveUK7loX4SLqOy4hq2I!E|N-|y~> z8)J@aFIN#!eokTn&=O?YBTAon$2#cUS;;Jw%dyr&+U2){%A4Gc9Pv8YTCOIPJTPN6 zyDroz(lea3oP+fO-*3Hm@S+3Zx+h#jn#P;s#yemPteZ09`4V!ZB`Rusja=s1qbKId z;#YVCt!Y6L0TfO!X6zjc_|NGtvU#+4pq z!1-iIfPL@0yd1PAQRz-C#FQ~NF|2+5wb04$MaM?FQ3)SKSc69J10s}gZe*2=t(`fK-8l~r0le|~cl@5FO|&1`^e zH1V;+Ma9&{n=~p#l1*+b)Cb#?Ur!V`COtU?4}8c|gcg}3N&)`ySxEV-r1~EfL-Q7`LRw+{p+ZI$E!HdOgt(0QC_fkSji}?k_RaA-Ds!HEW z$E<#E+UXuGb7k-Tu=C4hLEVGAK6+U3+BzcfMKikGgo(bgBEKXYCf2%Lx}(f1M{dOp z>@D>k*b0XDjP&)z+@rOfPofVjq6h?o>K>yKtL5nhX3o&^^b_d`D98t4uwuqd_D*7q z6n>1(xQB`vD89LuC_~zr&l2`PdWEkue-D2=VD=cy2r%sOqQ90-f^1g2kaCHLsWQ#e zal8LYj8X}gH|>>mcu=}*vo|z`*-D>E(0l`l%!5RP11;=7f4?``9?GNjBG0d8MF#HG zBS)%l5SF(?!Ns#9t_k;T(8 zVX8arf9gNWpa&GSs{@>yoghc~jq07OnIMN@Z0KLp*jB6AA0P~lp)g@MMq9quCnTav z#kMLgP@9}_LtANY5JAsq2yov68wpEGLY;97On40V(UoKxghRP5gB%=Km z2~oclt{Mm;Pj6JYCBCl+Dm{aKSvdx>$Q+Y^zx8S|D1&9!9TRt&qHjLfO)Gh~6*Leq z|IWiKadb61dWh%$QpF2xP)XQA@cLl2R^iMw^y~U)wMexUZg3J=zm|H3;X01#90?_H zN8XWrS0j3)S*slS=3z+FXRGx;Ga~GBuV@QFstBP_y@9M(Xa0KFs-I^^oxJ3cOj! zNhk@FC9EdB;TRflee_%N7B{DIoQsXcabx<2*uAo2!!_=c`8N-0)nOBK!J2Cm?gZcf z9ROH)D2!Ij5+0P1oCs#!*kQ-8UewXKv)+u?bTuT4HVU|gbvM}FSf$?yDNyS9`T5zv z^zx<{OWx#&sl>)7?J_Y5&6!$Es6||oep@VI0Vh(UnB`a3a|5!4Sc?S5r;13Kbs{#f zjhR8*wVc6Cv?Hp1DsALpgPc0WG_edN6O*@Zmr=$3eP*W{d^(-)cDqu;WoS72ZK|p_ z*byJ7f4GPFK)KUoJ{q!BBG?a}9MFsG=-7Gewywt1Lm76_(iC`wVS|wM1~iGXNx8id zYnP?qH$VtrHaaA!;v#q{p)c_=eD>Yc+6vo#OJ zIvI^j^-CPC#7@+apg5~4=QGy}L2wm^F}CXDO6a4-^3>Y*N2g0iBs9ysU0D@uI0^#szDdnxD01LiLt**9tMSwTrL`ao@;K4SSp0f8{oa^N!S+1ksk)L5*ss z9KVfdY+oNW$&S&0lSV{P0$STc~UGoSSWI@6A=~qAkC0s!<0KU%4pn7GIw-80a;EMaq5H zw_$Gbo;h_WH)u#WUb|j6@c;;fMte%aeC!=tA3Nb;POA~VWQ=t`le~ZXqPHy7h>fhsw?fDNbZVDm~xzA z!f;@X(nr%Dh2z*o_f2~h6y=74X~|2%C?%hH)>G(JpV0UYL`Nr)ZE$LG(#a`MN5?I6 z7Yg>tAL7A0vgadcup&jzKOcQ?6BjT;`QYW;_Et|M^Dy}d(>dL1~ljK%r}pPR|WN``!d?Q|=t(lH~vBpMBdE062x@*o^vT^OgsUdT zs7|}&HC6ZSgU8*O8mf1Hs)*>SaNnu~Vt)NdFHh^d{yrI{a(I=~=+*C`5CUp1^IV;W zEzwt=N^SA9+ZGcxd-ZzpB^l%sr=kW@;65q-Cn6k9>`Hh1xh4-Ktz`3e#yUSdPM1E9 zxG|=r2)3zO^Ot{Vb+;!Ai#CxbnI?+qsDla%{CsRK1>qo3eB*%m!t& zT>PM2?b1#F{kN%iXFOWNEHa|UFd|{Q??ux5>a5tbiYQbpIq>0LVK6C*udw|%b&d@3 zJX$BL80W3z6Zvj_f<`3@S2OjMfQ21S6%QVk8Xjj}>q_imCV)wd6V##Q``97AFztRlvIEr&y7mP1`8Z;_qOuuvwx z14!fHFH3E9-zImHR-9>S#&oePcYLQyX2=p65W>U)j|8MkM-_Ex>{ORazF4_ z=an1j^pwlSkogz$gm+0Y?d`BUM##&b5_0q}=BCXz67sB#>3(qCA<=8sZ7Q8bJX9U{ z3BEbkMxiPl!Cf-03-)X}pISHkx|wNojjDJ;%@rM$5W{iXgNyxdx5v5ZkcY-7v8g&8 zp=?bMw9F5s$LJ0-rgImQT)ThR>Acis@+`_A-UgGMi+b@W_OOrRl^s%MPMU9J0)b-*t2XJ**0-0PKx zusei&z{)C0P0|9R4Z-CKqro&mw3E6D(>4Ogvs<1-JHnvr8h>r)3YLtDH*euYG|@hY zBD7E9X<=E?98o;>jM3O|e!kNz5Au>5*p$#ELzwMiL#D-`?RW}L(^EI-gTZ`z{0~x0Zpq7r7gDTw z$qDbKfemuwq=tNY~BH4iVkaI$iI=!90+8kxs zYHBFf!qHn>D=Q?m<0OKicBiR9k1{14gpzde@=E({hOoKkKdi-?w?P(QdUdLQ7_x#% zu^7f6pV*7{@>Pzf6O}TCn62h@mv@>w7;t>?YbYAeTi|e`o!!YR#irbqL?1h`EojCm zN#_Ah!>^SXiP(f!uOos9y*LS(UMEqt-uqmO#ox+#HSW&7 zmsvI72cGLL`*(Zg8qB0uFo8%*A@WeFku~|(!j`!cZQ$m{iXMh}SsG5}{so&VMISWh z!a)H~n5eE_l@3>7GVW&!)~Qtjn3i&J|I|IOvLuBfqY%%7X4(fz$DfgRi;$a+?$5~c z7EqKQ_BWm_rl2v}Ck=arBi^v}RbRJKu5w67TSOJi{S=I$U8wEI+KNhY@(PK+4_m|9 zW+LzUAZFF9gO<+38F+uxS4Oug-W?BNP5AEarG;{u_5B*nQYGf!YH4MP3P~aMdmEX^ zMEm`8;cT*ToViq5m{g1Wn7t=bAhQV{WWuW`6|lC>QU#dw;}ToGkG&BEKTc<0 z&1I>I*_Oh+hSpZZ>b2_(zOH$}7e^_CqhfF!5$}8%2}ce5MTPW?DQNLSVflGe=!vev zH?u#PPEdA?6NT2Lux^6A}@N9>-cz|vo>IRsAld3XY6KQGc%{2)f_gQdv|(~vmG(Dw^LKT zedvAKORh6w9Z~|rQtPblg0I%Ch0X3$xUje}#kE`gCkm1Qe?N_oksgO8-2GGHG_~@6 zuis!&z!O-ghV{D(tBI1Q8NF#2uDgf0g=(n$d{<~Wk*qtl4l4;$S)=put9+)Me$naK zr^18H11UEUwOBVaGhUM=Z-;7_mxjwl?WsP0UB`wEn6=R3v7ZC4c=L57%?fR>B^|-T zPli*KsBwFBY5XvnBEL&+XIBa#jgZjxONonzH1AVsn0}Qag;X)M%#8LsYbN+{FpmKT zRVMf;rP&K-nHDTxw4blK5w2aYlM>UM%)aG+!4B_bsF0e83)YfS7rNkl?ufLDo0_us)dp^StE)b}70yUZ*4BxmZ0hvsc_1AWTs6^E&gbv9(N< zn#a@ymQ8o5pNGm82oBO2Z!oAMR{fB)7qVGuH&qYjqxllB7;hH_RS#j?&<~O;zVf_C zY_o}re~oU40pF00x|L~Sz4q&L>hQ2eop!!agL9t>025z4OCNK5MXQPk|Qe^OCa01&>Hm`K!T$yRlmV@?a>LW$c)_+MnOCL-pCuU|;it(V5`NDaKXH#H7K*ahJ}VZmc3jJ#(SxcShf$Y}y@=+`Z6KFGM|**CO&%wkQX`|An3 z&-SV5jQ)z{U0+RRu+8rbz4;|SCtCDiP^JTkceZEx#|uSQW_V^d6mjDP>c_?9awHGlY6VK5sQ}BrVhsvQm4p>IOfDl z0Ec>wV{tXvu01ZOwJCA3Zo2SUh^goLrRGpw9VR={PjjWdR7N#A+NN76zB5xRdo#Ak z*hav~4quU#`11ZM?@CWQ=n@yIHR<$Chhq_k-01U4#lBcUMOA3m)BMkplA7yc_{Pvs z<071@jz%72vy{e&?dXwINN<%G+nGqvzda}Q7T1^x>&eKaqGZNu<+5HPrmRu4`fVxl8=D=+qb0$DL@(hKjhV85E^&0?X;@PSj? zDlQt~%ym3;+0Obr;O2(pW}4Yu)@%2gZEubSBTDJsic0}{vd-rQ*ZpfD1cvTPukDM> zK?#lgXS`!hm-jyIz*9AInvkY(V|s~J$ip;uVl8AoQy$*>D3h|aN|5SHe;vd_9K$ z>=OyI0N?kW8g7u(Q4^hb;@=%@D@Ls|OUOPq$egO5JR_hr+K?TqBBAeZS&l23?avLK z!mC2Nx)!C&KsU>h?@o5aW?J01G2udMkdc_GZfuRDX`4*RRUtF=Z^tW3r5*fQw1>>i zOr~%5+4CH18>dvMVUJIO;sa4*6VDQp1lwFg`(+|#-fa%w7Q*^$v{j#0Ua!Zc7JKG= z8?1v7TmexKLY`)h*OSe(9r7!IlW6KEoIU;n@JxiQ zF2kI`rUqrsSMeJH0ldN-STE-tKaZL<4D&$bty0^irojDAUi_XfsZ${rX;+@V?DGkx zKb^M$Tip}ic6}s#W&h*I^n8-<+I9#ZD>I``AmXQH*Li_;*ZXi~le>>u0|0!}`vtzK zO&h`Z*x;&kw48+dd~Lt%7Yn6aUtVN}0^0CaZJD!xAgg>|Zv&~Q?;CD`%NZ`)Jed&Z zVba+ZXBy!fna}+AsUrIIHT!wA^8=3Jt0D<_PiD<%EI%R^x)D1PJ|e9DV=y5TUD?$Y z_k$$-*eY}BbD$=3I{!2*9DILkBh4Lue`p)<;azX2Q>6K`ilU+(Sq>ccq0DqF^wH+s zsvqPTaQo$*HzU=l!NYF+yq>Jm(9b2f@mNmi_6s`bl$uJQUE8PDIAL}fD5#~^OTgg$ z1Yog6Vl=%(i^4-Y9bF5t)!JU>9wU7!o1`JRzS#qukoqqzbxe`pu$I1wMUtI}IGCku zL%`6u%NpgfzTZ;Cl#s43No?{u)C&^GYOg+B(V92H$_yBefVhfD+2i~_eZ~!iSn2LO zXcMcKl4%zk3o6+sqP$}`@05PxT+mfll9SM0LT=PFjRv=BN?_2ZEKD6$TLvee_@T}M z)wD3>=<2eIuaZkWwXOx502ni_pZ6I=M3CnRB4Sd_HH-hYIBLUGP0e3(JHD0dPp9=s zQ#uRqvauhXY=|)5A#*@^#;G zLVo;hI%A;AzO(EJ&!@ga>e(Hi2JD*Y-b3Z?dmWG zO|URBTM}Krb_`KirPgfb(Qusm{c?_Jn8e@P_=Mq}Q08^AZl3rv>3Um3uIAVmUPaJ% z#hR8IjiI=OmbU9=zEfg;C9h?L@g*4T;08tynt}8tiDh_n`?Pu*v+&y{pBcLj~H+3_`AT`RL~d{hvnSw5?%N5CYjHWT{-> zf{Bk-cwm6hNN@Q1S`KS$^>e<&x~sgXQ}SPkLB?YezpkrFvnzYgy2gKW00T9p1@!@P z=#*T6kG&Trf=6B;i;ALN(-;GY+`*HH1$0bfFm`rJwCNHT@^o6ckm6K>4 zDC6AOqHYACj7ppnU)G3ArhGolk5Z4>Cv2Ml?t2ex&3HYeT9I%(WM+2gKqe?v7_wU| z(qG)wp9}zT8DBzLb(OV2O`s|-%}TP!L<0M*t$=n#w3u@b!~ zwn)>b*y}1Sm&Tv&dPD@y)VwwI4brNO1mD*I>Mp9@E(+k*pP>$DX;s~zZC@7O&T8OG zI%gq^=}xU`atPAY@lh(+jQ_cbH@!-X3RAz^9cL^Xv_|44w+-GmhuUf#;KP*d2T78h zQn&Ed`m$Mj&6jbX|mTxxJRWOiP%T<266? z@%l*X@r@^DWzW(=YZmzKJiCGo#VZ_5&mLkSh{t zLZEo#!<011gAgQ7B~ezqaBZ<_&vDsf)|~NRLY_DK&)ZQk9zcc~?O@ZndoAofINmar zv^q!lmIo8&ZRpjkm+?<8il|D@^)B5D#hR8dbEs55Gqqrg?qbQ3aN}{#0etMvS(U`S z0EMc9T(9<>c~f{!SX0$VOTiT(=dj@WxX-m1ZfrqKD-1)4`@$wI0HNG9sG^<$lLK;!{r<0{Y5Ak?t#aye%CRbch82@5~3Ra{F*;^ zMufVEx(Ds5776nL#{-0tQ)kLe47^Jt0$SF8S62@4w~4pgd!Y3&lpOVUY0J`}{WEUl zRq)*!Cl;M6uY_RP`kguV?cPgLla}-FWIo^E8L9=tuVW6zF#*rs?yG=DL9{-KiBB7fyF>q21bgr?Io_r5(KHL z7yjjKLr(1Sdl)f-x?U;?7FtbjE}ibm%A0Gqw&YSKo4=sk1qgbhN%-)CU-3t{-|dMX z=1E5Po?_o>_?Jw}Bvve~_dRYZi%W3ceK5Xmrx)pVh*Jgt4kcFdh9j|Id$VD5=V&tx z;3BoY+wf+q;^pFoUJ#u|I_2H~h897cpG`egSA^{8a`Lo`zmP0^maLmyHC4Du^tC@# zaLpa(>vdnNxf+TlWp2$UhPl|bNv|3tJob;{Xk|CM8v<*POa;lM+SvQYg93yAn;xaHE zzT*xjc4bOeog-#!tK{4?%S1#Wy_o>gk$i!wR-WKDS%0&sFQfT+j9CbFvG7+J(6&eZgL= z#B3pBpDlhereVYN)f(z9O1X{?y@R#Y2Gy(`_lC^wUQPh3`8c)Spe%io#kNC~hF#VLd>{a~=8m1u^X z2vc^|(`u4iZhuNb+;yL<)5(H< zTHgFtkFhL#zo#qi5cyEJS}!lL8ywxU-M(WBw&D;YsG>G+IiWM}9)T(+&LoaIY`KKZ zaxTNdx%<3gLzw5q@y5;OwNERrrf}PlI#RCPuiRUY^>>ZgmvB}#XG)wTV zT0)+t(ttI5S(FnoZx-z4Ec&>ow^X_YvjCWjntx#b%hWADcu_>)!^XhnoRDF?%hZrx z;^si7NWgDUBte-RQ1bh0eMO-|~q6sTMU>R~>}w9vH|cU}}+ zEXUckM~_F{HFXwsemC*isCI5&UwMlSC@ABsHv00IAzg*B{Y;1tZm)jwpkg+SltQ2n zhZP{xgqoEkn_;cZ2jv}Ii(v;Gm>&CDLh_Tt{T6k&-vY4P={!W`O^ke_&LGh{eaxxGz2i;j zrEDLQ|3vQU-!M&86eh!==DCmeN-;r}G{2T%AB?sQ1#fSE{`woYK$=0C)Lmhx+5kNt zMJ}!&4{q>a0_PdgHC@W#A3PS|>X!I}ITASyEZ~%-$#aQXDJad{I%den{9JPpbi2uc zUsP3iQpiFRKzh5l^NSB$Q_}Q`E02rGi`GpQ6*gNT3fxIr4s-{1-|eK;>QElhN<$Cs zH_l4G*o*QV=e11K@K7&6cj6OXx3OIjhao%`JQ-~au{-Hk*0DNrD$%E_%L)28KI_3b zBgcpkt-j93cT@njM3Wk)d|6}Z?OnZJiIn+uR{`bq9!vP$n>g*(Py@8@Qm`D5@c~P` zeDeWzqauD#RxoPxF?ktg`~x$$XjM4j3rS?~)XiuwUoEVZ>wKFx1fnCalyP1d&0q>@ zyTz16vS)7Uf{MW7gAE2fbkGfLthen_^mcO?Ch(49q>WCyH|$B2t+iIQTZqlOkX16Z|ogAn|#`p+_^mzmiW;QnV+ z73FCE?&27;_dqPXKGt~rU&RxGAr;t`=}=07Z3Qz(BLM$YNYHuu(%PW!M8s@d8>g1> zsN^kkTQkrq1I*D+XkKxe>J+Zh=;7pdj# zwYP93q@oN?Vog2H{z8#byyBwCS`3fiDaHk@eXGOYJJ)76%tE!t+m|u##x5I*I{h9MrEtKEb*JKK@fx5?1uPs;7hyxz_Ow}y zIag_K0VivLOrWlYY1JcVp4Wfb=`%{LTAq1!%WMjiW~94RRh7iyvtx-lx43rBsJ7jH zmsN{j9uZ2MCs@@1;ZnggKMg@dov=^lsPRb&%P4Z;O@oMD zNQ<=z&W__x$6S22ZXx+Psy{q>yiqn=;@b7fCd@+n$eReY_PG?W?b75KZ-D4FC?rFw zjkICK64kGZIV=9Uc3ha?G%ENuJWF}LM}hk>7+VRD-m6u>*o?b>WVB&rq}VB7wTR+d-z*~Qq`WGNczpk>jZP}%IS z-&P!B5+C8PmR`bwtQ}7wKGR&+xR+wSXJzdeGs@Zi)mlJ=F*g*ae7HwBaDKzgPCN0F zn+i1=bs{4aTGy9SwLI*#20HJhS1{|j74dPlJ&B(BQn6Z?ncCEA-Y(_17zZoihKBB} ziP39nlg@(j!ZC7%imUrpD*&9W^1P7(ieuTFyCXiK^?fR5)9_ zNVsa`FuUR2F7`{LMZZBAoJ36bX#ZB}<)IB;^}}bnj9`AyCw+2)br}c|(u&~7V?Z%_ zu0fv81ehGsYF30fZh*epc}?v(p=0j{u2Og@zOOQO=1NN{d4KKUyrETmUC1;+H6=Oj zixpKbf5(S`;gd~XV8_;l%@GVFguz%vD#EBtY1qc^!SNr5N@5A|-t&S9r5HtMx|)g% z*+JA>f+pCU?sJef(GmT+oF&y!GhYZ$8nK$qH{)$;H~aKA`aG9&p0ga(FIKihyZ7%r zaVWRHsv^_l={~xMZT&;y`kR1H4SbBs&FBRhV+&D8B-WRS+JrJ{U##tXyzKI+8{=ve zfseArE;qtP-(n)C2kh({bW~*km37d$VYf<|Q5&(c6ylP-u_!l8%AIgbpO$NUQf%_U zae<0gv)(K?qtMjr+^b?Oh|!*0kmf2zd*#Xy`Ena0gzXAxSqi1FC}{E6`>&3n1z#2? zwNm)mbZ*2$K66y51hz0*@_^h>)Aadao!vlo_@kYSe9-U{>F0-KIF+R4jp4gRms^>G zX~zH#=PFa zk5J1nKQ}U#j6;Ioax%-r#`H`E3RfQLfMm=JQ^H6o+{QkMWRGryn3B8~h*n0&mUxhs z3ak^zs1i<)F&F(NVD7kZaGN^&`QjRtN{JD~tn3vZGW2>XyeT0g;+olGf0VBdV3fiX zDJ%PWvgc8|J+bnZ1yw&3$yPlRh>6J$=W*?HWYzIPFP5(qDaGMeqno&B(pA-e{*DV%o;bT3oLs ze_Kc#MpG31x|j6(v(P^nQT|N@yvO;UjH(=jfw|^GvNd7YQi$+uXJ*Md-SxxyonH)Q z@~(s^h-hiJ$Anocqz*m&46z&!90RnJ%*5BY4Q{p;0tvrQRJ{Ar9MGT6Ae1sH!z>xt z;CLk`^eKTdp~%*B+jR#&%s2HDk=H^9!fNgOCOVf!n!;@| zrZ|Qf&6g;k^fAJ6`8m?dfn~oEi~1=dY3-j*1fSgFW8zzisb?e|X}yW1Kxys#Jf%IY z!GT%mQ2r8$esPx<<-I|hWKjReUnxr=(A3T~yriVi^nuo*vHku(9DMN7S=-5N8!Mi=3$_!J z%)nh;>5}EUlBC*D^Nf^g;e(c98t|VN3qV!Dy%ClE{VJ*i#Uyva`d!}V5zKV&FigLH zf#9hb`KV(iHOiY|W2dfG!VYu7NuoY$O{F9831jx&JhBUEaYkjnl$OM{>GG`rLhYR{ z-$)(k)h}&Zqa)83QT>f-`-tRurFa#!o_OV+?_rh&8d=GO2i3f+7s_G#W1HDhhdm0t zspnAOo7zg4Cq5@xOsZptx`(5Gulju~Savu;ZfAE>(R^Y7V|9LB0SFAA%r2zhAMrk7 zF)JShR3bm*@h5xSPh@+S(kpj=yy^>0)TYrJk6nkr3@S5*$67~=CC1pI*rUru09&cy zuUgDX7IMY6;(}#5&X!_cAuj2a_3-p+dW(W;V*ssS0CamisEZ>qYQ$Uq#2SFwmTo>M z-OIHCT-|c{C8LJc0xg}uxU`ww;r;#>%>@!hundJ{aOm#Dy9+Tz(xj=z-qfJ3vqpf3awXL zmR00@s>F)W-1dhQ&yDo*uJdeqdWb^Yd5ESE=6tvhc)>=BPM0s5nf?B3RoYk&&@l#} zb0c$^=l!w+L$wpNyqVLI^mx&J`pY^eN^kgYlM~UJZGB6Q{aJsG_tni2I)OHaDza!6 zc~NF@&3h+9gzCCbT)7;*f^<{Pd;DizR_=QIPEF(@$A*e;PQY!?*@bu;kWVL@kJd~S z2fY2;Qr#ouCb%}lRCyF31g}+P$>r}^#DPNW3|o#!EOM}pf90tbRWz|`Ff4Yhxc@}9Q-S|(#`eQz z=d(ha3`$$Xj8Tj8tvaB8#2JJDIchX+FoHw=#@dJ1LS{GxYspO&z6ZN?Pk*}5Wjyu| z={oC3P>B^#x~#Kpk9}F@I!jjvAg>zb@&!FaP9Bl?PM91gW?PY9U<-nuFWM|eC(Dz_ z@Z_y3M3rKer0xClVi36Q%zXpcO>eL6Ru}v6hSr4Bw_;6@Vy($a9yS@e!FFxwMIG0A zaiP(Xi@iZ>J^Gdr#Fx&VpQB*b<<1Rqb6~B@#LT{GlY!-+eK$S^fL#I%Tx4`a5Wq4u z%@S3cthuN<-lPCcRRJ<;UF@1zt6nStv$n_u1_pg7zNMX^6$~Zm-7UBp*x8+He7CT$ zJz1C&KQ#NP-=y+l*y!jQWloiM7#Yb7hxIPlWa$Yv)`4?`(Nm_cks4Tkr|||Kmn6t- zx07O;mkWEG&dIMv%Bhy8p7jWx{`Fx+$M35w8`8YH*<~-h9^}6N_`_-iPg;k%?^wsq zlEmYg7l=0-RdCS6RaSLC0MHBraRefx>QL+Pt(_II0_ehX&V`zbE5^vPd`ia5rtRH# zM&z@Mq)lb-=yA-?C2W2%I-g~;hp+Ld1U6R!VAcRk&6g%=gUs5xpdRi|#`JV=RB-6I z^2uuF8zQ3f&Kh;F37H^W-$?Mv``$_D*@M8Ka-642cTcaU;Y5NLS2L?60HlGBer;3)O~o~dA+bb{%%y`E9Ij43DNMuHHBds1+85RGE~SK|E#rHo(LQ^ zo2b&fzUclIU|ZVbJ$11d=S)Ks>Vnw368Ln8_U?o749yajfum9b&}FwfQW#ryVkT;y zh?d!7{VuH9Zc;ZNvOGI)mU-UU0e@Mw(n;Le2+T(|brw+ZJif7-e98CMwmo^*mHf?y z-xN-yWOlg|cL-$lilpAh#|7Ky-A(eYZ2}Yu9_cqeH?SXpU8!S&(F)7N0Qc#=Vj-%^ zl*d48+b$>aEp9(@doGpYB%gtoam0rkj{hMc1}A6Qr)Qn9eYBd$>EFXVH)?gh6)>>X zB20!+P*qpz)a=++J=@!&Upj;LxMGJUH!CFm6P~Z&-rw-Z;o&-aWxM?{n?q( z`BJt;doeX56=l?QV(6Ay2>|PZZ6vdhsi*kde>^@%~#}3U->W;$@sg zvMkU5Dx{cjOh0dPHPR>G22SwUk-mV1)x$4+;Pvg$E%cYz1YbKnHZA2rPCTD?*ztMf zVf}K)hWV?r1;6F0)o(Re>APJB@d2CiYxkn;zIiHP7<0%3cxrL>{rA{O-P#!61-1Xeoc2@9z$kH8t~m$CsBiTBnJ)ATfy1yHI{2g)6!V0{x` zowAbMhLKr<^CR)YR5SD!oo|lx)%|l@|Mxc1r|VY#>DLwjfY)t09OF_80dxZT!0GhD zL4Y$B>Gkja(p<+OXk0R?2cF(|!*Qhb{rdA7W$c19g-t{M60|357oaE8-XL$=KE76Y z^t&UR2Y=6y+wA{3L+IiQn%Yo6>CB@PHx%^JQOUg|^56kM37@?Vlk4L%q@?Qe@AtO4ByI z{>iBTP#Y*=IIG_P=`V$aRh?4#?}dLD))733nD;j$#s&ZG^WP{F2FDA>@GNu$&CUCD zk=k>qhbji&pDY&3`ARrpfd@#LB($jT=mjX}O35&dpmBL*f~{nXqt393kNkR!Vq?rJ$O~=upW1fG6 z|E*0C?edkJhn$q>W2}GQpCbOlIj+YAB;v`^AOcE@l#o$CIzd7)1QI%g7D5{7_n_i@-@W(gK79}O zyump+d!Mz}UiG)uj(*`^>PGBv`qrRxO3H-LSK^B2qXV&ji$#1jY|bFw)s6*eZCPz; z6RE0ZG9;6Gnl>;nFgNvSpn6yvP(G?MmmOFHq`jTp9q{)8e}F4JRo2Y7>{GJg|8sxf zm^NZx73mJpdce}WQ2E)$f&_Bpnq8#7dso?48xmO^Z4h#Nil%lYM(bC+ALnX;*lo~%UqV#KDsI#F7>U?ud$i`^zZ{rBKB^8N=Z#0rTyLWaO2je zPsAeTzwbK}S*@UG6mtA7XFgsey&mWqG7VZMz4`Cf)4<~odVY>;!rPX$U<$p0`9y>F zti0?<_t$*lzCb)u^zYmdK%t`Cg)>BG$hE+AQe8W8M`+q-Gamiw^TYph&CKtGMMbeV z0mlO%XBMU~KmYGf4>!91qAmXq;E}f^td>F`Ag#jE;My~-rynjI`8kcqI$*l%?*i)# zh_0swTck`%l4H%B^$#EAa|u(cG-4rv)Sow1yzT=w?+|*eR3rs<&&_SKyxuZ1i3CjG?bC)Ew?fT<@n|D5xh zSO@U@_saxwz+~vbz(eWVu-LZvbyEL4KQHKpqF1$77G`^v@BOwo;KtXlnAQG0VF2T} z_so%jFmc5SRqh?=1_cgaS8q>W0NRM;%Y@DmR=z*`Ij0Drzy0TU-|u}>U!iM;g^I#| z`ESe|83s23xWgN&VmR;hYx?Wg_EepT-s+Yj2N;6}t^@tng2A6ZlNz#$G*IgTJ}P%i zCg$fqO(6SCGW0?gvn-zgi}N@au~1I`we}h7^LLni21r=^`qw|<*izX`{mMpr1hR_F zD>J@qA)r82fUaiWudJFxM5F=ksUETV=xH+pKpmQ?xSCNmG?KdwpMq=jId?7Whe(($z84O*XFOY zqF>yl{Hzm39~f`*bfbGu6f5WA2+R2_^%s9KRrB5m^h*fLWNGYXoT8MS4g44v| zXxsLD?mSr~oIBr!nU>iq^WObrYNu6Nn3qj|37lOu>>I#*QW1!P*o3@j z(QuvkoPHS^M2Z3rcpjJ*CU;o2P3%Jy2ZEOhOt(vUN+VXSv8=)*=ScqW>oYd$IbUZ_ zZ7SG(=*hZTH`UPMbC9YsG$MOq*P&7;#@V-vW!EPHZS1y8xTpfY0F-JppS{ZA@T5|i zODcN%jE2%CPV^eEcf?L6zZUX>FYnrU_oUPHwjJ8)1!>-_`H0`8JmnDZpY+dC2Sy&4 zCgL0rhfZnS*}>lM=LP5Sks;NBBv!Dz$VJp}liQV-Q#T`)J{)csWR#GS<;I`>`D1{f z`0e>oC&P02`|jGGmu$E4alEV>D8V{qH|6ahJ>{=8KTuFpEm_k#aLU0|31;LIaZK18 z9aR8*4l?-pj;^HEqbAPbi#Jvz7Wxe~UpC797CTp;+$AxR&nUe; zSrZ)4X5cL<-D_UcXMDy36q8Ne+VcBSl%l9qDy&%B2=TQi_)AJENh{#?#h*2hCy@982%&vHgDOjoZWqIbJ6Ci&R=Y_)Lc!76WXbr;!7Ne z4k>MCiQmZH^5tXK!tRaqJqR+&*HI=D=zFw<$$lAy_Z6S8%VV>h_(FY=6{e=iWG>tk@`SuYK zylz7oLHvgcuhrp;6CuJYJ($9C98wHuKVjp^%GmB{oG8S4O%;#B>>D4-8f{C{g|$pd z&{!e*r4cVZWC~c*$7-1`JG|m?;a;UCB)mirJo?pKxx>^=Edze5c0wcJ*33D%Mu57L zd-OaniKjW|Gku7-TE7kbf-n7e(XiPvRHN$K}Wzqy_NlVF`H49YdKa?g75)h4hE86)AFRIJ5psJ5_c|)dHFW& z_V+CTDjO&A(l^2)h-sq>71Wka(!`y%$qq)}1y~Cvl{@`N68B;3%Y-e03iOUFty|qB zCgzD&fJplEkU=)qnzBVe{35G|>s1{0sEEEbEfr7>##HDCiJHB=y$r6dR;(c&;Q^Yu zhe0F5BF3{-Vs#a*wjFWKjVK{uQK&&iY)O-xZ`DDUFeqfG{ZRMuZ-FW<+lZQ}yLYYI zI3AD|aJUjiRz~yMUxX}&%T6)nS+fofa`cx=aUU0|H~5WSP;(>B6Ax68FAgmX)#!qP zi5_DLt`f8B*c+#Xq<4-yxCLR%aT|wQTO)j> zca?-K)mFkv02+(>xl6C2)BGukR~JX(8!q_tkg<|^>geqY3eb|cjrq*qO-lN^n~545 z@@2ZIY5Y#Gf%lz>GBrr2Y0t!gqKExYJw>%FrZYjuFf7=W{bYaBS&7&!@cA!ouV8l! zXyFH%2U;5{>Rn2Mu#KH6y<1Fs8pX4(qep?6wm!(nHjD3^;zY1X-`q{>GIcDUjZftMj#U*^CP=GNHM41M@v1raX+y?j>4@7KChMq%aY zeo?^-hWsWSJs0_L(?4%2=fo;(8 zMAtDGfR5!Z!tdTxa5dE*`D@20K{Bc}S|?a&>onf5B3C=9uvgkg00Df(-u?^W1(Tlu z0lgwXy(in-#=5z+YUrb~yMQ*F|MsFyyh?g%f`d`T-)dm4pkA)F?%Kc)`T-P`6=@_> z#DTN-5_doNVwHP(QbKnXA50e2gYOXK@UuWoS@fjGr@o7&FSUo+TI%)<4)(}{X-SU? z!Hmb}{07zIBhEINt01&@BbVzLLS`?6|Dhy4-Wi3sm>v>_FsZKOm<%%M5@l?Jm`U<% zhrZ`d<`^M`P?D(81sMt%WRVD8l&Bw7)2tu51yCg7H}ipqk-Qk!FH;gReaw(rppe1D z(ZgZwXi|WJq*TJEk&Fr+24Y=bEPivppbQ2tdcE`0#9WerLZRL>v+?D$$!4z`twJ zL#v7@3!Vj(KajjO*9IyzBH$e^+(#!PE<*Ilt0eYxM-SOj3%3&$J~ryTlc~L4LP`Vm z-)We6Q}4!X)Hx9UPN0(2e^x(ItaQk*FAe5*0tJn z3s(DMct4ij{+S5fw!BW|WlHQ_XQOKPxyDF9Ys8F}fI@)pe3VbTu^@JTAcS~_{ZUs| zsX3e$=N{3EcI`bdN6|vk8>lgO$9vA{OinX_$WD6Hg$}u8YoB@C&$O1` zThiI;bEK#F6W~tyl!Blk25gTqio%rZy_<*-gCu(^15<9G%d;+lVw56H+H-DS`M8&EXIC-v)Ve+bUh|?aZpoMNq%!{(GizUEhHl* zM31z!Ko`$@swWXL(>CwRwC?FR|8NreO`qLdFWWB}GUzCB9toD=_KF5mddaBrY|0;n zSX#>U7fHmh@ZtM{gs{SnyclEP*NgO->B-)ts!=;Ai&I5Ls zy1UD6v}1R~VSX$u%r@fAh&i*s06P@s%`Xk%nCSwR<+xxvpChGeFV?K%_fJ~5e-ZJrl@!G~O}!K-m9^*=;SM_W`flhO*wryp)?bRrm?1eSxKpzGVtC49b3L^on5dwa>Y zj#b8N9r2d2+1dFP6%c8zxt{~5oY_hl7zy8WwSO|91bSCZL8UX#!MM!|*O{uJ2XsN!m*A+wHq zSn2$my2V-7rebHqvZcpr8Fq4mJ}d$4Y`un(&GNXd^#%Ih-({S?zim8Ne6`k3xI7m` zduloALB>vTdq@tpI}9_Sd-k5(GEh0eg&3dGIwxddz&_f1KSq9&8al#%vk%)}BpOXB ziEyi3*=%`8F;mGVLx}X6Da^LPX>)oKAa3^;`*GsH6A{bw$$+%}!CRGw2n9EK@@;Hv zo-1G56mKNdHA~n2L#7s!89M3l!2{y#S(SANibnIt)n>700&YrJp}NJ`s}Qmy!N;$c&N&Eggq=si6r~!0wX6G2^e*I-dxb+#ZdQ@x`$v zrxc!A#}gdHH1woxV0V|!#0B*C>KuUSzB@}7EN*fw!};)iV@_ADFer|wWJus(G`RFQ zi2ose!gfeaCJ8VG^VBlG?Q|f#N{E#gW`cPMfN3S$H2f)vhwIQ9b9F$R^5xH2Ir7E; zyMPnk*e%6=2QDgX>yJkughNz~*#0>ABr8n!cC;LFyq+<zhtvsz!caf@!-{#i$6nEwvgk63&;8Omgs-6=90*NKK*=Yx63umBCzv zLsPy_$oQae9o&pW|5{*|cQydc;4{JAz7kif)Ka%*Kxo5cL5{grXj)g^t zW=Ze>_7a_XVP+mS;0lxG(BlE`A?LU9QyDP@43OBL(L z*kIxe7e49*op;_=tq=W650|Uzil6BXL~#M)KvC^=+C%GQ4Wzp762xhP?_ibu9){|2<7JGBl$3Ob*gZPDLrK8w;XiZ_ikDr(bi z7_uQush-CSyv4$Ox+|aimOS`uAU9{-ce^Q0$&u%P=U)-BEBe0(rrICF5VKB>fEJCb z3j=ifoouXYVDH>9DHkE`DW@kG@+@wxKw^(LrQqxwlIkD$zETItm@8uph0|jNFUTjs z0>`<53Uz$S<-5K+XS5HCQWF&7g+OqbU)#eS5?7Qo(sodKrjt8YqcU0At%pqJ{fB)7 zBRe1stYC?sup=4-KYPC~JMh2=F@>Z3&ck2{e(oIEivR76%@*!LQHv>D$;Q@phZbJw zZ?CNFc_~Q!uu(M2gTX9NwAy93sc{mlG zP`{wXJ8MQ*8(ZkN~~S{EW^8QF;VFR>#&Tq z$~4l~>LoEQ4y9bfjOUs@p?gb&8PpcbK-Eoh@f$M?M68pe(GcULp@pl=j*$6$8=i5M>Q!9l9>1z@ zg)wH=#a9xZCmQ~-1q^01Qpcj_jh33>a9+LBw;tGrLFt%tSx#K*foP+9RZ~ zOaw>nyC>`c1j($p8)b`7$FJ>=Z{;5}O+Da3}l`L!XR88g3>VnFJ)#(j4 z|L4*fSz?l0F}2FeL1$)W)>Xm>sNFvqV3;NA>;@P-gm@GmqxD61;iIa8U-g@$91n#GQ#FQs5F!ydqd88{WN8O6obd_z$dGkJ^dTSlhi4y6o4a9cjdM20mKZb*V_DjSU6|yF<_h@QCA5b^z)nh(BwX$$}V)({VUK_ zG~+Ol8yCJ54Zhgt1*wI8P8)5Y8lFHqg$cOy66#h=y1&1o;rR4S_CYJ%-jbM^`70%1 zfi4Dr;!!gNj7N;IvL!+#EPiGRO=vY9@^WFO)~?@a2)K}?=%%d&f?MlzlQL1{7jetf z{#a#WgPMw*BEf3@MDI_x9e~AE;_C87BVe)?EEY(|H-?MHAxrwsW9M%X8$Xc;1|It5 zBll~-=-S7x9}l_b#)7%!j7-`AUg(J?#e^g;FR!uT*AE@_uPigI(3Tx_sUf_Hl(MC) z(+_WT{d3w;|5VQ5cfpG{d|d7hQ6d}*MyW8T`qbR}3mPl7?1$S;=Kwx-tUA@WaFPR#f2F0mZ^O;P!k9DfG@5S2 znC}NMauSv<3zP&;9PVVF$GC)yL@jr~kO@gn2^)h^qQYu?+9ThX`qp|BHyB`sG6E+J$Yw*GeXE{!g?L(&U+*a%g4HwUxv~6d02SQtemTNKGt-(DvQRvYbmd& zV)Rk-yldBj8=Ms%_y!o<@DN@!&)fMd_4v(O>0vVkB!-^xA)9Ls*F)W0?#{iAR&KQn zvC65gbCIY2gk+R z0&Oqe2?qo4PZ6`6 zVtCiiZu6dJgs7z#3uUwv&(50R2YTSD+^HAaRn5k^mn~!fMOE9gPYDs$bTZaFE7K^{ z-(^P7oy8hU%k+e+_b--%z5~ku@^$?H&4A`}Q@Cb6K0ZJ+CZh1|Q3?*}}k1Op(2F;UzT9ig@(?#-$MRCK| zU3W#9mNwDJcrZx9MWK!2uK<>7)d_B@SLya7!#5wFNSI>DQ`t^I;X9#Rz+^)$o?mF6 z63lfDZZro8N8Y7_IDo((8u`c0;2JlY85Z7_gKh9gnE&dJJ13m;AdkT&3Gv^3mrcx- z_zY4_1&>ax;OOc2Z>zU~t z38Zbpp2kpk=FzcIh#X{@02^9h%_s$zE@pRZ%m*)xY+y73@;>6|=qWX{BIHohomR*I z3OK3gk@{*NUn&j%T=QmSBRyaK;M6E}p|-H;FU+i4OE+-Nj4L3$Puv{+>z%Ou zq~RzK-f3^uAXPD^7}+X5Ev;;KFJ7BTO;K=coA8N?Z8BC9pm z>x8vWkJIBiS~t`K4T%_IkhWF`m~E$X6z)~&J0uSle~niaEz~P~iI0DMCVG-ve#WJU zcD~(72iDo@0T>orf$ofrPSZWMu5z$2qC(kH0z`ZqnGvg*0eW-KvNwtNJPWm4uLi_w znmcx=S$qGUsVUA#1{~JqcY~Q>0vZIB8kJ#-!7K1#rqrR%EuWMl%gG`wqJjbW2Z?*|=4HCu%tw^)l+QD-gqvn6D&{ zI^f?rHGSE74u}*)%sO2m*~t%srqTo6pd@|dAknpn*%Wb&aX~f^r#>p9h4ydtD&v{| z;cOpf$Y|Ix_U?{hj&kVo^JNZJqOn!+urdRO)EH$nslTEIT`USnm{0LFkEG131X|LKL;Yj0P@-mB1>ifXZneRvLQsCZl%;|%-_s>B zHgaB?o>rk<|7tSRy&Lkkc=sBbELuxLa_dj5=flY_SV!1)dq! z;Opm@Kw*wGINMj7!}>s@&k1>iGMYj3=8K<5aX+FC3;Ysp?pO2WEu_O#t0^i23lewCiV_Tcr5Oty!j zBK#SndYo^!fvB(p!uybVxz<@`p{wX9@SW!vXbA&NVy#%9dQ9 zx9r1wPi%}TNGdhw$nB%Vm54^+e~V}sL?liQtgB!Ext)Vz@g$xwbv8Y8@)ZL`Swh_c zyjcbiRP?yNU?;!(hnh(bv)SF}(^=``G&a`n>MAjW4jU-3Pj0D`0D+|rQ`|~MIf$xK zHzqsAR{$xOAlZdwv`Qt5AkMh?62^5%5E8t^5XAW! z#r6S?Fo9WB(x4dJ2}exI*oP)%OJ$~LuFg)r!4*iA2YMPdUs{KAmJiDWl5B6mYNyy~ z<3>bf+6P|j7`4ljMbl0fxwi1i(%1Gtjm7_$WV+s{4L-aOz<7OG0o?(>rSDfH5VrtA z(GR7^u;DJ97=C8*+8&zcb2PtaG^8o9dV6Yk%rOXg5mJ+*x`hfSdfXlg?w<(Q8qB9* zn#@7`uOamx{dnp1#^Mmc+MWXKX+dvE#6f!`Pf{qmMK|eDRf%Bn!Al@XoA0T*tsDr4 zmTfO21|Ig5|7dMa7N_B>G_8$jIT6<3Jsa|g8KNez*sopEC5bf-=QMzi37SIcml=Vw zwDkb;$!|rgPx)3C*z7C5DsPdA=tTp8l}|}YyumM=zVdBZ)@YNHR--hGZj%QPh&N;R z*OFjD#|rRj=;^RA|YWbDvVz0VL{3g_Upe}=O@ zpBT_^&X)R01M>~A2Xw?s>r|RCt!E5IS@T$w2KuhIyMHkc2sZQVLPHv#-7Zyk`bkHr z#LJ}h`9RWulM6&RwB@qIi{`t(n1eYVBHjiU#ADr~CO;QH9b z{|t=`1#*sKfNK0=2xN!++D|l%$NRbf>48~S6|_^ewi&MSu!(c6K?@~e!Gw(UiK+0S z$4RS%?9+i!UJSAGeuF&$&V>xGosFmNQp>#hA4A=tq3)QZlyT9{_x4jy11)QB^~)Gu(|FSM?Kbm}+_hK^}dyRVkH94E^pV z1lKV$uHwa?_}kN(Vigo%uULIAPuwUoFM<7+py+>xk}$GH?0Hs9utL zyEu_DJ2PKH#c35{wbN4`<(7q|fs79wM1=!s`anGKk@gbN1_b%5!=;>|_C*tAX`0&bX#}JvM zq0e(c&AKMezhx0THO~%Rjx^YE5ye1@Pbf^_3y75EFINS?)aqDv)|K8AMOX$aIpS?- zyzer754knzB>x3~Q62lXzXtSl#!3{I+Bt4PK#ePaH^Y71!C%S@)sFz00Qny!SLY$4{;RG-;-%7?L6SC^m5HYqgWPlr@LPB44gY^mWzxXpi3 zNBIT(35zSWI0D4E*lrTh!=}-hs{V0XoGap+wmKZa(iX+|3~hSS77gVOKlm(kU+|hJ zj%$&-bk(}7t;1bW2WzhW&c_n#UJ+u>9H?jyx_}f+@0RMyuGsaaZgQq@3@v7;lmRE! zRZqWxx+wr{lLK`ECMN(D(=E6rgW;Y+;7V5Y6H9SoS5|R;vn7I1gMq{Y*=K$9jNwcG z#_{!3?j_=>qwlV~9%LooHE_OUgkSE{pGofdFcG*Q4^R1A<~eKMHGnYJaJ`Dr+>sRx zC7kcE(%`jwj8&3hTI5Cer>SJk*cG5r#M{#bTDX#Q)mf8Hdr}1BC)43HJ^}`|n?f@oy5PfE* z(A9A-RZv2Ao4|@bkw;)MJ4zxZ)R{*-DF6m! zkM=zvIQerAUAhKTk+|IkPQk1_3rpO1dG<_!Bgge@MV-At!O^Ww7weX*4PT(-d&5W6 z0t27-L56lp1^$JB%KmrgTs2_U+t}>S`bwU#>8-u=vUM^M{RTsMRdMO6{moDkVf1l1 zMTR(0d`T`8v2)j&NSz!6vpjy45diR&rb&kUVUr>5&fg0ipH-BYJItcJMFAS^N0L65 zritGrNZ8)aPhbc~CZU3{Q>sguKlyY1CJyRtQtU`T=lzZw8rj}=egApqk4l{UQQ9ca z_)1+JiK+kYTka-Q$s%^PQ11%(ByJ+=^SVt$Qc+f}Kg6zF+}QZ$f>hQRtK*lC0bo4; z&`L>KIqY@a3M3bCrA+Q2#G?}1D+jJb-t6eSn8SR6i8reg+V_e_lp0YLy{9}+6REJ}uXZwJYXHpTC7bOlRW&Ps;qMnbwsgxSI{4l`7m8B?#K!ts z3~{%)NBq_5$j>glgP%8{a+c$Ff$J`5cDKKCDXO{)we$F3`5v{xN}6uM}Ph|ZXTZ4J@$l7JWB4n?0hPNT3(md z2`i=$X?blMqzoMX4?Zm>f+dd08@Ff=C@Cd8x+{atdqVf3CExW$hFR`PfjV0T_QH`j zt8;k;lMCylEG%7D!9LTPZc16odN*#k08sVn7r)TEUCylrCPR1>evjuk5Wbkc3{^3j zk68_H!=nKVF9sNH2}rfU4eZ48rH<-N;On$m++R z%w7X{W$WDdRdLTc030!!lRcLE9bz5lGJp+AlKc6SEHwZCZZ?s1+7viA;a=iQUAo@k zU%PsG@A`bF;~y_~0i=n01p0+c-0|BbY}N`Pt~7+Z4-#<*RCh0*GV71 z-d{nFcPT?v4CXCY$;&#sw#VnZmLUkJaatMaS^_772-5yJ9}~zylcB9mn~R@{HNn7c zEV2-^L#caz%{F0^Z#0n9aXbBtr3Oq`58=Lg3%f1L7qcmwwnoY%rn}C6z~dKOopwOc z;D*V$bDQ@X>8*?iEVur=`mgzHT-&3heZwT`dHx!yeJPw+B51XG%mnszW#bXoNN?@= zl8FmnLEj$+lbVU2iL2AqQZ+&B1c29H06A*9Ypv8JYd(9{L3arrwYd68T>_vWAs2w) zmu+Kz8V6?03KR)|6gzQO3*_f(A}K((#T_|UUIHkU?Ldh~R*8jqx^+E~_Me`gPe|dC zqyp6jfFe20Jaq-NS+NVxteDlnn19DVGskoZepu$0#J988Wg#RXH5 z*+6w$m-?M-Ygzf1qF;3yF+j~sGf)vy_BH>Cu9TFq4Y03)VWff55W=d|nN9;9x^%4< z`nSOrTL8@udHyJmxCobj^K0#9fd@VT-a^?v0BtY5QXbRA-nezovke#^ae)5!MFQZt zl?ol;y98XB%fN&JH*QsY7HiHu;UQ9yAHP})QzJgm5&&-+cvgqJ@}1@l*msPbOZRyM$oYhKl4$XxN_nckN9 zu1{xxf+=a)g8-H^FAe+tvCmzp$V?5T3)PEd3%kGKT_xUUX-JWvcLOFWcXb)!p8WTd zyf?e93N_&UiT}gT?*9L^e*f>ae*e?XVgG+!O9>i>zn@(AZpT@A^v!)+`VUZaLZi2C z*H%hV{NuOs>pLRU_TwI`eK7x@zmGrpL*oI})Y6}9_PZy1;~P&IjeR<1&-ZW4_EFb3 z>}cA(u3`Iytop}=Cr_K2dQckb9iK#q0L(JNCu`k1%Y2qDsu&`PQ4EO=Yy z)lVTbNqF?k{r|P_c6#qxK%1;yk^l2<`gOG(Bz@O&Cc)t}UU}&JTNV{rYfyxUFdwK@ zCnLX&6wvcqpaVyyatU>XTCoJ39}08G{D-XQQm8DlcF?F(EgEFqs(k%~B$p#y;`Hp0 zTjBSwHsNkJ`7U?a%R||;0W9U^LaRM-J0_Q|3D#n}6`mPbs)u(9?DNFlf#2|W5e5D= zw2e669XQkIkS74D)SYghH?+@{#na=;`BC1u-D`em!Y;PhnFC%K;`qxmg6{MUB>!&| zbmm~DM*q#JFKodBOg+b(Sgk?wR|&C<4P;vQkm>8};u7PH6~8oM{oa{KSZ+~mB2*IsN1 zBv*~dL7gkpFT3&yGI@8}e$ZFz_n-6)n|}vyy-D)F%Dgi}*ezY*W_wL!T6x`$PN#<$ z6p=GEQdk`e2?s;>RQhVJf2-mLPt`f$^g}0wy^#7|N}k{UiAbEoy0-yG6Ni;<`*yvp zGj^7{(E@^KN{5?OUI$)9c@HY;Otv*R(1Fa@+^QPH8rRl$`l0K?h^MtueoX71Z9(9B zXHv_?8QvPXCzB&iIfEsO-bJZ$#2GE;bdd1jRnP(Fkg;Cqz&!ebgRwpGeH;}69nz4) zaF3x*IHe`{id%A_l-H!YpE8{Zc^#7SBSqoDmhzx^$wep01hxQWE)G8!t(3`+OHI$~ zFglSD(V7l#Wwk&DKapfTY%Jx%h*dKzVgUO>C7}f7z-ng{UH9#Pl561cAG3wpYBwpn+*?ZY9eZBh2NG> zDN@b{BPLk0-9_8~N_Ee}^S@5enpGkWWCW0z6XwJkO^5=>AmB#TTPs>0v)IbD<>VEu zkJ`pdBz{rb0IVwSC zr?H=TDuZ!_WPzB}xR&Y*!dXbM@SqHPioY#?Fh1zc{lSXZH1QXf9EyRV&Gm0p-l@Wo zwC7>G7V7@Y$iPQb3{Uw>@!DCWS#YLxi)8W%p7~iJC09L8eZEP}ArB(v$r1;gN$aHb z4zO{v-SD`p1cAMPluq<)vTS?h!~bEuUq zlBI0U&c!M|*NCo=B6Geq)YYUH}9%Dgm{VhHExY}uC+*3I7AtyCFbSG z$n|W^r0~aPyPZHG=FXppH@R91R&uzhf%4M#D0yFjD(!9jF>8n3?M6x*?=Wp=_I)tK z!T8c`bl=U{w@%&Ro%}mt0uSWN1>QMjBPExMhPv|>H7>38g*+4*$nacGw-z_w_(U9O z>_Ks3Y3s0TS)sF*QO3fx%V8a}8)Y-MhU*krtB8A+dcfa)z!)NR4*d{dkKDpijU~^3 znG3R^L&@G%Pcjtx>06#<8Zf7|kL-2{_BP0_&AkI`djDrq!$S&vm|l{q<9umQ^szNLyWvRN2^mA ztfY%dsx^%Lr{2IPUL|ndJ|S<~R^J^_f(~svjaOYVMO~|2OfD~73O;Do|-C8c8b^tH&uxtW;P$k4bRgiVH!ih zn-eLs53w%KIzdMTBQHAI=;wn)qtD^<2V;m8vS)|+`|!HBdxNklT|}<^u<>_ETdw~@ z^&G#5P^&%Rv}r~5du?c~dzm@-{)m$`E%A2QTwBR>X~pIa6XOof7%^njQU8zto^s|7e}C!b18W$;^9ixaBK<&k`6j)l@<{qAAFA!qQ_d)rscn)%~_UMgGHZf zwNH^1PdB^r7I^5TmMDWy}w>Shh6?9PFbCd=Kf@aMQkDH1=qJNNr7 ziKoK3fmfd`9D{-14@U6E=R7_$p==G^9Jla?7}*O@d8$RChaz z<)Vf|p01wc%XHV`=uA>Mi-^ssu|Q3Iq`@xsM~F)9N+_`@c}%X@#i9_k>^al~xaq)E zzGId4V=mlkZ7rv*T%$g`?s70Ht-Bh1CkEU&Ei(gj0_=2z#HR{~lis1==+28U{laYx!tt4M zcc)6Dd^uuuwDJ(vb!%EeC)#PgW8Q*0Sodd9TVJ=0O!qt*5uR%QpPnLj;crv9cq14U zG$@=G5C^Dk>ap)>^zLqzIWl*Q;s@m~IhxbkBAgpP_M=mL9kt7zDG8p#XXDWw_K%is z4J1fAsRV_j22(cBIt|tT{5z#Bi8gmDJveHNP-7$-?~CcKJ&io1bV4gb(&t?@_pFCo zY-x$-GTd!;rnQ|`>Xb9DR_#iNM`b>9DwS-V5>K5XOE_E%TKWTOs~UFXjm zy0j*91J0)+W*#6&Z;544(%Wlms2ggOt%DITSE;2QCgegH^IgX|va+>g{=T=qQl@6@ zXszk)sZWY8;!tE-ftHh{mQiY7AER=nVnKW*iU7B$9v#%(DsPmb0Or=h8$XaNTB>&y z+M;?z*E;qW(A#CYg8@NP&FkJkTUG^V0vw|%p|@*8o2mK-l)^72Z^6wqYKK$O4Uk&7 zU?X?BvKA>Z^PT$3?`M8W4aEGgX7*G9)>dfLMI2ehYPRxD+Xh43VyeO(V|D{AYTx*a z{yNQ!7aDopv!6FKMh+o#N}MaS1g!1M?pIB_{#2Bm$frSrD)*fJM+BiaCz}_7 zi!|?`t;^(cu!^rVWG9A?KsiUE2v7mgmAAk<^inhiDA0>y4MDWdBceAr|l-fC*X@^Tr&wM zqt7=2T8TO={_*+PB3~ zwDH*)E0WQ{jL@L{%te{_JTwS&b$G6fh5;Wl;tGk~udb|n7Mv?~-T5GMO1VZW8}584 z7b^H8pE!dFcDKwf$CXo;+D8tKxqx$77VNa3_V&*o7!lXF5cc{Y$OS}+`o?ly5?;kD zi@&c@)CMa%_=8myf3oTU7nHYwy3^)DU2lu<(upRHSFyaWjTxae!mZVWeqpA7HSDC{nxUdveEHOBQw3NMrX6u$d3 z=7CP{Y))3y6Ber&rw4g>NiX-#l+U$zSB(pa5R0+036XJ{cXj7bB!~h9(JbjV&qQ!R8s}Bl$zWs zK>6V_rOX`>iD$g^bF+zax3?0K3qt_K%eIXnepZ?P2A1L6jKS906*TV=(aR*W>?exzw$aFZtu?uF~5fMbEp*F0-(uF zWfO^$)?fptx9?qd&tixJv4XYn0+n!Wr=$9Bw3NcnR$f0R;rd>54IORtE*gK|LYq71 zkY|o$^|FTWwc3c$+6_?3e#nI}HDwpgv8|oRAUzkbmUD=A<@M5q;@$#DuJ^$* z!E0%vb9k8rBlb|Rrnnr6a(X#4K2N}nzTzJOH%F8)P(5y|EFYJ!bY)@_r%I%e%lTm~) zA(kG>uQVKNUls78rnrPGW3w+xV$>bWbviH=cRiMj#BjcI_jfxulB4>=1~i))&L;i_ z>rAePWL^|8B*|hHaehMM2vFGs9RT#f(LAM-Cu7@^iK%nkU>2qqe-w7aaQ3g$rFyWi zKM`6^n{BlWv@*a~yOs{ef*CoiWN-aHeW>g*bM6LXSh$`DD_uGPKW`vuPv8)aBO)(A z2Yw^n`nK#nkgO3`(2_u;Gz=f~l7xCx+F_Ikc{m31ATzGG+4BeWo*wzB-)}`@#vsSm z>*5zfW~eW^7dj-`+y3MHXP}E~YO;pIDbY8$edWtam<43vJQN_KgGbsQ*>Gz4wY~2w zOV|?f2iLyIsuF-p?!I`emdhjzaFLt{x)_d1mI`MMT!9V&mKmTiac=JOT;m)5{Q9v? zU}j-7aUkgLEYRpQJYs*@a%M$Z_oR1~a+)s4W%;5-Q!{XEWZBPYp7=S zxLY^4KXk_7U4s39LC_KIQ@Q!O6fNKIb6)a4U7eJ7l#fS?oYRVOUO_vbzFwQEmRGlf zd+t0H?-zKD`z?kv(6#qnlLU20ZtXMMYl5CKQuj8Of#-cb@0cFyxSwv$I7VH?It?A6 zH&ks*ezxM_?EEu8cJkW>+LO-#J~#oa+aWUqLC99y20c*j;KA$zhjR5&?NB zduyOwccEjtc1jwRQnys-oqMNwb00C@>VNU})=^QmU)%5y0t$$Tbg3vHAt2o#rBb32 zGL#@)B0~&A%B^%sNz5?PAs{tK4&Kr=beA*?Fwzb0HK@PmdEfOuYkl8Z-}?9in7QK2 zz4x(?z0dQsW=#NCjN)(Ng+8~yd9U`fbyxD$-_}|v z`sn>q{}318U8fyBk~$3Z$czU7S`}LefKB73KJq6DD>%2M;i4gDNnBZ*qlU80tT4rD zXz$KL&UNFJB8*YYBHq~PpRk@AdKLpV!5vTZoNjKF|Kgy}kxZOGB%GQ<{M6loa-fMO zi2E`;U`>JSGv~?ZsnX@W@~sf-{*7tNl(|y}tTwOn%haj2qxt2M<19%{0<~v8wkEf} z{a#{dL-l90;+ip^;ukGDRIIx`A^0hO_SB15E24}P}f^MUg~~^ zM@da9^Zf!E!i&}mJ5ia><$cwFW^XTm);G5?>V9r3*G3CTg*P)HXD@Z%)kGNpp4h8k zB#tK=;R`+5u=F{;WP&?u;#e6wkOp@1ot`e4nH5&kzC)X;pr z2ySZ%gw?BRx>sc3~mZR9V#K^2u1UJ`uk}6S72`_X}YX5-_vx+&yXK5)43>kQqjT5 z;7XrG8D#{dF%{CKU#vBpDp?hMs|MxLKmDobu-WhNaHssQ?cXHYSl}N-QGo}qw4)zP z<#~5ceNje1w!+f})ZyNbq%5uAj9-4dQp?#+oJe4+-FydW14n7~b6Sn7H?xaMcKKp$ zSCW|%vrT<*tB|%3$}1Z_r?rruzKR{s``(j(*hCaC$orU7&yrVWnchb~F!md*g{ZF7 z?P=a8dBOgXQ!@%}ZBo5e2GFm*wS2N$gB|_vo#mt=*mY*ySxsCZQpf%3{XaVBv+VD~ zW2d%`LXIulBTtVLJW}CB=!zZTN?#fHsbLbZ=N<5agtP9vef&pFT}VyO;Oh&ix%dF= zM)%=&dw+jKTK^WQ|9>yC;H#*?TUQQf>|b*`5$y8GLrx*Ekw;^miEhj7i$5U(q90&nva#>LMQ)l4MG7n*H%fEYSeY)h%k9jL! zV+S?X)Q7#@bfk`39qMcm+E}NP;X(CY4))m~sp}NRh61R?BeV{aHCAcV^{lfxPkVcjms+Z2y_7PqEcwJB!n?aUT|J6MTRM!+HAE4FG_a34?UDc)ux?*&8de7Dn^=U{h z1!WbMQL@`2o{66`{F)l{CVo($tpV#7g%#Or zAB(;Ec$*GTuM!)-*8Xm>2RyaRN?K?1k-z^1&_5^GzS6C&caS9_`pmhGjtYwXCDIZf z-J@Tm8;=T1AK8upFxEq0U45=J=&iJ$x1-b{Fvtt!&PEAE5=FIS|5XuV6_Qn_eLUIr z=BB6DM2f&8aZ_)z|D-VhH||^^7%*Nxgq-U;J?cQk(M1_XJDsn+6WN^!_#HaAco~xZ zLiRPDuRr1ysJU&tYruzjEPg)UCxYwQA`;!n$=WVf?`ZEJ?}JkrMMnw>Zctk?NhqUdCwnXq|GrSEj8AGdc5 z?)YIjR#L~)1O(Db<_lJi4ucXCQ!JOFiZ!`REAwi-e@1ll(YD0XlP>7VfkZxnn+SUN zoM4m}Rqo*%djJxNMNufs-N0^uAXgc4!8Q}&bN!5?TZjeT{`U3JCs_dY*WX|wLulm* zz`Y;32L87>i_|9r1X90N=yYj&%WPQ_07G;+V82a=vF&H5l`MJyTx@9S^Vf0|c5^}_ zrLl34_Qrq(A{l_Q7drp}RqL&($fyz|;IO?MvHcjsC4$QSr8c8fV zJc?flZyWua-w50(Xopvh)UJJ^(+0^KMi4`(xyClzzE`7rkVnbxvv{6aW#v1L|A{qw zlH>0P39?}F*EVKdzsUF3>hIalw2n+vFSTSyu=d-2h$04Mh%e|835Ob)f^5o9m}rcw zK{g_efCgP?O#!^wJt%auK@fqZ07Yc;6ZJ?mxsZXRf++rzQUjz`VXpm-qFpovv_tV5 zPyhb(|H(zlU%ya;D?oLoi&M8xpO>lO6^9M|Z)647$;XKCGG5tQ$F3>-%xf5-`CA{& z(pO7!3w#4m`~zldm;V?~e=$hUAZZrW$qdm0ef$`TeEMrhZJu`w@yAx-4FU2y@t>`^ z8yKLT$BePe9+MDyXth z4teoq`MW>8jkHBTH&z@jODlaFNln#I6f2+qgfwSM22LI1XAXRS8%gU&W<$+Q`f6Hh140O>wYcM_;23 z8pn}C&%6x}!^4XhqQC0jij~JN?X5il=A7}-dF7w(YyOks00ME8ziK&h$aA}`bR2pn zU3$@|59)vCVR(-#dOV)LLhH4S9}uFxm<9en|L{h?tMfm#N_PU#fUHk1Zd&%f5H%Zh zpt>&dz})*g7yOHMI$gEd zuYLWB7iAq#@?H6JLWRmax*IC@|4HgDENAg3w#%I~+M}B%d zyY0n7pTR}KCC(P?^iA~_OobHXdL82+&1^$j8|q z>-)^auB4)C#V5DJSZ$Z4q-SeCiJi}GLa=VbS+SVv`-Kw;j%*b`W5H#8OsSR6CpB&&~2*DN)(7qpLO=68Qpfzm-cXi_FDZRA(~<$0lVjd=E1)IpYFS zQV|$mvAVDKeD;c(Yws)Mgi?HpS9}Uc2t!q?GD9WAdY+`?pO@-FVOA!86^26bb%n_% zXqZT0?dhR|CEy4Q74+Zb(s~T=$zrVDkCMG60PhFhW$7kf&`~%3+^>5Nolq~<9 zsD^d;dUORZ&@$sDQYLu>VqVf*2s#L<55Tq0vQei|5Pg))Mr~RDsM>4Z_(+3jIhV7a zqfh+2ewD~9S64N|J>N>nlTT;?$pU9fkq$tzDe@}x!wPrUFkdI?0TcC__q=8hj$i3z zuV2eWQ6nUE+}%pMsbQh9K<2en$mRT=QTKI5H#&ZxWL3rgi=$=hEVc|2b7)UUUYh}! z-%flKkaO|-04Q#At>XTwe(A3r{-xcf5v9j zRPd@KTBZ!94&XUZj})-Ba(LxKnqQHT#v1U2jV$0Pmhn1HMO%MFAhH20Z!c;7BpJ@1 z%`e^8?NE|ua+^joCom|s&h0z@ZHMab9mc8$jP1()F3Bh`xZxB z7BSt8B8))2S~}X=?^Lz~wD!i^-ww=RT0%GaK0mL3z#|X2H;PjAwMHFp{Cq|J5)unS z_D9ho&b>?hT{+rpo@7WheyG{w>Z9)kWfiAz@13@l0CM)$ArR7o4_wJ7TW5c>V6=K2 z+vj|~Ub(O^A#HLV2^sZCw6xzn9zFI;SVUW$A%~}*tiP3&TV4$;%^IUitCiqSLf-*1 zPvmTzzA!vT59OGi?+q_JU@D0+N9&E}{D5lrTwOgZiipDzJ++l^`=l=$u^`B*BbPSxsQqcbE1xg9om)d2 z+`cLflW9cxNRG|?U4>x>xQqQH?{YAQ?b+oyY;omSCIDYP(>m zpIN{bwjjB(m}Zo`$ZF_|`~c89Kv?D`>G*%#dCIN6ZQFHm`Wz}hdo@W47G;KAp@z+l+v0^cHKx6m z0oY=Fx3uOx@kBPv!ZG`pt}xo(<>WjG2g*S5_xscvRJ#v>x$YfpNaOfTEntIquc3bY z>?YVg(vck|2Y$ELcXgf6jmZW9pb|XnDt9P^dPmucIx_k<--}cN^+>)(IwdLfUvnZOCOnwttChuhkj{=g_h*%PAYaN-WVR&L;Qz@8>R6{<>V zgLi1`Nr2-U)^8OJs}J#Y4&`$0^-g9TX^ZOY-x~_KEhrZ%b*+v&@-&9{3E@}mfm?Lv zfz^!R6}2X&e0W9sg3G_!k~5nlhy5eJ^OKZHtFQ_OKe56tx_+J5{!9m#fwKzv$)h|> zruLb{?Qg%6WsWwog!?_e_bWt&ei@(^*@^y63|i)n`S}aAD{g83g624P%73}_KHFW+ zRK?;pt{Wi1w_p!>;|&XfW<8T)feXBz4@|g+KEhL<__H9xV`mO|)n}$MM%`CgyFUrZ zf|=m5+LI}Ay9g{k2vbrnCIt7Jr0?spW`#FX%24!d-^LmNs;u%g7}yw}zOpAm<9E6! za7vc|ph!iWnm1R%9iW)4c-w_NfN|T?UE%eRO)K2Viq)2o0kc&V7C31vd7D9xZ0{ft zfFpkb(A4_f2#H@e6%s?ggf)%pHhb}Jdis*XDvL<{hdPRff6yjbq>XQnI~0nh7e#B= z5`tO_M?9^ooCraJ&p1QyIyDkN23}3f{eE%+R3E0|imG3`RPWMbvSkGCbVdpMmwQhb)?QK zGN{@~Y&@9Vi>7hz@qunKOf$s(CA7n{T2*W<{Hxm``jea+jjtXs)~(`waPY(i8$$@~ zsYhmzs^BdO(9K5W^~(Ec7k*@q0{}Bt@#(?x7crB`UC2>;j(qHyjUpX!-OMaxwIwkp z9$eZ&m4Z0iIuXxy8C`uOyPv`e(5JQ5{rQmIBxuj>(}l?L`^P}$n7;`#{#l8s!g^YO5^g>AKKukA`W4Oh^=dcUXzPeQUGirl3iY{tq&Jvm^HJBo!~ zi3}&f#Lop!SEkX@V5~_ZdPSfwE&fV*Aek}%fz+jzj&rQQhFCy#C9Iz8TiQQ`qcrvz zHPH`?5#r{h#uFu(qbQ%XLx;=Fdc4Q%SlZnKJ6zj-HZ(nwDMaSq0xlMKcBH%%9M%c5 zLusxU6;^lCHo4sJVeMTaz*C6-8z~g0;+~;Jj<|~VW)P!*7hqpI!bKhYYj@WV?nq#G zw_DwU7JsnGF4H&QJnL-Wef+)jpZW2y#puE{^O^t@UVNEV(>V(_p#;YdBZ?11g$poJ zPmtJ(*@AaeQU|#oND_U;3{8UD8J9DZ^%Oz*<98dv*se2v z7$ncVyBdM_D3qDoFhSi>8b-0syN#+7jG!9w?wihAMxZ z{=LZBm0$DS7J}X`*6@R~8|n^34EBJ43IO*ebhLXg(c+Kl5lI95Zz;o)et9W?k7H7q z#M564>O>aob)ryi!W@oh!$oJnu8;oaN`S=j_0f2K0#HK_YGdjd;uDw;llT1LOXBxx zZw>$wiGLh;zQ+p~g=GV&A&|dnJ1K69@ag8qRKdx+bkbelm^MdFRXZ9BFf%wvIHIrP9UBlb9<||w0j>Vri&tr= zxNM%$uXT(AAnJzy!+nF=#n87P-#Ea^nW5XJUF!!EPj5gHBoJXOZ{udaO*gop&%Z@u z0o5J>B<1=?UT2irihNkDZ<#Yfc|Nzj5Lsp(;utSa3LFhH=2tqO?7#~GWl;NA4ZbU| z02Aj3z2#^lt_3(b7*T3iz`KeULopfi1phRdtqADdYf8(4RoLr!dksGg=-mz7)_2k1 zZvViF-Iv~1IXpT--S5@$rW?(+rK0|ZAeE3WcU*oP81FXE4V{4r8B`}9jCO3CKV4xk5KcF9 zX0UI%OuX}<$(}Bu+qi;eQRaCyf)?YW6UsQI?{I?Z90Y+V@Hu%>Pau&B{KxO|ESZ75 zWAHMr9UoTQ2T-X;nV9vW%^ zY-k4?D_mwy;WabJ?9JrIfV=Oqc!hf$Ci(rn?xZ_CuM)tvg_BGcNq7P#Uxk`%b^@dz zyM@v(i7H2kc_HHIy&j*je)fQ&aavmb=;kU_1Iz%Y5^KY&@i%3z;-Md0V(g!9xGwXOQz#x4KxWWxgW7S-YSbky(3IK^tfeV0v3{pg9+s%pQQQ zsWSV=T9eSW5=~m(w9$<=spA<3>)^>(*|E_ri_eTC?-ZH+xB@*fk;c_+3O@GdZKOm? za394%9hUvcyBiAP*EeB!mhhO?;sppqjMobPe2ahenl@MqkW52+fZtnC5FDLbwj`Mf z@S-FhN=m&n$@y-Z18B`UJm=d}kuF>@BFS_K#8BVb{=AcgenFbqa`kg11s$JmrWj%my8r93?f2YVkhT;Tcoi-~0Sr-Y#^LPGiG>Uhj z{*Q~_G7tC0LaFvb_zYlW2Jzh`F)N*tAKXQwlQ71YmajH#hdQ`}=WqO=Z$JtMa#{);^%grJ)M*NH4t8A2@^pvMc#ZKA!SMM$&> zymX1<=LtCLcu3MI;{@quVfwYws&OV~_X)f7JIy7Z?x8aj;wYtut>r1#1xia6TU_h% zFB#UHJusU5H0d_ORB|pFu$oSpG&~*TdPcC9IIL-@p9AO?Am?M<;@uSxND=5HM}8^6 zbXUW#yb&L%J~q$XQz(9*puo}i&z%6ZaAi3&6FwoA>o$=~dnfMBbm}z0h6RR5l8nTC zT9rIbbT`Gxgo5fMrF2>=+a<%S(a|pfL@La65--J{)KrXbSKDSBXX`oCKA$Wg4c{5N z?hy(c=Vpgju$uAK)EM;)X;lISxq#;l2Kf|=;cus(%kxlqdeA5;`M1qjaI99lIEG*~ z5jw-ltTysG`aC4ic3;X28jsDj1!<_N|Cefj;S$6fLL2p%-ltz3Js4UK$^|4;d)PpQ z*ON#ro}PWrVb{Q-iq+Alm1JWqtq;k{6B(8_2Ew0?@pV0?$ZscCsH=7~zbfdSj#y27 zSQ92TXhRe{XAo7AWKJE|`9{~CqeW6AGRIvyv3V1|PrKM8L%OWL8sqzrF*z}#eByl-y^QIc!hV}=op%e$a}8x)bUQ&%UY{fsP|5~jz$9A zpIu}s#;Zgk4{M7B1pwa2f9hf}kAD$8W3E6At`zXFLK1tEo%jc={4 zTlkQs|Al?dczxKRpHT?6)_813eh1&0|jj3d{=v)!=ZAk zUM*eulh*;=9cgE$H3eE*>sCT5ML3i^xAwP;&4Dh!FBQ3fEdJE7RhR9s<(;kTnM3sj zbPf18jWOfMjiq531wMgYk~@tUw>dci26bllFPH?ItP>^%58Yh8E~|fOChruptIA3X z>^q(Pj1Fkz6`M-pS9El#4>vpcCJbb#L9PR;MdA58Yw0@E3);%V2tMW*7eWv+$^G(! zpln0Ha`HvnnTpjpAsWw)3$#Z6tO*l8(2xU1(Xu3 z*t!Yp=}Y+~dz7FX6E{Tz35yw`!WQaBS#7byov|A^9p4X{0bqU6HvjBa>DS{Gv5=b$ zu!?|d$ehNTs>JaGg_b=Gh8dqUi;Y(gn`Rb)k^hJXEN=zd%w|)mhq0e_UaGSpuQN{F zetJs)ShJ%zpj1avNR#%yMif+Mb!VxL>_d04H84q}$ECiJjb!*X-Q4AlX3m9e$m=+N z%hx=EVYeg~j+}3t>HsiY`hw-};kf;Y?_E2xmpEvUE+CpZ7c?MHFOCP>g>ESGH?F6n z$G+ft6DqO8+f+}56f;gKy|u7qqW)8YW7|_jeHFjzb9_az73-WmJ03omfPLFDKQzB0Z^HgZ6#(GU3twDY3Pu+N7Xmx8H-TL(QTGe!a=m%^wvI zfa|gmo>6Jg75bVnNE3XUf#XR|@Y+ZzAz{yb#KfU;^)kY>MMSX7Ta7#*UX~Xfs4q_3 zM|HxVGc-80mL+CeZ@Sv$sqI!ml~FgC=NPi&o5M~5LJt3q1lbV1>#4!Ww_lwoyB&SA z{Sifik8~-a>YIOcGbjX$ZxEPoJR?w$o%N!oi4ScO{l_VG`g=hl;R?*{p|gfp__jeE zO&V}8;>PU@(CMErI;cdnT^bstQM^&vO2|4oPSR1&ddx`(y)Dco=mkX`d`blkEPd|t z;C!G&4**RmaIpz_h7;?OP*QGN(hcH2+Cgkf!w~LOQ0yLy>o6xkdNjG+e5d5USK-<9 zZ{04+iM2KojRm?mu29jfj;Xenca)M@)1N3K#yX7MVU}>-YxhXR$i>7-S3a}Gr(vW9 zHdom(eTds3SJ-gxHEF}iT#J>j`HV}~Gqu%lCh<@1v7 zsqKv3q$^cGNw&%dozjoDZ}$*M1=WNlSbXMvkM9J(uy+5>~3jNNAD zCk|)E5!Ei@!20}3)$9~X?**%OjM#t6AbZdSg+xB@?CY5o(6f7&_O#w4z4KQzR2@DD>4*rFo9EAid zvcRNLw)JF$A+rpG$OM4j<{Nh1A~*<8nBAjFaGal{m8y_a-)>z0GMMJCRRQpsb&lU_ zDMSxZ&zmabIX#kYQq-+{epZvk%;kCYE@4Z>tNbu8nI9RG{U+(f9Gs8&UY5OaHVq2- z2y72>6Bq}k&yV%-8GWRY?riEbf=!bR$Bd*LqJM2N<%)OVqc{f^=O-OTU3dieC)8K^eD=? z;d0uS2&Tcy4n^p%xcjf6ZCsD8THWloH6k6h1|wZ1BUeZUIky&1Hl%b|BnH0mAQcAbl*a!CzY^A(^lG{Sib>Rsk~THi3XBd(DjiPQ6!-GKfL#;Qz(}!<#D#lr(MmP{d=d37s$8bjNrv`FJ0e)uEOQ|7@ z(iy)e^gK>y=Qyg6u0!ZyB)~iIBG@A1y8F%T>B@rAK#MPs+g#Q3Wf@!4GDXzQtlr$@ z6&E1Bt@cWR?NQw;>)os43~@q&L_P4^pFF=2G{JqvqYAT*Vs6^mI77kQ?ZDX$ZKyuV zWH)+$)lmObbLKH(6J)>gP_Ng~jqL^@d^OCmPN$Hn5%!*tbm!acPiGmTwyPbmT{LM* zh}X>$WFQ3?K3Gk8t!&rg+ATs=>A??h?d8tTwp#(H+Rj70v<4Vx&(7`UsDf2q03R~K zbDiuuH3cdB$9_f4r-p~Z%JNZX)F5OmMWyrY=0JST%?`k_x1t0;&1Oz~ZnTc!N)ozC z2`|9P+kcJ^IDStc0Wd^&m!Y$@PeUi!TtM$t)toEf1cW}&(A1_or&G7s?zr&*k%=RKVZVA*H_ZM(3Myt7yA2!y9Se8^$j^v- z9wcriW)1>W6e7>%4&GMsk%K61alT!k@ky_BJ4}V(7SSDSlR2LHfea1yA9Cp%GrxLA z{|b{o_dayX2ygP&pSY)NoaMT(dn@f{&+PG_Edj4f(ZQVanF~HW<@Bl8aJ zZn58Sm~WVW)QGf{d-Hgy4Xaeg68x-~;b~;Rg4T7KM{^0o=u`1DfgiD^)3=f7{X0+I zj4F+N>7iB~UyN`s3prP~-z+s&5lQE>w(oeRgBp_Q^^Hi^E)hEh>^2PV>NrA_f|Pv2 z(0b?mo5@f-b>IYzj_D*?=a-V6`xHz;tKiJgI+J7_`SDhCH*L!sw#Er;I)Xk4xfa%4 z(Q)Yc2+r`j{#6j)`*00kpD;1+)H&&wI{~?_5BEdujivWbBiz$pl}qk85>`pDE3+0F zmtAU+EYMAlq>t4A*p(DOCn4``Da@KP^5lMFCK)G@iqmo&oxV;)OvI?FJqb1+Hy{~m z{W9!(+g^`_hJMo336{<7sCghaN*Tm*ff_8Uo z!vmZ4H;u&pWZ9wM+s)_B#&S8e{GqNSAQi9&Ir6FtZ1co$G_vwWQPUl1|L;x8u|QtU zHmjQxsh5(*Ymp@CWzmfyAUITqR|-F_!xv&`SG3R1Cf~8ZUJ$PQXEcPF1JDOt(t! z*kYr!Xo&|2b1l0}4D@5?Ma`T-K#&jsO#qSG=?4@y)g{kIwWd;q%VUoZ%nxfS+#sPp zE_^=NE|H-W)7!&H!Xf;Ix9e0o5gxUo4EAt#;a|QvNP>EOH*;7yVTR$F`y?yuH{Opo zE*e!}4yn!!k&mF=O<6tY__-gZUKBwR$5h{lXl z(c7Jg@YLFLc%z)C&PNlvrH>`a8aE&7?1#+VV^@lJtvh+=ItRrYyWnwVdo}Ii$qFiE!o{x>QzXpXIUQev)NQDGkwK45|%mQC(x=y2?P9)h~ zbvWZ}GV3k#OFCxL&23Q2ia;Jx@GS!3EuJ3vufGJ~i5ftC?7s}GO22Z*1rycr6@~Q5 zQ-A{VtSNXta@{y?$an;#I+POukmLm|F}1bXoVTPUv#Fab$?6{2}q(YST@F@XPjfZ?Fz#HZF2NATrDE zOCN6hak^uC6ssCz9-cD*(zlxN$!GbYUwT}a^Tvm!-uyd*_GU-J==8~B+bUaTjV`RibVY!9rB&KzgCR4NZ9q=k)O*baDSwpZ&0H7 z=vP;tJ)8Z!6PQK16&E{=%X%bEA!n!r_7bSpf_uH2d-+`Yj=A6Ea>G=rMehP|qTmKAn|bMVi@=qx+K3_nZgO)wy~zRO|P&O+ITo>}+{` z^m#1M!8JqhgJv3D3s7~uKsDBKbP3X`OsF=(Pg6;2`dDX$`MYbQu9t15+%*wkHxfv2 z`bae}59E%0dC1ikHGPaY(m6aZF17khK#$}d%!|owSh595UAe9R@Zm>NWKo8{=7b&PKG(qOxm4aQFOv)wwLouHw3))4Z+`!t zcRWccG_-C1Z@qn>#<@}(_VM5b-{pp7W7O)$x6(a{i6MG1(FA^$qn&OPaaGVuW zl!ls3IOSOkTOiqdlP`y}m|>j=1|UT?Q7Ua*1}`jbYvQ6;fPM4p7@f=kb}rg)Y3t)~ z!R@)@N6lojyW7)Y*D-FmMRuzwLI^^lc)_9!YHRfjVK;kXG%7&7^h*LquP!|H!R2c4 zkUpZag8-C_cY5*nI=MN8^R zDXDjB-LIuRe_fM*V|~=A&_X}9avFB`8{gTG;rBR;yY*aSvVsqwNbQn;KFu(_pL$-P z_0VTlijn3A{rL|CV*!D{L1&K356|EQCmWOltU>as*Dy)vCTBIh#Z$RbhFm-3!eXgP zHkrAJ;g;tLsq)*G`E`KyN(S(8YBodB26rk6K-DpT?Aj3ZDpD)<3ke!JX1nDWlx%7# z2Jl~pA2tQNGr3N5TMD~;qx2;~U8Z4lTJ_IgD#lz?JMDon7U`DKVVE;xKW0Ud94ikn zgVzJolTHQrMt9$q0;)juQhm5yc^ga3aXAe2v)5ZMMyGaCxXW%oyiS6O{I#1bv z$`8}ZXLzyfHZHg4jUp}-7o0(z@1Fv3){T9ied9}px4NZsG-0r#nJ-ZZ!xjn!28qGj zy2R`+yyksQ1BWC-m#QSvDDQR$==eprtJVgQIj!y?t5riVuT~E-`okZ$#LIWsrRmoU zz-G%LzSJ!&OoF|KWN+4wPx~zSj2a$?v=v8O^Ru>`EMg5`I4gELT~=8?lU=@BQNMj? z9ao9YA2bG{#d++MOnmkFm{TYHDymV=G#+9E=5lC4vjOL6+(l9A@1tu8ZG#|EKSwJpNL09r{u7 z`4cgQ-t`P-o>-4X7Qu$680^tis^tODsvUeV3JI@##HyewzMy)Aqsa(ca|_d_s1E+346$KNoG5!7|Vt%U4B zmoWrWwQ|9MQF2GFD_^r45DE)854=?0jA>#}5`0`yQ&%w1?Db>*r7lrCZwu*kwRbl` ztQrZk63dgE!EFmZ$A7=+TiFv?*z`mK4kCn;U_dtg9L)ZBX##PHK~qM`XSOUclU)1L z$7WU|L%)@#^4&Jw6?^qWT$h&Y#+Rs^Ao9)f`%iQ=m1AKBqm7NWlxD;<1o zuz@Rk!g{a4Hl!8r?yHR*bg7svHa^QJ-SR9L%2R?du5F7m3@}ApK}K$$3D=~_A%6%X z(1u*PuUUirvDe@>LUVrjSlh1A#8_QYlChMEw_Hq~5aAWzA18dows4*1`%Ld;Lue`m zZ7muV40CL3tC7$drRg)s84!`~F~9{w2Ca0@dEu>k4G$f#39+tMbOyb?1!E zVz=1+mLI6E3sEq))muDs*p`X%#o)f(Zw$L0NC#@22HKX>2oB}L2lEllpv9H@u8DU)rm znJB&6y)Uk6!DYlQN=^25DV3Vu?#YH+l}+F@>MYwK!bF#a{fj60M9FP*<=6RU?vk(ZQ#W!3v>BuqQxj3llW@Nvb@r^?u8K$(V047m~A@#2XlT#2N?l=>{po2i3x7^?0go5Pa! zIWEHIvj!U4a!9@|3W+8^mK==4x?i5n=Ug=sy8O8pZW^zlN?5J< zx|@e2Km|4nxB;*20Y|ww|C0T4v?0O3Ep(H!?A8*HPvg8lWXWOXTYh?tkf%*WR&@1K zhHSQKfzJBefi%y!5BzrbT(6$ZPUExb{Z*!zUr6rA8mp=IMg4UK=$KT2HX>jq7a znj|nKy%;^7h~!$9^%|#f?cMw#lwrKgv)cV$5oWS8|_j^^L(JnluKj z`3~)>eb3H_u8p6&!B&s4^*im;Fv}wx&ga2+wYD5D3UPJbMng604t;#Wx29h<(xKa` z%VrpwfTyM>hZVN#$6|V?RpzKK3@7k6%Z;Z~*hjiz>2nq6@)`SuQ0KC;1ZA}o%Y8o^ zbO7?Z6ZK$-9i>>~p0{AJXKK@)1f4m^OC+5JO`T?1r;ygI_icNnEQuT^emHophHg}R?)(V{s~|hiH;%~9o~JRvW+7zAqCH|%NaYN|+?vN{Ny>0Pk!dI}+_Of^ z>&3H@9unpc*8x&(%`xS@M2^}u?~2C{T#VUP>?2+nnT8IhFK_3%2i54kVIk~s+5)Y9 zGf`!();s1uGMRH>D5|Yztgk=Iev9@K6}taS`;;%T*F05WCnXB9|PS1otg{_`o00V6&(60G&2P*W&XsMHH!EtWM1b72bSaJ+^U z>xr)Ur4B4O$ z_f&4G%nP2)?IsOt?Y9nu4PvgM;1O^l?y412vM(MjEyWP*Z^XEWN+mi z$I{}8>s&fIys$y>ZdJAQiKb=p21gTNX~t8GFV~$YZ%wId z+JeFFq%^w+vDw|c`);}%{?(}Vv+vi^R3;X7B!hF2f z3L*=z-y(e{1d8f_EwsesV#lXZ>2}c?<+k_F(Q;+`qO7ZDW*5`e0Rl=IOU!y9WKWgLwvJpSr6@16{VO8yeRr|4P`jqL*=`ai7 zk8f1YJ+cl)YfX{hdB^?(`DXos5FhH2D?$iFlm$h2F}uXx(?Yo9N6#1r*vPK)Ifh`p zG#9gk`^+tuSm|rs?Yp}vlg%mDpBo#cT%t9(7ShwZ=)K{ASx>xnzuEEw(_9D|nwyrV zCWHBbDFSi~zX;%#YS(q4oh@fk(w8Xi`pVCy}CaG+-cWLE2mM-)00x5~nX~UpxTcCP-M-y^7_n-Oi{ht|R zV)cg#F2yZibW&C5(c*nKSII&{6yvX9Xd*t;zGI=SXcaTMf#WB+GIy$GDMe^&@7B7A z<0_44-fqOH(zS@knOHZ^^S!Bfak+z%_;y|PiQKFQhl2Ol5074aOIMQS**kO{&Yz&C zr2AyNRvR^wcCy{pwAbnnM{BL2kHz%Sls;24t>=hzηTTZbw$G7&>Chl{QyxKzU zr5}VB$m;8yXw|lOEwmWOSg&DdnbCFoZgU1Ok7-zeFKUxX9=gu~n^vzF?%H=<3;b57 z!+q8L2G@-jHE_QZliD+9^%n5joJ9BAO@p=R+2i-o>>Zmvp>5Q+BC8~{c+i-Jv)SWV z(v8Ic#lwpHT*Dp*U=4)}+yDT{fG?prU`u!YY!?#CotcB{d~DK)Q8brerkV!s)O<0;`=^V4@G5O6fid( zx1A>(jLJOvQhJrCm>L65v%pQBc)+kxO%_=OOT^0%m&kB;rwTK$mm2YeN{z_T?R?EI zJWOBM30J9GhSR!_mVI}Bpnb=z%AtfDQnQGXEF)XoN~@Iv@~(^vl0c~RG-J@Lk^_VDfJE~F_d%%L?8?Pq8xbN zw7N$aLy%2Di?P;i_gl(7h5Ah%!t6o8N7ihu%*UBq(#-399=mlO&OV-BkF15*kk4F< zZms^%DUKi(J>0$#SUk66UdtVo$Q`<;Z&|~?+S_YQW5EbLl|I_sI*3TGr8J&kvk(+= zas`&!S%deqMI^2t`iA=$zi{|L--K0x$u+O?j1%8FTRb@4IMd!wV2@hL8+0dwYCNg% z-|}iccuTEKb82?DbZt6=s|+KQ=$E)vGu{F#aPwdTC>;1>klXNhjGl?RO7&`q0V3&a zRV^{+xrxqyRu{a4LKUmG1IYU}7ZwPbK&!deCQS10s4f&3zHtf;RfRC9z0c9JhP@C; zJo)kOTNMs^S`7DLgE4V}YL5L2LXMl_Qb88};ylGMP!1^r;6!8$`_m+fUbEn(JUyL) z8>|ica=Sl1kTfd47n4F^p{Gwuoaa0&toBS=;_Gjx0;`Ad6)>Dk8^4M`MUze_I z%FT-HHDRPty4ah!6|(w{?iMd5I2d@+s2OPI4wz@l(u6IYpx_sVa}vSf3%v)el)Gnh z&R^V9KLq&5T+3vV7ygQqFY`FpC2tT2ofEavI-Bd>CCmp&Ju*^DkL3|JfM-23S_*}~ ziP@EUa6f83c?7#<@8iwZTczvxRaR=7Gok3T#<#M7Wh51c8QJe$p<~q(y0x0$Ct5Q? zZJ-GmQu(T2`PB@PpkkbEc=KU!_rUWU_{*Y#fpG#-5S+WHOK3TH_?7v;9>G{QBT`bV zg}>Lr>BB)1@r+1%Hx$D>E1#MC(Uxl8(vbRAy(mGh1;-If2An2VHApzxRC9`iwM|k2 z7Z~A-A-lqmXOr0EQ8q%w+@k-Q7sGX3jbz=$a;$^6OqFx zA~!gddnu#t=7&Y7(`E$oKNYom7kWD&4}B1MK#cs3&&AIip`MlV9yRF~bdS_%U9tQwO7imrj+p9snLrV_@u3 zX}q}9Wj7m%jZF(#e8hd0{SCxF+2lR;;ZIoy{w|lwEbj9m?u&={gqjV(E=ljQDx%s3|`00s@_V>w@*%=6nnim;tlrZ8#kl=>OD)z;QOcW z(Mr@1q0cXWYlX$ulk2b&JBWE0XX-w3Z}OPqNq zxFk8yEbwMAI?q$Qq6$h^a#fXS3&y_Qsj$o`hc@7Cqzic&Dx0TSKC@^?8^T>mMHzg+ zu=&N!2bD}5l18mq*bp2`dA$)28&^@MkH!P|^1^y-!kyRv}llZR2(RUZN&x#ySg=h0G< zvZ+Fg+g@u%cU1&+SHT4ro33T}srN8i{{ve}9m9@5d*L(a6O`I2wqvnbU1u9-;#0aZ zwWLfB>Sho6uMa0qw!-ARhwKLV1?p>p45OHV2YoR8%8E zwBBL++3_`Qa>j(quYu6yVO|Z2KE{q&b3P+Q4kJ66Qcl^M3d5m?%n+T$`TGE_UeEI6 zI5j)fCL-(N)9h}lvZ5^G>)F>Wy?gzo7b&CBu{l{~oJrB)4>(B=5xy6mt5`Yt?YJd) zmbG}u9L^bwp0(G)6eTd`%j;=uyl})JwllbUepGdFFgKVIyKfvp!~x9! zCk5)Ij%M?v4vmV%Ke=`$S{nlUoHr{gCPr~~7&6e}(6Tq8;dzIO3r`lxIu+_|k=x5D z$4hD!6SXF<4ixH9f+Pz<>5??}Q|d;!uSs}@ki=A1wmm5!8{?;8F2Xnzh6$_Xx`rW; zKXkjse&(W(n2N4GuC;ts%D`haul8t{>XoJZSB!vaDCS@S-{zIDWD2*9ptl>y59V1L zP~P?sp8BR0SSd3rE2@zy{rdW&69M$PRC#K8&+V@mB<1aZD+_MuhH3MU#QRm!;+A_u z;gy`U5IAM);aRZ{I^VjmGPlp>VsC%9T1J0Pbm6vaCZW%nAHRfL`A4H4@Y#k>cJ+(2 zNMWm1f_LMw(T%hxUW*<4F{xEr5e%#6`unLd(H(4G&wPdvA3bJs%{n{_3btiwmm*1_ zFT-Vns6M@E#I*ard$Z{|=V>PtesAG}C4GeZ^pEEeHE-rh`W9V`PFeP)H--eWVbiSP z>uo2mHV zm9^#3@EPMHO|J#Fxz~?Id|W|F+YmN&$m*fDMf2DoHo3*r7aez*__q$-7G6fIcIS9K z)>XhJ{3!vff@Frt%IrNFr|#n0jpA{KNBQ!_M@t|q{c*)VN%_$YU<1#hmCuSvO*(k~ zck=#5hk!R4V{x#Z(p@>cpLL~*;a3#^#!yr6?M|?^@hwR;RZNLzn5`9_IT(&8Ej$So zcYZo~nC3X_Jz=wMH@WQ;<}LK+G+XzH0JP;s^w3Z?0#Tn7G8Z(b2QrOk`_==>pb1+u zr!8Mo4LJ0Dk=o)9FZAn`8X`hYjDhjEYl>6XMee91{f+hJi2yJuZu?rTB1l{dMs{P6 zWu$@>3wG5WTZ6mYURbPx1Jsi&Oj?RSI8rd%sSX6(6)h6MSH|T!gxrXvHj152IL5by zD*-yg%Nauqd~96Jz|*w$@Gv5}SnBABNRff>@xe9+=RxgwF?m^Sli}E2b1O!;sK?=P zAH4VG`=X1=qQy0DjV&F?mSG^(=d_N#)WXa(+HN=tR2bKk`+9r}{kPqGp;iddxdxn{VkX z=kD=PL&Fw}%RE7dPoG=tWNsU@PsjWF4t?M$LUGKJCqNcv?=5$oCf=%=umvX}wZ;1oQIU2(>_dbm(_H?S#dnbYe=IfLw)(z4{8zIj8!n z!!s)KqcQmG>a~z?Cz%YvCytvc(-fhqC6XUzQUaU(mj0Q(&nO5ck|aJ zew7aABf+0Q-ZhF8mv1NN&{woP(fZ|}S_oNd#w|$4rMNxn@vO}`u6poBir+9MWZn6TIqg`&z4OkKV^VVw*_nGR z-Kc;!J9N?`$29N+uO_tqRD<}+g+siL{FyoGM*VnV}AYI zJ9%||qo+>@FZC6MUzM|)TvOeu@0FF=2e0|YmEpm@+f&a<8W+)luD0AOcg5{K)}PVZ zjK#gt5$%xmbt?y{(joN`)eHlKG<~|aa1M4F77(N^ifw%}C&Q!%1W3*OGK!PM4Cqb& z$2+qG>t=+0wl`h)Fl=A*G<+k@_q*i!rKK>%V+l=T-#=KQo*{(Z286gQjZ-}8_+)hY z0JkXQ8cOGGyDo5rdDx5X+!<+gYOTrC;c`Rl9tQqwtyr>2Fn2mVJ!_n_OO+8}npHbx zycVvMAiV9j$O{dzgt=pa#~2^2)aqPFmMbhc^fm9Yv%we~a2^<)LoixpnZjXwX9cBwq z!XqKOz2RxC(1P2Fw^Fw^?Q~TuGVb?I>`=F&7?W*O&2mpq(d$};Xipzo=t0~;%$#gR z-NR?}8Iva=2mDDt`bxwf^-;U$XgX_1d;ggmJ}BBE;X@d_AaQEwl5!({Gkx}^uUX0n z$5;{Ob6V>holE2jyvX$w;v8Jf(1zi7Tg22K6*D|LJ`5|*{}7KEBs7OcVb9lzuI+(y|O9vc^(UtS5_HO+OBQLWhaJU>Bradfh7 z$}s98kDl0Zwqu@bjmHww%gR&_;B%LknKKikfJZmVNZbngU8oUlV4KU&3ULm}m3lT~ z^H(P0UUu{6e56aR7ajNUl%1{{>`19EAk4x|R}VV*fO~eZ^VoG*JCTf%>r3thRL!z4 ztj&s2TGwoB+m%>^1y*enpz)R8RMXn`*)NZK(NsOv(f@MntTS8ujIYY{>|z@ldeU92 zXvV0jKQRY#yjvKnnF~2Oqz|&zegArp%>8`ElpQaZO*`FmL*zCll@bI2(x|N;20WB}`GJ=h$mkVtC%OzOgg=M%CdT zm{}RNCCtY|qQ$z~_u3&+>U{=@U&gHCmZp( zd9EPg-}NasM{t!PfY#B5n-fXpIn#Q4@V&TN+| zFCK)M`=~o~XJs#_PyaKA=%4a{L&W0j67)|qtN5{AiiRn7%VO{G8;?VBJH-nm=xlaQ}6SIIh2|%M)K;x?erVWuc>iCPcETM4ipAa6JBIGRKZ~ zdy^tZso%1sOojE78Y@a-!C3>Mw>qnsaZ~>8dn^pK7fA0m+n%!h^YV^9y z>x0Ew+EfJw6_xl+Zk}f1+w**M7-vTk{t32ctg6!%X$j+1WGYZCd%eI$#O{7SqO*p2 zg-_~=N=NI$)mk+al}MYM5$tx7P8eiu_KBAGxYlZ0rRwld#&*G^9_|`~Ss6ia9o~x> z6p}P-AE&a>WrGTgJzh~7*{UZfd`Z{H*0p|pWGzoM;;V*R&xw6Sg@JxV^{q3^0(Lp7=)&aS*FkTIHF&_~< zuZsoOx)wy8y4tTKIV6n`&SkVoje9-XjoEhHAY5--Ep0@v@@)IzdTQ-G%~T_l~aEdzFI&2JVyC7 zY_avQ?=p6K|9b~B5ObOtbevA?htm6!m!S{yHqo!6|!Xt0zpR&>ZfKh+BR1=Rul#pBH=+xTLkEw@# z!i<8O?M=-C1cd4*Ni=84wmGmr3-)#)t`%0YbyvO|fXp%pk-P$@VRDG(3w-vr?QrAy zN*xMyR9CZ0mrMN+q0YyfnC`gH*AH28+BWs;j_eo}Gk9qQB zx0@!2wHlwjL9-iYnk~u3FJMb<-hVhG9{;@O-@kn|`RMfcDax>b7gTga|K!Y$2>%u*MV(-SlP{LZ+t*S_>3xQDclug_``Y60BWyC^Zx>R zIcs(BpHUbz4SwIl4oO6<^OKe*yIiC{5h~_ubt!SLbc#O;K4AnJVX&u{p zOGtH`R6Z4v*%+af&NTfQiaX>11Ici;Su@_&qldUkS0abP7wWX>?i}qtG?)?@XUvLu z!@gzzlBhlWW^)K$^Xrqfbk%epn&kFv4k9{SKB?j>!Qs*RMN<_4SM39>b_{VoJLqxh zr&4=X;4-gA`^M}{%z*=7rSP1zH>ddQ?#&Lvx@0zwySakJb0W%BA5YPNce*`><$-x{ zIt#h&BTCs%Gw{)2&8ly@*p9hNx!3dFNe~WY-?7$F-O*OV@$uQ3oQm2y4WstjqL>6? zVVv#-;?A0q!;dz%yQJ;>@iDHqI^G~KkAZCkpa!hn2nztAh$HpS2uYm5em5Glkv3pK zQ8iT<+m|E6^gy8ZsJu$>JcFt!n;aRy+{QC3A!nk;CvP2slX$f#90f6_7@Ham)Q#bF zl_|YohCi7konfEMQ|&V!3knG%;tw*UK-K-EwYeb@s(w%GVP&3WMbOv#MS(umKlhpk z;+Z@hD(XbJUDHXKS=e9g+YKj#s@Yw*0e5l&{_SE_A?}CrdwH#%e$A~l_tws$T`nRO z8@f*p7wWhs4XvrJ!`ua0b1y(RUjoom__;@uYq+U&gjNh8jYkVk5xu9P>o0OlZGo+m zzBk!+e!mbf+%0+#o>Kv8@&H_EcCN^iSr4!gj12FxI&I%vs-r2b4uwXw`~TL#Lfwr$?pL?fplJ>EG%9v#lf{pfZq|%+n2?P&;M1-wH{QY^t-SXl+Av1!c#R&C`c+{CgVA${ z$iO~!Xk9%BaY+y)Br=tWosoJ3?=P z_WAGtV0}3M8+Ibi&!p|t1S~M7F6|BaFqlx_3BU!*yHvmvScR1%Kr*UzVO8idZ-X0} zbO@Wi3qY^P=qcNX=#$TdG|t76;girl!mCc)zn~~x5KHHFIQUj~ia!vqUA|afA#F?= z(yY>q3qm|lgTz@c`I0=8D=OuwY;U|ntGgs?5Eid~Qj7p8cknHHWb~xK8IR~NVD3Wq`k-K4lvo>Uld0(HR$u2cAVZ?NYNHc#D z_vr90pW5)pqLe!4ZFk;Lsq(}Ht-cTkNeFxjq}Z5~yUTi)l(!h_|M268k(o|-_%@pv z@A-LBsA9ft4>Zn;MHrSd?hqYoxloyv%s%T|m&}+p(+asDH1p=u#jDj*v<~%fshG;s zSo?BE6u44#We_gxg!&__>q8{uZnBSfa*^a*A}@wlEh{$tmLKjHJ`3X4$Wea2{lvWy zZNL5H>Dkx&&sIM`7yP%qoo;^bM=mRZV-DJ09s*;fj|BiL&QRb*ELWWM zSAT3Sylt57gE4ip%2L|a3J-UjRKOQ&g}*oSrqW4jS>O+munPlvG00NYJ` zq05JHyP1UBJ9e}C$&U875D$C(S*DvLE?z5ctuEJnllH!=e0Vd9EXa7p$000m`WJL)mb5t<{##d?CAukDsI?Jcn?dZGtbnxG|!QW4Plk{HDp~X?= zuD+IBG}+2dZA;MVV~(s)omX*kUc)c>X?icb_BH+jg@QoknLfH}%vhm3!(+R1hy+#F z**|O1jO+fFhd}VP;G%$jy*)?fKht3FKs!-bQp$<~5n^X8N|zd<`b zWKG`jb)Ia!bEtwZ!LOfVl7$Pe0gdptEbM%Ky~lNDhmq4vkSzwgV(pvT)tPP$Izps! z9B4PrEEqu<`uI4q2ChD`t(|gFB=(UaB z)V7;bYfpamw<{fnKEpXW0U^YfIT1Ru{xr!y9P%et_b`qnT2{xZ%9`keNbbi5${DdgJ)k{owIRf2>4ozpOX&heC## zNx9Nbo~62eSuyCDqQT6Tz+uaYh@NYlw#xhUJ>{6sudzswz#X?=+L zw79S+&cDDtZ;AQ({MM(&h{Im#p!Mqz2{vrf*`MiAM+@4FYyy^OC6rNG*|U$bv#P53 zXqX`HporwZTIvxX=|iRCCHsN5iFY(OKrT7q>=CX8+P@-CPa~p0lg{l3HJ)cQ_$Rj^ zi##iR6nQuK^()7FZx54jWY|EP5z;d}YjG^kaS$v#C|MDwzkHgg$OEBQ%)gjhQR5Ln zYTtvP`49JbaF!)W-Ggk(8#{UWR}6(E3HcJIn7?&WE#;_5&p4LHn=S~+FqiVQ>AfdMeN2~P0laTG~s*r?$YlErBCO=FpZDUu`bY=1=w| z)VO}iFJ7+j>I=QXa({?l2{tX!L(w^bnE&dH%La*aIn|>}sh@pPKka#o4}tIOO4i8` z+(8Wjc0kLN-D6o{c|GD0>PfAKhvS-4{&pdyP4A6uesP8-wXb+f41 z3%8`xzE!ZZ@wYV%&v4QzEsquWSPd zKP(Qpb9L7gU?^CX$4lnLOE%f@t&oJ6vpTTL&QCtaJ;E_3RlVr6yFC?-iJ*QmtL*Tr z#Lmxw_}cYANNVh)6*@Fge#`wYG_ucVDbU4Yu96pu4+~2 z7_fOxc|Kwm==a`LJaLc5Fz(rv%fFWOuzBlok2APYgcINZJ^!aM6<_g|hsRPTxC0(Y zkv+qK^k@pL=Igkk1Q;jc6eD&{BazwK!M+Mag8T}iz6>j7@Vqi)YL;Pr2{|=0t$Ruy z-^CO3^*K##D-jU2TX=Z7KrzCYgc&$1@<+sk6(zSF2o_=b^I2j7)~8=KCI|{Rd&=0R z3K5mlrz=sxIA>V;%2Q*p9#kwe{>lrm2pSSqJhFEyr}vJu;U*>M?A|mhSVD&cT$9-KJmlTYBjYYhH{1<;_m~~aiSl&| zS;?vp4{dzPgKzJ?E=CAKvF>5ERPskB{OydwcTXOIhU#M*`X!y*A~RX8@Q>-RsnXlj z_?vps9Z&1N2aMsT;5Zj~!EFlGY1^7UR97x-vvC%4n0-m8GWezQ{G(sh2>qAglVmO@Zh-Yt}`+a;2Y#bGM!~j^7rRGgoGnIEq;i)lj}8HC6Vw0x2o# zPR&LpdP7nOM0Czq=y&}=(r*LusY^&S*a`SsEAIOmxJSOP^kLy2kZ;?6$y_QdbLU1m zZL>wcJAj>gq?=CJs%IMP^7%nY;ncIl!qVh>I|QFX(2_u3&b&?)WLp#R7}7rj&dn0n zv30F#kR{RMT?eV75Bhx0z)T;dha}9sPorj1Dds7gw#Q{QGE(E_b=8Mc2XAI6C96~7 zX!IOI1$2`b7f$HOmCXyP8J18U*yT7o z$tn4G1yvIH%s9&xd!97H`g{5_)Mk?q1#g5}@rS+DcS zdmUW4qxP_fut#(ZZ?9Cd`vwUS6RyzuweSX(I;Wgtny^te@&|0JU<-4`1xkd3+OVg> z!c|Ln)m0h*5F))_d+M=Ldo@Ggy#E@~Ra#MITzoP5JFdjpF8XS}yYWW?0&@Q(I}SqU z-p5t+@R2ooP5{V`!FCRw*lxqQt7blvp9$Q~5N}ZNmswZ9P4tHvGZ^>j4`~>HOM=g=3r{C%@ zF)}hKr)KJQ7Ig(jHTDbZ62UnffQg1h6eGSjC5KwA2qkwA*{RQXQ*b*#cNZ5jldV&^ zTFn6bg`^1n=1;R3-p^GP0qEQ_42v9quj6*-ehY_vbDZ5TVwqgr83!})!9f&d2pWK> z?_tBU`b~C;TCiL8NT=kC=#bQt0I|_O2GVGX=Scip5qH8{^nbLHKyHB4m*~d>53nv> zCR2_s@}xsEx{Q6R#f72e?8N*}*Khr6M+0U+C*&UJbpsj+ zd$7U!aYVV@qEIMQYBBocqE91~E25SCaRCPMZxE|y+ZWpIHLsWwpVc>zj0|*_c`n^2 z$iRV&n|-Senz)&9TQXw6Cot%fdt!R>`X1ENyhxPFL(pb0Xi@(moWAxM!}w;AY0Dzr zRS?&KhaDl(e4_Gj9txvRuyd5{X9Fblm*Bpuv`Zcq+Mk%Ko#DU z8{XD@1fI44RclvbLW!6sQz)GzUU{-R#KQ#59`EU03mn$~8|w2&ZAEaD$KvG67hWu6 z{o3qM)?5QGGo2^RObs2lPtp)eV@fN=Jq0JP^qFCy_G z-QflET2vx-aeQdMx^4*(+7$yqF2gdxJWYE_2w!iam7?NXTOKgkuw)Gg3DD4fXLp%k z0Dx1$l#zK2;MDAWzPi^{rizmOMaie>)zMHsKWhECMM4PpUQ8pNn)Un6xGxmR^j`&Y ziA@&(H&J4TW3qiAiZYgvcczvez5@!vkpOI7a>Kr8>*3OuyfkVZ0^9r-Gb{}-2hGSs zan4BUlzsb+%tic!!B<9~Wu~dSXF?OqKq6!nEBw1WF{B@X9Vi>v_tX2xX2pO<05qv8 z5FYOKy?!Ff=&e2ep4UP0fr+l4QdY6$9@8O5aO*YLdP0rDm4rbq7_~ZWvDl~3wxgN} zf-Oro19#tpN7{KH6*3lRb?}@eym7Kz0BP56o&4FIca}Bm_QbJy%C$xP_{TWOciwKN z6H(ODHjfOrX|kgd-r=j0xoCdZ0;qBWSk$H_p9TIDcMW(wL_!OgwH#SSI-&SIpT?wW zgLKbs;ty8NzH=Zi%PQe5A)b}03Wm$iv8_PKu2SPfwlV4QlowZ1tV?B2by@;9`Xxk} zh7p8_sZ5dXy0Bh{43Q#I+kY3g{~=R#?6SL*yS;OY-~9xZ;*ZYcRmb_rvOl3_>uOfUE>aVt zC2oU}PJrCkBkV^!q&)w=Q?W9tQnl`yxw6ApCv{6F9Vr+FEXpQC>?erMh{hW;{tj6o z=FX+0zBgsD5V~;&L9p?d*`AD}kxX^mkj*9Htjef&coNLdF6zE70;yoduF8Awtd==m zU~>pdjUi&iGo?IkWpVZ7CnB77P)FO^(-*#fUf}iHlf{8i2QC+?fgDxk?Bf;R2rNA4 z!^`R=N0t3>He;e|1|Dtm-H_}KQPzLWQOIAir*Gs-*`115Q#JWW#-Q>=%VPEKA0b_wDO{HC|l~GE(_$dn?L&f#V8o4o^m;dW;XQevIDgAnGf*L=|XLsC_+^ zUfiMY6J193S&;!B+t_W}c#;f<7n+py+aHP0BUsmI(5Y66u)d6SyVBmPC780i&}M}lr`=ct4y0cj^jp8{uHm(+ zcd+u^-th;Je)&bxqu9+zx@={EdJFw~?m!9#=85^?{u33S{qtfMQCttvF1&Qlk;O$OG$6 zY@6P-@MP!L0Q)u!wdD(vUcb)*!4AWsgUML58&{RuWt~!Z8|c_uVF^P-m^v9CYqzzV z{kC{oAmYq7Cc0U|JxwYr`##r<VCh>fR&0fp&P=E&qD^0fJswa=>HfL2j>Yj(_&hR!l8}iBCI82sT0Ti3m&O6DB%-DWAsyAT2oH^KtDom{w!S&9Fj^c-B zVA1m4y)*tZcrFMs(!8}S>*m?A6jxnU45eP*{HFIx*dPg(e#Y||4M-_2AuY-gqEk{J zkkm1&KN!sac&TFUC{E^=wf|u*$C!r9(K+`_Hw=K`Q>vwfY8eFJtvnOi^dxa z;{#R?W7S7KGVVc!bpk}6{co9*Ze>ceCg|-P52hV zS=xjRzkp3Z+lQUST615rwNj=tm2;~Uw~&7X-wXiCf*}MoXY4jlMB?9Zt36C)Bg{k- z>0u9NyO4$F1e#2|&dh}PA*rhRI`;kA_#tg6o*j+b)Sn$LAo+)f zoTA`5|A2vN0iH|{eCna1D3mg*XR@Iy^1e z`}&-L%-6SKqZ98$X6mm*YRWWvBq~b>wJd16UNfaDLN^tua_2&KcQ99^2do7?zbgrC z86Z3T@V%_G5X{Gl;@*KeJT$~PJIf_eahR5&UbD5^w6|7fzqkH9q^=yCBLHYU z#JPOMK2jB&iXev8(vTM=5S7MAd}0|11Cvh%(^;g!+2Pw8bx&lnF`}Lh*l~>)d&(Lt zY{cLtcJp#?0@@EBmTgpc_MKEmk=h&Vidw~j2dUL0;wO0`K_1^=^oy0Cpsx8jVJXDi zHy~;=4YU^$7Grj=h{!&SDc|ED*cd-LOS^Zo4!(DKg1eK^$`65H z7mE&0G}t+_MuS52RE5v>-LTIA!>VDcs0#Pn4!>S!%yd@P_OEmQ9A1_%z4UmQJR*CS zlr`JPpNLZM%X}XP2Kz;3I7QG_(k8E*9GS#`QM03;bQ33BpzNw12A zDfV!iMjq%bVVe(80ZgGKyZ?KLg|OYh5DQdH$50otDpOg{QQG4~WOs}`+Rck689kvp zs7ET6=qhmRf}?Xsm>>sx@}-Rj_GC@4E+QVVyuVGT?;p9IO=Ry3mE?1dI@LHQ<0F6+ z{#sKF7l9O30G|t1MIDxfV6?{1pVzM8r7EIMk%-k4gN^4R2A!2Tltaq~qA9kBtyF*}>uX$c-kf7`--)tX5E>oy{L5T)O*(R5*F|W;bHqs?95MdC@*D}b zS`5vmU{UXpe(<oHlD7AWZX4cV$zRs|=i-X;ROuJ^oZX-p~tk*Ynd5s@i_* zNRS@q?Xy!55iN2vzXw`u0#f-wc$ljv3<5t}@^8edJe4ZNo$zS0&K?C3lM$dy(XdXw za2Llee7qF5Ljwx1}pBLL&iSwsF% zz*vvmVyTU%&#CcRUkr}$E1&%0G~7I9D;IpFv6~T^6kvrw`ZtodzxYuZT9~(Z&uJlU%$Ahf`0D({@|f$;+sbl7rETT=6Xj2It-rod z&ORw?%d->VCQf0dg*(N)CxOhgxzGU#UiD4WZJY7@tx00s0ffaRdtpDvzwz!)KCVM6 zhzxd3ZTH|-X?}9JO##71&}Bt zuDfm%CLG&XKdwTsnL&mM(om7L2n#5JKG6NzV>VE;%$b;qO?t8}JfdUcf=|}G^1cBW z5u*HSSAs+aLysuK8`yQCT}_oI&Qp{V8{MR8bLm8r2g}hM$_5lzehUa!X3!!_3?5y$ zc)I)0EZ8UDW3BI6yTFSA5`)?tVtnOPbo2`gWec0rQ?SfX9%MP-k%GHo%h+@op&K{+ zj-Smt8-G06wJ_+dg!B`YowpQNUjt4E8Fhc}ne}groWq*9usd7JSoT8r0EtJBd) zK-CPggWFl=WbBR|C=oKKW1JX&DK%iR_4D?JuHmQ<7DP#6Dvd^8Ojl* zJNbgSm%N7yOoQJLIxn&L62DD4rZgLIZ^pQ{U)XUZcYrdn}li|Co zhX=l*dT~`DZeqy|yQ_Vf;yRzMNOD#SCLix0fl{%t(~_OjD0&fQ@w!s@W#5^p>}1sA z?TPY7dd{W{&SL5{%!S=Pz@7pwnoTT~4qyNifDQg|yx~%!NOjP)p-NS~)#kvBHT?nN z5lDM5;9n9c;M8+C%Ua!Nj@mpmeODfdEt73qSQ>Qn|2wZdjw*#MJSVFdIcRE6P&dw#ppk0CkyiGm71%1{PMg+ zt4h4=Cy4oVvKzEGpLlYm>;$pr;s*4AsTC;7Iph-S>QAkdv!3f#WU6k1T^Le5vYT* zm_FY^#KELzF(MU&u;nW&HFPH>6&;EDR_z~NtRL5npAQsU?xa00dtO@1SL7`Yo85VD z*Ot+S>N}I`z9@fSFDgHmOyYb&@}!Sau$ExaGYIvtgM7{eLZFO6-GC$s{=gnVoAd`s zcTXp@An=qdgYf4_alQ*BTO7fLtOljlm1*A?{91etTxKmAe&aU(pL`A$koQ^U|=0$G> z;z$}W1^F$0yexG;(gz-E$|W*u%3Zg3eeUa-kTF8j)tPs2ax5Ll4kc9xdEiNYkx(|C}ZwEQ!vvKKs2MrD$rDm8= ziws_}ad%w*{Q%wA%XI%%VZW%UOw}l>V$6u6$6^lpC0zWXIA*T6XV5m=RS=pLpWBTx z*ACBV+yw_4mF;_21A+ZCTa-Ce#PPK)&YGM&U$vp4ZV`?>Pwhqe`;rAV@#*}`%*&4Q z%aFtc^jHwWFP~%7c6UqESq3sE5+UIgB}Xo*MvO_PgJ@APfIZh`um!amo~SV~u!uJ} z2PA@m+|S|Qzj?oxgO2k01aR4-+Bj2%D@wMj?pu`A1j-(pr}K#-`;X4QK_INbyRmhQ zf=vg0*`eW9Yz@1P;h=i#HQy@Dce@VQhOhR zujqci8`qsx<$^gNK*&jkRl;{L+3;>aIC;$Hx^zf~1Hmj~#iHP_ixf7P`@IHqiQi)K z@9Lf>-1X>TJoRq;4w!f_3uQS9*b2eBs9wH{Vbl2@7}lO=dXMvn;S8Auw{~hTiJ~C( z3sG44yS(tYEQYsRgNy6yHETd`x&t3Bg!PZh@uE|s46S(pq$d_Togp2?L$rwtZ@Fze zqvbMKP{2C^NqTTOI$RG(4nDpIFW+)#*&X^CPw+g`VE1|B#*TL5#_u)4{^h4jiHDn8 zT6YaJXNlXLGz+*5|IxI=arXJ6!9KQp&~W$P4SzljdIQ7qB1hZ~YZky>9d9q+nV@Yv za?wg|@b{Mxd(q1mX@VbunX#yRA1f-@%RcG%_o{BqKt9WuJCK*m*fwx4|)I7 zGCI!7FCh}*MQ7979iS)a3?4K1A7dsW7Z(at%}_K=C9(QZgJxgkGJII8u;tk2{__ab z^Y{=e%$sDn1_n0qWTdalxyB&ji0pRdF=$58?D!WzcLCa>{=3z-VWOBlAQW_f#)^*8 z&FiNw0{M-b5_uo{qkrVde)^cS=ltFii2X)ZyuSG1ayX(ZLHzLKaEQ|@zbez^^0$A> z`{m!@pS2H{FaP&Pxb5XQ{2y>LTmdVB4aYTeeb{M@y{PDZ_XUR@SN7wENgn30-`@kv z%zxGD@S0+utpiw_rv7YxCcodHnQVB-_4e1dn3cuEE(ZfFV?QG=g_HYWVyL6z3T$%! z5tuN0Ksc~#OXx-asR(x7VK0)1|4}~)hKJD$acmuatFlDqr=FZm{{fzD`uX{vS=Uk0 zmjZ=ePS5D#${(GD*s|$3{$5tGA0QGlmow(_Z}87B@&7?LW#_R|3hNvOAglOmFq4DI zl?^Qhw;3T;juc^D7QN41#16>hi1zeX$t5ms`gRXCX34B5<`N?QG-dZda41D(J=wwe&q9Zu0L~5tJJ&W4N-XBu_Ca8*P3<~GYnvu#U^L^>a)zaB zf%kEr(de#C4aq;<1J*3gYr|4DW@ha5Rt^PT;$sJ9x@OZSvJveO5sv!k>{k);SgzQ= zFRXww7$x_KlPPpBt!D2ybTgB-^$Ewc*A-J_Ev_E^Bp-rJR@Zx>YOB|a=YqFh(_7DM zvxg*76cl6T1nbjsKu8V*-mAy8oL&vK&fx1IvcF4Kg5J#iI4En^)^%UR(Zopf*YKIc zqI*V$gO)AL=nYt313SZ;H!|hK+ZJo*vk?)!?8Cak3zc=kMp9%-al#@gr%^j%o7kdU zw!lKWlVfnGv5Bn<^mJ4N{@UH)v;B)el;gCZw~nF+v*xjIhL~-Q?Xx21b}PGBxv>D_ z%MQ>knAN81c+b1=U8FB!?om|r??o6}%=|qTe~+s+T8$jCWMdVTOS!&}QF=GRgnU+K zFhVr=b#+F_fQa_NgOE3`J=tq=#M=L@+mt}J@k|-D)FBlI$zk8d>9+c)Gculh-0{)0 z?)e!xL1vYj!g$coz-DyV3ixZOxlP{aV-MMXUoH~3m`_I!&Rxt%V9)sI*&U8}Ghk0i zU_Hy@Us{+;EJiZ8t&S?0EQ8}o=g5Y7S1hpGm7uB@#;K7JyVyB^)!gy#b(f9?jJK(k z^&63o19TZm-9K-v`j}}?mm7LJ%Sx5^BvU%|GDOB=D^3!`t`g_vF%mN|m#Zo0h5qO8 zBEtXF^**wq(2ANI;M0Q4`YoBa`L?ktM!zOI;n4zaBa4yv0z21j8>%C! z_(yo9n$`7v7?&xWV_&Fh9F_1|jBW_kURS2;XkwRmG1g0m(-cX2w*6s@0nrHI8t*!Y zYds-KdaZMD>{MdmdEB$Z>;(GVs7=!+&dT20D;DR@(JEp7V!{Qw^fj#)v=}@CzN8Wz zNtFVc1s5LXlh70B$ccW-r5xc%fMDSf-1Y2s77543b<$?>$FO}|0C#PFG(+RzF zm|}EKs*GDWXDT1J?8mP{*^jkowi`Tt?M_aMox~kIGnv?kxmMSfc$t+QUO8uIDHA!Db6OD?8 zo`hh(0AC&7_4jy?>}u;};}K7UfNmjn8s4n+^b8^O;L=) z2nAV`gg1MVgOBJh&FhO3HL-0-)RT_fpEY7MM&I0{7Tar_7Av-U!Yp8pyuY97C@f*_ z=%5Fy4^|U`%pbaTb-br*xjS*%=(>Cr5yGJm$+U|9*#D+RH5;BA8Zq*wszQZWbmiQhe?A&A;z718rc_FDq|n$o<9d z8~@rqJKlr+Uj5|yr3y%=_xpk0zwxWR`(M=q-dGT{w(=0D{0oaG(5=T<-L@VW?13!Y z`LCbA7Wm&5_y2HHFmJ|p;G5P?bnL;SD48=QeXObg1Rrs!lR%o}Y=GL2Z49DoHSEQu zdJb-5WG;_6;4eBtCGinXckD0`UHL~xu&0X~J{1wdO3J+XzqAhU>Hl**qNB3{=s(!k zvqI)e12-n5X@K>g9t@AfszDr7LmF%kv77V!%72_o{^xDA|1yI;eP4lbbY1F*;rIXP z2Cl%DrOtD^Inp&i#RgbSV1!_D|8yKt`WN<@OAXNeR1mrNH2mi$>vIDUAc3;BdyTaE zo_puGO@?f9u$$Yw5F&mc$bvdxotrn`)gv(cr7m8Fhh6Oc(MmztDgwW?dhjRqa*_I{ zChVvG|J+nMs{RBM%-t06NoK~rIrj29=67smb;557diL*~0*wUGr2o2|b?7e4LQ`!19&WtnT@mLuYkUAqGDosR6fWSBwq$?;ORY9be5PC?W zV*x2)q<0)?(t;5PHGm3<6ced|5I~3_K!6w^2@pd58*t9=oZt0c@2CHDy$-!$ybR+8ipccl+4vj0yGh@r&KZsO!j}l;X*1{?Bf__woI{U3AW+ zM%U1Ymw3V_$>JEKqO|q&>wSmJO>l?ak6thRGP+M`NuCt-f)LP%|Q%m!)7yfV`2cMX3b&*-pR(Kgfe$3|O8WZ-7beqnrYD-k z3Eey>&d5TQ-5$=s*J=PEa*TVpU1bYnse#9FzH$W^{}^AmLFmw?{moP6@*f(Yv?*aU z)M*Lm+n(Aw_lr&&cRWKF8xJp7s!Vv!Ov<<#DCmf^-yIY6l@Q)`Do806+vhK`9en|N zd4bgz5U(LUn!m$`d?2i$v;UxfUSM8A2+M8^nv z+TjRtEy~8cImoouGGE3$M=Fk&^|YlnqF|09>1(U{tobZU(>*X#!l(t|hW@VoukF~= z2?{m#J3Wn`I@pJ+@%;XK%|E^m@+#N7&OZ*gz0}^{KDHW0Dch!Z4QOb{Ma*wCuv>Bs z@(%dMVmw{N4;vULP@eKGgd$ml`+otx5QRx^U}rUI1V+JX^T38zwH8hCEv2laWvg7< zj9eUp5P|PPc;yAgo~Z9W|8~_PPg>njSO2~x*vJ8&Frpl(=j|sa@88Mqjnn6yG!JyB z*TBw5iX7@nM>Q;>^S{jPv+*RPAkVk%M7$3!{`xQ*9?~W-nr@WQu`CcDAbv<|C3+<} zqNSfJ=dmec9{qryYiC!mTjFU=n>{ZuQ1?xDJJfe~jwJZwexU>dX6^B>%p)4v7whpT&Tbc9x^g63U&A|+OdRBnGW6Y#ftA>4!r-cWJcn{#qr zAA5i;OIrKvDRUwuA-3dvmXyd(?}@TEwgN0^nO1}kGJDz|EG)&P9X6CQn^d2MEMmUo zb0w7{w|DnDpG;nqB>7{e#)D7IJ=3`BcZkPG=ru9E& z4glRY_Ti!ISEkEnD{o~_<0tj&!~pNQMt%bUqX=9TGXvZmjP%pf8IF zTB_1uu3rFM+{sLIEEjGAi|zlE?>u;0qj%pT?|6#Zs0~rLk({9Y z3Ge(+LrmJcwXDznGC0#BCnDp-)9;xs`H@+uFZ`L9TVwo15qOf zNq5amfz5wpW2Of}P7Eh7s$2EN(R092&=2-dMvw1jIfz$;U%lX>tB~LFm!j0l2@y$^ z0zH2>r*M9yJ$oUX<|%7N8@Gc-(wo^7&G3#r0F3jYC1Alxo@Y2L>Igk1B4TN~v$g1U zaZXyf)|?iA7Gj@gTr(B9bFQ`{_x#+8(?+nVzw99-k3Pyx6$%!VPQ+<0Mcw*YOJwgG z1w_kipZx|x6y=+w$g3rc+y$G7-1&NeKB$uBvxirMLN6ys>3~W;wi>y> zUV=}3nXz9Ph^SH~0El05@^2RMg>LUapakSIS9t0rU=YOmmSi#ayWqI40kIMy>R9s6 zBD(*S%DHN6#$UBx9$fXRs9T835U-9Sj@SKDWR>RXHl`lVdCUs$;MjQLi_A9ECDiRF zYh3QmeN6L+*9VdsnJj!wDIvEaSaF!}R22j$%TrPTi_z6G)G~;{Xf++uHX*9(5T43h z1$LvH?LN2%gxhVdJvWNKO$q^q#;Q$WYlgU@XtgIOB(G0=fC6oySK0F~0`L72LHQok zn5ri*blu*cvH_6Rpx_kXnJhH!yIn$&C0yfz8;l&a3E47uz`3UFv_1cx7%`7+rVZbk ztN&g4p>U!+QCGi1B!=RfxzN{dnsQ+o#0PakG;YwGI}z8l&l(YGQTA% z`T|$wrw4_8(2&-=_g*NB1?Yzd@C$_6AGb~9u!~DhGsE+oX2C0!obhDG?>_%yaYE9U zT!geTi({P^s>STi)ki}qC5vZ0P%19kqmr7e4ELXly+T_2035WxmMaPZ9JH>scBp|U zd2?!c?MC)uO_}11UY@la^n8 zDU`)<1fguILMN>64-O8A@$WvksNrFc2HHE&h_+u}baWrS1SFZvJj`Der0}fqW^Qyw zWYMPDz!9?GqGLi;Com|>dfPGb-K4xc%vb!6m{oz3(VtUdv?TQep{AM`dj`Na`11)*GE`!V8?}!lN)TgNMk8|W z0B(QoA6Rs@JYdhR@dC%=B74Zklo`ZlF!Gmq5%Ww#9oyc3XLzT*fIX)je>FGxH9d3X z>q2JOYonfl!JiLr7VlsoB9`|LLN*=tR9z4_l zk^|$H&1-5aG#w;m#reKF+j&L+WW)&8^cqExL3sR~a2hy3Ab0IQ^9GvO`Zst4m>`kQ zMgK~dwtxLQ72f_CSOow7MZy0sBJcm+^Z(I&?xZ~11{C!DNRto3spSqFeK7qQyM1NF z%>ioVzc`%*5iIBHI(T;Y?x(ol{DwK@e%Rryx4*c3{clcUCyU-r=N_bV>i4+yf0S+P zmm{_6DU^KKrQd!7YyM}#{H-~qLU)5b94>D%Hcdz{Vi-^NjBEhs z-!jM43E8<9a7GHG=JVo6n1eY;RYyiyJ`CR;=fHPO<4?Vl_VqGZ&!|INu!=@J9p{*KV=%^EyNLBWg;qfZuH>Mw`t2w zK)pRMSd4$hz~_#-qSP(qWv-T?gVD%H%7Q@)5DeuX zLsAVj@cF~n9jW7WhKSc+o;8Hu$~p+vn*sB$JL#OVsxB!gh&;&2Ed+j@&8C5(mCog!W17>QEY*a)XrDO?AO*`&!q@5tq8y1I4q`yI z)y!^)y#8byOXYM*skB@7!IkMcRmOz0fFw&GAxNVq5MpO#SE14bZ}jjt|<_YffEd6^)N+<6r8t zP^9|FBX$$(UMfx#q7{>)l8#0)k2Q9eKKH~fPN@r7q-w3!7#~i)u^8Tw;u<)Sa@n~? zS-YiCEnUq;TxUGm)hW93pvGKzGYw4-?09;3aCw!SEIa$MoE{3aS$7;17=2c_I zLp0oBw}E~6PSlYLdJaV2c;@|&D1vagnO$Qij!&*mnoPNd1LC2X(3;)#(f<4!hMwDy zO(3Ld2FnBjh*fiNbh2(8$iWk?**cWXd?Ft>vK@~D*XO?pc*{Ftg^RqaUn1-b>c7xO zUhCUgNquBHwq?Elowv`h2PljHE*al%vnJ-yGw9uyPFq{=Zs0s0!g&8;GT0p6S)!9c zsPxo>UP3mx(WzM)bID*j7Udr~DjHEb!kQlH1O15(3rwgBWHbb=Gw_UUx(Eo%chvzv za^(nxMf)ywc=Zk)tVr)*xxmAFk$0W_W+|fMY4_%r4u^M?@Frr@;w3uL>?V)2!5BT5 z`^GLK1&4(i!OgWC+KFxVe>ZERawVUc2M zSl6tK{9l^HxGV|ONXl3EISW+{mbwty3mW_1D_$w_2E#l_aT2D7SR1U&48LVon>%_` zVdYH`u%|kx=^4%R;uKs|hG8@SrYfxisW18hDVH;G%gGP6&3MK9nG65zBd}A%v`QN} za;{f4cslg9ghbaS_2BbvfrGcM#;A*)(;Qg$&g7 zxP?}o|F&C{*zP)U%RBd9Y*PjzvjlFqP)*57Lak8+{pXIn85sG$nGpWSvA{N*8sQ*h z{Fp??2d6zd&06|>i##_dIK$zVWSS<$odP=8xVr*#XTL3*RZe_cygX}RUt7rgL9gS( zV!Pi~GW^Cb!2Rrbwtl8>zpl`l;8{Gx8DuE%5i%{_Jxh+P=oe z!+=+x$Hv3P++aS^h9?||IWhw!2oTy!R#qWqXjta_nTH$GKQevUy-Ml$+#27zSN={5 z1;;^X5OIUr2w5-ohs=R4A?0#`H*W)2{N^pML} zd)xR+4QqH0%*=Z#;*ZfSgk5DMo!-DBhRB(@anx77mpu9J0E*?&Z9UVN3#v{Z3O8yZ zhj&y4sw5G|8>fC4!XZMb7g(JmL4OqNB}n{Dk8k%t>9Ie%f7%11r(~sLX`llTQ~K;0 zR55$0;z}f4pFMpEKnf#B%Mn6)rts0n#^4t3B+mg2o~o=BL7FrvZ-szIUzpLNF(#Ab zTQb}wfB26c{lKb5wMA(WJZn2stf(>rM87!v-NwbilFXo~1pDP6AZ4-eEpUXpef%9< z`Ri5fq$KQZD2-hq6--0VQb*Fi$FH#eo1JJPaY=^fqSP1AOdvR{Eu0CTY>^ZxU{iz0%k zW1`X%s-Ab{0X7B?5dek5U*~0Ax(uSe@pQB6|7 z2W8oZNnsts0LXHsUKPO1B{+~p_LuQRjdIU32X+PD-%OIw>w64sWr(^A1j;w2nVRSZZ&x8D-X{|e>fYg+BYawCiLR zc_YGSW7ED?vC|(@(4zBVMvI8)zk z$u<0HxYaRK?7zu|KhMtQTyPrp^*;mARXk!h_6A3pzk_?YW)3y~aCdVBX!}r7`OXX+ zs>7G`X?V<60&Gx%S5J*)phN3f#PVIM1IEAqn`N9^^D%jM=59mw1*+1+hWe@rGH`AS zKrG{PugS0kD{sYKHSh2;fk5I3GB{~bfYJaN+$Va)Gu1vqD&y;s&oAkI|8EQJL7{K; z$tjyCQ{=Pvp}2{+38+aur5KDMplUe8b7z($_T|xrF)?X|vXJ>$uim?U3vpK*kgN5JtxF{U0e%jqJ0~U z!OfSn^oUL8VjPD%DTjj%iU3ocujO4ijoHuu8Wg_#I&~q z0%G@?r7`+|k&c7Xj3f%6R=?RlKqP;?CHC0Ja$m=p=enJZsn0rZahhEG*G0_zl8gdA zveSnk10StAx_Q_u00_bD!;XC2c#|1wXYBXE!UpXdpRSNu7y0f=#uZ1D4p?~ENqEgT zWGmP%;MwdFMY1BpNQ&$VRtkXMH4XruAD3T>tzuQjr^U14;{zw4YGMdsd|#9gx?S{N zc1X_}-xXn|>2I7LHACMM;mjT7cyIW9NS4B>6mC?bz;Lb|>V19>_ zy!n+$G7)8W(n>S---!+5i>61Gs#7-{ZB>i6(r=lSwRsF#&j;SF`_Dg}4@&g3O!6#1 zp3ZL(Y_9cZS!kTECwvv{3+`LjVEq*{T?e3POz%d)wS&HKSjSeeYOsdFeV0K%FuEKD zh-jkc%Dpl7*2CR9^ZeP4kN&K59vD2T>WS?LttZ7DBN@pmZ++R6RE*5N7_zNt+)e4W zpZGu=AzVFkm@XDqT(8N{(TM z;#wEmZ-oBcr*;!}KA$4B zeAJ~~)nNObh)uFphkLUG=VrHgXQ~-h$5*2+Z9W7yW)x{g<#y%3&bHs!<&z$ybPodv z3wXbtTPPV#J%29oV<9bmpjD=(G+$mUJ9rn;iK$Be%{f(WC}FzN+||GXnMJrQTM0Dq zbXaM5=M%XjVS%U&ajKNW)na&Yj$CRLgGsNIW86`uK}&tA+gSuR^t$3;)?>P#JP#*Ijk#12HPvo=;w(O^|uLdgHnn)%@tKIs5a?U~Lr@b1J!VTE-6d>BS=iJh73?XG!P}XYz-rS7X ze0IBUIG4E>>H{@z^a(TI<6@?#l(P#pbao6csL1+{AfzJG;>3h&*StUY^Q&E?w|$>K zGkb_#D5IpR6Ahx2Yg!&2eBB3G&)ZMqLmj7AcY)9LU)FUqigcbg$nT8czhD#HsY+)9 zO|)yi@{?W77^D087*@l!t?6v*|HQGrK5^+?sv7?D8ML_JK8;{||Bsij0)fu-US-$# zG*&>uL1y^n@T$6BuUc$|N=-JIKY0UD_5QfVVg7lihIJJl6 zMOA5dLaH#r#R+o5(fs(pr-9a_e<{@*|E7}UlMbaKAKpedGK**`px3z$_DhdhT9G?_Aw+LpM?DqgJy zeEv-RsoIVVhq39G@u%=3HY+a`ZcvSV^<5H*g^R2xLLq=wQQ{yCOT6lYVL@UiqClrj zj11>=XNGYxl9#uq?5Fp%<}Cv`{wxVvCGWwVMg+&hhTcZi7i@fNsG7=Y{7J@tcirm# zbujdM&bJr>>JOB6Y_?=-N4r-#Z8F#eh)=gntjFzW9oUG?soQ&Vw>KE-Qha;{V zg&uoM^DVJ$>%qNE2t=jKHKD!^9?GLikyRw^fv*7cQo3P-ip#o3fG)*wA)z=Nsi2nG z`pc5@M57XrR{>GGFme*IlW+++#(Pr@@>UZ@0oesiKe*>2AQ!S45A*ER^I8shqpYWY z7KrS8c{34nm}c5$#S&2cQys50G+GqCG@|Lgc6aQUuebz{@<CxWiq_EB;xg7cWs`>e{t>r5F2wFQH!msF|WgAaOwnuEQ{Ge#l0aXb_3ywq1k`}p+ zFQ`5#9GH{rZy(|3$99Y>GmL?cM3&nS7pn)x_oL>@~-uc?#S{^hncv6DR26ANFfeinrIcV-Ej%oD+!m)T_}g3h>iY`M%Z(qdP3e;*iFeWPAW@ z_46}E9mMfS{KTkcK%$ruAvb0ziEK;X@}=22@k;M^wp5(Lzf#E6TDNP900lH}EHyBn ze2mHcjXGkWrzYG;pscsu0Jnxr8X%D>S!3c=7LoMdO>b^l$%5WS{-#anH)`JAEUIFq zU)Htph0nQ}{rW#OyK!EovHSJa@fKvD#>=-&@^#yy+8A@BNk9KtXBuQLFa@74_gVD^ z-6mth9jinx(fIfgq?jIG@6GrwV+~TE&vImh- zC-S2_qj$W`aIVB)NN0B=bb3Ook@79$3Fds(bDvV&@8*r}{QA_3+Q|I;Y+%PrK);7; z;7Q}r;k~T9#l|ye$FINjR@|TBV&;aFrb`0bhV<`4hHE3(&)jl(nunxY!^52#y+{Wb zdydzS;55<^v{w5y7qonk-$nxMH^N^OPdj5O$KK9K1LjkkI3{f5Z3H>1Y!({3duJIS z0H|Vmbgu7B0hGeqvZ9>>Z_N{XxG-UOTa603F_MJ4EfqQ8!Om+OC=9KJmAz!Ge4fvt zmDcC#cXVw6Ga@4MjLpQuW%Sm(BMnPP?}W-S-4^-I82Bg85;;*#`E|?6h7}-3>w4X%;m^%*H$|76 zoM9lo4hVWrx2U=}yixMLfUhGYKLYq#1yYvWVN3bvzM|4wN%Dm3q~PNDM<{!HlYA5; zqtjJZWnaUm2>h1@I-Au|(`k=9TQ94XRy6h8lcV__db|6}+&gZ8O41WR&L?UUkOgZ8 z0^|in9m0E^o5SntJX+wywj9~dQ8pg%EPG_@CACjnMd%S?%FiE4{3r&y|4TLTh)cT9 z?_p4w!RDvchl*B8r531BotyU|lgqZ1f?o)GwN9wD)ih{NP9Onq7ZC=FUhH)$^Nv*? zM<6%7-hP){ngU@EhN?hnA=Mj^A%Nm{+sR(*K8&XSqqJ!1WooEDtp2-8fbeX4w3S-+ z*?mk@s$NpM8#0Dn>)S7@A$)5LSS;z;N+~hH#HLM@O#Pb~9g`VvUTKQ)09fKSCGYhI z__|KAS{Gz_J_mrOWbEKy2i9`zpt(QS0-VvAy(alfeMf8%fj+Ax@0OyKhZn#%f-F1< zJl;u9@K+t1|{?G4o+$mH(t&YCp zMjT9+gi#RpRgJz^K=LcWJPVEVoGrUUhkZh7u_*cWginQq^jw+YgrxX5U%llQG_s7R z1=^r3EFb~7(5P7%GxseeKsIs6w0U*!8#6y(*0VX#v!eLv%nGI z(XE1~4a@m-%!WYFYhd7&{gU(U8OUW>`?!O?dfbz`1%2Xnx$uANtD&93=kJ4s zUt!=|M@yLQoJm{a%5}c{*(y7Hd=D#0nk0RvgsV>OOd@{)tZt}tJ0+Cq1q6j*-l&xA zhbLHVVbAkdk1uL0=Kd;KuSrZ&DsbKw2V8r@7L_zSQLccy+wa&fy5`X3 zHGg*B3(|?5E%^oyk_~A*|2}xz14^I!#`6ayONxq>0LlOtdr1<}KVCgi+UpM{dgN6{m&02kBM>bT_?;ZIzj@U z&XoxkUizX#ircp*J5+zHLBp1D&O7hi>(Nr!g^t@P8z-^C6EVfEVfcAoaBdMqI&QR- zAPmdpWXZ@s%X7v|<%MFRhPv%*@7-33o^5TI!Go?e?jn0lVWTdz@C(sCBX>^b0%Wm2 zAa(_OddDZ%{-880*dIRju<&m?&#=jH?;gC(UemN9H1BMfgR<-|RAu*>Vp^fcbS&=Y zOdeKQ5BYlAOWm#nJD3c9EL7IK%uD$!lS7lAB;H-#EYvuo*l?(!_ZduCQ-kH+GlTot zW9-x>UnXw^E6%v3H8rV2_quacz{hzSPrNR(a)b3DL+!0%A^g`3!|E!i95n9NQxn z{WcHUt}*QT?lpF6>_MUBuD=K^#Y9G%d$*mR0k4j)jjn`uBzo$6Q3PwNx*juHKWF!} zU%lDw(xIfMi)~aGo{1NQ74V+T+X-v*+(RIi?}K%(TYLoM@`5@2_Z+r-s~KyKfpkZy2Z9f(-MX^A&oZcDKYjT9)$+1mqjZq|@kd~>*N zar~zPXLfPy!TOmlNY8s6La$~p?mmzl90tjlzDFutoq2tKgSuT;5aW8k{tGH9VYBy@ zj7J-=P8tS;y!q6D<`ZFzpch!#lMummu{TL+oXMo-{0aLk)#hIv(3zsjE)jYz{2rhN zW4JamZ0d^_IM6M@UrJQL2BcLD6cYeimP@J^bw8qW@84ds*@+`$CC~0=3U}7NG)#Ak zHuk%}PoJ;EKf1?2z2Z`q+jQus^{dX5m$iEfz+JD6(U6Y7FRutUKm_;Ct5ZV|s-*NG zPHosyoCRos^a^QYXbjfg9Kg!RT7`AaOSn!bi?%Fs0TFfHVqZT)dpnSP@YjG<|5ulN z?6KURj3l*~doF+8_X?Y*>{-=UPHB*X4L_C0td9I%z+8J((jG6L8R zn|%lkgil&`?H!8`N}BFI^^30+c`&7M)Ov3z^g@K*BtqG^tHUen)1NeJ=;rWf8kUEU z^8HEkQB!1e?EwK%NaDGB!@>$p4iz5hvgBTG{TORT3SYaHSV&=Ico86?c>Z z5NZms0bi3tV=NM|WoavvH+r^!R!MBs^Atg%!p67lYgA>~DKYsE*aAGyAQQZG z$7@%0`1vun8j$-}G8rAX3T-@hypeq*v<-J&9G9*91Ygl6>FU>=cCmkbQWKN;Nc`?(z64{1+P!uI zJ%sKVTyeXHUewT2+ragK)E1(ZZChJ;33PrP)HnJo31paU=^KYh8|`!-$B+E^{?X_L zPk8Qg=r(Th`OjNo&r%$6&9Zcz&^9FI|AZv1fY%Rx3x}{1fkOz$hgF|PF`4nS>@hn+ ziBl3FH+OUehd-#{EyX4M^yldo@1Z}|-j^!xtFIbxn~78iE>5~zNShDD5*|BQ)VQnT z)=6$#O#VGpo-ZF(;XaiIa5(!X;t=(^P@i7!>fYZ4D`^m>Lg%`?UvJZJ=gYBxL}tN+ zj4(Tg>lKTkRSbBy9W#>j`h)$;@yNba+|MK(gB7n{=QJNowmg#;2IcrF!)+T@QdYPcAwsDZOGuUBos=XcJ(0bF^!OuEk9b(moj%Q<#@^av z_LI@c_g4v_JYp=?G+7N2&%~Q z8g~EGClheU2v-%Sq2oDc$vFP|k9}FJs+B+|IU}hz2Fp*q1UR2*0m?p#U(o5_6VGq1 zAPKCS4*@KYFDo8~yq;jOdq9QpmC4J~Z_xfM?K{`L}`-2SHq}<*n{c@qOP#Yg@>7;WwMue>4eH{UU%po_tEj$(9!W@>fi|8uj7GYB`)IrjxMO;K*rPI z<7~Aa`;Q;$|r#azmoc?KofYO3Y|yTW0xrRiLC?{5aY~@Qlz=( zPn1y4skds@ogpVB;x$VNH6*HN{ehs~Lvu|Bd{I5DNTBEk+jw4o4RIh4<;^oVs!sGz z+E+S*{qQ=armjQA7Xgv)Ei;-Yi5RcMa5=8yGW4^mr#9g+d^u9u?f1T|;tDhn#WWgCti27V zbyXM0i&^%>AMTo6n_hSc9Ab;FfvU&%kj7w01wygFJYK0_-Li%KVbyC zlw=@lO_HM3%4Vof!)uBafU{m zR^bz0GC{Dby#|bdM$T#Pd>I{acU!Z{wy@w@OPL!cM=WVthTAZVeXuUs!``M=(DiO@ zZn=@&zB(-@P4;YwgSJ4Hs1b{M_tPG7`ywxP2BHj$ju#(xwwe}5lOzZ{&NQ`xKpq@~ zMH@xa=j^Rjc7(eq`|N41#RtwEv1MOLH(NyK>@ZVtb!|uES&V5UhmB| zKsr+nf(ZTquzVi2lS`GuC}Yu+lImap5h>V8Y?x7>h^uM_3L1gyg)soGgfWsM=5YRv zQ|;I`oJxRmEr@*L)aI5%A!pEO$7rK$EKV$0{9?He4+mJINbsExK)1zmY(FNc8?D*W zNl%tY=-KG4XARRp(@WGHN#p7)0vbvUrB$)leHnyJ30q^vUj3KNddJ@a^{GsNW@;oX zq14AGM5TkF0Cl*xIzMygfRyXT8>bGkj%6rTf7_%(!6V$ODh{=+I=@DJiOlFsA*m!3 zgsLQlyQ-QJ27ubj8d~~_)UuT5SR2pPeGay(s9Y#+1}e^ZBgqXj8-X2+vF+N$;Ko&H z2%9XQxUTLuu_E#L?>gJ3j2-RrV0`Pd4jgCIUiBEjyt10wumAa*^ImM5Agbji(O_H; zY^#zf2apjvsNC?8LVTXQj@p`~LFBEh%Q%Hr-ps}>vloN@)qzslSwkYldwYI4wLQ$Q zbl@qEfQcIZ>43(*!O$1LUL|v-);TyJK~=1jFQxs`9&r(U*^knyY`p#P-!9Y>XhspI zZ4E9c%sAdZjxJsZ9Q+KhIr+vJ(un%7kA7q3Ee4(f|-~%!9!JZgZY=(3%B;#OKg96=i<`X zi$X;?{t&XO4v{(!{JX87Odnv4IX0X=qsGKR&TLQSVx5dG<>YnpqZ#hnF!fnq=5!v@ zeM^2qifqIUt`yv{^M{o2C6wL!l2e_=kN86a6TCm}9!#~1U?2+orT(6C?6L4c|3V|SJ1itFrJ~5;zA1M0PB5g%d>+ZSx zeYAi{i)N7WqXA8x;HGeaBPo6j?JOz}ziF_gr+y~-&Ft);4NQ)| zQSX+M)D(Xx&fidNV%XuOfF>}b1N+- zO}sR;ACgLNTvVSMGu9h^Q{+*XoUy2e6_AmOXUa%ch&QFF)|1`_{H4O1DDTi*;NMb= zHA^Ay(rr^Z!p0HFKRp&zUnS$(#PV@>&AKfR>ufCL^k}@|*xlfg3qoYzgcsDndF4dA zCd3)b?k!*bymz^CTP^%7{muMVoKN%{dvALO4SvH&gNip-T*}oSrZzui6yMpBaLfkza5}R<1W6?)SAiOV=&6HmyDk zXC<}{9tYNgaMk4U#hXtz^-|DNZNv`#3?VHBQG6IO**5 zbY?%;)}`6nftTWa`-UlfnNp|1On3;3Hxywhy5dPV zhNIl2M#S01GW)@`u9wO9H`igvGZ~8X5))7u&L5`Qo0OK=Bgh~tmT{eLamVfS(93XT zdH2P&9`dINen~&%wtDrg{^o{%bZfji%Qxw!siiUPbW9jG_3MHDtehHW4*72|6>#c@ zl~IjUv_ZkianCtK6;AyNJdJNQp32Ms-f&vHP~Frv!MH7Tozf4br=;$~Q-Ew1Ib5XM@BfWUV zB&m4Nm{SvMjT5K0mb>3pQ(jinBAN-`q+rq1+o|6$D&v@9xZVD!u_)=4 zOhDDqo&iSDYF3%S(?R3Ot^U6*i1*l-6nKXpbN|z@I=vrevlvf9VpYZ8hXv+b|66iH%sdMSn86#4u zzf?cu7+_&u<^Bbxebklgbg9F?s9G|Mlkxh_G4q0((_RaVBJ zRaIN}-r11AVurmQx`?%23VX3ZalU0(-@|ZA-YAE+p50hX+n9A+eo0x16BZMK$=5ApR|)zuG|dqn;FHSJy4eO^-8-58vV^9 zxOf!`MzP{V+knTV>ljX=JVAYWNC5{9w&2l#F83RE8b)Lo^1)OCG)3tT$)Lcv0F-0&JJX6HeycX39Ia8HGY-4Gg=%H69Zt+oR z^=06q*6xNs4;8nx)@|98C0wMq`Y*(Dt=DC&iZ_YFe23HxlA=GE*OXT6J4BKOmiWg3 z_Eyt~9jifnKa7=ANeKu2G(_fy^wJwsyEy|`xT`|BpqO-Sj-v}s?~g3;@5k`E`Pbm7 z?tMH-*dO4-WvXelE_zF?OfuF)(d=&IA@6aAxw7Fu-l)!vyV?#{EY3wbAneS>IJiJ- z+;BSI{4gE&d=Q0Esw^9Ky;6p_YC37wKT^WJrc53tNf*Z~yDV8z!YODLB#rTjmwgJy z7+tx}Z05D9uktoe*Uzz{`6#GEUD!mLbsClh6LM~Q$PY1X&$=?BcqwDPJ4?onU}6QOJl5L&sdJGyXXn76oEgk1`Ld zn?}i6S-aMe-gOFYA`GhxOB}UB3*^k+$8iVYO`&FyDTpC%0jZFioib0~cWk5C(}$L7 z#uDUal*Yi($hd*1#$WyI((p!g#e^;FVxWBS1o=_-FS!m0kV(FxzYW*Mus$lO^t1rC zmBtt;a4=e*Fyncir8mt1+s^dFqpH;b7-}x}O-S3Yi%uNDno`uF)3endW&(1l!!W(e z^Q<;>xT}0;!q!PtrG@IMl7VR)Lw0R>9$E zFRtSw1P$uqcnJtZ^ly08{u+hatWVPK{52z^NEs z!hl5K%z2LMxfEl60=OHVv5{{TX^zW@rmk!yQ{vob)zo|!?M=P;t?rYs+_FKbW%pk> zj;O%?W=mVOYExdg9$__;sF`RSip066Fwb1`r$dIhb|A(C{3bUA{|2xw?3=bXXH?Dj zXxU312m0gSBn5%rt4)JZUgm)NhIy9ZemhcOSnj&652@h7$nC`~)8b$`V3BYNhAsHl zikp%`)Qhn5D+>?KGC3&MQCHF>i$JFz=uvZWWddpI8>VxfadpT?<+;WiI`yj)c@r#O zq2WzG`+(#P3*o8je1t6!c+p|u$@pJzn^_cv+7#H3=R%9Q$6uG0{k*WRqIqo6uv;*=wTNSZ=?JwtnQChvUr=CrX0bUeP&|pww&Q?HYFjoB1i#=^jGd%M z;TU5!RX^g!zLeDZEJkJ!s#%8CxbOr0zWxOov*V`%GvvA<=FH+17F2e|MPJo;mBIGs zo4p{`9*d&vrYQh^dG02VW5lZhdWvvWR2s|Jydf%VbEqI!UZEb~2YnIWYP4OVMSG-_ z@}&{V-3kt?I6;+-!f|8XY(}v0`uP+B9I)d$TP%qknIFE#fRarf&6D zTJ1oTDNEwP^dqY>s@?jwXZW4DTEXN`2z zix<>6&s#r~B%>74R+G*bhXlZdQ|A-AK0F5l<8YG$cSvUz(XJ7)QH>N=4%Kuncc3A| z=)$$JjGh6mkSCO7lZ4a(&qaCCS)B%=I2$jsef<_H*I<3ala>l1{#m21K@v(Q9AxA<876XWRi^ z%!RpJYZF3pda1Quo&8qX%OZmH#E!=nNTth@yXe=XFy_ndGEBzsIgwr z()4CI{~F(lkCwBO%D8|A;vA!^x!Gu65BavUXqtLFA8HJdDcB0JGNIX(yTqIv-# zxV&Y~%;1^DnR!PxT5_V3Lt5t6v*gU^;8Y?D$)AiejXY3yIo(ZIDo(fov?x_9?^R@N zR-{I$ATJJ1+Ybm*<$TGngAdGY6c;^7I!iteE|FIQVtb?nTX=T3dh^O7%19~gOo5n* z-sGDqgGr!s+?yNHjH4Ngx9HrVR@1VHdR5MIQwJmG!d3gBaCtf~2PXX(!Mq(-t5nXwEbWEfk88T&R1W(=?Qc#q@#dVRP*T*rM~*LnWV-$HA_X{ullVKEDJ&IASH ztjSE8lQA`Pdikj8*p^?7I#g6$pR{~7V6WIvvF50H6gopHCt!oZz)O*ROis!jgPMBW z^1(0+vk?VZHf;x#hp44uRyDepj8)!WOtxf7y$fx-z0{mNWI{H%mXv8e1`5Nm(+!^8O;zc}?SrhIM5d%hw8WKgHZ&kip;Z^m9fB z>D%&N*05$p)JG%gvR~;acrX5CXy9-?mh==@=DQT#$5mY5JbiY#3_7e??l|M}3!;K? zhRZKNDua!!CM({h7{0H^VS{EvtQ=9d4aPbMD7-2(L*CjqrgCFDr(g3Hw{*@{(Ia)+ zMD=~YW94l`IkZ3EzC8HxhN1eeI(tG^;TM0R^@3(8a~SddHXK|Av6=&O_upleZ3OwZ z9~71>a5HaLK0_|hn~bPOTar)tM9a;p*slER?pXih5ZxZ}w)u)R>C4@S)&StAPxj#s zFD{>VjZ?X+UVG(q)3KEF zBoIF0+^iI=rjD8yz|H&{(u)(YE(|-7EG@$xk_QYSO8@TPdJq1jFYhcijybVC0#tBO z-Il%R2mf4xqA7wDBsc9A7LJStl{P=lzs}a8zrU&+%O<~HO;GdOA({^qNB*~kbJe_a zV&3yp&IABiOeqEgN>I7mB9;6sM9+82_t|=Cet<4{fID6OC3h~PzKj32PMaWfXG|ZV zpRVCyQXN^x3~$Iyoh&*T&>Ju~+S}$+=<3wbAxw=Azls5khkW+`LU-TBI-y}*$PE|d zg8PYer&^(kALsy{-)}W*!x`2%Og2gnx8JX~_LBg*nP`VMvteI-_)T-J3s~mmQIGp@ zW*H{-SbOacIr1eEB$NXrX_|?(XDO6;bVd!Ql6C${+?YkPqQ>8x-ZuF=aBidItJt3B zUAWm{zRoMX$3dQ$hOKAG8A9rDWF`6SP0F!QS(v*2PM&~YVO<8y z)qcRv1zrKq?uu57=v8wif6oYkua}=O^7_^;+Oeq(!8am%d5QGoJs;1uMqy@;bg4R4 zdGm(Y`Nnh-)3PAMH=owLN2J@%W{&@TmQT+j>%?7}Z=l`j-R3C+D4zm8UU zJyLb=X+rgVBPN6ZJE+B2$f6oI7o8+sN_=5*o&C$C<3*!Nh5Fu;4O_XQm5#*b-5<(@ z?SNy%>=m;#A(*`|44HH%l9*RKufG6R=K!lZk0M5uw_3W~^ha9rc-}(rRQ- z$jQzA22_ATn7MJP#Q2~4*0LS1u8PHaV7l3-lJa#!(-GCK`$v7jL51;Zn)dvFqGrgn zA7C(Y7Zpc79#^wB-xep`C94s(YBo0r+SG^o$Qa<29?H~HpKBV}ecRnc(;C>BKNjyF zqkI!Y3m=rkJRM9_P{ys33#^OfZqG@MHYA(`MHI%t%yP|0b>>h?7SeHWs}$g?DiAun zD}e>cbA=J5V)f6NyvcyoXfG$ue1fBXVc-ROPy${o4TAfwgGPd5B%}dDrl_s+{Zh46 zo{gGF>SdZJB4s2+)A4qkadZF>MBsmv{XF2Gx%nvwsu~JiP8i2pES>8TtDAEg%F^WV zx!98vyUngRgHJPPQ>RZ9z%n$QLQzm2U|QIDfi)t3+3#mbh(Vpeg@&BrfIxPnycGY_ z8kh}Bc$u2ZdCc8}{UD>A$f$)VwONs^h5yzyjH{xLTDWdONuG2z;@XHm!a=@gc%c9T;PP!tgIh>biB*-p7u!l@dqK$L5D=jL) zk8obob&cUGfZ9RoN(b^--Dchie?LUM-3#*nZqL~ha~5})WOu5p3fh37)@3ljZ0i@P z?+B8%XZOfH2{PxNw~S$r%3mbmPwh?t4mXdew-%;m9&IOSegl7QM6dO?Z~rt6MZ~h- zsLfbJFLFexklo}9gDx(q%D(KkS+20xgvmvwL>|5>YG1Kj-M{8*AK*cP} z12@O8#I5KRnrtn8sX%j9yGAm|hu@Yk`^3XAHuJDA0@cwxlc6XdP1UXtLLV^TUle%r zPb~)pO`)2^ylWvWWU8$Mq)LnptN zfu$^2ACC8E)))KmS6<;hBF_aj{SfLP0!RJ$Ch=8raYizRtBVpoo^j7YDc{A;$Xi2m zvpr19YJ+T?#_DJ&5oj^Zy*cZ(A%u(^Q)W~STzoc;FRyR3n7n+x9v?dH>SOW4cIM#{ zd6S*IAZeYN|0TM1lyymd<|7O9?Mgd;Pxjiwy(PV((YmJPaltm%mu-I*iiK{Q)UMAt zSD^C;+Y+Je3mZdf>>((D@2-{Gq-*14#H|ujSpu?`E~2n+@ZZab@s;5G2e|$1n=fGt zn|Bu5T+g;vF&?eJs_~?zodMWjDy=B^Sf!!MAb2pHt{fA4pyCc(Hjl2l(pCv+tax&KM((JZY^QHYGx%^j_xf~YRkI+>0MZN zQxDFOYc$0VKXvf(G3Uv|8hHPbV**iI!R~pF>@~WIbPU1$V$4{aqJwF1;4%-iVn35t zR!tjPvLOSFy}Si4A+Fd^h&lpEu9KcJX&IHK5W!_bB{C|Eon(8s>$UdvtcUfI9!KTp z2-$~XuawqX=Vg!U5c@0S1ru6(96&iEojvD3LT+DpL=I;Ur8J6DVwaun`$cCaky>s| zW6!VIO9`!wd9%urwvW~ChUd@B$iWl;{Z2gJDTQCJ`=s?fpy>sZ^t{K|7K1y)lOzGN@G zEc{oXuR}(~T9z%I0+wdCkMXb{?VS4)xhk|6UhUji@Gl^jrE#q5(~d@P;Ol6-wl{7Y z3IJP2eBU1mCJs9FRTw{g=n1Q)3^g$WN92#q+qModGeg#9(uj` z3IBNZ`#t(W(`{F3D)`6B+`ARs+pF2rp?gG6Ifi~3_@I{`$9JoJ^c?rpQ#nY<&V#au z)fIhpFG4#o>X1#Y?otZ#cEiPVnufnY-ck}iX&l-K7tWoP^N5jeedTUXK%Mc=XBLS5 zzAwNkYg-$hdqIBo^y}cY+591@0V~WLZWdF>uWFyqQh;p9-XTD8Lat5HqxCe#wS)QO z)=rmTBHrbs<#r-d-a=k~F09!8X0Qf!2hV>~LdNAaytX0-;$l?bGj0wQB-qf7ni+ha zE%~vZJhH3?YoFO17!OUtzYlrXAv*~rQ2$r#5lEfldx}!FG8)TudOfgoA=N2{tg%Eu zHkw0n4Sp=|8zhLUWFMT@mDTGBrQTJ!Fnn{_swn28Q7H8ukX;;;V&Pf@o$obVDZv=_ zzgl_2#GhN^oEGu4ml>~w%woRZ?hh#_pT9S|L$(mv4^!Kq)!uDR<(G3h%=3GXf7DjX zu}*BiI``pKYn zmCqP8it-VD@AokT()2X4qZNWp+728mG~!S#l_kN;?7L#;^fY%{E^6=^4*m*qAB9Tp z@?n>v9yH46Exf4z0b+_bs==&u3Fp&cNUgKCsm}u(X7pb2fnf71?+Ake*X?GrHaDFc z$0zGCtHxgG8mz4@nYJ02=)!%9#yaVZ%uZ^t9Prc1h)5ad!P2&@#(v~dOdES|xWx3> z&z;njoB-V@5mrkPW=V0w^R9HXgK`HuVkRcrMs$bxiB5^FF=BN$dPmhw25e^f9<<$j z3zIMIiRNP)l>@v8vLei$W_b3$3Z@KlI3w|Cs6vlN@z6(~PQ!K!gcvJZPfMbccjzFa;qPXrOPCT}$Q z8V-TkSh3NGJ%^NcsAjw2v)dBOdGthvWg*Ivd#%`%CL@<}X!Y?*4sunyP}T|mt_6Rj zn>vn#;uMed3D13))$y0j92+TQF1eM;9%JX|W(*31#mQ?b86WbJLGyK@j6Zeko_-@h zvK;);WqApSBa+#9R`RdNsY6oOCqQakxHx+(m{SAk2?)U9?-JWO^j31-k?(`8;h31w z#Ls=pCbPgxt}{Dy*NDn+!|(?3&f(p5;CT7xdxt3+p68k_;uB@nwB~O;6oa8Cxp2P5 zZ=lp%+V$m5>+Mnh47&bgnZiSYIVjy=yyt5JME&Ak2(G8Dxme6-OzDZ-uzF_mJYjEB zKP)2DDcsRDHSRz=A*)XXO&AZ|fJ9O1*EMg+{5 zt}D6Gxq~!s0^gamoAqq_dsITmxW~k?1LT9z$aUQQj@|5$J^^mREgwO%N74ESvL(sb zux$%>IJ6XTC}bMNZk3Iu?G(yKA4Z@&*9)~hIinSJn6rRdRj&)*loCY^W>3xi(eehC z^$fJ-%m2CArK>0Rpvm1IR`Q{J)Ghx#KRdS352u^NOAIMRL~94(ii0nIZww?C>Z&&6 zZrPqv-s>h4zsR_K%fDpGb5)t*lVKF|?ccce-1ww~I-kKGNz&3EKU+g?f3$F4BIx$$ zOG`q|c7R@7!?DZH$dYu;TN|}Z@z-$l3KQrL{z?%-d^MNP`D>PQ$*h^WJis1f;sZCE z_e|Hz{xJP3B<89IUIlRdaz`bT|8b}2L&`BkTH^sv?JC^q+!Lo3;9yVNf2LZQSI)7* zfB)HRdW_g!iJN}|e)*k=8f*WT&o%ifDAa(stvS`k!=SR>Aw>o zED&xmm+&24hA8BhZT0kG?MMZj-0g4hmYeV=F@*%S)94xbd3&O;6UYHs`N~fQ^kaKP zyIo<^1Mn#K#%1xL+2w=Cy}f%sg=%xHsU5&_kY1Qfz}tKvKrJ}jh` z*Xr4qJ>NY3C?pjz__q0F>q;KTQ6uaIRjSmMc2mJx%*hgdTlLXSHjt5gfJ%Kqj=Vj0 zs;5p{_w0o{EW{;lC`B=LRt zmtY}AU3?1p{+tCFVM+PcG)JuCAsm0%+?{Pu>1P*?NCBR}qXUc}B|&0Jwbr;ZOZHx9 zomT+pi4^T}S+&kWh5c;+X{(C+V_Xm6Vzf7(vW zmv+vg@_(q4dgjofR5MOx#0J_~zCI!E5(7 zChe=`hUxqJ|GgBtMg^ZO-Za0qvw!eF=Xd)n;6v|(&6HEtDZ5c8i{lGs60*JefhVGN zO%f9BxQ|qJ)zL&AwRXFYBo?mR00T2iK4XC0h3H?@y!R~Lz$aJA&oHd#ZUKiHP8z*R z5{RO={lYTl?)`^vxmU?aQY9|ta?X2)@|e_9`W&f||^rt3?mLim1Id!C1UNDbu! z5wu~KcoE|H)p5jcg9$Cvr;_|ygZT$>bQCP{GW$u5hwrbW+Jrb&k{6(7ZHACnHqxaM z;|~6awRZRey(F41ciWlJZ>ID%gwDTi5Z((74wD1V3+%HQ)=UE;`%JYpQuRH~rc*`S2igYEaci zCH@L^vdt?MhT;l_#j5WJP6yjWwSGMoYVzMp^&Tb-E+jzz%K>}~sJK{tizFHa!X=$P z7mQ#1>ZcCE3wF?DOGyE227j8i9-s6=Qg_=(5b>I8pDG|+a}JQpP#aFr^4H)BFC?oi zM3`kqDzJKU>CwUrwFIuT6TUjB)+DSnAhAWEW)iP15Jte%biQgFE?jE`ZR@;qER7)q z$TSk~uy5HMDiCrIsP`DY=?6=Fpi zF!e`dOA4R-q;lql=3QA3rn(=Fxa(q^<7JaQ_UgbXVWk?U(7U1;Bi+qRG+Mg~8Iy1R zbKh3Ar{YYHxWs}O~J3j5Z94NQy=7mx{4afR#jsGWL&7XFH#l ze6yb5770~uOS5go+aM<{Z00KH3WP_tSDab=gnJ8!A}S&`vyX6%N!e6b+8!@X-;c4OSN0nX(ep@R7OE`VQM)5j)dF4ESkyd{NmFSZF9*1Jsl$JGU9lnXTL)oc#4 zNfFAi40Dv!x11;kaEdw`&#;f0mOFkYlN2YJbrysDj2Pji80t~REObmYQS?ovNbNvg zJU(-)Dt;sS<_&$NVq?1`Klt79HFtj2zA}ln;NI!r1gt+`_Cr z*B)~aP!YVEV!g|$Fg@JZCsdg3NiXC6Fe;GyrXU1axus-_0%luRxWw!e$amsw#blr} z*e~!WU70yDr+wPezvB}{GxHU*O7cslc%}tcOYE0!`UeGK-}ij2Pr0IN*&NqZF5?Cn zmTzqS^V>Fo0m!@�Lug@ySAX#MBI3mu49B)ZxT~e7&b2D!;g1g+IX1i03#Aa;Y}q z%587=Aw4+ND?wJTqqkpnSnW+U9m5-gq3sKxpLh0a^7cSaT|gwq?p zz*Q?;`3OHDu4M`coigsqvzGNgW17CEyF3?2c~m9s;85^K6uUfYSV-&N7xZ|5Z+wE* z*E=Y^Z+o3GGUMVsbcZXUz<~8&mrqjmh}9bvZrz+%+LlsRcdK3wMl)hlB4Z2lY=dry!wGl zPV@IiYKoG{oqGo3E0`XkH3`ZsLo?h}j+D#+t>H-EcAY|*=(>A2%o$co{}T9q(7%Qz zH;geDKgF7xY?hGDF9J)$Y7p~Ae!q$VpyTDn+jDKge>=q#*CrB-=j##*q_qi-`Wl%_ zGEb5Y)YS>Ta2ZDW+?jDdqqar^&jmBQ!e%Z}mWQEz^C@kgA~a6gEIW4l4>LLPH6YJA za73tgVgD*0(6PwE9i;0WJ>IvQiJs%_Ii0G>J5L1Gi!0bC?S4BuXQ7+E#rM~ozxa~z+4C|wWfQfe7iaQR44beK5nb!adN%M6{K=jek=z&>dOl8Gw z*4_1d-W#T;Qrclo7P!WjRVBvU65J%2{{cLsJ>uurp#i#zjn1QyyBychSlJ!)!_<&i z3tfu%+6CG7dvoU>lPkTFBiBOOW8JM89`|ElpMMezNHAM;BMFJ{=UsYkopWW@Gn4zzWMZI6A*=L z_DrDjJ_hT3Eewg_YdPymLL8Pn*bFQGR6+O!5Dt4qZ8J-!U7UNFl2NB=Pn7mL30ME1 ze4{bfFK%{P5{ba|wG_}2;3T7xl|0%X|EC^?eHMf~O-7#FT#;DfggD=-KS64-3r(N= zU@b2NW`~`;y2ejQQ{LG{)4nS{3dRdn`^juZ;$vgTEj=*4c3A-7NJ2fYRbg0a&+rQz zM}uHoZ#0=N^unXb&sP1@?gnD)heVd2dnF{L9w>8_OL>(7yAW7{I!JbQQ&=k|oU@8b zIvi#e>NA6h;}&BLYOjT)xdUJI2X*68$jM7x%^#*atZz*(_3Nu18SWSNY;7{8^CS=c zzW8mxcJ%Q3N*V{;lx)s=J_E@LH1X+f$U^XNpe#O(v4Arz&jYGC_r4vV+B-8Y)h&fd zF}B{NrWp!G%eB-MqQ~yZyR0|QM2KChtMdJ_qt141^g6<>L%x{2gp;PUn>@jRq|m{# zt2;2Udv2b38taKM6Kjg3Fi5&&(UYXK1Z<5|Z+<{aTNW3Nt z$}{glaFp@FSKGgaJ|H%q8PBC{s6Sj@|MI$@d^i2zSZJuEb`B(sI@-?HsZ0*U*pEA- z9l;TJ#~f3A!uZ}jB199)>-&1L#lY~8k#32< zb+R?=DnfzZRd3Lo{lwVTI2M@efR4`TdcpmUq?8q*i*)DNaEulnU_yBYs(tWCJJ{l|=QmRA8WGKHeuHwAqg+Rca|9qA(Y!LQmHqs6dH z*uZ_o4cAZBR+GdpTAoKodDm-DeJ@9Wl^ezVE?1ijh(4dM`PS&}&8Xd zFd8~w@JGPEnf^%GY1g(K0fZCV>ztH^^@fDjqz#K~4J#|`7cPIx?B5^qi&jlIiA~ko zYzA`boLe6xAY<9U!%hP&4AE-->uEZ}m z1GIcdY(gFM1ary{R`A(dyNroLCI)ptnJqa;9KkVKkH!@P4Na#3Cfp@za-9CRL?ClaL{PI2iIy%@sS1D#ComF!zzDGVmHL5k0yAs9V8DOxOe_Zm)=4HRDLwh zP$uX{tpG!3E*@KVnh;Jo9kiI%0FL4T_S{-nl^AJ+NGwaE9a0?m7KoBV3D8{au%&pa z0n@yXh=d-W!`dEp>PGv`6cet=ag6O^Ce&L#_i07!SKgC9w{DvKg1fw!p#6q>v!N&d z&P*-Kf~5E_#cuC!_DimWHJ0~wN-MMh2=#qY58OS@ivP+if34TwxJS$X2)mFc^J}V< zN{uv)Z4KlLHj&Xjh-B*#9Lw3~RgevdM$N3K_P-cPk9DHF-p}Nwg90GF2$Qt2ErQbf z?S7GZ&gxVG?MB~HKhr9g>8{3Syt;W)LVoUKltiNiSKDhnVM>)1f3>^N^d2Jm$@!?7 zx5h$0?uC2Sus8vBOOTWtLesjlYw!Br;1S3ZI5+FgKPQ0ox0t-f z>B);&c-Obi#_~pd9VOMxoyB@fT$L9dPHFkxSZ$540UO%55^|b}U-cX1zc7mznV=ZFbdc;@dZY;Ha zDT}QgB``0Y3VzrXZkpMC%Y;iWU;dY*vA=5_waTmQ^5bTKEK9>4(~**&vmBda7;d0a zH4TZLSu}l%B6Z)OrgEd2^;5wG4`RGbdb(6P26@I_1PHjX24Jxr&Ff|~UrkORf1{si ztz9m%JUI5sg8>6Z^tNoY-YBq)mId!|^JAGkl%Rj#x-W9Bp53^Ty7SWSmyDNDim<*N;eVXU*79w3 zc~aQoxIh>*Uxa(nQF+Z4J%JwDsH2rvhj%P4ht-mT`ySKUWp_?|@vkxfa6S`*;MM+;kO1H<907*=4M*W`^L59nYKm9(Q zy~4b^LJ9a^JpxcR&-S#`1OwSEuJuKEK|fZddYn@%=5?-cUd!<@%WI%1&T#J(9{3+w zS~N3Em!8$Ky!Kl5SsEmLtL*Zq306b3Toe+|Uey2$xiCEAH}bQWMO(85<4039<0fzy z5X9__wP>%D9jI5zoha4-GF8qGKL?_+uc~vBl@X=3{+ya0;T?_FW@8>Cgz+`_CN1&j z0KH_c;wSVLl)ouTr_J5On=Chq;O9AyeWWZXe9L9_UUII4D{{b$qoLW(=NKxC6w?YvBeeR>w6J^pY>|xGnSIhO5A-1A=8$h zMeQltWN^*baTo)c#Zt6H(~`F5)nUQ~+0#=zH1SdsW~RMOD;(7_#;Ztnz;OFkbLJ!) z6={?g!fU53I}T1M-`Znf29str#ba2!DRQ`+kTN4?DZ69rNi`t1QN>@P?@_!~Tr{Sl zCwGl!I>|$&3Z=_qu)A#lMfzLdnW}LnSRtzZ>*)}pQBj3)%3A&{xwsXr3C@T~(_5~z z05%vKheS6AnnNmrqTiW-N3>?0vb9$FggGOPx~s}U1ZLgxNOtJ+MwFy^YrswLAa9tL z5hWjzzZF->zv`lqs=Ne~>yZY(Lymdl`J56QOEJW^P&1E5)e37HIkiv<54-Hl64=<= z@T%unouhOymPt-Xr zm!@SHM|0mVwl!YgcP#YvI6IQ0_E4CSl@SsGOupz$p!d&d&Pgy+!Sz5r4bNy<# z!yPo#qpmxi5W{8s?gg6y|1*#&IP8{zifjlqn{*`F&u|+@nr69UVNbjNaoN8=$L3*@ z4GQ;XD{%WIE4<6NBZt2H7#c2;P5ZhRb{cKU9XLF|6e3wJA7lP9G!AD5_(#!u#Jvyd z(@&36rk?WXN&KsKHpuZh#m4ZRuHI$CkC>#;w_peO5ji(1XR< z_CF}yG(Q+s(8hzS3czAKZm%6*vvnIR<04_FhRZb&k0dFL@mt@15QVw;bk?ZOPM^=^ zrUj7iy9woF^F^6*RC8aQuSX^Ss#XES-a8Gj zFLF}K8=d}yVjyZqDE&s3?`*BQ9JROM1U$Bd?)a>1O!swxIE!EUS3*Iqo7dzj%;Xqy za`q7|C!Ev`$|lOtP{c4Tve2^>5+G7m*+IEjy^P(SRW+C_ge+*AVu^}v|Jj!V2K^@| z0euzuCR!y1$KJgmJ->7zX8(S_mPF~qw$ZbY@atZLSak9 zHjJ98uE2PRBE+d&?LU#d>13spfg9B8!&x>tkmw~-=%O<8{Dw>nrfx8!k%|(b{PJ(U z1V3_%*^{V$v0R6EawNYFTeFes7hAee3?P&yBXdgF0aCOu!9)S>X)xYc`0~J&S|FM_ zrVBAEg#ST>1ep*2FS+FR8ya!gp<{38f>%9z6F|Afp) zy(RPFZ@S?Fs^wP#V&%tuDg>jsQ2#2aB}O`g_uog-+}`)fmNAH1E~y;@oK z2zWR?B*yKF#rnz&;kG{MEi9CUbE@C23gIKI)Ct}Dd3sm5&#U=Up(ALlsLiz8rl03f z|5q$O*7ra*Ve6dXkX@FzblAe^ggNMsraH$v@6tc2+LFMc@GA??vZT>-e2f7db=jtE zahV#>q7GVyn^{Aj) zexNHE2l~*y@U{)Vx%s~+8F|WPoZgP+Dgy2RsYLBa`M#~+$^AAHb=MG41X=9M{cHjL zrRud&S@j_mS->yrFseB!_&=qcg`dCHxMr&SRf0Qir|wZ+BTb|4uNrIZsmpK`B2Q}q zSKTgEG*umbu0Q-Z5C8qrxD|QMSv7qo0|;oV{z@G1t{#W04O>doCJVlSU=o3=@1Y0h zZ2lp*znv*@NdT0Z4Pq)*!br>_v6V{BN`of3H2~T~Ew^WXfi8bP%G&Z{cW2RJ8AbxbjUFdJeaw>g*3hyPYjorsXZ zz69*omoxcSh1ltB70af%?qTwUN(m-JYx$~`$XcxFsJd3B_zGP_Mw*@^^Dx>>?zY!d zXou2c%g5GHohDCI{jg;kbA0&Aa&H5M!vmiX1Jp2e+~4qt)#&$E76XLCFn-m}$mz_T zR@rPyh{pvIWwT|rF6$KddQBG{ELu|EsZVrDu^6IB32g1`y@0NYPUmXVe}Cmw{2#Ee z!jZFlEv(S;=`Trb6Ry$--sd^wFNkGRn}WE-_do;qLGWvWKRL(UKG&Y>u+G*<2hIIi z(wmKD@#sK*Tn}mxuC+pUJS>cot&$5pxmi*Z^0QnlN0T4=2)Mc|4>1M1s7?bR{mc(y zjmZO07qzl;h1>yK;xSrcCCeL3@a-LAasQS48+*4Z$he-VBsv*+V9 zt&5EgdW?@zrp@7D=W{|p%KviJGN2A1=6l}xWZa1bV`2G=9c~DXtdpFOnwNxjNJ`Z# zr*l;~N~1FM0od}Vc}I#WXg#5#?#r?Evy-z?eZF1BOxIo4i5px(NAZTCi*Qt`J9}JN zi9K$Oy_RmgoUt5rUo*BN>r7QMW^PqyHx$kJj|F68T%e^n4%x{Ml);(z6qE~`mZ_$o zV%xF0#*7CPegYHwh?DYT4y2UiG^nG>t2?ZB@HHheSZGSg-s2$-g0> zVlNXWQmdlcK???PE4#_9sS2 z?Y-YL&`8!He2?KVyJuUGI*Rm4z;U!=*bIUR^H~fv^oWl5ed#E30=LIlX`v+vp}B!{ zJpQ|=&=dX)8Bx_$r&)-8IzqK%J;iV@iN7WBbRX|B<&LMDs!*VXQaXX1r?vQppr>Xg z-Djouak&{}oyWh%E+^{?uxb~|NdpCht8{+sTRB~N4+nn-zG^U2d1UQ>O3DpiVWKr* zNleq14Zh|IdG8;^wv}n*-OsnM6D*0}sA6TB| z&ii-kEA8UndUb(NhO(17#5}xjQZT7kchdEOk0?3!Ko<7< zSBOELT9Q=AgT7k`NBeONE!~=`jxE}`-#SwV*j$svj0uvyRK|+fcCn~$il!z(J+LBE zVc}s^*1sHDk)y>e zL-l-(V?BZ;*FIFB;xp<5bWKd>+JcCOa=Pa94D1%zoH1iZBjQ1~SJWN<;~dDh}5jRF3i4p)27 z-=+)3jM|H?gLHt?xIIfq=a${*Lg8iBsyva$4LpDkZHIP>|< z%t|gl(5`Iu>wt7)&lf;T&miJ7c4iy!Xk*XS$A)L?Y~|sj(6wtg*<5H^^Nx|tkkGTw zHOP*A8aqT(5h^6_hjtj$T^V_$dEB>XQ|cV8#~CWA<++YOVw#iS&2qe zcB03AM-7Wv^7IoW;Nwv}v$3)hQ@h-j`ddTl=S?XB{LJrDGBw6*SJ)Ld!pZDrQrrA} zKCIt?QZjDseGQ>eu_60N401G_b+j*)MQB(2GqY#>UQvrt>^6ER>8-=CSip9D(yI!R zd6Te~o*+|-TLj@(;t-3^L5-^}55ITxVhz2Ksu2co6y0bgV(*>!#;?9|GS<3SujFu9)DWVUvJbARC&}NzN|ptkHfS-M-Dd6 zwYinbZBmRk2Fkc{$pYY!*<;&myfUFvGqL(YeD(HlhaAjZ_KP~li|27vV~A zEOExqn?_X6p$bzN%4YJnqp{{Becb5W1q4dN0rb5U95REE?Ql5Kn=XE`U@cEwm=Eh9 zSv>tTyQ1#9A-Y&=N&%I-F*>l(0~%fpLBftS`LZb2*Tx&h77u=!un)qnv&7Rl@6rNv z=Ln{A@BgPCG4O7wU9A4RpXu75@3Lxyzq-?16TqVFqA~|->5dvHLQmXQPv)dY#2O)*(+v#y$GQ#2bG$u>(F{hkldi?Q zW4&o2aTL5{6$n~%Ub*0RjUG`{p%VnffrK!hpBCsgGw3aLvZz!%{MyZr7Z~T67?Xuj z-bx6%#lc7Ro#vB)qC&R9iZGo&EjQ#(#=&F_kk5npEy*xdF+6hm`&ojMI(NybISKYJ z)aC=EntDQ?EIwh4bee5J4VftF4xcSj_XE~W-_swu$LTbIa}ez9+@tFA0PNDiGlBNXVh`B;r@*6~@xJ z+qlItxyqA6h2f6CH|`ac00x>~c9Ip`@8%e3v;aE0JI*V;crLb4!lF9S1v$H6xZ_cG z!_B{*bYgepLaKU_{^n=0fMt(^hR{+<2b?t7c%@5x|lr*2@8h@3KX zLgQn_>}m-jrEBWjD1QuJg zC1EK2YUR8(-Of7@Xg{wpYzOr^-#y08Sso1+l7uqk43$Ywa4x{;ouAW z>#f~${Fa6yh&GBq=#T4@8RyjWJ|)Lps-9{3ru{DD@R15;t+ecv;7EJSZC$I6@~CIfYOPMD@ey4SV4BKG@=7iBKp zxcv=t-b8hx!g)K+6!%|2u|A&o7Oo7Oe?@nQ?C1np4)zBg%{T0aQVxk<*iRzX@H=f| z=idgWBS6`$b*zTOw*NdF7WBge)>K7mc@teO#hZ@Z#@qNC;WIK9yS?l!e6zyWU z&^eGI*4L&J(wAiWB)XTNXd}}j0XzKnydt|Q^<6?>$7dqv zMsYQZIkgFLbCS7ga2eSPWh-`afzwr@8aS!jH7%b*pI~bE(^{eozjaBh+K+Qs7SQeeG`Piu1n>YiTK z7o2}KYa{UdaRtuw@_pN)oiEGZeu=TY(?iDrpD)kzPnpdBjokW^;Aq^)=dpZB0`R>E zR&?w>ZXTiU=0Sbget}`ki=5c0qm`@zNT1Hs#5IR|Sa_y}8gKR2!0j9eVx=HX=em*j z@*r}#7Ne@;pULMMc1*M7)vsUjP)bjTeocY#WNnJ5`3fQQ?=*-{f7fr#hy|Tyd=H8Bb_K2x=sV>jXQaQ*|B&&f#yt!MJWq> zis_n{#EGXUV7eCm-PSkNg>#=fHu(a{wB2y9$W$o3L z2{Y<3e=5Fw0o}n}2Dc-`Yz7LdP|Md@}Qjq*PfMw&Np>xXb?wS7BI# zpm7t_Iuz@AiZkN&1Bg+hre}?DwEFMyy?+kvN7)C*IE?zP<{XY)&D*vyr%>akq`c!h z2-h`EVn04}hg>=le4>UoV#QVtlfM75t?AABl?f*X=jHN_i4v>L^x$Fdwj@(N6oTN@ zoZ0~xj>f@`_q@bC3-*k~e3|#H5v3PykB`*&vTvR6b+q)EwM&gW!sU;=R2S(_z@GPE^nf?h>U#pBk=HY9K3MuDo|Ua zb8E;h=nJ|7j?@qC1nCGqMI`4S^}}aE5iNqDK}N5}?G9y20L<)`z6!esDMDTs2!#j1 z@!Op_n>fRYEh}`uMcOT}s(nF@%CCQ1B`L2K^_9HsLRWuiMD%nlWz2cx*R4Nj^zh1; zf{AILg_g<{U4|&8egO`tD?1hjG|&ZTCS&k_rb?pfeAXrR`k&C{&uolv$mL|s-=0!9 zb&rZ0G3HF`8{iGq$brEfzp5&8Am^CpjsRKoQ0x6p9+oivOfGQy-P~;z%MK5V#^Q`K zAU#-3IwS?99DDMA=v=3(3+9w5Uelh{)w-HcuAfa?;g)AQqR?w*W7;tL->{Ds=|_}@ z1BP`}%D1hNS4rW#iXC)HqActi^8tl#JTJig@EvSL_ff|1Zld+L{VKap|Dlc9tk=k4 z^Zz01J>%Kz|FD0fq^hK?)~FF3s;Z^98bSnB)!L=44m-6jtHg>CtF5YCyGFQbR%_JW zlqN=uqC)JI7>Sh#Lb&t0@Bg#^qw|$V=cAn8&-Zg2$NONVbp9vzeKAivrCZE3={n5OV0D z92lX%L*%}|vN`q^<`n!)sBLggs+F#ix1X+XI6IcomM?G0pMSPXB2$G)4hkC$8@G)p z;9LDLIz9qcb>$AO@`-Y1%WJ*)KGO*NXX^UyLzxGHhcm*D)w<^R?$Q#A3_C(}%G@_K+K^PO*^zOWi92)+iDj5qfB8bZ0AB^w&NRCLxv`H+9w z?qtE}8PECiGZs4!Yd~*9rTRqmXa4c(Yz#RsEUNRT@S8XtfdA(^eB9%{i&!!$J$Aab zWog?psQ%1-P`GyOBiAmctUIPByDv&V4G6i|Fdch3%jY8C7^`Xf3dXn3)K`{+?m_a; zU&U#N)JJU zPZJ_>wdpnUT+&B#v)Q*R2c?Z|;FabYgNe1s&|84+uq-oo@14B&ZP00Dy^1K9=kv2z zdE+$xR`wf`>)%5DmiD$n8ok^kqR^h{+NW3c43+SgX_A^4&)0g?WVhNu@T5|kLkRti^@h6Wt}LoH$g8^YVZF%X z)qp2PZ(XN`Yd(|wOvMvDN2Gtd>Lc6t(?uD(KcKsdglF6SWuIcq!Y$)|LpOG<5h>K= z24fI?w;Z^xDXtYH!|A3J@>nbTIQcw6jBXtGD}7E5Ai0~M1T2HZT?y_QfUX$ZC{u>w zT_qe(;(Bd(t$FM}Mvli|J^uj2uGRV~7=U^?mwj2uqQi<+-t2PcP(bm)Uptd|unyxN zxfQ}Wky$w-O*N!#HnEwV;!@=HR1~jE$}km=cT|JYc*lCK1P_>F3OB`!YpVdrya7Im z+~)(R1w=@>#~ee+m{6#}bk-&C$7Y(U$&ezZTk6^!dv-YmQmjEYTqkZ_VFkP5Tl{`3 zI&2UrSN6@7Z8q}+!(1wc$6VrwiulSigmK-fx;6FP zKaZBhq@jf8fqmu^F3nD9fnL~Rx!|YHB+kN!6zkrjzpyt1_j!N+B(WdD)7MO;@#nJX zW^Ux|32BK_7?GdFTnuqD+#m33Dv$b)aSxCW`c<$~>&Sf%5r9qoeF$%~6J5iw#$bp@iFOmKX zAAYsp-mmp{x;D9?-yY7O@59D7t!L!uKGC(euDp#6_<@z=T2EfI`nqB+zSBBWV}~1c zj=+R;HOJ+xCzx7dNB~^#2!FzEp6=eY{oz6{O-lyp9@|PhD&4Ki2Ip*+&#(Dh+%12` zQzM`7rTQ69i`-i|T1)xWz>BR0CX?Zgt!u($@`y2Pyhs5Czm^lj^JY3|h4TvtGaz=o z=uSez>_n!C#;U~=z{(S4)FJhSa9kaP5aWph+^lLnL-c+J&+ZG_c9!lwD^M2u&r9>tCP+z>T zroDX1E$01tiOcon${+H18jKcHG2kN)k~x&aQmG6upY1OcyBM%BtJx{W4e#i}9+Od zVYLS1KpTuBnkld%?fbKP#qmD!urMZgrOJKk^@S!xDP_8m4Qb%sEkKRuW0|i-$o1|B z%GF5Ybk9}mU|Y+~R0ipV+GrkC_kOlCTET1s-qE0@8^yI=z zz2GMtBy1MvTDP69#%fvs#?5j%ISJE&R75k|fq1rJcfyeS{aGF#_Xyun{D*D(HfkbV zU&}? z>Me1rgq7ds0_Wc?yUpd>khs);z(dG?^xD*BP&ssMCI0hv1IrD(eZI}hRI-orLKj)y zgj9E~A5N59ybkl;eQdGz#{DBmgtpS>*=QRXj-ql5ry|2>RXqLs?wz>Ub`;|gd7-b_ zWMF$iI{3Ua!sh!(+OeO)mczFDa=dcZjLkz_=AUm{#6O}vOqc$@&-F>%1l^RQ37ulO zeU{u`d70;(?`; z1nx#Ip&^+}guCKq8hJUO0F5}Mn1+)<$X0V%V8b&v{ajFz%IDeh=00b>Gron7@7*xF zu?dmncioHa`A=bIzN7EnHb^EQiQSxO+xe<9n}n$GgeIF%j-%X&dbG}m8CCze1xa?4 z|Ek=0FiQ)Sz0uzJDE?cl$Jam@1Q-MNlYoFMIMqp94QVmzq|?Qj2)<#*U-aa*H=McU zv(CK5ygHc1zA$i+`k`Enbax`2#`9*kjdQynf6?b=ln>EPD8j2r?5=}{*J*O5C{t?4 zcvgfnc*Kez2|Bo(Qqiw~^SmBo%OP&pM!Q@gnuL`xo`%yD9ea(%6R;w)$(+>`0vBX- zleR335Q{r+G+7OiC*;q1hQI4^Aa zob3HGE~inT+uk));qGzs7=)iBiGRf)*rsv9!^5WnHk7Oux|@gM#0e|i*e^a(Mr&`4!0qzG7VC+>A5OIM*tdMn2`jr zQh`6tIOVm!*?e1EZzly**eJwqAgi$Dk_CIWY*Du{{8(72;3C6V++L>A*){Zq3ihDl zuP@9sxMb%PH9{b~<>M^b$8nkR9W2dFK5(#YAa^-kGm6pq{=C zl)R=cm{V0-6Yn*r>PQJ_4WDMYX-ub+jRUVPZ>hlx?@+^=A0ZFjBASHX?S!Hzj4v){ zX%Js~MQ)!F0O@v{zh8N;nh#D$g+gpbuG+T9%McINFYmNmbKF z2(j+s9$)4%iHwth3$ENQ26r_n2R}G$)WPWkA^6J&3qiSZ70Xa^=*hI$^{x4!P?-?ttZpeT zpBd>{d$;FI_nzmCz>2@$%yRYr1fC(5z7f$25jmXtyZ+@~e*~KqkSD5!59zkm_bWVd z8kEM_Dk&>ml%=5dP_OJ>7P^5|bMU1-?!eDu9%|!Q)<&}xW965$(O#io%ACgDqNXSz z;(kkv7ILRsesYI*zL34@qhi-v&)hv9o!$o+P8<)7kc~Var%ckaT$V08l9B}U_Y7DO z&)jseEH<{-PYrK8KJ_!Fv?;wZ?Kt_*a|U31{wV+A_um#ts~Rj&^V|WSg>eJeOKe}^ zp=KsYlTm4X5EBoZl&%QdgraGDfBIZ5XbKMO-j-H0%Cjyme}{@fVgQ92Zc9^!nN9nn zo(3iGfl=Ff`DYrgZk+ zSKHm-HaSAFg<3N|CvBT=I=-rIPN*n}4B~%!j%nietnW>m;2Wc0b(PSdL?(dR*^uJW z8RQ_*lXq$YPzaUeUh<=7>5?9vA@UK3M_vO?vJ_ivgWw1%9FEOa<4d+^l{V30 zJYEAXN(CV`y1R6rdHPxv|?z2Z>(9P*Y*0iv#L!?-+%2E%^^iz*9IZso@kzxrC!AIPX9|5Th$oamTOz zhX=q_tyTo&>oD-sPy-^dt3(vT%E9851oK*GwmIn`CyM%*kFcka>!(8$;;eI9%_)9s zjfv%lQ;20mTk~G8J=@Q&pjptBtl2~J)%DWDQ|+(_=7*3U95iNxB8GWqCU95Yq#V{8 zhg`OPBj+%%LWQCs_kEQ;qNcZ95$fl(860UrCgN|tr>rneN!?tbZEL5Mz$&aECs#+V9W(cBS?pPg0p;cNKCB{F# za)G%L3c9#&q>WBJWm{9@AirJw-LEEp@xU%o)FKZ!2Y6Bv;db~wGh-DrG{7CFn5b}B zw7nMe;3~ZBZfi^j-z9M2Np%s+JVDMJo|+4JXXZW-r&5B`SL#cbw7&u2CsR`%FAmdG*J7h! z49*`rhjG@)ymLykQyjCj%hAl0!#-9gC6It>C5q6?qf%V+mycv$;e1j|)626QX+3uy zcvxM_I>f1K0u2Ifdf}VqA-e`+Bz^g8j|8a%<a}%ZL;Os3d@E_+=Xui^cr%HIa+PXhpAlox$mgX;~ncr1PtlAU%)uwN|GR# zaN=EpKPQwGu;oWF(e;T^*qOKI_cM@vi>`pR!dTk+J?7`*cZ=_Nb{+VguqX!Gh&BDZ ztgoWOnuLqTeRY{EQb`g_V1?K(7y2Ds?C2WGj0}_ro2C7b%T30OD_VubetdKYZTxXUVw0-4l!CVln+2Cb+CFBC;Y>&`l-o+TM!{l&g zRA8B<)fT1ZIu^l$(S$5=#%NGp)O8uh`ABe*!heR?DiUMy&^9&T5PdKxy`8SeFHkWoZ}3^FaD)pCe_Aty zVHz1Uki~$xw-%oE;R;rAkg!5ExT5k`%BPnBvK`&xU`jC z2Pz&|{qf+*@3X;=E*&1-R|^q$ct7;EJU$ELF7@_t$jPwDQbr|I?xlp2l6)?xz3U-h z+E6)xB>a{#UmODYv7*{8RLD^89scV7>SXsY`?9=C`7Y<$V9f`Qu{yj==xrB4#;dQe zxuEP#^OSigEg+M%Deqk{KeMLv+}1iS5gg}?mn$%l4i@Hg3-Y|Y7Vm4ttKvcR(RPxi z9OSk$Z$WrLN7&y}L?ofS`Lxt`Ie_|!jk?AgO7@buYkwyF4X_&Faz2 zlJcf19?9c9NNyIhZq48%w6z(`ftWb>4zYDFG)m?1F3{xv-1h3Ys4)oIB>#*y6XBM% z$z;4<>Tn(wF+)hg?aqF#yswq*^APpN(={Bm`wnR*hEP`$41U8;Hpjzl_)SqH<$3>@ zD|q%thnj)=Z~2q+1`!j&D{%J~3Ccpm#KW^o{F5wHOFIxUxQlzWz(^xCAj9L#1NSur zRrxhF9;B{cD5$@40~@XGYJB%O`Nk%X5Ym6NGCz8C0>+)?EJ;?A*nW%G2wqj6szP5N@vo zye~}F##%Y}<8%PH4A;`mUF&$ex)TM$jz5Ry>ik*#s&xAITSgW7mVB)7-G@PBuQM@P z{%2c!ZwQ0BJj%4^4w6*|M5s)RXwgWf2P`@aa`MQs&W31s(%k^#q7NZ9SSL%8!hO7` z8%;)oy6jn7s$JryKw&4GYHcIbyr9iAS}5xr6*2&lM``2aEwAt%nHxb8!VdHv`(+KXMg#YaNUC8ZYnB@rkM%ocyfp zk-XGLbsxBQK$Cz{`Kmxpyu}(B>DZ+qNYA~iTM#^Xpls_^_>YFNo0d%@5}LY}iIt~a zopjGE^@!qjuY7cD<+tk18%k?0!-JMqv>)2UE8xb9rn}s;y6w_nq{mLO)c--iu8qnC zci(kiRwDk|WnY{812k#d;2c#`iFA?pe`UA_m8@4_I;WBcBnk)U3j4C}hlh6oC$leO^&_dVSI z+g1Y!-^VCNtlYPSE_H32ogp~lq9zy2B9C!@tJA=~4Hf;qdK3uCKi73nl%-(uT>uhh zejw+stfAi8*tRw-jVmWgp^i-dp8A*CsWB7db?u9C8_LEIw2S)sP5hD|gBwwsk`>7` zdE0OJZD$^$+>kQQiBVpIhJ2kVzP6su1-eM>C;$nw-2UwiM)eY2hP;HHz%rpARqO8& z)sHjjs&*y@h&r7I(L|3WocPRGqukFpeo zZL8IHYLrGA#gLo$C$dL##)fTbZZ7QVSRBa4Uz<~b!03!}T8|X{gu4QIwA2XjkyJ+^P&$^dOjvUHPxCo)p+DUjdrY&{kXjrRBkwFs zX;ZUdirhA!Aqj((!_O@^g-4)OUL33T;;wHVGh4!@d7!^b&T(8T=iqVXsQN<>qR#89 z@qYAo$7Xni0ncyc%?c&~e6@TNV#*Xc`*MbSSlqgH2;)>OT$-r??4AdHi@4+Ui-89O zH?$U>eCMUUu`o4Jt}{dXt3)C+9zbCh360@Lnr>H;To7gV7+XoK3la$i6DwDMD`WqG z*WQ`T=2h8^@nBRum9G>9{KWi5llF{he%6Wj+jOz!nO$^Pbya(3>e27WBb`TfH*V-H zo@eNtL|Ni&l=)Am!&OLTa|j{bT1uP%5l26M?>OdPlXM+cj60_UNW&yiMibvRQSm-L zlP-FU|FZ8rn(Ey$Q3M*dS=6!ywrd|y`#&(_0eZ8$G%O|$WmqOSoQKv6wk_y4Dw=v(LD@Y77W#zt)^G5Tf5@o^*&zgpKwMAd!H#3k^8IPB#h@)j>qz959qY|{-l4Ov9V7@i zFgOPI-a6TWCBb*U#y%?7~SVJRk@xJ!O@} zb64v$F@Lp$ZgVdG?Qv0S7lN$Kt{&K@uDf{zj7g--jCAM2B=G7V7W-B*;o9{o6< zi9KCGiN7yqzOQ4xHljX?O}axf9Lv-)9EqL-Qa_39D6K;aQ64f&Ao+HUEbUvCXeBK` zKrha}I1}GDwTe1uFdxk-_q)^T`q}LGOZV3W&kYta*1~F{zgOt|0QMu&$jGvb|kWryu^+*v2MtL zC~xaz3}IMtiviK4T>$Ru!Nl^@gVAgBY;`y$?jkWkF)!}ek&Mhj6^SQQst(rLVqNOM zDV;ZRgBfVahjuy}!;tLUXuKclbh!Mbo%V}+XWs)&%owVh5WQ@?=S;kapkmAH?H!wH zpfxXj)nlwRJQB@?f7w>?i#d??X=bylZ12}J@a?qC?2vVHG5z=#>vYE-CS2XFJZY;; zyNMYEVN+@cG+b1zwR_N0VMs5D-|j!gF+sgN43cUtO{{`13Bp*(Qy@Qn|<04I)wP?pS?a#Syw95f7r`{`cBB!}V zsVdCPgq{8QSEaWp!NA;fFhuwK_U_%q$ym<7&)Xyd>wwBTu31-Yo&Q&ScFTGz0U&`i zPE&wK44Rl0>_&H~Zj!2v4vavfSA0np!$!Efhwm4kTIK3CSwDCNP&;ZL=#xjUa|Ba1 zcZjOzM*kFf3`pgQ|Eo-iN@lU|M$&3+?n8p8gnq_3oB|0{Q2$2PCK|TbpT1fZ0>FNT ze*s@;QW^T|_iHgb&aDYXu{S6~TUx9n$rs=8R^M}iqrBXGVHUvs9>^u)XJLoi-H)|2 z^4(ApHPk^!PrORUG%Nh)QHFUW6O_HYTeFFG+u__V?Yvhlx<_E>pXeMj;ij@(hTwRARur*@>d^2L5DQ{#dS|)=Yh^Nmp#GZ(&HBrOT+a zsrflxN`|PnWs(x?%{0C!&gadxaYnAjGMBF(2>_GJ1`#3HFpFVu1(kONE60rCJwr*4 zhyg;z&W`%B7?Bu@{V6oFs?_&4e|d{UnQ`3`sKKt@j}cw{EFpxqjW{b-N?(#PF?wR381m8aQ;I?nv*F&c)>nCd8qO7d z2>Mnq9x`@1V1!3u=Jt+k5FdF?RAZ1|6qes1`gG>ej;0Rd+K1M}nza%#XDuviA<}&2 z1ySffW`Zc4Q&uO$@Kmkw!YeYMuumf2NHo_e73xt|84Q%xtZ6(zy%8X$r zyEYLSc=21@gx9rP^afI#*60*1REf^av_fsUXc~IPVa%A_ph(i{l zn+A^t=eB}Xxapk1H~fYG%>9gt5YxbY67-#w&_|avB^vI@FW-`QGNRdwUg{_};)jte zCT&-L4JKWq+>C~yXn<8E06{AiZpDeTw2yykh+P*fzqb+TY67nilTzQ(DVJajkQO;-9~M~n>ZR45V~DH^1aULcy0-;J@0$EPiA#W*Y{ zM|F+SKK_knp4%Nf2tTrxU5*@koKiLlD2UXU3EKg8o<+8hPMD5&}q#gu^t==juURUzWQFgj6n5Ez0f;B*9H%Won{;CDFPVP$*@B#Y9)K|C#BKh z$M!PE*jLhp8&KVse$y38nLcDOl0}n4pj)b(;rOWB3%giLl(ZLg=@*p4>mM!9g8@j( z15)Vvm=_koW6Y)2V5%w>tHg3P1f7k)vRHs=mUi<0$lgYV7>7o%)kL=aR z_9dg&7<9hIU~ulq691ZX<#G#*@-Lj|N~qxz-*bm3izrQLWn6C@x3Th%C7HV&M(cGw zS6%uXc!nA9bSa0UJUFFwOxi)>+qeIsOz-)jD~x*fds;S^dpISzvFN$_;HPc>n7`!! zDdzDJvUw3YVj==hBdLPtbWPPTX_ZbR+mMHES_ayyBjXfl=VgBmDX5`}b87P`YM*fN z0fYj%kCO_a8Q4Zmow)&(KjcV!pN zjI7&zmxdmTXo-S9H(1dMK;AFYP|mn*T8q6^-M6+)adXjfY62j0I*{?o@N~aV7XDH1 zvFT`lX5-#pd+lVWx@p;2`R8Z1P>Fz=qrQOszp%hF8;^J9?)e^CUH|S&zf9W;e0M2A z!SihF59-AEr>_(D^pX`f@x%iqM3x(lZ%Kv3rIPkIk$z#EeIM=OP0$>pzC2mWx424? zO*_@gM!9eI8}yGa%ZyZkC5~y~c57$Z=-EgCX*yH$(o- zZbt2J-LZ38_k5>k$ybSjZg2)T`FwAFh|gSE|F)?^=F<4~ghI5~4Ll1Zf8M_R4JCqmS?9fIp)(^eyGpO;%h`n)TsiA}~ zu{Hs(Y;e=BJI!@a{in|qTHG$vUntQz^V$39r$4i(GeEU6_R?5I zE82u1B3M4bg!}_}am+8sXf$blmti({ib6>hXD6w_}y?2(;Sk*cEff=eyY%IO6Dlp~dQqU|ZUEP6~%n zqhSxgtc!UG(6z4>nzQ&guw7VdjT7!U6G)Z7W-o^ykp zX{x>^6A1#du5O-551T;WR<2dLDBU1ykrT#^@ivT=Zy~bY30tMm<9_M`6<|wADl9hQP`rpFe$~ zGoi3IL+HiQPLy`HwkDP{zv9iSR&OMnCn%zxe^5ErwniKXljQvNk3;da+~73d%$C*9K+Yd%L#(Ut6$T^&z8t@ki?cn@^nK z7e`Vu^;)9WT_1!*mG$@s4S zSH-HfX9UsR!dz}88kDzZW3`ZXv49j~wYByb76%O<)kx@&Q_QH-ABFuRVbkW5;3->r zbV_xf{v}_1y(Ich?pcGOP1cFnTNs=doL+fhq}e?`XWQXWs%@;Q`P3p?iG0i^*g;~a zxl3^Sh7*4FU9E!#g@C8iuyCJ*r0nuriSycLisKfIr)ep*VLp0{lzR@3(RcY15NDkn;}AI$LR*@|?e!Y_ zS@nGHq~XI+yV|{+YTD~}C#kn}HVyUo#rJMa$g_6Xhu0)dFLyZay=tMS-nLO`*$a2G z887t(ovD)lgUvHppa4Sh=O5qOl6)x+K4ec)E!ucy!Q^VZ8mO{xe^zBYX|J{UOT5CL zwl|GN4aE{~%KZHsPsIN1E}aFAoYPNXCeSB*2Co3o73J>->ZrV2FqOSSPU_FiMweux zjq{T#{e3y7M&u|~W)_Ihnx0pP=|oyln>@p}@f5 z^{>`9;oDX56o3I7_(g#oeqj~RGA3_HY~vr?szvRm)_)T52{0t65@sMyr6Is%3C}qsBuw)W>Qe@5i^A zp*tpSWu=#Y9q?W$*o;uo&*w=Pnd$h8FOERPH}(GYW|=W(gS=JMZ>tt{EjO}oo-!#) zSk2k?N@*^i-S0(r?4vrMMQ*tj-L&k!v6x+K)^0#u5K!8MJANCeURff`5)0 z5d^D6lVAx={|vH;x7&;upgb7_!%m1b_TT@Na*D8`r7Cil`w%w(uM#%F=rl1}=E!e+@h3OkSJ<1_;~ z44O2y2$jJ$6@Kn5K^@B1W`?3|u@>aQ&lIL}*kODYX@zE$M;p8^RYhQEQFQ1RD)t)s z9&TRxM0Q4oegXV!K)of;hzGW8zBryD)t;5Y?(2X}u88f)XYPWK2o~-VnA`*=b-@zW z-*AX{r*N&^_m0)haMnLb@bxj3bNiJ&mE5mz-`;t!Mpn8SV9IZIZIP~0VhGn~X1gt4 zm{7**AJV=yXDKMS4L0y4`_0yLigL_y|-&9}uBv`wTG3x*9beJ|vH^S@%1 zmSd(wrQr(PRMM9ZCDCVnND_=3JrSE$r;$QmS_+C{7B{St=hYMCY6g65x28mgg1OfQ z$|VI6|A}Rxeh$`F7s<~nc+27X$g?cB4tR+ip=4wyBqple`%QNAef4^uMS48)JG8tC zTFUl9dFkm;TVDzT$**b@Y%+-V@Y$8GesAsE79hgu1Q`ny-(%ig(!=6|&scr6-xP3t zy+ikm*Z(;f@-#gED}QFNe1%?_En{Wx)bHaIm$QBMGwKy)cnFBw?b}JBn178B%@B_y zrJ|*2c{D`6cl8>J7^$?!roQxm{zO@B*~ z|M{D?LomqagP-pmZ0Kt}(YV-EI6L8J5K$hq=O-V=Qrx#zNSSF`an}Zgwrk}5{$G{Q z>!3N=|FQW#krryu$Ezcy8oOre#d8nCVxyEN$8UdXNMv*UXPit1Ymd+Ob9%r^j4LcA zuII^)7I{M)Gelp6S{^^(^VUwW`6|Wb<%Kyb4D13TL!)Hk#latwyKaWji};!D!=_dS zNUv_;etY^?eXF@@PPv0exK{n0ID3p_j-&K?j0VY*Fgek7imd8EcrdFhW`kL{0YZ*B zyYFO=G;CvGsyNON!Kg^Wd*#pdTGwCIy&%RZXJGlh-VZVP;wIH6j|(n1LRAv>Yvl^s zeNT35QkdNLypfB&<;W=+Q=!&B(|H=VZwI1Yoc_35FC*d0=Hbn}`+fFBIXI-_)oZ$2 zBGbN&}~%`)xmZan~iPeXtw+ya#Zx0qI)+w zcb&RzX`%K~-{v^uj#o%IV?!{zF9$mu#^}|E{tIlncsK}ucJ}DY*`GG`?soylG8E}& zJJLSr=MwKhZ!|mlM~W^96yX0t%AnUtM;z}Fr^S_QnhA*{gviMD^-y zO(-RZd9M4L604{Me+kYCYWb48OQmUW12%NG;|y8X&nmOp#bN~D6Th9Z$X;6z-xC$1 ztkne356nmt;q<|L%+XX{F4Q16_xbv4Y4a1IIL~7AaZO~|p^#g_u^aLau{!dOmD1?f zeDPbx8ACab8dlM8lC|Aymeqzmz7lfiTxEGPX`At`Xa z0`6S`F!KKD%)FJdA`2tU_)EvT@@G)r?WG|H>cNsp{uXVV-Pip$$F0Y{vn+Y5k><{?v0W4R`V`&0^S_{M$JG0v14c){TmW}=q<*eOm^jRU^hIvpg;#!bUj}JBtMDE6i@e9|3UViurHKK3 z(&2*&8EK>2Dt(i=F+0w(woMK1OES}aac-$`5vt|V3HX*p9~<_`r5K;9`=6`Yh94<4 z=YfwRP;cwHcS}e&cDe%@brOGr##|!bYG^g54Jc!?5gacv*G`irn%VnG)RgOPuK0sGOKXWo~@60auW@&ipxTL;}33? zxT)Jy@0BwWsmbpU)O{Y$Vp-pHuZ$K69=q_93ggVx2&M(&J{_QbPgxU6T`uqOaRJp1 z*N1{P6EPp>@3s)e0aOA%89EJ= z;CnkUo8vemiMUF4NLcH4cz}@DoKt?1FhC@j{RqZwL)HlD(;9O|>PqCtq z+|BHe$r?amY>D#nEW39!HvAzdm7l6^4XFIhc%*@0%SNoRA*}%!A#q9)^oj)aRLP5@ z5d2gf2=Ys4Jt6&hCY+4mnEV~2VdL6>Nz?@rx2xMikv2zD_~9GAUYo$= z-!LDU7#TSIUIf$#xwbZ*#!L&1f!~v^)5@^4`WTOofewz$$lYLEp0>aJC|n|QT+(>_ zu}GL3(KU`!Q4pWKZ7X#69T6-^BH744o7A^Q4}!SL2ZKSSH}`*DKp*aKt;|r$yJsO9Cps-AIzBO?tm$57WX6qKj1lqAqKq$%E3%*q zkJ}X6eSZ7=KFIfobF*XQZq!4Ch~q_f7)1v61Bp;ASHurn{f6o$7I*ikHfhQ3auB<`neap5ZHwJAES@(KSm!*8Miv@K|}waHI75bOF6}rmBU-NI0JTf zweaG^-smzE?UPTFq>rmq&S!{T42=`L#G;TSk9#F?LI=RPL8j-{Hw^P0C@WX&OG=Cb zwFD=~XO?!Qhp@0^)=Hm|tGdc*5SXEz$EUKq?fAW2_;*3`ADE2q;!rt#1k2CTQMO+V zWA-ynst=$naLm)dt5`M_F#k~r(EnN|qc9fm)TWGU;ql?P-5t%bi=cm~VO3in+~PP9 zRBe^g@pa(QH&*cqgMIaC+l9UQolefuh4+fxfM-%1UY!MU?noI z4+gB;dQ}m4EmJbYO1pweNmc=Nfd}LDhxi$$-s{yTKUDrt#p~LJ(|18ayoI$n7WT@Gv0m-NIzk~c$L0}8f zE~5YY?d2MQ7lUBi951nMnP^~oMZAIn0AiI;8|ryMZL=xYGbuAZnC#8i$9fF#wHt;! zdik?SSh7{gDkw=bsN}0yKAJ89C$Q1<2VCq-0BROO^}fV$$nrQZyig|%I@j?pta?7$ znz`=x>_d}ZXe)~CD{Z{9`B|W+#J6)~-9uZP&y-Wau#CC{e^>*^A$~Mwaps!&964<# z?XlLObwQwmk26(>#V^<%2AtpUny(1^!$l7as*LrVy7T%-wTAw2*)}%A^DCf&RDN3F zIjB|epFPiK`+v1g>|%3ey8@vxv)19#smCd~T*ROLRK=?1XZu3|r`*J}txNk81CYVoG__h+K)2Vi z*%^w-03zWtc!27PD7#JJ40^n{oW|n2rm(P4EPU*&4gbr&W4(fvd#{fV&ef0teglQ? z_i~_~?fhtzGhuX;=Jp2I2aESVTeT3R66Ct0ke+T!>93TDTnSZgh?ft6pqb{q-##%@ zCJWM}evA9A{g7W0fgnB4I?RCf+TN{>rJ}+3zvd1Do78W#W}lp4p^KQ$i@6(>>qD$0 zOl{YI)!M*rGiM{wPl#F;JJk@PDNn^Il*=T5CB*bjVhRSsD)50HdhgM@@=PyTtGRVb zY!6`~S`c!9QII!u&&WFr5PyHv3RIoCoVg zgbFlMg_Y!|;v3Dc_cuS>I`^QJSW&NbeR3@~M5dXnXmI?m=tZVf_D)~g;)6Io^>u-? zvh!UAGQvU+av)p(D>23#;}_zmi^feW2fRxe%NO3?fO*o?efc{(ZkkXO0EaIwIWWvHHgak8G7M062uAMR+by+|pxT|D9wJlm_ptPq&Sv7LadO2qU$U3SqV3ddd!+AS(V1Y%|XvIUhBa&Ftzs$f|Wi!ISQ;opOb0@rkS!w zHx6No{}6{vAhLtH@56HkryN?l%J^Q|-8a6aYcL@1)_p(BgQ4hPzoD%w_F0Tb{;`$QzuUJmeFH;tVx4uF>9^@8{Yk*7rUFHYAn^`&0t|!l~ddZ z71knz44_%A8FSd1NxW_DtGq~0l2_h#9k(p%{X%sX*e|@8%~=ZCrA5uK%t2SR?V@&L z9aTS3N`qL8HIL`!qD$I0P|JY2X4^ZJO5c9?Bx=-o7l5vou+l*`k}(FC7>kDPvt+pl zrj`myO0l#$`$BfD57;l6Y194hO6Bzhf4=o@0p;clPO4P3&apa{h`e59sp4lcPy)b98b*t8SS*T57etj9%Sbq;8YNaoKbK5It6Kt5@?1X;K z6bq94RSxAK_5mg56pEtD_=s&9G>dBV|2nxF9H&xRxBBL2A_I8lW`jtV1i9u(8{vvyEP->&JzBzob{$x77 z8V)=2gDhsE#pW2XG2%tBvd8M@*5*PUPFD@aQ+p)F^6N*BdPiik>#*1A^FW5vA${uo zmlp`av9*^tikrpIs&$|A#Jqi)Rq}j- z#HN+=pg<=ZA!%tI>eI#={STDa6u6fMXUrri+NXWH%XBVwO?d3Fl=@Zykl zr81YVj8nxA&fI(GW%>fE?=XjoZ5$P@Q31KL1@3j19%(GK^d5bH1X1Gzr{Z)ab(eBO z;h{JP-|Hc?T@^r(YI)5lB zM(X9;@YDcWjO@L?Z`$?1IA+2R7tEuKMp4K|ZA5=KkCi@lR7kDULAh_XJWOxzVI)tJ z(#v^5!1YC`U#mtJfBk`(ri9gIQuhm+zC1m~QzV~td~h!UqHZvp9Y-f#ZL8*FQ*QI* zX<_q5H6M$T|$irt}=p83xr+)Gd#Ut9aa z{~b8RBMF-P5ko2lKOa7-;2t!GPT_2hV_$u?IfXmlIRE*@T3=ICt>wBis!vOmG@E2? zpPmEI+bcDj*PkBic(DB!)W_=9k%a{K5Up%-?h;X35-Jh}@U`FQZDmZ3ey+8|Gwaqd zNWmcuN=E(h0XORP)ld`(?`gYf+Mgy;8*Hy)ds2b48Z{r^kgX| zj5w^x%EcX^fCKuzVTq>n9&szxLM3M)y*ZQp?{L6rQa1U#F|alCS#B@x>KOM?;)VSj zqXR3o!j1rs8M+t1HD+A5z?b|;t(dk_TtLt%&r`qcTh&f z%9OY6_pP9UODl10$nE^71Feq`Ir_mXO+Ur`Mv=O%Y@N}-GsvD=+xgU^1**`O4oY0_ zpBCx3{Q0a1wgtOLnntG}0j{P$;!u`$^!^UaE7v&p0WmRe_h?c)9Qm__*781JSqg4v z^vhv{Q0_T0$v*Iq$pY)sa$34{v8rnpo!vGa2d$%U8=uYJKSt3K+K2l=^&I@k>f^RB^OkLAhPEe@3t3Gccr|&UacZQ* z&Cj&}{$N3}1zUJy)k=;-+~Q6+5nmm)S_C+d{bB)wJB!+$X39*OX%SC&`Lpz%Yyunk zt6FWVA;T)G1@fU!nAR^=BAyDGss`*71m4ja@FEGwmgv6loxa~c(~J~XqN?R6Y1@;D zf+B!n=5IhH#Q&~Z>WwE2JpA^HaUX}!z`I>am6|~H$Hq6t{H-e5?RFS=-Aff{|6|>1 zTbR@|b7y|jC3q)5@PjBGKG8w|`edD{^F_Za=wLL(^TCaa{9kAJlLD@rEcTs)o!e&{ zy9~TK|1}TXKN-FJU^)jQzEj92hZ^J|zI8nPtJ43be+=Ru<%1^n>Dm!22!Cc7Ad}EC z*p$+4t(%if>Ump7;TF(IzDKUt`z>ch=Hl)=igQ@%+N)GC$HvYhp*!r`>E~BH#h>>C zXxU70pIp-dt_lO)=^Tt3X^fiV_r8mP!vxu}e$g}4d~05wt8!A6)~?*S@{lvc!mARe zQf#dbxg@=&^~v6t`G13LiKzO^&!je2W9ifMjTSqPXYU z0=Br!#?&CH6?`DCJ(0%)Xbh*wCvV(ceU*2{SHD%mvw8O+`bv;=^Q|}{C)QYxMhK-< zDH(KXTv&Al!UkRk|FphHBx^?1tC)B8F6dgXQ$LLg$Nv=LAr|PmNR&pBV!lC0Y1U6# z-wV^yGSUcX7}gjMOBBbl$m)o9k9Cgz)B9UE4$vQ)cMsY#X>13V-v{=+NM1#- z%yu0oX%T08>2nGB0RO}ora))d8pj36!D&oW@O);kqp_OcCu1@MAO3a_y%9i zRCi$CLF}Afb?D5f7GOy^&PsBqYtM{v84Jhyl7XA?mJj&m*~s)<*4r_h@` zJ8})ODegA}KW*-S?Gz4B8>qpIOpfIb3fuSWDSuks1}2l@RQ3S&FFmN(7;&;Bj<+VE z4fYzF&{^+!xGI@Nme;ITNYgDUsP^@W9C&)^%chp%!&BqfyF|{(3=7Rpee{XQ{Ued* z+#V6DUKS;z^;JpoSzBib)osdH&U@H1^)87lgAb8KuGu{B<7SAUhL4#ASCFN{xXNfkC z?8f|cFYV;Crc>r~_IUq29Rtf%25(CTI%Y!x7F1c6V3Xf7PYH5+eIC3kfMp@|nrM*A z+vVAj5yTcK`M@Ne39V+efMg~?5>biXYC9@i$;A*W54xexO3%y*_$l1ocohCDneCZM zYID@efxz&CrC55sOYT~u^f;2-fjcX{ecdy0+gHD-LG21O#K zo=myk%dd>Q5NO(}yHjDoXy8pB&7Mp}8fZL7ToD?p&ev>=nLI}mshE5d7S)1VZoOEZ ztFofb^48`mM<)iDiQj|_fliaGJyr>kqUnzmlf}P~R67#!poGaR!Nzi>GE#y!r!vxR z%-?hgsp=-8G}h=V!Rsr0T#eRsPv;=Mlt{G5O|NIgVv(w8M5B4ALU%1${N?B%I1N@v z4bU^9o$glNs~9;rXqlvgGc;W7cTR6gZV^^?z1i$}6P55hL#Y0{ zje|oXsK9OE&oO%PTLf**kcs4*2={`qd{S;pD`^NvQ(I)OtL7^}*42|4A+rXnt-HL{ z6Jlw%lqPp#Zs^DJC9?+-9}@9LY?Wcz<^T=3y{55E%(*Y!6XFNR9GlTP8X`egdm z2<4x2)Woh(ytv&=<<(g9$0)Gxqxpsh&RU$kD@*;sxZ1&JUuXJMW|yB|R~I3%og`ee zLE{;pbEEU9cD2R3sU+UmPRmBr<3L+9X{ylPtz@?LbdU_I7f^gwOt7mI;oF&{wLFMe zC7S7Avqd8xPTP`OQD&XY-?zfjf9BIT-2Co~rn#y7RBNgj;NWMT4XS0GH|8b_*xV;$ zji8n;GY^RZgzcM@dtX2J&Fso1-Y_TqbUP=2^ms*)=4_qBw+J%zZPjiKjpaXy8S&E7vy z(t$pbl&D%eP5Sgwcgt7@{v@P7csYpsIj?E!uUO41?(#0!3cl!SC_0#x$-$yXmRaMX zMvaA9_=MniUX8W*L@h|vH~JLLcF9bzz!b|{pQaiWELVH4`HSo`8*|`$U7yWMC47Ai zQ}_doi(Xoj7o{~<&6$(umZ>lLfk6cp%o6Hr-4aY94g`J&zrr*ibA0Be!r%+Ed$ z6$}#DzS7{S?F$v3$a*&AZe!2GNV@|rH_W5F2acAWsF~gCh_2Ht5pgr*Q^L?2aai1 z+_z9{17_mg`^vl52qz}HN8Z4B7(Sy2$cF$J)w>D8MyE_bG^U$IUW_ojUzip^xhP;X zItp^BrRq?eeD9|X-VQ^(i8GSol*R+ApMZ<2=BMiqn<-qVToJvZq|G~9X#BlxXSe{f zS%{5M2yjh{D{3x*n&z{q{#)FU5XVft(%xbVn*R9u=ab_R#^Yc0c^fBL(3hiE=Y}@b zZ5SxXOt-FyFPrbxhAdT`611%?WW8MlzhrW7g$1JtY{h#dSBXLg6>Nflr>>`}RLt!f ze+G}gaoP|Z2+0TTU=C_Z;VGaKl#~wb(e#E%r$FM9i>DXFFf7AhKhIkCid2vl{|UIo z80LmtihI5_x7`*m48B6ybKR{9sss}N>Vf9zy7**G?cbC$Nmz7nFrBkbL~3Ty%CfX` zbo2_jx@Fajpt@!|@ERuqRo*_ic|&jlAu$^23$3PI4sQNW;*yYbS3Vq@c~sB-EDKg? zPLHmat#$W#6{>#xZ2mi&mBd)G*qVlfKK%Xuu4c6_WSl&*p>6IgWGX@w8BFrFPSspX zPwNs+k8{Nubqhqb9P3X7%r6UVJ8uLeFuMX%H8F#zWF{jqx{NGg{$c|Z*~A`poT_sN zy>ah9N#F1jK#{Q{`_8Wn|(Crm))D+3%@P9Ph2& ztc-WgYHCX}Vp1H#nDg=gt9&|4$*lE+hRgg&rduMcN+q+N$A{a?rWuB!S6Lj>iunuX zy~{NVj!)dJQlcA*R4}_BZ>vcs7?!&3zS|8VvmHpKSxya`5sfHpdY3Hg1(#X^-2~>h z=HzPlJ!reKSnl1UUak;x1Df7^v7SX#EBE(#&-=G{78bK=saZM&D77G|%)$63oUH8G zSO;GwQyIQ$(rIZMQfZ0K`Wx(9q_~x+6;$tu8dG&eL2`978ioT*$ebgr)`=XzK|xDa zAjAQivUz%~6t(ssrBTrOi9i;)pmvH}d%o3H#s`lKO3njQ)jj@bUlQWXS~4H9{UNn` z#l3;j_L(DPut^fvCn(5lTytfG!wTJI18>~rE>mGPZB+ejlU1VYioqftvq!XV;Axs~ z`e-_pnd-YdcF39Iv)9YEJIJz3mE<$Ditf7=#6a3OgKGDE3rUl-4Wn#!Nt zL5A;-Abj2%V$Nlqnw0e;gHjzL}A z>=+s`S$KY#Mf(AJ9oOe+((Q~!wcYdzk~bHH4H!1~qrVFC+d7`j3XQ-2cu9QpqwPeu zS#MB5E%Mb3ic~ zx}-uBTr5}RAf>0I*!NY-`Q8maNAl&hO_z%xZ?&N4wO7shae+(UvU3J`e3tcso~eOf zwfIhO-;_A~K@N@DENM^Agor)d}cJrL|v)8A)43j=xQS0BxXxDNX6(!D? z3hgZk+e;1w4H2knV-ah;Xf8k4|Cik`1=O{n zE*z+4hrIle>zx|RlY;jf!RUiaB=nJb?L(Ni;+pC1aiQp6us#fQ-`kyy*?jbG?e+vX zXHsJOoA#_UccJwkl)rkq!x#I}-}usv4F8vhHQhyZGA(?uKd~qphxT_|P4x|nE2H4* zMy2j&6bcvqzE9PjivQ9;#3M~0+f;g~`H{Rew_Mg%l-P4TrM_SmQJB22odWW*R`)#H z+^h$2hMzho9L_g=Oy*p?akmIBDnvDZ+aM1<;-t(FyVZ+wZgR5)QX z=yAB^*<6h89T=)!>FU$Ckp$YbtMixCyQH-hZ5`c*X$7ueGyPIenpXa_xSp zts&Z$U$GxxT;+O&=2sR=_)BXds}>Z3*vzigL4Db#g?S0zL4SnDqyVdDDa-@UOKK+p z*S8C9xHj2UU&{X5la^UN({AYoyc$}jx!Wj5Q^IGu?4rbYC3;_2TqD@%RzsIe9&i2r z*FJu^n&YP`d9m$sb%RX@tl(;A0n2;5!KC#693g51TKZlu4}ZKGvnL|d`OT<}+P?Sq z!wY{i`Lb5@5`J$zvm^XYZ%qy;$myYk?u`TL%HCtWKl{BLd`dTnvl|Z{a$n+z1+@J~ z(0L*^<_?QX$dpzzMmj$pm79ISif|p~7~(G%tiv+Nmn?%D6DBX9l5c`3V;S|HX!3k9 zGzFUHT^pY9TD3}6QG8<|KFj+7HUsYoHnjF63MIy&jJv(BkgwwnXI}6Hd=L&hzUJOf zhVi7VfHNvLx6fw2(D_o$xuFr}coAp={BKd5JojpwA~p*8hVnd+_ZsvT*NqSl$k(_? ze02gIm{0`+>jUYvUkgm@Hx_|6hn`?m5g@@!z~YTa-yR_-$0muAAbGS;Jmoi7O8My> zf)vu4kf_ZTVphbUU=PsC`c#Th$}WuG6}KPYclV$SV(X=@_fLp*csNvtSAHoiPVj$b z4iy*v0*jVBh|(D2>Qh~Cc@_L@q@e5nv?ttfJty8#fWe*arfebXDG&akFQkD@O_>34>fn)M zZjH{t&cWj{Ia+Q@Ymf+dCE>^;f{3W=LF2vt8;) z3(ug2Z0N}AJ?17XxBE^-exl1`B8vvX$-4i7r(h0sheY?tbI_?1c`@UQ3Aa@RO*)UN z#mK1mobavH9qDOyoI`1#BGf6{gdBDvN2Ao4fxkGZdVZjkpW@h=0y&t#1TRBpr;_#l z1jVb4mU<;0Nhz*nID9=9PMH2Pc0)K4$upzwr0X2{U>IK*JHmVygdI{>#}^_}$vbZj z_UI}y(Wse1oIT4!Gz7f^8xSAQ?2BXInxsaQYy(%QG zMiJTIXAmc4ZtP85v`--S^cSGsSu~H^zXQLGuY!Q9_{4LkNqHbod-|X0+sM2bGMj52 z1uzsaO1jG!RPR+(O6;jsFJ&0RiR89&-dI&J1=H}oU*J~|4G<26SV@Z-KYW#k;c>m9 z)ORV8F*9Y{oPLtteV5p$$ynkYQ*-we)g+WA1%+drq~fiG)1(6v1-E!7XTA=6t$|xq zyZ+vJx4&fvHZ5m99d+7#5)hNfBd~!S=-lYm< zD>EdPwwqXj!BJJdx+aS}l?$R&?JZs^CGVJuJz zCt8=B$%cA4o{NS0_t%~Ish+022Bt_*DyeUim(})W?Khwpv2KKL`^f^WHh_ZdJ1|b4Wldo$~XUNvUyZ;afI89kHSWj@U8-)7f3O?w2*W0SgARFV|k` zwA9@RR8fycZLUQ~$3l!d7k&bkW*nXa?8wiNdZ}x@AE58Ue9mvCL35;0xmIiL{5A3+ zxfav)8E|Z%*Y&;VECxHhPjKJJZ%GDD@Dy!n-SJsh3I3yAmz3L}i6Bkj#rImN!jv-gM{+^!qCG`atQRnBl5b(RyL25QH4jx3-!6uRNpKMR$Pq~&peJdUH6jHkl(w)B zL@9Fk_joo&RjJ;Z+YFteuy$EC7T3r4U8fAzsy`h^s00xJBe^BB#S-kU1($ zo&|9=gcVR-)kI%!Xn(5v(Cy4}nQSn`WO{-NH^c*Fy;c8lpZ*)9n+SH!z0JRZKDbRS zigXrT^fO%%z1jG6PE!!LUfk=aC^(UiOCPi4+*}Z2r>Hzhp>qZ<`DA^a^?ujI43eczz?G2o=SA(W1I7^joshw7r3M(aIa{?9@cYBrDRAjnJo$8= zbmth>yXin_!B!=|vvFV3Ol*QKUBK-*w7exL1DtW@t=Vd&l1Qm_TM}=Y+>k70mAkNgF@H%>MkT+Lu;e1Y`sag}!Y)J`Bxx z1^xwdKu{K&ecxLHJIK))LW4)lcgdQPbVO83A7RTToQ<@r6n#qKbV%ncpeECAou_uf z0rzqzzooBdg38=Z2+*0k0;sC7STG48LiibfgbuxKjseOlt!Rewx6L0$>(KLizoTxx zvDDDJW#ijYfax@TU>62$|I*lZL%L7*CLSo=i-fP9aDv$8?q4e4ckZzJiX+NVM|dEM zZ7#|8b1cEFf?s9>HWr%wM)26S{152K+Z}f?8&|ff?F@AL24}A|zxikI8RD$FZza2N zA$iAYLEEir=Ky(0kD>y?{~#$JdWaxwrM-DZyxFsxKfRQInl71W>_kC(8g3l)OF`Uq zT~^uLztd8g%g^I(UpWb)1zmQH=c;@fb7iLouR??Vx(ar9D)r9r%Hs`veDrI&qtL&B5i`pRPf zR5I~Jgvb}&yf3t99rEBFntN;KSFYzgocR_Hr`FAn>{j%CqbP+decQq>-A#^ddm=vs zY?g7_t?kNQ>EVV9h)o&E53z48mqxDc#Z=oxT;?7A$}(;aeF+-K1DG(1$DTAs)AqHO zt@Bv5uhhp; zlJjDgcawjn=s^WlJ8elY0jNrl=lbTZM|+7u;LY7@eLwt0IMaV778z*ViY)i3J@LG-4-}WX$@> zeNJJ&4)QjRU3Q{Jt&+m$2H6O&_gu{ug?GLth>Rg;Uo#n=y1j~{OFWmtUDdL>gk@45 z8(HGHNQ0o$2Tf1(|FraLi^@uC{)(;j7-~J@WXeN^#^}@#F|4V4wjeFgudGL`l>W>b zM;OoAPJV6Del{~lJ{=WBzDXZ}Q9?p6k@ zW0-pjg&CWf`1+{}-RtoRLWa>@AdQZ%0*O+p(}{5br;zcu!-p=WOERM&RM+`Uj{@VNrDuxIyX0LTA6JJuSqI4#K5UhfK$B!Dy7)ZNLT{9VQEApB`ib=Ndb-wp7iseG=w^bK-a%J2 zI`_@+D*66Rf7Ju5%2qn<@f4s#kx?-*nISZzy<{P=ZFyqtvCc*(qw|u*?S&qpxpc0! z{n#IHdK~8OrsVx|1#gm(E~%;ryVu+$^6hqW0L&!l#A?5l*8HTR>cih}6$WBX&KYx% zU6y977C@Ae6HaPOV!ObdZN+0#L*Vbv5p*3XM1||3gnL6VzlNs%YEKGIf{~( zN`ceT3P`^$UlVx0(~;Yt`{nXw7tN4|nyP z&szIaR*8p!lYK=qIAL~lrw4t@3Je(pJ(VdHyCuVeO4g{BPtXniIoR=Zj`>~Z?*=#j zyAM6<^=MxrhnPYZy*QiIlHBA^aDtC`ZQTC5mrXlK4BN1lJ{Ut}yY4)40es4LkDn)` zi66+U_F4vqDJp0Hqb^9DUjC~0x2nYBr>{k8i%1kyd}oHRZ#)F zwZ!}GP}3qF1(d6X;QHs<_*?y)^f!(#mW|`)hTAo8n%%&HtGD?3&ndT?QA7|`}Xo*cRT^t)!^OsSCn-A15&rIpQ_{6M8+mgk;K_a^Hq5?hz6XM@hF|G zcW+g;+|#5`YPZ?td`#@4>c}7a+WuWuAKDuS-qiyo3+`Xy32s!J8Wgy)p=m1aqow;m z^rb_(mO8!1Xfw8M{<>DB;T-9&`77<?!>o1CpvGNhp|BuY5ZcQr8C-_kIg}Pg7mMta*U#H zwdm@Y?q9a4{GpBoAMRE$TUL(!+tKUI7t8zi*dl_iZ>r_iktdr$eW+F^7*|< zb7$?niL8}PsrVmIZ)bC0g#Pq{6jq0Bo15N$N(SLqw%JSgAMx)UkW4`o`aGrGqzIhzbk)dCx?x&Ui=EgOrZ)(WaCh^J)e6X!4CfTCoL^d|O?^ zU;?CgkmWNVn#*Fq;IPHD6La8!cziys+t#m@OH7luueG!^dDH4$`fZPw4ji=6w&Ei( zZz7=~X2m)({Q-l<(@^_K^|kNWB5`$GkMJMP;0Rmb0fWOU&g$AHmOr96akJ0SUO?S% zo4dwKR6nz}7QK0S(hHYepIR_{#A4ojC;*82=Gf@ z?Z2YCcZN-dSgz0ZJK^(jEsQz(sgjw96P~QKU~(#FDiUwxXjL?l)@XY-o3_z7GShcaDuErwvKo=i8~m0bZ#jyAklz4|X_jWul5 zRqYLIFdY9UfrtM4=YLC-*%3994-mySep*XYLJY1(T+CBrA=-bi)8r$Qj}U*O<&*`{_%0!N0|Z$b$+xCf)Q{gYh{H;%E92`H+4;zXB8asHG1CEzyQA_xUMpMW5VW##r`r>_4Ic zG~OtxX$&QRc!}z=%?7yrm7qzGXQx_8v_>E0V%-aF_y#tFYz*h@@hf#Cc-OQmUpt9{(JdS)XK8lsu$T1h4}ih_@D55HmZh zGF77-U4J34`CF`yH43bD*Jpt$WRv7^E|tou!9QpFr%TP6w%NrfcL&_s^u?!HK_^fUWXwo;CkjVv9w`Ko@DSf3 zrzO}(T(GrG>z83h>*e2!KT3OJZ}=eJ)L722T+I}aCO07$f4iXaAAz;-nDWqO3_09^ z^6j8itgU!6X;He_b!%x8G+{nC6l3RXz;M=-REb zHoDq;4GI}2zgS#L8>&^80PWqoaMe=rs7041GghCmeJ!3zdL=p_v z%xh)`L+&5}>au=fxxF3Kerf4`)*&r}x8#9hxC8A}a$&FE%E^TktzEnZn$&{y~e!MlU#o6~F$+`FkV`Lu9pYwnU zRIcdHh@-$e=YYLO_o#_8CdiWzV|AHjVd>3){3h_ju%oLrmzO813;ZJD+wxEETvw?U zH(-29+FKiq5aqyC-%h2!9rSJscgsMda$0CJajui;T3@q)T)HPWWP3PW+O+FM+K`0ZSNh7;1A za5^DQKG$*|{_Lry9>GFmX(X7Z1mm--X=Go`{vdqS8y~zJK<9NY(6~uX3Ek{b=d5GX z!v1NpO=foFcDyIeasK`=iEsLdzt$gai-}Ru^My7@zwHi&gosH(tndON+o!9vcG(&V9A&&uiW4$-}S10 zl#Dru0m4((#omgFRriIR$*8U8FB=X0VhUT3q~LC+BdoK;tR7C^=vUhRC$w7QA|{eR z5jX*7ch})sQ7UvAIK4QLADE7OBnm_Q=gGk5pa1nGi6XyoZuQn0-p2{|Ct7+Mv-opF zqG~s~#&y}-Tb^tCg_q^u1pfHb$Wv2~^IycE-i)IL0BNkSxACFGewZ zrbL=s)P_1IdMPZfg$!$C@~q|bXe%`+r7Ye~0gIL*eX2+E_;FJu)T@0>)}si}wDudM ztbyoNoD(FEsAz7IdmHseMD0H(@f2p) zCUrr-@llA*zoS*Z@9@&)_WBOblO|bsxB1A&%kaF#?Po^=BJRgsHnkge2F(8+i2pi! zV05C13>jMD6&OV&7p?UzW?R2e-R2H#iwfRCA=m>M{|;kTDc-N>v}it>@{`|}?$6n- z zF>3Qafjd;`EQXDR5?Ab!iOHRPpF3aAqey*FJKaPWUHWDgE6S&@mwR6A_5(E?zn3!O zLQHFPkt!&kMluF}Q=OJ4dt5v4&@+x4t6d@#Yc$`&uJC&3>CP4zF5M9?Y@_Fd z^n;EJ7=0iOTp*DfD>a@u>GVtKoL(X>H)92j_#Ntoy$PqJ2`q?piRj(*r|-rMb*~#K zo(c}~GY#j+(KDK^)WjWnJ@xn4_V{mW>Ayf~jO=*s^uFb3#GZ4mT6f##8uyKhq9%&I zFPSqx1DPJp%VxBrvu(fL0nI-``!2%M_&~j1-|ZKm)P}Uj(x|p3rXRoYJwZJ9reXSO zR{P=2Amsk%@i015Fj`;Gn=NnppR~xJ_Gro=$gRDEdMLU55zzeaH~U(F-i&Q!`e|*9PM6rk=)OWm;q0b;=Uq3> z{cVRh74q4fyQ|ivkt11}G((JvCEShG#6b&BdZ>ASrv$Uo6R@hVXsx6%{WA+&%B+;? zv`4S_2ht%N&i(?&j4^;3LFwqzo3MyCVyCs3y;CUM1QGCbnsu6#1elIC13m3gR=$ zt@U2TqwqLU-pi*vTvWxVU!J>WG|3#3@nBocMVsK}kob6&yC9eTZ!yC3onxXPeI=3w zmjwXp-HacL^_rDSjQ%#OUmu8ld0mp3#_@|S68U&*E{@v~guFr5&@~;aRBeI%Q+=SF zLO67P4rJQ^`&aQ}wVDT~#bPv4*&J~{8sA2%M(~f9pp`>N?BDBiUW~Bq58^Qnzy}5B z`toG?`Jza}-$&;b^3O{gJRNY=yJ=GgENjxWb*nz?;-()pGbXr5B&E2S3UB@6oN-O_ zU!B`WzO}p1I^^bPhcV;d%VXlpsdj5sX}Z@&P!Rv3NK!dNJegGS-+Td-Y&%oYjW}D3 zbRllGpODrZO6456SG*?Pas%8kW1}2ABQ*_|CTYHQZgqMqByOx@OiI!g+MlQrx1M?h z^Hp}*A&1HxpiW%^;=@Pw+wk2WxED(;usKenNn06FePm18jCF!SKz(_9zVkAA@Td-c zd_Z+V(Z;MTz(Hw$(M#gxUsNjyL%6n&n6D*}%~pPP-bjQAoRxy+V!Vc=e31rrg~)~M z&4U+ll*eIt)BD_n7rw>*ZpxjbQd`R|F+14{PBRfnu9a-!tLuidqryFIjKP-1w~@qL zjaaQ^DN29uy6~m6fCn$6xK@AuqI|x_L^S=?wZ+vhYVHtihRJwJ&~@3TjjxB<3E#7f z!~2FP&eig3CdgVf_$lV_kvKa&n3DEhBXh;Ry~vkH|9~RjKatBS?{f-kbaEVAcrQG* zugthtz2h6c56?wm_o|&Tqed6%};D+c~1~34Yz(b~ALs^D0h25zIlijCus`Hj7BjNKQ{dA5;G@Cqc z-+6_i>-6@!oLNrCDfh>8*spNgh<07n8`*r~sP((&^DbEPdh2>XIW%(nQEFU+r5TWF zbI<{8_<|nw=01Q{E986~v96sr=^me2DtCLoTvxK?Ft}Z{#ud!4D$76j;?;W&=+31H(wB8+%aUT-v>geg96=^NQCGP*8Y5=nK> z|Ccj~*2nFC#H2(?kcoU%NIjoq45Lgn zvIy28rZ8l<-^5o#q@4)da^2NfZm{K#P}%M1i3T={g6oHAo~r+dswKz-#qJ^CnP04!6G$o*e=Fr008TTKSiJ900o<5m-4? zRO@udBV3; zoux=IHrl|V+t%TqMm|JM3VlB?TmRpMX7T@bU{P`An2UfWjlvn9*ud}O28dvy@BI&ANZ2q8q~C0Vdw^m zUUu+jpq^vMg80Q;gJ~U7HEsSt;^4k@KbcSK#mws@Rr3pdY9aek&5>r1{bHI~x$pk{ zG|}(cw1FhoA8M&4T$`=#yn!U)8WHcjD?w4SM?p6uBsqGxJ41(Krqj7I;lGuTBETcy zoKBv^g?+a6b6bw1q;H=z5nmo69%KC7goYmt@GQk}SLZ0{vrrUlIT~4K^Y#sRTJ~m! zrPtOl*i$E4DpiOM=vS~#`xfojX{2R6u+s&^J-}9i(24L?Z=SA3%D`TYy-9N?U$u8j zHuqAlhKsqK)<-=DIb|apO3;>F6!>{_!J)DHB%(hSptRR@=ilXbK&Mk%4-q1RyuBz5 zXX;{1EqUgq$N$>witvW}mVT>5{ysq6@{eHoQo+>TfF9{hfF*DnnC~98RB< zJ=~P~H>#%?+==IwgqOk!i^RiF`XNioBnZFrw}2=w3;W*NpfwI!(m6M=cpnqs2$W6^ zm8I=!J6o(T472ak3)u=K0;HdDAZ=PDMzc#J=vqs(RC3~`m!UI7byk}@3S=JMn!UVz z$;8pIaFmvy8`#gOxvGJ`b8IbJOXHv=LKE|ECH7F2@D$)JCsc(9b0LYaTu^cH^;k%62~;lFI|g!SHOB?%Qe})0gwqPZGU`h~uK~^bzo!xJ5R#45 zn)&6pJ2pUP-8`E@u(24_X-$wOqSDig4*z_A1&yv;`=l~v^yYmqgX6;Zz74uUkNn?| zaFj2!D#&jDfs7fCLoh!+#rhaizvrJvUxXU7RE&<1ecrhJu--!x18p%u-u*aY(P-S` zq@2+8Cy}>jjf)G+P8%+Czik#`9uvYve>+aJR&U4z?4rJSrX`77xtx??SBreYW17eF zLv?J*O8=VD6_ql+R~hBcyzEL%pFX^BW9`=RP<0SCi|Fdc_gC`IjWfs(mnOVg)onTM zLmr-4-o&k&)ps35>^|C#=lYk(WIe!77_Osw9^+OY)%X?#4+a{hz;q_EFPLWiqEUK49MCyU4QG*vv3Rk(IDU`{eiFrFb^6lw7(0Mk_m zD_)vm>Ue(x8~@clfil_s*;EKDkqHI=iLv|$WGDsRAh-Xia((?GQOQN3N;YNpb{B{T z>B%qWh*?NG3|JFlYFa`@RTtD%k1K>1u5rnab2rxH+ODYc--L95=^Q?nFvj8gEgDP? z+?7OG_*60gSARm&O48q4a4BZj7N%D^FPL@sm^R=KF8#(bXJ3=SR7LG#aDbgb8C+pt zq8?7(Ja57(n|G`K!$%Y>041g+QJ`pAfNpuEN_DgUre0vF3&H9_qucj2`4YV{IF>@4 zpk;%PBl40z!n|@Eb(yD^GGA&CnrpbZ#GZNq{bY0y65qylP=(X5IMQ@=G16<$r(ZZ<8c_u=FI_iA%+S z$xq!x5&xPmVN_RCMtg70k1kul^pD*!6**}l>lNWCrg^@cC~{Px^=+ z^th4v@(1G3snf1Q0MXX*2`3|ord2?NR7=JGjxAlZ|f511>cGuby~FlL9Tu& zaqkttP0?^EAC^l2j5L20*T~6c@cg=0&F9l<0Y#RM&{Athpj_(tiBkCYlm~8|JBfeN z+v`*H6&DSNj(KFl2XBzjQ!HAK8%D zi02i6@$wj*rG#MIwfYQS4W)`bD!#g|t79mkiwk398JUw{-6Wdx8gv@z3sHQ6F;Wmm z#&4IGt}?c0{})r|9?ygy|9ea=iB&4CVM<8l{ta{4Tq+fze2c!W8A&Xc=c|&{iGv1B8Is}~ z<>~Y*X?J@GoL0uGRE^e2Oi>xdO}@M~+5CU(%KuPkevE~x3BsJh+A?+BJ8 zDBoI)XbA8!h3?1p?G?$rSSQ}D|9+9mU?skK#{D+#R$-2RnQuRZZ9@&^Zb3>Z>*!#erzQ)IE=+X1gSB*KnRV{>(pQCnQR=Q*#7AQHDW@h6(>V=I`x--A?4BBOaCtau?Zbv-z(kj zC*XaF;FeV#gdvv5`u=%xZ)sF+a!s1s7e8ihE9sTtYK*}g-Z2(jKA(PK5UqKT>GWgj zU{YN24Tx9v@QplidVqRZaNLGisi8@zP9{2#{jJ6AM3Nv;+=Ks(@tjzKA}CFz#Zfry&C5RAMDC6#wLqv z<8vCubebQOuU-EQy!q<2cKgtqInL7AsC33yD>1%RYVR0JHBMq|Y8f){kd!`;jc)X7 z*PeV0`L*X-4$YpJ?F7x(Oy#?`)Vq1fr!MZDuy6lbiw95?J)a|-r_z)BnjTBSCe1}? z`ddkN`1NFWk#(!9KIPC|+4#Rji`NN82@W=x%9DY-vO6O>4Fr0a?VFm4NjJ^~Bltga z^%29grOp2?W$qq*{x1D|;uZG4Gqd-9$nLhvL9-QVwYNa4gcN4=8$Gm%&dlFKeu+b$ z)69qoNAj?Y2(68EjyZJb7!}R$z?p`R7U@aO{WXOqUuqe~o#1Q*f1nG_gJj=l$Rc)2 zWb#_`Gg=R-&u$KCkv`CY9dh%jE;4ea`iTMP^Y{6lSlTj+ z?h~)wq~R-nzp~Ti+byph{u{G>%+&7Ncx+$BS?`Hhk#iRhsQ*$zKcs$NB!K?giUIVI zd|s!3M7Gr?PAf9>AqDD(Y0__HPc{F+d0R75uMOL}Qz_eP8Z z;h~2PjUd}Y4P5rHJraZRook6^gtYoumzD3TC1W;=@fg(Ft3SOy5j#~%KZM)eyhXq( zYiH@wQZ27w<2*pa+IYxjrfJY+(atR>?9_-;IrMv~bIY&Eb^Fb|&(-hrmP@xw7mnSf ze-9pUySCCtw6D3WyRE8G8U=UxE7*plbmeocHad+6(nFG-TYDeha=W2uxiLN3Ro?ZH zh|F1CN;Td864i)-_m6biouL_~B8-`c5cvbMzzTESp&P?1 z_a1*IW<`r|?jx^i@mqkR*knzAw(3zkj2K{Wwwf}gD6#!~`R#=R6$Hz+a(LBjT^W)b z|3Wt)M}3xNm<786G=vTlEf}pq)u2a?ie;H$7k*LopD3E)=s;q!uaX(FI7GjZ;XC<4 zw{sf<)#D>SzLh5Zu*>V#P7US-?9M}VpyWTH(*HDf=%;4;J|u0iV?FrZ9ts2{WgXn2 zU9&5@of4W?&{Q5r=UAN`G#bcPrvH#`39KCztAqM7n8n^cvkviNVeDKf6 zF~lD1r2Uw`d8VI&L-3%251Gl&UQ4`E=`1b{j}$L7+Z?Hw?&BBihD32#cImMkV~3yjkYy>plpA78U_a68aE9_VZu@*P${4)_yvzX6P& z8ZPJO@%?U&p4Rq!4J_Pg-2U$V{TKDem(Hm~xP(R5MT4gC1T@&Q!wV&Wzl-&$jy4d{#fJ}fb(K|MNR zJ#>hPfrBO3I4G{hChU2+hl$Qh*!86K!JjABrbKss=_t8=}bCxip~ z!-)kl8q+@)U!sH_yqb6-O~yr{d>GlQ9s8>YX0_)1Ibh-2xC}+#r+PI8yPgz~75}xM zs^GZm|Dq|32oCI^uVB3^Ri};zJ(Oh696UGkBE9?k4Vaf%5^COk1_0KA(}v>x9vCyp zJr*>4fQ3;==UHmL+4zQBp^4VZ=CdXib0DmicW+Myq~ZMffN4}1GP>3GOX^-6_>1Q^ zx8wC=bAQ{#iZgV(TJIfs>KvzyGV+&xzs|41tBL_7RI96xCJ&h|^l2}pRfg#RUWT_m zTx516Yc+`IRY}&5jDec?i)Xs)Oq%z*d@|6sXq{2z%`x_~BMlQJ8+Z}duFRD}s|^0R z!mXp_O#T}y%?0gCM}$eSBmHRE%R%SqcN0OXuitv}zt>_o`dk;8kF}7rK_{~0yr>wy zzh+iU^LHuwLXx{fd9zBT9H-99*!%KEZEA8e$kNlamnfz_2P0VS9+mUnUXA8)0=LUP z*`6bL%QM;MpqJUZk_oJ`Exuxmcs;ThsqRusOZsd)w*7Eo$1u8nO#$dvW)5xsk#~zU zu;%jl2AMIVGvQEADwB}DA&sHh;j&kB_L}&Pq@q%$zD0WKNV(W!IWA6O_PRrkR26ko z$U|G^&|~*zr56nh({b0XK5qlmTQ6%BrNlH?w310?5O%3e`=8%cX=}HbRLm#oTwy>&FNL04vV43AWx{%g>T*xJ@qoK2=Aefz0oyD zEPfmOhT`BWx$pTXuWJjVgd+isFP~_OXzUw8qz1HSN$?{4pjq#1y&C!pfkP`hG2%W3 z1_T%+M67$bca^GbZXD>niMUC#Wz|3OC}=zq@35e^sUd?hyF&;L|Erz00TbN(q80kchmIAPFJA*Au`y=!!YC12s z1v+>^2AMY|LUT+CJ+8;Sf#cr#*-Z47oOxXwW~HkGEj-vUZ!dK9D#`kMDJrUBIOOX& z^#FdVCg<$jZ~!c*O*y6QIE>lyCF3#o2W4b!>cNN%I?13K6pe#KqSzE3DkQ~o=)LRs zJB_gM>Hjuaz{AUO#YHGgR$2)5PyUd@y;tS*%LYVYuWtW$OFg_{BY*gtEK`smZsAyg6&pm$;8G_~&_`YrQCbc>ZbZbZUWKQ$M;8>l)&cOw{YkQ}0{utNSsZYW~ z^d~uUs^Z6}nd{R!_vz|X50jE-r_ls?+|akdNDsxf#*nW>ZM6cAl?p9f&1-KNWQhEV zPSI#8yaJ(Abz4bsj(R04pdarmxFj?)nB%Z80B5SHCejCIvD%(-jPpZf&8c_tP`g8m zUpMOVjp7wIib1y+14B>g5*u%&0KG5utHYx_hO`VJ{^%#)toYml!w_ppwU|Z~tK)c3 zt;GZ4@^)rIsA?`xl>Rm7A#|Wt(@Ap!*(f6_ZnoRoJkTUrdJG+F9CFvyL3$21tNU;K z-pS;92OA!FNYGyJEy}$wStZI#Um1?B;TkBP*aRuNF4_m-5qyJMsbJS(w@CFw2}he( zCwsju3O{?7&~Bw|Trv8FrL{qut3;C1!XlcbB!w?w4ILmqI~God^do1=ia<24+6-Ao zYESHgZ_@$`CVGrU?8$E}w)-5T3%8s;hqTy{xj(YrW*1j2wVJM&UfQZxO(F42)*hEe zs1mwEF#xC2Q6)KB;e=bZ*7~|{k=@=lTCSUu&UCR5(d4Qs6MC`VreXg zNlHUZHJp!OjBI{bFN!JAO@6JEZaA^C1iK_l+IjA@r1V)z11yU2qgDQEs-X~CxKbDA zv;zx>6eryrje8KXkr#J65>f?nL2DH`BPIX zcJC;GJS!%i2hZIY-N%)=M7F%yw89I&RVO`^7p&rC2my z-?sX;&UMnfEv)7_EviP4e(QAVdY3Dtrdsteuv489%D1L#v^J~>c1=I`6(A*DRiv9k z3TT%HuJh-b_F>Ha;L6HtVA{FTBqktq(mOLmuyfNr#y_hH;dRe$RB8Rp81EV%UBcV5 z`uX9toD@QhA;YfOm_R+I+z}~o>cx2NyIivKJe#wS!P<}_|7ad2B!k{@4-h|NiT{nHK`ab zjM#*3^QAv$daS%m%Nk*6tzfbmdXMt)H98$t`fZQz4?0EFp~pH&Wk9pxbj+W%a5hy1 zZYXAfx-dPKSSJ$c<=pr)M@(C)i%**Dje(p<^y74jSrf0Io?Oz_Ctj7tG6mu*g_S_% z3HwrtOHUaoTetC$(B1!$h9AH?rb>_T%!c*Zsw!tIXrHo#hyR(SX+H|Jvr$a=b#es% zPeP|IaNg1Y+e=kQQ!g?@7{&x}$_wK8QZx9I`{m9>s9#CAk@SOzSS{!8HC{s&oP$+n zTKYSN2T5aP%4_W=mtq6GCE&@u-sdmXg!THnPo5{hUh-a6Zdj#Y>GemTlhgO-#hm}P zDX^1Rk>VSxPCX>ZaI?rVNUAzo>`3I7jWxoGt?!K3{N<+l+vNBMt2w%e+U>qG*5Ptf zgurPbbXW!eheIFCfH20bJnem%GYc0MAD^wW2=WCj{s zC@5L)>L-+}Zx6~`b$48h6@3gnjb5PGJ~o3T1;pK(R@VUI!xWl`i`qrf6jiUAC=-s( z2M>t(KIaxVLdDRcP_9a@DoI1u%}@q{j#ny$#=9TjDN1jBK^@@p$ z_mWKDJg`IqcFS5D#m>!ky<|OZL6%wm12(_DE;?*~s<1`L z*9qFc=@HwDi#=908Hmr8VI_iKn~UB*k3U)2{K5?FwPhMeUaqHmY6j~?(odirj^bBf zvg*^phT@o^x-vrJ-k~V$rOEyyUhq$6hF1b)%t~8~;?+-Zb7G&6o*^>sFtN|7HCMw9 zJ;xREwp&k)ExpMyvsF*sHd^lWFOl;9TeoHNW5VRXgZI*^dr6*gCS0X}uyPx};j0CU zdf-`KgOhdzG^L%I3A-y3dkPD>FQ5JXa|eGOn@c1?S><6t&@`?^^T^7?-Mae$oPyDx z*Qz(UuQaAYjiej=LDuTE2fZZ z;F|Q#JMXXEG#8@ed71aM#hP6yZE_I}dMkyXKkG<;ft zHS)xqE^y(ZH$0_aT9+84)C1h$Xq7){mz)L9p7MV-uKFazZ8B_BqvmQ&ar$~cJdIs& zSvYyRbJJ`$P4BT?S9a=NMHZZ&-P3cSy{O6gER8mWWOrCr#$zW(H(gqwDt->v4^${L zeW*Cdg4`S+$k%+8s4X2Q7RBm*4|u!_J6P5ah__R?tH^rfVX*0yZUqg-CzY4qAGbT? zsxMjIb2`!bx4(Ss`wG(SpX%6a-y=UIyhfZsA91)=evx8z!jG|MWiWT7s`nx!*6j?qn#G}O`0}N&> zC$*5q_lK{0#R7HV)4iL`k{yoxfEFj`nLh1<$VMK<`sH0d9d*^{GFVcrWx)dSzBMez zg7{(G*kERk+7R#@^L+5E;Y2kbMqgG=R>F4vn>iFnD9G~0qLd#%O9=Tdd5*3IFP9~` zqTm3Iqps^qXxi2tMOB8EtAx@&?mXlWU6 zmhEU%gD1_OKQLYzEkUt$L)(=>mexZt!9}8ePg|?feDh?oEhYpshSSz4HCOq1%fo@4 z>}TVA8)&CatOHj7!{m?Gv^$QE0Z|(<8Q9`AT0wYGoc;%v&P>M!kWbMbLdO}f^e69Y z8BjFb9@!gC{w!+w>()y-Jn!U$^)9Am&rupJT3h@|9!;X;(t4)3B99Nh`z$7ReK$$TF z@F5MY`8DB+>U61-!)e%-I^`=#Jom zCT07_kN^41l8gsgDUKzUw7!^#yZT)$_8C%WJhnt_Nnm;=)&%jCHg@5mA~siM>8_4= zp|t{n#KOC6a+ z-J2T~Dm{X}u>9mZF=5)Y4$Cls^lna1_2XfSx*#6_(#} zsz_^2?CvR56hR=-j4AgQc3&cEO> zPC?w#ffvh`Zo3Y%)daqXZy1H35yTph!|A{Zk_`{4{I2vQ87fT5IE7Z$iJ`=0iw$3& zx36dGNZJGsXD!08FlrY76}Vh%cwcWp{=uw%r%=)zacAye3{Dz26Go=qH-XY^>5lcmYB{Kus zW>Q$$O7N*7ZRGI{6E~anh<;XEbFvZsbap_M=j29C+{wozr>tmMl;wp~4W;ye=h$=p zodrW8bgEXogooZzke5saH6!DDY~TUX0f&MSh5nW4!v$WhN*R|zy9YDF?tYLW+edAr zoB&I*8y^*TFV<@*<~ums9k+|O2J>3=3pU;G4PZ!b4QXUHSRK*%;M`~BoZUe|n!}yb zZwEgtB8pIT>vDzoGpyX~Ws}n8nkw_n&8$4;n(nWVxHm6=k5YcbOy0uPXfk4@r!x-I zuRNOaZK+Aw4_mES=#ME3Y58|=Y-Iy#kb-sMC(oV-k;{E=P6w&B!XvBQGL}JuH&;0h z3>AIDrRHL%q2kx-B}4@jfEcCv_n+V7&+12=7DN{2X0q@God!SlwA^V6h=0G0jnrR8 z@FINXx7I9tHp*)^0kS{r5fyG;(u)Q;HLThxg#hEV3?>vA1^-cJfqd(Wv zoR0KQPzU`j_OCYL-|XXdTPCu}QY#I=#wX!DCWE}&A^u0o23SsRBr9ot6v=A;*we2{ zL+2({LcjhVz+Ta2LPyO0LZ;qg^}|C+GI#e4@WZCWnlC0pM->v$@xdirU+T^3pP$W@ z=0^=&vo6u}x@>;_M@GkBs=N;{pTXG}e&7T9ptedyhcX<&?z~k0asb$&tue5C@exgb z=sr$;4w+Y5kwaK#NCXpD0=bihUqyc3-}my4>q9gwZJoGRkg(Vj^JILrxxL!VF1+l& zO~;jMciTD2nlb#bDV=ed+0{Ta~4LTW6VB8}{Y)=zG374dM2(QaI@QMu2W6~eg?@r4-(fQcyi)j(! zBPZB~<7g5;#Q3B`XuG&W)ST#k`R!nU;_IK)uDi0NhJH=Bo9^!>OYEYA5|B~eiNV!Z zDv#PYo0mLb#BkqijR(HEJqPGr_N-rd!PriUY|QGq>Nq<7)>g==a&dAk7!YiEO=JvC$6Q)%}UZ_1#m_ zE)C=IRXuR3a1kEc89kD!4gpkTea75zzWIdSi-x~l(>3YdiLz* zJU`I5&^!cpsZD_wS}HtB_dT59kwX>p3%va;6%=$;X%qX6QXV$bxCkRvrV?N z^sCFd=djVSzk(jtJhYTX6@=4l;2IMc{m7m%{9`BRTs>PKA+JGHemro$OMWTlfn?{Q z4b8n=o*H*ku1eLtB{-JKQpSEstr$31b{mYt?X7%paZBBc^)|-dVl(+f*Y&R-f>WDo zSRa3?{$jY3;uh%sBSV<@#Yv<51LD}Q`VXZ$Jkcle?I97*k?%gyFL4t$=RW3Uu4!od zF^s?oiv6j6#WlHnYE|Nmk$No$Jx*_em1Ay(d5Bc>FOIM^&=pqH7;u0zJd6FjB7N3p zT<8`m(tao+z~%X%LwZp()4sAQPt3>wd2FCm-U@&BTURl%{!>F+3}S^PzT|#+ZF>OW zvv+VoRA-#su!#2yi{Qj+h8qqWV0@kw;n{1Eg-c7Ytic>OsV%Vy){}0jgJ`3RJ^AE)HC3$NZhrBu z53A1pAMV-MzkM>nzxXb;_WX1D7usSb_5V2=yz6hS!~vA`uf+Iw!@@C42&4IqBae6V zM^4dbQ1sa3iMyMAzGol(org?aL#H`gpquV0%G~-rV(&SPXJH76v+rOpX&s8|YP^ER zJj0=YXLl2Jer2IE%NHOHZ83|XqOzdMLqo@h6e*}qj^Ok+XrjW_H)(r^uez?{PWIT) zy0f_Vd-uOr^g*2ySxpI>z1k)&6Lm^{-akY=wv1$bi`PRw6FaqK4mX4-C;xGO6RPLW zNDH0j02yFd+6^9=)p96<1WV;8*60_k2tTv`pN$YlKD!5V8|t{&Dpjz?I0K|rt76O8 zw%RnlYb`4FW-hfPS}8rO$AiI-jf4)sk;f=)U0F*5q#_*@o<}TO7i8$fy@&Zb7Ti|% zS7k+tq4EvrX1NZW3Or}fNM<4MqzykuNWUEt4}kq%x(M_DVD?Wx=WHY_4_Z2hbFwY$ zhtWOq`3A`Re-6@H`32C?N~yL#$kiE(T5DwV8OU~w^TMJcwqAzuQDDKn(GW`#d4cK3 zQX`}^^7%rsppcPL{eOrZ8|v!l#2R6)n9OiA7SM95uk?z&q^AnUXyjvb`}^;})!WqC zH_1ZW*lSK!N8fT^$Bz2S>Y)Yvjzma>+z$9UgR_TdVBNe#kj_-|*acZq^JG(a?Y##o zwZS5DAibdMY-rfKaB4_)UyMgotQJYUA`|+zPqKZu_F8L%cJc&@d@57jd$05K&6rKa6tyA zUCuU>CD`PCYyVMFlXbYKWi?ocJ{M#4y%0s~R2Pa5DV3t@d=kFXjk#Jf>Eu&ZYrT3q zQ@=C3vI}x$!tNWwC+`#)TkW%aDX+Mi;!H$;=6Iiqi;UtaObSzhPPJg~^Iy^bk*nJ> zwn9#Jy=jlRuOfTqRa(>e0*>ZM(2xT~@DJznw0%1Fk{Ta>lUS{KU{)vKg2XURctA3n z>M62jOmQ%Z<;VumC5g2{ehaTs^lRi{TlVQUS{Oohj&Gld8Uy1q1(@lb4BrKQ-jyb~u#a&%w+4@uA|o#jc94rvBz*1mT9CLew= zwW>gQS5y3k9y)2wL)E|Ii?!1S^j>v z2AF>3-leAeb8-G9gU#2)Q1Yk;K zmNH5h)Xa+~pwfrq8l;Ze@k#68`W0k62lUHDDzI!HQb@7io#(h!!a-836J15ERGHCn{Szo%#dV4-_hwhg>gZ01J@J5dFGLp-&*J5$ZwFCNQ1fQ@En@Tg60>-SZ*fRCDIh)Pwfaw?2 z&%Dcrrq#=u2A2iK%Vw)^m{X*TlZ9rpnr&m#G(LcX;(2cLF3?AR>FgrRWK@6hv>rAy zTDr6Bewlo=>27CM!v|T#18f6C;vE@TZEUYML6<7`i1c24Q38gv4C!)AvPRA1BJVg1r(!#8cyZOL}JRGZfoHtcNKix;y|QnO}oWPXi*SDzgQ*Lt=tzEJ}vUrH<`KX925 z)!8?Ol(E!9zcP=?smoD!bH$f7kW~MRiFmmF%QT=7*9|7C68gHz6e~2k-RYU>t8hT; zwA_j95YBx2?4O9{B26p=^v~pX^;WW`q0Hz2&?HiGQA3#>$p5HfnEy^3SUAsP@olv0V zC{-ExP@;u)Ax}U3xK>nhy@yc8c|N1+(%Y;lg>%|h>PO_}6vbV!l_j^C+YbwC7(_F~ zIi%m`M~Eda-5iY?b&G`onawm)<}LnpfkR7ij@7t@$WXOytEjknr2Teae0G^C`h`kG zGNp)ZNXw$1w1y`rK0>e?lgzoXX>01#Dhz$(^o1V2?0QlanHq(^w52%5QVlw=$p|x- z)P7a}I<&H@fPHrZAhKOpLAof~LAOj(+*Uh$ER=;F-+Qh*=e>I2?%}Y8om(S?1NqRp z=DUj*k?ez*W)C(IMg>lGIBrQz)e36`N}J8bR-g5XF7c4jrD%JR=k*I5-hUu}ZB6wY z8*?7rG?bnW%{O?gF96|KhGc2du34(ZW8TLFmHi3IX4heS!{-gXU02T@M5Ky|R3prF zo?X#d2Idw&bEwziwBag1Ai%!V?+M7KVGT z@ET|l9ZJSw*5N+?T|U|!bVEUcU+E|YVv?%lzsc)0c$uiQDoLo`W8>l zEH-T*MH;2%^xMXE{vC+v?iis_?4QUH!qSp71EV;xq7SeaOF^8@OFLI;Vl6Qj5;y%5 z`A#h@UEcg9l==QOGdE?GBZd9|y|@(m=!4xN%q zs;lqScW5S;9ShJIQDLr{kC?g2t;4V19w+amSP(@u9(mo2)Q3IqFewtR$pp;!Il70o z;@-UmLwRW=CV=9`DBrg#j<5GkI8&+~)zrQ8h7}O^!hrg#f985i_KE`UTI7-|@%Ci~ zsqVthljpV2|0E%))#?a)30Q!c;drCSY;;YUNg`4pcWO_!+420p_Omi^pvytBInV&< zS<>YfB%;s(|DpG47W}nfT0`j_M&6dH8~4AY*UP}mYj?sZ-~NUvE?Ky`H->Dh>w3*0$S^woa`1Tv$Sr6Yjs$L{4LJn*ieZZx)s{I zX7~n_{m&PsH>ddYFde$puUGgj^^~&B5^fHEM>*eMp~(s;jV^W@HnVXF#Jp zvif99=#f5qk14zo@>LpP`SZFc^f zAB~;57Ui4($2)K=cuI&;AITOD=+yfn@%*2m=-^Q#(fiF=se78gh&!0igi8#-Hx^c3=cFo*B=EqCQm&vwk$OF7mb;Qd z?M`fPo{{XeXr65XW{VBoaE!r02@>RJLrJ^gjgP~iyDM`$7X#|saq*z1&%^iT-Wo{H z97u-yrfHLI?!9^+^cv|X@!E-yxFf`}4a%Cb3y%uGVa|h=C3XNiq2329zDgRV^Ulo} zNcQ7lSE!vg6L)}|GU$bs(>dxJp})r7JCV}0`W}`ff5*u#&4V0h1=6EQnd}JukFQYH zCe`Wzd<;0u7^(l=0xu7=r-wi)K>B4U)sT@&w9tVySpGT` z@kh~)pMg#R0YU4HA&5=wma1{@QtPQW0T!7i(3wlQ_^6bg+=18K4*2@jC0Chr$lg7f zzpD?}Ooi*?AIrV8r&gKe#I7fMP}TFjv(38zUS!sd#|aus^O6R{bQ{{}C_I4vF8$Cn8%(a|WYFX3P*fNp2-#9?Avby~TOw+DKr zUH#GJ*RZGYC{1^mg4b=l-;5j=oG^5ln*ki{UD^7pEslykCjBt}kvT2fn{C-U+proO z0gaTduQ0{BK{0<7z2}6pwjtrj(#&#We$rZk3m7C}F*B@J{H^yaurOR{HF9ZmV*A%9 zbRWI|LV06IkQ!EC#C})X|9L6M@F*)|GCPD7T7nQ-?yTRy*|F6-4N2W$GxO=`KTZy3 zIKRCcVr(~Za7>_HqzhNCG^MTmb}V*>IMj2rVSB4*Rf>9ee!%yOi5@btYnZ5Tq#kCG zU2^}#nx25b{>II#M*kWT;}rZaB+5II+I$xVULr`i)>6o3EcgLRkR!WZMHS11_0$RI z(z>MtP;W{~r(XmY;nhVs!#XCMa^L}fbbbaag0MrW&&eWzp%v&j*4d4wAA4^n)&m>w z9QpEfQ|?i#N!eJ6i^#eUvH1jKsfEc>^F!E{*NRT;QnXvQTYDA&*Y|{Q-p45|!i0ak zY!HdAH?AjVe9MC|y_m=rMW`3*YM1w2h+-%nS zHLx{tYV$CL;Mp6-?8`!5%Y-_7^kxV=OM21ehNzi7tcegocEkvS z^|GZQ?C8DTZ#QnDpGotT$gzow{Zv5dj=+^te#Yiqr>(%6P5`n}?Bjqn>}ZMTgr0K@ zwl(ySsH?^C;xPyy4|cq83rbnq5q#!Uyid3XC?)Ys{k|bufv1>aoc6cnTIj|iq%=lp z@Fs?Cu6w^LP4aNfab^to#{Pdq70|Hfddy4x2p9=4Hu}YQYptDjW;Vu?l5N;R7uS;+ z=m>Ucj!V<>GP*bSspT52&vDrqL}`|c1Vf<;n_*uGa?EsL)(>@1DLKXrB;MU}{V?zB zzdx;-3mL*^WWO*$nDC@W!XRUW1AN`)YOwybkGlZs?o=^i_>-f&tmEG5@p?L7&?BMz zBroqoYsOaf%j{0@5pw+x8|uR~lbH3F+!!%-4x zy0T7e9Q#}vOKB!`%lLcwjnP*pT@uzL@Kd2P*v^xF z)?sxji}6>!b6<#pRwZq_8=Z2zu0vP!f(kDp^k+cQ+b_|-&(o0^MBr(?6GY2V{(x+- z(Q_|e0;jscugO^#c%XFOPolV9nEzjk{So9osZ^(6qc%|$jq-oQWGB`B3c9usQS}}h ze-@byo#fP>7|io|`heO@(o!LO4gGduGJg4r#=i-mL&kpqQPpD!;nS^kf&t|T_%v-~ z*RX|P2;;b)m=!fcO#z)Ot4J6Z@hV|Fr5!DB@wXt$Ws5h3xn^ww!J^!ZMuKmTA-pt` z0tp*~fT@=S{V1ATSSNJskW2={=GyS2d~A0{0cvH2K|4kf&FejNKFb<&l5zB-uJAC>;3qXLd+!h zLW79QxngNddJsSV5E~7ARd4Zq{93BpmlYoEW&OnBJz(iP3sSg)ENsTxPH#5tt96~e zl`s@gXv8ky3HT=kJK?wBhE1be$W}+eQ7fnA6gx+#-ms~U z4ZQjVea2TBJb$A;Jj~oU^5XjR`s^vsAiQ|)z=Dtg5#F^b;S@FLUX!uU=Z1@T4h;{^elrIqQBA1HasGb4ZDF;*o0}G2<0mFq;b8IoCQI^bL8Dwcbu^*Ah@GW1yir zckZwyKk}xu&snUcZfD{fuoUmzYtT;czR>^6zVXIK<$laY+-@(*yXy4x;xKbYOhEgA zK>gTu>);R4*sHRnHW>(XdiD`H?kLs}a@1k+`QeF;byjk+@i#s@*&<{xuXFF@*kucI z$kXLmdH=EAKGoCO+B)lu5=@&6J%Bmq7RtF5GUo!=FOCa7K4s$KPK=Q*)bG?{rIlHJ z18cIS-y2Tg+{Gi1gCM_YvHq7m8%E>miGb;<7?t{$^8nXIju0*4ld=A8hpyHw_ICC{ ze0&zb4ZZ8D}T3PfTFijue5HDcOlrzP$@Mo;RL9#1EyotV=^So!o!=_}yL;MuN? z_nk2Nw(6<{_wk4L^ms5p{ga8*Sa$A-?N6#hZ|TnCt8JnkT2s;Si`DELZTU_Zh`wqi z4zfht+w;CB^*7Jbd$c>erYjgt1AXqGP1QlORVp3{EWQC<+a>C(%lz2{42B{%moBPC z1l=lQk=6)T1;2%O0|xDCbaT~!hOi`Ek=cp8!9PJ12z;Zm;i)-9AdTMu2ac|hC}0|K z>=h+ENa}{z*Bh#_fKT22npoMzy4T9|N;vYqIDz<25?t!DZk3yOuFed=z>v#q$Md=6 zWJ|Uuf)|WBd!I1M3by3^dt}%*>p8MxE15(I%@pL=j%enG-tJTJ>!8>#n%b$&2rp@> zcGlg0m3expfGRaFH=1Z7lO7f3cCF^@TfFdIoScRA=MUYN5v}-K#l4}+#kqNH>i}D@e&ymidXz2(KAjA;~({^|4 zudO*yo@1)SG<7FZVonTo0drM2ISabW-oZ4~VRnpuXm87n^3h{bRH-@8WR_(Cd*V_Z zL6{U)oTB@TWyol{7JBR{1){V9jGP}OxrXAVQtu4EnR`h%T;1||`E!Xh!+Gn#u=bki zTm6i)g$*s!fQ*D#4Ds>kBu^&u-pyv)x1rXVK3`j9TGkKjvU2W*m*qAI-E)}|+;V@j zW9p;7OKR4G3Nd4DtU6669B^jj^^dijqW9{>Vx_bfTNI0<@2WwYD8*h*-@KxAJNB2O z;q1?eLEM7ha*x7(rx>{IHF^TAnOjQyt(OQcusLTiA8vXvUs7H1SEnwS6+JfVuRL@% zU^e*rOZD$3djltg8?2mm8iK*AdHVc?85)cLn#MN2G9J z3hT?8Z#>nXnbT}XJ_bBr8cAqYfAB;CHXGhssh-@6#NzBq&FXE@H1}Oz}t>U zcv#kdOFCIp*`G-wr|!$K_ZD9(aGVISrqyyCA`(#y-%xtT$pCjn_aFSvAcCHwn9jOE&t9u6cw+Pz} zPd7$6stWA5m?sO_F_PBEo(0G%G1&Clc*}h&64Ol>WVex};B{1@Bi`R=+_t=X0x&8h z?b2m`cj&{;4YIya!cRT`45N<6WCl7$u=8&lLU^khI!}=?j1!PptYeG{tN1D99&N~p zqh?dUJeSTsUfkSGYBnTuMxU#@LNIfuSDi>|F@sxMQ(;FMy}SM2V=v}8Q@h}aO>@7x9KzbC5AzEv zn)jW*NLe~D#TeRm*kP52*`Te-3lk+AxZ~6n8M_G1n&86vOz8(HqbAo8ll4S#CU@~4 z8*UTR^P(vX*htt=mlx}s)hCbXDNbi#e#>A>t`r`@cFLMT7K!|e5iK?n8^_s5VjB_< z&bb>fKK^JPLhR;-XUDhZ({n8k^UzBS?DO#5%XLrNkKBgPD0Q1DC=qT&I8j(?)QvsZ zy+)utuOVmJR_9ngmED|2);t{DH0CxweI=EE{O04e|u3$@jhKvfi92GALE~GSv+C<+1W8? z2|vdkgZ_(;SjpvwLLUMqoxA2Qjjq16ba5QXBzxxZIYemkU}{+y*Y z1hn1!%6trkt1L6Zzok*zF5-zNt48rUCpAfo z2;FQv!u&kYmXSg86}9>4cvRJ&!Z zvIe-5!HAW-{Qm{t5E~{vqL8RDFI&o<{;9$7XOG+IV;k}#vD1$_#p2R+T_dmpdd+?K z4sq`fq3yy+o6)CdqyRpj^RbB<^AKkVOz2U$T$Hha|0b|XbQKv!hG1#GuQQIPkrjO! zy8ztkRNnNYvSLk;`_?wm4wn#+njU$gcnj?KRJqcMr%TYJ`~{C_d^?(a-l)x@Aun{Y7SzL@3)GK1GMfRw|UoKKXNBc+G{e4rGi|XYL;!x zj37?IKLSr^;^7u0x)TRS`;FM{SIUJv!d|UKYfRJ?bP8gBYq+iR9F_5QM)kIW&eOV& z{sN}?c1U~%e0!>t1T>HdW^SG1H^?2$PUqY;J@a5PM|u51VIree^r`-_2Y~0_=QJ|Q zP;b3Ts`=mogF(bD)XPPFHc=V=O(rAo4I^!+1@o$9?PBTuPuE$s@h6X$O@EG)EXO`p zj|!*@o6l3_)x!Pm1{XO%Xa(Sc1WrB%+kbLZiBRApLr*MMFKNSES7A-`L2>eG`Yvx3 zSoM{{F36l}OG&h_szO6pHf%)+o_~yzabd}uuU`(MiG&cHfR`bN&lMWdwXZfrwBoh=$N8-#5j(b3O z=>gf+^Ru5Y4W2?p>eWSxrxij5Tei=qRZK` z;Wc`!=}usuz75S?W$mTrVhMJvq53&^L;`uMQgPi`7hL6M*o~)#am&SI6~HPDBe1Bv8TR!p?Ig}v=HLS^H@Fk z?N!5#Tz3aX!< z%^1|lHb@BcelV=bPgaE5ryFwR07L_D>c)VgR!n~iA8vH%iC*H6C`{`vR@XMMBRS>j zOIai~kN$uT7k99ni&E8p{Utd!hq08w7X8(YM?g#MWb%z3=Alo7HhizwMc3OWm*dpg_ zyhFBc8zpSqURWAGzO-|HP#U6Yhr6(X)70O7pHD6kbtP~l_aq+IX<3< zW#dl2t2~S6YH|jip zkhp?rN85xQ+xB!&^-RQsamA5QMUj8sIk*c4an&{Yk!wcwpT*&ApuZ|^A? z(c_UV1wp^|9}Kz23P6`wvfoRnJ(2bbdL>@31C8y!_;$LvkKz$g!MCpjDAq%|AA2_` zE^TSuoSK92#Q7V!RGpiMrjeMhdp}2Kcdsa6jKBf?Rt+*CrP^Gy!QZQnP3ZB%-)Zy~ z#0R#1=Q<^pF~?fr&cxdevm$T6|0vrVI=YiPkF9{74^Rw9>H281geQE|v(>MV{c=_5 z%{lx^t=PK6lOcr1Vny~1dZg!iBA9{+Oz2Ru;ToXg#PZHdz-i0%evK#tALEKq|Eh8a7kRAw|z47K9b zU>g|r2c77Rr~Uvp=~aUkgmd>g7uwtDrI1#x_SbO|l$K5TuV+t^OG0)GrnPo>g%t(A z&N{5l?{g+V6#l~IZ!zB>%KHcHP^fj&@2@Z6=d1n_g9X5}l2hSjYkWX&9K9*uD?Nj1 zX#6KwUZA@A?{@q(zY9yns}I8ksb3uvL?hxrM=`>5XN%;Zb&0Z3N5)?uo2H~X!jyHy zz`VzExo^5*sdO@iIG25Be`|V1Z6qOr$-7)4obQWuAFRVAnqTr50c^g}f7}tLOxQ+W z^M2@=mQ~YM`r30Kqo0lzE7`Hk-E@ln6;D7NFW9GRSb0_DOAp8ty4F*E1fvR7{v2)s z#R&O4*~$9*JYx54qR})@Pn9jt#@^u*-hR+lP)of1w$tv9lmDXOE@hfTr+2FtZ*${{ z=vUQ4^yiXWM#7|ceSDR<^i1=)h9gIx4l9~Y$FO^u!X>59?0p2;#!{N|-{WNscB^A> zp+>~{gnuvkYJw&NKz_SduPzb;R{~OQZ8>!h^lu`DC7JInZh~IvB4D`OYcictEz7Z| zxN5k1o1-1(F@U{X^81ztLZkCZvx2_pUFbO;xT5!(+(s02zL6&@d z`tUtT1Fb*t#n~F<%J8>!o?KJ_>#17L%YGTMB2ZPvpwoZ6an(|X##N+-W{QQA&K{QF ztC3LK%Ff*5Q3|VYXKDsD1r`dWmVk7aqVRC1$v}mSFysTr!;4g1g<+abq9}1z#Uft= zHbggm2k~$m%Y|!WqOJtM4_@i*cb$$iIgni|qiF6}$#z$B#tvs}#|_Z`1PcVv484aa zFO&!Y$LiFAmRA!4!4vA&mKNsX9wWxRvgZ zsEVA|%vye?nWk@VvgG#nW83g+3S+4AIO#&&di8GvHB6cA*XwS?r{GVk#lox2#H2?u zPLi24!`!dir58pYK*|?aV(Y#-bb*E(Dt9Z?EXWU6qWQyX2ViJ>g88>iZLX=XR-@1N z>B(L{y{AaCN5p%wiSZ%r#tc_wTmyt%21G<&>FgN;eSYXoO}AVoMF))nZO=&a4cVHW zVJy^jIetCZs@na#O-X(ZwyN%q)q(pCIDTRSUyjsfts(D&T^x!;me+i33a{XMQh_$K zZoXd}zM2=6|BfVVmS!LC_8)PGp_FZM3LC7RCj#~iOli95y1@RO*awqt$I4^NekuK8 z26>d}?+|pIP=(=(8djo8)LtXt;x`a#o_ua<^MvLCZpN!&^E|LCysm<8Z)l$FXExiL z@q0z(iyiqo72RgqH76=sr2n&deU0`VqN5as9i)og*FksFyQkmjg1>dI@U_;3vCqi3 zXZ7x$>>pQtvt1hPn~YxAjLF|N0Z&Uxjst&45e`^EAW|K}4vKbRu z&46B(LJ4P!lD+&rF>?hp-Y{RsXuBd$!k|pDHS`NF;z$& zo&1xS#w@_}3bmkbl4nKUc@dq`+%b1st1}9p9!2KYer{B>M7{g9r|KAW6+4=>HX80G zS+m=t5q9IPaO4%Zrv*>TMD>q~Z39+6pGy(pG}^P=TC+n+fjRT#Dnh|pVQt))nA;@H zm+!Iw--sgo+@Ss7c#HMukj=jVlSi}wyEhXtQc@zVxAfQ-2C$2NA@F4Oie(jW>F){$ zs*g4)3@PXPZm(9jQssUXvOdq`S=GBcy2HJoo}+tlDhBm4eZO+4@`$SWouUuy9|!d! zR759o^VH1UJTHx%n|~?k_f0EyI$XOmN~vYR;e0nCvi^nPejwTr??M}OmYzipXfQ13 zQJ&WY>Rr@1a(I{q&`QWrDU`ep<=5i211ro)!kV^qVM8bEV}pnCxtp~+Q+;jvHZ>ix z+baO^4{}@CQ9VBU&$0KMjE_E#WOGWMFaBjOZ!{vTT1(9&2gP-BVNZG2Je7aLPd8is z?_1y;rw+sbrUS2patL3$@4dOdNb{?zQXIAa|8@Hf(v zwm?CsskC?N@Wz>nA*sqy_XIs;aYdH~rq|pdLBb}1@9E17Gdw6$V{z9zW+cf5ll@n+ z-5$WqM!`m&@L;e{mV_ah&_6By`j#tlTd+B`j?<}to)|E@2)1gS{bF1mnj4Zq$j9&9 z!R!krJWVh2&lu+dxS5cptAtx1oyANqmHQD`lRp@h=Mjv`gft`Hn=;sUt{a%y%@d+t z+SO;px`tnL;$W`P+WszY-%9VC(pkm`koTXf(DYAMQrKHBlcz0E_2dh8DF6B+4n8~4 zg*e}GS$1i%kfdiBJi zw%36Dpd}F2UC|*8LKNWR1S7D$O~2w4p@$WfRPS<8frR6`1f)=9P8kaQb64?A;5m-_ zF>cckDP1gL4Zv-5f6LN|)TOxOMxiKb?9;iy)T|w>gpw{(6kn&eO(6N^qt2AzWKeQ= z1Gb#Q!B=!^#|n6MVgH~V#cFh9Wi?vv33vL0{o>n!DVL_dN2_Tx>>T_Ff<^3iGVO3X zd-rMemh&AHor5jLGuM-w>`!F1gYM4y_Dr}0Dh{mIFc{bYl z7jMVFs*Aa#mt4ATo0_hGGL?ep)sMaeD?>(SEMJ z13|rkn;g(}^Txhte~z)5?C?KUD&|Ru|LQg6FAmSt3NMsA4SSRW5M|zdIeCuEw1GSv zKeuqX*>HnTls(#w3ilmSq{(Y(&umIgw}QcczvTFWiO~V3y^V3_#(Z;2PchFW%maql zU8dC$F%imab@Is-->+-2ZkiP}7{m6n#L&WPunVMYg>2Gb^j@Tss{-O2+(TN6^h=-Y z9|t7afo6EKFX+!?psLb1)mqTQ^AI`wL@&Oa;*IcJ5gUqcl5^I!`&9t!wOI6_Sy`+)KvOXi9~ zXUsroAh}cWU!vC`XtxOAH{c%bJl(bzj?WU|Ag2}EqrsQx4CXy&NK58?pDghIWq3KBOjB)0-Av zyK37n&zeMaH7zp#epp>Yx+pfso@fXBa{MiPEbP3@)}O?^P>5Z=-RSFg?PZ3jko*t! zB#rL~ixUgTK*SD7isJW{^Lul5dB()hCJ`|IoezC}R)=c%5MzJ4@8)>9zKm1}@C#w4 zv^McV6sD={$I~GWHVz02vM^J=wXRE%Wte-P7ChO>5W)mSO+F@yux7sxOyLKL*j?H# zdLhyo`FZ};!%~GLHN^U};-Nl4Y+XqOSwi_M={4yWKRMUNQE7CtxN5#-JNE;+@ETpJ zgdEV|oI*{ZrnWE{UiiRo!c(GLI>TDgXC|1~)cK&@_1|JL2Lt8b z^gqpuy}{OZaZ`#~htW5YHYG!Wth%AyA)b5?ZUA2V)&dQzh3xf60ogOdD&G{+PA?Nu zPD&F}(w(s}{~l%qx<@obuQ!^{#IzCaXH5)6v8QagHYnZE%|atW6eN4G}hd0Sbr{K2l>jsE**`_2jg2 zQM!kp<~~m-9U19&PY?hP#;@J)lqLK=@ZNdy56BwTN6Oje>roZ9pm7okH_osvpL!mo z#`g3-lh=^CP>FPn+U_Y8p{FHaRxhdn%22SyYD9r>z^~@su#2R#T9xOPkJ86GJBmC78xAg3tK4FGL!6ntEg@z{=^j+}WondV<(tu&d{~1S zBE0=||Az@1$JTNvM?6;3gG6na(C`_kD|EnFoOt?=dh@agm^q^W$W{umxIk#4-E?k! z8IFx1S7L!j!HerVi4dB# zh}9}2#A#j1jtlis4`}zBgLkGhT&$`{9L%zEBVH1$a?bU%L2(g6#Xx9@L%^VT$(4&f zB%`M*Z8mOU*iX*Xzay+Ry(YEC!TY74xBn|vS5yeca+*>EtI__jO;`hBE z)$G-^9oiD>tr)(xo%X2HmDM(KP>-{0r-QsVD{Nk{n)7BH_gY5g9a-!35i)8Z2v zJiZ3-LKns{X770do*BD&e@w`i&T8(ffk+0NC!6FSoKF7fZmkzSsKLGt{S(a1-9AW& zhWr-`YWoM)NShdng{hJ+(zEgj+3wWg?H)IVR_I zZ{(q`h4Sq|FRdt@kU902M|8r9lJ&32a28}O#){c>G=8g*i`7Pn_7-@^PE~Wy7%3G{X=pik(!?PlzK>o`jY! zxL_l>!%@%vTNv5x;3ef`%(7ox0;fnt?hPk@&kPHwI|$FOxEF?QxP{)Bfwz_9+pqGl z<)dntRPDAF=c5XBz+#7Q4j#C=5|MnL4N+!w#A;D4o4NhWU|#&AXoG8>zQ_0{vfxrp zPAJ+H@+n0k3kJ70fxcQjzlj)ZxEP{!b7VQx+(bNfS_)cgxoi|NQ2OB*CzaIALWwtF z!+VU==^4vSfcE*Z3{t;lI;z_04R7SBng?b9^z^Ym zefz$Q1G65E%>qsiBv0b2({<^zH@-4hBCe24T+4E$-W_^WSv~)_q6Fo{``k;xAmR2& zu@JSOr_U-Q*btl&Uv6y3#*>}>PrLqCkJdF9qB;&U)1OSbOJDxb9*POIN0DCsD%z`WX?wqkXmI zj1~iVkL~rZmHN~F=J70)o+J0uw+)>w_i}Lj6Q(DLYUY6bL|)M#Wb2O_o^k2)H}Osi zx~B(?Rca4udlhi<$!9RZ>{2DU)OVip)&ADg(m7XX9WbC=nhKTtL$5yY`1wWc@*ssf9}`^xmo( zE1*KYSQ#NI^S}RCNToHHw4e^PQKsH)ooL|dck^FzR|ZRyY5kWJtP?f2bmYlgDFy@j2Lf#14NgXLJ?Pvdl-WlHq#p?!rH{DaIq38h?(cD$hX74ZKWUo zo^B~7&lFt=7#ty%t$77*3X)Xx3*+0b0d;C46iz`7FNag3=%8jlV9p>CfgLA$t3i%C zyc1^CzDE1rR~poZiDwl#xJb*cu>_X-1gHY;J~cO{xXDtXJ^F7o%!+%Byz1%O&-?u4 zyvejw5u{maY3A1AZn9)^9tZO_0nXZS|# z!~}GyKR3t?xi6s!?Q6DC#{l5#C<#0C$C_1aG4S8-^m}@)QPnQ?+-plZ4NwCbRj#`u z>6O|3Ma4>0zKNH}LbTD8*M+{vw;Qx7>vY@=8~M(WCagnZjr{^pRPVs}(q0t^`btop^`$7epErh)kdwOv1GrPEVvs zD0jJ|KN`3(%_ODxpgZC-hof4o!*++WJrrDJ^_?3`UQSG+5=14Obp6h=xH$!fv?=Xg zJ_@uF1Il)WB_wJ#&T=CP2=!G+B(uVAq^u!bpRR|q!w(_;Uck}u+dABELc<y4*3?7+ZwvEc9#1#ifQaqIs+! zciZRwA%w=&TMjRSWoKeviH^r)&Ck(}EUa%8$7*!_H*|7Ka(oOE2mTw(ZRMBgV~)7b zbqzgZI2#{DlZGhGeb5Vijh%+!?x6gN;iKcW{UyK%?0fOx>#ydHx+rIvt$yOg`Aw$N zUsEoDm$t=c?Il+$u9Hi%Kvy0RjUJEZYV`)LC+jSDI9_#B`h4F0k?q0D-y01B&pz5+ z?Q1~zszg{Pp?b5XQITKPR}um{zHVfm_q`z*_@$Y1VU)4jnVtAC%Ref{&f3z7qm zep$gs%tUlCN9Y@8Z>3XTTW{T60I2a#%MD-fxWWrzTM5qLf;CA?r$ra}VmtgFoMIoc ztRKJ2DOiiF!Sx$g2$NL7E@#3uf8%e+_*1C3#t&D-(7A+8(Z^I?P5Ge3P?W6Z^x@^~ zQ@@j6X~~ahO&jDq%Ka-9>yHHr)XWrQBcCUzxVnKTwbufqy2L&52r4_+uWjCXe2Mjn zRczXL?9DQe%Tt0M@vw_GH&pX4Xz3VUX)GytP~xlKT!X83WN;jxhX!m=yja-9-SIJF z>w*A#yaV%T#d2?=AXlX%qDy=pW09Yw@%+w|Eu+Zl@JRsEvpa-aP)hAT+nIq`9J<*P zyp0g@kknHg%Ntb~4D3=t1;7rGEFF-!ye%JR+GBWNNs`#Q9IMBEY!cR})FEoab^0c^ zQ3ZUS#Or|3$6)*`(q~6@w=5NzhGR9qqQ}|~&f~*C8HBEjmEHS`t_K0eDr{LZ@6roz z2in^Rw-Yj+C=Lb>|1eNo$Js1BC{@(Y}>dyF0jdzY>mrpSh!RJtKgJ_~qIQVed`{8)Je#^6nx*_EG zwWRjlxkNs6X~nh^!tu{IJdacNOJz;G;$?N``=@wSsDb)Y^}rr~soBpxlMOB7H`B7$ zM+~tfuR|P(3;fKJOp@dmnM+Tf#Kv~K?*s*ZM|6EYQLX~=cqJvYAhhtkNl~CAyfgOF z=fuH*Qr%nV8&HdkS}ElQ)t%Hh2zWR?L;VodLPl}u_ib#WNL>m;JT}`Pz3ES#psknZ zX|;PfgNlAq{Y84KD(yn?x!e!nSmX8|ud)VmA0^&?E>zZE}flgA}7SuM{@-BXZ0<k^=F%>2MNdk4sgbO6inzr&RpKG8094am)t%^y1?hXEh;bZHVp#7-E_`amn8o{kk-yx*<)SOvF zvG>1Mx-JPn2QTcweGGXwbwb_5g`!!U+kT(4JiV$0`?u>eE7KGx~~E05y4xVJlY&RcXUL zUzTO?=c!SdmLWVn*1ty5^eYFxAAcY5X9QHu6zci1bl$NGq(@$&5zE>ZxRbjWv{dYFF#B7a zJ6oKf{bq960u9)>Q#YjZw}5uaz$H}PCSBWC8lcnUOI zaBPTgRKarnn@_l39P8P?#C7j3(;d0Oc+H}n<1 zm1T8w)ur69a7tLZ45*9J0(3R||1jwJzMM6xduX6>gA(t^9)DAX4bCqDT~TyYa#UnI zDO)sG^wm$iGHdO%ZQ>L`g(pdl?P{NFj$Zy-9C#wj1e*Z7af&C9SsbGEYVJz<{%b0- z@qN9h=i?kn0{eQ<9=|6rB`!xL;{NO+&pV*N18&EZCb?n&vc2{gp6&(AwCB4%eFiCt zjhPS~{z_dBO=`Y-(T`)RvZjRmkLD`*hls>Ky!TjVEgvj?Z96J@F@u4+{%r>2w7wp4 zDNL(mNL#-m>%3~s^vD9rWp5Jz?boM#1{?`EriuxAA1M`(uvD)H*HbK(W(G+6-OsBo z5@v~84yOcYbt>_~%xG}fe|37(ul(hJ`5yCQc6W57(#2i$oLhT_OX)rfst0u@$&_1Ex#HnCp3&I6=2)ot zNZXHp=3%?JUOs4Wa>J;Z-Sbs$j!*+~$P7>_rwN39@{E;Al6=nPo^*_N77f*>4Z=S` z5hc2qL-;{K|LZ{?3HBO9jZE&Lfp6<;C%^v*cSn$#PgPAOp^q|l7ym~5gZ|6V{i&52 zu2ebelFVx|JFnc0!B73`X>e@uO}6V5=3#PBqOJDyVj!YDuZHQTZ=KsAg!-U+Kg`+J zL{aP77{)RBdYR^i155HnKQZ~V5xD9p?c9esR*RT<6F?iqLP#s{aK5kY8u)JJ^qwv! zkg8}2u8J3b%jtuz-E`cUmxWeD%;$!o5~@dHAF#btelB!A-B^q|b*#SAAul8;{<8nb z!f`D|d<{tb^`t_`%p@QyYeGRuWF^*V_Yxg-r18E+X}2R)r}To*6D4Z)-KC9Ir|7Y5+gE$~75Pf+pa) zUHfOj$W=q=VIpqMj8)FJ?agrjmb{Il-jbS7tCYgU07(Ng;^Qs}QP8E@T-d7LFlt=2M^t^CpV=-7Kl1O*PWNP9W$evz0$sb0URfDM z6V6I0qlhaM4c$z!&+#%b$Yd12|F#vyKN4|9U+@f)_Sa3`vyi@*r>|hilXE)29)5oX zSDiEVyHVw=LY1ZD?1rvWBH`jaX}HsHlFFIVTq|WvRZTkS1ajPmE1c4FyFv}ENEq$Z zd;}Ko9ts|X>-K5Mkj%d48OS(HDbQ`SO|YaUIX2dQh%cjBqLQH(b4bto$B~JYnTr)| zhqtuf?iooy9xYy*IZyozDi~cb_fE_bIPB3q56=Xg?@N zj1@U9nt|rBYHaB!`Rzw~8w%&bj$^!h-ZZ;dkUq!I>U{zl$M+c1 z@`-C?Fr-SLrm`uFkJt`TRX5G*?SgET2Ng%G^O&ywrSAGt8l2(?W(M4`_Q{8jP;3Ob$$OBB=py*NdH?eDAK&bm{S@sp)BcQs+t@;1;vq?z>w;7?MIkxo)J!96)!CLU zh>%YPXsIJTlmtCSv?6zlBV4xTbD2ltS;Tt>V*Q7gGxGv;)1)*g?|NVTlcH-!@6Z|jcshhS|D z@*!-J4A0Kkk?v_uo3DT=tw|$k|HW;YAiD?I@0BxQaqqZKy(^LDGeodTg_Ho#)eaQJ zn(#tZWDKQsLav(NN)=Xi7LEn%_J$zt2b}J{82GV4(klUQcIW--#nHZu$C{@>yKdHin zrG93d^c2#12~T*e?$!?CzjRcl|Bn%L!DYb?Artw>?d6B{fFG7P88u)15Tm}vw@zw* z_bJy+-%NpC2I;u=SqIj;Bmm!>GHV0RG5BEMKa8kj;Vi>Kf9En^k7;{+g+?IzM zlypy`eq7hW_NizRZ!EWzh|t`#8X{Fcg+Sk{<0kzbny-28WQ1Dly=_%G4Zxo?O2`C~ z1kR5GuO_9h2FZl#J3mxtF8CWOrdPsJ!Sn7FeD|uHzL$1M8al`nbP@U3C6x>{#SW#Y z4#TlLGL|m6rS7@ztw`Q1t!d`4xLau(=2d8bz{|0`0JKXUgGWW}us%v$(R^Qr5=!lm< z5tv&2{ujqa)c`Llp@1+MT*S;oF(wMK?{%h}XB)8IabfBGnnfL?;Gv)WfGRNJHB#wS zotiVguJvP(eS)Fpl=rw#gM=tN7iEnW*Uh^KW1d+adJ?H#FYu=Ng_&e+E>+-X|Axzd z>S$e8+RbSIZ7!F;6!??IQ5=zWll4z{12?(g#((~&R!f|U6d@*x?9od>9dom;;ySUo$82Lo(5C8 zUaL>UHn^@c^EW>C0(r5hY7tiA;mxZIzQLZzq34C`HF+)&c$qiNQT*WLWW>6AS|w4) zsglrm74#1EJ2sswZ&7$Y0-pnsuW#;(V*JI-+B09w7%U*KNzXq9G;3^Bd~_ac11%#p z-@o~!+>+-e#hbMepBW?*T>7;I`OQ>DihmF=Dx&XseuHzS{R^S%3=4(-;DNKwxxJQV zPMoB7Ws{meEh8R4ospk<|Ng~{!BM^bjR3Yh0xD2(Q*eEt{&1<%1ziitlWg6?Sc@zs zMfY*yNtbIx`UP0D?qk4=h^>EAoXI6%Q_NqydWM#Y{B`6#od3^CIR_=_sXK+(QU)vX z3+WMf`QdaOVegTu*c3cjt1`koW>DMhYLTU49GGNKoK5(Wt^S&B ztgot8xCTD5TWd()zSU)+x|^lS`|iUZyZn3Oj3g&KyEPF13LhFj#k}Y|g|z@pN$06V z)+N{M{ug^s0;(<&{MDtYv9}{tPj*pzWg)Ho1^Ch(ki(-fH0U9nZ~Chlq?4E~qPh-C z+`hAu?tOeNZwFP->o})(3-@Mbj(iKbWk9EVxUj0biR222pE7&m&TZcwZW9SO*2{C< zcGef=R^iq$78AgD&57mv^YleU<^%5lbcYzOJkV6}or zx+pUp+nZffK`~hTS$B>nG$H*75)89r{v-~HjUFiSVt!mjM5(@lKBwd=QbuDE{EHJ* z44ik5P*Wx}S0DHW=|pPS*>RriG+zqAf~x6r$>!<#ia+GZK3J4E8Bs?-BB7 zI)#tzE{SlhhC3>|{Fdmf?^LBH4LDXQO z{S9K2o-=c3QAu#nbFAYvp{J86)!OjY%ZMwLHvX@{W%UFQ4zu?|U*^$9tHP{#O^JQe zV#}Ii8uues6ky#p+yBiKB^XSp$k!FLH7L>*Jc59)u3qPYSC1D-n?eo>wD=PQB&ht} zfSH#EDJUoOHN&ZRxqql#p^iVJLFki-;Js)Z!IJGXc^MLRHW7aKS{-@+^yMAy<$Eo1 z)yxsDxw{kz?DS>j8ZR^>@ivblxV>rH13;&1*e?|J9k7E$#z9z+h~l-*CAAJGR-ENG zD&t~>j|OU4r=hVUiwS6i>@Y|0C=3RHp<)1BOaTQPhGBFgTB6hU8%DTC;s{n6E`kn1 zQW&6vpCfqAvZTH{>m#0yYru2<7&jl9MN-Ar_aQU`Y4lnDp%%a*TEtC`C zkr8a>6xgpzN+@@JZum6X;WZc3G?C`2*T9I*lWYJeZLHo+;GzB9|Q=^r}$it}SxwFYbY?8%f(~zdUNVTZEJgp8|m0H?ncJhlBc|?$%HLIh+3^ z4Mx!Y7q+P1Q~&Fb5WN>Rai_w#3mf7YPvw~k?%VULZR#9u1gxI*V?X|7M@Cio-JX5c zw0ReGm|D6W@xjtMFh3F5&wox%v?SozkUz1DzTPZ69@E0>5a6>=3-pU?_nro93GZab zrj}Sp4TmszUS=v}R39P`u-B1(-gb=UK*c*1r+tHU4l|z+M(MBC2hGqwe6h5zdh92> ztB#vL2Ng3g2B1u^L7!B&O{kVz5{v6Fq zOZWe4c{z^}r-&e~^`@r3Lan=95hU9cE2A;oR;#L_*SpF|`&AG~O`mIEb_PDJ>i>Y5 z$=o}nC{$IDvl5XYkYC9qwGqsuy+{0ytp6@Chn~pJD!*!405NrOP7n&qbbrnTbE*NP zAX9f`NQg^IB^#IJp|ZrZ_HqRe$kH2a*vbVe@9!LaSyebpoZ4^9SOK!^rvoAkz|#U9 zzsEu=eot9dh#-nnVg4eY-hSxOc66 zPZT?%jxGRt8zM(BEH1&P_mKXR5R)Iiy_LzkTW_2?2OOipRbk(Uvr0aMTk*|+8OjZ* zZ7lcgI}vLp0ZoNvbjXc*u*{%SA`a-k&fswi@%-jYlX-YuzE>j(Z@2EtkB<^&)GRYV zCXeGj7r{2V-?n%DtXwqMJh%|U;p_g)XKJ_*od255AfLyd4IYG7N1)pK_pYl~Q>t#_ ztYjL_=(_YqSMQ(81;t2!GMhHhpAh37s4Skpjvrb3^`H(r}#okjTlZTb^E-aGIb z>8L<o&U!4`swKqNV6ZlG;QX(DkXwgwPAk3*TjGdqx=eH>hMkYpL(mJ?5&%8HB(N+3Fc*dnYy$6$_IaONJAme z!MFFwKNWRn8Mxk&b!?(JQI=YKK#R&ITqlXr1Xp`*Z*-XgGE3EP{ICi!Z{phNre~A| zTXURYgHng&_;2b-(aZ2Qfju<`^gM>k8Vab(3=bTw$bqf!;?@+Q8S;{ zGF1lGVyg2Aj+1+q5!j)Ot%I6#dgyx3&0 zI;ir#lmW%ss{BppE$1;ALuCZ=0Ztdw8!zR=)Wv7~`Fz ziZf&gqL;$OH@T|HSWD=T3NJYsFdfQ}Y|Y;guXPyR(2^UAskSFZiz9)C;n|jsJBJX@ zi+e25UmW^rJeqDI;tF5=bgg8~vLnSUe()LQsaHG_skGfsJ-<`IIr^rOynd&7cw2ak zTWZHFd{*dj&=MY@x^spd8@gB?zKiv- zkWiS;Zxi{AX)`3g$xhZOuqU37hCP#iA~ekRC*$VPp%84Xu$?6sna`x109Kb8t%SOo z4|(Nq6z?!)72>8**r%0yXAg}=DTQ@a7p73&>E508l198#3z|u~m1z=5&J=S3qqr*o z@4~`7iAw0i;jiiwzw+*B6+p1Tn^}1$M(>IcKhv;>JVtcG%J<_!(gw5=UQb^#HH-^a zH*TwlU^^Lw)fqlN)1U}UN?X1KEvQo_)Xq0IrvQ^wxpI0wE(Sbw&}3LnMLP(Mlt`j5 zUv+vWGTS~ai!ExOkRVA(k~58y=%?K_;ze=gMvSH&WzB=9fK<_$B*DZXZooL4pPvw6 zqC4AJW0_k;rboBPcB+k>XRmdu952}$&Ugo~%ZzxbHD$qwdNhbO6;vZmVvnn-8_pBF z0q7fqleLM=)zFdlvk5wnmI?=?yIrHFBT$cCg+1jwyN&uZUpe2Zw_z^5i8~#vm;TDr z<5M;@8%or~1`%MRQ?k2+U`%8Is?Ti0PTM2x)#&$|sBbbAn0AUxl6EwW@=EzaN zHYDr4a;SG02^FjmzHa0@v#@;=2&;5t>JZ+%4i?ZR#I^E^ZJKD@2r_%guYAL)0826q7|ld z6yX;#IEKs=hWTg${*jL?nP~jkR|{tycvAepm48rJ^6B&I;FB3ZOuBx+`tjMmbz@&* zZncNkSKzy}_&QLw3I2(Ty%b4&Tm&d6rC}W#0MYv$j!E z(T|u(ke`ilu14|4SZ@|i$3=Q48aFxJa9E{F*aH+A(?7(xjLIG&npt5oM&Xt5H?dngaDe5Hg_u1(eRkf7;hSOlGcq>C} zapLQ>$eX;SUg;eZ(IGxRG|ADXmLu>SZFAIAo;oo`igM^A=>L%Q-hWB|;lIDDd|N5K zv!OYtT$!nvd!)>dp?>{3AzMB%y8J`CC2S{Jk?G+F^z zmtQkjU2RgJ$dD^;BVQQh4@`Ixw(@KMKC*qLUEG5hGns|&=-ElB&bx`_FHt6nL-0_2 z@gBuQFCu=r+^92D?QRhwfTBY^aQIdT!E?B_H35Kbh)z2y6<-iwV=FK}vjBp)(3{qg&yN5-wUfKR~Te+*XM zDKY{aaVnvDgql8g+NX2(YySj7f~O%HVA6SwRjoeY{WF zPKmG1lhjtc1o(WiDwArJXN7gPUokncvXlP?1pT=4-qRLaqleDSjoZx&M|PoC*=IiC~>h~5jo+6w*X3Nn>E$zB}#<6($df}w7@#K9p5gk z5&z_h9Je<5-;Zg`c+rg&y%@gHxP$ir@dEnCJuCdP2Kh48*Df^i8~#|MSqS-Ts5NK%pLm0OQ5^h_z=#sQv}e{kjVKk= zR~Od!ke_8YR6#noTVDg!=|$bZ@r_hcPcL9P1iY)p@r@JLF`(p?rL<5Nnd}@444%+~ z4Ox2nfxsBvlQt3~8i0osR?lQV>Hj5Sz;G#F=nwxHC$XU;vXMJ4k_sZ6+kqS?zr_RT z0rpKAOTyUaZ7XgLmh7|QWn+{=mcW{DHUJhp`<27YA%ONU6UTu6WldaTM1g`5K;~rK zA3H;+uWb7!ce+CrW4cvxZ|29W?zB= zu?A%Ae5(eHR1{%YOG^sOR7Od~ksUL&zDNB{T#*c}yfgtMP+l@Lfa#^6K_`Cj?s!b3Te@%DNGaA-3PZfbc{Q=);|4$Ku$iKKt|{5 zY->;F1OF7V!yeR!t&f`uom*?>sgYo~&mI_#BaO4V9C?ypAo>TovoW z(K;NM$Cl}i<0R;5l|ae7*$NJh@K?nASodh>js3nxNCT3x!G2F$?c9r}^|gfOZGQjp zf2#i3ogtj6KVLR)r#!~&*np=}sg+4Wpyrc5<^w+k1x!ER1;NTS6!u+t-}(M|sd@5h zELc=3)Te_BF?)X-`p1M_ zUif?xHcZJm34GtD;rrKe>v;9tW3;%;r0#Bvc-7i}dpYX89#c!2<&WZIh<_8y0ajd+ z#k*1NL}bBJKVC7%^gquBS@r$=zB%!7Os)nF2MF9|AVh79PNuqZ$j-=C)mT8V7a-U$ zUuJ+;`5<$UAXem;XhjG)FP}f02L}uX4LS**Ack8lh91Z3+#1&mjDKXeWa!glz*MnS zdjI7w*MlLmsokiH<7rE}8NS5u`;I%>PU!x)W>>wxrvck|ez9=)sqL#ik4RPseD}r` z+XS<1Sf z5?4#2+(Ld+;c&evvj>gJh@GgxI>F*2J`GB6m-i>3`qrAPP3h!Zq-Rn`7H^k-L)HbN z>Uk&K44#mk^dS`EabrlwDr}w z_oyebIId8AgPH12{G)jxcnlI*z#3zi6eyJFeft1?n+y;UL_R(`n3sQ%lb*m&dtMF zg4P}}oSiEE4f_;k)!UJVJkj!TulvWDu=9w7P}~y+-FK zJt4T$DnRn}KMH%$NsmgC@Yd#TtrhL>V4+FnsviysmiJ>Z9S{8fuKu?+#U1wD>zr`m z>?QP-4u;s*0m-vF&2x0t?b*5?7DAtH0h*QQ-G_E374AmEV>veCEIW9PMMxr7J9w|9 zOKz*etY5&J;HA<)LGV6{(Qw%g=UV!r?2aFm#NeWXgF8*-@h4I7QbPcJi3kDOCmksE zJXN@;V6KfB;V8nZ%Z(q$1?XTDgW;Izu+bSnJ`R<2SkD({R zu0Bp{lbu$w)qGxVaiP6|7b)PP;!#%{Sd$+}H7yj{6$ku%oHb@2g;}UGCb@Z;9_&Pw zQzMPCr-Phm&y{H|9_YQZ!!i5Oo;YyIX6jsXVB+8zKZ2i6tGgSg!Pz?y=?gL-tst;= z&%@M<`2yRl@fZ#VG(NX^ysNG>QJz2DEQF@oSyzdDYl`L#E)OT3?LB>W&zs! zmO~Vh;a>-HxM8->)2gR!LJw6>aUNjAGQ{jAER8k)BI^EUD$Cn}%)GGre9a+f8|-&v z;5g+pr#u`tsl9%C51dmKWF*$BYrhea*Ae*lCo>{?>YMUczpo{(Ip}l2x}@0VUfs&B z=>m)9RQsl#NMDMty?%GaD@ET3I4w-v8N&(|5+DFCP3?F(Xn1n2E?@fYQy zK!)X|lCKAY)4d05RYctf)=rB4tH*qZAyLv#M1HXZj<+f*oT2GHkKjz!)npN_j&$Bq z0A8x7L)FJkvDWt({7*1f;HHJR6A`D$(NcmO$i@=hVrsjfV$P?}1{(*U-YFDB@X>k% zN-d6hRZB|A;`hxDbP0jDSBefXS7Y<3UGZbaVd%bUN^pwQ>^t2X{8JX;*niSF?2YR; z_?K-=pG3lNAC7Vw@ukBYE14FC-<;vA+20DiCI9*zMs0|wpR`}Vt`W?R+%uiH#t}Uzk0H%2u-K)C)5JyY zuvn(T`JLcI!i0}QTC>FuJrFo^fqi!G@YdXgs|N?fL?lY)8Rb=V`FImq8d%%AOTmZ3ZcW9LLyPmH3XJ068wJ}&vMB)*v)iz1g+cOl7UkRGw+QW3+6zhT26>swQbV71(l~A{pA-*WC^~N+>pShm$<1@>^Uw$w zM71%C$q@c^M-B1(4k?3R~TehnE@q14nF0N4SF3Oa(MUug8+O* z^JQmJeLtZ-{OM6x7xl>D5)*^G+>5$SoV-+ib@Ep4bPpH0s9?zKvKRoA1dCZYZ84JA zS!d4~ntBsJWQQn}hV}g?^7y_&eU2liCN#5262bpqG%>7Vm0qzv>3Lo+$?vzD1P!q!=5Kd2d-bJg zq6bSsy@!wU9|o#2ZEC*jx`X9B@5b%xPFzN~ShQ}Pbq6*G_joF)%5mA_u6Snq!%cxL zbPMy4DBT}!LXr&U+3M>CKO|8 z+h-Lp#(4moSql0g0Ow!Q3z`9;Asx19mx;nW5jxjx#!z`!^Dq@ z_v0%=21Pitm$sj`@eL$KiV^RR=DwO1a06!OR)R!a9nGlWT&^mO9Ww1EMl>dB!I!)4 z_8TT|mOqOaX@isi4(tJ1C|sYR-B5_)4D|B)FA-<#DK!tQi>{~+exIU(*ZOuZD6e+M z>Gzq8|fOTgHFcYJVFzY9eQb@q_hhNn zrq12C0O}?wwBI@+?7!-@HJ1O6#W)9shjK|08e{|s17mG}RmfFMKxdwe{zUD>rmptX z^8~OxgA}t3AI!vY^25y)?>|7~YdUWBx3b-H;z2__{wq1hdk5hbOLDB-75+*5#Hq?D zg}ISUnPap+D~Y!%QEj1Fv`uMCh)NZ3-#e1LH;PY;Rw_4dNy{uy!WQ6 zQh0F$v^Nxs380_EE_m&Pe>MN;(a{^^FoB5+QT`EjwKpcgjmQaXAM9+X&xQ$>1sf1g zbKF^}8ZNy}_`!ZGo^Bas|q?JRb+IizJ%em8bRkjE=X7NNhcs(UJaa%jC zwIi-NZFhNBkVHXs#!!n@%F_7uHG)nL zQ=6)O?SB>y~S>+ER$txSJvi2oFkYvC%q_;Ad) z(85!4@OOgpZsh8mdgmJuO?oq>M|m|-d3Rntnw*Y9L+`b$e-^W~oPkQLlglD5IFO>k z9@Eo|iOm)Cyml-Hf!Q7juH40iC04mkSuE^6fpCSj-<@1UhsEAHHpR4cd0Mq$W&>m7vZ7|(x|N1a(WtYCZx2zVXe~FtQ7#Y^STe-g84>@cY;oc<0lMPdFQKle zu|9Dgn#l|6hL^N5HvZk%(VOxjuZad4W7ngvntBIYbH&J!VG*5PqKHs>3%HVoVf5`} z4Mz|%T|1~xKMIB*|NSH_+dpMa(1OVS%tj?#HV&(Zd#8(mIwY#OeJ;tbOKYhkzlByv ztxvRJoxE<^DmW<&eGmue7I}$VSs#=OmyvV4VnYfdh}Ax(S#TN;%SeSJqp|g#Ow5Av z_hhf$&U;_Ez_)d}e8HK~DCo*wsQa;cZ6p8irc5vOZR(y)_}iV$%GFJSW~SEcDskCM zC15(}6JfQ%6%b|D5urnzaQ^|?4t#zJM8|UnG+U2kfO2yh(i9*SiUn*};)I3~(papc zXRJ>v;pEj!)VFIsq%kwj=W1N7VVO78DnOUyF)dmK=UJM`I@-`oo#yTuMq*vlM(xA4RdwWg}xx&2G&FblQ%9#&B0GAg7^Qcb8~n$ zzDZ}3IN>gL;i{ZT%s(s1iRpeZ-LJlP*u6TST+uYTG#4pboYR|_q%XwuDc_N6bJmxi zc5=Q8dBQlY7?@zM<#6?LYY0cX(t;i{;lwMEQdV>R_0DhAUBMDlGF=;QV*p7-|Gc_BzSUw_X`QDdPN0D z3J7ULt8{Y2rugAS(`qR0EAktUuf~E*B>n`6S9pA2Eyq+tC&H~x5NQ|6L^!rQZBEME zJ{hVTJ7zeJv4(r%Zy4z0glTz)4F(=&@urX;9=9iRFcb@RhD(F>w~&lNGbT1q<11-B zII_AZ=@Fh-Fthtn2w5)mIcHJ+nXuSs9xP!7@t*qMea9QJRN}NJ;QkQQnJItZEnEk> zJ2eN8eQ2tC)l59gW8rU#->Kfg!+2fnt4dq6Kk@9Osr^dcMy;MolH3{&d|)2d^Br_N z@vYwPUvJKB!Tt+3+AecZn#!KO79!d3*C26Pm>zfIXW;bJ^ONM7pbuwzMz$jfXaX`N zpuTcJe1jvFMZ9Ls)|_M}T~zR)|A@Y3IA8PNx9^@)>plG`6*1o^?YZy#%O1)ZfRrkW ze6o{og&<1iC6ee5u%WS>AHG}vEk6U~po~iTt=E5xSZ(|)TQHdFw))BMbe`2vPx@3% zzTTXDZf~?vsL0RPP`mRL#m|!DKA*h0>bqRf__Q`Ma0o!{-HzYB^g)98p)1ey z(#fxK$-m_8pKr~v=(zI_&djc(VQIr0+*a(0UkK`;7CMS;tFN5JpTw}^Gn*GKKk9Hk z(Lz5>1Kc&4s2~5Vjm&9eq0}kw)w`b()X`qc;+P95xUoSXM&;x}FE@TOYhaAC6)A&N z^|x7usS9+n{|O^GM>eb^pc+t);=v}XBDhm+MaOx=>F%Eu_^{si%ZlFMFd}q zWjXd+H8la6OIM|@4FrAwEV$!$W}5t)lE@)m>z8*0!i}ON$hlxHnqQF#a*_XP=9YdK z+CJYdTZF$70LrXW%M-o8@FSXZ2Hdodf*D<}1#lK`yIyN$y2C@kTcJsI6^1HE*<`4= zyagKnjuqVkt+7XvYL!#Aq9^TEvBonKa90l@j2ae0O#qEy`y)QmtosnV&kxAi(#;UX z;F-8-sWxlQth~kwidwzKeCV)RDyOxu+W~xq#!MyEY%H>imfF3ha29hVO6$Lszttb! zv9I(Q-~F<8vFZh1L_uF>$kLv9$nmU7&_wT-Gz^Z%cBcriX5S#2ZkEdjtDVtGyFwGd zc0Dc5*&2&aRL`K@oD(&4&~*1^5H^=a)xD40?(;WuMy)zB(6Vt5?iV`{5ScfT=zaJ6 zQHH3RHX-K5@vcb^pRs>e2$oC2Bbh8oi`;n8ZobWAwSLf@c`Z9Bwa(Gb%3VKo2MmlW zf4|q#HC#bU6J*XBSq9Rl`NX7W(vK zjuI{>NmQ~&@;JwSC{|8bEhD}zC`)`8Qa!YK5mpgI+OOg1FuTaOOV{1M-n8*wL?Zq9 z;_uKtD>}1$=6nI_(}M4xMl|!}>F*8^DwEI1ZyOGID(*}|+Z?O{)+o{HgKY~T>up*Q z=~uQfbWUs`<>pk5$u@tD((urMrMnDALYiFtU}#&=DK2C|+c2BW2U`XfK+QSx!UyFE zHGK%u+>{y3u=B$5B?e({AWbDOn>W}y-$b6p)VYO6iq~XRk%LQy1XxfnqJr+G4ZGh# z2O4`fVy!ZlgLG3OF~mMQ?rsFJVqeQ0{e+S-k%Oehy&X-9BQa7w>s946JJ&{%Vg8%t z%M|{fd$C*#J8W(zv#}Bh%gxT5w}${Y1NHwcj76}-j&|r5z|5M;NmL$(4aqy^8~Thn zxvrH+I@+(M?S(Ir8iADgclq_KLEnQvG)>N8L|K~SCFLZ|GM%jSj?PSJhI^k5fRnL5 z%)C9Xb7)a=bUq0&GGYwgBGHl=HDJV5uny{9Sz|*dl7igQ&~ZkDG3a zHwF4#fyQi$v~0%qB$@h5Ch+Fnq*pFfHxl_qu#czj-`t9<*mO1bEXhASpq!LI|ANk9 zAbn;{cx3crs`+Vz`N6pWyXR>ju zd6iqv6Wl1?0uJ@mo?Wk^S;x)N9s8a=pFLftv=uo()23iUjvh+gs#u%74h(;LCBiso zu7D~QppQ;nc%2XLuvgtWkJDL(o+6ZPS4VFbihK1Sc%!ZN~&MqBM#c()w$` zT0m=R9NCBmq>?^Q0~G=5B8#<^V?YQPF9>UphMfM6tg9`Zc>+ z`eNVj&pu7L3Oz6w``G@-q48U8K-z>0JiSmbdWs(}_|Q7!211s5Ykx?~Mni@MdL<3r zH1zOIl}*V4Ukc=Cek-&-9Lf)GXlGUg-HV(_5G-s1uNfwVWQMR!i&4>mI|-LrQbX_m z=%%0|<0X4B9*1dM?VPnW!!D_!57anSB9TSzk<}b*4p9OHhL0$ck$Xm)I(u?UeI_V!$r{4q|$uh8Ni z(b_0X74|q&;mLw^jUR)HAdJIi6)JA0gWiw%Vp9W)EegC>7B^k7F*EYYRU5lfrE}+& zJP6GhUF9CO75pos{4itU|)(94FpxCnTM^z zRw)**e0V(foB>1XJiV~JvcQ4$NL=7qt*_?K0s=TKPIfYabD;&Nx1R{hoZWkP`2nN( zO;UTKv;$aVE7QLCh`QjvAuWs2E9&&GJiG_~8`46|`vRuV@3c1F>Da5-PkX1fx|SwC z1vJ@nxhwWV()PpN3z=@iX%<ctha6!GMfdy7ly!JSkmNA|~O%y?ueV=e#+Lsv>bIh?e0YRgh0rx@sli$_iI=D(OhiE)h`>n-?zI~$F3bG z+A~+=M!^9g_V5{#p&8x0)uXe;kfbzy9b&t1`h%pV;dhg6rXk38J=~TKbNJW>0YOGy zbX-ArrBSZN_&;7#d!i$d9)#kzdmF#G8NIQ)zx*2ptY`@X?afU1_&>CYV^mGJhNqG% z<)8e?oSCaRKIZBR&@LZP_3$dNAf3Cb5>lBVUfZ|Icy|Kp!XTAu2 zG@Mq;Uxv^=s%@9DN_h>4aS5>qiZ&h&nZW7Zu?bVi?sZm}xDn8U6<-MRM#n3|p4pUR zc|K<+Nk5*7nNs2oC0dAGg2`DN42i^c^x_I>{rYRkz>OJDV@bH{T19EPjmJ5$0((6~ zd<^IXY++Y^;-)GtJw$V_IRo{mZ!?i?QR!f;eg#mIU6n+XzMz#zcO!9*$c=2Z8KAH}(L9jrG-^6StsTq`c=-14pj! zID1bzL2w@F$=chQ8seJsS}a4BNF}tlh22v}Zt>Q>W>XLOY(evk`U~dhlDhfq!ItI{ogc z$}$4PPfHvYD)t5kgo6m_hROm}at6c!mXKEz*yTosG={z#4|~xItuW)z%7>eNnVEOi zeGNBRb4t^OuRM(s=RGuL$?oKGxUX-&S|05--m#Hqwj&Yt@}hkzd!Nvin~QsomAAY+ z51%VBPp~JyK<>`=Nk$=WcT5T!MyxmE(r@q+N(%*fUi8L}-J#rnpN&reE!&+<&ITQ$ zoVvofjrr5fqQ^QDoUntKZM}46koPCGi8}D;9c*iKgKWJz^j`LY{2cXxocp%s;<1LJ zz|^{bdAIvRGAAzP+p69n^D&~9A;xnjM1_0UZ77+crm*CStQt6}g?8-KUMG17*16#f z%tJk6Vj&UKxRP(b!dAs+Ha`Imm)iqRKx$b_N9{I^7Sob#Cz%Ru3yVRaZ3h=5ZBVO* z9OYdi^Bafa$}FKOe?S?0tXT2L+0^~!uHPc>#KFqD{oup-tW1wlu>AYQQggxj#Ba|H zYqyShXRG@71n>&Z{FRy*Y>UJWx<7DNN>%~K`_j744X5ZoSrWOlJGJrn zl2OPr2TR9-A(5u3dENp}0Yg&T$3@0BV_P=TcdhORqc>Y*HvM?n{xPH~6xZN&A|T<^ zCPxknvT-b?8S9HrJ|2|GDNx=PI-s|*{hArf4~g35Tiif^%Cr)9uj5>R6(+Dlz7)tM z-??Roz{iZTDDcKU$Gw{?ZT~J(G&BkTnSN<4>|HTi;P=?JGj zUNhJeuGDTVZ7=1-feOm*B$<`Ar(MsbR(T`X*D9;t3&CyaO_zTHgpVC?Ww9M}7Lx#H zYk;}zaS_)&DEM#i7s6agS$GR^f86unRKpueFog#Xb~=)87T3hd^PBg{TJ-rPC=woJ zHGt&W@0ctq!*%Y))5-geV#a^6H%}n!ad5>e15Mk- zSICYAwrQd!so)95m!UwH8ym?`hN}76B`%oJ5`Q$j>tjaNa6=)6TDJY~I<3ChADdgS?p0@*{ zf1sdGri@i932za^TC*={H)O*r539jV=Kh0!w8*wU50a<&UkhX(Tr4knrTAcA7+@N-sb>^fJI#lJUj>-^Cu zn9`P$z}f`$OrQlxJL(K9+vd$)dgUS4GS5KJM!gA|Ge|W&9<5p9e+KP^Mt-#P?d6oq5N-Ufi>l#8Y9)%V_j*D2(6&#o z0>4zKC6fnN@4KWOdGW4c#=-kK{*_6mEJZ9SYLD{|Flj@oxkEv(gUR`4x}iU^Dyh;?^h!#A=Dp8h~K{H zSP>zqOV#^+?(yzI`!8(euZ8-5TToRv@j?K#M=MppwK&J?2ZHR6d%~%rx%HRi?{F`T zS`+6*>?3u_7%yAfQOb#(v^dUxX-}8B)<$PJ-B@%~`i3vX=TUy4FPhUldwQizk)>J@ zGHX2FB;gxSB=wWbI=v@!n4<)-(NUMZLF?MdhV{NK=iRl+)HuW>E!!UCtcJt5tmMcq zqfrdm`pUH&In3(uGYXuQ=Pj|00Mu76uL=P1T~>|X_;Qf$@|?($mE@JA>(CtJB<~WV zL-hGciGI!v0@mPRnjGlDm6ax4ZG=vV>Do`L5cqt=r+p@8i(`8o1~$}}B?NA1WufGR z8=UjEd|*GpfGkZF4v=b{)io98WW1~aQ6Bp{EcE`3G2B_M#5@E)yRb2Rxzh8m`av6e@R>P+M^0`SW z19O<5LUL6fBpB3S8*N6xe;Es+LB~C5W2wxKIBLUNnDVXgeJs7A@i3kS?_3Ac8|57j zXpT+DJ;%^)I*|#O3HpxfU`)X3p}uX~aYqLZI zpbkbK6x7xTkO(jZa&Gcme7$Dx?|#-Jo6YT?Bl50nxu@jsR@Ch@{@8p~12p;^I1I4J zRP1a0CM}xiK-~Hl-7s1=il$yot->6=`kcEB-;Fi@* z;NPzE;LArLyxGCbZ0vI(ronTg_Flp54o?S0?$FjxK=#=r?3;bcpeoRB#HL#hY%~K% zx1Ivrw|!o6kD*iP$H~>6xgWHi^9fzL6zVV01b)|7e@Blx7GmnC-;9REsa60Pe@iUx zAqv31^?K*?J8FKyZJRd$qAM7U;pXcg@LHs<>C9ZuBiD|m6z-G{iE*EikBPoMI|eHr zUkDtFEF7QsWjpaDQVLoE*lL`~&3c=qz~E+uNPbPGCvS%jfRUtl3zdc|de!;H9&jp6-a13=vu|;b^OMk^XbJv?5g{Di0Pd7uFr>ZeC8tksi z54_Y+tcSL80Ihz9?*5G(3UBAjvUyz|xKsJ{oo{l8A$A+^+#m|cuebl^N>=19I6p+YM0@oZht;MAWrGOA;?jWo{C7)j55W=0uR?iU(>^4N8FT=zUp15cNvae`+_*^*f;-6zJqI zsDp5rtG~>>SUQ1ndo7=WBaXb9FxpEvLo$AmAT|kRQB_bWar9+J(UmT{6HKeSJn$@S zQLp}|=1#V`W51L_qC86~f7~9oJ$i5b)9dQQ_4Nn(mCgxTf&G9>O2#DroG{c_rj#vo>$YwB%iE`pr=#wDoJ&6Ll z{u8)d)g5Y^hxmQH#nM#^(y}24r;A8Y$5Ecxp3Ztn!}3_(aa=aMzlRBXFk4BRSlCi* zZVYQz!W>#kgz0ZcF^Pi+b)eIoS;(0}R6Q~H8Ijpzm&tVg;^L4W#pDlf%D*JLcZ&Jj zS^24cWm1r6CDV=?pBR=&4{EOcwc4pvY6*M+yO|8=o--rb0g{Gmo?+OaH!CvUwZJ!H-#Vz#W9P|zaIrU5&>VeSdw-qtsbw+Bqy)q$ ze>{zA%9vIWz-7Z;8){)Rzq-}W{l|Z8%nCT1SuM8=1q^gd?%kRE2Cf`bE8Y<}M)OoN zw&koZ`kRZbSWg zMr3!@S@-nU_P1FTAs_7Rao?)7mqy(oxTS-suYs4KJd77P6d{7f3iej`2nttVrq3D^ zkvoQ}%4VzCT!A^;k1O5ovNf3#dNf!_l_KAadWmsE4_st%ziLi z=yr@*=--^R`Pdxav8;pRDb@v-pvCY9?&oxtn&Z=ZN;ysW$Va5oO`?Z4`uCXD*ds^# zU1aSVuVlx+B>)M^7Gw7<-uL6H1mQ3R+vG`iJ5{nti>G>0`DNcVZg2PXC@%!#^tc*7 z<(v4u$fI{cTUK#$g2A7re9g;rXSI|9cw62x47E{Nw;UVR z(kL`iNy9$7!UCFdH=@;nSBI#i3YhIFn4LRg)ewA|VeQYpe%|YKeRsEpu6Vm^V&{`x zu}3#|4nUzDh^H=Z`7dRtZ16pi1LX0KOkdTcm8Pj$t&i!Nfg;;MHhrRVZ+d0S4+`^Y z#!9YNt?_?0NdhFw;9iL3)5ma0&`hr8NBGjT7ZN;`eW|);u4_wSgRw+BW@q`SCVI+3 zAfS!mlz(Yds35&-lWJ!Y*{;hEI-XT4Z3M)Yb@-no4?4Qcx1{=8jO698CjY}FlS_H= z*_@E8DMy^?_KqW*cgt^3bi+bB{Txz3n$3_c#6=2b$Kl>b}t zApH{bohZB^doPE#>Wzu{)zzo{yXk>7Nh9@x{?SluH-G*Jrw)#vw$QT1)}{X7r>h>1 zPs);8QjgYVp@%S$s1gfWmIwKsbE>dFTYoyUQ+4?*`g{3RXp(|%0=3~V6l=&c_Y-AS~#y7>0q$@gPH?Wn&m zD%D=G&!w~!?Mgi8-!+e$Dy5o=q!1E8 z)+!(iTiy`VwE?q4MwN_(0#H}WE?C{8@Q>M?+R0NDk~}lD9ZW_l>?W(8`3yJpBDo z(1xL5*VEj%0#FjqP<^EK`Z20)ameeb?R6-^r=Qt_;~iVv+jn-ZPTE_{ufLVr0!{4f zIuK{FhIjrNw3RdMd4rD5C!zR-*F3<3^3XI6^?E;&UGzTY7a{+PJ3$(Lsv3mj<-E1B zp}xo~ZZib!7i#W%qq2*27T2+9{vWcbVf0cPFq79q(wTm4NfT+IvvQh>m#>RB(`=&h zAWj*6BWu(2T8vLX$zGsS0@gDtf3*VVAtb}>4*_y~>IqYp>wgBc3K5oghU}I>c(NEy zDypqoZhN|EyK+_u{6dv~*5rfa>46-zEa7dQT{Y-O)QS;PI|)EFQeFM|OPOyfUs~mI zOvf!p3E~K#a2DK~rL!%+saHyW&W_p2$LOK3$ARcDPG`2= zRetE4$5z%lb;my`p2eZ!Niyr5BpRrlRdzK58!CiE$LY#^vt`tM{yH5vr<)~(fsOJb zh_Rz3Etc8Ka-LzTWGDp2?c(oH^;Jmq5;YOOeUpC7G;C}Yu8JruNJLVftFlUzm)Nj} z=T{Vrh1_!kG7Dgl;u?hmq$#}5o^?fp7!%7e6t$J7+zLBB-d|O z9??fT4@|F4ro?dJ56KXksIgs?_g=T{o9o9Zix*GRbLzD+iHE9+X-_mOWQ+R#ye zMt~JaT!1URsDs)nxkSCy_+bAUT`fS1PLGY$pc(FNrUXR7wxQXDv$|$8b%H@#@#&ft zS=n`h=>~GjmXw>**+SE23wnak)qz1{9rVxLvOVY=SOaOI&a;&%038CacC%&Q@e-{a zF7g)I>#Uf!!t>)wHqz~iHkv~4g0t+8|M2d7IfXv|7jJPxIcp{o-dFuGJ?IhW3rNBW1w(?n@q=C`i+{(MB;A z_6rKpffQa`45}bh^6q_|I?)*L8RC3Y9%@vJHtA~mT!=N;#}_DZTw#L zld33(pZ?FLuIz#|rvF*yf8Q_1fI+K)PRdWNB?3fR5#t*-+ya2_fmdsC4^vmmdDS8) z=_>u(T}swTUPCbNy&SvygphPZsf8H5R?s|zrcct*=yy5lQx)01{oK)>77QMV<68px z)=O1+hz-lq@$m2T<$)Gs(l0Zj(&U6MV|_Xq6Wy{#y0A>OMxnAQcRuOC?!PHFjMH+5 zL~4&K=q#qFbrN63ukqf>*iX@vE7e3weagx28|BybW7J5RhH+eGKxRSPaAr#uKc;#~ zYPF|)!h)DGzN`Sr96jD8g!D6?yuhqf#PsTZ+tEWxr-$j7c2dSsG4j6Gov;ZGbTbf( z3sG?3A|{%wA{~U;W&F&m8w?z9jYg7*Kjs@6rW!ltmHY1I0Y)6L>JZ+9%ZpJO9J(D? z?fBx3+gdd8#qI`!ICSna@^c40O10MPD7|h{qR- z`@pQNDna`$KkBb~rC543X}IR^nr1B{FUX%A+r6Ob27R;#_#13k8nGLznQ`Xrs>H(y{tgu1dGt6ADftF|r_-!cXytt>xxMv+~e?FjjRsjhA5XycI(d-mSMG^peT4ibC$d>1Q?~R@m#)=AHM$ zIiPDnDu)BmmRX4jJHn6IdU3nI5ShjL31&=Q zUf*`IwmmnD$8g((cNdH9FJm1=wyx~SthNUAt zuEUuAj%1)0WruAXRm1iW;t-Z$8TXrfbNv^mQ@F&=O#14twH_pxEvh*ZOMEP|?Cn!S z$riwX^-nPt_X_ef=F`oPh^g}&qI@B`rk~qeBu14nQ1gU@gk-@M`YllAdv8@F^EO1` zc}0o!E)^G2bii%>|BI({acBDf<9-rCSw%?>(*gAr!Z*xeNTqU&`bx-QSYl3#4BH$k zm2;HIVU$ymvmtVBa~M-jvpI&%VI#(3Gw%KF`?~JGVAr+leSO~V*Yov!J|8zLj`u3( zn|{#y%l?+PsJTixg;>7CD-A1mZ_Ft3@#5}j!7Ortiq#TJIFE9RFiaZIP67S>?)$rN zZ&W|9d-sEC-foqZWgT+!#;-?SF*_@b=LnLED#&j~qfx~P{8}4m=D#91UmICIXEiKc zZQc#n={M$WSJ^MuKX*2B?*)}3WiKsOKjP)uit9f?c113rMiIGMz$?i{eou5mGS7n# zAFVT9C?lMr8-P30%Cdg74Bi}-iU0Q|X3kBX+7XUAOp|qL(4!=MV_+va({i>JZqXXSwI)oV?GwYsPPL$ z5nMdpu`R<;3`5~J;aOX8l>&%@KA_>QXvlp>gs zDtKq1wr@{;iUH)a9ujX~3Hhq~ANPQr11Oa>Y)EqMB+A!5LXS4B)!iR^r#5Fp72)O=UP%`k_! zy0#hWDqfenyXT!&nXx0(Jkz5lE@U{BsP66G*wfQJ;j za&Ktlp#H?6%BL^$e(&$;TBEs>uAv;*KBWR|Z_&Mn}F&ctM?SgK-A|qs*Gg(wRa>;C;VtLF}9p9B;O#Q?B5j9iv zDqvs)+xC6~&-{H%aYg8pB5GcVBQvD#U7z@3L&q&GOE#TmN9OnG!J;0lV?j66`AJ9y z*V~tyRIZzTV58~uFMkRtJgOXYGkII9xciBX;a#Rp#zfkEQtTJr%OjABAYk~b$^n2; zna+M!>Y|3QkH!;~iSI(rGaJIGC^PeS;A3iG;_Iqc->3inK9Y1~V|NG7y`f$ryHKX_ z_Pn?HTQ2Qcf0Z{3F6q?#ADsEFnt&Z#`0Dg5KFK%PX$PT(|H^v6?orciGNL_)Lv*p1 zd>}1!y~-SJf|5=umUYlpBeQkDJ!+a;=lA>T3O)+es|~pHL*1J$7BWTF!nfp`BG*e) zfjoAA$0e86i^qnw|4Yz>8CAUZClymjL6E)8jI&pDYK<=E$kp#j_9Sdo3oHUz*tZRG zgzGdJMeStq*~X+`kr2rw*{ykXY8}D(7)=30&?m@7IGSfK>iu^~{ItrzrF2~3k*bAA zAm`F^X!CJyVq)Cljf=dZe+ioPwt8HBxbIuK zimq)$dQweDH*4oGGhQcZ{UV@*|6rz_0fm@Q4tCF~5b!(Z`!vm?MQ9vTwuPrE(P4Pk z+HDL}$*NjeOBY%_Zw}4}<@ckMYt4A-B6e;h6j^0X>%YautPPz--~iZzpv$dR)AEk= z#3r0&o)!-tGCv2o&0wDmCE8H3A)n=T&YdEGuF2XCCdqA(dx8XHVx|0{^0&r`Ds3%6 z@wIcfZk<%bRR2*ru~vuy;h*?&A)-0X#?9fJicqMWNVE%R&cjunfjMu=&GzlRkZ8I? z6ku}9`H(0AZi64_-iQkA4q`2r(n*K&k6t9R$$ZjRRcC`fKp>7PIU=!k&(APwHmgLy z_rs%r^t^>C)TL15aFv3GQUAk}t-KfgPXzt`8_8N>9^`!2x%NIAuUGqoC#@XL4ce88 zIQl~2ru3rV-s?_{v|%V!#7gVs`%5hkSsR7=^j15_#XmRv!?NmTW|u^6?EUk~xPW;2 z_>X3FeGM^k{d6BuXjLXT zQhpF|`6xeH`UG*s-&64$%^4ZkLnxT}Mq4Ug39l=M?e^XLS(qyc(g-VAe*Fy}Cx-oXKKq=0j# z@@->764Yz5TBfo(7jNKacT9}RU$4D4G`!a_7NlNo_Xp7y>E*9A{irl(T{}jvkpEN3 zsJAts_ipS?!%wuM7Eac+jYDrD7w_h&)>Rngp>X1a$56mAn*lR7^sn;JMe(NzQG`QM z%aA@DrU3Pog&0+1+&C7InqGmgGFfwWAzu!lR>`q(vWS;InYZHPr_|XUg za_%5!??u`nI+W4;kS_^#4k_z2yD~XaVb=7DKS9$UuEx1Bf1++w*Bq59qR;NdIxgo4XJV7T++mL)6a8MFulYi!E#0 zM1|5DzERGL4Bhmg9m%uE#vcF*F&D>9kdtd;ksRx0JXEf`aZ0G$8=F}_K`NRsb**ug zBtP#yi(Maoj}l_LekN&j>zE|@I+yfs+DtOcIbORn!)KFeTGY_)?0A=)(8rx3UG=Pc zWZ@UF@&vH#T>{cFlJW z0^$u#)U*QN+3+KF1F+x@!vd*I5?m!5^z0d*C55~-IjNH z$1O4nSrVjQ;-CBj;n`T4k(#HAt2q4*#oEmxCZTiZ`X!>;LhWlg$bpgea)dSTnz5rw z3ITBMnaERkWU(LqNa!Bi`9$PRxb=E-{h!}|(fgEd9&vQ^tbsKN$uJnrojZ=0?th_S z+X!u)2jsM@kL|Tj?Mc4@@+^wcbIH6v6a;kp{Qgiqc5mCEJV&r17d21a>skV!N=E&X zyC8gVB5l8a$!R}U;}N&@dL#{))3TDx>lwoCysq2F{TV4<+m_v5TBdIsiPkl)>9|Yu`(c8+ErSINI*<_D7a<>v9RpM)`w6(|VQqW9h*QCo(@>sFvARA1JfI`j?hkr66NPw`!6l9Wyw~Md;Vgcw?&M zsEj$q%s61v)Q?&YbT{fIpJ+m^RAwNDTfltme5e6LxL3J2xd2j7VG(RFkbOb27kYWI zh{SKFEZlv*r;X|kL)47GKeW_|Ok%e<+Wdam831?c>1bRa_wD!e9E{dt8i4t#zQ%av zK*)H2Uo25+7~OmK$umkSrz>~vK$U39Z6~%GS54j?*PdQu*bb*QA($~WJ zrY1Mw#fuLmt8mf!{Rag*KK7VmPnix^$RGl zb}nFfUi>o(0)uh|86U zN8lzP(O`b(lfAWLY6w0rc|xSSFu6cWrhWqypaio&MV2TNY|PnHE|S2d_UrK-L)lVz zpHXKM z0Dsf^6fzy9iI7-_N8tK)o#Vz|`n}cXmRl}I$w4rj4xRa3M!{sn)n4B*9_<(X%vft> zSLY)IT{2wujQF>&(pxerwdx@Z2*2_?yDFwO3(W5ju~$#!7}y%m6&}J1)&?1t#TuXa z@}2(kODwYGE1C1?I*iqsHddO;x(rh@KVEofyBN!$8uJ9}m8#3V7_o>8=I~YJ>LeJ_ zJ?)o1Q^sbMuBUf%x%lYgoV}vG3%3v@mj`pTHnE2DoTLXO4h{;0+b}`P3ppa!u<1+o ziW=JGJQ z)Ok3Ur{51yr9=Db+2Jr}+nf8}!C#S^Rh{UrI0=9aYxD`D_NzyQ&2iP87GRPq<5AuY zqkhr<7Am-`+XX95xy>uZ!QG&q1KQJmpDK)Q8?iBIQTnN&QJPUM@@`Ze_aibEy{F@G zWbfRE0a>;V20r#NwHZ2o4muFY&*FATFzSLt@3mBYybx2vR%99eY}KYoaRKV=Uz*0L zU1*_;f`N%P5w{nr3RR< zZb4tM)qT!6v1R{QHc_-Y`-~0kG}+-58hH5M$uR=YW&Dmi{M~$BRBT zyqO~0oZ^1Dr<1BHd^i}}P!KY%u_8oGsJfo?jrcXxZ|qcHc64x3f3{#7awQ!O1QP2K zq|Z1|90=i=gY5M7M{GiP{*ix$sasjLuHKAgB|HD#6^9uTYh*w-WH7StGi0YE%nmG? zo^{B66z1_q4yaGs+nYweZ}Nzj1~15#h$E67EE&1&wM&;wRA6*%_gm3_H|gj_`sjc` zX89wZ@8bDoSmE1`XN{>$nIH{QfPd>^GrkAF~6EYw|0TSK0v(ltHYG7 z{tn3^iE4@vjjaE}!Nvi>i(rL@*-k-Ps~dWcQx|ew>4B&6P@e zVn7PaU@!@Zv^;oKGS1VNAF6IY${hQWD7(^u z5KLXsFd|vy@CEB>scSu_%{G<0PvKxi!tuuzG7AT)TkikE zTpA_X3o*N+$AwF%OD6Q& z*Bn_nJSBBdK0k5?;~Y**-Gk3f)LRrZV?w%*JG>W#6g@Sls?qTIBt`c2Z>xRRcts+A zy3EWw34P6R7orQ97?skY*vfz%!Jfnf z)I1X<-*)1+v|mXDliyRSCw^!lW73D#%KE_8wt&}JK}Ax2+UkyDfZ-obp5BHAkDo~! zvoodBW9Fi_Kw!-I8XF1=b0>?fvY_JcD@mCcuYu8}kI8C^EyG({j_DlyVm zVo-S^$Cn&>B;m*Dl284i`9{k{MpSioT#XUot%i4QSxBG#RW{GfE2#X2qgk~Ab1NWq zODwNo@1{$th_YLS18GEr2F^LP~#0rK>dfw^O?WhDb>cQTz|SnCeKlJb63*voT2iBU zh83b7UAiGdz>wY#f=IHrfg82O-JaUf>?yCMKzcZH=W09HxtYnk=YBpP zV3%S>bI%>>ui1%PcIy+lxP5ufu%BK;-r2svF>&60I3KQ>y5T1&c)@?I^py|V<5AiH z!nCue_19vlZoO7#!Ux}ZqSLaXcz@x!zx%($ltrQo&!A@)aO%)pM=pGiQG zr2SGP4s%dDpgYY8jQ*?w-TuI@w`P36Bis#?LijgsCe}qB5xVEIc6U(to1I=j4G@0_ zHoY@GZf1^jH-6n-f8wnL^)x{i;#fU!cG+8Ef~0RM-*{G;dTIPGrmd(Y3YRYOdBxDm za))0{W+BhBYVEd#N-Q=Ff|k(4yb?1Womjykiyd@ZtU8fLkvKNi%%G_h)h*eKZ8c6mroK3&my?v;~NFVCAyBsvn zh<<T z%8kGG-u3d+2G19xdB0GX&>cE=xQ8W|w4)_>SdhsyYAuq4I;H_XCQoxkq4G4=)96>C zcD@20?Dw;b++@9vz>Q)29{fK2Jiyjx73;%VKsR0g4vrrr3%rJZi%go_xoO9f6%1_o zjm{7xOyLq_3NCl+{mde`y^n%%lVCj2xldBsv~;NP+WkU}s3fUwPlXyAzs~Uh_F4Ta z%;+TtkcqXH#-f&T`Hihh!}o#YfVn$}lgxMFBfont=qd9fa)WATz`LnwSo#jA5BC?41k}HO=?j<6wfp{8nGqMH6v}%# zooy!;e4$u@ohgzjNC71Ws#`jm=CGfKaIo-1(#EbKL7JMrNUSTF`xtq!Y0-pVPh9o# zb9}bcDm8-vxB`sfB}8<5OP6HGrUh>oO(O=ZPdnkN)T-*@mHuvt-=H)dT~dv<;twj{ zfnn=Str0Mns!kSg?&$DzBbx|Inynl7_P@zHx20-ZLvDY9AFEg1>q*}Gvt+QpY|h^% z80;@@rU8wC7B)PNh7odPZTrj6*nVyWm^bQ5i!YvzAw}+Pp5I?_H9~UZ_A7?W(d?0- z&LRG~VM`W0x8#c=tr_xCr+1vJT{pJ>XE7);Mse)-MLJrWb#142erte{I~jW)h8&Pv zt!U2V{!{=6xsz5p`^L4Z0sMbm1gz=5$YMHpcebv6rlaeJxw=Vch`K_4yh8EWLMDay zeIl!K1ywF(_(Md3*s)^+CPxx_=`^? zy1XpLHy#G}LgIwvLpRV&juH@}b~_qhwA0!C zp+W5*5e}DiDkOB){0ZC6KQ{X>nXas*#rp%(1P&}_(YlUVhgE)>zT$%1wDQ3HHTxCf z_eTZ8vIv0m%l_`r<*h$AZ(OTic+pf(Jc`TgOSsC4g=Ea#WemuI9E?uY-j;PuYLnBT z_8l$Q0YDhnAhITxX3UUaM`0l>mzZ~2n;Vy~!jxCDSEAVC0r2`hZ7R){+L3(? z7yZqrQ^(Dca%fQ-SZTDCs0JzOCn?~dSUKG`!{@jmnseXjlr20LV+I@@xW4?%ZE}YedMjBV*;zNlJTC|0rQJA;!qcM0q-IGnuyoeZm*S zERw{H`7$5}|K+<69N*OfX_RfUFackpfd`eV`XR!OoD|+Io{Qn=y6t4?J792^PMA&N z+K>%)XX=CRfi04j?$6htf5y#gCV%WGPPtQFwz@yS3bPozH!W~qA?z(Ek*V;x^7uMsaSmxhGg|{8F&Z(S<$~^Qxkt(7{*P+xKg7z3M?dIOM?|gJIOI0GEM&girO-|NkjF@!XT&$eAJzmp?HJHl6{XK+g2y4?1l)<1QjG5YMY>skTORug(aLaAo^Abj^sk7U^mJ_xcvu%{DYIN8p`*Q4gB-1bwbGCR zUS~s+;@Xm>x*i*6!fnxH%veTp$AAci8`Bb4-k|};7|5{vTdw>yRL00IevsTH&SmfE zwUBR0%O0e1qUSs14(xjFSH*)j9~4T>`{vgC)K%aW0%V80Ud#xhzF zhXx#8G5pmVk2Vm(o-O2FolC6qjc)yC{klA1V&7{r$2VZ~`Do)!zM_7n>|3#KcI)=S z&=T^prSt<`!~bxRGh*Ang;y`J>|V8uF(#rml{R8Dro;O?hp-?1UXI78h$N3m7~ix+ zCeZvVV5&CdP^d&oNWU6pp9tA#H7Jsa1Gy4gndA7zw`+NoeIYt zv>j*6J7F9JZPunyP56D@n01PIWMk5?t{csumhKvCBN0R2w%*^q?M46@M5}MWcemop z=Tb0Wd5t@i>Ky&ee5Qe?5f%Sjf*)(`m8{w6!|)%l>5}Z9+ot#g*d?M3?ID8T4^7rt z*`AoI=$!1du3MMst7}>$%|0QK?8;f+X@3=>-CA2q%XC&{x8e`JiShY+b0=wOtTLeM z2JtncCeZm}p__lOe;-J_(q>;~q}1Nqz9!od?`Ms+vcWyu<^fM7m4 zy{j|m)$YKw95m|k(*!0U(0FB#1QOgz+xXncmII`2$i!7z&Gf;OTWYZQLoc)j_f{i4 zj1By$58uFVrA_=56`uDqJ~vb|)E0UusPLDwnx>y4=^reZ+{n5*Ec- zAsIM02mPSSjfs?0gqer!MYPa2C;p<=D5{luf6n#37%aSz=5~Dc8u7)ep_E@jm*O)K zLGQnH+&GGQ?XDg;^3(hrPfXpzvCB@3`(onLlu2T1f=vN{DMOdI1R%$;uZDcJu+1Qb zGAa+Nko?x3s(7#3zl6cDZkO}-hLRDz&V_)dMF7T1XQZWP4qJYIv(Aj;Cc9H>zKi$D z#tpsyW)h(wJ1P#pd8r}gdV)A;bK_O&Cz{|A ziYxt^I00BMB8Y#DI*H*Dv-V7>ntg)m*|=sO)A0a#sGO>9oc-^+jT6b9ihOj{G)iPo|cH8MYI~vdgnT)^7*0?<{43ib0J*_ z|Bmh53KQh->UqkoT~BsjXGYA`Cd2)m^})YXx*r&N59VO+8k|;9YAJ@X&J$&>hc3JM z;-VE@q%PR5)Mi<0B)N8%Gd!JF$;`vg&1z82nz&H+iJpH z7FZ|lE=hvulmP|uh|FJTmD6VIhq{@ZGlKZpZTR02d=NW4m6>B?LRClO{pGLFJU#e3 zOdolZF~}XG(N>!=&}`8umMOBP#&qQ**gvs4}me<0{6B zo>T0g)0zgYEWJ5_C*_(ucJJoi0FU?rFd5G^Rc_$=;SwRW2GU>WSaHspp^JK}ZMfZp zkCU1~2JQ;${ZOUCD>^(ukB*jb8DQ1idC13sa8G$(cT14#lzPGoyTyQ*)2aLw%DPf| z)AL$^&DZi9i{Co+=>~iYo9#ySQLVIaRA_!6io9 zzO*}*!7?dG&M{iH6V4ilt5wUc@iCVjt}5^LJrQN~cuEUAPjj}%gAZCL*tR-BLIV<= z1o!KB<<&!f`TTjqsb9D&-cgr_McQ=lsl!f77m^oHBFF+4SRs0O$(3Ujm>WHw1fE!` zHi9ilf6#`Qybo$8NO0^F7?J#)50uLP%aVR552fR9M2hi!Ypo8} zwLmt-^CN`|Wb2X)rkteVtp&}i%&PCyxP(&60>i+8o=t41oej{y)A()2+%J8`phj|Z z4zsbsaEpB2cxg@tIqz*eS=?OER3l)7+X$m>bs|}}fm7;5Y9>eU(dft6l!aY!dkx5c z4yPpxO{}dKJ3}()C#&K!PiGFLOX4AZCbC_gPIv}?FgRDL5PBg`!m**ISV1a)kgL@Z zS}wUGk!gpiAf7~n7FK(KW4+G08j>(oNQi3#Hsm)Bm!O*;nFQkTM~by!_^xI-TekA( z2%lZ!=}bPa3wkfQFciln=dMSm=j5sJKCMh(LZdc5$J)RvI}O#NanHbQjPt>4EA%-S z*TzomQ{|dCjCP+3H8B`mMbu=4cADNJ{0(s1Z^%U z+PiDKAkpRJmyy!wra+pTz!7T(Q5+L%1KPhK&om3sNk<W%~1~65Xt4ja(zd0eqYRdq1;6Pv6G=Y2$mWK?T^5n4=^ct zU~0-$9i9ME(Ey^}j3L>%te@Hx95!x^5Db}bsKOVnzqs@a8O3>{3U2-X6F6^@KY?%N z*EzTawP~c}iw8){wEb{WmvfG6IM8$HR!;n#Q~c4y+sob@%^A3HJa~H`{-oD~z`H6! z(p8&rb{7HLAEnNq+0vE~CXVi9xAVdJLlP$M!}vAQjHb|c76``MG@`}1t>=iRm+b+L z0qG^hfK%_>D1zMu@|*W?DE zV|R28GxF0Q5W^^a?o+{qEb)6q;Gw|JY%@T`2zt0{B>+k&Tu%Nqx%9fzd9@m@YTbn4NU?A;+Tw&~VF{2e~Q!~dor<2O(q8Q3|Y z`=)2NP7^TOGBDw&Bk|*foI~5`E2Pr^xY!H05#d9278eAFV$;K{#9$^o&{zr;xy0`{ zF-$&L$uhmnFaW%s4#j^0x?K$Z)S*9hs@q8r5%Gy-@@Garz{~yeW4d_C;_9iE8!xkD zub@K)!i);`X9e{hCD<#V+x8Aq`o;BYy(Avn(AGr<`}A==%t8_A{!4{zFCm9n9j6@` zZs$Wz^ooU414TK9>!7W(5U=>4CE5GphUr~q)`kg6i=SM-=v55m7E+_a6p`61lJi~b z6yQ{zz_>8^$A1Lp#vqg!-aW|*|3ZNy1FTM}k{X4U_h%w)oO&f=^=q?Km%Sr>oE~d4 zi-GPDICmRsTY{cv&G$h}Fa=gG)$|7!EO-g|*XbS#n_Ps#I14 zRIyAI>vC)=x#``v0LVVa+5ME(V;YF?Bc+1i8 zbX({Hhv((vzsp+)E@Yw&OzZpQQzDlB}&ookw- zt%zB#fmzC_AQBXWv&mW!Z;4vL{(6%E_#1u0O77mH+MFA|Alh+q%cf%5z`Z8LVjwXuaJOW@sA|8GTg= z(0L6f@XfHS1Hj;fLk{rH{E{8JL=+ilCyLVT2IT3vEI=g;IbbylX4BSn=&lD_`6L;b zO63o%=S!a7|@*Id>-`zX=Veza1hlfQ9gfIihy=o&e#^FR-_tU#AOo^)#_6yM>%_Fk%3@ z{Q5j~92ACXADHH`88reCqVw!WbsB!uSZZ>Oh{v^tSqiD&}k<@Cc*$146R5m z{)jb7?~p4_CdMl`I^sbc7rzXnh%t?2-4fY?NpZ+)RlHrmM3{-%O_PNm!~ROe&11poxJ>n z5l|-TfE}FG_*(FNuTS+P1ZkWU~>f9bnX;cU`YDZeHKC4zf{+u5B zdZLD=Z+3tALH#^ZXF28Xp21-UqgFkQyKg`$bZ5b&Iuqs0HXR7vITtQy4EGNtE(Gan z@&suKdvJ9D`a=&qLbMC2egOAf_ao4czFxI5R11+`FdVo9El;tKzf^g4NCNXMN+($x zZ!Wg*O9Tn`W7y!E)pnZ!~ zg*af_GGnTCu_*k8?7FuBXM1W#Xwk&fsM;1iUS5%@ooA?Yh-{`qSGC0>QW0#qRQ==; zw*1E3g`mmrl+DsLQThxqOOxS-IzcAjtpn)OD7W|^FU!>9wHB-ke`;hcNCF0@raY?otj1zXUj#*7aLLfGZf$V$Ydac3{tBPaPfIY?QHo` zksrxV>Q!W50 ze61rS_TH(nkeX%r(Q#|ebI4obx@nWEd&9}k8DghTvWnx0S(v|>~{Brwe?w`&Z1?8yp z9w95O)GBJlW)|C}dpxekF#w%59GjlS?HyN6=n0FWe#Y|2hl@~&(0hA#g9NB{H*@AC zh9Plbd3ghk2ep!;kf%N)f(q11%D8D9eirg`#gV*6`=g)ISi%pir>|itMp6A9nNNa) zK0j<^#*^6M8GG&-IQY*^ht^83nST`X5%8P;iE-V@e}{zwq-L4H2Xko4l8eTUkLF+U znQ$}82GWG7nN?JVob(XV-6bU8UU}q!j5x;~*qE+FRVWS|5V|!W z2X)UHd}L?o7^z%*nRSjX%pU^enZ1XYn?jECEMye#r3uW=!K+n1Mc}J^FjKZTtF9kA zeXv_!=WQHZt_CtuIomJ-cD*ulf-NoXYrs8V%9bYU&|7y7dnk%(!ITWsFr$E0hSdPp}969p@+k7dy1Qd`n(BHZvEO~{d=dk$2FY9}g5vT#R$a>^f? z10Ej9)q#DQ3iW!Nwp?(O8pTxHnkY_O|3F`r*M8rXF4=gLTo1(mqopHSq_J|T-~B`Z z;Iqb7L3e1Xj*ra~xLuBC(p5_ruU~O|g~)_5qJs(U2;WKHuPG)<@xij#@}hD zdFGW$s`QSD#)u$!HvbC2WdB-LdZsZ?rkFb_I^S=zObvZ4$}NC>o9%2VGg3H(*v0?> zS1qDKEGyja42bAaV7#%Fv?nXUKd&BLai%71^^X%1>;C){F8wJ-A+TRtj-S2BsDm}I zzxs~qfS=UtAJO-e^@xwV$uWIfO=?+rrto0h^cYt(TJud3Wa}%}HU!ps>r1x^x?GM5 zD`x*R_E0POPI~W$_P;mo8goDATi3|g;WC3Kts~2=Ey^VzA9Pw_a8l!)S?6vL9(SnQ z(`5i#7MGChE5)wL2;9A3YdtMVu|Bx@ef7{V*CdnIO5d#dMyU1djQK2=&pbmjRG?u9h>B*UMMRa7IU zz9Uc{1LCi7d5+@i_qPZ;{yl7+g}1F;jQYk}*YqCI<@9T^jGik~y%KygyMt z@L&Qj+)`%dFEUq4I>#*PcNA$a2P3_W{$!WCF89RIi7wM!u$}QGMJ!SfqguMNHx7Xrktpm0uW)C(>ZAC({C=0DW+$M%&Wz78ooRhw9OeQw2X^kA6= zLNN;nUB}%~Ag=lf{JsqA{rfGpn$LNi5%9gUd{*V;*vG}MQva4{{1xeYUd&ld-sr*A zKbHdrtIC(oKcCBhTwQn^czZ*Gx<(uUz7TJk@N zTe+W}>vw$wE;9t#@QE3i$uuIij4iZvg`3QGgI%ngjg;E|i-Hy>yeYq%5b6;@{qO-K*y!4{9FQir6e!wAD-Vy*6+ zeG-$AcKy+~MK*>xU|e3$D2BBm-6GR(=SghH{H=vD+CnqgBhgEJI`ivuq1Qn5(YWD0 z&F--Fqmv&Y85&-BEyj?lM!#oHRci)n566oB>p}gyMTb|&zd4XuPC%)qTkVzJx%PMz z2iAUhJ}Cz7`8IqFun>k3PuY0gY3|pVGeOk09Uz-f1UN4;mTT1bvF1!ikwmE0++&DZ?QjLMZKR>Al=6C%TY-%*gS8z)2c=0AU0D4DO z5aq}yK>ME_aIN{7g!E&*zl2iD(HxZ6%F(P$tSGZr$a*j8X|JFt!dS6}ifo38oAKcy zwQ4l5$&^qY@&t$6FX;6%3skf30UZmyLl@V2w0oGH>6;!CmQviDj(ts3)Bl{Y?&YUA zm&HjrT@dIT>{lEAdXF`8WAd0-K>4S5r29WlvUD`gWpH2WmU5XvBQt_W56CvVd;-R( zg}PkbQ1L_9&i4dw-gq}ue2%o6x!PPJw2N5CXiT+gDb0=3e&5LMOME(YA=vq_!+G6i zo_zw!R6Q(fDE3@{ws2QEh9!JOT|Q!Gb}SF18?11cm8AA`MB2|_G)DiG0xozW{aJ3Y zQRE*X_m&+~#2r!^BGR^^r5p09zQt%~K1X|81o4mR}pOD>uUIzhW^p$QXQ?jBc) zAguofyN0;KSP@rTaqb1(;uf^&Y%D>}@yivi=O*WW&~bT#DkaQzl+Gw!0BD)a6ahOS+M3O zh+_ul0Xb?VX*9D38C&E5R(zyX`fYB9&MtAAzj^ANbH;w3`h;av=n$Vx+;FLyCfrM9 z%ES^RcYZhY!LL&v2DyAmJ7_?WT()#H0WLyvYwjZXkf0aXw`kj~W-G@(dD{`MnwGbH zFAZK*UO6$U4ep94H>|Hw_L}U=dO!O$@4KJ-Xmdl$*}uCB8BHf+HP4fI&RUB=Sl+Uu zeTd`M_)WjEG}h7KZg%H|cOt32>cH@UIur0y3PK$j{T(&*=&4OMIjn_JjTJ`jAnx%qXLuV9Zx{PQ}yuHJGRDrL z9D|sTJkizN;92%w2HF8W=&5~^@GI2KIgoOY^63IFJ*EL8o_+mV&(dpktRjy z)#pb@ zD6x#Oze0V6lxmuQHqE6CVuSIq42dL*q(sjAc>NT8_iBKGfsEOJOLPFY=k(;~P_T^v zD)6YoNGg3`Yk974TS#kO4CL{GrdU;40~iM+TljgXCf{faUCn~PJi!A0%X8xP-WsuV zqRYY!EM4w!F>E`ubga~(!K&M`tmID2rp@E{hC@ud$M)lVb(=7A?s~n7#MLNtZ>cKYd&*55y)QP}%!@58S&1uB4XBU;R=PYzk{D%E!uJLu`*#teoOzBJNIMe9V-+ zw7usOVi5=IGZ0Y~!vfubsNV`&g>D0Z5_wWf5WW#Pmu^%+3nPXJXzF84adRSEoG7;? zzMXFy<6UF9- zE(!i0vfe$O>Hd%ZcL+(WQsHte370chn8T1tl0#H-Ic5%t5z81hBuV5DIW&wAO3vnd zNSGLNA&1TRG^Y_`Y_s|8^S%9k-+z98?f>`fz1REod_EqJ`@@;ReqE!Q#YE)vQsAqu zPfoWUO|S_68=8^a2l&VdIJR}DHO6;leaiLZU_!^nl`To1|KIO!(w{E;F>XD!rA|fqT3`jP7Z~wEe`TH zT`&}BYkG2l5bH+*?fEN(^KV>FjDGUoUZF$Q##(YS8kQ#G8CC5%lB>Z<{Z&~#8RBL% z)^dwkP1b#$U<=&ayy*}*Hs|FUo+jovzA56mMKY)#LmTtKH`22;idyN+P0GliTBdk+ z{qcU~s6~drBh_p6P^0Ni)mvzRM}F{VUvc}d*xPIH!)deC=czgayLYNvdQ81PA_7|{ z0iL4(i3x26Xl!ASJYM~MIaK(x&KNv=zV*qfz#hx4`8GD_!ZHXbE1zhDRk*g39j=yj zZBK>-yYPg|lY?$Nl`B{f+549Vs%GOa)PXtOy`#}>zu;Uj-K!l|_N^u$=;ALOfDOgw zJUni?`_B*A-gkT=IkHiMA3xv8HZlr**o+gSh6|Nr9xMC>IJN541xsQ6BNFc?0(C8l zoWff4kay+$tQo)91VF1I=X~-QO0$9Uq%pcQ_2!yop@CP{;jG)R8I^kuXkWzaV~Y6_ z%w>MIS`8Wah?EB8<^)|$b|wE^qFme_1r>By{EymQmkRsRwxvTHw!_ZnxiBkCS?)0GCPvtBe|5R3=8T*UP6Uov{GnmdnNSS{KF3 zy^ocH$ua!qkFfUo=ra6dd%J?`nV0_kAQQTq0!AsFLU>li%9S1~{XiBk0`x?3|5zb>17$b~*( z^zy~?I5K7YAoim`>4Ytk(|^)%*p{#&n18a@DjEOzk6zTs5*9`-Rw4Wu4CR89Oz(60 z+20YD(vmqypPA}Hld|i6Vi*nq9g}$&j^KW)kvEljcoFxB>*UpJ?c=|_&oW`NA7b*8 zOa9haIPi8&c=*d{Y4Q}a5dW^g^BvBiLM=-l?i7@`)Io`JKMA>m?V#?;<7#QO)*p_( zi%@SEk&K68&7KQ&(HYp||#P=|7xi!gC7u5-1M za_sgsDyUg?GUT7mgV|kwiqoPN^+8g0jZ7U1a$eXU?C59V*B4ROj8JVZ|wGdwrDF;cZdy`!{ z8mBi>7j(JUA>g1>&4OZE4_iR7tXux#!o^v8quv-!YBeWW_~Mvn#llt^!_F!^T{HEd z+r-EKV8VFw9#~TH-HbClqQu%fk)29Hb!5vE((ND5O!3w|6~E6me2zDzHA`i7hOHrQ zo}6Qkx&LUVQg~$q!li5pzTN|r&4}WiN!uR_@YJ8^B`sJKQ5dj4sd_EFD zcX_K5EfF&@qE4vASwX4L{Y; zKV>3L!@FfKbIW*H)Qs^9Q%tmR&?SD4^O=B@S~L7yXEF%f*#etUdi&zTUOByAa>clZ zMXD^t`yF=Lt51XD)~?s{X{kbh6AIjT)`C&%|GZ%A34D?qkPk{ZaLo#mj#zE@KlHdu zXtV#9B1e&CyV09=80~EJ`A6r?#JHO<0f301YQ*)`o7|1(rZ~fuZ;nVcr^P$x3fOGy zekAY5x?UMkD>#c-%JdlK5gU#}cen5 zk&bgl8>crFcwmIbki!_^We!ydkuRM7f3DsP-h9wBtn*#T<(~AjaS6HgS6_I=cfOuM zIz)8zmwfF=|LpAR7A-!>;_n$xhGCjCwoys=k1MtGvv&5aF70rE8*)u=hGJ!n{`-Y9 zT*42Y>%eg?XSS9I5!X0-$53yg!jiKa1h8V7_-Wpw#T>2!{+!L0CV@KHoXGiXZ>y5S zww=Ky(}|0F%>Ysg6OHGpb7-7fI~%tMd<3>@qezhcp}pE?ZZhL4__2I&?J1v_ z>NLK;cR+P=SslkC*+Epkj&|}}NHl_)T5N4&6L5Aaq_Z7VV?;2HgPSY*wY)^~Dkd9x z6kr3SlIz6dhWUc~ODw@w!+8ixnQwy@S$bkI8fS(MC|Vp2Nq}Xi@GL-8!a(a}2HEbi z8=kG|Wu`!=PgmdTZ9NT1s%KqT+X><-9|*&IoSPD^Bg&g7YXN+?9bWF2Y>oYQQVh#~ zZ^r4ocj~|BmaI{}-5P5j<CZn6YVq-7=8;{nr60T$2=dO%D%e}(nU=}i)|^nS zAiE+N1peau@1a6IAOwq@MuYYwcKm@_L5(s6CW@q^s-|B$`&K<%Xg zQPmZmk=|A}N8Jf;9KZW{Kd3*>*Zhohf47QVrIY0Rl{dCNdzcp1rnnMlys7Q6VtB=a zCyePbyqVN78^dD%t)uc$;;V!60p4EUuh9z{_hKAo)%(RE5u&+jexap86)ECL7BO$0 zd5or4S@qXnsIJ|nE+khdg-H(kHa_=ueW8-#8`jyJcp$;_QVEUF2U;roTiig-tv z-}hGX?uJ$VwJl*!&nb1{8^< ze9$%usRvYf>gT_2+MvM+g_dHeL6i5wqsY^I+rCQR5uufRp8b#7x-M(Pbe=IwX{pSy zz})Un=TT95+3MHTiGOA$v~SF9r{6!B)LsPVEpw;(?MoH#B$i6FNWWvNIgz* zhgBhnYduYdp%ZS>_5~(O@;3Y6DrJ$Rupzor3k4ozWS>yf8|`b?K?laI!l1jW#7>m}5) zL%8>>memlcBBaEH5EFnE!N?)TM5_F$&PB?@^6&>q7k4MjwkJS`e>s>zPi(4KtDIgN zVT11~s|K3YCtCO?%HU`Lgx{*G>v~7JJrL}ScLm`N5ZsL0GeH&Y_P#lc8)go|6zcN| zfl~qXTOBz2Tv^x}J@!WbX4O_WKKGSD=cC{yN=&e7q@3o+-ep`X!{HNnDO5&@CuqnU z`!dOu%Q^W+oornuE4pTs?*Ph*A-Y>_5rzjr!+9VW6Qyl{wR^61uw^>F!m2Q$BBVN~ z3${YT*}&Uq=(S>*$3xudQ?3Ql1^D#Qk}9+?;a+$rND~mBY#W+%wel)?%l-10zEoGE zSHzYn*u2Rh6ZuPnR&=r3pyTHam;B0ZzX2G+B$b+IBDs;9!0#agdlvc7IZ50m14Dl@+m*wK<&aO7szWVWPVcO zUKAt?{#3jrT?+($;|Ep)!}K?YGdIpGOJpV{Ai$rNKJ4=Qg*p9Y*UM3^1?bB1a=I`F zy`LS|CoZx{4T=nFz4vu?FZbp8A@h6HXj2K`(d@Sm(o2VJWw9Bquzytu$5n%0S**sp z@CZ>QdyMpW#4q+At)ysnXW$gggM80b5I+{RRSiG1!0h);#rxONIkT z0S}g)xT)ve;zA8pIbot^?#3`Wfo9C-0C9a6z~@o@QP08Kr6&;}gVi+&zCh>;TP|Wg z6=#$83f}R)X7yPu^lXi2&V?~VEQLEq{Hl=cIE;wwP)In(lKL_8f2#$R; z5s=S1a1EJlT4cM!NdMOzM$0WBz2kfGVmZd4X`Vy%@1Rdj>JQi-HqjhRr_<@^>7gDx zWU}YllSZ;?dv$_Pg1^70Fgt>D`JVEgg77+<7RL0b4lV0AF^F=GSndJrhDLLEgK$vl zf460Vm*Wxy=^QxxRvb}v5z6T!RL99yrIFsVRhd+8T$z4BWN?)h<)+X5Q4w{%NfR}; zuGl6ivALzjpsZdTz9VvJM=)*cVr@l7XYJhY6p>oy3vTUq%$1k_!E8-jkD>J(!1mLm zf{)?9?fpMM?P%IBz@Nnbi`bpf>54P-kAiAVi;uF`sinH~JE4Q}b9dPvTvS{M4N6%x zzz%9$7)vz`!X4q8-c(#%Gu}AT8mGx{VnP5pHSdLjPy(ODkzXp2efgz^g#RWkU)#P# z;OJBO$vrmQWIBs3)m{^6ZDl6&y_QPHG4<5iD(SR~bAMEeXnCeT=QLctZSjG}LusU4 z#%j+^yxN{Hu+TL}Eq(?Z6%n=7f+)5G()(%YFm^ye%WU$s*2&d|Xo?c{gT|}wD|Ke; zG4d^0xp8IRRNh+k^6qDdi& zTa#`2EJF4Zk`pa}=389)RB_2?2A!;uBw&6XOH+hxHs;^lT1qr|m&ot1bgtK!^t#XzcJX_h*0pp)#Ry*LM*+7ox?U{Smwe~e?BIRUcT5@?Ci#<(_omk z&~Gx>Lr%0JARUYM_rKp!r&MB8fat>n$EWj6hO`5<`}37}h^x0xrmQ1I@Az!3r!}Wr zuc2+{@gV-C>}vKB20-fdGtG5=Q~RmqBFOCr(eLUQuWNVJorS0T6D{Qy=yLpRR`=mg z&3)V40QlQ~tF~7XB$2PL4}WBO`C#lFA+QLVv9m1o_oZ#&xoRm_h#+kFCh6O~*=?+= zz{2;)h63{AlP7U2pB!zHagH`My_wGuf)9XCCGp&=FTCxC8`zL7=8Jo@XoI&`CFW)% zIH0Tt%|dlT4kQ7Jze|W#W*cVXI9AQC@$rb%pQA$mp%4=L0ABv{-LIdUFbe8yM=Hl^ ze-X+bq7;$WM$Dc$`6TNl`>^Xk3bMKHG14@akpw<^WNUA<`Oj9H*B^BHewHs4_=K*1 zl7&P~{O!Vecw)L(fyg2o225O^Ul@q{t#+4*(;}vT{)_C$s(mkE61;Yj6Uo2)1+Z_J z*`(SlWIWz_rO+rk>0-pwLGB-8v==^a7DYWHwntYRuYXC-8R%ZAmaQWh;Fbi7l=ElU z*C$!NBXzS^VQ;4s;k^QrI)^^ee{8l#g9T6SI}aRkH~-GqxI({P+HJ*8b^Y~s&5X^J zi`BnqDYW}~QW?sCVSZ>zQ$G^td&4c@7iprnV`tRt1=Yhjy>pg;=YOrA%&lT zK5&=-mcr#GLqPv+!=0)ulWK=ThgT%Jod%91M8Lx#gu7coy+23Q+FlL~w&J5yFlj!H zj_I!;$q0`EcKyf+35cScCfQ-MwDS#`3fbqV*V$Lv&vHhk?jnpkB$B73)gk!-pj-Wu z-b;5Q9ne)$b3yJCBI=uD?j<*I^pfjC(3QhDhe%S}EHbJ&h+S%p?3E_nTl~ANY5S#M z)MQ|uvoSgx5AiOLuK5?%c+&URXk1032<_LW$YI8Z?!ER;sGUffADWT5c3LVAc`(#8 z)Y0pWPCWH+F>l8oYG0EUF<7Lx1yOHz4!H%SPNu|X$Gaa+*YR|-k{|_c}>Ve zEEA`Q=0=)jcgA57WfxdOndmgGxpTNCvXkXiNkX*tW_Q|W`#F7Kp{MH`lNvd zc6~!+KnrG1E7LXw(BU|}@=}M@XTr#6ufj(ZUv$!`zVqd!(?}es*1z`ygvvK`l%k~? zoDv9nY(yxvl@z&8lx?XP0M%$v)P=RS;`E z)7a{28Wq-w98&Z9*_jZzt5)${3AV;pn($sizEbgVAXvHGcqNZHvdKMW@3c$n4sS26 z!c;acriL!aBNz9(teIo*rwdn6u~`Wa5tR!zXpi#?uM*Naq%YnSyLafhXgecRJA3rU z5*297_?S@QbSx928a6Io31Jr^L=(%rRqIo0gndHfjeunNw^|}8nfWR|w5fYs7TM|C z1UBWvv&0}PSn-==c8!Bkf$}xL2axfhcZ;5_W0yo%j&Rvh`_lZAPUN|oewKUP)(^sT zr1{y$JRV&-B0s;gY(56j`8xQT1$lUMC@UHi;e zZ6ee75!XJN!?C%rWYUz5+g!R`2F$adrX{OviH|1&e&0^VOqp&hJx zJ4dz)WOMzysr8wSgOZ4C0pe@z@=PByE~ZNjn}*$BcgJy6e7J%v#_gsWIP}j%{MaLT zh$8x6+j?X*)O1sLd^_JCWC^|nv@jwqC7tRfwYHVu^SQo+(m(1!$>X{ zt_8UD8fuUgOCWyN(+l$8i*Pd_`EBBD;Imxng(*xeUyTY>&Q`4K$Vjt2brj+pf$27Fe%047#qe(ql7Ir)!mEj)d2 zmH58ifl)PK|4niwspJBOv+uff^S0L@-thPgLnhRs&l7b)M5p4?yCd>++=6h(YF%`PDJNiwTBIm?jb~bb5K3~(Z$iTgC za^2d*$|<#hioP@5Gf%>noH8~9Lj`?u^f697?o!4WV!#^kvWJkCh&~sIpV42E2RlWa zw3ent`Cx1OZjyz)sXU@FHSt^MhhDv>>Yl0|TYcE1Oh{FF{kVfFQmv@9oWX@ z%qTG;ez~3ukw$A!l<%aW5uP!i+BX#Q&V$ud80ryvdD{GI62%LWFjgBWRte_wLu;+;lFOX6C-?Z*Gj8kSKTrxr{l*&a}uduK?b2>#26piEheW2*2GI>1t;> zg*@}HUl9922Kj+r<@{{7A#%3%yW~@3>42_(Okj~@0p9$F1PVRF(ZK}=bFQl)zDVS- zhid1kGH4lc!ahP1nG*i{@E*&*dZO@>8tLABHeKUN6 zr@A1h`^_NXP2b1t4B@18xglE4k^z?aAb1#jWwY*k$3DRJF?``DT;kA4f#GlndmEX8 z7!tiOCI8g*DIKG_a*&ESkTW5is&#&*v@+EoU7HKe<15W4Nqan#y|aV{7q>rD-AsIM z`)*$#eeRc)Xbk8;-(+UXv{$XX<&=l#hL`h(BO1q2ur>g3WY)Y2d|?2Cn@c{ERy^Q@ zn29*P($i=trZ-$!3F*UZ#nnZ2Ax$?1qdjkm-)wTCED>`~ucZIxt;udkPa7+4MR5C# z-dWe+b_h69f7W$WHWARvUyfQr>5ctb=U(GG6Fiz^Jk(xNfv}dulvr=hTCq!NhFMi@ zF+pqm4FFURW}_gp2o9a^H+y!QV8^cIsaKOu7(s?;D!k zg1F!*4&q604B!&*s9PB1XWHB&-%KTUHU4Q~XmWW;*9zrk+YuFFHB1?00FS00o0Ut6 zW@~C>;A1hqu@!3$8}Ul=`H^!}A+*6a=swY*8*zEIWOoK&a?@YwQXI6(uS z7Ugqre=5QYqVu-!?ktn*bBFSid5F^Cx4DQP^J=a3q24pEz`UY5bIN^_v~$Vx6Z1^i zMn6i%Y&d5!(pVA#m*8Y-+H$^hUuWZ&i36 zni`+kIGU+38bW%TFAdD(*v(RE;MvQ`Ty2VnJR2I}qe7F4r2>Zx>^3rO*-T_rgw3lh zqq`k-p85!MU~zJ`H#_6Mso;&H-__qjrA1z=JwT=mtdDUBpLPgO4BzO+7j_uS&d&&Ca>c)#>Q4?`1Q0}F-%Eeyf; zCjkR>CC7CI&IkUc8_y$G|3gr6^|ZYE^x|MtV>Z$WH&Dk{7cG^;T7JE^Zp~bmP2DqQ zbf@MEO#w5;P~68^ItvC-!u>xJO=(;5xh;-JJ7b_zg$n@7qS zGoRxi_O{XE`q^gRA_rD{=DF3cgNG6=XYuZ_q(Ho=Z zUH8i-z@*O;t&*-fIS2HCk`3?IFEDl<-+Ks36HW$y{tI9x7#s$hrpD1+9QQfBx>_AE z?96+8HD)sy&i_Y#0RF<}j}48v5_2HS8Z2x7MEaUEqlT+Gc>arW`T38WEbI#SccuP~ zKE!lJ0gdAwRFM=%976el4RS~_L=H?=p~DWVjwc5X#70v>LJTa$IkLy5IE`AH5_G~h z0J$?a;;CMdoD3w-ERnb3ab|IMpx5%go7uF`o0pqqDKk!4VV2gO|8AzQU`2D&Q_`Dt*%(-7fJ^oL_5 zgw6@MZ~w0l+}J=%m0@NG_we^LIw)|$kheSyWLBmgo5@%Y$dZRp8J?K>_ofY#1Y3T4XF;_Q>WA!onF6$VKp<#Uss^p$BVW); z)#TG3wrc;QPp;kx^q8y3U5c;%6+Xc=)9S@VeXq$-tZ~j42w{GC=~A4;{Nk#`w;$W6 zAWSgBMsRFM+Ic=3sLp{uMI-GlI1;!C=Lu1LVlnwipdvvw#Sr<}BO-69c4zfWQ??y> zK}3w++irs_NxRrLicaU-9?b)qhOEB3Xz)NYCA^ynTN7 zhMcu)#*oP)?5;9j%kjQfRMX1k-#mS7wv>~PdOBsM`k?radOMD1#{X*l_^BBQg`Jn| z{Is+(0?!s5dPF*a{?0@p&-aNxN}V`&{Gxn9YJfLu>7O5s9^1l0wK+OKuZ=GV(zm5A zk!!joMk@s`Sh8_Dem_+i6wyY($^Wk7mPFi?7^TVkD0TU=B@kqi?asV%7|m3d@4j9W zP%EfESz?9i3oj2k?%3;xORAL~3YPHi?|)tJCYaBq1>IXfnGb_^7>tWJ_1e$&3s_tyRl<(ho? z1Xi_u-=u|k-Gwl^ofz*MzD#eoe6M3<)4%iDp&@R77hQ5J!>?mtFk{rEr*m)vaS}|t z+ROw-`aj<%MJUpz&ZaCN#pWp}{L=~X#`90^ZmaDTg9-h*46Sf+>ElX;2 zGrO+2(@8T%8d}-%cGU3`s*ZxSxAb^W-_ z^Wut$WP*UmwO4fW0_T^v*m(@w7KbjU#-XD%fPlumA3Zm*&c&|ogGW`j+wQOFL`!Lt zx&#!^VN`R&;4gWTzZ4TBsC}G$+Vhx$G`4*=px+{Avfi3VKeg({w;?(Dnhtaw1j<3O z3#VsI7)L=;z}ZP}&I*6RSS6^X{V1=8KC#SI-HmEPLwa~MF1Gb8M+>dW{dT({eXTXG z;gMI2c6_D7rAxCk>ygW}8bU*VpqBVzt+a;&lMEhXFGaEAQE&hU~8#hw1w_&`v zFx@Qyb{b6`WuruHfHozn7TiC`=O6ZhQb=;HhH&X9h%ijv)cn-7 zjg5%(saZ8mW`0tN`zY+RZ(&;_ zAJtk_rYEyI9j(#QG1_2tlK2e)CaP8X@hzW4u|;mO+(+xbQ0kdBRk<2d{^-}3e7!z_ zjWM(2&m97pF;K-3lo2nH@yeO)ig8TkrL8As2HAH1ylh#Y0I!74Uz~0 zSP6B}Lw{J*4maO0oLDumbYRxx9+>}t^y@9Ke zY-{K$KmMjj=;@+xFY44(Lpl1z^tx}w%3$nWaiCH9qdaZDnq(D)f2H(&rH3n#>fc@- z@8PrD{7ttulOb%h@>)@z4d9HQx`@1;2%Mo>uEpyH3feA>bL!nhYOV~A_MQSR*Txv# za5jZGm1xenad-4QdT9tvQ`_A9qu=(J!K&v(#3UAj)q;j>Z-uk#!;Y>oHU@I<6IIsu zijr(D8>T?qlC;R{$&jGZ++09@4h4-^h_I0-PL5BU+5@5krYZ4HPGRcERJqUIbMY$4 zVuYMXH@Z1a+fRPPwZ9rN6dlCx`9+l-Fn+HQ#=OG8ak6iI%jDsZ0q^?2w$q1Q&R))` znb2$LI!WfyDxl0z^;q&ge=hjKv!5V!RUh%MDDB@ob(UO&L7g2#sfM)xw-d%~*b zCY|;pHa;ao#^2DyhMKA;YKt43W_8UlteS#07lCxa9pUiGEVetQk3^HG$?N1j{D)R> z?NBG4pXTx{qwVL=SN%mt?G4wR+Pp}ch~mHdakR?k7v~&<%YBETf)SJ7sH}v*+d+Di zGa)Y|B-GYhqoMZ;?!q+?&6fA!t;NZ8-`O>YhtrudM7g_jt7g)pyf37gEE9+2C}xHr z+&C9mr!sAgri$lthh$-*nE8X$r0-jSLN5d>os$X;tnV>Skps4^amZ_b228zQUa63l7>mKW3|q6 z{apR)veC+Peu!MXY^m@V5p9u$e7}-=Axk9Lgn@<|k0TGW+F{oNZ+g_rjuqT*xf%Wn z=H91qPodHpdWxJ--3%zTIQiinIkEHaP5Z4sUGm7*xtlZd5h%A9~BI1hjPF3^X zS4C8P0i8s7>z;1!$X1E(&Sd|Oj zPu=TRWG}ado@iCXvx}B;CRZGW-y{Qd-uh6v9Uxawdq{j6HwK`#@jgKl93`gZs%-RlONXqiK2hYVDy<~1PMx}T>z>rZXCvoidcXRkR<;Lk zJ3!b6`T2WuqH3~7k#r}?Ro@`F2JPaEcAU=gCgl~kZfSjzcqNH7SH*^GU8 z6~PVmnl*4~YPWm%T9NT^D)g&gAUF5$cz};0RrFHRf*M_L?03Md!mL7UF+sQe_(cbS z_EERnt0Vf$F~3wfyq=yFknfd9`wh3uZs>ISF9hm|R=m<=;lb*&Ulq&3mk%tdZbm~! zlWk^48tde9#79B%5nb@;AkqLl1*BY`4F%JA_Z@~L<^v9SpKlpcPT$U}yC|qIU$@<2!+3-=@lnjgDeyTW#|Mb>10r$kLb|=exJ{jHm}{aHG#_@s8vHxsG5nKn;lXxw*vM2fz-|JGmemTZ#E+z&}Yv3$7gG9gQZr@iGWWXmt$ zs+cE_j&7g1SodJgcd|G=7^M6oCg>Gkv8z&vfwGE|&h8iBZ2B+L4l|*zNjiUw!M)n9 zYk!6{knupLR$9v4$Eq*$zO_pa-RX!rW`2$7?)o{}t(ZY9JVgbZ7#I(tWhR`-dD2me zR?hoLzNeK0m^@;X&jeRlv)kjZqV!JOEvfNdMJ2clG%n&ax?t? zl>u)~wh<8Rn|FL%wKF-;E7SFsUp{A;{My^QGWX}1s-`?!gYb$>FZO5Ku!EN2mN1bOWHJ zTjDR3--}K1(N%?~PWUOARJK~Z&YiP#w(4m_7c$LvB5T8KV)qf7EEN~T@)kX&i4wYt)v`lJG$cppW(`V(504mjv0C*q<5w~Y1F=Wl{lyni@77YLUnBhM{`72@mVh=9B3Z$qTLO9^Tzp`uv6 z!vdPcKiyMC50zU8-)2TjE<2=gxxj)tmeHMNslG9Z$$_K=sqx2MfwP0+#_Vj^OK}HM z)QeWw67Td-^*?VUVW6sC8gBp_wU*&)*S5IHwl>!KIaswmNR%pw4QYMEYOHxcPK10$ zC>v!~QMgMp<1w_iSt{={TmQg&CHmmTOamh@vHF&XIJ^An^199vC(bBh(KV8gKNdM| zay-Q1z{IO;rhMG6lCf@MSCg*FkODJS&jX{Z>G_bQb9Gaybqy4rtvHPmL$bGI^YHYR zpQZlIsWL6k_k4?HuEq}cHWhh@RJ$s&!JBOEt#fK{3!JEM-}b7O-XzRXJbmn!A9qWj z#k>*tMR@N=m^bQfNB@rdO?0RXS*xT)G)HZ1Yc(2qK`f?8c-%X;vjTeSK5btxQ7Je| zV=5zze%_CgIk7$Nz0vkA^PqF7`tsW5(O%kN>QT;ChyNQ%_n&inQ`7{SX7#OvM=KYM zq9lUJmCy9i+{g8ed_A&&-$Md`S-T?tBsr${oWWKKq#vx5tXPf0XJZ?}&$>&C-BXm1 zw>W}Tr(v}8QD^G$wM0oO^t(#Bd8 z2_6G`dsT;fkaTNTsvOj&u|1j&GdNj{v%Is5LsBM!4|TO&qQRCb#A9sWiI;Qs`f zj@eyYYy@Y72lBrN|euvRI3Kk&BP46-6DTLXF!yN)-sxt_Ba2j$U;J zg8f{-LnoA0P&%y!2OCr9HNKRuD$;)zIP>fi47$1ZCi4UG#BrCZ(97=v`wavrt32P; zOB}G;2b#qFg~q77k}K9^I0-#vjqQg-OHeX^Uxnh#B7reh zzjwI}#}&J_Yc9ibH=46PFv|=7BwXG=aF`k>Fw{g8dGsnGlhi`TL2H;vY-Z#Ew-Lq6 zcfDa^n>eBaJ|V!16zLUj8uHY!VG#+mEs8$^1qxb_W?-o*ix%*MFhE$-&r1wSO=o{<6%I z7w)^pL%FGc6O}S2#D4HjFwJqX=SWx zGLj)%*WyiNlwzyBa97zT`jnp{HQoBbMKUgaTdVTb)8FB+%2jay)9?|n(LOK%RQX9=S2w-xvqN}3_#?Gwjb5W-X0);RuwI02 zN1rsJKXBOS;#wd3;)s9pIw-wYEkzbP+%D@a&8g|5b~tF_PUR~FBlA1w#sbmWOQY7k zSWGZ`W7}guf}&`(2~$TGy5B0d593?ctxq@{f`0D4hFcTrwdB|UZ&z!F;ENHn;X>Q)^^&59qHKybJ1-?k2%vQzDh0FPO+%)D z=Z^aVQeTy|5Z`|QI4Azui2GvnD55NMiF2cWlUegi#Izi`{%syV&~dy+Yj#o>N?O4w z(ykv4PkYBJ(}#pRb<378#&Nr^xm<_jnI(o%i5i?_+f@UezVpI3q>Mn>0O*yX4tQLI zXe^$R89V)b`oWpr`X(x80Bf~zsrJcCBIhy-_j~(D-32r{uL=q-|Cz*oUK|?&_)AA_ z$j+I`(+7%aT#=KXkesmnR1MORRk+*sH&dS^m>Y6FK~rO8J+i{OR+2#hi6ET?ba}J2 zNY@YexB9cC5uXz3O03ut_+s=c2)K`4mwMo~&ApTlk&=!IJ4n3Ee#sbwR~yf~drgc| z!4igX{O+hGe5jMa#!peGkx6@1hq#W$5=xsxGiB1V3-(ks9mUG=tTJz^fxmN~hAtqK=0-IE( z(ZSRhCwx=&05s_%MRftLgjGUVMCa_4$=W^^je#hFBl9~eC{Od^{47rDd}1bbVSBC7 zAlS3REc?t0xj3<)%GMw@VyaO78TNXyx)fQgP@Ru|Gtr2$xO*%6fTLv%C?rgXjTY%3 z6UWCd>9e_?Cx1+R@j2ee%HVNBhUfpd+ic`XJj$D*R1tddi9x8WNm$HBo;ZQ{Yr*E7 zT&HSw+|6xuL=;|OeC&+YnscwIs@*H9-Y!LpKiS=X-$Ge+T)x73GD+5DoW?~rr$9D- z`Hb*L%Dvnrcco&As588Dt~DT1^=wJyDF&UlSLsKhfa9c>cu{Mh=+d>jw6SWoQDWL_ z==&V)Q+5AV3v&4u8-0IvB(OxvZB9T1IvZJ&9(s9-Wc}Jyb^oyfUu)o`PH6&b>hsdm zM1nctWY(oTl8=_0FyayU)VZ3`dGF*yXew1qc&+*Qw@Ar2orWfp_Ld)HQ3>!(=MN{^ zRo2f@;0~^;L-?zYKp_(vgjK z`=I=bt>wb1w83VJZf8MkU@NTsIlALTiibh;?hxE<%UolnfGj33IhER%wP}7Jys?nH z)F+-iYrK1O@2c-vVLHA#ZA%zyfc_h=jSc{1^=Stii*q?_S8`$A%!?aou^u&T7<>77%MmyxP!{7#!5~#9K%P(y)Fd%@?#9hd4E=rngGW znO>DWNP9J%Vzk-@MeF%>@iR#g8NJ*iBHA=t)1Vv;RW31VF*sBE+Ck@$3}|;pFO@lF z-c)p8jQ*AQt8v7?e+Z%IXYDw(jnqV#`b8xYpL7<3My#4pzKPjXnPI8PDT^Z2u(S&i4m8}L8en_9`rByc?#{E5eKX|o&Gb0IYm7Q>aMTSaEWKbt^HE^c#%w8O zVnEy1Gg1Z?^T13b)a{kdeN$;rMtNH3H>Z>M2k|2}v2`2QT&Y!;H;jDR-&#-Hl7z1J zKi8`)`PvRHWB01PW`0?_x(Sr(ieYO-?|ub`R@QRq(3{?J?~MjvmBMm4!rSb%L+{d_ z{?hhENxSZ4K-OI2noO7?V(B=*68kZ&{rtzk*mv8rELa-aW{PlSTofC~(ZR{4z~frL zjw5-L7w$uY=_uqi7^SrUJ@!kR2o=X6EsJvDxSdkS7L6uHo=95T(v9B2r4%4P3MYJcOs|Pi6QBJP|_F3guFVXlT~a`YmG>X z51Qm#^V0X_l2XV;PZ=|iEE36_VCiL$qPNVUHw@2W-W6}cvm?WwiX;uOoq@x7fu4?- zk{ocMbpaswH-1tRn*vQIEi8q>QlVE!_2!jvbB~DdV-t1*a5%YP@b3d2AJbm&&GC+k z%&VLGH?o&>m#&DIN1d(S8UO4~mkFtzlcD5-UekMdr}&bpJrcz>>-Z-hdnMIUd9CP8iOS%&CWg$}JIqcC!#|MCA0NfPU- zuH<7*X#LuJnr`o1|Wk5o6N0&k=?kVjG@gg=^?$ZV`np6(dI`fCY62r z^q#PmrT@mRyHxVW`>-dSFbp-;VlPk}w>Gui=~Q`~o%i2HPT?YV7_Ax*UnW@UQ%jop z5W{+!EBHOMUnqhJY-{MF-Z;AYfw)VzMgJDn-fH-L3#Wd2=r5{bAb#QN(^I-_W~5uJ z5i8Pj-!PTcK;JHWgIfg6ioA6nMg@$17g%jSyXtxkFXS%UdmDY5g194|WYWmsRdZpA zvv16pC2yJ06w_@ydd)k4oB@I(D@a0hYH;stH>D9o*e@uZwJaZ7TdmyZ-HvTkquXp{ z#w^!9VLJ$h&&*98eitosrJ9CGCpE8#B{R1S3M}SU_5G$k`0_m=2N$EFS^1(t4S!`z zC5Y>o&Z<(s(rG-0!{6O{+}Db$XL^KI#wHQd?)dls&K`+t^ufb4-J5y3zxWP>zc9c_x9iOZV`cD6tE~PU;V4n&B3v7RK8aQ;4L?k+*vY8*(F#z)b$6l?K0#XymL46} zc#~#*(r3$B9H9EES0MjbnQ)o5G;Nu`M5E5VWdL3TNX~6E0t|tQ*N4)AVGW?g+IxMP4)-FQ(hA!QyjXBOCZec#z$qDQ&zXi82j@mhVQ$+vCe6GE5D*CmJwmy?iH zjscIZRbBT}**&b0En^ItbljAmJ}uSz2`@-IfmJ2zO8-J!#w9}0ZTJ?HCj zWHfM3%3ogW@5?2uCx9`n=Lf&=ljXT>2EbxMSt+4dYdozbaenh}B^fG5#_BtD_?0vv z)nz(NWlD`z^6kED3EXI#NeZc@LKi#1KZlt`fWf5TpTCcrg8Obd!U*W0!*$k4f%#vJ zw+*B6m_F~PrTkcCkZ-7!55N?9|C7=yzNxHV$-S<%ZWy{;uSgij`a-}X-TCpPt7_@2 zhXh@bs3vYoVToUyp_Sd15zESAUu9oyP&OvzA5O)REVJ_f0|9wqn!UNhdjoL&lHicG zVvs@*Pk3#FFI34d;AT`?5O>_lF0#Osp*aaRdb7^ZK43IfO9A94+-p-H=)f|cPU7>m zx6J*O>NIiBu=g6f6UScqE6w`29+Oe4#dzDqBh`CXpLG?Oy)Jcj^)s~*i=tJ_3!w5D z-w!1s2Aa7MHyt5TBCk_e-+z2;z@@l22R`~G?B)qN@$k~WfV$i&(IbB_96_Y22&IPfB$KRWq=@`j3&fD(~5h@g?k#v#=e-rkb&FaD*M31 zFzA*4ei4Gemn6zV^n!6>Y~J7hmMI-0YV`AY7iifY{ho|9-kbqtT2$61JBCH-B$SQ* z09fyK4fcusXti0KuVL}uB$^M>4H75g2{WC_-qF|DWBXpHknyJtlo9dk>!6j zcK%Xg5;7F6wVN$(23FXY3t>uKimlMv7I@uN9n<%(Hd$u=54n?LG0_fsjbIh#qu4wN zSA?pTiS{1&I?bS4OsV2i?mYtKgDploA9poJ!;q)VY5I@*rYwa*xDfIdz2L^A#JeE` zZM@mgmp0LgV6@JB?x=G_&p9^o(W1u_n@1?pa8`m;)f{Bhwi%Swe#M+Ew;R0GslQz> zJJPSm`JH=#c>aHUU~|%AC8(`ct{3+j`R8CTvp+V7_(M#g@SbCwp>a7vvgqJ&TaA{I>>!+9+`_>GcGv`5DM8d1 zqr#myu?Z2%CEsaaLKnB+80;0Z^mR6nGvz(=4)Eyre2X2@rfuHX4>n&5Iq_hBCpL9V zkVDmrxo)r{Lb`ic`~C3LCBr`caTKPKFUP7cG{Axv-g}Ti(JoSz!AJ^IUo*~7vKB8W zde)AgcbnTh$FllE2*1>)uU((%&gW=^!u=(Wh+>`Qj=5Ju4mTj%l~=SrwLTg(eUugN z=wk$7**N7`Wj%NJ$(+vNy%Q`wxxJ{ zEUbf&v}~KpY8UZ>-ms_Wtp~ZUB`TxV51!6g#_q+%BCLcm^3mmUn_5S>(pFl}Ce#&Q zTebM6dz+;v-G5d+`?EPyrVt}(ygH`HYgvO+Dl zDq_jXF~i8;`YGNUBj;XATk@XME|+9yKNIZ}Dy*EiH=;NsepN{~B+H5df|rVxuxU!Q zB9ttJ^!GDpH^{gL8CB>&bB5&b2PP=fZ!r6dyagC>NeJ|Ll1ZkPn@{PG-b#wQeK>b1 zCg|Z5Y7$~#I1k{@pSBZLPZl&l^TR|9PzsZa$ef*P{^$P!Iu3~cLu7`+Ohh%Y(i@IS zVr?A@puM&^B+F>?&E<-{gF^fUw^StB!51lz>UQ6y{~gbBF1;}Rq?_aGh3GL8ZXe^k zWTF?~U7GMCdX>m)ztQif?f@s~&&!md8{eG5s@GqFZ{ZDy*n}zE#mxb8j@dhYF4rnn zpi=u}Fe@!Gbx)BuOg!yYELs|zRNCh4Ot+Hj zN{mGFeCWewmH+wL$RiG(z0`N}CgAe%LxZr0IGHb(DTzX%LuEC&?Z>DtubV3;5#@J# zHoofm8yCFPutEF}%9#%nu_1?N3KCyBACI)yn3u&=; zlUfg}7<{v{r%^`I2XBX@pt$0m%HOt#s#^})(&bHf(;wRRmDuqg|9z6Mt@oma2rs$| z_n8Z8&M}(Ius9R0%{0ddHoJgF5~UR+1gKBZ6BLJ?ugT-QJ0WoGB=tKpCBbCXsiI^0 z*)Mh|_`Mlh(`IS?OxHVxq&cB6(Ux;(N5B@6-0Yizny%BOt*}V7|8ycjLp{3%1Zk6TESD4gUrX4J^z!6S8$DSSe$l)*mqmxPfDUy z=h`CSo?2q$3!AUE$cr@VG`!_kNf}ibs0pFFr zmxMvFd0W{xv*}B2mP6(P7d>+kZotNT)QYa_oC#n z5Y|+uU222uJo)>x)|MB_ArhM@)BP5!H17UTzH4t0aiur#jHfN|v^^$v5q7IbozxFK z8zCN#_G*7oR;mgLkfP-kgLZu7HBwP9NGfmlv!n4eu7jZLMc9aedquhFuGYWpjEK03 z9i0eJIs8uy>wuI3ELq4IyI!;sYUWtU`}tVoW1g9U@ri%^%H_RtziuJd$X_&H-Fmd! z6`$VP);lXl-p&^MzNS($OufNY%{Y)fbu58n@U$uox{e!?G~ecdW;4=I&&|YI6O4}< zWU1+aPCtT$@W6)ebs5}EBQf@{&GozQP;#&Z_)Q_~@0Xj7@0@dqTYG(N1cPQP?zO`9 zUlO{$q@g^Xz-2$uY4P`kmag)g< z@|3nR`A`o=OLYP1X0#-F1OH-8`mfrd#2lrq+QnR8tp0hQv*!4sA?#+K!Q0g3kkP}h zhC634bAQdf%o@GDR+~TG7F@5n8BCs2-7q)k)r&|bjw)`Y)dHvFhyu_zh&gg{o{)?S)Af_T%JU$0W*uaE?^-R_p3 z!de2H)WS{Zyqp_iF41;GEa$2(Ln)c@%A7X+YA9;MH4Z&;8!VvScCeM*eOv#Wet4td zq9$dv*(ew$qop zkvM5!5MIeLRwhwh=c+1X>ifS{l0R8$rlVx-i(BC>3sHce=RerN9eaqF(aj-$tx~#D z@=h&`t40AQ`Cl3dvAhGTq3bGfJOy)m)PgrXGbM%(yh1)+VEieo*?IVVne1{Q2K^aQ z{p#04n=e$%FPB$`nk#?D!j~=-lU!=W)~PuajKPzXmjIPGsPx0;&@Ym050KWyR#R#L zw3vswPQEtZhHBaAngp4kD^l8+n|>{d$@kIU{BbiN3+%vsF!UP)8y|m&0Ojhpjn4D# zb@j@5`RJc=T1cP&kd&eT*vp$&JDE%n=O+&tvV5gVsCIvzs@^c>9&tU99n9`+Z=t+- z>tEQfuZ_6}*So!H^ZZ<6z2OJSg{mZQ*3MLe7ogZDIt&aA(daoP5McB6>k?PBB`z`Y ziqjc^rbJE&8n>#K7;Y!J^l0x!HFZ-Em+8YT);HLm*kf$?;c!#BAnCo+$v_=Zjot1p zhxfM#c05h2QPC208E=f@Xr*4d&8$PDV}u|7VF+9d{NQVnB}MWyKube4NI7nB?+fD~ zAzyd$<5-fjel}N(uo(-$Lx?#De(|$~j!%-A$(jv4SIM$k0yW)6`Mu-gW>xF{FEHoO z6HwDEB~K)�?m}EoAuTt4MY@^oMj1_EqAu90Z#kT6Kx z!>3H?^-5VO-mQ7%?ot2N{=*bN_hEoy;NG_sHF1Pz*2B3yhc6=12kwReFCr8FwNsi0 zYP(cSUN6u0+q=ZE#fcREVAZHdu6i5O=RmcOQE2~~HJ7@jm6m)t;s?qvZ)kD`V zImYbXl-Ta=;Zq#o3e!}!$*jAg53t&hxK`@rI6JgdymS(2E%dR(hPj&SMr=CCO=R!k z7ySa&8j+24ewEfXwN?q)aZyws+Mt|nYrbB8?o^iiV4k=0nsg1l$4ZIZ>OxqiK zAh0uboV*6yx4+Z)qSu;retd^m9(*^-5HHM_UhprQ8K0UR{CMG~so19&D}wNZvbplv zh4Zsz(*}-gtIR|a?5f}TWc?o$p3b_2s{9+#N2`C6xPLLs>0yGdww(mS+0fmpSS~l3 znUvtz^9491_E0Z84)R) zWlRZwSC0%4SH%xMlEbXQx9}owa;DybJZ^~5mbkngDx9@I0 zd)%$WI@Yew7!M|C5d7jAuX1VE`(lhKoDx@w{uuDv<{Tbg!G2up`oq&JwL$ACqipOJ6Xx zQ$b338sFrodMv2U-RTT5d$Z)gAcT*jP#1&Avp#-u^|@~#2apV(zC{|GK0%YDa#d$~ zLi2T3=I=UUN(>4-$tcV9O9^VI;<d373cbfk0YX%&9 zcEy3g3tnkV>f`iJgx7D-({zv}uk7M9#H+^DGW z6fU4N+Ix#-u@upxq?D}b%R-Y4?pWl$B|cDsj#y2^Y116`t0t_{{}`^+CXF&ISXJ2O z!{`K&5heV+ZGr~fOOyVUZZ}$RhaDO0rc6EH_qDNfgx+zRQOJM6PL~EIgGJRtvzGHN}g-!H{fkG?`}y@YX2FcXbE!Kf3cky5ETZMG~uY4iRXP5Lux)@a5r zNi}?0{tR-B*)`(bZag!(TS;W@*vhU8S3PwW`OUdFvP=1NWfFiC1-czM%yg6kEKQ$b zrXLI(K{cM&`4ZXhmKW0VfArQU0+=yk5kedH!8L%Y|JC-HdS9dF0|L;sV@qYk|7I}lC z%%iRUmtVTF@W)M6Px@uPN)Qp+sqS~HIuh!V5if>wyjEirW$Ar)+i{Nv_28MBn0K4_ z>jyHz``_60|8-FKr=o)os`{jaOB!ULBke)mtFBksooFUka4)K%rsC9i|IhH5j{u+6 zt$}P}(X3VqGph03EigN&9ARn5?Y6UsKV8N^HO_<aG@RlsV0+Axj6{E)qQXkY2wi4;Kw6BQzJ`))9`_1MKhg^rY){o~Mw%hJ+7d>Do zQ`#6k5;tn|9a6_6FBj&`>yW8Pf8S7s&`F;=J?YhtuzJ0Eun&fKWiP-`R+bXJ|4_5T zq|?4<&US}zZtA!@YdhdO=Rv=J?R>wS-?ejikvYCeKvQ|1j-syC?7}Z|_qkgrCD%2a zT*6F29*EVE1`|Ki`Tw>vaN$6sE&-I8>oX^ zH|bJ6CiVp$efKF(+~za{S0SIeaJpxP{9OkbSZqD=c2u1JF}C}a^dhVEqGJIxu0o-Y6-Tr$~wCAk)ec|v zT+r41)kd?&dN0r4$u^Sa9?F2ywc;^b%zNSsEwp6j+~>5h`- zC{;!WGi0h@qFm$$H@W;J0@e9K>uE#+-~$jQ{~2uDqWQGHJ| zQmm!}_2fjU!Shbi%UH#*;l@D3ot*2vpEFFs`pSE*$AcenGFK$#xHkwy8R>32rD{xY z;;nv`ATvf%;N&0Y*1?#60GD4Ojn44~L?a{OZmF1y0k_K{c*KJC76va)tQ!R8tX!UT za4bzjzy5FIz4=7s!HL?4hfk_i;2ywwxB;=o86Q$O#}PBV8yjoAl&Pzr#Wf0k!h87e zv+z0Ve@{p}GDR&aIqOwN~EdXuE>q^oT#4K_f?Ijn2 zje?oES%sWo|3b@?loV>ylwy@R+h9^1qEkAmorJF2ogYVD-o^oh1eH~E%w3d z>K=o@h$#%gu`}#!&4)$7pmJf(G__>_@m5Uh#X$8v{gszKUA+!UHFJIq6#-nqp@tr+ zQuox<(0*Q!7x$~xdPH!~4E9q&cY>-r>jrUwkSZAZ7{?W49j%BC*kMM@_OO3ILt4UlC+A$yV zq82?HX^GSAa0{-=wa_u#r__L%jD;J_=jly1A1x^1OPtqcJ_numRtEDY7^7g$+9XeGo69#<~`ow!xN}iW*XcGP!r#hU)2<|Y#T?1wPf5*dLL0^ctRqMh# zKmkjO)5Ck*+#K_Il+S^4@vpx9 z|M*v>5v=Y1c^z`&0e$_`X&g~RjTLun8TjLS^o#GnK-n88*1a<~RRf%RUd}9xKN2LX z+=6dDl)EBW1s1zYGP8ilEKYxLbk=15K3q*|>fh#HijD6`KfBkV7}SRkz2c4&AH*2t zC+P2}`FrqNQ<2WHZ|)UYlK$$OvP$Nnm##f6gsl*dD`TI7$-`fq$^KEe%ite7bz&(D zt2dg;jghPIcLtt)t|;)6u{Hac9B=3AYRmA4i%a<#$-OwGEuDMq%V_T4v{BFT1>=+L zfeueGB0lX-y`tKm*f$XS-_Y&!&?%Q(wW%W3(zDV}1k^iN=fP^g!i7b!qFP|>`Zr0gR-#;+5SRYQsfu8y zS72nnkSKB9^zi)XTRjS7DA?RNFX68p>B#}=CcZGxpgs=KZXsBlSj`ZbH-yBWYW4H+ zn*cdHOH8mmqC~+s9@9M@=o(9IrukHrAsp7sOc2uOHGDgMU2z*;a9NOIR{}OKgE)(x z!6}{9&vhPXOhPfredOqUj;+;hbHzbs2E?EDE+pk`)f%#0BaCRpiR1StLQAE{CKLDNs6-2?Rd-PSO$>CP7N$?O z_5@r~+cee<;@NR#N}b_)nv=s0_*ce%g6O2vb32Aiku; zdBRN`dP!}OO4eyQkNd~(p*SUm1In3gLxL8lATE8-Yy>0KYXYQs**Na(R7zT}rK%-w zMWPloJeD2=%=S&&{#>@~pHpsceXJr48M>(Muq8$K1jSw#1n5_Zx?}lrIguXp-7vw4 z;7Z5Gj%YKu^L}UWZtb0-N;DZShx>7LN=hDgz>hamyW?UV;( zdac`bsVHjU-sh#Q5oIub%jMXdL;T&pd;j5^!kbqEL3RoTTbN&4FIBvaDn$0=60}dt z3cilL&~NyM{`pW7+#5rCGzkb5^VVZ_6&zklkW>oE>b>!8|73_(IlQ7#;j zdkQXxw)Ar(puWWhp6vL6;V|WGwJ(Eti)~rBn)DtefyUMO*3gAoQe5U}Jvbn)P9X%r zLkwK4RlCtTR1!RTUXN5YqqiDv@uTO0zu%8(t6bWrO3s3Uo4s!-{TAsOx~~mUjpDO! zc5sO1rq*BjEjpaYTlqUaChhh>B4 z0p)d2F?a|}_+jQYhGQ}3a6Xgm8j)cx_B~gNfKx#wU1Y^4_75JJWVW7HVP)Qx!S2^> zPOTZ<>}Z@|R8sA^9`=9!k&AiFEb^34=+eyrT50`ImJSvksQy6ZU5+(Df3*F5J+5@} z(xaN;9NgsMqh1njuKdaT4JlaPHh?EnsQgL*5Y)kBpqQLaoqkHMo}^+q|ekfWA>{ zoxmuT%{|#4rVEZEoN}_NYt`yB=|YR*YFtlJaMLmr$4gfB>iSR&U_wiU%WXqSq4um) z!-!8{i`m_YJ)@q1%~5GB#qwMj@!rzl^!tT$izv|ifcx7~X?qv{{Zm+c1#6;g9fEe1EC5-zyEEEL_bGGvKTL7vB;h6U%w?vCs+Psivk)s<|m~K z)&xH~<{62dP?e4p7VM0c{VjJxV(qeNKOZ6XWPNOTJTbj=*hDt4lP~2*(vW~`QxCw4 z@otBw4gWpSq(Ya&vQvYXR&%G! zE9j5&T3X*nm>{Vw>jJ9Uh3qSV<_$Z8=U2a)Vx;}yU*}|HpXBUJQvUFe3QuKdc z;3to2vG7uK9n~|Q`ax3@ZD9%Tp+Y{T+d7-Qyrbwlf%)1bc&^%;^>m1*wosfJPbhTg z?B@lSi#@y=YbR17dj6e)KIpyd7mj#O1gxb4d;^^us5idPXd9n@0NFS#!ZNrc^0ZAO z31f^iTTy~r%qRhs--|H7K`J)0>O8bNL1zZW@LA6*Y}$6?eH!R;fRml9Nyw?a;A?%u zA44(>{0b;p6xgb^bm8r?WOaTP_1-yTwOU-1p3$jqhj&wg4UgA2veO^f%fRkNTJ+p9 zlfnia-j#T&nkXMsYZ;y3EyizA=yC9zlxsAHB*v!C>=Rw6T^EkgSIHjpV}2CBZ#oGI zoRS%pjQgU6;9hCFb=}e6)dk68133*sq!Q=w0SRs|6Vr#j6KSXuCBnvzXze$qzn?CZ zYLv{a<33;*MTlZ&8SZ5ooqwtdtYIwS-&fbK-bYPdv2-nwCNEX5SC{#+1p1a^=EW&` z=Vg*qt);DF6Sio?3Y9u(sj>uX%O{?e+-$N;Py&CDD@klsK675BUUS9$hJG|`=h&ux z2;Hj3y&1+mf-|N5UD^8&Fy8;QjDH!jKV>x^MS66eTU#;`dHFsk z?cG{CpMvs++Q54>wO?MPu(Ik`PZP(yfj%AeiIE(h8LC_}(Y0%bThs8{?ke27EV0yx zeKlRI3RpBL@`S+Sn6&4hbyIH^(Z#-#2EzZ(-(m;Mi%&NAdZk8BgVlM_M zTOH}Votn&~D|=wKg@~r$qCDbMJzzu!VU*!5Qla*&Hnxo#Fy-2ul=e)=j_o8ry0%lY zWw`4t!9@4BHZLIhj<-jzZSY4;PLO`6mtsdk<5EV6FzpCR&*))U8g|TD|Fi^p!o@rs zSG!LAgoTkl9Aiaw=gb>I1>Fyks}*z*&{~wQ_ZpOsB2#pfpte{l;$lyk)w`S01TT-= zVv3!F4(df3Xk|tNY)zk7`t1fLb9C_bLayG(@yf;=!?uq$wF{r0h#oEc!`@tovA!2OmeNLWvl7^ zBZ`SWt{K)^h(&7=O?OhXAzodqi$P~u4+Nj)qw?;E-;KQB?eFxh!{a*OE+boQ{qaGI z?wKuJ5SXD$a2;Szo@sQ7w^eDhymel`T+>nb;Z8N)v~>N-v(gifzlz|)m!ZN$<{Jyp0RUc;KUcuk%1DAjt4` zfMXG)=X#L{HR_oP^O(VAy|MF4;dY?$q*V!lFF=Ltn@0O;tvk;eBaCyhL=lG?v7@lSs_wCFo z)S2RzAl!<1L|l4(`q`yiZJf>dIg0U;&AuDyY6v%^hcltA7nWDM9qOpnsz#zM1EN8o zIk6|J)IxTHK5xB|V)4?f4sOA|7z8hC zJqHT~aOKpvMXf%j&mb(BKJ}I?L$1BjV2tyy_l@x(;6faMg>Br%uqI6f{r&xBZK>bU zD=?ac8ukNmvVn)DpAYGOl&p^C*EwpIKu`42w)|%%)jBS!{Bi+X8ORHtE?EE zbt7@z8RWV$D`-kzwVn2%5Z!$)(L5qG*auO27g5%FQDg5mK_|ez@dfG!IN$+RhE6`7 z0IBga6w^M4lbRAUTxcL&{9{mFWay)F92qrqGsEE)Gp5YPwplFca-!UWi1A&8Vvbp0eYS31K=-ffbgRA|!T zynu50lfitsd&h@>z_)jR24>+)N@pckG_%pv(BNy*Ml*t)0oanGe2~TS1k7XzL|*}A zlbOZHd$c?H-;XwCB&uUNE8dKK5z!UVBU6`qZ0TzfB&5ZnPCzkqBaG@ zJBLhxm=%K{6tYSlaYDQR`vCdKYHl?xEaD+tX=-;Q={a~?od~cRi>iBFbuf_LGHR+tdQU z*Y=B+hbQA!TiyDdwc{X@YsX;Qjc7l6ss=lx#<{|Tq)TyFXn89+2HIgrT7|U{7(&H~ z7maR9Fmj^5#O0$|8de&MKeO*h=vzuMKiVGo=CiCH!s~e_lp$3adChNA0Z&&HwgO9i zwxkUL%9^O??X=B*7uYcv=qS+B}`H8E87ZT*-%ICRKPk3TW z$a+hrJ4f-ul`KT(T%C|oGI??1%qIWqq!kY*D_KyoCdMFQN4?Au7NKG7r(tYCK{%b2 zp*wpB>PdUC6{O%9$9k>`eUR>zgOy5F^YkQFejbXMPhD^>1(b~?iNa6*ES2J0(SWT- zRvxQi6(Y}BLK=SPNWP!XFtXRrT}a%OZi^_kEa#^C?9GzV!`jnP97(pJQznAA=iB;L zVST1@%Xr6@%bX5ss{EYsUShW*zOEUv+ETUibI) z`+6Z`POQ5l1q{)QM{{;*%Il2?p+4cDL?CYX=bT+N&_G+t$0bm4`TggA59+1pzzBqC z6yMIo6Ms!G0>tkme6VHTBkVEj!v4q;Le1d{5D>5;t3ZR!@fa`LVkF=G%yqtE4E&B% zGm8r_yA!!uK|r%q(zZRMGvE$s&K@^E9bL7^MI?y0sxwz(^hef^*0Vicm*-;_a>950 z5QCos!HYpCnh}Gy+?=IJ7~psPRuL{st1DA7?#UmS=;qqozT~%>_k9Kaz05zepx{93 zvc$UwQ00q!3#@1tAG|s=m925QSstEXIN!gYE?#+VJVYmdNmCKiSt1#kk7;?aI;f|$nKmg~Y0^Jcz~y!H5<`adl`{W~AOAY; z0oEF&!0tdsE+~!_1R=iPOol%Sss#G-W#g^~34**A(_6XzBqu=2@3Su-wf~VT&_^+B zQDe)fm5;&ymWCXJAN4O=9ag9rw(h4m@!K4ETh6gsArWDwlVHxqEcl3OML_n3l)r!3 zm_n4zoJzNnT9{UGS0)-X-9JaWxYNLJF(jXvcKaH>>+ojw;GA)Ue5f$)>~$5n2hVxHh`Z!z`?3%r)5Mq~Yj7Di&r(rWajkUWd&MTEp^=BQwx^fA8+Y}q#(U9o`&e3IH> zK!{(02FT6ceR<@k-+#*e3PA%e1H#cC-`$-{Q2qI;NsIO-HFtd4n&M*rTLd<@z~^Vl zc=Z;d@{02+*d_P#zp{FqUuFveV|l8lXLKVbYVYY%$bb%Ce#S4PS(Kws6QgSAW=d54u~;=p4RQaWB0)*E@6L^GaX?M(QMd=T>4AHI;`YZh^NR3@O~oR%c*HB_7&S3s zRR?$@T)dX}J$wG%CGQq-te#Mka-1Qdv)fXPttlw9;DJ0lR_U*#gV*w*Af2K$#QAmS z@?!=@GAGSQPmZo~iU>rnTu|QNP<-RNho&Qxul3j2C4KYDM`t#8cd(!i11(Wcq~>uW zaE*9Ojiw-J>dSSMf(9j9Y^8R1ekLdevCw2pHW1V`^&=t1b zaGE@`_cP>=3%ynie$BrOB?!~BoWza9!ryxHPa#B8P2+q9K6A%N@Dbg9gVHZTZqyxd z-N}nP*|GTm8b(6uKnRmPIK5tDv6iuFV?tX6Zw1u?pM6xX2KcMZXb3Qi0nZ#a#O_bW zKd92a{i^+AC;DY;X(5VLXxWks>3rF?fiooCEYJ&=1Fkt+nWzZuI#>Q!Y#%j0+TN`+ z5R2PKFDgOTB*joCcsAn04*wY$eAEt6*|}Yj`1kEhcifSIe*0Hez>Z;6ZAa$Dn#$8Q zUTB&7k9I}W3+te^rTw@!V}ka-qDJYVnIawN*x0`rMT8fV(mldUAu7^JcAr6ezetZ2 zgKP-Flgkdxdjzcs(4_Rls%YOgam`hPJ;!zO?ep1VTNHs@-P^zRbqShD9+o>+UJ}ev zKPK<$F2Y`G{jvVK@u?I>WNcZ13Wa^FQ*W*Pxy_>YX%JB&Ty;hbe2`j9In5~goiJ!H zDC8!7+4~}damI^)aB1T*v~)}Jclb;wh_;F|0Mv-LG~!&Q>-nD4&HI@~!0sp&M&72Z z_T*P}{W)>|Et0h<7 zXH&P+nji<+2({)^o_o*k$>Q1Ex}1Kxy*rPycT15`Bhhs%%*+_$&L_^PEf^KtSsrUl zU4{x$c$=KK{I)zEAaUkPeRhmw=A|K+n6O1;m%aXf{= zK8T1@R!h&_a>nVZejU?HVhu=MJ+J)wC&Saj3PG+D`cX8b{##;yNHb`=_KLCowTSMt zEZv(EjX!W#K|VsH;DX2S{61NTm<>kpVhe*Y#<8KR-Ubl?omH0*cI@UZ-%(I_kC8C7?j? za#w8ilxBPL9@@ zT97P`y8HD8cQ>_r9^W!}s7Fx++-#n9{rE&&yudMvmVHfv zE`pMO+@pm9IWjCgVuCq|rD|^`Hw{5C(uIdOKYc?RgO20T_pay0O(&dqE0!yvZO@bM z#)r_ll&N*sndSw=gI4>cs^~ptzu;)2w;Db#4$_=>+HrK(+rGZ#JT={)4pi4(aD2xL zJKmlnzQo^S{-wFe>QF>2L)=dF9II~~`>RjfgY=hgH(YWdDzw@V+H-v6+r3L|wLphA zN{r05v zt9bzF8^4Z~7}d{xecaewHY`Fh@ZBG!4#Va;D%2?=K0Z4VzpUTAk(%xMuStGEWIg^m zMr^QByI1*N1&;G^!7pM6rb8ivEDlb(`m*q%~ zKCDYyIevAwQLDgd&aZ!wiOo5>KQ6 zj2XQA?I*hqg!}>}@BOXDSev#fe-;y4iml9RvnwgY%5Hq;>FK-P+mbydDOe?phjbGP#IV+>?xv|MD{@tW?K+vZogX^D? zAj$HgfzrKC_f{B)`x`EzV?%<6l&Sig<>eC6x3zqG%^M|3p~Z9cIs3V~0?O}EE|$tK z83{nrW+ez#4u4(QD4I@S@c$C~r@c=LI_Q5 zcqM6(O?nbg1249emHOB%Q*+)3NA2ZU*84=zk{y@B$X&WeO2E($Yj(kXSyV|cz)4mJ zRFE`Ux^bbktXaZz;OgV8RHnl^DMrZ61_UP{;fJ5zSLREl;EZS|P zAIq6&b0()Tr!qFo*$kWU+vofHYq#BQ`{Q+c{jux1p4a1XzqeQV7967QFKOZDIpHK* zb%pvY6;oVc>u4qt*V5b0`h%$R0}Dn|{8{U5s_aQ7eYP1eb>OEHz8ncV9En;Vm4@zP zl*mZQwIVZFb>G#V{5Xw@Qe#Zk_hZLfdtlbGwI4B;NzmGogq+Kd14!@xCQgq~J=_k+ z?fof~j6n0eS93-I9KCLkv&QD^&IpC+HiL_(rKBMFJF(GMarzhW+QxDf{ZQV2_@fC4 z6)BJ)yBy46C!ke)pP%A)V=%d^O0A;NK05SYR)o&q-n%r36Iy0``Yhyw5YaXCIQTa; zKE|l=@iyLP)*3!)`(H+m*3J_t+tyyX0$!8i;$T*(g*S>o4v3u=9bZaLr+{8HRiE5c zI7t@4JD0~JN^fA4j0m0rVuKdvRE+(>Yr1w0!kjI&N<{tuPq56E#M}Bs?g+C-%$Vyu ziF-L-s3L2|pI~4&Jvt*jxI5h9)zK9rjXxfx3(&%?ihQ86PMbQ~FRt6W($rc)L&_17 zJ;;z;7*?o!&)Q?s`&9CFqI8a5pba&YkWO_Prs1KU?1x3mBYrC!`~f2wnl`x??2i3SmJ|-3pum>`u&2BO zj@W$m=Rdd&o3?0x-fTJBVdbO~GPi#PolNFOrywVRXan>JaMwN(nUE}U88K_gF>>nz%2 zHCTq2;TqgEKjJMp)Dhpp=F>EH)oUepwuXH;jwu^0nwpB@uhH|&PSL22wE%T#zTD+o zg7)ZaxA0m3mVz*PwuSV!_h{da@=kkCE4UixjpU=tCt9!G=dH2YHXA!QJIP;VHqflh zcujj(5ArkXXZ8rJJoRHiuf8%K>=ktWE4SCDotuq5^+>WV{wH_Mc?s3p=(FX(P5&_Z z^q6=k$=q335+UEUW+%*Y;WUCr1qQhCF6^kYFpapT@oV|KmFI*tu`UoIi{KaO5dG8j zVKjlG#3&W~4!0Hk{2R=H`E{e($HZC8D0&FZx@LelD2c+0MOV~dRX|s{XIYV9O6n;q zQWlq}>hC-jA;_9c&C&jgngjaJ=;vzqZAn6yru=OV`5Ff5UIjT5vL11kah)_W!J>i* zTrQqiJTF?`kzBA&|3wQH?TkZ6jGAf1*c<<15tYEQi1k%OrkSYWw1d~lsuok}3NBFv z6dwU!r+X>WPI!rAXR?ab( z7)5fa>*n%tMD^oLvosHG{ZR2s+2$Hr)Dgk6d!F7-4@}Nbcts0G8%b60RO`Uaf}RoB zC$6|Y@9SJ}<1dRvznxCl5cyW0*JsVSHW@8C4#_pK*E<%Gz|k{_#0f#p%Bi|umGZrd0C zvx<9P-5gShX_z54j_PhW?$ z;I%plvzl*~a()1fX8(7&@f64a7vhw?n$|FY%5k@DGVU;L;>&ggQQqA3cwO7*#A=nV z_MX4~V(fo$tf{*d{T>0y9UD{SB`%PO7z*94S^}{Br#1LD)fV|#Y#aAfhSbmd&_wNT zxk>e@9%p`WT>tvaZvP?Ilnnb0TkJJe#K&(RiF={K!Is8EU$j4NLsYoL$y8$AJ?MnI-ddW z2`?HUzUMo!ss&6b)HZG$Pi()(N|N)q>Fiz^VmX;**9x+f!oO#&KVq7GI=r0`ln{yR z^N<0$P{m$)mLlN0;KXJH`ea68(e9zGp&n%7%hA@Z^&&iP!#CuidO7FNYX+ z_v0Vo-%&WKNOsa!<6}i_@a`){uRpd0(%iIK5KIXG+ES3iT;PRY^yvLUmR({}D)!I5 zE-i@p#V}1%Y8KZ!jZ-Y;tScwKjZIKH4t zZ}Lulm*&FdaXjV4D)e?SZDvBlH|v`B(0cvfU{`HI1WgCMA~_5tnP@ufwxg}myRwtO zW9e|^5H8%w+Vc2LN!!tO`GFcrpxR))ul%IR)Gr#^vU#*peoZY4eSQd2Uyzv^YnwTG z(H%!mj&8HIbGt37+vp>-C;47`02LEb?q0_p(#5raJ}G74wemHkp1A+9BwG};AdP>2 zpzNBRnQRsP(bpx{P^;55gJ9I>t(((d1E9iE`GCE`)Tox1k+a30Z@2ndDpXl5Ag5Gm{nP0S9})z!OL=%?LSId8X8CL+d7MpN zy+7kt0d133ZpN7ZAc#sUQL{L%zPH<&h}0!8TC3+!W)E{w>)Lxi&PG3cspx%OS5$`?bz?z~Zla!b%`k{`uQ4WB-%kw?v&uoE*a|xmOoj)h z_4Z!&+)mu;M&F+bxXCFb04JDCX{`od8;)8u9sk*$w@;M_r>VF)+HvhB?B04LW7g_> z&8JgkFdEH~8&{QF=bb^<5KCb8I3L$=u)wrq4cmlYpsJE9E~D%{HfP{65aR3=T#>0V zGL3uI>_GQM-rl!mjyUhD#ESLGmZ3~kfn4R65h2jpKq%?zqB5kYSi1{6#aGu4e!ZPR zPTiaE`rgpGzu}R;QTqKLIBl`@?PnY09tVJuS~ng}v8I1H{@zygaO_jvcp_I^YL4@IwPTbPd~^1>?Y`WnATl}R_wtQ7+9Emfe=RP{IEn)6#Jnw#8@jd(1T%-1 z&A>a;%Licj!<|+d6?&X_yhJU7R>eUh2eYBsM{yhKI!SY1=5Mb@u~lS{q;5;8-;-BU zzip!?ThGouVlXb9?1D(D2oVsT%iozAyRGqh;*K7Cm_&-TjuAdXujU-Rq8i+hujBbw zcwj@76e__PqtW!iF;(}#{D63u8r_EOoZ?I*>fR% zvss0kHFMKr@f}I@qZMz1RZpTvOEMjnmaOaIdcz?2-bW9IZ%Snnn^` zy>7kn=MJ3qy7P_y;2`W~_ItYzQQ+sG;tQko>^%F`ONVJGU$M*b3wpzE^sG9obW zYb~;u8b*8vW5{4j&oU@cAsN^|i2EmL4C$2cYYU}$uWFnQoc$$5ajrWbkGI8tVqS<( z;jM`UUd(#h%7#1KANue&a{g%Nv&)_}1Z)czdI*JKF znS6nMhWsi%qp`N*bjvASJ5d2OxnXMG0gBI{^*66DtUrB)U+~sH91@KAjZa4RstXmA zpNyc1;6y?{hczJHio(v3Ak3*#qkp7A2D}ItsBW^_Sx0(w4tELJLJ8K;xH9m_Rje;9 zgO-2OrAh~(nB!&GisD>bxK19vl{TtM6W36Uqk71F7RR~D(TdYm!+iZ{)`PP(m--sX zkLVmr11y8&r;2{N{-Xe%?3coM8HQR5V;*hEtYJrXhGNT|6>%+uI6qYj%Y__?rzbbo zcV5cvw`S?b`q()~@O|7}O@`qu8QJ~Xg!mW7fn2J^&5%0^XU5M`Jj+CJO2l5@M4xfl zw?s97IKz*a1NzcxCp_WipnM9hiv*pNP1)@WisShk>8~us(pu5g20Jr#flK;B13wXF z`t;eSzH?7~RuiXwuW&H4ZAGg=0ZoT(uyPAHb#J=ZGNidcDgVV_1t@v-aM$J8GLTff z9y$Jn-<@Mq6K%c`@cn^op9JFw<)7=nn)7`XU^-$@y9==ReH8I@Fb>^!ghQ&r$vC@%*}tgTX1icVXm~5Cy_~!H#~lGu)@CgF3SslWXYJPhTQ9Lr z*P9EP{w1DUM!zTcfe%P74uEl1s#_X=>NR|5j*)%U<5ig%oqRbf6%I@qOl4Ev&mG15 z?>~1iZB@V_)Ls8nqY$9Q6F|s*Bg>sY4(@4M0{O?e25UJAh(6PohPj$`mjicaKl)o~ zIf`x_9uMPFfHY1!fs-EZfa{2xiNi;Vtv%?I zf|vk6O}BU3C!Uz<@fb;M_Z)F0pi+UJ6hLlMoe)6Tf^{rrVUuh9e;jebE&S0u zWbL)I|FK17haN*y`EmErD|Xu-&gS-SouS9-)uXRae*I*@Mts4^0EEM)QJl>P{TYb< zDE6O2Vsl-PUc1Mu4QMg7y|Ae+VzI2<;n1NuZiaZ|jHEKVMI?;ho`{}!v%aw#4nZZ- ze-q;Ov*P`O4bzz&%*$p+zqx7cRHe8&BVtub%h^cA>!TC5jtt#$srO7zQhrhTGJa%` z-}qWs$pHVb((0_o7f19pRb@#n2@sr`owfe@mq~`1;Lj-+k=1q7NP*+|hM=yT6pbnc zWQ~~yBBARi8;+zL@|n|DX4*-O7bYn-eh-@mRS#ij@bx(2DrZk2 z&L$d|iF#_a%1S4=9e;-1HZ=`u+mcEMkCP4wlf{@zyJ$;cnr0Wol6Kw>K9uEW-4tk~ z;!PdpH!G)b5B>>W0S_6aLKBYIw^=V$E^tX01n=yaDB^HN%i^wxz_4hMYnQP5-A%qX zQi$T0h;B(RNOYx&Pu+oon^%+)w zn-&7~qO>hsn|{^TDKJXSMB`(w5Pt(Ua{RN}elBf*z84aYlph~#t2nlK95n6qcm&T) zeXr^qguVj%?HawDgBC|+%4>ElpxA_S-|#v2Ig-*e1Kv26@oe9XOO*Jb$nZPZ4vr8p z6H4%Yh?A1kT$0Y`Ssbf0Sz(=r#Ss@H&{{Wwt(7~Te-Po(!L5+NoK|%R=b`IBVlwRjV5-|u-*@}UjstQBoGRn>GG&-J zSRd0(ZU7$!u@K=y%E?A%^SSy3p_S>yi8SJ%@VFrT6r%FGD5jO(`7=1F?^ez5WN5^+ zh$Bu!Nnk;1=d%YAPqH(NL-6P>*mNA^8)>m+&LF#5SW?49zPQNY1~1Z|`_KeUc%2cw z!V802sG3r2dq32%fePgH?$A4Kjfm+b+~)**8d3cjWJQrHOV-jLL%frl2CeGhl-R}`63pA|0^ z+Ko%XK9{(ZZ=6dPH9E-39`rVJW=B;)h}G-rwPx;TlU00+c-9DY9jhJD6xNOA@zDM? zRrKAVGHqQcq>sEt*~+p*ELW8oQenR34O0?2PRx0SvM+_Patcte1Fa#qc|!SZ+&AhZ z#}?b&&{viVPM$|^WPlI%D2VgTv({FbOj9iT-FT<5qQAxY3)BbQJLi9<&-HM4L<88)EV(=DLpEyxiqO;~7zsZsD9-v1#SjJM4y0C@ zXe9V7-&l<8ZovHlS2Q{p=V~)|GF-L$xS^XKl}?en;~qH+;uQQ2?}0WZXVdN^lMOjZ zZhlZ+{|RCfO-AHCI8E>Uu~4;=23S(>+vW7=X@18nfrwo&fFRcjL_xQOlW7m7TpMY= zckXHI83}=Q5*TmXttJ>o_=Vc%z=Vg{ql3xmW8F}}Of=pb^tzeW~DPMbWjxbN}K+9#reqH;L-~ zDKd&bNymDKun9fOA4Smx>hu<=No{2rTM;!9cS(;<;4rFxI`;B{`B`zm_DP)g)wK=J zqHj8~$%9QpgFm;QDkE!eig?9oEmjWhJ`tS?-$l<Tmi;h>B?8t1Ku13!r zCDZI5y&zQ*-LiFXS95aahT%di(3YNSl9_DzL}QJ6zALrw!_D#Hg(M#xm#`qdAPlPzxdz~YIW)R0Oh~(-Btej0U9M> zNq1$vl_BPIfC%NbREzN`ssjhb99qmpe%VV|L4JYeycCwC#J@3jQ_#ccZT=Ntc7~VE zMk_Rc{z#uR5tXJDJHCAcWiyzL}ZI zp_f^Z(L8VtrAIZa8O>s~6>2#Jv0lf?xzb38X% zCwKzfKQwI3r*FCOzU!}a6QK$fNOCUe=7{x)yz5KxO|X2z;6jE~t$fF5PLf1i?6Aq2 zbaSr^m&?2C{*=kr4TP=W?&dt+fL>cd29f&cns61Gi72#OpZ$U*ojv|4r2X z6PYy7AeX|Q`HoeW!mJxs^ciwjG_f4JIgmBNhSCK>1l`ybGR+>;)i5wNCeZR0Clrsv zW>!vlg%A{9hGkK%?|z>2Axt6x^EP^yIeeXMEV?j5u?YdRc8k3-I$$FUm%6i zbJJtEUeEkobT!aPNWXB}CrKGDfnkZHRH^U$2oavy6NY~jRyjuE)?$`{eX2%8Sl|9A zr0?(wLKaApD~in!Sgdqem(_@JH=6*Ax_nzvJ7Up?T57(PPM-SXP5cmx))#jZHV*N& z72eWaS)$-lYtt%#&!u5uj%rUFxeCeqZ+)K+3fi_1hr8KO{rCd%DD%?2C<;gDD4gduxay>X+y>PSfXP%1yFYB{OS) z7MGwkjlT9;*ehr&o49I|15cyI8h0F@Yst`lteFipPc&KV!$l*0J?lP3cC1Oe^CUO5yPo82V%+lHb3fple!iwhsu;+MF_rWs5w7p2he(v z-k?dZg_@hKKf5ix>t{TZ$)gxj_+cr@h0*=KwpHBvbvf6hWkrhTj0+CvJl%L(uXgX| z_H=O9Rya?n-rM2Sm5PwPX#Km$f%M$@Dxz3n`bsNmqW%Fy#K-e+(h^);w&-1Fr5X1P zR)}gDlA6Sb0>56+Be7HUiBtt)>^*iF#PmYl;D_Jwmnb*wk1c%j=!ao`lgU{lKiw3% zQ?}{tDbFCB4!9<1by|cZZnkRg)8h@1C%;=0enJ1Wl?6TSVz7w?md^W^KA>MFHL+pY z-}MW+wJJM~JC)fFaJfUvHb!^UnI=P7N=ROTaNkutFc03NJ75J>$sucpa4qhZwsJB9 zc!r|3>CtuR+%i5dt%2IQa~nXTD2gIKp_LCh1}-5wsEIsk1SQk-@889%zm%lB+q=}9 z|NRR7;CkSUXyK^^>)_ynS88H7r?P9hQh0lZLA@S5LSiO9=x}%-NXxKw6ufI6%rb+? z!aNByri8ME`V$gk_@`4{B+K?GjmFWEFH$u~$#dtK=nj9lepF?e0j_(34M+Hj>SI;e z$KQZfJ3g0^l*c^(s?Dcvoqqk6$v^38r$O$#Nf!K6ARq3%gy_*{B;Q)EF_>wo`4Qz?r#hfaB!d7sKgQah!k`zrs zS##c9P7SX*!4hlggc-+~6=Jr?>NV`2OhTN8%R*#tBwi_-7S4s!lWN%MtOKb1;&O)+ z{`Z-KLjit)fXrPm27Vy`Bj^MA7$O0!qcG=RP#8&)IFI1tw=tU1^eg16KfR%RS?Pm| z+hm8ZX8nzn?u(V5?Yo=%P-eOrC-(+Li677R+I@oyG?aRpFYjM}@@W6#ldQ;vGpPs! z`DtZb37iWl<+E-FthrL`1s|*z=pt)Yfxh3c1j?keMP>8aE3hz6MH_;5dMJSWI9yM- zNIdK#O`ioxxX`E??nRqIbDBg_Dyms0yiwatJ3<7!+}YtPS1Fi9?a#IVPz@q_nFys!2FX&2HVdcn5x|(ZSTJPDc zfrDdp9)*Zc1zB;?8#AkTw^4lXMy&w1}g8R>ecA@6{^|91t}C2{}+LK328t8ZrNa% zLN+G^FsPA~(Dvn+Kph>T$}0OxEAt0*=0b(utC4?E*VJTwnM64v*r%;e6DryD=-shr zUMHQ9`_y>HDY!@hp?#Gx@)_Qy%CKB_YCCnbc~fJ+yJ5=$9Fwul+MxAttR+GdRqw_{ z`mSg{QVzW(%-HWpQB@Y3x)#{tKHaaP`;tEBeT~1)#dK3`?mvm7N_P6T3HyIhXqcLO zB76iD0VdNE3Yrhi8=QPpPBs>z(i7Iw+b^oQv~t-aC%imAOh0-e{Re24qB2MI;rJ4* ze(3uOm;3BIBSfNU0(ku?h<57zq+@ZA2nh@+!dux@>^{rclOTU$N)-G<4c9Ip>mAW3OfIc4}S zHSDA3MMl&@Vw%L%Q+kMi38}Rt!V}+*U_AeBCYt!4uV*0hlPWQDhmKgs0$n?Z2pAq4cW!6#CEC<0gL+b=C10@oP zWXkBL8HRQymKUAN1|%C{v_SiVwg2dfWy#SazRGJ)3ug9L(^cIXH&2NGzX$U32BoPz zZ$bpS_p0}$DRH>CrC0D+&mG(sUBLokZx(5LFWd4-+!?6kqCL|7$=wa?yoA8@Gu*k? z$3G|y92s$Tvj6!MXE0Hh^bmAiqKX%7Z|=5Q0+_Fy+aJQoAMjDrq4@6ewz~-ZEPN>) z58SB4onLzQ88~ck1UdX?1m9 zDs(HdM~WjC!)dV{4a8je3<>6m2NXg=kw>P51|>oVJ5v=>-UX2BD`J@V&lc7^$4`1x zsJRbEHzT~h{q8kQB3^X!yTJGmdCTjtA)~UQ)#><6DD~W9)J3wd4Q`(tevbBYG(db( zBzf5iBKMSj`F6{mIdWh}oC_&7>L^Eg>#q!;*83I6r|cIK?Q@&+SE&ykvygiJVGVSL zoNp`fpUhAX_gUZOU;h4BZ%JA0)|B?&{2gsM0pqh|nAcoSh=%+7(<`fp9HY$5*Pk+p zl7k&1yX7!UxlNW)Jxe=@9&TCn6D0*250M&b*K3D;9~5OYjPFWoM`2-}_u;yWMHDj` zePOaXUgE*vfjh^UkVT}S@iPS$<=YR8!#D=yaq=75n30^iaMb+aBH`QyJRX0{)^5kuPK|G z`;~OM9LHf({<}nXI|_q!)UnL(#Fr9-3%NbS5LxBNj`?2(f#xX~pUFL`&2G&NC9ul6MGKs77I2T0dgTV{kw3mi4b)Tf6Exf6S zkbq_j->XcM6Fc+477t);@k&KT(<_&g>U~6nQdp0BX{CKs#jDPPBjC{67!sl-t_J63 zWWxYmxpr|TxpSFbFth6zRP2p#ImCNUnl0vg z{WRFl%t~eNl|W?T%R$LiG=J8t~(XosO}sOb%U&LQ06M#{*~9Jmd~)f9(50|sw`%zxg`IH4M50? z8TV0=t^Nh8A@F>pKDFhk48Ny5}j=yu!wGlxzt311|^7Z+SGMvlUYo* zsYYKt2;=33O#q5*F|phGshYz(`8~~w#Rk2(;MGG9VBZl{Qc$|C>qFJbWJ-;R4LnUH z0Tzbt*t^o@Z*m_pp0}wEtC-tr8|~HHw1%*WI;Hz`qi<3=6KRERXmO%ZpXyq@>h6^^ zKOex@-GSYbj5Lh%RXR65FPlIn0_uPXTF#ioLJg;?+M zf;(n@D?e|mhqkw@B;Z|N?U%xqns2~=p+ptFmTQ;18raME{P#}_){RTq>UO|iYsqaJ zzk#I5ax^lLc9Pjw>2Y6lUh=q$kOeN;i1U)D8c8{5pW|fC7VpvH%QRUoj3nBdgZ_Z7 zpR0qa!FsEc7)4T(R6s`F=#K^K=1G81{57$8!_`$y)az|}#Xc1uGGo#5EZQw}-GFXY zDEw*rUfymP<5(D8d|30*)@44)lLF0d0sr5zTKNBt)h`3s=v&a&be;`5vOwXman6qp zJyviq4?Us+kK#8V%vub+X~Zj-tf1AL=U@c+#tM$N0Za#vmf~9)Y{Fiuv!zFIsS5L3 z@`i9tziylMckJzt5x$|OVh-);ssYAkw#Z^&Mlwbs`+W34f3j=$epUa39yVE6hxq%) zw$L|b58DHmnp-XvN#1O`7TBLoCv*rE?sDg5L<&@gqlZo@g4Z5Kpu_2CujraQ?JA+9 z2D(vW7>ZE^&5xf_bP;7OYoBFmPYElkIdQG}+EzARcL?pa2|(6Y&VV#Ajg7%P%g}AY z+oqhw$d@mpf;uLl{QvS*y&M^_H{~v7i^hehOat8ojMff|M~X|C_F+3&lxs?4NPKtv zj%Onxr6TswHYJ55_JU5ovqzzJ2q)ZOs{#H~{~Oz^e4sO0pvQ&n-3-ojcLLq8hM z{q%dgU>%4q^;FB!XhH8<6otlB8g;8PRgU9+1FseqOBCL43liuR1HIaDTr#i{yB30F z*B+?k`_M81$*#fp?zqdtt3z;iu;0Wk(Cn*n!0;6%UP#HVtD3G9)7^t^!f1mb-N0{| zbuOS(COulkt4Vc6Hti`NGyHTCW%lHD+fL+FNnNZ?QIHTJ@c4x7ck_#_>i&@~(a>R! zImr!xnx~_t>)A)4*qn#)lSYgW!D13bLCC{{Z4t`;b4dF0gYi=J(a0xLrpwmRU9{*Y z`^8f4f^pCmHWWXzjT?Sip1AaIJKG8J*KQPdoiLkJ8|NQJh&GMWc{Y~FK0&^cbo8m^ zyZ+u6zE-t3y~$-AuA}0)3`sk!vgS7-7BhC1A5nk3#}T3rrd&a|mB<=3r#=T5+Hygu zt`b&cG9|^L(JWNf)taS|%Y2f1gvyO=m}2EKW8=UFpX_ddJV>#2MjsBhuQge&Mwg^y ze83_uQ58%9_K+DXIRv@Z;n*8&Ln}X}(13kY8Oef>sjHGTcE}l%3L4}0NJh;yDmk-v zNa_LIEJqx$lKf@FO>8O($Qs5?DCt5N(o2KZ+CVu;WtGnhXpSN5(>Ie|wyks5(^2P7 z`7q~0HoR|kcujuqAR1X=*{OMO+0Ar{>qlQk7=lB(Z;X}N=Ss5=ZUZw^cz=HQvoE5r z>~iQ6zBakxJxsgP!@bhO?^ni+HG$G+w)px~I#nfUvh6k9mGIl5^Xx_XnN0z5$=%zk zv-#YyGu~{B;ek6n*3lEGeZfIT{-sGvXVzW_`=1S`Rq@D}$x1)B+)!nW{57Nyq=_ms z+t0G)=!OKClN>!$NlKdKPfe(rxUz9@1S6PcQZcVue0V<@*}5AX;4{d-aqmOszCWY= zBMRd!3E!z!nKSw+51_V_ezc?NiJ<8dA?vV!Qh_BHpz$`e;$hGFhTet;w-7hba%96{ zq?yxj1_rvmlKOKxvXGLoaLa`jH9r+sf zu-=+pR~Oy!`86QIMgYm$nP8$ds*<%Di9NOZY3L>P>b5%kFD@g{IOgV|UbcL;8B{A* zF12c9s>~}*vFF_?qJ;3)(ak6~X2`T2Hr^uE1I?olWMO37<=wP!)b7!-GO%{tO-7h%a(G#H@))2B>kM7F=2a~nj$9TtO9EelFzB z4*`O!)r$<5q#(8cV96fO} zGbEL-47GtpU^@N=9l6J8u~z;dD|f2#dtEq_`p<_LzF_`Nhkwa~~L-uVAlM1|SS zm|x$a-scB1TMmj_mZ>deq5aFYt$P%+jitc9M1@S$X6vznY3rUX>U*dsKzpkgwX@vH zkZ7e);|j#|jz>5*m>)v7sD03VZi-9->l*pok9HRWSsmZVbm2?w>D-`-b1yBj)6+W9 z-PUlU<1n5JG=#+465p8k=<}|9b^0s6Qk73l{hjt_XS@LipMKUe0#SR>a(<^@uuqBc z>h~zef@o;+i4f2EsrrXz$CIuDBAC1(1JVQ0Y@i>3W0)LP!P}Y95rJ zFqv^Q^eF;`UBim_$gP@bJ!;PB)k|%x^oj^OGJDjN;Ix^aEZh+DS$6vr|7=B!XP0f> z0{-@4t;Qd1w%U)xHRvhZ`mPLi{HZ+3b)}uN+F6p~ zI0TbxVUzu&hIi@OYda&AvO5E^icy^4Eez3QL6D7J_D$=zG`9XPfD`Zl1C)5aKhJSO%(Udz({}gy{C(x0npA zp8$1T*re3ThMeB{Nco+b=%zi?UsXA8|mc8wZH4G77;gHhtf79RIi*SRBJ zibeIMBXws2ygh|Fc)RreHn}HF3`Gk`E2AS-$%m@qlzI&85~+u_ikr)8S_&;Io(jvj zu%Ga&X?4|=x6M=@&W_V<+%8gy58qDS|Duttz;qTH>qo)=Z0x@|ayu&yU(M$A>0H(} z1MREI=VYi3x6;s)wKBm&7tah>p?KaR_L5Gv*cJRza<&_fiARPFi14#USnM93CS-n( z_Gg4sc@l8aqhjTaCqlN9eTTg5dO*HgX3#{)#qRPIi!-nXK@%Udxa0Jp0XOwh>b)JF zyE*)pcO`zi<6!%%*4a~QMs(;(9Xj40>i^G}z`ixz=to+=oTMa-p(T9cqT3HH^|Mm` zwduMkKb2L%=Aj<3i#P9zp?af5+nDe7et!uP`-=KX*f`g_!hcgNmZ@j_yT{{Z5FZqw zM{cGwQXU8-6FG!X_k~`>RYBy=w5IOlaMN_g#(EzHmkewG6jJO;_X=Jxo9gE`p#JjN zQ$dA2?8q?tJ*NZ!4@->JWt0UizQh!#^w{9e5uJ`nj4(w+Uo0?kGbdGXWA$Lpt`hQi zp*@-7RAX4Hz{Y4x{{Dlpsks%7<~;=xoD9nM0+l*Cv-Yzl`|@(3QCM)c$m@lb$~ewI%? zp1qM)@Br%9?D`@T^2B3y=kumGV(@8S^fz`&{V_VU#0?LUQYUYGi%fj$p!HfFr2e;E z*l-$wM%!*TbY!F=wx133Nj5mZ+{wF9O5AJaXAoyuTfd-fK9)*C`U&x3SsqnSyX$;< zP(Ti|rkIEI1DEQX;jKafNEq94?^Z$Z?a(3^3!gBMF8*W>;P)UJsvXoMLx1`dS7y@I z;XPgKU%wCw`IljH0vGN@(tu^pRGwM;BJA=8t`(F*8rI-X@!?#WQc{W+m2lkPu;-35 zyWE<66VI-j;{DJxU}YNfrcNSgZ#|790qexwsWUmIChC(@)J5V`HN-bMJ4Gt;9aqF# zt8+5ly#NSyOMz5_c~+K4v;%#AK;&O!Wt)9j*Ut1@q`Nfpa-$(=VoN~u?f+)g;h1G^ zyBC0oD{(b|(JOxDAq#@ikD|dla>HU(J58Kj#;w1$-cj|Mu;PW)EuM3+?ze@4ol>?{ z)d!P1leV1PT7F#L3E^2ZianI%TL&T(tf?Hmkq)*&b2h}kyE8$_O!3Fdh|n|zwr$GD zYRGS*OKa%wh6mwsc=&62Z&a^X&Pe2jA{?h$S7@;4aWVk<`cT1dr52G zS58*H$s$`q^c%REr_sWgahqv>79m2_08L0m@AJn=GfkJw1f|O@(Vr|5=F;4V67u*2 z_=J>HDyoneIZ=tBh$B9?Nd;2-15_2dtly)JRtbFjSpi#uQ%g?z=YSdYcwSLcc zdUE7gj$L)y_IWhwz6j~D`cYcsXkT6^cffTUx* zhxQ}s#d_?%(VmuIif)R(9lsFnSM;nO>iX(?ommEvl?m$uw8BL-1Qxu<;`95~+zO^g zTIYXYTDDS-X=N<0^F>UcU1&Hxar1PmI1`IAg;G6GBg^)!OB@9ohesH2Wj}KtLjSuw zbxc1Fp$njx3OdZuAIDx8p;lcc5rPV&Nr!GXEqfC?NNpw&uT>CE#8+WpHT?vbXXX7I zmZ)TsnrBS7KlM^q$5ciOX%Xj}qY%vTfH3ij;VMs5^CiAnIWe!CvMlUsx3WrSXS$~x zxvRPC^uq}eHA2wrb|MX>HFksrJhGQJ1`iPcu^b)PB=$ z;^VK~`4FD`?tNK#R4~yfTw!*4R$-iPS@1cL{ftk98QD`qy}}!I z*b*#H@uhEZA%~;Q2am30>yX7>2n%6X4*X-VofxYPNQ{GA64?voL9r7NzXR6&%Mo!@ zd(-oi+$vD9ryLY9MAEhx!Tc2md$rq_X*ekv$9ZvM)waE#oR)9 zTHl^`EM*}dG)y1hI7b&rRt^;UsVBQCxayec#1bBJ1{UWwm)3NLU!1v#|kJq^}bnq6k>WO8V7t zhBVDqfaW>y@#EHl@k$(Pr*UuPHOL$$#J_?=DMuL?YUzygFPP`QB#YU614!x`7hPF^ z6(v~#6xK&+p|;)o7RIJohXzrEi)UM)MOtTaeW%7}^c>jXn<1n;smXdb^4!G&29H=q z3luhw|Hkx?ZFb|TyY6zn0optEv#`z_C1^&GdblakrubD%$XIjcs%8+LndTV*a)C1k3y}%j& zxMf{Y;Q}Sepy0=!VAUoPfaCF?z%pHL!s41=1w``n<+I(K#=V@DwXOgM|F!!s4ARB8nFLj0-g*|6CR@;=y6unUaL0pL zB^D-qLXXjPps6bx+kdz~0R+abl>azUfJ?ba`W&6OKK?Xdhq;rYcChQ-(q<#&pri2p zm8jkT#j0nB*+APCaJ9xx51-8?|)SQ`e7$RlwEGn9Khrougg z2j-s-6H@w4A`PLWGbegQhX$Ma=yo1j`EvEN%Z^%ob)Ev9>WpNJBoVf8J8pD`V?7Nj z<0_^eHQ_zuZaHA>+@t%x?T|2ox&9Ut`P|L0CnxHxiQB>$Tvmgix zas-J*7D*>G*F+`w=~!B^wyYOAlh-EIlw6bn_n&z;opUbvb!$HG{?(KN!&Y(1)!kak zF3>n=VRje1gNV`KeSlbNGCpwbxy_>1BE6=A)_Sf0RS_FQfcT(68=Rei5p{xJ-vQ6~ zxA+zKUo~J!C$PJ^+{@;IfIr(5v?^aHu>W4c1|nMfdHUl(9XGQphrO2$$FvuwBmWz^ zF$reXCqqk4_34j>gjOGJErMr(+A>oMv4@*@Hb#p2hZ^^h=`XDVPLubCPg(Q#u&6m% zJ7t)fG(3kBl>URWtiAh9Lqf}FWhkp)@=ES`^vdmVYOIG__bx@nMbtkuS)px9;>J&^ z0(IGo1oY3m0_q7zYj7zSjEJp~GLN*S6CO*gajnW2>8!1Azc9L5pi|VMDkxDNOLdVS zyGXM2H|`sEqJOz_DRkk&&~UjXdLWVU5S6>Ur1&Al`Re3br#vCwAE{ zf%#Q|pI@nHw^++esv44QZEyiscBS>nJ*gSExB&z>BcLIO>aOS!eACE9dKi0$s5Z@3 zLl4&nH0-x#$tSVK+3v?lc4VF#FPIgodJ!>NY2nnZ2w|-UqnAnvubj}!?1{E~AxTFq z!tXL^r}E)CgKw1WcV^80-+KD2WUgaKkMQx2;+ccB4K!*6gWRM634_SR4fGM28L>i} z!l6(|$$ifwj@?9-?as;Yl*Q#gu$6mUwHZe1xe~K(ZTx&1rwOxGq`N#tmM(uesHZkR z2#BpYoFU{u;-c%>L~lD)L}hQ#3V{d7k$KKT*acLOy%kakc%JF+0^wnWza22R*u5OW z>5A0|L`%gquC}W9#S!z)V}CYky%TMt{d6RLS+isZm5fg>_T>k`V?872Pb^oojVDKq z-;2Bdx1{i)@qVxG1Y^y|yZ%e(;|Yx~AI5fXtfjl<>YitC#$D0CF)d{;-tAE;Yy)5) z|BbZ^w|eyv*tj$KyA@a)c1FIq|MCl+P;W6QXvzf8Tex4M;4hf7o7i7p26TyF7>iYT zq0?`A;-wj{f)KWWsWrdFHSYYvdr!>YQv=G=xE1d99I3tifExUb?qB9Sd330SfZA_z zpS=CNpV9WFg8I}m?D~kt-BYzk3*3ScawjF0L*8lF3;hSu`z{ia-lwvh^~JhXr3^$s z^K|}uRmHWdq0mTz%oYig2Q3=GN|BJ(@^Y=%(cPQ-dbyw=rH18L@jY+_adQILVdHq9 zCw&~(+?>K8fq6DuOHHF4Jmtu;gV4xMNCj2H1iZ=y_B#?M$BCpz%cYDKZHp@1kzUhd zdKwmqm&KP)0;)L`sHU2Z0nd=anEFBWJj}K9ZE);6+PBhea!~|tNiiJ3E`&fQ6)F>L z-+q|dCqf5&naOv7$DRg*iCGm1LLuG)k?PALsT~64e)<3GT_?+?4HogNI0Fc!>*Am+ zMk{3PdG~!8cCDK@ID{|Q_QaKM1l09>%9VJYxb~Wrj6fgN$B0ktwzjJs{hPdIQQ~mug1!jo zj7(5qnb@zRuU?Z|8!ujOlg$oHwO(oYr(Er9E`vX8)S+=qL|i*aI?=%T(Di5o@ZPIe zhiwyb9cend602&Oh@Tq&q5k4>v(-Ez1F$OaSdCm5bi}9BBgyzwWYFw1@PSotLK%~Z z%V(0)o$fT*^@qOITKHi)b`VAVxHFLWc_q}+2Juao!Kh|OAIkI?-?MEw?sku?=llid zRpsoY??i)8p-E0BuEq5K{ukd22{($F`i&6PwR#SpjCEq#y!*~ipZQnm5`H8cWx0bU3eMa&gc=z?`{R85NZN?V9D?donkMLZTq6N)Y8UyUQr92y`_jjJQu#I)jgzv%WqT((AnYc!}LOUW!ai@qMG!A z>5YwY*ki~uD9rSWaMF>klJ5syeX$BpI~RLrtomHYxf=bQ1+SOOr!SdebHJVxH<(&a zT_V;Xqh2z*)54#pda6Z3j@DOc^C`tgrq`y_7T@GFaqN8>3U28ZoZ#E{8W_D?!vN@z zPt3$th56nWAb@B}CTZa_zg~}xhrPc5ye6`DrE_aZAv||OGJmiC0?^1|+CXYem~(9I z;q9q|RVH<%n*m&RPH->ntK7L9g|ZFaX#{jGU3#E6SykL3ie{agT;Uu?l#0#YI|;qKiEkjVi zw?qXbVL4n%l3Ey-Rc9=l7`G<&uRJcYcaat)P_g3CrhZEN1AH_}{WX=*87Sy609o4B zgqiz!drQHrKC2^T2GtX*^Kbs%O10E=4SE@D*5$tAbC9uafA{tJ7&$KyNP3EK};M7fWOO>U+I4>17c_Q!{PG=H7RKezmcKSoSYFH=l# zcPhlJK2(1?o}YRUBT&-U%`XKUNoRF$;PB?jh3 z1pc=#AdZ3kZtpYWB-HBfesP6XiQhRh7<<#I z4{4AA?Hbjn^(qnf0=~F^T%MOA8MZSC_$Q>TF)Z>6TRjrl&?0C5LVTICSKVZL{$cccT1mw=WmjTR%xyv)(qm744;Y?@ot@yCt=5YM#? z1IduN-=3hHriu1GCOqC*ccx@}-+|*k?$fQNrYBZSY6f1wFXX~8q9zLS)>ern{ zh-}Lt?>#I)0{G8Bt2#C=>%g^lSv1`yva$0ZcfEY~HT)a8zEzVAm1u!?szlgpM+Wf# zEtnj)NYSL&J2kF1ip{YNE|I8fdbnMAwIGuP0$rHJ@nJu-llA+Oym!XAR|#h?MF)K{ z1}7tjMVyIal7)BffjC0*ZY~w%uGdzZW^S1#sh?|ig~VA)xa^`C{jXgHSxs^WV`&3kL&jP3Nm-Uosr zh*RP%H44>*x}NrQ>(khKPW_mdozo|&1-j(+&rAvLOVMk3!^Mr|Loo`0QE@>sG9GO) z^eNyaK4cA3iti2(jpZJZ?1ZEvZ|lO+SqkzqR!I zxFg{fuzjMX_c8Tg_A~|Qw7bxQ4-+fo;s)sO1eqS~ zgLuAo&uk2}h!N-nwoIk1+@Rq63sLyu?Yru0s*kXlvj{k~ypSi}x<7l^9d~@NXsBn-{y!?_{1n)w{o!InRjae@lxV=GRkYHHO z$j9pY#hy^TKArJrm=n{!G|a=t+3L%mcM_e_-H>Iak!zm<)Ap_)eQdNTh>no&i%6HI zs=Y5NKCoBWGgnAA)|vw!)}y*?C6d=-{cWEDM$4&dmvKas%&G;aONnL#%uPK93&E@9 zxBZNRgi{J3@utWHHAzGj=T~{d?2z2%xV~F3i!shlPbaZuG}}c`RScisIkf8EKk<^t zl~DMw251%h>(dqd4N$K7cmd>kb8RAh4v1cB9*3O4e(e(<5Rb+U+P9{vNqU!A>}d8( z!lg+TP}Cycm(Lp`Qk zp9X?W(TXB+FXQ&(?jI{pQlq8Th^lx`?I>R-Tr~!>YVClFilCl$VH@4gRr-!wqxbCr z8S*$wg(_Dcs!;lJ+AAg}`QsYiReRHrS;74CcVv|`{b1>3wj?+Dx0SNK5(P7x!-HlR70eknM0`fq$}0x$*N_8rNG zchuOy$raGwkO)rOjBv{o-rztpP~`(#pB5-;9*Hlhj`~T@?ZaMlgdh zbe+eEx2UK=!br2$=x&;NRU1Jcs^?Ho5nXc{VLi*gv5ejWiZeH>7Df!O;BLXXLUdv} zh0>60o86T#UgB-M$cV)i*AcXTG>xywH$eilD+eqBdWO+0ovd6MAlbpPqo!cjzF5?o zZH7Ctygvl^=SyJIWi?V(&|4rKK=T4B7aQl9ofF}%DD`J<{fNj5SDo9Bf1(`_92f6U z5}DqKn*s_6zpN4cu#n?#zG48ihAd<@!sA*sBUzF3)K3#iQT4=zQ4Ay(n2AkieUqnH zD=u+vFXHi5q$@C^h6GOrTAdU2M|zLT!{Fp!iK-tr7vEjQ3}gf{9YBH}(I`)!>YHDg z&DeuwlxP^IqeiGNvZWLG<8*!(D+eW^_~eggsk);f5Y%P!0UN*00FoP?95n*RFnx=4 zCOB+}c*>?oMg})2Eqn-9cS0hN+*Z=Y%P9YJP;9P?TOiyaR_HQ6rHO(%n|X;zcX9h~ z<#EntZ1mb~rw2$An&LrO#4`ILxIdV#xklGjs=k|vxlGt*#OqkyW2-Bt_hi$Kl7?#F z=Lj1j5;81?ModTX%x-CA?*%)WlGSnV%gC2}0WI4R@<&h7n8yjhsH0uZ|2405{n^J_!1^exszvqp;@C>HjnttUTq+?@ zoc!Pt8!Pe=IRXrLP)7lbs3Y4sis17-nWo){Y3D9dyz$`DzuZ)G1zY9Su=^-g6z!zx zb7N0w?*fy1z1tLIeUd31vUP!B5-XPL@R^??XctlD5cKX-y7tKRfoc67F{Bp<3GLhz~Iq`xp2Juc8A{mJOSF zy$-z8?HEz3e-&j?M13ql?~9QNB!bS5B3i+2>-K8Ms}1aJ4C;D}6H-4BtQ&XnU9^}C zt8eGuSD+sv4E@~i1BZgv{n6(~B^r=KBW)qM1x8oR9Z5F#W~>iF`Vgo*V8BpwWPxqs zgdSEyN0T!Qy2L@C8zt&659r+igt2yOu|M~hNoc@YE#9PSGfJ2QBFP16?eI%r7C0ri zv0R<{&vb4(P>tZ`Ej8TmAKWW|O7o^`@J^R-vM2ML5OICSZ^FE2Al0&-P3lNY9db0^1m%5WmtAz>y@jb`j&q(taWvKGypYiYL-s;5Bsyi)Mjh5Uzyx<6?7dAVO5OkH>R zI8l78@Y6j#$K5{s`AfcQ+=jKlBsS!mpl;M1dySfWGWqn84Sw}8^YQ9-!QM9MR zdISBsH;CRX1^Mi;F}C09>Y4))T`uhr4A*OxrQhi|6B-3ci;N%*5!vwuCu#GnORKS14j)BwE0 z?0Ff<62f#eT7fWI2qTSvINqjzoAqg~`Ir8p!9gLE$$kfS3GY9*WalZ+PxYORTX$nAslnf;>u%UJMZCUcqhT8u1Hx{^{tG6p$Y{P^A|K|o$Gm#^&2se%cd00eUn+$Z z!6k|)@?j*ExX7sTfNv6&{>5W1A zJ-Y-`U4*B+2J21#IqjbFa`uU=BhJqblFr+C*T!Eq#}-g+(S>~F`a&ev7IgcQPZ`4L zD^5A3;JFW{O5DBArEPKJIH;3ntt>!SNKz^!>BMR{sF8oAZh17Dr=f;@M(W+C{4!Gw zVxFV(xqia=Mco(6pG!z28-5a+4W}%(mlpw^s$N|Il^3~GA5$~AZ2OTNNh2Y~cW%J-A|?I5V{QiW!>y&~LM{*f z;vtd<`<2kd_;^NtOY=5i_|~8D+ED?W>1r9FQvUf$&?NRm#lU_p()%`2x0uqj@&!2S zc<#amr~0 zeA<_7UB5<9NuQAzgCFpf0YgbDYk>kn+7hmnz}lKhhvoX%9nE);z00AIUs`H#DzhQs0$@8OsF-G41Z7@Z6R}5qpHq#-jtG?khyVHnK0HdpR1Wl)L zo={yiXpr2=hgkaMJZnh^ZK55CVnTN@Lzp+`pzZgmgZpHs<4%rx9K?ADKZoj5WV}q$ zK`T5`dIGUq)IHh)2YR8I;wj*~EWMa1BSbG~R)o=f4;&Kb**03upSiw9OV$$XRsVVu zmc^qTmI79UyP{dQCm9(>pcv1|>NN0X`c#N6+f;cw=_p%>%=kzwSQK&Q4qdhuqs`$1 zFA@NO)1dJI6+ip`d(lSp_=0;J%1;0-cjQMum2|JxQL~sGUXw%P?49xf9`F@YJHPu! z407LgS6*?xZbTzuOe1?5zl#AQ*Y;4XRSAa(p76ts?B=yZwzLBp28G_`hZwUWu$$8x zhy5dJ9b&W31^%Y$C5v61i{bt{sdVZDu{@<_STa*Ag4g&f_msCMur_ZWWu1a? z()kj)c+Di>B79=T`k*>9?RF#aRQ6@(oX`E6^?_|4a1+lYn=RHx1`X*4W4+C+fi6%f zNm8r7YaAho%i+*^UH#}lT4Jymi0T*zD4u((z6J%&bJTE~+HXl`JPYIglG!!CU#3E= ze;S|K`0je&5c2b_X_iERFwTMOp-^f%+&S!Xv7Uld=D%_OM!CvVCq&Y?+{< z>eFi5sJ^J#*C()SmLd1cOT#zm(9(&}b1y{~J}xwE0O`;LU@z@r1tzZwRQm`QnnaE>6^R~He26;-W{pUn zV5t>VrJ;DX`u9`&9{4GosIQBWiQD;DdR{WjM{ow-niDz^rScLkh@B(2nUS3X_q4LT z&N>4`sY@ZWl+XeYQ_yZdlU`);s&+-q=f#1mgi&K);=2~n%KCKCZHe z_c=_uOPelQP619bEs?^^w14>;KQol={{!82qdYQ{-czBZtlM~p8=Fbty#C$DG~MH$ z13jWYr=-Pa;C<+3Nlk;Mt~>het?B!>Jh5$UmvyeqqvDYS)1~8%=TJR+k6%kd$2zs) zzsSK?PL|yPmc{POodX3lmpjSZA!`pGY&51K9iKaWKZ#nhQBFht{JG5@pbDR7TK>AI z1wm$I5}kk99NeysC@_+*s&;G9AL@TAaYM%t%1rrIR2T%~eP+mAoYs0g-GSwR-Zl%6DUk%~z?xkOUnr*Lo@k{rKlUz_~n++oO1 zzPCP@VhZLS&IICa!KPppO)cq(S-D6C#&ejEVoC82e%}aw`CE>e>D%&f7O_(A_^TCp)$DvxYLX`U8CwiuktT@n0i8`b-_QDK)v{+=wTaukCP+KGP|=K-P0gT{ReFFIDVMFLcB-pPNu;zKvB`Jcx>` z!$)A<(SG6yVF|9nm0ErPq-_!O`KQ}q+6`4;B{lVg;4O2s)Ivs(hnjSF7`2{1YwyVQ zWPk8i2NS1}ioSDN)33M8ML>aYXgwizr4QjO87RKYE0*Aoz0MgB0Kle)EwR)|CFtP*ZJGG(?|>bIiOoVyL)IsMb`pWC3Sy8B4e|fj!(U` zeA?T@0srZ=tz?6$_5CJ*4?rZa?(LS(tmqmhgCd-r6v}>@lPMb#s$pmLhSoKddmcn0 zk1yTpEl6v*BILQQ3w2t|qc8*SQr^}HeANc^8`HT-ZvAHo)rB|LeqDx_1uY*W7j8Bt zP56p!q)GnRi3WTPMM7FdY#tHJ!$=K@{mL?In@$6y^(krUJS_~)g_Z413FY=2S-Zyt z3Ux3FWx1|@n;bvjb=cN`ab@7afPH00*nMm>4Vt{gm9WGE|FsmkmWB>H^}q2Nw%X3i zIMROFbb&8tl{*&=d7I(|DhyIMfoqR9?~c_+djE-=z$emv>jY4;rqgX+EOBid0DYGG zbhK--087>~u!-NHf}LU**H^H^fuP8r8N$wIh;+?zFYc<1=Y%fpivb^Ds?xH(`Mq8| ztry!bhNX-yr6bb!POSgc&YotB2lA>yc0LBWqGTmt2|@?pw3UXOCNL-U-#zZp3L z_tWJU-(E%f_o3urH^{MgQz)qY#j%xMzrh&ve#HUguTuNqYsVRk9n5l5G7j(6EoQ_# z@thU1#3J!V2J#*PC z_Ol4$i7HpQP2Te8Cedhm`~SpO_Ly(^{LD%q!QDX?t+hXYxGm1+S$xoOcN|vV&Jd>f zTR0E+4@T^*A7tH?Bml}1b(VMn`m43}T^;mDcN_RXjD?8sG4>$62}*|-yT7;55x0!k z^3_Jdw%)T3+{ph~t#?~-(3dZd-n=aL(FuCFV+R{yc*g9ZaKTwQa+1y|t=LHZSD8QR z-uZK1UH+I4jY9btmB}eKUB(xQ$-{?z?0FHXeIjK8Z8uY2p?vA#FT7J_|2|xw4N+B+ zwEkB{1@-{4lliijQ>=RG8WB5{E*!N!wRznm<@q>C#UCs^*~_P*?B0HLIfTQ%e(1&e zNMdLnvXuWtfij@#vNo0Ke5_bZyABshFrH0A@WjiLqO|Ch{9^rlp$l&LEZWxPBi4VR z252#(l8K`rztzwkTfb8Wx12qC2_3Zf^!LcNU!Cz{F$Qv)KoOIIgG1@Mx9Pyl74EvD zlx2js{dD>=zpDAo28JHg${~UI+pfEt{r9uz7Mlgk3-7-UDs6dycMa6tbrVBn7iC>BCZ|us0;`#M1N-&n|=kuavda!?=AD=V14Aq%+14Ia<0D`3V zIbH#;%?sU6rNcj)_1CR9i77ldz0>){Ic~5~Szwz>P3gA7lK$k=(8=9Q z_Nlc`CQv<43+^28z-q5YrUsU%F_RTdv5O{hQk-r~G!cD$iAUqAf*>BBh~rutf(5HODH;Ya^g>m%G)><<{*CB zM?w|*xA8ihn>{70aeYy3R%^`AC)2s`KFVVv5lIZA0zO*^#MqP%^}AMu0E| z*e=P_i2#3og>OBWHknNejl$X&Szbrg+m-J;?euzdY_iP-tUIt#TOY4$Jt_i~%5FY2 zfdb8Ii|1}7Dt+9NX*U+d{5;s?W?qx*&OP7xtYWWPK4Mwy5o&^FF|-%k7_?__tphMd zf6n_C2%`KvxP=bbA){rSRD}3|enQdyb8DX}PE2U)JPHr!;tN;Yi$A(z7oW|#)xyIM zjQZFC+Ip8z7w`wQf$G{S+vz#07T8(6%zu4{?+1(Tefx}T=(cI2E4FAMDwVW0+cvfr zP6&f_jFGs68e28{ZhA%CO7hb1%p_%yE*46;K?nsiUjLDRLCJ4cxsOX(v=bswysya| zIA4Mqoelh5zCGcd^DZM~=c$=Oj%BFNuur2TU%)GVlRV$_idX4w_IjamNpR?`{+Gtu zTN(b3fZ|?(tF*9|j~mrb>YIZ49paC6>6apMNKG|KKLQ|wJOPK1^-S*h9CnQvQIV65 z`!jXjUc)LRP9h+6E%IaNGl$`UaG~hjh<;AhUS0L<5sHmW6xBFSSJoaQ$Xw-ic-{_0 z6%3*hq7MTF=DD@irt7eB+4A!X_MQ7H=uj1DXA3GR`Z#QtD*!`(Vy|6CC)JaydGRVM z%ySpG*HYFzN^jv#oj^SQ)F1Yo%rO}^g9=n8*$hetvt`BH;RO*18JW#RX^nv>v9i^v z-cW9apLsIhTu836BqC`*Bv=LPLyW(hzSrASm+p0M7I==?n_S}|+^y%`zu!trS+KHM z(Li$GJg10vgr6*Y*%DPdzxb~1O3W&#p`q@f1?U^hsirXN;9^y%qaZhlWDgjT#8fIZ z`Z=O07lwwUU1vwA$64-8gjJkt&L058}wC$!kiV zgq`1ak4qJ8Axu>A7brN3$E7?lV~q~YRJBF79$Mk7KP3+QJP7^neVQIdxmky$V+@7= zCPq&O2Q+$(;XevTTyP zyxliGB9rPPKN2d;MN-a(C;UFR(-kJ=@itvb$#oqYc53vXoOihs6*aR8n!%cdhbK5k zU<>0GBFRL+3O;c#r~aV+jbdOj=^2;yvU%>n?wg=eIb}q3nz99}uo3e1UHNkLMYbr+ z?SIyq9T7rIHoMW1UWYosd$q&qvhLx|!M$GGVWwCTVq3j-X6};5`fFU$Et6hP;7nkg zXLK{A7+HENH(}?toFME=;CCh-vDJhfs1+i%?M(&Bp|!B8rq6N$#k)0&zZo1>EoqJx zW`%NUYyYXC^n?^CElN>^s+jc_aw6<7l4Dk)3)We(Ya^T=fw?U=z5RJ)74EB@1`cXf z37S<@(xyDWH?wxq>lRF1$xm3VVkavPbOi%lSh#fz-(5ap|8nputzOqKpL9;er(;t7 zgxya4gSS3U5L*OFaxgBV@A?B3B04#oZ}jd~im*g|I>=9ZLmN`Vu*<|*+gwYZtk_W- zVPb;P_Nft!6>2%BQ3$j4FSd?bwf{a*s6Ec1)Je2Qu{3KYdt;&~>!liiZK=ib?KGO_ zY9MrL_*=4^>^HU=c5cBQgV^nj{zEkZZ|?bpIZp_#<7o8R6q!N+57aEA)kPB24WroP zmR}3(Z95y;;o{g)P4x(4*f{>5I2$j7-ihV=f$Ate?R%ItA`~oX*I4#$|DWHLP)2^S zdjtj7FUH9mAk^8R%e#{DEtAD%=Lz)i`gQi1Q19}4fHk32Di(sE7TlaI4-)e6lRE)Q zP($2&uS+)7U^#48f4IJ*oKbVR^8#3mmV06yRuY>L&~^64RihLRx5B zA**HQ*ESCaT_hqbZZ&#NP&Lf=N3)y*wqh_>fTn>4mH5yhD2BH6vTbua zdSvyDR!i*xJ5~9;acJqDG7lk&uoq*=Sk)LKl@Womn;BH14GyK}*!ToS`9^wKb`#+crkrQCe0DJ5CKX{u$B%ON%Hthoq`W|O{ZSaj_QBN|)|^m@76mY81HId$ zpe5XI%F>?9UrcAigSG3VAugZI;kyU5A?@}oUA+6yg5{Hjgm2o4rv?M3syz8#rJ}P~ zp>gJ(nMXqHpOMNi=c6ox$?c-fQFBoD$mg|j*kr0XXt>#h(dGXg80W6aBAy&XlvqJ0 zFcEs%>YDx4rKPIxM$uLpxMvbB3LF!SdDSWTS&+Uaq_#tbRlmqcDKaU&vFm$QD~8kv zQH}AoPK3Q^42>_f8ptYEYPEg#N2eGWczd$S-ONe~2Cb+2D?E^ytZuPW!e%x{;zr!C z$&EZqaw+Y9AaC|~4xX0i!v3qe$&{g*&fU%By7ZQ;9{4T>T50-oCHt=p z1*x|bIh-D~)8;fZ%R1T87%Q%lk87j}uV0LB!adwJ8((-7ONl76`-^+|iqXA$&063F zn~{UR*cn9TO~|W|qMGWT$r{o7>j^71ozYRxq4Cr7-<=_Gsd}Qd>JxfUaH3)`YS{6d z2lUyok0foNuC;#o+2!$vXzNGr4wV*+D0ZslNnlfrS7V7-7r2|rwy5*2mEnEA<={t# z0qq>iB1m(t2Jx>X&2!=#*q35&4ve&bSp8LmfW!QYM9z@}z5a9$JVt9rTa*rExy%4uc_AB+_ITrRDv3|AfTNr^6f_om<@*UO&rQb>N)Yk zhJ(qDSGM)Cd04FkxiM4rZKE?PF-hTWsEG@`9YN3Y(&^%1WkoouIg?M3we3DOUxS0! zO|9;MU%dORo(FPmPtErc&}qq~c4rz)a@0&{RK{X)qh0&5@djZJ8oEN84I=w^YMLak zB*aIwTO_fb@G-EO%tAw1IA)3;2nvsyAxR+Mw z(Q4i(RVjp2(b_iANo3J9{O}7D|48X8&as*}k=S4pjQ(J|0lq2Bev#j*#hS};zMQIT zPhg(dYY88E14u9D)Jp7BKzDM$uh!fd2ycmYGUB1U$VyG#e>+L(a7mdbLMhr{-o|G3|NMt<(dPJ&Qr@KcRWuvs4Dgfi!axVU z*3GNMf0x`D3ACcb&b>d*s#W(z)%FO5UfF<#$gn^@i z1tTIr^Q5?!<)kVmgB%d?Za^_qGj5;I=F-S8@zBoGO{#B&><%AO(BbDruCjTmST7OW zAFqbJvE?sQ2Yn&ew7?dVK37#EL*nE!;A$5WQW$Tv>t3@Ob_`PM7N*<%9nZ^t*>z$V zFN(ZTS@(#0!gdXPRu-U%wq4iG2-KU2yBsREGINmeGSoY(YE@%Z+^>JhU6XdbDg`!l zvSw)OcUY%EdPD5y8^nwFH2YAb(c}aazF5u}q*rlDC@oZ3YFk6$SD7yHi7kt5Yfrf| ze-JT=YqqcQ73|7-vo?NTn7C?n!{816zT*fJabfNirXx$NIUSq1iN`YQVHFe(mI#C#*MFGFVfSpn=3*(F4#0e{?|_LkU1flgQ7=+!VyAk_oB?B zGT(a-n4eI6h0<{Fxxs_b5b@EeIuTeYT|2})+$lz-a}Kgc+Yx_6)E-lqZYHw{8Ukb0 zO`O>MiN*Esp=In~$Y7U_m8F}#poFiwpP6@iZBOjrMHVqRGF_@hP5qSW=WjoHy(=9E z=%Cl-b*r%}V!{;(=1;!$d0R5aQ#hERy|qnIFaf=84qQy!7FAEYG1<**IP_0W&}+ox zW!S&EwpxWKRC8dOxgdyxSbav5pDlm?6F!4 z9ml_EL!M&blrDruGCrx5x0xFs*?$Cj?hGuEqOmw8Rp}tlT@BF%{Js)#8Y!g%OEa7i0Xd$Wzv(WK&kO;MJJy~Nb zL;P=c&U4&EV=N*ye74n|v1T_Fq6v|*y;BOkBencJu=4rz*A+3T547mRRz13Hv{?a= zn3beQG0+7PCE+|6bH>egc`M!Ao`z>Z-l7*IFbOqd`GvzmBCYeaLw$(UIM=+ zNLw~7@aXG5#i^y2i6e?tK%&gxuDBYMu!uch&<8fl6rx>~3EqkDG?(`7sqLGw*H)8W z+Rb3CIA3+bM{d_#2FO9z6YwoK!7yiHesb_%L`hE_l#}D1$H!J_x+;(frH!gpSGhf50+d*C zf8BZ7yKgW%uE`uImMhB%k>V`rK26I!lJPKn{M{=Xc-1LNJnI1QwF8dW`NJ#T{ zPTLa$9c|Y2=L|Sq9!lGK+Y3M}cWQprbAVaLK3HknPu;Z*vo^HSkaQg-+5V;Uu+<%< zw_q1nLQ!nU`S!~2XH;tC&-XiRBd)*pls~MS?(22LN+!fgn;33`3d26HD-Ex-jifdT z7)WNVwa~3tZ*Qq^2*3RsGVyDCbg$WT(tMHJGQIX$m;RW%`SPiO&IZZ!BADSSAqkAq zymxe|gIz{z8(I7h_iCylO5phZA7Txfd~<8{I>Y4a-X2ZPZbbzj^$&EhV2`TgZ@W?NXbuzuX1<-!E)p`@~tja&? zCUBRj)NfwsHM0V9xj4?Rnd&P&nxDo{XNsbG>tM4m{h-}CqcFXt+=77asZDu8^?{yyP@9p9LI|$Vc3)5Z+`~@0(^Psh;J|C zdb}$)j31)$9s~?$s(4x)D(-Ks%X1m`a6k7CTW2vUKT^WJUsE>x1Ji_YF-+4dzsc)M zvbFajm;f%ZZYeaQyPq8uD9R~=AXJxMqbLT3lOu``C0iCJn~qY2tH6-Nlqzk3Z@`Rn zq@cteReV5Y7z0&-Yob=`%GQdrhar@YfCy?libO=s_pDu@*2Pq3(BXTwVOMZ%glhD* z6w)=5htRDvZaP}(c4CXKOA1TMK5f^{eTfLB-amN@strJ=fAXzXI(L&({VX#)57abx3>ss!b7V_ z&Y=|NZk&3M4vk*O4sa{{c3KH%JH)XR#01==JW5|;g_S8P#4r@;FK9HNWyLkAoKS1fsM!T8K1)rjVS$G5m$j{^rw$hUq_5d^+m)(pjQu z)X$9TCkONyhli0`E-Q{HR`6ZMM(udJrgT7fY}18e*eh&yZ@G)7L;;nje%U;G^e$WE zf^5)S6sXD9K>Qw^l=#8d7wZ)n%Dx|>767zj-CbnJ)8TtitKi#qgVmgILl9PRc1a2h z0~_9Y#wV8W#G+wEFdF}pN1d%dwJ66utmijbdt={VUGRj}h}v^X{;In4WC4lE-NuHg zCV1i`7H^44K-*sDIx%6>y-yM>Lv$uZt(SEYe;PzOb|)>2js#0)1}dA{F)7zK+k&7RrGze(McY&jr&e1w#PExXTw*pamD}Yby=&EQ z=0O2u;gzS&CD3`gtHkpU4~|!rC_4WOz+b`;qGF3f-_{Q?-L}IZ-u9CT|M!Y_`u~pK zXaBd$-!EwWAAYsna%xJm?y~>F?DE{XPxoyG0ab?3%&Muc`UxWdvGX&pnSSqCtuM(& zf?{8!>7;w#L7NIsra3nAlz?xy_0i??pa*!L|KO>|62XIt=i`L_gRKjF zP;GfvbrYGJ5nC+(8nt88&`^YGdkeIMi*L8OLT%4dG%`njD8W~7v7fi_8leM!Ntuav zpHmTzi!rw%zKcVi=>NUBT|8D7C7^9_=KOsCVXtwnZ39*NYunAff<+F`W2ek&e0n52 z>iEu&mj7h-*Zd{s@fRgnrD=Wwu2a?}}SSZ{?yZ7zn>RPOxpuFDBH6s-Emz#t!e0w#s0`)<{-4o6kg2cA|_?wj~htzBg-k$(A?xG*4o$!vPEq z^2S#StUz{4FIW$yI$7-{aG8w<+s*EUu)khzODwqTv>LPp->{%-yxn<<2R=EEWUbN1 zaE=}Lc9h`NMGDDQjBX$q187OlEvk21FIKN1iPD>`V}YcuES$TZ>|jlwndCAC;2 z=%mHpV!kdH`T;XKz4S2wfz4|%K6C@GjT9SYT&r-LEXu)k_Ypi(xSoWsvKydR;LF6b zE%glJ?`R|K005;y{TjTrR-c%@lLb-i2(imsU*j%d&rGPI>F@KT>;EYCf=vD`g0A;@>H38$M-!X_4{6RaCxT49ywOW zr31CC5C4bpOn&|+YOk9uz)`98OxyIW89 zh(Zs$9^C>hwv4~owak_D52Ka=?JR0)s8Z2U!>H&N{^H@Z5J4gDmQU<>|N;H>787+ z<-&fhN-Uplup!xGJ2(4wYE}rip$BueF6Cz#Ikc+y7b2;i2BPU!Ia9Uc>4CCUC#s?( zspKJc)Kh80AFJw`r!1-0gdf=kRB?I?9DUQCRGd?PvM?^JitW7>rK0>8n$M6efso14 zMZkn|sz`vsV7PuVSnwNn<;~wBix!RzqiRK_Tkk{@RV={DiTb4ggmnB>SUO5(MI!>6 z>fbf#a#_#fSnr9R6PXlJbQ7s10N0WsR!R^*k0z=Q+2rh5CgOLe!m_Rhpma+BZ9o`I-}hfXuea$cZwOzj&ddJ%vtsGT;sb8aWAer<@}W}G8u zBvK6BYu>H0zw69Svu*O+xV7|PBjn1_YeqY~CTXVQ?quq(pZm0inI`u8GY^!z?v&eq zkgLcZUH|_rZ+^5aTXv0h4LRC?gfUDbwg46pyS~U3+EviL(XsQRra&m`l@nWe9WmQ- zIAFPPaQ&|z_XcZy1ERZgw1>&(>cNSrjsDeqTYcZ-{glU2CYRpoy%X9yF(2q7u`~Vo zw7z;48>zr`917$06I%G`tDE+p#%hPgL;F2b5~n{V{oN!b9qiclIE9Q zVlL@88|^Vp8Y&+I8h%E;2pP!)##*^LER48~hUr@UmY#WE`e4&}Z{e;a*M%uFRN?rg z>%sb=3;&9r!8-}OSb1SAS+dE=Hw6CiQM0qIC5tobD*nY4*K_d1+Q(4m98Ku{-8hZM zN7J1mHqcNv!qI_8Wa-+8;~nuAtTr0=Tfh@qkXKuv*yJxxhfeP6dpF#J*QS;Y;@9Q1 zS&v%@}+HFqMgP7m)Nollwl@L&qF)nvO+aQ^jt zfL|joR&Bs^S$8L0t13mNn|CpbI|?~qia&E+(mhk9_DBj5IpxE0c$yLkC}z(H-rabj znq}}ld7v5mP(|A}tC*!1)K3KM(lT5=Bc&wN4r`WEb3?iuvny@u$(G06=3?SbWVM(ozSK-`2MB-T zZ4S`qJ?*+UjA0xxTBvt%-gh}v2IHj3jI8@o&WY5|JG8(pOyu}KlZS0)Ws`pXKcdb( zoasOO{|>PvR;g4DQ_7)}(>pO2#=YPO%~*E;^rMdbgCtn{<&M}m(s-#v>?rW$?J>XA z*8x&QQw?aP=hcxRX|iN6=J_~L+z@+mM_E3VC8Hi^*zXl91am6wFa=~#0$DABP!&;y z>Km{Ngj(*U$fd8P57SZ#&j(7YawaM_Fc*L$Om~L`R{fv06Jj(dE7hyVVb$$cXrLtx z7`P5B=>Cplyy4s4b zF0sCX*4N6oKcNgOSQA3#>-sBhj%-T~dUngdM1vbS6`f610Wxcf9d=bEmZ(P`w?)B$ z9a=IhHcVx@kQt3R>mPg>3h^L~q%?W>{ zG3y<{-q4d>FB2@%3K_!v(Op!nwwg81QL35Qw4&6iVPla(Ef|q?;tjy3RgdSp4%^A$ z{lD5XAT8&{2AJMWt_M{0!FkwY4|H(JKQmU6isxDtC`yf`lHSEj_?z-aIBk+^6~ zoq8DakU#MdA|v>w@`_mGM*r(Sp{hw7-hj=4cI#g$oT%E6Pg%7Gt- zrn4fEE2VU0azWgodIniDHZrZ;EWQ#no`_OROy)xp@)m`4+A~xvDbgWhgBl}oux@_8 zYV2|w7wUZ$nuQ$uC`NHwvNn|(mrUl`TZ&RQgf^Fh)T7>LM=&Zu(c7<3tHaAzg7sVd zQy_0~9g};PEGJfafsaGFD)lv2-EQM)Vw1AMtEaGbaKco!-l1Tj?dI>k$TbYya_vBW ze`t}Bbiv=L?RF=TSv(0!O56K1bpDQXSJZ0K7I5gHMAw$m6U&cufA%rT)b7^=t?<54 zQ)$Npcpt)-GboMh_Q?i!?>+HXXu9K;;Oh_^ra~HD4u6TgEsa~g4=DaN)A@{52F=bm z7oTg>$?}p(`z^rQW1Id}H%w-=sow8ZLRrIUS#N}XWwENN z1b)C8_P-`mWomm1Rq~HE_o@YaOY4-`D%77l?;3k4T*OJLENBdUGIdz69oHxt?gp@- zrmD|p4H~y#Zc#t#s%u#(zD3DPRxK-&HBCWXFH-dqf`rEKt)e5J4YVHYn}l9iA&n_52zF#oJ+0g%yw{vBo*W(96G-uf(glWYli8A@C`12a;m zzUp4Xb*XG(TtWAc2GWh6XM-sn8!bXBBxCQAP?{r95X@-a{yBC^r*Zr4!Ng+v+-=Q# z=w+^Xo3E}glpJh2ZQVHc;fe#uj(e>R2WC%ggZ7ICA0O8vlGBzZLQnvre|FPq4`C?S+=MyWVRF??f zUv;@6mG6DjlG9KV)^&*V=D&~#=ybLDhr(>_Ebum*kiR~j*qDc`Q#bDieBUU{%2LvZ ztV7)Ftgt<@gb`lym{^aLf_=!6Qjg@QPQ?n@c=@I^EP7!Ok%r`e zl@KS+Ln~) zGP=n|$Bw`z-fi_>(m1kv4|2FAuPvgMG?Sb3p8N#>H4Mp_mg8h-9es~rr&h^yULm- zi$nBoA1y}jE<+5jVms}gqCrQL?@+>kNw>@LD?56Rq7;?&Px7SM6r3B`de#MypZISh z3g9Sx;4?aN{?)*SQqWHy=>^$TdI-I7B3xGmz$i|wucq4RSb29xE`pBb!Czh-6B;q%UjV4bze>#Q5~VuWvj*j-X4Pa^T-QFE$t1{xCjAMJ$GRmh^ST z(6jk)t4IMSKW05w5B3Z&((-gHk6^cmhK6S%##=+=bzP-T;SN)^dEkAqLflNsNoJv_ zdQHy`J+q+K{SXore!t)AxM8OXf7IJQus@j&UySCv1eJ z2rRjp2*FHrMgUa_+Q|4nnukbm&o|oL1)>${YES}Z=oGikfqu(G8U$NFE_Cwr6A&5N zLxmzly%ozTM8V1VKkjV|g39I+UL}_ZxjCm@<7lF)lEAedliJj&$2Np$nixehiAwWy zMhsTVG^s+N%6$+};idyyQe*Rtcb(sKBkbmT#k)l_e%VU6fy_G#w6ck=9?AuFd1Gqe zG*~z7ML`^i%bfGh94%~OwDJ$3A*$$v$yZe2+cZ~!vJ9F&%lLV|SldbyF!&2m_ZY8n zf0Nm2S- z?o9bW!08>;3w2A7+)~owTv{7^z&$T?>y^j^YBuQmV`ik<_NzwPeV(sPp^9H3W0X1?E7rdE+@U^QUk{=-KM3_XH_lVll>MP}G5V)#-^1yHz8b;)RKBzE*Q$W{ zM(@mvls7AU!K-mHdxst*VRWr-jotUlzNtiBpofTcABze8BuUNpF{0g@^-oj+fS>G! z{-9g{xw;T0md0$Pr2Tff#7+?YWf>kN%Wr{5o=Jo25 zYB0)G|1ecWCCxrewK1wInVC`pk)x=n?0)%->F5(|DMXI0vFMqA-Dk^)e9?f+;IYEe zjYmgFjQuxmK*&zJobnyN04B}so1P*o23PYm@Ez;@$7R5G(z)6CFQ82${FD+{;i;yG z$*+V{O3DtKGneVcP#Ovj~UO--*3g{+kw%gX~s| zstXllafznYBs(Y3SRvDXT$Axj{W>S9-;!5WB&S`JW;1z)^}(tx9$K$UmURmayCd*w z{@;&cy2Sr;d9XjOzQXI=3z_@ai1}8Yu<`u z`&`!D$q1JuzFv#uXCCP9aaXonKiE;JO&ZMeY%;cTnfXN8__cZq~X_cmUrKPAS40yV00O= zeFk}z2nZM;z`ZZPc>M<4B>rjC6W}M}kk$uV*`*;f^7e{fQ1KO3lZc+W)f-R!ehzeG z4k?cVFY=kz9;eY+Zn9imcjK_vMcmaFN zqAhBp-u>H=E1DJTBw2DodBd%9cpxG+~2G-ndYgxXn?a;>d=5# zCHb@K%+$Q9RiynWJ4&tAs-^87q>;K-a9vw!IUy|!le8^~AE|^K;XdDXXWNF(XVg@AgZ?#zR{}4Io1ojr|h{pGv&p#h7*PTdt12s*)&75*wN6IW}R2c?cLTu0>0&|TKBLJn8BQ6hqdgZ-Vtx~XrA?=q}P$hZeI`NM1ls?y!uX+$4_uA;HCt0D9XEG*a9fiRu?PD z>aWYWcs-4tCqmVt%7n+COr^DEHDxv4+hgYSr1{1TQb$_whQF=f_&A%mC2;KV^i0Ff zioneNvS!6}F~TQqAWAGB^H)B5KuaKVq(TejfdFg}Ftb7B%X^yqK7EKI7 ze7P(3-15Tgf65E0>Gb}SX+PEwx6|ToI?ZZ=9GETgE~T34Z)AneF}%35d{)h3Pc1hO zKPNI@a8u63wtM4dqLN@(fVz~&i;%*$%S% z9sg9?U?E$3PlmCA&E>pOmM^FvnJapyhb9Ag z#~Ca7a+`IToeQ$Xjdc3=w2UV&T2D(%*n_1G7`e?E(5=|Tzd^iQ7*#pUPEjL?`DSwz zH-VSXyU~s^4?i&=K`PO1i%DcXya?>vEUyCHp}Ra@FBk+0Ar40dH4Ak-wan`Qw99$L z({pV-zi#*b`8wmGs)0ewRCzh(HPW{DrK?XRWTcbxpg#f~iYljHTG_8hr9!@i&4={0 z#bsXAR5~X0U{?E0M9sSOhL1JoO2~BZ>xHc1uby?Steie9;1h5k>fVR-=}Sv`&Aybf z#o`svhBwtlafdvfyV$$j3PN|gtn5~HI<@&x_}VL~U}qd8RuL&(>GSYEE090sO>~#N z61&BGG+ej*z(J=f&- zMn)btV5GzREJ}vn5JjC5tl$WwT~bm_5b|@qo&?2bM}G$L2eHzc8StV|fYZBR4X-gZ zmj?vn`FG;>xu!SwQYbRH5RX209cAfoGxiJj(kl#VqoLD4x(XB`R|z8ms&6aCy2TF$ z_CGubBQc_B)hbY))Uooy-ciBW+cQ0Mdxf=;0wwz8?>YfaTZ$gWI$cJ7YT)P7qD*Hj z*OX&^TJL#m>;T8jt5m#9I9|+Y5pmU$s7J9QT|n|DxHf>xMc|F?H;AV3!lNmLlkgxG zZ+BsQPA{Fvv6rX_bl@9X;cuv81d1oaQg$}@Wh#2Y5fKcPsW}pydIwAmcRypLM-I^Wlw{Z zZ^@IU;%xO<-L<-|Ymc+hKU7+wW#=l6dq>P6haRwRgj!V4otco13)wut=?hC`|Qvbo_ zApGx=iGKUEki(33dc!a_1P1sK&?^YC>_XXI0AK@{|&Je^l~PCi>QuTGWVt>HO#7Sfm0Ia(0U zy|TgkRb2ap8^F+r-@#9s3;*c47Ga0}p;$Z^t!x^)(g3lq-yM8~)S{lx`VF09PSlHZ zoh!x+PU#9tyGlk#_Hj+?4ctN2i>Ii zoto9I@#Bz@7S3I(?0?g(KDFFN#FF;5lh0mpGPw|Y@vS}F+!;G84kO)|Ao`EhX{3jq z4|{x}Sl1vMB;Jw=aqwG<$y^f}GF+A2c3=xEP1$y8?oA;^Tl7Om=bX9ZQ1;0|{d({j z9D||#jw74UBRF$o{!=Bk0UbxnuTgQnU8?5bu32d5(sv1&othn)auejne~=xy2y_f? z9xl*VP8eA8xvn$%Fbm5!TGo3-@Iy1e%=LKi@tat)g)~BR+-Jkj{BnTnysUdF`kH^) z0+h_?A%e>wxfvz24-w22CZBh zS1TOZXzXVmsytg&KDhzmQ*A%|o)@A8SrETxsYuX+mjA3Z3T z1RjYf%SCv_6KV3xUs}b#zPw3j#{IZ^gjEwhm`!vq-D|{rw=mLbGCycCCv8NBV+_s&>T`>==5TOE_*GLGlMg(G|T~0_o$PC)&QUkj&@WFhojiePH&K_Y}jz&v%2AhB+XCqn$>03Ymv_+hq+$Y@3;9W45TDfeb zm*m@W;(Mw#h8OuzDoOq4AX=42hvF|D9G?UoOX9~ELw`GbC5}`g+W8f`f(;ik3{ zzWXdjimX)tJnIJXFUz7NsIMphhj?)?sdT%8BB(w$I`kmt=f%oj9djeSasCfT?eX=P z^t+CsEba$$Nu~3pAikALZasnRLj(F-?WI{zVq{%Ibprna&9CRKwO_x}RpFSm7oLL}c=S}F z1qE{gLA&5`Pyiw`sKs4eJy|hhNc=zXYB8(6ODkQga~gzTqs?x$Xj)r>Bu4ok9WC52Fw9@wPuzYH+VvQT zR(Mfp;~T<5u6<5vsCSTRdqJ@Mgfw`Rf2hgfRpzN~1jK4(L|(92xl^3FZ>-7=3rw=! zcXr$EZ2>l!KFm4^3Yh>&Pt2ILi$tP{XJfIkBB^ zKtEMWsI|I*pIIQ;4xZ1PYtuy?|3mcNXBo~6@gw?}B$Y%yeu0R-uaAZ0|~awpQh;I8FJi`95wpiEdgY|_y z_C`!m_zwu=ST!}^+~l6Ttci_96+XZ^{Vh8sY{!if1g%B77p`HgAyX3ToUm=Vv_iaq z(@GuZP2Ha@KyR|QZhr+ddV}oH0)4R9_eu&O?3KeV<3q;jZ|WXmV_WUGdoIRv?!Qn5-4{Q#$>)ej zj6J$@4s`GA;YyH`K5OiAZ2ApjefO8E*9w88f1kA;tojXQaO3JvpZ0Bpd1QdOka6w8 zY8?mve}DxEwMrg6T0frs0aqi7Tn{x+y4i@b#VgvFC(;z_Zt~Vh1qeT{qVM zRQl68i~DZ;BYJjKz?xj`=bs5%Wb0iBeG~ynp`6KJJKvhKsqFRl$bhum%qBt6kGS+B zyUy{@PZ+}i=7+E|NzfA3p`T)`X7!OXTEH6|5Kf;9)L8^aZ7L~>|gCyk%42ZMH!1D# z;YOaVPnN)iyV&I4P$$=^LjhYEU@qVt(R0J&X!ihmFPL)VrDdJaSh+yrr=5T34WW6< zr%U`c5KDo8Mo6LkO#Mg76Bqn&+}R5U3MNuZZp47Z6@$@l>Vv~Hv42Cv`ExFi2DWY@ z^V@c3{GOjc*>ZW=tbjvw9mnY8UJZQfiqSyXc5z?A;iktGo5Wsi+=WY73k=}l#~`%N zS$#Gh1g9WRP)`2swDak;I9)4xt5jsX*Pr4EpjE18is$H>=-g+N6gD2J1FKyp3>2A7 z28?q92I$IeUh;YMoX!Pr>2D&5TPY96Qk%n=k`iR6v2<>{eNs~rbkYr2(7OnHc?H(D z|G#?mAS;gz8et#JjnlE}{QWv$fmEfP%0Vkmgp&PTD?6W#!gE9jcWi;>%2qiIYg+6s z?fO`>HhbpR{?DzPEhTW%^6SvW#h7}I!p__vk)2((e{udG^tTz3u(-`xJ(y)6*|+OD zO|cum{oKJ|>}u*(LMci$aBXioG>FoG4VJn#FL(2Zx^81d&dKib_v^g^{?9`_!6!WL zyy3c8-90=!`RhFt&inhFc)j#++a6U@^q`enBM5mZG}~$KfBEd$PsDoVs0xPMOWKlh zF=5H1&bRNpm1tnNd-i%!j%#5rtx8~rIy&)9C^y;cv!plR^WczV#h0DZ0kVqvhwNrKh+Je#e zj;wvF$qQp}C6%Ev4&P8v-M3a;tU1E9t~Lq}p<-Uf8|s@s`(0X9H(Z+(u$Vg1&Da&+ zSciMArhQRAx!wxc+COrmbf8yjMaw$VAYO|ZmF@j=T`n7*R4d?Hl9i$%&USn>8aeSF z&wL^z@_Ffg^2gX#pfYQAX^E;WmnO!lwHW|i%=tA}xYqGOoCy#Va0(1O_X4;}B2tck zdEPuU%z{)>49|I4)B1i%7W#K*2G95(3u?ZEe{@re(4mWUL(^UC^~%nNS@)xT5V4s= zWuLN;=KL-t^X;-9C;jh!NAmxerwdw68$9!!Ef}b~>dSv$Q*fX4%n2WsrbbAakEj{a z$oz2+F>}T@(@~lqX?A395a9p5SIdi;ORd>g|E9NfqBJg@|M2yW*Fs%pppiEXH5h(4 zIi%PB_9XEq{vqyLNlQHQjfx>%6XuA9f1s)>*osD;4m z3VB{3GiZ9Eg-nk&URNxyEWDbaWdNdRzTpJO6r6=%F45%U>!=s9?eP@igi!phe{SP& z1$NyEHcDKdxwWb>QMhGvB)_fgkK}8}2DRrk>R#LOm4FdIh5Q})?dJADyUv`I)T7La zwULA-R9YnFli7SQK@26fLnCFz7%LW(*qyY?B8x2`V7w5WbsS6 z3f@qL&Yd~fneop6enEiF@0?QSoV>P%JfwY0Kl_)RQ5dY2Q?qouU;< zdCWWeh9QB9ufOQwB6C^X5CExA9|=BMVs%fq1pe~qgi)Q9yWp^Te%W-L`kt}EC1K!W z+p^XRZ87HqvfTUc#x$Hci>4-w^yb42-)G^Wd^#n`)pw<|%fhtiHR^Ch7tY8mGBb&(iPor>=#Lp`c(+Wl&b*CC`Yrw4E9Q8kh_>r8Kr@iDYgOC!1BAE%=ZueZXHW%g>G zVX#sm<>wRNE~W2G{y-ICNfu{0W&TIJkOt8Z8(@Wyz5OrY>ZLXLO*S93L##aWteQQ0 z_ihqAd>@}F8zZt;4e+J$oqg}oog568Ss)gc35@C^F9WmMRrFarJO_;Xw+tU!ZW(4% z-E0@QKZ}a?qM5C^rfADp7&ir{j;q?65cme{6PaqrA=M9{H}1`GbZL@0^g1M z^xpxYV$YgKcl9>P5F{;E!$K$mIuw*R8gW0kL^w1nQ;VzzqrAj1l+Re=-s-{iSB}G} z&0RMVH5CN>aR%4l67Rs(2yQe&8q9bqD*6vsQXeYRtz#AeEqjQ8#$zXJd)dU72$5c_ zbhtr=FjjJV@5hC;S-SLD_olVp=Xpy=0xT~Bc2x=tAhCLz9vvj(Kyh)97V1hQ{AcN_(Tu z-A%hrdTv)L65?%Y0R5AeIJibDj6*#jeI?Wez!P6>QF%t?pk$` zC(eAQAI%vo4(JeFW4HW5XDh0!66eMOs`c1PO;ko4WYgas%UzHFZ*dcaY_m82Z-k+J zu<`c)BLfdyPl5E*;v$7SZ3>@(Cfpq#@ zjE4RYG>)9PHAwn}9mn4XYW?Y_bDOYR@h+s}e0qfrz>nB+00L?5va1tB4BHt@n zc_^JK9+P?c0tfKHKh)Z&=bhwb>x#t5L;1Shl<1B}-R`32`E9%0#cV(>7d>IZtU^61 ze&1#K(u~+g@nIg$n+me#yABPWX5L;@%Nfrec=up}8E!7xy^Cio_C0s?&YfRJGDB_V z+0E<#ZuYluRafSUz*c%>bKN_b$3GvxCoh%)2Y^{ZS&-TpLjjb^m*6`Uegxo$lS$B} zKu7+D2-H;nb@=3;46Ck(y*i&z`eSKu zZkW?~G|Q(hAPL?K1U;$z%nd3VOeEgudib@`bT#aH0I1kU*MHluUVJ%ye_Q0;hqc;) z#%@wu%mv>F^LRwWWPfzi^&Q7Zp!9eCBOqpnXnl0q&CrpSF>4iGO8t0opJsZ7?zFDH zs?u=pb8bMowBp7Jg7ah09r{yvUl~{@8b+7lS|Lx2j{?fG;APG51SC~pUHGNhsk(52 zQjoqv(DlSN9kQ^?b)8>NSnKyt3vD5ZW>kSR@!B@Ltbgf^5FIpyE5^dZX^up}cda44 zjgu}wJeT;FSm(m=^M0bC@8O^58Uln;K(z#S( z%(^G`ZT&ror4LyN??($8srM;o_uzW`Aq9Qa>7DT+f16{JI(cY?7qf@%2Pbmx7UbtQ zK{y%BY*I^UsOkwl*b$^DmSP6cx=#9sZNAmbg_5NRR71V^Mu$#S8yl9CJcn{+rt!zK z&Z@C@(At>uXf1wbzUkN@MC?8z{Ndbf&#}CV4r8s=KRt18k5&W0N4vg8mQC*F_8zJ@ zo~jKu8OyAvw5agD@ca23_{z&QGN_MUomydH&uSFSk5?VZ+VgQjhRRn9zn7)*fM^N6 z3AY~5#n)uk+S)6ADQtdOg7om{;a(hEj)Rd?^{xd3&TNxEVj(BrJX`fzMHhqiXwmGy zX~A+dCckWTRtC%eqz!`uVNQXW^|bo6`gpZ$D(rRnX!8MQ1+}6Y7X8MGW8I&mv<+)BB{LqI5#)|G;E_XgVu}iZvSxE5dQ%9n3 zc6{x(*M2gMzEm<)7FoB869H%G2+@L2 zK7`iZAC;9X5WW6~$dUk(*{tr*zvVh{Z-6%`gX5+TTM*yezmL-F6wOKYF+~r=d0B$l znQ7dSpVqAmE#goztCGMy?Z1^{%@vHYVE?d#N&VvFaJs+v-T4A!4kxQQ3)p^`?Y%TU zYA)}*kWO;M<~mOLAagSjd5~{?&Wv84yg1_NTp)}0pX-4|w7Lvg2U*Z9)SX#|<7LMS zv#J#XoEIvZHl}Kh&nrgCj7RDQKy;lgr6pNEq%UYB5uZ7fuZsUP2Ch_^rPP5efJ%z& z4TwU?4%AAmEk`j z{vAX-_x@V8jxZ8;TFGy6$M4iUavgX zPa__Krb>8wUg{V4nXx5N!zFADAdxr~dVfRD8x_L|E34h7L1M012&DDP(pZ{I=uRWv%jLq+|9>ub4@-X14fncteQ*Pu>RWLLws=OpDh^FNAXy`k9<3Uc`Nd)Sf zk_;IZdCe{J{#*K$t;2rSWz9on2U?!7>FiS@&HsrODdjqBB^`<1%j~LramxS0k6{}{ z1$d#=vOdb=-JUe*{D?;7f&8fj*Q@3EN~3qsf4$;!k#BzQewdO9Q0b3?nqHO^cRIv@ z10%5`bK2icL%t%Xgn5xquOrKWqd?kXD*G`AR@A0Ma92po4_T^GV!E;t>B8R*#D@Od zlAaz0!ytSsD{YMQT6|5>2UEglT^ja`Iet$;+5W3r+3an7Bj&S=oOiFUZPxMb@s|>I z?RfOq&_|z`Wl!4R54WlJJJOWM5)u{^;csQiq|bD%9^TG)~q!2Xl!is5rE7j z2GRQT{C{J_J{vLQ7Jw&r3KP14S)hY(kagXafCcgI`jrkj&2vw654NUJ$Z+m<#?-CG ztzh9Rp&b%9c{Ba3X5ddq4yS(bbqG-d-CRJ&-rFbwW^uYg*HcYHllS$Rwx*%`KyGo= z*PA^XH`9@Tq%r31>IcDPc}?J!{3G9e>HL%JGHp>GMN%_FHC!D%b+3kgd)N3i>2_F(cIBSiU76|gh921R1@^uud$za{Y7=z z0Q=wDzdOpJxS*SzCJM6e=UhF+GSE#EZm8wy`74L=R!fHF>8eir$K{kPC5fd_C&CPD ztG8Q4I=K)@gM)YCzUGjyt+9LPT<58mCR%=%cKh`q`^O}?r}Mi>Uo1ARM+w3Z7Ryfs z)Fx(_O7jo^NlS8&f9%!^9+N%CfTf5QlLuOl<$+&ncx#ikh&wVGgGhe6pjsOWu`Q7t zbygo)cj&BVGg3w1!eHeksL2UQ3+7Oa%w38dm~>RKnE#ShLB{^5K0?y!4(p@(%GP~k zpD;Cw8P=>^2m^E(jRm3(`vt{!^Z%ptEKdX$c=DWWj90Axo+qXK9+Dq>mee^ij5WfL zi*q`K(QXahG6-B6Q+!dg3%d`JsIE&$db?k*H68ETyy<%w)SVhgP-)Q`Rmz8! z#pjZ4(pZGg;t6S|$5Ur!;WhrHhNMIksHE}@bb3DUc#ckm4#s2nz0DNz@H7T|+D(JN z+by3Wn?-N&Aq0NYQ$aS)i?y64Wy7l9ny{`V5uo!MgMioZHmon|1409`C1Tb0Exe?& z2BXq^`|C1cLqfC!l^nYtS9-#bzm#;WUG?0S?tup7-(6)%N%%Vj_Z*^L$2Tj>?e?Gf zE(^20W4oo0p)B2^eqy2ckfom%PC_%W{bkmlqi-A06iNOS0vWLc&L5`%kDIX0_7m>4|R$t)?Tj{(O z@eUr}b&8aV2-=QykIOjx$mj?r6YyQaI(0Lr%Neoi61B#+jAazsKo}vbA-N{BH~37m3LwH!ViyA=r!RUf_z7F;Z<9-$*k(nM-L zF4818jOCF`)$JVoT*)$W=$G5xSu&qe-Cwse{EYNj8gRu4o~0CU+#v_yoi=wbR}^6v z;oUk)vC$``i!JpqSP_Zj#Zp6-u%BEHLzu-545FBHwmY?gmXtqI1dJJ;lfVxWC4Qm}vk9&)z$7~fKZ z6C`8l_2dP!hO-*odp7HpaQ?2x5|54h>#vY^H`J=_4_!jh;W3}2W1h`^zw-RWoI@CN zzMe28rc@7syQQkrh zj}H*TMq{6pvaeunOyM|j8myNnr*lSGc z{L%RjooAjtuqo03%P#1-Bn2RWPwif~N*Ko9=treoVDcfcm@&-Z&7M%?kQwqVqh~+9 z>y@xx$rW?tF+9{D!%pqQXXh*g?XPJ;CivmB$&<{uPAD9@CkIDMS-a1iqqNhtJDp~75^jSW=*4j`{_Ylg|TECVrY?D#o9IF zGJWRYwoOdW+bGuh^zs*WKUk6NS2`}La7AD@1;DyL?!jFcNhAVtz*`Hp7+wt2t|}YD zP%4{T-^vNSxwnv|%}CH5H;liu6)%O~b+i4>Z7l4wqR-a{kj?R%_8E42>*-)8&`L79 zXnz0fsm+n^l@1Nd2F$NV@5N+sagx>Xqe``<^b^$($D5y3TWdzC0!xL+&;ZPs#hQQp z%YulDlO0z|Yv+1fxl0^anGiQN z-3K_*5;zMgy)tB)`YKVm+Z0= z_W5$M46pr%B_0Fx8I2vrB~|Jco{5%Rmb?i7^OKCkJBM-kASTQARsAsV2tx7-K3qki%*pkM$Fktkqvt~c( zG?!@VX62R~P<*XYN#FLW4*t9WyXkKydQol~_^|JZ2>4r|1#+HWcwp;NC;tlPQ^Kjd zcHIIqZ;k6M)_c@RN%^sz-WTJ!8U6qZeS#sZ5IBFSzL%fTR{_8ahkOkgn@KHPd(qI9 zNy-12UrOrQS(ksL!x5f0F6qoAfJFJ;MFl1t@3~^6w;a$_`}fp@^K}zt+(MKMOxGkF z@j_0Z;`z@@r*>wndl)^ndC>tyW+r=G&goq~#uD?nu)!!!;t_vYYLMgE%F-(-=aX4O zw=b_ptM?U_$SR2~<%ciwQ3Hzny9(GF(nlY`rF5`Rh$;#m z>msokUj{q0xoYMS2|jST7*Nn`Z#R+JuK}|zD?1i!kzLY3m~X!h`p=j=Q~QT&Yp2%4`r!$QGNKNn2(UGqg%32$Mj7;iIiCoK8P*WeO!m; zZ7+Z4CfiLskfwX@jQCy#GP{$aTFO5R!#y6XYS9#wRPg3mu{PQ4Y6X+B{mX8Q9i zy7@1Vr1-t?poOzKb_Av=JsPT<5MF`a07x1$?pIt9OIcy*z!U+Bu9+_A9MaJ%f1 znM=>Yw-7Y~}HfYM~X)GKTh2)n9h*3f#sjThUU{F^&6*-f!LA z^~-Fu@9)&yM)@Zxscnd#->I>0o$9MG&H-4=r`4BUVq`2U?b&BT^%Qgnr`Aph)N5Zu zFk4EB9k%zr0E<_ie~q{CLTdd$3k^;SFw<#LBk&KaueTzbPOGy}j*9JIrf*qz)7={y zyY4}{gpi^+_IFB=!9nC)75R#i>jv#FTd4?Z@Ut6JL4$|IZnluTXBn>+DM}^lw83CW zDxhL(cRmvY7Zsv@%&dLfxsloPKE*%ka<_xhr>8(qws_ecrQ~;8LFPg9{ZrCLW4&8l>c7D^3GxbdYO0<%lY@GK zwsqR3qvv$2G)@gaO2zF>umghpI(e4CN~!3jBB2Y zB+?~pkPkw`NqhpVA-(YI@wuZe`eB{J_CLgN0j!KJ^==Dw(#<;LH_6bA;0s&F#kD$5 zY7r7VlDTVIi0K)A#zI4o@5EKvSrmw=c4PnhV{c^0k66-!I&hVE ztxr}-V{q886^BJuXpwGJvJkFxSgs3z6DT!lD0nX!p^GpmQFUQjC2sG_ICLM0ufkg& z?zTh$RxU()ErOV%cc~nn@8YKD#N;JY1vE?-lc1#`>59|Ob)fz=MLT!(X{Dr9j#?@2 z9UiJ0bOlv{aEGkROuJum((lHro142fahySRvKD|~?O_P*xaAT*=pZlDkIa=QyiHga z9@4Lme_G<@v4kp0VYiWX%MR=d*EpRN55O?{N z_RS6k7n?fxJx`m+`zD`He1h`CDY#a(-ih?}`tJ}KE*ml7S;eoAr%s{8Xd$Q0NOp9KY|789GiDHezC-6re5%+0VVcJaAXfzo9Ny_ap-ZdRZ7UoTszm z&VM0V7jp=sRv;-nZh4vSMId5CEn2NL6Wq|y_oDxYq;n5v^8f$1Ly}bM zjE}>VkjSEPwvh@s6%}%r%poyyZrGfXNlD#^|#U6{aPha(F-bf&YR3l1z$m5r!q4`dygLGqi)o-MgM=@XO z%>+N)t9)umYhOP8=yF|<%7xCc>Z`dUD&s&ppEE0hDBr^V2jU5m6=#v`O2*iL%#r#d zzJ6x{g%@&tJh&q-uGwnUK=tkfEcgB>8Ly$|Hs>4rutu@hkD-TcZ zhANThf2=b}JVHb*cz&<98S-&{q?*G!>fd((($^#F2kH-s;(vh&ZrA*%@~Qf;D{{5A zZwE65TS62D8m{@#(XNKxF(ELys!iyQYnGvg;zL+<$TGMF2x`NV(_L4<- zJ}N=&Ogoq_Zr6zMi;8g)UF+gU_H*ow&~$HHU-~UTGzI&*gikczlA4W2N4LtD#N;FO z^OB+_04W9EN%}(lnSicbMTVcSDX>+C@}xLEOl{>MtLiRpn{0{bF9B6ii>;tGk&nrs0=8tu z-Oy#*1#oOCH-#khxx#ls2T%f|XXTqsBUpAtVeBl~Y(2EgcDMdCDCJ-G^$H@JsPx?T z=$ni^;mbcG@?t$5bCfQivnP2W8=r1i)fUw5Z~JEh@ZnJf^te3aST#O9)HnZ+#cus3 z^K~*~S3ZxY5@@eNk*r7b?@fb-GV|I_&!%uyA8_BJn$Cwr!03D%`!G9u;o|%|R zWsR=>GgvD4BVvCOW+VX_hECsQ7RnFH$e>f1_Bnga*Q6T#!i045&tyIZu6BQS{@8yu zp4azx#G;Pkk3e|=;IpH(8k@;VPT?7VdRIF14|)w*b-L7#cK!^AnRv27vfK(`)Ywgh5?08TR&3(;O^S4}Xu-{`$?rro zkDsT4)QHkAs^xKX=EaTo2T~e4;(AE*gDE95;L#9e*7pSdrH%OadL;7l|kr zgDFvn{qLF-!9Nv!0>z%}PrR_E44J&&zgCD^rQ}HF`-Mx2`2Co@&rK#m)0iCCwI-?fpa0dvSZ~1ExZ+A z(ZFAy1ATJpV<^5+3tGXnZ$4EIqftsW4y;*XntVIBDbF(9W4m;8sm;x%`rzrpFpIf7 za4tPil0mKM00J zA>UKW2a5++$_&+_Lq)tZ5jvXQ`i~zjXh6=J0loHO*(x6Ao0zuT$_Yp0JqA60m?_!o z{iHXZX}cp+7RR`j3*p}`E4-eFDo8=^uTR_*mQ{7*s`%R#)h5R`Cf%;F-<-W)d|=qP zA}%N#r(g8O(7Sm|OwVRQ$Yc#2=n}LGu*7GWsaB4326Q$hgT?vT=ytX1HPDR$lTB#6 z;}aH6L(e9^H{ae!cx|#I4$WB{x8RwU*?U~!?{)-)N=_~@awk+n&8{a|15BJUBX5|{ zS5@CN@9Gtnk0@nO5b`I;|HZq>7MrZPaF2cozPjdoio;KligpgVQoY(_puq(U{u`0{ z(>_uFy!WMKAGI2+0aOW6aG0+7btCY|W(h^#W)Cj7@c+pg(Q!Xe9xvA_-*z6XZk8g?b<8_(LW8b?7O6mf%MzKN8~o<($>Zoh$meF@~>kYdgz*`f!s5z<~$%c z8|NLW2k(>9!Q_H*2Ke9TxOpoh4~!hI6cqmYD;mg{+3arHRi1kC(*2R2ozC()_qwP1 z2d9=S!HwhBw{CJ5w|hm2ZMpD{T$v>&e6$hzt#bb~K>V-i?8xQz_rwAvP6Qu$__Q5W zAhQ(ajvbSKd}d~-2B9$*;Fei)e$L%=HY33M#H-sYr=qxPLe%OXj2`I0nSXH(Ta=Sgk{^2L4hBEn`B(n+;cU8Zf33_u+sOjxn ztMS#lY!l$eu8XCGX0!D-cqa7QjoewA*c-3r3LltaeZ;hn%%V2$vFt5UuLPEy+AC?c z0+wqi&oJ!*6Zej4V=i6rcpDGC%;;&rkMz<0G@OHGDy+^ZytXL1+c)!GK4fa)x>cc|=Q2UXCwsIu&T$}hL70?(BFOU- zuh5~3Uk}B7;SC9gHJ?<7*J8akuMUoFzG26x$Hz|$hA|VVvrYN~v6XOf&s@(Z#zMZ~ z=**`*^^fOfM&i7%=f!k}Ji#YcUS3!pIk@1VXUUo!wTGPj7IqYoYM%*)Tz_$lxUV`_ zWm8gA39nQ`W~&^Q1s&lvwRk}P`rmYf7^{dg3EZ{Dz==e#p*lW;&67Ke*_o0LA+YuAyHSHf%ATce zyyQ~;lWuc$)5CyQ@IPDxmVuKgRLuBU_-fMPc-AExSGXNtfTv zyT1wQ;%k|j3TTb3IKhj%b^C!2RzUstcTUrXi_+Q-=C5L&X9nu-2%RuxSn(40W&u>_ zLVol0n6lLjz#6?fS8ZEUKQU$0P%~42T9XHlabU9oaa(2M?5slpE4YM^PJLX2V*Z1F>ySE0KO{kxa z1K%V1AlGE@3^RPDz$9o}_#0h?SChBFgFIbKDYaBYLgf^~;Cik7XucYtWkq~iDFpV) z>aQntlr99M@U0dD#3|uyf#BC(Y5c{kxLCBclTF&dJICAdT5~DCHMESm91Ahc7(c%L zgkEDhsNBHRDY(RD1|)JhCpv+#=)|8pOR8BO0Zs|1ZqwcyhTd(glo)Ww;tqH2*7!7C z-v6Za0&X*mk$`shie!2J1`i(0>P5{^VCnLQRS~V_LZD|X=`cf z(l|qV@GUl`(*ZvkZ*4UWaxepl=iu(_jR&u}rT@Y9^NS_;Vc8kFAsE>SSet`UTCX(} zO{$w>JWkvr_~zcM8Q!dTj@^2kc=w%C(QmEfv%O zlhibec)sDl7w-asu1jQZQFHdDp4i=NF<(V^TvA@}NUo742-w0Oa}itvjlLgN$K{p{ zxjNTvG9BwkMRKj#II%{{H9}>eP0`I#>!zkk$LKoYz~+n0*OX2m=hx-dGi1`>*f7a( z3f`uoqrMu)RHlbb60FVHb#wYoiAT?$^*(y~gf$&FJ7%KMi1N`Wn-@q3h<~AW-FxV3 zuPPuEw%VZg2a)<1yyo6HWE-}U@^=RDTLD=BCAhp?=WOMmWQBF=>LJeU);D5WP7dq2 zZVat(ZVA@E(Oz0DP_}FSZa`Ii!W8$_;Mi+d2|V>m*{rwkfE&q`QWPZSEUcA<&aCf9 zS)1|fyX%aL-eb#P+#Ao+eHRDv;{#r4Itu+@H$1!}PQbI~@&{9D2NmkX7B;1+nrZYq zl;Yq%%k)1ps25B78Z$*fqP1XeLrpgR2RxO1a+(H?KKuynxq~#kemgXFHm%0+`}pOx zT4Rg!!nPruxCjb*xH>dN!o-RqtWHsNPd}j;FUGLbn1F-V>UdIWb76-OLg|E(0Jd*Y zdyV{8>AFsq8-(~5pb$~&U!RGHPwh78)o!fNZFPHFDbeJ@oSq(dJ-+`W0;TjY<~_la zS%Kck8EIU#t7L3w>pgzXo=Lt0+vQ_NoGW|dbg6w4Ge9KY$dH94z?oI+);}huN#{s> zX3OlEs>{@P)68BcWhq2cyFk7gn>6oJ-o{cXw~ZWtlM=OtbJz$pDSKb!o)oj{w6B(? zX~Ogrq1TQ3LaiZ7s-|T^*G`=&>3g$1ubtGk_M?2B&B8k9i_f+_?9#pQ!+uU2)Llo= z+zDbIbU-N;?3l?V^Pp-^wljkNSNa(LL@C{{)6@Q2`+T!)X0N^b!}$;S!9R9?EY?h>JF7>zqxz{a*4%e6V4%JH%Xi^PeLTesK^XyQXDXzdrLS%B z-!gf9DpUsEtXcza^lWa{6XW_sJ`F2kK2Gpiqdh*dOWoDjoBT##*PU3)EZ^UEm^R*u zFi>Xgw*TjKfuMnFLkY26$FCrEsU)x4fMQ9n`HM?3nrzT}BLx+5rYxKHFU>An*a4is zDgdGP{1=#th7LSzyZ_6RO}&LG_WpwP&U#B~Hz$&$QZ+J2i}hkV`I=)exq*7pN0Q30 zbBIxJx0Yp;ro*^1+2Q0!gsq-xlNQ5rHIy49O#A!&*w-sdyC!MIV@DTg-7DcA?SIJZ zImc&Vp1zO=uB=8YCI~o+J*PL|ks;NCdA$NJ9^W#7N^w%Ux`bVOA7nU-IMmm*6b@y$ z_QJ43Y9a7x%}}{&^W(2Pbiswo=3i--T4*6(KdKKQZEq^>Gn(Zaj}k&^&JPM0zv|SP z8j{Nti|=P7r0$VD>9X{{@WfX2v~rpFvZeSBnvRmw0cF|sLerK#)_&X0n$)PQV^hL` zA?5TKWSVwJE5Ym@0zXLZmPz*zv)YRjH?)` zuQ!*z<=2!EMy&2bD<9Jqk3ihIhc<97ojL|$Dh0Cz?iICjN0{)o<_#GR9= zV*Nx^kwR-+!WH?Pf+n)odS3F9^MUHCbtb~ThCk_ES&Np=a;A}bc`nX#=rW_ZHJVh7 zm|=_fR%-f8$MChancZ%_y`h&r#s7ojq0{F>y@-FqwREX~RRyAx+t{{S-d-4tahfe_ z$T(Z!of^?{xb1f}Au~c|wNJ*TYBU&3%UB7ry=w!jQAgMp^h)UYr%yrnb1{(?3}$ z5_oRRu}0Gu1=5+Q(8^W2Il0DuF)FjFu4y_@k?OuJ^8z|~r5BYT=x53DHy-iFn78hG zI;Q(gn2&N=KQ!54-iXj8L~pRGLtLI(?mYN-cu5H5lA&3E6K+>tnMH;@n?imIYQ4VD zJ6XJ6pW2V&D^g_g|CPMmXWg5-rIg>iVtUe>_-=|DJLLU{6|d&=y?#5L7T;hO2OzK4 zauKvE_L=SQV$X6DZg)do0JQ%nyKfdCk0Ey}nZ>{Lj!-p&pBKFMXNY6g(%n z0ogTAXX@D$ZpQb@d~x5hP^S;5*u0h6diE@5x=rnUIjg)aPo|@}q{e7?;hZfwhq%@#K;9LND|s0*hG( z!F%UO|RkbSrn)b7-2^iGJ)q>tWM8d_?A_dNJ*lg^JtfM6I;%0 z+^xi?^0pXHHQW6rI44Z@IQN2;{o(`O+|{idPOcjt#*~?%INxiuX)y~wz*lLm9=z37 zjt$++do14!wO7#S=C51vf+=*;CwQyk;HLFYtsR5WCkTt*>O(P`R_cGss@gTy*6dko zX}EXZ4}}H_NXLZs`Y{?mw1#BY^ozWT?MfoaO0UD;jsS<0=}s(_b?YuEZv!X?Vs1-X z4PvEGT6i*g654ZiPXwKoL`pXm5CTc_qHqmw3>%v+d=KWv&ZJp5TLpa)N#dQ*c8pjP zpVQV_7^-xWeg*Q+pZ7cX`$iwVuF2)Ykm}FSv-8+E>FR!x$nyW;uPVBXDXF9 zr_}s$Ud<0rX_1#$wFFX9j&5{w;wR};UkUHUCgun(8;5b2PMPv6ep%Fxl z>AucM^HL9z%7C2i4@`h|W(Ki<4cuS5f>?Q^26MCK**l+;TWMdyFM7VvgU->7{)^JH z`iXMLtncHej4s|i({k35X=Z{yR%?QPD8)#E`(+#ziM;}jN~5EKXQz}VHuKboCiK%I z?85}Ga~OfAAuizNd+R)=tK&^}2?=JB+Tj19_vTOX?rd!Y+GFZUp1EczYOlFyII7>Q z!*}LLYenZZ>;8IFxT4}#w_Hiitz1kYpyynT7-^tu1jmo=Q_V*vqON#|83X-E7%Jaf znFm!r&lK>_B8MtFmply97Q*pQM+a5h53Dq+|B@Y$FG~4^KCSvbN+{f6*>vz-Qypx< zO&5`?+)iI6Em7gP8rk|kYc95*-QVPT5B1)%9ynP>&}7dN<3E=W|L?Grizyqp)OK-q zb+rFtF5`Xh9+{Wb!VauP01o@cp{zfQGe>KHLw0Zy;(UVEOXhtytxqlkE9KlXdQG$|fRr{pheyxXFByv7V|q<-`+PbN zPH6P&ot@16Pwb26wd=XCxKfONE_U?i(izH;Ja21ACKQK%wyE zJHX4O{`BKA;p6X{_STPo8&+ypy2r|2S<4_E7P~Co=Q{rjs^jUzM*+z{U4#}>(uw%i z7ASf!{cF7619HfDICH}oO3yIlf7&_&C@70xJfIrANf0o4J0s`P-$};3KHB5&K6;JW zJu70`s+SwN&K&EwikS5j|Nidng?$mT4BYYg*e~~a;mq6cv!ZCvY_B7ATXBkbaMV0+2Up|~e+C{-09UFvRETd& z2pV7csQK=1{xbVXp?pn2QJcrhvLU1>sImoebNi1;Rc~vJX#8u)@OO8Aet#I36CsfY zV>0$|PTw~aO?#NISA%;?`xRag2ijCS&a5gWD9N&fERODK+7^n@9H8m8yqodj$4}iu zSxmSZB|>QZ1{0N+8IA{&emzr)L!!ViL#`pSVc77Rfv0hfBGCykH9q9|n{uUhqM8?f zwMY)f(MzwEc$|I=&6ysghoA^6@1S3}>XGDukqEY7Cuzx&bcqN0SX8S5TcX zw|mh>!gvGCG|LM4e3OKL@f7D%8QwEbAFATVr5$9<XWB}-aDJ=x{c)MzAEgW> z`)SYPE-h;q?(on?N`PL zDX-TNstbdv86RB_UNpb^0*OXh#%hkobT)R(@~#*vTsu>)i8{dZR+XDYe=5;yNiJ8nt~C`yH|kMgpV#|W)iCtCc^+?b{(#G-y;i@; zbU99aJi4blc~=&?+#*&jk3{?)72cnDwg|PCX|&F=0DREydF-AnV>!{X$JM9oZ4y}PYz&8cL%Ruaq zy341U2U7*s!&MB!*{OIy1$YAvMZS7HLGMH#Its&K0U;(LQ>yng!Yel*Rc2G||cU*8A(? zuHpOay?%bhbivqWOEIpi8Ssh843crOWIRSJZ~NZak2T9Z`cJ)5QpjLUO6|-uh>PmV zGDW8Hi3>FNe63g;j?P`&qTeq)s=5+3G}psnvmmfsyZozKsojmcm^c@dg;66)!;`7* zwK;p5fIjot*e$2cmfiBWrlhjfLybcFi9nIPEkRRd?I(DT+_L}5i7kt_b7hb0sL8F; zPQe_=PmaGbuCd?7no8bB>Qf1ngY5-uHxi#A(x=f{KII zlQ~4Q0ovT+iQ6Ve@$NNR9>U&f$PsjKN%Wy*(gwkuOeI_Q=d#zMOC zE%I$ev<8d4o&P9X@iuukZTQmjqmRllfV@_gGHPiPnZueIj}Q8{pN+D?1uHC*NR&QZ z)*zb&AsmciHc=~8nYt|Ies=3h4szt}-(D1fya))g6TLKZvBA=lv>PDI?J)bMvBQ&e z#dLW7)hp27-~5{uC}U5D`S$JA*?$LB+Hbt)0FEas({5&*hS2{GIosbG z)a$M=qTeFQx;ATEW@E|92bv#?DV8+oU%jLevdt^)60QMzR?3{Vy*e-wF9%jv`bmyN=x)wfDXw#PQ()<6`sT|(xSLoSzikGpeu&S9X8+);Iz*-9_l>T2X{kiM&X4hs*Xa!q zsO{Q5oT<}Rb}(-6a}BRAzikbyiS<5Vi1aV}&HN~^89FI#|0jBM@vQG9>W62NNM?as z!N6snm5H|JMlXo;2;4K?n1WL9FO+uam~|xB42c@lkk{;r@x9A9Ho>E4YbigLt94ZG z`U3{KAY!)Dz!5~ozCzmP)^Y?_o8-XRFyflA20}=yOFRraC|C%1EUvs#BWpkiDn1pq zVDUM6#l~vyAk7iu44Js8G32#}&!e;RUO0`;G(hoWqzzrGWoEWtkvwBUXrEBl7AidL zRe$b)w^V@KLCdB}jnqb$Dm_RuVx*i47cn^WY59E9_uzU=6!ZAT{oJQ{JOP%td47px zxCgfv`P@R~^g~u%9{9tAb_*$fFNVLjM=QAYrsHiYIX-fFP#T{Pui~sO%?pE(Ul|kz z--GCvX5qwrM;hadL*>9pCt(In(LzlnZP*#k1uuN3Gt9|aZ`@?Hq=nBx!`T#&e`+o! zG~mr&+u3)mh8;)RA|T9X*ycHctZhX`iz!2oD$CJ$G`=t?03EaQxN%^2q@Re4RXM^w zqIKQfy)Ao&BM-TGKW}%}ov6RPm!u_1;va0j6OivisgyFGt(EkS3{*pcEGzytUq=ym zROAsx8Cpt#+jwIW8_MHiQ|9G8(UigoGo)7=p^)FdzbK{|MLwRFX#? zZ~&jX*~;%3%>RPM4xK&~AY$^=V3+@r;oE~}@V1!77fnCm-Qqj?TESnRyEIOrKQW)S z`PdNoc(iU(Hl&wiGTY@yB3n$$MEOzxc_8FXoeSbnz9aIku5CckST9bkz<7OP9`-_d zWGg@c$%AO+!+N{1b8FedZ>CN9t$FX`v%tv*C42Fj!J~j9#`LYS)8LcP4_7ZWi*%;b zZa6R{j}4U|mQ)y_sErp{3TLU#)M?*IMIT|^z40_PyQ_014E%YQU_3AO+4RH@t31{B z_4+LnvJ&pKiS)7ekHK&b;vZum73$^ZfK z3Yu17kZ=1_=tq~NCdNCCBq9rMIulCfoyoI&Z_{rf!B<0V(ga_+JyN^GM^_xy9tvbQ z*--T%iL>2yztXw%DbTO{QgWU0ttH64H2yW2&gbBz_Tp-PqbzFi>9`yyI&}8^A>jv+ z0-~_hsu=^|1VAPacApm063{;6j2{R4Vhbu&1wtJ+R{u+7+#NIUOcaZ@lpZmeMQcyf z4cxokmeY4EG@3N%6O08lCy!44_Le*=Ukr8NA!e@lB+}OZtE!?ia2{rqzpu~iy_@EG zyWJnU+L`-!xaCv&ux2^ZodBJYUXf+!@4wrgmhm>3VGKW8=N~HXxW!!QRhltKK8f`i zJjo0fxtosGrmd(!@)_LyVvDNDHNpzA;!}Y`|GM$=WmKBmnOj4OxEpq7h8mwdsMHMC z70Bel4&gF3-hd)d>!6DhuDWl*2{mBVrBJ%KmO0%GSYzmrqg17i6kpR=*BD#U7h{2H2sjS4<+O@s%NCj~!L=SX zrkJq&@~yd;6YhAfbhq}H=*PW;eYESz1_26)_d}~cNQnA1`{b?P3M^R%VmRzN`Zlvx z10HSZ1k8t};Lxs5=`rO@0sQZpocRhMR@Yc_QyRfHq5DzHcky^n4%m*DR%$ZlzwX=9 zu6R92CJp%dla|SF-K2EL&8Zu1Um{7;@R?xKg5biRO$0{F=9zlV7#Ubu@J8{eV$lxA zh?diVBBOz^W4Jn@x5yoY4cIe2cxAE7XP=T7UL*R@F;d&^}R+m$p;{&74bE>&G3XA zX3iepKy52F_|W7q%J)t88BovxlkF8a&>iOOkzYBIw0hqA#HZYwb=Gk>xN}Dhx&IS| zqy1qI76+eK=v)+mchDx#<7xLTa`Fyy+N3SGMS^J)OhA@E4vcUbZ~t-w8`yx<`L9u` z-SdSFjh^8W>r#?iGLbIfMwfzf<0p8}*;YB-@Pgglnz8WWzJCL2vifmRZnl!#S97dJ zLqy^dRT$zx*S6*I`59Wc;JyMFJ2L)JdUVg@{2zF`JW-!mTV*X2YV7DN3^)FX)^&F; zA3qMhz>nvF4Hv8T{#L@z&wd6cqD%oiemN^$zEpvk%L)6(kGE|vwIR&l!2#|!(L+Vk z2Zq68&iIrs#%<>dj@CH?l6k%{pWi}=iBx{;+2(e)ZC=w>Zj<{Rj(R2RL#|9nIW$uq z*f?|2HW#&Nn=_s2r4v&cQ|7HS0EP-0+=6`lob4?KyvjjvAte!QG*>)J=v@9VQiB^L z$j6w+>lV>_@L;c?XW>R$_~17k!rLcI>j?!SMAs1d`qY6|c=e}0S9*6tdpFF*KF0&z z2`%Z~%@dM7y5Fj0!e|e<`A<wzE^XU@g{WT-1rF%dQMB_vffXpVYt5`1Rp3Lp1t|WCvaGF6p*AB zO|P+b7yLnp^xRAj{-;XbExC<&8N(ZRyI4Jc+I2<#xYk@4Y-R4~eK3b+LV%25a0MsA zW=%(E%Dk=co>f7G9Cy!0FC6ARy-TelyR!n2Ky;?^+K|-XBDIDL7oClPA?g04AE6eA zg|6tamr4L9fo;w3BVz9+Y*X;EJonO_dCku@5_DJKRw^a9r#tCLr zT|)mc$J#SF2l|k56P2ODgEVT6LE%++{_VNP?}Hx$6^By3M5Cm~q_omzzE~lc4=ce! zW~)42F2EVcYK(W>9GnEOA!YYv@R2tpP%_*Yb?8X=_n1)F{Tm4{Q zqi<(Q_l`E7k z$1M}gcd!yPDxfX2+fVDd(*Ol8f*Mf@-`y4tw491Ko+(#EI&P(Y?32ADToD^m2t8IY z0g7205JG7s1IMZ$F9DsDOwh$$$sa*|@1U2j9T8?Z86ewSC&;M=*2>aC&ML-eX*Pg7BZOS6$Dp(6de_!fzP-6H%!gz6UJ+$Y44>fh{_mG=lc z7k8r>(aqA;OXN#)2imse*4Jwh3Km8l?HLZ%|GKN&^U6zr%=1raKJ)ueMUi3g-yooA z#8DMJMN3-ET4Kz0T|0j6g)=8^Iwk$;fL+D84VX*d{9`hT$+> zIt3kU>UBIgw`D@E&2zcWy4LP5qj(&xkam>;-a3!JAGsw1w{Z5V_x|VP~qH0abDp8rw2Fqo0ZckzsDdZja)L^e2tK10<|e z0IA5Hj`~>uRwCSHxl?84xGIi+W;6Ui%G7C!lgQ_=A>bpXA`t3(YmO6pUN^(a}qJgP5)8-(r^P)e#yKl^(gm<3;-P%YtnEWhd3y5_qM-6V5?17yD``?11c(pQE%Q zXAKdd9_o30OMXVPoQ*=x`?Rhx&*#8Wn4{29ejdeH0LHG^;gh?*7`C^0L7~t8M9mrDBRu(U;G{Q%V*yq?Oizb# zPyCDyCGlrknyAwCdA#tq!PqY?E7{0jKzOi9JyDd+Y zg4k7$NvU}!F0=@2khoS3g5}-c82_6#1EwPfgrw<4F}pcncEQPUjOtL&v9pn?$3=AT zt6dYE=ww)x4!+AjsNegs_=;fB>D4Rs9Lyo63!WF#*Og)*7=EmtJwW%fnKa@mPKI7b zfBBK`S4-S|o}^b_JL4V110VeY-~c8>)`B=@HJsVc4zhl{CkFOs1wz)5_3u8z_74n~ z3w*6?xHSF0&*k6U>o+{RK{taYH@8sC{#;vCb`=Yg9V4P+p-ilaG3$38qpQE=%sfY; zaZRaSO|_j)ua20~HE)UKo^T)h=fTre4m#nsN`{Na^luS$wnC`adT)&?8k{LSouTFi z;#Art#-2x1jdv*QlGd~{stvi2=%t^{O&(vo(!aJwkzv#FE`dzt;^YY;IW$LO0+2{_ z29_pA*)+EH%jjeZdocA(*MEDnw_A=O>prFPYOGB6y=T1&Yh%zjA~f-4u}3~-q{2se zAU@o8ewhLdwy2v?t`@xUZ?6_KP3Mva9wQG_b)Z6D&i?6;R?5N(=4VIhVE(=NFEEMs zZ_1ZSUvVhwZdn{k$>2HG!xVo&%IG_~o9YYm3on!R?T|_L9m?!1vF~X?o}h`PFDsqp z`RI~NZ_k$HLfrpwS0TwehQV&2K?5-hO72|&eLCQYKG=+R)tE5#vdqf#S>|Npq3?bC zCM-DQi9+In0Dd&9Fj#oZ1AAh)pif9$UljLw^+1W>W5`8G8me|6{9Gcp>0Quw{`QSA zXfV!P_DKYrF2UU(8{x(5k|w$Gho-E5GR)hRJ2Ti=Z&vB_*h27jokLQ&Dw(8YOu*|$ zJ;^oiD>uNpr*SfJ3WtAsop|Mn3R8;`o#x&{D-_q4WW%V^)>)Fhm<`bVVQbeyC0oEo zidIW{1_d81v72ZoEur{aan8ilBzU${KZtu{aZ*?UPw0iu&t6*Hgy4=%zpk9l1s#M8 zPUxyqOrb?hb&g{H1!oea5|>i7Lu_aP4TZYGLqEv_YHzJXkI`S+^k?}WCds5XA7kcL z1?iGW2`Q17B-$~r=G(dGEbtR2(N^62JE~)^8$i)SlgMlr{^mi?7dsq8obZ-6=;wsL z`bpX}%~LgIVu}WF+*o2m8OeUww+iZ1QJowwE8m<={d$Od5ZPKUKNj~+)96`_CBnL> z9O_esT8+Mr*$?ix0j?-PBdwVGgsV`yK&#Z+-p%w6(J1Yht=q)e{rMTdqzY#7ODf?s zgNN}ZJS|zue0`)k&fE*7D%IHUzVS_?CrX)~`$>JxUy>PJj`{J{am;yNsS&D*>Z`-Y zL#=P_$u0(|4z#Twqy2i$bd88H=tVo&;mQc^b*a8sD!A|o9KU4da6&vEs^3!^+)+kW zl4Yyrt*35&H+Umoml+ z6XbrH!D@*~?$VTW-i@-Am7`P31|p=gntwOR)f>!WjpYmF%cLX0g*OzFWu0FQgDR3W zLDZKe@eRY89TU|g0~>t8AJZJQ3)n|aSqAOzYv>+8DbpWS0DZ?XaId=MiK@-W#-tjw zq+4xmb#mJs_xz8;`%foZr&)HDsoWJVy8Xe|wIP0(&$5_LOIwJwj#x&`Yv?4d&MA>{ zB@tN@ugJK$D}&2q{?LS5{{+R})rvQ?C|>My}-N9w{_EF)=o z8hBtbUaYsHQR|3881(X)x%?mL8vI&2?;+R3oYI_x!8Z?m$CNKnTW@0R;L#(>Yd~h- zdG!+({NV7RvEfg@)UH|@%)=6Q+Aa#NHs%Z;SjVq7J-ix9zjkS|d|LX9C*cD(?Z!Re zmQ!WUz);6CDFzMN9Rq^UVj6-qB$(fom^E*!84t3)d5kIPiHu~m@6^Y(KKeOaD7&Oq z#^%pFz}86Z8;t7re=MKGd>u~5N(&9B0+YmX4}t!-`sCyv?=0L~t3~IWcM7d%wM(eA!RL@VGe7J8ZqE zs!Z<5roY=Yi__S8z`EPlB>e_%=%l_;En|q>z&L+MP^=y|nTu(mPpD>M6jJe*=;VG` z-^Fs|e98k5Rm8u~lYTsho356~H27!}?4GgC`=VuVTk zfy#Sm*V)%P-np`Us435iR!-c_>AxuOQxfmqTJh20kNzw%>9WHf+=NTX*m*>(NT=GZ zxYgUog(?Y^%zlLsv;iW4=y#`HAG;A5P^9Kq&xyeW)o4~3-J>BYjI+3c8~xT8lIazW zF8rD^V?dI6F+9%OW>nc{xYg`Tjj^$zP+Sq4OU3zfPoIncuLh|1A%6KL33`1G+APXy zUiKqDe%j3VPdTpAQF);0@PD#@?Z2s=o8^a>JOuMJm=g@J4}Ehecwtk$bt}sRI!=SK z@0Wm0*bW5w#~A}rWtz2ZvMEQKL>td@tYIjMwb{lPsJn9g(7TPHZo(o!u(LhUK<36K z8*@}XMSRh4r2#FIYdH{%_Dole$D z7yf7xt~z{09IY$du?trUCHhfPe!p$#=z*+XZ=SHt{{s0S5B>B^>L+qL%p!Yr>f=am6~sEgniq#K)7?RIL==X`7t~>BK+FLCeA#pP7^4`;V^zCAmJsv zIL7lme#={k_SutAzaC9ih35BM1j$jjKaWjTCqJ74ls-DHs@DoVb_4H9j|raj$Pczw zs@jU)PVoHtjT8+k)NbO8^Y-DiGV{L&J)d^PU@!)LzR)@;ppp|I%7I2`-rybKtvO?H z=K8dJX-&VCu1H3fbzV+M%CKulfS$A1HrXJN=9X2HFe6wZH;hQEmk(9qi*r^dn6Qxp z1;dE#uk0)fBjNOUoqmF}cn*Ss_1NQRnPTf5F`X02FNIC7k5#afHN+A_R!Vx5jZOas zzhlpn87UVaCh+$c8~RBeuQVg3Jzab>_ViroK#G^IRlG;V_qf=-$5tU;c9v|M4n@aa zV5Dc)a6=s~sRu1KrW6DEU*tBCGhz6W3Kt0Sf%&gJYm(}5^kk$yvIY!SJ-%8*=jg>& zpRoW;z-P<9#48_u-BbI`pm+JkL;sRMugL=wl8lCa1TEB6Zn|p9(r2lo?t*6O9me#> zXo3|kw*)S?VC#zDY^XP4p1(z0dLGt%|K?7Lk?r;~Q2BDJ?S5`dQ%&;pvw^_8cf*o^ zP@vozuP%AEZA@XsAa9mG=_K9V>hL{t&$fslip_PYu0AuvuRc{+r7XB6zz|?~!z$}f zRGnsoSJ$u#vHu1-;>(WSMm6~~H@Q{qMBH(nzoM|(y}5GqIqoG3>$e%kEUW1%w3vnM z1cv;&8Zdix54Xu!>rQe>0-2iJ-n*l^E`7?!vXFy?xZH7@4~rPObI;%RFWb?H@j)|pG8z~t~xV19)(R&cTY*ipZx|h}MO3Wb< z3bm&n>tde?Y?%dj6k+I?d3z>4&_PeATxq>!hqL@|OI+s-hwZ)j@VXC57|P*8k5sb} zHe%iYR|~a%xiDUNte5I*&Ks4gz`X8U1^;(oJm9c44FCYez&{QI~<3?!)IeqSR zZnfBF{$=iIVvyK;>l3KV)|Kfu0w$F^=(X9uvABWgjSUNNw8LlwVEb=+la!~VcUS3| zK#;W%^~?P2pyIt}zD|8f_P+#6<5}~8t1nV;?_k+?*2fyToO7DLpGmMCOKfj#<#hbV z$>k8@i@}x_ln`#ix<0w+w_Lrn;Z=c6QF`+fPef2ITWFGRHS8Q#Ng+8lEI`#>(Y z$Bz?5I{QTbMGOOX`Ew4!jP2CPo6*yu3l>}4d*TD<#|0c?pj5tD1nNbkbx#Kj$yFQ6 zwrD=C&_EmJYzm-J&Q_ua3%g}O&nVezs$IMG-&`iI4A*^B2qjy6GbvuUpOz(^8s-(` zD}n=Gi_)a$gQ$FXgZ`l!(GeZ!r1nWuUi2w^Em`HII+=KQi9M5}Wj>t7y<_AGcL5G> zG*T1MGh1d)Aa+#FBmuexoV%Y8iK$PiO%pU=d^im&gclfNXU?e7zll^}7Wu_HJtz0G zzjK>*F+SP;X(H6#*lMxuv~KeVI(|Ie8h4 z-IqbAWr^Rn1(WZ}^Hw;fwcv7~!2$le|C=?ut16zFADZ?k2dzLFz4BCAcETY)_tO7G zjSn9}>pWjKlgM=}Ouv-ZmjsNFUQ*39FSlDWLHTSHk`K3{vSd=M{`sov0qARkIdw=C z_lWXKo2$fdvVbY7&lz7G?{Pku;)*|?PEK$ptq+W&u0N1775h4_GQ%1S%q~1-h!ppg zk};sUNw@_HcNeLV?c-Ivl5ddS+vz4YX-!Ew1+G=)C_MVA4zs^e+^SK3YUEg2Y3tuK zY=-6BhUBF^o*Po8ha-kRWK!!wFd-C-){ok@3?LI^V_v7i$A1RCBGuBNG!Q$0>luAhXU+z;FosMtr|Hsq2 z$20x^|No&#%3dWQhtYv37IJ1Ii6X}eIV_e`LXI_Tb4=wdj2t#{h~!MpBW%ud%K6wZ z=i>~U&3Et5?{fM5yUS&}>~h)jvd8^#zu)fHD`MxBGjgcq={@*zXR;1#@kqrOAMG;` zu04H%WBk#Vp&YS6JF&?=T}Z@k=700H*cjqt4al05$xK~wSXJGN1q*HEA3*E`Yg)E@ zLL;NN4En4(p!ReLrfa-(J6CRGeWUIxR#D!QdZ+i~a`VkOs4s?9Y%ypQGO-U=BVb$^{A zZzQzd;}>Y}C=(^ov?=0Ty&5q@jLg~YuGQpRwzz;w=NUMRb)w4qA*I(eS zIW--;nW0%rSOQ zUcIqU`Jro|zwz-$^wz=O<+`#Jc+j}OeR29Nk*#Nd$=}*k-S%o{&L;tq)Pi&Lxob{C zyU^9swI&8%dUt@n!E1v{xEC>_v`!>L15p%hH}ma@T)QZjB9oK4jcz4Z%vtYjiiP%n^6PZhZ=OXFI=>F<`yf z!qc7dQWSWYMIT@)EIF>;uw&7UkA5jX83MTzp~v`f%u}~82%;z9PK^r}+xARUPO|Is@ibPdMY%U5pGy@WUiGML z$Q?l|NKZ%%)>NoFFk;PT$o8X*lD(BKQaTth=3zM_^)HLO)m+xUYBi>1MJRe*$W|y{ z_OTL{w9C)_$MRnT^#h8>UpA{q2D9@kmNDfaKvHuP(bZ^Y*M zbB1t~^L^p|`^I_nedtxX;Tl$;=B@wdmC5s?c@MPd>BSma6tvo`J#+vuDvoqRN zUHxk!2K0nvcn#C8Iq?-f(zJ9=|HhcoX+UV*1$zE9raFv~X~Wl`P7V833N^}X7ls9> z<{Lse#C6Z9H%V*0&<{`Hu&F<{{52Z;?l?~)EcUTf@;$yVl|&@&1%Mp`~(^a zul!5v*=w$R1_UJWQUfC8gG{u7mATaHvw!UxWrd)Z=eI8?Ekt+~N)Fgl2|lNBQxk%K z8jb}uxqKqhe3Oze$J0^$7*(Jq-atrt0uUIZ`P z?@|*RI7GwP*}%z{BPby5H5aP}{*<&18F+Oo^>pWQ3>{yF%a{9}R2{!ZD4S=LRmP%1P5uNV z4e}y7@>RHBr6NvNc%AiiHAjw5cdT9|tfYIV^^{+t4|q40a}cKPR zPSy_p7ZpDDLtEAozba#G8F7pUq$2}F?eIVIF!ZH7V=|;^tDDt8U)P|8YtINyBWM{w zTJ&JCjlTOC@ZMJzzep4kK!eg|*Yh@Mu;Y$p0CKPa{O!YzCGvnWfjRCNBzkNq;T9;6 zqZHI3rKm`a6t<5$rcD21Q|5D%o3g!@Z0{Br>eR%QR`oqP>Qw4E3T?}xM0$JCwB*oZ z_uzgm3s>d)mgf|;y5FSkAp8~yCaQ}A8otd^-@bc(t5m9M5Fc7<5>_WIQGoZ z`w&-PWr!SX)++m#p|T*rHJhk?d?BB2UcQWWJ5%nJFzMvfmb#=i{6?BU^d`mH2oh9P zZfv(y+Uv)7fcfF2`>0M0V}Y5iw)UAmU8{R#Tle_M5dBBv143SwO2NPClv`JL-0#@8 zUjd5I|H$UgN>l*8^C7Mp`Wf5nCjK^g{c~sf&VG;2A!a=qASknU{le<&)pOoe+qnxL zo}1-sN57skynqEdUd*SMieF`PQ;oXwJYYGgAmB<3# zZ$4Su5b%lH_Nf>2>h}Qcu(dU)=N-o0Wx46N@`0DH9X-D?BsyzC#yw`t5y?FE<~ssH zZC~TzR`bMl#au=o?C;#=9tpUEnA+ooYLD(CDSLY!5f$yTULv1_$RhWA?IDY`oUbCH z#nACjq^j?z_}z+#l&2Yl0X%7$ngl(1yOoFB_JAqpH?T$M;LZUF|78 zekkCN`o!YloZutG77rUOdo=O-%Lasa(LtxJAtps2K-{>tC6di=`8~b|d_}69IBc?% z=gPN~D@U#~JswIyWcXBA=Ji>2{t_70$;1sPU>_}5aDGTrp5*HP2v$(8GP{JkrHdPJ zTqzV-d5DpEX?tP(ff0NUI`AFJy)Z{PcPzmW7!NeD`+jES4^+AW1O9GH1wg zDY}7Z?r}fi(D4-gGPuM62*;kJ)&AzyLCyWNn$kekGRfZC%We}?(Ep*C zq54`yBOz^q#$F;*Q&c|Ui23Uuwt$<>aci7Uz-do`u$TIGos?ddsPKhF?`HKmw-;Xl zOQWFYgd@3k4Dv z>P6OhK=>5F1M+Z(ip)Cn)2}j)^Bg<9fXS?tANRvqMU`Es%%bf?Lq9f9(>6R1f;-MW zIx^zF09=|p)x8ZjJ}^J$KiatYsb&s7Uvq~TQ{xsSt zIWhuL1R59$+P08n*kr`^07XWfP&2Ql^2}K7of{>>uuH!QnPfGR?@bFEIOeHTmcNum z5Ip>Aa^YW>*g-t8(7B=%M>7&yE!q1sDoBi!t`+iMIpr!*^(kGRy9Fi{xr& zo}u$h>c2?kp5QZCzg|f6+)Yq)^KzU0$?$CCS@3eZ^H7X`Qr8bCtssd-1h+pa(9%Ix zc}xZKEwx~%KfS4we^O2jRrF@YFeJT%UK|He_Z$R(tJ?dY8v$CKSP(rl&DEEtnH!Z{ zim4j<&B5@Iy{O-fx5|ZhhFgcbdgnkB*8g33i|W9?MSDE{ZK^yW?t+VRipnAemt}x5 z!|T2BiyfZiJ~XHDXJ+-2QE@x9*;7B>GaR&b4rA#O@klvKAN|uZF@>v^kUExC%4+m~TqcTT)I4!WbCgS*Xr zsuowj=(Gu35u7%yetzCxlNsFgQ|Y5-$UOWS-5^eW{&JlBpWpgEAkJND8x-5SLv`57 zw0bK&u=<~&l2%NAO)mLGuSrlnOb0$3w?95tDo(5qOZpzJrdeF+xWb`E$a5E=-;H&R zB{lx_w)GYnz?KI zA)2h#qV*pt?$b~eH#w^MqQ4HZSKx)B^1GGLNO`jHhLC4KDDursvV`A~#t{CTO$jq1 z%lzF-2+?YZShiJp5+U?gE{>XZ3x4EuYrr{9z8$&r{-3!~yhir9oeXq&6rt`QL>i-w zA!W_9_o{BjGS}Y<@+`b;a9u(h^~t$v+UzoTUFN>+>)OTGefk~fgsjnY{eg-WFsMtx zs9#vhl-rXRYwbT|3)^n5HpQ%(;sIOM7odJ*E}nbGuPz~EdzAuj5zVYzYBoG~Kh_&e zaNW2m6hAG{hspUm|KQ1xxzAe2R!sGW46Neb#=k3NzxJvu^;g*2ALvASJxX&GQGuzg zf*i;k>h)sF93;Hbw|%`uRgPYHOg;}$ei6?*YE-8PS=|mp{=?LkBiYp)|Bc#S{(jOm za?VvbA+vYSgM|=mk-T=luj$5Auy{btl@|7LAWAUn-7yNiw5b_ZKAM-zE7 z3@J!u^fZ$82)7BBUyXEvd4NpGaq)?vYaM| zXzUydhuY`2uE^nd|3pY+K?U=X??|iRq|Z!o><{QXG~bIZq2l569jYMh&kpwB&7kNX3np3|CXPPG@+p@LM%^w{8%=Om#?P zbUA#f+JYv|IMvz@b#pKZgDn-;-9`pOJ|9;PgTc1R#s2l-%%~Tc^GQD!cK80LLF?PH_O&f)(TDxOZ2pS^S>i9+<&vF z9pA|S|Hm=xX@JBxCC8tku;!^d3ETm`YyB-A__J{7A+|hCrvTH3kiIUlRQdDxJETJ1 zy=*Y{S12l1 ze;`ZHU}vZB*yE8Cs@3p3=T=_M;7DA4;qi-TnE===e!Yk+sKDWLx0fBM366#C)B@(Km7= z81<+Pz2(Iui~LT5uV3!c;@%g58+%39>5uypWxN34EQpL6IY}lx*B{a)`XH?+OdjY;1bp=%p=f zVbl@Tk$tod7!$BU)4!SL&mO&u3d-1Z8C55$FACt;K95}7h1Y7TDCi-m=8cezALFGJf_Z5bQap$Z!u9AM80{PrL7)qN4(ou5mqkwEaEgA z5sATem)?mUvm{*B8p;8$b*p|I;d&9p*Ox>MbsBYsK9uQetC!$k8WPx1qm_=-?#Uk?WE*Y=N8-w>w{e-Q$4e8ue9ay0N# zcQYL~if_D+8qXa0Wu_7Xv;+bZ5skO>fe!d|xqX4o-tdOrsr(w$@8P04CD4bAYX6|i zYR37x-Fa!t z_AyZ1_E>NxeR`@iZIu%~++4r8c$AEimnI60-=)-uz5crMn&U@Hm34UG#d=n!suYK{ zs`aKTVOLS0(s6+Do*Y_mMyCqi7t{ zfl%{RTXG@L={B7wA;*2KXXl$UusjMeL_Ap=ORPSz%MfaMc#6oRvwrP<7iaFp@ocrV zol~|OQJOP~oHVpk20feR`mo054{YMm4~T2*zNUj{yUQ8%$m@Hwd% zBX*R(mvZuX@Tb14efN2wgM6Q^{8XclC9LayLM0aMmdlpqLopA&veTFD5er+CH$Cf* zcGLi|`!$;2dBNcHsIizr4hu2C5ko|U$-(nm7vPa1R4uM-1Dm_>@eA(5SrlE21XkAL z&LwN9Td7o8lfwjE6&9%Z58*iC*zy0W9~?7i&>!`Z@jNf{dg%GZ`ycnv`IEad2O1_b zj3^u_;NAa#O8wjaUn+wYl0vQ^fa(m&|0ww0s;vkJX5A zXJU@&%Q&Vxb8^|Fev{Dyq`5;r&_3gK@=1z=#dP%X>@pW>e+0Z2!VN;sU(FMM9L@Pu z#HO+=^&#n*V@#L!e;^B0q|IpAGuNnaKk0QV~+~-qcwjNFabh4n(cFJ0`S!rBm8Rfb3TwJyl+sb zudKzP8_(0+RkRuK=N%8W0J$DjsTqtV81~(#2bi`LpWu5;=z(0_Joqm-+C2^+5}@`;ebl3{W(;)t zNh9>G^YNvCfBt<5%X-d9lrr( z&l^aI>dKET?Pc_A+{*(sYgcc>J@H)by?-bItNkT)@9=>T;p3w!IMT)7m3bOji`l$EVfUrjte%m(h; z0y%D-8031r!d6$cjD~5CU7yLuie$?oo>*4t8IKKVamRWY>&AJGcfxkHQ|}59jBxkH z9|OG1pS{$W_?9USP}+oL|-0_r-t1Kbm2h6RT_sf$M|}xn%6MQ&H*G zt`7xhM_ZOPjrWsy_ivCDVH@`Z*cfx_-CegB zSDq&dsklng+olWtML5@ETlN~dD_ohY4bv+cerl}Df>_c*xfN$tk<+=yVq&YHF%O! zCtGKzk2R*mI6Z|+t4fJ;d`ETUOzck4xvKN?b!jY1ipPFeLt+~7yCbG6gRo%oHK2d@ zi~>fSDg^o$DYqeiRB15(^T)yi4Jg+%X+_EiTaGB?tW%HOE}3a1`U}ouJ+uF)`>?I+ z>^a#;zxuhka-|qD+>ft6dGHN}c`WBDW$qkI8oHYB-KcIu>4q+J{M*h-NE6up&DXL7 z-GvLED9_A<77gu_+IhLRiphZn_ z%n%pW$g@*&un}3sP|vO37hWB$0#eGk9OK&UoZ=a%Kxtp&DlzKy zx-U79;di0kSq;8dr}9Yn;LV!eipjX->qAz?#alGaLXO*rZ&=uJ@ZQRa`R%U8URtP6 z_v;LTXH;3?#6|Z)o-igm7tejs{bE@GFCs~uLM({#^p#3e_iD?2d5L~CkAW~8-=_msR$aCo_e1WQ!8-%wCmXLg&eTHn%ts+x zru(rbUm9bhY`X;LhN$bhUyLa=Qm)Wa8}Xz=h4-|vQoBQH=?{eCPuv~Xbi-(x40zM! zMgh#get=WP-N>yqXSTjq?SZ}Ve9Z~Do}QMxCm{3RG-tZz;?Zu{Sh{O3pjfJ}lD$AG z#p~QG2h9_AB6o(*0y>j-@xkg$l*h(4|Kq#yiB)F_B87g%W*$tvlBPvbsHY#hmHc$hbFr?Bg=J1&uJxzB+@*82{2URU=kp z_Vv=Qw7-uUy!3^K*8}eM9^NYD8p@NKFn2qW4Qc<>&9QFbmi}5vYgH8p4udfsm?Mka zN_=t88X@^wqO65NZpxs-&g>TKuT7CrQc7Nwhws~aRdGv>j{;Y&_o$wkNLC5M1D<_k z@dGL)q{?`61Mb>T`;o>qtwW?;&i&tsT8p$G^^(G_m~IzS5L0lULaa$^eE8yRSE<)! zc384<0x4O2&ZDYBvu|3bv(Iv#=R(P2%+05WP3foSU(I}Jh_q&( zrGcEPa0$)O)4LHX_?)(92bC91rJ6HQ{X3Ci@>pKs!t2?1|4w2JsKU^$)89y@9889+ zdmTO4Blx!a#b~GWfwhf=P7kGmM>9O>sTZprn3|+CI%^S<9?Y+!Ykw)?I2OiFXyCBQ zl{d}rzYu!-TOlbAgI8G7Jf1*K)G)uu{twq^0%A7AJaucUJbwtgA^r6q8a?UXNLw}8 zl(Y9}<9FvR!f7KoP-Kb#>Hww1=>A`c0fZpvn~42omy!O|FZw_TV|kO_k_S0zu3fJE zE|c&^EzouDg#eZ+&{@a`<2s-@aR2HS^93D?kbP3L4eZ!PL5-~x_5S^f1` z0=mdXtDL*sRFnQHC9z7H*WjVRe72n)ViFKt(2f3T zaEi={OH09Z>!MZn>s(wvbqgIB;=HulECX6+{B!6vm(fS3R+xRCw`F%^k$86{*S|$- z-!h9HQ;I8N^Dx}7d<51cflqNswQt(%BMq*lFg?hNecQ{{w0(P z=@*k{wM>>O?ps)HGTJ7U3?2oLFXwB=&_a<-R3M{^ldN1z3;X_8*>$~d#}L9S5HQ`S zF3^X2UzO6MUu1A5^(R41+o5LI-ov|T5@o?G)Y2Nn^MHMj^~e}#Nds<~XylK0bolLz z5~AtCLGHamL4_5+nm+D6!NqQ;aVF#ud#{sD56G<#LLIDEYEOFq#0YL(s6?&;mcVJD1l`{>nFhK_(ThP%%8IOFFb4cp6SU%iHSZT2q8 z5!B(cmH5pWBp}UEfiU$V>DbjrBVIQP1T=gs+lQYyFYv-2@MO(pPljs^GN$Noj0Zb) zWaf5#FG5M@weyI_bdC_7kinO1j{e-Somx^yHm5Ww?A5oXF**>Xvn>UL>$;Jcn>GL$ zCkR{Y_sv1p;u@j>6#s=8Z2MQsrgRgEMJ^}-KYa!bQ0z+siG=A)$D>$P^l>m6e;9F1 zXMO;y4aAe(3-vzL80jZ3}xZhdpIpiO(#w}b9(D>P0y@$Ozw#?2<~jsdiy z5Sv|>ZtO3dZQhvzHCnD5-i{+DEA*nIRe{x#YrRqv9?Vx z$m(Mmi^%I!ZMnin z6-Z~-0yTIuBRrG%zW>HCj3`DxU)t;MAO$B80jeW%%&adOS}(~zK0CR8imN4wl+^Sx z+WilbsH|bFeb_IsXevq`kN%cx%p%0lAKmsL?T3O%!mv5PPx&p#?-9P;K?xFo#17`O z-(OWALD3XpA)*vkbncK-ujun2Yaz(TS?f@GQ+_?tX5}5r`#Yhn#c~3yT}Wty5_Faf zgZOySD@?iC}Z1h>#W7l*nWa+s{ zWXzzmftV>l;4*yU^YNPHlnn>IUGVh3CS)$moRIwc+Xv6-c@e3-dYDm~%^xrJ6l<^y?%a7A7BOM242z|;Pf(Sg7kx`Z9p=CAf{yL} zTtd5qsV~c{4>;jFoeL;a^5@IXLY5V?yJ@X!x}67C_0)LzSYm{RXhGdM)n3n%-PcL| zauZoDMW{~Qb#4aimh^kM)4lw-leZ>+?k-p@3VG?E`r@FiT(oxis#q1VKu_tuJg8TZ z1i@MQNam*JFb_SJQF6R;{Ryy98dG|bHVrxqN0Iejd1Ou`ot3UlB^OkUpnla1-(C4k z+z2pSy17u%aqv#F>jWV|Si}4DT2J+o6n4ELjvVg6H{^VVVqH^sHHJCnt|PpbeGc(Z ztMfSlInFh|G~)_3L9DTTh;CC1ZhsmbqxU^^oRRbXZ9?RT&+~vC zFt8@689gxKVNdJZEh(}o9`gHRIH`?U=7H0%h|Gx4Px@E`+I5FzN91(s7rofxwg-9h zNl8_ed9pD<>Ve>FO{PNb*nQ#b#Z!cXRHyE9l=xh&K>JRI@g@}gY8X@dtw%bMnKeT_ zp$1EJ4LiKWA@W@p!#eddeCK$_>tD>MH*LEg9dSKD0gDoQ?5u zr#pRku-Kn$?G`RDeasy6k5HLiZ0`9-m4Fu=oSH~BDd}kU9ZNd*ML3z;r?)nr<43sX zI``3jbJRiDcxc)AtR8f+`h+PSzcH}r#`U%~NH@v%%3e;!8lfVN7AkD9#Hox}Bo=S$ za5=S4FS-A%+s~MaLfsx0i0~{RZfSBHEF!(7u`#AvEoozRh!=5f`Q>vx)dq z@Esu_%Y0yhi)=uQInZL<0z$?!8^EI~H_9Nva;&;bk9l_60%W=iJz)x><$?nc+&n(?f0pud!IX+=v zq--nTVr*1_*Tk6%kMkH3>jL6msCf?HPFkAW1-^fzWNvRWDOIW@ub_oTYq7a>y9OYX zCcZb60~j)Fo^(L6i&0tM$50A*W-IS6=I8GC_McNwp8yDiHd?Y{{(BOKUdgQWHn3y-KGWwE@QV)0NKM-Q&Q6k62)1f}qGJBf`isQpuGPSYQ*0o2C)1Uq0@&vLnu>0*tZjPK48LGoHr zxfn*=rFKNxp^l=qZl2*KWCfydPuN2|i_D?}dOZTPa~alDMPnvyq6jkH>A52|VX0FC zzNzbvPnK`pi0tB}8Q9pL2*L-c_sT;%1M_2i&HL`B(~42qMQusf-|~cQ2M5>hhI<6j zvdsROS)ee-LPEG0eMh2wmlc=8Dw5;!xiphS`0_YbZttm(OFB zyL7&yK?1 z9oT!i0&q`licfJpd}^0nl`}=LRy(?f-uvC$*zdOTy|6fap&T(s`;&nekQ|Lz6xjbD z5v0X4(u{ti^&x+l11!!JpuKCG4kOxq2>P~qd41I746VIi;P3v7{izS0qv_Wjt zKjil|{Gdy=D-8-rh5AIcw8)UriDQO7c9jUzs*ORSQ@G0}X#f>4_8()J!*=}?>dc|p z@^d>>(Mhc9LDOes3#Eo`y?T>Du0@d+UaowpZ=Lq<&LYLSPDy3`-cQU4L3FlU7O>C0 zG`a-6Sorw1Tx-s8{HcW;SpF(9pa6e7s7F-|tno%6mU;qGU@em`k#W6yVGQSD;hJen zQNPjmGUzSK>#$l1;u8AW-G2YSZ!oSs_xnBin2A8R@xGe%ygJvmJpCTa2`ElahtI-O zl(qNEmkyI@g?vjEF|Jtbuk}G??MLSr`5%|-$=7A=8ddRW+W{9D@n*|JuiX7QrQ;g? zZQX%17i7sw06s>ljrn!<$;X*7LMtm`gXlz_=P)mx$_^HM7*uFv7g%3fA-5`u+Skwq zHf>y1ViC&92qmKGC)G!F?RKmsDXAYaQ}4(x@KKZqY}WKw8Y^i~)@W!_VS!7M%$4xC$&%%kKy83LgmPfHmPKeLyjhkgii(H%EZEU79NZFSd zgQ=T}%DXi};>K#!inx`>z-`7*UavPeQ-x*z)g;81iQLq-ea-cL*Mxr1Ti!M~afN7) zpmOwaPAGDkKkxF8#8j~%a)H^w;905VNS^h4m-Ht@MOMBf=rRMok%xLtYc2JPXZBCa zpQuCL2rrJ-QV*oQeA`-d)+$=ehnp1N1a*3k_|~rE;jD{d@%1#)t8|>BZ@Q+Kz}>H_ zR!v)JsvMdfqbqwN;xv!;)d|R2q?y=ye>>WB+0e1iNsqF(`{azkWa@mzeYp9e7#*xLX?J|m zsTU;t` z16t;A=-On9OS4&nTY2}X$WqSnPO+kDL? z0VE>mTMg%5lL+kd5LkesTJ2bIOSs3*cUN8*6(fx1P49;?ZSw z=4iZ4)&hH0Ciy%5*qy(*EaseU{Y|YYAyB>efV1MEFPREPOvlz1pV5TolL^ zKIp7eV0q(|HZ$YQjE^r2&YVU(r7?nGf_y_We%#&1V? z-nq8d@aEPf{}T&djfP5wvZtD>#z?P%6J<%-E0-h1USyWYkrnp}x7&BE&sIS3fy&`g zSN3{t@Ex@6CQ=Sv|1yo2=RjyGIZkhrq_ZKKJ*(cUbO*!pu1E4^``$BPt>Aad0tjGv z@G^)LU;1#~w$%M|hLpUV8m|8SA_CwQsJaNx*Ie_Gz>5=P8HzDlDOfRtmmSB@IxXnz z!YI)%n*5A>G}fc!Z5F;>5h0?`-dmG|NG4_Cs<>Y3&hRBUsk#1D(mCucn$%%D&=CV- zv4zB-9pUuR%Z;`yN3tjjbpfT9`n;(33SjNj*s)$>rFaGYhrKwc-})1Ce85tF3Xi(l zpG^H&_9sz&Y#Mi}L$T%yPN zMAqJoxS{YwoJ@#4`GT~OO?yA*nJxoYv}@CK(ed=^$8JDH&g_JiA=xeG>B5AiX40}< zpzwV+5!w6Ei%wN-LS+F#mzNZoFHjw{X0wa?QMUm97zdq<(RCj>IG|tVt}pZI{MGY8S#?BtCpqCRn>{$H+dSza0mWu` zu(>LF$C3lKf&$gU;O69*^>pFLO3xvWXdUgtAv7UymyN#U9Mu{ArG9odT0d`aVzpk9 z8MqjIeHR~oCs-XHS^94dY%7PEcq-b>#W!!Jb5!?eiKKT0AGOWrU7VD{O#IAjb01a8 z`58&!Icod<)AhjU!lH2g4;G;N%yJ|^ya*m0Bmy8@W4Wff)pwRG@|=p$k=y`B_`53y zf7_j1j1I&L=@#9)-+?FjEzQCMJ(dAM_eEzz6bXj!dcD5w^w62NLBYe!C)X*u3XX(; z7xvTA8b`D{2|&0qV8X?ylyQ&wCH(5QY9z z2O6&r8_#Jn*E?Wk+ zgW1eUpKuKAdh?({5~vIyx|*e|I8|8*7~H%!92Y+JG2w%DK#KC33QOIr)qaw2i6#&Y%vbYPR9QRrCDS0iT52tdOe6RC!g_LWZZ~_m3nV!sy=Y9NV}C61Uj9<`twGD2r6TPek?Aa?Pk z{S0}>cir1w{pXzn$m6}>4g^TGh@oeHZ!RpMg?wei`t5jVEtCCOT=f>4cjJ=J3$zLL zt`GBTew(uZpr(@&ZoE~Q1;aJ(WmL~YFD*l{Md0&9E*}=H!Ltk>RE58zGb8CeVZ&NpZDeoj?+8t3NTUR0&#NHY?L9VM(f}0XohZyc%={HmSf(XYeyQb zt@M%Azt)mV8%ui43AR%F=;52%X6I0Yg8uLJQ<^-nXDn;U_hKCl=aYG|>M|Zm#g%SuEzl zpZlj`0DSM;;r|vT_2akO@OPB+)s*B?lckBbikU=dlW`xtkLPI$&%whm-krN*yJnQy zCog(SU+UmygHsEo8IH`FA5rx=_$2ZLb)~)3Kcf>*A?8wtOniE&^bdt0~4wHkVHDRuviI70G1EpmFyy)!c0RP2XTSM2X}X3n|h z%20Y%9a#6r6z z%wt_6`J}7flhoHoIS~{k(#L2abST5j9;=8@E}mlLa)OG{i*CU{w&##PhIbk9;)rKH zfvZqpu`gV7z7`+Rv<5x32lihjzxydFSS0q&y+124Mv@@!l6qT`;b|{AIIa9???X;N z9Qni6dS!0F-p#y0jW>4ZX=#nQe};j|u`q!4Su6Kt)CR#H3iWV+$k3d2z6jGf_fJtj zTbV^0?+Xv1zOnAYYlIIDS|k2CllAC#n?ch^UOP?d)h^Vh#u`)y^31;65xW9$)=D=G zQ+Ei^Bx*T(7I}dU_D@zhYpMy%T^Al4E))>?Zer65@2sqA5!P}(xjUO>DY`m6XAd^iYx_C^cu zc4x9Dv}M;L8LNe)ITSl9OK1&;8Y=h^zAS??o%WKO`~@z*cJmGY{4Lre25DO z&|~R+U9XKS;7l^t>+pJJuROMg!Xrc;r2wq=?`R7-1bBi6pH9@({0yMQRzJ>oenc;? zxq_0F#%P^jv>s0h_Ee9$f9g#jk0Om{RC9YGyC&WkeUM0UvH!D&K`0J|Rh1ezK;Qp? zu*5qDxB9pq(3gb~$&5?UCU`9DVQQbQEi_6#;e>zd$*it}>$&GfWZM&#&@iT!Z5e9v7Gl(D%CtQt_(_NCgGR@LRILe~se?wCB(qPpK~ z`M7U4ZUDWZ$h<2$F2yfqBh{|!{r%>iHCF|TOp3xAh*k{Rv)EY&A^&4%fsQJH5m2nX z$NILKb4(&_Rq3VuwUEz9C!azx#2=C`!n)+6k(+A3gB{TBK7h!xwN0~LN} zA0#|t7Zy=AtLb>p@jzzuUUg|EBQ4lg1b?5TJBLkl3h>`)*3}lf?4_80d4;Cq7MHmOv2YH#+neKh z_R{ws>sSmr@6hYhp_E%`8^;^-Op_+4&rYTSAouXp`A@bSz?(A!!DukM{l{e}s^B6n z1wa)@R#EEB4iC7qqzXFYR$Wu$%XBugsVH3*yG#20F&-16LhzuPnTzAzQNo zwu=}SvJMEOI3LkKOQT(2WWA5`T!a8#-Ge0DOl3ST>Ljb4PgP@z@YS_!JPh)*FzI|4 zC^up=fu$!}_Rf!QCqm)sTKkhVyjz|-40Wl4FlXWuJMB~C=I!PYcF$Ifpx4HHG2gmr z0lf{>ix3-K;COQskXU1ZoNu`{9Be=Gvk;OjXx55o*|?6r0?a_C7tv@*EC1+YG=rPLP9UM_0vtU!kdXsTMJ{ z&1qaiKAK%BFU&8+=7C52$N%ZC7#N9)7;0-wup_SUQ#mFS*1Jm|vI1lO=^6+jDsh2j zgqqXrtZq*8(u%&{(Xy)>Q4v6pvld;$QIdbR3T zkkOcw1g+N${MIzkxs3A$phI@?AeRdoA$gmWG9J~@a4bJh)~v{K=)2;xZrdo^c0*x{zj_~ z_|R(qdKnuDRSMhN)TXHwe`V)quwBp;@L`UI|59E{6?*~^yY$%*hvmcnSH8(U*kB@a z`D;#5RN|Lu3#SwX<{6cX?RV|H=SCQ`Q!7et*nPRby-3HeqgPvyTNX-1Ai{Y5uj|uG zX!g!+7{&_Wz45po-uTtJ>3;qOy2V4O;4-7^VH^`n*gb(wl3#&JLeD**h57P6O--wN zF5+u6>hiso5U-l{({H*m>Yu#w^}&6uzbf{BLf4}7#wXJ)60CU*G~~{HaGCM0@?}ZYmyB!twD320va4Rte>|i=ekVI3z#v1_k=zUuy~W+U&f&Z_ zv+0;mb0$Qzmsv~@)VB}4#m`mwjZI0CKyk5__;PVcD-6vC*E>2c|PHb*CncoE5 z7mqfp8k0asK6-H+MvAa!Eq=deaE-V75l&?ZOU6Ee?~E!%Q!QdKrR*!o9&^_}`(FpF z4j>C*!8IeGp|_UO_o3t?vBV3?=P?|8Fy;txA z#q@VeFQ+L0He1hbopIavpsgmIc^RfkQy6_8^yZncir62g}rVuF}slB)?l3i7Tu43AlFSA1H?qoH*G z3Ipo|7Eu7b$kPi0kcsR7{HYhB@YO9BJRE;^)};4)T55YwgKVGF__#6auW^0wQZhkX zG>xlQFZ0()Q2dm0R8fKpzft8{m>;oXe1REQnF}4V8#yz(7pLrn`D9O(Ta&L0I=hY- z8}Xd;`Kxt#wku)Op%{Oe>lfh?oirUGSF*XFr+C1x1i;lEL{HyN?`POE?trR4?lbha zM*%^duz104@@LtniOY&5YJNY;zU>^|a&6R}LU~|^VL`Tps!<|;M3o)@m)rw2@qpYN z*yq3LTSD+*qf^+l{wPqhC-=1%rJrxe|K6d^1vpJxC=*|=_B9`MS0%jWP-}t*M|yf~ ze7QwbGZxDV9KavX9EazJ?^phWEWW4|8peU68P5;j27q~UF?HhAPHj2_bg{#YAGL@bK*u< zcrcI40dk52p2=}`d`Ibn|3}k#__O(jeY(K? zt)HfZ_%&m?@zTQ-p3V9;GuXD1@$RI=2F=X9hGd91$jns#x*z8B>;75AWh&d+n3j40 z(l8=E(D{ls&|k{rAjV3#gI)^9)gZgB%}lrq9&+k3)m*va=!h(DIS{gh^d;uaW!jUT zBG=ex$jx+>{Xxb>TROEWNO@}h(8HmLT@*P4jc6OHq)}>}H|b9jC^i3T*~#Bnk3Q@%h#h0mmbNSXuGz(edsw_|9|E}S2o*fO>^cubYpq^Z?rM4l!0|& zE`Ks2|5sejvbW+4KRW^-{wQdBG?DltSTDL4WLr)uV#lADiuf>pn2!$4V3PqjJqXOW z-UpVzeuv69b6fq$AOy-&p>RJi(V+H)(>(#_#;MmXd6;OXZF8sjG4NbC56*g!18Aw5 zMt$q1Sh2iNg6e8b!ol=2T_ix*fp1(LcyV<5o4z_TWrD`>BP;t+>up-fWNTxo&Rj%w z21$=zx7+oArB_24%FIp|Z&d{c8YLgl3&%E5E?sHh$Ab6tkq1lcZ+i}ugE{o{2zjFo z|2fu1TYPEs=qY=W+{kN1$@LtTzafUtFk2B>GMC(*N%^^~QrWiUa%Q*lc9PVCmRM2K zg(x758%ai8Nh2^h71%nmc%`1)pqzYIE4S&0uc+uv<+qkQ%~W516?&k^`UPPcy!}Of z7!963LOpab{=m5VMhPr3%f?}V_E%mGqfs0Sj=Br7-TNo%N}<*G+bhmsGA32q&pwlv z02_P-0xj-|j8S-=uG5@sGmOCp$6cy67l-w)$vzY_Qj$*kQtBLE=BcznTN%T%H&q!` zl;ZTip~oEVr5~t#nncJJ1rNvPu|kuGTVXcaw1x##fyB zGZp{w`rLOi0$`F?LAnWSEzL=0PzW9`68p6;s=#c8X{D&$a zEky#sY|HoDZ{%t1C{UNkyv^|6Lw42=NBRpzOh1^g%9duF@6kV(yObf6?AD^e%i9FC zw)ZyXI1qP%crJWgaAa-O1<77;t@@oZ>(B#vetJTykMq7qt5VP z%mY5;BS;SERNH*zme(rp-9Pu!-nOW69?OXn9Dkc~?(1JHis8>G1J?aKrZS=*`CKnX z*F3};q~O3Fa?i>`Tb8|nPbJ@)lH|pc?u+0R46P)OGtWON)(&aT((u?OXP(nis9Lc4 z1|;)GU%69kPfvN7?G6yqBqg}7=jX6_2rIJoFx}KBP^qaN{=o@TO%BLT?jkVIa zDornrBx0Ag!mLtq`UgV^g^vhyE8F^dK0=jTmwSf>T_eOWL3SB(J&kx-<+yDgaQwcS z!UPBEZ3}C0;_`X0Z|O#U3s_K($-ue3Fut)+cek%$OY>CHFW~g&=FcCfEb7c_eWQ)4 zF(jtHv1)*+IHFv{e7bpF`@oVtOZ${6?fD$xE++`WmHA@-w~xtg<|WBbO;E>=2iM}0TI1&A?IJ-ZcUy~D zc=51L`|QYb=Jsq@&!mJ~yw57cTt+4fzak)*;TYs*GWRzt8Y?IX7972IG@EOzP^E3H zP^^vJ^}oAG(|hnl0Q09o0N-HX7F-b&bJve5_&)P$PHf&%cK1aFl^TP%#dQ|q%y$bN zYH3Q{qEmd(w`RlD;%&~#31GdJvBx9y;iiX%v8HE1%H$ofj^Vmh`Zqcj$wDHP*=NHG zYCAVCZ_5gtX=!F8L{zJr5KpCjJn*sGR3N*8X^Q*((LDJ0zC)lVF{#$X)Pa&r?X0GG z6@14D`Ldr7>ED?kM^CId9K?8`Xvuib59G+!j~MDpujP|Xg%Sf|h7C(P4}HG)bA2+& z{Vi=6keQO7_?8Q4+8r*;Z1-PufHGgNW=?I+7MLG!!|;LUdl?8JDeN|9Eavd zHH3-%0yabBc4C+zL}<@E`TESmM9sEsdbbSBQ2iRW6qdJ&Zk3b?`QcUj+Ik|P@tg+! z$+XZ(CC#0hJ&78d@^rN`ork*)fK}13uGY5LMz?RPoF#F zCcpr}2=XWgJNfz$H3xu%=ThtJUs_ek)H~FltJ@>2and8lpOb?doSlf%s@KY{n@}R3H}2PG<-0u@`X67kh+N-E6V*+DS6F_$C_A zI|JSL<4~%49?4Y+3Qg9ylG3l7DftOqQbnzwoxs=o3@G_6XU7V_jf6(Ti zB1DkyxZHVZo$^o0IiYuxC151IVczrIyuEsN0v@nnjALc~ajaj!bP-vrqHY=+hdH#X zl|X*-7l_M%e0;5tOoQgI1>2=|z?n;=l z@W|DjIO*r{odOuXLtYPaBXTi)+@#Hn6!gS{j_^}~5Y!)Hh}-8kx=qprf2=FJ67MY`aH3a~m(bXZ#7OQq| zEH27qW>9+l%2=I$#pJ2{j=k1?H~k&{KGFa|qiw{w?5_HI2arU0uiQK7X6n)I_ zZbK4oa#+Uymvr13VsB;sAI^TK615a^cQZpol~NIFxKRP-F*$wO7G*+0n4gZECu!aF z>i5kpsRW~Z2Q#dbhB4>mpN8!feU8WEr~V47uD+^0*y@`>@kUGQFE9{=J@ zlk`X=(CgVd;9+fdhVGjEZJY=Fu>4BWZ~uy5yW$7c^Jw;80PC{)0boPWt%`tkxF*-@ z(Ei(zya*4@$Q_kQ^Wz3+YDpqu5Ww~82RhmJx%t8_x6$>3UnguAm{%PxRjh@wv!VisL1$Y6j)oW0Mc;}ji4~#*SBL&+z99iUl)$OOQ;rRN zk~Q=p3+FGLtb|Jn%tM2uVg;*hBjE0b5!{C6%z**B2R~K0M+sp+w*I@X_ZBsrgz>p5 zYu&o)yQ1VAu+cQ(Qzp>@1lux9sjpXP35Er9tM;F(9&{-yv^Qb}dte7@Q_qa@e@joO z_J)_Xb!$Q#l}T#CqVzK?>(!#zOEJ>`+2o1Q(8C7=|JEUazux)h8$}-N@++A|(^p}m z3t1&XcjQ20S29J40j|w?M3#xUtB!(23#9lVFL>JJ%@=U9WUCA+d2vc$!`p-^>ZZz? z-a8qWT5316)j20hT`V>qS2=#ex%j`z(~dqQ{1nUXxZiomm7VAPG9>4icDxpO<*r{q zVD!h%$8k3~VViz$5GMj#)0kIi1fx;3Fy6+0^-^synpy*lY&gzoHI#P1^i8*1lvW>C z@DGut3<$T7CkL}FU&lj6;IWW#iPdfA)q-cX^b<>`@}}$_uWi$wsC#bkCzTDKgmV$U zH>Bi1tN9oF!i4F=%6gkHZqVW5TjO5n9I}qdtAMCwbDNFVHW&II{Jp!=!Mb!S)aQyj z=<6Ot$$@Z+d~I|d4bL73dF?bNuW`;xP&8a2D~(CGOdngNMqBlCkFpR>Ol@s^F1=wHQCMRYP`h3>`z4ueZ6%f(_dyN`YUAdDeD_k@s?uHnhyivv5IkJ+eXI0 z`sMC1>FzrX)sCidyR7TQsO3D6#b3Ry^i_EVL(8<$5=_^#%9KU$03lZXu9PW!4t4Gt zosBThqaG{|6Is|<$q`JJ&rRh>I%)C`B&Cwqijk??!(ukm+NQK0T#aemurM|>YwCf* zmcvGf)(?Pg3NdJZ4Dw&00_;da&K%00S_vHWLSpLIUd6z>zYu$RvRaR*hJm={l7$}a zO#vVC22v(pBqjh`C)Oe2xz76IVn==Q)t~;yt35VpJ(YLHr@ZnA1J(Pp_%MbQvy58Ljr0^yQ+tTMvlM?95JD8gq*L z4kC)r5JbAgHqfnbC)idPtETD2+yfs)6(f%qAcMt_isviObXwLmkRw3XruwPw^N;j$ zGMVeSig4w77yf%!xinX7LzLh-qWCYqT&(cKY8~q|>JmriwQV0Ut0roHu?cJr!^r3X_G% z{pdW5r0sckBdE5~;qM0BWwF=2+>#WtZ{BYj0N*p-Vs1*=nrI?5l;k^*mH+r%Na+>v za;AZmnJ`YH`KV@qWuSbNPVM#yK;fTWS|)lueu9p2MFAt!0-CN7&Zxio3^hQeogm%V^|EYv7kAp3;p?GMg0*G6^4aVUdrxEE@GhR{$gWkz$OJLWDpGV zMTZj0VxK7KrK;;l!Y{4G%RH2-tpWy2=PBcrH(NfyKCo9Zr%s+bFqG)P1jgvD_!c%x zCfg{a)>ie`gi4cCxQfY=F=}gHNRlYfU+iY4+m0qrjyA~bcWwLmG$=;wb=bipDnhY| znLC60nxGY+m~0m*P3f5D^%F!189$W5iXH=^iV!lL5>NV+|bqHQ72w(QL_(r z$*V2O7+q5UMQwAk{_Q*u{N6(dZ)FKN<1XsE2k?tuxV<8AzQ~Xt>o)5Uk_i`!Ki|8t z5Q_)7s-X(Dj2go|lj57JGsr2{4DU$k=^vv1a8|DY4aTVNyCq-||?1 z3oSfZ)a&_EhcoU{OoUvk@=RfDU>E)q_Wg~!8}_aRJ{9Q6R%`vQ*b}< zEbm3i{_fy;p_~+-RE&DXzO3Vese6&s>Fob~RucE2`#0phXPIo(*O=1|hRt)UEUqne zNeutSX?23v&4bxs?!&UYIvPXw?ZBO8CIU%!^<_*Sz`{LoT*8ft>oO2>8-t_v@jlfD zaiMUqo^*PWI;(iRYuib_ceAp|+fLGb$HRqx!g-k8VvIlH9f^KGiQj~xK7TL2nSQ{kCGDi4+S zFeOK8lNWj3d0B!mKOvV^SVp zQQKM}IcgIQlGCSYJbKQ}XLEtA`xeGBRvhM8du8Iehj(VG6;<~KSuHG$I8v9(%^nl* zv|7|-?~=h`qjtB{U6UiWMDi(|px%nrL+9-qc(E`lwI%E4uhG0$yz~vB6*m9SH|p0K zZ1&REfD#W%0+FS5NwXzUs2HMfBM=JUEr(onn1nX`vT4L zv|FR|6dcv#(|4fM|9hLRjg? z1^Hy)8&xKu$ve^tS2ZUnLY8I@daM!PV=hR|*za#i&9VdDE$>fns3n*+A~zr@Vz^kh zo>Rdr{S~+D$i%(ooFV@J2_vhsCX10*IWMliJ1mF3Ha(U$ zl|<9a@0f5@^=(-(cna^ATX~ZNW0%sdzkYuIC*3e& z$Hcc8M4L|>_#@=l4*|LadwlBpRB@L&i+;;J^A832I?>>MZIuT;K_3{Cr*m?va){N| zr!WzBX9Y?eM89up&&BcJ1pg=OzMoKQe7lK&LRz{*uEkktZ&QDNfbD?pK#6_-T>{;| zPKz(AH7;}_V3kdR|K?%SzX1QgHt`P!4$!F;#G{m!l-BCuu%tpK^o`)C2p^3mqSb(% z#&i^ePcD}pJTBNFf)mN(q6K4?1LxX{!TEty^;H3-;HdZ5E$F*ncs@BFW`KL+R%{(y zF0N`aG#RBvWFLb5iYW)5xj6oG1PR68mLUj%nsfPa&{RFCck| zgNpLGM;(JKFqvl0pVQXVzp6dyd$XeiquA^0o+tgUJ?*y?=~`2G9Z z2uek)trUG^w#j4X%h2Nvs3(cCKu*yPsETbR9Pxq9l!{=Jr(mV#y=5rEcNa-^!~bZ1 zvU8P>)mLR&)P}kVymq(L+Moh6Tg(gU!46?Dw^v*sQwXWqyqLnk&3Fh^Zo4Re~I)&GJ z^C|k;QG$?B4818(-$IA|^B5J06b|DFrmZiZX}rXPjHn7D}_K0FaLUK|QLZ(bAuzWvk3uOlCX>=sxu z*j2u;28q#CAFeUD8%sO(sG?nB3Ot$`mcn8;^J3g4YQoM9=p9f@%D}nsmQstWj;i`3 z)`PE*p2d!<%V>JBt`u02&+p3y_4-8Qug8npr$Zo4R{hb*weTxf4q{e2-|Is>F3uRn zB9?QXf1i0J%0k&IIa2z@J>Bzw76C?{Zan3?9+#v(W(`%=`Hlx~wC(jXa{Tics%s;~-u&n4skuiRW$7 zF+J@srgG|R4dFg2yjZ}u)}NJhqq`BVTzDXcbfhmoE?=AMJgA}EFv%Iqgni!4laj^B z(iy84sT0PWVf7jr~W>nSoNa(U8d8yt2={T0#dYQOxPg2g7+>Fx!X=K?O7S3ve ztT=7hb4*-=CE-EW-Jb+G;@Y$XojKe&M3&PW%hk0;ok(9r?Vg<4F;LqmiA@xqTYjY& zmrE(yGsN@OweK#P27ysDr)8`a9g9C;JEDpt5kp zMxMqD_)*D0xBMXGnqgVzx@?*KaCnQL--Gk?(8c@x9vpJeTh-Sa$tmRTj2!NLtPe@) zNSB~%tiKclY6H-0@A$#aRqD1Xx*l!1@GuIG@P0?T*xr@54u}5+mOm~VWOE6IGBNqZ z>TMl1JFFSFI_(|&NA9udGbct8BU;^!NSB_!$R+<(e>{BhjmzckiEA@s_0aNVO?pt- ze(w92o3{Jq9mtw6q)6NR6MtbPUHF>uTA%Tj3mXF=H*AMxjf3RsbsNvg&`v|CHB{u z1mRZOYAEK7<*89(thTjvMb=_o#Q7dtOrRxAHzdqtzGXpm)r7A=R|Ah*X5 z*uRHl0?K3^yeT-;k(#TXqxumXPtcnRGl5Z?$?N*bJUR`8IX|ThmV^rPb!faRuU7UFU$$uwY@Y7deqm zv~vX^a!W7)7Gt_o+JG_)>4p+2ip7ek0GAeTuYGd5rCv*Dm2s?}51xiu<|uwD(GW_x z1fMDt&ox~cfthjLc@6GS)SVnF0z@QY$90U_r4to&E?aFJI%xWAJ_Sg)_>YNk()|e`{f}_zwY9GKHHUfk=(*ibd$`FFJ9Kq@rjtGT-%tus^9}HOUz% z1YG0KE5pXGH7#;V5t2_#tx3K|W09^`aMXV_=NvQbUOAbdLz7QiauPUsT?n?`$*i+> z+PYUj(JN<&UeBG{6EvnJQB#rC!|0-!k~g1XBd?uRrboh_^(X5V%U!TNl{n4_+;dsz zg;tjxpabyRb@l`B$<|O9k4Go$2PtWK7PUD$GmhRfgDX&Ul1|aVc5! z1%!R;{cRrEx`@w|Y4@S~qb+W3LN?wW!Syqg_4y-Syny!(ooknFD`ILh6AfgO-CF2J zne3r%N!9dz*Z2z(gtfwd_7P6Jdw%a?1TOFaL)3kF{NBi7&_&H&~A z`(U2syA+qGtk-AhDZhgPp;q@qd+mjVdt$t+kUV6|IL-Om%%&nnvRgwkruTtt;k*QE z4|!I3y{e+&Hqy$bJ4!-yKbwBm(j}V3L?1$!HCdG3w0Q$jPq&x&DwN^a@a#}j2sv>5 z%@N7C`Z|2!j}?&*A6IkJO$l=)M;yz>YE`+zJkovHw>m>tU^{>BEWh7n6r>l!J{qy5 z`kPoa+iC|T+hygHy8Uf51V@CpWO)QP!&L{RgU_p&aQ@Z-gShgo@%Wd>aV5ufseW17 z8MnbWJu36rgYzg-^1M-eMgG$T^~3YkHuPNmVqLzTIwU&ZjLn)S>%$iuqUWOjoW8f7 zK3q7$>MWQv|NAO3&-s66Lwi>~QsMYU=j{RBBr0!KpD>JH!XeH8NCX*lI*rJt&8FOJ z!yQ2{&Tr3G8R+3?p$cUBppyAMvcJj~p>@b6duJAPCtmA!!TrC^uwr#R1+S}%CazLy&&>@~($d^SFEhRIO zmg(0q8Q9P$j1cLtYkZq?Ic&8GflH8f$dD*7y|HZJa6ofMzP_Ol$`XDeMi^7iAXd%n z=kQ%@S+b_IAaXbzXMeaH2Gp%xD_-nQreDbqCMt##3$6p7l5{gT?3@xC%!#?zWqMW` zRMu2Z^3s3Jax*|41T>u3ZuH1C+>o$z%5Ta+hJ>%~0Ve5YcN#$Bfadb@zY|h! zpvjXjU6!*abXqbv;g4#Z>8)t4G%l4-|KY}xX+K}D{@rn#EN80j{5S5X(o4{{$8 z$Bs}}HUZvgN|@rjSo>Mowx?+CS8+DR8qpc;g!4deLhh?mgl-td!@F)K?(4boFVbh7 z{%B*qy+~ik4_-hy+UKm^-w|u~OZOixEJu@`d#&rTym#qZsnLqpN-~ydGBYZ2QU$JY;hCd! zC^?yO8A0uPOA*ZVvB8!rG>+oab>RH&ubwC0B+98Pn9>O>i5rpUm(T)Ot%m6$yO3r- zVR8zg<$OXm;ReYJ8cs0g>nQ>m7GY8>?66OX^t!uedJ9oPtJD?zO2?TriNBWU#5ZiL zB*-dFE9b~;EwB%hUd)_18c{u0!Ho;ib1*nFL-`GSmVCT)_gJIuw$Om3)Ls>6bz31 z2}?ItJ>g}v_bD4!04e4pyE53WlQo6omNG3A1!meBoz_wwHV|j2gQfkYFEP!}dN@UCCGSu-#&~s;Qhrn`lI+3F0wmu+}}?7B2K~d zhN0C-h(lX-l$4-Dfz1)az!>Ya$N{e(;*$1!i;}YQ?&HYjp-$6y%iKBJ5s6Q&=3y{_ z7g;M6RZrh1r*wkuo_f?HQ4#}>zhjCjpKpfM?!2Y?$cYWRKQ;_QH$jXvPX_(}rikrF zNSS|hs5+KW#yZ-W2zaWF!Uz2zjq%EtpIje*B`VO1*0}LW1N{c@>h#lpjCZXW z@4c@`lx3T_vC+lLC;XUYEZ;~XqFeZ?W18;cD}3BJ@xZj~`=eT?DtCTed0zs&jXh!v zh2Y>AxoAh+Mbi!ei@PZCg6_&ik8FhEV7r8}n#so2iJA#hzwMgi7LYl6BbRDSGz1!c z^UyK&ro%smjWmoh#irKFG)4@2D&%GSIM(v7OYJds=XIZNY^m7b!jFtFa#RBHCO;=V zo5kU`ZNlDF3v1KOpyY@SL#;+e?A*Iv{$aEUlWgt6kY&I%hv&4wzO^s^%iHt82B7opL}0wlNMN~qukcm66|X6OnBYT^nC^;aQxoD zr=DhN$unA;DLWQ*>^7*V53zjRWP?M`Gi{n2vegl%rE8-ko)Ob*#8Zx2kN+IrGbBDe zZKs!<+9Cc{XI!D5$_HyKZOgmouzUm&D@JtHCtnx#raRMuu=Rn_72|m5llFSNsk5-wR$T+h}pFyEBf!%Wp8K^7b@RI zH-)6pQb=uB%WFKc@dmZg!80xQ>M+%R4-w?c#kE&}b(`*=Mu1-G}{UxPM=6UGq!k%Zn+`?MsPg#X~ z6#-CU!!Z#OUb}5c=NG#QHT`X^@u@M9T7J@`^k?9`i<`QAq6H=~TsFfJj-W38DFMk_ zC4V5PQ$?S;6m|`F}C4r{L7d>PCZQHMgd)bR`P*D38Lx z8m38Wq5&I{sz_d)X2iupJ)e16+(zn-I^He~xK+3jdCNDn?C~9c;4eE7{lw`&x#f4h=AwA~z~DhqX^STcS>{$enGy{10JXzU*jhDU#2SlT!* zLJ&oMLot55_K%%NbkD~hnOgUY{!X;y8j{A=g;;E*Xwt!dLynR8(Iy~?ho-YepUPC% zFFineo4O{p#3|w=8#hw2cexVgQr5%Dx4q}c@7p{t)^#TGqqtPs=A{?-cBkz2w60q~ zKO)%l2xW#OMWl^kzw@3=R(lnCS)PkSivDw$B=IWgzP8+f5;P>j=EEF=E8-z=9`=p? zq%C&;8G0JxFeJcbvupvKEvzD6@j04mGl``NgnBec+>K6kR7`#~;cuQ6-p$wpJI=5G zalYifYZ+qcC=3>?bAqYs#%w3kbE=+vGGd&07K@vM_>gn9?~|p-nIeiIU6V?M>z}gU zfEtl-qAvj5@*8~0Z+HQPml}Cb$^;zt&Mh%IpEoVI`HU`{?s&L|sW06`b})>=8e?`o zg(;l*FW+6bRdUAV^hdy$YCsA&IxxBT6OcgIZmVy`quRC)_&^tODd5wi4A#yJ?@vDH zI{5Nx-^CbIky?)4b|xvMIFm_}CJ>3q)z~-96f|P%fJ!?X#`AW331;9JS`Aiv#zGMH z_SB6PXML31a))4HQ-{)-`P>}zsUwdmTrGuT@LzenqLbwlYz{UL$eK$I)?&{=nLPA_ zIO@*t)6@>jhhG7@TN@#i4JTy)>#nDsTZDt0PPLx|k~(5ioxD;2*bA7vPPAN;1IeNsi~hN z6VCBgrkm0`lZg87FJVpk>3lSkjL4ckZmdHKuD2QX-OVX90l9gGekq>8>!9)Eu%?p} zpn6SVR`7MGxS3x+9aehVh>DB`x-fbo`l9)D&l!o)iZ62ruM6QHOp~NrZ}{|fYBGkU zoPC=FkndJUPhcjCaKAXGG`M^>u$j_mmx}BRuoXNlAm+dWGZo7nKR?{_BZ2dRvBIjP3NCe$M z#y|ia%dw_{RZaq{?}5L8o*BE{r|dnQ1CLkxjbzF|#RA>8GY-e3NtXhB-^lGmpy1}S zWK0tfHzyBlebDt4%r<38zOoQ=J{>MyJ}k2=nLmG~;4sj@HS(_OVL>yx+u@UoIEqwg zci79Jxjemp@1$E1XN;DJL!Jn$D?Ml|!Oox+s>P7absKT?%KogTy z=5fXfzjYVW2xyb=wjBkb=u#ET9)W8CcM{*WVhgx5fJ3V}l#haWvrHartT>2eb%Os) zmE(HRc|Sc56tFW@jq8)rdN!E%%@G;`aos*jkN=LJJqHwk<2MIF$R`!X^@vwCw{KcI#%0Rve1_PYo; z$&?_T4lyFa*FU3{-Fj-~cUakgBT3G|*C{8D{nr^we{#rM-%M!qnda|`R2XI26iJFx z10%@4t0ZK%R8CsY!r~O-=+NJAocm8EkFJw6ZP1J#4Eo`J*RNMR>}_L(Gel=v<+?y} zl=uJzuB~hhZ2FRx=_&T!LH7}qF`L2b7*_8NM7DQ~>Y&Wa#9#DtJ2ck_cIxcc4d?~k z=5hy#NoR@ifLIsc3j{C~RT-ghns{52PBrsEw=lT!lQ?!V6Wbh)(G=~H`KRun4!J&* z^02qDCAx|j9CL+Rhn!ZI+3CMW&z>VhQ%aa4$a#1{I7@*utOSQ=QK<4<&brb$r^ zePWHaZVX@>@PWfQ&jg?hWFA{iXx)}j#yIKdf-hdNrn{t7(xBtBPH(?>)z3MJ5FDTM z`!OV;9(uGS{d=%^u`&s-Nq&(5UzD%ppY%>-!gFJ{C2ZMkcllzI^qCCW`YxSv$^3Tf z^c|AFs(DzP7eBnswJ8=x=Qr&hU76$TUSSebkOJ8C%fRQy?%3$2TvBDl!q2|~jL9+ZTD5*TDdIl>{#ltzNo**T zFl5Z9YTmS|{ZmoiBqWD!+VD8E-ia3VtE7z^tM8v0B9Qpx9=h1J*3t8vNp`LvO6KK+=>=hKI^Ouc(1Uz`Gah%XNPbx_R0b4MHw? z3*duIagi={J37Bwnx+db_T$%KQgB{Y)z!a z<HLuEM~hMjOD-A##1&fip9;p{E=DM>R9jV}ZR)R2xvGg4*% z)r@0ZC;Y6%H5yqhmRGp{FrxJKdYP@RCe4l(nIiCGL9K-xz9B8ZnV|k^0n4gY+!@cy zv_qlgsn)K?8r(Oa@P)H%=RM{OpS6Uup%Ep55c*!xwc|C&;lJ5~%xBP0>v@3Qf9M{S zJDqqo&IXab_Ijd^w!%_hn}}MD z335EFR*;<2Nk&HVoeszhVNTSvb94|`o{bQq^SV|q;Eqp3mNbh<_(|j7-b_i41pVhi zRQ@`!#1h@38xsV-+MzzzKezXwKx-}NoUJ|f}l3EYtm zf~Hzz?Kyhg&zfeJpON`p7*Uitw`79ybaGu;kT{%qbC+@}W_-LcBe6!zaq|4zu{4tC zt6{hrB>mZw$0JaR<6w9#W^T|v%1B0X?-S;#1o7wjP1xBV%@ZaYZ!(lp(TBZ&LX(Xx zQsZvQugUb@qwubh&tN1GFp{;glWS)$w_}dl?74Qf0ze$D{%TutHbq#(0QdPfkD>Kv zLr8K+3hMlD*Jpoq<6?T!_&jfcY!3z@?MdpA^JjAlaG%lR6M?(54_%#s9j*1Qxir`B zv5VJoA`16%8M4-QBIyknVmSiQf1)Dm);f@sDcD2ax;D{)Ct)aQ!X$q;>ehyXZKAf` zOee4J@wRk(5sb7@ZtBfg@n|cRuGh}1k4=4~%HTJ+*r-}5D7H@Qg`?^upFJ#+sKQZ- zmOP<$pRA6a&scf&@If8#uJc3U_KUV&MLHjRllS@$Dj(hkFM$3|RrxJB;-sLQ?6S6Y zXA0Zl(l@*&H~H-#$*{Gjv%8a9eytnwMh!inG_dTQcE4Z0c$p5o`n5PVn8VIoT1d$F z?m9wTVu|0_Dcuf1?nhSn(!5BcOt_HEa$|M^mbLp!u#Q`BtIT~5+@Gs159i2%1KsG+ z33Pc~Z-f|3F9XWL@m&WC-I8$fbK0M zPY-o0XWuMhC8lYr_w6!xEGMlcN6ZDKT~RFQLtY>S9Hhnq^_;oaWboDB?1^Cs#Ky3Mn}m*TY`NC5Cl;#2QVA;#7jMJcujux zn5VnzJgH|5?R@0vw*hKpI_i!46oS0|b!WxgE+Wd6z~F8{M~Mo8b3VG-$Q3AE2LrR# z;-qtQ0zO;)t{;_Qybg1^UA+|+^5`FCzx*RjTU=U2tsV_tu|!MGKA`oEt=4ml#S7$~MQ>FEPRfvF4t2O~ z`zdIOnP@2TB^*f#-)|PYlU3%UVa*t}&M|1Hr*dY!opz?!C3?}`71;kRBHiazD2naX ze^B9!bP-zt$P@Z^Ktt17%WbR2^0ty6ek}Po`=7G=N4OI2lin{iU)shg`rs7EYI4K+3(JzpzA8O6TL4LFk!5%!&$QLbB3VyG!sJoh{&P^p57WEthKJbuV(L zW6bFfZP*z!{ayp zaZ0mC?66F(oELl8s*HnfT58Qx@mGhB>l;#{A@E1pT6QD5xn#PX%37m?J1}+oGtRRHBz^H&)=i0mN@gov`Cx9ksCEx#d%LT1KbC8~pOpsb7V7x< zvwV2*di^8qIQ>x9$(LpzeNrWPMdEC(=n{54i5|H+%1WUjUVqS^u;X}nCT<4m9j!wL zw2hUL=0wgElwnmGJaQ=CE}xN`X#~Tf)X>o;4yVs{l2_?)h&Az1x3n&-*%m_T{4gUifHY|PTHX`$?;}@ z-};lVCrBYb;RL}k*m$eN@YQuou8#^TXa|Q}sp27Vv*;mv#HEXp?RV^?#5D5f>XbJ* zAt)^tI^qi+GR-9NIi+d|I+6SwDt~6os=Ql8)6E}nGvv+D$t?Qer`adU*=yLt7|X-& z_|HThyb-dt2JjwLX@ZKL=#+cH?oak9V}eg@do|LfrO|)8L9#XdEuti4>ZS!7g*}fs z&F1;<*ecgDs)NO%q`%?TW`#K;_XKfk5--!L?e{b5qzfXBH#=+BjV+cHCUZl0OCzefm-0iRWoQb`)r>-3AYXRNI?7KQ%;+Sj#R-GV*YSu%e{Pwz( zb22dultIxs#TJ1DA+g;VY$wyhLOrLEKN60%95TxwFlt^kDv8Gop zC2f?)0R!x-`S=L>QA1@n*E>25{`p#kcy-p)%Boqb`%TPQ+`|X!0lR%&Eqh9Lkw``) z^oe|#Q1LR@RmF5t77hH+ztCny(E{z@SuTN}_WN>j;~TGUS%&Qo_=GOdns$pW1I~Bo zwp@X#Hp>xy=YACnFkYMwEB^Y7_|z}U82+m{>9U_%Gg}QFq+*W=kz|iU)(5VN%Ny?> zZ`W|mc`m;=g-on`3BNK%G0RfU1`jW^1+1<%z}jTKQ)+`4Ex(M(t@ej{{%|vu9i<1M z^Y=V2bhSaMy}Gfnbp(F-8Bi`4PLaarAzE`YJK#ywFB1X(5hm=VZN#T2rRVOZ|1Ff^ z7@JHEgU?-?d-6YX8dXkz!=y?#&Zy&insgj-&8@a~zZ1jOjljB_U0fd0RJ; z`voben{!J{UME9~I}Br>%cc}Nq@T}R5hbq=cF_^wdTL`Bk4gCvuz`cMuIQIQJFd=O zWcLBi_MzUVo4Y3)hD|naJx)s&2=w9$#3lq`b=*VjT;mV+;(P%lfA3)q>wOa@U!H%# zA-?z66_6{x7ycC3;ce5(R8je0ldBci(ld^3xz4kh(Yr2bWwBBEo;AQvzKfj<;LyS8ov0eR{`##trQ|WWd#g2ooaCgZDbqS84|DU- zcbgAJ!T*P(^A2b8ecQNEMUBvf8lgt{R;5<##Hdv*sslx$Qbn!Wf&`_awM&cIqgAaM zwTcKW6+4s)vBeH0b|QHFj`y$UIG(?see1G3v>n3?nl6u@bcF zgjcT&J`6W1^1IHu*aT(9jZP;Eb8S}obkk%15xPiK{kmZ5m#H&ogY5&>gItgbnH0O zrubLLiw;1Zq9sf?Ei--Q-jver{nPkxgYjhYrcLIa+4%c)*WrL=}H} z4=l8XQKGc>>{ZK~cyw>a>!j;gOFeZkev6rjoMxxg-OaxMj9!o0G|+ipYatROD%Hp> zMxIlj7z`;*Z!H%<^-I4u?d0R?H}fz(4Q^2kQnk-F7G(Ae@{2v-zIWMaQlDl|QSU%M z4IZ6MGCD{`T%iPDXhxCeoNzIWN=$ZtSZ62Z6G~H z_s5dzgWfnYmCFIfzBK#qPtsHaSSH{`w7R0JZvxw}WanJZK`JAD-B1MI+j)MxVAEC+ z;ry<^#dU=`d*Mt6flo2BVq1QC*ZFB2Ult`W7MCl)SNsice7yJ4m=hf z|C%^TU3%D%r0cP!+tu6FXZQq4{~-9qBBuuN&tON|`QYbcwTDDZS-WfiA2pkHrui!Y zPcU@!w_d#xQw2sqoi(tzCDL3iXDs(|BAZbbz{>Ni{&zyH^4f%k!ccU!(~*k6+yOZ= z^YeI&)`KJ?6)Lv%drVf^7bKnxq?o%*KU{~*SI~REo<>L2a=J^%x*F&Or>)5Z#bU0&9Z|2ha zao^ETxxS<K5G2gNb1A;8iKhwcQwM+dS8e?|`v-!egaj+}>A3j31Z`Mv3hf1L zhXDo!bOMK=42cmfOZ{+%o)_6~XsMf<7x2kMlSSz}q-PvW<-75yde)k}mzv#8H!LZa zSD8=SM#kC3HgxSs>2R?l)~ydL3nZD{4uQ2@mr-PIDhYxb+a^U_>F5fqK_BKLI0s7l z>YfOdJ{1k?7**=|TM+Nuk)b*0$9Y(ajiuoJa~CT2+}`qlNUwA~_HKq}eEG?A^Moi*OsV96%Ig@-0xm`d^~{&Mdh`x=!5b zjE!|G>vDYquasUO+%EpxW-1=ABa^ueL?3rgMez}%GBSKKGC-~IE$)FcVZ*0l@>$t> z7z^2@0AIUR%(p;_)E0PLA22@~G*KtIThO#5!10+?Y~DNdui5}at!0V#%slJn3Q#Jq zQ6i$(rzJVC??}J9G)AzA;kco0Eo&BH**$i_5X5fYx zZWYBMzSe;V%v0*^ud4;V9=FwQ_7c}p&TdwfKY{Wi4=qIdQ+6=_X%nx4LGe&W&osgB z0MQ!{k zt75rD9n$G#Jr^vul$WBhxlR~&KH~S+dWrV~kP=aOp)4LGwc=z{E91@-q!t>Y{d$qr zfyf{YOu;I$_q-3@X}61L?=w>L*i@jS-HJ<%&ZtTD?ClOF%S3K`FTPAwM0lhYXY1V# znrnDq{%kenXUi~BK{K(m#G$5&;MWsDrR;sV|1uAtzsaJyNiQwWHH()+F8S@Iu&k>= zok?fkb9(A6|8dz>64-VEfd{#!ZQ*qZ%C7fY&MZKhAk|fetoJ}Z6WVvT%<}J>9*?kY zRwLxxtfxktMbGS9P|4ekpy#qO?M@J(DX$~cUwBOZ62|TPdqp&Uf;&Y4hHE#x*-gOa z-#Op%r%~B*2I)jN zka)EzFf)B1W@V1<+J5PKFnP;ujCxw%djSc~K3Me|T@v~9$M2a_s%9%y)$d02-Cs)f zF>~Fegtq~$qlMlcGJn~5O%jhSZf6G_UI^+Q5OuP@gVo?|2?VlHf*?r%DAkSMa^G{| zZXwdnn`p}$fK1&@M7}d+-L@@@i3 zg_qkP)BeJN;t0D4@a$g!QzWvN)uXa>!#ZmBAu#!O;&x7B@5a+fPeF%+M7TywHaQdX z%(FU{CUa5_`c;M<2+vZ^)`1$zKg@N(gBr(S_}&Hiq+$BCL`CY@Xcqc&-HzjlaS z4ZFih{Ure}>rT!DAO0G#+{vit$=@EW`}RIaCz<_^%PK%CE)4lW^_)1B^im$)9|EqL z)gy#Gxq;tl+r)Tj_wq0wxwgb!QuIxnDB#XWS1P)wFGoqWRWs_}y;I>P6MiYj{SUUC z|I~<46Qx9V5_6UI&Qu}dK6-c9$^I@9IC@#je=A4RMDGnet&iD#63ut3nR3a$|ARZ# zk^M7ZO0P>}PooZk!Js`ww!U)zMICeBfOY|@_fqq4k5vLTqZePj>%7B%!gir0z$0%{ zi5s-I8POAfeiJj$rk)L)lm_%2=yLj)pq<_#=tx+ZMAGr^B;J1Mu%ALtTWf?HFDH4| z`Bm9hQCVRFIj3b<2**3KYThRRV9i>J#r+DvMT2-a>FJ$(A+GqNn@Y!cqxq7%To>Xl z_~P8e5_L+FG`#qigutC#3u(2d8*TGH{j9-}0hk-I0X5S$%uWVSQPlU<*x&8SD?34d z3J$asUm_zRvmYOsoW=zIf^>0i`1s~{vRC>{Mk8(t4As5n8TfBaby|o7*zN|GDSr|( z>kiJL3R->Q6w=M|_$oEezs0ip*r=hdYMe-{$^9C+jN6UEB9e+pR7Qr`>Y>5eq>$e4#K5T7qEJ*jvI4e9d|HZou z*6p-px1gkMMk4<<2(wh<65}b&2CI&M!L?w7n6WSmyR`=G}?;|cpX;LH1HWkiJ>p~sG@JOAtV^lc(pNM9#h#S zNHu=W2kB%@1S32|vp#qqwQ5qBk-H|F2=*2`%x<@8BAV-i#ZDFdN)tEzami@XE6uE7 zW@gVd?c$f)JG6Guxv?k5qA2L`jZC4uxMwrT7A%3|e%Nl_DZ~D0G2oKUtGJj7jvbwA z+84Dh3-#h*_FL7!t{?g`DhU63NfJj3V-wnWTpNn%QWfETI?-wiehK>Q=aRdEHxxm4 z$642JJDl|>!l|6P*JvRy>W_7U{KNCRH7Q@DhH$t#gKC%by@Mw#;}9Psg^>JDi74DJNDN?R9epO_z z-kR6hNgIK7?1AV3XNN?%!?W8+{Q2ii5sfkE8hORKlVOCzG7o%9`hN?tnFTjkq1dQr z`-??h*GMJm6GHv!C)WkLM`6ifc_*(=+tOEv-?b)Yd7&GpmG1PVh-;7qKLt2FlVE>b z-LeIMhrOo%J(=uLp56<`F$zwcCP(2gKiYTi1B3LFA#eITWxE(I9(<@ZjYJ+o57^Z% zGaE#hqCh2O*Ft9s@ob!Nx07rmDwki(&dwP;|J>B|I~m`qq^HqnzxgB%3$9%0iweOa z?_aCrU52`f=^{bdSu=lg>87v1QYfkEe2*l70T9LRa`;{Xn{!YoU)ZuQvz*0}#Q52n z`kW@O9o@~?MT2srI3=?2RlusZLwE8OZ!%k+WsL7m=qVt|lk_QPr=*88B4j&_E@ZdU zs^J;X!Nn}HJ>M!tAkJ;coUeS{>7XE-81F3bbmGUao|yM)-!c~Be9!?1bbTFLKM~ALd=H#o1NWW#_1;4fPtHCWg+N&|(Gb+V z9-P!MR=bat1MFRL^F?Y@h#OdQIdyZ;IISvj*4qEE+AU~7YK6=4HO~>_*N|+1c7x&V zx!AA+2q3>LDpU^VRjfN2>C$z;qAFZ{0m}y{6>Ms2lD(=X=S&NlG`VJg{uq&=94Jl* zig4hVLY4h&`xJ{Z(pi5pfrR*RgKmW`p9ryb5xD$|(=gBccM^g))UG}(RuixNG*4?|ht6gNL~9r?!zrm73!{bRo!7jR z+HzWL<7;}Vb}cDhmI~oXs`qtCXVTnRv?_)JScE(Hk>hm~4{`6TTiajT)cv(}$JxJ` z*&gPON^%WvRQaw0ZiUhpc*mcuq4_ct~_9(wO!t=hV2WCgfY zUf_usAMSt=wO=ZYT-;^}*fK3A){H*o{eYB3E`>apNrYeaXoXr&0~_QPOP z9?VB|Uz z-JQCx{gMaGrMmW7)loo}3|x`CufyxvVOdaHVJy5)+=X@L^U}?*sX2 zSo?djpUoGT2&2m)-qnSdaV`gxX({vUZdBq&_7hqmtVLIc3P1X}JM5kwEl;$tdMaMt zwQW!}8nqA{%nFZ~i0_&=P9salh0{aqt*v!@%Nkq(lo^cJmm4UCT8P1VwKGGl3M%qL zW~;v{SQNiwwvS)y=mH+Tc_&mn@qVA8XM~NJIb7Pg844J_4=CuV zz-lQBw&}kHJ?38tXNs|^!l%I(jLIN&C4cvwm505`f{N0ba;<5Qq{haH5nJQ5+ zL-5kQJK5_X(t8;=JeOC(TRTSfjyCKkGTq-vuuYWJ6`C&bqDE&~X>O%~EoGWe>yWoq zyp?JUT&k`QmEF77WG9I>e0Olp?V zC3#w%9S|dRs(y233bS`f8q;+?=%Uz76uKAGv zdGq3e7XSdm&=c8m^UxednSBEnalQzruInWFLYn^GwQH~3?jq6i|L0GxenIuk8|&lM?T^{Q9Z{jFZ=xJg`J1jJrxPgvlyRM zJ?`J_ zc2Qi*&24TX(@r!d07uR|)JAL51|=#AP8YFF2vXxg{Kmx4M};v8T~gbR@G^{J-!^VX z=fZGci~wY=_Ta1;TK1%%1TW&8$a6FZ{x9i!@z!(19`y20E~L46*?kB~h?Y3`&#z=y zO%d{Av|zfF zfBUmbgX|p$dJ%`>{<-W8W8=RDKSJ77cH5Dp5ND{9Gmr(rtnp7gQMm8gPnYb@ao~XG zHeXn~e|-UjJgvPf`LF*YsWP5XZjI^b zCY_OepLDlrb=m75fXm8KpNTlj4$Zj{pIf=dZU12GJ8hbC?$B=c$wIZTK>-KnjTdj~ zTe3iwhb*c-u&T?z6zA)njO#O}Z^Gv$xrT#&v|-K#fh;LH`VmEX z6-XCOuRoFm4OBERw5E43pzuvDh?nTeub*L&^qh@ekz7-Ir@@)I_U}g9{{ns)zT)Dw z3U)Y9S$EADZsH>iLLB2RYAwD(C%-}XiANvT>EQ(YN6gl%HLr-co1vYByKxse7v#pr z4MgMgz=z|Bfdvkyb?E#kQLWG0mXr(gDfJ1uwkFNVA^odYJORzkmH!Ygo*5h^sDgyG75F;|MX9s179v&6>^s@Shy(pk@#wN6Ab3r2M~${G%- zb+v{rMuDZ@P=#_urUrh-$uUY>xg#mohfb+H65vZ=oL6WCX@f3~V4ay-rTqSoS8Mm_ zNf@rqnQPG&L;aP19&iYM36@nDPU$G2hF|;gD{oDYfjPIMl#ofN!wo0lnf04|y8rKctm^tOE_-#)(F8yERCKJfS7sAt@*!XM5+-?AWdqSp$? z7EmAG^B~C zb)VaAQrHPz|8&QJ54V0avGq1=YtLus!07lEz_Fi_ett8daq^$o_}PVyPqaI=f9dnY zAjTS9`8le?2)Vt*g8F&c)4fmxCutNn{ISe33B9rYj-(dw1&}!zq=#w)tH_S`{B@ts z{D5J~D)dVHV-!m(QbR=2BXGI2KLenS(Vs1`w1I?xIGXoj-qR(}Q>meKA`EXT?4rKL zRtHCQ2DT*T4pZ~@ss?mq@fNohsq+%B>_y5D?g!IdE&Xj>@fDFGXtbME_i4@XRq>@+ z+E=#Os2Gr(3Lt=71VADypV&DNUwFv7#(93&i~iw7&=9-Qu&g%Yv$PfV2icIO9-WMO z7+-{EzmlvJOHkGR)o%4itn!Wu@RoKk@`FvkLCPi{l&l1L_DLjI#~wp&3GYXe_@2c2 zC{sob`be1Tby?;qv(qdUK<#1HYs3B6kw6Q$xU~Wrm2kv)4EbHWt^_1JzWAQO}|fM;iZ3 z^)C=b@a=s2X%k7w{W@K9v&RJly+oIZYdUn9khw*sBma1uAp3pOf%oGi*^OoUe0#Jw zAu}GisNbbT3p(G2Y2K@PVmky=80L&DXH6rk4MXfcWrsPqG;2H7Ph$I~on6oyx-r2! z89I~;0UP+M7~}3;>cfHtaJiBVMu#156ZOYyODj$IB(>us!rtnqdL9&Ne#FU{L?sI5 zG3Qnbjy~F5t5WpVaprLr53p63JR3|EXc6visNis8E#w(C-TdOd^NNwO)7yMqA#pHq zNTRyCH#EsJ%k^^f>DuAA-kb@3C_e@^@wdG&fI~)zoobCJ<{Nd>k;g00kj;ui6aSoB;m=Rv=f``P!GzCbY5FM*qq>sU z$pI2>E8jk^}Gtx=G%Yvg!MSVdovQr_Lg85s1q5&*CQ*4SI5OK)?JJz);zxY`@$y zqi0qFrk+hlrt@?G5!bZij?|${UBln#QW)+qNL+;A_(%gb zd%sQ6?6@&+hEj&Tsh_Qq+qyfNnnvV}{&p95YaJfsz*0l^hdAx0&DvW5 zVnQ^zD1pu)dY18Xo5k#()rKvfJmjAp+o5iH{-+^psMhi3jOJzbB9kda72@Nh`%9Y&Y!-V#I6z2|C{LkWKFQ2>0{WLY7XJ|*Ur@7e;K`GRN6QjKk zSdN}_-o)3E-+P<1_p?#&`Su-smX=Q>R;`#6bAv_$t?+S*;Av-vpqO%mwu%s5M)wV% zu;*J;phI_Xd5G9*R52l!pgXy$R}z>%;vHHRMbxN6S(%SPj4Ev@Eun zkxfTV^#u+KcIn^S-yU9C9DSc^Z27F&=Lg&AQ~|w$)jrw_Itli@-XGP^)h$Jhi}OBR z4V`XNrcJhNy?(gK#pQiG zwb6`SImCI3vT*|i+~qItrOVmh`7ywxU}7`cn=tbD7V|-}i{e>(28V!*OWb*s8!93U zc*?^4dQ`-NmDT@`r0{)RVCA?qf4#KHQ&2g$sTsuPuS?&OwFFg^ww4Qz={D;hIm8g; z#Zv-)PV6@E4jlPhPIoZMN`|PE84HA%vQdK!aFKWC!NA7XjSTx|c{KviH0seK6 zt@;(H7PM6eO5YD=4i*L7z{eF|ptL)FcDe~yiFW|+%tS3FLL8O+mfxDt<{J%$p!KHY)m+W;fT6vz7-PZe=9XyG z-=(pMXydccd#V7PlEp8q4xzy0Y110i+DtX7J*Jyn6%Lw@;Vbk@1=*4ftP(}U zKo(pRvxeuL;-Q!prZ%gh#H72Q*gDGKT{{-P`9oWg$D_?PyVT)yI41Mm3R-e+Teo6P zf4aaO7{p||#JoGIVAAm3Idz39vfUc3_=l`=6>oGAshj0uprPWys6Sx-4k;Fhju0ed zyzxG4sH|s)YdkC4dP_Qup{DbEP4k8vu&BeN7PEN?T!RgbWQiJR1SXyjAo2M* zs=bwl=K(F>(`}irs%+*%>NcZRXcQZkJvRf<_G(Yg?ySdCV1KPO*5IGz-uuia8P^4? zI~6PR5**IvbfyG0RPwHU-;MyaBd>ArGj@9XY#I6;Jh1ZW?5IhZRk!-UE5oMxz?~A* zO9DlQI$rVBMQOAyk7_pE-VfO6g1Dco)sL_J>G!&h+)bYLjBMtfE(-bvRV7fh9`@dd0;U^PpLQ5(f?MLIZMlq zMGZ9mYoc;MHauRDW-~p}L)*Xg`Gn|z06Y5z@fxoJ`OO?>F{T|3e$lzWkh??fMn zf>L|oR;K?6h67V=dNobPqz!U?ck1C=$b@;4ocbPH2Kj5qP4*&5Pw9WFk~tL+%rg6N zu{t3BjACK#gXBqk2Et%^PrbKMC%WQb1mg0c`RQkiX=vt2F2_es*05c}D+K2Au+irdniF|=iL$6dnR_#^8r$uvGgSkjp#PlAO*kg&Fkdiot&5Bm#WR z^8yDDaV1L`BBWK&QI0=oexU8zB;}g7uFa}hRdSbXa3?`HVD;l2u>z078^92IfE;Vy zQ4fkUytcjBFn{vwKF7Xqh52p94}$BR_-+VT5Woi+I8J2V3OzXwdvK>keyke2j%0#w zvy7YGUhD54ACX8)ZMW4*zE)n`S&M!mx*W3ZFVy%Lc=O|(P@eAAOo;P5^VV41BKwvW z_~2@uqr6W=XD}%;p6V9rGH|wT*LW075f;hY{aBe~!bVsB_k+f|gVea$s1EtZ`F`VD zt|nYPP;{JP9@$$)e$T5&$+9x3b>42nEw6`uQr=8o$wki5o!4xa9`z1Sf;WHeO6)@| zNm)QP7k3Z7L%?4O=LtYFYPEdifoBYVheUY2eBAC629}ZA4x?cTKjN)ncV)t_B zxE3o~$a>JHD#_!5NBNLt5AE-2DJGx_5R;_xjy|Q{zx@(Zb{O+Tj5g^~{x{yN{F*Ya z!RUq9uXFM;hlM@#9VO7N#`3O_W6$9XlBPC{FgWtWgb;d~b#m&DuZV-C9w++dxM$%At2pJS}m*4G7wb~~~k)iFcJ!Osvx){KhH3l6<%hZCg)9=vA6 z{vGeOAA(#*Y@M2Fb-8=U{m5FreT806i>G}P6p5#mMK_+@@uKcJgs)nBn-J~aTs+x_ zH)7>ZubH9k16(ZpAk6!q7t|2uxy<`NQRQq9JpVYo9nbhfz%uMk+Ll1josl5eRNrGt zDs*Y<@K_+vkJkP;iJ{K|&%;sKp0{3Q`_8@k$^=v{)uRjSVISx!?0R*f*xW?;ZnNv0 z;KZkCe|GJ|Ia5(xt(S-+KmLzU%kgV~03AylY8t6ad5)U7vk1-_y;)8g3QDq5Ti)Xb zUe5p4f@Ib_L-PVg%ZJ!#r|O>usFur`e<}GlrR4L(`mQ02?V4~h!MiyKq(t{?;gG?h zES_&groaert&h!FOp8U&*mP+e5Jz@(w3)ImI4~(R8e<68r~jSKQ%6 zm_(|!iZ1V)Z_PShR;gka`Mdh0I0cYhaeZ=N53%A?7H`RYHL~MwwYb)?IQvK4s}a?D zVoG~^td$-We%QM$GJt!=By5|7ZLDDX!U_=1yJ&y=Z&TT6*q zT&Nhpim(&rL~kF(3;BPVIu>9LU)2KHjs$e%BSljA{EP^~BsbLjU#f4!ov2ggL4OiN zI<4}_k9QXlwoD6kNCi9(m(YV{xm1l(Vq8IKA%|yY*q3B*!F?aO#Wy+{_DbEw{wP0R z4nLIbvd}45>&1=N^Ap6~OB$wR4NY zvSxM((TDkEueWIDRO!D?O{LX!3BF=AgTU0=I~IQ$$}iDcF^u?W!viG<9@cVG-{x0{ z=um+$CF;Sz<&Jnza=)s2^Z|oI?j=b?xKvl{x76mhw(Hs)vyv2jr`wQ|(^;8b9 zNM~LM$POn8#9TEyG?H;X=or=Y8sWWID0>UN^YK_6@2h?B{cDNFx9P4C`_kE%>OG!@ z#R#{9N5{HbdnXAMSFYPd^X&SmA1lNsel!!FFEKTpEy?@LlI928vv(c&B$w~4ablCF zgbOX-8TISHYV+Zc$058~>|31vb+~ea5v6ql-O0On1~yoJ`e<1rGDGQogFvQ0iWOeN za9UAdkX<>K*l_dWR0?1WL<>5qab%g(D-z>u<%i4NsDTZhRkNan@>UE|% zHh*NB;F`PA_yj${Fx_+{AXA%%qB-r%q%+B$Ps6IMpwyT0W#(-y^vk3V^nbYDUW^&@ zDd;H=^+&k2#!oFozOKub)o%1frOL@Ig~+y9Z#@@d{Wj!Ueh(6pBLxWoMl=nvvHDtj?<%pTqK-t?#sn?( zmd)SCD_|z)#}!DVZTJG%4VE7>NdZQ4P}-4l9JkRhmBij&16l6QCi5L*+p-cHP~$V_ z8^6QYJ`2wdlo{%8_J{Bb7a+=oUl(^JV+NK0XRF32HZG9swgKCK- zQSbbIdDM}S37r0f@`FBFFxWKm;%M3Q7A;DuYCQ_+KlBb?*Yx1{-Sw|T#FDRO**V{{y-Xbv#n38ba-)u`ZdRVfz@Cu z!}q|eMDESjL)#%axC;8d@fG1AptIR@R^tW_WD{eBwoQGtwfykD=e_*5R1rFy&0@7mUog_+pgV@POV z3fWiytE&^vEQ_`L4zfCy{1orp8Au9rJTwvA?-(6588VKOXYGp;t9WwWdCRGXR(Rwp zs;jeEib3>t(H9y9Ifu2jz`ldpy{nXIFcrh&S7w! zf^?2IT26;2R)y|c76+rQB3UQG}5#Fr<1Uu2)Z;i5O9s0*`Oz54r_w~+>h{TYUBvGFbbS97n;QH2=(`f z|A^o?-=0Gyh5QQf7#O%{f+`lh|HO9u(x%&mP@R#@xb)$IqKYbhG+4N1sIJr9o1I>x z2m5SrvV0$`1Oxy;A=FwyG5y%FVjJ+%?*cxQD~Kdijn!UEVknq00_uWihe%_JG?J z;$rwwuFmg2`_5*7Vt0vM8rP04xE~BIftDGOf;9&P&vUMqbd)T=LXrf(fAGiki+#v= z2xTREiZIh-byjLxR@Vu+)ujfYc~z0~oBExvsSV5A%Z43PN!aaJaMR)@^!GXjr|ono0`e4I_C74*t~5Up zJkdTgbLGo_U}|2s#=(Pry)CW9SzXfI|8`y>{mN+v_rT8-c>7c}=IJZoc+jSlGNmNh zPw6*E3b^lOFZpb;?&;pBU=$+@IIPN8~1S(^^!}?iaeFe~hMw zOG4@%Rm~#r#_ntJ%v>iFV+P!6pQxGT;- zW|EIc-b=y^GSm~<|GcDtDL3?ge`MBV*oBa;t%k{ojxEgzS6w8#bSG!EJhf7kMqOQE zaW^Br-Rw(`)JMB)rn%h0vd;5)ElGL}xYm9lVy3jBzc<(V&PA{rlm~?il$%m3d)@+2J@QIKO$Vhj5GxQ=lt(Hg$OC?kp0qaB zkk3<)$G6Z~)10YRU!|9&Zglx*BKuke~bw@IDY3olEjcc-)(a^TMtRKOvJiN^@y(#G%Bj>Zh}z8_2GkO zplr4~pTFu&GG1lAW$SM3hslKhmifoc@o?0;EYA%)nuc$--R zzBT8)13zqTf@;SlY`o7R29>DpA>SZIPAjdCgaj~ow&H2{u;!j@YUtEDRu}EG*bpFG z)03;=gcTnrh$I5J(i34xe$ZPueB@H*Oh$T%t%dJ2CVWz+>Ahb;Uq z4ss*{*7t)Npqa-%I_Y9VaH|=oE$u9BtkI~EYnx6*&J*O0%V^;CUET5E!DPyEjkM+a z2hRmlPs6!0T2FO|J~&8U(#T?wc-3D>a*!Bbucu#2Mq21A+t6>CJ69w<9rF~q@7h~I z;3xsPc}%uw%}R@5;rJ~v&A2vOq2u5_zxC-|l4kXwlq;7ey^#bKmAoV~meptK~oTC0}DE-mdT!dvC zIJ>W;?ywiyUEsxz+`0en$b&KzlmEu$MSCJ3Xp*2B>z&M^i3X%>r5%*;a@39(~>W-COG^ibqXcxz^nf9vRQ?Je-#U6eT2HHvYi<{ke@&ACcAKQ~MfS8H9amHi%hvKJB1!vJv(#ofRkwM+#i7CjZ>WSGxq+n+ zj>q0LwFylz+D;8Fp~RG`uNl+t*ofaxM;@0+R3!a07;ZWHk8Sm*7HFE@bke(UFI;wJ zVdhm(%hJ{bs9zs+k2CKDavkrmtwu$rSDYxr18HReJ1H@0C+}K(kwfs~9*S$E#C+i! zL+(;XiFTPzPc>%`#?IACl{1b$e!_ba*P#pASjTM`NVYLQqL zo9>_v8V+AVE+>s%1tiysc0@d|RN(Z?=gKRni}YL_o~+WXU%xbLu-WaMHNXxkCNK%} zq1O@ChsCR2sBZB87!nkNT*T@b`eT=p{B7N%$`>h?^HfddC5W%y+(eq2_s~Q1bSulXT51Xz_0x~u@BmFdM;eQw6l#3 zyCih@ahy`+T+;`+cD8-|2Mg?J=zmA1w*~f%hHBJa9XEvcOtd?3(=P$39y=%D%HgSR zR=#fPwFf2n>^HTZi1{U)=EtiHfyd^d)}mEQ7A@c1Zo%SDHV4g}tRX)NMxEPom8?Ac z?c*yH&#L_7sibKlNdz0d?emVVFqUYc=kq_}HiLG`0=3)C$FAx@Hw`VV#-FOKBR>z+MqLYC@sd3IcNDa>t z)PLicc|rJ_pK<#7u876GcjTg_1)&ztn#8?XW$MS?AF3aK>iZIu2G?ev4v6-lNF-`! zA89tVEmU)3r-0SLrK&nfeAecw=jZ)9c>#4TN2wLGJ_Iq?CPu-d$i>@?;_51Jeowm5 zE~(Wxw$8VCrRtM~XlSHDaDnnkqbNNZc&O27#JE5B)6DWEd-S&9P%+tHtsa89EDG3a zukJ~c_=u%&Jbd4n9x8WVI_toq-2!$cq0`04f(4bbj)uSwB*H42d4dPMfWBVo0jsD-;N&JwLQp&Jj%r^&n_81 zneXY1@VUzAlw#Ly`?XpuidUcz!FasKeB949>~;4+RgDk@p1 zId2XXK8SQw%qbzK#4v0~p>m9xQ%2;_!MVw4h)hl+G7NK`v)OEnng71O|D|2nZ5Q`` zKVQ$+<8i-#BHK6~T=whm>k8HBV|+wo9lwUES76TobfiKnKSGZXx(^fL&~cAmpn*Ha zGG1*sOk6y1l?QgSWB&K^weC(EpFpF(NX039IZ8L)q-v{qDs<~}!c1y`>-04SL-cjR zpTd!-JGvC-z!bdA+^9k}zZxA^0;1GHb7cC0-)gEXE?#Hok{ zy>?^=`)X?XRnhf+Zun$YryXDss&w?|Hw$}GwfYJAWj%wfjYjIx=eXb7)xn1%e)yF0 zr6cy2cS(88B7xUzvXelJc~ zsUkk11Pp|onn+q0;I4u~4OOVJ2f8Pzf0MA5FsDBu^5Igrs3Fk9*D~lwh#rZ}Hfy%$ zo{?<$BULxxMy5wusWY+kzbBiBGIsSuOp}3a7|0WIRBZE{;+1KbI zJBj>}zobo0wj_(kGqR!7(NJmySa6(VQxZwt~QzY_9K7&SC4_>s~my#71uAM7*i%s8rdIf zu?b4kgyc+@nv~N$TNU^um-4-n1nW5FzI!QNkWskk6HY52 zJl~pDzScA}f<#>ih_oL5WOF9*OD6R3^e!W-o#Hl#rt{vqyb`m+$5ZCpgmOX7XY~OK z7-92beeR9z?8mFABCF3PR?%_x$6RI~)TSfL${QTnZsJKW+8+EPaJu>=JEJ{tcwUK! zqu!njZyS+E=8wj;C)=5$nm1a3!|&vQe%UkZXoiC=t8@@`N$K$>5h1#Y*?@++;9C zbf!-)_0BVLPg6$t*kr+>ssyQy0*Wv)C?Z-(3|fLdJUItadKrn-Us0mjx-GxM_htIq1r+oaq{H(Tz6tq_ZEo^}gk!{Y3|W zAoah!2Ykx~H;*=eoQXuX5!Xm(j{HufuUP>$C(3V$@0Ga!+1On*g)<-U?Hr2SzRH+L zlBa^9wBhILr_cCb-}aRYgw}$`Nt~qWO5VvfY2`6%sHx%Pdll5)h!F;PCTZs^_o3YX zS%L8@PQnY_3iZ*{rV<_VnA5kWO|1;KgcyyEoB6bx@^WG!6+ zXs`~;Q)rx};ZghAj*u(ps^rx)A6TA*{Ii`c-h3%xHH8O0TPFN10LpX zY8grZr#)y~1MS%*H>aNmbOEOI-bvRTO6R<;80 zMu$6pZ%P$*xqg*;+qqUbnLIGd8{jm>8)VYpM13GAyv%&?gP>-+6o-xWGapSlQl;+7 zU$yHz)*>+k;Y<2Q1{=}WcbQ<`P{plP%8Ru{P5Aox7A!=2Yst!q4=2ls3q9}VJp(^> zalXW5wrr*Dyt`?d?Cn;mrJda?AiKaA1MT{+5`lC|=BL%4_BZuF7Yo@)3lF+cH2yESFwF|ch`Wt|CF;I5tEpB2ONGv202Y}4}vLHSijh9 zlAlxBg5>N7lD}L zn(s(&vg}k#oo~m0B6_+VYCL`m<>A4M-sni&HedT0PO1p!xR%qub03)|UnS%rMn{I~ zHZ^6{Q|b8x?d&1<^&1M$*tcq+dgq{@K`&-Y z>g#xJIH`SlyEna4qgvfXw<14(|_Q*T`i zsOJp(8L7$=iH-nbOYu9%$8>mC#QUNS0gx$GlmEP1^o0oim$`BJjYIZNisirYP089R zD(V9qza*e2XP*Yi1911aJngRt)u2oD%EgSH%V1BY|I?4c*Y)sq8pgPr`*i;TDid2&UY%(w8JbYAv!Fos-Xwf*nK6b@`#f?*<^mw)F=n>QYV4n6-Yj& z*i1InAu8!R<%qEIjz==R5A~A``DrfXa6Q@~$XE@2)F6VHI(4cRdZrL`cE{q!TXjnK z0P2BErLnw?zuqL*PrLxa0jnl|+|M~T{}ZVQS?9lnXT|4K5cHBhOeY7X?W!t2N%+h( zmL62_?lfP#h0beS<74&vW?x%5Mq9EZ<_jX?pGSCyC;yFGhgwKfnPX9M>#=DhyiXL@C=yw&>>)2JD~V`%*phs*!sC~MTwtoAoWu5$ zOFwoR7owFLwwfMDI(l_;c#p1Vz-;FPS6)bEO}lJ(41Z}4U1}`@g{BF9V?;!ctpvmv zb|7T*A_!+C*sr9A$6_u6x79@D8rE|ROA4C=ukRNUA75UJ_nV1!GW{xn@;PSaF8DSM zYa{l_bnzJS_rv!fp%vb;>JWTb`=*6FROF@f{Z$MjtN({}*7K;%&bqFj&JMcyR09RO~U!MZ86G;0G&%Ui_HIBqF14LER4uc0=#sWn~SUiwL!G1_eq<^+iBq`6& zC$3+2``zbk@3uyrh9n-!a6$n!0Fp{<&oQeU-8TG5*qLk8E88Z1X&o^?Rp|}9QSN)= zcV%(&RzM#CZJTBmYv?q`A6)t4j@SYU6s9yb0+zndj*8%zc&-Y~H^}qu=X4JRsB2&i zrOdf+5Q2RD#?-tdy!vL=-;1i~!)t3eZ^z@a7uzGKUlOeQyjuZBexlDoULaDWN?2bV z12=xG#6W{GtWddU3oq@w{SGnH;VL1mUR2J)TPuTWc*4&EO#ngC;YTKJZkdAwJOA^b z8pK>9O&O30WJjlM;lKn!0^#zuW7(yr9zQumWK8N(=0^P4l%IlT6`fVz-k0k*Eh|kJ@|uy(8+L z+`S<`Slq4fCkJi8l5P)F!d<(R1pKLnIs~FCEv>eFE^~8s$f$KVCd59i({$g6T)iQ( z+4x3U!MpGuPKK{jeI&B5rf%iXT0z2tAjA2D6unW<(fak%)2Uo|9(FktrKuV=rFL zojq9{mPFn~xi1Qvu ziyfb)L^Vj22yz43%?{(c{L5*-Tfe8Ee|O>*4>j2?^NZ;`8mY(#Fpepk6yjI^bV{9< zYbd&L!6DR`7vZxWb_hG(&P)9fIXs@Vv|#}K5#`XrOXiM)`?X()@56?g>-mId42aw zV;=0N?`@ft(t^Yk4c_6l@+SFCy>*g+tnFUgb({|V!rC`K7&YICh1_yQ6%tSTQ}YTmOMQM6 za3-|nnUB#puXfCnG|e{_{8;I*LRZ{lxhD3E!)R<=k@BggV+RQ;FuR6LO&I}`gj?V) zk2H7T&b$Ei_aKpwXM(R@@y8*Z3pt+dae{Np%gJcBnpgRi8 z34L}be<45P6|?T;szHQ^hTxfN1!g6D%}pPK7FfvJsUv}g9-M;SqecW2K;O&0iHb)^V)5`xn=Ulu-Sd z@IZ*x(enK9fcV7lhhd(6Z&II+`LHRketSNNr@prrFhpFen()Wlp}PUMa6}o8Icnma zoFAsD+#}y&QMuw;%BM7@D62J7JbAj|dopuY;JGg!Y(FnCc{$KX9?%jR+M1zppnr%G`d)S-PY0z`uU9HUq7}oc8Fn>6Nly$@T>mF>ckWN*3 zu|Kova6k3DEK!v7{)vwwUc<=A_25>*!Mn}+HmdXF*%KXs%cU1}gRbqn*y;F(xuHh-aD=KLIa@7mT~6|Cqz6@BA`D)~-N*c-S& z7*py$XUOm=%aleZ<7|fSi=N&4R1T>R$Yc%Btml+hEl%7eo-5hKy!!^2G{ zGrSS2^wS|$y&2f7s{$*PtT8m$#Gt}e{y=ew@G@6ixX-1}oqHAQG}zL7H*VEH6+ih@ zkUDJ`dGsTwlCG?$v43s#Eg9I`E8TMN#jVjG5PByv_c$k9MsB<6GU9sBtW7eM?aEHM z>A1u<=>JG2uKguFOG@1QH(N-(Yxix)#C9hfm{8VlKT5sP99T}Rw~!}-N(h34)Yw@C z8Mp|c*zrlnDDA6TQP~paMKB|zR%2GV{q2qR;$8&wlgu?l9nYr-Sm=dtt%IV3JI`>s zeU!5qDQ;3uj)LnbK2me-F-eeuo(WbMQvOw58~*HjCFbx75_VxtFttNi)`O^u*VB2s zjheWFMq$Mwu(xgf9ZJYV*{&LK(gRm@aJ|j@(x_cg!vqFLm3!y)OMC0x8!g!ce*#aC$qv7BK_gMI z6J<5Vcq3Wi$jdTRHv*EtmjzNde?+&6xo(MlD;Tg}NtlLB%h;JI?|(u%yHQsT@HIv5 z{tdWCT#v!;euq@h`I3K3u>Y6!FBIn~cOUaZ;MGd=tozVI3_}U`+9sp#NfWWYKJqWv zT&oNL;F|Gv^=<5K!$)=}-J*U%Juq*L&oye%q(DRdSvyzq$U(1<0=qj&}QjaE^Ki|V=Z2ces<*%G4xlHOV>i& z=H|rHF)e6%d2pUA2jOx-FW*qWbVa(P?dnwq?}0=3iqz}Pg0*Tw-Q(yLksbD}Z~L+9 zk+~d1B|v1uVUP8JDMQ-A^v$D z7{MXq(NQ>@2s(hF1bpCPoo;+%-6~kg`r@rOG6&oo@7&VE;dBRLoyV43UUJ>W^ly$^ zf*EQ!O9^Gra`u3>D@sV-l`|T7hkFapbb9w|sSYphk^sEN0F1m=0R}6b zfZguzeXzc1H52i=YajJ6NdF%NzAV=3pCdZ6RJU5DlT8Ck^gxW zd33)IUb8ncSbtI+2>ddg+gB-$X~_*l=ZPO9=XC#>9C70fv^EXMTkC*kJO`}G8pb^uFq};qM8z@3CT*_m-Y6QPI zAM@*enZjZ~e~Whft{>o8&g!g$>Y%Mu<6cDMuwRQ(yVJnD%!26a)Qs}5;*YDMLXpo* zmi4BkBP%ubT$Q?>5PbjfbhNhTv5U$__F7-Nt*Xl$oedTskpdJqxUd7K`LuQF!X+FL`;~F ze(oa^WP`lXD1;NW+oPS3>3g{IfE1A3_3o19Xub-H1``@$!KrPUYMb0|v zo`4(=?VaZyU~HP^unWpZi$S&xk}d%ipra3Le^NbL0Bdh0S~NtjM3oIw0??IH0Y~C5 zng8!e6`_{FL{B)#t%>4bwzbHuI@{%7A*mngg;Bc$alp<}5v0xdpn0`NP-mt*;oADl*%+{Ua#WFRem;w$*;t8F?D=bND9mVV~2k9XIh|V;9QXNK zmk*IWlUf}HhQ+J6TS~!ctoJH*(T=|1oOMWu$QaqMsQtgA)RGYYi=_~uorL|e=7FkZ z(?n4I*wyx5QqOY{Dc$|y<)B+{t8sUL*+;ppy=(U#_;AE#(vZ~tj^kJ=bW|*wuN*b~ zoOPDq)SOTN!awn`8wr`SJ#A$|m0W)X0Lc@iOHsbE-x7d-tdP9@>| z@<_=#^CmXAfYK4WsM`F52i`~>mVKCT3@@Oez5WYwlDcP`3~?KL=lyPqkOkRy=t7uF zjV8$iV}wD4$!Frt{v}$U35akh*jlUx{bf859D>XEI{O%h?;X-pR8XvNM!;3S3C1Jd zPVce31`8A3Iu=t*p+2q>)1vt`8u>DRe=`Bi+jSGH+&h==>yb}avy1*o0R8(=a;VT)h^glgqHrPFJ!Ki_ZpC2#qXi zv3S(Q=bH`t=5(bLyDQ^hLWb!f4SidOx)}b+Z&Qj_!sOx?2637b*jR&(23Aio9I|?i<&)C zv3XiD0HV+cOPf}~)f7|o_I)w`tn2ZIoR|T5$&2QjJx%*FuI-dClMDWQ6Yr=yJ<#?L z`Gh8l7U;4^^zdhYWM5tqluz&YXt@1X>ZjMv@-11f1nmZGh%*;ItoOch+y8sUp&>=2 zifYU3KXu-A{P=#gx^V9<7?&6+Dp!4ShG!$DhM$%%xkizD%(*JCHIO#I|FMk!p4e_H zbmjA`C?xfK2t}{g^6o2h=&A`bUwNXaxNA4wRL zq1e(2@68_5SYgEWhja)+-Xx92Sk16UAGuF&&+F3i+>G`r&78RDgd-gQS*ADs&PR_H zViVUBy82If=tu352HQPyUL%+OuM&Uc<@!XN7ACq;9~#C)6H^)U4tw87LtKx<=SyXA z^^Oa^_lcX7c}&;US9f&qTK>I&N6OXX`uxnzKRV=-G4fQCjyo-udxcH81_q9?hO_Qs zV}kD2_QdqgIYR}K!h-KR%I|9h@E2|irsaukkJt`2%*uRmc&doo zpg$6pe!%M?Yg{M8L|l4G%OrCV7&Zu7`Sz$4GDQA2#ct&swvkm3Ea2=3DW5xEGX^x zq=gzpM4S&sJ$?_a;xtskEODk?S<&cXB~)jt!mR_H3a8_-wZs!qD@!1pGrE^N9G5^EwA(6C28sr=nl2dx%lt4#cjX zGh+c(KBzMWfVFDP+&l%-L}bMCp3pf5mjo^!cR40FbX}38nG5isD2DBh^9*qd_YGz1 z{dg=wwtuj`qJgCNBhm3($MBm(=}gJ((-Cqn1H0g7_CuhQQ=nxglEA*K-Og&Q8*%c) zRLacYd$zUI6*vBkREJ)eo%xdj-O}xFkEg%fyNk+u7FGXVWVLVPj5WdmS1H(KC>E}K z_PM*!EfI?7H|-1PL^dALV(5qwt@dah1C-39$Jm-a3d_K8HQ@>>C;EP9Y2P4j4SeC_ zeri;n4sOD<0ivDcXU;xtedvvQ0Bq{4j}!k{d#xza$SQnMzt426ra7f6ISr$ndzV?V znSetS@!KR(+?h`$BjlJnbh%m7@sG$$hmuGiHuX&GlOIhzh-+_mWf^LGZn8`|4wQgO zxBWA-fgtK4R`E{~Xey1t*FGSS>)rQ`Am)TM{&t*i zBpQwhV4^~{-?HM6*X;VPj)i4d<}x=ZQ#6I+0NeV@45g1x+apB1CpGd_I2S|=~H zJs8=aN_p;DeK7x#Wz7DoyikrmhoT>av*#mL>RuzF5R$(aW7`MNUe(f($}vtg&E<%~ zjuIDrf@_ok(eW`CYLB{mPw}lIzS#YRjwZ~UKZ3fLjdxAfU|_{b4G`q%33$Nj$XUQt zflWPT8yqasY3i}YpVTU#7jkqtjj6|e(J4y$w{|RjZT^FCUotIa9WrZ5CSrXRp=;nL zw7lf!=CX5%h9k3b<4s2Uo;{*Nk$Zc?18-KWL>8`Z{{!8`CC5vZkoSkX?;j-9I!rRh z^#nlNd{Rs|;9%)sf4&yER{+aBS>712smL8UVcx_m-%)KW>tnXZ)m3`;=oO8%;IxlSn2qEE_4<*n2^z=s&$YYX2D zcR^dDJL}AL^@8;Yi3p}FC%TMyvb0}i$E$m?+_}~!LJ1y%yqP#EH+B7K>x~|tj>3fA zC06ynd7`s^E*frfB_@%NJNzuIIw*p8X0A1C+FyyRb$lRzar?u>B6*rC1#21enQ9Xn zT zk(OPN?s*$*;js!jO26*ip|`{vkg*J%U;vl)yL9t&+7vm1Yit9>e6vvLg6)0-<8t z1?BrkN1UsheET8bvL&-vh!c-Z@MW%lhEr*=$15AOGS2tiYy>nkFqx~ZO3&d&ba279 zC&T9BUK-G@LiOYXJ=_Gzo&-p6e?F)O#C`UOFRxkdSzKm5Xx?%#Z`vnw!2Pc-&5c2K zKDxELgs?{Q{F-kZfoh+L#tF?_Cn-&jv`Z~FfLeD>H2u%&RkCNTum*Lx7q3@fYn-l~+IGGr!5@n*3zR8ck?@!Dz008~pDB#}e z9`>zpd@$``z=Rv_gIxe6D+32SPx+4Mnn+fx}$!KMEvmD+!!GkZ^>AC zj^*4Dd&Kd^$bYAa0>dbKYPMW#D~6!Ag{=(1f>tE`2bko|I%;0lK$R-lJYm`R9`4q^Hfj+)dwB5!NBHil9gBDuQL1sk^l^&nLhXum*Bj7~1gW4vad{~nPs zl3JfU<-a!scWNEnO@VsH5yj`u<~vsG?b|_On))h-8Anmv22iQdlWlw1!kwnQC@0Cq z=hSOH&3g3XrOqhkbtr5xvstWW*^dumyi-v?3X*-;2L{8#|5*OQ)$SMi0zNi~!&l+1 z+fp#nqG)-?i55a-L`ube!_6oNi(!geFw+KCt~pFDFYv#;ntPbm>D92$k0pnhO9Rbo z%Da}eMv{M6V5W#TCZyUMWy1yY=D$UB3vNa2tU@Pm2>b$VRQ5&q)buu4kNL}Vr_gi! zWLylweJRwo!*`s+nY~?tWxYFHVZNLT79HXS zf|qS?OwFEk>jXoBKEl9H5@nvRSd>}H)M^;#q1jUzy~I};dn4C*2Ur>& zJ343uOEGzytpQUv&Bk%T>dD7F>)W?@Pc#~dA<8?hE>~Dkfeh`_ z+qnQK)26N%(+EEzZ8-(6yH4+dhDS1*OOD8xCs@U6^fvi78;N_W+_?_INoNNg{Nny8 z7e0?|Uq?F|h`kDdQDO5hLXF2w1DY2MWv8w$78h*5-X94ZBkUjB{M@CH{%OT5++VRg9jW0SVju9@eN9em` z*AX=rq4X29aQ5k6`Jcaa|8BdB3BkQ$G}d)HGL+PtoHR9r)<8dQ>YK9v6lWr@>4kJl9m8-JC}E-h-@;~Ah9|t zB6-ohXN8Xzz8a0Xm0X5(Bd=q5!d?r5Q^+ouG!aG(geJMm{K4NJ$y1J+SisPRMR)pK zqY{&Dx7AMoqU6_8gE4<<#WbcSUxT(WEK%V8D>=BMjr{yjEr3}aT(3_1Kje=;hs51= zURv)`%a$rJ1!6s@6iYB zY*-{~r&P7^zym~R-1~@b9$Nd!rg5jAKJM+mhV!^%H2<=@`%iI2O$pNVy6X0VeOquGtDy{0KhBm|$p%d1xmn}zm9`!$c?vIx*;l(yF#}hw zo-~s20x#qZx!mI&-|9PeHhU=5!_5agcN?PJb02F@@4a#N@2G>&$KN*D;o+PAz-y4& z97S@Y!lRXJqm$i>zUO1v4dn&B7wrC~m4%^Or5&{-J>z*!+!R-*}{lAsGa zl&V{%&D3YEFi5-@zJ0vc0{j8tk_D2Xy&I8azm-x8SqSZV9%b`^pn_6%R4Tw4oBnFP|tF zybr%WDGBI64Ddg{&HJ&ommGzQDc`epdZ|%~xs-P5gHrh=#t#_1BS3{UCh-#Bcat$5 zD?jE%N+EjmGg1Hr^RH@L)Us6LYJFFuJUY1g8O_nve{qgC$UnHR#Ln_C#=w*28;fD$ zHDW`%RATUdwD56uC*NS1V0x_$?S}*+nx%gGCd@bLX65Ye+Xq2TCSBV0D*3X8NhgFz z>9+w}X76lZLU~TQ=Gi8&?(CH6kMB5Mw-M&Dm$c|+%L~(^!-8^Jljdg?M)J@GvdGopBl}M~Xm&YZ( z`JKGGvFhc`-@KFQK~TxVR2qaH@2~;N9yG-!dR+C`>E^lw(aHZn>6>~m#O%;&%t9? zDM5A`zAd1?ayF#0yY~yJCkE(Ww$LD|@s3K^H>C)7Xb!LFKPOj2r3Zf6#r&3CA5`wvvkNZ~iem7MlQ8N1bB<41Y%PrHF z0Gc04+v+Q0nyF`@8gKe!itjT@8~-MlhTiY099pcBp(t^tMuDeDBwrMx0w5udlvu)V*ckLR~%u`x8}j4RW7=jBZ?!gZ$1I zCrqsL*w}#kvbz-z+=ANI_>O++mwKava)~zXg=U_wZz+Mj-Y~zb+$;L>JY|yIF;cjH zk^H0CvEl`-X5w}LMSevR!M>I3DdSeP*Y7KU{ARHI zivS6)Gobfyb2qBR{_HmoS6)&Dn}@tv627AXw}p_o0}An9&()4oJ;IP;Bf0eJ*ZdCA9CjAp&^vlIXdj69) zP|D&NiUw7Guo3Jdn~^{QXudTE)1ZK1=nn=)-*>M&=yD@@ovL~Jpp~?71alZ zk(*}(Q=no`5UtS57 zXft&?Gyk)@$@z`bIKh$q-Arb|V%VWmE(-dmU3F>E;mWl6S%`U#NBbLkSVTL*Y0g}}AKGbZ6nXARYb!9;@ZN`Rj|+(LsR#13 z(sE&ia>q%jz+|gOsk&ozQJFGt+t1ExDcE&{PIGcsq{G#>{%sj|rA{|*p10?w)i#hp ztIndId?XFq(=<#{X-C5ZE|RaTG+f^rY}Cody<{@phpQ~+zc#%XNmW^sP7sf90w*XV zLc@HFhFnY|>v8k^?h?l%ooC_3WItO@w$$(0i*&hmw(p}#l^TD8gc#11TXCRW)%4ri zhXKApjmBLIWDSnRt>3I)q0@q=a=Lq7(SOjc)A^rw@OdT0ee#6mcEnKw*H%|l+}^Re zbiFz{uGYnBWsbHvT%ntE$pf*kuzaM6Ynhv)E@E=sNgngTyvxz}9k|^5u^lg9xb+Gy zHWEE3*i8ya%AV5z=|cO@MM;Ei-IeLyeIP%#c_cEt=axtK+0~LqRT5i2(^^n!W0Mly zgwW1eP~{}vobEzk1^SB)q1L!c7NC)YBNa2$JYCBf{=cRfXFVhq#SUv}T#8T>^Vr6N z>~MsGLYwA@NtOLK-lK7{|MQE6M6_wcCbyJzH&CPHP3+g@2QRwA54w^aCa+JFb6#|B z?C01>W8*ff(X~zU*60I#GAtRk-OMWL#_ZFGFj#Na<7qe8zA)N}&iWs7D1jDn&Flcg`4e&B1C_R3cyDq>I|q&h1Y(_WM!e?&5GG^)u6amq^&H z0lls^Qg?#zOF;=-)$7uwtsDbBeT@B`O~*wIDTAiEo01uKp!^Go1okkY1mTt z8Wq*ri6jssg?GoRn)4+&%)(R9)6x#_t1+a$sModmRS7D}z{ylKT>_}yuJtO|_f!>Z zE!c1-`l~U6gBtc8%-q=i$+9-o{^93i9C7zmaOG<}Q8@3hN18MEkmV-Ns|i3a+M6I0$H=56}U3r$J}#&D{@J1!jiBTxQuqE1B->g!IW3AVHJsV< z?qf+(SdEJ5Y#+Z@Dse=*Ty&IpfB3*=s9s>-bP7@2#;#zQVQg-rxvA#UY^0?b~{A(vKj-O zSLi?8RHBFd8@G2Xjlw<7V1l>Mj5kbUdXpUHi1Nq8rMv3odOf1Payq4hagKe=cq~`X zQc~O`DQnMs-~VB^=BqUW>|~m~`@DnbPJ>5;pnSiO?jVfo+^X>IB)jt6Jiq%y8MhVV zm3%qF!fVuath#7-7-O6&YO)u47^Ib+0 zVx~H&pMN0!CKzM<;Eo%=A;q#954kuBG=z=3SA^4 zY>%Fru&6s3HjyKxd=I5c!~f9AfSdfa&sH0)>xg|aQAMjD=Up+P>6zi+<&Tpb_l(9o zPvyQBEEvp)Yg|bE>z8vb_gI=7-8OCaEX0UnE7B`Uqq|kv7VnzxSJ25Hw7pmJ3!Lme zxK}3Eu+#7P(nhjKxI`u3sh~uf_9dT6C@Y$72fyqqU(Wim&XV)e)OsY*YW` zw5C2myQE9dKHrF!Go)|&CUMP52cQ0QUVjD?N-ApuaW1;5(We4&CIZm(=U;dzuR**2 zP0E@~dOyoGavpnMxmuTxoMwf{^0@1jeL^%iltdDF5F8T|+_~|7biDYH3?i+ zz9IT0cmRQiAsl*^^{=)KtdXnM#1-TGZBLXz{uAHF?#~$}Q!UAJYj>_-;0>2(9a9N& zqw|OG9lWyb2}7D(0=ei@`up|IHu6~ErxPDGP%D;b({yitGm_-U06G@(pd2cW^pgKb z1axi+F;OIvfR7EylXu9=Y{Ceb6*;VQKBgCa`A=~u#|@_B_@`1Jrzv$u52V*&w&HkdkoFS-WV8Z&&v(o3{q?xlj9w7d8Afb7T^RQeB!y}OJ)%eImdxsAh_g;v8zpk zLe`#X$8Qdwu3R{+t-!=#F@V2w8!?@6t&dF53#~nD zn>|r$;omstSYsXodWYSy18$v^<63}u>C-&%Z4e+a#Wfpu>fB(;?CA+;_@52YO{KW} zA%qPDz*)$RIxf$Hl@`=Gc~3<8eqec;8(xc|6^WQfp3lA##8yPt*TlJT6G__qCH#R0 z$50CjBv%*NUzv9S{}3N6>tCIZV?v@Uj?qp3NSK%Mxj_RK;Zl1o{q0X& z*uR)M@~6%w=Q&46Xb4JruRch88XpnM?vH3W$F9c6IUgI5=-LG22PD$YNaAijD*GLm zm|>8ZSjviR3^yEIs!eo|;(h!m$*oT`Nw+qvsU63u(T&&vS!XXx-z`Y0zY8{aTcRv4 zHv1k>mBguyG=quop<1uFJ^k}mx2)>5^CSoLQvi~#l1uD z`OS9;D-vA|5=|eB<@C*#_Qtp4wByq3g8NZ&PwkDqppb*ZePm5>ccbnK@G=8@*a`pY zj{Q+L8wHVW2d%nb-UTzy3)B!+kJ4{RdP_p(NcLuG(AK%N6>87Nl?R`ERN3^ll;2u< zR;6#Py~e6LV$XEcUO?*!P|c$#ue(}n-H$HYr)04DV>ssz9+Yhb3B$eIQUb>n7 zbwh0VUCI*c*Yo^jTOuQL0IO#Z`^_vDjF~Viy+84*ifP<1_ZvpWn;7OHUwJmSSFRM? zSZVCNfnL&osU9So+`Ax)qWi-{)}IN$a#4Fwdyp;Ww8k2XbC3OtvPb1M@>}QQ4mKjG zw5#NUspoau-Jr;w0~#C+VvU3}?bvM=!17RJfYT(6aqw^6tdktR&$YJ3U?N%VYg=*= zTWRRXwc82~xIIu4oi-`QX}<_vqmd5q7|mv$J#A_(@6mS8u>psi zzTN~$Bg5-Dl}=Ef``D9=Ca*JmaciCsd=Q>5APux*`RUkO74{@($l zi0PNvhg}X#Jd6l_X3ohdl@rwUfLM>>q``-w3aG6I`qeyJW(SAoNBl6A^StC5w`K7| z?ql~=8r#!;F7d(H_RQyA+RF@hGZ@63tQR^?I(lgjm~qJ=OVVbbFUCN^L^z-YLVr_Xf=p9j~PlvzpV znVG}T10qh_?3X2cI+6mwG9thoYSIV{B++k z8f-K|rkqnKp)>r)OvF6+!lN6rSLW|wOzscyP}0vNmHYK~j6~~|VF!Y_YY^7cLm;VD z=4l83@NBuX$J=Is?HnWGj1JHQ$vM6)lBY9R|NLO_>y?MeBh3nvhkV@p=Udy)*XOa) zN?rqfkEYXqtN_%)TLJ|jPB&+y`%%q(E|GSWIvW`zg(Z=I*5E^aGn}=0y4t{=pXvEP z&zkq;8!6{Kn66EXTGqXG@|Sj!lR$#yN`WsnU|Z%OehIY(#?Enxj*V!ZZojXBOrA3} z{K9kd8c%k8<$eDqF|5UXu}&Y!zAfE4f!5cN_!@V9_rIBdVc}(YQ2MqG50+cpomhBi z7IspL=z{&RHVtL3q1{^oEoF9cWJ6@)HE-wlVnv;!7nH1cppJ1aln3bJ!Hzy1d+(M< zBuYQnZZf=vFPy+~`4cj-(Mg8tF~3b&_OggYSGM^phHnf+30qi`2FCVjVKO%IRsMq5 zGJd$jDj6$?PBw0coDA_?I5=cDLQ@c&!Z_(weJ}HEX?$=>-;fg=S)aH&>8rTWmhw}J zh&d|Lf1o8`XLh=C@3f_pioIxn<0ns`{_^s3*7?&Ahp*qw@CI@=h;C0n-6lew4>mS` z{V4ks(mmOGkzKc$;(d6oxcO)#EpVdwp7>O@Ap35o=pZByAC&bhDx+f_;XED|#ofR> ze9~1Q+LN4T=bTNA`{8!56nMElut)SFPFj(=@yhlf^Kri1V}@SiXp&^{RX4uQ!MvyJ zw~F58`vnmSKVPXSXQI!V&`oU9XB(uoT(W{(a3y`Z=2j0L%{fqii-ew!liSWUdq=cr z)$M8e<_{a-^D0>T^cIm7aJB^f65>Mn2HRVFJpN9^)z#-Eq}4l{ zAHTz2D)nd?bw_MYaP2Kelokk!T{xRd-GT|wM;cchW?zPrG6z2|j^I6#t*6*~8eS`3 zHUA$)XC2m5!-jDY1(leJ3P=o;kXK}Mj*XI#6kf$3qy(h9#wZb#6p&6qz*nR@Ml-r) zFj8v3$PESy#<%a!bDit_v2&es?K$^z-@n^eySFvvjxye}^Rf*J$S96c`8_GiN#3U? z7Y#WGdpvmYQrc3TCq4wJC?3yae+jw~D;P~-b(Tt?qIvq1+704RMe zYel)Az&&~Gwi^G`sy?$!bPFEz&46ri39P6y-2S)bo|S=B`xWb832kDPLCvF#xdh$T zz$b#UZ`YVc%W~ca-`%MjJc{Uans{O9KdzHO4h$zL*x$L99s&zXrXb=qidid=H$7BM zjkU)uV4;}EGefT9Fhv>zC>VJw?eHfO^zz}uMyTqi3JbZzTU^7Ko{HewC-wAX#PB)a z8iD#g!oSK_(8pk(gWKaZ^4|a+s~q{Dd>=9K_Z%0acA-cA?BT<ybT= zWS8RK+2Uajmg{zJ8CN(I@h^feLxQhP?|EKA)F@y?#b}Bzi)Fsh-O5H3na+4s?WxM* zg4TEJ8}FyATW;lC;Q1|Oe~Z-ISF7}UB5&zaocL7S@_=Qpbs|sc6FYzw-hvih2Whg4 z@@|hoTT0e)vYxQ+--lzwue;eme%NIJr0yi@yH)nui+CrOJgR`KGWGCoCVQjTaDqv) za=xq0gq-XC_82}tlK596TEWHkWmCFj3h$Q+h^Ol$WK(mKu2WeC^pnP5fo3z0#9;Zy zyu7Cv2w@#Pg?55_jJpl^H1{)XDw%kN`+@*G7vzL$pRE!0bz$jn9p3Zc++ed{} zwALQZ-FOxjtY@?N#t1ScyJ>eVm0kpst}xzYl;1oJ=tDh4{2SE6sb(21>3#A_DxODo zxD)D#2As$rvJm${|HYmBx%I4q1M+j+Lk2`>E=Mhk6^q>9Q@U*=@fuffmrt^n8-c}SAV9Qu%(@1}Wl}#lkWl`@ z5I(3D-<_I+SN8BWygi=A9Mu%{;79C@$&{*kD+3Sbn}rBXAFwLYKS5ZM(wlY;i98{g z7@BV+&aDmm7{IIMU1|ZLtzNwr!lNY1g_k8_CpRd*0hn6Z*%^L@?a&XyPf(pt>Sza^hOH^IU6{*=o@mQ7W-LPG`4QyUMy2+`p`ZK z?9tSU`JGll#nF684Be)!N4Qh?SjGLNsKZgWcFL!V4NC+?fC#!cPisF7g4%o;JBlC5 zqv$Vt6Q!C?+RZHdh^YlVp4{uiQ8s zKFs1hI6IrgczI&3NB;@ehp}F*ZQ`tsiGBA!b4{GIns|qege=TAAg_P&oX!fFbJW)1 zSCtfe@-_D#eTDPml}`QddQI_#nA~dPQywBiapQpfRD4#T148ZV4Otia2+ui(0n z;AC7r%kDe`MejA^4w@CJC%}yci_V(tO;)5qX@6eGM&or}*+7B5U7HC%0Ih1u_>pp7 zolt!|d#m7}8leI6=9K=tm2HEN_pUH;L=gDTEw}A}=;qlJ7;zx#Eu`b?r7ES-s+-8T znkrkae4?k$G3hRHP9^^0HsG?t1#z(ZDBNdIfR}G1qV1Tz2c<{!`gZEM1kPwW*e)_m z7RyYlMA1ucThQP;pK_7vh8mdUn0|cqz4MGj^W5lmL*a+Y8Rtl~kv}b@zMbYSD{T1{lWbuyt(DKU$wtW4hO4eG#;~Fg8u`hJ$ z<$5O2SgtxaIHGor+JUV5h2pzi-v-f{?O@+smYq4-klm$Nr~Q4^QXdB1%V&M_+?RH?!X%uuHiJlR>?2oOvOz*8 zt_y}l828a<2_S}J1rA9XB@=$rJdy+Ce8#rp zIzWcoLljS$I*BBAM@j^NxMnU5a8*0E{&MAgoZ4{V#nCzu%c{O^G8VW|b*P4SSMCXP znb8qZve-Ad_%GRi;Fvh>^*G0;fMyyuSiUw)3z(OYtJ*rDv!mzM)rS0$LO5W;94t zd;s;%@?kW%BmXfl7M~;r7!UAB07HI=s88KFD-FuH0!~w4Cz;VB6}YpMp0g70T63QQ zN3X%ho{nh^)_O6l(;DH`f7-`B%cU=W>z9d_)k%0gDzb5148jBAUO#)WZ94H>h z6My~pIC0rwbN>ut8PAeof8IeEX&d?#ldCt^mS}0g{e}0l!k6OFI%yhRxhZb#-<2TR zOpF(=2kM5%j99K;2Vp)2U+F)s5{Cqd{)qo;pwjj`uX*0PQ_}L1sOP>GJk5ujd(A6n zOe7bc?*>Uc$Rl^c4E-L#Hd&sLLq;G3haZDpA2!x+T3nUHseyl~XcNj| zXR{K4dp8p3=K=2GAL%=u6xk2TG6nA?Z>Bbz+ySoG1tdgGpih=i;H^sKRujdD#;npRHem1`uo zR#%zeR=Rh9xe?%h-yYLr(2!$jJb#={NJ5K!W@p>JSaX`-C#2htaBxvn@J)wc%N{^M zn;h2{UVJ||&vzk#_4C?XLa)J&vYhMqOICTnQC!drxye^&?9);furc=V5#eb498g zznH3@%zQQn!IWo?ebNLO2oY;^(#!jW0#FSaH5#%d)l+2?d`izb4rwtGZpU`>QYQgz zhbX_jpBK!F3@0BLKUmhGJi_yuyEVN>^qse6x3m|lu6OYVKI|SUxyi-ZI^0h`keeZC=$ekid9FrdETuzY!=N5w*=byvyNeXAjL3Nj9z zpw&Ks3V>U`2opb+Hf5G=+pX6=K`8t{+*JVX7H3l-;NhC4VAg+-{B(#KUt$8Hq+RoQ z=zV0TbL}TaKDp|Gsw_pCM^vu3oXbgH1o=skF9FlbPPy@!6RIHeG?(SIg})Wsh6V2Q zwqdkBr>wcZNMBsr+!+&q^!}lYG~xTTE~o=cSV%#9G|A<1{qt%|FsgT}*I-gWt7TDR zJw~AQWA`VnwJ^j-``ss;Ymlm^zS}oz!VCkI!IVAOT|~{cBFIQ<%sIMmzsK~v;9nph zv)1q`vOA%o-zYXWk?Yr>%n2-%w=a>)m2cIwD3VtIBvn7}laZHm(Pk)g zc~eFgJ*Mk^Pk5l}v5dT{#m|#H4RezrmBZcU%!Tj|CV#6)Ssh~Yxb-@#*tX3&bM4@Km+amu8d}VyEHeqDS#i1Hj zfoXK1qrA6ClYL&2lj^3N%f9}!Va(@xz@ARR7b!wm>tW8(+mP{p*uN>;y33S#jVAp5 zCd*>UQ82^kn+4WeD*z}4H1>B-!ygTb^mIooGF&;<{l0e`fmOH``@}jirRg;NSN=2* zaNOSwJeR_}_=j#ieQmTai58^L`_gV^15sSdb`nCh=T6a~&I1Y_*jSf`>E?m~zpFOV zGh~)0m2v!gT=Yn~N~o`;;5)rY9|`;_xIOx_a}ou4QiLiiEj%D~%I&R_=z)PN+XIJv z>I=PqWMV+Z1KYM%)2FD9>W;iuBK^m`z}{w%E-<#Dr{*NQ39k8GnM513JcXwzWn*qC-ki|3*OV)Kk<&)Lj#Y&#bc5ruCs$NM#{? zUp-HbQZEr+{~C$t8INr_JX(DaV)cm;Rrt@wILt-NW1INdEBxPa_FT5}cZ|}O-H?yq zvFUE2LKTw-7~^_GAAE}!`sIFbtihhn;Jf$$o83>bQa=N3uR0A!->nk#ZQ+$N!VfGUKW1BYcmVDU zxA_|X_wwWq^v1y2aMwajq`&HW_{VCT+>bj*lw0=0KLPER+LEH3%|4s2!q){2nVYQFgNtD~`O zy;5~P2N?KUPg%N3cjE9B&`1~e!;EaTiNzTQg-|B4zRiJ)$ z@+o|jnBm>0m;$~6KK1i7qxa|WipTPXq>eB}6c8o!*WFzRYHHbytAa! zl5bZ=KdRuOOU!AX7fA4_H)*>ZKzRx6@u24^_PV(LAV;F=T0YuG_#a>3bm8(%`tYi5 zj+5q|qqCG9>GM~=cvBQdf4{B~i`}O^-=Po_p!6T2TNTTh6Rf7;W-wb+qO8Nz8PDL~ z?6vy4nLwRCFeJ@dWHpq`umR2qJ&W0TF18*l$l zs6tsBW8keC4DXV{xUTb~bh~3{wa`0jk*7pDkHPdQzQH4r?@{`+y3bz@*4~8|FXytA zbOebH0=>+fw>(X3FtbUQL>*_w&%QFB>(=3~d-n@%!0dsa6#6{6msmluY=xfzU+BJs z&#!{~f_!d+)+o<>FqLj?ULq3`{@}{Ud2qZVwo>3v$&uXl zosvmeNnT|+@p4gV%(ER>6X)g9&Ui$}yh;>A<~e-0t%+Aof9}65gql`Aq>|Y`*J>V@ z(@{96KT`N>-hrp+Vy^}MhBcOvTD9rhrq8^sOVD=&U4kdFo-%xf~dTrlkp;*9Slv^KdJkhLDR{xw0SD1=9A zHMjQ_o3M*VnZD2Q?D$J{p24IyLWxrh8#irA&R_UpoxbZWhmtZa^L2yKNf^DTfN|2a zI{EWM38Uic9eyuakDsmpLOvtg*RRqY+vv93R?*xcJb1V#iJMC>U-n2lg|93pjxN zSv%n|VjuV9VzgxR32vnm+6p@e!TE_DJh=GPzJd#aV%UoPk%sMSzth?eQ6cYf?IflU z>;@Aw?&_$EKLxjfyja~(3i(Jew1k}av~S?tD3d?WR*rIMZl|+$a_5j?kKf@jhSE2b zcLs%zY$Q;EXv~4&c6@^w{J<1`w9ycK#08%hFicU4|y^zmb@@~?)(ZBY!?*AZ_An3uER04uZ!MQ zJ7;cT-MbC6jUL$#Ax~BJTX7{Zz9yrnPQiV1>~mqTo1Xm*JTzz;(()QYxtMz%zH2o^ zW3dH4yTGz%-Na8afYIB&#j*eWuJF%;ie#i&fPd6<1cWqWpQ%&V&nq3b30`4DAtWl0cBXWXosMFkd_DdA6Xf0Y&-F z+_c6p9yX13fV3^Ak(pthClD{bfxCsz%7|udaX(I)OcVZp-L4 z2bNo*x;uwM9@jl7K5IzeoXpd&Jlx){#kVJuQ2FoZp05sMlbriTr1&e6OQZV}I}F<& z^{OWXh?8$ixR;%9+_r~I@zQE^+noi{3I5ub$*|8tMNBFBmYN;!tLqhJ6hwBhEctu2 zol%~ba+~5KH-CKFDgs>+IXXZT3_`NmZPkjbchnEZ&<_>=L}gnJIxTC4^gDSXiH8B( zmjIJvW9x`;_R>@GAI^$_Qw0?=rHa#cvtW9a5oQ-aHNHXZg`AftSd+%{$^b$P{0M=5 znQW)vhc5t~vW80PIQRXd>IL#@p-qHBILLS;W=Ldh!G&DgeG>LFzYX8@5TB_T{@Ly< z`UoMk4*CfOK49odv+@7zvHAN2tGgXl(NMeLaFNW=F9DL5+j_kD<=pIMuwE_}s*yL2 z+Pc{yIxZ^^xU_Z5Jnf^r)=%^CnB}v#b4a1(Z}P|b+9pTZ3Rvn)0o|#F_P)dTZFD801o_bJZa!gTX$ZVOwWJQcjI(fgOLgy-$W#9g= zw>>eRrDrOZe^Z&PF1|br_`?p(*WE-YuQMg@F$9ukPDnmT+k-*MyFoA?HaSlj)GOEbW-j8dEOcRW8Kn_q}KUL#z?P{X3d*74z(sj_8Y zk;P$JXYQBZ!bvwauRB+2!V}jRC+tw;z7|~@t&%U<3X7OcDb(3(UvF$`%~WKU)NJ11 zyo{=-)y+>OrUmm5*c<^r`7}p=TU+!SfVFd4;&dyB{cmNlyMJbAh>_*9GEv*PItSP$ zD{{iT(i9aPzb6I_&GlNFmuY_N;2?^tH+(8*B5f6DKq+3*8T7enusF zm?_^l9H!jPAXeSod$OEe!2u+Dwa;k{W_#HHbuQFu>?4AQjQfe#zkNQsdpU@?<8@ij zYw$!QueJKc*BfWP@V*ClVb2++_=yRW=DS=8)%N4r4qnQ!T;Ly;aeW@ew|Oc63djeZ zPjhqsOsF4|c~pyJ<6R4nT|rwZt!#4?&{wmlL9p}pP$lGyL!GOj`gYQv47}GtJH2w5 z;o;DkjbVF#1H2O1bPy8ha%nxTEqLktUMp7L?M;(X_TP~ZELqHrl(pO*AM1joln+Ek+vDm6dFvBQV+JQ^RyBHJ?MD+h{c}g&& zbdXULYh$8ERrhx*Vg8Egm>cVY*f87^dgwMMuyMpU!p3<3BjH_N`fu6^nsK_SF+7m} zQDRu$oeV-`>)wH<`aUs9lQbp^S>-j>#gb|t=Cvm6In34DS?s!` zex%Y!Z{}PAklEb0<+xI>?&8mJ_hq$a$F10K_Ug_WmCr*Bj>lL$qq$t#hq` zIbzLLxTvWx!*Y~KNC9O<)g%`xKbuW)AD z*aNJ?K2{u-s(+6kDMwIk#^i4hpQ2<|950%c6DG-zILR-*>s9MfE!3soXkPBW6ki9T z^FFz6v~$GJS+P8mT>qQFuA9;D(M3|me_Z2$O9`T*Q!j#NCrj$bveSgjFt0=qqfFX# z0N?yOaRw-%`hl|~f_zxNAGw7FW4%7oJu*E4_@GuZ`h_Q2b7Q%V`3v|XFM130@B7~Z zAy5d@7YVd6Fsc_ z;(h!ZH^4}8<9bw__sSC2@dv6t6mho{n7hhE9YMdFFvCx@dvCa-FF%K%GVY8<@H|2$ zp)#6ABSS29%wYH=mq8QnKU`#c4GP6-vh1h&tleU-kKtQV*3LHB@h$5(%O?7h`T9K` zl&YlR}O&_DN>RlJK<4%8ttp40Tc}KOdp8eR)KSH|!tdAvAY|!%Bs%b+s2Y&qCwXT|L zAp%I}u_tD7uiS=XmiSkpSS^2cz?EV3olv2Aeq1(vE%C>SLNQp83^pEf3D^`j9>D&X zrhuNlYZ#N@gW>^p3h^xpjl5&rx$2KL8Kz4V)Cn@-g>tVw)Gt1CABtW+%>L?e(27i* z112`>)pa(9Mfp)U&^`?C(!vuRLLDT$JS`=1te$T0{$YJZ{MCNX1i>MV`NaEAYS@h6 z!u&wKiV7DdIBupP)<@;_b%y0B;AA1IliQ^Dp&Zq9D48b3IKN5Z zT5X3UJdo8`%vQf`rz31pB+_`Rfn#TOzXwfs1N$QFQ@=W;!8jU(ui5WB{Ry~bVv!{7 zdzuvDIl<<5DdhPVCy2zUoyck&_vw&y;;W5G-!}Wc(=!%F_TkwZps2N9DYyhR%YOE! z7_g3@M$pUTcwa7HLfylGHFOJ6HNk75dN6BQ2Qr!W7~C-Z2l7mXSI55$7?h*<=TGab zFU07jq^$m-Cky%jl*5>5s2twvGL=^!F>LUg?KbZaW38RJo*5>+=wDT2^Ly4VHXOG- zw-=y8Vf20?J1YHg_QS;FgI54G-5hLTWxT~1pMQL?^3F>IOaj*EToED0V-Z~kxM5zw zjn|uFD5A`|QU~7}$MXjZ{$U6#WqVGvX3jrMO1rK^0>kbYw_nOaIM|@MGMio~gWi|G z;CCAR)8`XN6uGKvaTT*ErIz9hDf5~eIweoW>m028U3QbgQkga8CVZo-{~d<4I&kRx z$BbL}ka?o)$n73nE-^g@v7!(tbS+Ef4VV#5(3V$^tA&dORF!|CadZL z^zwRB!(nfS!&L`0={+{$8-uwexru5~t;R*CI&u zyGe>1#;ZCPD5X{2n%-FO_~Us~yG4(LK_J4NbtE%}DC|_V#@$@aaw{a)M7$(`bu?yU zKoSb*T2o*-RQ@L+*t@-&Y2y8vFy!!)D19;@K7C1T%JgZ<^;Nm5Shif~)(h7evmZ>E zN4GLPczJZCXjB zV&`tTD07?TLcABv{^oSLp|Y4HYu*wF$cL;+zG^MYdhIuMB4fADrCK?9f*XYYpvc+r zBY8J2!I5kyQWlTQ-Z{rDrbzJe`@c8qkAf$OF%`SS$VX)#9FBNK%O7U%%VU^$xrDc) z4Q;`0ksep42!c))IXTbriQ}|Nvs5K}Bi8p~w6ysnx4e_iJN>^z1$=`Nkv9pPEw^nl zJWtePg>!+@&)+LgBlc{Aju~&8V*>Nov-xqqwG@I;Lf)InS1NqvUq)4hizbVz`&In7 zN0Zw^)&>{qkIUAT*fZcl;oP6vPtmih3D|ChM1Ec*vcCD`s> zRHH1Z6d?IYnxXyTAO8&6`P1dYn8~yu```)Pkmsa`4QMZU*@HL(J&eF_><+b!wNMx@ z{<1@NFTxnp{3sokZIrqYJ*pk8d(Vz0)=`vbs+>7(zRC;U1+~KnPEcQXlz$Y!aDnFg zX9eC_qAV^49pXZKE$BV9T_Y>uuTewyk# z>TCphGefvx8{pnYUpw<~yoSsSL|C``&<~bIAbq7P<3<-5bvOWnDYxpg0te0IZ(!#U zF3=b{SerY;;ZIYG`A2UqSmz%CM1q+Ll6#MMP22=N?(NUsz}@r=2fKu(>@@~B2l)o~ zEqIamylbSwCeLn=SqCS;>|3#%8|aVx)0)6KkgJ|v%gzNyi+4D z+$J>T2Ky5RB;jy>wCtTjymKN>O^CziWtXvO9cKZs4KVU-YGpkAQuU!KS@e2cME8t5kyT3q4@q9e>Ld=VGS>n7mEM&pA zJQE!(p7laR-Y2f3h`e0utNsJEhtJN8yKH9YTl4d})qc6Q#EsIb!AGT~K$2HM0tY2~ zXW_hFkMNaV5-5PwJ&WO2D_0pi2>qR@ty|gMy^obUWRD~3xm<=VU!6jk*|xS1F9`YH+YChIZ+eeUIW$vcS6 zOLJsQKzBT@B9X?FPlgA;}yOptWGm)z0RSe**2H48iuI0_pPGj z>nDpouG!lzBxMlX*)z+$6o6MQUjYRu=S5i%!meN$5aA!0-rOUpIBUwE%&qwa6(-<+ zp7;1KC^H8HU7%IbV%^7yloo^jQ%D2SgwcCpNXk5o{!Ok&@zd1v&(h)U>waXWjXIWe zuH1NT2rvRY92D;-ZUe@VER>=+ll^G|cgv5Ee_V#s%yO`an9e&mQlU{NsswttvU3AB z%+1Z&IC`?$D14Y_mL9PvV$b^W2_GQInR?@SskU<`9y;bZB$^z5#_zbcE#c6zbNRgA zoo7usvQ+x~h*|Mm(p!08-42#F{L(wqjIb%V0P9=NZHZ*gnn{gk!|ySw)dzNOIO_d z#=xhO$}+LB0H!aZ9%82t@ShW#!iXKzSS8CJtegj35g&%xVOCWbmM*e_%Il7*j%L)+ z1fR%X9=~BZU0(O{wa0__!qe#q8O~AeP+iEcaPE2cZrXd7wV5#tz znhDptf`+pT?Br=^l0dh8kD^wwUne`_xIK><%XHUt?2^q@-ZF3~G`qdER?OonE@gS+ zk{BnRvtF(b2^Q}8vFFQ`t(wK3fRTppHe(mlro$=>@dZf%ecFM$eEsv;ie>W0S-4cD zX!Q^cqf>9J&G2Q!^gs>EsaPkkAok5>drcj#KZY4m9PHL)y;*S6lzZ(_IpJneRzycZ z07nG4TZ(}*Z5?ZWk+vGbkFzNE7^wN%mT3R9Cvi+-#OFW%b8Lo=)L15m4IOz@dF&Hw zWm_CH?$;q7gQ?YO!T>YJD=$ z(VgHL$J0uT5XB1`$9p_v0r)!#Hz>WKq3VrPhfmQj*^gvG%U$3ME{l7fvhdC&u^gn6 zRgllyTYZ^4OzEWrC;o50!Z5#60g*d)~?CZu@MT1z-e)dE#5EIUMW# z@+FqOu~<)=7iVl(F!F}gvG&{cAG&}d!34WoCFd!Nif<7CzC3TKtk_=*k*EKU4!~&9 zftHX%CC~vWq#R`AQjRCvw}lMV?ZwJiq{S1&gF%;C%z2UJ?Yz=|b# z_&Nj$x?nHmG7O=bnp0PSE#&(rT8~{vH)+dJr?96HF?ZQT$FB|;bOlv2nziJIn#qO5 zu-)Sp4K0efv-)G(EZcj*JjahKGwQ{V)sBWorz0ce8MiAM++`8{ zhV8G{Gu?635|jC{ZNJ~0(x2Q!x9lyic*@Fnq|tk4fSIQYqzdcYmX)J9-(Npq0Jyp`NP8DrusHdDkQB7a zeOGjGR-m!+w>i%@6=F=@T%~E!Bk?moUF>*=CJ*WYG7}#EBHy%)w_98hlD9JN<7{@0 zw!9rR{90j5fcpJt>p|Xn0uJSL<>c2yinHk)I9kDBx!WC}+SGV)NWBZ?Jkj zfeOJ)&OQ|-yA8{%^T}>>%k`&7)j!L)JuWK+)lk@v0^Q4S%ZO>J@EpIKvyZ z)%jcl!GIM-XVg_-yIwnqNwUJ46hJkF4<$F<@9VXKgfP9YR$*4V_hqf4W@4J(gseBc z?@(>y?2hp=1V9_5w%=*A26BLo`3B+w1oXzYwBF$IO9bL2+-r3y^-Lr>=bg-nIsuw0 z-Z^`RKgNt#_LV@Q=QgmS#{nQ^U||LmxvT0oIYI*4GS{|a>G#N^-DbMT9s`_63@jK( z$er4rkI!FgWz1fTP=uFw49@v1O-Yq+GZqbOOzew4pF+wXio9k5`5<#3uZn^b=M_!F zD$DiJ>*>iavyM*;CJV8y3FW|gg4U}+&&)6C-Rrif}RX_CU2{MU7KCrE`aK!d$D^xJawk7 zQz&!9!!u7yiaJHyPA@NBEMa(_%Rcw>KKEe;Ms!dYuHf~eYh{6aTQu3O@6zicucZaH z+Z|qfV}IRAy5$)7-KPBtzHVwKpbA~=ejp`RKEU+Qp5vmd=H(LcwzQ|+?ZiGJ&{}E+ z>~&LWZDS2rMmUN&TfPlp^VBwhymGpQUWMC&zex&0tM8Kq%nY)MxP>_Pr@ShCZnfYr z6re^snLnz9Xr+apefuUm@FUua(PP1D+i&Xh-fOur$iKFUnx2cYZXa3<3^R`JBA+He zw`LX4j<*7oFQi3m43{_P)(6G3(nnzp+fPKb{_zv`@g8+i1ZHoj-jl4$JE8AMuE-mB zH}tP-i_#ND(@~#6k8@dUX^OH_%o(S&)XSt98Gc8Fl+6S&J;2weiklBnj|U{Ld|XU# zS+zIgPu}rV4iO6Z>a z%uAoRwbdGCspoHdx-cyBS0LGDxnEtDtipZPVkI~D^EgH474kJg(TH99+&2pclOe4P z{#d>lbf?7N*KkCWkUv#7cTOrH+lo@dYS^4>ze!Xi7PGu7--xZ+;MKb?zxLT0XIb&L zO?Vs?W&`-AaK7CxFN_M8l}$YGpS3%F6_yHmBkZ6!OU&PXYk!hU8FFb^UuNNiv{)xa z8+JL-l8xhbvey*%TMG7CF;4q@vCT)ZQZCgCs-cjHOHTOjd&qW@9UWFg4}oFhPUJwNPi!$dHwin0hp!44Q~s>|`##^VLmEhEiOwU9;X_t8gEc8rkV|sji%L$*?~V(f z+^af><(lks@(qPacoc~Q)OW@gCPYqwl3&K}7tLKuISmxy&eA6@8my`eNhk+TOXu*O z_K~F5wW8o>VL-aQg9PJ00)gMA2(u5)W9y41hiQ(2`O{(9(b*(wzV{jqf)5dqxQPMy)Y!&U$V(m%w_k2-4uw4Hm>fpnPErc}v?-TJgd|$* z>-t~xF^;u@CDTb}Zsk*6edrl`Z$*PQ@Txx*dfsjh2S$~E z+?rq~zzftEz%ZnCjTnXMp6bK|@3@&EEQMgBv{5Lz{b(dD#J%r;ER|+|(bXPIoU+7c z&_p@VV}Gg}Y-l6o0<|2{*4VAZvyV^-j6det#sdV-CU*j zworLvNp0$|qMPIk8F~nE3}a$ni<5$DVeY03to|B0q03;b2v8}Ey{q+lhdI$PxChSC zAH_ZAgKO?K^DY1~y1YKVZnw8RF#t9yO*mP0nGXDH&``p-Q^MNc_rAWs?sJXg6-h$9 zbgX2GopLBDDE+T%N<&<%+pglGkYO#@ietO9<>UYsdI(RX%P}w?9dz*Ko!sAUb2azn zR*Wj`Ru=*>V}4jYif2J2g{4)|Sr|Y{uV2TwKLV8wcRfgCbk%MGa|1b5Rx$jFe``5i z{I=2>=gzd-t9jS>?8{%Gzwx@?Bw#q1zCK(E<>{Ib{7ad0ndfPmb#!x@53f{arOl|E z3EeSfyOrUZZA24~qv1_MWpBFjA4ZVoqmF*?yNljWUMv5ca0K1HcbkPcn8Z~bF6(RZ zK{k}FzS>}Oz;b&a%YF@;CjM|pU+t(<1A#MM)u|P?{5$mJkfxaIy}PUSQWoNWyzWN)RaiA8sj2#GdL1g2Vgl#-p4gNPDoRp zwRn9UdV235+7I{!z=UmQ4dq~edA+l65Iw(wzX9rYWMe%%lKcw4rkUEuI52O-Q%H9b zS=;M1X5M~d>#TE=`cH22!T(KDwZDDIuj2_Wi8H>5amoNxs?O_0xsIjO4#X#C(i>D% z?aWhoAU|2Qjf*FNdoUs#S61lgGq$s6HN*I&KCN=}a9FIvf9i%e0So$FkH1WbDU(w2Q+{Hz%K`oSd#%;XAn?&9HpWV0BseS02O+JR3(~wfpnKo;+Gev2 zusOS$Sb_GboObAHV@JWcY`UIUYr z^|8arX8!A)9(`G#HuWIQRB%&%L3&z(o;Eq?Mm6$@;vTj4P57OflzSjo)kV}~k~;py z4$a{s2*9|qg>AxB)bHSHpE#l}46{vFg3i<#JhMiGr~Q&m8x{VT-C7@ZExUg5ikj)@ zSa9k6WLe6etKTij_ivT_D1-HDo1yaC3-xHu4wS*rJAYWV25hc4A0{y+^r)Sga`Cxm z>=;2hT}Fl1(T}b>nW67X=DlH@e~qN|_PDqrr~KRussRmFawKYISeLm2st%`tp~05p zdVt2trmh=&Um{2#`b2%Sur+`-Y6AGVd=q4~F%s8ae~A>Zlzi&9(F{)jjOIN!5w()X zsr}r&tO6btc93^p?7lI$JNUa^i-&H!ldoWsmfJEKFE-9Cm3-Ko{NT&apaljSzW5@h z4WLyhvz7mNV!Wv4+#ra<@62@@*)&3iK={OS_Kovh7;0z>jRIA1vY_QRZkPl4pS;8r zzOthzzJ1`ZGi{Ay9j(2!K8~?egrLQZEr8U>2bY&uSWW#zDo*r1$T9Nf-ilLS@Jnz) zG={V;lJwJ(LSkgzB=X0rGbJPukP&@h)c_{aOp?<@_>g%haYkyN=q#=aVf5EV8$8vE z(9c!6HUk&jKauXbF_lDjvxVTtr{(9Om%e=&ODS!9#T$kbl7Or)OI-um3Y^g7rps(W z1-7GKM!SsK#25<3(6t*|gORLy_r_2dS}!b_{v#kRw|Y3V(xF`8T3kq1xpa@Zmn@@v zV&D^t?}oFITOIS^jZ4LD*60EjHr%_s2nzT87gPs#0F?YSsy(mOj=a;aF!)w zOXOMtkz;;He#9_6QhsVf<>D`FxTZWCurk~{cj;2qu+9^$iV?%`{7TvtR|v;|$i<9d$1vQb0S)akd4Z=V@4KGQipdHLb% z{j_uMgtu zm>S2*9%=COZ%5eRS;=3Bx`=I#`rf9$msd?BM~oK>8vbm?g#2P8DZf3<;mpxVLP!Vf zSnXoV27VQm8FWdoqIae%>362r2zXt+llQMMn!TBV5*d~$foIO$&!l@Zk zh?Z*IcY=zV#01sXbgIAmyFV%C(aZ-4EWaQ4O?vxoX7G=kvAoXiJ;A#Sh@hh=AC(I$ zql>N0bZ)MD&u?4b4eje@{K4hLSYf!~fiexUt5#2@PU5Tu63!K`N4>36PSo2J@B_-5 zN8(@f!E-jldAuI>1vG`1MYC>7z4+6#079*nr0Ui-#8Z1=gvJ89urP!DQl>#3V% z3t!`k$&Dat{~oUmm&C z-d*Rt(x%lJ)f9J8TJTA(Jk|&ZRpk_Ky0hw(Lij4dWXqEaNu4$^bf&rb(XjVVf zshm>Il~Z85UNJq=iioM)lF!Q*$I4wXtv$N+Bc@?*@4*xgtfl4!Bp-T@?zM=^C7j>u z&7V!K3Khi)KxwIChboo%pvga2Elm(QlfnCz_j4WGy!~>qfyS*f%Q+r3XTkozr!P0B z7G#EXRkv&8?tp5X=TF7V?~R;tMP}4F@{K>Mb5-bwm1R9hl%zYlc2!TA@k;E7+9U%1acb46vC%_+l5h7(xT}nVL%a=$&XL0_RX+vK zk3Az3uAgZ8#V$J2tZt&|rHU!xBcPk+XALkJ)a5^9+Xxc_#nV`^{t+9|J|UiaH4$5< zp>3ym%4ISxkh#fI8PIU^9oRcMMC?N4{ngYihho~bc0@U{!dUtshc4b>u;RPsQV%4C zGc_3SP$-vGN*59Z$~~1{Xc}bR)*fcWihQ#T1bmqh5wb0_^q*eeVR^qhGiZtN-+tWO zTXyXE(e*T~a)0&C>l)q?k$m3tU8T`+FMh^1iW&=E6)tqyvQZwRGMQ7m@ebQzW&Hco zAGSTCEg4*pZ`x|3c8nIbdFm~}bL~}Uw-9~rE#n0t_5#mgg}u?q4>l%%x^=t7VDFs4 zB(W=8`&^Mee8`KICWU@w$KZ^iszlZ{B0_I8kjAl60e)pQW z3}BuL37*ZLi>+Yn*HOXR(*XJho9}l`O)vOF)n>GC!Yn3q5akHY47nE_s}>VnzZe}i z7H=zDCg3+&&~M>Q6Mh}LJ&az zmtqVGRcXp=xDo=2B}lF;%og+Lc+J1DB=u2sMT;_5sN5jlG1*G9RP)2@U%; zK(+DSF-h)npXeC$+DMCbt(o~UkzP9-&eMtC2kk_M8EQ4Kcn|+y0VwR8g)nyZ@n3LrZdzQf_u3_57 z`iU5>(mW~5^X}dD@9r)0oEcc3y7i1s&aQ5$&6o%M5lYdB$Hrt3-%8ACZ83gM=|z>| z?$5%Xe6#iY@p;tRIjg`CHK8+)r1UBd`G79lxp9E!EWgLJ(wMf|G<}w*l24!90jOrc zqDO6(Br1t2%+Z!hacLCDdN7aIlO-}m=!NV_iH~!Q{`36~y{YcOadYNZF-aGcy%I`) zdGdL3?t@ZRlG#7lMNN^M+Ok4+7d#)B@Lg+IqjAv)Wa2J(Mn9Yy}V?OCi! zUP%d_$ao;TF;Q0Cle7d2B~~Z?Yn#N1|eqm@$`|#OEChFLG-*kZ!5Q%?TCiX9{yEKc(RZgwJcg(4l!tc4S8MIuR zGiXSdHsqFt#9&>e4pQE+&F%Sg-bjxnDLm3iFg@(QGPjQ+`le4QdrF8%;mmw)l}e9C zNHV9Vc`I5K@Sp}V>XVUUKuO;9X<1w?eEGhVuq)t+Fc0K*UhAp`wQJ17OmLa1)t4R? zUpzCx#B1vX9g$m+{gyfus9^gAH`2MDBgMBS+&-xEMRUfnm}HY=c5tDuaAdeo(Tm3T zFIz;=y(@dNP!W@VFxuc1q9DE~mP*UbQ5y^9Nq-gY7_;SGGZtQ3B<^_5SeQe7M6D;l zHol*8pzLWjMu=mB4ZRv0|7GYrRc))a-uP|Ll#LMawy?7SZ#*Xt=M+3&HA_9kX4+u+ zM7#F=o+O_Q&))ShS*#d{`EqEH2ssBV`?2WT4MJJ;=bkf=(wfdy#tUgMCW&_z{&B2! zxm?t9WPi?9XMYt=>|CkZglH zr3+0!>{wG|Nigb=tk$}DDQak zNOM`RC}9$r7J_rIB)j8AD6Quk=&bXJ-%Z8@-aJ1dSMYAcao!qT8kme+@*(D)OHu1A zsXv$Su(T0>3yk?uQ6BX>QUQL5tIRcf)akE4Sq@L?j@(@dBZxF&RUT8a|J|S&GAUkw$vXP&S&gD_t~lAFEjP==GQ{3%`dK_D%_G!vFZR0ebd>Tqdf5%#6-A;Wt|qg_u?@U z!TLw+!npm|M!g+p+P_Nf^z?D}AY=fmc+wrqc; zEVHNU3H@R-CCer^c|yn^DS=lS(-_-r$U%C2LlrqRD$44C+2NhSadzKZuvNZe4b}Zx zIS$n&cN^w^IaL(1Rof(cyYgEz%9l*5u3K|Ly*5UcpL9$IJp`XPWu=(rBqj6{Hjxa{ z#Gt354p>>((lwHveDwQ~naMjL<8oJc%&&jdfldEZ*u=PvA9?z;SbS}e(@E+|?_!c66U!QSXt0PX*J=l)mE#1Y8}MHXm-UNn~~udy$^z zH!u=1QD>@IaM<9SXgFRxE17s@XG9nf^D{4C=Lxy_>D}g@ZeKOjwW2>=TO76V!pft= zTN8NsL$@7&hE?4<)UOMCjjKbAgV>YRe2t7-u&tPA4~Ut z8t7S_C{A$0zb?Qo-WoJHhOD-j{OFcX=*|>ucpzrEw&97UDy&cHt5+6_`?ul=efty! zwWRx@i;+5fUU%?Zz%z~3yKfs`hYaXQZUux&I^8@kwA|L*l{a|(nb+>p5j{hCph17u zUp92K<2O^NMGV;|%x1h_DgU~p$Md3HvA&?b6+7T?2kDpCx_eEG*B$h;Ddv7Lm}979 z!7DP{crBAO+*A^yH;um=a`~UvCcH1Zq3h;wSNw@lFzM9X1=iR6*qG>|aKN)QhZi|q zStdNQbiP;V5dw1zS@|3i!;-#SF!CQVvfbyt6xSL6C*|jCn*UnSN4F_vHF{EL#p}JK z<3;A;5)19xDDGw6Rl0Ahg95_tnVEk=;zgEu~N z{cYOYV1zYE2a0Z68QI@qEJ^X2mIEQKIPmc~1&$tOW(%@$FEH>pJGGAM5dxk&Q&fxD^j-lnEe5Pksc5KvrQj4DB zjRYH>aqUp}NQuO)_3eh*R{Kljo}6yFvJ~0`d_!lZHA??x z2+yFyLcN_uU?|;r{O+U87zR{lSIQ!hHsEFwcpt;XP5#Xi16dh_Nr~jFR{$B6PkmOu zU(96baAVQhdsJI z9S&+`^9TDFx+mAyUl&6x0aH;e{wH%j3$@+AyPtxpr!{Z3k_ z5~G?BD3V_`R_C{K*w#PjRy0jNGz(oa-dgYk~Lcwf54%FU(aS-xE*HZS$;Cq$k4p#6H9RL+O=$?Th$x zs`eb&$Z823PqCUyHiQ>qdX`oSn}y)jojOa&QoUN#&p(yvymyG)>{F*SmWdd*(}K7XUzorhXkd9Whg6Bx(n zEHjEnlRTS?Q{?|Coi3+C9LlV+NgN;LYvOr(aE;oFyT{tKN=HT->rWmI2FUrQ_6DUK z&CSxV`^j@|w3yyu{rc0bcy0(fuN?e<<4k+cmMktUMT*-C21>zbF zMf@$IM+FltRt=p_{Jxz{wr}^$P<}QFozbneM<3lNUx=MX%cPK#^SYqA6%k1YCQs@R zohs!@VvW0JpZd6B8DnY_9u?c_FPK+FD8xk} z^*URRj;s!M9$n?SlG*0Lb+j}sE_x2S{V@pe&IEko%403Nm%rFGl=tq*>`WKGErwWB$`cpmEL5@G4EzEtFuM#tjBz3hSe!6*A} z^=i*$Je!42op|=cA!v%m^?&81UPE!Ad=ZvKr|+*VIo{O{3^g-1Rb~9V`Fr4cZ5xsc zaAz%%ram+cEM)(Q_op94V+l-?h4pfKxKwWIg zEyIF+ajbVYf0c6Q0uWF-JOAajKnZkpaEA8fFZzUg;B1`S>_2tarHXR4+9d8Ngz=*h z#_v*g6)8u2k-;s6V~=N9_!M&b3K9$RNWmrWw#r3T`fWWGV#kF&Y_&uI4e@7w$`QP8zjjo!pu!P()OD`mdIAFQ5~H~0`k z<*Ow&iGuc`Ne#!HqEmHcF2A?}Jd8txU{-ooQxlARJigq0ucX!w@!PqAkpUSkYn$`| zu`1{<;u?W7@7kQaTN%Xt!hE0L8;I;mjx}prtz?bHg6DHqv#9S|{upn{&;j#c)|JBeh7a>gB;XkUWpcM1H(^Xz{C2Tk zI>S>GAQcik1dC~`8Wsa8MNVgIrmpWJcn|CK%QF=;ozqL;}fL{BbARSsWO&%G#3nNcS`6fHu&nKPQ1}C z9i%OI8J!IO&O80~EW~?M(hkSXIWwW~WU=ODtXR$cgGB~{8jZ#Uc(t-l)`VeY>!)j* z?=B2Uu0Ii-v*bro1;ptT85*+_mo&L6CVq>|lZ2&>yw%+OD7sEcb?pIk<0|SUjYN9t zf~0EkQ3QraYcIX}Es>!1;NAU;N7JVlfQoB2@^*=Ha(`4DEB72kR-Ml!ifN1>`F1GkQybC~ugzM#*1)`d#((Uo~IWyJ~i0Ijz9$O=&JmFqp` z>eJ<6sCWsCtee^6U1?l`gaGxb2|M%I&azF$wIb)Rj6?d8HBXZ7iKG z$%F0r4~D_&y@F?^8#Rj)#TG6z$^{Yj7u(HlD9a&gP(jUa^#FhU(`^^z2c8mb<{ zfImTQzy{UA-A~$YtlN`id-816>+@=nf8F;t(%ye}&&&2z3?-wxAN=7t#aKJn8QKDi zRePSNIYB(d_u(H;P9P_lt?XP^Px*qZX2bqdH+B@XzznubUM(<^Dspb24z0Wl5tq^x zrYE@!zMNcWRd*sKS+0H-{vj;63LJfhO}txHvC!mKlhbbTDxk?b3Xx!4ZRjvpOOn+} zH|~#9`v`GTSXV5gU)%5(X_Te_73>KGecF86nao=s0bdRqKWL`ANQ_3D*1EY%?D33J z>7w?>)}A#cqvR}5sWqM&@K?|bA8+!6A7_2Y5cEAW{g{Q!6ycuQ$p-AC^-exhm%>J` z!!If9C;o)(it_|{rX;z4Pa@X@d_#`(a_He&JbPZ1io0BqYH#zjHrw5}>-uLWkkv9w zcs}c;J%XAVzgl64d(v*p<6zM@(UfEi&C2vuU!@)A1_qm1UY=BZC0xQJ(PeV~@gm%U zJt=qe_~C`s>hw3^gYTvzywQ2oqai^}O}fEq?~;{@K-c~&fJvp%hF+Yc$%(B7;E5 zntXF5NPd7fw@UG_h-#cHP^0#84wWryvEv)cqzspDRfSY3ASwe<36?#J-;NBR?{kL_ z<&FYIQ1U4Qbe2%Ee)paFDs*P=$!A8gZGrpGuYLTgdeQfh&ekBFp^x5cZ{?-Fm3sd+ zbv*0yTKn(HG?UY3S9$->4&(W8;_=j{|84TLG5k}#tY2nhhsH!J@=e2h9nY9yCmZ?< zO?GJGQ9Y~6!(4#X53Q9pgaB)Emesto$J`3PHqehOS~Uz+Ve2jwpR=}c+F}<2(%|>7 zNAj2TBKKxSb-Q7j0*E5kSdm{@*W7nLImep?+4i($btn-)Rcs=^IDEP{#v4X@ZW=Tw z6wTX^vFMU6g}bKBMMl3Gvs)q-KBphwb|{+XIjK7?~8MOfSXCH8Z?Py z8*w-(_r}Z-{lPjC`V3aB+PA1NGb*j_^iHy z)e^LmJRRL56GYkv1zhNQ4@%fk0ZiV`@C|yMj_g3e(*_2!^#Hr$rQ5efKUH1PayvkE zUhaC(o0d-uV~377de&EG{Bm@4E8Mh_W<0gKonGhvv=>g#hc#nq3mTsPhrcUNf^DA! zIp2lGKf_+)cj;}=%;;ebpSgG4a5r?D1EMN11y?ncnNt);hTxaFOEOXpD%uK zKWO%n5ueaEf|#@O+y58p;_)G4TaME({z;9FGyxH!U}kCm$c^ zO}p%wcD=K_PEFxpkI7MKj?wkk6Wr_#kK-#Dm(*#jk?dnpyQ8*knlXw=+LcfjZkS~< zmk4B&zXPAnH;Sj*?8uGi>eGEacODHfs5Na}ug2(VWQ}u9=;-E=%LMvp9*_$Owmy&iC{Tkhjs1_5FgJ#(783N2Tbg&h)F(5(g;VQB1 zFg@huN$#gCqefgDGs zl^=X{usVBR9y6K{lpr}mE@!Npq0jA~1SCm&3_5^Xju|t(ZaN&-qc}+c8&xOAh`LBG zH{E7L2zKpGEOeO5T2J3h1}(d^54YG;R^hW)ITlUz2)~SSi+dR0M0!T~w8= zi;_)}tjb-51-IzdT|jfM7}r+r>ThcY_Xqk*IThA3+B%_h2j>P?dULg6-$a7(`Q2-L z&HjUU0oZ)%aSOlB{S&e9gVaRUE5hbfr$qRx`IU?~Ple=7`I|cCd&<-59r6*iW5Z)^ z1#KbUdRv^qJ5HU2m^Z;3HMv&1J8m}@Yp4}A7IP(znXge*&j9d#HT6TYlb^PK zx`eh@ZGc0^(lE{x9Y~IP(^(~)3xZ`CkqQ2o2_91_9-99(IaISb+ zVJ6U~_3ION0!j1Os_#2==K*dQ*S=nmSZ#WNW~-jtaQeGjvb+)ImxVO(Kni||DD;LPt zJfafF!DHLAtNb;n-eR*AmYWEM!KI$M`An*_*SNen7dYmjH8VZc$)AtLJlrhiqo&)Q zntU*jt)tx#+|FiTQzA)KqNQQe4jHYkuY*3MB#oQAia=&?eUkNC_2fO@v+pH%ZuHq1 zoJ{Vf1b?mHhExr3Z*(*}{-=mK`Do#7zHre|U%Q#r^c{|(u2hRqw?*m&wBJve#Z3y_vttW=|AP_Re5aK3oJ1QSQ@0v!`3kD=b&U`l*=3n( z(R+ZIIK@W%_hZd;986Tkm2$8fFDV^Ne8-2J0bLHrG>$s6xam!_^2$VBYFaILA+C(h z;&}q(X*HDlSsrhnKWJo}#!kSiwfI0fsaD-~EqdRSTcuHTJXWpzx`RDOQYnS%422Y+ zbNd-1|F(oGlCn|H_G|_Y9%v97t#%B9k{qUGg!n;_i|E;FW$HVN`}Ljf-nJoys3%8& za_}(S8no#?w*)!B2=BxSCT*wHg|uT@PvB{8XT$jNI971LIa-P`bJ)n23urkCA#bixs3V{>Fs=#OtfsR&0eEe&>Mu=8EyP*1lUfhOxN z-`O?S%w;203l9hW-&NcL)ftbvI-Gr@oP&6M#KIO5wMZXYYv~+aLfSXbsd{lLK9(yq zW6r*#C!C!OjI|M4)%e!VV2_NXd>iNMsyQdu&iOsCp9LE*a)wim+pbC0Q1LV$Rdoj zhAt*{=DIVCPm#^PU)@5vahp-GzDK}uex{x!Y3sA?bYC3E&IC16^V{1QG>l~5<*l$G zXw4>RyStWwl}#6;;p4%+1?5r3ZR_ZK!#W;51!?n5%}dkHUK57WMU~ueibh#+uqZx? ztE@2T()d1{-47u6(?Y&_y6|16{;1S8SG+bbYtHZ5cf+XKs8Gf6*pGrk_rTIg{P8mH z0I{tP6TwyOiu)!h({HZs%wXGCNtbI=V7d-(Rzawa3RmQ?7lVCrPhhvx_gxKj8>L2; zjfeh4dAnb#Nuw@XgeiLW1hU$^YJFhvL>6P+{w2KfUzSjo&EQ|E7g&lh^&DuUhRv;d z6eGlj*GSoA2KH!^Ei#AAd(o0}x||k08xz~>Qc`aJKzz->H9|Wynbonu;6Kt1nPa*5 zbx$$$%(-TW#?ir6RdQ|izNEpN?{^FN&SnTv=HmVPxKyGtN`mV}ykxpwA&BkVNutzI}B=zDA)84LSaq^x^M_1?$`*h~g{ zN!SBLHr_BuaglE=B4@rd>9B3!^G=msg`>AE(aeZP+Ys(umtJArFG^i}Yc zfBw_;`)A_KILDkMnB<-MA4P5kum3WyZZN6~K;zKJ%2IHY6WvbO;^dHebEB3@e{C3w zm%*2NwborT2n8;Yb^5_uSeI~Xs&Zd!kFHVD!W_JN?}Vogg_ln+xBn%3YRb5}9hh9v z45k^c0g<;f1I`3q@Ri}Z9)LEs5vg>%O{1HOVu=xVR^}%@qhN29l0utlyM9a!9&!ZG zH|Tv|SZQgsftOCW(qY?W_b@cn7KT3)l5yl3O}yNOEnv`Q$xOCKJ;xZPnuf`q8#y5p z;=%66gOc?Q!vZ-q5GRqXAY_wrOz^W7to(s$j4)^qE9*({cvIdP>e376%#2bBJpHr! zEYhpRdZ@mfHTN0bsrT#uVLuUb~EEw_1IlwkAS;#~jyfe^jsVxi*<}Jc`{1 z0iF=|3JkK+;C(wuiq|9V6{3ItE2+RkHh4FFn5BwLlnR)>J9{t!8C5-5V|1hgaQ%=a zUUcn5#IKho+;rF)0HV#1#xQ2t@;@?+$06x0pHJ|qhug^!3E?=-H4a3dP4naoFSCyI zNfxqxCtPe2ktcSP7$2$>Nw5tzQY5ZSi)k>ns8h81K=(sVAvSLTx5_*#)2br(e+-mv zS|!F4+23_I7{0Yz4z>&l%cLliJ~QSuo+~k2zIU{86(BGiz-Cpzu>LpiP$Spb-%!2B zDG7?MQSJHUIXI!rv1H3xV}Vs@TI@{ORe6BqZ^e5ZC{Rb=Bqvrigs>+)@#?!ysr9T7 zC>0mfxjljjv#}1Y;O1+%2pG&1dBhiZ_gY9+Y;a_~or*L?D6HP?NwmVVo=@81+ZQT| zG!8KK=ql;Wh!1OTH)}HF&J**!tqR!Ur=U+q%}JhY=w{!8axp7uE2hT#kjKf`VRGBO zwMrbQQGjsy@}W;AxG}1^7}D@rlH1B?O&TSddLKf%v{D1}?_@X=V^OiS?aNgOoY1NA z-{Vr)s~zWA^r#iR-jtOIjt~akNZ*|MDZ8A(fN}0xWQQAN&*Z{@Dze3fsvbKP&eGWJ zs~xzOZrPip=UAI?`pq6!9!PyeW9u9@4DoI$QN1P!I6KxNxRGHlY@D>jw*QFn_LqRJ zrI=Jk0$Pi4hG9WIymk(C&n~m(r|kASh-^0aRZUamchqS9S58kiYrd`<1UybLfUp-( zBxVzhC)3nF?gfZQV7dRdN4Sv-dz-BQ@5N5F1@I7q@nbLlv-MyjaOZ;N;!JP6>KXqC zLk%MTo{OwTYk{#z7hg=%4A0|Ur_l!~Ay*<>#pBfe$hIA3CtA7<;-h2(=M>!Ow@+Il zg9e0xbmq+WsMWsOuaMQM6M5*c)8DnUTQ#9h@A+IVm*QQ2m(`(jG%2^fi8RSOr~NHv zob6K9!amJ~kPoVcs?!*miWlqtt$9>X%@^^M3`q6=!PnVAIaqmqLgU(4$(U$Zf1PxQRTO zbiRx(zd*qS5PMpw9BG^7zrxRbFF0aoI-u0yd4Fplaq)b4PIBo8ewb&fJc-wDJzZ)h zACH6}Ef!V^nEe<_4v*qG8e7*?_|!Bcww{Q~SpC+&Ppp1F_7C`FMaw>|`>pO(PFD)M zb1UMman0bc*dQIfV{YHxN92blB!5oG@C!dw($G|Y_=vX+NuOK(^WB zz>Hh`?n1>ZHfN?g>MjmV`y1*WVadQBf3p8+s@Ue1#g)6v@X3!yu7c%;Qm$e#N7Ymj zB~{AN%D)>s8ax{0jc)X|4_-r&|SEHzYwhz0r^&ss|Vc>lV)}}8S zxGZq~t7dINfJZB$Fw1bIFoSV%T`h3g>J+HxTqkS%fhm(ylC!hdjOY# zzj$n~`kJpD))p;Jv%GC}eco2zu3jJBSrCLs>nv>|YlKwue^WNWU&wQHn>*1&Y?(AC)v@fyagl%`4R z>UgJpfHT)D-XX7H+>m*vk4&==s>MY@HAkx5fRQsT4B(%ONX*okdVK=Yk&c=jST{$N z;dGEU({3%)$hj2;kJ>=OSM|}m49_st_fO`%(vXZ= zB$WmNW)4A%0yB)Ik+mXk21fPam@oCl{iyGa`|n0%0_8SOSdC^~C+OW35)FH;_49AF zxh))5tA@jd>VxfDdZm0m@F*}92M^L0eTRbzeoW*Hu@Qbs1Iwy-qEmXjSvQ%&pg(kW zSx!B5i_2VDR=DQVh(0Q1TjVds!9M&(JlAS>8Bd+QyRN(K*{tj6#J1vSBlT=}8*_UE z=b*=jrK+*1rMw^D{>m+DY=*YYCt~SYGM)O#xgCUB?Ee(I%uo0E^YhEKjLu5 zJ4wCQ|0%7(4k8BhL0B5wIXXwR;E}f1?w5Idk z3KVovjCs?^MXY>rfCHi=ujd=ga=9;mRGtHDZ7O$2Z+vKBS7PLN8?z1mS?%JiLbo?e zX#2Mjt}aSXg$aPFhBLV8!~ppm0pqW*j}U{fNe+`kgqzR-8_z>%NB#LIl%C=k(jMWA zika3l>%euffw=pWIxd!Yjpe)2M-Z!dW!hZqe$)Lxv5$|VRE7zw zA4Bj{8BrEHp62Fta4v1CdyDpyxJ2(YLwz=a6OsI3uG(<)Qo^o;>liCyhtMS~0%k4E zw@^6st@(>K>4B@UwSVtQ>mN6Fcq!48yYQ!kGZ4qtC7bB+Q83R@ym6OFS5S1RDs)>V z$#w~*fe59xS@g~R6Tdz(-FXVi4_2~^XF5q8a5OnJtH(u5v287NDlcTHT3f16#U-4s z&eEd@zOd+>&AS6%`!344K&6}*&-Xm!eB>;ac4GSyX0B!6MxS-IHk!@tJd5P9;k|Qb z*x8NSI;7_*%BHLsY+1k{o&uBayx|45So=omj<>w~Ik*v&vq58cTeytl@}ENhPJy-R zbz{}xL$wOyx#d!c{yF)crD>NQIr$>c`kwE*L;*fiYjf)QeJ9s*_RxKXYsTDz6Qyfs zEd#)++~Kp!ra{s~*sGJ7z=OttxLx&dFA==qdDUdlPf3()v6TDy=C!-7baL+!buC}F zo>1m$f6hxXs}AD4#+Xe_pnKh#!W5NR;p^(Yz!{zNQ$!Dob0EpBU~X4_Z~sZFBF|>{ z$$~;+o`X;vGuHaQ&&7`>Pg=@7<6j zSs>b)*`ht&PIdVzK)V{SdhIV9`x+HhCh}_Zud%R1;o%Taet@Aci+U5d%k}uM+FGLBs5Zl;m-*2Wb+xqiHHHCOsyCfQMG=3%*>{pD(fqYU4GjV;gfUW&Bc%%NU1B*8*%Q=9^2 z`Q`O>;T>?GR!!i>K|37|wD8KD**lwV^JNd~LnFMYMn*Lx#eIF&sO`+R`PfrUGfVtn zqBzc8X1C`&s28u-GflqHUvan}A)BMU!I+yAXcgEupB@|`Hl6ymoXHPU z=B|M#)n0xz7#iH8B7gR=t*11i0)L$H^+pUOvrQeyde|a=YH|JG_{dpypUxB{|IqEl zkMd)>kOQF$#ThdMw{0ipQDfjA>lE#5q^P$ty?EQ0Q-;nyB3=&4dQ9UbR%gVry!-tN zAic&R#pN79^C&*L*)+L@8jrHTC1;B=eTLJl$|q_ZMvW)adXs}3s?1EQfgyk9+G@jV zp-B?6?s>)z5IOXO4%xHziEV^qojW~iX@r*x{%>^!A1PAX-W&RXLVuQ8zN^{fv&{Vo zR=1W)er!J0x?yFE(9;2L!4llh43gkySNHpxQx(s2ku|Ne8}*lz3U$gfh6j1;5k`1b z=TD$$h^_A55-V{2;#`8$ot5AcD8HR{(l|qjhki zZTk!JrJT0BL$vv0uNiSyAo;ES%aaCf|3mojH`&u-H0J0pVkDZrHIl34P-bsz6z6fW zWN~o45cz-xw|1YgL~O|N(uJl7{IiRSLV1LqaeXvSaG}0+PHjBqRlj;%RL=VC9N_0~ zs7gwrj-hApEecpa;iH^o@u1BaEv8$CT2WD7vwo;+Tr;|bd4gd#QunzQjbCLtEetU2XBo3W*9K>S3Mu;h{(srm66PH0ZwpbUYi}}H>O+RH*YbVo(!1gGP~)i zg%NM`?i{HeK>Rj;T9dU`qav=>s01a~Ao6yM0e>&S4S{9JAEu#C|$V$PF z|FM}Xrk(OrHm=MusnDLTJtl~10D%&opFUB}PAKy6h!lckv3tF@O)hb}rkc+}xkCXEVzxCAUGZBpN*reO%;;Ob$3V7e`09D->XS=7M!s?)8 zOjD&githq7jqsvMBqaz5s=A$<^sxv06%KO<7d!m?!H@Mh>~zb@0V$xp4#%JF6+=?s zYV1Go$|0CvuHMvXsT7o|BCtvq?RSC3jF=5NvGg&4{<}_Z?`$W^b;|168_dhMTeB`M9#Ya8NhjavAEU~OG)mEf>5j{u~^(Ws*0ZiWZm0RlN3Iq?XW zbwu;J`5ozJv{?}P+eoRgeV=nkrc6c^j>eg;VsU&KmHBnXb;OSP{(?QOQO zjoUN3TS8?@SBK3jmL)Oqy^U>%qI?BDs$^BVyH^zSZ`^d2RyYJN*jXMO*3-Lfy^8Ip0J+xi;1mG z1jlg0d|=7y-gU`1qWgCbYe4$HC>t<-CaJH;o7$~t5YvDbT+$rx(v1gljE9v7z}Hga zY%Z9y|Eb-W4)kb!+dlMO|W#q z=Q;@M-s(b<*zY<&A@J&cqEA!9%rbStctP0Yj*nD$o2m+r$%QoJKx*`*MKYThPjws9 zvZ#wUNhNmO{3_yf^`=q5Q}oa~)SrT2OF}KY=gFpP-Vb`yOVJO+Ru^)s=XY4wa%ds! zgSic2uffZmj<_AH9_$akvA&Gi`a2)*Ivb%t7x%TBf#Y=fdcOgk@ZJheEj17pf0IiB1erjpHoQe+Xk-N+2 zpP#b0SKAo19qv#w>o=wr^pZ5Jz1D=(=TbJV^;kkq2s@LQyF@dZzI-UH6 zc2$b_d*-bnHZ5`~rQ}QVj&T(F1y$1#=ZEBXD!UX3>01XQ`0#b+Tn}7ktB1AQoY`cE z9$6JR&TCM0KK^99y{*w|3XgT+274ogdufOZAqyzIyrF1!Bd`?kH$v9_KM%^<_Py!z z@JZRbc6L#HoU@)^72$x1!xWDlJ>t?YN2_atnuIlrVu0);?#>e}g`5x$#^X=R-eJ%q=e#f9&png zn#oGK|9ISa)2F9g(Thj$)E$>y+IvVxz9^`2fOgl{dfwT{X6O^hRT)((Hu~B;(rU5=HzWZSo%rcC7!xDc@qomB7yLmLl~kI8jSfco7yM3V^zQzbLb8a?Yufs} z!U#d5(ACa$ypj!r8=>Cr4F(a;)~6ypj>h2u-|su_j-FfxPW#2rci^1KZa|8cy<|($zPv@+ zDCjXRxN;Dp$Lz{s(ep=QXX;r z_`#V4fv)<6tVu1MMk;Pc&~6_~$?9Fn|M4F9J|JB+l4MI( zGbfR~@UD`k3_TGbpv6br*1Q@}=DWmX;MzrfgI-EB(*B5?+je>saW4#l7( z3C5-6sv$lr)N7m0Evi{)`tp<-ekUiow};G&sAr$xtNc6yNV9Y^$$}Hh%mP47R-gcP z-iDa?gmoFmcuzrM2+%e9sB2aK$@=P}&4#CAR*=C)$OPkc;AaU&{1s1l6toG_H9sw8 z7TF=pTIm}b+Z}5=6!+BuS&$iEUnC;+Y=%?4k_9{CfPB_Ke{y@nJ@BkmG6gh3W2sQN zEn%wK1NkJ+as>lx4}r-y^wI5Kx~gDn5ST%v4F+GT_-utq-jQ+&F5 ziRjEGJR;P^hfmn1!zf=`hPhS?+^5Bxt&CJ+v_;27!_B&S0w*d#AkjKaxv&=!k`P{> z$@sXHPtClQEdDlpsA-k828rg_hGc|mr;$zoO!ETdAR+}o*9{NPr0l{SZc2{}-e z=o|#t*Twcu=QLE-o=L4DX)j&QR&RC`x-CMJv3Fe+5NXM>FK?O)a?Hg?dyWmvcH}7M z3Je*2BlZ(sNDs%I%`N1tio>Kg94{Mh)3rFG$8eggpYKHAV@N(th^<&#V2FT@UmZea%|ze|0bL%!pf zRQ|}vwKkD)*px)8WT$jm#(D@CYCT39oLv_Lgw5Xa@*ri@%UK)eXwPhAJm&to8K=7w zv4;$SvYx$^JjquP+5Z8OZa;f*o?4vmS#+{R!oL>CYuT&b73Mc?EaPX%!(c%hf(u@% zm3sHOLKI4E-R%smeH1C>$mpI z#!}$l;3`^V`QwS75!@#|3NO?Qr!hY}i3V}IZ0_0b8CNqA4%jQM53e=4ovErldC!po z^{e5X%{?1;FQ}6ql(T`TDd4m>9$4Fq!dQ**6w$|9xLas`k~*^S14?I*^ON zp#&5!H8-yA^Ns6|-zc%ANHdn34uu>kw{RqN;G%Lk+MnAssu8A$-akLLLCIrLAn#Fx zYx?{B^o&>eN(;S|K0aPgsTSZGa;ei11GejaSar|ieW zLHOsoT0USa9|_(Q^XwMM@q2>Yt)`Hzpg4n>UlEpnel&25JYKdR3lo$&o+O?*v!u#N zRh5Ic74Ug9E_9UG_)pN263XF(K#2riKO$`RE&^Dyr4Pw#h7Lxd_BxU^H7p3U5tTSu z$!#Yo-C5fy+q(1CdFLqC$JX{1cD5Vz*-dtN>p^R;zU6q|e1ASEY34X|4}(eJQ$2GK zbPD@&ZA{riB1@dHe0fIZ_7vs<0p@iqU|hTtP8KBSY2*=}pnacOn4%7hjzKK_6c)@!TOZ_GGF^a?S z>L%fchqhLEvMDpnQ|?atYnVW)4u%Hd?l=W zjQ=R)Df=TkJTp1Ya@Kb(bAOic9(B{@LkPi%>O0Ypo3CRX{M9XBWqTG;#;>=pvk!Wg zF@9(}(CWMxRf9_pI=~m_3*<*CR6iRlWgjKvdbD}Fm#R01dxx<&R)|2R%Jp}W46aCI zk!P5jA(HlcD@9xwnK(yICGf>xE93-4&m~-9p25*$7s=Pcdd)w?#{wOjXvRAyTNFxX zma^;&tK)|4$R2Vn9ba?5LdkSTniHF*-S04^_sNS}H)_5{K%JSsB6dNlp9GB&rlQuO zRak*7N4cz$g#E!!#o(HY!(U(UuCupF99&X9YyfUbApx{TY00XZVs57aUc;aP+K+7V zbw+PanCW?`e8rqcYtM??|O6Jh&>8-oEGG``7B>&oMQA~1l5Qnsbg zvy51h4Y0tByIAO6=syFXMfbNHwdXi)-|v zkiPRkPXczT-eh_V-Ey9-YA^Q2en`pX20Q*fU{@7*#a@ z;CRd($OI)x-W(5WlJi4ep#;N zfR0ozAG%M8BzeLDV$Svi1vxIS;{rv5vpYWyyus-v(+<~pAm`3})ocZvwgq?1N8@zY z2J))XNCUo?kW&Gz+VA91bYHJfw7tz3?j038BvbK6lp}vwaI-)&R|s&Lv$?jm1=KNN z%ZBjH<@@IO(hood9pzuHbvE6kN{tY7GvJ%cwcxwKUhOesCwESRezOz34ddV=bzxS$0aDU&l9AkajDrrsDxm3)m zx~M1<+0CLSs1vCXF)Xz8+`IEA1BSc&OO^s(O!JNI?T%#`hoIbBL*_b3RTdY?VY|O1 ztxw-B2M|{eC9aAZ1O}#XQa>Y_QI=qci&U!jNDPcaOB&l>Z|T*M`v1wt zLs$~E$J)0UM#1EwgV)lR{D>n}#4j*J$OTH$c=O`&*X@Igq=Tk5u{K&^?>k)WDbxuN zNeF*-Zub?Zz`GGr?Oz$&fVl8M$$BhPe#9G%sgyX`me1Ud&jd3 zQyZ2S;XcC_!;ZLQa4}%hL{B3FdARxGes83}5?C^~B}E}}qid*ExeG&T=qOzjzPz$s z(&gTIDDvxrv8&-Lz@%s_<313nq%&7i_Zw~l{rz_-3-iJqEa#)+Sus)ygC3%hvtz@*oB2u*iHqU=a>rCHQCaPsw#QPfFRV93HuxdBJzS*?(3e^U#{lyqfE^_9O`R^lGUs0J4)E}v+d&QO@YCLhAz%l?MfWf zA>wLsMZu-jXK12D!RJpxRl1&8R7?ur8e=)(pUObnM9Y6>x{k|5a4LO(aw!5|x#cG` zFH;Rqn4f=V*lw2<6#&8KrvM4`rMzC1@{XEns(j`M6~(a1h^{-A?a zm0BG-ke$%ES!FF+Hd#xMRjK+wgtXCgdRVmZ_PExEXyiIMB8+#+jfUj-98i4Mt{IQYQS`4*$tmIzh9e>YlZslYl`7861(#O|%(Kim zgKyYJ7qP748kQB_cJp%U(t3QN0^H=Vl;#qj>~*61x^1zEjWua$$f9~IFsXML{_&a} z2sb*wJ&Kb=KVL8g2gSC>HT5V5vcbg>;t2@gi`n>xuoJqcV;3(iE-WbRLuB0cC@WeoxXkcM;W!s_O8VdLCZr_2-N`CN5n@ETwb zT4D=MYX8q%=b11rq8fT-YU%Yaq4A?)rY-b&Qwo4B<#vt)+GYPfD>*oUd8B|Z^Eaq3 zuzD{gy=}pn({xwxJJ{J=#u@b${w8$4+4i=-C9_bc#xC;V>Q^+ax8?%3x6@E!klr^3k>9P~E}=Bna}d_H4}M;%Gw_)9$QXv| zp}fC16m0t{2>Vk}X&_v28IBX}qP{TcoA#1-a(b|-FyT?0eQA}rPT&m1Ak4SlHTR0E zjSSvy-w-FrV1L8bUZWKk!DJy5O3$TTnGP4PFyH3FwFR->qfGokVFp4n>v>lkSPJ91 z;tgJRn}IJMET}LI+7KLfK3ljDrGpMxeCoB2$E~r3sP+eht*}VS*Omi8fcbAA?ylXf ztiznLSKQk<)7hvbkmnJHVxZ|OI9Gz_sZda!9gcXzbwQ_q@&egbtjmqr55|u>jMIjL zdZV|&)jnLg4@HjT+yz8z<(B?qaZ!(}&}Unkl)(?;Ki#UprF2K^i)hcvL@K8==Uj+p zmIvMmQ*Q`Y2c~AoKQc8(gpx!sLbRt5iK*r)e0yfH_l5*0Lt5{*cZAztM zfLr}z5pO3xDrb-EuTk(d*gbGdk>R_1uDddS8%R_u2q*IRs~NW}9sKD@ufeL_RmNSo zjxoM#1e|7*wW#@PY;!#-e)}LY(BzBVKGQ?4rE(F!fAUlsA<-PmxEx!2w>cmt+Hhei z%d@Cx+aKqdH-yol-J12&&Z3xK4eY2>wwDZS+Etsr;Lh~R_v8`UPTC>W@xZy8e6b6)?4TAD7sEDNsWzmD11mF;i3O1{U%F1Vsdii#zCR0$ zheM89T6(EKt<-@<+MQ%kD>aSKl_d&nGZfS5MO5tA-ODl#g+q<>(Rh|rKBzd_~Z)3h>zjb zJ)bxUF3yqXrCrHxCfU5B`kze#8@Creghn{{Go&QQ@wxPxjXD64Cn}TgRgoYkTmuvg>yu=B$3JuDHltdwME-d3_bh_TG zEpB%wM6}L>Ia%$sB3Rf0JSNacY26hrr_Ge!ew=TFp~!RfAR3xWyxtSdf0ve^^m{|k z%u0LE9t^qCE()4;)q#inxJ}dTL6=fZg=2cI$hIaMTpFE;)NFht-93H{e}yPyI7lJ- zAMm;dGEV#H6;F$_gec*O_Y)VbzeVE;%f27TNm*j~k=5Mof<9~`Pu&%1%Q~7JLAPro zlyxoKAvfK)Xkgf&2{UDu^OP2$yms+oC%}MQJ-7%=soIQkQ&doZ8U)PbCyve8n00cD z7LjJ)SrWK1c&3FkSElW4&r!2c#rc<(;h}o%ZyX2aU6!bkUYBoEF6U3tspNxi%mFfkcZT)NaFc(0c4x`d=TZjA0gZI4Gn|~I0^_o$BpSSYJh;6xz{i$lQ zP30=w{PIzeDmwT=&^6Yn6noh)+?$0`>8J(2&cp;$T{z; zmue^`|BjuswvyYUIXHyY+LxNpDH8^_YyWc9<_9Go?Yf!KvoHrz_WMUm-*Y4 zq8A=&=VN$PYCR;?|FM=vfferaaECp&TX>f2&t#0>+i~VjYLH~>R~yT~Mf9)S?5v8< zN|ES6x4Mnt2b51zTh~VVg&Z<_Gn+{Cn8|`^_pjP2Q&qJO%P^BhqcNiL*5mOM61|X0 z*R!-yO&`>p^7YYgVB5vPcBsq|Gwe6He0$>s`BoA0EIUr|Z#bNu$!!QSWOIMBNB5YEoqq#|aL9}V6LYIIO zAE=0BN)|zGYZ+BSV&ld~1dZC~PiGUQA5tA3wwA*DVB3+s`WfFweLE@Z*NhRPY$#uv zRtf%E0^Lls|JwuSjCL0rwUin}D&C@z#isr54@^Nu#Ko*;&XrF1DGsw8eKH?OOjiDW zFdfQ2ptPy?W%ELgP?)CuW@AtB2|@9_4f^`N$k%wWjsDBEc&4yy>D(6VhsBmeZm!nF z3C|r4O_Tg#Y)~1-#NlMA6B18xRn-i((^;^iya?o;F8_nd**t1&%RZqi>%KGkg{`u9 zCgNm?@&0-joQ`&fC@l-6AYYskz2;JPuftqm-Z_KG1WNhg8JYS{3 zGO<-Ue56(u=HkBvRVsYonpi$Ptphji(cAW&ye>E$IWZ^2I_zPJO_L*%8Z9$iEy`%~ z-4Idr!D@Oa++Rs!&aDPG*Lw4w`6sUw{SfRfx6tYF_^zYnsR!n{6-+aBuNLOWeQ4)# ze2wkDMSwY~rXy0vQ6EBDsZcg!$Fe{zOT(VUslV1c2gRof5XTyOJ^lr4l-dmVZL2xw z4q=ZX)Oo)IKlII^TgrEQUKkJmRAIfznCM9#SEkH94w>A2`@?7iOHxsM`8<96&q2i5 z?`5_rB4V?ePoqkt&chBm9aOt-mz=W60r&?xT=E+KZO!ZoUQ?a;H@@c5V&}|4*p<{B z75p$gW;GICI(@+mYNh_FtD~;Sw>+l8#S|zGn-l+pR^5Rzt2-XC6H1wK|304nDK}q!#KT8(b6}=#XDgiiLf^pX>~Fe6eAv&Kf-zxSKU-rHJPLm)C*V%YJS%uf*gwW2mSxaPC`O3lmE~r~F~? zYEhrb@ziTgV7lo)#JWMueT==w4L@BSeOqLH^YQJi&>GqsCB|9^{m2G9ZqHYhc;no{(3@EO`*SLAxX z&bZegxh6i^@-SKDr=XXgszS=N@6{Q)oc1PX<%UFlsT0pAmOv8|!s0k}+gHL zCLOx@+t`c(zvYpr3WUB^tOgAKG^ zExbw~k10!;!HX9KK8$AB8v95Yi>EBQSm7n544J3kW3802#|i@*%pMY=HgdgHkR~Qp z{IQKXWBgF-N~yF1JL4Z(N$`J#t}Jw?p1TiX_7-gCCl!9UvgHosj}6z#Wv?Cc0ahA* zeXtyujlaKEI?ix|u<@7A#Mx>zCOJyhiLT>4t5a}S?2P{BaJLVWynL63Uc~6E%G?x} z_0FF@r@y4y+SV~lj76jI-Y$_AQB>T$leU8FeY# z8d4*xVZCYOTWYwavm{rMzA~{E5}}+Q4m4;p**EOrad|Oy$AxWdRr9ri_G3i*n#>9hCAb$kZLyM$fa{;4?0>bX?@+ zN@p9;>%1w7;n)W~P)Vq7TqLxz_-aXdN+f%PmB)7CXgB&Pu*{+A1WU*Bp%+VRR563q z!CmC1Y~P_il{==*G_v!Me#z5&A)lKX_>&$)6bsYI_`gI2IERS|t~&p902_jPNrC7EZn zh2fL>P?$p$*K3s&#s2Zo?5GJx>*ze+dne{6wH?T;dJ6r=6}5M4)`RMy;rA-O111J4 zj6M8c5AH7yL4AKDP+F?S;t}lrE|&D)fqS|^2ZWiysLBQX00BoH#-nMzU~YEfjI#OE zX$r$Z+rQVv0tTAUCSNK6p}_V%kxaF$jbUnbH&$KdM1t$+N^b8^Q)KO5%`Y|EJLvfs z*-c2&?iq(~@g)yVDpWBvsuQwC;=5rdSSgl#!3q6U8L7S5X1|&8O=^ANN~8k9r|q~0@Fq4iZ#atuj=3^7+{;q`VHGOE1O0>l zp#Fw$B&}BtUG8sftUtra$Ao~97AvD~oLO66n8o;N&EYG4mb6ZAHo#lO^Hw&~!H=q4 ze{5$w-5FsI5>Giw9NSi}uMho%PZ{QIUrlulcs6}H9JdfH}H zxmDX*Gt6qHWXh@nw&AduBf%-`$}Bk_T!DDF2er)_K5%B%bu#^&7?8Sa}uOV06rp%7FqhurJI zjLuDd{&SnLo+Nb@d0AL!pk!V6Y;awM`S?9Rgtup-`t5MA3fz(p`c$ZLqG1aJUShM< zw(Wo>@%?v&kXzv%JaLQ)WWnT%GltVV%I*5^4#*5 zBAv*|;Ce!i!r z^3ZV7VnD-pUOnvvI4JbNDeaao*qa6Z=!}*KP9a`4msMh9Hcc31{LFnn2GjaL9l!s_ z=uKmNEmZQP5Dvq_=e8<($7U z`TdY)yGTz`8OsR|zX++zj!p0~lG3)SyR!qO^BbAGP4 zLggreq4R!_4F>G<*-(PfakpviriY>ODCciFWL5uHD*HE!El>}uu0D6_W%91?rBWE@ zyIJBe&c_AkGr&(U)|k)X@^W2%!iS0_nHW#g9Vd+SG*?aTBg*NhHq&a4ikFWf>}m%e zpyNCr+X=7UmZ$7Ea$L%-Mhy|8hk{G-cIPs9*8ei1V>b zzPk1!1XaFyoAe-9x&7I9c>3nR(ko5W3*T?zV*xZW$UnHW+2A{v!Z?y)-jOFQo7YdV zi%^y+E3&5Vl;N^s#acK{%XGBppnA_mReSL3hp*fEwK&SCt4OHt^W}9Yw??ENs;QB12AV1`IbPuY$(9H9 zQN6@to?Ej;*hs!db<>Oo!HCw(12)HOY6I89EBwtu43F)>d0s2s3|0DFCA5V9(|x?W zn#IExOib)l2DtED{Dc{IT&`klf9rTDcAU?EFIHiVml+2wAuz!C7RFBf!c-D^kqa%5 zU)PZe1&e6$6fNp|} zz?6YjkM9+;ivbQ-aQpRa8}za0`&>iFJ^Ze=(I4&UCdDg!w&>^xi^P<_|jSBnt4d5 zs_9O6bW}DiFa&n>nHpPA1cnENg}$?_OPO`zG9qLEh&z}_7h)Y~L0i5VWJ48on7-Qi(^xDm#RfT%mQpAVA$?J}H8Sr2kJxP~ zsQ#N($}%5uh55HT2f5ldy}}MP*TgR=i?mTzrbVDLe*)i;uqhr0!=WAs7Q_*Q6HB0bJCoZL0P_o2@R2a#Z7WCw-a z;>1{(dUWD=4K}g1_n|74qF$`HT7N?-r7-Rje{DUHoil#%Stgh(y-%b&&hD*B8;;K5 z!EnO!LHY@BgY|ZzcY>#Ds~(P)5`1qur?i&^80&jp@T`yn-?8G0{kW7l=~Utb$5VBu z8Yl;mP`!Q6eBc#}9UE(GeGhiLk&cqDO}l-lj}{#yce{acdk!Y*fo~P7xHjrW)t1i3 z)GV`pb5Cr;?W1P%Nj+-$UP4INZHa)+A?XN%-$%?kk1%asXvgv|{gUX-K~!}!T4=M&NR6b0%{xDr zI%aE@HR0212M9xUs}}~-YSUA)(iWt+b^xML3g3UPjE>!^Uz>mhtiR*fe;I9v>Jh@- zGioX6&JR?EkT+PZ`T;(}Rg&*!W$U$XI&th3U+Bqdv1qH?HrZ{iH)^agw$Qq6FvFE} zTJ*SXE~8oT*#-730#1RG+SXT1+futL)NXDu3;BL)voKw5x?VEgRzDzaYV=`F5uc*x zc=}UEvVz(5*~|K34aZF!QRVsCSK3cK&rXjfAMBTWhB3eG5u&a#btTpt_SVEXrN`wyWkrqB}>~sWp_0sYq&i2 z`@xEqK5yDr(F|`dvYow+hRxJich1_QoZ?d8JMDp?Iw5Rb+=L7Qg@!#H8W^Zj`46#*AEv#M&e)whI; zmq|J9y*bO;1ro#_p*WL`rRD&T{tL4wJ{C$?M*QaxCBkOY7FNy|WxFxA<~w1zt@ibC zrodsbj(5-fbn#+T3AguHp2G1%|7P~wMe9=;fWq1#$NpBh1s*Q%SI{h5Vs(C-iOttz zx#+?h&P>4qQrf)3)rj88d4Vclg229s%15bjzf<5R=N_VY!KPd1MnP>R-Q#l{PeSe& z*(UB2^4V0L1&vC-Vxh~o;;g5G*mR&iaXWZXf{ydl^z~p=ip^W>qi)y0w^BW}gs{zb z>)FdnJ%sZ$1)d6o8^_d&yC1|JyO}b16f;~BHBLvROvP@}5mL7F>z?UZZk5TYYp*G_ zjdq|)tetq6hu(oa3_)IH0ETh33nkVZxlWknewE zHi8c7`7)Cnn9`dt^h10{k9R$(L+pURU{bDRchX@_^KFGaLiw)pRutG4m#1SrEPona zBEWZg9|F-xbyT$2lRdF?Pzrm3`PHElfpB(xteHCYBB})ez3p zUf$vXJN;%ZNeFzE$USR#BXIP0-y#F42wQ5Nm8OtK`XLTDo`;JVXWy^FzOCe-(OJIw z&5sXQ>U4L>Cn4q4y}|4ab`B7olSJcO%fdBl+b-FG23UTTF&elB_*$oJUJa8oTJaxq zmLA?PPX%2Sa?_3U_}nbuzsnn$Y^m_A*7jJ-Ui^kOE+whqATU|$tLpaWc*pe{Y7y9$ zp3n{L3_UaMe}hO8cH^F776ME-brtJ6!~DAu8MpFWpW64jo&N+Ak*NgHr zujs&sj^jAS^>!S;$t+HNu63+T)aP+z(->b{AaEGv5r24dwmrHXx@H1$NCABRzIXi* z(F4yUVyVmT`oa2MSU(yXc)#EwI7dYMsyaG+xUMp}Is-1xx~_7$;USNG`EX!CulGLo8%AZr-{^}OhSfW<(v1Fd=xq+D!SPnq=UlS%vum7) zq`gYHYV|Jv*-F%1avI3=U}ziE3ev>y2KN2f$_Uw^yY_a+a8X3_z6hu??+=|@OzJGD zR^;)47{@5DIOa(3Bv|Zs=(_>+?H5%72(htxp5ruxLjAbSPHq5o#0q{^+37d=OmSnH zemdRS!_}%Px8#o+3rGH@C;w7?A?n6yE9Xe+$b*FwSxC51Uof~peza%uKv4U2CUF== zCC`rWGEVl(t0Q(8kMw)_w>1#YMH?qE>7CSwUYi$yV}q{5iK1d1H%y_%;jBaaS57$%Ndt_8<~22FxrWl%`$JTd#^qSQSR@hi`QIBmC$9Lyv?60`(~dFsN92J|G~nR;D0`MMK49?I>sE4 zBv^HFEBZOZy6tD{)79(PYfoYFl_%eKCBqQiD)I((WsD^mh0_JYrq5v8AXh#OtDSJ} z3C}{@by)A7N0)pk!@W2I636L}MB)@v=l(4T_k*|rdGV+gWY`-3DXo+D_6$5|f14Jg z8v(3K`N}vf0kf@})y~@v#b=$BRX87p$GzcA*o+9U<@FNGs_H)$b5MW0F3d(GYB7Y} z{K(rxr@+Z4yca(_2%XOb=KZA_e1(@<#Glix-8~2~yaO&wa7&hJ-Qx%NO)i-(|M!7R zizvkc*R+&(B7U)O^N|2GN23EqQ^zX@S&3HzF>H}Y8hxOef;%!{#bjo%^kC{)xM6>A zj>y^cH~ZrUpFQg05^E^NFYh6?og6#vpUm|B;;YXg{&&YZpzHQbpjA(F-CP6+c5NsnhEF>s-zu+qc?(0t=U1Mr$?o+xSB&O&P`$Yek`l*U`V^aDOM8 zLPEb?Fm~LE*zc8|F-TiamBXg=^c?|*bpJ*ydxV+yDYT@kMwoJYT0a~3Y^-dW@~@1#d^S?$@r z`~>*}x$v{%(SHEhoqF+Nc~#~2rteCzJ__1+wUgYgb9tL>*Lm^O^x|(YJ{=M1jeYw9 zZX6&Zsn~GkXquPvpkC0sX?4MDTd?7QtvCFdlk>jW`i&(lilLq(>{S*y`qX@l!E_J< z4)4~2^z{me&s!RxbTX~KM2~12lbGt4h6aqeRSx@IENt7hyF>mw#GqBmkZeh2gOdE? zo(d;=XaA_LJXnO6NuF=N@0d8@xBd1BvR(ur-yvFBb0|oCA7ih69}=VRNlX;#Syj2i ze3-X>Ak7>TMLYG>_QQq)ro&8$YIgZmfbI{;;dkO;q@7d;4q^5dAr_j{&$3VwPJ zk`b7hh|p!<>)zqPKiC~>`}050cEO&(*__%Y%e<=&tU9m8ec*F2Z*(Ku5*#$zTU48= z=9l!=To@eskt870-BZB@q(5G_QopD>Vc5d+Y_{KO-9rO`=B*$slM1)TqAbmEXys{UEiIXqFy#&%ZGX5kuJi-Wv>EegZQjhR3zYsZ?=kM=$tS*vux3E5 zTaM%XE5BF@%$uO6p|Ql=V4!mWK&YT9*=w0`hVYC>r|1`uU7Wr}*B593$g&=_$ikwf zN3SzBbek{Ro(ZA{oaocxK;2VvDFwDH z)1D)LxiT0yTD!GMD}c`K*Q*6q2PT^zt1x_}k|b4)8y&J(4WwwdvyCV9+quS_n^;e} zT)6^RC*wnY^Bv8anUqraP`d<8-;cm%4+QYFI)?ujCUwbiA(L^EBS@mafJ9#9C<>5; zbDqcCnWd``D+*2YR1OQQI32qih7TlNaGSt}$|D=(1Tkbf>R_2x=EC5+X(7rML$1>+Px6$BK6FjHMkW|RC|-`TH=)PBThfriYWi9yK|2f6 z#OX%Zy^RH)&ta_2jpJ}YI&=eTcJQ2I$m_j$E>j~;onUdG&n4cD_1!G$CtR!uouZuJ zEAqr_3eTI3{=Qk1n%`o|cGP#0U2`70>S#AF6N_36cV74BjC3Bf&1x*;h|T2F&X=Awb`kh;Nz4m#03l6rOb5`em5ooU3{s2p} z!6f4#%!>B8biOt8N4!4;GT=30%Vu9-Kknqi)1{wE44Fj@ZB&&U70VBDWy{>~-zV3< zopZ&*!X(n06STkAWu<^p&S`2Yc@CM+Q+#$ACwF_AjKBv*S{4ZBamFol!Q~b&m0maN z^wX{p+>m$jajG`?*#0=^lBS`kQ2f-Vi7#EM^tXT&D|~YUQ^c$7N9!Ug-~4gIM%v-Y6(w?BT07b z=PH>j*>I=aH#V;FduMcdq$sS-%#4{FiCHnfH?gfv0?lg$KZ9tA zx5=kU-jtbzi)^@If{^7}Eh9P2oS7GRVWEoKlhpSB#d0N+kyjS@;JFIM-ip>lF{7gz z9Nj)K4etH_mzuk6p#++34BLretJ;M;>K{tAI0#c6y}eQ0d_src!#zP^ zk?r31dkRsl!%F=wtj=a64VLK0CSxstjtcf&hGf*z_;+afJ$VBM&HT`HEGd|S+~vv` zmvNV2E5|&?j-<;J>gJ~{7g1$Lf;&@Sc^>aD@5;NEsF9;E+&k4{N^4y_N-qk;W zI4vL}OEZTBw-4i8o0tcA{P_5T3hcG|o@mc+gaI@~n)`NOcPRXy%QO@*rmSt9Tv%PJ9H=#5fE6lR`IIJm! zozuS1Agz5S)dStG7K%YRD?*r^p|1Ic#jp>>C3Gsuy20>--44=l*B4DZ4K@L>p$k76 zD4Ssqyg3wXJ%OQ**b2abdryt_GSRA11npcONx!8B%Bqf;!K;s)ZW>wdL^Ht5D8hX> z{|X3Q6`0}>7n)c{h4zXh1q{;oDGev4{%{h~xNu_yc$<}N7XHp?PuEWB=tJIlut&)U zsjDQ^_~xm&}iC{vmTQC+(q z{;{8D?@U8BQ}f7S(8l#vSJQoK@gco8xFg%|dt?vs2FMxabp)nZ!n&NPo7s zPpxLO)<0jy?H=O&#K{keZCM1A2S>pAQj~x@f<^ z@woi@h#~-U-VMv13Vf5@G?demK}UFCrkSB{BlkzhXz;?^+D`W2EyEd+%<*9BW^fLr z*Jd|s=CPAb+k3oZHp#Zxvv!ULzz60?POF~K7VHBBhIdw$%$eG`QM+C}eC6d)$L{R)GM!&tF}ykZ`q#;05(mx9 zdMj-b2X7|0J5|hkL*-sTMupU8_a6UV>8-lh<=k))G7jbJ%iHb{vcYZIX>E1jK3b9DXfye#mP!&(tKD&pbEs${#L!3`aayd1CFP$`y3o1T+Y%|-#cG-yelTQ*m$lL#7e zGak)`JmmY(`(1VZ)$1X7sq}hAGCEsdZcCTWsZeqxaH& zZF!Q|b29ztY`q=w&q&^;)sp?T%5JjUgmXRGxzudr?o_^;J*$gc>yQ3P!%=ovZm5iE zLVO-C&l$(n>A##SE#T?{FNln%C+rvZO7iA3o)|z6 zxz@zq&z%HnyWU1CF)4^?n@&fyYQh8#UP(Ghyk@J4UP5Qw7zwAaA5`ETS6)x%&ukEh zplKs`_54`zE0?khz5K!FzI^&;ycC{#XHe>y+&RDdw8CXW&tJI83_Y_}CTrZGifR%N zb<&j*y4K$o5q6rQC+~&Q`m_0=>0OX#BpApTWMtLwCQ)veDd{{e;)&+blG(i^ciWkF zz90B9A~MqC^FIt&f}f4@0+N(_)83D#5aWpYWBw1D#i6vQPeru9sIqHym(RQ`%04mB zehHnMp#B2DRg@o2zb)&moa1#7U)KjZDy(zr;a&S?qd0yeMV2L$cM)_JNa#MZ|3VJ# zx%XGygT&^-fUQz7W8bsp^5~}dYl~u~= z+ifYgMjhDC&zN9y>ZQSR6yG(Ro$dPM~|EV;o+m`DvTV~6|6<>hoBT+ zcX19L>>#ds=UBdZES^*h!LTJ(^Pu@9#(>jLf5;ahEGnL7l~+SqWXh!|0duTP0d`nJ zff1^GhkN^^!TzQWDiPi{VQP)3bf-P7!%TC>M=Lm{&bNRay1Oiv#>o?%U9BG#gsb{B zwOL*f%ctJFJj2S39MAgUKFcj$Ga@I%ixuK6AiL2VeH;B<#V#1CR^w#Q3%EM8_2pxC zBw4l$`7l_8Vvd=g6U!fam&e&dMcSN^*$GsK{#Kth^iZ;$Gzbox@hukRmRUJ?T*eLd z?|$HcamLX2Wfl1K42%va}&)6E>oYTSw;eS zjW|770$JC)?*bd~KTqqo)vUQv?`m=Y=uW*k*FyBSXLj4Q?f@hw$46g*_Ea1xLUo`` z2I@1^zH^PO9dhA$GU3$)7rNCPeR(GqXW#Ejupnosc_l7(_De`-+(P)ndyZR=TjwoJ^tlXEj z8B2IP2dRi-cMIk`G+dyQ59!Wp6g$=mPiYW)TBmncI^(R)b=XsM*zU6gnzZ1PR{K&5 z@x0^wQmqxW22=L5cIIA@kx!pmIh*VDlAPVD%2^hJ*Fqyyp)7H#p^M3 zV=pdMSWE)^?yS~W2Xo-iW4}d*TEm4m*-O8BvQLbK4l9^M6Qaa6YCO|lLx%(VUKn)* zT!h5U_&FIaf4zqnTK8hSmY}yxGQozz^qVsza}gvKz4;g^Yi@AHnL03AUcqK(h@7`E z1P#NjJIEgQ9-{%TuCua>v4uUxkBWs}QMZ~^?@sbNK)j81+6ZY6=GL}WH;kU%+}0wS zPQeMGV&^}(e3Ic?Hr7ER73mt~4GXJ{Uj#S_X}Q>X_-X2ZKNG**p62V94Xq&vh%wS# zN^NSc&7VmuTyil4_ON#8^3iFd6lA&Q={~#Pz_FKjXjB1y7OpwEM=);1b+Pb6o}E&X zdT)FebUoI_7;MJ< zR==*BiVZpqf_^gH+K8XvIXhkE9Dh=jMGU$|Hdd<|Dl^9jC!@^ob9Qqq8aA56tWF=+ zb=q{Pj}8ENm-ATCX{IB&w`mZ;(qe3zdcp}ZKAc_V2W!dYTXBU}!md5HxxKnn&RCH_vymYA!r2{leO>r)4?utenOvF2hpS(Dg!f_<{x)^iYX$fOBzqN zl6opn{|5R_YAVAYLA1XN_C5m5lrwu09*ld*lpePd<%BRzo#*4!2_%&w{+hxMgq!@7tj-=R-IYt~Lqxwnj z`F)1;lKoHWZ~Gt3-PNAtcx`AVia3jDaO$JWL{j5N=|_BT~iESY5S?=GW z5sfxnvs6cMz1pCGIB2S`I5ZWMQ%#v6ZN5q@j4u;g_Z%{RlGA@-iATrHy}?hq+pH4} zF7vGT;gIU7YrzBC?|aV&m8LOibF{| zqM7z58ON-(F|6!~TI{v@z0!PB%$<66eBX;~F#8_wlfj+i=Ne2gtDNAgJj5z?;zDL4 zmOtG1>#Nd>7vpa6uPQUq!s%Q5I<4HgcF4pU@Y;PvOVK)q>dr2uvF-gHUOV5tr9-M^ z$E14eJp<(}f_z6qQi#;X%QlawkKN=oj-6;bM_2P0j`e2+>W)u8>DAwQeD`K=1^u== z5l`P4REfHS7xk-2vROqLq+=D!jV~(dh8AOP9`bJ%TcIo3*Anx4cWRw2DEeY1Z6Gs) zl$P6S1$2Q7xX^K$|70cb#**SJ;eRc*Qm8z>Jq=lXD||j+FM6?IeNR0mBef0#%%`$~ z&HM51qO*h%ZnK1*N|@!l5CRnnWjjW}uIR3gk{}#p_xN-r6c+6UuD{J4D_D$qwdII$ zN{aMJo;?HQBoQY@;~#!-ypSinpLD{o%?+Ep0d!=jW^4K<>^+Y@gS=TFci@_BYb&?$ zG~0Kzn?#O*o&&-2cY;e>3RsVsA^+-Ym5=GRxQIb9(szj6)Kb9Y*V(^lNM4WNIY+jm zstLT>pB`qAv>%iWCrtxCFm{5*=4o&m_>6TgpHhdo^clhjOhlDrP*O>$*wAkU2vIj+ zqWmMDzkHLDpIij4F{#*epIrNBS-aK{%qmL&FxZ59Ld-mTD@d43p(!=$k7!~@P}LkI ztMglJt!9_#$*u+d_4Wls}6;S#!EG|1Hm_;KvVA=d8N$ll`Tt^728i+1?O6g7e zemrY|h?>&T^@ttVA%FcklO+7S7|5rZ8M6ZbmX&RK|1*Prq^S&dQ2+DAVti&M;1Z{_4@ zI^DY>?}Tz)UKpViO4HY>VeR_<*C#t=->(%R56!hlY>(J{pMr%Bej*mTCE?yDarH{R zLp=i@_i z*<@0$O~e;SF9q)3nq||Uxc>O4`BfV`S70agiM)MHbD^qb4&_?aEx8zzFJrFsqv|Xn z6c~&}E6&_z)r2qAR~5 zCDU{F;|7PEL(H#^#Cn zVT{zpQboB?t_y|hZxjwL$k9>t?aQt5a6u>PR(=gJj_Nmw%AC058&>!kt8`s|;oHm& zQcls7TVq3DXWcU#D9@<@`6sPjGr-bCciGh-NQZ~~du*%Ij&U(=IPZ4>G}h9lgS&TuFJ+s743j&D%U#kzmCvB$9h} ze{wUIg_nD;`e|(2hP;}!Z%7fT=s|2mrqFueQkb1`=1{IqR!b%|9bUT2a)rF<^xH6M zImwH1Wc++4Urqng-mT+vr2We_ea{>g%`qzLpi9f`KOBw_EY}o0FL0`w&)sfcYHsMp z0t(^v{;@WfcHnXh9ao;rWf6-Q(nv?uX%97D8|A;$`prt~lYuUgVbSJdh+nyE@#o8A zXqLiTRiade1al6(hEor<0{-8S6Rvau@a^CJ@B@*%*I~-Xv4+XbL_O{4&wdnTtd(aT z`k4o1eEErr^bH|@fK!nsj{Pe(3^5;Q6&cHod#^#)y@g}M?HPVn9WGCB@!e$nhGVf> z7uJZ&skv}bV|5d zQ3de%VJGAEzLMG*nr~BbB}D;QpsW2^-X`VOqK&I!Eij;*-nTlbNP_KX)KQpVI++y`p>(Ks+fkae}Q+DzO{gpzBjp1i5 z{a>NivV%Q2G4Cr7!77BlS;3)m5kQc2@8@J<{30|%GU#$8pSn)si9PGAQZ)cG;%Sc;%$OefwsvC0b^G#tWV{&hn)UFA=J`+`7U&cDs5s&7NX--Zc~ zCz3tl2F{00q%g#UAw*FFT#m(XqF3aW6gcSJf9v6vP4_4VZ*5aF6P8lp>$ZN3XV0&1 z(QY*M7q*=8aC&zqK0XwAE;|hKU)I%Tb#P^ScO@TY0`;#}BB3pqj^rr16hTiAQy}3E zYzxOb>xcT7tt~+%c0aiDL#h@6@zzDs;gmzZFrG%O%@9~hu!eiyi>B}E{+bQTogN`G z*U$0;lT}XjCprl(*;VP}Au6&V{BoD1-n6fZ?ka$2^iT{vQZ7L&P-J!I_&Rv3ruvUs zXpKrB)X$rJG6MH5*KlJ--D@mGboj!*3Az)yQlQSS^M3jzBzg2Z8`}vZ-32U5pMB!0 z%CcuS4{*~8VeW|bx^YoIgwlc?8*_x(wyGksJg^{(HR?J)uB6zmr8i;p7oYp*!W~s zUw%jLUd!-|4YZub?<@Vp#@G2H1IF-^z^n!#`)ssQRSz(q3lF*`X~?1Xz9Q7|nv_S; zWxbOp=}|n+%&w9voMSz1V#sOLnoHbJUTvWM?LtA`uWwC+2P#LM^LK}au1%R}J#@4} zR+s)$A_szwAJNUuO_U;yowtp`*wn6_-#(!4&AkPv)+W=ls4+_|AFp2Spdc@pXMq)C z3`Eraakb`Dg6BqW=ipMmE^<#k#{P_)KC%mT-}+B>edFxQ;iHJNs*T|3=}WPrfDG~;)NDuSwldTrhI+916*P0>Y<`|(Bj8+c z!HM#hpu60vT56t&OMJ7S!gwURp0pKk^HIK_uUK@4{7rJC4&Kbt7tWOV+>WgIaJ!jd z`@vA|Ij8a^bIWzwg#|8&p1|ui^ZAI138#fu2P=<5!*rW7TE3zwo3$N6h9+Of)$bH0 z{4!zs%bZG8efo=?T|jA( zE2CGqC+Y9l_~j&{8Hbvulp(JNV^$Yy=c$vFgs)Hd0Eh0SbNmrRM&ct1UiF=sZTtJI z=IcqxZ_HbbhLMO_V|MA=AxG-+e{l=0Jr1tw{c8948M4I3Js3?RtKw26lmTNb4eL*- z$@m)57uC=$JyT!*kP9U*U0HDQi(&}l-AzZxEb;FtbM$-raMx$VSI=5c(3Ha48utQ^ zsBmTCz3|EC?@@;%9I&4|5xPHr1kaX90j7rU2JdAJZ*EZLRqBm$_DZkK)E$@rWG*L% zv%x~y2a}I2&}~8hAOyG6&+WV|ypq=_0Q}GP>bq&jVu&pjya08>^{D2AB2p2V#`TVG zgN-C=`o79VtJg|0&W7E@x$Si>=X_qM$w>wtQPpR(LZ zNBMqI*?ODJ3i#)nr0r~R2yM~j-<5*fmZ##(_W@rr!5b)g9>RnmACgKBzs%5ZM?aGX zd_6m z)PP?E z&AZY7`sgp{kQf9mU9)Y6H3{{nPHuPK>Jh&UMSK+soC-m0<@)3)CasU)>VbF_2a z_Az;0c#p|5ZO5n-mbc~;is*pmJ9{#p{b@^PA22T(av7d-Q*Nrdbk`rTVQwHgd1S?4 z9+wFrBvk#I%iOFRd8d{}pI7VL-bf8DSsp=7vW`|E(BkH&o=1rkPNUq&kWJ=kYuSt| z*FtyU^X{XV)7_z}P)Exr69A~t9b78OrtgR@`YeD-N=ru0-}NWRQ!;JNs+vb1Rt z8rH0wxaFUDN7*vyTYnJ{%E->vBe?el6Fp{Kuq$!#Fx#tYbl;fEFExF^KRh^hJxO_x z#IRg!C_FbVg+a6XQmqpOy%{!Q7z$u3y>@|%p9_!YM%*st9@-(t95k0La+kK(?)P8? zw)``YA#X8MZEIoaCBnpW`V-9+XR5s#4tpEr_7L$H#TTlv_oe1ja^jahbgII{y^=Oa zSx{$2+l`J(d2B(Af0&Yo8x1&e~ewp))FNRL#UrD%XMwG&L-d^ zjY!X={0vp(yhoy3Iy$MC*=~dDt&{N7d)(n4|5RlI#w|_M+*(qtpRTEPGfT@@QFhs} z%>2`@dQC4BeA#hbf1KaU`Jr|bJ#+C@g{GXpPI??7o@)>xb|9F28Ebj|{Z26Jo&>?E zqZ62@ZRn%jFNLssf4HhNX7|%pr6W$&)v<9+)_Tw{mf`PpluU{={@veC%Oa(=*0d+T zDZS7TH>ojJV0M70iN!;gk~MGOu)j@Q@RJzYA?XwDjEY)yD&==h8;)`gqR_+pr)&;4tMQh!`i2h-HnbD6yb#G22;UA{Oci!REdb*%+42AP^}NbY%F|L-`t5L zip;7v+Gqs(GFQ)#a-P@MmM8sB1pxLN#z$WWqQ6&hZweizvo~t2K>2&soPV?o7J6x9 zvLxzP7i>N$$CRx64z03kbZb7D>)^l{TyP&}9IWQa+(40+j8Cb_QCrkV_^QCdq~9&L zz8sOZs-)sZcGE}AgKDo{OBo;6qmiNcn|nu{;?G}i^hHwog`K2j(Uz}GagD@C@HF&H zTry30Ia~aIuZYkysSq7E+okzMo|K!#cSLqN$9JV<%Y?}1L?I5;dT138&9{92aj`h{ zv@frSUwEa+7j9)!YoYw+pxhOp^_8iR(8=t2wydh6Rp%j>OGQDvBcerg-K=<_I9eF0BIh*Roo7uw=|-xF2+J0J(7abibZ-2|v>l%OhBv z{+%y}H4|sZca|Mc#9!k&Kir@DOY9- zDpD_QyZ%ze!G{%k)hB(pJ_0^40Xk6NKDVviz%!Hi!6ICt<<4BwjSlVUnjB9AJC@8; zxfzZ(`YA;b=bAX$6+iaA{Ug^wykM8-AA2c}zIMg<;`b;2dV@8-1HX4jLx8kyzUGGp#4>N=S zchzjo>fdg2@_Jn1_W_NdrAjllfSvKfNN|9Rf=2kjY@{!h33ms>Ei9Fs_|qUu_4_ia zIvQ#_ z4tm}9e(rC^uC^z+No+)d1PgylkhiD1wWleM4+X6FuD9QT4yomo%nR?5@#2wn9Hh(7vtAR2n?tY)tZhiiOg|-> zi#Th@M$8bf{?V3CkI_umHtU_+-{=Tdc79S)l~;TDUc7I=e!_?w_=AfuF=4^xwWYz#c{N3D9XzNkN5ozk# zddQcA+y*i4Q=gMTcDc>=kc{V8b=3F%nzf=m@N>tAG5p|)G zfOfd+5&3KpXaFrc>~^{JN55n&34=TBzrjTqUBl@Mcrw27-1k#fo4Z&}^H(!Hr|L?d z75-M&o(_(j)r@j6@XesI*1L++n{3Y43qgcScCR)T92 zTi*B#n$9n?yfpn#s0wE5XEQD)xbJE-OeU%*UvQCg-ICbOF|z@h$5{%cawOSy>EDOt zM-)B>wnnXY9<0;pe`ncaOHwf36l+6mOwuxVVSo64i7lGt}F?7}HvA4KC{jG>ockTX7?{I~dbgLWt&8Jp;(rCNs zK|3?I{xiB8Jn)>Y#FXRftG|@KN#fOd=`O7Rn8)?k&d~T|?;^P|b#BPbF-`RxA?5%i zBCk5VO|vOyism+EEkNNSol(-kepNMXR!v_r6E-EZ651qC51+IJNdk#ILgtqV`G@+mb`Un`~3#U)|J5^N{Lr zuepG~{g$JTh?qL?6{Ao=y>w{qPI@S%5INje+jiD7jM=T(}*Or{x+;HEN8{x=5wAp$@)BEp|A=Ree8CkJ{8oaeGCC1%4=a&%{r4ZZpWq9a0>1z(4)I#GVZS&`e)w!_ClXbR@)V-Ce*G~wK;OmJ+C5Xy< zwy-(ulXF$2-~0(mk(I&5etG~;7o(}e!pG#N=GA_6KAgZ9R={8XLwnHW8 zxJY(KE6O#z-zAG??hM8G;Xij@ilT<+_i*>R6U=;dChH5j^*Oyq)_8}Yxor;#;o~tu zw600|Ju+^+rj|8Q+^TM#)ZK{OaogNDK4eXq8W_`g4JNlJ{UP1`8fW7!jXGMY^sZ+& zN4{Z;*B>{#0Dx`6^B$M-{jG&o%yjrp)nRS_#(jC>B=f5Dm6B0pahBw3d4^>ktBwY9 zWB6X$G==^&f5+T0&94aY9T7&hLBu?mW=*_AW8*q5>(X_eqRIM{He#ERQcSV}X-kn} znTn$N@&#WdcwHBe3_nSFmJ#64#(O>x^vzcQy8{sUa%CP- z)@<>ZZrx^Rd(C)^iC{0r-8jUWn<4xTF7%Cy;A)u{=c?DS`Li@@QwooC{7=7XALa98 z|7wKGF;D0sFt=s2WH0Xm%e~;Omzb|N1S*%4tRz#CL37&Q<2RjS{3BkGCbM7X64Q~1Y z>^sn854J7|YwpV=szLTz)BbBtjpo`8Zs&7kh5TV4p=!bBfA#Y+Pf2z(!tuyRbL&Xbqx@Dgd=C(3L%Y% zt0fy$Qf;pfcY^1ApeHe*oBy8DQj0Ju?0EU|{L$!onn+zQCRfDmjubrceZeM{5$UzP z{DtAl%-q?0jHyQ>E~-v4>zXQ*dux~zi4`+#$YDu-ol!PMHLBB(J>QX3-?+6N5v(%W zAfcJ_D`qWYJ)!vD{xVye*OxKBS+`?~V2;aL;n^7|YCc`fD$G`-nk-BThsIVV^@$t8 zED$Se-a0s_xajY|7lVvz(xNUqvE<2PSA`_6JpG zVOcJqQu*q>a8nt&(?_vc5))~XI{iJ}9A&gno`L9z5E>y4c*kk~N1&(Ag}voT?^&81 z-U+3{@?erWn0Kwc(qOdxyIyRQiE+U6{TFbVs#AP>)%V|{&RjU>LfZ~+`K{ulh77>P zUG3URmP^a_mIa-#QbbgPii|ILh_zFaO2-cTr zYjqSuL}qT1#@xNW6MN(Bnx+txOVmU4MxgDxRhB?R7tqbLJjSxTxw$zSpDIu3B@hv` z@2Rx2LiIrzeDdG59#$Xu$(g(JugT&Rtuj~#>GxQ?VH*N!xz6690O?upwmQdfAXOg4mYniS`O9M+FP#}SXhfpWM>h)*!V|5k5+-p~PEp3TYqI&!1?M-vw zR~zH>#Z@*cFfAMd3FZXI6UcOL9A!iI@4vGov2{ zx<8x6M_9In`}jwsX~c8Dy+8RTh6TW zbC&?DQVYuSp++9(yyR>&!JJ_Cwo};MSJkfewMjqq$82s)tz&V4e`q258wDzwrS&qm zo_%WQQ#0t7mhrms&%u%?vjYy`XvHHtgc@QMdAS{PAeYNyGXv7e7D?REQYTr^WTXCC8$ErPJgE~7b8keIgG#ISCpf-R<>Xw`@3 zDCTeO!$h1{50yHA&KXsrCG_?R*Q2~C&n8g*fK{WccKw;L68$g^jBTw@q(O9=F zel4P0?{+jf^U1aVKFW$wzFJzY-~%mme)3-*3*)&re*oMB=qn5 zT=PjfQP`U6A9WQ5cBSsvD<8Aj2OB|x68Fsj zGXbu868(ezpSPHtPg85hkj%4%JQqmbnk;#d?(=3n)qp*FhIMoL=gF;|kkdWg+$mx+ zs3UJM2e+s>T{1MNCuZMN=}+9e=wGWTeV(H^S-u&n)4Ks%CzLU#Vrt$ojh*xHe^Y(K zI071vhyDPRX|y2t6W_*xl)88bg{|Ewnp@7j7|ZE9JbWDP0n z#`@3ZMXk#{-(c%qpe%>!aZbmI5E%-BM5r7U_^)zr6SbDcib{fI3lU^f< z@}0F2sHcE%V{EVo)y1)px{_~a+fBBqnIxlUv3!LS_EAt+4&teNsXB(Q|R zrZ3GSHJ@I^vn=uKK)Fgt0Bk-)P*l_b=|ObG1ZA&Muv2=%b8Edr9@eG(qFKsy$dPvYgidpG5C7lmJlT|Ij* za-z!}(!~L7Te}cHUOsYR)AI&==UJ ze_$#U*X}}t7=tnzFzE!#q=Kmy<{k13bw?g7%09}2Kb@gACy6qDm=*>qXms&xywYG6 zMLpOZF2ArdIen%ZvuJDNv4{1`7A*4Oz;hD#A4EXYF4)fBpp~Y_TNuBo08^{@( z%@nwuQgoN=Pue#M>$jgt6h_i14lx6b@^(|hf3XE2QHNKB37sV=fBnS3`_I}!jCyXo zXS|}J9v?U^$?)Gj8R05L@nh-Sg%t}pwso@(N)=$L3+h_|+XFtLZnQe-n|fBjs8~IG z3rj-R3`Apz70LuX(Ckoh#%VIA0Y>-YzEdrOei-Ghv20U6v^(!0djawiL&Ftb`X+VN zVfvTFc6Eq2$mP$yRyA0RV#sx`fUxu3WuJK{^pLJ~6(`-cR%g|^j|1DwY!+>-+o%mI$tEYl^EmXwMramq&GK~Y??Sx(R1##pL z9)yk!b~Q<)$Yt4s6h77a`}88|S|?uufYYfMP@~*VEF+`{uVH^qwbNRhw0i79?d;Ka z^K?48lF~TfG(NJ+Z0GmcqT>r$-g?I4?yX@O+CD>#-6fNqo6otnqku=uUZa#y14QC4 zB5&VYsx07KNN1V)>XsBFy+XaKCe-xaahhYMrFH)Ujah$*944e@tzy;#*JhmL`=Ez) z%|Bg%xHT6q8}DVGRI^*2^4VTQ%ti9JMiRckdk1D)-x*lTGH9MS&!`W-PKtuL zYRrASSyTR?O+g6xm^6S9&wIkvhTAt~*C-;sjlCRc1bY`&NZ#Y|Mi{+|&#%Aq4_RWL zaXex0tqX%%ih*F}ZPB@OUOE+!#KgR>^3jT+aV5>LA6Y2c2BmGbr%aO@93-Wc^SR&c z3VMAX0g`jU#8!#lC1#<0c^Qq|y8tP-KGQujGYx3P3OZf4NLR{gQ}%+!@!h3+{2A_p zkQ45c^x0`-asu&at)-*^gxFQ4T&oQ|bfA*jp++=E% zG1Cyh&+gjq(r2Do5 zTW@fwKW`NQ#nd<*aQg$u`uz(-cBa7I{s+4GZk&Z2@r%8Bkt(AsaN&^}TDO^C`@TAu zT|AcM(Ny`L=ZIHjL8hUQ!RWlxx8SM$dX&Z~GT8hg<&-~+o)pH3xn4gVYFxJQ8R4@1 zC%)wJRVtIrwkk0aBEy>)%y%(&{1&TpBiLCJ^$a*5sPB1ts5Z**%O#Hgk}liC6}XXD z21!lY>7F*u!XiISvJ5QkTdH%uVA(Lt-}$bJ#9Xp`X;CNYNp7o7%RQOwB=_=*9f`-v z)voVLyzebZY4n#ZX7D|%B5ZuumOqe(Q@$w%C;OH1Azg`KDm~@1<)jYQ)fT$X74!vO zB!zvd7*$Q}rYL=?blCwHhcL3n#B0c|e4Q5Z&m7MeG8sR|d@$2mTT>*4Tb<)URNEDs zT*2Jrjw767hCkqHWllBTbVb zS51e0!OZK9X81&Q4U?q&Rx+P?`x&ShJb&$eMk2h?9=+xe&%r9J#?Jlg^%>LAi}=ynv#M*}KFSx-FQfgV0gVRC^! z-0xP`@m7)3VP*_k*Cu*2Q)%YTviK0`{lmLY==~h-AGpNs?iaueZXBdt{)1Db+{V4* zTQ@q@uTxf=;a*kh=i%+)=sGVxw!UNXpJ4A=jq+`oJe@v;i32J;x!RQ7P}oRPDUYsi z+p3ewDA^x8E%q2Xb*1aAeZIabW!QRr5UL@M6BU+s%Kf@4qh1`>~f=J znwm%REIqB5;ed^$IBzX{D@)fYBj{5@|I=JQ%KyjLd;hc9_+k5IQ7T4D)rgo?BSvdW ztQt2;>%Oa61f{4|dnHDNXw_DuMku92#j4u7s1-$x*faKsnUHw$d40b>JU>1ELta;| z>wKT*ajd13%e7xTKue0%TN5#4xuzqt3EFbn2Hp-EuJObe4n$*3`QqW!RfzV}3GNgo zw7QJd(@?Rtr!fb&*DuR$gImH#7x#|#ML_Dw-nIfsUDUdBs8-0u-|UG;?0KKX1siX+ z_Vuq8=7hGF?)0UWnd9GZe4gXj==}zM9Ny#e>MZ3*Tu+Ft;Cq3cL{hE)oY4^p=E|72 zXHq}R;~#L%NQ)-`COvlY)OpNyewFn6w??>pru*=ZxIVg(-`L{WmV3Aw{cnA7`KSILsHjg#{F|ZS4MHCP*Y} zc}gN(`ttWfvFM3+M4^HQ>QuO5$uH3U$)_`9L($JYSKW7vJqpKN`rR?)S)ilRr7Gn2 zl?s~fr#icBAEQRsBwSQ|;Cg zD(N=Ja~{DG=RV2tyfdedlPQcNh+5;j7U&PrP`~5gt6yen@oQ zNQ^;#+W+EBM}4DemW?86v|}4{cE8ylew7S}UH9JhsgyKsjc#Ik7;ut&X7Xs@2DUV5 z-UKLF?CDlFt^^AAwzazX$D0=6aw0wfv!OrSPU^n)DG&D2?(+^e|NbA3crigw6tOG2 z6X*3ix5?9mWV9liG5$RQA>C47<`PB^f5T2)?(lwXQ_Ny z4)uNFv#HpVy)zwN?@H>ZRl#m$UwOmFJBoMA_#69Wg4pTa-^OHAea!`8kr}dP-sY|j zy4XaPuq|un1l6Zu{X--#?F=5_2YPzOt%O-~H)#8CTOadT`mG8 zf;P|TxA+fb8rN6nNzX7FL8|uw7MCkYlFHL>>cn4XeqLS8QFyB(D;o`lf?I`{KrgUH|J)&qsbYWBdtH)!t)| z@-ON+XiIkduk>x#TJy%i`@hlWAnL>+&eG=7y_rmk0Isg%_v$O42Wq_Tx)9ga{0-e_ zHSG;^CDE_ARMX5q=T7(KZVZDpQo!w{a;wa%4`Vq<{#l_e@Dj5RZlmSRGh*kA@X2*n zMq`cM(?zrTF>H%$YZJNScNHU7dde_UMYLj?cgAQEu7F|o9+G`n=P5YOoWGx=+Rx?L7D!VzkHGD1xq<%}Nyb3|`TN*aW4?ZOqOXWfO(jkJa__ z`Lz^wg85=G#H;DB*TRFL51?sY{DDf+*GEmrUunYM$d1FHJjPpV|E>p5!eY*kOV%Cy z%z@0)dM}?64y2g=`@-~l3$>vIfKiB9Hp zEK2PhZSoJLpIzLLnE4vbZ@CXd2w$(n77I=)=yk2T+lei*l6tvd9FAxoTx+^2rBp~T z4GAJWzAWQ+fN!gMLQ{e^w)($yaT<_QXZ5UXo?(Z8pq`~B^W`l*-E^Prhvp+5^u6%a z=r1lq2}+(N&)7|;UY~izlg93?<{eA*Kb=#gCJ%#z!_X2bZsHj$^sD=3E=3HnW*uG< zr35k7sEL?657dxtEv4-r@Leg&O!i+jA*4natVx+~!#OBfEm7-9=cMbdtdqy)wxiyW zXGiA)RC2INh~%&yuAKV=g>SIRm3*eLPgHB+G|%FeNnWuMuche-&t7AO z8cu!%A-JFcH{sisxL*6|bnWevs!;QO=hfJK_j55ZKu8osM=~I4{`k$Q!R|x2GZ*5l z{fEh(TjATjh*h(e(~!8gm<1fZ0<@B~LHm-N5{$TR$DECLm}6k!a*Sl+C5k)>1NHd< zkA#fpH*QHzB+_?rl+IlUM|O^4z}Ei>Jb!OBACuTQ3fk0tO0nIs_IY#pbmG39qvo9{&J=_#zHIXWP8=qSMG%8N`BI)_J9z>CNfGgv@NVhcb zj7sKAZB4I><+}UfG8tj-%=N{E+_0~fWR9)geGr^JsewW8H}bu&ZhDRvlT7n6<2nG0LkOKLmiM6F&!4TvS3*8+D--9AiQMwLOb2ZiT_`_vx?vMq}Kif&cx#$6ltpB z=neW}_oelrYbcR{fPt{t9Cm)DuFo1_%11;p%@0Z}%|dM+L zdhVYIYHSu_f)Y&Nt9rzVFpG)wH+nmlV`{ttb}OuZG?|X~?-KO-N@-5cCPM4FCAw=! zz|t9P$%Ucr`ZAGj-c4^SU*5Q9a_fw>cS3%3g9$<3zVx(y>mK|AhX}B(IuqYOtRHH9 z4tRS{mPy;#X=uM5b;%6!*naB!$qMZJx>jSAIRW)_td4U?<)|b`>B6%U#S`gpEAD3LPbowH? z?e3z{@W<}Nb+5~Jh=Eev6^Cz-*G9|4>tRyWKe1gkVK~;3MC0D9374h;9|S}MeAz5Y zS@_nl+164)t@*M?q7{Y}g#2f?+vhmW!Q^i#8);*1;y}*b>36e&w8M;b+N^0L`D=aw7Bi`Ep}D(qpKgpH-a8li+>uPS*?zsMCcP{akF7!!V;ui}48Z z?{iF`Jvj5-n{iR37VaHF{HE2doh^-yiRI@IkA*Phi>c-hpNgEhKiQo6#Q9~{h1beT zdVDh`9G2b86lnRGnCM1&vBQdC`uvd1`@=3SqaNt%`t)N$B@4u(HsYtb%uiyW@9nMT(E8UY(~{N#~iZhf`4j=~}3vISztbH52+;h@cBRrgss1e@9A z0L9l}G)Sr3yFFcUq1lWbUi*HBab*>!=G6CJcRo{!N&quqVRo;xx-8H%XOKHBsN~s| zZRa+bQagJ!c+;o3cG`l;JSOdQ7puVV8!NRGT7!{;px+sXg`uKwVvf!+%hw5Zkp{8F+ zt;daW^9L&9gT4!knHL}W_s*?aC+v4425zqm?R zXb(7O*L8%MZrcu^^u6ekglymZV%P3&)-REWA=xvMQgbY1`Cy z;Rl%u=yemdfuXkggNl|y!&e-%%0weGnSi>yt{FPDq!(KI(u!sApy3j?%)n%2-Y{*u zL$jP@!zV%?A<3TCfGtzg_n};95*5OYDY@1qj*&cZ_n*!WgZ8GYyiZL=G zd;k^^9rUQ@KXVl;b~UMf*1JNl-*?h`0)x4IHa9R{D)*Fwt?``tVVCGg94&v0pjPYW+E=(r2*_3+oa{V$fTZ2iIOdOWHu zjD+fsDJQ|k4_)?|V;WvW6b^!vy@X% zr_biIVpv?`pF{X2N%vNQ#s2Ha!C~Sm(;4;DtCHdQGwQ;tsq3+w9H;d?>6icb<|*9s?k8Ysvx@>MNwd* z3aQmX)oJD_^N*~)%kQfEp4TX1{?IpPKtG#F7w?(#W7iZ?*9~XaqaSs*F!c5Wk>dAd zJyljxB`=@7yXIOna@+4+Gsp5}!pg%5ItWfI{JDqPPYA1Ej+9=nfa(N$WaSFkX#JZ~ z+s{K$%aNXHO&=#kirN$X+lfT`-W-cvh9-?xbu6JKsKgx@7_yhIY)+X4_WH?jXkRH@}eB`>*(9Vkh++tRa?nMAI`vboS?pM5c;H{`+0FbGVH1J5XrM*XaN}q`SgYA)j{k!@ zdi$h z27VhRnm$No*cGO9H6+_YJ5v+Vr@8aT=k-s|{)H}!6a$fia8+LD(1 zR*pXrZn}(ZSQjLkJ1%o?W(;FY;%k?$GAeIM!lUOWoINwy#j4#{&SRP;yCy%Z!?_&I z^4)d~V8!f5_jGEYb!`(V(a<%m4m`+6xM&M-@RNAM`m4!XDLg&^VcHv;pYcuc7*omOgrIjK|U$wLgwk=ky+#eF85 zDU8WW?S)_+Jo{#$Ew_ss&TZEF@W(5&XuAoPYBqHVGkdTz3!~ybi|NboA6tp zUhYN>WS@=z*3aqC2hp|lR&C9hcrU>FD!DAk%e$}5zV2nRMHDcxoX`rnP=5tOj1wBR ze)=5H^~w*u2&Q8lgH4?(;Iw)Ni;L*1`)(F= zo3*~$jFt?j5-3!*esOZIyAFEmMxtMbAkHZGafa_9ld3j9^ACh$-*v*yb6N`*w@gNX zXzsphO0|G;SOjMoubt6+@m|oi=*uIwzQ9?&r^Y=ty66(10&3<-tB9zkChQe>Bh+;j zw3l$!a2_Bb;h*m_=V+tW7?d>Z)XmBKE#RGHMv#Q zppr>?!qa><`vcl$1;3w$c(CHQD~}0q0D3|+7BP=JHYQ4Yf{Rl3-pTYr3?8@U^Eo1( z2?P4f4o2iPD>+^T7*wOrPHW-80lIAn!TeNsTlc0R!l&$@t%%6`(`f!KItU zx5>FL4=3K|a>rm_9?lI0uTadMDCkA3uC~@2pqO|AlSIclb6;F}&kG#%No4cr7$0|= zl9or^JM%D)GFKnE-6RauCk@b({7Gi@YdUlgbXpRk0jWbrH|t zXNm9@B0Lt_>D5~9))LQBbh)rDI0J#N-CI4*kF|5+v`dSZ7(_;@)nYf_>@Cy76xa(k zev&e_#ttm+Zm+vH8%jiMZfGPUkEBR)e#eUwEA`+%=en+>LR6_9@^z3DS>b{1uEuP{ zL!lZcH@c-Js5})4EH3r}%H5Txhh2L>`sT+g&u~)x8^aFYm#4rvvqgbyoH0{|Fn+c< z6~4XK07k98E*;<&jK#FYd0*;Wg_OOu;d0U09A2J+%P1VuF-FO}{+5pvl8jOL|=u__XlxBrrFP2K$x3)ml=HkPegZ$*1QJE115rxweeh*WKNE$3hFJv#^Vm@cn0okVX2kLTXEbL`pl(DjlMlMh zFZWkl&#*0r1Rxy$Epy^AE-JHSf$%IG@7;LOGEZ{O1cz3PK>GLFp+reA@a)7JW<|^T znu?wMN9OSm&-2v%TM!}2ltmsZr&Cq1NCM-yP4dT1RE*L#tMI~WOTkTNhiCz6Y<4b8 zi>|?gl75L2+VYdU2>937(6pg71*R9WdT~OV4!9#mpDhuaf`?u`HTQ1=a!MdNxZNH% z`apNqNFaGwj!C_mFAJO;@12}r)_YF0@1yJMk#N%mnoife_dk?e`L9;pIH+ASB>z|> z^URC)*bCi-HpX*wGpuQw+!viaGajqWR^Bq~a03rN&}*l6n#AzS)%;5CF}QB4_1jw{ zuqC?ITum7h5~8f)WuXR))lp&lh>W(-T;aOPZa!~LIqU?#Zm6ms&K#2d{$F3j*be?9 zI(i_{&?2HdkmES)hf5ikaV6SyL!BNH;!Z;kI8PTw z`DW_GuA>0DUizE#x^^^#U#j61m94?wp)Lrzr6IpFnh$>b8)-n*k|E(snP^yL^ee0( z`!djRdlwwrP6Y)TN{9h`HOxJa^(iaaAtigEb+JPiB4dm!o>}S=HLIp`T?6x;kFP9t zm5;hRbKY?$q(Y8qM+=%0X=Yu*V`NkcfuQ!>LAY5QoV3(6)H9sWDt-Mg=ZBWf3xRA7_76L39@44Pu_jPejktcT ze09IJkKka}BJnf}pel5J-w%iEWax@P>Z+2cT6J zLvE4b?(O!rWU3f4oNMUTYATM}UarY&Y09iY{Rx6LAj9?9DX1oVA&TBLS-BuO>^W>W zMtz*BHw(wTzn9xKB(w`Spi3itrJvTwPjzY5KDUgEeo@2u+;G?m^zTZ>cqSgzPz;3F z5%%wI@u+bUGeQ#7gV8*bmh)n{fh^neX^WkSr^T&REhT8?;3ReL1!2L6gXJO+iAH;f zypE6mArQLbXTIL{hSDPNDN*X(&YsgnndV5ab#Gw{zx#R&Nqhjw^z!Jv1ws@&iDxU{ z@ph3q9n594M&1#ATjD6EKVM@$Wr_0DfR?Mr zyUiaZ9ez3Cgb$%rOtnjs$NWu2>P_aknQkK0EA z_+LD8z?OZR9FuY?L-4ED@1VF|DA z8C`lr=mPT1p?f$3n&KW6_ zfYKY2H^NWTN?;m^C^Oyssm%7$L-L$gEw8^a;LR75Dmj{CZYpR%3C4k1nC_8mY8*UK zJM-GYayY9dC&Cy5W+Pn7!kMjI$iNV895Xbu?%4&Rc8I^ z%etMx`WLoEUlB_FN53CN`Wy*Ry^p3$HcI0SyK@8@cgL1B^$62QPe7H0GxmY13JCQ> z)BOfsu1n~+Z8D{Ma!b`*W2jkejV6DVdD=UpY&z0rV(h(!Kb27J^8j)&IYLx+N>MYc z3-5-G0n|0Lca%6CrVSlv6)*n85*ljFh>$R0B@%J*yY^i#^lWulHfS8RJ1U5fU5$`q zgrvNtL`KWNC)MuhAG>H&k8fup(%Lo0e?(JDd=EUTtEVy-pRXT-jjx3aN%@&7+08_} z@Gvf@8rHxuzVd8dxiMsgY%HHHnC+Gv>K0E)W?J zs8Xg_uMo`6<91Kv&5t8}J7%d537I2lMA2z_jULpuVmO?`(d!1Bc?oiwus?db9zD7S znU)0n4MT`fNXt!I&h0RJ2Xn5F{US??)>>NS?+wD{(8J8*eB}DxQf;WE`4*H+gCPgt z>8ZBE*IBPZk}uD2K%ZANf6K!9YUg8aV$ZX>Pi?rbPOQJ{YyqscG!QtaAp&1p)?YJ} zq=v5pvYwN8TgGjaP7^u$5s58&O$G2^Vu~gfcCmVhknQ8`I3JQ|`tQflqV(RgQSUPz z5d>Pp7=6knKh3IU$>X!9rYp3k=VBqTqqS4LgP$?O==o2!c9ei1rq6q?P*|}!B2)B@)=nrx(^fm)wq*rP#DnMe z8(WqLH!df!cHw7Cl!^X%+mOuv$L;fx*H`rY;8p6RfzSs#cc(T4(juK# zC1*oL`dj38YG?SQPL+F^#_o|{h6Ec0(aHcPO4>D?bqH@Zc(v!!4O@1+wY+OUin^cG z;x2nl*4s@7PMm89Ka8{2X-f3k6f^XmytM)Bh|QsvSLzo#;unhSSkAZ3x~63Ci|=_^ zJXEV4w!7&!NMCg*e$S)iCZRvBbFIHfOH5`8Qt1>{ZkwzKzqQ%@PcMff{nA4uB#|;9 z$}N%y%^W}n@5JAA5>GJ8r;MmNH+;E1TVK|!KyIMrt?T!@YuM*$o5tVzI&E2Q|o1j^`hd=9^tOe9IOww`mb zByIyaTkyomZWM$Rn>7=@{afYipLCdZIrui|HBEi|@PGVz*P}x7LWhHJ68+|Mb|b$_ z%Morj=11_F#X}Q@2lw<_AV0x$pt^0ys6o-oS?9@ zb%AiQUy~re4ES=4yw;ac`X;?rk`R$yTHWIu==uGsMq>ZXIsEq`Y>7N5a(*yulk(;0 zUN`fJwiYvo2xd9Se_LG3r*&@b)hF9)@nydv_GMn)x(?auo^(I=%WDd+Og=)skKX_2R{OT>ReZ{KHB|0$D~htli)Bzumx)WeGB8|H2v6x1Ky zsI52&zP04s;HSRz&Rh}wsI&{@@0ZL!!n3Tkax33_rNk?LLkoGU!%i2!K)SA>D{#{u z5d}NIp!kJHO!pA)!+Dw~kq;JibalshIKa1#ILSpVj8aK#eLpc>JexCTOMvrZp&Z3I ztNPP`<_wO3@PD|r_?R!hnxEH4Ot)}BDw2H6!UJN5d$hVwWX0L{dMwot!AD;>oWEug z`Xz!rCPJWk8Oxo#m@$;kwrZ(d$laEiB;g)Zt!N`k-jQ{y2*)jw)Q}2bPJ;{jeQRoo z`Yaka*3tzc9LKn?jdMoz8rvbVky7yrzi+w<7Ez;L?cgwBpZD&K5`6z04aRdQ-b4v@ znhX^S;@+7}{zIHn{{C#FtVpv)+8O@9*q6oCa^mK1QDocb*_O}dq)XI)kqVSZg|rGV zL#Ea-w@)8840&zn*~*q*SIWeJcBlA*4-D$1zfK?#Vs;~J_@_8G+1Cm*OK2$dce0J>&rFq zKq2JBcR1ZeD`vULPyaL1hsQ$BHY!wyc$AW$?&WdMyG4g>h4MUJjmLWMH&rlUyM_{M zNj0tMVy!fcq`H(1f&m6_5)e_r%I^qf=&crpp*4UD# z!ZWh=s{biHulqaQ3;XY=bH35VO0=Gh5;n^`w^AFns-5Eg@MbnlIN0WA8va)0X^Kte z<aF!=ju&`&#HkmITR;zw(Z0YCdGYVAH9DF{QAsPTX^Tvhr_~g zC??+|937_TR?8LfqcT1CbcMTz?e<`(*3m89H|IMih_&nNiaJ9#CcCnzldE&mJM&PU z+oFY*PZDd?NGVqh@xeSKP71Bpf@V)jhX?oW%gBJ45 zKK4+j$NP-W4-u&I;^s}mcTq#nKzPyKyOZY^27+0QBZYY=)%8AaNPgFN9U=6|d;h+X zN^V17u_oJdfxr34Jf&g2eBApsg)0gt)|hhv@34Q)*yp)6Gw0|(qFoB+P<}h)tV;F% zb(j8yMue%EZMQY+iPn=ok&g_Rg9}viclW-BF#Nr7$bk85Qb)c8T2Evl*TQ{_- z;G#z2L)+(Ve%{Qc zJ(#QJDumke9=e}iNn?Kx6njnP06N%TzN-jLV}|XRhwMlmWZ@U1Q`ruH&IQ!_$U4&x zsX|Ht8hCowNwP);+d1<(81IA6E*Uo-mwBpwrXAJ8~7kc_w^M3rdm6jZ8 z0o?qAd5XC|-bj|z=bPuoh7?_gT*pZQ5TI3~a`cmPzl2<8+zlzLo%dcINOy$#x!vFN zJIVZ}9|z(suzsJO7ak=~rV|+T3zwhQ#M}4!?2-b#>4|}+Ld=L>$(2xDp#s~EChJ&L zW#c?d_i>F6?wtp7vU1^NiC1u7NTUs*{pfG8wLva1`=Xic_(Ryg8own|O4usSxq=ZA z^fQ4wi1r_2A2TCHy&DL#i`Ow1V@cFGz;EC)t22M%ahvT2HyEX#Ix283{GT%l&G)~0 zTud8vRzT1Lk}hppsE1WwB1q_L?zOpHILYvGd^Lu|W8xm^R46o(10KPKFW-Sq^QGbX$(rRH9d8 zXh9{12hldRcu?avU?+ZtHLd)mUbzR3N!KFTc48qkqQE8Ex*X<| zjZWFFPPw|u#hJ@;9bjk*6ymsNI4iQs#r}ZN4Sn0Z`Ji4tyOzUQ4Qogp2pDA;(+?rf}+8_+9 zEn$zEbP9+rs*SCfzQ+9Iy)Cd5P{C<%3;z0Z4rlX&j!Fl8($&U4^sBf*_N$^fyz*T9 zw>Fa(qV~nB(!bdSL}##|4;(YA%iS-o9<`5|%}a@2YYbNF>Mqm2)nzi{)qylUY*PtW zN(ld&7&}?LgnRr{6A{uFXQrS$yb*E4dG0E8jA30cq4CM<-zu3WsQ90`|GtkBt_Qz< z!yUU_baWrj@aJe)bLE;$JVaS&>iV)cMId4_PH;9kJ$0PkH|4)uEz;uOndm$5NjEE6 zbNr3>9NK^3uY}-ZSC*kYM+||f5+UZ#;q;qZs&{MhyWcxBOD6x4;0H@hx<(t(_r)SZ zd;RRtMD6Cra8HYFzCmA`&Lo(+w)n}%bm-)(iPV*AL;P1^xMIq41LrLoN zM#rXAU%yBAwElljH`%vm0b=YA=2#DY0P&)`hs_g`vx@JYtvzM7Hf0;)9N?REg*nqW ztFKx1v|R5$&Yg&RWQA`sZVS6Uk(Yy#!RLfKTo>{%dWrzzR&{=G*ISf}uBlFHczS7N z3bEf3O$z9*v<3<}Jvj|Okaxhlb~vPYS~pfWq)>UrXyI~c1>W<&Vi#}oCV!Djw|GWU zGmE7kGd$Z*sda}Dss9lzHAT>5?98=Rf;QrYVDm&FcSwy|lXPYLH|2Ic=+ zS4JuOmW@!FR+gpn1lmf$rj56kRXgQ+Qox0E_$~T4^`epk#>0wAsIch_MSrT#=mV^( zXahIFm5MQs3e-7E3+#Z9b=64Hx*npF@i)}U>6h0Ht+;iiKR1P#qi{@vfLU#>jl;1N z4t1P(Om2KI=NI&|q?;zNvP)f04Y}VKpNWQ7!mC7%>6^AmGJG8272kUuTXY_utk?e&RZl#J*C#JX^hBY_VWXuy~f33#gC_+Yi?1f9@mUY`PLpwPf#or@oIpCee9BjnI0+5x(|lF0$|B;AN)$aK+a zCzL3Wl5{DsRkCGBkc1Wt{ctN@yz)WVF|;=q-*H{*V*8@^LPoV~^qA&k^;Uh)9DT>@ z3`v_Vo8I=;xg1NczV#0xngidtSUzu)0pjcb+@8X|z;&6bn4`Jq&VeBR$5%ePI|4;AerFXI6uPUXDzP@1x|Vf7V%wxC zj~^P(>T|T|*n=~gP4!upBwKN%-$#f3y_Sx6(ds)aiWaNUv(vc+SusR$O;`>cF>VZ5 zrurpSnOOlqkPqq}8Y7ZY{HCqaRylQ#_hd{$S;*_`3PbCtoqpjrM}HD`ime0{c@Jhn z-kFVZxeU_ea)(GR9jYKQ$#FxC(VEvcl%}Z%+RY@MPHw9Sp+1W;qb?s5-0F{HIuZo*;zh{@>XWmyUMz7`K zo7QsaKV-P~*1rz?Xe;d^YH=ycmEn?L)DSTc}+~d*&_vdE^ z3aH()PVl+>`YP@&`mf-*pB&Bce`iNO;F10;7G9+AKCNiaLXZAuFH!?t<9$f6q^JvR zNP$*b&8T7tH@fPFPo_plde2i_nBYa2b-zyJ(JZ^X=ff)vC!`1bi$7+HXG`}V@L8fJ zlJ`RiX1Y9qldUx_tIZepVATw>i){a^PX-iDFE@zF8F61W9wv10yTyk`o&Dbn1XSDv zyv}N|G7HfTfqkDdsMY|z)er|b7hUh7CT}%wl#Zw z3u#}pgsk<8o?N_<>fHLRvNH~0c@i!+ZkpqarAZ1j?xS5lq-Ek;vwX1rQ^}Qj9}U+% zj3lo6Sbpj_7<~w;f z5of;uEbJCPr{+CP5LdVNYGH>(E?lw1{cFeWoc$8hDZX8$9tGD}aDg0C9LH#|yv+C@ zaz7;U9foEaxQPP6KL6wokcwx6$2EIBwO~?lYl?9CjCzPz3O`j5)fJ@~1(rDp#Kma8#+r zdhqM3*BEk$-9qmOd-CUoRfMMn&}W3po7AVF+{L4Y9p1ZGvOC|p`pl5pZz895y(oYw zw7HPJL81YaWOdx7E|No)^v~J0x#82eYa6G<;X{CG&7IkV`yNR$EgAo06(_dhcsqnt z%Gcd7hC(gltz4+4C)_$h1>IhD$_}i~>v||mCpMP1cDzX$omKjZ!)J!nZ;i!c z8OiO&&KmbpE!cy%ycCgq@`XE`WngQX z`LmUp#tK41ecR;d9zw`s{iZ>1k=}_P}m;c{dg6R9zuiG68o+O zljjY30947uDU=sk|>F5_A=N5Bq?E5FOe=_<_)co0f!Z^+F zG4Z`&qd2T}(Yrjm+M4whU}r{9r1|t4^3Ey5nE3@7854T8WD_(rwYS0IO;y+_N^QB8~-?Q)Q*&Y{NR9tRQj_h!q7aPNt&we%`r(BTk zImZ_(J>J+8D3EZHU9u6LvP-Xi&}CvHs^Mk^BNXB|3=iJv<=(Y;R^#^@Qjyi+t0hx+ z@w{)_bumK^qpzWfBK~t%y<$UiqJp6j?jz3qFrOlo`(_} zbET!4H1iMa$E^monl;zSVk>YD(h6C`9u@T=<`^&tfgSx&QizJIy zic0#|=b}3~Zrn_KdXJbot~*~0QOX3IPVEhFc|>Lc z4j&Om%qPYIzd(}`3YTdDmbPOmc(nyXp~mZlg(O9IoiUJkiP|zC>$NC#1SJNHZZjpj zwbd@>k9QVH3l@J6G|8LS9r&cq+*%i5e~aSl`6uLV2$y0VZ>7+-4d=((P@d;n6aIZA z`8B)@SOs2D03VeKGIow#V^vsoBp7>AwIi+5Sn-gfdSUOq3GV9L9Ru9>d|j17PaOJlfiI7Ar<8L5IPhU}Dl;_WRJ^7rGI9uy=(C9rOXVH$?#X zhNl>J4?uTGFfqNsmrVxznoS_^Z9v+UFZ`-MYrQ~t`Z(XQ|M)avwJm=}L(4SDx{XE?9flTj zrdH68(!pYd+7I>c}3Z))t#B2x)L zdA&}9)xiAPYmy+QZR)mPn=-~~ouRqD?jg0M>UL1fIWnOUbRUvMuJ+R*ARsa7LKWg{ z@N4LROs3C#e&1b=f`;CwuFWav;xM0zXY1H2uh?UdK>JT;gFd$kP;BF?C*0G_bd!aE zBR-@|cR1#^5;VM8)?>o@vGwrLPoz)T7gABB%>$7H>fp#P=`FPl8b#Rq%BPsDmZ_gMUFfF0oKLjMvp~t1>8-bvq6R0hyH2=>cr(+ z=m+dARz$U=Cu@o75`6e6*jAj$eD0}y`9@CD^a6(V2xIm;O);{*idtT~W;NBhCLXc5 zfRPJ+^)&ou9qd&r1fG4@llFM}(L2rXCx3rHm$N9`z85-vGSjH-qLWX3Pkp*o591GI z888Tow*GqvwN6Z!$U~p+k{33HJKS7{>G!=AH^5 zpv3o}5)WpGld21rU@8KHZ@18I&#u_v+yr4WwUj z01Ez+xJqA0FLNEPS}a|Eo71mdc)@RRsY8FjeJPCcG`LAz(7I1!^b=%}zN)s~m9FoD zg(Td|!j-&*(!O^(P&0n#;UUaa;zlW1$EgZYz1Sx6~l zH#&{7RtT@x@C=PFx4&K>vH*~m5;a|sr)~<|B`48y?U|6FnfIa2dhaXSVhw<=44(pB zxj$wOvM5)~3cpf%&E&PSw0Ob-229?L7g90nI(qLZ_M&0ogR&203SX$SZ((_<_XWUY zT#7!<(MG|RucyH@7WE{mJS8_N`)=#TM{j(nc8hK1uRR4SiU&){eEp7%Ko5de#aZ6L#{|_xazSJ+kce`Qm|T> z7aO>{l)QXmFdvtI?wvgUS6Ah-26OQTd!@{o`PJ5|oqImCVHH8Rfh4xGQE!0;+9SO} zvmtjHAEDUS@6A5_`O(ARXEw8Nm2MB0<>0BXtldFJxHfK}EhWQSrJOz`Z_h0@nS9+4cu#5)<$E`Z0=4MM#`%J? zo@Wr>*Jr5J!U%7{iS=@zQ=sy!IUYY*iD7(Ocz7k5H>ZT14sT$mc-(*a__pi(ui9aD zC;xEvO9#4%-lSPN!I|?4>eFM0U!n-`+9#@iT-L4ToLVs<4oC=JZi7o4w*C)Q=l#!S z|Hf^jm8ue@t*Q}9jk}1^+7hFsYIf5it?jnAki=f4Mp0YnpuMYh)r>uAMNuR6j1@By z5{W0@=k@&Z{0pBSuIqfA=lgve9i5`E_uirKK#F$D>tDpzAYV0bqO%z4&Y0egfV0tN zw+`oY#!){&D|MIEa=1N%%9tokP z_aGZxznY)DHRwI!a8OI}@jMP|BE3uR@F4IJ`q@1{>(s|Mm!$sX2>NST8vH^MF8=Ox z9R6nqUHfX&P-6 z+*|lNDw$^GI}CQbBBb;iZFj@M30G_vLL~HYKo38^merjez4gf2(ny&3D2$5{*74_d z%>ZodM%hvXr?UcUk~Jw@exC?Dv0n)96;*ICHTORvrN@p;FYX-u@-!7H9bB|St#A)< zwnR}Pz){JFKY%>3H3U~4VDA>WX5&fU_92qG_fe?VcCdrSn*F-Z{LKA8^PWhYFjo4z zaDX~OmquabJ7kLfx}3uLyb|{l1SuX$anYR)Y%$5b?fLiWx|wsI9>nEP82pmlD|tl- zw$9b8K^Raf!bZmZ7)&jh`Q};<(1wr!y`(Jd?izL}zQzSKDZJX%u<@Kv@sIbdl0*UB zqIuCHvwS5X5#UK$G1oCE*jPie)E?Cd+vyQT5&z=vIUj$Gy}Bos$dPD<_t=r;$}-U3?dKH=0stBochuf6kFufvP`*?7-o9bN>42VL7LT8O6mCpZ8B z?=1Rf_Z~TCh2%d^GwVfHL0nZR{6)jqxYgD`U@%yywU8ZA^R)3B8cuQxTD=H!q zIM#SHaFiz&Pvwgza%c*MecT~3bD2LfThwwM!X7;fQoZ*lMC<60jr~-Q`n4DH1+B58 z$=Xm06pM{pZWb~qv&7>tT(d75{<;s>!!mgyl)1i?-HCs$&{MN;<8e^rqkDb@oZAa=38! zZcX;VP^v}>^O;qkttaqfkIM0dG3+-tFCcX@XI&O3`8G*Tu4hU+%0>{Pme3yH<$T~$qik=%}>wD zYgtdfUvfxpVW}IbC<)zo^KKbAmsKZ=1R%^EfFRbxVwpAm3BnHJut2?G==CE{8mLSa zp^Nmp9e;)1=~q>`={phW;hq(iT>lY<0IGRN&pA!{Yf)vxJX_3ICxb8!5JbB7L$pdh zN|@IpQMzm7Y@^@z66r)_u~f_jqEg|JB+T11e#yJVN0F5^=N~s9-a7uw(w{PXxB;0MhD@eQ^(cRqRNfYA2n3Ixa+67 zf^IVFXY=9YMDZ_Qr&oa3E(#qEFU3BX#t(h<5Ev+Xbfz;|+b8GE$={IT0%!+7?@&4Y zXV8SU#pI~SW-rYnRL!RNOSjwdN1MKu3(&b(o3x9k;ouZ3OjYEH`_>+a_FC^yx4Qgt z?DZ`tD$B1Ur}4i)AI^8h^`tLRl=uet1>F9&AXD7LgxDZn`1DPId*lp7`($E4d)i?o za!NR9{#F@Fv1XX`>4oah)V& zp2mr7$MiQvI(6X3P5Ln$N%d0MO}umFLkHw&9`9>@+y|EOCPIg6;x!y~>?W4l`D&f# zQrDKuMa*&*Ue=F7T93xqi#`7+t>$lJdCNOoHW04aVm@9T0B?42^_fZ;STYj2?#z8z z{w1Xoa*nd!-3~I;RttXfeuykaj>W2*3dO7#vYbFJS=e~FPwxhZdcN2Gm4MK}_!rWN zcULd-*;=d2y`&2z*cTAwN8K`~x|d32xB9tVwD$4>RF&I!c4C<=zLAV3MQL}BQesfU zdQ&yi|6ui*0bk+q>oF{ly@koNUU*&z6D<9TlO1~oDWawRt5Gq}^08y9W3MOblE z_5E4E-Vw*>cR9C(flR=MY@c2-_a7SrT$5c)?MsIXGcx|h^L2pFo*3ST0@d}Ju$!Vd z%-0-Bz{zFmw?kVnSI8w+%Fy!q0{2~-JxPG7=F*vaEu406^Xesi|F;-!W0+_sJ2h)_ zF|sIfzrUOODbYpc{@?S#UKgf6K(xvDM&oI*go1 z&o~Dky4_xzPrtf;Guit);R#gU`fCgFpF93-u>}Hz55u)lfzZ>!(pi-VqLybdfmE>X zI$GmuTduK64mQmm)8pAQ2Oq0muLi|Pf>1OC!rn_%p;o$obIhef(oxLm2$$b3GSZbg zpRI{3ri}tVcbG{PgHqWoyZs)d;F(qXb`H5+a3ZR8z2Vp2XXzC%maOV>C!2RV`~5Lh zO9Q^+|od# zc#K034H%znX!txR@a+iQMIHWAWL;++7Dsa6^a>VTH?4i6x;Y6Wb&KZn1%Arh}h zHF}PCa!DWy@B9k)$w3zYGe>jr^Co5>2Z&*>kF#CZCLTvm6fXjCUuf})k$l^y8c6}x z+H7R`-Nvx0d?&Wq;#gK$gpkMDa^w{?9PgQYOm?Ifr?qbYmej&sq+J3K4qgvBzM@f& z-%$=he=02s^Q5@pI0ks10LahdT#L9K>|I(166z+1sA{4}xoh-l=(IWa{AgXWSpQv? z&$GQxhH9o8-hl3GxiG*%&Aq7-j+BAwd2^e>(VtZip3yRPe=z`j6yW6WKnYpa%;^U#m(h%hK1v+f#{y5?;;hVC#ot?=@t&+aPq9Tf@` zkzcOD(bIT2lF_k`Op)`eJ3NiN$1i|P8#5CD0y+9wXM6j^YDfW6?=@@;1odOy48uvL z+^^fy6+C-9K7HDIziz0#q`92SdNMw_uEkZU>YGaFVqD)N>|(^0Ca#R9UB}<1Us5FTd$$+ z3>;owjtAXT+STOf6V?2l-0-*2?w8%cG|*}48!#)do3;3oTdwbw$yH{&PTw{yjCG8> z%nNA{$x{(oW8yLz1ts)q69q$w9Xt2$D<2w?*Rl$)t@l)POdJKp(J!ThXujgi)#&cf z!`KamNTqdBBb(q^-o)!+`IQ?R4{t4RS54 zcY2%k`!TcI3e(+1uFf6k9f@H%oH5VXx5G~^{mNm;NhSPH{MU_OSGx0H2w|$O@GJy2 z+cO~u4%@fQSvCPo<*o{MWv^7-{ud((idP{pz{)EeB>a_AbHfc&w?{eKr%p z1S3CJ&>uKW!AcM3O*9qwvsKTf!Hm&Nh#jlQ!2e zA!JmFcg2}DpTDt@o;$jPYE^K&-%Q0y)oz#U!^8 z7gqqnn+4Ddg{p3Z$lD&36a{77;~YFp_YZ~vNMtN#yFGnvA}dkFtx{3KaYrwpOd_GJ z8eeHWKKZ{EjN&+!`veNJommk;kT~~f&DJV2F%T=l0=*n=5XuRLu52Zrlo_tBqDJuj zE2EifGw_*D@*YLI>AG?G%LZ97Mds*N#{dc1 zhw0U2l6$*5PKbMrZ}?n8P%7@327KA;tE6YY>y-Gol7ohMg%{z_Gxi*17TMqXOc!ae@d*`9%oZA`({t$)zhE-Vo?D?o1*U&*JPV0AbRZox}QX* zBXX#-^;GR>Y@kc=tWY7QLkLn`r=d+dY?CaoLTv-i`WX|{=>I0DmbTHUq(B&ksDdP{ z3Y=w>V-Tz5d%mNdz;YVlh1~oR0qT1)0^Rd(IR=_z!RFPk_q;eJ`A*~>JLV*)T)Pq? zJbRg+yK0qJ?tx==y_YC*cjO?~bUgz2iqut-kXJuh)RdrnEP7}dda&|vvLW<`Elx5n zL~E2;-Qx4$dFkO_d%Fn=42$IY`B796!&DuT^^q%z1-|)ZKyLTacnI!RsaTPAK|qlF zygJk2kLiKYVoTr>Pq2u4u?)dfE8`Pl#r(x46#07YQc2US`WU^OYZYFE#4R5yFy~OPf;n9?R5kk8tSJW?tC+g%Ib*#u#I2bOUpL_FjWj51^ zd7>tlIhkXu7q~AX^DkaMqK?+>*WQejbTdH6y76g4!-z_2fl9YepP#REQ_p;yIZe4i zApl4r%GZTg7CCNm?J)zSHe)Ag;_#+U-buzn*Y^pP<4XN(VSH)*pM|vyXRQSe?=b2Y zg9*O7ECNKNidXMMx2Wf3{Z87tFm4tp9sa|hRxzHH*PxBs&j`H*JEdI!&m=ckIVQb%MQ zUVmp?h``LFAb3&Kte}U_fE^6xaX(hzbqy={n@vCBJ_2xf(OpU9RjR=aU!K?^)pN>Q z8_CyePP&oyy_aj)eOU#P5@!RuA^pS3D~GH7QyyzihM@j3rbld}<&82pq4`}0CYNdm zZ*LK-2$t=1aKXi>SA7uP3+Lupuc57?19)SN(eNkb+DWtoKi$QnJl9mG*&)9y6?czU zKeqU6>PFJlwzS+?NoTWcaXy0FVUF;g(yG!}|C#W-S;sA%0%c?P<}lY0>qxy5_uFp* zQ|Z~L>F_3a*bU23Kxbgf-qjPlZeqV=Wq!zIup86iI#uNOW-c|(m+7_5YN4#}R(S8` zn&0tG&xergj%K|7TpVhR@j%^;VBZG)@ZFqe=J;_Wqwn^>iC*!8QQn|%=lyiSk4EKx z`uN>7!}Yn!Y(32$e)fS+!}eErz~uA`;7IR{5S2gKeyR$55`7ih}U=f*NrTi1l&#|gUiT%wAu<8I!W0CfHl zON9>pI=*E6WY0DebY*b=Pv#rZ?(0n*S`4c&W!!Pt|M@{FSUHZty`DJa7^CI*mhn%_ zbnNFYf1m^hd5#OsYaj1!bBvJ&pW?e;v{$Bq>MJpRd8{}gTU3xNGQM2awC1#meYmU| z8S4@asW^Lsw_Ay|0RAeQC}pt}v?vlH<{tel^Ceu&0ofMVH4R{5ULIspJ~A|8xKChw zN6XWv0l=qSY)D{ATb{xE!fWid03&e06D)R$&ADy-QH%1z`*9Bv!jzs7<;z0Yp!MHr zR|Bx)`}uc7nnQh5DKzFz{=NG&9^L!Y-wVl&l!!FAu6aorUiZgt)n)F-z}_0%``Uea zZ1bq&uJqV5Gy(Fy={bP>Vz=@m>C4WO9vYDh$TndAw(iGnhi1J2>PX@yX|qhb2`o}t zk=ZUvBuo9=-_^h=kLn~F>>c}oj>SHngESB5Wvi_x;Wo7IvQOqc_zpfRfuuYs*>xL@ zu)PKbyH{N0?tuLfM#GDbpF$bcQ!2=yhW(^>V8uceEeTb}ZvrDIgdJZr{5udpeF zQWHk^n9*A|=!zrDy?K~zoeHZ_tWB~k0{K`F>0Xn&oKA<2qq_w8G$f#Qt8Rj7Y}?aL z>I)qo)aJq*q5~dhy0kF9E9Xwin*-`KgZ=Vdg*6{&bm4Vhk%FZd+(_Og4U=uG!0r0~14cT(>KAhFFPe<6jfuc%Fzd9UjGT zwb%U?+(MsH&E_2n)&<{8r5_4!UcZuVhxiy$)?@L>YcjA*sFCe=mM)|gtJf4Y;E>{y zc#9NtJ@T^#DjK3aML98qZR741{}L^e)XGJ6YW1=H!n=b&`?`Z$J$ut z5#F_)Psu63v}c23`|e}#u?wIeA@9Yo9zX7#Y9VI>TTkuhv{1+S$FO13T2FjK5jvP$ zPE9;gTCEiL*c^Pu@Z78D4pfz~2>j%GIychaG;o}U%-L@hd?3{uo9w-aBE9p@Iu%%E zq2GTEGV5tdImJH)ty@S+yi@mbv3|W;A>eAu_FHf|tmWT!pk(7j<1i)q>+W12JFyuf z%1PgtE?RntUf(^$-PD=QHd@O8Q1fDY10h$|bad4<-f8ab#m`9r2dR>-o!ASC4O1O9&Ir2)hBjpSk7 zs0f;3x9CX({rQ=M+dPA-{CR%3lipfRuKowZGB;$lb)#?y9-;R3E>0BO2@78Uue)4B zM*O_xD@#MG`3RSlKxWWuH=2F5gDQLQ^oucVdDpan?^8FtmomM~!L_3D+czc6wkMFc5Oi{++u?pq%B52M3UAwiQ!RF;Lm*IkD}YMz+13UwqEEit5du$YzF8fv52dK*-E|!)Y(pO@;PZ(qU3;%zAH{h4(1+3$XQ`4+4+Xtf8meb8ri+L_D22|S*y%?h(h zJmziympWcp`)CL2tXub4RO&Z@DDWvneRz}<-~yb4i-aLIrO(FjH?L-SSlY8Y+zy>k zGL%YIuLyb~ywQg%7cKiTy$Q@#Ih1CV0DD*VR}V0vd>Sn-t%_lO72x2KnJKpSnlVbq z2|#Z(bZ0f)Q-~3783l&+@bC}-4419S>W5a0`rS$-D^?zAi0KGQJwuUQ8Y!r+}KEWJsNxT)S7CDA!`-SP`tic(Uf=X3N3hk&eyVk<2}@ zT+LO(bw~Buy@QKB9cAI9Qtq#xU~NV>z>)6r`BAjKfLxl1mv$CuwM+5Har95{c!3qj zz3PDIyTxt0{C=>=gPSu)S+A22I;u?ggOc0#1nRHdXsp{wP&v)NLm!hw$$B27&Q>k__g0Tzd{x$C@Fe&5$N)C!R(--F z<*^3Ia}@o_;?Gktl>Keoz6qi>guu0dhlf6c$b+)FB~k=l?xjIjO+(sb!0b!7n}jTf zOC{E=TZSKU#S_g?TP*SXPJ~Y2urSm#G-UbVitm}zlL`MdBJ+taJ^zlodOst1V4rIW zlGe-Dkmgv29ma``LRh|W?7NuPOCQ?z#Z3>5^t|b2)`-7 z-i75N!?$`P%a@^-<2gg|bIXwWm5zDNL*j-V%EVS>X!Jv3AimDdh6}x-8o0AfOWhid zqSp^fI!zoTWO>b`g+Uim6~)Vax9&E!qDfHsRTp0STwe6B&F?#FOv?YNE$sT9Z;*mg zd7IY9x$s=NWxvr)HDF&YLdZa_DE?qGHYHo}vw{IxTerQoYbnD`?0C9BRdFMQY_s!< zlB(#A|Ll_M+l1uqYvEa{OZE!7AyAR2X5$PPnACnSi>Tjeey6!D3JOWBemg7TMl1;F zwSkXkRL!;2Nxs*o|M46&yHNApwb~qh2x3%Tf7P!Un9N9~o?j@R<0e%~bd=+IaIKY} ztx#9OtuPOswcINBe49syYf;eYM~J;MSmxdY!#XzxkBH)?#E%#>)V>oO!GUa@U>6-r z54&7P{`#A5DKO zhPg8ownv^{{b3Yo5g7fhP|Nk;Fdood*-o5XSrg?SP0#ul+!C{{egp8q6YZr2#BDQ( z!e~0XkezR;?-j7KG-;mNROWY`JIad|K`gAp}YMJRGY zfv_IGUUk<-#6n_VG4t^^uq#j8QG9HoaR|f@iZE1PlbaYP@_8Mnc>RtFqpZByc|r0} zT7WDXd7tW7x{KfI?nH5xmMjwS#PAIzbe;VThkIM~ZW6PaBlw@*4v+XorJ$=}mcYZ6 z672rWqK=+LHkjhyX%4H9oCo2x_Y~0#K*RSS?IbD-4JcgjEkD6|0y15ySh$S%(=(!S zZ1n2%I=|(6SCOEK{*$$JNu#7gEni=k44sU_1i4mB(q*__GgKUlx14aHxxW1VV=u#f z?Wkb99_+K;I41O;*^6L*$@XKD$MK}Fx{}!*1NQTFAGbQV{dcsU^!@bj;QM^J2YAdn z)k@Lr)xJh36x@~kjiJb_{$QoMTldC-CkE~c+HUF;uWtao(sa;OyX?!9&BQQ>dAB$d zh(4>(pQm#Y!(`K~?i=kJ|GBwQtJusLo99VVH<34`J!02 zW1%NdR2>5Uz1hHeI0g(S1+E9fR}=y)e2FQ~?K8O3HTzUexmF?+o@xDd<*DLnPiP)X z+y6Eg77SkxXcb4D3QKSi&v2s<|9S0ae zFy1?14E;n$N8TvhJ=Isy{eAz|jg;U=sBXSDB}XwW4l&JxK$GS#Ovs;i8^+8u4kG{B z5wbK5bBDihl}S$^mQEPJcLF)-ui*eQ6?4ufTf+|b6B?6~dEq13o3qymRu<~QP~+%M zY`3fH&V5gjOj5|IJFL@wKKeCeeNR%`=HgGV#2l$}{0qlZ<;9!Z1>^_v9Y{ImaXuCm z#5Lql7f(tHCDtl&-B)APCm#E4{&HE3*rlVnyegF)tSD74l?5mECX>urR`QN|x7Il} z*5*fORX;1QI&xQmHgn3{4J9m1GDM|2Q@@f8pbqRVdb)*K<3KG}K_LwLzOn40(M6$J zaTAbXX+pz?p*%#5USTqknQ*_rl`s{fYx4lA*IIjRa@&@Y_vjk{=I@r#8WE%C2$n_J_ZxTwnS^2@zC-wTdR6Ms*r~(#rqN zP*@%G8^NVlUpi*oXA42mMtq$R&O{t-iF}qgAZ*Dn)6FnK>vuwm=6QN|>;LGvaG6PzOEW@>2r(f!pzk zzEz4uJnSU`M!${u|0gtOu74$o*`Dz+~5Lt`4lLS<{Ycox)Pl(ixp?|=Vj)KlUxq@WjInakWAUNTX`u! z7VP^H@Za0JE7=Vq@+VFW^zjL8q0zH6Cef3WiQX|s9rdaj%PdE@`{S4*dtucyNzky| zfhZ$AJQ}DG*5IykG;d)pyA;lQ6BDr&x%&}vD)FOf@|^74@|zE?ONMC(XLtI~oK`5Holw_pdMwq! zdiAqOAM4P&P5l~yf9{H%RX%EG-Yj9CD*acnssEbuF99{TDt|rss!8$97-U%^dw5dR z>^;s(1?&Wfa?2bx7S%(lc7Lic9S3QXcm)$K%&Ote1){oQ4%Tap51;do-jK0gotX2_ z5B+W%dX8<4yh*J@4rw zqwLD7!?m{E?x5*Gy5lrE?iXyc(3%Ht-sOEEK8UWN94VD@)^ItR#y-W}g(qfvR9IX7 zrf6HvL@;A3*2_l8=*J1>|;`(--0vqI@p4qtf>aE{y2yq_xw7Gw9S3*D4=^-5c&$| z7FeF@JKc1iL>d;}*>}5v^;x%-%O)5$_?jT^9!O+MviG6)b5-1rg2(kxn;QLk)dQyO zbq=cXeJ`*g6OSiD&gfrY1qHs})C1;CSq^g8%1@b2s%M8bKhCfH+>iflpz4sLGFqbf zI{?xz$yG6{+1p?#sk})9jxDu6a;}93Q(z2h zVCwd&n)AWhdDsfC$upBL*r#?Q+82z84?Fq1cOB+d7H0UrVu`%>Xd9sFosktjU%9qx zo6j3{XM2HWTeT~!DekFtXRv)Os4s&+_ljdC#j)zu0iYB1zC9s&(vJt)q zy*o(nvDC8)jubBD6%m1u-z3yUp=iC@v3!t;V z-S^K?%uw6Q6t$&21Ib|7$d>KKu_6M`NalO2?iD+m1EZ;N!>5uCJpV@0SlZ`Exl*o; zu?sqKXW2>UjRZzN#cYtHi0fhH)^)=hwADhZW$H=kIG&eS?!{A4n&sWc1mA{oQ}RD$ zZDqmYluEQeq+)$0pLaaRxNc|o-M^WyqhW+mSOn$4?f3qc?=eaXo|a;|6AqO>pWaiP2cJ1

&$ctUcMUW{>U}@Rg#mO+WXXIK&AD;id{Ju`{j$Ff!0d{d(%qaL!AB)G(_I2(zAB0E9WU+$%a;9`8?R-ngkg9I+Jj=qje-h$e2?- zI}Z@_ue?WmPwrxLDvNf2@_78(4`#epqpw_c2+IFW_y4kqTB$K_|4^y!VyMv)&~i%q zL47HW#x-h=6IDwR{j@`k)I9actuxzqpPu{TvUn#EgiX~fRpQlY^vdFcYB+RQBT!u| z&D6;ZQ|7Q59G|M2K7R65bPswF4~ZW*_}1RAX0rwIyU+NjyS3f9@#j}d9iK*;6`dt= zXNjKVMOw-b0d2BuhUXEqX^jM9Q~N_-!=~2KZP}4ml?1q4aWf-Ukl(+NED`n zPRZ^us|xc3b;goGrTo`3z^Wu{r+Pq7dRg4*hRQ56DwOR{!QaD|QuB)#U3@A!n3!#* zPQAaV?9xovRhij+paTr>66M&{JpFT(VUPi5@Y&-{rIqBO>aiG7M3C(=x{xGi(=ol| zpyNQr`{umenTdv9^7NzvVqsj%`F&Gwcvr8}_vdSrYLord%#IMO|8`|~?2B#s^RnL) zt{0Bn$kn#j6noKMO4waz*N!Cqu=p)>+NQStL!-Q39{3Qj#(gK%I*%V%z#Zo_A+&bx zPYC5VN#jlv6IT*}?J_D<+e0#Q>YMZMi!)a%?_EUO$aOHCV9n{c^;@id^c1naqIwLr zP;3*osp>aOoc_S!n3$F~D=A)%HK8p-f?mX2)I?wWmJmF&Io*N?BpU__W+p~YFEKFg z`7wrhT1|Ridp0(;$ZPfop368=?x@XsvavV1c>K?u1jo180acx}0FUu1n|ha?6Y8R~ z0OwX>s{9{AEq%RX+V7fC%G5egM%|Ip0@nbpWDId-uzW>ZF9O5DMUL{aCTf%$!Q?tx z&N1*#Y1Xidk4@@uCB*%fd00H!uKZrbS2$TZzX^5pIu# zoL0I{Y9FQ-^OjX4W|sS@9NOid`h31VEGD3%6|&jbt$uYrkAxzfkL~@gDP}dPSkk1I z(@(fGhw@~_MfF&v6ufgBa@hP=)CJ((#He41=2^ekD(%d6|FemQZz{P|T{JSFAnf_QE({CXkJ^MfB~@Zue2& zIV|V=^pD=QE?SDzt_THpN6*M#Dd7y?HtkgFN7oa--HwVNnbny9TyqhyJW&}f;N@J6 z8t}Tgn;shjprex-D)d8g%aHAN45TPCeNaCJ(@emMq!Aox_O%OJZ(BC6d=X1=jxZ%% zt5?)NMO_quw(j@zIh{ka(R1GXJV*b`8f@(^5qrnHBeB0YC$Q^G8FPFLeQ+Dt8|4Ib zyPwJ(KE&r^c?gfw?r!N17dEi9eD0CIsY=L4nsxZAyl-wNCD}Z-DZyfZIq}viC%wq| zPv;T|YTj@$^m|VKnNa1HUM)YZh4SWR)cNp%BE~?-_;uuKY*V)Ffwpy?p~tzGl< zMH%w4m`zz3ulk1xSv4(4oX>N4KT6FVg9`v4O;9Yp7w>feQJbna`w4bBknNJ9am09LV2ek2si%^hfpT z#6jY&#eL~h2VxV42wCqD#CYxQN*%Xig(5KunnXNHWp~b-dF*0aQVynfQkrBudrfag zb&_^KQv$(g_8^_kT(y0f+Dn&ki+i1pA&Rc8!#bX*j~Hz=bgzl z<^zANz0Z7n+WmQ-PCoBk>U}syf9kp9cPzj#O*t)^z$WJOQ|%YmWofb8x_Aswo;}5U+L_8x2*&p!MO=ATC22C*F_{v z`rzSsueV=(l1oX-kH&%BV0k#prD~ws@P&MRs~aB{l#!rLur5H|Nh#+eOw?q(YS&U7 z671mMQT_0i&8y%u5#%dZKWifmfrLw6KS4G!z+h$1bEJ{`Ao27RXn)f<76kkXfHFt4 zr6pACbjWw>GcPed)=7AVc<3ziMeQ=%p13n+$MSg7pNfpeItH^+#K5c25Lj|rjI{K} zR@STiha#H(>f?3U98~qs}~(`*7}j|Ci${xFYVoHzp4-Uz_#8 z@tSM?$wh)@6`af8{ht2z7NzF!8ktt=6nEOi5# zlP7H()27=}E}13zkbDOdEwihJXbFE1c={n4zPEUXNQx2$f0rzVIB5Nq5V0iT)!sJ` zDGoNt(R$ZgV_il?`~wD{c)ytK=>Lpp#dZqF8A_>*tzFSkB~M;MHE_Ub`BxrsT~V*1 zXnnygNFv==_!^d(HqacdI6SeHwY!VLm(b!c&96zPkK8v)1Z@mJn6lH&5;c(Ltl ziL>%II5HKAlCwgvcs&9}RudHrrEQhAiU95}r(V++VP%e4<3CH9s$X&%rjD|< zvwe^y&qUammZ(#kzYjK^B2w__(L`he(2}i+Yx`z`GALUcAzSvGJoQFrjXL5|DyyvC z`D>e6?uj#<;a9Rq=hxd)9{JEuP){~qxb3s_JVD_G9??4sTHdewyThW|V7Eu`xyF?Q zZ&;T0S?*X1Lusp)iJ4n;Kh~d<-z1Q_@?C;jOx*HX@?Tz$zV|)(r`yBjm^ZbSLWk7b zESJ92&Bhl=Zi+oTr_y7?k~9==2FkH~luyP>&T#SB08VbgcDhF<6MaiA!djH9xU+xlz1)J^ zc;{)DWK}k{eK&>A9lp~ZF+Hh8Zrn71Q+NoRB}CRLpoDnNcW)l)eH6~ zz;qi}W>Ld@f+lcTeEgdc#J9&x_05y(yT%8^nQvaYbmpr@JUKy3cH0e|SZ!JodxAfq zRTyd7{CJ(Zb$61BwEG@`vV}RK%#zP_r%x^Z;B+?5Dweg zy}%bQI_l@9ANHuc-27|j==(of08BQ)RT(bbI#c`ioA`Z|!zG5d(<3gp7hUYha21}f z)8|0XZc{wxNU8&BCE38*izI~I^(d1H`|DP3nUvEV;A~<5B=pv(@g%A*~j8GQOUkB0Yicau0 zf8^?Xy1#kL+^fE%5=dPB3{-!N|bh6Y;*XQexu-D1@9W@tz ze9-J-aqT{^7W&a#bch)lcm4ao#@*bH*Q~wzVWj1srJ-fh_f>b@OAma-&5B+2$0b(Q zO||g@M6Q4RqKmLnnF2Z~n;0kEz{T0dP?5#eM0(+M7D<=vi>ESKb|TjmBLl3`xzreO zWFrr$m_^Q1l%oDaO6PTP8YfBy-8HhV~c{c+&h zrXy1h187#ci;u1aY}Wp~{&(lPXpAFJ$}Ap}1n&b4M>7nKyB`-mpuEvhlF2u28*2UH zK5r=-Xth>N2fjLqYRNVs64B)2F{#3c+>o~h)V9n%V(VRnO^4}A4G>7rH6Ylb&%CwY z&=I((hjY)dKp44JJkd&_QEi`N*}j~0B?6k}fBh8=f&ir&E`hP_%jt6r$tR}9M0pA? z^y~3DB#$UQDj`aYx}PtTN#85=R!4k|zI*es z)-O2E)A-@%Ce9af(--!cPuzpvaC6Zl?kw6*dhu)gFIjp4lSf|aqH}DVOzL{h0?X$n zdMmVy(-+v@&lzDL0<(TJFs#QDpSn z6-av>pX0PAyju;^fbqhF!tAquaB+-|G0zdTTEy}D>jp%^Y~}S#OBIyco>J2AA6sL(-&0ONHS-Efpb)w>wA z=8ddFC2WzXdUyQ!7(&b&U?sL>51qV$?@+t`=MZI81V(w42J*r$bAkGkYIx4cIeIUO z3SDOf=(gU^UBJF1N_1{9stEH6i<_|%doKf$vJ{qJ{t%p=6K~}o0vtzVsZ0G^mSiYl zKqxy18WbJ)dQ!m^NW{$if=n{Jac1-Z8dcHrUE{pt^IcH$3}+)I9bG^(oza-S!cEAb zp*>ScnTv`u07zFF!LUa;TfO`q8Zp;tHHUKkD54&b5Wqcn>72rCk%`;uS#H-5xsz0BIhC8WHty76!u`3Tn@$X= zTNWg@?lseqL7n{G`S0HLwu#n4<1zA79Qrij?-xu5Su0wO{zkF3wQxya>rMqjyv`O==#eX|iLQI&Xl7^q$dl4g~U~kD{qFBxMO8RHVRH7@-7Sh*?YEj=J3er)AFq2n7SH)rRZ1*-c zCYsY+%})7$x=TL3XOg>6Up$hnqK;u2Qye80iW{b;z#P~U!b#SfzvW2{1Dm~Lqzfk zJzsr_)2S%FR+QO<=~5mIAKJz)xONN%*(zJ!nD~Ky#AG)Mtsa*b*@f^?q3HbQ#X#rl zJ{d(sj(SN<*5Q$Ii9nyl`ZB(3X=J(n8E92B`Y#p$xfX;+ zyZ=n|HSuZMQ@R=9wEAFR*&RsGqP4t&$&)amD<`}B=j}P;oD=CVwsZHHp6O?(C$oW` zB6=qa+Ow89gZG74*`TXRoBQ~rF*SY*T9{Vr@~0j@1*4rZ|Ig1WDCsHEoFeccM9UvT z=Tk)Uk55NXe>ld-Z{mmPpWEP}`Zf+T1U>xEdAlR1o=w{Nc2;n#hHu}#%K7SJ z2Hwd_BZas`!x8A#X62!>8AAzP8F2ZJj`}Hiv8^0%f`7|9PWymaVt*hw=d8<_p!f6u zv>fZH`)>7;^<1fD0_{K#aRlgf8^W+e(HQ#+BEoOaP(3pmnvWv(df$DQ=Y06?*$w$t zFR05Lb=_3!a+SyI=ksq$*MkSQ=%3V$ruc4Ed#^OZ6MhONtY-gvYw(9PeS+-gY1S{P zT?;J^A_eW0r8&G91b=dy;cjGH`kLh)?t{UA5iH%3T|ypNOxqj zZ)bW-Hl+^-#x1z)OJdA2=sK(uG~n|FK9fPWgBX2v+>=|md6Cgtyd>v@T=|N7{Qd+sF$A&h;2w3`)fc($)F6(Z7u zws(;J0P}&771NG|ccbtp3F^p$lc{|$;~3W=Z$+0!cqO7|R?q1C6#F!9?3rS@M@jK9 zOLZ?fOU(F3)kt!W#?}D-g&#E1MXN#oT~EhZ5k3Uqi+sPib{ zt5oX~W4mWXP+H{mVR%D81`>=2X2fcFZ)jQ7o}Nw^O85!n=6^>tH%=4Sdf4LgVfSCN z1*q+BtqRcz8kjt@q)}n0JIj44Vj}u3HOzHBIj`+w9rxw*6Kv4^9M#@bj>`8@TEue^ znUap5));HlSrXROO0E`h#_6Xp`gBk^^vaU*^Ab)ckt7}>w9cr$gg!h)P^>hYn*; znODy^#rsP`H9G6-=@!YZ-Qw~=k~U{G2FI%_0uA(f*@M4u_&jRs1Z$|<-mIVQ`r|)d z#r7w>(vsWyD^%%^m&jMCOW((Ee-Zt(|FtDH z{$m{g+2B58fw$)K7a1-0rs7l)21IFzTCYDsk*2Z zcJ!Hu`4suLQl+fggY+kB^+tan!zSQuRGg`TrDIQpi8*zk%KP>@w?WKb1pOaNL+%f; zl)P+4_p9_#egXtc_h5)1HgErZwTgn|e{a^Gk5Gkcwhdo^u>pc527Na|keVf1Cnah~ z^~T$F6YE_)C}@#ER8jY3sNl`3XuPgHZd+n~Y|U3Iio4$j{R9g-Xs=q;$WK1IQ@mB} z0_k-rkPhmvY@BaU?q<3xe<^=OiO!EJUL+$o_o8hL<};x?rSuB-WCiDh%&>~9N31%R z$t?Fd3N%f&sl0%I)@!0oLsWw7fkB~?Nht9i(_!u{1;xbc83n{U#odJGKa< z;L|b+ZEm)tGAW^2CC1G7Tw&zRv_=s~>k&7QAY;1dr&%J~0@RQH?f|2B!}EC)-;HQg z@dS)F4|CaL3Ng%py(%TMEmS`4$^eGuJM!#S@V6}sMXYyzeHnexD81=RH9(UK5tyX+ zJq*C$LSkjJv8@q^kVVdd&z%BjT!;{n8f-c9i~+$wHq%&jubZ8AKk5Oix+%}yd2sB6 z04PgqP(|n&a>oDcd&g4t*l18+20o*gf~t?OD(a-FMce9GRB%ziXRXD|{~{}Q7Q$Ge zNB~W>n-LSxXJiB+1K?yK>=F{f)>Oz6E;` z)iM*=j(`0P_<;HssVQ^MWbsmVcuBDRfF|`qP5Zi;)QjNnSMC_5zu_T#NsHfzoJp3! z3BE8mjqBsTtoYHB_Fz9~8$n9aPNBb?y?yG=JvsrHD7~u~^dON<4j`8Jme<6<@2JWyg`u3j*}ijv+}qmp?Pn&E}AFuuQy%U8gzbt;tq>=pENQQ#!Xnd6Wi1W z9cwHzB_F8N6Cv_Xq0~`X@s>B841Ll=?TNN^QGZBpx(XEA6*k2SK5G)h-{rlBM*1%B zQzLZh`k!m+X-%N(+ZI`q%MDGiabW2qRK;^5(TtmO?sMG<-g#S`lR$F8c#z(t)9-Gf z>Sf#%$e286q}N?r9iBR3#GR$_`uL>jA~>D7Kh}Gc5L6FW&pG0$QD97?1OG9z+IeD%@Ztkyn-jui%x~y*f^rRkvatU2ZzR$dK<2;U`)g)JsloCoYGy=KL?q_!LF*(IT@Os8xbkW( z>iE2jNmx^Gw;PjFKyT26Oo?uM7hrXy^#jy4A{27#W5=8tcw+EKm-ux!#NeLM<~+pm~QxkY} zr4J@x>R;aYc2EN;p|J8b8CdD8GNWkT7lB~@K-e+)e4`C;kcX?Ykc9sLrtt$Y9d!us zi@=@w8BU)40({6f!;@p`1kz60$-OIjy2|!C{d$fUcm|Nlc+3ilO_Z(H63i~bK(o1r z&}>=-ag)?h78|gb*IHx_je&iD=snN;w8lal`7V=Yf-dxL-G1-$T2W)K4jNTWREx(` z)k!y@ARlnJ@=?Gf)9cv>Zk5x(GICHoSd($68b^5I`?RsV8i=?((WYLb{w;}@PhbcB z0MQ10pto6U0!e=n7)Dl!j23D8LeNF~jpdxpbKsJ(Pb&%=pZfIo7lA6Etp- z5})Utr@%5T{#FHyQ42HOgIUdyHoDJ!u)}!uxHGMJyEvqq<0WahQe0L?ij?7)Gr`-&j4z1x-ICzT=&IXHfNe$ zAQZU6h%`^$jjNB0b;(h|IeN*fgEko$1@!Ye+R<;`2*XD9C{2Q|2T4)YYR{-gI>EHl ztZEP>F~q**&M#sIB8V+{Rwo|cfhTu=3$9mb@BID61;~KTCsI}~MD;uatop5D{H16E zdB|qEzJ~u;=&!g20{I?FP8_SKoKu!H_9Eb2iNk9xBi5Q7;&y}T{(NpIm|5`Y3o)fR z6oF2fo?dHHOvPz^a?xA?Qho{CAn=xKYZ;&I{l^#@qzeULmPo%UMlT&v_DkiCGT^Tj z9hLSP;qDZ~Xpc8KY?sLH)P3;30$}%GRV|p&sBa!_kXOF@Gw~Qt4J=Xu>En>NUqM=34 ziO36?@grl$d?ei2aTX^SMBUcEKI(fyK*-jV3w?<38V8?VUYr;u@8p4me+%vqUW9i9 zB*K-l;)ZE{yD*f_pqeYjZ}Z31!Y^hF=vqBz%X^h1cnFwSeaja!L;_fmvx7U00U|QT z|9(8)2F>jC&SoO1q=|>nSGG8gxP|E?mM&Q!jUK9Bj;}D}-LcurCaNVYy^xG_Y9jaC zS>*k4FIdn%qR2ijG%E=~SY}p9OP3u5U}@vk)=~2+YpDRip&YO|45jHy%!>EJ+9h%S zVg9FNfLBC!IL0Y^PP2=UPG(2F8*n?^-P98)&f@Dr7|M5lB6(SrU5k->mHIZF+hbfk z+LAAGkPd5PWO*T5;{wP_oDkg#6b`X@$pPk_&&pCHPOrD-=Ms^ECIf$!X7JO0+Z{Q^ zmyy8_#fe0|5E2~XGzo&ubBd}<4o*95h7}M?{qpKf*Nh32(Im3UngpJ);DfT1)b&+wm#h^4N*4=``sz| zYGyaz2-t={i)cn@o#4$rN9d1F7BalI5M>$|ZzK@)cM1TcU&`0=q=sgmVU`Qe9Kq6Q zQU8@6J?Tm}392+^e;`NN9#kL}`!m2?7k+gMPQreKy9k8POz;VEAhPIbRQ>(y9J1Bl zhGF-TT6uw-Ya%lQ_)Vs;>2scjmW@+te|BAgo|*_I^0siyxl+=zV1ex{IIQwB*`_z8 zo0njfTV)y_BWPY(+|AoPl@ZlW`X$-_j|Yo^$n!~_Kk;=(EMaM5%*$;%Ae$_)^>^o( zC%U0njd^g%B-5Z|@&o47U#(3wEW+t-g(zMu0z?@IM(H6+^n6MW`rt5-e04|i4tSP{ zh|#@O4BxJ*h-);5N^1ao1tkm^6g0kJiMN&T@RjpM*MHz4->S^I$$Z$=Uj z0YViASxblii(IVfIUx-N7S?~lekJO7*4ejDzCD(Vm}XUb{-pZWYEbpT_*DhJj}u#I z%{x#b&|1giz0u`iJKsJ$ya(E1xs84wZQFXkY%7O+~twfbrHb|&}wely2|+Rj6k-8E4~OE5i@uYEe1(H|EsQ#%Iv-6hEF>10HDp7~x$;Fv22fwGkFF!pd>e$th3>N1bQjG882+oUZxKtNAZL=I)BUYeg zqKi7vhsb(ERP$6sH>G+ffKh2DXqc{ZD#(V@Ix(__pJz(_Kk_)BU!s@pH3s&%N6Bs~ z4!il)TxpI*NX3G>m7AvcM`B7Q<+np2>Kha6h2q+i5Or{+cJA6E^S z)BOMcFM~HeIlz$K*U8PG*?%AR@4ufB=9$X*Do~MwFEQ(a>N^Y2H0Km(4<&i_m;XzA zkxwHa^+Fl+SL#9v$e%`yrp~zr8&32xdhBPj&x8?mI(Qe~!#qCUxd4NDY0#TBU<&wm z&7*tcr*=od+fELo`g0%L?bMjEA*i(`<2UDDx`StswBNJ>5W8~~+hGMCnQ{Tzo()1c z_dzqr9^xU(DJPgnNI0OG9fSk?-};-wz_>GUz!!^~dg-lBo)6&$>iGS?z7=WvT2bi+ zJ7DEWO!NY17i0lq-3Prs{wwR#D_?85>qt)fj2pm2fF!cuKrcaHd_mEU_B)UN;akN$ z=!Drk-6&;O3e!$4&ArVhG(0f%b2-9W@YDKyP;;Bdm}0&A?XzJ)*dmpwGVG^Zvrf13 zRPBnfkppYRI}>mjpyjC{IDe!AE_3)%&5yn z&pZ(VzoyeFjy|VlJ#A1vyN8YH)%XpOjK2L_KGF7vLiVNfoAQPl#=HzkOX=elWcnXi zwH8jemRpP8dRigQhg@HWuJ7_EYMtSz8wk>b<3=NFYS_N0ot)%SU>w&6EgOm0xB6>a zgiSA^fk7_B0>p?a>$>NQG9kUc>OZdxQvY#F7}`<_olCyHV01Gz1zuNd_T z)oPoQCUdE&R(uoko^Pit$hgC}mRYBN<(1S6L)fY;G*M&ZufE&dU)Wx9U+j`&!kRfh z$@Az=c?nAbgcS8?Ku9qaIH%l6Xs~?dvLyRFPE{FTd6^z632+|K$`73*)k*n~a^oAQ zN3uO+Xi?-aR1}Mk+w9Hwm=P%0D$U>STzalzR#8_p*eG#9SyH=|wExa(N1=U^=bPR8 zw%M2aROZUPIgKlE$Cl1lXr;A19vaAg7La>MF86AzRpq{yw~mLeg?#mc(by|8y+s%6 zJ*#a_e$;;YplwU!{L4o7+g3$&^A|p~-+tTqw&Fs#x0tV0-nXVoHu)207C-5{`Bugl zdZ&bjimE#v$S@s6?8=e{w+e0>{>@MS7B|#4MZiTRJbK=y@!Y4w$d>7M6;&#c7DOU< zh2l6Fj_Tz!5fq|Tw+v;)>2u4T+x(yjr=Pm~Zx6TLBd8xCXIDtmqp@;a?7lzUXmQ~F zWzeu5Mm3+_eyI_(mEl=>evGBuVX4YoBGNLj%p0Sklytf*I&ybM#fxom>BbfBggY58 zW&+5cIfn0F8>T*Yzp1g{YW|d?cc;OQr>~^1%9rmww^V!&4nAzm*)QEaeEs50#pZ;k zeZ@3;dz&*4ZhWj@S>y$w{w3|AheWR&OVoz*%ly1?VcnAxo_9kchLnRW-L}*oF21Z5 z`SJS!y-nT6c**$-V~1pA?_ZH!84!B9e%ngfDxHlj^VF??Bblz}0w$0oh3Hj%bT6kT z-O;O4g}7KkVh;XWRhzMPYE`{9Q>KV?DPl`ie|5{d9BgSzdWW8YLOY>bI*Up~$zcJF ze0Kr;t1Ctlk0bT&St*??I}ti~ta0oG%mTF)U~1T>N=@6Qz1teXoHf!fYpXg2*{j-w z8T?Un@iGuc3D~vlObk?Occja=Te^?4lCk+gXqzm6aJBtZo1F*BXhsr5t(O|Zuz6OR z_N@j?t$Cr|$HeN<7~b>|qQ&Tal%N|QxOaW*c&S- z2Tl1+YOqFa4O-57!vZl24Kugc>bz3>QeSv>T)jE=ZMj;Of$ojK;59|l9U5y7mBQw05g|%3 zSfhC+rk}^8}%F8tnJN!wofe_cUkq=!iR3# zsS4DOGPOxW7VPrVv+P{(fU`*#hVtwH^eBn;@EBhI-4qvre$yTBwgefZ`*wETZs_+U z4Ji0kI2GMABDl1oobp%3q2l9Gmf1p)7!?^kA0(t`LSfAZJ14~)zA}FIreyK{&QjVF z&%}txXHBL>?YzNJ9Pi^nwi1S)1kv*WTwV;nY0WPDObG4&i4&qerTWVeL& zLp|I#DWjejnp$Jw-p-hqrqZ}7&+m*|qe}dzldBkrao5`F`^hG<)1IA73e6+xAyG}* zHTVZ})F72ajaBSCD3HaU!2MW!rT;YYck!4UTZj-2rpm1m`pO=3$4fzdWIdPPbCDI* zCw&71n4GlM zy(Win_4%ojF@VDUzq0S&byE4@IxijSZ?b=fAVVZ#T0J%x&l4Dt_p3a#I;7PQq=HZV z8Y|Z3cHCg*+r2m^qlYY7z|ASy6}bO*?BwWhqxEOU0UXnZ%&!9ukjkcfJeL`r`XD!} zNTzeXhWoDfaIZnqJ>~*}gV#U$EA>iW>D?F>R;@f*ns9$GIuR_ZzzACcem)=&2haz6 zl~pOl>qu2pJ@2n61P|Oyd$SvJ#jfE6d4)IaXYDFd?%l6QZIT;>?Y4O9DO}P zWg4eK3uOn2)c?w_X7ub?(Awk*gPAN}Wo=4vDbh8iYkHro>p%GX_~y;&`QAa2H|GP( z_t`UFHMgt_UjGDQQhY(19Nmg|=_*I4u5;MWIE|&oO&gBLJ)#VAg%+D$3gnB%3&CsY zzO8y7h2>;@)tWV3Us^xc`>RH{XQyGG0vL4XWl4KDyjNH@{t>3D!tQNHvULi$N3wGr!4*|2&Gl~~16IQOg zx$UB>LKK5ZqQ}`zvw^Asqqeq=@{r*FF^5w3~=C-DL~oM&?&?C#HVo!W0GhP z*^Tc&9m&v_kfy=*B`*NM@*gGgQSA-OPzi=aS=+YS$nbSx6KC&2Z#{lqOCJk8?Ne zuFc6994+!tB!&NE74;iIb@TAhbe8~%BN@?dUlco7LSUY9{!j+zVA$N1yD&*PZ4 zdw@>W)F<6E&&i_A#=zJD(YiILWa&Sbp6&jo?$mY5tn@OAZTr;xfImHs#NJl6V1j36 zxOPGv33zFkWB{CK`9{`a;R96BOTz_^Ih{0CejlZ9^t65P0EA2P(ofBU;248A{UubA zJmlg@yBq1hy4ZUTWuR%Z@zDBwaWA}e>xFl2 zmV*X?#OclW)OFS!3Be(J9o;n)}oZ z*D$CwUm>e>xT53Ns`LY@Q%n) zAh(i*B06!i8?|Lpa28cW{bjtG&B5y|VULn2kN@|ZEwrWn=bvzE5}jsCQNWoI1%)=P z9N26t^>M(nyU9D!7UcQAxE6BY8IaN=p0?yHv}Og5;`U*?#bBUivj|F)2-m=*qJa{I zp75px3glsdtdS(3Lqvc>wRLi-T~m8Sc^4Ijg~cL(lJOX%7O%_Bty@p~eRQ-{k*YvC zjtmLe)Vt~8YwVi51s>_};QUGOF&;3TifCo|8@W2r_DSQQFshK#4)(gx+_cx))=xoO zdlHw?qoo~;W&0%Yf^H3WX>7Z0>sq+zUPppI$kcP+Lw8s-jjkp5B8V#{m*l@Qn-2z^ zGS{X$pKjOQ3S`#-Am%2rBw()g#4`Oj^sqqbC8eGK zfxqz5Oad^$Yj`Ipg%>gaO&UM%H~%)rf9nzn?5E*8;x)|t2G4pj9}m2a1%R~gfN|IR z$SU^2-=qkKZHR4?11ZuDsIvL^j}CyNv+F)Bvr_OQ&{Twd`A_yAM5+-DB*;N%svu1x zg_Kf|*0=8H2s!!{TC64t(YN~l`K21PlR)v)i**m0u=bL1K1vVCQhX!?ga8QnSEr{r zA&sP3vZ0j>5ZR3ifkquh{VZr7{MFR-_ye;Q>9D@~fz!*JLp=2lGyC@pF$PG;pc`C+ z@)u8uu7ZAn$`OD#Bnr>NEsJ~j{}3T2ktn!5IK_9g5Ui^;e^D0-hXI1TB{9LDb^(>A z6p{f6s{8)6eCE4|eFFOb$+rT254B!8md+iC&dvO@!+M%j^m;Vk%t~HCbHbk#X)8m{ zBm`406g%eQ!%-Mef&G7asqlt*4Y@r_+5TzNLRkG%rTI&tsi_ZwaZM6j!hq(>r_Oey zVgxc^e)-QZT2g*#TvL*kC3CN(r%^)gnUjI*RoG{=yaHk@`tIo$v(pzdYP85o8l`|aeH}HCXb^&FV%N{sb^PECPQ$MF(#EtDyw?S~ zG)a*gG^@APWzzq)_i$uj%<24l^t#dfe@S+}j2?qFqjXZJAYlvXU5V4Mumaj8m0>?J zqraXx<~HP4R`-+2=@Sz^D+vE%12-0evnQ(B8VwBj?cKPRc>p^DJYGMxi@m-?4W7j6 zQ9~k7exP*l0Lll+HZn@-eqW=&!L)k3jaLU@Ul;%EO$?vjb47u@9zHN zM$?eJd2ZnFg*Qxn=eotE?1P&Jk4bx&&R5L;VDZVb?^m+jB6OLnTCC@^T2^SHy1UI) za%uT53(l>YkA=`v{x$KB8A6qVEE?CIi;~~|Sm@R)Q1?49e^Gkhx@7z>mhB+75#g~0 z79iAu|87I`=r%r4pK|c(V7UFF(zc5dZqu(+H>$GCjK}|h9i`f!AzV)5at79+ zHSPCqz3yq@YApj=m2f?by%#7Kq#*P2O6Y?r>MxU~TGORzQM}aW@QI&)sUL)w_vcWB zUK38!DEq-cvzqeec;;e{Wbg01CS3Dx+%J}t@H&N@;`N?9_eFH!$}8=*JsCUB*_Gky zpYfF`!dBykL=t(#at6n}&#ZrBU$Pg;bIsz_8UOt)D5`a+PnESZ&5Y{5Z`rB`-?Apd zMHwFJQ#xw*ZeZjDtmg)j?3k&*+IU#n6t5fYX-o=Z$%Il;Qqb6*+B(-Ec+Y!tk*QrN>-1yhIz`DeU)S5 z&4Xk7T?u`AuDw$eLChsTgpl)AlL-e@I*n8eSr#u&aA*(0v6%q39>t|hzE9Q(ccFdu z@1{?Yr951iyoJsX;?D6fsB4-{lO}Bx#hG&Stld5{TGm^q+8&rL)ngubcc(uuh+ou- zzqx<0K~FKGRl;#7eR^8_2o1#w)Pf;;AB`9E1V_3RA$N*;g;p_5-F>dtZO{F0Nt`W& z@xi`4WqIhNuj^jfhCJP$o4Y(z;7xM8FxSP8PRVA=YCVfunsZ4CbRRNXi>(8+YAOk; z9`q?4O2sLv4*xqk{~%I)*Y^U0;caqJ`|BvqkwGl#{}+oy;0z4aOZ|=I7QsD@Y z>2tSHwJpOwCYkl+MOlwDY&3V3YMf@^tzxq>qI;b>HCI{$v zlo?Z1HVypq%09!=wAj?B){(}GQ?x*s!`6_#g57G*UF%C)yP#a+?2#4NvP~xutamZv zDgE~i(b;a!*|(A0mjZld?BsDLe5K;bh6_y>Wo>)c4yL4NW?{n8iDskh?J+Gr0 z*&a#1T_Eb_@Elsk$I$g+L-O^M<`fyQTO*8xSR%d~uZybj)6V$UtJ)u)Bt%T-X+(D? zen_eMToBW{+arp4d71Fl)8m8q>Qu3TbJk^lEfVw2_~0DGaLV4S<1#H48Zs1hu3Vpx zG#3Oq4Io*`+^dyKB*lup2GwRKNYq;+&ul&}5MH85v@5mF4!dAn_)V8JV-86B|q8g>{ zS#p+SR9BYM$mQN~5?UIgyCXcu?u$w!)T8P+)K^&sy;VXXdu9}pk-dd%B73jQ?8@F-GRjCYva(lo z%t9d}vMD1(X2$RSqLvEx;IJN-zxM%J$_%T--~T(29lXzWC&r2SW8b#PD`e`Q zEUy0Y%ERnwxLToXCBWWOm2#(EE43}!j#Uf!M;V@%4_4?blbnh&?OZ6#Rh{dmniA1`GU4~?x^L&@x*{Q#dnSRh3*+*sQf52-F0_dd!7JFb@Qv{h%nkgQ zX+%SfK6UYGX9IBqMp##-5YTjV!T@_`d;+~eE&A1pnl<`5iDQ5m^l7*~*GyTZ^o?2= z=_ed8nv;U*fg3XAFqy*Yb z7rZY1IdP6^6B*1zR*eT#0pKh3=kVE%6jM};P%hUiH&;qO{h8a~l@O`J*IIJ-(PvA~ zVdtO3grZIgFaae{onDG)rG>Bc@8vB8(;vgw83A~pH!mBpN&o{euWv~{<~DZaLZs}W zX3=d^5ghF85D0vb*SE-IJDrCx?MD)$O-IsS;ppkt{kHV@7ZneHO7Ph#S#?keT?u?7 z*;h`*E!>Rsx}qGkS+hTe{XVAvtAyhFn7oYpI7n!Ux}-IK>ITs~)4mxMd_)+uM!cos zrepjMv1PCHZ9WWW<=XY*?EW(b)yKrExS)TKmJU_YNz5#~`A^xwB8pNWKdyk4Nek$? z1VfXL8Z^qwik#YFhJU6rr2TUxS{}>GXbjW z?*I~AV$w$NRk7QdYgUcxM^3{p=-i890dw=TR2bPCP}$h$zR~udYM8_^{)9A|imZDB4cBau*YNK#!mUKTiwL!yrgF7TI|z`Rz^dh~1mZirP?$ z=Tf1q>~Z6?5DaS|N*}+rgW%aWPM+V)2Bq(dgRcx_0JH%o(z%#@kb5}^WTU=&iCl&k z0bYaGUY|zqf-U>;c8A#Io43oEP6 z7Qt|OX7%wY1~5}8cAm@<9IrQI4bz}OnvDcpjpJuuTlJj=fZ1CBNXEv*oKO`B5fYp4 z&HV}3nrx_u<$@9wu>cl@2H4>zV8zjI954$w&tQQHeReR61;#%CguqYn9rwEse`bk2 z4j|;$F!=ZFtbe23G8RU_>BKR9;Zkx;#d`4T;P+RgF^OY)-}pw@P=2P~ z{+1em9xwPE?pH;nmw0X27wW#ofuXY$074M`Skdn8XL2+y*dZjQ+pCjqxCy_FBx@=` zF*Ga6e_ou_m2%7RopB5EnLyM7DVl>XSNFDO-9wu0b7(6})E_TjUpmO+GMj_mmVmUv zM;$&e@QVOgyT+N=1lPz*qIWJmE0Hb^Kw-)?56Gj_w8$8sxytL{$r#{I^tQ31?~|nCY?_be4~5j+p!PhDq@l4-NR=p&8}C;{-x3m zcSqV89YUPKV-{v+7k-wbOn?0WDl5RB7xKvQEV5^i89)3cMuh0@R%G3WgC`&*>K?3I zkDFqJlGoVv@@2G}To{T!>7sE{m*kQ1{+SH;2Z*mc-nNs9v%0L3aYF!BE`3I@5M8Mp zy-+aK+N5f4GGJ~Sm;oSA_D)~jPZIIF7^-z;Be-}2OHN7$JoAzjpSu6}5(uGUq0}q` z+L-i0glw|Sq%z{8cJl(`G=#t(=Fe$Sl!GGDafIFnywuT$8rvg4mnpW+oGt!c49eJ}i!1;HF4x)h06K|JK{r*v zX$mzn(hsPQ6pb74{xT@FG7|1HAY$rg+Q8igtqm${*|Qh0uk8bdRY3D+25dk{ELbo; zSRUs8&9C;*PY?$r z!0(QsWJA23VL`IWkAY343(eXOPxyPr)t$VLLVrkNh>4>{5wlL`5Zy&OGk*9-5-dSb zgvYLtnW{Urzdaot^G*{kEdqEOiz*J~%~}Xt*EJjfw|xZ!79pDvt0Esfk4+{3YJuJW zPKTiO#4VRN&>~}e;;$A!aK0mPO9!YTzQ$}xGXnb7b>R|bZK_!~e2Hd5wR3V*GVeS| zRM02G!S;|i>dbiShNsYb5RgG6j5QjHqX3T%dLms^noej68`-&3Yy1T|1x@%riqIE= zj;-TW${wgLKt`fR*vlvB|2KFfbjRTIkI`DDR+ z=12;6KLldqg6P8R9PGvlRN)@rh<}jLsd3qJljxxz<6Ypxf_$b=gXDarmHO`D^=UNE z7FM+}gK?DXQ6nROj^?-n-S3hHYL?CTM{9lgsFpig6O$^C)vx*mbu3SGFl9bsz6pSn zS0@YXhe|LYvFUY^LQ65c{uM;>{dOV0B$^ei%Boq2rCJ}d2Qa+dJ-`nxuEQ`X5(XiQ zHd?;2=4qhct;L_{)FK7P{TPX38-$kd$Cl8sIP<{2x<_`2z7&mggdP!KK!HzT5vP}^ z9iD=r9d84GbR?q+_{ankmhYmMGmCB+-v<=UooM&{U00BIvh)Aegg8wa3W-Fwp@cM_No51>(-`z07(mPQaW16lwk5Wwi)+@rw? z!Hq;EZqAH%i4}5@&BZ*eaK0r=LxjR(i`C#%lCmLAzEVnzoZQutMy(MPf3uA;sjL}P(Uupo70MYsqjgu^W`^$*mut4$# zA~)z`*fCUKd%dybFpirLBtjtv`5}HH%@7K zi!}#Dpa!@mu})g+CQ6ZpCu~~RBE0~fT=PQc2(9vBxWnz|hP;;D2-@gxy7xiSglF=| zTKR1#BuA6KDn|7eAM;uOYXGA?m}IA!|FNju{Is^(1|gtubPySyw{oiEXImRO*3Q0F#JQ$y|m}IX_PMa7|q> zuXXdRJ?|-Vil_>|m8|_?=fg=U zqldVql2q=d+Tn%GK3n{*g;UTogeNO@p6Kas#q&4lRg#%CY2$pW00+MNt_{*IT23xV z#9Ck!N=sp^$laAN^UxGn{xsa=l9E4Zf1n2qro55SmR}#+yeFWJ3ff4d!mH~;&}j4o zn2&uL#|g(32gl1_XP#D93$)WthXsNVtCx$Hg(#dr+Qh>ZR-|ZrI<*;as=D4>#2ogh z1<=qhh<^Tjmq_zA3wC3>0MEsiDwHHPbJOoNg7!L^o0+p;L(F6S3ZyXjK~v1KwYec) z>=w2M1T@Z+Ql#&BG;-TUu%3GvIxpGW_!N@8RL$X(q6X#4!TEbFw?{NDdd?6>Q1G6H z)N`=K+(<*!8PMYdK>nly-HB7GE>wcfC=l2FIKCQ2F4OmXb36j3fv5crc9@_dDddop zd`729M;GV22CR(Ud~drZd{_N-NJ*m!sIPw)f!b1#amDGhxd`FB zD|)mS$3WT}Ny+bBrcBZHp>QT1Lr|6kbB(iIh;}8Y{_#n6)Cro+&%EALCO>ngAAXVqLGQietGVCitXi2ez!c^B z$&rg&g5D^v+@WcG_I+NFEdZW_NmzfQTKG|D>W3s>P?D<5+^%QYwPVnB(h0#N?`8G3 z=;vhlLaVQby+y$D9zgZXvRWKv!$zh1^J~)0O#wmg_eD$dziANBY1DTS($j>Y3V14* zad*@o^j~L{G&V8W2yTQ*bbjgj?T|;`)@NS|Gto4y$LTCUr><}1K|N_D-AY8g(5IBF zBu>7{ChU5~fMy~hv)0zx0&kwj+Tr@@0kwY`oRbpBsp_ezZ*`uS2saj%9%h{VpeE=O z2ORFN_*nu65pC9A`jzRG9HgJda*W93*X)PF>V?(_I_k)j=<<6YUj8Vr7WjmY@?yYH z!A2UrX?kSrS?7V)dg2i{eSOJ*ClF>eEkVN#r2wvHEQBXz6k9 zGt-O@FI3ruaN0~=zUYmSds?8cqsg!vSyz(Kw4tYo&!s|;wmx^ki974~{*K?bqfZ=u zS-YecMP-Hwj|n^8sVnz6kKth zs&DMIr*t3aX?R*7bXx;i#S?4x9lz;yp3oV974yq-w=U?(!66v+>xc6nt!N}wC%TRF zS8nf33h%sXItW9$09%ZpBLSXMlaOpB5i?{<12>9etMZ)RQbP#NrNhv3j4RK@(P!~q zWe52)W6o|Fj-B+pL?tywJyOUApDA>f!x-lI?|s!r@|eSB5S8OUFX*8FE6XNJl_$g7 zW&|kb&|znkcS~El_Uh#?FQ`Y$ZTg`il|3ORL(wpYy5RTvRGspjd&mhRNsZx2IO7wu z^;UD{Ypi>_<_vYbJI4h{eM>~hTu91>Z%L>|7_{Y2K)CzY=V~5=4LEI+MJ$qG`fz)x zyyqjZrKz#Ffulj(Xfjw?1%M}XTbtg6@K8UXA#iNX1$wV8^2{5q4>+Iw7C_=@|5Ub^ zvORYG8dW{^Z9C%={FGcL7jxew&JI+UjOv=JXAhI>K72RKr)sx!VV>0{)M+*+v_@_5 z)=Mj68->TNROPZov{rXrcwFA($W?9Q0Ks$uVxv!MJ;J|-(T6LmQ`GUUv%wW`E%)WspswuSzoL9NR(*O2(U21pp{dch4?ysUdDtQyPZN$p6b ztlwOr`qTy8ai3E)x~b? z5j#|^@_RFB0%_bTJSLRF>Cqf|JZ&Xub|Q&-?@4|?DRLFf9#~UP9IvS+^)a}LdS9R1=hs*Rcf!O0&Q zaJ!B(s~6=9Eui!?t_p>A?(o<5{F6QV6}ZmV7)*9YJW4lDxzumPX@;cC-iQ~=A!SbB z6&A5gJa_q0q{IN8rbbx(r5@!9KNYHR2G7Bn^zhTqaFzIOm@UWt?b6a96lLs9QmpJ9 z$WcO};5C9psv3WxFR|?&bc^*HZM!a#l9E0I^V%WTM0VavZpj%FqvQ+cZsp&4U?4OW zBgRK;smaI|B%CtdEiN@i>S?GL>n=COqCR5~@6LH42UKZZXqVFYLbJefh;s`tPRC7G z=m{f{?^a3hx14eB^~lX{=S1DZ5)lASA^*Zlpo+L{IbXJNx67?`ew?ojlpQ_B`Y2`0 ztn;3Qag&@TB#z#ORO2jd?%hl0LM@;oBH3ycJ{~k9p+JloMq2b_8R990laLI!U-0Ht zOj@Zj1L40f_O>#ES?|7)F7+;lCCV1m6(uparF%+Xs>(j~+Shj*po=Tk8lzM{zPc@3 z-RT*z0`Z8;cx-h~|0ds`N$xuFw;#! z%}u;?t(%fWHc0Fyxk--N-QS9uHiY_R6>i>nRO5mQ@388b0mcOKc{zRS(i5BdV`|WB zq=9bOzbNC45GjymuBr?T?VH`mO4~b_pT2Z9mJi%+P`g7v7i4!8nLUsNIPd&Or1R?0 zFL!kMd+`Q%Qjwg&**anPcnM?gU&(E;ID7MN8Idc2(iBW=0cPuyArc|OUb39uYw z#fU??%roD9`Pon!nD9l&V&oc0vC_FzOKAPREuX&l?|X@7itTx6%8u{HKvzMfeUn2Sqbsum1`Y54V>1Ch%@0;jifyN>`vtF(mJ@b$t z%by@#t^5=U!IT@GJOI~Ar_lS?4SsYk0OYRDn7why#Kv(`KI*2yaw;Z;_yg@{&|7PPgXjWU!_oSG|2gCajBO;xcxMr~8NBFX zj^`2#nC`6{|5;e8SXZ9Qt_}rne6Z@fu*89h26I`;~qTAPzK-Z*)6IOkbZ0`c7vP zIr*TA8u-6i0Mvq4k^ZDo!t|eSk%lFQ2@Qat|IU4o9poR3TKm=$4c?qakm6m8YPSF19RS1& z;znuio4-y%SWY1Gf&KFgdiLKh^@tq4>DvV-r_$rU`=tC75SJjDaPt`_THG^)E$!lS_i_aaY+E_-cEKN9!N+ zuWKz0?{F>f+dRV2V13-uk&(=$9r5|S=ERr80&D!Ave~_w-$J_{_RcPgze`dgCXHts zZp_#Cuy1zF&Gu`M^=s>=_SvU9u4QHq)cY)ozPMAzxNy}vCFg704sNN_kNdY@xDe`A zSS99UoSM7|h!(KB$TzS=Thm`alp5|ofu>GW&=2QmTFIwF=?w8YN~xxuI>*gSB6}nZ zS24A2#Q9&sTKd|Eo4Ii@q_0hRDDq${6+dOOOFJS>h@!^WekGbAJ+!Su#xIVMU zi<{(Q2QDRQj(nBMJYGbsQ&u&%os^Nn?StJ8ih-JD;tSWs{;f z>@UJmmf*M(-+p(}OeILDtp#T-!Z6)c$+D-JvHVS>VJ=#3F z@@7Yw8Ij)xtK0rQVRt6*2IxHB-epU=>dUvFVA>sbN~nMLTTfb~!J%ziu+N#N()-uj z3)zhni3{S~y*69g^QU9D)fvl&3f2u~-%rLyMNcox7Hqq>DThdP=7yj9sT@6)(&)lj zX0Gms55RK;%qj78u>S=}>BzVhSFqVJVDKn~3Av8Mgs+#s1<5sMG)KQ{`h*iJ&JdP$ zsFg``?NgUzA&E6#@he+`w<^muRb`saAtMU83mQL->+%xfro!BUYwCX3q{`xPtI&Rd}O_0{mg!PX4-WKf1$NYScC1d&8BU&^E{8TKs6JXap8)p zq&y-IPclBtXb54`K4G@Z8E#Q0{L1(#x`rF~iS%U*)gYJ$gJ^-J_x30hTSv*gBuxMe z$4;4P-3~*Y5@KQq_()^KDv6WK8Wr~XWlhD4%qGg{%toFE)M4T?&v7hke`iu_Z;oKr z^Q)5m^3a6t_fsEt)C1&l4D`5%j_AKrjQ$v%Nw|&vFU+Y_aTveaSKBG6V zDG)0tILp1|MPg5^<4LY+%TM)JBKQE$g6sv~3a7}>d zW?Mr2!P8D|IT}SQDj`ShR~5I1dpV0dqRS3ae!V7MU9A~=b$2vu`21npisi;kw(dOl zLiS{}_hi;r?KT4Ho7n>#SFFc+w#S#`EO$SitUb^&qjsL&#s5*PUKbjRVg4p!d}+7) z_l}m&_*|&W>)z&Kp1>!?gdzcXsTD2iN+ZUTukL+(p@F;it(;cbYgDi9^z|ECeDuG& ze>>ElDji#uF4nJkux`z+8dRuraAwSV-c?xtS(enT*UvS6@~!o8Rfy+k=aX#vw$c}l zHJoZH(h1vDy72Nmee#vnNZ;y=n29d6A^x}6GhI*40SNUICm(U6hCRU5|5jal(vS}| zJ)3Q&C)!0tu>}T(ptN13e0L_9MjL%7b2}5?oK&fh05S9!S`%rPg$Uu5mFSw&8R9tHBSeV>6^m{c$U{*>;VcJ?y(Ky!LTZ8*) zZS20KULJR?`+ed=+pz~(3}KTElbM#9JeACfj8PX0@^+dIe@S>I(rJx)q2+T|exLlc z@ofK0iOp{UhorZSs$y_{Fsis^Y%t?g_Hg`5fMWcQs?Dq71Y($z3N-hwy_-3>|6uY< z+!tkny*d^Cn!M)?dcoDcx7C*U*Ava!?rz*2)0DI_W_w(2ep7K-U%+dDuRC4j;e*cH zT=CbP*Eu^547^IzYCeRRb?nWTyH-6X6!EuZWejfLKJ*Rt?6a7+*-Hv1`(vcy}$Cf;Lj$|n>Kmt{GYX0iIVJEe(wDYwZ}Y>o7^#|fQY*t(KVGd(l=v>R#SJ(1HRySE^wX$D8 z6FYUUA>a53-oh~9IgJYjGBr7KqQ1XUW(A6-o(sr7oKw28IhcQ5(2Bj+LiW{HXLH3g z<3Zc;rn-#=?O=9IP4;l#%vVvKyV@_hvt&CVMR9IL4{c22V+)XkL`F(MjN}HCR@=*v z+Dlih$^M`A2Ecrw*tm%Oh&Z4c-JGpAaFG8nz#L3$TRvO7e=Lz#={g`g6&Vsj9>PR7 zKx-m$D1XKvJ+JV(WR2Qud!@P)1w9+yr&OBu>M!0Y6hlk3lC$X7zp$_L`mHQ`VrZsp zsbX_DumhCH}!Y;-OF@4UWw;LBMZ=U2Fsn%E>@8n`iPyG_vJmL8O}MmAxD5yI)AbBvyl}AK{?^+8}I1z zgyIBfpOZ0Ln>q|3Ny`vlgOA8Zj@;~Xd7#1KN)49eX zv*msYud&7TB!XKuZuO1_^~xPKV|Dk6pJG;qEsa)EXEttO6f-@)sT>mB*7oqUNl&q# zt54+h*S{W%(mfQ*xZ!yVbR|OEpM^%`wHO>9(eVE00!D4mqc5bB_ho6CQD5A>&YsZ_ z>oJi3$-Q6q^NVqgSm|2<&+QZjz4ud(o3a&K)+}?#nC~(5pc|z zxy1AH^#*IXE(FM-;BKP8Y67shIf7! zqxQyHj{Ha+2_e6AC{7|Q?5H*Ks+HrN(Y7-eyrNO$TR1Blf!IUHmQ%g`Vz8 zeX9KMzIML8f2)ts1zsiB;n{RY;HnXvP>B^Ql+EBGsKk1 zPtK&rN_!Nj{zI)!R73s8m;E-|xw8*H?vgtCbsnA}Lhsj3~5TAr$?2J}i-)wyR<&ii0IKo!WdvrSSl> z$88Y@+uUn%@09KMZnS2|%&nhB_kS%$zRC2ny*4ZO01Gr~E`aBPwDV6>I?IAl9Bcsw zN7E;}RfcBV=UUo#rU68};44&OB3Lq^U}P#Y@#u4W@BI0E@(j__OCu+i3<#O2-Cok! z%?jiWB&4O?I_=#%H(}WE+UR&}<==$xDaIcOieNv&0D=L}#N#_(`+xFuzZA>)rjjXz zLH_7!LD=}SMxyfg4?1NNvlu?By+RHX=*jGu-=jOm@fGnj`F;Ex<&jj@6c#gup-DDv z;i>qGceiJ_+f!4521`Y1yqu~YANJ_(lV{YJV+XyIO8>x9eyEpOPbe4{&(3~n{+{wZx*kghj#Bj| zU!Nc1>oCy{TDo3y1@J2YDVSn`&x3lWrdW)kc}(l?-!bD-mww3mS#+l-^4FL#dfaPb z`#3EPK=gBsJnvm2!SclPysd~YsP_9r`ZYUUvfK98DoQH+X~}g-b{slGrDiAbBPdv?d@Qg{t;W1IJ$5ww+J&!jwhy9!`_iwW zN&9@Xor@_hhBuudmMBr;kL@5i5TMYY`D&y z3(ipgI#e%&1_az5hiD-o4Y1QG7S9M^B8K> zzlSQPk5%Htwvmf%i%Km2&0YC0`T~OlAeAjd9q8CLMAh}(Z*Zl(GHoD18+SFilSF-7 zpbUM&SeQqpr6tLh_s^K9nW#OH?c5BE$o_IBj;f;VP-s%s?=BQnyUdf6S@1vH+_mB% zP{5>>6?;&yd?)+t^A8f7>DS77WZllil4t}xaHkkd)|mJlXh=CI#fJIgG^Q~%owG(w z_uh>Y0(*odzl0Jw3Nc~?gH_x9;II!aJ8mxN9O7rZZ3pR241$6`NjOa!R(lm2{sq2| zoP3iN_Qi=PZ>fnC`S)xA8hH)bqsBoL^Y&&-KoQgQDJ-{5J*8h~Im&u;N6(YV<}P)V3=ria$7yeloCk3EvlC8KQFo!}OI}tt z(d%;)+*N%>vD!?zd0ROJLn~3LBeUs3uwD2_RgrrhA!ZAk-iqg+i|9aOCA9Ec$Jakq zq+kGm*(MQfNL;0a!;6xOn+l&#T+~dWiwoQOogC12#{b8A|LgKHDFyzCo%8#lgYrAT z*m*rVtkL?RTVCflPa$GKuA0Eq;q&L<^GBdbkr%GM1>6$?c;ulCctKBd_Hviu8Aevz zSjwN$AN$*ahMvkQ%Bera`L)z{Jnw$IO#&l42c+pwNFUW@VCdPbR=lA;QM+FoMH8!o!Hy(N?SX)0P9K zGGpsC?(!*myA~4U`-WUZD9l*$B_6=g{pKDagijJs5z1IQ_p8eGLDge37*te{R?i=` zhd*_6gIPuZ^}Un6Z*#JRN-Z-wb_9TyLIoh+Jf~3rO3ET=cx`UxXW= z#~MB+X$Q^Lf1)nlm=x&9`XE00WD9i~jdL zJR60B^~jGIV9Brw4OfKcWD+dmX(U zusVDoitr*tko6C|%kJicAS#R*=QFqB+5D!d-;?uJBMW%eJA&?l@tt!^IW11Buvla zz@@HsoO}>2JQRU?n=JMP-DKHSKCPVEbUH_AuV-bCD^gdk{P&Vt^;Z)kQVjs_N%Dvwe`+a3Q@-YU(~$v=L-5u}4=HBVh+ zN61U51Y!Fu+3$~r54BgioHOKTbg}UaeF-XQrDl9$w}PgG17thI?;$8Dpz8S%6U|CB z#s!tVeJqYXw{9MN3MWxrMu%0>`%SzFvK9Sb!JuS{EsiP}rAD)=Ms0T4=5oQNS?k>8 zg+*o@YrZcDaT+owjB)`USzH!O-1oMS9r)Bog9>=Bv&1K`m)v>x*C7jX%oj# zDjv5dRz=J>ZEZsehy6SzVC2{xAW&0+ad`wKPVm+815*dUDcLJ>^rb_8(Sq@^y3L zDcBzh6@3;e=W{yWua;4L_@t`udqv%d3O(~+sfW-a5$UJF=rIgW73M|;2+LgctC-!H zqR}ZS=f?@UKmF?=$u`TNF5dr6dns~e>rHnAtB;9_mpb8wx?NU+hb35J>ehZrHacg54xb!KM7>S*M-#g`L zinIbw*96pH57HlHA9K}?p7ePVc+z;6E+kVOpwLoA3Yo0ibEQH{y&FM?-PI>a93%a6 zz9x?Qc@lo-nZCg5pc{(Y1^fyp;Egy0-8KCCa3MGgU{-vNu9oS<55&cAqWG@MrhF;5 zhso$ifI_Fw+y%~-aWu#Mphxdb@D^R~-0;Dh-W82(O~0RQn|`)qd3N`8!M0QLz3Oah zpZ>*F`LZ8sFJ%iJM40v@5HPHpN<&ak6$pBqtTr{fSPK~DY+%o%T(i3XeYmN`NoZLd zpKtK~hy$4oZGLi^glI29BjT-+A^U}7i{vz_ao>$Vd3iiEQ`AGJPE!Ea-gUN<97*+m zUwb7vhKIe~<1P_%jUy88-Na8uoviAQAlu5yo=68DrBrt5?7vR{ktwWIEiJ%EUoWxWVMZGDKr0spF!Nz{4{=CM^+iyUM@C6a}yKlCfJ1{S91Ekiw z-oQCP(f_|&Sm9_8dvsB06kIGAbcrVcK`t3gKz+rkm;T1y>booCbO18FnJb@IV)OKohuGu{I?Sr)`fI867bB-T}=h7T}Xa`rOGg*nm(6 zlqWvuN-rXm;D8ql|A)n)iu-`_h+vEb0t&Jyk=yp6$dP)3mPSVoAj5a(cfm1;7PwAH zUUwde1^`m^8Du|1=Z@T6ZBS@=(|2WwwLf=r*_K;{;F*B=q&uGUN+o=mXgp%3u)7SeC)$KJACOn3x!nk41b6yGUxUCyn=L z3)3KUBH1r#M{{`b7=LC-bdfQf-v~Ma@w(U-bQrA;0D$lfIn7j}QOx9-(-@uuHiN}2 zfb1RpPCokl&ny_&;i?Y`RVqlhzk$p;kkOB_E`)R_*?Mn=6A5t5mocALI*%v?2^d53usZ7 zfu|q}Ac}54PftIQsCDNJgxL6syEnIqV~h%b_4N+o>N|&@`rO#4kRvvZ^$|S& zh@-H|6eF@%)l7XH4m`x4y045?`~1Fk$Lq({nfBPr!?cLqjA>5>HC$8%;!~VmTi7Zs zoE9Y7TKblV2i~fVfR;Qa1|5w&A@8)Jvw9c;zIKfjRG6DdElE3Wzu;rS7P*)GdL^}2 z4eib@2zKDGj>!X{3dr_5JYZp!{D8l)G)#&J>pOTxoxq$TxeCoVtRd_w$Of2b-rS}( z2^Hw33nMkpa>{w@g*?dBG_kGaue35xVg4mX`t9iVN0N6;6oR7Jk%ARcG|zOtuu4CEd(0wA{|``jN8cM5{cQM^c&42l~^v#mx(BF=wEhgGH{ ztpCPGgl}*({Cji>St~1#om!6nJN*PNfDL|9$P!hheWqKWLi~TD z3-VxyAbub~Zh$>fh!;$$$Os>%tJbaKQnlMgs^|OTX`}xW6tJA|1Ti_FU_#0S z0Zc2%W2=RoBdcFfQ0!agLo>Q4Ev(ODDLj**bWFi0=dgMw@%*&-8U?4>v6o;(gksp^_$W6Jo9_ z%KI*Zu{i0eaAxU04=jQTPoB!N=Anv(Md$CP7MEOHxXNL4qmYV5@6z=<7x5ET{ zT-blbH8*U1*1Vh}cRTDwxnM?bT+VZ{QdYIX{qtMxsP^M!aRyly(}GTkw2DuGh1;IQ zLCI;@Sg7TBu;&S}gOEW!0c%p@o)!Ix_No(I8oTDT^G;doMjJzxR?c6{k3=4 zu^zCjadrrqh1J`vb*p*<7TNN;+foP^$6e;+?A{`Elteb>@eAuFW&{h#1HwQB94<@% zuu28ukT0EW0FY;$7l7fINwgd8Uj@9(Ls%Q)h1|e~BV>63+`-DTL@i?oW2`?{s8M09 zxI;+hr*Hs_xt{?j=7adOR|VDoKzWa_SGnF3`3mUkZWFyQ&{K3MDuXm)wBEB%x0#Iu z>nl9`pn^r4WH(UXRhBqMi z1YEuof&5Z&awX^&gDn#+^ZNwkqtp?oY~++EqAQ9BMpQsiVLsXZXTQQGk4X@N@BH77 z6jOVYdWo(JKZ=r{KpPee}-(gkDh1sqLb^*K53%E1mgx$o) zg(|X>Y0zM1;SeHB#K}~T-OgVKs_%o`6d%$f&U0e|bWO4=^(sQwq{+dO#ft^{p9H%& z*iAhEbhd5+$yAvdDBt-PX_Za33z@J1{W_ptF987iTlwIiK?L{}@Vz)=-h11e;;Qv+ zRu1j|hybC1Fac3_x5@$$6HiP-ir&?E=3hU1} z26b+)nI+*Lff1&58Gc?j2G-|Je{B`k26rdiW1L zCUqkKu%b`l$CrIFuzSBFxE?myBnYG$t$q$U#K&sFmv>YuZ4C(xEoDCd8b}N9Re;TH z!LmLWh|)-Omlz|~3k<23Kz5`QHaj9TY6K1%0{4Ge8Zph;{(DlY3V+z_#{2zezdr^Q z8ObXF@WpM)@9DLK*lJDD*Mq;$7i;w9J400V#K2pBsM){~mxxI@tr=*3MOy5|q+VOg z8;c9G)I^I!o=j9GA?|*9k@l5~>emX!`XnAILo5{a5Zu@?f0SjhxAQ-*773-~0VOcwtO~fO`pUBP*^D z>`+GKv=@Xx5i-af)k@)-jPqjs%Je&v|Q zY@*h!JY{Nybf$AaqHlwPq=GF-J2PedI;Fy>ha|09y@!*ixWF!>+6?0Hk0K}X(hD2I_ zHzGV|UmCG3aPgjvp`VZy6PXd-|JAn*jKELNl_+J66y#U2LTs=YJTcdP>lW(^ex;$M zy?JI-vGKM0)qgCd+aVrpeS-}WErHL0%=iCO$i-j=fYi`rZt@**$bw5Ck|XSOQ1E*L zX`A}@7FI(NqQzh^tjh-ZLEkFh)!EvU!b#PoRM%awpuzR&88h>_FQ=zv|2ouY|5)|O^cBrwEwuQbslx((m{)-I{5Zda5=*eOl9PIXB zF>E@PHaI*gdK$e?B>Kn4W!!E|Z}R!=Mi%$;FbO^|6iNcxdvhOv`1J6Q32%}D@tZ7) ziQlFMhc?r=#A~)gM6k>Itej`J)%SFPS^mDmk%zE@9fhOk9`&;k& zIARUee*CpG%qtK+QTT6b(ND-A`93jcFe8Q*R(r6-&QsGd4=ADg2u?}VwpxXOf6@r#0#JJZdnoWm8hh!{--@YWg4bW38a>O|q~odq-X%}KmZzJbb;gBn!a^Dn4WsZ&eoGafzz#XP zyg;DH6qLmM86jT=CT=UpA`6J61x3lbYtUa1 zDln#i1|A(!D%hM$1?*)kczYd{M#$%qA)mY84$QeKe#ys8=n@LwV|6p&qm_kVBYRWqul)r0y7OXXb# zY)PR2@H;UEI;4Q(gjlm`!g2He-wh-;1M?8hYzCjFOfyG?2oW=rDnb$JeW)$G35vHOiV2G3XWQ~_jxLg+lM{0`WD`edry(SS+ z%T-_ByLG#e4Vu82uMMNpq2&NvLOV>bM`(eJ>#J@Kv)X9_s_&YYt}zB!(its^lPdk5 zyYp{<(l1z#z{y@Qa3S^s( z;&^iU2^Nf>62JJmicJR*x9c0}si0Lhg^jul9Bg-dY5&0xzZ77_{arV(#c=$dKfk5- zR|^0eT)4wvMy~6Im;~w;&x-TlA>7H3@bGfBZ}`}ca4|x4feh;BRiUr?0`L5l6To-g zdLb$l3tSRNWK?g-URZ?F9A#S95`J!Q9d&(GSL|Aw+`l#c01?Z(*>Y*LqeV@!H|Ajc zaQ_lw*$cV6Dwid=eOP{}V4=SUwcy)W|2${|+>y#JyZi3+dJ@vKpu_|bm2%T(r7Uv{ z|84^N5d)Ifua)kYe_FE}A8}}kR1w|3{q13r!vXO1Uz`~)GsE@WnLRhINQPD=At8An zAe2XNIskGe6QHi#QQ6Nt+-qCp7Jzlc8emlYQIb_*`G4GpYJ@JSe?WV+uKK+uwuD4K!H|>Te3GO*gG&Js1s~zvUEDe1!Hlvfr+7V-I127L_ znm7sq9kuCoPlO>Q9w^cahLCNoby0Ol7B2y))$|nda|RWbv2KrJ9JtNW;jkUxR*>9T z2XZ^({JRJ3^fUu1jWOfCzwtp6K^p3FZ*HnKnt%{GM=;%gYb_!LmY4iX6 z+!7Pdf-mB9q7e+N*LP*>-)%o&FZ>eEQ+o3JWT`pyGhmqxn)f}o=|YqYpi}kTO~`FY z3|fUFv#r>`2NH`FNJ`2_nXF91GxASkzUHIUZWfWVH5A6x=a2jq^N$$&? zq?|8C-Km{^JNt||+OW)tIqVWHi}&%ov!R`Qtd zWkB@S008&-?D;j0UGOD(InvG_1SjtxhJH(yB+n@Lz7fK54q{dq^Ilg1b(A-`5j0>K zg+^FZ)n%tUZr%I!UG+oiRfRTB5T;-Rwc$YM>rR0h#{FWT=p&eQ@Px9?S;Svs<)vmG`VDxAgcZEZtDUMlsmi4ymxyNV8 z*JMgfD>@QbNTI60bjR%i;islvXN4ruqKnUabdBomDYB&qJ!0 zDl9*V*Ug56KVnvvoV?2_IkO8{;h^2Ot|xgToslr$X3FfJ-Mfj|jTNF1tIeYdKes*; zO>xrN$_jvqZ@s+DGALBWfQ@?G%t0=5@SVVf*i*weTXFb7J&p2FOZFd2^G&*bAqyfM z2HuN0Y($+&90mLD%rvs<9V^IU*AN%jETUL1r(i*!T|S7%%vIlExFZOtA@U|B)Gm6^dQcboP4{| z`<7$%HBl^dTT%8`&7UN67gz7(^k=$GN{Nx#BOr(^Bv+{(j4!$z+8j089Npl5=?+Gt z`nZDkI@OhzY*#j@C!(Den)c6$J5(hrpCrA1p>5Ky-f$d`ywiFlE}>0l5C2MyXvy-o zgptyuqhp$872s1O9&@z3+03|9uS!(hR)DKpyY3!~)b)j~@zuWYq5x;rNRctg}EQ7B5n@~NzKo;4RVP~ON7RS^n zu~jU9P?!1gZW%w%#(T%JuvH z7DN;QrMsk4q*Fq=1*A(i-Q5k+-QC@}=~B9-8w(>^&rK8J|jmf&lGAM!uY; z8X;Pd#jNf+S%gLFUc&feP+8@GhgiI8Bao(}(?r)tcn&AG);;=EK^x&6x+YD&3#g?$ znh>cX4GTLPZqsjzzBM|PXoeDWtaF=EhlP)|U*1VR}v|3Dy*}s$e3 z6e6`#f=!Klz_nb9NVyj5E6ZnHtHSGhsDQ1!#;vQI`TKnjbYD)sBlVMiQNd&e+V-SQ z)74Z45wnB8bX{5LOu{H07WFt^0<#lC{(Wdas%e?ymdTx6rjt@0j%01DbzX&P0>J|| zXMVU7bftv?$uWn(dEYfdl(=&;OE+eW<)tY)?O&8umo3l96(H&zD zbf&55&S_Z^8ZPbE5%j*J_Ca;`K`RduQYb`Te=X-WGu8vW4Q+>|zrvcmO@R)z45SJ@ zGdnb(el^6*WqGaC{ZyVw&}>Otc<#6ZUm&TejzO#a-kkbR>b#BvHU?e6l%wdXgRR2; zgtD-;{DWGisB1~I(bh|_i1%`I+7cG+=h- zQ(!oj1#}#j^X7vl#QlYmhbB7I4L=&o*0XroZR(3-b6pUdXg~EBbuLj)H%+-pq^lm; zax%ix6s8(ycd&@NnodUS0*cy9maRa{Al1lRtXZYk--+b~1!6THABhhrk)8__e%usI z2&u~PSP=808G<$6HdUwG!A$1wt>xp(v?9JV$Aa1zq{fr{gW2?`B7vGA;>7ZaiKN~V zgX~Ynsnng<0ccukE)6=Q?A3LOb5pGJCc2<7>j^3-#sULj1?cGsM11N879mp*cnO)0 z1DNt=ffW8c3Mw?1!t%Cfyzg$%^C}z6Cl=1Rt5oULds+>sg~yAbaq|9UaT>T_RQzo< zxaRhG3KJS*g$~Q5f&2~HD~Puuw9LJE3LV)`+?&~7DTwzus`}o*^IZpJC^vo>3kPl$(Kg5l4rj}z+Kecqh>TVMG@F*Px@cl#Bw4_9R%S*mAhjq*aoQZ{o>T_zW9H7S$GB ztnYA63?LNWSIW~4AXGU4s73=eVsz_n9PBT?UTu?m1%aE3GG#)~5#~~AVWw(*@9Ie> zR^yCy$=YXPI&nMo0XC*r=&CuVU!g>D@s_(Cc$NmnV! z23`gYzS*{uiDF-+R4hxrHW3Zl4^Y4rT6Kf8^>^Cv_LqEn`_`tL`DE-i22bR(mr|2CR|;HCt64UJ0&2 zgsa7YaeYliqPvhduO>MSP`%PB(~baq+DvAR5~#i~lQqf|q$xsTn`xg77GA|EhzghT zM%T17xWLTT%^Im>C4pB_d@mQo&XL6+&&Q9JhuoUYP8Fj>p?xu-gS>&z3g_~wz~sR( z-LH=y2}%0}P6n~bt$*+<7_X?jZliu0zQ>WDekblwB~WGw(wVgu&zW zITwM5f^hJ-FueqDX8JYR?{c&Ma>qc8Tp3*#ZrjKQ&8?GIhh3I42m@}#?~|~!&cHye zmz4aC=A^0SB7TmGYJD{O?x`v}{A`t`euClyT*!t`S)kwd%@t(C*HuXUsl)2K^vi+J z>3d1ou;At(@r~)8xrq$4P-vyXp7@iA!2y?U^rES6R*I-9nQB5_hV8Q4!KHH;{qt+o z*`K4I6s`u_9#ra1OYxG+c0w8=f8**fm&2?d~^Iw zkEcQcn%d-r#Yqs3JT+ZiU8quE4g0ZRon#!Xxa=G7qYlM^LHI*E@LB_P7j1&OBhG&I ze8YYn_1)7G?gh0pZrOWM`{~F>(`G*!%El^SPQ{cNirAr|sGwOBqN?CVF`R@66xuK( z=~14uYqFo0vc;0Fez2L_K70=byHtPox5MEc2U?(w zFyTmgB-esus>v$pt~kT}d1!<6nt!+iq=&?MZ}jH(o6od;sr$3?S(~NEk3eb#N|I?O z$W=X<7MbTI*qBa>0@qPN6elLy;OfI1&+AL-;QEM%4`^txUzde`iyyXPV`6+lL1h0_ z$?;JK@m{Gi1k)8;P;i?h({8)4n_4Bz9F;Cp*++Q=Bi)* z6Z*2@I%=M>hQzr!2lYcYr@e-rMkEH4Pvt+FiXI^ho=9~#1DoMfm>?0-ZO0kV=@are z{O+_?J$e@v%DEc$J-Qy@2X%D)if1{W>x)DUqama(b1bxCNBXeV*!wjx4W!sPQx16g zu3;Ed-LC$|0rja-!Z{IkQGt6Bh0#0HM% z`58M5!Vv9qLN0XbATJyki_WipZ@XW;B9d8()5xMaVqC@te5_%n5$NKu^2phxS8 zEb&M9;G=WaMkr4)5*2M^vF=a=A)?nn7)%fqDHA(6a+w-|u7$XkwZ3p5Um6RqMAg}fz>fh+f%$B}}>h$rhvc)AXj2=q&CzL>L(@2GkI$t|jV z@(Z4?-ukVHRH)NH>E+<8cCAHnl#2>%F3~a){pDO@296uA{spy##nt@UuqASco&&RKA)Q6|hX{$2=m@V#T%ou4s+w_hoy#>w| z+<*NL_(Y54BZ~R`V;(0*k84at#SU}*kCv-^7iRr_ML5wS9J~Wb;rC?EDrzxbJTEu3 z)XPP+4Ahc}lg}alY@mz!G{{NZEZtl$n#nheedh~~P_9fUReplVU~^b))9?)rt`50! zke>Y2_Lt5VWXIK*A_KJgxs6v3p}Tu-VP=`D`HRL*Rh;2;RiC;#$TU1QJ<6ERs5`@O zq~4}8#PQ!No~8g{WAzP; zs(wfqj&N|S{E-Ne;+739OBl1JIB!_ag3?NzXa()CSGgP>5yt$FFE>$dnIv-qe`OxN z`+O#BFU>r@t=*_+S+cA?ykW4WUEP`3eKcRJxQwSTZWdbE-#noYGCwd68Rq4wJNoAx;_-fh zS*XDKTFnL(hKaoejAp>}jRvU10e}8j0aG0OV}5qJ>j76R;~~n@pwzB5h)j5NB|Tak ziGgyOTq{|JbCsvI=>US`SPmPQ*XABL73@b8^1!o!NGq*ttNBjmN1kFagbv4j?8q!nav_tZefCbc2aWT zOEH7`XgdSpoWAPN$(S^3Nswh(8xM46XF-jFk4Mv`ojAqe3C##|7Fjx| z6~!O~!J&~Kn^jnsrL34lZjOqea^7e?-{j88&bHH}bL1u?B?X@R0oeWzuof?VD0U%Z zc$Uw%IJ1y5tOA!vseUS#ExZlL`X&V4rvP1-dB7391)BAXSFd`^>z549Bo4cQxLuBn z1krN>V@r~DlpB*IfT}tG2SzT2E-Bqs+lDIr^=%z`v#j&-NL1X~mfDj3Zec9k&gXt> zSjdZ8qkB~8B^!8jqLx=@1AZHqshlu}Xz%KO03ElBF4!39=%r|eBB`;{h806Km?Y}Y zsIO^4HWEdVh&ok{?Q1zv97jf3-yn#6DO^&{?<5*^bg9 z=hq1c)o6ITJUF)bNR#pJEC;X*@Aw6P z3*vhqDp_e#pkp7<)IU~c`g#_d#GOKysV^37BiK9EM-RV6OyT?LeF(_iAq4s4%H^6# z2nlSLhWnbME=0!{^IELO6>GqX!>+X zxou?40RsYCA{P(->jjehuY9{6qG4IXl%^%jvUMs_L*+B7F;kgJzm+xO$_nHUNj5XT z8Nk`Zep;n@hppc>v!btT!WC{?tFCm%rB9Glw~yAQ=0a4!I2YyvA&EWgjyBw#&=vOD zh$e-<0A}{S(PdC9Z$>Lw7$BJmS05ik5RM@I-jCkJ|lpdq2J9)z}8nY0k!J zgvg}8t1-bOHe}B3dcOA!jQ46wyGV>z+FIM<1qvU$==!LQTE$k1yOD&jhJ-GMvPcId zg)l#8o5_vVCjl#hxGvP5?`JJ`_rb|_8drK5!qGQyu^Jx5Q+4hKAIcORV;P;$8_&QB zh8_j-U4xoM|19i*0cJ-toDs&++fjn4=}f_=U;X^vu(M-<0mV!@YC0vkasw!4)rY|V z&yPR>V0wZcG3vWP=u|spY{1X;9dz1Rt%7>HL9t*OXY}-I4pEYCiJx4HN|&ujD4H?N z*w`3ACxFs7Iw+)S3)EObz*tsRhfXE?dyfr`SOm5{xK}!gbR;EKe*_E>#$e+0ESuoe zhDS&Rm9%sNPd_fddB5A0?14JRm8Pd_uF;ITFk{}eT*hO4r9dyKLLJ~=mXae_VnD4h zl%!uh!uOHs0*naFJ1I3A2?D-;h1<81&v)aRG+lgA#tl`x80pEsVll|_x-PHXUi`Md zXDFBE-O6*3d54@P-!36k&m0~XvUML27ZU#=)bq$%<#x@{2i$SYS9B`n znL+`tX6t11U&@P=i+F$pfTmF}5gLPKLV&Xa|+*A

ZtcopyXANAWkTZTB5@>`=)CB|+sVJJwduIS=X!J&TNVb}8u@Fe)!#RDCD(QqCt%r^v2zLgATxxelsd~=5}QfNNaDEtgB z?7Dlyo9p{P#~Tg8M&e?Y_pNQ?DeF*znDBq^5-gx|pM-eYIYe2}M>g4Yv6t!#Q0sN< ze?Ih7+ny7oJQozk)w?(M`PsyNOb*f8O|5_6<=zb}(8z*6IN6_;3-&G$f|A>Kw)c%chxmNLh|8pT^zEuuctNp(%cy&_Na zo|g6Zf`%6 z7cE3)6_i(>nGbuby0IK+JpT^f;0_3u$NJYq0T-Mwgc*MR^jpn`uDX48eDn>$8$}uK zk_;zP4~~AKUTsA6{*^+`gHvPTc>iNX*~_2#4QY@6lLTI9L*ZvBiW6gLTNLNNJ*r23 zTdtPkZOrE0#JItC&x=-F~D_v zQn;Gu_D0e>acex5`%P7nzce4x(JbEzs-13UvwnInPjS(x27bT21IzySv_W{yw&U?L zo+C;3%evwi?#^$a)SCxIR=gi%9UhG;pqiWLUK&+}N(1%h)?b_)%`g7yyfsQpb|AZQ znH@%3pWe6jg&)CtYu{zLYK~!keYLy!l-?_bW*KWdg4T4|j<9*8St%zI~R^9b3{*e66L|F3cVZz()2$qdyPk&%JhVLB4p*~{+IfLtv- z7V_u`B0){$kA!ac2SEn=1q4A56p>!+^!rc%qkOprU-G1!5 zFTU_=z{>5yvhQW_HJ1CW%P|Bl`+C(Ua-yq-)!Sptj`>U{hs?p7x=8@x6qC7ul2QR2 z@2Ie$F^KsfC@T-FP{QRuq&R9ylNeK(#v7ldmvHs^ut1RZ{HuguiDHEO$=+1@uUMza zRk8ktT^f^G?KhNSatgwb(NF6aVYQTN?HVgNi*iXrAM-QY@88$&&ESRi`TItj*BG}$ zoaGdAiI75xqJ$bIh>B=(Z0L19#B~DeMuhq?cJShNL-nd#Dck)9%eU!7Rp2c0kwZB1 z%827Awvcj-nWm>Ytv`Se1uBvPn2QG$i{GSy!_QUK#h@cYQ|t-`&taGb`+1ha28-#I zQ{2>hn1w67@C3T~%yZkcR&XQ>?vnt=2ELY)(9Hp(-&w6J;^nEKReoz)s=X{K;6{7c zRpmgcK!nn;R-NsHE0s*?sfti-Cf=6z)xts1W`e^u!WK8*DLX*;PW|H~zljyKmOXZ(e0HyK>ox^G(QtWTu#J!lAgwx= zS;8*vPS0oriB7V%ewR`FXKsrLR8buE70(63?Y+Nm&Zcp}bu(pxe#2&cm{-DHuXpKu z1^J$~AnZ|$y24tD?DgAUya@?XJ85Qfyy)Vl*P4o4C4`h$g2ATCxoLtz4mW0;Dq`WW0QRx9H(N&9Ad>eSBTe?quD#LRn|!=q=gJ zx|ECx5YAIxp2vh0TOrmoSp>I)mWLl6ro{TT5~U9l#Zy!qOzC~Xq9QwUtUjdXM73f= z{FmCJ^GW9sfW6D8@4G$(X6A5>k#x>3#)%+qpbHmAqp$Zhm_xUtYOw08WtAD+YSkER zXpU*`Px9Z5j(F)I(0NP~NqRV&^ z?g34BkF|!oNfS0i&1A4jw@Wlv`r9o_Hh_lP5Z1mPY}uD$jCG54On}G}saSK1xu*KF zcBQLiwYW-WC=bb`dz~{XR)O*oLB`eL70#N-e=_1&J1>{KTRfC64APp&ujYMw$7Di~ z2-xhUv@5*R-U%1{yoib_mG(sUYxn2}WyLn29)49V1d&NMXO$gmzm==G7{ z{=Z#AO=;`cr_mTq>R}{tm0TPO8MuCR`-v*_g>bF$r%1R7?bw^669KbU^=Ye(io!&a zWkdMcFT+A}l*6(p2f>y>dwFVnuptbb4e_AGUZ&tA`(Ve&bkbDg?3@Ex`F>^%D@Mt( z&zwtCS%KbW6Vvb@qW*1ttWT*Ltq7Dw5*-bRSFPrsN_1#AF8*5?&&El43f_6!#$9oSq@ z(_d^8?v|((+`>dl8wnmIVE0alhx~ggSh)d#QbOSd>x!R{@JKAn?D4ykZl@xg@k{&G zm!tapZfTvODp6WLM(=)y?>-co%fRSa5dqIdxk8E(PM%eOdgm0#S18->D!KW06JZdi zzxbT%YKv#~aqy{ny~$rrAoQev#dwF{__oCK^MUMv)PeIUuM>~xg46Y6cpYcqHQyLR zlaA-(vD!A_bYazIn`;#Qmt9HR%vzc~c~$S3yoo$ReqqULH|)tk_^`1g*G9EkdH$Dg zdRkEt{;eIvX@DsC_T$s*tQ16&Qg`!KY8jxoZ13O_`=G-8D~&HXw)qDW9|O8iVE}B=WDXvA4-v^sqVnNTdN+&ci>uAH0_7a6e9f_ z0bkEQ>P#ypq`WQ%wD4Y}nKr|1y5|k%4b?*w385D*4SOdl5L=CwSW0J&>TIrI(Aye) zuPc7hk=;PXF48?~0-rBz1BxeqJk{Aoge%A0q(u1MIa)Yj7kK{o@T(G;?Lv1J?|P1|eoqcFN$RAo)x zfrCcWF}ye|j3DkYDjGiPi2RrTcxTlz6yuuPs_*vmhp2-otxpUXs)N&T=dz)Vds)n8_MM@{D zCZuF{>$1_S?*qLSZTTHuPxIF#n~O|$rEo|R^u`hJ=u;PvgB8NmTo}UlhqsZ-;iFL& zU?Fp;V)|UuxXQ0pjw2WHfTYQpabpy{fMoZbex)zE7*dG&7RbQF2jEenQ^t6w*YIF*AZ9eP_1ng1J)J-Ete&`eHbI&o{|u9{L*d83`+rfi|&S81ti z;;r}k!<(S$RDFJ~R|kLl>_$Z{26?T%c|p`swo8niD%Q)jj!@1e=s(CEQait{gwM?m zU3;>1fuO#(3IUo7_(asHt8JhAmORMEC9BiP3|VmHppED*xk)VQy~&*$PM%Be`9;=) z&uQRdOkB7g`l>#dul|Fpxgg0^tg4(o^*L-to5$_mwo6z*W-yZTqK(Wp*ySIm-@2fh zx5}NRCskMdM&&wV;$Z;ZF-}mGPFGEQzig5$?Z7@xRXcPm8xzB6MO8dERy?Vt@`@25 zsVGF+EtQ%vQ93PQPr6HNemoa= z9X#_%WGiAQbL2TVW=FCGTpi1!eAqBm;1w#JJUM<@XyWttW+k_qXV!ldZ(r#pEdG~a zX!&RVYGa|tn2;`CVseeJpRv~0WnV1pcYwA&usG_Ugqo;C%TqgClrNJwE@?5kIJv87 z&f%p;aR{t%W-7S%-6!c60uRz8NL%ci>RU#W9qOXn+7qQ zsOQQRCM8@&!Vc_ymZlcKpnbAT$6ZK;P+079g)WnScdxc87>=$yA2BDc8usyA%-KRo z{k|I)&%1eM=3puVXIE?<_A4yCg6oQM*Bq(3n?gQG5yw}Wa z7PRNmQbzTZYPRlvGlvf2ZGX8$)KnuFyFAr}*mF@4s3lF2g^U1^3Ha9VW$K=idw;@R zu8jR>5B`L>pAK*Uaj~@(4`)DUa~-#*lxqY|x&K)x$qUNsobWM``ufSP!q<9x*mK6H zblDpwmdwRB7gCu364ruJid`QqsCUyJ_WA>%rFS-WnExF?WK(3!V?vaR98D@}b;cDx ztsgy*yn@+odm$)%xWE=xek2l_QG}}D>}<{g-5mtUd^Rix$cOkxgSCo;gSce9e*&f$ z(Tf<0A{-}Ykx2qOi*GlG4LAn63rwqu&OwLPE|lYElypwu zW}+)N>MEnG->J1yKgRhOM6VW-iI^1DZU@Bc89mbhedH$?b^UW`x90#vOI2;f&~6Em zEX?rF+*yh{oRH}Ae)6!z`Emc>SDd!qQR^>b(ghY-*J@Vusy&D4`A%YHJX%*k#RA|z z`!7z%!QI}?_RNGxV<)a=zFF?7V`d&Z3k!g55xB7aL-qo$COhy$E$h4qLVYfOzNXr5 zb18VR_Dv#mKnX}zU-C6&P4vbyGn?Tfju~8mijc72lQ_)jQkwY=cEBL);~wMu?%EHc^Q1sj+j{K{jqh7xjlE&ee8YxVARcTTK5w%wZ2|8d`7DEt*v&Jz0 zb2VDAlDn6(qm&(5GFdT#h+i^5#g%%z)g7RIF%d?L#rog0-9NcUmP-prx1<X%NB}zi8)*9o)mV7RciF0E1R(54+ZX9< z<};U^7$ccHkgrw!+?yqVMmf&PCxE6KNNo0hlxOCD_i(j4F`o!Nz9j?ow}G!O-6jvb zTG_RGlkLMgiM0sCWF0Z5*FTc$!_|teS6PBEXAdU}~E7&(~)wcW{uzOT|{ z_Me&ld1T|4_&55~yc`ep=-e|2&*wt9>JmJ{AqyOPq~^ANg3;4^rIJUXJ#@*ebW&(&ukDNuwt?)?Lid9ekh1@ij55FXKsNMguIRZes68?NX5%*a+uIjewMn_+7!W`$Q#K4`qyZX*G zRZlmV`{*bivG$qqr!#6kFY{#o`BPNTT#){SYpkn_XeA{I7=gUD^lPlEGVObjndK0W z`xD61C-!uW3r>s_ku%wz;GNX)FMOg4Im$zYhm!lFPM?$nnDRIs^rHQ9Pbs6^DcJV> z+I!cK-X8DG=7iFJ68^Hb5X#+0A@2W&rSov4`v2p-m6152l8j@8zRAgmtYa2sSKmtE zkc6ykhjZ*gvLc&f6#6E|ip(QqbL?a9eT?HAXL7jbcklfp&gcC)uh;YWct~AZn@w-h zU?51lA~+)+iVg9c6Ta z6qY$2R^gt)H)s~9btu?c%rhWJgVV&eLV_fS;_(5Cz^ltK z3oG4iRTigF9GDnwB5|FO$AH*pF#sRp_EJemm6HDXySeNEqEp=wk5tR>l zPjN0*G2E`}62p{C6y1iM%R;WtS3(m^LXC-cwFpf9kaEeetIHTH&XI8xYguESf-7== zo)@$$FRko1Xf6WfT#*oXBt~jU*1$>0V@{Zk8#PZ$+Bb#x~^z-1Bm3Pi;>fRk#zVoF|-igMG5cyUaF z`_66ipa4|F^Nt0R7IC}JY9JU)-IL@E8v&v}SfEUYWiI*`gRCYMvph$Dx!Ja;PT6|u zdkv*cONgEZCfb?!t6UuIY}haPtRUx@mT;QuKW+j*a@aqANpQT3W#T=SD)j&#jcn8) zm}y>+llGkUF}syxU!}iLAzp9sHDc|blbgIX0M0ezo|)Mh%9;SpjV!F_cwgtg5LoS=%atTckxnq8teS9;p~TM+;wzy?5&?}x;8rc;|_Jq zXsg{nL#%lo{d{<;OV{I*qC`1d41Fl;$c%R5^K4S{)N z-oB?@#A;oU!LGuL~FNXlfA_cswc6_GXJw zBUKSqT9{v3zNU{qQQHp{RK1CeGSi^4p$ajVkiNrdEqgm@2vsssXD6L&l=ry6@xq5W zXIaK+_qOWuS@dywTVUeEyoEC+HDEV50@mGacm=RMkrOP6eu*h4*stwBV#Xk9XdmF6 zu)gX8wb{>y6D?<-d}m5@^6j7nJ}as4-1%Y6HD(XfR9i5AuoiwXkM_>u`Xzc5V^eY3 z$f`Pqa?Xo&toNz%9a_zyb+lSr901a#%&o0NxkN2U!n*SVXC(kfXRmg!?TXE|Ql8+F9Jy6zg3e;*WL zcJNVtQ%)C7;dTMbGH-B`7a1VYnum2fVLzk%IS%;Gtp~+Cl*1JV_(Pl$x1$$|rSME> zYf-G5>8h0XT=7%wp{`XnYL6bf6HXRhF&j{S%b4vNu zA63GWmXiDvrVAr$!*-{G!hRT)>}udxcMa_jy6p?^!`d1$`SArYlz!D9>Yc0}Mv3 zzcyIT_BV=WxhmM$=-v=+>(_Ty7!HS(3~G&jx&Avpn9*}nzra+b8|hiYD1cfC54&Y4 zw=wcarXkGKnF#qAXS^xnfL95LQr6aovVe)7E4-qiHTiu;^?N0z1o6>9K?VG#fy5Nn zIW|{_1+O&~VXIZ1+nZCkG*gGrC@Ut`dYSF|lskrf&vR>~q=s?G?SF{dGJxyP%?4yD zE!&=qsfNpcxCSxEmUPftknm0Ck`j)oxrQ6Q)YFRKTw^~6sHU?pLbWeh;$SYG50gh~ z4Mzr9kpTjuB;SQL2uXIPjYDKPGq1H?9_%fa4Id29jN0Qw6qM6*II3z`$JJVO^YET) z_xu56gCNVO8^CvQ5Ov&8ge0%PwzDdP@|f&C*W~cv1;8>jJqQjT{AVi2zBq=bSb10` z?czj|`Pya?b+93_LkCtB&!ZO6U&X3!Q7Ao7SCUwhk0WeuO28%te-yGk9h*wtUcRtv z4ITIOoGQX3C}Nd6e|eF@(Ku!gR$z|JF`e}#nIwy*CaBEN@8_|j9C7y-&GO;r*6@uN z#a?yDnoR-aIM?;s6(e{Cq8Mh#R0C)-d=6xZQiJ&sXz#Xg>z6z*8Q{WI%YWNLF9LGR zxOV}Te|exrE2Ds`v*Stf-8rg{ub>RZT@yjK;r~33!4aSK{a!ADq--`90GPayrSiJg z8)wt1Gv39XtUZ|ldrCLd`rCiNux?$V^37cveK{F(S@DAVi~cQlJS2}B%JvbQ_XC<)aCt0+RR zhEX1gjS_3>Opnp9+svT#67!%^GXdO-`?=qkUn_Z4FArkQZNeD5gfQtZC&D^w#u~nu z+HLB11o9jqgDLYF)L{>1F${FSmw0b@B7^*36>DtKjs|>WJDr*RtYNILV;?`RqYiw< zbDiP3ld81>*v0ztDgCElQ4}wjReMf%zozTD>+nKgpw8nDZuj;USyo)}ZVA-wLpNGeXw2&K1rm=(SeNqx}cF zS0)8CMY(588^PBVa5oJRE8WoONL6>+rMO+OceclN{ikj#S6a>bzK zj^|r5_Um7CnEecRwIp_nL_oIy^6OJ0f{`gd7n27iZuf-T(gVDR$zV-8nYhB6e14|z zw3|~e^tB$%#?pCj;>}b_gpT@dH+M7j*Nj+Kq8*B&x;_Yvx?2D)|C9V6%t5Sjk*QVY zEKO0Hc9+8oSn|`3bFpP@kac`&E+aRk=JS8?{vMDrD_NJRdoTZ31l4%6KVq0L068fW z21?QjDmnu-Nza^PRd_82QAd%36N1VVYcb{J!B6Qbk{A zNc*;?sY{*-OcPAI@ znhV&Wk9wVT`0K7qFI82*8icAvPO*RtgCihJ%XZk`%;iZjR$x-0Cn(Ju|61g-Z z6A=@es*o1pvGzxY5V78Jz%}Y21x=O(dOx_`jdKdq{}cQkXgT8bC-iIv6a3Ym{<%so z><#*pf5g?D#N#MQ>z5=D<~;(;mHblFa%bpdP2`;1SXHuR+w3n77mkzDDeSN}&qsr2 z#^U{~N2pLPrPG7JYaG*h8x@k~!tQX;C)N1&Bu~9(`e9ssYS0y^#cIO$&B`cX` z3Z423i@LBO(Ni7%o5*3KgJrr?=Z$t)B({BK*4a}H)4J~uvBy9dZep>o3_$@qP^f zSOP?uMl3xZyq-)||DC49^75-+p0Lhmv3r1mk1M~jq6K7T;88>!6kV4!I<-W z5>LmG>+!Q#*dOM%tqQaB@2$b&s6T8ddRi zjB7UZ=7S@R<}nV+dpDRt&V{(DKH&k_#$NqSf_gJv30a@LZ%~9szI5QN| zM!f`@kXF@8OO&D3~T%U+MN9VyE@98?gK(G>*!PT$!nz`jE*-Ib@~GmKyAlPDB}VZ z!nQ3KCQyS0bR_!NIc#=4@O13NBMof_!=Y$M1t6!W{l!D_+3A<(d@bab46zs4nW@?Z z{d)3pjjjTcyf$)-IZaU1#-#oI=lJUt6I_UxoLdI1ImOQ0VtYBkrHS9UcQr`WGb;Xr zC(5B~j*FdXKRq32X{;#!fRUd(xT?GURoeaY>{vZHMwYy#(-0Ibx+5@|Vkrk)vRJd` z7`i)v zX18}P{*>`!{-iH+POWVhdEAaj_!!cQ0zQga;adIBdP(|{yU&1*D7)peFDd=ccqG*B z1O@FCCQ;fL)0(Q{-Jn0`w7_dDYNFz50_neJ%#R14QH{%vw7VWl9Vy~rq+9mK5*@82 z5T=t5p_RV&(&)(QEyl8$bGm+7&i-*8Pq?S$IjxhL2Ta~~(~(glp=fLxFzig}4wfgza`bfs_vlaBL-(IHA z5bUEFgO+h$P#|(TW<$pOXrTjp_w_Cnu^JPFE-}s>au6#WfBt? zt3!v$7$hI|@G3Ts&F;Lhbz+oLk97rZFhM<>hYd^O3iP?1#;(<_8K zCnC8U+)V_gJMd4?c-70}f2vT&35^fPuV%TI8O+W$cDoBlcJ;$N6x<C(U$E+|v=h{f;zs3gB0CAM0DwP%XXSuux%<3|S> z{M8>av7D0}z+U>AR71T0M~~(tYs%wmp=}85wKU5mSwE%8Z}X+Y<)w$AcQie2qq5Bq z+l|-1-ny~}*Qx#;C`y(a)1T@yvFYQ6bn%_1DPu3#HCyf9A~$!YPW-k%5ig1{G9WnR zwf?vFz-XWUS&O!=DC)5K{5UH9`D%^b@-SkHdwUorkpORPHn>X;jL7>M0utO}{-nmM zrp(^iq@1J;%;9uR65bQg*CJ(^=K2eTI-32NVMfI<67sp%FNqXv+#DAv0Jplg?nFcK zyFle%z6-Vm!dg<0NmctlPygj;n=E%9plG@Q{}* zeZSmCU84**l!{oZQNzzD|M3T;gw~F*!1Xl`1?hV|7O`(#zRO*;_#`otpqfLF_Qq&M zsz?30VklCro#LyikY$wF)CR`fMfZ&T+Wl;#t#!K)!sOpByH|SilA$bUD@fv&L*vHv z!PAZ9n6EF5_ViJNgIcrkAwF1IeMR_hteM_SZv7Cf5>4*=3lyn~YDThr1WJCcKr{Iw zArt}PGku!rW^S~NpiN9qj%!SqFZ3&DppnykSe7CnsZg8Zvt|ag3Og{;2NpO^9$06d zeu;)>_)kRdQN`Rjx1DJT==q}?Fx76qu*%EXSf0?#6LDe@8vL;pqn{_svT-$3WC#F_ zWrYohuxck*h$$n#>6~{DMPGB1lp*omQKEdD#g0RJS!p(BX1U+gNv;4ADu_$#_*m%8 z%LAoN3SX^SaAx>|@EHo^kH=jFZ_(nsuc&{bLLuKkW4vVNCw58| zY;r*@AP=AU={yw8GWt7$F>2|U36YU}u~=fK=dU@`@p|65H4{914>tzd{vkkdCxr!R zT^K&iq78-G3(pN-obFSb)1%q#?=6)NfH}hRINROpWwwqKCE^f@{aItBJne*?xwlK(P2I?_VuL4MUOS zzt5MmTL)&cE&XmKpY{fa--R{)GvboP#B_G_z)Ql^>Q1QcWI3lS$XzhaxSEQEI<1zoL| ze%Nek^^2czc`zR77XTL_pGnLf>IhxYP?52|E`Fq?%L5|ckEJLi0+mRh_D^-?bN8AU zO@$8|ceBI8fMatvX&DH8uS3_KD32?J@?q8-Kzmj)QQhmd7vYPudk$hKT>bKNWj*BZ)_V_JIsIc^A+c^R6|&)p@{Dr^i_a z8@)HzFEROS_xr%+zoGVMh`eO33pA+M`4ktaavUGf-XHX>LOKQfqZ78BdjppeK+Uj` zH*kRv?AXobCeaaa&+gqb5-_$(BCbYdk{kZCA2xEVXROR;CR)%pNUXgG{4#l`GJ$h1 zW<#JGaeQL4*l{WQq`>{b^If&!D!y59ZZpv%EF5_*r)+f>(Sm+0qPWTEDR?AI(V2*b z$M!HIxlF8elFu36QNn^CyE?m@ub*yvDCd9Ia_+yN*Umfy4!|mi-(4Wq#OHsQ5k_S2 zje#8cPeLjFzQ`2B8R2g`&N`k#z71aK{(5g$S_9h_v>W3;YINMOFdisIGoK_Pdq_Ly zF&xpNh~;`@1nrh#$O!upKrd$mumPWdMko%GgvplryE!^f+d0JY-{g90cMGp_Ad-h_ zcwd8|<@1}`7NN+}Bg0Sqp_Z<1>+B2TuG?Chwx7|g$IYlAC^f7#-1UI~d`wmy=DAS3;JL%h0>+r$%xN+tZu{fU(UCd7K2)$gtUG>#eG7=ug zsK!d&asZgQF5aJSHn~J?uNI+jGWq@D}Rt9Bix z^pd7tN5>1c+NKxobrZOth)JYW^wjG$cywo)?MKttm=I3jW8h4-j=?=(B+OXZJ+DKP z?s`^sz_N2L%|Sy?4e%;Mn4OZqpevj`GO8}oDlpXz5#^XVWXAUtm{R7ZUZc@3#H>2IBnYOFuIV5g!2 zZBZwe#~u&S&69Iz1*=jWkvX$?(% zdjN7@s~_Y)K;p;1ry2;58p%?^HQvR5_oH`gb54}RxOEpa;*_uI8;!fwhEfzze=$dV zclCWIA&$Vxv*Rhehs^ASRi$4vW#mV(xcd=0kKb>9 zprvUTPnv*N+$pyMn74=BrCe#RWk=TqroJWY2t3|MyR}!{W}3s`4p(h*_nU|IYW2Se zRD_O1AjSJX+VbiqdemTN)~_L`(V8SYllF}J>6(=EB9Te;LAq7GQp9#$O~C$%jfv)) z1tDv(Es(M1&kTPpsmlIo!r-j;OYN_5QXUugqnFW+3SOMRn(&3aUE%zvj-1Orv5elA5N{^8f? zNU|f|NadT4PP`cBx8dK~*mupaW#GrR9`C%7Uk@m8rCW!@{~iUU0G8!eYs@`>)dFvm zypN;U(j3$`7D;+t=OrHWW;Dx=RkR4o4AB<@*1+e13@#9{l?|{isB2GQaZOAst zr4?X{dfL>!UB;9Y7fI}{=X2e#$g9Kwo&IsWh<^~}71nEL87AHN%kubpm6_6{<-ax<<9Py@9y`Bq+7*8o{&dHH@;*zw3`5yO*u9jvD)ZafMQ{n|I>g{r!eVR+n6kTLR7GA6p$`B@asLr+=7Bi%qdj zklZ~0d&3Vka2mTcL{^}6{rW@NLmQ~z=`c7)w$ap)C*p_8*M7f_E<*6{|ATN z@elFV)F~(Z$0ZU@wPr$V%TmY4W`R^7i?WKjwf+2cdlbS|H-ve6YEyfZEYwpk()r@7 z;cLy)Djnb9c;MT2WtncKJHb+&v`AQwGjKzOrVjso{AT2DP=QEiaKzzCri4e4qjWeR zt3o*?VIav!W|b|ApzeO|*?CZPgT0pe6`}6Lzrr5{WJSgNG)hufC$6{~KMz=w9XCol z(2C$C%8T~-A9#PR)19dClc;I@aA9XjOuEyb&Zqy(DLT)muW~ob6R!B3FxjWM-+S@# z<7B~iVZ)_c<4^qrx-BY9njk{Qa*knkuPXk1>H6qN;C$Gw=m(m?ujJDhSxID?QaLgS z^g`C74|~=2yt7W}0fHD-@fZBYS;yj`l&Ku7beGdrSgYc*3jeb!&hPq)T7AgRQ z>{s}5w+v0hfm&!tMbbx(+};hc#;F`!)85g&zH^rM*;V8TqtWJ^!V;UxAtGF1pjlxI zp>D-Ihm5G*a29&LaeTh}ew@0D1PAEwJ3)@)A9~^GBda_P<8)uE{A(gh~U|ZJfOf1V3g>KO;1*d(>j_Z?S-w@=JHA7)Th>WQ#N0lI5aZV`^V$fb)hg>ghzOibWLHmSOsj@|# zfKKE8$IorX?^r~K!fcnD1`!-@Xdtsz(Ai~(AGMl%wu<@sKNCA@>Gv@UarV#R$xuHE zm%sq&-dI7L{mM9Fv=Zp+@xEsaghyu#C~P5p@| zO#IWmGIp-Y@hsJ0706c@ z7%mv%40iq2*>u>j63NkS%`-P9sNJu+Col%u;2vK&J)J%+kzZrcZr)nQ_B3C;G&N)J zpko8fwkeSi^Th6m6whTMelK<2?9E@xOL^$`_Akph`xy z9eIN}wuj4PhsFMs#0f9=Y`Sq>(Q`jGaiNV0y4KCQyPZ5lL@y#SQ-aer%jjgH#i#<5 z5~VZSi!mxI1`%XPZmO}zUw)>|qjVDvsCX_&yM&#gAZ@9z;u!PK)Vr0xJQA(Z=zqNT zkbfseS_Oq0m(zalNe6lV0gZCK7dGx$eP^Sl^h~TT2#V_+9szTH?Jxwzyx*2Oy6i3T zsYDE?cN2JV-c+`A?O&Ufd3QyJ9?hBI+gtsZ%lnS^2SSH;&4)-I7J^BYn;^9EraiD;ZK{x|vEfgs#9mB#mR;h90V_N#AE3y&q^ zl{|+Yr*$X}cx!9rxfwq{unwfNSzGAc(LnQC($%Zn*2dIxctqtYAV8LUvNR~A-c?KQ)r`9{ojO;Klh5_g159X^xGX`T+G zMJdZsn={vLw$)jAbhW09yxA|MJB&7Bd{NoHO((uBdh@~WL|>}vxYhXccUJkfzPs3B zw2}Qw46D{EtpR@sYkghVsG{ug^s2dhh#^4pe&`bjm?P}bqLrs3N|IWMd(?}R&C@$( z!+d5al3qds;p_9GuJsFU0{i%Pi>YgRwH(7_N|x&MF#?|Ou8zmP<-JL8=2rH&>Rxli zg}%1iG-im)9)xe@7P*~HA^pW2p4+K&>ml;BZnCNZd8SWlS3^1okG9_TY$F0iL*JZk zOJof1va(En^XJ+5ad~knr@r5c!?ira1*^`EL-{iq9*?SZVx$-%j*}WXoy8`0lQ)b> ziJq|a9ffbl5TA+=s_uR~jxP{x*^H&mCAf19dmh_JE~#UMhm_&eY`6cEJ!CW#;;myu zY1j{o!+F6$~#%t*FBy{uOvy!Yx*yN9=Iqs9|_F_)7nn`*!@j+F`C0aVcs%E znW5=Pd$lIr1OLhT_C(bRb<#7d)Q>Mi%r>AzVsw#P_PD@ejZB0y2{VH+24N2p|5yV4 zmM)qCmo2t+&rc-sRhPnGug+NaG=G4X+)YzAe(wr$8>DB!F?BK)V7U5h#NB2*D@Crm z`RLUOz@RY!S{0*fFoz(6PKfXRs_FzH#nDLvQLyWYZU^rv8@`WDeU~5sDVNjv52q8u zLAH3=zYvJhM+XKd1N%PhF~xpv*e96&Eb$XS9cK!)G0_hx z$JWmJxwKO~(l0v)FI$naw3sXo1mf9F=joqLw%uGX99$mJ@|tknm(|V*y!#={Rv1uuV41R;SVZ~$pbT5_mKDTh zC@}4h&exEom$mudiwP`uH$HD2)HwGvFfFWteov|1n5C-wVVG@P{I@+RdFNL{Dtdl{tQFxmt=Z^*ve+` zdU1xvq0P|UKd1W>pyyZkKRIkP?Q>fNOlwy`q+%Eh?+sAO4GLcF17o+okwE{EmM0l! z0+h3SkG?zJWvf1++=C6+&}TEa|B+KoK*(iHeOyWMY zEz+jDg<*tiK?x!|jFoDeT=i0e;+0}_mBQ}X&zC_)Pm#Y|cW%3j)}IbQc7i|CaJx4% zb+*TjZeZdBNOy4QhdEmPA5WRCcf#XQ;E!jyj-1U07M9(`6;&Q}*Z;fdbveRc<|pCx zj*}1phdiK^fC)x-b_hG>Q%5ODB9o8HqufFIB+e`gZ_H6#CU~~H6g&?VpG*(YY%|K* zc7FG=DK`SP9;yL7&ehKi{8wku86|YyzeeWGSx+_7Y(NXeV21ET4``Ugmi(i_6#-_# z$I`D^vgdtxjLLKxjxnSB-Hgey(8jI0$Hduo*BmJ{(|#^C{$D!n|5jU!#1oJ;q>f;l z?aDZ2E(TFc7IA2`(Er<{6bpnDgbGuZ502CO>*;4fKCL#b6!PgpKk6_nc#1OHz6sl3 zCoD4L>S46Y3#afCJn95^_PzgrdIu3Qcr4#}QwJ}V@$7VRu`f68fiR1_>hmDJp?mRE z1$|SgFom)nMCbpYXlbkiz7bO2JMN`(3-*0J=o+%SAF}oZsIj`pWtsl|GWWj}n7VrZ zjhC6`cgG^Ow-CY6u9|u4-#G>}SO|Zzct+V^6b$RQyJ=0Op(=@H`No=3y6~MN6b@tW z!*Wp5INmI*HSjcrR$(rxkFO4duGqcowics1OQS0gM_GV0Xz^UZ>TMVNaod9zr!|WQ z8p*|8X(5&s1f+1RyIs1EsMSP-02m$o-bpq8>BY1v5&WBw>XF_J zUp39jsK_`e?n>ru8ENr`OLnSKb1u6wma$m1M_#Je=ES$zhgSZ0+Qrren;1b1bxpd= zkNQtEd20hxrJiVh9s#**w)5zFYQDh0_F`!%vdu4GP!0BQ@kTS^js4=iRX2&E8)5XfMf{y%2Imn#hZNX+gPus#f z=~Kp9uMIu3zdzATzt!{pErc!9{Ax*FBo;6KpB4iR^gIVqNk6YSg3y`)@afkwKiHEU zaat6x`3&&3nwEH7-`Y^7bDv%5cLv3|EURC{WYVeBIwPxVZji$2Yp&%6vWK>-Tj;+^ zzb8}2->%Fc#>nRO&U#3|@w;pRvtK}FdBX}(_$lr`Soln}O%8j&@mEpj#o9O=`90R%>qx$nG3Hc@L8VTg)|wE#;XEIA~$$M-IzM*>wjkg0@13xFeYD-wdHp`u zgbDi|W6%O%o09Z5zweAMMg9D-nwQ)hx~(&AP+gdb5kxfeVI8K1yI;vlK$FQFRxK$K z67}Sxd!jdQm+-~N?XT0IyWya5;ypagX}_~j8n_0ZgC;lphTMO<@$Xq}#NDYq4SbOa z#OJ{aj2uqd{F+v?chY10k#m-sX$os`sw*^rv2x7 z2(8QTiPL|E!-~ys{@mun#Q%nfkn^ICgj+37R7k@aG!~ihJ92Cma&PKyxd^_V%wM~S z4jxN-iQ%gw8AbPYbI$y{Uv7ULc$jrxXLDT&zQAbgkIWkI#2@+g^U^&q(g8nx62zd% zP_-S!ppVf;!qF7`o^WBY=(`K+azP1Vm{AX187wDF6U*-Rx$E~zvvJtkv{fy z5{5Y)m|)qpZ2XX0TY%=y#re9Ivl>tC^gQ?J%YS!SeeF>ftAIWNF5d~u*;RvQ-(~wm z*6_i3JWo7S*^C*A8(E(F%Pk49Q=c4i>07!M6s9d)Kj1huOSEm5&P(L(34L+Fx&Ry% z3yQ>q4(lk28JP*U)!K$I!)D|M-NNZ1V3i2-GJjj}gif1JpZ)&N#g2G&G4an^$CFZL zym{9v9>-GSH8Nt6*hHeGp$%f<+jWyiVjAIf&|61B{#mEWgMRB5F&qmAqODEnY=1+VmH71hk7VGNh-+1f9}zDB9LpFVDhb>BZN%D8^NDU=D| z6cB0AlKOe#PH59B_wc(H#&r}CmE_$mzo7`)Tq@th#gWz_v4;TD`#J@{4pwM*wvVXP z@-`)W%~namk0ofn!xsqH$>()^H_asRks5!<0m52RO?nO7H-G=-aPYzk(oGp%qHZZ& zu($rH8(jIyLiGX(f%%ws=F z4?J}1&fTo~MxNs%@S_EP&|(tS&BWGe|8C!F0lc49KfD`sv_4HGx`1*(^;!na~Gq2`5myYKCP+pjJ11Qq-M?fO`=(w|Kj85w+57K zF^$PvFSEYC)P8g@o${wuOXJ70`5!h=F2RT!0!LujYr6-brharD0NUhnCQY6LxLT(J zHrK9lB=O|CZoNE0sK3bt&4lR}&~ue=Bgt{T^R#uP3J}sQmF@azu|=v8q5%1v$X(cc^*K$Y8&*ggaf4a=~C%zs=ZIC=xdbqf;|5g0_{1hb~^0$gx_(q3KmzGnQ z=QTJ?YSI4*z$XQTcPYgPPPf9r5tZjeErYl;W7-A&E63?+_5X$p`jg)_C(>KcqpGYw zecLw-a0>r{^qoULWVFtO036C-Ri(TZB1Stkk3KUIaiFIwNwmaZd8MDH68Xa_tFfg# zS;!ex%TafLu1+obe4oSS9sRx>67;8qZg*Q_*{R$z|7cg%WM~e9*ZGWJcvo|1ernEJ zKe%j9NWJFl$umytq4#4$>?0G{%b2jvS>E0p$4dpp;B4@`%TD*=FVNZv&$KG#;n@#_ zdX3HsfD0oZy>dV71+^1^L^t)GeU%aaz@w02J6$rif-!7ZgXePx-D!LbG&t5fF&{bGIUKu~;7scD=a(Rv7b z4-__!vF)EUQ*yi=0 zt#ZUIGB#pfj8RYRbmvVo97fRuowci#gPxzwFpQsGXLMxaN%b2TiNQbHN85sN+KV+6 zEQ06xxY-1)UnwZD*$y!<8a#^Y%!0!0&9iF_&ucH>ZTQTz^{J0zP9Dch?Xjk<>sE1; z*@}ErMumU(B}Z_RL~8~TTg1A!h)J%G5$ipR5;~d0jW){3zf$-zLTy$pBSonT4beyj zO7d1Zc`dpwX7S<=HJ2-}s!gUf^CggEsl49Vv4A#Zi8E8R1@T~BmIrJ7k0+`q<;8p4 zW@W45ugkSm(?b>Lq}4*PBFs5e-MG4^#%&!p06p$9l+rOk{rjW}^e-RD zBH<>Xe`0RsOncc0s7KpUp?5b5pC@m~#N^0C7j3V3`G~6;-qjso4^U%cz4Gn!M$p94 z%~pkTL^Ei=*-6AAf9GHNlOqd0>g#681jWi>XfxGH9(T9%c5)^OaTou>1x)$8rSj z_cq(Mqvc|$0s*@iXQ!5`@4oWxwsYMtrGH}dCFql_P3^4lf;%#drJa{hh0!(oo&6TE z#(15xvt4@FuGh;Ki!pHa%m#9fjv=4UP|Bb)p*XotGS^f(XJW0^PTI-Nz+>q$gnF`(MKtK65 zsYQ1aRlDHlI?+EQW)p@7rN?wJ3A=rRI;ic5>ZW3>WhBHpK6heWGLT(;Wr#XXyO-Btf~hGP7=|o+HVc$?*%N) zVmT{a@k2^G-{xA)nYaoYR30 zkz9v;ekYTeSD?(Oy90eD{GoE~Nm)Z7PqttVHKUpF)SDM9UMc(D-^`V$^zD>XgntS< zvSJ3}4PYr>koA1$PaK65kFRal0Qpos#ts!;f7sDkTh_e@-7IpJY+gWR8fY3RP*{u? zXG_`|hg94n{=1^R_7m1wcc4OGvTlxE#{Kq>G#b9o=ceAebu1U^`&05m!>=!(mcz>r z{-b5JwDu$>cC@kPZFX>|hyD~$ZD^=KyzGuHFMbX4M8mQwi2Zxp$~G2{L!GjgCY=Jzcu+S;J-XvrTCCG**{9bAoLk)ycQL<1g&iAr-2UA`-evydfJ z?1A9#!!#?@rIbsPlfRAkk&LxuTggdFrvXDBN-B{ z1OM3E^U6cp!{v<2zal=>MN?1nLF8}8FvR1wf3qDY4j;!8Ct};sw{v~t<)R72f%>)gd zioDgi4_?;)>=v_nwa3D^^_Z<7H1Vfxjr2T?>%;+)TX}W>J~#A>mI$K>Itq%9;dO-` zBb|aiz}0>19=<>bM}Yw{N%V93Q0+4HT?SJD&Z$45n5Wpj&{3*H4cl?pHGUB>*OTQ~ z5-SE9g-7%_w_e_zIKukiE?73NJbE+ftq(j1o%DKz-|C+RE`Q5)#jQAERK8k-R+cV1 zcDJey!2FYw-1Qav&+l9~D@WIyFjWc&krG7LET~M!N1$NNUqYe%ED0 zIaFIal_pZZlz#_j^~NZjmd-n#>NfuWMmi*pQc1=kyF^B|V^s)I?mNj!2wBNIoMDAhMwIO!p_}YIj+xCd zJ4W_6_Tg}j;|%9_zQ6DHuk-)saX#1Qy5868^?aIH2B2WbE=zuh<`|_e*ToK1Sa}k3MU&yZRv8&i z98IVgOTdyue(4mOxv5^>yZJg`zbm^mE7w&=qZic2 zlqZ4l?rA?+?rTz*#l`}jmtjLvPoE96CF^4IlxNSfb~Q!Fuduyet;83=+T#ZRzgv#I z$ro7ieb#hhw+qzw|LKoxZuOC^0TwA3m6RK78`#r6a2Y;@^|f;6e|Go4zF_hhEcFg--Qg>jO}7s5KPK6*@GjnIZH6^ z(@|eHZluz$EIZ95@dNbk+(qd{un6+586*X{GoKDfsW0k?`6Z7^Yg9#hOmOdb@#|Cm z%5fGW-_`?7)%78p^Ax%D6L7reV&yKrJWPu;qa%W<=d6Onk47_iFZjl2zHt@(OQ#@; z3Buj(@9G5_3&9-jX1C=q>*6yw^9+fE;^>8op>($G(7&rf-8ANH^habX?bPRNf9~Yo z8KJ)IK=q8Yml=GQYJ0GSMA2pz>Du*lCwchd9sqp2S6}W$RRw`bn6ue=*2yro3-_&F zg}1+Y&@h@7$3@x=w2JHNZwo@-hWH&itE~^}1h}!fGHO=ujZ;r9A>?*DLvu}gQjnP0XG~kj34sf zhspyMk9ju-|7cx&`jE0NTJtL?(KXWy#6;L9$KnFcO*}i^b0I22I4FePWiisM>O%zc z!EWG%Q}!9pEI@ei;%YB|Fv!CUAzjd(_Z+wvBY93Bkd{pNo&(mcmT5QeQO&awc2vz0 z>s6k#{F}PwxsV2kJ<*^CI7#o-o63#EEL{DD=nsl?^Uiqbp!$z~goiL5vgsOGh4@c3 zkV&{<#%N{fJ#f);0Kbx>^Zh|4;r5h_Sf7v0d(t>FVAyJG`@|gDEEA4Y(_$u+ypr+J zWTk+_)f(dMffh%2M{kSlOkNWOIOU522Mk|~Mags&Y6H}Zcm5hN)wrnAzGMIH0)dKT z7NyBZ~dYko&XnflXTYaHK%Kd0~RKOG}$4gBt!Y?jYSR#DBIMfIl=hl?E_>pfGo z*Vi7cGJ2^4PA9zOt#!El87R_g)%#??ApSeYMh1;72&))dXVN!aDH~AWpgW1>{(?yz z5^b|!)mV5&d*C2qxDO52qVT3+NQV#n4IirBIXv0)I9gKa7c8w&tE1p`8G81ijonujW?Lb@hV5|B))Py0q8V&s~*{$Esp015qfnGAn=Lz zh{r3u4Zt~Bj(J=0+1@KFgO60oh^1+=q%y51ixo(Ps zCaR>VMB3#XY72?9mqd8)i>P~&(x()p(~y{ zLYqE|&jYK3o2wswY;7$XcYT%-_Bn8E!CPB;QQjl^mf}{mV7+8psl@ZPN!1Uli{#Tj zjYR4=3+%(|e1zZVrh-#+cuTwOgSiS~fthHW(ld3fZt@=jJ7F&aF;(l)sbKhWLa-~0 z_~1M|Gz(w;gE}+WSO^NplK*{|ezb9j)0iOQm5;#6fkM%KOPGqDd2fjG$UO&|sb*9u0FKvvPQReL zHUf$45(i4xlbCmiDAFfTxw`IM=7n%W1JIiPqT^+o4_{tx3 zWl=|0H85&dlc2YoZ%=z-f(OrGekW@0s1qW|bbi8UpwaV_{sd2UFuo9e+*o*i;97Sj zq)^ll5-O0F8W>&^ZlRwLBh0qdmkM$N)S=cFgzTV~zR7=Ae~eY)-eX4d@UbR@Rnlj{ zaDx=X5l<83?c2abE*+)RE}e-qpta@`?y-0--5LO&0r^cG;rK@ybbOAjDiBiBfA*L4 zK(lLeJ&G#&u-Onxv8@4DCSd!wTywk};LyKPQAZMR66tmxyB6+J<)W+YTx3|_5=uXW z9>R3WIUEPaKR~J*(3tH>huNq~)PUQ2> zh0w3OG^GYHhbeA%HULYkrh1+#WB3=(Gk3}s#Z{lTZB5XB9@ z8J~XgDhRO`NcmFw`#Gx;_JvfBhDQ!W7Z`fLyOt3|4!ftQa_Pc`Gq-UyH+jX%r~c#t z5`B(@=S?>h8DQQX)DBazeuE=jLX{cnZ+66NI1SotyIyP3)+;`Q=}qdtEv* zhOpD>IZ{OBUa;=TPn|SBCaq*XwL(`22}@Nu4UI)iS^9$%6gS{2azNTd*~2aa+Rpdi zz|Cj#ndU|gzpuny{^L5D^iQj{NCwNk-{Z`|>$9}+wv4aJz{D34fC06ou*`~6Fh}&_5x9zrL86FG_CQWTIw?0 zZ~w)5V<6)su}NUMS9KeH$m;b4bl0a+KFVh^bh_NGu|<1?zzanxw(zE}@|}=VUq+q$ z+i6F?x^$?0ooM5bcW1(?thbe~4kV|2a5@zKw+>=7itEdy0dtBD#h`1$Sci)>c5F?v z3Hk(A{kf}m0hM&6e_C8V?fVsVlbnZ@F!FwK*9ynZ%fGJ&SyW zlVI0__Xx@7hoQDBsXmAu${B6W@CH4*UWcJ9k@9J)Jw!7*koxD$vlUU>M8mdKkbaL3 z50r$3m-$D0fXcC?4;d$$W_Cw(HmplY%KQHY{&d2#AFX9t{Hb0uP7Yf6XcZA8S0l0_ z+vJ)-eEN{ayUjCuH^A&EkkgdF>4I}%sKs3uK6xNHeL*ekQr3^*<&~e+Tttt;wW507 zW2OP3hNfrQsK}}U56VbeYPx$}^5`|=wn5kV*EBW^`3-(oF!NZ-9nk_ zx6OIveWYh#77rxvLxeqNlwUcex+ZxaJhF`;;}zbkv!4gvC#W5-um*ts69d@g9v(QE ztJ+>BS_BmnH!_rW9i3|SDhN`}5+zAne&dOIoPQ;;I@>FeT3VjZ4T!9i$z4uY9itY& z$Kc$HNa7a#>G;f~BQNW=K{cLi*O@N5 zC=n;Z?ft~nKk$2}hcm#~yY1!CkU=OhLG%6#c7guha-sz7WeY|oa(?qy*wyD~?@v(` zQwBb#K`F+Y&^MYxb!?#mmx}g87_}U$>3<4wy4Rz*KdGzwk8w!65r#!^HIg+k7CWHR zNuSEEZ35UN6R@IWjo)mXiL}<7T{sqVPrngXOPlW7DfB9H{MrzGU0`F%@I!HedZ9xk zS0n#8SPsV?4*y0}4*RSA_!Do4{Mw5&DdxF{jo9m)5Omt<%bswI60C;}GoXU#spQBs z^Fe;#;&Io9yO{Cw7H+Yw^hF_F;q{0RqeS`V=(kIgkw2|eY)?bB4?^(Y&EaLkNT@#e zc;8DI()tIiyO_(}uB9>(Vib6IY4+^p><_PYp6Bugx@T+5ecz&>nz(L8{z?Q=%#!0C zQ!!+%qP6<~ugD#E-|5)C4V|$FXM_JcdR)nMGb!|4TXYCo;zEQ{P2k4z0p~ACa;a@F~uT&=~27P!Q z_nJEa1@ad43#_@_w^h~V)ZOc}344QVyT6GR-Pt?FdaaXIw8m>*nSBvWk0rb;yiC$h z6+fXi9PFguX;ZERj?|vt8>@A49tE2Lk*6sR9URcU7jbaU`7QKxDt(a^~#Rg zB$Z1+!#}`zS#@9EUH)iqDe1C@j?Ej+s2^2GtAO2AneE+mCDr>$u78m4Y!lSY_36a6 zY^xQ6Nez$F5WUZz!$tJkROQ{HsD|tgGF#JH9CXFrifj0=81fsZG7t04LTi=vv{aGf zEuZ5V@H8vg^fEYB#RXQnSE0M7B;zA6voU;b=SJuGf&3Uxf=u~(#6-`g*kjG3FW*w9 zT8vu0i|HrXXvTO^d(*ng0w9;Q7-p-Q%Dvx$uFb=~s=bR-X<3-|QLrk?5Xnh^U%Bj+U_tRe^Ro|v`wNdr=D~QTci3KokO3K(L|q23QD%?E zHdNUOln8B*aGAs9ZvKGxgW2tjZ?88w{EAGx)T2I9D_}QTs=0NL8`P=6v{Tw*%p$x# zLdHRsteXNMvnAz$nWCrc%{DTCo|Z?+la?x+h*OQpXfO&KeIVC)hBm) zq*g1@*b=e2i&{w?)Bilg?n*?ezwZj6E&HzUAO@NMB>mQesL`>*93vNTjed3~2$`!B zbS#h~vDaL^XtpU&69YO6pIZw*5Zw`X!?wp5i zpu$gSL>a#4f%Rw$iyY;>P=(c1y<$*OnCeGs) zku$5NF9w`_@{H~JLKQ*QX06&}kFdQQX`VKs7OVxd#JsGBZzRXU4ty)(8^4k}BB^Ax zNp;9+yNbFCH%pHgarEAC?T|F!-Ar8Jdc?b@m_9{tWF~8`f=m+r2iRY@V zNy;8@^r?9c!mW7(kymG#CKAjndX{$9B|QFPwN16dcB?va)p?T*c=IQ4_ zF@kVvg~!>NjeMW$H>&1OwA`T+wyaMU_(YV@%NogP=_;U?t5M z!nM2o3ejJs?g&F~uP-KjqmpG<+)d_+xBBZH?rvx#NGy<}%Hi*2qO{swFqtXVm@roZ zBXIaW|HqkUwenHmRg?~XhV>aeVrvT*H3`V)&c^-;jw&t>C__GtO(d%V`R^@>Q0@5} z#dkAACw;4&pZWk34K7WrY(qK!TVfX<(6xODAZ-5S>8t}Mi)+oj>;ILoWasii7%ZJy z(Qoivcs%u@J5e6FHMQ=?i!!$;B!9fy{rm);kISqN7Fg3qB;E&K8Vo$U|8VnyYv%h~ zB2ng97>kG6FMOnpFIfw08XTc)5aQbN8;>cIi!w$p2=CDve-_?f(w>#5^58sV|JqQ8 zOiyS%P(^&rtQso^WrCjSc+Qr{4*^n?78Q%y3^Ps~fGiUW!v1jD?Hbm!xu(qR+v41E5L(*0C+lpSpimewjt(5sjV8 zLK{6~*CrC&HHrT`m!C>4`8BpI^3-)AzF^{Pn*mD}Kac-FTz7GRT}73t@^-#>v1M`U z1UeC)bKEc)h?F&uPGKf0YSL3AnDIYu_m#9X+MnOdLcbr#w&+Q5g1lU?{cheY?r)CN z`__AY@U~Aq@AzXQy3qg6mwoiIM|8SaSCy#c?|IPgw*_6jBeAf$^R+U@Z z?qS{kTe=`i1~@xW$(q?`8Qqe+c5=hQXVyXI-5uQ8jAKsNB6+BVAA{Zw+YOD<7v9T< zcS{89sl4@Cus0u&v0_>4NBQGa(w&%FJk_-NwRsw-wD_Nu?$?G2uT0lx&)xo|##|49B5 zLtjfCt;s_U_X?9Axj94x*6iua1VXKENBt!$PV%W4uex|`P57K$5Y(wJ%e$JDX-G)n9+4U`qsRtanT_b5-wKLydGcQvsdk zj4J6*tLFRvgXs<-Dy}U)*(Wx)QvM_VCl2c{#FDX1`=c|HOP9V$roj>@M+dmaq z$grk~Mq}F-m>_BQ0#^N4<>I!><}nAUD?IPA_`Yf`+j6i=G2^(ZKd1Ojc^8`dd)s)e zP+e`(T~CDyJtkUm@;0*+w$I4|tP%c?1AaW254N{;S@#Ui!KISdT~w1&+S`XD(2l< zSBcIjhr5^vn8S8UDxqp4vQ_Zlg?9BP$d`~(KmmBrs4s7S(f|=tEB{;cqd(Zb8#=@+ z9t){E>A0XBYvy6Urf8VnleOQ{fw>QZu)Sj#s+Mt7bFJ<;vVUW~Ur49Kewtw-f6kW4 z-5tqg-fcsjLR5j-Vndy%H-DJz_y+U#H%?#nGQ?}uD4@u0N8rAfdD>jn2*Dhur+^rn?P5x{~so8wUL%fRSsK0RE<1z!a~!^ z9nC?h$Dx(i-rjkGtllTMvV+v0$(Y#cwK^10&pORs2_b|dxJ9cNE8>I8!5sT(m0U>% z>Khln{3qwaermqR8xpo&3JMuh=ra&X^bi?$Gd#wOT#1f7Id$3k{p?S(;7~O(GCrIK zTmn6gPQ)=weW=0Tse)N$@zXINQ$~@YkT~^4k-AgZsEw-;+L0Uh`wnQ`m`RF-R;6zz{L|lJbRtVc-6ei2Td0_ z8ZTokh3^83(N(WzIowxUq)d_*YwU#6ngSEOL4PUQv@&$RyI60t->LZQ@`&0aNY823 z;C>1R(hD+j#{}q*yKX|#HsX@-63&U1#%k$PH+iQ1<{s&?BgYnLJJ%U5G-PX9eCKSp zw&+PjTR!8C-k! zVaG48)OdhpKum}yqfnMYkyTwc4ucLFSC_6QB@A2EhcxE2yu^!vG84@NGpH$6*Ukk( zNS`2IRi5O$i*gY;nv~+0yON=?KEc=hRb2s+%l8V{?GRO+FwwTv{3(d6=?Cu(WGj$c z#xWzK%6{H4LGHCGuJ1Olb)f;An^>$a`=w)4lA^O-F4{?SG?ZIKoI5-|ow3jE)7^LI z4?dKh3&o4{NE}`8KqcRlvd%u5MnKgk+myQljtNWuI@=^rH3K#lpIA|O|D%a=`9E%D zb(XLYrf&A2M|OsFo_i?WET(7qm2jT+vu>}vn6b5b7QEcs%QW{MrRf#@%z6;#o)0TY zB8h(@1nK+FJ);_4S}E`w)3|pO=Ft6K+FSsVdwp>N%=18G2OwHR`@oR6qrB%uP8Rv) z57eHm1=%Nck7#mem0xrkZ0w@kfbExJ9`?TxuvR^LC3VCTn2aXLf~byibZLXRh9Hj4 zXEu*PYfYhNqR(#Gcaf;Po9ya(MA>bXmI%r);n=L6CoUnsLtoOrMIX z`w0_!6;B+f6%|QSNyfk{NRll*hGncfgSIEacXQ482di6it#cZvVbFoQMIY_Xw`^Yh zmJUhoJDNHFR0U$$RsawFU~r&O6pC0-2w)vB%arOjw57bq@c6#u_b)*zD;&Y18F%(A z+3k}G?ilTf&uIVhjud&*F7Pftv6T*_<^A?gxXpl>8%+AXq(TdX&c5n)rV}9pjNO|r zB8<$Kmr4E4uh$ZezDqwPuG0A{9hzXLs(_C!!D5;j{JfTO&a-8VyVPCgH^+JhuWoB}%SPIgi2i+56FPFXwHWrm=p4R^UMj%1)fKuj?iyolDiA*oDf(3K97o`nRpUfdDEI`y+F0gfr|74 z*4t1bHb54FcFjP}i?gX``paE`I}h#k2Pz=?60;+XkKefyHAj!Xn$LJ&6N#yL;LYUtUB!XGF;>Tk z_-`T=kf85W%!ZCx+u4) z4!3q5nBzy9uV>DWiw=br!lE&9(-nuK_hm5i$~ITVq@Ik_?QFaazBEODMizP`s#^EL z+_}B2y<@*flslwZ;e#O6R6z634&}kpf7uLP6<#SY8;4bsH4k>zDJD9JiJpq??wiyX z#1V~WVVmCS!OtMw9D5m84O_^uLA=t=HqV~vUYadOJZ+KzBw`>S)kEQOVjRw@H&)p7|+x2)$;sAuDFoL z$EXO3B7&UJx#_7Md<|q=81QNB^5S6?x4;0;_-l+D+t4V>dh@rqvL`i7*W;#59%91! z!)ATHlu8$GNU{v0SeLvU5-yUpF|h$$95!C!h)Zso9myU!m!ts~(PRl^B6!N+yYq%l zCt21RS~vsT>H2XZ7@AkCwQCBOX`yv`kOj|fb(RbDMH#-5J#IYr!o6spr=3XLlCB0$ z8L^jI2r11(688WF68AB-amfj$8w^vD#(ii+Rq>Q}E^0Q#;uL=w+j-9eIj29O^T|(v z66rLhK$4vXW__1T@2z&kQY?CT?$20E$J6zNVh(+1S>luW1Jy7YAA8(Yi6RKLxV7)-Ns|7uqtHZ!v&w`3+7fOZo1zsOLQH zO|T>#B6Y59s`YY5{KhbbI!blolMlD2ReAv_4(^k`cY3-Qz=aPpAW4ue-xlfRK*qb8 z7z?-o%;kpjOn-^zdaMevrP@SAcyB@u9 z4?VrP99O(~bAJ#{cs$#yNT1c$VBxh&-FltT1YFA}fo$U3i6ZIoqMR{CI*PJN@weT&|Wl21&u^ zZH3PA!o%nd=vzQ~zac*i692E?klD`<5yp;wP}ZMA%E8xF+T^aQGreCIJ#b zkk7+2QPHf5fvrriS@yN$8cY?Q=}3sz0wLHHfZs)oupa3?q1eTu23k=LA&=EDQ(r+~ za*Y#ntp$dC8Jno&hKRl#^PkfJtFtBd*vP$&Ap1D`9#d=%!)qWC0r0K;w-azFsO(;& z>W|>PrACH6i~h8)?K;L)+d1m`^K*UfFau+aU4BBC?sUBG;O`-JX{j{UEUE4C4nS7q z0s2^m{zPZ}hFvP)T}O-N#2m`F+SAM0H)2-nLupjpy4gybh&2JYH|kp2tts`sN#!Hc z919N;&?%C0C#P_j6wB7>XommEY28;^bw@ZT3k?_vm~S)5pak>maQ0rTGuP1!ln_O( z_Pv$&m^C(bwI#F~3tIN=paA1@IkWr^W-k^litqkHI*9{{=YaZCNlch;H6WvIpI&=Qp%;toY5Nd=Ly!S4ha?Lf0puCt+CWJ(&f1(T`ehvK!~tM zsO)wymwU--J>_~54(O8Zx@>TFl?9^3+WDQkSj2WxL9o+@;YRHhLc|R<=fg}{h?Qha zd7@Be$5u%}<^JZZ(^s()aiAKn7-IC1#+rqSKMMCk>yvrg8J9VQg?#PHB)vE_phlSI zlw>DNdl*9DNl|mJK1(+0P4)BEzow7J1^Xl#Q9NQ5aonvHoy6vYg5VEz-VWy3uNT>E zXk#EkOij_=x1o5xZGgo5X_a&O*YZ2xVoDQGk|3T5Td z_ym0575by*S>C?&j|2*JE$eT9adF>0_r!|w8P&iqh;Jr%Mh8c&{XDcE=1TtLy9?}@ zeI_F{tnTq_!3~2SG{kEVcl^2@e4@nDNlf-GT?zCCzN%-PL4?enxC{1>ZoL8X{Jra# zuSq-UTnA&vnMa5-p>-GArL!$RpY`|BIE|<35zslS#Av}kT?a3oJwHEG_nxWgRN=ckE|o^JXdf&7-OL_86T;Ru$TED?VD zweXIT&xZNSxKkcXp}8g>V=qSy7o!WOG8_sH!6SUd=0nG5bCeJ1V|jy)E-WFmB{CS} z8gaF)SLvWy=Rl{Cc0wmeeWus>(30yxr>hS0Cqq8Kwh&6612xrqkr-J*Z96FPq3_qz z;(HKEE0J_!8;qj6$^HDwF6Rx$lZ^SH-)w8IUVWS4yOo%Tkc+mpy-dEm^Jw8}nWrR5 zzK$s{5z8{HSWs=@-yzK~i?xp^U(1KqCh_c&CG>1{BIo5Nk$j<>m;4me2U(~cCc14R zGRSEHtBYxd)@*^@=8}Y1NHMl6#xPro7-HK2^@{MEm?^Vg^-=J;YPrBYE=m8&y;idN zsOQ`}=z$38C{tzP$$X53oLh3@X)SCS)=Txe4Vda7O9i~5yxno=wq&c*R1gl{*(t^D*ZWt4RDW4 z*Cs5%r(S-9f8{X54&CBsA}`zvGlhZu#Ne z^?Lsa{NG2TsfS#&8|nkIc{;9|#W5Cde?1PCQF6J39oTRmjF;)Eizx*AewCag$Yve_ zl?i+J*olOSuWBs}1^hjBsTALJR_xXBA-)Z>whHeh&dE}BVeJ8}gYkI{QP*ulz^P?V z&FOzW?V5vbZMjwduV>`37qoU8nqij+-x44hYhJ9@uL$v zU!y}dB>w1$@4+4Cw?!ZBsH|U-r#$k|d>HNVJK8{aJ&1p*ZF`@@bLHQ1-Hj3mH~dbw z6$ou`m(#938bFf$ff|V%tP{n1cBzCY9yG>U+s(xn%5HKBaV}^3h3&PVEemaWQ0JER zY*Q0a3kLs)ffTORjxz?3GEatKFHyk-zynhazAeF9%af8YC{ne;11V7Z)1$z)9>gu= zs+@lC^bff+X4l$WA9Jy6Y=b(4irxwB=n(3;UKrF0zttMI4KzckG zIj>^ZZsddNeeSI?;l=f6rJ9PBUY(R%kWkdSLM;lPA^~|?t3Hx`a>wNs!zb8%qU^OS z(h*t1>nGAti;#gv}7XIEj57PHOJi{FV;`5Kb_QrGyMYs((50 z>@7lASLN=~IF4n+m%@uzyLcS30=%6@Ld1wGCLh(TE}UKXGss^L{5i9XKLVJ7^xlfNz~k5IN>jmUJyE30x@>-hDNqTc~v|5VZKjFDc*+(YBPjh zKtC4RU+B{HT=T%3PqunZt7g;mq|aG0CPF+~4Bo_*cn3Fq&7xL)%B0_KrDfZl>#Ua1 zP9gf4L^6|eht7J3i4aF~8Rs4f^@T0R6@;&usIo$(yZ42^2J5_fd8dymN7|;|Zj^6L zNj!bI-!%qQFCAi3HFT7~*MU5+YYXZ=?Sn5EAXRlK<=!iYGp_uR`^yRsDUobsivG#z zO-|vCBS8lx$zM(q1s8^#QIUmq@p^y$osRe7-3Sdc4Hp+H3V80`3?ypb*@zlR>O!-> zIgAp*n1FH_B1T$=ru%W;73Yz>#}OJqI9YL|3$x1O%rNrY`RTt=tefAp@)D894Zn`6 zZ_3Qw)2WIUd_qZ7UwhIvdn41%b;mO7bktPynvD#vPMa_*S!|qbL!)OXYo{IDvyqlm zw$#u#;A7vZiFW@`r167;_Pn-QPS1;Dm>_3dnxGqv-us;bdc!U5_%R>4n4K#HPD9JY z?uYNL)q@h}B$W#L+qnd8qPHg0LF&lcyfBKN7A>3hZNDq3$@QJ{)p+aT2M;AN zDSLT~Vn5w7@AC&v$Zv{u2DVtDue_G~t#{@&j^YHNm$h}Xm3PxtV<4-)hmsioqK~+0 z=DoYe_YnPnV-7~gD?#VE4fL?2m#puzRJK*(vkAHpoK*Khv=oPvFLE z+1GEnAHOWX3%`hKoV@H!Z)+Ev3^MFO_l1)ZAly|(-GA*(S?;4y*Y1xg%w;H-SFUCO zugyN=Fq@+p_89#qYdlSCKXII813^e}sI3e;Q{+cWpFYa$q+x6C`f|*_s$^be90Uq2 z1WC91aKd-PPg$n=s_B zf+%x(zhgJ&?2`(U6ZIc9PlpYgqo~P4PV)Q_fZT-E3vL*$rLKkf*TEb=>GqEqj(g-u z$G*~m7W$z<1!}yRN@w{qD8K+hye) zW*|PC^Z8NI@J$}g*k(80Swrc>jzN`mQ@fYtVx#rjqryuBQ{KL($aj?|mPifuMyr_2 z4sQTvI&6M)Xpt71EtvcIFg_mgQm{{FP}E!>odQhcc+z8G5d<}zkE+{Vw99FV3Rwp< zbMsLrIc8YEpl7m?SpXn6{buH|3=m-y0nkT;4@U|Ep<1L5mOWn#UNY`;--DMV?o4F> ze|1yv@^DQMyi#d0X6nj8-m~KlRb|k(=8mvo4%jHTI<++YVV~oie*&0_+hZv`T6oxA z*Y;m|OZCvMY6SGZ-^@3-6MP#mZ=Tk}B}7q`Ig{aLnktJS!M7k(GX26amyrmjyk--z zM&SIm?4-0u%~)%<*O^U1brx<@>|5tiq}nbVQ)4ZpMgr_?z}~qvKYCqphCQM$$Y8v# z@D5EWy_;E}oGzk5W%Z2BI+39La8wd-_jInPl)N6V@N>_U-;K;4ngVUZ!3TU1N zVbF~ieIctL3T3USzU5MTek3F@xOuc^9lBy2;Rm_{;QWq|RyER{VZ|=OQy)I-o z_mh{pmlPLE=!`q^;D$Xeb;3|nr|PHbeGSG^^e=tlu|Ca>-`T`$)=#plY3tjYqlPNmkU17Zs-_qBUN@W&AaJ<-P8%RuKl>^pkq1gnn1BU z;o8P ziqdQc@{*kkHR-F@oL70h3eWsaG9C#H{X$JvWP9<2L3=ZoUR2rDc7XMb+H!wkpErDQ zwr={4^+SHMx#P%>qODdw zvz%{Cwm*W$Kn>mUQJs{?lR|YLW^6GB_UfSe>u~Ci17EkLYUZz20ZewBcj*#W_WaV% zjdp0iT>zBpF0R=>LU=94RsA%i4Xsj0OE(145W`{xBLzEc)+4oId7HNNV^QF})XThB zxQv^@Y}H8v+vKUIL0g^~`Yg#5Tb1Mw9g~IH%~=xrZk0|Pi@8^iRL3(;+Pn4KObhA2 zSoE3V`jcq^{`=TO;k@eO&lPH_f*xI4x5! zW*YL1ZzP-N&BpbA?Sc2xu1}tgmKnZwr<crZt=tM5qgAws4=even304|R{uv?WIk3iWItArgMe3im7%qy-DaMuFRqvT$9n!wY^n<86zQ;& zCgCrk_|ky3v{G5Mg4s~M(~tk%)H@5kp0Q&{e%AZUjcBr@ebr_; z(}Oci-4^}o^+~z=5B;c>H}KbTxX`i#l)MJnw1sg=WDJIF2(n%`-`zPHF&-t_;Z|b) zE$5&#ZPkoP*`wycX9J=;t?7miD|#V$J5Be#vB5hfsXb~^i@uL@Jk;>%z=OfQ)!*Ezb3 zGmO>W77@@c)!Qek~YeZVdiPe75{UVmJdo#;Jnm9I`3I(@^)Qt_NLjEb$VoyHvDCGkb>8FpHVDt40XMgMAQ+sD6;tW zOPcDFagJZ*kk0oRpYqjI@3x%Z{%7AbwW3Qc%=J%D@1!2TXO*0O zr7qQj3-@b%-b1CeG&58T z;$j+Jzw?r=%zJ)QI2*>peRwb%vot+1Y*Lo6GBpD@?MB?&I(xC;Fuma*lPVz`aXVp$ ziUG_-X;nOm83PdXRH(bHI7nh!cZ_GXG1`L1ipw%9{{iPe;K41jxQ(|P{G85E1573@(jiACvPNA*qX zqs0`<*0|V-^WHCv$xPo?K6Tqe&4mnVb*KX|w*n6=)rUl7#!!DhG@}*FGMvfX3sS#u zE$F{v^#?O}$wC@W8hEOF1yc3?bbz^Up&a?(lc9>M$hzr}a)6nl-H`vBsZGvJf@{an$W>^6!=$yktkD?XHNoh6R#%%ctHX(ObA>Am-G z4OpF)SYw^lI@%KOt{Kyj1Xk@<>uUcQZc&Ga<%KjvgTR<;1Bq5MmfV{fBI_X)foqk_ z7b`FA@qM44-Vh!e-WlDIL2JiRro?vu3cY@A%Ju0hS)m3)3RLzONJBu_%{B!7o^QJ? zFDBa$*!~zZ$3y&K(l{@~s1F#F^0 z0-c3T7g|#|^2*^HQ>gnp0@AvYFsyA{8s8hbDf6EnL@&(G%^inPqyIRQ6a{mQ#JF*?)-BqM9a{QX zHN@5~;yIAK{0%PTu{^;(mgAHJP~zXwm1w8L8bkJRWSvL2II<*oPDGXb(v;C6oi~I1 z^q`gc*WP9{LsQuuCB~R|em##7J^h>fSz1n`>Wor~^lu9A@shea62W z3uJ%PXPp%yaM0@B?9Arr!=bq?%=_-b-P}7Vlkz)pKI1q0wUpgh!f_v>#6ZBvcKlCm z+lS=WB)k65JV(v#%qScUx#WdMDA8wE1cKwJSQ(BY@~f^u@xy#n0pva{QD=x)QLVj} zJe_Uc)&XDa3uiYVRv7KNM7`y85L8waI4#w>P{inyAb{M75VvTO1k7MkDn@&-zs4GB z;*BA@aaID_GnuLAxrBK&Y8{~-X~G^=~oQ*tJm>?nMxJH!BP4oCnig;+k3m3vOzDbRfA9O*M8m4Q!1)y&BNIQDD5a5(4#JXT)^vfppriQ*EcaL~JU})y1Gt@**t+`?*ZDk@H&etFJ$pZDJ7F}PRX`_unE89dsDD$v z)$VAqg0MhgAhivVMs>QDs=|9Q=efDvU-`OUTtWug%{0D`UYBGWEw50j_+18HzB#qs z91d~Jt7^kK9G~Pr|6*{3R~yL?7E~EpmtLQD!3v6LDdfA!`uF`w$f&>HH4Q_HowS%- zX2Siw$2j5gY>%6r%t^iTor#LNrhHKBmW~q}bT#^x7w^f=Ru30m!dkJ^Mocx2S#txR z)NTZgIrRrUjH=elNn8EOsfN}B*|#*P&qNVw@fxu-%%>+G5oG1!gbc$#E&u0%=PA}N z1Cr1N{x~M4{3z|&3e${$Olb|Cj)~@q+ReIU0kavxHrwql!Cm$3Zdrsi4^1X=M$Nb8 zF)e5nxE4Tp+w>;%3)wHkxt{fY+#u$N?E;%rae=CEE4?zOwL8sw{>QP| zP43(vH4F`%sOsOM9w?e1lHI$@4?mU{jC|)7_d>1R7xslr2v=l_;p;x0bC~&N$)EW4 zsi3K9hH4kQ=Xk1l%g5>4E-jE7NuG)N!DqJZZ;40XU{@=qS#5UNyO8!XGTLs_2Q}UOoH7LrD9TSmqqa2%K4Z#aYeHXyihYeVM|k zFmh1PZGh(Jg~x#TRI3%knB%KV3Q5`_L&yr$7b!&$vRkOsNIJ-@jsmzu#TI5~HY%W= zh|*~fW?s(oXQ{>IWnlKe=3=-yT(UJy}ho(6__F znl*y04t-oL0@T|%4seg}TPxy5<@od#w1argjbFQfR!#qeFl>t@T+!fNUSS)}>=A?A zem!x$v5+bcoioNFYW;^|<`duz@h6$)*K^pPhGR_rLBXGXeVAC>Wk6yDINhICdah)G zuDlzK|CLVv#%p|jM^<7uOj15Bh z=cVqOTT>&<*x%opPqv1#LYKn>3q*Xg;^!Mw5LeHHr6+11C%rf7O)GpWH6$%-KUCd; zb==`!kY``lk5E2|3nA^zQ}&J4tAV>M5%yFvrN-f&TK>ncgO%Y+4@*=tFm%`zn;H}AtfI4TqM*AZU-l$cSTju0O8xxG_ z_zCYlM?z_7(^4B>(3Y7=iQ4M#v7+q_z72inEs3-5vL`?7JteHNE8afzx%}@682SzCqc6(;kk8sn@cN03|IH~aBmUGtdgoX5YF?aXYYie&8sc}fK{i1!w4b*|LkBXm^|#e7Bbs@ zi=EFZOoIILcCvm@axbcj4$rT5@9r(3D8g1P*z+Qg&6d(kQ%k&0AGLUK8v7gYHkr;B zan3$17`1g<;<(egr|d#K%mn-U1*3&EYg?LJZ%h{6A(-;OL&Lp!K4nuoz5AdK_ey>(PSLB_No|#OeQYzy zmTkExA=}4rJ9BP~`ml_nJ0~FM+DUz+TW>2s$-S?LWm7G>AdsUFP>X(_QsdfJ@jA~f zPxXuS%?uHPhhPTU7(5;-lrVImcXX5;`cbc^$)o_G>>Xc=!8aAkzt=?s^t8o%HtK8R zt9Bg(>?==F2+UkW6+Ox-FJ{JDO;ef}G8gbS`E7%0So(QZ0Ik_-zB3}rxyJe|jPThb z|GY(N?o~+;Je}cNDEywC(b6G_a(g`+8jjgP1j7kmKUv^KZviRC>tIgI7dYND^wv$W zQR!_n<0hcYBAMe~0p{$BPi!VOOkMzl;kwpDGxG`mop$CTNz)1F6$bl9`_z7RK0q|! zh{<)E#J!v7r)ha}#RZn_9FCUO%Y}Pwhau_Gd`afUMtcF>uD zB(dLKi2;^uW+LO#!S_bVK;#hd!+h+_C|hlOjH5P6ZZd};oG7o{YZ6;W_v{O=;=O2B z4bBwMTj-umdenqMM~LDUi@&np`*KonI&NlJ`EgUytJY>DyIC+tUj&`}sG1%Ik^>(I zJA7kcD+`a2)3clrgBAwV3{5hmF@c)#N;DnJXhI#}vq{A}hior*KPozsNhJ6|4diX& zn`~6oPA3?p7gZ*)>?qZ3tkjvQwzD-P0*1!BA_Ec1wA$Vzo=gUZyt717GVbL*S=$a- zKYOD!OBs5#3aqzOG5qtI4C!%H;4NlV*&}<{{wCEKDaCcGgzdC2WF&{Gig?SewPwV2 zUpA!W`EC1KsRzzXn~zb3+v-#1!#=+fsq2^Z^KD$r#y7JLa@o%(dDZ&V$TyLEeP29Y zhz9(!jl~tSFh$E7awjqU%^+P7>pRb(%K7%?4fDG2tMVq&&j1N+r}p-zVMUVf8R#q^n$L#%H_h^Dvxu= zE>s5I0_=jv9aVIs*t70egv0Cc4eMXemD&kiKS4F^($9N*`=L}Q>%r0MAa|#UaJHp8-!+U zBZFh@Fo)PYHbeQrSEIR+3ZAF^SZ1e7J>8a<&y)NJp@mTQxYvK!aYMT%8F;06u{I#m z9tn^(ySnMHX^%Zydw{>fNHMNfm3&p*Xhe)vR9XxLrFGn$8 zIYxhBSh1yrM<7RDsLQLnZ*h6^=tUD|=_M3STNth^P@@F?2*N!PUX93IGa`XW(eXqe zE~YB8t!rlsK9uXm_4(q)I-YWlx~KSD^pwX(_+z3tO1e5&Sa%M!UCXrx&|F_QHeWrk z{SmVjcF+>FT|~C|on$7I!82r8lHUPB*X-Sf^kOHl!@U82Y%XET##0}oUK_VGiy4eP zbX%I_usqJGudtQaLta6%Y%?6J*wD%m-%~U^uW%gxr7Y{6DyKAO>kEShZE17F)eu9= z<`jk@V9h6t!-X|pk|fd8`qFkTsQ#kWVNssph&?8 z2TlD`qjXh6CmSy*3a;NR?O0rV`&PqJJ_$QYiPAhhVkt9P&k}jm19T3!96^^F6Q=C%x~Ws4hHhy z)eSpOO;n7N27V=WF;J{FqZIwAJEQf$Q^>Id_*`}Q>BI4rHo-w-ZbJmh#Ga*3hp|?l zZ3&pS+YKJbTb>Qp4B#Jp!%a7-k#8SH7)B*H_GkKJ$~PT_Hn z{@!it3-3UHN$>j*30db~pikVo2=Xd+XQ0TyP5tdyFh&<}-MDIYlb*u|z1cZXfj%mT zdzYgc#^*y#Gxtj5+4FW+J%qtLvOv{tI7WtP%%K(QAI>~05S3TQ#Ww(c;L2b1fff*> zc_vHlDdgPQq=nO%(9KwCAyq3Yk`fV4-16U)ZpjfjxS8Co)A5|mT_)4JEx4lehatqt z)t~PBM&uc%2?`VC`a8bzaD>bL80AZ=rS>iH>d)=)Eri_M6o%v1-WT)d%Vr8$ zI6}mCThtZw^vN7dzWI@kynp^LQ&Ca|wRNholAVEgg$z^Tdc?a{xmQJ5?{N3+3r30g znB#0b49#Ae-_e>^f_{}jePoaYJzvjuevKDcDNFT@?e|zz?dZ54!ggvN+!g0X))b$4 zXxdw=gBZE+_;zhzM8?wiHQ(#IIVW7F+iIdcnJ^_b?)r#;_nCSXTlt+@EPG)A(A~Dz z$#|**{l4W55%7Y)`Eqa~%O@A>*Hw>~DkOio46N?PLajNa9v$1F*`LG^`W)lCm?iE; zUv&I|^Odpm3*~NqmUJEos1uq5PP?$k5S6NX?&_0kSpwL{67pObUd>4Kp50}B>y|%V zF-qi0YlL{a(dw@8?R;jyhTM9q`x!Xf4}mY-SO2zGDwhPVAj(Le|1#5bb0Dcewp%WK z^RGtn&Hwc!JVK@G8^I3VQ}*Xn#{PvRHZ1|?fR;X&uieb8nd6{eE!w>2k1gAs_^fq# z;aoR&ZG*?fmxxCZD6Eh5z}TTnXw*7}?tK1da=g?ZW&twyqpF7aJ&FW1D!#0Rs2ODC z=i1pD%3VHAg58b@P*UtOh{;uf0$F(=_!d(npy0#g15~KBoB({e%jlG@p<3 zo~qNlH_GGt=&I~+0Ag?VMM#7L(q8Y7(uR^TlColZ6WGr6lwV<8)^|=`&n|PMPh$#Q zEbNE;bZvE_KErB^m8pg4SE6c%>akQd2WIqL&nfS5_Li!b$f(TO;=2ryJdg&Sd9vie z!%#XNL1edQ#HGJORMz0F(&CeSraPYLorRrzvHal}y14L25b4|liQCn<7P3AEy!?Pz zlW|sa;#J;abl!O=Do~FN5xmosJ_s2UtG}s&bupw;y#()r{A980{^;}H!F!!f`_jG=$&Swe& z0d;i#$us@%w1+|siSj|444CG}-gt?n?<2np=2lvEjQ!9gqgr6L5E2 zLFK^IxK}O6oJaY+2X2C-LO5AH3})NyTwx37E51gaxKGz(>BH`;eRJ}-&UJ)vOmkLG^8-(yc-qoIR$ppj|0*~O~w(LLs7>d%sq4b((`iyf3&D` zO`ywMiecUxQ%wp6xSM9bmk+HBzh=v|nAX1cBi3?jDF9qh=8|y%{Bo_?S(0kBjCOOk zTNdnGns0)d($l}sdbN8!#)*48%Vo8zd~H7z*YpI#_;(OqalCcCSqtK&@u^ouMu8^E z%he`k?FvmguQ}$$&5pqfT|n+bJn(@BSx>ZeGR}X@m+A4LoPBo2M~;oJL;s=nNM?Ca zbX8jBY@RZ%l6JSDb7Gzv$UTXH-`G^$A1N$w`uM+flAOsgcLY`mOSpT+mxQ60k6s37 z-TPN>vhlt$Qo2joxaC#d<@!<{fio>8VO!!SSq)W|H>5~1jN{GK2*u!k(yO!XO1;88w;b;ZSh_m$!DcS)2q+gOX~2*De+ubh z0-0>)R+rI$c0qix$eb@)P3W{3xzBPWWIR`!M#6jvDugSy_A9R_I=5Rj%9f-g48u#f zT20QoCbr7$X>EG_#3-0fEEll=t&6NLGTSh>Qc)hH{5|%|Uy3`M)ZWQZ z_*x^CG!gf(9LlS#^4y;~9Q{y5x=jP?vwlNQ>=At1mhhL@fM;0cu(r)1GZ*~C$7~!kq-XkTw5iT< z5%~V`5tR5j3Hg=B?cQOwh+pT7$s335qYhr#+?#qw7dtT%6lk;o4BfP8<%hB$~3Z(&&j9$c8^a9yvzLglU*?e@zLHj|$n zv31uqm#F%T$`z;$+?a)sWhN>!fAbfr*9(8^erDB?P61jLkE({bdUtIV#`Mbq6T5=l zvPnk=r&O574zuL|0r^#blfu>xQC8Ti%q&9rK1WHX2aHCgq8@&?@_;J9w4~XAXyMUE z$k&~jvZ-LeXf5b6u{7`k=9#ui!ng&}1e6~f9}Va?x(`HWvV2jIzeitC7!_$298QQn zYCaYa+`Pj-2R?twdW!heqt(i`{xj29>l`mCUs%svoLD+63+waiO=LbR3oTcZQ;^)& zf?r3-U2x80sR=xw1n1M^u%(Lol#;`ynjT3{bNy%xg4UXXK=y$=szIF8;IF}7u5dF| zXipCYe|JdN$jo6O?GmJ61)IZLnhF&uG4NLGX&ft6llyZgS$KD*X0TEi29^}u$&9#Y z4vGhjW$G2W+euy2gcj((BP)i{~_P0jzB{YczD>*q)=ldDjeLr?0ta{>NY8XV;vsS_*6EBfa&& zSe4#62!zM4c+M!S_Ge&+ti54C-|LWk`#uzSWU0n|cPzpDxkU+#D%kW-R7mR&2H$;r zr=-$Ez9e_-8g1ns{-UaMqSqYn4}Q+qtjMU;{^X;p0#I-6Fks@6qfh&=cFB9dAkU-9 zW5aL=lVWmdkQ(!Dy8+UnN~XtNDfDGtJ-Z_49-whsZjZ8XP00$Nu~00yn%po)%D41_ ztz8KC&)~DCiC1>C0Hzl<yp+C-*x@aHnSK8_8~*00nk9;`)iay8!)Q1G|b&s zD0GMaXWAa$#Rs>jh)i{!aS_$z;JevH=G<>Vr_0%=?_Ciy4@XZ#UP)=xvj~u1mr#$V zsSMJ36JNhHd9H~cqMiL-CgIUeKWm@e25DifJ{8%Miv_Yp5Zz$9qU#|n0Tsn$e+j0U z3_7wjyU)j?#l+`)%WOc{uaUQkosG-PcPK5h&G`>)WlOxEFvtq^HG>U4!<;xi#U2kP z__5ThxHIG}rMOoVe9ViyIqjjAw(I%MHOMGbx2UQ09LpNJ#}gT%NK2ZH)R`@fMlE<< z3*OnZ`kTFMi5h}Nz0+)4(r2Fh3Sne&&G40D3R(dL65lCNrY5Zq0*OkLAdq=wj2Gt7 zJhGhvd$xOZe-FFF8@pkm1o7h2tap=$CCDT|<}#g`uAE~*82l;6_HDO5T79c6si`h6 z-Z&ESj8{49avPy)n&&c<9z)%8xEV^%MI0e4oVo42Y8&aaE_xw1kzS zA#Vw1ON2}WZi5Z7K}F@}HaDG6b0;qSko51OC0uYTd`0N{_0=q?z&uRZ={5zPebX&7 z%Foqk)*NlM{Q4ksC@u2Sk@~dq)|*$n=) z1Cfcx(TANp`W;*TpNVwQrX~21b(T||=a|LnO=D0HFm=q%dV23#Gb~+#D2hc zD}_kZ4CXqgrG^F6TnjKgIB=}+3#EvMlbRuYjUud+s7)ju%YhZ4PUB%#(cZh+kkX)0uFy&q}4gP(Ib7F1UuRxZy2rQdhN9M*R zjJ!PP3rh-=F+=D=@YDxdJO^P*(mxXY5NJ2)LWE;8>5~;woyD2PHsk!0@^z0Y5$?A~ z(Z0+26JHo)G|#x&SKD#uFpOo!x6F{aL__F5;??O0()jp;mnWJ-C^j#@twSM- zq8^lSFkicUH32|=OoAcohIInv8NA$G?+J8G@&hNfL2#n?AGsfhm`&ZDpqC5#_KQdaN7r zPhYqozE3VOWFJ-XArAL73fHZ2Vltotxy3vWzQJhG+glC12f3wFrYGK8gW>^^9%!&~ zVo+;zqk-Vw5(X@nt2vHp^^>D+%VZqiAAwT=&SJmOyO?Hoz_7I8yh|7>;=cGqg;GV3 z)V7+eYElPi5)i89waCbHmL)ljR|V9G8S+~!WwIk6PKz0T2>C>9x5cR_x5(lp=bRf) zGP%wwnqKZSB{lT>vd51Y(5ZSZY~2n^is^3rT+rjc38ht;CW`W#ihitNIBDAD zIv#$6^p>U9@5A|?#_Z7J>n4c_@gjgoX$DzC8M+PQO;Us#ks0|r!BVS+ZMVd{M^dcN zO-1 z=Pq6FJXz$my{Y@9UDH07L=8tk6nn2-I}2Aj>Hled*y>w*XIbYDETf)WT`L{RzslU* zoyFEe9I+`zE0<=UC2!r`%vKt?v#gkn#(f8RG=#EkAnb%5*|k>R3<1J#bB{sS>)JGj z07E)aG`zKQa!q@`rYBD|)-&F-ESPaCzj3tVkMTJTLFC)&44s}SEk3uV_K4kjJ;W;t zgH-sI$6njr;qfhP7qfnelHT&Kw??8aIgX6qwr{tIMOSquPEvW}A-7s)8FYDMR8S8W zEjAs!qxMV9d<~#2_1cS&lIz2{(v<&YrkwInnR6ns-9&z1T;6I8fbC_7TB&K5OvE`t zV<40(g}=r@9rY>3+C4Vip5`6BJ_w-+27wH7LqP<4-T%ULE$Z+7Ara%U)I;S%!IAJ<~InY-isTd8_`r=S8E67 z&dpKPP`ArURTD^a&(IQe#i8#cE#O3nlrZ$?ikNLHCW70<4mur{7IKSAfS{_A@ znRltrmDe&;NBHVZ&?8P9ZhQ7A8*y`7PPdN=mAR!!g{!>Ho@1y7vJX^esxbQH&ONm>pJcr#=KX8ek+EgA1cX6d@BAlj+bQ_N5(r~K$XCV>6dp`CEpWJu4-R%TU z_F5(ec(4SMgT+C0n6uP-bIws{rk0ZRzSmp*7mFrA)Xv+;UGAU-I_DIWio@)r2k0{- z%&%VU9?Nq3j31*jUWE*Hg*PygZ-d&brEY4A+Ny!+)>V;)a+KBS*%gs+flvMT2I4s1Sh-h`h#Pf|*G1KLCj!&W`>0`)y(ee zwH{@f{Z%&RLh+>^Vn~TgHYezH%X(0e(8lf3p$PD6rcUb+h94spVa0b_5XlQ*^J~1W zyT`V)$fPjR_wK!HqxMzf1U06=C|Jlw7Bo7hWc!ffR%k3`@=?4QWlO`3+d>L6Lb=O7 z7CDg-bTIdc_cSeyq4(UcU@AYbk-hS$lzmf5hbNxz?x7xE!3zYh!Pz?!Ej8D5HF;B` z2n9?P75h1b_m<>8%cBr+bfnqpIpz|G(}+@tl|7&JoQ#}j_x0E|s0 z{zc))ca&ULTSf9i?rEf%PXt4A*9~y9D~fjju|c83?-fX)X^$Qswt<;8j04U_>*XNQ z0n~x!K#ik`&{71^(I9n1;7unpbklA#;LoeW1eiE%&%$I%Y0#^dqt^tN0R?*kO9qmI@j zXKPNRYCA}S)O3CISmb_N>n(f5BW5TX%)>E_oQrpBfsNVBbrUjztTAWId_1?-qDd`9 zhf>^Mpm-f6eSk*$J;T7&PcfV-8VZ||b44l*3;KFGq8KJ>9KxlyPb%dDUvQSh5$+z4 zOH+&>B!i9F)pFXXv-EyE6~t=;V~w^<%wibHtm3qGQUSNL-JBg-F((^Cyhtr(7SavI z0iT#QISHyRQ4HAtho#TWRJ;5sNs%rksJ$+$<7@4%#L$Zzw6x|ax03WfnkcOwZjHLNA3qY?bF&Vyn6B8^SMkPbRLHGUC-vp z4^ew}*qUUGGs-c}(QHjxZ_-bWQmbVTsAoa$CWZ+`9#`S-hZ&xZUiBMlev*L%d{FUu zWXXQ0N79IOJf78P`4zl=TwO;5>e))Q{eEBA-siKIbTnDFZX-#2aw4LPSxjZxjy8bw z{Q8-7$^kFA_SbtuO0qW-lNZPR99=<$4i=7DNs^nc@A4Tu5%&1Ke@Z&$i=AoozCjq${xd9CZ| zz>r<7pz518OB!!Oi>;WVy~MnlgE&VTCZILs=$)w5AHUYuvayf7PpM8W-HMoAp zTXqy?mvi)zhWxj0-(`K%7|hDJsm9K>6}GG21h~#%sfaQ@+Huvd)+TzgOx;q08$V9K(+o1_UR~69XId&2}o+#yHBTT zrlxDb{*kX>B-m)W$?IsMB9QIQX>H=nWkLBlMF^6p6z%_mc_Py&3A+P>lt_|493eBo zK)A9nJ?!=V?V=#((OXaIG=Y!FDqf!0^bt#?4{S~ut_ktglo z@wi!^FS(3w{of{=OUQw7_Z2G(**jH>l}pnV9kDm3=NZD{t2vIQl2$1wudjDBYLzE*rKn=q5qb}$ zPDz9AZQWVs;iyUQOHWps`xV0M3><}Gf8ooj4BPns(oXhw18CAqLGMmo@ z%OYFo(7Zmb?)1i_J@=z7U)r)9e1JW|bh*yMpe5?uEWk|u6?F95zWoc}0AYWm`E#)5 zN^=Ao-xP3H>+U<>_HU!X71X$c)2Gn0$x_yL?Gad=MAnqvKuKDP4>izDR?}wDJyQnT zTTAJ(;zJJqr(jT1=WXtF{2%X^awC`b8+eiZ>i6Ms(=s#i2se;Ps-HuVP#67k3S_^K zOEiMq9T#$`FpoU;yBwJdJsn*13u-(VY)?LpgW?t%JjjdfevZTVx=U;WhjpY(#0t=i<(;}yY?=-lAZh`3LAmPrPJyy6_Mk43T@24@D*%p#RrRm)AQr6DG6 zPcD*udPgKbo7@ewwA8cy2{PfEndluz_h8?VdTAy1UyiCKygoCvqo%x6YNFFxr@}9! z0NoMnCRh#@qqZBWCX8ND;)W08gn_s+?%xGR@nAW=I8J$tM`5B+vhgx~4Q1!5mftOA zKJeB0Z`%&+o?@b%)tlhdA6ujBv>JaMlNh+GjmzE@AHRw-QrO}jU;C)wcJ;JV?K)*g z{wUwKppyHyz}XgUndjL*PgOO9#^uRM*03BBH=p@2E}SrNSqiG%_$X?S16i;i<3q{^ zp}Rf!EA|Rz)|h{s)~>m~KE>$iLyW+AZM3R<0RMQM$F7KaX7yh~UT0>Gz0&J3y)I53 znX>SsVoUkOK4dOGdwT5@QbwI_f=He5J_hbYb- z&7SapVru()3~!+Vx$g?TyNuFeg5|fFZ33hz$IfL8V>b_DCd#=&Z`S%`C~f>|Aj|dw;J>g> zBVMGE#*@IqCB}!FG>`om4Sf^tTngH$4+V|vjI1qAH1`{+t``c;QL!Bvb%9^#XW8*8 zw{I$IY_U%bC6TGyt?MEoSIp24FH5flY;re%{M{h;+!0juho62@XqK}Fv%mN@@&YTN zlaFGKR3?KA238ZRMWP5@LQG}U1pAS`C`5g@@htV|Ma1hEA2tG>PF{GM@Omc)O>dca zERzi6Vk&$2L6yi}c4{j=Y#IS-V%zMA_k%?U7rC*Dup-o-!mOhSzo&%xX(1@*Pe20C zhSb~z_YcFB8kWZQ{ijdi0BDHi4avbFSfH~X@sG3Y5chhr$9Q(Aw(&G7d8bow!-7>D zNG+VB!+jtA$|Gn{6K71K=6<6+Yu$jS%BqyZrQ}A`tEt~IT{7&`0gZ=tdJ9^Yv%kq4Ul30M zKFa6>RgA@iwI<};G5U+MciPYxqK-=7_y)y1kUk$W9gJc7R*?$b^Ct*+!s)-22BhiS zB>xFw&BDcz12x-@ZXqrv#WtzYWBAQZ1=ROx)-2!)mA4C3`N3ocX82c^um?})rDMgj z#3#<*V+2?ouu2N}ZsVBNQO_5V=Y^wx`v5k^Zdbo&-q)Atmp)|er}fFTHC?TDo{Ljd z?T-uud{NUx=C((W{)oBG?aXD#}Ac<#4_{Ap$#w4AIcJT-AHAYg_400Y&B^ zfHS~O>_*K!xDzxBU1Bg%>RP#FNHYN3eP^I)nrH$MQ2pdMID+v)-a~%`IHBA*?u`oG283bZi(zR^C&n+HKlyWaF!)a)ZgD0?^Z9tQ@qM-s-jJktj|Uv)hh@K)1mdQgB# zG0sltjylqqpx;067YqH@FCFmk1cp*zQo8kR`L)RKf# zKlUH8k5?CC#`MoqpWK9{FKh4#Y*5)33I*2NpppbO3cIBV_tVGRn0d_Jr~6mVB?DH``gCa8Pu^L?f+% ze@rVPvQB}!1(0#{XhJk@g)K;vM3u5c9b)rmI!NY^b@1DkHqtm6cl3ClwIWa%|F7}Y z&J(`1oTm6XPOs;eG60uI6GTmymifVom24T7Z_<$Ob)JiQ@8j4G3h+1Z5NUSn!jHbB|8ne7W#o0#!d*UwA%7Xj=N=!$ID^ z7mmEm&#y{DV`oCkw6?CF9qFx3fGuUkWED(}&b$1HOM)rOb^F*l@_Ya54rd|-WOkE- zJP2n}Xg!M?+iTa(xVr@E^CM@(=`)9oO{l-PDI%WqdZXwzaa65`?Kl9OzN_!luTs&C z2u*js-K_hQ#V&otFv)keJ+^kOeH!6Hyg}|%;@LfYh<21rGx}zG{3`XL`?*FvzG7fm zS(V+VWQpwwNS7!PGo)0?Pj%U)lXDsZ#kfphijorR9Y>a9o(wV9fQNz~a%_W35u(*<{E$`=C|a-My-T)SG8(m0+x3+7cY_ z-uS*mYg8QG@4@BW?-6hwEuYPRaQ+j1DxBLKo9Zx+m3&gd5MK*xi`wNNc<L7B@dc#mp?E^s;uIWVd-H`7!_G!*-#pK^eLQ>2%7^67A znh1!x!yfQDlL6N}A}@AO=NpmDAdtXtFV??8V4$;$8L=9g22ATF>Zm4lBPM| zX&9g(-U@7PN@*lar`exGXG+O}LNMs=It7&3Q}eh>xu@Kk>hQ4o{DOAH_im0_yI;U` z_4^?|dF@2$umT%~xypi+hCS?M=+P#*lQaX+L*UHwO*#g%>=Hx{&Z^wa_?uvg2|Tm$ z7s70YP|>``Q{lzEV!se!1U9L;cF?<`Do%QnGH`CrafD#)i58R@A<0-J%;S(>hRf@ zR}N;&NVRN@DnBaLBi@(Wf|ZxfOMp5~q>>c&=pW}3R>V&J+Cat}BU}deeZ{pt``iO% zu}UM3e&4>h*6Ibs|3Yfv>@;|f)BVHSlCoj;to(O})s{FsiL4lOR3A@q)U%goeUK2Bg-D7l~bC8TMbI$Ot+ zq%3{zYuQg%cR%DUIToz!A)E;5$uw2b>*R6W}PU1ft8@oKW9jU+O%OhH}jBn?kH zHHp20+)oV2RoHWCLjy*F*nGEPlttC0?rRJ!CP}AR=1KqoK9JpN(rY!IEveMgseHv$ zd2?9=>|L+#KkHbWwr9y+Qf_n8XGGWHSak?>EhtR+g3_`!gz1kTmtwVgeu6LWUwg2T z2ii4&hPSyV!MmGx{hSg2CRTClOE!FgC1rZE(r!C|NOq4l!6Ky5XS_`Lci{)0fnAxn zpuDgp|IPPDbC2#m&S*l%Yd3;Q1KZjk^DP|tgPt>-2nVs0D~YJ<21$o@NB$bB=-Z-t zF-N_)Zk`aNPka;`sHGmdjoA6=*oisQ4{_RLn)`SZAm_vUM}!=9K7Y&#B5m_S(8W@4 z2BcEGz<9jYrpm?U{pme{+DxQAV=b-blsj;2b7t!fbd94}$w9c8F#Nm?_xDIzT#LFJ z{8*6hhgH>gSj{Im;aZnton-yC!rM(92b6nUWBFM02(tunwtEhp3g+FYYQirSG6Yeb z;tG^o(}4z;>?rrCe(4q{=7_2;=9|QlPtZ5rW_XFrlMKbnfP1xGBZ8Rb42>0%4LiOOrKz`+ZZ430t)s-B+_t*YH*i7VGIkIl}orY+0Xr-a8 zRmuMSJyGukNC#>IzV3*KBkpFlToYp5&PZ?y79$+C;AsBsLU&9WF@`M4uUV23WKpU7 z4$-yAH4A`G{hQN&{V(S;H3BOFuyBs%(VXyd_tpY5^%F3#2Kn5 zL-+Escn*aQ>cD0L3>)wEPF>tj8RLO<$g~;o9JBeT&WZ2LG7D)uMJgp~yKgNXW!_Q6 zW#TxnBDNznr=9Fd{Yrz#Xdcfg4{3{sPjn8U`es{I#ad@(4m1~gxrVT+_T@ply5T|W zrjLa0%tb`-{%?i@q1sUOepBdio0LpTuU6C7q789bj6}?6x(*$(B&N(!!<;fmVE$=X ze?bGhE_;#M?YqQ{8wmLDxp6LB*5<{_1KFRys^a8)j6&1IQ>)Ly*t!x=?N>3^!GLDL z$`g$%Vg@F#sF+q+-m@tZQ6@;CIZz)Ew>YRGC*84_6Im-_^Unb4{nh6 zLHx5ft{+<=peU{x$BQq*d2Rwn$3|c5yY#u42BC4T#=318w0(#@zw$de!6hz(i#nnp z6C9m$xzOfpC$_0RYtO}b;3=Ym-ak!kZOpRUvcHz3Dyc~+w$}n*_^q4@p9|O-l4osa z*R8xMM2_=Dudv&pw80IwCGdO*nm_gT*rupEdH9F;p+@zLUHnUq}{{X$tN&HZCW7pR8(Yv0aMt~0fuZgu}u(dB+}t6vDpGZDzjcDI>clvU5-ff2wzE8+kx7ATe3mIeN*G^K4>V@w%kO7lK3N!3 z*w0iDJ3&wZN?F4-;b{C8UuWcPfG_;jpwCcSl&?W3w$h@H;Rz!bn`iYVI4&Gex4W)U z2N+L2u{Hd-@cX!O8WCSu0+bvOvz)9sW^xolY{eRqljS4kl zv{f}?m9`Y6wIxPX)$CL?Lai#QwjhZ;Y8OR~&?@?=+FOWCt=O~n-b4~45y_|d=AT76|kp@Un3AQitxMuCNgq%xGjb$Kh^Vv;(75X}1Ax5~sH-k-3c+LiybCR0T{-JuJ;BTX`O zA<5c&-oB;U4L6)T(6=@wo+wL=02c}D>2^6cdB9z_C^tTnvDL*z+RF7EU)1Lx#FOeA z-^^Tprm$G$crOHQxURp)v{K`mAPS%yYh5e zt9P}OD*9f*`f9aKU8iOK;#w6qH?oesBLi@PK|4&oqfB?Sc!_7GkP=6K#y-^}2c8^@ z24_bjl=1aJ@Jy8FZdp-%J`gPiyp5IZF`gJ@vcDO+H!ZUk`!38{ZBM%{{6uq}S9dvw z$z!l>FofNM^MS(O%SX@GM_s@vZ&Tg+YwfAWx32oBjJGi;ogWGhwP4W`+#i{KD?5^E zAH7xQVKdh1lm2oP9~v{A@QEl9cy5IG)5K~|<)0Ng5gl3BMwwg4rUA3|gK*9gB`pXw z4P|2ectIl#vAlZB(ztOe{I&8Rcc(vwiOwa|HT%(PpkN2J+G&{MqLh{F7yndj_fF|6 z-k(tt=WqG^cYLaCQVO#9I`uz^$14wJ;};86_a<~hN}$h-p17^(9W)y$VJG& zsGJ&bBx&@i_(j{YuL?%OXbrYGwYka`iZ3oFS!p}{1~~l(m~W$d@g7+%97}DUbW&>t zPCHDu7`HkYTmZ4wP)cT~~TZ3J&OQ|5_M#BhTB?72K!tEyP zmL?qo*zP#0_|0W0@t3P^26Wtj&<$VmHg=r>fytrGi2X!ta8uAF+%ffgT)KVwrvGjt zPzlexcln-W;~I9_PyI=oA0}2A1diJW$}2b^H5T&RJK>rouZei9 z92ouI|3Hk%l1eLfIY^!>$8)iz2ZKIOYW`TDbk(62r+R(crrEL;`_l~A4+Rppd6h!D zIr5Z@{-bLo0(uSI`P!Ur9y+e~g*2nH36rNmu)s=*IxbSLyCo!Al1@^tm&Pkip2@N+ zwW)q-Y9x?&j7ZwP3Rycj!Ft`KMAKQ~VWju#$q5uLPZu7};-gEeB-IqzgSIY>_e8R4 z!pr-o@lCxX+~zO$kA?>R_*5wOmn ziPIIK5AC+vFDK^M9%@Xs$O%*{W$WrNpv@;zB}#0(r`nK{K`|2o|2~$0RfpdmZ=gdu z^k&C89&Y~8iDZ9{c(ZjA^Ti0A95WEsEA@oRNz$r*h}ishEttbHQh95hSRnBh{nMHk z(HtkoB??T*nt5(C&i}8Z(P3ua!F+Olb3JZKEi_XOot?JKFjdrs?krUpPCZ@hD!I?C z&9>SQkw#R!OUO63t8uoST7UmKJM0aw1bLL^HZOTAQoFm=e)=on-9$_@ZvTK=mNUX! zOH)mT|GRz{GP3ROzwL!T7C-o>2M!hT`qe5^WkZQ7pVh5P^vA0mK>9aT;|iwHzPaWC zfWIb|NY(5?#3R%azy_~ToET;4EV4(YhpV4^NkcR40Dyk;DVO&eCb$bgb}@(d5d*;T&%{(F3(Fo734|g2d(d=Fh zhO2^gupTkh<3F`xiReKDHV1Y8aU|AF3{>h$_vKq=C72v$PnDc-yYQ`?4HGW_SGYxs zM+mZ}?}dli;QA8NUz5i(cUQAU@H5u`u5fby6}%X z?)b1FXt(+?`rI8pF830y$FP#wS6kII{Rn(+tvB)-)~j4>SSMr!aRR$E4F~PgSkYBm zz^yXz?5bOj4QE86svjQl(~BU2cTBPz?qF8bOY4z>nSgW<*=Fs8%YLS%9;U`r$^YW@ z1~-LfhdXb`GG?&?%0O5ieg?9_2a~K{N!#2|!qd#ZnucI|g0GdB_k;pWce#F}{^kA{ zz4LuS*I_Hoo)p7g&bs&S{CVxJTj}+aiIQ$Taiq+82=UY^p7$o8$2bnwJ-^@E z?XY*>wU`h@T1BaDWz{!kdwe_HS_1F+*+f%Q2LsK6}yH|-NGFu#Xt7D6CrZ8gA9Lz%>e+cZ+_ z^w9wBYbqOl2LoZB6)qkuZ@xpvO0+m*+T=A?LMngEyA z7nsq8^-SvSOkDC$z9?BWsq~&|2Kh~cFQQ4v@#8Bb7PEwC*5EeA{U`(}bHT|xz#twl zUz++H9}3PUu(ua9F08ONio?*8RgBgOBkz)cC#trurOP?JkNzkDz80#;zZ(1w9xPbk zoB%wR7n@AmUQD_)4lZ0uVsuDC^L8_Ph;)a%>Jh=E-%R_|{X;RdfAC*v8jfCj3a@$f zkQ5aY)|k~;eQ(!@tNJfB@UaExF~=jmg&zfr%j%CYem6|J;R1!+5v;z6C(k?O+w5qbGge_Ut?A%?DLLP%zbffkRD8Ek)@TF&(hOCka zMoOB%%bB!8K}`v|&N-_3jR&uwMR`)=9DUw9_O4HO?@FNwhfja(ZbD~2#Y)%Tm)m-* z;(6qfHze(s@ZnJSM}{m7QAKBR4$s7PYk02`Mjq~=OG!+yFU#tWK*27RW7v%Wg&CPA zfvwD=9jvoli`t^)jVAj1bFQN_r5R4-qa9@AUdz|eY1u_R`k>;Y-;pOGlzM>0b!FmT zUghUu13ICVZO(L~hqus1T~cfqe7{l*v$G~a58ozjH4YoUwsHJ-q&hlaqySiKd%@Ve zfNi9RiAkp^6TN>F9q-)%G)~v`A_Wqh=CfAo0Pd4GnAfcJY?#}&$17LD3mwh-@X|JP zI%Jz>uxfpuxG6WhP|)wAk68JuQwMJn&pHHeE4dh_%Ki-jX%6hst?1HDai-x~is^hW zEEwBb_iw{Nty_bc z%2Fo%C@n)toi)qDQ}g?aPWt010ZRcE()D?3Y*7Kx0QK;m5d8aYntKK46EYu|^MfJ^ z`@CY;nY0n2Y95oR^Xpt|dhJR52;B4t7ZQ}_8yphOAM!T?@{f*w!*Td&$i5l}n;Uk; z>JYs?CT+ZgiIaZZU_IZqqL$~pM2qrA?!0}s;)b!KJh_eE7PwNJ_2LqVU#F7QZT*rJ zY@4WgR`5iV#OkRm7uDzx5pUb60N`yO6(4-oz?fut^cuXmzBt@&lmK+K^-@!IUK`W^ z%7vP1-2t9^sof}Ne3uv`xfD%!1U(63ynF_R=nlAUY)s2~_B_2g!S3|TISseaCNN#= zG&X4NGc2EgaJqiEc1`#JxzicT6!2uOX-*IhW!C5ENd(EDPWW~kKU8xZx=ks*GAyI@ zkErcTgp9laxHpHlEf4(O55>{FegxRm0b5&Q#kq0X|Of?jWTHeKsS0 z^DQ5(j*@?}Q)0ZB;2JUtpPK7E+Gw#a;bKNlGSs>DZZVShu(|BZ_1JG(fU`WTY+;*M zr8G4m&}(j6Y9)&TL=JX_p4hkoe8nNK0}T`KsAv!d=-tM zB{B=-?{i#1I#ZG#YOfd&3aieYXpfIgTJi3Y1YMwQMCfss(zzEkR;cf1<(*wu6~SYZ zA0b+!1f6MC4Aa&e=sypD#~^TXKY=o@4QP(AMySHLs=c31=s&=L9BLMF|@c{4DdCmV( zaLCA6hcY-&Q2F9;2mQ|6CQKVbLZBGYedgA-iD+o#9C!H}d+OC9pR!m#Pgkj2Ct2Je zYu0w5MajEDUG2`9a>-y+!7!uFV?_+!qE%EWnsYmTi6fPwX&w~XGK z9G}PKeRulGeq;eO#>z%X=q*TO7YO9-%`zJ)9*-=bj>hRXUF(pNm;|C%4rY=6Op1|yo{^@hcEbN2UbKH+w4Mt!^3uJZQTQ_np~oJZmE&0URrwEvhFxMnjR z4Gd~D9kbwP%kD?HkWUspSp=Il?h#^QXWL@nQ$&7{Bq`BoC%!n~&Bw}JQF9WETC zYR(Ov{aJ?`;No1ckU1=6UCi-fN?PQO2 z*C1!Qu!yGSqY@;{V00~6`ri4J&;DeS<`>dVIj26i$&tsz=r56SBZ%}=Hw;@3NSycY zC@rxGm#vVBdLb~|!(r|NSMW4O&6#j9n&UbH`{pbYwxY?_;L(XjJ0B9mwHI;GLe zoMq!X5=&MRjAO$P!6=~v-lC)Ip}NaY`3|)@ra&9KSra1y9v=Y*Y;LFl0$GM zf;(WH4ZP8clicxxY5c5>{s@5hfW&}42?ObVI?L~-NoFR*S38aq=qbDgbTa3Pf$xUQ?4}}jTm*8cxfi#yn!}rB z9tej+`MFTcxehI1f6a4lluxRMM{{x8w|Z(V(8fIQT^GKRAhm0yvBC^*X^rlmtBd?8 zshoiH3iKPU1U1)8Xl1NsomuY74*sQe?%_DN>-sMLw_f4$)42*~?`QR?i#HM69hz|e z6Up~U0SyAX!t=@oJ7<#>5J5)U9cJgAWt#v43#Gu;w!}2QDlR{P1SX5AR)!q#HNKBa z+ZV%iqR>J2U7~VudiAyNLr@cwlWiR)LgWGUqk`FAx9m<>+G z9ZYdY9;O4cRr7F5EZ=x9fhSw%xigq9S6BLK3Vc_Jw0Sn{QLHU~*LV62OWnrlj@n2monfW@lq`s?HilOGdfXV$? z!hv>iPw|*k<&uD{^SKE&VYX>0A$b=or&FPLd7Gp_?guE}Rd2FycPwwfsmyYYwBzeW zL!|g9Hb2P3L;B>TtpQLtq{IIfH94mIM{>tHq<*=!lb_IH60n=hn)b0ZuQ=Gin8dQ% z72fkNlUA>fDPWM>Scnwl*lLi`W4f@N!1rn4rwpw)B%&GX-#i>mk#@JbM+i^x1Rm2( zK0S>mVG&xO&FJ%(prgT;C#3}hy?~GJXyvDT9ossIQLnC_7QlPMw<6(V_j@L5Y5_Rh zJAp{w`(VUXD&Pb*B{y@^>*(8UIBUwTYxJY_*nTWQ&7I5POe^l0d^D&ldqR`ay5Vh9 zES)P9U%SG0FH)<5%0+wz7tUu90_{c?n|7+=WdJ+Wuk>E!mL+yOntk^IJ z6${}<1Y%;9z^YeeCIBx`;4~RlSY^;IOJ9avr6X{jtrGcFXih6q3n0ByFBZ8QkDB{v zq^Tx)vzh#K3qbg!z6F7X-{-EFGP+%h!ZT~$nNiyLE^Uy|&kTvAbA=^un5cBr{ZiX% zdALn$DQI4WEH}K_@fp#dTy5?}|1YHJwmjD!g~9yP0!$v8BjQV@zH-cJg@`^A?-ow} z36XKAiEUNw{roHBWE99`*Y}W7NdA@gQlk&ZG|c19@wcNcOFgk;2z;+GA)$)aquO!A zs4DiLRuYXJ+%H=;wfU4k1lYzh;=rPB8p`BFqaJeDJ{$KyJ%v(jBNZ2VmG=z*A@7}b z#=S3PNdKg-&U-VYBv?JewlaqOAwaMD-=9(f`*j}XyhSI=?|4&@72me})ORy_rdm_f zO0I3LarMd;ggVw+6%qE8hln6D=MjFRw}S?sov=tiP?izy2t#-!i%-g z;<4vquT+tOH2LuFE_~E#A6)&ODeRI*F4vlPJ3I?mYq^jx`vd=wjrDYCfY9@5p5mOd z1GfKK8!R83&o&+H^Q8n z6=wsAJS*1gXUP`Jwj19igiaUFb`vZilv1*rCW!sBBiYUdR~>>0VaqtBZ1oPJOwg>%`~WUPgS*@%sWEk-mgfoXd++ zGnh|$Jhf?L)g*}Va0$?EFm z-m*mVdZXvDxH`kGY8MX<^)uwbvAAP=#Mz}_CVP$37eKce0kycUC>SedwWKpc);r@4 z9R@iT_}~)gh=slgl>>VEZn|{c`nTw4?4P{0_}F02?Vx$VU6YGRc_{~CWk=rlH7DPf zpRP?$(v!n>qVXzhC+Y*6D`jGCY+E<)PPA5O-SPS`Zvn#-_G0Z+>(;&_<@VK!@2|nH zAJy6GgNe|Gw4Zj-;OCZlTb=Zcv%QUQ_kCkP<6*}_eU%1+HKv{p* z%hw+bTYL4EqSOnIm!SGs4w+kY--V46QWy-PW{ zN-Ec?={r)XUANv$c;o2$qZReO>~%SBN5=brbj0icooN**#0?;|BP!FdNQw;o+{ z75XMSwL#2F+BVvzvR4tvL`==-Lf@_J~)&uwc-QEqW$odz+kK zL>5d(=7_(y+Myg9LN3xfp_RurE)m)t9RdhVgZ^6U!0s0eyejq#J6b2!zv`leK(|BO z9)Zwu;IC{XspAcSd{(CMOXwF1>{XxU)UjwN-Bu&{5~7QxIqn6r`OVZiWXU701+n@} zZAZKF)TY0idnfgYP1=d;(t-RQBL>x4Y_`-c1XxaF4A0@|n=)J8b}tT2l&CZHKznGYvBE zu!o*@N7ssNnO(vGu#u0wJ79J30Q3mCT|Dm7g3b9HrsjXX0|4@@%N2Ao7!k)jv+|$O zYym%@Q0qd3r8dX&50pdYJLrWRnS4a(!WLV#(Q02%P)_L+FOhTvSebpmp$t2*i7f%aUx00d#e5(vnnax8CI$ zd^2X7Jc`9W@V&*Y&NcZ%RR|OD?Ax zvbJb~0fNl9KIyM8{T~9&C#NfQ{sE8eRl-N_mv! zRm$STYI4Ngt)c!ed4~xBKb^n{4fp%$yn!(hYhrvI5s`P@+P?NjJ2aZp9rSs315%{%8KCVXN!^JP2}so>w> zW&^HPiI<+X_pT9W&OYWn2`1r74K+28zG%&v=T>9{Q1Ajtn^R1@>^6SNXjuAd3Se5WH^tBfEaafySl!cQmZ+yG!&bfFn8w-Hnc zjCnwL`s93$!g9WYwDn*GQIb>5;#vwD8o!=fhCt*?b6Ybd3o{5@ePweY@00$zLPH)z zFZHB9xKFv@GoH#fMCs{=~Hj*r&S>|dMkdNunKnJ4u^(1z;*5AaR&8=u*oUtWH0ySk3q zjF6Md!G7A$K7I?ArO3MJjI+fQ4qINMD31#VZOl1vma_#7oXT#x4t5p39G-jEd)Eh` zyBsD*IIyw~<{}q#!3u!0MK&R5a;B2PGOX ze$A{=dbUiZe)nin9(s*>$ujCM%m<$+}D zd+q3!w^UnQ)1^iM_vr|&J|(hWq}Ot9`b$OiSqD1j3TOJ3UgnA#<_TpF{T%LhlKa?a zC(Y|@(FvWIQBcWsKS08GNh3$Rn(v-6viS?L$!jJ4LjzZ+Ks8d%mU{;utRtKJ8 zL9)aFc+xEaz4`p*NnhpYHPUB!D+Fry^|>zdqeCu_s}#;}L{rfABi=(oqrhX`e+Xeq zAOw&GjsM8@_+aAsg9N22r^Ugdbv%IMhe?ZpEnIy} zy4cNMD78pp&b$u@2KtsVlwiz;$f)k*M-R%OGo&wM!Jr@wf6Lg-8FSJJ=0vx}dxSJN*sQ{CwvP4D~;tULY?-}UMEI0gHk2sNyEt|n^V zu{$d-KxTDl-7#JXfx3nD4|I`QK@k`ZGzY3Y(9a%o9()JP{eBp`&v?6HBI*pjcM(Vd zek0lQy_fL*QbP_@B`KWt!3K-uHXAI=wI;*j%QTF=k8i<~9P%oVVx#mV0SbS4d|TV1 z@VHkas95&kK*&$41X%9_65Ol8+ z9!cP72TDSF&@IY(FU@g za>&DGY8!K>tj#4$;QbFGxg^8+uIqLqQnRi%Nr&8ygc;g^1ND^DgE-LZOtpgvo8Ge^ zHJCpVaM7={8$`_%*v#|E&Hbof|4JIfi1nzPZ0_?bFqPmX=09x(gZL|A{CWOvW!^{X zt8akYF34IRA#7f3df;3ufi0-=liO*_7x_>3G&%>fVc;1DYyGrBc)$4Ln|694;GtgI zZ!Dl)LzR8>CEll6I%HWJ`D$+jo5fYN^-G!>rhC9-Zuoq<@^FEQPsD}IkLn}t@>3-B zI3dPW(;AXd+9-r0HqksOz$zgjz@8;jLQ1zE4^%RX-<5RBE%47Mt8kWEf57bx9emj7 z)_9{NAQJ`mDcR_3XcqBF1XO@ivMa=QE8dxOXc8p@$0h0joiV`QT^vBN0jZR^ZCVdA3))JQJcT( z{GrWdG9QnOvlG{YvAnwlJ%!KO4L~ zB{c#Fd~oW(Cul|%6T{M1zZKsN_o-gB96w7(+(MFqYUOv)s-op?Mz`Q66bqZ@ZU442X(m)o3xfrr6xJCdixOLC419 zyyE%8A&)=ST69OKIYfeR2w5p)U^trT%7Ad8RZhc+QqKETDSJE%>n1bnT`=5 z=ZY+SCT93sIM{+iNcK8d{%`(8F}M^vWT#|1UNXyPsy0b91H;&N=>`w|NSw-(X*sM1 z85?2@;5Xh@Km=`}3q^|-@*+EyeLA#gYP>|PcJ|^!NOP`Ks&c-h-aM9jTQqV>WV#e8 z*PWIvZQ0A7^A%F{P(imWMcE8MQ{v^=An;dvqz$tDH#)rQmw3hy`OLXtXL(W{^Pz@k zJ%?R7Cs~KRW?spP=1lqZ%smcymCIiCbSLg1if-~jb?Xa0fl}-BziO6MgruWjiK}MyTWTMsve@7Ej?alHgg$~ z)l_Biq@li~I<{8cC8f*gLV*G1C6fK@Iv5;NBM{EtJlc)|cMtZCZy!jij42%z>{l8j zfE8$t%boQPwaZ}5!Bre*9F1odx zaX@QRO=la6`xDnIMZkhrU&xmeseoGhta2m=?!&B-OSX`DoV*dg3B|DTT$;1T3hl#J zgzLYMv^VPH3Wz=xI7(bVvF)sg_FH*%h`F(>e_VRv; zig!y#+QRyb3Qa_DN|AjgS&>QB0*V+$SuQ$wbq%)+koG{GKckL_xK&Gx{`>u|hrtH3 zKoHQ^n)aD(4bAHwLw@bzi)eM(TenZs@&VO0FNVq+p`j;B*8!x;5G&;7BQ8ge38~*| z?NcS;tT85xmhju?rljlde8=MJnvJxym**-jB9qr46i)bvbd0X$Tupzq@_qC#AjRjq z_pL_*R?@^ndgbs!o26Y>9d`{FBUGi}J?J>!L&{Rt(FZE|%mndIF@LBh-+bSRneP%@ zE#z~^C||keTxA*Ijq>Hw(UH2317ic&N!(m_@Y#!zlX&j%qX_R^Y|PZ)?>l6Pi|BWP z113!#X0+Jcibz;xD{{bfcjXsHJdbm@(gHPtDIMR_;#Bl@{%Yp8CSeLCZ?}Wa0AV)Tj&)Rn>Z~dX{Y5E|ZnW}TY|p?amSy~AKv^>TeR4oEfp5K%qkL$r zG4vG?mZNB4^KzMu=!bP$8bwy$667!cb1wu$7;=_gVs{$siXY*ILV0K?!QN>$pSJ~a z3};FeMnG@=sIEPve&A|XoecTV24?H~R2XjjY2$NU0zB2|r)sFtB@YIhrOP{peC&d} z(3$gsbH8(=Idc>drZ`G?eL~K3wG?*!go=4i#-82%4X<$e&U*3{!MN#Q%u{A|am=iX zZH(Rz_`ur@bcQ%il+|oYI;m*UB6<_RyL9H*2d6$NV#(sgJ>4PyJPeW)RV&U?`g?-n z5rAEy0gqLx%62y;7!jvWI;hyo{b~Z1b(C`!zj_kA?I;JCQ-R2H+}w(W_hZp52)>Az z8<3(~YY4VLfKuPRKBXSfEW{k0fkZ5|A5F)hVBxzTNWb;Hx@qY&8dH0{OgtZQNp|s; zt@Kd}ep`o0d#anI)F)2c<)d0SdM@J^bhta?=pJ>FGW_tdXZhnt0(q)ulXXI8pEv&{ z6WgRMlJaNY*krvS22bm^TmS~MWZP-V6FVXobU-vtdHbPxqb#kJlK4R_0?fu$( zBT(>!|{GJ7BgN1rcik zLiNW94$hl_f ztNN+QQ3AzElRx}L1n!TBmDC{u4C|0Ouh8Imah&`cIVV+`3T879 z%N|{F8^!}YU%lpE`MVC@x1aB#{y{o*?74+q2<2O5^#5-tuHvK>!c}ylmK}D%#GR?3 z{5@YF_HN^v8(g6{6-4iDF9tuftA|@&#|#;~ZyHz_#B6 zA3>c#XQ>nOCrv3H3xBcaE4ySWC0>F0|KJ_}BkZ0XL7>VC7N#<_>K@M>G*g>1cRE(G zm>;h0*y$(Dcl3{zrPAc@!=#+e+3x_1LE8%VS8I*4&sc|HRi;&q zhM3uKiN11`^H&13<*em~6)ut`MNF$(3XJ+-TXXBw%Y*$4e9W($1KU%ab1d?WG71b@ z{RWixY@1ZM%%Pi^p@!#~0Vt|)!Q%Iif}-_WQ`3#xLh37ZZW^`2*vJPx9=rM4Aa9|B zgb1U^h-aF(#Cg^_b~l0GSTiZXR69%d1tz`&9%`QyH!(q0#3WuSwd;x_Ekp{W zul=Z?0wvA~6GMv`O;s$rOe{gN=_YY!WTtFuyFD*>P#XPyfj#W#=#2Ba^~h3^(!N$;y{1Q;Ojfz`w{lpI z#)p;Zu4(D(L4|wK?Cs-oDQu-2)+f${PNRBTfq zxQiZcJ7&}B{z8Cs@Atimf0YwBY!AS4->%(KIx;}3G!+%_;`)UHhIUQDA9t7Q`5YVW z;FnX%-x8zCn?(L)99DKZ?_QF_*QwJ-Bf`DhD)2h38&h*NH$fHAIjQj*f`$VDxw<878ykGK`U%O?7^x$)VAC-?<9 zl!f3he?hy?`N8vYJMKZVc@c>T^!vzK4}SGbZo~1*TmK?sjP0y5AH7|9mf(G<;?G6k z=C$~t7wfHQzlpEf0S}#U2@|I(gm#3!BA-WsbfJHGIaTo&_Dv4;JW{675H|Ny5{#0 zslsZpth<`+Gz6XN6a83ZCv(XKEa6gFGC@V-3DwIGgKX-S(!xNYASo{BSe=v&^VeYUT07Q$PF- zNGQGv?8Gp$Av*k8`SjpBx5tz1deQdixu)cJkYU61g+HgsW}$atrKgH*3>#O%94zGw zZ-`jbh*nUfHok>>1|$qtNw}>&Wz?@rSbhF;4x~sA>3T(e@}Bg^C3U-cJ9t-q`CzBg zM!I4o)R8DX{m0`FXC|exUl~X8jO?U+UV4AR>h zs4tBN>*N`{i&+VN$=8TO&v8rz#ky-uY)1#nu>y!@p2KG|*ROz%Z~ML#&URHm!GO|A{ZNeuE^2G$+s4(u5oCqCW5)wpMN}D>o9Rzc4-4yil zvAnP&t(UBppxX-U<*nwa}5u6ccMbl7EQ|J$MFmEN}+?lyV0@Mn%x8U^R8+ z5PMDu80Q=o*mNj@Y7QYVkXG4RD<^?H$e->0Z{q!L5(OpGnywMuhS$#Sp%z73YIAmW zjw?fsJ%+J$vYxl-(yC|dZ^~YF?4fIL9?7?4yp8pHI;pblGXEvHar+W3!%)%dbK_1W zR$dV7)#F5!)Z!anyMQA;DHU5U4~VMVzmJ{q4V?>mA*iV!6~!)w;iOC4^vz=hjVS$*!u1jr!{%YNaeioh?ce z5opj0`v2k*s5anb=6^d4?|dAmbh>YjYc>3t82(`-7${{oZpcX4t|vw~Zwm~EiPI*V zC>am$^~Kih@uk&nr%$EsZu_4{=+K4|BhuMQj9-)W@mj%k1FE*A}n z+q1OwZ9F#E@@2k|kAfDi1Bq+0Lu)fW`faWbe#NtT3E%5o7^GbuHB5f`Do%eg+hp=| zdaSY5&V=z<`1UlfMdyo(Vl1 zlZvV=uFz^1(zxoM!JU31mMtL6ve=BFApBrlGQ<|2}3Mk2h zY}YklNGqLwf@#lFhF@Bx_WhCz-`25XOeQB z%stNp=3(*97|Sq86~c(PGV-~1iiQu_4|gg-UzauUIN;V+#V8OX)J4{ts7>s6D7MuV zB>}ovstVzU#5PLdq19>QRDp7XTO&T44RDiyk=KFdTbt(Yuh#DFnhw5R`RjDk^YTe| zhO*e5l3fm3qU&la;7JBJJ7C;x97)S_V0Ilmd>dfEo5EW8$KMXnc#b-Ch(r6EFa1w` za6#_K!c-LR9jXTWuUERe{r^^?$RLu-f(Y^uS__{6+Pg6d1f1@woR1+7iCGx3UPI@Z zK@jF#!S#IkeDQc5f!o|AzM3bAyF&0?n0lnO+mg%0ALtObtY#Z@MOxw+enQZ?AKcU zf&txNNrPhUH&t-6lan!+O4vN%l4a$8K`d?~esk;z@6I65Y*qb}+_tC9O}ofd^pCI! z0UaCqyG!F%`oXexs?1wgw=|1$_T2*6rZ55)XE-mgs(bB{g;1`l-9AC1+6w{-PDSG) zCS&-t4V%ivY|6L~PSx*5k{6t)*Yw5m(sK}Dl3|MK_XHjf==}}2y4A923>A~!Q7#<{ zF`sN^xSSzMJM-a4O5~Ag&$GFH+L4@*wLC&D@@I>(%XXKbTa{%W1f|dlFV-i|WvXhkO#uyH1i8E~J ze2Qd64I+G^qVXSn?{S_4%I8Q0d+(q;^fTL7)fP&3L>@gLpzf}{sm=~oHkDQ=h>VUi zdT;SVYu@8hqSl>J6Hl&4KYbn{OwKUEZ6@0d9QH|(J}1WBiNiA za6@BO29wkg6FrqC#SW0=Y5$o=E@E=9Rgj%u`nkPX>kRqr5;?6kplO_Rj?Mrs~>h-hJ%K<7VmJul0{r{cUk^81;#4|^u{;;D@9Smj^jS(B@Rs5NiM7_ z&C_iKMgrV6qARx({~b=Z_2hcEWb_p~X&EIEldIc;m{l=&9{U@M)=|`s=)Ty5Z7;mC z4m+Tde`3Q&?a}Qw83%mL-?)|Vm&pK^Q7Y)|-EWUG4plXkwzO``TvMuuzd+&^^Y?cG zib76let2^j3%K7?c3G>J7acHZchn*A6R-@_S zki!WOydzajG8w)3Bd#hzxBH;k7V4eU`xMy|VS^;{VD{jf*mCq2s9fr$2jImnj+ zc()xiP{}7kwL(h$;*0O#y@FS$z0y&My4r`X!iclM6e)ZfP!h6_;Ue~L`Ump$sQD(W zcSN{O57ym*hhc2n-*~feoV?`hV;&v=nvHyI46NR&Fdm-S|9QEmQZ2qWn5=akY%Hf0 zWkzpSE3E)Q1lrH<2ZXsJSO}JkL@$OGaWsXA zkMyouPwQ)t6yg&xwcY>3FD8CVgRRMs@qf$DC-W0pD81Qu?R z&{?&6sTO;XdgLpX8~+K(jXx5Qxl)%elu@PP0U&=YvEO^QuV1>}nl0oH-K)xkD}<$^ zZn$o6^1skStbeB&7dF!LaE<$ZPr&kc!<;;A>9ePoD;;S8WjC7yE=A}4JJLR0bDs@) zkx18n=<)8S7sAp>t>hwImQ~iX z-XRAH#jn4~cJ=*wodp!>HX&Ud{!%jw2541m`xvMu@L?C&nA3iFn~x;2Y}{ZnG@b$b ziG3tdmM>bYi!Iw7Sc5j020pynM|CDB`WW=Ir$r+g7b5sykDX~ z{~ufD{Z94c|9>-N9HWwqV};6AWF0F>b}A_xva_ zzOf13jKUP1ap+Vha<46!5UTGSQEzdVLF*e8q&WBW#^eR4zG4?)1{`$jd%-Z~`NJyzJSG$~Avs?Gd3L ztMa zR$<3zUPi_OQ)<%k1E*6F^V|c40eN!Jp(fzyA3jcTnyVf(Z#L?v5PWg>gF|VbGLX^T zzM|C2cqm|*1W$5CPbBWEs%GZce3BZIZ&FkRdARSF=9kkRUuLyyfYaV{z0QZFwm2_b+vY8pLb+a%hXf zpn9hT%~Ei+YJe6oMGcv{AMq?<`_mcirlhL>m^3I(8PJDn^MTV2<($niQL0a-C7Fco z@xssoh>Og<0DcVdzj)+c2#1s9rZK_7b>w$(Td5Q+dV4`X-$Q>63Zg1La{SJ;Y0H4( z?Tn~|T%J|V&{q8K#x8J74#Lx5X$6i~*H)>pq_xWsl6lw)r7|&oNp2*1T<%@2hYIaq0|Qp7_M15dx9zO;k17#jm-TWB^mGfaSwhvIipe6sWb!VXw#_3IGOx>bLYB-7}m zkX(hGb;pf@UI(744|l0NdQgU0FdXpp0@L+O)h^Glxr-jB*jDN(y|xblB#HBL*7JzE zfH!*T=EaNlQ%9G7N2uVKr}W`Dzp&5&s-UeN4*y8lxe}bu&r?x1=G^n6tt!ZVovz8< z6wRq;nts}37d_~Db#DvekawK8ZV2W;e}6JDQ`P0XbFDrnKywbpxc}F4@>sz?%9!*r zho)jSR_I!5Yl-y&WIb+(O}opc->@53EYI(`&0@y1Q0jp7sFZ7HK=(-i=b9Y;$f8)wYYPscp9F+2{u|2Q zx+z*bBJheNPjR~=>_e}uRW+z7eVY~_0=5rNeT&uugAn|z6qNBMre zB161`B#0ENX|q)fYeX}b@C_IR&)p~d{>r_{;eRIUG9WLb#aI5THR&O^^OfLx_3@1S z)dRk2w8R`Ywb0-9CaVH~KfP)zR?I$eE$Yk|%rO(f5HD{Ri%!zb)X9H*Y3|Gd8KyadPa+w2D2t;LXod1k92=_6j!xxY3vgO&ghI5)hJEi~D9>x2 ze#%GM4nj3^-6E>-1mpa=- zkn8A#6FaO_p~AoHccIoJsJMbF zQ=WrHuc&+cw!O@(voASV_u1lbwn699uDG?po}|+GIs!~f1<1BeFV0*ttm?=Hz>9E1 zzYw%Xd%F#9HTJmm;qPDJ(C_PaHn)7$gr-6!pLV_6Vs$thoN0QTv4eTXXY|GgU&Nr9 z4yBt4w!RUo3tFF1j(-m$Mv#9=RogF_hX+LN&0oWTR5$F{*WvP0a5AC#~j%rN~0Bz@*B5#c)PX!^Bs|y*fYWp5hBXHa6HMVMgo`#L1 zZzwJ}C(jju8jYL@=d80WHPJFXq{V83@!+Yde{M$x-ivrTFp9tSR!!ZQ_>cPC8P93* zG1Y?*E*fw!o_#tvtizHYQzW1X_8%2(E&=EN1@mb`vrX$vr1w_qrZCrF2;ChQ-rOw8 zig>ad9sNFzWD&T(q~0ehRMNpQrbX}nQW<~l`p!5{nzSE4{^Osm6so&yZAF1sR z7woN6#4=4*#IMCh{!Y7WY~N)#KY0DmJA!Siek4w4+9pX5z`Ia6pHq&LA%v@#nJ_LT z9JP(b1iO5G_(-{f?|kkJLlbf^af4!`J-}gvIv}+0j!kE4&$!NaCRYQ$QN>^I&bYb8mQi<+7E}}{N9LiJP!H2Z z_WMrA$u~KLO(yh?&e#jY2TSBB%vcEN5bJUI22JnU_oao%Vk~j2?4bZ%j{GD>MhJp; zI;@nfM?8Q6vNfngua0NnoEdw7VbLikpZJq5t}dU>-gOB3uiR3pE5gn*LZgwP7D;-! zpqLYm3^-{A^o{#koZ^JBo@$97~>s>+6MNVa{1mdE`Z(kLUKSj>o}e zPzl^{sO@|oAj#;BY@+YZjhf2BNZJO<8}m>TrpQ$Z9d3$KRWxt-8)bBQkOipY31OoD zF#ceh75YdM?96gEF^RmHD;(QpV%-w@FoikPs;lL?wq00wZmD=~9CdJQ*o@Z@_t&q- z`sB!%=4scFyl#)>+OTpp&4v0UEz(Jm*ffgB$peRA8UyD8xYB+c$-R$gN+gFTTAm&6 zJ1(!1`_3Qbk6P^BSmX}9=Mz*v(<>9=&tJqeJn2O@G{2~%nsRmh{P&OTI|dTR+bj_t zF;>X0dj;w?rweU~-kQyIwCeT06$QJRRzZd#5tRJ5HrGz3T8G;Kq6W}Wd!~c0-22ZE zYeX?_DG3F)3~w1J?}>T!|Y<_%EeV&bm9R9TwyvxMeki)#?cK-mb+-Qa}lG ztEHVp8TBY5pU_&-GIu$iR{BJN(s^wN+sCbcxr8=ooXT8)z2LV#`qsxir7*pLj5ez-LfK!RksyGJpF;h*vun`G6?UO8zeMRV708lR^Yo z$|b5*Qq*c==>B?&&qf2wd$Wx-C+YWMN!6)}PpP1iu_YD3zl;?4UkR}H3JAo1cLfr9 zoNpCFY#9yVguRW|Zvyg}ur#Q3+6Ud8{OAt@v~xl98zXs} z(t$kxoY2t*9Kn$4Jkb7>VNMBU1X9fQ3eu&roYv>(DoGw)y{gC^jNgC!ZX!g`b$8#Q zj9Wt!7BGDJvCj0f0Iy&a0q_jYP&fiRe_^ul4`#v~LktHeEx4pfFb|PMq&mF?P=Z$47saG{s5Sj-tQEIksU?m-GwIGqmO2y|_cJ)zC zma!`DEUJa42U$N3=v&5n&5M0b6x@F#gAjJHkuRt>=yc%E5s8~ollV@xc?Xk^V>VRa zw@iE%PTNzw`({2IWex~NWgAH-UMm4VfwhCz4RXH#P(y>%iZW3BF>W!Y)jx*=mb}+) z``Llfs}bm_CV$^qA z)IP2~8F6w0Y)SDJXoO+*M<2z0*T7rHUd@s+uZ-E zD+9Dwk9vg8)0j7x?R3iI`A|QTnOC8AXe;kXU&d zg#Mh;xVOM*&%Ez_OHXvhaZ3mkGv%-`iF;k$iUeS-@Tc(&}Q5$*CJ@_>pmzwr6#oSE|jo z608hbT@gD|JyfC+fD8drmY9B9by|YagNO;CbCiK2J?t9+~ zy#v&wO(gEV#N|6+{Vpu#UIiuPEukYSc)0--5?warMpK-X&jx$S9wm}o`VinnA(Abb zA1XXmt;cwqNsu-OSOa=Y7_IjVQL0`e!b2m1nC801gpkTnzi4qW&G}E2C3NukRnxMw z$zpxH0Pn8BrzPDDGo_wg%U)dzLG_Za?FaSlDvsqSYf~hbT)3%U1(TVD!`}A5fIo_J z0*tRim6{@5_rS%KW0o4xSMTzp1*z8u5K=-7Zh;P_(vQNAwr)k&k-6XTinhB_Z)Vyj z>AsyzOpZjfxo+RcBi;Y6~l zyi~iqI`F`-n-pJHX+($ahgY>i>c1!&W-1-&G}gNJ+1y8tKHF|*mp>y}cwGHQ7Wt6w zmit8ZVB$vQJtajRWYuH?hXe1dNmCg=yXNf24{h8?#equs7SGs|P9)`-G3bBZG#p0{ z9F=JP>1Vo?1#pO~^GpJ$Lh42z+;O>}^+*qQj1I2NpQbR8jd`>+470kWD*08#G~|;L zFQx%yUMb;~^%@h>Z@5j;?%496CR!H*!9 zxXJ6>a*F{y9@=&!<^b&_8y-Y=B9-L`#b#lU3cy|wyX&#`a4JJEQ=49T<74CVcHm4M z3qjrLI!&I52%lwr15`6dvQp;9JcshYJ6lw1kZ8^4>k)8L)FcxrKG70B`6oqT3O5FHo=u=DyK|>$JDK_`GS6ZA32`sdw-rA*{~`*uZUXlhn1R3`im@$lL#d>LnC+Y`UcP;@5 zNhH`nw98->_M;l6<)vYa%7WfLru&Z%+8nSa3ysLb^DZN!SKU`x>+zDiOTk)@tzzmqz_3GWWPeOp}pZ zpLe^WUk7%k!N@)*P4yDh{W=lyDAhXMs$*WhDHo-QcV&i{gq4PkJO8lgRo+Zp_HNx9 z4{_Aq7(YW-p44m9p|8s89RFT^TyXK`tzlWqy7Z%qd7ES%cXiN+b@SgSSv}$0r2*Uj_ z#c(J394C|2=SvI=nTK#5yc1$f9&`g|P4BIB(#8dJyVlR&Cq(?!oakUQQ7Li)4e0V$3l5$WZq$?gRI( zWT{e7U7qSc(t18ZzK^=fQOcUvKxFwn_e|i2JzVi$h7fs83MDvEMF{}><5+_N!TsB9 zm85S`|9GEP+uM{#UeTd;BObzF;#y3hN2w#D*iGRx2cGuHQsRrbNhW^y5Eo`P5dCqs zLR&e!W=M`~+6ATBd=ombAOuw0_m&$>NnrEu|7m@;`>g)H_lw6(Q4^AGR-O3`zLB;F z`tpC`8|KKV((_%_y^k!@Qnf=ixVwqyKH9!lr&(cZ@P`uu?>`~Kh^1bYVT3`{9<@xT z7s04~7iV3_pteE{WD`JWULf$F=7E>(=hgGuyqBj3q_ESoMMTjFm>EAi4=0FRt{s3$ zGRRFZYv>WM>7wCs;nLf>oU#c#4A;FvXC-U6kCYa(%G{UD^O5d|Vt&9aLM6bhguZpx z4OwK>^*@{D);ESwZi@j?zdtJ6N!k|*tg{2=^~9-P9^p09ix3ROXTM*-;@bC-&Rc2E zxw$GK$y6L|Rcr#?IH)OE>JI^?$)Oarpn_;DToWPcq%P@3PLsn9qb<-bv=afB{YnJt zntz)mE#cf*wT9rFq40iF?x;{tut(&NK1>_Ts^`3pM+$#Va` z1K@9`XmI;l?0rws?^QOBsTRKUCjl13Z)gDie7swoeAMMmUZmy?U%aWG`3}4KJUQz3 zYNI6aJurBTV5=6rW)E%B{JLWAo6db5@|$}8Z^}?SD3%2D#vT_ z$zz%J=dk7zu6c`><0xHWF#&L1YhTxrz(;25e_aHJ5uCpp$u;pS7H)m*Kn zgwo<+#mx-#(TV{x-Q++k5#G!#n%xMq=LpLm@A|e@u2I*q z@Da{43K_qtF{755Gi zYxVC}mH@l|N5`-*}=&}uyDw`B(JY};HpZECE^ zg3JRA;X=sDnvYP=?bq2!F!Y?5Q?Mq&U&JfJnZJZD$jyAhhV&>uPG`s>oRkZ;IyL2R z{n+#;SX3NnP!bMNw!+VB&#vh5=Dp$aMqap`Z z!A;UfH1$(4c3fEZrsKa06Mdm3%efakveP(FJBdetsOFj$C~0iS&$YLV`MSxcSgqlE z75^J5Q#%+aVAeAP|1e&AWb*7WXIgT_zOMi~I{Pt%637`%Q<|jSthQ+R!o2ky;diuf z6>?~&{+)}>6 z-v>Xt29iB->bX)FbSL0lE$TT|_Qu14(=Tbty}IbV?MKQxQ^4fn+?~ReVl6edh1s0- z_Uqhvs!Cv^6}>I4p28I!MhG;AJ=k2@dz|eZR24Ym7Wa56wez2oz+dpAE!0 zc7-dJFMg_hnal7XPkj6QO!DH86_I_D|J2u|;yO<5N^aO#`>6kha*WN}wwpdO>|v7( z1+`LKJ|+bTQ<2UDLOu-;pP;!SKF1CJU{WpRTJ)0?KTZ_e+{)BSI1rrm?8D~4RKhP3 zAKZ4kApCo^an^BUlgJL`hmNramh*GHy8(&!j(tSw^xW7hH8Q^;|D;k9&BWse#f{7sgh>nx(+OzFpp~fMc z#?Oso5v)f19;D(Ip6)sJe+URn$4C3Q`bROQy?l{<7^uln|9o?;f5+BenK}k_-(Obz zgVN_?p!@Dl?tY{4g=04~vn=&i`0T95Hq}4>Cyw&G>=JNzC77g#n8k6`HHqfNIfIMf znk>GT-!ehp*jnf9K(&k@g2PsU*RVRxKE^ccem_*!Ckm<$H`G{DI1@KBAfL|90%YGu z{I8%SuqojMeuR6|zs^iBPdw^83%)*f2nmX;g6QWEu&HVr-8|bJfez@DULsEoOQA2V_awdJ8TK$2hLfM!x@DQu9|(@cmory^ zN=6}@Chf@@7xII~eSqV?lcl}tAKMM-cf~ac_jt^ar_`fTLrBcUuwXKxQ5%(lJZ9XacCcH&UOqbYNkFOP%jjQ{Cmfkt^;dp6EM*f{N1{4% z2jG)`x*yzUod`usiN~8Lr^4Rv(WY{*y*3p1)Fmt6zVY*g0DdLuN-$7g#U$VY&Fbjm zdZ{CzQUyM)%Dfve=^zf?L)*@?MzRAoApepojnQKxg?cuc7UL_MLNgBm?UCOp`3YO! zjbbZ{mkuPs5M4D*%-7x0-BLc4W7N1>UyJgr%u>t*zgvjM09FQ;$lx41+!_j1@k{?? zXv5Avt@l&%eCi9siRU4#&*WMAYh;G!r}or3wVw!*uoMfP|8kN+WtuC1#SFB&oPzeP zZBFy~9g#Ls%P&6t7PN9zd zeBC(ewcMYcbVh6N^iccQyz;Mom&~h?I%n}My|U6LmO7)w@?RJ0Ng_u1z`OpeX0}?o z_0SGsg-q0Ysr4o~JWNHPScI_hfb$y|i;@+>M(+0VZ}AA{C~uiDw(MsR({N6hIHMn) z&tm4dwd21Kp(k;ODC0h%2BOGj2>We1UjIEUk3Hbq$UL1Xf%=8b2icd$zhO${GT6*@ znyIj4yKI7u`C~BCc8DyLX(fZMrqy`{Q0Kf|yZD=0J|hKRsCthjPtv z6rb`3;#yr{#M>Tl|J>s8epOW2FY|m^)>n_o`YZpH7piGD&52izjbzDO=3j~UiO`y# zLxk|e`kjlJ#dtsJ+v1mzv#h0rISb>h$uF|IXBtf8r}-}Vm-W|luTv~yj92UQ6id61jNA%>zwY%~HG*?2( zP`TgR!OTya5+!-{=ShI7MB3n}^AKee%W-sOaGGnAnPN#Dye$n9YRhPr|6Arvi@0hq z!>obU#J4pWT;h7M*v^X>a|3A?po|nwMxM$U5;;z!nU4;d_Ah z*`0Fo*Zlgc>bm&r5Uz+P2G!5D8~Qq7ox#H-cDj9Yq7_XHTJu0B4m_mRTi#RPBu;Jc zd68%S=c^xB#yr%a?xggATm`EySujlx__?-nt9=Qs&4qmtbjlnhgFTx1DSb3JH}=~f zT>$FU^L)kjnMJ49WmA+bVp^5(cADF8v3RnFB#?F{Z>ui0m(!s}Viu~o;{~1i17WTt z{$@tM4l*iB$m?i*sbP9Jw5(-42tc?YlH@EBK&&E*N1V}Ntt;;?eeKhCTx#D z{yC7^ayqfgAtkSAUu{HQ35$oU^)qDam3WRP&^n4~?Iiq(qgdN|1mcDbn;2Jpm{=Re zSi4u$_A>dR5qDNRNUMD2XT-IH`%RBm)^xYj4Bi%gm1l2dQnkIC@A}H1FOdi9LPE!iBu6z~&2-y%G1BK%eu5%=;pq_>)uT@NSEH{6eJWE`6M}P1TkmX@Mvwmep<8lzL^F(}zFnrIfZ ztYek+TjaU=)b%YV0kmouK(!54kbz*bySIyzi>*G{_a@rAEHUi=9MVm&ev)&nDx}Ha zs1o=mgN1Qv?`p(siEf4d{Ja2#qMHoq6+*7 zLd$YnbU+6mM-+@i=)IA*cq zYk)@W()WF)au4C78nWdhKCLv=WyHK&2p<>{;S7+*pof$-UXWi1dOFOXWC9LTIH_DBCxqccJeLU#e9G;_6XXgN)`YxqfW16y z4+;F!9W~+r;&v(4r>yQwtri7a?@(dHmxq>v3-4U&R+T%x>XxwY&G+Qd=62~zJ%zQV z_O;)&C&srtDdr#Z%Ul6Ve zcFz|C#Nm}Pej;K~VRu!9>YqBRbDE4n6!RBs4`Y1FavsH5@ZJ*{Sg2aLWys&BKFBw# zcX@N_13T+zuupo4JBA$j#CWVeZDwXS#;@#e%5*5#f9?}0 zau?qr{%#C{aKtW@Y8eOP={Gnvp4VBl(hXa~`aT?4yfgPktQNVM z##dE7LJrN`>${NXPc5q>Jt{K%jQ}~|vlyMfeA33aPm80IfRD`wImKz!qjO&3%AnoI zsHBUaJySNJUE(-o4VRe3Mm=~HOB>XhuM#9QM&dp16Ec1y{sHTwb9vGk0j#)9nyQZU zLm5Mk=I1;lD)w}46$8ZZC#?2KG9tqnb!`l{;VJ=nNdfEMMX@e<7O%1*iO)pS!fqd0 z;^vp$q^9pM5yIWBc!)>khNz0iBxw~6j7Fc;yk%0oNHNXX?Ls)LKmHGFe5skdne8=) z%H|So`;ZKHHnK9s8}HdFtz)Y$bMUXm z0ArfzR9to5(_|ghdXw$f0@Ib8ZZm|<|0;fUMuM@h>&DL}|GN4Ptc2No@QF>PecCz| zm81u@BmeNy!R0tSXqVwIM|fmDbp2Nwi$aKgS(v6)?w1GJ4BEbXw0r_6u)=Jb+~K}#nt2zWm7{=iIs>%R3@$KN zG>0?WCV4IUk9>y@dsVIq7cw6H33*q`)mrI;*GYYbYm{*kGK<-5b4Z>XHt>hMm)bYJN};ENR|(JZ6GwRgeanqdp zSHC#__x_eOu?AH+2`F^HGDqU$)#OO+2w0Op>j}L9Lg1pOC`t$+Y-Ei-%1y-rL2k3f zzEZE0eBQ@Iey=YiV14(qJWSGkQ1>KUV_~*C`TCti)v1tMn=AT!w55rB;DRnN!#_ z8B*4?I*GVvnN)9NnHqJj;CT_Q===Vburv^~mX8EtZjXR_-{(2C)AF|q5v`HjLd4g- zjE>&`D)3;=QuqikVb)77@1!TU`&OBT)srV+$Giy0w#Y!O*lwuxRqg$@Q|2il23cxo zi`)43MAZ2`V)V3W7(SbwM5lHqAmPE@DqhP5lX{3Z7vms|$nglbg)$qp`>Tl=w>@md zso5j^jri}ZU;J*4feZ-4(HmOb7Z`HdYAEX-Eu*zicnaFgeJjAq>J zc7Fq&`D>Xk^WyG`yQaHde)~0fc-WuCUEgI=#3xUwuuOzGhn8VA50X;DNDDd5&fnZ_}4Fr+4`ozFH`s_pe@z0rnDk^JzQ!8g3- zct73A1wK_`lZ8kG`_gxS4|RGFm=V{oVK1GQbF6_KuwZ{yj~ zG!y)KoptKaJ?gC^$3ZUPEH{XCn9c@3xNTI=s;k{)Te+8mZ17ro{wzD=KDtm9V<vYE@Y2{z=l2rvKWuxK+>c0C*3U-B#1;ohg&i%a^7Wdov@b+=ga%2@l*3JQf z>RRZg^754N9;deAhhyF+)5$m4j~ZE+Hr_a=aHE*!;>ms$i#lqR2;e?fdi!1uR~tpr zjm8wCZSNXhaPWd zaX&VFsp6n|o$!mKM`Z`xZ--L{IUVT;Qi+ESoFD5q2Z}Tg-Z*?1(tt1-RQwdDFt;T# zJ*Xu`B7O$UPSQC}eK*{?4&THDUqSE>z>=yf33npn?>%d17E68{yrEUQG*EXgdLgT& z?7)Qb{mZ=ZH-@n3f~wO|j+zPgAdWS=RzkQ27B{;Zwxx(eM|l3YU22Q}P4i{UkSnL# z-R^NT?&oKhE%5e>xG=Axqd(i^b9g%xH+s?Jf4wTx*6t6jko6~BZc(c!_N#cS zC&Tn#SGK77m0EUQ@DR3*96zGXuRuT@HeZ`kwm*Kp56u0l-&dHIr8u|-K$n0^jHJgT zb^1Og8vV?wpQIyelA)oq{#2L}sm{7A&-rIzp_`090GGB$=U;&}Sl-QJ4##N1q*#zE z_ryQ(s=xQqXQBSL0VJzN^Mo;#evQGQ58!?g=s>2R5y=U(SsKr@7e30U+g?Knh>bIN z2yAEq(a*L83}4AtZ@bQYtkgZ`bO%X>3p{4js95_`52(|72_!Z%B^ho~@55#U*O7f{ zqRawMzB2(+f@Ld)MHatkp#IRhV0%VbzF@+??obc0=ge+*?G1<&lh1F>-M8gM9g!f? zk>TCUXP}_fNP*WUGDnU0=pMHPj90h((xtz zEpd<01?)HTi1XFfdbud@UayP>zU>k)TxPD&5TJJtTGLbFri1^J*z2;PxJ98vBhhp0Fnd_NjI_r5!BzLS z-Z$fA-kugbIyJJ6_ufiY&NxL5qBO;JNow!^wI?ZW8VjzwoA|K?)75v~k@s=Yo2!=z zFn1!Rv4R!Dndq_Ek1n}((BFanuCE93!H>0!DTk5gmoy>BpjxH^hFZug= zdeh;JCde1fPykIT!>q;G>r;i%qf|R+xErw!DcFK!_E{D>NmD&s(tgqiS za_KtdQMp}cgy*p|Zx7IhIIavUC7P-Q`<(aJ6U5W+h}=0E08zvJqqxJ~4$`S{^_h!N z!cZk9d9`XPr#XUP0gU+RW_KI}P-0jB|C!M$(xj;v>X>U5c&@+9$rp9VJ@~kVx<|!H zJ&x5x#DXhp6vPpRc-n6HSvFNUoDkE_-XWAtaGnXa#J?kI6z*KnIMB|bpbLCL9gH^+ zEOHtLp(2Qe+t^|mjuAyV`~X7n;!0ylVo=a)LHLO>bThM%CAgg1=?xcbO3B~bC|KKW zzN=wAo{#)Up7>8o(VXVr*63v@>P>IPeO=^NA@W48#pDS13zSciY9L=_Y`E27c?QKS z)9jqZvkZNj4>bd!%ak5v^i>@udeh0AyWRWiI_d8u%Hr=5`AOUa``ru21x$`pHld4tiO;p_P;I- zl5a99%K@wm_8B(ZkM6XW_wBh?eX@DlF7S*|PCT|kSA_9Rb2`)PG**?MBT&;4`e_)d zf9vfpnTMyxdy^v9`uvTiXpwg&aV(5|k$MG*8FlQ0A^W=04l$x21n&XjGSj@A0n07{ zGmJP#l%r@gR7YY}HDc9=<@#RzBFCLSJ3X|&h3KWUaGovKvYv>?_iVmwYQGclNqgK~s%zVMn9gUQXV}|4jG_3)(2I22ve=#xVXGP-h8aX~KBkWu;02f?O{j z-S9W!!C+@kd4f$3eGe_%Pnvv4I2<}<-T~cPc z*W5n@hbrTrA@c_LLRZ?F>WF3|Q_neVK0(qz#@b!4J&OdP@W71kEEfvUj@*IQJ0nC; zD3N~a^Vsq&8;N1xl}=-#Oc>7>8H)*xvy<_zEPeaRFV1{n&joK@`mz;xwTup&|9O}> zYG#dbcO<%hZHiCw8;SzFUwpv&!HD^!>v`<&gP+woQ1d^TrP4EM06fhB%9r=^jb;kQ zV=?!Lu9eiBAP0~Ogs6|=lg^BLRI!BaOnCAULInFqDed~`q!%Ghu)Jwhh8jSsawe9C zc3;_N`%?ZQymzH77+&)Bm-;-9;^{iCR=YPs9N1EN zu;jcC4kwRfIG_9rg_lUQjzQSQWS~e6>r?ALg#(q}!>Yyq+)94%^Tk$PpZNNv*Y_|n zwj`J&rI-62IO%NHRe=30^GX47KtbGZ>xGqAyR27erL$=x&YxzV&CbFlnB^XqmHIJs zCCzQ)>f zwIN-DSN2Ic-!6-)X@M_>$0DTCCdE45J1o6j&TWEL5bltwccON>fE)R}P_1v3qan94 zaa0v;dIK84tzKF*HiHErThtXs%Y(x3nn~#(p((W8El|NXR=W7Gb63*3PO}V&IoA99 zGtSz@nwF>zO-%ynE>}qw-yks(!vP*KWaQiu<)-p2bkbO7Kl6Se^#m ztv`NAL?N5NB^OPV)!@iqwh3B$6WF!PI3Dhux`YDKlO{1%cyQm%x}(RS`xiq?&dBqo z`aHUWr(qkg-`_NXSU3*PI}qMq87d~IFsDBYO^P9pxMy}3?z~R4GEYqS(+y)*Nt8FK zTfDJ#KE&6-=K=Eg*On;pFT?kIjW!H?s269uHzgf=q<`|nB4!rV=w4TM8k*VzC(SKI0`unTqeKY zq5j{=HQY@^~8{#v)1uHd|(g3p+DIS2V7@pSn$8eu!ed^oG zrBk4~m>{YRp#%tm)A`a&qs(KbeJ_0Hd9@HAtCm^8=OK$iX-3i{oC0*p===o(t&`2;k>*L)`1czQPM9ICt8=!>rQQGPQ$pk7J!r zEwitz(~MLK)JFr1Kg>H>cfI9PzWA%2qhdCNdtIy%d$0CF)Db_xx;tn3R3-RgP3IJw zaXQ7Vo@OL~aqa3u!AIp;3S;&(-Lo6LJGmQy(PcU$-kMHY#xD)l)pt$?1c$=dG0hCw z`mmX@w~^Q9qo7xY&=hLVC|T$oXX^}AAAO$I8R!<^nRRfCE==oAkp*0|MpygB`r6L<&jf%n=k%; zm=5fRl>srz7f(#}j=mEkAWUU%c9?^(aFryHtP;o?*FbdI4>cydFc_E=@q+S3x(F-iL%UGtF_eA%r7osX5y7WRvNm5)jq z@boE$mv|BcWgRn(7h3OSw(IaHrcdJMDfX_Kk~a*Wy+CALPvthH}uiW&MMUcs9Ah zIKIj#{vTK88P0Ye|9d;sh|y9tLaWA2#i|*JS=?r;t!fc!-}WAnpmuAvX6;fs)ZQbp zirTAY?O9upkO(LLbIyZvorigrE7$e=<@^17-tX6IFo+xD1^V0Zi50yU!oNeXp@enn z^AM?<=O~Re+^R7J@4On48kEp`X2okhn{sijvn=(Dc$nGOVp73y?A@m*z9#S3g{ zPZe|at0irkf%SWN*PE(|wG7j?eS(L@e$36Ner@}WI~S3bI%Tz2x3apuFoa#A$+WIs z4lEw~lPT!w0ODBB!q#JT-3MlHdfP?t>Q5z1<2#UxEjbBn2~PD!_Gl7vgK-5EZfT=v zt$Bh-4BLM^nnGvBF|-6}#&qwBuy zKARt_uCJJskt!R-sTs{Hbwg;6JAX^tDxHgV<=pqOybvdg^H?-#XNbgudwTN-({BUwT+rRC>|H=MM1FZ&6nZB@sR-{P>tyAyJ5fgBF zW-`KL?N1W$MUkhd&@~H2hZ8WXSQ|dVo!{D;Z(C=p+PcW0tW)M&9=>)Pdo>mW2^oCY zcbdF5-?}~$94Yq&@P^Z{tzH%OVzuB747Q;_Lcn_l6IFO{^(@(?9dJRi?sVkIIwF~NFxw(G*YEjY2QospfLp=FpVY2u;64wySCVOV6r285;!5SSmjdS-eH&12VY|>| z+Qup)aM_bX0Y0;Zl065?4|ob0F*@%xmk*bNv$YXlK#y)<_ETT2#x=!`d(8v{9v z35me7!jV~#T|#f=n3QNSJ**4VqkyE`CE%upp9gOiXX80?Io3)ocm9?l#hTijp-SR z*kKv*g=)Th!SUq6>jz+PNN(xTt5}yyuf*`~S)FgKk6At$od(R6BSEh|OjU^KSiVRq z&)%IfbC-|Jb;+T|zl)#^d@xow%=eJESXz*4oEdGf z;|5c^Y+BUI`KG=Pn$gVC68A{ix598lH_IE$kNU?0=Bu?3!byQd6g~IbZkFD|N%fnG z=zWD`Tf*>+QLM!J$i>Z0tMwpwHIsKn|MOF$B@6q23-&L$@06eG83*mW1#braq+5T< zZ#7l2_v(Gr9PaNeW2}wYsQ(KtlAyO;M7`sJmoEv#q1frj}P;*0)9PAWO}9I&5YM4U=#INb%XMYrY<%e&ATh8p(y zt#S+secYI8aIo=16I&<*TBE!=Lg!u$Bt32k$a1;FQ~*U%puP8xFD_deGD6Y;vTlHl z+|AESXT%w9*JCNx|D%c%3?3 zG%rN<<%dpHJ?bA$Po$4_aeaZuqr#Pa!AD}1Z3kUT6d^!(0jixEY5@reCq%4fpl~?H ztKXp)xC+vS#K$XR3?9dIJ#;<4KVOr0zDo-_XPmsA4fQ9V-e@?Q-icR&``Z_PHs(Qy zLt&S>Ry6%JU#KNtSUsh?YO0jEDZ!41_eOo@DUvuR)fn#xJUzVCXIo{csIGL{ESf7n zc+P?q3YXAJK75_wxs&XHHBLVGWbA3kC$Ng(X>MQKe~XSF{HxR#(+kyVG6zXb;2?J5pyVW1-B**e4$a#u&V(nONxcB@ z&kCQe_fDX8u9?&2-pj+}`cuP}orb>$id_pxXncm#kTc zrMgR3uGN`A^$#K`UDI&6LLN{`L;~SXo9hd8M+8%4%HN)07J2FOx6Iw|qVEPf17l^j zKibb$uK%P13mHie|LPoeCpMTyfeQ&Qn=X83?5EMOFqdZF+dN44yzu7d0?pt5tCidj`z)`)=c(}vE2hDQd)KuLV8fb~B!H(kBQhWAH)d0aoX(vbfbDJa?Ol_?{evwhp#D=fsy zdJOZYSQa!t&my=wqe>d8n7bf|6&4t_2FS*YAY|-+MU;>kXE-G)=)B=z{T~ujtUiMO zauH607eHqYP9y6ElChIssBnVz8AY+)>jpjZPbb!~%KP>mWz+>MgJzR!Z$ta3c0OOt zNvkhDQ&vmF8LFmd6^2cpMO3=&b4{uv``SDVjI$l`{Mil);)UYj8{u5PsJ9nS{JA-f zo;XyDuxzlea?Lw!s+$X+xqKua@8s}4R&-J&xd6Ej2*UTjBl)`I3+?g>xzj{W%onQwX_jkVlg0T!h3eB#(oNIH%MkqfBpmx^Ag#08>{o| zL$iES5p+#&i?AC+bv$%opXzMA`Cm)I@gvwL%k{s&;pbM<^(z}Dy-oFo4!|3qnb5`M z$puV)Z8gH$%XhaD7+$#9j$*5BtqQR1x=aauLK)m4%~jW2vS2gyGheAq$8|y99?09? zJKLqavqSFUanAWYcW^>0-W)XN&9n>c2x@rC}qSpmc~6EGdk(P zSIF1b)(lh*NO4E2p`q;YY%C|(kOeSZ3DX(fX9D%vDu&+!D$ws9 zcM<%1G=z@YtN^&`mwSb~lj1?+G-$1<)4%IAOQJx&vLHdyqs7#!-Pwxp{l3QF`y{4L zeApJ*K@gYJW=6kBmlw3mv8U&me@`eBiuW9P83~SO2h{Sk@_X(zJN^ZHT?N7KpDehD)Vi>^fDH z@WI%;(Xg#y)~jzzkId2XaZ8(KA`9ZnqZiXMp&<`Q6mUwr^U}^-k6mJ_<4`9h-zM&& zZnXa`?}5@A#AuC0P2LLKBmo@6r&Fvs8M;ZUD#dp9=0ws&z#F*l4?_T=j`7MFFkQ1RA_3dj=Blh%p=dr!@TWgB&i<95^Qt!K zbYaALuF|&ugGYaMZI|Cah1MgPj{X|z9AjqYEp6onne07_`09_|lp59mTf%M~$;uum_AAJ|0Md9}Qf7l=ZK6?6>&`5B@#Y zL}u(tIr9&$az}&yJ952u#N{G&P*I!VCT~NcY*r4{@t9ug16>T7muWN5`rPv2JUgP; z&H=@(fp){u<>DFFg-u4TDqSDsfRTui!m^REIn|BdS2O;&Y{V#<>k8Lo!(+Wy`Di0X z`QAm2G4EC+mGqB$HdniWF2jQJU1>XFaJw_B_i{*~vaT`!$~tG6qxTI|tsEL z(kRSvYNG4x&vz%=FIV)NR?+gc{3$w1?dSyuSFsJ>kkuvOUMyO@NuJzVgK~ z=T@K*s9j+mZmyBd*Yb)L-VZe8t#&89vuUX7k{tQ?mefwda}d5WaAnwFmS$ z!_05|#>@+RbNtIudtGy$VB!fnyhE0x_`b$q1`(94Wl@gY6QVlsr9&5M3=XYru4l4H ztQ}=>+G=27#g~xKhrtOZ0y;d+I$@V-rI4%YEkp?0PLjn3lp|Ag=wayK#VNzMN}ldh z{#v`7eFP0HDf0Lc;>HCfCCTO!^5QPD)Ri)44APABxjs^~_dJ5odP^XJh+T{KAz1J8 zKvmuD$X8x7fG$(*F>X`g!Jk6!`yt!myKBh1(5LB^Xmm>8K zOxs80bJa4E0n;aDmT;rRS*AGKLtb*~px$A4s&`@}-nQH^)JF12Vo(ieCcS6t&rh43 zwuKv+Tps(c>~$UZBh%kfaTF$IohM^>??U?n<3J3+dz`fAY@rywyDYq*#OuWGHu+ci zoQC*)m+l6jpEvy$*MtSO-iw8UqlPG_B){!`smkI(;O{HyKU0k<{=Dj?mXX`8}iX zB&s?z`oFHY|1ZuKe)MyG>$;r`kh|D|w~)H8110LMNb{I|+&^V9fk2E?g1~`n{QZW^H)p9AL+DX)z1;o%i%$Mtl2ET?Y#)A#{40v7~mzxo+-lqT^ z*mkTOxM$36wdDAL(Q9Kc_=S6VTQviyspfBencg(Cv4aKX|Y?# za;|kwT_VHSbzaQ!x;aR_?TF(lBNCcxJBn6sY~U6iGf;i8nA%l>rY{W@t^wq0zUeCC z#jXEr*e|OP8+b>}@^PC>o{I3A23E|AWZ@CP1)IQnZik|N0N!cT^YCykL^3srsn1kQ zO0>DsWdU@3P9l8wZn{1XA#Kd>CnWQ{tmPpqxo)YtvJ7SLtY6fL+7N5JerU8-B22e* zN$m})tAq0kU~D4s{7I!=pu2;l5>+T(oTCdbG75@lc`P3P;AqhswB5^@(D!YzS8+$>Mfb31hE$uz=3f`6SBrKk< zpd_#U>!ljW6f9+!Jy=?Xezv$LGDVi{9t(tjb?Kjw#9A^MSQVFk4f+R0A*ub;&KliB3q(WEsc=TBc-MRt` zEpz6>{k^49FtwdYFJS+ykvi!Jm(`BVL<||?Z>JI zFO1uA9BJ~3I%<;?ktMqeX>^Ui(}E{oI>u>tCXR?yJSGbC^EnE86Wioo;Ijo;AxriN zM>SN~gT8!}U_L@XO9{Jq-i{%E5?+)}t4=bd`?ywwgpe71E8Ry7U*MfO^h7I~S7v^1 zt-u&dO5*+SwsPn4OD1#jS~U#nOa-;`1cjx>@DiA78-~hFzl4@_81vmmRCrhD!u*Ch zcu%~2HR2bp7`E5neXrBbw`uSa8}7O(c&Higg@F!bmLL!Qn6e*seD`Da6a;+90-kqt z&lmc@q+4-j1a8{|lbH%vfLH$YPlM?s)1JZJ>-Uu&o?!%zsx6LVrM258I+u)NEX8Z$ z9qPtY$P>V;qiR6p5(?@Ai21Vhr@2A(gf>P@RV!ieqDZ-)?6QngxR)I;tg_64hGqEpw zXXNY+_GDsZ^G1zTaVU+CLhCveC&l*b&xR#|h^qQQ)Xdt4!m~l@XOot0#bjy;s!=(CJU-l%IG3$C;Q@ zN>m{#U~0s~E*R{5o1M_^Ut+asI{Z-VxFqUhx=YB<%e_C$%i=#=3Q!t5t}0ZYE*oQ1 z$CXND=(x?|?{8)+4XgKB#6DyLz1KA{IV4X7;(3r1eL7(D+@aWHgqPICe8w6N;TWR( zAZN`n-_BrEM6amA7&UV`1W9E|{+U#>9f5AEX1~|=OaDhF>0Erh-wj0>&7*6POz$1IE0bQ~4N{6M)c;`LDDIl?Bucn( zb>rpE%jU~LPD_(*r9-V^G0g||0eo27E}@kZzzU5_#VPfrHeY}xnPj9-M&c&BwHP92 zy7_MeSODieH5xjxmeqBaiFl z{TTGs!O$rWoIEFMy56sdz5+HZk%-q<^A&J!RNe9{*Y)`R2@LnU2eZ}?dbrGIRKCLLJ`KJHRC!jTcch>bwayx#8-bSb>T^a-3rQR;aGZ-%CxOZ>Gh|y1WT3Rk zrm=aCe6Z!;IPeaGV4Vl@N?UUQebi1Wx_PR6pArh4WwB=}8~$$s`k$kNKA1g`Ox8hi zl&GgABy2OB39^;k!m8co7~%R>04!nnp5d7yfNW?3H+(SS6~zY;u&>O|eUfVd4_5S* zyXmCi1qVI4<%RAJQ2;*G?I*Utpi;JIfn#@}PYTCb&x8u=gdQZxPB=MtKe3|2_gNaWFr+75`){w-WB8ki*C!SJkQw)X|`t5 z#7p(G?CzbHlUD}(kT}iyzr}pD??~^V^2BEq9G`B2`jS|Z*HtFH3+Gr@VXMiow_$rH z6Gr771`)09FJm)qyCzmsZbpk-z1im3auBv>Ndb6|p)y*1sCoMR;FnQ*p--Y0_xJlY zKG(u6iz@2jYK&6=MgEcn4gQ@v{5|X+6R*}j##zao_wD81_gOJ79IQ+ou&*AO6D-=W ztkVt(`>~As>M<_=_)+aNdCawe40-7HySl^uQgmhlsv;EaNU2Ee_F#(fGTCDC{T>qe zK{xxtRQ@XSq7yO?8uyQN2~5>7-+zK`Y>xJC5lN}%_^Xz$t6`2k@NT#A7qM14GaQ6C zTOG;}ZVk=+EX+bZc!1jNXoye-7Gr)p#5w;eG?7HiK zk-VF2{rT&MO22=F?hc=+%W{73f7WxiS9|(qw`r`SE`e^;qX5M)#+mqIywP1Hs&t=c z+>D@q{aT!qhx$hev@`v^di~JWVKd`(OrJd|F8>&bjUDU zz(1u-c$S`f-8F??c^T*LLx_L0o26I2(g_E8k_&sASz38~9_SWM)HuXDbF!Te^c#eTkODyr^K2>zHL0Mu`tsAii=|UJqNlXv8JSZIE$;3S_lfQGsXw_a;?)KWn1SYN7hi57Pv9 z7P_2|&A06AR^L4oCwh@FxpANoI zNh@{K{=ZPam}M7Ooxu&(DI>`)()W&b6gU4tP>dO#JWH zhGij0OYAee2|Xq6+^PI>=^d28E%$&mLwGS~1{osglEp(gH>5EVZ zdACniP0i>+@l;DEeJ+?93VkaU@AL>gYSNJ%}Fknro#@Jk!mm{+3O7N^hng%U4(o?;>*>8{y@uF;`n| z$W{q@i79s@Iyc5Pu@U_rCVWtryAyZZXPcw=T(^shV9SoW+dWNKsI#7z2uhFAjc_%k zAM`!xF6S{=L_$;oSk21qOaAb!i@9L4O2Ga~b9Edu7t9qqa+LXJO%?q34Pygn0d(Sl zf*$`xf%$XMfPOa9#Azzn`{B7#0PaMhaa$I%g_kHH6M4_Tp%;oCxbL@Wa-)zotD4Ki z2l<3avhseyUIq1M>`N95{zZBUH>v?8GG{X1EyiJ|Ow3gfj5g8=$I$cg%7S{UH|!*?>NrRWJd zFZ&Du5Bxo~be;q9^`Es>Mf7`)p6OO<5@gq~xq9OsxuY?j=A2ib1vDHC5dhjXL#$jF z+tFFz$5!w?>|hNvePzx+O%%9ZXtYzjcc8P{FYoa_gk5cum+@%dL-Irv?&4;9snZXqeOykNn9Tb;uuVHQV0j{FS zn|Gzlzj$VfOlyoOMTFf_(e!1~2qepF?}*q%6lFhs)-2lYF0W7Myh{Vj7PSaCVZgYq zm)uGnFa;q^mUMBwV2r${F`im+!Mo$zZj4 zr^9}3<>45V6*+jdeg+nv1c-X1TxJxb1J#RP?;AVjhdE0N#K2S^>7pTXiW5Svd8kLc zJocaas@#0cU;Dh}ddRv>mzpIw_kp?6$JFJXe4nuWzkgT$HWa;sB`~e{5RPljZ|CE9 z^6j$KET}^gG2D355*giHMUZlu!r35B7CN!4jH;8JGRFb{%-D#rxo=wE``ldW3)ppp?6YZZ&MDkM(Y zDa6BPKwNY0SPxvCCfAvJT0;=&TiVE~&qyn`=I%et^iD9I3sOA&x#Em2+bksa0GTuI zT~=wqS=IECvPcAac+`ftOREzeH40ACdh~k(wi#$8kCnMSixkU}v9y(VKixFT6-WkP zOgpm7XA&kV8k0`OGg}uN5p6#W6g&vL8i2Kq3i0p8$zbzGLFd2aWsH;|YEa$GhBatV zEe}~Bn!DQ~#Mkg-JvtMU+ah@R?{l`E9xkv8U@i+IH;%wkDgP!rl(!&&iguO(&GV#D z<%djuWaH7Y^7X&v3g(BNFXRpXK!1`3p#SEeCaHaKs-34%i(5y3+38_VXT{xUMZ#~Y z5CYf*f~qFazv{dKH2X|li!f+_=3mcWg>;TcAfXxIUtObS9D=tb`&Tj6L z`0F-Qd+>`2bA^ktmY{yMJfBFvw54k6^oP(xKr!XI9#)WpZ<11#byQegIYcqM?FNg< z#GzDIQ(REN2v|IU7&cPwXC<>8u#aeRfT)!2PZD)qyJ+aMhS_ROH-sX5>5;jm@;kTb zHFqP1#MUTxF3JF>48@0~aDhwhJQ+dtiyiZo;iAJRH{}V@S1KCuTN&w+q@M8mdAD{o zZzS_u(dQpWVd}e5{7y-eF-lm7+QJ*Lno}cEca{!@tux>%?c4Cgwz>WM@H@rgaq3CI zkp7y@GM%eu&*$G%r+Esjs%+AiRtP)X@gj*I3F58rxlSTGUK{X2-rgH)A8K<&UFJE= z($79D(_#(AqxC)nWcYvtV+fDwvXS!ljoW1lS@!Z>IAXmQ%?u*AzL+MvqsioVr}W2* z-Kh#>yHV9o>Fpf7X5}aQi3}_``#%3p)0|5m-&Darh7V^hQ#?F*3wXB|%S(kKJ|+Nn z<<@(R+hU@X^PWW4l_4{(mGkcA98>h2aDUs-gc_?K1yl(41mtD~_PWyYx2A_HKYwo| zzc%9MH=%j-gv<4NURpWdTzj27MW-b=QsTVD%2-(|c=}YZd_kuKa*Wbb(Xt$FmFm<- z>K%rty^JOLOs}2GOrKbT_h;JVv6?v9^Jb4*>Gj&<&c?BC&e<`afEv%YNY8*_mg{Ug z_x5Tfk5({?#Ecx=XOrnqUu`kP&A%IT-=@C7q!~r%+DLKnCNx z3ztI4B_I(>>{jDxo^~G@{g5*)e!;=X**7$O|ILop@}Ty%%|61R=*U6%m#{^qV=gu{8I`YHy8teI zFqcJT|7^H2#!uEZksyE>*_#abturq;|CIPclR+p)kC0b=`T!`ek@^FiVOcN&I}&mY z4y;WhDo>X#u3?47=R!F_k@1NM_rF6@o@E{J{c;5D(W(bN-gW|BpD zzT^(SEYwH|XwuO3gtlt?1Sg~+*aCn)}U^e}`;$g@E zBiXl#1B7~7J{XM>;O7i7@A_m>S!pMm_VcUfW6eOnB8~KS4>UVocLV4I^RWj5rYsQ2 zU_HEcTT?E+pxGXqZL6smGZy&*TqOfhC2C#UDy+CBsMia!wEOZs|jb zp&Dk}RJXEy&69p#Z23gcwgGriUlUwbME**tw~h?QbyP;Bk#x%qaE3F@)bN7|k;(5Y zy4yM$&Kjf^NLVOxMdBZi52j7oxnK>DmEca6OEeraUSpkFSzUPVYk`GAW_s?-wcP$l ze(dUqAMBT+we6!*b!9Elz~qflD#5=Co-eR!MVeyM3$C#t{Z&Du%>7aBOb@D6g&Xrw z(A`_rxq=;xWh7(c3C>Ffqt3RWcGoSd7)9Nl{OP;Qh|(cmy1;BNHuG9(+Z73(*;Wgi zqFluSiEIj2v}fPDW4SG(%K2h2>%0A=arv6Ob7uZ`W97Wo2zG4Lz$F#)z7{bbp2?Dr zYf_>T8>w~22`vuAQevA^kkAKbZA2Vse}vBj{!@K zf3Ax#wkR4bMo7#^-Bz4D8k(7=K4zSt>zb><{t1y`o-tcq+A6F8X(`}u^~rVe`!oJ1 z?%&Vvi|57ezoS|}gr)$UDsWi}8>~m{)H?%}ea*5ZYrxA#GlmsMPE^S0E_MB%Em@S# zjy#fqNkTVsIo1#w=WV4o1~?9|R;ke}z{D)$wzlcGzg)O2*?2W2D6lh#{oZQi~ zm@Q+{ASPTyPTDTItnAJJ4QF4^c}J7@apmpTuS3C`e}I;!=sovYecMz^si@<4LS>dM}Fy|lL_E~l16f@YAdQ;>+6Ny2(Cs>AjoGs_uyGsnGw!H`92RFq^DU< zhPmv)}j zn9P9sqXT(@=aPJel*16W!8J4vRgW@vn~S>mBQx8|NjB=8roI1+FoHuvdX=9qQxj?7 z2_H#e1^AcSy2`$HlNdPwx`)k&jS@!{Cm>{M)2o)_L)5hz=BiY}-t}JlTz40e7ZM>y zi%{b2&G!j#Z?~Fau>?jpz=1ayK3SgdW@z^|XuSG*?}n}eOZchJtlL0$5qbprMrr-` z%vol$Rv}nI8>w7QR_NpjA1zUC^0KH@Cf+J5yaASZUSjDj`=?Pj?g(OBV}hW7Fp-?%Mp-Fav6sZ`=^*J~DO zFy_|BlM|gR{WV98LL=Sm0E6tXOM=^Lf?8Im%VmMV`EtV`OJG}rajk`~7x+!)2o3TU zZn*1JuBZruUwbJAPUBk2M01FKKD!Y?j)V{>0)JTOny{$0490P7Ft~YMy@}kH-*rc% zoc=e(nolonOnJan0ad;Fqq^oADz6GB=kDWerg;{4;^MU!341{v7O3!Rb6;=k{w^I= zf$9R5Z5HnYmVQ~75^xg3@D=xrM@pfFU*sl%sXG70M1rv=@^eI^P_iEAwd(nQ=YqeN z!Bek2CVEXvVkF*!t%6W3NbFFgGhhWxfg-*<(1u+r`X%mBwP>8%WqJ~FRGByZ$xUDb zSk0r5W;8`Vl5cLKjwHzbNEq)nm2~N}qTPu7+8%l`8`qlU5t_IE5Xo`~Eg1~B9P{UB zVL5k$Zx|*+{n#A`$1vr4E!BFc(?*Q+nZbpMXw7sHDITGAsthpTo_fqv)Nj&{DL9TZHOP&ntf#?3m@xg_u>C$LQ2W|g7vhWzH+D!h17O&-rDR$@~DD z2>&Y7-1J)Up>6YST%RBx$cuj9>=q$&DaBURNb6BP&l;$ofv^bH;>X(%3AyNMLJ2Zb z0vFyAkPg|AMUEn+U3Jmc3;lOuiE$(~kV%8iUurkFUm0sQ0`^iUQ(=DUt4`?5ogGS* zWSQ;nW%py2bGJ9zZYlgr*yqik{I%sj)+pm$rIj1oh=&Ah-y>~2Y#(XM5ZT|DAqn|e zyU~2{z?|R7t2t07r6vmzC%X6G6sVmaAV>b0fvzPe_i9r*Vtg-j`E;!$-+!->ca6&= z?^qpFl;9-ZL5fl3Ky8Z!E~0ozNSqNI`yESxL?(VhCKcl%a22eiL`U!**O|Jj=A(SX z=)+THQ!)U|`-r+eK$4!puFw&!aCCI9(9QStUy-&EWibVkrmsgs$q(I)&pA%WWc8O> zEXY%{Sk%(7bbLF?wJYFF;JiYs{vZRv8TOjvKmO|IR;xr8ily8u7&c`KxkRK}OF(?Y z*OB;l0<&zR9SVph%sVmrR0~VG@F0=M)`OxX!reQVC25}7nIbSVW&DH5?{D;h`}668 z`x@Dse`rROL>3>TmiKu2=GOfV3{*D>S?I5jQ{QXn43%;+Lm`C8Mt0;nfW>5ugIvt) z;aE$Bt|FuY-%8>5?6G^AHE^3GywT%q&Fxk-s=n zFQ*OILq0c&Zkjx{SfnD)s#val`g9!HJhQ-3Zc`~36Q za-!IP|CT$9O9fqCy?ig82Uo`a7FBT1J`fPg8}bcey%fJq{mbQ*M$;ZpXoUQ(WzTd| zvw&rnj|d=@=fk~%>J(h(+5UY^rv79#7ciVuc-Enw_ohist0kMD$>K$8L{ULT01@S z!fcdtpgl^wcZ$bxjbM-F2-?3FqGiG z={#XjvzXqr%2iYTdTZ>EV1%Fz#>v?)C$&|GA)? zc#+9<7u1r%>`ZM>yl-q*E;m;PUo~unE9sF(E5DZJtpwKCPM*d_h!)7-M@Ji}x1c}9 zkWa33s@XL@A0n6~%4gR-W#2(5oj?k@n+y9cMCluIIQNfn5)gB z|0(i*o`yT*wO$=8rtpu3P?Y80Qt56G<+=ZKUh^WBeB=YOxP&k!s1#i1-g9MK?IgJR7Wp3n=Hw@;enKN%^{jdPGarQwKUp8e`4nXj z!Q@NmI~D=wuV-pl-Ne5DU{~#udzn{F|L#&&Z#1~`MmyxzrpeSt2%w2-$vv6)&*h4{ z&>b0)%tS|;Rp90y&u4U`df62K*+G=QiM(ZC8K&qhc>2Y74@Vg!Fph72x#(RZf$UEZ zwF$B}AgSe}Oq}K=dv{fh;nJl=?;bK-vDVK14haT*0qjSXnTu?gaA5^(=E_7aaa<=N zWq^zNxEbV)Wp)0LzB%H!WK8wWeZyBYe_MvSMaT=w;88Y|-A}G11JP}ESAx%smQ)`3 zj~<)_*CV#RCjxyKty%tV`t@DVc)>@`cnWpMWP?CBzkC;2dD5Z!)w!?l_O)&wg2ola zkVxv6u=TLzo+nJ0(TkiG6+&;3?*O5lC<+GB+oZo0CpvQ0JGpl>V@HEK&kP-EuKK~+#6Y;gKV09Io@-Vb%+6NR7i@0g^ zC5DQ<8A{p8WJ8+CxGNU%qJ}NJWmCVqJfKj@dkJg*c3W96D6%tnMgPT?Dtpi$P>cIZ z=kwCtlF2BLdX=OdQshha6;`eK%uh9O(A>C}X+P{T#joeEctZm|0Zn$ZY^!2cJ`1&y+c2JD{t4P`_cQ1ZoGnme`n@{ zUx)gzNF{i+-)*RWooK8^oO#|Z-PbmXlLcDTeclix_C*H0QS{#DIWlh14fum151Q*+ zkwt5^X@dHo(x5rmmzwQD?+{M*of^r}+VJyMI>PI8wz`M=x?jrh9{|I__vKjD^OYx! zzrcAFj$}qodv6{8bO&v0Lg<%$&WJ5FiT$widyv&8#lymO%pm4`H=!5cwM27sW06TI;LR|e z!C4mgh5k~%MXG^*eL|S`hHXVu-Hz05E0{Gsy1ZW9?wD7yZwaYlgSkZ*ML|dm$v-m% zZkpEHJ2Y`)s%@vkRTJ;edE}e5OHhMGp0WDxyVw&u$ZyS^ncc33#Sh5*#y5@1{!5m> zxs!p1we|q}-wHNa41U8(2BQvTD-5&%4f089s@ooH&2pqRmZJ}m>Q+ti>x8ua4Sz7B zWJd~w=vam;t^bG@jPvy+h-uCrpt%WThX>FliAU>%dNP@fP&6atWG2i^dBD>W`uOfoafpYg>CxAriU z349jm&sY2KFpl4`>~+$YGDChCdbM{pa$mLxdTa2Z7H@GSqL<`yJ0)o;vR?g@b1j2& zzqtwR-J@{1jtztVR5apDQE%;-8Z&lpTN@B^)>XY1W0rmPVjI`f%O?YjW{j`LB?FZp zLpq4$nx&*|*}=!MjP^`;dh+xEp8wyq&sAp$i~7 z)0InvAo$*`uXzwST#l81V-A;Uduz@v;f-%CyzfYmaY`P2%Sx+5Ge^F&K$J9z$Tj{b zB>d&kvltbtF+@_E*<*Ls3+vLm8i6nd21r?j?6d?Q6<%3#Sg2FDaqKJNW ziPOv16~QYfHeuRIa(=4{^7EwO%Lyr^o-atWznxE_U~hccEuZzRF8@w8IL+ zI^G(nEsTLG^ka&kPh7nwPRd-a*6jV^PBBL?RQGjFZd>{s^j)RMCLeHxAr5~hdpeV~ z#$&SQoL%!vsKTje4Pihc094`z0-g;N=#iQ0MvdiyENw@Aj0)1Tif@{2cy_oPo?9t| zypDiIbQ=_r#PoSBW%E(v%ta(|4$Ms=1{2|=9EL!a>#KuQ6WTm zl@vzKIVI+}134Bsm-8t?CFGd%v7F6$&S!=>j%{o<-@QLSe1F;>@Vs1hdED>!+wFQy zao&3h?Q&ssJp_Br&-H;L@s^+l)44vDFJYa^5oCyV^XiOujdA=MjYX|PlyO5v61168 zdsqqmCmbc8{hvKo@R;IsP+J;DPN^UJ_LUDeG`&Y62`+`(4w<55_#tI&r>P4vQ3@5p z8XRu#B`cY&R#L*Zgb3PQzd=s8(ZCxzfqI2AJ1jP7<;4fZrw#M+9)t%?DABvvZykZz zRslJG%E$$}GT73aVJXnMhk9vbcG&$pv3Ba)W_spkZzI)bX=f3*?5>N=F0A96eF6=A zD8Z!`9~pbqmJq8Qe%g|0&nx3jZtro$jelV$gD6g;Ym}GSn^7BH2TYKH;&KgR$*h%S zs{vxpz|Zc{@d}Hanam!(M~9lDfg5GN&w6RvWeg3hqNVzzA|Rw*3pIsx0b;fjw_DIRfm*p6Pw*CrCwO^QRKzxj3=GAKJ@nE_UxTFnbN z2%I-o4Xt{6@(ht}*gqO;2@x>f3{~XgKU0fI$%wHO2aS5+I}NeYA)Jh9cPo8xjXM1I zayEASSy}_P&!4Q->w#ynW~=)lBptp2BN%hfrU$#DFXBeFBaG|*3PBLxA^K0`^4J3#9aN0GGGZ<`SAf=5%*85u3Ye{fur3f05l{5tzW1N%k9FGTylt8q@gn@Wy)MR9RBg@Ol(w1#&v&>v)=! zZSvw@gG{WFzQE@j@eY&z#!{Ru;*qv|cGuvRJSQdiNYj7brDHY8&rU<_MYepnD%`(> z6l7d814`^l9G#^kY2H(xSTz|}ATTWz)MN52w%@Pd=U$&<-S#AHKREW?s(QfXWrX#u z)a?31|MQ8z=M zeQfTpLLMF~|p{GzhDePBYq_0$gLp?YP-VrOQt{60Gar&p}8qMrl`8rZG(ns1c5X(Ud>ZC z6cR%|Qu}Y3ua50wX~$vjSqAI-$dIL8P|tjDMJ2!A!t7-hF%PqqkmB5vm*EC5OeE3o zUl+w%C$l;DM|vNZ1cB`h3K)k!V3*XWihpZT0w;0!_n!=)YsU<0rf0rzCj zs8q}iu8G-1LZG&FEw1h;?1oT$ z>E>&Y`oFhBSj*FAMA^nL9XpMXL z>O({5tOQr>6ZYEpxrqI^rru_+#PjVhv8%o%rui^~=(>NC>ozQrA*SCdk$@+hOF5Te z1qeY@WP2w9&T^Ww2EJ3#rF?0Ry|3j5ca?C#lU9|b{DRzDeU zV#T@*7TuqRPE>lf^KP1@Y1@DRqJyV7{<-dmV(%h>ic05yO6mD-*i31@a$uSH-wW53 zX>Mpc(hcWGqbDm1&`rZ@%i8FVjdUD*We|l3Rv!vU(1lzUb3GAGi7h4tcZ_jXA$6w2 znh~MVZfgy+pr4^L`cr^ylbsUa3*QfjwHh_-Bzfic<2Y$m6xy}~sp406`< z-gGP02y3EZv+eak!}u`FPx?oUn$zcyH3y|TmOp^np|3H`JuLZwZkhW$5U;rm`mYiT zOoPlTL)gv01a+xcpC7gf>XNbQ4zQ=U@7(SNd1~6!I_IT$%Dc7xn`{=z1>5gbrzpew zDl;7*CZ{_f9r$-xT=R58;vIKQAH&JO`Jjd@XEPEX+lq|dVdVNF7*RSP~vh7QBa}aD$qU0yNp&UjW|1a#YDU zDGq=0euSv%rCxqlQLYxP$o!Z-P(!p0OJ=e*QocNrU#9r3A9Lr_{bsFtU?|a%K^VU! z?~#f+d@;Xy&e z3ekSx>wyU?com{W)fvB>X7c6TjWTS=(d|5ZT2f)Arqf-lXh(h4MYvIYPp~YIIRVoU zIyVvkyWm4D%ch@3ZP@mU-^zvL!j?x%vUBWTEidM>uO>O7b70JFvO@Y1R?^_nTR&o( z<@kO)W~X!Gq5m#G-dQ?ZF1Tq#tgszj6=1J2c<<;@k!M2C5pn(H%kaikY+p*F*c|RL z!Be-REU19{sVNL!`usARz_0?VuUFCFC*D7_f9G=eVf{{$2SR^-=s6yc@MC+`XO0@rSE4jV(71rlJ(c}$m&C4uM zBUPFSABWP=F)W*Z!Gr3zQgo*tPfP=awQ3zKT~Etn|K=|cg^&2M$f?|3S2w`6;>L8+;@{<&c@k%C z$lde#kGFCJ5ndMT|G;*@_fy8&l#eDrCPWZu3lK`!9MP{3@-cCjTCL7ryPh6YT{D+^ zf^LlR!#0RPVx8nYU92%nCBH=I5Hs&wb?@c$MhE zn}Vgs@o(8jzA7|-m`O0GpeLdUPKwpqE5=@6~9J|$9MAgXUu}qIJNRzzZw%>h|2Otw zt?+9@QJmm$Rd8bsPKNBWL}R)qEr(OrJLOo*qWj4^moeG*SAF*qwzJHns=JGa3@WkL z=Ujr~y?ca05xB03zOPdmIm;0GzQ~m#&0dJx$~A7_AAlyHY7N#M;AwNJ7tb5S|M3U? z#o55Y?hNs=PySuhlQCMDKX223b9Wx*z}61sG|~L%>KpLJf93eBq*i5G%&+br8A?cU zTchz{OeT)E+DHd1zUMmW81{F*s{1Aji1-e|s$T*aeU%wfklk|gd7r%GWb@8UuO6(U z!b0X82~|#*4$rMNOAy(b-DQqm49}fPRw!(a(+B6;M$_B?<%L~CK0h0Cs>f~uuJY}T zSq-!6UN6@?G#IgYh1QC)6rBz+S*p8)!sU7)!YPRI6_==8bG_c?3Xe&kTBR}2l{TI+ zTo)nD^2YZ9kXQh^B8#@qq~yfNMso;aCqh9B54i4I*3Q4Pt_f`J2KTn5D7i9D9-w#E zCGz{Wq>Kz4aMK;Bjt!N6?l6r#^nD57Xyn8XGG4+PPo6}fG5vz6nV~*Su zF9efl(n`rl(z9`k8c;rX+aSZc>-aO6dVM>l+41y!t-^EjgsBLAwWh1$CweRRna<(I z+~v{+>2IRE7_@Ky;#09dNq(L0>@v3#lw3pR_qA)sWk!z!Chr6{HfglGPxJeSPG}2M z$n`9DsNtj&lXpCuO_XT!R~_X{%Wi49}1)Uwu(s-RGk#&sR1! z`dF(R)Il$1!=J|HyUN2uAqSfC6$*^)lyDB0<^pY>Q8i`;9KHR&-)6B46{Ra@hR;St zAs5f~E8V%mgKK9F1s_|CZmoR1{~pU4{Z6kL5NPgX9MO_$e7#O5ipB*3kMGkq0sVv) zr91u(>SfU=ZVT8kue3*v%_cuXp+#MEUP;(pWAY5T4I7skF%iDAyqOFr5)gY_|F~Yi zd6R`3(d3jK_40_w8kenvl-yA952Xw-iv|`LXd!tOs$$>yUYR9GohErjs<|4bEOknr zb?!bi!{L&*#rU(D1{30HyZW&ZL-&plJGu~n(e9%JW|qAGgBvK^manuS(d8dnA7;YD z)+PVbR=HL0oav{c@%%c1a2TwdKRvVPR$z-O> z?n%b7otXO)=XCeHS)TakT0ECbKi}|n-(K^+6-_`%PkWl3`X_oev|v)wG5w#et4Tbz zIz%i_fbW^0wpJ17y_LG2zt>1UO$7189H!l3vTjCO{FCt?P@);+)h_TQ?dvV>Ro|3{ zFJQMP6ZvT|aILwO-!q8#@uo{zNJzXKl(KeCgYkPE`6nC5l)=^!#U#VNk zxJKBl{>pg=66YuA53A@=qLm%47Sp zzi6Rteo!kp+mKocX4*a6zuXGK$VGvP78`fm&Gl0ggD4KLM1Dh*zvJ~v76D0;k=ViK zyTnHY+W`)i^RYpYDH`i{U5&~MPP>)n+TrSk;w4}>%LvozUYL(lFG|yX*XL&hqziTw zWKj`kEPUSrb*SN9%G9iW3$&Y5k*oqhd2hklRy3OLVE4voG_Azdv7tb(9>NW0<=?4_ zR~Ia$LX+D^8suzDGhD_+1+&(S+e{@V555dH3-2i{j;#0xUH1d5pvcn7@VO7osWpQf z$MAd_Q`I0F({y?^_av|LxRG$1tTb@Aa2QpZ0& z-i7^laj1M`kIDS#?8!p{$hqUj$x~&*l_$-@(vP6BriY8w5@sG6vZT1CBG7h1gxsN+ z7;vuEL0cDKc3-q%N>c0xecy^RxMZL2dFLCpV0vR?()U{x+dufMl7y0V zX?n&Po&zsO_IFYurj&o)9tTeN4zk2=F>`Z0KKNRLz9q3?g0Hn(0&0(!2<`rCAVyfg zlKCY@9Y)Id(!VwNy|VwMJ7Ey?y5gcLUiql?V(!|#TBn!v_88uM_-w^yB$V(LAWL(MDdGvJ+~@j^84@ddiu3k>6Y%td#?Ba8*0;5UQoZRWeP!3RZhzd}f}6gQsHtC{dXgrYg;0&Pxxl zj1j`~7pG~VeVH%)8;~8fn>S@Ji%30|E*JJf|6S{{jwZ0aVb8{}Tyl#VRC8yWe0XSN zwMIReu6lHs!6quNbb4%PIMFURQN6-&vCRCw*T{s*D%>VBTxikhk-IPR*}mv(%jQx( zatlT6rPAz39Mc00lfo533#lh&*dB@KsOe_?&MRxbc9!#Zj+&adIFSEToiufh9zvHv zh(YQLKPRcxH5ktBs5Yo3qW9^5P7XI^0=F2^VDR9co=`2CVD?zOMotWwu)#}DMT`T} zR?i-jhe9EDOHmeg0K@0tDUYvh2l=3SwXAdqsf88`cNMO>M9Y*-m(nd`)T+9HcyOTFk&K1SE3jqw`a2|l2^_cPobU{wV267f^pJ|+mw&<0uLEmGB*hy{$D2WQi zseQx93<&&0&DutR*HTc(lv+t=69%h6!}lA6Ta4MOb403US^xwLbkp<`Qj3lk_P)U5 zLA%nJsDRPN45fpeN>cZ;Q68Cze-$|}4rSjn6r#tbu4M{7-#X#{QM{*7t=gY>DZThRr=Bk*cyLVZ756PL`1eaOk$uygc^JD7V8*ZG9OZM%$wyb zO4F)pL){6zld6*3vZdQ1 z8u>Pl0NW9T_<4)YLtm=7p2qZ!Y{_oR{z1xB zT-_e;2%@f=;4Bhj2mRl8LWH~D)x;U-aZ1R+MFPdlDqbLK9Q5DNx;c0YAW`w;Watd zOF2rVN>N$PM4K+35C{57NGJS;r?;x(gq1b+mLPe>IFlgl_(eQJ*(gQjVD%ms{;>9W zM9XdCdiGBSUB4mLo4-%E@VRdF7}$Ml3ezxd>X+fLTm{RuiPnaJxsF4YvJANCZPCz^ z|G6gVze9GJlmB=H1yKG2=VvUDvZ){-F{kP$t*TllO zw^Uf{nF$V%i4lty6QA3~p_HoNhN>VnVk%qGJ?R1q`+kLya3Kv)3fAn{dYt(Mz}0h6 zs>F4iJXJj%agurVokVaka=#edQ}-Rb%6{ZSGgD3k7+IP|(AFMB2wVw0Lr*4%JS;ku zjH~Evd+gVmRq7UMX73vp>&|ZSd@{%z;Q(n7625uX^nuF6YoHRX?O!7dqW@gqPbBi~ zWq*GC^>)fG_6 zonfmUE}?wltkNAiP-Q^qv|tQO??gDmY8e%{V5;AO zDd0gr|D@pRv@V7i`}9{@mF9c4F?^qf@46>BP6{lkGB6X&|IU8)-@f z7%C!bFy)LxaWRhTz)_6}M2maZ7O;(c>6ENh0Ov~tT5No4WG91rc@v}<7kNVg-&qUO z5coco6hIz9SPL%mxei6Aq*W*4sqEdDd+| z<oHS56Qg|i!@G;n?)r@mBaWkTeF_OL5%W2DiD zNx@*9y`Op=vcSp(ik&&ZL8L5TJm+_uujXhF=Qu5B5@4SNSW`%#SV%;md1o~0YT z_(gsy1mbh@)GW8bW<0fvhJUkj{~<1|tEu9-<+%WpYu^39YbTN1%@bVj`MbP5tRl}_ zohNs``7jM-y%|u(r`hiGQRk?Ix^uRn=x=2SgU^;vWHRvXMjIJ?Am!+Jr&Ne zIuQvCAc5XmxA*9D3}Eh}7MZx%&gE2SX(63=t6RUG$nG^OEvCGgT*o@TZmx!MR~tp3 z?yNGy+Z?Box>NjQg!XLrSzyd`Nu?gTQRT+fk)tm-PRhmY3XDs%<}{4qnW^Kov$fn^7HnQL<+4@e=U5ZLDc;lkb#S?efiloQ-Z)$yl`V^f zv*m+ier_Fy+zM`_qCOBd%;wWrTWi*i6V+LEzRjn>uiQtl6)cgFw?#5>b2a6W!I3Ok zAFTSd&%PdDPTF230qkjn6~%>K>jcC^OU1vn|(S1YMv2HMGt$_Kqfi^!Suo`5@< zY#Dcq;o}M{kTa7eiXhESDighi{dcw(^KHB)pQ0#Xh)9+vf>#kqgn^pF#gZp+iAp%z z{YO*#RW>3kF>GT5$Z{@hpSAcQ-8CT!1Rrdk#Cu z`T#K-ms%j_f+009jB?p-!fgkj6rlVfL3H{|n);&sFKy|# zPr`NiC9}eIf31sMy8o@Zw@7yRNHg%u@7iTBsV-a5Z7pjq#AzdB zW|;$57sW{2S*b*xq|SO?JN6QOKY#Z+#12(<5i!vICAiReB13i7wJt)(2D$TSqH)3! zwlwQ#<$M&$63NX*v%27k<`*W2WwbopY|Fsp`eMV%_Lr?0tOVF;__|A@@rd393J+A| z+_2r$5TJO%w;Q)C3JNPSNW8_pC%)=kG4S(iMDb8|naf7#2xv27yHmaW7!Gvw5`LYD znY%~3ZW8MAqxbpscg}q5&2hd1v&_!mpZ!X~kq6gsdg^Y(EY`@EDu5N9^Z!;fL@8&t zC#hpMW5NGY@l0AmswbWFb3RWTcYdEatx^r+-;3C;6#r45-DY*3eg)~M-4+J^ce<=3 zX*+cHoDcK+SvI1VYiLU0AK@i+=8jb~T=F}PUp=IySw~|uco`F7f zr(A0uJU9A5Ell}$sm;yh9B199_lO4iqwYg@+uo=QIZmxh5=)slVy{t=UVhLSo29Wr*o!jUEpN%PV^Q%&sfks$^HB)7`moMKf zd;=UrUr4WJ|96zgeGpd$k~iCfQ#?d53i^1BKZ7HK0*isydBM=;>#vQ()}vyOyU8DI zW462=T>n@50sK+N48=oU-QJ*F{7R=@?MNPuxU3 zf=Gz1!e5Jc!Ru*nA?X3D>B#@q6T3lrAqAMO8evAv@b0SdJVq(&gGdQ8Av3Bfn0vmq zpyni$aAH$Kk)Hlt)^i50CGB?9(@fs|#b9#n)dJ~@(~ju%i19HNt5D2I-s$=_08JUe zZ~jjz1akok=Fl%M^@Hf&rkhWJ42@7Wr58gnv)EZ0so6apn)ju~!2iBJ=scxCy{XNo z8_WHt2F-Zb9~wV;54iP`ioFRThk<4qb8By zM8>xksJ|oql@htlvB{L%OF55cJpGWAe@ZKD^;lDpY!*o9nd53=5@*51>|}#QxFJwWv?Byq5G~eeAmm-_qGi^4X_)dlq9d%9r-vob zd6g%(#Fdb#9I=4=J8R{;po5^ z= z+lP@%N(W5u+{!WFHA84fHluQpG9k^5_3tGpjGf!yS@cc601mQuQN zCo&E&sIqOQeJ$T|Qj4-u2sB8VChlUB9u$(-WYc|xranOTmjyKl<_;i()JCAmb6|=} zZ=7ROPV{P-%@lZY5uAF@RbWqX$L-( zGr0>v%Ag=Cqw63|>Q~e%c}^%(uw|pSj~gbztE5(ZP{u^eYks0~5@bA(%oa?(-)?#? z>GZI_<5?i{%4@EES7k2|&<*Mt6`!}TESC!>;mrl-U7a)e5+D-*=rUD$2qTSe&u^>;O*cXFZvgQ?^9QtqvnOp8%%BO^`VtKt%^8G5 z)u(k^6KvezQud&vmIhoiS#_gJl^J%3R%|LCdV#K>hp7(Y{_vEj)k8RUOPXx`yU(m> zG(}1D^->}N7~Jg(&$#CH-p2y(KF%KgGGQ~j4}{$dQLXICwVg8E6Kh}-jZb{Hx9*5o zvM8RP>g11?{J?O#`%6Lh)gXV9i;5ETnQYPZKtc8! zmyX^;bRz-2RK_}PMG-1iu&qqMO|_{xj$^2;pFI{_rWx3$8?6~Zf?OX_3LD&B*IJ-- z_DkOGK>Nh8?QJ0ka!NP5R&!Et+>Y8W|ICT)Jp?vp{c7)$OQ!cPw=UJs38 zgCLn1;fcWY4CcFz`>oJJm0MV%?cvseVzr2AIum3^)v!xFOy|8<20703aEE@@jIGF7 zAZt<6DdfqOyoQ7KH}z~G5qq;<2vzrh6Z?-a!zn~ep^%dlg7GwGT=S8nPM@WA_oZ)e z&Z1_r%z>0c^Ch>R4+7^`)qSTw zdS#alUln`cjQcD!?n$vPkn#93J5rf?{Jz}6*q@*in8gUV_%%i_p%?g9I1F=v2O-rZ z2J61M-(=q#nqi?%VSi`z8zw-agp;f{F@g&D5V!KH<`Hh*={AaTI6 z>&0Z}#fXEe=!r1%yKa-ZLnt0det&fGR7Ra|gOr>7<*wz0{{Z?0x9W7IbhL4;UUTe` zn7bqQ6BCt37ID1U&b@~~0`9~>_(0a{7k-h01g7DcW?t1h{Ablxvbdxe1?Nl-Xyj{y z(!Ytc+H57qfAG4JfiGfI^S*;!1Fjr%dk)rpvl2$lc5=G4A(?5J^|_zT&`abvRDr73 z6{=rXf|)l!ol=Wq|7sZOX+{9cYALGRSUYeDvX`RqXM1|lyA1g z?93N#ZK-^>LOXS6da&73;BHH@&8QE8Q(9^VS=v*m?1yZi01jJs{Li{g_T7^qM%!_h z8TOpDJiOmB;ubF@Y*6h{Gk2wM?q~1(Z{W}-k{KuaRo_0LLww8=!gHI^r?_t8=bh1) z3N~0M3#g#$LMJUD+;aP@>hl*t&~Yt(NXXgcXn6&+br58YOMCM$i;61wj>}*R*bT>2 z#wY(EAN!IxpAGaN4R+>_<`hxT zBe(jM-B9*aX*hB#*fTN>$jnaCPQf{&;g+9Z+g^edJ3s06?H^X!e_XVtZ3RZB%9IX; zW=iEnIf7a(2I}}USSjoC6GO;9HrXl3DcU37^sClfm0M7IcSgC~D9NYfo1KZAnA9-uJKVV+~n#L3X z8LD>@^2SE?MaJl@PkVBygBWj7so6da&=*0ps#y=Pe%+Wz+UtR8K=;z#MK^D-^cZLy z-dfziF_t970P~WyhY?A8Hr}Jnk5r1>1kw%$6 z+1xIqAD&W4-z;du#)#f+4f^-p2HHPzE%fxwKRk>wyiEk5_GiPGQ}%1G1{<<@@hrB5 zA^6P_5W}HSw{@0#&UW#2c?F+KbyUkiv=X zQ-Gaiwyw}P=*Wx6z=apDB#2$Ir(msZn{4L`s|(Kn8{F9O_&FuVvFJ`AQz6n8Q z5dNi(&x)(t&MoRevch_qf$C-Myp;VBIa|c|1G;dZW#~0`i^mE4XYz@`ajK`B_aCS% zy#7~z9L0`^GaC61xRU);!{_==!jqPe@0B=%S%w+fCb16Y8TsD*OD=dl_b?5P$#kTBD6J4*RR{ZoqCcD*}|ES)kC93GcNX#vTB4BW&7Y%kGaoW z&f~1}ezEht)++vcF96p5-%+pVF3pzZqPZ49g2a!RzKmh8oMBS4)0C4M^$=dHoZQzH ze4!F7hrj4=6e-thtT|Oa!Zk8h3W_U5R>TkzORSa&uy}itNgC<~mr-L-B}b_YAbXiD zAVUG9+qujQgN6J7$z$IEURPu!lNz+F2iBuFTsdI4#V-4jjXDv!HGLuMQLhBokbQ#X zI{(gCsYE>gSR*Jb!J^Fq0N{Jqyx3$rsSph3Tm-XVQP(Jo_Rt6=9?n4 zAu^%}T^>Vy=1m|mPfAv=a?_rPsQ>o?y!)+? zwF%MFS7NNyyGUWi|!2O-f`Z$ zMc<6wKa|;a6lyVWwG+Lnc7|TE`!w8Sy13FOP4@{GaF0dD&>9)+ez2boACf29Xeau# zyb?o6WabElUHsg6@|V)7PQqwQ#gZF|GR0BZ<$DTb;#;f8JF$Z_iUt^G_V)^S{Si>C zlo4z*rkV=X zh&eda<4{zI2F6>zqnaJ{71AkMPEsP*6JvmrCGX;tYoLyO3cIHK2zk7BXZVxHK;G;- z$etWd+9C^#VF!LV!XMo%OfgYlo?AZh+(Ymf=kJrr)t7&7X!t6KH1T-t$oa>CKBpiNs`@_sKR*maG6#S@SPRVvWS1||M${bn`?X)@{O zzb@)sW4KiyXqR5`P@Bn?DYL*-YAOoF1SO+}MX7pdL`9LCkjC*47x}Z7+l|#asQP>t zouR}FTC3naerVf~OnEnIVk7upl*XIOgbp$s zlMn$3yAqkO2d^8#w~s)fG`i|%oEieYqY+Wm`C7AEo$viXdY71D(RMB&QCHP@wOuOy zq5sxGkW>VglEE*ODQ+sI?H7y*N!kF%WaR6N>`uWq^?A1$z&!xDydXl))_9AbQGRpC zg}4-4#5^Ap`1(nmk^Ks7Qk$30zC%7dzLw5g&UECkLTdUn%uNiB8^eX0xUmD|x^L>? z-rC(4botFK)?L^iX?0KlYS*v^Mt<*w{FYz z$MiP1O3gV&@jGIj!x`ii@6~+JQ+(mG8_?qZD{o6~6k*Ke0mf%hL3CM5Ec4=)bUTwm z8bivaZU=M#g)f5_3D4CW#(vab5PDr-i9 zI#O(Co?nK7xNqJvXd;Z|_8)e8&9cFIW;g0{cXk1FJzku;{uHKd?HQM;A2c!1?bRxZ zlgf(Cs}4bY_uz?WB_XBec>lr*_`bPYT%W1WYGH%&oh$0faV~$>lk~-$R{)y;RbFM# z{f2ish`1c6&j%!hth{_y`ssG%QKcIf2Q_JXo#<-2Y&Fts4(rjlN59D0 zAn|+L-ff1;wDDaEdNm-cN(FuSgXYH)_^|SPn<{`0dft;B@${sDTxq#rPH*LJFk8~q z*&>rVHF7-)m|QNAbhC7U>f6{EK3XqpkFC{`2n5)(9{V_P99j5ahtCH8c=XvMWz^p< zCXeG-GRfAwSJv#sb1`5}yHc(@u@~b7a@(9^J&UoJf93GLauS*mu_dbruY`J>kCb>k?WE5JRU z4b%O#0t*FK&-=GBKzove-}h1+>dGC=ZKf_W!`ba`qOtP$*0wK3%qpW&KF3}15*|Iw znfwSlAld#lVTr5XMUAral~!{Mc*mgYzo6@G-%s?D`c$!Ei& z!}2L2$(%#DH6{U0*8rKklc?C9XTI7hiwn5xEiARI%=-&{YqcPry6$p@MDoD@WTUXA ztuy;)=mK6+!~TjWpml<+l6Z(D zoO^sSR;T#=J7oB)ZDT$v;LEVDazfOyxP=i)y&H-nl170>EEdM)C+;j~?OHDUCVD~$+`=kW_ zY(4brF`O=Rk7RLP*Mx~k_4oVlV46%E`DQ=C*SK*vI&KgBJ&YoG@tJoPc#&dCMV-mQ>w-gpY`E zQ=^<06D)Q)s1$QuSk9*5_~cFY`v4Iph~Jeb)dMfFhv{Ww(1DPIgKO4(N7`1?BKTY& z@mtex%U{W7lbB0IO8R8|N!MD4p$v+Uk6MrRz*dBAaCEx`^uz2?SkrwVA?G@6Q#0rB z!}xkXS1E;QfN7Sa$;#U@{!~bmx}$!+Xs0~q(3~6jL8Uy|PkVQF1bk@9H9K&x7VK)N zAXQ5jU6~4C_fB5;{N+}mrM|{$*jExkxMhMMELhE#q{tljTL7koyVu&`h<>Qz^J?6s zt$;Pfq`oZvFpjE;YUF>nWnv|A=126iE0ez}0zyhI^UHwN#@Qek?XG(wqJhpd7v`w- z^!~^6Nh$t6|9mR|28TA|^N43gOKA#Snaj=zl$Io^56&1QxMg3$A1wyK+fJGTn6p>M z0%7JHhE~Up2%N8bS^(2xCG7{-h*gl`_%9SkMr^3xp3|4qv(ma6su68HO|s}ug@I;K zm?nc}?_z^(oanzW{9#O{$?mrX0>@I|T;x(?zvvVg~w#x1`BCSI+c>t)uTtC<2wU(nIUYHD4x z@bU24AJ5JYyO~uP)u2xU2#LB~ws<}54CBpl2Ksn!e@5svk}~!C94&Q0&06`p0g2u_ z@vl3a150vAjs(36{qkII__aF{hLo1^6hR$XfH^m&7&%~G-hEpdfZo(noBB-zlx`cN z*VI0siW@G;4%7*@2E(6Nd21)#`(l3pROX8q4A~Wxr*)m|fOl!15ePsfd^pQp;9kG6 zUZ7~Z&&o?fT1WSLJ;;Z{8|{RrWM1t%$uk*x7(XuRq;Xro!j>Gut{)+o7es;$XCb#|`Q|47g!m<@s|ts;Fe-JBtt6S4IdM#! zJ&b6&3+Pj_zZo^yz!+T2Hj=P&&d_`=yQ%P|7Cv zdP+4@FZX~8yXIzw0yFmDD5z0dXc9Ogp(ve%B2Aj*oTErn0g*%R1PEPd z04eb2_ujm>b2Bq*uQl_0_ukLv@%%tNGa+=Y0o0lqtIMx3DgQ%?ovASRnHrP%pBlTu z*aXg>^nhu@J_a<~GjRnxA=RAXo5^Z!&09P4u`Q*OC>ty@1a@j$19qG1WZjtlJUyTO zA%v`dinnHtxg9mQg)D%NFkK;cmndl3w`G?5shIN_L$QPJ_&b)u-X(Jifo{&R6kRU3 ztR#VK@nQXTIZWFA100kUs=k$>$Z}lj9F;8f%{Gb6ZdVqz`Wz^W2n$LD_e3+S0Qk|6 z^oSL?^M%a@Sp3tHLIDy>Yx}d}$pr{jXEHVI3)fJ8s>BM~MJCqV8j(79Z{JWS9T`iz zxqZ)m20->}NwRrkgDQ)Vf2R5~TJsnGgV~--w_iMZ0|xc5eZ0QW4h<0F-FUTjJIUtv zK5>^0`Jm(MTs>1!?h}eG(-woTUjbGjTYx@dybtU7Me8=%YO*gTlscVN@Ci49$@f5e zL~2Lx66TzeOwTB{P}}70!n66=o4|6pHnd}8-kTXuf7-w`GRuAb4#pD(>SDF)zK~QvHf=<~p8g^0pp&!p2ef5V?CfpouU?1wTYxV~Z0aFDIcD)zb#KB`-+s>6 z+cxTFJbOHm^vm+za&6ZMZye|~ub|BWnA>dM;kbUUl5BDK=6&!ASX z*W|Xw*1bEewidt5{4vi)_@kjKM-NA|>Tu0!cIS`d_f$ZpnY>eKF8(6Add*qb74h{W z=vaB~6&u@1XFL842g%{(+Yi5Hj zCv3ft4!CQZ@f4EfV)3v4m(AhAMGf-SDavdfU72n-)U*i2JO+`QK$31#kkWg{AGwm* z;j6#B?y{xeimn&VoM-|xq~db!+3Ro^A9s8Nl-aj2%rCfeK~-JUMi+~LDMif4)GEJ0 zuQTgApQy|ruGEzIX8w=jKBUMbm1XrOB#7{;dqyCw>J{+rk1g^!q>2ygQYBYf!I^V9 zR6hCFe2h;2aL`^xV6+$GukpHw_EEQY#N5J4!b|IgywydAz60z>@9l`7;g64+CJL-c zvDVms+nJ%1Pyna!wu$SQSLymQ!`J%6LS3~Yo1zd_1W~-^>mj<(X&Dek<`WeQu$=b* zwsnnbU=E#5*Evi6smB>MS&}r&_^>+Mw~CXxR*?MIO!5gyazOA`w>8%N=1qe1cDV&J=7 z&XYZ8faX{YB^2U8{|?ZVw79kCkua?2+?Fzb-#v!y@iDgE)+^;JlG_ zpAy0_#LVJd z7Y;lv#}q=3b-0LEwjkaDR8{*zj{P5SUrk;GFi+uUGk!~67NmyO3u4t3uZv!;!27qG zUP=ey2PdtoU^$~yN|V<(RLcbOSDw=1-UzC?^9O_IXv4VDpk@K^LHE4jYpdb*1`*mV$J7VJ zP~D?Un338rN(#ECQ!doQt%57SS^RdIPm0%B)2ga;;-zEpRrmKVdiXm%Jyp~0)_Z?% zl0jV;SG+uIGvy`<$g70Kp&87V{hQG^lPBLja2jWjPx|TL+&qh0O|E_J0=I|?v5}Z_ z{;i~AVRs&jLWIJS{!`weTUDx}6QNTHRTJEQS5=MqGGcB&(Jh#GU&UI?^XkQ(ZARK~ ziWKa^H9pw~S2-mPG=1#GuJMJJw`%LK5lI@v^V$TO)gW!l>lBZeJIoPY`n_vSvEG>q z{1rG$vOc6uIF&Y`nZ;OH;_raA#$&%dqab-^M6*F998dT|G;EpcdIdTW&LX3%H>)m> z!x_)>m=`GPM*HD0Hwe*%HAutDcTw;@?pSE*49c44VN0zJ7;3fyzxmls9xyS;+MRsR z1Ro7C--qwE*xUR;6>s5j(2ZvRTi=Zv8|+1yrs~JVmxYmu#ko)6vZrILz8dK+!TZMi zW97um?jE&b$&i}5^~M{w(>g4u+)GE7j+T>(%`|)*pPr3pds#=VDHG}Pod=w;jDLi> zE-YgZpQBH@rgBfFDC*36&fv>8`TN1~ym)@xvx7n$IvW_l*84g*NPcl^1#Xk4_bldl=IO=hK~X20%AH&DQ3ibWT@lv_TUh ztp2d)Kxd?_#H<~Y8;_mR3JT8yx|`mYKp?VwV7?smu1IVFA4LFEp;^*Ona^F5UkJA6Sa(P=u!=XbX0Z&p^H-^vpdbvP}} z?E~hdhJlJg)QRxCWYYOU>lwD@dwXLJ)PI6nt0TcH9XXg=HkE|T8}RzBuJt>;Urjiq z=Ba=KeaV>O(n*IfCg;$0rOh<-6@BcI-w1vJJzdbJv>L59R_{fL59%~i*uY~ zhweal^Z%~yx2R#apn@UWfv$%~*m(vZv6N{A4xH(|zZVWx!_rHjk065p#>C#?sf}Bl z-MH3a^U{$En(=;l{PsGNn|Y1`JXuiSaaJbJuG?egpVGRC(Q%0=^lZ38Ppp%K)kj=_R?T4mLMU zzq@&E{EL-X_WDir8McYpn=RX1jm}FxNM)QEBtN0s(?A?J_jxDyLJDkQuPPova zebntAIU%O+vj?yOf`UY@oev&t-;~l`QC5>dnMZu$dmXHb&TFLN+1NfuB@;=&6(S*& zIJc`49%V>;Q_ru!VQ&*^{uq>q=-L*=3 zF*eB+PZLAXlTq$#!Urg3Vf)E@xcGo?)SpcCasKE>>;?fH=*b+e&r4lwQ;uHD8?D$F zE-&NRG9Xn;Qa-}usm$Lf+C@3NK(l5@6&G=$#6*lMU{;nZ!&eMi|EIb4Edo_IvaI?x zjDc&$g@iV$CCq#1$Pf=cPx%)KA{biuNsV?LnCQ5z_hkE8WWK);w1I zm*xI1a!*CY^$rA+%IBK7QVFZHZdq3A4B)`QFs+ zTYgQVR)9C{wO%4#Lqj~Zk($U#q!dayXoKyxq2f3nbH8nv6c?ThdLMLit9GjzLJM$- z2_^(7&RM(}nY1H1{W>vdQvnH+7&CQJE4K|x04KkfF*&{->ZsOqsioOcwU72QTSq`ot`5KH;TH%NyaW>qGl>V`A-`+ZAFhLpSSdAnxg zIakYK{hfUSWNBYWC1eD}WW-RW=s zLAg2_5?MvG1XXdTWFt7jDDBbKqyyfs*aLoVgjKR8@+qWD`sFP`c7U<1jK8G)*;MNH zX!&&-`v$Hdr9c$-v|qtmhbTw7sJ)tY+xsWMoq1>sE2&{}8W+IV2zeR)cu2C>c52^Q zvU=fRLFLc!gmYxHZ!LByz%(;eP^>)R7{k2d9P*<6hq0APB=zITyU-P%c+*q1L`Pdy z^ZqRgH+`scCDQ7OGBEi(Q2vfH`lSGqikF2MdrBWKz_I-Q%>Pf8)? zN(9}%%(4s5VCBg15`Bb1 zyg8~|bVaZJN0Rx$;I=31#Xp@NlNwrAmnXh(FLn=C{)$`+?|3YnP>(bq9HJmR?&j(uLZ=2p|+Ysw;Ek}okyC3*;d%SH+N`qrl! zrt11EfKv)|E_iH6T^@0f_`I{ki-9}3@}My=G4f@J{*#$HjAPV44kW9lKGPGiu^S7u z*If8<`U3a|)L;=>BWTR+ed@SxPq|kDCH&G0Ya%Xrh*q`L?#TN#TOWO`MMn#xLw&BKrnE?5Umtr#!(^RW>rb*P5Pu_j3e{D1E}(U-otW0N7w# zLK#p}LvqcoI2`kgMPF6iakPkG9{711uBdh`?atX;Mr^ZvaajjOHYL4PU3>^ggDO{r zc#iD%tb>LS5RL=>F8yxHk>vXc%kp~)+fgBS6`$`-W1PrXwiUV zLF`T?pH9^gIoJB)*$Hm1Ik6GZ7JNSjv?L2sQ>fJj9m~pB-*uyZGa3Xb@vZB6C zIZxX=M;oi89fuz;Ob$L4xDs$zd9wI$}?+QDwd3SN9B5coGjO-f=mpS!+*=k22Ma$oz?nU zWFBSSc2hg!$$qiTs!AXXE)J+q=-YklD(Uf0(8gz?4vf5i&+S`BAkz@<=x`#4 zw9vE?E3kZyc6~XMxwzde{rcZ<{w>zQqh$FUej^6zNYpX@+xFc^W_gf-Dx$!pFd?#1 zXv$;UG5KD8BayZ?b$tIXu{T+(l73tY1Gf8{$GN2!S7nyViGfH6A(_o zdRAw)7{QnG6oWD4al?Ky`#R`Py8RAOdUf-7#duU=voU@9IMO2!5q@C)i~_|H6kxU- zbM5!liJW7V$ekz~7SBWLd~9^!(Q#+o9RJVdz}olviEbz1@F2$C4&i8gFZyK4Mm=~( zU+LANOE-#Hj~8-V+xwB&d@P9eV!RguHqz;d&=W2j560a@a7=%VWeNh`N@}K9B;nsd zGI}7bn@^re&>UW&JX$-V%Cd*TJ3nU^pHm4j+!S@0@~qLl=`m`(aN)0l49+{U`|ifa zuK(?*T4`!6o=bg{t3CZjBh+If&;BJpm1kQsk!A+RDJ!jTyi+u;{pPGv$;G1jW$4he z$SUIak1gZha#DC;B{hreO0;RTzA7<4N*=Nk#Q2w${{OuM3nzyqh)GzxB zuiSj;U}N`E$K#`swa~?~x+g`6YuE0w5sq}ne5$1WQqIlR2X#lT8H)0ARe~4V=7Pj( zTJ#K+GTJQ4xBhjO;2tBatx2rCZwCgMlt3Y!Hpm#IE*3@7or?G+&W%SUUbIJGZ|%fX zJ>~Ml1g{gLcXRxK1*ZA58qrhDx2SJ|DrQ$_b9Ub;Y4MjY>P)L2{C%{tDlnlpmSU?p zWb$ckp%uq_2J5oW|qvPFawp!RM5jF|}YBM(kpJBZJ3G3Xti{yYHl$1*skD`$)z z4dHSd`ma1%g@qaj(Jp7NRj~{2)IH9rMGCcbSG67}bkj6L%&tHo>hq)OXb&&0iegq= zwe6Xqn&lmSq=^|ubGY?)Ea)qOg|Z!g3h~-{V+!hD1=<+e1Ck!;ZRrr)13j|y-`nP13IWKAsCdy)|a^WHgWD4M4Z-8fsZn=dae z(4UT^gyX#jsx(R!2Ld)}zPSo#Xlz7T_d2|V@5*28IqN7d`tI$M<*;~VrH*JEY%C$j zCXb;0H+Wyw#_654L;bO!JO&}_p@yJ`z78YTx_acou4?@9JRwC?_YBq*xbWWUC1$A_ z!-!nka?G#rlH8rI0 z1U3)ZA=i@2@MdW(VUy!WC;wi0mD?_`seb2eB}Au$bfUS{xDCmpIfdcgx^x{n9NUZ3!LSqjYtmbuYb^7)(AoSta6a$c}!&tvsG!biW9_&i;Y zHaG!&d2F2);4PS1g&6p3&g@lni81EpY&${Nk}W^Im@od6B=;&?3RvbsJO47tX#scV z+CB18L=)G>owk)%o5szt&@Xx`lvqP{!d>nq*eQ4-?ripJH9f-jZ34BD!mY(_j=Wg8b> zn?RaOQWJ2FMmEz4 z=+W$wAaUaBLKQb{G#;wZHJ+LRu^KG*uX??uYlcBjSVQW1pV2Gg~${L!|``%msN`gTebsx zyHF+izSAYFw>DcSU>+v;L!%1+Ps?QSzad~+an&bYnbL9LBXkUzEs zM^hk4fozsW?8{-G+B)uwcC+*~kJ_5t371`6F>);7i)+eqUTE{Z@%yU7-?{-Ec_7=- ze=nSLomRCw8);fh&a-$XS9j?cM#MkW{Ff9qmZS{@|7@k4){DhdyUsPYH08u*>PzcwHf>mG>L!$hua=9Z(keOOdMx>fj? zE~`&3Zg%$!Nf3cAZ#;iC0=-xc;=dmjlm^Fh7y6)8d|&RvF8mygcbie&fz9kr&V~jv zwIv1g$JYO)5q?z_&|*m%&La-Gw6@#69uH|_sCzWEMGof&0DA})#&17#xTu2rK$v@7 zYGEjLEgl@ypYb36yV0o1c4$7T*;GsIbJ5^^J=KaFoFq-u{t0|D^hWFRg?A=vwZc7) zrA6J+tg*6GA`@2%A1&XYvzap2K_+-imhcimj~LVFm^X^bCzIAx+@G+24%c>8c^ zXZHbF)~_Ese&1vQ%-QvZ3dd16DaMGw`XbE{9vZo5H~1AzsQI!E8G3ky;z<2l|>#C>}!YKGLwC*clswHG=6Z1NA!1;X>2 zD}6QSFIJQz)Bcoex1F>74L-d{vqe*MlVRuXvLWNS#yt+Cmf1a48sxgp4Xzgpg2m7X zi8TFmRh~;DRQV~TArt1sX!YH!J%4Ye9f7R`9u6u7KjbUW1R@iXJ*?>es$y!u-1D|Z zhwnwZhiokeuypSrUp0Ikf1qS}ShdhAI!+S3T<5wNMAfC`UZaMzQMV!!mB?OZa;Nny z=HI;1XHfJ%y004gF$GwNMzgq88F};uGi+BCu-3j6`Du$Ev&92{|AO6;0gZW*xU7O^ zVXQVKd$pxaA}H(IrHe8W8HCJ6|8Mw=mmBd@!a;h5GDDx8eKn*!~F)F*&Bu{!Kx~v!nMI&Mp`K=VqW`^uj$Ka^>1S|QJ z^yFxCPY$ik+(xrS&q!%RbUJ$>(pl77OsgQ-wzk5WT(mVIE`sJ~bN=eR@4u$3bVeK2 zc{TZ5&1)v1BytFP{`hvuWlA6FbyV|fFJ=XQda=+hbx9b{jbUl3A!&x76H3zUX>e^C{)Oq4c59uw+{cQo4 zMq*=(kQ?m+`(&={mg@Tz6B`m&yJ!D4()b3Vuu-0o$(k7e6&FZ`GP4Dh&qyV>b`_*P zB=PpiH)Ssx4mOV}9VAf|YL|r9r-r$B3@t}4PKt-PZIU@}rnNqk%DXfOP2&aMI3xwd!=>o2pf{gIA5vNaUl@s*vMR-XT=J|+@O`^4j2yQR*r zdAQ~x-z}TxJ#zh&y=!#CW*MZN5i;0V2x{5_t8|$;KWtJX21ykvii;->+_PSCNPOSW zu&8^H8E|*y<_BxkZ8Q4Ct+W`--3yUbT(2su2p#OCM!X|t)O4kIaLmI5D4bOApk`~~ zSlk1XQs3GCL%kaopSxFNG8WaLwqsj~G^G2>#iEut5-R5&X{59MR6Damsw}>eIp~|Of?cR%i?o;$;;f9QT8lYlY#>314Yqgr_=oghtsPLZ z7c+ia1rHe;xKB99bYbxmR+t`9MAL-3ld*$@a%h(3K}BqToa5gqps!Mw^h}T}h(~*W zllR&$o*1id`lK4aVn#|+22}8Ggn}rmzgn#iMY;>1N_#8=KE`-E4y3;g_?3~tEo%@5 z#fDwNAP#x(S;dWo1Y=4cR0pW!A+Nfx;Zxny@Xn==oAg+mV}_B>!=;I@9zdZ|zz_dg zBpZerlR$aQ-4B*GR9b+@tD@d6GoVE=v-xd&v5vLGDyQ4p_kOY)9wbsfT4g`2UQy!Z zNF>Cb2;BGuaZE9Kf`5NvP(kYIaBS8a1)TLnb6k8eI=EuAR%Mn_4%7VHhM&z89;IIJ z{bJvmBQiC*x>aRI;#Yr2da23SV^;DwV1r|ilbKIS(IrI3KFDRBL;ICnx^E2k*g%>n zuwOS{&+$%(8T7iGLV@Y|ayxe!1?(Wku9I!HOdAjX9^;**JjT&&pOdwEt&e+}XCD7c zd@rs(4MpsNiHTB(Ugx2+_2V+;n`)H&hL^LNsff!$d*JjXTT+bx^(bd;=-10*u(wc}kgbIYmVJb23hM{v{{s~;J#H^!drGQWL>^9j0pdrm2!G3NAEd;i@PKj})sqEI&KKqzocSqLeM-*U&g&jJNxznB~JHo*K{KxJBb)lPDb)^tWzBR9PW2F zEjhXIqYZWR@JZ(gc8wB6uP@1tn9?K2iV}k}#j!}COfG`U$vecHCp3Dn?aXi<<|Ald z1^@E0|3~avkIG*B^%AgEd-PG!;pYOUK;}OUDrVNpA}-N80dW7vHuJ+WRf5D(hqP%A z)Rm^5@TlWKlIkJn)>+VXtXzkN2xb;tRkA$%(b=W+iy6O~jLyM7h_3^yBX^sC4nhCv zG>AdxLB}h% zJGtRyhX?P2=z^b5{%A3ZuHJboM6SRrw%<03WYWBs=pP*qiktm@m95mz#azbr$bNNo zDRCVUU4AZsD>rVW=X_9U41UK+b)sLnR%CL_VecSRB4SyrMUhbV8C*`tKV;`;_eFg} z`3?{6Y0$Th0IHZNLP{qwmrX}|r;^9=j11;9(^H+`nRxma{|G3+EjuzN{lJlS{kk@s zbf5BJ4kq-8KLdDfWh(lH1-b*h6~KYR5RT$4o%cu?^)RO#;mVOashY@vq$ZW~*G0_Q z#vzG8ZfhD(h!tx-Cs8YT_HuC&SKnFLWqL2<$7 zsJ8g%1_9ZURo!Pht2)$!^X3!$+f20z4g-RcAw$K7+9IHK=PmrL1!VrG1=Cd3IAHos z`MUVOTH%on^VkO;#oH&J9b^`ATNVeJTncpK{}BD3u}acjgzZ>utdj@L)IA>e?a*{$ zR3lp9G``^m(D~V5r!l%PZ~S@ywy!nT$?++wfIdIba2oR>JI$xmW8+}r$o!d{z4WxQ z-2PX#zZC4tf8oPk5!3YH=hMYZ_67XiUY>k|1`W6|5a`(KB# zV9Mh&Rfq&0?(W9~+T;hDs^dXL=x)$gG^(Etus*#z@7cKBe7b`-UwB}uEJY~K?=uA5 z1J|LDs4!m!>Jf(XqKIt8K7dF&~SmA0n9;E zyM@^fW)nT1KZ4=!wQ;Nt7Yk}y6|ph2N6T4bz>__dxnWp8Ugu!syU>aPpKsW;&}&xz z+~IOJozHTaFAs@X{L%I#np%gn%=eeD91mFZDu;w^x5lKPX)E$(Ff zD0e8EWgIU)=)09)IMt-tcr*Cl(=o?e3GYMl5o=C|`gy3Y4aZ5%(J+EH(XX{W6u{R1 zTm9M!Ya)VY)``no=%w?1r)>{sD?>O+#Y(sj)O(Q~e{Y;q-C+5h6FG<1|A6-F&U3Y7 zmwOf@J^lf#FYTqfHiG%-GFCho49*Fh7r~va9M~P{#A)6AwEN96f*BCK5Nvn)zT1nn zp=dxozxF!+EAaQ*D+Pn+sZ|ld1j`@^WgT~ylkDJ585g!fpSg#1 zy~A8bw0k7JPqaraYxyq6A_5{?10B8+#Fl7@@TDApe=T|BT1z{)RrPmtWjJi!XM~K$ zQcFw<9#gKXYLk~7O3>kNR^u8Oj<35LDYrG@t2y@K6{PIlZ)*s7%>Ex^Ey1HOHt3a0 zALt|hX?vSBdu^(_v;9z~&8C`J(+cvfgz-6m55T<1&q+VqZ@<$e;}xlU!~JLVxUFIw zg?W}p*Vg(9{C=nVOnyJxTvZ?rxjN9_T`B;~RtmPhE#6oJdhDNm^>5d`<4~2><9@90%_Phpf$YS8><*C^ z#&0z~&^;X;q+PdAU_Z!9v(;3i@7|KtKWzPJ5Con}tY-@fliuf0d=1ZFP6j9b@;b@( zVLjVrN%l+3l7EUi8wH+?0s7U&bqH4Ug@h9f?i*;>%bJAe1(^vKJCpwd3Kw!$3Qdg2 zWn=YnTx81pYBE74jK^QP&sAFz)#mk3Tf)l3PE&>y!KaUWUNI%4#@yyFwJqLWPTZ~U z;53?6^j+8v>V*!0El8a?g9>?qo9TNwLzgdgywQDhnh%lL1lDODP=qAv{TKI!qZ3c; zCA9Le!YV=b^IbQJ%!7qpM<2~!o2Rg@dU9bT-7#8{s(w)qJfJN}7R6@krQ43NPDq0Z z=$k+(Gn-)9iXayhmTL_)2IhGu_-OaZt+VFb`;#Hp9}>yLdmIM*gI70C-u6rM;xW$;ye!d9Y>wL#d%8ceL6cxz*o00ztomx>ek{ zc`^IaI)c3>2={>tVc1#dMCVPktxdiKzr>hHJ@!hvNmJ+SCi?_S@xZ?SpsmJ2?!nd; zLCz5q{nFarRqR@$c#qAvNFjV0#TrdV0+Fb_mOKHnm0Jg_RWrRAcL1KYsr?9wJJAK=v5yRJF&x)z$b2 z*%3hJ?7&!x;s54kUT8c!W*v#X8p|PF>}6ls0fp)$vhK4&3lt!J!S8n)FSr|I##GKn z2jYT5oymg~BscW1S_B^K6!my;+pz8N1%&Y9++saddpva5cGl^s_Y(k7*4h%ZW0=jn zM=@N>QFxU@$sbgZP)omi$YVKl4jO$fP;74KKW^Udfr+3Bg-vsXDs+}(D>5i;xAW!6 z6Tx~`#QL2_-~}+_UMVNSMfT{4_{8mR#OM5mBz*xMQ@DkU(>rQw!yx>UNP#%4D;{t5-fEpw!Ex9BZ;$6|%;b zW)?(5$?03N!Ggt25oyQLp8c;HC1rNuUL1?uoBP3f^sm!q`T#Ep%IjaDgYRIzGHful zf;{sVAANJFvuU$Tk7aEiS@xRwlrqiS=isaFPbgWEx}|u@QuYn@gVA7vmlFUgi4Cz@ zy{D*C2KM60@fc*M49XYfS$%~594%@K){0^DM@%K&_mBPWOtq3I;v7f%%D@Wj^?qB+ zQpO_riW{P2Q8~|X!4=7s++7|BQFhZ@`L)vlLl5~$r6yOcvylUA`u1UY&v3-xjH$rJ zzIBeiuci9O_o72SO1VmAq#kSiKK}fmPP2IxiWoqGr)I0?=9I#aJoVVC)Lag4LxRIL z(JXUq4sS4qa?rm;%dPyn3>&;hFXL<;k=b&{(O)pe$yr7HC5k^M4PV%Op_Uh*?%Mmo z<{r;JPpus=*NNPTP>P4nM($U(-H2a<6q)-Erkn8KnR&dPjv2Bw9n4S0yYg#paS&ArX zVws(|{LcPU9Q`)+Z{nQ6E=~#SAvoK9f9B|1^+{Lp=;2E@i(O9R?fYoP@Bgo_ppW2? zIp3gTx1g;(l-HJ8qM0goW1yMjzgnvYD@A>trx2D8L&o`&c2)4$9-sz#^w{# z70Ntgpqte8AD|p?cwJMIWF}43aB>k23L4lJD zZ0Eesb6klTJu+Ys+j1Kk-0eWkbfV-c4%m{ua^ZU+x$O7O;}3odu633$->~-wDeP1$ zZ=K~MWScp^=tptZUiD@R1t5^M>5uy5eH&V=VWLcJ=~un46|%UjhX=lmdAWLeKb5Xn zv41K=l~8`!9vSVBY;v(>z|}@Rc81q+v|H!IyII%?d@;(|uKbp&anQdFEOx)qEdUL;-O#j_E4Rp~y06yrLDuuiTRxlecxS6w$L6n#2X$ zka~lR+`qACt-pD|sW4@{VDx?L8xZ^~7ZUT?x@vgld}j8;{A_U%5z6wNu~C3}q5tOi z8I~&p)jsKO`}7TLx8)>QuZp2L+0TAN;P7O;?GOKtQN19B02hq&m{K8bM|on<*oK#Yj~ zB~8#PQR+`RCzx7!`W8Qv7GuDSaofAjJTwQ2l_?d@_k+H294iDtW4WmJUuIUKFLF;O zEu7oJgKlWIskkka`oVLn(pZD)3>JV=jjqWkr&_2h%M#&2&w16N=8l6e86J4@1g;7Q z{rtG7MX2O%}zBG3lISYM$uj-CV>E(6I_28l=4tC-b4!ND))A|y1X%6pc&OMv;0R8EfW4~J|Q$^r>2s{*QNM>##ynEvUubX-=`>F zu#9pa8pbl!NRL)kPTWEtw4X>ydv2v#XdFMh9%8EASMjuzm!EdT+YlTJUQpc)_Vc+5X%#{tOUl9I6eCGj zdy~Tol)XA+^VzR9$7SV4;rFUGk4&A3tF8Q`ha*o8`H>uYSH0Lt6U^2SIv?P(>{1!q z{O3_cmY-73lPY0_HlMxl_+1{$-lAWx?~qg5Q9*4{-z{wFNWgA1h54$#d{m+v)Cim^}^F2EQH5)nxvqCMn<-&q^>4dQJOzwvj1| zG@IUfRnml}#^>yJZ-yL&?qj^{94!2V9?sheT^&V@cgw2~R_%pG%z+zS_ai_VDcL^f zCz?yyYj>79C7iop+vT3mU!wP?M)xMM5)Tex#R}*_pCf6`?Bra$Br;&2nYW=-g$S^Q zCSkFfO(W%}1=E=|+8dRFS8=mmChe#sCfSLUeQ&?)!f|qU;xhcAqnRdO=T%tN@g79M z43^FaFPH5;-y%cc<5MshM@gOCt(R@Mlfn^re8#%2U8DWgw$BcpDt9r=>u7&$$%HY# z?JQOY=bs)b3WJg9sGmZYt?<=u67WsD=w~-;ok=AGq6AX&bu;`BnatQa{h^Ng(^_6; zN*zF|S2ritc6%{Wzb$VS`nYJptKvKMbXYD@X;U`EHfulTbT%C|=gow@AR*svyVT%= z9wNz=zjh*>5G5b)mURAz@XUd*Q`#{qxY*^KD+uy8kX7=%gH2jmwy&g_0V9vVck5O` z{7H`eTBReZ?58>SWnH0#l9u4}VxISarBtkD?v?)6YsFThcDO?HIRYC#>a0@yOG&D~LM0RUA&by_t%ECYkVHa=i{CXa51R@}D&2o;a_X zr$X?9E6~JdnC#XrE5~|vyuMb5Pmt&3o19IDm(4B6^56G`HGevN?{n0GyF*zTCUx@G z6JsmmWj6*v8~#GFXHmJXq7M$;2hE?YaIl`Vy4gn_-sgdO^AOMB0KG5=X@o` z$AwoWC7sB5m^S?>8s-`%G=PZeTz1fbTazo4y_WZUF=@)ZB@ylOUL-rRu7^GxS9$fh& zQu$aHANUPG;aS^antRKh9QE;+6zr;nXvD$#I18XgSVU4n>G2|Abh#sg?TWY+!GoSo7z#e@1q zjSgqL6P~{`A&unoU--TTcRtFKkfCR)@iI|&NTbO8Bs7OR@v_Sxqkq3iMXyyl4@*?N z`^kO>4$1)g@1Y-ggQd2nlZ3=wU`kig%XR%+i_indlRmdGvZei`;rp@@Rhhx2#|Npg zlg~7XHOZbs8gxEA)i(wX>cV$oKF{-9vb()^pnttWDL@St&eSs0oF_Fs>48aqCouZ) z#fS@?)rJ$#@?GH5^R>53jmW4z{v!fU6P>sh1-*Z*`aGAk*P@_29>YYTA zArjNuq8@Ufol9f*tu#eV{`;rNdzRxU|FC2|d(Bt<(g+2Gz20Hd&`mhWI=0)m+RuUG zdYyDnjw@p-waO-1n#Jf z8;1b>y7UFWlsovvJ&pk8GIEjZ4zfmK|Ck zhhEJk%lIp}dz8#*_mRfHTN6vKO$0Ya{~gIE`iV~T5pwSNY(5f7-d^&sdGOF@z!T#4 z9JC;}h?^x4Q!nKG2zsf$*dL;#F^=+{O3?jLwY9SX0S%}27|#ZpKLtIA78f+m9)PHz z<#+BBY2dxDD)mJ)ust%G&E4 zafvW*Z!5ZNS|59P^4>b=sXTU8QY^1=PTlLX1#{C*o*FzML?=UJKPkEyiFr)ewEg()IAwg=8h z9euZw)UmHW7i7zbpuClMdULv~Qth(Kgs#A!HXC|ZncWujtvr{=NG+NM1twaa+&NTN zDYkYNAHI;hTuaOx0FkA@OLrI(fP2YU#U3_kn_#GzQ4cEE!0)L!W;?whW7No|_*Xm3 z{kdIZfVv^`4ZnxghmKTJYotg(#sQ%gPXVx=C zVHYZ)J_@ocwNTaKq=UHfxPx0lU0U7oD-7QeaMG_2(4Mm2F4LI74(bQxktd1#jz_O~ zo`W3&L!BxLO`1)Pn`RF_%MW?7$iw(nm+U^!|&60o9Lbf}#(rs~o0u z8PXA6WN+L6NA^M*yNVmR$1RKcXuDC*!xVM~TInfQg}m5)Se#^v_TYY_QQoQEpdLJi zaXgIN!(SZI_{L1_U?Uq(Z+A7h4cH!h-Ou`oiFNWd2`oPSvW_n!0eU_F!3scXK)8#D zq8tu!vor32dfCcgo{=?6km|~9%}Fq;u0Ar|wyOdU>{52RA>^tI-4!mgdD;d@6TF4L zo~+)lq8^jhq^OIH%PwU7E@2@PpiPj+)&_aPfyhVk(4L+96uaKFEa_G4DN+`cJy_?A zoG?VW!!FAmcqZ%`S0Mw8O7$g) z+Y0Y((+R+`Jh2MmJ0CkvaxDzCcH_yz$8=J!XWX0mhUd6s{WC=7FPZ+3s@w3QPa5T= zUZ3Lox0Qr0?(e=Yh41Usbcxj%%>RUg>hf;)Aq0O6@<7C%SRl-=zHVNQpb6_C%_PEo zNjkMThpc#EXsNs}HQ;50*LmA%`YM7eYxf#>%f%gMCUPOq>0E~LM77pR(BvmlWMK{N z)hDyAmA-cT1myU!qz!C&@s5)!IH;G_Eh}DvyhmP<@=Ex))^ZBL&@O$l4;xT%od1;5 znXb_rU3K>allwi|eHhS&ymf<+p^SW1xAHM9Hw4u5s{5xg0((t93Ay)$Tm%f5P41cO z5(M)bLcquip6c}8YWcD+PVWoF0%-Xv;Q~hI+CMC}67=zUOaJNHZKisiWT>{NH|yn_ zGMD6PQ$w~Re^f+HpxP7nBg^n#(w391t{u+Q7@UgVvt6;n(U;eg_pLvpfCuftu4mr` z{I$|MsS^+3RUz_GxB3JJbGm(}U(+t1>&>K9vkEKF% z6x5&_`(3J`o_&Q5CAWsV_r%Nccw{kw=$puXqY8YhJ$`dJKBVjZ|1otQPEBrYv_}CE z5hJ1^QX+zYH5BO~K@btCibq8t0TGZx?=^`?RWP9_y^4Y!>AfZ*Na$Tc4-go?#&Uz`ZUh&VhPYQfrAV-FC-!@XTuJ z?)ZbGWD{6>Y=e5i>D-U2?H)^6=R=xEFVdWc!6%f7NJx<2>VyW{UvPD#v?TI~Q_$-J z52rABrpm(TM!qabxk^x_zb)>)AOCBl6xQb!B8bwTsmyl-AV~LSfIa)I&`nO+*91g; z7BtAWFKN7%^31;FV?8Mh3L{}+MsHS-Q(;>N_!nSG^|JL+FrQ6FQVRD%z1k;i(Ne<` zl;^!_!IS?wo!s?K6Nz837C%lHmJ=2c$pY~=ETy|(#9olcdi}yM&mO{bQ403mm?#}s z-3+0M9IVvlzZ!{hk7u^?QG2z$iN>%sq}5shW`ZgghIoFe>q}N2tI7}EA*iiU3OqL- zF0tI?g~EF+G%fwvE(v+<_E4tALXLI3s>H4K>pk>6czXd_BOB{XG#vBh14+z#Pf@PV zY4+iqm@23ASVo59V`UwFsi9Mai)F!D?i;$4U$LERA(yT(L%5b<_NP2S=5MSis|xcC zzGHA{#bCpM-o06PZg2StS-wExlD*pQOKBW=1kT`%R&!c7ACmg(qkmDJPO?9f55=W2FUW^)t!GS7SNfG)r->g9xSIxKVZ^n5zf z_{s(anQiZet{00l?rAz0xV5Y8Qqma|d$g-{;~wHq;6JvwZ!g7UwmG^dtdUb^UrZ-u zd7S9SM9RQXJ@IIHMuL!++N$Jq^RtLPy<*mlTAFxt98Ihv=tJ-*nCo(Xn36bO6@X>7 zrk54+_+jWwr z@(bW|+8Yhb{@X9o*?pO?C)9+mzU@C|N;o;`GNCc~^xvC0pxq7^hwCAQ-~j2| z9=7E1Nk<7sjklzzGwj1j5p0lP#)KhqZh6hHxFeMFW-J+WgF63{5e>kQ_e z)jOYb1b*VoH#@X2)Pwd?P0l`$)kIQ8(tlzO*Gn5WpCd-e5~u|$&fjwlT9T|VDj0iU zgk9b*&e;j+JIL9rhx;DTN`vXN(SX}|BA55yZdB~_9cbfA+0(RE3a5|O3*FQ$?$BO4 znpaPa(zLDrue-ydwNZa>mHz->W8VZiH5nEO1Iul)j7dpui4H^CgX}(9kCEt|RFmfqGSp?=QYytsCAS-7i=d%! zI|%*UD(<~=%2rnp*1~#NjUwP=}m?{D{O*iQCp1lJqF1!NpBKR7Q zr$yTz-7{%A&vIUxy)W9eR9#qH;Ym4&{oUjmaZk0AOD*&L2I{YgIQebF1>&$*5#rr) z>TvUfd&xP`Ofl}cpd|3v*N7?k#z9d1FqH>qLbW}>qSmuwM6PreAKl4*Q zMlz46wbfbP8?d1{0r}DO%e;6+a+;t$+HO81{cx2O(KOQfyy&gi(mN94sv@Hg`PqK^ zujBTeCudgwLv?;|SN#K6&{l(RDcL3VL;gwf?vX`MdKt&vejDBU3}37z-fAA3}$nztHd^ zFI(~Kt${7p*4?-;pP6gDTT*3!qz6I5tK5H&nnaB{6xWA7mMl|GJi99Fx5C%q`R)o$ zdaeD{r-4J(yl=+?wwdC^u`2jN?5$9hUH=8faB6&0vBAhczbXxg608rQXVH&wSNm7mFy4{5AFpr zpE#A?Jf`n0IAO*{j0kjU%I6cbvs94#8Dnxi*Cg9Rdozqe0}?|e9Q|*BPVl%x0@B7NpjYGOfs9 z!23$=`26XDIL1VcfK&D-ec+YM+)~f1tl;?QiL%0-QZDo8ulerh14fk%ubH3m6?e;| z&zUMvlrev@O(DOtY+=y9P=vC>A4xpzQzU|RTtZ{RPu9VloSD2US87p7J&u2ls2=(+ zdRzWVGo)XUxs9n&#`HxuC02E&jjA|&8h4by+mZDjhIV%;V1-}q0t5{fanLdPVbe}bzFEL% zXXwg@0k-PvWr40iv(0a8M3iT8OF7SbP1I|n=9u=^GVdHAg`CC>>Zx?KwW3S;|P8(*d#S}M3ldFMr6b+9MA1@mgm;)}AX*r8NT9p|Jp zgxU~Q^;YE^sUSqAY3%dXTK~cKD*BTMM#9)fTy8>L$?c(9ttj2TX%{;fjagwVf5}3^ zULfwd-~KB{vaIWAvN&IPtO#*m5$qq-tLxs!TS2+!YNh(_{spw(R(T5FcWzZnxi0RQ z|AcKXE(U!3aV8X~9Wge;;MO+cJBG*Jibk&{Xe0>NFy4RFswKfEK2eS>{$^^st*da4 z%KF~Y<{|I2Y{|>v9H1Y0ORTEUiw#jT=dMaC3!x=k6lr=N$_ot*@{d%jskET5L zu_|}b_3NPp%6Fp-qidd3KFB$f=NngKUYw?sLWGIi5WTi@tsDC6CmSBQ$nh1iimH1p{fwn_sJnP6fHA3Qaq!l+4tp#3-UNL2)Hs4moZ)Z5%GKH(JpXYiu*CObU9g$vcWABK?XYorrD89hPW$Ak8`lTWsOgIj`bxLaogCpj0wYAp z!UM2Rep>7V+>=L^9nx}dO3}D~oCY+&GFXRi>rTLbGfM5pXyKAwMje+nCB@G<5!ET; z4e&vRd&jo8UNVF}8K5nT;3Xf}EWAB%>+W{OG(tZsjms?>&@yj&*05gvH&&yya~7uq zrf7^ZON8^>eBBV-YoJxVXY~_*Ue~WaV65r;LP+W|B{g5&G2o>|4kD{%yP?`b%*IZI4c9=}S}1>r3lr;5Ax&71p0{Oc`H;nu*za*H)jDl7HFL z1^P-qD|bH;*{LlqSs`6Y4=4y@i%Zm~DS$v4mHqEcRud-feV^Xtb%uo9y05mfOpaZQ zEoH>@zj(LjDd-V|*Xj~W3X5nWL7Dp`K4PiegIwS_7-U@`(+rGVHZLMGoc<9W3@OKYF{b>Vq%#$!;}04)R+QaR7mZ&m6_t9uGQ+iN_eVIuf8KnG-=>e~AEmGm zU3ivlTW(j=(!rd$rMA+&qq1J&fxLzs2Eda0oaJC!%P4@%7lQnI_E)&&9g*z7+x=Ar zyDct5ww!AulXE>{DEH+MZrnh|-bbw}!T4(Nt!C$s(@WZF!tySiePZ@%cevCBrw_Vr z{6Hn%!ME9ws_NDHq#Wc{w>}sDx%=na#;cVR6U}b)S1 z2Ngp6d3>_>{2NwNL>7WG!Ry{*RL4Zd?`C{N{Y#;>2+s9x>NpjVA#D5UlCmLG=81wt#fvL37&dKd@mfH<*jXUV?d~H|M_ALdQRnb@fdR&WAs2#tx z3}@|}HE3=|F3U`)UA1#IYW)eVzZa6hZwORLKHs($tD&=@J28~n}{vj}UkZKJ22Z7o658#`*zNmk`*w%WA! zoN2_Vz*H52GD5v3#uzwqZb@81Ep;Dz>#S9`xO517DA4myYYtYS`BFkGkk46!$*|CH zBw_sWUV{}NvNUN+8XxB#ymT&BroFd1T*1*gR4v9h(d@9gSpB^M$QYXH@?)VNi%w^K=?Ph{<%z#2h6EH18F`DE%5oNSV$ z(hrg?>Ajc^$*P9=oNHfZt!#~Oa+-3d;x)ed&rzgj4an7R!}EBR$8$N zR}nuVfZ^D?Y09_Ew#X~2f0&uSSl&|_2{mU};RS7m*wE4Fy@Z5qKBgkwpp@Sb+8NX} z$60uZSh46MIidPA2AnG46A$`!yt~5F9)KCkNr{lIR%>iioLk2N+;Slo;ntQDXG*>9 z5l0%7-C|0Sjhl5UhrS__HqmvCa*oRCJbawATDDdtW+dkWVnA=tq^*GmK1dD z%2Vq0^5LOV5%sZOZU?|HhN zewoa+PporE-CdtL$|J3RrUIG`%;rAYKSCVVXba!my`e<*9SV#lV$jstEI2AV(J{crqgz-#V^?m zmG>$-D1z#iNSES0W^TS}bjw4$vv}i|zbguiBUvc40g86YQ{)$9=$9z^C4Yx-?SZh? zagh`avSz-U(|GsH(>%hzrIhBH!dxCVwOuVsfv*CYO(6eivuyp1fWS1r8R`C@3)$N z0Qrzp)57I&t^1?OS?tv_qaH#-a%`_BzJYGne;5tD^dA%1=VK)J6HLm(nbn7jQ!Ftz zNSY5tjVecd@Va3}_xV`)xo{~>N_a;Wp<}?WGj=#&M)(L}>&)cLj0hg8pU9FPFx2>k ze1NL^3iOC88ul1!=YA5Qt4+dp?P1nwVM zBgM{fAx=?f6I%QeOV^_*y-1tA24xMe+u5X4?;r3kbwa{>N;ZBA@EQ<}9jr)lM;(2P z@lqw!fLaf&9*@RtPP)S15Wi%R5%>K{IlWWH#B9MqE(HI#hOXso5?76fdzG1|v8WSb z=vkmePS$XQ&BNBi^q1Z+zwYq%;vSx7f7~{l z=W_EY1|6H|69Fj0ErlAyncpw)0Xu&$yqoTi zS3kK6CM=Y88rne!b3SJ(zz(}U3$jj3nZ-H262%!SZ!kjbJ+M^B9PRV1Mwb$DJEv)h zdo5#D+-7-(;(B>;9h+Vu93shjE90&!5g)wuXC(!9AWNNdiXPtxXa(J^7rB(p$@9?W zbWr!dkj#+7jm+f*r|3z%)#0iL5Vdb5X-N+FZ6@Inye=YvI?&2C+K9y9v810EBJ(qG z&ttCgD7@s?{|zsZb_WvaM0PnT03is^J0rQjOcKD&Vh}^K69I5v)_Yc=Cvq4PodY+JE}2W(UQj{8pOHegG3m3K_Re z5oxy>F2oe{jwX|RU{|kc-EaJCxqOtPaeYj z4S7Mun%-UQ-GM=LxUfEv0<)zJBz66WCX%*D2ZCaTns6r7tCW^L#qqDH${t)Q z3zXiK<7%~>v=H~B2J5elhF9zC{3~-o;$MqRsJ#Ise6P~J%pWDVv7o!x5VP7%@X z7mWP;K&zsh(A#;zkZGat9y^8Zc-l43doRotexoTo%LINtdr~s$mzP;!VUv%0 zI^;^Pps`o+X(ohlg{jDXwJWI0s?2S`-ZEj0Mz_jVXo3bd78k3e;KqC1=Xp;J=6JJ) z9?TmMmZLIIzn^S=`(!oPYCTMQhkAB-vD)TK(o(IUTuGL+8lt*;ihsdN^6Yx}^&EfU z{U^A2x`xMD%q^L07s>OkRr`8Z?!ne)>@1V{z}VyyC4WqobF>%^{ZvL={tT@DeGaES zDoq3z4>)yjx^$(*l<=H+648|rklaE8^99i9DlghgtS+ey7kIqKC1dWc$-0d`pY&&q z1oZlEnMn-GvcW86pW_J3X2fn3;@5q|^Cw?4-JNf=Yk2oZA{kIk{Ee3(qj=e;Sqn6d=|r8ghmbDQ8r0Z1TM+foB4vJ zLf>uvbsr&;#v#9JYPEmGRc28a`qKb`5RwyX$bdX`W>=}h1!aePkZHb|n*=;h^Uema z-S|S=b8I4+9~t8!QCS_!~%2)`B1@KNxlKocD9wme78Dj7kKYCBja(-10);Q}&d8e`aLl~SC4IA{zZcyxNtm#u^ z@IoiNwPE-1&faI@BPVo(IPd*_y?k9?G`skrPtQJcuzgHmJ!91#NxG*wtL?R^NScQC=Y%Rqo7vBm(|UE68dy9MtgaRbI$1Dd=x7S!Vq( zFg`q4B4y(vxAAA?Iq<9!it9hI6u;HkqY3lAoXxDV#=XHAQjd%LzwC**8&@MEM-(f@ z{$hg+>!n=*x~`d9BN-aUG;?OPyn6YR`&3hrsy6YCR_8E>Z{+SB?i#u1W(jFSZdIHm z9v|feg&e5EIBknlA{r4P0A(~w5P7oe(zRfjztKGo)<6kzl~6?$U+`{qq`mEX5phyO z4X};xCcxR3x1&1A55&1RGucbg=6GkPFh!va8yRlOtmgCRm6edm*!mo3As~&;MZXO4 zR3aQJ5!&_JyaKxaO)!vVsDSa#nPHp{x|$-m|I_XB-QF*!kpt*ZM5Yf5x;&MpL_Qd% zyc!cRupsVECfr%;HaFj!<{N%K9H8YW?tPY+=d#KjFcfy1!z;bT?YqybjIO?yn^7Ze z5|0-5b6V?R$uF#lDPwYbzm4^`#Oc*C!xk-oU1RvWV-yd`-!6-fMYhw-L?4Pi>G5gj zpK{BTsz4uGh)0!R6BeJM%1}4Akl=TmlL|QKEanVZkhv&a^kX<#Tn+i{So_&MRXYAI z`a1KF)XCQ=Am6H}lbv!OSrWZ3bPFXBr+uM?i2=IgesTG3&eGZW=+hOvJsN3C%%G%S z*AxtC8XBl1{!~fOy!Tp^dZ|*!CZ2%F)s$EB>Z@&S6K>aEzurGJ2Y)C$w*0aO_&&X! zeI_KzsQzx){(fFuA)oi1h8YuZ-sVg6VQ}+%i9Q?aexMoDwm3Nd_K$$I0sTt! zI^T7mSYO*U9Ss{O0QUkzcgdS;ik?k}-^tJ+8~^TJ(Z?UhH%H$me<(De;NoeWfb{*J z+VkAX_)yzbMi@zFlt3y`+Bg z%iiDugSH@dzV_Ty+vjWO)JSDQ+1GstcI=xft=ze*qsAvHMeu>1}%ONqD;voik^|y7pJ6 zhSaL5k74^wTr1hZSrFs*f(`d?Wn7mg|Gk%OnfA8)QI-8g-)u7()x2Tc=yyP&$mS_9|BEjCcr;g_B~#z_=|pi z#x||hP0W8TIe`}fe;;UV7Oty0jHu;FBv%%itu?q(_;-T9b`x^ae0zuvESr4$(V3#QP01sXKsQw#~obPxvVqlt_OLP=6c zT@(SgV$$E_;izQ-Oh8pc+OGt$}o)u z+wsDKIPq3&6EbbX038rYtrdqK?81?uf62%-eRn({bcc2ww`YX`Zgd_AFq{%-;K52i zvyLR~)5w#}NvM^8AF5%u_Fuuvkh8+*Qf50bph+pE6t(yNJM8eR9AWAQAmk=;cK`EH zlwCLhyShIhd%B5Sh{hwg*YRq-!Qw{?yA=Xzj^+%kFs07%Mu!}c9B?qBhSZIux?t1} zy8R@<2S(71{T?$$8OZ>)w?tC=J^PY~dJvoFIr z{~Ib;KyF{D6$YC9u8LfLf^OaT%jB$@h>DL8?`}ml^^a$=ipZOY#xob6`P}O( zw^j6alb+F^FNv)~*kR%Xp1kd;GMg;G$_!I=~- zN+X#(6&i7z6`upqNy#8K-d>pAi7$cs2fdYy0lS?lJl(L%e3%PU9F9;eyVofo;99f@ zfBg7p;7}k>r*`mXYHG}os>0rW6#4#0)tdP&&eQ$W$@#j@X|>Z+9kL#9tmVqs(vlor zS0cu5_}{PP8tpmn*Pj3LrnUIvcET_6%-A5`Zp!)6d)lu&qS@k)krKVEORh&7KpyOs zt7Ao>Tb$VGY4J;bU$WUuhtPcZ*sZzxJ7OEPMACLkwydK#Hr}@1hxu=$mLtp-*+msU%3iU#5G20>KKKWYuJturU)rI|6r`P2vZ-l-;&Ag?F`$7OptT_n*winu4sGwZ5ks?tf^YT}f=m z!$!~6l6;oaPg_yDO|&`WpQ>`tm}Z5XKAU;Y`Tetut3)V0^LLe~RqpPo^e`n$+)kqXw(RSH{55ZUwxtg!SkInB4-2v=i*eSBh=m)i3+J%w+{=;8oSzLA1DFX-8& z`L0|Y_-Q`JWIL04V3H;|Gf+!GXM)6iwX-4Ju^-?0af2?=j{6acwuirkM+dVIrAxy-UfGL~L$-iM;{^@jkgjkx^%RLl)pj#|?V)$&yCvB`*dZ&)hRpmfMcCWJW5n zxW_j0KJ8TOX9CF|O1g#33-Nhl#_MqZxACT^FXnEGpjLlDQrKw77 zQ8B6YGEzE#<7O*nil`Fs5naFi#-JmV=^gr>nA~eE220LZOCe&PhBx$9^Kqu|jC|3I zLm$~jyt2o>gPDo?-h)@+SQ)5Wi#ndz2+73VH`_ky>D?;9twrzGL!(aSpa-f|#6bgn zY{7%D)0{zIC7jCs>!Z%9dP6ihY6Em@A-b)}7yd)*E9xq#=~R43GF8O98*zg$@K2kdISSvDThb?_i_+k0 z;b#@;Wc%j{9;M5W_8(pWMdBA07gr`%8k57_eS4~BvQtzuyheMOLKCAMd3=4|3E^#2 z`-$MQr?zyul+O2m1Ej*^;Wiz=oryAKGoy$ufyJFaJvaZv>G|jQylI{0V+(Lefy`Nn zU+8N38YnG=<<=rONq$j-BD)uI{7aW`Rb5T*nr8RqfRAPd0)5LI8}0mV2?J&0eL^!iz3dI4%DX9xppaQ%DxGQ4eKqf`BKDO zsIYK(__Z*?gRXYTRp|PLxOks8+fIC|=kIVzhtIkUhG0$hKfY8WMmHYAVLx$i+?G7e zXerB`eiL1PQD3g@`6q3+rPP{;3H?N@+jQ#+bpD61(pRImGbI77D=9VlfxU%0_Nx!& zha><6F0h)Zmbzt3aS}e{;mBaNH7@&+PJMh^5uwW@D<)};^d&8^HPzJzLh z$j`)bgm!VeE1T_@tH&{ee`yy_`Oc--!2L>sd$GqRk~QFe>$a;MlpO{F3VJ`r!cX#6 z)CitBDZ2*hD1ln9_BLw_5uCyuX3~`e^Z|zh!>XW7D2CI#sR?GX>n#DCSlYn<) zCVvd)O99vIntN2bX#%VHZCNG#_gFH>&g+zCy9l0sc%A>*Q(X(``Y`|s{xhF;Pi)*u zrFQ@CFCm@%tQNFZs-BzBSU0>fpRIACqY4nYkgsjRT{LW|ckYzyE90}8fT|px1;1%a zu_q9Rlk6kYK+1ui2}EH4b0@MOKSi@o$vm2PJ(euJW;W;2g2kZHo)4Va5y9`LGw(F6 z!6Ta6WvB0Y6$7s(eZlYvqi*s&M%|ki#YiL>0q9=j5BmukwnS2k*+>xc8{%ezR$;_` z@Q>InBer2{QDsx)l5|rh%)G@Y)Y=+7|B{O6(HV_q9tn=R^Se#9dua_?Pq6w?c$)d&sSUmIJOrh{J$1FTN38W{@%ElD61#9h14IIe}&Aq{6=o zT*g2S1T>Ph9H&&WgxgF+w_cI6y6PN<+l(i@OyHY+&^_|#ptg0Zz!LAjo5wLBy&n65 zpj?sfM88nFNN5WUkJIwe_av5bdPpvZZBF8)m#oPB0H|LmnS$)6v2Zl+T|?AbHyXFS z0`%8N`=1_M$`Xcq9(4XQBX{^7=h?*h2MUA-9%=-Ky=)@GB~TOqvW52NaNC=?S{m3& z3&O2=hXO$0rhPMB%D&`AR7&U=%>}Z93J9IuJH%nA47C&iS|f3%6jO^`jP3&&1MwIM zB72Dn#ju4p=CsQXZlU+DcI_;X8X#>Ff%||T4v1%%Rgyq$MF;9xnXvBCwxo~HBT5xHk0kX zyHORrB~3|DqL*f9y8r9%bHg`3x=E;1{JhNYnFrZ!)f4Yc&Q3mQEY258@pQjv1)of$ zyFU^dKW@>_15w`qb(GtUhh0MchY}gT)OIGubiiAeMP4~v&Xe33*WZ3pB*(8oJ#juR z6mfRmC`N}JcW_}$a?C{i9%PA&e2n~VoY3PmQ)zSCv$IFTaVh8-QJhE8Ow_ktgX=$v z_Y2g$gE>V+6G$#f!wtUb81xWl4gz3@c<;c6V`tg?L1iN;lX??p{7FLEQW;?m^BoxE z^>YXgLVU@%sH3~}3Si;srpe%=~1AXd#?(C|^Sh@CuH{KoO-IeY!v7(5mwvPeF z{s#6=^@`|(oDO5`9uVe>h*d&s-h|aF3HeaPXFX~TYc9VIy3!uVn^;_gh!os$W>kW& z7r1ZrE8>JLch+^tnKy57k6U8BmRt7R(WZ$f;>lASvg3br9S1{rP^$EG-Q#+Ta+eyx zO?FMn$w%)J{mQ$b@bd1Rh0Ks}pX^RAKQfdYxD*3ZJ9j%_r=@o#+_9@>2c?KjF25eQ&Vyc#*{bWJg9+Yeye&t#xkKBK2jtK`%eu|BNEI%WTq!Bfle_|PH| z78&95yCGr6)1(JAW~G7lnGODq1=RD+nc3zSlGN-7rm+jo$qOmV??zgpUC{k8BGn;z zey87)U}1c-Y8xWSuIpC#+c%j7rAMi=ojz#fI;qb4reIo)g|BBH_h^Zmy&gGM++HAP zlB_!s8|)Q2p(7EEoH!+Bwbzz6xplVu&TJGyBgdKRkWfld|Mw2*>Q@QYyDLG@Gcy(X zT-tbo>QD6Yy?$%ItU_+zv0r)I?sY6oDMA}iERBVdciS@XNn)DMJDQYAlUQJT%gP>! zuR$hHO+Wk)_IA(Ao3(C~v5iYeaWQ~%VXP-(yq`%AWpm_zV-;EVBpxvGR$$%#MeoN` zEb-lmQpJED3H;t;SnpM7?@TpxsFdm?+)PUsHbnjdd)u?2)#oIhifl?MJtA%935LVB zW3e5r6UfIP=d;rz&`!3LI)&RDceeg*9OS%pf5XL-gNr#y{%(n==8YXZXdisP2Kgv` zbz3Zme87_>c2>p=?>XOhqYky;-i7W|6q8Br9V4LatS+4AF zeJ$SZKmQ6S1kgFuDjwMT6X3mZX;efqLlJAg#Zh-TtsIGp9d=Fpg;@nu^u;X;tVqZmzoRnb}nKP?vHYeaj}6cEgxl_<1)nJHZzB=Htnaob=mv2v?H@)aj& z9;?6zn!7PyjRA*>&S*qcWJPzh0){My)%$&jQDA6bmc60UQcOUTVsg3&a3R$tnJKmS znY3>#Fb&mwkN)QjlBC&@E-~$JB(8SSZt;B)w|d!eYILr*p|q)O=V&u~HoZk7_IsBK zNng!}ALaZJ_q8<|xc^ulg!6Acr%j$Q`n@Vk)GR1w^3I;s?z4I8U6k*i(69On*s(zT zCV3NpA5abZ`}6dQ&@#fR3wQrk#l|FyiXCD9o}P^Rhx)!u#(&Qo7|DWcQ)QsXt&;Qh#|GGEkG;G&J=c*aONH|oAa7^ zTXC@v@k>dLmr>AnTy*{6a+8$$wq~pLxRLPS4!k3l;mqldP3XA8?h)xbHrBp#UfVin zIofAkufD^M3@e=}m5~9h41@ZN%yGwj6<|Bv+k#5dd@mTw2H!2(X(79W3-BzBsMlBE zsF-AX*aS)`ILBBV-a~Dw-PT(JvsNn-avdUo{P4ZZ)|__*a^Q-CEg`%<@ZC)H}pS|iGcsKG$lN` z63|nc>nNwJbHB$A8=#GWjxQbl&Wzc@zxJNi&USViUT7SoCR-1mQv!ao3^pF>JP7dq zGsj)JxW5!9XM7p|ApOL5p#4lu8OPyEo!PGRllU9Hx6eomWblobp{DQ3ZCrS`Q1bV4 z`KfKeD}6gZSC$lWq!eImA3;r$cJwc$>jLkz`+La2z%_bL74Unj=Y-6X;;i)-Gqx~c z>kYXrj^OBLI(eIFni`zzBbB;$=`2DkjH!|1U}mi44>v(5=d1`f&hQV(TZM?g>DO~A zcsAxP&upOs#61`-Kx?NxPqol_1H=f8e@@_-8Hy@~A<|H+l&oY7oSBAK3mre2@@9j# zu=CR%+B5r+hpjY>26FZvd`jq`oFp)Nj&UpWS2!NT?wG_AIR&*q%+j_5h=IZMv#WvQ zL=wk}L_1>0$%MndBphR?R1-T33T4woh?|0VvBQ}FiNk$3ZhwCha}ZCGo7q|-Q4deP z`+v-ce?ot?;zKxv9_m=^1qd2W%V8&z2nS!f zY1&Pjo>bn|Jzfwaon5Wm=@jv<>1U}U39J5Fc6uFhOq;h_>S-22=-ON%~qkFpBAo-6duF9vJ z`-Szr*KfCedFSd-ORUyZ!_rJOZe2u-|M4#5{0Mjwh!zZJRhcYSQ%x@@abZ53^y50( z1}5oh6(V<>!!c-JQ_aDl8fI=#ehUaMR#l zQe+?ullj7@mmf|9wbtpbL|s!W5~2!xjEFCq3j8S+gz%mjeDFO(+9*oQ&ON3+ZN1>1 z^9`LJ1od^*PMbeh`IHkH@h5Sm`=sT)dRk<5vH?u#0VKLm+cjeUAKRfLXf_`oAxWA< z_DUW7Y#mNpo>fp;Z31mKwnh(40qMT$n9YE^zVZ%iNBN{QJ2lJAGwo|B*VY*J_8|7$ zOOc_jm`8aguIIM(y|%7p`v*vqLI*A#X<^s7@2_7&O6W}pXsoSAL6LxRL4@nA@$A2! z1Bc#NNOo-UJ_5=nnLeJ@F*=z)9~uu#aPQevIh%R_d7Kp8HtQ*{^sX%#U(X6nNeUn&gVax$~E5#IruQMf@;46GJ$gW z^yHK+nlWkTuC0$o-_wX*&;y7yV`ZYwT#iE0Z`q1C^XpU00yUc7*6FiP{sGs_{0`mc zx$+Yfp#92hvb%cSmD1cJVIi`WWYMqUpQO!{9^wd!>P+8X*guo1`nU$kwVO3w+Lq;- zrMimIE_Ha(2R7K9sD6gXoWz7NrSO%ddY-)jwi7Zj7KQkM(Cqw>thi-3ab{}}OP9Q769Vhz zjW`O$LLj7aUokK$_ezVqpJ!o2G)fND(RDRe7mL!D%Wo+R%f<{tTAayspTccmAx$NF z6N&zLr9x&6go`E0WrRchu36q?xi{Tk`Q0kOuG$2<*cke`Kf^07J*8YBG;N;Mgt1nU z=K4D`B9{BM^X`8jOjnjRTaEM$bxda4zyK|W{=xk#vJo>Oc@xcbmEa|@-efW1(!6l5(S=Z{A1ITnQV5wtO@C);^v|1J?_DGwXcq~v zqZ#)Omx$MH^RfSX`cTumNW&RAl)(K-udx;ovCeG>ofK#>4>~%wj~*TmzP9pL#LCiQ z>)VQ)yUp6$uNEoD3m(-M0I=dDH&sIRZLl4pd|oA{wa+b&&C6O*uCiRW@_x%qn&Dfd zTteJ_wX*D&Q%3S(OGzDk=JHJTGW1>GB<8;E*LJI z<`Z%|b{!4S-I%-rs@&%5P|yr4%S4G}eJWoTpW)!p*8z$w57s5}Lcby?jBc)VXo~A_ zK~^B%Dh1FB%iO5Gf=>Z$cn7qESgv2E7lBW-^89!(x*|cb#6G4<;c-eSvpDFdz0zw} z5_}^lNGL&t2PylJk*T;*-L6P&Qa~C12hU$sBFip_dmCRU6M$Oc>Z3?Uxiy!(OZwvo z)Wo|7-Q^C(xUMul^KH&?l{OyTwGBzLP}0jhNDS)ajxx3d)TBP4V*dgKg3ei>DOy7@ zdS-9H-b;VTZbqv@TDcs96#BJYEz}a+EziCAnQYQR)1wnN3?^dK+zY&#e3)bEbTd!% z>53lyHQHuM`vTt(hjtru4d$YJ)XDyiZ7-QoVq@{V+f>vO)atWk`W@rK7XtKYc5Prd z!Z07BcWvrY)E2+i++*#GDAK(k2+!L*;`so1579B6La>iI^h1d4&U2=G=QfCWjsC}i z0{IGxi0qmBx&yLobBgEuiz~nkM_Zyc3G113eii}lEj|d7hJWJTqU+ve^lN|_QUt+m zbBMnlRqsJj0tsFHnirROoMvO%_6;}?A2?Z?JFvb!(L z6vP=Hl6^k3Vll95Vu}7)d7HnLDd&9+Zv@c%cX;dP{rKHBP`~b064kZ)!gHCmhEq>Y z(4<;|vJ5X9@6~OyJ`r%5LN=Ugp0VKjG1{k>084_d`8l`fpc=U|Pt2AsGl7fKb-Tsh z>PNMQFXcx-%kDD+$hR?sNpx@g!H&t0R;h#g7x2d9ruNF%@;(Jz@kV?li(7YF!F2y| z$GSN5;tr-zGCt@`Gu1GCFER^%j;Q+a*5YH3A2+MmD|1iZW6PZQi6lO|e{AfyHXlq) z?igF#(}b4YNr@$#ED@0a7$x5=m68zFxRYdxn#Dh1y~OzDO47@g);SM-=glUfUghqz ztF>2~CQ#{Z%he@?z<52!Us#EMQV@-ql<@zg5)O<+3Y~!6hTRc7?}ufVzE4isNB1R7 z)xMp{Ia*fz(IuNxO zhI^N6_s@Logz-{jOjYyEBw^&N7UdD!Jx+034t7IsGmo{$XRB|TAO~{<*lsTVv(4eV z&+oc^|L?l?*Y&4=O# z#cXV4XA#|Y34TaCVUS%RA=%&GfcJl5tq;MN?F01rZ-h862@Dg4SR<#q%b$3VyB5ip zvAfJJBq6xV?wCrJNv+V94yMa|PYpUGu!oJu?r^p>M!e0SfNvViibE0))&WZg1onkJ zH;C*?Ut#DUwk@$^yt|5^rJL9|_#l>zMne%Z;{BpAr0`z5GgIoNeN>Vu#ggy|Nq0z?h2g%u&nO zf6gxT(8He?fP}6n3Hn&*>~t;uBPl&$82C{^swCg9fbzN*&g+C>a~<(-s{OQg*Ode; z!pKXJYFpf$Y5nBWy6FGziR=sKac!8QK#r7;STn5Gj;5kcLjyBvQY3d|Xrl-CT(W|n zufFE!D^NFG4m*{glBYp5H-7QNWdnN9pl@E74A8vkZadcc!z{$yI^dhX#vFqR z79^3xilAJ)B=>#WaKpmJhr{p()7gGTFL+eX<;mAsF!cV8_+}U1?zQsVT+_Ben?bJ0 z``lY^Ei~EEn+_br2w{SD9%>L1%RZi-ErQ?huLu=*CAQJPIri@b(Vq4^ZSf$B_ejyl z^D1hv<3&SlmObZt@LnCtV5Q*uv%+50vWsBo-xq1Va0h^2+pw;{<0}+P=*T%nf(_#6^AgyijcG>n5iv<=iT* zZOLOXx{m#a_T6Bx%`1sMN8p*#ysD`$E7+a&DN60nxX%yD=|Bx{wa%w9sZRb2rjn9sZCfjCp5dv3?-Q@gdxA5dS~;YUo{!kkucV~Gpl(mX~hB;a&x&Q zHacYoa{d-L*Z*B0qPD`C)%QW;RjOYWu`351|JSj?_Z*~z^)P2z)8kmmL>!oRORuBL z8x0BEo*zZ&;J7_ksVz8O;)XTtt#B0>E+yB(JS9UjOfMOTN{`muk@gOzx)dJb-Ho>f zyF?_L7(Sd~_I6MH9WXD1S=^ieGw{^Qd6f3yNx^oAB|U6p|8=h7}jH89xg^b*K(z^kQ8}zUli{l8B9L)k-{JN zcwPDnDT?;An(nJg63V9+KVNC_N(8P)J*ook-%Dv;4<V;>eYsTBj6?;aasr>ZdX4b=2t=Mr{LiZpb{Vn+=6m~Ql*VP9M<4;tfLl(YDS?fU^ z36P}Y39UU-AYG@3o>P3(4oBv99qM9lJ7J$0Jph=9gpLxZ4?>6TM8~Y*nz+Ji>EO+P z=R!;B;a^Uy?te%AD*d?%?#TmNaTLJbem}juALBbD@`w&GHX=&y*90!b%@?qwt;Dxc zaq5~{aw_;{$t`o=L2@zW{?T6x?@-Tq75Jc0GZ$#bw=@Qhb7J&*KVT0TRrbD~nqx*x z5q!hC14GxzrYxWCQpw}%nZ}$59N7F>y4ntAJb@JhZnPlI9E>3pa!x*j(QYU{GVyQ^ zT6J;McG1KNvu!%1Cz&C+_1H6_H#gh;megjBe^xpnh5fU%2g`MvrfKc%Uy~MI4tVHe zFN-Ez_x+6MV)CslRx`MP_vwqgEOpNjs>5DDa?KR5MLDiU`-bxTkisIwpraLIzWY_Z<+08LnCUWl3b+2YES6JyWdJ z8q>HrlW%i?wB1KMTuDR@{wg5!<@qypwiDtC+oEX>4^Yk(fA+5CZ~LrYhunEC4zs=w za1{s@)e(yOy_I@?r(Mwf){F)@E$n;TW98Kt3rNRHJsn$ZK)`K3i;y1^d=iPn+BaVt z$JS???3zCs<~GsPpsZYWUpzjgxmgJt*0X(L!h^XWjTFk=@D*??eAk0KNCpHhnW{?} zAN79pZddke54%MJy}Jv5qG-(FfpL17QbD3_EbM%klGU?aT^Vj+xI&bUpLj^;08&Kk zl{Xxx8tC&)6G;7xOdC>F>&6c|sl%Q@iy-hJof?D>03~RBr;sudQ0^Xzrz+(;Z;2$( z7`|5rZpY0_8RzXiRfk|FDFeoGUR zpJ_~;g}7cP51}~6Zj#q@z%pKvv{Rz9g@JXy$J`a1BLU@9#V=BX7LujISm8d`YzDx7 zpXOXE2FlRXV&ywDiybilgoUbU>o=GOZp0~cKKR&u<+TsKaPFbMKlJvix-|@(kD`@!}5a=+dGtSeH3$i?^ z0*!IIL6>UC!nkXkDlQ|7xgv8E_Wr_&LLxz;bZGlo6EAO+Tpzx8&M5^`7TC$BR}25 zKd)HFm;}UX@BM~taqd49+__#y9zpQ>J(+)JrvKS`R_37(w{O$sFHu@d6kgV8cFJ3~ z?DmD-nS1i%8ok;$She>D7v;B;+ZPZJhHUJXF!TKM0!|ra=`+lck5Bp?%f+AqTJj1iqpycK%5NP)uU~+ap zEZAw~-Y!2|TcLU*DC8Clojy7J+JWtiNW8Ip`la&U_6BLVpQCr>)u~dg+j-a- zwbq8vB;&$+LVQ{(E|Gi^Mx)@1ExpqD_u78GZATW~ZF^bgvb*+*-oV`P?Ks?M6fXwQ z^*XsqLehS5;}4PFB(t2f)d7Ct0p0hMY-0UJ8n)HpGlSY17t4ko16u5tUT<=aeX5=- zcn;>CI;D_@IK?HH%HA>qYdja3qgpy9f^2r{cdv~&6@)BD#=TrO)Hj<057mkXZ5Foc z2%k$dJ`BJ3S(W3Bg7bm`jC0_nsMze^?P8Z?3Y?;j?}nfAPMA*m!{_Nya>}cvbu$L1 z{7^|m?18f|R^+`2$xe-xto_`p!wK~2NsNfyT#5EYj{d-Lj(sV83-P?`sjXO!}| zJLd%Fi#E^ponnq+Q3V7X?z3Rtl>FNAIqa#Yn)|(ko;Mrjb_!7<-=pDC)gUN2f)*^_ zs4vdGyHN8JAZG)=pPS{^=k=NZ5(`*wX_cW{-0VnH>-t1a^|?ZAw}xt&CdkWeJ7)x?I}>IW8>CjAmQEO3aLL_#mcp>80q6=hiRgH*(i}Z^yv+N3M9v z7HTK%6`1pEs#goTK70B|+4217mH8WdZ1-@&$@HHR{vGv_X5TM!Pf<&A`SyDe+QnkK zqXfRF8GBt-WjBu!Dj#PT+6(e{u{gDPrV1JYuR}XBP1u!@;c4b&c`!wEvP@x|QAl}u ztOrVOVW41llp^~boI`Bs)Ns_6PzX?fA`e<=@`e&x= ze24L0Mygc}mDo~;8F$v8jvL@oZywh9M~(Aj@V;eUpXPv^q?oJdYMB6R=c_j}LW`6$ z#p14=?9w~^bk=V@Pv_l5@j@8$v{&7^@NaLc5asM(cG}Oj{;pj?@og!H)$;qEP{V4e z;kJ$(H2frnD9#}>!0FG@qxmSU4S2Sl=|O#3xAr@?9yqCFFLMt%6;0-*2uX!T;Fmxb zQb3i~;;0AD0M9EP+brG}TxN{?x8(XfZ!_WrB)T2Is0KN>6_pBvarE6mIm6;bj(N87 zcjF8If5+7+@xAzltIn-ZWju9nAI5C((h*Td=*$>a$P?AK?t-#kQb&N@K5es;s&7&* z;>>xH1o0DL(138By+slIk-Hp%(Od!Fy%}lQIy_rrh?@Rou1M35e10T1Y#iT2?y;4; zeC77~#zF2BKaSmY*#Gl9gX(cXtU3zO_dd~AdPk9@X%wNZ<@i9WRl+gAnc7Sqr&Ora;8Iln@GVHMS>#f7l*uW4mz z4K0Gf_kP}IoDyBX>i_5UE5HiyBQLW^T`fg@2BJN7sCe_WGn3Pe*wykbwv!UMo^X1j zG@SKrEvho|gEtK$t+w$>Qrw?^Cf^^5Saab(H}#}mxVPQ775~M-0ecW%aZp=FS4`~O z9z_TaJj+CE2mCso3z1t}+?-i$cm<&hBTzm#bzkSf5`pTF?|;$Ia;u?w#hyXW>;@Z@ zt{+*^{Joyd8A^uNR=J(=ZAGR999{pTx-GI}6o_9t6ss8s@F;yu)8IK$KLDOjQk-cde1Rc$gs|Opl3*m&mMHbPgI$7!>E6(Edl8-V=!W z59QmSz#u_e7cFLIzrn9Bch#_#Gb24CYEJgBse~va56ZpshW5?F96${DIk#WiQY-ZF zaN<(G$MZlMQSaTDFvsB2c5A2O${i(Fe-6Zz@2Ye%>I2N-Johka1GZinrU%rgoe=KNQ(xp= z0@}^=@2`=d8)XGNT)?{n zlur_bb8j%x-uAY0iW}uKV8<k(RfYhqn*o`@ zS8}FwNt_>NaHj(sPAVz=p?+Ky9IaQxUDB|=;SgKf;sbiw%6L$j=ZbQo>C3a9Uhi>w^;@`2-s1%w&job+HP*{k>e@z0 zdiGWxpoBLRIWDp%Qf|izUui-ZW4Xx(%RM?iX|#KRO`q0$zpXA5gdH(B^Jh$FRY1xQ z|Cw;^TC(69`IQ}2wwwOWB<p6|OfySJDjVKsqdzd5>urIUJ-{`(_ulOxv;#AE4FU z_sf^U&z5}u7B^IW?+RJYt@V$E)pMkZFovf`8@%!YU6eo1J;+|b59T$?qg!Og-1fcp zZD`K!F$%;dN}4%(SE6pqOI^x_rOJimU4d_4af7Z6ut9eccb9BO=(aCA^pghkksH|L zWL^WV7#lJ+B1p!|6LCs)L#GIDFyfumJhTGG^XisRA9dh{y>7o_xYjYs>ZphJ1PxA?-b3?;`7LFCz%ym>&plw#MKAj?IY8k;-&W^4Uo=u%ZG`$S&i)SVZLK| zj;@N``3Pv}w*)Wd!$cb?7ui?M+rcX{&oa@tCMlCLXU?LN89kGuzfy!XV`qP!q?06< zjaZNROP~dyt3Byn@9A}~586||L^X!Uql0sfo5wCR%n$o!JrFdlk;_Nv;%{%*t%gZh zi5_qds+H>kf2Ml*EOZH-`&(VIeCF{R;?sto55eGTj#v8EfDa4~m#;nE;5^)T^2<6p zWqz|Y7ep|ey`Ar3(PY&yf15KwG)`~3Y&z@tjFgtfNz(U0-A``WDg`4tZtBHH1A4v7 zW!e}8!6{+#G%urQQ+@Zg$W|2GI8;9686|Xw{h4MeNMUlS{Y{Q?(pke^W zr9d6f8jk*pKo~!sfa;<5@$*Oj?0iS8u?ceWuWW)X@~2n|j^&a~po{LZO_D4yJsBIc zXN+E2PCc0*n4gSz>p>WaP{dre!b&jCS|au60HbrfNk=md_JAkDgm3S{evvgX1Xv36 z$MTsLkYbA#VvQ(%veZzeQ%E)ZE$0mYAJW*v-2QAhl^k%QW?lq6nr1H^sG~x|Z@A2N z@m--qiqLU%-v7bo$o1EaQI&T@jFznC2ZEK)P@bzSe{kLMF+9zM-SSnH?NyXpAKFG& z8wOT!@4+K=5>x3Z=YMeq_!Rz0*JdMf(x?i2LhiXOB@?ior+Qny?NxUY9rC?RBgCOKou zPgZVuh99kY;_*^Hy-6+(4*#4Nnxc~)mj~U`%Lo0j(elp6S*Bio1PV0JXppo9v3bf!I)pU9hDjQzm)-ZZJZdLNw^^fbQ!`0U%A2a}zYUYkm>;MYvL zp4&W+!=9!2T-0YiWJ^IZSA_N4An`}gmkCQR^7xxzy2i_w&|>e}X73Yc2y^M+qThLW zR5A}F0-G%;Ypt1e<-)iZV12_~BbGt=FBJ>)GI%Jc(G`zYQBZSrxk;|t`+ee%E-I4uLvELXML zPh)8f${ErKjfYO|)B0$+9)2rr`ygr1!uik`qcUnK5pcf}WL1ADhp!ZyCSqRE zZA$#-RNC{Sj)C1yk+fczXPcWa-tGnOJ$SWzX7)H_n66eYdJ9Z(6j>YA`5ff~1eDu3 zw1T!&s82!Rpq#*&;X3d*HXqC->t_Y2q+6$-yq!PjCco`~j7z$4(2M z9Nuco_5^l$x1AqAnGv1-O+U1+tHH~sgg7Mt4pbH&RzbS1NF%x+5lT~&?n^qA_7Sh3 z7M5pxZ>IC2rp87)A?dGkDV9cAXKH23Nv@UR>&>#bvqYf97*cBs|eS1Akbk62iJJB6Fn;vGiDSYP8COJN(UU zf2*I?6?a2y%pvIens^yfVjmdGEm$9{mGJ>r6+5K;ey1*_plbBuSeb)4pGURe&e3}J z2=-b8`OmAZKH+b!?MnsD6F8A_H|+5r<(3e};jxXFs>JjWa}Ffw2Nt=;#I%-jisd2`O3C_IzD40a(DUIge6JGu*R-2q{tDSm%Gmj61 z_$VWsD%hw)cp4$mo}&pdaq<1Fecun_W@J=*BKyk`&HDX5r}V zrpRjP@l)&7tFUo&h(6{eC*!M32{Nf!Gvn79q{xC==` zSGi4e!B@F&9u`IWhWwZ!9A9J$KwWz3-@N+u7Wq%jQen18S}+Gcbg=#W)}VP@MO4y9 zk`A`)LpO=?H+1Kdee$`S)^OkV!B7E?Gpj_V?{H+70kKG%yJ_7>#8VU50rA+&51VHr zjWbsDa|iB!6Y{sqWtet7LN+Pk2h;BPKHg3cZn7w_U&|F&D<2hhGLNI0$75)X9?fAV zA5*l;QtCNE>c1Xo6Rl0Sx6L3RxMR}yI{rAP!{!6ik&Lu3*MuMoDeEJhMd*ifv*xp< zaH8O_sc0Q;9M5vzes~{h!Aqrj*w+4K2!4e*F9sIUwrIcsesBx~*^?;Oriel>*6zxC z7b<8H%~YwXBE#B7`hy%>oR7h@VYO3k-6WX6QZU#ww$?3b9z-mDdfer^!u<)icN#YP zqNKRR`M>T_$qH4bh|*hE=f^yru4-#?<(6OVr6&jnd9cTnx8Ai1BKvE1!(ZO$ z@3o%hWfA^bxiTXt3`SNP*{NoarnJjFfPWK4c(dZ_3QyUUe*}Kx+ZtiDR#O(_6JhH* z<-1?w?s%AsmizraTW<=sQCVRUl6h*aHuxsYU`bk&>j78u3Gsh~0zM!mq@`d=+goDz z)Q#E{ak04e2`u8}wIaz!BN$QRXxGAbvH7h0b;BU2hM*|29%oN!MVO+FC(j(5hUEu%KobO~_!`a+>Op&;z9xPTB=!m6%b5YE09T~WeEp(XbgScu5qChD zJu9;)SsZp6H5I48{XUMV68n^1L|LwA|MTYhqL(_gCwra`0vais>W1s!eU6L_rmIOt zOfO2UU3K<6CPy@6opyB1<`*Lsq!i=#7`;tQ3*WJX;ndbD=jlEY2&JvGpn!exH zXWso?Qg8(HJ@Vxy$h#q$pIyK*l0$F5mQ1ZpFnKRfU7#Pb^&>oXUekl^K(D&?N9e9| z1Pz2`+|(M#dX2b;xqhi0c*LnTsZ*sS_JDt4o>cE9`Cygblsm$1QLIIn?cY{%GN)qgQ5H+&a#w|bGmMI6x%1$Vuu1Rq(Gjfp(#_RPcvYI-5eP!JD z(gf|EVCgcPU%mf^8euF1edw(eDE<3$5k%x`G{1s7^X!GVD&kd}Sf58ruy`m}=U3Lhk3cWsl- z`TsGcWo>fScTD@v^B-JYdRJ_(@M$_StvWj6xe|2f{wogM&G9oLM*X-LK5Rz!XLqdf zfOP!Ne?*@25tG$d*CV-YJ=Ayy>XX{VQ$EX_nNe2_u&|rpc5T5vm&EP8D^m`7I(NAT zcPC{CJZ}H@u)j!Je~!jLeg)E&*}#12y;Uo`)A9}#B3tUE?W7Y(I(}4^7BCg!feKXqrhhcP#D)y?B-reboxKSlZlXD9#V{$hcNPhuA{`UvI| zUdln_%SubLGH>S~%=0+@^oGI|E<#SKgKksEbDi?ScgkEXMIr2m@L82AeCB7qr~Wy^ zcSqgzeSalZ_WnEHZ$5uc*eX&z`0Sew*M}j_DQ6GdXZRX$dzMwrlUq=^udP;WbJ}h{25^QzsuJNR&@azOf=3`VynVt$iJWPKd z2K9f&kf>wewG9{h>h#}E9!g==%J`EVkGC5lC#!HUCJBO7Rq{U>b4j;8t&M>^7sX1vPY#) z6E0jo#}lv+qiw5kOPSbn^KCfIc1O73-9hxsZ=K0~Go}Ktiemh=GVV>H{jAQ~4aTFF z0;P-P7PIoC(YVRp^|s@r9}dPzOerLZA@?FLi zurA)d@7=$xXKCl%9e6Z*T(iOFxu2?r)w3x@2Uf556Z6cV{pOW&+gZKZ zgBqsL2N*Z-t&M58DSAuw36Yk{)UB<|u=RT?p-=?(+pOOel8z$fylQ=?2sTj?d_cA) z#P>!bOa!-$W+wJObC>g7Q8cvM;FgnqF>gY-4)095^_Nn4ky`AuPu>SO3>idhj;!z90 z0_U^$O9#zZ?(q=nS3%vGDq-0c4}C}f*~QYu(o|}drqf7ZA;g0mjKtoH$SyR-x=MyeeyDc;7Vr!c%e*>?+KTtu z$*)d1HJw%0TSFa1n2;M_Yx%<8_&~Yj28BqFE{S*4tW2iQPkSAFI0!kr{iyQuLYN<^ z$m6hkU>s6+Q)fMdt}C4?UZz$<7Mg%Wi6s$4;$iNutAw}tR6hd zO|2?9sWp2U zi9{vmOi>DutZ3ORYFnvAI!dg5r~WX*XA{lU5cg=-k5}haa|c_n@os%b7@m=s!6dZ(kW1faB(MpI{Gjl0}Bs zul^nQx<%tW-SGC%Q$VSe$2}dZFY+Zz|)5ZW%$ho^4`Q_vcCIs z?hzCrlOK_4Sw$K}Wk)JfAv=N_#tT|6df6|4*<~S4P0m^5%U|&1o*Ixr5cQt*wpf^r z{k&an)@OQHgPpZ{m7zBj}6sevC=2;(u;7j6jt;5~~X*>1&dbXxV+ z`0DrkcF`ps;scGXJ58X4TyGmk&=;m$cg!7-(X=wNWxew;OxYdIRTsfZfXxH##=N4+ zF?XwNz!TRe&=QNc!w2~zwBGSbPw&-ETFl??n`@2}!cA~tE^riORYg@RNy}5z+9+ee=jy@sSeoO_wjh2v$ye{rQv5(uyq%F zrW+TIU%$~+(7nqfGA4YJ)eHQz&xRG|$_|=qPkAvMCtbH9jYKKzwW7fhX0X?LmHB%2 zIkvGIh+cmTroIgnv?XT&4u9;WmmZDtW{ya{!zDp=XXhuHt$B|H-uu6=y@WZGLBF$8Q8U*z5WdiYNp z(Gx3%UeZnZ{+!N<@(J%lvfQvI31Pp!Ho@CSvy1+RM}Xyn%UD-5GaCR{IEb7D`-fBT z;U~U~6dH^`6NI0THPK|G7Gfm|#5{Ocbx?#jS(k+5G|{=X!h%tMnBJ^L-4p3}Glz36 z*?ebJ*Gg|26#kSFD=it~61bX}Dt!ADG5ZwCJNy3c_wcOEPVS})ThcmiUbi{?cQikZ zzw`n*{5iM%KpYyilBk*Z&fV?xsq&XVC{&?vd#Ar2qr1tHab16%6N<-Yo$lS_+# zxn@KF#mVokrEo-wsT^({YPbl%@ytaJ`g9eC!;Z@+r)`j8OM1>oHhQY)!Xy`%mZ)Jy zw?3&I{gHk9;BB%PBhzRZ5`N(5vCM~B z{fg^r^GW2M3cslYALJLG<)6}2=D#H2qzM&@Q)U$u6@*Nu+Ef=Wh%kiSBFi!LcE{TT zzq895IsC2m?6s-_yb^1-?El&x*D4lEQ52+Znq$+6hWwcKt}=fQ)m z<@;pqq0+NG+;H-n63XejKv})30^bYZrq!zK1j2_Pi;cDSZ5|w%L}R9K_m5S$@7|B` z&u@rQ!bvmC2usK1Iensc&l&u6qjiUJK)EK_r;&eCbH;V!t7qbAunygvnvMrr}W}+khwQpnj$AP+~p#8dZ zD{%&`P29Z5(r@c@W-98>^cjZhLay~uH`M*R=xI;YMRDEyU3izDg!Q^GzP|xn9+oAT zc2hvV{~YrR(1ve4;rI!!{%WgTO2PL5mk*{m6}^ofvW(a! zpL+ag_q^4CNhj>QjcfqgxV7O#Za$EUO~#;a^SIj@G~0`O#!%FcDPI#DQGCtH)a|1# z`p`Ae@Slly26xwgh%kdEAD1JuB9&H+(WBa$)3rQ9#_^VV!Mn^GtJjpZUP#QyAG3Gc z0dFm#TbkJZo!uaDKvLb>_Vmf7;=b+FvE-5~Ak7K9ofbkW1qZ;dwkX$l698jwWpx-+ zQ@otHHDLXG-cvxPObYi-1!ss`zsRVvKxGTX#oe%Z#-z|4`pcc2u;Xn=^@F>GHU_YK zIxE*YAKItRDrb*(Y8c&P!L~AyT%pSqGHZ{yw>tj}tL(zr8waGAxo@jcTgJ%&7ugoA z8wO#kZ7z_v=Vme*KX{&53zAH@zAbsE0=I$sR$m&mLQqB7Va(t+KQNF-fU4k#bF;|L ziZ4UL3f-$0Hdekiq=Ni6p{2jLm$gejrtQgN0|Pzzna17!q=K%BYvi~>cgJWc<}*C! zR`T&@@J}ZnL?_|I@OhaebO(ZTdczHR{--#LB_5HF|HrfX{G@?6{hF`YEp}%fo25T~ z%H{XkBk%q-W)L{w8uf-QsKWkL`x)Z6tHDuskR%@$b&A-6_*T%eKBb)mk~&P}tPcY0 zr6M{~BM#gzXrfPJW)f@wuhdH!wL7;zF4-@h2aI~-wl|ERfBHJ;lD=P_hf@o+JWkA4 zZ18|F_Q*HD@?h*=eEg^xDi48YU+(;9UeNu$9eb(dTBdvbvfMH`!K#slIerj=gr06j_S}O*_z2+f(Z7>&B1;I z27D1j27zBw(|mb1QE0~+-BMB`<0;tp==hDD)A}o->(*?8doQG`Ysc=UUd<+V_}Xu| zaj)p?yy~CV{|ikWgkEEd&7+}zWZacN|M@$oa<5}PeJI-1&HfEL_jG$P)0o@@YpOpD zlRT5+yO(~4#YMRd4vv*!_Sj<&W{EnUm&C)++2G>2%;IC#q5C`sG%Kn!Bio%jKQur4 z&G>yMh!OF!VSd}$fgBgI$fe)b4Em_xmEhcIz`rWU5fQgAgD~>*n_#q~$Elev2G_6s zbnUaEdEp$_9QIk-3nQi=^oz|yOGl*ZdH>-WnX~0D?E`jh7iBcRZgz*-`GnIl( zrAWQqhR~likfmLlE_x4c2L|uRfmKGf-~GKig#&roNre^wEM-#s+^QPg_Jz;QBp`mY zo0%vF{MUce;vt-4cZXiQeG6%^ejsGNe?~d3?GHkipUy5OO(lJEcOD_QyWf~JqL*2C zkyw7d`78Zv1tekhvYm`C<(#?hE=7YX7OF`8eHm+oa6rM-GxeB#|<>PpCY^t z+zfAB<4WNtqCvaEESVJBSA?R|`+k)WODH)pfE6NgjC>2ZJg&eUfb%WsJ6xWeGO z2k|>5BnjUsJdMY75$t<2#a#vOI+>Tr*^sNokWxT?J99Eby|H1tJB_iH-{;jCu3EqX zR^H>pPb2`#n{^SeNfDS5`(7~=srt8ugnxbsgcdLY5H=yUm0lMZ9c}R33QhSQO+yme zlrMfrY&e{dTMw2Ntp(Oiv==)OH&<-RVZXg)={uW z4o()5MW#!+!Hk$L*rnPkbSh$|AFYaymH?&~c+NLkKe)J0qPcs%OFR%ucRGEz$%lZuR#Aux%TsFTFS%agB-l&zBVWj zdHV1(l#4OzHKDqeCt`Ef#V94lh4snBsJI|HQ+?ocRvD=O!*HfKTVk{Ry7fm(ao8mO z&cNN2+WP4|Vq0fT+gYlGytQ^%tEHH11*PzQ+jh2FRNzPJa{(Rhn^|{@hnQhA;%yW~ zjGx3jn_ps`rc$t!lhf4M6d)_0N`Ikx7(rv()~%64B9bVE)3*QeH-h2_b^yF`P``Nm z8M}K3MjVn81TnwXjLEAcLfqnU0T#;Og9e4q&#Gr7rs|{j#lR;A_1RV9j$6zS_7}`z zmKS-{yfBN>&;Gsan0A(6I(wz=z(bqMb_#8I9n6bOf_iJNhgr>wH}h5Z-(`FeA0T z_kmXhJg3L9Nl?86`Oh!rgaO~{_|6kCa*Y%pv2iw3Yk?D;$2aFG> zK(mO-a-}yP&L1YDHsd^7?*RfwfYos#FPOhlsSyby*#}a9#uxTlW`lKi2S+aTCOzh^OM7uY87X=S8903{9F?H&?O|XEfm(L0-9+N}yet9QjrY z2GTiEpzG)&{SC#s#im@@bfU|GT*j4bDB|#5{hvdBNK*ro5^!Tf^46p%RiUc!cezPQA(49jv8Yy})OTylp~RQg_p!o>hY?Y6 zI?N*X(qx#jqReEipTW~X(}s)dPiCPBfGpr#6V|KJ^yu1e>FvZ>UVzxXp%U_bBXnb7 z<9!KKFi|Po&_XRJ@YrI;Y&heJ`W6ue5vWOamkA7#sC%Hxyh#SNT{EFYu6{gX81ggo zb+0`Vjgn;U*Nxte{nEjyaKQH$-1Q;N{80z#D3P99zfxOrQM zd)5^83QEm)M{Fkm{ff6x_K)uDMk(>WZ80 zN_80}e9M2B=)@ZP0$9ywL;cl&mNmv>87nu zl?uSVbq42AW7l_|n4BGZnK)#MNWnK_+cD1q3z8g|s4#$MhEjIDVT7LCaL`*&N5k7} ztDE?%GjHR3qC2*eN&M^N?gf0{X^Ii^I$;K2JzP^*6k^Y3{I_Ew-gx8g*!HnS)Huy= zvjDH(uhZOrpN;0`3T+50jI<62$5o1j{41(j1?vsb=WWJp41j{J!~T5RXnWW8wkjYL zfSvP0_g?!_hg4WOmy6E}ss_(kK#4f0x&Mj~@rKbf+EhQ|dvYd>dD3TcZ;e41Q%lz? z5A^i!0vCJgnCW#?4QlSB_f-}PFE%<{`epV&J3ZI=T$nCqZBF}?4t`KI`8>XdaxGWw z6XhuX#s{_FWc_qXC}Y)}Hl#1g~{gx^bhO)ml?Gs7im^nnH^z!SJjHm$GGR9wqG~ z<~ndQHCa(q_y}k1$>5U)6b%%ucE#@U4x5g8`E21Oc5>D5PV1s*sDsGTvvTME=}jZh4f^H`a>;io-T9jGsDvWhM(Z0b&~nRO!NlD>3-O&}A(BS3N5W^h(RbzM7$`AC zw70>z(!E{d+#aa|Mf|B<@q43_#?y3cfP9A zsGogmKGXDKop(LpJ}&Wtou9CQp5u-`ScRIM5BpYZIRp4iktXWe?Upp4?gdqG)`va7 z6ggZqb=Cc-d1sNWTQy`^eTciOaMXVU!0rs_gWUt<=JUtuU$vF5AEEf97_i8emd6)9+X>xHmz6}TnY^)+t_Y5iM@ zD^_DZ8Lnk#d*rofOhs*!n()kbXM8&fTKDv|Zl5*ymSS4^>qGpWkm<}_%r#BkUR@MG zl_;bcFr@9ZHOE`;{%Dqdha@}4I>(gHVBni=7rH1|4>4ch@6b8EGgZecKyCDX9a~m8 z1=WKTo)w(a0qteOx5#DeC`u}L-`sYNEj`aevmCY$7G)$`;UOV92PL+LaoZb2o~pxq z0w5eKmV)$4_CEfKwL`CCw2#MjSxp3I7$1)$D@9WhN!vxA3ilJV_1$HoMtopf%+#r% zgLj=QUdYL}#n86jvuMu(rNNk=1wP;MhJMBy497^3l`YQj0Ys8di;5_&H|XuP6wOfO zjnQsVZqcu~(lvj?`>88Th3}bXc$ruhnV%E!>uz3BL(we4BFR%Oh{%dd{ZZ{8$5Jtn zNJSJ$jk2lf^75MRlrDQF3+*aQ5zsunU*N*A^CNYf8?QoP8#V8>Ph5mY<9)uNTZal~ zHCk~tLs6?@_e_9C<$}oUomxSe-#?vDmE;&z^pNBAA6L3>N_l@nTNzK@m@x?(2JdGANWmv&ud zYh?S%pUp&3loXZx(|y#k(C66UHY9QERyD$A_-C31h1 zV=7|#Yro)nDKF;qKI1JH5CE18ft{4}oB&D0@GpjE^X9^(+afnwCht!n zNFpa*_SJ4DPsV~D?wFkOI)Us(X#9uli7lZIG$?*6YI`T}M+SPHkZ4ns=Q)n6!2soN z?v$F93yzrE3Ax&>xbBo$7x5wM$TAOX@8?66n|436sAZSL!sTmHI2v)=ZwU~8WnQ<| zVbM2r=r=8g|3}kVzcu;3Z=4VYA|VDMF**bRK|xwV5Q(9vh{RO-O?PesK|+w0iNru! zknY$hX=xeV-3&HZu`iz=zW>2<9MADw&v{?>>pV}h!=pXe051V(dz=s84P?N`QU)8D z=Dn;$ZuTluhCJ~^8Io8>r24s0&HM;ZR>z;Z^WL&Pux4A~R7pKILw$T`z?0@dfA~-7 z)I5Wk8W$UCY9GIhW8*fc(4M`Z{^sDg^v#s6#C0>Z5fEVy|Ad_KuaeEu?kNDf(WDWi znyJ>VA{@0z=SKLnMJwt3(|%fTM48bJ>}^qtWJ0f?pe$HI3o#UU8KhZL2$4^}U++0peaT3LT8AXmq?NJeKxm87 zgI|7!?G(GM`lGMOF((*h)D04R} zpn4NjQ6mg8IW8QSZ<1Euad$^{HN8Dl?SpaDGb8A~M_tMQDlqrP-SaAo6B0W!-x~iP z(wOIFQm-8R2+NOD`y2LQ*~iDKzNrx*oAw`XJW}*$eB=xjkSM6OGwsygJ9~V6jDrsS zpVhg}AbhBXwKC4@)Z*vWnJb>qFKZ zjW@qo5e4<1HEZq*p7L)57&0oEexA6Yv(}~9{W4-#cJ|v*-;unp`Lidmu=6yL&WIbM zort;u2%QK#aWhh}f=6-dZA|cH!E1UPp)EG@@zlfe`I`HASrhO_q1dN?khiG_*+L)l zu+@kl{ORNf$cdoiVRfFoGg3o4%6_(k!vChr$aWbW{BF01dNvRzw? zU1-1VqJ`UgP@2kXRatlBtbq`6=dQT?K7=+%Olzp}2`|iM+T%__!CrCtCs9w=<3CD; z@kh45xCkBNmpDvaCOqbg0X6x#;Qn?@_ZIbef?|BlIQ7)qepxHG1H z$1mh9W=k)GRlQPLcL6WOxJBpTWS{U;=wb^lJu4t@=@PT!9;1 zjveK{Iwuha;7n!DWoL`voav3z*JZn><{Hr8Ve)uno*Wrqq3dB30w%w>zZyb}4j5h_ zYy`)h#2M~zoYNp&!qlu%R>~b7`|-~J&L-rVaFEc2&Mwy#y8S2I$h|oR(|46am7Pw* zapEwmT!DBVud-(PnWU=wut>tjkBG*4bk5cRYx3%D^{|fay4INTVjT1z@R}s}>dvr! z*g0M6$ny_LSpPQ`YfYxtGc!v#HU@Zqo*i#XmrCFz>_?Pxq&Ixyib4qOV_mv-BBqar|crmT=LVd2;Zf;8^=|(G4GD{vJ6nm$iJFE><*; zTO5byE;{@Hxhx{2-uU^TfRlCbQA=Boe}3yKNq1$gz%9 zPvq(x{;S%eBKPcdi;B{+>$mq-d;LfS!O7c!Lr$rq0fM$qx;1wj;MeKKjWR==_n9(> z6AqhHy;;7W-2QlJ^QsMy@Doyi4#B<3a)=!!FY z!U2>gYgagK9&z(~B)5+0-`dBoM2}Oisc#V6XBghL(m zN3NN)-BjeP1k9btpscZz-VrdT{?v*J-^gKYfplqs7dxOLx#}M3X3_)(&fFY-=dwpq zJvvercw7s~vNuj8P;c&Kdd>f#fAyDe8aXhS+$TXO6_tMh-7GV{(fh^4%ayOlU`y$9 z@F!x$fuL~46H`vQ>bOVo4Oc8!iJhNPM4=*3$#IS=eO6qFw$6i2U25GMKH=x&yYj@) zC%QeRQvorn)Yumnt|yWp_n`)%%T;E^s_FM0r4&Vb9u2W6)Q90Oz-R`wV2pA9lG!U* zX~g0i&WXQ4{N2nM5HwN|{o!{IjrVzrGp-O9CQSRRO8N!}DelJEa!2~;J{b_mlR-#` zoQL)K{~#((bwaN1YA&B}5^| zSZSvut%FIA(c}GQdCK3_c@du{GUNAyF5A>0x2@CvVvW}p@MsD_sWH;2;h1h`+Bay{ z!l@qo-m@H^0t(0iXJ7cmT+)u}j zVc7EyQ#&aX(*nAxe7Og?9up5H&eWfsC#zBZm+Q_|4{}~?Bc9X!XO#`zFNtGy>Bu}j zrCj=a+{j^WY76ask+7ZP+W)z7P-&rQ=wkoq%c>lh zJP^+cwm)?9ADCT)wyXnS?wNE3Ym(d+4KisU8e4|UbH6n5od&+?lV9-<8G1Wc!qwe6 zIyHFNn?|%IKeKy!jjCYXuj%=@)b7*-(XMp#GXNO}t+A@1*lqqc)YTS12Ktg8*v0Vg0on<{MqpylpM)2^&9e zz07LzGYZh$KjG#)eHqU)eko9l`RwQQ#-%rwH_%*53mOrx;v*9t~-p^+YXs8 za;K|PegV^+ZX=`KyUoG(=eh*Y9G}@Y4V4tK=R=-xr!&~DBpycm6DF>|uhba($Jki> z6hsmFPqTmDPPaEAoIpj;v$rmn=3Z+^hZoDFOU!p^t9R&6{(bFT6@C163?>jPxeW@h zPmD$)h72j7YR1YnVV{lRkE$Bqz0O_NI?qazjh_~4xzy>Sa4UT94UCJ zoxaWxIP}@M;D{|ziCuU8h~GA`o~w}Gr`B!CEt^~vD5wkU*mIo_8bU5J$=wM%5JK2~ zFy9yS{Lt;ECCKlg|I{|xX;yLtu<~Ef)%BS1TH#V=wpLkBh{unn4+d{%-|+tfWSHt&ib(&6lV@K6}-Oq(IdW}9-JSl4ECa|}v;XLa*vrFU9at6$7z zH$lC=4=w0udi5!2R=Q`z*M)1HWY%J%9L;(D{i2&m?)2|A(@gtrnR*oFM?M%Kov=xF zYTg$F^FF#U3y&Z-eH=6Awat&m3rbn7eYn=eBRXr+J1MPhWLHok>QV(NP|cu=&dQHBDK31 z$o}K27sfCtvPmGE+pL)yDC>&nzv8_xj6$kpd{jEb*gJE=y@obJ=~v0efcVwX3NME# zxYl@i(0z-wWg3EUY#D1`U#tSb;A#|^(AD#jx=a`1PEP~m zs6G1@-;=v34-pA`|D}YtOh2ytja}#?1S@qG!)UwyKmv9Eq)Q2Und)>Gi6PsZVx`rE zE>s8qFq)y?hj|sDHR6;Ddqif!fZ`DMb4<gq@R%hd$w z6K3$U&qHV8m{RMiOs_|PaFSTzeusy^M~s{`UypFpOTPX0+-NbPRJQBBbe}%Yto-t{ zkvo3S={3;d1KTrR$mb05G)axraXUDa63A@6cSo;f6LQXhufyJ>1;31BPqT%U*Ykgz zWSURWQfA9?LWK8j8BBg-gVi24p8qUN<>lF9luCEmC&b*w|CIQ*=0dh+K zE&G$z%%({`-nBnJGp*C#alvG`hzLgoERa6BoH2ZGF!1x{TQ`f1>!_GtTh=-G* z=hMCp2`vc%=90%T6m{GMz1{47(=M%BPv!7gWai^5y=;Boqd$q|jXx9Gc}ROTRGU9m z``sfgQTA7I0zAwHxjD&>|5D?1o&~=eN)e}HeDN@anmPP>xY!U}>2Fb(hK6=XRKa7G z+X+8CR?g2e&Vm~%RVj;<1r%hmdXcDFM|;_$0BhZyN7tUsIF+~JW`JlU4x(z$(;jQ0 z?zoW_j8lCQgvn?{=zD^q%4G`N#quGl6Y5^Pf%2yYV|>t%T`=oWq|u;BdK9RL;`VMe z0y*$#ZOSOHN7(tt>Cp0V%+BMYllzVXi_x<_j87-jx^kGEP0A|APvsnD+{YY;<#-HW z)UlPI?o`sh*ti&pxMS*-tP7oVKAN8{4uG9X;vN3zBW*SE_rW|L3r)35HY<$C!&WZ? z=PR}Gk+>PVuaAs39ro`0A}3Tw1?>&adYQ)&s~weScxky`)Bfz^T+ z(CzEW0BRIg%TH(tz8T=U59I*Lc9!-zk-N!T!*xIGcgvH zX7QC5`!f$cNKZvB?p;-aTCL%F+$|#a0M%d_*Tb;qjI%lx(j!23^(6v@yFOd_t?V7S z+i)CPqYRfsq9*pXv2?|dFhQHIlO^(!InrA<)zNp(tZQ`gzl>O ze0m4^Cu?}Qc>M`=B#U6-?%&sOWTsE(2a)MRZjBFl6z5T9>ssfjPS&T{gK>~GP)OWq zVn1JCpJ(ga@KITulIQ0&1?G{+CO2L3OI}FFgYOvq#)CVzHP!ot)d^4L1mO?zeA;=f zM0t5M0vcIf=yq$M{|lO9e&M&$V0XVGKIG%ZLE@SIPz-9H27I&EmZI4uC!V}1qK0ep zXBgceOR$G!SaTx2^p2#8?X~Yt>ZfaaN#gT~^dUoI@z|kti0`p`$(@4?f@c;SdMt#fL zZ)k}Xws@`c@)ASs?3ohldQ)Uf+JM)7mIl$%MxgUSr<#%3RM@FiBs@5{m>VU@UalTC zbFXS7tn#xzrd^kF>$mz&*!d$T&U2A}^x~{5cdwf7|5bZFEAYXwPl$)eHiIfpL}U1Z zH8aH{6O;ttsJo5CmqqAZa^Qw&}NzxN@@n#sZmirDXkEqqB2%y^L zKl@;Mil!gIrsr%iWABN0!Du0;v~Q9+-Zl1QJ&ZK?I3te zwSrV0NBQp>Z8&ZQfKqVVK1F!I2Yu%VSnExZOL8CFPwJa9Y;7_c@cGm~jfob#zb{AZ z$s2zo`ay1%CS!Z;Da*;aINbZefQAUh8QjBk^0V+TnP4P-wy{2gpC*J5dN1?oB?p=e zc=+CDuBmG9QX_m`Dm)tuvQb3PnOwhJh&3Lj_Rl3C0aNuN)Lk6?}R~sfVj$Daign zA`0^ezIZ;6wS#qOamRDPj#c4662Jnri%F;Kmp76y3Ljxt7Dzd$ApEMj_a!+DzZiy* zMa56n2jVFstX3Dhur93NkS~&y3_m2Fz;?&1m%x}3DU{2^;fr$;_$Y25{bGj|3|+JL zJQ-WZNFa|lP2oYL>Zdw{pkU9IbxfReP|x~+x@90bd0vhxzP0llH0h(((-tDt*!CUp ze4}Gah+}v2)0GM;MGgh_<75kl#wVEkY;BeAm_>&AeyYjC368`BLS_h}$*7qRU@)DW zP|}jz#lxdnc&-2OKr-5Cpy_F$%BuZ>+`dr1EwvuG5g}hIWuGPP$L<&s+@I}|FYY(< z-v+{QL_BggZL=%w@U~$lpZqMOQGbEa`y*{cdCGJ0Yh}mf^&=s4vJCCV;PcJEFTx1c z#{86lh$rG_g(|-o{{{_Ett~Irh8v79%NAcp*SK2cT~>QO+p2h5R)Z;4r|kb_>kna9 zz`D&f+0ix@v z-Zm-XAShWB?YATNbfe7lT>emuX+l8&ejzvt$F3twS-`Lt%)eD?UHqRn7D(>M&T9SK za}^0^Wpgi=`t8X!Yoz- z&ak?ozP&Z9BMUpg2!dLSg8%jVhR+op6!;t*|DEQYto1e5=A3Ai{-o*Weor}soX>dR zsKClGr^Ik=AEV`Z8!PNK7F<02H{1APp4yYsEz|41*I4W-lb<&(^#*h%m1B;bE5cc% z`q+T@Px1ChqBnA{8~!c(LVUnf8op`O%u8DO!J(h*&-mDp!a18leWXhih1>=%NH4ejXm_Eac2x-^jaXmGX0jp0InRKF>$a>*bYiNt{1c)zgkF394KPUQ&Fm zw%2CJLXN#U+HVy;?2?+bvIE{A^_-D!BEx5UOjQCZw9xdG{HK-1sDjLMA&zl7bCgDZ zdwiQD%18LxFmC$D7_MC(Ytw2%KNY}X7H>YK^f{7k3yt094`NS15R(`E$M1N7`)^1B zs2bE2mNr^FOhmC6N|U(b^<)b{LxW6$ZN}O@x?EFL`!-E=Eb{^DJ@DGdBs&?QHZ+ic zPh$MJI1UePtk|4(Uz-P=S=?FS&CC#0G7X2t{bBPF zO9C{XRd{!wlh?PmCX^zjCH)9DCYo3SCT>{DR;UnZi}g!7@95s4WBs6!m|S zQomK~Hru4IX_A{GK2@Nbx;d6LE<;N!Oz6G%$=N4982zc7k0*--bA?j{eq^?3uiyVQ zLK2@f>UY;YQlJfTj!b%^I+*T+sa<&Ky#XzANtECR6I}(M{PljXrpS;Z+S4Lfw`sH5 z`^<_c|1pjAr`@>BF6*!jx#6l9tcD+?7|ZFCC^ zbDfI{$2|y)i{}y4THEJ#Pw+~+GSv9S^FUx&=m`lI#uXsO!ZY^wfLk(lx(we7{6!IB zrmQ|y2K*O{KVb6TVCnuCU0H}VxVvgF5XBt@1ZxK;=9K`~)mNJ6`3GAefS+H3c+!H4 zf8hPp>>DGp^2ilEHGj%r4m7IVyKZ0SPj9fiBN#9v!o^OT$A?B4f$kYNX~n$<%EyJ} zwX8zY;luO%D{yA{^{n$aX8yez;@f*F08<{p~q?vtslF&dgH#LiM|*uQA82q z`A3U!KgAHzN^^?*w2BY`YIus^{QzjOj0pd7lGuKak+@*Ll? z%`YAmW+a_g%KBM%C2Mm+EnVc2@?qIAO*Y|;lxv3eUDY;kDr^&^7#0`)*e?(<+y(DE=Kvrhb?`>5y;y ztRaOWlC!CJK|RJT`diSW4au~YxOP32)-cF0tvc8z#Lg;O)~NB5)5-I0Ed?wSzbE^y zJAohY=UER*Wh4?c>Kjt?vVRn@J5b4Lop{B;sr$$% zzoz%C`c(T`0NC{Cp8XCZaJVRTo6Nt{uh0i=%h2iZgyt?{g4@sA;%7+}dOw>k7kf|4 z;(aXZqHwsj=C0P^SALV8{J4jl;pY-wq(1^tVRKvLf{C*xJO|`wOgN3D{yO`k@^1|^ z;QaWCh_oQ4Sf5fjiJvP+`+@k!^3iNkiG}vK=%uZe_dxk;7crNqtWUuMCmy1L+f7U6 zy&Myd*+)%2wg?Nc?$;rtHEWn$7P0*TJ^p+BkQ3>h+N+YMSrpCl*c|PY)%)o2@_rzn z=`U$|0#9I*?+83?k^JIj7a;_ge-^o|x2uxc!v1&@wEKx=-z)dpfjGrZtuwzQ1%__Q zSLx8Z#H)rAV;t-W|G^CEy>Iq*;>z?vGsAYviT~BJ1=eR4k7w8UMLjHn2$~f!RV8e{ zRVE^W#@TE;oic${8doM47>5a(LnwCQ9iTZmMv(?s?>1t(6^To*qsPN_YS{`s}U)ETMAlN3pu z9!h`|tgl0dd-!q8k0tk6EyJc;N>)(rSLwEGwh!P@Cbg}BNLcxF|9OhQLFyrB@{kK3*`(S$vm0mhA8 z^uYkVzp~k?oQj@`m)itJQ5gf>XF`Kq^kAx+#w*8{rUw4~Z?THf$aAGiT-|+<|AyIQ zC3W##xiK!KRx2-D6!4;dTb5FMBzTl;d(Vn$6a;nyXI#g+em~S&dr^kXl>c!>OqYlV zpmH3)uva#iyLTouCdvsytVoWjoU_cYetj_ep3uRy41R?KeLj;5ktVHxYs#eHK%y?t z!qhD!Uai3(zI9}?+$xZaSs+!Yr0qDvuftNCIj9J%RAj29+$}sVmaxkVAu6`nl#A0qC zR=qa-S~peUelW^ec{3e{q@>bYho<+`QUVO0nZ(``7;Teu#N16ez@i4}>k?sfv7Uqa z{w3G2Cb}{l+DD$M>y1Y*6$YrJ#tN*qI%%#3bZKp9l(EX1NSEc9W7^i}daGHa7Mua& zDLKv$um~Beph1;>85X=+FqcGN{HNLy6%;^OH)d#&rEmAcl>=ek)xy z3!uQV7F9exC|c7e8Mgb&aqig(sJ1!aWhp6t4ij=OYPUd`(Kv*~_2A`@`3ZQOqiMNf zXIgzY^RYtr69xUzp!v?j5BGRO0&==_BHkw_{iPoJoPIMLExM0xx9gT;l_HPVJM&WU zhBsI8B5b^3gPju$Pjgh(b7}L2oaa=cK*yhTt+uH54wl6sAM2>qXbl&S{sPjKK#^}1 zifOpGNont*oSwXxqMi>}?knFN1!r5cz80D{r18;-;EJjfb#!IJ*yaMgc>%@FhB)qf^R8*2m;Gyr!NU#l|~T9L<5e2C+ML{4zRrzOHt-=Et_*!I{lnC z!q+1=y=uEb}#>^8ph!%W^{NUL><5LOxejnf@A&?>fh`mEo0XaMX9nym}C@ zC|-w?_0i+Wwh~Yh8Y)ci{8Y#3TjzRnaB67-_&y^>uc@;1PEu()SBF>Z)(6B$@B?j6 zv!q6_`CnTi!;9rzCWUh?2%`e#cFdXfw1EFi)PU=|a;K31{yd|?hPv6Sp98T}+I!Im zqvoT{*O#G|%ggKY?qLPQ&(0bP$=Xx(^9{GRI#tOL@bDIyoAEysxDF^*fc1IqaSkK1 zr5A)RaxBx_SsQFLknO(njoe9^Pk|%#ZvO!IsfAju<%vuzr!{fz?Km*UTz2@;E!zfY z=%K-;`(3c3(9M}J=s=7p(rEDQM2&F|s`rTRo^^2>SjAm1n7Tc*tp1NLo3-Q~!{OAW!>FHK5I#vUjx1&)M&5y$JQvp?F@R7cmk5+juc@kJhqqw;UJT5A2xA~T#`j0n83fC#Fv zy_njFb1SXOvqD+R{>w4i88xB@{6qaL_LL z5;#d+dSkmeD3&3>|B1P2a9N`p@$178;ATXy5sg>I2kuWBkJq^WJmND%#wql%GIr<1 zftF7yLhrH39vJ`O(QP~jMg0nldh0&@s0Uk@Z4}4+o#EZi>x$KKkCa*;ez$`{%hN|Y z#~)Kd3cePx;%k^jc=JBQDBhk`FR;8*uD`4ghxpH@QHlUwQtI+H3%8~U0Ms%gm;8Mo zj9ESq6nI#M%?B;X4bHP&NF~feBiNaaI>)shU!I{mk-P`B!v@ZaZwb_^43+COoR|9* z^&H?z$xrBQo4|gXP|6yGqTUWE`O&?{|bI; zF$WdxVWxf%=rCW;EZ|U<`0R9=A*c%Zr%<5?D9`&z0k%V|E9J>;fQ z)>pF>?^Lx`aHtKkzOLT~xK3x4?r$RxWvoV=ie>EQa7;ZxT12dQU~LX5<@+xVTx#Rq`QT@kc&eu`e> zPoL}zHqSxPA~_WtkN?Om0iYAV{}pGyNmJ^;nPK7af92Nw_rmbQ@e+Iq@)-PW;{xn^ z^{>a{HZvCNkhte~P@zRPpR-C2{|rNB1tQz%8*rXM+=Tp=NHQQ%X_|%{E}xD3ZUsyU z{N(p3`>V$=@=uitdvNmp`}C^g^<1@Vr;@G72Qcc?SGhT4k23`Uty<-pC*MjyQd`P6 zvoL;UiYoClP%)mRH99pVj19BKW(QSmKDnZwYv z%KbslV8p#m7vjPE`-gH+}4q$~15p+}(A#_-N$9{R0~Et7hN0@nRg(9)7gA zGhU4{26w||Vmz^s<(4mpS1;FEIWG0z>^Rvx3JnN;lwWSoyzg^<`V9!xn#o3{SBWnM zQjLjo@DxcaF=heoNJsWJXF9;m_Y2BBTw1>4ubg!!#*+wg-_apE-jaH}Gf7H#6{?~E z#~_#D_z|>CXuY~nKJD5&wb2KWvJuzMN8v=<;A=6!l_DB1E0{q-gDrue4%`3Y$9-Q= zKFdjyC}{(+Z(UblF;^Tu&I&oBD{cQ{Viu#CZa-9*cFvLzNjnM6pyT6eM%4N3KcAaU zH32zH0<0FWgC5Vs^x6Brdlp#i=MFtVy%@UBS$%DMsreiF!R8gmoAHO$`@Cg6E4%-eE)^-^dPQ54Jp@H}+)@BQY*p8WH>s1j<-a@ccm`~1K@MHMNd42%hHT=dIs5ChaCgTz?>A!BAzG>TUJ0p zsfd_Slm~tYOwpBsrS6I%aTxoH{(39q%8RNC)aCw+3rY-jDQiepkUk@gDF5VVF3Pc`L>!r{^0QZOp0<0UR5HHQ9r?}0JS>-lA*ND3`)^?F8w298e6#I-c@nzzzQmG26P72zwc-F-JFGA;Ul)>@Pyi@xn{<=;F;? z8CUiesrMEkny!-_f(=SQi&@Qm*zD8 zLf5U|e~*Bgwu34rL$L_k_vrh|jysn*=Z|bPEVLdzIf+EfK3w*(mhI9Km&NK*18c_Z zK9)hguiJ75OAwxu6xy`PBBOd0z%yDZQ<&^M5pJ8iRX0x6HKQVzTz%USLFXg+>2;0v zt08}{D|}PU?E4hATUE#W!HMNgMa~56THox~o8B6BD>=pB0dwEV zIA?(;UG1Ip4A}H5xtOJrL1P8|-sHzS<9~lYJ)OrV0R9F}f|GP*16>c=BrszS}ty?Ze@p`j*6i86jtZk3=eYG;GdvoCDw$dbuF z6EV9H%~Mc*nNpx#F*@2ek1AF9-rYh_hG!d6wPyDm~1N9}A}H6Oa# zeX6H1J8HGI`~69vCZsabKyzy(Fam`7b?3r7Ae+!5pfT;$f`Cw;TQjcI@l~;z{c8it-6oFV_uehxWLTkUEr)08~ICJK14go zZ?*=u)`8Ueg!d~974>``Q{KW9jjS_`a_Wt#Gk4%X>`n^9dF)Mj?6_{uv0~U0Y;7ex zz8buq32yv;q1Dk6fqFeydnM{HP{{Y6yf#ikb7KG&ZWH?M&6xSZ7m3AwP}=zqHSqJm zp9kl6m+m5b_FitLR@&?ZUg?eGxZ}rU-yEH$+n7Sl`%TwaU0z}OjxcK^n}1^8`R<*; zyUFfJw(l0gcddJozj)VP?xg%ZwrfuA@E>XDusy_GV$Q< ze=tyn^>yvVojV%y>@Ptqa!)4Vn7WKa4#h#oyqJRPAPZMJlm=*nrO2z@0DM-fftiu7 zV{O{zuwa5%W76&G#B(0b-Wd$$9tpVbdl3KM%laXv(X$(uyf~~c{vb6*m8Z}=k`3$2 z)AknBD*m$BPgTd#GkJ}`h8C`hEGM*a?I1dhdfyqUF0VdqR6nZbQGO%EBhug!l&G#S z)pSuGrG-49f=!jFF$%{iDHya9qKc_pc-M(j9WDW@-ve^oT1zRK+7>)3v-Y6-(1q`$q z(LnbG;`xZ|sVXlIBoOD{=&zezPejXG)P=s>^C@G#e1lt$6EHxfqRi92YP{wGV)dms zi92q)oovy=Q)Yd|QwM*9w|HU3Px_*ge>7PTc8~V?yUnVUS)f1}sn&LEc7ZSWKG#LK zHrfbKDFVIE2{xRSv&R9%qD5(&3>|Lls_yf*MAm$*6V@(sNfuCc%>nF3XU%Rd(J_9l z20b@Cci}d)~b8mKbkvJ&xg96Wnqdy5Jiaz;=T2Y~oAO8hET3ON} zzFXn;)a(57UQFFd#{;QSLBng$AYk>VxHDiH7qHUaG!6{ z0|_um9pdqe0iy;~z%AVlB>oa zG{3KT8NA?FeP`6~eJ$6(CW)Ruy`j5PDLY8p{PSll5A0rwj*Zr?&Iyzq zKL~cMldu-Vb;jMJldf|D_xUg<`_HsqPzy_A_L#7nY>s%-UT_+tb9ujoW7nCk!)U0} z`YNn<5?nZB;k6YHEn!*LYHVg3kymej`c?h5>g$HU~tX#A4&)%Vj64@Mr!tG`!7QL9&ou19y zf@*e855+U`JBNs2N9|JzW6SWJ43^(41J@UP;6lS?t1E!(jffy3JbA?W*o2qVI|q6H z&C)WtrwOg2FB<$W*myu+bs;5iM2gLYUTH|gjd6j^@#wL=_F>PPup=Xt`nC`g)cRAXRAXEVdStO)4a-Ubzua5$_TWPEq*K6UBCTy{Yhe zm2?ui?xZyh>q4MG8p84qm0;T^L$;W7DSR2#$JVQ(^rWL`JaJ98K;8St?(3Psv#ZTK zsabi89cyV7+*{!-8_B2T=IgXROAFroKpSY8+qPlL(M-!&xQ7XF`?j<0juG!^T1y_N zxkQjZIR009mBHDM+D7v2o9&uukr_Dn-_fWzed#kTCiyYx|E9` zDfRO+FY*{sN}Vt&-5h*kip(7#MjrJqHjy?_XF<~H-C*rzl>q#G%Iu}mT~`snZ){2f z$$-V6fML)EVe?Xxj)PCjt)Q!v(_*KYuAcvCU_VOq0GI$kdmQXupce!2V5Hnh?=S$Q z0u4AWzc^agg^Hko!IbqA*uU~tQVA-7ZiqSf$PIk4f*_9$EEZoJwgjKJkzd25T|3$a z)C2Gf>XdpZ;1WR}`ft4+GOuhhuF$PcLaGPiEf*;{uKz5d_;M7f3`Vvj56!EpHM>*j zc7~~-Bnl4IOxlHE)-53Tb=Hgg!-n8hFZB8F(^C$OO^zbVkcLq`v9}CWAvWzYIkSy? zjkQPLneKYTAiTFaQ`x9XL%RM7;lq@@`;`1kGJ zoKo@OKhvc>6Sh0Fk~c0c%tZS$0<@wK(pHJWW(!=Qvb8ZjKM0{_OHjExG$z#vrt^^5+LL@&mhWc%keX4YQJK2A?n3qB5d)ck;9~)C`*7! zHjZih+-vMH_cHte)v@#74cG8;)}ltCH4TEc0{`u2zR4Yc54_)>0y}gq{`@Ar!v3?+ zQWd*7F*JaBE@AW@jCv(r$D>&qzroQl6~3*Jn2=5z4q?3W%aSE?`t;<_FX_A*Ip+)U z|JD;1zOIE6={84Lw}3P#iF{{oo55EEMnaz5&Z ztq?P;Yd9**e)75jgsSRHM9NlJI*{fq@#O=KY8M9y22>}&4ZBQm93ymi=Xv!OyVR`- z$yY)8 zs4BqVgU>bj5E{=gr#M=|Sj{7ql+acpM%o7P37GkK>gyewY5EWqUU|0DX5OpM9A*dQ zXG9s^SSTb2Zk`lz1za{6{2Tf@GFE&iP{?W9C{uIpBWe=1_FF2a9qN*P>v?jhO6w!3 zYZgZ4*KnAVKaGloO8_ig)cFY88KIlZY|OeNGtl|w5|=vi*{uKiI)cy1=h?GH7a;rX z2KkHIbr-a_dOghxx%e|t^~ZR(DJJW0lSHDZXIctz>MK;E}EcT2G-?JZE3kL$Hs93!5ewE>)Mv$J3!!uDxmcb1^v8 zrb1>~*+h}5U-uU(29&OBn$ zbZ4JESGKrJvgE(edS&6fwRIWF=OnIgMW9bQ8EIaKpZlDfj=mV@2) z@>Pd?sJaE zhAsN&Z8m>19qAr!^{&6>KOKK0Gnrvp;$&#awHp9U5a= zI4)a`oBj(`+tFZiFX4{zCf2%Veld7(qI~d+*0H`jRPn-aK6G6xdAO$Tx(*MYrIp@$ z&<2%e`&YSOVH_yebfW+JDjoPzwf`Y`=D z51dYK)nyJ6O>5Lg0TG#sgMCxlD%Q4WPzf7B6a6|meb{Y3Y`O68MIHHuJ1b~hbE|mh zsVjX@&+G21xI`*)|0AL7#U@tK^U;gyVy?RF>UOaZ%z*k{EH8I8i-v86UXa>FllO#MLJa+nAuQO@Qe;Vmi7U^KJQL3oxI9sQhmd)tXN>5js z(ib=m1V=$bmAa}~T^Z91fP2bCBkZ#y(nBj8*yiw`yBC+2D_U6_AkNtKUNrst0*=O< z``=9xpDGX8ClNfyD)y(<9fo`0hfQiB2;^D@?dYNQX0BivflUXuX?sCviRDrYZpS(I z2c%Y&zd$!WP&zL4D2a816YkcGi5bAsJOmq7IkK(ClSC^Rm5jk-{3pwDypn~|8S?qu z{`QK!A9%GyhIMeSLt5&)&HOKF%ah@Ob!CbB7vJ2uXCMOa7ab{&zD!;Fv27FC^rm}( ze^gN&voSS+qT%Gt;vyZZL9ZRDqP&C3=#<>9i{2jz_ehsIyyAkce27j7ldR@*J9J}X z9Un<^u*q7@^-a{lWFl#1;Nx*4Y4lkJbC(=+rSSy#rc~c&*q(lI?Fr-EDxE zW#mPE$|zne?~sp>LEGTW11~kP>aiyY3oXriO9VVR&&(1*hV>b$nh(vD84 zo9iuI*lFL1`P!@t*SF~r8%^+sh!2der%|=786R6is{X8wskk4kmVjFNVPVt_fTfOa?rO@1C7(5vqdbj0p*b=!7JINX#qW6_4 zwQfzM$VD78y!?Dcl*r(T-(rHFQMS%3kteJzn;5iJ14ac`yXcE|a}9n(s6~+rrBT~{ zq$BLW7`)yBibau?rBH{9=ilJcW?DN=l&v0V*fEM=4L>9yg}0*1n@D}AQx`b+@4KDN zX#DHFfg6{K^5pPxxG&;s51WGdWlt=yt=nK|TTy)8Fk7zoIr-!52&Acy6_a1oe+|!0ZbHzOX*G-uE3;MOJn8pnFstw-NxL%?&Yt(NLJ^lX_(i@sNkTe*jfgv6cac<%czxPJIt z*Kxf+@AEugXVn)Uf}2e9Q?3mw%Vh{=KF*d(o4fJEVPS5K0vE_p+82k!Vj8T$9h__T z%LE|`x36zg0b_m0e3=1)vn)&9&rL3iYonFRGU(8=VX_rGwWyk&))x-_X8|~ChsT13 zDwGrKNv1vQ5x%~b8g-hvRiIhLxBZykRiyN)LdQ^IW3)N$QTIzkXk{Px=+O#ADGW*d z^lqC6teA94_t=qo5?ubidk|cq$daNa@OJvx%eWpVEw!|q`at0QRr@1B11sNG+wVkj zgE_!}F1KrKixAA;_0>KRh;?}OM-PGc^8+CAMMx^m9hZB<^79L^ z>ZgXrmbLxUB<3GR_M;_7Z(<8i0m6Hg6PBQK3 z_HF-m;w@Q|TY01)YDk*xG%bZb-n^jU*dyDhoCEIqbB)4R!_aY#gkhSLnfT-qchw{% zSaW{t*3ghn8m4Z1n|I1+_&0BW>sgAtqL2;x;Yc&{KGl>s=2GWorqv;E5}N2ruktct zsqnJg(@g+B&Y0y?H(i?iu9lb76OJj&sz6^R2KaT{L5JK=yH6^StlO$kwnb5-O!l8U zQC39jUj z8|l{rPrQO3X-T}IFIQHE2@F6OOUn%420#qW=oZjY&$DGkb5${u>MvI+tNF@cM0$*vJR#i~PH=4CfD*UaRQ5u9>d8gTk4_H<2Ymb!w zEAvM5rIAtg%u{|8!llaWXpPuC^2ZRqz6dn#OW!>DWq#>=-?ed0GPC=&GSG4I{w?}Gu+kO$Ay z+mFpI;vM)%NkqR3Rww#Oa5lEyhcpMuWo-SdEiq!B(7K$Z!a3h?H_TqBleuRS^d`sC zvNv^U(|csvuwYly)i>hXNz{AG5&D**3}ofi+wu3mueK+4*t_>~s=S~+R%ZRBIdqNzE<{b?iAwu9iDn?+|}QgR^ZU2*XIqJtL_VjEbKN zECY(}lYZNphwL4Qj0LzD9sX5(7I>GVmN(!zogL4+8T-E#0{9+d6CA<5beXuCG!1P}2^1-S;NDSN8( zhN$+;{#WYgh`Nv#(VZ$^vm$QXwd<=O{#(`-M1b>x%_d0p{skk5YZ9t}MJuHRGRB(%@M z8ZA(p9gMKx9}~5{&AA5UXKqn8#`myySW{#G$|Nr*tz+t98eskqU?n)dZt5^*6K}_0 z-hg5!eQ@;0du6xXuF)$UbJg1p1$wZ3D&rp|Z-zak=JzLO4L>^;+qyYKwp;7~MxXIO zJ!BXW(LtTV8=`jH-Q0beoLYa5PzoDgyY+-N4(2r7a<*|-`tOiX?^!)2y@<7*Bs_vlU-oZfUPDx-zrPx zaqG({mD?JAXHVY`;UTM(@b~Gfy0z`V_WM2ch_BMJ5@%~0pZ$-k7k}n_J?nAMw{Fhm zBT{4)hhcz2iMGU48L>UVt~l7^ix?FS{n9Bzwbrb0DtM#$Z|2`W;@Or0!}aYpg=2u@ z8yXz9BOF0ps;3XG(jPfCO{W~Nb*FQ^_`UZzCVL}v$P}PO{zj$1h36VZCqIKc%1Zfd zi5YwIeOL1w+V`4SdG@2%RNZ<`;HB zb^g@C6be*XjsTio8;&|PDI22EshJ<=Xg=)fKAx0+^;ng5DnfX=`}@Z^diG@9hU$OQ zT;<<)7oqD^3R`NNviYvG*vqj?wJUw-b^?|v(bgaxgJaU7I4@%lYDr$x;4D8|b(v^I z3WOzmya5?$D7LtFB!nkU%td-mt5bIsIC1r1G`zY$Gkm=mzY6JA*kK=Q3ynNuO+4IJZN8DOEzrrs1bvS))Q&Wp;0f1~0>X-xKh!lT7u z4W=EN=ZN&W3+=mYlPA8y|DGiR!_Yz0S@4HK*Q=l9M2gUm*7+h{opMaALT@(?J3OA` zJ(gckW}E@S9^O55C8Z*(++Ksvl%j?kvVc%E9s#X?4(HW3b`xNfp`Z82#JU`^dlEG1 zQv4%wRW48CY4=dXfP=9B)oGS3=O>~%=~JTug6jOU!tRdJH^V^%kyZ#_r0z8JOOnC8 zl;^q&z!nkj8xPJ@RLGw^zd(8LV&up3{{pu|K>rjOt_He+y6+aBWj~tX+(P3;HEw)Z zH5L4}?VPwYPZf2(p&CI9mR+V#(7m~)_UZ*E1lv<{&W;x!eo!@EZj5g3cv&qpeD!$4 zHTCAODwY-2D22fYLKTXZ%Ed#lK*aZewNCs6cuVrj*!NC-#Wfg1S~AG7cl-9 zxhKBV`bD=fQf@)zIT(}a_p;2C6=sFA3Is{Kjy<9=J+Rk699aiKtUQ90@0xeKS`=#fGj5+O8+lHL`7b*Mo z7m@DgIK4NWfk8J{23y5nq0jIRk%h3Ppe4i0lm5kCj3Es953_i29lT7Q{%}TaX?+LZ z4?bILLE+PAR64anb*N*nawXgq#85e)7)OV6J_Jd-iQczo-~YC?W=wY>{r1%X6_;5R zGeO~*3AP!(9{OrmC@yNe!%JGv!#&MSJMF%66#uW0%D23@Iqo8w_Cj?{2(`h(u?65c7tc5I_;A~p% zx5#ks^G|*%t{*j|#+Hk}j;4pTBR+zS@J~8RjI#b9msn~toqaH+nlapVA*GH-uZO72jwfMl3sNqyd*M)`RrcmlwuiUBhbU0u43{vnM)=&A+9sRHmO! zn$Lz#!r6Hn5IwwP+Bf9W4W0iID|v*UeEK)|KGFB@6#_ai`Xf+LIY97GI`%TfV!Lhq zVXF#%IxlQ{Rq0pjV}mGncE{5T^y$;JxmQ=_Hy`OVQr+05<7gCX;eDC>skyr3WQ<(< z*y+b^7$a1w-t9x8z^2ruUDioipQR^}8Ts4{)7D+^lh3l}e$p++Q{$zCGY656~gE|r~21U_*zzqcD_F!NMsLzBoIvyqxn zA>wr48dzG6W)RIuo~<>M_-2)(Yr-t%Bn3^q zzcT@X@_kqrd-f}Uw5+o|p={S%z!#L|jOeZu#u2roxy(U*PZ?_6s1DJw=v8qcN{iTH zdzp6ax_lP?>WVwN#{08vn&xf+V+VU^wIo5PbVL0Ce1jz3v36vZUdbrFmUgc+Iq=*a z6X&02$?(`gKV7!LrL{NGmLCy=d>CA}IGUi>ux`W*8gvrvii+pzrSW7qOG6Xm#=uxQ zWsvZ)E%T3k)f~a$)K?aYEb1b&hqn+TSJm_SqVFaOE*!>;+|Ry<)0;R<)E06#lsXR=huMjBYB1(A%2*hvaLN03P|3gL?lF(`_e1LkWX0WULFA zenGdF2DLB)b{l4s{YyIrY~&`t#CgZTFYaAee;eNU<1K>SfI+2I+%h9YoS@pecI(b0 zi^^9AE6yS20jHC{6s&TtUPBZS)0q74{Fsv%hk7e;k;)ZC^!)q$W-`vtRH*NbOv7wk zppRMdH@}OTe_Uy`(Mm4XjS8QxD3p~xFvKHiKVjW3gZ{)7w_%p`$T@)gog8Q1Bg&dRdB4+_u|?FM z&X+&rf(_$Std|}C?m1rI>aj4${+_szl`qq{L)Y{|BHX;&0@`_!t|nzb*k&}U!f{7&$}112It{|6U|ha8a~RH(|b3YOZ>e=uje( zLv)f_3KgEY@p>Rg#>prL0X?NEijwB|JV}!#)TPP#tU42)LaJqG)$C{e2UPm{H}vYP zPD}l}(p=;mXBp|-1gPbjk*`Yj3TcA$g)7?eAFj^GW1MXn1Q*Ngb>e+W#}r^|loE=- z&n>GDwYtZsO(mgOf57!uy^Ct|lSFs?Q<^7pPLFl_dbA(za#*pBe=b{n=@`>A^Y{Z_ zbi`?>1&5)0ZP|vIswZV~AI^&p?ql$!^s(eQ`{u|T*t*t#G1Cr7=s$a?L07v$$P4}} zbW-T1!C&#+SJVJe!E4K#XIpogk5IWQ-dTV+WfGz}e|@<}(|<^GlP6r{D;K2%M-{^1VyBl5_O z11{DaF?k%H6|2!7!~91b^TnbPR3;Y^Cv1n_%&y^)rvB6 z+Nm7kg~v~KZx$GUsOKCcyg=t?&~>sWA_ceNY6wABnv7$&sL-d3`zQ7*y}^YEVlAY5 zcDIo8_p}i_QcKuZ?62LvwTf_BniWR0u-a^tM!mFJ`_DlFolLXio)fK~y&B!kB{UNd zrhrcPrH`xV&Im@!3Y)8%oE9Ew;!k~I(ty2ntm4K0UBbAsQOmv9!{!u-WpEW&+Yy# z1i%PW1AhnY{bND&fDHHQhNyzbSK7&%!c^LRefwr!?4Rsc7jPU@p|u^OdD^!;g?vKF zVh>3WnkbBw!GiK*A^qsGzie$%64fzRD91(1`%t1vnWI8d{ZK4fhSb{Hbe4{QA zpfUhMwSKd7UexDOW#%n5z8nl@&M#EgxZwk#S7p&^oY4&-Og@-)Z z#C}4+7y9e|lLbvKqwUmiKsA7IX4vP|fhKN>tRj87A>tUs_vijiFoR&Fvx*3`kMh0e zgd`-eR%dy;j#>dc=c5nuk>eFAC5_I&oDPQ^*NmvQ-x+H6xmHnr`U_5Sf=#E>)82pN z8x<`)ck6E1UETi0Z`rCj(@sFaIiBPX`pSMw+P$MR@(|aA@-=wIqo?3QNt(L-SZuts z`+Ct<8S0dX2;5d)ZOA^z9+7XMz5IR;a|`^7`S=)ay)DDts&+Ygb|I82&@Su##UT@I zowDjpbNujLD^6muN&){z;%qi}zRWyr&y#7N6q8fme1^d4wVnAx|IdivK)(mT1+6YV)+{~diSC0sC1YC zi`8emJR9t0H0nQ#0C~~xX(GRQ9fdA*=Nf>AAu1k42}bqH=zAB&t<=09qltGegg(z? zi_9YpXLD93Wpcmqs=34mj94i>)X=)@Ry@ww8hcg(cDbJMA&5T$q;c?~2Gtom80lBw zMmE*i8L2iRNj^Al$T)IK@s`=7Tw5rcaa~r~#!zmaTFyM)(tM~ZHqtEu#1Zt5q#5lo z$?qsfY!QrSx!+|*l+Hj$Q1kB5OftP&dkC~b>hGYafz>W_N@{}^1`^}FrxDWOAs+{# zx;`eGekiRb@S&$3VfIBo>2B)P01t&GM0aAV#UQEg))`oBGlSFli^Mx?H1x3*YmADg zO^5#Z#^PgFJx_0x3TdWIF^6@RxFDNFni~Hn;wWviuN}&s}qCpjFiC) zpk0+rjc50)O8e9C(?eM~&M0=$VQql>+Q*QbG-*ZBFi<0CiSTk6l+naj>RE z7*7cPgX1;EyHdPMqBqS&7cexmq0xOQF=A46k+z32OMPmC?OJ;$cQQiy}f< zaR6u$9uGZuU%vpx)0Mk&2WJ~B59=@z+1u!l%h#1y8Hfe8zn9pqdg43f0HuT)UL3da zIv4)XwpG9(ILAKIk9z8h{EgkFS>&OO2$2c5)r1W75J6ATaU_Qd6!pTwtS#@MK?y&= z#en0~U9`|eFW0aZp^EuJAZKG_^QfZtr~(SmqD?kv`N$|X(%s)jQz@Y;PN-#%wuU5y)AzFgFzF&oH{1CmNl4 z;BEJt#%qKnT^3VUoYnrY5;p?AMq6MFKm!*(;($eMG{$#z}1r-IW`-vs3I5VjLHI8>WIG#)cj@%q

2@%d*EAwb#fO#O{(yByB+X%}JMeURKvC#)_zJi;vm;2Zd#SVmir3rCeNN@v zZQ>2&Jfr6@#I!fpFx3{$mp=Ql3>$^Q*0!QTz++wy`Bulfg`0!|BKR_jN~73hxGV{u z&&DKf&|O5JZadQK{nSOT3e7T{FYE1Ww9_+~J&eYgN{RQ=?REl?8&*_}M(!HH*@*O0 z@0~T%MhG1@+!R*Hx|9_$a`|c2_k0xQf!F~5H~W5DHE%ac6VyA_B*mxKv<}_4pZ(2h zQ7+Q)d)2Ucj&h;4wxsZH^Q4yoeSg)aJh#j3IV5Ec(Ch>$GjDf%3`4NW?q9C==50OU z0C-z#;2%kky;dW6ACc=MycYgn5@H^C-L}_u&O19BtG)hYePGWtaZ!a9|2jDF-Yy5S z8#3#9qv9Oygc&QN+QKnV7&GdVX%4-pRo>tFJK_6PUD`Q}RZJUtK~R1L;cm~5m8xgj zfKazh)uT4-9KCz@&=>G--oj>|x0)@f#50`i(#od(z>xcOLciaQK+?mxQ5`nm7XdTX z707Pg@;*SZfc0^Bl#j+NgmV@b*DJ4OGJ0c;pFnkrUEr?#{L*|t_IZJA(Cl&mL1pi{ z2Sy%H>p4dgQLVX3IUG}9yQB8|f=4^0&FVMH9(3L&HWoh7)-XGJ;QdFAE7&s9Xe4Lo zNGx7v>J##+)0lZks&B7FouLFka3fiMR=>WsAm?0PFj=vGWkwz^19zwO@1g!j{v0^;K7yl(uZ4c9eiHdg@3E0aX%<$>D_r;ZyGs+|8 z^H^@us(Dms_z3w)a|oD4<2Aa3w=B5Z^;8~k32@fk)Ebq6ZwP7|5NcriE%NSA!;xZjNug6o~$NPdhUaran}CWOJpc5I!+o zIn+3;f>5ddF-B|uoj0#MNXteY?vaYM!bQkt`3v=TMzHB4PRGzaKi zPy?qf=Ir^{0!MqxwaO7N|m4(cOA znYZRaZ5`fITGd&3o(YB{Lx^W|7^?%^H`uEu659aDa29z*NmoS|E4bdnS=T9M^Q86I z`v+d5FlP4b)BL?-OGh_q@0dBl2_Te(2go39)InA}7c6&w4pzts@X2E1W&1#++ihr1 ziz;9J?@^W{FEC7LXYiFAV)gI<{6&T1+6&Dr>%g<2UbI+fb^6800ncwZkzkYk}UeCwudGC7Be7YYRLcA-ni-ULyiQH`3~MTZd}!G^nuQk$1|1 zZJJb?;MP}g`cHYI-(n{Al`N)Vt&f~lL?micsWE9z$cxda`ZWetF<^!Ath*bH6HlT522V> zI|23mu+w-#zf~RxODQ)n>n|hIb0@5|04#nv*+^K{J>iaSof?sZL)WFQ}7c zls750d{e?+63%^`YhU)`7o-wG0Q|37M-{oFw&=r-s|0Gyqsu|u(3DWaW)2c8M!LH- z1684w_zO17@E&-6esN_aRmnvb`TZ^b$yyBj;uYB5lmJ1gDi^>=%hPl z4=s%g-rQp{=FgVzld4$$edvjn;vCo?`3rryh?8i zw6l0Kc=q26?{>fy?RT>yiR6&|s#1q++A9B*pcGCDi=dmks zifNuH_o~K>hQg747Z0t_cbw=+oZzlE$j&7`r{Em!iK!sGctn<~ z;&Hx#MAH2h+l_Kbmnl1NM>R0=0B!P}m)^6SDP+H5{P1w|5%6YuA^~p^YsJ)H^`W#5 zI`DUjc5w#zrG8R`cIPPUr1OX;c7y6I>qp#0@7F!5yXW+p=k|}(B|8wJJA=2pP5~>} zs*aag4l83lw@Jx`LnQVm(Y0L1RLBUq9FtE}t>98r?rw?-v`HW(bgol*k8!ekoT}w2 zp@JS9lIf&-@r(XgtGM53)f{0{YrnDg*BgAsw!|^=;m-BmBORyBkm4!}qBJ>_ne|h1 z*rg2rK?7-y($c+qJ24z(EO4gw?pb`L4gw+@T_@nMFqSi6nvN0bIGiJErfX-4%F1^($`6(Ll)fI_5Myl#dUL#_{~_ z1uaQM8VU{{W>xm8YO3s{^w%H~gT{v+Xdu^>eMAO{G^Lim^_$Ay*hpVV_^gP2uswcw zH=x?EOf4c1=+^mD4gpBpK`^fHz@83sMZKZ7VKS^4bP#gz@l;pGLq&MR<^gAwV z-W_P){8f1!3JK&7M;$QTO$GS}+TCsXC$Qmt&GC!z!}C}E{IgV1F&zu{DUzeyvGIvs zG_q-sdM4H{^p`UZkTm)5WP*LVrMYKa&}z|HLy862EqlRgMD}h72456?X#Vowc#bik zuN!m#aQ(oah206yS57I{?e2d2*_Gy~Mc%VxfWPd(q^r0u7BWPx4KC-##g>G%tk(sv zH6q!D#!hU7A4bju8;cVU;-B(ip3zeT6rgpwN+?c9T-zM#8k_6aOOpC2?K!|WB-7^k zN8!p8Dk_TeS9nNCp!ZtC!y2nBU9Ue?FE)fZX(nAjRet8SLjDTGX7y7s(aRm)I6yO3 zI1z)JVoPgR83<5HU{^d-Fj$>GoztZrO4~%#{vbonHM)1$5XHMUM~9jK1=*d-NT!QK z&$bHo+-yROZdY~qHuvXFJcYs{iBYeH@JMhTL$%Td82PYyZr8~h=NuQfV{w;=I47w!%_?B@3m+SzbSvCM;=n!VW^j3y!TVHsYZ9Ds zpD5;sP$2guh?Xb#(j~)D;-}x$rn)N)TO_JgqTZw{Wc$@FRLk26RAb&(e}DVw!A6=8 zN&~jDi1?I`7La0DAfF8RvOM|@TwyJS1MT!}_t=Z!TgDNuPj!il)9!|ChQW3pZCW#B z>rYXu$=5)co)*Tp?M0FZhm*vBc&ZettRr^M($M;w@w|1Ds|Pulr~JdR<2>KuoS53c zN`4TCV9zv{maj>xB-(k5!J)Cu3gw`J)${Lvv(4xUx@@OMp6y$ioYhC3(~Re+fSmzG zi>m^cB7ZZ24a-X@zygLc(k2XbQ3@nRS=w5nc7pu~u8YU^sde%gZ@`6e?O{XBAN#%?Zyp<;wr>%{Hr#&AE0m5MGyvi9zzsYI+51dcUdjN!#W!ngqWLKX zG9))c#VaQuDUSB`C zMZNu0=?E&f7_r5c7&Q`*5uEZ9v@l9D;BUjTq)#(99CYC9y;oBbw0=J`zvU@ObZl)^ zCurJ>$BJqK)EWLNn(9qpxVS z{(UU~h06ycV+p!K1l*17K& zM6{EF>#h!WBba@|Qbtz9P0O8d{1wmf*+F|8n_-}bK~Ii&E#;OzpSYs3HS;mZ#Vp`n zWwjND{QezubG7$wy4ML4BDRtW9S`~OsdRs;6i; z-jf>sMY%5}HiTcH@E z*Q4}WfZ;2ks9)dxy4Ot`x|wLy7=>geY*~r?*Ptj=u&(iHBf5Gv8z@NJzXKR?-|CL< zZ|3&M^ewgzx(8aNP-Dck;{%@EjVMN0Fi3{u<8^z>1hT$!=-vVef2HyNg!SM5Dtl56 zp+ev46tNqcAM2WpU%t=JR<=uV)PV1iw0pk0YvJZsxDJ-bwyBm}fuiw2v2V@o#3OPj zbFtP4@XxaZ3-s-L5K<#?t-qhJujvmbNWHQA*Ky8+uJkT*Hw5OKtW|9*2-%FoYjPAG( zW#!Lawv98b*sH+OF*}*CcNm6_*#(*9^fn09-3E$2OGOQ&H?H! zG4p|&0VN`X;-NpqF@q1Z0b%ico$dhN}zF#tD;?L;Q38-C)I3;Xt13oIz!4m=JMu}`+0;T(}vWma`CSh z)-%(7p-bN#XXIB(T@AJOcodZ~J8D*oh?QWtV5 zCIqEq=irEZpl>rvj!^CFFmGNdRO|HJij*P&9*+%>fhUz{0A1Z?e*>|MJ8f0+K_|db zYhi$*Ihkh*sZMbUU`hobrG1hd$!X zw09K25ttbUo%_Qx&j+vx(LyuPOO+X77db*)M`aZp`m+hV=LSXkMFuB|;lcoW2T2Jz#*H(Zl$zhwn(0D+EVhoGucP z9KGhPkD~W07TyL6e#Gm{atq;O#bVEgAmd_Z$(r!E*f_EA!p6H9C&(>ZBtyk6l<|`e z|1VqNqB^4K%X62}-VtO}xdyzXg(vhu0C}u3Uyy;^mC#2)4hM3>FCFIe1k%s zw@ykq)$ES4v`8?iNB>@Ecwdx2kFBm6X6`fvxM+Y+rO&@t9C*A+_k9V77LPv`xE3!^ zrUN*qWf^VE1qDQX=zLL;TWf}4`>YN5`Lt6g@& z+e1myYBo&*2MX%!7q6njjolwJKr8ElT!z)`ND^hImbJz6oVseX5*f+~BtUfKY)amA z7aN?&%t4KJ-`M#JBQeAzQ4o95ji*=SZR}pGzZl1^K98muBrBQ0l6GkDt)98QPv~5j z3Z7y_WYq|0=OC)rmJicJ9l5ZPGHizAN!y92mQEim8gsa)Md`kp`_7-vEerBewli%R zd2HrcGj_*BIQvu@aF+MMfNx5+o)v(e3aPb{E=h7OE7eKzC^Q_dMAz(_O8f?Mro|vF zZ?73#$ss4`?-3Gkg0rbW05SZ#_^R#&%exJp+($n1fekVFz+vTthS-MP%*9NAP&w(& z1uoQkVPJRCxElO8103(_!s*i7y}_oE^oI9dyl>`^e8lF=ZCZCHj9A51FQ~rI;FM44 zxSY?fo~|lEg!2{M)~1uPjQ0=Cp3p=m^c0lmRcaC~SO zSl#zOLyf?mGL($sIN(Qw!}g1Lt%FQ#;})a^)+tX=;kvEb$<|F~S#{I4#Axb4eEu%% zIK6nc!`5Dl5sMhez!LSM2~ET4jJ^BdyQdEU_(72xL`PYfCi8vN?36DL=F7To-vWWY zwYxpA|7q}F&P2))$Aoof(@V6Ozs)KOAff@_*d-RHeh`VgnG?v8%%PAewb{2Z)J zNN_ppsotzbQj`yd%Gcw<2z-_(E2+Ei7#9E<5Rs%8z=zsS1)=5aCA_!o;nZpG9e5%f z2yH$w=jMXQe|48LuqNSsPbPKuQ%;Cy%fIXy-Z=`(>;G3G+-U4p-#bkELs?k{cljS8 z7QG1&WH-QSVOZ;~{|**8hPk}>a#`IfcbP^LRk(Pb1R)TGBXuIs%-gI75mw>mNmstc zCMHFld*`*cN9X{2obz5cByO=W<)__Vrzx=42pPdGZ-srmch(R@3Kd1)=+(hx+F#js zNUvUF+>D@G|`)hW(EjPI3Z8i^(*3t+BgRL4=Ttq(>%o=1zcTD?mL{whRs; zFTh2Og0_CxbNd?acMFsCC#n9AIw6SAop9@KrP_$HmF<4D9vqvV)rh}c`+#~Q+g;(- z3?`Sv!}d0Vk3Y{Bf}S*Q85nm7FVVQ}CY^t6?DSRK6z;AS^syLucf0!jY3fX2o`s0q zpM~pA2+qmu#}7b4m5oju!o$_xWO)dB69N#bwzTFRK^tVyG6S_%0@(zwm2ch2Gp49t z<_zX%=w-IUSeFF1wazyPp@c*W;n$wP_<1sa?2f8qTSQ1yT zmNuLbn!MkCdvs_}sC3%SrXzyorq#VKvgBQw@n%u8Lv;;)oX&Mz;%dQD-$!X~VVa#4 zPGcI$t~81}IcI&u({-D%bS5lcpPc=lKS( zPtl49;m5N>L?!fOT40FlyrXC-==^yh@!6f;0!8ghH>*)Dp+a8k6Y zwiSf)bB0!ye8*4_ty%Q6@VU0}yWY@`J|_?x^{3py1;a?yv7!xlJ(W?d_szVM7py{u zxJN@`8T3e8NuUil_io>YA6BBI8UGm25J{ZaGSaT0T*UAA|62snp;GEalb^HNioZ_Q zg)Hzc<)K^*GfC-4^vNILQq{}Om(guy|0tz&$Syb5C$hmB$Mi=}Hp5+H53fp)E2&rC zojZNwhm40mS`qTscMsAcF=yB|OG5kpFJvIv$mE@E>?Xt-rt?C9)%9gR-vRykCd2Wd1kMWX-{JrVb|axx2(t?Vqtkw zo-KP+)0PREmKg==6D)LHw37kq)#WY{_irETZ42RiSekIp3&$u;K9%tSjx4@KcvNw;;+f;mb)n5Nsb&z&!ef z?`$gKaE2g|V=Zvh?>O}P^xa0llDA{@{P>E?b0*rfY=NM}~2hUow;UXav?=>LYgMuG-o>IJG|B zDI)e3S6JaL_vm`^u-QL(;CO|w*V){4L93Q6dZKnW7p_=#{KtveTI2Zr9%W2?Ji1tH z?LO=~=e-157Jb*FL_(p3y#yN~2<#wmVpQfV(E}FR>+a{qB9JyvaFGu*k`NRDcCOuj?||ic%>q?}|JI|%U7wR4eaYR9 ziu=H!AW3m=AKuex9%@>V3_sLV)uMQ9ei_fIrd8X#(C0hKQ+S&NJwAzdW~}kvxU-^F zW7rckAQF^D_;GL$E{HbYi9v*&Cm+4t8GxBrmF511WIuBt{~nZ-v+3`tS-q>q7VxB& zU2RV&ddB6G-%PJXgxOGHWXtwqMHX3dqOP5DQoSti()Q3`A-F}JJpW!(K@-2_ooH{I zFx0x`fak#cqNjxgQ;h4`j~0Ib>R31(RNE7z8Q?*1-jxMhAMh8T1k2C$Yp*97Wyz@% zZ_Dm@ADiu2jY?%BC3nbc2T#D#pT4mZ1<*!ks2I79J}SW5AxH?OPd2%3FO$^ z)=|DoV4KfGF=xBRQqDEoxsJQW0`QuW-4<+WIjS7LR-=Y@jK!4(u1!L`O8C~=6M}0Y zEG@^%zns$6Yl=(ryjl_X^cwgqD6wucGj&NPxT}FeRz#fU4H%teejD2;+@0fngS6Rw z2>;Sxccv0bUzGNHw?74|Mz7pWMbOv{XAQ3%mQg;C-C*C9%TuwY2oW_9FTSSo!CH`0 z??T0$tB;`&AJ_Z;w0c#(M`OM*t|-66K<@rXEx)bkCqN!TV$OLDqSBeiJy`ULvDo4Ah#b(3k$v z0f6i1_qRt-HOKRFcqRt?Nw7G%nqrFx-KqkwWi@Kb@n_v0UJDZZ(EOAW)P z=!{sNwyjJ(#r4obAvy|C%7MAwy;=65x?8lx z8!;3<(3j;@ zmt$abX^JKte(^RW%R>dg)``E!Nh{~X@_=L0G z4L=+)B_#4zsnvgIkbEd$$FMyb&RtlB$v)Lwhx!fq0)uqz$v~$AM?&k5& z;d2*Vgj2{x&Cv(({~_u;`{CD?7jCM5yX@K^SpT8=4I~tlbqLieZR+nw)sBDCL%f&!>F#NJT%Ik z$wUy_F?fhROK!++$$(WxVEA(&1!-dU43MES4|AfOpAW{|>UyF$9?T%#)cz5iq6N29 z@kW>S>XJEjO~Df^6R4okfA+NV74z5o;{m1l439)AGArDH{#G3W64&2uXsrB#EyL6m z0;oJjy!y`GxCWe)_CeMZ+dQX) zC2_pcVGV`)Gu|^Al|U+i&tLC1tAEt?-o6~fho!j(%gj`;O4v-`4uP)>#zMV*tKz&Q z_C*zgYnIIEpAx;qS1nnR$@!i0Sh3Lft%x78pI7vl`e|0o-2=-}MW0+vhiR%NB5GqV z_?svBtcg1hNUiG!&DP?$&+VV3`HQpn&K=gKe6+2P9oBMHMulu4_Na~YkA5*OsH+>u=#M50OFtAm2=)65qd08V`CH{oC?M`53kd^nu+{9Y5vE0 zCM@%v+j>a!>rVLi27sb8`h~O_!kJ5hUg9*OJcBMlZY_-ho&8-_<{~%x`{$F}C2AS$ z&rDn(J2a~jOPx_hc)=W^mC#O06Hb9b>1RIn*#Kxh1+M^bQ!1jOiu<09qc`YOvg=nP z$I4!`9mjiBzxcVA$Efldc6iHL0O)-CQY+nsb^IqlBewySw9o`>Gh{mwa{kh{C8iH~ zWC!KGw#{7bNQe_D-dJAJ(dma&TZSZasqhmhwpdml2Nj=|G}i*~L(ccoN&@VJDmr3X zPln#*5EXs%q`xG`%kPt#rPPZSuvc-Xxg1oBTf^Y6Aq$1fn{%?AckHZm4-L%dy4LU5 zCaGPY2&FkcA;kJlMWuAJUu~0qh_h@Mk)`}LF~|ZS!X}-PM=%_cLo&gTYIc}pFMKxVT=KPgy^y+5#>>R9#Xb~DPP>X#g*C-M7+yg_eNj? zd38IK?S+7GnU#jmjE(%er!9foc-V@`uACp`6*`#>S~n2n(f$0WZ!#C{U=cElo}ErOx(^)5 zD_!F@NN|{Yd!>yoQ9$C=$ZSf9xFF2dGtoH1s};3gUdG3v_q}#_siBQ`d`<^XS8^rj zZ6DSA-W9WTok0ruy$3@US~FYN&AkQX8#LVtRuu`2~S1ym4;^p=b~*capk;zR2A$tF@XQty#8H><*a?hHexTH zvVb~ZO7`MB)%I6SzC(Dj3nstW5P^5t4?-zHD4g8WKy2-+&&9Z4i0Vpz(0$;P<* z!t5c41-3K~tC`Z<$DC=kB84iksKPm4Nzcn7eM4VP@OX%I?Cne6>w`|gTX-Bg@TVv} z30L=EQ)Y8RXj3YnXKj5K7M^wfY267Vlj+gfmcqBGGs!gOy~g>$a2`MqQtAugw1i@vhidhsc~-~i_KxKOFK3e1x&18WU$A+)ugut7lms2p zD*Uc6AB~IuC&$NNW&}O*3V8UbiZ%ME_#1ENSKKYdZMx@cBZD&cfnC%a&q0``&steo zOGoBuP5l=lO)E8L;nO0dnjsvA4r@{H$kH-V-vx>NF@7{Gi zgx%=MKvJI_KW=|D`m-Ez-Vf;{VM1K9__YIUc>0MnIe`tQv^DIuoJrllWV~>HuzA2a%RMRHRD8TM($s_v3YL4U%X_7GiFnRr?W*aa1Yh-=GK9um7gireG z*HbZ1*|2x^`xlA5`u{vu*xiGF2RrcVlav63_i_IlEkpiyq0aLE8%Bt;qZTts_z@4u zaR5iPtnH5FH~Cx*04-*(LSWUq6zl>hz4*4~B~yGkEptb3&O3f}U&)7oXQ7gi)?BtH zB+6ZwDUGDTMCh`IhtMNg^w(NVM)2EOK8u6>zsXvn4eKyaV9?2Dr4mQPL3r=0KwJWm zg7eR@Af^k@`OD(voquKVpDvk?!sWyC`Jyy9#+zgY*Z%ADmBTewO7UEjNo9Duo$IK) zU9tll2iezRD22g}JnWIkfbh;w!hIK{Mv}aehwf~2Im+YkjGw`OWwMwT|86(rJR>H> z{7AhQu$Wi;K-EvwmcHURP$&JD*IN*8AZg^(U}a_<&nBC ziXL&h0{FGfKGg7Q`u8$03^0US8xL%?I{$H#mneyBq8Z50+p#UU%8!3RW8}i(ZH#WM z`IiUkV-TCnChD7}I*Yoyf1z?fj>n}?qwi%oyA1kCmN56K@v1lF87P<<)7bcxO_P7! z!<+ajS$@CS5vC}Q5QJ`oUcL2QP6K7Up8F18Qn%GHnB3k5;e`Rm2L^+f^!Nf^+s6fa zeDrfm*&j^f^e2?1LU{6Gj@O*lC($ol4&AuzD&_>)pENrr%M1aO?R8!vcx8}A z*EhC4;S1(yGJ) z%BytmWnBkDILoqS17!x#WbQV~<8)#jLrUw4$t{Z?g5H%wT#L*rk=$A*oW}E2?|sh zgg2VN^*eoAyGzSs>}g>~7IdagvN$3*$>CfTL<8|s&65>A>ki}ZaOiR{-!@4L=We1xqrKy=MPaZ zdQ!z#Pt;ub7<V+SH;yxuTDsK?8KpfLL{mbxqQwXuB)D;qJX_pb=nZ9+`dA(AbkxGeFJ21R z*WJ+d7Z<-DeE1LgQ}qrcHX%AK8u=9oyo&FA=+4u`bM;K}>P@~-0(3SdGU1A@ebq>N z1w7lwFd+J_yJOHrf{g6?__3zib5s7K>t8u0>iFz{udl0D9iGn67K@L6cTPpjL8~__ zu7t9|gR{{deU$3{dS-N*1No~!E1qt?_Tr)iEMyUFDptTXlZ0!D^XI32mO>0vI9ZQc zIKFqxKE-{X$*SR5kx%WTyNK_^KSQY9G@Fd8kTU8Gg2mOniYWXQ)x($I3aaU%A%LD=V1>QwV*C!phYv(n&Klt^Z>QpC z9GnRSWd_st!qc>(_jI4D)r@c<%deuoDdy>2-4|&pj5-^qK$nDVEVmUgd5~-?WH#WV zJ2~0#W`hZTbF)KQ9)LZ!jF|G+bQ;b67%#W3X#C%fsF&F<#3FKM{2w9C!@S6|Z+0*Z z*0QpzTRMFwKb$i)7W#W?;rUMiF}fStT!rmY8v1xeX!{r1bC6+{PoZEK@aR1BFb!L+ zH&)Q>Jo>rlK(nKEy1z8GQsjad9o6bnZ|3wV>~dXo`bYWU>QNej*!9_4dJn6n`^phpH)V>F~9R%o#NNY$|~XaB3%!p2xq zuZrd!tC$}ACdU-9Vo3Ge2CL|nV{$^r47(6NizZ+5-IV3|YV$>TNzkLSkVj|Y%_jXm z7IScWY6SOnD1Vw;byGo0x+hZ@>h($2Y=s;_mxUYSxYk7z9d;Qtndy-aM#~H48-?ad>m1!G|T}Q>- ze{IR~M^8sUUZYX;O@qRG>W?F<_ny1Kah`)gXV$<8zIqJ8@?o4}N@WlARFC5GP6N_q z!H;Q*CEkEi+t`3;{{Q60j)N3+^xi`rz8_WK5OE_{{EldlUQS%SxHZ{cnNec!3^&hf z;W#&}Ks(b-O3C}emIg{a9+z8GkYmcnd~yZ;-ZX%5Bsfr>#!vxPCYJbb&BW0SnxqQW zzT`OcW8>#HvgLXlqZ3z9*_7U{Gf(pQqB8$UFq#&B=e=B^KvQ}_04u!gxw!HSWoSO5 zmx_MC3TL3SE*ITjk$rn)aGve+HC|>rjD~SHo1V?k!~Plbv|M z{mwJ42d{~TQw@h+7yA$JMIhorcbx{8xBxGtnkDKXBSkqG^UdQmt38U){ilg@VJ!T2 zU$tg5d=BZ?y0MF@t&kx-cLwG?0sW^}RLK7T7&e>WLx?6Rg44Xk+0jIPXxJy)Z!T#7 zyDI6?X;RY9G`rk@ zw^i9Ri5c$etW+Tg^;gLGeJI~8pG}qv;$G^ncc(gU`Ypp}tHIiug@4~gt&+~W*gsh6 zG#i7r6Xa#QEepDZ+-Ro60X%iiM1Ojz%iHR!)ZQ!K9=X_m_wMjVJQsH|kTB1KX{K{K zDJ4vul=}p}TE*Rx?fSIZks^ceU&MF8Y@*vo!M$Ui@pZ1@>y?~BC{=#Ic&1AM zPU6x%&f@DSPh)c)IhR^!aPQA^p~zux2ktg615(3#d#RnJF&6cL-fWiqM3)y8$sqLW z3AZa?Ve0>X&53tnbi7a-!P5bdLT=VWN7swes_+E(o zXLNWW4`t0O&q>~Uxi;`crPWxO2#49P*9T#GI7J$&}@!i8G+k5DFmZUS?UMoS2&kam+%>o3i`8lA@IpJ(Cv^jMr~5I3^y^rlg7`Q zu2k+s@I=j%z{LBjnAL6LJ{Cii>Wo@FMMiVH{IGj8UY_4y!CPp(g+l42RwTgJN| z_H<_+i9v9*ljsNU>Y2**&ClMwE?9cED*=j+jiJ@b3VWlA`f^Tns3inOoy$fQFV{p@ zD$Way9VJ|&@K9!@hPUB+NKui078=ouMT9sd@99nzgeGIHf${}K^I_1I|8g15J-|%c z>L~`|q@s$B`T3WFy;P|Ei##Rw{e2;O;N#`jbqjD6)r|T{o~wj5zqRCgGd{2;`owrD z=;4~0wX!iA=3fw-dJygwknnwPV5At3q%l02k*hw=VcLTY2HjAJuncXMCi9$@QY=*A zIPjIMhB6~0Q#ZM8j51tu1cMn%?LT7^=#(VC&Afc>+FBh5vvS{Tc_{E1+lTx_m;i3f zp)4?yC+UE#fQ2>~36Hz)a+l@hWg=KZz0V_Qrg#KKVu64282Ct;V*W?+m$>SL88&jd z#SH#>jO>E6mr1z&SP4x{ct={;t=9Yspmk{-k6@_iKbEuAPa{LTnTDV>ERx9Md#fKu z^d!75=WVOi5t)3RM(#Tp6i;}wXYTeOBkJ})j+V>)z$$$x@u^$@r{?nprW`|q_xph% zJF0i;kC068%`ZwAopXHqu^ho@jgzL<%1&%~jZbkb*dx01gJ}CtMfTzTJ_HX0J@I9W zw|Xc)xa$#TbXYX`RnD9Dx7WcLeC}8Gokst3p4993!0(5|J{Cmdde{n?(?s4+aw02~ zA>7&eKAUPsF`eyRnY?`1 zu^c`^Bm);3(1TOW$ll{B0}xLpv3+ zM96TxM@taZm;i)p6>2;7a{72e*3)~2WpuCf@v@qizDI+76&Pp0xwDp3x=gtl9m9WU zb>>95bCYhf7oF8@k|ud_KZ{_4Y29h|#$ zys?(6L2fx)(?)DagN%E>{HcHROJtbRt0W%MYOVcFii4kmeZr1;F3(Hl+LdXRfOsD0 z3>EFy!CQ-v|YxZBR zjZ<5Y0Fl!ty0m8RK}u!EJ`|g=_}l*xM;!V_7%Zk>dN(_N#r_#sH`3|wCB!3XFSN{E zK1w|XqNS)*Xzw{kOxX#VSlu8{!j9EmhR5;BXMXkuj2Fv4h4`uN${800to(cP)5{k> z2^q@=ZnlJK#(kSLgb{+QVpo0*_hfZ$z%98qvT?BERUh;vN94fCfTZZbh!H%q(Y^wP zlUgHliDu~3$ROAdKjn%0Qh6#VgvULw-N34Xw#T8W(2;MbIDz8k=4~W^6M@Q(;hFpEa7#gBU(L4!6>uSO56(lopB!*?~oL?%-MvY zd+E8=0YP^6MWC22RENhN7S4foEOToOEGB=q-P>_J8p8^M zF=rgZ?#m|dG3G%3c7S6$<&ct_=BW7GS!-zUF%HyM2@@>?zVW)?f^OA`EO79_8k4`9 z|Iu?mt6TgDtETB(FNYUw+|z}#blvirEeqHLE$?)^Uu4Zsj}gROu<&ns7ugKnzV;Jh z?Z`CwI9e*LsaW{_K#ZZ|65AE z`aTlU@NkWg0Al>>%8c=uOu}jNtm2=->`-?#gMP3nl~o141rqw=&$P2GIFxgg5hs3O z5xE8>?z)-(ZYULp(+XWg6JcvF_<$Q9`7Buy_nuOomKrm@i%}jcE(iWRqOxh<{?qAs zrO$o7`g(%!^zVGwBd_1Eop-;cjr)@FC6j@@gxQY@J2FV}G7BJ`UA%X)8U&l~&Xb+z zwp@x6c^Xp4T65d59S&0iF@^XOeyxy3`PQQW78=fT> z$Mdk09I4YN`;GGCCfy~6;r)K$K5vaYuu~N-mkFq9ZG$RI%NFGE?;+SMxlM-8F+7%s zhH+(+JYo}6!@yd_FID#C&V=f;SFPooz1nF71*N2DR!%FgKnF@vIX5-V)@#{iX zXv-Mw`d)CitC^cNXQkKsG%Oii-FZ)uIlit`@vx#NG@8>N(zw_uGs^Q;I1ZXwxK zSZ!fcjtuK5>B~uUDvl+m2_cK1G{!l%(`^+-}Y^-ODZpz%IPY_v!#Y!rqqIz;|}(OFF(suk`E@_;C85 zb{PL^oZDCM{MNCPUPOvTl4ug*b(*kp*miM=gsXG*&zrP>kSnnQm{m2`{+N# z6_0|wzVVWRC2h&PUhx)11}29rEqTAyvUi!`eGjBNs2Z&~3SeE2@7$&LfU#d(ns4{W zSuu24Yej&yvbMFJ3`5(6S);}(1>i_)pQI{?6 zs^QA1aaROtlv`{w)`I(&680TPCPRi}m;;_b89!K>SZ$u=t$%!?<4Jk~n8B1pTDTNY z#&NjnN;KCM^yaEFMcY*O#&&nTo++V23LiC5PjH=x(4eSBRNa(JAa@i`L# z6+Z?P;tiBfRj1;y;5XnQytME#M{v%pGL0PeTb&!Z0_;fg6#4FQC zIYXd8jE#!L*^KX=g*P$zFtDmw#LJ;6X?#}z$AHmk+(?G}Z_Ez`Lhk_$v;ga%Q6GA1 z#Y}#iAXc%(Wd-z&7Fh1aFulPKz4+o3QcGe=WdG%RpX+iwASe38lfV1B{?W=+!IPjE zNJ*7z$>CQqFFHFtE-A_EXA_5W88s*3eA#Z%JnJc&Nh8Sq2D(&}b(0ZOzR4?2wanP= zn%!!)FWzk0Im$baEJ{|8ugC*BV9MSS4>Q&CVS@c@5@VxBzs}6R_K!p&bP>Yl9uyY` zxi1+VUmN(ITSGOqxKw0n_3z%=U68s~$*6FU={r`GZxqSIocm=b9L#eon~g2+_@RUR zeeWfMp7(f!loIGItIcsznlS1SpW>`UI@7gRudaDctjv!M!{y~N;xYZ@4{%$)d|k;h z{Z%HH6Ey3Z89Bc-)H`zQA+|wfeZEJ+eWjD0qPbgt)?;vm{G2wLXI?BF{Le~(x!^-Y*4S`n1=!J*5^-Htq2>qq#dA z4ty6Pu9DMS{9v0`*D~Qts*80{x*vCNe}HOWkSv3t{8Ew)XCsJ*RA(&OO8DV5hIn>d zDc<-{OED`2!VL}cr6I%f!;&_EXD>NNdQa{mOUJh9GvbmhDR@eBoi^&WY8HP}Hrie` zGQF(}S8BMKq@o@#NWLhfb_b*W_tjR3k1R0#S-#hFK|QTgR+o@oN%te}2U&j##>7e` z{NqQyL&O-iF;5lg_=o9~y*)CRnD$RdC+3EyTs5ISWW$R92ncc;GfJE^e>YbuZ=0N4 zx7m9qF(jlfA=^(9-7im(N9TJidzpET=-RS09a{DMz*1?lJa2%nmFHtfbx=uHsD>go z;lsfqGB`f;oiF`CW~Vm>=PEf*oN6KD2y1X6^zK0Ul}yeSRe2Be?}h4+8E|({=zO&` z415X>*P;dNtAqkOhfW%5sUfl+fnO8>4IW@mo0W`+T9qC;F4#{y?uwnaAyTjM&Ais% z%&dEPZZ~^*UL~Hsa$0|?ETEfluf-$eAo+ymjJ3{{^uFMR5U!zWqD1u4V%(9{PY90-ATv!I7Xv5qBy<5JUpHUod zX7w~+nV3cj&1o)iIqXZjbb*wAso+q)t!^Pj)#196a}_bpZ9^4yO+ zz_!mG{ejUnSzdCC&{sbR3Y(CluWc8ldWr1E$n%RzO%-C@Vd0#wz{c^ z_rnL+ z!6YhH$|xpwN3XAB;x2tHxWtaIBpFa;gtP_b5nozuj~`!%QV&3Dg}=Lv%_E79&ZQYV zLrdBUINnKJYo<>>q1m7LNjd+T#oP>hm?5(g2B_2LvI^LIB3hx6<{hVXjnP8<$7=M+ zE6JU6`RaKW$tnc6%Zn%M$6bC`D61%cK*DA^`ICcO8%@r8LR3m%uF^-4Cp6i_3PD8b z4j^8~a@|FwQ~tA)?Fv!fqh@U(0qe%wc)^tEtcp@#w2>_e1E#rj9rtg=J^%?w%L0!+ zeXk!Ndd|uHaIwlxvlje-eZbNH;sGq31DC0>&R-op{s(rH{UJFy9_%VSs{eg-p2ZZ=FJ|Qr+J2fH3-l6 zAu5**9FDs3xmi#CPA>~sm6M%Mdf{%;`%kS-ll=k;z1L8C*RR`!BxAh5UACUINuP8S zjlbs8F%xihd-e>b5bkZJU9_VH;Zk#XL$4tGBSqER_0xkLiFgxGTwfJEWIjc>N7j1c_fi7p7jL7C z-XHG6*9NZAE2WHYR6HD1D!QY((jJB?joroXrRQ=OylOE9GD7+^;u@j8o2q{wDR=On zgiu}0$#J(YFOT4T;Yf#qVl0aW}C?2)L7Jd<#D@mh_`4k3^m1%Zh+H)t zv$A!ch58rKqN|n+CW9%3k963~hiC-fc2B<1wl2(t>%#(=4g-bdQ8KR0a0RtLXy4_= znmmsy;``0B_nU9c`b!hfV=F28n<*{^iSgNUMh2*P*DdE^f&hAz2ZZ}*Lnl+csz`m! z2P57$L{usB@dUH`%e<_Fg9q%Ft+}%$hkYX53AI_-9KJu|mhRRc^F+wY#g?gb+VKv` zG}F&T#yXb*5b~VRZQX!H>B;>ljB~&$uYG9fP zuWgN2+ZWRs*?m`MZBFOaMA~1RP$q?`6e#l(HI-?Hb|@=PKCtV75+o?$x{4!MxtlUt z7dkF)&nYIxDPZZi*!9!O<-xwtlkc5UZq(FUToLu|x*!zPCKOR?+&+}{aI-YB^Aion zH8NjnjBnSyx(m6Sb6TY@NA}d82_EQq zzE8f%HBYwK&EK`=Txk&2Nnb!o5;tdjFnC({Rl&0J^<{n*j`$+``BvZ2Ps|`xcJZR0wmw450wwJr|eMR&(iFc5!bhq8a_?Yd0ivmK}alhj^dik+4C^@o*mSl z%7VX(x?09bY1ND<<-U33jI#GoKGuC)o1oO_voEwf2(3ump>eFcj&lh7T*TJTBx^XdyLWiF zDk8FP?xtza-9c^v{wZ8h^YEQgSw{3ixzkdyfgoMA4;!)$hezyk=iQGCLbHHPktWZs z0i3!e)cr|b*Rw>Ki`{=t@^o_p8R~xn zrAJ)9`0oE$L&MEJSMxyVCH32lQU1?nz*m8Es$h-JQJ3A zCHjoGZ0x|U8PCUfE*l%hA;WvZx3L&0Gh$IP*b~e#MT-cY|K7*i8C(ePv?vG}ws-hz zP>>>DLYz-#5Y=V*_KOt=8mMq$-`#rs+43B#=bYb}Rkb*j^6uWxG@7n&<~v_;`4F^2 zFls*oD7^07txz!>&6>2+Ja?KHSAwWDf{y6L3TEh*^^1?BU?1_4usoaXlLc$P0hew` zLbDq8&<8bv-H&EOBg7M#{c-W_M1${7BlMyU(`ecDQ2V!CVzaxaZwiAwvp4vl;w`Ho zCG%>jen&E~&Ji&Q0-Uvx31xYV=2>Y!C??jz@6;aiKAF8oKjGqChE!A_tIoUW+k1a{ zdZCSmI0&Yepy{%vjO~4l$R>{Sy!Fr1HwG+DCU5bsrxYyH`;{BQfUVw3lB#fd_z%EJ zSGn}4X{^H0s67z-2hK^-)1 zFH6;W!~cz>mVIX>kl7U+73yp?BOZ0a|FS(SXwtrqrP^)?E>wA+7RA4+WGOK#YtVW^ z@AUaQK&shAMt-gFR)CVx?ME#%o8dTP2g@ga+1?yovh{gH=;A1%`Z+8DOwl)aj-3XJ zd9yW&vLfanoI}L})Nt%-5Wjr8Be=)_IWwU9cgu#pf*Y}HQ5OZ;1jy9_cH5-<={Z1gEOh_|J{pA)dbyAbAF8rnuR=M+v z6U#uLKF;|ypIqZ6oa5C%VQ?W*2FGwHd6i@oGqc=4b0}R5^8!+$Q)!fl!hJQCo%!ar z5#=|n6gxd;_+L5q$Z_(`ltp62LmrMY4w$!)EUF_1$qbFA0IBmn$S4=fG3VMxoNojM zPsP`mg^V08*3TW8QuzmR z?CM4&<5F15aJ&~O4GFg7bcY<@>(t3Ai_NytI`rw$Bc3)<<5#QYr7zl^P7Vsrgn&D( zdc=P}z;hUw|A|>F<^#%eg*85bzv^^5IE-*iJDj78;x)ufomlX|8f{ zRj(s?$l>lRyR9&k0bA8&+SZqqCdTL6JrLq6u2AJ9M4fQ_Gf_$5aUC|DTje>+bH7e& zrnuz3&IHT*XDezxQZt6GK_$vlK^APUzgw<0ZnZ0C#m3DAzMzwrE|HWMEN8yr2( zIDa-<>K+?VXz&VT7M0G{)`VQ`#&l-jo>9z!8cSr%P&G(pX+a;ZB>elqa@B0^O&OA$MX$l}y0MgPA53fPL0jE__m`^eNb)!ziA2l$ zC3zs=fO1H}0RNOO?C({_@`v7lqvBda@Mski9PG7y8>I1aTT3ND;bW1!F0vvmoWE$v zY;nDO?RBG~UtimfEdSa>$Sg$B+Dsp#bkI&l(5?&wtJN9Pn)b`83>;Nd>J4Y!1Fa7NMYqZ9d5TXYm2Q8 zeD~J)D`oerSJ@bctUU3q{B5ET{pLq+cJ~|CYeF%USW@{*sB@3_6n~GEub*-Rf5&@? zzMcZZxW{^<%R8eTfiNYYW%$x$98J_-PFj5^x!L}V;B|%k?h%JG?6`d?ardq9bY9*PnKi$bHxxeGSoY;>-74LXin5j(O&>te)4 zjrT6E`9EKp%MeH1a(R2^Ds>jKYqYIFX>mq9S1ycnzOPhgQ?btNy6ulLZ?7I$keaoR-6-kzwqLYoQGP z#n*YyGm-mD@BW#DmS5_s6SYRU$2vpk2HRZ3oQH*jHgpRml;F=cs8>F3X?;n!&P@&s7^=rdhhvk5Ob? ze9RUP%&dwz)yaxQ@J}A{XC&e}QgaKN?WQZMv^1V>?RiObNsuT-jajtlI*AI@4k)mJU0*zQti{W8S2g&kzE|-ly z56$VX0yZ1#@d<^!v}Uv7;*H7U&$B@oIz_kM6ArXjc%w?n>7Pyq^rvktgh5)pQ;b}3 zE1M6ap$=o~9ph})t@~L^GZ%me4GVSJ>~eIl=YdA14He?_Z}7YHppQG7V`JYa{sJyC zoX>?})q<=!^yadFL)@1K3z^?`l-2I+sjm4i#_A^gCv_&5i)3B1>$ri!;AvckE5}Q& z(moRN_79{3-9BpQe8<-MY)1#A_iz8_x*@_XM%(|uuns%WPm~x)IY=@rk+eg1)$jPt zNMp%(?&&=D#;^bCpn3h49N1gzO$l2vL9dUar3Q$Z$nruvNxhC z`$gxVk1qImXMPJBUPeq_hK_{!={fo@5B#wwxESQ_vU9wZI)Vd z3Pq2ObNW3`)sV(IzWptYt6KSHhkeq)4@j6YRI#+3FOAJ;jPDBlUhgk-tCK@luK98^ z{XNRr$BQvt^Rr%RoiF71vgD0voJJTZ+JUHfAhGFxEOwHH=K|MVN8+(>^|~l~kVKe% zflB|sR&$20AnhyH*bCO)*hoGji{fBo@;Ydp4{(0Qlvhf|KWF~RC>CJoMsDgg7$VLt z?Nypvw0R|p3~pnQp>6tcZT{|A-&2tt$gxCR6r>Zf-lDOMiWlmIZ<3#|m+J>Zk|;C0 zsaDn8clbs~%aZYvCzgn`&qm2{H+idk1M7}hSkk9*vXBBm1gg=zZ(0! zL~+Rk6yn9u?B|%BC?5uzd9b=CnUJv0eje{!pd|^F4>p%ul-`9j0SJE;nU+k&Nqff! zW%oK-au0$pZkchS-|38g)Oj;Sp7*H~*uKv76(wJ=O2e`7(eV_~Z%>{^GivP)G2FZT zCjsL2_8-vKv@u+Wiy87jL%8K)pqp~hSoqBBcMcqc)9@Om;5rS0YO4ImMX0iG4y446 z6(nzsrS}5@1~mRSQ8xc|DwoOihbM;>0)TZy#d|J|dBo{)kqf3$qXxZB?JG!p% zzQwk~`UJz&_=jUR7m>JqYF_WuQ}9Ra5tlXPuDc0Ssc|uPLoY(WJ-xfg!XT7q{b~3H0f@T|G1ewA9Jm-IZc^QN4+w0=bFsOO*l0==7611 zr|!p#EQ?0$H(N2fs}Ct$<6PTSH_4B)cEVAF@ylkWl?ywWX;2FO#On%S|HRj0=gBuVEEq*7-nPtUlQ zxS?wqOaDCb0#PnA@x}e_nIEb8-U?to^o&Reii<;#JI-Dz*h%5W;|8#PGmqm?w4iU$ z(BQJYXmWC}D?YJ0^hD8-E;NNiqwnUs&=DMiP4(-h{JBZ(yYOW4Qmt`YJ7S49Yo`rf*Z!*Yka6j@xjaG)-lV zG25u(g2V>QkEALFmm(*>d->oGtS34B3QEX7e3%fU-%KHh1Qh)cK^1Xs7vIwLSGQLS z04^J55vB&>ieDTaMu1!C4;@~wvxh1H`{W~{3EG!|TkT;bbcmI1pPt0az^77i2O9kC zbcn&R&S}5AbfF?8+$~OUxSyzu$I*%`?AwCS?VRho z$Ln6tAEkC;*}o7q|Hem&XX?>Xqw~S1*uFy9N#0Jr%x2+OKl*X}f@OoR!r^=FCl>Kh zDhJFi)Ia|ZO=lg|^#6u^6_pZ(q9Otth=PhR1Ox0SB=SdKrR0KdLyef<;Q(AJk<@`;7{~(+)LxG;LpQ+ZUKVF zDGsW=Y(w(Y0(EbqNqvglV_tvwH3IXqGj zKsmXgweCh2f1q0Kz`=sD>cnbTh9Dm?N38u4R5fWUifM4p#weeI+n!ooW7e*3t$2sp z=jWL9!|o-78$OXaRa_WFN%hG9(}~<_{n9fawe4v&?61WKW8{rq(SzuWV(a1%PUplc zbE=HQ_jJ_d+siDS>p<U#!CCnPM{~K z5!EmE_wFfmGWEq<5n7b%C1EA0lNclI$vz8t?)Vx%UHGLBwAgifFCO$A`t-`sb1%96t(_Lp-k4Pt9J zf2bX&@!OtV>;jY1@+<>;^a9o_la49BLxBzd|ESL{yVPH1ov^LYNEtV`p;sbVK#f=QeU5+`xAAq>)31?DA> zJ15zX-W3On6O=TJ9D93=M(Zk`M?5)Gw)-o^oF1g@;~TbiF9%* zZm!h0 zyxaSEiU7W>UUci72{Yvk^Ke_j2c})|#(GkcnjM)c2?=Vmy^;?=bZ2Bf)qrcVYNt=R zq`{=`Q3?8PdTm_xABSOS-FLJvU&CVyg!hAXiK@q$QBuF>|0yqASLTr%)nRN6rq4;I zeCP{w4AdESAcY$UERdK7O>cFKQ82s==N9Gq0DO$jr~TteEH6)$O|}Yu-G(j2U7lGF zZnQb^r_I?io{Wnc^lt{I+CA-51KPX;aO2l(&U?#guktedd89zEybLuTy&TENOyuml zJy|!}D9;arw|%R)obK^0z^OO<{vAN7(cl2#U9Hed!URJlq@2ZE;R0&o4_&xdV*Zu1 zIHDdn3<7rgS)dmBB-Al4(xp%A&{6ox7$k00h)y!!TVo% zLy|8mv&u6RUqas%0xFe4SHFeq&pTlRK)@9Hq*e=PP1G#cR|sTD=<^oaW9DPqbDDAS zM3ARTTG+w#esXu04slC!z$o^&>TIsk_VuF$`cvS){rc9Y&117w%v_(n+ldjH9>aws zxMzx7Jparyw!#M&V8;q}rnlBttOl7Pbx+!rFR$~Z){F{|u*SW(TO+ zy9gul?8$;?mSGP##FP_1ckLcM4Q`D#Ht;)lUe-nTu|T{U2UwjCWLY`LD|uKEM@1Rr zgIkpeg6IKLr$=>gF3W!$japlaU+iZ^F&&O$#D7uywHHMvrX&47B# zT!O{<*QryaKNcvHzUAHNYN~kdSZQ13gE=YNN=2QQXWlejbLI)=8WRUN8cT9;eXk;+ zm#yVk%KPqf4YqBxvF`S_)yKKArdKBwgapYUo+-@Egx`#Lyj*X_-Ki_Rf6W2f#=N08 zTe%W<+?}RrzvlH5vGC4>Y6|4o(vNF(?mn=Qw~Z2X?QE|Ti7W^Nksy3=eczJ0OBvVa z(h}F&5~Dd6oNFyn?2qXO1zbG&t19%Tj6;~rx&zjiOJ=yfr?f~V^`_IQwh>W&6WIRy zqTU0}QQi@JFA{Y6o$Ga!vcEUY~;a#T;Y-8((d(UM3&iulB3ev z<}T(AiCbOn6K$sFY>V2uc6G|i28DcNO*IIqGSE~FfMm;`9*t3duY`8=R@&3 zA79|}biYMwJlF6meZBZNe}FAJvqT4F)y$&?{G{Vn+`TEs8=2S?g_(N5S+5tD(%Tn2 z$5`wz&23W&SD6|HU6X>I1>dIDkeB_(-+8^V-zsU$-T!ZA`MV@@qvfq&LRa>6e7?Ri zlMB`1TfOT7Zcil7`P245B3FOdJ;OQt3 z@pa=MiU7zUCID!JiGa1e`fGM;t&%JQcLDlEjao(x+l6y0E@ODzUTmEIJSifur3@VV zA__njupPfMxi|@5&kn8-M#egdlHa4~X4A`U5i~~|C8mS^H-V3Ts^g|hD?ij*7lR4J18Q2&@RF0%6H@Yao&E*ol&xjx83C|kn3}R zw-z?n$m&wmSqR?g?}XFOJFBd}K>RM>2pt@*{&Tt!c8k0W25n(=DU-s@0`qOrj0^Rb z%GzQLT_w&!b&nZzV(P_m;`EJ(u@@rB|KTMP+rE-j7@1&ZYz+zJNH!Kw&ZJGOlJy6`o1~vLzmWB(X@~$W=G)YVi3%WxTxEBdQjUy3>!1qwjWv#6u8Hr)8;bR9YnqB`OH%bl?tJ zZ^#=?^ZJvbvg5RR_}K31t9yg|#6oHDLyMH&^X>lN5V;;m%1@quJ8D^IQx0I8)L+`Q zq+u>6*pH+AiAxJ}=e;IBHR;+gni>@7JdfUwo3-j=b`?*CmH7Aj5II$nLgGH;f(-NU?1s_i<3W1A2I9 zDNu~RJN<9Ia5%0Df`9J9@XLB4oWGGofeNaO+Kp6Si2+e^eSTg zoDn1lqzF^L&6riVxF}#d0l@f^Z;1t9_k63Ivuo3V$*o!f>ue*iCJm2O4IYknha5q> zS(1k(PxqOAq`_S^lMDm*Qp*4hNd;Bx%PJ&ez@(pR zx@H1^k8@x`qJ86xAgK&NwYX1O4*nPeT`7{r7_Ru7h06g#On<$<1u+eGWtX;)%lAOVU0YpyNR#} zJXya%%VUo?SPhvSPas=Y4{ALK_q6AWxPc4DUbPQVf?%7hw?v|EH1MfHX_Jh5d@jM4!`M@rj@R3 z(!i1%>MXHh{FY(e^{KN@(p8~*YqL_EabGYzT%}cHSlX%Z72({0Mit7mRum@WIU^=; zA!M{EQ(z@l2D}Xa+xJpOLZRKler+1ZWSUt04!Hj~Rxp2A6x!cKZS%fZ>}=d+d~v1L zS-`W`ok`r{Bt?wiLVy#LTlZ;Pfe+-w(Pw=i`!|F01AzWG)KZ0_SJ`ZELG@yNsjoHm zV)ug@;l*R0x>4(?zQ@5FcjAi3DcbFr2vwv| zAzU3enwdx1z*edOkrS1;Kz?c?gb&=9Usp1%;Of(}!NW{qXuNhsBz!~e{-)mPogP_7 z7!!Q6&Gni++7JB3_$Jmd*~}&ow_KqVbz+$+5p~&)TLd^xII&H#*}i9YDc>$!6DgFT z{V-EXfw9wI@0ue`=WSkLfRulrARA2Wvec}U@zQObsv8cx6+_Y&wlfTo?+nP&`9HjV zK0Y}x+Si&WHu`WTeBQ(=0MW>m2EP(FSs|OxotqUAWj?Vr%{_HTxx8nA%oXKg5zDi8 z&Vxt?=SmTGX9*l#Q9crX9t~Zz?p&nANbdKRy$U1!x|maJ>DiAyi(CJLcUi7%(>$ZE z?}*GaOx&!O`gOz=vXhOT09SjzT?PKn#CiGunK~rOpy)IK;5H(G>xLOmV z{rHnwLRdLEE`{-j1u(N-|5OzCn^imi62v*>E;Axt)@zmPj^{P)b0d<&Q};9~i+LaK zj5zxi1v$T3ZOS3(z_v%Y_=ifY8IFa9&nu~YFnR1zBOTKRnRFQ)BW4Zu<(Ia?FGm1KO53`V=r>zp5ZQwVpv?!oP@P6C{VxmmD)fLizy>D%c< zRp9Z74__suo>&jak8iLKBNpJ6?}9wen_9QOLq)0(tAa{Nfg19=Ii1?FRW&+ ziE4v>r-2F}2a5TlX3IB_?pl?*5v`Z4q$QPXm5vrtc-GrZ{BuO*syLNO&c{#7ACzZR z3Bf!GWu8|tq7N7&#;TM$@#MSd2^`DSFVsCC5t+BW7@}*uhVGO|tG#{)B0IA&pE20q zz~5U>hI7bQeT^ao%_7OF~(8$2%eXDWx*C^ zn-|`$y$Zn}biEs5WiHj($i7HeKS)*%e`}1(3vD2ZQztIKeTu^6%H-Dkm9E zhPJCiokCGZtJ6B!I3nhsWYsBfGE78fNt7Avw)9FT(J3#lsJ^+t`@Cp*|U4S6d!mxT}AG0KT?YTj!$)#b>hUOkPK z8h@tm$vLCKDn^P)sbWv2J|}J8=ObzUW>w( zV!hAC>WW0+d`v;{DDmSU48_u?Nu#>IPj*|ij$+WcXycxrdMd!fVgtXfyo5ROy#^Ov zT81I7!WoXv5#<$1;P?I2{qn)zC86SC&(Qp4b&t}{^7eKSV(i!U8dGQwt>BrwE}85> z*jJR<^qE30*VwUcztQ0(vp|n{iKEZ_{>U@>%-tt&6P2Yz z6o1g@Q-6h4cR=Go<(=a=-IZUjK)w}PvPE$Kj~6OD?;jtOXwF9;-aTR}d@A)t^wmk4 z+F+z{wU|i*f@ZGcp)`48@Q<8>Ql9V#bS$~~S9tI0f~hPQP) zb*0Gh_sA-@D=@9?7TNy>hm3fv1KRZrL}FG2H8t8HBbDOIin;L4bZDN4S153Phc_Z* zvRqb`zxn+otj}h2j2<6nY*(te?(@dUdyX7?9AyM z>#5`Jj-#MN)=qR}w7t=S8rQV$sd2vl@Qw3%+5_DD#UzDyw&PFFdqkgWDDbt*F4IFKa`KMstoY=$T_c`Hl?$aPz8Gt}sj=l-#~)i{WfN%QzN!&omZLrIcOUN;ZoK~} zOKn+nhGUT5{56Qi4V{WQ5YeTJ{zu+$$5u7z)^#JA(zm?tNC#DwNuP?|&p)|S307tI zC=JIA5Tm&!VtP!O+oh6c&OkN6`PgEujpELKT!^vWYV8Q&1;KxPZUoACYX(T!- zo>~I3zDP4QYs}N`c4o_y;OspfvRROWrr{SXZ5M)lk*Kqg)r;#@Tp#lc9gpm>9$b4j z#8F1|^Y6%N(}IOp5LbsgaNsPSq6cYEwQswotXqL5F>11J5~DCSb|!FECb{0bVIbkC z{4GA2jC9Vj&{znq(>+g)o_92h`4dw1Hp|&)!DsD^4m9yIx$MPRXO3OkiA{Y}Pv7An zJ4pb`g?#sk?JIUzFCyMMW{Yl83kD75rfL zjuF1n&Eq}iU*`ocm1`YcLN(Jvm(5No2I`4>X<<9e1MwUt3hG`m(2tz08c~rWg@}ps zUJB`@xoNS`-dTkq&O3mj&=Iy_6P>(`_>{2l{z0xHO#!4He>N_@zw#fYKGte|k|5;H zsb?5?dkmrsp*ihn=}@F!{E- z`5l)W|1ko#6X;tT8)M2XX%}9-f%-@c`kU#s)`~~~%(>OF#ntx7GiO3ZtwXksN^lL( z%VwgE=T2KQ1*iuB!r)jyfJun<2^Z8#9rTO);%TP=zZEVFU=Nb7mU2j)YUk+b^E$O$5->_gr{TxOqqr5fB; zO&#SfSDtUv?)nagQ#f8-)4%d{#gMxDg&*+WPkD!o$qcCS(uL1B4kiuK#`*Pe)1Qho z(}cdkaV&(YCUG~)rSxrk890#DB4Mlkx9`)-$WSjk+8H;^upwLQ+46zjl9}9rgK;2S zVt(WH9sk?$?_^zFa`EEBuD4Qb=CizJL=2W)ZLc8s%JL#vm*Gwo1Q}#&uw+w`h!1YxYhsN%^LxmAU!Fnx~cAEhMl~xa-pnvHrFhR-hmhd1)GE` zcbw`6;-()n1zbQzl1i#n>4gjsgSv!%0cvT-r z_g+sqaS%8yJT7}Zs35`6|D}ssUuW=z-nMr-Oh@O+a`@G5viUY&U=;zH+2{m**_3fXXKr&( zMQ@aESGnd}q7SEr*qyrk@ESXwHaq{`X1~jQj~FlAm<6Cmzo~l>xr>2Xyh2VSE~-^)Mbng%rP19h+6#ta#z$gFowqyf@{f0NPWvD-W)9v1`!swA?X7@K%LB`S=WWV%6J(z*aDecPe9Vq66{EmI{!m8=4NEzW)EP{p zkd!a%Z8-#XQySW}jkW^YZ48YOd7;kvI~yAXcFqHluiE?Qf^vr)D8=Kc%NfRaOUI1u z@bbFyzNVFE_#);sDU8Yccfp+T(-WNZ2TM7sL^XC#Q)Z)Uh&QTbmcR zOAMwr!uFS)&;FiM=NwMAz%Y)gdSn&-qEBhDv3)JReVrohWr4-X0w*R=lVdj^AaB2GinIJXmNCx5C+y`|0m+8D8hC z*QJ4QWcA*wK*3mtGB#G%xE`!I7f$7JltVf9{b*b+Ve$zLzb^PNO@YtWcZ$g-ezUCd(9!BdlV@gdEsK{rW+aRDaVE znI`M<5}(yO5)9l4Xv22vxsmY;)D8z^e~Kk#koJi$L{{z&zgg;Sv5=66f-F(-{Z~AZ zcGYFH64mYN7J;W!$$I8ahwvZGj?_IkdNnWN?B=ot>ffz#ww(S@D~1#dqFaj z?RUU7h4bVr1yHmf?pcxl> z?OT28-vW|Gwd7YBR>DBxFJo4}a;fa7puw}vGi{doLfi#B#KGFJmbdom@GoENyY@e7 zpKk59`Sc1bKfBoXTr^s9Mb&K6p!&s5xbbj!&wI6gpzUbsZ$DOB=2i?~0Rw8CO$;vY z6~sWp)1@14Ny1HEs$W57OLo&VnP@HY@tHTqsl{R6`zhO!>i@uh9SE_$-Wy>+->=*} zd*4=}t}@mUSt-7GS)TZj6gVQGDf)pn=v+g3cAMb(*CsU9!N&wgFCH_RY+p5kqm@m8@II~Vl_$>}*T$?)nD-$-Iz&wJEJYzVn$RM}uXaJUfc$cjvTQ;;u1 zdkd3Ij&=I%#vv_njVxm>_z%i{QW@55$ACyI9efDtd&;i5ngp6_T0J%6w&8suVo?)v z#eA>MLrl(Nm2AqQR-JU+ZECNKarpzgS+m|z0;Dy~w-*^#OOnDQ8yE(3=LX0J_dSQh zEJPqM9C|VW6bc3=s%;Kvb5D9rQAH|Gl^?+MBWQ8stw#N{Kk_?n*j(otGEHga^)O+Q zjTT_v=-U@xweVNdt;ICkEK^1{@<1&TolZa(`ok!$WiC$aa=uicvxQ>_{#@2dHV(UeO zYWXPl`{!Fsl=Y`$E0ecoWJ7zc{rc8H2Z7KNj}iBjnwh>7b0Wd`-B?`KMo`1KJq#E(57%5*mZniOK#(LpR_HyG+V9+_3vkI%R%i5Ksl4mM zuE_`^gt?@vXgYGONlkywbWx_?sq-IX<)aPKFQFQ;hQD&?m@R7{xJh;k<-I^cIGDbw zE_PE~2}xuAJL$+y&Dzc`6qEqgU`gY0e|1x2UMK1?zMvSTZM~nR$;_eZ+Kw!(4n_P<_uV{_S-)gE8y5P}=dBx9&hexb#Nk8Egu_%ymiuu#N?Zy-U+cGBENhT=LpaI4D+5irqB)HO7LozB0&#I@kw;VZ;Co zTJyb2oL^Z=7wnoVJ>YF^!-3y;t;F2z=(Vl* zR6Y6e##@7vgR%FG z8klT_xXN3wU31FqHuo(&nK>JzHzdYR#B0bH!{fp!f)z7#Qc&c*j~yEcok?yqg``{t zc+t&adXlgO{PPb|!1x{^y^kq)av)*7ZDi#{zI@1X<6gm^eFn>Y+d5v{tErN@w=jYD zQ8%9*ntP|p5aac-^+3DUxVEk2#p0^t|EdX83sb|z?}esQ1{14i&6361b&KP- zUI%9T#vckCFw$3-PFOi9Ms4@M7(6TyJY9*VUKl}-8J*=nyi>Ib< z=&&xS>enKf*AZ*!&j;QNZOtXb{hjkOnxJbB6q*w&0ep^WAjC?X?MIW?(<@1atJd3_ zUIcm9(wxxTGak=DSN^QdcClmYFH`SL`p`YWUnE^`>mBo~#geyTv*Fu2ZKKuT-5*GR zU6X-jW|&v642*e_gR-qT0pZ_SJG@ zchoFRqD-rELtn@)_WVe0MEU-EykCFXFIv=U|6XR1;_Q0CtEqDDqTtSo4WE4Cc7646 zifqZ1gDY;;rS!(cQKv+0ig@3jlNqhurqfNBSKjOyBzU)+`ZE^9#P+$%l8odzB+}}r zv31mcaB|lQMRaySl-|6!b$;D}xsI@WrbwKgg)_Nr)0^kPUdNX7&^p>O7q%M4z2|p2 zSw|)^oKosg+xBz+85dmdyD4?~R7`4{+J7R(a;OxPQ(XhSzToUbsXYWjT9RyisP?N$ zh>41gB}ET<4P2=veuiNa;X`>;v3-o~06W;dnX{9ZuHr{tCZ=yWb%=)cV$Q)4<8P)( z0f`_oq6;spUaqj+q(rZ6}A+?n^p22 zTF3Scz8a8H5Y6A!LBjgi8YV5$Fzpk&G8chB`8hX1&XooUCwZgZ)_0q0U+Y%=gx9gR zrvWL$+=-y_e1%?HsYf1M_`$f%^%Ci3MT0|4yI)lqVKd`gH^+nwmt36Feua)4F@1&< zN-4-!yDED-{5!gLq$XfG{~Acff>*vxvQK_sxAk|OD(PJ}(|xY*KCo-Xja~^*c5xrU zMuFe*ettXt5vP9L(^Y&v?sP=YPQ6zkFh&>uq0h#d-aMhR-7x$>_)ufo%natiEA*jQ zG=J-xYc*o=+PL*SO>Vd9ajGlK!>RVy=pL`)sv8dx!_HU=uErfsgy&xJd$UbbvM^Z(TgAN%Dz4<>Dd<7^w_?6lO}1l zpV`iGESR8GT={c14__v4=zcyRM7k3%+im(i;!A7l?o&BXuRJ+8(HS&Hg%bTzL=*nt~YN|2@upAAOx& z>tynKzTiw3rqFKbzUv@pfB|uuq+-kSOO5v;#~{fB1iSglf~HB~9cSuM%u2^^46#jq z-0;9=x_KB^*W7jh;J-lxz;Di-MtW)~e|foaTQ1r^FqfQ2IBiUDv$ozFeB3PeDB%+V5C0 zCjt7?3zCxOI2xh^_Mt>yp6sFvY5fRf+VCf{1zunFv*@c;&H%@x8F=VnM~FF6z3zD>ZD!7T&G@EmaSIDBTJ%GhspB+lVjYanXwR5N;@wd|NFf62eWd zDd+mc$Dnl;;E(PDY>#YZDou8WBfU*aez^ueGLx2(%kPR~P|C%)fO{ikJ@&HmT6j@s z6lv|pu0dBwBQI!Q7?fTH=@86!8*de_a}2Qc;b%LB!X$L)8uJ-DK{ zY|O8xnOWJ&Ug4DY-VaEByWO0@3!REZuNznitE|*mh0#lkYxeJrv`@>H7%lH=B1C2L z?9lVUR3)g>H?7}aMio1Tp54bUuJkMFThi|G8Y`Ixtnr9x0jvJSmZfNrz>qQIJ=el_ zBCkucwc`v;~a|akR{X3G>*sSZlAXnv>h@lC%dd-`(3g(5*{(mvl^R z=>`wX`a&liR!TVFzsn#OCejglQ>BrNW>?9Qbnp%@Pi_}2h?ta${<8jpSPyDD_`OLb_q0}v$QM7f)+Or39HdE4V zH~$j6MW2op3qh>wmonJ)*{4uq=E+hByAQqlXLdBBQ?ciS7{N#Fir)U%4h{CelLf|b zzbe~HXVG$NT@RXgQ}A5tPf9xb_14nB8#aUWmM;8%AqT!DP8+oiLwrN3NvC z;x|e0+RjFO@5svBiKlk~t9%584GO|5B85KG$#7g9guzZv*4j(-XXmvJc4jzy(?=-}PQd~@P ziZ2Vckv?SbXz(T7y!kzQy056IQ^aIKlE4d0d8}Le135N<82eh<_v7$6f5Ow(O3=qB zr=$5IjjaB>NgHUVV^*K}#ZF1H0qNAdLK}+3owAvKSQoQNpzsLoJq-KIBj2*rsH?77 z-i2M0a(*HZ&{I>^N{CAfY1woo2I@LM-N(3gBf+TFlkrN*`NZe+2Y&jnE2`wBU(Vsm z*l>B;xq0V8^MNwqsqhjc32E!Da8QLTF=rumpRqr>vWe@TwZ<=E^@=szS235&u;;5B)+lsM62 zrf2@0Iw{!JQ*OVWNpo)dNg1aJGw)9skuxAe1Lr=9+?!R#Ly&HeB3l+Jle~a*Znivo1dtKG#@WYqT{c#hdXH za@^+jIt=Q($9Kys)R;LRPW z@eM21qiiyjvdm(rwLiU4*<}kI_mKXV=_bqmYR65k0dQIc+so`k2`z>)X1U4L+Xur` zk7t3&sNnJ=^1}apF`h}9T(F~js>uJ8$`AbI9P7UQG~l!sI^3q6HoN58chki9AUQQy z@QcEw87>K)BT)<9*_OjiJ-liLb(Bk>AFXV9uS-v`G238M7f*p`)PGv0Ytkw*SE-bF7!x>Cs2U?eVX$1d0 zJLCqYeAqP?;$6BgDbph9;h;3+%3TeQVd;_MxhvAU@{94MKu)^HcuC4?Lm?03g37lZ zW**PR3s^?ycq4YTpWD)td zRL5tE1KojK^UXj6Ko2VFGk#FX0bu$tU<*>pvh4`q`#^DR2Cya>w?BhF5sa5xYzPTku4Pe6sUm`=3K{IPS_UM&}0C9_OGT z=O$F=^wIn9(EzQy;62QeDVkI??>Z5lqa3jQ?B&1HebVu@qlafUhU&h9OXdBS<^CCz zv=tY=m`av`csBaNqVjWB?max=;~N?KdMEySX>GcRT>q+gG=!^B96oGFYKGx{iwm6$ zcg>uk)?RoEzueEX3P&A0N)PW<%kS{KNaO*3J%}V1>zVV>N6&lKF7meTE^~n2gm9m1 zFeK-0I7{6;6w2^oOd?x|7`L_Qw({yQ+2M6+^}vkv$j*V-%-RW3vf0Il>jBd^J{4waLKZ18nnfu8v|kG1G$$j_kBSTg6*-AA_LX!4ns zzQv>ThTO_EZEy8M5xtyt@~oxJTQw6V#lUk4y9$3yFa#V$r5-3vD63tLFsLMMKD`|X zp4oeZ+lR&V;p+L3wrC=Aw(?2!#u*y&`}%`C>W|Dv)}y=&KLRqXJPaM2+OL1dTNjWR z4}11=@2q#vtFM$^-;c^_VQLc_7`nbtUjXYddN%BN8kGa>hfY_fB31AwpSNM^rnyzI zKVfnk^*f;s?us9~@r$*4GhKy9{Z@B@-e6_fckr{Rm5AE_3#u&T+)Rv&+45J%3^0N) z_b#hLlAb=BwfxdB)o8<#oI&%K}Hl}Or1 zlJh(*p-4-h)!BKizii7P0y7TzLHdJeyn#F&B%Bzd1=z~RFp6tHuqh&1I4+=RM_1bqbpQvZOP5+#OeQKH| zj`x_a0gP#T^^1%Z|71B$Qno-bojxF&68wkYF)f*#H_!NlwzJ3oR;&K?v(s*G=Fpt$ z#`Gpv>Aqe`0%tD>yj=$_2>7`$D&p8*qx)+>6F_BOkhl$J_Fcz6B2pwt`{v23k5ZPK z+BK5OZXDdBkeozI_KiBBUtO`5fBXYVgPsP&5{)HIST1+Yy6WA;>g1scu?-4mQcNn8 zQII%n_K8_|`Mjet!K6A$JFGNZIrgLlCbP(cX_eOcUDJu1YI(Xo=~ee3Mr|q^?bxAd zfc*BiMx}@1UtI;f8wz)+`7Kp_4df^}doO3!jCC&s8eDnVI#gt9Z=#|7L~(56IQ{x1 z!^`a`m{C9K0I{#KDjZwGZ5d^KA5xwZ4KV%>^Of4+mI@6pY@}A)Clb`Y?Ci);P9{&SvgZw+o+J>pV&z3dgd0XuyG5WkNGmKhor@Q{!$Shi}~0f9mk03TfzSBOn&s8GIL{`(%7}5oQgASI>CyTlO}%HnF5N z>eJA}sPfN$(O8iqzfynM8|$v6B*l#`cH3gC<58}16*hS0wwisN!j?@0YJ6GkT+18b znv-*?Cy%{yR~;0VUfe?5jCOnp$1fIFx{YQrpBNPlAj}Exq&8-!59|)qZx`(JS7HjC z+j-}bSG^~w;sp~&DxcP|l3|7~fmVLtXKxY<=3;Fdg)^bHS7nmr*U@@YcN5B3@(bso zuDA0;NsYd?D%&|bXT6lCcRe~&PRVS{W}H_T7r{Zpr?1c4oL!-I!!Xk7@8>#vC9$nEw7@c*7SjrTsPY4Xl>1_P#ueN)AXwf!(?&Yy9|H99h%*?Q8vAJDTr~ zTW>D&(KJ%@bNF2F8Yd+Z+B8c|7k}3Sc-Q+YU=p^A08X1#AFJ@fN2S~ETX;j?gutS? z|6y+qRsla(y7y#^Hn``;z7jyLBb5$&vQaM88@5*jnn=4qRKMNCPYE9=vxk@lKigmS&6l(eR-;a^xb8er+YdSQNi0$7B&Ai#R z%HF_86_ZYiMs^=_@QZW0iGK6pS_>9o`X1d~sOH5s#!Jwm<8Hk{H!|RCS3S4MC!ZE7 zTan#H65)?{x^V_X)*ZWKyDG-_Jhv7}69$6!;4dG0ed3BjfxKi*22>|QbcHonYgM|P zdipwl6FNBt)~oF-LiUWZ``qy#LjIw|W2O0;|4x;>Ih{>2vG2$$pA~t@P zv@ifMXQGAd4+ng6{|k8B&YM@u>e~B7!ZnF-)^#Y@z*k9Y-p9Qu(ULEje zh*GfQ!!nO=GbV?`skRe)v$0GP-jTQs4YrJh(_FbbbBtHL#wu&rGDhIic)p_@gy&60 z(%AyUI;}S8sG-4p*u~m*E}^5=X!*L~q49HsjmaunwbSB;G6F-aJ z+;@PwIqdc9lFyM0Yg)>=@T6=%XWC7NW^Ei*?Ud|*KITKWlicd4R2adsjyk>LwX;m1 zmK@B0;ir-LFevYFZ#5XbeO8=nyGzjw!03VeBULbr>lB-QMBJgDEu$Zv(i|R5KH^9Y z<}uB$Zm(!+Ol6I(Rj6$G=trGOUDS0lwC4)&I{&8RrAksu4A=hB6F3)a(95UF<4?$L zcdG7027gC%<}l<^T&YHvLM{eUYQA5tHJBpdCKIoa)cHuUN$z2BP0y*&qy$#emFR4em|y7Mx$o$Oy%G=mUB*^vGI)aFh5O zQp%Qocg#JGpYYtkK)NTh;x+qjXeascuq%&$>e_>gpWFiUV!oN`!4r+uV;G9Bb}ObHA{jMbudo)=fk-cAXT&OvPHgT=UL&k`P*Ia$PEL3xF zMN>wN93v>3@{C7&CZ^6?uo*TH38a3t7v9MJqRu_9W_miz#n~xsKDf*=qdUAT?B%^W zZf-Q;eBJ9cLH0fnm5!UyNe6>cyf)<5j+O-Ob720Bks6;bnBK-eyUS8XYzsT$9L47| z+{)9QK2Y;`jakFm#&}&ELj7&GO|{WO3f@hk?TH)}P5m7k&pX6eP+wGZgtPG`)88wB zBWJr3UWTuWlVr-?W=^v7j3lbAS@Kke2%>97iNE>Iv=3dWaKPEEZqjtmLRZ z;GBG4nsj1au4_NFle!z;n=%cJU>yqt9ZN>AKdXKo4fIg1B%A&06)9FsI$969E&=uE zj37UhPY#anm0t`O@OGS;aL=K?SxuPsSi5mAF_$zS+?>)Qyzl`P5e+G2p4jIS9&BC6 z$-qgzsUAR_JoF`JkEY=aOS)TkT+YaeGsqE2qimW)SKadY*+DHvh#c&|gsIlv+oMDB z+Z)hbyGck@DNP!^mPh!B-*~+ov%PlDu7#_{<%|Bu69y{)JPRdkTD9?X497`B3Uqio zJW?=?L*)a0bDk)r@B4ARGP%45H+NTD&`6};t*Lx*Dv*wwUp~Y`SCK%6cMyUVq(@QwCqP$Dw zKg#mrs*h;u=&rlzh8pL}!{MS|E9}eV4l=7+fT9(Td#*%MCbwTSzJjzao1UZ&!n}HL zYRxz%Il9uQ$PWHSI{mnwQ~=J2@M^Z*CraOCjoS9Xo%xerBZF1o8xaHpie+~#gF{2iC9KE>JEp3)9$oeP$ zuKX!bf(Y0DgYHfdL*4p(I)@%HrHWH+jb4v-cVV)`Ny*s}>@I1GDQX#YUp>~`mzMGA zZea;k0kUo;F((_`{4U2EDM1?jvXNV;C1@1<#{hu^Wr7DRB%-+SCI*CndJ4a}9EgDa$g)$*5_ks{vw2Ko8Q&_Expk&d-#c4e2D&+_!3{ z&3>N0q-QlJL-uTM5eL25^#TU;c1s+)vD5`(Ey5W1Fe+|`*GjWwkh+OeQ<;rhArHkIq z`6O`1UmF$Mgh3stM@6cE3EY0{Yi2`SjTj;N(f$pts@Cy{*1MJzhjz$Dz7kvCJQjSj zDvXZ$b-$)D-)g;BmTg4<>LUfTPubJa_h zszs5J8spWOwzb=pmf0S!a{GhN+H9zEjB9h#MZx2qT>zC(siP{R(AuU0rl482N6u7! zc~x|{#;1~eDc+dO>>`5&70)(6VAw;6Km<@P&nm!SJo1N@MGbjk6{yW7$={ciOiqdQCq_oxGb1Gm&M!6mheJl8jh! zC|?7UBS%6??));S=lyIf1_d%~?_9L?hclGztk-q_#a=D|#2($1+`H+$)W%D!{wWI| z*Niv8WRo~SCoU!Qdt{x4AYOX08o`_3PfF5i*V<7XUh%i(vZpc|eMsaS6_z~tsj%eWyW z2l~!oJ1wOSy>pJPfR|@S;tsn$_qk}qd0kRWoa%iv9^~YD$Ed)i)!H7P#uATJEi}SU zB;|kK;e7;7dFyeu#>js|b2kNZISk0?&IDauqZ;6iAE!kj#^e=nIzOlJe4S_sx>k)0 z+X75b%cQwJ=HE5>=<*}u%VstAm`b(}ExeF83|qUdffgOr;CE6>$zlbpO*ZNW9zLye zUb^U-L+sV3m8HDg7 zgDjeo@LYBwF5Y}F$)FK?{Rw+jQl#NTqhVTvT08gT*UcRf(v!ufY-8RDTxCdoOl*u% z=s{bb^ZH8gH}L9{=)!3XIk* z0?Vy*oL)5#&+|wC+^k}MsM2;rPn~m{9xxH-Y$s*(*2=*VPZ%P(+Pxo(8GY*AqTQ9H|M#L|d;hp@eO3u4*jjHO2<{swa zagQerOQ$DKD;bpzEzPhxVkAr7nQEdorvf>-=ybScEM?lw810CsQ$9n?4Pgr=M~9G* zCdza7EzLm2k2Ct@$P9A}dSIN$G-*=$q$X**CJN$#ULscb3p+)W9+t2)ro2{|N53#@ zEzGM%JBvg*6M|-GgMG-e^J#O*><`dwioTY|&Q*_{&6X*kF-mg(6Ol@ZWeXi`_m(n^ zybk$}YL6?9ijH!UaT?E%*I5g_TL>CuxC%r?cJ&5K!D++ygYJV2c$aiH2f z0vN%Ary@TfY4se>8VLW8tNEmGgQq%$C>HRSp_h{|z5mEHJYH?hbDB^~2LhIZr?ftj zmHZ>61O4jPUVLa|p+~c+LIGK;l7Q zf-DC8ocV{lk1L~%5qEjvbDnHqd* zra#aA0i^S<#?=*)Jn}Y9@TIjqfwhi*C3tNRkZ-r8jc7;$^S$#n%Q?3iyQaqE7lMuh=DNbF`ux*z_o$;k|t&g`vp7DM6 zzHSie_*)+i&BUEvXxV?{#I&5_N`^A}dW*ocS$0cS2^f&V)Z^-ZG*eOeK&|~;0j3cfh0SmY6+f!eNt{x+TEO$SbY3=Cb&COkIyud#*Lk zanT*N=%Mi)Kei}cS|b_7hbHgak$Eyeu_U!1wvX7LfY-vavQ6e_&vxkwA8YQDF{XnpS-`+ihc}Ssq9o-S!&X5 zg*1>=_zBqxkAvkw%flytC>@=bKK-?bTv}4?X2s{a;qH4^_7m}*Tv62r6v2R}6HNYA z^=jB-Bg>uuRK7p#Z;%Pt=b`gEkVd*v#mY%i>*N_RM>j$`}9%lG-ffaUj}`Q*+- zmbYYo$Y^?Zdn#Eceuq~q;*up zPlgmDd}|e}0D6AS(%U)LG736bpD!tcgl4Kt=3L^4%(0b&A?@03f1mWGZe?$_=-c6Z zRA>|*y}CkCBx{0QYW46jUR-Fso!$TK^?k#$?9Pt}cnCK+1x$}P3yRqkQLM8XP{sUE5>c%T1sVC{S zCyj%VsEDiN;Qk$GNVnhka5ONN6zY#L$x~<#pGercJe+S7cDA9Q!dPw7^Le`LBTF;K zj{a$3xqQSL*Jw8inwz5AU&)=AN}Y{1q!}$T4yty39bi&1+hMOjRT_k+pm2XRqtCX$ zn)Ut8rIBQ9MEp(l9j z($*1{Z@`lW2n#(p(MaB%OnFKg*jpYLD!9QV+uQj`w^%GoJ|C9nc2Ub-6!96|dUGi2 z6$W&7r%ur9r3{N<)D`{d)qLLh@E48!aN$dSGxgcU%9!~#pC^neg{e*M(H4CTieXzC zi6!7Lx|{=>6LY`e_yNkI{UY0@)cs+`&{CWq;zKDV^pu5V_@%Dewj9c$^K2__8t$2pz?G5q0Sg_F$FVU zw$!u=SeZTvI_&zk@=uOqo{KcM2UC~2!kgLImtVS+8nX5ey=hk*wHI*g+;Rl?k-Yax zxhUi?AE7+($d&WZ@2<8yzl5=3E^wI4XOGF#{0U1-KSpq`rs@W365C_Q(Xq6SH%vEK z7wOArm;>aX(4UST{pq>n;_{ypEX=cAFc-l%uR8y+mdd`uSCOY}8$jRw!!{ZCMHTqnIZf=ek10FTakRdo7yv%= zoFn*dF0bNiFNGjLJn&uv_>Lgdlsy!j-gtfmhu@V%x=l~pI(8OD$7lyxy`9eBd)C3Y zDC;!{8~5!!&6=K-5V+?5JEQiSTbt&kXtje%T6Rdsp3hmd!@XzuT(0fHckr=V8zXrC zBptxRrHyRUgYRsNuY z={3d~Ln!@5Kat(+m)^El7_Q>Qe)4+QKd7H=zf#h<9bv?-k(c1!1a%2=62-I4sGcts z%4WAsLf9Q#Lz3&Z}_C|~az}?9gPNXl*oC}-BR9gfT`nd8)YI;TekupqK6}Kt3 z9hVInwZ036D_<;19YK!YI{^EzFm2y7+!}{z21+xwpB$QzKPVA_KZ)Ftd3fomJOuE& z;i6=hrfa4BytZ{4+4mmPN5godsRxVcZCK~3MohWYp68#uIy1%j3#tiH>7r!-$aHhH zeBmB#$_f6T=jD`eLb&+{@b!9FV)WhGiWb_;UcjosBi~utucm*A1Kzs57v}4!@y&~k z*wX2q*~L>}mJ71KJ}}a8zlC`GeIWlHyEi)x+=6OVnxcc?!K%B*L+4eZaMR(xwA%MP z?5(5SUeuN{yQQ9Oc~o*l?uG6++Rd}RZd(Jvbj*H!GmC)P_rS2oU)p-Dx(HB(>S?Z_n(ZrW-55F z^@mq5&TF1N?b>A5FSj4gHw?X%FA{iFS%2xv$4>q9B8~2|28UaTyXEJtxfvd<4+O@< z-VT&twY0T-);wqGQ>xZY>P)b)l>^0?M8Q+J#G zBE@}5hGNAxd43Xa;2%#YVK7^{;oJ-OWNcQV5j%U+1)wt!8!zA`fr#MNP2Pc3EoO^) zvy^{$s6EXER4}S$=92xfo$X22@_+;H5nRde$k^`EsA73cX4}_tZ$tq)q-5`jinkDq z`>pH4aGNz>vi{glZ(?R=HXIJ97e?_sN3IVBR$(sh@5VBo zak00f+_V{V2~r+DFM5WaSQiZs9e!+Zhk4x1J*X*V~Os;||>C?_YlVbn{y9Qb!6a{J_in zXp5^TMVWUk=(uRAj{!(JiikX7Uu2%hjEzeDS)^ujK{D;RijR?x!qjaN-&b(qNZg;aZrXW7Q0G?6 z(SsU`(_9$ln8%>NH>DZfrn-kvVUV(tuwHyIDEpwON~JF8b6@1f_R|^O*k6)$B^YN* zI^2d^HviCC)X&byHv@K4K{Yg@@!}Y-+ayrei>deEwr?T9Jz6L^`v27##K~2-{ z5Q{ju<~To2`=wr=e?@YRs(dLV8gzZ)-NEoM|BUC@+=A`$D5Xfe;^Q8LZ}a3S?`z6%@u|@?i0oJ|M>pM8c0*T$qp-MT_0NDQ z@q+AzY|R;iEJO4czmcP2$-MV}hpSoH2TcK+?R)YiifKoNplcclpu}}y!9l;tGM6`U z_AOu~75j=$CxZ|r@%gkr`k7^{5kGOnxP`8to=8uj>uA*wY1^FJM=G10q^pa-@i_JI zWN@e^pG&+l?63R42_1D+4J8Hp7Rc+L>baR{-Q2)Vt+_v{K-z}iiHFQ<_rzxraVNrq zvd%>AeUmdp*o}aCQg&`A>^;x_5xudkv(QH05#`r?ZFg&$MBj%PH*R?TqS|$WVY*pA zqTP6vH~A3NL+ZA-3=vTr<2`t&fu7Z2z`nLnZQ%p*V7!#q`Hb;dO4pBn0 z$0y~cGs+)5R@;B_3UM~tFkZ!baj3jOqEt2dOXQ^E&UB%ljx~>AmfqExB#+t?=Z*sq zJF*;b6Sl$7LLdIJw(C%-LnB(EFHO4n3$YhTx=-}NY(A>FPaA`5v~;%j6L^Icz|YPb zfNaUxv$Z8!Kcdd4IR8QAZSvLhKdBB`KUgrFQcwGfJ{nx^DL>NuwMY2+#44*8C=j~l za#KwDXzVLq@CmuR(X*r`eV5nIb+htNjCrDY2{)Ms{+b)B-XnpV_DIP#cb>aq1bxk| z&WRN_%rLeeNz>?quHN9Z~Ilam`rg)=F1UzgOt=lO8zPz1V zqVoM3sOfQ|o-)rh-AS+&9gUo1=WXq7`{JuO9Ry?F$cNwB3d=njU7S|kSdCZx@)5K-Z&iR`neRx z84A9cD!Yl1(>SYEtdD9Fum5~JBL&4c0YUtDh@`%duKqm)`ouFph_4&ELwm4>M=bON zo9{)&O6U7Swd#n1fLx-Mu?JV~~=59&?xoT$I|SgB_oI zY^^}pJ>0(ajjbBg**)b|R^Iuy%Y{i(xe|L|h|;fHj@1v3(`df~Oa0KDm{vW-7ErJ7rT$2CuGfk~tN>RPv>fivR8QHC$POHpY5Nq3!n-~Nzs1r!ygGdm>W zIV#RHv*rAvevR?zhX*ygTxWrdR9XLrCC7V9da)n>jdy_OX#(hQt!%$^v+fKQN%^P> zs7O(Lsk@sZ$Q+N2We9-$~%KX(=pNg=b8DCMn6t{NH1Y!QQ zkJ$0xbq%}-TO_t783fWk3f(Nq^&bYRMdsQLClv!%YY#2hX*aH}#<_nQ;1Ik-q)zD^?okT~k66Za*+Ozo9B>kbabe zS`68e%525w6Oq9OmTkv6>Hp>5k9`A1ky&s?E&J!uVXwyls-9g5WiMfOP z+Zm>_#{NcKrn(Jt+t$j*c8#k~Gp|<&8TM z@%Yq_i9{V%hB@K>st(oNLWiO3h9Rhi1gM|CI4V83Pf;MpNb%B)O4TA%^twmF*U?YC z!A7ebWBqx-{aU-9Z#$QWOWFAH@_FL(boJo98z&L*3m&dYf}eY0`$gk9DtE%Z5Viwf za>paeB_)46U0AF)co8nLczgYq{9P}^*y7-d)KjlJ{c>|cHjPUB*?#}QmSlJ=!W}0m zD;jCQYdq&9yRgm(zjgbzThFh1_k^#g2-udyZn-w(R;)ZtZdQ@K%&!Donfsp7*3fhR zcWgyMWEy0qTAD>t;oQTq1$Kgp7KJ0MvK}ukF9_zt&Pv+_r+Pwv$ zRPGk{l#xF@F{p0Fe~j~^D1G93PR!`^wKoEAm1mTSsctt+WqW-tgzm?e`yY7B-+b#t zFLeLC6FZdl{(+S{z)L&VvVg<6Y8C$JwjI??>2D%mD)?t4U%i-w51uy>FQ-XyW!?5t zI{pKm@4Y0v7(-V6()D6QSOQN=m?%h8hTnGkCESy$y^{@em}N~sgJ2A;B?tSa`Q*i} zr4aAeAAw84jO+O!FAjU29;V~U;saN`XE=3yt~1>wfX=P~V}}!-5w!tT=F=dcq( zYj{heovtOsNd9kuE4cZkzmBI1CYmmPTv|LFYPnA+FY{bFXwIfb{YYNmSbTvii61@r zYWb@LLk%79BEJXmyf&?AJhwn7tyJ#92CKl=8v(4OH(DE0HA$bPD(#cPk8Gn;mR>*Z z4*sEX<2vN<3%C|KaG#%j4^fIayW{y^K+9qnVc(+Qu&0>sEq67U?+Eg6#hkK&cS0{* zP6d)bCgRHn2y++wjuKf--@Tj3Vc5bUd4{9wD+L5~MbY+i-31}jjv@@t5O7Wmer5^m zJvJ&@s~ztsR&D1ndJH3?)BF=^(tKYRggr&% zK3MNREo4-xqKJ2^mDKPp7vs&(L|FWkUMT8KOb+tY9#S+hHC!G`IW7P?05rf4L%Wq| z_o5WN2}*c7twjsskCbI&@=7|h5G8Fg)!#qLTyW0lAf+(2Mn3_`J7{NC0$aun5ddxG z)cz!oTO{U^dq7GL;eqH|2-jEE7nnR>99@NH4?j3VNmV z5wI#^wx;xGJZt<)$#Fl|IR?`!h!fn*J>-uZOx$PeI6MyOpHyi#2l09?Iadr4z}$1+ zKUNUlu~o6z^n@j@eY{OO@2kqQL1o$ag|fqax9|N}zP~FWuV9*Z)f|qSddWkG^>fT3hDGOae4eLZuw)#?Kd^Tb0}# zP0}v;_ABBANw2itFJI5PnP{b_aT%~Df?wj6`?Y0E4r3_)XOPCebcqOB38d&cZ9Z$& zKcU(e8`dTtUTSnu#by7&VmWNVMnCYemXLCA|I!3dB*=Gm+B=Zp4~lysQ0s+NLJZy? z+j)!CI>?K~MJ^aKaaGRyeZ|yM^uem6s_lNBQ{D)ZE#6;8gc~fD4Y$-o`XM}=@Zdmj z!N^gtr+PPr%%2?fGF}wXpgVZ1U#7hGm zG^{@*l zCVscCj%kdtgV8oZ#5^eHZXcet=Zw(lF&DnOUB;D>=IDAMwOsngzLYn%+PG7(Rb1+5 z$V0+&tdwqD>g(#t2E{(0z6Yr?c!(<(?i!JTW@OtnHmdrkd@2%8sJZiQsOylFo^xMr0$mI0$LA3d#+~yFPhk!jn+KM? zD&9Q;Soprv7*`=<4!#h+G77lGa4}PXgi#Mn2A9RnPuh-~z@<+Z3zqF)-N65g%OYC#h zFb+ONeRq_VozSn+gIvaACUQlWjC?`U&a$1G6*=bNik(E>#!X1!A#>lRQI;Rmc0iy`A==cvRA7d~0^9!R{%5vD=2%G3j)}eM8ve z=1~vOv&+Xo>a)+G_IU2nCGj1jL0}QC;U-<;t@>W^&*K%*AcMD}cMO#PT0Fh`&1W=d zt{(MKqth2!vDH>8r=_#*J0>*>&zLgdGE-^>qHgGCYLh2|B;?}MdlDhUYXa$26aG&a z^WMxAf%>eS`F5zY_=2R6M-5N??Be`Zo%~{{*CYHw_vVk3+#d!FGCMe92pU;@QM;!I z(D-N&=IRPm_n8=PuF~Glt17XcgbePzP1B?%FT_Kj1%d_BUzlpoSNR)X%${d|H?ntv zM+szG=n4*t9RMgTvzgjjz`RM0xOGctNA|#@KFm%nhBg1`i1Y`wTZ(2GpgKJQc4n4vEGZPR z_lRW9u8-Z%3O?3c_XUnMSLmTR&ib`x8Bg9cpAJ2&UC6pg(HbJ>&K1oSv~w(()4jUb zE4du?rvuSUa&K?Ymd398Y_zplqAm*}aiSi8+`plYe)rhjWVTo!O(j4&5hJHeR#g!4 zpkD4-*W7!PY$Li{`1esN-8JKn)&rkhR zj&msgGP3s=bY4FvMHl|C&eW>AL6SC7J&u;`6YP$KT%J;;SL$XSG9r^hNeQla{famH zxg5x~yXdz##5MKNT0x$l`x<3_)Zs^(vXn%6olqjc^>j`K$1Ab8_1l8aW?f%|odt%t zU7mW7S9ZSR>sFiDVm?-7*+5GfsA?aA~LjhQ9*fYYJ^8dr_$YE-g| zoBq;n=a1_J9mqvTV{4?!OWd@g@5SrSR&k}oP6hw2oJ)+07RLquR!TPR=o5S98u%0I8wfj8X~7XbznEK$FUr|@LAF|-CWjB{c` zn&5x%vIJoek22>7UdwjKK5F`HACfqd4CruWbzEFtf5|Z-{9owGFJRrNr2=r{X;ok9 zEEWyCKCugY3*B>1Grz$<6Ll}qTCH~~8FA_mQ9Q=Qo@*A;;lUMN78jk~0+W>bzLb>JBa z4Y#}P2fB2X1nn`p&k{!mi0Sf^QF6#VUgs^O$qCl$g7xeii^cjH>4Bz44ic{ItKPR&QlFQOb< z=VNtHBr3pmFa+j&HKR)yRs@pq6tQjIyLk2G;*`bq=WA1OFwSj+*lOXhGG&3C%4`)U zB)x4`^GoB}Q-2~<8!b10<3fZz{O0v&=_6&}c7Z8o@*%{&b%6$5q3m41q$gu-$ohM& zP|elkXpFehR#xSy;D+tuaodfwL4pSDR^?g24b_wAo)o^GZZ#$KIykkxd*DhFtGpy7 zyAR6nfUPX+D&@{96Ka)$`hKA&?$uPWj}X&8Mhxe>(^|zTH0|0`f^u{VTiU zx{XNb&L_hyF$LT#v+LuU!z&c4+r~@OWvY+Ga^IzBN3>cM0hrXGo>aj3bcJ7i_^eCi zoAarJ61Ba+cAIA$V?Pl@b0`FPY!+|ZKfxwc1jZ|}uVkVhAjVG%Aj);W`Yu2$In^F* z$aO6mVa1?pS!{2aRgrArQwNs+ctv(lgEb=Yk+%tI%)&>w7ts2Iv-K=*pS0UeOqKA`U;dKmq+~$ z!12Rl5f?3qo9eg#@#%y5tTxaYu8Lx*`&+v8YzdpRmTa`YvEA=`-P>hpUXFrXS4z2) zghr(^!_k+)2XDH*o%az&jEs+^8JtCS#&E}99oNpVT(*!k6S&iB_4<4TsqcJ%3+X4N zs<=AS*Q&+P&AW+S!$0+LseWq(Dbgg0TFmA`bjR~x8cx=1a_}aXRnYiJF|eF(zQV6Q zl6g)95@5{eUgy%#`C{uRYaPV-D~Exi8U;E_CLUPjAR+DszuzQY>b-Z@9Q#nW&Ie*h z4(O{W&g*MT#8qCrDql%-v6?&f^E3&y*LMpUiyDqn6iWD6%gSNYrq2(6l_Jnd_=77s ziX=VZrZm-kkvfg@NSAxfaAgLga{23K2wW$KGQ~5C2-TVTk`8m|xj*iAJH%Bz$x$#( z_Vz;2FcfxFj+R)la}_ULxv$MZt=Xwqa?V;B@kc$vqQL(y;7=*Jx4i&KQ1uRi5L4>A z9ot9|9}S~n>%bMtvR=S@h!a@SfDx}*%CDS59n-xuyQ#J8GBfJMCuYqrISrS>==_OF zmQHMQ@TR`^{Y7KW%I9ne_MAhUy?F2Ft1PXqQwp9Qi27E;3LGYWA~vP%Wn$obWzFb5 zvZW*F9vY24oJe;@mU&o232AQx)PB+@DNf-Bw9sQ3?UNNN0V<%M8kaJEi#*k4j;Jgc?m@sbFMAW{%PP%xCF^XWF2@o6Y%^T=IBQ!QohsBjx5?at$MrwzpY$d zc1~N>uRTydDRdFQknkQ0U#(nIeGHMyN>U4_@p>3;9Q>C@?#AhANCtu;A~bWBtp92F ze4mvdzB68s!Sw)EiA-j^-$)_t{G8cS<8a*rhhOqXVI2gvu5T#w!L<& zYmG_Ryfwi$ClB4gwzYFxdW);7>iWU$0+J@mA%7bRvJAX3?|ZBcX~o}^avS~WAB>L-JW}nR?j>=8^mv%p^r0G^O&};FEzyVI5%&3 z^3}cbw*!(#{Vxky8 z6ytgJ@$7jViT}IFSAcrOYDqqt0<+biNW9h2F=uH9b6ioxsP{hX45nk)lr{{TD9DdG zwy2Pg&*87226~f)?|@0$|KDQD!TzIs_2f_H{0jR1s#Z?qCx~U}%ebabn2I1C>!!=m zsM8ggui>;QwOv`RjjqLEdjjO^cE$_rqY;ycqNj|P)>9yAERV55=Z2o zB$=osnzIf3EnV4=WJlpfG>V?RBedJ6`mO66+Ohre>r@6$bc8W#gSKU_Kt_{P)!@qr#IzA^qsCl)!s9QICc zl0}a;e(|61q@%p}#Y&sh`#f7pLWvr%4_^Ue4!Pp}sv?)1F(Tuy*aVqsef`pw4q>Y;gb-e&a9?uG zGG-5c!n3Kv;x@k2;nxr{Qmi5;ckidm6wh}5>G61RE#?{jnJKZ6@lPvUQ+Ca+snIu|0D9Xze1g-EjHb}_h9hlph4Uu z#&;b-s)7;#X`N?y4{z-0ahlSnqcZ+&*$-V0)vhOhoUAjux%9y5IAuMfaO%E{cQl`# z(KyEH*+=(9ZHKP3R1nC#!z zmhME(Ygf07yRhv9E$Xw9x#vZk?V!+EPsDJzI34~o&6u-KmUB&)@3#9&m+*7mC1MDR zPvOv$x0TTMPHV_ul7O_%dX&L4bOp8lO7Q97)w^6*u=m4V(R;M2EZWO8^|S(UA=gW| zUd%(k*=o;4V8lyIc8O@0pHxYjWkd<2VQQM#V7^4~jj!u*wh|#?n++`u3+)q)9ve=v zDo8hvc@<8uM&W3e*;#G3!CkEO=BLj+G9RxwMgcYc$O6~4%Zy5z`%?H?&cwGBdA}B4 zxB!sLq#%IFl&!V!%}DHjK*?Pd&x4;J=ib!5DtlvweJh-fp%w&gbmC$4b2j{F5l^f33<`mvlzuh!YmP-}XXJo?4=3E9q z_x3S=VO8Dx=lV%FRV%))J$&(|>qxCO9OGsNmKk{ZYX3=cbl98_f)^@RRo1dQ7AiwC zm`4o2b@UQI>Ki%2pYk!Z;E)s6oqhND&aT^XTO^`*G7@SjeMMCJhAvrm!{)1FvFnu| zU?SoJ2=;nY$S{jav5e?E@&4d4yZp}VO2jC>cSxm4FcZeOj#(9y-OPj^pVj=zs#~i4 znF5Z53}emzr5tj;2RnzV7^9pHjO9`q9b)dA2@Lt~bN4rdo`V@b9EkPfzEW)=a_8-Qrf_C?;9ElFzzvBRgNA@SJZ+>u_^wn&${Bg^5jFy1ea+!vTyMlslHgk_GR#) zAN+ncd%2NJ$RvKx5*Lk_9B&a=x!vpk_@W*}pbj9Hf;er-Jc(eAATJHt*&%vtM;%H5 zo3V5I)vj$pCwIpRN*@w>6oiw?1}#W5{_;B1y`2lD!(5 z9Na-g;*?w?h3+}i&cdK|l}I{nOPc#-V~)vB!s zT~kAHXQ|y=?0cgMJo~Ps&qm4=E8K{v$S(6lMN9*Ylb3o&bdwL&g9#aZF}4{{)p6fU zt@*;pzBVT;di$ET<9UAArse)fACT=ezoGfvO0x3!s16_4b!bo2o^Mq>QhE4cLFks} z25$rW*4(~<)7m?q(q@Jw=D&&32Mv!Zm*afAmiT2>_t!oCrInH&!!jfVI#GydzLbQI zt){K0Fuh!J->*ct!k(c@ar=VZe> z3dC)sPMhJsia$<}%HRl@y>o4sxA|;m#)Dm1*fEhnv2;+%diFvQdl0mW%+DV3O+0+0 zQrx!h0@o(U{z1^!cdmJ4uj1Z&EQ-LU7Y&StPs&+Ffuj{mngc15a0Cmb&d{qyqceO^<^K5w;q*E8=O;ZM#){ z13Xa!5Fh_rd%*|p8C!bDYcT7L7-Lu&p}2Nd&x7)81JHDG2TyR+;XGQ6MzI<=qigsW zvGkAKuIQ7!FO-e*wH`5s5_F%HhCwPMf?;aq2?(ouHpx6ZD>LOgC4vyY=b%``x^%0gfzQ;&RRaOs3(iw#m+0 z(H#;;q3Qa!sT3mZxr+p~dig4QMw_%Mi@0+PCf=E?MCKY}UmtC$eH%v{EGMX)UZaUK$#KEX~zLvc;&@gr= zf+qNKU*wAycx&C>d{@#;7}WZ8_F_S#24{s@jh~f{yqIv#$nP_k!0Rzq$|bh#zq7`I zg~OMcpY)`{r0}H1Ps(5?N$SV?w@cSn&t!W)Sxy=d!QMl-f7J5QB6*0=fIF2Xi!;ka zB{d=2_hcZX`6uwCXq_1m_( zX3=GmZZe^L0y^lwWPQS7C0>Ex=VHNn%)FcLJfG+rU7fv?PfC!Y{%327dx^aF-@#p> z-oIWwR=55xPr~+dy_fGgW4DFBi$QOp2d+?mCzbhWH@i-?)WD|7k3;q|{^KvKN2(CV z_@)uW+Z{{fi1?;SpKX)67j#GuD9sd4cib4oA3U?>ZDL&(05tXyGN%q`RnYlPC2{U! z0${Vt`Uh_RD)bEmzK}J%zV5H4NqVyXosvhcU61aWl#2|y{^h6ydgj%OxrR}Nxr~)! z%+!>Vb-PDUA}&xcYkyO~Joki8x%{=va;U=5jk%bjP>-)Cp1 z?>t?r_|Ftalh*7oG|MVv_ZVX}K&Nb}`tO8nzqnfG7ceREmLYfz3soR~Ere8C z_Lg#M9qa`QE);9LrfS=7$tB*w#Y`1M)wI+&Q_pgFP4|*ycpGPF+F^Cl>D}ZHl@dqN zxdC~VUP<8%@2_0lApbFQGpovMmbNN7`L@Z6xo+v{-}t!l#J#4IN0ok(@fH(d>QebP z8U!R~+Ev%izYQt&R%Vq%hutns<1c_o8vCFwN&^FoGYqB%)qhAM%O+$voL=x#A3Q~B zS*(~z*rM~&*A)WG zM!T=JN4tJ|E;x$rhycSHy5$8lIM#^3k*i7BmeWwxVSh9wsllrprdRFrv5}j+yy#s5 z9wS~5C}MVtyL0=8t*Elgd%md;YgGncJ}O4KF6x~7HFIm?AI*~dU*Ul5=nV0>;31X3 z{yb5Yl^hC|wdkfFt7;{hUyie+JIS1Lr^43X^_~z) z-+L~Gz#RH9eV9Y;fJ0$~7;EccX4$vsD$^{Ljkv1j6AEc--Ii0Pvy1yxQ_s2kPMa=l z*T3P8e;5;8sOAv$Iynu~hebFk?Tq;@E^d$8D&Z?H@NM-dQRdTkY}7EK zZ|^q!)`*MgI&MEKl{4;jnmq1>v+v}a&+G`bm7aRB>4Ktr0{x4vUYL6jb`|o6~YE92Gf!UhdDA^*6tR`eD!;0tT+k_zh_&N{52!DYh8WxIHqz{9vu4OB1uwy{I(## zr0YaRM`5yyafw`XOS1ISzEk1??)PN@j- z{E}BeKR!5)*3@U+G5*NNc&yg0Z#8Li=++aU2!4H~wnv_q9M%M3~B zWxSgwYS)*{B+`CIXioJlaD_{FdXc#zDM)v2v!}Rj@7hl;G>4t@wI8&@Kv@-NQ?@3RkPSt-$%d+9E=J)BOqw)#iwEZ^0sIx|$<3-Vz9M4VD>R3A0X89wN z)f)_EYPsw4fc$(Jx9ZkB$sFP_8>=HlQH*){Cl>y%sk!G!?ZSYm*gSAg+IGz3#oddf z_}g9n?MKD}TC*yHm#+&w;Xp?0n)Gbb{``AHxIfCCxcfZ0I6)Nyz~aL8y?CfsHq>z& z#}~Y0xnE_33K&(1>JNI*K~S=uJqOZos%fj9GOr@v-avU}e( z%m8gBYiQ5W9;xwppb}GM>~_O-+Stbh70z!;Ruva==5t>vwHb50xKk(_6p#S5ge-zS z1XETl`DFsbMmM>#)Sa-lk6F9au3O@@rT@L!LaY@f+zDFCQl~187*7?Gs2eBJ<+?71 z<0|E#e%5iFN^8&ee{IELjyPR%f9Yg4NVs5IS1_Aavy&WF!zL_(iciXWgee-J1U|Rj z)hk`FF6wV@_9RiKq)q+KP38Gy+GiFUOPsOf(JMp2RZurPvk- zzZ5lVmi0%^J$-&Y^o4wx@mH}vf`2U!Pt<`9&rK-@qv+v$2hmI^oNC8GHb@P9HDfnl zB_K@!_b9BIoBzOgXh-Me5ys~}e(I>$`M0_BlUarnHh9M*i^eB{_WK{6-aVe_#{c8K zx?H7FRFV)&DwPn5oTf;U96Cx4O9(M?Hq25E$zgIxh!HvDl$>(f%3;n%4r2~;nz5O& znH}tK@B6#&$Nl&I+-Hx^d++z_^?a6wg9CnDhc49?aB6=~#^accN7F51Z~sBea^sF-^B;NhOo%wcz3*O1Vv&m|at!X8%|S z`lAhTd1uYD@k;)~^%!Rv0?qe85sWanhh&qMF&-lQ^m+T)9upm(7Kf0$ws2OyzK;7ehQ3!Kb)tO*CjRaf_h8GdBl&9q1L4HX&mrA?A*q! zf$$`gkQriSWo=z?3v3jq@*pIDHrFWS>od1oQ40tv!myY{Gc`B5>-#I-^B_30z^g9#kk4)NdWom*8X8FWZGtS{?>_gU~R zo~1MAAb51?df?w`M!=nPmSbZj?51Hu<{#M|Wb!vbRk+(h9jTn^9Z$|G)YZqJA2idC zT5&9-9J8eUFxhF+- zKaM2igGF0?e@4m_EOI>g%(ny?XT`a83*DGqv|YY;B*8Y$ zaP6v9-rw7FsnC|t!ljAXdfM*uCi+cEt+cf&s?}b>(V#R~L}s(bR5~OCIve6_>%BW0 z6pJ38vp+w3H^wU{#hq^ex&u6dH(Aq{aJ)}>80ldT=96w^Iu#lld^d1*b{1^3z<*3? zKEu5I;&bw9o*F4C+kHC5@EhdUOxPaeZj|?FNk&0>qH$ihki?x`UFA%l-@szRMdqC( z%<0NH*3*->rX+9=BmWl5FDDIVuP=-`$1qKxJqXMAbp7w1Q~yC|OVW5>Yi@VxIM!0UVb120(H7e$u*yl>k8{jbo7)?C_V6?Q zM3gXqOToZ*&c}rf*d8`kb^4-x8`?Xa`R9f-bN!mSX0qX%$jEcm@(%CL2EVNKc}Z

tK6hm!b zh|Ki_UyV1AHAFdO9|)%ea)@}k|T?Yy~CPQ2}qkF*CThop#C>tNJT<3Z-{ zbSF>R(chjY`0869WmE0$qtn!*fdC;{dkUQ>BW-n_7l5YM#R0t{X4H|V&TL-Oan^|Q zSPXFGmewdDheb@$pD}ULx*p5(Z#9Zifa)f;hUswPGr+0Y8&F}$yu^1pNB!kSPg{LT zQ|+q3Bt-IM>j!~Mb(IG7Ih^TqP{n;(W`k1Jwvu2M--x8W@P}6>w$%!!Q@yXH8lrwo zi$FpW?A#YWb&gr*oA1S1kLth4EKQ9>sfzNW_l^Y4>8%4ja+663nSkOq=&iKK-UPmP z9rT^`chT9l#O?4MvSODny26jZ8qxMX^@*Pk(5(6t;s$h zeC1GIwr;6`5bHdUbNoORd-X~U_LPB@jPh>B(8p@SY$VjAY5HZ?WaRxQFh4P%{V~JK zaN)|Ii2L3|^9aus6T&sshzMCn@*d|N!)ZIM4dm&(q?f0z+pH*;Woq+`N$^C){0h#q z{(O}w@#h@+4v2Gl=~jFD^{+--S+x4={;i!b^{f1&9&y#Km;L6GF;ISoC?+PoaZEmN z<_rQepk>YwW0XS&&4>!XY<0xBy4x3!}6Yf#k%1^sb0zv8wJHf`hUN)+pl#(@G48r)tjGit8r5(=N>~Gy=yA{m7q*J{IZjKX&qa8z$C*t` z%loeB1O(#T`W$x@mLt*iQdG91d>!#rdx5~bLyt#%0{^Ow++E=z8$wcg;Ss`<6DzU) z-IjBN2B98!#icWkDH5(A4k_zrMjr3G{(B=I4x`!v#@zpA6}~V#3l;yMF5jD{8fLs5%ND)1d}I}XJ< zIcg?Y#h!V2>Tc8anUD|97>_fzGHnzkPIz5~1ra)IB=$VBzJ8ixh}-V@SY-I1zB|uY zEqX6XRVr_(6+*FH};>Yj(aTvfoc zm}Kt(KZ1W<8d89L2*>4`0Z>M}c1kU~ zY=(BWb?DBEf97^`DW)r3=9qR^T)>UcZ#Snm?Ch-(`?HKzt~izIEy)SoYgdHVy~*n` zhAK(;n1j2Md|`m)pA)#rP!9t#%h8YIygso{-Zuk-44oqK@6l$r;;9<^1CqXT;ckV? zc$Y>p!hm8BH`i;`)HPWW_?rFpmqSI-MGMDbij@PTXk5@ZBoC`}j67>_;ic?#Z+2G^ zRPm~)x`-a;)VHXANopBH-5zWqQ_K7Nb+Q^o_gA|9(^k{Yh{C)AN_BvG4~z-W`0Z+r zE5E#+`fIlFpo>Q1)0bJq5U*g-^locoaQ}|Z?q(JBbB$pg^D?6b0!(=DD5D`2p!4k? z1iSC)`GbP*i!3acf%vxl4YXRgOkYQlA9+}S?)c^yckkc1JT~p%E&t=GdMmAKarJrf z%OzW{U+ECRM?0+PdMWhD-#4g>W`^i}X26r88(t%Z$v>RCPyY52*UHQ+;r<1t?L;Vh z0h~1g*ab-(?BqX&UjfC}J$H&DHceTB!;qsvLzR(LMm<{Fady<^eOwUB7+s4oVsVlS zH`T_tmoFRC(mfZT{MWi(-+Qh_PGYCXop#jNoiAw6K+J?E&g$82sWu&(2}~YH4*Azn zoI)J<7e9Qd14>M985+JC0ghXw%|4FR__{0guFD!dP*85IQWccL1As?c|5)bUPN=9^e?(Mnwj)!u-2K^894aa?ZFK-BQkGx|Jjg=ce$1q_} z9Q(1Y?K@MsFqFW9i*_U?y9I`i?jeOM2><;A%j&uw#78>{R4okklX-!qhUudWm?Fg~ zHY~_GZ2D*zJh&|z4J>uL(_ss^tjgW2W?v(cRP&sNa(F`b{Qseu{{J8-*;bUuQO#Ry z%fv)(>Vx69*ynmyrIV5q_W&W{3txT5S}Bz!FU8JuRoIC zYRk@guP%raEc9o>1XS4NLmQ6*ok#Zmhx3y=A?H{c9Q9NB^fxt!VZ*~P>D)fj;`cSj zU+Xc_m-2dBM;$j-;z&q#_^kih@XwssJm-F|Je>^uE*LNgnIEc7R2%E6|MGyb!BZ;% zKLI*RZxN;@nd8nx@XRO2V~u5#nl^xg-%ai}o%XhCi&O1psFia|BkLe{?eV#yz62dX zhE*gj^tnX8&G*+(Q^h}57)LH67p$+gE1fNo1JMRVRtwl}zEqJl8+mUZOS^>}%eTW3 zGU{6yF?~jab7!l$CQSh3S;%!Yi23uv<*@j2aWh+p_bJ>zPl#VNGqf}pCC}w3YU5A& ze?FX=s`=!Z#N5F7!G)TUk<8pQg=0p|#axIvboGON;FJ-^G3ORREJ#cDmGx6$5um+1 zukV#!V48usZHsqeuJ!u{#MZ@MT(#rOuv2I5UMM%H)m=un8F$7wuZJ&=RmuhB%YO?o z5y_rwy%O1S2idCxjV9{dOg#@q-ah)k$&(~?NpK_LUrHO)ef>+Qn4x^J+=iqX>`JAik?iir zjKHz#-CuKHk*dalk8(0^lSqkr;^27z`+!B;$&UxB8-UE(Ug_CW^P7)jk3r2Qdf#=w z^z_tLl#FY=VeFJLAaU-63S`{G@eh>eq5at8ih7%n;(P1t>}0*U-NUM%sYAl@WA*y^ z#bx*}Tkt^_otTsa3BttF7M<%ltV4n_2{2?G-*BO5>75}e-PdWN378Jc%=^2$^x4>N zJIBAPUou-03q2tF1m7F}n%I^S9#QE@3drm#?+pS|t_M%{VP7=3Bg`1&Ks?dmRE5#+0aCNhuoT(*oD{t$M zb1ggDBEr}SdTpx2H#~l#BUJ0t(a;;#+;R87K1M~MP3T>ovnk&&{oqP@@&Te)FS_U< zrs_b|wcY}|v?B49pH-&Xj$Sz(I1<9kQPong>N=lh!FOD5Eolv!AvrdpAbevtgT9?eyl>H)tI5tbh zS8noMGKY+oU>`^7;oRJ%vq_1f{RA)X&|d3dj?WCGVbqZd)F;GY6Dmk87b|Rem!usv z19*3>X(@@KPd`85uErPoPpe>8(*!X*H>^p(vIsN*M&ZD!H?)XlAEW_ z*xc#XQUBIa$U9H%HCoDInJl06T=_)$5wDOJuJxZI_$=OS_f~T5mclDuu%V*_rjq}> z^-HzTUV|{EpUPs%SQkF;UXuxzu2;a(9g)ydy4kcXCvjH!%dUxQh?NaKYw5v4sy{I?uw+LFY!BuidB=|4rZQ zEmBIu^A7pAl^;VF3&9t?jfl)`TjmZ3JF&lgEukRU^!Tai*z-;WQT6*^`7X=HsX(|UVVPo?QUQxB$`_u*`3!1miyan_$XvSEo8YavY^bj zN5U#{eKs?*`h1|#>Gl0zXvZe7Q=QjnCMu#uM(>>vr|xDbkl`srJiA}{0I~YcX=X3B zEzVB8$cJ9iC+UQ76P4GI{R&k1!*o4U2Y;QU8k&j<9?4wc9BIH6-}=N*X{ram^{wy? zP&#h=sC&n}G_uVnbS*evE$4$HHP^i#59w^#`GSScI&R3bymZ!b%pwp;oI~cp?7XAi zxsI87a|2FK{tk0vni9phc^}E|-h&{ww+Ovt-E#)H*yc!$C)S_?A4hrSnxv&3hAk_2 zwQzDJZDzOJ5ND6d+R|a`oey%5{qplD0_*O6IiL6uo*rKrxp!Gb4%D4rWBb=;w}u9# zuu9J4_;I8GFG1{OIs84^`gpEQR*T?RHf*UeCgL!onm<|118@5vMahM&(S0gSW*_38 zNm{`yLk&e@w`ftSDXpD_wwM^j9mFeQGb3f}+TH5zCyT}u6t3Jw1sIkC8z3~5zo8Ubj_E%Txez-2O)j@2_BWr{Copl!{r`kS~2I%Gd7(jIUfM(;1y@tPF2n~$&T zbjM2XRdrbp^FhZ+`}W`19`9~Z=P+jMt%c4D6An~u1~Q&7*;g^>Xe}oXl8GeTkqFP` zr~1j5ApaZII6Uso{Dr@IQiiB7zQSr4V(SzkH@w1^zV1BWr)OHYF@%TWq{HhnAftSJ z$nCk0c6d+!>Bzu#tKldXrKM6dFX5Qmz$~w#j)z*Jjhhj~D2(E}1Fgs2f=z6!SHDcK z`-5N+)OVWr(wQ=?rnrN>l^FRxjmAOX{D_;&I<%A^wl>E;R-{uEPz1pi81Gh8=Gf?^ z&NbD!-CJtEXx-2oUJ7-`DY?ZzX-r5x|G=Pq)Gq-2!-iP8=5>JZFt*~teaU}=SnC76 z1svRg*rM;)bThTg2a3BX?v(y<+Y}u+4TBW<4C1?5+HWaYAi)7?{_o;?fP(s;dg6 zHa`}E{Dm01!CLz4NZ=%J+`vB0JM03{ns%7|oi(&tSPwG5b5y>-Krb{1kpQOwJ z>4)?sa<^v5{|SR*K^0YT!kr?&9I)lPbsWTRLoo^cM%yOv99&~Vbfcf`j5=>GYMzkE zZ3kV){vWPWS-}@2Yz)n-pq2=>JceMo-&eoUNPyZ>0rBB5rm%sI97r&d2eKsiO+tX8 z5ew@SiDjpZsPp-P@qQyZR?yl#h`Axuv-2v|jW(xW#mgNcg+Gr&Q&{zHex3Lc{^G|! z7b#t$>*V%7*FYxc`(Jv&ZtiPWd$D12US9SDE&W4Z{zLjBcSZRlGyMVJnr31BKAgRQ zQ2m#x2Y!V9R49x+hW7hgf*O=-CH|O#QA@!xA-wO?j@^BpbCn{%wU$HRr*BooE}{Gz z3<&AkeWSFHxe2%K`iL;_ZWI|keqvCJsrg;q4tjy|PVD+2iuDsQF^LUm@0|gI=fKo` z&M0l^n`=I|8Mh*j_8Rz=;Y}7gjX$;_mIG{LG80rdcX_ z<7++HK7F@qqabg_UlLkph7Moyo*z)(Hkp0n4tfpw{*8^F;%{ zSbp5x(h7TINPDSt`O$QmzT90!btZJJFQM$M+8qV;h@!SI5_^04 zAR4aJv5z-bu`j3))*bbA4V}%t`Fkn&^kMeh->nt#f(H0sU*unDNL5rK(fXVBGZavIO`3(wYE4q5=S_q|Myz~! zBl{q%<~>dHN9bgg<>}_*%wIH$X3t*6wMq>r~A40 zi5#JYVlFxySCvDqY$Uf(8Xfbd{@L&^YIN#r+8tWKZ_Oi{rBnsATbWEPA6Qjf(!V_a z75HYL)j~hO)%&KA3VQ=~Y>Xv{kig299b5xE;+{N79~qb} zF!Wu}5+4xz)JfH=Po7reEa?}7n9%>#Cs6z-kH8XZ@D5nPP>p`lWES=T|F~b@#g|_) z-Mh-&u^+;H_WT?Z6^es=eS%+GjE}oDPAk!R^5Y_ut6z)z*DS?%xF^5tMjjehFcqqY z0MSWniQ%I z*IREMPt8z0^C}QgZ32MMIPIT0w(XN?GhsC@PVon&S6;i^X|Yp(Rag^r<1+s^{h(_h zNAc3wf)o60UOgnJL3{h#0H$cB&8pS=_yhi>MUrHx=u5Kb`Zpl1OtUkIk|FOz-}ccV z*B1o96xlClGqo8ad;Fh&=nJCg^)MY%(J zJ%2(P=@rw-megI{u1})N1>pv)1TktSsG{JY7o5`;sTKHl-DRfxArcF*QSdss^vJnl z4gM>U8P5Bv&i~V6T=gf6mat%kuv9R5n$np6PGdg9_ZG~b@kd6-AC$XhCd#P?U4pSp z7`29?^#GL?x@~vL(VFU*ruocu0~eUv0^k*(FT_2fBQd0RmI2|E#0-E5*xv z{kwTFpwaU??Wt%q(A)q=W5sU@NB&0Z^{cC(l;bDXUqn97ktwm7``S=ZIaa+?Ml^-! z&F~b>OK7;cLht*t$Bce24xihHnmW1T$X^0IDz*aaet4e{x7xpQz zBV=avJh@eq*u6~$&T90&tl6kXoxV&`=}<1L`<7|%dU%+f|5@*U=1vy`K>1+rEbl;Z zMNgSy&#-MV%JuoqA0=Tbg()ghe_zp;2iU_7{rJlr{Alj59~fci^9a51ZrE!fIj+Ib z?r$)Aqx>QLqL@)?F~22mdCVt<8yel0=Hw00x2{(4B~B55qfPD)Wz~JV6|t_=pRa0F z6ZrUH@|)iT=M+WG!r^AHggUY-GX~Lino{U9fc~S6>~qA>ari*sKJjU4jv_ ziwb4`4bQrx17x!Z0^Y7*FjLSkSOLJ;E&MGQm0+}iTh0a1Xebtfe@-1%EcyXaz@hrJLS$lsG)n?UO|xnOLbU zBjvx=Y1^shRZAO@QePvVWEfnLd!7-}Lurj@>8*vRd>LHr)Av+Os620YRntacpvPD{ zqzmrAHkm!Hz+OC45|>O=7`ez4x1zL814|iYlx6De6v1SFar_|q?BNZh-olxa5T8^} z?O|6dU~v7~bEP^+e9k*Sfv37Kh+&zZ5;$VK73$5}6v@$@y@5E~m$sr1rODj)bhKA( z^)1U=!p1_J+|hju-*CQs%XlX@_`~c$&uZ_;F>bLrdXfD)(wa~}f<>CB(<{;)fak1}DKk_`a7<*8gWD!?(# zJI6C*66}bd^k}~N-yq73!4da#avd;8_hQ~b{mJXkcFspy6HMdKL0^;$v66rW+A=<$ zKm4Oqf#!4_TdDO(p%jJA$fZYNc<03wWn zLx?IHe#_WM=2a%TsV(w^tv%RKY%S!HLmHML4aw=-z5CMbvhE5%RP0p# z#+Qp?uv}g0ww=1G`;5g*_eFX~Vs6AhX6fAhxMhImdl2)&(3bup3r3) z&X4{Ya`AADK9zlA!e%%7F!#M%l}Qy*tgt}AAq{mkz+?52zT4FyUPW9AIYs2Q2l8`R zsd`|2VK6qf-jqVS62_02y!|kLi#Mr2K|Y-On92S2Z+~pi8Hns-drQ9A71)_TWYx*+ z>Q5?Kx2!J!wieN%-`=GkFo-wY%yAwRcK`9L*)#=%AktDUF&=Q~l7lBt1Vm zXrlw3UVtckc{Y575Zz}&ssQN@{6z-dI(*yqxbA4IxvK@=5s!s$7I0fQ&j^Q5za%kp z$4caTEKcd-h_o%Uzv*5E_u&oS0yVdqGoJ`#9fhaD$IOfLA~DHBmfU!`oDBD)d5~ni zs|01-HIuyO9_XH7A)^xT_I;kg=uqz)>9cUuIltX;mP4r@rQhUa7}Vp;$J6a**yuI| zfrO=8g3ORQYrEP2P2a3I)$y5Qv!1Z`z}0k}jrdc9!p1c3)K z2R;VeP}{3S|6l?WKZ%X}9_6pwO8aGd0?)pXOD|V6Dv|-{{v2L~a;yk9C^NfG7djw+ zstGT|_rxTsALV^EvI$jo7~5&N<3Jnn!)d0kD*EA0)Ch&w(Zz?2N4t$FY2RGsvd|uy zI6yuARJ7PaPOD-{FoV%KsMs)U?FxN6`IDgAL%A~DjWF~xPn)a1->aC<7BehR;cp&> z#V4khW42G<4r-b(FW6C8IOTtE@bl11Qkg=5j7s#P_+#y?C3oi@!ekV0^`$DB&+G`8 zkkG1%oebb!gslGYu^hF+z2>kyb$Ep?>&xbb(eiMM#p2v#hwYo|J4-dSfCmF%u>(pf zT|nF+U&_f>a?s;*)y2LMM>z4T@3t8VpFamn44yo?m>J-4mT}@!b8&34bk}y0Nz68o zJYb+o5k|iwr_So_yq3E=<`&ofY)(9VH_S!J86`-*EOYFeV4Nhmo@RH$e1C5_K!$R>I;KWBijjSD3{>f z-b5*L@-?X28Ii;O*3wI81M8z~92?cIsS2tf!U*Ej zj5&h)%p;1l}3|%j;K{pxf70 ztan%De(}t?T@IQQz=#K$J)916ZMnUT_1VULa!s~QFoO6)djFK%6TYp3&24lbXm=BF zn>5L5PIDGGcazA0VxyXB&PMRQk?{k!K=N1$ zTC(AI*B3apRSedmn5>vn8R(%x#{yd3O}18Y% z?l`8+^rg&z>fEZj&Ow~|n0C}o&91DazE>?|?f`XNsyEZ1PeTFuY&Z7!8TtAd&NaCq zr^!LH)T>voOIzk_cRWNV$BjR1BE3m>NBlYI1k2%?p%WLBVblK8O!2iA!)(!;bTCR5 zrdy{dAp^WMtV>N*=H2d^F%x=0D1`&vl$gF8zYAxfp(F7wX~&HoP5*Y3pbsY=pN(Zl0LDy1Tjnl1*|F+U@5N04dyy8mt5w4sienBhjN$}{ii3W+tpyMY9 zs6x#R&m`qW$cUq;5a5-ND}e(}p8#j7wnW?NA0(I#hdn{75oJE2$RnIHn%}LLbUGh* z|5ZcyRE`!<1yZO**T@9-O~x8oY|M1}#nRk~&61ONbzTf=Q`n=yFRCJwLU;F)d}RPA zZk2!{SSI-H5|REJg6msD7zkTm@DVmI6bI3vsVnYkk(-UYhD-pf62+L=%Z%biV%qM; zbeTwl9UontebLcgDDl~&aINJ_-%;(h-*FrMAJoR0=ijPDHk@MP&&F+>@b$pfh8719 zGB!%TxYLj_+9)sWqnD^vEmyZIko97hHX`hO-Rdlg{Ks2|F&5SM$tMSxn24}yIBwz||QNLSqHfv&FM+7&q~+*4c@1ECD z3;Ao>Su0&pVG0BN)14n5<2|2fGam5C1=@^lUAnmkSCwh`?xWvgLSz=ExH0J#s>c^K zz7M(coZj`)dPz_8x&ehe8rkbqs&4PH{wu z6-;65*ofFe)Bv6z7{@tj;a)Y!W1*GS_CUH~US2-mQH4ya(|VWn@u ztI-Lem%{B>r={QyzjQ{nuu?Lp`k|FI!uQp9BXQ@)QAr_ds9(*w4AE>HO6BWt?J zi#b!{UxwL*kJg*=PnXksA`CAEz2dwG->bKg6s{>2j8%@QejPm<6q2)8V%caU?WB-6 zr!0t>v7UQ~UpdP8DMI3^8X1|4vy)q#@+0oYd7oZUa+V$b>?vrGI^?fSZ9KVO z(PJt`_>ZIdrZ}q*@))sbJB>jX_l|1(J7A7DT>Vb5#VST4oF5%hHH{Y*Wp*Czdwc8W zJBSoO;n8!l#h0(Nd_5{~1n>gvz*{~3qjL}uhImKye!W<=y*b1=ORK7zDQ8X^F~vhm zLq_y`4c!M$NDJGA_QWivNleA}HxYJZ^2Pm9^_3lGvC=QBvr{evc740-b&9Or498c} zSmX-fJ_6@?=hop<3dNL=&)O2w^U;^Cxwgv+0BB#=}1Gv2fGC@C2+z!9|W!vU){^I0L z^sxq&Ffx<~L00BBliW-lYGUg7%%x z&KP{ab2+(c=MhDH?CJG@EB5suDYe01GA*tFQrZyHP+ek_i=fQw{?|W*A z{m7pxmA}b02=`x*^G?D0!YuXkt_RPjA)i&g@`eILJB4qN;zOdPluD;2T9g9|?=%w5@JhEq`OR@iZGq^mc`J2}3Pt%vU-|!Fq==u#^Bw<&<@ty7zE` zR!_bvBx)I@hCKz1mSgr+_vzPZfYt9AlI(@)W{lMPD$@IdY}BU)ra|et!T|1fE%fjX zZ$f%QkCEpz?mT_?QnmJ^2ehxYUlMd_E}qLg(;;Cf2HYrLk95#*as2yiZ{#)L`Hdd! z29RjU=xvHfNRJj&1&KAh6uGOk<>l%lvMXz(y!02bbDxA!SeWG-EbZ6ho*tU?j%eQ2 z_xwr1hXaYPcCw68K5KKURvC`{pSW8k4**DZuKDiJYHKdJ*?{)nL@`=W>K8IAnu>(} zIrnvfdq+{HWwscqQ~|yaKc|I8G_M<4xFGy)qBtkdi5sOj1J67Jor*5W1&wWw6z@$f z;R#D$7nP2#^sp6=?>;}2t}o=m8U}lcEvM8)gjEa=7q*+auT@qLuKn?tHBK+Oc4XsXeH#E~5${ z2Z^StkI4p+-$0Y@DZtl?scpwxS3fftx7{}0jl2Qp`4&z;VC%hinxnk~PE@y^px;b2 z33JujNrds_H;w~mE#W=7o?lg&yY{<`GfZNX=4 zBi$hTzQ9{9gC53ic~;9GST0%FOEtV76r0?&f^Tw)$GXvuFMh`)-dy$D9ah(ANxsss z-%cBr9%yt~^%W6JhQgR~!c!VGa`Svj2`}tTBzNOCp&9Dnn-xFVh?O{GY`VMCN=2W- zg~?!ksD#WVSu2V)YL4|!y5B3`n7$p-@@pf!8Mjx;u_bk+E*I9nk6_n-rgb?O@!uw% z?%hPI34nA?iJD>hz`TZge)jI{;5n$HL4rzlFmcMw=Y1m(N6&7Bp7;tH6>dSMH-w5i zzvVuB(5_*1crd=9dzyakFCxSC->{*Uxe7bPcm@!R4~PC?7=rp4>q|0sTGWIQM*-vi zB96!#@?lE2U<}S{$)2z)llcENuXh%Ofkz{?2rg|`NV{Ahz_@T;i#p1^Er7MQnPs$# z;?jgBbwRFhwTaW3|0J;hIMz;gf8MILSEnN06GZCh{kUu-<&am-NYZ(UWV{IO#Q;r^=c^ z^$wkg0Km9Ug+9Aa{MIn(&p9Y#zQ;~o5K2BTywy5sONopPT&O7_v z{7TMc<;$^gz^2s%oGs;I$A~ogNqmmTH)S8X3&OHIdNCy1LhiQqJKJq!-e(m7pXGHo$Xg0KE8(EbmQ%1*g9qZthKx|0jOtb}|JE3qm5~3IP zPl=+EI!AZ8tvlOEr^u^&-mB1ii|k8ZJijomkl_1N8g=+jWK5aM?+ss4Fu-D^>pm-8 z11y>IxajLkrK?5YjX1#f(Fe{*|IrQ|Qk?`Sg=qRj#}}k{lrAMmd8gUMVivreO~kf` z%Npcp9fS<$0rb?v;y3LmM?tMPrP5jDK){q1^VIYlQ7-69wcmPJ#kPLYnk1b@>IC;Z z!3>-w_u&r(TshC*c6ObuN^RGZd-!9t|=J6mSsjpOe4HPUjGC`8*9US^f7y6)35SdVMjLa8FdX> zFpp%8T%MlzF)2#QS3?vDpeK3JF&ZvtUy7sxDB1q8tg{ku;lCysz}hTo>{ zYHeZe@~KuW2gJy4Wh_!lyqr1jQmB}MBD1j`s=+GGp+*A+{^uMYSXIM5wQ2se{xh|$ zNa>}_T!%yiSBc?3xKOLUY+C5@wnoW_RBL;`#(NT#@_^;{yL&6aD0?&EBH6-x6!a#y zUK%#rz`XbTM+k6n)FxbAi6c}uw9G5n}0TZCc-iePSK83bRQ z__fb&D<8;Q6^Q{_O+;>t1dIP$TU|%-vT2L1MNfK~Kd*ksea=5-awl74h>ksJrQnwf zZ^*6m4aZ0`xf*U2fmcLNT5K~g?qK+&=&J70e2}x!>N%`K!ecH#ane5o(Nfkl5*MEbSRF?oJ z&g3zSB*VtVNueYOwX9A9ZF1`Hoc{8QyxEsV)EnV-xi1k)3wQINKyvr(7nOJ1YJ<9c zCMhotHasay`j5knNdM|5luR^R9fzjw6F&kwhV$C4s#IF3X~aari$=bQuC+vFX|}&Z zt6S;D1QR3x=eQqd1b6<{A07@^8QQMz$4Krp*&p4>er59d4UXJ<6|3^xS=mIiM(@miO9RT%!o6He5|IqgT-8MNY z2vs-)?t$YulGoG+PC=^vFl_iPPTwg(ff8@U5~?t#Vq4{I$>k6~>Oyhh1PW;CN|hAi z?5xBZ_asgsS2q|g7a|Z1I!4Uum~3P!q?fmGS^29s9lTsIBIxc}%v#ypx{JXp*_3M( z)Ur(GOR6F^7Z8l_cYN#FKf03(2$6Rt#x3YBoITf?@?Hk^z=uEhw27&@LFiy{MjlSJ zL~QMdee{~tYf+QRs3D}gPe>_OLRP(?>xO$0)$^`?uDLEX#V;A$hEj?xgIj)mX%OG~ z7;kYMQ=mw9$9^`2mT`Z>4CD@EFiEdrpUvWq0632Xy={J;-DPIo)Og$ZsND&7EOh%I zR29MvrKY(k4evvQm87@cvEO00Qua+~G5Q-I_W`%M#&4JPZ~w7b`p;1*uQV#y_=8)a zBLcE=UGF&h3}EuJ#zl6p+^=;9)K^de$2+}|5xzn-1)Hh|p{OyVLDPp}go`cOE$jY^ zllSs=n?p;amzR{VdZBZu+(0+IfT-^}GT#g%kJHYq(bj3s(;_pC$;%$@!52C+KYY4x zNS)OC$1qW8uH$|6oE$-FjxezkvaUA)eAD=7CaAf;JQ2NA|I)OIN}KdXl-$fXTxD^( zT65D2vkS?dj}x$A{QupxTlH?tHcV>64UHi7=eHhZ~egIOcfoP1nb$RGFiq-5b?VFF@%IOoLgkaSQp*HvmUSZ}@zwa#gp`+orBKpVgJJ8#p_J^x}u+}X$rv1vDeO4E*^(czRS9Fq53x7=iT{Fgj)4M#@xA-5ZV z5i#-Y+$Z=3oeTL~P8qA5k#WnEY(V~<1hz&}a(56?x3LNGj`3hbDIHp;ijCP2rg$}G`w{GV^PNjUgg>2HIO-9Q3x@VC!a_^8%CO?Cb>%x8M z;2Ni&$hv$e*H^akC3Kl$9{uwX^^N|Cr1$#VKNMLIISH?IsXm?OrcP0WpV_3_n@-XT z{N32xYsbogOP`a!*Db%iy{?R6w?k!aRPIJ)%CpOcmUTzto~7OEv*kX!hnDk4HsLW5*Y1KEZ*THWz^L_3+ABW}f zU-BUHj5&S|IOKYM03$9uLt*nWz<89ZJm~OUF9?M64~>07!dasvoR{Q>Tz3rYVG0M3 z{Aq9DA3^eo3)NQiBoob?mohuI-U(0FQxwFz=T&J9bF1^X+(` zHTYU#6 zTX@HwiQ{+tZXW{+o3qp2zLzGi`R{TH}!I;9n22PTxHFQ2RICrr#?&@u<%=^3>LsEnFu^ z^&W9aM{{UKC_N108)1s*Vd=atF9;_;B&>hk=dBI%z|08|$iDSi7&|7O-gF*aLVm0w z-?Ul6*0J@RgTbL(KK$U^<9|dvWS8p4Y&c^bo`Bt_jFhi^$U)b<7#?2}^-@>-$U8E$ zi)(RVpK)t`sFSeyOVIw+GHQ2)_>qqv=9w2z+~8sR@HhZqjN_m6wW(8hfF31je3E}4 z_l@Ukd`J3>7a{S=sNbQEEA2ag2l+wFe0f^GAS*I!Pvoyh)Gpeg_TP4!dE)5gfXOFe zwNJ_h>dlT9C7ue`*mUT!-=SjG=*hF!hwkTaa6;>HhmA%zc_U}~U5rlu_UG1zk@N5J zAJjSs2Z?N3O}zv4Wea?lz1b_WcYDlS9I8#)j?;dzwXl3s=1DKFzF=P`L)hgzhK~Nw zbN*`1A${f|yooFIFYB;**Ii@BP%fuFxxA?DGrT8GJIs6#-g(Z8GS}Fv%%tmM#_~S* z6~^Eh_Y&^06R){oo)<%|(d0j}wGCbvd&W-1_}sBo?Y8YqFFv5zgmdqDa!21Jh@!#f9=ZsGzjXxS6=HN$U^0EAT*f6xm0B9b>UJd(f$9^*hhy2K5DEZYFkxd4BkDc5v zs`BIaD7rE=vgU1>#U87gHu6JVyC7MbP5+v&j>aCYpFNjB<&3Ok-mtpWg z^YR!!W-W5YgYosy$O+Y->j=p*{YO8vj+!6lWw3P+7hiR#ex(iNc6=xEmi$;3h&*H* z^jw1+($|h`P>gS|SJL~eIHD7W*l~|R?xpkly`D_=SGsfzn6xH_rkz9AJQ0dnA7kC2 zX7dUupDJH*IAMt1pwSty9f!sM`?~Z2#T(HCXRds&vioMwDr!b zc_!C0ex<;+i|{_andg1I-w$ydqIMwEUkI3!%cbyjO^T-d6HTA7O~UHevE6!c%998G zp*h%=yz&p^Nt}GfU_6bt22^}hyw;gW+LWD3>?M$Y>Xr;M$DYLTi^y|bT?&_rbqJ04U&&aAZ_9+uGj&MtCK}%h&#ZM4p+8K%gUjcF%G`1DmjgqSSGve!z?OkJ zf6V+RJTg$_IwW5ozACfBL-LG02u?V~t#lYY6Pmkb{tDByeXf)An)2Vwp$R*uO+QK6 zsFFO$wYEj$7m4Gi#ua}p@+x0^sOy^YU6eR_xNiS&UB1TEzpSacCjEK@&f)OT_(kIR z`4F8BL9#vUd#L5ly}arXa$U5pQ{V7j*GW^w>89ZyN?vvvn+QY3lc|5gnYZ``T|@nN ziuSMu@MA(L=$$LZ3j-Pg|L5nr3$<+yJ78jt}QxTlpiDrZC7 zCU{fcXZ5(>e=}6(4(WftZyP+b#U^FeE_f=dRQv2Q^hwVeJy%O-2f zdxqM8-YskE+4ROKqw;s2Y>;P6s}9f`{DOhtj{N$ji~6H%K^XhVDACHYS}3IAR)5x^x34_xGuVR4c#2= zgD%}`7n>bG-+9mtpF>T)ma+e|ADz4q6K)WxZ}gzgY-9vB5`0CvUs?g>tAiJgd-I`? z=3tPG1}_{?>ws?#53MrP!!NhL9zH%YUkwQrvjLy7Jd?|=LV=~(^n_a#Tm zRaVRO<*1L&OXkA0m(Mn{XX9YmY;=xZ^OfNfHwH`H*;~2F+Fv-3j%9;A<|S?N*>Rdb zjn_KenL!*|Hs&R8WH>0{P}=-? zw37#`I;;PcEorqM-1>^9hDxNzPadFUC?K@&DM3t z7yA7>(Nnm@yb;>4UsO^@{b%<9Uu^dUF=63LW%D)8g!eMkSD)&kkbAqqE1Ly2#Xksr zlF0tLzjYIoin$N3b}>&ncy05tV82o^Yk}B|U+@8k6Q0?*v~iaOn|Ih(x;XMi=82Qs zmoxRhEi=Q<9B>mBmw%I&eCk`coCBMoNk^0NfHF$O=t`%A_xX!1=99bs@iQduo@T?p zUo0qU$L6y(a9`}}Eeq=Zs34CF;WMmeyiSj?g0w6(dbm_8Y@F7ef>Ipn;`t8Rr&`FE55y~uG0 zz`VS_lxZz72aT4kK90-}axZK>3-uX2L;RU)+jnOJ9-Vuj|EzP=<&0f>PU*|H)F+y{usYk8Li`r8hAON{0vL;CEp|N8@il)^^;;=KqQY*LJ>Mxpv=8%38`cfBa!`cyAuNfO6>uv{r;vnr6oxFXC z>oA@d%npDQ9Uo$bmv`br`fKM9`tl8ZWVdy*D~sIvCg^8uAS~qPU)E@|_D)*G@uTaj zyU_#4lYjhv=~{g8!!bDgT^m)ISaqdu&B4+$i ztntfQm#(j{?)laAj2PK>U;Ca2nlHzvK5R4XcGAgph<48bOyPOb8f9cBK5AdUCjU}l z+Pv#O`HhDKUN66Q`--bB?0cU-)Wf{0kG>OLe{P=tPg;b`;kEErXL|a&Vh^`8uF%iA zZ&ApKu=OQc=OOJe^MCk1mvP{WoX)k?o?pNE-phR5ly486G}r91=YSIo+836%`HI?yse@;8^>sLA`{KUdVB3y4 zsOXvw6}>0w{j~G@u}l3!{rVTI+TYlykj_SXY*NSiOUJFM$*;7DAHiH-RZQa4JH*!a zc7wY=li#`4_%jE`PTGLH1fH|!AnQt0*RHP%euvoVK1Z=7y2q98nVp+1EPD9W9RJ1m zD$?(uzQMNEBJ&06<?LeDx{6)0L`fnm}~t|&)!8bV`<{2zeT3C%*t}v@tI#IdT1wg&h6L(GuH73auUCF zg8JHg?KQUEYZZO${zbWt-TXzmwCY#;D8G7X_fH}h%^rt3E*-`Ohl*YAhE3nTB(1($ zRQ|mTJi>cgv3(O-M`e6!EMB_wIkPg$zwNRi54pQMbKSe)-|OA+A>}93b{&U7yuXH_ z54Z-lcWr;yyno+MB!_Q|?~)g)?+T1R^=)u%O{w4MAI6t0uh{;t58*WiZCP(ZihTJv zdVE5cnr`{FypxfcJJC^bUk^Ck8jRyK3T-w&z==#hVl~vR9>QCeIsq^+-xZ9E}&v54V zmRGn}#;bau=zwO-IpN0L_PtkcezDta!&o3qShU?b?tuHQ9g*vnCs3DJk36DYVV+AP zOIVD}@GqSdhsf3TRJy7u?_p@WwoPHiCUov$+poUVEb_9)_P>qVuXO~xT}SS+VasdU z?jO>cw_sCg&-w?*kE(-s`N=Ae&^^L;Z?|0s<)h{mkM~}F^%Ur$PH>Rs8*i=RJl9mf^@ia$m9-hP4lE#BhWi??5({!fXY^*!JH^6nRY&dZ%H*lg62*J|1LmW_a1e! zkhf`#L(sT8ns4*I3HS6}7QN;xXiLAnv0%~Og?n+YFKn52YI?(77w-3B49K^5gW9W* z{Zo|xY($igUM&mR%o{m)@09=K9b05^jg;>}OdHGB1?0A+e0MaxI?IQ|tIzfq`O8;S z-%@dpFWo_w_}WjX?5%h2pV7v>oPB?fFRu6{t8sL(d9`g|zvG>x%>f&E=#gvWLA)D@ zv!NdH(3)m>d=(_bPql3>AC&yFxfGIrZ^VnX`JmkDH)V!LW*ean7TE~vMs8m;-8ZaS zUN?8Qo*usEFi+B_9*LV5haNnYbwhcsdgq!fB;Raugu0Qg?w>?Hnuqwv{i&0F@vMC4 zQa=~!qFyaWI69wtOGiYv-s&vP1|d>kFQB~?Ro8xH&^11`w;^`2Kgvyj@UStO&F9eA z8&9u86c~F5lQx^QCyrb!)5hhIB?Z{HnzG1yh#3NTF)EWDQ%}^ikU&0|4W7eqlhh&G z>U=~E73{CxW7CuG(g9;GS|{Y54n0g>KR8MAAHI2!qK)}P!jiUiuW|W{Yrc?ys9#wc-wmbcAz!z1FvE+&_GRgO zPTr9nIoH^dO#RZ1cGFJvh07-5&OLGf+F=bZNz7U5+V&bOUNnj+D_}PpaL+a_x4o|R zPFhrbwE?{CEWGC(p3p9WEb9#<;d)tdm}ILzJtKG}FE>O8%TI*MtB;4~E0B}; z$h*sv&L-nm=YE5(LS7Ap~556nBE`dyiw=y z8AnD(;UPcZ@q$KJ^24Ed*%#^|<7B_oHqUsuE9}_SA05mAz{p6pt7@(z>anSR?pr1O zZhnjzs(crn&)Ch6t^7renXgdF{)vpA$e1{kIHC1c=fU|R^`Wl6W)2LGP&vxH#slG@ zJr~Yco4g$(g)@JK$ds<$v*xn+%=P$L$avf_IsYE*@?d#1e=daZs@)>9V{0+C(te>m z&-9wm(K$rsCAV~BUnDX4jIm@-8%*Q_^8zak_fsij#z#CyC;Igz*Yv}aKI`qsNN|Gfg|YW1X;1X(1y!y`_sHy+b3t0C#Q0uI@FG2^b3z<;2&AZkI!1ffjoV~y+W*C{OM`;#vE%F*GRsu=jnrr)>UNd zQ$6LeCVI6tO1%CxYtc~i&)PL*Dtp$=iEDk-J^voJMbHS%>WVmA+{FA_`3(sf{13k2Xu1!&cM3@uQ^CBdm|{GgKRhPm?^pa-I++ zdO(F<_d*=o@EdX(f;O0OH3bTbnIjT6Ym?k-+e=PpY@Y=8bei!!%HG!}rKuZPq1t8i z3(dMR`MO_*0FQjFBiH%tNbalUQcv}|U$(#7G_aSo^yLFzn7B@e9Jv3s%ul`-DP5mQNSSsJ9LOyof z?NC`@FOob?q<%1CPWb7}e)OOI)UuLy+CThg=F!Mt`=V*+WV>YWZ@g^d9*@xdw1c=6 zkFNGS>CJc1DTB|={!pmvKy|N z9rikX{4~m72k|HAvVn0N!|;}J-#RpdYva?bD|1a|?TOM1^W%y%$%V@&m6Ln=&)kV} zPj7k{Kh2+gzP;i0MXq21Te*hYJX=rQd~uS0ay zcCBNQuW>B@hP2JJWoZ5)Rl?nF*0O#~=ef2A;pio^`gct+`W~6{!HwA2H69_GkKGSZ zu-DBuIB6GhZM^Q`MD>Q+N1f;tH6G(|Xl#{GzVY2$e^V-7NZ!i6C_KVfIBCFd$9Iz# zcHY9Z_L{FWQ*!a3(K*VXY1i-&R|0W`xbj1Nzs3}>NnGvGZsJ0<%`VS1c_aSFXdk)! zN`A?SFX8n=jF~5L&tES834KRwg8FCu?Rl?hW0U09cS}=VqVeD_@b5Y0vT49Znr?nL5^DcweT20O`SUyy{x{BBX&CG4 z4?lVkK2#mhw9BDl$CWA@TIbe7*f`-%-*K?zZ9Br~6nX4tth7fa3CZ4P)7_rhIR9px z)~;9D$v+=Hk)AkmXIwh^USq?dLE;;n%aO-#W?dQTx*W;Z_GnynDAwi0FRY(5&-l)f zu`|_X;`m|5a-=@xDfV^!c%$uuul8>n-P;M_ZF`o=Cs3d1iwR4`v|pHn#qcMt?Rk;J zq>WF9CLa=?uw}xoW7K(!&v+6dvZg)>u}+&L1o5qR>j>(|pIgpOFF%sL{krnlx%{Yh z+xd%a&n>5UTtj8Sod>}^y=dO*CVVFESst?K)-vE(-dzW2Zz^NQLF3E6vE!Prb>Po? ztV0`>TWbEwE2Oz2eA9$?esw@_FE8}{f8AgANPCN?m@iQO-rsxf z-r}3#?H8!u;w`?tc>4wFxA^125B|&l^2-nWOaIcj3oHu!%C!fmpDpBJyV+VmJjGrF z?nM?@fZTu42VJn>EkPGEO)oaj9o+=qWpwdRj=li&f)w5@jipWPR(+tvl0 z*j-rOhRQ0v33zuI3Gth8+tZu3J0g(2~)b% z<0~W5T29igC&>dzHV_;dRF(%;58Z`yB8Cz_5AqTY92Viwt&1YYC+HBHy_2Wh{o>Vw z@zBz_57*Eay&j4|nhkz0%bLf7t_=z_yrFK40Vs(R*Ij ztp3|}-Rmc-HT+}$Ip7fgy?9~omdH?0*4}anD5@ZgDgu2O~zTFhB zT?R)dWzeUGh6jglnPpy5c0+yqFL9R-hTltTn}k;s8K;aR?$Zb}J=M(y-ZwHobe)Z! z14%ZlxPw0(`X2i&-=LeXjtUF;=Vg#~QP0rce3{Fo96kC%NaAcdJ(31qob1i5yFX8V zlCP1;Z!^tiayM$Wykx(qun}j&UpQ$AFOXXytBlbz%vejs$O!MIS9ufc%^%^|IW#Yd zj{OSxC#o3&u7h9L@&4-5k*!~#-3K=A@%70$%;Y-McEjo`3(^`#J#k)mw4dy+4|qxF z(2_$Qg$;82X!0(3+Sox3hdOM4Uz=;SsJ>y&c|YSlYJAh+!;W&iIvUK3_10D%`5^7=J_eIn?+&FRr79^oa`*sh(ccKa3DMn6&UpFC3DYF*W4C zK&o_&-&01)w62iXeEYW5&_ZF$-L#G_YsIdbr^EjzTn+R&GD zjp6DwaUln&ef8gZ)7P8!*U87H&UGQro};u~+YdkA+G|&rJ+Rw@BHwLYaL|KmLt&mD zYMnHNkB=nn6ft${SJKFyad9b0r@lNb=4JITKlDbgbUx{?`c1>gns&LAdbGj#<%-+w zHaaG)GK|kXFriP4%}yH3_#E2LYVG&;z6zrHLxTDSwEcbU>g(vlLrgd?(1*ga`|tFb z(*_l%jPbuy&-!9vzMlg>t1b08?WMl+qr~uB`Y9vznRPGW2lab#GuEs$`F)Ymm#(ej zKW{Cs@d9O8i+q^1I=MhU2b}t~+BwLi;s@G{UUV8Flqw({^q0TRF!u${>p=w(v#YFS04}7?X|4y+}pE<-C zxqPqvw(wOpUuW~&I`hP8H~P_Sk0k=_W}Ke&AzJb#{ZOH8=Lb1tk-6jz>>irF{NXOkd1BI~7DB z`tccaQhj4|<^z#eyYOdzKYv{*KJAAA|3CKr24>qWyX(X5@pxv=dp^h5RxCsUaxGg< zlmH4USgD*yE-kKtflQ)+Do|8a4gE;Nw56yKi5yg(@85pcv(NpUGo;E(|ao;(2==?OYUyq(os5|;h8;m;lnIz@SXA*5dD6pZ_HVLU7(D!D0 zd2hQ+TN6*7vkcbYd|CFD*c7U50(_(UKCv5PvPGhtD+^_T@)( z_#cP-*>7PC7uc9S^j`Y%0_|lxk#Fux5p=u?QGWRrFJNqb%$g^81oI~2FZ`V&Q-EqC zk3K!0o$4D2C==1WCVs(rc&-J=u0G@;qG14=U!3inp9sfpKz};-$8Af(b8IijFRh>l zsFMdjGru@}1;8V(ekkB|JtE)}$Ugh@EaMzk#gt+B)^nzTxfj55CM1vg!86M-9-K0X zhINktLHBJF^5sj`2wlG{yD`r2WoQmP>*x42YbMrCtgRn2@5ko^>m&LNdCoBh|6uKO zYmLOX&KlyiD1%&qk9D7A*qJ`X_>X@&)+vwpC(&pAa1Wiy(;r0<7qa+`6+zvneYozN zHtK;0Tord=Uhj`aNmw8rlMo7qm8aqD2SY}?7LnT@Kth&58UhrO^3P{08??^@#0`1Z zhn005V51kJ1bd&fC3aXz#p}`OGa*8ue-WNON>RwEUkpJW0Oz@zH5Q@pM?18Oe;LC* z%07>MP`e<9bjiLV7&G|GKl86PqD$K_wAUi)9J%%n)gp8suj3byiyt`W@#Ekg9!;Ei zx#*fJWEOm+YmgAoPX%@F9>cl^`*|+Zx{mU>>^!}$g>6wgho+s8jXyHh3fm2scg6|h zW*T#YB6;`UTr?x8ky6^^BVv6hX$f!YpCPuLTwCj z9i30#u-|e>?@88IZ?G^}&~FG&+~=>>-E?$r8EHt$Gfr@Q`Vx7SRXK#$q^oZ%TQ+j> z3Zn6wsj~85>--j)HmYAFR1RwDf7_o(Sm3&}_=`kTMnc-)&Nvk6IuYCCrq9|GGBEW4 z_2uO0{E&Onr@V$o9`_MDM!`SQNjjgcCh&{X|Af9g{rMKmi(Zeic6dI|fxFE^dybk| zIwJbqgJO(3-z8K|cnG8NeC)p3LketnvGa~N{7GM_yxf~iUVlt5>qedE6U6E7f@{}? z_syY%Q=T$!$s_*kAHe+JR@y8BOu3dRXjke;cD?>L2{Ub$0{PKi&?%1o$4lK>dIRyt{ZfK>dJ!Abi)K|MQoRgzi8Wnv0@| z)5To0xp+Az0~bR6dW!{!@L`TnO!>DaAQwWW!9Vpp7fL+b_0Q3UL?3`cWNF?dqiG4x zMdg&o^{vGi{ODkvX1a0&_YnN1O&;XCxD=KPnSa82vYeYb)RTvyWuiiz3DJ2jKGBP` z`LF^kU(7m!^WMC-^ab)0N0h8xK)35rl zyuuCS5s7T|8)a}v1s|PRx)~K7;v~VQ0}qyj#Yp6`560wOWdLQcdC3c|d1wGMeek=< zhMblc{oDZPW&(tcH$Z$h=tjCPG5gXBioa13bL+V*`bkd%OS2)`B` z=3%$?BibW*`ofy>mg4HuwuJ(8@}<9rF{ZC^sNR>MzJRh$_(sQ};T!fr2*8sfHZY|m zX!0I@301TqNPhiw>y<=&g3a9CCVuKnyY@wWLfb)D|7insqz=5uRR3<)E>zwM1$9Da zbaz~vHlQ6ChbBxH@QH5dh3fYa+CMS6_zC;xVnd`C$xCu=mJ4`xTF{N|kTwGNa}Qo6 z?4hZUWnfQU0P-@B&2Z}`5Yt16*V`f{>xMJoSvTMuX+7b=Up|kmQ#LdYD=^RT$m=+>B3G>Q%PV5cVR?d8NTh6U3Dg=lXZUscP0vwwgyj=YSoj7Jr>L+c40htixbpBpR#996n#3gjAj$H$VXuIa0`ACai z=N!+{_9`GxkhcEPv>^* zvJ&UG(RC7$Ltgs;?J$*Iz%I;#bNvjsp7&g~Eh4=3$@R2leS$F}j9>ZL%jK&=)8DIe zXafJBeA4tM(f&qyNa%m)&^00Qr;g}@Zr9shcU*{Eo@+Rq#Clycj`3%W1%!@6J%kl| z;{WaU0rf=&#Qf^qFQy*|szi~<;0I?Mlw0E3PR##2W15FF=6>Ys&(oh$^yRtma!`W$ zPaA`6qje7`n|A3ql(w$j7O>gbmgMhRF}m^RzWI^nsOB@h);(Fn{;dAFnR7+?R-VrB z@x@uEw%I5j8P2EjO@6Mi`?UQz|IRkxvTLz~^{Yh!G0=!=`}5-k`1)fTY;eew0Ji<$ zcOSNPa81jv$i7?eePaLxh^r! zwCNq&tZTw4A86+@9#9Tnw(H)=l0jeXmtSe$#L@3MRxjX#r|rAa-2)W0O ztBBo3_1h43n!W=sq&nt@}E%uly}+y;oRO@D^qt#=w~Ht9#Wn*$=tcv@LrV@KbKv?wV9^c=|Kx3y&G#7I+5c zFQssZn*ImPwmyBSeh;P0Wb5HiLXoO;{EXQ)uphP~B;WK`@}@gqUJGIKw%aJ5_V3zw z3cu-!sYU*$idG+O^|Z1 zMaj4mk-QpTm}f*e?m_q_c7rR*hA?d4z`LnBIot!NnhdzGdfnzfXIr-(Dcdn~_MO&4{w0s+%U9AC5OtdUA8<(cv#~p&P&Pmww0+Pr z&bY{?Tn;r;&$$OhozNNk_Rqw}N30WNaR3g&hXHcWJ|aJWkAs-ZMWT7agAenOjQPwC zsLrf0NoLmR73qP`HB=+I~U_!ObwOXutx5ekwS4)G8Fde{)TfxXW81H0f;(`H?3 zF4E=`_O)qyeXoD@EH-65FMQ@C6y$})=ek*r@*vBC#H~2gJcolBq%%de{U$b@?IYy7P5xRJ5)!E2pn zJ<$guZjOzh^@yQ~c<*@G3$k_*Esk- zZGJ0{pl}Ju%}?I?=cgMML{xmH!sh z4PNA)Wom<&2h2Q#lWxKMgtILrVE!o+oawh950FQGXx2><=%Y_geS~=@;j=stbR#ME zY@Q{!`ckfW;5)?BrzCmVz#-(J(Sw6*LiiBg2S;hZ2YfaM$iv3LrqqK?oF3>&$i@P) zD?fyMXm1iM>I?2cfDJKpVY8Z-kp1dc3N(GIwRBv;E~DuyvYTU4@^`}`@#P8H4k7u$ zY!AY{@v>;gulsJMCvGM63y-{PtwS{vRr+1h2(*!@q?Z*Uu6sA&~C~ralHspGmiUG zuJIW*J~;qUIU!_svpJ+6aa+5ue3iL)fSa+TLv)7vVdQtS3*TJV(AI?z{(x)K^?~-6 z5SuG(T&G#C?E)YClai)fDF3P#A^O6z%Y}E57q%Dtx1fDs-}Qx7UgAru46dW*wfyj* zf86qp91`K#lt^Ter#=yK4C=K+`l4O`2tfY652*(sn|*DAT&n|l2wUc=OZ_#1+K+PG zmx5C6d+;N?Zg#O@3_tzd_5;d9UhT>~2?_JBBzjHlgq1}lP_v%2a2;bq`#HmKqIPpLH+cAN5hnj!;6O+>>US(3Z-WX%q5I z+=>Il?RLGvGyOoQ*IB-C%9--&56jPPmErnU8OFJWnCoBp?t}R!&nYkPdN~W z3f>6ykT&{*yN@;Th|@*CHeavY%I@Bw8=%vqCKZp!rFBI8zHX03Ws zrelA7o3cYGuJNv$v>W^#w?f*NbaaE-$CHP=0NYv?vMVR)mSw%G$GQZL1D3@#iDXH& z`og7Qs0VrAwf)vT^@J#=ZB3;D^&kw}wLQ`fTpP!l33)Zlfv~)l$P}7>sC{mM_>QL` z*JHGo=nLEsUxU=vsG>To;~=U;dtvX+GbDx=Mr{bq{1R~xH?>SVq+Rs|oBv$VOM0E}dRW70Nxv0*$ ziI^>;>yPlVma^c=4v`oA$wQr$)%FS@vwVxcc?t5LYX*o&cwO&t(1(|nw3q9_J7gR_ zk!$kkBM?9@%H`k?FXu6#^|dUCauqlP1uEDB>gXna{zSMCX=WK(MM_;9C<> z_d0Kcw3Xlz{n`g9Z&9jn^{+6reG~5D_zyV=lvjHvg7J#EhA-dmeib@(;+DEKdI_F( zkiWhjSs z@+HJSA?pTli=QY1yZXZxHMwdWlyeJ{AHaLAQTIAD zW$?vx##rhPsI$*~+K4e#U@yjUZ3Wd&Nz8w>TS9!SV_@WPJ#Fb2K#`V3KF35FjDR0l zux5m?FZrmSvVra98qEG?Sy0zIA>ti_U1P$EJV@Q4>N)PYCcR@`sQwnRjJDmnHjjD{ zf+pv>dghunH;2Da!tvTRmnLjOLXsfMvfQ082sz}D1b!jSbnb!qCZBp4&l<%v;|RsP zK7D%1=6dFbTm!idg6tm)_h9vIe!B5c>CriXr4uR{QMAV;#A{ zSKWi^JP@AzYg#`(5hL(}VcWVyI(*?rX7@QGpS_qb55I?@@zo@?{kq4WGF*qmT}tGm zoH7n@o22WSCB${>5n>#3AJ6%;8A9k+`#y{6nEngVt7C8?BJNIo;JIT@%S(t&@r};0 zNtDP3;DgZ7^%Xycg!D~{#U%A7<3jE8#6>{g`z^VX0DOCHUQkDDFCpJ{XTHEs8ix3h zA$t{keYnpoA#|V!sP#ZjF8aK+sEGqzXN4TI91kHXqa4uo&$Z-uI$@f&m}8;-3Z~tg zhq$EE_T*Cz3EVsGb{t*t_O030t``UygB<6|x8i8q@~vyqYCEqJ?|u&Ljsm1m7pM zHMO#5xwAZQD-O)K-L8XorcV%Od0_h4tj~RUCsD5%BC=^^zLNb-AA=YV-CLAz%So`? zVkLw6Bsy}lc$HVIP=D_D6#-lq2`7K6nd^@2#D!LHeTH77ZTEc`^i%Yc>c0uC2V{9$ zr$mRX#5)cD(!lg5Q9BMgVLP3e@#;=~4iEQ-Nhm2254+_3j?aJQhuS9TRE&xFx2g4B z{I$^{>O=kB%vwh|cghdycS$o(d6r;gVK?H%w3B*Clp)SIt~p%Go;y02KCdBweERy# zD_X~-s|Pg81(P3;evoK9+i8Ic7k92gS z4rhKtNvPk1atUW1>9i4;_mKL1F$B&y+TBCf(+3uP_*=jAn(qNO@zJ0B$GCU@@}-II`U`*I@{!OTxFEolyq<7zB3z_QlE*~{GUfxQ zVA112l3Tb{z6+&U26B;48VgQjif&%4Jc8nQh$gBJIE}r61%3bI;aVsme|9V6P|nmD zKI?e`Jx_ix>juxXtJgEma;?YI32=`*vQ*wRx`4G0DkG2lgvzx%;cC$X# z2~>{|d0c!dkN8=qSvLTm`U$S3v#eXN4D~pnP7qKwfZvBM^AcKbzM|31%4<@A7+#1h zaRnxV;OiIBQsxrZu<5Hzz(Y41-B8n$Hq+S{U~?M2ZbTtZokMImHSQW!klcsS6=^7r z+r2T5oY(pKMu>6&x^RGruc%PIaAU3^1wsqHAmfFDhcVRw5qvp~gIoBhplu}ETj(!m zn{v+s_;X3;tMh#r57_kls#hZAWq~|j&d_#6Lfg?$I|4T+)peE+Htz~eCZxUl)rf?G zcH*!bq2MbJd})gp*L>jvQfze&xd98%g|F0b=*7ct^Q%(Un|kuX2(rF}{kjA;xJI=s zH$hr7v$J(F+tc|u7rPWV?Y zoI4!Pf%-uwYSWcIW%kfa;?bcm#fE~cD?D&4u;J{7gtIO1cl{e+7w-3FE*R-O*g|-{ zExO4Zg6#`QnDZoYwM*o+PU~JEkAs+mLOa36yv7fM$S0b8rS~!y_(S_uY~ots#4LBV z!;BZ|AWd+8&h?XrHgldA@Sk}|zg0FcZKlsS>gQYp2xr{P2k=)(=wCCO`A(g!Hw3Tr zc0eAo`vNw)@v>?9Uwtz7j0HXPGs@@7LvN%l-#pq9JthfWRDEUgeXFKCx*t{>8hYtITcL}>D4DI@+?AIUNUDpxk7w<6d+19s!4jqR=K3}|D z9;!x_A=U!tS1egqd=@>zz?bSC<$9@C+wd1L_|wt)ENQeOFSl3J9XS3%qIOP3Lf2PQ zWo+rZ2)OK;hA?tmlM32^@!uEYjJpDRw-1K2J?(4T=kn?sqr9iF$7^XbLcudFU(X@D zg^0Q{UUO}^anG*V@NI!xcAQV>c%Tu`kF|ri6NE zD0Q0Yb3OssKpDt}Xd}V8ki$vX?d8`)kFRwEurW@|H5WGa2NVF_TYfO>(Y0KRLp*;{ zzc25XJlJj=o%bV^|KG>Wka0+Ll5q$jGrL4ZM)o>p z_9lm8&ts2c9mhG`zW1-!KXAR?&+&Mo|NHHL`*P{X;%x!4Hs4|V-#OB2vCaj|;oMVs zm&bhnx-*0*GN5Hntt$50B3Zl_io&tydpiCdq@h zPt0~t1ERo9x3OukK9U-b-3Tc1zC~}jn`oM}tGlpJ=tcZ9e9qI)e)U(Y7M%9|D`#j;MGC9FxaAn-ViqOYbN<0&*-VLlsi*Z&EAYdmdXx{rSq(UNWB=0a^1mTrgh%PFD4vA8PdgaS5<)WjBF zfC4il=0a9~(CxGPi8m<-8UBL9551Q*>h+W%%a1XKzdze+6(4I2KfH*O`65x-w^;^p zWZjl3XbF?d@|xkimg|*1c9_Vd3Y0DHiU=?Vb`+ByI|5uHD(In0IU~Df=Zu?C+~8(L zq9r04CzzbBw_8s1=$*c=?g(FX8*J@IfyhNG?6LOgz{VJQGv=mbd5 zR_Zq4>$erk^yu!I)L(K07taCH%d4jBt8ux$LCb$XdvnuW2(f1b)L z`&(NRDH!>lIo0I9v8;Djl~vp0dGo^XyZ0hwK%>wzT)^jFx6=$or$6?pzCPJ{(6Z0h zcxHmSQGw`x_T+j{k*|9$K3oU#x4O|BRbJfFCG2aVDZda^uC3>C9OG$hPT5H>Cb?2K zEgr7}noi1le~W%!9{{~-#c}QEj#ct;d2MkmC7kkDvMaIjM|}NiH#;C{X$HhWkbPgi zNj|k!vgm&O;iuULXm=pNI{Yf&S@?>Gj~n;t41vi1=4%F5 zV1s{JsrOl|*{Ke2R^uOImF6yktgM*@$cU^{0FI({uAWZ0(eD?Fl z?cHfrwu>dEI9(}>mJR2%U)}N*;3$cccE$LJ@)(DCPu>3p2z#uQD@N+)6)!{RDwwTi zbviNX5o`Niy`ETMaK(LBzz)J3ja`GN;AtGqG4i9nXCJglXx;H<3mJkxu36nERF=Qo z_9vOu`KWKI`$hUfi;tmYMs5ZCK&L6uYa>%qsbw{JTZU z$QRuNI38pTUS+psQb_8@yc?BwPX2x@+(u;x*ps^$!F$99P#d63ps!hC%%qPI^@7{9pgBCkDjCCBymakFP9P}(R&mL}`(i(J})ne%&Bqg|FhwGEOQkH9$e+P`Og;z+> zx)U9`B@|JmLtKAUa5|v0NV9Pw_7-bT*oI&`^LFP4>WYRcsicdC9rcTSPts}ne;9jl{a>Ie;x`m9wY(meOxBc*;q zq*;EBwW}*Jw?e*P`;#6G*LcD#y%N9trEC{%D_l!}L20%&Fu7-_3mqwzMnLzuzY&#C zJZdpHicUsP3F<9M6Jt(%=bLF}I1f;c4>Y1_6N5g_IPIzXQC}hx3+1&XIo@}wPQU5T zXM=63JMZfH2ot`!lHI|^m<-iz&wSoze5~p+UCS_C!a7rMmN+Wr z{L{J4Nn|YY=gzHI+6g$`C(!2TEol3t#iQh@@{u+7I<2yT3Uev+ltjuJ+CmmXh2@?8 z@!O~ffqj+5D68&CpgFZst4o$-rO}orPg49KQzNOzUny{unVoQ_e5>%Z z?G_!abL2g}>(FguE$g^x*qZF=w)SA5F%{O_b%*MZ$wf#}1#7JCGLT;1z%+wTuwPD- zcb=6ICg4t+ll;t)on$?W)#rOmgVlYsO9ilR!?_ zDW*RQB&>f#I45(qYVyweD0nDYb%XFaEC-bGp|0x>;+| z)qwI(WDu>}gAAY>$Dmi)W<8~O%q+}i|E8+AY5=t>66*dyxaY|hoBK~HOx5W6?u0+X zJ5jw3h62~)$k-j+Lj~&KK(m)W-5T5?U4wUquWvT&wW+NT-5qfXGz}W|J$bNMnXYm8nb(gvdl2)95bMr_aar9Ya;zokz#DCGe|3Y8Y!*PE}c&C_YK>_T~hry@ew6+V+0qNg4VVGyh`F$ zE0X~i0!NxX2(2&D)#4lS#I69$R6SZ2>R*G2h2iDRMW;{oqwEWN*vJA>2n`2ZL4~GL zswY_#LLiQ>_0=v_^KI;{%z|;7;xy0TeO|h=JJX#<<_2n$E_uAU0WqWBBR$d-*c`JV zOZx7|CH{#x9#>1CQ!Lw67z801uswFFJ^o8|KXDd629K@)J#$&F6e+vjljbY1QXSp-THfv2e{Iko;Irj(pa-`?X7y zDE;gTVweig1?#-Pr{9eN{Pkq7OWZmJI7R*4K>NhAbM;_~z_W4;sjf#aHmCjaHGD1! z%I(XYWz%JmRc|u!Ns_%2J~b98n-66jtunqM}MD4 z+DJLe^k42~naFnmNEgDqwNxHczj+!ciY@0Ze zFmE^xg4gvq$`e(ghldYtYDu}u(|qm5N*Ei0V3#=k2n#M2KD5B$@h~v^O4*|4m_u?H zM=!H4w?RS(s(Ep#Zu4;w6cm1_?~_Bz*=qM$Xgq&dSZno*{8`D%vWA{5IjQiey?+Pp zBd|`j)79rHx~$JF`c>1AdNO;6u@>Ju7-^K~)8;jgY|!^~)(JHC=kz}$I#cdZBFY5agz2bt)|TJt-San)+Q_yF`7a9o7_ zI$19jZyoux8SsKamKhon=|zu-XuA4%cSV(xFv`S%NgBp1|CN$i8$rbrtrEhJai%GN zg6mvkcY5bV>I?Gxmd8P)3kFxt*N+~et%XClA=fWLb}3u!sdV3Y@~r>VC30=YhuM8D2;yYik4Ub7<_V4$7Ym!JYeIt`k2c(0Ky$8H zp*Pgxd99gQjJm-Q(KQQio=E#9Vw(GibB@p01Nn5S^d?Y2NC{HJES)cx3U!NS2}tDE zwWjg?f8Za$Rql1eI9c+LZH;=dmvN92UgcIHed@M%h~zhHHr!4*D!71G{7?~ZBIhT8 z-+xR;@=tZ5@{Z{W^pk!Q1)jk^vVCVmH~U}3_!PIdzh|&{Sw1D)#xZ+e@b0#isq(fx zn&M9*65wkP*iU7T6re?FtvUO<>Ely{S<2Ro|6Z1+`m1dBDBi!2I~X&(FXXsA`sd}X zh<_uGOOGcV-*=CkY_mhMPV?5adyDl4RsUeg$DP5)y`fG@b72AY@}ouD-`QrJx8)r= zq;dS@^F^-Na50(;IBgnigS42#{)_C-YvBa@q9HLmX@~w?R9(NQZuTD@C-o+4+Gt#6 zq_oeWgYRj$8tCfxjd+M5m2$(-PNKbQXH@6nE{j}o5jW={wF! z#d#oReer6leE&545poBDj~4-5EQ2h(X+@rIkXfyGiu&;B4*FxcXd61hN6gl!ae-}B zI&1SQeC+Srw>pg4b~-~j8^YjS0QlO8K7S8H2!URDqwtUBB~UZlt%_qbAnL9M6V;NN zvNJ))7>Uop4*PGff$@{pAvhx{o4I1~0KQh)A)n67^GW&)V>n5kXA$WE{rW$<(G$qW z{tV6o3U>mQhV`A(FweA7b>s}iNNnJeo~EI_+q~KGd)~vK#m>3{`~p_FJ3!G7Mmk z6bsF%6MhFMJ(R65QI&OeE6Q2~c8cdT7m=6wlIqS*Y!GGj(B?R7vO#mMd^vBxmw*7@ zBVzSXl9mbb=^wMCx$M7oF7CH|&Vw_a=HU%d_<>JZoWxRQJk~wNplvncP1pQZRa@;B z>)_RJ*3ulo`|s))tzaBvR&vftJg?s@x;*TxXPkoPgcxi#P4rM2CMltyI;<)LlRn^` zo^(-)M$b~McT&xf@LsOmT2Vdo%Ik^g<5NbeAC&OUbPH;=0eYrKuC?M=CvAgyV-pe? zBKjf$(=D({7PtUMfHvw$hh?_kSgbX1xx>E)<%79ZTyCp$=qg*p(KzWKygKEty%Z9U z>>hrp3Abj4eH}cvx2Oc^Rvh)Gvvao3O_b+;HC_>P>{H!B3-IWW9X5>#*ae<4kJS^r z(PeP975$Tzqi*pmaw(Dmn#)(S&(=MJQ}mm+_#R)MP?htgK#rT)hw-E~+p5FDM&N;< z12gMi6S2&_sv^&V_=+ePvNE+$e-r8O)aB7 z+RT=_p;6fMq?cYC{DuXLb7zEpy4yd{`WZZseQcZQzmR^oJ_&q}Bsr9^5Y_jzu&z)P z-Nya5v6mOPjyL8x0~#5>TMq|bOnrlACzKadzFDG;Z05fgYm#KZ=Xq3tnJjP&?%io_ zEmNC#)BM35%S~p1wvTc|-0-FSahIWy3BLBXrtSq+J|H$)&^B0btD0!AY<5bI1k;~1 z{`2mI^F29Rvb^Pci6T>pbF6j)(1-m3xp86CcOxJz-qpO&4EVjgX!+B{W5bohpB%$) z$yj+>tlzihyRm72#iEx;Xem<`IhRWJZ$Oi9Gl-?j=hA?pB~9`Y-v5(!Wi$N>fQa zry|zt?NC2{5#Nn>+V#jx2ORK*qutc(wUAfG8_o;nY%?O!Q62j4OqkN1!{J zWWi%E%6!u?8d)odHRgEkUr|I9In6w!bjBD)Sabe4%!_Lf+`E5RfCo9V^PU(5GvKaH z_ySl}B#9*B*B?6h8V-Q)^xdE0PE}Vy&5YQNtF()yYHsO0*>(lnL3gu*L5cV!tl`>G z4Wvt~cA5F>Dny4n9?=kdkz95mFIKL_j>3s4dVGVGw^TT1R79_i%_<`{sKyDr!f?xG z)OjKbL}Bou%&{tr%rq$v8+OKNNt8r={O#={w{Q0Cwle|^vNY1dm3DHq0P}3v(@%jr z4}#$T1fI{cq_aRN{UHHr-Z%dO+rHxX!OG-nyw2`_W?#>yNqWU!V?BS5dQE;1jpnsy|+ff64?8x@Q08IP_XOdG5*+Kb@Fwgov#voQCdC zT9YgsXLi{aXWDQ3l*D>8=W#Bms)vB@RKdBz0_`##X#@VV79BUn{StL_RfZl>J;k+| zwJr?=lm@rHH@v3bPL3`HjcW|MMO8+-X8@j~db#kFk>y;cj(HZ?M`fH^|1Ih+nCS0o zKV{2zok5I944&15iU0Jie~@T4UcHw!=3I7A!hah2CKvx^`zZI&Z3Zr}<_5j|YP`Sr zb(S=s;8NBw1k|Fr(t=a#&Oa}bAM*=KQ5djv^7%p-<>0HZS_|MxK@eXGk4TcSV%a7; zzczr>{@s7oy4of2>okPwzd+T?8Id-rMNQ@1otU`vXzsby1UzY3BwhdJF?XXn>6G9E z>hRl@CFf+NxciWv1=_b3w%YNr2daKinCJZWOOimu#v!8{OCF_n&~Mq1>Lo{CqsNk^ zWer8nvg&5ZWh2rJ_Sbj($vPBgzmfb=tpamKp5HG;gnJO@!1p&%_9X_N8oI+1+EL$+ zR0(cz`UKt~jY?52-~607J(_%`*R7iimMW-?kmk7f5Y<(}U%sMufIU6-GWKV8TV@Zw z#^fth#ye8AWh|RP5CN2!ti2s0+j+G8Rpz8445j8X(8k--D`(kcT+AlkYid&# zf8<0XrvKTCxz>BCL17OyY+?fKi2RD=gPKq zHgb}YKxAi*#~)6lA0Q`3%Jm{xY@@kf;v$G(lnbE?{@Xa$-Y@vT^rvZ_v4DL_xcoE* zPg7lfqmU0Nltk+e2kHbr$fbH0>G%qEAV;<_+=bxm*nlQHXCT_1Wp3Zww$R>Q4E&jT zi`DtlVn(=_Y_X-_K9B|-UK{E{>YwW@kl;K^GfVyD$FNqFxxK8c=P>K186=kZ_BMkq z%N{@w0JAO4#LkJ^Beok#)fV(uh1=|!Zl6+aKT0SWvSV2bjtaHom`D(C1E+|q;+ZM( z0@e5tX$3mH&(@=l;TwrA%Qs@iPMUnwpS|PvX%fpV*rGvGTb;S_KEK8Oot6UpAU;P- zKLO=uvKU=OHy+rX>YXNfnSVf6497Gnjju_83#b?QuDbHxN&)J_<)HaDr~`nT+9yV+quHZ^Th3F`t%+otlvZeF#E1g5#N*`a9#bmG|KU6y zY8J`A@k0q=A}@<{j;W80B2eNvKHJX*IN8$_?+OtIJ415+rqDW2D^N7*1d?D!=-(;_ zCb~HW+i~f>oZR

w-4k!3PIbgH9Wp&7P&rL#GduB`4w-RuDYbl z*)2CE`8uOG{aLnJ|4^?N{7AD_np0bPeEG9K7X~GCCtG!W9 z>B@)+=k%d*jI3XEo`!u*w|Z#k%qnRn$H2sp#TEZq<76^#c53NA2i&p>Aia@@1QLTsyT8@Wg>28WO8zqOM( z;&yZ~!>UB5SH^$XnY4AOC4)&&pxbm`&r*J*j_o|aeO0=*{fk4j2HgeZXFl^Kgm<#x z>4Wa6=)L04)QMuBHdt^=Tz^~!=Afo@KO`FN)hPvrN!HQf>4@t?quLEuzq(r7b)Ot2 zBPo=@HuS!@(yJA9kG;DcNFwt<35c3OLy=)f^&i`xm5~Lph3dRWugh>J@NQ>bZyqwH ze9S8r6e9n_2o{MvCuNl`AIr-&<kp}bK%GzmiT8KiVDC@ zCwu0jRPn67&`+&MwwEXq2J4$VF-7D$Zeo`0*-icn$$Fe3*MC3dhIusN`-j@&B3>s4 z^T@+&PW6UPF@SdS=)N zWXW=9Mg-*3&w8h$S*O@?Yy72{>gvZ<49&BW1^vZ@zre6=-AGjcwiu7W`J#tCeQK|~f znJ<2cR~*RjG2dPv)2ECqWXtm3^gp70dr++%h2<>kvu7pQ(afg$k&_&4U8>;9#y!{d#*x9OkZw`F_GG%25arwL9#fcWci?}vJgeKBY^#mINvwGJH`4pkUcGjcLwaGA>GQvl7YLZ^<}PX4_2GV((Oj^8S| zAWxP)3e|dW0MWKcPaAez`w1p}qV8y?EK(;v7TFfZ_H8s$f)nus+F=}KO%k{z|G>QU zog%zgL`y?a@dZam9XUMS;_$+njL1IkTRGxmuVA;C(W(63SfwZ|KX44leiN2=?_=yX zJcTnQTvDQ&6=j_kF@>jdRnYs!z4z?%7;zovJ{t+)UKKZ)KuCrSWqMpF4|Gk^a26l4tn)E_De6o^zkMn>aUad*Cs(f? z+Qz;}?M8xD9Tonsu&$NQ>P!^ra>dg%Km;b81@w06;&>e6fGgk$$f@hDZ=7mfTXr9* z=9%G#o2ieRt5QR~u$P)<{1F@1{5834~1xwDT|_ikMoteltxFhFbIr8IUdxVL%9 z%(rb;ulL9cWDxl!k@?DjM}8*Ks3t|GQkg9%lpmmk z5Q$>zc?y@v=pKWYKPH5UjV`zMa4AF58xN>;bJ~4fufOGPFilZ?@r%k7NeH*Nj z_Ytvg%ZIKT6ai)+b%y-K07=Klku_h+2EI-Eld+k_x0un$4ZmV*{q6R*BeI1c!~?7! z3E1XT4#@-ANtqHjtsDvYZlTO^de7+okLKR5aTU1)R&I_a*siXHXYt6T*>kz0f5w}7 zrbT+Y!3Glj*@4B9!H=r`CM=``x9g9UV-z>`B0YuyX#t$F3VAX~28Xfz?^4g4FQSd8 zoI0+~1K^Hk*j3fR%H5hrsuefa;vv(z?x;w@Ss_0mEHwYnx^;^x=i4f6)t;QjZ_#?P zLJ}5YhkJkBMf5jeE|hB(9qcO`I<~e7`F_FuHya_e<|H7l*af$6lF8Qq7w%FdgPbfg z9$CcPGW+lRb7~oM$P*m~!raNVq}woAU$0griYncOZ7f`T7@i=!-9Ekdss07x)o^@s zulH191CtE;(^~Oz;6n>|hJPGv@*?~YV&A|mT#9P#U=n*+N$8ETD~eUL|NV!=8a#44 zwzESO9e+oRPqpfIL&Tv>Kt_Y{TMg%1Cp5U~MGC)~-zWv*KNyKM+cT8eIxCE%S{22@ z1*gW@;6TdGjKAOoCV8gF?yCFnZbog^r?=*APyrU0c*n`pN7V<*=7b;#kfIvDU* z#qY=NvF+d66gv+K^ZQO)t(!{4>VI6tv3Ag`X8jvm5N)Km*UKU6YX%sS{~a36!8vrA zli~cFkf|=?akC?E`SPp*0T{;4_%~euXYLE7hi(%M`zE>mSl+a&_{lRTylKYh#mb!j zY*T?9!a87Jo`YGdG*Z2$XaUTmzG+mIS(&}*kZf*6z;P$lkSv+6%lAC7t`wPzhGK>) z4(y#_M=sE^Ng7r^r-t-~xBc6ndS*?bNA_G}DAH+B)#N@;Hq{Iyd6Ee};yWn6ZK>E1 ztWW5JG?a0Cq;-{XJqz;x-_>|?J$fH4xB{k5$@km!Z#;1FTwMm02?f2yHDtJR$pg(20I$O? zNdfOZ9+fpW;3hkpX2lt3W)0tfCLO^$rC387A$R`+7a(Y7^la>z&XZBb#| zMxK}0q7b5=`|8cZ!cqZ%PS#tbTbsKu`P*TW+&?r!s*PgBPFgGr$u3Q=?bh55(<6JY zd|@YV8-4mV3;?elzhakyMR|0lf2NGaxyXh~Q>1<^hg!aET&kiL55sdld}JyW^@eTC zN&X~XT}7%|DE)JxOZVgFY+dS4SexfZTRyd@{(dXcJ3%OLnT|+Vx4Zwr#Ei#gVokSW z^Xc^(GqzgF=F-y9DQ?dmPeW9P)~~X1VXH3NwV{B=lM80Wv3dF|lgjT^KKc&JIHW_&hojV6_`SzZs^)Z}c44u(UxnP(_kVWZ=Z;ry#+vv}rtAX@8(%fJc-^EWA#8j~ zmkRFg<_l`SRYlPH9jHGkox z&pSwv_Gma)?oJwQXQL+5-`xqcGHZ0Ex+%Jr@q&Gzk4jf&xX0-6)@kP2DLjp|o~K1# zFqczpAKbR-9qV0qq{h8PUTK6$NueDnR9K+cE`_ReI|`O%@x5XD18qLP)-V$X)jR-ToR7r^@nA~!VgBS4o`Uq$~ zsDfOMTlj3&oj{{1GeEE9wb-oL0-LX@aK1nPS_#8P*J;tn1U)tn#%7|*Xi{Isa4-pj ztvE=!TaM~bC8Oa>dS~(iodjY{SwUW@Mb&>;Z!XtA^f~M|g`2QuUL=6C`>#x{*I*b+ zu5xS+JL-#1xV0mO!l94czd6i-Nvn3ksT@zU{C=LmWs_fp{TDg9Gw-F{GwiS&`5$cu zBfsm9V^ZJF#ZXTzYXwxLurG<<#Y&jhNUObUM6qwO*PY{A4(x&|Wq`5YS0MA>-d8P@ z?DoKQ=+Cpas8_u2v*LE^V#Q0?E`ptPzbJlVu$r&^w2gvS;`$;76|TT9-H1W_YbyVQ zmxaiu7woUY@a|FtBjmj$ON`iz&;S`gjbe1Mx!p`wlBVX>RN~V*0?G?e16bz>@jj1N ztD*E=8VQH1U+;bSe5I|KXF){60p&;s2p@gc`<)Uol&lVdw-GNy{K0o*#+?T~oH<{9 zoxAt1S)1xXRLONu)LCdP%ia5yzNz}`?!Of`zFoH$>u4c$qe}*RN!Ci(qNQz=%p&oj zGpXe}(VL3I)suT@X83Uwd-_Qnst*OyOv3b!Gb)(L{gU!?A99%|THtF`uYSUk?ic*o-1)>BivLOw6!?niQiBKgr=J^o%5hXA|IGy*l)xyC&3%t z<~R}N*89u+J)k3bMCL_c*!lJolStFBQqzFBk`b-x=t|Q5)|Y^kh{MaWZ=#Cs#w3rYq9dmSsMBTLJk)iu?rlA{Ct^^3KCI5CI{B0Ir88>%S}cad4Mg5H{8 zdr~+eGfZML2HXT10QxpaKZ86D>AIkv%QHQoCh6Kb2vY>M-0zgE(Z8XbE06oJ#KPs@ z&J0g+NI`SVNB514S5>RZv%@F7E8r*lM^96poPPYEjn|NZ;(f!MOX}}s zO)4j;96H0l4``=DBcubI5!lG0ZE9Gv7Mtyo@j5Rc#c{Vfkh_19qX7NbE%thM_T}RE zN)M{miWvjjX)c@fu8))NvEA>}PqX88VcJP~+239}qVElw?A7&kj%RGNj@qYWk;4<_dZfSs%?Q&7KADq z<-B>@bZafkLu1zSZ#ouqlv7Sg7xQ6Okv6`oXi3Wz77H1|jo4J@gIvVy*nsJ$77PBC$ z)w;C4(HXq*0+JsOSJ!DnPM_vr1NG8?KbmZsr4NdCH%yK!A$-gteoZwz7nCR?NghZ6 zK68Ue<8#E@pPJ&a(V~BgbEHswBt^VjU__#F9aYENBN2p9A=2^f#~9T!Tp8pVb5n@^f5*9zaoNl=Kvdc{ z^MP3c$v9s;%6?Ax&L%~4pEY_7V{8L%@ua8Rkp%VQ88y1dd`;x>v=UL_g&FHKK$ytX zMi}+*Uyg)DwOO5S3?dfc0sXRW+Y+Z$PT1Qv`d=g|x1|QKejCiaMLG5?mfNs2D}q@C ztwjKqRz<~1A!t1rYH((vD4*ge4O4=`owZre8W(k*kB7n~uzERx4kfMR7KLtaqtfBO zK8-}<7wf59KEkYh6D^s`s6te#0k%KIl@{eB`y>#0tnaHzSYLh9Q#icK%6MCH*O#V^ zHps=Ps8mk&>v1DRVM75ry)-6_RkL{)5y%81pDik(5Ay2i?i%uH091#A=B9xmOfA|4oRmQ%8x>G-e+^4_T{27IT8xw$A<4fRypJk=EIz3`~x3tgi&$t z)7z|n`i88;osg&RmcDpddQ^aYJ5jn;jPOtIxn17hYlV^}>W`ky7v|aWK!n2PgXrRw zApP0vFR7&73i!7QS$t4{)y#vw$#72;AYDn2 z7|5w0%->RmzAMLVBe)k+)GUrIA0*kyrVCu@nMp;f%ITPekXcxGeawi|wFM~CtS|lo zL~qUx4W-7xr`&E3S6<^?0*jTIqXaA`%J&|}by84%0MQ(mO4m^(wC`M%S#-6e)ASmE zG44nPJ@D3UbtpIGbjFgeo5YFjCx@2diw#5yP7OH>a*xChO3N*Y9Vq)eS~vQi48$66 zt;*Rt&$<5ed5I4CFP(Bv0qlX&;rQnon$<*)xK#mn7HiIKAb^`%gh}G-yX&;fLx(lD z1diWU0^<@j>qIVhpwW{CgZ~cttYz)Vf`5%R%u|KF9(a^D- zI0X-L2sMi?zbN3)Rzam5b&1=azT~EbDC;X%&h7Y9u8X`p|4IH;&ehD7)`DXK(x~_1 zVp{W>=i}ZlP|}8U5a5rZmF^Sigf-&_m$8WiZ8I@wc`9WBW|@~E?~pte9_SkUCtwTw zcJWZCE5@(&SJ7}AQvQ-*_1<%qUd(sokD)ZWlRf2_0_#1R-~LX+(>-rds`ti7@6?!# zZ{TM-j{?@R)+PVF~96596Et~I%ubR$G?@SKbq&;B)1umFuZ`wHS!(_y8R!e*{G`1NK zhPmkdhM<{h8O$o}qlE8T$q}+EftO@cZMe2)VBxM{PX_pz z;M~A1Tpo1(FIWqcY3L~gCqBH5a>fZqD(Afqea#n8=2l>O1AXO#56g6Un@e~W!BbQ> zahCU|6SyxKUmu2#^k`*6X}^z%qnNk(m={KTF4$B@+?4M0de4V|4q$&(E(7?Vn4tuX z+}7Bz|COjc*U!3Fy63rx2O={we*iH$^4gwy=NLO2*#>@u4eI#(xZs0$D7dji32?Oc<-{oGTS#fc)1v< zR_VmX6~=NAc?%6x6({Zml}4cQrvv?_DcLVyFNMWwC=A*a9iD$9sr=5K#e=^s=|MkM zYtrU2ZGI{4^^2A;9pmOaYnyr$ekiJlOvGjQhr$>GGYsnQJFukGt&rM|U6uM4)4%if z(c)gL2;Qn_$~7}7se*W-0|Ucn&o4&kNY}jspAyvMxLAD?B=Z&Uu@68KdCI7)H5w4r zCvLBA$Q0?0L|gJ1?*OpQ4dAKw=w?Qyc`jaFvG_IIGZRT`DSY`dCpl}p7fM=BQVvU!5+@;F_h&eRjaGHyIvQT-A)f>&Xc-}_nsjE#(KKR1URg&WShH_TuD&WP_8inX zqmj+qfob!YGH%zMMqDs{KEzf45fYG6P7GTY^zPS)^Ln+hR+np^TjkvRIEBKuScnU- zt{w8s^AYGkDnIjD2=unBQ7^|i%hW5;POh^jeG~&d3W>x37qWC`LjJ6O2dmWK9xq8P z{>(cKrjXrC?p3_`{c#gK%rdZcEcO;k@c!%|pdxyw`*@HKdG7aQm$dH49CApW9#2>T zShzaH%*5Ro&SlVi28E^n>4k@fl-u52i%7!gbvsz+x^dtdhj4>i;bU&DKj7cv@4@j` zC;CuDyg208iR-V@Fv)O8NBg{w)`FJEfAxyKW+xev6Akx^X!={@5kc~zc=*|dzE%@m#bzk)^c_qs~2wz&lkdY+yG@Cnf4J+j|e?iwRC4nQN?2G=hHW9fy zeeb^Dqxv6MN%6j90K2AYf7$F-p&mn4@7TSeHlL^4+0e!1yYNI%&u4R*b0)brveZTS zdtENX4i*={O`bll-m^5Pj<&8!z;l;VMR%Ib8%wYDGd}`vRg3?5(WS-#rE#<7HWAEn zC-`Yfl^Z?oN-qXsu3#&V1|-Xd8|s{k_L}-0esx}BHceNy95DS;FT$()QbbBYDW^8L zPIgwZz{SfW=hg-Bw;@T#^|VjhiD7?}jUr!NOD9 zgGc?PxdM)tv@gQyZH)fxv5Zf}+KxJj6UNeWyHX)XjKs`P4y;-|pYy~&$Z|&w+ z1R44Vm^(^o`bB6`b-DHh;67mFCpc`L3rE7mdd@bt6hxji!0~u9IOMOR|3qssygEHF zpEd0#q;cDN`X1?F-RDH0%CobeQ7T2;rM*r%-EL?AUNCFDOM-poa3J*UmFeB7mSz3f zY)O($&RA|6von2lQZ z&0H&E14jk2k3d#C51Vcr&l5(&ZQ5Rxs2gc*L_-D-$nRxYhC>1&M0(pMkaIjyd^X1M za)t(1N$wdlA|{~SCO1Hi_0?~<$w_Sfrx4_zWF9Wtr!Vb*)t6zmpG&dk=<}rEa4z9U z76O6ftiDhPk$==B2$f5IHjAneO|d$|2CS(@GgxDVzm|)0h>J|1{1f=yvP+g75xu)m zfeh?;*$86)MUJ;htZGnYx2h&QJR#vC2ogUsAdRsf&%gQ}RoKF?_-I^!)qL@Jj~Ujz znHUrDgF5BJ_RFxg!kW4Qb3@PG-={r%KcKrW*a?-RPs(1cb+`fE8S>)f0P456D5zx- z;Oz%_(%8d0HDGV6cZAc>>C!##&*<=cK^Skqz&p;13f@}YunL7^G1&X7uW=ipA576j zU~F>hYQS@R`g1n%6JAeOgsR3$Szis!WP`1_>mvns*^M#-&MK^v=}IhMd0@?oFsBHO zD51Zp%6!4=iqU_i>lF)1cOHPXJoldO0`L%(@74a%J#s{s)M`y%fKhmZ^=^jh8p)uV zE(C3jz0V~5msOHifxv%@`CUfP>lbk_9nV=xnUpQId8}%0f2)1U^}BPmqL^PV!W_&r z9z%pW%MVi<(v}YOdtwYOk5{T(&;90CzcG5PtF=VR2SkMYhe^>wp0W1>cCGPpjt`Ua zi~fo*5}AJKM^ed+mdkMVHvS8p+EiLNqrvhN3f&OGf#Yv7_hw_FW{Co*Q-JT3L|8NW ziR{Or2c29eEyz!VtMm{}@cKflO*?H;JBvrl`;o0sHf|;diB(cs_W{3x`cE%}yDZ{f$>m>j?itON z8n7BbSF81j=Ok`L+%QSfQ7D}2>#J@%JbyMMG(Oe%>CYHZPVjNw*5_y&DR+phj{%I;c?Fk5>2gy zyuLxQ)GoUWD4Vwy3#s<&p56Ils&1oMw`~O?l&k!Y>w6LQ4`@nfelyDb40h8qlCxv> z`)zg+yZQy|kS<_qDG0}rtuOxDAG-(og}@*^=01G`AKuJ5=34Xs@MP9*h^Pb~JZ(T~ zYCr;fczHvA<6`3=5=SO+X&P%n`1kax0}zO`4a1Ff94Bfl?&9wHZjy1u+Bxv; z%eQN`60#ARkyK0XN=>8dxZ6^`r@F1ffie;wQbb_6$=jf;n0J@Tv2}+d@|zdxV%Y3A zzK487{L_!?F!SItgI{P|oM0c7+q`=x%6$oXk*IwUFT^f={v34AseAe>1?W~yb zNNXXbXp_UDRXzKp-|PlMxC!^Ao?#R(uV`oJng~Wt^9~m=8D_8Bnxiu^{S=z6p{%uyl5zjhKITPMD@>O=dzRQ)bn-|d- z@pcmb^5AJ=oKCctYS5}yPs$;To&@sDX1O?J(OVf+TEns`Hxf_e*Fz*n$0%;|J1G#qor!Z);>mz7PVJfYPR}R)kxJYp+*oSp=fyQQM0z9t-ZGz zMNzw=ViYkWLd?Ya<@+1w{Qk*(PX4{```p*{x!&*BOYz;3c2BwiPAzMX)Jq-0s<9*5 zC!b5!4N9uqOU~&+%Cw)PyfsY#0i56d|?xGq?DQ?=)xN;@yQ|g`*2Tq5-4n zqe-?!AO6514&%Q}yEsMO>cW)jk~=Q0*Dp!`No7t-xGGDyG-}AN%Mt#2%J$XTb$dph zMc}T`UXJ)y3DfSFi}1LDNV-c|XNr-kJL{xjudGEb`Ck+kGKlw&Rkl zQ~T2=TxJ-+{iPeXSu-0RdXA5EepRnoLse(H^J=SIpKOph_Jvznv(;W4Q5*7p55^w- zQ7NbmYc)m;22K_Ci+@@(9SpetW|F5Hm$ft9avmmZ9?eIR#bsnxW|EgtOwKJ?wbqewy*jtH=f`xh$Sp16G{|NT$ zan*TDxpYKZ@(ij|b&Pi(H<_$i%y>NRv0!i-Yfqy#R$Kd~?6*DIu+L=?UzTTpg z;{463s1&{4?+IMj^Pq?7-Se5n{-I<;(f7S2kH*@?cS>}Iw8a_!EU1Ty07Y`H$U=FE ztVKh1yc0j-Kv5<@rLM#^=bL16Pi4DaLP;X_(Mx6F$~VC>ATqmu-9HNMRpwU_FBsLx zi3{y9^DKA01>X45Wh>c4QxfK2grtV;X9$uk zP5M(m!Nd9Qb<9WP-gfv(tX`F&^}t@=J>-e`LHDb$wA^|CEshpe-EjdsSt~B^J$qhRK%N^~eN?CP zQ1*hqi>^VSmV$GNnde&2yuH27x7>=lV_EOXWQ}wb=KxoANebHOo~gArqF_<6vokrr zJd~QZh8ua1A)C1||l6fv=LxeVOqlWj0{D{|xVt(U)rpf1}(oyQkTYGPN|0oNGJQVp= z8>=`brfp@R8EUCfVmD)SshHq2U63!d)8UyZl`Eadu>C1U7{_REW-}=rQ=8-y|<-C~E{Kwg9J7C-O6WPSrZnL~efmT6i%DL6)6J7~cYQq!= zPcyo-a1p9HPh%ccpheHwYzx2dJNPKJ@v2k(SNHw?oBh|%l>2OYVhER{=aW1~bQUX< z>KAj*z_(;q#Cf$A_n2+j9q1e$gW1$(#70}1(Mb%TRRh_)IIk9eHVYTVF3Fs9ne*4f z-i~wQ$3?w|bl@INv@q1!wwGC#&B%bibsK%ISDCI9nfD;SRaWDc`$H#LyW#Ji?d|X^ zzwKyz8G1U*KVKu)NZo&p>NzH0jq>yjtwCjyng4r~Fd8kaYkpw>c=`yJWE_B%dBgVk z;!BJ_IplQ7_~L;|l;GjORB7PdSZyN5v--7$UyQzh?zzLg-(b|I2W(N|wa_DajS&ls zWRibVMT>t3!=9M}K|r70N&5y@qsgg~*7ZFG6vVPJUFI|5!OiQ)E?w?7tgCYe+;3^F zvrJ!!rRpw_b$irO@1Y+@4B=vWDP0$8+T@EbmrZ4mN3WvX4VJV~3j`jH1hzezxur{E zI1bP)t9*_RxXXiSI8WTtz&3dp|EWVJ$E)LKYxU+3Lyt|}8UUmUvx5(<2&3EH`ro8n zGqGyKCvAelSnCAH@W)166Hz;K|KmR34w6r}81c~%QTWsBRgXow0)KU!*mMnT&9VKR07t2>=dYlhoa;sr(8_hypaJS@;h`~(wjhf#wJm%0bINF7p+P_RA zy`#CLvcYXKD2KgkODwr2m;kRR%P{Q_QA>La#|K?u|m-u*g{OF&npm;v#1 zM}@ozRTH4ifw@2T$lq5tK?Z^=1On~@@D!KGyycr@oA}j0 z+jq`vifnl}YqFVApSSgfydPrvr9&?&iR$R>tIB;4v@)-M=5(vlBJhbF$ZnFKVXPJH ze|H3k`BRlo2G4P<+`g6qsK27?UwRx1m_F*PxAyxMYq;J;5ahJ6PWba^I>E(m_ZMh( zVAZH%voKG%^pf6-IL=TngI@vfuRL%b+Y+9<74E!*Qon`EWkL+r%@mC@7o;bjJS0yi z>j7~*6_#DaHCw~SI@ouwv%E`uAC$K#q)6GQ!1Lg!275A6Fg!});QiSQO0j}x{JQuNibx?`B0sTBbqOiu%CR?Be74?-n z7zu<{f#^%R+i>@YoQ)U4E78mqJGOc`wf}%3z*Cxy;!#-@d2XC1r_Tmk=%`ta) zVu9N~&-4(qaz|gs9@00OrMY912b>1_#AnrYf&O;y_Z~a?$Hv6oXgji&{uORqMhSJX z)PUwdE)+C`Xg*wgXQM%gEejuJYsO0j&Y=Yg**+kSs@P{QpsZnYn!>q%U>n3oWMU5N z4$Owx2SN(<8*n0Z%UrbqHjP?h_tTVzYfY!@xJrh-E>gXj`?FE!7RY=Xp`||KCUqPu z<<{_?kH;%WdnL=|oNoh6{tRjuUS>XSQ3Rm-r2>d*#}q2wrDk#-I|c^0i%ecKw$t6B zJ*U!qH$;ogMu%l*hMejFU&T_|FoOB(?SE(%IRFf0z`f7_9?ErzLkmg5 zAn_nO#NkFWVvA^Yb+F_|nk#>ryBtH2sRUdd3v^A%FIp7!_G^k$}1jkb#fiMNr=sV*u%o89!}dU*lkJPYvr(0XO&4p`vr zYcoH|uatbuI%K?NqTMN)y2f-Em8yAV&@0M2kVzC8lG94O>QpeUW7!S!5rXD9X3?fX z$+!_^9Ed~Nsb*_~`mHiwn_<1*DqV6{d4OmuUoS2T_0Co66G~nG)q<#K_x)Jop{=Sb zi$}xE`|k+!P1!UaPoM`BlDU**#Am2U7T?u+X@19p)M~!#M)b5XZPz_x{SLzFZR&93 z+~1=M=Zr@}j}<)6EFN8p3Np_V++NYd7PzgnJJWmy(TP^2DlGC4h5Z_RF|w(}h#Z}> z#`qYO8`-kH3)Km1yM0IeVs@bjPCSg(@l!* zm}ubhW?DG>k`RH~ugLx-jNJonRal&El*r@X!;gOI+g>qmNkzOLCPv=D5Ql-3;X886 zevW;M%q$H0Bn{AY3F){rg_k+5tbY(A66ytn_cudhBj)^Y?H)9cu7u_r3;vu#KaMAN zhG%Ouqjs0mMQRoP=&!Rct83dLH{PGIyDnIqwpLsoj~)LmzHjqflUNk&OWRUa8Lif} zbNhZJ#FQ?atUE}4=!j%Vv_GAr(1LRvvDIp*Qmu>0&kR(#>aBvGj_CyO25ll{*<=yM z)`Mi%&lb+$@qCN2OU{*Q@t{Dte#Et-?}x>kTg*v@L~CVC?!67I-Tj*zZw&1E5AVh_ zr{x8%-<^`_MZhMvq*6Th^VY_K5#c#gWkRrr{w$dCWrryUq%wAYgv^{e(ni8_NlY_D^ElHpLe~2zy8a-K@6;;8uPgRd4kHfPA>;e%ZkyAf0LeUjcxAot0Zp|!Q*eVuvl%Uj zD~D^GDK{v6U;`uipz3r8+JlrsB2l0?*U;-Z9d`qQ2I3>E5`9W*@AKW4K$j zgEd0*98fhVSOc*#nGY*}N^0q!%~6zll>(SV`889LWb)D5x`*~s6Y=fmN7_~x=(BCq zmG8UT+vu;Nv7M1XceHIO9YOeL1}T};AM>k{F4?R@Xs!aLXTLzC;QAgHRz825_7Qo+ zG&k51d^UJp_*85-wD|*jj>zrFOZ-4gP`>CrCk}ts8`&IlAZePVdatC>2G!&yr`MrB z%xuma2cc0Hbw0XjNYslK8__ZZ-!t$~@p)RuX|tq9O963^r&iX5zg^`;On&?KaLYau zjjwvwc0V$3NG3iem&~VhIM9+LeaHP{WXVr4GVO=ab!RPteTz8^K5?V< z-+~&)P=9e5g73RJbN%oe8TvN3+bcMjSA@FuuTQ#Dg;T#>$4-rw_f@T0`+qt8Bt^9{ zRUQS{Uzqr^cpGP&=<=qZ-VM#&Krq6E#?O~a@fo0?Q8W@_sj{CO3cs$kJTe8euneoJya3#9a;x(&XpUh zmLAiqr@8UXaUW{U%pZi(ZTo{n6?BdO4bq2JeI4GjG<;%bh{lr#c{Bo}-`Zrjs#r7H zGJ2b5@?J8$9ILh)E*xQBEoI0x2!Go&{Sh(x%5#_Zi08J^ZC`(T*TzW-Pp z4+>hUVeEOq6zv|eYluN;Mk%e}atS|gM{1Prsufrgw*uxKEoFLhj$0mC!z5#U;}$^5 z>z!xM4IkgIdR283eI!WahIYPmY`Z7Ze#eei>K>fSyBHOebZwhP7#hB(l?SWf$#BT= z4pr|PgL1WlS$;JXcsC7Er8=m<#HXeQ+~v?u2E~gJ(_DAP)N^o!eB|G+h2d;VKQ2$( zFTzRL`{`ABY6yek6BIQL_TiFnZonTb!=LM%Kx`RYr-FXi7>-HmRiq16OhwtS@l1}H ze{nAzYygl~Z}kVR;N3|9cr=q0c-!}RoD2PcSxCk5KVBzyi860k-0~j1tZYCPn`RH* z$UT>$#xwrWWIgrb2zsW3Dgm2ykhGC?8Q_W9QDBGN#9fFxr~}>eBO9X4XPBqcpJjF zpUwt0vZ|5wvvtbP`oTEt5|Wh^AfIlt(p9$+2dj>1{5V)?UM zXP8OpFv@QOYoX%NUqP)Ar!PJoOg37&H_*lB3{23%oz5c0-;eghxsHo|%}EHKrzw?G zC3%r~*DOuXlUyF1Ps&q^AoF19m*((rl{uOHa<10?(8(}XUOn-^!-5xdwpHEGax4F* zF0X@OhLVZdUoOJu z$m;adD*JK7@I^k?jKlVWryU|Lx*beS0^A%M{2Id$!8_<2$;7%cO%od0?ZS5Qs}+2e z`K1QmH@&-_!?i%U$2sn8>#SUGRDjy(@&(y%B}o*du&1FbvbhvNd&eAE_?@Z+S38ap zo@n9B6rRfz`!e`&BmSWs7m;T!FjoU$j$xbXkNB>FQ<IZyH56Gi!_2mGJ-ZW6 zG`r9GtHh{KUUAxXU_mo&tT|~Bw)702v%U?Nb?uI3?ihrrA(P1ZEY-&IG~&6 zz8XgQNd2=2?kouN{CKx3(el%IiP@Xdd}O~G?XwX?^M~2z_P&gMy`hv3%*v$61YA3; zSUj83a&cwJ$}TW6j`mqww!=0~^SrZv-^!5~d>EY7xNOE&XVu^B`86y=>Bgu+zUQOL z^BI)-LdFwy<1aqShO^k+?$SY@aNRc+l_(4Dqx|Hp79_I(OfWLnOt1`P{fw>Ao+gcl z$)msa&!e^`foEEC1t2LS?^HDK zAWAIfIB~vtN^m~Au@+3q$3pdPKZv7b?M!CVrf;f273GAVXLA6bfrkEhC`dLh@e#%v z>~WOtdRCH?xeGOn>Lt}%oKw=&1&2PAjV+Gw8|>q?%$C}1a@9LL%j-y!aYx_%8(0~s zp?3eaUh;^%{8^!R@k0gQSK=|&++qHh8yyx zv4Kk9qu_CGQjpcViZQ++Kn{eSA2a4ha5)!S(ZQ>)*fJ!U&cW}=6E@Dfj~>W=$M7f* z0=*Zi+5Qa(Y`5FjkmFGR{CIi7of%Pq5o4#9b>W@=2FzHyufeh%PpC;+qo0-y;7~74 zaTZ*$MhlOY)%V+WtNM>BXCP^h^s!m&udCW@0s@_Jxla{v8Hz zhSjmik(|$4O1|dl7-GQLqxT(34&K;wtO7*D$r`H~40dFf7_P0rF{37uPpCYj)kW*8 zIiQc=KjH!^3ECk^anOFgq$QY&lU&+&nl`Pn(}(AG`+^2lwr@AOIB3Eskvu(Z0L{~` z`DaTToMxM0*JkK1CY8|3N6ZcJIW~$64-yPxbK>*t^h!AVcT5nDhimo}g&2C75e?d8 zwEt$wdF2xYvN`*KgSoc(3aN@@w|=ptNN3hE=oL6wl#wOx!76Qs1!Y$sep*#u@3@AE}X zo3Ox;D9|HuK@IjKf2Re zR~bZ41!robyMlAD-&qdV*9ICCA6&yXLvHEbA=e!H#!6H|I6F1`CFcDlKx<<6;)xt6 zmYMX^E(YFYXbOYrSBF!fou{vy(iz!r`NieDTj2IJZIAS|Ysjn5;rXi9^BjJbH%2*7 zP}Lp;vUn4q8)&Sk)-h-;>rp5%7~`-wQ;k49{;#PP%^y;os};qG$JJg4j9Z)l!aU+Xn;RZKxK@xUF7B zma#jBe!WVT*U@B@n2~(C|0L;QX9kh@=%1r|A0uC)ss+1E$yx3<=i+PVFc^kl1KGZW zCnRRg*_NJmQqSCg9VX?Y^$vN`Sg_rC8mNzpQM<+6k!!~T${T-<_EW_ z4Ic-o$v$p|mub`Mtjq;iGVb3uoDn%)`Dfe8ib9B9JU)qtcPOT*tZC8OWE-|V|BUZ+ zR06x6=>{eSceIrFGVa8&D;2fRxlRZ+aeh6t5-d1wDeC>+xO5F)k0!RH#U7Aq-1qk^ zLX!VowpyTS71Iv6t6UF9x3!@5{xtySFTHrQ8%7kHthqmZ4KPB^ZS|EoG+8N&+3GiY zHd;~d=4GEna<-dt!7#$GG?IB+{Uqav%M$2CdUF&1Jr*s8N^rLGh)Zs`D!n<6<6|)6 z)thps5%@<+E3I%tP$XBL8bNG%%v3Xq+=|128}{W2`fS2&*o+h_g&2qf_}FF0syUAD zEz)H6{vVM_@ogw4DY=5oM83Xy6Y(NA!lw;x`J(D>ddt#AM_ueOT5e9#NYA{sUV5$z zw@=w{DNnqZRHjcF}4gfYR{A`)sDlf1ezVN4!I1U}R!_JT{1y;6NJFHgJKB-EJg z!I-%g`t9+uog|0Ig~l|imj2S-jD*ycbe?Nh8+3o;^5FtoWcPPc;ZM(606@LT{! z%>IOrtLcQS7u~vUGzhKzxR{sSY`^hl?qfZzj0+|xvj2;o7CTKOu!Y^=xsxIP$XP9W z=juUREBiCM_iC`~fj;RFW;eMBF2CAwFQ-|yeQT3P&_nk> zA-5L8t=e?qAAWW~Ix$WKjb<=k=T91Ri{28R}hsofa0mZBkwNF~14DP9=c0WMbI#`IUP?fGnISO`hqBR?UN!-($Da1_4l9M$<&%Xk=*O9x9um(ZdD7WFqJJs7G;kP)ETQUu*wd-6Ih zcs(hWW}QZ0Sh3&a>~8S-eKytM47c&<%#A~V)e?St%}FY9fu!m|d4d(U(5JPqyWy>;tJ2$zo4sAD0X9>>cU%E%=FAY)efE?8(}hmo z&jr!q$kZqQN58VPhu2!f0YZieo#$)F3uPO-ZGqF-HMOfuLUV{$Ro5D^6oCz}Lzq=7eLUeb{cjVNRga;?KIGwW2Lj9rRW)|5f% z>AE&~Ws4JW=I=KD<8kuH5=rOzwf=A4jfYPcKIO|YwYA*46`SW>vD!*-9=uNNj}V21 zHq#d$8G)x|f}m55pQ$@fyQ{vEEGB39z%{l^7`8s!qW|Vp`kZ^F>L0-95btBRK&Lk& z)Pa`)K`Upqq6*Hu-~MZe2Kgu_(ut4886G@q@{C0}Yf!J%3I8+_Q4*ocHX|2mf8hIk zL3W*dPF)kd`=R*Zxak5*+`OP*QEbFDO$_ToejTX#fs;*AbK}yJwj!{{={f^v;ovns z^?P*q{Xdpf`b+fnKVv}sd3D$S-kx`Jxr2`AL7;$zAW&xGXzIs%Y~+f!H$5ALhk0}h zA-o9ef0FhgGKcqw8uL!vc3FGwNVfOBtb5rTd2O6pY@6M2AL&DQb*$Szg@7cxI`z~J zM43826nTwX!l+N*VxP)or`CTHTIfv`X$ls-ea{W6!w^#EV0tS_xH{91IllV?BzxB7 z*l2RTILYgp=8jlb2Fb-hohZB>W}!7{X0`Y1!$sgWWqk>#;oaex(4KP1Wk0bn1vEw6 zxdzSu;_Ei;yFEyq3=->2MFDhD|7SaKbRSS~@K4B`forLo6!vX)6OGed>vjt}Kb0xx zi@tJnrfEy6;T`9ert`OUX2H}%gKJhfZF(O~a+3fsi;6|mG(HXC!dt9|Euv`3Zn@;} z%;j;VW7W!@=t=g6clH?!PLI>np^MXcjcgF#3&x(6@)QC3mbV0tZ$qNnM|JdEX6R!B z_{+9tH7Cm$oV0UXeF!1cDtjlCB&3jyidXlg-OOF#et`*|HwEXIJ@HrI@B8oLiQKm~ zc%_&moiEi`mA|)-x7f=CT=)zYHCOg>DnNBC5D#M&5`Wq-kkYAC{>oO@ptyLMcT<56 zH16HZQut=EJN&kJJgD5fU63^Lv_1z(a&VqR($%(IGB1axDi0IF;s*2ai+s72@c46U zt#{ZXr|kUQ(;Lh53c^EXAp<`@()KKR|5h1#JU03A#|%M(#G?S2v-icz*c>F zo5|GmyE?s6C9OAXpa2QbV6TOQfk)V3S*9 zU-IWCwwfACir%O*X3I=ex_~?E8@KSk4N7hlyD=o9oyOomX&Y*ipUJx8;=x8pVFIW1 zJMB_%ywP9*H=C9uMo|nA)jTKO7q}Syz}%wpJOA?n(B~BGGbTUp@~5EfPQd-~s)7kw z{M4zMX~+0|#UnA}OUITX8g%sHk)!5z)`OdMp5IQd3CuPA94we)ISIh#ytDDCbhPVG zG<`ZM(j6wYe#_&$^E4l+Lp{wPLW6wdR)}hz>0o#(zkBVN;A>KO7H&=ir@Rg5XUQE< z-KCvQ2xdH+L>tqD9G$vo1+qpSP)LyWuzb}=pg{jTHm2=K`Jm?!j*Oau%NbQ4t_R17 z)ZZ0`L}_fyy{R&K^a-!asDQSWZ=o zs`8~9MAzEIF1zyWzq#(X1GB$jgJj!3Km3voN;g9_Gt8EP3C0F$he8H7Ti^SU|Nhe{ zlu2azhLcmEpgFV*{n{n9zzl25m5!=#OD&Xs-d$12f53Ljusy?-N2(BG-~f8A3dLG> zCRMf6pmQNjyw#?nJw%OIq0;O>5NFVAp+8+&-K#|;dXaVVyg9uDs5@t2YQkjgV;zq@ zO3SQJ3VsCJZk(f%o2lj&x*>2T0^po$ zv3TFx`sdd|I!x?ltj`mXbF3qn|4br8N<2eb1Dkbb%?hTxzSj?1f#F`;VgPL2!ia+v zSDme?5LjT8`1f%qDbBa+)}n80KIC*Cc=6=#!8uhd@cQ44!?TvoZU!}_SC57W{J?wuyU1LUi@(eZNli9i#i97(y>otPjgVoT@C?5g<3dw zzF|{7L&>2_cQEY^;qY+<)0u;bqZD9)Ge1x0SL@SCGV=z!M~x9wduvL#lG^GEjZc6^ zB5#}SNP%H)?`p$>uDb6l$-2Vf+yFhQmgC{kbQJ4LwUc*sy4TKFH2rsV!mJS(mr?Qb z;>_4j@pq#g%fIuKs-L)|GHC&kp?{HwcVH`Cg<7ht0I_X8EC8zUUiMq;qYW+W_0x~x zZ#8T*{0f;Yjy}xR;<|;+)JBgVOGL<+W!H8Z{riM(Go?{qIiw#bM0E|&^@?`qRQhc4 zJhFil#u6HrJKR3qDd;@dUPk0b7R>MMUH*_CQlgz3@z1Qja8~1H0R1d)SODW&gZIWw z%f$mf#ND0a9Nl+ab_U-``sv%vj~kCsw{?G+V4f$O9{UK;_7pjDGghDwe*a#tcqO_W zw`|3d_uG>%`rGMK`Xd=(sgqc)Hzm>WeBWe??dvibmfo=Ra2LYm_JdhWT)D|+OLQMR z!XL10xz{BqIFC0bYPv>wjoXbM>P-DYd%M`5B*>>_G_9dd4sQ9Tb{F{cca?xRXG;(I zqlzxem#ZJ5dj^|g*&P)9?-`{W_Del`1IH0LNgiFZFE!d$Mj z+N`HLgm$#nRe5n@uXi{;j8ZbMRo-9f@c`JX;*GMk`@6w>(xRV8?q}F+Cly)aEK=%& zf>^`%8)C_}L3ssom5uMP@b{Tq&7Tt(jVIffXH9H7Of&uW~ZE%yd`Ti|*Rv`M*-= zzYMZKrkit|*<=3nbsdOkBK#CX87BrK4!m}cXax5{ugLq9rVZ#C@N|u0fF_TfOQz?& zW<6-S1gGbPkSxK7Rb@nVFtyhi-S!yrm{@dm9AJ0(7nz0f*}H$$G2C=DhCEvPrs&nx zc6z!NjKB~&&dsh4!ENWCuXYGFi1m$J8_kaA|15?a5bKa2*mAFT=u0Bvs0L^`VbT4`a`Zi)fgKGY3LsIpY+8 z5&T%{RSARiMW*!}m_$ysx*M@|udXs;NV3>&k@?{IT88E2PnyI?yb$EaeaPE($Cilk z`U9q;#*hO>sDhv_n0riJ&Rh`OF7X_+mB#;Wjw++lgwA{Bx2pE$w*?_5U2KvWXNp-F zI{IfR;yr6K;OlzN|>Dy2qQkg5F1r_D3%^ zZWby>R0O}%AgRm`esH)NaXAgFLB~bxG0EN)m0ozhH!MmHzq~&noev(pr!A-ao~p6W z#Z`3W&ujrj2yp}MK4s&+l*9dXD5)EY=jWXbtx9G`sS}?nIiBhCKUD(Ht*}RCZ_lyN zV^{b-0(H$+Ciuo*RL?BVh(hxPQwb&R82M?{N2Q2giT482a^d?)=bkTgrxVEe3CDVL z=jkSA163%&fZUx~{VpdDtT^o|>X@O}dx^%iz>OK#ui++jr*QEIw-#MI|BUlbv)+qo zYvy>VIl&2m>@U90+uD;t+FAbQr+`zJBNc-7#qk4X7721buYuszZ|a0!w}xW?JAE-? zj=CKHudboaofy^%2HHIErBEmG&4b2*oL5(g7j3IL4P7$=x{uDR7^m&9qVn3B7nMr= zGf9}Z7Iu{Z^Ff4SmGH*b2Q$MC^LN7oH#9eLe-1b7D+*jY{@tY4X9IAdOm4qEv%hEr z%Y0?!G}RC%xP)8?`}q7}bq5UMiD~n8@$R9XsT=8*YCvI8$Y>xcgd7f)1VJTo*zI(c zCNtB^nO#Z38_uM)8bqn-cNLP04KAdfVSfw^y+zgKO*2s0zRL@$aLi@paT?+b>^0re%IwK265e}6v~80JNlg`{ya;4 zu734bzW%=qm+Hqwd(Q|g2Nz*X4#oC(_BXi?(cAn!HnC;jAVOLcAU)Y^;+KFc}Tr&&TPe{(dZU^_R8Jo{Dec?V}ucltTDA$QuT zDxB@wp7EvN_uiy8FL%I8!!pmmhkKe=Ho^5ia`l*6+bQT`y%x$G0^J7^{l!>`bZcp- zPZ2prdW^b{JGa->dt=Bak`XTlCM0hK2fvnz05w zJiqb`hj#YU#jyjoKoOgO=eM+`1MGsAeMl{qvgu}#Duc|+UZ=N+i!1;TIoK8cbolzR z7-nWU@~hX7qx5xmrWXV5J0rP|zUgK#L+&qY9M3~C-3d8uCNx8t?VpPufF0gvlq!jJ z);oTeYSx#wM|x;y@IjXJ-rogpzZ85Er&}`0+S2!eeU-)`4&jm4H-p_DuI@_#o*nR? zh)uh!^-T%RRx~oh7@aVoe%p7%pNN=|dT4;8Xq*m6ma`N>=U!S#&0uJ0f%dzYY zH#kdm06_2=L-&il-Z1DamOS=%!U|eLHGi56d}VjryOs+Hz5{S(?BIUdHKRwF_M{A^ z9B%UkC7-F!p-a!&6UWJNo=>?$xRv1_ab&qv_vGmUA-3EfrEzAHd7m#vI1%J{T&Ro^ zEMH%%NKE%s3LY!pxhmENjSBjkadF$VvZ53w`LHji)%2DY&LPJ|a0OfAd7`t(wCmpW zyxL!L=-+j#%MrzcQ3aMTSiW^W%UqvNej49OmIwRgE~3!==86q)JYnl#1UHI5cMSzS zJnKCe;AQxJn0_GLcrXw)%f3DPsUE@J-E8B)HKW`z?<+NS`*UBC z>8oKw!JcO#)`H$l*=!tURzG`Zl}ZR_@GL9l5K`i$PZ&7=)ot3nzu}D2g*h?; zzrTo=@s%zCiD25Ri-4UqxFnUim}BZ`@R^x{Am>S(R=j=A<6>;@I!8mER=|?+r%S4* zS9Kw-g{}I640geZ>Z{rpveW*3nl*HAzQTm#0Rg77PoLTkHF7`t{?QOTCMOO9pfyEn zZwR{_72ZrMc3nQ@6F|lqNDa}KaN4c%a8pre`sB7DK~?<7J9cN5TO@(RPk2S^!!rR2 zGRVV;-IpzJ%%h&Ibg#vRl_~p6MKZi^f_$tEZ8V;Fp7dJWjID2(qIp6JmZV#r*uJ1(g@)x&10 z_$L?lV(}h)Tg)A7H4-7WeIU(Cpr`4@qIC!6x@At~?ss?E0O%`vEjP}s#)R{-CwYB> z0e+@~FDKfWCO#|Z{#N9`u5}*Upl5a3RN1H@n#=G3)VMLaWbhdqdQocWn4(oPuyP@J zq0;ma#C5?QQ%dKSV<^w*{Z{Er9C2dozt>dY4lHrhkYOl_RlS6kFg0 z5B0;?VNA3ra!1O_S~d-CB$?l-4U0&maB*oQfkFay+r&e+TwL8{GU#x5>8Oo+mBzCG zoPU{(^j*;Si~pZqIYp4Y@S~S{8$?gISW5?=Ng>vWl1MB8teJO{Ac@$iqO4L~mXIWJ z)y2p3D%ie-mxlH)bS`q6f-|25p6aD*ou=pA6R*2i3pmGhZ4e8PDk+z3@m)xg(Na-7 zA$05#DSLGSyo{!-a0$qRH5`eEr zv?VHeU_3Iungv@|RxC}GAcp<;SA!I}H3;qAW@bDXX0T$|ig3zIs4Xjs@hoUJOTy}HV1pVCC`9=Eh)Us4-Z4Cz=1aY zI`Md$F2MdZffXksWuN3maI=u>Q{^WX=oZ+MOb_itDqTk%{=xO?yI@lhI8ZuFUrGmnF-^K}-Ah^7uhr2~fn+RqSbPs1t91 z=V*(SNVD^5WEIML{F=t>tu6Yo@qsw*JOB9g{?Z~XnWXLo&`6$q?p`uE>6`c@Da?(rr=6xfhzhCqRn}wDYmbmDMp6-}uAP1X<7T7Wt5<~*4Yo0=z3eI< zOcR||6J3b^=)vN%6@h51{H}4zU+;L%q35z#qjG&)Cz)rXOhx8xU#p_vpbspL?oEQ# z2A|rAFS*I1El@OcEi5^t!g8gYc|2{e$=DnDiujyU1T{a#7gp4y|t4(n4R-6S**We z6mz#XJtFZ!(NMtQR=B_h@4%bnXU0CC<&LEagmH2@Ru;fc(?i`r(v#u?N8D}As0N)4 zk9GDTz%PYVWt>=f@W$lB5U0f6>gg%XZVFDv^Gg(iGQ8$RPLGATV5-q}g3NeQux&=m5cCAQ?FrMjH8=haa8ty%--P1P!n<&e zSj65(W$lE~&-0RgY>+GcDt7g8Pt;z+?oYide@L=~d#6>CeiH#E6kGG7hPFOWd|s>m zzxX!%I5lie82vXbPI!h(Q0c3N-f2Flt!**1`eB)7kZAmQrCFAwx1ELZ@NNOFuOMEv z;UevF?%Ty({ErU&*!|~fAF>*^zW}E_2W_{8c2p8O@nl}oM~)Zddkm6ztwf@|Yi=UE zr4P!e^*6UZJ|QVAged+N(-InB>lqQ9p8d1}XLP?i5g#kU_GJ@ODm5>D;s;p_HoM?P+vND zgr5#tdEMK9{8^I8R|ENyO6$19cmptSgU#4CFnt4Z(Q+wxd}C=N#j(p*MW{R=_u&h_ z=tj}b->Y5hjn|4raq;bRFh+4g&9{^9XS~V~P0Tg9MHtn>rJne47T<$D$yo*6dIrRw z)8ZxKeL1Kw$l>i(GT>y8)gCz)c6_fAuBsm#7&>=prjFy=(aO8{Y=Pn;8oynPc#02l z6+Jm^KN49W8&xhix7ap`S6=?BLqe77PlBz6UyqT4qbMJ|{k8sr$BeIWV&Dr&>?ze! zY>_(K%SJD;%CVt{0!>9!^Dck$(1(90w_D1YPkS~L^uu?Pvn7~+=wcDTH+#(ZTkt35 zzC_>s`pE-=>biMe#Ot}2ZM#aZNLk9X<3+)K!XAHceZr&aO$IH$WrVdtBZJB!+l60S zV?d>LB}X7QvupFrqBQ=fAzida0^>m5;9UWI;Fii4 z&4OdK6?>)x;N?3HU+MA?kDEl)bK9;IR67Y!Xmh}4N&2=sdu${6XCurTKd@}rID}+2 z(0fsai`!H4ufQ&hR`}9Rybud87STL`9N3h;vh~^oo1fYHMV5dXMq?P>fug3A|M>3R z7F_!?CRySUMTzu0?xy6Vkoc_@n<@JE;al|9Oxw?a%+PtmTz$y*|7-@nMO6@{0Ew%1 z2t1Q$I|tQK(7aJ&g+8y)M8FQs=k4GT6yTXNMvG%UWq0vy9xPGHY+MNd!i(2$l@wbl zbNW_*AIN6^D&NMew>IW5CH9sVeYJ8|JCo}SLJxk}5{vveQ z0%guLrT3WH_&VRXAmn#ln5S`w%^rlHIu5#{=Q>PCDB?bRaPd{iRAR}5d5S7%+*5H> zej_bmRDMu>(-l4oTKg-6yEXtK9@_c^zBZaM!FGdALsJQNq_`I4{&F=~?>4cDoXD>^ z+GPVd%`X4a5hFj2LUaB4%MZk2dl8g(TMdtWQ4aE3zl^0(+abhlO$4f+rBp z4s`7LRPLnA^r%$CPF?bTvWk<`68m?pDG0K6%W_4!Y78XBdbprR)WZfdBN;nnP>qyn zXg<@L31v(z>HMHHj+V4<#n!&vcAS~%wcbUC)4AjMF?ceQZQQMY>1*kUqJF?xZcrjv zLri|SEdfhCl;y2$Hv}v(9UnB$L6II$*K4Px zEC2{>3mGGjebdb@0f>dQ?5i4zVzKCI&=PDHv`@M~UtbD22isltpCHH9z!$4|FOLT2 zFUKyV(z}kk&Lx)$2{~F{s9t*P$@LXT+x(;(@7sDa&o}VChXR?UXBJvX(wORg0Bc_; zrVuX+Sa?3lj(TqW)nziynP7S*vA_5A#Kwchv2BkljmhLssQZzko!Xd>WUBny_s5#J zD7I1o78gxw0#=LAJu-0QrpOad{$c#Mv<}_?3@IJ8$o1QXZ27o1-mUJfcnK zphKcnM>mRlz1kKg1BR%!weK-_7uou9ym+qbd3D~O>s;sD=l*@apHD91)5I-yWm{)Lp12jISHP$EcW&Vca!YDbQ8#I+ zTUY}hcBX%_(#c)dL@3hn12Sh5O18Twf3Ygc^R>-8I&eixx{TK?((OA}t}jofI)|Pi zJ>Sk{%kjI*Z)o!&oS0t|Z?ju*^&6zVnMH3dh7}1XK1D{e-{xHWQ z`J;-~V|TT=`c05xFdlKL4FBn{LI?c7d@LrjC|`&3O0??AcLs7xAg5!3=z5O?^=Ev? zZ@*rw+99+Z0<=>4$ zC3orj$$DUtbcoXN7}#P=!6N1-HIaO~IVmq83Fk<^tFe&Jq?TpS((YiAZrm>wTodmx zd+{cWLDw2e*Do6=N~JOjXYHr{7(JzAmB(g#ZOyEJex=`!N<#XQV25}+p!98Myf2;e zjOf|rSQKS9q0yjErdzkspcn#wv1&XUj(X&pRKeLsU%~_iI~-~1yt*g_5US^k29%r} zW|k5VY}l1z;25Z)gMClvmKbb6#&VqV;StPOr_SI`UgJhjymYCYd8_jQaCE# zO&g5N@z&7PU*?-WaM9Zui(v16$EV@j%9bLver>B;&tN+0*HGZy{$FilDQzgRb%1;G z@qn{ zpfBI;1D=)0Kyj>u?k|U4F=zHv8@(vxdTfLR)yj19DX;6J*L0mKKNfD2c*Y~jzhR8` z>yeyFta;$G#zwfzBat1nQ+=PFoufeB= z#${k#g7DEI2gTA|k#w(3+r&iAc$}lN&zs#j)!(=`A9iEudsd^qoNb0o2I*Yyq8?Sb zb+4!8JDl5FV+Qz9i~G*Lc+{6>al@XB=|Bq#bzota9O6Q0C1(D+znhM>V3#sH)c9Z- z1NzuH#a1d$*g&iI)YvzcTNVD)!O^Fb+)b@7V=*}in!Jkdwc*_(NPM;UrSOwM8*eMm z=1irEwqy7a{Fc>LlxwzW3Di|$z*lk@IhL@D+d2`T3{gCvlHPcEL;>87!A!h-LPy<9 zQ(0tqA}$!*7E~r@5H#~9c_RS1@s5f4zKQ{DpfMd;mv!<^@P7{Lz8+$Ue!_{)CSxP- zT1H!8Bl%Y6`48ip)#e;SwrMw@KDzpU*u~GEW-H8?@$kX<4#82xKgnU9dl8A{%k^sP z@J!Eaxzq7JQHN?qGnq!eoP?Yq5+%a(4KDkv+ckd{Bpox4HWJACZARl~rxFLOc(LqJ zye#UwX{AmJU2(1(e|_Nc_l~~Efz26f$ZiI!SHC)F`^Ji*pN-5he{@9c^0cE~ceI(= zy3MMgarNLQx^LQ-PybXPG)qYwmT_gAl(SgPe%cR*$k`rUu)Uo-C=>>;AtrVHL33{N7K;)%Tx zuq!bO*PERk3&>7?V~^6opCisLCM0hMgsK$TSsg}0e> zGLIzIXdG3AGYnh40$*z{n12eihic|IUIY8)sRnbWHBQqvB)RXsChALL8gj@lkfXe= zA0W9V_A1sYC$mD+iJGT0yYnt$;2rK_uX4yz0LXr4DJW5y{(=xHe^uc)RD}DHGWeSP z_ijH+PnIL|p2q?e7t(uj5$YTGP=e$Itxd2#1pOfQP4GJJ!gN?8WF5yPv#Wiv$VLF1 z(YM$otaekN=hiNG{z1Cmb~_og`eJ5{_sj|hGzTHwcBNC5v$ZT6SII5af%+|zc31q@!t>wfj0L48plyj<@Oo}CA_>_AyIkrP zC=*CtN2gvyxi^1&NVCmwHlPZ2^zJm`v2C=|*EBpwoXWuG9B6?~q#WMdU=07YkQELN z)Q)o1it>zKS|VbYv%rGXlkze|mz$QF+|aXD2syIsdeZ%R=QQ&`WB+f!L72Wy9Q_+o zYM!gg+Y+7<89vuI;N2*of#UGd<1UmpaJ=Si`AKzYFpP22s0S&!Z5h2@KF7m;dhOX| zxEt=qd`R=1C2hCK0pj%qO;iF9uH^k8u!PEX8P8YNmkSSXIl#lCPqHX8tEhMFI1QRM zcA^)+xPxYwz4dW-O|+4s{6~krW+4@P#P~nyt&oGqszu>zuK;fGROlRgWval%4c;7oi6}C-w%V4+=pvBMqju0Qo2TFQFzCCO-2m?3_EyiNP*IEVybF~_ zTAPY=Ji0ktiI>XKsYTI{a@!F!DjpG3z`59Z#nBAL{G)PPY`B<31^dmrO#9Z*XY3y6 zrR;ZLk6qKTtU;|;NBnyWi`NKH-itxl8PpOv#6uba^Kb@txF?=iBA25=E?1S2edRmo zy^v3bYl-K|>Fy*1PZQ_@c_fhzC3Ljlan7G{4E{p0t!k?5*1yMmM9YT$u=3_Mic4CY zaSZXlgM=O_n*H|p94l;{iQb9f7UT#ySRuouq+caQwR8LfMaXNPj(bV-VRxKGeysiXiNt%tF|J`TkP&@dty6F1iRuf4IQ zFuuj<VRhG2jf@eU~1MI9};f779NgR zFqvd$cPY*27kb_TaV65b_!tX;X%rx`FdW+Cskz48(#Ex{Ea=Y{dx~+XZ-N}tja!4Z z`fVOsbrBX`-6w}t#cytG_R-2ui5IR~6z;V`wc`eYLX{Xd`{;1lfNvrZ2EH)f);{~> z4H`Q~ko0DevCr3>)!HYi>}Dt2Yx$t{Mar_*7av|v$yGeOr?cO85joAt;u7a&y)qa} zIM^?LBrI^8yBxNhcRs30PuPCMCkK7Jt=`0XtVgSBsy({lKK$W6iZk%hgugz+??~+| zimj3I-`x`%2Apv#%WE7ZREWF~ke`AzS<#b^+~~SXq0vRNtsd;j8x7T+Ntdca0mK6o z4Y*z9!_0dQs0ZDYuS`=Z3QPXx!*Z7Q*3P8XRXv?8P_BwIxB9oq9(tfiYz0NMWO&^I z)us_V^Re-DRW%)N%@2>{h{<+}rbDlrUnqMfw!U-Q(4sl+yeydD_BYPOi`~nXFP6~N z2Dg6$@`ko>u81+sO)VTB9GTHz4*C`Flzk{q>-4}*EG20ggdV^$mDTlh&`iz_yS^@C z)bacQVs(Eh6j&9X$AHo4N$uo_a@i_sGR(?3Q_Hblre-_$cKF-W{K?JtTqYBfQUL2y z-{%90;4M0C2K0cx-|lfZTU-@;GhJ-Jpm5aLmPwRmQyvi-FO(~YOD$gJcjel$xap;O zeU>bEzm04BAtR9q*=4z=-s?ZFroHy~NP~$q#2ygW#C@ryQWEU+&wcPO_rB@nNm+uf zqGr=}Sj+I04{_~`mXD{O*zvnYQ1&{O!$_dfTj$0bi{e4=L3QI_?yW<5UO;2K0GEqM zHoBy^Rj8+1wUPI$RYUabJ~XQ3#%SUdG?Rwtzr#h*d;nS3Txk*x`j^y8e@InqDG7HmG7?~OH>X4fEDleNz% z3v#)hX_k3dLDsdph$J)IfuPzLA%4O(Ro&cIcM=R0%xRVwPL&dr^bE}G@Zg|7fq+zx zw1%E{J|Dytl8haroh?s2*|zD`$@LRzwW8&Lsg?Y?Sa2$!JP%G1aW=w$C)kloyt{FX z2ZGVRz2`PR-(rzTaNhc!6JskO3p6%Wm9B)Y=3!m>(i(bq5*O{SHcjDEgI@^XgD{DiXSO(n!CZo(cl$r}@GpdJyMy;67eVNP(qUx3CFsd5 zZw8k6eJQ7&!kyawxnyjMA9J&6IhS94&Nh9_Hr&VHess4sfqDUf|03?#r%B@}>QM5$ zHC#f`QO>m&&vY{RbfY5&s}UFR-55pJ60&zuWmHh+S2u4(VMI&zT8P+FMH7BFZ<~L) zi&;Opig}uby>s$cQ>xMCfqnk9M{&v!$RLI%-=CF)tlb=>Lp&CcWM zW0e_v_`~lryj`I^^QFf=$%>bq{Hu0Mk#~hl4RL3v0*oL*O8zP!x<=%Uh0;H<&S!GAqcj`Iy$V!6KsI{J!A>@@v#5ew6Vu_TGDsLa(!Q&(5!;79)9r z4|bL_6wlMQ(TJEr81u(JaABQvAs5pyM-11gB+k!E(xN4+c(vP#`S&Vkf`9wEz|!DD zsBvz+LEysflH0$-7dazgxXP~xYYk#8=~LiprWz9xx++x!t<^B$No0K0l4C2c8VC|8 zDjC$P69z(ccyIMA0RU&rVTz-^Os&A}%SoS2HQ0gJPsoqqmus)Iu?_zK%C%aOZ+2p@ z1)ft(;xW!`51%3V6CisOPq9rG^c%3aFg&lqs?Ke6|J0 z;->cFk0_*w$d3?4uN1vW6-Zg-^J!PZr##WE+9n^ElRMs0dnUCCA&@r~(`Wrvszc|7 zszY0m<-&~ZYoWM`zh@NUxD6`iWx1hur8bYTqMafeCg&nB?lk6^o5qq#i+#`LD)o+{ zhIM&wy>`n(8D{=GStw@kdCNGr@y&EWg5MmQR3ijxs)#Y$3W@o`a8|er>7^p1=9{ih zAMkIhR5>_K7d8(L^q4l`cm^JG_S*|I1pV?0E4qPTgG`%U{=ulCj=v4jxhnLoABZ}0 zO(-K0ibEr*S0&7AWt`WuE*xD&u1L0gxp>8F#!XbpoImYG({+=1Jck~Mp$`6mP_ZG? z&OcJ^`gVae&S>~3zNj2>tdfu1yS#SNe4x9YqR2F?`r`(*ZkdE=x>wcBR^Yw1jLy?> z2k|YAQOS)H^8na8o?O|ppNj5-PH~8xL}jDFK;~CbyTHHn<=z5U4@=7C#g~$1zDE#* z3L>^6>xb*4_M)v}W8zM5cMrOSva3~oBw0zq*xzLWILe!@BzZo(?RmC9NV<5$o!ggv zC^Ak`kL8)9!AEt~sKEOTN}Y?X5}xnCq!KG6ZtHOo~hu5ZaXNN?OsnZ`vX07T34^!SL1XvBRFZH;40JCDTkc zAZ0P%D)J;%MxCRVMi?rc;yuCh$_QT~6ZCQaXnlbdi&yzyj^!}kJ|EC117b3UE>P@C zUR^I=wk==caD~v*VlWo(aMdQ_L~*emcNVULv1hMUeTeDMtCn}KS)DontH^q(4BS~sKF97-2-`3Pu5DwZ7J&3FwrQP?A8FecUy9F8Iaf{h#R^@ZZu zQ7*%C(>wkhnd!H}4Q!Wqw=}Z!&!1tFf_l2&lR0(soF26(F@N~Y`1!Wgv!zwWlo0CO zwb(Qegi!=LpdD?{;p8wK!DzkJg`!;*OaLQw(5mP;oo=BN4C7`bC1(YP%C5!>I)>vP z`mPsGr+&4*F57MCoUd0#%k>f?Wc8BaB!qoE>Hs2D5)13EWCqB)yZ} zc%is~m?pCI#KJZDLK+}Fs;+X|ZyO9a(njiT<~Z_loTq7oUR zx6fvrLJ1tc+1s)!yq}o+J#Wixd9No~mf-!4cJI1>P9hL>K4{46K+ca-jl`u%M z6x@BQRSDCcdv8+l*P(i^0?~<;Np{5H_q+~rGU=;Zc)&^rw%UIS?qFH3 z!TBi|cAB1erIo5-EQ5!whdm3FI{;NB^zl+y` ze@dS$Efz8)I|nbp0v3bRIv#T16x9Y^GLE1wB2})h`VJtPBzm5xeptLlgoHlgvQ@V8 zWGod}7A%me^p*}#^gZoezE+K3MkqkeH`z`1zp~>pAqAY{x%c+{nQ%1sz5a#^nhJTl=3AMGiG_hsNHe2*2MWL%1K>T?u} ztAWn^C{_5D5UQxazcPJG)5q31e$4_&A!|1~|4E`HOw)F)U+ZII^axScBf|-CA%C{H z0Ed})fJq$Y>0_Gh4|Sko2(sc2#a6H3#T2woTYlvR zNg#6H-!LI3o$h|ET^{~^gcmfO4EoQ~HybgIbeh)J4gso|v-jFvcru0i_u7{31HtaspjFgHCbpJ%XEl|bI(KSPN3>-?9*@lPtsOl619r#1ooi|10O55F(yRl;Zw zArp1MDYYonOKuA1^u2H8Rf>cEivB`dlHwVEN1aB~y2@WJ**@E4&xGf?aLTQI@Rm?B zKf$$Q6kCt)1gZ(??CKFcZy(FxIj_C{jsjZ|m;GfHyU|3y1g8hK`CB3U>-5$I-`nZcNye6{9YNKOPM~6IYlmd9|%b58N zH;WF#A{g5;+OnaThTB)fWE$U(giM2%cL!(vscm>~S>pdul%2FH?F2C(hpX~gk5{?5 z$OYqo}AsrzunJ&6dw=`CqV@3k63p7t#HzsBQ1& zSys)L+{RK8rT26%6>J@UupmF~a2z-tNjRRB@q$tYBLi1HVsOJ>$@ZJ>TaSGM-PtDQ zhvyT1 zi847?0_wCy(V^f`1nfcJ;qD$)0%T&O(P8=<)n?6%M2>_}N$mER47XWt!z$oS7azSU zrgxoU%hX|+Rzh#KK36*0R9kA*fD*@Q7&yTg9QeTe72CZe3h-hIIO!|XHPsc3&H>px zI3d)&zWy$y&m#cN2_I$zKk+aB0hn$7-;5;a|8XxbFFUgmQ5II@abZd5`Q-p50KSmg z7qZPmTumjq$u`N3a@;w<%?Odl``N%$=}xl zf41EF?Z+p-oEki#oqe)WEm0*%yyhgH&9|gu8z!9c(C&VsF}G03r|U%w+fdigIajJM z&^m>}RVAOC#$>%E&Xez|x|%7pqHT2a6rc|A3)g-<`>|Mm^yd{4g+%z!$(p+mmIENx zsq1iYGI`^Z3+jp7IQCf=1zM|>W3|`48?l(-*{Zw^H%iH*uC7~f7jYcU*X@z702{eq z+6x>1t22j_#zo;MNv|i~RZcXhM|4+z1YrYW*{Q3L>P*j=)M+hao%EVE&Fau5>x9dN z7L`-rc$atF$Gf37wJkZ>ry5rljEpJel-IOWe2;#NyKtd573JLwjoopY@FucnZzjNh z@;X9_ge%aper@G~4(n*6mb>e!r(sod@Ue5?QLi>IS}wBQtT8mGN=3D+@2=HCM*6W# zxhtJJ;sJ!Tbd=*}QFNg)TopP;`lSNjFhC4wT(~I|{}it!=x~?N%k5F5@|APGig7o0 zk%Hk&^W75O&3?;|-(=rqjT!TwXF7dk*26iMr_9-0hqC41HA0Azf)<`UOI?wN%0ZVePQGAmBfB3ce2{nf1uHM{VX*LBePJs<)PIPxo}clo&S%a|HQ`Z zgM0_i#Q`Xf9Dz&yS_6k{c<8TEfd>OVp_eAW-YXmZfKOR+qBz{PrdrA9g6>FbZ`@|< z#&49I@|Y-45p8kCgWTHxZNm|{^qyRJMleZ#!D6P1yWc6*HGZ*pICD% zzgRP@vJPYw7w*)&VBh$KVNHhVtw!lXq4)LF^$&(*)_im0RJe-opfkoYFm-YX>8Fhg zyIgEp^@i8toX$391DUfGlh_9v9X*@u|7{qZ{bRl&$;qi;z!^PM~G!ue99Jmlg|Q+p4WbT=Q9$T8qyEKe)I z=;|Ki5cxd0?q&h*ECefhg5TX)ub<~#ZqEkPGN$Rj^+D<(QZSXGqyv7|snZ+Me?A>F z-@P)FlC&6c=S`y`Gk7rBernx{G{g4iUNBnKH4C`*u%RUs z$kFpw6k-4lVdrDD-J9XNDVVz8sUN!xSnH*r7;MI>V!{m< zJahDnvTd4*-e9l`6oVMhsrk*>nsQGjYAiA2_|s8--9xD65)wNnZFEw1_^i zshuizr^s#oQWR&m&T1;WRl-QDR7df>XPnnp;Xo>y0lQ%KAFwfi@;aB$|BdgvBUr2B5?nFob69X|qcD z_hmqpu#_4Kb-ejSW}fwfnfozcM$5CCs7cql8~GMz>0ZR|v&$RZFI$C*=ECf(x6@&R z3dE;J=1SHmcH3ei^?&fCoKUx09UhdA^+%2iKzuXI2zWw)SLe`S7S%W`HHi7fc8nw{ zMg`i@_*qGwF>oZ^m(8nUDn*-gBzUcy99%n*b6|)pTlAOkPg8Ham-Kh>h`j(0)O>$%!|`+m?rm;1qMI&JvDuE>6tP zF2)YY-E{8Ud0I91M}*Lqd41wM##ZMtS6+dJ3HhgTvIxyf*+FLJeYlt-VDoz5w+KRp zYY8qpYxBZ31As)fAtOI>SST!V&4w9Ir}i4Cro{(l6K-5 zq+zxewB9#d{x$blW|YisvvE1{?p$xeH2mR9%$dSIrK%gs7Wo?=$EuD%QLZmVCiwM~t{;4I25C^l-`1+eI6d zH&ARqrxX$?&W;Sol&P^~3f!PjCOqmIuTx?Stk#47IQRQ{Q{Hpd$~VEIx6QqB3T~Ch z>xDVXHifuoi{}qoDTEo{_U#6Zsh-{Y{Sr0eUE+gI) z4;e^0e7{{cRIv-e`NV&gqHfa3!Y#-Q#faM8O*^D(u`6Z1lFm^?JtM29Zo8^qzTd#~ zt3~;3n3i<-ycny^h>0W~l)k4~Ah|*;}M+d$H*&}lk znS^!Pb@u%D;v6S!gf1#)SB|*vy#Kd6#E50up_i|GwS8Op)|HDTFhi#} z*g&cJ)&RNth^sM4>HC!#U`uwJYG-{oliliywxh-dFEYyfqSx+YceOfykvj!;fbf<} zuVl3*km%H4nEjQ)mQg=S;>lE;!mjrgq^&7%K?R|i;l8FV`A9W?klE`F=1OD>XO-F0 z?=VE?9wA&PCrEa1_0?;4>`l-U~y4kY##x0R@8?gjsHy$3M44y4)WR4+shfc>)mB)3%UP4QYUcNl=>WY+AnI4w+t(eZ5w+?3Go%-_E9l5M8o-OrDFFba6(U!Q zBsiQ|pQtujh4}W;&`#fL19yZrNr{eOwK{)q6UTv1~{d-}IiVP~gMFv;; zxOj1qL}F1*x0W7&WQb8;YbSRp(V>1O?x72!?_+uJLe*}+xb(v z0&v_t#Pja-O+L)fo@+Z%O2Z-|fpc+LPzc5VSHUzcWGHMW!U#Vgb$b}2LbNSwepIoF zqa5(R7pVv0sW-YN63SzCEMg?ssPKO4M@fhe&7G5drvN1&knDrJF$971$Lg*r29Tc$ zI*oH%&j=f7j$|o*I3C*%lgnxDnw{e@e)n`>Tis%dd03L0lckK`yrsGG{Tr}jneClr z#nv{G&5SJtr-yMSA>R4!&Y0$kC~dHDksU=>K62IfkZPJn!*BfeOqqqemr3YP1s|rm ziOe9I5UZowCNFnCW6J~`pQ@n(fZRBc0_L-%?b2AqT1Dk;O4c47As%((k?EhD32d{b zLXpJ7Z<#rev4q*>ma=jq&eL$W>CA}yY>e&sEK#Ul>{HU&F1T0sp5=8*Qbgq`rELap zNOw$6!4-1Tyn(M6@aUs<(}D9uOR1?6ySgScS7HN&|D!}y;ky3}cht6;3_)#ozeQQUA0)IK+^m1`)Y{juX(Tv7P^IJ+ z|DKCpKm*dsFQOD&Px|;z0aswuU7IZVAG+tW^F?a4Y9A`0dv+#7mqCh#u0ZKxg;LJ3 z>>bBg#lU4^(`DkhpYA?6Z)h**`(l~*`x|=EJ9W>zxVO?CiSYfR zz2wDd-;9km{vQ3gfaa@i!;gX^z>^;$f>i}ioK0W)E-O4r4XqykPWK2`@40igAgNb( zVjY9eM$hv!!uEr$yvz2phwwX6Nsnt<+*L3#-a{7N0}nA4GAFa%L+cmP09z`=vvG_B zEkA&}HaCLZ7#lU}WuWv+nN0(o!1~rqd)?b5$@q#qr|Z%#v(|JtTUYu4er;?8WmjLL z3+*S+FaHU&m`8&~YT_CP4JyCdGIZLxfz|)wn``zJM}92F9diY}6c5rgsB1l4W`4M= zeKd8S#Il4!GdQ_xQ6P23f?rD82cOBTyF0;v4Fg2$ z9@ehpp|-zQbLLLZB061we#S#&my2F62S`!LbvOLTe}en`2bixf`Q#oB^##PVIf_jR zQ1Kphw?^LeX5H1~Is2&j-{dVa<#Ys8-8srgS^M>}^&ydMb5d)#p#$@5Eff;TId^JRQekiVZ4f@`mV|7-{ z8FWu%y6=GeGPH>2j{Fx%T8@eR0LnpMQdZuXu3F$nCrZ_6UTB8Xc)Vm9-vG(A;n$+= zeCl=h!7~0mTD9*+d3BiQNSy*CGCY{q`+l6H-gV@gj4#P zk~R-5=hIxGiE)&(#EmP7T0jZz+K~+2gP^DAN)w$WIUfT)bKNt@ANB@+?_lP@}r{MfhZnoY-i~jjI zJpQ`s(zK4k{7_)tE^u%4IfjpFUiG#1RXcsp2;V$?PVDg*x;1Q1XSsfQ?Z`k>x=y&D zHjil{AKy$U^}BZD4WE=;AgDC-Ll%{iBLKKf2Xr} zT7N*paIy?e^LqWL4Kys;*t&3j-BfK3#Ea?CSj+WiFNyUVtczKyc~{4Lnce&X#W?UE z3DcrcYGRc2<^&xwg1M-MiTG=nna-;~uj0ALX+J|9m9g{%2MOwABuVD6(aa}H9-|HQ z!=K)xl{dXZw-3LeGF+=HhyRu%i-%N!8$dqoI<)Gj5oeF>xx)nm1@Yh?mIX@O-*oxB zO8ZV)N7?rUY6?6N2A%t)eKF;>#Bzow`la3O4c+@kQFJCVbWFQ);K_YVmGtM#t8D+L zDR0MNSSgyRuLJ-ur6N%Q9k1a!*ZM4c{L|KT~*KU>ahV7pmLLQOn@evl8YaxnUMr5z6Ot^X*iyT0#&9s$L`QDD1>w4SH#*;y^=?wSvtQN4kC1N@hkqWxzttg_I3XzaLA~wbcWRnH7j^&2pcx zZMu_{F>?`O-3_ddhiNW|MAH;wC z(2OQco%y0GveNS1gv7m;Z)tR=al_|y!rcGz8qT>$GS|~-(E1jN)T`~v2WXHecNN#K zspY!38&`XmQRR7JePFxacg5!PIk}EV&K-g|7x^*bGpN z$BJm+j4hM%FYFEI38LHe9@Q;DXmEYs()#;EEPsZ7YplHL@Tj&TJt|^y|It3|h6>~1 zx*~B@wz&f}qvJh{AG-KqTw83`IMpx;lUhLj1AnMh=`C)ir7EqTxs*S z&N7^|mm+I$_WxQ_*3M!Q;!!)!hDE}Av<9otysF#D8&M>9zvt{14&L@JC-6{=Pd9rxphEw$vW zrtN$K)Oh9y50Ez^l)`1Qd0l6MQ^R&x$y*=;q*R}t>7aPW!=P<;8TXGQ-IKRAoMbA{ zYOhhV->qI=18j1YvL}x3^k19oc2pwOxQoR{LsYzLlMCq9QZSSPE>!qlN#=WPnhNwdamWk*O( zR-xX0U`*B?*O~#iKPnoyHDRDm{`~1HwdA1hpD|QQZc&BV=P?{ z2<$^7YPOu)P^BD|X*sgJDNYoyiYxfC8NA^DeWH@V{CQh=r>E)m zj8sBBjX^F#O)2#iyPO?^&C8wQ{WcntGUx9H>fREangb=V0!&jG&uxbsh@5QITVE+x)X@Ps?B;X_k z%s=|6w~_y)GXo;$g9v`XGApNfy=9lS>3`;NHRWC}Nd#DUj|Sa#n7{3zoI!lHOCZ60ude+xS|BtAw|HuiR0 zOUxi7GrAbf0B8;LxYaIlgn*gWmYP+t*Htj~u~G*XJKB;C+?OB4Lwx5_!k9zW8Im|l zCpY_=)QgOk^TS(@ureVYLp~SBNX2!pwOgl(uNHwcWWft&nB^zOOU&pAEFOHEDLW2+ zDS5f#9Pi$eUG1a6G?(-u__0G5_0YjAJK4823Gn)k{EXycdE&dGkpDW$PBbSI$+Cdq zlf|mPA)$WL^O-}C)FRkP*cVnqj_Dl>pr4i1Bc)(T&@69NSsCjaBX~}Y4SHVeBhysP zr)=UeZpE`Oa=1JS8S|hV8Y_OGj@x!)c5qRz|hMoMh*583R2uHer{r(9qW?q zsnEsZv;piNfS_qTSJhO*)KebvwcLmUca`4PVya228TizF4k*ZQz0bn(4|UHtHx;<3 zSH(RFveNAL9&9p3NW+86Xru2N2Odcw05+KNNYYNG92QNnb(P`C&v1n%>Rou;YS)_9 z$}$~1n}S+U?FNPW&~Y5gH%^}IjBUyLRDvE5L3+ zVesXu(NKRTL77!EIY)hYsExviq z8l;S0Tq|1kK%e6HB*2xJVTw8Exz!3@^T8fp9N%{ST)$_J!O^e3{46e6%h}GoQng;N zeAO0(II9EWBNo5Eo4$DrN9I#JahuG)_yUxQN7<$;-@Y)!Df;xa>E>Q#mddMyV(T*N zVk5qSA6o(YB2S%V&qH-_59m(#$Xp9Nq{NH&A&btX>Gki!{gb_ceo3L?8;KIpu*=7W zZ~b}FJpK?f)c42oYyLGdAD8Cv({o(M@EXlAD;b{*|52=N&U7aIKG7rvr9S4~J#z~P z5%pUEi1fV4DY5)rUQ@2QqpmS-t4c5dXf#+t1^6$ZX*Pvu`?6z+S`Oo5E?wqfN z4mMxam8rTpvL;G|U~_M8{mk*fS|j|8)bARJENLD3VO9;As#ne-XARINd6soWu17hl zaST2$K)-%0-@8La`sJrfO07Hz!}!j}$XwHdgM0maxy*7_CVl^?zN4 zzuF;hJrOAP7EJMR9YO=;JmI|0f{75zCm_Ovdm%lshHhALp*m0|Tpq-%XHY_Gy>i&A-Tp8AF) zkrILl$+r|b^PZPStf4Rws$}3r1TCUd<{%$$pBNz2ye@;PL!4mACuKjhAxk$yiVshz zw-O^!&F3Tg=K`4YOn0n5c`Rfx%B|(+Ax!gh3p{_(4?P69UB;r8mTKoOzJ=g{bI5gf zmq9A{1(xU!LWX^Q90n%Gpgb0t3@SXNvR6H{+2dOuCuGFx<}(Bewwh1^OaT;lJiAUMW%d7^9YYG@)`@268_+Ybkz}ysT1L z-0yPP+5@YMWMJrAwszy`^s>^#+ z*!ILbsGrXrG2T=?s6H?$%YwdqYU4RtpYLzUMi_XG%1f-@%&D-`rYKX9{bI4TADjCx zVN0ghQCt7gFS0rHmrER#oE1rM+1sr*#7AUy$`erE}MN zXTY9tvLO756R{kZ|ND8QB zMHUXN#ZO6(Hz`il`Gz~M`>xH#1fNk7KW@yTSLl30uU@HFy6>Mx`hnLb{^71pIomp8J^cW!(L9lI)4{gxH!)OLc;Acb5qd(Q+(KwoSp4NnpOp`0`BV zm?J0|sJNfIJixA906V{7aw*w>tB{Eruar4n)%)E4j?&U|^_z_6O8&xFC!Zw6_k+;{ za?-hG6XGxZ7}c)X2@TF^StJrW@JMODYxa zb!{m(gm7JZW#?0paS2IQX2!in##Py5WLzXva*@4P_9mAruD!2)FD{p#?;pRv-~YcK z=bYDg9@;#9T}5k^+}bBMap&j9`l2jOsVj&Pxzm`_Uq?^v+kycFc8)In>QdbnC@LwR^Q0=hdYHO`i$b5?!L^VS6^a zM+i^zgMqq8q9TI{{o;G&ks$@$;U=K{W50<|*AIr=cAtX4lAcOuLkfn*K>7UnFockS ze>jnt5IMM}X;F5WRSp_!{x8vZ?I5{QLUjc2OHNHGS+((ldU0XI=FtgDs7vY8@XTxe z!M+OGQ#&qIvb+SnLxPc2Rnoo^OSr4C?CYDCMC|_Bn-SF{qshQo6KsR>)vmY$F2Ned zItX7{b9&dRQeYk#Q#WOnu3!V#-L)J+rdF(Se&1ZorTpp6N8U&F=YIBm9>ThZO-&uj z9{O2o67-`pHQPT4;A&XZi^{KKsO&l8c{>amB~P+!lcR z(K`G*w~56FUo1;vbM?CXTzXRNf~Su)7?HuWp5mnKeCC{wcb1^ zq7Zt#qI9Rma0!*+Zl4`dm%><@ut);(EDlzN$XCI!(HIj}A9G$8V?fpY;naW9jqen& zj9dPLg~hsLta`Ved=u4rP2(!$=z*&-FPF*6y~5^fhmIahbUIrzZ0KL;zz9dYWAk%t zCTdMU*M?1^41VNoNBfk&IAML)Zj5JNOvuB_4uXB;V`^ga`eRi zTbb60{cEz9LwHSaS!W23a@MD>m`!mN-dMoI>+WoLMS&=64#{~4yComZ5rm+X2ekIp zwN=s=!Nh3G3U*aW)-qF^w=0pew|#8cHHOHc2jV0fSeZoWYRoszHOt?VctBT@AAHr64beCxxp*|uTEeloLq9BD>i5 z5!VFvYZr45R;%a_*Tl)E;0|i@O0TGPyDjnZ&g_^%@ zpzx2xi7&~=9vSD^L1B_uAySN8Jd}(gvgN9n?H3GYbJ*xNrZxs_?n7L&}zV$iOM$b{;QMHVZl#(_2kb=DuGKWW_WUvA!9z9D0aM zF#eL}YcfmcnJlEQesj4%-{0}}W@+|!2A@&v7VVIv1I%F%hEDSdJ9tq9@KG^Uo<>eU z%rAc1GcOxXajs;^(oP7)M_edxGLNqXk{<=M(Pr&;jI$~Q?6k+t!`Ep*i~Xuc!CXjz z=R1vcVesSjV81^=4*>dN1k`vz|7I_(>QaN<7#0BTMY*#90mi|CXD^ghC`%-+i*L$) zr;Qh#86Sf8Nz)gr!N(*>&_&kRSnF!=R+k%^+4b_;m^L5EWEL>nc6o2Vd4HI0*$NT5 zp0_d(F`67x?B$_*GB&WYF4(VlP+JuPwHGzD-Vk=(kT5j&0^hqZUkUe_yv&y5VIbJf zylWoDTtViRgRG7@1j^qT#M`6gnSX-Xw|e?vSE;Bj4s|tD|I1K^j_*F8ZzjBF_0g)n zp1(Kzb>LO=lv(&o+|5&!F<%@LOt)L^#8tql5Lx?7LSt;miy1j~2N-IR{>qBJUC_2t;UiR5sX^}_RdlKfSVOOR6FT9&TgthQ zHZJ*uwAN|OApF8xNdOVkknGP-W;}hONJ9?i5YHMV)Njbi-xHN{4H_pMag5V? zoV&wvt6Is(hh4Q!(b8iBv?OA2nDdUJ+f6L~{r9BuLTKJMaNKLi0UGXHj%G|8_kSy7 zMKk=lO0YNbaLep%>^E}ovFu5~k`~oFcdosjd1UMosx(vc zWv&iUXpr#vVVC6Gmq20f5_o)j9p@;?(MqH{)hz20fx~o)NbzV;r=bY?74Gl_?`H(P ztg=f9p0NG+n|Dcku{g|Ix4**FQSCGY;f&~6=MXbXa`^;-0EQwNoKlf|-1ifu9A1%pM zz(up*bWc~(UTv>42Oa91BRpngZ1<0o8Uo!OTLWw~h6-~4~S3m;x{`IkW>ES%BqTkGOq!dT6561btl){>WiUo zV{hy4J}H5-&7ZC!aa`R>X^Dqk15dNe7}tIRA1+#F94FITz8eg@zbGnP?`?3p?)V8@ z#lXwf_|a1Ora5ZChz7ltnO%eO8m|wEZwOqB(Ib~4r8hj49(Ls*HMzkA8{lOTcaV1> zFOR-0p$)uI3ltgT0*?F*n)TAySSJf13Acu^w`eQZ-z2GMt;U4YC}j=J_S|s=hU-sQ z!_9Z9p7yp}tn^_K0=F{Q&y3pIYw{q}s_;@259k&xvUE|2vTMioC;Qy;F;C^2Rd8Iz zEDEdw?9Ua307LHqrWdBsA883qhUQz+mkjC>ubMSnWC#7I;z|$vH8SDBliVLAec9PwvFLbE`H&9fPTrAGVdySj>N#e$_f% zHlMn?*uAjJs{H;(EOV^hS!#PX)ZJy6G+YsCciYrw+sM8u$*0{CL$x@W>H}J)^B1`Y zKPdIt8C?ndcct_hHMQjpcu zYx9%x8T+(LV@EC=>k!H~@$xWL1%w_MCye*`>T}cJbb+~PP~1~??Q?S!VkmKy*yyOc zbxC1?_hunGW(QFuzqUR<8#Ws}-KcWUCbmGm#6L}WZawwiyy8~d@exXO#)xSEwxQ%| zX~Ce(zlkj#)0&WKJ9#zxDFH8;SavK6Le5bUd$o-wp8oZ(q)jr1kV{MWuIDG=>$>B% zZ~yyh0pkv{%cy%qfF?SN7=h}z$M6uzr>mQJaiavu!)`2lc9 zXheb&hbzVu8;6}Vi`Sib&_a4-R+*Iwp@Hfi+Lc-O4( z?t}Cz7YvK{JhmdBqvq=Upu$6c1>a6C&0vJuMd!1%KLHM0y7@yJ>Q_w27(`fjlL>F0 z_i4a#t5(3Y;x&tZ@d&lc#H4DM4Nj%T=ehsn_h28v3ZmiUyDPaLUo6uFF$GV5flee1 za#tLDwp6zj+Z6(;&gyml?ms!3u2R{&cF3GM4;Bnl$zi_4#WPWbRxS9S&K!;*wkPnm z2rG-Q zb&mhA;m0CJ%T8hE5gG3m74;>U$KRD$`0|dY%c{(tzJe?NgxnrmGq_NCbZPRSzIa+v zG+N8%K`hQ3qgh>=T^3723S>7)@R2I|Mq%^7G1IiN z>jMf;y=oy;SO2!@&5#|Q)CtuKU7FdTQZf76_BiZ@>}DC;TEB>qyeW_lk_J@?PPe-g zzaGNY+ZeC$nUN;BWk3CGFoql82xF@Lzf=06F%8N~SLWfuJnmuUO z$X<2DYdh`1$CDcT;mw|2s`5n{e&FQXAQxY9WF6xnb$;cb!`EK!h7C4-l%X{y!2x&A ztGAZRu;tVCQWjZN#X@g))wuBe#5{gk%c7*&1;NtjXZ^Ba=yg( z8Y!)HL;}6VvOPy~2?pJl_REiv=!e5$((oo(+IsLq++g%++2po`bpvpY4?(N`#QIBs zjB9ufMJIhY?EI<;9Ig{y1m~@Am6L4~$+aLDo%5HhE8;ZK2g|y$(2XSUsqd377To(% zX|&HV;ggp6=fJ8l38qTc*HhyoT#{h9-qO_NN70lwYoGIVAwriC45@NKSx<5zzb*W& z5d7nsbCS5$OFKYYbyG^+Yd{M;Q=73++23`|PvH7w2_@^no1^$Ac|@5g&m2NfXJw9$gMb@$*F3|{x z6a^qV&o)EUr0^5j(;Z9q3cNmFU_S4=)PCVZdjp{=f~efxQMW97I}vMvkfd z>ZNn*I8lhfE?w^hJ`o^!C4RjoB%XkN>tOxsOb^jT1(#bo-;6vH+_1Mknx)|=blbM& z>0IPfy>LMs|8%#1b@zKL&%(dtfUmvQAoe>9Ymi%Vlsf0OGx367&O>XtS)nHl8k-H7 z`(7|Y8UWMLh!0u`VhHY%le%_fZ9OH{r{qg(4&G`BYUSB$V8rGwPP{ z;-IWiW6OoC-#b!Yc>W`tvGeVww+1qFG%`@ioC)Z7VU@LZ3NYBlptbP_JRqa$DoDz* z{dHIgGABX1(dzdILvk3uI(OtZh1}wRq{%S1o}Gq_M+$p?OeTOJg1W8r}9@}#_)zbdWS9PD9@5Ws~z8_7iax)JZdjvr@S6Z z5yeWF6j#(L!XPlukZbAe>uXW2j}E)QZ#t8Q-f;Xqi9dU?u!c|l#Z+6s%d4DwP8DCL zzWce2_G@mdAeJ@@B~U2U^PKN)<%UM75dn%67_UNY6s~=aCTre!jx}Cjr!3#O4lHs` ziDn?PEAkm-uRR##d_xnm#XA-rlm}b<>+^4niY=FJqDd zIMLx~Ps~e$%VK{|f~`H)`{2Q?Wy{r?n`>OSTz!@NquHBELMlFk0{!aad!6;z7v(-0 z=h-`^E0k_^w=N^bTFdH^f6AwJk-N3oYhM(|Pi9v*eR)LQ1E@At7@}%ib1vhiIXu0- zF^LG=1PQ{eQ_NfYt_t0DQ*t8quu5@+wxHvFmRS+3x_pb#AlsHIx; zE)k|OS?jx(t8xA`$md1H4w#9UE0yOM)m+*EP&LP^@>sfZm$vgwln=p^G{;*HaidoD3w8P6&0$uE7 zGsI)u2PA1tZsMx+GwTj*$Cxe^uop#f#pEB3YKS6@^xbEUe(K@6C7@E+7|NA%ohT`a zr999(28XaP@-BTdn$_iJuwF+wId;QRaUe;l9W5Sl5{GZ=Hk9)XYOsq5y!8S9AXw3H z(OqdxciPE$>F>~pVM_^^QR$8pKkV4!c4NeBXGgBov)DRY#vs#$?fR$X~B}l zAzO44MCXP2Qa;OxQ|D}$l;QJctz66M*ks!Q)E1djBVg;`embW!Cb?(X8k5UKsNwPK z<_A82NJkR7ghfh>@B)&UtfaO7)lr_3^uJ~vbq8X3q0J1J1p<<=`sO~cc4v_)xDA@WVy>GFLXU^zyY^;Rw>GhKe5bV086ur+b z@Drq_8C|OE$s&Kw*Za8*->@qN&)Q?%*f^k72;TtBhgJIyXddP8)4qfrY;|-Y=n%i* zwKqG~X*&gn&-d{HC!(8x;9p-ayKLsevJR_aD#riHnCN6FT0cA8O4W753sh98U-2|8 z@zvvb?0X^b-<3ZbBH<^1;HqC$SXq6Vd_CznXBB_j+oJ_Ww0+Or)L{M*P7`7}UMw^R z>&)FAG6&nAO<;UhQZwc8b$UyEivB#<0t3-^3&6NX4w(+M*#Ua_o7h5$PDcxUMW*Ln z8hzn!rOFkm*&{8n15&*-M;mD;?{=qlPMdB7b|@j{9fJxEsg7L8X}+h~Lx>g9Oz!_Q zay$uuV}eEG5+PR=}_5lN?8d`Q)z32Q$!K_21Fn zMv#RGJNdDBE^Fr`T~i6VcxheHF5`6RLH;NEHqwAcQLyjgEgJXSDHSLwWE$Kl=VTN2 zTFjFVNaIhnf9IolKu&%J#JB_r4ju5TNe4U{2M>_Eln62hTSq9 zk-UOXjjDbLe$&qn?AU)?)&j6Y1lcexql&BkwBsvw?GRt_FA?Gza;mri_925&Q{>V$$Bvsqv#=$ z7HkeNv@10B<>||+CMNzkz6;gK&M;(A^qG8Ni=q#ItK<*L9<3{N-*(&|jC=gWU|;og z2F4n4f@o#nE|0QN_r~eqN&-V=|3i(N$|O1R=1^|k-rNs6k=(V0%7s$9cDRL<9(Ap) zgj%dx_QZU5I!anWw%0|O-t-@rK4r_qz89PpsnKwdWu5xx6H7Juv+4W25_KF$RjE6T zy-c9y+)kt$V{O!Lh)z2QuXm@(5Qvm`A^X8aYg!nmIIV9P_y&+O5Yk}zrAh3gX?24Q z&hmsLL|HRqOjhiHdHyP3@mXP~MQ!f!Vy`cOgL^r4{Xanx0?fT(qwDhM9YkX+HTSQc0~bEXjjpls!KB1HK_PS^o~*9MT}_ z)wyDy8R#)GWx?I1S&X>_9zJ#f;_hq0CN<`qTCFB+PjhOotMz#URRV=;X zBC6O7E3cJ23Xs2<;a-lQ!;2qol!Km;h;|UWN0IWgs~r5zkSpJW+&7{nkY(3@%%lQ_ zuY9LGPTy5ewu^q49~%$(6j&6P0H=v8!uV(cH-Qrz{Ha*xrf~sCj+^&>4waqQz3Hl? zf~<9STW*y;&8n%!0++U4tqrMYB1ePY9L_|*AEM4>K) z>RTJPZh2B6x?i&FiY(*vL2GfxJO?s_VrAo?o{h}UqG`Ihuq2R(@XE2KcT@u ze$JABjITY*9146c)1YL}ar^k?jzz-L=4eFs9=<`dlGP_7lQY4*5wk?06+H?n=h;E5 zsd}y4VH+nhZ*}Vj?Yv8%ogexqvS+u-Mfp;uUyqNAKDyS#m38ua=aBjwg;vxSJgo~3 zf8*Rnj|_pjLS4WA67DW^^9zB-ld^raVktDq;%KBDxj5C2VzX|awO;`cu*2gM4KIe> zvkKerd#Mjcvzm3Q2`87el&iM{!)pJ5J`Br+NKK}c0SffH`k79{PGzBi2-Y_<>VOrh z39Azn4Dq)V3=4a%#_OH0hqB1X3} z6!+I4qCKe)K@Azg5B%0D#u(kVj=52t=};!=P#Dzt7v(6;)X4JJM9QiW{J@nl8IUe_ z@F5A6X$th%7smGsJOtfBBc$;Ubsln`cRRWsy;-rs#$`!My?T!7K~d@;pGjVC(`oy+ zzo|41zF2Gw-wiO*fFaw@1yB_;m$6O-EW!xpu<~g4t>@|&OZEC}_D%X;?`%dBwDKkH zqi1nt;MNY{W}c#FhQzhiYg1LUl_v%z3YEX@ARDSfk zorJ$3#dSbyF`Jtlg(CNJRB2#Vm)MV925J0Jx*-Zz1N!KYp-pUq2;8L+eyCDq664de zA@1_LNpo`M9M#|k7kIOmju|NBcul_HI+1Lps6KxCCEMz-d~od$Wz7B^p(o%6h5JYL zPXy82H6frB+O&yFtBs>2ocnyYBsCdFsCBOa->Dty?h&+kcn-TnOSwMqrK*2sR3xxu z%5D7|1nW_$LQG3^#T)Ai`J+6{tP-*=xfdyef9;>#7$MUvAAo{7c<{dADQnN7GD+H_ zn`ovFdcRrOoi=9MTSjbWf6uG3`36uwmLYN2CGP3Ng*L-vQO5wQH0iXAe_gqhL+W)L z^?QcEuk}In8}8Ij6ZMH#_Y<(P7{Apn(Z`mr4y>>_00ouZzgK=*w>4H7tx>!_yvg_B zSV@d;6vINI_hL8Wy{OiSU&paZ&lmDphq-8wmz+J}E$jv;J@IHzceXu41hQNtQvL#e z*V$j~xB&LcdGARnc7*zIMYw@b-`WrnpZ}X-`1Dm&P;T7bH{IT1E7JmB9v$B?G7Be7 zEa>zf@y%GGd-Kzs5uC|dys>yi2$NBC(9=NpPf(}`mKsiG;U^IY)@j&~!MMH6QWbpag*yM7^Lj=1BM zg*_^wcAJP>^ZJ;P+eH`XbZXk7`eKXp3s0o2#-C?Q43Nyko5)Kf)8Kt{1Lf$VI`|i= zuzB(HLc3a-$Pj!=;Fj5BRzC9j)<~89iJeQrE0pGvz{n21xF2+w8mQ0c(!P|W|GPwYMfLroa?3egiP-|noMO%-t}o@ut7_$RM)rfK5nZhU zeOEns>lUzj5M%#l`UFF3GLbs1Pz-oH74>+3Hbm@63s>T>i*$eu^H|jMtdp4vOhJIQ zze<#WeO7>C|NhW4wp{Qe`lpD3VbH;$CLd@?=ByygL9Zieww*?mzzqcQ`L}Gf3!fwl zC{brTJrq*O&bXc;E_2W;ZqywU-#S|ZI88wVns!w}yrWoYWjp0>jdIn9LBJ>m2ig0o zpdii-#U)ex>{`KWOMqBcYV6E|^#bkvJvq5l(+ah{<(LjvM@W~~aqyYsRdW8%$-90W z@G2GAmf0CUMgclzvd`CmW%KLF5QR!t@L5SfWZis%a~e}l><~Nmg&Qep@RgU`6FXXK zQiVisxVw9iaID1IFsYD_dp1aOJmz|7ImD_R@TOYO*Ys{Kx1z;rcL+n!2AAt1e`^6Z za(?BD9gqe<7KDz$A3+CXuHsV3@^JZdRA#7@XyD~7Wk7F~ZlN~3IBekg(n8|*f*bwv z^s~LsBR=>>7qnC%M=K%}#rjg=&^hIyDU}9Q=tm*sHZv+l#f5OjU;j#C#p#xCvz?LxH zf|5xFe%JFf+J!mMOJ2}h8R(!0aei&(lgPzBT~J3WpY@AK*Hc%y7WFmksY`d@9lN}v zDE37fHQ2!wJEoI{;vFS9;@hFM)VG|zhS3N{t_uDnwP@wx33~gy}XMK?Cl3Cvmm=RnxC&>!CU2kauGIL z2G&q04${40`}FKSG%HEH)Rci^;^#O>MsID7nLpRfv0g7f34qVz=h(=1v(_T^4R)wg zM*S7AwBx`h9=|kiFWE3-eg?X8AmWjs0!yNtbUtVUDUMws59^Q0X(4JnoM z_7q@FmEq>!o3SQXScquAI_)ksVYKXKBk)kB!X8)Px|i1{K;kaQu1c1trNandwEH@2$snXW4eNi4{p?gc|^&)zUW6r5!vV} zKy_eu@*PTt-U$wMZUGIUYJ%x?aPgHm0qQ(1fzCe+6y4OaL~LHHljJ-Vj~rriBpBtF zZ@X?mueYW0#>tk~&E!U1*tjM*|3nqx)myqnSmjMqOnQ50O;XeuaC%S7PU1B!UgI;$ zPE%i+UiSSC$au*p`1D7V(mQaDr+z=obS|a7+`i>I5St2f(R#BxoA2}@k5r;gETx(m zxJ=_vcNN$;x7BY(tFMTU#_mnZYG2_LL(T??+y{3kC2}1pL3zplEu1-^%@))LLyvEG z0lZjAYsa@QA5VqPr%r@4UQk5#jszBUmy}{)N-Dg!q3=euIuXrVyjF?G8S&KEF?7yG zrl{{S^P5TQ*W&X^*JyCQxI5)M705>qg7z&U8_aJGSCb%VQgtFJ#?vFft!YfTmRVxy zec0+*U#7PIGyuN3zqF3U(We7l{)4Ss2}H#IJ)x`pxTL)JsR8o9(G306$OjArmi%OP zO6cN=FQ{;@K)Eu6=Np_775rDJ-*|7&_P=5FT86q*Omism0r{004V4i*xaOps3`&ONC%C;SxD2f;NZrrVV_sKUpc>jzgKdQ0yB4Q8?! zO27F`3O^u-FDW{xrO@_D8PyKur9-07o-*uJ`9r(kd&Kgg5f1LDh3`CbElcCmV1kCiNWGahgmz{f$q>?Xc7w@Z)x4LjwD z%meTtV{5I^@rMv>T2_X7ztAzy8{=$GTuRIStsfNG{kd<^J-cFIv|~HMwEI*tCs5}- z{maqxGbjgiltyN(izGjF>r`o*#l&+3HQk+VinUzPC8}u?Sptv#)RkB~TyN=vn3)t`WZU`%>YCYqLnk!M*PgV3b0898!w)P@6;o*flOHSNxbM0(?XE z`wxa4rouWc-?L`+G4_m^6|@kPKRrmw(MOx5TB>c>D@9}G@XPT1A}oZ+iPF(78B%gJ zE2+;0jBri9{IC{WVpsP{S1FonBT}lr1jAi;4>qMv5# zI~PnAc({}Gzm+jIE*yEql33%G;hFRO6CIqO0v*9YlKZX^{z%N|J*G+Ca>hsajh=YW zz!oMu&-+c0aiKF}A;r6P(pb6o-X&Q-72ujMRAT zdv&hd`vVLQG3AWlYu(Inid2JzZJ%I1vP5-bNY@#k|4;C^ij`*c*4LD8nRa(zj5pG* z+&04Dt0pDJ25X>6W(p@u9D(0Ll_Ry{v?{_~p@8usox!r-ZMOS0 zFE~W02VEtTMK2FGKjgdqX-cd;AlicM+HdP_DUo!DL3)z=gm~L5tAs3iP0nSw>G?Qn zJN)F`R8%~SjgTs&`*n+!|2`Dj{rkg;o@{pH%p-yP#^Y!}`SpD3TzKUcjgzC2^;bde z5Elh7gUMx05w*`EKa$!UL)dEfy0b|8(1H2~n0zfI*w~Lz*L^lbHe2x3g<&Ql$CbHE za;Gn11`@u;{ZcxXi*d~jR;6Q{O`2a={xSe&`|QE40YW!9W*X=sZo#j(y#mXPxO#2r z-MKIaAV*^|<6qbQv7j)R#Ii=ay$rvL+B|fSLz@*Vs7%pQ1(nX-5I{VSMXvSAtZr0Suc z3uLbeaTQ&&DtCkk*OTLlpuFGcE;Mk1woUuL{48em=cAb8X-8g~H^^qKz^2HT{_DRK z#ZfAx=9AUckaAqhIoNvuj+>)3;jR{Zq&*lX!j z;M{x-`zhNY&;ydCa9gf7+o$!bzG_{-my&3bUDTsephKxd)}{DpRK(3^Maef`vA$F4 zcIg^ED9F_9mHu}b0~O6+Uw9x2nJW6&KH_Fv)%4&n!eZ@@WpkRdb=2Pboa^<6e_GH=OX@i1=}*HE zWFDWpUz>DWJHFrKM;mU+tmpoueHIV(>)zqH^J75$&n`%mXI`@@LM(XYyG zQM_d!q*L(SI4KS1>0OmU8TRwsxR4ZP7iqyC2YiT#`1YGh>>N`Y(Ewkocw>B8ldDp` zJp5T|hgJK!MaDDSptSP5$k^M_N2LR)7%-c}eqJ=h?qI(HmE4Y1F9RkUzVK10X3uG^ z8;sg{3nHFB#^+SiftvpLsV(e^0=l)8RUXdnBSUL?Rjh-f@|eMkDeSlDj5SwYK!@q? zwtoYVvg%BVH4o6_r>V(bfetN3@R#D`gDvI~`TlvXMtH0hTsJCaJ=I+oIK+NPe|XpS z$MQD}xq|XDtCI7L&S@i0@`w;go$1`mACh|;UUzXnD z+11B`ri}Pv@t4Z#23O-`+ht?_mfSYOE4Wj2Mle|MdHb+UK^k?L-OGWZcMk&4Pp;Yxx@HA|Mzu28<2z=DG*lDM~SJvlnrPps!*B{EG)ROwHm zp=TGh?A6+)S3&dIg2urrYKR#6W^a5M=YIN~=C*7eDvy%ZRNYk=Ulb__?TNpZj#e@3 zJwiR{IsNZgIiXyso1I?FIj~;X(%Yx)FPr)qgN=w2j`^0MA~u+T3A5K)`|eRFi@g}; zs75sf1A;t{H0JG&J&qOFO5Y-7c-?}1G&tG2fg+zsk&`UddAlxKKQ_X3p64P%x5#>< zS)hZRFGid%QXHY3j{@n}H?BDyYNN$C8M=jI;yo`f)c54)rUFNeJZY23^;BpPYFd9kL7Fu|PcFve8JvBp4X zcAsvmsss*H+-B(g$2gLIEkN1){!6=5*`$1nEp z+o1>i7sJbe-7HZ{VT9a7CFi>TCns|J4K3c2nv6Q2|Dz_pk){yVxX6`8~^b7X+Y}VphmqfQ#$x*?e zSL!kIhZ+)i{K}fD`~*w)Jt2IJf=Q&acTSNPOE_z44T%%FCS?50iRrd8RNM{UEmK^~ zusu+CNh1=u58OYDIXh+doy`Lz_bYFFd^zY;V1{L{^KbToKGk~QkPQo5SXzu!=LWlA ziT8aOhiH}5NUb^&BW77UdtKr6koX%fl{SChDmM^Vs`Jk+IeJ#d+U5Q{PGy_3RKoYy z@)tHgxNnd^@!*4^d(=K)X%E<;YEpldQSDlgY(wq6<+`Jlc>2qQCPTeAJw-KLDvkG1{ZZFJz_qz%y z;ssnnJgz@9J$~*ei7#W@K9})=kJifyEA9Ui2OpX*>MpozuU0RP?cDctYq`1Y{LP7r z1MajXBm+7^Kj;23rVkAe|=h2!a9 zo!gxguIWfSyZw0p%r3$8PKVEzacCEtQ|xat?}_fa*%quM+j{hnL#(y*TvK9`pK@3% zC;&DfqfF{BD+N(75JhV%JM0dp8gfNfUX2_HjW5oVN)*B8S5aRD{NY>4hy6zRDlX~4 z&L;z37cD;cFdk}jEKv#)(5oFC%~kF-e|YL>&5UuxPiyaTrw(nV`vMf>q{g{}7JiAC ze7IPVNiML2>AkKEuV7{OX0!0X`Im?M_=yI=a-{7NXUwOergsPIC=c_KiRZ+LYJW4l(t9Qaq%`ozdWVB;qh+ z1z4}3&rR#l)BaDV$*Duvi}27P?TUFJQB@5%Hjn{Tfo*2&b6rZ<=P0g&OV{URp;l;C z8NS0QaCb9$=l`e3t|nmdu|N~YdFSN_d2ulZuct8hKR*oxo!eChuLRs^1S)LBA}D-b?zW|duRs6g=<0b-Q?>O+5Tpbxm+XZ zmqwqtywkI-Wabh%p&&r=!#1*3;5;;wF#8obxc5iEd7jBnfadeHJ(C;r$=o9@BLTySpWhMONE67D4{f98c=+%qNDs z;h1GBj_;_zj|*71H!HaLWVf)?z-#Yip7;=YAHQ(2>V;GdOmtI=qvVM)Y zyf9BYpUv!kSStH|BnNMf8jx$bwMLyGYIxQ$Hhy4um+$)c4Cq?)LpCF2gRJ8d=e=nx zYUicsX1kZ_fE;@Dul{j+Ps-F!|oMnRMS-a%^AAK zE70MITFAblW2D)&W0r?wbjmDypU&UI+?xfx1^%w&p_{aL_$JiXXZ{m%9glde6I8lN zx5RP{$j&i&>5x%FTOwO3*XF-P8;u4i-y+Q823(L|qx6>+yq4r=sS#QA1(~}~99(U` z-h3^wJ}?v^%XLI{D99p+3_efyQnI-x35fdN!Lf00s{7O=>K`k;PA2umiJ`A(EppD=U_hi>`v(O_`rr z`h(cVV#kC-4g-%~gs=oXXuD(Ne@G1%2pRO}9xgoBdEBCn=E$y_p*`7v>SiJpRe4M7 zL;&j(amC+hv4yO{+dX4w$1ec}*RoJ*2f)MS$u%+32I33R zhQSWmEui#OKc*H-XIU^z8hd>s%IcAKOYVPat!EvU53vjrt;kUI$2>W+&VPl_wM&%b zK*$j^_${5aVJ+5l+rjVVp<_2@XV9&*nGSn+BYd0uxl~D-usFZ0a%Yi!MxOCOSDZuq z`a$#>bI}r!YFVx8A($&+6@HJSgh}Qhf&_LF#1zwH?D6M$DJq0l1q&ZqJHt)oWD#P- za^XBHFG4GVdZ5;+$+!Y9LPk7DenSDPv9UDWtbc1`ZVYnn8nj~COCWqoh%q|O(;s&; z?!C58L!oBN+Tg;`tIhU?RTFf+sJsGv47bGYFKCZrQ=sa^uP!_F;Az!4W;ttedF(6K zdpeFtmLaXH;7{di(MHq#&pux~`^@F3%K2K4cy++Bd82emo*~U3s-%!tv2$K8`$$IB z8vE?Oy4`E=|N3z$7*U+ukoqP|E6PoBil|>Vcf9zPz`bM15KrPse7HbN1tgLJQXhJk z0zw#!ndk{>R~im!?cw~Z?m=YVQww*&!4~GZ&H0@Qj;7!Enp6Qa5qutvFH%2|9zx11 z3@hiHzrQHl)Aw+Cc`;tw(vYR7h5K7WHz~5`#dAld*eVaD9RJU&jRsO_zEh4;IW7I5 zAkZErJ(jYoLRMJR*cCEF@W8nyH;E5piED~Z$`#nJR_3-cr+4`cP$As;*j@5rNCvPs zmeOtWuiQ5&+%AADF7++&gv7j_RjmaqZ_W&At&Sk3)>u2`1PX|aHF6LQfmJQSTOK~F za!ZXI0_UCm&gjlG+R1E>^Oc{~(~yX?K&tcZxDfD07o@;_*I_h>1Cj4~VIc+dR#UO+ zz6Leg<2kcFv?0GWg74(1XvN0J19YA+U(oGqEq)DqMD>3>{bwK>{`-av+oMX1lByA_#<#86+7h#<*=maxp>|QN zRV$%rQMGE%(AFwNt=h3_Z))!qD@G(jh}`)-&;7q&=S^PabLG0u^EiAJjW52WG@@=r zgEL`FPIGEdd)}%-p)PQ>s_)y-*duLU#a-PxxQklv1U?FZd+AhYqGZG5Pzgqa(9+FP z+31hTAI?^S)6K%vQnhq2D$u_p65hjZ#m~M%Ori6#dt%1Qb>oqR&92^p5FaU$rK13* z1|Jb9y_Klqs|~`kC#R5)7m0pzx#5GCQ3X&7W4?vy^iG{p9@$C|G`pL`t$W!PdXb-p zNRMgwQ|-5Uh4L330f`JMTkW-`T%gj(f+$KTf=CDPMOL#|3ddZOUVY&0mbRa06)om< zi157a*`})${P}?iK)qbme+-0a{ddg!^WHb66UvjYqddh`#(qUl0cwFid-lbKfzRfgxhIOu$jm~}e78S$^5m3H2NRN8K zec#P0Y1Ee`P%P*Dvjr(ywAL~`y6`HZ7Y!&~H^U~XPZPEq%ZDu?RZg z4u;`xY2wMR#FIx3euz7bgHCU#W2{bSSFd!Ex_;wahBv8Wv_l4?jkoIvZFtI18D>(8 z40Cpmv1#{t8=+W>djjG-r?+%-_r?)+*1l{!tuS+ZZ65`8%ePr_ zcxH4t@IIMUBg%!UZ&Oi(jcHJ_g0D3CF0~O-#;1ac>vrgQpG)&!qt3nrA8h}T{7J{jS4!ujKA7Od*^G;kD08= zcodOmOrowQDTxfj>;@f2>0ajatOJW7LQ5ehedryrkP|fMEa<8m-9zdikCz1q!YrAH zIZe8n_q;ljT5i3(s)(p6hs!8zksvEX1nGF5v>!tL;0i~gS2$64gG9w&R(O&IL-ruc z&PVarTEWDRBrK#w^ldcb!rwTK%dU-&6qk}+R5 z06Y}^6lO@2MytLVaEJe3)9g(hklm*9FpRYqxq16n5)!=nKbU>is$~;~Z>?l-?+}wcwobvhSvb7T$&LfzJR@~fpTW8;F4l;rD!SBU*h8t?pawE%s0eH9H%f(cR6LRJejKRNRo00?E^Sa5Ui$xR;?|cb=*D8e0$eA{XVWIpMF=iNJ|C@V>eW zz3HPWX7orQIZ#nXI~&Vy%i;;W&-S$r)A&haSO{P`(%`u4$zF612DWf?6(ikt1*QA& zs$ZT(weH7svz`P^Up_8ZUIF^tr^;))swF4$?slBOX^MNo8w)PY`7~h?DXdLz?%Eu1 z_UDtfN0uPlBVGD#NKufI+Cui*a=ZM+5q%PCfk)ZEe0+@Jc0BV-u3Xd2&KR|?e{hyv z#Kg3Vt3E%QEzGy&iq4;6r+-0BSC6QC@83HWed@fHY(K~rSzOdCXd|t`dw;ccAuTd` z?~py@7BFZ`f}r=U;OT_SbzdU8&GhRTpjssl%#fEz8OYO7#QaliQ}>qNdfs*V#Hfq$ zW_hFDc6%1Fcg5c$6IQJ+n=gYOU<1!=Ovoy3wX1A-bh)wM?uKOqNGk6|592jS;$$g7 z@(Bc^0TWTy=oJA3u~Thm^seoZ16W?TcrUji*7Hk-Uj&f@UnDq3v9;T7=<4XJY@$AC z)comaU}{U}((x>?LJ%SN6|12ljx67Sjvaf{ujOsB`!RE=ZMD2Rkb9c8P4lcI&z_~g z>sQM@b$5jQAY2~Q@!UkwBIDg-5Ti`$9BRjYD>l8@&+cyES^93W244$H{&N(llk+gY zLt3%t5id=oa)lUZZ7E{t3AW=4{RvpIw*swsQQ$v5o_u;ZJw9CF$Uj;U3J=YxWcD_m zXYY78lP|TU73`bS>3d|WbvOX>!EWBcl}vVCb`ZypS?1`InIV%>#pS;@&5Q8nG#n@; z#(8o=n3!lBPIqO=>#8G4KP(4>jsD)zR-UP8qc@V za?nB}lF3Wb!73543yogon<~;<+)aa0Ca=tM@yq&`Iokj-``vvah!*#w>_odqOOrsi zTklRAZKiwe=e6lnqd)H}LZwW5RB!uq=pIA;w_X(E+gZQGPH=6-@UslpdQ270rI>Cn zf8p7(VnF0?|F^_y&0W2{>hGY|vk>dK4Lt=m&%zBf>=C_h~I7<#HmrukuPQ=#tEuq9GZG-5?0sI@(sq& zKUQ*Spv<|SG8N%l{C=HXH5stFUeVoi?0w0q=)?_Bb4WRF3j*VnP!5+%zJ^3dSNk*g z+qQPs5Q^?*LuHzfklVG!WGFI4# zJ%*Pkt%Hd(TIl_M3cc&+TkAWeEs%80-a>vX`Ol^YMYD6K`O@D*0`)VqjTe!ug$p7F z&julUhrI@r{j5@gcP3GzB3NogE0F7xNs~@ZS00?gJ*twInrR}Q9iMu?+Z;k7Vk7gX z88?e^lI<2>ynxquRQjCG-N%(@|KQGbBv|pSakFYdleOZ1bx{TSHhO! z+k*?9zj8&yoVsdix5a=)_eT9&#n6&?Y&xB2%YDbf*G zttfa$+ziMt#~tM=r*64|iwKDOrDUA2x9kA4=wJl3PVtF+EcMLoH#^@$f@X z1F_E+N8)QeJS`%1knx)HjRdFF3gQN7>-2(TP8vg_>iySMJVUlmNM~sLWm@+%BUlJb zN`eGjEFefHd#^wIrQBGvljJ%mV4Sc_W~IOT$fEgeeSoBGgkzzqBWnl1s5fMaa_Hko zSBsm=)Ez5z%R`9~`9Z(XKJTp$8FW%iz6>qr7$JG~3A!*N*{cm@JA@v4+<$m2=3@9A zmdrGgXb6ZVT4aNkC5!I4?m7RLxHm6N{S}|irtvoHckUNQ(04j0Ea)u^-AZznAAM6a zbTF%K`bn!O;T+2Z&HGe1QgaaARrVd(2FM9CoGcgIIAL9k4bKPZj9^MjAA!>WoZtxh zznwFCkMl5k0N-`FX{QtEW%|>E*JC}+N9%VcVUM#CjHt($6Mj_p$=J0l3RH)MDmMmj zzI+T32&I4Nqa}iw$!2}yk5kt#U+kUf{ksn|4RC~8KwMxkH1Mf#8+-}3O5@Aiw4cdXg7h1>D@pHB_9EC zvquZe#fuRA3X@ME#{rKGqjsiE8U-_718td{OL zv&ydh*Nw;YmXdyHpZ$E)5r<14QbmnB&0M@v$;b3+DHIM{GG-a(2m_IDabprf zJyNNtHcq_PfWgD$#f4gz;vrLfLh+F15k7BChD+~o^Q$h%E1ja_aFx95d8gkN4*Na7 zR0++4H5;zzNWPVX@uz1kil2{OCR8oq#a2GHshvCD+WXodpFuQCS>O@4uszkfQ0h_@ z+EUYj>#%D^rMte7l;u*riKnp396HSAg=Z(PK2iu;E)b`8kV~mCo#nq_ENl25Yr3@! zCjU;!=xH@zALuU1=t0LAkYHzdqK~L+nVvQSD^|3ZfE+R}ryRvsv=G&A)aOLh&nMoO zQH$@@d}!`(5If{|vK*5!IDf2r@&5R{pwU{S8CM$tjrVB%1e>y;1q znS3(+w282rl9(1C8{@NIil8_;h9O(;+}$`-mx4_75jgY-~+)~ z)AKRMO-ktQk$vi2k4N7dQv+5EqJ!Fw?04jpSNH7ijiNq`Vn>GYj15H|1whlkwV!c* z%%PA7)p}K=la8<2U`}OZW|A)VXpn3`iPo}Ue8v8|(76jv53ceLg`_c)tCi2dj%Ig+ zumAQtg0oBo5VCDh#fsNE;3HSL0P{Rh5atvG^O zgZJhuqnxsigjxprq(DX26b*d2n7LZem%B%odB2Q_Q9%?h102T1{>dFnFm0PuBZ3(I z>&mo1{^s?*0ZkzU|@06kl>?qbf#6VSWIFimo15@TSc zvw^)*J?~s^+0r^OJWJ2AM83X@2~mP|rUmwSdp08gtS$@t(G~dCrfF(0d9FfVYuNs# zR*3C_-=Qdhsaq;i9m+4C7jJpu^F?!Y`Q(M>c>W^GF2mz;4@35RTPtjC6;p(QY^^=u z`^T-D`*O}3IUcmI)55_7F>Im)AR7kQ!nFh%3z%1xt=_Hh97b6ZmK*d&!I#ve3iuru zwGF1c^r4FaPPo*qKH}Sn9;PU#GmzLOxpQSZ_TWAG%a>6#;!iWZtN5@agd1YH7}c!% z7~$Km%Wb<<0!-hD2vf2J8CuWmkztVKwQ+aOSe5C%&UTJ-3zr$ux-J}Ad`}o&-DwMJkOL;)7M5ZK%FUG%xQMc6 zXSGAvFX%XT%&1HEZ863i`fnG_CgxyeOMz`L7xBx)J&Q;TgVCg9i?fbAdhWzfJ^TJ( zhmBc?VrBpMA>vU3Jw9}!0iJJ^qA)iAcSHaqb80#5&>SzGxpNPax+uMelWI6##qGw| zKj&YO(uDm7*HC-x8B&%nz@K^b$a)ZO3Ml`}B8Q-*@_k9(3U6Z{?5+gX@gze8!F~r~ zFAB$Qo3s8p%XiH@$u3pEGh0KxrbSQB}F z0n5YZbHFbAqL8A|IT^y+JQ~T3VCfBdS)o7@#duI0SV7|jl7eq{PKnkztDb(1*FG1mZc2^6O?W}BAWo;N^aOtn@r{cPX0ep}Bgj!>Vy;3c%i8v_J^;dV zY{>o$8~qltdy=>f7=o+x18KM&JDY+O8Avc*TY%Agb7?gz5EgN_X=pYR-zxz|&1L4m~wUG2*|WJ&68w ziT9i9AW%!&Kx2(rsb)PhV6&(COm?%FL`!oE@dtyt!NM zvEFm;998M0L|wupNNJ)X!WEBY;zz0zgUKTx1S7n+%Qn(UUUHJxot~2af7<3QSe@ui z+5*)R%OO`W)=9{tm&ijhY4a-b4XM_=q8TayCZR}|kpNRBphd!4Ho1wdS8G;230rF) zSu}T5;dlNOJ z$`x?;G$KYtvz=}rW_5MCesU4G%cNRgY1uC70D5Hx%ojimC7+M#@(M+Fa++}b=1GWE zyLbibBWJCnYrHicfq7V}7iYp`BZ`LRJ~^hP#=@(i`bh^G@RP*wd%xEwEx~&T<+uJg z38ZHHuS338@LpaSe4#?@hqdY##LZ_u!YM`~9dAAfy7e{1+;HNc(v0Jw{FVi%5wio0 zYgJd+8J*Hy@;a=wJwp0)&|Udw_K?XUw!$wfSEq*XU9&&JocOl8Yh5QLK>FJX3Y4=6 zN4I|wnC4bv-p?yJCOmJfx>jz|^fcNxmVeRR`eSigMg37#I##H8(cwDoOD>;dqK%dn48K})%a|sVn1kQxCY75;!56q4jq`E*j zxOc3y_$=yC8SPUtf|B8VhC=ZbEi#bn zwV43HKUYe!J0bdm_fh&6AC%MI3wSIC+M5&jAx?wjFMwE^34JH6ko`ga#oDc&wDT|_ z$DafJ)Gv{~_iD^PqhmwbT2@`LVR{=^ltu!cua+R$F*?MMzVs^zvS+>@vR6=6_woG{ z?s>Hyl_j7Cs0878Av1To2 zU)OXWy~N5@`vnAHGpB}LO+Mj0El-Qs$1SKZ+jd~#I}mEXBf%|-@qlme&VqM@w-CKn z>9ift*1Cu}o%-DC$SU$3#Lf`cS~X7NO58<{SLj)HS%)hYeePEruUm;|0ep0@+kYE> z3=ZJZiEL=4m})wS)5N|+ttLzp*Q^=ZmSvhpADBm-U!WsA#r7K=+B-*uOXNQVw#njs zHQN!~@O!chAOYVe9`j0=dOz6b@=zSt9(xo{^fVV8tgtzD(h^Z2dicQqxs6Ar&!u*O z#2x}%l*7xg;WpYKRf+j(q8fYMCd%7lZ$bjUGpJM!E1`y}PW!59bIE$p(*CJ_I;J#L z(NyHG5rS2B?)8&r4l%mryDIE4(J^0wN*kx!7vQyDP%*7tzv_Q&-8@ zl}yLr7!hRaZvyQoy>(>I@(2Vq)G|Rm+r{MmkvY#-O#^21dW&{S+Y_bqA@HcA0p14&Q)irb8?Xkg(9P8IOj(P4pw3sivsuFTbw!0jU_vQ^+ju;NQX`d0K zacER}-HrCpalr)IpKeFeSn<bbF{KBxWHDSlFs8dUq;#!T?lb_osE_BRSV*v^< zCrkIyAdMTp8C+*T-#mwLYVw_+-jD|)C%nP)_AhSrlh1oCks;$pCPfeQP)2FCp z(j$j=0v#1QZ`^MOKJzKJKE5h>-=B`Cv=2_sC)w!p70jS(ed|r$o?QzXZiti7Rry`y zHfZha{w$*oRU?pl9g_INV?XJ4o{%LC%^>8hOZBYI5<_i~vzmU?p6kY|#c#8sAKV+{#q z9Sq8;7LKo_-~0m_jk{e0t{z|cpz848P@>l>R-l>Q2Xb$RA50a6_BPs}top=Q?thi# zN=O@FB=dNi>5lrLBug|UVZ%_X)s4upv4g;loPSwgU}j`FdM8bb&=DDV516#_E$tXA zg~!mGrf5I?IdZH;b=41wC=WdydiP_5(c?eJsPPdZ@9yb`WUXyA<{;DQ1rWlouDqm^ z5i^Kz)ak+xT$bh2oQUH?rr(^j9;p2~De=-Ohl)?&OxMCd{xVJ$@wu3Ep2j-HhUZ88 zwC~gtd>OrKZOa0eIe+9U2yP~B-rQ)!tL$31mQA0(<8W;i$CazP1ciCrSRj{`NXQ3=tg}l=sBh9zpN@&SM%o=*-GDt z$njq7?t^%T>+k*zNPT2NDR1q2WZh2i_l>iT3ctUTTJQ$2ei}oP=N;9oCt?p7$Gz2CJ+0}Tekt|@gi?% zD53t(n#pORPGLpHL+g~Rz~$KNKSGC#y`02$_jG@opTiPJUzWu^0~U;suw|NL3}_PI zrA7pB!;C*Z$#t$sB-%={TZfoZ?j$OmlzQ)Of<2Z?K<>SWz0zUNnC)u7FUHR0pmz?@ zQOK(*i*2iy6i>hRr+pn!jb^4Du?l_`Qcjz#^7}Vfr2f=^oUg;uPaPB8jMp=00tQQZ z=PfR0I#IGP<-RoJ54(j<*)qZx$%94ebYLLVC>vz`HZ<_Gv*GZ^^@(9H?U8D+u(I#< z0zS_kFe~0Ol=t8rIOeyF;$gm+c({$XwL*SHNtP^wrT5WQlIhtMR_mfntLN%(++NlC zc|pxbq^$0Xo{Q0>D@9Zg;q5X{XOlJ%D;61dZdcl>F~)BdcdyspGYW*f_H?tAIX(}( zRS3RQWQFK^4qnHlOfvuOwv&od%V%~lU@DWCHHkdZx*U^p2zOekO}VQdNUhMGi&SN(KkuXb+0D+0T;G2iwQ^0adNrb+OF zY>;R|1Y!h#qb5IuQ%a<N$^aPfQKvR2-@&oB+Po54({lG|uBq{b$Lzyi(w*NQD(~i-LVT&n zMs30L08EmnGG-gt>ny&@R@3g6O%D59R`WB=j<-o&9z$AR#n(rUkb2`6!1(WwKx-HC zeXz&%;Z_Rd@EO@%hmHE*b}*bQhXgSq8v&-IzacyLMff4|w{Ws?Jfw|>91qg;RWFgr#_dN0S9y8Q6pDivd|@YaxiHZE7y#w{jD z)+w`*^$|xAFXk6$KHIOo6>P}pA6%)d&@KZ3yj69;9Ce4UFVt)wT&sE>MCqo_Amc81 z#OopNN}MROtGtD|+@<8KGy2ZLALlX>$^UhnYutlWlf~qWmO=Z?u?j?Dxvy=258KT$ z+qzq=)^%hfz+qLX^u7Pmh-U9jrq5I*;(js#epe>8ed5_Dr)*|@_9E`CLUP+e_MLhg zS;Dxy7pm}Y6qoACNo`W=bv>xM_XHo(E-BX3$NufV_J_EI>!LH~Ot^+^jh9W){#(EwQzq?cIFJA3RoSFD(FbaE!3(q|07!ha305WARMu!0u=5%rc@UFbeS<-@&2(WRxkoETMb{S zg-LJD4ku0;X|ROK8sic2(Cai$nJo)l&#pOk4Dj|7Uz)1bWxamfjroBH_$AEd6F(C$ zU(vcTg45y_I7^zXK*_InIM8m)O3ek%(W(xlCegf`PLZ=79a)$ksa@Bn;wB8zY_^2# zsf1yYGrXLdEX2s>0-{DhLS@+|#Db_(i z)r=(r>ZkCx0~iRK^8V#N2(sR)Qd82mrC2O@qRKhO(@xY`PMB_Zt4>H_V_@{r{9hf1 zdC=B3&frucV$oC)4hdm3PLaz|FMTZ-Ds&; z?MH~aIiQuFfc738{H0h#Q@8Ps2l;O295pAs1vMMaKzMkoq)NT_=as}yJEn^~H~iiO z-42Kd3+SU(Rcnq|xKoNr9}qx4Yznj$(tasH&)9miTky3_GEV2c_!v{-hkGbN76*UC zcm4xnjrr)|#t=?Y3Y8n77kH1JnVvaA}8;DT*|xr3u8@Be-w zG(2=d#ip0 zG1S(;8Z#vSO87FZj_L3?f>13WY1!ji#L*j>uo-#HqBE*mwQAKlI02QZ+ssBHF#jg; zbe_&|hIHePjaXE6O}?P{w{DVE9=r2mqg2ZNo*)0fDzQ(I6Hpp-a#u}Bo{?3Lh??q< z;AE(AJy{^OQ_Vph`L5_@1l@G~=%>9kfN%1SqL9b?0HG!#T9fNV}&2S>?= z#ya0SbQBq>+GM(Ss;QDFW|Tc`8~1PGJvL=Wsv0t^tY?3)CBQ*kzbbfpkW1{Ab42c@ zHC}vl*(g^HHO2?uvdsb~#Ra3VN(k@xooNmZT?IYvUO!$o(Zeek1r1XySX>88FMq+K*)ju(gx zuZp;U*Z2s);X1Wf$TN$*gdSBjjYYEe=4$R|xK~SvB;DHJ@fW#`gPL9uKI~G<9ChQ? zH5tFeX;`#i{BJ#*J;q%F*J^Y*OQ~euboMv@KX(3ZFIjN8y2EICb!-q5Unwi zy^)F*%A^-eF#gAYOXTDJNaG+X_Yxyo<#GHO&CP%2$l;vdF+1B>nn&4M~{5D>plE z{1R>}`4j51Yhlw?dy&S=k?Xxf<3_uscWGWpBGI`$HkOc*PCa()mj70$NYDlOEakO# zxl()cp5%D5uVf%Dhd@t79oYp=u$y!L38L2m5aEA1g0Rh?cBV1E)9zuZX>U^47VJBW zmQBp=nEKmS7wC9=B9Z2iTG=th;S)$koj1uD-2=unM_L}1fr$V1__rFhiskSx;Wu@} z>!-au#cuS68uP4NTspoiUSD~%WXmf}$3VMYbR*#JC^My%yes2(kctxrhPVBKL9^X^ zHz2gJm7Tx)R^W8^9mF?c`cnqP$~x5`=Qn&M=Vve=@~u7!)?;_>I-=JZ4;rMDP{bG5 zGF^lhPE60uK0x>tk_NMaH6iI&hTuK1{#>HG42)Ov({Z2FGWNDc=H3IGCJSUZL{R$) zF(A2?89$d_?o1RT8LFIr@SxoGE0-zYNPfp05Z7MKk83HCs~3Z+-Z9}GnJ{7fb2a;3 zliMtAdSp^bJLxkp`$INlVE07gjEvA1Rrb zf%M*qZDC=5r>saK=SyKev0U~iL8ieW z=vnB*CkFl8$eIKLKKmBSN^WkMU*Te7pO|3zY#I?^e(KdZnprb%&U5MaS3921Qq9iL zXX3<@s=lVf-nLrFjbJ)jUzwFcWp65vCXb72L9v^r{W!P8ql#dEZWc0O9GD#G&zcq? z75C!a`VC59#CHAu7Il_ovxTV5bvpUQKCv6_*DCwAKXGl)aRwffTcS1%eJh_nWO=^{ zWSXZAQBc#ef@UQ0WqQzTUqd=7gyh}bqd%&S*06Xi41hRNf0qJ0RGZSj7zWS9Gne`LS)%%*T>;%t^F5sI+dJo?K&`o*{iN;QcJoft@L z|D@`GIQ&^;{dFcyVCRmztBTHi<($w$gjlT9?pszmC?$dNhX+|Pws9gvm33|W@)@L2 zXrT3B7nVBI>f6*aqE=Yv1Jrcj=NsRyH%{ojL`J-6EC27syLpyhKNx+qJ$?2>7gw}Q zwj$o_SKP zbg8iGo6JGn!$a2?n%(alcw}|#O29Oq9OFbSPw?N5Z!{!#5_GV!gyBcjKeTWt*8C$A ziIE{)>>=V^_izIkZ3v4?%RlJepsfw^c^&nNbF_@A$(QAY>(Q9lpw688Y3IJMMkdVI z{}kz}p3#AuFQZ;+FjievZdYDu2PV+^^7-oKDpT~=bAdM*%TpK``?CXQ>Bc=OFbu9t z_kDA{#7NZ?p7Z+iD=A!9-vhxq{XCw{BI{7JX~>5c!HME#ef-pe(Swj??e)J)Dh(3H z<66OLu9qdn&3px|5TeF!4 zb(970=5as`(5sP&-GQGv83&cl1c?Ivtw#nuYeXv~z2|m8a%5`#FA)A_Ox{mo*wVHO zF;sFtJ|>m-Oi+WzNXpAPHzFl%P$FKx9{z($9Hu;1QpM#kf zZ&llSnF4GEvy5bw$$T|-LTasIq_>sXFx;v6^2wCsp%b65OD8WL)!Z@gw-`72ykQ00 z9{V-X-p_4BmzaVmZk?J`W&m+x{4@eIy3?-T@h(L1v;QM+g zM@2sb->Hq!HGL1e>np(V#Cd-K=hxfXwq)`uNvyOY$mE4@w5#r@yr{c1qaf~Kd1S3J zNITwg>1~eBX41eM1rUfsyGOCSolbB=< zPgGR^zm8J=oVa;zhlS)hhlFxs4CV)8rF6KUe0wis^5XcruG^)}!dK*^uEeqW9nO?W zNfC9Kb!68XM}r(F`?9>Vtzj$Rp7+F`?t_Zh2F7KFoDaki=R3hhD&C4;_5$KHAMF3| zq?-D7P=JdG>EzmuT+EyG%LfgqVj(EXiU7k?rpdUMKKo!}286AVy48xVR2NIl%Y^P= zqfGn1jPZ4|2ftEXze{dG^ADf=$-682Rw*~Sa?Hi)(D{s?sD#(mKn*4Dv;?ZFELP7m zVphCk;TX4x9Vsd>v0pag^!DaVc(%6;T5%IcE+31;gM_`I2#+v4)^olaC=b<}HTgty zLc15K#51mV>Y{&%L~qFpc; zByIi{xfeh8PtaGoNNbP}m9IIC_vU-JxWF+={WrDohTJu#r+%g!I!ymqIAG!q@!F;;NVuxPIaONDw5UjxB_I6EASu8 z<<}SEC_#@AYo?ly4MYuS;o7Wk~uN)3izy(VGQ+gt8t{D{aegnz={dAAK-Dhex*;S2b5C7TJ&)PFQqtB;j zCgwhIp7^1$4a(c${JWs?dfm1i`paW9l`=pC0gLDV0m1~57&e#bh>sUB%Qo|p0Y^=u zIdw8fB6Y0lh7ZW`nBtV@?3>~`VpCYVr2`}|{=Vff4*uOn-ybAObCUaTA!s`SuTCBl zmp+sGiRaBg6Q0p6^Cy6idu84)I+Wl!o?D7i&(GuV`EvFp>vy%SSR>;v7>P=U;bqq8 z;f1_waYmDZ1McIEQ1-PR&R~mBXAL_L+qJoWpc|3NDvkN*bl2#E^NK6p4lbb~x8-%v z+Wvg>Y+A(VzFYrLEpr0>w~-_0i4j6JR9W1upBjaW2sEnaKUCSfA4oV+v#7WVp3=R@ zH}kAG+$p<@nz2mXlNf2VF3R=GeP@@0-{o>{a9ys>o=&!N|MuPQc227{WPnx3^J~D0 zC0d$jbtl}pk23i;h=x#{cFxiNg#X;cYh5C!C|2NJ3zdD7lltuEc@w`{v<1!yr#i$pz;&x{S{aj^Fj&Ipp455 z$*DS4shcYmhW%p0bd0W#Nr5+6gn6E+1#6}^x2ET*uZ2XTiRqj+!Y4SoFg2np zPJeyAzG^qo)~UoQ&;(BLnxqw{{+-AavS<3|$HkMzb22If8*1eL6SNck$Lg$xv(ocX zv^fe3^I&_G)SUp#a^J?V2kb6%PzH83;Y@D%`U&4ydd9}%x2UrC`L*zxIyE+i`GSi-aMMF2yVoSC znO4+?#-qa^w=hOba!>Yf;cAhl`-XHdWpT{Bg$|o4`h8Cv~9{S8l973}q+1Z>K~|C+pH<%OnLy12yBg z4`#7Ic*CZ+h|BEJKYsJ^f1vFf!QZc_=VITUZi;M-ilSOhJHK%xr-`e`IEJ0BCj1&K zQBWR|wZ(9oE9f;Oi5_ekY>g-%oGaq$kKgaoC*MxEO?J&RG723yovO{3#1QmCDaIzI zEvcoKo7p>xV5^_wla_%uxI;$4dMP#ZWO#9j^CYD&->nXma$-brmhwf12u{7(A60sB z`Hyz*5SAnD+HA2-o8`KYqzuuGTaP0(7x_aEGhZd*Z$S5lf~QzitYq&_Uwoz+>0idD zMU;DFrn0KtOj3vU?6}Gu*47xk=FytT7gQBwx#r!wAoZk{$To3#B*@gTt&fm1@mg`> zXqGrxp|y;v;5iYB0=hIW-!);FXRjN1QYa5dq$`fRV6B_9;dS_d+<8BzhVFTw_x!`n z@kU8QrL_(8`TeICpN^V5P)CI8S{bLCyJj{%f=pMf+n=Iz-yGH^)?R!djD@@HGLJoH zU`>czC`XitVO*Fp_}=%?&&-3%`BzD=U4|PB_W9!hpJ?;@fh1IU)M}-8_e|LaZ{u^@NIT#t)H=f3O zef3rrReuWtJOkak4w(!z6KfO#!74o&Ge*mVcY=MtKo$pQ%B{5~Qb%dh>VQHk5On2$ z%bmIGw`S!zpuED5Ii~Xu3u3(5&o=g~2V2Ts9?KlmE*#b)-dV&^Kj<$~4KxIG`pqQB z=zxR&Wn4zgL@LVf%os2*MZ}(YEl(T;oVkHxkZGq!)VEFrEA5_^BJw?F>UdGYYqoPK zzP-Wv*LRA}3Xb)2I3$|?OUHc0QC3XbJ*-fYG;YIF^cqR~vhtQ~+a)G|ANuv^bx5sO z|Gs^C?`s7NKXVWQtOarDXqP}Tzi2d|u%<>1#Y}I}@-Q6srIszvU}XJ6QG5V=u#?ai z1Y~}meVgn85&5pj2Viu_(@JmdQBxn5|9EMBvBzx^s<^X2={%$h$V zcMP08CI-41?lZUbC_>ObJVISZs8un-ZvDcGt?Y8>GBc%!vg6z5V)1WbG>t_(kV^Qb zNy4WJ>2C2UG8$4pQqsP>PQ>O>D^Z(B9Gejscgjh^8uixnTcBl(Sf}Ig>PmZ;;_6Ea zs9LaZf-_ansnn?mba}(3qnbL>c^0)BPtE>yIP-e|x113Z8~!^jydl$TAuX?=aJxDC z!UAtMz8s%W)XaNaAr2n)6M0~6w@Ls0?Oh(LHNHVC)UuWyp@&-|hCo67$ z7x-Ecs|hwdGnM?q>$+s-mvI!_d#WGd1Y0U^;4EwOqnX;pFqEW z)Yu+arFgG%hBj#OVg7L6G0#PEW33!v(nQGnRDYVUulg=H@zux8tV;I8F`Vz(hUb<# zR2b={YhC3Rr|(~Azjyi$t1cO#>fWLF^FWaJO;uvz^fpW4fi`gw+WR6#My;oLBUr%oeLodW9NIy69Tj-Ybauu{2U2|f zVy=xhCnOE}@z1(~9hw^2Kbf1qbq_wBvbf$3k6zy!#8-R#5Z3D7d?261D5Wed_NdqR zsQpA77*h$sM7`htG&*p#eha#1Rf*XN!F>=na6vxFts+ofjRg1cmc$$6!IA@j^R_4g zjNx6a$!=G@$4dqkts80TpM&25HeTfuIYkfHJRsx)IMcIb?L>Qi*+$Q-(LO4b+L2X_ z=h)p30piC?h$k}_`4^>{%WPdJHMGd);C*$a6H3|UkyQ4gTTuYFv*UZj{Pn=cjdfRZ zb-%8>nyINaw%x4uFQ$A0h<`R@KPl7Sy)rT|g{ejG-QE>~^xgJRXIZCJI>LUJA6+Zx zAJ!5)&2i`13%bH^pHRP{ejNAyPOqC9eaSR6!e(Y#@ccIn|4J(L8@O)Yb9u0}=}%#U zZfcSVXv`$yQf#=!`S?Rnt6~W}Rn=z)psCfLb=f7hNbTsdhyOiV@AkUW~ZOitT529>PtYxl#FZY2WQMsRnoyqHROo;|5sM-03s~{MJpBj zKfx5>d`GYmXR!Ch!T*V}$&tP&C>xlx4uM}-ZA-R}Z9qQEYj~5jYm~n!Rw)y8*qQDL zF8jRf4@zKNn6Sr$lW$#<=(Piysy;0R4NI&dTP$rMK(wJwp-D1Lw-+} zlkJo2Ii-r(*EGs`g01e>7k{Jy#N5wfy>EeII~pDA9-gB7bYrDG1R`#uqyu8uAw7;v zKW(7zKMxgOX_@|ixH|K9DBnNq+lMS8J|$roLLw82tYfK=J(Xn7o{%l;%#b~6D7zs- zCHuaN?6Qq5!`OGmJ`7_Rp7|cn^V9P;T*rM}*ZqE<=j-K&U455uKIu3?n{XbX-daBJ zgWq3f@FBSN7db;ZQK8T#XmRA*c*_yh5vD}Q!hAex5tI6 zxZ$Zllh7L3i<##c*2$@Z4DGCG>d(!rUVEsf5+5_jDt%gM5*xiYK3GowZPtk~-Q7rf z(N$TT;!74*K+rGWLdor1FOSO%;u*Qzs+O{8{iECPB|K*~Q9DP$c%0RZ&0_yqPO8tx z!3MX|u+Oza$4MC0mi?ya$cv!A)Ni(H(2;PVMzwU5Qk;1!LoQ+|a3FOZ((~6biyf2j zDC2#gaA)^DSm%=@h5WUa55o-8v#dEUpzpoB3vCjPUof88m?*NDk-E?p40CjZym3b> zX4Ly`WL@lQ^6PGwqrXXj5|_}eYdbNN`-Ju7NWjb6GbMX5QP<}6u2_wI%e2QKjH1d^ z?_EEA%bAxu_^xoI+Zgn25=Oe|+0V74DM3yqnT0SrT8*v~{}k*PERt`KqefoJzng%J zhX<4^hXEN$FJS>rO!IuMKeLS^7cwaFnQNw&qxaqSw-8<`tK~EhGv~cC=gZXO^_d6L zGGx=+?uDCMbNtPsw5w9S%%i&NR8M;h)I;JXS-K49A;rXJSIG&*bWQ2cyCx;hl78<} z`F{tO9h&(C2++VHhkAE0${L;YT{bpK$KjL3p_T6}&%U<`o)QiXfv*!=acQy(2Ffzk zr*C5?Nv!ncUR&C|=FCjr%7^az&7;k`xQ4AC?ln+>d}u5+#%0J1X+^t5N~)+dR30MD zz;rjp7@}T^^eN-2Vw19GBo>=Yr;8Lveo$=>5RDGR_nJ74I)Lg$*Hgrzy?FI-d_l)oVW@1^$)_(Zet@TL!Jn_RG(GwLsg zIlKr~Sx5fJKDpc1bW9b??QL6f723u-%LTvhM-69NlrmvlyJ{JB*v{sZEEqH(I&^Bm z6;~O%UiQR_WLBQD*dKiUA}x@Tmtoy#PIBToF=ZPgpg^Wxtaa0TMEs~fz5Mz22zZ5W z!NJGT)|mzBKhc%(_t&~X5>HaiHH;m{m!jekCG zw6eugw?|*fM{m%MAPWo-qOTR>?q|eG>?O2lW+2M}b(%M&e~H#SCp(nL)MR;}O;6q$?M6(^^v4sG(JV815@mZ2dQI<$Li6&@?SRXNvj9%C zE^5gNXhUChOp6x?AFSQe)t-~7KBR}GI<0|e1-5PR0w8FK0ls#)M#4~XzVdSoV=y_n zakRu`HdPvHVBOJVrK~sU0g@l)rM>7qZ%Y{B>hs;t4;}y3Syu9^^m(_k(J;zHKDVx$ z9(_3ZXaNb*zd9M+u|bpql!D*Uy$vm}MZ>%9b#l$op>G?s_p3QCzAm`P@n%HOT*XxA z)UiW2rU1<^lg_D{46Nu%r_F^Pw$jiXVSE)p5lW)USP=yz^iDHN}8{>|v`{g1FW#9$=Zvdlb7~Zk%mY_JbFWwVU!zI#h ztyG3DKiW+khiQkd5$o$jHDs5=9Aw-omXAcI(-Hh{-C+ymE$1~SNe@;-jN?y-n!H#e zXDc?3r1mAa|9H>=SM}#O4+XeXKJle!n)bvB7$EWtETiD(&OU9_ViVt#etjx65zGfa z^d5bQQ<**#uZRggNy&fL_fS*`c+#iulp1&pliDl`qpa^v39-zrrG3r)ye#`KPM#A? z_l&li(qy0GZaDrP3|O8)fYA+;_tyq4f(2|7_&#I^yq-HZs#~!A?HKq;Qh*1(9vJ-2yk_^y z!$C)hs9O4*uzfWW!#DXGfE|^OzSXYN4vV&vE+&3$_s>HdL0Z=s&v;dNQYA(eda`#c zH_rJ*e)>T?=XD-kc8MUP_=9xLt{@sKA1cC;y(&ihjLp=7mV&NaM>U>T~=zo{M5#Z_hGQ#9PI$W#2Mg z)DJB2Q!Wm(Jtkh@7pfA?$^6wjz3mhTT973Bf6r6DE7ndrk0b>C+R1Ias7V;cI9YX@ zaXn++SKClEa}50P>zK*ap9u#1{I=9l{u-lG=wYy``CdYp7+o~38P}GhR5ek@^>#TW z`$NY`n&Jo{n>b|-%oDNC{qDIn456TdY-HB*8bpQ4nO$vm)XfEemz%s1gOPe6#~c(^ zn|Rn{T}~io0J8lZ53-(vrkN-gMYrTKKq4yV7|yzs8}hk(!k)(%CU6Ch*>Jp}n^5s}*&t z%wXU`6Iq7^oQEVV-~Z?r^=3UFYWM-~LqetpIDzbXdjL6%&~5F3j;++O_{`AmsGXc< z^#rh8m^w^^=vp)CH&Kn<&ezS8SU4O>2`n_4SiigUH9O6Rd35)0zLMv#OJ61Hgq+V; zegWpDY`n;&F2w?KD@3WZB3mKTz!V4dsRa=wAP%zy=;pxOpp#n~N?iK{=VP-ld=c;H zw#$25AzUU~I{0WzF8K6ob}r~JTp=&={ysBe$)MzaHtRO7;-Z}{Tde-sHuN|_86NUK zZ;QzPye+iN{$~xB`&@3VlSH1n*1f&UyLFhG6gk+r!FeZzUjumm`J;IDZl~bN%s@({ zv?M;7){RBdoMm0TOTj@|Qg_^j#y>^$sE{^+Kyw)0TxD5WWa${FG;n)9S%Z-sGEL_K{RDdLdhBNoW z>oeIF#+So(3Udw(8lJgf8QJTg`j_|Mg_{O_nWWnZ)i@OP&Jn>h!p}a=uzGjcFfX&_ zV$(}5;<&?Su|d%r&hJoo2k97X|LbzArCr$|dQkPvwK zU{~1uO;aS>U5eQWuJ)0u_ZMiC^dOp@N8sK2V(7vuJ!Ic+WCX4wKOMHau9KKXICfiN zZ$z=b$#OG>g?;B}#nxyLpvZZ0m@qmoZ>=Y5^~xoZIrw zO}#olGeaC$?%|Cp_6_CqN11zsUYREyP3QTRJ`66)e3R$q@XTTIRHxwry##a@-IFUA znE%o^v;Vjh;`v)6Qq4{7pK*?1u6ygljM^(CCS<9 zIz1exWQTaRhmXUKLTBgx!50g`R<6qixRd?OKiejMnwQUoHUZ-v@j8{S-AfNPtL7KR zBLCTAgY~>d>HKGmvzT)?4e03seLV8-SWu}XWFfGbKlkk2n0%r0nS7BVrlC#d@U!m_ zT4qI9hNpdn*>z2gIv7&f7OgiFDn&=eB5>JOm7LjJxn02hP}CottkbAKo&eh2Yv<92 z7wL!pK73LyuZF7Ay}?x2QQ0le@wOwNSAz)E%ty~~e)63mt|L;M8D817fZ`YbUUI)P znqPeLPYtP}*OZg{h^XH*Y0&o{PRq&s6$4qAA{APGfNFafUIMhR?8{=%Ybnf`5IySI zOx}MeYWA|FBs0jlmdldt)n03aPIt@{S;+L!SJH9R_Wvs6Z4s%9L)!2C6=Jh8uSL8) z{daU4%H{NV%HQ5X2H=;lPAOMZTs_PlmIJrNq=N+XXC8dAWp7`6 zT{GyJhkMi|j+VMD%>`RP>#yd)LvxDRRX-9^Qw&?5-1PTCiN?2-3;SsF7r4p+wgg-a zn+o7;vqto*Ys#^wZ%PlSNFHgqJ3SFE%Y#e5`@~uA30KqeaDF;o9hY~lZRyo|xp%i; zLeEaJ9m8il=6i7BMIOMW@u#paaUR~g9o%X^@xXrF&r{+`k38YnMQ`%mwTL=`F#1il z;+V-^_?z8n7csQb-3MXuu{~6Y+Z!pAAL5hQ_r3U}dln*l^f>69LtvK9E346C(%)jC zZ46L_L$Ih(4ESxJk)dG6|Lkc16=k#)?6Rv-aKP~i8m(~V)=iCvQ4Wx^cLDYHX1M(K z!{H@0t2M9|&plt2hiPAq#FFqzfmd_>fF^-vw*7+PFZ_SbJHFGL?O z{C=v~Id6!EY0CG{^{}O`BSi>-k={j)g6dI*g#QE1?(^*oAB60OyX*v;3h93N7PSdPd%`&Lin%>k2YPOfBOREg_Pw54arj&9xj)kE^!y zxnAS7qKoK;lcbuxE*S?d*zq2Fr?new4W%7*_4D_>67#O@*z9r<1-E`Tu&pf2H#twI zCJ*t_HOM_7Y}_LVh`Wsk(rwip`hSsUK#2TzW^LC_8Y;fzp@#ntFE#~ie+WK?o;}MP zs@VyH$=lQHcw!l|oUmQZ_`IXZqZ0Yq(RC12g=WlYf* zO^e=ba$uqW&~)r;itk_ouYGh?Tri$p!H|uc(pMZRZ_Ga_r+#?oXgSlw54tf114=ZZ z;IuabUk|L3-v(|!F~0`|Uq{+oW}|d?->|mv>ea9R$fG8;rEm^YN~_zxt^?PXr)9^1vy;kYWeBQ59S4O+fb z;AXmsws-N*7+bH6Aq<*t&UZQP+l0)HP)~0mj)lPr~UDE|U8Okdln z)h+e|Gj2GQ%3tOZRCu{*LcEd7rzCWbjZicjzn>g|GA;v|>w;WY1opRuuz!`K9+Wxv zQW6r?oA1_NLW4dAET$6-{NshxvI50V8cXK|a6GvWUkn2hcjz|;Ywj$Z4AAy&-?ZCc zP})$yEK|fVC9DS_lor*?$^V*c%DnC-{BFXiSulfg3rnpA&IvK;J&Dxik6ZtMaP@4( zG5BFop(dX&I@!*zUM#+-?4z1aJ42nqlmT0ZgrtK{5%-`|n%oU$@w zhn8IDPtYeuqR~~#*?Y^5?6ARv_!}B;62gtTeT0wa@B;~!b7N1QK{ooe)wyNj)Xl#} zne+nQ2R#F#WO{^9d%S{;Di&~e)5p&=$>Qh6;kaumb}%PCo(rt?#j4{}h0Lv5rqNzSdnNSxQKmm|)Zjp9b@o>ooRSu6#IS8qkwk9|yW%uKF7Fxr zOI(VSMpHJ^*;cRWlc|GB=Ra$lPFd!~2UiYn3;Bd45M=GR{a|kOjU6-W>?`YPBMXT~;5d@0{U_tuXlRnd20mg?M9eCpYsWqi^U zkc~g~N=(v3sc{pQ8&9B|Z0GGrlJWnhfg98~#M!*1NrE5f|63UEth*zYT2Ff1F8iTp zh!*k&lH9q|dU}T1I^GdqzsiN$KZGLA+R@~hj;@a}ydBXElj$+cZA%Opc>Q`tPc?5~ zfd)mW$B2HMAU&wqKcKLhNr{6n`3)U&K({D_j94>hUO7wRi9emhUPy8wpMj~)Yx?jL+!1*PdVR5@9W=_-D^)*dK@H^e%q>ZM|6YxvS4z{ z_-uRAdpeREVxpi)Tzv%b(CC7x0VJ)hlvYcieb;Lp?7+HTN`-s z2<(m8w?FKT`{edgRXb2sn|6a)*g5pvtse5mIE>ZQvfXob@P}|yb?MYSYtAEQ;>^m| zpJRdp6yV3-UvQnF!WTIZlKN(jo9_NPmeej=p#LtZS{W=WIXR$;{%_v(SN2OY@XB7? ze=8C1fQ^Ui$kU9+Ir*hfTV9;a27Bha?GW|Aj>xBM1sN_g5l0%Z=QLFbU=RH0-bJ*H zs?-*8=U4rCB;VA%_Uq!CB=^rF$G$5VlsI2t`AGj-`Uk-T)@rx6`LwZWRslty@Ae?_ z;=|b?d#CN!5E+>rGDmz?7&Q_6S-7*eIRVqkQHv;st{|UBI{nK0J^o@gIUaIEPi^mI zffR(|#v`9a9Nd+f@nLLQ#$5I1d(JchVv*X4 zEV8w!srbh>%U=!7^RUhg@yF6HW#CVC|8n97k6gzLchp|vg;g`GQhc&hZCXi^EJfx< zr(wg1+6uW@ofzeM-*u}&h6RHcP3FrnTI%Z#2Uf-BF%Ep#$2=YnLmEk*?w`Fx^9;M$ zOq%W5uLvCFBMy9bq}Aj}u^!c{EHuZn%7Hn~De^s$p27YpRbs30exeDlJa=cifE}ZSBrL_F3_zquZ;R|Hk``EE=6CptRx4XbUB)t4MG?NJ$p41Av z)7kPlP||9|9wD{8s#b`X?;U|m-Qrt<|0^?dL@mv=g;-xmh@Ws+9($0Vs^cF%-$cI6 z?#$XVUsoFBMfWqaaeAX-^oL-ER&OM5evJO}2XF={=98NkD zOagCx^;-$;Zn*g)wxjy#(9tIrG4_1GcIlJ!N$yx#8&_1plQ|y~ZK4*$S_ogY4ZBve-CGPmXEfUzrJUc&Sq}A~-2zw%sz2WW z(r^R`M8;5E;?k^Cu6&+2>NGXWIk^i=I--ZQ&+YlaUxOh#fyI$7yNCuLN6}E(Bh8>` z|Ddu>&bQ32pW2(xi$F)~H@Q<5|FA9VHaQy`TIpszuoF9F_nMIBqIvfT>du`fe@W%v zrkh!SU|u3%!iv)z!lhCTkIMOvfHxcLe^-WWl*|R^o_q|u{(bP&gn}i?wlEe=kp2ea zo*u;vnT)xgEhp`t{#zM)B3i;c{9#+P=5F2(&U--EdcWOaTeVow{?Vx{>R|LJjxJs~ zh!Hn~u=jP_QjQ~WL}JmuW_p&5`I!&sZC6!$H(+lH2$qSSzvVE*MR}m1XxQCF(QN54 z2GNkijAH{H-I@MO`?eHOs9lEF{-zAO^YEjjvWqK&g#DG8);jq9pC%%~h|x+G_HFfJ zyj76xfokuEJ#`JCP#wmGeN(9SZsZ+ij?RBBqf9G#p=WoiTKfK!hMnNpWfI;`*z*;k zK0aVa%d)Gm5;U*wSwtG1E;1b`BSh^$bAX@2Ulg8+qWYBvV878fD&#uzx+!shmBYsv z1;f7V@rop+@s%EK-l)~!jAhn`CroSb-R~DvDGjp~5xH}Y>i=!NdKz#tJS^j*XBzC% z!+Q#2P=`3JZpsXUGQmkdx2zW%PddM_fpyF_F!#*LPIil7I|u439ht+@HIJPRzq4!* zU!~kp@U4<8YJCJ$6}crg0lL#{momt0!lO^ z5^5F;4pL`t9YgYpIy-1DFK&8lEN>dDjeDs)jvOkonXYB3u}ul7>)u7(zt8}|(E;yCR{avACbtY_mkNIS?c|iBq_Jb~?2O&j8RZG{`&W?)`7yTT zq*CeB+?cvejrAeszujEDp#%9rz}u1hr>j7B%uBHPZfNYay@OeUSDQ&elf1f$G+0L- zZ2Xs2-S<(3Vf4TzxmUdy;Vc}+;#X`q?=(H;Hu0_I zF;M}{`exTJ9G3R57+_xmJ?%8jZ`3F7sUmvYfPF7%71@&YLBm!!0MiKT+j$6COMrC) zV(8EDHjo@$9pCb0jCC9%)MK3)+63bXg5!msUXD+V=azKdjH3 z^iLKz+dJCRhcrC(6@+{SLkc)bme){0<Tm%drjL?Qdi8;$Q8&Yd7+sZRmtA5b|6I{W z{rWA401(vY94V&5Q6-VvT#qRfGe}ce{Up{=`BAC2a>A8S!lfY{km6Ao8t5~0!G%ty z-c}?Siag>*n%o*4NVQtR!qc6N0X{rRNSM+-$vkMW2thgyhrkaS|3pwVed?*sK2}Da z4(v3w9xC!6Iw{sXUl1iL$iLt|cL71*FN4lv;t-TzpvoZ?!K#(qfFNkPL67+_=iQ(L zeu^cJ>r%XR^O)3kS`!aB|JNG8mwR1*!&)ErMpA^%Uf}wFqn#~5TvAk~UW-CdZ=68F z0aNM7PG?DApE8%ZaZcr$APzLogil1M^NCQ%EijN+C`9>JAU``7fu7-1TP6BZzXc04 zB-vmcFv;&H5ZFb!^WF@eMN5q$mU~+a$&M4kgWg-^%>hVE_g#bADT=FY%#0(MRyXlm zFBoB;7imYBwlb`37sJ^zk5p!uNxw4{-=&n+ZYB@53IUw}(?U>uppy`RKo5@sAV4b( zyP<)7%k4u(&nZ|B*4b)2_w3&>&<93`4 z5Ldy#+L4C^?!&TJe{~R(wIZq{^6rm1EBI5!JWhp6X`WoLtW)D>bv`kg25lMwJ7h;Y z>Y}!8=gOfXScUQMe)D<5a=rGn{_Jej)9Og?>o*4xlWkWj06&Yc0N9;D6n$q+!iUd* zZ~CNpl;h!~inm4)&x_Q0<`j$bLKoU#)a#;4A+0P0yTq((%ToRq>mNnp zcpsj^7VujoCA;xW^3WBy&qaqK|bM9!ouzKGu*e{32h6`@N?-0EY6H0Hi+ zeR#d)tSD21m@K=C++`+E5q@%6bDmRMxy?r=(i`=hR)%`cRx^JQwQHq5dof&hctNXI z{p!L?rVQ>Qyu<3+JE%VDEaSSqbt~w5F`|~|lPl4aq`0`8|KMC+yBLLjrJbI(PQrPN z=3x^8G!^BQ22*>9SGcbo8(QPPSFzUm*P3Oj4*eaj+Ll^pUuU*?z2^lUKF^c+V-n;q z*wA2h2}A%D+Mz3w)Aw9-%)t^S95DSM_nq}@l2>@MXo~1yW1HCCEiklEHx0gI>KW#$ zLdm9PI?LVeXpL-oX#Pj%{@Tl!Mc_aPZ0hs-GJ5ry^@g?bFVf&{Rm7vjN*I{)=lQ^k z=G^5~-pFfyOqf)UXQkZvu{u421-MQ`9%*wl#(#Im+Ox z?ybhWP2D#JOv*BiJc;`0kS)FelNr1e0GC5Wn3iP>kolw``@H83>obA25uQV|@yyX3 zJrGSXHsVSDU&H^tC@IZ8@GAxsa}Lpt&se=OR_D_Q`}_4MMB}$QEW1j8pt+YWInJke z6ijIlyLwH)k0Q5JSGJqQoiey!i@YBrkDgT?-_Xtqi!eFDuN#`bLDPf6Dy;VNL8?rP z3DYNHsqFG(;Dx=AzD9yGP;c)IQT%Q{}rIE-h$5$apG6(D?yX60wpAV@fzdsQ0Q+^=6$^Z!&*D9_udc zAH{_Io!`ps)q19;D1a?Zcz4#3;&W?PBN z59A5@n)zO4)BkX3Rlk4n{>SMZBt5iaMf!)b(GMA>l^@S6A%|+Hw)7>Jr+WB|U5Cdp zx3hgsHL%?9Kl~Hs^G1iwV3~iCCie`5)C8S)mKv|%V>Z+RU1h@f8Vl7Nh($<4hLPb3 zu}+_L7*jNnXj3aQ} z`8u=dvY%y3ons-on2XI+D0XU--aq6@4~V+S9YbgD8rpat*46SP$I6;^zpmBz;?`=m z=Vx%#A#b>Duj9`uB@6}Xoc3X~v!vs|R-y9{fC#6>@q(5#GlMAl*@gnECG(3?w2A(T zKg8C(!YX)B0n5*zX5N}y!??9Y@xe2EmSysv;Ue}^lQZ7q9CMglq7NqJp?2w@T^1v& zp6yD5kGOunRTS+Fjg$jxIgb^rh;`Ge=|mp%bn(-t&Ky+g)k z%8#5pl2Enk{5{_jz$tjC*dPJi+Zlc}Gu99DptZ-|e@CA#3QZ28p7J5zxe z`M_(BZV&Dy=;e|u!@T@55aH`Wuo8xO>4C%v;c8j*09^$dAeV?42 z4&Gm{$X6kOqu^~dJPOe4ebZTMH)ucj$B$qmejuKaCN^83vM~#eMsEk zC&l>3oPJVvJ9#2t_fYnheLeIwf6qhlchFng^#VJ6clt8#NqkoUggH4P5xLcaaEhvX z)I5Y`&v|ZsO?m&;A9CygwEuak##iU{2lwXqwK(C#`P$IF|!2^+o`vnt@? zXW&54==#mkAg#nwh@O0VC60l5wYVobU9#+vnS%=6;_^o*A%L}&U^;PsY%c_#oY077 z$G3!dmY5`}Y#EB=xx8}KxGC+3pq|#qc$8Dw{M8+IZ_I%tVQ^bl&VAy6N0}ulr1lkh zKt}Jj@qA*>R-p5c8M{x6m|08ks)P&^kH&G)X74x98O6QZ8|i!e8M7v)X^L}jN*{K- z{ES$68rW$OwCD(53#A>$9)TQ4WyYwz)1kd)*omV`703S-ope(y%m5#42Ffu0*KoW{t=DubHdlBrSi9TWH!Jee8EX z6#hjEe)zsq)R6lvHU9f8T|NW#ugq>XH+^QSP6p-)EDBp`&MD!y#iP($(Se>V41Juv z9wW;9xbh}MQV%Idv-ur!pq%y2um1WUkWP(PVWg(f4Mn|U^fsp9)?wqh`R+-%n0fj&&n|FZ{pmQ=zW;&x zciLldyFk}pJHqbxfAEoz)Mhus>e5763Pyi6NJr~+Wo|(W8nybWvgDca!B4PZ>+CaVOM_ozeS-qc$JaWH? z00oI`ErVnIV&j zo1QzuaQ-LfKaQjQ|2&9V)+Oizwr!mrqVl|G;vK|y7je>`P%bRn8p&;Y2HJoA#yeqF z5i~k<^6V?*%Vdt*u9R~);OvQA>$sjG7+aWZ8;SjhpOj|d%{lH$pez5qsNL?eZnxBz zCJRnLWd;Q+Ee%W^8VG*;8prF??LcIPfG6p8BYa1Xs)l&Lq$;iSbMZ~D31(}dGMG+U zn47MK_VS;M%)us5-~N2k3;!6cZZ0Odj5L8qf#o{usK=zgD96a7&uPEAFazC|X9uw6~b~uAgIhmVjQMEY8s~3|-*X(Iej}-46 zu3*X>l-bjohvoX99$)3PvpjQUEDYUR4fO-oM+-DSZV_3T`zCB3J$r-Ct_-K9>704q zvDvE%(>~FBM*IQG=i>%|u)3aqJH}wtb?u;=Isri3t(EUQq-T)BG=+OWk3n8YL-*oi zZY$^#yU*TfgX~^NEW3c=^*kp&?JXttLM;4CXqK(^T1MqKtv`^8j=i?0mTdGGuT$_H zR%^sJ9x9&?N&GDyR2`WL-4{sp99U0uVV>dS8#eW?(c7j*HhzwmU<(i!NPWGMnl%5E zLCYPfnFH~e-u)5*AD7y3gGFFgJdM^n?0b@&!Am6RlM!61-y^tc!`If7>gVL(NEd|l zR|I|KcYrNr2GG}Ke%Ix;2XXr!@eaISxwC?Xki(%IXl1(Cqt45?s4~sJDiLzb3VS(S zVu*dBhW21etFvfw(%ED6I#2r= z;sVM4nFbi6X7ENL{>r!6-zqO6ecox%&+s>(cU7*~0M37&pC%Y)wl9acjKE`qqKI_o z>&mVi_#&w4fQIt@Ogzc;>@J3VXV1L6H1q4F7QmcKyy`=NlF!=B45i5@XXAx24%wtw za^K8^Pmp@C%lS1NfKHsG7RU)%g+|luI))LOM?pFwZUaPNtJq9XV6+FCy+H+;cz-q66t0G;*kVo2ILpMZWFDDVf5AG`% z6~pw?)sBDM$CQ(ZcIjurJIU;}!}8Hv#X)iX?1u`GaTP_d{5Y{aa$REqul0au@Xc|I z$YcKPfWkJ-&!tjAt+qEZZ&j$=%vhzO_MfZ=q$L{$vZSpAg(#C34J0Apy!apcT0?>C z@D&?fEDN~m_`4<6AniW!w2b^Ts7+<(JaucAHY^PS?!NJ-t`p?j3uBmS1#WZyOcU0z zw3#{5Ch?_hTGkB$^iY3oGvzpZW7{wVMnfJ8G7PHv>tc2h0Bn@fp4eB4y$$JOE_ph^ zbYtvG6KMxD%$L6XWj~*h7&K8C$G&NXPINvA9%#Abn8HSR9yU;`N8Gm`x<59^i8Vr{ zUwYy#W7x_f8ugh6X~%ilR{#=2TVj?AG4a5)g$pXFE!GLX0W`=V;@m^fOLp~h4UEU4 zfF3VjLfrH%T?enFzd7)5VmEC8JW}a2-)%&lsAHp1hAS2DHv=^1ETBDM3Lx#4R(@P{ z9BmX?^~uuQ35U0vgeR-kR#4(FKFo0MnzkJJ{MT?_FvGd0Bs}gd6Bf~uU!aEeM04wU}+0bmB5dSKbzQd_+HhKx{CV*_4n}n`TcfIec7+{B z`=zCiw$#OH^sB_^hVkhKzR>#GIgxOFtDUy4#I01UJi!eL)H-S4%QgqpKsb(3JWGwR zFVYj5%7t}vPg!2(EH|q(`1SG7^yVs)|8!D%cuyV<4I7;8O5J$#j>AOu75ar@!{Rk`M_>@yQA zn|zsIYtY~PRsp>BJm5&1$3Cx5D5YM1makPO74T})h>SG`-_?@RgU==~X^@Y5lUx0ixEjK zl^8}beIA-D)Q1(FQhJw?$_GYg3DPaC>e0(`y>BFXBX1zaL*i4^WLLo?SSw)g!Tm)A zEv1e8UWK#&>fJaB`h6h=$PCnQp!P<7yMlm?^2!55_*uY#+1@SOu|-o_qnYJ<*Wv-r zuSlij>X73FIP;GLA7;PbTz;&1igYFD2PO7}&8Dr=5}Hors#uK7j^$Kzy8B#9z5Qp{ zgz_oUAeQGV)K6c&hb(Vu>a>+NoeDGy9Ztf4dG|L9(iF(GN^dPqDPo< zqEo46{5Hza9OL(%uvobe5YAJ0H>)-;t{S0oPPIdKDG_?NT4-}}DM#0pVF)h+>_z-< zlFEK^vtUZ9-`H@s=U408D&(NW@7nZ=QN2$CC@}xTs~GWUoGv%^MmH*v7e7;ib9hn; zUcEYc0g%;N;jKD(r>uq;IG|IpOAT|#pYIQ{aDY7sNxcSf7+{iBgEjo&dOhdA`4*tM zeKwLnTLf|_hDeFzYgpZrt1#u_*t@~wd_UoXFDNK@6IQRa|HKK&`pi7~uc28s>wfO3x$xm(9ybmPegbj0=GQ$%NP`tY%<`Xa;UwoB7YVa!uM~Hv*q89>4;kEYPUUS?_DF%_6>eeJxwV879}Atefr@9f&T$O zKB?g2cV7kR*I4T?`<4psr;5w}y40Q<974}w^{&^eX zlf0hBrKgkawQW*#-ZQyJT`=(2*wJd{=;9N6f$FOZ9d39?bH*W8JUNBOEC+lJ_F zzVf}gqjd5F*W_Of-{Ch`2I!3-)H`r-{CqqaArC+OGMUMr7#)Z*;?vl0x%dyuk7Btv zPPT9Nbm{&X6Wf51zA#4%|MD@8tSx{8ITpNXkp^mX9KqPxK#~T2U4P!S?I&| z+36oX=8g!~=(6Q_P*0^;5X(qg&v2{bQROQ2wOC_APnx!Rs>Qa00?uhEp?uBx-Rap% z;q|lAi{tep(%xCqEC&%Baap(9c$U?Iu?*a)!N9X%$9v~Z`YhQynZ32{er*!`)`3Ll z7|}f|)WC)UhrjZ3;ANkNKiWRK1GB!ktoq-UkeUM7>)07*PktMk_^$+BkPyIbA}@je``{W{U^+E{TUf9_IaO^zS42p3%VYA9QmEo@0Nnf72zR` z=Bl^=C$OEe>RIQKShvra_j!zp;6e?=hKn+{{)HI)1R&koBAF`n1!{ zYgzpqu8dr#QCyyv+iqTYrb5V;?Or=dSRaLA%e)M->HHNg^jnhVr$ISuPsrEZs%=MK z@xS*gK`&(L#pp5MpaVVr6Mhm3>KpB^o;D%1nwN7HtkWZPNzEL5&qEYchOk0*(}^Ey z0Rh^~oqK0Iy%bt^g;((w>r#NVq&;UOETW$IdYJ&iqYukky{4r$#o^@HQ8wGT(#k*t<9G2t zB$p)u!^8>^e?JjYf_>xv1y{nR&}{)ueJi6i5C2$!m@f0HHSvk}lOSkk@U~E*W;&$^ z9igCxqhUJK&;Bz2ZBvGqLwl;waprx&#d~bT0ux%#-@tO^&aur)sLaJB#dLmiS!Pdo z_q4wrThrHPz;Em3$hza#(-^|d5KYJ%t>5xif zgX_032rc)~yWfK^AATg9Q{~tFw)KkEy8oy3{2`laXRZ1tKXtuT*OD}c*W3GdnEHKG z(q#qEv|djH6-6S(F?Q|8)8<+57G^~!PB8Q~XYGbO$Uwnp5j zYnr#uVZ;S6hA$qW%bSkyzkbJUBm{B&b@0IWwQ2Yh z?=$WZBgNz|Ei@MyskLf=wOOpx={B^T>=DH;rCQ4=eI6q`k$lUJqjIw3h1q4Ip%a#MYV3Gl1sTwyI;-BS znGQ2N(^a$;vip#I&9rLX`Z8Ww+uiJbr8fNuR!iWf=e*~!!LW*)^m}dP;+GgZGX-2- ztm*EZ! zKj~WSt`7^}aLtAKDWzBSa7Ah1^^GGAZn{(V=S83G=C^h;P{(W?+hgW?9$(wKHUXU$ z-*9MlTOKcd*wY1ZPzk@)$=iO>^JUp~A9nf{B3-gUlpCGovFKWYsz zldNBl=Eqno?x|4M`?}iStjw1C(t>`BSrM?(g}*v$t{K(NSPIC#bjaFLbxLK8n)adU zHo6#Z3)C`ezA3>eh<0-vD`HDV9^M?TsRhFTrw%;9-0QDx)>Z@NIsmN&@tl}GhEA8g zzxn((iUzce9m|-0oLOGB1npWdgJWc_ZAfa(ufAu6E|!Q3bRtcm+>YFdKIs9kfGgYn zZ9~9j%^X*m{cSn;a${rH1Es^?^8w9wy<;%lW{90KWw^=4^}6~l16DVn;ryQ+Y8<9r zX3rmeP`gKr_o-W0iV6>pVeuGjd8JjaB~`Y7ef%j+x;}@wh6(>v-9NBdHU5@))L_?S zipII6igQ(#>GgBTl}@sflJe!b{Xx))8>l#^UZT&E^^A5vV!2Bx5u9MCbU}P9Ya_=n z;{H7k3cPqHE9i(wTOW<|T?`ckMz~?K92jl+-830SNtGT_V8quDHq1Mws4)YFMvUZ} zqQS(P{!EYBk+4n0@qekf-;&6y@(IbkON zn|!6z9G6=(|I(*!R*v4)uA9K$xN!67jMS2p$LZfPWn<_v6I0OL>X$)pMs*|h^jB0J zb5g=f-`z1BLQ%IxXsmXzZAi;0)~|$b_UD~0^)xy!J3E%oho(D{7)l5gHeBA;LWzSt zO%QX-l=7>BGv_w6hck5JP~+~Ro(N^7^esWpzqA$39U^~c-ZR_{j1f{)sX77r@#{@B zpI1swMh;3}&?&v!f5rc&-`TQTc-p>0-t%7qN*C%lcJQQi$j3)Z%S`o~)$G$SK$+?^ z7yqXqIkW?}TRoCTS9{r7fu{`geEoGI=DbGU$HM<}h!R%Sf1)E!e#sFwxv(UzO3YF< zk&POejSD<9^o~IzbmG23eL{Tx^Z1+^u&nW}1kL&Yrw$EWFEL@nEHU4&Hyqzm^9?CG zLi;_Nqk%8HT1xXGxh)Y#g{@kvf!}njZkM-ZH;zhf2#`1=!flbjxd!%Cnve0U;8i0e zTv#-E!+oA(V>H!RM^v>^scC4fNS`S0C7hS&-baBf^a7adJ|p2Wgd}9ejQ7E0erm(1 z)J^LU+Q8;v0p2(I8=}|*)ZNC4)+z2+^mzUG0;-_I+KdrP-x)3WDi&-$vX9C=7c_tu1Vs;ym zE}L-f;=<3^fP=G+$-T1*qA|s+=)vu98ko_xjy{pynh8LYQt1zLtTABKpLL<5aMEGz zYVT+rJ?gHWZhA7ZAEf$;*D`6bu4!-X{lK48o7g(o=bL2St>MeA*iCj!uKg$qJfXxp zv(Od0ARq+cd8jHt!rwm$qddG>2<7Tg9ZbB$rQjrMwhf)6F5v4YD^k~UT&BTXre16U zPRIg_cAbEpi}0Z>nY zN4*`XMV5?s=T$jAO>b7C=gUXv^#G(q1QR%l61@?04$+M0eOnXX*VDR0qQqLi`@ zhIW^7CCdWU@#+_BDE1}$oggWex2Zm?^(U(;kF8_m^#kO6N@Ejuas-PKkz=xfB%MZ^kf?ocLTeb2p*J!0_IH#X8W$@TH6y53Z zyRre{YclSZx76rmdK}(wlB@m7nzs$;7OOYnH6)xgIh}ofU3&Cb{*%=I;p)8q*?z+= zZjUO_YKt1NN2^k!w#242J5eL{_*%6Eu@$vT?O8S2+Ixmtv13*2P1F`*gz$KtAD_SB z{@kzo`kd>W_sIkgx~m|6ZrNp5S^`Rj>hI#xMO%Z;U;9s-cBM=|y5RsIp3enc;iKX7 zC!+qK?&qkU&9LthREhh1%K0t-#=p=wIWkz4UJq_v zKyNXYzO9T+BdlG0kw_=qod8;PJL5t4<;*zTw0XJHJ#B(*Tec*#WTnVjoEM*?FBM_b zX_o5UWF$9$3}}t2RzRwG7@YBRE#Z^A7or#9NMx72S#ttp9C*va>m4r(btZMqo?=}8 ztk?%IomgNG^*g}a9&t|v(WFD$3VnFC@7kuRqMO%17``hnUu*|@I&m(_J;wbri1ABf z64_uyOQ%k41s<0cvoOh$!$YX8%9 zNq}4*4u-^*2iN;di%qa2+cpdy`8?Ulm9)`563-N;i7Ct_%=4C}9X^dijMb47{Oic! zSzRe-O5Jr!reB2(HpaXS-50bCFfy-D};{%0wt1A6Y#G*ea1Y#;X)a4tGAaezs zFOOu`@&avKog{FGDauye*Qvl0D-Ox{|Tu;c7EPAuPVykq-Yl2-C?v~~+Mo0piM{3l0Sz4+j|j4fYo?2-Ez zCSdz(>()|T=PS}=z!vu<_{FKy4HH)H=1N3mHT$5O6h7K{|CBP7Qz@Tzl63k&07$UU ze5*Uo{Tq?K0~g{kZ6o&sfmB^PBZiZXa~#g|+_PA5yTse0^A$VpZ}zEeGl{n+fE>NR zZPG;btYug>SWBSft)QsUH)Es1foP$tm}ZjQwU5bIZx$&qs_KV-K2IHRM1rFX%)$k1 z)?TWOwUN(|zYvIqrNMGqgM0(0+VrOG>|_2!xQH^2L!}jB*~seWbJs9F?FH9u3jr+7>Xc)tBna^?(0-nF`P!QBDdRvt3+KnBM` zbEpZ%5OAI+II+Xb4}=YYgj`b1=SNU=$8AU0mU?)a+$bS`w9c2N{OrY{7XLv`V2WSa zU4j{ka&!*cEwuJAhpn$SKL2NX<*s0~6oF*13BjcTA`m5@Z5hq7dy?Z@(;4;H#q;la zzRnG|ZU2`!_4ir%N-yz$PVJXZ8e?S_@jC$vx6`1}{Aw|M?omXDCp>yhJ+A%uv>op6 zT>-iA8PvbNT66bpTVvR`JYZiu)4ln3gt)=CBks=L|D?d56?*w%sE4jqc*u)nfYF_E zHDu#)B5Fra*8uzvZMC;yItWUJ48Fb=BF%86_r&Pks|8K*B{-NacMgdr4uAiUK42X> z9A41;&{#d?^*~DVQH@9Pw48#f1miTT+>Oe3mp}>YHoWN>yTH@i74+;MDbI+~%;+UU ze(T#WjeWjSkz?tHLt~1>!C%0xf06dA$i&A$pY^UOPD%P{qP6D?fL-uOgM1(5hegAw z$-tENT)<=p;K#(dSYrEcr0G-DjKq?hQanRNs*>AEgl5%Myr=Z}BNs`A6<%FlvfuO) zT7FEK66{BM(=GkP!@gTAqcN!+tw<326bN;h01*#zGi1q^jPRWPv1UpV=4hdmcY4S3 zElAY2FRk&%`--hWidQPxpe@dwU&5o^`{)x*iFG{2$2~hxb^1xMT&9z+9nqby9`%5m z_r-nKaH=!C9qMO3vfn@ET;XW8**Z-F!d2Rpaxx9SB}pk^JGosXsg$Raq$F|^emf1O zI*hgchRhv`qTa*ezegU!N*g#0mpt+ZK{ zEvVMikN*<;=uZc_6_49brY81xTk0FFXsc&8HiDY9s>oF*P$zRIQzoyanLoD5hzP*g z(EggLxL-fqrB0wOgM4H3rMO(TEP%`-04 zn+$p-;kAmr@Hk^s{pT1Q4(3#BpEJqVM$O5#H$TXWIUy{$6I_Zmx{J2Zudg?V*Fjz(gk7^KOSacF___ zlD`dWA-=5k!e6y;H%NT6*4Q$aBY!JFt1f#YZS_yXYz}q$Cg7L%h5K>87>08tOCUZmM;hE~+6Ki_(t!4EDFt<9Cxl;ob z=E`=1f0n+~v`WDOE!o_cR1BQyJ94bHW%yISsS0c$aE9-A%J=S;sSFsuOvTZf3kc6a zw%^>*Mf>%;mtWooB=-8ui%54Hy!i<3O(RKM6b+q?0gK@&-Je7lN%?o2-??|tSTsUC zQorR+1kO6Y{}9@7T~7()0nHY3x9>g5saM`|gzR^HF~*y~KP8G%Ef*O4qVUYoRuQ!C zyegHve7K``NuJ&duh1VI?6ZR!;%jLcj~J!!?|TvgRN&j*zn#l8SHIb^QO7=Y8v{SK z*{W6jTIq*odG`Pz_%hxStsI;VC;uh`vIkT zto$UAVZDs5=+bX1&mxA`c4^&^;E{avG<4$1VEtk+9*|9Nm(4Xx8Zq0XZ*fU;0J^1Zy$LefxckWPwX zQ;hsxi{2{Fcw!XYJQn)y3DZs#w=Ob4HMxsbh53iK&Ry47dDR2uK6m(f8nG7Twhd2e zTw9G}6xm#b_2<5V5xt8?C!kL3XILD|>oULT1;`+dn_7Bu0J?Px{?8vFuI>50IAJ>e z#WpYt@ak~BQ$P3e>TC6^KKD{nh{#Vuq?{;+q`WF2e(QAZFT2FxHZ# z2jGw`WBGxepSa;a(VDgcfzlVbx|^5LKG$8JlWi}txlYf~t&dGEYE59NcLi@=nbMzG z_C=Qd%k9)$&36B2b1s-`@n&7Ry*@@|6FUy+vbR=?NK# z-eW15$%;pPQ05;m``0|rE>cQ+D{9lqP;=QJcYqrVUk3+%B9RCOR+vKj(rE(*JsHqILvXIjX3PzkW4{6 z?k2=lLcv4~qHMXf*V{7^g}e0qWeNG{j-BVyi!@Heg=v&i+BNMj8NQ_^_#m0*t{=Y! z;AuTcqMA?6RE^bD7`#k2X;bL~Cbz71*WZ< z-rSjsl6dpTtMOjLx>BD7Fi4(yE?bHS=*kw);6}Uh`<%&=Oq4V1Z;EEG&bs*lk?HzQ z5*vnXPMw+lV)Pz?dc}NE4H{sK@?X5*XvMxBnfx~?-O%p~gG7Q=ghuIP9YMD@hGcI3 zMG`03kk~}B?v-(YAiDbW#eA4segv1Bj7vU^D5#1*jmD(wK!#{17@TC2-uow_+yqc8 zKWV~d8=x`duq>YsvxPALG~c>-HkcZt#3#Jz`H2V8TsJe%D){2BChE5xukGDMF1+;U z5MPYo;i9k+fiE6A8HnA5mGPq}z(6L-A+Zh8@xL%Es``Yvzm;K~8(@FUG2*7= za_bro?eU3r@L7Uf@E5XF((*a4#X+yap+5dE|G$bz5>dQ#*0yvxz!el87Al>8@{t3W zMw66|A|E!MRuFn;5Y}iW;@s=EXQ9egEL5@LX>Mr>zjtyz5s#H1Q z1ZMk|m~U8zPE{~eUd+WZV1=-%(igwo@d^aN?!hH?ErZ(~0Ap@_$$qGpEUNxq_siNb z{}Pd$otO3eWYhC`0j)wS;h*t!TpanCjVhiDJXdwLRI&=wd7e?N&RK5SpY?-fz8Ib% z#G{9)iiHci?Q>@$G~RChWV}!i-dbB24M%MmmzGK@QT=ExizkDChv{c%w$#`))I}80J^}q10>f-(97>z9zeHB%{^hT}oW%tFKJDsVxgWFRATp!-A zYJf>56g=?1%|I4<(#i(J28!PhHFW=5n-nZfWt8G9UkR@K9U;{HVkuiCfG*mAB4qwK z*=z&LaQ?q&NB;>>!Rf`g!250erSYtV={2wr#-<+R%Zo=1ns5H@JdrN(jB(tA5Dp(x z6BxPJ)7|Mf7h>-}KMoN**p<{Yh1+eBN=J-bFbDp;;R2mWZg!9hJJ^mZ&F3b!7U{Bg zBCIqF7c@-IboQk7-<|4K?TnEB&9QJ~I;^o3_dEDS zwJqec;UikZ`pyC@uzMWraLyA&ig!lcAHuftbCSxD^@8u}3*5_7b-!J}-#-PuyP^W}Q$0>6%_;Fn90qT@YO z>K3VmrcvXfHMj0<8+VALwS;V zlGOp^+6>LjsGvBlE!JtNHOXIBTQiqo^Qn0ytJa@G5R7)aEKCYj?>G1I6%~1|3de;!pE^c_1C#35381ib}t+~KZqzt zGchuOHdpkh(x(_LgbvLp!3zOEa9%92@0J5XJX~YZvv+etdn>A;t$EQw^Zwle*5N8hXMOcJtl6I?JKMw8F1Pqo-Mck^p2@Y8fxuukmoy zG9KjpK^Of=e<~yf2P#}XKJ+MoUdMu9C^@Y}c_G!nkM1Rw51^szX6zA%tePJI9c$(aOD;s(`5@`OC;M%-g>34@JH$8rX+)jtDkm{y zhiUU?Jt%qSCC^V#SoQ+t1f~rs$mGC8`ZJ>B<`2;P#Hd%Ey3O(CIKxdJwxP*>#g(dh z%PbY{1HCt2_M{4gNjF!;6_=l%^)pTe{C3>(2kU+Wm{#Eb+0MH1kAE84-&tSuvTI9h z+g@~aO^$c6^TQY6HfO?*(wajYHaugPR3O2PHVYb3A}~&*h?R)*1uqyWT+TTz0(QuG z-!n1QqV?SKfw6-u2k5yo&}hf#C!pERPa&gK48XbhDk1X9$o||%&=rhcAx-ejQ^Lf( z?iE7%VX=RmOJFN6Io>;m@m{drWYnK;dvoUS`_sKfjC#?_%+S??j<##&B5J3ktN+5m z$38BzFG#jSTiI&-2j)G^?l!VyP?ng)FRz_Kk*}sm&h}Z6?z+ZPr0h1EO+Vp~4Ux~! zZ)PA&AVTmo19S03mFC&Vn>YuH&bR_S4~0jT;tsUeAE`NAW1*Y@d*_;b#~#K8a@eLR z@>5D|zQb|3+z-VQTR=jJd@H80!6&0qp*5mEQFS~Es9?=6?l3(fO);}xbM^L`g9K?? zE=#HR$et6P@Ckpf+gXEZy z@)U)81Him3&Ue6REXMO3QzoS!nnt#hsmO`|ex%X75#F1TcxlyckR^4pbKb6GHLin; zQk};@C#ZXm70s6&Z#4|<}2T0)}u>>(^36d@>NL05tV8A zwb&o<@%z8H-&X)J<%aLNIV9KD?7-%Y&{ zHYZpg;r;D_*{IX?Z2$98#Am78y{3G*-jgYjAHwUBx}Ju3j%c}3=4R-?o| z19r?!POFCtcKac?+XPVFT^L_G*;45N7viyhbi~>zk>??G?*e-sqUTw_f3AK(c#7MV zbo(RUpbU^6%_Gjcn_oE=dt%UNjUg#UF;l2!%OlixtKOGe8>>hu>jYcjkfQm1Kxx4l zgK(C{+c*zx2^w)a+l$-2Ze;}owe_F15WNnE0LCEqL_-0;dks8Cg`}oAhpQuQMpNGr z-86qtYQ~IVP5**X-zYYgvO|L7@he<)cEk`!;K25g;vHn7(6_ZZ)T+J#X zJiD%IXv>4W8V;goM#f_s`)rYE+FR_08UdaZlMIa)uhouc7@>?<@I6X&pn&R<^(=*NtYHu9aeu%56LoDs>}z|I&XHAz}WS# z_ysw%m08akim8Pp%w)9@jJaK z12HEO!u@9Jqk#G8%%pY?9Ww?C+dy)vima>u@=5t7YZw!$q=p@E`iTqPNNI zac>1A=auon*HE5(SpMNo!?$#+U+7f(CiFomX~3b|xaBwvRKRdY?DZva!c=S1R`jQG z2KCGnB2VWMF-kkt9Ze(cZ1QyO!B7qM4^?36mm0=^$%sZ56XmkC?{L~NSNmYs#~BFo z7U{T5(?DomsY3wKUV{4d%U%|p44L5*6Wak5Oq1~2)Rzpla*}=OIwY5qF_L8w!12%Z zPOwLt89Fmt!8!r(4CTq%0+0z?s-mz!HLPRZ@KWv zB$ym%&k1NEQCjU&4#14?g|V(N%gHR#veVVM20dPVI$1Ayttfw3FmlTxuqhFZ(k$6K zk_AvqyWpZL*(z*zsYSr)6#*StiGI+?uXpR7rP6-1(YxfWayKaFM82o0PkBl8EC9M)-NbNseCw{@R!_f!<6lQGRYxvEpV2rQBYlHu6l>|~% zzY1GFQ_|Py;13pLs9wDA7M&?)&-zRx#}AFHO2V<%mja8IrOTI%aF*rDR?d)*$AVC~ z9ephZN|ECGBbWzzH`GTltDK>;JmvsGAuE9Swa%Q(B$(N7n!JtE>W}jEDe9cIRdqEI zl1{pROBrw{_mUj;>Y#h+EZ+K;=9Sf@)_8Hdg~Jpx+wb*T717b>GeU2UdlDyLerL1U zp-2w^;4g1|%0Qe~J{=2QsH3tZ-na!vNHK8%Pt{Cw^_9B?ksz0^l07D;v-g#33HRNo zaY64L2PpvtjMe2NKN69)lvaw<`G}P6sb~51nc{!s{Rwxxl}3^pry@!$afRi-5oLOP zRq3uDp}*-?^5wVv1ioCU-W#h|))JJvwT*d+eD=()G+S~L3M2D8A(yaZTw_v!Q;k(! zDAm`+ifrtj26x9~ZlI(F($<7GexqK1?Z8hHaK>lu?IjNXJQu1#{U25=X8Y<#HVnlF zzv5XLx!C3tn5AF(e!3cL3toF7I@*5pGRP}$0$i-sHp-Y`-?u9^Qb~(Wkb&HNOKGiC zb5}zeJsa7jr^^!;TKD*KRcx2e`Oo8Kc&2L-WFzq4XwVZG{ui^t4@EciMQWiXujca=`f`tMhMKBO};uZMx{q8l$$y#>WhTdtN)ge=d3k2za#XMiXQCx)iS~$c=3oy zk}BO!1I8i=5H2OPR|AY_zqlhQ^I)YeyM=;I1fU0^n&U{Sm!0#xS7{Rc5Q5%uDt)NX zt-bVl8O=|x`p`M~o`rG7aq^ab*j*1@D&p#|fQ6P1v1Y4JfY0p?f1f%N;xBmknJe+XIuUeg z$p<}%x{F60-~*j?m`yR15Tg$5L+&y7r#ro9sAN-AJKt{bJS zOSpfuO538@p9XaL=)n@#j)Snts@Z())1DbWV&qMCj}okh++rHQQ&B$8VZj%%xn}@w zYbDtd(mg433qws@=Owe`V6|NS-Vr}`*E&O2>jJl=nj!*=;fBbe7aNEMYZi#?lhBQ~ zO(%s~X0RI8a!w!lPh~mz4Tz0rsjUr-U)b{kZZ6Ra&rE3>C`(*iqjNv{&&E^k;2$9` z<120oq28@+XczEx=Mw(Q8smvCl7HZANnY-F-y7(1S?^-x9UM)@NY@)xWgsQ8wz+B@ z5gjiG7*eHK&LF>&4A(}YdecBd#q&|%vU0%8?)>Q86&A-xFUv2joQf@HnrQ{>hi6ais%SyM9LMiW$MfD| zRD3x)Vt*L+lKJ8aQO%Uy+AT>_w8sx`z1}4i6=2>HNL}jaO|{|ETEU@>#o@JXNg7;x}IMD;EXB_f8Qsk%py#g{WYLF z`<#_$(DSpSTXr&#cbGOhLNWdGgiXhhTO27X(<=ER(1dTE{Zn>{hLzXx=H(tA5ixtK3dj@$8$>9p=e_c!ASgKvjdOTy|V=pEUm}cjCqg6YG zaWbzGXKKSGoP)vB*ux1+qe#&{iNRdTbMopz#%eY)>>(T`U-uVT&e8aN&6tY!+=|wj# z`ZjVt%<^OHg-{aC_j%eT$2+oCg>L&BkwMHT^xF&3_k%ALuk9F~s~tXci*!(6 zBOGsRU3M0vdVbrQCI6Cze_#HL{$4q-=>CYHHY)IU$#?t2wJg#@Xm1bRRqO?xtTeQ( ze>TSlZ={lw$4*dk+=j(vYw|VYzj0w`w>z4@di~A~3&cN0#A*9b(4Q|{vpx?Vwq&?a*^!1Y|F>?czD#mMqr>W(0AFb{JqrSSj9{J2j!9-Z&5&G637_;!_LSp zk)ER*rIhc05}amcDIkTOZ4SoIlFni8GZ?Kwf_SiN?pwjvTWyvPp&XP-!jvgeJl1og zALKP2%5OHVd`yBhuAv3GQgikgqH+<6?*J;9?91&jRqn!Kw9onRU-c>_u!qcGqMIDs zT!)pyyO>$Qcxgh^^tO)DUU6dOV08arE*yPIHsWwN?(Jt_7yK)XkOd4v(Z`KFvm^%Z z+b$Jr*NO#f$Z^^kwV#`|Yv$MAv&*H`Zt)rY;@Nn~L4t^53vGwX$~kqZmC|=G24L&7 zsnt#2Szoa|@xGf6NQ?j(C}EiHZM$prSu`RtdsdP-5fTd(P4TezmpZcEj#2WJe~Mx1z~{(@%1C)cp>cUk`4YkrX$T0(jD(%EIUGnh6&mq8%AEq z8$6gLq{Dc$=W#Ml#0QSH)6J&L)+#D%k2fLKBvYNb$cmiK1~v9=vKh4kKV!NI-dMfo z&SQ_xosdNd4&;uV=CrE@M!3OoZWnZDw91*!Auep8AkXQ;V@50Wg?y}ivEdo@?h9gI zy#l6KIrpVuQFy9|`_}-{w63Ebq*Pm7M zEZ?~Rddn$r`O9_7n3fM1dmxNjJM&*(?lnoPI)bOjRLO+n>@oQ^ZxbPB=e3riG;SfW zgI6WcZMVqS?AC(Wq;|V~`n98V+a<#;PZoe-x4hK9A8`jTZQTI)H$mZ-&hXb|Mvc95 z`+-u9DEY?A%>@z9H|(u_Dpf4s0r**`azc#pl;7Aq<-a!m%o>s=c< zcU%n;q6ODXx7{K6TTnpcEiaWHuk0T8-BEe8N|;nUY>C4=MU~-INYSynUL1y7a}e?R zV6+KL^--6Laf>l6JHZQGOk}~-Ac(Td`LoH{Y+^f0+cIWK!g3-_j`%|}TDa3at~WDaZsz1t;yDE>MRVIw%4`b^U$@GMGs_T(Y;;|^tg)f%5kyq2br?udq#*r6KX(w-Y z7GL@UKTyw|J*^xw$6L+uP9lO@6BKlF(=W5`RhJ1O&xGq)B+S6=?X%UTPv|@Ns1MBD z*`$yzNHHY;uR&ju)d;QIdSxe}(Qj2@UjN+DqW~lL^Jcif`Z3dAxPPLSv!-)w=u)M2(Awee+&Ne1-m)4bkG|3x$o+Z-?{fT zx-6RnU!XprC6ES=A|L08KYZm)n1#0NJG(|3U0K~qjX2u*+d=Y~-0|Mh z^)m5)jBIAYQbcD$>_wR{ZH?KSIITX_afBX;J4w2AqXEOfS0aDm-i~DQ1b*tBNPp}B zzS(Ehnismgit3~Scn*iw)GQY1%AaC`qs0a@ucmB;M@P|&g|ZNI1;2c^@&nb_C)s$# z?;D>-(|xf$ul5vXx#Nwu6%9LT?SE6_(tU~d`;|>LNfpHWzNY3G4VkO}O`G;k;)81tipDgwQfc z{H@ZKv{%cfTDa?FnNZ1s;grZa(@t`Z3ty)+{%_D3%)|Nwx=T{#Cmb2-v$E&IG2i|D zS<4v18!$_lJQaKPWH8Z5+r!n5TYy=cfgQDp z>W31&Q?~HXUou6)_Wf=vhRcPFl(kZ39L>Zms~h|3Ust&M>2cBc*Q;_I#C9~M z9HOT3_uKZny@AnjN-Q<~gXWHib!n#oO_%mXnkS&@gqQxv?XZy6fLJRF{Pl@7o)1)W z-uXM?(9Txk$mgZHUxy`-VmdpOxRu_XCGPYO0>{v)skNuo?j1TFs~rsW`p)d;m^#JV zJbXbxXhM2d9q&d5oxoW;8jf6js(_YfIdt06Eir6P2Y*lif$1wX9b|0s8V%rF!#3wo zoz%VNA#f}AB^$2WCN8~pc$YG$zE_RkO`A0{55(us`P!0cn=$(Ql>p^WawlBn5%U66 zBmJz(-Rk>gtO%Ds+#az{oS6$ZWdGuqtYw>YW4apv!ky6Hy*T*%PZ-XH#W5%viJov%43q~?0#ahiJzT?MpM)a*uy*?o_)n$IC`0f{ zGwSk{PJ5Y+aZtg*YvMU@O5*z9^2o!%V%19wm*TfAmYBfSlXX;}>?3cv_`B)} zi{yB~afiV7wW^)UFKpJYoiY6hN7I={R9tQFPm@tyFG1lxu1ghZ5k-OcPUVDmZA#g) zU%?oFV@-R>0nJ4WDeEuQ5AgpAIYIq|s)Qb{q!&|i3fgltSW9%}f}L&^a;4?ub+6O* z)U}zpV!Io|T9WWqSLreJu-KSyUWd|7;!TMrz&6ar=#43qD}0+fQ_is@{uEqoSkPEM zIc334@o=bEzK>A}GZSY91j(M3HrDrZ<=>s%6e}PVhh}VJ)Li3DO(u#02(K4LgFTlQ zCZia4NeJ?l7wNY@V>L?dPEkX6?AK7*G^2k@Cdz8;AW8l{<(pRI%4Q6s`-md?b@-RI zvE1*--M8mJ&kTpx0|mytBmnu@;y&H%NcUX@;eYH*$z)Dua~}IN!}Y&M-u^SI2VL06 z1Po=#5N2!*I1^m4kFD#|6t)KOwJE_U_sV z&-#kb^&C6`>`BknI(&OD4RR@$`cfEhvX<6}7u%$vYd@NQ5jd*h_uScx0Wm!wmCwd1 z$q^9d15vtI`6)$sMNWdZ6ttcWRhS`hNhT4 z{mL-Vx49)YNmjAV#xbvg>n9ISM%m3v+_y9e7UjF!{*tnt|Dkc$Jr>^x*@uJ&U+6>9 zx7-<-Kk)62g*I|-+6NGAW>FZ-fper&^(Idbk=NU$i<4+u$lM#(bN02e0NzJN7bmpG z2?o+(scD@K1FB0u-9eJqVErHP^jsg)o%7c5=37SJZu!}0{*?pU&{a>P@F>vK|1lI9 z(BfjxFk0U&oG1cUG6VVN9Ga)$@=F6?vev)F?iwrinYa556fD?@;bZ+8zvIJ!Y>tw% zA;3pwVym+wsXxBgLs!V;YF{`+q!tXO_`5r&+R7CAj_PH;?Z02(-p~@6%CVNVB2wh1FEepH6 zn%=uGRz8d>oTC0UclKe$98!QUbn1C=b1;2BV12wdVUOaF7%%F2ID|FdnRab@CJ={H z|FWO<$KO&`B=9+M2=-ZglQ$A)bTd1(+42^8%ggxMDoFa(_VYr6#)bQ?#iCVtCQ5MY zdB3#bC*4*BZ>;iA*}M_3;~XhAGY{pK4ci~GO}JM1famlaAE|Lks*m@apE$t#r}__& z)Y=k0+wSsG&Ric-UBqz4RFKR#?QkLy+8-fy$@OEBzr;59ou-uO)e+3-%kCtk>k>B7 zrJd^b<&mN61~Ga6M6t@Ui&yx64H zx|@n$jHlJ`D*8f36BA(B-s2NX#9W0(~Mj(tKOVqc;13?Nmx>AWI?>YZth!A&{J3{uXgjtj)LlEo>tZ zN#I*sUK65LNe>eDsf4)3I7~Qi^Fc;)tA>Ba_H1iEmQbW~oqf;YH(r|5913eIM%6CQ z*yb4OH#B~>UA!1sb6UcerLs6hXiLiGcv0MBU~xpATaO^$&O46cDz4j*z$@YM8=MS{ zZ?s3(9O)#(t!@dWLHW-$u(|l1wV+Kp6!yQ9>RYUcn0z6pqDhByl0aZ4>qBzgPrU<) zJXW8RB@3o|7oo@jd8JKKTuw@cXb-;DDeGhuSA17>SV(wgWIIAlp4YaJ&<1+*ryLVj&3D_lXonrbhYGU5y`gW} zRc^^S2@dklNWt4d3kf}%#7c@YKNK&yfV&cFzPH3wUU3-Nh9UnSQ1b_W?BBsr8q93*6XICrB- zf&=6_yp`|dcZW|`ZW8C<^N;)7E=WA_WKhX`Y4xP~vLDoY(_HnnAz|+kdHa2TQj06@ z>9$uLhzm~+4}|%TMKeycp3|cpdF?8WHQx?SIbVAeq@T7}G&rvcU6T+};?}B1`6h0N z&g$FNg7b#u)>U3RZxPLwvS9MJ?|B?Y$Z^8W@cfV5w9v zX^sMBnsfhAd%O2==`b$~Z=bv%Swiy;iXw;$I`vT;Iu1^e&QL?XUc!PN*8+#@U%}!a zta$-)c(*vB)B*H_I!jL<{q^F76LucdN8c7HTbi+SCgbvcL`EaN#< zq&N4hnnSS}@r*TdrQq2qxNVSV;NC2KTG8DO<2`@G94zHes^Bxr)J6hGgKcc%kRl&5C>i6ZKh6T$L?*kU7dv?MUi5 zySP29E`;cZa?##DKjtKL;8=0;KqyqT42~NG&8A#e^l1g+`>wZ(}XBW^m z+Wb4Il6^{l7q^J}dy`MSvFk3dHky>^DLI+y*}X9<9fYWAS1>QNz3T=i-2x$gJS!UE zykc>iS1nSIIC5Y7-d;BpcA}0?v0T}8>Afe#N8&#t9<&v(vEgQPq6$~4uIydwnMmhl z+};gtwjWo4nlj}uf4Jz=@TreBzXL`lJiLFdM0$+}hA^I2bFUFuAMAZda4{hne-e>U zs~S=jSLKNC==}_br62kSliuS2?Tve`f9;ZwC_bXa?m_cfor5vhh}cC zy3Ymbapo!IgYY3~+tENnmzn_ct43Elj!(lc7cUKwLlyr__a>qPXT4@iKoMX$rzTq0=}n z)G!xC45BA53qt?WO@|&=(C>}4Gbwf|0qBu@rF&tn=Mc(bpQ&25O37+WzI^9(6JGCj z>v;dt;Ob+?B5PxpWEBgF8PD1AZcD^Tw?%ORjjK?B(y@fs@?NBeZlPEMRw*6i8luuF zDXD?V>Eyjay-#L$

5LgG54(f{zXh`(Pripa|W(OoD#N+DUtqA>|5zJw#eft0S~x zA>7U7ZPMSPS`_27^SZU;bty(T-wtAmHem>}0j|&b1(_rOfPrPvK$}l`Se1dD42R_3 zH)g4DE#kNrlY{k-Hs=U{*^S6_pn!J@Ua)>20z$Q}MOb*L%`JvPOtcHCY_kDhWGU8$!aFr&J{0EMbJIdhAY{I9{JOU2& z$ENHMj?45!b%Cr)s)yE*6=G9jbh6if2B(LZ7T{@pYs;w$ajh``_ZXs>?D;wMEk>XZ zUa5QU@@}aUPyR@9s1SbI*i7$gH`I8J@}jZlcr2>z1;TTEij$B zA?g+Xsa&K(PMC=i-@!cN7V&>ma88eiNoKzQfoyD-VdK$Vy1wn>EqeD5uYD-Z9qR4j z8d^)eT$J+dsWx0*ab8AjgwDc_Y0rEN&YtHe6l>=j>jF88 zU&%V;bfEgsyy>%k-G5rJoWob^A1x7<4MQO}nii*{tjY>Bpyyxg56a^8aeMwZ4*1gd ziR&Oxm!8J}V<0KE;%D=z>93WG(~pmMrz7^gkLbHdurbYDkhgwi*c;cOL2c0&p%-?J z{)u9Ff9eFC^R3Q7I{-7qPqb}ZmJgii?b4{X5ab*yd__H(i*OscGtaD`h3b+9WgghR zoqyc`v(xslpnaX6EH8{R&tHxCq;@jR?sJPH;-e)hS9}*Bpr%VpVbT1D0@kr^VIdVu zy~Jv8vBh$4hFV1b(dOqb+i68+Ap`5m$@v%DTEUg;OZI}hZdL>7Nc(9su?3}}h%(Qp zU#5>=3CIm5fkh^dgg+0?S;RWG4)kH9h{M5#-a?(F-f#${0V&hz;)qZ9;-#80)+QzIwWtO6IaR()(mrG)`hrgjahv6jwIVI;}c0@ z-q(h2dlL`4g34yLgXa$7fWahUERB$z1f-l`R*3)6#w|kbdK~~G={2O94?P}PgX?RY zhtW|!tsNIu^cXKHbgnUR*sCg9C;q zlJl||P3tH<6(yWFiut@pd*y@B>l`cD+2=gK6%U%@bVd7$;T$Js+Ehz7EiBFj= zLS()lgWfk77-&cL40Im}CGMI;$NaPj_+mP&d5fvmDSw%4fIqyS;iW#;AJFwrlGHn^5`s$9LvD zN<#j0lW6AcRgM+g!dKn^1t!%yb9Y#dAl#pLYBwHS(V^91PU?eYoeA!4>1ot~i4qp2 zGOu5&XxJODe`yC>5@uTHBHqSmagqz=iGP)11l|RvyyLFA(CGs|r@ZcedsRClJjjU? z4VI}mMNjpZFz+N?X)=e1WeBG(#N~WLUDFijYKW9aPKW9U8g($0_?6G7rFzc zpGUX5xeuRRMqF>&9hLnItucFvGK6&zs;;dVx%8}umhx*n_)vHXo#%&vF?sCV8YJALm;XrT1A zCH9wcgg>Z{=D#|iVf$o{$+u4zjbkP%gs1m%zNvrQ5?2NVZ}>d#&)dvz;FbwjxpMbP zaEJ+J)udQ!Iz#;EBbB4cDEW=ZHB&@hU>PLtqpJtztG(y4xYl%`@4X$zX`bEJqJ1?w z7>3INF-Jc%;E{9C1J_dmn4u(rV#yOUemjIf2O8wTH9oLOdcWw8-Xb`?3}Tv2bG_OP zj_pU8=Az!xe0>ICwGPx^a747_SSN?cX*vAYk)Z{O;?3Nl+lP0JWg}NBsM%74INU+{%P#Zv<<*a30??hpO!8&Nq z_T0`Px}iT5l<`~UeR9feMy0Ifr!?G!uhY4%xP%mnbJ@3#Cc*d2v^Lm}Z>3ZWJ{O-U zqDUy^G*)(a2zsjSx|}Ty9Cr+9lt~!?;}3YDpUn%MT$kzZE1k)SaKm(q5?) zhv`eE7=~$q5g?Z%Y_Gk`{_QRBTO4cGnjqG{?BGGG@MJv%e!f5SXy{RV^ZOZz5_(A zuP=3z9tYs-v(VU&>|~~6dKrpZcAY#D(t2`D$Hi?HD`yFKc2{jjvcZBS)*D$n?Wsy} zpJsbmt}PfsN5&0XA#Rnoekljyf4SD^+cl%k6{Fx9ONSG-QFanS=k`|HjH+gGe74oR5c&k z+y=&K6qVr_J>Ci8kV1(@fG_LUgk7?1m0j}=r8gx26N%B?&vRl72|ZQzte zXA{S~FB%H~==%|4@9wPVudP1oYtZ?*-_uK}2S4?V)D*9SSCz54`>yq$f}SGGvT!v? z;;X42!ls{REKT!Z=3uo+o@zICU9=ut@jnSKWB4H4N}UYO`0uz&TYdNP^8r_LeO0k? z>|3!$%7rL7^_EY`6iD|$$8C0$BPJ)pALiBgU_ZSV=FBDhO>IBw9BXI`(mtcWF8)U{ z5;1Pq7vWq0%p8(Asf8VzgwGsb{MWi$wM~c{r~xdk*1i-q&m&-m%C4>IlzN|G|Fr0c z&%C-xID%0XZUzaQm<8clAvrk_Ss&Rz7Y(_R+wuf67doCZ;zd%jP5>e4}l1ZiG8XT z+5SFyaex5?p--!1p8 zBGAmmysY(4xv9V?yP#EHu5{Ucd{pSQ!Q+@yX?YZ3?KIwpzIi@b|Non3`dV2g$)cWl z3_{GMPyH;Yfy9t(sYm#c#zCYd(jTg%E+%Zf5@Uvl?B&^)y+d4KP? zx2so_R)D1-XkkhFp^?;$-IV~Gb=t9u=;qwPC4yiwoClUK684Vkg{2~sW z%vS*NonoVF&*_&lrq&7#`2ya-T-}AHt4vowJq(@EvO%rE`1@fwC(7`QTMIRzDGI;? z9LI8Cw(j!`+24k7zXGREIhWs$;|qJo=J13j`p;ChrGM`wgcAL@{xXaIK#0&~iw2lq ze=8jX)(%723C~;h3a`F?a+dAg!;CYV(gL1zN4nTAi?2pbr;y~eE_Yiv%p2aK7`y+* z>&g~EHPS+l)qzgvO8Ae#ztmK3gcLo zSwaSH&u+aVQwcyYj`FJQm6o18)318{?AE+enFfycJj>6%Lv-5#H@y|EZ|!*7IyDv{ z9rtZ+IbEJL>$_S>{jgoj*tC-AXCoMgK^y0FltZd1ZT?%~yuQQPDZS#N_V}^l4Ec!o zBfONw>6P-O`cx0wk~mXAmC?7}+FuRyw}ME9%Qt@9*=koVN1ABG`A*TUgAbBKC+nP8 z)7^tvSDBddWH!pR7mep_PwrZ_z6k3ul=H|E_t3SKrDZ`&n3s~j@^JfTTzN1E*S7n! zXeJ~W<{{+gnSE|62{Hz4L)1>453$~DxjY>idT^}5Eo-?`ECpT9w9UMcc@{(DZiji+ zJ9e6#J%fzqR>f>8=U_FeXFLWWA*nzP=bLr@vpr8{4Cj-T4P%H(Rmg=M zYV)I3>$v;ZL)Xqq0MvXxnfc;%7KmHpdf;o-$ZAYX;#QS|4ezwVmq9|L2Sl;_mJu~e z)p6DXacI7CqLa1tSw8p!E5Yv{C5|fo_N9#!Is!P6X3L-6n(cz|FkOfDJld<#dXr&fwIy+r-l&&t5iO#o57y| z!D%;gHS;<@b#(KJR0Dbeh&7;XX7ae97k|+7OaN<9*As2A>nr8Eskz%#0N~?sbPYdxA6iIlxUOUd05XBm7F#_KQ5XoHm8*3*!9~ zoYgb2OXO<-8n|wa0~;Q4pX&d#I(Jqx|7LsooSrm{t0wU-?k(wtp|!y$e4>QTl}Dw& zN7;sMdQY=bG(~TrcrpaW!6rmKLs1mJrqw*qx@x_TI#kQBkY??>) ze~PcYoTzv$PyZU0ar`xcIDO6)U3!n=uRqew)3#T$ug*`wssrgPee(@4Tvz$bU)KR zGp8>*egHGB~XjHhDJR*<6-Tq#8bJ%|g;Nk!9iS_d2bmLx^dOht|fJ z81t);>-Cboo4s;es3|B5$8->>QgZN!LzgsL^)W*Op?!tMW3?c-K}+uoJ@RYyr&@WRXRc2ikGM zFCr@!^7zkY+TDvD$+Z@RVtAfcZJWs1eXvbl_eZLmeK++r(62d*E?5bTx+(4E(35)_ zbkS|V9}&lVp3<_tmEaHf?sSWE7qL6y%OPIhxktQS+fI!s7H)fep{E9**Jz#=8Pdc) z?)h513qUGz*X|iwI}Q1{jttVy|E!Jp*_am!}v~1fWHKq_d2!b2mc{^_s)^$}`r8as9=n9Pj`&u3ztd zv*A9j{b|@+-E}J(BsVLqAY)ElghGb-cgPNzkn+_h{Eh-uyYgYvrhNg2_ai~L6C~rN zch4}n|07OQ+LxU8S!`Fyu;J|CTjkf{-kF?HI(<}11Tb-H&^lNLCv1sTr_Wl6W$ zlvRBcKVhy6sXW*#D2uw{J9nUYir-i2ESe=$vUa`f5{TE}5N=?YTnK$&!5tr{tv3GB4i#(LYj%CXnme7gVG@B%y#p_0?S~se zHvFYJ*5MDgO~@mhxkFVjP8BfdLpDh)IhZ^|NU0;=n^+glu3&7a5~)c77Y8An-+BME z*%*_HS-|=eR}J2JYQhM>fXblzALW1Y6`TQPCHRGZJXtw*(&-+IfXMTHib$r6T#S_} zuNc57njaxw4~x_20VaU+6fn%fKKQdl!Qlf>A?pF{PUaEKavwWJen0acrM5vR@upRP z?{4@Q0r}Ye1@FYkxq8`K=nRS2iIngSSW&F4pf$Je#_fsg;~YcwME+g*SJ^lR3znQf zE?+JPxK$ghGgHFgca$}q8Jz0wB|G|Ze?~3M=vSUF^F(mB_86EKu7~Y2%2~bcl<`%8 zrH0b**uE?7Lzl*-SJ00X(2F@r@MY?cxvQg7hcA+9F#i?*m&dw&*!|Get>Pa-Tc=;; z5tzjx&?ruzP zq(*OsCOS zbu}&=(v7nu70d>)=|U`EnR+lsE2gaw%@D``xGkcSMHgswn8RSgIytN>lPV-Kgt+NL z3#Kg)x4Ra|x@G@Mf4m0qpar?OjKk)oQm)XUjBO%^%r?F`46MklCOO{`&zT{tg$bTq z_RSPuxDX2*vYtJ^(&tC;a?UHmG)jt9VPaYwzIkH_;}? zHAX9I$%DuX2a)FUUW*FN<6LQ%Ln(B|QFfHjh38eK)t-9b5@!3vO4$*!n6b;+Ea6@s zQvp19*QckiZ|=VPQ|$T7aG7VcKfXIR)c-e7f|J2Y#_|G5s&2yhr4psiFE*ggyK0V<6H0-INPTI*d{tB-pwx{UK0%r}tlq@7P2e zL(N~dBjl^mOXHAK^ik=8WK70yPUE`05xull(C$vrm7|Btp9iMy5REaDf`tes@TcJ=qFA;)H0-oEM&a`(btyFJcW+rgy2;xSP z-i?~H%XKh!Eekpu)B^3-O{!j#mtv<8*ZFDu#5l@hb+$zl;U`5*Eo1yC15rHoFPoD~ z#P^~C8LBAp3o|Y1K0klT039N**Ww9{w}aU6gDKI8Lgd$|nu{U!oGbce&Za3v(@XuS!YtwA{9r>4k=yj%m$erwFqX*v(9bSG5V(xqp2rDY3 zXGhyPf{S=o$7xT3_3gbBkT+?O2F+!O4b)3FcU(L*D&b2ZpE0#6aw}1W3(mFE79(n zR`jFPb%vgTa6ISR=-?>tu(_9NHp;wPZ-{F5@z`Cq0$j+7=FN)OSBDo_9QR3(*^eo7YmYHgvo zF0f3%4~N{#G+;*rD^WH-4Qh43NoxCY#hRip$1H15p)jeq&lq&#JTC%Sb`1^Lg5f7xFG<5|btkDD^eH zg2ZXLlMRIu+Pg}BtNf7nZd>{NbgW4>0KCK64_f{Nn!YSBN?rg$x*rjZJ@(suI+0=SRf2dpt`INShKn`n{OT=4O$8MO6 zb&wc3E6d#9Bhi{&_Y6}01Pa~PG7@k;ch$%k<O^X|aQST8Qi`GaNhlS*^H zNq9viy{^x!E$+8BogxSKsb0s03@epbp_26?FewIWQZ0vy+s=7Tv1pE$6ZJIq^oZ%3 zZX`*#L400o4Xcuw)?LYR1$Xw_8+>8NAY%I|DJy0~g0+Ru=l6 zdS^s~d3_I}2;sa)L{V4EFB)BQ;-GW+ zQ&La+II()D4{g{+ZHJOl4b`m7lvjbEr(}kR#(6>C|1JouaKMEBST;VpcjLq#JbQf0 z55T#i9pDC$~*nj)p+l~)vKB612nSVmV5EJl)7ne!N%4uP8isDqb40w43u$DaMy);(f)AHT^MXZ_42KeR6v4 zrT6XfrQFk{IaHW2R{H`=VD>XnViLc;G}b*#R_sp6oyj%>#~6_4T9%WjGp4~?0MSg z?HSA|F=BzduxUPln#bxn|9TV3cC;u?lvlLQnQUAGVc{{>ni4xR|n{!RQj4c6yl0~0=X{Sl?2 zv36A=UHn3{O)bnC@xO9{^PUa4dinpH?&O^ad7mZiKMi8s!!pIkvT|nE$2;a{#*h^h zBpF0GC0_J_p0Bg7i9gzFr*qReov*0Kj2&a=$wI&7!8{Y-7*$}%CCxBR|K!J%5Yjo!Rd{GGsm>Y*4-g~w75u2 za7%lFW6P&JW+5lf1*CzfVc^E|FX&g$|y1o%Rk>3|VU7 z!uLt1DZtOxXY$`5jWkr9$Y+Y{@2kD* zJZ=xslMtD?6D}3g^O#m-JH)>;)WDyf;5jwaK2Fa-(<+agj725PXvGNq!<08ry^W=g z)y56RAqqTrpRO3UP45rBp|NM)S7~t-3DGb#AImTqPnTI(K`AXtm4BRI0#7h)K~B?e zWBc@@ebiZ$vQjEzR2QXQL-+Y7gGDVoZODKyeG(AArB$%WkYNN}uDDT9!o8FmabXHQ zd&DqdZAhG^rs~^v)!{ftN_xlYvE-Xo?5TQ5?mxBgdcs!BBUh0d<>^s3P8n5WUpBrG zz!Y^D^sq~{($E7OObo?e~!Gx_xqH$7oPrNGCmjj`ZxwdKAMFr0Zr4TlJ4rnR4(>JF3B zvX3*rY?ntip~YU1_0UvvV^Qp)5S9S0RYob+^<{PtpHIkeq}o}V*wZDL-AeI?DYfW< zDE5}Cim;2_b)}m$2K&N3X9X%qnjk*OV4~^oF1*mZgZ$?qb zff`Q_Z1@FCpp-f)t$Y!_ye~z88{<9?b?KJ_#Ah@)yM$VflZ!CncZ3;4rvL}`hnXVz zNv0&0;kFm7yBv1zb?*3qT4+chHMwJtSfyG%FwCE;O;3Im;@cny9X(OF5K}Gw2rMD{}b8g=$u?(#cdjfnltsh5?3g zaovjV-*fdm(kfL6SSZxs1kg@?HmlLN(_>DSi6!TyG?Rs@d4isoJHmI|s>< zb~FQnUoerU%U+SUbN+l3-@{*M`E-3(=4vKIHxvUF+zMq)fy4$xRGe=K%0gashv3_N z!*PFmp{zW8>=qrEr;Qcn8%U(HMS@+SZB*agteAivnM=F+9H|AMvrA`xAs6Ss zkx6n%;4qmJi}tH~$|%?c+X2hOv~hNslF$iKcudgSN7(lBUt zyTbF7m1i_#*g!eyVeMhA`u6V^ijP5=sv!!yhXUXB!SfOgcP{CCIT*mY-d>v7?Up)p z6Uo(b(Z2S4S0blzYP5rU=tdbV3k&qWcjk9>(~3w7uZzaubCL$%iYz7r3gqCL^z?|s zWz$cC(`AFw5I>xm-uP7<2QefgCWrID3bT@Ub=h_1i@VU^PfNw(U-Q9tnuysETzBMr z|Aev5L$^ies@~>KDxPpC3$C9A17M8u#u^s!SFW zr;SZG8du|-FvYLAwj?92OBwnQe678KX{$F+DonqMBFkP+T01KKwcBZ{qe@|YcC!xp z@Xo$McW%FLO>%>9eSH^Mg9mJpt7dqa)nhW2cRy{Iw z{DvrUhoVp1(Y=vlm2#KnWZ}E^0NP8Z1tC%)6t6<3wg$Fh3&Y10l@|^0ak$Ix4;C~1 zf%+NyCSc|0G1vJ7@He^0Otc|t6t#IqHjdPI3N1o2Wv3KVoEWEWsf|@S{bhPI6TEgG z91xA+oIbYYyS12IjcpsZUCWAmf?`un#+M&6Q(ofz?|l#-_ahs-?b%szM|cd(@nyh% zf^tBUnuLEDbRTBCh}&K5`|Z9tv8A~CI}->2O>%uYt(>yuv{^IP zv$pCSsL~f^nJ87ucL(bTdl&%>+t}mR1)|qEBT@;$x1HDqZND5Qs!RZTtfVbvdeKER z5jtqQK=`=Djjy+oTSo(s#t%9(qHX+-nDB_GBwSXYEVEnUI&_@9B0X9nw@^2euOJ)(yzan~8fCW+r3yAPt z2znO2s@6my{(+qMMI=~g(+|bbQTtl+)NIUV@I=Ge-5~(NF*mb-`fY4{3f7#VGT-lIaisk+)(g|UFExOa&cvfUsWN!%+tZ2Cy7;PNo5D$GIsEc zR1$}M0TQdlTNh;aFx92)cp=qg6cO$35ZKQ#HqfH+o z7-?{#gOmEUk;$WX_?4~y?X}B)UYKPmdv!ARe6~;d7~a0GI=!z8_z-8j9W~NI*PHbH zatRgfllA_5UHK;~8Suj`DJVRU5@3kv{#4Pt+zTJp>L^v;1hqo7EpeRyu`{{bSh}4>h`iBv!J90&$-@W zA$#M#HUQo5>{U<%Y+%Hfn&{+~inbT^rf=3bs>wg^@>22@ejcrNC+SY`cn$0I)_1&W znv2Er=xVXhjD=o^4XZeDS<0y7y1tkuPqdF9t>tBVx$#){NupnmfdM9-@M6;d#J z1DXRy5jGN*me@%G1!7qZ*Ut(1rxd`RRtQCLfnpZkO;&4XJL)|q4mn-=Poptc4r1zA zZkr7q%EJwE$*JMZHROQ#(D`KZlh0|{Qj;+Um!6yY^O;Y&R{Z-DBXkG1dJ(n(Wj;4m z%+cM3>xIoYeq}VsG_>7K?Isy7eNH(s|26UdX>FcgB8$M?gr(8RVl}`${U$=i8)oEQZQ!T`lV;Q&b*F zK*;WDK)+XG!~ruhW!RaUq9up#l-?tEaIgyrpfkYnH)yQrRAhd3vv{YWL4W3t2?v`2 z!f9sIuHgo0>mLI!JCJ8vT=i(}6V6dn0MSo#nayMC4l^q|2%A>Xc*Kc#AzXnWZW*3k zFR+Vb2MkpihuDrJRsSg1c&7i%m>?3o{p1Q1tr5ihNO)^}J$Na}TT>MO;rat;GFg@x z@fC6K@b#Ten+Ps?T`*VaoGP|Jm53Xdnsh$M{l!$a6q_jHf#S$gg>F4s9~!fsvMLtn zvTeKiV@EKgpaozIluG^Oa{Bz6jb&HTrua&q=J~=!EB@X#)kL$yE@2!myV=?Kt!Cn# zF3rgsC-PyvK@y(2!ds%lxxg2127br^+yyZ{fOD3Y0~2(sDc!JBA*9GS>KV6Md-9!M z54DYNcL&bC?Xg=JW5XD~DHINdkR{ zFQ+NZ(8=6svW0&{I!qJyxQ>h69JD=Dv5d!z`-+s=EJn6r#r1|uQ{N<(d8QRLk?yV> z0i|pU-zMGT5E$%MrlbJ>J18V@1MQ&G++4P=t-$%*6hDUs?BlrNHY z+VQFc9P|dwpgY0cRYEh;}S85qjFFOlod+Z&QS~4%Q3Pu zwmE)qx4rE?uPrl)1%2#vpxl{kpB#7>zFzSl&Ixj7{k%?zSr~%Dyy=n#*;zP`gM}Te zUePVi`2ss}^C#3a?4lAukT>V<8@ouN$V2l%%{5@AMa-3CbQtDCIsA{Yx|a?g zE@hC?Vx7@2B64JO|fL{2C9QCZ;Bzrotqb3_RcakT~*f zJMRZ9YfwLN-Ef*_u;!^S_&v%+G{jNd{U(KMPGOW)+0Xqcq&s0kSrAq ztyCL01QaZ4yreastGKZJ*>w5e6Vr$a4e}^qECFqBj$Rbi%=9^Fck63?#}ah%9(p$8@hL;rZO>IK|EL-R_QG!96(OH{ z5O0*VWQAZ|A|QSASBB})Cg{yTW=UBkiI-(s?f8X-L(<9d$>q1-M`#Lx*@Mit6qICp zNlxSZbdsn`rMb(1daC#%HOjVjd?s;>&(s*>p*3Y#?n6)dJ` zY;$qv)BzF$nY+F0#f z9`Esf*i`}Pq{{rk^?FQ;=-*V-P2Eowl zgjI%9vXa_R+%&A{8|NxVSHY-^AI9Jjjr!Z^pV7-7)OazH`GSu9Hn?Y~^N1BL#Psco zhb)@%%NmbbO-Pbu7TQ~3P>I-|Cf9Z`Vl~+$MGJK3JrjENN}q}4dclg5-G3A&Jr5g! z7w301uSg68N`D{TZs>;5mT$)37((ngV}7e9J9unq%|Ec{ynoW>!QQI_C7GNoe*EFb zRUTP~?Yt3PyK0^Fdo&1{I*p>!MPKOY_q`$M;j9A>Y?@FrjfXO$E$Dd})L7Kgz6P4w zPwP4!o|a=zfsjs;$|=qZK4dAnnvK->9_~y8u4R5-AloFJP@;eucmoPBc^$Dq90-c& zJs^08(GKirqS7Ps5r}cYWZ5WLl@%cR^~SS7zZ`| zTU&=-DV`rB|HJ%`TKn}~MowYX|E!08|K?HHSZmFIRI-*QJO%j#>F<3|k>Okc?W~rq zn!{IYZQ5Iy?b7NX#teA5wkEzF>2&F`Ir?Ff=bUJq9s9nk5Nz5LxaiM&e`?$!1wCVb z`c8kLKpF5v)XX|OwyUAp$mS5ggRh23^q%TDA%;gXo5g!oXFVm~P(&utWA8|MX|6pi zI7emtqScw6omT4pD?xGZ?wG+$Wsi#%5Y`f=eq@N{Wxfga96~!JXazxkRm~I!{>kj>; z+w8)y;?g>M=ink7deq>iD-dzO<98VOMMU$Qip5^B*;2HNz(yO}t(63Tnnw3Fzx1t4 z%j~OaEr!YL=9Evp^4>tq$ynGw(YYFdlBVwKmufkmKQlKL%+~@2xM&+6O7)u*h`el3 zXMTYU%x3(g-jPdXItud`V!ow(`*pa$b<3~W=QsGX@x`=({xA8f0D;MKxf(A=lvLq5 zA^{QdZYiNw3#9+HzS$M!hCGOZm?5P9Oz}f-6^}hF$i|-)C^z*tI-l~}cLVE)9gIF1 z0@Sdj5b%n$CZpizi2BCm_KVOW?VqYz0t$FpcAKNsp7DX>xs+NL?1z4DRpQoY<_6yp zaBG3<{z^;5N1owmo4WSqt4zsN^-b&;I9$9cj^1EX5(9c} zGIp$WL~^dQOnhJzGD%S>v=Fn(E%J&)BfY8s*F`ar0}vTamM3*7zl^m2ajSV6zH)0G z?6=pwmdk6T6g}5!=EQV!LUNG(%BQQ08#+QwMDf#@5NX<2$DL8LDv!;74sclG<9U{o zCNXETf2z%D7tfPE?@B-2uZX&5N8ye4<4B3V;A{pVCa&y?Po?PGapH8Oe6QpW@aRIN zm0;B8{Xh2Kj)seU91Yf01IP^(|AqiK^V0&~fJi)kH!x27)dDb`a$Y1PC^7myhT{6Q zz-9Y3KEE-5u#lny4)nACBxcd|CtdvaT5nxAV(|V(o+Yut-e!#@=cQ60QGi;>tFfP;YxBNzgMpb~qy5n}p0XgZaT))mgbd~eR)7!~tzBIK{uBXnQxn{0O1mMP?Au+FT= z+;7XSIXZ&Iv@I`)kH!L{n}*qnF}8)*^d2>)XP-Wo9leWi(7o{r$#d+&rM8})6wN!v ztYyM|Z)0rT_=Qvi!JoL;f+C0JL{vw}PaNfv^mVkur8jK_T2UVrJ@?G%9Gv8L$k!@y<;{v{wzY)+E+GoWI!^2xGs}X#(V8tcZ$%t`TC8d|)#BQ}ZK~0T*xFaV z`n_a%L!rS90VajPtpd{S`+JL-y-&uzrd;!|{Kp4^yhQx_Tf5H=A_)e z--%Y@0A{A0?vJ`R)cg99^*N5)M%At?lrd{sh@yrj)~9(9*hLP%)0-P61)HKekTmJC zDaUmcgK~--^vk!=KIO`;L43E`E>rd8{(oTD8-?y?x%Asqs82ieT)$k(OvLh6s4kpj z;L-6BvtAI9)qDKJ5strc{}J#8%`w}$Ypc;%EJ3M87I<82wto2C(!%*Kkf6kH`o_Oh zMOfpAgHTjnBm7J1_G+7!rwdh<*`9|jOmKC94_05%ctbmSdQGpz%_sYXuLu@5#k5rRq^3tw)8ta z(G3b0aP9$m=Dpk{qor!FtN%cNZxYzeb$Vm7@@dUrZ@YYjQ$<^{qMs^(0(SxRQ1w>| zYN*-=^Ge{%l@jEs;Iu-u2Vk!ZPCXHn^;kf2d}w+MngvT3#Gk z3@v*1_*zNct&iDN#6lOIB?^XKw8luURRh@-#q@4-#lfT&qK*k3XU7Zbv#I@q8p z#=T{GIs4*USr;Qup*TTJqoKigbU7XFbtsJ=PVP-bN)xgShRV+Te?|UuMt*E0>awD5 zc|^PMJ2Z|O;r<&vQ0e5FhKihT$=uLtuK|9#VtWV)HxhkyYEY)Mk(_9=v+8elWIe7w z-6H+g~I6=iFcStQUQN5_B7_TN-o0s zMJ=SntYWhwV%r#XVb17rry=+6^W(@fBcqT0lI^Bi%L!DNC@S;GgN~`7xLg0yn*86f z8)v&4>%#YzoVhe6YH`1=WF6M)c%2ErReCl^QY_yO@AJ(L{39 zUH-=oU}32c{>2?o0Jf3|uK9^4fc$qDm1*4gPlz72_I)QAH|k~B(O0s&=h5Hi3w~rgEm-(1l?!u5G{Wqz ztB^G35#%?}Q&^W8?0o3xLH|DGJ2rtPg^U*^+R&@cvhuUAuCuOcf)Ck1C_fK6OAYA~ zMNI62sWKLKmad>)6B8kqO1J5vZIuHw;wbt4y57YKOPAE0!cJBJ7C?eTH;DW;fmKGb zJRw2j_SJGzV*!Qxka&Oy3+;NXlS2PD$ z=PFpK(Zcp7`-I2WEB_#6u$Ms%G{w`6@t3GNHj6=iz48B;I`3$<`|$4@wQ59ZiyEO6 zy{pvTVzsJv-K|lCs#TlXkx;vKsZk?Ti?(X-6lGKj=@a{r!jp8s>s_vD<< zH|NUrdB0z;uvRzc#?(?3Of7Gw$8e^Ko7_7s0Xz9!O*ge88peUc!LZTn3<5{@qC~6s z4-Y+%y6Q*-JWdNBXE_xZfBkQNw-rz0LL76^4!6zBp9r-X=NX2bC#GvPlH)!vA@Ye@ z6MuuvPj{+t-s3TZdi#|#)4x3bXax*LW``+xSMK#Ou30qz*f$os6u@nvd? zzkNh5q*ifz{-&TBy2p;1#B~yZb6qpnUAg#j@8U}>L59U2d(X3JpSWE^~{ajtX zoh`Fb~Y24mF0a zLt857*6C^z?k5v0bdZq98mo?P_CIAL*YlQwiv}3*9WS>27?T`J!A!b28ap5K(Lp%b zr}HV_K#%Hi4mRa+5yXhB1RxJvsLZMHW&tZyi?k}pt4F>|l=AmAo{}i{GIoWAMrqzR zTUqZ-JJph~+IY70n*$tF_#4i50H+ylor{

b(zyMKyp6nQts|2>h`?)bpJP41Sf^ z+6^NmB|pclJ(wQRI&ndBtA=H>Dy`Diwf_!TaB{~KiX^Y<5(%YQ>Pp}N6r_NIgBu=ye?cZSAd1D@pAtvN*CpS(t*2Yr6I3$3j9zmw-N z6{tGD{z+pGy#kXaMVH?2lXVRzxFNFBg-Cl58N{_h2M+Lo#SjQKF4p5P;-_iWi$sp4 zt!EeJv;4jy1w0UnIsZ@li+X;9D!-W-RDd-tXM%vf<0B^h^VSA4|**`|%4{ z;2(_1h$)%GBpGZQS!q$s7mNdz5JGjKTWI24KmNt!a*U;%iY_cwhEQqhJn)m{xkm}O zDrw^^SPk5z?IpN)S8|r`^$<)pB>-iwwh^aTLAlBH=uE^w|MI(tGc3V?-pTqc9NAf- zGiNH(Fy6fe}wUt{E3qVqkZ*W#x-}`bmoW^-65l6=Jw-2 z``%V*W0Nd?d0s5md2vt_$;yuiy{74S zI=2)fzXpk5fdqzWO^^{!W#j3pe%X-Gq&zhB#GdQ-+4Rs~Ob#e1eR+=^yY_#vR-RAR zbbE}xv|ns#J`GBKN4K3zy7pS(!nz5n0~9@k(r# z`ER(!^lO;bjqdV7F^7+?TBA$hD6tJKk-aa_8WoJW2q1q0!vkm(Hb5>0dnG5NoR>+M z=V)}bspN<5WV(o;SrYaW&}D@dIRV4%llx`U7gcJj&Vv!p`P2?9e}u&A>Gav3@W9RK z>Zh^xK%MI$XpXIp&zzcc15c8DRX<`v)gG0N{G8+a!H75h=VqshmXth>_4UASLG*fm zynvFNzmr?HVV2W(h)aXEzuUT!_x#|O-1O=CfrozVUrB;G<;uxy(ITCSN&Z&Dp)bt6 zr42w!?Yn<^83khRezNY2r=N$es_cwV{J)VYcBR| zcC^p;a{sF+xIOY|AE7~VP3PCgJeG4H{~^ypwu6RcuMMH`sDH3x%G)?*Xv!dfv#`|U zH6&D2z*TUkXKEyb$?fy3*OqXXFCQ9>{{~gO&hEN4DT`>7uJxrSI_UgIOkYV zp$(ZYD-44&HtzWbY~`BiB4JynK5MNFm*EIPa0e&lahBWG+QTLEgK!oFeT z@F)h|+6GW%+6*=b?@X_v=Di3nxOfS5*1xamS!T=Q-7L$}V z=&l87f@sL=orV$%PkKdUZdOo~gA$A3UBmY=3i+9Pm9`1qTV-5_KB7c*=W7x;%tAq^?qBKTc%)%)r4Y#`83y`A_G2 zAv=zn0-79LuLq>t&U0FuQmCRkR5aUvud4bKG%3sq9OgT6^&LV&T7xy_TGVr7f*coO z{;=%JU=O7z8C3wxElaCts0!2B&*!BQ+(jb$hTb(#*Sy0%)6`ejm1v(un=XD&yS7yK zus9)Ry^VGp86mb+XUi9e-{mD+WV7Yq6BxmmnEpw@^WnFbkpmHPwuy}%xTiN>6tDsM zA9HR-o5LOmE1GH62EGrO=JK%nY}RXmvPXMI_|c zHfbce$*bDR714NRrh!3sdB{j2!yRW0-Y}oKTRBHZX>-Hb z!@H&-)w<(a?#4tuiRZDVP`nXTKB$E<&xhDW**s_C{96cktyMU`EOa{8RsQ*RZaevM zP2IQi7VmaQ9%MpFy2px05{4o;U(QnMN5rvlc~P*uCx7{-9{o;FaTQpnC_nTJ-cxUP z9jmnRgyc^)0@PGAg zXT%TLbn~BoY>%Umu%ZfK7DZFFuMpgE6tDpNB@9nLzpUBZQGHJ9Emv`TVRN)&)t8d!P6mO z4kz@|A9Ui9vjghCJ+iiOh;Vf&-lZ$H7CieCm7*Ydmg2X=r#4AlsrlPdS(#RI%7E6v z+$r{)H!V}lW=oCjq@ADHubW#@FF{RCe*O=GW<7T%6)z`HfntEC+p1fSDfEU?|K5=n zu||R=*p}aiD@TCE_83jW>Hnx#k!9_(WEw{x&%i$FL$n-rOayJ$gaSAN|3>B)~6KW7qh{B z@upv)g=->0VDP{8{AWWoPp_hpi@>*K{zvK`C5&b_wGx;iXE%cHo*G}}USf1-%}7+@ z**`gBRLZnpy)eRcz(=L|J0|DSCW&HqQ4=laalY+BdDUFfTruWFmY(XI8EhwWo7eXZ zxc^1;m@r+-+3q-JQ^0yc(h{WqjR|VA&tw>?K?U_0J@3!!Sb(pI+;p1$8iV}saG$Q0 zGBiV``cq6}E#>~kOt1ett1Yw9-#{tg7G+%IZoTtaXIU~{@Rz;i+Ox_`dF`OQH2Snr zGE9)9M}INFh*@F;$>C#kamp+T3b*$z;@Y2-CaDOTu`ACq-c}f|bugFkH7ozCL5R>8;GY+^L(XR#-8j5of#Zax&0G3zwFw%qI< zJt7Eq_y=fiw0qt0%rr(7;1apRQZ-p479=d-%9eHZd$&%hlD6Zx>Va>R-0=Fg{qsxf zjoN!9?s7;4JFZC>I_SO3SuKsqg#Cs|-x>!-t5{G^3b@2)ZNR>ub!2(Kvi_>?+Ubne z)zTeNqiQDGCiXsy$prR+u-Fo@Q>2%lEU0Y76}6|V zvDLxTIn|q~WO4Nl5nZLc7Q>SzB8onX@=o#zI#}s=J$T)FxRdOTo{_CxRhyvv$nR6i zD3vEa>$%PhSQXy&n?-M$^eylyM8Z-lhlMbI+rLKTMs)III^|@5(*LPt_UBDqwG0>Q zF&RF%rZxy_Z`hT+@8qJ;P>&hjKmcyzNkyxh6ET01UKO?1!htZvq|)~IZM7H4x| zHGT=pz42)BcBC}wt}UYM;!j*Pn^6clhWg<6`w0l*)(dxUD0jCn&8B~9g?>LvoWM-$ zhF;oW>?W>KlC4OT+Db}_;?ghK^Ka%z=XdDXobKSHk?pFhWsC}>w3DZw@zHduQ)xdz zrl&o+B}_YaR10Us)AyWUV})d^l2EITaE=1R`@E`*jvw@%&h2WO&YH2z=O3x7=?xXY ziZVeNTzvq(2T*-KH1Qa=eg3IzOOMaxL}2aR!y7AHny>RTF4o{d6sH?zw1wNs=_YNW zuB_h0dP|=nV=AZWL@}~JWjtKTu=%#ghHPVTQDO6eRM@FBNIT^47#|)P7Q8Al@NbfX z?t9$#?jgYw8JC?$XO8Fj!oyM^flJm)T@K@m6OJy8_bSRK3Z!HdTf!{idOE4$asb@K z>G?m%&E%`&=9b={%cdI?c<)WZ7kJ1iA?A=g{7nN%gyX${i(jh}my0k$Beq8ElmKjB z_5j*{;DDbcL(nJ3NoEf*ld2J&CgYS6c{lq!Ax5@bLKWRSv=r0y2eqopO+OzsXmJFw(Kk zB^nGpPpTk))RXi9?04P=zKBl(W6r)WUSX&GkVNnb1wQ^X8AuR0RK1)+lT)OwS3Dt* zvty10Vtg`cX3sm`=;8nIRW<|~{HGs+{udaap^m@5HM8>jF?FFWWuIn_7kGk#Lm={_ z`iLbYH>ICuTnsqW3qP{C%+?7SWx-mf$*(%vj5{{t`OCTARP_&O5hrqF@adjt&jC7Q z3K?q;px9dOGW4?j3{Lk+u~a!*^KMo=zMN2r0#mL4$9pPRdQyd6qRSHs0X3J$d{sJ3N4xTMxq*C%+?M{R7V z-w?)~(lnMDeDtj6aPS?ZE#&?^MQ;mZajQ~v=JDTyQvs?>pdLTO3Iw@Z-S&>ZG1&ZT zE`r;OPy71N`{7)$X^C1H^9A99r@b( z!;APLpXE-c`guGBWZ&+Gf8Q*iDJ&gv(UI$p{5>^0Y}aR|9zvjgIO}+_s90~LAeUBh z0bR>RcPXqJC=0rd0NB@dg@bm!LNRIIRz2vI8D@Gf$!eR>Z1$Bqi01}wA;ggn_N!M~ z_p#jq?XQW^2_{B~RatVLq-3crorKItQTf?~`a{OKuQl7P$)^McC4HFAOE8k`Dslh; zp_}*R3J-+%e0SY;*5d0WtX~8~4lkJMwu7(Qb^+Fyb15y*cWr;y(aYwh*=YOm)8qVIVi@<)wgttzpY_a!>U^}m0U^Qp3aY~EnL0-m;1%u z;4`9hVb(>gl3~lc1SG*rUailvfM}3|Sri2pxPpzpCO|c^gC3oRDJ@wN29Q4g3=BH6 zXk)&ZJ)*iG_fS%W) zQYOx%dg2wJ6dl3pwpWw)mG=B;@p&d{=uM8n?Uq9BT>&?)*T3#=WKj>3e@?eohkAFI zL_faj6KOZ@5eV#TL^~Bmo>L@6l$d&Ardh)DF14j5Ft_76gS*rW%qJ5W;QR0t>r-_g z+Of6v=7`C~$5%Up0C_i8$Wl!{z`<(yP65cBs_ORZyyyh(r5rlVt{6x(cW;K0%dOiyLGs^Nl@SrNRf1;4uJ#S@}5BE*c zga3U*mv^^W{$HvcChHm({Or!3%?sqgatR@Qb>~{c&Dk$HpY5*|b|8-Kb!~8+TdlBH z&dQbccXF`2{8N(kpenUMqAsD(cQbp64{-=dujV0hL^Ur%1kQPa=VxR6-|S`CE1_AH zI$I8XT}=IM&)E;Uwukd*n#@Y@ZRF}yqW+-<+U~6%aqiFJCy?=;IIllH1h)b|EHDJL zxsfc?R8$&{?$x-q4-(-QpU-$Ynt%BlAsc&$v)-OFx*?b(%3x z$CGBb*k=B1p;{wGDPipB2E$OGsv(|BZ2})7d`p4%?7)@sq>R7@BE4MGY4{GuIH^vU z&actc5AJ1Jg#bW7PJ9k1Or$#n!-4$k;l@3zdS=QLbtP@?xj3Z)?gAnNlM_q-4{d?17ZJPM%rQex2fM2CpnjuC9z zeLy4~$$l=pX-BO8RFITFo#Ox9NZh*i*i8=-Ez9G#BJo6$HkTw(7}^$pVNPDA7QueI zsz&7Tr^$>~`TWk(MP4nX^&g|8FTGTMfQ@Y~$cG0wjuPb#F0d~2UR3!`haub4DeFs; ze3N6|(aTMqN<=@o5oP~JZSj%QBb^)9H`rzvKcmAWlc>Ls@*IwwFmK3uwTE8|${wd6 zL&-8_d3k|Gv(JjQw@6=deco71o%W(~^@(QqB0=zNc>bQGe{QnaJCGlz z!)OJODZ@Bjf;?Z~g;e@4J~_5|CQw#U`UGw2oU3_E2#+??9huN+j`^E!Ogvr^z6cy; zooZ{ENRvh4D8qiVi@bE->7B__%M#nCW=GnhoCdngD(^MFDV1QZ+o`}$`+=%YbS|u0 z5`9vBB_Cceur~A$9$I>#b8vCfpdm&|3~}MZd{tfSnb1&p@&i!7w1x^cY}znc zsEvnQli}5(X*Yns0hsCM0i!L1Awygv+@$jxDkP`Nq5sNED%hj|9#6(wHTym8b6lsB z;H%x_MSMmBj%Z2+Bria7F1O@)U@M1xhrRa6K*SR4ym%EIdOY3_BIq3|EF6dSA4h>t zZuLuhhK?TxnMHRVUiMX}To!pYS;78aJL%#Pr1~7tO}dh}`nigR`A)v-+=QKxg9R&| z#GwB92Sna}a_aW$m>)eKxO?-cg_AAg0pg6( zr{a@9ItXhOK~{Wn*M%rXW8z$^m+d6+kAhm5)SGKpEt}?AMq4w?a=tSb+Dhk=Q=?2h zdCL9AIMd8W(;mo`b>1Ka?LwZh<9LgP?9DrlL>m|-s!nF*S6Y^Ts&M&s#KTH%pV2&} zSl4yKt(-KYys1d%SDwlxT$$!m(=y-wWtSUmb3zP>76%;bWDu3H}GF`FIQBQniadF zjS-Wb5yx!W$c?)>V#&R+ZkVbn!O@=pn?LapD0!BodkacGq`lH28$ALOWaQLSHAHI} zh*gEuc@`fhnXd=FRCT77ODXUe>W~|t7;pZNNQL4ggd_M1RC(+y_>|oA$6cXPw8R; z=C}~LFT2idaEAQ)!)@;Lfpih8+pc3f_--{UEXm-FO+g~zx&}f!tUXn>->E%%ufaeQDkjQPuy^H<71!|@entokQ3VY)m?2NJH2r=l< zs_s?bUIP0c7Warx<;0>pnb0`MP8%V*k*~wyAeYYj_<=*svO&)MGzCP#3q)jbM^IZB zD9HGCl}FNI3;lr8-ikvYLN3ogpRP2hseh1JGh8hXMh7n_BZkDfURb}gmuVh8&OAG| z*>J=4`eSLXH$Ts8-?KUA}nQh;!MLIty)M7@gzO8Xhd+*VW7gg%Kh2422qh0eN^QXK}(uv#x_|AR1YIwh)#mClppWf1*@{X!~K}#0NU<-i$;ad-CXCVSlxpDD& z4B4m6!dAo=2VXnB&PXeb`i}!f)EWoqM*K$iVCvwe4ZRkCfqpiLM{zvz6#41e*{Li% z_|FF6k8?&mu*XSFaNnQ!BvwMtH!!>LcH0=mzEMb8hu&_zNw&CKg5VALn$|B`V+Gv! zgniJ%dibh~HedF0;tM2QKI{x&QFxN~xJ&Yj{VbKe#IE;w(5c@PKiW2IErUDH9CX6k zHJU6-ajAl)A}FY#I3unC+8W1Fhr<>6f|RaG#-GUay+;L&8{s@L{ttnd+jMz!lXasb zL@bY|GVNoQ!j1rPz2zOut7)JE%2!Z)cxdm`zdhlgHfwxxkVa*y4f!p0bY;GFWxl7k z_zNna@k;7aaZ`UZnrCT81AmV-sMXEy(0zuHTt-#icYSeID=avh%>>O;ulXi_e+zJX{s$yLsA{tsvKHvSibq% zd-!i4b$2wfxzM*fuHO;o85^Q*ob$MJ|FX$A|;!Vlva!{pUJ+0=RWiy|}dEv09-zox4_OIt}M4 z;h0t%{>5#dx)z-vXcWY*lef|DrjF%znorAAn+;yecS4!nQB#-m<8w&SSA!&-+K!5@ zoTqvs-sO1h^h1_g1+sDq6qD)u6gCpdAE!3(d-i8Pgrh>NbtH0?qB75J`Q&S(&?TV) zZ$edw$AX68feXA$-gMX;Y;N<<7?)ys@t)RUU(os|zi;8?yPA<-+!wnTu&Bgy^Z?2UvMogVV&%*oGvKw67qj|C`Ru3ZXYw40n7iIJkbi@;F*kbU zc*T!E2`P((NGvLLJH-tLz%PnDE&nrClKV=XBAlo91HzR4yP|*Ce@H4K*QItN9=z`6 zhEALDz)k{M&j{xtFMFa7FSixcFRD&~;424kK&$HD(rT52;w4TQHh5fo%o)lKI;Ja) zFClMX;TIW;kn<5T=M?DUMY;{Y{(l=$|6{pi%p=c`IYr9ly-fsfNveI1r)sz5k@Flj z%AoD7M`w}e>9W{TG~atyhf3Dhl-eqEv1jXtbr{k9V1`|(Gu;NJ@nRM$R7inZ2z|Qz z;QU~tF!*zI82;W(H(CPC=bcsz73TD(gMdTo6OzV`6h{hjYL>$*naMA!*#dyIR9mwA z%#XnMgN(^4-UvFIF`?xPG5d}AL4IcFyY+PeD7k~0_JVCQ^o`0<3RiB<4rcF2cs9PY_yMmuMzh{EPTr`i)f1C&+v%v-NJA zng=~Tp1F{2?|ti5XP>l$WG$t)s`E`H`6pe6PZ)Cm^FlntD#bB_)kv!R%FGJrDEIT{ z&{$^Mr;nwNW_gDd$@1^yfGnGYL_YC$%DX0oGjr>3HgPvUA3IJX$GVsGcf{$IiAm1XOuxDHm0ER zdwqASNA^nVPc`~@uW-&6LNc?ll8#Avgb>65m-1NtrLKAET09Eu!6+BXPbN$;d~i5}XiNEAM4oblSmt+EgX zX_4!;l99>F)b+~hncNpHogw2i4u4Kk_L-Ai!%Z_kojlS%8|z678AIQ*<=KaW_#C`g zH>ER-FjVo-L?a3+`I${Z>1YOGntwWEG{bl)1oGncm#zBUd*3VPB{@9gNHSadvA#)n z-&hqU7DL@qQ2u?JiVDnl2mHn0AXDd{%;k<2;H9`N-JXUGIO|okD{J4ck-D7r9vAse z9zDK3bt0FKk@6D{u%l_;y&pk&sO5R^qe zxXO?Nk1h26w%%MXoC{c*?1N;CVnAG)ziT}1dPs!idJB6SeYG*_5gyqmgGVpEb2wXS zo?mC%>LLR>^=A2=<+piK!0&qybjnTL?n+2G)yFmy-rmjj-I&P;gebmUJaT%IYP5$z zN{?GywWiZY1K;JZUME&VmnrJ}>le!Eln?vzkt$Uy+h(7n$2h<7q(h+K2UbR5?9Qvrx*LaNnG5B?P=4tF_ zKW*AwkFh=Jsbn>6M9sOPjsRS3cP=1_j?xV$O}e8FLS#Ko4>jlS@1}dNQdaX!k8U=t z{(UAs-(l1%$NCU!yMs~Gg17@xs;UIHbgDT{FVxa3~)i9nv8h`Yd67fv!C;UW*P zp_;V(xK-o4#y;t(NtCvAn_3kv9}dS@PEL5*P>22|w3VsydIj}G1=LIX&h?ufCYu~u zQNP2v_FRmg==PgM?muZc+B-{C{*7>pbl!_AHPQF`7d1~?)@`mgj;28+LrhYE{D-az z*J^BZ=t3LXYwUaQO#<$VPEA|S7PCXIy*wCxpQti7+ODyd(%<<8;) z=v`?28zJ+P`Vnw-j%RmED~q@7Xfa7L^XvQbQ5)KxW>@gy>Z`Ip^p7{IS%aE&Ztdt1 zC}zi4Sn=Dwt3%wc7qak`kV=w(|C(zGPl3uV1B}Kx@HkNw<#I)Xev)94KtrZ4r0+{J z_>}JAx&^f^zd+Eov%q=kfREIOq4ZKv5YL>ebwS{dbq*WXN+JS}co z^fLy(Nevj7r373gD=qI1+)frbOcT%~6Z*)ig4A+%P2m8m=t)^yh~v2vq!@gju$Ffb z(ZY+;wWUd1(SsBc>9TEzBlcYs3KgkOf){T z_!qIpwZ96ixZgU>7VP%jtB|8-*XF^Fh{f-gcTcMFXCJ4lklLkM%l>S9j+f#LafZ9@ zYQry(Meyu|VfetE{qJRrO|)x&U$%(ZzN#~1ESE3fX4eSIpATXDE`4`btE!)w`{9q}?E7WpJm4DgYw+$V{x|FXXkE__fWmHyKIA+S zU%S0EtN1!2wYENVZOKWsk&|r_Zu~FLnQtMC#XA%Fge$XjARy7ODbz3=mF#M} z`E+XJ-&`O5Jr;W0y?rW)5Q&LZl|YZz>fW*alPk5pv51v!H9Vw#<%kD`nfT^9n__E~ zat%EqnB+d97&-La*M`PMc|iRm>TfyF-9aJ$b;vn~g;Q`BvHK%J>tdsw zqPmoOzqkU0vAn$A=ERfQ5bB>E!{eqSy5xH~xW)#h-r(nf=?X+q*Klr&Qfzgb92@TM$gm)r^}Wj7w(fJxu$pps>qSs! zdksP`3*&+Nr2gs$(CZ=7{`ryGC@R8&{H{?BJeQXw$39c!&>J1qd@tMOru?$bHS=9v z`U(%-=vJJ9-s3DbuC-3;ijJ#<*o=?hQh*X$my~A=97in;Lvc+7D%2vm|%1|LmvCpVnuH}Js-!|;~7Hk|=i?|2F> z+dUPbXaD6w=^Fq4UKR8si?PJq#kcxm`|IGO_MtE1i2+f^mv{XBxyRW%kqEiyX* zYvTfHoF}#5QKH}+O0dwlP8`C!;2zrKdrYa=dj)`SE>OJv!FAqfRF$O+T&Qc>IOGhnDk=)qZYu)&6uwW&yv)V|eMW zyAhS!@e~fAjN1ORjp*BWuT`ELV<6N(qLk79J=|dPg2E4A=%xO)*QB8-e8M2MPzdiB z@y0{VpZ$?a$|!(!a4-Wp9Mt=V*3DM)rYWLHD6e4TD2@E>zuD{$!JAjbW+w(Q#JXRs zg-Z_aVD}6+Xyk;?1{D7WO1il1Wb04k;?C4{U7$td7jrI?pJHA9z%riKhwd$#h4e6p1eA!lBJ! zLi~QeME#AnkJkx8%NkbT=)W9h+56)JkCjF_kC|^5Zc|YP8t0sd#JAsE{xv5_xlt-Z zWia4A$JD0XP4l0_Vc38CHc{~7O?szh9qlU#ja}BOUdEWg_wh0!W1)vm#f!#!Z)D5KgWzzn<8$$b*+gF|alg@c`7?B>j=Pi<69>4{7qLc}htOFJ6ZpB}( zMBngwXd<%DP#Q8T?LCw1A4)u1B@D5GKQAEUhE-Y+LOlR}w~pKoo+PSp2@t-*HuCA> z?u4SuNv^#$_;X~emdBY{{y1FA3C;lDpDT?X?dmm)n#xIWGDNMZg`=xvo!ckv^$I*L2|=FC-yb znpfJ%z7Q4*qfS+Oc9aV0@nQ0tvJkrjKc0N3RIv9`qD`MlER?~VQ7sh`pjF;;?IBo@yS^+-H8@9rn(24$w(g+nu6IkilneET1e!$62hyiRNLo-CUrU8lSKOv z_^|ozTJvW&WI$3qj%6?l(siuOD?tHj&<|8aAdApkB2dFBVutHO!Y#k(u+~Wn1@VHx zGrx0(fOmxp1ZlEIq0Yc@){EJM1G7(Z3_(O@zC+4%@zyYfs}h~&wMtrzNe%;^L4{2R zw&XjERKhb6avJ%yNLIUtwVPSNLs1|r2p_=#7PvXVa2HkQ?LJ^!?%YzMzpJ3ZvyIt% z#6sHXp9#r0yhxzVaBV5&_O9m$@(&tlzFi_d8Y%bO3Cm{lbG1(4;>EOd;_i0$j8ojA1Njw>H?E&ZpUO6SiRGhy zCvcI2mR6|CPRqW{4@tY^B$2&}gF$!rw(s0iqzyC9q_1g9byJ2ojg4ECFo?r)DK|H^ zYwD*TStKg)*Gs04MU zMkW$fZ#&;Id4=OOiFMWRf9GWM20S|!b0(k^B6_(s)4KJWkNUEFgkXKjhq2O_ciueH zVwtQmsuI!ay?uh9co}LYFwxt_M9hdOr8%8SG){pNhvYO5vmIfyR;QuQO2LPQb8`qM z6|HkfQb{`=8#xkj;@kI4haDAieMYbs9NJ%vx-np@Z@-Z3o8-Ho)YgEC*F+UeY)5cBf{hyy2x(i{NTk2p>p z`jeV0F)aDMb)Q{Jf~+WqT!Tm9w*p!oH}`tx4+n<5|diR8{OZ&$84q32M&& z-B$e&Xr3Vzwj0pJyEfl1yl%)<1LB9G-Sk%x-V%@ApS=txeI=VcAp58_?i&;8#NdI&`~ z@PJ8)u){!D=BEnf1>9dU4RjoVDX#qATg+23x4vO9Sn=SzxZ;vX_pUk42t$GAfb|}O zL+5jnjC_jw%XRCQ1s9?Q}|*zVe=+<66g9=L%ZNF+5x8CIrj z#S>HF6VX23UE_HrIeMCMaxlrq-`t@#KRSqCfSKeC)ogx)Jmp>fb|D%t^Tg`GqQaJC%Wv3lu?vM!{<)J*k8mjW!;t>rPuO1@9gbm3v)j7J;cgYf zAmMiI0o`{+Q3Wt@6>UDG5*6m9VfnP1%YxpRgS5hFN_`|4bfVsk+vL+uV&NlFT-^P; zvY!c*3rbH`F|j^sIl|x-%$W0uJkbkG^-JZGa`~?GF5eO&v?@J#48BRr%Qd&h_7(qV_t6@Td1s?hPDABi zkk!dG;vH|RL6MYI9)U)uw1~z~YaeE$eOdy9SH!-<)t+<>a>Tm8897;AQR|HKC&pVYh4PH{QPZZu$M3<)^1l!0}>+?q=&(lZ4dB=Su_w z>FhFJpLL}CJr@I%otXGIVcC(DtqBT4`#J06Gf^xT#&*Np&z&H&A3y(RD#INXHhV;^ zFLSrscnUgd#QR|ne@>1|NErx*f)gDZ;Cu=`aTMQp|Bj74k2$oU zFXcfrQ|z{Sh6}4ADLRsKjBd#T>MX0Nqo0RKb=3k#P4uKB^(1M=AOpM7%%FTqtB4XZs?6kKOlNYr%MGLgx{ z8_KIsoq9Ee5Yor~{54^#@^H#+knI{lMqrk#TX7_Ned(~~i@mACqXBzF z2db|`XCs#q#YS+S)`VJQ;rI0O-eh(*b5C#mG*!3Bm^5m==|6qE^jyv`0=fFuF8^J! zUheIqTi&!6?G1X;2PR4Ws^h~pcO_i2Z$;`EgIw;n)vaiBX4RF$vSeu&)y6QDeW&~U zOB4C2I*`qD6D(y{6>oI$z?2P&{K`IzTgCa!IFv@<-f?Eh-_vP4p>`yf{Qd@md%EjK ze?AFTEh9x4w$;;(`G2(~SdDO(Uz*kH3vGfG2>wQz`oya_ZmCZH#u&*p#~p^Gyhg?T zl5ITaq$5y_{1d9eN{js3E^dCyZ!7|Q54KN9d z=rquSjih@AwaoT&UzdcqvOwkUM1MJB($G?hj~xSOKZnq^BxH&aTs}gSB6@a6StGqO`OJOa4;^m|0? zMRG8X%O+CLR}m?7(-fcZ!g1f)KXh({S zDxUl1bLLXFGW~ULW!Bz@P#mZG<`Y+!ee)ES!G-sro;54#ZM3O?mt ztI{+-Xmpa6V&hhu`-Fznr2}rmJyXNhkF(G_7TduO+K}8__9Q2>#aSTj$ zm&I5zVGCKcpXZ-vzIHBzs=WPu-up@qE_fJu<>Z{|cF6Q5&4EBoo`l0xUSg zB-zv=u+1fcMU(-BJ8`%o-j{<~@6mw~5}Yn#`GDXgqa>t*7MNIapCs#V;ko_$%fS9T z^~c7tpi8|^_)}1}C+YSQlK1aBPyw?$0na4W*&-P`a8xD;Mh#=`5_MsuTK9G_dnN?J zVzL&2c9<=f21_#!}`yX{ULIlr}@Jnh&OwK`A#Y2PO6X>}Jf1mTqXzEClQ^ zLA1(#8jA|UD)HM&RHS4jylHq1&VwrXiDY9nMWJm686!o8XMi?#lD^3&q$W@blvd<3lY47oSdZo}@nY z_QO2I3Kjf677TpYVAL#p_`dD1oK8>;$nfgOHxYX@9*r@u+nqMmr=&rQwICJXKAg(3 zx>s>JY(1Uu9$+&Oza1sN_VR3}fbedd6;|*=OxN?e&Pw^w(q}gD06*gZ)hp4$I>FU$ zOI8_4ze_BEOEFL&-D?vho>wR(6ob2{b}E!j4k+XyQ9a4sAV2)O-_}1$RRz4g5T)?) zeK)+`nq?cnBR&Puc-a*J4kk(8|2Q;PZRaetbODGJ!gLvMtL2UrhRQ;7w>+lctKT;G zg5w2*1?vrROlf< z?YwWu?7*o8llv${e*Nqfha;~36PhWZed-Lc-*daLkAJ?Qtq3{($2@tmeDI(vqCJlfISGac^rG-a^S;jp96%w}sw%j*&< zUWN2mijo|4m=E```l@E%cz5|Sn&=$`zfwFv>eCe`cC)|ox3HaN5BUxMVlt(OZw=-_ zeS&vyKM}mhmZ%6X{14V>=O(!$dkYX?XuKh`*mwkH`k5K;MAi$%$(+XzrTLs`smO$0 zbp=_7d9b@BxP|$@5c#M#^MYc5IyGL0tP8fua7VcIRm`69nK3Hz0rmhB0i>LY%@5iE z*=uZhyB6Df=IR?oz5OIM)y3;hzCz)w=yX16-x;I=|0Vn=HbQjCz5s6Ym=Q}S5Fvmn-&wqTMzoWe=LQTg!LCbJ93tK;iMGY1ts@iyF zSt4RypYb7mo*~R%A3N-EI+={G6;N2I3wVR9=dK09+UQGEcrbM${dw(0s#Q1si#U61!`HNZ$7?TQRmsp}$Wh=lV*_P^U~9xRy($-rmb` zlo0}c!iewSqn z4^qQ>qkP9_LG`NYPqDSKP77%~^lm+zb!$E8X}r5@nRxy7QyHg+&1$Q*psLvjs!j6b z{H4(=HnQu=G!7yn!hIh6PR~7Ga?0`H^u_1+bE!_C$i`xehEu)p%Vj4ea~nE$Sf@%= zd%6|f3o`7!mQLKt3H`(><-0Th$V)%Uci;S?_>*!;d55Uz)z8_HA`+zwf1fN}9H(UlIL+t=h$kql`9h5{d0On2$@u{V{`bW?x}`ALGi zzGZu%Q@4ftWFNbP1s0A?WjCcirvFa0-ooaY;8$=;Ye^ifUV z`^E&if+`CoN^cLZ3@MTgAYH>rPK8o8_~oO>bX<9@RhLY3DILEOXZaSNAs>nuw{{ac zblkyJa<6Mt^Dwl$6)8dialST@ak%$0oTThRKg0D6cAWx{1A5!;nHuZ4Xjiac1vYE9 zqmltIvNf8BrB>|Bt#k{KKGM$RGOhCb-XwVj8J1@b=FNi_F9pZ+0baB3{~D#7Buj1q z>BU=3hAT4kqNSayz=cQ0=P1h;WK0?5P#!@L?EG0)OgLkVKJb0LT*>lN|1-N%K&LMA9kR^SEUE;N(|FRYbCqw% zb9>E7ul*r+%ZpD`p$BQr0O0&pd9 zn)iDC8!?Ck=L&rp?k~wx7J8*ITHu@2JnB?M?wCt%3XKt6<2J456ucdBQHOGmS7@i( zx@jS|bl%4@U$(N%*2Rn?OxM^6VsrJDMB_pC#W%qI*cy@6lJi#eCt|-0PyK95gzQVk zK>yF3KkrWNqy`OXJ^li{ng{1HA3IW8Nh#ziWtIy@(fMoQ%_b zmpCrwW4<%M!JLHQ%=0^cmx2d^eK#2~Ju7pU3vlf?W<+zxPO9YMJ+iK}g-U32_psL+ zqqxsW-)B4?`J5|3uj7iA#E(rk^22Y;0ZGc&mR~IPs`qafeT?0G8oIDgrV*YC7^M4! zaGo_i{46Ow7elh`$UllEbt04DREdqpeWRjae2g&V3j5+#wGd(=;Occy1-f56ho1Oo z4hc0tPC4v+rZJN8WR3Nv-dDXS&V}_%)5GR!4J3TZXfDXtUBHpsQ)iG4;|Wy+13|p1 zy<}jXY$BTCDe5HBeZJ=@YF+VzMf?^#FhA5#WH-9*@HxypyYdenby^~r@wkh|>Y888 zt`?v0p)lT6*vK=^o zq%p+~saT2H@dPGm+sdV~6oBLh-VSnCc1)8y1|8!_(xO31yV8rP5recBHWU{hp)5Ms zVM@$d>Q1gfvAG$hy(&4)D95Q}ws6gnnvJxjZCJ|bGm%AWoJV#+I zIo^~H0dn)d$}Qd)sDFq& zWnQ(0s{*69PI;{#@6HRPA72(9?zv)-O4^MBoZmFc;>m> z1H6DRLH4n;O{vNk+v>d`wfn+H9&kq^LHMq0@dWNQr?Z#gL)KKdZ^H3YUzQH+;yPU28o9Tu7V0r44T51Y<-x!q-~YS+K6 zdeR#?B>)4%z7pH{bO!*E0L{M*5%i=Z>NQ{8P8jR1?ORavY-)+%AI?Y=$uBF<{2#Tq zci)8FJo1pLT$t&I=55)2UKMS7&j*{~INGbxC>s6yvoCCB!a~aBvr)Ihavg@?3FXgg ztNP&>E_b4x5?&K({=Dkra9Zx5O?odC~@MdjiWWl^&zb;>(VW80x&_FVRyL)9SF1)p>`hP+8?}hyGW& z(*?F$Jm#1gkZ0sv0tDX#zfSSjS;#qdC$aB4Tsw%60l>|M?8^F-J#ZtIlhO z3B4k&v2O-gAbVAQSe2zZ=EOBE>%l8L{o1AVugM|kVaYp)S+Gr(QF?s_%`a76y7`EoP<-YKz^nvT(yJ)ygvQKxEVP5yS z)Cf3HgrgG=H0qp2wGFEm7M-CE+;~IZs&42?4apYxQ~c(|oN*v(NJvlHJmsIN*+6JT zOVBEaT3s=lYSd(4yCELNJaNvmnLP2?{&Bnp%1({ z@R2`NkHG6wI)!eIyL-p>c-b{0zju+dJlP`lPRQ`{EMFA;*qunsp%32Fp-nOi+`Ogl z&+52_>6S6z>zuo>A=KPKgh06kuA~OY%W@4wQ@gl3^<^(yCU|nT<9qKV@*l` z3E*9=qtc=d$_h1T*GX|pbCy!xFllbQ-%^4oP?dV`mZl(!EqhdN@81c5W~umwNdnJ& z2#d#rL#pw;ssb<%C%Xuo$YHq?Ata9TU%eoI^p#&?0NZPKN?k&~zV!)9jk_d<)jpI> zA{*m{??I{OmVi1uT>DoL+;*Ff;JOEss|Wr)H-#9FFP2e@6>ztAo#94xhveO@kBxtFznQZD$hwLGc8tw)J zZD*g~w;lPo0^YKJc3BE_=EGJsSmkf7P8mhcN?R6#vrm}WAjEyCwW7%W*t$zoRxhVe ztO1wi0=;xRy=mpmVOn~vV5(?-12ZMdy{G(Z<823~`a5zIp&R=5Pepk|Bk@b1rAhmHps(&#Lj2wb--jg7Y2;T41{1uwjuQB_ak0t7 zw#0=C=vqH!fuc6cpdly;+R8c-41r%Rf^fk9L&#%*3lPA3K*ScU6b&E9%mckIGT*a${ef>NLY$F0ZB+7M zMf3Ugn}d!!yl#(HY&Ha_R=tL^Q8)y;ggegwdUnnz4XZgflg7W% zy2F}s%IE_LUC?Ia>-tE~uSSz2#~hg6?%7JVbAU z{ng_8jIXgmlm$Ncl?lqVXNI!9fntBZw*}Qp>gr22c5Ey?T4EyuASeoJJ7O`&9@&B2 zqXJOS!#dV}e;uQToN}HdTB8aqdT@(e2oVN&|hBDFA6$tAPb>vshTXOuZA4qQl zHPBrfCy6AS9p*5I&X)QE_!ErBU+VqeAJ|MI!d9I%O9FHTKe;9W6(NngrH?BLM|nOKInf1 zq*Ea8Sl1tY6x|c+Joydl(*(50@9|Mq8j9mmZ~j?F5BG^IDy5Xeuy^6=Hr!B95pOT2 zK%lAhLS$}%&T87&@0eoSVe|Ceb?(_nach&O{#M+SM#|BYY+q}2?tfr4{DjVOYt(DM zQ*DShLhIb>58foap>n9QPQR>ln>D8y|I2*OHL-c*kQJWHjBL7_n z;^q|{?saYtx#7d$Qd-U*;@|^V~9P ze_xH(4STJiWu~{dl=}PGdMw{W6jVGG=`G+*#cu*_>-4^>9P7dq;)2`u&XIsH$Ee<@ ze37ZAQ_4#Rm*Yj{EF5G$-fh)3*7rNS4BX?pMmZzBxEqz?!1*>%TkR9aAMk&ucEM+` z&!^$2z8}~ENa%{xp;2eRqktH~&)?C_&Nq-85OroT$Al>8Bhy8X&^R2f$bJ|gpg%Jw z8b7+dSn;I2y3!9oOuuJO_+6Q!ajegQ{E09mm{IgY5McAk=*a(4^DHZrr$P47T7jwc zDh^9tov}cCcx$F%!B)jcIG}yG>xRy8)pc=9&)Sv!tH98P;KbF)s%9SW*_U5*mNdo3 zQs}PzqMsoIu)5i62un9Rj9vk_UuMdsHa%_n>;uG2CXf1}`yn&xK)m(oYFpHX^`(jP zj=IKQcdTQDun~zyAUKetiIume5RX4{mI{dq5g5?k=OmfuOduo}yG~uO#X!{Hb$E!@ zo^l5}?4o{6??)?}gy!Q#BB`CvN?RwrB-lf46BHdOD%d>GDtml^KdwYO6A+D%&pLcQ z!`@AmzJWj6o@qdZwkF#^^f5h_E+q10lSLhcu9Nma*5{J1<%*M}t>b@MlRLai4ZIH8 zK9mUd#QYgcu6O)V!PX?aSEvhxT-0z4NX+{d^*81&kdHrh3kCiJZ+oEv-sgHL!5q3N z49^r486Zn|$rS09z!rkSpfUpJR;=FbhMH?9kQr>W1q2IG2^8s5NZ7rqC}h%7nLzmp zgVo3MG?p&}M~rpU6jtk1br-6>oGnFXxrxklL}6~CMcF7Tbhcu%>*W+-Uf7qSUTXa8p^{(d5RNq_gIz7NsxM|d+{Q4j58Me-}phX zZ3HNc&0>h<>V0~D#f>srV*)a@rCA#$v?fDg2q)brhWJ(<_YXN2kjHBS)-zd*M92_9$$`^k*wWXnTb0EIYkq7e*XrG ze1yHQR?|_h_$p0kE5_n?;OE~5vJa#-<}@u@SICQj$VB!{#JH_3<0fI0@=IugqZpRH`@^0i4vIy=}4ni!|nbK zLjzh`GuhC0unU@v(5$;3wWRqP4%pif3fxUwF4*dUnlow*&zC-AR!BNIH>J)u*ttGi zPxh|n|KNm>aXSVuYd5;j({0C>q9gUB!8yK0L{ceADdk_-eF=Ch!F*gB#FPb_64M>OItyD+UcA!oW0ITZKBrf1V?5V&7 zi16$^_T?*w^54q8&Kiv}b&KA?E*S>`)gfHNV%dKtRZKriHs<$}D+g^IU`0@cyRI5A|{eZ`n10~N?X|c^tG%Z(@e!V}kDDOHg)s9>bwYvn|ipZ-(A%c!!(|TIM zQpk)7cDhyAf2t9ciz!m!cD$bQUPtHXC&2C%oA@#9X^&9mr6K;J)xmt%@ zS9`^2nP)gli~qgr-S#v(fUVHs;|BY5bQmIKl6>#$#><2AVjoqPHEyi@ah$Ps>glKQ zzOX;@BPbpuN`lXT`jjjKNo)0uWG(|wCfrU>3mU(Co!6{OJ-hS_LR+7X;xAh$iI zm(ygC(yW;NBg7Imj51G;*_h2_=0o8jxXk9QP(MB-sR@NAUSmZ852$wYyE`!Es&}YM zc|tOMm6(g!Ot!^DZG^#mDI)WJT{V=O`UPN-t!8 z(_OBkQ_62R84zwk17`X!sJ}U$*1w=qEOE|FVO)CDTEFUibn5^x5K#;#ZEsz1X7cX$ z7zvec+Qms`#Psm-_HDX7g6@Dv75%tJdNzyz6!yRlb$EDgkw^dUs$Cgi!lBJ=y{Tug zl+v^crKar}qwwIJkP4UbaLASF-gZtJeD5*BDB4Ralnway75?=Dr&y{!+88JoAac!^ zijj}dPzNtN8&j4WVQSVs6(&$M#aY)?NMlD+t~R*OUnHrLUDxnKnk)yxCS>A5%aF|4 zthI^S(Mws+^o0eU|D?dJg%|B#QOQDv_JY#$jVUlYQn|Q~?s0tStv`yb_iH)DC1xw0 z`s`LpO`KtRjDKmoJe_{{KB@Np^RLMCX5OUVlE>sPhCPS3GPbg&y>dECY7~cgF*C5w z~;r&DQf(MhNct z=DyR3hssd+_VT3kEuI1&EtEeW5w&Poajyr80@XfM0Jemllz%w(YA;3dzga(LExO9D zDhYkrF=}-COJ`k+p8zesvG!Ix|5T=Bq-bM-Rk!$oj1mFNInDl`G~1)T66O}R|D42~ z(3}1qXXE}IKm29&qB~CP&dhPg{bTRRiz17m3%m2tUpmv@%l|6VI#X6P%XWL^Wfibp zNZV3&%LVZiOSpW=UOUTsM_j1lBT05}@zpa4aD@P&yTy+k{~rE9{tAn_fAQ1hlO{f6 z7TS@&Rz=nEL@yFHWKXsmiL@P0rc@^C>2`}ApYJ~IOHiY7FI|Q8t?^}yNvfdaiOCy6 z&66J8alT>XNxDgP(ogyMliL?7Q1{4ggLjW_8R(;vkH0Sty_$X)vpg^TmfS7Cd(yN0 zZApHObcN*sni?e!dGxPYIK)!h4z5wMfnPz=Na5>tc~{nHU}Z{V@3Wp~j7`8QPoUWH z3fsmb3;WRc^bILj^SP3TfrHN`(xD-fnU?bEw6nf5r#2zWZ-pUl^BG}1(B9tuXJvc$;3uI+_&@6g?Dp}|)GCe6pGQ>s zLKB#J+BhGHQ0;um`KcV7Y&3Jg&I9T~u`{>P819MgsEKYZIUlNH_jo6*U2y$gOOduL z`IkwVr0jHw?}Ao@=#+#Sap+%KkSmoPpSZHMks%{^S4(I0QDV7b=AGr6Skk0sQX=8r zJWEY5lJv`W7_azjL8^!(z@Cnk8gv*t4A8$<}OMq6+6S zB#NB97AOOh82@(7@*oe1EZ<)qnR%~pzkqU%UA&|Y_asQTa{tQ3zLb}_9R6k`_z$2_( z$++SI;g9d(kXwb{(cN2` zIjS%sN#YbN7iA?GnqYTOmm3m3&5TLvjRFpL4NTSsb<6jaczjBBxyhW-+m|74g{pPh z4i*bAhcOlrF*dRuS8ZU=CE4+Fb&?momrKe6w$5qS%s)?TZT8y?57!}bis7-RQ*|l~ z6vp&K=6Ch8=}ir%@fh*{7*3=0KeC;AZF+pzyYUy!B4 z64&m&k^N1hcD{|1Xj;xvKWSYF2IjS?<<`b!LP+xJPJS(%nuM5E0`aCVk1U9CXwsDG z_Ow40TIe(WW#dRFP>1>AEc`1fp8MZ?%fXuN(zwRKns2Jo9|&A0UKq~Bqr;S)`{7ko zWgUW}6mhm5=HGed#013#kHKMtdWH1*RJmZ;mp6fR3!a1#Oz&ImATU9!9;(d2b zyw~4~G*}qhvexb68RHKZ-@|ox0YB9I5Kzyz@6u+NW>5=%mP}r7JUp>v{EQ(Gh-k`J zRV*zd3;r1#A_p>p(*0)QvH&#OxQ_(91-LIril7yeKKkjJAcAK0UdV87yDr+}8k+37 zGdS$_XgW7AA0`$gaDC-)DO8AfoNyKtdgx1Wh$nNnb7J%aK;!=a&_BJTQLIl~I;lV7 zsqVml;;Y;LkpJ$vx~$)qTDJ}(#e)- zruxy2UkEw7HBkQY`^{B86n-;$Jc33N@W{-t2{H2q!Qgj?ZFJx4WPV!PS%V&csf=k@ zqwU<8&~4&|UU#)6Ndi{j>>+JyY(o{_m%%m6?91%Cq$Q{LeA#y#=`n(R^Zew;v^yHD zBwk!MwQDT9Gg}HWhJG?nb+jWRQO=Z|dWJUC5f zTYG9-s1GcHAsLCHI|QE@jlF^Qf&#l~!qe}CkXua?mzt&N;b9{ml=0`OK9tfF7nOYN zj>A<}d%!71b8TGENYlTU%|ceh+I09IB+>ssLnKYEx$H$=K^y7$zNp@2L>u5U>}9`g z6pjJ(etK^vZr$ruMcSUi9sC=f zW)IAGQqrBig2X}l#qM*g@+Vf?6QEIQS;W9By*C=;&6|+HpXDlCM?F{W&qa?A`4e-nK0UnP+gIc7*@G?GDFX|P zF$i9Y$Irl7#`A7S1RQ+{valHX}`^aDGK$WII-57 zlwco-3-ZWAM5Z^r1+_7JP=H>tD8e!t9Q`oYOBe>l*PMYpg+Zu>Eb zCx}6HNbKZZCD4{R)`TUKbfI||QTQ6`z#YP(ioR5&CbYH4LLmQ-2T(boA(TYezPJ*L zwBn?s*8yOz6PvJ05E%1m29+g;)n};AvwmmJQ=&kd!h57!us}*4mdFPQX>aN%IlnDv zIlo}Oz(DPL+t0CJ3@bcWg7`N7I{#|4@X>od?sIbL=j7k`fw>QxJ*1yGlct|b_^z}I z8bUGf4pA&mh**0hck2P)^Q{5a}WR8$tlqRQ!H#dt;aV}oqSk6^T1x!%b_Oous!-Ncr%*8~OZ=Q&-j`XGa zQV%q*FM&xG(23D>)0`X1WEE)D;Nqt2y5r0MAGyFZ*vO+u0z!%vtl`zZB9CmAtKWN> zQjwcnc)KJKc&ImX&x<3)Bn$Y0GvXch8}sIj1i^nYSv{kfr`-3?{kxkMmkTCZ#rU=a z3z2+lk~4Bz{slT!B#_a2)SMNzcs9i%%!0$Tu@<%6^o?7Fq=YG;Myn7c#> zcZExTJd$s2iK?mdaE-_D{0=SLf7 z`kd0MfnWEV@i^I;FDL3T! zCPwc~{jzDc)OG7lL_p6^K#Y!Uxcw?;DX<7(nyZBs(^?nXbbGiZ+TuRi^N87u4!Nwn zG?wj?v8UGd2j6@CKo2g6?$P!9qbG6RN(ddUL5+@<=({XrZyle$HzH^BY-T*};4+~}YBA@=aYNkSZg38g%zB-*`^HJKD3Z1VFVXmf0HRP zP??9$WJY(Ai2;4DuYdHnv;m)pzC>2|PWK&tdDzq8(OuYmnNOR&udQk;HMe5pSZm8! zTYKE_Aa%*)6uXWspTB+h9Uj~y^KU&+^WVR;JhXhb?gxd=j)ZbkT62`t>M0<@t=q$! z^f$RL&$t=$>Z6m37VO+FZ|lzmk4!bKluZZ7pT@O$^JtGCo9zLx7sUKpA9JT{jK{s; zMEQoydcYx{gT!^{!X=W03Zykj;y=IUios%cz`l{SZ#T88x=A>6B!5}Yx-rcai9qPVrMo1%3#B5L z^crh_6cWBL_RAsFh15=Ia+w~O1;4q$Be(o)5AGv+l~UY;+rx<^zDkkPUPZkYg%X*rz9((chfGxj$D$Wq&E5vm3Dn zAjJ*@b%l*Q-!IHG{up-iw-L233RZO7w!**Mj}<J&YVDC>P56iK4o$jsyzIpX| z_QORHf3I&?fwz3Hl?=k3=0CG+FOro1)X?#v!dH~8KCxIFx~7n9WFl0%c-<8E!`O1g zMeb+-wZLZ!f{w;)Rw|7B)DeX2u1m$tj|L#m>h6O|^ z;BLD0d1Ino0AJa`fn8SoVCoe=wzaZ`iV%ig;Zbq=9T0&!5O@`=$}&*2D7l}tuo}7w z4f%SnTl9^}_m4}FL$uFOxiy`)eL2{xMu+kznfAGiFK=fd`}&iH z=J)=3@7a`1((2-DmnoErB zDTEZ9m48r7R?qVl!hctwH<#OkQ7IcwJKgi6=#@NFktr{>2M8b=B)Hg&7_Cn;H1q=F ziK|})$MV9+_NC>_O#hd{Q}$#|6#{ljZEaVx;(Tj4jSWmSoxVVcP7?;CSmpgI`HCh~ z1t$L}?fh(lk6h5jo-()YBQJd4EwK$ZBUjD|JIz@6&Av%I{Gm|@E4ZA8Trv1-w0zOy zNxr>EIe(eaSW{!Ppxr0fsBiM1#J|qT(Y3OObu)?)zO&zIu507_#6!vi*Z&qC8yXp5 zFF$?vA>adl!n~#+@-CCnozwR1Rjq0Y@%)7%h#CaP(-1;s6&X5xpK&d0jr{?>%%{+{ zsZ5a@I;5;dUllLq^bJFBM+l=z!NNfH_ISNzk(F~KW$yq%b-(a@A=+&D?M$mh=pZXm zbHFq0Bkity@_q8G(AKBC%6Do;5=FDGEJlnvL(ng}_YKexLyDEXr1rmJ2fw@zo{XQ) z&bfQ!_ZuK0p)KvfUnr_YjuWZm^pkSrXq>L=z=Xk$1*4I=@aAQm>FC5&VlZA;lX7`d z?T<;k4m@NJFB=N_ulxm8=vPHL{j14Vb8rCW&IK|!Nx1-;_>|nKCbFr0$QEG8s0gsF zN`H`R*uebP8JM*d&bgZVB>?7}u#UC`lsyUeX|IYE#<*)D#Ht4JVHf9q=J=m; z(Q!)I>$#wCieRb{Ak-HV-b|?6(A&qIRBaa7mol1T<&y119tm2?79{E-aH=6+IkPCO z1KRy;Bwvg8r1yKIn>Z=xC{dQTj2Jdv-0mAjKegvVx;!$*wYgSp*8_n1@3mBiJ$y7s zaFe_fSMSmsTMu}dE{CiBMAIX^M;FqszzD|@N<`+y$-sumXy}mlk|@YOoK;>zzFDS` zvFlcd(-H+3+c`?e$%Zxu5Q}0)WkQb-P+v+Ij$Q|04Xj}INgj=tg13k#yI}X7upJ_b z@eV~LWd@h?BWLlp%vNkANC~+?#FCRTmFABa(0~x-ARH?OilPi#uu34TMJTNS{g)hs z+$7C;Q;afEJ2>6u+cUJq{vBp4ib(1o{e|=iUOu5}_wY*dhcci6vIzu5j5|RW=4qvn zdip+^mDG<2Ms9x2MANcai=;f(iZ_Oy#9Bf((OrQSQK)*(vGSJPT~alU^{YRKHOv|% zZZMB=2NQy1770Z|At~#qoL}Sri;KB|fa7Gz?cK&;+!1nzI9tfcPf+^z^<~i;$(&Bo zBoIWq`=zp|cx*d-)RPDZq}XCVo)J`$LgMtozBE`1@5WkR;+1+mn?lx4BJ$W=GU!7) z+3BA-O{!Pku=-?fQp>t?8 z%_LoT>WlA30(=COXZiLD=FP_oRYNxJ@uJPNIyvhdK0AC82lj3hw^qtm{_Sd4ScyHA z3-)-uyX&nvLM6tWBpb~+?9VkGH0hGa}T*pT}fxWFctDD!{+~`aTo`Dj;2DKdhh~;V_OE>Hu1U= zZx?0X51|bh^8TwIk1=e?-cxE{0kGq0S|HtKH>ShF=nsVW?_;74f4SAFO9Du(QFMCV zk=}ok-qcVX?tM2>)ZZd{BG`yK^gkFemUpk~kJ4~zCb&EnZgBCyWs9v8@HcdhGvL$5 z84|v1~qf+gm=kMz%ZJj}gmI5!;-!9R;AIVbU)Dx=xKcde1t?4%K{&Yx; zQbb?_Q81Z+G;EZB((0q4G*S=UwT%{$7LgJLf`E@mGkT*OT~edFYxFk8`0n$K>-z`n zy57Is_jPveb6%%|aYm?_gAi7_6MLKdWXJZ(+3Xk2;kk>RvGQ2k5h0@kRww4BSx0EQ zUd*NOy~DpAoH;Z|Rx{a9emY5WjgnY7KVHhjvuQ$ujP0!u)*>U~`P zQxT(nuH$$L2*?Kz75Z9Ur|tk!YXTmaFm9>AX|Gau+!8`YF8883Wro-19KFNt^|5AO zrWnn0S9>pOt8VF4$W|&D7NW)TdC>y^ow2Y(Aa1Ov=(0sedbf|C@r+0Mj{Nqt=BZP( zAcXUG&@rAW3(J+hSv?>VwauAK$W;kvDLGukp1+Nv;KlGajid3m~5@hX^)d*7|@O+=B zjyM`#ryX=Bf(;k) zH}=wKoufnRFHno}>(_3W ztyD-dZJ#eUqGe)`82eDOk$S%yYJa}b}X`H&~g?}`}riY|?^_Rg%ZV{60Khs!zC+xv<9!ccy$+qjWJ(QzlT zwhtu3P?%qPtGMje)Y>mhXg3mes&_v1fUVG_MOB)$d9I>2_u+M!mTE;j0^2PJveAcg zzFPFuvpT_aKXPNn%3bB=X)vy|Ep*pUAabX3A>7}9te8%2pPb|9JjM; zEA-x`xi^iB)2o;AdziVA8H3$n?AH%%mlA(TA*z=4-R z#z4fyLaQtv9Qa9<^<=yI=;OOHyw~q1kdY^u8*O4&@)n}M!#uxI1DruFxtD&1F_s#g zZ$&)&jBuI0?dUnKW43YNs{EAoDOl}1xorE3td2C64SUiIcwPuC7?9nTB*nmfdGGv_ z9Ur@QVeL#3@P06SP7Rct_8TMDUz;u+++Dj$v&Y<1?1)!u?}LBP=^ zJcE2jl2d1OqAPpL+7sW5O9OLk^oP=LjWEeG_e7e~u#=6qQd;L!4i|5rh)KCNT$&*< z;i>4ZzM5bk$-7&n0|+F4yxu+0{G(to1(&YWJs1{y(!y}607FQ#;rtKTI>P0w;Y0Lo z6|@wGceWVHkb-N3i`w%E_B>M1-?ge6B=ylF3qsqw5bcKnMuH;w7xWgf3lDn|@j>rK z&+^^j_k8OYU&lh#eiUq-rov&<|0e|M%C-P*cmMW4Ruq}gF&v6lqm*1sXgI! zf!G7v2}PR55x-HLH)UJG?_@d^$4zRMRdKTaQ9P(t7}zyp?tS)=l500k`_9I6U(|*l zE>NlfelGdw=oSjM+xPm!(h8`5C+Ll$)_j5LD&xX1h$(8ws1Na)lFzQPMqneS^Fjvd ztS=c!Oo229FJ5(eZ)F${fzt=?Mn4=f3ehRGD^d1L@Fn|3r@hEQNE+8HpPitJcdzZ{ z#=m*7=PciP<0a1*QE*2M!&XFh@04d7xi-wJ^P5aA;%Ks`DQSE3=e#S->|KLo8Yo?b z)@kcOKh;yFx|KuVjlpA+^T);z&(9(TJX@NOlc|#|;+Vshth(urRno*So40$jE+RYAz9hI28M zgO$A-RW{fFAFdImHRP&ayDagC}N%fzf4`zo}lQFJ`s2q%owDkgv?b z-m{lC4=SkAHX&qP5IldJo`{5g0@qVPS(nu8LgOAMYyqAYvRz9#=mnHb%SV6-nbfw| z#6BT)7S+^V1pL2#sOtO~Jk9^o;c~Kp?TuPFE}SYTJ;HsS))j}IlGPBBOZKMkb!`r# zLGP8C&iLi=u&tl>>oZjBLd}`j;FgZ%gve8AHDXe_FZVb|;K9GRx5rFg7fDUm$R)JHefhjw&TybwzADu zEt;QI+dXKHMLYeCV7+^J<<#_mLFjMPXCKXZ`WQi%x6z22$SnNQApMn<3ymo&%H2yR ze)%@HY27@1ryn&ghyT6^-K+U$W6aDuU<><+x;-ZYe*yNRzT>>Ry@PamvY%(UoKk8v z8sXnJAKeE0w@r=B<*;fp5n!*o{W?qE-WIPOM^BWdfz~7AlE8P|j;S1l#rKV&jtOTDIQB14u_LB42<+9lzyGb>2kP<_;{0!U{A`e*K(if_3rIRp8qV52ra8 zIB%=Z@`$F$PgtXxDdp#UXQ|NlqY}>(hMbAXme3Cpl_Kh>sY~j`jq3UV>AfrhPMHlA zDYDSbr~hIW3uu+9xVjl@HT56ZDvA3!lhVOoej7NgR3{tk3NjnK8c1BwQ_`?#F${6f z_QLc%4_RDg=ui(wM7g>MjEa8|@N#m7{SrN!*a&y-^ghy`59pwwy$FZU!K6M~*rWl{ zL+;(TgBmu*DV+k9IU7AfG{p%-YH3bYUp!-tBC) zmuWLO&rukew;mtoBUBQ(u2|4LvS>RmE(`S!NPX4h>WK4n#pz$#?AT`y65It__Z2!S zzJL0Yb&5MTTE7#ZHNfy>VGOj;VLt#8*~xO_BL^*rs=Ntq=F2*YfQBJM)-G_OO(cm5 z&4HLmm6SgItXy(?8fEa!irxz@lL@e2e9$9c&?78=Zh|c8-L_}^znOz97vDXXipv}^ zx+;qtNa6B+nn1_<8fSObsOV6&tw$3HsN1>h#MH4<5W`k%?8jg3>{*KL8i%j`tR*gH zh8$$IFmFPs%%asOkk&ilXw?s#RKj-mL|gfhbNAUj_Dfg**8bh*EqSO zsmsLxHT11VftaM6x+!+%OgoK_KJl$p-_*ul9$Nr23(*!T7g-EjhUNDm#D<=%29)%j!U%o7D+2fK#uFhN76e z5!-j)(S6Mh;`)27H0myMOg8f)0CO=i>a(9vE@zJb# zvBWR?bKTGAana}pNv+q}JXLpDT^w{h)FL!Tk95Fvz_H+$--3rh1-b1>5eeqZw|5j$ zd`?=#fuYZQ)Tf8Px(;?ARsD^C%6(-Pku%ER*;jt!n#vHv0nQ#x2k0#?;H{S-t<<37 zkg4*15Pp^!p++7Z%7}0gvWH1m-bH=7hU3HRkOB`x-#@U6v&%ksJC$|t&ya)2)CbQX zU{JyAhMxSmWQ^?t2Lr7x!k!Gr(4_;&W45p?KZO)FSwtJC@HiHIvCY0%lX>!~!&egU zI~RVfh2G*wfG)d1J}gJe?QpUv;I`air#@ZCs$$qdr1+lYZlk!bi`76gbr;|^Ad)=p zgwKDlG>!C@n$T3)MyvY4bQq)(>97$45f!xCOav)#?BUy6j+$%PrHs$fwmr|eP^8iY zo<`}*#0~ly*S{Q& za*+7=@^}$YV3!4yLl~XTMIoHq_~Qh;pib`cfmZ;={zf+4=}OD{z$8Y7llVyU`#*;5 zk;9g{`|4c>wjOPJ?i~>Chxgoje2t<|rn%^vRpTjyVP?ady8%uPVXA~=;yi%c;M~~K z^9!rzZwQS*0>@NT-QL)(0mZ4u9>*C5%vc&kz@^RFf0PAS4F_UkJ;3LaOHW?x5Yvc# z*9$khsFz_}UKqxfJ^UrF6?iIeierxS`rMWL6tU5%Iw~dQIpTKQk2Mu>R0`znG;43N zRXX~UA~5JKgwoGESqn@BogaGMXoYoVdX$0^>hK~rrejd=cJ!z7^>*caPKgdvElMW* zIXda{iAuNQecR(zGm?U(2@m9ix+gBSMt`_c7yk|q7-0WdeoCOApeOrb>jl%7MP*W; zL&GFvA#}1cVHCg*bupdiFbD|;M*>7b5(HPKC%zNBR>fU}qYYb>UeA9KdGf8K~#yq|Ps* z3-YY`lfeRZPc<^|3ilh=nZ3VF9_=g966tydP|25)wl+ptO@lrAXKJc>roi0Zy}`Ap zQ&b^&rb=n)IT*?9#_mIl?olw(C6z{0MxG^|j(lJn4C$>c`KA1A(HCIsyx;uLkId6` z_R}Zy2D$H?(nYly7ZNQH4@NGQ@qL)%)HHV@+!9W#S~EO&+|N!cqhwnK6H}qb(USAWX&(tX zfuTyiMAAH*I3Mz7{Y)9;PZ;f(rQK3)x=Za=h2W{(Dnb~wTV*w{6dKjzbN8>=OiVqO z_3*L=XCL`*WNea*08P%2{`yJabg z-~RzI()~BejKAemb+>@)cpo|1_g{i@OW{+@7Ed@*@Ot>6OP_DMxbJtw2-9eOmYg2* z->!E{N_;ZH#ljT^y2q>P_Xh0U0WEQlaY5<2^&Osx&A5P;leJ!1&Sm_Vi5!sRd$|7fHqbY`;!+7jlFZqu$)=n(o)R%3^<+#E|w9@a{%@&_$55rxq1Cv>IliCH55JYMzcX=dU1qpc81M*U*ang=&GsvqbPzl;DdsG^z2$8 zaJQ@XzJ*QRFF6cH!FlENY=QN{fw(LL?Hgk-RvFs>Z;C=L*Dg29(XLE)s{hSq{+Pya zkit9>7rPOMPS{+EW0sz_*yvTQEPZ++!s#8IupxmJI z7rM&EvuR%a6W@baz?HLxys<(i#gt%HFn#<+owP5IzP z4PI>affx{q>+PDT)Ub@eOTiGN8?IQgZ0E?hlyn~Od6c%&^?x>Mytt{CI^U4wd91;Z zpN5|gb&Bc+{3IUN2My_A6I=s@fq9MY3}x_FP}O?IiF$(ftz9O3%0SIo*}=-NJI$Z7 zx`cz+Z}L(cxdY_fSkP+d*Jg`rqwupD^a3ZZj}B6?NTKjK(y8~C&1*qozW8wPVYF5s zLP%|Xmkv}Z4IL%?)d1Jl0Ry6*+n?Ftge)B~+BG}W)#DJAPcU}S$KOAHEezeW#8@}E zD;Jsn21lO-++~<_`}52JA-?~uD6!RjD~j`elLc2WqY3O`Hflr;-?7JK=a{Cb<15TE zXl!{LweX^#L{4et7Wc(ZK#2WG{!eh;Mxseu1fJ$lqifAd_`PgJ;CdKNzkW>7cK4k$ z1gz8Y1?)Z_=N1mq;{fF0WO+#85nZe#= zQ2t+`(wQ9wEFt0I4ThY<3o?$dbS&!< zbe!?$qcd?~yQCTKABOb)_cMM&@P5He)wXl889m`-oA46^rx(6{|4(epIr9NO!uVFs zv&nT0mCK>aBE)Eh6w}|+(P;jBu9IaC*$dZbcu=RQP#_%pg@KsCX0Nzs?)*7T(G_(z zpsAldK)CBr=(&mWXs3OZGj)5_jgUY@k^3_hIt!;rnWL1R>*k7V)y$ABC7baWhuqV6 z9I`AP^Nq6lh9wqQ_;!LO`&;8r%k|gC78_*ln|ysKeOcLHUT7z_9T^jySRHCKdilZK zqyKFp7u-H8FeE>rex2Ekj0*#)3N<_sK~m(`AG$1%Kkj?ys?9$!o5t;Aw;P|jw#?djEve>EDr{Ckumh!3l%OQ68>fs>tW+BsZ!d(6&}+5jzqg|bB2 z)t)!Ub%s*PZ)(jSTQNL|U2$$sGkCH-#pM)7oD#x|Nl=y;Lt<<-E`7nhQs3c^D1>Ww zCit?VWu^c;Pc6CNhO8%@e-_W~DRqH88!(!U;q|72-G@D8$m!)c*?~izGN+_UhlJf)-v8`NVw;m zRr>+V+_5Gq1>+Aa@lUMdq7H%PzM4pm0G*jb6BRgJr3M$);|*JrVSO*cD-o$my+ysf zPi?>PoiyW@I$27VP0tJw4&~6C_&+<&$2w%u;Xu7-=I7sP0{D{Qwg45UFQR}Ge{(8n z2##xi?X7PO3F-Wm);q0;uU#6hfL6u}rymXZPYDQ|BeXixz93P@pC9!H%<{YTUH$pW zcLZGMJ_=5QT+q<@T5OJvth?`O3^{z_B7I*T6#(7iV8j6q#l%-!|0WH?ONOeOA8b1h zvnpx;qnpQ77p}pb1~u2Urd}OpqD>Ha0_zAZB>n4b{X*3MG@PrF{lrd{CVLkyiUhlg zt?z;@Vqi3?_wN_XE>Qby#OKP}&)$DZ;1<=Og$Cf#ieT?^U+gw5C8>jsLvIJ#H=fX} zUHnS?{OYX}GlpvdYc0gvK9j=602h5Bc;vb=Dc=gX$|awtW<=VB$-QRA78Jbz%{^)3oor{A zIDeQ>DO}65M+sR09ei<})i}P3P53FJk}EhXedap-$5z8K^YOYY>~BqKDF2xQh3@10 z$nifKPn&?yL&NMk9iwr#EoN3pQ)vHL+Q$gSQA+*h!bw~NW?ORwxv4f|@ik~?J6IjK zt?YWG(8YSj6fsSeP(9xo!n)f@eaEJ!@e6UxXImW^9FAKCa~VkIUXSKpq*BJ~zDnM% z{J6zPIo$FhV{5G3`i&%ah^@d`odajj{3(T+;<2MCbC?WU$DF*I7!w}w@DB}Yw>^aa zC!8w@ISlKy44glgoTs2c&176+$iiXDHu*>fvy`0%rtr`C68|fs_N2@)OVp8E`B3Ef zuf>z15W->mC6a1vRsag1jDzG)gP5l65Gs=Dl5n3|oN~Stg{oYzK(1YcY#3Rhca1Jq zdjXdn?Y+)vwHA0sK$ARGwVgyMLkM5lV{jM;{$BA}<5e z{Ndf0AmXP>N*Nl~y3F}ye(?p}|A9_<1clyM&~{*B;g?HFk2W^%=<{h|5Jh$yDas@? zef9TrQF4;}v{ukwvwHgY5z(m2UGjVPHvQwunh3l)4Ns5wjGd>uu)~d5aA20?5VJDR zl>j~Em51O!qaC|NgJ2WDl-vBH1vRcsR1}HZ%VTdftt`)qdqCYu>U@k!D=BA}CW%d_`Kd0B#YCmx;J@~V3TkTmah zhAAAl^uS)N9Q|`mo_$7$2Jf@1a-gxBY7%~cYs1yS1>SA#A;KBcqllonwxB##J?2f zf!WIL!|}IdNm+v$IX2T0%)?hm`zNRk-=&RT7czm%9$Ze*mmH31Q&NXT=0Ppd27e;? zBEI(g5j%^qdSLra-4dHxYj4IqKpTe)Zy&kv{=@oi{ngTyPD7SCEbERaEA?5ot|$z( zxu9;b?zS%P_&;Fr-+XhAO(or{+tIWt@MwNf!s9$g!AMrh6#re(i%;K!l_RiGj)E3{ z`})U8Z`({iP&x{wY>9dq?&p8fe~3XAU#sKD_D$Ir@e#ow#M+H3J}Y)o#IXalo5dqS zSqcI8=PS*TvtDp8-6=*hhumDmg(2Pk&|?Vqxea=U}gG| zRrz%%vYn-O%%nRb*)&#-bYqoqw>+_^n%JuJ%jrKHl2 zgmbfAvtej$7HW-IAg;S0cI#^}^-ZaF|L7j+c-6ZLqA?wEXFD74frjjUoJbYxJJ{cH ziEl6TcrrP%k(a2b0azbQ#@81H~YX$2CRw3Wgb0t)c~zCieKEQG{i6c-~{`-Ft7Zx z9$7;_afNMwocR9DSFaJ@KlkXTfekDVxmTmzZGOp2=khZQ`2(eC$K1chN+t}#o&3i6 z@ma0;C%TBOe$ppdffKsj^;j30m0x-jk3?a5ErJ+hteXtnym2_hqDzRVk;7*3RLB_> zXq=AuAj&$Wk(IZ-!u-*Bx+M7XvlnSs>97wFE}a0n96fkWeqsW9WA@^%9^%+tS=1^_ zZdXNSr>&{$n}!NJ#m~ec-~|%+wO(6xV4at`E`k^Ue88K-8Hpz=Q)yuB=gSL&r5!Ke6Uc(4&yS`fRD#h@MEg5G;}$d(#`H zrQ$gNApEGLfjeCC2oCxJya%uQ*yoG6&5l5c_eIILyl6j^XE92r^aFc2v$Vt{Gt09@ zXY&%khIQ?R9H3EF@ltBQR(afbs|2iW#x^p0ZneYhfbPEWsfN$h3upCXxu`&sy5M#1 z(22jbl@qJo$=I^E$*)0vQVcg@)2D=P%QSZ^Hbz8?Ku;nc=vnwO)gEMr%4?Y(?hyxj z1a*oj)PIG$qq6QR_;zm2+-<(`k%oV>gh%1EVXO8CBJGacQA7r3o%~+kKu@|Ijh*BDLk*N*o-gxCu&N`! zu4&@^BFv7LL{qJsGNvl{qu1vR+O4jjdwDI*S#JT1fJ)GCPA=u)2ism zxG^$r00gHaEM^%an>7Zk@eAmXS%-uC6ve@Egw4Kvsq2CHUN*HVDKjBL!obsp>R*iH z$}poB+adVqlxWS-aKCL_YLK-cHpVX8=(ycP=M(A2%bZoS`=8Q}M64T)IXKZaAlFdD zD(~}$KKtIjH+L{qMveY^z;zb~P6N#HV(J3z&>;Qj)+csr)xzxGwLt?1+$eHt3z41e zKb_(e4zR`|l{kbILVt6VmWElV?(k^5?plv^a`=0@71tH?RhHg2V~}e!N|pAM>GQ5ded# z1a@bQE|lodB@B{<)}G!$Ya_P;xWAjP*U@U8C<_H`vMlPTOmkl$AG!wab?QXzFAk-u zKI(!*%^T~mTwGT^?w_C!+GDQ#Q>{92YKIlfxws=i3j0w|k%+J|_eT&qT-AArQ3v${ z*9a?Urlq-)wVYRRe*7mo?N7c3xvJDbSA`emPig;-Jvq*&5!Eme{7`N_&+Ve*F&pR= z7VIKwosuz7Q>{5$WZ2R$^GbftVP-UjJUZg@ceFcb2NomG(FJPwy<>qW%XgvyHex(D zk`vE-xq@Qv@_PJ^ug@We1#!O}KaA{=Nq}`_QCJQ7l&%mB&1r~lilj^5e&7PWz^+6? z2oj4qoR(Yana0;ziY|Ir=?5=6FAo|T+6Cs0=FTkUF3jhBn=@P`_LdUrfVos!0H;4; zWnquze>d!pkYtH8|7gzgkLhg5v)N{jYBrW1`=YL=$2KaO(Cj;?Oz^1WDGuxMVt-ES z7wwGF)&-;lcwAdKqn(l-;dD6seOBH3AfK$75ZvVx2C1f38MHE&Sm6M<$PsDnSP4fO ztfsY-fwe#It&Wl+5>Tn_Lh@bPsXmx;A7Dj(xkFzDK2C3H!M5FMBZNeEU18Bg3L@Dvb5}IaT`{}Wr zm}Vxku!TWJnx@bimlL-4;%d0~Pwy@R-dOy?w)!3#0v!Pu1s8Wt1h3EQg5DN&ocwz=~r(4%F4a+^hTCc zm(3B@z^--8(o#230i&jhxg`#@U^)vmBJ^?ma!dH|O1AXWbm5o8h0Hfm^aI(;a?Ep*j%o9CIy@m2q>FHLGE> z8aFnF!zUZa`Go7foA@s1ut`5M;0g^W3-;I$`W|3i!hxyMrhhi-D?1 z>XYZtPOVwiTFpXbttOM2jvQ$I^wEU`yeqzI&FBVdx|aqPtpt)|D~}F&Rf-aU|41I} z*Wa3{ zGZ7LZLLRb3-3B`}{LJ=hI<|`N+rsfzcQ3QO!zwPe_9%{T>O~gI+8JC_v!Rb3`NBRL zttj2|_qH68*f!FWph4!t+7bjOrVCA0>-tZ)^lv7Kx{Pc(TcMVZ18w_T%9B)Y2i8HOAP`5p?&AN;J+i0;h>*{Q~{T%w7PmG`HoVBnh=vx9W)9PkU}O%x^7}DQ_W{ z#xl^RB6sov4%U|NeWpGEt5^Rlj0#;3|sGp z@M)}TMcs+cXY%+8I5N-Mbm?3_vuOmN*16@&p$oC_`z-QM1Y{}3m&(1B7$KkayxLO? zx+$!eJIo&ybQ1~69R`w^JTFB5)kXecf^<4(Ifp)xG0v^01*R0Tj zciG4={_h{F_JJRwq-3-Qy&#m+zg*(Uoqg8bstDNB`|+xfrw_06gLqDe>=+}+sY3C0 z71KtqVj#u}hC7dDRu<-OsqlM7cyusy$4jM%#b?ZN$T#G}kUWdO@z=+?qlViJyQs;Q z27|`C$jUD9OMS3W29w!4ZDpt=<`IDCAfU1Bs^DrR?4X+L`mLb}I6CIMN?)4%^v-@~ zEA^-zGR2(<<*td;c*IG7oul}rOY2d$@EPwj6Pku5gIJI%Io*dl1Eeluct!Je{Lr`Q zP4D9;Ib)^{eGYz|r!pbpkq&_m5gXm`6LZfJ0m8xEL247A<4FxYiP4t9lI#H4$yskh zLpJIvj%`%8WLg;l{i@+SHtt9$O^IT9S5D!@qmsn-ZSoJ%(f8{FJ}*w{Pj2 zx-=~%DBdO>nDkyDvk1HIq!p-aH`kBzj)s*P9hupvE_y;QVtr|YCg#&%#*JE%0jI0m z>L%+fJt(wjZ?5IM;EH12_A`=5d{yXglGsq*(Zxn@o?+XC;wSR)ImOr}RWw_v66C!i zY-bUp5BVJGvMYa{WE-jGzw_qZT2O7PS?8y(fMIbEY>h-us83T1)Q>azC)&9*-4uv z?Z_*t!aW!rtYHTafVETnd~FBG$X*=bD?j^abDUcxt>PeCDA)exn-&(GVckT;& z@t|Ba2r?JGrS*3-i9~c!u*$DEAA)-u>H02e$=E2whSH8)vwlx9n7@0QuX33V`C(K5 zlLz5ayQajlvIoCX%)I|~P1H-ueU;YI1qDqz3FKG*3PnlM#< zWH!yR4(Y0y_{O#=?`=jC^WPk4_t-p?ya_&x@=F>#)7*NrdIs0qbKj}?9jsMt6eV{x z_Gg5%v0cAE13ty?QHjo0lqoL6oI6NpgcqT zGK>!7U&XPQZy;f0XKX(XD3*DjVS3G?E6X#K{ACtydzfV$4d1ziIj8$78d+^)*>R+=N4%Ifs5YnS+ci1-|R!hTK) zB$fr8&X)a9<&C2*U7EM&E!I-=9h#n9GR*Pni^8TqUuao-yQmQJ0|ovhNEgrZ^`HEQ zZ{Adh=zM5m^gOrW0{#{7Bk~?*okRlsGRS?*1Mhz>3qY5Wpv%P=Ih@pyH^+*R?ze+* zf|HIFgy0y_ok$+HT~}OXJyIJVmX4JC6dwUS8Ts0XSSiXLj)idrhElPguS!Wqh>mt% zL^%H^m0YgF#|oAFGkP>8=p;87B}9x63R1Rr4j#U_jya#tmW?9`!R|=c5>9#vc zSL#YqkbCnpKZtwuYj0C zTFQCqQttfQ;sbL?m1|Nfp(_A0m>1e419g1@Jl{!%AoqZd zf>AE_7hWgl^pJZ4*nHzSagRIwa|e`@-ESxe$_w2e%Cjs7C~AxFXWB9m1Ff-aP;y+o z2kFKs-YztE-gD^MB>8$2=tuFj66MlJaI|es!a!uWPp3+&tmi1lr5#1_rM?QBtZg<_ zpZrmvo(6Pi^iH6&>CcH7AgpIV)l&u-5TeOPYN#AI|M^3h1fS<#LD^bk(ho`)+US<>dH_*PTz{$#o<`Ih!cT-@K+)Fzu_q{x>`4 zks;&7PX%LuX!7(>kX0g=l4cdVSJ1B+X{<4zY{;#B0B!XR zw$bR^EO3d~2;rF#V=Jc}b+X`y^i<=iTCv$QH7VyCVKlX&Lxnge7${7Px zxVi7p+!kg(9=riLXO^yB=T1vfsa7;;1;Az|O6z zt2hVyMtAW2)Q;kG5Q-7vBaBBy~>-BBxO_~zbAXs-hwsDm@2^K$2C^+Cx z*8=3cmt3ts#mA?9K%ANk*|ake?%GyUt_&=Rf7qbUuGQ#@Xx^pU;E&sEgxWHXbDCh@ zbN5tfN7ZfC;lA8|TND{bPsh9HP_~NUw`}_C*Pb?`{LR@7-l{6Au{af6`jhHX;o}(t z>W$hhUANNeOdP{X&Iw%;QN%nDbMdfm2xUQlb_zaDJx=K)*29J6$bwe)wxBKm4?a1v zfh#j)OQoH4X(;&2;n6nDsKl1V955Kd{~!2)f{yg547A>m=7fX5k+bt!is?9-PmCg$ zq$4~KDl7n^($d*?4d%DK{g0~@CP`_}UI8Zn%!WHp=JVa+_{oH*c29E&#u5|aww};& zB?R~SlD^73bXQ>-tHiUVERzjqcFS^)(tv(*Weo}7%Cr|C8d9vbYKp?{^1e*_ab$W! z`e_k#-_pi;erAo&?icg1wr3$VasFbiO9oO|ql%l3k7%V1_ATMvu|C+8v?YFTO3S6< zMvs}xKOb$g!`zpG`;3l^snd@#{vH9J9sEld0e5=q_Hdj@)9dJszJ^%MZzZyk7bzY# zS3^2xWBegpU+j>T=d{UHAy;{ot!>xFij6awi^E8CK6_QVcmBN1LEIt<)wac$uHQrd z-L|b*bgszWeG<54x8^uSWmJYNCRXG$YLs*J z%>ZMC6%t4IYF}_ljL(!$LXo+$7RYCdB>s5ayr=(NfbjOZ+2G^Q`JLcgnH0Gy!>)b) zVtM2-(LhjNxv?{z{c!PUDzRMd3Fd0_Lz2pkwt0o?eRcPX&nXjSJ!y5V?^*`^ImaVZ zAnkMI^E9Rvg3eunCmeb0I}xK9{`5`!9)m7ZUsO%T`6I(4M%>vfIlb8a*;GF<*o({0 zLH$w#`bNH4ww6Dp!|!?{U`|ssbNH4{d`&%eB`_{=lIu!u3;p~Lb64k`IQ=EvU&7jr z^^)RzU!sO$D~}m{*CvnD-$Qi;&J0Zb%P9ez)8YjKFC4b&^Od4;w!yM;!Llgt5^F>F zO&EduYjAav2O`~@W|_g8Rn(j9y%L$Q9oy%WaJxWC4)K6^Lqqe!(Klk4H<#zdc7mv# z@ky-&WtUX|RB`dbovFXOK8e}R_`dBpW?~aS+a`M%@JNFkWW>35d(S?#zQET z*D~&7m$vk9jzAJaOSC;^`Pa}6ZNMAT>X5wlo7;bCC@!B%6{bYE25d`Zuwm zGlT2FE5^!QThfM5Y;tci<%(52a7BlI>la5T`c5W4qumM;Zr(_UpkWo0OU=2^bJL+_J)j>K` zMF=^TNJiDzvBZqX7rBI5KGz|LPzST^=vS-!v6l0!H8VQ|j90zIl?lj^96y5?Ur80W4ULF8p7(^txm$tiDLI1jJ>D`%Ue3K-+6vtBG@ z@PsGn5L=qQPx91O^1Bt(5fw%(sIh=blV^9ngc;(Po%OJ_&oU;oMZ_#iN52s2|_860ynhWTmTLo(?GxA9zg1cbMUF(&hiPzYea?F;CYN-}uQ6 zuok6rDK?VMb7!BY?cegorCW#PCT+l5re~q;+OrE(zB~VO=FSMXOFC}ZK_JX%XdL$m z`jqISJC9|;#TY1#eRqU=aR(L+d~teLDsA@8fhx{)-=#!5AD$TvM6LDKSgc_LebwdE z+uYiy;a~;q1MFM1`dQAz2-}0@M|Lge@n56>%#Q=r2kY#U&BoL6ruc4pdP?DCOWGs@Y(p7; z)n$cr^XdkD=)vX=jqkZ4{DEg<*+~m$P?XlTYkE_4>XWHgqC2nVO*g|8-|~$3+gnJj z>mDeCjJVqi9BP)dE8zdwFK>5eI|}OnEzGPc6C?;vT>f)!sYP5j?NPgmmH$}gsvhml zq#D}sp0^rH$e!g>76aKS&c#-hv9k7r4vPf243VHi+Np))dS&v@`-_nEb%$)>0t)qEo)yRhWQa|d7N+);9p81Eu16!Tso`r{_nW_`Az|P?Tm!^ z?#Q>sZh$xbJ0fIHo9NG@o|cwjKY1(5&kue`?bRNmZXf7{DN^&YH^hItUiD(nmix`2 z;Vs;uriR=MO(Jw7E54+4mfeP<*?lci+&1`fWgyr^C$xrYXA=D{_J-qJkB`1IUiPOz z7mfonkpT5U~D`6hfM|4G=rCgs8dbp{vaTogF$O z@|Lk}Ac4Vh9m&dlzma;6?MdB-+;|wjpb6UyU}HU@^Z_N`g{pe z8+ySv+WEq*ROfnhz_m8{LLsRW#&I@j*R^5Eq=i__F`T3nq8K0XnQ&`kDjbsObNQB) z$S}E6w6p z4H8?AU7VxPTdvc0a*+QQRqy@J_WQ^G+M`O0wzg13hf>rgv8hp7eOFr}_NY}gY9z5& z?NXaiYP4!oGxk=YRP4QDM64v%$>(;y=iJWu6JEbu*Yowf9{2mB31wR401&r#Y4upT zp;nA>t@?cD6p~*d*e5wxa~jsK$9odE#K-iWFYv21-FCLptsn@%THE0Va(3D;)5z<2 zQUf4E|1G?`9b%ROFz8QvpZ_ECW2pJ{AO+Z3Py<~m?%5;7LpMCr=ND>=SSFg~;^nn` zVS-||pdBUerK>5upCOb3wu**|pOnJCg`OYHe6>sqHv9X@oO-VjY|>sJ6F&#~bnh3G z()T~NT`2x&CBb6C2F&|8p$Y&Sa61QT}I;Xw#ZVF7NiD@|IFKiR7r#^YXw8YAd z+r5A4<%&i|jV#mmqd`);6oCu$PsH;Sy>l+6(yaLI-kSGhfOJMRJ^-^!RG=Q*qrvk8 z$bA1kUK^~!zDr|p`5u+%5mv8U`GrKqpTR(%PNt(q5N4KbreRg4 zNhDkI^n^-9#($~i;^gOc^CiUzX)0Sv;|UV(*82Xn*Bd6OFySW1xHNmLea_L2#s;bp zJ&uxQt5})Ef`!P0%0PeMI zqo!OI1Gdb#TMr{q`hNY5F*3WkxBVKPn~4F8>>*uaz068j`3mx28_egO@TICRTiOf#L;DaY&`0GT4I zcnhse7r2E(NcOiKrSrRrxeROL<#Il8;w%pVZ$-uvDc#0VUMksZcb%kKX8590Zbu>CU}X2M4}4CYG}`Mb_G|Ozzx7E2l#y0! zN6r-k36l68XakvXpDcxpd9JMhq7qOXt;pA)tsF1X^IecOVgkV0pEk17z22}FAq9Eu z_fUZQ;xfcg2m1jcIWrKOv=^fU&*JKJ`c9|8jLKpqmlbFAl=K)=e*!$kzV7(>%DO zCsq+L#ZO*>^TMl?Zg>~?Pte1h(g?-9P2_i)m$iAKB5Nseb5CE2tKlsZiXOL8yZ)t| zSPn)A40%y#h!C+(uN=lJBKTG6Ecwk3OAg)6HG>k??O-Cqif z*kFJVpK4h2nU7r|Ne=W+{;zf9~(TPBrZR_2GCf3me3a2yz+jA)&Ood3qRV+cZ)Ks&c zpjdBMbe_bh&_8H9r-fz8S#XU$Cq2O~ag3hoWe`3xLOR35hYU%xIxD-R#6J%e>wFynn!Yl zo*nomV;pQ^e)kmdy%%r<14(Bd*+USE^g~agn0GLH{?gJhDJ{xZNK+F}Hx2kXx&Xgj z4*Jrxaul+%)&n_IXHvm5hv7xNx-}heiR^Hwsxxw}`1v`$FX4toe<2bLeA0B zkg7h@=6%YX!-IJDXw348^UIEo9iru+S8Jx+xD(~9s!?0{tQzbPQOgZ}*Y!G|#T$0W z6(Zi^h}j0V``sAzX(3x#2M}@=sXUZ^`s8GPtS#NO47C_u4qf*aWq_+=jZL(*>T>V;8X9bz!49zsEyc1^f zN-0R}*b{#mDL}6nkwo+TW*7-CWVb_&AJ z%QiUtqQxI|Cc?r9Nj?V&-Nr>n`1^VixrZC7srAZMbkjK203O!~;L8Zk%S83_ewImr zYo;j^`g6hG%W}if;(f0HYDe<_4NGruR>rNhqU6Ti;*Ywm)OAXZsMejpJLhK5jXi-2 zzNNt9!&A>F{0qJSY|CG16h&k)gi)e5XCTX)%-GPWXk0j+WhENBhMJQ?KM|W;Ko_R& z=^ak|4BQqvwLLK4LFpT7tvK3@o-lh3e$fUWGEB^TQQ%#D{r8_~!<`IQC(|TU^|*}L zDmp$tdC_(Bd2>#5F+^=cb>YG)^pFA=wCkCtAXL%BdIRH{lQ&?T{c@2fHN(Rmu1EeTf1zV@kX(h>K-{`cV};Ls828Wpeih^Xh?f&qgn{}DN0 zdz)i|wtl^gJLVOrw-u>zexstelEsr(f0?m~h>msO zkn&8p^0ct~b@HB-I2?JAq-DX>x+sUw9hFXd+9zE;n<^NuGy+Hw|Mhf_6g<_wdsdiM z{(^damdR*M|82WuaeC~Xe!XUtWG$`2m`op0{!G)iq${jv;P*MrQtHPkiWM$?cKJkw zr%a&epD@M_^%?b5&x>=&~4dZc-e1y>L!&GEB&s;TBprrr9c%)4LXOhM)}cpehrbeEz2hkpTOXvoW3ipTl2)6K$PrK&e~y ztuMl$Nro01cR(MPAS_fU`{x0Ddgw-j(Y}jA2_&qbjd~r3tYvhSoQ(qVHB^W$g_1(5 ze;r&ViWd?k*A|=ZM*2r|>$6WWc2Fk4Y z`V2+)GKAW`?7|y1kH@Ym;;`54S>8g+AoLf6G7TW3TDbQt93!ja1$5osk(2lPhGB$J zqP?fuDQ77%K&E@^ZlSOg|9u&0*GbCDQ8e*@`0n7rqSBTqJ#O4BB?B6Ml5^1@Y?0~Y zusXU{3cF7mRXnnW__!YZywD)hG*EjdCyB zw+YcR4{c4KXN8ommS>%?Gz`@w^|34ZvW)(@>_}jY01X4$J|>e%0NAeik*gawo%mk8 zc*;%d_Nr2%6EVKQA7*KjNIBtlv(tpTa$vM#wrA1}*^UY|3}Q^>&XRdR`G>KZU$YMJ-K^A7rH_p<(Y?v0P}3dC(PQ8{}n(3ID{rV z+>*AG*fNi9(n?O}eOqwRKa{3FzkhMs1vwT!&w&&O-kP~pb-`)nSn@$ZW=edywRk*I zvEFz&96oT*wPz?-ps5|DRM?+RLD%#kwU+PL{OOD7d{{E87o|^saXg;=;?NHI{F?Sw z_@bppE2K6dwEIP%i|z%;Cqqk4RelL7=SON{z|FX&N>L4FfBYAI)vNcCNj)R>A%Ss7Cos`V?n?4hfI$7c-+s!qAFBP@>sxiVL)d5!xL+jz) zsuv&ZuZ@rURmGViqOJ&M-XCJqULEyRtlm4KL6fy@$))Ii>77yl6nl@S%To)A%DJ)l z?CNH}e*yGm`|AU(a_7sdH^`ToDryE!EmENu4BN=147^020Yi5L$1ZR7Cv~qqCZX@c zXJrKbq4veLt4{AKoTqQB!AU@wHtP*0!z0aK1e@>iC?~)S$GJL@%WAWz`|wMU2dq9R zeKq7tdzIN>mVPF-KJu8cH-`Bc40)0Q&ti z|0nTdeQyjCC)0XD^I?BVwo$}wD9Z#~E5nMte0>~Lb2yGPXH!uJx$ZCD?3kKYkD;#% z-&2{AQ1p>xYMh!+Gle;Ev;KSZ)ca1I-OgZ4cKx0jCWE<^lYvfibtP+fHN@Xaoav2` z$cD)|Gc^78Ksgh(c>esaC|bji&@vp*uq(W)ja`(jdxkkO#X0y*f)`O~iPYYF=1&*en}%bT>E&df^@8px#HmgWZ$M@$&Ba^wXG*<=2r>3#eJG zLy9a=EJxZJcF&fj+Ar_VUA%K(V%i}o#KOT&h1{#rJAaj7HlA(_*%(#nm%ed{uCT9R zn%TU6zGik**?T=Ib=Wbr?_#!!!XIs3C{M)x!4L5p?aA&?LSKA_)IZ2`7?+?5|nJul=#|h3-!u*|i7{hX*jNjf1I>s~s~9xD&FIC3M#-0Rp|Z zF@o3F9oBFe@9RXp=Pod`2QLR=V*II|7h}vlZL%x^=v2*7Zr1d!!Dgy~hm^XD+gqO0M_lS9sAHvwr_D+e*Hw z_0m5IGlj|@MZ{3UjULfMY2`JxF|s=;wXNoI?VU$yFpu&`?J>$){oV7O=Q{vCQXp9YTx5+j`j>_|+m6>g) zonHnUaAeU_gYT`+8T=shlLJO6Sz@RZ)M`n0^{$-2{&)D9jR;&NSS|EF0yye9z?vPd zwAaC;+WZ&bU^Yd z<3x$rbb$XgclaYiTbQG4f)xxq&)_cR2?izw?t{C=k>5rr8vj}SgYD2nB~uX|)7os7 z59>8)D4GxuwS=~uMG4<3wKIE^ZxFEB=R<4jxmdXAcTRTZ$eU+##}0}gYAT$4iDkiV z!>*oug)W<|TgMNbubIz+f;L%pW(d;v6V_aLUn2!?_9v7zmWRq>1SjRoYg}d3&NSm} z%yC{j)K;1EBkO6%o8Uoxi1zKx{esL!w}9}b$eVGJSO20*J&(%dpdr--nZ+uw-J_Ub zq$fYR#J9Tx8g$CxL+ocyh{I6b)|WB0+pe*lvdKVN0)MCxcg44nl*m^g7pBbJ}) z0`bF*^{5gjTQ4n|?bQ4~3DQKzio2a!T$l&r*@)?I~IVS63UN-pF77Jx22L?Z8VjXlpU}v)ku8iJniSxz z67%3w?289x!8vJ6TjNzvJf<>aGJSe!uNM?qiwx;6(#79l#`oIvobY={E7Onef$tIT zDj6M2Z>-Ek*4!lb4uQY~_hh^`axccw*iV-W1ezQ&$@`0-2HV87si1?si~hDDyPSa| zPB7WrqM}q})jLKrBU6~&hNQ-z}2Ljm7(*h^e&yOftL;$dt*D2l83OALP47a!p z6Ag37tIGgb_>KyvKoQm%8Px$jAwvil|F=l@h!ZeicRmIHQx9!|< zw0Q9cv%};-!ix?rM!Y{o_AUPpf!N0m^!(pumSeILz~yL@>4z5*>K^q*8T@Uvq11FS z8E8|y{Nqij5=Uc^&#FL##%(W3W@e^Egnfc=<7 zavtj#WTl^^SAR;Z$7sv%$a!xK%`)4oKK)@>-#_h5h7-z`Ea^g)Xb7J=XRX2cUgedD z|DJmK{c4u6JM%HyB13l@`OYUmN{YC`$VnhE*f0nQ}I^uFM+1tox|8scS zA{lWw`6At62Kb?Z*8sz=_?HH?5gewDa`7ED7zAwpxv(Ci4vSm3HyfC_ z-Qd5-jEddfMNU~ZrqLS8s8VGKUHr(wHx7_ZY2f*3*>RSC*?l^!77Jf^Ti6a)$nw?$ z6`D&sI@-A*4EG*SRM;mYSByd>U7{+jwh<55Ty#!?YmyFiIp|I=rU2i z^sN6DxS-s!fi87K^DV?7n}JJGh9U>X0x_0~ZFEe|)l4kP?bXq8Dxj+gMU6H}r_-U? zFIbm-Kc^tqISz_i5!ysY)3LZ(tfdqwNhdo|DV60kzhwa(B*zL5(EG_L37Ht!0 zi52okA}oU6bjEfxU~X8`iMUGJ+(^~ZiN0lUY@WFQAhMmi-3P3~IGnJEwTmU#=><kx&u4)JKA4R^|4uG}>T(gD$67-o{Z>=4 z;y9}<;>-JHn~EnSBp#HgGqL>Iyb@6Yj6k~tAmjVcy7(LK_U1a#PiGRkCS!R&tK4P+ ztHo;e=fWzpDy!Cc5Zj@#6=DCxm8e&bhX&61yM51INe*{i_gmT^_uASw-1`oMifQkf za9L*n{4E_1GhmsF?!C{|f|0SjShlY$cU?=2k3o{}S5;SBT$g$BAj2{_@UFUaw4zRO zm~&z|q%-gowGBymvCutY2)PG5v6_F+`B#A{jOxFYJF^a-TjmUNMReX+^4MRT^sr%{ z+}TstF)gKVQXXcvo)bzCmm}YsmA9M_>`9QX=~#*s7ia_PbJ^8W+q{9$F=)8emQV4I$VQ88Ef*w^D@4a2r)}_x-Br&(6x$CX-Gu@}W!V zQo5fQfRtK73;xEQ4V%X=L%P{~#r&LC&D&d7M>OJ4BZSFKcAo8A=J zSay2$!IcZ8;D!w?ZOeOKdmHnsz(r?CBu8-alBrLL9AI&*GDF?CQ3En&-{*8lFUXDO zkMa>`MIYL%6$g$!k3(I$I0Q9zmhU~j$mBXoLDAsuwFd&OnXrr-iSZ6&v!_L7LU3}dYIM{9iVX8=zBJ*S%opuoDW;zWQ zH_X2peYpNq%I=VSW2-c6^_Jf``tjtYYaZ{+9f2QeStECAxrq-$T22Ln67(0Be*R*{ zy!auHm{$EUa}s|+wk~ji+T@V%W3W4Ilp`pz-;DU0bgsT>bj?c1{-6d^maw_F{ur+f zT~F_NNdqGa>TwqB9)4dk+HCItT<_<0m(coU0;Z#jFSU~DxhEQhZl?!Y1zSgb8zwJb zCKswgMc#$Zpzk+~g3^_g9#!#e%Ezbeg^2Gj4Dt5^G&K8scbNTMS$>lpnx(mmVPjh7 zPwS;n%bwh)KIC<hF`!h1!9_8$gz}{r;Rimgb`3ueTa`G>qcJ$|oCdsdP!%zRg z)y`Em@suoeM6R^B*D`^3K15|wPV6gw{-K_MaqV~)>{Aps{V6D-6evGR z-vIhaxs}b;h)?GiKe=1TXs4A23ssftYM>`3-W)H0uG8hl*Ddha|7_Gc@PCNJ zUx!DF=Wq*tAqI;G_0TVy1 zU*8;0dxfWY`Q_nI++o0xLvKm3byxABVMfA5SsF*^$fSXWX6s+k@n?6#xT`*?-)~yv z?0Qlh6R#++v)p%sC+1;l?Sk^>`T)oDC4XQ}-5!SgMGEA-62I=Qh5kz7T4qasQK0^H z@#A?!>hFvGSr7*<;}v)1#uxE>jx|Lhw|3}ah$lL$xtH#_`CrG#X2Y8{x#xXzng2;S zDX|^eyeh%0Q?GgNQBZq|Zo_GakKYCz7Nc~#MoR8KmZ7~rBkg}&lCClA@>tT=Uil=`b^BdRtmDM*y=ooo>LhKx zi5j-YXOb;vI=}`H?_Bxi#-wgaMnLW5w+|b!@)aY5FO(sPM;4#Y#HrQrPJNSOBR=>> zlO>*h5cDUXMJ8GO6pSdh#3gNdZKls%S(F7-ps3qx>J_dVYKoKvr)7QE4-!JG6H)4YFIR zJ6r66Xz^5YZenSP`*A>FYP0y=KiY)9^x(BjF}x#PAaCmC+`KH79Wfnk^G6B|!>2g# zQ#s%DWp093Co~~v(D6Y^%#s3Q1n9Qln#Af?)vdW-y8|eNcAHL|Wtw*8y^VM7hQ?sE zBPFJBF|2Y;DaM-I(Cs2~x!S)XeKBL(?ojL5htTpp^pTwp`u7+E`tZbF=w^UT+O z`L=5BzJphgM%kP1#S5p|-vPO{#94Rty+L*_X_b`fVTO&eg746jyE4I`m*|_4T3f z0TU1ZApe29c+L&oTdJwvR1--RkItBTeq?dKNYUiob!3r2Wvnf;NyI(%U!2Oj3G9`E8uLzig~X$@Jg3seDK$h#Cbsy@SQ|O1hp1Y-K^(p+poA8H zb)NNaxX4DC2oRsk`l(|(12~Pm-WVv{NPxq{Nl&fA#E)n~`8T3=8>-r&^vBAC z4qt7)#0KM(I8OJ7<2%t7+Z*Z7^5)+)u~oaYdg;hl9=TZV3HcnR;6tjJUE)zuLQb$F zji;%hK(OGgUTKgW2kjnkDy)v)nWtQ}Z)mMY_K&P8&(lr+J+&xTx;iO8&YIzt{)NE#%MYu3yVaU|gq5f8&H3dKOX&p7WmFadGgb zh~*?92l1K|^Vx!RF+|q8Sr^8%_^%Et`^7xA2ou?J8Z6)4rpx8bS6kOOvU0WuT^G`A zzO+9^`VFU3^WC|`nnOgSt^$JEr8jugefWncCyslr$OOIbl+8a=mkmZD=jxC_) zVxe0^9LizxH3pLXM>%AZmgym`tQO4vW&X`VXLCtGe>$FMa zUVI%}^@(dfR0F%)2XVpXkrQUXQ8mL~7DInB+dXJ$m&$508My z3jOow&udXQoqqDvI|yy*LG2$=+rcAx@Ik!$wLb@tA+;h6_?<7D2?n@Eu zn|Wn;6|4Z0mg8a}EqzmsUU~6r;Y)&sRx@Jkos&q7|7eMv7+bm}LC@zjUH3F2&aRw% zptRGj>I6+UM4(izoa{;@Z(ze>26dsjXBk_pe5|L^xWZ%yO|v>Lr55+CtSc4pKX^c( zLa0vx9W{_XLyktD>+8xagL48+XcTugyHqo#G?h}=}=-szhOs+#nFM|XHd z_UXsBkqP$$cL(I5tuOet^1-rAbTK0jX^14HDB-b7Q2Ahmel|QC6#$;{B+&AS#9pz* zj5|>Z8GcQA2iL8j-4iGQ5)B|2f${s%kCQ?#-rCbX!zrI;X$j>pRJK*}#Xn=s$f7+-1?P#Z`-gZ0u@0U;Q9?E_EzeA)pFxK zEbi1YZb?CP)O~9f4?^*KwsthMD`Zq^79kt$p+(m`N3Lo(kBSP^$@4&$bjke;l85K| z^Q4%%MI5_2jRF3>o0jpUp@R55B*cz!`=Acu$hSo~T%7Z$F{1x>-bFz?`KnkLFXvBz zn5ow_d&|!PJo2hM1)k!^z*~LCfb-4cIFGmdt2RT|)9fp6t@i;E?3EyaTf5I}KCPYh z$>;@=1(rW@=pkQy5IPc@^lkOK$20e7dcxxgF@Z_zp4Vi+C1$av=n*FYmp}fA(nUT6 zaafN@>w0$AiMsosjPru$48Leu1TOv(*b>nW&-N&m_AaBueEZsu?m4@R0V*ngr(hFZ z2~&dvt-z2^mk+9|B#zRKo;V^#TGle2hxgxrV`|T9!TB! zCbUSQ^;tA#dZgvdkyop=53a{iD5JZn{ley)20zaI}zgA$jQozY8NgASS> z^+2v-qvY2Jg@Q+t#fZ<_&e#Q5!N-qayQgB^zWwN00Y78NZgkFx1Kvk87`0-@ns}Dd zLB3;R&AuUyC~z_Cql=Q;ln$=Yu^xeq6tI<76y5Z@&%o+{Q5e)2Kl5pi^j=wubn8c) zq$?Ow7=z2>Ld7*eURf9Rev`~KqEF_PNM#l%Nwpk-d<$GoOLwdw5M$pKR+W+?wjFgs`N{XTph>)zvc+Q!}HR z7-3#FR8Cphve!~dpN-!DcOmqvxX;6FQ_PDv9O{%$=;i%L&GPb(kBju_-cy*N~90Cm11pW0cgT zaPS~KqNbM9|8ZiRTm9h#+wfCw>cXGgGHa@9_+;A@M(@=JYm`@qSYYLY_6T-v-oh8L z0WzA?DSHB;VMpunS=E2^X8uxFcC>2&tz_9!nV7R~^5||ASq~Nn`fJbg(FMU|o7HAY zZuxEPZ@hI{U&TT&w>!5p#k-4W#9w~@UtW8D3Og(7J(|S$YNseirmS08CyQV&Q}VkpST)7y~Hv2n04zV^NB?%zaRgKz!|~VD3bn(W9zC- zcG$5E&pYa2Vw^QZ*2gXKN`JCf4Wm-(#%R3!zhHGPS)u| z%?9K&ZVm8{UHzCFngfzPzps*N!|?)lYiIP(KWX(*Aw1=UMYS15yP@CfqV|gVpd*s* zh!+=ax~}`cVev;G&8;3EX7s3sWi-|46WRCajJzjA=onRH0Xv z_{}SBsIv4Isrd946V!yOj6VOrf0L%yifgnvKUp`SYs66>VWr;2wLW3I_4-|7-I?9C z5V`FL-Y&{!^4&XCh`cqsoRmP>L6vrpJ5LLb$-8Y?vY~hJ&L>n-Evf^X3RQbeB}6gS z9qIu{9{9vkG|MKb9e(qvFG_Y7R-N5Ht4`#N)HRn^^@B-Be!e5G9s+o%tPu{J>E4-YPYA^GJpp$;#cMmDT|oP1Tx8S zd`)9ag7G1BPIsStg;U=&6%IdUxQxXvC1Dk>~r8WTQrlffRt=nH|-HWd*aHWAG z9{7LnJjG63&R@s~tSqTMl`I2#Z7}@Rz%;21$&YY2@Kr8Sx;&t_8*s*{QzN{$k}C@i zrUFg-Y~QzSDA;Bt3m*(5gxV|He!)dI;pn_Yb!p%i-DzLxFOSZ_z*UsQ=ay7wGp;bX zzUz?DEAn>yWK@6c zYiG^noi@EG@Ahg}G;H*Jb7+6s$CMt?czPpm`WNEf>wV-Ga&>`Dch{4Op}CQ+&cu6! z;=i)Hwf%a`KHqz}8$YC&j>3s0#CZ-y!zX*-Z5bZt*X8UJgPOw{n|_>Ro2)@hs5rqn`7$AwBxI*;16!GNyr?d~K#L)!$@L z86~+zOF%!_O4|zvd%3T?GUWM;!@1E1a1jGtvGkIT&^T84y;ZlYLX+qE6m0Uhl82yxfz`-ze z?a1*hkaLrLyVZSBPku;!qEF$2VeW<=XD|1L#^lErbwrh^VGlBklsXbYJ<|ilKlG?G zRr6)`v9MU5O<4N9qf?a1xHzqq5}vloEBQs&n`DH=#mjb`!%k4l z`LB3cf83O$+M5EZJ6P6&aK>wp<@c|E0r7sY z_uXsG6i+S1weKLs7ZKP$mZ`MZysOC=U&I+Fdw{)Ftx!83b7De${=ydz* zlH6@NCbH_PAg|1}`_#!I>1UH3xUmuJ651UqFza=^8O&47E3C7)I(;iBYDETdb^N@D$J?EckiF2=dzPggO~b<-hOhMRQVhnLm(>BMO}ccW1E?c>C6^7U=;e$7vRi{OQZoV60L-Di9{4GDbnv2WyL z9%%7!-{gyOI({wgYqXJ>-qgd#7h*41$Q|``xATIcF6+^}65SNtRii_Bw{*~N@u?}D z9W8GiFPpo5^r#@^IThNu0A{ZlU4K&-Jd;=KLT4JPCEe4eH%`!N5u+M)#q5HYmWSY3 z`^RXhLBXKW++I7t*e_?5918I*)|gx(>+i9PkxDs;IZMl(F$P~ zj1-vEBI?1MNx2ecC!Mldtx(|{ zQFv$A%i}ottW6NG`Y>OP+i)Nh+P9nmd}FUhm~{Bo;UW6Z+Sj7nsHE=9$@P5n&X(Cj;?|1t42YOt2A#B~wQE{EKK_ zj?e+Q-C2ExUxw_N-&ZKUFGUds#Zkoqhc_<1)5Rkoo~ zxS<47x42SeNCI6i*+qyE_FHaSs>;;qv`di%R`}Nup2FWl@iJKn&uPOiPM6U|c*GV| z{qK=RfUKBRn{Eyi`84|qS|mjqPb_wovcP_eVYkpGj+U<1SGO={MTU?QpsHFh?bEC` z-tMjHSRrw{BJ-tj5C`iz_g!xBKk_QSEU041dGM<}|E@$lhrmAH5Ya?tq8JEU$NTV+ zpXmnIKd)h3gp=YX=_APiL&ovLk9dIr3a(GvTIdt__zg^b;D%RxFU>7mH5YsCbh4Y& z&Px=FW`s>S#~TFcMYUXZPA|!xm#j9(#q_;Co(A=`KRu*c8C~a^BU8#=6b5gkz<7-6 z2|kIRHUgFVA-NOJKI}n7I0B(QMv69FRQ!L~hZ59;?hW9|^8-d8aOo?_k*%?v7 zG3`YmJl$xCBxgKMBd-08tT6ghQT>u+Is^6p1Q+H8;U9#0L^?$M4$p-VxE=!D;vai{ zTV=k;_LQ3-{Sf2ta)>_^-?pB#dDWI@C^_=v^)zam;qu`Lm-9B*6RHIY9Njb*c@tUvD|gRUJl3PT4vK&) z4j>x_Xwv3bKvg8IVQp}_tD&inK7fQbpMCS})LqrHD7y{4G8Q>Da@f@d7FHlm%#Tq1 zMtAO3U`F1Kcsc$KuI48X?90sodPq?jy8%jycC$_79T(~rqPimRLZ>!mzi$#a3g!wZ3Ny4b6Blc5+tsndDLepJ7P!jLTsYm1T_DA=UGC+^ zyF3NvkI5~bsJx5L7@+@lU}Qx&JFFP8&F6o#+n#SjT7D`TMgaE4fE967OqWZm9WrlN zK1|Y+0(=FH$e4ANcY;= zKXSfS>S|WA`kZUuFAH-va|1&h3{Qhz{-X4#BC-jPN~HQc3|}FiwxUK6I5b%(7he6jyV#^lI_8>^lA5Q3K#YpY@(-L+DAXV>;SP8|B3r8#VpL2DJA(LM-A>n>oE!8FC!EN zYNkJ$2RaY`m6FV>XYmL{obnd}hY$Jsoo5)C(dsENRF@HcmLQwe;_{<-SVT|4)4?Zb z49Zx9>BR~R@yAOS#9hYbFNWs!vw6nokS?+P{KN>+8OsXclZdqK zZvhPbeAFeG|J1z>etY47elK%R*bd^N3Y3ctoF2_$$oyCx#~U3q+WBsuZhKnnNTb90 zLpR{h#Elc{y7WpHhXm%6cW`ze9ACA1hfK{bPsi8a8LmO1Z=L?vd#`Vt+s2?=QR^NI z*et3@kiTY8pm`HUsfGT)l{x2Hj%)v{B-@ay+!svF%&A+&>D8D?Gis<>6>_+o+Ct(>2HZM7~QB!jjlp z0{im(KDO#~b8Yi20;tLh3u>W{Xri_&8#=Zdt^F#d;64_ytR0|6>^-KX16Jmc6Yg1A zKsz_dVYI+zV3uk>3+{XD5i&27dOgO`YJ1T)k4QEJ@Lh#2+-yqx%b#w3( zA@qCN|E6Cz?K2nN{=%OCjYDiJ_KVD?A_D%75nKXUGwn$F#i4VLyfPoSVB+Rl>yVK{ zWpOa4EXImgz-q=t54YW=WQuirak<<|3eU#IdH(v1kseZ&f}t2J`s`8H%ixy(t<>Pb z3SN=uzIpclgH0ewZWGz?v+Hg{vxVzbFx*^2C#WYh=Eka9mzhz+7JtgpGDS3XB7r*MI&pqqEPs`0?IYo-f=yCT=7Mk7K8*)gi#QA!B^)MB#_HvCL7{ohW zo9k!xs@gK!JNG0P$MRM;8$cWkX+UW31ks+=mlC2K)X9|18gSoAA(I6>ewJj2o}YkJ zJ@CmZhXe9_^q}!Zq3n&Jf@^4FjJ}63t-jxeEHHh0v=&G(AJt2hIOZBfo^lcr-DdY) z$(G!?69LwOS>l{sM9VfeuEJdwpszuHM86W9H;-6v>N1gNC1j@_r7I2;o~;U|*@o~ib!*E#*+Lhiq9{coF8ygw4gGvoY8fOQ$N4$|FWiw#T>`Qe zx;WhWsDK~+1GfJX(oCb9kfZ(79&6)bkucWH4nxT)nuOYCI_aPzx^uZO^GYdLzQP)D zYR>dbJD2%tmU5!h9h3TThF!r{Ank@2?~V;KLFBur)>7sl7!hD~6u~hIG=CJB%a8aJVW^jAOjYW&cLM}5j zZQoqJRVb`k0J~8-BDo)tZ-RrW>4orl%~z%Fry!QZtAFSJG4Xd*SZ9ALspF zBz<`}RPXTnV3rNWJ@JkCR;;{CB`~4l924lzKo?T+4p_OzK?yF z!Pxh)494ufKEK~T=eo`x=bz_1=X&mQzwQNgk`@L%Ke;?=+;30d_fna`see(g*_yPj zE@b^Ihm6{LpQ-4zf1kAx-ctX2whbE0Sn1~CZdYPdD%xwC_wQprl3(3W{qc=1ygMQ& zmx6;~D?84KG~Vi@_8R?BQf%Qm=Vn>I$yGne_#7cq1F<^?H$M*;J$4i@*iD{G^}4cV zWTvV1IOSnyjlXib%KjKJ%wHK&RNuZ|tVSI@_RbRG_I+bsM zn1cZ=p!&y|Sdloz(mQ_o|}xXGT*`#+Vj3q>_~6aj9k7<_JcyUV^DBe7+Uo9@Yx#9@qX`ay`Qv=XkPpa zvz%KKY2OvUKZIGc<;Jmp1eit%YPxG07r*<&oLL2{O!%5$_}0BI7WcR7G(1n5zk&<3 zxiF7cFOqfNQJv{|?-dyoDRVxi z%I$L}rwNT2uW*fYu5s3}@55r8noEDiGyPW8Gy7{Qk$3ig6RRDaeDtNhajPAh zP2}*g%Buu+8pb*2xG)#6N`)6^j0v|> z+j|!)08f^SbBGs!We2o??u!1iV*Cmu*|RD>I_j_==3|H zxtpf;wLKp=T0%B@XH9vu_KF3`R}BulNDWq0!Ja;j@&cQW#^@c7cUNs8?d+oi;GnEQ zwDQU8pD#?oW`M%|R2O>k`hOiGugD&-NP7Ssymx%)CL0d|QzE+0&s(9mcLLz444OfS z{oBPss4^~y-)28J^QfsiQaSGD=`;B7kUODrsm=!;^X?-{>EsA@f8C|~qg4r~h3w>% z74~Ou+mpTT-{p;mjcN6IC;#DER4OzS+bsh&y|>zQ#R&I$#6lW_erXH4B|q#*na~ve z(EW8>z((C$JI7vN(@=paNVMG8m`tX)hpx(LFChV28Yo*od)$Ojgr-`KVBKD%(C578 z*88IH@BMq6jEm|a8UAr>@mBMr=|=%?JY?28sRUm=#TveC`^q51AuKtwgPd+usBG zc&NUd4o^56?El=uAy-f8lfWr0re?I!bp@ZAO1txjXVwOVi+K6LT-P?yIfCyTdgVRP zT(;ukK>;p@>o>&EoWH>1d$GC>zvq4REPH|GnhueRIL}QZbo5G`KErhFF=VmUMP!RM z`<5ZTcnRAfTZ;-Y#pmp1d0*&&&kc^!7by7bOP`!HVU`VvPy_TT_#6J*L#H!dV-3IT zJ#Q58pE-@iON=ZtTD3SCL`j#2p-kMq0XKlAH@;8+O77v zm~epe_f$7jQ!_pvRIH6a!)_Kq0s!e8@ZUw-$?eLKK8%F*jVHj$4E9L0tl9F#1fTCH z#SYoWzHX=mJo}Td7w?c1DX%85SP5azhzU}kmQe~&PpzCApvWjjDcI>RmoMa|S_Hy6 zhupa>0N)w(_nk8NB;Dz6bnaEUnEYcqPorl;q~!`=*Of}!f1Zm%zp976mx0D0T7pkZ zhAGm-#xX8UY5G;p)~V=Q{jvrnO;@S6T}$`hXr0<D2xUaTegOPw`}vQMdxAKxq) zEq5ZtQj2RVwKs)`-(2djXNHu6V6=FFcbO%i1=ZsSBfSKn=tkzP56U)X9Ig?EQ*ZmB zN?p_hBi8ZSzeTQuaQ~54EvEuxv#Ploi$?Hv1amqWkA=P<71s zkgPc5ge59SZW8H751ZdSqT2pn-Y_H$(~{Ke!yZ1|BX4feQsI2X;~y-zkMmvSg-W?6 z5iaV{2v?hh>9;R0iJ?PyW%x|LGEsgZW9ESy(v{TX*&UwB#RwGq8Q#1dP zZ*8l`*`i^lL-aG|mO-EZZXmx1t~ej0+nu<-;5`N=G$&Q7fp*GodYu&4UwliT^WXV7 zHxRYZ))Z>EX{`Q%`Lb#gkE%Rgk0@E^SbPa8^P72pX>B0(M4SzP zJ$_!NVF((^2~jStY3?D2MWeGUhBE}NbE(#2FT`gD6*L3aK6*b^mlyWFF?&6xwe{)g z9f!9-l@=~+!{?g%-9J5x6MMtLi7$*7OYTu-vx}a1ec5o#2;@xSb__h*U3#zYSzwx^ z(v2{LLrYKO?Ti@-5nxb$-69yVniw}&Kf*gzezwG5NT(rP&c!kOGxzg#ji>>Mnx#MX ze~}!WsU!!1i*J_)F7lDTH2qi$6ojOqxP^mnZEY-1>j}F{Euc>NzGw4XTRI94A%K5b zMW68ru^P7ROq)J}jYh1glP}n}=T3*VIH`w~D8VAa(A{dBrvj3!ry_OgDXNhSV}N6I z>vSg(-p-!-5;i?&u?cxjyu;l44gQ+p*nL(CX_4yN!583>rqy^Rl?#8`mO*eX@9PZO z+MHK|1R|81w}SA%AXFM=BxJf$7?b9B_`GJq5zj^b@QXcpjDPz*^Vr(<5IXJ&LdcKL zDnq)%@)w(b?-cXzFWK;b6Uc()&Fn8eOy5`+gnP38o*6kMY@#gUqVFB6_5{vXhiAKA zrrh~xtT85lQXMMp(R#&cOXmuLD&A=FGv+xv5ZElAq&{8wKAp)|S9wo>78>zoDPb)d zkT|+=+UL8dgZyp=l+r?Rv7L%8Vm)f5HeCw`1&@L3_OnkZH2in7NMPfOzHdcg8K&O3 zm2;UdXJ4$-J&A;p$0h24ZXpC`tQbAm%1!N=qW zGpG7zEP$wMqNkRznF=Il;ys6LmyK2x8Bi@^zWfzz2YW5unA*pemSu(M0y! z*YotD(7^K_Dq8vWha=hNx}g{E2!3=hra!pM3lR>qtmLR~3R>BPDmrA<-~GWZXnBRd zwbAnK717-8gpkRt*>V8l-L9(AhUU0?IHY*fTe~n@S{&#F9T5KNv(Yu~+$vV?_sEY|pY0a7}EcZ*q-tB#~nz+_$M--HP!N@XOCpFRL)J?+f=n^v#U| zdxJvWO*r|>G0aUo^rnCQ9vpLNQBcEod2xwJmVgn#&qSz4!8qp55fIVLdHu0p;oy}K z>G-EXRt_(c*`~hQLd$aB)?0;b-`@yt(Q6gTsw@o7*$NCYSlKu(E=tgfMWpGy+C1!- zMoW@P1y?PW{$BJE*1!L010(rgTsk2#;iSH5RD$lUl*nIC5=I1{7}WB=^U$8MPXD6j zbh+yMdKAyYI`KX|O9R36>|T zmMkX=d;cA3l{s7cRGNR|jz-kQ80qrZlyYV;Gd-Mp-?tX7Q;qXvoI+_d3a$xv`_-iv zXt#^d6wHd>Hu(|?V7a7C!%J-oD(`+fpqw8eKCjFSmCS?)nS|s|1%Q3^Im|zBfyDRo zuQRy&-25?Z)GhN2mEPcG< z%Y_(=KWO$C{6*NDqm4zVOwc!-_^8ffyw3%CkvtF0iiO>`*C0jiA!bs4Bbbq?`YNTY ziEJO1C{GtC-T!9}DXn%y26^-bXobX~|I$NNN(mLp>Cnic*DDcBn*(ixEm zR*5U3+U|sQ$T>84BgZaoe^OI<>%dl#u66?9ZMd*52_dolG|gp?h(}Ld%7lc4>{Z;) z)g5v!D|dOm;U&6G$Aq`35lvmXJt79Lw_op3eDmut)=!V|grON77Mxm;|3(=wMC5za zm{5^`xoU3YwUS;x4lKGyq^tiDw(OA51+788bOL}`FHy-xGir7|tf88V5rU6Xd%~l4 zk|ge5kNX^r9lv!y{!B#mEv2DCH_HvR{-JrPOm$;dos6P^Vc#veg${ve$A~QbJ0v9< z26(9DaB}T)zX{fv?%g9%b+9EuLYVp6uUZeL?`jW%=f=1K;2%+~vZvR)kqn>!4DxkS z*K+Kt(Ys;|7RYKxN_y+5fa79U>c5pJq(7GqL3tZu??wU>R}d4c3L+recrRu9U9M3> zU;0j*(=S_4*QZerSqtXmH~3+f)wv1d9sMH`fn_sER`}Gd(*e-_Ou9n2wGT$YJc>rB4+QL#1bm_Ic@JAQwpPF zv{%@1Q(XDI_K4U{5#qw~(Gc|(#7oTtDOZ`Pa%KbwaWU(;Y^TN1wkels#eTK9z;d56 z7+dlWM(ytWIh`3EEbMCjN|MhSivov-ZbcuNbU?2)^&Ono| z1mfnzOi;hP%s7(V^_BqbxWjR|z47|uLbk&uvNdZ(?x3M^k&_RsK?p3svM)je@R%`K zvH-_kQEXJ~vc$f*OBE3EE{-M=%|Yhr=0iD;0OdOCkZ0E*_ts}vGS8BI(#T`kby_7z z5>H+{^H1Jnr5|;}IBS5zI4JyOI^~G~%*euS-Wc@_imrPf2%so^jI&QzGIF2)HK^Xg zSq5sf?tG!kWZmVA8D$F0rIPw)D{wL7l=a$z%yq+M$s)Le>8=x9=XsKu>L%UXmgccz zuvxwv8wSm_Gc!>I!@c+#dFOOfEPC#HAHG~J{HLd$mh3+AOtNgi0ZfhuCoLm_Tg>Q*&(cdhF!97IXV!0MQ3f5UmTo`vdB^b82mPZ2P)L2P^wX zKDS3dwPsX5ckulv->it55_y*7O9d9z`JMX59+o;yfpW0j<+uPLNm7qfC12_D2_~V# zGF}Y^Tu6oTTkI-?Pi|p+^oI{bO=upHn`v~Bar!@9iGD?!_qxoZRe~m23W$N=mK+ua z{1qKtc_AvLs)D4a^5)Chy-O`ctD|-Bqe|5>z8;exGNBTCI;eJ#R4e^zS>itW$Pu?< zD0(=Oo+BBs|7Td*0ylW%W4kMl<~~R|XQKznYq;JBcbw>JdDQSCd$T*%cefHTT#1Nd zC!_hSC$2q?_}z56gGN55a?43xsv}VTYSYKBIywT;Q}L15_8=W#(hQMXr$oS)LP}CN z{AcvPF9P{mNg!ZKzqp3d76b;e-g7#X{rhBEvsZSu#KrN)NYI3~N(SE8*-8BG_+zj7 zs6PJ6npV!K@+eSJDmW%Mg8TZy_1}Yor$?(?S)&UFpKEP>HY>9rgnVS4^6($d95W6B z9=*d)nhx(VcZ$+39Kt%wqAK=p;_eVBeC#Ez%X>eh z!wxGYSA89m=U1^*J22-rB78jTy?He?Iz8xzQ#$Dcf%6NIelhaM663~NVXap2c4I>B zLc~W}{HhPKg$31w2W7#2>sAIGX?tjRk*f5NF?bCRj+-R7(l-} zw@hD7JwX2(pOpPWm-4AZ*GZX=TetZ$UX^rwB!>c%g5(M1JE6c#acua+8FUVQB#>z| z+z-)%e7M=PtJlF}E`L3fh-R>|k0OBUkfwu<821hFOKSX`4!{W;IUaK>czW1Am+Jo0 z*C}X?_wtEB^&4Gr{t=a`A@i#PNtiJmCP`(#3-jR<>qb)9uoUCA1u}2__=vleA*N#a zm|b@#C)7v2fqkk^nlxE%SPse9;EL#vl6GD_>BTRz|AB+%W9w-}+)|IPv2kdZ4*+_` z+Y4TnS~K1bmy~P>$V~}Cp2s>mTroMnMZzk?>Oy!T&;Z(x1!(40c6ZU9 z53|mCCW#|F8lo!TNPG?)#)JeRX60zDTYOS(x(ptqLTTf5vSy+4yLo?+pty53n>6TJ zPJ_qc7e0633y|>Vfu7nX{JP}Jb}D`4{qK!QAG_u2pP(>P$n<_e<4>azblFu!a^347 zzWw-Om;H;l00x`d^=?9@^r&tFX1P4*XTFqCX`fz=sT-;7IK#pxhv1iYs(1+4PntV_ zB!T2oM|uK>jMJz`r2A8xn(ykrDfR5C(TJ(hcc5^$v9dN(r z=eXc4k*$kYrpityhSY6xbDE>joNMJ=hfQ02WthVNj*&Q?IbUYqz@Ar-@)tAm=g)Ku zWh1fJ*4K(q@0p@X0x%sLzi^U;1}IR(;#QfSMY4z!u-G}i)AhS^m9d7_-uXSlM!Bg1 zMf(BPB!>ZX(E}UN11^U5AFt2FIoe~Qp{-rnVGSdb>|EmVhlMNIQw5p7{0fmj=tf=+ z<~K-tAj}UZ@auumTK9DabBg7<7VoFJP_$8EDS6XBaPkhsOt@R)21&`{-?Nl?>T~9T zOOn04uk-|CC@GbPy~|0~=^=w6)O;Z3y?6a0uM-;b>9uYMFW zIsa^KQo&JnnIHcAS80-6s!O|56)1`K^99dBVp2U6bU4~-435l8@i3_p@-Qhi=}43- zK((NC)g_I@-+x^8Aj+&SWN!j>XaFq?SO#_X>7gd?(m>}NRQF-*Vow@4d;0ur+6N8X zR=YGt=%+Mf$An2OU>Vr|(>xS4jQ6j$HnsD#CMhEe$-cZJ0>?p5_fFKtqx)G;R=4IP z3trzvKO%kG3dN?L%N%Z zFCeZBj2pkP=lO$nLkG`sd%t`3&YIoO-1)wKyt3RtrClMs9BT6{jG!PIk+03OPv&bH zu_|1}bR0wF0k}f;f?eEXS4ukGC>}(0ijx?cz7K)Ph|YhR7*BGZ&@X~bCh;erP!E5% zF%0i%SVDW<$ZHZCxO@1heUS`Ld!N!**2sipjSMh(ZcDY+0* z<+7DSI}bA&?S7x{q3>wMi}u5)n?xn|I@}!}X$g5@CHh;O`2oUZN0D~fjJVYh)M~b# zc}QsV)JGd{cj3}HVAuW(*|%or?ZKN44FKuoGgJxZ!>F{@8Wn@!j_D;i|9qZnQ5IP;>%Rxx2Y=y>Av9Z7A#8?BOUv+b*_y&>l_e?L>oUZua;iV{lafg<-ZXoKgwwVGt zV|D9=%CJ4(>$(fe`qQe~^@_bl#PemB8q;Sk~DR#&G;Q+sfm;hL$u@cu$VwPm^Iu;G^?L=$om?; zvN8N)3EQrdAhEVH#1JLBv*e%5kM?ohz~QnRdjD{){6*^s2|XyAP4zC6#kZV}loxlf z-XC}k7#&qu=t0Y*P+dvBUKL8|i=0H|lHGpMjBRnAVWcHIl7k`XTZ`XC#ZW6~x zg)7T}aXrZ$X-L?rpToO9^-9x3{dQ0wbz?ofeEqzKeZHtfho%FEjUAV z+%Scxvmz~==I&R6s1LoaX7U7-p6R6t9B79@JGo*?{)Ol&sBEsb$`JLpA8mnYj*w%El|@ykKdm|_i2IX_{*n~d zd|93_B6t3&9pT2}j@RUpy1EKnYokE&#~wq3?ZAqOOEsDuv1n$!sVGJxtlOo3nXJAR zx&jH8i#9^B@DW|ioQ=xS1-Yw-Q6(GVH#@I^MwC7`E>y|8-)*5Q)E3cIIn46GWJ{;b ziYar6y59KRYHrH-h)W)6%tgnuQc1S4;^m-ASmc2gfb&Tx^vhFZ?#3;+xp>l4zmP&G z?CBAF;wgU5Yc3lKLZejdvFU~AiQ6a*V(`4S4u|`wR)LoPZM0N-4#^iW8b8;;->yjC z;CODniMARx0%E*&I98yfKwvG{YW`{nFo3#3}CJq+A?M-QyH%J)-=#R z4CGx|`T(_@pBzp7UM0{m=@7KK-88RdIV=(Sb4P6iDu8GNLY^)N1RcCfmOckK+I0Mf z1D&HwIhbdrO!(YSby_)6L=EP($QcLo^=^}nq>8w#6o!y_XLcOItTFo=6#(M)mEHav z%CmCdQpK5l#Gr0mLFm*N{hJ^1o? zpF?>iNK%aZ5MOD-MP{6Tv*NR*7ewZmLil;xVvmmjJ586hxK>U0s2=pDzj{k066J5Z z-&4iGe6B(MSL43lumRQW?Q7&om-R$dONZjo5!bvZcDpl?MM4^+Y!lw2||PQAJW zsVDq;M8UnWwMezcDK6@&imCkFEsu0rgXNq)r(N;uI8Tc~rRUW1*@qHD?-%xm_oy~w z^SoD?683>-X;Uc7F<8tFt#k$-E7r*`l^9lKDe=bxS)%2dY)UpKdQP_trmK`V~ zkHm5e&Be@&-LI?(>1A4X&^?NGIV+RBQH!-5sJ&XC5X0O*50afCn`W5-t8yDHh5@c~ z2hkQl#gRbNwNWpjXQLgppP!>Eg!9h*$ha-5Z<<7@aV8{imoCekns`N%|5@+ETIBt# zbtiG)J1#i} z&2_Ll8xih~FHd7NO`fNQ1#ZLjuZ-fX8pIg+)EaS4MdrLn1t!d7A;? z^+3T6S-}k3=kLbFn4YQWea83>SKQwv?Fs)kdprICE$G#zi1EJ3kKyap8W8N-oPgK> zyZd`~$8?#4F9Gu;y;2*XU&lF-(W1}9Un|+uMoNj}G4u?e5<5&j-D23pXvybM_Kf5nt-RHH+Ph3j2gt+89_opC~y+_mG+VvGbH zFJ38RoLDm}wXY*jF+c1QJg-6+4#G;0)fQ%}0hCk1ZUnAFavJYc_ns|OCH_ZtwSv(a|i5VzIY zzmD<~--|^oteWpUHJC0NGs6w!T3kgl)$gXoH>Z za{aQ+6qA~(Lu{C1M&z9_7 z8b4^MV*jh$Nl6c*Xs<56yIc6$bOSZj9(HX*zOg7f|6{3z^aea#7yl%_@7rxa$m-AB z`94Rf`tP4@4V8@f>a^O20!Hj*D^PXeg<pA2>AWzAJWtnc?M;O9FkI(7FneM`Ifx2F)hOYC^Vl z_ni$GO=t8E9kk(F?|4-pwU-M#CW>po_lIphe?I{Ob*gHyOIe)lyZq4&x7;60a0OLz z2ppuP$6esL@8f4MAR%oec7JncZ?2>b8hULaN4*n>NyLyi4e(049Yfg3t=n0Ae5a0+HP`@|K zRy1fc%6}FQix60QYr>biHMzTvA~924@8m}qIWWDbOKGjtcCQ(?`3*hm*|5qK+7Bh} zOZl=xInLSrxTl%%+iLH5vd;q$g52|B&rjL6wvGONA}Xb$E>H-!4b?dZRM~RJwI9y?W2q?JR!>!0T)B|6 z(>p!{d|C|pD?}Zj-EI=~3am;aogZ^*kM4iu(;HP}^|+^6j2Q8R+}2=~Z^#x*5=>^d z!N1VIVVpsArj??3oKRX>ix<9PhSy6&HC_5QQqF5F#9+5pYVx|C{(32_2|F(>YyQ3Q zd{YmdFKh*$l7&g)>ZX8B#Dxp9EkNZEg>Uex97ylXTvSdU$^w|(k) z(4^uLB;vk(eDf|Ti2RZrZm`%f?PZ|;nqcTODf0WCRtz8Kliis0kRO$&*i4}}&sb@g zNDfixL=6$&qxmJ?#s56#QvRP7aT zVOxpytKA0)uDfd$vP_OELP-?QJHEN(b99v%zv8g4pX4tz3Ncitzu~;-V&3M{Esx~& z@CY}>j{a%1iQJ7HZ$Y$ulRs?f>v$`C58#O9gR^<9^>?p}zM#3#{f^dSgGL zhRbs5LpFy<#rRVIRuz~MoHS1sa`jlnzd6k{MK6ly`|qXdyz&XP?e9nU$OMm1j464^ z24P;J|($^+Onal%I3+dKI{w9zKd_d+0`~9r*#PO+cqdhCaTHQIvF_K{@LDVq~b;U zOw`r|JG8}q0ChmG;y$Jn+sqiLKd3tJ=FB$<-IwP17$$}2L>%y; z)!E&8%V(YzfIA%;b$n~Onx^NrJ4h5*qa|I*5i0|n(WS9E3I2&2e%9c`JE{aL5hh6jr ziAkb_wD5%=%zFVBcF^^8ps7WpoH<6cHTwBA{wN7v;(_Xo z`y+lBe%>nQcm~wnAQeV!@K2)K&2oma&mFHhoNW}uL()=9Q6rJ84a2d=YEEWSt?e!L z9i*C7$IK82{_|cRzC<=w@`&LRI*3p?>@J2>$MFoQ=?%r?uQ-Q^Z08G) zzVXcPwsTq>ph#FnHY7J}P9CN2ZWU`M!65g<*9CEwalwQwkz;yn^ggvS?>u6~R_Gcb zLxTHZjmO=dQ8I_c?iY9|JhW)#fInFrh<)_;+^mwLFbAj$cyS%p{=LTvoeqzniifE9wNu{?QU*ne}q?(xHz3;M| zS5M$&N(0ZKdeIfRpO<=K2e(ED_#*Z{qH4rV?5^*6oK>ciuw5Ey-YN)G0mJisIs(XE z-+irpZF3??`9b8pF$Iztp5OqCY-CT$nv4}?IUIfOW=A?esur-z^bnb58Uj{JEw;I76W>?y4x#0n#s5P{IAKzp5Un6xxUbVR!v? zjnmk~UjNeK?PJR`W{r{T{M4ifA1)8T$GXj@`)%120&`}Bt6%l$dKqkH2z-nDSje6C zL`wP2m>#XtsRZsk9&q&{JaKpb|?Kva6V$~9a5V$p^@8@vh zD4Xk!s-rN-or{f)rhxWn`nl@!=(DTdzEN8z>hP`FjY2f))pvSv=ih{8n15S5R>25X zgsTs#c2#LQr})dvJNe4G_wnVNWgZ_W+qK>(aX42&cs7r(s?tHHd(D<}m8~w{^-CHhQi7x+e2TY zZFAIK2=pD^M>32DRbdAt?ZUo6*2QBJF@Vyw&AjL4LnYvjG{L{MLzhS2u9?Y4BN_+# z;_7%#_H;ba+1{cw{IEd*5av*sH zhMytQTWYH)_j_%IQ*ljEh9d#m_dRPHNlkaZetK;>?%^sSB$vJ)0M-b~zbLlJuv36& zVHhi|0*CWOd_bIW8IMIY$8-p!JuJObzbL)VvagD~e2ToF(mDJJ7y+ECPxuB@ zDc%bDO#M>7MCzR~5)_!pg>8c%&Q-`LvX5?-xA`ybHvnd++St9mMkKf{x}pPA+b!nH z%5?wt)|C0D^6UD0fpxv+X2m#vIbLyJg6E7dn!0rbNQ|#tZks!zd(P3u-QL`se|6~H zsl@5$K8`sV7G3pv1DX+$jTU8_Mm8*FONxnd=1Ak+3)MW5i)xVceQH}McP&Ek1LWWtrUL-Ex|cW0~bsr*QnBLRq&}jXwxu$LEAA2k~W|)Qj2D3(JvhlhMAp zaj0azn&`2Tkp6Z9_-fB6JJ>lg7I@`l3K^PJ0bh3u$eoAlTv?aDZ`zUd=EDcUMQK~` z)ywj`#J0`wptdDSSbxxgqTKin;v?2;lh(zq&0x|j_o%;ic9=sWa=(Heak7;jBXf*X?oL&)u zGh*=G!<|*JI7D?va@$rb1^)d|nqY-%*=>(#ybe(V+@Z>37e4@6rzWi9J^Ur3Z z)Vj?X3y_Y^2e~uoZiyiZ-<510mBWX)`C}F9XQSbZ*!1?OMzqoGY~cpuapkH1TnAf9 z9#_HY78<&*)CrF^io8CZOwwm;P=%^`t|lowS4bsjzj}6vm*1B;?PJ&6tbP;|4CF>W zkZ0)EnJIrDHflC4MgQ`51AW_SKWOCl-G2HO$@@mH)^i$cQj|3h^X-e=s#+I#qh>iEfy3C-&xKU6Ks zJv2+(OkFZu#>)_8{;KRc@YVwiu>I}A&I(DeX{~VD!b7@$pRPX(^^n87VHtu2yfchrY zClGIrkO93An#%^LU=l6)IopuKrWkGMswW z9jM6oP~4+8m#T6*T^$nBBzQRC0l@F7ebA6nlLyJWKY#Pi8m2OzyKGN?&w!doQtC$ zebj!Z2+Fq~5Lq<;J*B2>oAt8MM-)Nz^gkR8lUrWxwFAtBgr?~V_fLe#*#TB~rVqin z$~WEDBR^O;PiKmuF~FvVKG+w|;PM&zv-18O;mQ;?F5-Qi*=LZdA?}`QIJhL2En_jO zB!M;M#k?V&b@G1l^faU`ZhB`hUDRr7n|Q<+8%9o%QLDC9D@rupM+erInM9w<48l`N zuh{Df=3Ps2x!&%2C>&fB0r=gaHNIRcg@0QoNt*Ri*Yq(xf$({rq*p2c5---!ZuJj3 zr`!3^^C41dt7Py|b(UdpUlnR-YRkL0pazR*1L+fF*{TN!F|0lq#?d4p8+Z{&=FXjH%zg+U**G*J5cncjP z(vs0VckVZ7FLuhMfFtEoT?Va+*{%PKX$NH{nmh25GYzFj0*FhT)?(ObSTK2QGksKPKmJ z+!^GMepfvwoZkhVWbSaMj%){UzNu@Xgb+Ic^_c8bCC~e;OJAK7NDdgJ+#c)ZYE|=u zqZCm<2oZLJ0*pk*onLsF>`r7?u?{E3z3AFJU-g94lWKnp8%>KQC3Lj^>C-3KM!!}e znA#Hr+c;o(eLG!&OV(>kypsZDYmZvT7$1^AVOtkylnF=OQH3ferUdB`op$N%MH8m2 zttZr#F_V2%N;MtFK@DBso08zJN6hlMLmN{+m@?Yp1o5XU$}k&o1LkxK>*3)$c!1@Q z>x?Gm9(Gw_GQ%ulCEyMZNfaT+d4D^xoR7{O=H1A~_dfn9mCS+DLe1F!$xu%&y7Uqu zf2BzFtAlU|j_0Psq48_T#OE^uvjHcZtgMjkJwbGzl8*khSBFH!b-%`98+q|6RrHPz z->Z)~wg&R6kKUA~uUU$2`TyiARZ=Mj5cOpsxq{ty_9N&1nJ&z*E}&@~^6G1D_yYMB zJzBZ(uvqxYZFUsYVJYqC>7czXQ93t#@@we)nhZ|uOqSEB=vcnvc;CXJ9)TTy5z^~{ zMw9`|*U+RVq7r?X9CMB|RndOdMTb*4c95Gk({qjKSs%6EHp-BJ`; zJ`r&CUKK}CbF(R9$HR#{AB(Yu0WW{@CkI2`Tn$yt-*msXWS;E4EBpItc}>kf)xnym z!gl(=!lUp_uQy0R+nAyO;VTnGA2G$p6PXoLV-tc)i$=?{AINomt)|~)i*^QT3$2J(KHtxv>{8n|LYbW@DpmK@eZ-e*VMF3RV(|( zTheV=NhG;vkpJ`{v9Wy53~S6VBuFIiFj>EDNjp`d(XBjSa(--k)_!RXugq+h+L(|% z>W1uc^pXj^wtd1Sq3VT6V@<@?C;t%0hU{^lXfdqg`iRvf%6K{QIaAkv{PWz^qz4{F zy{mfEL2c^!zXKS9-@ay_&9*YEv(W{zh*={r150w;2&;=M$BHmq&34SK22Tw?N}~Y?V3i1W@Ts10@zFCyH_~8kkJy^eFL+fhV>eFR*$kM7%wSS zwm;~dvNm;b+p%h3-i`z-d`HpAzwTWw&@-f1!u>;s(Pk0x1VE+Jh|%@Uhipsku*zhn zr`28QKY{uGLPDc5w@46w#Oh1tw^gSQelN3#lv0cJw~AjBbpdr63{cHke2MEV_YN;2#irs{(zdvHJy*ZNoq6^F` z!}HWU)HW>O!1erLxlTjn#g;B`ZG-GBb6;?PXAtkVHuklDjA6&Eh`#Mp`xFi-OqL$< z!Gd!K;GdB|V=A-R%r`9>E~~u%jB*t71dHwW+5`&q3IuqyzjIXWqkp1?%B4(R<5YQe#YF5baA$cs zeOUoJQnGtfeCDvO$Z-VH!-21AYxuLXZftZacGc# zZ(jRZ1;{`{9v?d6_vIL|^G<4yr}k}s=pIcFsHyP&@*uCn$+G*uN~)-qB>yGl^-Kv} zYu3MHG&n%}pQ@qh>QkH+kN)9e2N>OpOkV$|vYs|r;yNXMomNxC^z+%t*odJ2{D}Lr z5wsuTN^@}^PyZ`H(P zF*0GJIaQCx;Aini1Dpx4@ZbIu%*>%TowfNO-S5DcY&s~CWNr-FWIBB3K3=q|^fbwO z`rTzBrb)PD{||4xN?A}aFtMi+z}iUixtd!v^;yO7C9}*WlPyv|{?r}sO80^5cStYs zFZM(Tf8J@YdQNNOxEy1Y49>`5oa$ zs{iADWo2dR)n!WN%DT$TL8)O*DlfItw2d3ho!|x+LSCkp+f2>eT57rX#&YMvjfzT6 zRK$S_f{dH*egAm>1&7CZ9L{-tUhmiI`POx*B8OCzgJuWvrK0gcE%S`)GR6z`OC@U8 z*tVNsNoMFnZrV;lYijv+=yUQvxW{1)U~qNT6OHPRGWF|6lo_wsF76YJ`M}jtyBBuCS%K+|QU;T2L0szYuh`BptGpz;t zb1}%4;faS+D=7wm7Y{~bErE`Te>#_!yO!4Uz%NKK>)XY;^Xkp-l3GWLpuCm$u|nr! zUU^Hin$B4|h}bWkN%;{Xcq+}MP(mrvDbMkK+IOshOJYO$+71H}Cs-qSz7h9h$*4fX4dIX=4=>TA917O(INCZBD*>h7==qY;qA@Ls^&VlNl{evVSZZm*4X z9<3>xllY=LK5#Q|>K*q1w`K2gk$1rt4{XN4R>~kQ8-6z^>0-pzVO~4GUdrU^T#Q3R zG4l`o>~v6^{63zt2+{{<9m<&MSOZ6ovm4*VWamcYA0i5URO^8e(>BM_Uc&I8$+T%u zLsJF!_Qe2t*YbJf+nE*CEGKb7P}ythdec%n(2F zLEa^+K6eDN*1@SO5tGc^Ji0I9Z!8MkZoi*_;^mQ_uup2* z8ybkJYalg+_4UgEvepH1B^!HX*K(?W@txpzVSBNT6PlSW@2_BF=oI< zWpDYO_$vVXCnw1GDAoZbCN|)FRt3G~^{lxC@ z#H}&6b3&hvDQ_ahe{j!V(FDq;L=n+W(alh%)$%*^zg4Z?SA6e`udBzIyH+ zgAY0Xjx}XQR`k`&Zl)K+SdMsKsI_;#;b{jIo&@;}t@a9$0((9$0{b^)9QEGysh+!T4rSq-VGbze92)H!=gAA4iITD@D}pDuRVDs3wD9 zn#SjwW3>sU=tV%oM0xM|5evg$(}wk8X;M`@%6D#YT=Y1VI~|2AOPB!C+c9N^)$Mho1*n`ps5`@x#cm16m!IsK-=D?pxD!N|0Sth=J*5Q? zbV!LTvF{TbnV!%hCU<8#r&=3MqIUEilNPbZR#PXl%(&jGX`MiFjd`oY=$Xab+t|=C z8RN##wjk>t2d@*bzs?qtE8qZoQUkIWd%t!j*dU{cenJUajU5oiIh=5}AZ-8p1XNZ-j{OH-A#rsqe}3b0eCg+oVf=Zam5EXRtg*pbB^L9FdLKgpVKCG$1G z<{Rzh53`vTqhEy5B2Wapfa}8G9+iN2_nZXytU|iGO;VlU@MHZoKJkr>0@k&7dy-q? z95!dk$^D&iU!nFp#1FYm*jCco>5{AQd@-ott@Z>vo#%{89uub$ahjdKR8cvaTRe?j zF&wZ~zg2?S*4&R8OnkM(aOAl>;lv)Br*S*Tp1@}s^QsavUhVA%9VsObDLBnZvpetL zSw-ZZvvcbiWmkhh*mZpw`^egjz@fk&wu`ER%>FNUv?V_J#M1()tk(r~?jn)y*6Jq6 zq*1cqat+^&bqUmUj@lv|zfE=PuABG(v3w7~Vvny&V(>!t!nKpk`az!o(FjwPQ)D9*Gm4^t1t<~`b?cG1{4!Z44X{qyYmzFIWw*54ufno_09}r@;N=_K3Z)pY<~)3 zZ;zA0fV?@d5oV~^QET46H^*b;tvp2LFUSniJtU+oEzM}}F{PgHOc(UJN%Bh_Km(yz z9n1=F5UW`1DOD=E(h;ubnxZgLQy~k^te=LF@f8ovd{II%LN13g^)bcm!rE~p4fK-e zbo^P@X*D*5Dj03m1ymN_NMQIqu&z?CgdK^{6nD$(`-UqmXkITULl|RYK-;0XXT!H& z;-3ZupD@MW1AIZ|XVFQ4k?FZRg_;|Cx^d}l!wXhHMu~o9`s3eme!d#GqCYuVEj26B zf1XO!&P<;LFTXo6rS$xTDdUR`+BYQ@sdGaw-P_WjViY;`IHt(ty--uhREL(*eAh~a z>}Iv

$w)pq`C)&50jERKx&LWm&-?t!j2mujxz_oHSjA>h&Z41Dzuc)Gr5M=%(e6 zk=8x4G&mYw7c+o?LV3R24%F&@z8um2Efa?GnGRKW5Za^ zsU;#(PaHudYePXQnK`UJz8Q$qK2ckt4yC@83=eRT9@7-{FBjx$JN~KI6-xqIH?-Jx z+ly4kzRyae7X!83j2t`i*1f3_b9gZ=K?}gYwWxv7gO!IfPnJw{zB}r{K6*^vJuT(k z*sudjyb7(%2;Agn>FyDl%OAo=bGN@mfVsxiG27-bvQr(gH#<~+ zD7fZ;care4EK#E@;kt6%X$omEl7XgHHrvv05l)^V`dh4WrcLgYi_c#boP{S&LQi-B z>o5B1HS`cc2~W^lc(g0IWxii*wt5;a-X9DKQN97|nf`ovu_+4XdCZ4|yHrYxlbSM@ zlH1(Fd8{x~p0A|kw1x2PzEA1ck!O{v!J1ra2>B9?HESkSsTqQl9AE_Yg12Qk!`+Bo z(Hbs>dAV%RT|J9WMZFlD|EUU|SkK<9%a9uro9;j?l&QUsi5LN{mhTC~-qDX3Uo<-9 zBQh%nwio91$xaJ&rAn9o7T9hd)VQgBZdzH>*z7{kq1*OI0=0l#61(WWGIp%_!8MLvcqs^ zUh;mDgljPqIJ=*{VEJ?R+(y3198=suQJg?GN+e|3mi~@IuD-Nd-YSmDV&3r$d$R@4 zJD5&-RqsCMV-uDSsF)I!BTglk8l7lDC@tHbTzsLF`yx)sJ}wG`l*-<}zfaB}jcM>t z>vox^QxHz741J;^(iTp#z4uh#fqh%DA~amO=W=9Yy^qQJ76 z@cI=Z@;5u4w(~cPn@V)Pq!+XdJl`MORc`p~uVCPfgC-g80Ixja*Z!#c0P9P0jlIoI z=m7vJ*`&8oFl)XETE4hAH~*db&kgLYPYv6V3J%XnCA-pptlZBXozmi7nvxr|NZ{^Y z)vX?kRGaCU39!}+9x#HVlS98M@C9$_2HMTg@nFRZ(>~JmM6bQo&)2yc$TteET|X_k zMqW94`B&Yq^FQ4FoYqf(T$}+i)nT0B~5&ILP`%^%ezdwpui@5r;^fm$GR8CfdD&(F<~q;Yw6GAa69Lmx!ufOEi}M_27I%@4A2 zsdYl&xVE&O|sznjZ!y!t%!_YeD{p|oR@!~%%o!q=hD~md={D-^vdhh z0pFs{a)RowO=o3Qpy0k~X_jUd$GEKcYWV81;Zeu$#q3ANOMGOdzWd~CQq?vPPZ=@K zli>;Vv6L_*JWy7LcF}U@D)0-TV8DE*#vDAw$8cB(4r}aL?kFpiyw3@I;Qt@mcan3$ zSAX1*yir6Q{RBi`aE4XqSAljzX=ToeGte(r`R(js<{K)0Q$~~gmF)5@LJq5e^T7l2 zZXGd=UG>Q;FXScQr(&I(a+rAeg0=C}pg88^(JQ2nh^j_qhwm~QNSBr(y{DY@h|IMo zhuQbB>n~wCZdvfq=LYK)2kMHw0&jK)OoNe90s$b=73B?82=stZ<<1ap_1|3~rrkCc zx+9e8%nPXBQ^QR{B$1_B9ueaok1j<2n)us1MK{+6o+XG-%7)>$Y4d}v?&mUq0bv1i zyXL!FdX>KGn#GG!){VMP>Lf2=*ZIEp&Gf~Y`s#(Q>lX|v+-LL?=~knjZ%rKm?#nUC zE>l-o2I_|r>aF+CGiqOfhh(FmEfxMkLpcnEZxD&Rqz-PY?;Yy)e?5RlFdD9n4Q{+D z&kG(b6>U;hs+9JDipTxI5)mPD1=yP1u<~<2_fC(00)Jypq7!s}x>liwlVpx@P(RsD zi8PsByCL}q!(Wg|SxkV_dRh(h<((fkjzyLKu$qu7O#~-%aF$9bD92DJSa>|^9;;E% z6uw~oPEwnG^cAHTzVppzPQ%Le#qWA*y`Z-asyJf-4>5B|9Q$s3GBd0slmw-+4on?(41GNGJNyZ-4_07fQ=ucOKVJ?{Sw$C0N%u{&NI+`kFm{`&Cv8ZoGDy z;%GwULj^ydt7C2bu&2%ByZZyW%y(C4&MGnDsBGk+?7t_|qSN0!@1C7lm~4QbaBzS% zuD?eTs!O&P=mftkMisfo01{lcl^(xEc8gwytv6EA1EF^Jl(F}lAtUW4Q-pRb6O6F? zjb6^T2~mg)ugCSn2tuZeb%b|fp@g+h&H`=rbk@$7LV#D1i*MD+B3WVp%0AvIu7Af zDREB9yfLVP{5Fitf4ysnI>oZ+rV;`=lEip=jC+$;8kwtA{@>V?yT|j{p#RfoqOFF_ z(%I+I~!c)xkuTi(i>idKQ!b>(*{ed+m>4>_p86Q_IJL@XL|8I70j zD}|P+xB!PNP7W#$?I}SrleZS&DRrLYW8Nuzbn6~bYwg_imJr-TAA&5)4d^*-(m>6u zC0@r=?o3lAm>!vy1*3yQ%=EzaQjg(4kA{QsQnBWsVS!~Xfy&2FkGpS5B9|$OL`*)4S3ZrO$i7oMj+NH(R49hH~h^=9L>^l9dv&IvrUHbj2m}j4LFNr#_xZ5_{giBrBdw{K7ztj!_SU8 zH~V|~U)+iL(<{8r+Bkwx?}Sx$$(tq*1<4WxC_d4I)#CkBMP%bM-EP6;*$Z#6*7R4bzb}XGZ-Dah!9%KEPqbg z+RZ8G9)bBY)~BZ*;M{UaQ~mJPB@E%S3EgSFqrgOum#QRMcZWm1Fl%uy-m5V@WUjgG z)fzfLxk1qZ8YkH$R$L2)AEZ0I`&Xw#={*F2#VR4WF zaxw%>EnXQ*uJMyakA^p=oms}wmW3y3(dYllVEpPd1+j7Uznz3KFSbTHQ!R!6;S}Re zv)6**;3d;X7&V%gf8q9jy%03*=o_Z1!t0j3?KU18NSCM=L|r8xW8`zH*%HIim8$U3*f0B@YhFVj*{`o-!%h+frO&@O99r! z-UZeoezSd$Cd393>YytX6wojRN&##qWY#wKy2M7)_52I44+FAQ-i7J45Fh`#wWU*9 z%RO~JDvpf$?f>3`HFR_xl_9G0(K`XyDI|K}WUUoeSdxKk8uzP{B5aKS%F3}KjOSf= zC?vr!=(>7BU6#$0Ys<=-eaP76f3;yMi1NAYssT0t`fmog$I?sa>x%8%C#|fCkij66 zytts9CagaBv=pSzv{{4iws1$S2+el-Q6s^hE)3o{VD+hc=w_e`O(``;()T}2=oYr- z2D@}#7~)eXH~aQjQjAJMkQb+W@BHG1O_Z}*hPXQ-Vm33*?8YUruA7(}S~wlcu9?MI zo;5KCzi?Or`q@4mDC7Mx!en*oqB42s`(pA_>~c$G!P60sh+5?eHWnH-0_^H)V$JJj2(S&&rkjJB7} z>ruxo$DgWJbY$arK3&?WYKlOfxRppts~@kDbUyx>~{}4T|xnXeG<#) zmVj0;KJ)vRXgKMRR<+lc;D+r;OK<5aaknH%U)x=jj__u?6g$1H+Vq;pED`<#1y9C9 zuyrfrR}>G_rS~+mym(~-=cOWI3)ns~Wuru7Dzo!b)vH7QgH4&uZ#d|SgY}};OZ&6$ z_FSxg)iCoW5;?*7TI6H2Vif$T*#%x!YkH&a;Vz(^7PwTq6~8)wd=a!e_jRj1c|uyq z;-cG|))Ms=q!xgxz3Yt7$g1n*YtSWIGA_#6KFO%%oFHzu>)P!& zLt6rNx46`Sv{rWK;*&+r8yLxzd>pS&zoFGy`n6lO|3ja@7Kq^Hu}g$t%^HMi)-BD| zU9&!@N>la)=9Hj+N}5?3+M0~t$M4}!D_v`?7}V_0?Acw?JyM{&UTjQ?o%qdWSRD;m zkzkC&3EAHC`lFlrYY{7)bATRVs{^8L%b}l_7mi|F3bK!aX8FA<5H5_9`Ey|u*)#y3 zkj*sm-Yz|JHd=jZE4@~k1;G)|3}*Y^Fc&If5#C%`Fjao3>_c(CsG>9hHEUL7YAFFj(WCSz}T zuX_Gx!itnu5|dv2lNcDcX|7;uO^43(-Lus+y>7}{xnYWJAO~sMYG+5@{-Gn#wmQK% zzPUE0E5LhYF%uJZ46n3@j&SOaS~tj5ei>8w@RNI9 zo+1??z_pcBPMfM$6X;d8cB1KI;Ky;g!BWRb!>LKz&YfURPIPrOp)+n4rz3ukZK`o- z&T?N&ZM{C~-3I4pSUPpL3Q+(2>XynhC=t7ZB8Ofo)(t_wkq}58`}9|3!k8O-}cVgJ>?R-(meJ{PSqW_sWwUo zL$ys4!Kq>T!J);y1Z1K0Mbeyq>aE>%3_;kU+ZwmmG7eG4G?FwLR9;| z_cHi-&vgS6I*dqQ)Hc&*QtP8k=Nsu0A*-1M{F8njZlVHI$Um!hs-L3Mgx^DC#bQ>DJE`=z2H-cpg;=sCjqq$0{gHeNlVhcDQ>E28k@ZXK5Gd()` zi#wFpYWbfUQ$V-+|8Fu%JPl?Wa#sgT zIXhtNrr6n4vGb&r!p`upsPffg1DE@Ak5fMUI(CjnMlg>~7r-hPHr~~cOL2~BQYC&+ zZ4r>w{y|LT=vl9Qs$mQVfggOrt&ke`Z4uR5&cSBdY%V-KniyZBi&a5bx zCo*diNeY?L)bsw@K^Ly$rmF)5+(LV&sXs=|jUKdEPFclR{mM+Pv-vov{aZU56rdSiKh>cTd*R7Ft0gRPZe z2Cp*9_Z%D%X$_dY8;?8LF*kA>4vnj>CjQ}9qaw*PL3uR^K zA==m8ZXfnALa-*JoHn=Le-h{5Ze2I%z-t{AolLT`SK}{~%lwh64wlfIwlJGyRsRa| zh%8~PE6=UCAjZQ%8;uRfTA%elufxE{11vn?^@JmnkRW$0H-5@IkE-p*bWxxA0af2S z0+rb69U?g8366+>rd!3S%O$e_TlziExHpI4odH{pTM_mO(9Fj&R?J(+XoT3(H4CjQ zc605tL`aPU2=;EX{xsuB@SV!!=BA8`ZAr?&rjZ<5EHIpy!t(ANlMcI8vuHEyl?;y# z4ayXY7`!#h7Kdo3J->@ywJLXn*=SL$!8`rX$g@Oy4YQ{64OJ?qmYh$J> zdO{lK?ebcn?}{y$4Zuyte;)>h>Q;wA%UL+g`U)!&o_?Q~H)7m|0_J*y{IOPz(E z9KPuljDDm5EUy5YZjlxJvyfef=4l|)MZZS!-yL0gTW{LC1fqg;vjVG4zsa0^7c*)~ zMbJ-s7^8&vpp1KmPAxu+y$OK#DWBhdj9)A|rvq~bodD!+)zGVdG5*VCrz@|NvO8bc zVGXnqDke);x2d;-ZHP0AuBE_CapCaIyhDoP@ndwZ9rHUE^^&Ge#8--K}ajYQlZ(XUX-?ul&Xo-2|k{Ttm~>z!y`^Ax)VF z2MZ?j-dCk79WW&#fc%bdxSsxmPt)oDNa{4@Pe&*O5NSBF_mI0awk7rt2R zo0TVNjHB|x4jDdI{w>Jq)syRBr>#kF0}pOni{o|~mtFTSVqxzk@A?$};ryDt+Y09@ z@p2Q1rx*FDP{(c8ETDvblSwst1A>n%Rp`vH_u^fX(lp%yycOfej(l~`q&uWdvxwjR zvi*p~Rm5n-Uctm}@-12~Kqju@gYs{!clh618=>nyt%WHNsfaUVh#S5|8nR;ZTo^Yk zVpSmg;<_3xq3(f!v{EIrQ_ru;#tGs523_!0DPyuS*g&19zu(|GM!{ zAFBYj(vX%du#~Tyn8|7Cl10yw_Go&p6q!;ZeIfX~s-3I`NcwZDkLSEsni@mOv!0>m zb(epi7+fBa9HTekjM0WqS5}TYDP2(li-}V^8=zAWY32SH)MDO_m))-jhB~)rQpcAb zCu{GK`H+-Pv59Z$HkjO>q}}MC9L!Z!fy&jdD|Qq$mAjy6&Cs_tYwHON6w|pAM$B1X zGu8M<>+72>97bpe;c-9cA|L8p*>1SkceaLG&WF~I?n{O$Mg|jdddy-yGfw2Q!?!X# zsbP?T<0a%bwO_!rBnX(%%%`%-=sS$C6C9GyiL?ohkZB$>Xen>_*RNl3`Nas!9I1ne zOgGg*Az|q0m(Gp?c=Xts;l~*D@B@qbfT-qvdXaIK>^uD~()_&VG~_wBdMxt!i*In? zO$U=I?45s#O7@SsoukQdQ?1p%cIzfWQfJ*^0lLsiui%R&oP$dNk2+O}m-2k6Kir*J zD?#~-Wv1@pms(LQ1E%tG;#1y8iMhdw3FXbfm!_+9=x~oLtAMCE0rtyVx-Smf){K0g zntnCj2O1dDrr&jvpgJ@KiZ%LS8x^Xf@D`=zSyx*C)rd3F`KfLw9#_XN5oDF8KcTKN*1i^61ba|JHWi_q@Y8Z)mSc4A+i8Cv&Vwn*CEbX-a=;tg%Myiw`t*j)V7U2^ zECo?RumNO&l)kD9rz)&v(#d6S`H4JZ1iHaD_MX*w+;m?LHFAIzzW4<|?VLJoS9h-h z@rJ$A!gg_vM~8)JCMvz4ooEdyRtWk_LNEYqHw#o;-6*rSSRA#lOKad1Rm8{~tq|Sj zC!YY*uE@cCzJ<62r=m}^-n1!CJD9G2V3|XfCRQK*J4D{QjpCgTo>tXWug4g&Ja&v) zm$-Vf5tIHG4P_W9@RIt40;bX zcE%HySj(*UXAPF>c@oy^!%HsIsz5rFPHCnO%SrMO|NHb@E*^ zLhlunXz|bbO!YjOvl!vH>GpVj6Qynz{{EX9?E(a0er^2_sgN;Qhg1>iGSb*r3PV zkc{}+szLQQQC}^}vp?rB-COSoD(_|$%Au~pufW$|hvo$Tv9V$Q+fe@fZHSTXl_v66 zXk}zS0)N|#Yi60_BfVtE&NgKWvkt~o@8UW8S9{p{JiovD;Kc`Bezp zf?VC=VfljijJLGlK2-vW>`+IQb;!CvW9AE(2 zB)VV}My|y(nzNgc9NVr9>NzK_?K*PTMsw-N{`OxtaDx&BgG+ejg`F8qVKmK;GlVbegA*klryRhWz##wSc!Sc^481~rBtZyU=nEK6UEIgOuQ zf^%%Z?Nv+C@Kq>kGGS%sEs1pPru# z8UwZ2grbs$Robvm-d1I-86_Vqq(;XKH2n#BV56-L{z*XM0g;BrDC9Vjyqcq_= zQ>(@675rU*%I~WGD8k%n`T_*RVsSm29C=pDOGKOSBoQJGGAWe&PmlX&Cg)0i?jN>NVZ;U=j`CX?+=r>hEno&m@= z59L^@vt=s{>tNG5xgWb~nXi~xpq6#MimIs7#7csdbEt`>iGTM1drooY0X8Qi8JVh5 z1Lj`e<_7xb%!2g0t9W@DTlIIELhf1_pzuNY1stms$0F&#q(;;uw(sM*wlZXTca>!B zCi=w%Ke7Cd*$1MYZW|`nwsttB$Ez=PPF~HRf7DAsefJ%K**MKWHIE?b27NnhKoM0^ z4=5MxI5$sHQa*QZ-f;TNsP6+qR{HLEhEQ^&0vEtukCDWhJaX0JH;bc#5?eQRmD3qn z`%|S-)%T#;8=Bt+!l}>RxZq?~jCV%Q3Q6fB4F!D7WypHmgB&P*Nc}i(TJFgY~H)qcT(ul7rFl1z0UouBM#)g_bGh8 zLKnyY(e-(S>BFm!*U`&#@1w2Lm$X+hQ3zF;@6ZGMrmUAF)LACC35-~{zSpWOk%YJnepIG6O zTG`bA7*Cbh9e#CrvF#@EhzkG5Zb+{wQ5}~Z%QrHJpc!-vE?QvU-e?}V%Uzm!RpTgVu>Dp=E5l;2x z{J?}DUZb~f*>!)Pc!&*Rv))IZUYD2vPP`AYQnmS$=1V|VMfLx#?%K-P!cUZ$VyZp;wRB~a2;3ugcLJs4vG}OR! zp*k^)4wWYxxdT0f$%ta8?ZKl!g@6S*B(M&OhrZiSyo?6TM?T6JaZiN$>@&VzFQZ^m zN3X@oR8+}yR|~;H3^8wga)CB*lgoYkNXQ zc^`!HpW@0MYc~e_-RJK2s&YT5IDPlx2Wm zqQs9f>Mb^wcj#&%StB0JAcW0jj3uQckJI$3iyLWjJVNQ%IPeBMEqAj^Cgf`v*yvZMwh7J*6M7qnj|HvrL4N^#Flg%e;dU% zk297U7@zq6CRTQ;if>xtMoc_XOM)uO1ey=BRBE2?&R;EAdrO+DD-qHU*o)Vh_Et4C zHp@^59w^9|J;Bu1uk|;;%vHPR_`QwEq_$#(Z2J~U^BM+8uY&!t^=%r z6t19#ob;%^1`$$;h{onL#FYSApRNt~B^sKIDc^5SWPdy-ooCsFy-Ft+5hLj+H{H4G zap%*vZ+v;jO2Xzy6%4LHeENjiE-%Zyt_#up)+@47R%~q1{eHt5s>YS}N=ydoo31iD zwY7TU%C1$(3^<$>pLQG{-gWK{TC5-e$ROPgp`d7gWa0RMy-=n5<|Og5!z|J!UK%Ri zSO#=piR%)e?#7bh=1}O;zD6R zliLo0lMa=#b%*WR+OFz0>1Y*ij;}=Qj+Qiy&{@d zuj%YdamLg777-st%)=fFt8jsR;Gv{n6GcUztl~Q096j&cc1&h5&VT79A~;2d2aOG+ zo&AZfv75KiIwJs2*q?v5SJ)ErC8p|?w(y8^dSjc&I=r8NIT4fPAgEp)oWGnUrTi_W z<_Ar_u?z3KX7FmtI=4;LOF^*zY6kI4@b$%&zk|BV@>T^$xGR zGpfE`wR0GChE<7cJh%;(7OxB6KF(4FnFK+AyU;nA$0&?h3;l#jx7CZDEUe0W$wr$h zJ8GhWYIk^tcrnSnCpo~cP~SpV`(ne-?*6_e+&}J{Vdhz9#|utmU@Sqn6M!~yu5-55 zv)W5VRL~#~PrZ3~kbIa^xdw&kG&+fjm1KD?T2pgewW@`bh`m~}g8`Ado>i43%~zVw zx~EH418>?BPB6Cr%5;4qZE3K79&+ObG_YM{LBP?CFJS97xvdI$az}T;A00BiTNJvG z)V_r}zQESSo3NbdMdG6Sa?@yM2Bqy_Oy3myz=QQiE2=|C*@FSDwx0lbST?`X+0bXwYwCdE~aBf`i!@|s8lWV%vOncY<7A-J3#gZGbw0cCC8gHdJ))^6s2*5FaQ2dcAua%A@{M)&6$GhPc9u2=`|N-PTW< z7PH(E;1{Bm{s+i!W%=gvL2k&ot2TM9;3F34*``oU&&vg^b0R% z>^K(gAKmwBjq1VR#tk|Dnuc2gXnm5m_l3Y~pRGKtZ*1eQO)r^yY~*3r zDL+~(OjLz0)LMVVzZ$t;)Nioj%Cu%SaToHs0b5p~|G-x|6EukUe;B4YN~#5lQrOSh zm|Ns4{5PA`XnIZSX66K^p}>kd^cBU`vS#gR+XDFsr0eZ4#;*t!5e$vEx6?pmTjgwL zseo{p+ksoRa(o^!E~sl3XosStFhc_7p|Q>+v#~=}LGRWDYVO*Cp4bmbuaPe{UpMs@ za2looh`si|pvMe!)cz~^Vfw7}7mCscd!TC{QsbNeYAE~z43fGGt3;}~Uo6_hYM{Qk zr7gMn=R};U6#KGY;2g=k*!ufo@B{D>M*=Xj<$%clt<_BPiA4BwZ#*Pxy!N=d zW11zN8TOSK)SW{HJs6g{<-%^2X+Wow;$O`7^*d%Tp+= zt$)KVUfo-^h7X70+AdS&C=)(^mXmMCgep((ERH8%BKLBRi37slh21LVM|t~4{RgOi zDU2XeK>bG6e%WpFYxIbea?kin5gy^yPgx7k{(}EbNfI<6u$*o*U4FMkK6%=>2qs48b zqsfh-6Kg<zBJw=8uI{4j*rX)@J*x-HIj2FyhYry8y{*A*L z!MipzVr@#SCLnuOD9m}$G^S2OY~u8UN89~bsbiuiQ`~>r=%m-e4fQ|iW-Dd&5PR&p zhC_wR<(g{CREhsMWAeai(;VZhNw>9cSe^G*OK3BbpadJO$g&avFC2`A$CTGY!}BQ? zE5<}>ju$n_k1U_bDoI2!rT#Weu2t?87WJ0oph3XZ|`E4>;cDF(KryIh0lZuum8ho+VSh{_6Rx1o81SB=MRe=acm{ zk)VG`56jNlWmZ+Mi5h}~J_$V~^94*ak7V!0mAefQ3PrkCcH$J)!&xHlUcKg0qsj|g zUEC1*)BGJ7Dk~~rrhciZ88BGYp`IFg+2=uUyIJP#rGtCjL7*P!b#VzZtsV<=TcSFD zVS(|lQgd9aB;}3b8%yG{TnTFOrv1#)?*sfh0obSNhEn0%QFl1^)bW=#x^GUdZg|YP zbroC(H(KiPX2O>SBM&R94 zy{8(nXZL)v8J2<8UcJnx^(1lHk2(DV;fkpJnN5LT2x|@3I{CP6eEU{Z108=dinV{G z+QDC$I{pTJcdL5##^S|8OGR;rF6bNHy>3<0K1>5=_H}mb=$IT_W2aO^J-W%@4Hkfu z&~UbF+~Q@FVaT=q!Od@LS62M(?tzgrnuara5qTH5L)J&r0MCgHZxJMFH!W_Bi2*fPKMd>yrW(B8hbG~;<((Pl*!I(Os5*4;_9 z+UD$xU7Pt2*y(}8b4&624Z_Z6`=lvPLHlW3)`c1EhHeu#7hhaVqSM(e;h7;P`tPx= zZdDSG-3#KF^CK~k4t)QZHqDIu@pMx`nar5ST}`m1{^h*Avt6Ci+4+4< znO>lSk~_g7x7pT=D$r+@3_iMc{Z4VQcN2JD8WGdfeTui%R_P519@YD+fGJxjjic&~ zE^p=hnRuw35$}B|#>9HG4)wG_0=Fc#FzBnz>&-5hHkEhV;K7LvArYjbWG^OIOa0l6 zUWgT*dZ1PUDl9qJUKOK# ze81}$hxI?$ed- zvNgq10*E_+q$h-VNZNh!^BIZ$qg2{4Mk{oVw|A;M_S}BXM=sSRTDIoTkpD?sm)O z!3tgC6dvnJNP!+Uy52{Fpx3fe{>Er5ODf*z8O+ixXb&s&PAcPRYf|{9L)M2svQ*fJ zI~w%lej*Sl{l*IhkIjy>EsE<-k;+YmOex{=O+c7yikrMzRb~amc{=x3c6iK0r;tM~cPDG?) z#WjN06e;b$Lz`z-A_ylbz~HAJ$vJgH_48?cr>EBU1o4l7FrlD_(%w2RM3+c>?6$rt zuWd=Krp#5d`uVxZ_g^MB>y>nLwFkQS$*NAiL;~f{|X$`&#gF!#*;Fx4r z%bd2AvOrTvpTKeG?K%5*8ri^8)klNQGm9&pnYsADQ*Oo1tv=%0p)3w*FYL8S3R$Rp zK^zKXMEeBX(`Xdm+N|(D^VK?h>^DyZAgAWp>@i0Sqbtgc|C&_nvQL+&X2?yB_6yMZ z5WxQddW|P`QvqRN4H^-pavSw8(EonV(p-ucv|u8XC%t}SI={iYt~Ft17?5;U&WEDr zy7GljuAlXSdU?)h3BpzwZj*M85YR1eMSJ*srIFvOSF{(2UJZ1PqnAqHHAUCa=#WO% zr+Iaajb9Iyzv`|w@&li>dV0cESnDHuzXJ(lw;vERT_q@tV~`ce3ubNqZh>I_aaR-b z`(Hib>UIl;Iov|hkJO-_`VOv64&CbV|L4YH*A@Amqj0qZsiLj3z1GWhXD3m9Uk+7E zU4m&Bu0p%}tPOM-et|j7ft~_esPbi{r-Sm8i#@tE~7&XfPn=-E7vnab;t*wsQ-tpbB|~8|NnpGT`Hm8V##6A zLFKTLoNXag5~V^8OGsiQXLhL2DukSp!$?97bDXmk<;X+lHo`uj;8uNPyV>idd*D2KtumlQPv%Unxc#H}!}YbiT54tnXjZJI6# zu&~Dxp_pwOTYJr3TZT#f_IU&^oi)SQJMt!`xNTOvI8dl)O-l@^xmmWUDo}E37s(wC zsFa^K`~(vs(sNQe4Ozdnl;|e)Jy#Q!`7=80&(wmqgeJ0m(99!OLaMRDSezryf)WYP zOoazC7q;kgw4FrfNejx!dq~0{bN1BgsEV7I!Ml&?ZFC{O(=wsBU7}`ysOzwlYh`29 zDn38UvnyRUd|lNqTsj^qo7B30czKFpJVZvs+U73=ey z@4;Km`@t(mvr5~ccRojt1o)rJQ;%r9Hl)4XTi3gHx~PSQ6r6Xw>)_wGj7|gI?3lwo z$$oH#cmq891-Klqb9}N<{jd1O`;nk(i@^`I2-ePwU=uo$`*})#=^QUz>=T*2!p~lz zDcpyBbBvyn#>_Rr_RR5Fd#ZF^L)zWDE4;`3mZENnkB z_WMOKEBh8(k(2fG2ggU4A!-ea2dBqbhae}7EZkebW+R#FQ?Z$Qdu%pWHY-lbj1w|T zZ0ugh)vc_#oDS^s84ZI#fB1AAFJUwiCB?nQ#MLV?sd)w|S)YyhNe4Wt&54^nzQ1Pl zu3x9^?4k2&cLLRyH)_J|>|#Onr8hUZms|Wg00AFG2K2Pv=z7WAhtfBdsNr*y-=o+q zJKH&zHl_;2c9c(B4xTa*Z>$vIW}qW2u*l`V*xRcCbesG563nT9-<~&W8+VOfF9Yt> z0$_A}0G`~l_c=AOw0-C2s?@n9eh}an>J3vVtU~hucg+wIx{rI8wkmNlyu76Ga6d8Z zQ*2vF)!9k-yO=RF$UgS?Bv5;tk@JrU{~wF=%Y9!M{~nX(US@I5?3N0%N1(JX9LWG; z)UDm8ohKoTGV94QA~yx5G3bhLj-5_e%azS?y5n86E~~Qqh4Un3sUgjly6`j|`$BxU z`jP2(*rtn6Axc4{9(&2AiVZ4E#EEm-L{oiUfnIA35#4jT?8RrBSv{=hu6e=%G zXw|BasyS(v?>#>@TCi*vyG)**fSW5Fl{1i%tlyP_Ywuig-YiA0gsdMDAC|G6%ShHS zfwad;sI`;WhyT=Sjt55XI>NtpnV4)yNUaxsbY#)11e$>iz-jkI%2G@+1^yG^k<-ww zCOOR$%ZYPxtXBO%o)cjjbk%342b8#nGvhESm1V?rB`7g3eG=pB+nNxIqCb&CkoG@7 zSM+kcq)XPU#6vJuWY3uXQgb-B=ZB# zp8S~nSTOKy;hX+5@GSf2}_e*=Ez>x{p@~%)B8}?pBIf#@2N9R&^0>j6a?N?K;jm4^kN~46nfja1B}2w`UbS z?DIr5tx%wIo#L^J50ILU%(EXA#@^DqM3-;2p}I=#;w&N*(3jiHVo2MvZ0GP3!pn zsUAejeP4C^?6r7gLJ{3jyLVr|rxDbjJDXTFX8R7Zv^V%Xv8Sf7!0>Oc<#&iZIp+5e z%UAk5spq^*O~M9FLdQvEEV2G;`PkC_0t;JWpGIkIfVV$6zw_-dXL3x;D>3Dm@x-N1 z=@w^Ty*n-}?Wz6HY!6B%li9s---KOsHzN4G7~d7F0g=TMrz(A7K= zscRaf1R40ARSfw82U=#z7SRT~4h)aYIq?z0d7Bfk+Wwvw_w~C7)n2l+)%Et$n-eO{ z-ofsty$3e=&F6C8G|qVNG$Tk}WMAXZf}e#hFUs0nsdMsr*}8qoGRx+WoqHEcd!0$S zP;QwGc~k-=pgLrB3R`WwK?e>b`=R(|`Fxr8Midv~eAD*CGhafpM z_0sy*jp{#@kyEs(5{oDLkuLl`V5J?;RIBfo2vhdB=A@(GrBPo)pRi*rfW*n(CA2<) z>(V3<$3P9dWw&rn&wO}gHTF}{;yEn18zs^ZqtrUc?{m#(&VRuHe@xDO8+|^9mcMrC zt>nnw)~jwGx?TK0%Qf(_sFf-h^y$eK5u&?8Tj95|Ki`6!e&e|^!i{IDb&q=!Hr10S zL8c{~9%FsEC-{mc2^un%;6aNTomvl{v^k)#CA;Gh9$&Wo>NbvsnwhdQ7|R2-K}R1C zOq7f6-o-0!7%VSm!tT>i1RsHZ|&?eBRH`*LUR8 z1j-aVys&~#aIFN3k+k)<*S>9Zg?m|JE>@b0eK>McOKN$&doLW`_1P0XLsObxuM+ie z($+ibyyK_sOxB?euTlvZim!Q+N3dtHh@}5J+ag zu(0^fOl9W6l>~};zhi9W_EEg5eaAS8dj3f`McHA`x+_e>wuW9ATmX6Imgg)H# z?oQfpkS=t%uYAH65O~)u@F26$WiCg=X^|Hn9mg-00Im@Gzu(f4t!{hwQNd^9m%{T4 z;XhKmMpE7iW^_i}#y|U?nc$SQrH{HDxuBvud3yUP z`L@gg!5y=V5D!Uk6)4vBb9Ah&A>SA$Xu95Vag%BF+Gtz@hT*qwl)u1Al9bMdNj zqpvLj9ca66^R2G3$-P1Kw?{V~a6Ou^O1&vYc$u)xpWfM-G-2`4SeR!UtRl#nkk_a- zS_%s?a)OP$JOxBBlWOA;TPFi%{Ipm6Tw3OB zDpffSZAhdrd}ZBNfkG6=#RB9`&-{|ON`4noh`PQrdg$zFL`ttfX3)6%j9a5Sd+M(P zhIMhhTp=~{!rhSz#*3APmA-#@{Xzk%UC@{w-V)_#qhosWfiaZqVy0xJffYG~M;d#$ zm;hWCGq4vKP2hPm$>A@bsX+Km1Ec(K>1t++cA#A8FYxmBoiH)_UL!)q{@NR1cFJ2; zrd?JYlW5)XjnDA&u&6t?9Z<*F#KEUrBld%boBUUZ0sH$hoHZM9EBpErK_76hORb>5 zePtMi%ek;(6(F#>^@qj6b2wc*X&NZI^hUk2%jb40e(R>W?6wqsy=%%Dr7i5+#yC9o zy*&z1F#m&Lkq_q`C`D|ZL^}XV)-JW}K5L&eF(i`328n}tg_O9h=WWW1*{oR!ugpM4 ze)(MbdZT=7w1Prapv*Pb^Fa#^(yfub9%1rDF_EZCfu%uCnTs55ch8)Az`hXfkE@@y zJZz}f?m9Jp#l_Ga27PiLD1jf#>TPG;_xFmPNc_XPRhTfdQ}=dVZ^L_hI;Bct%-nkf zC1Ivy5-kyFjw^c@Ttw6MRlz*6$V_lXEUR%94*mDLYEt9C&;*H_aN%?#>ye=oqV91{5!$_B!{N!1d^|M@&#y*GoZF3q z7yC`sQp8UQ(#_lPa(dz;kTfa#!)BBTz?V(MmCn;2CWrSAtxW=IrssX&td+Quh59HU zlm-&S7y7{klgSX`@S$1?{|U9rJCasg1XJo3e}4{k77|=n!+E9&(f#uDHdI&Kie=4+ zv0?3mBGWFhDwhhQ_t?qIwC(HE+xr&i_H7y zXxsvZG*0DIjwk;Kk(!8IuKj*&Q|GTVWL8iNx|d1dJUF*VdK9VvEjW+AqLTT?#bfx=sEl#{kA|#a?U`PKIV4UXmgszC?fwirr54WCV*QO-UH{iG zk*OuZ2>?BrKJj z?{Wu*yQ035u7Nj3L00Ac#Va|UnpUe&rh&su+{7p6g#NjModHq-12+weN2GN`AE*a{%J3lgcJ18?AEig8GR{vkcQE>Ab6^f8Fb`}a-k(B!foFssQcw@DLy6yF7S)?Rn% zK}X7b#4=kA@=W8dn5SDyKuJZJD$_`8Yr9a8GSRYTJ6DqTFuXOY<-K^rLt{3XV}DW< zHk}W{+t~r{LFedQ7s%jT@kLa<_#5ztZ|)5lrEws?zD`Re+(#$HhjInXS(EL{5lH@8 z1m^a$mTKXDqr$?yn>j1<16xM02I^0kmpgw{mzv$($;f@+^!!K;$^daVz}jZw9$O)> zeKa)p*EVe-k1u*SUME{U59|mhHXybOn`e52w$JjlK7)ds9H`X}bAg zbdV}o>&!}e{R9=i4T7wk$cR3>m96uH7Z$&BxstxQp0&D4^1FZEp6}qjDP6dlaS0@c zkQ;s+vezdx{=t0SR&X1o7`sCX9dZu}<@>nOFOTNFbmcAkdxZ==UOe8=AOZgLvRLgrIvXwcG5E&#NP5I}xI}9PDZoEzW(N<|e1&}~J$z*7 z0v$Kq{yPOcd2tyCoiYb~EAi?Bq{pSy3?67Uk`6V?JDaC{A5kfi@m5#d2kE+HJxnq@ z39Mbxi1ZvZ`gbhXK1F2Na{@tF1%hubLy1NF7s*njQobG18b2w{bT373MrB-bgr#2gcAZ)J!FeBcWz8Ea+jqZ-V})Y#^~@#p4=h)s?XsL^D?P0j!IP$K3INl8#l1UC~0 zAy{=%{*vy(+SLWXx(}?5NYIKk6k8~|I;HBu8M;5$g*AJRJrVqFCk*~^V#u>I#F@9I zJKk+sWaf8%Wj1KP)!~k^}zVlKMo=Zn;ehyUawilD8hnUyT2v=|9Ucd*u zb28@tvBe{bH6^5MhWF_T;DCx{F;G#}W3_|^P3Mu%#wOEwYRyI%-pi2+c-+pGJIulf z)DIjKA>>I5pX+wW?oMij z|7RhM-t}@tXy{5c37!B_Qj0PeIVXej)d&}8c$f>8zku<>vxbP(vDPYf=MuFNBRaH* zf`%P^blrZ#Q5UbZ$$cRoM*+%6!ogr2XIC^A>$~+dUI*nhNc*UW)YHw{7kdu_In1px zZhaC`>8`9{wZWlMHS;s*TETbjwb6w+YjQ-B*)E$c*fk<@=4&W*5HaT7G`AJ)Nim~M zp%d14zLaQh1ko1Ch@yshQw-D(v7{iGwx4y$wRq|Xa zW;vMA$^;&|L+q`l&~kkEIf~eSW=R76=!^Yj)oEvS=hN4|3erK!6E35()56Vdux9@L z#B76J!1ghG=u>h3e7Bf|)=>I5S72%->Lt2&6%de38WchFJz{^_DVL9U{nc6hlmwe^ zu@+k*_>6`Tr?1aOxDNr=?qfGb-|DQ;(U%T2nMY05^wde%72ox3AsU{mD3%R8v&=_6hz%o|8%I zl){uL>@(Pam=Zcg)+yGlFyFRBtV2TXd|T$bkNc2yWmooW(5x5aj}z)6%4{$leNJsZ z(7FdH&8x*FGX9mI3XI=mbFRb*>QMMZrh@)zL&PCg(G^WwFYmtzIQfeW!fSx|2HeKJ(F`1lB(Kl{A1^Kzx5lqg>$ew@ZD3hx31hlXtK}q*a@WpACYFV4@Si@0*(ijv^=53$Byo5;Cy@i7*0gD z?h|2sd2aQP;uXxD{RT>`T9(-mp%2&nCAc-Q@!`Y0O}_|_q;XZfW~o|V*l&O$>C`KZ zOSG+J8uQ5pjYWWf^@(##>aP_3>D@nI69B@p4F6%GDc`dgcoI zNNsw9k=uO~!>_-G^{(o%FSe&7G~LgZSQ0~M&=DM$w#F=uRAvZcIvJ1d*HyVuY8mCH zO4dw>m2>nf8JW{}1H99#&NtROffBKztH->Tw|w3$odgJK7$)3@1V>;TZdMbbxSQy^ z7}9C|T~m$Z%^5P??-cCaI79cdz$jcaZen^$hgAVBcPziF(x9aR6KLoRcXhZrhsf=| z-&8LqQ%s59so3R1@c%h86fo*gVgH4m{O_8r@a|tC>fT$R5aWN0m@5G>YnXi0BjJNgS%`Fen;7mL-5`dSBp zk~K4XV?zJYcl~RjOO6u)V-zl<3n-Zu1Y;YIaxqgw)uZa!>}Iuy4&1AWfM#=keQ@9u zf=L-w-?QsTAeDiH0_*q6WCnVTU)dhV_%a4|9XS^2--t@)vr3Ew6r7B?ybQ=!;ze29 zlF?#g((C(rBE8Eahm%%Frtf2tz!Bu@rLP|w|9l!D8t!dntmXd4T_S+FB6`s6kCs@u z^NYJ!5gEz(d!SJn#p^i_&dAi}$8$oYR~{FMy5U{(T;gNW%hGz}p^(zC!I&2Br@_t* zk$sf8PY^A3m$Y%DR$|8bM6%}bO3a7$tx2nd$gy#0QsAcINXm(OUIRGLl-x^$wX%uR z7oN91wUPVzGzDuJR#2oC^&qQPAfw#?HCWAKw5mO3Ge5qkZJfPCi9t zeAda{`tvsS+j-2=>6OWL$vZ<&*w1M1KYx>9l0FY)DnUZHxfUDpZ(4|kmcKu0Y7CA5 z?xiSzJe$5WXqpd;j^e4EhhW1o-y3x#5q$w7VtDSOla$-_#rz?DVMlabw#hg;CnDuQ zh=)e%qE?(mmj}UnJrm~N8ARwsTwXV#ytWtEIN)+$AFjlC$*nI!R|M-DZLcQsJjQ2L0uCJa+k~ z4tB{c%){CM1Q6r~KZkYH(+q1iLcaom!I!@e#%AWEwA@8qZK{PTZp^j{=3Bh%iB6H2 z4(kkV4!>1*Edzu)cz0?#s3|z<9C#YZZ*gVU70GOu5lRg0F0;*0+-Z&%G9NyO9ewFmtN1+N<|y_DgRr_~Ft(*4tN+ z5{yVlhzTr^6nR8tT|)Rx+U)?pqo1&S%Uem&3z#Sjpq*iPU@2`-JPUz59VqX-{t3nN zhSgpFjTzb>y)DR7@$HsGC#W}D3~7Dys9$ca1O?s1px1;R&%VIZIUq_(CtkjS5SpMz zq&VHbZ0iYHn);AhRL0Zro94ES6xMCde&@@i0^1u_I%w|a>H3FPK+YRyxgnn~v-ZtR z+nj822&kLI)_1QaP=Z^PijQpuf3BG_WLVNj3X6AXrt29@RU=v@!}Z_1UUEwLIa%YA zz{5KUSBGa*62x|OI(x)m{5RT1pBP-#iV`0#&?4*YUHO3`Q!g}WZiZun4{7WJUj zZDIW?JC*n_-Pp%vt_?EYnKV}Wa1}N;Wr=+6z>@H;`IP{!zXC8 zOj9YB9tTl|a;zsv{=^nPMLp&fa1!m!Su8ZH%eng|1vSv%nH$p~-%RDd$>fXq&YmI; zbZL*!#_zpG@qK4O98CCT)BVYfQgDX*hC+Zkmf>!_kXrrFK))pw9W9e-e|K=$7(yW+ zf3%2s8DWbsDM)ydYV{HN#XD2WZnhE*Zk*MW(-iTSyGg;7q;@csZ4KP6ES=oAXA@j< zC{l2C*@X$O@k8L!;EtyOb8nX=E#{ZL^F;El7vO#8TGrg4D^gNor*vOzWp}XNlrI;k zP$pPyS2UpSfg?}jjBQ=25ubgHoxlxl*J%pKg%_VnIpJ$b5dP{hsZ>v;SRnj_P5&HQ z%^qj4sm&(;UAA7s7RnvOee8~ff>s;hUI^=KMqa zp}S7V_lurM+46+WTb`kJo#3T}AbT2Ts&CGQ=Y4TzL!Xl;p&PwAE6nInov1b~~&tUp!+um)sG1@zZiq5}HI9YB}+tTY2PC*!I1ng`fqcum*Nc5UVHI zx#vld01chy=6*v?O+IVc5jkyhFRrep=HD- ziFtCBL%cz+G=?A)3V);GX+ro+_D+zk+o<#J)J8*elu>X|4-oj$@dMYm7hI5__TlC8 z^h*talx(0=vtnSpPsyTm%9A2~h^kXk;`$TUxVH@zY^clkYY;wa$zh7WBAxK{L zuHHG1wH(HXT?oq|)7gyX*8`O%sX>l5t-L1WN8#hzO-~iLn2dBMjOZGXl{XrW;v4*R ziN=4RGi+ig(9Y-P~__r946oH!uTmin~#b%{pN(v_qt z?wSF$y7W{+ua{3zSY0)Et?J6;n2q!kNW)NesVUp#vGbHUlOd`zj>fN5vcBN>!NCZ9 z`j}!FWk3Dqrih-U?wIWmBxpBa%)W*t@#5?_SGZLhoyWFq3mS@F-f927aa_p<8GM;- zJFNA3cjIwD**13>-?#MRENX5zdmg-CK0Blnrk)pT`F#-my0>634!($oSv?3G7uk*9 zJuJ~}u|=;${pn{p0rmUG93&_ix{@2d&i(31W=aif$EDySJAjLIffbqEx*qd?wmy{q z<6t!39`RK#Z($e1`m9ZW6~4Oc)K)ji{2BUCZdHc`NW*(_hXrL5UG_CvNxh-n%SV7XEX*pMdCO$! z)o&l^Q#*DBSxZ6A*<#y!CPzda1kj0kn5r1oT6irjR(#bn3{=@<8r_59KLD5ThH=3c_N7I?Y5Z;0%w$MhuB}!O^B)}f_ck^j_1?E!vG)?fRU5}&FH3`2=L6L(M?+oQG=Fb zQBE=;xqpA$(me$zON#mQ@^yAREWFNSW_x%w=$$U=5zAKPQzWjpSg5v&kUUn|9KgMg zw{EMDe=&)PyAw#B(wa)N`#Y#Pp}yutj-q%pPWtxMWvrFXVl~F&ZP?dYHE&r_@}h&r{X(Z_mWKjzB>V7HFj=@%)5V zCrt$Goxk6W7hPf34K@$YnaV>=Ie$i*%?FzS|MudDAK`;f-wLfX%@0^ajijlPA9Kjp zHtyAca;qU!$K94I;D992i@d{YJN~5K7`5TKvC-$5uTusE4{P4VBv81vMgjx{U${o| zzC+(6ZRP2jn+DuG*{I}I_XMt(W)+)1L#X202$~NKW8Fm{(diZmrHJB_2C$z239I|( znsi~(Nii7g{nIZ|2TA8Nh-2en*M+_AU6l3OT_cK98en}!otATrhWV;;bxru@;Gswh zuK{vpMOg|6;-K60VK-T!HwAta0W+%1h;kGb5Po(Z^P8V~@!4HPNnF4^qyDVI&yKwqscoi5pxPX_jX7K2E3e%fK)>NaT^T2Q9Bfrzqppl z^W3yKGdBD_2BopqqYKfxypU9)S8pwy^~3 zcS-%U>+Gw}xNx7yE?NDM@dKRASK)!9RH;h2F=s+q-(^urSjEKmm_my+5827>fu9v2 z9B-B+duMegc)lgu&q1P6R8aKD4@)1ix%aeLF>U;G5dM_%SJr$QcIp950eY(oeutI~ zyYIKD6BL&1EN_gc{bhEI7ovbkakbIWm0}LNA9lGtJtV9u^Qto1glKRkM3Hk0Lt?F8 z*{uj%l>~uu$OK%pYetyM=UQ9a69oQ+-EDQn&wM@!V}om&uepvAe#MvE|MJyGM|^22 zppfV3?|+ALa32?cA_M=D^Ef!Pwl6TJG<-iv742j8@$xq-EfwhK+7B&x%AH!Dw~I>m zgWgg0HWi*+ux!m_%%rKa;V5U|y=b6VW9dSjT4h4ul=aASp0HbI8}u#oSd*E<>sVxV zBWYxL9AMtsJ*c(nzAJxn44EaWZg%8BfE(M~V6Fa9=U_4Yi2tt^-A<7^_XSEq1cIE( zM}{OjPCimjxXf-|kBoZNyZ*sDFP{~cUH5!mF%AW4+ z3brq4q^Lr=OsnbA9nV?9xk-h^AJ+&?ec0Z<(D$d^y9R8)yv6>!!?1kSg{`jGJS=>D z2^{RQ@K0lm1$X3!J1@<56N|2f1WfAMVS<0fI0)5_7`5yEV^=Rlx+LsqV?P-nQ$MTY zHjn!i;bHD;lh{!8zyrJgu)a5?krVISKwi4^E<>p(T9nxXa2i3$R0GmkURW94+H3h>5S|Fjz>>EYIFn%sb0#5%2uSbIM*`T) zXSFNIx95*~h_`Pa)lYjWs>zkzGTD$@bA+&^HJfx6HZHR_B(1*R*TCk-epdrp)Zb^U zFW3=-cZLYP#s}953zp?E8(6G+cb`^|2d^+8OSIZYEUHc1b6!VX_L62Sz*W1nvP~2Q ziw%$MZwt;ez9_W0%l*AQJHJJg*y-;B&v?s%@xMn5F5q0TB(VsB)5<{q&I4F2v1|b% zfu|mZ&4y6_0PdKz=1VeL5-m-WfqBQ}80?b^n=}P45+;fz|2&Kegi@$kn*&YFF^Q>G zlhGD8F2#!_D7w6({flT=rM57*zak=;GtW;$Zd?m%&|_a4`*UUgze@CVM;Bl6{=^QB zeqSajpxPKaTejol)Q;g3ON>ogr4O8V_OJiM0Z!vzNFq|tx#^a6rTDL(c5lTH2MQ@L z4SXd%t*0yH0Fbj{5$%+lTEW5;sQ|+xDQAqRedP zyBwoNXq|ohg^*cu&o5E2`UWR1QoNP=K+$r|k1x3EtNSd+9y^g1W8k_tYt8yE0J{KkBX$)>#-t+Avq zCSs=H=Wjbgfu};)%*UenfW%tZr85g&LNG6EweERfhM6VIJq8E+wZaIMQdIV1Ci+NE zulhs#*S}qXdh8gKxD`h!4fGlh+m96~qn`tBE@JASzA~x8XyHtgHtR62G2mz2VPg9t z$1yP2o$w-mb22b~cgI|`^WO6X75qz^p%yJuTn5= zml2~ZK=pT58H@QNb{6y#ZbMWB^L0htF1>?QjapkZJ7o_Y6fbOn0YV+?8nqL3m??EW z)Eihq*r&vW=7Dhst2PqE==Ree=n{u$P29p=+=64Qq!Nfb^AvIQn}cul<&(Txwe_56 zW91Uqg(Hr>e?AJ|T=$-E!JHGDHU~(;3%3Jeg-tjX?@}sZgX@vfX%YLuBZ*}b558X< zOk!k>oc6W$Nz}aev{NcrXS@+ZA3G?%X=$KTqP`BKLMW~$_Z7fB0(q=;gX) z+{o@=o{dAW13W^{BE&DIw6AT=xrycSPh9v?LcXeciK znbwE%pQ^}Aa9rbbr_6lh7d4CRbzY~=Z-aHa`1pcei4<<|LCn|CtDOP-XYXRmMC1;$ z`Iff-0dlSAew}{eq$krBwKMAbWx;gBZZLThS#ElA@npeLVNvmq^{f=T*qD{g*u#Wo zM2gd2`_8T9zh}J%4>$?_AIEijnRgnlNwE)9exI3b7X)q^ZAW?unv(DrYc8mX zHrcCzaHMVLkj|1dGQ!SnL>?%rLs@w5hO=#ooR@sBJ%y9I)p z43R=Q@+H0QUu-WHO{4_;<7*G%&c8x0Nf5lwf~4(NOu05Y$-;T*i~j!(eb1l$)_=&r zcnPG^iJcOkLa!@lj{S(kYI>$71K4V5t>PuPqu$c0P^1;|#UiyyFdTAnVyxY@x!&zv zz{orm;$j)%j4%J`d5~t+;D2UA`;=Js1nS|6%cX}$KhJkXdfyw>?!=6JL5mEpW|eCV zd9~{#x}N@X;F{AVCwhz1}K5VJUj9MTx$}oh+lz0NkUmu|?1wX*-4=J^jxbHoeuV z6fa-jf4JGNdhfGo00j9cyP>)D^<`^}G2z9>jBaQ6{L~t!^7 zHdfth&b&DveC7i5fXpqy>!`ZA)1XJkjs6Vxy|J>Pe;CtTMaHT^y|8{3Jy|ogp?lER z1lT^;Q0L$~!*a&auC(UBgcfq2+^@wO9h(GTWo%x5e4pVG)mSq6vylGiU5l~9QZ|EL zE00fbZ|%*k6dG!p-}w~>P68aVXVfN7Xz%_B^O=OV*r}~rR5!mtsi#(u-1Ly(*2m7yw>JkAg+rfuO|mp5^uV5k=q}?(fBN>^FApcIy;3w+!rPdx{L zCCP#@>oLZ;GXdH%DB*iJtRYXON*%deS=;JE*1_TZH-{C`IpiVc*4N5SA8E}Fg`Wz- z6xla}K_Zaft5rF78J7c#1Ca+zlG?u6U2jR_A3o96`2(KB<7rAKuAgpdOgNdv(!*L%ry}^c- zY~<=eF*{!QJ2&yN%u;X-=g^AKjo>N0`?h{FQQwxX$?f_Qo^m^@~&qOm}su)Eh;DCc-ywxcP825iQK3~reFU04iFyzO!o*dsg9{-ng(Vf-Wd zCMPF;!z`sp`J$)n3}}D4L>0CU3E|3pQ ziHsh6(}{P@zbEkDNY~dtR8gmidej?*g*lxjXG^2b{umy53+>$o7&m?B0;fUSreWsl z^>^H`6{71lmDECa^0U16kXcP}`&Ag^=(Crt-xZ$Dv;n0dDHg@Cy})$*o+hH3Z{c8s zISya%g0Qu2GpFmpO>?4oWX%1xW55BnL8 zG#O`{PHZHOSbz8XBNFH9xDR||tVFNy3eGV~<4!0sl&$M_c}S~UYubdd^|QlLV1IVV ztbf}Jl$3!48FXHmN`y{_FFP0U%iX*_1-&CwLiU2oS|UoSd%fnnTG|7$C)AyeVD33L zY157e9L36){bGtvdn04>ziot|5l;rQ-!#FfeeFLcJ1qRcP=$8JemGA_gJz_UjA$}{NEtH1aO*#-%uPWDRkEF`gZM3?)Sv_V` z_t)hZ(h5hcuev#VuyCK#T45JN^V`{7NvX2_>$s0`H41fwo{RNf2g@s=$gm%AY9^V?|j`!pj zUwIIeR(@#AKfgG)x;Tg<^|6MTylX=F@=w4+An`(A$D8u8Koy|xvhy+o;Y=~!a0TIx zNwFuVMhp8&at3m|a~P8t%c1$(Vqq1R(*T_Tl%~<8yp=G<#sgxojH7V}t&aVQSW$75 zd6>Yz*6Kuq{q>V?SHs@hIDaw6*WDoSEn%t4sa6lEE-w5VQ)JQnrZ=$Iy zK61B!C9w0%n*Y5W3$j9~zg)I=pU}Cltxa!?S^dWmpX&Y^>z*-xvi+eMI)AEi;qS(t z4RNaA!zcFt;9ll|joc2%P8)F|QjH@}TD(Kyu2r;`elD%i0cbNdcRR#0JI=@YO`ScH z7j-g@(%dtBpf}QD)<*b6ai(B>8Nzz9a;u`2N`N4>h5*rd4W#^MjasnujW~I3ifX@_ z@%R0k$eK8O)Qr3OP%YmQNjT+GuqCKI-gN@&QPk|?Lbgo%;d;AQ9NjPo9p&7(Ov(dWaC1I$}qr10T7oajsoP3(r^43adO(5BLD6Ja( zfi?A2=0IG}beX5t@ra3MwdJlYN>JHUw8+9(gk1nrQ4yMzDGU+c4U_0IP3nnYS;Bq3 zWfHH}v?9}b4jdR84P;H5q%M8WhVOj)BCHC!4*QV5B;vqd#l5~;!w`w}wu&eA+=gcE z42VP=5_w5}DznnDx^FP5+xt|Zxz_yiu(a1=e7F`kQ8Xj!gX(Dz<>ATvNwBNi=?-F^ zKqo|hV7FQvdb@rll%tqr+_>}2M{^(g;yIWlZ5HG;d+AHhFE6lMQ?|ZAouR+?bxFck zSajq=1DMJhh;B97P;NJSDp=DJ#C!c}d|XXoLrL51_|6f^?$wQL2-+dMX;%1#<_ZB^ zFhc6 z*?t>=!sF~p8SD3odA;R=aZ_u%JC$5mq+1qWYHPwu>h4WBr3(73gmSyltiIK2BJLu^ z?Mx1FY5q!$e?gWJINk18FrA1w5ahTA<-(xuSF)CAinl*yLvB%fMY#s~>WoUX;E348b1E{DJ5m!5ST!CV z(;g`MzB72aq@8|NupQ)%8q5_{Hk}9*6=W^Dtf+U=*W@Z#PyU|7q?y^)%Au?AVG5pb z3Mj2IDYP(ey(gBrn9(w5WAd#)=u^cQQz@@@Y{T_ z;YA}~Whpy1yo()n~Hfcuqy>4PW*erqpeA zN!<3)f!M`2cPQ_o2LkZElL>ODCO%F7nrbh=di(t2(}5?Q{Bv4LUZ~OVdd~%1$u2e5 zjVy$-gT|)zwe7LKJmc{bqq(5%Mc`uJ7IBU#Dd$(jh$`W3_uX*GXwXkf1%kW0Pe=>!Z4E5HB5DyRkHSol%5MhOleR;8T;N4&5rxVf(g2gFEYW)1!*i z;dWo`KjJrveGhA{JDiDr-UiDTEXy}d)<3R@EDUO zpe4Ye@(}Wwa&hUY%d^w&&Dl&B8R&GULs!l8TXlfAP9I0fnh43%6f76_KWF(n%OX%- zGdip@oSoX5dZ!;iE;C)DK!xVOTGJepezEJxlM=a3LTt2mFn!P5Uq zWtgollB!$)db7f^)xTEmi{1)S?xLiZe|{&n)K+Lc^$_n$D=)b!_D>^%4$ z1|&6PdA4DIZ`2))kIw{6-2EKB;066lqnhVK{(;HRar-t+GN0Rrlp-%=4 zG588Ukj{=S69J26m?EvrvB&LODxOut5Jgn*a-A^IAOI@+Np2cN+Sv?OU=;!R!6nS_ z@7vL;#$DB-p$zVJ+kf;OwL&;Q(ShGVMcHJq|7Qe7x$(tBjh{nHoW&EKd8r0l8rAok z;rp&TZ9ck0!>t|{($yIuG2Ww#e{WXhAD)-kE_7`a69iJhaKY?d(HG`lC zK>@_xlk@M6JUTi`Iu94!L6g3p@4zlNOsp4!|EZ(%7Ajgl`N226+#R2Sc@+x`$j*}$ zY~b0P?TVjxM3H8eS-;M$w2Z8W7`G*cYYjse=S#0XL__j`rp4|71){$LDBnVF4l?SL z-c&jp-a#~u1~l-vNv5WAn^PW#vMBU`uN!k_BSKx?i;1D48Po$ZTR)@S$Mq$|I`2Fq z^u$(0;6>+CV%dMImu_xa5r1Z~qBzI&O85(d*5c0OxfzDylLD8XoYsiiZ9Ucj1&TSCH+R3E~~jrgk}y4YJwx}qnc?G*LMrAaEDO-QOnS|9N7 z;0f|UR@q08OqqH%SLj6}qxy)pKyiO_#}UhN8Y4Z10%G*s;y}mogPO6eT{LM@rpTBQ zKKa;6uoVuCbqv?fppgy`GFe}MfsN#+c}1`k(KW&Ml9_c3{_KU~muXPjQku8Tv&I>{ zS!1_{YJtFBRD0}qU2#d|%(C*3Q7Cb9sJu+>4!u#t~2 zA^vmY;P0MY2a}Ae3iz&c;m>oP2IE?gxG`OWmQ>1Ba8-v-I7$8yg?XHr!rBwCv{w(} z#q~EMKjePt{#O02awbi;uGO=wo(q%yD^DinEB-&8&c&bUKm7ak^(~cHrAWh)N)pQ< za$J#Ajz!Kar{wFD^I-=`sGLL6Vj+ieKA*QF=a};_Eap6?89UkL-tTwckNbake?IT) zeO<5X`E(`ujvc;4>J>F&J&(;`tAb3!dS{jDPPM&ZXQB6!6#_{e1&{oE(@u&fR#c7{ z9|cff_-b@-c}lB3-quiiF3$FEPzFmE$a`I%fFDLlsA9cdKfTJA&Z%hL;NhZSe3l5_ zf58`jI&Bi#atLMWc;SaYD^Bnf@aDXi>UUGJ*GR{GD~zQdB@N#hBY1_E6(d_hUn=Bo zEt&#{LVm%I^zNC>8+@lkFy;C z`?gZ=r+vE5?G7DeI3y<+{y1HY(|T;G)2Qq`QyOz1dQ zo45XZo#kw&sfzJqUA7AruD)o+>!shOdqfS%qw=hp1V_qs#3L@SkY(P#xBCCiL&ow)^35f%mJV#t49fMY>8F`}S%1iAiRY3ZG{nN`qK@MdkxdO%rY zI0EmoqiM9dm``^5@QO7#>Y#O9FIjK3to-(*|0{#gp29A`-N|7agHs`iwb8tF7Hx8d zx|Y&{TUX<0@aWN%4D-wgb(dbSCD5jjv+_cXnohBjkg%MAez-^k98U4P7!@H6KKCc5 zGy2!3&keOEqjcq2lP`XuNgXF!yH1BH3jGOUZNHE6O*GahTf-ZRa>qvZ9l+bKN;)>9 z*Dom*YKb9IuuT2U!SQ0VPJzN1{&<8KU*YClYwu}vdwx4MC@E1?tgr<>>5_$f?&#c{ z8+`hz!&JJ$idE#(*$brT^0}bx)GZ$YJpM|$EE3bsA&@Q1B4L@BxaH@~OK^($}8FzZCpXoPqgGjE>Go z7E0D=c#?*#TEhw1nRs2(*TNT`6Z9x4vgze8whDeAnrG8gw)~NT?|Bb$o(!A8Q!BD=JbJ5!Nx|9hlxhl0=u^ zPEX9%%e@-v^tP^W`%SoJO%3z`x!qwaFne`6whDKX+EqUOWoLF}?GBHt;z5Md(+iEW43}jM0PxJDIxdz=kf-e|o|$A6XcKq|FH!gPs!MnyPoo zBITRPNv1%8CTr*dp4(sZaQvcPbkL{vkze93FG$u#R*uXa#T}x3-1!GS*5g+6NYEZ% zH-zXdS*Xd`*Yr`?R17Ps65t2uWMQt}7{1CdSsKAlyi1sz0-Uj^FZyTy3RhZAc5CRo-2dBYy`=?;X-C!qk$I(sLN)sOY-&5@?DmLuFkd5$ zt`)-6<<4%sS+n~4eWMwD9;t~~586B)GZRE~uawWjeDoJRHE9PQtMznGz&tpzYw-goOu3JH;!(09;H*msqIj2@rqZJcOU>nw1+dh*@SE;33 z)tsPY|4{Xb+K5BX1^-K|xY~cgrzog*;i!18{0F?4lm%wF>n|$-qlNn&VVLt%s%CYpb7Q6)l9jS3K4hX@8|n!<``M1^ zxTe`scQA&n$jAA3;t`g`8_Z01X^l8s(s>J(?*20EF53s)x+$>NKQ}=EdFGbnd_<0G zqZ7HH#7*FsCeAvw@J0gv|NMoai3hFcfn{tOVE`LiIog;b^q>_h?pWk49f97c0%+C% z%uwVi6R2M{xN7!OGM02}jeS9G>o}xg|M?5h*SfCNFsFly&qjN!V=6(@%rhG3e>ZgN z+^=l7ug`cA9)?pn3zLu!UCp2>GacL&s|(ExMeB$L{@0ED=V4nZ#@5@d@t%FO>=E|F zoe5R>?R>d|dFjrNh*zJ5q;$a*p8FpfkJirAWB_z8Zt%Nq3y00~wxl@9d*`S2Zkhyw z@1SyNvVLRl`WZ``zRBT^c(t-{IV;3nN7`_NC(w^6s}nk+02lh6I-jAweOO{{j%63mzDfMR1VBr)s?VimcWQ3LkM z3YqzxxAWVOU3V^+`}uE!S?3ufT+sjb=WZSB|6jbI#`lWQYv)8xi(JqZ zZp%IyjwOX?%=T_Xzui?Rxbfkex5NM}O#V>7M98Lm4DG0&>f-jaf1#1&uNZM#Jp|+N zX#0=lO7;{h*BS4Y7xWe9*u9})xiwCylgSQ%2HsuUep=}rvG&SWG(A1(y*ul=`ON!L zkh(}obPdOOY1D}vl3Nw-pn@z!Pf$ul8w)`V$hr#4*zvrLilG+9c%44`#jqNdAxs!{ zdfjh=F$b@kNs6RN_yel;{!Wy3U7mvXw24xZR)Fc>jmT78>^SN4x!?7%|L)&CTtJ?- z__n^#6SZGMpA~;)uW(rPoEL54T1DpHij4lRzcN-^l7AKgKg?KEiaC*v%mDvuZ&o@p z5Mk%ee-aDK#@cf9fIzN82f7sJ^^g-X%)u)z7FwG1)FJ|s# z`JCkc26m@9>}fJ=%8@{^f=fJr`xcQbf~vlUCmE+M9g$aZTsXv#YQHPz#OhtZPU!DR z{w)@AG@%~~S_5+*@u6I%o+3{07qooCf1EsolCcQCyROvY^6zw()o$j{V4Nc2!Pw2s^Oh6$`eU|!yw&^1@$d1~kLu({e5{dXKOTtjo)_HbWR*0R z(53cYlwGHN*T)y@lg58u>x|{RMECbmL@@zA>va8v@+A%ENa*F62?Srvm`h96DOB7T zUjTI;)1dUBZ??IKfhGlXE*`IqT?zRlLwdN|GsDMN@nr?&^mAkp+@0~mYwz|?f7aSq z{;=|H*BH7q3BKn4(*2Eq?*q_qSMEWFKzEkHc|!1w(8t6SZ{#VT?h(q$7d_K&lID^r zc=Zx9ntZc8JdHCV1?4v9I`e^VZ<`&o%AV-W4tIq&uZY=hjgx+_^5fQ+ZG&B@%Y4ij zns`H3Q+XFYLI=ZHWWeV9a~=N_J1*^#$;`uTqqVA+h|{(Bi>zYsGnl%Y_)F`loYh|f zr>JW>4KM6B<5B|FOWJ1El2)-+;-O;P5_F$*gd^N2`(p}D>gs5t-3Pze^+c3ir}AZ7 zS{E+vC|Kh6t&QmE9Lr^gnylz zJFX)Q;oot?IK%TUgN{UZjO3%4B(_(|?wnQg;M3-d)O(ICfOBv)a(`clGRV>G`-O&K(P$CU-r3~X=jz zQ@bC-{4=!-NVj9Hj@VF5t#@C37G2y+aIUvomfBg6veOwX|JU$ zuDg|SEJTxc=&L9N6H1P|`H?!aQETe1%5*rnbNc{z-}ccbV2S(W2A~kZO)iJ(R^Zr* zqqmI5!$(AfKCIU+CR}eg+uz5Pa@${g>DFeuMwLBi+^_VV3SW)CWmOP-8v9A>nst>* zO63S3(=bvv&T**xX-a|;FjB7Vfpxish?W0brRjG zzGH=SmQ-w5fsppfd3(1x9w1vIaSIXCePK$Y7zC=4*`_8vFOggmEs~-xyn(Sx0M%|> zr9rb4FDlWp7$eM+irR=>a(Pq1KU6E$kcL@wcgTo$!F`wcFCr?DE-$@>c;LL*Txj#G z1D%VHj@N!sRu9%?Q;whOjERpnjM2Uhds?N=rf?YM`zjn(`%o+NwZYCwkm_Xc@v}+!FzlN9q=Rtdx*B@!WmXyq|F#?|MZnxh#i)Zr)4*(^s=42r12B! zyB+O!YxnsDM0eFOL`=rIs2<+N*WM#!dg62@-+&&>^|H_Kj?3p96!GjplFH-(@AFxj zDUn8Bu1mbwxS?61OHjBsKXlEB(+5v(ee+H0=2+kC2RJuyAEq~_ZWe+YEF!&fj~D+> zLTS8`C@Gz$ji1J@ifPtgw|v{Y(m5YOx(>VK*Svm_eL={ZxMKUjI<#y_ZoH6vzj>D2 zUCbUJDxwa8N?YB|?u?m+aO{0hi`k_yHQ#f>1aV=(YtX1`EOwF6Qqp%lw`bZz0N~q( zWXev{Ea~F5@Ne*X-tOeT72bB1zCkX^oL%blLdo&-R-;t`$W29{w_uHC!Dt-OH@Pb#D`72 z=y^MQhk`1P>>e>$#g%E@I=0;tdtC&(ZuosU9kepViNu6_7rYJ)b}q2iKUgKhbMx% zlRm{P*QpK0;cW3+w1p`11)r^6e%#iQHj+nXF+MML39QZ-KggCC$488Y)!Qw-$zd$( zd#e>8>nY&mnW}vzeG^nHxprA-)s#hWz@?lU7oPtO4*;JmKdc7Z=f4ZqZGiJYq_|yA_~@>%DGoBcWB<7LWs+>1YYOEO_cUHVdITY9s0R$^7J+i_w#NBgI=i z50BOa{DjPGemNf;`Vmpb@X7|ww+?lv*J^vM%e>P{cCfJ43OiO=w03x{K%nWdKe$Zt zp_++A)17lfjqE+Gp(8}jA^1YXGT%_IRX6bndRIXu;#hB5ZddKi8v`0e(1Vh9p}?JH zqcE?=P*L78iJ8PB;Ieij!>YU#^acK{{!eCbE?mtNb2h^EE!cP^vI!`EWVR6EX$OX; zA1_tOP`$Slw&LR%*2Ju`CSEKTIMDhRyqpHvs5sGNq{lZ zXa4z2_L;q_dZxQ@qnNHQ#l zLX>X|UX&C^JW2f3UI%{H3Tg4g+I1L)G=O8B)^96~QKsKGElala-Zww{=t-IPo2i$c zu>Syix{LxI6|U3ybm%GCosun9Q(pPk+JmA1AyM*(mDC!-VBpP8SmazZa>zK_RTMjX z??Y+^TNEqGU!;iMceu<8D>OTJ-S;r8t2(_P0Gkm%|@~d8L+C%jWcRHlhr7b zQn15z584lY?}%|k%*5aS>Mx;SR)o;V3XUm;)n?}fZgm;Sf7L%*!&zA}RgF-iCTXc{ zJEzJC5#cGLdjlbNExvE3Z{Z}A#Ow&K^gMCxG{*7wUZ-o#;)c#0J zsr#G`QY(%5KS9k?AX_&^q@V>Vm!*zUlfaHk^CZLy&=;+P>VZk_Vts=Pz~H1PA)a`u z@BaKZ;?u@633KyGvApxJS;%mK`((=9_>mgOIwn97O8*i;xx|0xsxRi0szv6m|EGV0 zvQxg9e&)YKJ`pEr&r1@GB*ls!0xBO&Dd<&h-TV8Hwj|0a5Ts2@AlDR~JP6;Z&Osu( z7aR75!p=KR4j-?x96&$EPRw|(^6q3Y;ZnEv<~kL=YDPGf&pDKBVci1GKzyNYfBLEe>>t1NkW5kH|}mw{AO(gBKL7Q!*p zE7o2}`FJ!7o9v(#L1&eOyLy54H*~VfZ@ov==rXs(dT;+c8X(wvuqXqV3OJ+<`CR!x zC+ss#p~AaMm`%49ZwxT4+g&sNhH)`+l4`W+g%PA&mU+&|ZYY`=9?BF@(H~!Kcj3+q z%23bI9v1iaCH*Ohy{f&VuVAz4k{Yr9nPQASRdqN^O)0w%s*_r;kt!Abu6ybVWmrav zGxrEp^oNr->n-y_(o=t<0ea){verFGiK`c7z(X=AEtidT9b-%gfs$uFlc^tGuIFJt zp#(e+ai>Kcy$fHPfW0d2)N6A7X*IF?SpE{P{poEDKIN3AsrA7$n_Zg%E|I6YkVDV6cSe%#X~ zAPjk|vTx;461BXT;n$T^m^gl2U+9F9rhv;+OO%av89{?l8aElMgW<86s!AUR8^P7ZdT%5Mo)z-&M+N1F?wQj zh3n&PUrU%%cfCAsL4{_aZnc)b+9Xz#jxFlUxq$y^kJz1VTIF3Xc8J8O&mSE{+PWn_bz5bd-fdEGz)V+i|sLc zi6uyx(*gs61k1Jd*P|5IRGq0``eO&QOWprMgxan7TqW&;V;EC|Go6OM1se48)jH>$ z%ggWri{GV7j;(j=^=5ezwCe3G8Wy@bMVppy0`Bv{8L>>g11)?O&s{g`9TD&`=qvVW zCxDvXU;KnU4XyUSU5%(;cUbcFw_#MSTEQ{gnJtaHO2mDb4c$P73h^Zij13RVdAstT*$y#~4S^6C*9Zc`GW|(r~0H+MF zTi3M3y(;fQAgNvhO|CCWk(+aOiuLyXIHfh3qDt8djuie?qTZy7_+1oek`;t}@cOEw zeVb!uy9qKGKlgx>SSA7bN9&vZJA<>`uP36)TPq0mRzyZ?mB5Ao&pjlKY2kGK(Z_PK z46L`AdPNJ^duGafUvtzuRdua^AWxXO=H}T=X3WD+f1W+7DZk<2ZjZTUAJ> zAfVofh_s*cYo>AvBx3SjL)f$Ou5!eeA#&UtJ*o%K{kehFIH_}?*LMCQS7Wp(H^H3! z5AWRM0aXbAN+TrTQDXe@^uD&Lr=5;(+|_iJx^U2RfPTHsi~B7xFopVa-^VfVrDX*})(CcQOQ3e)bZAwrVTb|K12 z_uGm%Fh#DLhtg*p&^s;wPtm$Dde*?@|71eY1;&k`k!V)C8Kfz1t7-n{eBZ7p^}?C;>mla3@&brb((hoH0+=)=$`8l?}5%iEB4jtR%KP zS`CI=)~fNnV?ksuhZ(+6hw548*760v&B;>S!+o3MRb^TeRnG|ZFS4%tW}HqUR{LMv zGLh)9O|I8S$7e0u(PN*{1&csl zKz;6LDJ`@!qKgvZsUGpvgt#6MmIINnOac{ZDe&zz5X5||cZ|bv1hnAg zg7b}*fpNWj!)XM-L&U`oiu}hH!Y;e{eL6B`zaT|2ySwRV9Z7ydW|px;YTtSJx_SSW zpSvDVRAeH8*m?v$CojF;ZrrEhjW=(AN=}0;(^`3m-bo4Fpfo6a@!b3cXwAtrj1@+~~<^i3&NlfI7Hjxhd$nJvAUNqC`7{u>Y9u zn5lU*4cWNs%{U&q>wD1Xx-wE2 z{}X?5){%U6p6U#XeN zrT*U1Ghd$;RDPU7d2pgBCw>$7`t0tAeHWq!%{WO)pI4)k z`hyscfcJCKvx76khtHX|pvt}^JP9s#e23bXrI8vOPsX3l2>{>^uN|jgv1#RrM~JmU z9;EFl3muQI+P?)->%s&#Rb#yTg@W|BrQCO2^U@BpKA*SPW|=N^1|VKHF|1N!*V*M1 zx^fF-!YPOTM5}lg?4uDguZ%Sf+@?|fOSQQiH@evM-b*b>9gHCMmHh!z_`Zcx>UW^A zkkuAM?}kEb&RC&_(dWuvpKkCdpb!gJ;d4Y;(xJkrE5??ijEapaGK#s<35JS%BX6fZ zC&jA;_z|_@bYkD{2+w*SrLza& z4tgiBC!wevtj=7Q@Xn0((IRZd303l5Ld*7fsC|W;Ss#iHtrr921I`7`tL?j)v!-)s%$*TDR+FmwMD3=~ zkQhZUmI-ZX(KskA;>YH>FHIhobFtQPG-*^1Uk`Rm!m#Cf`77HIUd3Cgn4Lj@Ouy8B zPLH8e{(Dl$lCAFnUdZ9?cix32@T(i)!*{T5s2NDZK!e$@H1&T=^;S-w;|bOj>BhHz zwln02;CLV88;dKhwK`RpW=6k0`@_H@^VsV@AU(oI8&i|FN=qk`ERPGgq*5i)+tga& zdryO?1$$AA4ci~qtzuG=?fk!gyrQOY>Sf@BhewoD*!uyyU%4E;2Je&am-5+-N4*P6 z{a4&WIj@iAPP8otaC)wJa_k~j4r@1kPAYv@u5*2~Cq}8z*apg$_3w5PziYjs=Kv+% zwHPncvmzKRYypUn^B<7)hBuFUBn<(2%@ zrSKc*2=G{by%Sif5f`~`QZPi7{(#25QuLffx*Logctt=A;(({(=k9V#DRTo^iF2>iJ5X|1sQc-)`w z@E5&-12X{+hHkr9<5AQt`QHbz4EQo$dvm7iE*z0LjDKA!yYqBh*6KGPrY=G~jh0|K zAgaUcx2YDH*rEkxVtypXYTSm?wCP?DZSQe%$2cP_+3$oox9(L`(8&K z5Rsp^qQ2`~dXxD9Lk`?I=GM9&z*MnijTdVPuy)0MypVL;@;qF}qtLKl+H~uT-_+mL zwy=;1r-pEf*-CiR1B0@*+bafbctDgXe2A6KUxQ;t4i%C_)$bqzD@-ZCcK+4gQoA`a zuG7WoG-5d$w`9r^s+C9^`v_aNP}550n@4c3EyOvPBA-zigWJwlE+(lxIPt)F-gYE^ zy^VmD4BsV%Nakt3{|;DCsTgKuN|F>6Frt^BiEQ;e|~RK_z%^_a>Qs z6vD%6=C#PoJ_)XEH^6`ng5O>&oYn7a>w(<;SwPnwq6V!8|{`F3C4OJ8UTs>|Kbv1;ig5HYF3a046zSjwvky;z5v>mcu}W`hZ42`4KtT0II*9uWU3xi+Cdh;3hwlJb0asVUn`+YUYd{A0XZ z#7(%{CZL1tS=pDoJS@6s;r!M#P~irnOmCERw-dH1W9;bZM;=Y&o08oR)0>MH!7_$( zepzp2nm5|**;`rRfb4$J85L2=e? zcr`!$@ME(?3}xmakml0b0_t<@`k2(m}JZv&T=(2&(ukt&B-K}&`-cNe>Y0G16=A zpMLmPg(Jk3PsbQUKzp$s5vVny?lR~RdnpJVE=N%GBByAaoP_m?7R8(gBAM1iZ6^Oa)<}vb|Edk1CEsgl z4t4=(e9BPW+;g$xT!WnWQq_!>k;`%b%|Op$iGbWKGY(0?)FD_qm)h;B>>)XnZygZ#b-Gri z-)iVrukYmfkc4;nU_Oba*wd|6VsMnB1&Fe?CH^*OQ)1CC*2@F9(EX!ZYCWW~T3PE)xE;!zG+CniYR5RK?_#6P?3!s_akx*GUAaSj zLbDNfQ#?xB2QkI_FVHPSbe0RVv3FfpH)xbIOJ zw6d2dSNCVle-W0O`fU`kCOaz-gPD$+x@X<6J8v2)F{nv9bj*g++J>1l=_U#U4$y^Z+_!!-tw$M$b6?n?=Q`dKc?5B*K^YI{Cf-3V1{( z7(ws(zxvE>J$-$>z@Ee?L&kj2@qmq#T?(=7d*lTirLCd$MD-A>x=Ff2GbU~K-k%{R z;FliS6VzfpMaT%=>sU%6Tm;V)Lc~Bw-?9kby}NGxLy!yCL0k@VLU7d-1i#5`Rh89L ztxq~EoWBtZ?-zg|3RlFR`3yoR@>A$HK#3%~I@Y`T`(cz1FT2Gns6Ng2mUBv#GkMR- z(ABq}jY2r--tqjq7`z%dS&}E4hpGSg!e_QGFzG$QcodWkXHwUcJN2KkQ4!Ckn&nl*qwSXiNZZCULg#kUPvF$8Mya#`E#{vPQC z(Ed#zS`kh0dxlPwec3N-Ad@WP($llQ9=*`ZT~Th=gU!jn_Jsr2Pm2sICmBD7vI!88 zFApmisZak*RLn#!9A(%7s&yE$-!S< zhvnR&6u)n*)6vZ&$`F0>;8O0L51dXnpA?geSL@7O6CbJ%j;whXE^Y%m*$_CRfxC5U zXZ>aveJW=oUO_bu-8a(}sq^_))8d5;--dG!F-xr(?;;Z5Z_$YeTX$Bc0(|ZF_vXF0 zjHOIzgG^5DBTgD|uV#Sh%wQ;G1rJ!mhbNX>=cuy`D8RaiU@#)mZEZ5HbI1EJH8!tM zbP|u|hwi->3e=&$8G0zOU6yzRv|@rGip#Gc_OUSqnDJr>O)091 z1g`bg623DSWJ)SBbR0f;F6{AI@{hJt=nli?%m^*g7=Bnm8ilTG2N@w~(0U#%oboG| z`eR$icRr3WeSch7I!+_*=}%&H-KPqw6G47M-=I#@@4Tz8+gsUuH#l_AM6Q0}fp6uli#+KA`xQKiViDwy9^dbe)~O@GM8J?}qwn<%d|3t(RQ3>IG6 zCg1g{SmWC*on_?I@psMuzC+C4LV&Thxaa)Fj@d^zw}jCw_|X<{jE1NS$-o;5I?=0O#Qjr|PwT7gj^o z;o1k;DnG2_7(892Ea*rP{8P=q!)WZO-S$=75*vDf$0YjaFAycq+}vN^UJuJ5k#4C< z?s6||tsWLGhj&`npZ=V7v86x=1^uv$eItzBm{h%Ac6Tzqr#70AdQ=`Wv0!itjr+7- ze&UmrSdVz_?EeJRGaHJFyw{QYUP&M()V}1!4D!dk%N>Xv8h7l ziX_N>3yGaG&=M*;9)}DY(ZSss&r+er2NfgU>9pIFXPWL(S8nd~yF@Eqi#;TxVzP<& zJK@SGlujd>MY#6yhwvt@yZr5@)**jXW#4VPWF+OkxzXfNh3?5X+WGgZI z%=9(@Q;Tw`F&ad}+`O9_==}uVjeSadN=alvdmz4k!@T!`^5oj}+f`NEiy=O%Sb}-x zlp)*199M#m(FlI*04H20zjfpDpi{CFMliw`NVMDkrjT<}0@~{;%WDdV7d4q;YNj!N zKwaf^j=LQ)7|bp>yXV2Q+0u_8;;lilBOv_zNPh&UZ2_71NK+c08UiV=5p>*L&C$Z* zqOUELfbR}3V>BYQ3pVo(h8ZldP*AIa-J)+`35m2veQoFP&Da3jcT_Ko2FwkxGISo_Cj-YcNIC4PjG(Wuqsvvdjt%uttY=M_IzMxqW$MxY3xn% z$eb|t(>@QFrMPkJ#zLI-)$$UU_}>6AzJimE@a0X<9_(Mo#@&1=%$U}%m76tv`gkd3 zKGZYzZs!1{Z6iWfgLTb4{{EO}ZaZ);wLBJVGvw)dOaPMvVR_ASUh!~hb?kg>j!4Sy z@y@C4UjaQgEm=R4Dz>HTp_$y>V(E4#hWNI&hNnRU<)`xFZ27Ag?-S|tvkU0Hgeh!t ztoHAGEOD4@`0Mqb!~kyZu^(Fx`X+8S{m=yVTK&y;cmpgoc-_2N58Eq~pHL4v6LyM# zu6PRpg}1?SCMl&l9d>jXxacM`$8XPpQ@AbKhjeE@JNQZn9lT|m&B@qY2(hoG)#wO| zOd4Fs9{14i!p19~p9xeE$rDJ;c2D#up75XY-%2=74n8(!V~_7J&KR#Wv-cn&p>MaS zxPg(%I=}CP!@weJMRK{tzV}~+D$t~|+Rsycj-lwJO!WxK2zJ{c$eV|Hh3 zpiB)qs4#9tfVM=*YU_F1xAuy@BY0OxwKjQqIqEYO=ewLJMx73^?rhbPhtv$;PNbOH z;KQihN)J*0CdZaac%(cid*Ux4f@O;SAEY90&fR7z_Wz7~%>Q3rLKuMA|1B>YL0@K~ zG_?X0{qhCf1y!jUI~&OvdbmTV)FJ=G^2JZZ`jR6Sa7tD7mW@9y(`(9Nuvci|sYA z1wHv3K`5o$6*|;6mu)tygLxF>>=l_7J?^jsZox}iEv@{|cee%|3< z`6h*lFZ1t8e*Fmg#+Z)Fz&|&1)DquylRz>b;mv|7axM!t9t9c*nZj(gJmu-Z&8qqz z9liU+pP*R22krDpbOJd}k3S?CeC^znD8vcj*%yE%bzf4(sQ48D!~fo~0HljxwKo_c z@SP>yodu;##Iih^d>~pv)<=&H4f3<7;MCm;m#ITpVp}HCT6VEPS!=zbo&9V?jSH| zwM;uTc$ez@;RbyV_)}9uGV5m_kfQ+YMtNFD?8m);3rng+q!$SVwmdX*9n^%JBB5zF`H$P7ORHHK z!JMyQ_3?<(vZCPBalehs_^GZSsRLLZ!G~CGDuMJ|^H7Mx3O)7eOprw*l_`Q?iwuKP zUyANEjdzvZeu+Ktzy(Gw0A<`US37x9eAVXUvoXqj!5-KMD0#CsM%Mg6x{p}g$xEMY z6qtq38rcC#Ap^xXXlnvW)aqNG)xSL}XB-Uox2@s|yp@BuvvO#6k$=17gznw`dj!5* zY~JZ49}pMyAH>?b2RfW&Fkp36(mUe;YG`h{*kD&zm`Hx@3u9stUj*4dA8}bdZe6m5 z81?+UhR!A_oei4+Z0JhpC3GOiHOV+_hQ32L__+h~K~SX8Qo&Pu!EY~v=`uzKHt)vN zgEck!M#AP+C)CqKPo@$9HE`|lBnL`iI*~!?VYC;T`|6~BOBYIaXmA&HTEq^{1p`)> z4U*Oz&R2`S+N~XBY?`Ik=0_b7y3^=mkJod+d-<(7Nw|vND0#dO|4>d9ss%V{&Q~w_ zJYC&RNQtYl=}hjx-Tk#?M5vOOg#9tci9fo)32vsmOU&}s)=O4gtSi+l-+AaZXcB_$ zORn^8rs}#qScbraylqHkvE#)j#59X41<;C4gIgwlDPO!t+Fz<_dN zJs#zQyhb2EPz|;ej7Pn_=q?#9OSq}qdKs7A?ZC+y{?7$`@c;I&6+OdVF zCPH+1B<=#%+c`cu{p@AL3%oSv>_`giAo8**sDCe3FLqHNbaQ2n_Sy?3;K~qQgyTXF z{hmduucD$BmotgCpJ`pI@k7G|yk}I)%R#{aO|)M}bvTmI>UE+Se0SfkYd7)T->HUI z7s}Dd4bP{gwOO*_or`1=&s$ zlPwr}_j{IP!$(oj-n#+An5uliu=I_oCXKh{*?W;Oz^(Vv%Z^=ByF$$TE!J83ZPiMh zoN{J9&g+$F5C$*&E#v);==A{8Jn(-ytK1J?YAu%kJiUy}P_)Z@$a92(ybh*aD#(fQ z8)W@Xnv_#Ydpyb0b%kF-@8|h8e1dRUB#1>8zouc-I0H$e=bhw#9PsKhT2gzoK6tWj z=wodXd82u+b%rZiV6;<>`;Zb0Y;rZS-SAJK%RsDwQc#!tc0|HEP!x8&w*HG9F+%WlJ8cTFX>!xZT)n%4?X~oz~j{^boLLYL|ms zRwoiW%c*?ptNWb0Lr6(iD2}b`U4^(~kD$%^Jb;e~tkNrUNY#qUuv2=P9I@@L(pCHJayB)lhF}ewG)g|_Xe=#2#L(eSRQY$Y2D(_@KVoz-30(yPW?bOe~ z{AAOBT)$%OH{)9QGZAzLW|gIlMwe9@=m!Y9h~x9?sS6Sf;XS6d&ERYHqC2WS8L|uv zcl4HY1E%X-GiHNt4z_!#u<7h3Wo=)4FlkR2$>lV1U8lM)h@Q<_Ssy_cy{RNk)CD~{ zP_K~LS)Cv@fBj{J*obva3L2(i(t=rgTfA*_AQ3UD-~3*#=Jb^HeD-Tc;3o4!I9@?B zi9FaCTX&8SdY>_4#fFe;959P@#`K zlnB;n@SYT;gxHPLY7j@hdWh;($y`jG(Sq56M2RTc0V)7|gZ639YkFnEc@_Ug-3$=GY9KACETM#zRhGe@B2CZ_IbMq-d_F-}sZ7&l(5dMYHM zbCL-&4(_v#t%#~HtMlX!oAAvGo@8I}nQG{gwbP+1%}@O6Ew)>fJ&!u7Npx`c*b0xC z*qdG9C5WTZp|g|Rs-Uu&&13^xvfW!EYj)$t)&SJvd7tkzbxUiE(syyIt}hGCO1CXc z=q)x{L68IFs2jdWs(tIc*lgG<)~qUFDdHZrRq`vcLk@}g-{~;=xGy5|!i_+*eD=@R zIRc5fe)i=8McPtt2{j|O!F}Z1LIHFZ6oYiCa@($XS$BSYp{6;q;fWvq_$>z;0gK>Q zSx1LgTfNP=!~bE|E^-z4f-^O-?q-;{5l3HSx)sU?xoHUO{km557%A9x3*+hu3O`#) zN{ss06;Ze+305UXwPiSwzn}b18%R85kAGgaT1Dn-Eyr!_)RGjaC5igHb#nOWOi4X_ z%;r2kT*YK^?b58?LR}nV3L7N1-9U)e*$GcZ6x1v2s6s-^cnN85m7J@^wrXc>jXkn) z_s7(rC=8Q;<{g>6;{s5`&d%`tUQWesoN*$r_HlJN@pR&122ROTH;*3k&U3M+aUQ2# zZSg=_-XA5C`;(#5Ob5xua}b}K+0&r)Hvpai*} zmxFlK9&Ek{x|Twe1M9dks#e|eU+&y}9}KvVAl zm@HOGKu4(JJl}u4HQET7UoBDPe3nNDT_Kj*6N)O#bc3gno9xcHfho*0VqK4kv;y=F z)KC46veQ%9gF8Cs&i_)5Kt!G4@6;mo>ad@|6_2Jbe4mNlck7mz?~eFLOMQ9nG5vTA zQPXedtkt-?zf8(;$8F)**q7@=+Kz z^{UAqnT|(|AhVO{oF81zSO|m~%Gg>yeW!IysDW;EZZqZS^*c+F?V)*<~& zBao`YXZMZp2I}i7-mjZxUpWc?m|DQ7X_bylUL)=HZtedU3R#J{IpXO}|{%z(N$e==PUIn+&+kv2J-1vXUI`?p<`#=8I?Jkv2DN9Zr z?kL5In)6m9x+RLpaS5rMaz1WLLggGKE#?%;`Ft2#$ed!%V-Aa9m~CblJNfPY{;uoy z&+oeS|8{-0>+^Z<`Fg${58;zC_J?UseCXrdfp0$!*kko?KlXYy<1s6Vb6T{Tf2A~p zNe+?`EEY<)0%?$iV~PF1MdFhlT(FI8bG!@Y>{R1*|Gb9h)Fpi$o!nC<65us`eHFfC z$yg~g&*@`iIKFp>efi70N~t_#)3UrpdH}jpR`C)Btp|jh_v+!%V|`|L>CkZ2qOOMY z38~DhIpn{RW92S~dH`MK=zm2W)Tp6?Q3LF2HSu(hOp8Bq)j!3;E zgniIbY4zuioT8Sx$#yR4?7a$Yf~6U1f=byo7_E{Fi+yzw~F$+@^7MZr8b`k2#k0p&R_I_1^3#+r*?h*^_66L zT;@6UBZ5{~B{H;yRx#-kC0K9QHxFF}J%fGx&hv^|d6zb+x1cF`K#W&MaONIf z>@+wP!rz*1E#lg0~&)$y(f6i~>>A><`>Od9XXX{E_fg3TR}b*x)UdB__4*y+?^9dJM_yoFW@T)oC~Y%n|5b#d`vE#fGOG3 zyF;FYs00s{QwHx~S3Np_5IaA3FwF{xOYI+X|^F3v8x667<;|=7r zS-`SXyYfQ^orsG{)g{0x^RI$5%*!PMaz3gkp@dXe8XeutnMk~}q#zrgK!1auokNL@3ovm3SbBs;29 zks_P47^3m{;cVFtd46Wggub=y^OCLWv~7R*w_5`xPhmZ>Q1Su!oW5 zFUfGpl?TL^cqPx_FcrL#Nz=Ate~V6>2huwu6{lU0t($(qX zoEx;79Z&wuzR-LUOYhQfgh}Ir*)W1V#>;(kV)+le&ZqDt2`}xyIuElZ?Da0sn`Pc5kZ}aG(gG)RN9p)2ym!@>y!SEq&VYCq3F&`IeZ> zbLqM0=;=(9-hw~0BKZtt<%!9uP!l;P>03qsfxS2D9J8&*g1sJ+F8>Q+dJY>%HnE-I?d?yq`b1^LLI1pe2j9yM{jM`hfCz7t_5YoA zG=*&LgfW!YR&;~6u=lUVK%QoEb%R2eV%~{ob7L}Uynog@kdN3-T(ebYCX*Hi8F1cW zUvNy*;1mB_ju8`;vZ|nk<{KOPl={BWf^?TurYs&M4?>0yZi=1mxtYh^yCUZ*^WgS| zTJ>Zj`RDE7I^amG?B-dokSWP5t<$@*M0RTReO)Vl)z!cyzYnv7gv(paK8Y`4i)r^l zE^0iRsEV3GHD5%`&1S?$?EBMFIcXz-&1w4vx|NW~*_LQ28g31QEc4pMfMqgQ$zk`pK81KA=^&LC#>i}_Y|HXY0>{HYph)2N`UY~a6n3~OgWrmz?x_`w!uuk2 zUX#~I5!*hvse>&?99DDvx)2@T^FWn*%NX8p#tL*GXe!hp{ZDks&CqL3q5K-wtCx6f zTqyO7^B}l5%U-$xC)!<2fn1TiSPxBWo2i(2zO6m~(Xi^0#9(GwaHcci=nst-p;l=m z2{AVv(uw&qI}cD~fW^dIU1lkuMD|y3qN?21hTBQ!Sr3TB^!8p0$+Upg0Zq|77RB|$^5D<1r!E`#zg21JVEj*Ie&`hMokLB)@w+4KghU5lr^IETgihbF zxju?Ad`$uHC{EQ!^bq)3E{=@yT2&(z3VEvf@=hsxc&40n#-bIPTl~s`w-4_pIqplGQjSw zO>>Oh#u@=UKmXW7_0QRC;JW71g<6E-dR+rI@5AlA4G z_xbA z@kBOdy~fUZa=OP`HH^6$Dr=33byhPRb8dk+4$-}j$H~jzi0Tek)HOBgH2vFW!rjD< zJb|W8^c6lsU=2e-T}d}$R!{*HV+(!gNQ)Yj!R?4+;; zUEuZiDgse9zC+*995-^Kj&xtd-A(V1KOJ$yjv3sN_+I+9?~8OD$q|AcO&5hls4!7&WLYZ1p^|wf{c+ z4R2?OHF1Ow(+}rHf_<6;DiXc&Ywy(_^lLd|jw8;ZC%eAxdeYu5t;W7yDoU?Zk0sv@ zGCCO!K}oPWHpD=QjItn8l+3Xqm-@X^lQLflcY-q<@om=y27x!(;9z6I=?t6LvZz!M znse!oQaU_aAFBiFmgG7{MPw@1R@M@;49~fQ1bDvrEi*46L%J^{nW`JB>6$(%cqgb# zQib{4IzRZsewnWr+vII-7dmfBpKUZZL zd-Rn0hmsbi1C!G+2G7}oy&=*IiZcGBU?ZW4Ljm&?waw2d@W94LTXt?mUZU6dX4R*Q zJD5)T+F}o9*pLwc884%{aMQz>!x45{TaqSIO|cbUwfIMA0Wgk&&48v$Er+!60m8et zlN)|+62(p215=-GOxOi*62Mgtiy9Q~&Qnrt9zE-l5FUr+^Halb%^t1yeg3) z)cqZFJwZ}L^6BFb9z8;y0z}IP9|F5N+?vx>l9ngbl?OT)$U}Q99VD2NfvX>$il?>@ zZzzx^)k|!C_l+Hp9jz4_wt51@YZjuK*i15DJ+x*-Z@Bm3E|x2V`0x+y?7V8I3#FoWyc>*rX1u}(2bSkE&i9PTTqSNCHd5 z%70N8W)rYG#{yaa%6)Adgdq5qxfA8vJO7cO|8Mh+Q~%#+^7X#ReMS2|QS~bf0oLJ> zuD(MZ#sC+zLwRq#*oz8_W}o~n%ciJ!wF8<>pO*~|oR}*n$eZO2k2UHMGwIVSj_i#G z%Ag;K+jCgAYh^m$cyrHPEO*dy7F%Whtb-qi!IQV~#8SOCJzlo>09?6fGM!^Mx#z_C z^z%I5fOaJ9{W{TY%BG;14Nza7S@6*hp3%*kux~6i9F89PS9aYy(>Wcu>4aR}I%^KS zIXAvY$A3e^2g0mwZ>BH`qZ7^|8K4BeXDikqB+gXn{zTE|f!1Y`fKKiRnLTYNVUffL zaB;~)-k3h{x&lj5ycXtT=X&nxebD`TU)o&h;#Wf-BuNFQ`=s`X-9WB@5+ZEB%X&{Q z`CiCFYH!bfo3W5MZZ>u25OF`cblu>K+|II=)rqd{o-4waj!Fzl1OlQs7Ps3U(R3q= zdK?y8gSt7KdRq8CJ-)9B858dRuYK5F=Tvik^+e#%1RiGzI%Ok6Kz3^6Pv2|!SY%c= z(`~QAjt&~QM=Y8XNOQqTGD6g5BFj-%NOeUpSuI3a`m4!{Kl=Q zDooi(H@Wl{Iphn+^Nvik=S?TtWr1>E`+A_45=0nb1I$RMZP?p(N!R9=fm-EYdz269 z547cWG%p7!K@h_?BAnHC3M8Daj3{XJBXTiz$7Yd-4w0* z^nMDZH}*H~v!gj*sDEy`@*MG)uN#oyVG3Db<+?poNW%PVSq&XJ&0IHkU5xrTUS+Qw z>-pPU9x?FHb^W3BVBKVuCN|iaT3H@f@p&8ddNeLVwf+oU?Off-$)^fB&a)W7m-I(2 zEG2T_&hR;v9Znw7UG!j^&({}8_{Ebu{uVE~B^EJZ5BDQ)r{^r&`GAoDz*6BBeamtO zxmLJY3yqnEN5f~0_oESOXn3Sf?5~Z+)crq>fs-cl`7bv%&w~lUum(({Ba#0t?X=(> z>ad7^vdAUO3m9zB&PZ~@2Kc*hJn3qD)qcq41w=CGb4(DFQR~YDOUh_2u!s2M+0EJ< zP%UVcQFHz#*|DZM$LDiXbg*XaK?l2)qbny?y#@&?#GCo{ zihpt%=R}poN6tfM93y6s)Ce=5o)ER)P!(<3PcTlwWl#anx$#k;OW^i)J!j?A;M2ZC z_rOEz>}cPM8szT!>ASJ*G9?#1GF1W|C$dJJ!SjuG1*>D5V+ZghzUp3)5E8orvbaD)t!B7`} z_&cX@#L#`hvv6@`Fu8knZ$cgtd*e?ztLA|0=Bs_WgdihIo>x@cnYW~X8*W`8dwO1& zds%`#S7L8v>95UdPWH>-W4AbwPCP>p292_OlLcXNTOD*=HCCY6KhC%m$8T1M-S_AX zMm5YFP|_D%;ki&6%@tK0(m@5yVN~hzAxxHt&2hb#f0n}C zQU8c~$21w|n<0EuXvVs*dw&)qFT1T?82N6MOo&3HT%}n}TD15d&!18eo{|j1 zOHvhN{JYPu$EUs#Ez=HWwnDSqF}+xZf9ZyaP{k&6+TTjnh-?!3E-G?3A1SlgqYuw4Ic19nv z^pySFuw!qy8GkeS@7PL;j}n4Vpt{##+bPqMvD>}v%c1`Pp zlEBAEh?dp%(Fv~Swb|%yMf<}$+tHkn{Q-=dvs)MW-*j{z@#ET)oQ&YSeA3+mKDF(Q zGs0sATJhh?kDuPQ7kBBkfu!(H$K;WuY3#SaOtaDPXWwM~Yu;TDHBH!D-!nNM8>Ar} z$a^YQH&{S81%0iM@M1c6{@neYVQEC+;|nwWngC z5k5Mg+V4~Yr#Cy9sNVqtAndg|X36K!=cm=E0CJmi5v-50^+?A!SWd*h9(?BVisaap zo_e3-?_|$1z+(?a`PMQY+|EXSdpx&c&i7(VhCSuAsACcU>Pk?$c|+hKr3Y4AcelM0 zqd660%-_zmF~e5*N#_&TBjJZ_@wSt+Ku01hgw9M;7<4O}xEcfw^C*Vy3I>#u_?NtD z2sWj?*0(}KHyp{LL{NgF2}C_x@0eyl{p1rx*SjTNIHfFiUFk|F0evH#rhC)~M+RB! z($94$Xl9b@4TX(-rwwmY_-~)OBob>Xw6Ty<&2$Q%yLfrXy6~_k{115UUb9E&a(>zT zs{1iSN{>c*#TC3T@6URP4}Pq-G_`%0&3jCXENP6t6RyyKZ&Kk_-H*8Pe9@UJY2Mj? z4*Dp~hxN31NQvY(kAHTB$mJb( zw~l(py2Akk|Gdz+o^d_Ui>QQ^iJD9&`I&FMpK8yPhXgg$_SWxipuqp3PUNx$RX_{u zT-$$;ljb=}+<$6~?Tocj>_+-b1Pi)Lz)$~tLN12FEC4p)DFYpYP8$Y*Q_^RI+oq@B zu&Xv9H=jj4dSt?wD{DR#H0uX#erUc!7dtN$SWza#?u_d>*aQtW1b5i;1o`{^X|PF> zw@HRU50Gs#m%=Hcqd6P^sDwiuGidSpC06WV7@@fr-jm+x$gXQmoYdtw%$r zd%H+sg#OaPhIPY^#PEVBEx!rTk0{Q>fXg|(Lpv#2p+UrIEgr~AUd3MjP}qyqy54EY z8_f}}w4ftEMqOSFN3kmkWG84#tDp6e`Wcf4aM0JvEfTpnz1H%wjHL|4YRncUg}?kf zWa=mSr75)1*XxGo)SR~pHlh2gp3hZ|??lV@zvzaPu=Lv5!SuJGJKD5CaZS!s^3x)t zG#|LE`t*Q`-$0Pbqb~PqeYbm!D;FAk3QVg*11I1`fl(vmBeL$64sAE&>TjsJSat!* z6n@;!)14WApA@vEwPd4ZbA8Q$%p?7egr}`h+oZ0)+_BDJDwrzSrL$he6IvZ#? zzc4YbgxnOxg7yivH*!eZbI#Q?4;6(*sok^QbScdgf9@wQp1pYybKv} z3P?1US8%PaY`@j!>5w?B`5}2E`oxpCq>$xmiiOJy%3LlPs36}tIkp+cF^Pn%ZnW|Bz0(;HmI@V{YF>JA z;n$Xu7Q8BYkFx(-@TEND*YyWXh0(7q@hzwK2g56RM{tbS%`4x$=1DB2-veulQQc&G zlQd4#-x_gYn7L=$nVZT~a#QKq3g5h3r<*tH{qxpCFlzJe?^&)>Z@ zwNLQ3ZLs5%&;WL)pi|$;i)!#-v5#4ec|-)ley3Pg9 z4tplMk(a{K&NAH(z`pI5`lRTnTh9o4NY@yEc#NJu=kbFKuq_n#;-B?zGo@QdNPeZ3 zO7R|V!$7WfT_}qDN`nDT)N3Y_L?PVFG@K=+HTBocCw2fPjHa%pu^i1<&gCbJoUQrepAKcUaIZU_MsbTC13tk zfH^(7`_RySL1AP0(Myv1<6uF97ZygbB~EwB8h?~j+Kn}oM{c0YGz4L2^^vMJ!2vq!p!}DY4P}1-xYN#q z?!4c@v1K5|HmU3P2JuQs<)>GoB8Icf&XB4V-#k`QsgqM8G=NfYsvAg>Fpc-w1q$Dw zl%Y42T}KkE#!U;F(NQC_XwQ7Ja>Z#mnV*0;&1dvrM30b|r!Dd#JaDidH{x+GBgCVB z#Y*;s=ReQC^D26~J9kS@yvX~7Hcl#fV~ZVje#TYLS4%~7)+Zj(O`FKZB+f58IEE}< zD?21cKK8+Ums-6jEPD78ZW0|S>t>LzHR0I6gnUhCCDm$`;>xp#q)qSsqkA{$!^47o zqS~ePCB5o}q?PjDr6zQSZwHsGxtPmN&rC&?Dvcnj6?6`7?<4uYZ4xL$B^1ARIuvG^ z{sG>V4`1TWuAcoS_tsdHNolN~`B(dz-h1B1p9k4GcLiI$OBF*)87*m(r`}*T>VP|< zmc~|EcVG0^S|egO5BL}TS@WBDwhxyh?w{;5q$mubZbNtIm`KKS=Qph8X4fodw&UKW ztc51IPzz6g>-acVqo1wt!ky{qx=QkB5Idw8;Q4&g=PiDg!_1)HJj5Zpco~e1O?>s{ zrH~w65rh8nTfbYeedlN1<_YPYy_-4iKb7yOYG~4pOH*IcDG=X@`jeS&j3k&6fxL0r zey{R`L|Ni+lj);?_;E6+n(5}kb&A|YUn$$&Uf)>bc5(c7_9IX34oVX!`#mSP`?!to znooScjQ-YApvHMPY4*&C0=1b+Zl05}(K)RP>HiXsc-$-_+hXxr{9wnk(~3}5RI+{Q z-$H>MsHWr!ym9F5dtmKUX$dbFVQu1Q;E&AD(||lgP;-cTAh9tl!uflSnpjgxj=|vK zAY@Gy+g5scl-$;hCYP{>KzZrzNqpX!Rl9s&*Wi+GpFp>awjRXqm68tGn3kE%5R+Kr z;3!GhKVJlp=ac(uClru!Rdw)h?e6EO)y$-opH7R{@V4@ze~)Q+K1AluIR;rn=q7e(c99nRBj^ifOo z*p#a~?_pBTw9+3iNLCwZ7U6ecqT6Iu1jk)XFmpVFc-Ny*rVXHz?K*YHC0G5D>T$;8 zWd-GhP}%^ovro_riK!7*o^m7Ud3?VQG0Mka3+YZ zL*uSb^CHDlngd8~LJ$eU3t?_4X{9~Hbvi%~v?}n=;^g+RB-|E&s0Xbe$6x?{E-0gi zBXuRb-mLfIdnbmLn+I1O>x;*J_chwQwSXR;g+nuu>`ww3Viv0P-nk@VO`&%RqK~Sqn;`t-!Q{tC@U<8|?$y}UEt3ULD?TrB6hbD}_e%%HVy)QU5 zb@2FmYN02=T#eedYTHwx*Gp3q2Vj6f$N31EvVD|!{^(%$QwPK-?aoXGqp{D z8mLuOp*m#l#!jDk1k5;sDugl;3}-m5H0EWPq0SGrQPD32P7>=ZBfY(-Bybq-bG9B5 z%j=rt%@3>Di}IQ*Z0NS~!TQW8$wy{DkfdLo#1K6wHuzvooV5u0mJ^Z5x zC{j_Ilh||HwtHPDZ<-{9rb2oh#!s@`H>)%Bn`q(*?je_+Z(~jb$7!wTgZ3+_A|oy( zHRS7ir+%Uy^vZUvCjT4Z>9>*W4zh4M4(9c^TuCRyq^Pq+Oy=mK$)36MFT0ZifG_GK zrhgOFbEyO0Sm!PdD42XTtBzpaYP<1OsjnCNOK*MLmHUiSlb{5eq`k)K9AG4u1FP+o zeA`9KakCQ;ly|#%gC=?e9~a%cky0jJAI8+pzf5&#aw0r}NhAwlV~BK`txkNHer+ihuAF0p654+*yM^mYX$~)-4Anr&Jz|S0?Oj zXg9$SfA~jMwpauumHuh+#p6`dD#gsE?%C{RqR$Q?A&)jAKW-Y#DWR@xLfu{zdZou1 zle4bsKk86(Xwq-3t{H=!^NkIIc1@~Z;+PJGovPnqC}0yt+3nBAfc*Op7F%=b5~_yc zSqp~gHJjT};JoJR<~K>j)29>$`kGD4`$g8*fqqv3*vYq~4=t30ZQRnU;FU&>^2pPV z3DA%li6Qe-5#VO9j6CLZLt+Q)Q%%{<%F@>8vLgQpz05c;Jcs3wS{oQNPHZJQr)s++ zqJKxsP0(4BdtZd=02@dy|8~k>E_i1UJ8g(<^ws(v*6ky>+OJh{Zr3lW6F@S?DB2uf z`B_KQ^eA5tCH{fGJo+MTL{WYRkZ+!xKAZ|));x}$URQSHxpqCeNRGs=&izTNXh>oY zGSUVYW$;SydIZnefb)#NZ*!3UMO75M?^9+qxA47Kd#N(9%#Eh;R2sE;&!c!3a9z<!Jttz-ETl=aBke&)DbH z$Uk8UhYMb*&#``QycG`${UeDx4BlTk+4IZ#Tdok@5%H3(>2fS6)1OLxoIyRkYexrj z2F`$vdslEZP|@41su-G|k8H&9<{&S2YwwGop3lIqTQ40yd&vFSY_=&v>fwqY0$s_W za^&LUK=^4Jd8Lvs^Y5^e;z18Lm>av#6=C|ZK8p&+KQ5mBcr_Wmad{*Pj8>4*tldoZ z7zsL2U>566ZdhP1a>Au3d-9?vT;TV$%%zv>0Pmv{Fv0!^bXnjl4r3eWq>x>P_0Ama z-TML`gjpw+zX4?W`>6bVogqe&hY8c=4I7`WwZlHDa|&x?i8ycGFHLbZ+0_Pr#tln} zn!}^F0~#l1v|MOe<$vFE;O*q!1lm~P{)PS3QEzYinHpc&nJ8mpGdoQ>v`llsm4HR1 z=QNWH99T3SZ1IRRZ1K30#}{S21l0Pf`>?w&SG%#Q$LGffcVIwYtzqcBlM0*Nsxvt= z_?V<}xO(4@ev#CjGR9V^E?5D(gZu5QSl)V1usvzPd$|)S=)`YQdQm?pQm~uQ$^r}J z2s-CNJ;8vkfirXH)OL$Yl(pq5+=@{*3`VZ;;kJS7$d#eWUOCI{oLEKJhNvPiKw*Fj=g1v*6dc~69vi~z2k<)uZ1D8eQhzXUz=`cEb{(}* zcQLKAg|921HCGI0?JZxic1?0$amFH{^FUP1y>o1E`B*<;DsZW7eiHPiP72K4F&q!& zznA?`=%2b>uy-N_LSSnj;+%^n8V|X4GfdmXNFNo0PP{s}aAGD0Gxm;G53ZFIth@UX zIM0tbxu*Zp|KL|H(RKX-0~nK&n1V#Le`hnz2RFVQWa+i69;U9Ba`u~_E%qlG%4UZ6 zI>uS;pCCZ>zj!e1F`#|{F6|P}+TBcoC0q}x8)z+ z*|3=pIa!8Af)|ln`fk=+8(uC~)~AsA*=`~({*vl|A@oeo_?4PaOmpb`)pS|1?mY%& zGLeL4mL5fD`L&=|##=fW`dMVs)k9&smnC|r{XR*Qp(gmlM0$84QLl&Xxc-=B@sujr zLedG4F!(sCK38+_hum{+=26>liAkMW%mVwcJw-Kd9ByF(B&|4EVH(z>7A+I|?%xce z+B&`;15DGl+^W7<|FLWdju_lXz@6dHH@4~VnFe)@g(NpC6pqm>N)n9K_y1>n*c2Oq z=>M;^bpitYpW*@k#)mJvi$nSdkh_bGvX8!Q&y_{hviBMJLRY(njS`IXh;fJGQq96= z-I#v6>V7*1O=a0y1nIX7yJT;tef%GJ>Cr{YxgQyegPBnB*4dPJqN$F`Zo&6(w*%$s zNkssQ0P2B{m+m!s89$rpF*fS;&eVT%CgN9b^}=8!M|$B5xCvq@h*=j(EM(Y45ra6# z!rrItd2yS%yJ*Lz3ZE|(7W-!h%b$&UuQR{5tfiOh)`X+z@Hd_0$xEFNiqPk87&f4xnRo$+W_VcqowcO)&!qg1!)|Qu5JcipdIH#EB2F zdKMwyTSZT0CiFfB#1`3O9gWrLvX6VxuH;hcT2&POq_;n8Cq1>+G!ct{AZ zPQ+iF2yNvOYr4x3%<(-5E$Q*@KfQep;2`M|5qpz35x>k2n0^FybPIkS_rsEhVhiD` zx#rc5!_(D}p19^S(B;OLpK0YUV>`zWDGu5@;i{PDV{>)KXq6gJgD(vVm_5Mdb;^AD zVYh$rICt_{b?+dUrPWxL{}472jMX_b-)TEK+ch?3?>@o|PLExw*;I$0^Br|I41K=Q zL=_E|X(6cxajF~ZbZi7!kRJ?K*7HX6;Qkc7SP97+)^* z!2XD^4ZPrz>Q{J~RHA6ewVpMqsr1M1mG}n+{wE?PTt8^K`BxYfe>dqjH@zF9zRc)r ziHZ#Wx?f2{X;qsMt6w5f@xK2R`b(TN$ndYq1l|1s}6~g-Db3F`5K(&?kq@4tXXBx1&|lj-eCeJD|&= zW@v*g@bciphIOAJ_aYfl-nd*N^uT*oZdXJI`}$UIgl+&cqaujTdBK*PSbEcz1hwhB zGx90J(lz3iTv*njfRk)A*;(^FZ7W5;3pRrJ^k8KD7d z=M8JD8WxEO@r8!$>;Ht@=OQs#*JopN*3Qx5S498vDoy~}_vntOvjx^D*Wz(c&*0@N zq2=w8%7;*LT1MvzV{aTn;3g#vqoXS?dyM9k+(|#PeTz^ok3LQ#ex%zLUxs&I$@Z5> z!P{ZIZephW0`LU!p3X5u`YiZA8nldvB$7szaP`G;UJx)81&uIJt^HNS+@ZsJJK4*izCzNBuYm~_lE<^#$mrUKh1)n z*$Vn{ZGmrnxqDIF(EE>kpO>HUuye7K_^UQ+T>`j4{V(4#=k@CSn+I(mJ6)&iU~6D*adY-2ezQU z__;2Y0*csd?+5C(B;SkWf2EwFHP&&9rcI|gOl*=;c~kTKmszrk?-)a~`6(1pk*@0l9|hrk6_t{DP}uBz zyC!w|p>-HR8jsUqhdfP)`1xsl4k{PKRK@%*1R}Do)XA>rNkLY}UEFcQLhxc>g#W~G zb&_i4!+h?~T>l1E%CUUl;Utd=j30rs4TS^o6@yXOY|E)W*L77?3 z1mxFJQeb09fS#2mU)g`O`@(O)cI;;zU&FCIJC}S4#fKzBY3=9PySv$XWp#>HN_00A@-N@;u2S;_@OG(fXwO%H4%>q}d}xnd9GtFJa!ti!c8H;tFW^_47$b zhLLc2%@vWkChymE(823+(}!5^u)pR-dqQY%1=+*p?3nQ({+1@q`)+Gq5pAKJ9p@Ic zCL2z#^}15E^Awkna{h8w|3%{%EkY6Rjn}0oMn1ZPa_Dwzey6qYM&sZKiPTH5U!Bvq zSiPMg@?%ndBejk)-)G*$^+NT)ttFj%N_ZFX`+L<^i>FI3rOsLxxG+EEI8I{vn%bY^ zlDB#_w%!;KJi_G{Wx%4P^{{k_vUBi`2~*JL{Co5NZ2j!}%b~7vZ0tU=p!~&q4|T66Sh0_-?dDrtxXg|2IzL*pMPv z<-Cvnnpcv8uim$5LXV)8_QfTr_# zg!4uHwJu~iB&y}72R>0arLiAti}J~JGBR_6WxB8*!_&A5)@!|$ z^YJt0L-%R^^{t;oRmZD`h-ZB#*9wbOOEYHKV}^E`HcdN~n%Yf!pAOHre~&Bo=@ISenGDZ% z(Tr=FsKu5ob-V$@J3Tp$)%9A3!nd_<@n_OB-&AV<@rjD-t8*%~g08uyE}ll_1$@3j z3z|FAVTb>vxBAWLEsTx~f>k!SjX$RLT6)!KMtL3Tjw3$ZUNF+Pmz>r_|JvMld(ww$ z;fRXFgp&`cK_00u-BI2e0vXT_ZFq9Fsy$ztg?KrX)ybVG2P*1TG)S5ZE|l5O=FabH zOsRJ5zCBIdGxDc>nXQ;a+IgY7sv~VrQmv;o^i~wtdFUErQ>D8QYS3&w?{~TN5f=EC zi3^?z&a`sHdDD?%V8udp<^9I4FS`apHt+RM{Ff2+K^oqtDDkbB*RFSoP~9P*0YxA| zCZiYkFxlEa$Q!W4J$x1VOq~)V0K|_}xF0vUYiS@&Z-P`p->sm*&v7`tb}S zw*zAA?ROrThn_E(u`x#`vdMkI(x(J`p2b1BCh*nN8kcZtPx&@|d#y|gY_CiRu`YDPK5@ey0V}WSUx2EFkwEXJW7Ha+8qYeO)QJ3&@5XIIPgS*cu3bNm`&!dZ&_k=syQ3nb-NZ>(?Y-7 zXjz~)Z4aHd7#JC}Yfb-fNc1rYwNYLYmZz4SD}Zoz!;ReJU6Z%5H4J?zKqgFK%^j1HH+;Oa4==Uhh9JR`cbS# z>Gw~E^0ucW7arrL>|9Pr9Ot}nMS1)oLm=!ee2q!9qHES^$>9h6oJ3J6FZhBjg&w5a@**tGP zuJhf;%WA{v0`~epXlYM4DZJ<~r*D0w&)72PuKZh;p=UnoX^GByq)$;4r}^jF6%=nV zI2NO!o+Q!Zo+U$!SP2(mIU;B4tBjqp*vh|(oA#||@#I+ht^s3pG@m-T-Vj3{rnD*a$1Xu@PdEdvAaeDWlLhV{N(i5=^gJ^DR<$4svFp` zsFv~4_s8Pcb$ME#ADOvA$oEk;^IFgpTlw$7jQXz6$TVnZA7FK+QhEg_k3=Qlvn*we zorGHQ0+-isIw}yXOa55NaSlBx0Oq=bT!!%D|hxU4T&?dAS2Na0e9)Ed- z>>O-SGRvxi)a_jDvLFY+Fd)`|kCsK_m`TVB1lVsg$lX;pXPF`()>-`VR^M&9J;)HK`!M0!8p|OiM;vL#oKg zs_sq@>EP`fo2t3(Z*m#1flY6~nn;L;f^d_>7?%I?&h5M>#oy1ZR8#6qTcK6;+Fr=Y zXE*BM@>tNZG2Dh5%F&$j>HYD{-Cs;et6j_bGHP%J$F;o|DjE@aZ!ASd$TfDHEYR)W z)Gxc@eY^j|&8OVE*hFGbauU{q@>_8(hVfYiaowYyAn#Q?{q!Y+g8QI9W`Y>(r@%YA zCz<=ob+L&Eqv&J6*`QkgjF6Y5sBz>Dc_gP@eJR4pPy1H#22OTqnRY) z(XPeawknUeW?mKzo{&Uy^9C<^1&i;oB^dgGtpc@ojEeSJ+BR>TGg?9~Od!OuIby21 zvp@+Tctk63!|*VpM(0!VAXC97&<*(-?0Ncfgrk=5{}F>rjsy1(_`bbB7)Ac>`OD>B z!e^t^5QD10hxj{PNmVhOM~}+z_r~quz&RB~yFQIx0N?M7rK!3cMpmy%QxRPE=F}Fu zrteO8IyUAN+8_zbDw~A-;Y)v%&E$`e_R! zXWMaq()RFLeBYwN33+AqmtuxO3G-sOC9@N9Ma!bFhM!DD`>JKyG;ezD+R4Z#`-YOi zS;qxmC~WNR0Wth6%x2oRIRt0?Rc{QPXinKFqA zw*b{CEsj+g#TlWW*lf^DnOhHNp(dIiv@^cdUtP;a_y03hF1Das_}U|)uXcN$|0R9L9QLmf;G(pY^gl&z zzid5o)U>v84Kcb9f@8ylYJiF^!p)@qvEdv&4!eFMI$c!ZAFldE$-|-{^J^QkmMRV; zz<0-0j7OysoRIf22}*rCa&*?uDaS z!SatOdB&4fAflTC`JDN4a=#*WNPhj(`w`~a`OdTO`jm#6zO?SA?L*Uno1|x11wM{; z##Wv!X%p%A|q>r)>&1 zNIE3VlGCz6ls7IKxV8!i37rkO(3H9TXQ?T4;<&#FAw76pDOBOkO|J?~kMZuCH<3?4 z;??)VhZd?2rkTw=R60w^TJ_RcT9jKU=)V`+q4KQh)mPPG-rB|d;nEzP6La*<10+j^ zr-K=6%aq|MCqB{ewBA!Evu4D%4ejF^6?h@+ZTLU?oc{iJ2xztaHQjr&xD-0sX!K9u z@bw94;9I9v11LdA$3ES{k#5xIIL$_S@DZJ}1ZH~KD8PZR6>96xn{QkC*f-N*kul~- zm;#~}dJD$K3(1+4B?u?|SO+BpCBy=xXDv4$bWC3QDoE`_8MBhSqF{dp^k|>XnET;} z(yRj^r&+(>*x&9=BP>$B{0?_u6hl*Z!^Q{%rw|gzz%TC()v+F@i}G~wx7f2ntsCGW zI$m%6gV((#>Ji-;*9PNKpZgj1baR8AuU9-8r~bL9ZdX`pNIeiN5B^e9p49gy`f3>v zMBcpw!L9;|7Pyab7O>g|3D3)DlPu5B&=Wgnd#l6LVCP4ZA{M}+dFk-QYgO#tOXT|! zx8I?t!kXbLD0io&9PqGke1gtjbQGdPLl=OKkD45^@#o$;PL%j{xKHUE4Tk4Ru^Yta z%>+4#XjT6OMQ!xmPRsR=If$Hy?Yl&=wALkeI`Q+Wq`neQJ)CsL9@jcrV8yIGZ&mwd zOed3Cm+&)?B7%AKBdWs9`Z~e^_3rmV+%H9Cz``)c2z>1B1aYH^|7K@sA|B8sRv`Qy ze_ql7lEnK1N8iVv6Lp%G-@xpgxi2+Xf;L$<`_<0?2Yxl>eJO01i?!YmNxe|EDN0}# z{vWp9`zz@N{`*tej?zJp(X|sl zmJC=B!en$iCV(=#wnN2zHjoP|p(!F*)nLsZ0U=0B*ND)X)!R;LYR0$q+GA|nJdv!$ zJ~FRK*%Lvj5V^g4Z^U;&%2Wfdai`clkQy3rRpoh5r^O?(SNCE;)R=fyXo-(nw#hdM zWWdmF#km__!5$+TbTiE^*~P;n*pWNwMjdh72cpXp0$*Y)l=hl8nQ!b~dto&;$&=1XDCRl8^tIuw@DWT{MJZ{ zqm@cY_)U((WqG%p*EBMv1)m5f`r-)ib^^!{tzulVAw0X4j`2I7NU5`(%=X@7YTo!=DRG zN74WBUHmSg;1FLmPf$*zCzl03o|rsXZV!E($2*IBy10A^!G2S^7dp?d@g7TaT}V0$ zlGb=v>cl*U>EB+6O6a&Rv3?2%s^_>2@ES*>_Kh?RE>I2iEDR@WcMc&sfOqOz)zem7 zP42s?tUxi?ZU%LS{MutD%rx_StX3^&95jHrRw-`}y6-{phg! zq{{Vw*SZ;W^g@?!TATE}mb1wuU`8GRM|;2rR$PI;O52qg%|4GE%SKy9*-e+f4m-2hcgM!r$ z@oqsjLw6F?*B#Z6bFd%G-!d-VkH;?bz~Dh_90Z;bMGn@1 zyZ3c8;y9Ia%QZ`9m@y%buzyaZDEb`bedZ1vji!~vRGZ0AYkE?ynA$ zsO@DZ9-Y%}O*T9fR5l}xDR3k#M^A-2dg*ZMO9}me-G(@RYZYhyH}qhY(fTxvbwa7E z^_r{nrtLYN|9$D1^vX*^t4|&(WxdQ#2)XCZaE;C_ zSD2~EBUZBwp5eTqwIDwfPd0NvRy%;KHD>sO(l#5?es}+t8_7-LQH`J3wcOXi&Wakl zc%KN;x_xA^6uBGXwL`ScQD=o;S;6~%2oBD*6|2=xX~W;;sd-$4Z_3Y~AM9S-2w_ZE z0kLm53%bnP^Txvkx%4K6-G=*8ahp<1lGU6^P(j>H<&zhTYt|-W-fyZsukGJ!nj06k zVJNnwjpQmGrh*vgeReSm8}~Ee?q%X!Q)8#^oRC*+P5YFi$!}tH!21(uBXjzm!Yc{K zsm!K`-9>7>s%K|3#BADTy{V3N9p7+bz``BAD#ODvUZxp@hX=!mwIq9j0EXCTEGXDK znzbIOn<1UYN%wO}4FvSnYy)Tgto?x~$9F=0^^n~&dn58NGiWA>AAEs3_#YVQ{~HqV z;D6eS;q`-4sr7$F&Q3{Im7eyr89J1rleifeW3c0g=jew{0M9% zy^%2)y+L!Ur2uC24k6Cw(W*7ons$fKCAHfhSRkv0k0r5*I;1$`x;(_QzZiqMkayCj z3ql734y3>u{+gIsSo=-~!cL(!oYSZM-RfrJ8%LX*G_vnPdpzzGt0%5-rDz_~eHg*b zPo~*zb|woi*42ZTF4qO>lQ^3E&~qq2UrlGl3O;<&@rTcBKowD13x)q_x z({@B3?n0NI!*DoZ5r1*DLrmPdljsIK+l3+|b|o)Gz?6+QOY0_r*lsF)|@ru;x>eQ(5#K{Z~Vc&~whkrP3AYLqeP&Ep4yNt+fp6tkxvkrpK38H|n zl=3#oIwKnBEbVZCaHZ4m$Bv#OA8rZSM%HIs>R&ubM8kD@S~N~F;bfE0nTG~tR8Yl)Od=7meu>!kRIwuec#tG=+|D1DRlisr&8PLv*k|(NSyrr z4BD`qnqO-Zc0&yTT<>U!uho4fUyMNL@Wjxt!8&nKDNMM<%UkO2{B0gB(1s|>`|^{N zGUni8PVvRs+1{z%xVh(mgV}k{x5RBz;=hckSNZ*u8)uA?Njq0F{ijZb1Ir=^?uS z_Jx}DtDeGPMjE1ZxXmm3MWHNLAzvf7totXS>QpPgMtyYLahnhriFN4N+aXT1Ite}Tq(qVwXn;@s7i{X-QUaL((Go{WXy!E*50+eG~l z0|;}%hy9(6)ru~x!_Ac*-^5$0B|Ylw*Vb9{Sw`%EHi^5i|HhKDTgR4CYq!RE(vutH ztM5)%5*mDp$sW8<5=5^Q!H$%|qQE>o$xc&@yZ8bG!x{S;@KoJz(l#UXE4Ff<_h;+V zI?&$-dlSMUB;VP(18bR8)`asnSiz1h^rUC-YN*AZMUT))X$|0+q^Tz zf>-a&Q{D>Ws4US`6PU%c7eEU023D$4{@ zRW)ZVl8}th98n?o&PIcR&_aSo>gTRJ96_M zzg^C!zna32aCqlcyW{Vn5*Rj}3h=C!=rrAT6<3$KFal$yMEDOX@rNjtfnvox1UEzrEGXcOOW6_#xh)E?>UJqloTr-6e#2sKR4mr!%FLZcz zy@hxAKa+=-9y>l##=8#_d^0K@bD6A$v|0S_8krrG(~Z(uWEHith9c$lGpb)8^@lF_ zH0!0R$qCDKUQ^O5s{$`fF`~=D=&$W+J zo$EeSZ>ZT=jdNU^7tp55-KWW8B`iw%uD^veN7sh$b&YtUd>7SJkAQDz0yUn!OI^y# z(CeSg1-!~#^Xd$&VD!~O7#{E}kvF^TCl&<1=nB?g^RR^E~!B%CGRvu_p?#i;By zkKcm4dXYC15-`9^@Qp0vzeP!zj+0I8wl7bxzVFrb``;x*j|zxfEv92Ha<}J?F4p@^ zcy(Pwcm!E(ep;_+nELC|6Fr04p4>j$Rew5D6!dJ5~AjsBke?g!@Op-T{zCC{45RA6((}$nuEBiT9f8cnOa!7xW89CktR-LN!ph zNhlu^-8R6jPI2|S3m#R5Aw0;H7wk`382EPuOj>h|_|BbmM(<&9^aOIkMcdkU>SM8q z;f_Y%w-(6O4bI=51M>h`K2&o2#xs!Uz1Il8SACl;hjGet(k|jGbJBuU@6}8FuUyi) z)$cHiLXIiblAcHvQpWcQ4wkI;y^OmDHtF4uxN>-*RFnrAO1aD~_Vi-@uIn%fSU)|j z=57on!(|l!pkFVgOG8-bRt>$rGpt#7V%6i7mPE4adlB_p_ES^_EVb%)nI6S*XCkOX z>gTY8gleYM)8P2qo4m;^Pd|6FJHnW1?TOE#4WHBk+vIa^t~g2-lC|1@qElD2fZiL9 zPDRcK3EhK1{_7>Vo;ZsTH^Z!j~Tlr}h1J=gyfJfB?X zChT4w+HhUy7X3X$^qj4_eIw~bR*RNlotKl@WB5)aw*R6 z;g(I0YO`>6_z?Gp+{dqlishe}PakvweXZMu%esPnzE7*qHk1;hGjfD(W-{Is+WR5i zV)8`Db`lCjk?*W}&stz&H!Uj5fPSU);Ob?#Y>j_%%=C4B$`yFa+;zF5q*NCno#?S1 zL1;MJl2@NKn=+YNVy`^!KAAjGnYSOX{1@ibQy|r&M?{;?z=8@fGx;xS8VF<_?FP#$>PNVGrdtrWmimo{* zwmgjQ`v5Z7W5$D_qGipM2h!GlfP+)-_~$69<(az6;*&F~jpG`DN)qKqzZi7uj%f2H z$15vRT(YcQ*?GHh3c+4(>29L+Gfe%PV95=rcfeWY8;HiPyMaCer`%e`?>v|a?rPs0 zxek6K*Pm>`Fr^bcU=|O!@7M)O?AZKmPu}b9$LM!HSNV6g3*qp+=8y+a-?EU$yKb}} z3f;v2Do=%(&PiUXxjkYqniE3XC^=cuTGGDke#>j(jLH7z1n|86tI9q31mGpx>+a48 zr>&GyW6D;RM^D^(;~M?vXaen@x`|saJp4B_kKj3VD{9agy>q;nB7VlZrJ>Y^d8(nl zGID~u+6$~OQ(pMf)-QLWI2q`J4yycuVW2!{;rM}<8j&&&r!_g7Nmr>o=ds5=0Mm93 z+CZZ|)IWzkf}ZPkSPD^PXdzn1oL@l@9L13fUs!WT8yD_Nz}OY7A3v?_wCmF6)Bj0B|;g^e^i;1ZE zwWK-@`q%q4zCgvOXai-NMz4-EFXje*t7%F%Egqj6lg^52Ff9K%WMpKP4k89m<`zBF zw05guFPrA>GiADX$IRMsb;{Pig^*IK57nSXWV56=2xc>`ZSl~%TQ1vT$EcsQhhY2h zH7nFVA~xj}6HNf6V)*e%34xeHDjZu$)s?-EWU%N@anxt8D%(e1qtqP5uXZV*&y+t# z*xu0Lv&4II#4nEOl-ig6r;I~1cujH&^3M~1=ynE`qqP43i&tc_1Q|lI&Tt*Lk*$rzh zvZMs-;NiZ8_gEeccdQYE763xocAvbI;&)hC&>k`40fu7pm8v>&cSJK%hx19wNF2Lx z=m*a2LZ5$4hY_W27hfMITyXS)a=D>ddf7PIK4bP;w{f z9pq@=SqV=WK{f6h6@qxyOP%bc^v;YJiR)ID!teE8&6jqL6YnK^xzniCSi=1;es;+_ zWo2?|qGN0_clBbJT1g>cuYIP65%5WsWaeKR>wd7#;uPUqK zCC*k5>p+8LbMb1b*t1vK+oY+l#Cve9sU$O{rnh?!J2G168XG<5CE#D+E20{@9_=W! za1-#XY1&cOtQOxTbIm2QH=s)=kx1*L>0k!xTO`I(EF30kC6XV#)eyw(^Tx(gb>rI> zl0{g}16*D007S*vZ+i^N1-yQZpGi%eJrsnT^ESveWRhEnjmZAu^;%)QD!gYP ziv4ySls)y;z;cYS*ueg#4Z)XkD&D^B6Z;E3qTbqMq)osZkk)B8is#;kt_m9={Wos0~Zxx}rDBrW^lKO5q^MZGOl&!ZQcgF(qigwXC@RECX z4aB~v!yzH}R=*i#@hvCB@d}bLxt-Y5EPe7H8Hm;)X1h0D0|xp>5X7R_wjg+>xf?BN zi5)_9)7*bLSG`}}$Ai_hxE2*2d+RcA;J7YmH|t-`%)aVxF;gbG)f>ca7B`X>tmwM{JA}FIcSQM@z&_d6 z2Lr;y7*LRFS(wL)vQx~J9>hO6o?=2znYLb1Jc!rojzDn420aM)JU=peao?UeB})=5+Ud_C}Y&7WK{4o}3OyweD5; zl(IZQx!p8Hplhp4#=Uy!y!+27XeBi)rKECc?|{NfQI!!@ukNk}6535kcht3qUm)V_JOlVdeWyJK@!}(Fe=VTY_StHEcVdS%OkULXJseU4u#kQNe|dNXRyo}gWO2a#Aj>y0Ci*onjJUj;|ba z1`%Qvq37$TAm2jCQ<%Lhfss~O#7D-DT=}%*Gj*g3ZwMNebP)(}^n_)#Mvt!L(kXf` zZ!-U4>Go8I8h)e7fO1Ac-=5>t7BmmiU(V7D2f6WPH;Hqw#r-;bt!6+?zs+@d7+^D@ z@KB+V?E0_mklXE(Q@5qRMD`vn@e8{2^X&pnjy^SK&Nz{4poA;fH~6MeP}XWceAG5s zR1UQN-;^SW^JYI|3bi_i0l;C4mLp|CB*edcoaI!^Zl2E4qnU?GZEVR24!B#tzUrT$ zuhXIFErG?6c0vmc)0NfUhv$n0CKt2`nQB;9T`w!r7%bat7omigfipfo%&D@+AipC+ z4PneAD{GzYB!VIW4V=E*sDz`H+=vHL-$(q(C&*vq%ENjJV){z#|I9@m+p03iI5AO2 z*&#~Kvo~7>*A;XYGBV$VCyn|9D~m2<9G60T>4yb4Z5MFbul6AK2FG&qQm^f%Av%g% zbHY9MCm=z^)?3HPEFt7fpYr}JGpT?>(_A16jTMW2a&CDbDS zi1Fw3fza36hnb8_!5e;?&#HwA`bwhl4IUfUL84Ui7~kQIc84$n=3u@=Kv#vr;Lku{ zUxjm)y*VJ=cKzwK`o9g^A$_)4Ow4Ev0DAozU<+aMD{b^=qcXT#LI#LWiJBebM>XKw3DYV12&g4EjjR+{o_7`#YNQisg7`W)K_f@ z+Ijn@z8-L@l5cFcUfPGkXBol5%t1m^=c@7H{VC5ZC8A+6e9#$r^&WHi&J%W2GgoM&e4&kCIsT8=6(3Iqf5^&v+NJDn zqum$L%l)}b&rJL*Ow*&z^n9Ig;zNv7yl}TPKNXv#U!(#PLDtL<#Kb1h^o`NO3mp#h z0wQ@axpi1W%iY$|b5G^c`obnp0c9WqJ|70-%S?B)r}SG&&;v@}H^3jpZ@ojUHRbOd z)OGy*y{Y09BzRYrNfKa<+t9i_ZLgzi{AHTw0jQZ5sh6wZD}DqO=$CKRqGJHp;R3FQ z6NC?y-&-vb7@@LQ?(TtAePM&H%O4G@EO6sqdGwHxg+D!wUk)-JPriAjS-j29-ymXJSUL-zke;jaM(x98SM+4i`ej0#9JE#V z?M8I8R0CMtWSYSHeC3z#O@W~~q#nZBGR~MK%a6YYDaKE6$v8;m)wBgufJCWJlUx12 zKbwTB`=Z$J`5{r%Q7X2QeW_L!Egef<0T~fXP_Ynecjw9NF9ze-7t3I!F{bxW#2-;9 zZ@0^io~U*<*rC4eJSVufuugIe$(B%%mHQjkPqY z-%`@ZY-yN#xCh|-QTSKg(x1T)glT!>u>2LU^+O)-@_BNvuN6pooRBX`E?!zYAG zjjX}H1C0+2+Tshe4C!IrI-E|uCzQ&OxkG1>y{xnEI5bnBphpPiko-!B7lrlxP zo%JSHzArrLXSL~=3wGdnr}s3=bLGDUU!8udg)`E{;AH_#v7dv2n|QNrgRUsod53DAL=K3WLr$VtbMWPG*-%UW9d=?nwtgr2RE_2Vx$}L1DXhEFuD{_ z46}F`Ic)8Y>p}3%@_^!`r+~qRb$O6!hJf#eo)6-i=CDGvr;oV#n)Nbw_`(g-1G|HC zWA=lsJHoNpmgTT73Ns^9D21lVB@9!B?>m_-tP5}30102oxV(`;s>5DHn5u5BeGhP* zsMB;bcq}!dnvd~HzgXSgDESPD2|LTy^iau(GU<{k2BzNxmv&Gm2 zf$NIw=!LP5SijZH9{A+z?HD;-LS^&NM8@~Z)-WB8m;si}!jA8{HSB)cnA7TrwTg%} zap*?ea)19u@LO#$V6?B+1C*YQ!&balRXhh8^4!U&;S9HZk~muiXA{$|YpD_+s0&#{ zZy(sssCH*3gIe&GlYSA7g0{zeSG*mT{9Fg9z3S`+Z6aW#VV%Smk$I<9iY9AjY+S_Q zJF0{bLt`_EVzxLRNqmc1*}WZ91LwYB-^YM=iG%!B zHe4oc0nq)0YG%l(U4sv}bQD;dSLz!4T3R2%Q zfQyW0%pFIs2mY$T@~^OR)5hDe-uSOD1j7QmKZA~mltL8;HoO${$qa-1*pKf=c)E^b z1=Hpzw)8%WuRxw43_01UuCbHkYy|c9eN_!l(^#`u{JZu^Ps6bD<XNE!$Z&t@wk3j9Yw(5$BO)k+_vQ+{Qjupk z3bvbDDcoLu4OsXuT39fXx9}go>@8U>sR=(E5d(HA-s(1Iem;{;Hv01d6<%zF@D$~* z0d06s`U#H3)wu7VcxkE4rvSY9LL`o5rWTc4j5+feo&ah9`2Lu{k3awX60OB=I&)I8 z+C74%CLEtozN>?eS2dPz;baKFv-@hr!TeJSYv21KzLzlvKcVZlQ#GXPVl<-9Yi9o=kbBGIo~AayC_{tf0Rk#7L+?QntFB0Rde9-gB&; z0kE-AnKJG!9)i;tQ7N%axSQ?j|Fr0G(AI4l8r>gGZ8HQc;MrRJy^qmVT2d?6)=y&aELbY@1Y3yxJ@ zR7zyJJME?8+G3A=v&8hS2y0qzuId#tSuF-WfqCmp2Os=A0M@0yCW_^{zM{R zizKr68_!$1izRzQzTL+gfb7&%0jjsnQdf}r;KK#UxHdOXn3zIeJ}%q@C@_qis?$^8#P!j_62V z+q^0|7(jD;TD|(rSn$P0afvHyni7zE!ai+l&-v`z{~0;I0_>`9ZqVuFOR8=@C#6LK z&$m*Fb(!wZH*#jpnv{RRp7NYSP5hD_(U_c@7@O^hV5(}Rrd>d~R#AdNl_Q@$ncvH8 z)9!I&7G2bAnr2-=?uM>j^#oN%6)7^=GDqfhVNxzNjG}{J)wSvvOXiAdqqbaZVdHbj zUzX+Xj2(}Fz%T|lTe;f$y&$tenzlUWzZL%}=t zW3Me(Yi#qHXi^O+!mzWQ3LMtN@R7gI#-isrTE|NgYBmowfp=y`Jy;g@E--i{Ehd*Q zP}Y=0BwI&g>KzsRCbXDr^x<{;=eIYG^`WBNqvJ*;7HabS&n5>R$Mil;ENT(N`wtxOm7Lt}AjonFkzwJW_~lU6 ztcApu$YS`|rpThs*fmmscgLY!cC3Yz()1{R%3VHZ8(Cbok$ssevlU>bU^7_usy3ss zmd|d^QQaJ7tBW;AqAQPEMfF-}w`9T}`j1-OpMF4{V$1=N*Od9>|QRIE-U2CQw@)L|-d8 zsRvm5Ux-W!UKo~%Z4r`#8>%9s`0%uyI~e~A%Ty8N-JQ2qWS&ANAut)=ZdH846nPHU zlkwHQSJEx5{$sA%FO5zz9t)6wH?a25^2C!A;&cOyIcp4OLe=cDz$gA zuA)Eww}i#t<i*?k}C(3Oj^x+czFF z|4SRb=t%1QS>~jFwvKsVqzl05ZG>Z7=aRL}EM`cRbxIpz+gmw39@<-vp|&PzG7gjT z0&q>jQdEHkzNx+}H1v2x1vnxZ6b4Mg;S`i6D5mOCy^O-Ip$7yf~Zi5`%rG z(~SEic0c^#`4sk4?#65Iwz|vyl+qNuepxY0TRgSq7ch0$QMPZ={>0jAqf>k=Xr3~f z>(s((Vaz1aLmm%Tub6hyyHUzaV(oG=wl9@;td*yCe5+|DOSnCeEomP!sM+!Wp?O$E ztb*A0YGbpKId<0=rEc3q`RNf~YoRlm&%Sg59lk7?E{(vT0B}}Um`TU1HuOW9>Mvh9mY!R{Tn}gd*@f zMW&Yc?!n5>3%2nzZTwhRGC1|#TX4}z9{@kQ7gBxJWjLrocEeK88;U&92Z!eyKeWW8 zbc@weajdmy`Bd39P>fztB? z2NEzH(Ce_F>;zZx^p9&dG}F8&gbt8o_TnyET#f6#x-|x6snPW&CoyXWm~Q15cNaE; z;O^aA?e&>~E~NnNg}lu$GJRGXJw|!L-Bw;sv&%B_3{~g&ojP3p=D{xBCG9z2gW$%g zo7XQ~z3etp{nn>p=eYs6e7A~IniLv)|N9JC`&eu;TP?Ma2 z`Re4SkV*J&@6_GHnY?cTBji+ux_goISnk-pReA;SipvxY>ZjJp&DdGb$8S_f2{ue0 z@PZo}BX_F~NHZ3T*p(e6aVQ>b&vC0Jr&wG$;+XwJ+T?QeyqxGJH}-4#`cOum;gArc zPmwS{787>@r6W9xaqrwPpqKIis~ZzY(s}Rg3`c(@WVtP-qY%8?c)j?!U9fa&eOkjd zMSV&I&QG1J9F|(Qd2#7M475>CG>*CwBu$h5qZ&57z?FYwf>qy8I_-(H@A-|Etgl|P zYM+v?N(qLJdB8L`7N%%6$#vS6q>Nj;=D32MseW2MDzNH!iq-ZU3`*!)LiFTqj5<>y zKodKiip!r(E8|G=Ne6bcNSZ?fUu&4NRm{<^FbU7e;qM^NN{ZSZ1H)8YS{22|3yf$} z|66Z@FNpJw%>Q4**lFJO{$}6)B66#wk*UlhFY&~&4F1GrvQL9B7pZJgKlcQg1>2um zf~GG%JnSnJrcq01sz$E|&r;=`p4xs6qr#ug*FSed%qrt^*27QxY@M;TD?H=?C9cJ4 z3LkH0 zfX7?*O1c~ke~7TxnwQr6`M}I?nnni*9`?TFG20+6=$!(UWv|YCB~{M=nHRESg!+iQ z%7y$M554*doAvP>A=;Gxj!>-60BpBCz8wDgI!I~refXld8gzl3)P~nHfQFZw$@Bs9 z8U{~qnIKS7Zg$?&LdPUbu*PoNd8IR7pql_urFgbhT1mSh9IYRbY){;K=4waR8(kx5 zmN(0;wgX1p+20#U1ip1oItIDDU6#6YmC$Ev3`7Sic?y9+ZnS`mU;RoMh{C_`4GdpZ zYLt6LY&geC)T7!;CxCcb$X*|46IgtPRO}w^cnd2 z^g@AhP|!+#6McP;Q3>0c8Q*oaT|#;I`JI3K5#NOMz_^!MD|j-!Abf&ir8ID|!;Ip7X}G%p}pSV2Q6}_U3)K+2=;>AmL-*Z@DKx;OaBsZS0n1lEEu@ zYch}s*^v@ds<6o}l0#H1_qf?d72r;+kn(JWr9n-~cj#VLS3c}cdH(ia?YI`C-X~V< zkx-*h-7ijf=xFSLN;PBa2iV9k+mai@m%blY-(?+5lrg#ULJ6X(_-XS{e^cY?z=h1N zuQ>X6I&LrZtVCYNs_t$%wVryfc70ZBMmND@xM?%|C3JCn?wrK5_3L%ug^BmMB+J~X zuP6R2yW&lfJu zh}a}O=(@3G;(*g?R<>0>1atw`@5y{Ay@GoY8;Lw2E>WZ$n0IKN-CUWhtsl(O^m7=7 zTD+?bJ#UW6orbzr2U`Uh2Lb8% zK-***y5rj4FC~xU_jH1PfKPCn_sG1=XG>SOEg3iG{_8l>xIILweHxIElZI_XC$EX% zI?nhm`9;1Nu9Pm!D?ZSGU*ZV`kMA4L82w;p)+)gxYFNzk$q5kX zS2Sx4tl6aOnhcP1Lzg~+N-I3veNFrMw|SlMG*ZT;O&GPT<>d}V7HDK3l0a^E5#L}p zL?ZoCET7<~1iHLi;sn(+YLgygWw56jY~Bvs0cTgGRJ%4L?MpXUEom*p@5Dvx1ix|W z=`dsuGb2rqb9Qju!MJ1~=CLhdKH$Q|ZdudbzshwTM%JOR}$yQ z_HakDJ{IHnQDOucYWcKf8kibdeN$+HC@fL6FNz!L^WAln32djv73*XT5LPuiv&kQwTWNbIS2(~@LI_?{;vvxzWTd`}CynSFT zh(U`PN^TpgA?5H}N5#tOLUu^o`4PuM9CCf%k42`>eyyadA9PJxu&>>eW||TKWn4Bq z?2afJT~hT*c-bB3)dJi;&r5*ScnTZiYwzc(1BwMkj1h|;*awmd)npA{#mk`-RF<`O zvoI`nvxhKBT0CKWYk}W8?l?+5vP$prBF@a5nF5jN6xOgPVdSgF!* z?LL29OO&14-gLr_*t~c5;d5ODJ-qq=#dFB({o^e)lIP}tTd@g;%Qpgq=hluI@(N*- z;6ugS`m5VQ?a$%#XAZNGewb%G?>k=Y59I%*>d08_+eHd_d%B;iyV!&J-fc9svo(dc z@A=M}yk6tWa|=Tt7yrqq$MI!OyBijatXQ+H)WL-|BKv5u@2sWkY~kfEwBJ^X5#^Si zY)9g|#H#9Q5E|ays_kUye|yYef|Zb0s_OjLdK$=^#nl;cI_ueADy4wE@T`r;VmZP# zjV-fhYBP7amdZCKaX%~ASp_?%JKKe&*in0}VOAs$D5dr%^3}y}ifS(t?jy#0#g1pI zNPgtpB1GP2K+`%+ZRJp<9{~oI?J?d+o8q~G7TsO}kBE;iK{lUpcUH{W#MA=bU`)hH z`VY*I?)geGi=myMWmi~3_VmZ1!I84I3mzU{#{~`l8&lHm2DaBQNHvHX3L|A(&uoUu zU$*Jq+uf;sTOjaXuJ+aF{UI5h&1Q>l~+((-2>WmM?@n11jsYx=ue3n@4rM=K=+x!730sS<>TVR2sRsqtAE{h zWOZ;?ceTp*`UYJH3A{de5(Z8PY`8b`HGw60?ax9TV^8Ipp7U6+n-0=B0d<3hYk84rVR_VmfTUgi`)SkKhGU?s^5k!o| ze~TM~RX#}h>kP3yH-nmC>TZy{Cr3jgAc7TKKD<&v4%O<}(KvFA<^fw0npX-747uc)~pjo_990Xb?TGe*YQ=W}4 zP2>o{{p~si2Ls6!pYBoWxsw_48y2}%pz-qifxb=?eI@SikFAV8R-w|a>sm7UBITdz z;v3~8CYfD?M$?qgUzp8H?F`#pS9i7YftLS6!3PTF!APR~Mdu`h2jFV|f0z3g`Jk}< zr_I3q2{eR`;&^-VM!{<8R*m~GEVBZ+g7M!bGl_N3|B*woOz~48Iw1$7==FI)?#k@) z;01x%8?XH)1z1P{aoLB&)K3?(WMLDYu2-{8$!te{?gqUuNM+%T0NeL{4FB} zt_0tTPPnyM%HXk4rFk63#^njYl(Ea`77ij$q9-WyRdp8ajN~~r%O^%yk6Ea(-^Vqp zn#^jKBlC-bheZe{>SG%(Vcu?2WKLjhTA$X+fW?&zMq$ZC1~u?NygW%$3nEd}@*t_h zt_;Xzn!gVVDJN2R;alpK_}B3zFJ_sKj?K<#ms12MM%0fpV`BB8faOJ@eIrcDJ#?wk zs9V&Tse44Ytgl2l(^mgoX*loq%M}p+=ho-@@hjk=cGynn3HpP4!b!CUdj=`)y1sjT zM*l;>mhoL%d@-Z*vvnAu10BEAL%5~RF-$2SgK9l7frou-FWR<#LKciS&@isG+cA5h zzy>1;QfGr_9P#73(uLuSror6CcL9%Yt=BCbIrxlf+jG;pu5vJJG(=N4Z?!m+SRw?@ z!>t;#$}38~FZ4h~G+U=Ong=N@H(Q0&)`Q;;Z483DZ!})V=3zF1izhT@285*fQOCa9 z@3{3U0xTDfZNO^X^i+Z{|6aV7XJoxzO+3R86A*?5rr2&@Usqt=)ng4!I*Ketsn48; zT1gVoHLKajSRTl$WKEbQ^G@6 zQ)~(0>vY0kt0%a~u|@h}k@Q|l^gy-Mn3%|~!`nWRVUlO0iP{O7W#I1=6Ry_NfTt6c znt$u(OfyndG0IKj1PN!OsN>ar@5z&Z!{6J-spL z4(H>SzE$yg#%LU?d}1?gi@6+nbFlr>gQ3zB3n$wXkv2-hmkUxd#*;jxy^5k>$Tr*Yp2b6 zgsRtEUO1Jj$=-CJzfeXj*)K3~9a?wPW!Gw}-2aR04()thW71KrEu#s4-SxU42rHBz zTaUk_WBv71j(iqhPd0zvHAlv_tV|o!w5HBaRu@)OrQj`H(e4z16qK;2aO?ii@rT0G z^U+B}#+lPjz!8_zcO8N``Gl@`73ZUa9h~#VkcJXwlpZ57SgWF`DdTe{p8)_ue8s-v6I!y59D7|$^w;vdCc#HYu?nhk02 ziL3goE|cu0m%N-$wyk^3&tg1bF<4H;P`FYWFRBso zZ6(0X&YkSj=K!9KtP6DDX)j^(Vn?szUGyNTJtXYQY@<7pE)rVS4o1jf%|p`b9j&W6 zh+vM~9deNTh^$1Cw;*CL1!1$(Ne~Eb(5F0u!@m2wCu}Wg@t$#U3TOHN_hdJImE52I z4R``USFLJ3KM|FH2E`9iPf7hjJ<;ks3GoC**xx2=_bM<>K zi(op>xc{h7PiNs+jr(s6;x-Uo@8?7`^b~*YBbCEj3`sGUSL-<|6`@Y0v~r)lxh1|Z z!ONBE)kAPTgS=0Sddyq0PZj82FVdK6SmIeNqoUR|?oIMlTEya{%8bI`DX76yfcgGO-5 zcE`cJ(TMSO2r(ilAKn12C#k*@p7M>eig@{a_JOy?1jxH8togF-mAO0Y8{BO20%W*i z0ESgvXKfi1z7o4+bk~c(t{v+V^t&|Dm<94g6-nil_?OC!t-pdnYcY_`C7HRuxr)NH zqfSyLc-@g!tVvGLp$564s74L-7qReoL{wIF===Kip9OtJh|L3^z!CWYw{#5az-{AyEPE22%7otJ zw&|jWKV6NfTefz{s&>?+)9w2;(yZagj~W#`4d?O&!Q=!-(Yo{Grm1weO_tx26mP3} ztj~G*5w^%dc0uUL>1Q)9^Hv&){A31q13yv}&s(_e3MDApF3Br0D-GCE2(;s?J`MJC z$KwMMo@)4oEJ*C@z&)n}*LqV^s3`T1CADJVbnS%_I(@zXk(80AIVkgdOoO}`ElnHo zoHxnYEc{WnmN;n5DC-6Xg+xQ$6ug4^Kq=bg_Kl_;m;WpXrFe^cZ|pvW8(&DnR7BVN z#r(UD^bc_8-#9aHj_W|6|ZUKjT zc-Fu+Zdd+eQAn}IbG^e-4C2ecVEEOvd)1a+;gzi4@+OIxolK?i>SuShuDjK{s?t&& z>;IHQxG4HJ7jUBL98I_I16$t7UEELL9nn9*yY&qnSk{=Zjy>5oUdNJB5v49ReOo+D zN1!e>>zHO@i3JJ*zfHgxv1?GrWUu+Yp~QsuC@XlTZN|lA6dXO+ho$_E7)x6m-GJ1| zt6R+1x!qo^_ZV(ZTcXysH~svB#bc+813vaQhDblXPxzju9DmnmCgloV%C?bnFEUQ< z1S)2|8gLp? zej%%dgH03p_WzE_OqV1Z%p3BEtL^d%Tg`LU1jjtAdqaf8E_>BX%@moQ>!)MkSiRIz zOy%^>sc5GUh}cPfFKtU)vY?j`17}FcRio4bKEsS(-E1}z-{o&(Mfs-5S-xo!bc*A+ zPlT`*srwKlua-$f()%C`98Y(e(~2ZuI;R`K|8S7JVH|I9e=nCeME6}S;q2}^dh#&) zROXHz{*XGTCuR$?pN6gM&jFzeN?G$JpK!|PjJjWT^p9K6DV60(Z zN>8^tU>|!{ia+G-NClRiGF;p|IlV3TZ8|AUwSYPs`MOe7gzMZ*rug+Jpv66L!E^zY$AGhY@DXG(*0Tgq%dQqK_t_jb0 z@n58XZ(z^<>+Y%>^=k^9V6BP8KUd8m8wVS4kN#b~sE{*WRSwmXOtp63BZ~K$b9p(>`HVWM*CQPfhyGf&Qa`_TI7`jU-APq23*qx00gcG z_mJAAS>P8Vl!sQ{HXt|k-M6jH86Z4AS>Rd%NOUJUiYGyPmi`(RX$|-N#VA>QAo0KG zdat&o);HRFxme0llvqKeL@Xd6M7p#fA_5}4NsUPFA~isWE(D~5G?5Z|?JebcM&wY>a8$8y}_E@Htd|4O`DGfZxj6iF@>;_92 zo`2y_kM7gP`}*S8e_MW&k{Q+-FU3^Hu;Yq)i^{E8Kx(k_=bTq!pdxTzET~}G%K9H1 z$P$v_C&=E-ZEG^asj-k0g1#TC0QvT1Rl%kvY@X|KYoep!gh0v2*h~^4`PyQ{3c#cRBDQOS9-HX z@)c6E;v}ypN%y!JmJ`Vxu)QCyz?^9#|P9$D{hCUfj+@JXa43j zT_xDuA`5lf;y$nk-rQkp(>)dru%-YI)vrT9LiDfySjd;DzBprEB$XVBpA27t49kxk zv-D;HirGtC@2pKLFv53bxu?=*D>v`3?6lr|6P=1~Hlx0sJiGZY3K0Xu+23jK?Vkcc zVEj{OKGB-wuqx}X+i6nAPlHB{VGUad zB=H7S51>wnk#dfF1{cniMN@A3sg3o&uUkYHc^qG>ssY~B3%!O;VALB0^M$X>$1jMfRdIH#s2a(-m{-K`FGl2rO{SH&#CW*U=k^V>m% zm4qL_H}1SHRhOpN}@O`Ug|dk&**|1Bb<=TAy(w6bvzW(XNb=qNs_Ms0&t!)EzTgkDhwRw*}L3kd5p_ z`_dD$%}EVxPwr#V%WMq${q`Q$&n#RP3WpfL~uH$*t#u7AUCwRySm{dfF=GRCn}lCerrV}T=YL(x*WZJF+^yO znt}?nZVh#NH%p(0w?2gI{5{W^luX9G+6xe>d?Ik0n|R%yNX#gozhDLgG$L&xr9GbdDl;yOySa$X!{tI%y_Bt3_{Pa19`?7D0hV@AVfDX6s$L5c% zdsHKVw=wTFT26y|m(WUvSrUhJ@$_;+i{Nd!gYkjlN(H~rfNg)=h8#4zG=}%&1JEfi zT4w#mC1miYkq$tSZv1N~$x?nTLky1JI*T$#1LNEtRLLy9nFAf3cCMH2Fn&CwjlkPt zjqHCvw6W2N@$8qq2)pp#@%K~39VGPuo0+%D!i?Ft!A~(R+k@L!=L8w9!DjK3&tdp+ zZL28PQ{Y+0mN}X9d$Uiv1f5_9mqL%Q4{>vE)iUm`FlJwCG-Xl53U<`d4)bdLL$_a3 zLyw93{xw@kT&%Fj7P65EaCpI&>k6E{MZM#n94ORf+P1@bn-$EL4dGtg`DFGZh<@i3 zp>wJm9>W#l(}8*s5_?VSO9<^}XIMpACe6+dNAx9_$~2C9anw@$jamIyz!N|2t@ z<#L|APjghZ`ISff9476g%rg#vG<nPx&b$>io_f5r+i1*5b|NadfJAB=x zl4`Gm9@3xHSnC^DdT9oSQ2o6ranjyR7bdP#KkFa%Swg%hX!HA?uw)|HOFz;R`yCNS1JoE9T$u!?~a>a(tg!@ReD>E zH{Jwd*k|qRtt9)}uUWEtTrX6wZKyIhw5ntM7RJ4v@aQG;Q*bN6Z zaW3dT=CaMtr9i>1N^Xd1*sInJ7FRo@H;QgSpX8JMrmftd!=%L!7H^k{+S|&Kr7oO0 z*=-qy1s3sX9jv2z#S+!CgC}NN6K=LuN-gyo$Zm%C4CThofQWlx(SirF{fpxHB_qh& zW_rsW)FH=w@{KINvDEm2j~xbSEuEMRuCn49iV5;xwyhSr`0u9b+Gf2DkE|?p0DgIka}>xTkI3&iT^1TeT3d6jCk+eqkH5V!n;vN6Wc*^S~a~w`G*$@)coY?`<0`N~RQif07la4NLvt|=2D!1H06V@R+ zIho{0rQWi;rReg1!jJS4Yu_94vSYd-pFRL4SiJ#rGZ0K<#g8%CXWf>O^T~5Z zA7z(4DeP1WelQaH`@+n$97_Bq_khnI19)9(MP+{jn%?ygXz^$L zN&M;W3}&BVeh2h_mO~P=2~HeY45D8mH=Uk=Xh|dRjYTndmpks!LICLx{}p2 z#kZ8UgOp16WN!wHyJ(A?R*t-tN}s`Qjetf+K$%x+GI_YLfa@G&4oP3%ybUfHGf^Ir z$cIE$>qq2hNEoLj@C^SjTcT>fSEu+R;MsucWcNkXkOZTek{FMa6nzk?nv_JDl!YdJa3;}*Y5{ZVit z=;N#bFSR26RobH7+Ow?A1D#y9119yvE0vJ76x7&>XqNY#t$upf#99XoQUikSK2KBD z1q0%!_~CBon>=+D%9m#LAEM-U8^=H{!H)OUSu2v!5XL>m&d1I#mzL{o-v470)bqcD zfiL~tB^daSr zZ4meC-@w-q$ErTZ2^qkvm%}h)l-Oejv6sx2SvW<1lTEJk_Q@{~Z_!HbuOJzzmkS<2 z`x98s%@}FV*lVX5^ze)8;W}OiEpPiGzOqUHYR_Qe-Dbc3=p$LngnMrjaf@dxi+GqP z)&aZ5m2BZ+b@CtH|Bt-wz*w}~hQJEGRy%i8gF*UbU$P=f$`bn1XWO-p&3I?Q{uwjA z^vf=a5iuL=Ntx^!5{Ix(c|QB9<%0Bd3z@I^HL2n_g^bQBz6*4^MzTejoRb-qLA95B zL(Xzf&i9WZcz92(`HJ^`_3JU)ly9x}jV4+57o&4Q3hhP@v@_qHMvp*l26R{78%Hd- z^%l>W@s(KV9cI(*|0`a(x-cM1B_}Z&Uw%4qI~tb#?@55=g(SZyNI%Us;I?C6O+B=K z*Zdhh@Y6GgYs%ja=h1V|+Nj=3a98a1QG~qTRvPMQ^sR=CMjF!|wZ=Mgb2Z7ZU~@vX z`zHKix`1wPTM9TM2|`C0S!R<6(6&Tt_eS zzs^9&kN!E^tzgxD|ZwfYjL7{_R`y zN#v3tupfcuk?-=qisF`gP82+AeT{e*pPQ1LaX54#dtH1z9z zGz_Z^vsWDB+nT(w0ME>+aZHUlcl)A1El6W}Ann1?IlMdbxg@_PuWE)EnBnL|cn$I) zL%0AH^YDQm;Myhp!!-%EkN!^KyAt^2|$*Q?7V*zNy-~Ig{P_&7JqV z+;gmEJR=;*%fIsDQ6;acI=E!0<`q|GhK&5$G$Zf;PAJ`Q8PzwgR{S!eTBCtt@%U++wV^Evr5z5;Gie~}teypHjqwO?+$=NmnZ5uJxK-(9QVPlAW7)P_pv>Pc}(>(%z08vv+` zeW?9Y=xw7;?VK~5zge5ihtH0>uS0b$XRc`}+rI6s_XgAv*{1viIBG_;_V~3|e#}Mt zrJVgO!CeS}7+{p!SzCeiXFh%*<@5&4-uodB8Fx?RFYQUUo@Rv>s2(8!sDr+3XLb}F z@P3*5MpIwW;o`VTg7o;GkuaDo*uT$5wtp4AcZrydHbo|7{}qsy*MzY5cc*{%&*xS}st!l#^9t_y_}Mxt$A z)ftEBGRfvLJbV#Cql#C%42}ddQTxZIc#!)&;#A8p(V+G2NoUiyZAb z_S6QRFt$QxYN`BhFCYJ?ubn*pN)2?13M*TzXRxZM^o#oh<6zr4jS7zme;u}uhx_+~AzrSjC1JT$lV-+rC%>_(uBi-+w z-KVQoI`)ZzkvP;J$@S+p{2)0yx5n%2i%sk|b>GcgR$ZPzr(N?ywFe(1hs7n{y0`x@ zI3jerJizpa`X&3lQ^&_L>lu=Yeb}=Wx^KZ^S~*U%ffxTOm~d`|s&1JIx|!K4Rm6lnyC`YP zT*_&s6w7b6TT^*N2cGZE*mv}4Ngqo+8OHmqh+4V7rQ;qm2A)p(L2Xb;{^}2zdUOG$UA%0?u`tQUjCrVP!l5~AS@!gK@ zZO1*PtNh0~&?McHS3?gOP2-#LN6?pdaKAB$6c!3Z^P>&(Tbp3qF5Fch6@Rz+#6H2# z!hS?OJh9{GE@&As4-ieZXS#E424n!9Ufxk%v>hZwXi`QMgWMA;JGtOC#I^Z@{#?I_ z_|uQT)_Am8Q!VMr6bhj^OV*G#KRIK%EpOg86!&t^2P*U>r^~DNu8vsN`W%=JfU|sT zmR2r6*E0qSz=KL(@5!Rsh(Io%=9wSf%wG;HwiV*v7Y>ec1=y4%*K%l)4KK-xZ<#%9 z0|6?kPgmea+rFqou*_8{^5qF)10I>{c~S$4(q3{p7|n z`I@`Pf9C)%ID|O}9A?m`iGIJw0@vyznQqUcZQ$(lDxZImLRsg@Jc+Ruko=lQq>rrt z?OQ?jLtg`=WDHt0S3VWyRCj#}I#^w+0fkWhvg4>xcl6!DXj4@6Z4Qit6;BRz<)nit zcl8_RH+0zniCJpIvKw<$8hT&I$36Nb$gfjA5H97@*p>WHm!Y%?>-8AAz>!Z)8W~Z3 zdp|mWZK5rtjIwR>F#3o;L*n#t^}c9DV2LRs-{zC&grFUZyj#YvcjW6yR~g~&6!Z;e zSW1yPLG0IGw<JnGisv^MAWJOE2MP3@-jsh+NtaG9+TEqt-&H*_2V$SYQ%w}cF%&Q-i*6tL8 ziMXi+ztN)Ko20F=qnD?Ssy(bppGJY~Q0k`5o(sX|H&SRbn{&1p_|TOEnBv-z;49-% ztUXg$6P+&eUcVf&fg<~{JGRhG-m(*S$@PBK`7zVt%+ndx*2BdX47@sG>moUqlK)Vk z1mT{=SpHp91Io4{mUlh58xdPhT2BX;UomOxNxe}LpY{(n|73rilkYPDL>mG!N5}T! zAOu-;Yn#W@>{z4*N};_;l+6o=>VGvKPAncMstU3!*LArMaZuYa><*E_Xc#{;z(SPP z1I2)xtlJ1Vm$`>4(Tb{vFB?Baz#xC0_~7S8lvQNU&#GAj{RIL88OmAg{izTH^_G2? z&Br}yKfVzBt}Z+Ge#t#_KGe8`^+#H{uq2;E zn0g(VZZ$?z`h#n*J6xN29iL}CL&#>NTE80WSx|tO0toK+?KI!}FHu(8`!aQqo*ln*lKdHShc=1F>#JLQ}bF@}vHaWcbs&(`h!V`A~IIfp8Svt2jp zXZTyxex~pCUqdwU#^+jbc}~&DqN#Q414?#xs*QK+vSao;(-r$f&QwDO&#V6K1bdra zr4ya+X;X_pKE&mCzJ$qnFb`j+utUz!(4rymPn4y7+tcjcO1{8>*}#cS-08tE=J<29 zqC4EdpJ6sD&$<1H)dh9U$Lg17u$)W$QEx=`$Qyyj4bk8d9*c?U5C5EeW{AM)LDl}5 z@-A#El8D)ePIJG57JH&Cqa zkl=;H+`tBl0iAE`wLAu8=oAE-DX^yl5nJ2)RUsZB!54A9;Bp0OMin&RBngt0|*fXog@e= zzyIERxHS$xa&{k*X^c+DN+=1kXtHYFaO@&XF3mN5{ClRAI{5V$OVRQBUFF*ox|^NU zMAPi^SwuzoFkC8b<{Y1)K!u5!uh6Zi2~!e<%>d_7N?%@@xHB02mM(VGxG89Os&JNJ zbCvQtZCr|-;+(qAdfX>Fn}lw<^08_iUFp63D}cp*Vj%)ZKQLt!hg{`af%jT$(J&!m z(ab9s*@BGmX~4nE=I7(+Z~&axg{;0}2E4U8t$WFL3=*R~^TLVLu^oKRHaanWs-yFb z-MrN4!^bBrJGyfpp3_y?=|*8wL0DnGbb3sXmU7_yhDykEh{I)r>P@-ZexB5#hK7-= zjDg>$Jj|COqYzc95B|>@5|tT#eCZoE_|#`xVd~&L{Q`{9;$?GCZm8~EFLN4<<(@YE zOk}<-@j4uCf4@B|@Alin_GlaTRWi^WVrZgJiy@N_3xO@UJqLKV@xw4{T3-h;Jr zPQ|*ju5+)L-rw{g;+}X^+k9)tqaClYr_6y>Y13O>;E3?BIosQOwL>4ZCmo9y+~JCq z$`z`L7EXjpQrH06p~OCnb{*?wwsE<(v^EG`4!U#tQE^2vs8G35zD_BG7PRyH!YSj! zBP!nLn`vTeQU}1~KjJiJ0ppS~khg*vX(n*RlYI8rSW`UsNNY{Y154;0Fk))Wjl=1X zGmvHwN%WR$7SK`4xh3Rpvi;6L|O`Rmhbo%_^f(4@85T>H<|r zLF`t|4jzn#u6!8}x~c#G*=qLnC*8E$>v|($qauhlc@CX!(}SSL&eGC{ZHfGhQ!gI( zU%*)b*LuzBklL+o$JcTnwuOQvsIiM+BylMj3ZHv6M}M=SS5~_q-{ty@H7@niPaF6r zx5t}(FG1_O`GT27BJj4#1hW?o^MF z6axmdhJPt0>GdVgv-UK_Cg~Os&M}S?k)`tn@CGVEs>Wj=-YYGpJ>vFYP^&7;^9NuH zMSr6bM{{Tg3NNFE;-AOk*ys!OW}P!i1tTYgujy~p`d%8{Au(ted>AAacO7M z;fiA|d0ae))bfu!XQe7_E)umZAx|#P#mOF*0ht%s)*9E=tpc*eSz+)1^JthNZP)kR z>g64;`|{d+1-xVHSpi3aaie7MRwqel6s@Yp$xCMFjcfx>hA`Ttewexlo~gZDCi$js z&nq+rR1~sF+~6k=hLr^0MgF_je-=TL0d^dJbv>(#l0J3t;8n=!NNrlENewv2|6QmvGwO@Y}CYGgcve;v^=*~i&U>6x}-Zxfm;D&SzjhO_`DqdUU&1%{+LCt&2yjesX z@Kh6E=+$$P1{p?@45Z zr`uSHi(pWK@65k>`5~#mzRZT2_w|=*4Be`)te>8$vWLt9LpTp7_}gmcpTFAii8%@I(p-+8 zC>Ub`8W7k^7IQnQ|3y+m%n?x2Kh4)jnxytQfBpsZY)~z0=;Gr->+ExPVC|%<9M}cy zSZ8zm%no4v^3t7DtzrN|3}ka{+~nR&;J}6=T02qOF$Uuh>37Gr+gfNqE>fO z5lpL=;5g~)~c zXVks_X54N4!ot$5;!VPD&b8%I7sKhi0fad>h4XQCrLzOf`zK3w^{zc>CeN8e-0De& zljZKkh>FJs5s^JOhFWQlA@DCYz!MkmVc}q-qSVXqs<1355o?9Ow#a9S(z{|c%qB$a9jqEFe_R!{Og zw|=Mn^=0!R_~fiaF6o(Hc-qp>1?qoCbwUio_}u3(i;Y2H)D=qY+eL2mgo@%|Ju87> z)uGR8>N+QdauHzgX5G6)E^{o&45wik|_#D+b7*#Xd}cGyC9g ziG~e;Ol@hAjtm~09YH#aD!|C+iO+DZwsxfZ8S?qIxne{a<<*eqj2Q=R7hnoyCP3b1 zTcXZDo%H9k8mizIlHKOp;*nzD(rp!kokzmldQB%5kXcamL~%HtXV7Fri4W3z@V9#} zN`JeS*gR|s6Q~@w(4`dic{DYUW*-FdKC)mAe3Niaxi3%p)4i0Yo!3*%xDQ+Uk-7U2 z^#ta>rC#lkZ6(qF;(jQiC&Umr7I6U2m2CT?bkm|rD#!os;zhQQpMWqmA3o+gP~G7k zUKa0|G=M=`&n}X?9j((}x1q!nPlk-lm}U@(6k1{-`!hH#z)Z8ZL$|)e?>8)5XRZNp za#}!%7wF)238phjyMnqrncqtN*qJSh5`isw6Ekl08?CWO-VFXuE4$O%_#*z!L2P)@2n_TUamB_?z;^kNMAG zwPs-Iv_JhX22CQ*U8sY>gBrdyBxdb#tn#I1OaO1ZIH@d#;Ol&`tP7_i*%K&=ME>^8 zC%I&t2TO2WNH)!f(T;nfyqqq-2ZGX;Hq=#r{nFxtU`Eg3Tbf>`53x-P$?JB`t7jm6 zcZ1opp*%Dfr7gE@mB#wIs+95_By-xI%mu0(UsLe8lQ)?%CZuSiy4oO(b~wp2QH0ELUcmoy9+k=TRXXUGxwedqhiC%Swj!xi9klnL z(EUg=`ts?!_hZnk%ph3L5g~p$TC8^ZYAV)l2>pG3)7U8*KftOVZK)VumPZ&qH@Uky zfZDj(`X#7{Bw165EUa%2sJBw7?uRHIBTiQL^Rsl&Go|t@R`$+8JAQ_#JX`GN)<7wl z@&f`L{}9cAW5)B;UTf5C&XX{p&C@{VtF#o>r8APQ<2+ zl-M1Yy4G~StaD#t%SQTr@cWNgi1Ag9>=B}1pkSQU;TBSwA>pTMF)bFWMIZ5XYyaFY zFnYr5X=nld*xsPf1Ji_p$0b1nFZbx!|JrXVQ*&LF{0Ix$eOK-R8{K>(W&;QJY9pT- z>pVT*>eo^;$m+c)oh&{9Fr@tatQ*mH6u~1VR+E6p;*~kY7Wls&8_jZ^R%J9@*&M_) zjc!TaNDC1^iZC8KrM$E9x2xgB{Xx+i#G6Sz{P8j^*{TuhDjg>&aThBiKm2i-`)KXf z;k&+=ebk>0FNK>KB)Rz*#qDmF@nN6H=?3?ZihJX@}+ zhdNB_onFV7Nh~q}ZHX3x_2vSc(kfI0I#&`Ne26g{hb?CSEf;)e_5z)%WIE0|et+m} zJDnWHs7B=3E<}R^h{cTf?eOF8Y*UUyCE2C{L>AbbGp^INPS5JO;FaznIJqhJIkp4s zxM9dx*ZIGEB@COr>$>r5Gm@}SxI~37jB^&>yB0x@$#E`m!pIC zRo*bzWIBRR9+XtG0J8HBFdLNF2NsNb?oZeJ>4n;y%DLkGOB!H*;9hi>#9Kkz;B1%x zsO&ZJkHrbfZwavcVN4U=Np_a+7uhy1divP<8E%8>=0w+3;X_>GI2`Z$ywpN$>Y$u9 zUxm;LwmZ-b6;s{-)p>cF*Um} zyofV;tU%?a_|*cgBbpVO*-AxZcn5H%kQg`NyGKkqgZ3Nkjkj&P@Ew>dS$+e-YDIFf z=rO~58vI=qJ+WVqRD%pDprF}u_xLu$ut=!4p6`>=1@nl!v2ZGx15 z6RDnq)QRQqcCg!W$I=y*fSU3M!rE5AjQ)$(LS8b<&5lkz0;$20e{$@}QOagpc52)e z&AFjgw>`r_1l~HgDk9MG11xgm9k%F|D#q);v{!Lakgms%kbG1%>zVS>(L^uv)k~-W zaQsP+jQM8nJmA5ZAa-=N>QJn-ZLP{(mLFG*f>p#b40Z}_3|Z~XZZrYvLkRLtG;?EC z^GtG)B4%x%ZpO=LUNO52(hZTsKfV-b;vI$l0=)jH0XC&}E&ri*Wl<1&R)mjI5pX{e zs3c5l<@j>^is8VWym@c;sXDa(gEMT}^B)Lg<40@eoa`+FyPr*_iqNWlnAD0PHa*^t z*WRkQf~Atf_spO^sfVb4w5q6IFXCh|QpMrB4B}UVB}&?vR`3Wy{m(y&q6a7E+VS@O5S-#9xQZ1f(#r-hyRHY& zsF|Y!x5?Iav+!XPies2YSK5Zexc4VnrL3Ll*Ykuewt6r0#?bPKQK|@Svkk1gQR+tf z&z?^!*sonkAJ%k*=G`(Vv`r@=`Yrs#uP-#ZCi>%9MsR@{#4Tvv4$lzTkG z?8%e}?>wac?b{Kr$!IzJ%PO9;#l|Dvp+~cv?!Qy%J3<6a>_W(P=^x}12%Pj~j8J)K;Dp*4N!D6%{k*qF*|g8k ze73bOIMr%SbH(FEZEu^0t?wt#r;Fj2|DIU!fE>>x4Qg5U6W$5&`BAUv4sxStFHHb{ zd|C1%8d~2>AO; zs&LxvWKh9WnNtvQT06$$k_0;H&B3GYFo)8n0{!V?V&d|1Oz2EJn<{Yq-E`A>6L7UO z*N0RQ+yoIA%Fu96@=>YuHKMdTm*6C<#Yit~J50EpLq1E0`kvl2S}4%3tZw)^GZ5Sp z=6O+br%FnCs>hobAr&#nLnF?D6zwcqG;YS@i#qvVwWb?`r_?0?SQF=&a##JSh{4=o zXbFITg*O08^`{rff60+}Lrt=c1zkcnJ%!7k^CJ3yjQa1z?m<1qAbeNgSOv|!c=C|b-Ey<%LdUbp2*6;5 z2hzJuqpOc_`dFLCyvRqsUZKpHH>n;SooCE6l=IHUTLP1XS6Tmd!4y#3l;2J?(6W~C zDSqQ(O~eFxNawT472Bu?D}2T)49Rfsauh@p10U4-E>E~LOcv6}2tT4kANt-Phn$Gx%?)&xZ+t(R7mAu1fFKT!;`(i*Zyfl=P@2KP); zC6G%7%``x&pEB~YAByjKrNHN8vcq97!q0qdwyx{-sA`jGpFI=*A*k(z^5Iw&hSGq2 zLitqYHX{7+^xy1GnvJ!r##HHs1t|${sp6Zu(Le0Hs>|q|91M0!r1@-XJ)G)$H;tmb zg~1Y_Ch~ZUOIr!%-ei1$^kG6Wd`$sgsRuDZeDSs|4@o|Hlppb0m1N{FVtQ7=7DER0 zKDtyn*5}Fhi>WaN?xZ{BP(%|Lh!O6rN0b)x4M4*egU260$B}F0-x%i zt>@veoM|-oIN%lYwK-^1GixBHj&Poje7`V>UhgCYaT=O4uZkEv z_%SnSV&TB`0RN+y;L{j8p_iS%C|q|v7a1EII405P-vK{tgt&CHvbzYY^0n13B){Z= zzXHb38E5;(>BGyFrzQi1&t}f_A3B6guwF28`|Z(gcWB|2dE`EuS60sF80ulFB!R8 zfB4G~8M{0!uEXEtiqR@r9zM>Sr)|3GoAMEx%fQKX?D!8a+wFPRR2emKy5!qieh$*m z0qUSQ*u;_ zT8>kuZ*4jGH9lJJsSXb851iB^SHY~Po_nEp&?~Q^6jS;eVN-?!1?wBMbWl&Os}0$> zk`tqmZEavB1ehx(gxtU8E3NC*Er$!87_c=M46GPRw2$#os`@&gBUO*E&vyNumb-M# z$zToRrE5PZuT91eCdI+<+;fZG%+#kCs2LuR;REDDy{u@4arYpijb*aXx9}gzJQ{Q| zJ&lUW=8dXdxKmX78fZ_`cMQ^05A2kpk=2B|6Z<(fF4^a)0jc7XV|ST>I_bB_c?6be z?P{8+1dP0Oe7(qX_O~RR_Nuy*{R@;2^}L$U_^QF4i?gw9R9k+b`~>Uh4K$wey8LKT zykx>~zFhH=CgD-Zom(GTa}Ud(N|kv-57`#);QLx<=_&E4sEhh^PzSzv4@v=XCTGLxohIrM9JBGBGAUs9s@;9MP8IgM1^o zAD(DI?u@an3K1=ln)pS$?490im-I8kWqXK?>RLbWk)AdF@FC^mbO*-NKxtL7Y10*4 z{ep9CjCRSa%8$`NkeeK~HL^lqU$M_B@CvI~k&ktl7iNc*K8D+fR(U2L)xD-pk@R7u z5_B`z?Kr_(rn0%372j#VSx?Ga+ktQ58J6^He9;2y3g1YmLytFGqeTiRRz$n!7QG#i zBsd8bFOuXkTH~UW-ba{NMoDf6F`@27!R73DMRldr6Sa(#1PQpdIXDKqMQXI_HtXTU zxfVpJq7R4xMdW(M3xX@KKji)^5IrkZ4f1&C#g;mg1Qmv)@~^cQY!!7RcT&Rqvb8ku z0C~&?S+zhv)!EXkR53j+RY+$#Y?q2tUeA#zF!LE!tZR6x3u#X$&G{+`trF3vh1vd?|GAb&eONfp`U?uLpN^c;2_}hhaiz`X^ z7ce!iwa%2C;@k-%nkIAvX@qje)TfH9b{4Lf%zV0_@5nb;ArL#uYFU+$H##UUl-&j$ zYjir5aBO@!gO(^5HeJ`|SIZJQ4rtSn)*VXTC^$GCy2sbxacHAxJ6uM*<$F{1U*B{L8)o~SM6QGpHzs`-$I@Oz5CXg7d0_*J57Y>1 zk&aKH)%v1w2Z83_X-BP51h;AL0R-Nq4ai9r>BhI^$t%r~B!}ap&HOKEx)-j5hD)y* z^T%J)?l5-l+sFfpOr|}ytq;l7tSnL#Uj0y64$kS+!)OPen@~+P=C@~7)037>GR>8% zJ7{>sD^>kz=h8SVdi`enV6i(Z8}-@$U;GjcCo`GVJXuXZFIK;!#_?lcgrL)(Eyb2tnL4|;_H1B5|Q z_Xjtw*;(dE5d)vA5T^Qgth(^t(vAKGkl~QhfJWHBhiVxuajS_LvIP%fNMamZ0=zWw zq*QGBtVaj0$4{CU5sh7~(UaHDuInrrot_!2$BWTknH2@(WuK9!RJ}?VQ(#(jT6Tvu zWN@5QEF8*z_z>q7{T*qVoK{?Srn}cxPc_%H;Z9_u8HM_?e?bhJ(52N_XD>F6d zc-pU<5|A<=S$#U?MMtW=bYTX%_`BjUk`~jD6#4Cj=2{2_Lyoq9vx4;Y03f}#P|N=- zwKLr3njEk@AeR$G`p+a;+=g!4dB$cfG;O#sOdLGUp)A?c+Y``t1Sb{D@8#w$+~7>f z8dzufRa^eONM&wJ8YIpdKj&9 z-w|~&t^6K*E{^G0%aJyYSB+j4R+5RmckOwdpk+f9YhH;~%f{VMNj-oeWJ?xGX&^n9 z1Pv0R(J|_w6(UNv^|Gb_4m8@s8}m;1a4n!;skTp-<%URgrg0kWhj6Emxm||C!b8y8 ze2;SK^>G)*H@u#lk6~%XRrjh(mBk^8w|;> zZq@zH31GdcE_S4`{Ii-h1@zw~t4CLQil_4-P$%NHWIf6;7~6v+lWQp|}56;rFGa`boVRK2N{h8I^*)82kj}TMcaith0=iUAhN)r8@8k zl@1s`dF;8adUjv_jJNdVVRYyP?tIu>SM)}|UJqKJWkzHOSlK=;)l z33>No+PIAIbuXT1+wZKf8hz#02G;lw(SxadX@1ZL>r_4GHLts+9?M;BK!L*IhDNKY zQeyGHnd{DQebhkUc>87--||oQi<&@o{E$95;qZPz$0xarT-~;X_FRTCYr$d4V^FqmYJ&BN0NzCwX;3r>!xAs_XYB%Vb^gW?{3bk)y!>=EcT_8i@te&U(R34qecjc}TS0hzP=2|yb2-Y^RAdxJTpN-W znT#JZyT0Ok`U2Nit3yYwr??w{-KL*N!{$hZouw-u*UtmXGR;hSEYeK-oO%isA2iyg zYSUlxU?ji@&LO2KstbO&4+nhbYN3ts&2N7(Pj8J9nE)NEx!iG4s5Cr^E$O=cwYOfh z?oTBR&w-sDwP-Aoj**>L@qN;3rVZ{$i^)^)aNs~|f5U^iJgE5e zHF`FyRniOG@e_iE*CvbZ`Pe?iMKE?D{$Zjlv-;J&<)NGgLW5LgKr*UeXWFz5?R1+_ zr3cjGBDg0jpYP#3du+Ac^U%jy>9^9q_=)W#;)*MZi}*yo-&z}&$ubkq0`inGXo0zQ zh*r>~ab1T|xr_v2Fep5Er`(1T8e>5^e8@My;?O4KzGkdKji>F~49X0hewRR-rCL^y zm)qC)2?pdUkSO7#qJ4{}iewOlG_?>uCbnv;c?km^mTG9>9=Y$J2R-LKyhb0LUGP-w zhZ*-sqF9rS*Yk@q2WMiV%A~`IUa=8yu4ko+@SJo%YxGP5G>?u4SMEWb1AYf?HFe%L zhS_EN{`KWp>y9>rL~7|(8)~e=>+X~^;9&yvsKH!sk7`A_-qE2;MrQHJ1|xBF%f)os z)gXO2GbzjmE2Ek*GZZ-NK81wFPVL&}f&8}=b8>oKflUpsCq}p^#Hw>CmBcq5{p?qD z%_tE;t4>8voGrzOdtKW}`c)DN&fL>p^RfkT%E`wPuoUT}gr~*OJJhf&TA<}lc_7-4 zV@+)*KX|S738|vsezm#!kkXk7Go25ef|~qh8m{f$PErl&EeP%-b| zH{4*#c&nZ+)maVPDWV;@V==WbeI*ewvaS`Xp?Xgmt1fX(N;yvG`=h$wzC72daDN=0<<3=cBk7!RP6qGkM%q_Mv2#WG|E=**QhvS5r3aFy1QLCCX`n zCHp@JFZ?d9_Y@u}ptdNM>TVvF*c`gARj%spfHl^t(V={Ys()Sbg&H32#T54|id570 z$wZz4cWmE)M%U!b6K5lW6nTSx($IQ&_78vlc>Azprh)}ScSy<#Q@`vvG3M2TMN1ng z375d8IDF^sDaDsprNw|$t&|5#zE*8G8+n!Q=56!gkH1T#Y?)WAvz1M&7G<#ox?3!l zZLHP4=07?b4qYB#NKWcA$4QD-7-zFKQDaD1Uo+>yR$%(_2&^K#Qqj3}N|kXvy`&RM zE?UV9Q8V8dh^ZE;p4i{<kpRPW!yPXBGQ!A)(5d{Dv$g4zSeY*4?>2CdMz#+RssUgJ@$>}3%Fc$j z3bF(1Unb>h7|)BBXTj_O|K zn^yI(NYf{%GY@T|PB&F>y^UXKDjxY{r72$oRj=f1T7)!U#``FCBkj!~H{Dakab~$o zq!b5D@FQ*Vtww(Zc$6xm1G90~QCnRe>;N%vlP4THREY8Zy)@sRj^HnA%xF<+9S|5TC#b)Ud1@`&^Kmpz^ zlKCr~=^UHco|otZ)?{c(#ytDo?@(+jEPlSEC6KSgn)LuUcUw*OFzT4U`MMGCW*Pgh1c6v;fCU1zhRB<7vLZ)>OKhP^wHnpt_8 z2v8~C%>J>QQ*Ft*9M>+*AEHo$(4c@u)rnV-_RXsITGT3epo%6tL-pd7yU?a+k_jo4t z7s?!lJv{Kp*6mih@8Dt8NRNvc+pe<%2b_j{Q1%wmXRN2wHn9l}5yNk_WxnENth-xB zwhpD;YvP)89pAp33plLSf-0^&Y%z81v)=qw>0><8MZ7XiPs4tMp|eAI=xsLUaT8ubC+`^P#jsb8x#uEP~( zJ~$EF=7fKGctIo=r*p;A{Adv;?aps;%#>T^=V?qD!Eya3{;{&NgxAV~$jD$jy$IW) z^SiZi0NN63Q8q983*s^IwJPY@#!|y+l@X;iD_63!AiAR1KFJ(OdQd9Y6znj?u(A-% zQ951u;B(9s8Cd0sHZA+Po@Bhe(GT8I)2)%zW`boi`Xy+A#S60W!cd7TmxI6 z-*_RHn_zb!$zaQe!@x;7Pe~W7V4pinTAe_-ZC7y04W?F14xlY~o*?(VZB%=5aG^XA zOl0eAtwvl1uL!T6ZuKY|%|n;z)T$)zS4i<@ANC`^iHytCnf?X#G_%NUffKqjKzt^& z-|j}=_PvL-$Y3;q%Z_rDOH%L)6q3Van4+_63mXkpJoWJ?k!%lnv0g4~f0TwO-EzSB z3~Om~$}^`mY|z?@Xe*?{6;r72T)S0k`Z>jh1Bzx3KqJ+Y;&%3cLj@AdFh`vq9Q|>H zG>++#naXgU@LcO;ZYXulACg+|GEs8CA-%NbMb&XcP7Ysl?)vaxrNS$#o~A>7_H{P$ z&=5I0+VR#+Bf^q#QARQ_q|Jz^l%xPXembVIyGESb5^>>{)D2&A>=$@-winO`BG7+h z+KkEyH?mnP8biAZPoeB&QlLkZX9vStC;}5|wT-VTkD25m zrmlm~FBpx9%G^=P85`JbZjt|Qz4euv8{RwN5m~-&q3FnJhV!Ry3*37$9n7i9%%&IR zw6JF1;jVP?KGx{L%1ZK^9WyL))9tyaJ75Xq=*$gzK%RCU(yy z7psd(yc20RnwBAT)DdbBSlnWdH*RU22UYOm0j3dMbC3B6yrQk2im7H=6=Ajxojvxv z@TbCS+q{GnYj<&(qVDElL(DoTgm z*#n$(1exM>nf&X+fgo+#Y41^~ACQ5gH9k`9WrEr*?I&3i!34#H?P0+Cdy4jQk(Zqj zRU7O4ORCUVXxlZa`Q34u1cL51qSO^Vb3(C6wU+;wdKK};W2jM?9LFcguGw7^?W7y9 z89`qk_-yfA*%a_9%FkaiK`p5yg9nQ(sJ#I`2la7do+j*Av+E}?L})lWk0(BZbRpo0JXSTJD@_mQP{{xRh^CMJw-}g z{(330Cz+`~EX|b?I?6(icP2Ju5B_eCdq9iK?vC_Lx8+j-nFgvfmCw3XyGA3vK8<6} zXm#{ae;B8KV2|Vr&nLbSBq>1^Jb39@5Z<&vYf4dB*gblhu4l9EF@%3EPi84hn{DPG z`=mDifm3{hg5(*eSt0ubNpFHvZMO&NVjwU3cT!ahdW=TP66+56W$$o4U5CE18$bel zW$P+LGYVbZ#a%Qn+ni?7>VufQOX;DAEPkap`c5U|AAM-sKo*kZ(kRI^xYqgMYag}? z!z-&<$5O$R5jhfVuGre2$}Qi%J*u~qLXjS0_@>%0E*-VUXZGYFQj;{2nP+ytZCszq zka7U^4P0^%;Pm;~-Z_D!V(%n--{p#43C&@o7OoEYnwf;iVn$SY8yFNSyYETzas*5? zb0iugB$ZW{NEgU6#B9s~xl6ZySyP&FRw<_2t)#0l&y(j270g497Hy#*wN*C$VJ^pO z3239yMq_Yk4ksMSFsjQR^7Gmz1O$veosH$I>s077J(K${^GA(FvAj)7ljv~N?fHSK zKy+{YFaX9rt34B1ZYy7G#j#jNt~ib5wL17d;iXS{Gq-;<9U_dOY|-k0b>8Qlb|O7| z8)Eb&>W5y0g&HJ&Ub7J-_cuQ;jMFg^;Z|T#WE;g%|3kg(o12K=%LNU4>eX_u6af)Y zU3lA5NR6k$zRtQNWq6sE&&v^PDs;X~6KBIG=H_?@=P@SR(C!C>zm-vQR%C5;p~{aR zr7=Lox{e@wH$5aoYGff2@4olQEY)6EZf$~0n8Q~U3FjnZ>whku#VyYrBHr*}(!G7^ zwrVIa^i##gFIs*tN0O>rQqrzut%^DevV;z&OW335{r27vpLc@W?awI=Q^i^7=SMpD zFr+ZjUt{yP(akcK?AriV?bG2-dwD2Cy%X^}BOhe#huJ~$)yT0*0>V(KpfwG~&v?QN z&s2Olz3!Qi=Dl~VRDIqakxprOb)vn(Nadb^0bz%p&Vsl=7rLY$+R0@DM>>a(5pbYc z*f8Kosr>Qk)nZ(2;A{IEO;jCBBY8O98;yBADC)l2YhOE0vR^N%W{u|ze4d!AZ2eWf zI|cTllhK>(Bi~-2f1GKMfZwz%4Ga-=d<%AIxDFO>8m@LNSMm7J;Sj3$ahojhx`()Z zrNSC=sb!P+fVV+UNV7y3Uo;jd-f-ln_HH7(L;&cDa(0@+! zZ+vLKJ)$V(%o}#AL@^25c!ny?ilKr&SeO&_g2#P~>np3cWU08eloTNsDexIVt!mGJ%sI-tv3e8L~n)wWx#qg|

BYJ$w(Vi*g}7+Vmm%P?gaQ^~U| zty-Sjj#rkB(s@T9f(CA~-B-;orJ7KNOBoIQJWoVnkwISwt|NKK$!)a!G4-b&2Tccg z*6o|EkUK=R#=>hc zW92B=?u!%Z{Q+e}rZsY<3|d0u8r7{h_q2kX2QkG_7Qmt&qcYGs(F|>!f~l<=;^u_h z$HlbgFTE}*MGo=`u&`#Bp$`P}gX2;=(Q)X_Xs|D`*1+C(XPT;FHyag>%*@z{PRlDr(|ddu~F(NA=RVV(&{%KXlsLld zf``_gZ3ji5dXlbW{XRCxU+e9fvu{yOwVx+x#e)1PYd6*wQ4V0Nt!pVHR9=JQSZ8B_r$9Q4>s-0H4|nPt1cOaI@n}xu7s+ zLkYR=lBI1DG4U~5zeZ8T$ISuI7Xnt?G;N_<4I+lSK_+PP>>|~Bn1D1Kta#T|T@})`sDyBX@7tjL9sm!)j zTDZg^UVaI|W+LzKh|Po$-~_nI%41lyUW+i1mcrZww>Kv%{CKX#ts=%L zQuVBFOGIN&rzburISzr&m$VKIE=&I1Twe12i!|1(Xx8pPA6~(0DH9WU+W9o#vy@2d z-IHy--yL_+y=$pJ19)rTQPb{-op2v6tvwqnij$9=gohHb~Cr$mhu>#7Gr?kkl2X6oY@_KNEgGg=*Zd?4ZtblzA`;f6C!*P-g1^_0FS zPM5Y4-a4Uz*Ie8+{0!&Ih`%^s>WGgWhm6eRKgC}N#;Tvxo zdqUEKLYknrvKN+0ch=5WxKv64>m%g@xCZ*8oi>{Yv(OU<3Ml7eFsD6Ps#(%gDPitP zR*be+gtv*Z5;iu7WcY20Z<_JEvwN-LRn?V$z;3fj`!>Fqvf87CYQ*}M@}Mm(;C-vJ zsLWv?>W?{F>h0Ehmzlbd-D2j1b6pnkx2CedAFVlBS+i?}5jhLoVsT0S{gVD{s+HaW zZuXnezG&V<4~3-l=@oyg>587*%n{TbsWR`!S%LcWjUD2rCB}Y*f?Uai+yp2)KUSS| z_PM!@>k`AtqoZWQaW_o9xQids5=)L7VMU*tQZapfP6El%teI|ES=*s6W7j}=Uk2@z z3VvwSnMY7i5i_)fgqMd(Fn09?flYI5FtR1_RtDk?CPKrRT8~I3IvkYWuVdMFv{sUZHg!1I^0^crkcjBXXPR;y@Fn%6YJW&=QH;Uu z`Aam{o}8zi*+wZ_)q8OY@qwJk;WkLt;p0rtt443X;D=0(J&jaFCK95G6_2Q4Qf-8} z=*a=mScu&eWM8?13)6JXym|XV%gU0l#-?@tIck#TY777uhBZ1W$O|^S(^IP=7G3IG zUn|}KarAL$XGDRn)6I#il6Udj4;~pSOm{0}EByX5yllryF7H7Z$RVsRQQS7mbWm?^ z3U7ULmx@(ABe)|EwR2}w*{dD3m4!hqPf<)vRY6p0`f zTOsE0(j0E4m761}UmxLiDDt~sLap?B&kjLl27-z1;yo5};-xyZL8YxC`WF3-yDQ<- zJZ9L0_^~4Wv3bPhW~0SN2^#ixZff;C+3Wf*1W{R4`eqT6$n#|^`&!~GS$IA4$RH+1 zW#gfxo1`O&c09=FH5VRw-p+6gFjRJgx&Kt*WYR|7@61~iPZ#oS_)D9zO71TgdDEo{ zd4xkSwC(OX3de|9_GgA8Uzam{}ei2fUjo9LzK$mZ8fQDDxDVMB0y{gDw7 z?X;KI8`<=C42$k36>W2WWm-_;g0?X3tHW(VzWZtNzPvC^(`iS zcjO49V_1nUdm~*fuzOwQ=YpafuEX@9#)R1}%u7y68r8%%XcOj&o^ZWUrawA2#;qsT z=5TOKL^hFinp6@3tt`?_;up^jHw{8jFH^+XSOe!pqHXSyKgfo{?J*Ct;qK|=~w?n}ZowyUsPue*Zjp&bPA&Ai*S>%y0j`I(?QPpWu5UobJiwCxf1i@U92@rRo_~ zWuhj=?95~q=D~e}!pEwDI442$=3(^1dSq^Y41ay4L_r6ahXXD~$O`U^rmcN=K^}f* zg|zPqyGiF5TqNZh7>p(rC{C}yXXdB#ShdUu@JCL*w9h3=su?^NwMbc;YnxJ&EUjs3 zRVgW!mfUvBAu4RHd=J$fu^7qTs6D>EKcqdGorbKW*};Z`&4ZQPDTH}Mxd*=(K(RnK zjSX6~bJI;j`+fxMk1{NNZQx8{ED?x9Ee^2&yR>y5^rLcb#BhZ|E989NT!{jkd0-nz zyvbNi5@=*J<2+3y61*D^>6(3xaYuGVbntE zFUYaO=tHNZ(HABPI4$C%l-3A&!cwqOcD#Ps%Fq%?!C_@O%AM&F+YiW-erApYCfH3y zqS#S#+!7u-1!oB7fJ&OWBm0G#g5<(W=Wn<|b$de4BTyHpPK(VRrvZU2VH{TEY|K^3 zz)%XI5MEo!QEmiiMfNrC@o_ug0iK6`B_@r(o`z5^>kPDyhM10G~B+bRKMC z#ZuMeWz>;GB%!eGran+SKD z-4wJm5z&(4fp_H{?ba&AAv7yFkRk412|avd_d5EPAm?$5TBjAks%9!0oFnRd0P3OZm+>y={EBTs{#73|W22|gLzQJ8HgnB27qgAhU8VOHc zkDQv@@~%2|boeIOA2mVmL}XPkPBoHQAt&XrQaa}q5Ck8i-sqJN`B{c>S0*VDP1Nud zC|bde;X)Y;rmOiDOVGavDk^aM3d&%yXG;LE>>D?1j0e@^AC=qJmGLY1L4o;UNdmo~ zDm@fX9=ux+3&^fd`T|2HWR0Zpk9cI(V{6NXcSq+p_qk(!b&{zw;`^?prLQ5XNc98_ z-`|6xnu-M9DKbx8LDxDM%b8HIA54*CGB{-~tYw4@oo#eYsn{HD}#w~1b{~kkdN{m3enFtNmW$BOkh@m&qV%!(F%A?6tXG9^(&|O^p9@Ji* zdHEq#L(tHZ!+Rp^V$>yh@{KrFfS7)7#Z6FENHx>c!CF4rvBRr%L*Jn)IO*0{H`+W; zB>8mvpbM(t>kZBflet64@`^dyf^TbO2-Xq6|1Imc$`5k|LDSd~YgN8osyHHg8u^Nv z1T59ZE)mbElHXaWON=5m;EbLdNBBsE3zOHDs!H^w>drarjvl0LW`;mgz1=rk^LZtF zY9~rIOLoJ=(Uz>RZXw}kE>AkXPFL%XhMWf~4!3N81-pfK3;u8YCAi0+PK&0ZIH(?v zJBMjEaePy_;;1WcKN53_cvt6ooiR${7FCt3IzbVU-N4bkkp(~H1Mor^Gx~a&sY-AU z3m21slNLVdC_;TILO~Rq9HzIIPp|Bgf|e-SX4}c=Sla5^Ksl+F3sySv=suJFq+RB= zQ_JG59EN05p|yw{77+_=g-n_Jb**)-b(d#TPZ_BEW;$@5_?)YjLF8B9uxg)cbmsv{ zY^#>-3(Uc2{NP3PB-IyaJ(0}>1yl3bMCs&i(hawgd zycMtWjVx#kfl8i0*uda~hPpW!dT0549nr=p=^t4QnI79wYZ{*p?H{^qt7A(!*th3$_EbY3-(N-!*l`B1CI{ zze4a2VJ;5g()G?X1$Gk_^GYO;ABvv^e>8Of8%rApC)-{@?>i|yh4|9&Glhhd3H{&# zr{Bb#1Dz~0LHsaQxNSv8O&z&7o$gePS;nMzHUr36SgY(WDM^d z`4Oo(A4~9S`gW>u(HMiVFu_A91S2N>K@qBk5sIi%1HGzQFU3W(d&hhRBpoT5h}SxBB~vFB~W%k1qokY<}gt{ zZh{PjUzSWiHFTF)aTqT^~@FhdQr$gOlLeDf4z@dm0aQsW z=Nz#*Q|U~<@nnlR!+9m2Lv=dS1i}xBrbb^zEDt339;*dMbP?{24E2^>8FSPiAtZ8q z<6cOLxbAfDG3b!AX=<)j=9qCU1n=)o3D!8z32_;Dh*u>sxt_r6BAASk|5{vNv(ZPB zAD~4mSb=5KO}QVpgW9Eo(>7|2SZ|$+7yJwKK?0xGoHDldRhhD9oK$Jx&!y1{S7E{; zgP>Q6!nVV7^Bpb-m&(B!+Qt`$hd|n@%J(x_|0N8p6zw=HoI!bXFnJ zjCEA<6Nn3sCOgi?@6K}5jfZbfCPKMcuu9+H+8AnQ^ayPw1U+kvc;>7AO=`W%!s(&% zp*RF^M3_U-g1R$|xwIol1qX-0`Z~hz4TT~OyNZ5fhf~@j9QPG~2zb)elji4i)1e@ZhMtD23AaLu*k%i29*yN4?jgY zv3~@hw+WW6Z3Zfs)Up*DgU+iHsK$Gkvs<<6<%nAMgMGsX)ry$qph~Y6o?6Ur3sFxR zj(bbhLhnVX4XT8H=R&zuk;YZb+`=Mm?sY|dIPz-R*=;tCm&>p}qU(dQaO$*&g7r&W!IgD1Zvr|nJDWb-#tpRCCG%PldG-HG$mci5+wgN_Po62 zA2$r6Z76H%AJ?iz!Vgx+u0;%7en;%2Ab#Cry_x@@H(mD8x!$|BWt|n5iYsK{$B;Lm zPUA}??T;bo>>||Fy^))1DGhmYMC+l31z+SsOQqh^d0~Jok_SW5<98V*6^clTuH#L) z9;lpR=`&WRu0kzD{tQsj>zUR+xLS`U&r{&T%VC#OD^K_|IZqF0Nm@Zww(TJy1ZRG1 z#~ulD>O<#QdC5lWg^Ry+=5L>J7Q)bGUy#Tn*Qk)>Smy_dqi4jpVJm1{bcJzi-odMO@Hu4Clq_;ohXg>%Z?Du0x*4S zjCuVf7n0JOgA8_4Wp%6-92x|RB(y<7wbjRh84=aHCNmqNCmHN!`_Kq%dmc7!x{E#| zc+%lGHa=~%)k^OaubAUdxi7edyQLdiBg2E~q)3?u2E$29yvW$rL;Nahut_so`}6v3 z)IsK>VbTJ>49F}(VC_yJFs}$3(x~LF=%CD{OSu~UF_7&IHdEBqS<5*&Fp|&zRb11* zQ#m7BSJS!i6?w#Hu_n2<%BQ)o_t3?f8*R#$dI@IkzPGL{G((AQ%&ogP@uMk$Ri576 zCN)H*T3lKx$WeUMHm#*fM@M0s=OKRvBH46);39E5fZjRR)Dwef@e!&7mBhSCQ=e`T zO-W3hKQ*XpbyKdiQ*DIvO3UHQ(RNlw(z;)kQ@Rb#qImA8^7Snu+xrn?CB^NosML0i_z@~zn~`#yFmtT;;AQS zgo=2cH_Ob`*$lp;BI|DU-%pGEI=AnuHvgEmO;5mxU({YG~mzXErpM_k%Ma8qk77L&G0r08ydU*e08d!Va-9P5GmOC(y#yD?=bKC zu*+Qk#Z{E;HqpEO_(^uNPDYm-uhj@|0p2IUc}q|2*{>~(ED0Zo`kp14Q571BT4O6j z-Z!#tO8;FJeGPVcj3Dy4K1HUBwNJk=;$dXY$sxG*YRjO~yv2x6h7J!G-p9KSIu`zq z2j$q(PicT7+Ci%X9!hLg6_TTH%?xrcTy%_OFF=YT7B-HTRjugxWmPVpW+O{X~n*!UlmJ6?&2 z$q~aLZ!f2Vd@N9*%;>}Lg~qH2(bX4UtDOp+0AO|r!hk+74NOeiyx8$*y?3S zPRA~L`5m&pzaCbVy0ja>dPmz^1_`$DT1Um+e~9#sf$yP?vc6||Tgkz4_{kCLRgQu>( z+}h_#CENjSF=cpV;MVsC!$lT9o}`c&-tFdktB_gEFt993@q+jbKSLnZ33 zXUlufkqyaaf^FD%H}|qvOT}&C`#|@%#BCUJP_Gw{Z3qm=_(p8z`$h4bduyW)YaMPwf63fM#`wSNW5{?`jq%Owz;KMc;bo(4RKHm)WXnKb{?gX8ZH{ zSXBUfF}nZGH<$RE{fF@yP#j(~m%lBXx^b998h$YQlPr1#7Dj2Y5BTF*f4O_o-fL*? zK~WWc*DyWVRu(%}3<21XW-8z5E~O!PbKq&g9~(?ZAih9kNCVkFq?4rwHB;tMnPZF-+BQVC&k|9(rPBsrqEEv{F3(hw~CY6 z-5ud+Ukyg&-&8&w@Z0dG#O)<%s?J3#+f-jaOS($yddg!^Bs7l!dCPIzLzlkZiMo)r z-S|{DH=YL2qdwL;B=-LKDd4Vl6#1ga@T%qU{O6a$1c30&qcZ;CQ-*CIaMNab*q=$5 z%uj`ro(x9-HG64>RY1k}QL)m+gH;HBipYKYcdqV{KOmr6GP0s^osmem%zlxvHNh9O z53}dhj11$z@S|e-W=&=j8L$yE3BMI@kuBUc3?E<6GT7R%hu@MB7n`{ga0`=dr&bg&`VGD~|)a>W^yxr{viY}+<`d3eh zdJDfD1cZ(BZ$sI?)EkLPlmavi7(GTJUG{3Q+M7GP^+G~yCWjsA`y7p4xlMyb2mDb! zq05=vVP$_!=>F>^mUMbXjATn6K48qjkn!UqQ+!izySOyqX1?O;xMEnEt*l>LOv>Qy zYbRkOO>KAQYo-1wiDv`$I-FA104RJWRQ!OLTt;}<_~Ue^ae|B9L~Z9r{u-LtkuIXzdV&`?3>wf|#&`fZhEwtq-k<=fG1?@%$Z z2+{VAe_Y`3ahv<RX$J6dD*3f}TryI+SzL?%0^ArkR1N>tXQQ zo)#YtIc?;tiT78q)xhy5U`*%3Ciu{mDKF#Hy|42ZwCd!~8i zm23bt-#>tat3DzomIJ`-H7)OfdZi;b1Mu18mhSVfi0>74mm*mcgiY!_fj^5~Vs1$J zdD@Nmu6IhYHPHnXgZ5fS>oSyqV;m~IAq-9r01LXc1bnXEg!75=cWlKTR^}8Q1}~DU zx6byJMl*a}DLW_6Cn`a5zJ8@*N93Y(Y1Pe&wU07&E>0Hiw*M>}Zl|$NZ01Qq!xt?9 z&k9$onX~|U>3;M{F|l75`#(3Bi?&@`X$p@`khMD@@^=nLs)N%ZUg79f>nL8g`SQH? z3EfW=aAPNChh4TD`!2et$%~Z+@{uHr@C!sRGVyWvH2e3I;a(PSb-x#B{qmah7Dzas z5jLu60iU3@5v(J)s{vy}D2PEN{IyBrwhxALPa*E%DBh94s8 zg~O2<5zi3N%k7w*EjsHp6pSeXh6-OU01PXl?mg7phG%T$$bj2%n%gTsO3uXg2)kmZpCh43w{v$vTvAT(ROKpokhW6-NxbSd%M20 zn}te~50W$+c8z4+(GMfOEKP;6HET8@$;+tzw4J^^CL=orWUwEBe zu}d8M69+1WM(I+kn}d8IIEC;w4OgRz=Iw>cBJg7(DttG7=2N%s>4jx{(E^|z2fVmX z>^&%~rKMu$?wGEt{+#Su83g^<|HMSkEwW~otETI{GE#sDn$VuQ>n?QwT*5OZxi+$J z3y~uhdFIvfcd202ITI!7#x7>&8s%qbrcXZUGZ{1CNh57RWZ|RWfb(;nY_PB!DMZor zMmpRLyNNdob$5iEG5e0BahfcAen=2Nqzx?^^?AF zRkEVkEB5PVf7f@qzISD&H-4*Fa5nyWJ8B>c)X|Y{>st42`_VNq5swiU#F6&HTO^-} zf^6XKH&Er?>Jr$5aTFUn4(KU%-!|7>`KU!tui5RY>OP8US`GB`-E*bHz0 zgKVghej1YXkU$UiAvM+YrPnCE-tN{K%eLjMk^%YnLEQk&{H3}*4=^AOePa4QHt$Y52f$<~l7;NmPC3yPrUdWY z>_a>~cvfA>gY)B==1^MesTTimPU7sM$M%EJJu~bBOBrp~`cMC$ z=gZ`j|lPr>K5gh5T>HQ*lgEC)T&;IU?G9*3d@XtB zI8GiPxh5aBRGzwa^vefN<#CfZZ(3dUEa-sP{eXa5#PxUgkR zW1jc6ma7W}$?xsfqmdV^@!xqLssErHwlMKDI^Y)Nq%3yZF^r3Z(Lda(>pD-oSDG~} zk&bN>kx}2Bn zYWUNbY2UuPUq-X*$(%BSujQmm8wSh^^}fceq@Z6Vn?+k}@>qkBP8xN-lqv7ST<6Kp%YX@IGWbi?Wf2kVq5}T0$*sO$ zsr`)7CQimFL}_8&!n#a#IsS}^x1L+Q;iOnoUVB+k&1DjW#zO@iGw=IuwMk22+21>< zFB25LdUSaAh>ZUx$beQkvnqmJ60xG)@8E4cu5< z8lMz}U;bc#DBf7^7s11~CRoUX9(WJDz}Sn%Ojq65{0Z#p3dEMW*Vk%;xE~&gAZULC zmRG-O69*wkz#DXunW1TMlwXng? z4RjLr$;O;5cvqMFI!Ouh@RUIB4b6)lm2YK`Z@K9mQ9s2sf#CkcJYw(He4b@J7{DUn zUt@AYdeK*_B(!c6eI_kFJfaZwa7yscd&fI@@>%IcQ@?A2T{;qfZ25h+_exY_)!To5 zMeJNza?Q%=tkHUKO&3ux+?Vy8oRsLR*)NNHKE=&P;v>djVeOAZ0?_g^{)NTf!{nN! z8n20(By#ojqV$QcM)j3y!pg0%J3mD623^8d>y@Z=*{tf7NY5e&TV{kkX9OF1PUGhjY;Z8Z^tgPD zpeFLe(XgrfEa{4(Kbf*@e#|{e={9f4h1|aNkkdMm*7wS{zG`Ues_Rbgaan1O`d1WR z<#JE`66P7g2S1o*+1j^2C|v_F55y!u|A}t@ozeHU1<)J?w+`n@0YelN5_m}f5s!i8 z+o@hjo@|?D(Lndf^}xla!D2pH9HZ;dkQnk8iOC}SDOQXg35H$*N6dv z1fd3|o0_nUuQ5+sGePiGGH;sc?QGVGNB9a_s{8Qefc%wwcH{?-x&J9>ipjYvBWR|_ zNL&3rO!&9N|&=24ceQInlo zGKfXxBGT7PoLXyzJ<}5}q{CC5f&zBUStXEA&~04}sGnqz+A(ja4|Zf17AU9t7f1Ma zY2VxGj(|25m8pwbb?Z^}6Sd})@y9y_zV)nnApUdISAf2<(tGm3b9X`Gr>wQK6$@py z&~&j=p_WCg|05Eg#-6OKV8Q2qYt)4=;kCR|tg`g+@rsGIyhnR4X{&D^)lYfU1EBt{ z^sjChuUQ53KD^r4cfQGssM#%@l{8QZHwzPP|~E^#V4e&>luaTgFjnG zP*VS==y~M6Ya01o9HUaxUR>mlA_MSNOts{e<+r1py*SZuvk?MVL?_^z<0k1m8Qbxh zZK}X-Z8r-k)DW!I2Fkh>N|JC1_H>-{*fEziz(2DI4AvAowtk-YeaJL@yt%S(#iFsT=(DO!(SSPl+#Q)_!6U zcfAC(D)~p!i8gXDN`Hn21!r3*(0W5sLJPbyfkaD1M+jiLX?T}+5P}0yMWO3I()fD1 znsBUnnNx$gLk8)$7eWm>nwk%$9`I`GKk4c025vtA<~>30t=qy@c8}DpuvTil6+hsUyL$AEU&29>s9H1q|1>oAHTnz{{JlgRU!WmYO&)_X!6hZ&zRg1(d@9b$vPO*MgHMg-J61Xx%PkC{{JcQCrbZQ`&2o048EpqAK)euOaD4f zc8NNR{`KQu|Lzy{BK>6)a!GUsG>5)R4Z$z(xdoD(D(Vq5RL7T3x|p4BGcj#H%TABF z6yg1cPeq}gR_&4M#%hrDWSM*3xJzS7JKgu1*Ir!IUH;e~DGn1!8ufk!LiTzp-mh=L znjiQzd>ih4(>z=HRCj3-q_qa3?ov+)mdMyZ8|1cg)s#o`-IbqYFY+7=_g-@Xc97Zc z$mqcG1d!^9f93Sovwuk@B3!Yy_zzJ|t5J#WM&WSpzd8i^BLG0%)fbKUx0Y7u`?-B` z&Ga*G;S5CyrH8-~j~A>jxd-R*VGg5a8N?m&1T|6Wu1Bebw`@l zoH!xjy16CSyx}U2LEgrL_kMTZOfbbqdS|R*#wR>E#EY9hL2+qQV;mbnmHNMuyMIR{ zT>taOeWOT4%tM2tB6!zFShL_g;&hBZNk(M$}AmT)>2X7Gin%qV8@z%Mf5F zxm0~!&$?z$CsDB(!Iy4h=_+rS1c7wn&I4?%;ly6LxW@& z@$?A!Bda41nQ*~a0G#aE?(ni zu8m$HGC>pDzX|UI5{-ae9a9%*I3ZmpGjxTXUIhEnogXkcKOFF(?N5Q9AQGj#lJ@Z0 z7|WKL4qR_~5xm%{h0$*<|CuyA%hfISk56btRZj^hN#h~l*IxEy71F=j;{AWX8T0nK zaB@xac}34QYYlB`hxOYkoMVTmMQhld^_I2TBvP+HKe*hg!Oth-f@|YSHtAN7ET*Y9 zZFxCf61UYlvBTV}TceMnAljk(SNydHctik83JI~YZp$o7C>wfD4J zLH}~;@1KiQ-oNr=-aaEF)@c;orYvePn;t7}6@023HcP#@N%f-HnQc;iTDJtKeN7}$ zd!CVXD&%>Oi^P!Mwdfp_jpCYug)E#j_D|vB>-*>-f5(lPlQMp1n|g{}>k+Optm!9% zo;|m~bW9%vRTnRLncPHHnG=6NRWYoIr^d;`O=N&x*0P>+W@*7R56Q{M7~sy18Lh6U z_ivZ9D)V7D{3df3EU=QP%Tix%vM8ip7MEI}{!yL5)@buw`uI-QmwL+| zQyWE#jG9TKEe{_HdRrFQNM6u%_fUu44sxY~4k1Jo@1IAcNSFERR1snrt|)*efo_;t z5+)Ryi2kb%5O&`%6qD zM1J|{tnk8ed|Uui1kh?o($p&KgCyZr)NlZgD?K6IA;A{1Mre@%{a6>TvT4=ff4y<$ z?cdSYTCuvY^iZX-Nf8PZQwv9g21HAaSp)6%XAD#c4KRdg;<7dKGV5 zMQbly`M0(F8=ELX+&u#(yR-rU<2{#}Nn63QZ_dkt+1UEllhDE`GNl6G`FHa7|0DT# z?{q&U#iXrG;qQGI#s}JD7zKt-wC*{`Aeup+TdV(0N|5k({h^8xP*As0N^le`V``%% z760iDB#&N0(=}=xDFjmOy*6MqnwghI`y1~CZ7syYVG}<@bPs2>4n;~Rl>%9CZX9gV zeNC|=Mbw@3&*Q%SzpFULm2y=O=i&*!pW#-IdWc>V$>#kwW8+PIh5^o8EpBCf`1oVz z>>3z{L3MPVz5^x9`?}WUYy6+RIeq(o#_F)Cg_4ysefOtS%mj{B+ZIv)6d_Wv-|nG$=8eZzVA5~k-{B+EB2afUhw<9_tbbafddv0fSe*89|(B8@15W8 zmw%_<{Neng@kL^K31}vF%F5cWdww`cPR{%NN_Fw6AA(Yq+kF4Nxv*EiM*hTyyxD&b z@82@>Ij{>2GgJn+^Z+uRz))~UqVQiz^oJkH-}w3cH{XnZ2Rvc8?e>OEK5XTsk>8)| zZJN0`Z<~6}-sf_2Q;m<8PhFZTz4rOhKVQ@De|vTQ_H+J+Pv87~wd+9Iz0cRby?*-r zM*n`>lgr|O?fCN&A!21b){9Rci?5n__v-_j-(MeZesgOhWGDdSlmizF4((nCoR@-t zOV5DEtAmEPXGo-O7Qdf-eV&{!REpt&a}T^@++dee|I_^G)6DXdsb#AA{xJOoF^EKe z<9J^6$D^jHH4|d1pQh(TAlgatdGPX%A-DS4tyS}*l_TO sAIt$D`oWH4qpf>TKn%7Hh{KG(|99n_J|`UfaVp4tp00i_>zopr0FG~@egFUf literal 0 HcmV?d00001 diff --git a/models/tacotron2/NeMo/tools/text_processing_deployment/Dockerfile b/models/tacotron2/NeMo/tools/text_processing_deployment/Dockerfile new file mode 100644 index 0000000000..9b2a4ae3e8 --- /dev/null +++ b/models/tacotron2/NeMo/tools/text_processing_deployment/Dockerfile @@ -0,0 +1,38 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for C++ (inverse) text normalization backend Sparrowhawk https://github.com/google/sparrowhawk + +# set base image (host OS) +FROM conda/miniconda3 + +# set the working directory in the container +WORKDIR /workspace + +# install dependencies +RUN conda update -n base -c defaults conda -y +RUN conda install conda-build -y +RUN apt-get update && apt-get install -y --reinstall build-essential && apt-get upgrade -y && apt-get install -y git && apt-get install make +RUN git clone https://github.com/google/re2 +RUN cd re2 && make && make install +RUN apt-get install build-essential -y && apt-get install wget -y +RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz +RUN tar xzvf protobuf-2.5.0.tar.gz +RUN cd protobuf-2.5.0 && ./configure && make && make install && ldconfig +RUN conda install -c conda-forge thrax=1.3.4 -y +RUN git clone https://github.com/yzhang123/sparrowhawk.git +RUN cd sparrowhawk && git checkout test && apt-get install -y autoconf && bash autoreconf && ./configure && make && make install && ldconfig +RUN git clone https://github.com/kward/shunit2.git +RUN echo "DONE" diff --git a/models/tacotron2/NeMo/tools/text_processing_deployment/README.rst b/models/tacotron2/NeMo/tools/text_processing_deployment/README.rst new file mode 100644 index 0000000000..5c1df016d0 --- /dev/null +++ b/models/tacotron2/NeMo/tools/text_processing_deployment/README.rst @@ -0,0 +1,10 @@ +**Text Processing Deployment** +========================================= + +Introduction +------------ + +This folder provides scripts to deploy WFST-based grammars in `NeMo Text Processing `_ for +for production. + +See `documentation `_ for details. \ No newline at end of file diff --git a/models/tacotron2/NeMo/tools/text_processing_deployment/docker/build.sh b/models/tacotron2/NeMo/tools/text_processing_deployment/docker/build.sh new file mode 100644 index 0000000000..fbca6722b5 --- /dev/null +++ b/models/tacotron2/NeMo/tools/text_processing_deployment/docker/build.sh @@ -0,0 +1,18 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +#!/bin/bash + +docker build . --rm -t sparrowhawk \ No newline at end of file diff --git a/models/tacotron2/NeMo/tools/text_processing_deployment/docker/launch.sh b/models/tacotron2/NeMo/tools/text_processing_deployment/docker/launch.sh new file mode 100644 index 0000000000..ddc35cacf7 --- /dev/null +++ b/models/tacotron2/NeMo/tools/text_processing_deployment/docker/launch.sh @@ -0,0 +1,44 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +MODE=${1:-"export"} +SCRIPT_DIR=$(cd $(dirname $0); pwd) +: ${CLASSIFY_DIR:="$SCRIPT_DIR/../classify"} +: ${VERBALIZE_DIR:="$SCRIPT_DIR/../verbalize"} +: ${CMD:=${2:-"/bin/bash"}} + +MOUNTS="" +MOUNTS+=" -v $CLASSIFY_DIR:/workspace/sparrowhawk/documentation/grammars/en_toy/classify" +MOUNTS+=" -v $VERBALIZE_DIR:/workspace/sparrowhawk/documentation/grammars/en_toy/verbalize" + +WORK_DIR="/workspace/sparrowhawk/documentation/grammars" +if [[ $MODE == "test_tn_grammars" ]]; then + CMD="bash test_sparrowhawk_normalization.sh" + WORK_DIR="/workspace/tests" +elif [[ $MODE == "test_itn_grammars" ]]; then + CMD="bash test_sparrowhawk_inverse_text_normalization.sh" + WORK_DIR="/workspace/tests" +fi + +echo $MOUNTS +docker run -it --rm \ + --shm-size=4g \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + $MOUNTS \ + -v $SCRIPT_DIR/../../../tests/nemo_text_processing/:/workspace/tests \ + -w $WORK_DIR \ + sparrowhawk:latest $CMD \ No newline at end of file diff --git a/models/tacotron2/NeMo/tools/text_processing_deployment/export_grammars.sh b/models/tacotron2/NeMo/tools/text_processing_deployment/export_grammars.sh new file mode 100644 index 0000000000..393fca3d5e --- /dev/null +++ b/models/tacotron2/NeMo/tools/text_processing_deployment/export_grammars.sh @@ -0,0 +1,59 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +# This script compiles and exports WFST-grammars from nemo_text_processing, builds C++ production backend Sparrowhawk (https://github.com/google/sparrowhawk) in docker, +# plugs grammars into Sparrowhawk and returns prompt inside docker. +# For inverse text normalization run: +# bash export_grammars.sh --GRAMMARS=itn_grammars +# echo "two dollars fifty" | ../../src/bin/normalizer_main --config=sparrowhawk_configuration.ascii_proto +# For text normalization run: +# bash export_grammars.sh --GRAMMARS=tn_grammars +# echo "\$2.5" | ../../src/bin/normalizer_main --config=sparrowhawk_configuration.ascii_proto +# +# To test TN grammars, run: +# bash export_grammars.sh --GRAMMARS=tn_grammars --MODE=test +# +# To test ITN grammars, run: +# bash export_grammars.sh --GRAMMARS=itn_grammars --MODE=test + +GRAMMARS="itn_grammars" # tn_grammars +INPUT_CASE="cased" # lower_cased, only for tn_grammars +MODE="" + +for ARG in "$@" +do + key=$(echo $ARG | cut -f1 -d=) + value=$(echo $ARG | cut -f2 -d=) + + if [[ $key == *"--"* ]]; then + v="${key/--/}" + declare $v="${value}" + fi +done + +echo "GRAMMARS = $GRAMMARS" +echo "MODE = $MODE" + +python pynini_export.py --output_dir=. --grammars=${GRAMMARS} --input_case=${INPUT_CASE} || exit 1 +find . -name "Makefile" -type f -delete +bash docker/build.sh + +if [[ $MODE == "test" ]]; then + MODE=${MODE}_${GRAMMARS} +fi + +bash docker/launch.sh $MODE \ No newline at end of file diff --git a/models/tacotron2/NeMo/tools/text_processing_deployment/pynini_export.py b/models/tacotron2/NeMo/tools/text_processing_deployment/pynini_export.py new file mode 100644 index 0000000000..80ccace9f0 --- /dev/null +++ b/models/tacotron2/NeMo/tools/text_processing_deployment/pynini_export.py @@ -0,0 +1,111 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import time +from argparse import ArgumentParser +from typing import Dict + +from nemo_text_processing.inverse_text_normalization.taggers.tokenize_and_classify import ClassifyFst as ITNClassifyFst +from nemo_text_processing.inverse_text_normalization.verbalizers.verbalize import VerbalizeFst as ITNVerbalizeFst +from nemo_text_processing.text_normalization.taggers.tokenize_and_classify import ClassifyFst as TNClassifyFst +from nemo_text_processing.text_normalization.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + +from nemo.utils import logging + +try: + import pynini + from pynini.export import export + + PYNINI_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + + logging.warning( + "`pynini` is not installed ! \n" + "Please run the `nemo_text_processing/setup.sh` script" + "prior to usage of this toolkit." + ) + + PYNINI_AVAILABLE = False + + +# This script exports compiled grammars inside nemo_text_processing into OpenFst finite state archive files tokenize_and_classify.far and verbalize.far for production purposes + + +def _generator_main(file_name: str, graphs: Dict[str, pynini.FstLike]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graph: Pynini WFST graph to be exported + rule_name: rule name for graph in created FAR file + + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + print(f'Created {file_name}') + + +def itn_grammars(**kwargs): + d = {} + d['classify'] = {'TOKENIZE_AND_CLASSIFY': ITNClassifyFst().fst} + d['verbalize'] = {'ALL': ITNVerbalizeFst().fst, 'REDUP': pynini.accep("REDUP")} + return d + + +def tn_grammars(**kwargs): + d = {} + d['classify'] = {'TOKENIZE_AND_CLASSIFY': TNClassifyFst(input_case=kwargs["input_case"], deterministic=True).fst} + d['verbalize'] = {'ALL': TNVerbalizeFst(deterministic=True).fst, 'REDUP': pynini.accep("REDUP")} + return d + + +def export_grammars(output_dir, grammars): + """ + Exports tokenizer_and_classify and verbalize Fsts as OpenFst finite state archive (FAR) files. + + Args: + output_dir: directory to export FAR files to. Subdirectories will be created for tagger and verbalizer respectively. + """ + + for category, graphs in grammars.items(): + out_dir = os.path.join(output_dir, category) + if not os.path.exists(out_dir): + os.makedirs(out_dir) + time.sleep(1) + if category == "classify": + category = "tokenize_and_classify" + _generator_main(f"{out_dir}/{category}.far", graphs) + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument("--output_dir", help="output directory for grammars", required=True, type=str) + parser.add_argument( + "--grammars", help="grammars to be exported", choices=["tn_grammars", "itn_grammars"], type=str, required=True + ) + parser.add_argument( + "--input_case", help="input capitalization", choices=["lower_cased", "cased"], default="cased", type=str + ) + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + export_grammars(output_dir=args.output_dir, grammars=locals()[args.grammars](input_case=args.input_case)) diff --git a/models/tacotron2/NeMo/tutorials/00_NeMo_Primer.ipynb b/models/tacotron2/NeMo/tutorials/00_NeMo_Primer.ipynb new file mode 100644 index 0000000000..db3bdfa814 --- /dev/null +++ b/models/tacotron2/NeMo/tutorials/00_NeMo_Primer.ipynb @@ -0,0 +1,1281 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "00_NeMo_Primer.ipynb", + "provenance": [], + "collapsed_sections": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "7LfkL2r2Q1tr" + }, + "source": [ + "# Getting Started: Exploring Nemo Fundamentals\n", + "\n", + "NeMo is a toolkit for creating [Conversational AI](https://developer.nvidia.com/conversational-ai#started) applications.\n", + "\n", + "NeMo toolkit makes it possible for researchers to easily compose complex neural network architectures for conversational AI using reusable components - Neural Modules. Neural Modules are conceptual blocks of neural networks that take typed inputs and produce typed outputs. Such modules typically represent data layers, encoders, decoders, language models, loss functions, or methods of combining activations.\n", + "\n", + "The toolkit comes with extendable collections of pre-built modules and ready-to-use models for automatic speech recognition (ASR), natural language processing (NLP) and text synthesis (TTS). Built for speed, NeMo can utilize NVIDIA's Tensor Cores and scale out training to multiple GPUs and multiple nodes.\n", + "\n", + "For more information, please visit https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/#" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "zLSy94NEQi-e" + }, + "source": [ + "\"\"\"\n", + "You can run either this notebook locally (if you have all the dependencies and a GPU) or on Google Colab.\n", + "\n", + "Instructions for setting up Colab are as follows:\n", + "1. Open a new Python 3 notebook.\n", + "2. Import this notebook from GitHub (File -> Upload Notebook -> \"GITHUB\" tab -> copy/paste GitHub URL)\n", + "3. Connect to an instance with a GPU (Runtime -> Change runtime type -> select \"GPU\" for hardware accelerator)\n", + "4. Run this cell to set up dependencies.\n", + "\"\"\"\n", + "# If you're using Google Colab and not running locally, run this cell.\n", + "\n", + "## Install dependencies\n", + "!pip install wget\n", + "!apt-get install sox libsndfile1 ffmpeg\n", + "!pip install unidecode\n", + "\n", + "# ## Install NeMo\n", + "BRANCH = 'v1.0.2'\n", + "!python -m pip install git+https://github.com/NVIDIA/NeMo.git@$BRANCH#egg=nemo_toolkit[all]\n", + "\n", + "## Install TorchAudio\n", + "!pip install torchaudio>=0.6.0 -f https://download.pytorch.org/whl/torch_stable.html\n", + "\n", + "## Grab the config we'll use in this example\n", + "!mkdir configs" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6G2TZkaxcM0e" + }, + "source": [ + "## Foundations of NeMo\n", + "---------\n", + "\n", + "NeMo models leverage [PyTorch Lightning](https://github.com/PyTorchLightning/pytorch-lightning) Module, and are compatible with the entire PyTorch ecosystem. This means that users have the full flexibility of using the higher level APIs provided by PyTorch Lightning (via Trainer), or write their own training and evaluation loops in PyTorch directly (by simply calling the model and the individual components of the model).\n", + "\n", + "For NeMo developers, a \"Model\" is the neural network(s) as well as all the infrastructure supporting those network(s), wrapped into a singular, cohesive unit. As such, all NeMo models are constructed to contain the following out of the box (at the bare minimum, some models support additional functionality too!) - \n", + "\n", + " - Neural Network architecture - all of the modules that are required for the model.\n", + "\n", + " - Dataset + Data Loaders - all of the components that prepare the data for consumption during training or evaluation.\n", + "\n", + " - Preprocessing + Postprocessing - all of the components that process the datasets so they can easily be consumed by the modules.\n", + "\n", + " - Optimizer + Schedulers - basic defaults that work out of the box, and allow further experimentation with ease.\n", + "\n", + " - Any other supporting infrastructure - tokenizers, language model configuration, data augmentation etc.\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "XxAwtqWBQrNk" + }, + "source": [ + "import nemo\n", + "nemo.__version__" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "H01SHfKQh-gV" + }, + "source": [ + "## NeMo Collections\n", + "\n", + "NeMo is sub-divided into a few fundamental collections based on their domains - `asr`, `nlp`, `tts`. When you performed the `import nemo` statement above, none of the above collections were imported. This is because you might not need all of the collections at once, so NeMo allows partial imports of just one or more collection, as and when you require them.\n", + "\n", + "-------\n", + "Let's import the above three collections - " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "J09NNa8fhth7" + }, + "source": [ + "import nemo.collections.asr as nemo_asr\n", + "import nemo.collections.nlp as nemo_nlp\n", + "import nemo.collections.tts as nemo_tts" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bSvYoeBrjPza" + }, + "source": [ + "## NeMo Models in Collections\n", + "\n", + "NeMo contains several models for each of its collections, pertaining to certain common tasks involved in conversational AI. At a brief glance, let's look at all the Models that NeMo offers for the above 3 collections." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "9LbbC_92i41f" + }, + "source": [ + "asr_models = [model for model in dir(nemo_asr.models) if model.endswith(\"Model\")]\n", + "asr_models" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "t5_ax9Z8j9FC" + }, + "source": [ + "nlp_models = [model for model in dir(nemo_nlp.models) if model.endswith(\"Model\")]\n", + "nlp_models" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "bQdR6RJdkezq" + }, + "source": [ + "tts_models = [model for model in dir(nemo_tts.models) if model.endswith(\"Model\")]\n", + "tts_models" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iWKxKQnSkj9Z" + }, + "source": [ + "## The NeMo Model\n", + "\n", + "Let's dive deeper into what a NeMo model really is. There are many ways we can create these models - we can use the constructor and pass in a config, we can instantiate the model from a pre-trained checkpoint, or simply pass a pre-trained model name and instantiate a model directly from the cloud !\n", + "\n", + "---------\n", + "For now, let's try to work with an ASR model - [Citrinet](https://arxiv.org/abs/2104.01721)" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "n-XOQaW1kh3v" + }, + "source": [ + "citrinet = nemo_asr.models.EncDecCTCModelBPE.from_pretrained('stt_en_citrinet_512')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "YP4X7KVPli6g" + }, + "source": [ + "citrinet.summarize();" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MB91Swu0pIKr" + }, + "source": [ + "## Model Configuration using OmegaConf\n", + "--------\n", + "\n", + "So we could download, instantiate and analyse the high level structure of the `Citrinet` model in a few lines! Now let's delve deeper into the configuration file that makes the model work.\n", + "\n", + "First, we import [OmegaConf](https://omegaconf.readthedocs.io/en/latest/). OmegaConf is an excellent library that is used throughout NeMo in order to enable us to perform yaml configuration management more easily. Additionally, it plays well with another library, [Hydra](https://hydra.cc/docs/intro/), that is used by NeMo to perform on the fly config edits from the command line, dramatically boosting ease of use of our config files !" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "RkgrDJvumFER" + }, + "source": [ + "from omegaconf import OmegaConf" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CktakfBluA56" + }, + "source": [ + "All NeMo models come packaged with their model configuration inside the `cfg` attribute. While technically it is meant to be config declaration of the model as it has been currently constructed, `cfg` is an essential tool to modify the behaviour of the Model after it has been constructed. It can be safely used to make it easier to perform many essential tasks inside Models. \n", + "\n", + "To be doubly sure, we generally work on a copy of the config until we are ready to edit it inside the model" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ISd6z7sXt9Mm" + }, + "source": [ + "import copy" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "N2_SiLHRve8A" + }, + "source": [ + "cfg = copy.deepcopy(citrinet.cfg)\n", + "print(OmegaConf.to_yaml(cfg))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W_V3e3W7vqOb" + }, + "source": [ + "## Analysing the contents of the Model config\n", + "----------\n", + "\n", + "Above we see a configuration for the Citrinet model. As discussed in the beginning, NeMo models contain the entire definition of the neural network(s) as well as most of the surrounding infrastructure to support that model within themselves. Here, we see a perfect example of this behaviour.\n", + "\n", + "Citrinet contains within its config - \n", + "\n", + "- `preprocessor` - MelSpectrogram preprocessing layer\n", + "- `encoder` - The acoustic encoder model.\n", + "- `decoder` - The CTC decoder layer.\n", + "- `optim` (and potentially `sched`) - Optimizer configuration. Can optionally include Scheduler information.\n", + "- `spec_augment` - Spectrogram Augmentation support.\n", + "- `train_ds`, `validation_ds` and `test_ds` - Dataset and data loader construction information." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sIwhdXkwxn6R" + }, + "source": [ + "## Modifying the contents of the Model config\n", + "----------\n", + "\n", + "Say we want to experiment with a different preprocessor (we want MelSpectrogram, but with different configuration than was provided in the original configuration). Or say we want to add a scheduler to this model during training. \n", + "\n", + "OmegaConf makes this a very simple task for us!" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "WlSZ8EA4yGKo" + }, + "source": [ + "# OmegaConf won't allow you to add new config items, so we temporarily disable this safeguard.\n", + "OmegaConf.set_struct(cfg, False)\n", + "\n", + "# Let's see the old optim config\n", + "print(\"Old Config: \")\n", + "print(OmegaConf.to_yaml(cfg.optim))\n", + "\n", + "sched = {'name': 'CosineAnnealing', 'warmup_steps': 1000, 'min_lr': 1e-6}\n", + "sched = OmegaConf.create(sched) # Convert it into a DictConfig\n", + "\n", + "# Assign it to cfg.optim.sched namespace\n", + "cfg.optim.sched = sched\n", + "\n", + "# Let's see the new optim config\n", + "print(\"New Config: \")\n", + "print(OmegaConf.to_yaml(cfg.optim))\n", + "\n", + "# Here, we restore the safeguards so no more additions can be made to the config\n", + "OmegaConf.set_struct(cfg, True)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-nMDN66502kn" + }, + "source": [ + "## Updating the model from config\n", + "----------\n", + "\n", + "NeMo Models can be updated in a few ways, but we follow similar patterns within each collection so as to maintain consistency.\n", + "\n", + "Here, we will show the two most common ways to modify core components of the model - using the `from_config_dict` method, and updating a few special parts of the model.\n", + "\n", + "Remember, all NeMo models are PyTorch Lightning modules, which themselves are PyTorch modules, so we have a lot of flexibility here!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qrKzFYkZ20aa" + }, + "source": [ + "### Update model using `from_config_dict`\n", + "\n", + "In certain config files, you will notice the following pattern : \n", + "\n", + "```yaml\n", + "preprocessor:\n", + " _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor\n", + " normalize: per_feature\n", + " window_size: 0.02\n", + " sample_rate: 16000\n", + " window_stride: 0.01\n", + " window: hann\n", + " features: 64\n", + " n_fft: 512\n", + " frame_splicing: 1\n", + " dither: 1.0e-05\n", + " stft_conv: false\n", + "```\n", + "\n", + "You might ask, why are we using `_target_`? Well, it is generally rare for the preprocessor, encoder, decoder and perhaps a few other details to be changed often from the command line when experimenting. In order to stabilize these settings, we enforce that our preprocessor will always be of type `AudioToMelSpectrogramPreprocessor` for this model by setting its `_target_` attribute in the config. In order to provide its parameters in the class constructor, we simply add them after `_target_`.\n", + "\n", + "---------\n", + "Note, we can still change all of the parameters of this `AudioToMelSpectrogramPreprocessor` class from the command line using hydra, so we don't lose any flexibility once we decide what type of preprocessing class we want !\n", + "\n", + "This also gives us a flexible way to instantiate parts of the model from just the config object !" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "1Be08R4szkT3" + }, + "source": [ + "new_preprocessor_config = copy.deepcopy(cfg.preprocessor)\n", + "new_preprocessor = citrinet.from_config_dict(new_preprocessor_config)\n", + "print(new_preprocessor)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UzJQ7Y8H4S_U" + }, + "source": [ + "So how do we actually update our model's internal preprocessor with something new? Well, since NeMo Model's are just pytorch Modules, we can just replace their attribute !" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "WdtnPKX84OJ-" + }, + "source": [ + "citrinet.preprocessor = new_preprocessor" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "OMz2KR-24xTO" + }, + "source": [ + "citrinet.summarize();" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gPb_BdPN40Ro" + }, + "source": [ + "--------\n", + "This might look like nothing changed - because we didn't actually modify the config for the preprocessor at all ! But as we showed above, we can easily modify the config for the preprocessor, instantiate it from config, and then just set it to the model." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IV8WKJkD5E_Q" + }, + "source": [ + "-------\n", + "**NOTE**: Preprocessors don't generally have weights, so this was easy, but say we want to replace a part of the model which actually has trained parameters?\n", + "\n", + "Well, the above approach will still work, just remember the fact that the new module you inserted into `citrinet.encoder` or `citrinet.decoder` actually won't have pretrained weights. You can easily rectify that by loading the state dict for the module *before* you set it to the Model though!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YplQcgfG6S1U" + }, + "source": [ + "### Preserving the new config\n", + "\n", + "So we went ahead and updated the preprocessor of the model. We however also need to perform a crucial step - **preserving the updated config**!\n", + "\n", + "Why do we want to do this? NeMo has many ways of saving and restoring its models, which we will discuss a bit later. All of them depend on having an updated config that defines the model in its entirety, so if we modify anything, we should also update the corresponding part of the config to safely save and restore models." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "dsxQHBV86R4a" + }, + "source": [ + "# Update the config copy\n", + "cfg.preprocessor = new_preprocessor_config\n", + "# Update the model config\n", + "citrinet.cfg = cfg" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eXRRBnJk5tCv" + }, + "source": [ + "## Update a few special components of the Model\n", + "---------\n", + "\n", + "While the above approach is good for most major components of the model, NeMo has special utilities for a few components.\n", + "\n", + "They are - \n", + "\n", + " - `setup_training_data`\n", + " - `setup_validation_data` and `setup_multi_validation_data`\n", + " - `setup_test_data` and `setup_multi_test_data`\n", + " - `setup_optimization`\n", + "\n", + "These special utilities are meant to help you easily setup training, validation, testing once you restore a model from a checkpoint.\n", + "\n", + "------\n", + "One of the major tasks of all conversational AI models is fine-tuning onto new datasets - new languages, new corpus of text, new voices etc. It is often insufficient to have just a pre-trained model. So these setup methods are provided to enable users to adapt models *after* they have been already trained or provided to you.\n", + "\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "B7Y7wt2x9goJ" + }, + "source": [ + "You might remember having seen a few warning messages the moment you tried to instantiate the pre-trained model. Those warnings are in fact reminders to call the appropriate setup methods for the task you want to perform. \n", + "\n", + "Those warnings are simply displaying the old config that was used to train that model, and are a basic template that you can easily modify. You have the ability to modify the `train_ds`, `validation_ds` and `test_ds` sub-configs in their entirety in order to evaluate, fine-tune or train from scratch the model, or any further purpose as you require it.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1hXXdaup-QmG" + }, + "source": [ + "Let's discuss how to add the scheduler to the model below (which initially had just an optimizer in its config)" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "cveKWvMZ4zBo" + }, + "source": [ + "# Let's print out the current optimizer\n", + "print(OmegaConf.to_yaml(citrinet.cfg.optim))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "XVguw3k0-f6b" + }, + "source": [ + "# Now let's update the config\n", + "citrinet.setup_optimization(cfg.optim);" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1JZBCQeW-21X" + }, + "source": [ + "-------\n", + "We see a warning - \n", + "\n", + "```\n", + "Neither `max_steps` nor `iters_per_batch` were provided to `optim.sched`, cannot compute effective `max_steps` !\n", + " Scheduler will not be instantiated !\n", + "```\n", + "\n", + "We don't have a train dataset setup, nor do we have max_steps in the config. Most NeMo schedulers cannot be instantiated without computing how many train steps actually exist!\n", + "\n", + "Here, we can temporarily allow the scheduler construction by explicitly passing a max_steps value to be 100" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "mqC89hfE-tqf" + }, + "source": [ + "OmegaConf.set_struct(cfg.optim.sched, False)\n", + "\n", + "cfg.optim.sched.max_steps = 100\n", + "\n", + "OmegaConf.set_struct(cfg.optim.sched, True)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "r22IqOBK_q6l" + }, + "source": [ + "# Now let's update the config and try again\n", + "citrinet.setup_optimization(cfg.optim);" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "U7Eezf_sAVS0" + }, + "source": [ + "You might wonder why we didnt explicitly set `citrinet.cfg.optim = cfg.optim`. \n", + "\n", + "This is because the `setup_optimization()` method does it for you! You can still update the config manually." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "THqhXy_lQ7i8" + }, + "source": [ + "### Optimizer & Scheduler Config\n", + "\n", + "Optimizers and schedulers are common components of models, and are essential to train the model from scratch.\n", + "\n", + "They are grouped together under a unified `optim` namespace, as schedulers often operate on a given optimizer.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6HY51nuoSJs5" + }, + "source": [ + "### Let's breakdown the general `optim` structure\n", + "```yaml\n", + "optim:\n", + " name: novograd\n", + " lr: 0.01\n", + "\n", + " # optimizer arguments\n", + " betas: [0.8, 0.25]\n", + " weight_decay: 0.001\n", + "\n", + " # scheduler setup\n", + " sched:\n", + " name: CosineAnnealing\n", + "\n", + " # Optional arguments\n", + " max_steps: null # computed at runtime or explicitly set here\n", + " monitor: val_loss\n", + " reduce_on_plateau: false\n", + "\n", + " # scheduler config override\n", + " warmup_steps: 1000\n", + " warmup_ratio: null\n", + " min_lr: 1e-9\n", + "```\n", + "\n", + "Essential Optimizer components - \n", + "\n", + " - `name`: String name of the optimizer. Generally a lower case of the class name.\n", + " - `lr`: Learning rate is a required argument to all optimizers.\n", + "\n", + "Optional Optimizer components - after the above two arguments are provided, any additional arguments added under `optim` will be passed to the constructor of that optimizer as keyword arguments\n", + "\n", + " - `betas`: List of beta values to pass to the optimizer\n", + " - `weight_decay`: Optional weight decay passed to the optimizer.\n", + "\n", + "Optional Scheduler components - `sched` is an optional setup of the scheduler for the given optimizer.\n", + "\n", + "If `sched` is provided, only one essential argument needs to be provided : \n", + "\n", + " - `name`: The name of the scheduler. Generally, it is the full class name.\n", + "\n", + "Optional Scheduler components - \n", + "\n", + " - `max_steps`: Max steps as an override from the user. If one provides `trainer.max_steps` inside the trainer configuration, that value is used instead. If neither value is set, the scheduler will attempt to compute the `effective max_steps` using the size of the train data loader. If that too fails, then the scheduler will not be created at all.\n", + "\n", + " - `monitor`: Used if you are using an adaptive scheduler such as ReduceLROnPlateau. Otherwise ignored. Defaults to `loss` - indicating train loss as monitor.\n", + "\n", + " - `reduce_on_plateau`: Required to be set to true if using an adaptive scheduler.\n", + "\n", + "Any additional arguments under `sched` will be supplied as keyword arguments to the constructor of the scheduler.\n", + "\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "V3pQM2aj_6WX" + }, + "source": [ + "## Difference between the data loader setup methods\n", + "----------\n", + "\n", + "You might notice, we have multiple setup methods for validation and test data sets. We also don't have an equivalent `setup_multi_train_data`. \n", + "\n", + "In general, the `multi` methods refer to multiple data sets / data loaders. \n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "g33nMx9WCJdj" + }, + "source": [ + "### Where's `setup_multi_train_data`?\n", + "With the above in mind, let's tackle why we don't have `setup_multi_train_data`. \n", + "\n", + "NeMo is concerned with multiple domains - `asr`, `nlp` and `tts`. The way datasets are setup and used in these domains is dramatically different. It is often unclear what it means to have multiple train datasets - do we concatenate them? Do we randomly sample (with same or different probability) from each of them? \n", + "\n", + "Therefore we leave such support for multiple datasets up to the model itself. For example, in ASR, you can concatenate multiple train manifest files by using commas when providing the `manifest_filepath` value!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BjI2Q5LECJib" + }, + "source": [ + "### What are multi methods?\n", + "\n", + "In many cases, especially true for ASR and NLP, we may have multiple validation and test datasets. The most common example for this in ASR is `Librispeech`, which has `dev_clean`, `dev_other`, `test_clean`, `test_other`.\n", + "\n", + "NeMo standardizes how to handle multiple data loaders for validation and testing, so that all of our collections have a similar look and feel, as well as ease development of our models. During evaluation, these datasets are treated independently and prepended with resolved names so that logs are separate!\n", + "\n", + "The `multi` methods are therefore generalizations of the single validation and single test data setup methods, with some additional functionality. If you provide multiple datasets, you still have to write code for just one dataset and NeMo will automatically attach the appropriate names to your logs so you can differentiate between them!\n", + "\n", + "Furthermore, they also automatically preserve the config the user passes to them when updating the validation or test data loaders.\n", + "\n", + "**In general, it is preferred to call the `setup_multi_validation_data` and `setup_multi_test_data` methods, even if you are only using single datasets, simply for the automated management they provide.**" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZKURHn0jH_52" + }, + "source": [ + "## Creating Model from constructor vs restoring a model\n", + "---------\n", + "\n", + "You might notice, we discuss all of the above setup methods in the context of model after it is restored. However, NeMo scripts do not call them inside any of the example train scripts themselves.\n", + "\n", + "This is because these methods are automatically called by the constructor when the Model is created for the first time, but these methods are skipped during restoration (either from a PyTorch Lightning checkpoint using `load_from_checkpoint`, or via `restore_from` method inside NeMo Models).\n", + "\n", + "This is done as most datasets are stored on a user's local directory, and the path to these datasets is set in the config (either set by default, or set by Hydra overrides). On the other hand, the models are meant to be portable. On another user's system, the data might not be placed at exactly the same location, or even on the same drive as specified in the model's config!\n", + "\n", + "Therefore we allow the constructor some brevity and automate such dataset setup, whereas restoration warns that data loaders were not set up and provides the user with ways to set up their own datasets.\n", + "\n", + "------\n", + "\n", + "Why are optimizers not restored automatically? Well, optimizers themselves don't face an issue, but as we saw before, schedulers depend on the number of train steps in order to calculate their schedule.\n", + "\n", + "However, if you don't wish to modify the optimizer and scheduler, and prefer to leave them to their default values, that's perfectly alright. The `setup_optimization()` method is automatically called by PyTorch Lightning for you when you begin training your model!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "g91FE8mlMcnh" + }, + "source": [ + "## Saving and restoring models\n", + "----------\n", + "\n", + "NeMo provides a few ways to save and restore models. If you utilize the Experiment Manager that is part of all NeMo train scripts, PyTorch Lightning will automatically save checkpoints for you in the experiment directory.\n", + "\n", + "We can also use packaged files using the specialized `save_to` and `restore_from` methods." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NzMxga7QNYn8" + }, + "source": [ + "### Saving and Restoring from PTL Checkpoints\n", + "----------\n", + "\n", + "The PyTorch Lightning Trainer object will periodically save checkpoints when the experiment manager is being used during training.\n", + "\n", + "PyTorch Lightning checkpoints can then be loaded and evaluated / fine-tuned just as always using the class method `load_from_checkpoint`.\n", + "\n", + "For example, restore a Citrinet model from a checkpoint - \n", + "\n", + "```python\n", + "citrinet = nemo_asr.models.EncDecCTCModelBPE.load_from_checkpoint()\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W4YzAG-KOBkZ" + }, + "source": [ + "### Saving and Restoring from .nemo files\n", + "----------\n", + "\n", + "There are a few models which might require external dependencies to be packaged with them in order to restore them properly.\n", + "\n", + "One such example is an ASR model with an external BPE tokenizer. It is preferred if the model includes all of the components required to restore it, but a binary file for a tokenizer cannot be serialized into a PyTorch Lightning checkpoint.\n", + "\n", + "In such cases, we can use the `save_to` and `restore_from` method to package the entire model + its components (here, the tokenizer file(s)) into a tarfile. This can then be easily imported by the user and used to restore the model." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "P6_vMSwXNJ74" + }, + "source": [ + "# Save the model\n", + "citrinet.save_to('citrinet_512.nemo')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "HrBhgaqyP4rU" + }, + "source": [ + "!ls -d -- *.nemo " + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Tyht1E0DQGb_" + }, + "source": [ + "# Restore the model\n", + "temp_cn = nemo_asr.models.EncDecCTCModelBPE.restore_from('citrinet_512.nemo')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "dqNpmYYJQS2H" + }, + "source": [ + "temp_cn.summarize();" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "A5e42EoiZYjf" + }, + "source": [ + "# Note that the preprocessor + optimizer config have been preserved after the changes we made !\n", + "print(OmegaConf.to_yaml(temp_cn.cfg))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OI3RxwpcV-UF" + }, + "source": [ + "Note, that .nemo file is a simple .tar.gz with checkpoint, configuration and, potentially, other artifacts such as tokenizer configs being used by the model" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "jFBAGcaDWLiu" + }, + "source": [ + "!cp citrinet_512.nemo citrinet_512.tar.gz\n", + "!tar -xvf citrinet_512.tar.gz" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mkau4Q9jZo1l" + }, + "source": [ + "### Extracting PyTorch checkpoints from NeMo tarfiles (Model level)\n", + "-----------\n", + "\n", + "While the .nemo tarfile is an excellent way to have a portable model, sometimes it is necessary for researchers to have access to the basic PyTorch save format. NeMo aims to be entirely compatible with PyTorch, and therefore offers a simple method to extract just the PyTorch checkpoint from the .nemo tarfile." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "qccPANeycCoq" + }, + "source": [ + "import torch" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "A4zswOKHar9q" + }, + "source": [ + "state_dict = temp_cn.extract_state_dict_from('citrinet_512.nemo', save_dir='./pt_ckpt/')\n", + "!ls ./pt_ckpt/" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ACB-0dfnbFG3" + }, + "source": [ + "As we can see below, there is now a single basic PyTorch checkpoint available inside the `pt_ckpt` directory, which we can use to load the weights of the entire model as below" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "4ZAF_A0uc5bB" + }, + "source": [ + "temp_cn.load_state_dict(torch.load('./pt_ckpt/model_weights.ckpt'))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Hkq6EM99cS6y" + }, + "source": [ + "### Extracting PyTorch checkpoints from NeMo tarfiles (Module level)\n", + "----------\n", + "\n", + "While the above method is exceptional when extracting the checkpoint of the entire model, sometimes there may be a necessity to load and save the individual modules that comprise the Model.\n", + "\n", + "The same extraction method offers a flag to extract the individual model level checkpoints into their individual files, so that users have access to per-module level checkpoints." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "LW6wve2zbT9D" + }, + "source": [ + "state_dict = temp_cn.extract_state_dict_from('citrinet_512.nemo', save_dir='./pt_module_ckpt/', split_by_module=True)\n", + "!ls ./pt_module_ckpt/" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DtV5vpb5d1ni" + }, + "source": [ + "Now, we can load and assign the weights of the individual modules of the above Citrinet Model !" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "rVHylSKFdywn" + }, + "source": [ + "temp_cn.preprocessor.load_state_dict(torch.load('./pt_module_ckpt/preprocessor.ckpt'))\n", + "temp_cn.encoder.load_state_dict(torch.load('./pt_module_ckpt/encoder.ckpt'))\n", + "temp_cn.decoder.load_state_dict(torch.load('./pt_module_ckpt/decoder.ckpt'))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "88vOGV7VYcuu" + }, + "source": [ + "# NeMo with Hydra\n", + "\n", + "[Hydra](https://hydra.cc/docs/intro/) is used throughout NeMo as a way to enable rapid prototyping using predefined config files. Hydra and OmegaConf offer great compatibility with each other, and below we show a few general helpful tips to improve productivity with Hydra when using NeMo." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DfY6Ha3qYcxG" + }, + "source": [ + "## Hydra Help\n", + "--------\n", + "\n", + "Since our scripts are written with hydra in mind, you might notice that using `python --help` returns you a config rather than the usual help format from argparse. \n", + "\n", + "Using `--help` you can see the default config attached to the script - every NeMo script has at least one default config file attached to it. This gives you a guide on how you can modify values for an experiment.\n", + "\n", + "Hydra also has a special `--hydra-help` flag, which will offer you more help with respect to hydra itself as it is set up in the script.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gEsZlnfaYc3X" + }, + "source": [ + "## Changing config paths and files\n", + "---------\n", + "\n", + "While all NeMo models come with at least 1 default config file, one might want to switch configs without changing code. This is easily achieved by the following commands : \n", + "\n", + "- `--config-path`: Path to the directory which contains the config files\n", + "- `--config-name`: Name of the config file we wish to load.\n", + "\n", + "Note that these two arguments need to be at the very beginning of your execution statement, before you provide any command line overrides to your config file." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZyNHlArpYc9A" + }, + "source": [ + "## Overriding config from the command line\n", + "----------\n", + "\n", + "Hydra allows users to provide command line overrides to any part of the config. There are three cases to consider - \n", + "\n", + " - Override existing value in config\n", + " - Add new value in config\n", + " - Remove old value in config" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "96CKbvn6Yc7f" + }, + "source": [ + "### Overriding existing values in config\n", + "\n", + "Let's take the case where we want to change the optimizer from `novograd` to `adam`. Let's also change the beta values to default adam values.\n", + "\n", + "Hydra overrides are based on the `.` syntax - each `.` representing a level in the config itself.\n", + "\n", + "```sh\n", + "$ python + + +
+ + +
+ + + +
+ + + +
+ + +
+ + +''' + + +#class Converter(): +# def __init__(self, model_dir, model_name): +# self.num_features = 24 +# self.sampling_rate = 16000 +# self.frame_period = 5.0 +# +# self.model = CycleGAN(num_features = self.num_features, mode = 'test') +# +# self.model.load(filepath = os.path.join(model_dir, model_name)) +# +# """ +# # NB: Save the graph +# definition = self.model.sess.graph_def +# directory = 'saved_model_2' +# tf.train.write_graph(definition, directory, 'saved_model_2.pb', as_text=True) +# # https://github.com/tensorflow/models/issues/3530#issuecomment-395968881 +# output_dir = './saved_model/' +# builder = tf.saved_model.builder.SavedModelBuilder(output_dir) +# builder.add_meta_graph_and_variables( +# self.model.sess, +# [tf.saved_model.tag_constants.SERVING], +# main_op=tf.tables_initializer(), +# ) +# builder.save() +# """ +# +# """ +# builder.add_meta_graph_and_variables( +# self.model.sess, +# [tf.saved_model.tag_constants.SERVING], +# signature_def_map={ +# 'predict_images': +# prediction_signature, +# signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: +# classification_signature, +# }, +# main_op=tf.tables_initializer()) +# """ +# +# self.mcep_normalization_params = np.load(os.path.join(model_dir, 'mcep_normalization.npz')) +# self.mcep_mean_A = self.mcep_normalization_params['mean_A'] +# self.mcep_std_A = self.mcep_normalization_params['std_A'] +# self.mcep_mean_B = self.mcep_normalization_params['mean_B'] +# self.mcep_std_B = self.mcep_normalization_params['std_B'] +# +# self.logf0s_normalization_params = np.load(os.path.join(model_dir, 'logf0s_normalization.npz')) +# self.logf0s_mean_A = self.logf0s_normalization_params['mean_A'] +# self.logf0s_std_A = self.logf0s_normalization_params['std_A'] +# self.logf0s_mean_B = self.logf0s_normalization_params['mean_B'] +# self.logf0s_std_B = self.logf0s_normalization_params['std_B'] +# +# def convert(self, wav, conversion_direction='A2B'): +# wav = wav_padding(wav = wav, sr = self.sampling_rate, frame_period = self.frame_period, multiple = 4) +# f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = self.sampling_rate, frame_period = self.frame_period) +# coded_sp = world_encode_spectral_envelop(sp = sp, fs = self.sampling_rate, dim = self.num_features) +# coded_sp_transposed = coded_sp.T +# +# if conversion_direction == 'A2B': +# f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_A, std_log_src = self.logf0s_std_A, mean_log_target = self.logf0s_mean_B, std_log_target = self.logf0s_std_B) +# coded_sp_norm = (coded_sp_transposed - self.mcep_mean_A) / self.mcep_std_A +# coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0] +# coded_sp_converted = coded_sp_converted_norm * self.mcep_std_B + self.mcep_mean_B +# else: +# f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_B, std_log_src = self.logf0s_std_B, mean_log_target = self.logf0s_mean_A, std_log_target = self.logf0s_std_A) +# coded_sp_norm = (coded_sp_transposed - self.mcep_mean_B) / self.mcep_std_B +# coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0] +# coded_sp_converted = coded_sp_converted_norm * self.mcep_std_A + self.mcep_mean_A +# +# coded_sp_converted = coded_sp_converted.T +# coded_sp_converted = np.ascontiguousarray(coded_sp_converted) +# decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, fs = self.sampling_rate) +# wav_transformed = world_speech_synthesis(f0 = f0_converted, decoded_sp = decoded_sp_converted, ap = ap, fs = self.sampling_rate, frame_period = self.frame_period) +# +# # For debugging model output, uncomment the following line: +# # librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate) +# +# # TODO: Perhaps ditch this. It's probably unnecessary work. +# upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000) +# pcm_data = upsampled.astype(np.float64) +# stereo_pcm_data = np.tile(pcm_data, (2,1)).T +# +# buf = io.BytesIO() +# scipy.io.wavfile.write(buf, 48000, stereo_pcm_data.astype(np.float32)) +# return buf + +## Set up model +## This should live long in memory, so we do it up front. +#model_dir_default = './model/sf1_tm1' +#model_name_default = 'sf1_tm1.ckpt' +#converter = Converter(model_dir_default, model_name_default) + +#tacotron = load_tacotron_model('/home/bt/models/tacotron2/tacotron2_uberduck_Noire.pt') +#waveglow = load_waveglow_model('/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt') +pipeline = TacotronWaveglowPipeline() + +class IndexHandler(): + def on_get(self, request, response): + response.content_type = 'text/html' + response.body = INDEX_HTML + +class ApiHandler(): + def on_post(self, request, response): + raw_data = json.load(request.bounded_stream) + + # Request parameters + synthesizer_checkpoint_path = raw_data.get('synthesizer_checkpoint_path') + vocoder_type = raw_data.get('vocoder_type') + waveglow_vocoder_checkpoint_path = raw_data.get('waveglow_vocoder_checkpoint_path') + hifigan_vocoder_checkpoint_path = raw_data.get('hifigan_vocoder_checkpoint_path') + hifigan_superres_vocoder_checkpoint_path = raw_data.get('hifigan_superres_vocoder_checkpoint_path') + inference_text = raw_data.get('inference_text') + output_audio_filename = raw_data.get('output_audio_filename') + output_spectrogram_filename = raw_data.get('output_spectrogram_filename') + output_metadata_filename = raw_data.get('output_metadata_filename') + + # Optional request parameters + maybe_clear_synthesizer_checkpoint_path = \ + raw_data.get('maybe_clear_synthesizer_checkpoint_path') + + print('synthesizer_checkpoint_path: {}'.format(synthesizer_checkpoint_path)) + print('vocoder_type: {}'.format(vocoder_type)) + print('waveglow_vocoder_checkpoint_path: {}'.format(waveglow_vocoder_checkpoint_path)) + print('hifigan_vocoder_checkpoint_path: {}'.format(hifigan_vocoder_checkpoint_path)) + print('hifigan_superres_vocoder_checkpoint_path: {}'.format(hifigan_superres_vocoder_checkpoint_path)) + print('inference_text: {}'.format(inference_text)) + print('output_audio_filename: {}'.format(output_audio_filename)) + print('output_spectrogram_filename: {}'.format(output_spectrogram_filename)) + print('output_metadata_filename: {}'.format(output_metadata_filename)) + print('maybe_clear_synthesizer_checkpoint_path: {}'.format( + maybe_clear_synthesizer_checkpoint_path)) + + #pipeline.maybe_load_waveglow_model(waveglow_vocoder_checkpoint_path) + #pipeline.maybe_load_hifigan(hifigan_vocoder_checkpoint_path) + #pipeline.maybe_load_hifigan_super_resolution(hifigan_superres_vocoder_checkpoint_path) + + #if maybe_clear_synthesizer_checkpoint_path: + # pipeline.uncache_tacotron_model(maybe_clear_synthesizer_checkpoint_path) + + #pipeline.maybe_load_tacotron_model_to_cache(synthesizer_checkpoint_path) + + inference_args = { + 'raw_text': inference_text, + 'synthesizer_checkpoint_path': synthesizer_checkpoint_path, + 'vocoder_type': vocoder_type, + 'output_audio_filename': output_audio_filename, + 'output_spectrogram_filename': output_spectrogram_filename, + 'output_metadata_filename': output_metadata_filename, + } + + pipeline.infer(inference_args) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--port', type=int, default=8000) + args = parser.parse_args() + + api = falcon.API(middleware=[MultipartMiddleware()]) + api.add_route('/', IndexHandler()) + api.add_route('/infer', ApiHandler()) + + print('Serving on 0.0.0.0:%d' % args.port) + simple_server.make_server('0.0.0.0', args.port, api).serve_forever() + +if __name__ == '__main__': + main() From 694789cef87147d1f7c8ef862903db1d2257926f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 8 Sep 2021 22:41:09 -0400 Subject: [PATCH 0592/1042] notes + requirements --- models/tts/requirements-talknet-python36-38.txt | 7 +++++++ models/tts/vocodes_server_talknet.py | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/models/tts/requirements-talknet-python36-38.txt b/models/tts/requirements-talknet-python36-38.txt index b37623b1af..71c0613b89 100644 --- a/models/tts/requirements-talknet-python36-38.txt +++ b/models/tts/requirements-talknet-python36-38.txt @@ -66,3 +66,10 @@ opencc pangu jieba numba + +# For serving HTTP (existing vocodes pipeline) +falcon==3.0.1 +falcon-multipart==0.2.0 + +# For metadata +python-magic==0.4.22 diff --git a/models/tts/vocodes_server_talknet.py b/models/tts/vocodes_server_talknet.py index cc1888d898..32cc859623 100755 --- a/models/tts/vocodes_server_talknet.py +++ b/models/tts/vocodes_server_talknet.py @@ -268,4 +268,16 @@ def main(): simple_server.make_server('0.0.0.0', args.port, api).serve_forever() if __name__ == '__main__': + """ + Launching a TalkNet server: + + from ControllableTalkNet.controllable_talknet import * + + app.run_server( + mode="inline", + threaded=True, + #dev_tools_ui=True, + #dev_tools_hot_reload=True, + ) + """ main() From 6db5f0187bd981113df27985265441fa05fb36cc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 9 Sep 2021 02:38:38 -0400 Subject: [PATCH 0593/1042] model loading, arpabet --- models/tts/vocodes_common_talknet.py | 51 +++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/models/tts/vocodes_common_talknet.py b/models/tts/vocodes_common_talknet.py index b50d5d37ee..9aedeb771c 100644 --- a/models/tts/vocodes_common_talknet.py +++ b/models/tts/vocodes_common_talknet.py @@ -29,6 +29,13 @@ import subprocess import magic import os +# For talknet +from ControllableTalkNet.controllable_talknet import arpa_parse +from ControllableTalkNet.controllable_talknet import to_arpa +from nemo.collections.tts.models import TalkNetSpectModel +from nemo.collections.tts.models import TalkNetDursModel +from nemo.collections.tts.models import TalkNetPitchModel + def print_gpu_info(): print('========================================') print('Python interpreter', sys.executable) @@ -100,6 +107,19 @@ def preprocess_text(text, raw_input=True): sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long() return sequence +def preprocess_text_talknet(text, tnmodel): + token_list = arpa_parse(text, tnmodel) + print('token_list', token_list) + + DEVICE = "cuda:0" + tokens = torch.IntTensor(token_list).view(1, -1).to(DEVICE) + print('tokens', tokens) + + arpa = to_arpa(token_list) + print('arpa', arpa) + + return arpa + class NumpyEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, np.ndarray): @@ -240,9 +260,38 @@ class TacotronWaveglowPipeline: assert('output_spectrogram_filename' in args) assert('output_metadata_filename' in args) + # ======================= BEGIN TALKNET EXPERIMENT ======================== + + print('load models') + + # TODO: Singer model? + # tnmodel = TalkNetSingerModel.restore_from(singer_path + talknet_path = "/home/bt/models/tacotron/david-attenborough/TalkNetSpect.nemo" + tnmodel = TalkNetSpectModel.restore_from(talknet_path) + + durs_path = "/home/bt/models/tacotron/david-attenborough/TalkNetDurs.nemo" + pitch_path = "/home/bt/models/tacotron/david-attenborough/TalkNetPitch.nemo" + + if os.path.exists(durs_path): + print('loading duration and pitch models') + tndurs = TalkNetDursModel.restore_from(durs_path) + tnmodel.add_module("_durs_model", tndurs) + tnpitch = TalkNetPitchModel.restore_from(pitch_path) + tnmodel.add_module("_pitch_model", tnpitch) + + tnmodel.eval() + + print('done loading model?') + + print('Raw text: {}'.format(args['raw_text'])) - sequence = preprocess_text(args['raw_text']) + sequence = preprocess_text_talknet(args['raw_text'], tnmodel) + + print('done with arpabet') + + + # ======================= END TALKNET EXPERIMENT ======================== tacotron = self.tacotron_model_cache[args['synthesizer_checkpoint_path']] From a681778046633f3708072980024326f2bcc0620a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 9 Sep 2021 02:57:12 -0400 Subject: [PATCH 0594/1042] end-to-end working? need to test the audio --- models/tts/vocodes_common_talknet.py | 155 +++++++++++++++++++++------ 1 file changed, 125 insertions(+), 30 deletions(-) diff --git a/models/tts/vocodes_common_talknet.py b/models/tts/vocodes_common_talknet.py index 9aedeb771c..f7b8c59105 100644 --- a/models/tts/vocodes_common_talknet.py +++ b/models/tts/vocodes_common_talknet.py @@ -35,6 +35,9 @@ from ControllableTalkNet.controllable_talknet import to_arpa from nemo.collections.tts.models import TalkNetSpectModel from nemo.collections.tts.models import TalkNetDursModel from nemo.collections.tts.models import TalkNetPitchModel +from ControllableTalkNet.controllable_talknet import load_hifigan +from hifigan.meldataset import MAX_WAV_VALUE +from hifigan.meldataset import mel_spectrogram def print_gpu_info(): print('========================================') @@ -107,19 +110,6 @@ def preprocess_text(text, raw_input=True): sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long() return sequence -def preprocess_text_talknet(text, tnmodel): - token_list = arpa_parse(text, tnmodel) - print('token_list', token_list) - - DEVICE = "cuda:0" - tokens = torch.IntTensor(token_list).view(1, -1).to(DEVICE) - print('tokens', tokens) - - arpa = to_arpa(token_list) - print('arpa', arpa) - - return arpa - class NumpyEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, np.ndarray): @@ -264,31 +254,136 @@ class TacotronWaveglowPipeline: print('load models') - # TODO: Singer model? - # tnmodel = TalkNetSingerModel.restore_from(singer_path - talknet_path = "/home/bt/models/tacotron/david-attenborough/TalkNetSpect.nemo" - tnmodel = TalkNetSpectModel.restore_from(talknet_path) + with torch.no_grad(): - durs_path = "/home/bt/models/tacotron/david-attenborough/TalkNetDurs.nemo" - pitch_path = "/home/bt/models/tacotron/david-attenborough/TalkNetPitch.nemo" + # TODO: Singer model? + # tnmodel = TalkNetSingerModel.restore_from(singer_path + talknet_path = "/home/bt/models/talknet/david-attenborough/TalkNetSpect.nemo" + tnmodel = TalkNetSpectModel.restore_from(talknet_path) - if os.path.exists(durs_path): - print('loading duration and pitch models') - tndurs = TalkNetDursModel.restore_from(durs_path) - tnmodel.add_module("_durs_model", tndurs) - tnpitch = TalkNetPitchModel.restore_from(pitch_path) - tnmodel.add_module("_pitch_model", tnpitch) + durs_path = "/home/bt/models/talknet/david-attenborough/TalkNetDurs.nemo" + pitch_path = "/home/bt/models/talknet/david-attenborough/TalkNetPitch.nemo" - tnmodel.eval() + if os.path.exists(durs_path): + print('loading duration and pitch models') + tndurs = TalkNetDursModel.restore_from(durs_path) + tnmodel.add_module("_durs_model", tndurs) + tnpitch = TalkNetPitchModel.restore_from(pitch_path) + tnmodel.add_module("_pitch_model", tnpitch) - print('done loading model?') + tnmodel.eval() + + print('done loading model?') + + # ==== ARPABET ==== + + print('Raw text: {}'.format(args['raw_text'])) + + text = args['raw_text'] + + token_list = arpa_parse(text, tnmodel) + print('token_list', token_list) + + DEVICE = "cuda:0" + tokens = torch.IntTensor(token_list).view(1, -1).to(DEVICE) + print('tokens', tokens) + + arpa = to_arpa(token_list) + + print('arpa', arpa) + print('done with arpabet') + + # ==== END ARPABET ==== + + # TODO: Either pitch prediction (if model given), or use a source audio file (TODO) + + print("spectrogram estimation") + spect = tnmodel.generate_spectrogram(tokens=tokens) - print('Raw text: {}'.format(args['raw_text'])) + # ==== HIFIGAN ==== + + print("load hifigan") + + hifigan_path = "/home/bt/models/talknet/david-attenborough/hifiganmodel" + + hifigan, h, denoiser = load_hifigan(hifigan_path, "config_v1") + hifipath = hifigan_path + + y_g_hat = hifigan(spect.float()) + audio = y_g_hat.squeeze() + audio = audio * MAX_WAV_VALUE + audio_denoised = denoiser(audio.view(1, -1), strength=35)[:, 0] + audio_np = ( + audio_denoised.detach().cpu().numpy().reshape(-1).astype(np.int16) + ) + + # ==== HIFIGAN SR ==== + + print("load hifigan SR") + + sr_path = "/home/bt/models/hifigan/Superres_Twilight_33000" + hifigan_sr, h2, denoiser_sr = load_hifigan(sr_path, "config_32k") + + # TODO + # TODO + # TODO -- Auto-tuning + # TODO + # TODO + + print("resample 32k") + + # Resample to 32k + wave = resampy.resample( + audio_np, + h.sampling_rate, + h2.sampling_rate, + filter="sinc_window", + window=scipy.signal.windows.hann, + num_zeros=8, + ) + wave_out = wave.astype(np.int16) + + print("hifi SR") + + # HiFi-GAN super-resolution + wave = wave / MAX_WAV_VALUE + wave = torch.FloatTensor(wave).to(torch.device(DEVICE)) + new_mel = mel_spectrogram( + wave.unsqueeze(0), + h2.n_fft, + h2.num_mels, + h2.sampling_rate, + h2.hop_size, + h2.win_size, + h2.fmin, + h2.fmax, + ) + y_g_hat2 = hifigan_sr(new_mel) + audio2 = y_g_hat2.squeeze() + audio2 = audio2 * MAX_WAV_VALUE + audio2_denoised = denoiser(audio2.view(1, -1), strength=35)[:, 0] + + # High-pass filter, mixing and denormalizing + audio2_denoised = audio2_denoised.detach().cpu().numpy().reshape(-1) + b = scipy.signal.firwin( + 101, cutoff=10500, fs=h2.sampling_rate, pass_zero=False + ) + y = scipy.signal.lfilter(b, [1.0], audio2_denoised) + y *= 4.0 # superres strength + y_out = y.astype(np.int16) + y_padded = np.zeros(wave_out.shape) + y_padded[: y_out.shape[0]] = y_out + sr_mix = wave_out + y_padded + + #buffer = io.BytesIO() + #wavfile.write(buffer, 32000, sr_mix.astype(np.int16)) + #b64 = base64.b64encode(buffer.getvalue()) + #sound = "data:audio/x-wav;base64," + b64.decode("ascii") + + print("done!") - sequence = preprocess_text_talknet(args['raw_text'], tnmodel) - print('done with arpabet') # ======================= END TALKNET EXPERIMENT ======================== From da16826ab53edfa8c8025d48e73e6b5e154382ea Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 9 Sep 2021 03:01:06 -0400 Subject: [PATCH 0595/1042] talknet works --- models/tts/vocodes_common_talknet.py | 33 ++++++++++++++++------------ 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/models/tts/vocodes_common_talknet.py b/models/tts/vocodes_common_talknet.py index f7b8c59105..eb0f961c8a 100644 --- a/models/tts/vocodes_common_talknet.py +++ b/models/tts/vocodes_common_talknet.py @@ -384,27 +384,32 @@ class TacotronWaveglowPipeline: print("done!") + output_audio = sr_mix.astype(np.int16) + #rate = self.hifigan_super_resolution_h.sampling_rate + rate = h2.sampling_rate + write_wav(args['output_audio_filename'], rate, output_audio) + + print('Saving spectrogram as JSON...') + save_spectorgram_json_file(spect, args['output_spectrogram_filename']) + + print('Generating metadata file...') + generate_metadata_file(args['output_audio_filename'], args['output_metadata_filename']) # ======================= END TALKNET EXPERIMENT ======================== - tacotron = self.tacotron_model_cache[args['synthesizer_checkpoint_path']] + #tacotron = self.tacotron_model_cache[args['synthesizer_checkpoint_path']] - with torch.no_grad(): - mel_outputs, mel_outputs_postnet, _, alignments = tacotron.inference(sequence) + #with torch.no_grad(): + # mel_outputs, mel_outputs_postnet, _, alignments = tacotron.inference(sequence) - if args['vocoder_type'] == 'hifigan-superres': - self.vocode_hifigan_superres(args, mel_outputs_postnet) - elif args['vocoder_type'] == 'waveglow': - self.vocode_waveglow(args, mel_outputs_postnet) - else: - raise Exception('Wrong vocoder type: {}'.format(args['vocoder_type'])) + #if args['vocoder_type'] == 'hifigan-superres': + # self.vocode_hifigan_superres(args, mel_outputs_postnet) + #elif args['vocoder_type'] == 'waveglow': + # self.vocode_waveglow(args, mel_outputs_postnet) + #else: + # raise Exception('Wrong vocoder type: {}'.format(args['vocoder_type'])) - print('Saving spectrogram as JSON...') - save_spectorgram_json_file(mel_outputs_postnet, args['output_spectrogram_filename']) - - print('Generating metadata file...') - generate_metadata_file(args['output_audio_filename'], args['output_metadata_filename']) def vocode_waveglow(self, args, mel_outputs_postnet): print('Running waveglow...') From def7e4f3cf644a58c22f7867fb9ac29eff6d18ec Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 9 Sep 2021 23:58:40 -0400 Subject: [PATCH 0596/1042] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2d1a0abb44..9500af898e 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ https://docs.google.com/spreadsheets/d/1BEdLmwOzo3r83-iJn9sj6co1VT92t3lIeJ-lNH25 Ubuntu might upgrade the driver by accident, and it might need reinstallation. Who knows. This stuff is a nightmare. + ### Database Setup Install the following libraries, and see the notes further below about MySQL on Ubuntu 20.04. From 743f2bf560a2182931c0ecace7e6c487d3eb48ac Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 10 Sep 2021 01:27:04 -0400 Subject: [PATCH 0597/1042] Create "squashed" schema references with central documentation --- .../up.sql | 3 + migrations/2021-05-15-071601_assets/up.sql | 3 + .../2021-05-15-212913_inference_jobs/up.sql | 3 + .../up.sql | 3 + .../tts_tables/tts_inference_jobs.sql | 107 +++++++++++ .../tts_tables/tts_model_upload_jobs.sql | 119 ++++++++++++ migrations_squashed/tts_tables/tts_models.sql | 172 ++++++++++++++++++ .../tts_tables/tts_results.sql | 100 ++++++++++ 8 files changed, 510 insertions(+) create mode 100644 migrations_squashed/tts_tables/tts_inference_jobs.sql create mode 100644 migrations_squashed/tts_tables/tts_model_upload_jobs.sql create mode 100644 migrations_squashed/tts_tables/tts_models.sql create mode 100644 migrations_squashed/tts_tables/tts_results.sql diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index 6807d24d83..91e1e70987 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -2,6 +2,9 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile +-- NB: See the "squashed" copy of this for better documentation +-- and the current state of the table. + -- TTS MODELS CREATE TABLE tts_model_upload_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 6efd46ebfc..62cbb5454e 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -2,6 +2,9 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile +-- NB: See the "squashed" copy of this for better documentation +-- and the current state of the table. + CREATE TABLE tts_models ( -- Not used for anything except replication. id BIGINT(20) NOT NULL AUTO_INCREMENT, diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index 0b25e2e584..28a68313ad 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -2,6 +2,9 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile +-- NB: See the "squashed" copy of this for better documentation +-- and the current state of the table. + CREATE TABLE tts_inference_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 12061ef51c..f6aa7ab2ae 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -2,6 +2,9 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile +-- NB: See the "squashed" copy of this for better documentation +-- and the current state of the table. + CREATE TABLE tts_results ( -- Not used for anything except replication. id BIGINT(20) NOT NULL AUTO_INCREMENT, diff --git a/migrations_squashed/tts_tables/tts_inference_jobs.sql b/migrations_squashed/tts_tables/tts_inference_jobs.sql new file mode 100644 index 0000000000..e25ae6680b --- /dev/null +++ b/migrations_squashed/tts_tables/tts_inference_jobs.sql @@ -0,0 +1,107 @@ +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE TABLE tts_inference_jobs ( + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + -- This is so the in-progress results can be looked up by the UI. + token VARCHAR(32) NOT NULL, + + -- Idempotency token from client + -- This is so the frontend client doesn't submit duplicate jobs. + uuid_idempotency_token VARCHAR(36) NOT NULL, + + -- ========== SUCCESS CASE ========== + + -- If the job completes successfully, this is the result token. + -- This is only populated on a successful result. + on_success_result_token VARCHAR(32) DEFAULT NULL, + + -- ========== INFERENCE DETAILS ========== + + -- The model to use. + -- This also determines which architecture we're using. + model_token VARCHAR(32) NOT NULL, + + -- If set, use a different pretrained vocoder. + maybe_override_pretrained_vocoder VARCHAR(64) DEFAULT NULL, + + -- The raw, unprocessed user input. + raw_inference_text TEXT NOT NULL, + + -- ========== CREATOR DETAILS ========== + + -- Foreign key to user + -- If no user is logged in, this is null. + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- ========== JOB SYSTEM DETAILS ========== + + -- Jobs begin as "pending", then transition to other states. + -- + -- * Pending = job is ready to go + -- * Started = job is running + -- * Complete_Success = job is done (success) + -- * Complete_Failure = job is done (failure) + -- * Attempt_Failed = job failed but may retry. + -- * Dead = job failed permanently. + -- + -- Pending -> Started -> Complete_Success + -- |-> Complete_Failure + -- \-> Attempt_Failed -> Started -> { Complete, Failed, Dead } + status ENUM( + 'pending', + 'started', + 'complete_success', + 'complete_failure', + 'attempt_failed', + 'dead') NOT NULL DEFAULT 'pending', + + -- We can track this against a "max_attempt_count" + attempt_count INT(3) NOT NULL DEFAULT 0, + + -- If there is a failure, tell the user why. + failure_reason VARCHAR(512) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- Failed jobs will set a next attempt time. + -- Subsequent tries can increase the timeout. + -- Failures because of permissions require human intervention => [retry_at=null]. + -- Failures because of invalid files are dead => [status=dead]. + retry_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + UNIQUE KEY (uuid_idempotency_token), + KEY fk_on_success_result_token (on_success_result_token), + KEY fk_model_token (model_token), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY index_status (status), + KEY index_creator_ip_address (creator_ip_address) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations_squashed/tts_tables/tts_model_upload_jobs.sql b/migrations_squashed/tts_tables/tts_model_upload_jobs.sql new file mode 100644 index 0000000000..47ce1d63b3 --- /dev/null +++ b/migrations_squashed/tts_tables/tts_model_upload_jobs.sql @@ -0,0 +1,119 @@ +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- TTS MODELS +CREATE TABLE tts_model_upload_jobs ( + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + -- This is so the in-progress results can be looked up by the UI. + token VARCHAR(32) NOT NULL, + + -- Idempotency token + -- This is so the frontend client doesn't submit duplicate jobs. + uuid_idempotency_token VARCHAR(36) NOT NULL, + + -- ========== SUCCESS CASE ========== + + -- If the job completes successfully, this is the result token. + -- This is only populated on a successful result. + on_success_result_token VARCHAR(32) DEFAULT NULL, + + -- ========== UPLOAD DETAILS ========== + + -- The name of the voice + -- The "name" of the voice model, which might be complicated. + -- If maybe_subject_token (etc.) is set, then it's authoritative instead. + title VARCHAR(255) NOT NULL, + + -- NB: DO NOT CHANGE ORDER; APPEND ONLY! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + tts_model_type ENUM( + 'not-set', + 'tacotron2', + 'glowtts', + 'glowtts-vocodes' + ) NOT NULL DEFAULT 'not-set', + + -- Can be linked to a well-known voice + -- maybe_subject_token VARCHAR(32) DEFAULT NULL, + -- maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, + + -- If we need to download the file from Google Drive. + download_url VARCHAR(512) DEFAULT NULL, + + -- NB: DO NOT SORT! + -- download_url_type ENUM( + -- 'google-drive', + -- 'web' + -- ) NOT NULL DEFAULT 'web', + + -- ========== CREATOR DETAILS AND PREFERENCES ========== + + -- Foreign key to user + creator_user_token VARCHAR(32) NOT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- ========== JOB SYSTEM DETAILS ========== + + -- Jobs begin as "pending", then transition to other states. + -- + -- * Pending = job is ready to go + -- * Started = job is running + -- * Complete_Success = job is done (success) + -- * Complete_Failure = job is done (failure) + -- * Attempt_Failed = job failed but may retry. + -- * Dead = job failed permanently. + -- + -- Pending -> Started -> Complete_Success + -- |-> Complete_Failure + -- \-> Attempt_Failed -> Started -> { Complete, Failed, Dead } + status ENUM( + 'pending', + 'started', + 'complete_success', + 'complete_failure', + 'attempt_failed', + 'dead') NOT NULL DEFAULT 'pending', + + -- We can track this against a "max_attempt_count" + attempt_count INT(3) NOT NULL DEFAULT 0, + + -- If there is a failure, tell the user why. + failure_reason VARCHAR(512) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- Failed jobs will set a next attempt time. + -- Subsequent tries can increase the timeout. + -- Failures because of permissions require human intervention => [retry_at=null]. + -- Failures because of invalid files are dead => [status=dead]. + retry_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + UNIQUE KEY (uuid_idempotency_token), + KEY fk_on_success_result_token (on_success_result_token), + KEY fk_creator_user_token (creator_user_token), + KEY index_status (status), + KEY index_creator_ip_address (creator_ip_address) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations_squashed/tts_tables/tts_models.sql b/migrations_squashed/tts_tables/tts_models.sql new file mode 100644 index 0000000000..8e036da5b5 --- /dev/null +++ b/migrations_squashed/tts_tables/tts_models.sql @@ -0,0 +1,172 @@ +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE TABLE tts_models ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- NB: DO NOT CHANGE ORDER; APPEND ONLY! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + tts_model_type ENUM( + 'not-set', + 'tacotron2', + 'glowtts', + 'glowtts-vocodes' + ) NOT NULL DEFAULT 'not-set', + + -- NB: DO NOT CHANGE ORDER; APPEND ONLY! + -- How text should be handled. + text_preprocessing_algorithm ENUM( + 'basic', + 'arpabet' + ) NOT NULL DEFAULT 'basic', + + -- If set, use a different pretrained vocoder. + maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL, + + -- Optional Pointer to a newer version of the voice + -- If there's a newer version, we can disable this one. + -- maybe_updated_model_token VARCHAR(32) DEFAULT NULL, + + -- We can set as a combination of ['username' + 'voice-name'] + -- There can be public aliases for voices, eg. a voice's default model. + -- A user can change this. + -- As such, these should not be foreign keys. + -- updatable_slug VARCHAR(64) NOT NULL, + + -- Can be linked to a well-known voice + -- maybe_subject_token VARCHAR(32) DEFAULT NULL, + -- maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, + + -- ========== UNSTRUCTURED METADATA ========== + + -- The "name" of the voice model, which might be complicated. + -- If maybe_subject_token (etc.) is set, then it's authoritative instead. + title VARCHAR(255) NOT NULL, + + -- The description of the model in markdown. + description_markdown TEXT NOT NULL, + + -- Generated HTML (not user-editable). + description_rendered_html TEXT NOT NULL, + + -- eg. Bill Clinton, Batman, or Goku. + voice_name VARCHAR(255) DEFAULT NULL, + + -- Optional; eg. Sean Schemmel + actor_name VARCHAR(255) DEFAULT NULL, + + -- Optional; eg. if the voice is "happy" or a singer "a-capella", etc. + characteristic VARCHAR(255) DEFAULT NULL, + + -- Optional; eg. if you train the model more than once. + version_string VARCHAR(255) DEFAULT NULL, + + -- ========== IMMUTABLE PROVENANCE AND METADATA ========== + + -- Where the file was originally downloaded (if it was downloaded) + original_download_url VARCHAR(512) DEFAULT NULL, + + -- The filename that was used at upload time (if available) + original_filename VARCHAR(255) DEFAULT NULL, + + -- File characteristics + file_size_bytes INT(10) NOT NULL DEFAULT 0, + + -- ========== CREATOR DETAILS ========== + + -- The person that created the template. + creator_user_token VARCHAR(32) NOT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address_creation VARCHAR(40) NOT NULL, + creator_ip_address_last_update VARCHAR(40) NOT NULL, + + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- ========== BUCKET STORAGE ========== + + -- The pytorch model + -- For now, this will be a hash of the file contents. + -- NB: NOT UNIQUE! We can allow duplicate uploads. + private_bucket_hash CHAR(64) NOT NULL, + -- The "full url" version of the path + private_bucket_object_name VARCHAR(255) NOT NULL, + + -- ========== RATING AND RANKING ========== + + -- Calculated average, on a scale of 0-100 + -- Null with zero ratings. + -- calculated_average_score INT(3) DEFAULT NULL, + + -- Other metrics + -- calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0, + -- calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0, + + -- ========== MODERATION DETAILS ========== + + -- Mods have to approve of tts models for them to show up in a public index. + -- Models can be used by the author (or other parties that know the URL) before + -- they're approved, but unapproved models won't show up in public indices. + -- This is independent of other visibility controls (is_mod_disabled, deleted_at, + -- and creator_set_visibility) + is_public_listing_approved BOOLEAN DEFAULT NULL, + + -- Extremely popular models may be locked from deletion or modification by users. + is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE, + + -- In this case, a moderator disables it from being used for inference. + -- This also disables it for the creator. + -- Unlike deletion, it remains "visible" to those that have access. + is_locked_from_use BOOLEAN NOT NULL DEFAULT FALSE, + + -- If a moderator has comments. + maybe_mod_comments VARCHAR(255) DEFAULT NULL, + + -- The last moderator that made changes. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod. + -- It completely disappears from the system. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY fk_creator_user_token (creator_user_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), + KEY index_creator_ip_address_creation (creator_ip_address_creation), + KEY index_creator_ip_address_last_update (creator_ip_address_last_update), + KEY index_creator_set_visibility (creator_set_visibility), + KEY index_private_bucket_hash (private_bucket_hash), + KEY index_is_public_listing_approved (is_public_listing_approved), + KEY index_is_locked_from_user_modification (is_locked_from_user_modification), + KEY index_is_locked_from_use (is_locked_from_use) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations_squashed/tts_tables/tts_results.sql b/migrations_squashed/tts_tables/tts_results.sql new file mode 100644 index 0000000000..7b09574ac3 --- /dev/null +++ b/migrations_squashed/tts_tables/tts_results.sql @@ -0,0 +1,100 @@ +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE TABLE tts_results ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- ========== INFERENCE DETAILS ========== + + -- The TTS model that was used + model_token VARCHAR(32) NOT NULL, + + -- If set, specifies which pretrained vocoder was used. + maybe_pretrained_vocoder_used VARCHAR(64) DEFAULT NULL, + + -- The original raw, unprocessed user input. + raw_inference_text TEXT NOT NULL, + + -- SHA2 hash of the text [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)] + raw_inference_text_hash_sha2 CHAR(64) NOT NULL, + + -- First pass at text normalization. + -- eg. 14th -> fourteenth, etc. + normalized_inference_text TEXT NOT NULL, + + -- If the model uses arpabet, we'll save the arpabet we used. + maybe_arpabet_text TEXT DEFAULT NULL, + + -- ========== CREATOR DETAILS ========== + + -- The person that created the template. + -- If the user wasn't logged in, this is null + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- The synthetic id associated with this result. + -- These ids are incremented on a per-user basis to help users + -- sequence their own work. They serve no other purpose. + maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL, + + -- ========== BUCKET STORAGE ========== + + -- Where the wav, spectrogram, and etc. are located. + public_bucket_wav_audio_path VARCHAR(255) NOT NULL, + public_bucket_spectrogram_path VARCHAR(255) NOT NULL, + + -- ========== METADATA ========== + + file_size_bytes INT(10) NOT NULL DEFAULT 0, + duration_millis INT(10) NOT NULL DEFAULT 0, + + -- ========== MODERATION DETAILS ========== + + -- The last moderator that made changes. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod or the creator. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY fk_model_token (model_token), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_maybe_creator_anonymous_visitor_token (maybe_creator_anonymous_visitor_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), + KEY index_creator_ip_address (creator_ip_address), + KEY index_user_deleted_at(user_deleted_at), + KEY index_mod_deleted_at(mod_deleted_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From c529756ab7532bb720995bdb9b7a02156ef0c0af Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 10 Sep 2021 01:33:58 -0400 Subject: [PATCH 0598/1042] rename migration directory --- .../down.sql | 0 .../up.sql | 4 ++++ 2 files changed, 4 insertions(+) rename migrations/{2021-09-25-010101_alter_tables_add_vocoder_type => 2021-08-25-010101_alter_tables_add_vocoder_type}/down.sql (100%) rename migrations/{2021-09-25-010101_alter_tables_add_vocoder_type => 2021-08-25-010101_alter_tables_add_vocoder_type}/up.sql (71%) diff --git a/migrations/2021-09-25-010101_alter_tables_add_vocoder_type/down.sql b/migrations/2021-08-25-010101_alter_tables_add_vocoder_type/down.sql similarity index 100% rename from migrations/2021-09-25-010101_alter_tables_add_vocoder_type/down.sql rename to migrations/2021-08-25-010101_alter_tables_add_vocoder_type/down.sql diff --git a/migrations/2021-09-25-010101_alter_tables_add_vocoder_type/up.sql b/migrations/2021-08-25-010101_alter_tables_add_vocoder_type/up.sql similarity index 71% rename from migrations/2021-09-25-010101_alter_tables_add_vocoder_type/up.sql rename to migrations/2021-08-25-010101_alter_tables_add_vocoder_type/up.sql index 17b09f19b8..75e4eb67dd 100644 --- a/migrations/2021-09-25-010101_alter_tables_add_vocoder_type/up.sql +++ b/migrations/2021-08-25-010101_alter_tables_add_vocoder_type/up.sql @@ -2,6 +2,10 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile +-- TODO/FIXME: I originally created this with the wrong directory month. +-- I'll have to repair the migrations table. +-- version: 20210925010101 | run_on: 2021-08-25 03:01:15 (development) + ALTER TABLE tts_models ADD COLUMN maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL AFTER text_preprocessing_algorithm; From e3233f4c3258cd0ab260f7be647c75a271ae4c9c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 10 Sep 2021 02:35:52 -0400 Subject: [PATCH 0599/1042] add talknet as a model type --- .../up.sql | 2 ++ migrations/2021-05-15-071601_assets/up.sql | 2 ++ .../up.sql | 1 + .../down.sql | 1 + .../up.sql | 23 +++++++++++++++++++ .../tts_tables/tts_model_upload_jobs.sql | 3 ++- migrations_squashed/tts_tables/tts_models.sql | 3 ++- 7 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 migrations/2021-09-10-053520_alter_tables_add_talknet_model_type/down.sql create mode 100644 migrations/2021-09-10-053520_alter_tables_add_talknet_model_type/up.sql diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index 91e1e70987..5a377f2be5 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -30,6 +30,8 @@ CREATE TABLE tts_model_upload_jobs ( -- If maybe_subject_token (etc.) is set, then it's authoritative instead. title VARCHAR(255) NOT NULL, + -- NB: THIS ENUM IS OUT OF DATE! + -- Check the squashed sql file as an updated reference! -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. tts_model_type ENUM( diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 62cbb5454e..6a9a8ee855 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -12,6 +12,8 @@ CREATE TABLE tts_models ( -- Effective "primary key" (PUBLIC) token VARCHAR(32) NOT NULL, + -- NB: THIS ENUM IS OUT OF DATE! + -- Check the squashed sql file as an updated reference! -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. tts_model_type ENUM( diff --git a/migrations/2021-08-25-010101_alter_tables_add_vocoder_type/up.sql b/migrations/2021-08-25-010101_alter_tables_add_vocoder_type/up.sql index 75e4eb67dd..7a16e2c67b 100644 --- a/migrations/2021-08-25-010101_alter_tables_add_vocoder_type/up.sql +++ b/migrations/2021-08-25-010101_alter_tables_add_vocoder_type/up.sql @@ -5,6 +5,7 @@ -- TODO/FIXME: I originally created this with the wrong directory month. -- I'll have to repair the migrations table. -- version: 20210925010101 | run_on: 2021-08-25 03:01:15 (development) +-- UPDATE __diesel_schema_migrations SET version=20210825010101 WHERE version=20210925010101 LIMIT 1; ALTER TABLE tts_models ADD COLUMN maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL diff --git a/migrations/2021-09-10-053520_alter_tables_add_talknet_model_type/down.sql b/migrations/2021-09-10-053520_alter_tables_add_talknet_model_type/down.sql new file mode 100644 index 0000000000..291a97c5ce --- /dev/null +++ b/migrations/2021-09-10-053520_alter_tables_add_talknet_model_type/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` \ No newline at end of file diff --git a/migrations/2021-09-10-053520_alter_tables_add_talknet_model_type/up.sql b/migrations/2021-09-10-053520_alter_tables_add_talknet_model_type/up.sql new file mode 100644 index 0000000000..5743c05bee --- /dev/null +++ b/migrations/2021-09-10-053520_alter_tables_add_talknet_model_type/up.sql @@ -0,0 +1,23 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Add talknet to the tts_model_type. + +ALTER TABLE tts_models +MODIFY COLUMN tts_model_type ENUM( + 'not-set', + 'tacotron2', + 'glowtts', + 'glowtts-vocodes', + 'talknet' +) NOT NULL DEFAULT 'not-set'; + +ALTER TABLE tts_model_upload_jobs +MODIFY COLUMN tts_model_type ENUM( + 'not-set', + 'tacotron2', + 'glowtts', + 'glowtts-vocodes', + 'talknet' +) NOT NULL DEFAULT 'not-set'; diff --git a/migrations_squashed/tts_tables/tts_model_upload_jobs.sql b/migrations_squashed/tts_tables/tts_model_upload_jobs.sql index 47ce1d63b3..084449a325 100644 --- a/migrations_squashed/tts_tables/tts_model_upload_jobs.sql +++ b/migrations_squashed/tts_tables/tts_model_upload_jobs.sql @@ -36,7 +36,8 @@ CREATE TABLE tts_model_upload_jobs ( 'not-set', 'tacotron2', 'glowtts', - 'glowtts-vocodes' + 'glowtts-vocodes', + 'talknet' ) NOT NULL DEFAULT 'not-set', -- Can be linked to a well-known voice diff --git a/migrations_squashed/tts_tables/tts_models.sql b/migrations_squashed/tts_tables/tts_models.sql index 8e036da5b5..b6ccf77563 100644 --- a/migrations_squashed/tts_tables/tts_models.sql +++ b/migrations_squashed/tts_tables/tts_models.sql @@ -18,7 +18,8 @@ CREATE TABLE tts_models ( 'not-set', 'tacotron2', 'glowtts', - 'glowtts-vocodes' + 'glowtts-vocodes', + 'talknet' ) NOT NULL DEFAULT 'not-set', -- NB: DO NOT CHANGE ORDER; APPEND ONLY! From 620de4c528e0d067cc95c7e79da0f529e6bb6ba2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 10 Sep 2021 04:21:47 -0400 Subject: [PATCH 0600/1042] Update tts_models schema to track TalkNet-related details --- .../down.sql | 15 +++++++++++++++ .../up.sql | 19 +++++++++++++++++++ migrations_squashed/tts_tables/tts_models.sql | 16 ++++++++++++++++ src/database/helpers/enums.rs | 2 ++ 4 files changed, 52 insertions(+) create mode 100644 migrations/2021-09-10-074547_alter_tables_talknet_metadata/down.sql create mode 100644 migrations/2021-09-10-074547_alter_tables_talknet_metadata/up.sql diff --git a/migrations/2021-09-10-074547_alter_tables_talknet_metadata/down.sql b/migrations/2021-09-10-074547_alter_tables_talknet_metadata/down.sql new file mode 100644 index 0000000000..1da108803b --- /dev/null +++ b/migrations/2021-09-10-074547_alter_tables_talknet_metadata/down.sql @@ -0,0 +1,15 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_models +DROP COLUMN private_bucket_object_is_archive; + +ALTER TABLE tts_models +DROP COLUMN has_self_contained_vocoder; + +ALTER TABLE tts_models +DROP COLUMN has_self_contained_duration_model; + +ALTER TABLE tts_models +DROP COLUMN has_self_contained_pitch_model; diff --git a/migrations/2021-09-10-074547_alter_tables_talknet_metadata/up.sql b/migrations/2021-09-10-074547_alter_tables_talknet_metadata/up.sql new file mode 100644 index 0000000000..e0715f0878 --- /dev/null +++ b/migrations/2021-09-10-074547_alter_tables_talknet_metadata/up.sql @@ -0,0 +1,19 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_models +ADD COLUMN private_bucket_object_is_archive BOOLEAN NOT NULL DEFAULT FALSE +AFTER private_bucket_object_name; + +ALTER TABLE tts_models +ADD COLUMN has_self_contained_vocoder BOOLEAN NOT NULL DEFAULT FALSE +AFTER tts_model_type; + +ALTER TABLE tts_models +ADD COLUMN has_self_contained_duration_model BOOLEAN NOT NULL DEFAULT FALSE +AFTER has_self_contained_vocoder; + +ALTER TABLE tts_models +ADD COLUMN has_self_contained_pitch_model BOOLEAN NOT NULL DEFAULT FALSE +AFTER has_self_contained_duration_model; diff --git a/migrations_squashed/tts_tables/tts_models.sql b/migrations_squashed/tts_tables/tts_models.sql index b6ccf77563..41a03f70a4 100644 --- a/migrations_squashed/tts_tables/tts_models.sql +++ b/migrations_squashed/tts_tables/tts_models.sql @@ -22,6 +22,18 @@ CREATE TABLE tts_models ( 'talknet' ) NOT NULL DEFAULT 'not-set', + -- For models distributed as archives/bundles (eg. TalkNet), we can pack + -- in a vocoder. Note that this vocoder does not have to be used. + has_self_contained_vocoder BOOLEAN NOT NULL DEFAULT FALSE, + + -- For models distributed as archives/bundles (eg. TalkNet), we can pack + -- in a duration model. + has_self_contained_duration_model BOOLEAN NOT NULL DEFAULT FALSE, + + -- For models distributed as archives/bundles (eg. TalkNet), we can pack + -- in a pitch model. + has_self_contained_pitch_model BOOLEAN NOT NULL DEFAULT FALSE, + -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- How text should be handled. text_preprocessing_algorithm ENUM( @@ -30,6 +42,7 @@ CREATE TABLE tts_models ( ) NOT NULL DEFAULT 'basic', -- If set, use a different pretrained vocoder. + -- If not set, use the website default. (Currently HifiGan-SS) maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL, -- Optional Pointer to a newer version of the voice @@ -109,6 +122,9 @@ CREATE TABLE tts_models ( -- The "full url" version of the path private_bucket_object_name VARCHAR(255) NOT NULL, + -- Whether the file is a zip archive (rather than a model file). + private_bucket_object_is_archive BOOLEAN NOT NULL DEFAULT FALSE, + -- ========== RATING AND RANKING ========== -- Calculated average, on a scale of 0-100 diff --git a/src/database/helpers/enums.rs b/src/database/helpers/enums.rs index 6e49c85ab7..71f915ab4b 100644 --- a/src/database/helpers/enums.rs +++ b/src/database/helpers/enums.rs @@ -26,6 +26,8 @@ pub enum TtsModelType { GlowTts, /// glowtts-vocodes GlowTts_Vocodes, + /// talknet + Talknet, } #[derive(Deserialize)] From 00131daedf05c98622e4ad9489bf35142c0ba5e0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 10 Sep 2021 05:16:58 -0400 Subject: [PATCH 0601/1042] talknet downloads wip (BROKEN) --- .env | 4 + ...eck.py => vocodes_model_check_tacotron.py} | 0 models/tts/vocodes_model_check_talknet.py | 66 +++++++++++ src/database/enums/mod.rs | 1 + src/database/enums/tts_model_type.rs | 40 +++++++ src/script_execution/mod.rs | 1 + .../talknet_model_check_command.rs | 112 ++++++++++++++++++ src/tts_download_job.rs | 71 +++++++---- src/util/buckets/bucket_path_unifier.rs | 16 +++ 9 files changed, 291 insertions(+), 20 deletions(-) rename models/tts/{vocodes_model_check.py => vocodes_model_check_tacotron.py} (100%) create mode 100755 models/tts/vocodes_model_check_talknet.py create mode 100644 src/database/enums/tts_model_type.rs create mode 100644 src/script_execution/talknet_model_check_command.rs diff --git a/.env b/.env index 0aee8e094e..fc4bea8051 100644 --- a/.env +++ b/.env @@ -41,7 +41,11 @@ W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py TTS_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/tts TTS_INFERENCE_SCRIPT_NAME=vocodes_inference.py + +# TODO: Remove generic "check script" TTS_MODEL_CHECK_SCRIPT_NAME=vocodes_model_check.py +TTS_TACOTRON_CHECK_SCRIPT_NAME=vocodes_model_check_tacotron.py +TTS_TALKNET_CHECK_SCRIPT_NAME=vocodes_model_check_talknet.py TTS_INFERENCE_SIDECAR_HOSTNAME=localhost:8000 SIDECAR_MAX_SYNTHESIZER_MODELS=2 diff --git a/models/tts/vocodes_model_check.py b/models/tts/vocodes_model_check_tacotron.py similarity index 100% rename from models/tts/vocodes_model_check.py rename to models/tts/vocodes_model_check_tacotron.py diff --git a/models/tts/vocodes_model_check_talknet.py b/models/tts/vocodes_model_check_talknet.py new file mode 100755 index 0000000000..64e21dce5a --- /dev/null +++ b/models/tts/vocodes_model_check_talknet.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +""" +Vocodes Tacotron Model Check +""" + +import argparse + +parser = argparse.ArgumentParser(description='Check that the model is valid') + +parser.add_argument('--synthesizer_checkpoint_path', type=str, help='path the TTS synthesizer model', required=True) +parser.add_argument('--output_metadata_filename', type=str, help='where to save extra metadata', required=True) + +args = parser.parse_args() + +import os +import json + +# TODO: Enable model check/unzip/etc. +""" +import sys +sys.path.append('waveglow/') +import json + +import torch + +from tacotron2.hparams import create_hparams +from tacotron2.model import Tacotron2 + +print('========================================') +print('Python interpreter', sys.executable) +print('PyTorch version', torch.__version__) +print('CUDA Available?', torch.cuda.is_available()) +print('CUDA Device count', torch.cuda.device_count()) +print('========================================', flush=True) + +# NB(bt, 2021-05-31): Trying to get everything on the same device +#torch.device("cuda" if torch.cuda.is_available() else "cpu") + +hparams = create_hparams() +hparams.sampling_rate = 22050 # Don't change this +hparams.max_decoder_steps = 1000 # How long the audio will be before it cuts off (1000 is about 11 seconds) +hparams.gate_threshold = 0.1 # Model must be 90% sure the clip is over before ending generation (the higher this number is, the more likely that the AI will keep generating until it reaches the Max Decoder Steps) + +# Load synthesizer +checkpoint_path = args.synthesizer_checkpoint_path + +#model = load_model(hparams) +model = Tacotron2(hparams) + +model.load_state_dict(torch.load(checkpoint_path)['state_dict'], strict=True) + +_ = model.cuda().eval().half() + +print('Model is valid') + +""" + +file_size_bytes = os.path.getsize(args.synthesizer_checkpoint_path) + +metadata = { + 'file_size_bytes': file_size_bytes, +} + +with open(args.output_metadata_filename, 'w') as json_file: + json.dump(metadata, json_file) diff --git a/src/database/enums/mod.rs b/src/database/enums/mod.rs index 7ffceb80cc..a30b2cbad3 100644 --- a/src/database/enums/mod.rs +++ b/src/database/enums/mod.rs @@ -1,2 +1,3 @@ pub mod record_visibility; +pub mod tts_model_type; pub mod vocoder_type; \ No newline at end of file diff --git a/src/database/enums/tts_model_type.rs b/src/database/enums/tts_model_type.rs new file mode 100644 index 0000000000..8a5d2b8ae0 --- /dev/null +++ b/src/database/enums/tts_model_type.rs @@ -0,0 +1,40 @@ +//! These denote the TTS synthesizer model types + +use anyhow::anyhow; +use crate::util::anyhow_result::AnyhowResult; + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] +#[serde(rename_all = "lowercase")] +#[sqlx(rename_all = "lowercase")] +pub enum TtsModelType { + #[serde(rename = "tacotron2")] + Tacotron2, + + #[serde(rename = "talknet")] + Talknet, + + // Currently unused enum values: + //#[serde(rename = "not-set")] + //NotSet, + //#[serde(rename = "glowtts")] + //GlowTts, + //#[serde(rename = "glowtts-vocodes")] + //GlowTtsVocodes, +} + +impl TtsModelType { + pub fn to_str(&self) -> &'static str { + match self { + TtsModelType::Tacotron2 => "tacotron2", + TtsModelType::Talknet => "talknet", + } + } + + pub fn from_str(vocoder_type: &str) -> AnyhowResult { + match vocoder_type { + "tacotron2" => Ok(TtsModelType::Tacotron2), + "talknet" => Ok(TtsModelType::Talknet), + _ => Err(anyhow!("invalid value: {:?}", vocoder_type)), + } + } +} diff --git a/src/script_execution/mod.rs b/src/script_execution/mod.rs index 2b606badaf..a4c05b7424 100644 --- a/src/script_execution/mod.rs +++ b/src/script_execution/mod.rs @@ -4,5 +4,6 @@ pub mod google_drive_download_command; pub mod imagemagick_generate_preview_image_command; pub mod tacotron_inference_command; pub mod tacotron_model_check_command; +pub mod talknet_model_check_command; pub mod wav2lip_inference_command; pub mod wav2lip_process_upload_command; \ No newline at end of file diff --git a/src/script_execution/talknet_model_check_command.rs b/src/script_execution/talknet_model_check_command.rs new file mode 100644 index 0000000000..dc7d2968bc --- /dev/null +++ b/src/script_execution/talknet_model_check_command.rs @@ -0,0 +1,112 @@ +use crate::AnyhowResult; +use log::{info,warn}; +use std::process::{Command, Stdio}; +use subprocess::{Popen, PopenConfig, Redirection}; +use std::fs::OpenOptions; +use std::path::Path; + +/// This command is used to check talknet for being a real model +#[derive(Clone)] +pub struct TalknetModelCheckCommand { + talknet_directory: String, + check_model_script_name: String, +} + +impl TalknetModelCheckCommand { + pub fn new( + talknet_directory: &str, + check_model_script_name: &str, + ) -> Self { + Self { + talknet_directory: talknet_directory.to_string(), + check_model_script_name: check_model_script_name.to_string(), + } + } + + pub fn execute>( + &self, + synthesizer_checkpoint_path: P, + output_metadata_filename: P, + spawn_process: bool + ) -> AnyhowResult<()> { + let mut command = String::new(); + + command.push_str("echo 'test'"); + command.push_str(" && "); + command.push_str(&format!("cd {}", self.talknet_directory)); + command.push_str(" && "); + command.push_str("source python/bin/activate"); + command.push_str(" && "); + command.push_str("python "); + command.push_str(&self.check_model_script_name); + command.push_str(" --synthesizer_checkpoint_path "); + command.push_str(&synthesizer_checkpoint_path.as_ref().display().to_string()); + command.push_str(" --output_metadata_filename "); + command.push_str(&output_metadata_filename.as_ref().display().to_string()); + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + if spawn_process { + // NB: This forks and returns immediately. + //let _child_pid = command_builder.spawn()?; + + let stdout_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/talknet_upload_stdout.txt")?; + + let stderr_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/talknet_upload_stderr.txt")?; + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + stdout: Redirection::File(stdout_file), + stderr: Redirection::File(stderr_file), + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + p.detach(); + + } else { + // NB: This is a blocking call. + /*let output = command_builder.output()?; + + info!("Output status: {}", output.status); + info!("Stdout: {:?}", String::from_utf8(output.stdout)); + error!("Stderr: {:?}", String::from_utf8(output.stderr)); + + if !output.status.success() { + bail!("Bad error code: {:?}", output.status); + }*/ + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + let exit_status = p.wait()?; + + info!("Exit status: {:?}", exit_status); + } + + Ok(()) + } +} diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 95f30b27d4..fa695ae405 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -20,6 +20,7 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; use crate::common_env::CommonEnv; +use crate::database::enums::tts_model_type::TtsModelType; use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; @@ -30,6 +31,7 @@ use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_failure; use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::tacotron_model_check_command::TacotronModelCheckCommand; +use crate::script_execution::talknet_model_check_command::TalknetModelCheckCommand; use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; @@ -67,7 +69,8 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp"; // Python code const ENV_CODE_DIRECTORY : &'static str = "TTS_CODE_DIRECTORY"; -const ENV_MODEL_CHECK_SCRIPT_NAME : &'static str = "TTS_MODEL_CHECK_SCRIPT_NAME"; +const ENV_TACOTRON_CHECK_SCRIPT_NAME : &'static str = "TTS_TACOTRON_CHECK_SCRIPT_NAME"; +const ENV_TALKNET_CHECK_SCRIPT_NAME : &'static str = "TTS_TALKNET_CHECK_SCRIPT_NAME"; struct Downloader { pub download_temp_directory: PathBuf, @@ -82,7 +85,8 @@ struct Downloader { pub bucket_path_unifier: BucketPathUnifier, - pub tts_check: TacotronModelCheckCommand, + pub tacotron_tts_check: TacotronModelCheckCommand, + pub talknet_tts_check: TalknetModelCheckCommand, // Command to run pub download_script: String, @@ -176,11 +180,17 @@ async fn main() -> AnyhowResult<()> { }; let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?; - let py_script_name = easyenv::get_env_string_required(ENV_MODEL_CHECK_SCRIPT_NAME)?; + let tacotron_check_script_name = easyenv::get_env_string_required(ENV_TACOTRON_CHECK_SCRIPT_NAME)?; + let talknet_check_script_name = easyenv::get_env_string_required(ENV_TALKNET_CHECK_SCRIPT_NAME)?; - let tts_model_check_command= TacotronModelCheckCommand::new( + let tacotron_check_command= TacotronModelCheckCommand::new( &py_code_directory, - &py_script_name, + &tacotron_check_script_name, + ); + + let talknet_check_command= TalknetModelCheckCommand::new( + &py_code_directory, + &talknet_check_script_name, ); let downloader = Downloader { @@ -194,7 +204,8 @@ async fn main() -> AnyhowResult<()> { bucket_root_tts_model_uploads: bucket_root.to_string(), firehose_publisher, badge_granter, - tts_check: tts_model_check_command, + tacotron_tts_check: tacotron_check_command, + talknet_tts_check: talknet_check_command, job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as i32, no_op_logger_millis: common_env.no_op_logger_millis, @@ -324,15 +335,13 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho let download_filename = downloader.google_drive_downloader .download_file(&download_url, &temp_dir).await?; - let private_bucket_hash = hash_file_sha2(&download_filename)?; - - info!("File hash: {}", private_bucket_hash); - - let synthesizer_model_bucket_path = downloader.bucket_path_unifier.tts_synthesizer_path( - &private_bucket_hash); - - info!("Destination bucket path: {:?}", &synthesizer_model_bucket_path); + let model_type = if download_filename.to_lowercase().ends_with("zip") { + TtsModelType::Talknet + } else { + TtsModelType::Tacotron2 + }; + info!("Uploaded model type: {:?}", model_type); // ==================== RUN MODEL CHECK ==================== // @@ -340,15 +349,26 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho redis_logger.log_status("checking model")?; - let file_path = PathBuf::from(download_filename); + let file_path = PathBuf::from(download_filename.clone()); let output_metadata_fs_path = temp_dir.path().join("metadata.json"); - downloader.tts_check.execute( - &file_path, - &output_metadata_fs_path, - false, - )?; + match model_type { + TtsModelType::Tacotron2 => { + downloader.tacotron_tts_check.execute( + &file_path, + &output_metadata_fs_path, + false, + )?; + }, + TtsModelType::Talknet => { + downloader.talknet_tts_check.execute( + &file_path, + &output_metadata_fs_path, + false, + )?; + }, + } // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // @@ -362,6 +382,17 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("Uploading model to GCS..."); + let private_bucket_hash = hash_file_sha2(&download_filename)?; + + info!("File hash: {}", private_bucket_hash); + + let synthesizer_model_bucket_path = match model_type { + TtsModelType::Tacotron2 => downloader.bucket_path_unifier.tts_synthesizer_path(&private_bucket_hash), + TtsModelType::Talknet => downloader.bucket_path_unifier.tts_zipped_synthesizer_path(&private_bucket_hash), + }; + + info!("Destination bucket path: {:?}", &synthesizer_model_bucket_path); + redis_logger.log_status("uploading model")?; downloader.bucket_client.upload_filename(&synthesizer_model_bucket_path, &file_path).await?; diff --git a/src/util/buckets/bucket_path_unifier.rs b/src/util/buckets/bucket_path_unifier.rs index cc8a11c6d0..28bcd4daf2 100644 --- a/src/util/buckets/bucket_path_unifier.rs +++ b/src/util/buckets/bucket_path_unifier.rs @@ -59,6 +59,15 @@ impl BucketPathUnifier { .join(model_filename) } + pub fn tts_zipped_synthesizer_path(&self, tts_synthesizer_file_hash: &str) -> PathBuf { + let hashed_path = Self::hashed_directory_path(tts_synthesizer_file_hash); + let model_filename = format!("{}.zip", &tts_synthesizer_file_hash); + + self.tts_synthesizer_model_root + .join(hashed_path) + .join(model_filename) + } + // ==================== TTS INFERENCE OUTPUT ==================== // /// This should include the string "vocodes" for downloaders. @@ -196,6 +205,13 @@ mod tests { "/test_path_synthesizers/f/o/o/foobar.pt"); } + #[test] + fn test_tts_zipped_synthesizer_path() { + let paths = get_instance(); + assert_eq!(paths.tts_synthesizer_path("foobar").to_str().unwrap(), + "/test_path_synthesizers/f/o/o/foobar.zip"); + } + #[test] fn test_tts_inference_wav_audio_output_path() { let paths = get_instance(); From 3697c353ccd1ad28e3ff7ed2f4927e609de23732 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 11 Sep 2021 02:48:04 -0400 Subject: [PATCH 0602/1042] stats for both queues --- .../jobs/get_tts_inference_queue_count.rs | 118 ++++++++++++++++++ .../jobs/get_w2l_inference_queue_count.rs | 118 ++++++++++++++++++ .../endpoints/moderation/jobs/mod.rs | 2 + src/http_server/endpoints/moderation/mod.rs | 1 + src/main.rs | 15 +++ 5 files changed, 254 insertions(+) create mode 100644 src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs create mode 100644 src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs create mode 100644 src/http_server/endpoints/moderation/jobs/mod.rs diff --git a/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs b/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs new file mode 100644 index 0000000000..4b2767b15c --- /dev/null +++ b/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs @@ -0,0 +1,118 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; +use crate::server_state::ServerState; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +#[derive(Serialize)] +pub struct GetTtsInferenceQueueCountResponse { + pub success: bool, + pub pending_count: i64, +} + +#[derive(Debug, Serialize)] +pub enum GetTtsInferenceQueueCountError { + ServerError, + Unauthorized, +} + +impl ResponseError for GetTtsInferenceQueueCountError { + fn status_code(&self) -> StatusCode { + match *self { + GetTtsInferenceQueueCountError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + GetTtsInferenceQueueCountError::Unauthorized => StatusCode::UNAUTHORIZED, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for GetTtsInferenceQueueCountError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn get_tts_inference_queue_count_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetTtsInferenceQueueCountError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(GetTtsInferenceQueueCountError::Unauthorized); + } + }; + + // TODO: Not a good fit for this permission. + if !user_session.can_ban_users { + warn!("user is not allowed to view bans: {}", user_session.user_token); + return Err(GetTtsInferenceQueueCountError::Unauthorized); + } + + // NB: Lookup failure is Err(RowNotFound). + let maybe_result = sqlx::query_as!( + PendingCountResult, + r#" +SELECT + count(*) as pending_count +FROM + tts_inference_jobs +WHERE + status = "pending" + "#, + ) + .fetch_one(&server_state.mysql_pool) + .await; + + let result : PendingCountResult = match maybe_result { + Ok(result) => result, + Err(err) => return Err(GetTtsInferenceQueueCountError::ServerError), + }; + + let response = GetTtsInferenceQueueCountResponse { + success: true, + pending_count: result.pending_count, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetTtsInferenceQueueCountError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} + +#[derive(Serialize)] +pub struct PendingCountResult { + pub pending_count: i64, +} diff --git a/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs b/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs new file mode 100644 index 0000000000..e622df5f47 --- /dev/null +++ b/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs @@ -0,0 +1,118 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; +use crate::server_state::ServerState; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +#[derive(Serialize)] +pub struct GetW2lInferenceQueueCountResponse { + pub success: bool, + pub pending_count: i64, +} + +#[derive(Debug, Serialize)] +pub enum GetW2lInferenceQueueCountError { + ServerError, + Unauthorized, +} + +impl ResponseError for GetW2lInferenceQueueCountError { + fn status_code(&self) -> StatusCode { + match *self { + GetW2lInferenceQueueCountError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + GetW2lInferenceQueueCountError::Unauthorized => StatusCode::UNAUTHORIZED, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for GetW2lInferenceQueueCountError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn get_w2l_inference_queue_count_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetW2lInferenceQueueCountError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(GetW2lInferenceQueueCountError::Unauthorized); + } + }; + + // TODO: Not a good fit for this permission. + if !user_session.can_ban_users { + warn!("user is not allowed to view bans: {}", user_session.user_token); + return Err(GetW2lInferenceQueueCountError::Unauthorized); + } + + // NB: Lookup failure is Err(RowNotFound). + let maybe_result = sqlx::query_as!( + PendingCountResult, + r#" +SELECT + count(*) as pending_count +FROM + w2l_inference_jobs +WHERE + status = "pending" + "#, + ) + .fetch_one(&server_state.mysql_pool) + .await; + + let result : PendingCountResult = match maybe_result { + Ok(result) => result, + Err(err) => return Err(GetW2lInferenceQueueCountError::ServerError), + }; + + let response = GetW2lInferenceQueueCountResponse { + success: true, + pending_count: result.pending_count, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetW2lInferenceQueueCountError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} + +#[derive(Serialize)] +pub struct PendingCountResult { + pub pending_count: i64, +} diff --git a/src/http_server/endpoints/moderation/jobs/mod.rs b/src/http_server/endpoints/moderation/jobs/mod.rs new file mode 100644 index 0000000000..1b0e583d12 --- /dev/null +++ b/src/http_server/endpoints/moderation/jobs/mod.rs @@ -0,0 +1,2 @@ +pub mod get_tts_inference_queue_count; +pub mod get_w2l_inference_queue_count; \ No newline at end of file diff --git a/src/http_server/endpoints/moderation/mod.rs b/src/http_server/endpoints/moderation/mod.rs index 9494091cad..bddfab2dcb 100644 --- a/src/http_server/endpoints/moderation/mod.rs +++ b/src/http_server/endpoints/moderation/mod.rs @@ -1,4 +1,5 @@ pub mod ip_bans; +pub mod jobs; pub mod user_bans; pub mod user_roles; pub mod users; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index efff6b912a..48d027e8e2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -107,6 +107,8 @@ use std::sync::Arc; use std::time::Duration; use crate::http_server::endpoints::moderation::users::list_users::list_users_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; +use crate::http_server::endpoints::moderation::jobs::get_tts_inference_queue_count::get_tts_inference_queue_count_handler; +use crate::http_server::endpoints::moderation::jobs::get_w2l_inference_queue_count::get_w2l_inference_queue_count_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -419,6 +421,19 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service( + web::scope("/jobs") + .service( + web::resource("/tts_inference_queue_stats") + .route(web::get().to(get_tts_inference_queue_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/w2l_inference_queue_stats") + .route(web::get().to(get_w2l_inference_queue_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) ) // ==================== TTS ==================== .service( From 71dc65a5e48797999375fde64fc31920a239423c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 11 Sep 2021 02:49:38 -0400 Subject: [PATCH 0603/1042] codegen db --- sqlx-data.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/sqlx-data.json b/sqlx-data.json index a51c4ffdc8..58451f3522 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1838,6 +1838,31 @@ "nullable": [] } }, + "196d879939ea1aac35bf1a5413a9fe28ff7c1bf641d6c6752d7746df211f2e39": { + "query": "\nSELECT\n count(*) as pending_count\nFROM\n tts_inference_jobs\nWHERE\n status = \"pending\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pending_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false + ] + } + }, "1cc62e5cdc8de18969d7e8018d91c38794e5a7d23f117e5b0a4126845f300e60": { "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE\n token = ?\n AND deleted_at IS NULL\n ", "describe": { @@ -7687,6 +7712,31 @@ "nullable": [] } }, + "d9a30d5fcf83f723e696000893ef60774bf286d77b609c5c5f2b51a738fa283c": { + "query": "\nSELECT\n count(*) as pending_count\nFROM\n w2l_inference_jobs\nWHERE\n status = \"pending\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pending_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false + ] + } + }, "da3197d279b94817c6c8a6277aff7a0cf44d66e6777f46680b1b07de48875dd5": { "query": "\nINSERT INTO user_badges\nSET\n user_token = ?,\n badge_slug = ?\n ", "describe": { From 4d928bd8def045b0a1ed43f389ad0b6dcd65ba37 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 11 Sep 2021 04:19:06 -0400 Subject: [PATCH 0604/1042] Rate limiter for TTS --- Cargo.lock | 309 ++++++++++++++++-- Cargo.toml | 2 + .../endpoints/tts/enqueue_infer_tts.rs | 8 + src/main.rs | 24 +- src/server_state.rs | 3 + src/util/redis/mod.rs | 3 +- src/util/redis/redis_rate_limiter.rs | 68 ++++ 7 files changed, 384 insertions(+), 33 deletions(-) create mode 100644 src/util/redis/redis_rate_limiter.rs diff --git a/Cargo.lock b/Cargo.lock index a93c002336..3c166d2b24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -180,7 +180,7 @@ dependencies = [ "copyless", "futures-channel", "futures-util", - "smallvec", + "smallvec 1.6.1", "tokio 0.2.25", ] @@ -250,7 +250,7 @@ dependencies = [ "lazy_static", "log", "num_cpus", - "parking_lot", + "parking_lot 0.11.1", "threadpool", ] @@ -406,6 +406,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "ascii" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" + [[package]] name = "async-channel" version = "1.6.1" @@ -495,7 +501,7 @@ dependencies = [ "async-global-executor", "async-io", "async-lock", - "crossbeam-utils", + "crossbeam-utils 0.8.4", "futures-channel", "futures-core", "futures-io", @@ -786,6 +792,16 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + [[package]] name = "bytes" version = "0.5.6" @@ -886,6 +902,28 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "combine" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +dependencies = [ + "ascii", + "byteorder", + "either", + "memchr", + "unreachable", +] + [[package]] name = "combine" version = "4.5.2" @@ -993,7 +1031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils", + "crossbeam-utils 0.8.4", ] [[package]] @@ -1003,7 +1041,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f6cb3c7f5b8e51bc3ebb73a2327ad4abdbd119dc13223f14f961d2f38486756" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils", + "crossbeam-utils 0.8.4", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 0.1.10", + "lazy_static", ] [[package]] @@ -1301,6 +1350,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + [[package]] name = "futures" version = "0.3.15" @@ -1831,6 +1886,18 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" +[[package]] +name = "limitation" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e415413963930f7050f21994bfd336dc75ef8b5b3b142e99c55b72b61f726d5" +dependencies = [ + "chrono", + "futures 0.1.31", + "redis 0.13.0", + "time 0.1.44", +] + [[package]] name = "linked-hash-map" version = "0.5.4" @@ -1852,6 +1919,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84f9a2d3e27ce99ce2c3aad0b09b1a7b916293ea9b2bf624c13fe646fadd8da4" +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + [[package]] name = "lock_api" version = "0.4.4" @@ -1915,6 +1991,12 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + [[package]] name = "md-5" version = "0.9.1" @@ -2136,7 +2218,7 @@ dependencies = [ "num-traits", "rand 0.7.3", "serde", - "smallvec", + "smallvec 1.6.1", "zeroize", ] @@ -2241,6 +2323,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.6.2", + "rustc_version", +] + [[package]] name = "parking_lot" version = "0.11.1" @@ -2248,8 +2341,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" dependencies = [ "instant", - "lock_api", - "parking_lot_core", + "lock_api 0.4.4", + "parking_lot_core 0.8.3", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "rustc_version", + "smallvec 0.6.14", + "winapi 0.3.9", ] [[package]] @@ -2261,8 +2369,8 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall", - "smallvec", + "redox_syscall 0.2.8", + "smallvec 1.6.1", "winapi 0.3.9", ] @@ -2436,7 +2544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" dependencies = [ "log", - "parking_lot", + "parking_lot 0.11.1", "scheduled-thread-pool", ] @@ -2447,7 +2555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "182473b876b0b93e353682ec58e207dd1cb4a62278bbe0045fe52b86b74363bb" dependencies = [ "r2d2", - "redis", + "redis 0.20.1", ] [[package]] @@ -2574,6 +2682,28 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "redis" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb1079bc5692c03e1479bb7086ac870ac2aaf7115b5e72d314c01368f1a747e" +dependencies = [ + "bytes 0.4.12", + "combine 3.8.1", + "dtoa", + "futures 0.1.31", + "itoa", + "percent-encoding 2.1.0", + "sha1", + "tokio-codec", + "tokio-executor", + "tokio-io", + "tokio-sync", + "tokio-tcp", + "tokio-uds", + "url 2.2.2", +] + [[package]] name = "redis" version = "0.20.1" @@ -2581,7 +2711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a32cb439c4e89c1e6415e5b3b23d9d8cc6dc1bf5a8cade19adbd5418de803be" dependencies = [ "async-trait", - "combine", + "combine 4.5.2", "dtoa", "itoa", "percent-encoding 2.1.0", @@ -2589,6 +2719,12 @@ dependencies = [ "url 2.2.2", ] +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + [[package]] name = "redox_syscall" version = "0.2.8" @@ -2605,7 +2741,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ "getrandom 0.2.2", - "redox_syscall", + "redox_syscall 0.2.8", ] [[package]] @@ -2739,7 +2875,7 @@ dependencies = [ "base64 0.13.0", "cfg-if 1.0.0", "chrono", - "futures", + "futures 0.3.15", "hex", "hmac 0.9.0", "http", @@ -2788,7 +2924,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" dependencies = [ - "parking_lot", + "parking_lot 0.11.1", ] [[package]] @@ -2974,6 +3110,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + [[package]] name = "smallvec" version = "1.6.1" @@ -3046,7 +3191,7 @@ dependencies = [ "crc", "crossbeam-channel", "crossbeam-queue", - "crossbeam-utils", + "crossbeam-utils 0.8.4", "digest", "either", "futures-channel", @@ -3061,14 +3206,14 @@ dependencies = [ "memchr", "num-bigint 0.3.2", "once_cell", - "parking_lot", + "parking_lot 0.11.1", "percent-encoding 2.1.0", "rand 0.7.3", "rsa", "serde", "sha-1", "sha2", - "smallvec", + "smallvec 1.6.1", "sqlformat", "sqlx-rt", "stringprep", @@ -3086,7 +3231,7 @@ dependencies = [ "cargo_metadata", "dotenv", "either", - "futures", + "futures 0.3.15", "heck", "hex", "lazy_static", @@ -3198,7 +3343,8 @@ dependencies = [ "diesel", "dotenv", "easyenv", - "futures", + "futures 0.1.31", + "futures 0.3.15", "hex", "hmac 0.10.1", "hostname", @@ -3207,6 +3353,7 @@ dependencies = [ "jwt", "lazy_static", "lfu", + "limitation", "log", "magic-crypt", "md-5", @@ -3303,7 +3450,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.3", - "redox_syscall", + "redox_syscall 0.2.8", "remove_dir_all", "winapi 0.3.9", ] @@ -3454,12 +3601,44 @@ dependencies = [ "libc", "mio 0.7.11", "once_cell", - "parking_lot", + "parking_lot 0.11.1", "pin-project-lite 0.2.6", "signal-hook-registry", "winapi 0.3.9", ] +[[package]] +name = "tokio-codec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "tokio-io", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", +] + [[package]] name = "tokio-macros" version = "0.2.6" @@ -3481,6 +3660,49 @@ dependencies = [ "tokio 0.2.25", ] +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "mio 0.6.23", + "num_cpus", + "parking_lot 0.9.0", + "slab", + "tokio-executor", + "tokio-io", + "tokio-sync", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures 0.1.31", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "mio 0.6.23", + "tokio-io", + "tokio-reactor", +] + [[package]] name = "tokio-tls" version = "0.3.1" @@ -3491,6 +3713,24 @@ dependencies = [ "tokio 0.2.25", ] +[[package]] +name = "tokio-uds" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "libc", + "log", + "mio 0.6.23", + "mio-uds", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + [[package]] name = "tokio-util" version = "0.3.1" @@ -3560,12 +3800,12 @@ dependencies = [ "async-trait", "cfg-if 1.0.0", "enum-as-inner", - "futures", + "futures 0.3.15", "idna 0.2.3", "lazy_static", "log", "rand 0.7.3", - "smallvec", + "smallvec 1.6.1", "thiserror", "tokio 0.2.25", "url 2.2.2", @@ -3578,13 +3818,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "710f593b371175db53a26d0b38ed2978fafb9e9e8d3868b1acd753ea18df0ceb" dependencies = [ "cfg-if 0.1.10", - "futures", + "futures 0.3.15", "ipconfig", "lazy_static", "log", "lru-cache", "resolv-conf", - "smallvec", + "smallvec 1.6.1", "thiserror", "tokio 0.2.25", "trust-dns-proto", @@ -3675,6 +3915,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +dependencies = [ + "void", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -3773,6 +4022,12 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "waker-fn" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 617394b102..e31f0c7e9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ derive_more = "0.99.14" dotenv = "0.15.0" easyenv = "0.0.5" futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool +futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate hex = "0.4.3" hostname = "0.3.1" lazy_static = "1.4.0" @@ -66,6 +67,7 @@ tokio = { version = "0.2.25", features = ["macros"] } sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } # Redis Database +limitation = "0.1.1" # Redis rate limiter r2d2_redis = "0.14.0" # Files diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 30eb0f307d..a41d83d1e9 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -44,6 +44,7 @@ pub struct InferTtsSuccessResponse { pub enum InferTtsError { BadInput(String), ServerError, + RateLimited, } impl ResponseError for InferTtsError { @@ -51,6 +52,7 @@ impl ResponseError for InferTtsError { match *self { InferTtsError::BadInput(_) => StatusCode::BAD_REQUEST, InferTtsError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + InferTtsError::RateLimited => StatusCode::TOO_MANY_REQUESTS, } } @@ -58,6 +60,7 @@ impl ResponseError for InferTtsError { let error_reason = match self { InferTtsError::BadInput(reason) => reason.to_string(), InferTtsError::ServerError => "server error".to_string(), + InferTtsError::RateLimited => "rate limited".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -69,6 +72,11 @@ pub async fn infer_tts_handler( request: web::Json, server_state: web::Data>) -> Result { + + if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { + return Err(InferTtsError::RateLimited); + } + let maybe_user_session = server_state .session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool) diff --git a/src/main.rs b/src/main.rs index 48d027e8e2..737633680e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,17 +36,21 @@ use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::default_route_404::default_route_404; use crate::http_server::endpoints::events::list_events::list_events_handler; +use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha_easy_handler; use crate::http_server::endpoints::misc::enable_alpha_handler::enable_alpha_handler; use crate::http_server::endpoints::moderation::ip_bans::add_ip_ban::add_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::delete_ip_ban::delete_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::list_ip_bans::list_ip_bans_handler; +use crate::http_server::endpoints::moderation::jobs::get_tts_inference_queue_count::get_tts_inference_queue_count_handler; +use crate::http_server::endpoints::moderation::jobs::get_w2l_inference_queue_count::get_w2l_inference_queue_count_handler; use crate::http_server::endpoints::moderation::user_bans::ban_user::ban_user_handler; use crate::http_server::endpoints::moderation::user_bans::list_banned_users::list_banned_users_handler; use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user_roles_handler; use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler; use crate::http_server::endpoints::moderation::user_roles::set_user_role::set_user_role_handler; +use crate::http_server::endpoints::moderation::users::list_users::list_users_handler; use crate::http_server::endpoints::root_index::get_root_index; use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handler; use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; @@ -95,6 +99,7 @@ use crate::threads::ip_banlist_set::IpBanlistSet; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; use crate::util::encrypted_sort_id::SortKeyCrypto; +use crate::util::redis::redis_rate_limiter::RedisRateLimiter; use futures::Future; use futures::executor::ThreadPool; use log::{info}; @@ -105,10 +110,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; -use crate::http_server::endpoints::moderation::users::list_users::list_users_handler; -use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; -use crate::http_server::endpoints::moderation::jobs::get_tts_inference_queue_count::get_tts_inference_queue_count_handler; -use crate::http_server::endpoints::moderation::jobs::get_w2l_inference_queue_count::get_w2l_inference_queue_count_handler; +use limitation::Limiter; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -184,11 +186,22 @@ async fn main() -> AnyhowResult<()> { firehose_publisher: firehose_publisher.clone(), // NB: Also safe }; - let redis_manager = RedisConnectionManager::new(redis_connection_string)?; + let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?; let redis_pool = r2d2::Pool::builder() .build(redis_manager)?; + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&redis_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + let redis_rate_limiter = RedisRateLimiter::new(limiter, limiter_enabled); + info!("Reading env vars and setting up utils..."); let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); @@ -262,6 +275,7 @@ async fn main() -> AnyhowResult<()> { hostname: server_hostname, mysql_pool: pool, redis_pool, + redis_rate_limiter, firehose_publisher, badge_granter, cookie_manager, diff --git a/src/server_state.rs b/src/server_state.rs index 13c809a071..9e29cfb02b 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -5,6 +5,7 @@ use crate::http_server::web_utils::session_checker::SessionChecker; use crate::threads::ip_banlist_set::IpBanlistSet; use crate::util::buckets::bucket_client::BucketClient; use crate::util::encrypted_sort_id::SortKeyCrypto; +use crate::util::redis::redis_rate_limiter::RedisRateLimiter; use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; @@ -20,6 +21,8 @@ pub struct ServerState { pub redis_pool: r2d2::Pool, + pub redis_rate_limiter: RedisRateLimiter, + pub cookie_manager: CookieManager, pub session_checker: SessionChecker, diff --git a/src/util/redis/mod.rs b/src/util/redis/mod.rs index 7626fe14a6..fc70e62d66 100644 --- a/src/util/redis/mod.rs +++ b/src/util/redis/mod.rs @@ -1,2 +1,3 @@ pub mod redis_job_status_logger; -pub mod redis_keys; \ No newline at end of file +pub mod redis_keys; +pub mod redis_rate_limiter; \ No newline at end of file diff --git a/src/util/redis/redis_rate_limiter.rs b/src/util/redis/redis_rate_limiter.rs new file mode 100644 index 0000000000..f7442b403d --- /dev/null +++ b/src/util/redis/redis_rate_limiter.rs @@ -0,0 +1,68 @@ +use actix_http::http::{HeaderName, HeaderMap}; +use actix_web::HttpRequest; +use crate::http_server::web_utils::ip_address::get_request_ip; +use futures_old_for_limiter::Future; +use limitation::{Limiter, Error}; +use log::info; + +/// If this HTTP header is set, the rate limiter can be bypassed (eg. for debugging) +const RATE_LIMIT_BYPASS_HEADER : &'static str = "limitless"; + +#[derive(Clone)] // NB: Limiter is `Clone` +pub struct RedisRateLimiter { + limiter: Limiter, + limiter_enabled: bool, + rate_limit_bypass_header: HeaderName, +} + +pub enum RateLimiterError { + RateLimitExceededError, + ClientError, // NB: Fail open for this! +} + +impl RedisRateLimiter { + pub fn new(limiter: Limiter, limiter_enabled: bool) -> Self { + let rate_limit_bypass_header = HeaderName::from_static(RATE_LIMIT_BYPASS_HEADER); + RedisRateLimiter { + limiter, + limiter_enabled, + rate_limit_bypass_header, + } + } + + pub fn rate_limit_request(&self, request: &HttpRequest) -> Result<(), RateLimiterError> { + if !self.limiter_enabled { + return Ok(()) + } + + let headers = request.headers(); + if headers.contains_key(&self.rate_limit_bypass_header) { + info!("Bypassing rate limiter with special bypass/debug header."); + return Ok(()); + } + + let ip_address = get_request_ip(&request); + + let rate_limit_key = format!("rate_limit:{}", ip_address); + self.rate_limit_key(&rate_limit_key) + } + + pub fn rate_limit_key(&self, rate_limit_key: &str) -> Result<(), RateLimiterError> { + if !self.limiter_enabled { + return Ok(()) + } + + // NB: This library uses old-school futures (pre-async/await) + let permit = self.limiter.count(rate_limit_key); + + // TODO/FIXME: This could block. + match permit.wait() { + Ok(_) => Ok(()), + Err(err) => match err { + Error::Client(_) => Ok(()), // NB: Fail open for failure to connect to Redis + Error::Time(_) => Ok(()), // NB: Fail open for key parsing + Error::LimitExceeded(_) => Err(RateLimiterError::RateLimitExceededError), + }, + } + } +} From 591d828da39d962312f12c909c1c13341c38a467 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 11 Sep 2021 04:27:40 -0400 Subject: [PATCH 0605/1042] more rate limiting for important endpoints --- .../endpoints/tts/enqueue_infer_tts.rs | 1 - .../endpoints/tts/enqueue_upload_tts_model.rs | 7 ++++ .../endpoints/w2l/enqueue_infer_w2l.rs | 6 ++++ .../w2l/enqueue_infer_w2l_with_uploads.rs | 35 +++++-------------- .../w2l/enqueue_upload_w2l_template.rs | 7 ++++ 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index a41d83d1e9..3438922477 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -72,7 +72,6 @@ pub async fn infer_tts_handler( request: web::Json, server_state: web::Data>) -> Result { - if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { return Err(InferTtsError::RateLimited); } diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index 0747e85529..79165bb12a 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -44,6 +44,7 @@ pub enum UploadTtsModelError { BadInput(String), MustBeLoggedIn, ServerError, + RateLimited, } impl ResponseError for UploadTtsModelError { @@ -52,6 +53,7 @@ impl ResponseError for UploadTtsModelError { UploadTtsModelError::BadInput(_) => StatusCode::BAD_REQUEST, UploadTtsModelError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, UploadTtsModelError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + UploadTtsModelError::RateLimited => StatusCode::TOO_MANY_REQUESTS, } } @@ -60,6 +62,7 @@ impl ResponseError for UploadTtsModelError { UploadTtsModelError::BadInput(reason) => reason.to_string(), UploadTtsModelError::MustBeLoggedIn => "user must be logged in".to_string(), UploadTtsModelError::ServerError => "server error".to_string(), + UploadTtsModelError::RateLimited => "rate limited".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -71,6 +74,10 @@ pub async fn upload_tts_model_handler( request: web::Json, server_state: web::Data>) -> Result { + if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { + return Err(UploadTtsModelError::RateLimited); + } + let maybe_user_session = server_state .session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool) diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index a14a680815..c0c25714f0 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -41,6 +41,7 @@ pub enum InferW2lError { BadInput(String), NotAuthorized, ServerError, + RateLimited, } impl ResponseError for InferW2lError { @@ -49,6 +50,7 @@ impl ResponseError for InferW2lError { InferW2lError::BadInput(_) => StatusCode::BAD_REQUEST, InferW2lError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, InferW2lError::NotAuthorized => StatusCode::UNAUTHORIZED, + InferW2lError::RateLimited => StatusCode::TOO_MANY_REQUESTS, } } @@ -57,6 +59,7 @@ impl ResponseError for InferW2lError { InferW2lError::BadInput(reason) => reason.to_string(), InferW2lError::ServerError => "server error".to_string(), InferW2lError::NotAuthorized => "not authorized".to_string(), + InferW2lError::RateLimited => "rate limited".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -73,6 +76,9 @@ pub async fn infer_w2l_handler( unimplemented!("this isn't finished"); } + if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { + return Err(InferW2lError::RateLimited); + } let maybe_session = server_state .session_checker diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 1fb2679fea..68709a3940 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -58,6 +58,7 @@ pub enum InferW2lWithUploadError { BadInput(String), EmptyFileUploaded, ServerError, + RateLimited, } impl ResponseError for InferW2lWithUploadError { @@ -66,6 +67,7 @@ impl ResponseError for InferW2lWithUploadError { InferW2lWithUploadError::BadInput(_) => StatusCode::BAD_REQUEST, InferW2lWithUploadError::EmptyFileUploaded => StatusCode::BAD_REQUEST, InferW2lWithUploadError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + InferW2lWithUploadError::RateLimited => StatusCode::TOO_MANY_REQUESTS, } } @@ -74,6 +76,7 @@ impl ResponseError for InferW2lWithUploadError { InferW2lWithUploadError::BadInput(reason) => reason.to_string(), InferW2lWithUploadError::EmptyFileUploaded => "empty file uploaded".to_string(), InferW2lWithUploadError::ServerError => "server error".to_string(), + InferW2lWithUploadError::RateLimited => "rate limited".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -84,31 +87,16 @@ impl ResponseError for InferW2lWithUploadError { pub async fn enqueue_infer_w2l_with_uploads( http_request: HttpRequest, server_state: web::Data>, - mut payload: Multipart) - //-> Result, HttpResponse> - -> Result -{ - // ==================== TODO: READ IP AND RATE LIMIT ==================== // + mut payload: Multipart +) -> Result { - let ip_address = get_request_ip(&http_request); + // ==================== RATE LIMIT ==================== // - // TODO: Check IP address holds - // TODO: Rate Limiting! - /*let mut hold_exists = handler_state.redis_client.get_ip_hold(&ip_address) - .map_err(|err| HttpResponse::InternalServerError() - .body(format!("Error: {:?}", err)))?; - - match ip_address.as_ref() { - "127.0.0.1" => { - warn!("Allowing IP address hold bypass for {}", &ip_address); - hold_exists = false - }, - _ => {}, + if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { + return Err(InferW2lWithUploadError::RateLimited); } - if hold_exists { - return Err(HttpResponse::TooManyRequests().body("you already have pending requests").into()) - }*/ + let ip_address = get_request_ip(&http_request); // ==================== READ SESSION ==================== // @@ -343,11 +331,6 @@ SET } }; - // TODO: IP Address holds. - //handler_state.redis_client.set_ip_hold(&ip_address) - // .map_err(|err| HttpResponse::InternalServerError() - // .body(format!("Redis Err: {:?}", err)))?; - server_state.firehose_publisher.enqueue_w2l_inference(maybe_user_token.as_deref(), &job_token, &template_token) .await .map_err(|e| { diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index 9d3a9b0cfb..681e9f32b1 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -45,6 +45,7 @@ pub enum UploadW2lTemplateError { BadInput(String), MustBeLoggedIn, ServerError, + RateLimited, } impl ResponseError for UploadW2lTemplateError { @@ -53,6 +54,7 @@ impl ResponseError for UploadW2lTemplateError { UploadW2lTemplateError::BadInput(_) => StatusCode::BAD_REQUEST, UploadW2lTemplateError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, UploadW2lTemplateError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + UploadW2lTemplateError::RateLimited => StatusCode::TOO_MANY_REQUESTS, } } @@ -61,6 +63,7 @@ impl ResponseError for UploadW2lTemplateError { UploadW2lTemplateError::BadInput(reason) => reason.to_string(), UploadW2lTemplateError::MustBeLoggedIn => "user must be logged in".to_string(), UploadW2lTemplateError::ServerError => "server error".to_string(), + UploadW2lTemplateError::RateLimited => "rate limited".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -72,6 +75,10 @@ pub async fn upload_w2l_template_handler( request: web::Json, server_state: web::Data>) -> Result { + if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { + return Err(UploadW2lTemplateError::RateLimited); + } + let maybe_user_session = server_state .session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool) From d359427e42c782a96ef99565f5ef2a54c76eb7da Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 11 Sep 2021 04:43:17 -0400 Subject: [PATCH 0606/1042] fix rate limiter package --- src/{clients => http_clients}/mod.rs | 0 .../tts_inference_sidecar_client.rs | 0 src/http_server/web_utils/mod.rs | 1 + .../redis => http_server/web_utils}/redis_rate_limiter.rs | 3 +++ src/main.rs | 6 +++--- src/server_state.rs | 2 +- src/tts_inference_job.rs | 4 ++-- src/util/redis/mod.rs | 3 +-- 8 files changed, 11 insertions(+), 8 deletions(-) rename src/{clients => http_clients}/mod.rs (100%) rename src/{clients => http_clients}/tts_inference_sidecar_client.rs (100%) rename src/{util/redis => http_server/web_utils}/redis_rate_limiter.rs (92%) diff --git a/src/clients/mod.rs b/src/http_clients/mod.rs similarity index 100% rename from src/clients/mod.rs rename to src/http_clients/mod.rs diff --git a/src/clients/tts_inference_sidecar_client.rs b/src/http_clients/tts_inference_sidecar_client.rs similarity index 100% rename from src/clients/tts_inference_sidecar_client.rs rename to src/http_clients/tts_inference_sidecar_client.rs diff --git a/src/http_server/web_utils/mod.rs b/src/http_server/web_utils/mod.rs index 9e58e92491..64a1b8c708 100644 --- a/src/http_server/web_utils/mod.rs +++ b/src/http_server/web_utils/mod.rs @@ -1,6 +1,7 @@ pub mod cookie_manager; pub mod ip_address; pub mod read_multipart_field_bytes; +pub mod redis_rate_limiter; pub mod response_error_helpers; pub mod response_success_helpers; pub mod serialize_as_json_error; diff --git a/src/util/redis/redis_rate_limiter.rs b/src/http_server/web_utils/redis_rate_limiter.rs similarity index 92% rename from src/util/redis/redis_rate_limiter.rs rename to src/http_server/web_utils/redis_rate_limiter.rs index f7442b403d..9bfef9f180 100644 --- a/src/util/redis/redis_rate_limiter.rs +++ b/src/http_server/web_utils/redis_rate_limiter.rs @@ -1,3 +1,6 @@ +//! NB: This doesn't live in the `util.redis` package because it depends on `http_server`. +//! It's only used for the server anyway, so living here makes sense. + use actix_http::http::{HeaderName, HeaderMap}; use actix_web::HttpRequest; use crate::http_server::web_utils::ip_address::get_request_ip; diff --git a/src/main.rs b/src/main.rs index 737633680e..f0fd377d17 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,8 +16,8 @@ pub const RESERVED_USERNAMES : &'static str = include_str!("../db/reserved_usern pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../db/reserved_usernames_including.txt"); pub const BANNED_SLURS : &'static str = include_str!("../db/banned_slurs.txt"); -pub mod clients; pub mod database; +pub mod http_clients; pub mod http_server; pub mod server_state; pub mod shared_constants; @@ -90,6 +90,7 @@ use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_h use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; use crate::http_server::middleware::ip_filter_middleware::IpFilter; use crate::http_server::web_utils::cookie_manager::CookieManager; +use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::server_state::{ServerState, EnvConfig}; use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; @@ -99,9 +100,9 @@ use crate::threads::ip_banlist_set::IpBanlistSet; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; use crate::util::encrypted_sort_id::SortKeyCrypto; -use crate::util::redis::redis_rate_limiter::RedisRateLimiter; use futures::Future; use futures::executor::ThreadPool; +use limitation::Limiter; use log::{info}; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2; @@ -110,7 +111,6 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; -use limitation::Limiter; // TODO TODO TODO TODO // TODO TODO TODO TODO diff --git a/src/server_state.rs b/src/server_state.rs index 9e29cfb02b..c740483434 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,11 +1,11 @@ use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::web_utils::cookie_manager::CookieManager; +use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::threads::ip_banlist_set::IpBanlistSet; use crate::util::buckets::bucket_client::BucketClient; use crate::util::encrypted_sort_id::SortKeyCrypto; -use crate::util::redis::redis_rate_limiter::RedisRateLimiter; use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index a6cc2af609..4f6949868a 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -10,9 +10,9 @@ #[macro_use] extern crate serde_derive; -pub mod clients; pub mod common_env; pub mod database; +pub mod http_clients; pub mod job_queries; pub mod script_execution; pub mod shared_constants; @@ -20,10 +20,10 @@ pub mod util; use anyhow::{anyhow, Error}; use chrono::{Utc, DateTime, TimeZone}; -use crate::clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; use crate::common_env::CommonEnv; use crate::database::enums::vocoder_type::VocoderType; use crate::database::mediators::firehose_publisher::FirehosePublisher; +use crate::http_clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; use crate::job_queries::tts_inference_job_queries::get_tts_model_by_token; use crate::job_queries::tts_inference_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::tts_inference_job_queries::insert_tts_result; diff --git a/src/util/redis/mod.rs b/src/util/redis/mod.rs index fc70e62d66..7626fe14a6 100644 --- a/src/util/redis/mod.rs +++ b/src/util/redis/mod.rs @@ -1,3 +1,2 @@ pub mod redis_job_status_logger; -pub mod redis_keys; -pub mod redis_rate_limiter; \ No newline at end of file +pub mod redis_keys; \ No newline at end of file From 3e9ff8ff7c9bd876f276efb4860c805468ea5612 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 11 Sep 2021 04:44:23 -0400 Subject: [PATCH 0607/1042] fix failing test --- src/util/buckets/bucket_path_unifier.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/buckets/bucket_path_unifier.rs b/src/util/buckets/bucket_path_unifier.rs index 28bcd4daf2..3e24289dab 100644 --- a/src/util/buckets/bucket_path_unifier.rs +++ b/src/util/buckets/bucket_path_unifier.rs @@ -208,7 +208,7 @@ mod tests { #[test] fn test_tts_zipped_synthesizer_path() { let paths = get_instance(); - assert_eq!(paths.tts_synthesizer_path("foobar").to_str().unwrap(), + assert_eq!(paths.tts_zipped_synthesizer_path("foobar").to_str().unwrap(), "/test_path_synthesizers/f/o/o/foobar.zip"); } From c6018b4ebd0e7298ff113cf0b72687062726593c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 00:37:47 -0400 Subject: [PATCH 0608/1042] add seconds of waiting to job status --- .../jobs/get_tts_inference_queue_count.rs | 23 +++++++++++++------ .../jobs/get_w2l_inference_queue_count.rs | 23 +++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs b/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs index 4b2767b15c..991e13089e 100644 --- a/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs +++ b/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs @@ -23,6 +23,7 @@ use std::sync::Arc; pub struct GetTtsInferenceQueueCountResponse { pub success: bool, pub pending_count: i64, + pub seconds_since_first: i64, } #[derive(Debug, Serialize)] @@ -84,11 +85,17 @@ pub async fn get_tts_inference_queue_count_handler( PendingCountResult, r#" SELECT - count(*) as pending_count -FROM - tts_inference_jobs -WHERE - status = "pending" + NOW() - t2.created_at AS seconds_since_first, + ( + SELECT + count(t1.id) as pending_count + FROM tts_inference_jobs AS t1 + WHERE t1.status = "pending" + ) as pending_count +FROM tts_inference_jobs AS t2 +WHERE t2.status = "pending" +ORDER BY t2.id ASC +LIMIT 1 "#, ) .fetch_one(&server_state.mysql_pool) @@ -101,7 +108,8 @@ WHERE let response = GetTtsInferenceQueueCountResponse { success: true, - pending_count: result.pending_count, + pending_count: result.pending_count.unwrap_or(0), + seconds_since_first: result.seconds_since_first, }; let body = serde_json::to_string(&response) @@ -114,5 +122,6 @@ WHERE #[derive(Serialize)] pub struct PendingCountResult { - pub pending_count: i64, + pub pending_count: Option, + pub seconds_since_first: i64, } diff --git a/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs b/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs index e622df5f47..0049787b61 100644 --- a/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs +++ b/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs @@ -23,6 +23,7 @@ use std::sync::Arc; pub struct GetW2lInferenceQueueCountResponse { pub success: bool, pub pending_count: i64, + pub seconds_since_first: i64, } #[derive(Debug, Serialize)] @@ -84,11 +85,17 @@ pub async fn get_w2l_inference_queue_count_handler( PendingCountResult, r#" SELECT - count(*) as pending_count -FROM - w2l_inference_jobs -WHERE - status = "pending" + NOW() - t2.created_at AS seconds_since_first, + ( + SELECT + count(t1.id) as pending_count + FROM w2l_inference_jobs AS t1 + WHERE t1.status = "pending" + ) as pending_count +FROM w2l_inference_jobs AS t2 +WHERE t2.status = "pending" +ORDER BY t2.id ASC +LIMIT 1 "#, ) .fetch_one(&server_state.mysql_pool) @@ -101,7 +108,8 @@ WHERE let response = GetW2lInferenceQueueCountResponse { success: true, - pending_count: result.pending_count, + pending_count: result.pending_count.unwrap_or(0), + seconds_since_first: result.seconds_since_first, }; let body = serde_json::to_string(&response) @@ -114,5 +122,6 @@ WHERE #[derive(Serialize)] pub struct PendingCountResult { - pub pending_count: i64, + pub pending_count: Option, + pub seconds_since_first: i64, } From a7d986d3f0e27d5c8471d6f0edd3dcbf2b1fc900 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 00:38:19 -0400 Subject: [PATCH 0609/1042] codegen --- sqlx-data.json | 126 +++++++++++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 50 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 58451f3522..baee85c6ce 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1838,31 +1838,6 @@ "nullable": [] } }, - "196d879939ea1aac35bf1a5413a9fe28ff7c1bf641d6c6752d7746df211f2e39": { - "query": "\nSELECT\n count(*) as pending_count\nFROM\n tts_inference_jobs\nWHERE\n status = \"pending\"\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "pending_count", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 129 - }, - "char_set": 63, - "max_size": 21 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false - ] - } - }, "1cc62e5cdc8de18969d7e8018d91c38794e5a7d23f117e5b0a4126845f300e60": { "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE\n token = ?\n AND deleted_at IS NULL\n ", "describe": { @@ -5405,6 +5380,44 @@ ] } }, + "a9173fd8148645bda75529aae0af63dcfb7cf9e114604ce4dcc962974e7268e1": { + "query": "\nSELECT\n NOW() - t2.created_at AS seconds_since_first,\n (\n SELECT\n count(t1.id) as pending_count\n FROM tts_inference_jobs AS t1\n WHERE t1.status = \"pending\"\n ) as pending_count\nFROM tts_inference_jobs AS t2\nWHERE t2.status = \"pending\"\nORDER BY t2.id ASC\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "seconds_since_first", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 16 + } + }, + { + "ordinal": 1, + "name": "pending_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + true + ] + } + }, "ab6f25eb0f6340d49aa705835ed428434d8b6ab14b0740f6a5ebe7841b9ece20": { "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -7712,31 +7725,6 @@ "nullable": [] } }, - "d9a30d5fcf83f723e696000893ef60774bf286d77b609c5c5f2b51a738fa283c": { - "query": "\nSELECT\n count(*) as pending_count\nFROM\n w2l_inference_jobs\nWHERE\n status = \"pending\"\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "pending_count", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 129 - }, - "char_set": 63, - "max_size": 21 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false - ] - } - }, "da3197d279b94817c6c8a6277aff7a0cf44d66e6777f46680b1b07de48875dd5": { "query": "\nINSERT INTO user_badges\nSET\n user_token = ?,\n badge_slug = ?\n ", "describe": { @@ -7963,6 +7951,44 @@ ] } }, + "dfe818ab849c5f6e2a5931142aab640d249aa96cb85dc06bfffae4d7feef541e": { + "query": "\nSELECT\n NOW() - t2.created_at AS seconds_since_first,\n (\n SELECT\n count(t1.id) as pending_count\n FROM w2l_inference_jobs AS t1\n WHERE t1.status = \"pending\"\n ) as pending_count\nFROM w2l_inference_jobs AS t2\nWHERE t2.status = \"pending\"\nORDER BY t2.id ASC\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "seconds_since_first", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 16 + } + }, + { + "ordinal": 1, + "name": "pending_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + true + ] + } + }, "e0ad2a7fe4ae795e4f097f604d5cfcf9605298b58cde5ca23bdf9c8ec7383b00": { "query": "\nUPDATE w2l_templates\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { From ab974cecb2e8404d5f82f7bb5acd3da5e9c9291c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 01:12:56 -0400 Subject: [PATCH 0610/1042] add new pending w2l templates endpoint --- src/http_server/endpoints/moderation/mod.rs | 1 + .../endpoints/moderation/pending/mod.rs | 1 + .../pending/pending_w2l_templates.rs | 182 ++++++++++++++++++ src/main.rs | 14 ++ 4 files changed, 198 insertions(+) create mode 100644 src/http_server/endpoints/moderation/pending/mod.rs create mode 100644 src/http_server/endpoints/moderation/pending/pending_w2l_templates.rs diff --git a/src/http_server/endpoints/moderation/mod.rs b/src/http_server/endpoints/moderation/mod.rs index bddfab2dcb..bb4c0313a2 100644 --- a/src/http_server/endpoints/moderation/mod.rs +++ b/src/http_server/endpoints/moderation/mod.rs @@ -1,5 +1,6 @@ pub mod ip_bans; pub mod jobs; +pub mod pending; pub mod user_bans; pub mod user_roles; pub mod users; \ No newline at end of file diff --git a/src/http_server/endpoints/moderation/pending/mod.rs b/src/http_server/endpoints/moderation/pending/mod.rs new file mode 100644 index 0000000000..9f34ef78bf --- /dev/null +++ b/src/http_server/endpoints/moderation/pending/mod.rs @@ -0,0 +1 @@ +pub mod pending_w2l_templates; \ No newline at end of file diff --git a/src/http_server/endpoints/moderation/pending/pending_w2l_templates.rs b/src/http_server/endpoints/moderation/pending/pending_w2l_templates.rs new file mode 100644 index 0000000000..e502cea6b9 --- /dev/null +++ b/src/http_server/endpoints/moderation/pending/pending_w2l_templates.rs @@ -0,0 +1,182 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; +use crate::server_state::ServerState; +use log::{info, warn, log}; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +#[derive(Serialize)] +pub struct GetPendingW2lTemplatesResponse { + pub success: bool, + pub templates: Vec, +} + +#[derive(Serialize)] +pub struct PendingW2lTemplate { + pub template_token: String, + pub title: String, + pub template_type: String, + pub duration_millis: i32, + pub frame_width: i32, + pub frame_height: i32, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub creator_gravatar_hash: String, + pub created_at: DateTime, +} + +#[derive(Debug, Serialize)] +pub enum GetPendingW2lTemplatesError { + ServerError, + Unauthorized, +} + +impl ResponseError for GetPendingW2lTemplatesError { + fn status_code(&self) -> StatusCode { + match *self { + GetPendingW2lTemplatesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + GetPendingW2lTemplatesError::Unauthorized => StatusCode::UNAUTHORIZED, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for GetPendingW2lTemplatesError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn get_pending_w2l_templates_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetPendingW2lTemplatesError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(GetPendingW2lTemplatesError::Unauthorized); + } + }; + + if !user_session.can_approve_w2l_templates { + warn!("user is not allowed to approve w2l templates: {}", user_session.user_token); + return Err(GetPendingW2lTemplatesError::Unauthorized); + } + + // NB: Lookup failure is Err(RowNotFound). + let maybe_results = sqlx::query_as!( + PendingW2lTemplatesRaw, + r#" +SELECT + w2l_templates.token as template_token, + w2l_templates.title, + w2l_templates.template_type, + w2l_templates.duration_millis, + w2l_templates.frame_width, + w2l_templates.frame_height, + w2l_templates.creator_user_token, + users.username AS creator_username, + users.display_name AS creator_display_name, + users.email_gravatar_hash AS creator_gravatar_hash, + w2l_templates.created_at +FROM + w2l_templates +JOIN + users +ON + users.token = w2l_templates.creator_user_token +WHERE + w2l_templates.is_public_listing_approved IS NULL + AND w2l_templates.user_deleted_at IS NULL + AND w2l_templates.mod_deleted_at IS NULL + AND w2l_templates.is_locked_from_use IS FALSE + "#, + ) + .fetch_all(&server_state.mysql_pool) + .await; + + let results : Vec = match maybe_results { + Ok(results) => results, + Err(err) => { + match err { + RowNotFound => { + Vec::new() + }, + _ => { + warn!("list pending w2l db error: {:?}", err); + return Err(GetPendingW2lTemplatesError::ServerError) + } + } + } + }; + + let results = results.iter().map(|r| { + PendingW2lTemplate { + template_token: r.template_token.clone(), + title: r.title.clone(), + template_type: r.template_type.clone(), + duration_millis: r.duration_millis, + frame_width: r.frame_width, + frame_height: r.frame_height, + creator_user_token: r.creator_user_token.clone(), + creator_username: r.creator_username.clone(), + creator_display_name: r.creator_display_name.clone(), + creator_gravatar_hash: r.creator_gravatar_hash.clone(), + created_at: r.created_at.clone(), + } + }).collect::>(); + + let response = GetPendingW2lTemplatesResponse { + success: true, + templates: results, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetPendingW2lTemplatesError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} + +#[derive(Serialize)] +pub struct PendingW2lTemplatesRaw { + pub template_token: String, + pub title: String, + pub template_type: String, + pub duration_millis: i32, + pub frame_width: i32, + pub frame_height: i32, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub creator_gravatar_hash: String, + pub created_at: DateTime, +} diff --git a/src/main.rs b/src/main.rs index f0fd377d17..092c7fcaaf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,6 +45,7 @@ use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_h use crate::http_server::endpoints::moderation::ip_bans::list_ip_bans::list_ip_bans_handler; use crate::http_server::endpoints::moderation::jobs::get_tts_inference_queue_count::get_tts_inference_queue_count_handler; use crate::http_server::endpoints::moderation::jobs::get_w2l_inference_queue_count::get_w2l_inference_queue_count_handler; +use crate::http_server::endpoints::moderation::pending::pending_w2l_templates::get_pending_w2l_templates_handler; use crate::http_server::endpoints::moderation::user_bans::ban_user::ban_user_handler; use crate::http_server::endpoints::moderation::user_bans::list_banned_users::list_banned_users_handler; use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user_roles_handler; @@ -448,6 +449,19 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service( + web::scope("/pending") + .service( + web::resource("/w2l_templates") + .route(web::get().to(get_pending_w2l_templates_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/w2l_inference_queue_stats") + .route(web::get().to(get_w2l_inference_queue_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) ) // ==================== TTS ==================== .service( From 5ac58bd84931864b1a1a9a0941dec9b83e4b2ee1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 01:13:34 -0400 Subject: [PATCH 0611/1042] codegen db --- sqlx-data.json | 155 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/sqlx-data.json b/sqlx-data.json index baee85c6ce..7c57b1c1b0 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -3753,6 +3753,161 @@ "nullable": [] } }, + "619d18c87075a86c01f1e0ac26744774293ff8ca6b0011ce611b3f47d8a97a0e": { + "query": "\nSELECT\n w2l_templates.token as template_token,\n w2l_templates.title,\n w2l_templates.template_type,\n w2l_templates.duration_millis,\n w2l_templates.frame_width,\n w2l_templates.frame_height,\n w2l_templates.creator_user_token,\n users.username AS creator_username,\n users.display_name AS creator_display_name,\n users.email_gravatar_hash AS creator_gravatar_hash,\n w2l_templates.created_at\nFROM\n w2l_templates\nJOIN\n users\nON\n users.token = w2l_templates.creator_user_token\nWHERE\n w2l_templates.is_public_listing_approved IS NULL\n AND w2l_templates.user_deleted_at IS NULL\n AND w2l_templates.mod_deleted_at IS NULL\n AND w2l_templates.is_locked_from_use IS FALSE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 2, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 4, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 5, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 6, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 9, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "63ac1d3f54c8258b7d20d3f689b2613809099feb4df925c680594a1e43e74bcd": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", "describe": { From aabf5579e166ada9cf6d2445782a23e14c244725 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 02:11:49 -0400 Subject: [PATCH 0612/1042] fix pending queue bug when queue empty --- .../jobs/get_tts_inference_queue_count.rs | 16 +++++++++++++++- .../jobs/get_w2l_inference_queue_count.rs | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs b/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs index 991e13089e..8f6641dd0f 100644 --- a/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs +++ b/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs @@ -103,7 +103,21 @@ LIMIT 1 let result : PendingCountResult = match maybe_result { Ok(result) => result, - Err(err) => return Err(GetTtsInferenceQueueCountError::ServerError), + Err(err) => { + match err { + RowNotFound => { + // NB: Not Found for null results means nothing is pending in the queue + PendingCountResult { + pending_count: None, + seconds_since_first: 0, + } + }, + _ => { + warn!("get tts pending count error: {:?}", err); + return Err(GetTtsInferenceQueueCountError::ServerError) + } + } + }, }; let response = GetTtsInferenceQueueCountResponse { diff --git a/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs b/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs index 0049787b61..433e431c69 100644 --- a/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs +++ b/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs @@ -103,7 +103,21 @@ LIMIT 1 let result : PendingCountResult = match maybe_result { Ok(result) => result, - Err(err) => return Err(GetW2lInferenceQueueCountError::ServerError), + Err(err) => { + match err { + RowNotFound => { + // NB: Not Found for null results means nothing is pending in the queue + PendingCountResult { + pending_count: None, + seconds_since_first: 0, + } + }, + _ => { + warn!("get w2l pending count error: {:?}", err); + return Err(GetW2lInferenceQueueCountError::ServerError) + } + } + }, }; let response = GetW2lInferenceQueueCountResponse { From 5c8731eba9a548d5b5a299e7f2ef815197b34309 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 12:24:20 -0400 Subject: [PATCH 0613/1042] silent k slur also works. ugh :( --- db/banned_slurs.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/db/banned_slurs.txt b/db/banned_slurs.txt index 1bb81205b5..fd6c54f0db 100644 --- a/db/banned_slurs.txt +++ b/db/banned_slurs.txt @@ -4,5 +4,6 @@ fag faggot fags +kniggers nigger niggers \ No newline at end of file From 0d691ded80cbf2f4f27bbbc6e24528ce70f0cd43 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 16:45:22 -0400 Subject: [PATCH 0614/1042] fix tts download job from breaking --- .env | 10 +++--- .../tacotron_model_check_command.rs | 25 +++++++++------ src/tts_download_job.rs | 32 ++++++++++++------- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/.env b/.env index fc4bea8051..c2366ca4d6 100644 --- a/.env +++ b/.env @@ -42,10 +42,12 @@ W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py TTS_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/tts TTS_INFERENCE_SCRIPT_NAME=vocodes_inference.py -# TODO: Remove generic "check script" -TTS_MODEL_CHECK_SCRIPT_NAME=vocodes_model_check.py -TTS_TACOTRON_CHECK_SCRIPT_NAME=vocodes_model_check_tacotron.py -TTS_TALKNET_CHECK_SCRIPT_NAME=vocodes_model_check_talknet.py +TACOTRON_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/tts +TACOTRON_VIRTUAL_ENV_ACTIVATION_COMMAND="source python-tacotron/bin/activate" +TACOTRON_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_tacotron.py" + +TALKNET_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/tts +TALKNET_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_talknet.py" TTS_INFERENCE_SIDECAR_HOSTNAME=localhost:8000 SIDECAR_MAX_SYNTHESIZER_MODELS=2 diff --git a/src/script_execution/tacotron_model_check_command.rs b/src/script_execution/tacotron_model_check_command.rs index a66fbb9be3..e637d20290 100644 --- a/src/script_execution/tacotron_model_check_command.rs +++ b/src/script_execution/tacotron_model_check_command.rs @@ -8,18 +8,25 @@ use std::path::Path; /// This command is used to check tacotron for being a real model #[derive(Clone)] pub struct TacotronModelCheckCommand { - tacotron_directory: String, - check_model_script_name: String, + /// Where the Tacotron code lives + tacotron_root_code_directory: String, + + /// eg. `source python/bin/activate` + virtual_env_activation_command: String, + + tacotron_model_check_script_name: String, } impl TacotronModelCheckCommand { pub fn new( - tacotron_directory: &str, - check_model_script_name: &str, + tacotron_root_code_directory: &str, + virtual_env_activation_command: &str, + tacotron_model_check_script_name: &str, ) -> Self { Self { - tacotron_directory: tacotron_directory.to_string(), - check_model_script_name: check_model_script_name.to_string(), + tacotron_root_code_directory: tacotron_root_code_directory.to_string(), + virtual_env_activation_command: virtual_env_activation_command.to_string(), + tacotron_model_check_script_name: tacotron_model_check_script_name.to_string(), } } @@ -33,12 +40,12 @@ impl TacotronModelCheckCommand { command.push_str("echo 'test'"); command.push_str(" && "); - command.push_str(&format!("cd {}", self.tacotron_directory)); + command.push_str(&format!("cd {}", self.tacotron_root_code_directory)); command.push_str(" && "); - command.push_str("source python/bin/activate"); + command.push_str(&self.virtual_env_activation_command); command.push_str(" && "); command.push_str("python "); - command.push_str(&self.check_model_script_name); + command.push_str(&self.tacotron_model_check_script_name); command.push_str(" --synthesizer_checkpoint_path "); command.push_str(&synthesizer_checkpoint_path.as_ref().display().to_string()); command.push_str(" --output_metadata_filename "); diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index fa695ae405..cd695ff799 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -67,11 +67,6 @@ const ENV_BUCKET_ROOT : &'static str = "TTS_DOWNLOAD_BUCKET_ROOT"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; -// Python code -const ENV_CODE_DIRECTORY : &'static str = "TTS_CODE_DIRECTORY"; -const ENV_TACOTRON_CHECK_SCRIPT_NAME : &'static str = "TTS_TACOTRON_CHECK_SCRIPT_NAME"; -const ENV_TALKNET_CHECK_SCRIPT_NAME : &'static str = "TTS_TALKNET_CHECK_SCRIPT_NAME"; - struct Downloader { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, @@ -179,17 +174,26 @@ async fn main() -> AnyhowResult<()> { firehose_publisher: firehose_publisher.clone(), // NB: Also safe }; - let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?; - let tacotron_check_script_name = easyenv::get_env_string_required(ENV_TACOTRON_CHECK_SCRIPT_NAME)?; - let talknet_check_script_name = easyenv::get_env_string_required(ENV_TALKNET_CHECK_SCRIPT_NAME)?; + let tacotron_root_code_directory = easyenv::get_env_string_required("TACOTRON_ROOT_CODE_DIRECTORY")?; + let tacotron_virtual_env_activation_command = easyenv::get_env_string_or_default( + "TACOTRON_VIRTUAL_ENV_ACTIVATION_COMMAND", + "source python-tacotron/bin/activate"); + + let tacotron_model_check_script_name = easyenv::get_env_string_or_default( + "TACOTRON_MODEL_CHECK_SCRIPT_NAME", + "vocodes_model_check_tacotron.py"); let tacotron_check_command= TacotronModelCheckCommand::new( - &py_code_directory, - &tacotron_check_script_name, + &tacotron_root_code_directory, + &tacotron_virtual_env_activation_command, + &tacotron_model_check_script_name, ); + let talknet_root_code_directory = easyenv::get_env_string_required("TALKNET_ROOT_CODE_DIRECTORY")?; + let talknet_check_script_name = easyenv::get_env_string_required("TALKNET_MODEL_CHECK_SCRIPT_NAME")?; + let talknet_check_command= TalknetModelCheckCommand::new( - &py_code_directory, + &talknet_root_code_directory, &talknet_check_script_name, ); @@ -343,6 +347,12 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("Uploaded model type: {:?}", model_type); + if model_type == TtsModelType::Talknet { + // TODO: Finish supporting TalkNet + warn!("Unsupported model type"); + return Err(anyhow!("unsupported model type")); + } + // ==================== RUN MODEL CHECK ==================== // info!("Checking that model is valid..."); From 0450adb396be33142feb5bdfe2219e77d25f5194 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 17:07:41 -0400 Subject: [PATCH 0615/1042] attempt to delete files after they're uploaded --- src/tts_download_job.rs | 74 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index cd695ff799..7a97041849 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -51,12 +51,13 @@ use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::File; -use std::io::{BufReader, Read}; +use std::io::{BufReader, Read, Error}; use std::ops::Deref; use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; use tempdir::TempDir; +use std::fs; // Buckets const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -336,8 +337,13 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho .map(|c| c.to_string()) .unwrap_or("".to_string()); - let download_filename = downloader.google_drive_downloader - .download_file(&download_url, &temp_dir).await?; + let download_filename = match downloader.google_drive_downloader.download_file(&download_url, &temp_dir).await { + Ok(filename) => filename, + Err(e) => { + delete_temp_directory(&temp_dir); + return Err(e); + } + }; let model_type = if download_filename.to_lowercase().ends_with("zip") { TtsModelType::Talknet @@ -365,18 +371,28 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho match model_type { TtsModelType::Tacotron2 => { - downloader.tacotron_tts_check.execute( + let result = downloader.tacotron_tts_check.execute( &file_path, &output_metadata_fs_path, false, - )?; + ); + + if let Err(e) = result { + delete_temp_file(&file_path); + delete_temp_directory(&temp_dir); + } }, TtsModelType::Talknet => { - downloader.talknet_tts_check.execute( + let result = downloader.talknet_tts_check.execute( &file_path, &output_metadata_fs_path, false, - )?; + ); + + if let Err(e) = result { + delete_temp_file(&file_path); + delete_temp_directory(&temp_dir); + } }, } @@ -386,7 +402,15 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho check_file_exists(&output_metadata_fs_path)?; - let file_metadata = read_metadata_file(&output_metadata_fs_path)?; + let file_metadata = match read_metadata_file(&output_metadata_fs_path) { + Ok(metadata) => metadata, + Err(e) => { + delete_temp_file(&file_path); + delete_temp_file(&output_metadata_fs_path); + delete_temp_directory(&temp_dir); + return Err(e); + } + }; // ==================== UPLOAD MODEL FILE ==================== // @@ -405,7 +429,19 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho redis_logger.log_status("uploading model")?; - downloader.bucket_client.upload_filename(&synthesizer_model_bucket_path, &file_path).await?; + if let Err(e) = downloader.bucket_client.upload_filename(&synthesizer_model_bucket_path, &file_path).await { + delete_temp_file(&output_metadata_fs_path); + delete_temp_file(&file_path); + delete_temp_directory(&temp_dir); + return Err(e); + } + + // ==================== DELETE DOWNLOADED FILE ==================== // + + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + delete_temp_file(&output_metadata_fs_path); + delete_temp_file(&file_path); + delete_temp_directory(&temp_dir); // ==================== SAVE RECORDS ==================== // @@ -448,3 +484,23 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho Ok(()) } + +fn delete_temp_file>(file_path: P) { + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + let printable_name = file_path.as_ref().to_str().unwrap_or("bad filename"); + match fs::remove_file(&file_path) { + Ok(_) => info!("File deleted: {}", printable_name), + Err(e) => warn!("Could not delete file {:?} (not a fatal error): {:?}", + printable_name, e), + } +} + +fn delete_temp_directory>(directory_path: P) { + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + let printable_name = directory_path.as_ref().to_str().unwrap_or("bad directory"); + match fs::remove_dir_all(&directory_path) { + Ok(_) => info!("Directory deleted: {}", printable_name), + Err(e) => warn!("Could not delete directory{:?} (not a fatal error): {:?}", + printable_name, e), + } +} From aca73c32e300a6c27a8dd6a47c1e5608eaff6b10 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 17:21:47 -0400 Subject: [PATCH 0616/1042] make shared functions --- src/tts_download_job.rs | 52 ++++++++++++++--------------------------- src/util/filesystem.rs | 24 +++++++++++++++++++ 2 files changed, 41 insertions(+), 35 deletions(-) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 7a97041849..e08dc1cc2f 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -38,6 +38,8 @@ use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use crate::util::buckets::bucket_paths::hash_to_bucket_path; +use crate::util::filesystem::safe_delete_temp_directory; +use crate::util::filesystem::safe_delete_temp_file; use crate::util::filesystem::{check_directory_exists, check_file_exists}; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; @@ -51,13 +53,13 @@ use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::File; +use std::fs; use std::io::{BufReader, Read, Error}; use std::ops::Deref; use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; use tempdir::TempDir; -use std::fs; // Buckets const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -340,7 +342,7 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho let download_filename = match downloader.google_drive_downloader.download_file(&download_url, &temp_dir).await { Ok(filename) => filename, Err(e) => { - delete_temp_directory(&temp_dir); + safe_delete_temp_directory(&temp_dir); return Err(e); } }; @@ -378,8 +380,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho ); if let Err(e) = result { - delete_temp_file(&file_path); - delete_temp_directory(&temp_dir); + safe_delete_temp_file(&file_path); + safe_delete_temp_directory(&temp_dir); } }, TtsModelType::Talknet => { @@ -390,8 +392,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho ); if let Err(e) = result { - delete_temp_file(&file_path); - delete_temp_directory(&temp_dir); + safe_delete_temp_file(&file_path); + safe_delete_temp_directory(&temp_dir); } }, } @@ -405,9 +407,9 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho let file_metadata = match read_metadata_file(&output_metadata_fs_path) { Ok(metadata) => metadata, Err(e) => { - delete_temp_file(&file_path); - delete_temp_file(&output_metadata_fs_path); - delete_temp_directory(&temp_dir); + safe_delete_temp_file(&file_path); + safe_delete_temp_file(&output_metadata_fs_path); + safe_delete_temp_directory(&temp_dir); return Err(e); } }; @@ -430,18 +432,18 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho redis_logger.log_status("uploading model")?; if let Err(e) = downloader.bucket_client.upload_filename(&synthesizer_model_bucket_path, &file_path).await { - delete_temp_file(&output_metadata_fs_path); - delete_temp_file(&file_path); - delete_temp_directory(&temp_dir); + safe_delete_temp_file(&output_metadata_fs_path); + safe_delete_temp_file(&file_path); + safe_delete_temp_directory(&temp_dir); return Err(e); } // ==================== DELETE DOWNLOADED FILE ==================== // // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... - delete_temp_file(&output_metadata_fs_path); - delete_temp_file(&file_path); - delete_temp_directory(&temp_dir); + safe_delete_temp_file(&output_metadata_fs_path); + safe_delete_temp_file(&file_path); + safe_delete_temp_directory(&temp_dir); // ==================== SAVE RECORDS ==================== // @@ -484,23 +486,3 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho Ok(()) } - -fn delete_temp_file>(file_path: P) { - // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... - let printable_name = file_path.as_ref().to_str().unwrap_or("bad filename"); - match fs::remove_file(&file_path) { - Ok(_) => info!("File deleted: {}", printable_name), - Err(e) => warn!("Could not delete file {:?} (not a fatal error): {:?}", - printable_name, e), - } -} - -fn delete_temp_directory>(directory_path: P) { - // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... - let printable_name = directory_path.as_ref().to_str().unwrap_or("bad directory"); - match fs::remove_dir_all(&directory_path) { - Ok(_) => info!("Directory deleted: {}", printable_name), - Err(e) => warn!("Could not delete directory{:?} (not a fatal error): {:?}", - printable_name, e), - } -} diff --git a/src/util/filesystem.rs b/src/util/filesystem.rs index 1ad312633e..d3c84000ac 100644 --- a/src/util/filesystem.rs +++ b/src/util/filesystem.rs @@ -1,5 +1,9 @@ use anyhow::bail; use crate::util::anyhow_result::AnyhowResult; +use log::info; +use log::warn; +use std::fs; +use std::path::Path; use std::path::PathBuf; pub fn check_directory_exists(path: &PathBuf) -> AnyhowResult<()> { @@ -21,3 +25,23 @@ pub fn check_file_exists(path: &PathBuf) -> AnyhowResult<()> { } Ok(()) } + +pub fn safe_delete_temp_file>(file_path: P) { + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + let printable_name = file_path.as_ref().to_str().unwrap_or("bad filename"); + match fs::remove_file(&file_path) { + Ok(_) => info!("Temp file deleted: {}", printable_name), + Err(e) => warn!("Could not delete temp file {:?} (not a fatal error): {:?}", + printable_name, e), + } +} + +pub fn safe_delete_temp_directory>(directory_path: P) { + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + let printable_name = directory_path.as_ref().to_str().unwrap_or("bad directory"); + match fs::remove_dir_all(&directory_path) { + Ok(_) => info!("Temp directory deleted: {}", printable_name), + Err(e) => warn!("Could not delete temp directory{:?} (not a fatal error): {:?}", + printable_name, e), + } +} From c934e6eeb40d5419e12ec5d105ab8c433e0b643d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 17:32:39 -0400 Subject: [PATCH 0617/1042] delete intermediate results in w2l download job --- src/w2l_download_job.rs | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index d56c94a258..e0b49db743 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -17,7 +17,7 @@ pub mod script_execution; pub mod shared_constants; pub mod util; -use anyhow::anyhow; +use anyhow::{anyhow, Error}; use chrono::Utc; use crate::common_env::CommonEnv; use crate::database::mediators::badge_granter::BadgeGranter; @@ -41,6 +41,8 @@ use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; +use crate::util::filesystem::safe_delete_temp_directory; +use crate::util::filesystem::safe_delete_temp_file; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; @@ -379,8 +381,13 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) .unwrap_or("".to_string()); info!("Calling downloader..."); - let download_filename = downloader.google_drive_downloader - .download_file(&download_url, &temp_dir).await?; + let download_filename = match downloader.google_drive_downloader.download_file(&download_url, &temp_dir).await { + Ok(filename) => filename, + Err(e) => { + safe_delete_temp_directory(&temp_dir); + return Err(e); + } + }; if downloader.debug_download_sleep_millis != 0 { warn!("Debug sleep after download: {} ms", downloader.debug_download_sleep_millis); @@ -414,6 +421,9 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) Err(Wav2LipPreprocessError::NoFacesDetected) => { // Permanently fail. warn!("Permanently failed due to no face detection!"); + + safe_delete_temp_directory(&temp_dir); + mark_w2l_template_upload_job_permanently_dead( &downloader.mysql_pool, job, @@ -466,14 +476,20 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) if file_metadata.is_video { let preview_filename = format!("{}_preview.webp", &download_filename); - downloader.ffmpeg_video_preview_generator.execute( + let preview_result = downloader.ffmpeg_video_preview_generator.execute( &download_filename, &preview_filename, 500, 500, true, false - )?; + ); + + if let Err(e) = preview_result { + safe_delete_temp_file(&download_filename); + safe_delete_temp_file(&preview_filename); + safe_delete_temp_directory(&temp_dir); + } let video_preview_path = PathBuf::from(&preview_filename); check_file_exists(&video_preview_path)?; @@ -528,12 +544,16 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) &video_or_image_path, original_mime_type).await?; + safe_delete_temp_file(&video_or_image_path); + info!("Uploading cached faces..."); let path_copy: PathBuf = cached_faces_path.clone(); downloader.private_bucket_client.upload_filename( &full_object_path_cached_faces, &path_copy).await?; + safe_delete_temp_file(&path_copy); + // TODO: Fix this ugh. if let Some(image_preview_filename) = maybe_image_preview_filename.as_deref() { if let Some(image_preview_object_name) = maybe_image_preview_object_name.as_deref() { @@ -549,6 +569,8 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) image_preview_filename, "image/webp").await?; } + + safe_delete_temp_file(&image_preview_filename); } // TODO: Fix this ugh. @@ -566,8 +588,15 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) video_preview_filename, "image/webp").await?; } + + safe_delete_temp_file(&video_preview_filename); } + // ==================== DELETE DOWNLOADED FILE ==================== // + + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + safe_delete_temp_directory(&temp_dir); + // ==================== SAVE RECORDS ==================== // let template_type = if file_metadata.is_video { "video" } else { "image" }; From cf57772a396caeccea32bd12f09200a429a3e834 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 17:37:17 -0400 Subject: [PATCH 0618/1042] delete intermediate results in tts inference --- src/tts_inference_job.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 4f6949868a..f379a018bb 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -40,6 +40,8 @@ use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; +use crate::util::filesystem::safe_delete_temp_directory; +use crate::util::filesystem::safe_delete_temp_file; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::hashing::hash_string_sha2::hash_string_sha2; use crate::util::jobs::cache_miss_strategizer::CacheMissStrategizer; @@ -724,6 +726,8 @@ async fn process_job( let file_metadata = read_metadata_file(&output_metadata_fs_path)?; + safe_delete_temp_file(&output_metadata_fs_path); + // ==================== UPLOAD AUDIO TO BUCKET ==================== // redis_logger.log_status("uploading result")?; @@ -741,6 +745,8 @@ async fn process_job( "audio/wav") .await?; + safe_delete_temp_file(&output_audio_fs_path); + // ==================== UPLOAD SPECTROGRAM TO BUCKETS ==================== // let spectrogram_result_object_path = inferencer.bucket_path_unifier.tts_inference_spectrogram_output_path( @@ -756,6 +762,13 @@ async fn process_job( "application/json") .await?; + safe_delete_temp_file(&output_spectrogram_fs_path); + + // ==================== DELETE DOWNLOADED FILE ==================== // + + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + safe_delete_temp_directory(&temp_dir); + // ==================== SAVE RECORDS ==================== // let text_hash = hash_string_sha2(&job.raw_inference_text)?; From f26a3318c45a20da72e41b98f82d9d251bdd394c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 17:39:56 -0400 Subject: [PATCH 0619/1042] put in deletion of intermediate files (but not on error) --- src/w2l_inference_job.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index f9354900cd..98bfa2d327 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -41,6 +41,8 @@ use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; +use crate::util::filesystem::safe_delete_temp_directory; +use crate::util::filesystem::safe_delete_temp_file; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; @@ -49,6 +51,7 @@ use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2::PooledConnection; use r2d2_redis::r2d2; use r2d2_redis::redis::Commands; use ring::digest::{Context, Digest, SHA256}; @@ -62,7 +65,6 @@ use std::process::Command; use std::thread; use std::time::Duration; use tempdir::TempDir; -use r2d2_redis::r2d2::PooledConnection; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -572,6 +574,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An let file_metadata = read_metadata_file(&output_metadata_fs_path)?; + safe_delete_temp_file(&output_metadata_fs_path); + // ==================== UPLOAD TO BUCKETS ==================== // redis_logger.log_status("uploading result")?; @@ -593,6 +597,13 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An original_mime_type) .await?; + safe_delete_temp_file(&output_video_fs_path); + + // ==================== DELETE DOWNLOADED FILE ==================== // + + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + safe_delete_temp_directory(&temp_dir); + // ==================== SAVE RECORDS ==================== // info!("Saving w2l inference record..."); From bb10e6fc899afc80e1e0b8cc1cabc168ad183cf0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 17:42:47 -0400 Subject: [PATCH 0620/1042] add temp file deletion for a few w2l inference error cases too --- src/w2l_inference_job.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 98bfa2d327..a7d29b6aa6 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -551,7 +551,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An info!("Is image? {}", is_image); info!("Running W2L inference..."); - inferencer.w2l_inference.execute( + let inference_result = inferencer.w2l_inference.execute( &model_fs_path, &audio_fs_path, &end_bump_fs_path, @@ -561,7 +561,14 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An &output_video_fs_path, false, false - )?; + ); + + if let Err(e) = inference_result { + safe_delete_temp_file(&audio_fs_path); + safe_delete_temp_file(&output_video_fs_path); + safe_delete_temp_directory(&temp_dir); + return Err(e); + } info!("Output filename: {:?}", &output_video_fs_path); @@ -591,11 +598,18 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An .as_deref() .unwrap_or("application/octet-stream"); - inferencer.public_bucket_client.upload_filename_with_content_type( + let upload_result = inferencer.public_bucket_client.upload_filename_with_content_type( &result_object_path, &output_video_fs_path, original_mime_type) - .await?; + .await; + + if let Err(e) = upload_result { + safe_delete_temp_file(&audio_fs_path); + safe_delete_temp_file(&output_video_fs_path); + safe_delete_temp_directory(&temp_dir); + return Err(e); + } safe_delete_temp_file(&output_video_fs_path); From 7bcb9090ae66a92370e2dec770d97b0b4f516c7c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 13 Sep 2021 05:44:47 -0400 Subject: [PATCH 0621/1042] Add NewRelic tracing to tts-inference-job using vendored lib --- .env | 3 + Cargo.lock | 15 + Cargo.toml | 2 + src/tts_inference_job.rs | 87 +- .../.github/ISSUE_TEMPLATE/bug_report.md | 38 + .../.github/ISSUE_TEMPLATE/enhancement.md | 27 + .../.github/dependabot.yml | 11 + .../.github/workflows/ci.yml | 38 + vendor/newrelic-telemetry-sdk-rust/.gitignore | 2 + .../CODE_OF_CONDUCT.md | 5 + .../CONTRIBUTING.md | 30 + vendor/newrelic-telemetry-sdk-rust/Cargo.toml | 29 + vendor/newrelic-telemetry-sdk-rust/LICENSE | 201 ++++ vendor/newrelic-telemetry-sdk-rust/README.md | 108 ++ vendor/newrelic-telemetry-sdk-rust/cla.md | 16 + .../src/attribute.rs | 252 +++++ .../newrelic-telemetry-sdk-rust/src/client.rs | 971 ++++++++++++++++++ vendor/newrelic-telemetry-sdk-rust/src/lib.rs | 18 + .../newrelic-telemetry-sdk-rust/src/span.rs | 569 ++++++++++ .../tests/blocking.rs | 252 +++++ .../tests/common/mod.rs | 250 +++++ .../tests/spans.rs | 219 ++++ 22 files changed, 3134 insertions(+), 9 deletions(-) create mode 100644 vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/bug_report.md create mode 100644 vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/enhancement.md create mode 100644 vendor/newrelic-telemetry-sdk-rust/.github/dependabot.yml create mode 100644 vendor/newrelic-telemetry-sdk-rust/.github/workflows/ci.yml create mode 100644 vendor/newrelic-telemetry-sdk-rust/.gitignore create mode 100644 vendor/newrelic-telemetry-sdk-rust/CODE_OF_CONDUCT.md create mode 100644 vendor/newrelic-telemetry-sdk-rust/CONTRIBUTING.md create mode 100644 vendor/newrelic-telemetry-sdk-rust/Cargo.toml create mode 100644 vendor/newrelic-telemetry-sdk-rust/LICENSE create mode 100644 vendor/newrelic-telemetry-sdk-rust/README.md create mode 100644 vendor/newrelic-telemetry-sdk-rust/cla.md create mode 100644 vendor/newrelic-telemetry-sdk-rust/src/attribute.rs create mode 100644 vendor/newrelic-telemetry-sdk-rust/src/client.rs create mode 100644 vendor/newrelic-telemetry-sdk-rust/src/lib.rs create mode 100644 vendor/newrelic-telemetry-sdk-rust/src/span.rs create mode 100644 vendor/newrelic-telemetry-sdk-rust/tests/blocking.rs create mode 100644 vendor/newrelic-telemetry-sdk-rust/tests/common/mod.rs create mode 100644 vendor/newrelic-telemetry-sdk-rust/tests/spans.rs diff --git a/.env b/.env index c2366ca4d6..e6cbbe3434 100644 --- a/.env +++ b/.env @@ -58,3 +58,6 @@ TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4.pt TTS_HIFIGAN_VOCODER_MODEL_FILENAME=hifigan_universal_g_02500000.pt TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt + +# Development key. Permissible to check in. +NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL diff --git a/Cargo.lock b/Cargo.lock index 3c166d2b24..db14c98bfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2149,6 +2149,20 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "newrelic-telemetry" +version = "0.1.0" +dependencies = [ + "anyhow", + "flate2", + "hyper", + "hyper-tls", + "log", + "serde", + "serde_json", + "uuid", +] + [[package]] name = "nom" version = "4.2.3" @@ -3357,6 +3371,7 @@ dependencies = [ "log", "magic-crypt", "md-5", + "newrelic-telemetry", "pulldown-cmark", "r2d2_redis", "rand 0.8.3", diff --git a/Cargo.toml b/Cargo.toml index e31f0c7e9a..ff8e2a0d14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,8 @@ hostname = "0.3.1" lazy_static = "1.4.0" lfu = "0.2.5" log = "0.4.14" +#newrelic = "0.2.2" +newrelic-telemetry = { path = "vendor/newrelic-telemetry-sdk-rust" } rand = "0.8.3" regex = "1.5.4" time = "0.2.26" diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index f379a018bb..c79a4f28d1 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -54,6 +54,9 @@ use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; +use newrelic_telemetry::Client as NewRelicClient; +use newrelic_telemetry::ClientBuilder; +use newrelic_telemetry::Span; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2::PooledConnection; use r2d2_redis::r2d2; @@ -68,8 +71,9 @@ use std::ops::Deref; use std::path::{PathBuf, Path}; use std::process::Command; use std::thread; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH, Instant}; use tempdir::TempDir; +use crate::util::random_uuid::generate_random_uuid; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -109,6 +113,8 @@ struct Inferencer { pub tts_inference_command: TacotronInferenceCommand, pub tts_inference_sidecar_client: TtsInferenceSidecarClient, + pub newrelic_client: NewRelicClient, + // Keep tabs of which models to hold in the sidecar memory with this virtual LRU cache pub virtual_model_lfu: SyncVirtualLfuCache, pub cache_miss_strategizers: SyncMultiCacheMissStrategizer, @@ -279,6 +285,9 @@ async fn main() -> AnyhowResult<()> { ) }; + let license_key = easyenv::get_env_string_required("NEWRELIC_API_KEY")?; + let newrelic_client = ClientBuilder::new(&license_key).build().unwrap(); + let inferencer = Inferencer { download_temp_directory: temp_directory, mysql_pool, @@ -287,6 +296,7 @@ async fn main() -> AnyhowResult<()> { private_bucket_client, tts_inference_command, tts_inference_sidecar_client, + newrelic_client, virtual_model_lfu: virtual_lfu_cache, cache_miss_strategizers, bucket_path_unifier: BucketPathUnifier::default_paths(), @@ -315,6 +325,8 @@ async fn main_loop(inferencer: Inferencer) { let mut noop_logger = NoOpLogger::new(inferencer.no_op_logger_millis as i64); + let mut span_batch = Vec::new(); + loop { let num_records = inferencer.job_batch_size; @@ -345,14 +357,22 @@ async fn main_loop(inferencer: Inferencer) { jobs, ).await; - match batch_result { - Ok(_) => {}, + let mut spans = match batch_result { + Ok(spans) => spans, Err(e) => { warn!("Error querying jobs: {:?}", e); std::thread::sleep(Duration::from_millis(error_timeout_millis)); error_timeout_millis += INCREASE_TIMEOUT_MILLIS; continue; } + }; + + span_batch.append(&mut spans); + + if span_batch.len() > 1 { + //let spans_to_send = span_batch.drain(..).collect(); + let spans_to_send = span_batch.split_off(0).into(); + inferencer.newrelic_client.send_spans(spans_to_send).await; } error_timeout_millis = START_TIMEOUT_MILLIS; // reset @@ -365,7 +385,9 @@ async fn main_loop(inferencer: Inferencer) { async fn process_jobs( inferencer: &Inferencer, jobs: Vec, -) -> AnyhowResult<()> { +) -> AnyhowResult> { + + let mut batch_spans = Vec::new(); for job in jobs.into_iter() { let model_state_result = ModelState::query_model_and_check_filesystem( @@ -428,7 +450,10 @@ async fn process_jobs( let result = process_job(inferencer, &job, &model_state.model_record).await; match result { - Ok(_) => {}, + Ok((span1, span2)) => { + batch_spans.push(span1); + batch_spans.push(span2); + }, Err(e) => { warn!("Failure to process job: {:?}", e); let failure_reason = ""; @@ -442,7 +467,7 @@ async fn process_jobs( } } - Ok(()) + Ok(batch_spans) } #[derive(Deserialize, Default)] @@ -513,7 +538,7 @@ async fn process_job( inferencer: &Inferencer, job: &TtsInferenceJobRecord, model_record: &TtsModelRecord2, -) -> AnyhowResult<()> { +) -> AnyhowResult<(Span, Span)> { // TODO 1. Mark processing (DONE) @@ -529,6 +554,27 @@ async fn process_job( // TODO 8. Save record // TODO 9. Mark job done + let start = Instant::now(); + + let span_id = generate_random_uuid(); + let trace_id = generate_random_uuid(); + let maybe_user_token = job.maybe_creator_user_token.as_deref().unwrap_or(""); + + let mut job_iteration_span = Span::new(&span_id, &trace_id, get_timestamp_millis()) + .name("single job execution") + .attribute("user_token", maybe_user_token) + .service_name("tts-inference-job"); + + let span_id = generate_random_uuid(); + let trace_id = generate_random_uuid(); + + let created_at_timestamp = (job.created_at.timestamp() as u64) * 1000; + + let mut since_creation_span = Span::new(&span_id, &trace_id, created_at_timestamp) + .name("job since creation") + .attribute("user_token", maybe_user_token) + .service_name("tts-inference-job"); + let mut redis = inferencer.redis_pool.get()?; let mut redis_logger = RedisJobStatusLogger::new_tts_inference( &mut redis, @@ -540,7 +586,15 @@ async fn process_job( if !lock_acquired { warn!("Could not acquire job lock for: {}", &job.id); - return Ok(()) + let duration = start.elapsed(); + + since_creation_span.set_attribute("status", "failure"); + since_creation_span.set_duration(duration); + + job_iteration_span.set_attribute("status", "failure"); + job_iteration_span.set_duration(duration); + + return Ok((since_creation_span, job_iteration_span)); } // ==================== CONFIRM OR DOWNLOAD WAVEGLOW VOCODER MODEL ==================== // @@ -808,5 +862,20 @@ async fn process_job( info!("Job {} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}", job.id, id); - Ok(()) + let duration = start.elapsed(); + + since_creation_span.set_attribute("status", "success"); + since_creation_span.set_duration(duration); + + job_iteration_span.set_attribute("status", "success"); + job_iteration_span.set_duration(duration); + + Ok((since_creation_span, job_iteration_span)) } + +fn get_timestamp_millis() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} \ No newline at end of file diff --git a/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/bug_report.md b/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..71eb0035eb --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Describe a scenario in which this project behaves unexpectedly +title: '' +labels: bug, needs-triage +assignees: '' + +--- + +[NOTE]: # ( ^^ Provide a general summary of the issue in the title above. ^^ ) + +## Description + +[NOTE]: # ( Describe the problem you're encountering. ) +[TIP]: # ( Do NOT give us access or passwords to your New Relic account or API keys! ) + +## Steps to Reproduce + +[NOTE]: # ( Please be as specific as possible. ) + +## Expected Behavior + +[NOTE]: # ( Tell us what you expected to happen. ) + +## Relevant Logs / Console output + +[NOTE]: # ( Please provide specifics of the local error logs, Browser Dev Tools console, etc. if appropriate and possible. ) + +## Your Environment + +[TIP]: # ( Include as many relevant details about your environment as possible. ) + +* ex: Browser name and version: +* ex: Operating System and version: + +## Additional context + +[TIP]: # ( Add any other context about the problem here. ) diff --git a/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/enhancement.md b/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/enhancement.md new file mode 100644 index 0000000000..0907a9ff87 --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/enhancement.md @@ -0,0 +1,27 @@ +--- +name: Enhancement request +about: Suggest an idea for a future version of this project +title: '' +labels: enhancement, needs-triage +assignees: '' + +--- + +[NOTE]: # ( ^^ Provide a general summary of the request in the title above. ^^ ) + +## Summary + +[NOTE]: # ( Provide a brief overview of what the new feature is all about. ) + +## Desired Behavior + +[NOTE]: # ( Tell us how the new feature should work. Be specific. ) +[TIP]: # ( Do NOT give us access or passwords to your New Relic account or API keys! ) + +## Possible Solution + +[NOTE]: # ( Not required. Suggest how to implement the addition or change. ) + +## Additional context + +[TIP]: # ( Why does this feature matter to you? What unique circumstances do you have? ) diff --git a/vendor/newrelic-telemetry-sdk-rust/.github/dependabot.yml b/vendor/newrelic-telemetry-sdk-rust/.github/dependabot.yml new file mode 100644 index 0000000000..f1d26d808b --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "cargo" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "daily" diff --git a/vendor/newrelic-telemetry-sdk-rust/.github/workflows/ci.yml b/vendor/newrelic-telemetry-sdk-rust/.github/workflows/ci.yml new file mode 100644 index 0000000000..72a0b5a5ea --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + ubuntu_build: + name: Ubuntu Build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install latest rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + default: true + override: true + - name: run tests + run: cargo test --all-features -- --test-threads=1 + + windows_build: + name: Windows Build + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install latest rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + default: true + override: true + - name: run tests + run: cargo test --all-features -- --test-threads=1 diff --git a/vendor/newrelic-telemetry-sdk-rust/.gitignore b/vendor/newrelic-telemetry-sdk-rust/.gitignore new file mode 100644 index 0000000000..96ef6c0b94 --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/vendor/newrelic-telemetry-sdk-rust/CODE_OF_CONDUCT.md b/vendor/newrelic-telemetry-sdk-rust/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..9503218f9e --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/CODE_OF_CONDUCT.md @@ -0,0 +1,5 @@ +### Hello! We're glad you've joined us. + +We believe participation in our community should be a harassment free experience for everyone. + +Learn more about our guidelines and principles by reading our [Code of Conduct](https://opensource.newrelic.com/code-of-conduct/) on our Open Source Website. diff --git a/vendor/newrelic-telemetry-sdk-rust/CONTRIBUTING.md b/vendor/newrelic-telemetry-sdk-rust/CONTRIBUTING.md new file mode 100644 index 0000000000..e16db5528e --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/CONTRIBUTING.md @@ -0,0 +1,30 @@ +# Contributing + +Contributions are always welcome. Before contributing please read the +[code of conduct](./CODE_OF_CONDUCT.md) and [search the issue tracker](issues); your issue may have already been discussed or fixed in `main`. To contribute, +[fork](https://help.github.com/articles/fork-a-repo/) this repository, commit your changes, and [send a Pull Request](https://help.github.com/articles/using-pull-requests/). + +Note that our [code of conduct](./CODE_OF_CONDUCT.md) applies to all platforms and venues related to this project; please follow it in all your interactions with the project and its participants. + +## Feature Requests + +Feature requests should be submitted in the [Issue tracker](../../issues), with a description of the expected behavior & use case, where they’ll remain closed until sufficient interest, [e.g. :+1: reactions](https://help.github.com/articles/about-discussions-in-issues-and-pull-requests/), has been [shown by the community](../../issues?q=label%3A%22votes+needed%22+sort%3Areactions-%2B1-desc). +Before submitting an Issue, please search for similar ones in the +[closed issues](../../issues?q=is%3Aissue+is%3Aclosed+label%3Aenhancement). + +## Pull Requests + +1. Ensure any install or build dependencies are removed before the end of the layer when doing a build. +2. Increase the version numbers in any examples files and the README.md to the new version that this Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). +3. You may merge the Pull Request in once you have the sign-off of two other developers, or if you do not have permission to do that, you may request the second reviewer to merge it for you. + +## Contributor License Agreement + +Keep in mind that when you submit your Pull Request, you'll need to sign the CLA via the click-through using CLA-Assistant. If you'd like to execute our corporate CLA, or if you have any questions, please drop us an email at opensource@newrelic.com. + +For more information about CLAs, please check out Alex Russell’s excellent post, +[“Why Do I Need to Sign This?”](https://infrequently.org/2008/06/why-do-i-need-to-sign-this/). + +## Slack + +We host a public Slack with a dedicated channel for contributors and maintainers of open source projects hosted by New Relic. If you are contributing to this project, you're welcome to request access to the #oss-contributors channel in the newrelicusers.slack.com workspace. To request access, see https://newrelicusers-signup.herokuapp.com/. diff --git a/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/vendor/newrelic-telemetry-sdk-rust/Cargo.toml new file mode 100644 index 0000000000..0434c20be1 --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "newrelic-telemetry" +version = "0.1.0" +authors = ["New Relic "] +edition = "2018" + +[features] +default = [ "client" ] +client = [ "flate2", "hyper", "hyper-tls" ] +blocking = [ "client", "futures", "tokio" ] + +[dependencies] +anyhow = "1.0" +log = "0.4" +serde = { version = "1.0.57", features = ["derive"] } +serde_json = "1.0.57" + +hyper = { version = "0.13", features = [], optional = true } +hyper-tls = { version = "0.4.0", optional = true } +flate2 = { version = "1.0.16", optional = true } +futures = { version = "0.3.1", optional = true } +tokio = { version = "0.2.6", features = ["rt-threaded", "macros"], optional = true } +uuid = { version = "0.8.1", features = ["v4"] } + +[dev-dependencies] +rand = "0.7.3" +env_logger = "0.7.1" +futures = { version = "0.3.1" } +tokio = { version = "0.2.6", features = ["rt-threaded", "macros"] } diff --git a/vendor/newrelic-telemetry-sdk-rust/LICENSE b/vendor/newrelic-telemetry-sdk-rust/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/newrelic-telemetry-sdk-rust/README.md b/vendor/newrelic-telemetry-sdk-rust/README.md new file mode 100644 index 0000000000..08194ef18b --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/README.md @@ -0,0 +1,108 @@ +[![Community Project header](https://github.com/newrelic/open-source-office/raw/master/examples/categories/images/Community_Project.png)](https://github.com/newrelic/open-source-office/blob/master/examples/categories/index.md#community-project) + +# New Relic Rust Telemetry SDK + +![Build status](https://github.com/newrelic/newrelic-telemetry-sdk-rust/workflows/CI/badge.svg) +[![Release](https://img.shields.io/github/v/release/newrelic/newrelic-telemetry-sdk-rust?include_prereleases&style=)](https://github.com/newrelic/newrelic-telemetry-sdk-rust/releases/) + +What is the New Relic Rust Telemetry SDK? + +* It's a helper library that supports sending New Relic data from within your Rust application. +* It's the foundation of [New Relic's C Telemetry SDK](https://github.com/newrelic/newrelic-telemetry-sdk-c). +* It’s an example of "best practices" for sending us data. + +[The Telemetry SDK](https://docs.newrelic.com/docs/telemetry-data-platform/get-started/capabilities/telemetry-sdks-send-custom-telemetry-data-new-relic) provides you, the end-user programmer, with a `Client `that sends `Spans` to New Relic. Individual spans are collected together into batches (via a `SpanBatch` object), and clients send these batches. It serves as a foundation for getting open-standards based telemetry data like [OpenCensus](https://opencensus.io/), [OpenTracing](https://opentracing.io/), and [OpenTelemetry](https://opentelemetry.io/) into New Relic. You can use this to build tracers/exporters, such as ones based on these open standards. + +This SDK currently supports sending spans to the [Trace API](https://docs.newrelic.com/docs/understand-dependencies/distributed-tracing/trace-api/introduction-trace-api). + +**This project is currently in an alpha state.** The API of this crate is +likely to change, therefore there's no API documentation available yet. + +## Getting Started + +In order to send telemetry data to New Relic APIs, you will need an Insert API key. To find out how to generate this key, see our [docs](https://docs.newrelic.com/docs/apis/get-started/intro-apis/types-new-relic-api-keys). + +The Rust Telemetry SDK is not published on [crates.io](https://crates.io) +yet. In order to use it in your Rust application, you have to clone this +repository locally and include the crate via the local path name. If you +cloned the Rust Telemetry SDK into a `vendor` subfolder of your Rust crate, +you can add it as a dependency to your crate by adding the following dependency +to your `Cargo.toml`: + +```toml +[dependencies] +newrelic-telemetry = { path = "vendor/newrelic-telemetry-sdk-rust" } +``` + +This is a simple application that sends a single span via the asynchronous +client: +```rust +use newrelic_telemetry::{ClientBuilder, Span}; +use std::env; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[tokio::main] +async fn main() { + // Obtain a license key from the environment. + let license_key = env::var("NEW_RELIC_API_KEY").unwrap(); + + // Build an asynchronous client. + let client = ClientBuilder::new(&license_key).build().unwrap(); + + // Create a span and a span batch. + let span = Span::new( + "e9f54a2c322d7578", + "1b1bf29379951c1d", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as u64, + ) + .name("/index.html") + .attribute("username", "user") + .service_name("Telemetry Application"); + + let span_batch = vec![span].into(); + + // Send a span batch and await the future. + client.send_spans(span_batch).await; +} +``` + +## Find and use your data + +Tips on how to find and query your data in New Relic: +- [Find trace/span data](https://docs.newrelic.com/docs/understand-dependencies/distributed-tracing/trace-api/introduction-trace-api#view-data) + +For general querying information, see: +- [Query New Relic data](https://docs.newrelic.com/docs/query-your-data/explore-query-data/explore-data/introduction-querying-new-relic-data) +- [Intro to NRQL](https://docs.newrelic.com/docs/query-your-data/nrql-new-relic-query-language/get-started/introduction-nrql-new-relics-query-language) + +## Support + +New Relic hosts and moderates an online forum where customers can interact with +New Relic employees as well as other customers to get help and share best +practices. Like all official New Relic open source projects, there's a related +Community topic in the New Relic Explorers Hub. You can find this project's +topic/threads in the [Telemetry SDK section of Explorers Hub](https://discuss.newrelic.com/t/new-relic-rust-telemetry-sdk/114558) + +## Contributing + +We encourage your contributions to improve the Rust Telemetry SDK! Keep in mind +when you submit your pull request, you'll need to sign the CLA via the +click-through using CLA-Assistant. You only have to sign the CLA one time per +project. If you have any questions, or to execute our corporate CLA, required +if your contribution is on behalf of a company, please drop us an email at +opensource@newrelic.com. + +## License + +The Rust Telemetry SDK is licensed under the [Apache 2.0](http://apache.org/licenses/LICENSE-2.0.txt) +License. + +### Limitations + +The New Relic Telemetry APIs are rate limited. Please reference the +documentation for [New Relic Metric API](https://docs.newrelic.com/docs/introduction-new-relic-metric-api) +and [New Relic Trace API requirements and limits](https://docs.newrelic.com/docs/apm/distributed-tracing/trace-api/trace-api-general-requirements-limits) +on the specifics of the rate limits. diff --git a/vendor/newrelic-telemetry-sdk-rust/cla.md b/vendor/newrelic-telemetry-sdk-rust/cla.md new file mode 100644 index 0000000000..27af9b4dd7 --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/cla.md @@ -0,0 +1,16 @@ +# NEW RELIC, INC. +## INDIVIDUAL CONTRIBUTOR LICENSE AGREEMENT +Thank you for your interest in contributing to the open source projects of New Relic, Inc. (“New Relic”). In order to clarify the intellectual property license granted with Contributions from any person or entity, New Relic must have a Contributor License Agreement ("Agreement") on file that has been signed by each Contributor, indicating agreement to the license terms below. This Agreement is for your protection as a Contributor as well as the protection of New Relic; it does not change your rights to use your own Contributions for any other purpose. + +You accept and agree to the following terms and conditions for Your present and future Contributions submitted to New Relic. Except for the licenses granted herein to New Relic and recipients of software distributed by New Relic, You reserve all right, title, and interest in and to Your Contributions. + +## Definitions. +1. "You" (or "Your") shall mean the copyright owner or legal entity authorized by the copyright owner that is entering into this Agreement with New Relic. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. +2. "Contribution" shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to New Relic for inclusion in, or documentation of, any of the products managed or maintained by New Relic (the "Work"). For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to New Relic or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, New Relic for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution." +3. Grant of Copyright License. Subject to the terms and conditions of this Agreement, You hereby grant to New Relic and to recipients of software distributed by New Relic a perpetual, worldwide, non-exclusive, no-charge, royalty-free, transferable, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works. +4. Grant of Patent License. Subject to the terms and conditions of this Agreement, You hereby grant to New Relic and to recipients of software distributed by New Relic a perpetual, worldwide, non-exclusive, no-charge, royalty-free, transferable, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contributions alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that Your Contribution, or the Work to which You have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed. +5. You represent that You are legally entitled to grant the above licenses. If Your employer(s) has rights to intellectual property that You create that includes Your Contributions, You represent that You have received permission to make Contributions on behalf of that employer, that Your employer has waived such rights for Your Contributions to New Relic, or that Your employer has executed a separate Agreement with New Relic. +6. You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which You are personally aware and which are associated with any part of Your Contributions. +7. You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON- INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. +8. Should You wish to submit work that is not Your original creation, You may submit it to New Relic separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which You are personally aware, and conspicuously marking the work as "Submitted on behalf of a third-party: [named here]". +9. You agree to notify New Relic of any facts or circumstances of which You become aware that would make these representations inaccurate in any respect. diff --git a/vendor/newrelic-telemetry-sdk-rust/src/attribute.rs b/vendor/newrelic-telemetry-sdk-rust/src/attribute.rs new file mode 100644 index 0000000000..4b9c787aa0 --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/src/attribute.rs @@ -0,0 +1,252 @@ +/// +/// Copyright 2020 New Relic Corporation. All rights reserved. +/// SPDX-License-Identifier: Apache-2.0 +/// + +/// Represents any valid attribute value. +/// +/// According to the [specification](https://github.com/newrelic/newrelic-telemetry-sdk-specs/blob/master/capabilities.md), +/// attribute values can be a string, numeric, or boolean. A numeric value is +/// represented either as a signed integer, an unsigned integer or a float. +#[derive(serde::Serialize, Debug, PartialEq, Clone)] +#[serde(untagged)] +pub enum Value { + /// Represents a signed integer attribute value. + /// + /// ``` + /// # use newrelic_telemetry::attribute::Value; + /// # + /// let v = Value::Int(-3); + /// ``` + Int(i64), + + /// Represents an unsigned integer attribute value. + /// + /// ``` + /// # use newrelic_telemetry::attribute::Value; + /// # + /// let v = Value::UInt(42); + /// ``` + UInt(u64), + + /// Represents a signed 128 bit integer attribute value. + /// + /// ``` + /// # use newrelic_telemetry::attribute::Value; + /// # + /// let v = Value::Int128(-30); + /// ``` + Int128(i128), + + /// Represents an unsigned 128 bit integer attribute value. + /// + /// ``` + /// # use newrelic_telemetry::attribute::Value; + /// # + /// let v = Value::UInt128(30); + /// ``` + UInt128(u128), + + /// Represents a string attribute value. + /// + /// ``` + /// # use newrelic_telemetry::attribute::Value; + /// # + /// let v = Value::Str(String::from("root")); + /// ``` + Str(String), + + /// Represents a float attribute value. + /// + /// ``` + /// # use newrelic_telemetry::attribute::Value; + /// # + /// let v = Value::Float(3.14159); + /// ``` + Float(f64), + + /// Represents a bool attribute value. + /// + /// ``` + /// # use newrelic_telemetry::attribute::Value; + /// # + /// let v = Value::Bool(true); + /// ``` + Bool(bool), +} + +/// Converts an i128 to an attribute value. +/// +/// ``` +/// # use newrelic_telemetry::attribute::Value; +/// # +/// let v: i128 = -10; +/// assert_eq!(Value::Int128(-10), v.into()); +/// ``` +impl From for Value { + fn from(value: i128) -> Value { + Value::Int128(value) + } +} + +/// Converts an i64 to an attribute value. +/// +/// ``` +/// # use newrelic_telemetry::attribute::Value; +/// # +/// let v: i64 = -5; +/// assert_eq!(Value::Int(-5), v.into()); +/// ``` +impl From for Value { + fn from(value: i64) -> Value { + Value::Int(value) + } +} + +/// Converts an i32 to an attribute value. +/// +/// ``` +/// # use newrelic_telemetry::attribute::Value; +/// # +/// let v: i32 = -5; +/// assert_eq!(Value::Int(-5), v.into()); +/// ``` +impl From for Value { + fn from(value: i32) -> Value { + Value::Int(value as i64) + } +} + +/// Converts a u128 to an attribute value. +/// +/// ``` +/// # use newrelic_telemetry::attribute::Value; +/// # +/// let v: u128 = 50; +/// assert_eq!(Value::UInt128(50), v.into()); +/// ``` +impl From for Value { + fn from(value: u128) -> Value { + Value::UInt128(value) + } +} + +/// Converts a u64 to an attribute value. +/// +/// ``` +/// # use newrelic_telemetry::attribute::Value; +/// # +/// let v: u64 = 5; +/// assert_eq!(Value::UInt(5), v.into()); +/// ``` +impl From for Value { + fn from(value: u64) -> Value { + Value::UInt(value) + } +} + +/// Converts a u32 to an attribute value. +/// +/// ``` +/// # use newrelic_telemetry::attribute::Value; +/// # +/// let v: u32 = 5; +/// assert_eq!(Value::UInt(5), v.into()); +/// ``` +impl From for Value { + fn from(value: u32) -> Value { + Value::UInt(value as u64) + } +} + +/// Converts a string to an attribute value. +/// +/// ``` +/// # use newrelic_telemetry::attribute::Value; +/// # +/// let v = "root"; +/// assert_eq!(Value::Str(String::from("root")), v.into()); +/// ``` +impl From<&str> for Value { + fn from(value: &str) -> Value { + Value::Str(value.to_string()) + } +} + +/// Converts a f64 to an attribute value. +/// +/// ``` +/// # use newrelic_telemetry::attribute::Value; +/// # +/// let v: f64 = 3.14159; +/// assert_eq!(Value::Float(v), v.into()); +/// ``` +impl From for Value { + fn from(value: f64) -> Value { + Value::Float(value) + } +} + +/// Converts a f32 to an attribute value. +/// +/// ``` +/// # use newrelic_telemetry::attribute::Value; +/// # +/// let v: f32 = 3.14159; +/// assert_eq!(Value::Float(v as f64), v.into()); +/// ``` +impl From for Value { + fn from(value: f32) -> Value { + Value::Float(value as f64) + } +} + +/// Converts a bool to an attribute value. +/// +/// ``` +/// # use newrelic_telemetry::attribute::Value; +/// # +/// let v = true; +/// assert_eq!(Value::Bool(true), v.into()); +/// ``` +impl From for Value { + fn from(value: bool) -> Value { + Value::Bool(value) + } +} + +#[cfg(test)] +mod tests { + use super::Value; + use serde_json::json; + + #[test] + fn value_to_json() { + // Attribute values should serialize to plain JSON values. + assert_eq!(json!(Value::Int(-5)), json!(-5)); + assert_eq!(json!(Value::UInt(5)), json!(5)); + assert_eq!(json!(Value::Float(3.14159)), json!(3.14159)); + assert_eq!(json!(Value::Str(String::from("root"))), json!("root")); + assert_eq!(json!(Value::Bool(true)), json!(true)); + } + + #[test] + fn into_value() { + // Should be able to use Value::from or .into() to create Values + assert_eq!(Value::Int(-5), Value::from(-5)); + assert_eq!(Value::Int(-5), (-5 as i32).into()); + + // cast needed because integer types default to i32 + assert_eq!(Value::UInt(5), Value::from(5 as u64)); + assert_eq!(Value::UInt(5), (5 as u64).into()); + + assert_eq!(Value::Float(3.14159), Value::from(3.14159)); + assert_eq!(Value::Float(3.14159), (3.14159 as f64).into()); + + assert_eq!(Value::Str("root".to_string()), Value::from("root")); + assert_eq!(Value::Str("root".to_string()), "root".into()); + + assert_eq!(Value::Bool(true), Value::from(true)); + assert_eq!(Value::Bool(true), true.into()); + } +} diff --git a/vendor/newrelic-telemetry-sdk-rust/src/client.rs b/vendor/newrelic-telemetry-sdk-rust/src/client.rs new file mode 100644 index 0000000000..0efb8bf4ff --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/src/client.rs @@ -0,0 +1,971 @@ +/// +/// Copyright 2020 New Relic Corporation. All rights reserved. +/// SPDX-License-Identifier: Apache-2.0 +/// +use crate::span::SpanBatch; +use anyhow::{anyhow, Result}; +use flate2::write::GzEncoder; +use flate2::Compression; +use hyper::client::HttpConnector; +use hyper::header::{CONTENT_ENCODING, CONTENT_TYPE, USER_AGENT}; +use hyper::{Body, HeaderMap, Method, Request, Response, Uri}; +use hyper_tls::HttpsConnector; +use log::{debug, error, info}; +use std::future::Future; +use std::io::Write; +use std::pin::Pin; +use std::thread; +use std::time::Duration; + +const VERSION: &'static str = env!("CARGO_PKG_VERSION"); +const TRACE_API_PATH: &'static str = "trace/v1"; + +/// Types that can be sent to a New Relic ingest API +/// +/// New Relic ingest APIs currently accept batches of traces, metrics, events +/// or logs. +pub trait Sendable: std::fmt::Display + Send { + /// Return the uuid for the `Sendable` + /// + /// This method returns a version 4 UUID string which enables the ingest + /// service to identify duplicate requests. + fn uuid(&self) -> &str; + + // Create a payload + // + // This method creates a JSON payload representing the contents of the + // `Sendable` object, conforming to the requirements of a related ingest + // API (traces, metrics, events or logs). + fn marshall(&self) -> Result; + + // Split a `Sendable` + // + // New Relic ingest APIs reject payloads that are too large. In that case, + // a 413 response code is sent, the payload must be split and sent again + // (see [the specification](https://github.com/newrelic/newrelic-telemetry-sdk-specs/blob/master/communication.md#response-codes) + // for further details). + // + // This method removes half of the content of the `Sendable` object and + // puts it into a second `Sendable` object, which is returned. + fn split(&mut self) -> Box; +} + +// Represents a New Relic ingest endpoint. +#[derive(Debug)] +struct Endpoint { + // The host name or address of the endpoint. + host: String, + + // The port of the endpoint. This is optional, if not given it will default + // to the standard HTTPS port. + port: Option, + + // The path for the endpoint. + path: &'static str, +} + +impl Endpoint { + // Create an URI from an endpoint. + // + // This uses the parser of `hyper::Uri` to validate the URI and returns + // `https` or `http` URIs, based on the `use_tls` flag. + fn uri(&self, use_tls: bool) -> Result { + let port_str = match self.port { + Some(p) => format!(":{}", p), + _ => "".to_string(), + }; + + let uri = format!( + "{}://{}{}/{}", + if use_tls { "https" } else { "http" }, + self.host, + port_str, + self.path + ); + + Ok(uri.parse::()?) + } +} + +/// `ClientBuilder` acts as builder for initializing a `Client`. +/// +/// It can be used to customize ingest URLs, the backoff factor, the retry +/// maximum, and the product info. +/// +/// ``` +/// # use anyhow::Result; +/// # use newrelic_telemetry::ClientBuilder; +/// # use std::time::Duration; +/// # fn main() -> Result<()> { +/// # let api_key = ""; +/// let mut builder = ClientBuilder::new(api_key); +/// +/// let client = builder.backoff_factor(Duration::from_secs(10)) +/// .product_info("RustDoc", "1.0") +/// .build()?; +/// # Ok(()) +/// # } +/// ``` +pub struct ClientBuilder { + api_key: String, + backoff_factor: Duration, + retries_max: u32, + endpoint_traces: Endpoint, + product_info: Option<(String, String)>, + blocking_queue_max: usize, + use_tls: bool, +} + +impl ClientBuilder { + /// Initialize the client builder with an API key. + /// + /// Other values will be set to defaults: + /// * The default backoff factor will be 5 seconds. + /// * The default maximum of retries is 8. + /// * The default trace endpoint is `https://trace-api.newrelic.com/trace/v1` on port 80. + /// * By default, product information is empty. + /// * By default, no more than 100 batches are sent in one go in blocking mode. + /// + /// ``` + /// # use newrelic_telemetry::ClientBuilder; + /// # let api_key = ""; + /// let mut builder = ClientBuilder::new(api_key); + /// ``` + pub fn new(api_key: &str) -> Self { + ClientBuilder { + api_key: api_key.to_string(), + backoff_factor: Duration::from_secs(5), + retries_max: 8, + endpoint_traces: Endpoint { + host: "trace-api.newrelic.com".to_string(), + port: None, + path: TRACE_API_PATH, + }, + product_info: None, + blocking_queue_max: 100, + use_tls: true, + } + } + + /// Configures a backoff factor. + /// + /// If a request fails, the SDK retries the request at increasing intervals + /// and eventually drops data if the request cannot be completed. + /// + /// The amount of time to wait after a request can be computed using this + /// logic: + /// + /// `backoff_factor * (2 ^ (number_of_retries - 1))` + /// + /// For a backoff factor of 1 second, and a maximum of 6 retries, the retry + /// delay interval should follow a pattern of [0, 1, 2, 4, 8, 16]. + /// + /// See the [specification](https://github.com/newrelic/newrelic-telemetry-sdk-specs/blob/master/communication.md#graceful-degradation) + /// for further details. + /// + /// ``` + /// # use newrelic_telemetry::ClientBuilder; + /// # use std::time::Duration; + /// # let api_key = ""; + /// let mut builder = + /// ClientBuilder::new(api_key).backoff_factor(Duration::from_secs(10)); + /// ``` + pub fn backoff_factor(mut self, factor: Duration) -> Self { + self.backoff_factor = factor; + self + } + + /// Configures the maximum numbers of retries. + /// + /// If a request fails, the SDK retries the request at increasing intervals + /// and eventually drops data if the request cannot be completed. + /// + /// If zero is given as a maximum, no retries will be made for failed + /// requests. + /// + /// For a backoff factor of 1 second, and a maximum of 6 retries, the retry + /// delay interval should follow a pattern of [0, 1, 2, 4, 8, 16]. + /// + /// See the [specification](https://github.com/newrelic/newrelic-telemetry-sdk-specs/blob/master/communication.md#graceful-degradation) + /// for further details. + /// + /// ``` + /// # use newrelic_telemetry::ClientBuilder; + /// # let api_key = ""; + /// let mut builder = + /// ClientBuilder::new(api_key).retries_max(4); + /// ``` + pub fn retries_max(mut self, retries: u32) -> Self { + self.retries_max = retries; + self + } + + /// Configure the ingest host for traces. + /// + /// Overrides the default ingest host for traces to facilitate communication + /// with alternative New Relic backends. + /// + /// ``` + /// # use newrelic_telemetry::ClientBuilder; + /// # let api_key = ""; + /// let mut builder = + /// ClientBuilder::new(api_key).endpoint_traces("127.0.0.1", None); + /// ``` + pub fn endpoint_traces(mut self, url: &str, port: Option) -> Self { + self.endpoint_traces = Endpoint { + host: url.to_string(), + path: TRACE_API_PATH, + port: port, + }; + self + } + + /// Configure a product and version. + /// + /// The specified product and version will be appended to the `User-Agent` + /// header of payloads. + /// + /// See the [specification](https://github.com/newrelic/newrelic-telemetry-sdk-specs/blob/master/communication.md#user-agent) + /// for further details. + /// + /// ``` + /// # use newrelic_telemetry::ClientBuilder; + /// # let api_key = ""; + /// let mut builder = + /// ClientBuilder::new(api_key).product_info("NewRelic-Cpp-OpenTelemetry", "0.2.1"); + /// ``` + pub fn product_info(mut self, product: &str, version: &str) -> Self { + self.product_info = Some((product.to_string(), version.to_string())); + self + } + + /// Configure the maximum number of batches sent in one go in blocking mode. + /// + /// This configuration has no effect for default non-blocking clients. + /// + /// If the number of batches in the blocking client's batch queue exceeds + /// the maximum given here, the addditional batches will be dropped. This + /// mechanism avoids accumulating back pressure. + /// + /// ``` + /// # use newrelic_telemetry::ClientBuilder; + /// # let api_key = ""; + /// let mut builder = + /// ClientBuilder::new(api_key).blocking_queue_max(10); + /// ``` + pub fn blocking_queue_max(mut self, queue_max: usize) -> Self { + self.blocking_queue_max = queue_max; + self + } + + // Configure TLS usage. + // + // New Relic endpoints exclusively support HTTPS. This is mainly provided + // for testing purposes. + pub fn tls(mut self, tls: bool) -> Self { + self.use_tls = tls; + self + } + + /// Build a client. + /// + /// ``` + /// # use anyhow::Result; + /// # use newrelic_telemetry::ClientBuilder; + /// # fn main() -> Result<()> { + /// # let api_key = ""; + /// let builder = ClientBuilder::new(api_key); + /// + /// let client = builder.build()?; + /// # Ok(()) + /// # } + /// ``` + pub fn build(self) -> Result { + Client::new(self) + } + + /// Build a blocking client. + /// + /// ``` + /// # use anyhow::Result; + /// # use newrelic_telemetry::ClientBuilder; + /// # fn main() -> Result<()> { + /// # let api_key = ""; + /// let builder = ClientBuilder::new(api_key); + /// + /// let client = builder.build_blocking()?; + /// # Ok(()) + /// # } + /// ``` + #[cfg(feature = "blocking")] + pub fn build_blocking(self) -> Result { + blocking::Client::new(self) + } + + fn get_backoff_sequence(&self) -> Vec { + (0..self.retries_max) + .map(|num_retry| { + if num_retry == 0 { + Duration::from_secs(0) + } else { + self.backoff_factor * (2_u32.pow(num_retry - 1)) + } + }) + .collect() + } + + fn get_user_agent_header(&self) -> String { + let product_info = match &self.product_info { + Some(s) => format!(" {}/{}", s.0, s.1), + _ => "".to_string(), + }; + + format!("NewRelic-Rust-TelemetrySDK/{}{}", VERSION, product_info) + } +} + +// An internal enum representing the state of a payload. +#[derive(Debug, PartialEq)] +enum SendableState { + // No retry should be made. + Done, + + // A retry should be made. Either after the given duration, or, if it + // is `None`, according to the backoff sequence. + Retry(Option), + + // The payload should be split and a retry should be made for both + // payloads. + Split, +} + +pub struct Client { + api_key: String, + user_agent: String, + backoff_sequence: Vec, + endpoint_traces: Uri, + client: hyper::Client>, +} + +impl Client { + /// Constructs a `Client` from a `ClientBuilder`. + pub fn new(builder: ClientBuilder) -> Result { + let https = HttpsConnector::new(); + let user_agent = builder.get_user_agent_header(); + let backoff_seq = builder.get_backoff_sequence(); + + Ok(Client { + api_key: builder.api_key, + endpoint_traces: builder.endpoint_traces.uri(builder.use_tls)?, + user_agent: user_agent, + backoff_sequence: backoff_seq, + client: hyper::Client::builder().build::<_, hyper::Body>(https), + }) + } + + /// Sends a span batch. + /// + /// This asynchronously sends a span batch, encapsulating retry and backoff + /// mechanisms defined in the [specification](https://github.com/newrelic/newrelic-telemetry-sdk-specs/blob/master/communication.md) + /// and customized via the `ClientBuilder`. + pub async fn send_spans(&self, batch: SpanBatch) { + self.send(Box::new(batch), &self.endpoint_traces).await + } + + // Returns a gzip compressed version of the given string. + fn to_gzip(text: &String) -> Result> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(text.as_bytes())?; + Ok(encoder.finish()?) + } + + // Extract the value of the Retry-After HTTP response header + fn extract_retry_after(headers: &HeaderMap) -> Result { + if let Some(dur) = headers.get("retry-after") { + Ok(Duration::from_secs(dur.to_str()?.parse::()?)) + } else { + Err(anyhow!("missing retry-after header")) + } + } + + // Sends a given `Sendable` asynchronously to a given endpoint. + fn send<'a>( + &'a self, + mut batch: Box, + endpoint: &'a Uri, + ) -> Pin + 'a>> { + Box::pin(async move { + for duration in self.backoff_sequence.iter() { + let request = match self.request(&*batch, endpoint) { + Ok(r) => r, + Err(e) => { + error!("cannot create request for {}, dropping due to {}", batch, e); + return; + } + }; + + let response = match self.client.request(request).await { + Ok(r) => r, + Err(e) => { + error!("cannot send request for {}, dropping due to {}", batch, e); + return; + } + }; + + let status = Self::process_response(&*batch, response); + + let duration = match status { + SendableState::Done => return, + SendableState::Retry(Some(duration)) => duration, + SendableState::Split => { + let batch2 = batch.split(); + self.send(batch, endpoint).await; + self.send(batch2, endpoint).await; + return; + } + _ => *duration, + }; + + thread::sleep(duration); + } + }) + } + + // Create a request from the given batch and endpoint. + fn request<'a>(&self, batch: &(dyn Sendable + 'a), endpoint: &Uri) -> Result> { + let raw = batch.marshall()?; + let gzipped = Self::to_gzip(&raw)?; + + Ok(Request::builder() + .method(Method::POST) + .uri(endpoint) + .header("Api-Key", &self.api_key) + .header("Data-Format", "newrelic") + .header("Data-Format-Version", "1") + .header("x-request-id", batch.uuid()) + .header(USER_AGENT, &self.user_agent) + .header(CONTENT_ENCODING, "gzip") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(gzipped))?) + } + + // Based on the response from an ingest endpoint, decide whether to + // retry or split a payload. + // + // See the [specification](https://github.com/newrelic/newrelic-telemetry-sdk-specs/blob/master/communication.md#response-codes) + // for further details. + fn process_response<'a, T>( + batch: &(dyn Sendable + 'a), + response: Response, + ) -> SendableState { + let status = response.status(); + + match status.as_u16() { + 200..=299 => { + debug!("response {}, successfully sent {}", status, batch); + } + 400 | 401 | 403 | 404 | 405 | 409 | 410 | 411 => { + error!("response {}, dropping {}", status, batch); + } + 413 => { + info!( + "response {}, payload too large, splitting {}", + status, batch + ); + return SendableState::Split; + } + 429 => match Self::extract_retry_after(response.headers()) { + Ok(duration) => { + info!( + "response {}: retry interval {:?}, retrying {}", + status, duration, batch + ); + + return SendableState::Retry(Some(duration)); + } + Err(e) => { + error!("response {}, {}, dropping {}", status, e, batch); + } + }, + _ => { + debug!("response {}, retry {}", status, batch); + return SendableState::Retry(None); + } + } + return SendableState::Done; + } +} + +#[cfg(feature = "blocking")] +pub mod blocking { + use super::{ClientBuilder, SpanBatch}; + use anyhow::Result; + use futures::future; + use log::warn; + use std::sync::mpsc; + use std::sync::Mutex; + use std::thread; + use tokio::runtime::Builder; + + enum SendableType { + Spans(SpanBatch), + } + + pub struct Client { + channel: Mutex>>, + handle: thread::JoinHandle<()>, + } + + impl Client { + pub fn new(builder: ClientBuilder) -> Result { + let (tx, rx) = mpsc::channel::>(); + let mut runtime = Builder::new().threaded_scheduler().enable_all().build()?; + let queue_max = builder.blocking_queue_max; + let client = builder.build()?; + + let handle = thread::spawn(move || loop { + let mut batches = vec![]; + + // Wait until at least one batch is received. + match rx.recv() { + Ok(b) => batches.push(b), + Err(_) => break, + }; + + // Empty the channel. + loop { + match rx.try_recv() { + Ok(b) => batches.push(b), + Err(_) => break, + } + } + + // Drop batches that exceed the maximum defined queue size. + if batches.len() > queue_max { + warn!( + "back pressure, dropping {} span batches", + batches.len() - queue_max + ); + batches.drain(queue_max..); + } + + // Block until all batches are sent. + runtime.block_on(future::join_all(batches.drain(..).map(|b| match *b { + SendableType::Spans(batch) => client.send_spans(batch), + }))); + }); + + Ok(Client { + channel: Mutex::new(tx), + handle, + }) + } + + pub fn send_spans(&self, b: SpanBatch) { + if let Ok(ch) = self.channel.lock() { + if let Err(_) = ch.send(Box::new(SendableType::Spans(b))) {} + } + } + + pub fn shutdown(self) { + drop(self.channel); + + let _ = self.handle.join(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::Result; + use flate2::read::GzDecoder; + use hyper::header::{HeaderValue, CONTENT_ENCODING, CONTENT_TYPE, USER_AGENT}; + use hyper::{Method, Response}; + use std::fmt; + use std::io::Read; + use std::time::Duration; + pub struct TestBatch; + + impl Sendable for TestBatch { + fn uuid(&self) -> &str { + "" + } + + fn marshall(&self) -> Result { + Ok("".to_string()) + } + + fn split(&mut self) -> Box { + Box::new(TestBatch) + } + } + + impl fmt::Display for TestBatch { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "") + } + } + + #[test] + fn build() -> Result<()> { + let client = ClientBuilder::new("0000") + .backoff_factor(Duration::from_secs(2)) + .retries_max(6) + .endpoint_traces("127.0.0.1", Some(8080)) + .product_info("Test", "1.0") + .build()?; + + assert_eq!(client.api_key, "0000"); + assert_eq!(client.endpoint_traces.host(), Some("127.0.0.1")); + assert_eq!(client.endpoint_traces.port_u16(), Some(8080)); + assert_eq!(client.endpoint_traces.scheme().unwrap().as_str(), "https"); + assert_eq!( + client.user_agent, + format!("NewRelic-Rust-TelemetrySDK/{} Test/1.0", VERSION) + ); + assert_eq!( + client.backoff_sequence, + vec![0, 2, 4, 8, 16, 32] + .into_iter() + .map(|d| Duration::from_secs(d)) + .collect::>() + ); + + Ok(()) + } + + #[test] + fn build_error() { + let client = ClientBuilder::new("0000") + .endpoint_traces(":80", Some(8080)) + .build(); + + assert!(client.is_err()); + } + + #[test] + fn uri_from_endpoint_ok() -> Result<()> { + let endpoint = Endpoint { + host: "host".to_string(), + path: TRACE_API_PATH, + port: Some(80), + }; + + let uri = endpoint.uri(true)?; + assert_eq!(uri.host(), Some("host")); + assert_eq!(uri.port_u16(), Some(80)); + assert_eq!(uri.path(), "/trace/v1"); + assert_eq!(uri.scheme().unwrap().as_str(), "https"); + + Ok(()) + } + + #[test] + fn uri_from_endpoint_no_tls_ok() -> Result<()> { + let endpoint = Endpoint { + host: "host".to_string(), + path: TRACE_API_PATH, + port: Some(80), + }; + + let uri = endpoint.uri(false)?; + assert_eq!(uri.host(), Some("host")); + assert_eq!(uri.port_u16(), Some(80)); + assert_eq!(uri.path(), "/trace/v1"); + assert_eq!(uri.scheme().unwrap().as_str(), "http"); + + Ok(()) + } + + #[test] + fn uri_from_endpoint_error() -> Result<()> { + for endpoint in vec![ + Endpoint { + host: "host:80".to_string(), + path: TRACE_API_PATH, + port: Some(80), + }, + Endpoint { + host: "?".to_string(), + path: TRACE_API_PATH, + port: Some(80), + }, + Endpoint { + host: "".to_string(), + path: TRACE_API_PATH, + port: None, + }, + ] { + let uri = endpoint.uri(true); + + assert!( + uri.is_err(), + format!("Could create an uri from {:?}: {:?}", endpoint, uri) + ); + } + + Ok(()) + } + + #[test] + fn to_gzip() -> Result<()> { + let text = "Text to be encoded".to_string(); + let encoded = Client::to_gzip(&text)?; + + let mut gz = GzDecoder::new(&encoded[..]); + let mut decoded = String::new(); + gz.read_to_string(&mut decoded)?; + + assert_eq!(decoded, text); + + Ok(()) + } + + #[test] + fn extract_retry_after() -> Result<()> { + let mut headers = hyper::HeaderMap::new(); + + let when = Client::extract_retry_after(&headers); + assert!(when.is_err()); + + headers.insert("Retry-after", "7".parse()?); + + let when = Client::extract_retry_after(&headers)?; + assert_eq!(when, Duration::from_secs(7)); + + headers.insert("Retry-after", "seven".parse()?); + + let when = Client::extract_retry_after(&headers); + assert!(when.is_err()); + + Ok(()) + } + + #[test] + fn process_response_success() -> Result<()> { + for code in 200..300 { + let batch = Box::new(TestBatch); + let response = Response::builder().status(code).body(())?; + + assert_eq!( + Client::process_response(&*batch, response), + SendableState::Done + ); + } + + Ok(()) + } + + #[test] + fn process_response_error() -> Result<()> { + for code in vec![400, 401, 403, 404, 405, 409, 410, 411] { + let batch = Box::new(TestBatch); + let response = Response::builder().status(code).body(())?; + + assert_eq!( + Client::process_response(&*batch, response), + SendableState::Done + ); + } + + Ok(()) + } + + #[test] + fn process_response_split() -> Result<()> { + let batch = Box::new(TestBatch); + let response = Response::builder().status(413).body(())?; + + assert_eq!( + Client::process_response(&*batch, response), + SendableState::Split + ); + + Ok(()) + } + + #[test] + fn process_response_retry_from_header() -> Result<()> { + let batch = Box::new(TestBatch); + let response = Response::builder() + .status(429) + .header("retry-after", "7") + .body(())?; + + assert_eq!( + Client::process_response(&*batch, response), + SendableState::Retry(Some(Duration::from_secs(7))) + ); + + Ok(()) + } + + #[test] + fn process_response_retry() -> Result<()> { + let mut codes = vec![402, 406, 407, 408]; + codes.append(&mut (100..200).collect()); + codes.append(&mut (300..400).collect()); + codes.append(&mut (430..600).collect()); + + for code in codes { + let batch = Box::new(TestBatch); + let response = Response::builder().status(code).body(())?; + + assert_eq!( + Client::process_response(&*batch, response), + SendableState::Retry(None), + "expected retry on {}", + code + ); + } + + Ok(()) + } + + #[test] + fn request() -> Result<()> { + let batch = Box::new(TestBatch); + let client = ClientBuilder::new("").build()?; + let endpoint = Endpoint { + host: "host".to_string(), + path: TRACE_API_PATH, + port: None, + }; + + let request = client.request(&*batch, &endpoint.uri(true)?)?; + + assert_eq!(request.uri().port(), None); + assert_eq!(request.uri().host(), Some("host")); + assert_eq!(request.uri().path(), "/trace/v1"); + assert_eq!(request.method(), Method::POST); + + let headers = request.headers(); + + let expected_headers = vec![ + (CONTENT_ENCODING.as_str(), "gzip"), + (CONTENT_TYPE.as_str(), "application/json"), + ("Api-Key", &client.api_key), + ("Data-Format", "newrelic"), + ("Data-Format-Version", "1"), + ("x-request-id", batch.uuid()), + (USER_AGENT.as_str(), &client.user_agent), + ]; + let expected_count = expected_headers.len(); + for (header, expected) in expected_headers { + let value = headers.get(header); + let expected = HeaderValue::from_str(expected)?; + assert_eq!(value, Some(&expected)); + } + + assert_eq!(headers.len(), expected_count, "Unexpected header present!"); + + Ok(()) + } + + #[test] + fn request_port() -> Result<()> { + let batch = Box::new(TestBatch); + let client = ClientBuilder::new("").build()?; + let endpoint = Endpoint { + host: "host".to_string(), + path: TRACE_API_PATH, + port: Some(80), + }; + + let request = client.request(&*batch, &endpoint.uri(true)?)?; + + assert_eq!(request.uri().port().unwrap().as_u16(), 80); + assert_eq!(request.uri().host(), Some("host")); + assert_eq!(request.uri().path(), "/trace/v1"); + + Ok(()) + } + + #[test] + fn builder_default() { + let b = ClientBuilder::new("0000"); + + assert_eq!(b.api_key, "0000"); + assert_eq!(b.backoff_factor, Duration::from_secs(5)); + assert_eq!(b.retries_max, 8); + assert_eq!(b.endpoint_traces.host, "trace-api.newrelic.com"); + assert_eq!(b.endpoint_traces.port, None); + assert_eq!(b.product_info, None); + assert_eq!(b.use_tls, true); + } + + #[test] + fn builder_setters() { + let b = ClientBuilder::new("0000") + .backoff_factor(Duration::from_secs(10)) + .retries_max(10) + .endpoint_traces("127.0.0.1", Some(8080)) + .product_info("Test", "1.0"); + + assert_eq!(b.api_key, "0000"); + assert_eq!(b.backoff_factor, Duration::from_secs(10)); + assert_eq!(b.retries_max, 10); + assert_eq!(b.endpoint_traces.host, "127.0.0.1"); + assert_eq!(b.endpoint_traces.port, Some(8080)); + assert_eq!( + b.product_info, + Some(("Test".to_string(), "1.0".to_string())) + ); + } + + #[test] + fn backoff_sequence_default() { + let seq = ClientBuilder::new("").get_backoff_sequence(); + + assert_eq!( + seq, + vec![0, 5, 10, 20, 40, 80, 160, 320] + .into_iter() + .map(|d| Duration::from_secs(d)) + .collect::>() + ); + } + + #[test] + fn backoff_sequence_no_retry() { + let seq = ClientBuilder::new("").retries_max(0).get_backoff_sequence(); + + assert_eq!(seq, vec![]); + } + + #[test] + fn backoff_sequence_custom() { + let seq = ClientBuilder::new("") + .backoff_factor(Duration::from_secs(2)) + .retries_max(6) + .get_backoff_sequence(); + + assert_eq!( + seq, + vec![0, 2, 4, 8, 16, 32] + .into_iter() + .map(|d| Duration::from_secs(d)) + .collect::>() + ); + } + + #[test] + fn user_agent_header_default() { + let header = ClientBuilder::new("").get_user_agent_header(); + + assert_eq!(header, format!("NewRelic-Rust-TelemetrySDK/{}", VERSION)); + } + + #[test] + fn user_agent_header_custom() { + let header = ClientBuilder::new("") + .product_info("Doc", "1.0") + .get_user_agent_header(); + + assert_eq!( + header, + format!("NewRelic-Rust-TelemetrySDK/{} Doc/1.0", VERSION) + ); + } +} diff --git a/vendor/newrelic-telemetry-sdk-rust/src/lib.rs b/vendor/newrelic-telemetry-sdk-rust/src/lib.rs new file mode 100644 index 0000000000..0cc8270150 --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/src/lib.rs @@ -0,0 +1,18 @@ +/// +/// Copyright 2020 New Relic Corporation. All rights reserved. +/// SPDX-License-Identifier: Apache-2.0 +/// +pub mod attribute; + +pub mod span; +pub use span::{Span, SpanBatch}; + +#[cfg(feature = "client")] +mod client; +#[cfg(feature = "client")] +pub use client::{Client, ClientBuilder}; + +#[cfg(feature = "blocking")] +pub mod blocking { + pub use super::client::blocking::Client; +} diff --git a/vendor/newrelic-telemetry-sdk-rust/src/span.rs b/vendor/newrelic-telemetry-sdk-rust/src/span.rs new file mode 100644 index 0000000000..29591866b9 --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/src/span.rs @@ -0,0 +1,569 @@ +/// +/// Copyright 2020 New Relic Corporation. All rights reserved. +/// SPDX-License-Identifier: Apache-2.0 +/// +use crate::attribute::Value; +use crate::client::Sendable; +use anyhow::Result; +use serde::{Serialize, Serializer}; +use std::collections::HashMap; +use std::fmt; +use std::time::Duration; +use uuid::Uuid; + +/// Represents a distributed tracing span. +#[derive(serde::Serialize, Clone, Debug, PartialEq)] +pub struct Span { + id: String, + + #[serde(rename = "trace.id")] + trace_id: String, + + timestamp: u64, + + #[serde(skip_serializing_if = "HashMap::is_empty")] + attributes: HashMap, +} + +impl Span { + /// Create a new span and assign an unique identifier, trace id and timestamp + pub fn new(id: &str, trace_id: &str, timestamp: u64) -> Span { + Span { + id: id.to_string(), + trace_id: trace_id.to_string(), + timestamp: timestamp, + attributes: HashMap::new(), + } + } + + /// Set a unique identifier for this span. This is a required field. + pub fn id(mut self, id: &str) -> Self { + self.id = id.to_string(); + self + } + + pub fn set_id(&mut self, id: &str) { + self.id = id.to_string(); + } + + /// Set a unique identifier shared by all spans within a single trace. + /// This is a required field. + pub fn trace_id(mut self, trace_id: &str) -> Self { + self.trace_id = trace_id.to_string(); + self + } + + pub fn set_trace_id(&mut self, trace_id: &str) { + self.trace_id = trace_id.to_string(); + } + + /// Set the start time of the span. This is a required field. + pub fn timestamp(mut self, timestamp: u64) -> Self { + self.timestamp = timestamp; + self + } + + pub fn set_timestamp(&mut self, timestamp: u64) { + self.timestamp = timestamp; + } + + /// Set the name of this span. + pub fn name(self, name: &str) -> Self { + self.attribute("name", name) + } + + pub fn set_name(&mut self, name: &str) { + self.set_attribute("name", name); + } + + /// Set the duration (in milliseconds) of this span. + pub fn duration(self, duration: Duration) -> Self { + self.attribute("duration.ms", duration.as_millis()) + } + + pub fn set_duration(&mut self, duration: Duration) { + self.set_attribute("duration.ms", duration.as_millis()); + } + + /// Set the id of the previous caller of this span. + pub fn parent_id(self, parent_id: &str) -> Self { + self.attribute("parent.id", parent_id) + } + + pub fn set_parent_id(&mut self, parent_id: &str) { + self.set_attribute("parent.id", parent_id); + } + + /// Set the name of the service that created this span. + pub fn service_name(self, service_name: &str) -> Self { + self.attribute("service.name", service_name) + } + + pub fn set_service_name(&mut self, service_name: &str) { + self.set_attribute("service.name", service_name); + } + + /// Set an attribute on the span. + pub fn attribute>(mut self, key: &str, value: T) -> Self { + self.attributes.insert(key.to_string(), value.into()); + self + } + + pub fn set_attribute>(&mut self, key: &str, value: T) { + self.attributes.insert(key.to_string(), value.into()); + } +} + +fn serialize_attributes(attrs: &HashMap, s: S) -> Result +where + S: Serializer, +{ + let mut wrapper: HashMap> = HashMap::new(); + wrapper.insert("attributes".to_string(), attrs); + wrapper.serialize(s) +} + +/// Encapsulates a collection of spans and the common data they share +#[derive(serde::Serialize, Debug, PartialEq)] +pub struct SpanBatch { + #[serde(skip_serializing)] + uuid: String, + + spans: Vec, + + #[serde(skip_serializing_if = "HashMap::is_empty")] + #[serde(serialize_with = "serialize_attributes")] + #[serde(rename = "common")] + attributes: HashMap, +} + +impl From> for SpanBatch { + /// Creates a new `SpanBatch` from a `Vec` + fn from(spans: Vec) -> Self { + let mut batch = Self::new(); + + for span in spans { + batch.record(span); + } + + batch + } +} + +impl SpanBatch { + /// Creates an empty `SpanBatch`. + pub fn new() -> Self { + SpanBatch { + uuid: Uuid::new_v4().to_string(), + spans: vec![], + attributes: HashMap::new(), + } + } + + /// Adds the provided span to the batch. + pub fn record(&mut self, span: Span) { + self.spans.push(span); + } + + /// Sets an attribute on the span batch. Returns `self` and can be chained + /// for concise addition of multiple attributes. + pub fn attribute>(mut self, key: &str, value: T) -> Self { + self.set_attribute(key, value); + self + } + + /// Sets an attribute on the span batch. + pub fn set_attribute>(&mut self, key: &str, value: T) { + self.attributes.insert(key.to_string(), value.into()); + } +} + +impl Sendable for SpanBatch { + fn uuid(&self) -> &str { + &self.uuid + } + + /// Returns the span batch encoded as a json string in the format expected + /// by the New Relic Telemetry API + fn marshall(&self) -> Result { + Ok(serde_json::to_string(&vec![self])?) + } + + /// Splits the batch in half. This is mostly used when the API service + /// returns a code indicating that the payload is too large. + fn split(&mut self) -> Box { + let new_batch_size: usize = self.spans.len() / 2; + self.uuid = Uuid::new_v4().to_string(); + + Box::new(SpanBatch { + uuid: Uuid::new_v4().to_string(), + spans: self.spans.drain(new_batch_size..).collect(), + attributes: self.attributes.clone(), + }) + } +} + +impl fmt::Display for SpanBatch { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "", + self.spans.len(), + self.attributes.len(), + ) + } +} + +#[cfg(test)] +mod tests { + use super::{Sendable, Span, SpanBatch}; + use crate::attribute::Value; + use anyhow::Result; + use serde_json::json; + use std::time::Duration; + + macro_rules! assert_json_eq { + ($x: expr, $y: expr) => { + let (left, right) = ($x, $y); + assert!( + serde_json::from_str::(left)? + == serde_json::from_str::(right)?, + "expected {}, got {}", + left, + right + ); + }; + } + + #[test] + fn span_set_id() { + let mut span = Span::new("id1", "traceId1", 1); + assert_eq!(span.id, "id1"); + + span.set_id("id2"); + assert_eq!(span.id, "id2"); + + span = span.id("id3"); + assert_eq!(span.id, "id3"); + } + + #[test] + fn span_set_trace_id() { + let mut span = Span::new("id1", "traceId1", 1); + assert_eq!(span.trace_id, "traceId1"); + + span.set_trace_id("traceId2"); + assert_eq!(span.trace_id, "traceId2"); + + span = span.trace_id("traceId3"); + assert_eq!(span.trace_id, "traceId3"); + } + + #[test] + fn span_set_timestamp() { + let mut span = Span::new("id1", "traceId1", 1); + assert_eq!(span.timestamp, 1); + + span.set_timestamp(2); + assert_eq!(span.timestamp, 2); + + span = span.timestamp(3); + assert_eq!(span.timestamp, 3); + } + + #[test] + fn span_to_json() { + // Check span JSON serialization with empty attribute hashmap. + let mut span = Span::new("id1", "traceId1", 1); + let json_span = json!({"id": "id1", "trace.id": "traceId1", "timestamp": 1}); + + assert_eq!(json!(span), json_span); + + // Check span JSON serialization with populated attribute hashmap. + span.set_name("I have a name"); + let json_span_attribute = json!({"id": "id1", "trace.id": "traceId1", "timestamp": 1, "attributes": {"name": "I have a name"}}); + + assert_eq!(json!(span), json_span_attribute); + } + + #[test] + fn span_attribute() { + let mut span = Span::new("id", "traceId", 1); + + // Test name attribute + span.set_name("name"); + assert_eq!( + span.attributes.get("name"), + Some(&Value::Str(String::from("name"))) + ); + + span = span.name("name2"); + assert_eq!( + span.attributes.get("name"), + Some(&Value::Str(String::from("name2"))) + ); + + // Test duration attribute + span.set_duration(Duration::from_millis(10)); + assert_eq!( + span.attributes.get("duration.ms"), + Some(&Value::UInt128(10)) + ); + + span = span.duration(Duration::from_millis(20)); + assert_eq!( + span.attributes.get("duration.ms"), + Some(&Value::UInt128(20)) + ); + + // Test parent id attribute + span.set_parent_id("parent"); + assert_eq!( + span.attributes.get("parent.id"), + Some(&Value::Str(String::from("parent"))) + ); + + span = span.parent_id("parent2"); + assert_eq!( + span.attributes.get("parent.id"), + Some(&Value::Str(String::from("parent2"))) + ); + + // Test service name attribute + span.set_service_name("serviceName"); + assert_eq!( + span.attributes.get("service.name"), + Some(&Value::Str(String::from("serviceName"))) + ); + + span = span.service_name("serviceName2"); + assert_eq!( + span.attributes.get("service.name"), + Some(&Value::Str(String::from("serviceName2"))) + ); + } + + #[test] + fn span_attribute_type() { + let mut span = Span::new("id", "traceId", 1); + + // Test String attribute + span.set_attribute("attr.str", "str"); + assert_eq!( + span.attributes.get("attr.str"), + Some(&Value::Str(String::from("str"))) + ); + + span = span.attribute("attr.str", "str2"); + assert_eq!( + span.attributes.get("attr.str"), + Some(&Value::Str(String::from("str2"))) + ); + + // Test UInt attribute + let val_u32: u32 = 5; + span.set_attribute("attr.uint", val_u32); + assert_eq!( + span.attributes.get("attr.uint"), + Some(&Value::UInt(val_u32 as u64)) + ); + + let val_u64: u64 = 42; + span = span.attribute("attr.uint", val_u64); + assert_eq!( + span.attributes.get("attr.uint"), + Some(&Value::UInt(val_u64)) + ); + + // Test Int attribute + let val_i32: i32 = -5; + span.set_attribute("attr.int", val_i32); + assert_eq!( + span.attributes.get("attr.int"), + Some(&Value::Int(val_i32 as i64)) + ); + + let val_i64: i64 = -42; + span = span.attribute("attr.int", val_i64); + assert_eq!(span.attributes.get("attr.int"), Some(&Value::Int(val_i64))); + + // Test Float attribute + let val_f32: f32 = 3.14; + span.set_attribute("attr.float", val_f32); + assert_eq!( + span.attributes.get("attr.float"), + Some(&Value::Float(val_f32 as f64)) + ); + + let val_f64: f64 = 6.28; + span = span.attribute("attr.float", val_f64); + assert_eq!( + span.attributes.get("attr.float"), + Some(&Value::Float(val_f64)) + ); + + // Test Bool attribute + span.set_attribute("attr.bool", true); + assert_eq!(span.attributes.get("attr.bool"), Some(&Value::Bool(true))); + + span = span.attribute("attr.bool", false); + assert_eq!(span.attributes.get("attr.bool"), Some(&Value::Bool(false))); + } + + /// Helper function to generate a simple SpanBatch + fn span_vec(count: usize) -> Vec { + let mut vec = Vec::new(); + + for n in 0..count { + let id = format!("id{}", n); + let trace_id = format!("trace_id{}", n); + vec.push(Span::new(id.as_str(), trace_id.as_str(), 1)) + } + + vec + } + + #[test] + fn spanbatch_split_partial() { + // Note: since SpanBatch::split() returns a Box, + // we cannot fully test split with regard to the returned + // SpanBatch, only that the originally was drained as expected + // However, the integration tests cover both sides of this case. + let mut batch = SpanBatch::from(span_vec(2)); + let uuid = batch.uuid().to_string(); + let second_batch = batch.split(); + + let second_uuid = second_batch.uuid(); + assert_eq!(batch.spans.len(), 1); + assert_eq!(batch.spans[0], Span::new("id0", "trace_id0", 1)); + + // confirm the uuid for the second batch is not the same as the first + // and that the first remains unchanged + assert_ne!(uuid, second_uuid); + assert_ne!(uuid, batch.uuid()); + } + + #[test] + fn spanbatch_to_json() -> Result<()> { + // Check span JSON serialization with empty attribute hashmap. + let batch = SpanBatch::from(span_vec(2)).attribute("attr.test", 3); + + // json! macro imposes a sort which is different from the serde-derive + // specified order, therefore a string is used + let expected_string = r#"[{"spans":[ + {"id":"id0","trace.id":"trace_id0","timestamp":1}, + {"id":"id1","trace.id":"trace_id1","timestamp":1}], + "common":{"attributes":{"attr.test":3}}}]"#; + + let marshalled = batch.marshall().unwrap(); + assert_json_eq!(marshalled.as_str(), expected_string); + Ok(()) + } + + #[test] + fn spanbatch_attribute_type() { + let mut batch = SpanBatch::new(); + + // Test String attribute + batch.set_attribute("attr.str", "str"); + assert_eq!( + batch.attributes.get("attr.str"), + Some(&Value::Str(String::from("str"))) + ); + + batch = batch.attribute("attr.str", "str2"); + assert_eq!( + batch.attributes.get("attr.str"), + Some(&Value::Str(String::from("str2"))) + ); + + // Test UInt attribute + let val_u32: u32 = 5; + batch.set_attribute("attr.uint", val_u32); + assert_eq!( + batch.attributes.get("attr.uint"), + Some(&Value::UInt(val_u32 as u64)) + ); + + let val_u64: u64 = 42; + batch = batch.attribute("attr.uint", val_u64); + assert_eq!( + batch.attributes.get("attr.uint"), + Some(&Value::UInt(val_u64)) + ); + + // Test Int attribute + let val_i32: i32 = -5; + batch.set_attribute("attr.int", val_i32); + assert_eq!( + batch.attributes.get("attr.int"), + Some(&Value::Int(val_i32 as i64)) + ); + + let val_i64: i64 = -42; + batch = batch.attribute("attr.int", val_i64); + assert_eq!(batch.attributes.get("attr.int"), Some(&Value::Int(val_i64))); + + // Test Float attribute + let val_f32: f32 = 3.14; + batch.set_attribute("attr.float", val_f32); + assert_eq!( + batch.attributes.get("attr.float"), + Some(&Value::Float(val_f32 as f64)) + ); + + let val_f64: f64 = 6.28; + batch = batch.attribute("attr.float", val_f64); + assert_eq!( + batch.attributes.get("attr.float"), + Some(&Value::Float(val_f64)) + ); + + // Test Bool attribute + batch.set_attribute("attr.bool", true); + assert_eq!(batch.attributes.get("attr.bool"), Some(&Value::Bool(true))); + + batch = batch.attribute("attr.bool", false); + assert_eq!(batch.attributes.get("attr.bool"), Some(&Value::Bool(false))); + } + + #[test] + fn spanbatch_from() { + let vec = span_vec(23); + let batch = SpanBatch::from(vec.clone()); + assert_eq!(batch.spans, vec); + assert_eq!(batch.spans.len(), 23); + } + + #[test] + fn spanbatch_record() { + let mut batch = SpanBatch::new(); + let span = Span::new("id0", "trace_id0", 9); + batch.record(span.clone()); + assert_eq!(batch.spans.len(), 1); + assert_eq!(batch.spans[0], span); + } + + #[test] + fn spanbatch_format() { + let batch = SpanBatch::from(span_vec(23)) + .attribute("one", 1) + .attribute("two", "too") + .attribute("three", 3.0); + + let batch_string = format!("{}", batch); + assert_eq!(batch_string, ""); + } + + #[test] + fn spanbatch_attribute_chain() -> Result<()> { + let batch = SpanBatch::new() + .attribute("bad_dogs", 0) + .attribute("howdy", "y'all"); + let expected_string = + r#"[{"spans":[],"common":{"attributes":{"bad_dogs":0,"howdy":"y'all"}}}]"#; + let marshalled = batch.marshall().unwrap(); + assert_json_eq!(marshalled.as_str(), expected_string); + Ok(()) + } +} diff --git a/vendor/newrelic-telemetry-sdk-rust/tests/blocking.rs b/vendor/newrelic-telemetry-sdk-rust/tests/blocking.rs new file mode 100644 index 0000000000..214851955a --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/tests/blocking.rs @@ -0,0 +1,252 @@ +/// +/// Copyright 2020 New Relic Corporation. All rights reserved. +/// SPDX-License-Identifier: Apache-2.0 +/// +#[cfg(feature = "blocking")] +#[macro_use] +mod common; + +#[cfg(feature = "blocking")] +mod blocking { + use super::common; + use anyhow::Result; + use common::Endpoint; + use newrelic_telemetry::{blocking::Client, ClientBuilder, Span, SpanBatch}; + use std::thread; + use std::time::Duration; + + pub fn setup() -> Result<(Endpoint, Client)> { + let _ = env_logger::builder().is_test(true).try_init(); + + let endpoint = Endpoint::new(); + let client = ClientBuilder::new(&endpoint.license) + .endpoint_traces(&endpoint.host, Some(endpoint.port)) + .tls(false) + .build_blocking()?; + + Ok((endpoint, client)) + } + + #[test] + fn backoff() -> Result<()> { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut endpoint = Endpoint::new(); + thread::sleep(Duration::from_secs(2)); + let client = ClientBuilder::new(&endpoint.license) + .retries_max(3) + .backoff_factor(Duration::from_secs(0)) + .endpoint_traces(&endpoint.host, Some(endpoint.port)) + .tls(false) + .build_blocking()?; + + let span_batch = SpanBatch::new(); + + client.send_spans(span_batch); + + // Four payloads should be sent: the initial one and 3 retries. + for num in 1..4 { + endpoint.reply(500)?; + assert!(endpoint.next_payload().is_ok(), "receiving payload {}", num); + } + + assert!(endpoint.next_payload().is_err(), "dropping payload"); + + Ok(()) + } + + #[test] + fn retry_after() -> Result<()> { + let _ = env_logger::builder() + .is_test(true) + .parse_filters("debug") + .try_init(); + + let mut endpoint = Endpoint::new(); + let client = ClientBuilder::new(&endpoint.license) + .retries_max(5) + .backoff_factor(Duration::from_secs(3600)) + .endpoint_traces(&endpoint.host, Some(endpoint.port)) + .tls(false) + .build_blocking()?; + + let span_batch = SpanBatch::new(); + + client.send_spans(span_batch); + + // Six payloads should be sent: the initial one and 5 retries. + // + // If the retry-after header is not read, this should hang for a very long time. + for num in 1..6 { + endpoint.reply_details( + 429, + vec![("Retry-After".to_string(), "0".to_string())], + "{}", + )?; + assert!(endpoint.next_payload().is_ok(), "receiving payload {}", num); + } + + assert!(endpoint.next_payload().is_err(), "dropping payload"); + + Ok(()) + } + + #[test] + fn headers() -> Result<()> { + let (mut endpoint, client) = setup()?; + + let span_batch = SpanBatch::new(); + + client.send_spans(span_batch); + endpoint.reply(202)?; + + let p = endpoint.next_payload()?; + + assert_eq!( + p.headers.get("content-type"), + Some(&"application/json".to_string()) + ); + assert_eq!(p.headers.get("content-encoding"), Some(&"gzip".to_string())); + + assert_eq!(p.headers.get("data-format"), Some(&"newrelic".to_string())); + assert_eq!(p.headers.get("data-format-version"), Some(&"1".to_string())); + + let user_agent = format!("NewRelic-Rust-TelemetrySDK/{}", env!("CARGO_PKG_VERSION")); + assert_eq!(p.headers.get("user-agent"), Some(&user_agent)); + + Ok(()) + } + + #[test] + fn product() -> Result<()> { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut endpoint = Endpoint::new(); + let client = ClientBuilder::new(&endpoint.license) + .endpoint_traces(&endpoint.host, Some(endpoint.port)) + .product_info("SomeProduct", "3.14.9") + .tls(false) + .build_blocking()?; + + let span_batch = SpanBatch::new(); + + client.send_spans(span_batch); + endpoint.reply(202)?; + + let p = endpoint.next_payload()?; + + let user_agent = format!( + "NewRelic-Rust-TelemetrySDK/{} SomeProduct/3.14.9", + env!("CARGO_PKG_VERSION") + ); + assert_eq!(p.headers.get("user-agent"), Some(&user_agent)); + + Ok(()) + } + + #[test] + fn drop_payload() -> Result<()> { + for code in vec![400, 401, 403, 404, 405, 409, 410, 411] { + let (mut endpoint, client) = setup()?; + + let span_batch = SpanBatch::new(); + + client.send_spans(span_batch); + endpoint.reply(code)?; + + assert!(endpoint.next_payload().is_ok(), "first attempt to send"); + assert!(endpoint.next_payload().is_err(), "payload dropped"); + } + + Ok(()) + } + + #[test] + fn backpressure() -> Result<()> { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut endpoint = Endpoint::new(); + let client = ClientBuilder::new(&endpoint.license) + .endpoint_traces(&endpoint.host, Some(endpoint.port)) + .tls(false) + .blocking_queue_max(1) + .build_blocking()?; + + for _ in 0..10 { + client.send_spans(SpanBatch::new()); + } + + endpoint.reply(202)?; + assert!(endpoint.next_payload().is_ok(), "first batch sent"); + + assert!( + endpoint.next_payload().is_err(), + "additional batches dropped" + ); + + Ok(()) + } + + #[test] + fn split_payload() -> Result<()> { + let (mut endpoint, client) = setup()?; + + let mut span_batch = SpanBatch::new(); + + let span_batch = vec![ + Span::new("id1", "tid1", 1000), + Span::new("id2", "tid2", 2000), + Span::new("id1", "tid1", 1000), + Span::new("id2", "tid2", 2000), + ] + .into(); + + client.send_spans(span_batch); + endpoint.reply(413)?; + endpoint.reply(202)?; + endpoint.reply(202)?; + + assert_json_eq!( + &endpoint.next_payload()?.body, + r#" + [{ + "spans": [ + { + "id": "id1", + "timestamp": 1000, + "trace.id": "tid1" + }, + { + "id": "id2", + "timestamp": 2000, + "trace.id": "tid2" + } + ] + }]"# + ); + + assert_json_eq!( + &endpoint.next_payload()?.body, + r#" + [{ + "spans": [ + { + "id": "id1", + "timestamp": 1000, + "trace.id": "tid1" + }, + { + "id": "id2", + "timestamp": 2000, + "trace.id": "tid2" + } + ] + }]"# + ); + + // Skip the first payload that is rejected. + endpoint.next_payload()?.body; + + Ok(()) + } +} diff --git a/vendor/newrelic-telemetry-sdk-rust/tests/common/mod.rs b/vendor/newrelic-telemetry-sdk-rust/tests/common/mod.rs new file mode 100644 index 0000000000..cea9ac979a --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/tests/common/mod.rs @@ -0,0 +1,250 @@ +/// +/// Copyright 2020 New Relic Corporation. All rights reserved. +/// SPDX-License-Identifier: Apache-2.0 +/// +use flate2::read::GzDecoder; +use hyper::service::{make_service_fn, service_fn}; +use hyper::{Body, Request, Server}; +use rand::seq::SliceRandom; +use rand::thread_rng; +use std::collections::HashMap; +use std::io::Read; +use std::net::{SocketAddr, TcpListener}; +use std::str; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::SystemTime; +use tokio::runtime::Builder; + +macro_rules! assert_json_eq { + ($x: expr, $y: expr) => { + let (left, right) = ($x, $y); + assert!( + serde_json::from_str::(left)? + == serde_json::from_str::(right)?, + "expected {}, got {}", + left, + right + ); + }; +} + +#[derive(Debug)] +pub struct Error { + message: String, +} + +impl Error { + fn new(message: &str) -> Error { + Error { + message: message.to_string(), + } + } +} + +impl std::error::Error for Error {} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +impl From for Error { + fn from(error: std::str::Utf8Error) -> Self { + Error::new(&error.to_string()) + } +} + +impl From for Error { + fn from(error: hyper::error::Error) -> Self { + Error::new(&error.to_string()) + } +} + +pub struct Payload { + pub headers: HashMap, + pub body: String, +} + +pub struct Response { + pub code: u16, + pub headers: Vec<(String, String)>, + pub body: String, +} + +pub struct Endpoint { + pub license: String, + pub host: String, + pub port: u16, + pub timeout_ms: u128, + server: Option>, + chan_payloads: Arc>>, + chan_responses: Arc>>, + chan_stop: Option>, +} + +impl Endpoint { + pub fn new() -> Self { + let payloads = Arc::new(Mutex::new(Vec::::new())); + let responses = Arc::new(Mutex::new(Vec::::new())); + + // Three rounds of clones are required to get the Arcs through all the + // closure bounds. + let (p, r) = (payloads.clone(), responses.clone()); + let new_service = make_service_fn(move |_conn| { + let (p, r) = (p.clone(), r.clone()); + async { + Ok::<_, hyper::Error>(service_fn(move |req| { + let (p, r) = (p.clone(), r.clone()); + Endpoint::accept_payload(req, p, r) + })) + } + }); + + let mut runtime = Builder::new() + .threaded_scheduler() + .enable_all() + .build() + .unwrap(); + + let (sender, receiver) = futures::channel::oneshot::channel::<()>(); + + let wrapped_receiver = async { + let _ = receiver.await; + }; + + let port = Endpoint::get_available_port().expect("Cannot aquire port"); + + let handle = thread::spawn(move || { + let addr: SocketAddr = ([127, 0, 0, 1], port).into(); + + let scope = async { + Server::bind(&addr) + .http1_half_close(true) + .serve(new_service) + .with_graceful_shutdown(wrapped_receiver) + .await + .unwrap(); + }; + + runtime.block_on(scope); + }); + + Endpoint { + license: "license".to_string(), + host: "127.0.0.1".to_string(), + port: port, + timeout_ms: 5000, + server: Some(handle), + chan_payloads: payloads, + chan_responses: responses, + chan_stop: Some(sender), + } + } + + pub fn reply(&self, code: u16) -> Result<(), Error> { + self.reply_details(code, vec![], "{}") + } + + pub fn reply_details( + &self, + code: u16, + headers: Vec<(String, String)>, + body: &str, + ) -> Result<(), Error> { + let mut lock = self.chan_responses.lock().unwrap(); + lock.push(Response { + code: code, + headers: headers, + body: body.to_string(), + }); + let len = lock.len(); + drop(lock); + + // Block until the response was consumed. + let start = SystemTime::now(); + loop { + let lock = self.chan_responses.lock().unwrap(); + if lock.len() < len { + return Ok(()); + } + + let duration = SystemTime::now().duration_since(start).unwrap(); + if duration.as_millis() > self.timeout_ms { + return Err(Error::new("Timeout: no request received")); + } + } + } + + pub fn next_payload(&mut self) -> Result { + let mut lock = self.chan_payloads.lock().unwrap(); + + match lock.pop() { + Some(p) => Ok(p), + None => Err(Error::new("No payload received")), + } + } + + async fn accept_payload( + req: Request, + payloads: Arc>>, + responses: Arc>>, + ) -> Result, Error> { + let headers: HashMap = req + .headers() + .iter() + .map(|(key, value)| (key.to_string(), value.to_str().unwrap_or("").to_string())) + .collect(); + + let body = hyper::body::to_bytes(req.into_body()).await?; + let body_bytes: Result, _> = body.bytes().collect(); + let body_bytes = body_bytes.unwrap(); + + let mut decoder = GzDecoder::new(&body_bytes[..]); + let mut body_decoded = String::new(); + let _ = decoder.read_to_string(&mut body_decoded); + + let mut lock = payloads.lock().unwrap(); + lock.push(Payload { + headers: headers, + body: body_decoded, + }); + + let mut lock = responses.lock().unwrap(); + match lock.pop() { + Some(r) => { + let mut resp = hyper::Response::builder(); + + for (k, v) in &r.headers { + resp = resp.header(k, v); + } + + Ok(resp.status(r.code).body(Body::from(r.body)).unwrap()) + } + _ => Err(Error::new("No response given")), + } + } + + fn get_available_port() -> Option { + fn port_is_available(p: u16) -> bool { + match TcpListener::bind(("127.0.0.1", p)) { + Ok(_) => true, + _ => false, + } + } + + let mut ports: Vec = (3000..5000).collect(); + ports.shuffle(&mut thread_rng()); + + ports.into_iter().find(|p| port_is_available(*p)) + } +} + +impl Drop for Endpoint { + fn drop(&mut self) { + if let Ok(_) = self.chan_stop.take().unwrap().send(()) { + let _ = self.server.take().unwrap().join(); + } + } +} diff --git a/vendor/newrelic-telemetry-sdk-rust/tests/spans.rs b/vendor/newrelic-telemetry-sdk-rust/tests/spans.rs new file mode 100644 index 0000000000..7bdd98e354 --- /dev/null +++ b/vendor/newrelic-telemetry-sdk-rust/tests/spans.rs @@ -0,0 +1,219 @@ +/// +/// Copyright 2020 New Relic Corporation. All rights reserved. +/// SPDX-License-Identifier: Apache-2.0 +/// +#[cfg(feature = "client")] +#[macro_use] +mod common; + +#[cfg(feature = "client")] +mod client { + use super::common; + use anyhow::Result; + use common::Endpoint; + use newrelic_telemetry::{Client, ClientBuilder, Span, SpanBatch}; + use std::thread; + use std::time::Duration; + + pub fn setup() -> Result<(Endpoint, Client)> { + let _ = env_logger::builder().is_test(true).try_init(); + + let endpoint = Endpoint::new(); + let client = ClientBuilder::new(&endpoint.license) + .endpoint_traces(&endpoint.host, Some(endpoint.port)) + .tls(false) + .build()?; + + Ok((endpoint, client)) + } + + #[tokio::test(threaded_scheduler)] + async fn empty() -> Result<()> { + let (mut endpoint, client) = setup()?; + + // Assertions are all handled in a separate thread, so we can await the + // future in the main thread. + let handle = thread::spawn(move || -> Result<()> { + endpoint.reply(202)?; + + assert_json_eq!( + &endpoint.next_payload().unwrap().body, + r#"[{ "spans" : [] }]"# + ); + + Ok(()) + }); + + client.send_spans(SpanBatch::new()).await; + + handle.join().expect("error from endpoint thread")?; + + Ok(()) + } + + #[tokio::test(threaded_scheduler)] + async fn simple() -> Result<()> { + let (mut endpoint, client) = setup()?; + + let handle = thread::spawn(move || -> Result<()> { + endpoint.reply(202)?; + + assert_json_eq!( + &endpoint.next_payload()?.body, + r#" + [ + { + "spans": [ + { + "id": "id1", + "timestamp": 1000, + "trace.id": "tid1" + } + ] + } + ]"# + ); + + Ok(()) + }); + + let mut span_batch = SpanBatch::new(); + span_batch.record(Span::new("id1", "tid1", 1000)); + + client.send_spans(span_batch).await; + handle.join().expect("error from endpoint thread")?; + + Ok(()) + } + + #[tokio::test(threaded_scheduler)] + async fn all_api_attrs() -> Result<()> { + let (mut endpoint, client) = setup()?; + + let handle = thread::spawn(move || -> Result<()> { + endpoint.reply(202)?; + + assert_json_eq!( + &endpoint.next_payload()?.body, + r#" + [{ + "spans": [{ + "id": "id1", + "timestamp": 1000, + "trace.id": "tid1", + "attributes": { + "name": "name1", + "duration.ms": 2000, + "parent.id": "pid1", + "service.name": "service1" + } + }] + }]"# + ); + + Ok(()) + }); + + let mut span_batch = SpanBatch::new(); + span_batch.record( + Span::new("id1", "tid1", 1000) + .name("name1") + .duration(Duration::from_millis(2000)) + .parent_id("pid1") + .service_name("service1"), + ); + + client.send_spans(span_batch).await; + handle.join().expect("error from endpoint thread")?; + + Ok(()) + } + + #[tokio::test(threaded_scheduler)] + async fn custom_attrs() -> Result<()> { + let (mut endpoint, client) = setup()?; + + let handle = thread::spawn(move || -> Result<()> { + endpoint.reply(202)?; + + assert_json_eq!( + &endpoint.next_payload()?.body, + r#" + [{ + "spans": [{ + "id": "id1", + "timestamp": 1000, + "trace.id": "tid1", + "attributes": { + "bool_attr": true, + "float_attr": 3.14159, + "str_attr": "string", + "int_attr": 40, + "neg_int_attr": -40, + "string_attr": "Str" + } + }] + }]"# + ); + + Ok(()) + }); + + let mut span_batch = SpanBatch::new(); + span_batch.record( + Span::new("id1", "tid1", 1000) + .attribute("bool_attr", true) + .attribute("float_attr", 3.14159) + .attribute("str_attr", "string") + .attribute("int_attr", 40) + .attribute("neg_int_attr", -40) + .attribute("string_attr", "Str"), + ); + + client.send_spans(span_batch).await; + handle.join().expect("error from endpoint thread")?; + + Ok(()) + } + + #[tokio::test(threaded_scheduler)] + async fn two_spans() -> Result<()> { + let (mut endpoint, client) = setup()?; + + let handle = thread::spawn(move || -> Result<()> { + endpoint.reply(202)?; + + assert_json_eq!( + &endpoint.next_payload()?.body, + r#" + [{ + "spans": [ + { + "id": "id1", + "timestamp": 1000, + "trace.id": "tid1" + }, + { + "id": "id2", + "timestamp": 2000, + "trace.id": "tid2" + } + ] + }]"# + ); + + Ok(()) + }); + + let span_batch = vec![ + Span::new("id1", "tid1", 1000), + Span::new("id2", "tid2", 2000), + ] + .into(); + + client.send_spans(span_batch).await; + handle.join().expect("error from endpoint thread")?; + + Ok(()) + } +} From 58af7c52c8664ecbdcb3b1ac0f7ede29f342a576 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 13 Sep 2021 05:46:18 -0400 Subject: [PATCH 0622/1042] don't spam newrelic --- src/tts_inference_job.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index c79a4f28d1..f3b3d73db8 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -369,8 +369,7 @@ async fn main_loop(inferencer: Inferencer) { span_batch.append(&mut spans); - if span_batch.len() > 1 { - //let spans_to_send = span_batch.drain(..).collect(); + if span_batch.len() > 50 { let spans_to_send = span_batch.split_off(0).into(); inferencer.newrelic_client.send_spans(spans_to_send).await; } From ef4a25ee6e3bdd2ad32c30d33b248407d2db6051 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 13 Sep 2021 06:03:40 -0400 Subject: [PATCH 0623/1042] freaking dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 42cf3a7527..3d4660b581 100644 --- a/Dockerfile +++ b/Dockerfile @@ -153,6 +153,7 @@ COPY Cargo.lock . COPY Cargo.toml . COPY sqlx-data.json . COPY src/ ./src +COPY vendor/ ./vendor COPY db/ ./db COPY migrations/ ./migrations From d615a0593088c9c096738cf983b5731b488c6dc7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 14 Sep 2021 23:57:46 -0400 Subject: [PATCH 0624/1042] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 9500af898e..d1b20f5ca4 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,10 @@ Notes / TODOs: * Actix/sqlx runtime compat: https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734 + +* Redis caching + +* Jobs for analytics queries Docker builds From d29990b877dca654b38ecb13c46f18abeb3c2f7c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 17 Sep 2021 02:06:53 -0400 Subject: [PATCH 0625/1042] rename endpoint directory --- .../endpoints/moderation/{pending => approval}/mod.rs | 0 .../moderation/{pending => approval}/pending_w2l_templates.rs | 0 src/http_server/endpoints/moderation/mod.rs | 2 +- src/main.rs | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename src/http_server/endpoints/moderation/{pending => approval}/mod.rs (100%) rename src/http_server/endpoints/moderation/{pending => approval}/pending_w2l_templates.rs (100%) diff --git a/src/http_server/endpoints/moderation/pending/mod.rs b/src/http_server/endpoints/moderation/approval/mod.rs similarity index 100% rename from src/http_server/endpoints/moderation/pending/mod.rs rename to src/http_server/endpoints/moderation/approval/mod.rs diff --git a/src/http_server/endpoints/moderation/pending/pending_w2l_templates.rs b/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs similarity index 100% rename from src/http_server/endpoints/moderation/pending/pending_w2l_templates.rs rename to src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs diff --git a/src/http_server/endpoints/moderation/mod.rs b/src/http_server/endpoints/moderation/mod.rs index bb4c0313a2..71ea71de02 100644 --- a/src/http_server/endpoints/moderation/mod.rs +++ b/src/http_server/endpoints/moderation/mod.rs @@ -1,6 +1,6 @@ +pub mod approval; pub mod ip_bans; pub mod jobs; -pub mod pending; pub mod user_bans; pub mod user_roles; pub mod users; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 092c7fcaaf..51a0c13297 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,13 +39,13 @@ use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha_easy_handler; use crate::http_server::endpoints::misc::enable_alpha_handler::enable_alpha_handler; +use crate::http_server::endpoints::moderation::approval::pending_w2l_templates::get_pending_w2l_templates_handler; use crate::http_server::endpoints::moderation::ip_bans::add_ip_ban::add_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::delete_ip_ban::delete_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::list_ip_bans::list_ip_bans_handler; use crate::http_server::endpoints::moderation::jobs::get_tts_inference_queue_count::get_tts_inference_queue_count_handler; use crate::http_server::endpoints::moderation::jobs::get_w2l_inference_queue_count::get_w2l_inference_queue_count_handler; -use crate::http_server::endpoints::moderation::pending::pending_w2l_templates::get_pending_w2l_templates_handler; use crate::http_server::endpoints::moderation::user_bans::ban_user::ban_user_handler; use crate::http_server::endpoints::moderation::user_bans::list_banned_users::list_banned_users_handler; use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user_roles_handler; From fb320f0338fc9d9b5e6ba3993989e39fd310903f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 17 Sep 2021 02:50:14 -0400 Subject: [PATCH 0626/1042] endpoint to count voices --- src/http_server/endpoints/moderation/mod.rs | 1 + .../moderation/stats/get_voice_count_stats.rs | 137 ++++++++++++++++++ .../endpoints/moderation/stats/mod.rs | 1 + src/main.rs | 9 ++ 4 files changed, 148 insertions(+) create mode 100644 src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs create mode 100644 src/http_server/endpoints/moderation/stats/mod.rs diff --git a/src/http_server/endpoints/moderation/mod.rs b/src/http_server/endpoints/moderation/mod.rs index 71ea71de02..afd7a69415 100644 --- a/src/http_server/endpoints/moderation/mod.rs +++ b/src/http_server/endpoints/moderation/mod.rs @@ -1,6 +1,7 @@ pub mod approval; pub mod ip_bans; pub mod jobs; +pub mod stats; pub mod user_bans; pub mod user_roles; pub mod users; \ No newline at end of file diff --git a/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs b/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs new file mode 100644 index 0000000000..dc2c06880a --- /dev/null +++ b/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs @@ -0,0 +1,137 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; +use crate::server_state::ServerState; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +#[derive(Serialize)] +pub struct GetVoiceCountStatsResponse { + pub success: bool, + pub all_voices_count_including_deleted: i64, + pub public_voices_count: i64, +} + +#[derive(Debug, Serialize)] +pub enum GetVoiceCountStatsError { + ServerError, + Unauthorized, +} + +impl ResponseError for GetVoiceCountStatsError { + fn status_code(&self) -> StatusCode { + match *self { + GetVoiceCountStatsError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + GetVoiceCountStatsError::Unauthorized => StatusCode::UNAUTHORIZED, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for GetVoiceCountStatsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn get_voice_count_stats_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetVoiceCountStatsError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(GetVoiceCountStatsError::Unauthorized); + } + }; + + // TODO: Not a good fit for this permission. + if !user_session.can_edit_other_users_tts_models { + warn!("user is not allowed to edit user tts: {}", user_session.user_token); + return Err(GetVoiceCountStatsError::Unauthorized); + } + + // NB: Lookup failure is Err(RowNotFound). + let maybe_result = sqlx::query_as!( + VoiceStats, + r#" +SELECT public_count, all_count FROM ( + (SELECT count(*) as public_count + FROM tts_models + WHERE user_deleted_at IS NULL + AND mod_deleted_at IS NULL) as pc, + (SELECT count(*) as all_count + FROM tts_models) as ac +) + "#, + ) + .fetch_one(&server_state.mysql_pool) + .await; + + let result : VoiceStats = match maybe_result { + Ok(result) => result, + Err(err) => { + match err { + RowNotFound => { + // NB: Not Found for null results means nothing is pending in the queue + VoiceStats { + all_count: -1, + public_count: -1, + } + }, + _ => { + warn!("get tts pending count error: {:?}", err); + return Err(GetVoiceCountStatsError::ServerError) + } + } + }, + }; + + let response = GetVoiceCountStatsResponse { + success: true, + all_voices_count_including_deleted: result.all_count, + public_voices_count: result.public_count, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetVoiceCountStatsError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} + +#[derive(Serialize)] +pub struct VoiceStats { + pub public_count: i64, + pub all_count: i64, +} diff --git a/src/http_server/endpoints/moderation/stats/mod.rs b/src/http_server/endpoints/moderation/stats/mod.rs new file mode 100644 index 0000000000..a2441d1780 --- /dev/null +++ b/src/http_server/endpoints/moderation/stats/mod.rs @@ -0,0 +1 @@ +pub mod get_voice_count_stats; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 51a0c13297..6c68977882 100644 --- a/src/main.rs +++ b/src/main.rs @@ -46,6 +46,7 @@ use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_h use crate::http_server::endpoints::moderation::ip_bans::list_ip_bans::list_ip_bans_handler; use crate::http_server::endpoints::moderation::jobs::get_tts_inference_queue_count::get_tts_inference_queue_count_handler; use crate::http_server::endpoints::moderation::jobs::get_w2l_inference_queue_count::get_w2l_inference_queue_count_handler; +use crate::http_server::endpoints::moderation::stats::get_voice_count_stats::get_voice_count_stats_handler; use crate::http_server::endpoints::moderation::user_bans::ban_user::ban_user_handler; use crate::http_server::endpoints::moderation::user_bans::list_banned_users::list_banned_users_handler; use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user_roles_handler; @@ -462,6 +463,14 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service( + web::scope("/stats") + .service( + web::resource("/tts_voices") + .route(web::get().to(get_voice_count_stats_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) ) // ==================== TTS ==================== .service( From e3fd37725713eb7afc8315436c92506e6dbaa543 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 17 Sep 2021 02:58:32 -0400 Subject: [PATCH 0627/1042] codegen db for new endpoint --- sqlx-data.json | 128 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 83 insertions(+), 45 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 7c57b1c1b0..ff285fd464 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -133,7 +133,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -145,7 +145,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -157,7 +157,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -460,7 +460,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -472,7 +472,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -484,7 +484,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -786,7 +786,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -1000,7 +1000,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -1012,7 +1012,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -1024,7 +1024,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -1848,6 +1848,44 @@ "nullable": [] } }, + "1d0f302c0b7c4301000c024d6bbae4d3b7370ed7c944e4c557c64b0bd2f31c14": { + "query": "\nSELECT public_count, all_count FROM (\n (SELECT count(*) as public_count \n FROM tts_models \n WHERE user_deleted_at IS NULL \n AND mod_deleted_at IS NULL) as pc,\n (SELECT count(*) as all_count \n FROM tts_models) as ac\n)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "public_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 21 + } + }, + { + "ordinal": 1, + "name": "all_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false + ] + } + }, "25e7858d370ac1359deb42ef5292e85ccd1fe7debf3a1ba73303e5882523a4d0": { "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n download_url = ?,\n status = \"pending\"\n ", "describe": { @@ -2019,7 +2057,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -2105,7 +2143,7 @@ "bits": 129 }, "char_set": 63, - "max_size": 2 + "max_size": 1 } } ], @@ -2292,7 +2330,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -2304,7 +2342,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -2316,7 +2354,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -3161,7 +3199,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -3173,7 +3211,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -3185,7 +3223,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -3197,7 +3235,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -3341,7 +3379,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -3431,7 +3469,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -3802,7 +3840,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -3814,7 +3852,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -3826,7 +3864,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -4821,7 +4859,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -4981,7 +5019,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -4993,7 +5031,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -5005,7 +5043,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -5650,7 +5688,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -6094,7 +6132,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -6106,7 +6144,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -6357,7 +6395,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -6659,7 +6697,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -6671,7 +6709,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -6683,7 +6721,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -6695,7 +6733,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -6877,7 +6915,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -6889,7 +6927,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -6901,7 +6939,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -7555,7 +7593,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -7567,7 +7605,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -8239,7 +8277,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -8251,7 +8289,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -8263,7 +8301,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -8883,7 +8921,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { From 40b35fb642b642d3ec3f84d43b3c94041b06677d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 19 Sep 2021 19:33:57 -0400 Subject: [PATCH 0628/1042] warn to error level log --- src/tts_inference_job.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index f3b3d73db8..03e182a1c5 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -53,7 +53,7 @@ use crate::util::random_crockford_token::random_crockford_token; use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; -use log::{warn, info}; +use log::{warn, info, error}; use newrelic_telemetry::Client as NewRelicClient; use newrelic_telemetry::ClientBuilder; use newrelic_telemetry::Span; @@ -508,7 +508,7 @@ impl ModelState { let model_record = match query_result { Some(model) => model, None => { - warn!("TTS model not found: {}", &job.model_token); + error!("TTS model not found: {}", &job.model_token); return Err(anyhow!("Model not found!")) }, }; @@ -852,7 +852,7 @@ async fn process_job( &inference_result_token) .await .map_err(|e| { - warn!("error publishing event: {:?}", e); + error!("error publishing event: {:?}", e); anyhow!("error publishing event") })?; From b9e4087e3ea4df4ec652f4ceb7aa73c1d2175964 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 19 Sep 2021 20:20:06 -0400 Subject: [PATCH 0629/1042] mark jobs permanently dead if models are gone --- src/job_queries/tts_inference_job_queries.rs | 76 +++++++++++-- src/tts_inference_job.rs | 109 +++++++++++++++---- 2 files changed, 151 insertions(+), 34 deletions(-) diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 63ff083d13..55f17f8efa 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -11,6 +11,7 @@ use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn, info}; use sqlx::{MySqlPool, Transaction, MySql}; +use std::error; use std::path::Path; /// table: tts_inference_jobs @@ -192,6 +193,30 @@ WHERE id = ? Ok(()) } + +pub async fn mark_tts_inference_job_permanently_dead( + pool: &MySqlPool, + job: &TtsInferenceJobRecord, + failure_reason: &str, +) -> AnyhowResult<()> { + let query_result = sqlx::query!( + r#" +UPDATE tts_inference_jobs +SET + status = "dead", + failure_reason = ?, + retry_at = NULL +WHERE id = ? + "#, + failure_reason.to_string(), + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} + pub async fn mark_tts_inference_job_done( pool: &MySqlPool, job: &TtsInferenceJobRecord, @@ -374,7 +399,7 @@ SET Ok((record_id, inference_result_token.clone())) } -pub struct TtsModelRecord2 { +pub struct TtsModelForInferenceRecord { pub model_token: String, pub tts_model_type: String, @@ -388,17 +413,40 @@ pub struct TtsModelRecord2 { pub private_bucket_hash: String, pub created_at: DateTime, pub updated_at: DateTime, + pub mod_deleted_at: Option>, + pub user_deleted_at: Option>, } + +#[derive(Clone, Debug)] +pub enum TtsModelForInferenceError { + ModelNotFound, + ModelDeleted, + DatabaseError { reason: String }, +} + +impl std::fmt::Display for TtsModelForInferenceError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TtsModelForInferenceError::ModelNotFound => write!(f, "ModelNotFound"), + TtsModelForInferenceError::ModelDeleted => write!(f, "ModelDeleted"), + TtsModelForInferenceError::DatabaseError { reason} => + write!(f, "Database error: {:?}", reason), + } + } +} + +impl std::error::Error for TtsModelForInferenceError {} + pub async fn get_tts_model_by_token( pool: &MySqlPool, model_token: &str -) -> AnyhowResult> +) -> Result { // NB: Lookup failure is Err(RowNotFound). // NB: Since this is publicly exposed, we don't query sensitive data. let maybe_model = sqlx::query_as!( - TtsModelRecord2, + TtsModelForInferenceRecord, r#" SELECT tts.token as model_token, @@ -410,31 +458,39 @@ SELECT tts.title, tts.private_bucket_hash, tts.created_at, - tts.updated_at + tts.updated_at, + tts.user_deleted_at, + tts.mod_deleted_at FROM tts_models as tts JOIN users ON users.token = tts.creator_user_token WHERE tts.token = ? -AND tts.user_deleted_at IS NULL -AND tts.mod_deleted_at IS NULL "#, &model_token ) .fetch_one(pool) .await; // TODO: This will return error if it doesn't exist - match maybe_model { - Ok(model) => Ok(Some(model)), + let model : TtsModelForInferenceRecord = match maybe_model { + Ok(model) => model, Err(err) => { match err { RowNotFound => { - Ok(None) + return Err(TtsModelForInferenceError::ModelNotFound); }, _ => { warn!("tts model query error: {:?}", err); - Err(anyhow!("Mysql error: {:?}", err)) + return Err(TtsModelForInferenceError::DatabaseError { + reason: format!("Mysql error: {:?}", err) + }); } } } + }; + + if model.mod_deleted_at.is_some() || model.user_deleted_at.is_some() { + return Err(TtsModelForInferenceError::ModelDeleted); } + + Ok(model) } diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 03e182a1c5..5f67dfd22b 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -24,13 +24,16 @@ use crate::common_env::CommonEnv; use crate::database::enums::vocoder_type::VocoderType; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; +use crate::job_queries::tts_inference_job_queries::TtsInferenceJobRecord; +use crate::job_queries::tts_inference_job_queries::TtsModelForInferenceError; +use crate::job_queries::tts_inference_job_queries::TtsModelForInferenceRecord; use crate::job_queries::tts_inference_job_queries::get_tts_model_by_token; use crate::job_queries::tts_inference_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::tts_inference_job_queries::insert_tts_result; use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_done; use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_failure; +use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_permanently_dead; use crate::job_queries::tts_inference_job_queries::query_tts_inference_job_records; -use crate::job_queries::tts_inference_job_queries::{TtsInferenceJobRecord, TtsModelRecord2}; use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use crate::shared_constants::DEFAULT_RUST_LOG; @@ -399,15 +402,40 @@ async fn process_jobs( let model_state = match model_state_result { Ok(model_state) => model_state, Err(e) => { - warn!("Failure to check model state: {:?}", e); + error!("TTS model fetch and state check error: {}, reason: {:?}", &job.model_token, &e); + + let (failure_reason, permanent_failure) = match e { + ModelStateError::ModelNotFound => ("model was not found", true), + ModelStateError::ModelDeleted => ("model has been deleted", true), + ModelStateError::CacheError { .. } => ("internal cache error", false), + ModelStateError::DatabaseError { .. } => ("unknown database error", false), + }; + + + let mut redis = inferencer.redis_pool.get()?; + let mut redis_logger = RedisJobStatusLogger::new_tts_inference( + &mut redis, + &job.inference_job_token); + + redis_logger.log_status(failure_reason)?; + + if permanent_failure { + warn!("Marking job permanently dead: {} because: {:?}", job.inference_job_token, &e); + + let _r = mark_tts_inference_job_permanently_dead( + &inferencer.mysql_pool, + &job, + failure_reason + ).await; + } else { + let _r = mark_tts_inference_job_failure( + &inferencer.mysql_pool, + &job, + failure_reason, + inferencer.job_max_attempts + ).await; + } - let failure_reason = ""; - let _r = mark_tts_inference_job_failure( - &inferencer.mysql_pool, - &job, - failure_reason, - inferencer.job_max_attempts - ).await; continue; } }; @@ -486,11 +514,42 @@ fn read_metadata_file(filename: &PathBuf) -> AnyhowResult { /// We check both of these in one go so that we can reuse the ModelRecord later /// without another DB query. struct ModelState { - pub model_record: TtsModelRecord2, + pub model_record: TtsModelForInferenceRecord, pub is_downloaded_to_filesystem: bool, pub is_in_memory_cache: bool, } +#[derive(Debug, Clone)] +enum ModelStateError { + ModelNotFound, + ModelDeleted, + CacheError { reason: String }, + DatabaseError { reason: String }, +} + +impl std::fmt::Display for ModelStateError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ModelStateError::ModelNotFound => write!(f, "ModelNotFound"), + ModelStateError::ModelDeleted => write!(f, "ModelDeleted"), + ModelStateError::CacheError { reason} => write!(f, "Cache error: {:?}", reason), + ModelStateError::DatabaseError { reason} => write!(f, "Database error: {:?}", reason), + } + } +} + +impl From for ModelStateError { + fn from(error: TtsModelForInferenceError) -> Self { + match error { + TtsModelForInferenceError::ModelNotFound => ModelStateError::ModelNotFound, + TtsModelForInferenceError::ModelDeleted => ModelStateError::ModelDeleted, + TtsModelForInferenceError::DatabaseError { reason } => ModelStateError::DatabaseError { reason } + } + } +} + +impl std::error::Error for ModelStateError {} + impl ModelState { /// Query the model details and see if the model file is on the filesystem in one go. pub async fn query_model_and_check_filesystem( @@ -498,35 +557,37 @@ impl ModelState { mysql_pool: &MySqlPool, semi_persistent_cache: &SemiPersistentCacheDir, virtual_cache: &SyncVirtualLfuCache, - ) -> AnyhowResult { + ) -> Result { info!("Looking up TTS model by token: {}", &job.model_token); - let query_result = get_tts_model_by_token( + let tts_model = get_tts_model_by_token( &mysql_pool, - &job.model_token).await?; + &job.model_token + ).await?; - let model_record = match query_result { - Some(model) => model, - None => { - error!("TTS model not found: {}", &job.model_token); - return Err(anyhow!("Model not found!")) + /*let model_record = match query_result { + Ok(model) => model, + Err(e) => { + error!("TTS model lookup error: {}, reason: {:?}", &job.model_token, &e); + return Err(e) }, - }; + };*/ let tts_synthesizer_fs_path = semi_persistent_cache.tts_synthesizer_model_path( - &model_record.model_token); + &tts_model.model_token); let is_downloaded_to_filesystem = tts_synthesizer_fs_path.exists(); let path = tts_synthesizer_fs_path .to_str() - .ok_or(anyhow!("could not make path"))? + .ok_or(ModelStateError::CacheError { reason: "could not make path".to_string() })? .to_string(); - let is_in_memory_cache = virtual_cache.in_cache(&path)?; + let is_in_memory_cache = virtual_cache.in_cache(&path) + .map_err(|e| ModelStateError::CacheError { reason: format!("Model cache error: {:?}", e) })?; Ok(Self { - model_record, + model_record: tts_model, is_downloaded_to_filesystem, is_in_memory_cache, }) @@ -536,7 +597,7 @@ impl ModelState { async fn process_job( inferencer: &Inferencer, job: &TtsInferenceJobRecord, - model_record: &TtsModelRecord2, + model_record: &TtsModelForInferenceRecord, ) -> AnyhowResult<(Span, Span)> { // TODO 1. Mark processing (DONE) From 535f9574676fbafb52489aadd47937f69cad0e65 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 19 Sep 2021 20:20:51 -0400 Subject: [PATCH 0630/1042] codegen --- sqlx-data.json | 320 +++++++++++++++++++++++++++---------------------- 1 file changed, 178 insertions(+), 142 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index ff285fd464..8ae82cfe4a 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -3332,6 +3332,174 @@ ] } }, + "53f0792791317fcd4038308a6630f541d5508ae913cbfa9bc3d4081fba9a048a": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.maybe_default_pretrained_vocoder,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at,\n tts.user_deleted_at,\n tts.mod_deleted_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "maybe_default_pretrained_vocoder", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 3, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "private_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 10, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 11, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + true, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "551749bc153f9c8683a3a1ca5bb50744322aac046db34327501cf991485c4dee": { "query": "\nUPDATE tts_models\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -8192,6 +8360,16 @@ "nullable": [] } }, + "e0d442313b7861debaef984c5809647bd5df85106c194b66e864674b6af95330": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = \"dead\",\n failure_reason = ?,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "e3a0bd8073c3c0d84b9ab0e2e1ad8371f14b10801b7e4658b73c42ce1b27fa0b": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.is_public_listing_approved IS TRUE\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -8583,148 +8761,6 @@ ] } }, - "ec37e0acc4c897e8ef5aeeb66dce9df684ed8ea1ec29b0d9e92f44e0bd6d645d": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.maybe_default_pretrained_vocoder,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\nAND tts.user_deleted_at IS NULL\nAND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "maybe_default_pretrained_vocoder", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 3, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "private_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - true, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "ee05f95e45f7aaa5811d70b1f00d3f40a361536dfc6451affcad97586dc3338f": { "query": "\nUPDATE w2l_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { From 8ec7f0956b4d4104303724ecb98e71d60d1abb84 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 20 Sep 2021 00:47:10 -0400 Subject: [PATCH 0631/1042] TTL cache list of models in memory to de-strain the DB --- Cargo.lock | 7 ++ Cargo.toml | 3 + src/database/queries/list_tts_models.rs | 14 ++- .../endpoints/tts/list_tts_models.rs | 119 +++++++++--------- src/main.rs | 6 + src/server_state.rs | 5 + src/util/caching/mod.rs | 1 + src/util/caching/single_item_ttl_cache.rs | 43 +++++++ src/util/mod.rs | 1 + 9 files changed, 133 insertions(+), 66 deletions(-) create mode 100644 src/util/caching/mod.rs create mode 100644 src/util/caching/single_item_ttl_cache.rs diff --git a/Cargo.lock b/Cargo.lock index db14c98bfb..1aeed79fa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1956,6 +1956,12 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lru_time_cache" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" + [[package]] name = "magic-crypt" version = "3.1.8" @@ -3369,6 +3375,7 @@ dependencies = [ "lfu", "limitation", "log", + "lru_time_cache", "magic-crypt", "md-5", "newrelic-telemetry", diff --git a/Cargo.toml b/Cargo.toml index ff8e2a0d14..adc8d730e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,9 @@ sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "off limitation = "0.1.1" # Redis rate limiter r2d2_redis = "0.14.0" +# In-memory caching +lru_time_cache = "0.11.11" + # Files data-encoding = "2.3.2" ring = "0.17.0-alpha.10" diff --git a/src/database/queries/list_tts_models.rs b/src/database/queries/list_tts_models.rs index 420a632c5a..e0f727a8c3 100644 --- a/src/database/queries/list_tts_models.rs +++ b/src/database/queries/list_tts_models.rs @@ -15,6 +15,7 @@ pub struct TtsModelRecordForList { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, + pub creator_gravatar_hash: String, pub is_locked_from_use: bool, // converted @@ -31,6 +32,7 @@ struct RawTtsModelRecordForList { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, + pub creator_gravatar_hash: String, pub is_locked_from_use: i8, // NB: needs conversion @@ -56,13 +58,8 @@ pub async fn list_tts_models( }; let models : Vec = match maybe_models { - Ok(models) => { - info!("Model length: {}", models.len()); - models - }, + Ok(models) => models, Err(err) => { - warn!("Error: {:?}", err); - match err { RowNotFound => { return Ok(Vec::new()); @@ -83,6 +80,7 @@ pub async fn list_tts_models( creator_user_token: model.creator_user_token.clone(), creator_username: model.creator_username.clone(), creator_display_name: model.creator_display_name.clone(), + creator_gravatar_hash: model.creator_gravatar_hash.clone(), title: model.title.clone(), is_locked_from_use: i8_to_bool(model.is_locked_from_use), created_at: model.created_at.clone(), @@ -109,6 +107,7 @@ SELECT tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, + users.email_gravatar_hash as creator_gravatar_hash, tts.title, tts.is_locked_from_use, tts.created_at, @@ -134,6 +133,7 @@ SELECT tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, + users.email_gravatar_hash as creator_gravatar_hash, tts.title, tts.is_locked_from_use, tts.created_at, @@ -170,6 +170,7 @@ SELECT tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, + users.email_gravatar_hash as creator_gravatar_hash, tts.title, tts.is_locked_from_use, tts.created_at, @@ -198,6 +199,7 @@ SELECT tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, + users.email_gravatar_hash as creator_gravatar_hash, tts.title, tts.is_locked_from_use, tts.created_at, diff --git a/src/http_server/endpoints/tts/list_tts_models.rs b/src/http_server/endpoints/tts/list_tts_models.rs index 455518c75b..de5c2eb9ca 100644 --- a/src/http_server/endpoints/tts/list_tts_models.rs +++ b/src/http_server/endpoints/tts/list_tts_models.rs @@ -6,12 +6,13 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; -use crate::AnyhowResult; +use crate::database::queries::list_tts_models::list_tts_models; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; +use crate::util::anyhow_result::AnyhowResult; use derive_more::{Display, Error}; -use log::{info, warn, log}; +use log::{info, warn, log, error}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; @@ -19,7 +20,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -#[derive(Serialize)] +#[derive(Serialize, Clone)] pub struct TtsModelRecordForResponse { pub model_token: String, pub tts_model_type: String, @@ -76,68 +77,38 @@ pub async fn list_tts_models_handler( http_request: HttpRequest, server_state: web::Data>) -> Result { - // NB: Lookup failure is Err(RowNotFound). - // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_models = sqlx::query_as!( - TtsModelRecord, - r#" -SELECT - tts.token as model_token, - tts.tts_model_type, - tts.creator_user_token, - users.username as creator_username, - users.display_name as creator_display_name, - users.email_gravatar_hash as creator_gravatar_hash, - tts.title, - tts.created_at, - tts.updated_at -FROM tts_models as tts -JOIN users - ON users.token = tts.creator_user_token -WHERE - tts.user_deleted_at IS NULL - AND tts.mod_deleted_at IS NULL - "#, - ) - .fetch_all(&server_state.mysql_pool) - .await; // TODO: This will return error if it doesn't exist + let maybe_models = server_state.voice_list_cache.copy_without_bump_if_unexpired() + .map_err(|e| { + error!("Error consulting cache: {:?}", e); + ListTtsModelsError::ServerError + })?; - let mut models : Vec = match maybe_models { - Ok(models) => models, - Err(err) => { - match err { - RowNotFound => { - return Err(ListTtsModelsError::ServerError); - }, - _ => { - warn!("tts models list query error: {:?}", err); - return Err(ListTtsModelsError::ServerError); - } - } - } + let models = match maybe_models { + Some(models) => { + info!("Serving TTS models from cache"); + models + }, + None => { + info!("Populating TTS models from database"); + let models = get_all_models(&server_state.mysql_pool) + .await + .map_err(|e| { + error!("Error querying database: {:?}", e); + ListTtsModelsError::ServerError + })?; + + server_state.voice_list_cache.store_copy(&models) + .map_err(|e| { + error!("Error storing cache: {:?}", e); + ListTtsModelsError::ServerError + })?; + models + }, }; - models.sort_by(|a, b| (&a.title).cmp(&b.title)); - - let models_for_response = models.into_iter() - .map(|model| { - TtsModelRecordForResponse { - model_token: model.model_token.clone(), - tts_model_type: model.tts_model_type.clone(), - creator_user_token: model.creator_user_token.clone(), - creator_username: model.creator_username.clone(), - creator_display_name: model.creator_display_name.clone(), - creator_gravatar_hash: model.creator_gravatar_hash.clone(), - title: model.title.clone(), - created_at: model.created_at.clone(), - updated_at: model.updated_at.clone(), - } - }) - .collect::>(); - let response = ListTtsModelsSuccessResponse { success: true, - models: models_for_response, + models, }; let body = serde_json::to_string(&response) @@ -147,3 +118,31 @@ WHERE .content_type("application/json") .body(body)) } + +async fn get_all_models(mysql_pool: &MySqlPool) -> AnyhowResult> { + let mut models = list_tts_models( + mysql_pool, + None, + false + ).await?; + + models.sort_by(|a, b| (&a.title).cmp(&b.title)); + + let models_for_response = models.into_iter() + .map(|model| { + TtsModelRecordForResponse { + model_token: model.model_token.clone(), + tts_model_type: model.tts_model_type.clone(), + creator_user_token: model.creator_user_token.clone(), + creator_username: model.creator_username.clone(), + creator_display_name: model.creator_display_name.clone(), + creator_gravatar_hash: model.creator_gravatar_hash.clone(), + title: model.title.clone(), + created_at: model.created_at.clone(), + updated_at: model.updated_at.clone(), + } + }) + .collect::>(); + + Ok(models_for_response) +} diff --git a/src/main.rs b/src/main.rs index 6c68977882..b836857e19 100644 --- a/src/main.rs +++ b/src/main.rs @@ -101,6 +101,7 @@ use crate::shared_constants::DEFAULT_RUST_LOG; use crate::threads::ip_banlist_set::IpBanlistSet; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; +use crate::util::caching::single_item_ttl_cache::SingleItemTtlCache; use crate::util::encrypted_sort_id::SortKeyCrypto; use futures::Future; use futures::executor::ThreadPool; @@ -245,6 +246,10 @@ async fn main() -> AnyhowResult<()> { None, )?; + // In-Memory Cache + let cache_ttl = Duration::from_secs(60); + let voice_list_cache = SingleItemTtlCache::create_with_duration(cache_ttl); + // NB: This secret really isn't too important. // We can even rotate it without too much impact to users. let sort_key_crypto_secret = @@ -287,6 +292,7 @@ async fn main() -> AnyhowResult<()> { audio_uploads_bucket_root, sort_key_crypto, ip_banlist, + voice_list_cache, }; serve(server_state) diff --git a/src/server_state.rs b/src/server_state.rs index c740483434..5ed28360cd 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,10 +1,12 @@ use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; +use crate::http_server::endpoints::tts::list_tts_models::TtsModelRecordForResponse; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::threads::ip_banlist_set::IpBanlistSet; use crate::util::buckets::bucket_client::BucketClient; +use crate::util::caching::single_item_ttl_cache::SingleItemTtlCache; use crate::util::encrypted_sort_id::SortKeyCrypto; use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; @@ -39,6 +41,9 @@ pub struct ServerState { pub sort_key_crypto: SortKeyCrypto, pub ip_banlist: IpBanlistSet, + + /// In-memory caches with TTL-based eviction. Contains a list of all voices. + pub voice_list_cache: SingleItemTtlCache>, } #[derive(Clone)] diff --git a/src/util/caching/mod.rs b/src/util/caching/mod.rs new file mode 100644 index 0000000000..cffdc1415f --- /dev/null +++ b/src/util/caching/mod.rs @@ -0,0 +1 @@ +pub mod single_item_ttl_cache; \ No newline at end of file diff --git a/src/util/caching/single_item_ttl_cache.rs b/src/util/caching/single_item_ttl_cache.rs new file mode 100644 index 0000000000..c40c261164 --- /dev/null +++ b/src/util/caching/single_item_ttl_cache.rs @@ -0,0 +1,43 @@ +use anyhow::bail; +use crate::util::anyhow_result::AnyhowResult; +use lru_time_cache::LruCache; +use std::sync::{Arc, Mutex, PoisonError, MutexGuard}; +use std::time::Duration; + +const CACHE_KEY : &'static str = "ITEM"; + +/// Store a single payload in the cache +#[derive(Clone)] +pub struct SingleItemTtlCache { + cache: Arc>>, +} + +impl SingleItemTtlCache { + pub fn create_with_duration(time_to_live: Duration) -> Self { + let cache = LruCache::with_expiry_duration(time_to_live); + let cache = Arc::new(Mutex::new(cache)); + Self { + cache, + } + } + + pub fn copy_without_bump_if_unexpired(&self) -> AnyhowResult> { + let maybe_copy = match self.cache.lock() { + Err(e) => bail!("could not unlock mutex to read: {:?}", e), + Ok(mut cache) => { + cache.peek(CACHE_KEY).map(|inner| inner.clone()) + }, + }; + Ok(maybe_copy) + } + + pub fn store_copy(&self, item: &T) -> AnyhowResult<()> { + match self.cache.lock() { + Err(e) => bail!("could not unlock mutex to write: {:?}", e), + Ok(mut cache) => { + cache.insert(CACHE_KEY.into(), item.clone()); + }, + }; + Ok(()) + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 00377d97b0..cbd4db727f 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,5 +1,6 @@ pub mod anyhow_result; pub mod buckets; +pub mod caching; pub mod delete_role_disambiguation; pub mod email_to_gravatar; pub mod encrypted_sort_id; From ef8711ab2e20f5758492e14fa8b22565e66fee58 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 20 Sep 2021 00:48:36 -0400 Subject: [PATCH 0632/1042] codegen --- sqlx-data.json | 1213 +++++++++++++++++++++++------------------------- 1 file changed, 568 insertions(+), 645 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 8ae82cfe4a..5d4668089e 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -375,6 +375,148 @@ ] } }, + "05612e384bdce558e4834c720d1eb7ca9a09e3cca64b72f1497929d860345ec7": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "05c3ebf54cd8a5374d6f4bbe21b3c6027ce6314917af8acd0ed136d375d53342": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -1699,135 +1841,6 @@ ] } }, - "1556b7ed6b0cab002b52dfa8a5f81743a38967631ff24fd9555a168823b1dfc1": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 7, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 8, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "17c8d210d93ee9c0d8e8422eed605c9150f19f9bf72affcf111b15a67a5b9e70": { "query": "\nUPDATE w2l_templates\nSET\n is_public_listing_approved = ?,\n is_locked_from_user_modification = ?,\n is_locked_from_use = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -3742,6 +3755,148 @@ ] } }, + "5d1f983fd2555c0fcb2ba065e9b3c6174346ee0fdaecacc2a557715a68cf23e2": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "5e54fd8569b2d6d3a0d11acee6bd03b75400b6325952eb16c8dac68a7933b13a": { "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n is_banned,\n disable_gravatar,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", "describe": { @@ -5465,135 +5620,6 @@ "nullable": [] } }, - "99f1869443b8331b3099af44c53d6db4573fdfb2df5af56668f3256bf446b096": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 7, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 8, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "9a879f76345ef5682b995f99026fac8b2f4c64c8ec4bb078187d1e53effcfbaf": { "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address_creation = ?,\n creator_ip_address_last_update = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n maybe_public_bucket_preview_image_object_name = ?,\n maybe_public_bucket_preview_video_object_name = ?,\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", "describe": { @@ -5922,135 +5948,6 @@ ] } }, - "aff28dad260440114ea616087a6f29a8a81e8ad1329d0c3dd4a3db4a613eac29": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 8, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "b1ad806aa93f66cab358d7cdb405d57c643e2ff131cfec9f1ced55c2e6ca9511": { "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { @@ -7212,6 +7109,148 @@ "nullable": [] } }, + "bf779826a10dd532f442d5b6a7ad3f35037276e8f822a3a633ca325c85c34234": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "c0f56e2656b566d3a8c869d39b0b56d1fbb75d7d9e817a24b10add44d348092d": { "query": "\nSELECT count(*) as count\nFROM w2l_templates\nWHERE\n creator_user_token = ?\nLIMIT 1\n ", "describe": { @@ -7889,135 +7928,6 @@ "nullable": [] } }, - "cc51b77e88c2ee5c046ea2f2287b9e9770743d50b0db90481662e5c1118885c5": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 7, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 8, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "d12757abef811cd6951e0e1427fdb496a8931b547519d320f5ade56007d3eb5d": { "query": "\nSELECT\n id,\n status\nFROM w2l_template_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", "describe": { @@ -8183,135 +8093,6 @@ ] } }, - "dfc7d0ff70e135e1c4d50c7655b6064e5034f16eacd9bb94070933a68815d270": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 7, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 8, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "dfe818ab849c5f6e2a5931142aab640d249aa96cb85dc06bfffae4d7feef541e": { "query": "\nSELECT\n NOW() - t2.created_at AS seconds_since_first,\n (\n SELECT\n count(t1.id) as pending_count\n FROM w2l_inference_jobs AS t1\n WHERE t1.status = \"pending\"\n ) as pending_count\nFROM w2l_inference_jobs AS t2\nWHERE t2.status = \"pending\"\nORDER BY t2.id ASC\nLIMIT 1\n ", "describe": { @@ -8771,6 +8552,148 @@ "nullable": [] } }, + "ef239806d026cd59106f79b3dc7c850da2bbc325a1adb831f319f87c0b07b56b": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "ef828549a221fe6fad3105fede0b38a439a685d6ea7f83a0657df9cdbb80c589": { "query": "\nSELECT\n ip_bans.ip_address,\n\n ip_bans.maybe_target_user_token,\n banned_users.username as maybe_target_username,\n\n ip_bans.mod_user_token,\n mod_users.username as mod_username,\n mod_users.display_name as mod_display_name,\n\n ip_bans.mod_notes,\n ip_bans.created_at,\n ip_bans.updated_at\nFROM\n ip_address_bans AS ip_bans\nLEFT OUTER JOIN users as banned_users\n ON ip_bans.maybe_target_user_token = banned_users.token\nJOIN users as mod_users\n ON ip_bans.mod_user_token = mod_users.token\nWHERE\n ip_bans.ip_address = ?\nLIMIT 1\n ", "describe": { From 383a5a99ff5dad5488e1606a93f6dacfa67c4154 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 20 Sep 2021 01:25:44 -0400 Subject: [PATCH 0633/1042] tts model cache duration configurable --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- src/main.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1aeed79fa6..06a1e3fcd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1212,9 +1212,9 @@ checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" [[package]] name = "easyenv" -version = "0.0.5" +version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852054bee794f364fbe890bd947cbf678c35226b81ede17720d6a17fc687a0d4" +checksum = "e8044ea5153408cb704ee2a9dafbd33aa5d14c8a34c8405910d90e8c8ef45944" dependencies = [ "dotenv", "env_logger", diff --git a/Cargo.toml b/Cargo.toml index adc8d730e8..30f67728b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ base64 = "0.13.0" chrono = { version = "0.4.19", features = ["serde"] } derive_more = "0.99.14" dotenv = "0.15.0" -easyenv = "0.0.5" +easyenv = "0.0.6" futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate hex = "0.4.3" diff --git a/src/main.rs b/src/main.rs index b836857e19..5439431a13 100644 --- a/src/main.rs +++ b/src/main.rs @@ -247,7 +247,7 @@ async fn main() -> AnyhowResult<()> { )?; // In-Memory Cache - let cache_ttl = Duration::from_secs(60); + let cache_ttl = easyenv::get_env_duration_seconds_or_default("VOICE_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60)); let voice_list_cache = SingleItemTtlCache::create_with_duration(cache_ttl); // NB: This secret really isn't too important. From 5187d25e9185916e94b3526afa48a9a68e176817 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 27 Sep 2021 17:47:47 -0400 Subject: [PATCH 0634/1042] add fakeyou.com to CORS --- src/main.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 5439431a13..2eeb5456ed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -319,8 +319,10 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> App::new() .app_data(server_state_arc.clone()) .wrap(Cors::default() + .allowed_origin("http://api.fakeyou.com") .allowed_origin("http://api.jungle.horse") .allowed_origin("http://api.vo.codes") + .allowed_origin("http://fakeyou.com") .allowed_origin("http://jungle.horse") .allowed_origin("http://jungle.horse:12345") .allowed_origin("http://jungle.horse:7000") @@ -334,8 +336,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .allowed_origin("http://trumped.com") .allowed_origin("http://vo.codes") .allowed_origin("http://vocodes.com") - .allowed_origin("https://jungle.horse") + .allowed_origin("https://api.fakeyou.com") .allowed_origin("https://api.jungle.horse") + .allowed_origin("https://api.vo.codes") + .allowed_origin("https://fakeyou.com") + .allowed_origin("https://jungle.horse") .allowed_origin("https://mumble.stream") .allowed_origin("https://trumped.com") .allowed_origin("https://vo.codes") From de802846652d951379a851568889020a436f8144 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 28 Sep 2021 01:09:14 -0400 Subject: [PATCH 0635/1042] util to get http hostname header (turns out I don't need it yet) --- src/http_server/web_utils/get_host_header.rs | 18 ++++++++++++++++++ src/http_server/web_utils/mod.rs | 1 + 2 files changed, 19 insertions(+) create mode 100644 src/http_server/web_utils/get_host_header.rs diff --git a/src/http_server/web_utils/get_host_header.rs b/src/http_server/web_utils/get_host_header.rs new file mode 100644 index 0000000000..fd47e41dac --- /dev/null +++ b/src/http_server/web_utils/get_host_header.rs @@ -0,0 +1,18 @@ +use actix_http::http::HeaderMap; +use actix_http::http::HeaderName; +use actix_web::HttpRequest; +use actix_web::http::HeaderValue; +use crate::util::anyhow_result::AnyhowResult; + +// NB: This is extracted because CLion is having incredible difficulty with these types. +pub fn get_host_header(http_request: &HttpRequest) -> AnyhowResult> { + let host_header_name = HeaderName::from_static("host"); + let header_map : &HeaderMap = http_request.headers(); + let maybe_hostname = header_map.get(host_header_name); + + let hostname = maybe_hostname.map(|s| s.to_str()) + .map(|inner| inner.map(|s| s.to_string())) + .transpose()?; + + Ok(hostname) +} \ No newline at end of file diff --git a/src/http_server/web_utils/mod.rs b/src/http_server/web_utils/mod.rs index 64a1b8c708..8037e1b8c8 100644 --- a/src/http_server/web_utils/mod.rs +++ b/src/http_server/web_utils/mod.rs @@ -1,4 +1,5 @@ pub mod cookie_manager; +pub mod get_host_header; pub mod ip_address; pub mod read_multipart_field_bytes; pub mod redis_rate_limiter; From e05d519e5a7a51673ca83fcd926c5ebf9e606d10 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 7 Nov 2021 14:07:01 -0500 Subject: [PATCH 0636/1042] NEW OBS GATEWAY ARC (NOT YET WOKRING) --- Cargo.lock | 77 ++++++- Cargo.toml | 9 + .../down.sql | 4 + .../up.sql | 75 +++++++ src/http_server/endpoints/mod.rs | 1 + src/http_server/endpoints/twitch/mod.rs | 1 + .../endpoints/twitch/pubsub_gateway.rs | 86 ++++++++ src/obs_gateway/mod.rs | 0 src/obs_gateway_server.rs | 200 ++++++++++++++++++ test.js | 10 + 10 files changed, 457 insertions(+), 6 deletions(-) create mode 100644 migrations/2021-09-29-042235_twitch_oauth_tokens/down.sql create mode 100644 migrations/2021-09-29-042235_twitch_oauth_tokens/up.sql create mode 100644 src/http_server/endpoints/twitch/mod.rs create mode 100644 src/http_server/endpoints/twitch/pubsub_gateway.rs create mode 100644 src/obs_gateway/mod.rs create mode 100644 src/obs_gateway_server.rs create mode 100644 test.js diff --git a/Cargo.lock b/Cargo.lock index 06a1e3fcd7..b194588ab9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,31 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "actix" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be241f88f3b1e7e9a3fbe3b5a8a0f6915b5a1d7ee0d9a248d3376d01068cc60" +dependencies = [ + "actix-rt 1.1.1", + "actix_derive", + "bitflags", + "bytes 0.5.6", + "crossbeam-channel 0.4.4", + "derive_more", + "futures-channel", + "futures-util", + "log", + "once_cell", + "parking_lot 0.11.1", + "pin-project 0.4.28", + "smallvec 1.6.1", + "tokio 0.2.25", + "tokio-util", + "trust-dns-proto", + "trust-dns-resolver", +] + [[package]] name = "actix-codec" version = "0.3.0" @@ -130,9 +155,9 @@ dependencies = [ [[package]] name = "actix-macros" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" +checksum = "c2f86cd6857c135e6e9fe57b1619a88d1f94a7df34c00e11fe13e64fd3438837" dependencies = [ "quote", "syn", @@ -190,7 +215,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc7d7cd957c9ed92288a7c3c96af81fa5291f65247a76a34dac7b6af74e52ba0" dependencies = [ - "actix-macros 0.2.0", + "actix-macros 0.2.1", "futures-core", "tokio 1.6.0", ] @@ -335,6 +360,22 @@ dependencies = [ "url 2.2.2", ] +[[package]] +name = "actix-web-actors" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6edf3c2693e2a8c422800c87ee89a6a4eac7dd01109bc172a1093ce1f4f001" +dependencies = [ + "actix", + "actix-codec", + "actix-http", + "actix-web", + "bytes 0.5.6", + "futures-channel", + "futures-core", + "pin-project 0.4.28", +] + [[package]] name = "actix-web-codegen" version = "0.4.0" @@ -346,6 +387,17 @@ dependencies = [ "syn", ] +[[package]] +name = "actix_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95aceadaf327f18f0df5962fedc1bde2f870566a0b9f65c89508a3b1f79334c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "adler" version = "1.0.2" @@ -1024,6 +1076,16 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crossbeam-channel" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" +dependencies = [ + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -1655,9 +1717,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" +checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" [[package]] name = "httpdate" @@ -3209,7 +3271,7 @@ dependencies = [ "bytes 0.5.6", "chrono", "crc", - "crossbeam-channel", + "crossbeam-channel 0.5.1", "crossbeam-queue", "crossbeam-utils 0.8.4", "digest", @@ -3346,6 +3408,7 @@ checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" name = "storyteller-web" version = "0.0.1" dependencies = [ + "actix", "actix-cors", "actix-files", "actix-http", @@ -3354,6 +3417,7 @@ dependencies = [ "actix-service", "actix-utils 3.0.0", "actix-web", + "actix-web-actors", "anyhow", "base64 0.13.0", "bcrypt", @@ -3761,6 +3825,7 @@ checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" dependencies = [ "bytes 0.5.6", "futures-core", + "futures-io", "futures-sink", "log", "pin-project-lite 0.1.12", diff --git a/Cargo.toml b/Cargo.toml index 30f67728b8..b46d2161a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,11 @@ publish = false name = "storyteller-web" path = "src/main.rs" +# OBS websocket server +[[bin]] +name = "obs-gateway-server" +path = "src/obs_gateway_server.rs" + # Jobs for managing downloads and inference [[bin]] name = "tts-download-job" @@ -89,6 +94,10 @@ subprocess = "0.2.6" tempdir = "0.3.7" # Web +actix = "0.10" +#actix-web = "3" +actix-web-actors = "3" + actix-cors = "0.5.4" actix-files= "0.5" actix-http = "2.2.0" diff --git a/migrations/2021-09-29-042235_twitch_oauth_tokens/down.sql b/migrations/2021-09-29-042235_twitch_oauth_tokens/down.sql new file mode 100644 index 0000000000..fdbf0af1b8 --- /dev/null +++ b/migrations/2021-09-29-042235_twitch_oauth_tokens/down.sql @@ -0,0 +1,4 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + diff --git a/migrations/2021-09-29-042235_twitch_oauth_tokens/up.sql b/migrations/2021-09-29-042235_twitch_oauth_tokens/up.sql new file mode 100644 index 0000000000..eed73ee95b --- /dev/null +++ b/migrations/2021-09-29-042235_twitch_oauth_tokens/up.sql @@ -0,0 +1,75 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- **NOTE ABOUT DESIGN**: +-- This table contains _many_ OAuth tokens for any single user. (many users : many tokens) +-- An entire history of tokens is kept in the table. + +CREATE TABLE twitch_oauth_tokens( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Foreign key to user (Vocodes/Storyteller user) + -- If no user is associated (anonymous Twitch user), then this is null. + maybe_user_token VARCHAR(32) DEFAULT NULL, + + -- The user ID / channel ID are the same + -- https://discuss.dev.twitch.tv/t/what-is-the-difference-between-the-stream--id-and-channel--id/4423 + maybe_twitch_user_id VARCHAR(32) DEFAULT NULL, + + -- Usernames are between 4 and 25 characters. + maybe_twitch_username VARCHAR(32) DEFAULT NULL, + + -- ========== OAUTH SCOPES ========== + + -- "bits:read" + -- "View Bits information for a channel." + has_bits_read BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:read:subscriptions" + -- "View a list of all subscribers to a channel and check if a user is subscribed to a channel." + -- eg. Enumerate list of subscribers for a channel + -- eg. PubSub subscribe to `channel-subscribe-events-v1.` (subscribe, resubscribe, gift) + has_channel_read_subscriptions BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:read:redemptions" + -- "View Channel Points custom rewards and their redemptions on a channel." + -- eg. PubSub subscribe to `channel-points-channel-v1.` (channel points spends) + has_channel_read_redemptions BOOLEAN NOT NULL DEFAULT FALSE, + + -- "user:read:follows" + -- "View the list of channels a user follows." + -- eg. We can use this to see if a Twitch user follows us. + has_user_read_follows BOOLEAN NOT NULL DEFAULT FALSE, + + + -- THERE ARE MORE SCOPES... PERHAPS WE SHOULD ADD THEM? + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- This is when the OAuth token will expire. We need to refresh before this time. + expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + + -- If the OAuth token is deleted, we set this. + -- We'll need to set it for all of a user's OAuth tokens since this table stores many tokens. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + KEY fk_maybe_user_token (maybe_user_token) + + --KEY index_has_bits_read (has_bits_read), + --KEY index_creator_ip_address (creator_ip_address) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + diff --git a/src/http_server/endpoints/mod.rs b/src/http_server/endpoints/mod.rs index f585acfc72..38b3143d3f 100644 --- a/src/http_server/endpoints/mod.rs +++ b/src/http_server/endpoints/mod.rs @@ -5,5 +5,6 @@ pub mod misc; pub mod moderation; pub mod root_index; pub mod tts; +pub mod twitch; pub mod users; pub mod w2l; \ No newline at end of file diff --git a/src/http_server/endpoints/twitch/mod.rs b/src/http_server/endpoints/twitch/mod.rs new file mode 100644 index 0000000000..f1d932d175 --- /dev/null +++ b/src/http_server/endpoints/twitch/mod.rs @@ -0,0 +1 @@ +pub mod pubsub_gateway; \ No newline at end of file diff --git a/src/http_server/endpoints/twitch/pubsub_gateway.rs b/src/http_server/endpoints/twitch/pubsub_gateway.rs new file mode 100644 index 0000000000..96f49d96ec --- /dev/null +++ b/src/http_server/endpoints/twitch/pubsub_gateway.rs @@ -0,0 +1,86 @@ +use actix::prelude::*; +use actix_web::{middleware, web, App, Error, HttpRequest, HttpResponse, HttpServer}; +use actix_web_actors::ws; +use log::warn; +use log::error; + +pub async fn ws_index(r: HttpRequest, stream: web::Payload) -> Result { + error!(">>>>>> ws_index"); + ws::start(WebSocket::new(), &r, stream) +} + +struct WebSocket {} + +impl Actor for WebSocket { + type Context = ws::WebsocketContext; + + fn started(&mut self, _ctx: &mut Self::Context) { + error!(">>>>>> actor started"); + } +} + +impl StreamHandler> for WebSocket { + fn handle( + &mut self, + msg: Result, + ctx: &mut Self::Context, + ) { + if let Ok(msg) = msg { + error!(">>>>>> streamhandler::handle (msg = {:?})", msg); + match msg { + ws::Message::Text(text) => ctx.text(text), + ws::Message::Binary(bin) => ctx.binary(bin), + ws::Message::Ping(bytes) => ctx.pong(&bytes), + ws::Message::Close(reason) => { + ctx.close(reason); + ctx.stop(); + } + _ => {} + } + } else { + error!(">>>>>> streamhandler::STOP"); + ctx.stop(); + } + } +} + +impl WebSocket { + fn new() -> Self { + Self {} + } +} + +// From the Actix homepage: + +//use actix::{Actor, StreamHandler}; +//use actix_web::{web, App, Error, HttpRequest, HttpResponse, HttpServer}; +//use actix_web_actors::ws; +// +///// Define HTTP actor +//struct MyWs; +// +//impl Actor for MyWs { +// type Context = ws::WebsocketContext; +//} +// +///// Handler for ws::Message message +//impl StreamHandler> for MyWs { +// fn handle( +// &mut self, +// msg: Result, +// ctx: &mut Self::Context, +// ) { +// match msg { +// Ok(ws::Message::Ping(msg)) => ctx.pong(&msg), +// Ok(ws::Message::Text(text)) => ctx.text(text), +// Ok(ws::Message::Binary(bin)) => ctx.binary(bin), +// _ => (), +// } +// } +//} +// +//pub async fn twitch_pubsub_gateway(req: HttpRequest, stream: web::Payload) -> Result { +// let resp = ws::start(MyWs {}, &req, stream); +// println!("{:?}", resp); +// resp +//} \ No newline at end of file diff --git a/src/obs_gateway/mod.rs b/src/obs_gateway/mod.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/obs_gateway_server.rs b/src/obs_gateway_server.rs new file mode 100644 index 0000000000..114f8c4dcf --- /dev/null +++ b/src/obs_gateway_server.rs @@ -0,0 +1,200 @@ +#![forbid(private_in_public)] +#![forbid(unused_must_use)] +//#![forbid(warnings)] + +#![allow(dead_code)] +#![allow(non_snake_case)] +#![allow(unused_imports)] +#![allow(unused_mut)] +#![allow(unused_variables)] + +#[macro_use] extern crate lazy_static; +#[macro_use] extern crate magic_crypt; +#[macro_use] extern crate serde_derive; + +pub mod shared_constants; + +use actix_cors::Cors; +use actix_http::http; +use actix_web::middleware::{Logger, DefaultHeaders}; +use actix_web::{HttpServer, web, HttpResponse, App}; +use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_RUST_LOG; +use futures::Future; +use futures::executor::ThreadPool; +use limitation::Limiter; +use log::{info}; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2; +use r2d2_redis::redis::Commands; +use sqlx::MySqlPool; +use sqlx::mysql::MySqlPoolOptions; +use std::sync::Arc; +use std::time::Duration; + +const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; + +pub type AnyhowResult = anyhow::Result; + +/// State that is injected into every endpoint. +#[derive(Clone)] +pub struct ObsGatewayServerState { + /// Configuration from ENV vars. + /// Some of this might not be used. + pub env_config: EnvConfig, + pub hostname: String, +} + +#[derive(Clone)] +pub struct EnvConfig { + // Number of thread workers. + pub num_workers: usize, + pub bind_address: String, + pub cookie_domain: String, + pub cookie_secure: bool, + pub cookie_http_only: bool, + pub website_homepage_redirect: String, +} + +#[actix_web::main] +async fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + // NB: Do not check this secrets-containing dotenv file into VCS. + // This file should only contain *development* secrets, never production. + let _ = dotenv::from_filename(".env-secrets").ok(); + + info!("Obtaining hostname..."); + + let server_hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or("obs-gateway-server-unknown".to_string()); + + info!("Hostname: {}", &server_hostname); + + info!("Connecting to database..."); + + let db_connection_string = + easyenv::get_env_string_or_default( + "MYSQL_URL", + DEFAULT_MYSQL_CONNECTION_STRING); + + let redis_connection_string = + easyenv::get_env_string_or_default( + "REDIS_URL", + DEFAULT_REDIS_CONNECTION_STRING); + + let pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(&db_connection_string) + .await?; + + let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?; + + let redis_pool = r2d2::Pool::builder() + .build(redis_manager)?; + + info!("Reading env vars and setting up utils..."); + + let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); + let num_workers = easyenv::get_env_num("NUM_WORKERS", 8)?; + let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); + let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); + let cookie_secure = easyenv::get_env_bool_or_default("COOKIE_SECURE", true); + let cookie_http_only = easyenv::get_env_bool_or_default("COOKIE_HTTP_ONLY", true); + let website_homepage_redirect = + easyenv::get_env_string_or_default("WEBSITE_HOMEPAGE_REDIRECT", "https://vo.codes/"); + + let server_state = ObsGatewayServerState { + env_config: EnvConfig { + num_workers, + bind_address, + cookie_domain, + cookie_secure, + cookie_http_only, + website_homepage_redirect, + }, + hostname: server_hostname, + }; + + serve(server_state) + .await?; + Ok(()) +} + +pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> +{ + let bind_address = server_state.env_config.bind_address.clone(); + let num_workers = server_state.env_config.num_workers.clone(); + let hostname = server_state.hostname.clone(); + + let server_state_arc = web::Data::new(Arc::new(server_state)); + + info!("Starting HTTP service."); + + let log_format = "[%{HOSTNAME}e] %a \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T"; + + HttpServer::new(move || { + App::new() + .app_data(server_state_arc.clone()) + .wrap(Cors::default() + .allowed_origin("http://api.fakeyou.com") + .allowed_origin("http://api.jungle.horse") + .allowed_origin("http://api.vo.codes") + .allowed_origin("http://fakeyou.com") + .allowed_origin("http://jungle.horse") + .allowed_origin("http://jungle.horse:12345") + .allowed_origin("http://jungle.horse:7000") + .allowed_origin("http://localhost:12345") + .allowed_origin("http://localhost:3000") + .allowed_origin("http://localhost:5555") + .allowed_origin("http://localhost:7000") + .allowed_origin("http://localhost:8000") + .allowed_origin("http://localhost:8080") + .allowed_origin("http://mumble.stream") + .allowed_origin("http://trumped.com") + .allowed_origin("http://vo.codes") + .allowed_origin("http://vocodes.com") + .allowed_origin("https://api.fakeyou.com") + .allowed_origin("https://api.jungle.horse") + .allowed_origin("https://api.vo.codes") + .allowed_origin("https://fakeyou.com") + .allowed_origin("https://jungle.horse") + .allowed_origin("https://mumble.stream") + .allowed_origin("https://trumped.com") + .allowed_origin("https://vo.codes") + .allowed_origin("https://vocodes.com") + .allowed_methods(vec!["GET", "POST", "OPTIONS"]) + .supports_credentials() + .allowed_headers(vec![ + http::header::ACCEPT, + http::header::ACCESS_CONTROL_ALLOW_ORIGIN, // Tabulator Ajax + http::header::CONTENT_TYPE, + http::header::ACCESS_CONTROL_ALLOW_CREDENTIALS, // https://stackoverflow.com/a/46412839 + http::header::HeaderName::from_static("x-requested-with") // Tabulator Ajax sends + ]) + .max_age(3600)) + .wrap(Logger::new(&log_format) + .exclude("/liveness") + .exclude("/readiness")) + .wrap(DefaultHeaders::new() + .header("X-Backend-Hostname", &hostname) + .header("X-Build-Sha", "")) + // Twitch + //.service( + // web::resource("/twitch") + // .route(web::get().to(ws_index)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service(get_root_index) + //.default_service( web::route().to(default_route_404)) + }) + .bind(bind_address)? + .workers(num_workers) + .run() + .await?; + + Ok(()) +} diff --git a/test.js b/test.js new file mode 100644 index 0000000000..e5e40c533b --- /dev/null +++ b/test.js @@ -0,0 +1,10 @@ +//var sock = new WebSocket("ws://api.jungle.horse:80/twitch"); +var sock = new WebSocket("ws://localhost:12345/twitch"); + +sock.onopen = function (event) { + //sock.send("Here's some text that the server is urgently awaiting!"); +}; + +sock.onmessage = function (event) { + console.log('got a message', event.data); +} \ No newline at end of file From 5dd7efa2d5c6798d3a7c9b3e0f8c1ba4d876980d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 7 Nov 2021 14:35:34 -0500 Subject: [PATCH 0637/1042] Massive refactor: Two app packages in a workspace (5 binaries total) --- Cargo.lock | 8 + Cargo.toml | 137 +----------------- app/obs_gateway/Cargo.toml | 13 ++ app/obs_gateway/src/main.rs | 4 + app/storyteller_web/Cargo.toml | 133 +++++++++++++++++ .../storyteller_web/src}/common_env.rs | 0 .../src}/database/enums/mod.rs | 0 .../src}/database/enums/record_visibility.rs | 0 .../src}/database/enums/tts_model_type.rs | 0 .../src}/database/enums/vocoder_type.rs | 0 .../database/helpers/boolean_converters.rs | 0 .../src}/database/helpers/enums.rs | 0 .../src}/database/helpers/mod.rs | 0 .../src}/database/helpers/tokens.rs | 0 .../src}/database/mediators/badge_granter.rs | 0 .../database/mediators/firehose_publisher.rs | 0 .../src}/database/mediators/mod.rs | 0 .../storyteller_web/src}/database/mod.rs | 0 .../src}/database/queries/badges.rs | 0 .../calculate_tts_model_leaderboard.rs | 0 .../calculate_w2l_template_leaderboard.rs | 0 .../src}/database/queries/create_session.rs | 0 .../src}/database/queries/delete_tts_model.rs | 0 .../database/queries/delete_tts_result.rs | 0 .../database/queries/delete_w2l_result.rs | 0 .../database/queries/delete_w2l_template.rs | 0 .../queries/get_user_profile_by_username.rs | 0 .../src}/database/queries/list_ip_bans.rs | 0 .../src}/database/queries/list_tts_models.rs | 0 .../src}/database/queries/list_user_badges.rs | 0 .../src}/database/queries/list_user_roles.rs | 0 .../database/queries/list_w2l_templates.rs | 0 .../src}/database/queries/mod.rs | 0 .../src}/database/queries/query_tts_model.rs | 0 .../src}/database/queries/query_tts_result.rs | 0 .../src}/database/queries/query_w2l_result.rs | 0 .../database/queries/query_w2l_template.rs | 0 ...ist_tts_inference_results_query_builder.rs | 0 .../list_users_query_builder.rs | 0 ...ist_w2l_inference_results_query_builder.rs | 0 .../src}/database/query_builders/mod.rs | 0 .../storyteller_web/src}/http_clients/mod.rs | 0 .../tts_inference_sidecar_client.rs | 0 .../endpoints/default_route_404.rs | 0 .../endpoints/events/list_events.rs | 0 .../src}/http_server/endpoints/events/mod.rs | 0 .../endpoints/leaderboard/get_leaderboard.rs | 0 .../http_server/endpoints/leaderboard/mod.rs | 0 .../endpoints/misc/alpha_cookie.rs | 0 .../misc/enable_alpha_easy_handler.rs | 0 .../endpoints/misc/enable_alpha_handler.rs | 0 .../src}/http_server/endpoints/misc/mod.rs | 0 .../src}/http_server/endpoints/mod.rs | 0 .../endpoints/moderation/approval/mod.rs | 0 .../approval/pending_w2l_templates.rs | 0 .../moderation/ip_bans/add_ip_ban.rs | 0 .../moderation/ip_bans/delete_ip_ban.rs | 0 .../moderation/ip_bans/get_ip_ban.rs | 0 .../moderation/ip_bans/list_ip_bans.rs | 0 .../endpoints/moderation/ip_bans/mod.rs | 0 .../jobs/get_tts_inference_queue_count.rs | 0 .../jobs/get_w2l_inference_queue_count.rs | 0 .../endpoints/moderation/jobs/mod.rs | 0 .../http_server/endpoints/moderation/mod.rs | 0 .../moderation/stats/get_voice_count_stats.rs | 0 .../endpoints/moderation/stats/mod.rs | 0 .../moderation/user_bans/ban_user.rs | 0 .../moderation/user_bans/list_banned_users.rs | 0 .../endpoints/moderation/user_bans/mod.rs | 0 .../moderation/user_roles/list_roles.rs | 0 .../moderation/user_roles/list_staff.rs | 0 .../endpoints/moderation/user_roles/mod.rs | 0 .../moderation/user_roles/set_user_role.rs | 0 .../endpoints/moderation/users/list_users.rs | 0 .../endpoints/moderation/users/mod.rs | 0 .../src}/http_server/endpoints/root_index.rs | 0 .../endpoints/tts/delete_tts_model.rs | 0 .../endpoints/tts/delete_tts_result.rs | 0 .../endpoints/tts/edit_tts_model.rs | 0 .../endpoints/tts/edit_tts_result.rs | 0 .../endpoints/tts/enqueue_infer_tts.rs | 0 .../endpoints/tts/enqueue_upload_tts_model.rs | 0 .../tts/get_tts_inference_job_status.rs | 0 .../endpoints/tts/get_tts_model.rs | 0 .../endpoints/tts/get_tts_model_use_count.rs | 0 .../endpoints/tts/get_tts_result.rs | 0 .../tts/get_tts_upload_model_job_status.rs | 0 .../endpoints/tts/list_tts_models.rs | 0 .../src}/http_server/endpoints/tts/mod.rs | 0 .../src}/http_server/endpoints/twitch/mod.rs | 0 .../endpoints/twitch/pubsub_gateway.rs | 0 .../endpoints/users/create_account.rs | 0 .../endpoints/users/edit_profile.rs | 0 .../endpoints/users/get_profile.rs | 0 .../users/list_user_tts_inference_results.rs | 0 .../endpoints/users/list_user_tts_models.rs | 0 .../users/list_user_w2l_inference_results.rs | 0 .../users/list_user_w2l_templates.rs | 0 .../src}/http_server/endpoints/users/login.rs | 0 .../http_server/endpoints/users/logout.rs | 0 .../src}/http_server/endpoints/users/mod.rs | 0 .../endpoints/users/session_info.rs | 0 .../endpoints/w2l/delete_w2l_result.rs | 0 .../endpoints/w2l/delete_w2l_template.rs | 0 .../endpoints/w2l/edit_w2l_result.rs | 0 .../endpoints/w2l/edit_w2l_template.rs | 0 .../endpoints/w2l/enqueue_infer_w2l.rs | 0 .../w2l/enqueue_infer_w2l_with_uploads.rs | 0 .../w2l/enqueue_upload_w2l_template.rs | 0 .../w2l/get_w2l_inference_job_status.rs | 0 .../endpoints/w2l/get_w2l_result.rs | 0 .../endpoints/w2l/get_w2l_template.rs | 0 .../w2l/get_w2l_template_use_count.rs | 0 .../w2l/get_w2l_upload_template_job_status.rs | 0 .../endpoints/w2l/list_w2l_templates.rs | 0 .../src}/http_server/endpoints/w2l/mod.rs | 0 .../w2l/set_w2l_template_mod_approval.rs | 0 .../middleware/ip_filter_middleware.rs | 0 .../src}/http_server/middleware/mod.rs | 0 .../storyteller_web/src}/http_server/mod.rs | 0 .../http_server/web_utils/cookie_manager.rs | 0 .../http_server/web_utils/get_host_header.rs | 0 .../src}/http_server/web_utils/ip_address.rs | 0 .../src}/http_server/web_utils/mod.rs | 0 .../web_utils/read_multipart_field_bytes.rs | 0 .../web_utils/redis_rate_limiter.rs | 0 .../web_utils/response_error_helpers.rs | 0 .../web_utils/response_success_helpers.rs | 0 .../web_utils/serialize_as_json_error.rs | 0 .../http_server/web_utils/session_checker.rs | 0 .../storyteller_web/src}/job_queries/mod.rs | 0 .../job_queries/tts_download_job_queries.rs | 0 .../job_queries/tts_inference_job_queries.rs | 0 .../job_queries/w2l_download_job_queries.rs | 0 .../job_queries/w2l_inference_job_queries.rs | 0 {src => app/storyteller_web/src}/main.rs | 13 +- .../storyteller_web/src}/obs_gateway/mod.rs | 0 .../src}/obs_gateway_server.rs | 0 .../ffmpeg_generate_preview_image_command.rs | 0 .../ffmpeg_generate_preview_video_command.rs | 0 .../google_drive_download_command.rs | 0 ...gemagick_generate_preview_image_command.rs | 0 .../src}/script_execution/mod.rs | 0 .../tacotron_inference_command.rs | 0 .../tacotron_model_check_command.rs | 0 .../talknet_model_check_command.rs | 0 .../wav2lip_inference_command.rs | 0 .../wav2lip_process_upload_command.rs | 0 .../storyteller_web/src}/server_state.rs | 0 .../storyteller_web/src}/shared_constants.rs | 0 .../src}/threads/ip_banlist_set.rs | 0 .../storyteller_web/src}/threads/mod.rs | 0 .../src}/threads/poll_ip_banlist_thread.rs | 0 .../storyteller_web/src}/tts_download_job.rs | 0 .../storyteller_web/src}/tts_inference_job.rs | 0 .../src}/util/anyhow_result.rs | 0 .../src}/util/buckets/bucket_client.rs | 0 .../src}/util/buckets/bucket_path_unifier.rs | 0 .../src}/util/buckets/bucket_paths.rs | 0 .../storyteller_web/src}/util/buckets/mod.rs | 0 .../storyteller_web/src}/util/caching/mod.rs | 0 .../util/caching/single_item_ttl_cache.rs | 0 .../src}/util/delete_role_disambiguation.rs | 0 .../src}/util/email_to_gravatar.rs | 0 .../src}/util/encrypted_sort_id.rs | 0 .../storyteller_web/src}/util/filesystem.rs | 0 .../src}/util/hashing/hash_file_sha2.rs | 0 .../src}/util/hashing/hash_string_sha2.rs | 0 .../storyteller_web/src}/util/hashing/mod.rs | 0 .../src}/util/hashing/sha256_digest.rs | 0 .../src}/util/jobs/cache_miss_strategizer.rs | 0 .../util/jobs/cache_miss_strategizer_multi.rs | 0 .../storyteller_web/src}/util/jobs/mod.rs | 0 .../src}/util/jobs/virtual_lfu_cache.rs | 0 .../src}/util/markdown_to_html.rs | 0 {src => app/storyteller_web/src}/util/mod.rs | 0 .../storyteller_web/src}/util/noop_logger.rs | 0 .../src}/util/random_crockford_token.rs | 0 .../util/random_prefix_crockford_token.rs | 0 .../storyteller_web/src}/util/random_uuid.rs | 0 .../storyteller_web/src}/util/redis/mod.rs | 0 .../util/redis/redis_job_status_logger.rs | 0 .../src}/util/redis/redis_keys.rs | 0 .../src}/util/semi_persistent_cache_dir.rs | 0 .../src}/validations/cashapp_username.rs | 0 .../src}/validations/check_for_slurs.rs | 0 .../src}/validations/discord_username.rs | 0 .../src}/validations/github_username.rs | 0 .../src}/validations/ip_addresses.rs | 0 .../storyteller_web/src}/validations/mod.rs | 0 .../src}/validations/model_uploads.rs | 0 .../src}/validations/passwords.rs | 0 .../src}/validations/twitch_username.rs | 0 .../src}/validations/twitter_username.rs | 0 .../src}/validations/username.rs | 0 .../src}/validations/username_reservations.rs | 0 .../src}/validations/website_url.rs | 0 .../storyteller_web/src}/w2l_download_job.rs | 0 .../storyteller_web/src}/w2l_inference_job.rs | 0 199 files changed, 173 insertions(+), 135 deletions(-) create mode 100644 app/obs_gateway/Cargo.toml create mode 100644 app/obs_gateway/src/main.rs create mode 100644 app/storyteller_web/Cargo.toml rename {src => app/storyteller_web/src}/common_env.rs (100%) rename {src => app/storyteller_web/src}/database/enums/mod.rs (100%) rename {src => app/storyteller_web/src}/database/enums/record_visibility.rs (100%) rename {src => app/storyteller_web/src}/database/enums/tts_model_type.rs (100%) rename {src => app/storyteller_web/src}/database/enums/vocoder_type.rs (100%) rename {src => app/storyteller_web/src}/database/helpers/boolean_converters.rs (100%) rename {src => app/storyteller_web/src}/database/helpers/enums.rs (100%) rename {src => app/storyteller_web/src}/database/helpers/mod.rs (100%) rename {src => app/storyteller_web/src}/database/helpers/tokens.rs (100%) rename {src => app/storyteller_web/src}/database/mediators/badge_granter.rs (100%) rename {src => app/storyteller_web/src}/database/mediators/firehose_publisher.rs (100%) rename {src => app/storyteller_web/src}/database/mediators/mod.rs (100%) rename {src => app/storyteller_web/src}/database/mod.rs (100%) rename {src => app/storyteller_web/src}/database/queries/badges.rs (100%) rename {src => app/storyteller_web/src}/database/queries/calculate_tts_model_leaderboard.rs (100%) rename {src => app/storyteller_web/src}/database/queries/calculate_w2l_template_leaderboard.rs (100%) rename {src => app/storyteller_web/src}/database/queries/create_session.rs (100%) rename {src => app/storyteller_web/src}/database/queries/delete_tts_model.rs (100%) rename {src => app/storyteller_web/src}/database/queries/delete_tts_result.rs (100%) rename {src => app/storyteller_web/src}/database/queries/delete_w2l_result.rs (100%) rename {src => app/storyteller_web/src}/database/queries/delete_w2l_template.rs (100%) rename {src => app/storyteller_web/src}/database/queries/get_user_profile_by_username.rs (100%) rename {src => app/storyteller_web/src}/database/queries/list_ip_bans.rs (100%) rename {src => app/storyteller_web/src}/database/queries/list_tts_models.rs (100%) rename {src => app/storyteller_web/src}/database/queries/list_user_badges.rs (100%) rename {src => app/storyteller_web/src}/database/queries/list_user_roles.rs (100%) rename {src => app/storyteller_web/src}/database/queries/list_w2l_templates.rs (100%) rename {src => app/storyteller_web/src}/database/queries/mod.rs (100%) rename {src => app/storyteller_web/src}/database/queries/query_tts_model.rs (100%) rename {src => app/storyteller_web/src}/database/queries/query_tts_result.rs (100%) rename {src => app/storyteller_web/src}/database/queries/query_w2l_result.rs (100%) rename {src => app/storyteller_web/src}/database/queries/query_w2l_template.rs (100%) rename {src => app/storyteller_web/src}/database/query_builders/list_tts_inference_results_query_builder.rs (100%) rename {src => app/storyteller_web/src}/database/query_builders/list_users_query_builder.rs (100%) rename {src => app/storyteller_web/src}/database/query_builders/list_w2l_inference_results_query_builder.rs (100%) rename {src => app/storyteller_web/src}/database/query_builders/mod.rs (100%) rename {src => app/storyteller_web/src}/http_clients/mod.rs (100%) rename {src => app/storyteller_web/src}/http_clients/tts_inference_sidecar_client.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/default_route_404.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/events/list_events.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/events/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/leaderboard/get_leaderboard.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/leaderboard/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/misc/alpha_cookie.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/misc/enable_alpha_easy_handler.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/misc/enable_alpha_handler.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/misc/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/approval/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/approval/pending_w2l_templates.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/ip_bans/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/jobs/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/stats/get_voice_count_stats.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/stats/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/user_bans/ban_user.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/user_bans/list_banned_users.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/user_bans/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/user_roles/list_roles.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/user_roles/list_staff.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/user_roles/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/user_roles/set_user_role.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/users/list_users.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/moderation/users/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/root_index.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/delete_tts_model.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/delete_tts_result.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/edit_tts_model.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/edit_tts_result.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/enqueue_infer_tts.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/enqueue_upload_tts_model.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/get_tts_inference_job_status.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/get_tts_model.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/get_tts_model_use_count.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/get_tts_result.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/get_tts_upload_model_job_status.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/list_tts_models.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/tts/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/twitch/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/twitch/pubsub_gateway.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/users/create_account.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/users/edit_profile.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/users/get_profile.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/users/list_user_tts_inference_results.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/users/list_user_tts_models.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/users/list_user_w2l_inference_results.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/users/list_user_w2l_templates.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/users/login.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/users/logout.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/users/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/users/session_info.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/delete_w2l_result.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/delete_w2l_template.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/edit_w2l_result.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/edit_w2l_template.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/enqueue_infer_w2l.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/get_w2l_inference_job_status.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/get_w2l_result.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/get_w2l_template.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/get_w2l_template_use_count.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/list_w2l_templates.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs (100%) rename {src => app/storyteller_web/src}/http_server/middleware/ip_filter_middleware.rs (100%) rename {src => app/storyteller_web/src}/http_server/middleware/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/web_utils/cookie_manager.rs (100%) rename {src => app/storyteller_web/src}/http_server/web_utils/get_host_header.rs (100%) rename {src => app/storyteller_web/src}/http_server/web_utils/ip_address.rs (100%) rename {src => app/storyteller_web/src}/http_server/web_utils/mod.rs (100%) rename {src => app/storyteller_web/src}/http_server/web_utils/read_multipart_field_bytes.rs (100%) rename {src => app/storyteller_web/src}/http_server/web_utils/redis_rate_limiter.rs (100%) rename {src => app/storyteller_web/src}/http_server/web_utils/response_error_helpers.rs (100%) rename {src => app/storyteller_web/src}/http_server/web_utils/response_success_helpers.rs (100%) rename {src => app/storyteller_web/src}/http_server/web_utils/serialize_as_json_error.rs (100%) rename {src => app/storyteller_web/src}/http_server/web_utils/session_checker.rs (100%) rename {src => app/storyteller_web/src}/job_queries/mod.rs (100%) rename {src => app/storyteller_web/src}/job_queries/tts_download_job_queries.rs (100%) rename {src => app/storyteller_web/src}/job_queries/tts_inference_job_queries.rs (100%) rename {src => app/storyteller_web/src}/job_queries/w2l_download_job_queries.rs (100%) rename {src => app/storyteller_web/src}/job_queries/w2l_inference_job_queries.rs (100%) rename {src => app/storyteller_web/src}/main.rs (98%) rename {src => app/storyteller_web/src}/obs_gateway/mod.rs (100%) rename {src => app/storyteller_web/src}/obs_gateway_server.rs (100%) rename {src => app/storyteller_web/src}/script_execution/ffmpeg_generate_preview_image_command.rs (100%) rename {src => app/storyteller_web/src}/script_execution/ffmpeg_generate_preview_video_command.rs (100%) rename {src => app/storyteller_web/src}/script_execution/google_drive_download_command.rs (100%) rename {src => app/storyteller_web/src}/script_execution/imagemagick_generate_preview_image_command.rs (100%) rename {src => app/storyteller_web/src}/script_execution/mod.rs (100%) rename {src => app/storyteller_web/src}/script_execution/tacotron_inference_command.rs (100%) rename {src => app/storyteller_web/src}/script_execution/tacotron_model_check_command.rs (100%) rename {src => app/storyteller_web/src}/script_execution/talknet_model_check_command.rs (100%) rename {src => app/storyteller_web/src}/script_execution/wav2lip_inference_command.rs (100%) rename {src => app/storyteller_web/src}/script_execution/wav2lip_process_upload_command.rs (100%) rename {src => app/storyteller_web/src}/server_state.rs (100%) rename {src => app/storyteller_web/src}/shared_constants.rs (100%) rename {src => app/storyteller_web/src}/threads/ip_banlist_set.rs (100%) rename {src => app/storyteller_web/src}/threads/mod.rs (100%) rename {src => app/storyteller_web/src}/threads/poll_ip_banlist_thread.rs (100%) rename {src => app/storyteller_web/src}/tts_download_job.rs (100%) rename {src => app/storyteller_web/src}/tts_inference_job.rs (100%) rename {src => app/storyteller_web/src}/util/anyhow_result.rs (100%) rename {src => app/storyteller_web/src}/util/buckets/bucket_client.rs (100%) rename {src => app/storyteller_web/src}/util/buckets/bucket_path_unifier.rs (100%) rename {src => app/storyteller_web/src}/util/buckets/bucket_paths.rs (100%) rename {src => app/storyteller_web/src}/util/buckets/mod.rs (100%) rename {src => app/storyteller_web/src}/util/caching/mod.rs (100%) rename {src => app/storyteller_web/src}/util/caching/single_item_ttl_cache.rs (100%) rename {src => app/storyteller_web/src}/util/delete_role_disambiguation.rs (100%) rename {src => app/storyteller_web/src}/util/email_to_gravatar.rs (100%) rename {src => app/storyteller_web/src}/util/encrypted_sort_id.rs (100%) rename {src => app/storyteller_web/src}/util/filesystem.rs (100%) rename {src => app/storyteller_web/src}/util/hashing/hash_file_sha2.rs (100%) rename {src => app/storyteller_web/src}/util/hashing/hash_string_sha2.rs (100%) rename {src => app/storyteller_web/src}/util/hashing/mod.rs (100%) rename {src => app/storyteller_web/src}/util/hashing/sha256_digest.rs (100%) rename {src => app/storyteller_web/src}/util/jobs/cache_miss_strategizer.rs (100%) rename {src => app/storyteller_web/src}/util/jobs/cache_miss_strategizer_multi.rs (100%) rename {src => app/storyteller_web/src}/util/jobs/mod.rs (100%) rename {src => app/storyteller_web/src}/util/jobs/virtual_lfu_cache.rs (100%) rename {src => app/storyteller_web/src}/util/markdown_to_html.rs (100%) rename {src => app/storyteller_web/src}/util/mod.rs (100%) rename {src => app/storyteller_web/src}/util/noop_logger.rs (100%) rename {src => app/storyteller_web/src}/util/random_crockford_token.rs (100%) rename {src => app/storyteller_web/src}/util/random_prefix_crockford_token.rs (100%) rename {src => app/storyteller_web/src}/util/random_uuid.rs (100%) rename {src => app/storyteller_web/src}/util/redis/mod.rs (100%) rename {src => app/storyteller_web/src}/util/redis/redis_job_status_logger.rs (100%) rename {src => app/storyteller_web/src}/util/redis/redis_keys.rs (100%) rename {src => app/storyteller_web/src}/util/semi_persistent_cache_dir.rs (100%) rename {src => app/storyteller_web/src}/validations/cashapp_username.rs (100%) rename {src => app/storyteller_web/src}/validations/check_for_slurs.rs (100%) rename {src => app/storyteller_web/src}/validations/discord_username.rs (100%) rename {src => app/storyteller_web/src}/validations/github_username.rs (100%) rename {src => app/storyteller_web/src}/validations/ip_addresses.rs (100%) rename {src => app/storyteller_web/src}/validations/mod.rs (100%) rename {src => app/storyteller_web/src}/validations/model_uploads.rs (100%) rename {src => app/storyteller_web/src}/validations/passwords.rs (100%) rename {src => app/storyteller_web/src}/validations/twitch_username.rs (100%) rename {src => app/storyteller_web/src}/validations/twitter_username.rs (100%) rename {src => app/storyteller_web/src}/validations/username.rs (100%) rename {src => app/storyteller_web/src}/validations/username_reservations.rs (100%) rename {src => app/storyteller_web/src}/validations/website_url.rs (100%) rename {src => app/storyteller_web/src}/w2l_download_job.rs (100%) rename {src => app/storyteller_web/src}/w2l_inference_job.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index b194588ab9..2a3988d4cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2222,12 +2222,16 @@ name = "newrelic-telemetry" version = "0.1.0" dependencies = [ "anyhow", + "env_logger", "flate2", + "futures 0.3.15", "hyper", "hyper-tls", "log", + "rand 0.7.3", "serde", "serde_json", + "tokio 0.2.25", "uuid", ] @@ -2344,6 +2348,10 @@ dependencies = [ "libc", ] +[[package]] +name = "obs-gateway-server" +version = "0.0.1" + [[package]] name = "once_cell" version = "1.7.2" diff --git a/Cargo.toml b/Cargo.toml index b46d2161a5..6c5e053d99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,133 +1,6 @@ -[package] -name = "storyteller-web" -edition = "2018" -version = "0.0.1" -authors = [ - "Brandon Thomas ", - "Brandon Thomas ", +[workspace] + +members = [ + "app/obs_gateway", + "app/storyteller_web", ] -publish = false - -# Main server monolith -[[bin]] -name = "storyteller-web" -path = "src/main.rs" - -# OBS websocket server -[[bin]] -name = "obs-gateway-server" -path = "src/obs_gateway_server.rs" - -# Jobs for managing downloads and inference -[[bin]] -name = "tts-download-job" -path = "src/tts_download_job.rs" - -[[bin]] -name = "tts-inference-job" -path = "src/tts_inference_job.rs" - -[[bin]] -name = "w2l-download-job" -path = "src/w2l_download_job.rs" - -[[bin]] -name = "w2l-inference-job" -path = "src/w2l_inference_job.rs" - -[dependencies] -anyhow = "1.0.40" -base64 = "0.13.0" -chrono = { version = "0.4.19", features = ["serde"] } -derive_more = "0.99.14" -dotenv = "0.15.0" -easyenv = "0.0.6" -futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool -futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate -hex = "0.4.3" -hostname = "0.3.1" -lazy_static = "1.4.0" -lfu = "0.2.5" -log = "0.4.14" -#newrelic = "0.2.2" -newrelic-telemetry = { path = "vendor/newrelic-telemetry-sdk-rust" } -rand = "0.8.3" -regex = "1.5.4" -time = "0.2.26" -uuid = { version = "0.8.2", features = ["v4"] } - -# Runtime -# NB(1): We switched to Tokio from `async-std` for broader ecosystem support. -#async-std = { version = "1.6", features = [ "attributes" ] } -# NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: -# https://github.com/durch/rust-s3/issues/198 -#tokio = { version = "1.6.0", features = ["macros"] } -tokio = { version = "0.2.25", features = ["macros"] } - -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 -#sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } -# NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade -#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } -sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } - -# Redis Database -limitation = "0.1.1" # Redis rate limiter -r2d2_redis = "0.14.0" - -# In-memory caching -lru_time_cache = "0.11.11" - -# Files -data-encoding = "2.3.2" -ring = "0.17.0-alpha.10" -# NB: This is the source of all of our problems. `rust-s3` can't upload to object names with directories -# in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2 -#rust-s3 = "0.27.0-rc4" -#rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } -rust-s3 = { version = "0.26" } -infer = "0.5.0" - -subprocess = "0.2.6" -tempdir = "0.3.7" - -# Web -actix = "0.10" -#actix-web = "3" -actix-web-actors = "3" - -actix-cors = "0.5.4" -actix-files= "0.5" -actix-http = "2.2.0" -actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade -actix-rt = "2.2.0" -actix-service = "1.0.6" -actix-utils = "3.0.0" -actix-web = "3.3.2" -jwt = "0.13.0" - -# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime -# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. -hyper = { version = "0.13.10" } - -# Crypto -bcrypt = "0.8" # password hashes -hmac = "0.10.0" # cookies -magic-crypt = "3.1.8" # encrypting sort IDs for the frontend so we don't leak info -md-5 = "0.9.1" # gravatar -sha2 = "0.9.5" # file hashes, inference text hashes, etc. - -# Serialization -serde = "1.0.125" -serde_derive = "1.0.125" -serde_json = "1.0.64" -toml = "0.5.8" - -# Text processing -pulldown-cmark = "0.8.0" - -[build-dependencies] -# NB: Only using diesel for migrations support -diesel = { version = "1.4.6", features = ["mysql"] } diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml new file mode 100644 index 0000000000..377d0d234c --- /dev/null +++ b/app/obs_gateway/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "obs-gateway-server" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[[bin]] +name = "obs-gateway-server" +path = "src/main.rs" diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs new file mode 100644 index 0000000000..c4357a72be --- /dev/null +++ b/app/obs_gateway/src/main.rs @@ -0,0 +1,4 @@ + +pub fn main() { + println!("TBD"); +} \ No newline at end of file diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml new file mode 100644 index 0000000000..4407bc2745 --- /dev/null +++ b/app/storyteller_web/Cargo.toml @@ -0,0 +1,133 @@ +[package] +name = "storyteller-web" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +# Main server monolith +[[bin]] +name = "storyteller-web" +path = "src/main.rs" + +## OBS websocket server +#[[bin]] +#name = "obs-gateway-server" +#path = "src/obs_gateway_server.rs" + +# Jobs for managing downloads and inference +[[bin]] +name = "tts-download-job" +path = "src/tts_download_job.rs" + +[[bin]] +name = "tts-inference-job" +path = "src/tts_inference_job.rs" + +[[bin]] +name = "w2l-download-job" +path = "src/w2l_download_job.rs" + +[[bin]] +name = "w2l-inference-job" +path = "src/w2l_inference_job.rs" + +[dependencies] +anyhow = "1.0.40" +base64 = "0.13.0" +chrono = { version = "0.4.19", features = ["serde"] } +derive_more = "0.99.14" +dotenv = "0.15.0" +easyenv = "0.0.6" +futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool +futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate +hex = "0.4.3" +hostname = "0.3.1" +lazy_static = "1.4.0" +lfu = "0.2.5" +log = "0.4.14" +#newrelic = "0.2.2" +newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" } +rand = "0.8.3" +regex = "1.5.4" +time = "0.2.26" +uuid = { version = "0.8.2", features = ["v4"] } + +# Runtime +# NB(1): We switched to Tokio from `async-std` for broader ecosystem support. +#async-std = { version = "1.6", features = [ "attributes" ] } +# NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: +# https://github.com/durch/rust-s3/issues/198 +#tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "0.2.25", features = ["macros"] } + +# MySql Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +#sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } +# NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade +#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } +sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } + +# Redis Database +limitation = "0.1.1" # Redis rate limiter +r2d2_redis = "0.14.0" + +# In-memory caching +lru_time_cache = "0.11.11" + +# Files +data-encoding = "2.3.2" +ring = "0.17.0-alpha.10" +# NB: This is the source of all of our problems. `rust-s3` can't upload to object names with directories +# in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2 +#rust-s3 = "0.27.0-rc4" +#rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } +rust-s3 = { version = "0.26" } +infer = "0.5.0" + +subprocess = "0.2.6" +tempdir = "0.3.7" + +# Web +actix = "0.10" +#actix-web = "3" +actix-web-actors = "3" + +actix-cors = "0.5.4" +actix-files= "0.5" +actix-http = "2.2.0" +actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade +actix-rt = "2.2.0" +actix-service = "1.0.6" +actix-utils = "3.0.0" +actix-web = "3.3.2" +jwt = "0.13.0" + +# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime +# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. +hyper = { version = "0.13.10" } + +# Crypto +bcrypt = "0.8" # password hashes +hmac = "0.10.0" # cookies +magic-crypt = "3.1.8" # encrypting sort IDs for the frontend so we don't leak info +md-5 = "0.9.1" # gravatar +sha2 = "0.9.5" # file hashes, inference text hashes, etc. + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" +toml = "0.5.8" + +# Text processing +pulldown-cmark = "0.8.0" + +[build-dependencies] +# NB: Only using diesel for migrations support +diesel = { version = "1.4.6", features = ["mysql"] } diff --git a/src/common_env.rs b/app/storyteller_web/src/common_env.rs similarity index 100% rename from src/common_env.rs rename to app/storyteller_web/src/common_env.rs diff --git a/src/database/enums/mod.rs b/app/storyteller_web/src/database/enums/mod.rs similarity index 100% rename from src/database/enums/mod.rs rename to app/storyteller_web/src/database/enums/mod.rs diff --git a/src/database/enums/record_visibility.rs b/app/storyteller_web/src/database/enums/record_visibility.rs similarity index 100% rename from src/database/enums/record_visibility.rs rename to app/storyteller_web/src/database/enums/record_visibility.rs diff --git a/src/database/enums/tts_model_type.rs b/app/storyteller_web/src/database/enums/tts_model_type.rs similarity index 100% rename from src/database/enums/tts_model_type.rs rename to app/storyteller_web/src/database/enums/tts_model_type.rs diff --git a/src/database/enums/vocoder_type.rs b/app/storyteller_web/src/database/enums/vocoder_type.rs similarity index 100% rename from src/database/enums/vocoder_type.rs rename to app/storyteller_web/src/database/enums/vocoder_type.rs diff --git a/src/database/helpers/boolean_converters.rs b/app/storyteller_web/src/database/helpers/boolean_converters.rs similarity index 100% rename from src/database/helpers/boolean_converters.rs rename to app/storyteller_web/src/database/helpers/boolean_converters.rs diff --git a/src/database/helpers/enums.rs b/app/storyteller_web/src/database/helpers/enums.rs similarity index 100% rename from src/database/helpers/enums.rs rename to app/storyteller_web/src/database/helpers/enums.rs diff --git a/src/database/helpers/mod.rs b/app/storyteller_web/src/database/helpers/mod.rs similarity index 100% rename from src/database/helpers/mod.rs rename to app/storyteller_web/src/database/helpers/mod.rs diff --git a/src/database/helpers/tokens.rs b/app/storyteller_web/src/database/helpers/tokens.rs similarity index 100% rename from src/database/helpers/tokens.rs rename to app/storyteller_web/src/database/helpers/tokens.rs diff --git a/src/database/mediators/badge_granter.rs b/app/storyteller_web/src/database/mediators/badge_granter.rs similarity index 100% rename from src/database/mediators/badge_granter.rs rename to app/storyteller_web/src/database/mediators/badge_granter.rs diff --git a/src/database/mediators/firehose_publisher.rs b/app/storyteller_web/src/database/mediators/firehose_publisher.rs similarity index 100% rename from src/database/mediators/firehose_publisher.rs rename to app/storyteller_web/src/database/mediators/firehose_publisher.rs diff --git a/src/database/mediators/mod.rs b/app/storyteller_web/src/database/mediators/mod.rs similarity index 100% rename from src/database/mediators/mod.rs rename to app/storyteller_web/src/database/mediators/mod.rs diff --git a/src/database/mod.rs b/app/storyteller_web/src/database/mod.rs similarity index 100% rename from src/database/mod.rs rename to app/storyteller_web/src/database/mod.rs diff --git a/src/database/queries/badges.rs b/app/storyteller_web/src/database/queries/badges.rs similarity index 100% rename from src/database/queries/badges.rs rename to app/storyteller_web/src/database/queries/badges.rs diff --git a/src/database/queries/calculate_tts_model_leaderboard.rs b/app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs similarity index 100% rename from src/database/queries/calculate_tts_model_leaderboard.rs rename to app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs diff --git a/src/database/queries/calculate_w2l_template_leaderboard.rs b/app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs similarity index 100% rename from src/database/queries/calculate_w2l_template_leaderboard.rs rename to app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs diff --git a/src/database/queries/create_session.rs b/app/storyteller_web/src/database/queries/create_session.rs similarity index 100% rename from src/database/queries/create_session.rs rename to app/storyteller_web/src/database/queries/create_session.rs diff --git a/src/database/queries/delete_tts_model.rs b/app/storyteller_web/src/database/queries/delete_tts_model.rs similarity index 100% rename from src/database/queries/delete_tts_model.rs rename to app/storyteller_web/src/database/queries/delete_tts_model.rs diff --git a/src/database/queries/delete_tts_result.rs b/app/storyteller_web/src/database/queries/delete_tts_result.rs similarity index 100% rename from src/database/queries/delete_tts_result.rs rename to app/storyteller_web/src/database/queries/delete_tts_result.rs diff --git a/src/database/queries/delete_w2l_result.rs b/app/storyteller_web/src/database/queries/delete_w2l_result.rs similarity index 100% rename from src/database/queries/delete_w2l_result.rs rename to app/storyteller_web/src/database/queries/delete_w2l_result.rs diff --git a/src/database/queries/delete_w2l_template.rs b/app/storyteller_web/src/database/queries/delete_w2l_template.rs similarity index 100% rename from src/database/queries/delete_w2l_template.rs rename to app/storyteller_web/src/database/queries/delete_w2l_template.rs diff --git a/src/database/queries/get_user_profile_by_username.rs b/app/storyteller_web/src/database/queries/get_user_profile_by_username.rs similarity index 100% rename from src/database/queries/get_user_profile_by_username.rs rename to app/storyteller_web/src/database/queries/get_user_profile_by_username.rs diff --git a/src/database/queries/list_ip_bans.rs b/app/storyteller_web/src/database/queries/list_ip_bans.rs similarity index 100% rename from src/database/queries/list_ip_bans.rs rename to app/storyteller_web/src/database/queries/list_ip_bans.rs diff --git a/src/database/queries/list_tts_models.rs b/app/storyteller_web/src/database/queries/list_tts_models.rs similarity index 100% rename from src/database/queries/list_tts_models.rs rename to app/storyteller_web/src/database/queries/list_tts_models.rs diff --git a/src/database/queries/list_user_badges.rs b/app/storyteller_web/src/database/queries/list_user_badges.rs similarity index 100% rename from src/database/queries/list_user_badges.rs rename to app/storyteller_web/src/database/queries/list_user_badges.rs diff --git a/src/database/queries/list_user_roles.rs b/app/storyteller_web/src/database/queries/list_user_roles.rs similarity index 100% rename from src/database/queries/list_user_roles.rs rename to app/storyteller_web/src/database/queries/list_user_roles.rs diff --git a/src/database/queries/list_w2l_templates.rs b/app/storyteller_web/src/database/queries/list_w2l_templates.rs similarity index 100% rename from src/database/queries/list_w2l_templates.rs rename to app/storyteller_web/src/database/queries/list_w2l_templates.rs diff --git a/src/database/queries/mod.rs b/app/storyteller_web/src/database/queries/mod.rs similarity index 100% rename from src/database/queries/mod.rs rename to app/storyteller_web/src/database/queries/mod.rs diff --git a/src/database/queries/query_tts_model.rs b/app/storyteller_web/src/database/queries/query_tts_model.rs similarity index 100% rename from src/database/queries/query_tts_model.rs rename to app/storyteller_web/src/database/queries/query_tts_model.rs diff --git a/src/database/queries/query_tts_result.rs b/app/storyteller_web/src/database/queries/query_tts_result.rs similarity index 100% rename from src/database/queries/query_tts_result.rs rename to app/storyteller_web/src/database/queries/query_tts_result.rs diff --git a/src/database/queries/query_w2l_result.rs b/app/storyteller_web/src/database/queries/query_w2l_result.rs similarity index 100% rename from src/database/queries/query_w2l_result.rs rename to app/storyteller_web/src/database/queries/query_w2l_result.rs diff --git a/src/database/queries/query_w2l_template.rs b/app/storyteller_web/src/database/queries/query_w2l_template.rs similarity index 100% rename from src/database/queries/query_w2l_template.rs rename to app/storyteller_web/src/database/queries/query_w2l_template.rs diff --git a/src/database/query_builders/list_tts_inference_results_query_builder.rs b/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs similarity index 100% rename from src/database/query_builders/list_tts_inference_results_query_builder.rs rename to app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs diff --git a/src/database/query_builders/list_users_query_builder.rs b/app/storyteller_web/src/database/query_builders/list_users_query_builder.rs similarity index 100% rename from src/database/query_builders/list_users_query_builder.rs rename to app/storyteller_web/src/database/query_builders/list_users_query_builder.rs diff --git a/src/database/query_builders/list_w2l_inference_results_query_builder.rs b/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs similarity index 100% rename from src/database/query_builders/list_w2l_inference_results_query_builder.rs rename to app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs diff --git a/src/database/query_builders/mod.rs b/app/storyteller_web/src/database/query_builders/mod.rs similarity index 100% rename from src/database/query_builders/mod.rs rename to app/storyteller_web/src/database/query_builders/mod.rs diff --git a/src/http_clients/mod.rs b/app/storyteller_web/src/http_clients/mod.rs similarity index 100% rename from src/http_clients/mod.rs rename to app/storyteller_web/src/http_clients/mod.rs diff --git a/src/http_clients/tts_inference_sidecar_client.rs b/app/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs similarity index 100% rename from src/http_clients/tts_inference_sidecar_client.rs rename to app/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs diff --git a/src/http_server/endpoints/default_route_404.rs b/app/storyteller_web/src/http_server/endpoints/default_route_404.rs similarity index 100% rename from src/http_server/endpoints/default_route_404.rs rename to app/storyteller_web/src/http_server/endpoints/default_route_404.rs diff --git a/src/http_server/endpoints/events/list_events.rs b/app/storyteller_web/src/http_server/endpoints/events/list_events.rs similarity index 100% rename from src/http_server/endpoints/events/list_events.rs rename to app/storyteller_web/src/http_server/endpoints/events/list_events.rs diff --git a/src/http_server/endpoints/events/mod.rs b/app/storyteller_web/src/http_server/endpoints/events/mod.rs similarity index 100% rename from src/http_server/endpoints/events/mod.rs rename to app/storyteller_web/src/http_server/endpoints/events/mod.rs diff --git a/src/http_server/endpoints/leaderboard/get_leaderboard.rs b/app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs similarity index 100% rename from src/http_server/endpoints/leaderboard/get_leaderboard.rs rename to app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs diff --git a/src/http_server/endpoints/leaderboard/mod.rs b/app/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs similarity index 100% rename from src/http_server/endpoints/leaderboard/mod.rs rename to app/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs diff --git a/src/http_server/endpoints/misc/alpha_cookie.rs b/app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs similarity index 100% rename from src/http_server/endpoints/misc/alpha_cookie.rs rename to app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs diff --git a/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs b/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs similarity index 100% rename from src/http_server/endpoints/misc/enable_alpha_easy_handler.rs rename to app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs diff --git a/src/http_server/endpoints/misc/enable_alpha_handler.rs b/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs similarity index 100% rename from src/http_server/endpoints/misc/enable_alpha_handler.rs rename to app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs diff --git a/src/http_server/endpoints/misc/mod.rs b/app/storyteller_web/src/http_server/endpoints/misc/mod.rs similarity index 100% rename from src/http_server/endpoints/misc/mod.rs rename to app/storyteller_web/src/http_server/endpoints/misc/mod.rs diff --git a/src/http_server/endpoints/mod.rs b/app/storyteller_web/src/http_server/endpoints/mod.rs similarity index 100% rename from src/http_server/endpoints/mod.rs rename to app/storyteller_web/src/http_server/endpoints/mod.rs diff --git a/src/http_server/endpoints/moderation/approval/mod.rs b/app/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs similarity index 100% rename from src/http_server/endpoints/moderation/approval/mod.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs diff --git a/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs b/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs similarity index 100% rename from src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs diff --git a/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs similarity index 100% rename from src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs diff --git a/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs similarity index 100% rename from src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs diff --git a/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs similarity index 100% rename from src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs diff --git a/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs similarity index 100% rename from src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs diff --git a/src/http_server/endpoints/moderation/ip_bans/mod.rs b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs similarity index 100% rename from src/http_server/endpoints/moderation/ip_bans/mod.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs diff --git a/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs b/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs similarity index 100% rename from src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs diff --git a/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs b/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs similarity index 100% rename from src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs diff --git a/src/http_server/endpoints/moderation/jobs/mod.rs b/app/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs similarity index 100% rename from src/http_server/endpoints/moderation/jobs/mod.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs diff --git a/src/http_server/endpoints/moderation/mod.rs b/app/storyteller_web/src/http_server/endpoints/moderation/mod.rs similarity index 100% rename from src/http_server/endpoints/moderation/mod.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/mod.rs diff --git a/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs b/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs similarity index 100% rename from src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs diff --git a/src/http_server/endpoints/moderation/stats/mod.rs b/app/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs similarity index 100% rename from src/http_server/endpoints/moderation/stats/mod.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs diff --git a/src/http_server/endpoints/moderation/user_bans/ban_user.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs similarity index 100% rename from src/http_server/endpoints/moderation/user_bans/ban_user.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs diff --git a/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs similarity index 100% rename from src/http_server/endpoints/moderation/user_bans/list_banned_users.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs diff --git a/src/http_server/endpoints/moderation/user_bans/mod.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs similarity index 100% rename from src/http_server/endpoints/moderation/user_bans/mod.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs diff --git a/src/http_server/endpoints/moderation/user_roles/list_roles.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs similarity index 100% rename from src/http_server/endpoints/moderation/user_roles/list_roles.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs diff --git a/src/http_server/endpoints/moderation/user_roles/list_staff.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs similarity index 100% rename from src/http_server/endpoints/moderation/user_roles/list_staff.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs diff --git a/src/http_server/endpoints/moderation/user_roles/mod.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs similarity index 100% rename from src/http_server/endpoints/moderation/user_roles/mod.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs diff --git a/src/http_server/endpoints/moderation/user_roles/set_user_role.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs similarity index 100% rename from src/http_server/endpoints/moderation/user_roles/set_user_role.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs diff --git a/src/http_server/endpoints/moderation/users/list_users.rs b/app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs similarity index 100% rename from src/http_server/endpoints/moderation/users/list_users.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs diff --git a/src/http_server/endpoints/moderation/users/mod.rs b/app/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs similarity index 100% rename from src/http_server/endpoints/moderation/users/mod.rs rename to app/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs diff --git a/src/http_server/endpoints/root_index.rs b/app/storyteller_web/src/http_server/endpoints/root_index.rs similarity index 100% rename from src/http_server/endpoints/root_index.rs rename to app/storyteller_web/src/http_server/endpoints/root_index.rs diff --git a/src/http_server/endpoints/tts/delete_tts_model.rs b/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs similarity index 100% rename from src/http_server/endpoints/tts/delete_tts_model.rs rename to app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs diff --git a/src/http_server/endpoints/tts/delete_tts_result.rs b/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs similarity index 100% rename from src/http_server/endpoints/tts/delete_tts_result.rs rename to app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs diff --git a/src/http_server/endpoints/tts/edit_tts_model.rs b/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs similarity index 100% rename from src/http_server/endpoints/tts/edit_tts_model.rs rename to app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs diff --git a/src/http_server/endpoints/tts/edit_tts_result.rs b/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs similarity index 100% rename from src/http_server/endpoints/tts/edit_tts_result.rs rename to app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs similarity index 100% rename from src/http_server/endpoints/tts/enqueue_infer_tts.rs rename to app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs similarity index 100% rename from src/http_server/endpoints/tts/enqueue_upload_tts_model.rs rename to app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs diff --git a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs similarity index 100% rename from src/http_server/endpoints/tts/get_tts_inference_job_status.rs rename to app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs diff --git a/src/http_server/endpoints/tts/get_tts_model.rs b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs similarity index 100% rename from src/http_server/endpoints/tts/get_tts_model.rs rename to app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs diff --git a/src/http_server/endpoints/tts/get_tts_model_use_count.rs b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs similarity index 100% rename from src/http_server/endpoints/tts/get_tts_model_use_count.rs rename to app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs diff --git a/src/http_server/endpoints/tts/get_tts_result.rs b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs similarity index 100% rename from src/http_server/endpoints/tts/get_tts_result.rs rename to app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs diff --git a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs similarity index 100% rename from src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs rename to app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs diff --git a/src/http_server/endpoints/tts/list_tts_models.rs b/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs similarity index 100% rename from src/http_server/endpoints/tts/list_tts_models.rs rename to app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs diff --git a/src/http_server/endpoints/tts/mod.rs b/app/storyteller_web/src/http_server/endpoints/tts/mod.rs similarity index 100% rename from src/http_server/endpoints/tts/mod.rs rename to app/storyteller_web/src/http_server/endpoints/tts/mod.rs diff --git a/src/http_server/endpoints/twitch/mod.rs b/app/storyteller_web/src/http_server/endpoints/twitch/mod.rs similarity index 100% rename from src/http_server/endpoints/twitch/mod.rs rename to app/storyteller_web/src/http_server/endpoints/twitch/mod.rs diff --git a/src/http_server/endpoints/twitch/pubsub_gateway.rs b/app/storyteller_web/src/http_server/endpoints/twitch/pubsub_gateway.rs similarity index 100% rename from src/http_server/endpoints/twitch/pubsub_gateway.rs rename to app/storyteller_web/src/http_server/endpoints/twitch/pubsub_gateway.rs diff --git a/src/http_server/endpoints/users/create_account.rs b/app/storyteller_web/src/http_server/endpoints/users/create_account.rs similarity index 100% rename from src/http_server/endpoints/users/create_account.rs rename to app/storyteller_web/src/http_server/endpoints/users/create_account.rs diff --git a/src/http_server/endpoints/users/edit_profile.rs b/app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs similarity index 100% rename from src/http_server/endpoints/users/edit_profile.rs rename to app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs diff --git a/src/http_server/endpoints/users/get_profile.rs b/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs similarity index 100% rename from src/http_server/endpoints/users/get_profile.rs rename to app/storyteller_web/src/http_server/endpoints/users/get_profile.rs diff --git a/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs similarity index 100% rename from src/http_server/endpoints/users/list_user_tts_inference_results.rs rename to app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs diff --git a/src/http_server/endpoints/users/list_user_tts_models.rs b/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs similarity index 100% rename from src/http_server/endpoints/users/list_user_tts_models.rs rename to app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs diff --git a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs similarity index 100% rename from src/http_server/endpoints/users/list_user_w2l_inference_results.rs rename to app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs diff --git a/src/http_server/endpoints/users/list_user_w2l_templates.rs b/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs similarity index 100% rename from src/http_server/endpoints/users/list_user_w2l_templates.rs rename to app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs diff --git a/src/http_server/endpoints/users/login.rs b/app/storyteller_web/src/http_server/endpoints/users/login.rs similarity index 100% rename from src/http_server/endpoints/users/login.rs rename to app/storyteller_web/src/http_server/endpoints/users/login.rs diff --git a/src/http_server/endpoints/users/logout.rs b/app/storyteller_web/src/http_server/endpoints/users/logout.rs similarity index 100% rename from src/http_server/endpoints/users/logout.rs rename to app/storyteller_web/src/http_server/endpoints/users/logout.rs diff --git a/src/http_server/endpoints/users/mod.rs b/app/storyteller_web/src/http_server/endpoints/users/mod.rs similarity index 100% rename from src/http_server/endpoints/users/mod.rs rename to app/storyteller_web/src/http_server/endpoints/users/mod.rs diff --git a/src/http_server/endpoints/users/session_info.rs b/app/storyteller_web/src/http_server/endpoints/users/session_info.rs similarity index 100% rename from src/http_server/endpoints/users/session_info.rs rename to app/storyteller_web/src/http_server/endpoints/users/session_info.rs diff --git a/src/http_server/endpoints/w2l/delete_w2l_result.rs b/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs similarity index 100% rename from src/http_server/endpoints/w2l/delete_w2l_result.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs diff --git a/src/http_server/endpoints/w2l/delete_w2l_template.rs b/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs similarity index 100% rename from src/http_server/endpoints/w2l/delete_w2l_template.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs diff --git a/src/http_server/endpoints/w2l/edit_w2l_result.rs b/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs similarity index 100% rename from src/http_server/endpoints/w2l/edit_w2l_result.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs diff --git a/src/http_server/endpoints/w2l/edit_w2l_template.rs b/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs similarity index 100% rename from src/http_server/endpoints/w2l/edit_w2l_template.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs similarity index 100% rename from src/http_server/endpoints/w2l/enqueue_infer_w2l.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs similarity index 100% rename from src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs similarity index 100% rename from src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs diff --git a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs similarity index 100% rename from src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs diff --git a/src/http_server/endpoints/w2l/get_w2l_result.rs b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs similarity index 100% rename from src/http_server/endpoints/w2l/get_w2l_result.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs similarity index 100% rename from src/http_server/endpoints/w2l/get_w2l_template.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs diff --git a/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs similarity index 100% rename from src/http_server/endpoints/w2l/get_w2l_template_use_count.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs diff --git a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs similarity index 100% rename from src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs diff --git a/src/http_server/endpoints/w2l/list_w2l_templates.rs b/app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs similarity index 100% rename from src/http_server/endpoints/w2l/list_w2l_templates.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs diff --git a/src/http_server/endpoints/w2l/mod.rs b/app/storyteller_web/src/http_server/endpoints/w2l/mod.rs similarity index 100% rename from src/http_server/endpoints/w2l/mod.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/mod.rs diff --git a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs similarity index 100% rename from src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs rename to app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs diff --git a/src/http_server/middleware/ip_filter_middleware.rs b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs similarity index 100% rename from src/http_server/middleware/ip_filter_middleware.rs rename to app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs diff --git a/src/http_server/middleware/mod.rs b/app/storyteller_web/src/http_server/middleware/mod.rs similarity index 100% rename from src/http_server/middleware/mod.rs rename to app/storyteller_web/src/http_server/middleware/mod.rs diff --git a/src/http_server/mod.rs b/app/storyteller_web/src/http_server/mod.rs similarity index 100% rename from src/http_server/mod.rs rename to app/storyteller_web/src/http_server/mod.rs diff --git a/src/http_server/web_utils/cookie_manager.rs b/app/storyteller_web/src/http_server/web_utils/cookie_manager.rs similarity index 100% rename from src/http_server/web_utils/cookie_manager.rs rename to app/storyteller_web/src/http_server/web_utils/cookie_manager.rs diff --git a/src/http_server/web_utils/get_host_header.rs b/app/storyteller_web/src/http_server/web_utils/get_host_header.rs similarity index 100% rename from src/http_server/web_utils/get_host_header.rs rename to app/storyteller_web/src/http_server/web_utils/get_host_header.rs diff --git a/src/http_server/web_utils/ip_address.rs b/app/storyteller_web/src/http_server/web_utils/ip_address.rs similarity index 100% rename from src/http_server/web_utils/ip_address.rs rename to app/storyteller_web/src/http_server/web_utils/ip_address.rs diff --git a/src/http_server/web_utils/mod.rs b/app/storyteller_web/src/http_server/web_utils/mod.rs similarity index 100% rename from src/http_server/web_utils/mod.rs rename to app/storyteller_web/src/http_server/web_utils/mod.rs diff --git a/src/http_server/web_utils/read_multipart_field_bytes.rs b/app/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs similarity index 100% rename from src/http_server/web_utils/read_multipart_field_bytes.rs rename to app/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs diff --git a/src/http_server/web_utils/redis_rate_limiter.rs b/app/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs similarity index 100% rename from src/http_server/web_utils/redis_rate_limiter.rs rename to app/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs diff --git a/src/http_server/web_utils/response_error_helpers.rs b/app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs similarity index 100% rename from src/http_server/web_utils/response_error_helpers.rs rename to app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs diff --git a/src/http_server/web_utils/response_success_helpers.rs b/app/storyteller_web/src/http_server/web_utils/response_success_helpers.rs similarity index 100% rename from src/http_server/web_utils/response_success_helpers.rs rename to app/storyteller_web/src/http_server/web_utils/response_success_helpers.rs diff --git a/src/http_server/web_utils/serialize_as_json_error.rs b/app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs similarity index 100% rename from src/http_server/web_utils/serialize_as_json_error.rs rename to app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs diff --git a/src/http_server/web_utils/session_checker.rs b/app/storyteller_web/src/http_server/web_utils/session_checker.rs similarity index 100% rename from src/http_server/web_utils/session_checker.rs rename to app/storyteller_web/src/http_server/web_utils/session_checker.rs diff --git a/src/job_queries/mod.rs b/app/storyteller_web/src/job_queries/mod.rs similarity index 100% rename from src/job_queries/mod.rs rename to app/storyteller_web/src/job_queries/mod.rs diff --git a/src/job_queries/tts_download_job_queries.rs b/app/storyteller_web/src/job_queries/tts_download_job_queries.rs similarity index 100% rename from src/job_queries/tts_download_job_queries.rs rename to app/storyteller_web/src/job_queries/tts_download_job_queries.rs diff --git a/src/job_queries/tts_inference_job_queries.rs b/app/storyteller_web/src/job_queries/tts_inference_job_queries.rs similarity index 100% rename from src/job_queries/tts_inference_job_queries.rs rename to app/storyteller_web/src/job_queries/tts_inference_job_queries.rs diff --git a/src/job_queries/w2l_download_job_queries.rs b/app/storyteller_web/src/job_queries/w2l_download_job_queries.rs similarity index 100% rename from src/job_queries/w2l_download_job_queries.rs rename to app/storyteller_web/src/job_queries/w2l_download_job_queries.rs diff --git a/src/job_queries/w2l_inference_job_queries.rs b/app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs similarity index 100% rename from src/job_queries/w2l_inference_job_queries.rs rename to app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs diff --git a/src/main.rs b/app/storyteller_web/src/main.rs similarity index 98% rename from src/main.rs rename to app/storyteller_web/src/main.rs index 2eeb5456ed..3a933b5675 100644 --- a/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -12,9 +12,9 @@ #[macro_use] extern crate magic_crypt; #[macro_use] extern crate serde_derive; -pub const RESERVED_USERNAMES : &'static str = include_str!("../db/reserved_usernames.txt"); -pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../db/reserved_usernames_including.txt"); -pub const BANNED_SLURS : &'static str = include_str!("../db/banned_slurs.txt"); +pub const RESERVED_USERNAMES : &'static str = include_str!("../../../db/reserved_usernames.txt"); +pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../../../db/reserved_usernames_including.txt"); +pub const BANNED_SLURS : &'static str = include_str!("../../../db/banned_slurs.txt"); pub mod database; pub mod http_clients; @@ -114,6 +114,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; +use crate::http_server::endpoints::twitch::pubsub_gateway::ws_index; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -660,6 +661,12 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(leaderboard_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + // Twitch + .service( + web::resource("/twitch") + .route(web::get().to(ws_index)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(get_root_index) .service(enable_alpha_handler) .service(enable_alpha_easy_handler) diff --git a/src/obs_gateway/mod.rs b/app/storyteller_web/src/obs_gateway/mod.rs similarity index 100% rename from src/obs_gateway/mod.rs rename to app/storyteller_web/src/obs_gateway/mod.rs diff --git a/src/obs_gateway_server.rs b/app/storyteller_web/src/obs_gateway_server.rs similarity index 100% rename from src/obs_gateway_server.rs rename to app/storyteller_web/src/obs_gateway_server.rs diff --git a/src/script_execution/ffmpeg_generate_preview_image_command.rs b/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs similarity index 100% rename from src/script_execution/ffmpeg_generate_preview_image_command.rs rename to app/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs diff --git a/src/script_execution/ffmpeg_generate_preview_video_command.rs b/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs similarity index 100% rename from src/script_execution/ffmpeg_generate_preview_video_command.rs rename to app/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs diff --git a/src/script_execution/google_drive_download_command.rs b/app/storyteller_web/src/script_execution/google_drive_download_command.rs similarity index 100% rename from src/script_execution/google_drive_download_command.rs rename to app/storyteller_web/src/script_execution/google_drive_download_command.rs diff --git a/src/script_execution/imagemagick_generate_preview_image_command.rs b/app/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs similarity index 100% rename from src/script_execution/imagemagick_generate_preview_image_command.rs rename to app/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs diff --git a/src/script_execution/mod.rs b/app/storyteller_web/src/script_execution/mod.rs similarity index 100% rename from src/script_execution/mod.rs rename to app/storyteller_web/src/script_execution/mod.rs diff --git a/src/script_execution/tacotron_inference_command.rs b/app/storyteller_web/src/script_execution/tacotron_inference_command.rs similarity index 100% rename from src/script_execution/tacotron_inference_command.rs rename to app/storyteller_web/src/script_execution/tacotron_inference_command.rs diff --git a/src/script_execution/tacotron_model_check_command.rs b/app/storyteller_web/src/script_execution/tacotron_model_check_command.rs similarity index 100% rename from src/script_execution/tacotron_model_check_command.rs rename to app/storyteller_web/src/script_execution/tacotron_model_check_command.rs diff --git a/src/script_execution/talknet_model_check_command.rs b/app/storyteller_web/src/script_execution/talknet_model_check_command.rs similarity index 100% rename from src/script_execution/talknet_model_check_command.rs rename to app/storyteller_web/src/script_execution/talknet_model_check_command.rs diff --git a/src/script_execution/wav2lip_inference_command.rs b/app/storyteller_web/src/script_execution/wav2lip_inference_command.rs similarity index 100% rename from src/script_execution/wav2lip_inference_command.rs rename to app/storyteller_web/src/script_execution/wav2lip_inference_command.rs diff --git a/src/script_execution/wav2lip_process_upload_command.rs b/app/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs similarity index 100% rename from src/script_execution/wav2lip_process_upload_command.rs rename to app/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs diff --git a/src/server_state.rs b/app/storyteller_web/src/server_state.rs similarity index 100% rename from src/server_state.rs rename to app/storyteller_web/src/server_state.rs diff --git a/src/shared_constants.rs b/app/storyteller_web/src/shared_constants.rs similarity index 100% rename from src/shared_constants.rs rename to app/storyteller_web/src/shared_constants.rs diff --git a/src/threads/ip_banlist_set.rs b/app/storyteller_web/src/threads/ip_banlist_set.rs similarity index 100% rename from src/threads/ip_banlist_set.rs rename to app/storyteller_web/src/threads/ip_banlist_set.rs diff --git a/src/threads/mod.rs b/app/storyteller_web/src/threads/mod.rs similarity index 100% rename from src/threads/mod.rs rename to app/storyteller_web/src/threads/mod.rs diff --git a/src/threads/poll_ip_banlist_thread.rs b/app/storyteller_web/src/threads/poll_ip_banlist_thread.rs similarity index 100% rename from src/threads/poll_ip_banlist_thread.rs rename to app/storyteller_web/src/threads/poll_ip_banlist_thread.rs diff --git a/src/tts_download_job.rs b/app/storyteller_web/src/tts_download_job.rs similarity index 100% rename from src/tts_download_job.rs rename to app/storyteller_web/src/tts_download_job.rs diff --git a/src/tts_inference_job.rs b/app/storyteller_web/src/tts_inference_job.rs similarity index 100% rename from src/tts_inference_job.rs rename to app/storyteller_web/src/tts_inference_job.rs diff --git a/src/util/anyhow_result.rs b/app/storyteller_web/src/util/anyhow_result.rs similarity index 100% rename from src/util/anyhow_result.rs rename to app/storyteller_web/src/util/anyhow_result.rs diff --git a/src/util/buckets/bucket_client.rs b/app/storyteller_web/src/util/buckets/bucket_client.rs similarity index 100% rename from src/util/buckets/bucket_client.rs rename to app/storyteller_web/src/util/buckets/bucket_client.rs diff --git a/src/util/buckets/bucket_path_unifier.rs b/app/storyteller_web/src/util/buckets/bucket_path_unifier.rs similarity index 100% rename from src/util/buckets/bucket_path_unifier.rs rename to app/storyteller_web/src/util/buckets/bucket_path_unifier.rs diff --git a/src/util/buckets/bucket_paths.rs b/app/storyteller_web/src/util/buckets/bucket_paths.rs similarity index 100% rename from src/util/buckets/bucket_paths.rs rename to app/storyteller_web/src/util/buckets/bucket_paths.rs diff --git a/src/util/buckets/mod.rs b/app/storyteller_web/src/util/buckets/mod.rs similarity index 100% rename from src/util/buckets/mod.rs rename to app/storyteller_web/src/util/buckets/mod.rs diff --git a/src/util/caching/mod.rs b/app/storyteller_web/src/util/caching/mod.rs similarity index 100% rename from src/util/caching/mod.rs rename to app/storyteller_web/src/util/caching/mod.rs diff --git a/src/util/caching/single_item_ttl_cache.rs b/app/storyteller_web/src/util/caching/single_item_ttl_cache.rs similarity index 100% rename from src/util/caching/single_item_ttl_cache.rs rename to app/storyteller_web/src/util/caching/single_item_ttl_cache.rs diff --git a/src/util/delete_role_disambiguation.rs b/app/storyteller_web/src/util/delete_role_disambiguation.rs similarity index 100% rename from src/util/delete_role_disambiguation.rs rename to app/storyteller_web/src/util/delete_role_disambiguation.rs diff --git a/src/util/email_to_gravatar.rs b/app/storyteller_web/src/util/email_to_gravatar.rs similarity index 100% rename from src/util/email_to_gravatar.rs rename to app/storyteller_web/src/util/email_to_gravatar.rs diff --git a/src/util/encrypted_sort_id.rs b/app/storyteller_web/src/util/encrypted_sort_id.rs similarity index 100% rename from src/util/encrypted_sort_id.rs rename to app/storyteller_web/src/util/encrypted_sort_id.rs diff --git a/src/util/filesystem.rs b/app/storyteller_web/src/util/filesystem.rs similarity index 100% rename from src/util/filesystem.rs rename to app/storyteller_web/src/util/filesystem.rs diff --git a/src/util/hashing/hash_file_sha2.rs b/app/storyteller_web/src/util/hashing/hash_file_sha2.rs similarity index 100% rename from src/util/hashing/hash_file_sha2.rs rename to app/storyteller_web/src/util/hashing/hash_file_sha2.rs diff --git a/src/util/hashing/hash_string_sha2.rs b/app/storyteller_web/src/util/hashing/hash_string_sha2.rs similarity index 100% rename from src/util/hashing/hash_string_sha2.rs rename to app/storyteller_web/src/util/hashing/hash_string_sha2.rs diff --git a/src/util/hashing/mod.rs b/app/storyteller_web/src/util/hashing/mod.rs similarity index 100% rename from src/util/hashing/mod.rs rename to app/storyteller_web/src/util/hashing/mod.rs diff --git a/src/util/hashing/sha256_digest.rs b/app/storyteller_web/src/util/hashing/sha256_digest.rs similarity index 100% rename from src/util/hashing/sha256_digest.rs rename to app/storyteller_web/src/util/hashing/sha256_digest.rs diff --git a/src/util/jobs/cache_miss_strategizer.rs b/app/storyteller_web/src/util/jobs/cache_miss_strategizer.rs similarity index 100% rename from src/util/jobs/cache_miss_strategizer.rs rename to app/storyteller_web/src/util/jobs/cache_miss_strategizer.rs diff --git a/src/util/jobs/cache_miss_strategizer_multi.rs b/app/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs similarity index 100% rename from src/util/jobs/cache_miss_strategizer_multi.rs rename to app/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs diff --git a/src/util/jobs/mod.rs b/app/storyteller_web/src/util/jobs/mod.rs similarity index 100% rename from src/util/jobs/mod.rs rename to app/storyteller_web/src/util/jobs/mod.rs diff --git a/src/util/jobs/virtual_lfu_cache.rs b/app/storyteller_web/src/util/jobs/virtual_lfu_cache.rs similarity index 100% rename from src/util/jobs/virtual_lfu_cache.rs rename to app/storyteller_web/src/util/jobs/virtual_lfu_cache.rs diff --git a/src/util/markdown_to_html.rs b/app/storyteller_web/src/util/markdown_to_html.rs similarity index 100% rename from src/util/markdown_to_html.rs rename to app/storyteller_web/src/util/markdown_to_html.rs diff --git a/src/util/mod.rs b/app/storyteller_web/src/util/mod.rs similarity index 100% rename from src/util/mod.rs rename to app/storyteller_web/src/util/mod.rs diff --git a/src/util/noop_logger.rs b/app/storyteller_web/src/util/noop_logger.rs similarity index 100% rename from src/util/noop_logger.rs rename to app/storyteller_web/src/util/noop_logger.rs diff --git a/src/util/random_crockford_token.rs b/app/storyteller_web/src/util/random_crockford_token.rs similarity index 100% rename from src/util/random_crockford_token.rs rename to app/storyteller_web/src/util/random_crockford_token.rs diff --git a/src/util/random_prefix_crockford_token.rs b/app/storyteller_web/src/util/random_prefix_crockford_token.rs similarity index 100% rename from src/util/random_prefix_crockford_token.rs rename to app/storyteller_web/src/util/random_prefix_crockford_token.rs diff --git a/src/util/random_uuid.rs b/app/storyteller_web/src/util/random_uuid.rs similarity index 100% rename from src/util/random_uuid.rs rename to app/storyteller_web/src/util/random_uuid.rs diff --git a/src/util/redis/mod.rs b/app/storyteller_web/src/util/redis/mod.rs similarity index 100% rename from src/util/redis/mod.rs rename to app/storyteller_web/src/util/redis/mod.rs diff --git a/src/util/redis/redis_job_status_logger.rs b/app/storyteller_web/src/util/redis/redis_job_status_logger.rs similarity index 100% rename from src/util/redis/redis_job_status_logger.rs rename to app/storyteller_web/src/util/redis/redis_job_status_logger.rs diff --git a/src/util/redis/redis_keys.rs b/app/storyteller_web/src/util/redis/redis_keys.rs similarity index 100% rename from src/util/redis/redis_keys.rs rename to app/storyteller_web/src/util/redis/redis_keys.rs diff --git a/src/util/semi_persistent_cache_dir.rs b/app/storyteller_web/src/util/semi_persistent_cache_dir.rs similarity index 100% rename from src/util/semi_persistent_cache_dir.rs rename to app/storyteller_web/src/util/semi_persistent_cache_dir.rs diff --git a/src/validations/cashapp_username.rs b/app/storyteller_web/src/validations/cashapp_username.rs similarity index 100% rename from src/validations/cashapp_username.rs rename to app/storyteller_web/src/validations/cashapp_username.rs diff --git a/src/validations/check_for_slurs.rs b/app/storyteller_web/src/validations/check_for_slurs.rs similarity index 100% rename from src/validations/check_for_slurs.rs rename to app/storyteller_web/src/validations/check_for_slurs.rs diff --git a/src/validations/discord_username.rs b/app/storyteller_web/src/validations/discord_username.rs similarity index 100% rename from src/validations/discord_username.rs rename to app/storyteller_web/src/validations/discord_username.rs diff --git a/src/validations/github_username.rs b/app/storyteller_web/src/validations/github_username.rs similarity index 100% rename from src/validations/github_username.rs rename to app/storyteller_web/src/validations/github_username.rs diff --git a/src/validations/ip_addresses.rs b/app/storyteller_web/src/validations/ip_addresses.rs similarity index 100% rename from src/validations/ip_addresses.rs rename to app/storyteller_web/src/validations/ip_addresses.rs diff --git a/src/validations/mod.rs b/app/storyteller_web/src/validations/mod.rs similarity index 100% rename from src/validations/mod.rs rename to app/storyteller_web/src/validations/mod.rs diff --git a/src/validations/model_uploads.rs b/app/storyteller_web/src/validations/model_uploads.rs similarity index 100% rename from src/validations/model_uploads.rs rename to app/storyteller_web/src/validations/model_uploads.rs diff --git a/src/validations/passwords.rs b/app/storyteller_web/src/validations/passwords.rs similarity index 100% rename from src/validations/passwords.rs rename to app/storyteller_web/src/validations/passwords.rs diff --git a/src/validations/twitch_username.rs b/app/storyteller_web/src/validations/twitch_username.rs similarity index 100% rename from src/validations/twitch_username.rs rename to app/storyteller_web/src/validations/twitch_username.rs diff --git a/src/validations/twitter_username.rs b/app/storyteller_web/src/validations/twitter_username.rs similarity index 100% rename from src/validations/twitter_username.rs rename to app/storyteller_web/src/validations/twitter_username.rs diff --git a/src/validations/username.rs b/app/storyteller_web/src/validations/username.rs similarity index 100% rename from src/validations/username.rs rename to app/storyteller_web/src/validations/username.rs diff --git a/src/validations/username_reservations.rs b/app/storyteller_web/src/validations/username_reservations.rs similarity index 100% rename from src/validations/username_reservations.rs rename to app/storyteller_web/src/validations/username_reservations.rs diff --git a/src/validations/website_url.rs b/app/storyteller_web/src/validations/website_url.rs similarity index 100% rename from src/validations/website_url.rs rename to app/storyteller_web/src/validations/website_url.rs diff --git a/src/w2l_download_job.rs b/app/storyteller_web/src/w2l_download_job.rs similarity index 100% rename from src/w2l_download_job.rs rename to app/storyteller_web/src/w2l_download_job.rs diff --git a/src/w2l_inference_job.rs b/app/storyteller_web/src/w2l_inference_job.rs similarity index 100% rename from src/w2l_inference_job.rs rename to app/storyteller_web/src/w2l_inference_job.rs From f1668370834bccbb1206e27d594acc4c9625d55a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 7 Nov 2021 14:37:13 -0500 Subject: [PATCH 0638/1042] Fix up dockerfile for image builds --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3d4660b581..ec3f882214 100644 --- a/Dockerfile +++ b/Dockerfile @@ -152,7 +152,7 @@ WORKDIR /tmp COPY Cargo.lock . COPY Cargo.toml . COPY sqlx-data.json . -COPY src/ ./src +COPY app/ ./app COPY vendor/ ./vendor COPY db/ ./db COPY migrations/ ./migrations From 0c409c6af1bef2c8e568f28473401409fcf26df8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 7 Nov 2021 15:03:42 -0500 Subject: [PATCH 0639/1042] Shared config package --- Cargo.lock | 60 ++++++ Cargo.toml | 1 + app/config/Cargo.toml | 13 ++ app/config/src/lib.rs | 2 + .../src/shared_constants.rs | 0 app/obs_gateway/Cargo.toml | 102 +++++++++ app/obs_gateway/src/main.rs | 200 +++++++++++++++++- app/storyteller_web/Cargo.toml | 5 + app/storyteller_web/src/common_env.rs | 2 +- ...ist_tts_inference_results_query_builder.rs | 2 +- ...ist_w2l_inference_results_query_builder.rs | 2 +- app/storyteller_web/src/main.rs | 7 +- app/storyteller_web/src/obs_gateway_server.rs | 200 ------------------ app/storyteller_web/src/tts_download_job.rs | 5 +- app/storyteller_web/src/tts_inference_job.rs | 5 +- app/storyteller_web/src/w2l_download_job.rs | 5 +- app/storyteller_web/src/w2l_inference_job.rs | 5 +- 17 files changed, 394 insertions(+), 222 deletions(-) create mode 100644 app/config/Cargo.toml create mode 100644 app/config/src/lib.rs rename app/{storyteller_web => config}/src/shared_constants.rs (100%) delete mode 100644 app/storyteller_web/src/obs_gateway_server.rs diff --git a/Cargo.lock b/Cargo.lock index 2a3988d4cf..210075ab48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -995,6 +995,10 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "config" +version = "0.0.1" + [[package]] name = "const_fn" version = "0.4.7" @@ -2351,6 +2355,61 @@ dependencies = [ [[package]] name = "obs-gateway-server" version = "0.0.1" +dependencies = [ + "actix", + "actix-cors", + "actix-files", + "actix-http", + "actix-multipart", + "actix-rt 2.2.0", + "actix-service", + "actix-utils 3.0.0", + "actix-web", + "actix-web-actors", + "anyhow", + "base64 0.13.0", + "bcrypt", + "chrono", + "config", + "data-encoding", + "derive_more", + "diesel", + "dotenv", + "easyenv", + "futures 0.1.31", + "futures 0.3.15", + "hex", + "hmac 0.10.1", + "hostname", + "hyper", + "infer", + "jwt", + "lazy_static", + "lfu", + "limitation", + "log", + "lru_time_cache", + "magic-crypt", + "md-5", + "newrelic-telemetry", + "pulldown-cmark", + "r2d2_redis", + "rand 0.8.3", + "regex", + "ring", + "rust-s3", + "serde", + "serde_derive", + "serde_json", + "sha2", + "sqlx", + "subprocess", + "tempdir", + "time 0.2.26", + "tokio 0.2.25", + "toml", + "uuid", +] [[package]] name = "once_cell" @@ -3430,6 +3489,7 @@ dependencies = [ "base64 0.13.0", "bcrypt", "chrono", + "config", "data-encoding", "derive_more", "diesel", diff --git a/Cargo.toml b/Cargo.toml index 6c5e053d99..fb9eccd2cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ + "app/config", "app/obs_gateway", "app/storyteller_web", ] diff --git a/app/config/Cargo.toml b/app/config/Cargo.toml new file mode 100644 index 0000000000..fc10d38795 --- /dev/null +++ b/app/config/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "config" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "config" +path = "src/lib.rs" diff --git a/app/config/src/lib.rs b/app/config/src/lib.rs new file mode 100644 index 0000000000..6faad52aea --- /dev/null +++ b/app/config/src/lib.rs @@ -0,0 +1,2 @@ + +pub mod shared_constants; \ No newline at end of file diff --git a/app/storyteller_web/src/shared_constants.rs b/app/config/src/shared_constants.rs similarity index 100% rename from app/storyteller_web/src/shared_constants.rs rename to app/config/src/shared_constants.rs diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index 377d0d234c..bbb0580112 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -11,3 +11,105 @@ publish = false [[bin]] name = "obs-gateway-server" path = "src/main.rs" + +[dependencies] + +# Internal +config = { path = "../config" } + +# External +anyhow = "1.0.40" +base64 = "0.13.0" +chrono = { version = "0.4.19", features = ["serde"] } +derive_more = "0.99.14" +dotenv = "0.15.0" +easyenv = "0.0.6" +futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool +futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate +hex = "0.4.3" +hostname = "0.3.1" +lazy_static = "1.4.0" +lfu = "0.2.5" +log = "0.4.14" +#newrelic = "0.2.2" +newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" } +rand = "0.8.3" +regex = "1.5.4" +time = "0.2.26" +uuid = { version = "0.8.2", features = ["v4"] } + +# Runtime +# NB(1): We switched to Tokio from `async-std` for broader ecosystem support. +#async-std = { version = "1.6", features = [ "attributes" ] } +# NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: +# https://github.com/durch/rust-s3/issues/198 +#tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "0.2.25", features = ["macros"] } + +# MySql Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +#sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } +# NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade +#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } +sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } + +# Redis Database +limitation = "0.1.1" # Redis rate limiter +r2d2_redis = "0.14.0" + +# In-memory caching +lru_time_cache = "0.11.11" + +# Files +data-encoding = "2.3.2" +ring = "0.17.0-alpha.10" +# NB: This is the source of all of our problems. `rust-s3` can't upload to object names with directories +# in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2 +#rust-s3 = "0.27.0-rc4" +#rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } +rust-s3 = { version = "0.26" } +infer = "0.5.0" + +subprocess = "0.2.6" +tempdir = "0.3.7" + +# Web +actix = "0.10" +#actix-web = "3" +actix-web-actors = "3" + +actix-cors = "0.5.4" +actix-files= "0.5" +actix-http = "2.2.0" +actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade +actix-rt = "2.2.0" +actix-service = "1.0.6" +actix-utils = "3.0.0" +actix-web = "3.3.2" +jwt = "0.13.0" + +# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime +# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. +hyper = { version = "0.13.10" } + +# Crypto +bcrypt = "0.8" # password hashes +hmac = "0.10.0" # cookies +magic-crypt = "3.1.8" # encrypting sort IDs for the frontend so we don't leak info +md-5 = "0.9.1" # gravatar +sha2 = "0.9.5" # file hashes, inference text hashes, etc. + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" +toml = "0.5.8" + +# Text processing +pulldown-cmark = "0.8.0" + +[build-dependencies] +# NB: Only using diesel for migrations support +diesel = { version = "1.4.6", features = ["mysql"] } diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index c4357a72be..9c8a7c14db 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -1,4 +1,198 @@ +#![forbid(private_in_public)] +#![forbid(unused_must_use)] +//#![forbid(warnings)] -pub fn main() { - println!("TBD"); -} \ No newline at end of file +#![allow(dead_code)] +#![allow(non_snake_case)] +#![allow(unused_imports)] +#![allow(unused_mut)] +#![allow(unused_variables)] + +#[macro_use] extern crate lazy_static; +#[macro_use] extern crate magic_crypt; +#[macro_use] extern crate serde_derive; + +use actix_cors::Cors; +use actix_http::http; +use actix_web::middleware::{Logger, DefaultHeaders}; +use actix_web::{HttpServer, web, HttpResponse, App}; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; +use futures::Future; +use futures::executor::ThreadPool; +use limitation::Limiter; +use log::{info}; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2; +use r2d2_redis::redis::Commands; +use sqlx::MySqlPool; +use sqlx::mysql::MySqlPoolOptions; +use std::sync::Arc; +use std::time::Duration; + +const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; + +pub type AnyhowResult = anyhow::Result; + +/// State that is injected into every endpoint. +#[derive(Clone)] +pub struct ObsGatewayServerState { + /// Configuration from ENV vars. + /// Some of this might not be used. + pub env_config: EnvConfig, + pub hostname: String, +} + +#[derive(Clone)] +pub struct EnvConfig { + // Number of thread workers. + pub num_workers: usize, + pub bind_address: String, + pub cookie_domain: String, + pub cookie_secure: bool, + pub cookie_http_only: bool, + pub website_homepage_redirect: String, +} + +#[actix_web::main] +async fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + // NB: Do not check this secrets-containing dotenv file into VCS. + // This file should only contain *development* secrets, never production. + let _ = dotenv::from_filename(".env-secrets").ok(); + + info!("Obtaining hostname..."); + + let server_hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or("obs-gateway-server-unknown".to_string()); + + info!("Hostname: {}", &server_hostname); + + info!("Connecting to database..."); + + let db_connection_string = + easyenv::get_env_string_or_default( + "MYSQL_URL", + DEFAULT_MYSQL_CONNECTION_STRING); + + let redis_connection_string = + easyenv::get_env_string_or_default( + "REDIS_URL", + DEFAULT_REDIS_CONNECTION_STRING); + + let pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(&db_connection_string) + .await?; + + let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?; + + let redis_pool = r2d2::Pool::builder() + .build(redis_manager)?; + + info!("Reading env vars and setting up utils..."); + + let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); + let num_workers = easyenv::get_env_num("NUM_WORKERS", 8)?; + let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); + let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); + let cookie_secure = easyenv::get_env_bool_or_default("COOKIE_SECURE", true); + let cookie_http_only = easyenv::get_env_bool_or_default("COOKIE_HTTP_ONLY", true); + let website_homepage_redirect = + easyenv::get_env_string_or_default("WEBSITE_HOMEPAGE_REDIRECT", "https://vo.codes/"); + + let server_state = ObsGatewayServerState { + env_config: EnvConfig { + num_workers, + bind_address, + cookie_domain, + cookie_secure, + cookie_http_only, + website_homepage_redirect, + }, + hostname: server_hostname, + }; + + serve(server_state) + .await?; + Ok(()) +} + +pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> +{ + let bind_address = server_state.env_config.bind_address.clone(); + let num_workers = server_state.env_config.num_workers.clone(); + let hostname = server_state.hostname.clone(); + + let server_state_arc = web::Data::new(Arc::new(server_state)); + + info!("Starting HTTP service."); + + let log_format = "[%{HOSTNAME}e] %a \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T"; + + HttpServer::new(move || { + App::new() + .app_data(server_state_arc.clone()) + .wrap(Cors::default() + .allowed_origin("http://api.fakeyou.com") + .allowed_origin("http://api.jungle.horse") + .allowed_origin("http://api.vo.codes") + .allowed_origin("http://fakeyou.com") + .allowed_origin("http://jungle.horse") + .allowed_origin("http://jungle.horse:12345") + .allowed_origin("http://jungle.horse:7000") + .allowed_origin("http://localhost:12345") + .allowed_origin("http://localhost:3000") + .allowed_origin("http://localhost:5555") + .allowed_origin("http://localhost:7000") + .allowed_origin("http://localhost:8000") + .allowed_origin("http://localhost:8080") + .allowed_origin("http://mumble.stream") + .allowed_origin("http://trumped.com") + .allowed_origin("http://vo.codes") + .allowed_origin("http://vocodes.com") + .allowed_origin("https://api.fakeyou.com") + .allowed_origin("https://api.jungle.horse") + .allowed_origin("https://api.vo.codes") + .allowed_origin("https://fakeyou.com") + .allowed_origin("https://jungle.horse") + .allowed_origin("https://mumble.stream") + .allowed_origin("https://trumped.com") + .allowed_origin("https://vo.codes") + .allowed_origin("https://vocodes.com") + .allowed_methods(vec!["GET", "POST", "OPTIONS"]) + .supports_credentials() + .allowed_headers(vec![ + http::header::ACCEPT, + http::header::ACCESS_CONTROL_ALLOW_ORIGIN, // Tabulator Ajax + http::header::CONTENT_TYPE, + http::header::ACCESS_CONTROL_ALLOW_CREDENTIALS, // https://stackoverflow.com/a/46412839 + http::header::HeaderName::from_static("x-requested-with") // Tabulator Ajax sends + ]) + .max_age(3600)) + .wrap(Logger::new(&log_format) + .exclude("/liveness") + .exclude("/readiness")) + .wrap(DefaultHeaders::new() + .header("X-Backend-Hostname", &hostname) + .header("X-Build-Sha", "")) + // Twitch + //.service( + // web::resource("/twitch") + // .route(web::get().to(ws_index)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service(get_root_index) + //.default_service( web::route().to(default_route_404)) + }) + .bind(bind_address)? + .workers(num_workers) + .run() + .await?; + + Ok(()) +} diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml index 4407bc2745..ed5181cedd 100644 --- a/app/storyteller_web/Cargo.toml +++ b/app/storyteller_web/Cargo.toml @@ -36,6 +36,11 @@ name = "w2l-inference-job" path = "src/w2l_inference_job.rs" [dependencies] + +# Internal +config = { path = "../config" } + +# External anyhow = "1.0.40" base64 = "0.13.0" chrono = { version = "0.4.19", features = ["serde"] } diff --git a/app/storyteller_web/src/common_env.rs b/app/storyteller_web/src/common_env.rs index eb2b6cc1eb..9013327abc 100644 --- a/app/storyteller_web/src/common_env.rs +++ b/app/storyteller_web/src/common_env.rs @@ -1,4 +1,4 @@ -use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; +use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use crate::util::anyhow_result::AnyhowResult; // TODO: Move more shared configs here. diff --git a/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs b/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs index 6840023eef..011629933a 100644 --- a/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs +++ b/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::database::enums::record_visibility::RecordVisibility; -use crate::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs b/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs index a0e5c6b8ea..0553bf0552 100644 --- a/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs +++ b/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::database::enums::record_visibility::RecordVisibility; -use crate::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index 3a933b5675..b1ee31d84b 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -20,7 +20,6 @@ pub mod database; pub mod http_clients; pub mod http_server; pub mod server_state; -pub mod shared_constants; pub mod threads; pub mod util; pub mod validations; @@ -32,6 +31,9 @@ use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::default_route_404::default_route_404; @@ -95,9 +97,6 @@ use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::server_state::{ServerState, EnvConfig}; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; use crate::threads::ip_banlist_set::IpBanlistSet; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; diff --git a/app/storyteller_web/src/obs_gateway_server.rs b/app/storyteller_web/src/obs_gateway_server.rs deleted file mode 100644 index 114f8c4dcf..0000000000 --- a/app/storyteller_web/src/obs_gateway_server.rs +++ /dev/null @@ -1,200 +0,0 @@ -#![forbid(private_in_public)] -#![forbid(unused_must_use)] -//#![forbid(warnings)] - -#![allow(dead_code)] -#![allow(non_snake_case)] -#![allow(unused_imports)] -#![allow(unused_mut)] -#![allow(unused_variables)] - -#[macro_use] extern crate lazy_static; -#[macro_use] extern crate magic_crypt; -#[macro_use] extern crate serde_derive; - -pub mod shared_constants; - -use actix_cors::Cors; -use actix_http::http; -use actix_web::middleware::{Logger, DefaultHeaders}; -use actix_web::{HttpServer, web, HttpResponse, App}; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; -use futures::Future; -use futures::executor::ThreadPool; -use limitation::Limiter; -use log::{info}; -use r2d2_redis::RedisConnectionManager; -use r2d2_redis::r2d2; -use r2d2_redis::redis::Commands; -use sqlx::MySqlPool; -use sqlx::mysql::MySqlPoolOptions; -use std::sync::Arc; -use std::time::Duration; - -const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; - -pub type AnyhowResult = anyhow::Result; - -/// State that is injected into every endpoint. -#[derive(Clone)] -pub struct ObsGatewayServerState { - /// Configuration from ENV vars. - /// Some of this might not be used. - pub env_config: EnvConfig, - pub hostname: String, -} - -#[derive(Clone)] -pub struct EnvConfig { - // Number of thread workers. - pub num_workers: usize, - pub bind_address: String, - pub cookie_domain: String, - pub cookie_secure: bool, - pub cookie_http_only: bool, - pub website_homepage_redirect: String, -} - -#[actix_web::main] -async fn main() -> AnyhowResult<()> { - easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); - - // NB: Do not check this secrets-containing dotenv file into VCS. - // This file should only contain *development* secrets, never production. - let _ = dotenv::from_filename(".env-secrets").ok(); - - info!("Obtaining hostname..."); - - let server_hostname = hostname::get() - .ok() - .and_then(|h| h.into_string().ok()) - .unwrap_or("obs-gateway-server-unknown".to_string()); - - info!("Hostname: {}", &server_hostname); - - info!("Connecting to database..."); - - let db_connection_string = - easyenv::get_env_string_or_default( - "MYSQL_URL", - DEFAULT_MYSQL_CONNECTION_STRING); - - let redis_connection_string = - easyenv::get_env_string_or_default( - "REDIS_URL", - DEFAULT_REDIS_CONNECTION_STRING); - - let pool = MySqlPoolOptions::new() - .max_connections(5) - .connect(&db_connection_string) - .await?; - - let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?; - - let redis_pool = r2d2::Pool::builder() - .build(redis_manager)?; - - info!("Reading env vars and setting up utils..."); - - let bind_address = easyenv::get_env_string_or_default("BIND_ADDRESS", DEFAULT_BIND_ADDRESS); - let num_workers = easyenv::get_env_num("NUM_WORKERS", 8)?; - let hmac_secret = easyenv::get_env_string_or_default("COOKIE_SECRET", "notsecret"); - let cookie_domain = easyenv::get_env_string_or_default("COOKIE_DOMAIN", ".vo.codes"); - let cookie_secure = easyenv::get_env_bool_or_default("COOKIE_SECURE", true); - let cookie_http_only = easyenv::get_env_bool_or_default("COOKIE_HTTP_ONLY", true); - let website_homepage_redirect = - easyenv::get_env_string_or_default("WEBSITE_HOMEPAGE_REDIRECT", "https://vo.codes/"); - - let server_state = ObsGatewayServerState { - env_config: EnvConfig { - num_workers, - bind_address, - cookie_domain, - cookie_secure, - cookie_http_only, - website_homepage_redirect, - }, - hostname: server_hostname, - }; - - serve(server_state) - .await?; - Ok(()) -} - -pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> -{ - let bind_address = server_state.env_config.bind_address.clone(); - let num_workers = server_state.env_config.num_workers.clone(); - let hostname = server_state.hostname.clone(); - - let server_state_arc = web::Data::new(Arc::new(server_state)); - - info!("Starting HTTP service."); - - let log_format = "[%{HOSTNAME}e] %a \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T"; - - HttpServer::new(move || { - App::new() - .app_data(server_state_arc.clone()) - .wrap(Cors::default() - .allowed_origin("http://api.fakeyou.com") - .allowed_origin("http://api.jungle.horse") - .allowed_origin("http://api.vo.codes") - .allowed_origin("http://fakeyou.com") - .allowed_origin("http://jungle.horse") - .allowed_origin("http://jungle.horse:12345") - .allowed_origin("http://jungle.horse:7000") - .allowed_origin("http://localhost:12345") - .allowed_origin("http://localhost:3000") - .allowed_origin("http://localhost:5555") - .allowed_origin("http://localhost:7000") - .allowed_origin("http://localhost:8000") - .allowed_origin("http://localhost:8080") - .allowed_origin("http://mumble.stream") - .allowed_origin("http://trumped.com") - .allowed_origin("http://vo.codes") - .allowed_origin("http://vocodes.com") - .allowed_origin("https://api.fakeyou.com") - .allowed_origin("https://api.jungle.horse") - .allowed_origin("https://api.vo.codes") - .allowed_origin("https://fakeyou.com") - .allowed_origin("https://jungle.horse") - .allowed_origin("https://mumble.stream") - .allowed_origin("https://trumped.com") - .allowed_origin("https://vo.codes") - .allowed_origin("https://vocodes.com") - .allowed_methods(vec!["GET", "POST", "OPTIONS"]) - .supports_credentials() - .allowed_headers(vec![ - http::header::ACCEPT, - http::header::ACCESS_CONTROL_ALLOW_ORIGIN, // Tabulator Ajax - http::header::CONTENT_TYPE, - http::header::ACCESS_CONTROL_ALLOW_CREDENTIALS, // https://stackoverflow.com/a/46412839 - http::header::HeaderName::from_static("x-requested-with") // Tabulator Ajax sends - ]) - .max_age(3600)) - .wrap(Logger::new(&log_format) - .exclude("/liveness") - .exclude("/readiness")) - .wrap(DefaultHeaders::new() - .header("X-Backend-Hostname", &hostname) - .header("X-Build-Sha", "")) - // Twitch - //.service( - // web::resource("/twitch") - // .route(web::get().to(ws_index)) - // .route(web::head().to(|| HttpResponse::Ok())) - //) - //.service(get_root_index) - //.default_service( web::route().to(default_route_404)) - }) - .bind(bind_address)? - .workers(num_workers) - .run() - .await?; - - Ok(()) -} diff --git a/app/storyteller_web/src/tts_download_job.rs b/app/storyteller_web/src/tts_download_job.rs index e08dc1cc2f..7718e356b5 100644 --- a/app/storyteller_web/src/tts_download_job.rs +++ b/app/storyteller_web/src/tts_download_job.rs @@ -14,11 +14,12 @@ pub mod common_env; pub mod database; pub mod job_queries; pub mod script_execution; -pub mod shared_constants; pub mod util; use anyhow::anyhow; use chrono::Utc; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; use crate::common_env::CommonEnv; use crate::database::enums::tts_model_type::TtsModelType; use crate::database::mediators::badge_granter::BadgeGranter; @@ -32,8 +33,6 @@ use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::tacotron_model_check_command::TacotronModelCheckCommand; use crate::script_execution::talknet_model_check_command::TalknetModelCheckCommand; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; diff --git a/app/storyteller_web/src/tts_inference_job.rs b/app/storyteller_web/src/tts_inference_job.rs index 5f67dfd22b..89aa39a78a 100644 --- a/app/storyteller_web/src/tts_inference_job.rs +++ b/app/storyteller_web/src/tts_inference_job.rs @@ -15,11 +15,12 @@ pub mod database; pub mod http_clients; pub mod job_queries; pub mod script_execution; -pub mod shared_constants; pub mod util; use anyhow::{anyhow, Error}; use chrono::{Utc, DateTime, TimeZone}; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; use crate::common_env::CommonEnv; use crate::database::enums::vocoder_type::VocoderType; use crate::database::mediators::firehose_publisher::FirehosePublisher; @@ -35,8 +36,6 @@ use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_failur use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_permanently_dead; use crate::job_queries::tts_inference_job_queries::query_tts_inference_job_records; use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; diff --git a/app/storyteller_web/src/w2l_download_job.rs b/app/storyteller_web/src/w2l_download_job.rs index e0b49db743..1d0c3344f1 100644 --- a/app/storyteller_web/src/w2l_download_job.rs +++ b/app/storyteller_web/src/w2l_download_job.rs @@ -14,11 +14,12 @@ pub mod common_env; pub mod database; pub mod job_queries; pub mod script_execution; -pub mod shared_constants; pub mod util; use anyhow::{anyhow, Error}; use chrono::Utc; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; use crate::common_env::CommonEnv; use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; @@ -34,8 +35,6 @@ use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGenera use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_process_upload_command::{Wav2LipPreprocessClient, Wav2LipPreprocessError}; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_paths::hash_to_bucket_path; diff --git a/app/storyteller_web/src/w2l_inference_job.rs b/app/storyteller_web/src/w2l_inference_job.rs index a7d29b6aa6..75f42e4f63 100644 --- a/app/storyteller_web/src/w2l_inference_job.rs +++ b/app/storyteller_web/src/w2l_inference_job.rs @@ -14,11 +14,12 @@ pub mod common_env; pub mod database; pub mod job_queries; pub mod script_execution; -pub mod shared_constants; pub mod util; use anyhow::anyhow; use chrono::Utc; +use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use config::shared_constants::DEFAULT_RUST_LOG; use crate::common_env::CommonEnv; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; @@ -33,8 +34,6 @@ use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGenera use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand; -use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; From f049a6f7d0b807d2ae154cf73c5ba0126600c0da Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 7 Nov 2021 15:24:28 -0500 Subject: [PATCH 0640/1042] Make a shared "http server common" package --- Cargo.lock | 57 +++++++++ Cargo.toml | 1 + app/http_server_common/Cargo.toml | 108 ++++++++++++++++++ .../src}/endpoints/default_route_404.rs | 0 app/http_server_common/src/endpoints/mod.rs | 2 + .../src}/endpoints/root_index.rs | 0 app/http_server_common/src/lib.rs | 5 + app/obs_gateway/Cargo.toml | 1 + app/obs_gateway/src/main.rs | 6 +- app/storyteller_web/Cargo.toml | 1 + .../src/http_server/endpoints/mod.rs | 2 - app/storyteller_web/src/main.rs | 6 +- 12 files changed, 182 insertions(+), 7 deletions(-) create mode 100644 app/http_server_common/Cargo.toml rename app/{storyteller_web/src/http_server => http_server_common/src}/endpoints/default_route_404.rs (100%) create mode 100644 app/http_server_common/src/endpoints/mod.rs rename app/{storyteller_web/src/http_server => http_server_common/src}/endpoints/root_index.rs (100%) create mode 100644 app/http_server_common/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 210075ab48..4e12b9dbc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1719,6 +1719,61 @@ dependencies = [ "http", ] +[[package]] +name = "http_server_common" +version = "0.0.1" +dependencies = [ + "actix", + "actix-cors", + "actix-files", + "actix-http", + "actix-multipart", + "actix-rt 2.2.0", + "actix-service", + "actix-utils 3.0.0", + "actix-web", + "actix-web-actors", + "anyhow", + "base64 0.13.0", + "bcrypt", + "chrono", + "data-encoding", + "derive_more", + "diesel", + "dotenv", + "easyenv", + "futures 0.1.31", + "futures 0.3.15", + "hex", + "hmac 0.10.1", + "hostname", + "hyper", + "infer", + "jwt", + "limitation", + "log", + "lru_time_cache", + "magic-crypt", + "md-5", + "pulldown-cmark", + "r2d2_redis", + "rand 0.8.3", + "regex", + "ring", + "rust-s3", + "serde", + "serde_derive", + "serde_json", + "sha2", + "sqlx", + "subprocess", + "tempdir", + "time 0.2.26", + "tokio 0.2.25", + "toml", + "uuid", +] + [[package]] name = "httparse" version = "1.5.1" @@ -2381,6 +2436,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", + "http_server_common", "hyper", "infer", "jwt", @@ -3500,6 +3556,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", + "http_server_common", "hyper", "infer", "jwt", diff --git a/Cargo.toml b/Cargo.toml index fb9eccd2cf..a5f41170c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "app/config", + "app/http_server_common", "app/obs_gateway", "app/storyteller_web", ] diff --git a/app/http_server_common/Cargo.toml b/app/http_server_common/Cargo.toml new file mode 100644 index 0000000000..88c14d9ca3 --- /dev/null +++ b/app/http_server_common/Cargo.toml @@ -0,0 +1,108 @@ +[package] +name = "http_server_common" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "http_server_common" +path = "src/lib.rs" + +[dependencies] + +# External +anyhow = "1.0.40" +base64 = "0.13.0" +chrono = { version = "0.4.19", features = ["serde"] } +derive_more = "0.99.14" +dotenv = "0.15.0" +easyenv = "0.0.6" +futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool +futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate +hex = "0.4.3" +hostname = "0.3.1" +log = "0.4.14" +rand = "0.8.3" +regex = "1.5.4" +time = "0.2.26" +uuid = { version = "0.8.2", features = ["v4"] } + +# Runtime +# NB(1): We switched to Tokio from `async-std` for broader ecosystem support. +#async-std = { version = "1.6", features = [ "attributes" ] } +# NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: +# https://github.com/durch/rust-s3/issues/198 +#tokio = { version = "1.6.0", features = ["macros"] } +tokio = { version = "0.2.25", features = ["macros"] } + +# MySql Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +#sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } +# NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade +#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } +sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } + +# Redis Database +limitation = "0.1.1" # Redis rate limiter +r2d2_redis = "0.14.0" + +# In-memory caching +lru_time_cache = "0.11.11" + +# Files +data-encoding = "2.3.2" +ring = "0.17.0-alpha.10" +# NB: This is the source of all of our problems. `rust-s3` can't upload to object names with directories +# in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2 +#rust-s3 = "0.27.0-rc4" +#rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } +rust-s3 = { version = "0.26" } +infer = "0.5.0" + +subprocess = "0.2.6" +tempdir = "0.3.7" + +# Web +actix = "0.10" +#actix-web = "3" +actix-web-actors = "3" + +actix-cors = "0.5.4" +actix-files= "0.5" +actix-http = "2.2.0" +actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade +actix-rt = "2.2.0" +actix-service = "1.0.6" +actix-utils = "3.0.0" +actix-web = "3.3.2" +jwt = "0.13.0" + +# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime +# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. +hyper = { version = "0.13.10" } + +# Crypto +bcrypt = "0.8" # password hashes +hmac = "0.10.0" # cookies +magic-crypt = "3.1.8" # encrypting sort IDs for the frontend so we don't leak info +md-5 = "0.9.1" # gravatar +sha2 = "0.9.5" # file hashes, inference text hashes, etc. + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" +toml = "0.5.8" + +# Text processing +pulldown-cmark = "0.8.0" + +[build-dependencies] +# NB: Only using diesel for migrations support +diesel = { version = "1.4.6", features = ["mysql"] } diff --git a/app/storyteller_web/src/http_server/endpoints/default_route_404.rs b/app/http_server_common/src/endpoints/default_route_404.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/default_route_404.rs rename to app/http_server_common/src/endpoints/default_route_404.rs diff --git a/app/http_server_common/src/endpoints/mod.rs b/app/http_server_common/src/endpoints/mod.rs new file mode 100644 index 0000000000..9ae4ad6cf5 --- /dev/null +++ b/app/http_server_common/src/endpoints/mod.rs @@ -0,0 +1,2 @@ +pub mod default_route_404; +pub mod root_index; diff --git a/app/storyteller_web/src/http_server/endpoints/root_index.rs b/app/http_server_common/src/endpoints/root_index.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/root_index.rs rename to app/http_server_common/src/endpoints/root_index.rs diff --git a/app/http_server_common/src/lib.rs b/app/http_server_common/src/lib.rs new file mode 100644 index 0000000000..34440e0e43 --- /dev/null +++ b/app/http_server_common/src/lib.rs @@ -0,0 +1,5 @@ +//! +//! Common HTTP Server components +//! + +pub mod endpoints; diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index bbb0580112..d4c5d95952 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" # Internal config = { path = "../config" } +http_server_common = { path = "../http_server_common" } # External anyhow = "1.0.40" diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index 9c8a7c14db..2eadfc277a 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -21,6 +21,8 @@ use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use futures::Future; use futures::executor::ThreadPool; +use http_server_common::endpoints::default_route_404::default_route_404; +use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; use log::{info}; use r2d2_redis::RedisConnectionManager; @@ -186,8 +188,8 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> // .route(web::get().to(ws_index)) // .route(web::head().to(|| HttpResponse::Ok())) //) - //.service(get_root_index) - //.default_service( web::route().to(default_route_404)) + .service(get_root_index) + .default_service( web::route().to(default_route_404)) }) .bind(bind_address)? .workers(num_workers) diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml index ed5181cedd..3744dbf292 100644 --- a/app/storyteller_web/Cargo.toml +++ b/app/storyteller_web/Cargo.toml @@ -39,6 +39,7 @@ path = "src/w2l_inference_job.rs" # Internal config = { path = "../config" } +http_server_common = { path = "../http_server_common" } # External anyhow = "1.0.40" diff --git a/app/storyteller_web/src/http_server/endpoints/mod.rs b/app/storyteller_web/src/http_server/endpoints/mod.rs index 38b3143d3f..3842da000d 100644 --- a/app/storyteller_web/src/http_server/endpoints/mod.rs +++ b/app/storyteller_web/src/http_server/endpoints/mod.rs @@ -1,9 +1,7 @@ -pub mod default_route_404; pub mod events; pub mod leaderboard; pub mod misc; pub mod moderation; -pub mod root_index; pub mod tts; pub mod twitch; pub mod users; diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index b1ee31d84b..3d1aacdbb7 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -36,7 +36,6 @@ use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; -use crate::http_server::endpoints::default_route_404::default_route_404; use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha_easy_handler; @@ -55,7 +54,6 @@ use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler; use crate::http_server::endpoints::moderation::user_roles::set_user_role::set_user_role_handler; use crate::http_server::endpoints::moderation::users::list_users::list_users_handler; -use crate::http_server::endpoints::root_index::get_root_index; use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handler; use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; @@ -68,6 +66,7 @@ use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_u use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; +use crate::http_server::endpoints::twitch::pubsub_gateway::ws_index; use crate::http_server::endpoints::users::create_account::create_account_handler; use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; use crate::http_server::endpoints::users::get_profile::get_profile_handler; @@ -104,6 +103,8 @@ use crate::util::caching::single_item_ttl_cache::SingleItemTtlCache; use crate::util::encrypted_sort_id::SortKeyCrypto; use futures::Future; use futures::executor::ThreadPool; +use http_server_common::endpoints::default_route_404::default_route_404; +use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; use log::{info}; use r2d2_redis::RedisConnectionManager; @@ -113,7 +114,6 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; -use crate::http_server::endpoints::twitch::pubsub_gateway::ws_index; // TODO TODO TODO TODO // TODO TODO TODO TODO From 18ce4e1c6266c385552ecfaed6b8ef0b20317625 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 7 Nov 2021 15:36:14 -0500 Subject: [PATCH 0641/1042] Move websocket endpoint to OBS app --- app/obs_gateway/src/main.rs | 13 ++++++++----- .../twitch => obs_gateway/src}/pubsub_gateway.rs | 0 .../src/http_server/endpoints/mod.rs | 1 - .../src/http_server/endpoints/twitch/mod.rs | 1 - app/storyteller_web/src/main.rs | 7 ------- app/storyteller_web/src/obs_gateway/mod.rs | 0 6 files changed, 8 insertions(+), 14 deletions(-) rename app/{storyteller_web/src/http_server/endpoints/twitch => obs_gateway/src}/pubsub_gateway.rs (100%) delete mode 100644 app/storyteller_web/src/http_server/endpoints/twitch/mod.rs delete mode 100644 app/storyteller_web/src/obs_gateway/mod.rs diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index 2eadfc277a..04256e0009 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -12,6 +12,8 @@ #[macro_use] extern crate magic_crypt; #[macro_use] extern crate serde_derive; +pub mod pubsub_gateway; + use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; @@ -19,6 +21,7 @@ use actix_web::{HttpServer, web, HttpResponse, App}; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; +use crate::pubsub_gateway::ws_index; use futures::Future; use futures::executor::ThreadPool; use http_server_common::endpoints::default_route_404::default_route_404; @@ -183,11 +186,11 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> .header("X-Backend-Hostname", &hostname) .header("X-Build-Sha", "")) // Twitch - //.service( - // web::resource("/twitch") - // .route(web::get().to(ws_index)) - // .route(web::head().to(|| HttpResponse::Ok())) - //) + .service( + web::resource("/twitch") + .route(web::get().to(ws_index)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(get_root_index) .default_service( web::route().to(default_route_404)) }) diff --git a/app/storyteller_web/src/http_server/endpoints/twitch/pubsub_gateway.rs b/app/obs_gateway/src/pubsub_gateway.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/twitch/pubsub_gateway.rs rename to app/obs_gateway/src/pubsub_gateway.rs diff --git a/app/storyteller_web/src/http_server/endpoints/mod.rs b/app/storyteller_web/src/http_server/endpoints/mod.rs index 3842da000d..1bb6fcabce 100644 --- a/app/storyteller_web/src/http_server/endpoints/mod.rs +++ b/app/storyteller_web/src/http_server/endpoints/mod.rs @@ -3,6 +3,5 @@ pub mod leaderboard; pub mod misc; pub mod moderation; pub mod tts; -pub mod twitch; pub mod users; pub mod w2l; \ No newline at end of file diff --git a/app/storyteller_web/src/http_server/endpoints/twitch/mod.rs b/app/storyteller_web/src/http_server/endpoints/twitch/mod.rs deleted file mode 100644 index f1d932d175..0000000000 --- a/app/storyteller_web/src/http_server/endpoints/twitch/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod pubsub_gateway; \ No newline at end of file diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index 3d1aacdbb7..71713623e7 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -66,7 +66,6 @@ use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_u use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; -use crate::http_server::endpoints::twitch::pubsub_gateway::ws_index; use crate::http_server::endpoints::users::create_account::create_account_handler; use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; use crate::http_server::endpoints::users::get_profile::get_profile_handler; @@ -660,12 +659,6 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(leaderboard_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - // Twitch - .service( - web::resource("/twitch") - .route(web::get().to(ws_index)) - .route(web::head().to(|| HttpResponse::Ok())) - ) .service(get_root_index) .service(enable_alpha_handler) .service(enable_alpha_easy_handler) diff --git a/app/storyteller_web/src/obs_gateway/mod.rs b/app/storyteller_web/src/obs_gateway/mod.rs deleted file mode 100644 index e69de29bb2..0000000000 From 1fed826127c1fc97d44c1bac3af2be7982c13fd7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 7 Nov 2021 15:39:24 -0500 Subject: [PATCH 0642/1042] line count statistics fix --- loc.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/loc.sh b/loc.sh index 7f422b148f..babc6e8a00 100755 --- a/loc.sh +++ b/loc.sh @@ -3,9 +3,9 @@ wc -l \ Cargo.toml \ Dockerfile \ migrations/*/*.sql \ - src/*.rs \ - src/*/*.rs \ - src/*/*/*.rs \ - src/*/*/*/*.rs \ - src/*/*/*/*/*.rs \ + app/*/*/*.rs \ + app/*/*/*/*.rs \ + app/*/*/*/*/*.rs \ + app/*/*/*/*/*/*.rs \ + app/*/*/*/*/*/*/*.rs \ | sort -n From 9b2d2957ca4dbeeca160f592f7f77a5e1ecc4307 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 7 Nov 2021 19:49:35 -0500 Subject: [PATCH 0643/1042] websocket examples --- test.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test.js b/test.js index e5e40c533b..c477d0f0a6 100644 --- a/test.js +++ b/test.js @@ -2,9 +2,12 @@ var sock = new WebSocket("ws://localhost:12345/twitch"); sock.onopen = function (event) { - //sock.send("Here's some text that the server is urgently awaiting!"); + console.log('on open event', event); + sock.send('on open message from client'); }; sock.onmessage = function (event) { - console.log('got a message', event.data); -} \ No newline at end of file + console.log('on message event', event.data); +} + +sock.send('this is a message from the client'); From 60e0ef5860d15604591b1d8c70529fa498e1fd0e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 7 Nov 2021 23:32:21 -0500 Subject: [PATCH 0644/1042] Twitch secrets handling, AnyhowResult in common crate, etc. --- .gitignore | 1 + Cargo.lock | 354 +++++++++++++++---- Cargo.toml | 1 + app/container_common/Cargo.toml | 18 + app/container_common/src/anyhow_result.rs | 2 + app/container_common/src/lib.rs | 1 + app/obs_gateway/Cargo.toml | 6 + app/obs_gateway/src/main.rs | 9 + app/obs_gateway/src/twitch/mod.rs | 1 + app/obs_gateway/src/twitch/twitch_secrets.rs | 43 +++ 10 files changed, 372 insertions(+), 64 deletions(-) create mode 100644 app/container_common/Cargo.toml create mode 100644 app/container_common/src/anyhow_result.rs create mode 100644 app/container_common/src/lib.rs create mode 100644 app/obs_gateway/src/twitch/mod.rs create mode 100644 app/obs_gateway/src/twitch/twitch_secrets.rs diff --git a/.gitignore b/.gitignore index 9f00f24867..49d8c6ee90 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ production_database_secret_connect.sh secrets.toml secrets.yaml target/ +twitch_secrets.toml diff --git a/Cargo.lock b/Cargo.lock index 4e12b9dbc1..9be78428eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,7 +22,7 @@ dependencies = [ "pin-project 0.4.28", "smallvec 1.6.1", "tokio 0.2.25", - "tokio-util", + "tokio-util 0.3.1", "trust-dns-proto", "trust-dns-resolver", ] @@ -40,7 +40,7 @@ dependencies = [ "log", "pin-project 0.4.28", "tokio 0.2.25", - "tokio-util", + "tokio-util 0.3.1", ] [[package]] @@ -122,7 +122,7 @@ dependencies = [ "futures-core", "futures-util", "fxhash", - "h2", + "h2 0.2.7", "http", "httparse", "indexmap", @@ -446,6 +446,26 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliri_braid" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4961918d91b2182cf3adc6e3cd8d6f44452e0f42db367691447af31afdd1ad7d" +dependencies = [ + "aliri_braid_impl", +] + +[[package]] +name = "aliri_braid_impl" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e21d6298c933984056b381a70c7248a97370b76dfdbc95dc3da0fcbcd994342" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "anyhow" version = "1.0.40" @@ -578,9 +598,9 @@ checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" [[package]] name = "async-trait" -version = "0.1.50" +version = "0.1.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" +checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" dependencies = [ "proc-macro2", "quote", @@ -1005,6 +1025,13 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec" +[[package]] +name = "container_common" +version = "0.0.1" +dependencies = [ + "anyhow", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -1255,13 +1282,24 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "displaydoc" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dlv-list" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68df3f2b690c1b86e65ef7830956aededf3cb0a16f898f79b9a6f421a7b6211b" dependencies = [ - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -1424,9 +1462,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" +checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca" dependencies = [ "futures-channel", "futures-core", @@ -1439,9 +1477,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" +checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888" dependencies = [ "futures-core", "futures-sink", @@ -1449,15 +1487,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" +checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" [[package]] name = "futures-executor" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" +checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c" dependencies = [ "futures-core", "futures-task", @@ -1467,9 +1505,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" +checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" [[package]] name = "futures-lite" @@ -1488,9 +1526,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" +checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" dependencies = [ "autocfg 1.0.1", "proc-macro-hack", @@ -1501,21 +1539,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" +checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11" [[package]] name = "futures-task" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" +checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" [[package]] name = "futures-util" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" +checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" dependencies = [ "autocfg 1.0.1", "futures-channel", @@ -1610,11 +1648,30 @@ dependencies = [ "indexmap", "slab", "tokio 0.2.25", - "tokio-util", + "tokio-util 0.3.1", "tracing", "tracing-futures", ] +[[package]] +name = "h2" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd819562fcebdac5afc5c113c3ec36f902840b70fd4fc458799c8ce4607ae55" +dependencies = [ + "bytes 1.0.1", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 1.6.0", + "tokio-util 0.6.9", + "tracing", +] + [[package]] name = "hashbrown" version = "0.7.2" @@ -1700,9 +1757,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" +checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" dependencies = [ "bytes 1.0.1", "fnv", @@ -1719,6 +1776,17 @@ dependencies = [ "http", ] +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes 1.0.1", + "http", + "pin-project-lite 0.2.6", +] + [[package]] name = "http_server_common" version = "0.0.1" @@ -1743,11 +1811,11 @@ dependencies = [ "dotenv", "easyenv", "futures 0.1.31", - "futures 0.3.15", + "futures 0.3.17", "hex", "hmac 0.10.1", "hostname", - "hyper", + "hyper 0.13.10", "infer", "jwt", "limitation", @@ -1757,7 +1825,7 @@ dependencies = [ "md-5", "pulldown-cmark", "r2d2_redis", - "rand 0.8.3", + "rand 0.8.4", "regex", "ring", "rust-s3", @@ -1786,6 +1854,12 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" +[[package]] +name = "httpdate" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" + [[package]] name = "humantime" version = "1.3.0" @@ -1805,11 +1879,11 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.2.7", "http", - "http-body", + "http-body 0.3.1", "httparse", - "httpdate", + "httpdate 0.3.2", "itoa", "pin-project 1.0.7", "socket2 0.3.19", @@ -1819,6 +1893,30 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "0.14.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b91bb1f221b6ea1f1e4371216b70f40748774c2fb5971b450c07773fb92d26b" +dependencies = [ + "bytes 1.0.1", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.7", + "http", + "http-body 0.4.4", + "httparse", + "httpdate 1.0.1", + "itoa", + "pin-project-lite 0.2.6", + "socket2 0.4.0", + "tokio 1.6.0", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper-tls" version = "0.4.3" @@ -1826,7 +1924,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" dependencies = [ "bytes 0.5.6", - "hyper", + "hyper 0.13.10", "native-tls", "tokio 0.2.25", "tokio-tls", @@ -2283,8 +2381,8 @@ dependencies = [ "anyhow", "env_logger", "flate2", - "futures 0.3.15", - "hyper", + "futures 0.3.17", + "hyper 0.13.10", "hyper-tls", "log", "rand 0.7.3", @@ -2426,18 +2524,19 @@ dependencies = [ "bcrypt", "chrono", "config", + "container_common", "data-encoding", "derive_more", "diesel", "dotenv", "easyenv", "futures 0.1.31", - "futures 0.3.15", + "futures 0.3.17", "hex", "hmac 0.10.1", "hostname", "http_server_common", - "hyper", + "hyper 0.13.10", "infer", "jwt", "lazy_static", @@ -2450,7 +2549,7 @@ dependencies = [ "newrelic-telemetry", "pulldown-cmark", "r2d2_redis", - "rand 0.8.3", + "rand 0.8.4", "regex", "ring", "rust-s3", @@ -2464,14 +2563,16 @@ dependencies = [ "time 0.2.26", "tokio 0.2.25", "toml", + "twitch_api2", + "twitch_oauth2", "uuid", ] [[package]] name = "once_cell" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" [[package]] name = "opaque-debug" @@ -2797,9 +2898,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" dependencies = [ "libc", "rand_chacha 0.3.0", @@ -2987,8 +3088,8 @@ dependencies = [ "futures-core", "futures-util", "http", - "http-body", - "hyper", + "http-body 0.3.1", + "hyper 0.13.10", "hyper-tls", "ipnet", "js-sys", @@ -3011,6 +3112,38 @@ dependencies = [ "winreg 0.7.0", ] +[[package]] +name = "reqwest" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d2927ca2f685faf0fc620ac4834690d29e7abb153add10f5812eef20b5e280" +dependencies = [ + "base64 0.13.0", + "bytes 1.0.1", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body 0.4.4", + "hyper 0.14.14", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "percent-encoding 2.1.0", + "pin-project-lite 0.2.6", + "serde", + "serde_json", + "serde_urlencoded", + "tokio 1.6.0", + "url 2.2.2", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.7.0", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -3080,14 +3213,14 @@ dependencies = [ "base64 0.13.0", "cfg-if 1.0.0", "chrono", - "futures 0.3.15", + "futures 0.3.17", "hex", "hmac 0.9.0", "http", "log", "md5", "percent-encoding 2.1.0", - "reqwest", + "reqwest 0.10.10", "serde", "serde-xml-rs", "serde_derive", @@ -3197,9 +3330,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.125" +version = "1.0.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" +checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" dependencies = [ "serde_derive", ] @@ -3218,9 +3351,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.125" +version = "1.0.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" +checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" dependencies = [ "proc-macro2", "quote", @@ -3229,9 +3362,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +checksum = "e466864e431129c7e0d3476b92f20458e5879919a0596c6472738d9fa2d342f8" dependencies = [ "indexmap", "itoa", @@ -3239,6 +3372,26 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0421d4f173fab82d72d6babf36d57fae38b994ca5c2d78e704260ba6d12118b" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_repr" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98d0516900518c29efa217c298fa1f4e6c6ffc85ae29fd7f4ee48f176e1a9ed5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_urlencoded" version = "0.7.0" @@ -3436,7 +3589,7 @@ dependencies = [ "cargo_metadata", "dotenv", "either", - "futures 0.3.15", + "futures 0.3.17", "heck", "hex", "lazy_static", @@ -3552,12 +3705,12 @@ dependencies = [ "dotenv", "easyenv", "futures 0.1.31", - "futures 0.3.15", + "futures 0.3.17", "hex", "hmac 0.10.1", "hostname", "http_server_common", - "hyper", + "hyper 0.13.10", "infer", "jwt", "lazy_static", @@ -3570,7 +3723,7 @@ dependencies = [ "newrelic-telemetry", "pulldown-cmark", "r2d2_redis", - "rand 0.8.3", + "rand 0.8.4", "regex", "ring", "rust-s3", @@ -3660,7 +3813,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", "libc", - "rand 0.8.3", + "rand 0.8.4", "redox_syscall 0.2.8", "remove_dir_all", "winapi 0.3.9", @@ -3677,18 +3830,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.24" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.24" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" dependencies = [ "proc-macro2", "quote", @@ -3809,7 +3962,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37" dependencies = [ "autocfg 1.0.1", + "bytes 1.0.1", "libc", + "memchr", "mio 0.7.11", "once_cell", "parking_lot 0.11.1", @@ -3957,6 +4112,20 @@ dependencies = [ "tokio 0.2.25", ] +[[package]] +name = "tokio-util" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +dependencies = [ + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.6", + "tokio 1.6.0", +] + [[package]] name = "toml" version = "0.5.8" @@ -4012,7 +4181,7 @@ dependencies = [ "async-trait", "cfg-if 1.0.0", "enum-as-inner", - "futures 0.3.15", + "futures 0.3.17", "idna 0.2.3", "lazy_static", "log", @@ -4030,7 +4199,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "710f593b371175db53a26d0b38ed2978fafb9e9e8d3868b1acd753ea18df0ceb" dependencies = [ "cfg-if 0.1.10", - "futures 0.3.15", + "futures 0.3.17", "ipconfig", "lazy_static", "log", @@ -4048,6 +4217,52 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "twitch_api2" +version = "0.6.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "600ce224d5eafc176688f05836bbcfec3d314956b5f4a5a360218b4289923dd5" +dependencies = [ + "aliri_braid", + "async-trait", + "chrono", + "displaydoc", + "futures 0.3.17", + "http", + "once_cell", + "reqwest 0.11.6", + "serde", + "serde_derive", + "serde_json", + "serde_path_to_error", + "serde_repr", + "thiserror", + "twitch_oauth2", + "typed-builder", + "url 2.2.2", + "version_check 0.9.3", +] + +[[package]] +name = "twitch_oauth2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f454ca1e9b8dc97a7255ff50e13a8a1d6ffab8ce6b9445cfc331d6420d40a953" +dependencies = [ + "aliri_braid", + "async-trait", + "base64 0.13.0", + "displaydoc", + "http", + "once_cell", + "rand 0.8.4", + "reqwest 0.11.6", + "serde", + "serde_json", + "thiserror", + "url 2.2.2", +] + [[package]] name = "twoway" version = "0.2.2" @@ -4058,6 +4273,17 @@ dependencies = [ "unchecked-index", ] +[[package]] +name = "typed-builder" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a46ee5bd706ff79131be9c94e7edcb82b703c487766a114434e5790361cf08c5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "typenum" version = "1.13.0" diff --git a/Cargo.toml b/Cargo.toml index a5f41170c5..2c2cacc9dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "app/config", + "app/container_common", "app/http_server_common", "app/obs_gateway", "app/storyteller_web", diff --git a/app/container_common/Cargo.toml b/app/container_common/Cargo.toml new file mode 100644 index 0000000000..eea6a8b7cc --- /dev/null +++ b/app/container_common/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "container_common" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "container_common" +path = "src/lib.rs" + +[dependencies] + +# External +anyhow = "1.0.40" diff --git a/app/container_common/src/anyhow_result.rs b/app/container_common/src/anyhow_result.rs new file mode 100644 index 0000000000..7784374637 --- /dev/null +++ b/app/container_common/src/anyhow_result.rs @@ -0,0 +1,2 @@ +/// Easier to import than anyhow::Result. +pub type AnyhowResult = anyhow::Result; diff --git a/app/container_common/src/lib.rs b/app/container_common/src/lib.rs new file mode 100644 index 0000000000..1c12804a22 --- /dev/null +++ b/app/container_common/src/lib.rs @@ -0,0 +1 @@ +pub mod anyhow_result; diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index d4c5d95952..6a362ca5fa 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" # Internal config = { path = "../config" } +container_common = { path = "../container_common" } http_server_common = { path = "../http_server_common" } # External @@ -60,6 +61,11 @@ sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "off limitation = "0.1.1" # Redis rate limiter r2d2_redis = "0.14.0" +# Twitch +# NB "unsupported" pubsub features may disappear from the Twitch API at any time. It's an unofficial API. +twitch_api2 = { version = "0.6.0-rc.2", features = ["helix", "reqwest_client", "pubsub", "chrono", "unsupported"] } +twitch_oauth2 = "0.6.0-rc.2" + # In-memory caching lru_time_cache = "0.11.11" diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index 04256e0009..a0668374c3 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -13,6 +13,7 @@ #[macro_use] extern crate serde_derive; pub mod pubsub_gateway; +pub mod twitch; use actix_cors::Cors; use actix_http::http; @@ -22,6 +23,7 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use crate::pubsub_gateway::ws_index; +use crate::twitch::twitch_secrets::TwitchSecrets; use futures::Future; use futures::executor::ThreadPool; use http_server_common::endpoints::default_route_404::default_route_404; @@ -35,6 +37,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; +use twitch_oauth2::{AppAccessToken, Scope, TwitchToken, tokens::errors::AppAccessTokenError, ClientId, ClientSecret}; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -77,6 +80,12 @@ async fn main() -> AnyhowResult<()> { info!("Hostname: {}", &server_hostname); + info!("Reading Twitch secrets..."); + + let secrets = TwitchSecrets::from_file("twitch_secrets.toml")?; + let client_id = ClientId::new(&secrets.app_client_id); + let client_secret = ClientSecret::new(&secrets.app_client_secret); + info!("Connecting to database..."); let db_connection_string = diff --git a/app/obs_gateway/src/twitch/mod.rs b/app/obs_gateway/src/twitch/mod.rs new file mode 100644 index 0000000000..ba70f181c9 --- /dev/null +++ b/app/obs_gateway/src/twitch/mod.rs @@ -0,0 +1 @@ +pub mod twitch_secrets; \ No newline at end of file diff --git a/app/obs_gateway/src/twitch/twitch_secrets.rs b/app/obs_gateway/src/twitch/twitch_secrets.rs new file mode 100644 index 0000000000..2cb6c72e55 --- /dev/null +++ b/app/obs_gateway/src/twitch/twitch_secrets.rs @@ -0,0 +1,43 @@ +use container_common::anyhow_result::AnyhowResult; + +// TODO: Not sure I need all of these. The less I need to keep, the better. +// Oauth delegation in particular seems not necessary. +#[derive(Clone, serde::Deserialize)] +pub struct TwitchSecrets { + pub username: String, + pub stream_key: String, + pub app_client_id: String, + pub app_client_secret: String, +} + +impl TwitchSecrets { + pub fn new( + username: &str, + stream_key: &str, + app_client_id: &str, + app_client_secret: &str, + ) -> Self { + Self { + username: username.to_string(), + stream_key: stream_key.to_string(), + app_client_id: app_client_id.to_string(), + app_client_secret: app_client_secret.to_string(), + } + } + + pub fn from_env() -> AnyhowResult { + Ok(Self::new( + &easyenv::get_env_string_required("TWITCH_USERNAME")?, + &easyenv::get_env_string_required("TWITCH_STREAM_KEY")?, + &easyenv::get_env_string_required("TWITCH_APP_CLIENT_ID")?, + &easyenv::get_env_string_required("TWITCH_APP_CLIENT_SECRET")?, + )) + } + + pub fn from_file(filename: &str) -> AnyhowResult { + let contents = std::fs::read_to_string(filename)?; + let secrets = toml::from_str(&contents)?; + Ok(secrets) + } +} + From 5809dd7c404800e71ac8517df91619f58b55ee49 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 7 Nov 2021 23:39:54 -0500 Subject: [PATCH 0645/1042] Share CORS configs --- app/http_server_common/src/cors.rs | 44 ++++++++++++++++++++++++++++++ app/http_server_common/src/lib.rs | 1 + app/obs_gateway/src/main.rs | 39 ++------------------------ app/storyteller_web/src/main.rs | 39 ++------------------------ 4 files changed, 49 insertions(+), 74 deletions(-) create mode 100644 app/http_server_common/src/cors.rs diff --git a/app/http_server_common/src/cors.rs b/app/http_server_common/src/cors.rs new file mode 100644 index 0000000000..5e7ea5f0b7 --- /dev/null +++ b/app/http_server_common/src/cors.rs @@ -0,0 +1,44 @@ +use actix_cors::Cors; +use actix_http::http; + +// TODO: Take an environment. +/// Return cors config for FakeYou / Vocodes / OBS / local development +pub fn build_common_cors_config() -> Cors { + Cors::default() + .allowed_origin("http://api.fakeyou.com") + .allowed_origin("http://api.jungle.horse") + .allowed_origin("http://api.vo.codes") + .allowed_origin("http://fakeyou.com") + .allowed_origin("http://jungle.horse") + .allowed_origin("http://jungle.horse:12345") + .allowed_origin("http://jungle.horse:7000") + .allowed_origin("http://localhost:12345") + .allowed_origin("http://localhost:3000") + .allowed_origin("http://localhost:5555") + .allowed_origin("http://localhost:7000") + .allowed_origin("http://localhost:8000") + .allowed_origin("http://localhost:8080") + .allowed_origin("http://mumble.stream") + .allowed_origin("http://trumped.com") + .allowed_origin("http://vo.codes") + .allowed_origin("http://vocodes.com") + .allowed_origin("https://api.fakeyou.com") + .allowed_origin("https://api.jungle.horse") + .allowed_origin("https://api.vo.codes") + .allowed_origin("https://fakeyou.com") + .allowed_origin("https://jungle.horse") + .allowed_origin("https://mumble.stream") + .allowed_origin("https://trumped.com") + .allowed_origin("https://vo.codes") + .allowed_origin("https://vocodes.com") + .allowed_methods(vec!["GET", "POST", "OPTIONS"]) + .supports_credentials() + .allowed_headers(vec![ + http::header::ACCEPT, + http::header::ACCESS_CONTROL_ALLOW_ORIGIN, // Tabulator Ajax + http::header::CONTENT_TYPE, + http::header::ACCESS_CONTROL_ALLOW_CREDENTIALS, // https://stackoverflow.com/a/46412839 + http::header::HeaderName::from_static("x-requested-with") // Tabulator Ajax sends + ]) + .max_age(3600) +} \ No newline at end of file diff --git a/app/http_server_common/src/lib.rs b/app/http_server_common/src/lib.rs index 34440e0e43..5f1e14a349 100644 --- a/app/http_server_common/src/lib.rs +++ b/app/http_server_common/src/lib.rs @@ -3,3 +3,4 @@ //! pub mod endpoints; +pub mod cors; diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index a0668374c3..16bea80049 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -26,6 +26,7 @@ use crate::pubsub_gateway::ws_index; use crate::twitch::twitch_secrets::TwitchSecrets; use futures::Future; use futures::executor::ThreadPool; +use http_server_common::cors::build_common_cors_config; use http_server_common::endpoints::default_route_404::default_route_404; use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; @@ -151,43 +152,7 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> HttpServer::new(move || { App::new() .app_data(server_state_arc.clone()) - .wrap(Cors::default() - .allowed_origin("http://api.fakeyou.com") - .allowed_origin("http://api.jungle.horse") - .allowed_origin("http://api.vo.codes") - .allowed_origin("http://fakeyou.com") - .allowed_origin("http://jungle.horse") - .allowed_origin("http://jungle.horse:12345") - .allowed_origin("http://jungle.horse:7000") - .allowed_origin("http://localhost:12345") - .allowed_origin("http://localhost:3000") - .allowed_origin("http://localhost:5555") - .allowed_origin("http://localhost:7000") - .allowed_origin("http://localhost:8000") - .allowed_origin("http://localhost:8080") - .allowed_origin("http://mumble.stream") - .allowed_origin("http://trumped.com") - .allowed_origin("http://vo.codes") - .allowed_origin("http://vocodes.com") - .allowed_origin("https://api.fakeyou.com") - .allowed_origin("https://api.jungle.horse") - .allowed_origin("https://api.vo.codes") - .allowed_origin("https://fakeyou.com") - .allowed_origin("https://jungle.horse") - .allowed_origin("https://mumble.stream") - .allowed_origin("https://trumped.com") - .allowed_origin("https://vo.codes") - .allowed_origin("https://vocodes.com") - .allowed_methods(vec!["GET", "POST", "OPTIONS"]) - .supports_credentials() - .allowed_headers(vec![ - http::header::ACCEPT, - http::header::ACCESS_CONTROL_ALLOW_ORIGIN, // Tabulator Ajax - http::header::CONTENT_TYPE, - http::header::ACCESS_CONTROL_ALLOW_CREDENTIALS, // https://stackoverflow.com/a/46412839 - http::header::HeaderName::from_static("x-requested-with") // Tabulator Ajax sends - ]) - .max_age(3600)) + .wrap(build_common_cors_config()) .wrap(Logger::new(&log_format) .exclude("/liveness") .exclude("/readiness")) diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index 71713623e7..40c16c4de7 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -102,6 +102,7 @@ use crate::util::caching::single_item_ttl_cache::SingleItemTtlCache; use crate::util::encrypted_sort_id::SortKeyCrypto; use futures::Future; use futures::executor::ThreadPool; +use http_server_common::cors::build_common_cors_config; use http_server_common::endpoints::default_route_404::default_route_404; use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; @@ -317,43 +318,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> App::new() .app_data(server_state_arc.clone()) - .wrap(Cors::default() - .allowed_origin("http://api.fakeyou.com") - .allowed_origin("http://api.jungle.horse") - .allowed_origin("http://api.vo.codes") - .allowed_origin("http://fakeyou.com") - .allowed_origin("http://jungle.horse") - .allowed_origin("http://jungle.horse:12345") - .allowed_origin("http://jungle.horse:7000") - .allowed_origin("http://localhost:12345") - .allowed_origin("http://localhost:3000") - .allowed_origin("http://localhost:5555") - .allowed_origin("http://localhost:7000") - .allowed_origin("http://localhost:8000") - .allowed_origin("http://localhost:8080") - .allowed_origin("http://mumble.stream") - .allowed_origin("http://trumped.com") - .allowed_origin("http://vo.codes") - .allowed_origin("http://vocodes.com") - .allowed_origin("https://api.fakeyou.com") - .allowed_origin("https://api.jungle.horse") - .allowed_origin("https://api.vo.codes") - .allowed_origin("https://fakeyou.com") - .allowed_origin("https://jungle.horse") - .allowed_origin("https://mumble.stream") - .allowed_origin("https://trumped.com") - .allowed_origin("https://vo.codes") - .allowed_origin("https://vocodes.com") - .allowed_methods(vec!["GET", "POST", "OPTIONS"]) - .supports_credentials() - .allowed_headers(vec![ - http::header::ACCEPT, - http::header::ACCESS_CONTROL_ALLOW_ORIGIN, // Tabulator Ajax - http::header::CONTENT_TYPE, - http::header::ACCESS_CONTROL_ALLOW_CREDENTIALS, // https://stackoverflow.com/a/46412839 - http::header::HeaderName::from_static("x-requested-with") // Tabulator Ajax sends - ]) - .max_age(3600)) + .wrap(build_common_cors_config()) .wrap(Logger::new(&log_format) .exclude("/liveness") .exclude("/readiness")) From a727e261d1e9cdd28978159d417d43b22ec710ca Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 8 Nov 2021 00:19:56 -0500 Subject: [PATCH 0646/1042] Figuring out Tokyo runtimes between projects (ugh...) --- Cargo.lock | 56 +++++++++++++--- app/obs_gateway/Cargo.toml | 11 +++- app/obs_gateway/src/main.rs | 51 ++++++++++++++- app/obs_gateway/src/twitch/mod.rs | 1 + .../src/twitch/twitch_client_wrapper.rs | 65 +++++++++++++++++++ 5 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 app/obs_gateway/src/twitch/twitch_client_wrapper.rs diff --git a/Cargo.lock b/Cargo.lock index 9be78428eb..3b50f46aba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1930,6 +1930,19 @@ dependencies = [ "tokio-tls", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes 1.0.1", + "hyper 0.14.14", + "native-tls", + "tokio 1.6.0", + "tokio-native-tls 0.3.0", +] + [[package]] name = "idna" version = "0.1.5" @@ -2347,9 +2360,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" dependencies = [ "lazy_static", "libc", @@ -2383,7 +2396,7 @@ dependencies = [ "flate2", "futures 0.3.17", "hyper 0.13.10", - "hyper-tls", + "hyper-tls 0.4.3", "log", "rand 0.7.3", "serde", @@ -2546,11 +2559,12 @@ dependencies = [ "lru_time_cache", "magic-crypt", "md-5", - "newrelic-telemetry", + "native-tls", "pulldown-cmark", "r2d2_redis", "rand 0.8.4", "regex", + "reqwest 0.11.6", "ring", "rust-s3", "serde", @@ -2561,7 +2575,7 @@ dependencies = [ "subprocess", "tempdir", "time 0.2.26", - "tokio 0.2.25", + "tokio 1.6.0", "toml", "twitch_api2", "twitch_oauth2", @@ -3090,7 +3104,7 @@ dependencies = [ "http", "http-body 0.3.1", "hyper 0.13.10", - "hyper-tls", + "hyper-tls 0.4.3", "ipnet", "js-sys", "lazy_static", @@ -3126,17 +3140,20 @@ dependencies = [ "http", "http-body 0.4.4", "hyper 0.14.14", + "hyper-tls 0.5.0", "ipnet", "js-sys", "lazy_static", "log", "mime", + "native-tls", "percent-encoding 2.1.0", "pin-project-lite 0.2.6", "serde", "serde_json", "serde_urlencoded", "tokio 1.6.0", + "tokio-native-tls 0.3.0", "url 2.2.2", "wasm-bindgen", "wasm-bindgen-futures", @@ -3613,7 +3630,7 @@ dependencies = [ "native-tls", "once_cell", "tokio 0.2.25", - "tokio-native-tls", + "tokio-native-tls 0.1.0", ] [[package]] @@ -3951,7 +3968,7 @@ dependencies = [ "pin-project-lite 0.1.12", "signal-hook-registry", "slab", - "tokio-macros", + "tokio-macros 0.2.6", "winapi 0.3.9", ] @@ -3966,10 +3983,12 @@ dependencies = [ "libc", "memchr", "mio 0.7.11", + "num_cpus", "once_cell", "parking_lot 0.11.1", "pin-project-lite 0.2.6", "signal-hook-registry", + "tokio-macros 1.5.1", "winapi 0.3.9", ] @@ -4016,6 +4035,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-macros" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "114383b041aa6212c579467afa0075fbbdd0718de036100bc0ba7961d8cb9095" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-native-tls" version = "0.1.0" @@ -4026,6 +4056,16 @@ dependencies = [ "tokio 0.2.25", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio 1.6.0", +] + [[package]] name = "tokio-reactor" version = "0.1.12" diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index 6a362ca5fa..3cdc80d250 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -34,7 +34,7 @@ lazy_static = "1.4.0" lfu = "0.2.5" log = "0.4.14" #newrelic = "0.2.2" -newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" } +#newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" } rand = "0.8.3" regex = "1.5.4" time = "0.2.26" @@ -46,7 +46,9 @@ uuid = { version = "0.8.2", features = ["v4"] } # NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: # https://github.com/durch/rust-s3/issues/198 #tokio = { version = "1.6.0", features = ["macros"] } -tokio = { version = "0.2.25", features = ["macros"] } +#tokio = { version = "0.2.25", features = ["macros"] } +# NB(3,2021-11-07): Actually, maybe we can do this in a complex multi-project monorepo +tokio = { version = "1.6.0", features = ["macros", "rt-multi-thread"] } # MySql Database # "offline" supports typesafe builds without a live database @@ -66,6 +68,11 @@ r2d2_redis = "0.14.0" twitch_api2 = { version = "0.6.0-rc.2", features = ["helix", "reqwest_client", "pubsub", "chrono", "unsupported"] } twitch_oauth2 = "0.6.0-rc.2" +# HTTP +reqwest = { version = "0.11.4" } + +native-tls = "0.2.8" + # In-memory caching lru_time_cache = "0.11.11" diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index 16bea80049..b802557a75 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -38,7 +38,9 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; +use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{AppAccessToken, Scope, TwitchToken, tokens::errors::AppAccessTokenError, ClientId, ClientSecret}; +use crate::twitch::twitch_client_wrapper::TwitchClientWrapper; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -64,7 +66,8 @@ pub struct EnvConfig { pub website_homepage_redirect: String, } -#[actix_web::main] +//#[actix_web::main] +#[tokio::main] async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); @@ -87,6 +90,52 @@ async fn main() -> AnyhowResult<()> { let client_id = ClientId::new(&secrets.app_client_id); let client_secret = ClientSecret::new(&secrets.app_client_secret); + + // ======================================================== + + info!("Getting app access token..."); + + //let scopes = Scope::all(); + let scopes = vec![ + Scope::BitsRead, + Scope::UserReadEmail, + ]; + + let mut twitch_client = TwitchClientWrapper::new(client_id.clone(), client_secret.clone()); + twitch_client.request_access_token(scopes).await?; + + info!("Getting user id ..."); + + //let user_id = twitch_client.get_user_id_from_username("testytest512").await?; + let user_id = twitch_client.get_user_id_from_username("vocodes").await?; + + info!("User ID: {}", user_id); + + //std::thread::sleep(Duration::from_secs(5000)); + + + // ==================== OAUTH FLOW ==================== + + println!("Oauth flow..."); + + let redirect_url = twitch_oauth2::url::Url::parse("http://localhost/test")?; + let mut builder = UserTokenBuilder::new(client_id, client_secret, redirect_url) + .set_scopes(Scope::all()) + .force_verify(true); + + //builder.add_scope(Scope::BitsRead); + + let (url, _csrf_token) = builder.generate_url(); + + println!("Go to this page: {}", url); + + + + + + // ======================================================== + + info!("Connecting to database..."); let db_connection_string = diff --git a/app/obs_gateway/src/twitch/mod.rs b/app/obs_gateway/src/twitch/mod.rs index ba70f181c9..a021988480 100644 --- a/app/obs_gateway/src/twitch/mod.rs +++ b/app/obs_gateway/src/twitch/mod.rs @@ -1 +1,2 @@ +pub mod twitch_client_wrapper; pub mod twitch_secrets; \ No newline at end of file diff --git a/app/obs_gateway/src/twitch/twitch_client_wrapper.rs b/app/obs_gateway/src/twitch/twitch_client_wrapper.rs new file mode 100644 index 0000000000..881584e87b --- /dev/null +++ b/app/obs_gateway/src/twitch/twitch_client_wrapper.rs @@ -0,0 +1,65 @@ +//! The Helix client is independent of PubSub +//! This just wraps `twitch_api2` for more convenience. + +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use twitch_api2::TwitchClient; +use twitch_api2::helix::users::GetUsersRequest; +use twitch_api2::types::{Nickname, UserId}; +use twitch_oauth2::{ClientSecret, ClientId, AccessToken, AppAccessToken, Scope}; + +pub struct TwitchClientWrapper<'a> { + client_id: ClientId, + client_secret: ClientSecret, + twitch_client: TwitchClient<'a, reqwest::Client>, + access_token: Option, +} + +impl <'a> TwitchClientWrapper <'a> { + pub fn new(client_id: ClientId, client_secret: ClientSecret) -> Self { + Self { + client_id, + client_secret, + twitch_client: TwitchClient::default(), + access_token: None, + } + } + + pub async fn request_access_token(&mut self, scopes: Vec) -> AnyhowResult<()> { + self.get_access_token(scopes).await?; + Ok(()) + } + + async fn get_access_token(&mut self, scopes: Vec) -> AnyhowResult<()> { + let mut http_client = reqwest::Client::new(); + let access_token = twitch_oauth2::AppAccessToken::get_app_access_token( + &http_client, + self.client_id.clone(), + self.client_secret.clone(), + scopes, + ).await?; + + self.access_token = Some(access_token); + Ok(()) + } + + /// We frequently need to look up IDs from usernames. + pub async fn get_user_id_from_username(&self, username: &str) -> AnyhowResult { + let access_token = match self.access_token { + None => return Err(anyhow!("no access token has been granted")), + Some(ref token) => token, + }; + + let req = GetUsersRequest::builder() + .login(vec![Nickname::new(username)]) + .build(); + + let response = &self.twitch_client.helix.req_get(req, access_token) + .await?; + + match response.data.get(0) { + None => Err(anyhow!("No such user: `{}`.", username)), + Some(user) => Ok(user.id.as_str().parse::()?), + } + } +} From 06a327367cb180b34bc5f04cb5e1cfb9a84f6fa4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 8 Nov 2021 00:57:02 -0500 Subject: [PATCH 0647/1042] versions of tokio/actix work for OBS gateway --- Cargo.lock | 451 ++++++++++++++++++++++++++++++------ app/obs_gateway/Cargo.toml | 33 +-- app/obs_gateway/src/main.rs | 64 +++-- 3 files changed, 424 insertions(+), 124 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b50f46aba..b6ab1a3613 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1be241f88f3b1e7e9a3fbe3b5a8a0f6915b5a1d7ee0d9a248d3376d01068cc60" dependencies = [ "actix-rt 1.1.1", - "actix_derive", + "actix_derive 0.5.0", "bitflags", "bytes 0.5.6", "crossbeam-channel 0.4.4", @@ -27,6 +27,30 @@ dependencies = [ "trust-dns-resolver", ] +[[package]] +name = "actix" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3720d0064a0ce5c0de7bd93bdb0a6caebab2a9b5668746145d7b3b0c5da02914" +dependencies = [ + "actix-rt 2.4.0", + "actix_derive 0.6.0", + "bitflags", + "bytes 1.0.1", + "crossbeam-channel 0.5.1", + "futures-core", + "futures-sink", + "futures-task", + "futures-util", + "log", + "once_cell", + "parking_lot 0.11.1", + "pin-project-lite 0.2.6", + "smallvec 1.6.1", + "tokio 1.6.0", + "tokio-util 0.6.9", +] + [[package]] name = "actix-codec" version = "0.3.0" @@ -43,15 +67,32 @@ dependencies = [ "tokio-util 0.3.1", ] +[[package]] +name = "actix-codec" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13895df506faee81e423febbae3a33b27fca71831b96bb3d60adf16ebcfea952" +dependencies = [ + "bitflags", + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "memchr", + "pin-project-lite 0.2.6", + "tokio 1.6.0", + "tokio-util 0.6.9", +] + [[package]] name = "actix-connect" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" dependencies = [ - "actix-codec", + "actix-codec 0.3.0", "actix-rt 1.1.1", - "actix-service", + "actix-service 1.0.6", "actix-utils 2.0.0", "derive_more", "either", @@ -68,7 +109,7 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36b133d8026a9f209a9aeeeacd028e7451bcca975f592881b305d37983f303d7" dependencies = [ - "actix-web", + "actix-web 3.3.2", "derive_more", "futures-util", "log", @@ -82,8 +123,8 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51e8a9146c12fce92a6e4c24b8c4d9b05268130bfd8d61bc587e822c32ce689" dependencies = [ - "actix-service", - "actix-web", + "actix-service 1.0.6", + "actix-web 3.3.2", "bitflags", "bytes 0.5.6", "derive_more", @@ -102,17 +143,17 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "452299e87817ae5673910e53c243484ca38be3828db819b6011736fc6982e874" dependencies = [ - "actix-codec", + "actix-codec 0.3.0", "actix-connect", "actix-rt 1.1.1", - "actix-service", + "actix-service 1.0.6", "actix-threadpool", "actix-utils 2.0.0", "base64 0.13.0", "bitflags", "brotli2", "bytes 0.5.6", - "cookie", + "cookie 0.14.4", "copyless", "derive_more", "either", @@ -127,7 +168,7 @@ dependencies = [ "httparse", "indexmap", "itoa", - "language-tags", + "language-tags 0.2.2", "lazy_static", "log", "mime", @@ -143,6 +184,48 @@ dependencies = [ "time 0.2.26", ] +[[package]] +name = "actix-http" +version = "3.0.0-beta.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9b510d35f13987537289f38bf136e7e702a5c87cc28760310cc459544f40afd" +dependencies = [ + "actix-codec 0.4.1", + "actix-rt 2.4.0", + "actix-service 2.0.1", + "actix-tls 3.0.0-beta.7", + "actix-utils 3.0.0", + "ahash 0.7.4", + "base64 0.13.0", + "bitflags", + "brotli2", + "bytes 1.0.1", + "bytestring", + "derive_more", + "encoding_rs", + "flate2", + "futures-core", + "futures-util", + "h2 0.3.7", + "http", + "httparse", + "httpdate 1.0.1", + "itoa", + "language-tags 0.3.2", + "local-channel", + "log", + "mime", + "once_cell", + "percent-encoding 2.1.0", + "pin-project 1.0.7", + "pin-project-lite 0.2.6", + "rand 0.8.4", + "sha-1", + "smallvec 1.6.1", + "tokio 1.6.0", + "zstd", +] + [[package]] name = "actix-macros" version = "0.1.3" @@ -155,9 +238,9 @@ dependencies = [ [[package]] name = "actix-macros" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f86cd6857c135e6e9fe57b1619a88d1f94a7df34c00e11fe13e64fd3438837" +checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" dependencies = [ "quote", "syn", @@ -169,9 +252,9 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "774bfeb11b54bf9c857a005b8ab893293da4eaff79261a66a9200dab7f5ab6e3" dependencies = [ - "actix-service", + "actix-service 1.0.6", "actix-utils 2.0.0", - "actix-web", + "actix-web 3.3.2", "bytes 0.5.6", "derive_more", "futures-util", @@ -194,6 +277,20 @@ dependencies = [ "serde", ] +[[package]] +name = "actix-router" +version = "0.5.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36b95ce0d76d1aa2f98b681702807475ade0f99bd4552546a6843a966d42ea3d" +dependencies = [ + "bytestring", + "firestorm", + "http", + "log", + "regex", + "serde", +] + [[package]] name = "actix-rt" version = "1.1.1" @@ -211,11 +308,11 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc7d7cd957c9ed92288a7c3c96af81fa5291f65247a76a34dac7b6af74e52ba0" +checksum = "4a0c218d0a17c120f10ee0c69c9f0c45d87319e8f66b1f065e8412b612fc3e24" dependencies = [ - "actix-macros 0.2.1", + "actix-macros 0.2.3", "futures-core", "tokio 1.6.0", ] @@ -226,9 +323,9 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45407e6e672ca24784baa667c5d32ef109ccdd8d5e0b5ebb9ef8a67f4dfb708e" dependencies = [ - "actix-codec", + "actix-codec 0.3.0", "actix-rt 1.1.1", - "actix-service", + "actix-service 1.0.6", "actix-utils 2.0.0", "futures-channel", "futures-util", @@ -240,6 +337,22 @@ dependencies = [ "socket2 0.3.19", ] +[[package]] +name = "actix-server" +version = "2.0.0-beta.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7367665785765b066ad16e1086d26a087f696bc7c42b6f93004ced6cfcf1eeca" +dependencies = [ + "actix-rt 2.4.0", + "actix-service 2.0.1", + "actix-utils 3.0.0", + "futures-core", + "log", + "mio 0.7.11", + "num_cpus", + "tokio 1.6.0", +] + [[package]] name = "actix-service" version = "1.0.6" @@ -250,6 +363,17 @@ dependencies = [ "pin-project 0.4.28", ] +[[package]] +name = "actix-service" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3dc6a618b082974a08d7a4781d24d4691cba51500059bfebe6656a61ebfe1e" +dependencies = [ + "futures-core", + "paste", + "pin-project-lite 0.2.6", +] + [[package]] name = "actix-testing" version = "1.0.1" @@ -258,8 +382,8 @@ checksum = "47239ca38799ab74ee6a8a94d1ce857014b2ac36f242f70f3f75a66f691e791c" dependencies = [ "actix-macros 0.1.3", "actix-rt 1.1.1", - "actix-server", - "actix-service", + "actix-server 1.0.4", + "actix-service 1.0.6", "log", "socket2 0.3.19", ] @@ -285,21 +409,38 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24789b7d7361cf5503a504ebe1c10806896f61e96eca9a7350e23001aca715fb" dependencies = [ - "actix-codec", - "actix-service", + "actix-codec 0.3.0", + "actix-service 1.0.6", "actix-utils 2.0.0", "futures-util", ] +[[package]] +name = "actix-tls" +version = "3.0.0-beta.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af84e13e4600829858a3e68079be710d1ada461431e1e4c5ae663479ea0a3c" +dependencies = [ + "actix-codec 0.4.1", + "actix-rt 2.4.0", + "actix-service 2.0.1", + "actix-utils 3.0.0", + "derive_more", + "futures-core", + "http", + "log", + "tokio-util 0.6.9", +] + [[package]] name = "actix-utils" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" dependencies = [ - "actix-codec", + "actix-codec 0.3.0", "actix-rt 1.1.1", - "actix-service", + "actix-service 1.0.6", "bitflags", "bytes 0.5.6", "either", @@ -327,18 +468,18 @@ version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e641d4a172e7faa0862241a20ff4f1f5ab0ab7c279f00c2d4587b77483477b86" dependencies = [ - "actix-codec", - "actix-http", + "actix-codec 0.3.0", + "actix-http 2.2.0", "actix-macros 0.1.3", - "actix-router", + "actix-router 0.2.7", "actix-rt 1.1.1", - "actix-server", - "actix-service", + "actix-server 1.0.4", + "actix-service 1.0.6", "actix-testing", "actix-threadpool", - "actix-tls", + "actix-tls 2.0.0", "actix-utils 2.0.0", - "actix-web-codegen", + "actix-web-codegen 0.4.0", "awc", "bytes 0.5.6", "derive_more", @@ -360,22 +501,80 @@ dependencies = [ "url 2.2.2", ] +[[package]] +name = "actix-web" +version = "4.0.0-beta.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a4b9d00991d8da308070a5cea7f1bbaa153a91c3fb5567937d99b9f46d601e" +dependencies = [ + "actix-codec 0.4.1", + "actix-http 3.0.0-beta.11", + "actix-macros 0.2.3", + "actix-router 0.5.0-beta.2", + "actix-rt 2.4.0", + "actix-server 2.0.0-beta.6", + "actix-service 2.0.1", + "actix-utils 3.0.0", + "actix-web-codegen 0.5.0-beta.5", + "ahash 0.7.4", + "bytes 1.0.1", + "cfg-if 1.0.0", + "cookie 0.15.1", + "derive_more", + "either", + "encoding_rs", + "futures-core", + "futures-util", + "itoa", + "language-tags 0.3.2", + "log", + "mime", + "once_cell", + "paste", + "pin-project 1.0.7", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec 1.6.1", + "socket2 0.4.0", + "time 0.3.4", + "url 2.2.2", +] + [[package]] name = "actix-web-actors" version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f6edf3c2693e2a8c422800c87ee89a6a4eac7dd01109bc172a1093ce1f4f001" dependencies = [ - "actix", - "actix-codec", - "actix-http", - "actix-web", + "actix 0.10.0", + "actix-codec 0.3.0", + "actix-http 2.2.0", + "actix-web 3.3.2", "bytes 0.5.6", "futures-channel", "futures-core", "pin-project 0.4.28", ] +[[package]] +name = "actix-web-actors" +version = "4.0.0-beta.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b43c8e03f1877bc010316de455ee479aa0de0ae0717d49041db5f38fa192de" +dependencies = [ + "actix 0.12.0", + "actix-codec 0.4.1", + "actix-http 3.0.0-beta.11", + "actix-web 4.0.0-beta.10", + "bytes 1.0.1", + "bytestring", + "futures-core", + "pin-project 1.0.7", + "tokio 1.6.0", +] + [[package]] name = "actix-web-codegen" version = "0.4.0" @@ -387,6 +586,18 @@ dependencies = [ "syn", ] +[[package]] +name = "actix-web-codegen" +version = "0.5.0-beta.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfe80a8828fa88a0420dc8fdd4c16b8207326c917f17701881b063eadc2a8d3b" +dependencies = [ + "actix-router 0.5.0-beta.2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "actix_derive" version = "0.5.0" @@ -398,6 +609,17 @@ dependencies = [ "syn", ] +[[package]] +name = "actix_derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d44b8fee1ced9671ba043476deddef739dd0959bf77030b26b738cc591737a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "adler" version = "1.0.2" @@ -437,6 +659,17 @@ dependencies = [ "version_check 0.9.3", ] +[[package]] +name = "ahash" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" +dependencies = [ + "getrandom 0.2.2", + "once_cell", + "version_check 0.9.3", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -666,10 +899,10 @@ version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b381e490e7b0cfc37ebc54079b0413d8093ef43d14a4e4747083f7fa47a9e691" dependencies = [ - "actix-codec", - "actix-http", + "actix-codec 0.3.0", + "actix-http 2.2.0", "actix-rt 1.1.1", - "actix-service", + "actix-service 1.0.6", "base64 0.13.0", "bytes 0.5.6", "cfg-if 1.0.0", @@ -928,6 +1161,9 @@ name = "cc" version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +dependencies = [ + "jobserver", +] [[package]] name = "cfb" @@ -1049,6 +1285,17 @@ dependencies = [ "version_check 0.9.3", ] +[[package]] +name = "cookie" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f1c7727e460397e56abc4bddc1d49e07a1ad78fc98eb2e1c8f032a58a2f80d" +dependencies = [ + "percent-encoding 2.1.0", + "time 0.2.26", + "version_check 0.9.3", +] + [[package]] name = "copyless" version = "0.1.5" @@ -1383,6 +1630,12 @@ dependencies = [ "instant", ] +[[package]] +name = "firestorm" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31586bda1b136406162e381a3185a506cdfc1631708dd40cba2f6628d8634499" + [[package]] name = "flate2" version = "1.0.20" @@ -1791,16 +2044,16 @@ dependencies = [ name = "http_server_common" version = "0.0.1" dependencies = [ - "actix", + "actix 0.10.0", "actix-cors", "actix-files", - "actix-http", + "actix-http 2.2.0", "actix-multipart", - "actix-rt 2.2.0", - "actix-service", + "actix-rt 2.4.0", + "actix-service 1.0.6", "actix-utils 3.0.0", - "actix-web", - "actix-web-actors", + "actix-web 3.3.2", + "actix-web-actors 3.0.0", "anyhow", "base64 0.13.0", "bcrypt", @@ -2026,6 +2279,15 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +[[package]] +name = "jobserver" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.51" @@ -2075,6 +2337,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" +[[package]] +name = "language-tags" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" + [[package]] name = "lazy_static" version = "1.4.0" @@ -2108,9 +2376,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.94" +version = "0.2.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" +checksum = "fbe5e23404da5b4f555ef85ebed98fb4083e55a00c317800bc2a50ede9f3d219" [[package]] name = "libm" @@ -2145,6 +2413,18 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "local-channel" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6246c68cf195087205a0512559c97e15eaf95198bf0e206d662092cdcb03fe9f" +dependencies = [ + "futures-core", + "futures-sink", + "futures-util", + "local-waker", +] + [[package]] name = "local-waker" version = "0.1.1" @@ -2522,23 +2802,21 @@ dependencies = [ name = "obs-gateway-server" version = "0.0.1" dependencies = [ - "actix", + "actix 0.12.0", "actix-cors", "actix-files", - "actix-http", - "actix-multipart", - "actix-rt 2.2.0", - "actix-service", + "actix-http 3.0.0-beta.11", + "actix-rt 2.4.0", + "actix-service 2.0.1", "actix-utils 3.0.0", - "actix-web", - "actix-web-actors", + "actix-web 4.0.0-beta.10", + "actix-web-actors 4.0.0-beta.7", "anyhow", "base64 0.13.0", "bcrypt", "chrono", "config", "container_common", - "data-encoding", "derive_more", "diesel", "dotenv", @@ -2548,9 +2826,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", - "http_server_common", "hyper 0.13.10", - "infer", "jwt", "lazy_static", "lfu", @@ -2565,15 +2841,11 @@ dependencies = [ "rand 0.8.4", "regex", "reqwest 0.11.6", - "ring", - "rust-s3", "serde", "serde_derive", "serde_json", "sha2", "sqlx", - "subprocess", - "tempdir", "time 0.2.26", "tokio 1.6.0", "toml", @@ -2694,6 +2966,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "paste" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" + [[package]] name = "pem" version = "0.8.3" @@ -3701,16 +3979,16 @@ checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" name = "storyteller-web" version = "0.0.1" dependencies = [ - "actix", + "actix 0.10.0", "actix-cors", "actix-files", - "actix-http", + "actix-http 2.2.0", "actix-multipart", - "actix-rt 2.2.0", - "actix-service", + "actix-rt 2.4.0", + "actix-service 1.0.6", "actix-utils 3.0.0", - "actix-web", - "actix-web-actors", + "actix-web 3.3.2", + "actix-web-actors 3.0.0", "anyhow", "base64 0.13.0", "bcrypt", @@ -3911,6 +4189,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "time" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99beeb0daeac2bd1e86ac2c21caddecb244b39a093594da1a661ec2060c7aedd" +dependencies = [ + "itoa", + "libc", +] + [[package]] name = "time-macros" version = "0.1.1" @@ -4740,3 +5028,32 @@ dependencies = [ "syn", "synstructure", ] + +[[package]] +name = "zstd" +version = "0.7.0+zstd.1.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9428752481d8372e15b1bf779ea518a179ad6c771cca2d2c60e4fbff3cc2cd52" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "3.1.0+zstd.1.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa1926623ad7fe406e090555387daf73db555b948134b4d73eac5eb08fb666d" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "1.5.0+zstd.1.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6c094340240369025fc6b731b054ee2a834328fa584310ac96aa4baebdc465" +dependencies = [ + "cc", + "libc", +] diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index 3cdc80d250..c2912f888a 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main.rs" # Internal config = { path = "../config" } container_common = { path = "../container_common" } -http_server_common = { path = "../http_server_common" } +#http_server_common = { path = "../http_server_common" } # External anyhow = "1.0.40" @@ -47,7 +47,8 @@ uuid = { version = "0.8.2", features = ["v4"] } # https://github.com/durch/rust-s3/issues/198 #tokio = { version = "1.6.0", features = ["macros"] } #tokio = { version = "0.2.25", features = ["macros"] } -# NB(3,2021-11-07): Actually, maybe we can do this in a complex multi-project monorepo +# NB(3,2021-11-07): Actually, maybe we can specify a version that diverges from +# the rest of the monorepo. The Twitch client needs a 1.x runtime. tokio = { version = "1.6.0", features = ["macros", "rt-multi-thread"] } # MySql Database @@ -76,32 +77,16 @@ native-tls = "0.2.8" # In-memory caching lru_time_cache = "0.11.11" -# Files -data-encoding = "2.3.2" -ring = "0.17.0-alpha.10" -# NB: This is the source of all of our problems. `rust-s3` can't upload to object names with directories -# in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2 -#rust-s3 = "0.27.0-rc4" -#rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } -rust-s3 = { version = "0.26" } -infer = "0.5.0" - -subprocess = "0.2.6" -tempdir = "0.3.7" - # Web -actix = "0.10" -#actix-web = "3" -actix-web-actors = "3" - +actix = "0.12.0" actix-cors = "0.5.4" actix-files= "0.5" -actix-http = "2.2.0" -actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade -actix-rt = "2.2.0" -actix-service = "1.0.6" +actix-http = "3.0.0-beta.11" +actix-rt = "2.4.0" +actix-service = "2.0.1" actix-utils = "3.0.0" -actix-web = "3.3.2" +actix-web = "4.0.0-beta.10" +actix-web-actors = "4.0.0-beta.7" jwt = "0.13.0" # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index b802557a75..547d04ad87 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -26,9 +26,9 @@ use crate::pubsub_gateway::ws_index; use crate::twitch::twitch_secrets::TwitchSecrets; use futures::Future; use futures::executor::ThreadPool; -use http_server_common::cors::build_common_cors_config; -use http_server_common::endpoints::default_route_404::default_route_404; -use http_server_common::endpoints::root_index::get_root_index; +//use http_server_common::cors::build_common_cors_config; +//use http_server_common::endpoints::default_route_404::default_route_404; +//use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; use log::{info}; use r2d2_redis::RedisConnectionManager; @@ -66,8 +66,8 @@ pub struct EnvConfig { pub website_homepage_redirect: String, } -//#[actix_web::main] -#[tokio::main] +#[actix_web::main] +//#[tokio::main] async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); @@ -93,6 +93,8 @@ async fn main() -> AnyhowResult<()> { // ======================================================== +// TODO: Temp comment out to debug actix-web vs. runtime +// info!("Getting app access token..."); //let scopes = Scope::all(); @@ -130,33 +132,29 @@ async fn main() -> AnyhowResult<()> { println!("Go to this page: {}", url); - - - // ======================================================== - - info!("Connecting to database..."); - - let db_connection_string = - easyenv::get_env_string_or_default( - "MYSQL_URL", - DEFAULT_MYSQL_CONNECTION_STRING); - - let redis_connection_string = - easyenv::get_env_string_or_default( - "REDIS_URL", - DEFAULT_REDIS_CONNECTION_STRING); - - let pool = MySqlPoolOptions::new() - .max_connections(5) - .connect(&db_connection_string) - .await?; - - let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?; - - let redis_pool = r2d2::Pool::builder() - .build(redis_manager)?; +// info!("Connecting to database..."); +// +// let db_connection_string = +// easyenv::get_env_string_or_default( +// "MYSQL_URL", +// DEFAULT_MYSQL_CONNECTION_STRING); +// +// let redis_connection_string = +// easyenv::get_env_string_or_default( +// "REDIS_URL", +// DEFAULT_REDIS_CONNECTION_STRING); +// +// let pool = MySqlPoolOptions::new() +// .max_connections(5) +// .connect(&db_connection_string) +// .await?; +// +// let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?; +// +// let redis_pool = r2d2::Pool::builder() +// .build(redis_manager)?; info!("Reading env vars and setting up utils..."); @@ -201,7 +199,7 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> HttpServer::new(move || { App::new() .app_data(server_state_arc.clone()) - .wrap(build_common_cors_config()) + //.wrap(build_common_cors_config()) .wrap(Logger::new(&log_format) .exclude("/liveness") .exclude("/readiness")) @@ -214,8 +212,8 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> .route(web::get().to(ws_index)) .route(web::head().to(|| HttpResponse::Ok())) ) - .service(get_root_index) - .default_service( web::route().to(default_route_404)) + //.service(get_root_index) + //.default_service( web::route().to(default_route_404)) }) .bind(bind_address)? .workers(num_workers) From 5ba21a9771f3c2d27e05ec4b45129ba67b14742c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 8 Nov 2021 01:52:38 -0500 Subject: [PATCH 0648/1042] more of the oauth flow, fixing tokio versions --- Cargo.lock | 172 +++++++++-- app/obs_gateway/Cargo.toml | 11 +- app/obs_gateway/src/main.rs | 115 +++++++- app/obs_gateway/src/twitch/mod.rs | 3 +- .../src/twitch/websocket_client.rs | 273 ++++++++++++++++++ app/obs_gateway/src/util/mod.rs | 1 + app/obs_gateway/src/util/random_nonce.rs | 24 ++ 7 files changed, 576 insertions(+), 23 deletions(-) create mode 100644 app/obs_gateway/src/twitch/websocket_client.rs create mode 100644 app/obs_gateway/src/util/mod.rs create mode 100644 app/obs_gateway/src/util/random_nonce.rs diff --git a/Cargo.lock b/Cargo.lock index b6ab1a3613..b6bb41e755 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,7 +47,7 @@ dependencies = [ "parking_lot 0.11.1", "pin-project-lite 0.2.6", "smallvec 1.6.1", - "tokio 1.6.0", + "tokio 1.13.0", "tokio-util 0.6.9", ] @@ -80,7 +80,7 @@ dependencies = [ "log", "memchr", "pin-project-lite 0.2.6", - "tokio 1.6.0", + "tokio 1.13.0", "tokio-util 0.6.9", ] @@ -222,7 +222,7 @@ dependencies = [ "rand 0.8.4", "sha-1", "smallvec 1.6.1", - "tokio 1.6.0", + "tokio 1.13.0", "zstd", ] @@ -314,7 +314,7 @@ checksum = "4a0c218d0a17c120f10ee0c69c9f0c45d87319e8f66b1f065e8412b612fc3e24" dependencies = [ "actix-macros 0.2.3", "futures-core", - "tokio 1.6.0", + "tokio 1.13.0", ] [[package]] @@ -350,7 +350,7 @@ dependencies = [ "log", "mio 0.7.11", "num_cpus", - "tokio 1.6.0", + "tokio 1.13.0", ] [[package]] @@ -572,7 +572,7 @@ dependencies = [ "bytestring", "futures-core", "pin-project 1.0.7", - "tokio 1.6.0", + "tokio 1.13.0", ] [[package]] @@ -1920,7 +1920,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.6.0", + "tokio 1.13.0", "tokio-util 0.6.9", "tracing", ] @@ -2080,7 +2080,7 @@ dependencies = [ "r2d2_redis", "rand 0.8.4", "regex", - "ring", + "ring 0.17.0-alpha.10", "rust-s3", "serde", "serde_derive", @@ -2164,7 +2164,7 @@ dependencies = [ "itoa", "pin-project-lite 0.2.6", "socket2 0.4.0", - "tokio 1.6.0", + "tokio 1.13.0", "tower-service", "tracing", "want", @@ -2192,7 +2192,7 @@ dependencies = [ "bytes 1.0.1", "hyper 0.14.14", "native-tls", - "tokio 1.6.0", + "tokio 1.13.0", "tokio-native-tls 0.3.0", ] @@ -2823,6 +2823,7 @@ dependencies = [ "easyenv", "futures 0.1.31", "futures 0.3.17", + "futures-util", "hex", "hmac 0.10.1", "hostname", @@ -2841,13 +2842,15 @@ dependencies = [ "rand 0.8.4", "regex", "reqwest 0.11.6", + "rpassword", "serde", "serde_derive", "serde_json", "sha2", "sqlx", "time 0.2.26", - "tokio 1.6.0", + "tokio 1.13.0", + "tokio-tungstenite", "toml", "twitch_api2", "twitch_oauth2", @@ -3430,7 +3433,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "tokio 1.6.0", + "tokio 1.13.0", "tokio-native-tls 0.3.0", "url 2.2.2", "wasm-bindgen", @@ -3449,6 +3452,21 @@ dependencies = [ "quick-error", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "ring" version = "0.17.0-alpha.10" @@ -3464,6 +3482,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rpassword" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc936cf8a7ea60c58f030fd36a612a48f440610214dc54bc36431f9ea0c3efb" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "rsa" version = "0.3.0" @@ -3535,6 +3563,31 @@ dependencies = [ "semver 0.9.0", ] +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64 0.13.0", + "log", + "ring 0.16.20", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls", + "schannel", + "security-framework", +] + [[package]] name = "ryu" version = "1.0.5" @@ -3566,6 +3619,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring 0.16.20", + "untrusted", +] + [[package]] name = "security-framework" version = "2.2.0" @@ -4020,7 +4083,7 @@ dependencies = [ "r2d2_redis", "rand 0.8.4", "regex", - "ring", + "ring 0.17.0-alpha.10", "rust-s3", "serde", "serde_derive", @@ -4262,9 +4325,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.6.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37" +checksum = "588b2d10a336da58d877567cd8fb8a14b463e2104910f8132cd054b4b96e29ee" dependencies = [ "autocfg 1.0.1", "bytes 1.0.1", @@ -4351,7 +4414,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.6.0", + "tokio 1.13.0", ] [[package]] @@ -4373,6 +4436,17 @@ dependencies = [ "tokio-sync", ] +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio 1.13.0", + "webpki", +] + [[package]] name = "tokio-sync" version = "0.1.8" @@ -4407,6 +4481,23 @@ dependencies = [ "tokio 0.2.25", ] +[[package]] +name = "tokio-tungstenite" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "511de3f85caf1c98983545490c3d09685fa8eb634e57eec22bb4db271f46cbd8" +dependencies = [ + "futures-util", + "log", + "pin-project 1.0.7", + "rustls", + "tokio 1.13.0", + "tokio-rustls", + "tungstenite", + "webpki", + "webpki-roots", +] + [[package]] name = "tokio-uds" version = "0.2.7" @@ -4451,7 +4542,7 @@ dependencies = [ "futures-sink", "log", "pin-project-lite 0.2.6", - "tokio 1.6.0", + "tokio 1.13.0", ] [[package]] @@ -4545,6 +4636,28 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "tungstenite" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0b2d8558abd2e276b0a8df5c05a2ec762609344191e5fd23e292c910e9165b5" +dependencies = [ + "base64 0.13.0", + "byteorder", + "bytes 1.0.1", + "http", + "httparse", + "log", + "rand 0.8.4", + "rustls", + "rustls-native-certs", + "sha-1", + "thiserror", + "url 2.2.2", + "utf-8", + "webpki", +] + [[package]] name = "twitch_api2" version = "0.6.0-rc.3" @@ -4719,6 +4832,12 @@ dependencies = [ "percent-encoding 2.1.0", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "uuid" version = "0.8.2" @@ -4900,6 +5019,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring 0.16.20", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" +dependencies = [ + "webpki", +] + [[package]] name = "wepoll-sys" version = "3.0.1" diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index c2912f888a..103b64da24 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -49,7 +49,10 @@ uuid = { version = "0.8.2", features = ["v4"] } #tokio = { version = "0.2.25", features = ["macros"] } # NB(3,2021-11-07): Actually, maybe we can specify a version that diverges from # the rest of the monorepo. The Twitch client needs a 1.x runtime. -tokio = { version = "1.6.0", features = ["macros", "rt-multi-thread"] } +#tokio = { version = "1.6.0", features = ["macros", "rt-multi-thread"] } +# Async runtime, Futures, etc. +tokio = { version = "1.10.0", features = ["rt-multi-thread", "macros", "time"] } +futures-util = "0.3.17" # MySql Database # "offline" supports typesafe builds without a live database @@ -89,6 +92,9 @@ actix-web = "4.0.0-beta.10" actix-web-actors = "4.0.0-beta.7" jwt = "0.13.0" +# Websockets +tokio-tungstenite = { version = "0.15.0", features = ["rustls", "rustls-tls"] } + # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. hyper = { version = "0.13.10" } @@ -109,6 +115,9 @@ toml = "0.5.8" # Text processing pulldown-cmark = "0.8.0" +# Temporary (just to ask for oauth credentials during dev) +rpassword = "5.0.1" + [build-dependencies] # NB: Only using diesel for migrations support diesel = { version = "1.4.6", features = ["mysql"] } diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index 547d04ad87..357bad65c9 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -14,7 +14,11 @@ pub mod pubsub_gateway; pub mod twitch; +pub mod util; +//use http_server_common::cors::build_common_cors_config; +//use http_server_common::endpoints::default_route_404::default_route_404; +//use http_server_common::endpoints::root_index::get_root_index; use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; @@ -23,12 +27,11 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use crate::pubsub_gateway::ws_index; +use crate::twitch::twitch_client_wrapper::TwitchClientWrapper; use crate::twitch::twitch_secrets::TwitchSecrets; +use crate::twitch::websocket_client::PollingTwitchWebsocketClient; use futures::Future; use futures::executor::ThreadPool; -//use http_server_common::cors::build_common_cors_config; -//use http_server_common::endpoints::default_route_404::default_route_404; -//use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; use log::{info}; use r2d2_redis::RedisConnectionManager; @@ -38,9 +41,10 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; +use twitch_api2::pubsub::Topic; +use twitch_api2::pubsub; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{AppAccessToken, Scope, TwitchToken, tokens::errors::AppAccessTokenError, ClientId, ClientSecret}; -use crate::twitch::twitch_client_wrapper::TwitchClientWrapper; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -132,6 +136,109 @@ async fn main() -> AnyhowResult<()> { println!("Go to this page: {}", url); + + let input = rpassword::prompt_password_stdout( + "Paste in the resulting adress after authenticating (input hidden): ", + )?; + + let u = twitch_oauth2::url::Url::parse(&input)?; + + let map: std::collections::HashMap<_, _> = u.query_pairs().collect(); + + let user_token = match (map.get("state"), map.get("code")) { + (Some(state), Some(code)) => { + let token = builder + .get_user_token( + &reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build()?, + state, + code, + ) + .await?; + println!("Got token: {:?}", token); + + token + } + _ => match (map.get("error"), map.get("error_description")) { + (std::option::Option::Some(error), std::option::Option::Some(error_description)) => { + anyhow::bail!( + "twitch errored with error: {} - {}", + error, + error_description + ); + } + _ => anyhow::bail!("invalid url passed"), + }, + }; + + let auth_token = user_token.access_token.as_str(); + + // ==================== PUBSUB SUBSCRIPTION AND MAIN LOOP ==================== + + let mut client = PollingTwitchWebsocketClient::new()?; + + println!("Connecting PubSub..."); + client.connect().await?; + + println!("Connected"); + + //println!("Starting polling thread..."); + //client.start_ping_thread().await; + + println!("Sending PING..."); + + client.send_ping().await?; + + println!("Try read next..."); + let r = client.try_next().await?; + println!("Result: {:?}", r); + + + println!("Begin LISTEN..."); + + let bit_topic = pubsub::channel_bits::ChannelBitsEventsV2 { + channel_id: user_id, + }.into_topic(); + + let topics = [bit_topic]; + + client.listen(&auth_token, &topics).await?; + + println!("Try read next..."); + let r = client.try_next().await?; + println!("Result: {:?}", r); + + /* + Result: Some(Message { data: ChannelBitsEventsV2 { topic: ChannelBitsEventsV2 { + channel_id: 652567283 }, reply: BitsEvent { data: BitsEventData { badge_entitlement: + Some(BadgeEntitlement { new_version: 100, previous_version: 1 }), bits_used: 100, + channel_id: "652567283", channel_name: "vocodes", chat_message: "Cheer100 testing the cheer", + context: Cheer, is_anonymous: false, time: "2021-09-27T04:30:53.627717085Z", + total_bits_used: 101, user_id: "650154491", user_name: "testytest512" }, + message_id: "793f745e-8f3e-5f71-bc41-4808c4b49a53", version: "1.0", is_anonymous: false } } }) + + Result: Some(Message { data: ChannelBitsEventsV2 { topic: ChannelBitsEventsV2 { + channel_id: 652567283 }, reply: BitsEvent { data: BitsEventData { badge_entitlement: None, + bits_used: 1, channel_id: "652567283", channel_name: "vocodes", + chat_message: "mario: hello everybody Cheer1", context: Cheer, is_anonymous: false, + time: "2021-09-27T04:47:32.943872952Z", total_bits_used: 102, user_id: "650154491", + user_name: "testytest512" }, message_id: "5844328d-495b-5585-9d6c-3c99949f9a0a", version: "1.0", + is_anonymous: false } } }) + */ + + + + println!("Try read next..."); + let r = client.try_next().await?; + println!("Result: {:?}", r); + + println!("Sleep..."); + std::thread::sleep(Duration::from_millis(30000)); + + //let text = msg.into_text()?; + //println!("Text: {}", text); + // ======================================================== // info!("Connecting to database..."); diff --git a/app/obs_gateway/src/twitch/mod.rs b/app/obs_gateway/src/twitch/mod.rs index a021988480..a7c619f5db 100644 --- a/app/obs_gateway/src/twitch/mod.rs +++ b/app/obs_gateway/src/twitch/mod.rs @@ -1,2 +1,3 @@ pub mod twitch_client_wrapper; -pub mod twitch_secrets; \ No newline at end of file +pub mod twitch_secrets; +pub mod websocket_client; \ No newline at end of file diff --git a/app/obs_gateway/src/twitch/websocket_client.rs b/app/obs_gateway/src/twitch/websocket_client.rs new file mode 100644 index 0000000000..619b8b680c --- /dev/null +++ b/app/obs_gateway/src/twitch/websocket_client.rs @@ -0,0 +1,273 @@ +use anyhow::anyhow; +use anyhow::bail; +use container_common::anyhow_result::AnyhowResult; +use crate::util::random_nonce::random_nonce; +use futures_util::{SinkExt, StreamExt, TryStreamExt}; +use log::debug; +use reqwest::Url; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError, MutexGuard}; +use std::time::Duration; +use tokio::net::TcpStream; +use tokio; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::{connect_async, WebSocketStream, MaybeTlsStream}; +use twitch_api2::pubsub::{Response, TopicData, Topic, Topics}; +use twitch_api2::pubsub; + +// Reference javascript Twitch Websocket client: +// https://github.com/twitchdev/pubsub-javascript-sample/blob/main/main.js + +const WEBSOCKET_GATEWAY : &'static str = "wss://pubsub-edge.twitch.tv"; + +const PING_COMMAND : &'static str = "{ \"type\": \"PING\" }"; +const PING_SECONDS : u64 = 1 * 60; // Twitch calls for every 5 minutes, we'll do every 1 minute. + +pub struct PollingTwitchWebsocketClient { + client: Arc>, + is_polling: AtomicBool, +} + +impl PollingTwitchWebsocketClient { + pub fn new() -> AnyhowResult { + Ok(Self { + client: Arc::new(Mutex::new(TwitchWebsocketClient::new()?)), + is_polling: AtomicBool::new(false), + }) + } + + pub async fn connect(&mut self) -> AnyhowResult<()> { + match self.client.lock() { + Err(e) => Err(anyhow!("unlock client err: {:?}", e)), + Ok(mut client) => { + client.connect().await + } + } + } + + pub async fn listen<'t>(&mut self, auth_token: &str, topics: &'t [Topics]) -> AnyhowResult<()> { + match self.client.lock() { + Err(e) => Err(anyhow!("unlock client err: {:?}", e)), + Ok(mut client) => { + client.listen(auth_token, topics).await + } + } + } + + pub async fn send_ping(&mut self) -> AnyhowResult<()> { + match self.client.lock() { + Err(e) => Err(anyhow!("unlock client err: {:?}", e)), + Ok(mut client) => { + client.send_ping().await + } + } + } + + /// Start a pinging background thread. + /// Per Twitch PubSub docs, + /// To keep the server from closing the connection, clients must send a PING command + /// at least once every 5 minutes. If a client does not receive a PONG message within + /// 10 seconds of issuing a PING command, it should reconnect to the server. + pub async fn start_ping_thread(&self) { + let client = self.client.clone(); + tokio::spawn(async move { + loop { + match client.lock() { + Ok(mut c) => { + // TODO: Future isn't Send + //let _r = c.send_ping().await; + // TODO: Need to read from all call sites due to ordering. + //c.read_response(); + } + Err(_) => { + // TODO + } + } + tokio::time::sleep(Duration::from_secs(PING_SECONDS)).await; + } + }); + } + + pub async fn next(&mut self) -> AnyhowResult { + match self.client.lock() { + Err(e) => Err(anyhow!("unlock client err: {:?}", e)), + Ok(mut client) => { + client.next().await + } + } + } + + pub async fn try_next(&mut self) -> AnyhowResult> { + match self.client.lock() { + Err(e) => Err(anyhow!("unlock client err: {:?}", e)), + Ok(mut client) => { + client.try_next().await + } + } + } + + /*async fn start_polling(&self) { + if self.is_polling.load(Ordering::Relaxed) { + return; + } + + tokio::spawn(async move { + self.is_polling.store(true, Ordering::Relaxed); + + let command = pubsub::channel_sub_gifts( + &topics, + "authtoken", + "random nonce string", + )?; + + }); + / + }*/ + + + /*let message = Message::Text(PING_COMMAND.to_string()); + socket.send(message).await?; + match self.read_response().await { + Err(e) => println!("Error getting PONG: {:?}", e), + Ok(typed_response) => { + match typed_response { + Response::Pong => { println!("Pong") } + Response::Reconnect => { println!("Reconnect") } + Response::Response(_0) => { println!("Sub/Unsub (pubsub?)") } + Response::Message { data } => { + match data { + TopicData::ChannelCheerEventsPublicV1 { topic, reply } => {} + TopicData::ChannelSubGiftsV1 { topic, reply } => {} + TopicData::ChannelSubscribeEventsV1 { topic, reply } => {} + TopicData::CommunityPointsChannelV1 { topic, reply } => {} + TopicData::Following { topic, reply} => {} + // Not interested in these events + TopicData::AutoModQueue { .. } => {} + TopicData::ChannelBitsBadgeUnlocks { .. } => {} + TopicData::ChannelBitsEventsV2 { .. } => {} + TopicData::ChannelPointsChannelV1 { .. } => {} + TopicData::ChatModeratorActions { .. } => {} + TopicData::HypeTrainEventsV1 { .. } => {} + TopicData::HypeTrainEventsV1Rewards { .. } => {} + TopicData::Raid { .. } => {} + TopicData::UserModerationNotifications { .. } => {} + TopicData::VideoPlayback { .. } => {} + TopicData::VideoPlaybackById { .. } => {} + } + } + } + } + }*/ + +} + +pub struct TwitchWebsocketClient { + socket: Option>>, +} + +impl TwitchWebsocketClient { + pub fn new() -> AnyhowResult { + Ok(Self { + socket: None, + }) + } + + pub async fn connect(&mut self) -> AnyhowResult<()> { + let url = Url::parse(WEBSOCKET_GATEWAY)?; + let (mut socket, _response) = + connect_async(url).await?; + self.socket = Some(socket); + Ok(()) + } + + pub async fn close(&mut self) -> AnyhowResult<()> { + let mut socket = match self.socket { + None => return Ok(()), // Already closed. + Some(ref mut socket) => socket, + }; + + socket.close(None).await?; + self.socket = None; + + Ok(()) + } + + pub async fn listen<'t>(&mut self, auth_token: &str, topics: &'t [Topics]) -> AnyhowResult<()> { + let mut socket = match self.socket { + None => return Err(anyhow!("not connected")), + Some(ref mut socket) => socket, + }; + + let nonce = random_nonce(128); + + let command = pubsub::listen_command( + topics, + auth_token, + nonce.as_ref(), + )?; + + debug!("Sending LISTEN"); + let message = Message::Text(command); + socket.send(message).await?; + + Ok(()) + } + + pub fn is_connected(&self) -> bool { + // TODO: This probably isn't accurate + self.socket.is_some() + } + + pub async fn send_ping(&mut self) -> AnyhowResult<()> { + let mut socket = match self.socket { + None => return Err(anyhow!("not connected")), + Some(ref mut socket) => socket, + }; + + debug!("Sending PING"); + let message = Message::Text(PING_COMMAND.to_string()); + socket.send(message).await?; + + // TODO: Need to read from all call sites due to ordering. + //match self.read_response().await { + // Err(e) => Err(e), + // Ok(Response::Pong) => Ok(()), + // _ => Err(bail!("wrong response type")), + //} + Ok(()) + } + + pub async fn next(&mut self) -> AnyhowResult { + let socket = match self.socket { + None => return Err(anyhow!("not connected")), + Some(ref mut s) => s, + }; + + match socket.next().await { + None => Err(anyhow!("nothing to read")), + Some(Err(e)) => Err(anyhow!("websocket error: {:?}", e)), + Some(Ok(response)) => { + let text = response.to_text()?; + let typed_response = Response::parse(text)?; + Ok(typed_response) + }, + } + } + + pub async fn try_next(&mut self) -> AnyhowResult> { + let socket = match self.socket { + None => return Err(anyhow!("not connected")), + Some(ref mut s) => s, + }; + + match socket.try_next().await { + Err(e) => Err(anyhow!("websocket error: {:?}", e)), + Ok(None) => Ok(None), + Ok(Some(response)) => { + let text = response.to_text()?; + let typed_response = Response::parse(text)?; + Ok(Some(typed_response)) + }, + } + } +} diff --git a/app/obs_gateway/src/util/mod.rs b/app/obs_gateway/src/util/mod.rs new file mode 100644 index 0000000000..b0c7022656 --- /dev/null +++ b/app/obs_gateway/src/util/mod.rs @@ -0,0 +1 @@ +pub mod random_nonce; \ No newline at end of file diff --git a/app/obs_gateway/src/util/random_nonce.rs b/app/obs_gateway/src/util/random_nonce.rs new file mode 100644 index 0000000000..5f9034d3aa --- /dev/null +++ b/app/obs_gateway/src/util/random_nonce.rs @@ -0,0 +1,24 @@ +use rand::distributions::Alphanumeric; +use rand::{thread_rng, Rng}; + +/// Returns a random nonce for Twitch +pub fn random_nonce(length: usize) -> String { + thread_rng() + .sample_iter(&Alphanumeric) + .take(length) + .map(char::from) + .collect() +} + +#[cfg(test)] +mod tests { + use crate::util::random_nonce::random_nonce; + + #[test] + fn random_nonce_length() { + assert_eq(1, random_nonce(1).len()); + assert_eq(10, random_nonce(10).len()); + assert_eq(15, random_nonce(15).len()); + assert_eq(64, random_nonce(64).len()); + } +} \ No newline at end of file From 5507e3bd6a82419a74877dc42da7b23e4cf5e434 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 8 Nov 2021 04:48:36 -0500 Subject: [PATCH 0649/1042] setup for new oauth endpoints (disabled temporarily) --- app/http_server_common/src/lib.rs | 3 +- app/http_server_common/src/response/mod.rs | 3 ++ .../src/response/response_error_helpers.rs | 30 +++++++++++++ .../src/response/response_success_helpers.rs | 19 ++++++++ .../src/response/serialize_as_json_error.rs | 21 +++++++++ app/obs_gateway/src/endpoints/mod.rs | 2 + app/obs_gateway/src/endpoints/oauth_begin.rs | 44 +++++++++++++++++++ app/obs_gateway/src/endpoints/oauth_end.rs | 0 app/obs_gateway/src/main.rs | 15 ++++++- 9 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 app/http_server_common/src/response/mod.rs create mode 100644 app/http_server_common/src/response/response_error_helpers.rs create mode 100644 app/http_server_common/src/response/response_success_helpers.rs create mode 100644 app/http_server_common/src/response/serialize_as_json_error.rs create mode 100644 app/obs_gateway/src/endpoints/mod.rs create mode 100644 app/obs_gateway/src/endpoints/oauth_begin.rs create mode 100644 app/obs_gateway/src/endpoints/oauth_end.rs diff --git a/app/http_server_common/src/lib.rs b/app/http_server_common/src/lib.rs index 5f1e14a349..61a42e6bcb 100644 --- a/app/http_server_common/src/lib.rs +++ b/app/http_server_common/src/lib.rs @@ -2,5 +2,6 @@ //! Common HTTP Server components //! -pub mod endpoints; pub mod cors; +pub mod endpoints; +pub mod response; \ No newline at end of file diff --git a/app/http_server_common/src/response/mod.rs b/app/http_server_common/src/response/mod.rs new file mode 100644 index 0000000000..31ca5b7882 --- /dev/null +++ b/app/http_server_common/src/response/mod.rs @@ -0,0 +1,3 @@ +pub mod response_error_helpers; +pub mod response_success_helpers; +pub mod serialize_as_json_error; diff --git a/app/http_server_common/src/response/response_error_helpers.rs b/app/http_server_common/src/response/response_error_helpers.rs new file mode 100644 index 0000000000..2977eb183d --- /dev/null +++ b/app/http_server_common/src/response/response_error_helpers.rs @@ -0,0 +1,30 @@ +use actix_http::http::header; +use actix_web::dev::HttpResponseBuilder; +use actix_web::http::StatusCode; +use actix_web::{HttpResponse, ResponseError}; + +#[derive(Serialize)] +pub struct SimpleGenericJsonError { + pub success: bool, + pub error_reason: String, +} + +pub fn to_simple_json_error( + error_reason: &str, + status_code: StatusCode, +) -> HttpResponse { + + let response = SimpleGenericJsonError { + success: false, + error_reason : error_reason.to_string(), + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(status_code) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) +} \ No newline at end of file diff --git a/app/http_server_common/src/response/response_success_helpers.rs b/app/http_server_common/src/response/response_success_helpers.rs new file mode 100644 index 0000000000..dae959059f --- /dev/null +++ b/app/http_server_common/src/response/response_success_helpers.rs @@ -0,0 +1,19 @@ +use actix_web::HttpResponse; + +#[derive(Serialize)] +pub struct SimpleGenericJsonSuccess { + pub success: bool, +} + +pub fn simple_json_success() -> HttpResponse { + let response = SimpleGenericJsonSuccess { + success: true, + }; + + let body = serde_json::to_string(&response) + .unwrap_or("{ \"success\": true }".to_string()); + + HttpResponse::Ok() + .content_type("application/json") + .body(body) +} diff --git a/app/http_server_common/src/response/serialize_as_json_error.rs b/app/http_server_common/src/response/serialize_as_json_error.rs new file mode 100644 index 0000000000..7a36a6d595 --- /dev/null +++ b/app/http_server_common/src/response/serialize_as_json_error.rs @@ -0,0 +1,21 @@ +use actix_http::http::{StatusCode, header}; +use actix_web::HttpResponse; +use serde::Serialize; +use actix_http::ResponseError; +use actix_web::dev::HttpResponseBuilder; + +/// Turn error responses into JSON HTTP responses +pub fn serialize_as_json_error( + error_payload: &T, +) -> HttpResponse + where T: ?Sized + Serialize + ResponseError, +{ + let body = match serde_json::to_string(error_payload) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(error_payload.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) +} diff --git a/app/obs_gateway/src/endpoints/mod.rs b/app/obs_gateway/src/endpoints/mod.rs new file mode 100644 index 0000000000..436ec98e87 --- /dev/null +++ b/app/obs_gateway/src/endpoints/mod.rs @@ -0,0 +1,2 @@ +pub mod oauth_begin; +pub mod oauth_end; \ No newline at end of file diff --git a/app/obs_gateway/src/endpoints/oauth_begin.rs b/app/obs_gateway/src/endpoints/oauth_begin.rs new file mode 100644 index 0000000000..9f45f59a1a --- /dev/null +++ b/app/obs_gateway/src/endpoints/oauth_begin.rs @@ -0,0 +1,44 @@ +use actix_http::StatusCode; +use actix_web::{HttpRequest, web, HttpResponse, ResponseError}; +use crate::ObsGatewayServerState; +use log::info; +use std::fmt; +use std::sync::Arc; + +#[derive(Debug, Display)] +pub enum OauthBeginEnrollError { + ServerError, +} + +impl ResponseError for OauthBeginEnrollError{ + fn status_code(&self) -> StatusCode { + match *self { + Self::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + Self::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for OauthBeginEnrollError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.error_type) + } +} + +pub async fn oauth_begin_enroll( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + + info!("oauth enrollment begin"); + Ok(()) +} diff --git a/app/obs_gateway/src/endpoints/oauth_end.rs b/app/obs_gateway/src/endpoints/oauth_end.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index 357bad65c9..bbf36156bb 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -12,6 +12,7 @@ #[macro_use] extern crate magic_crypt; #[macro_use] extern crate serde_derive; +//pub mod endpoints; pub mod pubsub_gateway; pub mod twitch; pub mod util; @@ -45,6 +46,7 @@ use twitch_api2::pubsub::Topic; use twitch_api2::pubsub; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{AppAccessToken, Scope, TwitchToken, tokens::errors::AppAccessTokenError, ClientId, ClientSecret}; +//use crate::endpoints::oauth_begin::oauth_begin_enroll; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -319,7 +321,18 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> .route(web::get().to(ws_index)) .route(web::head().to(|| HttpResponse::Ok())) ) - //.service(get_root_index) + // /twitch_oauth_enroll + // /twitch_oauth_redirect +// .service( +// web::resource("/twitch_oauth_enroll") +// .route(web::get().to(oauth_begin_enroll)) +// .route(web::head().to(|| HttpResponse::Ok())) +// ) +// .service( +// web::resource("/twitch_oauth_redirect") +// .route(web::get().to(oauth_begin_enroll)) +// .route(web::head().to(|| HttpResponse::Ok())) +// ) //.default_service( web::route().to(default_route_404)) }) .bind(bind_address)? From 7256630d87cf6f29e7e464565cc172dd4e31da0f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 8 Nov 2021 05:02:42 -0500 Subject: [PATCH 0650/1042] http common package versions that work with both apps --- Cargo.lock | 9 +---- app/http_server_common/Cargo.toml | 34 ++++++++++++------- app/http_server_common/src/lib.rs | 2 ++ .../src/response/serialize_as_json_error.rs | 4 +-- app/obs_gateway/Cargo.toml | 2 +- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b6bb41e755..f3a7fc09bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2044,16 +2044,9 @@ dependencies = [ name = "http_server_common" version = "0.0.1" dependencies = [ - "actix 0.10.0", "actix-cors", - "actix-files", "actix-http 2.2.0", - "actix-multipart", - "actix-rt 2.4.0", - "actix-service 1.0.6", - "actix-utils 3.0.0", "actix-web 3.3.2", - "actix-web-actors 3.0.0", "anyhow", "base64 0.13.0", "bcrypt", @@ -2070,7 +2063,6 @@ dependencies = [ "hostname", "hyper 0.13.10", "infer", - "jwt", "limitation", "log", "lru_time_cache", @@ -2827,6 +2819,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", + "http_server_common", "hyper 0.13.10", "jwt", "lazy_static", diff --git a/app/http_server_common/Cargo.toml b/app/http_server_common/Cargo.toml index 88c14d9ca3..400c6497de 100644 --- a/app/http_server_common/Cargo.toml +++ b/app/http_server_common/Cargo.toml @@ -69,19 +69,25 @@ subprocess = "0.2.6" tempdir = "0.3.7" # Web -actix = "0.10" -#actix-web = "3" -actix-web-actors = "3" - +#actix = "0.12.0" actix-cors = "0.5.4" -actix-files= "0.5" -actix-http = "2.2.0" -actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade -actix-rt = "2.2.0" -actix-service = "1.0.6" -actix-utils = "3.0.0" -actix-web = "3.3.2" -jwt = "0.13.0" +#actix-files= "0.5" +actix-http = "*" +#actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade +#actix-rt = "2.2.0" +#actix-service = "1.0.6" +#actix-utils = "3.0.0" +actix-web = "*" +#actix-web-actors = "3" +#jwt = "0.13.0" + + +#actix-http-old = { package = "actix-http", version = "2.2.0", optional = true } +#actix-web-old = { package = "actix-web", version = "3.3.2", optional = true } +#actix-http-new = { package = "actix-http", version = "3.0.0-beta.11", optional = true } +#actix-web-new = { package = "actix-web", version = "4.0.0-beta.10", optional = true } + + # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. @@ -106,3 +112,7 @@ pulldown-cmark = "0.8.0" [build-dependencies] # NB: Only using diesel for migrations support diesel = { version = "1.4.6", features = ["mysql"] } + +#[features] +#actix-old = ["actix-http-old", "actix-web-old"] +#actix-new = ["actix-http-new", "actix-web-new"] diff --git a/app/http_server_common/src/lib.rs b/app/http_server_common/src/lib.rs index 61a42e6bcb..91c231f521 100644 --- a/app/http_server_common/src/lib.rs +++ b/app/http_server_common/src/lib.rs @@ -2,6 +2,8 @@ //! Common HTTP Server components //! +#[macro_use] extern crate serde_derive; + pub mod cors; pub mod endpoints; pub mod response; \ No newline at end of file diff --git a/app/http_server_common/src/response/serialize_as_json_error.rs b/app/http_server_common/src/response/serialize_as_json_error.rs index 7a36a6d595..beb6d25a25 100644 --- a/app/http_server_common/src/response/serialize_as_json_error.rs +++ b/app/http_server_common/src/response/serialize_as_json_error.rs @@ -1,8 +1,8 @@ +use actix_http::ResponseError; use actix_http::http::{StatusCode, header}; use actix_web::HttpResponse; -use serde::Serialize; -use actix_http::ResponseError; use actix_web::dev::HttpResponseBuilder; +use serde::Serialize; /// Turn error responses into JSON HTTP responses pub fn serialize_as_json_error( diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index 103b64da24..3a9500509c 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main.rs" # Internal config = { path = "../config" } container_common = { path = "../container_common" } -#http_server_common = { path = "../http_server_common" } +http_server_common = { path = "../http_server_common" } # External anyhow = "1.0.40" From 8a00f37e9e0ecf2c8c6553658f7ca3249236df32 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 9 Nov 2021 21:13:44 -0500 Subject: [PATCH 0651/1042] wip using common HTTP utils --- app/http_server_common/Cargo.toml | 23 ++------------- app/http_server_common/src/lib.rs | 2 +- app/obs_gateway/src/endpoints/oauth_begin.rs | 24 +++++++++++----- app/obs_gateway/src/main.rs | 30 ++++++++++---------- 4 files changed, 36 insertions(+), 43 deletions(-) diff --git a/app/http_server_common/Cargo.toml b/app/http_server_common/Cargo.toml index 400c6497de..cf22ae1e20 100644 --- a/app/http_server_common/Cargo.toml +++ b/app/http_server_common/Cargo.toml @@ -69,25 +69,12 @@ subprocess = "0.2.6" tempdir = "0.3.7" # Web -#actix = "0.12.0" actix-cors = "0.5.4" -#actix-files= "0.5" + +# NB: Versions aren't pinned since some apps use ver 3 vs ver 4. +# This is because of the Tokio runtime requirements. actix-http = "*" -#actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade -#actix-rt = "2.2.0" -#actix-service = "1.0.6" -#actix-utils = "3.0.0" actix-web = "*" -#actix-web-actors = "3" -#jwt = "0.13.0" - - -#actix-http-old = { package = "actix-http", version = "2.2.0", optional = true } -#actix-web-old = { package = "actix-web", version = "3.3.2", optional = true } -#actix-http-new = { package = "actix-http", version = "3.0.0-beta.11", optional = true } -#actix-web-new = { package = "actix-web", version = "4.0.0-beta.10", optional = true } - - # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. @@ -112,7 +99,3 @@ pulldown-cmark = "0.8.0" [build-dependencies] # NB: Only using diesel for migrations support diesel = { version = "1.4.6", features = ["mysql"] } - -#[features] -#actix-old = ["actix-http-old", "actix-web-old"] -#actix-new = ["actix-http-new", "actix-web-new"] diff --git a/app/http_server_common/src/lib.rs b/app/http_server_common/src/lib.rs index 91c231f521..ebb920ee22 100644 --- a/app/http_server_common/src/lib.rs +++ b/app/http_server_common/src/lib.rs @@ -6,4 +6,4 @@ pub mod cors; pub mod endpoints; -pub mod response; \ No newline at end of file +pub mod response; diff --git a/app/obs_gateway/src/endpoints/oauth_begin.rs b/app/obs_gateway/src/endpoints/oauth_begin.rs index 9f45f59a1a..d0eb413cd6 100644 --- a/app/obs_gateway/src/endpoints/oauth_begin.rs +++ b/app/obs_gateway/src/endpoints/oauth_begin.rs @@ -1,16 +1,18 @@ -use actix_http::StatusCode; -use actix_web::{HttpRequest, web, HttpResponse, ResponseError}; +use actix_http::{StatusCode, header}; +use actix_web::{HttpRequest, web, HttpResponse, ResponseError, HttpResponseBuilder}; use crate::ObsGatewayServerState; +use http_server_common::response::response_error_helpers::to_simple_json_error; use log::info; use std::fmt; use std::sync::Arc; +use http_server_common::response::response_success_helpers::simple_json_success; -#[derive(Debug, Display)] +#[derive(Debug)] pub enum OauthBeginEnrollError { ServerError, } -impl ResponseError for OauthBeginEnrollError{ +impl ResponseError for OauthBeginEnrollError { fn status_code(&self) -> StatusCode { match *self { Self::ServerError => StatusCode::INTERNAL_SERVER_ERROR, @@ -22,14 +24,18 @@ impl ResponseError for OauthBeginEnrollError{ Self::ServerError => "server error".to_string(), }; - to_simple_json_error(&error_reason, self.status_code()) + //to_simple_json_error(&error_reason, self.status_code()) + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body("TODO") } } // NB: Not using DeriveMore since Clion doesn't understand it. impl fmt::Display for OauthBeginEnrollError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.error_type) + write!(f, "{:?}", self) } } @@ -40,5 +46,9 @@ pub async fn oauth_begin_enroll( info!("oauth enrollment begin"); - Ok(()) + //Ok(simple_json_success()) + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body("")) } diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index bbf36156bb..ca9e038d9a 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -12,14 +12,11 @@ #[macro_use] extern crate magic_crypt; #[macro_use] extern crate serde_derive; -//pub mod endpoints; +pub mod endpoints; pub mod pubsub_gateway; pub mod twitch; pub mod util; -//use http_server_common::cors::build_common_cors_config; -//use http_server_common::endpoints::default_route_404::default_route_404; -//use http_server_common::endpoints::root_index::get_root_index; use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; @@ -27,12 +24,16 @@ use actix_web::{HttpServer, web, HttpResponse, App}; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; +use crate::endpoints::oauth_begin::oauth_begin_enroll; use crate::pubsub_gateway::ws_index; use crate::twitch::twitch_client_wrapper::TwitchClientWrapper; use crate::twitch::twitch_secrets::TwitchSecrets; use crate::twitch::websocket_client::PollingTwitchWebsocketClient; use futures::Future; use futures::executor::ThreadPool; +use http_server_common::cors::build_common_cors_config; +use http_server_common::endpoints::default_route_404::default_route_404; +use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; use log::{info}; use r2d2_redis::RedisConnectionManager; @@ -46,7 +47,6 @@ use twitch_api2::pubsub::Topic; use twitch_api2::pubsub; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{AppAccessToken, Scope, TwitchToken, tokens::errors::AppAccessTokenError, ClientId, ClientSecret}; -//use crate::endpoints::oauth_begin::oauth_begin_enroll; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; @@ -323,16 +323,16 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> ) // /twitch_oauth_enroll // /twitch_oauth_redirect -// .service( -// web::resource("/twitch_oauth_enroll") -// .route(web::get().to(oauth_begin_enroll)) -// .route(web::head().to(|| HttpResponse::Ok())) -// ) -// .service( -// web::resource("/twitch_oauth_redirect") -// .route(web::get().to(oauth_begin_enroll)) -// .route(web::head().to(|| HttpResponse::Ok())) -// ) + .service( + web::resource("/twitch_oauth_enroll") + .route(web::get().to(oauth_begin_enroll)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/twitch_oauth_redirect") + .route(web::get().to(oauth_begin_enroll)) + .route(web::head().to(|| HttpResponse::Ok())) + ) //.default_service( web::route().to(default_route_404)) }) .bind(bind_address)? From d209e7fd79746381f72da741a935e20e66bd053a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 9 Nov 2021 22:16:18 -0500 Subject: [PATCH 0652/1042] HACK: Multiple versions of actix in support package with features --- Cargo.lock | 32 +------- app/http_server_common/Cargo.toml | 79 +++---------------- app/http_server_common/src/lib.rs | 20 ++++- .../src/response/response_error_helpers.rs | 7 +- .../src/response/serialize_as_json_error.rs | 14 +++- app/obs_gateway/Cargo.toml | 2 +- app/obs_gateway/src/endpoints/oauth_begin.rs | 6 +- app/obs_gateway/src/main.rs | 2 +- app/storyteller_web/src/main.rs | 4 +- 9 files changed, 57 insertions(+), 109 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f3a7fc09bb..2363b85ce8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2044,47 +2044,19 @@ dependencies = [ name = "http_server_common" version = "0.0.1" dependencies = [ - "actix-cors", "actix-http 2.2.0", + "actix-http 3.0.0-beta.11", "actix-web 3.3.2", + "actix-web 4.0.0-beta.10", "anyhow", "base64 0.13.0", - "bcrypt", "chrono", - "data-encoding", - "derive_more", - "diesel", "dotenv", - "easyenv", - "futures 0.1.31", - "futures 0.3.17", - "hex", - "hmac 0.10.1", - "hostname", - "hyper 0.13.10", - "infer", - "limitation", "log", - "lru_time_cache", - "magic-crypt", - "md-5", - "pulldown-cmark", - "r2d2_redis", - "rand 0.8.4", - "regex", - "ring 0.17.0-alpha.10", - "rust-s3", "serde", "serde_derive", "serde_json", - "sha2", - "sqlx", - "subprocess", - "tempdir", - "time 0.2.26", - "tokio 0.2.25", "toml", - "uuid", ] [[package]] diff --git a/app/http_server_common/Cargo.toml b/app/http_server_common/Cargo.toml index cf22ae1e20..6ce406a03b 100644 --- a/app/http_server_common/Cargo.toml +++ b/app/http_server_common/Cargo.toml @@ -18,74 +18,24 @@ path = "src/lib.rs" anyhow = "1.0.40" base64 = "0.13.0" chrono = { version = "0.4.19", features = ["serde"] } -derive_more = "0.99.14" dotenv = "0.15.0" -easyenv = "0.0.6" -futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool -futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate -hex = "0.4.3" -hostname = "0.3.1" log = "0.4.14" -rand = "0.8.3" -regex = "1.5.4" -time = "0.2.26" -uuid = { version = "0.8.2", features = ["v4"] } - -# Runtime -# NB(1): We switched to Tokio from `async-std` for broader ecosystem support. -#async-std = { version = "1.6", features = [ "attributes" ] } -# NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: -# https://github.com/durch/rust-s3/issues/198 -#tokio = { version = "1.6.0", features = ["macros"] } -tokio = { version = "0.2.25", features = ["macros"] } - -# MySql Database -# "offline" supports typesafe builds without a live database -# "rustls" doesn't appear to work with IP addresses, per: -# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 -#sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } -# NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade -#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } -sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } - -# Redis Database -limitation = "0.1.1" # Redis rate limiter -r2d2_redis = "0.14.0" - -# In-memory caching -lru_time_cache = "0.11.11" - -# Files -data-encoding = "2.3.2" -ring = "0.17.0-alpha.10" -# NB: This is the source of all of our problems. `rust-s3` can't upload to object names with directories -# in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2 -#rust-s3 = "0.27.0-rc4" -#rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } -rust-s3 = { version = "0.26" } -infer = "0.5.0" - -subprocess = "0.2.6" -tempdir = "0.3.7" # Web -actix-cors = "0.5.4" - # NB: Versions aren't pinned since some apps use ver 3 vs ver 4. # This is because of the Tokio runtime requirements. -actix-http = "*" -actix-web = "*" +#actix-http = "*" +#actix-web = "*" +#actix-http = "3.0.0-beta.11" +#actix-http = "2.2.0" +#actix-web = "4.0.0-beta.10" +#actix-web = "3.3.2" -# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime -# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. -hyper = { version = "0.13.10" } +actix-http-old = { package = "actix-http", version = "2.2.0", optional = true } +actix-web-old = { package = "actix-web", version = "3.3.2", optional = true } -# Crypto -bcrypt = "0.8" # password hashes -hmac = "0.10.0" # cookies -magic-crypt = "3.1.8" # encrypting sort IDs for the frontend so we don't leak info -md-5 = "0.9.1" # gravatar -sha2 = "0.9.5" # file hashes, inference text hashes, etc. +actix-http-new = { package = "actix-http", version = "3.0.0-beta.11", optional = true } +actix-web-new = { package = "actix-web", version = "4.0.0-beta.10", optional = true } # Serialization serde = "1.0.125" @@ -93,9 +43,6 @@ serde_derive = "1.0.125" serde_json = "1.0.64" toml = "0.5.8" -# Text processing -pulldown-cmark = "0.8.0" - -[build-dependencies] -# NB: Only using diesel for migrations support -diesel = { version = "1.4.6", features = ["mysql"] } +[features] +actix-new = ["actix-http-new", "actix-web-new"] +actix-old = ["actix-http-old", "actix-web-old"] diff --git a/app/http_server_common/src/lib.rs b/app/http_server_common/src/lib.rs index ebb920ee22..9fd1d17f59 100644 --- a/app/http_server_common/src/lib.rs +++ b/app/http_server_common/src/lib.rs @@ -4,6 +4,24 @@ #[macro_use] extern crate serde_derive; -pub mod cors; +// This is a little messy since we're supporting two versions of the Actix crate ecosystem. +// Long story short: one service requires an old Tokio runtime, and another needs the newer one. +// +// I don't think this will be too hard to maintain. AFAICT, the only major differences are which +// package "HttpResponseBuilder" is located in. + +#[cfg(feature = "actix-new")] +extern crate actix_http_new as actix_http; + +#[cfg(feature = "actix-new")] +extern crate actix_web_new as actix_web; + +#[cfg(feature = "actix-old")] +extern crate actix_http_old as actix_http; + +#[cfg(feature = "actix-old")] +extern crate actix_web_old as actix_web; + +//pub mod cors; pub mod endpoints; pub mod response; diff --git a/app/http_server_common/src/response/response_error_helpers.rs b/app/http_server_common/src/response/response_error_helpers.rs index 2977eb183d..4d1fda7942 100644 --- a/app/http_server_common/src/response/response_error_helpers.rs +++ b/app/http_server_common/src/response/response_error_helpers.rs @@ -1,8 +1,13 @@ use actix_http::http::header; -use actix_web::dev::HttpResponseBuilder; use actix_web::http::StatusCode; use actix_web::{HttpResponse, ResponseError}; +#[cfg(feature = "actix-web-old")] +use actix_web::dev::HttpResponseBuilder; + +#[cfg(feature = "actix-web-new")] +use actix_web::HttpResponseBuilder; + #[derive(Serialize)] pub struct SimpleGenericJsonError { pub success: bool, diff --git a/app/http_server_common/src/response/serialize_as_json_error.rs b/app/http_server_common/src/response/serialize_as_json_error.rs index beb6d25a25..54138ff539 100644 --- a/app/http_server_common/src/response/serialize_as_json_error.rs +++ b/app/http_server_common/src/response/serialize_as_json_error.rs @@ -1,9 +1,19 @@ -use actix_http::ResponseError; use actix_http::http::{StatusCode, header}; use actix_web::HttpResponse; -use actix_web::dev::HttpResponseBuilder; use serde::Serialize; +#[cfg(feature = "actix-http-new")] +use actix_web::error::ResponseError; + +#[cfg(feature = "actix-http-old")] +use actix_http::ResponseError; + +#[cfg(feature = "actix-web-old")] +use actix_web::dev::HttpResponseBuilder; + +#[cfg(feature = "actix-web-new")] +use actix_web::HttpResponseBuilder; + /// Turn error responses into JSON HTTP responses pub fn serialize_as_json_error( error_payload: &T, diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index 3a9500509c..cd745c51c6 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main.rs" # Internal config = { path = "../config" } container_common = { path = "../container_common" } -http_server_common = { path = "../http_server_common" } +http_server_common = { path = "../http_server_common", features = ["actix-new"] } # External anyhow = "1.0.40" diff --git a/app/obs_gateway/src/endpoints/oauth_begin.rs b/app/obs_gateway/src/endpoints/oauth_begin.rs index d0eb413cd6..236a96debc 100644 --- a/app/obs_gateway/src/endpoints/oauth_begin.rs +++ b/app/obs_gateway/src/endpoints/oauth_begin.rs @@ -46,9 +46,5 @@ pub async fn oauth_begin_enroll( info!("oauth enrollment begin"); - //Ok(simple_json_success()) - - Ok(HttpResponse::Ok() - .content_type("application/json") - .body("")) + Ok(simple_json_success()) } diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index ca9e038d9a..d1a2672ec5 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -31,7 +31,7 @@ use crate::twitch::twitch_secrets::TwitchSecrets; use crate::twitch::websocket_client::PollingTwitchWebsocketClient; use futures::Future; use futures::executor::ThreadPool; -use http_server_common::cors::build_common_cors_config; +//use http_server_common::cors::build_common_cors_config; use http_server_common::endpoints::default_route_404::default_route_404; use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index 40c16c4de7..b5a809d07c 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -102,7 +102,7 @@ use crate::util::caching::single_item_ttl_cache::SingleItemTtlCache; use crate::util::encrypted_sort_id::SortKeyCrypto; use futures::Future; use futures::executor::ThreadPool; -use http_server_common::cors::build_common_cors_config; +//use http_server_common::cors::build_common_cors_config; use http_server_common::endpoints::default_route_404::default_route_404; use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; @@ -318,7 +318,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> App::new() .app_data(server_state_arc.clone()) - .wrap(build_common_cors_config()) + //.wrap(build_common_cors_config()) .wrap(Logger::new(&log_format) .exclude("/liveness") .exclude("/readiness")) From 6c9fc98c6865041ad297b7d82bf21a7a4d2b6c1e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 9 Nov 2021 22:18:47 -0500 Subject: [PATCH 0653/1042] update docs --- Cargo.lock | 20 ++++++++++++++++++-- app/http_server_common/Cargo.toml | 13 +++---------- app/http_server_common/src/lib.rs | 2 +- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2363b85ce8..4383e03fa3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,6 +117,21 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "actix-cors" +version = "0.6.0-beta.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa59d36d7ad063401d94ada05264a303791796ad5222d0954cc21f2e1052e99b" +dependencies = [ + "actix-service 2.0.1", + "actix-web 4.0.0-beta.10", + "derive_more", + "futures-util", + "log", + "once_cell", + "smallvec 1.6.1", +] + [[package]] name = "actix-files" version = "0.5.0" @@ -2044,6 +2059,7 @@ dependencies = [ name = "http_server_common" version = "0.0.1" dependencies = [ + "actix-cors 0.6.0-beta.3", "actix-http 2.2.0", "actix-http 3.0.0-beta.11", "actix-web 3.3.2", @@ -2767,7 +2783,7 @@ name = "obs-gateway-server" version = "0.0.1" dependencies = [ "actix 0.12.0", - "actix-cors", + "actix-cors 0.5.4", "actix-files", "actix-http 3.0.0-beta.11", "actix-rt 2.4.0", @@ -4008,7 +4024,7 @@ name = "storyteller-web" version = "0.0.1" dependencies = [ "actix 0.10.0", - "actix-cors", + "actix-cors 0.5.4", "actix-files", "actix-http 2.2.0", "actix-multipart", diff --git a/app/http_server_common/Cargo.toml b/app/http_server_common/Cargo.toml index 6ce406a03b..8ac3c31bc8 100644 --- a/app/http_server_common/Cargo.toml +++ b/app/http_server_common/Cargo.toml @@ -22,18 +22,11 @@ dotenv = "0.15.0" log = "0.4.14" # Web -# NB: Versions aren't pinned since some apps use ver 3 vs ver 4. -# This is because of the Tokio runtime requirements. -#actix-http = "*" -#actix-web = "*" -#actix-http = "3.0.0-beta.11" -#actix-http = "2.2.0" -#actix-web = "4.0.0-beta.10" -#actix-web = "3.3.2" - +actix-cors = "0.6.0-beta.3" +# NB: One microservice requires old Tokio (thus old actix), and another new Tokio (new actix). +# Thus I'm being extremely extra here and supporting both. actix-http-old = { package = "actix-http", version = "2.2.0", optional = true } actix-web-old = { package = "actix-web", version = "3.3.2", optional = true } - actix-http-new = { package = "actix-http", version = "3.0.0-beta.11", optional = true } actix-web-new = { package = "actix-web", version = "4.0.0-beta.10", optional = true } diff --git a/app/http_server_common/src/lib.rs b/app/http_server_common/src/lib.rs index 9fd1d17f59..fb953e5635 100644 --- a/app/http_server_common/src/lib.rs +++ b/app/http_server_common/src/lib.rs @@ -22,6 +22,6 @@ extern crate actix_http_old as actix_http; #[cfg(feature = "actix-old")] extern crate actix_web_old as actix_web; -//pub mod cors; +pub mod cors; pub mod endpoints; pub mod response; From d0c156718aa4abdf8d4342ec75169f50338dd544 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 9 Nov 2021 23:00:39 -0500 Subject: [PATCH 0654/1042] fix for obs-gateway (broken for a few commits now) --- app/http_server_common/src/lib.rs | 6 ++---- .../src/response/response_error_helpers.rs | 8 ++++---- .../src/response/serialize_as_json_error.rs | 12 ++++++------ app/storyteller_web/Cargo.toml | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/app/http_server_common/src/lib.rs b/app/http_server_common/src/lib.rs index fb953e5635..3285e4237f 100644 --- a/app/http_server_common/src/lib.rs +++ b/app/http_server_common/src/lib.rs @@ -12,14 +12,12 @@ #[cfg(feature = "actix-new")] extern crate actix_http_new as actix_http; - #[cfg(feature = "actix-new")] extern crate actix_web_new as actix_web; -#[cfg(feature = "actix-old")] +#[cfg(not(feature = "actix-new"))] extern crate actix_http_old as actix_http; - -#[cfg(feature = "actix-old")] +#[cfg(not(feature = "actix-new"))] extern crate actix_web_old as actix_web; pub mod cors; diff --git a/app/http_server_common/src/response/response_error_helpers.rs b/app/http_server_common/src/response/response_error_helpers.rs index 4d1fda7942..cf9bbbfccc 100644 --- a/app/http_server_common/src/response/response_error_helpers.rs +++ b/app/http_server_common/src/response/response_error_helpers.rs @@ -2,12 +2,12 @@ use actix_http::http::header; use actix_web::http::StatusCode; use actix_web::{HttpResponse, ResponseError}; -#[cfg(feature = "actix-web-old")] -use actix_web::dev::HttpResponseBuilder; - -#[cfg(feature = "actix-web-new")] +#[cfg(feature = "actix-new")] use actix_web::HttpResponseBuilder; +#[cfg(not(feature = "actix-new"))] +use actix_web::dev::HttpResponseBuilder; + #[derive(Serialize)] pub struct SimpleGenericJsonError { pub success: bool, diff --git a/app/http_server_common/src/response/serialize_as_json_error.rs b/app/http_server_common/src/response/serialize_as_json_error.rs index 54138ff539..47ea5c3b2e 100644 --- a/app/http_server_common/src/response/serialize_as_json_error.rs +++ b/app/http_server_common/src/response/serialize_as_json_error.rs @@ -2,18 +2,18 @@ use actix_http::http::{StatusCode, header}; use actix_web::HttpResponse; use serde::Serialize; -#[cfg(feature = "actix-http-new")] +#[cfg(feature = "actix-new")] use actix_web::error::ResponseError; -#[cfg(feature = "actix-http-old")] +#[cfg(feature = "actix-new")] +use actix_web::HttpResponseBuilder; + +#[cfg(not(feature = "actix-new"))] use actix_http::ResponseError; -#[cfg(feature = "actix-web-old")] +#[cfg(not(feature = "actix-new"))] use actix_web::dev::HttpResponseBuilder; -#[cfg(feature = "actix-web-new")] -use actix_web::HttpResponseBuilder; - /// Turn error responses into JSON HTTP responses pub fn serialize_as_json_error( error_payload: &T, diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml index 3744dbf292..cd9417bd22 100644 --- a/app/storyteller_web/Cargo.toml +++ b/app/storyteller_web/Cargo.toml @@ -39,7 +39,7 @@ path = "src/w2l_inference_job.rs" # Internal config = { path = "../config" } -http_server_common = { path = "../http_server_common" } +http_server_common = { path = "../http_server_common", features = ["actix-old"] } # External anyhow = "1.0.40" From aa38f0a71f83b5b3aab68ebf15a4fb4f8d134d5e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 9 Nov 2021 23:12:04 -0500 Subject: [PATCH 0655/1042] mostly working for two projects (broke 404 handler, index) --- app/http_server_common/src/endpoints/root_index.rs | 11 ++++++++++- app/obs_gateway/src/main.rs | 4 ++-- app/storyteller_web/src/main.rs | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/app/http_server_common/src/endpoints/root_index.rs b/app/http_server_common/src/endpoints/root_index.rs index bfab8f441f..415b514186 100644 --- a/app/http_server_common/src/endpoints/root_index.rs +++ b/app/http_server_common/src/endpoints/root_index.rs @@ -2,7 +2,16 @@ use actix_http::http::StatusCode; use actix_web::{HttpResponse, HttpRequest, Responder, get}; use log::debug; -#[get("/")] + +/* +error[E0277]: the trait bound `fn() -> impl futures::Future {get_root_index}: HttpServiceFactory` is not satisfied + --> app/storyteller_web/src/main.rs:627:16 + | +627 | .service(get_root_index) + | ^^^^^^^^^^^^^^ the trait `HttpServiceFactory` is not implemented for `fn() -> impl futures::Future {get_root_index}` + */ + + pub async fn get_root_index() -> impl Responder { debug!("GET /"); // NB: Google load balancer hits this a lot, and it spams. HttpResponse::build(StatusCode::OK) diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index d1a2672ec5..a2485b7a3c 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -31,7 +31,7 @@ use crate::twitch::twitch_secrets::TwitchSecrets; use crate::twitch::websocket_client::PollingTwitchWebsocketClient; use futures::Future; use futures::executor::ThreadPool; -//use http_server_common::cors::build_common_cors_config; +use http_server_common::cors::build_common_cors_config; use http_server_common::endpoints::default_route_404::default_route_404; use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; @@ -308,7 +308,7 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> HttpServer::new(move || { App::new() .app_data(server_state_arc.clone()) - //.wrap(build_common_cors_config()) + .wrap(build_common_cors_config()) .wrap(Logger::new(&log_format) .exclude("/liveness") .exclude("/readiness")) diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index b5a809d07c..23ede7aabf 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -624,10 +624,10 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(leaderboard_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - .service(get_root_index) + //.service(get_root_index) .service(enable_alpha_handler) .service(enable_alpha_easy_handler) - .default_service( web::route().to(default_route_404)) + //.default_service( web::route().to(default_route_404)) }) .bind(bind_address)? .workers(num_workers) From c55cc810221f636bfdbf646ccf618fdf587088c5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 11 Nov 2021 22:13:53 -0500 Subject: [PATCH 0656/1042] TEMP BROKEN - need to upgrade storyteller-web, I think The situation with selectively enabling different versions off actix in a workspace using library features is crazy. It doesn't work, anyway. --- Cargo.lock | 2 +- app/http_server_common/Cargo.toml | 9 +++-- .../src/endpoints/root_index.rs | 17 +++------ app/http_server_common/src/lib.rs | 36 ++++++++++++++++--- app/obs_gateway/src/main.rs | 3 ++ app/storyteller_web/Cargo.toml | 6 ++-- .../endpoints/misc/default_route_404.rs | 10 ++++++ .../src/http_server/endpoints/misc/mod.rs | 4 ++- .../http_server/endpoints/misc/root_index.rs | 13 +++++++ app/storyteller_web/src/main.rs | 11 +++--- 10 files changed, 82 insertions(+), 29 deletions(-) create mode 100644 app/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs create mode 100644 app/storyteller_web/src/http_server/endpoints/misc/root_index.rs diff --git a/Cargo.lock b/Cargo.lock index 4383e03fa3..94b50f0ac9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2059,6 +2059,7 @@ dependencies = [ name = "http_server_common" version = "0.0.1" dependencies = [ + "actix-cors 0.5.4", "actix-cors 0.6.0-beta.3", "actix-http 2.2.0", "actix-http 3.0.0-beta.11", @@ -4048,7 +4049,6 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", - "http_server_common", "hyper 0.13.10", "infer", "jwt", diff --git a/app/http_server_common/Cargo.toml b/app/http_server_common/Cargo.toml index 8ac3c31bc8..545e3e3aa3 100644 --- a/app/http_server_common/Cargo.toml +++ b/app/http_server_common/Cargo.toml @@ -22,11 +22,13 @@ dotenv = "0.15.0" log = "0.4.14" # Web -actix-cors = "0.6.0-beta.3" # NB: One microservice requires old Tokio (thus old actix), and another new Tokio (new actix). # Thus I'm being extremely extra here and supporting both. +actix-cors-old = { package = "actix-cors", version = "0.5.4", optional = true } actix-http-old = { package = "actix-http", version = "2.2.0", optional = true } actix-web-old = { package = "actix-web", version = "3.3.2", optional = true } + +actix-cors-new = { package = "actix-cors", version = "0.6.0-beta.3", optional = true } actix-http-new = { package = "actix-http", version = "3.0.0-beta.11", optional = true } actix-web-new = { package = "actix-web", version = "4.0.0-beta.10", optional = true } @@ -37,5 +39,6 @@ serde_json = "1.0.64" toml = "0.5.8" [features] -actix-new = ["actix-http-new", "actix-web-new"] -actix-old = ["actix-http-old", "actix-web-old"] +# default = ["actix-cors-new", "actix-web-new", "actix-http-new"] +actix-new = ["actix-cors-new", "actix-http-new", "actix-web-new"] +actix-old = ["actix-cors-old", "actix-http-old", "actix-web-old"] diff --git a/app/http_server_common/src/endpoints/root_index.rs b/app/http_server_common/src/endpoints/root_index.rs index 415b514186..fe50b3a901 100644 --- a/app/http_server_common/src/endpoints/root_index.rs +++ b/app/http_server_common/src/endpoints/root_index.rs @@ -1,18 +1,11 @@ use actix_http::http::StatusCode; -use actix_web::{HttpResponse, HttpRequest, Responder, get}; +use actix_web::{HttpResponse, HttpRequest, get}; +use actix_web::Responder; +use actix_web::Result; +use actix_web::Error; use log::debug; - -/* -error[E0277]: the trait bound `fn() -> impl futures::Future {get_root_index}: HttpServiceFactory` is not satisfied - --> app/storyteller_web/src/main.rs:627:16 - | -627 | .service(get_root_index) - | ^^^^^^^^^^^^^^ the trait `HttpServiceFactory` is not implemented for `fn() -> impl futures::Future {get_root_index}` - */ - - -pub async fn get_root_index() -> impl Responder { +pub async fn get_root_index() -> HttpResponse { debug!("GET /"); // NB: Google load balancer hits this a lot, and it spams. HttpResponse::build(StatusCode::OK) .content_type("text/html; charset=utf-8") diff --git a/app/http_server_common/src/lib.rs b/app/http_server_common/src/lib.rs index 3285e4237f..a7e0cb7a41 100644 --- a/app/http_server_common/src/lib.rs +++ b/app/http_server_common/src/lib.rs @@ -10,16 +10,44 @@ // I don't think this will be too hard to maintain. AFAICT, the only major differences are which // package "HttpResponseBuilder" is located in. -#[cfg(feature = "actix-new")] +#[cfg(all(feature = "actix-new", not(feature = "actix-old")))] +//#[cfg(feature = "actix-cors-new")] +extern crate actix_cors_new as actix_cors; +#[cfg(all(feature = "actix-new", not(feature = "actix-old")))] +//#[cfg(feature = "actix-http-new")] extern crate actix_http_new as actix_http; -#[cfg(feature = "actix-new")] +#[cfg(all(feature = "actix-new", not(feature = "actix-old")))] +//#[cfg(feature = "actix-web-new")] extern crate actix_web_new as actix_web; -#[cfg(not(feature = "actix-new"))] +#[cfg(all(feature = "actix-old", not(feature = "actix-new")))] +//#[cfg(feature = "actix-cors-old")] +extern crate actix_cors_old as actix_cors; +#[cfg(all(feature = "actix-old", not(feature = "actix-new")))] +//#[cfg(feature = "actix-http-old")] extern crate actix_http_old as actix_http; -#[cfg(not(feature = "actix-new"))] +#[cfg(all(feature = "actix-old", not(feature = "actix-new")))] +//#[cfg(feature = "actix-web-old")] extern crate actix_web_old as actix_web; +//#[cfg(any( +// all(feature = "actix-cors-new", not(feature = "actix-cors-old")), +// all(feature = "actix-cors-old", not(feature = "actix-cors-new")), +//))] +#[cfg(any( + feature = "actix-new", + feature = "actix-old", +))] pub mod cors; + +#[cfg(any( + feature = "actix-new", + feature = "actix-old", +))] pub mod endpoints; + +#[cfg(any( + feature = "actix-new", + feature = "actix-old", +))] pub mod response; diff --git a/app/obs_gateway/src/main.rs b/app/obs_gateway/src/main.rs index a2485b7a3c..e69ece9f53 100644 --- a/app/obs_gateway/src/main.rs +++ b/app/obs_gateway/src/main.rs @@ -315,6 +315,9 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> .wrap(DefaultHeaders::new() .header("X-Backend-Hostname", &hostname) .header("X-Build-Sha", "")) + .service(web::resource("/") + .route(web::get().to(get_root_index)) + ) // Twitch .service( web::resource("/twitch") diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml index cd9417bd22..a1e229c316 100644 --- a/app/storyteller_web/Cargo.toml +++ b/app/storyteller_web/Cargo.toml @@ -39,7 +39,7 @@ path = "src/w2l_inference_job.rs" # Internal config = { path = "../config" } -http_server_common = { path = "../http_server_common", features = ["actix-old"] } +#http_server_common = { path = "../http_server_common", features = ["actix-old"] } # External anyhow = "1.0.40" @@ -101,9 +101,6 @@ tempdir = "0.3.7" # Web actix = "0.10" -#actix-web = "3" -actix-web-actors = "3" - actix-cors = "0.5.4" actix-files= "0.5" actix-http = "2.2.0" @@ -112,6 +109,7 @@ actix-rt = "2.2.0" actix-service = "1.0.6" actix-utils = "3.0.0" actix-web = "3.3.2" +actix-web-actors = "3" jwt = "0.13.0" # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime diff --git a/app/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs b/app/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs new file mode 100644 index 0000000000..6bc7403c15 --- /dev/null +++ b/app/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs @@ -0,0 +1,10 @@ +use actix_http::http::StatusCode; +use actix_web::{HttpResponse, HttpRequest, Responder, get}; +use log::warn; + +pub async fn default_route_404() -> impl Responder { + warn!("404 not found"); + HttpResponse::NotFound() + .content_type("text/html; charset=utf-8") + .body("

not found

") +} diff --git a/app/storyteller_web/src/http_server/endpoints/misc/mod.rs b/app/storyteller_web/src/http_server/endpoints/misc/mod.rs index d5d7a2545d..c4da5e82d0 100644 --- a/app/storyteller_web/src/http_server/endpoints/misc/mod.rs +++ b/app/storyteller_web/src/http_server/endpoints/misc/mod.rs @@ -1,3 +1,5 @@ pub mod alpha_cookie; +pub mod default_route_404; +pub mod enable_alpha_easy_handler; pub mod enable_alpha_handler; -pub mod enable_alpha_easy_handler; \ No newline at end of file +pub mod root_index; \ No newline at end of file diff --git a/app/storyteller_web/src/http_server/endpoints/misc/root_index.rs b/app/storyteller_web/src/http_server/endpoints/misc/root_index.rs new file mode 100644 index 0000000000..fe50b3a901 --- /dev/null +++ b/app/storyteller_web/src/http_server/endpoints/misc/root_index.rs @@ -0,0 +1,13 @@ +use actix_http::http::StatusCode; +use actix_web::{HttpResponse, HttpRequest, get}; +use actix_web::Responder; +use actix_web::Result; +use actix_web::Error; +use log::debug; + +pub async fn get_root_index() -> HttpResponse { + debug!("GET /"); // NB: Google load balancer hits this a lot, and it spams. + HttpResponse::build(StatusCode::OK) + .content_type("text/html; charset=utf-8") + .body("

hello!

Are you looking for an API? Join our Discord!

Maybe you want to work with us? We can pay! Get in touch!

") +} diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index 23ede7aabf..d6ee668191 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -103,8 +103,6 @@ use crate::util::encrypted_sort_id::SortKeyCrypto; use futures::Future; use futures::executor::ThreadPool; //use http_server_common::cors::build_common_cors_config; -use http_server_common::endpoints::default_route_404::default_route_404; -use http_server_common::endpoints::root_index::get_root_index; use limitation::Limiter; use log::{info}; use r2d2_redis::RedisConnectionManager; @@ -114,6 +112,8 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; +use crate::http_server::endpoints::misc::root_index::get_root_index; +use crate::http_server::endpoints::misc::default_route_404::default_route_404; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -624,10 +624,13 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(leaderboard_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - //.service(get_root_index) + .service(web::resource("/") + .route(web::get().to(get_root_index)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(enable_alpha_handler) .service(enable_alpha_easy_handler) - //.default_service( web::route().to(default_route_404)) + .default_service( web::route().to(default_route_404)) }) .bind(bind_address)? .workers(num_workers) From aa7c13ad5d70ef746167694153132f651df5dbb6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 11 Nov 2021 22:20:39 -0500 Subject: [PATCH 0657/1042] BROKEN - rearrange workspace to group libraries --- Cargo.toml | 6 +++--- app/{ => lib}/config/Cargo.toml | 0 app/{ => lib}/config/src/lib.rs | 0 app/{ => lib}/config/src/shared_constants.rs | 0 app/{ => lib}/container_common/Cargo.toml | 0 app/{ => lib}/container_common/src/anyhow_result.rs | 0 app/{ => lib}/container_common/src/lib.rs | 0 app/{ => lib}/http_server_common/Cargo.toml | 0 app/{ => lib}/http_server_common/src/cors.rs | 0 .../http_server_common/src/endpoints/default_route_404.rs | 0 app/{ => lib}/http_server_common/src/endpoints/mod.rs | 0 .../http_server_common/src/endpoints/root_index.rs | 0 app/{ => lib}/http_server_common/src/lib.rs | 0 app/{ => lib}/http_server_common/src/response/mod.rs | 0 .../src/response/response_error_helpers.rs | 0 .../src/response/response_success_helpers.rs | 0 .../src/response/serialize_as_json_error.rs | 0 app/obs_gateway/Cargo.toml | 6 +++--- app/storyteller_web/Cargo.toml | 4 ++-- 19 files changed, 8 insertions(+), 8 deletions(-) rename app/{ => lib}/config/Cargo.toml (100%) rename app/{ => lib}/config/src/lib.rs (100%) rename app/{ => lib}/config/src/shared_constants.rs (100%) rename app/{ => lib}/container_common/Cargo.toml (100%) rename app/{ => lib}/container_common/src/anyhow_result.rs (100%) rename app/{ => lib}/container_common/src/lib.rs (100%) rename app/{ => lib}/http_server_common/Cargo.toml (100%) rename app/{ => lib}/http_server_common/src/cors.rs (100%) rename app/{ => lib}/http_server_common/src/endpoints/default_route_404.rs (100%) rename app/{ => lib}/http_server_common/src/endpoints/mod.rs (100%) rename app/{ => lib}/http_server_common/src/endpoints/root_index.rs (100%) rename app/{ => lib}/http_server_common/src/lib.rs (100%) rename app/{ => lib}/http_server_common/src/response/mod.rs (100%) rename app/{ => lib}/http_server_common/src/response/response_error_helpers.rs (100%) rename app/{ => lib}/http_server_common/src/response/response_success_helpers.rs (100%) rename app/{ => lib}/http_server_common/src/response/serialize_as_json_error.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 2c2cacc9dd..5a919a6a90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [workspace] members = [ - "app/config", - "app/container_common", - "app/http_server_common", + "app/lib/config", + "app/lib/container_common", + "app/lib/http_server_common", "app/obs_gateway", "app/storyteller_web", ] diff --git a/app/config/Cargo.toml b/app/lib/config/Cargo.toml similarity index 100% rename from app/config/Cargo.toml rename to app/lib/config/Cargo.toml diff --git a/app/config/src/lib.rs b/app/lib/config/src/lib.rs similarity index 100% rename from app/config/src/lib.rs rename to app/lib/config/src/lib.rs diff --git a/app/config/src/shared_constants.rs b/app/lib/config/src/shared_constants.rs similarity index 100% rename from app/config/src/shared_constants.rs rename to app/lib/config/src/shared_constants.rs diff --git a/app/container_common/Cargo.toml b/app/lib/container_common/Cargo.toml similarity index 100% rename from app/container_common/Cargo.toml rename to app/lib/container_common/Cargo.toml diff --git a/app/container_common/src/anyhow_result.rs b/app/lib/container_common/src/anyhow_result.rs similarity index 100% rename from app/container_common/src/anyhow_result.rs rename to app/lib/container_common/src/anyhow_result.rs diff --git a/app/container_common/src/lib.rs b/app/lib/container_common/src/lib.rs similarity index 100% rename from app/container_common/src/lib.rs rename to app/lib/container_common/src/lib.rs diff --git a/app/http_server_common/Cargo.toml b/app/lib/http_server_common/Cargo.toml similarity index 100% rename from app/http_server_common/Cargo.toml rename to app/lib/http_server_common/Cargo.toml diff --git a/app/http_server_common/src/cors.rs b/app/lib/http_server_common/src/cors.rs similarity index 100% rename from app/http_server_common/src/cors.rs rename to app/lib/http_server_common/src/cors.rs diff --git a/app/http_server_common/src/endpoints/default_route_404.rs b/app/lib/http_server_common/src/endpoints/default_route_404.rs similarity index 100% rename from app/http_server_common/src/endpoints/default_route_404.rs rename to app/lib/http_server_common/src/endpoints/default_route_404.rs diff --git a/app/http_server_common/src/endpoints/mod.rs b/app/lib/http_server_common/src/endpoints/mod.rs similarity index 100% rename from app/http_server_common/src/endpoints/mod.rs rename to app/lib/http_server_common/src/endpoints/mod.rs diff --git a/app/http_server_common/src/endpoints/root_index.rs b/app/lib/http_server_common/src/endpoints/root_index.rs similarity index 100% rename from app/http_server_common/src/endpoints/root_index.rs rename to app/lib/http_server_common/src/endpoints/root_index.rs diff --git a/app/http_server_common/src/lib.rs b/app/lib/http_server_common/src/lib.rs similarity index 100% rename from app/http_server_common/src/lib.rs rename to app/lib/http_server_common/src/lib.rs diff --git a/app/http_server_common/src/response/mod.rs b/app/lib/http_server_common/src/response/mod.rs similarity index 100% rename from app/http_server_common/src/response/mod.rs rename to app/lib/http_server_common/src/response/mod.rs diff --git a/app/http_server_common/src/response/response_error_helpers.rs b/app/lib/http_server_common/src/response/response_error_helpers.rs similarity index 100% rename from app/http_server_common/src/response/response_error_helpers.rs rename to app/lib/http_server_common/src/response/response_error_helpers.rs diff --git a/app/http_server_common/src/response/response_success_helpers.rs b/app/lib/http_server_common/src/response/response_success_helpers.rs similarity index 100% rename from app/http_server_common/src/response/response_success_helpers.rs rename to app/lib/http_server_common/src/response/response_success_helpers.rs diff --git a/app/http_server_common/src/response/serialize_as_json_error.rs b/app/lib/http_server_common/src/response/serialize_as_json_error.rs similarity index 100% rename from app/http_server_common/src/response/serialize_as_json_error.rs rename to app/lib/http_server_common/src/response/serialize_as_json_error.rs diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index cd745c51c6..6d411c8e80 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -15,9 +15,9 @@ path = "src/main.rs" [dependencies] # Internal -config = { path = "../config" } -container_common = { path = "../container_common" } -http_server_common = { path = "../http_server_common", features = ["actix-new"] } +config = { path = "../lib/config" } +container_common = { path = "../lib/container_common" } +http_server_common = { path = "../lib/http_server_common", features = ["actix-new"] } # External anyhow = "1.0.40" diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml index a1e229c316..1cde7646cd 100644 --- a/app/storyteller_web/Cargo.toml +++ b/app/storyteller_web/Cargo.toml @@ -38,8 +38,8 @@ path = "src/w2l_inference_job.rs" [dependencies] # Internal -config = { path = "../config" } -#http_server_common = { path = "../http_server_common", features = ["actix-old"] } +config = { path = "../lib/config" } +#http_server_common = { path = "../lib/http_server_common", features = ["actix-old"] } # External anyhow = "1.0.40" From e424297abf960afa84330c21fe9ea540b6a79103 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 00:07:17 -0500 Subject: [PATCH 0658/1042] WIP --- Cargo.lock | 2 +- app/obs_gateway/Cargo.toml | 2 +- app/storyteller_web/Cargo.toml | 2 +- app/storyteller_web/src/main.rs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94b50f0ac9..f8775bae7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2808,7 +2808,6 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", - "http_server_common", "hyper 0.13.10", "jwt", "lazy_static", @@ -4049,6 +4048,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", + "http_server_common", "hyper 0.13.10", "infer", "jwt", diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index 6d411c8e80..e97f34511d 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main.rs" # Internal config = { path = "../lib/config" } container_common = { path = "../lib/container_common" } -http_server_common = { path = "../lib/http_server_common", features = ["actix-new"] } +#http_server_common = { path = "../lib/http_server_common", features = ["actix-new"] } # External anyhow = "1.0.40" diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml index 1cde7646cd..bcce8433b7 100644 --- a/app/storyteller_web/Cargo.toml +++ b/app/storyteller_web/Cargo.toml @@ -39,7 +39,7 @@ path = "src/w2l_inference_job.rs" # Internal config = { path = "../lib/config" } -#http_server_common = { path = "../lib/http_server_common", features = ["actix-old"] } +http_server_common = { path = "../lib/http_server_common", features = ["actix-old"] } # External anyhow = "1.0.40" diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index d6ee668191..0636e91fcc 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -102,7 +102,7 @@ use crate::util::caching::single_item_ttl_cache::SingleItemTtlCache; use crate::util::encrypted_sort_id::SortKeyCrypto; use futures::Future; use futures::executor::ThreadPool; -//use http_server_common::cors::build_common_cors_config; +use http_server_common::cors::build_common_cors_config; use limitation::Limiter; use log::{info}; use r2d2_redis::RedisConnectionManager; @@ -318,7 +318,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> App::new() .app_data(server_state_arc.clone()) - //.wrap(build_common_cors_config()) + .wrap(build_common_cors_config()) .wrap(Logger::new(&log_format) .exclude("/liveness") .exclude("/readiness")) From c40d772ebcb7a084fb770cc9e15ac3578cbe33dc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 01:12:39 -0500 Subject: [PATCH 0659/1042] MAYBE BROKEN: Full actix/tokio/etc. upgrade on storyteller-web --- Cargo.lock | 818 ++++++++---------- app/lib/http_server_common/Cargo.toml | 17 +- app/lib/http_server_common/src/lib.rs | 38 - .../src/response/response_error_helpers.rs | 6 +- .../src/response/serialize_as_json_error.rs | 14 +- app/storyteller_web/Cargo.toml | 44 +- .../src/database/mediators/badge_granter.rs | 4 +- .../database/mediators/firehose_publisher.rs | 4 +- .../endpoints/events/list_events.rs | 2 +- .../endpoints/leaderboard/get_leaderboard.rs | 2 +- .../endpoints/misc/alpha_cookie.rs | 2 +- .../misc/enable_alpha_easy_handler.rs | 2 +- .../endpoints/misc/enable_alpha_handler.rs | 2 +- .../approval/pending_w2l_templates.rs | 2 +- .../moderation/ip_bans/add_ip_ban.rs | 2 +- .../moderation/ip_bans/delete_ip_ban.rs | 2 +- .../moderation/ip_bans/get_ip_ban.rs | 2 +- .../moderation/ip_bans/list_ip_bans.rs | 2 +- .../jobs/get_tts_inference_queue_count.rs | 2 +- .../jobs/get_w2l_inference_queue_count.rs | 2 +- .../moderation/stats/get_voice_count_stats.rs | 2 +- .../moderation/user_bans/ban_user.rs | 2 +- .../moderation/user_bans/list_banned_users.rs | 2 +- .../moderation/user_roles/list_roles.rs | 2 +- .../moderation/user_roles/list_staff.rs | 2 +- .../moderation/user_roles/set_user_role.rs | 2 +- .../endpoints/moderation/users/list_users.rs | 2 +- .../endpoints/tts/delete_tts_model.rs | 2 +- .../endpoints/tts/delete_tts_result.rs | 2 +- .../endpoints/tts/edit_tts_model.rs | 2 +- .../endpoints/tts/edit_tts_result.rs | 2 +- .../endpoints/tts/enqueue_infer_tts.rs | 2 +- .../endpoints/tts/enqueue_upload_tts_model.rs | 2 +- .../tts/get_tts_inference_job_status.rs | 2 +- .../endpoints/tts/get_tts_model.rs | 2 +- .../endpoints/tts/get_tts_model_use_count.rs | 2 +- .../endpoints/tts/get_tts_result.rs | 2 +- .../tts/get_tts_upload_model_job_status.rs | 2 +- .../endpoints/tts/list_tts_models.rs | 2 +- .../endpoints/users/create_account.rs | 2 +- .../endpoints/users/edit_profile.rs | 2 +- .../endpoints/users/get_profile.rs | 2 +- .../users/list_user_tts_inference_results.rs | 2 +- .../endpoints/users/list_user_tts_models.rs | 2 +- .../users/list_user_w2l_inference_results.rs | 2 +- .../users/list_user_w2l_templates.rs | 2 +- .../src/http_server/endpoints/users/login.rs | 2 +- .../src/http_server/endpoints/users/logout.rs | 2 +- .../endpoints/users/session_info.rs | 2 +- .../endpoints/w2l/delete_w2l_result.rs | 2 +- .../endpoints/w2l/delete_w2l_template.rs | 2 +- .../endpoints/w2l/edit_w2l_result.rs | 2 +- .../endpoints/w2l/edit_w2l_template.rs | 2 +- .../endpoints/w2l/enqueue_infer_w2l.rs | 2 +- .../w2l/enqueue_infer_w2l_with_uploads.rs | 2 +- .../w2l/enqueue_upload_w2l_template.rs | 2 +- .../w2l/get_w2l_inference_job_status.rs | 2 +- .../endpoints/w2l/get_w2l_result.rs | 2 +- .../endpoints/w2l/get_w2l_template.rs | 2 +- .../w2l/get_w2l_template_use_count.rs | 2 +- .../w2l/get_w2l_upload_template_job_status.rs | 2 +- .../endpoints/w2l/list_w2l_templates.rs | 2 +- .../w2l/set_w2l_template_mod_approval.rs | 2 +- .../src/http_server/middleware/mod.rs | 2 +- .../web_utils/response_error_helpers.rs | 2 +- .../web_utils/serialize_as_json_error.rs | 4 +- app/storyteller_web/src/main.rs | 4 +- .../src/util/buckets/bucket_client.rs | 16 +- 68 files changed, 473 insertions(+), 610 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8775bae7e..e20b79a808 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,31 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "actix" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1be241f88f3b1e7e9a3fbe3b5a8a0f6915b5a1d7ee0d9a248d3376d01068cc60" -dependencies = [ - "actix-rt 1.1.1", - "actix_derive 0.5.0", - "bitflags", - "bytes 0.5.6", - "crossbeam-channel 0.4.4", - "derive_more", - "futures-channel", - "futures-util", - "log", - "once_cell", - "parking_lot 0.11.1", - "pin-project 0.4.28", - "smallvec 1.6.1", - "tokio 0.2.25", - "tokio-util 0.3.1", - "trust-dns-proto", - "trust-dns-resolver", -] - [[package]] name = "actix" version = "0.12.0" @@ -34,10 +9,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3720d0064a0ce5c0de7bd93bdb0a6caebab2a9b5668746145d7b3b0c5da02914" dependencies = [ "actix-rt 2.4.0", - "actix_derive 0.6.0", + "actix_derive", "bitflags", "bytes 1.0.1", - "crossbeam-channel 0.5.1", + "crossbeam-channel", "futures-core", "futures-sink", "futures-task", @@ -152,6 +127,28 @@ dependencies = [ "v_htmlescape", ] +[[package]] +name = "actix-files" +version = "0.6.0-beta.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3754d4632de16dd68f6a5d379cf9d7200ec26020482e26aa16560a258e92f3" +dependencies = [ + "actix-http 3.0.0-beta.11", + "actix-service 2.0.1", + "actix-utils 3.0.0", + "actix-web 4.0.0-beta.10", + "askama_escape", + "bitflags", + "bytes 1.0.1", + "derive_more", + "futures-core", + "http-range", + "log", + "mime", + "mime_guess", + "percent-encoding 2.1.0", +] + [[package]] name = "actix-http" version = "2.2.0" @@ -166,14 +163,12 @@ dependencies = [ "actix-utils 2.0.0", "base64 0.13.0", "bitflags", - "brotli2", "bytes 0.5.6", "cookie 0.14.4", "copyless", "derive_more", "either", "encoding_rs", - "flate2", "futures-channel", "futures-core", "futures-util", @@ -263,17 +258,18 @@ dependencies = [ [[package]] name = "actix-multipart" -version = "0.3.0" +version = "0.4.0-beta.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774bfeb11b54bf9c857a005b8ab893293da4eaff79261a66a9200dab7f5ab6e3" +checksum = "ee3878dadc5d0e2afa0439b4e1ef3b03fa2feb72c208d8c6f196ed6242979f6d" dependencies = [ - "actix-service 1.0.6", - "actix-utils 2.0.0", - "actix-web 3.3.2", - "bytes 0.5.6", + "actix-utils 3.0.0", + "actix-web 4.0.0-beta.10", + "bytes 1.0.1", "derive_more", + "futures-core", "futures-util", "httparse", + "local-waker", "log", "mime", "twoway", @@ -557,29 +553,13 @@ dependencies = [ "url 2.2.2", ] -[[package]] -name = "actix-web-actors" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6edf3c2693e2a8c422800c87ee89a6a4eac7dd01109bc172a1093ce1f4f001" -dependencies = [ - "actix 0.10.0", - "actix-codec 0.3.0", - "actix-http 2.2.0", - "actix-web 3.3.2", - "bytes 0.5.6", - "futures-channel", - "futures-core", - "pin-project 0.4.28", -] - [[package]] name = "actix-web-actors" version = "4.0.0-beta.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b43c8e03f1877bc010316de455ee479aa0de0ae0717d49041db5f38fa192de" dependencies = [ - "actix 0.12.0", + "actix", "actix-codec 0.4.1", "actix-http 3.0.0-beta.11", "actix-web 4.0.0-beta.10", @@ -613,17 +593,6 @@ dependencies = [ "syn", ] -[[package]] -name = "actix_derive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b95aceadaf327f18f0df5962fedc1bde2f870566a0b9f65c89508a3b1f79334c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "actix_derive" version = "0.6.0" @@ -651,12 +620,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "ahash" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" - [[package]] name = "ahash" version = "0.4.7" @@ -733,116 +696,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" [[package]] -name = "async-channel" -version = "1.6.1" +name = "askama_escape" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - -[[package]] -name = "async-executor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" -dependencies = [ - "async-task", - "concurrent-queue", - "fastrand", - "futures-lite", - "once_cell", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9586ec52317f36de58453159d48351bc244bc24ced3effc1fce22f3d48664af6" -dependencies = [ - "async-channel", - "async-executor", - "async-io", - "async-mutex", - "blocking", - "futures-lite", - "num_cpus", - "once_cell", -] - -[[package]] -name = "async-io" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bbfd5cf2794b1e908ea8457e6c45f8f8f1f6ec5f74617bf4662623f47503c3b" -dependencies = [ - "concurrent-queue", - "fastrand", - "futures-lite", - "libc", - "log", - "once_cell", - "parking", - "polling", - "slab", - "socket2 0.4.0", - "waker-fn", - "winapi 0.3.9", -] - -[[package]] -name = "async-lock" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6a8ea61bf9947a1007c5cada31e647dbc77b103c679858150003ba697ea798b" -dependencies = [ - "event-listener", -] - -[[package]] -name = "async-mutex" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" -dependencies = [ - "event-listener", -] - -[[package]] -name = "async-std" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f06685bad74e0570f5213741bea82158279a4103d988e57bfada11ad230341" -dependencies = [ - "async-channel", - "async-global-executor", - "async-io", - "async-lock", - "crossbeam-utils 0.8.4", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "num_cpus", - "once_cell", - "pin-project-lite 0.2.6", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - -[[package]] -name = "async-task" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" +checksum = "90c108c1a94380c89d2215d0ac54ce09796823cca0fd91b299cfff3b33e346fb" [[package]] name = "async-trait" @@ -864,17 +721,11 @@ dependencies = [ "num-traits", ] -[[package]] -name = "atomic-waker" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" - [[package]] name = "attohttpc" -version = "0.15.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe174d1b67f7b2bafed829c09db039301eb5841f66e43be2cf60b326e7f8e2cc" +checksum = "e69e13a99a7e6e070bb114f7ff381e58c7ccc188630121fc4c2fe4bcf24cd072" dependencies = [ "http", "log", @@ -883,6 +734,7 @@ dependencies = [ "serde", "serde_json", "url 2.2.2", + "wildmatch", ] [[package]] @@ -934,27 +786,27 @@ dependencies = [ [[package]] name = "aws-creds" -version = "0.24.1" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad53a54cb2c99990e96eefacde6f143dc6d471ab70809d26d360292be421d490" +checksum = "460a75eac8f3cb7683e0a9a588a83c3ff039331ea7bfbfbfcecf1dacab276e11" dependencies = [ + "anyhow", "attohttpc", "dirs", "rust-ini", "serde", "serde-xml-rs", "serde_derive", - "simpl", "url 2.2.2", ] [[package]] name = "aws-region" -version = "0.22.1" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f610af4a396f07592014dc3410f6ad78fab931852a99bb6cfdc1ad04b9329b80" +checksum = "b1efb67b8f201dd0deea4e1240bce7da0366f8df90509a7df054973604b20b34" dependencies = [ - "simpl", + "anyhow", ] [[package]] @@ -1040,20 +892,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" -[[package]] -name = "blocking" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" -dependencies = [ - "async-channel", - "async-task", - "atomic-waker", - "fastrand", - "futures-lite", - "once_cell", -] - [[package]] name = "blowfish" version = "0.6.0" @@ -1143,12 +981,6 @@ dependencies = [ "bytes 1.0.1", ] -[[package]] -name = "cache-padded" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" - [[package]] name = "cargo-platform" version = "0.1.1" @@ -1257,15 +1089,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "concurrent-queue" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" -dependencies = [ - "cache-padded", -] - [[package]] name = "config" version = "0.0.1" @@ -1351,6 +1174,15 @@ dependencies = [ "build_const", ] +[[package]] +name = "crc" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" +dependencies = [ + "crc-catalog", +] + [[package]] name = "crc-any" version = "2.3.11" @@ -1360,6 +1192,12 @@ dependencies = [ "debug-helper", ] +[[package]] +name = "crc-catalog" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" + [[package]] name = "crc32fast" version = "1.2.1" @@ -1369,16 +1207,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "crossbeam-channel" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" -dependencies = [ - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -1421,16 +1249,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "crypto-mac" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bcd97a54c7ca5ce2f6eb16f6bede5b0ab5f0055fedc17d2f0b4466e21671ca" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "crypto-mac" version = "0.10.0" @@ -1442,13 +1260,13 @@ dependencies = [ ] [[package]] -name = "ctor" -version = "0.1.20" +name = "crypto-mac" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" dependencies = [ - "quote", - "syn", + "generic-array", + "subtle", ] [[package]] @@ -1520,9 +1338,9 @@ dependencies = [ [[package]] name = "dirs" -version = "3.0.2" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ "dirs-sys", ] @@ -1630,21 +1448,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "event-listener" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" - -[[package]] -name = "fastrand" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b705829d1e87f762c2df6da140b26af5839e1033aa84aa5f56bb688e4e1bdb" -dependencies = [ - "instant", -] - [[package]] name = "firestorm" version = "0.4.6" @@ -1771,27 +1574,23 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "futures-intrusive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62007592ac46aa7c2b6416f7deb9a8a8f63a01e0f1d6e1787d5630170db2b63e" +dependencies = [ + "futures-core", + "lock_api 0.4.4", + "parking_lot 0.11.1", +] + [[package]] name = "futures-io" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" -[[package]] -name = "futures-lite" -version = "1.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4481d0cd0de1d204a4fa55e7d45f07b1d958abcb06714b3446438e2eff695fb" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite 0.2.6", - "waker-fn", -] - [[package]] name = "futures-macro" version = "0.3.17" @@ -1888,19 +1687,6 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] -[[package]] -name = "gloo-timers" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "h2" version = "0.2.7" @@ -1940,16 +1726,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" -dependencies = [ - "ahash 0.3.8", - "autocfg 1.0.1", -] - [[package]] name = "hashbrown" version = "0.9.1" @@ -1959,6 +1735,15 @@ dependencies = [ "ahash 0.4.7", ] +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash 0.7.4", +] + [[package]] name = "hashlink" version = "0.6.0" @@ -1968,6 +1753,15 @@ dependencies = [ "hashbrown 0.9.1", ] +[[package]] +name = "hashlink" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +dependencies = [ + "hashbrown 0.11.2", +] + [[package]] name = "heck" version = "0.3.2" @@ -1992,16 +1786,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hmac" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deae6d9dbb35ec2c502d62b8f7b1c000a0822c3b0794ba36b3149c0a1c840dff" -dependencies = [ - "crypto-mac 0.9.1", - "digest", -] - [[package]] name = "hmac" version = "0.10.1" @@ -2012,6 +1796,16 @@ dependencies = [ "digest", ] +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac 0.11.1", + "digest", +] + [[package]] name = "hostname" version = "0.3.1" @@ -2055,15 +1849,18 @@ dependencies = [ "pin-project-lite 0.2.6", ] +[[package]] +name = "http-range" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eee9694f83d9b7c09682fdb32213682939507884e5bcf227be9aff5d644b90dc" + [[package]] name = "http_server_common" version = "0.0.1" dependencies = [ - "actix-cors 0.5.4", "actix-cors 0.6.0-beta.3", - "actix-http 2.2.0", "actix-http 3.0.0-beta.11", - "actix-web 3.3.2", "actix-web 4.0.0-beta.10", "anyhow", "base64 0.13.0", @@ -2303,15 +2100,6 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - [[package]] name = "language-tags" version = "0.2.2" @@ -2437,7 +2225,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ "cfg-if 1.0.0", - "value-bag", ] [[package]] @@ -2490,6 +2277,17 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "maybe-async" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6007f9dad048e0a224f27ca599d669fca8cfa0dac804725aab542b2eb032bce6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -2535,6 +2333,15 @@ dependencies = [ "unicase", ] +[[package]] +name = "minidom" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e" +dependencies = [ + "quick-xml", +] + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -2720,6 +2527,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.6.1" @@ -2739,6 +2557,24 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-bigint-dig" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4547ee5541c18742396ae2c895d0717d0f886d8823b8399cdaf7b07d63ad0480" +dependencies = [ + "autocfg 0.1.7", + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.4", + "smallvec 1.6.1", + "zeroize", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -2767,6 +2603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ "autocfg 1.0.1", + "libm", ] [[package]] @@ -2783,15 +2620,15 @@ dependencies = [ name = "obs-gateway-server" version = "0.0.1" dependencies = [ - "actix 0.12.0", + "actix", "actix-cors 0.5.4", - "actix-files", + "actix-files 0.5.0", "actix-http 3.0.0-beta.11", "actix-rt 2.4.0", "actix-service 2.0.1", "actix-utils 3.0.0", "actix-web 4.0.0-beta.10", - "actix-web-actors 4.0.0-beta.7", + "actix-web-actors", "anyhow", "base64 0.13.0", "bcrypt", @@ -2822,13 +2659,13 @@ dependencies = [ "r2d2_redis", "rand 0.8.4", "regex", - "reqwest 0.11.6", + "reqwest", "rpassword", "serde", "serde_derive", "serde_json", "sha2", - "sqlx", + "sqlx 0.4.2", "time 0.2.26", "tokio 1.13.0", "tokio-tungstenite", @@ -2885,20 +2722,14 @@ dependencies = [ [[package]] name = "ordered-multimap" -version = "0.2.4" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88f947c6799d5eff50e6cf8a2365c17ac4aa8f8f43aceeedc29b616d872a358" +checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485" dependencies = [ "dlv-list", - "hashbrown 0.7.2", + "hashbrown 0.9.1", ] -[[package]] -name = "parking" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" - [[package]] name = "parking_lot" version = "0.9.0" @@ -3052,19 +2883,6 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" -[[package]] -name = "polling" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fc12d774e799ee9ebae13f4076ca003b40d18a11ac0f3641e6f899618580b7b" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "log", - "wepoll-sys", - "winapi 0.3.9", -] - [[package]] name = "ppv-lite86" version = "0.2.10" @@ -3110,6 +2928,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.9" @@ -3352,42 +3179,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "reqwest" -version = "0.10.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" -dependencies = [ - "base64 0.13.0", - "bytes 0.5.6", - "encoding_rs", - "futures-core", - "futures-util", - "http", - "http-body 0.3.1", - "hyper 0.13.10", - "hyper-tls 0.4.3", - "ipnet", - "js-sys", - "lazy_static", - "log", - "mime", - "mime_guess", - "native-tls", - "percent-encoding 2.1.0", - "pin-project-lite 0.2.6", - "serde", - "serde_json", - "serde_urlencoded", - "tokio 0.2.25", - "tokio-tls", - "url 2.2.2", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg 0.7.0", -] - [[package]] name = "reqwest" version = "0.11.6" @@ -3482,57 +3273,78 @@ dependencies = [ "byteorder", "digest", "lazy_static", - "num-bigint-dig", + "num-bigint-dig 0.6.1", "num-integer", "num-iter", "num-traits", "pem", "rand 0.7.3", "sha2", - "simple_asn1", + "simple_asn1 0.4.1", "subtle", "thiserror", "zeroize", ] [[package]] -name = "rust-ini" -version = "0.15.3" +name = "rsa" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a3679dd538c876a7b606f3bb951c8a20fc281a0ff7795f59f7cb490e3f979e1" +checksum = "7b0aeddcca1082112a6eeb43bf25fd7820b066aaf6eaef776e19d0a1febe38fe" dependencies = [ - "cfg-if 0.1.10", + "byteorder", + "digest", + "lazy_static", + "num-bigint-dig 0.7.0", + "num-integer", + "num-iter", + "num-traits", + "pem", + "rand 0.8.4", + "simple_asn1 0.5.4", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-ini" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63471c4aa97a1cf8332a5f97709a79a4234698de6a1f5087faf66f2dae810e22" +dependencies = [ + "cfg-if 1.0.0", "ordered-multimap", ] [[package]] name = "rust-s3" -version = "0.26.4" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a04bdd0f5118d06ef0e2daa658a9e5135282bcecfd6c46b11fddf47b1d5d736d" +checksum = "18c58d4682844a5d6301efbf915dd7a9d3d638006f9bb821527a0bbbf2a4cfc2" dependencies = [ - "async-std", + "anyhow", + "async-trait", "aws-creds", "aws-region", "base64 0.13.0", "cfg-if 1.0.0", "chrono", - "futures 0.3.17", "hex", - "hmac 0.9.0", + "hmac 0.11.0", "http", "log", + "maybe-async", "md5", + "minidom", "percent-encoding 2.1.0", - "reqwest 0.10.10", + "reqwest", "serde", "serde-xml-rs", "serde_derive", "sha2", - "simpl", - "tokio 0.2.25", + "tokio 1.13.0", + "tokio-stream", "url 2.2.2", - "uuid", ] [[package]] @@ -3678,9 +3490,9 @@ dependencies = [ [[package]] name = "serde-xml-rs" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0bf1ba0696ccf0872866277143ff1fd14d22eec235d2b23702f95e6660f7dfa" +checksum = "65162e9059be2f6a3421ebbb4fef3e74b7d9e7c60c50a0e292c6239f19f1edfa" dependencies = [ "log", "serde", @@ -3784,12 +3596,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simpl" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a30f10c911c0355f80f1c2faa8096efc4a58cdf8590b954d5b395efa071c711" - [[package]] name = "simple_asn1" version = "0.4.1" @@ -3801,6 +3607,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "simple_asn1" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb4ea60fb301dc81dfc113df680571045d375ab7345d171c5dc7d7e13107a80" +dependencies = [ + "chrono", + "num-bigint 0.4.3", + "num-traits", + "thiserror", +] + [[package]] name = "slab" version = "0.4.3" @@ -3868,8 +3686,18 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1a98f9bf17b690f026b6fec565293a995b46dfbd6293debcb654dcffd2d1b34" dependencies = [ - "sqlx-core", - "sqlx-macros", + "sqlx-core 0.4.2", + "sqlx-macros 0.4.2", +] + +[[package]] +name = "sqlx" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4b94ab0f8c21ee4899b93b06451ef5d965f1a355982ee73684338228498440" +dependencies = [ + "sqlx-core 0.5.7", + "sqlx-macros 0.5.7", ] [[package]] @@ -3885,8 +3713,8 @@ dependencies = [ "byteorder", "bytes 0.5.6", "chrono", - "crc", - "crossbeam-channel 0.5.1", + "crc 1.8.1", + "crossbeam-channel", "crossbeam-queue", "crossbeam-utils 0.8.4", "digest", @@ -3895,7 +3723,7 @@ dependencies = [ "futures-core", "futures-util", "generic-array", - "hashlink", + "hashlink 0.6.0", "hex", "itoa", "libc", @@ -3906,19 +3734,68 @@ dependencies = [ "parking_lot 0.11.1", "percent-encoding 2.1.0", "rand 0.7.3", - "rsa", + "rsa 0.3.0", "serde", "sha-1", "sha2", "smallvec 1.6.1", "sqlformat", - "sqlx-rt", + "sqlx-rt 0.2.0", "stringprep", "thiserror", "url 2.2.2", "whoami", ] +[[package]] +name = "sqlx-core" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec28b91a01e1fe286d6ba66f68289a2286df023fc97444e1fd86c2fd6d5dc026" +dependencies = [ + "ahash 0.7.4", + "atoi", + "base64 0.13.0", + "bitflags", + "byteorder", + "bytes 1.0.1", + "chrono", + "crc 2.1.0", + "crossbeam-channel", + "crossbeam-queue", + "crossbeam-utils 0.8.4", + "digest", + "either", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-util", + "generic-array", + "hashlink 0.7.0", + "hex", + "itoa", + "libc", + "log", + "memchr", + "num-bigint 0.3.2", + "once_cell", + "parking_lot 0.11.1", + "percent-encoding 2.1.0", + "rand 0.8.4", + "rsa 0.4.1", + "serde", + "sha-1", + "sha2", + "smallvec 1.6.1", + "sqlformat", + "sqlx-rt 0.5.9", + "stringprep", + "thiserror", + "tokio-stream", + "url 2.2.2", + "whoami", +] + [[package]] name = "sqlx-macros" version = "0.4.2" @@ -3937,8 +3814,31 @@ dependencies = [ "serde", "serde_json", "sha2", - "sqlx-core", - "sqlx-rt", + "sqlx-core 0.4.2", + "sqlx-rt 0.2.0", + "syn", + "url 2.2.2", +] + +[[package]] +name = "sqlx-macros" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc33c35d54774eed73d54568d47a6ac099aed8af5e1556a017c131be88217d5" +dependencies = [ + "dotenv", + "either", + "futures 0.3.17", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core 0.5.7", + "sqlx-rt 0.5.9", "syn", "url 2.2.2", ] @@ -3955,6 +3855,19 @@ dependencies = [ "tokio-native-tls 0.1.0", ] +[[package]] +name = "sqlx-rt" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d1bd069de53442e7a320f525a6d4deb8bb0621ac7a55f7eccbc2b58b57f43d0" +dependencies = [ + "actix-rt 2.4.0", + "native-tls", + "once_cell", + "tokio 1.13.0", + "tokio-native-tls 0.3.0", +] + [[package]] name = "standback" version = "0.2.17" @@ -4023,16 +3936,16 @@ checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" name = "storyteller-web" version = "0.0.1" dependencies = [ - "actix 0.10.0", - "actix-cors 0.5.4", - "actix-files", - "actix-http 2.2.0", + "actix", + "actix-cors 0.6.0-beta.3", + "actix-files 0.6.0-beta.8", + "actix-http 3.0.0-beta.11", "actix-multipart", "actix-rt 2.4.0", - "actix-service 1.0.6", + "actix-service 2.0.1", "actix-utils 3.0.0", - "actix-web 3.3.2", - "actix-web-actors 3.0.0", + "actix-web 4.0.0-beta.10", + "actix-web-actors", "anyhow", "base64 0.13.0", "bcrypt", @@ -4070,11 +3983,11 @@ dependencies = [ "serde_derive", "serde_json", "sha2", - "sqlx", + "sqlx 0.5.7", "subprocess", "tempdir", "time 0.2.26", - "tokio 0.2.25", + "tokio 1.13.0", "toml", "uuid", ] @@ -4428,6 +4341,17 @@ dependencies = [ "webpki", ] +[[package]] +name = "tokio-stream" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +dependencies = [ + "futures-core", + "pin-project-lite 0.2.6", + "tokio 1.13.0", +] + [[package]] name = "tokio-sync" version = "0.1.8" @@ -4505,7 +4429,6 @@ checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" dependencies = [ "bytes 0.5.6", "futures-core", - "futures-io", "futures-sink", "log", "pin-project-lite 0.1.12", @@ -4652,7 +4575,7 @@ dependencies = [ "futures 0.3.17", "http", "once_cell", - "reqwest 0.11.6", + "reqwest", "serde", "serde_derive", "serde_json", @@ -4678,7 +4601,7 @@ dependencies = [ "http", "once_cell", "rand 0.8.4", - "reqwest 0.11.6", + "reqwest", "serde", "serde_json", "thiserror", @@ -4860,16 +4783,6 @@ dependencies = [ "v_escape", ] -[[package]] -name = "value-bag" -version = "1.0.0-alpha.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd320e1520f94261153e96f7534476ad869c14022aee1e59af7c778075d840ae" -dependencies = [ - "ctor", - "version_check 0.9.3", -] - [[package]] name = "vcpkg" version = "0.2.12" @@ -4894,12 +4807,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -[[package]] -name = "waker-fn" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" - [[package]] name = "want" version = "0.3.0" @@ -4929,8 +4836,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if 1.0.0", - "serde", - "serde_json", "wasm-bindgen-macro", ] @@ -5019,15 +4924,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "wepoll-sys" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff" -dependencies = [ - "cc", -] - [[package]] name = "whoami" version = "1.1.2" @@ -5044,6 +4940,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" +[[package]] +name = "wildmatch" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6c48bd20df7e4ced539c12f570f937c6b4884928a87fee70a479d72f031d4e0" + [[package]] name = "winapi" version = "0.2.8" diff --git a/app/lib/http_server_common/Cargo.toml b/app/lib/http_server_common/Cargo.toml index 545e3e3aa3..1dc4b28a0c 100644 --- a/app/lib/http_server_common/Cargo.toml +++ b/app/lib/http_server_common/Cargo.toml @@ -22,23 +22,12 @@ dotenv = "0.15.0" log = "0.4.14" # Web -# NB: One microservice requires old Tokio (thus old actix), and another new Tokio (new actix). -# Thus I'm being extremely extra here and supporting both. -actix-cors-old = { package = "actix-cors", version = "0.5.4", optional = true } -actix-http-old = { package = "actix-http", version = "2.2.0", optional = true } -actix-web-old = { package = "actix-web", version = "3.3.2", optional = true } - -actix-cors-new = { package = "actix-cors", version = "0.6.0-beta.3", optional = true } -actix-http-new = { package = "actix-http", version = "3.0.0-beta.11", optional = true } -actix-web-new = { package = "actix-web", version = "4.0.0-beta.10", optional = true } +actix-cors = "0.6.0-beta.3" +actix-http = "3.0.0-beta.11" +actix-web = "4.0.0-beta.10" # Serialization serde = "1.0.125" serde_derive = "1.0.125" serde_json = "1.0.64" toml = "0.5.8" - -[features] -# default = ["actix-cors-new", "actix-web-new", "actix-http-new"] -actix-new = ["actix-cors-new", "actix-http-new", "actix-web-new"] -actix-old = ["actix-cors-old", "actix-http-old", "actix-web-old"] diff --git a/app/lib/http_server_common/src/lib.rs b/app/lib/http_server_common/src/lib.rs index a7e0cb7a41..44ed722aef 100644 --- a/app/lib/http_server_common/src/lib.rs +++ b/app/lib/http_server_common/src/lib.rs @@ -10,44 +10,6 @@ // I don't think this will be too hard to maintain. AFAICT, the only major differences are which // package "HttpResponseBuilder" is located in. -#[cfg(all(feature = "actix-new", not(feature = "actix-old")))] -//#[cfg(feature = "actix-cors-new")] -extern crate actix_cors_new as actix_cors; -#[cfg(all(feature = "actix-new", not(feature = "actix-old")))] -//#[cfg(feature = "actix-http-new")] -extern crate actix_http_new as actix_http; -#[cfg(all(feature = "actix-new", not(feature = "actix-old")))] -//#[cfg(feature = "actix-web-new")] -extern crate actix_web_new as actix_web; - -#[cfg(all(feature = "actix-old", not(feature = "actix-new")))] -//#[cfg(feature = "actix-cors-old")] -extern crate actix_cors_old as actix_cors; -#[cfg(all(feature = "actix-old", not(feature = "actix-new")))] -//#[cfg(feature = "actix-http-old")] -extern crate actix_http_old as actix_http; -#[cfg(all(feature = "actix-old", not(feature = "actix-new")))] -//#[cfg(feature = "actix-web-old")] -extern crate actix_web_old as actix_web; - -//#[cfg(any( -// all(feature = "actix-cors-new", not(feature = "actix-cors-old")), -// all(feature = "actix-cors-old", not(feature = "actix-cors-new")), -//))] -#[cfg(any( - feature = "actix-new", - feature = "actix-old", -))] pub mod cors; - -#[cfg(any( - feature = "actix-new", - feature = "actix-old", -))] pub mod endpoints; - -#[cfg(any( - feature = "actix-new", - feature = "actix-old", -))] pub mod response; diff --git a/app/lib/http_server_common/src/response/response_error_helpers.rs b/app/lib/http_server_common/src/response/response_error_helpers.rs index cf9bbbfccc..1bd6bf274b 100644 --- a/app/lib/http_server_common/src/response/response_error_helpers.rs +++ b/app/lib/http_server_common/src/response/response_error_helpers.rs @@ -1,12 +1,8 @@ use actix_http::http::header; +use actix_web::HttpResponseBuilder; use actix_web::http::StatusCode; use actix_web::{HttpResponse, ResponseError}; -#[cfg(feature = "actix-new")] -use actix_web::HttpResponseBuilder; - -#[cfg(not(feature = "actix-new"))] -use actix_web::dev::HttpResponseBuilder; #[derive(Serialize)] pub struct SimpleGenericJsonError { diff --git a/app/lib/http_server_common/src/response/serialize_as_json_error.rs b/app/lib/http_server_common/src/response/serialize_as_json_error.rs index 47ea5c3b2e..d288c9205e 100644 --- a/app/lib/http_server_common/src/response/serialize_as_json_error.rs +++ b/app/lib/http_server_common/src/response/serialize_as_json_error.rs @@ -1,18 +1,8 @@ use actix_http::http::{StatusCode, header}; use actix_web::HttpResponse; -use serde::Serialize; - -#[cfg(feature = "actix-new")] -use actix_web::error::ResponseError; - -#[cfg(feature = "actix-new")] use actix_web::HttpResponseBuilder; - -#[cfg(not(feature = "actix-new"))] -use actix_http::ResponseError; - -#[cfg(not(feature = "actix-new"))] -use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use serde::Serialize; /// Turn error responses into JSON HTTP responses pub fn serialize_as_json_error( diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml index bcce8433b7..8221f12a87 100644 --- a/app/storyteller_web/Cargo.toml +++ b/app/storyteller_web/Cargo.toml @@ -39,7 +39,7 @@ path = "src/w2l_inference_job.rs" # Internal config = { path = "../lib/config" } -http_server_common = { path = "../lib/http_server_common", features = ["actix-old"] } +http_server_common = { path = "../lib/http_server_common" } # External anyhow = "1.0.40" @@ -67,8 +67,8 @@ uuid = { version = "0.8.2", features = ["v4"] } #async-std = { version = "1.6", features = [ "attributes" ] } # NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: # https://github.com/durch/rust-s3/issues/198 -#tokio = { version = "1.6.0", features = ["macros"] } -tokio = { version = "0.2.25", features = ["macros"] } +tokio = { version = "1.6.0", features = ["macros"] } +#tokio = { version = "0.2.25", features = ["macros"] } # MySql Database # "offline" supports typesafe builds without a live database @@ -76,8 +76,8 @@ tokio = { version = "0.2.25", features = ["macros"] } # https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 #sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } # NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade -#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } -sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } +sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } +#sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } # Redis Database limitation = "0.1.1" # Redis rate limiter @@ -93,25 +93,39 @@ ring = "0.17.0-alpha.10" # in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2 #rust-s3 = "0.27.0-rc4" #rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } -rust-s3 = { version = "0.26" } +#rust-s3 = { version = "0.26" } +#rust-s3 = { version = "0.28", features = ["with-async-std"], default-features = false } +rust-s3 = { version = "0.28" } infer = "0.5.0" subprocess = "0.2.6" tempdir = "0.3.7" # Web -actix = "0.10" -actix-cors = "0.5.4" -actix-files= "0.5" -actix-http = "2.2.0" -actix-multipart = "0.3.0" # TODO: Ready for 0.4 upgrade -actix-rt = "2.2.0" -actix-service = "1.0.6" +#actix = "0.10" +#actix-cors = "0.5.4" +#actix-files= "0.5" +#actix-http = "2.2.0" +#actix-rt = "2.2.0" +#actix-service = "1.0.6" +#actix-utils = "3.0.0" +#actix-web = "3.3.2" +#actix-web-actors = "3" +#jwt = "0.13.0" + +actix = "0.12.0" +actix-cors = "0.6.0-beta.3" +actix-files= "0.6.0-beta.8" +actix-http = "3.0.0-beta.11" +actix-multipart = "0.4.0-beta.7" # TODO: Ready for 0.4 upgrade +actix-rt = "2.4.0" +actix-service = "2.0.1" actix-utils = "3.0.0" -actix-web = "3.3.2" -actix-web-actors = "3" +actix-web = "4.0.0-beta.10" +actix-web-actors = "4.0.0-beta.7" jwt = "0.13.0" + # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. hyper = { version = "0.13.10" } diff --git a/app/storyteller_web/src/database/mediators/badge_granter.rs b/app/storyteller_web/src/database/mediators/badge_granter.rs index 1d8bd4734b..126409c4e3 100644 --- a/app/storyteller_web/src/database/mediators/badge_granter.rs +++ b/app/storyteller_web/src/database/mediators/badge_granter.rs @@ -5,7 +5,7 @@ use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn,info}; use sqlx::error::Error::Database; -use sqlx::mysql::MySqlDone; +use sqlx::mysql::MySqlQueryResult; use sqlx::{MySqlPool}; use std::sync::Arc; @@ -314,7 +314,7 @@ SET Ok(record_id) } - fn handle_results(query_result: Result) -> AnyhowResult { + fn handle_results(query_result: Result) -> AnyhowResult { let record_id = match query_result { Ok(res) => { res.last_insert_id() diff --git a/app/storyteller_web/src/database/mediators/firehose_publisher.rs b/app/storyteller_web/src/database/mediators/firehose_publisher.rs index 850f4a8a73..d9a1860fb0 100644 --- a/app/storyteller_web/src/database/mediators/firehose_publisher.rs +++ b/app/storyteller_web/src/database/mediators/firehose_publisher.rs @@ -4,7 +4,7 @@ use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn,info}; use sqlx::error::Error::Database; -use sqlx::mysql::MySqlDone; +use sqlx::mysql::MySqlQueryResult; use sqlx::{MySqlPool}; use std::sync::Arc; @@ -207,7 +207,7 @@ SET Ok(record_id) } - fn handle_results(query_result: Result) -> AnyhowResult { + fn handle_results(query_result: Result) -> AnyhowResult { let record_id = match query_result { Ok(res) => { res.last_insert_id() diff --git a/app/storyteller_web/src/http_server/endpoints/events/list_events.rs b/app/storyteller_web/src/http_server/endpoints/events/list_events.rs index 9274684e99..1822edcdaa 100644 --- a/app/storyteller_web/src/http_server/endpoints/events/list_events.rs +++ b/app/storyteller_web/src/http_server/endpoints/events/list_events.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; diff --git a/app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs b/app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs index 2437eb477d..0b2c49fcac 100644 --- a/app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs +++ b/app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; diff --git a/app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs b/app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs index 6071621606..da5bdb88b7 100644 --- a/app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs +++ b/app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs @@ -1,4 +1,4 @@ -use actix_http::cookie::Cookie; +use actix_web::cookie::Cookie; use crate::server_state::ServerState; pub const ALPHA_COOKIE_NAME : &'static str = "enable-alpha"; diff --git a/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs b/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs index fc027e963f..c93aea0d49 100644 --- a/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs +++ b/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs @@ -1,4 +1,4 @@ -use actix_http::cookie::Cookie; +use actix_web::cookie::Cookie; use actix_http::http::StatusCode; use actix_web::web::Query; use actix_web::{HttpResponse, HttpRequest, Responder, get, web, HttpMessage}; diff --git a/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs b/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs index 35ae74a502..b07d763759 100644 --- a/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs +++ b/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs @@ -1,4 +1,4 @@ -use actix_http::cookie::Cookie; +use actix_web::cookie::Cookie; use actix_http::http::StatusCode; use actix_web::web::Query; use actix_web::{HttpResponse, HttpRequest, Responder, get, web, HttpMessage}; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs b/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs index e502cea6b9..f0a6f7b1bc 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs index 99ea56639c..c95b379039 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs index 348a922964..5de8eb49db 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs index 5ba8b6bf2e..20b174bd60 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs index ca2cebdca1..1ebea79e13 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs b/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs index 8f6641dd0f..3558c33897 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs b/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs index 433e431c69..867e88182c 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs b/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs index dc2c06880a..b8810e2d21 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs index 725cfc5c87..61fc2a2ce4 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs index 76e8ba7044..6c79346f14 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs index 81c1262c5b..17dc561285 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs index 9a43ba5afd..d79a244d8c 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs b/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs index 9b855a1981..6b8fdba86a 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs b/app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs index 31d586bf6a..a30b6e1143 100644 --- a/app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs +++ b/app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs b/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs index 8197488110..9753a57a56 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs b/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs index 6e8d1d3b37..6db246af59 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs b/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs index befe0efaab..10e4cff029 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs b/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs index cd57d8db75..4560c607d7 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 3438922477..2a3c964404 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index 79165bb12a..e82fcad088 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index 25005e28b9..8cc570619d 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs index 2f0112070c..baf749499a 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs index bd6069d498..54768c33b8 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs index 61b606cccd..f3f9d284dc 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs index ff0ce3e99d..ea5b9c97e7 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs b/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs index de5c2eb9ca..eb1066b464 100644 --- a/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs +++ b/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; diff --git a/app/storyteller_web/src/http_server/endpoints/users/create_account.rs b/app/storyteller_web/src/http_server/endpoints/users/create_account.rs index 17cabc9dfe..48db69f9d4 100644 --- a/app/storyteller_web/src/http_server/endpoints/users/create_account.rs +++ b/app/storyteller_web/src/http_server/endpoints/users/create_account.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; diff --git a/app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs b/app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs index 1b69c5e187..c31c421065 100644 --- a/app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs +++ b/app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs b/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs index e88652970d..e3f464a7bb 100644 --- a/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs +++ b/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs index c632a6473f..4b1a64dc5e 100644 --- a/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Query}; diff --git a/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs b/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs index 95ea806ad6..e7de6a9d50 100644 --- a/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs +++ b/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs index 52dc38c569..21e032853d 100644 --- a/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs +++ b/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Query}; diff --git a/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs b/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs index 76300e85ca..071e91af1b 100644 --- a/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs +++ b/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/users/login.rs b/app/storyteller_web/src/http_server/endpoints/users/login.rs index 2079b4648d..a348294866 100644 --- a/app/storyteller_web/src/http_server/endpoints/users/login.rs +++ b/app/storyteller_web/src/http_server/endpoints/users/login.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; diff --git a/app/storyteller_web/src/http_server/endpoints/users/logout.rs b/app/storyteller_web/src/http_server/endpoints/users/logout.rs index c7f289301d..f892574399 100644 --- a/app/storyteller_web/src/http_server/endpoints/users/logout.rs +++ b/app/storyteller_web/src/http_server/endpoints/users/logout.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; diff --git a/app/storyteller_web/src/http_server/endpoints/users/session_info.rs b/app/storyteller_web/src/http_server/endpoints/users/session_info.rs index 8324b7bf0a..b13a984319 100644 --- a/app/storyteller_web/src/http_server/endpoints/users/session_info.rs +++ b/app/storyteller_web/src/http_server/endpoints/users/session_info.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs b/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs index 7864714374..8eba2b589b 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs b/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs index bd4f9c2017..cc0fbcea19 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs b/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs index cbda727582..f9f45cf103 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs b/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs index 5d5929e2b7..1777c3c5ee 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index c0c25714f0..1597201851 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 68709a3940..0e9e63b3e5 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -1,7 +1,7 @@ use actix_http::http::header; use actix_multipart::{Multipart, Field}; use actix_web::body::ResponseBody::Body; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::http::StatusCode; use actix_web::http::header::ContentDisposition; use actix_web::web::{Data, Json, BytesMut}; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index 681e9f32b1..54bfac71e0 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs index afde7fa1cb..5997d0856a 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs index 1313a8fd2f..f5b19d8eff 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs index c9bf1767dd..22a72c26a6 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs index 79583c67fa..909022b12b 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs index e1fc4dfaf4..c3f26bf6b3 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs b/app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs index c4d8ccbfdf..83b087acdf 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs index d2612a1d6d..8c356af375 100644 --- a/app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs +++ b/app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; use actix_web::cookie::Cookie; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; diff --git a/app/storyteller_web/src/http_server/middleware/mod.rs b/app/storyteller_web/src/http_server/middleware/mod.rs index a2369086a3..f7dcd4aaa3 100644 --- a/app/storyteller_web/src/http_server/middleware/mod.rs +++ b/app/storyteller_web/src/http_server/middleware/mod.rs @@ -1 +1 @@ -pub mod ip_filter_middleware; \ No newline at end of file +//pub mod ip_filter_middleware; // TODO(FIXME): Bad upgrade \ No newline at end of file diff --git a/app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs b/app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs index 2977eb183d..e950731d73 100644 --- a/app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs +++ b/app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs @@ -1,5 +1,5 @@ use actix_http::http::header; -use actix_web::dev::HttpResponseBuilder; +use actix_web::HttpResponseBuilder; use actix_web::http::StatusCode; use actix_web::{HttpResponse, ResponseError}; diff --git a/app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs b/app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs index 7a36a6d595..d288c9205e 100644 --- a/app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs +++ b/app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs @@ -1,8 +1,8 @@ use actix_http::http::{StatusCode, header}; use actix_web::HttpResponse; +use actix_web::HttpResponseBuilder; +use actix_web::error::ResponseError; use serde::Serialize; -use actix_http::ResponseError; -use actix_web::dev::HttpResponseBuilder; /// Turn error responses into JSON HTTP responses pub fn serialize_as_json_error( diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index 0636e91fcc..be3ba88a91 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -90,7 +90,7 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; -use crate::http_server::middleware::ip_filter_middleware::IpFilter; +//use crate::http_server::middleware::ip_filter_middleware::IpFilter; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; @@ -325,7 +325,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .wrap(DefaultHeaders::new() .header("X-Backend-Hostname", &hostname) .header("X-Build-Sha", "")) - .wrap(IpFilter::new(ip_banlist)) + //.wrap(IpFilter::new(ip_banlist)) // TODO(FIXME): Bad upgrade // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== .service( web::resource("/create_account") diff --git a/app/storyteller_web/src/util/buckets/bucket_client.rs b/app/storyteller_web/src/util/buckets/bucket_client.rs index a137184d5a..72ff1b2bf1 100644 --- a/app/storyteller_web/src/util/buckets/bucket_client.rs +++ b/app/storyteller_web/src/util/buckets/bucket_client.rs @@ -29,13 +29,21 @@ impl BucketClient { bucket_name: &str, optional_bucket_root: Option<&str>) -> anyhow::Result { - let credentials = Credentials::new( + /*let credentials = Credentials::new( Some(&access_key), Some(&secret_key), None, None, None, - )?; + )?;*/ + + let credentials = Credentials { + access_key: Some(access_key.to_string()), + secret_key: Some(secret_key.to_string()), + security_token: None, + session_token: None, + }; + // NB: The GCS buckets aren't supported by default. let region = Region::Custom { region: region_name.to_owned(), @@ -184,7 +192,9 @@ impl BucketClient { info!("downloading from bucket: {:?}", &object_path_str); - let mut output_file = std::fs::File::create(filesystem_path)?; + //let mut output_file = std::fs::File::create(filesystem_path)?; + //let mut output_file = async_std::fs::File::create(filesystem_path).await?; + let mut output_file = tokio::fs::File::create(filesystem_path).await?; let status_code = self.bucket.get_object_stream(&object_path_str, &mut output_file).await?; From a26e11a294bc8b867fd1f5d50a0499710c7a9d71 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 01:14:29 -0500 Subject: [PATCH 0660/1042] BROKEN: OBS gateway now runs on same actix and shared code --- Cargo.lock | 1 + app/obs_gateway/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index e20b79a808..eee8c145de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2645,6 +2645,7 @@ dependencies = [ "hex", "hmac 0.10.1", "hostname", + "http_server_common", "hyper 0.13.10", "jwt", "lazy_static", diff --git a/app/obs_gateway/Cargo.toml b/app/obs_gateway/Cargo.toml index e97f34511d..8303bb4735 100644 --- a/app/obs_gateway/Cargo.toml +++ b/app/obs_gateway/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main.rs" # Internal config = { path = "../lib/config" } container_common = { path = "../lib/container_common" } -#http_server_common = { path = "../lib/http_server_common", features = ["actix-new"] } +http_server_common = { path = "../lib/http_server_common" } # External anyhow = "1.0.40" From 30c26647dab253cd4bd51830910f20bbef42c753 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 03:01:37 -0500 Subject: [PATCH 0661/1042] rust-s3 "never-encode-slash" feature seems to do the trick! --- Cargo.lock | 2 +- app/storyteller_web/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eee8c145de..1c906d085d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3963,7 +3963,7 @@ dependencies = [ "hmac 0.10.1", "hostname", "http_server_common", - "hyper 0.13.10", + "hyper 0.14.14", "infer", "jwt", "lazy_static", diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml index 8221f12a87..d3fe11ddd7 100644 --- a/app/storyteller_web/Cargo.toml +++ b/app/storyteller_web/Cargo.toml @@ -95,7 +95,7 @@ ring = "0.17.0-alpha.10" #rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } #rust-s3 = { version = "0.26" } #rust-s3 = { version = "0.28", features = ["with-async-std"], default-features = false } -rust-s3 = { version = "0.28" } +rust-s3 = { version = "0.28", features = ["never-encode-slash"] } infer = "0.5.0" subprocess = "0.2.6" @@ -128,7 +128,7 @@ jwt = "0.13.0" # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. -hyper = { version = "0.13.10" } +hyper = { version = "0.14.14" } # Crypto bcrypt = "0.8" # password hashes From 2b3fdb87e0de2c2d987b50605165a993112c9816 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 03:30:15 -0500 Subject: [PATCH 0662/1042] BROKEN: IP filter compiles, but does not block --- .../middleware/ip_filter_middleware.rs | 53 ++++++++----------- .../src/http_server/middleware/mod.rs | 2 +- app/storyteller_web/src/main.rs | 4 +- models/tts/vocodes_common.py | 30 +++++------ models/tts/vocodes_server.py | 52 +++++++++++------- 5 files changed, 72 insertions(+), 69 deletions(-) diff --git a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs index 5b90e6f349..26e63be4cd 100644 --- a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs +++ b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs @@ -1,15 +1,15 @@ +use actix_service::{Service, Transform}; +use actix_web::error::ErrorForbidden; +use actix_web::{dev::ServiceRequest, dev::ServiceResponse, Error}; +use crate::http_server::web_utils::ip_address::get_service_request_ip; +use crate::threads::ip_banlist_set::IpBanlistSet; +use futures::Future; +use futures::future::{ok, Ready, LocalBoxFuture}; +use log::info; +use log::warn; use std::pin::Pin; use std::task::{Context, Poll}; - -use actix_service::{Service, Transform}; -use actix_web::{dev::ServiceRequest, dev::ServiceResponse, Error}; -use futures::future::{ok, Ready}; -use futures::Future; -use crate::threads::ip_banlist_set::IpBanlistSet; -use log::warn; -use log::info; -use crate::http_server::web_utils::ip_address::get_service_request_ip; -use actix_web::error::ErrorForbidden; +use actix_utils::future::Either; // There are two steps in middleware processing. // 1. Middleware initialization, middleware factory gets called with @@ -31,17 +31,15 @@ impl IpFilter { // Middleware factory is `Transform` trait from actix-service crate // `S` - type of the next service // `B` - type of response's body -impl Transform for IpFilter +impl Transform for IpFilter where - S: Service, Error = Error>, + S: Service, Error = Error>, S::Future: 'static, - B: 'static, { - type Request = ServiceRequest; type Response = ServiceResponse; type Error = Error; - type InitError = (); type Transform = IpFilterMiddleware; + type InitError = (); type Future = Ready>; fn new_transform(&self, service: S) -> Self::Future { @@ -55,22 +53,18 @@ pub struct IpFilterMiddleware { ip_banlist: IpBanlistSet, } -impl Service for IpFilterMiddleware +impl Service for IpFilterMiddleware where - S: Service, Error = Error>, + S: Service, Error = Error>, S::Future: 'static, - B: 'static, { - type Request = ServiceRequest; type Response = ServiceResponse; type Error = Error; - type Future = Pin>>>; + type Future = S::Future; - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.service.poll_ready(cx) - } + actix_service::forward_ready!(service); - fn call(&mut self, req: ServiceRequest) -> Self::Future { + fn call(&self, req: ServiceRequest) -> Self::Future { let ip_address = get_service_request_ip(&req); // NB: Fail open. @@ -78,14 +72,11 @@ impl Service for IpFilterMiddleware if is_banned { warn!("Ip is banned: {}", &ip_address); - return Box::pin(ok(req.error_response(ErrorForbidden("Forbidden")))) + //return Box::pin(ok(req.error_response(ErrorForbidden("Forbidden")))) + //return req.error_response(ErrorForbidden("Forbidden")) + // TODO: RESTORE BLOCK } - // Call endpoint and return results - let fut = self.service.call(req); - Box::pin(async move { - let res = fut.await?; - Ok(res) - }) + self.service.call(req) } } \ No newline at end of file diff --git a/app/storyteller_web/src/http_server/middleware/mod.rs b/app/storyteller_web/src/http_server/middleware/mod.rs index f7dcd4aaa3..a2369086a3 100644 --- a/app/storyteller_web/src/http_server/middleware/mod.rs +++ b/app/storyteller_web/src/http_server/middleware/mod.rs @@ -1 +1 @@ -//pub mod ip_filter_middleware; // TODO(FIXME): Bad upgrade \ No newline at end of file +pub mod ip_filter_middleware; \ No newline at end of file diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index be3ba88a91..0636e91fcc 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -90,7 +90,7 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; -//use crate::http_server::middleware::ip_filter_middleware::IpFilter; +use crate::http_server::middleware::ip_filter_middleware::IpFilter; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; @@ -325,7 +325,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .wrap(DefaultHeaders::new() .header("X-Backend-Hostname", &hostname) .header("X-Build-Sha", "")) - //.wrap(IpFilter::new(ip_banlist)) // TODO(FIXME): Bad upgrade + .wrap(IpFilter::new(ip_banlist)) // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== .service( web::resource("/create_account") diff --git a/models/tts/vocodes_common.py b/models/tts/vocodes_common.py index 4b2160831b..0f740a4fce 100644 --- a/models/tts/vocodes_common.py +++ b/models/tts/vocodes_common.py @@ -3,26 +3,26 @@ sys.path.append('waveglow/') import shutil import json -import numpy as np -import torch +#import numpy as np +#import torch -from tacotron2.hparams import create_hparams -from tacotron2.model import Tacotron2 -from tacotron2.layers import TacotronSTFT -from tacotron2.audio_processing import griffin_lim -from tacotron2.text import text_to_sequence +#from tacotron2.hparams import create_hparams +#from tacotron2.model import Tacotron2 +#from tacotron2.layers import TacotronSTFT +#from tacotron2.audio_processing import griffin_lim +#from tacotron2.text import text_to_sequence -from waveglow.denoiser import Denoiser +#from waveglow.denoiser import Denoiser -from scipy.io.wavfile import write as write_wav +#from scipy.io.wavfile import write as write_wav # Hifi-Gan -from hifigan.env import AttrDict -from hifigan.models import Generator -from hifigan.denoiser import Denoiser as HifiDenoiser -from hifigan.meldataset import mel_spectrogram -import resampy -import scipy +#from hifigan.env import AttrDict +#from hifigan.models import Generator +#from hifigan.denoiser import Denoiser as HifiDenoiser +#from hifigan.meldataset import mel_spectrogram +#import resampy +#import scipy # For metadata import subprocess diff --git a/models/tts/vocodes_server.py b/models/tts/vocodes_server.py index e940b9b240..6dde8583c5 100755 --- a/models/tts/vocodes_server.py +++ b/models/tts/vocodes_server.py @@ -9,30 +9,31 @@ Brandon Thomas 2019-07-29 import argparse import falcon import io -import librosa -import numpy as np +#import librosa +#import numpy as np import os -import scipy -import soundfile -import tensorflow as tf +#import scipy +#import soundfile +#import tensorflow as tf import subprocess import tempfile import json +import shutil from falcon_multipart.middleware import MultipartMiddleware #from model import CycleGAN #from preprocess import * from wsgiref import simple_server -from vocodes_common import TacotronWaveglowPipeline -from vocodes_common import print_gpu_info -from vocodes_common import load_tacotron_model -from vocodes_common import load_waveglow_model -from vocodes_common import load_hifigan_model +#from vocodes_common import TacotronWaveglowPipeline +#from vocodes_common import print_gpu_info +#from vocodes_common import load_tacotron_model +#from vocodes_common import load_waveglow_model +#from vocodes_common import load_hifigan_model -print("TensorFlow version: {}".format(tf.version.VERSION)) +#print("TensorFlow version: {}".format(tf.version.VERSION)) -print_gpu_info() +#print_gpu_info() INDEX_HTML = ''' @@ -197,7 +198,7 @@ INDEX_HTML = ''' #tacotron = load_tacotron_model('/home/bt/models/tacotron2/tacotron2_uberduck_Noire.pt') #waveglow = load_waveglow_model('/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt') -pipeline = TacotronWaveglowPipeline() +#pipeline = TacotronWaveglowPipeline() class IndexHandler(): def on_get(self, request, response): @@ -235,14 +236,14 @@ class ApiHandler(): print('maybe_clear_synthesizer_checkpoint_path: {}'.format( maybe_clear_synthesizer_checkpoint_path)) - pipeline.maybe_load_waveglow_model(waveglow_vocoder_checkpoint_path) - pipeline.maybe_load_hifigan(hifigan_vocoder_checkpoint_path) - pipeline.maybe_load_hifigan_super_resolution(hifigan_superres_vocoder_checkpoint_path) + #pipeline.maybe_load_waveglow_model(waveglow_vocoder_checkpoint_path) + #pipeline.maybe_load_hifigan(hifigan_vocoder_checkpoint_path) + #pipeline.maybe_load_hifigan_super_resolution(hifigan_superres_vocoder_checkpoint_path) - if maybe_clear_synthesizer_checkpoint_path: - pipeline.uncache_tacotron_model(maybe_clear_synthesizer_checkpoint_path) + #if maybe_clear_synthesizer_checkpoint_path: + # pipeline.uncache_tacotron_model(maybe_clear_synthesizer_checkpoint_path) - pipeline.maybe_load_tacotron_model_to_cache(synthesizer_checkpoint_path) + #pipeline.maybe_load_tacotron_model_to_cache(synthesizer_checkpoint_path) inference_args = { 'raw_text': inference_text, @@ -253,7 +254,18 @@ class ApiHandler(): 'output_metadata_filename': output_metadata_filename, } - pipeline.infer(inference_args) + #pipeline.infer(inference_args) + + metadata_temp = "/Users/bt/dev/storyteller/storyteller-web/temp/metadata.json" + audio_temp = "/Users/bt/dev/storyteller/storyteller-web/temp/test.wav" + spectrogram_temp = "/Users/bt/dev/storyteller/storyteller-web/temp/spectrogram.json" + + args = inference_args + + print('TEMP COPYING FAKE FILES...') + shutil.copyfile(metadata_temp, args['output_metadata_filename']) + shutil.copyfile(audio_temp, args['output_audio_filename']) + shutil.copyfile(spectrogram_temp, args['output_spectrogram_filename']) def main(): parser = argparse.ArgumentParser() From f2f39685a953ee2c3b890738e5e3fba4a89c2154 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 04:36:50 -0500 Subject: [PATCH 0663/1042] WIP filter (not working) --- .../middleware/ip_filter_middleware.rs | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs index 26e63be4cd..f75bcd4e9f 100644 --- a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs +++ b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs @@ -1,15 +1,16 @@ use actix_service::{Service, Transform}; use actix_web::error::ErrorForbidden; -use actix_web::{dev::ServiceRequest, dev::ServiceResponse, Error}; +use actix_web::{dev::ServiceRequest, dev::ServiceResponse, Error, HttpResponse}; use crate::http_server::web_utils::ip_address::get_service_request_ip; use crate::threads::ip_banlist_set::IpBanlistSet; use futures::Future; -use futures::future::{ok, Ready, LocalBoxFuture}; +use futures::future::{ok, Ready, LocalBoxFuture, err, Either}; use log::info; use log::warn; use std::pin::Pin; use std::task::{Context, Poll}; -use actix_utils::future::Either; +use actix_web::body::AnyBody; +use actix_http::http; // There are two steps in middleware processing. // 1. Middleware initialization, middleware factory gets called with @@ -53,6 +54,8 @@ pub struct IpFilterMiddleware { ip_banlist: IpBanlistSet, } +// Updated example from +// - https://github.com/actix/examples/blob/master/basics/middleware/src/redirect.rs impl Service for IpFilterMiddleware where S: Service, Error = Error>, @@ -60,7 +63,12 @@ impl Service for IpFilterMiddleware { type Response = ServiceResponse; type Error = Error; - type Future = S::Future; + //type Future = Either>>>>; + //type Future = Pin>>>; + type Future = Pin>>>; + //type Future = LocalBoxFuture<'static, Result>; + //type Future = Either>>; actix_service::forward_ready!(service); @@ -72,11 +80,40 @@ impl Service for IpFilterMiddleware if is_banned { warn!("Ip is banned: {}", &ip_address); - //return Box::pin(ok(req.error_response(ErrorForbidden("Forbidden")))) + //return Either::right( + // Box::pin(ok(req.error_response(ErrorForbidden("Forbidden"))))) + + + //return Box::pin(ok(req.error_response(ErrorForbidden("Forbidden")))); + //return req.error_response(ErrorForbidden("Forbidden")) // TODO: RESTORE BLOCK + //return Box::pin(AnyBody::from_message()); + //return Box::pin(req.error_response(ErrorForbidden("Forbidden"))); + //return Box::pin(AnyBody::from_message("")); + + + //return Either::Right(ok(req.into_response( + // HttpResponse::Found() + // .finish() + // .into_body(), + //))); + + //return Either::Right(ok(req.error_response( + // ErrorForbidden("") + //))); + + //return Box::pin(async move { + // let res = req.into_response( + // HttpResponse::Unauthorized() + // .finish() + // ); + // Ok(res) + //}) + } - self.service.call(req) + Box::pin(self.service.call(req)) + //Either::Left(self.service.call(req)) } } \ No newline at end of file From 5c9da5fae502693108e6789d5b97bc743bb8f19c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 18:12:13 -0500 Subject: [PATCH 0664/1042] Middleware not compiling, see: https://stackoverflow.com/questions/68944823/returning-an-unauthorized-response-in-actix-web-middleware-in-rust --- .../middleware/ip_filter_middleware.rs | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs index f75bcd4e9f..ad92cd9165 100644 --- a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs +++ b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs @@ -9,7 +9,7 @@ use log::info; use log::warn; use std::pin::Pin; use std::task::{Context, Poll}; -use actix_web::body::AnyBody; +use actix_web::body::{AnyBody, MessageBody}; use actix_http::http; // There are two steps in middleware processing. @@ -36,6 +36,7 @@ impl Transform for IpFilter where S: Service, Error = Error>, S::Future: 'static, + B: 'static, { type Response = ServiceResponse; type Error = Error; @@ -56,10 +57,12 @@ pub struct IpFilterMiddleware { // Updated example from // - https://github.com/actix/examples/blob/master/basics/middleware/src/redirect.rs +// - https://stackoverflow.com/questions/68944823/returning-an-unauthorized-response-in-actix-web-middleware-in-rust impl Service for IpFilterMiddleware where S: Service, Error = Error>, S::Future: 'static, + B: 'static, { type Response = ServiceResponse; type Error = Error; @@ -72,6 +75,16 @@ impl Service for IpFilterMiddleware actix_service::forward_ready!(service); + /* + + Future + + Future = Pin>> + + RESPONSE = ServiceResponse + + + */ fn call(&self, req: ServiceRequest) -> Self::Future { let ip_address = get_service_request_ip(&req); @@ -103,13 +116,13 @@ impl Service for IpFilterMiddleware // ErrorForbidden("") //))); - //return Box::pin(async move { - // let res = req.into_response( - // HttpResponse::Unauthorized() - // .finish() - // ); - // Ok(res) - //}) + return Box::pin(async move { + let res = req.into_response( + HttpResponse::Unauthorized() + .finish() + ); + Ok(res) + }) } From 38cc8f31a814c1864bb1e55a312d3b5f81430baf Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 19:46:57 -0500 Subject: [PATCH 0665/1042] NOT WORKING (but close?) --- Cargo.lock | 1 + app/lib/http_server_common/Cargo.toml | 3 +- app/storyteller_web/Cargo.toml | 3 +- .../middleware/ip_filter_middleware.rs | 113 ++++++------------ 4 files changed, 42 insertions(+), 78 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c906d085d..7f66a5dba5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3959,6 +3959,7 @@ dependencies = [ "easyenv", "futures 0.1.31", "futures 0.3.17", + "futures-util", "hex", "hmac 0.10.1", "hostname", diff --git a/app/lib/http_server_common/Cargo.toml b/app/lib/http_server_common/Cargo.toml index 1dc4b28a0c..19ac74b6aa 100644 --- a/app/lib/http_server_common/Cargo.toml +++ b/app/lib/http_server_common/Cargo.toml @@ -24,7 +24,8 @@ log = "0.4.14" # Web actix-cors = "0.6.0-beta.3" actix-http = "3.0.0-beta.11" -actix-web = "4.0.0-beta.10" +#actix-web = "4.0.0-beta.10" +actix-web = "4.0.0-beta.7" # Serialization serde = "1.0.125" diff --git a/app/storyteller_web/Cargo.toml b/app/storyteller_web/Cargo.toml index d3fe11ddd7..74519dc69d 100644 --- a/app/storyteller_web/Cargo.toml +++ b/app/storyteller_web/Cargo.toml @@ -50,6 +50,7 @@ dotenv = "0.15.0" easyenv = "0.0.6" futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate +futures-util = "0.3.9" hex = "0.4.3" hostname = "0.3.1" lazy_static = "1.4.0" @@ -121,7 +122,7 @@ actix-multipart = "0.4.0-beta.7" # TODO: Ready for 0.4 upgrade actix-rt = "2.4.0" actix-service = "2.0.1" actix-utils = "3.0.0" -actix-web = "4.0.0-beta.10" +actix-web = "4.0.0-beta.7" actix-web-actors = "4.0.0-beta.7" jwt = "0.13.0" diff --git a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs index ad92cd9165..b78c91ffba 100644 --- a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs +++ b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs @@ -1,16 +1,29 @@ -use actix_service::{Service, Transform}; -use actix_web::error::ErrorForbidden; -use actix_web::{dev::ServiceRequest, dev::ServiceResponse, Error, HttpResponse}; use crate::http_server::web_utils::ip_address::get_service_request_ip; use crate::threads::ip_banlist_set::IpBanlistSet; -use futures::Future; -use futures::future::{ok, Ready, LocalBoxFuture, err, Either}; -use log::info; -use log::warn; -use std::pin::Pin; + +//use actix_web::error::ErrorForbidden; +//use actix_web::{HttpResponse, ResponseError}; +//use anyhow::anyhow; +//use futures::Future; +//use futures::future::{ok, Ready, LocalBoxFuture, err, Either}; +//use log::info; +//use log::warn; +//use std::pin::Pin; +//use std::task::{Context, Poll}; +//use actix_web::body::{AnyBody, MessageBody}; +//use actix_http::{http, StatusCode, header, Response}; +//use actix_http::body::{Body, ResponseBody}; +//use actix_web::web::{BytesMut, Buf, BufMut}; +//use std::fmt; +//use std::{io::Write as _}; +//use futures::future::FutureExt; + use std::task::{Context, Poll}; -use actix_web::body::{AnyBody, MessageBody}; -use actix_http::http; + +use actix_web::dev::{Service, Transform}; +use actix_web::dev::{ServiceRequest, ServiceResponse}; +use actix_web::Error; +use futures_util::future::{err, ok, Either, Ready}; // There are two steps in middleware processing. // 1. Middleware initialization, middleware factory gets called with @@ -32,16 +45,16 @@ impl IpFilter { // Middleware factory is `Transform` trait from actix-service crate // `S` - type of the next service // `B` - type of response's body -impl Transform for IpFilter + +impl Transform for IpFilter where - S: Service, Error = Error>, + S: Service, S::Future: 'static, - B: 'static, { - type Response = ServiceResponse; + type Response = ServiceResponse; type Error = Error; - type Transform = IpFilterMiddleware; type InitError = (); + type Transform = IpFilterMiddleware; type Future = Ready>; fn new_transform(&self, service: S) -> Self::Future { @@ -55,78 +68,26 @@ pub struct IpFilterMiddleware { ip_banlist: IpBanlistSet, } -// Updated example from -// - https://github.com/actix/examples/blob/master/basics/middleware/src/redirect.rs -// - https://stackoverflow.com/questions/68944823/returning-an-unauthorized-response-in-actix-web-middleware-in-rust -impl Service for IpFilterMiddleware +impl Service for IpFilterMiddleware where - S: Service, Error = Error>, + S: Service, S::Future: 'static, - B: 'static, { - type Response = ServiceResponse; + type Response = ServiceResponse; type Error = Error; - //type Future = Either>>>>; - //type Future = Pin>>>; - type Future = Pin>>>; - //type Future = LocalBoxFuture<'static, Result>; - //type Future = Either>>; + type Future = Either>>; - actix_service::forward_ready!(service); + //actix_service::forward_ready!(service); + fn poll_ready(&self, cx: &mut Context) -> Poll> { + self.service.poll_ready(cx) + } - /* - - Future - - Future = Pin>> - - RESPONSE = ServiceResponse - - - */ fn call(&self, req: ServiceRequest) -> Self::Future { let ip_address = get_service_request_ip(&req); // NB: Fail open. let is_banned = self.ip_banlist.is_banned(&ip_address).unwrap_or(false); - if is_banned { - warn!("Ip is banned: {}", &ip_address); - //return Either::right( - // Box::pin(ok(req.error_response(ErrorForbidden("Forbidden"))))) - - - //return Box::pin(ok(req.error_response(ErrorForbidden("Forbidden")))); - - //return req.error_response(ErrorForbidden("Forbidden")) - // TODO: RESTORE BLOCK - //return Box::pin(AnyBody::from_message()); - //return Box::pin(req.error_response(ErrorForbidden("Forbidden"))); - //return Box::pin(AnyBody::from_message("")); - - - //return Either::Right(ok(req.into_response( - // HttpResponse::Found() - // .finish() - // .into_body(), - //))); - - //return Either::Right(ok(req.error_response( - // ErrorForbidden("") - //))); - - return Box::pin(async move { - let res = req.into_response( - HttpResponse::Unauthorized() - .finish() - ); - Ok(res) - }) - - } - - Box::pin(self.service.call(req)) - //Either::Left(self.service.call(req)) + Either::Left(self.service.call(req)) } } \ No newline at end of file From 6849e6d051d4063d2e2f2ee577809b11d1606a80 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 19:52:00 -0500 Subject: [PATCH 0666/1042] HOLY SHIT, MIDDLEWARE ORDER MATTERED https://issueexplorer.com/issue/actix/actix-extras/191 --- app/storyteller_web/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/storyteller_web/src/main.rs b/app/storyteller_web/src/main.rs index 0636e91fcc..39a8e3927b 100644 --- a/app/storyteller_web/src/main.rs +++ b/app/storyteller_web/src/main.rs @@ -319,13 +319,13 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> App::new() .app_data(server_state_arc.clone()) .wrap(build_common_cors_config()) - .wrap(Logger::new(&log_format) - .exclude("/liveness") - .exclude("/readiness")) .wrap(DefaultHeaders::new() .header("X-Backend-Hostname", &hostname) .header("X-Build-Sha", "")) .wrap(IpFilter::new(ip_banlist)) + .wrap(Logger::new(&log_format) + .exclude("/liveness") + .exclude("/readiness")) // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== .service( web::resource("/create_account") From 4aa9724d41eb9a098b84bb472c37909ca9410b91 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 20:05:54 -0500 Subject: [PATCH 0667/1042] BROKEN, BUT - filter works !! Needs cleanup and correct logic --- .../middleware/ip_filter_middleware.rs | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs index b78c91ffba..015e6b8d14 100644 --- a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs +++ b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs @@ -22,9 +22,50 @@ use std::task::{Context, Poll}; use actix_web::dev::{Service, Transform}; use actix_web::dev::{ServiceRequest, ServiceResponse}; -use actix_web::Error; +use actix_web::{Error, HttpResponse}; use futures_util::future::{err, ok, Either, Ready}; +use derive_more::{Display, Error as ErrorE}; +use actix_web::{ResponseError, HttpMessage, HttpRequest, HttpResponseBuilder}; +use actix_http::StatusCode; +use actix_http::{error, body::Body, Response}; +use actix_web::web::{BytesMut, Buf, BufMut}; +use actix_http::http::{header, HeaderMap, HeaderValue}; +use std::io::Write; + +#[derive(Debug, Display, ErrorE)] +pub struct BannedError { +} + +impl ResponseError for BannedError { + fn status_code(&self) -> StatusCode { + StatusCode::UNAUTHORIZED + } + fn error_response(&self) -> HttpResponse { + // let mut res = Response::new(self.status_code()); + // res.headers_mut().insert( + // header::CONTENT_TYPE, + // header::HeaderValue::from_static("application/json"), + // ); + // res.set_body(Body::from(self.to_string())).into() + /*let mut res = Response::new(self.status_code()); + let mut buf = BytesMut::new().writer(); + let _ = write!(buf, "{}", self); + + res.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/json"), + ); + res.set_body(Body::from(buf.into_inner())).into()*/ + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body("TODO") + } +} + + + // There are two steps in middleware processing. // 1. Middleware initialization, middleware factory gets called with // next service in chain as parameter. @@ -88,6 +129,15 @@ impl Service for IpFilterMiddleware // NB: Fail open. let is_banned = self.ip_banlist.is_banned(&ip_address).unwrap_or(false); + if is_banned { + let res = req.into_response( + HttpResponse::Unauthorized() + .finish()); + //return Either::Right(err(Error::from(error))); + //return Err(Error::from(UserError::ValidationError { field: "token invalid".to_string() })); + return Either::Right(err(Error::from(BannedError {}))) + } + Either::Left(self.service.call(req)) } } \ No newline at end of file From 5ce4bd3b37881c897fe90a13da9bcbaa065d0062 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 23:35:43 -0500 Subject: [PATCH 0668/1042] Cleanup IP filter --- .../middleware/ip_filter_middleware.rs | 90 +++++-------------- 1 file changed, 23 insertions(+), 67 deletions(-) diff --git a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs index 015e6b8d14..5e88ca0301 100644 --- a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs +++ b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs @@ -1,75 +1,38 @@ -use crate::http_server::web_utils::ip_address::get_service_request_ip; -use crate::threads::ip_banlist_set::IpBanlistSet; - -//use actix_web::error::ErrorForbidden; -//use actix_web::{HttpResponse, ResponseError}; -//use anyhow::anyhow; -//use futures::Future; -//use futures::future::{ok, Ready, LocalBoxFuture, err, Either}; -//use log::info; -//use log::warn; -//use std::pin::Pin; -//use std::task::{Context, Poll}; -//use actix_web::body::{AnyBody, MessageBody}; -//use actix_http::{http, StatusCode, header, Response}; -//use actix_http::body::{Body, ResponseBody}; -//use actix_web::web::{BytesMut, Buf, BufMut}; -//use std::fmt; -//use std::{io::Write as _}; -//use futures::future::FutureExt; - -use std::task::{Context, Poll}; +// There are two steps in middleware processing. +// 1. Middleware initialization, middleware factory gets called with +// next service in chain as parameter. +// 2. Middleware's call method gets called with normal request. +use actix_http::StatusCode; +use actix_http::http::{header, HeaderMap, HeaderValue}; +use actix_http::{error, body::Body, Response}; use actix_web::dev::{Service, Transform}; use actix_web::dev::{ServiceRequest, ServiceResponse}; -use actix_web::{Error, HttpResponse}; -use futures_util::future::{err, ok, Either, Ready}; - -use derive_more::{Display, Error as ErrorE}; -use actix_web::{ResponseError, HttpMessage, HttpRequest, HttpResponseBuilder}; -use actix_http::StatusCode; -use actix_http::{error, body::Body, Response}; use actix_web::web::{BytesMut, Buf, BufMut}; -use actix_http::http::{header, HeaderMap, HeaderValue}; +use actix_web::{Error, HttpResponse}; +use actix_web::{ResponseError, HttpMessage, HttpRequest, HttpResponseBuilder}; +use crate::http_server::web_utils::ip_address::get_service_request_ip; +use crate::threads::ip_banlist_set::IpBanlistSet; +use derive_more::{Display, Error as ErrorE}; +use futures_util::future::{err, ok, Either, Ready}; use std::io::Write; +use std::task::{Context, Poll}; #[derive(Debug, Display, ErrorE)] -pub struct BannedError { -} +pub struct BannedError {} impl ResponseError for BannedError { fn status_code(&self) -> StatusCode { StatusCode::UNAUTHORIZED } + fn error_response(&self) -> HttpResponse { - // let mut res = Response::new(self.status_code()); - // res.headers_mut().insert( - // header::CONTENT_TYPE, - // header::HeaderValue::from_static("application/json"), - // ); - // res.set_body(Body::from(self.to_string())).into() - /*let mut res = Response::new(self.status_code()); - let mut buf = BytesMut::new().writer(); - let _ = write!(buf, "{}", self); - - res.headers_mut().insert( - header::CONTENT_TYPE, - header::HeaderValue::from_static("application/json"), - ); - res.set_body(Body::from(buf.into_inner())).into()*/ - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body("TODO") + .append_header((header::CONTENT_TYPE, "application/json")) + .body("{}") } } - - -// There are two steps in middleware processing. -// 1. Middleware initialization, middleware factory gets called with -// next service in chain as parameter. -// 2. Middleware's call method gets called with normal request. #[derive(Clone)] // NB: Clone is safe because IpBanlist is clone-safe (internal Arc) pub struct IpFilter { ip_banlist: IpBanlistSet, @@ -83,10 +46,6 @@ impl IpFilter { } } -// Middleware factory is `Transform` trait from actix-service crate -// `S` - type of the next service -// `B` - type of response's body - impl Transform for IpFilter where S: Service, @@ -118,7 +77,7 @@ impl Service for IpFilterMiddleware type Error = Error; type Future = Either>>; - //actix_service::forward_ready!(service); + // alternatively(?), actix_service::forward_ready!(service); fn poll_ready(&self, cx: &mut Context) -> Poll> { self.service.poll_ready(cx) } @@ -127,17 +86,14 @@ impl Service for IpFilterMiddleware let ip_address = get_service_request_ip(&req); // NB: Fail open. - let is_banned = self.ip_banlist.is_banned(&ip_address).unwrap_or(false); + let is_banned = self.ip_banlist + .is_banned(&ip_address) + .unwrap_or(false); if is_banned { - let res = req.into_response( - HttpResponse::Unauthorized() - .finish()); - //return Either::Right(err(Error::from(error))); - //return Err(Error::from(UserError::ValidationError { field: "token invalid".to_string() })); return Either::Right(err(Error::from(BannedError {}))) } Either::Left(self.service.call(req)) } -} \ No newline at end of file +} From 23f5051acef11f038b1eb226ee9e6bab71221e1d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 23:43:33 -0500 Subject: [PATCH 0669/1042] cleanup --- .../http_server/middleware/ip_filter_middleware.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs index 5e88ca0301..c3fbbc1720 100644 --- a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs +++ b/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs @@ -13,14 +13,21 @@ use actix_web::{Error, HttpResponse}; use actix_web::{ResponseError, HttpMessage, HttpRequest, HttpResponseBuilder}; use crate::http_server::web_utils::ip_address::get_service_request_ip; use crate::threads::ip_banlist_set::IpBanlistSet; -use derive_more::{Display, Error as ErrorE}; use futures_util::future::{err, ok, Either, Ready}; use std::io::Write; use std::task::{Context, Poll}; -#[derive(Debug, Display, ErrorE)] +#[derive(Debug)] pub struct BannedError {} +impl std::error::Error for BannedError {} + +impl std::fmt::Display for BannedError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "BannedError") + } +} + impl ResponseError for BannedError { fn status_code(&self) -> StatusCode { StatusCode::UNAUTHORIZED From a3bbf7b42ce7807be15063c845d998edbfc0241d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 23:51:25 -0500 Subject: [PATCH 0670/1042] Refactor crate organization. Both server binaries build --- Cargo.toml | 10 +++++----- {app => crates/app}/obs_gateway/Cargo.toml | 8 ++++---- {app => crates/app}/obs_gateway/src/endpoints/mod.rs | 0 .../app}/obs_gateway/src/endpoints/oauth_begin.rs | 0 .../app}/obs_gateway/src/endpoints/oauth_end.rs | 0 {app => crates/app}/obs_gateway/src/main.rs | 0 {app => crates/app}/obs_gateway/src/pubsub_gateway.rs | 0 {app => crates/app}/obs_gateway/src/twitch/mod.rs | 0 .../obs_gateway/src/twitch/twitch_client_wrapper.rs | 0 .../app}/obs_gateway/src/twitch/twitch_secrets.rs | 0 .../app}/obs_gateway/src/twitch/websocket_client.rs | 0 {app => crates/app}/obs_gateway/src/util/mod.rs | 0 .../app}/obs_gateway/src/util/random_nonce.rs | 0 {app => crates/app}/storyteller_web/Cargo.toml | 6 +++--- {app => crates/app}/storyteller_web/src/common_env.rs | 0 .../app}/storyteller_web/src/database/enums/mod.rs | 0 .../src/database/enums/record_visibility.rs | 0 .../src/database/enums/tts_model_type.rs | 0 .../storyteller_web/src/database/enums/vocoder_type.rs | 0 .../src/database/helpers/boolean_converters.rs | 0 .../app}/storyteller_web/src/database/helpers/enums.rs | 0 .../app}/storyteller_web/src/database/helpers/mod.rs | 0 .../storyteller_web/src/database/helpers/tokens.rs | 0 .../src/database/mediators/badge_granter.rs | 0 .../src/database/mediators/firehose_publisher.rs | 0 .../app}/storyteller_web/src/database/mediators/mod.rs | 0 .../app}/storyteller_web/src/database/mod.rs | 0 .../storyteller_web/src/database/queries/badges.rs | 0 .../queries/calculate_tts_model_leaderboard.rs | 0 .../queries/calculate_w2l_template_leaderboard.rs | 0 .../src/database/queries/create_session.rs | 0 .../src/database/queries/delete_tts_model.rs | 0 .../src/database/queries/delete_tts_result.rs | 0 .../src/database/queries/delete_w2l_result.rs | 0 .../src/database/queries/delete_w2l_template.rs | 0 .../database/queries/get_user_profile_by_username.rs | 0 .../src/database/queries/list_ip_bans.rs | 0 .../src/database/queries/list_tts_models.rs | 0 .../src/database/queries/list_user_badges.rs | 0 .../src/database/queries/list_user_roles.rs | 0 .../src/database/queries/list_w2l_templates.rs | 0 .../app}/storyteller_web/src/database/queries/mod.rs | 0 .../src/database/queries/query_tts_model.rs | 0 .../src/database/queries/query_tts_result.rs | 0 .../src/database/queries/query_w2l_result.rs | 0 .../src/database/queries/query_w2l_template.rs | 0 .../list_tts_inference_results_query_builder.rs | 0 .../query_builders/list_users_query_builder.rs | 0 .../list_w2l_inference_results_query_builder.rs | 0 .../storyteller_web/src/database/query_builders/mod.rs | 0 .../app}/storyteller_web/src/http_clients/mod.rs | 0 .../src/http_clients/tts_inference_sidecar_client.rs | 0 .../src/http_server/endpoints/events/list_events.rs | 0 .../src/http_server/endpoints/events/mod.rs | 0 .../endpoints/leaderboard/get_leaderboard.rs | 0 .../src/http_server/endpoints/leaderboard/mod.rs | 0 .../src/http_server/endpoints/misc/alpha_cookie.rs | 0 .../http_server/endpoints/misc}/default_route_404.rs | 0 .../endpoints/misc/enable_alpha_easy_handler.rs | 0 .../http_server/endpoints/misc/enable_alpha_handler.rs | 0 .../src/http_server/endpoints/misc/mod.rs | 0 .../src/http_server/endpoints/misc}/root_index.rs | 0 .../storyteller_web/src/http_server/endpoints/mod.rs | 0 .../http_server/endpoints/moderation/approval/mod.rs | 0 .../moderation/approval/pending_w2l_templates.rs | 0 .../endpoints/moderation/ip_bans/add_ip_ban.rs | 0 .../endpoints/moderation/ip_bans/delete_ip_ban.rs | 0 .../endpoints/moderation/ip_bans/get_ip_ban.rs | 0 .../endpoints/moderation/ip_bans/list_ip_bans.rs | 0 .../http_server/endpoints/moderation/ip_bans/mod.rs | 0 .../moderation/jobs/get_tts_inference_queue_count.rs | 0 .../moderation/jobs/get_w2l_inference_queue_count.rs | 0 .../src/http_server/endpoints/moderation/jobs/mod.rs | 0 .../src/http_server/endpoints/moderation/mod.rs | 0 .../moderation/stats/get_voice_count_stats.rs | 0 .../src/http_server/endpoints/moderation/stats/mod.rs | 0 .../endpoints/moderation/user_bans/ban_user.rs | 0 .../moderation/user_bans/list_banned_users.rs | 0 .../http_server/endpoints/moderation/user_bans/mod.rs | 0 .../endpoints/moderation/user_roles/list_roles.rs | 0 .../endpoints/moderation/user_roles/list_staff.rs | 0 .../http_server/endpoints/moderation/user_roles/mod.rs | 0 .../endpoints/moderation/user_roles/set_user_role.rs | 0 .../endpoints/moderation/users/list_users.rs | 0 .../src/http_server/endpoints/moderation/users/mod.rs | 0 .../src/http_server/endpoints/tts/delete_tts_model.rs | 0 .../src/http_server/endpoints/tts/delete_tts_result.rs | 0 .../src/http_server/endpoints/tts/edit_tts_model.rs | 0 .../src/http_server/endpoints/tts/edit_tts_result.rs | 0 .../src/http_server/endpoints/tts/enqueue_infer_tts.rs | 0 .../endpoints/tts/enqueue_upload_tts_model.rs | 0 .../endpoints/tts/get_tts_inference_job_status.rs | 0 .../src/http_server/endpoints/tts/get_tts_model.rs | 0 .../endpoints/tts/get_tts_model_use_count.rs | 0 .../src/http_server/endpoints/tts/get_tts_result.rs | 0 .../endpoints/tts/get_tts_upload_model_job_status.rs | 0 .../src/http_server/endpoints/tts/list_tts_models.rs | 0 .../src/http_server/endpoints/tts/mod.rs | 0 .../src/http_server/endpoints/users/create_account.rs | 0 .../src/http_server/endpoints/users/edit_profile.rs | 0 .../src/http_server/endpoints/users/get_profile.rs | 0 .../endpoints/users/list_user_tts_inference_results.rs | 0 .../endpoints/users/list_user_tts_models.rs | 0 .../endpoints/users/list_user_w2l_inference_results.rs | 0 .../endpoints/users/list_user_w2l_templates.rs | 0 .../src/http_server/endpoints/users/login.rs | 0 .../src/http_server/endpoints/users/logout.rs | 0 .../src/http_server/endpoints/users/mod.rs | 0 .../src/http_server/endpoints/users/session_info.rs | 0 .../src/http_server/endpoints/w2l/delete_w2l_result.rs | 0 .../http_server/endpoints/w2l/delete_w2l_template.rs | 0 .../src/http_server/endpoints/w2l/edit_w2l_result.rs | 0 .../src/http_server/endpoints/w2l/edit_w2l_template.rs | 0 .../src/http_server/endpoints/w2l/enqueue_infer_w2l.rs | 0 .../endpoints/w2l/enqueue_infer_w2l_with_uploads.rs | 0 .../endpoints/w2l/enqueue_upload_w2l_template.rs | 0 .../endpoints/w2l/get_w2l_inference_job_status.rs | 0 .../src/http_server/endpoints/w2l/get_w2l_result.rs | 0 .../src/http_server/endpoints/w2l/get_w2l_template.rs | 0 .../endpoints/w2l/get_w2l_template_use_count.rs | 0 .../w2l/get_w2l_upload_template_job_status.rs | 0 .../http_server/endpoints/w2l/list_w2l_templates.rs | 0 .../src/http_server/endpoints/w2l/mod.rs | 0 .../endpoints/w2l/set_w2l_template_mod_approval.rs | 0 .../src/http_server/middleware/ip_filter_middleware.rs | 0 .../storyteller_web/src/http_server/middleware/mod.rs | 0 .../app}/storyteller_web/src/http_server/mod.rs | 0 .../src/http_server/web_utils/cookie_manager.rs | 0 .../src/http_server/web_utils/get_host_header.rs | 0 .../src/http_server/web_utils/ip_address.rs | 0 .../storyteller_web/src/http_server/web_utils/mod.rs | 0 .../web_utils/read_multipart_field_bytes.rs | 0 .../src/http_server/web_utils/redis_rate_limiter.rs | 0 .../http_server/web_utils/response_error_helpers.rs | 0 .../http_server/web_utils}/response_success_helpers.rs | 0 .../http_server/web_utils}/serialize_as_json_error.rs | 0 .../src/http_server/web_utils/session_checker.rs | 0 .../app}/storyteller_web/src/job_queries/mod.rs | 0 .../src/job_queries/tts_download_job_queries.rs | 0 .../src/job_queries/tts_inference_job_queries.rs | 0 .../src/job_queries/w2l_download_job_queries.rs | 0 .../src/job_queries/w2l_inference_job_queries.rs | 0 {app => crates/app}/storyteller_web/src/main.rs | 6 +++--- .../ffmpeg_generate_preview_image_command.rs | 0 .../ffmpeg_generate_preview_video_command.rs | 0 .../script_execution/google_drive_download_command.rs | 0 .../imagemagick_generate_preview_image_command.rs | 0 .../app}/storyteller_web/src/script_execution/mod.rs | 0 .../src/script_execution/tacotron_inference_command.rs | 0 .../script_execution/tacotron_model_check_command.rs | 0 .../script_execution/talknet_model_check_command.rs | 0 .../src/script_execution/wav2lip_inference_command.rs | 0 .../script_execution/wav2lip_process_upload_command.rs | 0 .../app}/storyteller_web/src/server_state.rs | 0 .../app}/storyteller_web/src/threads/ip_banlist_set.rs | 0 {app => crates/app}/storyteller_web/src/threads/mod.rs | 0 .../src/threads/poll_ip_banlist_thread.rs | 0 .../app}/storyteller_web/src/tts_download_job.rs | 0 .../app}/storyteller_web/src/tts_inference_job.rs | 0 .../app/storyteller_web/src/util}/anyhow_result.rs | 0 .../storyteller_web/src/util/buckets/bucket_client.rs | 0 .../src/util/buckets/bucket_path_unifier.rs | 0 .../storyteller_web/src/util/buckets/bucket_paths.rs | 0 .../app}/storyteller_web/src/util/buckets/mod.rs | 0 .../app}/storyteller_web/src/util/caching/mod.rs | 0 .../src/util/caching/single_item_ttl_cache.rs | 0 .../src/util/delete_role_disambiguation.rs | 0 .../app}/storyteller_web/src/util/email_to_gravatar.rs | 0 .../app}/storyteller_web/src/util/encrypted_sort_id.rs | 0 .../app}/storyteller_web/src/util/filesystem.rs | 0 .../storyteller_web/src/util/hashing/hash_file_sha2.rs | 0 .../src/util/hashing/hash_string_sha2.rs | 0 .../app}/storyteller_web/src/util/hashing/mod.rs | 0 .../storyteller_web/src/util/hashing/sha256_digest.rs | 0 .../src/util/jobs/cache_miss_strategizer.rs | 0 .../src/util/jobs/cache_miss_strategizer_multi.rs | 0 .../app}/storyteller_web/src/util/jobs/mod.rs | 0 .../storyteller_web/src/util/jobs/virtual_lfu_cache.rs | 0 .../app}/storyteller_web/src/util/markdown_to_html.rs | 0 {app => crates/app}/storyteller_web/src/util/mod.rs | 0 .../app}/storyteller_web/src/util/noop_logger.rs | 0 .../storyteller_web/src/util/random_crockford_token.rs | 0 .../src/util/random_prefix_crockford_token.rs | 0 .../app}/storyteller_web/src/util/random_uuid.rs | 0 .../app}/storyteller_web/src/util/redis/mod.rs | 0 .../src/util/redis/redis_job_status_logger.rs | 0 .../app}/storyteller_web/src/util/redis/redis_keys.rs | 0 .../src/util/semi_persistent_cache_dir.rs | 0 .../src/validations/cashapp_username.rs | 0 .../storyteller_web/src/validations/check_for_slurs.rs | 0 .../src/validations/discord_username.rs | 0 .../storyteller_web/src/validations/github_username.rs | 0 .../storyteller_web/src/validations/ip_addresses.rs | 0 .../app}/storyteller_web/src/validations/mod.rs | 0 .../storyteller_web/src/validations/model_uploads.rs | 0 .../app}/storyteller_web/src/validations/passwords.rs | 0 .../storyteller_web/src/validations/twitch_username.rs | 0 .../src/validations/twitter_username.rs | 0 .../app}/storyteller_web/src/validations/username.rs | 0 .../src/validations/username_reservations.rs | 0 .../storyteller_web/src/validations/website_url.rs | 0 .../app}/storyteller_web/src/w2l_download_job.rs | 0 .../app}/storyteller_web/src/w2l_inference_job.rs | 0 {app => crates}/lib/config/Cargo.toml | 0 {app => crates}/lib/config/src/lib.rs | 0 {app => crates}/lib/config/src/shared_constants.rs | 0 {app => crates}/lib/container_common/Cargo.toml | 0 .../lib/container_common/src}/anyhow_result.rs | 0 {app => crates}/lib/container_common/src/lib.rs | 0 {app => crates}/lib/http_server_common/Cargo.toml | 0 {app => crates}/lib/http_server_common/src/cors.rs | 0 .../src/endpoints}/default_route_404.rs | 0 .../lib/http_server_common/src/endpoints/mod.rs | 0 .../http_server_common/src/endpoints}/root_index.rs | 0 {app => crates}/lib/http_server_common/src/lib.rs | 0 .../lib/http_server_common/src/response/mod.rs | 0 .../src/response/response_error_helpers.rs | 0 .../src/response}/response_success_helpers.rs | 0 .../src/response}/serialize_as_json_error.rs | 0 219 files changed, 15 insertions(+), 15 deletions(-) rename {app => crates/app}/obs_gateway/Cargo.toml (94%) rename {app => crates/app}/obs_gateway/src/endpoints/mod.rs (100%) rename {app => crates/app}/obs_gateway/src/endpoints/oauth_begin.rs (100%) rename {app => crates/app}/obs_gateway/src/endpoints/oauth_end.rs (100%) rename {app => crates/app}/obs_gateway/src/main.rs (100%) rename {app => crates/app}/obs_gateway/src/pubsub_gateway.rs (100%) rename {app => crates/app}/obs_gateway/src/twitch/mod.rs (100%) rename {app => crates/app}/obs_gateway/src/twitch/twitch_client_wrapper.rs (100%) rename {app => crates/app}/obs_gateway/src/twitch/twitch_secrets.rs (100%) rename {app => crates/app}/obs_gateway/src/twitch/websocket_client.rs (100%) rename {app => crates/app}/obs_gateway/src/util/mod.rs (100%) rename {app => crates/app}/obs_gateway/src/util/random_nonce.rs (100%) rename {app => crates/app}/storyteller_web/Cargo.toml (95%) rename {app => crates/app}/storyteller_web/src/common_env.rs (100%) rename {app => crates/app}/storyteller_web/src/database/enums/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/database/enums/record_visibility.rs (100%) rename {app => crates/app}/storyteller_web/src/database/enums/tts_model_type.rs (100%) rename {app => crates/app}/storyteller_web/src/database/enums/vocoder_type.rs (100%) rename {app => crates/app}/storyteller_web/src/database/helpers/boolean_converters.rs (100%) rename {app => crates/app}/storyteller_web/src/database/helpers/enums.rs (100%) rename {app => crates/app}/storyteller_web/src/database/helpers/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/database/helpers/tokens.rs (100%) rename {app => crates/app}/storyteller_web/src/database/mediators/badge_granter.rs (100%) rename {app => crates/app}/storyteller_web/src/database/mediators/firehose_publisher.rs (100%) rename {app => crates/app}/storyteller_web/src/database/mediators/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/database/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/badges.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/create_session.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/delete_tts_model.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/delete_tts_result.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/delete_w2l_result.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/delete_w2l_template.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/get_user_profile_by_username.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/list_ip_bans.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/list_tts_models.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/list_user_badges.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/list_user_roles.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/list_w2l_templates.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/query_tts_model.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/query_tts_result.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/query_w2l_result.rs (100%) rename {app => crates/app}/storyteller_web/src/database/queries/query_w2l_template.rs (100%) rename {app => crates/app}/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs (100%) rename {app => crates/app}/storyteller_web/src/database/query_builders/list_users_query_builder.rs (100%) rename {app => crates/app}/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs (100%) rename {app => crates/app}/storyteller_web/src/database/query_builders/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_clients/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/events/list_events.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/events/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs (100%) rename {app/lib/http_server_common/src/endpoints => crates/app/storyteller_web/src/http_server/endpoints/misc}/default_route_404.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/misc/mod.rs (100%) rename {app/lib/http_server_common/src/endpoints => crates/app/storyteller_web/src/http_server/endpoints/misc}/root_index.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/tts/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/users/create_account.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/users/edit_profile.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/users/get_profile.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/users/login.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/users/logout.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/users/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/users/session_info.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/middleware/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/web_utils/cookie_manager.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/web_utils/get_host_header.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/web_utils/ip_address.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/web_utils/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/web_utils/response_error_helpers.rs (100%) rename {app/lib/http_server_common/src/response => crates/app/storyteller_web/src/http_server/web_utils}/response_success_helpers.rs (100%) rename {app/lib/http_server_common/src/response => crates/app/storyteller_web/src/http_server/web_utils}/serialize_as_json_error.rs (100%) rename {app => crates/app}/storyteller_web/src/http_server/web_utils/session_checker.rs (100%) rename {app => crates/app}/storyteller_web/src/job_queries/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/job_queries/tts_download_job_queries.rs (100%) rename {app => crates/app}/storyteller_web/src/job_queries/tts_inference_job_queries.rs (100%) rename {app => crates/app}/storyteller_web/src/job_queries/w2l_download_job_queries.rs (100%) rename {app => crates/app}/storyteller_web/src/job_queries/w2l_inference_job_queries.rs (100%) rename {app => crates/app}/storyteller_web/src/main.rs (99%) rename {app => crates/app}/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs (100%) rename {app => crates/app}/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs (100%) rename {app => crates/app}/storyteller_web/src/script_execution/google_drive_download_command.rs (100%) rename {app => crates/app}/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs (100%) rename {app => crates/app}/storyteller_web/src/script_execution/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/script_execution/tacotron_inference_command.rs (100%) rename {app => crates/app}/storyteller_web/src/script_execution/tacotron_model_check_command.rs (100%) rename {app => crates/app}/storyteller_web/src/script_execution/talknet_model_check_command.rs (100%) rename {app => crates/app}/storyteller_web/src/script_execution/wav2lip_inference_command.rs (100%) rename {app => crates/app}/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs (100%) rename {app => crates/app}/storyteller_web/src/server_state.rs (100%) rename {app => crates/app}/storyteller_web/src/threads/ip_banlist_set.rs (100%) rename {app => crates/app}/storyteller_web/src/threads/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/threads/poll_ip_banlist_thread.rs (100%) rename {app => crates/app}/storyteller_web/src/tts_download_job.rs (100%) rename {app => crates/app}/storyteller_web/src/tts_inference_job.rs (100%) rename {app/lib/container_common/src => crates/app/storyteller_web/src/util}/anyhow_result.rs (100%) rename {app => crates/app}/storyteller_web/src/util/buckets/bucket_client.rs (100%) rename {app => crates/app}/storyteller_web/src/util/buckets/bucket_path_unifier.rs (100%) rename {app => crates/app}/storyteller_web/src/util/buckets/bucket_paths.rs (100%) rename {app => crates/app}/storyteller_web/src/util/buckets/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/util/caching/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/util/caching/single_item_ttl_cache.rs (100%) rename {app => crates/app}/storyteller_web/src/util/delete_role_disambiguation.rs (100%) rename {app => crates/app}/storyteller_web/src/util/email_to_gravatar.rs (100%) rename {app => crates/app}/storyteller_web/src/util/encrypted_sort_id.rs (100%) rename {app => crates/app}/storyteller_web/src/util/filesystem.rs (100%) rename {app => crates/app}/storyteller_web/src/util/hashing/hash_file_sha2.rs (100%) rename {app => crates/app}/storyteller_web/src/util/hashing/hash_string_sha2.rs (100%) rename {app => crates/app}/storyteller_web/src/util/hashing/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/util/hashing/sha256_digest.rs (100%) rename {app => crates/app}/storyteller_web/src/util/jobs/cache_miss_strategizer.rs (100%) rename {app => crates/app}/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs (100%) rename {app => crates/app}/storyteller_web/src/util/jobs/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/util/jobs/virtual_lfu_cache.rs (100%) rename {app => crates/app}/storyteller_web/src/util/markdown_to_html.rs (100%) rename {app => crates/app}/storyteller_web/src/util/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/util/noop_logger.rs (100%) rename {app => crates/app}/storyteller_web/src/util/random_crockford_token.rs (100%) rename {app => crates/app}/storyteller_web/src/util/random_prefix_crockford_token.rs (100%) rename {app => crates/app}/storyteller_web/src/util/random_uuid.rs (100%) rename {app => crates/app}/storyteller_web/src/util/redis/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/util/redis/redis_job_status_logger.rs (100%) rename {app => crates/app}/storyteller_web/src/util/redis/redis_keys.rs (100%) rename {app => crates/app}/storyteller_web/src/util/semi_persistent_cache_dir.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/cashapp_username.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/check_for_slurs.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/discord_username.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/github_username.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/ip_addresses.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/mod.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/model_uploads.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/passwords.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/twitch_username.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/twitter_username.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/username.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/username_reservations.rs (100%) rename {app => crates/app}/storyteller_web/src/validations/website_url.rs (100%) rename {app => crates/app}/storyteller_web/src/w2l_download_job.rs (100%) rename {app => crates/app}/storyteller_web/src/w2l_inference_job.rs (100%) rename {app => crates}/lib/config/Cargo.toml (100%) rename {app => crates}/lib/config/src/lib.rs (100%) rename {app => crates}/lib/config/src/shared_constants.rs (100%) rename {app => crates}/lib/container_common/Cargo.toml (100%) rename {app/storyteller_web/src/util => crates/lib/container_common/src}/anyhow_result.rs (100%) rename {app => crates}/lib/container_common/src/lib.rs (100%) rename {app => crates}/lib/http_server_common/Cargo.toml (100%) rename {app => crates}/lib/http_server_common/src/cors.rs (100%) rename {app/storyteller_web/src/http_server/endpoints/misc => crates/lib/http_server_common/src/endpoints}/default_route_404.rs (100%) rename {app => crates}/lib/http_server_common/src/endpoints/mod.rs (100%) rename {app/storyteller_web/src/http_server/endpoints/misc => crates/lib/http_server_common/src/endpoints}/root_index.rs (100%) rename {app => crates}/lib/http_server_common/src/lib.rs (100%) rename {app => crates}/lib/http_server_common/src/response/mod.rs (100%) rename {app => crates}/lib/http_server_common/src/response/response_error_helpers.rs (100%) rename {app/storyteller_web/src/http_server/web_utils => crates/lib/http_server_common/src/response}/response_success_helpers.rs (100%) rename {app/storyteller_web/src/http_server/web_utils => crates/lib/http_server_common/src/response}/serialize_as_json_error.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 5a919a6a90..fef48a845c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [workspace] members = [ - "app/lib/config", - "app/lib/container_common", - "app/lib/http_server_common", - "app/obs_gateway", - "app/storyteller_web", + "crates/app/obs_gateway", + "crates/app/storyteller_web", + "crates/lib/config", + "crates/lib/container_common", + "crates/lib/http_server_common", ] diff --git a/app/obs_gateway/Cargo.toml b/crates/app/obs_gateway/Cargo.toml similarity index 94% rename from app/obs_gateway/Cargo.toml rename to crates/app/obs_gateway/Cargo.toml index 8303bb4735..fa9d81175d 100644 --- a/app/obs_gateway/Cargo.toml +++ b/crates/app/obs_gateway/Cargo.toml @@ -15,9 +15,9 @@ path = "src/main.rs" [dependencies] # Internal -config = { path = "../lib/config" } -container_common = { path = "../lib/container_common" } -http_server_common = { path = "../lib/http_server_common" } +config = { path = "../../lib/config" } +container_common = { path = "../../lib/container_common" } +http_server_common = { path = "../../lib/http_server_common" } # External anyhow = "1.0.40" @@ -34,7 +34,7 @@ lazy_static = "1.4.0" lfu = "0.2.5" log = "0.4.14" #newrelic = "0.2.2" -#newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" } +#newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } rand = "0.8.3" regex = "1.5.4" time = "0.2.26" diff --git a/app/obs_gateway/src/endpoints/mod.rs b/crates/app/obs_gateway/src/endpoints/mod.rs similarity index 100% rename from app/obs_gateway/src/endpoints/mod.rs rename to crates/app/obs_gateway/src/endpoints/mod.rs diff --git a/app/obs_gateway/src/endpoints/oauth_begin.rs b/crates/app/obs_gateway/src/endpoints/oauth_begin.rs similarity index 100% rename from app/obs_gateway/src/endpoints/oauth_begin.rs rename to crates/app/obs_gateway/src/endpoints/oauth_begin.rs diff --git a/app/obs_gateway/src/endpoints/oauth_end.rs b/crates/app/obs_gateway/src/endpoints/oauth_end.rs similarity index 100% rename from app/obs_gateway/src/endpoints/oauth_end.rs rename to crates/app/obs_gateway/src/endpoints/oauth_end.rs diff --git a/app/obs_gateway/src/main.rs b/crates/app/obs_gateway/src/main.rs similarity index 100% rename from app/obs_gateway/src/main.rs rename to crates/app/obs_gateway/src/main.rs diff --git a/app/obs_gateway/src/pubsub_gateway.rs b/crates/app/obs_gateway/src/pubsub_gateway.rs similarity index 100% rename from app/obs_gateway/src/pubsub_gateway.rs rename to crates/app/obs_gateway/src/pubsub_gateway.rs diff --git a/app/obs_gateway/src/twitch/mod.rs b/crates/app/obs_gateway/src/twitch/mod.rs similarity index 100% rename from app/obs_gateway/src/twitch/mod.rs rename to crates/app/obs_gateway/src/twitch/mod.rs diff --git a/app/obs_gateway/src/twitch/twitch_client_wrapper.rs b/crates/app/obs_gateway/src/twitch/twitch_client_wrapper.rs similarity index 100% rename from app/obs_gateway/src/twitch/twitch_client_wrapper.rs rename to crates/app/obs_gateway/src/twitch/twitch_client_wrapper.rs diff --git a/app/obs_gateway/src/twitch/twitch_secrets.rs b/crates/app/obs_gateway/src/twitch/twitch_secrets.rs similarity index 100% rename from app/obs_gateway/src/twitch/twitch_secrets.rs rename to crates/app/obs_gateway/src/twitch/twitch_secrets.rs diff --git a/app/obs_gateway/src/twitch/websocket_client.rs b/crates/app/obs_gateway/src/twitch/websocket_client.rs similarity index 100% rename from app/obs_gateway/src/twitch/websocket_client.rs rename to crates/app/obs_gateway/src/twitch/websocket_client.rs diff --git a/app/obs_gateway/src/util/mod.rs b/crates/app/obs_gateway/src/util/mod.rs similarity index 100% rename from app/obs_gateway/src/util/mod.rs rename to crates/app/obs_gateway/src/util/mod.rs diff --git a/app/obs_gateway/src/util/random_nonce.rs b/crates/app/obs_gateway/src/util/random_nonce.rs similarity index 100% rename from app/obs_gateway/src/util/random_nonce.rs rename to crates/app/obs_gateway/src/util/random_nonce.rs diff --git a/app/storyteller_web/Cargo.toml b/crates/app/storyteller_web/Cargo.toml similarity index 95% rename from app/storyteller_web/Cargo.toml rename to crates/app/storyteller_web/Cargo.toml index 74519dc69d..7ff25f0492 100644 --- a/app/storyteller_web/Cargo.toml +++ b/crates/app/storyteller_web/Cargo.toml @@ -38,8 +38,8 @@ path = "src/w2l_inference_job.rs" [dependencies] # Internal -config = { path = "../lib/config" } -http_server_common = { path = "../lib/http_server_common" } +config = { path = "../../lib/config" } +http_server_common = { path = "../../lib/http_server_common" } # External anyhow = "1.0.40" @@ -57,7 +57,7 @@ lazy_static = "1.4.0" lfu = "0.2.5" log = "0.4.14" #newrelic = "0.2.2" -newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" } +newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } rand = "0.8.3" regex = "1.5.4" time = "0.2.26" diff --git a/app/storyteller_web/src/common_env.rs b/crates/app/storyteller_web/src/common_env.rs similarity index 100% rename from app/storyteller_web/src/common_env.rs rename to crates/app/storyteller_web/src/common_env.rs diff --git a/app/storyteller_web/src/database/enums/mod.rs b/crates/app/storyteller_web/src/database/enums/mod.rs similarity index 100% rename from app/storyteller_web/src/database/enums/mod.rs rename to crates/app/storyteller_web/src/database/enums/mod.rs diff --git a/app/storyteller_web/src/database/enums/record_visibility.rs b/crates/app/storyteller_web/src/database/enums/record_visibility.rs similarity index 100% rename from app/storyteller_web/src/database/enums/record_visibility.rs rename to crates/app/storyteller_web/src/database/enums/record_visibility.rs diff --git a/app/storyteller_web/src/database/enums/tts_model_type.rs b/crates/app/storyteller_web/src/database/enums/tts_model_type.rs similarity index 100% rename from app/storyteller_web/src/database/enums/tts_model_type.rs rename to crates/app/storyteller_web/src/database/enums/tts_model_type.rs diff --git a/app/storyteller_web/src/database/enums/vocoder_type.rs b/crates/app/storyteller_web/src/database/enums/vocoder_type.rs similarity index 100% rename from app/storyteller_web/src/database/enums/vocoder_type.rs rename to crates/app/storyteller_web/src/database/enums/vocoder_type.rs diff --git a/app/storyteller_web/src/database/helpers/boolean_converters.rs b/crates/app/storyteller_web/src/database/helpers/boolean_converters.rs similarity index 100% rename from app/storyteller_web/src/database/helpers/boolean_converters.rs rename to crates/app/storyteller_web/src/database/helpers/boolean_converters.rs diff --git a/app/storyteller_web/src/database/helpers/enums.rs b/crates/app/storyteller_web/src/database/helpers/enums.rs similarity index 100% rename from app/storyteller_web/src/database/helpers/enums.rs rename to crates/app/storyteller_web/src/database/helpers/enums.rs diff --git a/app/storyteller_web/src/database/helpers/mod.rs b/crates/app/storyteller_web/src/database/helpers/mod.rs similarity index 100% rename from app/storyteller_web/src/database/helpers/mod.rs rename to crates/app/storyteller_web/src/database/helpers/mod.rs diff --git a/app/storyteller_web/src/database/helpers/tokens.rs b/crates/app/storyteller_web/src/database/helpers/tokens.rs similarity index 100% rename from app/storyteller_web/src/database/helpers/tokens.rs rename to crates/app/storyteller_web/src/database/helpers/tokens.rs diff --git a/app/storyteller_web/src/database/mediators/badge_granter.rs b/crates/app/storyteller_web/src/database/mediators/badge_granter.rs similarity index 100% rename from app/storyteller_web/src/database/mediators/badge_granter.rs rename to crates/app/storyteller_web/src/database/mediators/badge_granter.rs diff --git a/app/storyteller_web/src/database/mediators/firehose_publisher.rs b/crates/app/storyteller_web/src/database/mediators/firehose_publisher.rs similarity index 100% rename from app/storyteller_web/src/database/mediators/firehose_publisher.rs rename to crates/app/storyteller_web/src/database/mediators/firehose_publisher.rs diff --git a/app/storyteller_web/src/database/mediators/mod.rs b/crates/app/storyteller_web/src/database/mediators/mod.rs similarity index 100% rename from app/storyteller_web/src/database/mediators/mod.rs rename to crates/app/storyteller_web/src/database/mediators/mod.rs diff --git a/app/storyteller_web/src/database/mod.rs b/crates/app/storyteller_web/src/database/mod.rs similarity index 100% rename from app/storyteller_web/src/database/mod.rs rename to crates/app/storyteller_web/src/database/mod.rs diff --git a/app/storyteller_web/src/database/queries/badges.rs b/crates/app/storyteller_web/src/database/queries/badges.rs similarity index 100% rename from app/storyteller_web/src/database/queries/badges.rs rename to crates/app/storyteller_web/src/database/queries/badges.rs diff --git a/app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs b/crates/app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs similarity index 100% rename from app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs rename to crates/app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs diff --git a/app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs b/crates/app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs similarity index 100% rename from app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs rename to crates/app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs diff --git a/app/storyteller_web/src/database/queries/create_session.rs b/crates/app/storyteller_web/src/database/queries/create_session.rs similarity index 100% rename from app/storyteller_web/src/database/queries/create_session.rs rename to crates/app/storyteller_web/src/database/queries/create_session.rs diff --git a/app/storyteller_web/src/database/queries/delete_tts_model.rs b/crates/app/storyteller_web/src/database/queries/delete_tts_model.rs similarity index 100% rename from app/storyteller_web/src/database/queries/delete_tts_model.rs rename to crates/app/storyteller_web/src/database/queries/delete_tts_model.rs diff --git a/app/storyteller_web/src/database/queries/delete_tts_result.rs b/crates/app/storyteller_web/src/database/queries/delete_tts_result.rs similarity index 100% rename from app/storyteller_web/src/database/queries/delete_tts_result.rs rename to crates/app/storyteller_web/src/database/queries/delete_tts_result.rs diff --git a/app/storyteller_web/src/database/queries/delete_w2l_result.rs b/crates/app/storyteller_web/src/database/queries/delete_w2l_result.rs similarity index 100% rename from app/storyteller_web/src/database/queries/delete_w2l_result.rs rename to crates/app/storyteller_web/src/database/queries/delete_w2l_result.rs diff --git a/app/storyteller_web/src/database/queries/delete_w2l_template.rs b/crates/app/storyteller_web/src/database/queries/delete_w2l_template.rs similarity index 100% rename from app/storyteller_web/src/database/queries/delete_w2l_template.rs rename to crates/app/storyteller_web/src/database/queries/delete_w2l_template.rs diff --git a/app/storyteller_web/src/database/queries/get_user_profile_by_username.rs b/crates/app/storyteller_web/src/database/queries/get_user_profile_by_username.rs similarity index 100% rename from app/storyteller_web/src/database/queries/get_user_profile_by_username.rs rename to crates/app/storyteller_web/src/database/queries/get_user_profile_by_username.rs diff --git a/app/storyteller_web/src/database/queries/list_ip_bans.rs b/crates/app/storyteller_web/src/database/queries/list_ip_bans.rs similarity index 100% rename from app/storyteller_web/src/database/queries/list_ip_bans.rs rename to crates/app/storyteller_web/src/database/queries/list_ip_bans.rs diff --git a/app/storyteller_web/src/database/queries/list_tts_models.rs b/crates/app/storyteller_web/src/database/queries/list_tts_models.rs similarity index 100% rename from app/storyteller_web/src/database/queries/list_tts_models.rs rename to crates/app/storyteller_web/src/database/queries/list_tts_models.rs diff --git a/app/storyteller_web/src/database/queries/list_user_badges.rs b/crates/app/storyteller_web/src/database/queries/list_user_badges.rs similarity index 100% rename from app/storyteller_web/src/database/queries/list_user_badges.rs rename to crates/app/storyteller_web/src/database/queries/list_user_badges.rs diff --git a/app/storyteller_web/src/database/queries/list_user_roles.rs b/crates/app/storyteller_web/src/database/queries/list_user_roles.rs similarity index 100% rename from app/storyteller_web/src/database/queries/list_user_roles.rs rename to crates/app/storyteller_web/src/database/queries/list_user_roles.rs diff --git a/app/storyteller_web/src/database/queries/list_w2l_templates.rs b/crates/app/storyteller_web/src/database/queries/list_w2l_templates.rs similarity index 100% rename from app/storyteller_web/src/database/queries/list_w2l_templates.rs rename to crates/app/storyteller_web/src/database/queries/list_w2l_templates.rs diff --git a/app/storyteller_web/src/database/queries/mod.rs b/crates/app/storyteller_web/src/database/queries/mod.rs similarity index 100% rename from app/storyteller_web/src/database/queries/mod.rs rename to crates/app/storyteller_web/src/database/queries/mod.rs diff --git a/app/storyteller_web/src/database/queries/query_tts_model.rs b/crates/app/storyteller_web/src/database/queries/query_tts_model.rs similarity index 100% rename from app/storyteller_web/src/database/queries/query_tts_model.rs rename to crates/app/storyteller_web/src/database/queries/query_tts_model.rs diff --git a/app/storyteller_web/src/database/queries/query_tts_result.rs b/crates/app/storyteller_web/src/database/queries/query_tts_result.rs similarity index 100% rename from app/storyteller_web/src/database/queries/query_tts_result.rs rename to crates/app/storyteller_web/src/database/queries/query_tts_result.rs diff --git a/app/storyteller_web/src/database/queries/query_w2l_result.rs b/crates/app/storyteller_web/src/database/queries/query_w2l_result.rs similarity index 100% rename from app/storyteller_web/src/database/queries/query_w2l_result.rs rename to crates/app/storyteller_web/src/database/queries/query_w2l_result.rs diff --git a/app/storyteller_web/src/database/queries/query_w2l_template.rs b/crates/app/storyteller_web/src/database/queries/query_w2l_template.rs similarity index 100% rename from app/storyteller_web/src/database/queries/query_w2l_template.rs rename to crates/app/storyteller_web/src/database/queries/query_w2l_template.rs diff --git a/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs b/crates/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs similarity index 100% rename from app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs rename to crates/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs diff --git a/app/storyteller_web/src/database/query_builders/list_users_query_builder.rs b/crates/app/storyteller_web/src/database/query_builders/list_users_query_builder.rs similarity index 100% rename from app/storyteller_web/src/database/query_builders/list_users_query_builder.rs rename to crates/app/storyteller_web/src/database/query_builders/list_users_query_builder.rs diff --git a/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs b/crates/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs similarity index 100% rename from app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs rename to crates/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs diff --git a/app/storyteller_web/src/database/query_builders/mod.rs b/crates/app/storyteller_web/src/database/query_builders/mod.rs similarity index 100% rename from app/storyteller_web/src/database/query_builders/mod.rs rename to crates/app/storyteller_web/src/database/query_builders/mod.rs diff --git a/app/storyteller_web/src/http_clients/mod.rs b/crates/app/storyteller_web/src/http_clients/mod.rs similarity index 100% rename from app/storyteller_web/src/http_clients/mod.rs rename to crates/app/storyteller_web/src/http_clients/mod.rs diff --git a/app/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs b/crates/app/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs similarity index 100% rename from app/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs rename to crates/app/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs diff --git a/app/storyteller_web/src/http_server/endpoints/events/list_events.rs b/crates/app/storyteller_web/src/http_server/endpoints/events/list_events.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/events/list_events.rs rename to crates/app/storyteller_web/src/http_server/endpoints/events/list_events.rs diff --git a/app/storyteller_web/src/http_server/endpoints/events/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/events/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/events/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/events/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs b/crates/app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs rename to crates/app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs diff --git a/app/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs b/crates/app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs rename to crates/app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs diff --git a/app/lib/http_server_common/src/endpoints/default_route_404.rs b/crates/app/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs similarity index 100% rename from app/lib/http_server_common/src/endpoints/default_route_404.rs rename to crates/app/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs diff --git a/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs b/crates/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs rename to crates/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs diff --git a/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs b/crates/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs rename to crates/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs diff --git a/app/storyteller_web/src/http_server/endpoints/misc/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/misc/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/misc/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/misc/mod.rs diff --git a/app/lib/http_server_common/src/endpoints/root_index.rs b/crates/app/storyteller_web/src/http_server/endpoints/misc/root_index.rs similarity index 100% rename from app/lib/http_server_common/src/endpoints/root_index.rs rename to crates/app/storyteller_web/src/http_server/endpoints/misc/root_index.rs diff --git a/app/storyteller_web/src/http_server/endpoints/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs diff --git a/app/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs diff --git a/app/storyteller_web/src/http_server/endpoints/tts/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/tts/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/tts/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/users/create_account.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/create_account.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/users/create_account.rs rename to crates/app/storyteller_web/src/http_server/endpoints/users/create_account.rs diff --git a/app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs rename to crates/app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs diff --git a/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/users/get_profile.rs rename to crates/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs diff --git a/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs rename to crates/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs diff --git a/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs rename to crates/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs diff --git a/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs rename to crates/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs diff --git a/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs rename to crates/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs diff --git a/app/storyteller_web/src/http_server/endpoints/users/login.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/login.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/users/login.rs rename to crates/app/storyteller_web/src/http_server/endpoints/users/login.rs diff --git a/app/storyteller_web/src/http_server/endpoints/users/logout.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/logout.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/users/logout.rs rename to crates/app/storyteller_web/src/http_server/endpoints/users/logout.rs diff --git a/app/storyteller_web/src/http_server/endpoints/users/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/users/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/users/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/users/session_info.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/session_info.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/users/session_info.rs rename to crates/app/storyteller_web/src/http_server/endpoints/users/session_info.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/mod.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs rename to crates/app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs diff --git a/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs b/crates/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs similarity index 100% rename from app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs rename to crates/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs diff --git a/app/storyteller_web/src/http_server/middleware/mod.rs b/crates/app/storyteller_web/src/http_server/middleware/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/middleware/mod.rs rename to crates/app/storyteller_web/src/http_server/middleware/mod.rs diff --git a/app/storyteller_web/src/http_server/mod.rs b/crates/app/storyteller_web/src/http_server/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/mod.rs rename to crates/app/storyteller_web/src/http_server/mod.rs diff --git a/app/storyteller_web/src/http_server/web_utils/cookie_manager.rs b/crates/app/storyteller_web/src/http_server/web_utils/cookie_manager.rs similarity index 100% rename from app/storyteller_web/src/http_server/web_utils/cookie_manager.rs rename to crates/app/storyteller_web/src/http_server/web_utils/cookie_manager.rs diff --git a/app/storyteller_web/src/http_server/web_utils/get_host_header.rs b/crates/app/storyteller_web/src/http_server/web_utils/get_host_header.rs similarity index 100% rename from app/storyteller_web/src/http_server/web_utils/get_host_header.rs rename to crates/app/storyteller_web/src/http_server/web_utils/get_host_header.rs diff --git a/app/storyteller_web/src/http_server/web_utils/ip_address.rs b/crates/app/storyteller_web/src/http_server/web_utils/ip_address.rs similarity index 100% rename from app/storyteller_web/src/http_server/web_utils/ip_address.rs rename to crates/app/storyteller_web/src/http_server/web_utils/ip_address.rs diff --git a/app/storyteller_web/src/http_server/web_utils/mod.rs b/crates/app/storyteller_web/src/http_server/web_utils/mod.rs similarity index 100% rename from app/storyteller_web/src/http_server/web_utils/mod.rs rename to crates/app/storyteller_web/src/http_server/web_utils/mod.rs diff --git a/app/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs b/crates/app/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs similarity index 100% rename from app/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs rename to crates/app/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs diff --git a/app/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs b/crates/app/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs similarity index 100% rename from app/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs rename to crates/app/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs diff --git a/app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs b/crates/app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs similarity index 100% rename from app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs rename to crates/app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs diff --git a/app/lib/http_server_common/src/response/response_success_helpers.rs b/crates/app/storyteller_web/src/http_server/web_utils/response_success_helpers.rs similarity index 100% rename from app/lib/http_server_common/src/response/response_success_helpers.rs rename to crates/app/storyteller_web/src/http_server/web_utils/response_success_helpers.rs diff --git a/app/lib/http_server_common/src/response/serialize_as_json_error.rs b/crates/app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs similarity index 100% rename from app/lib/http_server_common/src/response/serialize_as_json_error.rs rename to crates/app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs diff --git a/app/storyteller_web/src/http_server/web_utils/session_checker.rs b/crates/app/storyteller_web/src/http_server/web_utils/session_checker.rs similarity index 100% rename from app/storyteller_web/src/http_server/web_utils/session_checker.rs rename to crates/app/storyteller_web/src/http_server/web_utils/session_checker.rs diff --git a/app/storyteller_web/src/job_queries/mod.rs b/crates/app/storyteller_web/src/job_queries/mod.rs similarity index 100% rename from app/storyteller_web/src/job_queries/mod.rs rename to crates/app/storyteller_web/src/job_queries/mod.rs diff --git a/app/storyteller_web/src/job_queries/tts_download_job_queries.rs b/crates/app/storyteller_web/src/job_queries/tts_download_job_queries.rs similarity index 100% rename from app/storyteller_web/src/job_queries/tts_download_job_queries.rs rename to crates/app/storyteller_web/src/job_queries/tts_download_job_queries.rs diff --git a/app/storyteller_web/src/job_queries/tts_inference_job_queries.rs b/crates/app/storyteller_web/src/job_queries/tts_inference_job_queries.rs similarity index 100% rename from app/storyteller_web/src/job_queries/tts_inference_job_queries.rs rename to crates/app/storyteller_web/src/job_queries/tts_inference_job_queries.rs diff --git a/app/storyteller_web/src/job_queries/w2l_download_job_queries.rs b/crates/app/storyteller_web/src/job_queries/w2l_download_job_queries.rs similarity index 100% rename from app/storyteller_web/src/job_queries/w2l_download_job_queries.rs rename to crates/app/storyteller_web/src/job_queries/w2l_download_job_queries.rs diff --git a/app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs b/crates/app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs similarity index 100% rename from app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs rename to crates/app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs diff --git a/app/storyteller_web/src/main.rs b/crates/app/storyteller_web/src/main.rs similarity index 99% rename from app/storyteller_web/src/main.rs rename to crates/app/storyteller_web/src/main.rs index 39a8e3927b..93007626ad 100644 --- a/app/storyteller_web/src/main.rs +++ b/crates/app/storyteller_web/src/main.rs @@ -12,9 +12,9 @@ #[macro_use] extern crate magic_crypt; #[macro_use] extern crate serde_derive; -pub const RESERVED_USERNAMES : &'static str = include_str!("../../../db/reserved_usernames.txt"); -pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../../../db/reserved_usernames_including.txt"); -pub const BANNED_SLURS : &'static str = include_str!("../../../db/banned_slurs.txt"); +pub const RESERVED_USERNAMES : &'static str = include_str!("../../../../db/reserved_usernames.txt"); +pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../../../../db/reserved_usernames_including.txt"); +pub const BANNED_SLURS : &'static str = include_str!("../../../../db/banned_slurs.txt"); pub mod database; pub mod http_clients; diff --git a/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs b/crates/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs similarity index 100% rename from app/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs rename to crates/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs diff --git a/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs b/crates/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs similarity index 100% rename from app/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs rename to crates/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs diff --git a/app/storyteller_web/src/script_execution/google_drive_download_command.rs b/crates/app/storyteller_web/src/script_execution/google_drive_download_command.rs similarity index 100% rename from app/storyteller_web/src/script_execution/google_drive_download_command.rs rename to crates/app/storyteller_web/src/script_execution/google_drive_download_command.rs diff --git a/app/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs b/crates/app/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs similarity index 100% rename from app/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs rename to crates/app/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs diff --git a/app/storyteller_web/src/script_execution/mod.rs b/crates/app/storyteller_web/src/script_execution/mod.rs similarity index 100% rename from app/storyteller_web/src/script_execution/mod.rs rename to crates/app/storyteller_web/src/script_execution/mod.rs diff --git a/app/storyteller_web/src/script_execution/tacotron_inference_command.rs b/crates/app/storyteller_web/src/script_execution/tacotron_inference_command.rs similarity index 100% rename from app/storyteller_web/src/script_execution/tacotron_inference_command.rs rename to crates/app/storyteller_web/src/script_execution/tacotron_inference_command.rs diff --git a/app/storyteller_web/src/script_execution/tacotron_model_check_command.rs b/crates/app/storyteller_web/src/script_execution/tacotron_model_check_command.rs similarity index 100% rename from app/storyteller_web/src/script_execution/tacotron_model_check_command.rs rename to crates/app/storyteller_web/src/script_execution/tacotron_model_check_command.rs diff --git a/app/storyteller_web/src/script_execution/talknet_model_check_command.rs b/crates/app/storyteller_web/src/script_execution/talknet_model_check_command.rs similarity index 100% rename from app/storyteller_web/src/script_execution/talknet_model_check_command.rs rename to crates/app/storyteller_web/src/script_execution/talknet_model_check_command.rs diff --git a/app/storyteller_web/src/script_execution/wav2lip_inference_command.rs b/crates/app/storyteller_web/src/script_execution/wav2lip_inference_command.rs similarity index 100% rename from app/storyteller_web/src/script_execution/wav2lip_inference_command.rs rename to crates/app/storyteller_web/src/script_execution/wav2lip_inference_command.rs diff --git a/app/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs b/crates/app/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs similarity index 100% rename from app/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs rename to crates/app/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs diff --git a/app/storyteller_web/src/server_state.rs b/crates/app/storyteller_web/src/server_state.rs similarity index 100% rename from app/storyteller_web/src/server_state.rs rename to crates/app/storyteller_web/src/server_state.rs diff --git a/app/storyteller_web/src/threads/ip_banlist_set.rs b/crates/app/storyteller_web/src/threads/ip_banlist_set.rs similarity index 100% rename from app/storyteller_web/src/threads/ip_banlist_set.rs rename to crates/app/storyteller_web/src/threads/ip_banlist_set.rs diff --git a/app/storyteller_web/src/threads/mod.rs b/crates/app/storyteller_web/src/threads/mod.rs similarity index 100% rename from app/storyteller_web/src/threads/mod.rs rename to crates/app/storyteller_web/src/threads/mod.rs diff --git a/app/storyteller_web/src/threads/poll_ip_banlist_thread.rs b/crates/app/storyteller_web/src/threads/poll_ip_banlist_thread.rs similarity index 100% rename from app/storyteller_web/src/threads/poll_ip_banlist_thread.rs rename to crates/app/storyteller_web/src/threads/poll_ip_banlist_thread.rs diff --git a/app/storyteller_web/src/tts_download_job.rs b/crates/app/storyteller_web/src/tts_download_job.rs similarity index 100% rename from app/storyteller_web/src/tts_download_job.rs rename to crates/app/storyteller_web/src/tts_download_job.rs diff --git a/app/storyteller_web/src/tts_inference_job.rs b/crates/app/storyteller_web/src/tts_inference_job.rs similarity index 100% rename from app/storyteller_web/src/tts_inference_job.rs rename to crates/app/storyteller_web/src/tts_inference_job.rs diff --git a/app/lib/container_common/src/anyhow_result.rs b/crates/app/storyteller_web/src/util/anyhow_result.rs similarity index 100% rename from app/lib/container_common/src/anyhow_result.rs rename to crates/app/storyteller_web/src/util/anyhow_result.rs diff --git a/app/storyteller_web/src/util/buckets/bucket_client.rs b/crates/app/storyteller_web/src/util/buckets/bucket_client.rs similarity index 100% rename from app/storyteller_web/src/util/buckets/bucket_client.rs rename to crates/app/storyteller_web/src/util/buckets/bucket_client.rs diff --git a/app/storyteller_web/src/util/buckets/bucket_path_unifier.rs b/crates/app/storyteller_web/src/util/buckets/bucket_path_unifier.rs similarity index 100% rename from app/storyteller_web/src/util/buckets/bucket_path_unifier.rs rename to crates/app/storyteller_web/src/util/buckets/bucket_path_unifier.rs diff --git a/app/storyteller_web/src/util/buckets/bucket_paths.rs b/crates/app/storyteller_web/src/util/buckets/bucket_paths.rs similarity index 100% rename from app/storyteller_web/src/util/buckets/bucket_paths.rs rename to crates/app/storyteller_web/src/util/buckets/bucket_paths.rs diff --git a/app/storyteller_web/src/util/buckets/mod.rs b/crates/app/storyteller_web/src/util/buckets/mod.rs similarity index 100% rename from app/storyteller_web/src/util/buckets/mod.rs rename to crates/app/storyteller_web/src/util/buckets/mod.rs diff --git a/app/storyteller_web/src/util/caching/mod.rs b/crates/app/storyteller_web/src/util/caching/mod.rs similarity index 100% rename from app/storyteller_web/src/util/caching/mod.rs rename to crates/app/storyteller_web/src/util/caching/mod.rs diff --git a/app/storyteller_web/src/util/caching/single_item_ttl_cache.rs b/crates/app/storyteller_web/src/util/caching/single_item_ttl_cache.rs similarity index 100% rename from app/storyteller_web/src/util/caching/single_item_ttl_cache.rs rename to crates/app/storyteller_web/src/util/caching/single_item_ttl_cache.rs diff --git a/app/storyteller_web/src/util/delete_role_disambiguation.rs b/crates/app/storyteller_web/src/util/delete_role_disambiguation.rs similarity index 100% rename from app/storyteller_web/src/util/delete_role_disambiguation.rs rename to crates/app/storyteller_web/src/util/delete_role_disambiguation.rs diff --git a/app/storyteller_web/src/util/email_to_gravatar.rs b/crates/app/storyteller_web/src/util/email_to_gravatar.rs similarity index 100% rename from app/storyteller_web/src/util/email_to_gravatar.rs rename to crates/app/storyteller_web/src/util/email_to_gravatar.rs diff --git a/app/storyteller_web/src/util/encrypted_sort_id.rs b/crates/app/storyteller_web/src/util/encrypted_sort_id.rs similarity index 100% rename from app/storyteller_web/src/util/encrypted_sort_id.rs rename to crates/app/storyteller_web/src/util/encrypted_sort_id.rs diff --git a/app/storyteller_web/src/util/filesystem.rs b/crates/app/storyteller_web/src/util/filesystem.rs similarity index 100% rename from app/storyteller_web/src/util/filesystem.rs rename to crates/app/storyteller_web/src/util/filesystem.rs diff --git a/app/storyteller_web/src/util/hashing/hash_file_sha2.rs b/crates/app/storyteller_web/src/util/hashing/hash_file_sha2.rs similarity index 100% rename from app/storyteller_web/src/util/hashing/hash_file_sha2.rs rename to crates/app/storyteller_web/src/util/hashing/hash_file_sha2.rs diff --git a/app/storyteller_web/src/util/hashing/hash_string_sha2.rs b/crates/app/storyteller_web/src/util/hashing/hash_string_sha2.rs similarity index 100% rename from app/storyteller_web/src/util/hashing/hash_string_sha2.rs rename to crates/app/storyteller_web/src/util/hashing/hash_string_sha2.rs diff --git a/app/storyteller_web/src/util/hashing/mod.rs b/crates/app/storyteller_web/src/util/hashing/mod.rs similarity index 100% rename from app/storyteller_web/src/util/hashing/mod.rs rename to crates/app/storyteller_web/src/util/hashing/mod.rs diff --git a/app/storyteller_web/src/util/hashing/sha256_digest.rs b/crates/app/storyteller_web/src/util/hashing/sha256_digest.rs similarity index 100% rename from app/storyteller_web/src/util/hashing/sha256_digest.rs rename to crates/app/storyteller_web/src/util/hashing/sha256_digest.rs diff --git a/app/storyteller_web/src/util/jobs/cache_miss_strategizer.rs b/crates/app/storyteller_web/src/util/jobs/cache_miss_strategizer.rs similarity index 100% rename from app/storyteller_web/src/util/jobs/cache_miss_strategizer.rs rename to crates/app/storyteller_web/src/util/jobs/cache_miss_strategizer.rs diff --git a/app/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs b/crates/app/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs similarity index 100% rename from app/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs rename to crates/app/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs diff --git a/app/storyteller_web/src/util/jobs/mod.rs b/crates/app/storyteller_web/src/util/jobs/mod.rs similarity index 100% rename from app/storyteller_web/src/util/jobs/mod.rs rename to crates/app/storyteller_web/src/util/jobs/mod.rs diff --git a/app/storyteller_web/src/util/jobs/virtual_lfu_cache.rs b/crates/app/storyteller_web/src/util/jobs/virtual_lfu_cache.rs similarity index 100% rename from app/storyteller_web/src/util/jobs/virtual_lfu_cache.rs rename to crates/app/storyteller_web/src/util/jobs/virtual_lfu_cache.rs diff --git a/app/storyteller_web/src/util/markdown_to_html.rs b/crates/app/storyteller_web/src/util/markdown_to_html.rs similarity index 100% rename from app/storyteller_web/src/util/markdown_to_html.rs rename to crates/app/storyteller_web/src/util/markdown_to_html.rs diff --git a/app/storyteller_web/src/util/mod.rs b/crates/app/storyteller_web/src/util/mod.rs similarity index 100% rename from app/storyteller_web/src/util/mod.rs rename to crates/app/storyteller_web/src/util/mod.rs diff --git a/app/storyteller_web/src/util/noop_logger.rs b/crates/app/storyteller_web/src/util/noop_logger.rs similarity index 100% rename from app/storyteller_web/src/util/noop_logger.rs rename to crates/app/storyteller_web/src/util/noop_logger.rs diff --git a/app/storyteller_web/src/util/random_crockford_token.rs b/crates/app/storyteller_web/src/util/random_crockford_token.rs similarity index 100% rename from app/storyteller_web/src/util/random_crockford_token.rs rename to crates/app/storyteller_web/src/util/random_crockford_token.rs diff --git a/app/storyteller_web/src/util/random_prefix_crockford_token.rs b/crates/app/storyteller_web/src/util/random_prefix_crockford_token.rs similarity index 100% rename from app/storyteller_web/src/util/random_prefix_crockford_token.rs rename to crates/app/storyteller_web/src/util/random_prefix_crockford_token.rs diff --git a/app/storyteller_web/src/util/random_uuid.rs b/crates/app/storyteller_web/src/util/random_uuid.rs similarity index 100% rename from app/storyteller_web/src/util/random_uuid.rs rename to crates/app/storyteller_web/src/util/random_uuid.rs diff --git a/app/storyteller_web/src/util/redis/mod.rs b/crates/app/storyteller_web/src/util/redis/mod.rs similarity index 100% rename from app/storyteller_web/src/util/redis/mod.rs rename to crates/app/storyteller_web/src/util/redis/mod.rs diff --git a/app/storyteller_web/src/util/redis/redis_job_status_logger.rs b/crates/app/storyteller_web/src/util/redis/redis_job_status_logger.rs similarity index 100% rename from app/storyteller_web/src/util/redis/redis_job_status_logger.rs rename to crates/app/storyteller_web/src/util/redis/redis_job_status_logger.rs diff --git a/app/storyteller_web/src/util/redis/redis_keys.rs b/crates/app/storyteller_web/src/util/redis/redis_keys.rs similarity index 100% rename from app/storyteller_web/src/util/redis/redis_keys.rs rename to crates/app/storyteller_web/src/util/redis/redis_keys.rs diff --git a/app/storyteller_web/src/util/semi_persistent_cache_dir.rs b/crates/app/storyteller_web/src/util/semi_persistent_cache_dir.rs similarity index 100% rename from app/storyteller_web/src/util/semi_persistent_cache_dir.rs rename to crates/app/storyteller_web/src/util/semi_persistent_cache_dir.rs diff --git a/app/storyteller_web/src/validations/cashapp_username.rs b/crates/app/storyteller_web/src/validations/cashapp_username.rs similarity index 100% rename from app/storyteller_web/src/validations/cashapp_username.rs rename to crates/app/storyteller_web/src/validations/cashapp_username.rs diff --git a/app/storyteller_web/src/validations/check_for_slurs.rs b/crates/app/storyteller_web/src/validations/check_for_slurs.rs similarity index 100% rename from app/storyteller_web/src/validations/check_for_slurs.rs rename to crates/app/storyteller_web/src/validations/check_for_slurs.rs diff --git a/app/storyteller_web/src/validations/discord_username.rs b/crates/app/storyteller_web/src/validations/discord_username.rs similarity index 100% rename from app/storyteller_web/src/validations/discord_username.rs rename to crates/app/storyteller_web/src/validations/discord_username.rs diff --git a/app/storyteller_web/src/validations/github_username.rs b/crates/app/storyteller_web/src/validations/github_username.rs similarity index 100% rename from app/storyteller_web/src/validations/github_username.rs rename to crates/app/storyteller_web/src/validations/github_username.rs diff --git a/app/storyteller_web/src/validations/ip_addresses.rs b/crates/app/storyteller_web/src/validations/ip_addresses.rs similarity index 100% rename from app/storyteller_web/src/validations/ip_addresses.rs rename to crates/app/storyteller_web/src/validations/ip_addresses.rs diff --git a/app/storyteller_web/src/validations/mod.rs b/crates/app/storyteller_web/src/validations/mod.rs similarity index 100% rename from app/storyteller_web/src/validations/mod.rs rename to crates/app/storyteller_web/src/validations/mod.rs diff --git a/app/storyteller_web/src/validations/model_uploads.rs b/crates/app/storyteller_web/src/validations/model_uploads.rs similarity index 100% rename from app/storyteller_web/src/validations/model_uploads.rs rename to crates/app/storyteller_web/src/validations/model_uploads.rs diff --git a/app/storyteller_web/src/validations/passwords.rs b/crates/app/storyteller_web/src/validations/passwords.rs similarity index 100% rename from app/storyteller_web/src/validations/passwords.rs rename to crates/app/storyteller_web/src/validations/passwords.rs diff --git a/app/storyteller_web/src/validations/twitch_username.rs b/crates/app/storyteller_web/src/validations/twitch_username.rs similarity index 100% rename from app/storyteller_web/src/validations/twitch_username.rs rename to crates/app/storyteller_web/src/validations/twitch_username.rs diff --git a/app/storyteller_web/src/validations/twitter_username.rs b/crates/app/storyteller_web/src/validations/twitter_username.rs similarity index 100% rename from app/storyteller_web/src/validations/twitter_username.rs rename to crates/app/storyteller_web/src/validations/twitter_username.rs diff --git a/app/storyteller_web/src/validations/username.rs b/crates/app/storyteller_web/src/validations/username.rs similarity index 100% rename from app/storyteller_web/src/validations/username.rs rename to crates/app/storyteller_web/src/validations/username.rs diff --git a/app/storyteller_web/src/validations/username_reservations.rs b/crates/app/storyteller_web/src/validations/username_reservations.rs similarity index 100% rename from app/storyteller_web/src/validations/username_reservations.rs rename to crates/app/storyteller_web/src/validations/username_reservations.rs diff --git a/app/storyteller_web/src/validations/website_url.rs b/crates/app/storyteller_web/src/validations/website_url.rs similarity index 100% rename from app/storyteller_web/src/validations/website_url.rs rename to crates/app/storyteller_web/src/validations/website_url.rs diff --git a/app/storyteller_web/src/w2l_download_job.rs b/crates/app/storyteller_web/src/w2l_download_job.rs similarity index 100% rename from app/storyteller_web/src/w2l_download_job.rs rename to crates/app/storyteller_web/src/w2l_download_job.rs diff --git a/app/storyteller_web/src/w2l_inference_job.rs b/crates/app/storyteller_web/src/w2l_inference_job.rs similarity index 100% rename from app/storyteller_web/src/w2l_inference_job.rs rename to crates/app/storyteller_web/src/w2l_inference_job.rs diff --git a/app/lib/config/Cargo.toml b/crates/lib/config/Cargo.toml similarity index 100% rename from app/lib/config/Cargo.toml rename to crates/lib/config/Cargo.toml diff --git a/app/lib/config/src/lib.rs b/crates/lib/config/src/lib.rs similarity index 100% rename from app/lib/config/src/lib.rs rename to crates/lib/config/src/lib.rs diff --git a/app/lib/config/src/shared_constants.rs b/crates/lib/config/src/shared_constants.rs similarity index 100% rename from app/lib/config/src/shared_constants.rs rename to crates/lib/config/src/shared_constants.rs diff --git a/app/lib/container_common/Cargo.toml b/crates/lib/container_common/Cargo.toml similarity index 100% rename from app/lib/container_common/Cargo.toml rename to crates/lib/container_common/Cargo.toml diff --git a/app/storyteller_web/src/util/anyhow_result.rs b/crates/lib/container_common/src/anyhow_result.rs similarity index 100% rename from app/storyteller_web/src/util/anyhow_result.rs rename to crates/lib/container_common/src/anyhow_result.rs diff --git a/app/lib/container_common/src/lib.rs b/crates/lib/container_common/src/lib.rs similarity index 100% rename from app/lib/container_common/src/lib.rs rename to crates/lib/container_common/src/lib.rs diff --git a/app/lib/http_server_common/Cargo.toml b/crates/lib/http_server_common/Cargo.toml similarity index 100% rename from app/lib/http_server_common/Cargo.toml rename to crates/lib/http_server_common/Cargo.toml diff --git a/app/lib/http_server_common/src/cors.rs b/crates/lib/http_server_common/src/cors.rs similarity index 100% rename from app/lib/http_server_common/src/cors.rs rename to crates/lib/http_server_common/src/cors.rs diff --git a/app/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs b/crates/lib/http_server_common/src/endpoints/default_route_404.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs rename to crates/lib/http_server_common/src/endpoints/default_route_404.rs diff --git a/app/lib/http_server_common/src/endpoints/mod.rs b/crates/lib/http_server_common/src/endpoints/mod.rs similarity index 100% rename from app/lib/http_server_common/src/endpoints/mod.rs rename to crates/lib/http_server_common/src/endpoints/mod.rs diff --git a/app/storyteller_web/src/http_server/endpoints/misc/root_index.rs b/crates/lib/http_server_common/src/endpoints/root_index.rs similarity index 100% rename from app/storyteller_web/src/http_server/endpoints/misc/root_index.rs rename to crates/lib/http_server_common/src/endpoints/root_index.rs diff --git a/app/lib/http_server_common/src/lib.rs b/crates/lib/http_server_common/src/lib.rs similarity index 100% rename from app/lib/http_server_common/src/lib.rs rename to crates/lib/http_server_common/src/lib.rs diff --git a/app/lib/http_server_common/src/response/mod.rs b/crates/lib/http_server_common/src/response/mod.rs similarity index 100% rename from app/lib/http_server_common/src/response/mod.rs rename to crates/lib/http_server_common/src/response/mod.rs diff --git a/app/lib/http_server_common/src/response/response_error_helpers.rs b/crates/lib/http_server_common/src/response/response_error_helpers.rs similarity index 100% rename from app/lib/http_server_common/src/response/response_error_helpers.rs rename to crates/lib/http_server_common/src/response/response_error_helpers.rs diff --git a/app/storyteller_web/src/http_server/web_utils/response_success_helpers.rs b/crates/lib/http_server_common/src/response/response_success_helpers.rs similarity index 100% rename from app/storyteller_web/src/http_server/web_utils/response_success_helpers.rs rename to crates/lib/http_server_common/src/response/response_success_helpers.rs diff --git a/app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs b/crates/lib/http_server_common/src/response/serialize_as_json_error.rs similarity index 100% rename from app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs rename to crates/lib/http_server_common/src/response/serialize_as_json_error.rs From 6cb6c83dbaceda65ff3fe4fda242ebf0002bcd62 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 12 Nov 2021 23:58:10 -0500 Subject: [PATCH 0671/1042] clean up dependencies --- crates/app/obs_gateway/Cargo.toml | 21 +++--------- crates/app/storyteller_web/Cargo.toml | 41 +++++------------------- crates/lib/http_server_common/Cargo.toml | 3 +- 3 files changed, 14 insertions(+), 51 deletions(-) diff --git a/crates/app/obs_gateway/Cargo.toml b/crates/app/obs_gateway/Cargo.toml index fa9d81175d..6f47012566 100644 --- a/crates/app/obs_gateway/Cargo.toml +++ b/crates/app/obs_gateway/Cargo.toml @@ -26,14 +26,11 @@ chrono = { version = "0.4.19", features = ["serde"] } derive_more = "0.99.14" dotenv = "0.15.0" easyenv = "0.0.6" -futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool -futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate hex = "0.4.3" hostname = "0.3.1" lazy_static = "1.4.0" lfu = "0.2.5" log = "0.4.14" -#newrelic = "0.2.2" #newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } rand = "0.8.3" regex = "1.5.4" @@ -41,26 +38,19 @@ time = "0.2.26" uuid = { version = "0.8.2", features = ["v4"] } # Runtime -# NB(1): We switched to Tokio from `async-std` for broader ecosystem support. -#async-std = { version = "1.6", features = [ "attributes" ] } -# NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: -# https://github.com/durch/rust-s3/issues/198 -#tokio = { version = "1.6.0", features = ["macros"] } -#tokio = { version = "0.2.25", features = ["macros"] } -# NB(3,2021-11-07): Actually, maybe we can specify a version that diverges from -# the rest of the monorepo. The Twitch client needs a 1.x runtime. -#tokio = { version = "1.6.0", features = ["macros", "rt-multi-thread"] } -# Async runtime, Futures, etc. +# NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.10.0", features = ["rt-multi-thread", "macros", "time"] } + +# Futures +futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool futures-util = "0.3.17" +futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate # MySql Database # "offline" supports typesafe builds without a live database # "rustls" doesn't appear to work with IP addresses, per: # https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 #sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } -# NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade -#sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } # Redis Database @@ -74,7 +64,6 @@ twitch_oauth2 = "0.6.0-rc.2" # HTTP reqwest = { version = "0.11.4" } - native-tls = "0.2.8" # In-memory caching diff --git a/crates/app/storyteller_web/Cargo.toml b/crates/app/storyteller_web/Cargo.toml index 7ff25f0492..5e32f25947 100644 --- a/crates/app/storyteller_web/Cargo.toml +++ b/crates/app/storyteller_web/Cargo.toml @@ -48,15 +48,11 @@ chrono = { version = "0.4.19", features = ["serde"] } derive_more = "0.99.14" dotenv = "0.15.0" easyenv = "0.0.6" -futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool -futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate -futures-util = "0.3.9" hex = "0.4.3" hostname = "0.3.1" lazy_static = "1.4.0" lfu = "0.2.5" log = "0.4.14" -#newrelic = "0.2.2" newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } rand = "0.8.3" regex = "1.5.4" @@ -64,21 +60,19 @@ time = "0.2.26" uuid = { version = "0.8.2", features = ["v4"] } # Runtime -# NB(1): We switched to Tokio from `async-std` for broader ecosystem support. -#async-std = { version = "1.6", features = [ "attributes" ] } -# NB(2): Really unfortunate. We can't use Tokio 1.x due to a bug in `rust-s3` that I filed: -# https://github.com/durch/rust-s3/issues/198 +# NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.6.0", features = ["macros"] } -#tokio = { version = "0.2.25", features = ["macros"] } + +# Futures +futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool +futures-util = "0.3.9" +futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate # MySql Database # "offline" supports typesafe builds without a live database # "rustls" doesn't appear to work with IP addresses, per: # https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 -#sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } -# NB: We can't use `sqlx` 0.5 due to the `rust-s3` bug that requires a Tokio 1.x -> 0.2 downgrade sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } -#sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } # Redis Database limitation = "0.1.1" # Redis rate limiter @@ -90,43 +84,24 @@ lru_time_cache = "0.11.11" # Files data-encoding = "2.3.2" ring = "0.17.0-alpha.10" -# NB: This is the source of all of our problems. `rust-s3` can't upload to object names with directories -# in the 0.27.* series, so we have to downgrade to 0.26, which includes a broad downgrade from Tokio 1.0 -> 0.2 -#rust-s3 = "0.27.0-rc4" -#rust-s3 = { version = "0.27.0-rc4", features = ["with-async-std"], default-features = false } -#rust-s3 = { version = "0.26" } -#rust-s3 = { version = "0.28", features = ["with-async-std"], default-features = false } rust-s3 = { version = "0.28", features = ["never-encode-slash"] } infer = "0.5.0" subprocess = "0.2.6" tempdir = "0.3.7" -# Web -#actix = "0.10" -#actix-cors = "0.5.4" -#actix-files= "0.5" -#actix-http = "2.2.0" -#actix-rt = "2.2.0" -#actix-service = "1.0.6" -#actix-utils = "3.0.0" -#actix-web = "3.3.2" -#actix-web-actors = "3" -#jwt = "0.13.0" - actix = "0.12.0" actix-cors = "0.6.0-beta.3" actix-files= "0.6.0-beta.8" actix-http = "3.0.0-beta.11" -actix-multipart = "0.4.0-beta.7" # TODO: Ready for 0.4 upgrade +actix-multipart = "0.4.0-beta.7" actix-rt = "2.4.0" actix-service = "2.0.1" actix-utils = "3.0.0" -actix-web = "4.0.0-beta.7" +actix-web = "4.0.0-beta.10" actix-web-actors = "4.0.0-beta.7" jwt = "0.13.0" - # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime # NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. hyper = { version = "0.14.14" } diff --git a/crates/lib/http_server_common/Cargo.toml b/crates/lib/http_server_common/Cargo.toml index 19ac74b6aa..1dc4b28a0c 100644 --- a/crates/lib/http_server_common/Cargo.toml +++ b/crates/lib/http_server_common/Cargo.toml @@ -24,8 +24,7 @@ log = "0.4.14" # Web actix-cors = "0.6.0-beta.3" actix-http = "3.0.0-beta.11" -#actix-web = "4.0.0-beta.10" -actix-web = "4.0.0-beta.7" +actix-web = "4.0.0-beta.10" # Serialization serde = "1.0.125" From 65934b20418f290c25a3fb9f2fad0deb9eb791ba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 13 Nov 2021 00:46:34 -0500 Subject: [PATCH 0672/1042] IP polling job works now. Migrated to Tokio runner --- crates/app/storyteller_web/Cargo.toml | 2 +- crates/app/storyteller_web/src/main.rs | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/crates/app/storyteller_web/Cargo.toml b/crates/app/storyteller_web/Cargo.toml index 5e32f25947..46deec8d45 100644 --- a/crates/app/storyteller_web/Cargo.toml +++ b/crates/app/storyteller_web/Cargo.toml @@ -64,7 +64,7 @@ uuid = { version = "0.8.2", features = ["v4"] } tokio = { version = "1.6.0", features = ["macros"] } # Futures -futures = { version = "0.3.15", features = ["thread-pool"] } # feature for ThreadPool +futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool futures-util = "0.3.9" futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate diff --git a/crates/app/storyteller_web/src/main.rs b/crates/app/storyteller_web/src/main.rs index 93007626ad..e391a9cd8c 100644 --- a/crates/app/storyteller_web/src/main.rs +++ b/crates/app/storyteller_web/src/main.rs @@ -38,8 +38,10 @@ use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; +use crate::http_server::endpoints::misc::default_route_404::default_route_404; use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha_easy_handler; use crate::http_server::endpoints::misc::enable_alpha_handler::enable_alpha_handler; +use crate::http_server::endpoints::misc::root_index::get_root_index; use crate::http_server::endpoints::moderation::approval::pending_w2l_templates::get_pending_w2l_templates_handler; use crate::http_server::endpoints::moderation::ip_bans::add_ip_ban::add_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::delete_ip_ban::delete_ip_ban_handler; @@ -101,7 +103,6 @@ use crate::util::buckets::bucket_client::BucketClient; use crate::util::caching::single_item_ttl_cache::SingleItemTtlCache; use crate::util::encrypted_sort_id::SortKeyCrypto; use futures::Future; -use futures::executor::ThreadPool; use http_server_common::cors::build_common_cors_config; use limitation::Limiter; use log::{info}; @@ -112,8 +113,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; -use crate::http_server::endpoints::misc::root_index::get_root_index; -use crate::http_server::endpoints::misc::default_route_404::default_route_404; +use tokio::runtime::Runtime; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -260,13 +260,12 @@ async fn main() -> AnyhowResult<()> { let ip_banlist2 = ip_banlist.clone(); let mysql_pool3 = pool.clone(); - // Necessary to run our background work. - // I tried getting this running with actix, tokio, by other means, - // but didn't have much luck. The task didn't respond to ctrl-c - // interrupts. - let thread_pool = ThreadPool::new()?; + // Background jobs. + info!("Spawning IP ban polling thread."); - thread_pool.spawn_ok(async { + let tokio_runtime = Runtime::new()?; + + tokio_runtime.spawn(async { poll_ip_bans(ip_banlist2, mysql_pool3).await; }); From 1e1906711336bbb62c155b8b2cf57c11eb16554e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 15 Nov 2021 00:13:25 -0500 Subject: [PATCH 0673/1042] Repair model/ files that were overwritten for testing --- models/tts/vocodes_common.py | 30 ++++++++++----------- models/tts/vocodes_server.py | 52 ++++++++++++++---------------------- 2 files changed, 35 insertions(+), 47 deletions(-) diff --git a/models/tts/vocodes_common.py b/models/tts/vocodes_common.py index 0f740a4fce..4b2160831b 100644 --- a/models/tts/vocodes_common.py +++ b/models/tts/vocodes_common.py @@ -3,26 +3,26 @@ sys.path.append('waveglow/') import shutil import json -#import numpy as np -#import torch +import numpy as np +import torch -#from tacotron2.hparams import create_hparams -#from tacotron2.model import Tacotron2 -#from tacotron2.layers import TacotronSTFT -#from tacotron2.audio_processing import griffin_lim -#from tacotron2.text import text_to_sequence +from tacotron2.hparams import create_hparams +from tacotron2.model import Tacotron2 +from tacotron2.layers import TacotronSTFT +from tacotron2.audio_processing import griffin_lim +from tacotron2.text import text_to_sequence -#from waveglow.denoiser import Denoiser +from waveglow.denoiser import Denoiser -#from scipy.io.wavfile import write as write_wav +from scipy.io.wavfile import write as write_wav # Hifi-Gan -#from hifigan.env import AttrDict -#from hifigan.models import Generator -#from hifigan.denoiser import Denoiser as HifiDenoiser -#from hifigan.meldataset import mel_spectrogram -#import resampy -#import scipy +from hifigan.env import AttrDict +from hifigan.models import Generator +from hifigan.denoiser import Denoiser as HifiDenoiser +from hifigan.meldataset import mel_spectrogram +import resampy +import scipy # For metadata import subprocess diff --git a/models/tts/vocodes_server.py b/models/tts/vocodes_server.py index 6dde8583c5..e940b9b240 100755 --- a/models/tts/vocodes_server.py +++ b/models/tts/vocodes_server.py @@ -9,31 +9,30 @@ Brandon Thomas 2019-07-29 import argparse import falcon import io -#import librosa -#import numpy as np +import librosa +import numpy as np import os -#import scipy -#import soundfile -#import tensorflow as tf +import scipy +import soundfile +import tensorflow as tf import subprocess import tempfile import json -import shutil from falcon_multipart.middleware import MultipartMiddleware #from model import CycleGAN #from preprocess import * from wsgiref import simple_server -#from vocodes_common import TacotronWaveglowPipeline -#from vocodes_common import print_gpu_info -#from vocodes_common import load_tacotron_model -#from vocodes_common import load_waveglow_model -#from vocodes_common import load_hifigan_model +from vocodes_common import TacotronWaveglowPipeline +from vocodes_common import print_gpu_info +from vocodes_common import load_tacotron_model +from vocodes_common import load_waveglow_model +from vocodes_common import load_hifigan_model -#print("TensorFlow version: {}".format(tf.version.VERSION)) +print("TensorFlow version: {}".format(tf.version.VERSION)) -#print_gpu_info() +print_gpu_info() INDEX_HTML = ''' @@ -198,7 +197,7 @@ INDEX_HTML = ''' #tacotron = load_tacotron_model('/home/bt/models/tacotron2/tacotron2_uberduck_Noire.pt') #waveglow = load_waveglow_model('/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt') -#pipeline = TacotronWaveglowPipeline() +pipeline = TacotronWaveglowPipeline() class IndexHandler(): def on_get(self, request, response): @@ -236,14 +235,14 @@ class ApiHandler(): print('maybe_clear_synthesizer_checkpoint_path: {}'.format( maybe_clear_synthesizer_checkpoint_path)) - #pipeline.maybe_load_waveglow_model(waveglow_vocoder_checkpoint_path) - #pipeline.maybe_load_hifigan(hifigan_vocoder_checkpoint_path) - #pipeline.maybe_load_hifigan_super_resolution(hifigan_superres_vocoder_checkpoint_path) + pipeline.maybe_load_waveglow_model(waveglow_vocoder_checkpoint_path) + pipeline.maybe_load_hifigan(hifigan_vocoder_checkpoint_path) + pipeline.maybe_load_hifigan_super_resolution(hifigan_superres_vocoder_checkpoint_path) - #if maybe_clear_synthesizer_checkpoint_path: - # pipeline.uncache_tacotron_model(maybe_clear_synthesizer_checkpoint_path) + if maybe_clear_synthesizer_checkpoint_path: + pipeline.uncache_tacotron_model(maybe_clear_synthesizer_checkpoint_path) - #pipeline.maybe_load_tacotron_model_to_cache(synthesizer_checkpoint_path) + pipeline.maybe_load_tacotron_model_to_cache(synthesizer_checkpoint_path) inference_args = { 'raw_text': inference_text, @@ -254,18 +253,7 @@ class ApiHandler(): 'output_metadata_filename': output_metadata_filename, } - #pipeline.infer(inference_args) - - metadata_temp = "/Users/bt/dev/storyteller/storyteller-web/temp/metadata.json" - audio_temp = "/Users/bt/dev/storyteller/storyteller-web/temp/test.wav" - spectrogram_temp = "/Users/bt/dev/storyteller/storyteller-web/temp/spectrogram.json" - - args = inference_args - - print('TEMP COPYING FAKE FILES...') - shutil.copyfile(metadata_temp, args['output_metadata_filename']) - shutil.copyfile(audio_temp, args['output_audio_filename']) - shutil.copyfile(spectrogram_temp, args['output_spectrogram_filename']) + pipeline.infer(inference_args) def main(): parser = argparse.ArgumentParser() From 7e5f2b8c10dc48adacb571c622272d93b96f4e4c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 15 Nov 2021 00:30:37 -0500 Subject: [PATCH 0674/1042] cleanup bucket client --- .../storyteller_web/src/util/buckets/bucket_client.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/crates/app/storyteller_web/src/util/buckets/bucket_client.rs b/crates/app/storyteller_web/src/util/buckets/bucket_client.rs index 72ff1b2bf1..7f06de635b 100644 --- a/crates/app/storyteller_web/src/util/buckets/bucket_client.rs +++ b/crates/app/storyteller_web/src/util/buckets/bucket_client.rs @@ -29,14 +29,6 @@ impl BucketClient { bucket_name: &str, optional_bucket_root: Option<&str>) -> anyhow::Result { - /*let credentials = Credentials::new( - Some(&access_key), - Some(&secret_key), - None, - None, - None, - )?;*/ - let credentials = Credentials { access_key: Some(access_key.to_string()), secret_key: Some(secret_key.to_string()), @@ -192,8 +184,6 @@ impl BucketClient { info!("downloading from bucket: {:?}", &object_path_str); - //let mut output_file = std::fs::File::create(filesystem_path)?; - //let mut output_file = async_std::fs::File::create(filesystem_path).await?; let mut output_file = tokio::fs::File::create(filesystem_path).await?; let status_code = self.bucket.get_object_stream(&object_path_str, &mut output_file).await?; From 807365bd6138c55779a0a75afbfbe2633de20b41 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 15 Nov 2021 00:32:24 -0500 Subject: [PATCH 0675/1042] fixup loc.sh --- loc.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/loc.sh b/loc.sh index babc6e8a00..50e23caf80 100755 --- a/loc.sh +++ b/loc.sh @@ -3,9 +3,9 @@ wc -l \ Cargo.toml \ Dockerfile \ migrations/*/*.sql \ - app/*/*/*.rs \ - app/*/*/*/*.rs \ - app/*/*/*/*/*.rs \ - app/*/*/*/*/*/*.rs \ - app/*/*/*/*/*/*/*.rs \ + crates/*/*/*/*.rs \ + crates/*/*/*/*/*.rs \ + crates/*/*/*/*/*/*.rs \ + crates/*/*/*/*/*/*/*.rs \ + crates/*/*/*/*/*/*/*/*.rs \ | sort -n From c7989106b7c280812cbbc6cbe812fdf202b827bc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 15 Nov 2021 00:33:57 -0500 Subject: [PATCH 0676/1042] Attempt to fix dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ec3f882214..1eb731ebb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -152,7 +152,7 @@ WORKDIR /tmp COPY Cargo.lock . COPY Cargo.toml . COPY sqlx-data.json . -COPY app/ ./app +COPY crates/ ./crates COPY vendor/ ./vendor COPY db/ ./db COPY migrations/ ./migrations From 9b2c0edcc5bfed80dc6ba7a67962cca1bf885cfd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 15 Nov 2021 02:28:25 -0500 Subject: [PATCH 0677/1042] Twitch oauth enroll endpoint --- Cargo.lock | 1 + .../obs_gateway/src/endpoints/oauth_begin.rs | 55 +++- crates/app/obs_gateway/src/main.rs | 301 +++++++++--------- crates/app/obs_gateway/src/server_state.rs | 31 ++ crates/lib/http_server_common/Cargo.toml | 3 + .../http_server_common/src/response/mod.rs | 1 + .../src/response/to_json_success_response.rs | 11 + 7 files changed, 232 insertions(+), 171 deletions(-) create mode 100644 crates/app/obs_gateway/src/server_state.rs create mode 100644 crates/lib/http_server_common/src/response/to_json_success_response.rs diff --git a/Cargo.lock b/Cargo.lock index 7f66a5dba5..a2699de538 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1865,6 +1865,7 @@ dependencies = [ "anyhow", "base64 0.13.0", "chrono", + "container_common", "dotenv", "log", "serde", diff --git a/crates/app/obs_gateway/src/endpoints/oauth_begin.rs b/crates/app/obs_gateway/src/endpoints/oauth_begin.rs index 236a96debc..4f4cd5828c 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_begin.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_begin.rs @@ -2,16 +2,32 @@ use actix_http::{StatusCode, header}; use actix_web::{HttpRequest, web, HttpResponse, ResponseError, HttpResponseBuilder}; use crate::ObsGatewayServerState; use http_server_common::response::response_error_helpers::to_simple_json_error; +use http_server_common::response::response_success_helpers::simple_json_success; use log::info; use std::fmt; use std::sync::Arc; -use http_server_common::response::response_success_helpers::simple_json_success; +use twitch_oauth2::tokens::UserTokenBuilder; +use twitch_oauth2::{Scope, ClientSecret}; +use twitch_api2::twitch_oauth2::ClientId; +use http_server_common::response::to_json_success_response::to_json_success_response; + +#[derive(Serialize)] +pub struct OauthBeginEnrollResult { + redirect_url: String, +} #[derive(Debug)] pub enum OauthBeginEnrollError { ServerError, } +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for OauthBeginEnrollError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "OauthBeginEnrollError") + } +} + impl ResponseError for OauthBeginEnrollError { fn status_code(&self) -> StatusCode { match *self { @@ -24,18 +40,11 @@ impl ResponseError for OauthBeginEnrollError { Self::ServerError => "server error".to_string(), }; - //to_simple_json_error(&error_reason, self.status_code()) + to_simple_json_error(&error_reason, self.status_code()) - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body("TODO") - } -} - -// NB: Not using DeriveMore since Clion doesn't understand it. -impl fmt::Display for OauthBeginEnrollError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + //HttpResponseBuilder::new(self.status_code()) + // .set_header(header::CONTENT_TYPE, "application/json") + // .body("TODO") } } @@ -44,7 +53,25 @@ pub async fn oauth_begin_enroll( server_state: web::Data> ) -> Result { + let redirect_url = + twitch_oauth2::url::Url::parse(&server_state.twitch_oauth_secrets.redirect_url).unwrap(); - info!("oauth enrollment begin"); - Ok(simple_json_success()) + let client_id = ClientId::new(&server_state.twitch_oauth_secrets.client_id); + let client_secret = ClientSecret::new(&server_state.twitch_oauth_secrets.client_secret); + + // TODO: Not all scopes are needed! + let mut builder = UserTokenBuilder::new(client_id, client_secret, redirect_url) + .set_scopes(Scope::all()) + .force_verify(true); + + let (url, _csrf_token) = builder.generate_url(); + + let response = OauthBeginEnrollResult { + redirect_url: url.to_string(), + }; + + let response = to_json_success_response(&response) + .map_err(|_| OauthBeginEnrollError::ServerError)?; + + Ok(response) } diff --git a/crates/app/obs_gateway/src/main.rs b/crates/app/obs_gateway/src/main.rs index e69ece9f53..35c5cd7562 100644 --- a/crates/app/obs_gateway/src/main.rs +++ b/crates/app/obs_gateway/src/main.rs @@ -14,6 +14,7 @@ pub mod endpoints; pub mod pubsub_gateway; +pub mod server_state; pub mod twitch; pub mod util; @@ -47,31 +48,12 @@ use twitch_api2::pubsub::Topic; use twitch_api2::pubsub; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{AppAccessToken, Scope, TwitchToken, tokens::errors::AppAccessTokenError, ClientId, ClientSecret}; +use crate::server_state::{ObsGatewayServerState, EnvConfig, TwitchOauthSecrets}; -const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; +const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:54321"; pub type AnyhowResult = anyhow::Result; -/// State that is injected into every endpoint. -#[derive(Clone)] -pub struct ObsGatewayServerState { - /// Configuration from ENV vars. - /// Some of this might not be used. - pub env_config: EnvConfig, - pub hostname: String, -} - -#[derive(Clone)] -pub struct EnvConfig { - // Number of thread workers. - pub num_workers: usize, - pub bind_address: String, - pub cookie_domain: String, - pub cookie_secure: bool, - pub cookie_http_only: bool, - pub website_homepage_redirect: String, -} - #[actix_web::main] //#[tokio::main] async fn main() -> AnyhowResult<()> { @@ -122,126 +104,129 @@ async fn main() -> AnyhowResult<()> { //std::thread::sleep(Duration::from_secs(5000)); - // ==================== OAUTH FLOW ==================== - - println!("Oauth flow..."); - - let redirect_url = twitch_oauth2::url::Url::parse("http://localhost/test")?; - let mut builder = UserTokenBuilder::new(client_id, client_secret, redirect_url) - .set_scopes(Scope::all()) - .force_verify(true); - - //builder.add_scope(Scope::BitsRead); - - let (url, _csrf_token) = builder.generate_url(); - - println!("Go to this page: {}", url); - - - - let input = rpassword::prompt_password_stdout( - "Paste in the resulting adress after authenticating (input hidden): ", - )?; - - let u = twitch_oauth2::url::Url::parse(&input)?; - - let map: std::collections::HashMap<_, _> = u.query_pairs().collect(); - - let user_token = match (map.get("state"), map.get("code")) { - (Some(state), Some(code)) => { - let token = builder - .get_user_token( - &reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build()?, - state, - code, - ) - .await?; - println!("Got token: {:?}", token); - - token - } - _ => match (map.get("error"), map.get("error_description")) { - (std::option::Option::Some(error), std::option::Option::Some(error_description)) => { - anyhow::bail!( - "twitch errored with error: {} - {}", - error, - error_description - ); - } - _ => anyhow::bail!("invalid url passed"), - }, - }; - - let auth_token = user_token.access_token.as_str(); - - // ==================== PUBSUB SUBSCRIPTION AND MAIN LOOP ==================== - - let mut client = PollingTwitchWebsocketClient::new()?; - - println!("Connecting PubSub..."); - client.connect().await?; - - println!("Connected"); - - //println!("Starting polling thread..."); - //client.start_ping_thread().await; - - println!("Sending PING..."); - - client.send_ping().await?; - - println!("Try read next..."); - let r = client.try_next().await?; - println!("Result: {:?}", r); - - - println!("Begin LISTEN..."); - - let bit_topic = pubsub::channel_bits::ChannelBitsEventsV2 { - channel_id: user_id, - }.into_topic(); - - let topics = [bit_topic]; - - client.listen(&auth_token, &topics).await?; - - println!("Try read next..."); - let r = client.try_next().await?; - println!("Result: {:?}", r); - - /* - Result: Some(Message { data: ChannelBitsEventsV2 { topic: ChannelBitsEventsV2 { - channel_id: 652567283 }, reply: BitsEvent { data: BitsEventData { badge_entitlement: - Some(BadgeEntitlement { new_version: 100, previous_version: 1 }), bits_used: 100, - channel_id: "652567283", channel_name: "vocodes", chat_message: "Cheer100 testing the cheer", - context: Cheer, is_anonymous: false, time: "2021-09-27T04:30:53.627717085Z", - total_bits_used: 101, user_id: "650154491", user_name: "testytest512" }, - message_id: "793f745e-8f3e-5f71-bc41-4808c4b49a53", version: "1.0", is_anonymous: false } } }) - - Result: Some(Message { data: ChannelBitsEventsV2 { topic: ChannelBitsEventsV2 { - channel_id: 652567283 }, reply: BitsEvent { data: BitsEventData { badge_entitlement: None, - bits_used: 1, channel_id: "652567283", channel_name: "vocodes", - chat_message: "mario: hello everybody Cheer1", context: Cheer, is_anonymous: false, - time: "2021-09-27T04:47:32.943872952Z", total_bits_used: 102, user_id: "650154491", - user_name: "testytest512" }, message_id: "5844328d-495b-5585-9d6c-3c99949f9a0a", version: "1.0", - is_anonymous: false } } }) - */ - - - - println!("Try read next..."); - let r = client.try_next().await?; - println!("Result: {:?}", r); - - println!("Sleep..."); - std::thread::sleep(Duration::from_millis(30000)); - - //let text = msg.into_text()?; - //println!("Text: {}", text); - - // ======================================================== +// // ==================== OAUTH FLOW ==================== +// +// println!("Oauth flow..."); +// +// let redirect_url = twitch_oauth2::url::Url::parse("http://localhost/test")?; +// let mut builder = UserTokenBuilder::new( +// client_id.clone(), +// client_secret.clone(), +// redirect_url) +// .set_scopes(Scope::all()) +// .force_verify(true); +// +// //builder.add_scope(Scope::BitsRead); +// +// let (url, _csrf_token) = builder.generate_url(); +// +// println!("Go to this page: {}", url); +// +// +// +// let input = rpassword::prompt_password_stdout( +// "Paste in the resulting adress after authenticating (input hidden): ", +// )?; +// +// let u = twitch_oauth2::url::Url::parse(&input)?; +// +// let map: std::collections::HashMap<_, _> = u.query_pairs().collect(); +// +// let user_token = match (map.get("state"), map.get("code")) { +// (Some(state), Some(code)) => { +// let token = builder +// .get_user_token( +// &reqwest::Client::builder() +// .redirect(reqwest::redirect::Policy::none()) +// .build()?, +// state, +// code, +// ) +// .await?; +// println!("Got token: {:?}", token); +// +// token +// } +// _ => match (map.get("error"), map.get("error_description")) { +// (std::option::Option::Some(error), std::option::Option::Some(error_description)) => { +// anyhow::bail!( +// "twitch errored with error: {} - {}", +// error, +// error_description +// ); +// } +// _ => anyhow::bail!("invalid url passed"), +// }, +// }; +// +// let auth_token = user_token.access_token.as_str(); +// +// // ==================== PUBSUB SUBSCRIPTION AND MAIN LOOP ==================== +// +// let mut client = PollingTwitchWebsocketClient::new()?; +// +// println!("Connecting PubSub..."); +// client.connect().await?; +// +// println!("Connected"); +// +// //println!("Starting polling thread..."); +// //client.start_ping_thread().await; +// +// println!("Sending PING..."); +// +// client.send_ping().await?; +// +// println!("Try read next..."); +// let r = client.try_next().await?; +// println!("Result: {:?}", r); +// +// +// println!("Begin LISTEN..."); +// +// let bit_topic = pubsub::channel_bits::ChannelBitsEventsV2 { +// channel_id: user_id, +// }.into_topic(); +// +// let topics = [bit_topic]; +// +// client.listen(&auth_token, &topics).await?; +// +// println!("Try read next..."); +// let r = client.try_next().await?; +// println!("Result: {:?}", r); +// +// /* +// Result: Some(Message { data: ChannelBitsEventsV2 { topic: ChannelBitsEventsV2 { +// channel_id: 652567283 }, reply: BitsEvent { data: BitsEventData { badge_entitlement: +// Some(BadgeEntitlement { new_version: 100, previous_version: 1 }), bits_used: 100, +// channel_id: "652567283", channel_name: "vocodes", chat_message: "Cheer100 testing the cheer", +// context: Cheer, is_anonymous: false, time: "2021-09-27T04:30:53.627717085Z", +// total_bits_used: 101, user_id: "650154491", user_name: "testytest512" }, +// message_id: "793f745e-8f3e-5f71-bc41-4808c4b49a53", version: "1.0", is_anonymous: false } } }) +// +// Result: Some(Message { data: ChannelBitsEventsV2 { topic: ChannelBitsEventsV2 { +// channel_id: 652567283 }, reply: BitsEvent { data: BitsEventData { badge_entitlement: None, +// bits_used: 1, channel_id: "652567283", channel_name: "vocodes", +// chat_message: "mario: hello everybody Cheer1", context: Cheer, is_anonymous: false, +// time: "2021-09-27T04:47:32.943872952Z", total_bits_used: 102, user_id: "650154491", +// user_name: "testytest512" }, message_id: "5844328d-495b-5585-9d6c-3c99949f9a0a", version: "1.0", +// is_anonymous: false } } }) +// */ +// +// +// +// println!("Try read next..."); +// let r = client.try_next().await?; +// println!("Result: {:?}", r); +// +// println!("Sleep..."); +// std::thread::sleep(Duration::from_millis(30000)); +// +// //let text = msg.into_text()?; +// //println!("Text: {}", text); +// +// // ======================================================== // info!("Connecting to database..."); // @@ -285,6 +270,11 @@ async fn main() -> AnyhowResult<()> { cookie_http_only, website_homepage_redirect, }, + twitch_oauth_secrets: TwitchOauthSecrets { + client_id: secrets.app_client_id.clone(), + client_secret: secrets.app_client_secret.clone(), + redirect_url: "http://localhost:54321/twitch/oauth_redirect".to_string() + }, hostname: server_hostname, }; @@ -319,22 +309,19 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> .route(web::get().to(get_root_index)) ) // Twitch - .service( - web::resource("/twitch") - .route(web::get().to(ws_index)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - // /twitch_oauth_enroll - // /twitch_oauth_redirect - .service( - web::resource("/twitch_oauth_enroll") - .route(web::get().to(oauth_begin_enroll)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/twitch_oauth_redirect") - .route(web::get().to(oauth_begin_enroll)) - .route(web::head().to(|| HttpResponse::Ok())) + .service(web::scope("/twitch") + .service(web::resource("/oauth_enroll") + .route(web::get().to(oauth_begin_enroll)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/oauth_redirect") + .route(web::get().to(oauth_begin_enroll)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/websocket") + .route(web::get().to(ws_index)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) //.default_service( web::route().to(default_route_404)) }) diff --git a/crates/app/obs_gateway/src/server_state.rs b/crates/app/obs_gateway/src/server_state.rs new file mode 100644 index 0000000000..590f44be78 --- /dev/null +++ b/crates/app/obs_gateway/src/server_state.rs @@ -0,0 +1,31 @@ +use twitch_oauth2::{ClientId, ClientSecret}; + +/// State that is injected into every endpoint. +#[derive(Clone)] +pub struct ObsGatewayServerState { + /// Configuration from ENV vars. + /// Some of this might not be used. + pub env_config: EnvConfig, + pub hostname: String, + pub twitch_oauth_secrets: TwitchOauthSecrets, +} + +// TODO: Many of these do not need to be passed around past server init. +#[derive(Clone)] +pub struct EnvConfig { + // Number of thread workers. + pub num_workers: usize, + pub bind_address: String, + pub cookie_domain: String, + pub cookie_secure: bool, + pub cookie_http_only: bool, + pub website_homepage_redirect: String, +} + +/// Necessary to run the OAuth flow. +#[derive(Clone)] +pub struct TwitchOauthSecrets { + pub client_id: String, + pub client_secret: String, + pub redirect_url: String, +} diff --git a/crates/lib/http_server_common/Cargo.toml b/crates/lib/http_server_common/Cargo.toml index 1dc4b28a0c..d90d35fc76 100644 --- a/crates/lib/http_server_common/Cargo.toml +++ b/crates/lib/http_server_common/Cargo.toml @@ -14,6 +14,9 @@ path = "src/lib.rs" [dependencies] +# Internal +container_common = { path = "../container_common" } + # External anyhow = "1.0.40" base64 = "0.13.0" diff --git a/crates/lib/http_server_common/src/response/mod.rs b/crates/lib/http_server_common/src/response/mod.rs index 31ca5b7882..6bcb009c18 100644 --- a/crates/lib/http_server_common/src/response/mod.rs +++ b/crates/lib/http_server_common/src/response/mod.rs @@ -1,3 +1,4 @@ pub mod response_error_helpers; pub mod response_success_helpers; pub mod serialize_as_json_error; +pub mod to_json_success_response; diff --git a/crates/lib/http_server_common/src/response/to_json_success_response.rs b/crates/lib/http_server_common/src/response/to_json_success_response.rs new file mode 100644 index 0000000000..6d981a9d1b --- /dev/null +++ b/crates/lib/http_server_common/src/response/to_json_success_response.rs @@ -0,0 +1,11 @@ +use serde::Serialize; +use actix_web::HttpResponse; +use container_common::anyhow_result::AnyhowResult; + +pub fn to_json_success_response(response: &impl Serialize) -> AnyhowResult { + let body = serde_json::to_string(&response)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} From 75aa1f81aaeb5c08307cf9ea1f677c0b6cd73e47 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 15 Nov 2021 02:36:05 -0500 Subject: [PATCH 0678/1042] redirect oauth endpoint --- crates/app/obs_gateway/src/endpoints/mod.rs | 1 + .../obs_gateway/src/endpoints/oauth_begin.rs | 15 ++-- .../src/endpoints/oauth_begin_redirect.rs | 80 +++++++++++++++++++ crates/app/obs_gateway/src/main.rs | 5 ++ 4 files changed, 94 insertions(+), 7 deletions(-) create mode 100644 crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs diff --git a/crates/app/obs_gateway/src/endpoints/mod.rs b/crates/app/obs_gateway/src/endpoints/mod.rs index 436ec98e87..bb1fc37db2 100644 --- a/crates/app/obs_gateway/src/endpoints/mod.rs +++ b/crates/app/obs_gateway/src/endpoints/mod.rs @@ -1,2 +1,3 @@ pub mod oauth_begin; +pub mod oauth_begin_redirect; pub mod oauth_end; \ No newline at end of file diff --git a/crates/app/obs_gateway/src/endpoints/oauth_begin.rs b/crates/app/obs_gateway/src/endpoints/oauth_begin.rs index 4f4cd5828c..ff0ca8e840 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_begin.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_begin.rs @@ -3,13 +3,14 @@ use actix_web::{HttpRequest, web, HttpResponse, ResponseError, HttpResponseBuild use crate::ObsGatewayServerState; use http_server_common::response::response_error_helpers::to_simple_json_error; use http_server_common::response::response_success_helpers::simple_json_success; +use http_server_common::response::to_json_success_response::to_json_success_response; use log::info; +use log::warn; use std::fmt; use std::sync::Arc; +use twitch_api2::twitch_oauth2::ClientId; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{Scope, ClientSecret}; -use twitch_api2::twitch_oauth2::ClientId; -use http_server_common::response::to_json_success_response::to_json_success_response; #[derive(Serialize)] pub struct OauthBeginEnrollResult { @@ -41,10 +42,6 @@ impl ResponseError for OauthBeginEnrollError { }; to_simple_json_error(&error_reason, self.status_code()) - - //HttpResponseBuilder::new(self.status_code()) - // .set_header(header::CONTENT_TYPE, "application/json") - // .body("TODO") } } @@ -54,7 +51,11 @@ pub async fn oauth_begin_enroll( ) -> Result { let redirect_url = - twitch_oauth2::url::Url::parse(&server_state.twitch_oauth_secrets.redirect_url).unwrap(); + twitch_oauth2::url::Url::parse(&server_state.twitch_oauth_secrets.redirect_url) + .map_err(|e| { + warn!("Error parsing url: {:?}", e); + OauthBeginEnrollError::ServerError + })?; let client_id = ClientId::new(&server_state.twitch_oauth_secrets.client_id); let client_secret = ClientSecret::new(&server_state.twitch_oauth_secrets.client_secret); diff --git a/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs b/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs new file mode 100644 index 0000000000..2e90635790 --- /dev/null +++ b/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs @@ -0,0 +1,80 @@ +use actix_http::{StatusCode, header}; +use actix_web::{HttpRequest, web, HttpResponse, ResponseError, HttpResponseBuilder}; +use crate::ObsGatewayServerState; +use http_server_common::response::response_error_helpers::to_simple_json_error; +use http_server_common::response::response_success_helpers::simple_json_success; +use http_server_common::response::to_json_success_response::to_json_success_response; +use log::info; +use log::warn; +use std::fmt; +use std::sync::Arc; +use twitch_api2::twitch_oauth2::ClientId; +use twitch_oauth2::tokens::UserTokenBuilder; +use twitch_oauth2::{Scope, ClientSecret}; + +#[derive(Serialize)] +pub struct OauthBeginEnrollResult { + redirect_url: String, +} + +#[derive(Debug)] +pub enum OauthBeginEnrollError { + ServerError, +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for OauthBeginEnrollError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "OauthBeginEnrollError") + } +} + +impl ResponseError for OauthBeginEnrollError { + fn status_code(&self) -> StatusCode { + match *self { + Self::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + Self::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn oauth_begin_enroll_redirect( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + let redirect_url = + twitch_oauth2::url::Url::parse(&server_state.twitch_oauth_secrets.redirect_url) + .map_err(|e| { + warn!("Error parsing url: {:?}", e); + OauthBeginEnrollError::ServerError + })?; + + let client_id = ClientId::new(&server_state.twitch_oauth_secrets.client_id); + let client_secret = ClientSecret::new(&server_state.twitch_oauth_secrets.client_secret); + + // TODO: Not all scopes are needed! + let mut builder = UserTokenBuilder::new(client_id, client_secret, redirect_url) + .set_scopes(Scope::all()) + .force_verify(true); + + let (url, _csrf_token) = builder.generate_url(); + + let response = OauthBeginEnrollResult { + redirect_url: url.to_string(), + }; + + let response = to_json_success_response(&response) + .map_err(|_| OauthBeginEnrollError::ServerError)?; + + Ok(HttpResponse::Found() + .append_header((header::LOCATION, url.to_string())) + .finish()) +} diff --git a/crates/app/obs_gateway/src/main.rs b/crates/app/obs_gateway/src/main.rs index 35c5cd7562..b0446dea35 100644 --- a/crates/app/obs_gateway/src/main.rs +++ b/crates/app/obs_gateway/src/main.rs @@ -49,6 +49,7 @@ use twitch_api2::pubsub; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{AppAccessToken, Scope, TwitchToken, tokens::errors::AppAccessTokenError, ClientId, ClientSecret}; use crate::server_state::{ObsGatewayServerState, EnvConfig, TwitchOauthSecrets}; +use crate::endpoints::oauth_begin_redirect::oauth_begin_enroll_redirect; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:54321"; @@ -314,6 +315,10 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> .route(web::get().to(oauth_begin_enroll)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/oauth_enroll_redirect") + .route(web::get().to(oauth_begin_enroll_redirect)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/oauth_redirect") .route(web::get().to(oauth_begin_enroll)) .route(web::head().to(|| HttpResponse::Ok())) From 6bf35a83816320c2df01ba9e87e83ec0ceb2a500 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 16 Nov 2021 06:28:57 -0500 Subject: [PATCH 0679/1042] oauth enrollment wip --- .../obs_gateway/src/endpoints/oauth_begin.rs | 7 +++- .../src/endpoints/oauth_begin_redirect.rs | 7 +++- .../obs_gateway/src/endpoints/oauth_end.rs | 42 +++++++++++++++++++ crates/app/obs_gateway/src/main.rs | 9 +++- 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/crates/app/obs_gateway/src/endpoints/oauth_begin.rs b/crates/app/obs_gateway/src/endpoints/oauth_begin.rs index ff0ca8e840..a4c3c8a715 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_begin.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_begin.rs @@ -62,7 +62,12 @@ pub async fn oauth_begin_enroll( // TODO: Not all scopes are needed! let mut builder = UserTokenBuilder::new(client_id, client_secret, redirect_url) - .set_scopes(Scope::all()) + .set_scopes(vec![ + Scope::BitsRead, + Scope::ChannelReadSubscriptions, + Scope::ChatEdit, + Scope::ChatRead, + ]) .force_verify(true); let (url, _csrf_token) = builder.generate_url(); diff --git a/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs b/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs index 2e90635790..6db0746597 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs @@ -62,7 +62,12 @@ pub async fn oauth_begin_enroll_redirect( // TODO: Not all scopes are needed! let mut builder = UserTokenBuilder::new(client_id, client_secret, redirect_url) - .set_scopes(Scope::all()) + .set_scopes(vec![ + Scope::BitsRead, + Scope::ChannelReadSubscriptions, + Scope::ChatEdit, + Scope::ChatRead, + ]) .force_verify(true); let (url, _csrf_token) = builder.generate_url(); diff --git a/crates/app/obs_gateway/src/endpoints/oauth_end.rs b/crates/app/obs_gateway/src/endpoints/oauth_end.rs index e69de29bb2..84295ad078 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_end.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_end.rs @@ -0,0 +1,42 @@ +use actix_http::StatusCode; +use actix_web::{HttpResponse, HttpRequest, web, ResponseError}; +use crate::server_state::ObsGatewayServerState; +use http_server_common::response::response_error_helpers::to_simple_json_error; +use std::fmt; +use std::sync::Arc; + +#[derive(Debug)] +pub enum OauthEndEnrollFromRedirectError { + ServerError, +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for OauthEndEnrollFromRedirectError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "OauthEndEnrollFromRedirectError") + } +} + +impl ResponseError for OauthEndEnrollFromRedirectError { + fn status_code(&self) -> StatusCode { + match *self { + Self::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + Self::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn oauth_end_enroll_from_redirect( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + Ok(HttpResponse::Ok() + .body("TODO")) +} diff --git a/crates/app/obs_gateway/src/main.rs b/crates/app/obs_gateway/src/main.rs index b0446dea35..45c693a927 100644 --- a/crates/app/obs_gateway/src/main.rs +++ b/crates/app/obs_gateway/src/main.rs @@ -50,6 +50,7 @@ use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{AppAccessToken, Scope, TwitchToken, tokens::errors::AppAccessTokenError, ClientId, ClientSecret}; use crate::server_state::{ObsGatewayServerState, EnvConfig, TwitchOauthSecrets}; use crate::endpoints::oauth_begin_redirect::oauth_begin_enroll_redirect; +use crate::endpoints::oauth_end::oauth_end_enroll_from_redirect; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:54321"; @@ -262,6 +263,10 @@ async fn main() -> AnyhowResult<()> { let website_homepage_redirect = easyenv::get_env_string_or_default("WEBSITE_HOMEPAGE_REDIRECT", "https://vo.codes/"); + let oauth_redirect_url = easyenv::get_env_string_or_default( + "TWITCH_OAUTH_REDIRECT_URL", + "http://localhost:54321/twitch/oauth_redirect"); + let server_state = ObsGatewayServerState { env_config: EnvConfig { num_workers, @@ -274,7 +279,7 @@ async fn main() -> AnyhowResult<()> { twitch_oauth_secrets: TwitchOauthSecrets { client_id: secrets.app_client_id.clone(), client_secret: secrets.app_client_secret.clone(), - redirect_url: "http://localhost:54321/twitch/oauth_redirect".to_string() + redirect_url: oauth_redirect_url, }, hostname: server_hostname, }; @@ -320,7 +325,7 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) .service(web::resource("/oauth_redirect") - .route(web::get().to(oauth_begin_enroll)) + .route(web::get().to(oauth_end_enroll_from_redirect)) .route(web::head().to(|| HttpResponse::Ok())) ) .service(web::resource("/websocket") From d370d13f2d68ef5aa2d7999ba35a680aa93ccd99 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 16 Nov 2021 20:00:16 -0500 Subject: [PATCH 0680/1042] OAuth flow compiles, but has issues --- .../obs_gateway/src/endpoints/oauth_begin.rs | 18 ++--- .../src/endpoints/oauth_begin_redirect.rs | 18 ++--- .../obs_gateway/src/endpoints/oauth_end.rs | 80 +++++++++++++++++++ crates/app/obs_gateway/src/twitch/mod.rs | 1 + .../src/twitch/oauth_token_builder.rs | 25 ++++++ 5 files changed, 118 insertions(+), 24 deletions(-) create mode 100644 crates/app/obs_gateway/src/twitch/oauth_token_builder.rs diff --git a/crates/app/obs_gateway/src/endpoints/oauth_begin.rs b/crates/app/obs_gateway/src/endpoints/oauth_begin.rs index a4c3c8a715..ac163d0325 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_begin.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_begin.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use twitch_api2::twitch_oauth2::ClientId; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{Scope, ClientSecret}; +use crate::twitch::oauth_token_builder::get_oauth_token_builder; #[derive(Serialize)] pub struct OauthBeginEnrollResult { @@ -57,18 +58,11 @@ pub async fn oauth_begin_enroll( OauthBeginEnrollError::ServerError })?; - let client_id = ClientId::new(&server_state.twitch_oauth_secrets.client_id); - let client_secret = ClientSecret::new(&server_state.twitch_oauth_secrets.client_secret); - - // TODO: Not all scopes are needed! - let mut builder = UserTokenBuilder::new(client_id, client_secret, redirect_url) - .set_scopes(vec![ - Scope::BitsRead, - Scope::ChannelReadSubscriptions, - Scope::ChatEdit, - Scope::ChatRead, - ]) - .force_verify(true); + let mut builder = get_oauth_token_builder( + &server_state.twitch_oauth_secrets.client_id, + &server_state.twitch_oauth_secrets.client_secret, + &redirect_url, + true); let (url, _csrf_token) = builder.generate_url(); diff --git a/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs b/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs index 6db0746597..9001d43f83 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use twitch_api2::twitch_oauth2::ClientId; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{Scope, ClientSecret}; +use crate::twitch::oauth_token_builder::get_oauth_token_builder; #[derive(Serialize)] pub struct OauthBeginEnrollResult { @@ -57,18 +58,11 @@ pub async fn oauth_begin_enroll_redirect( OauthBeginEnrollError::ServerError })?; - let client_id = ClientId::new(&server_state.twitch_oauth_secrets.client_id); - let client_secret = ClientSecret::new(&server_state.twitch_oauth_secrets.client_secret); - - // TODO: Not all scopes are needed! - let mut builder = UserTokenBuilder::new(client_id, client_secret, redirect_url) - .set_scopes(vec![ - Scope::BitsRead, - Scope::ChannelReadSubscriptions, - Scope::ChatEdit, - Scope::ChatRead, - ]) - .force_verify(true); + let mut builder = get_oauth_token_builder( + &server_state.twitch_oauth_secrets.client_id, + &server_state.twitch_oauth_secrets.client_secret, + &redirect_url, + true); let (url, _csrf_token) = builder.generate_url(); diff --git a/crates/app/obs_gateway/src/endpoints/oauth_end.rs b/crates/app/obs_gateway/src/endpoints/oauth_end.rs index 84295ad078..d0592e8217 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_end.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_end.rs @@ -1,13 +1,34 @@ use actix_http::StatusCode; use actix_web::{HttpResponse, HttpRequest, web, ResponseError}; use crate::server_state::ObsGatewayServerState; +use crate::twitch::oauth_token_builder::get_oauth_token_builder; use http_server_common::response::response_error_helpers::to_simple_json_error; +use log::error; +use log::info; +use log::warn; use std::fmt; use std::sync::Arc; +use twitch_oauth2::tokens::BearerTokenType::UserToken; + +#[derive(Deserialize)] +pub struct QueryParams { + /// OAuth authorization code + pub code: Option, + /// Opaque value used to avoid CSRF attacks. + pub state: Option, + /// List of permission scopes + pub scope: Option, + + /// Only set in the event of an error + pub error: Option, + /// Only set in the event of an error + pub error_description: Option, +} #[derive(Debug)] pub enum OauthEndEnrollFromRedirectError { ServerError, + TwitchOauthError { reason: String }, } // NB: Not using DeriveMore since Clion doesn't understand it. @@ -21,12 +42,14 @@ impl ResponseError for OauthEndEnrollFromRedirectError { fn status_code(&self) -> StatusCode { match *self { Self::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + OauthEndEnrollFromRedirectError::TwitchOauthError { .. } => StatusCode::INTERNAL_SERVER_ERROR, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { Self::ServerError => "server error".to_string(), + OauthEndEnrollFromRedirectError::TwitchOauthError { .. } => "twitch error".to_string(), }; to_simple_json_error(&error_reason, self.status_code()) @@ -35,8 +58,65 @@ impl ResponseError for OauthEndEnrollFromRedirectError { pub async fn oauth_end_enroll_from_redirect( http_request: HttpRequest, + query: web::Query, server_state: web::Data> ) -> Result { + + if let Some(error) = query.error.as_deref() { + return Err(OauthEndEnrollFromRedirectError::TwitchOauthError { reason: error.to_string() }) + } + if let Some(error) = query.error_description.as_deref() { + return Err(OauthEndEnrollFromRedirectError::TwitchOauthError { reason: error.to_string() }) + } + + let (code, state) = match (query.code.as_deref(), query.state.as_deref()) { + (Some(code), Some(state)) => (code.to_string(), state.to_string()), + _ => { + return Err(OauthEndEnrollFromRedirectError::TwitchOauthError { + reason: "params not set".to_string() + }); + } + }; + + let redirect_url = + twitch_oauth2::url::Url::parse(&server_state.twitch_oauth_secrets.redirect_url) + .map_err(|e| { + warn!("Error parsing url: {:?}", e); + OauthEndEnrollFromRedirectError::ServerError + })?; + + let mut builder = get_oauth_token_builder( + &server_state.twitch_oauth_secrets.client_id, + &server_state.twitch_oauth_secrets.client_secret, + &redirect_url, + true); + + let http_client = &reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|e| { + error!("Problem creating HTTP client: {:?}", e); + OauthEndEnrollFromRedirectError::ServerError + })?; + + let user_token = builder + .get_user_token( + http_client, + &state, + &code, + ) + .await + .map_err(|e| { + warn!("Error fetching user token: {:?}", e); + OauthEndEnrollFromRedirectError::TwitchOauthError { + reason: "error fetching user token".to_string() + } + })?; + + let auth_token = user_token.access_token.as_str(); + + info!("Auth token: {:?}", auth_token); + Ok(HttpResponse::Ok() .body("TODO")) } diff --git a/crates/app/obs_gateway/src/twitch/mod.rs b/crates/app/obs_gateway/src/twitch/mod.rs index a7c619f5db..422b2a3dd0 100644 --- a/crates/app/obs_gateway/src/twitch/mod.rs +++ b/crates/app/obs_gateway/src/twitch/mod.rs @@ -1,3 +1,4 @@ +pub mod oauth_token_builder; pub mod twitch_client_wrapper; pub mod twitch_secrets; pub mod websocket_client; \ No newline at end of file diff --git a/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs b/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs new file mode 100644 index 0000000000..3e11fd6099 --- /dev/null +++ b/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs @@ -0,0 +1,25 @@ +use twitch_oauth2::tokens::UserTokenBuilder; +use twitch_oauth2::{Scope, ClientSecret, ClientId}; +use twitch_oauth2::url::Url; + +/// Get the token builder with the scopes our app needs +/// This is used in several places, so centralize creation here. +pub fn get_oauth_token_builder( + client_id: &str, + client_secret: &str, + redirect_url: &Url, + force_verify: bool +) -> UserTokenBuilder { + + let client_id = ClientId::new(client_id); + let client_secret = ClientSecret::new(client_secret); + + UserTokenBuilder::new(client_id, client_secret, redirect_url.clone()) + .set_scopes(vec![ + Scope::BitsRead, + Scope::ChannelReadSubscriptions, + Scope::ChatEdit, + Scope::ChatRead, + ]) + .force_verify(force_verify) +} \ No newline at end of file From dd8945dc5e186d8b0a3ed63a69a76e85a7893e44 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 16 Nov 2021 20:36:22 -0500 Subject: [PATCH 0681/1042] Oauth redemption works, but I'm hacking the CSRF --- crates/app/obs_gateway/src/endpoints/oauth_end.rs | 8 ++++++++ crates/app/obs_gateway/src/twitch/oauth_token_builder.rs | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/app/obs_gateway/src/endpoints/oauth_end.rs b/crates/app/obs_gateway/src/endpoints/oauth_end.rs index d0592e8217..4c8216ef85 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_end.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_end.rs @@ -9,6 +9,7 @@ use log::warn; use std::fmt; use std::sync::Arc; use twitch_oauth2::tokens::BearerTokenType::UserToken; +use twitch_oauth2::CsrfToken; #[derive(Deserialize)] pub struct QueryParams { @@ -78,6 +79,10 @@ pub async fn oauth_end_enroll_from_redirect( } }; + // TODO/FIXME: This is a major security issue (part 1). + // These need to be persisted in the database and associated with the user. + let csrf_token = CsrfToken::new(&state); + let redirect_url = twitch_oauth2::url::Url::parse(&server_state.twitch_oauth_secrets.redirect_url) .map_err(|e| { @@ -91,6 +96,9 @@ pub async fn oauth_end_enroll_from_redirect( &redirect_url, true); + // TODO/FIXME: This is a major security issue (part 2). + builder.set_csrf(csrf_token); + let http_client = &reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .build() diff --git a/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs b/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs index 3e11fd6099..7f44e471f8 100644 --- a/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs +++ b/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs @@ -1,5 +1,5 @@ use twitch_oauth2::tokens::UserTokenBuilder; -use twitch_oauth2::{Scope, ClientSecret, ClientId}; +use twitch_oauth2::{Scope, ClientSecret, ClientId, CsrfToken}; use twitch_oauth2::url::Url; /// Get the token builder with the scopes our app needs @@ -22,4 +22,4 @@ pub fn get_oauth_token_builder( Scope::ChatRead, ]) .force_verify(force_verify) -} \ No newline at end of file +} From fe7b8a8d07c7d45bef933a2ca181ed2517003c67 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 16 Nov 2021 20:41:29 -0500 Subject: [PATCH 0682/1042] now with refresh token --- crates/app/obs_gateway/src/endpoints/oauth_end.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/app/obs_gateway/src/endpoints/oauth_end.rs b/crates/app/obs_gateway/src/endpoints/oauth_end.rs index 4c8216ef85..4b635607c9 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_end.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_end.rs @@ -121,9 +121,13 @@ pub async fn oauth_end_enroll_from_redirect( } })?; - let auth_token = user_token.access_token.as_str(); + let auth_token = user_token.access_token.secret().to_string(); + let refresh_token: Option = user_token.refresh_token + .as_ref() + .map(|token| token.secret().to_string()); info!("Auth token: {:?}", auth_token); + info!("Refresh token: {:?}", refresh_token); Ok(HttpResponse::Ok() .body("TODO")) From 39906ae9c2d67f4c3eb2992de0a39e7742e7fa2f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 16 Nov 2021 22:50:53 -0500 Subject: [PATCH 0683/1042] begin testing websockets --- Cargo.lock | 103 +++--------------- crates/app/obs_gateway/Cargo.toml | 2 +- .../app/obs_gateway/src/endpoints_ws/mod.rs | 2 + .../obs_gateway_websocket_handler.rs | 95 ++++++++++++++++ .../src/{ => endpoints_ws}/pubsub_gateway.rs | 0 crates/app/obs_gateway/src/main.rs | 86 +++++---------- crates/app/obs_gateway/src/server_state.rs | 10 ++ test.js => static/test.js | 0 static/websocket_test.html | 9 ++ 9 files changed, 157 insertions(+), 150 deletions(-) create mode 100644 crates/app/obs_gateway/src/endpoints_ws/mod.rs create mode 100644 crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs rename crates/app/obs_gateway/src/{ => endpoints_ws}/pubsub_gateway.rs (100%) rename test.js => static/test.js (100%) create mode 100644 static/websocket_test.html diff --git a/Cargo.lock b/Cargo.lock index a2699de538..c5fb675ecf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,26 +107,6 @@ dependencies = [ "smallvec 1.6.1", ] -[[package]] -name = "actix-files" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51e8a9146c12fce92a6e4c24b8c4d9b05268130bfd8d61bc587e822c32ce689" -dependencies = [ - "actix-service 1.0.6", - "actix-web 3.3.2", - "bitflags", - "bytes 0.5.6", - "derive_more", - "futures-core", - "futures-util", - "log", - "mime", - "mime_guess", - "percent-encoding 2.1.0", - "v_htmlescape", -] - [[package]] name = "actix-files" version = "0.6.0-beta.8" @@ -634,7 +614,7 @@ checksum = "796540673305a66d127804eef19ad696f1f204b8c1025aaca4958c17eab32877" dependencies = [ "getrandom 0.2.2", "once_cell", - "version_check 0.9.3", + "version_check", ] [[package]] @@ -645,7 +625,7 @@ checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" dependencies = [ "getrandom 0.2.2", "once_cell", - "version_check 0.9.3", + "version_check", ] [[package]] @@ -923,15 +903,6 @@ dependencies = [ "libc", ] -[[package]] -name = "buf-min" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa17aa1cf56bdd6bb30518767d00e58019d326f3f05d8c3e0730b549d332ea83" -dependencies = [ - "bytes 0.5.6", -] - [[package]] name = "build_const" version = "0.2.2" @@ -1120,7 +1091,7 @@ checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" dependencies = [ "percent-encoding 2.1.0", "time 0.2.26", - "version_check 0.9.3", + "version_check", ] [[package]] @@ -1131,7 +1102,7 @@ checksum = "d5f1c7727e460397e56abc4bddc1d49e07a1ad78fc98eb2e1c8f032a58a2f80d" dependencies = [ "percent-encoding 2.1.0", "time 0.2.26", - "version_check 0.9.3", + "version_check", ] [[package]] @@ -1653,7 +1624,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" dependencies = [ "typenum", - "version_check 0.9.3", + "version_check", ] [[package]] @@ -2474,16 +2445,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "nom" -version = "4.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" -dependencies = [ - "memchr", - "version_check 0.1.5", -] - [[package]] name = "nom" version = "6.1.2" @@ -2494,7 +2455,7 @@ dependencies = [ "funty", "lexical-core", "memchr", - "version_check 0.9.3", + "version_check", ] [[package]] @@ -2623,7 +2584,7 @@ version = "0.0.1" dependencies = [ "actix", "actix-cors 0.5.4", - "actix-files 0.5.0", + "actix-files", "actix-http 3.0.0-beta.11", "actix-rt 2.4.0", "actix-service 2.0.1", @@ -3677,7 +3638,7 @@ checksum = "6d86e3c77ff882a828346ba401a7ef4b8e440df804491c6064fe8295765de71c" dependencies = [ "lazy_static", "maplit", - "nom 6.1.2", + "nom", "regex", "unicode_categories", ] @@ -3876,7 +3837,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" dependencies = [ - "version_check 0.9.3", + "version_check", ] [[package]] @@ -3940,7 +3901,7 @@ version = "0.0.1" dependencies = [ "actix", "actix-cors 0.6.0-beta.3", - "actix-files 0.6.0-beta.8", + "actix-files", "actix-http 3.0.0-beta.11", "actix-multipart", "actix-rt 2.4.0", @@ -4145,7 +4106,7 @@ dependencies = [ "standback", "stdweb", "time-macros", - "version_check 0.9.3", + "version_check", "winapi 0.3.9", ] @@ -4588,7 +4549,7 @@ dependencies = [ "twitch_oauth2", "typed-builder", "url 2.2.2", - "version_check 0.9.3", + "version_check", ] [[package]] @@ -4656,7 +4617,7 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" dependencies = [ - "version_check 0.9.3", + "version_check", ] [[package]] @@ -4754,50 +4715,12 @@ dependencies = [ "getrandom 0.2.2", ] -[[package]] -name = "v_escape" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3e0ab5fab1db278a9413d2ea794cb66f471f898c5b020c3c394f6447625d9d4" -dependencies = [ - "buf-min", - "v_escape_derive", -] - -[[package]] -name = "v_escape_derive" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c860ad1273f4eee7006cee05db20c9e60e5d24cba024a32e1094aa8e574f3668" -dependencies = [ - "nom 4.2.3", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "v_htmlescape" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f9a8af610ad6f7fc9989c9d2590d9764bc61f294884e9ee93baa58795174572" -dependencies = [ - "cfg-if 1.0.0", - "v_escape", -] - [[package]] name = "vcpkg" version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" -[[package]] -name = "version_check" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" - [[package]] name = "version_check" version = "0.9.3" diff --git a/crates/app/obs_gateway/Cargo.toml b/crates/app/obs_gateway/Cargo.toml index 6f47012566..e87584f7e6 100644 --- a/crates/app/obs_gateway/Cargo.toml +++ b/crates/app/obs_gateway/Cargo.toml @@ -72,7 +72,7 @@ lru_time_cache = "0.11.11" # Web actix = "0.12.0" actix-cors = "0.5.4" -actix-files= "0.5" +actix-files= "0.6.0-beta.8" actix-http = "3.0.0-beta.11" actix-rt = "2.4.0" actix-service = "2.0.1" diff --git a/crates/app/obs_gateway/src/endpoints_ws/mod.rs b/crates/app/obs_gateway/src/endpoints_ws/mod.rs new file mode 100644 index 0000000000..1b9fe2723e --- /dev/null +++ b/crates/app/obs_gateway/src/endpoints_ws/mod.rs @@ -0,0 +1,2 @@ +pub mod obs_gateway_websocket_handler; +pub mod pubsub_gateway; diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs new file mode 100644 index 0000000000..27c20fd7a4 --- /dev/null +++ b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -0,0 +1,95 @@ +use actix::prelude::*; +use actix_web::{middleware, web, App, Error, HttpRequest, HttpResponse, HttpServer}; +use actix_web_actors::ws; +use crate::server_state::ObsGatewayServerState; +use log::error; +use log::warn; +use std::sync::Arc; + +/// Endpoint +pub async fn obs_gateway_websocket_handler( + request: HttpRequest, + server_state: web::Data>, + stream: web::Payload, +) -> Result { + error!(">>>>>> obs_ws_index"); + let websocket = ObsGatewayWebSocket::new(); + ws::start(websocket, &request, stream) +} + +/// Websocket behavior +struct ObsGatewayWebSocket {} + +impl ObsGatewayWebSocket { + fn new() -> Self { + Self {} + } +} + +impl Actor for ObsGatewayWebSocket { + type Context = ws::WebsocketContext; + + fn started(&mut self, _ctx: &mut Self::Context) { + error!(">>>>>> obs actor started"); + } +} + +impl StreamHandler> for ObsGatewayWebSocket { + fn handle( + &mut self, + msg: Result, + ctx: &mut Self::Context, + ) { + if let Ok(msg) = msg { + error!(">>>>>> obs streamhandler::handle (msg = {:?})", msg); + match msg { + ws::Message::Text(text) => ctx.text(text), + ws::Message::Binary(bin) => ctx.binary(bin), + ws::Message::Ping(bytes) => ctx.pong(&bytes), + ws::Message::Close(reason) => { + ctx.close(reason); + ctx.stop(); + } + _ => {} + } + } else { + error!(">>>>>> obs streamhandler::STOP"); + ctx.stop(); + } + } +} + +// From the Actix homepage: + +//use actix::{Actor, StreamHandler}; +//use actix_web::{web, App, Error, HttpRequest, HttpResponse, HttpServer}; +//use actix_web_actors::ws; +// +///// Define HTTP actor +//struct MyWs; +// +//impl Actor for MyWs { +// type Context = ws::WebsocketContext; +//} +// +///// Handler for ws::Message message +//impl StreamHandler> for MyWs { +// fn handle( +// &mut self, +// msg: Result, +// ctx: &mut Self::Context, +// ) { +// match msg { +// Ok(ws::Message::Ping(msg)) => ctx.pong(&msg), +// Ok(ws::Message::Text(text)) => ctx.text(text), +// Ok(ws::Message::Binary(bin)) => ctx.binary(bin), +// _ => (), +// } +// } +//} +// +//pub async fn twitch_pubsub_gateway(req: HttpRequest, stream: web::Payload) -> Result { +// let resp = ws::start(MyWs {}, &req, stream); +// println!("{:?}", resp); +// resp +//} diff --git a/crates/app/obs_gateway/src/pubsub_gateway.rs b/crates/app/obs_gateway/src/endpoints_ws/pubsub_gateway.rs similarity index 100% rename from crates/app/obs_gateway/src/pubsub_gateway.rs rename to crates/app/obs_gateway/src/endpoints_ws/pubsub_gateway.rs diff --git a/crates/app/obs_gateway/src/main.rs b/crates/app/obs_gateway/src/main.rs index 45c693a927..746acdfee0 100644 --- a/crates/app/obs_gateway/src/main.rs +++ b/crates/app/obs_gateway/src/main.rs @@ -13,7 +13,7 @@ #[macro_use] extern crate serde_derive; pub mod endpoints; -pub mod pubsub_gateway; +pub mod endpoints_ws; pub mod server_state; pub mod twitch; pub mod util; @@ -26,7 +26,11 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use crate::endpoints::oauth_begin::oauth_begin_enroll; -use crate::pubsub_gateway::ws_index; +use crate::endpoints::oauth_begin_redirect::oauth_begin_enroll_redirect; +use crate::endpoints::oauth_end::oauth_end_enroll_from_redirect; +use crate::endpoints_ws::obs_gateway_websocket_handler::obs_gateway_websocket_handler; +use crate::endpoints_ws::pubsub_gateway::ws_index; +use crate::server_state::{ObsGatewayServerState, EnvConfig, TwitchOauthSecrets, TwitchOauthTemp}; use crate::twitch::twitch_client_wrapper::TwitchClientWrapper; use crate::twitch::twitch_secrets::TwitchSecrets; use crate::twitch::websocket_client::PollingTwitchWebsocketClient; @@ -48,9 +52,6 @@ use twitch_api2::pubsub::Topic; use twitch_api2::pubsub; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{AppAccessToken, Scope, TwitchToken, tokens::errors::AppAccessTokenError, ClientId, ClientSecret}; -use crate::server_state::{ObsGatewayServerState, EnvConfig, TwitchOauthSecrets}; -use crate::endpoints::oauth_begin_redirect::oauth_begin_enroll_redirect; -use crate::endpoints::oauth_end::oauth_end_enroll_from_redirect; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:54321"; @@ -107,59 +108,9 @@ async fn main() -> AnyhowResult<()> { // // ==================== OAUTH FLOW ==================== -// -// println!("Oauth flow..."); -// -// let redirect_url = twitch_oauth2::url::Url::parse("http://localhost/test")?; -// let mut builder = UserTokenBuilder::new( -// client_id.clone(), -// client_secret.clone(), -// redirect_url) -// .set_scopes(Scope::all()) -// .force_verify(true); -// -// //builder.add_scope(Scope::BitsRead); -// -// let (url, _csrf_token) = builder.generate_url(); -// -// println!("Go to this page: {}", url); -// -// -// -// let input = rpassword::prompt_password_stdout( -// "Paste in the resulting adress after authenticating (input hidden): ", -// )?; -// -// let u = twitch_oauth2::url::Url::parse(&input)?; -// -// let map: std::collections::HashMap<_, _> = u.query_pairs().collect(); -// -// let user_token = match (map.get("state"), map.get("code")) { -// (Some(state), Some(code)) => { -// let token = builder -// .get_user_token( -// &reqwest::Client::builder() -// .redirect(reqwest::redirect::Policy::none()) -// .build()?, -// state, -// code, -// ) -// .await?; -// println!("Got token: {:?}", token); -// -// token -// } -// _ => match (map.get("error"), map.get("error_description")) { -// (std::option::Option::Some(error), std::option::Option::Some(error_description)) => { -// anyhow::bail!( -// "twitch errored with error: {} - {}", -// error, -// error_description -// ); -// } -// _ => anyhow::bail!("invalid url passed"), -// }, -// }; + + + // // let auth_token = user_token.access_token.as_str(); // @@ -267,6 +218,10 @@ async fn main() -> AnyhowResult<()> { "TWITCH_OAUTH_REDIRECT_URL", "http://localhost:54321/twitch/oauth_redirect"); + // TODO: These are temporary. + let temp_oauth_access_token = easyenv::get_env_string_or_default("TEMP_TWITCH_OAUTH_ACCESS", ""); + let temp_oauth_refresh_token = easyenv::get_env_string_or_default("TEMP_TWITCH_OAUTH_REFRESH", ""); + let server_state = ObsGatewayServerState { env_config: EnvConfig { num_workers, @@ -281,6 +236,10 @@ async fn main() -> AnyhowResult<()> { client_secret: secrets.app_client_secret.clone(), redirect_url: oauth_redirect_url, }, + twitch_oauth_temp: TwitchOauthTemp { + temp_oauth_access_token, + temp_oauth_refresh_token, + }, hostname: server_hostname, }; @@ -333,7 +292,16 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) ) - //.default_service( web::route().to(default_route_404)) + .service(web::resource("/obs") + .route(web::get().to(obs_gateway_websocket_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + actix_files::Files::new("/static", "static") + .show_files_listing() + .use_last_modified(true), + ) + //.default_service( web::route().to(default_route_404)) }) .bind(bind_address)? .workers(num_workers) diff --git a/crates/app/obs_gateway/src/server_state.rs b/crates/app/obs_gateway/src/server_state.rs index 590f44be78..76f750c580 100644 --- a/crates/app/obs_gateway/src/server_state.rs +++ b/crates/app/obs_gateway/src/server_state.rs @@ -8,6 +8,7 @@ pub struct ObsGatewayServerState { pub env_config: EnvConfig, pub hostname: String, pub twitch_oauth_secrets: TwitchOauthSecrets, + pub twitch_oauth_temp: TwitchOauthTemp, } // TODO: Many of these do not need to be passed around past server init. @@ -29,3 +30,12 @@ pub struct TwitchOauthSecrets { pub client_secret: String, pub redirect_url: String, } + +/// TODO: THIS IS JUST FOR FEATURE DEVELOPMENT. KILL THIS. +/// Being lazy here until I have a decent DB model and know what I want to build. +#[deprecated] +#[derive(Clone)] +pub struct TwitchOauthTemp { + pub temp_oauth_access_token: String, + pub temp_oauth_refresh_token: String, +} diff --git a/test.js b/static/test.js similarity index 100% rename from test.js rename to static/test.js diff --git a/static/websocket_test.html b/static/websocket_test.html new file mode 100644 index 0000000000..fa42446580 --- /dev/null +++ b/static/websocket_test.html @@ -0,0 +1,9 @@ + + + + Websocket Test + + +

Websocket Test

+ + From 885ad5338302663456599833bf6f8406a16c8c2f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 16 Nov 2021 22:56:29 -0500 Subject: [PATCH 0684/1042] websockets basics work, cors fix --- crates/lib/http_server_common/src/cors.rs | 1 + static/websocket_test.html | 1 + static/websocket_test.js | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+) create mode 100644 static/websocket_test.js diff --git a/crates/lib/http_server_common/src/cors.rs b/crates/lib/http_server_common/src/cors.rs index 5e7ea5f0b7..dffe077c62 100644 --- a/crates/lib/http_server_common/src/cors.rs +++ b/crates/lib/http_server_common/src/cors.rs @@ -14,6 +14,7 @@ pub fn build_common_cors_config() -> Cors { .allowed_origin("http://jungle.horse:7000") .allowed_origin("http://localhost:12345") .allowed_origin("http://localhost:3000") + .allowed_origin("http://localhost:54321") .allowed_origin("http://localhost:5555") .allowed_origin("http://localhost:7000") .allowed_origin("http://localhost:8000") diff --git a/static/websocket_test.html b/static/websocket_test.html index fa42446580..759776836a 100644 --- a/static/websocket_test.html +++ b/static/websocket_test.html @@ -5,5 +5,6 @@

Websocket Test

+ diff --git a/static/websocket_test.js b/static/websocket_test.js new file mode 100644 index 0000000000..64421b8fbc --- /dev/null +++ b/static/websocket_test.js @@ -0,0 +1,19 @@ + +(function() { + console.log('installing script'); + + const WS_URL = 'ws://localhost:54321/obs'; + + let sock = new WebSocket(WS_URL); + + sock.onopen = function (event) { + console.log('on open event', event); + sock.send('on open message from client'); + }; + + sock.onmessage = function (event) { + console.log('on message event', event.data); + } + + sock.send('this is a message from the client'); +})(); \ No newline at end of file From 2514ec3356a4c8433d9ec568e38032974bd28076 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 17 Nov 2021 00:36:05 -0500 Subject: [PATCH 0685/1042] wip websocket + twitch pubsub (not working) --- .../obs_gateway_websocket_handler.rs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 27c20fd7a4..7e6ecc14e7 100644 --- a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -5,6 +5,9 @@ use crate::server_state::ObsGatewayServerState; use log::error; use log::warn; use std::sync::Arc; +use crate::twitch::websocket_client::PollingTwitchWebsocketClient; +use twitch_api2::pubsub; +use twitch_api2::pubsub::Topic; /// Endpoint pub async fn obs_gateway_websocket_handler( @@ -13,6 +16,46 @@ pub async fn obs_gateway_websocket_handler( stream: web::Payload, ) -> Result { error!(">>>>>> obs_ws_index"); + + let mut client = PollingTwitchWebsocketClient::new().unwrap(); + + error!("Connecting to Twitch PubSub..."); + client.connect().await.unwrap(); + + error!("Connected to Twitch PubSub"); + + //println!("Starting polling thread..."); + //client.start_ping_thread().await; + + error!("Sending Twitch PubSub PING..."); + + client.send_ping().await.unwrap(); + + error!("Try read next from Twitch PubSub..."); + let r = client.try_next().await.unwrap(); + error!("Twitch PubSub Result: {:?}", r); + + + // User: vocodes + let user_id = 652567283; + let bit_topic = pubsub::channel_bits::ChannelBitsEventsV2 { + channel_id: user_id, + }.into_topic(); + + let topics = [bit_topic]; + + let auth_token = server_state.twitch_oauth_temp.temp_oauth_access_token.clone(); + + error!("Begin TwitchPubSub LISTEN on authenticated OAuth topics..."); + + client.listen(&auth_token, &topics).await.unwrap(); + + error!("Twitch PubSub Try read next..."); + let r = client.try_next().await.unwrap(); + + error!("Twitch PubSub Result: {:?}", r); + + error!("Begin Javascript WebSocket..."); let websocket = ObsGatewayWebSocket::new(); ws::start(websocket, &request, stream) } From 80db649b9f9e7fbde97556e77f73752152ccc602 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 17 Nov 2021 01:03:12 -0500 Subject: [PATCH 0686/1042] wip --- crates/app/obs_gateway/src/endpoints/oauth_end.rs | 2 ++ .../endpoints_ws/obs_gateway_websocket_handler.rs | 12 +++++++++--- crates/app/obs_gateway/src/main.rs | 2 ++ crates/app/obs_gateway/src/server_state.rs | 1 + 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/app/obs_gateway/src/endpoints/oauth_end.rs b/crates/app/obs_gateway/src/endpoints/oauth_end.rs index 4b635607c9..93167956a8 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_end.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_end.rs @@ -121,11 +121,13 @@ pub async fn oauth_end_enroll_from_redirect( } })?; + let user_id = user_token.user_id.to_string(); let auth_token = user_token.access_token.secret().to_string(); let refresh_token: Option = user_token.refresh_token .as_ref() .map(|token| token.secret().to_string()); + info!("User id: {:?}", user_id); info!("Auth token: {:?}", auth_token); info!("Refresh token: {:?}", refresh_token); diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 7e6ecc14e7..352ab36fed 100644 --- a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -35,14 +35,20 @@ pub async fn obs_gateway_websocket_handler( let r = client.try_next().await.unwrap(); error!("Twitch PubSub Result: {:?}", r); - // User: vocodes - let user_id = 652567283; + //let user_id = 652567283; + + let user_id = server_state.twitch_oauth_temp.temp_oauth_user_id.parse::().unwrap(); + let bit_topic = pubsub::channel_bits::ChannelBitsEventsV2 { channel_id: user_id, }.into_topic(); - let topics = [bit_topic]; + let sub_topic = pubsub::channel_subscriptions::ChannelSubscribeEventsV1 { + channel_id: user_id, + }.into_topic(); + + let topics = [bit_topic, sub_topic]; let auth_token = server_state.twitch_oauth_temp.temp_oauth_access_token.clone(); diff --git a/crates/app/obs_gateway/src/main.rs b/crates/app/obs_gateway/src/main.rs index 746acdfee0..123e9fccb4 100644 --- a/crates/app/obs_gateway/src/main.rs +++ b/crates/app/obs_gateway/src/main.rs @@ -219,6 +219,7 @@ async fn main() -> AnyhowResult<()> { "http://localhost:54321/twitch/oauth_redirect"); // TODO: These are temporary. + let temp_oauth_user_id = easyenv::get_env_string_or_default("TEMP_TWITCH_OAUTH_USER_ID", ""); let temp_oauth_access_token = easyenv::get_env_string_or_default("TEMP_TWITCH_OAUTH_ACCESS", ""); let temp_oauth_refresh_token = easyenv::get_env_string_or_default("TEMP_TWITCH_OAUTH_REFRESH", ""); @@ -237,6 +238,7 @@ async fn main() -> AnyhowResult<()> { redirect_url: oauth_redirect_url, }, twitch_oauth_temp: TwitchOauthTemp { + temp_oauth_user_id, temp_oauth_access_token, temp_oauth_refresh_token, }, diff --git a/crates/app/obs_gateway/src/server_state.rs b/crates/app/obs_gateway/src/server_state.rs index 76f750c580..f03b78e835 100644 --- a/crates/app/obs_gateway/src/server_state.rs +++ b/crates/app/obs_gateway/src/server_state.rs @@ -36,6 +36,7 @@ pub struct TwitchOauthSecrets { #[deprecated] #[derive(Clone)] pub struct TwitchOauthTemp { + pub temp_oauth_user_id: String, pub temp_oauth_access_token: String, pub temp_oauth_refresh_token: String, } From 20e0f265c1bb2266ddd10ed2fd4ff37c03036414 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 17 Nov 2021 01:14:40 -0500 Subject: [PATCH 0687/1042] read more topics --- .../obs_gateway_websocket_handler.rs | 24 +++++++++++++++---- .../src/twitch/oauth_token_builder.rs | 3 ++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 352ab36fed..07f2f6860a 100644 --- a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -44,11 +44,19 @@ pub async fn obs_gateway_websocket_handler( channel_id: user_id, }.into_topic(); + let points_topic = pubsub::channel_points::ChannelPointsChannelV1 { + channel_id: user_id, + }.into_topic(); + + let cheer_topic = pubsub::channel_cheer::ChannelCheerEventsPublicV1 { + channel_id: user_id, + }.into_topic(); + let sub_topic = pubsub::channel_subscriptions::ChannelSubscribeEventsV1 { channel_id: user_id, }.into_topic(); - let topics = [bit_topic, sub_topic]; + let topics = [bit_topic, points_topic, cheer_topic, sub_topic]; let auth_token = server_state.twitch_oauth_temp.temp_oauth_access_token.clone(); @@ -62,16 +70,22 @@ pub async fn obs_gateway_websocket_handler( error!("Twitch PubSub Result: {:?}", r); error!("Begin Javascript WebSocket..."); - let websocket = ObsGatewayWebSocket::new(); + let websocket = ObsGatewayWebSocket::new(client); + ws::start(websocket, &request, stream) } /// Websocket behavior -struct ObsGatewayWebSocket {} +struct ObsGatewayWebSocket { + twitch_client: PollingTwitchWebsocketClient, +} + impl ObsGatewayWebSocket { - fn new() -> Self { - Self {} + fn new(twitch_client: PollingTwitchWebsocketClient) -> Self { + Self { + twitch_client + } } } diff --git a/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs b/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs index 7f44e471f8..0f135aed74 100644 --- a/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs +++ b/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs @@ -16,7 +16,8 @@ pub fn get_oauth_token_builder( UserTokenBuilder::new(client_id, client_secret, redirect_url.clone()) .set_scopes(vec![ - Scope::BitsRead, + Scope::BitsRead, // Using bits to cheer (cost money, larger streams) + Scope::ChannelReadRedemptions, // Channel points redeemed (free points, smaller streams) Scope::ChannelReadSubscriptions, Scope::ChatEdit, Scope::ChatRead, From 898fe142e955931de23aae7786abdedcec2818f8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 20 Dec 2021 17:47:16 -0500 Subject: [PATCH 0688/1042] Uncommitted old OBS code --- Cargo.lock | 65 +++++++++++++--- crates/app/obs_gateway/Cargo.toml | 3 + .../app/obs_gateway/src/endpoints_ws/mod.rs | 3 +- .../obs_gateway_websocket_handler.rs | 59 ++++++++++++-- .../src/endpoints_ws/obs_twitch_thread.rs | 76 +++++++++++++++++++ static/websocket_test.js | 6 +- 6 files changed, 196 insertions(+), 16 deletions(-) create mode 100644 crates/app/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs diff --git a/Cargo.lock b/Cargo.lock index c5fb675ecf..44e91d03ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1178,6 +1178,20 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crossbeam" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils 0.8.5", +] + [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -1185,17 +1199,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils 0.8.5", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.5", + "lazy_static", + "memoffset", + "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6cb3c7f5b8e51bc3ebb73a2327ad4abdbd119dc13223f14f961d2f38486756" +checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", ] [[package]] @@ -1211,11 +1249,10 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" dependencies = [ - "autocfg 1.0.1", "cfg-if 1.0.0", "lazy_static", ] @@ -2289,6 +2326,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" +[[package]] +name = "memoffset" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +dependencies = [ + "autocfg 1.0.1", +] + [[package]] name = "mime" version = "0.3.16" @@ -2597,6 +2643,7 @@ dependencies = [ "chrono", "config", "container_common", + "crossbeam", "derive_more", "diesel", "dotenv", @@ -3679,7 +3726,7 @@ dependencies = [ "crc 1.8.1", "crossbeam-channel", "crossbeam-queue", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", "digest", "either", "futures-channel", @@ -3726,7 +3773,7 @@ dependencies = [ "crc 2.1.0", "crossbeam-channel", "crossbeam-queue", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", "digest", "either", "futures-channel", diff --git a/crates/app/obs_gateway/Cargo.toml b/crates/app/obs_gateway/Cargo.toml index e87584f7e6..58d814d3bc 100644 --- a/crates/app/obs_gateway/Cargo.toml +++ b/crates/app/obs_gateway/Cargo.toml @@ -46,6 +46,9 @@ futures = { version = "0.3.15", features = ["thread-pool"] } # feature for Threa futures-util = "0.3.17" futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate +# Threading +crossbeam = "0.8.1" + # MySql Database # "offline" supports typesafe builds without a live database # "rustls" doesn't appear to work with IP addresses, per: diff --git a/crates/app/obs_gateway/src/endpoints_ws/mod.rs b/crates/app/obs_gateway/src/endpoints_ws/mod.rs index 1b9fe2723e..cfbaddd858 100644 --- a/crates/app/obs_gateway/src/endpoints_ws/mod.rs +++ b/crates/app/obs_gateway/src/endpoints_ws/mod.rs @@ -1,2 +1,3 @@ pub mod obs_gateway_websocket_handler; -pub mod pubsub_gateway; +pub mod obs_twitch_thread; +pub mod pubsub_gateway; \ No newline at end of file diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 07f2f6860a..a735ab214a 100644 --- a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -4,10 +4,15 @@ use actix_web_actors::ws; use crate::server_state::ObsGatewayServerState; use log::error; use log::warn; +use log::info; use std::sync::Arc; use crate::twitch::websocket_client::PollingTwitchWebsocketClient; use twitch_api2::pubsub; use twitch_api2::pubsub::Topic; +use tokio::runtime::Handle; +use crate::endpoints_ws::obs_twitch_thread::ObsTwitchThread; +use std::thread::sleep; +use std::time::Duration; /// Endpoint pub async fn obs_gateway_websocket_handler( @@ -64,10 +69,12 @@ pub async fn obs_gateway_websocket_handler( client.listen(&auth_token, &topics).await.unwrap(); - error!("Twitch PubSub Try read next..."); - let r = client.try_next().await.unwrap(); - error!("Twitch PubSub Result: {:?}", r); + //let (tx, rx) = crossbeam::channel::bounded(1); + //let client = Arc::new(&self.twitch_client); + //let client2 = client.clone(); + + //let res = rx.recv().unwrap(); error!("Begin Javascript WebSocket..."); let websocket = ObsGatewayWebSocket::new(client); @@ -77,14 +84,16 @@ pub async fn obs_gateway_websocket_handler( /// Websocket behavior struct ObsGatewayWebSocket { - twitch_client: PollingTwitchWebsocketClient, + //twitch_client: PollingTwitchWebsocketClient, + twitch_thread: ObsTwitchThread, } impl ObsGatewayWebSocket { fn new(twitch_client: PollingTwitchWebsocketClient) -> Self { + let twitch_thread = ObsTwitchThread::new(twitch_client); Self { - twitch_client + twitch_thread } } } @@ -105,6 +114,46 @@ impl StreamHandler> for ObsGatewayWebSock ) { if let Ok(msg) = msg { error!(">>>>>> obs streamhandler::handle (msg = {:?})", msg); + + //let (tx, rx) = crossbeam::channel::bounded(1); + //let handle = Handle::current(); + + //let client = Arc::new(&self.twitch_client); + //let client2 = client.clone(); + + //handle.spawn(async { + // error!("Twitch PubSub Try read next..."); + // match client2.try_next().await { + // Ok(r) => { + // error!("Twitch PubSub Result: {:?}", r); + // }, + // Err(e) => { + // warn!("pubsub error: {:?}", e); + // } + // } + //}); + + //let res = rx.recv().unwrap(); + + let handle = Handle::current(); + + handle.spawn(async move { + //error!("Twitch PubSub Try read next..."); + //match client2.try_next().await { + // Ok(r) => { + // error!("Twitch PubSub Result: {:?}", r); + // }, + // Err(e) => { + // warn!("pubsub error: {:?}", e); + // } + //} + loop { + info!("Thread loop"); + sleep(Duration::from_millis(1000)); + } + }); + + match msg { ws::Message::Text(text) => ctx.text(text), ws::Message::Binary(bin) => ctx.binary(bin), diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs b/crates/app/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs new file mode 100644 index 0000000000..215cfa2c20 --- /dev/null +++ b/crates/app/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs @@ -0,0 +1,76 @@ +use crate::twitch::websocket_client::PollingTwitchWebsocketClient; +use log::error; +use log::info; +use std::sync::atomic::AtomicBool; +use std::sync::{Arc, RwLock, PoisonError}; +use std::thread::sleep; +use std::time::Duration; +use tokio::runtime::Handle; +use time::Instant; + +pub struct ObsTwitchThread { + twitch_client: Arc>, + last_requested: Arc>, +} + +impl ObsTwitchThread { + pub fn new(twitch_client: PollingTwitchWebsocketClient) -> Self { + let now = Instant::now(); + Self { + twitch_client: Arc::new(RwLock::new(twitch_client)), + last_requested: Arc::new(RwLock::new(now)), + } + } + + pub fn keep_alive_thread(&self) { + let now = Instant::now(); + let lock = self.last_requested + .clone() + .get_mut(); + *lock = Some(now); + } + + // TODO: Investigate use of a thread pool instead. + pub fn start_thread(&self) { + let mut last_requested_lock = self.last_requested.clone(); + + let handle = Handle::current(); + + handle.spawn(async move { + Self::thread_main_loop(last_requested_lock); + }); + } + + fn thread_main_loop( + mut last_requested: Arc>, + ) { + loop { + let c = last_requested.get_mut(); + + match c { + Err(e) => { + error!("Mutex failure: {:?}, ending job.", e); + break; + } + Ok(t) => { + let now = Instant::now(); + let delta : Duration = now.duration_since(t); + + if delta.gt(&Duration::from_secs(60)) { + info!("Timeout elapsed, ending job."); + break; + } + } + } + + Self::thread_single_loop(); + + sleep(Duration::from_millis(1000)); + } + } + + fn thread_single_loop() { + info!("thread iteration") + } +} + diff --git a/static/websocket_test.js b/static/websocket_test.js index 64421b8fbc..98bb5940fd 100644 --- a/static/websocket_test.js +++ b/static/websocket_test.js @@ -15,5 +15,9 @@ console.log('on message event', event.data); } - sock.send('this is a message from the client'); + setInterval(() => { + console.log('sending message to server on interval'); + sock.send('interval trigger'); + }, 2000); + })(); \ No newline at end of file From 69dad0d854f972f57dfa4339508fdc65f519571e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 20 Dec 2021 17:56:17 -0500 Subject: [PATCH 0689/1042] move unused OBS migrations out of the way (for now) --- .../2021-09-29-042235_twitch_oauth_tokens/down.sql | 0 .../2021-09-29-042235_twitch_oauth_tokens/up.sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {migrations => migrations_wip}/2021-09-29-042235_twitch_oauth_tokens/down.sql (100%) rename {migrations => migrations_wip}/2021-09-29-042235_twitch_oauth_tokens/up.sql (100%) diff --git a/migrations/2021-09-29-042235_twitch_oauth_tokens/down.sql b/migrations_wip/2021-09-29-042235_twitch_oauth_tokens/down.sql similarity index 100% rename from migrations/2021-09-29-042235_twitch_oauth_tokens/down.sql rename to migrations_wip/2021-09-29-042235_twitch_oauth_tokens/down.sql diff --git a/migrations/2021-09-29-042235_twitch_oauth_tokens/up.sql b/migrations_wip/2021-09-29-042235_twitch_oauth_tokens/up.sql similarity index 100% rename from migrations/2021-09-29-042235_twitch_oauth_tokens/up.sql rename to migrations_wip/2021-09-29-042235_twitch_oauth_tokens/up.sql From 4cb208bad742adf311d646e23464f4686b3b35dc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 20 Dec 2021 19:36:35 -0500 Subject: [PATCH 0690/1042] Model category tables --- .../down.sql | 6 + .../2021-12-20-230000_model_categories/up.sql | 116 ++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 migrations/2021-12-20-230000_model_categories/down.sql create mode 100644 migrations/2021-12-20-230000_model_categories/up.sql diff --git a/migrations/2021-12-20-230000_model_categories/down.sql b/migrations/2021-12-20-230000_model_categories/down.sql new file mode 100644 index 0000000000..ba45bcd3c8 --- /dev/null +++ b/migrations/2021-12-20-230000_model_categories/down.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE model_categories; +DROP TABLE model_category_assignments; \ No newline at end of file diff --git a/migrations/2021-12-20-230000_model_categories/up.sql b/migrations/2021-12-20-230000_model_categories/up.sql new file mode 100644 index 0000000000..84562b47f4 --- /dev/null +++ b/migrations/2021-12-20-230000_model_categories/up.sql @@ -0,0 +1,116 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE TABLE model_categories ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Visible "primary key" + token VARCHAR(32) NOT NULL, + + -- ========== CATEGORY TOPOLOGY ========== + + -- Optional foreign key link to super category. + -- If absent, this is a top-level category or top-level super category. + super_category_token VARCHAR(32) DEFAULT NULL, + + -- If this category can be applied directly to models. + can_have_models BOOLEAN NOT NULL DEFAULT false, + + -- If this category can have subcategories. + can_have_subcategories BOOLEAN NOT NULL DEFAULT false, + + -- ========== CATEGORY NAME ========== + + -- The name of the category, which might be complicated. + name VARCHAR(255) NOT NULL, + + -- The name displayed in the dropdown (if it should be different), otherwise NULL. + -- eg. "Gender" (name) and "By Gender" (dropdown name) + dropdown_name VARCHAR(20) DEFAULT NULL, + + -- ========== CREATOR DETAILS ========== + + -- The person that created the category. + creator_user_token VARCHAR(32) NOT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address_creation VARCHAR(40) NOT NULL, + -- Technically we don't allow editing, but just in case. + creator_ip_address_last_update VARCHAR(40) NOT NULL, + + -- ========== MODERATION DETAILS ========== + + -- Mods have to approve of categories. They remain hidden and unusable until approved. + is_mod_approved BOOLEAN DEFAULT NULL, + + -- If a moderator has comments. + maybe_mod_comments VARCHAR(255) DEFAULT NULL, + + -- The last moderator that made changes. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If the category is deleted and shouldn't be displayed. + -- Only mods can delete categories. + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY fk_super_category_token (super_category_token), + KEY index_can_have_models (can_have_models), + KEY index_can_have_subcategories (can_have_subcategories), + KEY index_is_mod_approved (is_mod_approved) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Join table assigning models to zero or more categories. +CREATE TABLE model_category_assignments ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + model_token VARCHAR(32) NOT NULL, + category_token VARCHAR(32) NOT NULL, + + -- ========== USER DETAILS ========== + + -- The person that created the assignment. + -- This can be the model owner or a moderator. + category_addition_user_token VARCHAR(32) NOT NULL, + + -- The person that removed the assignment. + -- This can be the model owner or a moderator. + category_removal_user_token VARCHAR(32) NOT NULL, + + -- For abuse tracking. + -- THIS CAN BE A MODERATOR FOR SIMPLICITY. + -- Wide enough for IPv4/6 + ip_address_creation VARCHAR(40) NOT NULL, + ip_address_last_update VARCHAR(40) NOT NULL, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (model_token, category_token), + KEY fk_model_token (model_token), + KEY fk_category_token (category_token), + KEY index_deleted_at (deleted_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From bbc268aaef2f119a482748dd228ca800ee3c8551 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 20 Dec 2021 22:45:14 -0500 Subject: [PATCH 0691/1042] Empty rust modules, change schema to add w2l support --- .../endpoints/categories/create_category.rs | 0 .../endpoints/categories/delete_category.rs | 0 .../endpoints/categories/edit_category.rs | 0 .../endpoints/categories/list_categories.rs | 0 .../http_server/endpoints/categories/mod.rs | 4 ++ .../src/http_server/endpoints/mod.rs | 1 + .../down.sql | 3 +- .../2021-12-20-230000_model_categories/up.sql | 54 +++++++++++++++++-- 8 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/delete_category.rs create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/list_categories.rs create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/delete_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/delete_category.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_categories.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_categories.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs new file mode 100644 index 0000000000..ed0a0cfaed --- /dev/null +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs @@ -0,0 +1,4 @@ +pub mod create_category; +pub mod delete_category; +pub mod edit_category; +pub mod list_categories; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/http_server/endpoints/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/mod.rs index 1bb6fcabce..f7f14dfbc9 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/mod.rs @@ -1,3 +1,4 @@ +pub mod categories; pub mod events; pub mod leaderboard; pub mod misc; diff --git a/migrations/2021-12-20-230000_model_categories/down.sql b/migrations/2021-12-20-230000_model_categories/down.sql index ba45bcd3c8..986333f646 100644 --- a/migrations/2021-12-20-230000_model_categories/down.sql +++ b/migrations/2021-12-20-230000_model_categories/down.sql @@ -3,4 +3,5 @@ -- noinspection SqlResolveForFile DROP TABLE model_categories; -DROP TABLE model_category_assignments; \ No newline at end of file +DROP TABLE tts_category_assignments; +DROP TABLE w2l_category_assignments; diff --git a/migrations/2021-12-20-230000_model_categories/up.sql b/migrations/2021-12-20-230000_model_categories/up.sql index 84562b47f4..3dbb313d4d 100644 --- a/migrations/2021-12-20-230000_model_categories/up.sql +++ b/migrations/2021-12-20-230000_model_categories/up.sql @@ -9,6 +9,14 @@ CREATE TABLE model_categories ( -- Visible "primary key" token VARCHAR(32) NOT NULL, + -- ========== MODEL TYPE ========== + + model_type ENUM( + 'not_set', + 'tts', + 'w2l' + ) NOT NULL DEFAULT 'not_set', + -- ========== CATEGORY TOPOLOGY ========== -- Optional foreign key link to super category. @@ -16,7 +24,7 @@ CREATE TABLE model_categories ( super_category_token VARCHAR(32) DEFAULT NULL, -- If this category can be applied directly to models. - can_have_models BOOLEAN NOT NULL DEFAULT false, + can_directly_have_models BOOLEAN NOT NULL DEFAULT false, -- If this category can have subcategories. can_have_subcategories BOOLEAN NOT NULL DEFAULT false, @@ -70,14 +78,54 @@ CREATE TABLE model_categories ( PRIMARY KEY (id), UNIQUE KEY (token), KEY fk_super_category_token (super_category_token), - KEY index_can_have_models (can_have_models), + KEY index_model_type (model_type), + KEY index_can_directly_have_models (can_directly_have_models), KEY index_can_have_subcategories (can_have_subcategories), KEY index_is_mod_approved (is_mod_approved) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; -- Join table assigning models to zero or more categories. -CREATE TABLE model_category_assignments ( +CREATE TABLE tts_category_assignments ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + model_token VARCHAR(32) NOT NULL, + category_token VARCHAR(32) NOT NULL, + + -- ========== USER DETAILS ========== + + -- The person that created the assignment. + -- This can be the model owner or a moderator. + category_addition_user_token VARCHAR(32) NOT NULL, + + -- The person that removed the assignment. + -- This can be the model owner or a moderator. + category_removal_user_token VARCHAR(32) NOT NULL, + + -- For abuse tracking. + -- THIS CAN BE A MODERATOR FOR SIMPLICITY. + -- Wide enough for IPv4/6 + ip_address_creation VARCHAR(40) NOT NULL, + ip_address_last_update VARCHAR(40) NOT NULL, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (model_token, category_token), + KEY fk_model_token (model_token), + KEY fk_category_token (category_token), + KEY index_deleted_at (deleted_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- Join table assigning models to zero or more categories. +CREATE TABLE w2l_category_assignments ( -- Not used for anything except replication. id BIGINT(20) NOT NULL AUTO_INCREMENT, From 18c3913f8f8dbf4ed6df753a8645b31760e4765f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 21 Dec 2021 01:55:32 -0500 Subject: [PATCH 0692/1042] create category endpoint (no route) --- .../src/database/helpers/tokens.rs | 4 + .../endpoints/categories/create_category.rs | 186 ++++++++++++++++++ 2 files changed, 190 insertions(+) diff --git a/crates/app/storyteller_web/src/database/helpers/tokens.rs b/crates/app/storyteller_web/src/database/helpers/tokens.rs index 9f906ebe1c..9dd61b791b 100644 --- a/crates/app/storyteller_web/src/database/helpers/tokens.rs +++ b/crates/app/storyteller_web/src/database/helpers/tokens.rs @@ -56,4 +56,8 @@ impl Tokens { pub fn new_firehose_event() -> AnyhowResult { random_prefix_crockford_token("EV:", 32, false) } + + pub fn new_category() -> AnyhowResult { + random_prefix_crockford_token("CAT:", 32, false) + } } diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs index e69de29bb2..ed6716b560 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs @@ -0,0 +1,186 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use std::fmt; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use crate::database::helpers::tokens::Tokens; + +// =============== Request =============== + +#[derive(Deserialize)] +pub enum ModelType { + Tts, + W2l, +} + +#[derive(Deserialize)] +pub struct CreateCategoryRequest { + pub name: Option, + pub model_type: Option, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct CreateCategoryResponse { + pub success: bool, + pub token: Option, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum CreateCategoryError { + BadInput(String), + NotAuthorized, + ServerError, +} + +impl ResponseError for CreateCategoryError { + fn status_code(&self) -> StatusCode { + match *self { + CreateCategoryError::BadInput(_) => StatusCode::BAD_REQUEST, + CreateCategoryError::NotAuthorized => StatusCode::UNAUTHORIZED, + CreateCategoryError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for CreateCategoryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn create_category_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CreateCategoryError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(CreateCategoryError::NotAuthorized); + } + }; + + // NB: First permission check. + // TODO: We don't have a permission for categories, so we use this as a proxy. + let is_mod = user_session.can_ban_users; + + let model_type = match request.model_type { + None => { + return Err(CreateCategoryError::BadInput("no model type".to_string())); + } + Some(ModelType::Tts) => "tts", + Some(ModelType::W2l) => "w2l", + }; + + let name = request.name + .clone() + .ok_or(CreateCategoryError::BadInput("no name provided".to_string()))?; + + let category_token = Tokens::new_category() + .map_err(|e| { + warn!("Bad crockford token: {:?}", e); + CreateCategoryError::ServerError + })?; + + let creator_ip_address = get_request_ip(&http_request); + + let is_mod_approved = is_mod; + + let maybe_mod_user_token = if is_mod { + Some(user_session.user_token.clone()) + } else { + None + }; + + let query_result = sqlx::query!( + r#" +INSERT INTO model_categories +SET + token = ?, + model_type = ?, + name = ?, + + creator_user_token = ?, + creator_ip_address_creation = ?, + + is_mod_approved = ?, + maybe_mod_user_token = ?, + can_directly_have_models = true, + can_have_subcategories = false + "#, + + category_token, + model_type, + name, + &user_session.user_token, + creator_ip_address, + is_mod_approved, + maybe_mod_user_token, + ) + .execute(&server_state.mysql_pool) + .await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Create category edit DB error: {:?}", err); + return Err(CreateCategoryError::ServerError); + } + }; + + let response = CreateCategoryResponse { + success: true, + token: Some(category_token.to_string()) + }; + + let body = serde_json::to_string(&response) + .map_err(|e| CreateCategoryError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} From 5db08c3b83bc27cb9b23aefb5a87f3b804cbab00 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 21 Dec 2021 12:27:51 -0500 Subject: [PATCH 0693/1042] Move routes to their own file for maintainability --- crates/app/storyteller_web/src/main.rs | 313 +----------------- crates/app/storyteller_web/src/routes.rs | 384 +++++++++++++++++++++++ 2 files changed, 390 insertions(+), 307 deletions(-) create mode 100644 crates/app/storyteller_web/src/routes.rs diff --git a/crates/app/storyteller_web/src/main.rs b/crates/app/storyteller_web/src/main.rs index e391a9cd8c..534f17254e 100644 --- a/crates/app/storyteller_web/src/main.rs +++ b/crates/app/storyteller_web/src/main.rs @@ -19,6 +19,7 @@ pub const BANNED_SLURS : &'static str = include_str!("../../../../db/banned_slur pub mod database; pub mod http_clients; pub mod http_server; +pub mod routes; pub mod server_state; pub mod threads; pub mod util; @@ -114,6 +115,7 @@ use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; use tokio::runtime::Runtime; +use crate::routes::add_routes; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -315,7 +317,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> // NB: Safe to clone due to internal arc let ip_banlist = server_state_arc.ip_banlist.clone(); - App::new() + let app = App::new() .app_data(server_state_arc.clone()) .wrap(build_common_cors_config()) .wrap(DefaultHeaders::new() @@ -324,312 +326,9 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .wrap(IpFilter::new(ip_banlist)) .wrap(Logger::new(&log_format) .exclude("/liveness") - .exclude("/readiness")) - // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== - .service( - web::resource("/create_account") - .route(web::post().to(create_account_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/login") - .route(web::post().to(login_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/logout") - .route(web::post().to(logout_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/session") - .route(web::get().to(session_info_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - // ==================== MODERATOR ==================== - .service( - web::scope("/moderation") - .service( - web::resource("/staff") - .route(web::get().to(list_staff_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::scope("/ip_bans") - .service( - web::resource("/list") - .route(web::get().to(list_ip_bans_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/add") - .route(web::post().to(add_ip_ban_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{ip_address}") - .route(web::get().to(get_ip_ban_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{ip_address}/delete") - .route(web::post().to(delete_ip_ban_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/user") - .service( - web::resource("/list") - .route(web::get().to(list_users_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/user_bans") - .service( - web::resource("/list") - .route(web::get().to(list_banned_users_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/manage_ban") - .route(web::post().to(ban_user_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/roles") - .service( - web::resource("/list") - .route(web::get().to(list_user_roles_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/edit") - .route(web::post().to(set_user_role_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/jobs") - .service( - web::resource("/tts_inference_queue_stats") - .route(web::get().to(get_tts_inference_queue_count_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/w2l_inference_queue_stats") - .route(web::get().to(get_w2l_inference_queue_count_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/pending") - .service( - web::resource("/w2l_templates") - .route(web::get().to(get_pending_w2l_templates_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/w2l_inference_queue_stats") - .route(web::get().to(get_w2l_inference_queue_count_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/stats") - .service( - web::resource("/tts_voices") - .route(web::get().to(get_voice_count_stats_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - ) - // ==================== TTS ==================== - .service( - web::scope("/tts") - .service( - web::resource("/upload") - .route(web::post().to(upload_tts_model_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/inference") - .route(web::post().to(infer_tts_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/list") - .route(web::get().to(list_tts_models_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/model/{token}") - .route(web::get().to(get_tts_model_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/model/{token}/delete") - .route(web::post().to(delete_tts_model_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/model/{model_token}/count") - .route(web::get().to(get_tts_model_use_count_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/model/{model_token}/edit") - .route(web::post().to(edit_tts_model_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}") - .route(web::get().to(get_tts_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}/edit") - .route(web::post().to(edit_tts_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}/delete") - .route(web::post().to(delete_tts_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/job/{token}") - .route(web::get().to(get_tts_inference_job_status_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/upload_model_job/{token}") - .route(web::get().to(get_tts_upload_model_job_status_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - // ==================== WAV2LIP ==================== - .service( - web::scope("/w2l") - .service( - web::resource("/upload") - .route(web::post().to(upload_w2l_template_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/inference") - .route(web::post().to(enqueue_infer_w2l_with_uploads)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/list") - .route(web::get().to(list_w2l_templates_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/template/{token}") - .route(web::get().to(get_w2l_template_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/template/{template_token}/count") - .route(web::get().to(get_w2l_template_use_count_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/template/{template_token}/edit") - .route(web::post().to(edit_w2l_template_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/template/{token}/moderate") - .route(web::post().to(set_w2l_template_mod_approval_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/template/{token}/delete") - .route(web::post().to(delete_w2l_template_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}") - .route(web::get().to(get_w2l_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}/edit") - .route(web::post().to(edit_w2l_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}/delete") - .route(web::post().to(delete_w2l_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/job/{token}") - .route(web::get().to(get_w2l_inference_job_status_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/upload_template_job/{token}") - .route(web::get().to(get_w2l_upload_template_job_status_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - // ==================== USER DATA ==================== - .service( - web::scope("/user") - .service( - web::resource("/{username}/profile") - .route(web::get().to(get_profile_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/edit_profile") - .route(web::post().to(edit_profile_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/tts_models") - .route(web::get().to(list_user_tts_models_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/tts_results") - .route(web::get().to(list_user_tts_inference_results_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/w2l_templates") - .route(web::get().to(list_user_w2l_templates_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/w2l_results") - .route(web::get().to(list_user_w2l_inference_results_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::resource("/events") - .route(web::get().to(list_events_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/leaderboard") - .route(web::get().to(leaderboard_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service(web::resource("/") - .route(web::get().to(get_root_index)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service(enable_alpha_handler) - .service(enable_alpha_easy_handler) - .default_service( web::route().to(default_route_404)) + .exclude("/readiness")); + + add_routes(app) }) .bind(bind_address)? .workers(num_workers) diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/app/storyteller_web/src/routes.rs new file mode 100644 index 0000000000..e3e2149760 --- /dev/null +++ b/crates/app/storyteller_web/src/routes.rs @@ -0,0 +1,384 @@ +use actix_web::{App, web, HttpResponse}; +use crate::http_server::endpoints::events::list_events::list_events_handler; +use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; +use crate::http_server::endpoints::misc::default_route_404::default_route_404; +use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha_easy_handler; +use crate::http_server::endpoints::misc::enable_alpha_handler::enable_alpha_handler; +use crate::http_server::endpoints::misc::root_index::get_root_index; +use crate::http_server::endpoints::moderation::approval::pending_w2l_templates::get_pending_w2l_templates_handler; +use crate::http_server::endpoints::moderation::ip_bans::add_ip_ban::add_ip_ban_handler; +use crate::http_server::endpoints::moderation::ip_bans::delete_ip_ban::delete_ip_ban_handler; +use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_handler; +use crate::http_server::endpoints::moderation::ip_bans::list_ip_bans::list_ip_bans_handler; +use crate::http_server::endpoints::moderation::jobs::get_tts_inference_queue_count::get_tts_inference_queue_count_handler; +use crate::http_server::endpoints::moderation::jobs::get_w2l_inference_queue_count::get_w2l_inference_queue_count_handler; +use crate::http_server::endpoints::moderation::stats::get_voice_count_stats::get_voice_count_stats_handler; +use crate::http_server::endpoints::moderation::user_bans::ban_user::ban_user_handler; +use crate::http_server::endpoints::moderation::user_bans::list_banned_users::list_banned_users_handler; +use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user_roles_handler; +use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler; +use crate::http_server::endpoints::moderation::user_roles::set_user_role::set_user_role_handler; +use crate::http_server::endpoints::moderation::users::list_users::list_users_handler; +use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handler; +use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; +use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; +use crate::http_server::endpoints::tts::edit_tts_result::edit_tts_inference_result_handler; +use crate::http_server::endpoints::tts::enqueue_infer_tts::infer_tts_handler; +use crate::http_server::endpoints::tts::enqueue_upload_tts_model::upload_tts_model_handler; +use crate::http_server::endpoints::tts::get_tts_inference_job_status::get_tts_inference_job_status_handler; +use crate::http_server::endpoints::tts::get_tts_model::get_tts_model_handler; +use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_use_count_handler; +use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; +use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; +use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; +use crate::http_server::endpoints::users::create_account::create_account_handler; +use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; +use crate::http_server::endpoints::users::get_profile::get_profile_handler; +use crate::http_server::endpoints::users::list_user_tts_inference_results::list_user_tts_inference_results_handler; +use crate::http_server::endpoints::users::list_user_tts_models::list_user_tts_models_handler; +use crate::http_server::endpoints::users::list_user_w2l_inference_results::list_user_w2l_inference_results_handler; +use crate::http_server::endpoints::users::list_user_w2l_templates::list_user_w2l_templates_handler; +use crate::http_server::endpoints::users::login::login_handler; +use crate::http_server::endpoints::users::logout::logout_handler; +use crate::http_server::endpoints::users::session_info::session_info_handler; +use crate::http_server::endpoints::w2l::delete_w2l_result::delete_w2l_inference_result_handler; +use crate::http_server::endpoints::w2l::delete_w2l_template::delete_w2l_template_handler; +use crate::http_server::endpoints::w2l::edit_w2l_result::edit_w2l_inference_result_handler; +use crate::http_server::endpoints::w2l::edit_w2l_template::edit_w2l_template_handler; +use crate::http_server::endpoints::w2l::enqueue_infer_w2l::infer_w2l_handler; +use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; +use crate::http_server::endpoints::w2l::enqueue_upload_w2l_template::upload_w2l_template_handler; +use crate::http_server::endpoints::w2l::get_w2l_inference_job_status::get_w2l_inference_job_status_handler; +use crate::http_server::endpoints::w2l::get_w2l_result::get_w2l_inference_result_handler; +use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handler; +use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_template_use_count_handler; +use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; +use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; +use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; +use actix_http::body::MessageBody; +use actix_service::ServiceFactory; +use actix_web::dev::{ServiceRequest, ServiceResponse}; +use actix_web::error::Error; + +pub fn add_routes (app: App) -> App + + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, + +{ + + + + // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== + app.service( + web::resource("/create_account") + .route(web::post().to(create_account_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/login") + .route(web::post().to(login_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/logout") + .route(web::post().to(logout_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/session") + .route(web::get().to(session_info_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + // ==================== MODERATOR ==================== + .service( + web::scope("/moderation") + .service( + web::resource("/staff") + .route(web::get().to(list_staff_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::scope("/ip_bans") + .service( + web::resource("/list") + .route(web::get().to(list_ip_bans_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/add") + .route(web::post().to(add_ip_ban_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{ip_address}") + .route(web::get().to(get_ip_ban_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{ip_address}/delete") + .route(web::post().to(delete_ip_ban_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/user") + .service( + web::resource("/list") + .route(web::get().to(list_users_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/user_bans") + .service( + web::resource("/list") + .route(web::get().to(list_banned_users_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/manage_ban") + .route(web::post().to(ban_user_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/roles") + .service( + web::resource("/list") + .route(web::get().to(list_user_roles_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/edit") + .route(web::post().to(set_user_role_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/jobs") + .service( + web::resource("/tts_inference_queue_stats") + .route(web::get().to(get_tts_inference_queue_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/w2l_inference_queue_stats") + .route(web::get().to(get_w2l_inference_queue_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/pending") + .service( + web::resource("/w2l_templates") + .route(web::get().to(get_pending_w2l_templates_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/w2l_inference_queue_stats") + .route(web::get().to(get_w2l_inference_queue_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/stats") + .service( + web::resource("/tts_voices") + .route(web::get().to(get_voice_count_stats_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + ) + // ==================== TTS ==================== + .service( + web::scope("/tts") + .service( + web::resource("/upload") + .route(web::post().to(upload_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/inference") + .route(web::post().to(infer_tts_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/list") + .route(web::get().to(list_tts_models_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/model/{token}") + .route(web::get().to(get_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/model/{token}/delete") + .route(web::post().to(delete_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/model/{model_token}/count") + .route(web::get().to(get_tts_model_use_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/model/{model_token}/edit") + .route(web::post().to(edit_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}") + .route(web::get().to(get_tts_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}/edit") + .route(web::post().to(edit_tts_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}/delete") + .route(web::post().to(delete_tts_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/job/{token}") + .route(web::get().to(get_tts_inference_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/upload_model_job/{token}") + .route(web::get().to(get_tts_upload_model_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + // ==================== WAV2LIP ==================== + .service( + web::scope("/w2l") + .service( + web::resource("/upload") + .route(web::post().to(upload_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/inference") + .route(web::post().to(enqueue_infer_w2l_with_uploads)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/list") + .route(web::get().to(list_w2l_templates_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/template/{token}") + .route(web::get().to(get_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/template/{template_token}/count") + .route(web::get().to(get_w2l_template_use_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/template/{template_token}/edit") + .route(web::post().to(edit_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/template/{token}/moderate") + .route(web::post().to(set_w2l_template_mod_approval_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/template/{token}/delete") + .route(web::post().to(delete_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}") + .route(web::get().to(get_w2l_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}/edit") + .route(web::post().to(edit_w2l_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}/delete") + .route(web::post().to(delete_w2l_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/job/{token}") + .route(web::get().to(get_w2l_inference_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/upload_template_job/{token}") + .route(web::get().to(get_w2l_upload_template_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + // ==================== USER DATA ==================== + .service( + web::scope("/user") + .service( + web::resource("/{username}/profile") + .route(web::get().to(get_profile_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/edit_profile") + .route(web::post().to(edit_profile_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/tts_models") + .route(web::get().to(list_user_tts_models_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/tts_results") + .route(web::get().to(list_user_tts_inference_results_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/w2l_templates") + .route(web::get().to(list_user_w2l_templates_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/w2l_results") + .route(web::get().to(list_user_w2l_inference_results_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::resource("/events") + .route(web::get().to(list_events_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/leaderboard") + .route(web::get().to(leaderboard_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/") + .route(web::get().to(get_root_index)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(enable_alpha_handler) + .service(enable_alpha_easy_handler) + .default_service( web::route().to(default_route_404)) +} \ No newline at end of file From 591f3ac2adbbba13c77bd0127726060f3f93f394 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 21 Dec 2021 22:52:00 -0500 Subject: [PATCH 0694/1042] organize routes module --- crates/app/storyteller_web/src/routes.rs | 600 +++++++++++++---------- 1 file changed, 328 insertions(+), 272 deletions(-) diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/app/storyteller_web/src/routes.rs index e3e2149760..ae59aa07b7 100644 --- a/crates/app/storyteller_web/src/routes.rs +++ b/crates/app/storyteller_web/src/routes.rs @@ -61,7 +61,6 @@ use actix_web::dev::{ServiceRequest, ServiceResponse}; use actix_web::error::Error; pub fn add_routes (app: App) -> App - where B: MessageBody, T: ServiceFactory< @@ -71,10 +70,11 @@ pub fn add_routes (app: App) -> App Error = Error, InitError = (), >, - { - - + let mut app = add_moderator_routes(app); /* /moderation */ + app = add_tts_routes(app); /* /tts */ + app = add_w2l_routes(app); /* /w2l */ + app = add_user_profile_routes(app); /* /user */ // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== app.service( @@ -97,273 +97,6 @@ pub fn add_routes (app: App) -> App .route(web::get().to(session_info_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - // ==================== MODERATOR ==================== - .service( - web::scope("/moderation") - .service( - web::resource("/staff") - .route(web::get().to(list_staff_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::scope("/ip_bans") - .service( - web::resource("/list") - .route(web::get().to(list_ip_bans_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/add") - .route(web::post().to(add_ip_ban_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{ip_address}") - .route(web::get().to(get_ip_ban_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{ip_address}/delete") - .route(web::post().to(delete_ip_ban_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/user") - .service( - web::resource("/list") - .route(web::get().to(list_users_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/user_bans") - .service( - web::resource("/list") - .route(web::get().to(list_banned_users_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/manage_ban") - .route(web::post().to(ban_user_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/roles") - .service( - web::resource("/list") - .route(web::get().to(list_user_roles_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/edit") - .route(web::post().to(set_user_role_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/jobs") - .service( - web::resource("/tts_inference_queue_stats") - .route(web::get().to(get_tts_inference_queue_count_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/w2l_inference_queue_stats") - .route(web::get().to(get_w2l_inference_queue_count_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/pending") - .service( - web::resource("/w2l_templates") - .route(web::get().to(get_pending_w2l_templates_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/w2l_inference_queue_stats") - .route(web::get().to(get_w2l_inference_queue_count_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - .service( - web::scope("/stats") - .service( - web::resource("/tts_voices") - .route(web::get().to(get_voice_count_stats_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - ) - // ==================== TTS ==================== - .service( - web::scope("/tts") - .service( - web::resource("/upload") - .route(web::post().to(upload_tts_model_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/inference") - .route(web::post().to(infer_tts_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/list") - .route(web::get().to(list_tts_models_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/model/{token}") - .route(web::get().to(get_tts_model_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/model/{token}/delete") - .route(web::post().to(delete_tts_model_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/model/{model_token}/count") - .route(web::get().to(get_tts_model_use_count_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/model/{model_token}/edit") - .route(web::post().to(edit_tts_model_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}") - .route(web::get().to(get_tts_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}/edit") - .route(web::post().to(edit_tts_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}/delete") - .route(web::post().to(delete_tts_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/job/{token}") - .route(web::get().to(get_tts_inference_job_status_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/upload_model_job/{token}") - .route(web::get().to(get_tts_upload_model_job_status_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - // ==================== WAV2LIP ==================== - .service( - web::scope("/w2l") - .service( - web::resource("/upload") - .route(web::post().to(upload_w2l_template_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/inference") - .route(web::post().to(enqueue_infer_w2l_with_uploads)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/list") - .route(web::get().to(list_w2l_templates_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/template/{token}") - .route(web::get().to(get_w2l_template_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/template/{template_token}/count") - .route(web::get().to(get_w2l_template_use_count_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/template/{template_token}/edit") - .route(web::post().to(edit_w2l_template_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/template/{token}/moderate") - .route(web::post().to(set_w2l_template_mod_approval_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/template/{token}/delete") - .route(web::post().to(delete_w2l_template_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}") - .route(web::get().to(get_w2l_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}/edit") - .route(web::post().to(edit_w2l_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/result/{token}/delete") - .route(web::post().to(delete_w2l_inference_result_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/job/{token}") - .route(web::get().to(get_w2l_inference_job_status_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/upload_template_job/{token}") - .route(web::get().to(get_w2l_upload_template_job_status_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) - // ==================== USER DATA ==================== - .service( - web::scope("/user") - .service( - web::resource("/{username}/profile") - .route(web::get().to(get_profile_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/edit_profile") - .route(web::post().to(edit_profile_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/tts_models") - .route(web::get().to(list_user_tts_models_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/tts_results") - .route(web::get().to(list_user_tts_inference_results_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/w2l_templates") - .route(web::get().to(list_user_w2l_templates_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service( - web::resource("/{username}/w2l_results") - .route(web::get().to(list_user_w2l_inference_results_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) .service( web::resource("/events") .route(web::get().to(list_events_handler)) @@ -381,4 +114,327 @@ pub fn add_routes (app: App) -> App .service(enable_alpha_handler) .service(enable_alpha_easy_handler) .default_service( web::route().to(default_route_404)) -} \ No newline at end of file +} + +// ==================== MODERATOR ROUTES ==================== + +fn add_moderator_routes (app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service( + web::scope("/moderation") + .service( + web::resource("/staff") + .route(web::get().to(list_staff_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::scope("/ip_bans") + .service( + web::resource("/list") + .route(web::get().to(list_ip_bans_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/add") + .route(web::post().to(add_ip_ban_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{ip_address}") + .route(web::get().to(get_ip_ban_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{ip_address}/delete") + .route(web::post().to(delete_ip_ban_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/user") + .service( + web::resource("/list") + .route(web::get().to(list_users_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/user_bans") + .service( + web::resource("/list") + .route(web::get().to(list_banned_users_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/manage_ban") + .route(web::post().to(ban_user_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/roles") + .service( + web::resource("/list") + .route(web::get().to(list_user_roles_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/edit") + .route(web::post().to(set_user_role_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/jobs") + .service( + web::resource("/tts_inference_queue_stats") + .route(web::get().to(get_tts_inference_queue_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/w2l_inference_queue_stats") + .route(web::get().to(get_w2l_inference_queue_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/pending") + .service( + web::resource("/w2l_templates") + .route(web::get().to(get_pending_w2l_templates_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/w2l_inference_queue_stats") + .route(web::get().to(get_w2l_inference_queue_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + .service( + web::scope("/stats") + .service( + web::resource("/tts_voices") + .route(web::get().to(get_voice_count_stats_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + ) +} + +// ==================== TTS ROUTES ==================== + +fn add_tts_routes (app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service( + web::scope("/tts") + .service( + web::resource("/upload") + .route(web::post().to(upload_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/inference") + .route(web::post().to(infer_tts_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/list") + .route(web::get().to(list_tts_models_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/model/{token}") + .route(web::get().to(get_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/model/{token}/delete") + .route(web::post().to(delete_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/model/{model_token}/count") + .route(web::get().to(get_tts_model_use_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/model/{model_token}/edit") + .route(web::post().to(edit_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}") + .route(web::get().to(get_tts_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}/edit") + .route(web::post().to(edit_tts_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}/delete") + .route(web::post().to(delete_tts_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/job/{token}") + .route(web::get().to(get_tts_inference_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/upload_model_job/{token}") + .route(web::get().to(get_tts_upload_model_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) +} + +// ==================== WAV2LIP ROUTES ==================== + +fn add_w2l_routes (app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service( + web::scope("/w2l") + .service( + web::resource("/upload") + .route(web::post().to(upload_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/inference") + .route(web::post().to(enqueue_infer_w2l_with_uploads)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/list") + .route(web::get().to(list_w2l_templates_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/template/{token}") + .route(web::get().to(get_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/template/{template_token}/count") + .route(web::get().to(get_w2l_template_use_count_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/template/{template_token}/edit") + .route(web::post().to(edit_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/template/{token}/moderate") + .route(web::post().to(set_w2l_template_mod_approval_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/template/{token}/delete") + .route(web::post().to(delete_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}") + .route(web::get().to(get_w2l_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}/edit") + .route(web::post().to(edit_w2l_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}/delete") + .route(web::post().to(delete_w2l_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/job/{token}") + .route(web::get().to(get_w2l_inference_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/upload_template_job/{token}") + .route(web::get().to(get_w2l_upload_template_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) +} + +// ==================== USER PROFILE ROUTES ==================== + +fn add_user_profile_routes (app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service( + web::scope("/user") + .service( + web::resource("/{username}/profile") + .route(web::get().to(get_profile_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/edit_profile") + .route(web::post().to(edit_profile_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/tts_models") + .route(web::get().to(list_user_tts_models_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/tts_results") + .route(web::get().to(list_user_tts_inference_results_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/w2l_templates") + .route(web::get().to(list_user_w2l_templates_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/w2l_results") + .route(web::get().to(list_user_w2l_inference_results_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) +} From 2119cfbe095f16d5c39d64f871c21e188c34feb5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 21 Dec 2021 23:08:10 -0500 Subject: [PATCH 0695/1042] add an ip banlist error message --- .../src/http_server/middleware/ip_filter_middleware.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs b/crates/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs index c3fbbc1720..68f0a5fcad 100644 --- a/crates/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs +++ b/crates/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs @@ -12,6 +12,7 @@ use actix_web::web::{BytesMut, Buf, BufMut}; use actix_web::{Error, HttpResponse}; use actix_web::{ResponseError, HttpMessage, HttpRequest, HttpResponseBuilder}; use crate::http_server::web_utils::ip_address::get_service_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::threads::ip_banlist_set::IpBanlistSet; use futures_util::future::{err, ok, Either, Ready}; use std::io::Write; @@ -34,9 +35,12 @@ impl ResponseError for BannedError { } fn error_response(&self) -> HttpResponse { - HttpResponseBuilder::new(self.status_code()) - .append_header((header::CONTENT_TYPE, "application/json")) - .body("{}") + // NB: I'm setting a string error because I mistakenly got caught by this in local dev + // and couldn't figure out the issue for a bit. At least I can grep for this string. + // However, I need to balance this requirement with not cluing in those that are banned. + to_simple_json_error( + "ERR64: storyteller-web database error", + self.status_code()) } } From 7088b8657d082637abf22cfb985e1d03b5329569 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 21 Dec 2021 23:39:37 -0500 Subject: [PATCH 0696/1042] category create route --- crates/app/storyteller_web/src/routes.rs | 34 +++++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/app/storyteller_web/src/routes.rs index ae59aa07b7..b4820a580d 100644 --- a/crates/app/storyteller_web/src/routes.rs +++ b/crates/app/storyteller_web/src/routes.rs @@ -1,4 +1,9 @@ use actix_web::{App, web, HttpResponse}; +use actix_http::body::MessageBody; +use actix_service::ServiceFactory; +use actix_web::dev::{ServiceRequest, ServiceResponse}; +use actix_web::error::Error; +use crate::http_server::endpoints::categories::create_category::create_category_handler; use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::misc::default_route_404::default_route_404; @@ -55,10 +60,6 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; -use actix_http::body::MessageBody; -use actix_service::ServiceFactory; -use actix_web::dev::{ServiceRequest, ServiceResponse}; -use actix_web::error::Error; pub fn add_routes (app: App) -> App where @@ -74,6 +75,7 @@ pub fn add_routes (app: App) -> App let mut app = add_moderator_routes(app); /* /moderation */ app = add_tts_routes(app); /* /tts */ app = add_w2l_routes(app); /* /w2l */ + app = add_category_routes(app); /* /category */ app = add_user_profile_routes(app); /* /user */ // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== @@ -97,6 +99,7 @@ pub fn add_routes (app: App) -> App .route(web::get().to(session_info_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + // ==================== MISC ==================== .service( web::resource("/events") .route(web::get().to(list_events_handler)) @@ -391,6 +394,29 @@ fn add_w2l_routes (app: App) -> App ) } +// ==================== CATEGORY ROUTES ==================== + +fn add_category_routes (app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service( + web::scope("/category") + .service( + web::resource("/create") + .route(web::post().to(create_category_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) +} + // ==================== USER PROFILE ROUTES ==================== fn add_user_profile_routes (app: App) -> App From 9f0763b5ea44eeac7ea99b8e50a683979285316a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 21 Dec 2021 23:48:17 -0500 Subject: [PATCH 0697/1042] Run `cargo fix` to fix imports --- .../src/database/mediators/badge_granter.rs | 4 ++-- .../database/queries/calculate_tts_model_leaderboard.rs | 2 +- .../queries/calculate_w2l_template_leaderboard.rs | 2 +- .../src/database/queries/get_user_profile_by_username.rs | 2 +- .../storyteller_web/src/database/queries/list_ip_bans.rs | 2 +- .../src/database/queries/list_user_badges.rs | 2 +- .../src/database/queries/list_user_roles.rs | 2 +- .../src/database/queries/query_tts_model.rs | 2 +- .../src/database/queries/query_tts_result.rs | 2 +- .../src/database/queries/query_w2l_result.rs | 2 +- .../src/database/queries/query_w2l_template.rs | 2 +- .../src/http_server/endpoints/events/list_events.rs | 2 +- .../moderation/approval/pending_w2l_templates.rs | 2 +- .../endpoints/moderation/ip_bans/get_ip_ban.rs | 2 +- .../endpoints/moderation/ip_bans/list_ip_bans.rs | 2 +- .../moderation/jobs/get_tts_inference_queue_count.rs | 2 +- .../moderation/jobs/get_w2l_inference_queue_count.rs | 2 +- .../endpoints/moderation/stats/get_voice_count_stats.rs | 2 +- .../endpoints/moderation/user_roles/list_staff.rs | 2 +- .../endpoints/tts/get_tts_inference_job_status.rs | 2 +- .../endpoints/tts/get_tts_upload_model_job_status.rs | 2 +- .../src/http_server/endpoints/users/get_profile.rs | 2 +- .../endpoints/w2l/enqueue_infer_w2l_with_uploads.rs | 2 +- .../endpoints/w2l/get_w2l_inference_job_status.rs | 2 +- .../endpoints/w2l/get_w2l_upload_template_job_status.rs | 2 +- .../src/job_queries/tts_download_job_queries.rs | 2 +- .../src/job_queries/tts_inference_job_queries.rs | 4 ++-- .../src/job_queries/w2l_download_job_queries.rs | 2 +- .../src/job_queries/w2l_inference_job_queries.rs | 4 ++-- .../http_server_common/src/endpoints/default_route_404.rs | 2 +- crates/lib/http_server_common/src/endpoints/root_index.rs | 8 ++++---- .../src/response/response_error_helpers.rs | 2 +- .../src/response/serialize_as_json_error.rs | 2 +- 33 files changed, 39 insertions(+), 39 deletions(-) diff --git a/crates/app/storyteller_web/src/database/mediators/badge_granter.rs b/crates/app/storyteller_web/src/database/mediators/badge_granter.rs index 126409c4e3..714d4affb2 100644 --- a/crates/app/storyteller_web/src/database/mediators/badge_granter.rs +++ b/crates/app/storyteller_web/src/database/mediators/badge_granter.rs @@ -233,7 +233,7 @@ LIMIT 1 Ok(_record) => true, Err(err) => { match err { - RowNotFound => false, + sqlx::Error::RowNotFound => false, _ => { warn!("query error: {:?}", err); return Err(anyhow!("error querying: {:?}", err)); @@ -352,7 +352,7 @@ SET Ok(record) => record.count as u64, Err(err) => { match err { - RowNotFound => 0, + sqlx::Error::RowNotFound => 0, _ => { warn!("query error: {:?}", err); return Err(anyhow!("error querying: {:?}", err)); diff --git a/crates/app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs b/crates/app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs index 106e7d4b25..1aeb5cc3cb 100644 --- a/crates/app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs +++ b/crates/app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs @@ -59,7 +59,7 @@ LIMIT 25; warn!("Error: {:?}", err); match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Ok(Vec::new()); }, _ => { diff --git a/crates/app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs b/crates/app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs index 505342d8d6..0f830bad3a 100644 --- a/crates/app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs +++ b/crates/app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs @@ -60,7 +60,7 @@ LIMIT 25; warn!("Error: {:?}", err); match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Ok(Vec::new()); }, _ => { diff --git a/crates/app/storyteller_web/src/database/queries/get_user_profile_by_username.rs b/crates/app/storyteller_web/src/database/queries/get_user_profile_by_username.rs index 7b84bdbc4f..939192e0a6 100644 --- a/crates/app/storyteller_web/src/database/queries/get_user_profile_by_username.rs +++ b/crates/app/storyteller_web/src/database/queries/get_user_profile_by_username.rs @@ -90,7 +90,7 @@ WHERE Ok(profile_record) => profile_record, Err(err) => { return match err { - RowNotFound => { + sqlx::Error::RowNotFound => { Ok(None) }, _ => { diff --git a/crates/app/storyteller_web/src/database/queries/list_ip_bans.rs b/crates/app/storyteller_web/src/database/queries/list_ip_bans.rs index 98e12b8368..998a0d9cd4 100644 --- a/crates/app/storyteller_web/src/database/queries/list_ip_bans.rs +++ b/crates/app/storyteller_web/src/database/queries/list_ip_bans.rs @@ -52,7 +52,7 @@ WHERE match maybe_results { Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { Ok(Vec::new()) }, _ => { diff --git a/crates/app/storyteller_web/src/database/queries/list_user_badges.rs b/crates/app/storyteller_web/src/database/queries/list_user_badges.rs index b333920a64..5e597bb9c4 100644 --- a/crates/app/storyteller_web/src/database/queries/list_user_badges.rs +++ b/crates/app/storyteller_web/src/database/queries/list_user_badges.rs @@ -53,7 +53,7 @@ WHERE Err(err) => { warn!("Error: {:?}", err); match err { - RowNotFound => Vec::new(), + sqlx::Error::RowNotFound => Vec::new(), _ => { warn!("user badges query error: {:?}", err); return Err(anyhow!("error querying user badges")); diff --git a/crates/app/storyteller_web/src/database/queries/list_user_roles.rs b/crates/app/storyteller_web/src/database/queries/list_user_roles.rs index da4619f772..129b581fa0 100644 --- a/crates/app/storyteller_web/src/database/queries/list_user_roles.rs +++ b/crates/app/storyteller_web/src/database/queries/list_user_roles.rs @@ -113,7 +113,7 @@ FROM user_roles Err(err) => { warn!("Error: {:?}", err); match err { - RowNotFound => Vec::new(), + sqlx::Error::RowNotFound => Vec::new(), _ => { warn!("user role query error: {:?}", err); return Err(anyhow!("error querying user roles")); diff --git a/crates/app/storyteller_web/src/database/queries/query_tts_model.rs b/crates/app/storyteller_web/src/database/queries/query_tts_model.rs index f15043c24a..bbc3b4f41a 100644 --- a/crates/app/storyteller_web/src/database/queries/query_tts_model.rs +++ b/crates/app/storyteller_web/src/database/queries/query_tts_model.rs @@ -100,7 +100,7 @@ pub async fn select_tts_model_by_token( Ok(model) => model, Err(ref err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { warn!("tts model not found: {:?}", &err); return Ok(None); }, diff --git a/crates/app/storyteller_web/src/database/queries/query_tts_result.rs b/crates/app/storyteller_web/src/database/queries/query_tts_result.rs index f8ce3a52bc..6b93c0a30b 100644 --- a/crates/app/storyteller_web/src/database/queries/query_tts_result.rs +++ b/crates/app/storyteller_web/src/database/queries/query_tts_result.rs @@ -120,7 +120,7 @@ pub async fn select_tts_result_by_token( Ok(inference_result) => inference_result, Err(ref err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { warn!("tts result not found: {:?}", &err); return Ok(None); }, diff --git a/crates/app/storyteller_web/src/database/queries/query_w2l_result.rs b/crates/app/storyteller_web/src/database/queries/query_w2l_result.rs index eb51f60bce..ce38cb86cb 100644 --- a/crates/app/storyteller_web/src/database/queries/query_w2l_result.rs +++ b/crates/app/storyteller_web/src/database/queries/query_w2l_result.rs @@ -119,7 +119,7 @@ pub async fn select_w2l_result_by_token( Ok(inference_result) => inference_result, Err(ref err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { warn!("w2l result not found: {:?}", &err); return Ok(None); }, diff --git a/crates/app/storyteller_web/src/database/queries/query_w2l_template.rs b/crates/app/storyteller_web/src/database/queries/query_w2l_template.rs index d32af7445f..29ce15a925 100644 --- a/crates/app/storyteller_web/src/database/queries/query_w2l_template.rs +++ b/crates/app/storyteller_web/src/database/queries/query_w2l_template.rs @@ -94,7 +94,7 @@ pub async fn select_w2l_template_by_token( Ok(template) => template, Err(ref err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { warn!("w2l template not found: {:?}", &err); return Ok(None); }, diff --git a/crates/app/storyteller_web/src/http_server/endpoints/events/list_events.rs b/crates/app/storyteller_web/src/http_server/endpoints/events/list_events.rs index 1822edcdaa..250a09e651 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/events/list_events.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/events/list_events.rs @@ -91,7 +91,7 @@ LIMIT 25 Ok(events) => events, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Err(ListEventsError::ServerError); }, _ => { diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs index f0a6f7b1bc..5752c04ac5 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs @@ -126,7 +126,7 @@ WHERE Ok(results) => results, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { Vec::new() }, _ => { diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs index 20b174bd60..6974d68473 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs @@ -148,7 +148,7 @@ LIMIT 1 }, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Err(GetIpBanError::NotFound); }, _ => { diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs index 1ebea79e13..153a8f85e1 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs @@ -135,7 +135,7 @@ WHERE }, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { Vec::new() }, _ => { diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs index 3558c33897..0ccc33e3d1 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs @@ -105,7 +105,7 @@ LIMIT 1 Ok(result) => result, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { // NB: Not Found for null results means nothing is pending in the queue PendingCountResult { pending_count: None, diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs index 867e88182c..3188055b6f 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs @@ -105,7 +105,7 @@ LIMIT 1 Ok(result) => result, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { // NB: Not Found for null results means nothing is pending in the queue PendingCountResult { pending_count: None, diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs index b8810e2d21..38eb438994 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs @@ -101,7 +101,7 @@ SELECT public_count, all_count FROM ( Ok(result) => result, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { // NB: Not Found for null results means nothing is pending in the queue VoiceStats { all_count: -1, diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs index d79a244d8c..054dec8fcb 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs @@ -124,7 +124,7 @@ WHERE }, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { Vec::new() }, _ => { diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index 8cc570619d..541a496ec6 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -143,7 +143,7 @@ WHERE jobs.token = ? Ok(record) => record, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Err(GetTtsInferenceStatusError::ServerError); }, _ => { diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs index ea5b9c97e7..3434facbb9 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs @@ -118,7 +118,7 @@ WHERE jobs.token = ? Ok(record) => record, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Err(GetTtsUploadModelStatusError::ServerError); }, _ => { diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs b/crates/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs index e3f464a7bb..5556e2d745 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs @@ -181,7 +181,7 @@ WHERE Ok(profile_record) => profile_record, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { warn!("Invalid user"); return Err(ProfileError::NotFound); }, diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 0e9e63b3e5..a4b3925b81 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -378,7 +378,7 @@ WHERE }, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { false }, _ => { diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs index 5997d0856a..e881cfb838 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs @@ -137,7 +137,7 @@ WHERE jobs.token = ? Ok(record) => record, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Err(GetW2lInferenceStatusError::ServerError); }, _ => { diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs b/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs index c3f26bf6b3..7ba87e2a2e 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs @@ -124,7 +124,7 @@ WHERE jobs.token = ? Ok(record) => record, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Err(GetW2lUploadTemplateStatusError::ServerError); }, _ => { diff --git a/crates/app/storyteller_web/src/job_queries/tts_download_job_queries.rs b/crates/app/storyteller_web/src/job_queries/tts_download_job_queries.rs index 712b6c4a57..1ff2073a5d 100644 --- a/crates/app/storyteller_web/src/job_queries/tts_download_job_queries.rs +++ b/crates/app/storyteller_web/src/job_queries/tts_download_job_queries.rs @@ -92,7 +92,7 @@ FOR UPDATE Ok(record) => record, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Err(anyhow!("could not job")); }, _ => { diff --git a/crates/app/storyteller_web/src/job_queries/tts_inference_job_queries.rs b/crates/app/storyteller_web/src/job_queries/tts_inference_job_queries.rs index 55f17f8efa..8ebfb02157 100644 --- a/crates/app/storyteller_web/src/job_queries/tts_inference_job_queries.rs +++ b/crates/app/storyteller_web/src/job_queries/tts_inference_job_queries.rs @@ -116,7 +116,7 @@ FOR UPDATE Ok(record) => record, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Err(anyhow!("could not job")); }, _ => { @@ -475,7 +475,7 @@ WHERE tts.token = ? Ok(model) => model, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Err(TtsModelForInferenceError::ModelNotFound); }, _ => { diff --git a/crates/app/storyteller_web/src/job_queries/w2l_download_job_queries.rs b/crates/app/storyteller_web/src/job_queries/w2l_download_job_queries.rs index 0cc6394108..77998b0327 100644 --- a/crates/app/storyteller_web/src/job_queries/w2l_download_job_queries.rs +++ b/crates/app/storyteller_web/src/job_queries/w2l_download_job_queries.rs @@ -91,7 +91,7 @@ FOR UPDATE Ok(record) => record, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Err(anyhow!("could not job")); }, _ => { diff --git a/crates/app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs b/crates/app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs index 0e77bd6c33..de013eda66 100644 --- a/crates/app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs +++ b/crates/app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs @@ -137,7 +137,7 @@ FOR UPDATE Ok(record) => record, Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { return Err(anyhow!("could not job")); }, _ => { @@ -441,7 +441,7 @@ AND w2l.mod_deleted_at IS NULL Ok(template) => Ok(Some(template)), Err(err) => { match err { - RowNotFound => { + sqlx::Error::RowNotFound => { Ok(None) }, _ => { diff --git a/crates/lib/http_server_common/src/endpoints/default_route_404.rs b/crates/lib/http_server_common/src/endpoints/default_route_404.rs index 6bc7403c15..15a0985f2a 100644 --- a/crates/lib/http_server_common/src/endpoints/default_route_404.rs +++ b/crates/lib/http_server_common/src/endpoints/default_route_404.rs @@ -1,4 +1,4 @@ -use actix_http::http::StatusCode; + use actix_web::{HttpResponse, HttpRequest, Responder, get}; use log::warn; diff --git a/crates/lib/http_server_common/src/endpoints/root_index.rs b/crates/lib/http_server_common/src/endpoints/root_index.rs index fe50b3a901..1d96a50fcf 100644 --- a/crates/lib/http_server_common/src/endpoints/root_index.rs +++ b/crates/lib/http_server_common/src/endpoints/root_index.rs @@ -1,8 +1,8 @@ use actix_http::http::StatusCode; -use actix_web::{HttpResponse, HttpRequest, get}; -use actix_web::Responder; -use actix_web::Result; -use actix_web::Error; +use actix_web::{HttpResponse}; + + + use log::debug; pub async fn get_root_index() -> HttpResponse { diff --git a/crates/lib/http_server_common/src/response/response_error_helpers.rs b/crates/lib/http_server_common/src/response/response_error_helpers.rs index 1bd6bf274b..5247186d21 100644 --- a/crates/lib/http_server_common/src/response/response_error_helpers.rs +++ b/crates/lib/http_server_common/src/response/response_error_helpers.rs @@ -1,7 +1,7 @@ use actix_http::http::header; use actix_web::HttpResponseBuilder; use actix_web::http::StatusCode; -use actix_web::{HttpResponse, ResponseError}; +use actix_web::{HttpResponse}; #[derive(Serialize)] diff --git a/crates/lib/http_server_common/src/response/serialize_as_json_error.rs b/crates/lib/http_server_common/src/response/serialize_as_json_error.rs index d288c9205e..548488e2ff 100644 --- a/crates/lib/http_server_common/src/response/serialize_as_json_error.rs +++ b/crates/lib/http_server_common/src/response/serialize_as_json_error.rs @@ -1,4 +1,4 @@ -use actix_http::http::{StatusCode, header}; +use actix_http::http::{header}; use actix_web::HttpResponse; use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; From 42637b2949283edfc0e3d979ca910e9f7590698d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 22 Dec 2021 00:12:58 -0500 Subject: [PATCH 0698/1042] Fix create category and test endpoint, add idempotency --- .../endpoints/categories/create_category.rs | 17 ++++++- debug/curl/create_category.py | 45 +++++++++++++++++++ .../2021-12-20-230000_model_categories/up.sql | 6 +++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100755 debug/curl/create_category.py diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs index ed6716b560..3a09e032be 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs @@ -31,6 +31,7 @@ use crate::database::helpers::tokens::Tokens; // =============== Request =============== #[derive(Deserialize)] +#[serde(rename_all = "snake_case")] pub enum ModelType { Tts, W2l, @@ -38,8 +39,14 @@ pub enum ModelType { #[derive(Deserialize)] pub struct CreateCategoryRequest { + // Fields for everyone pub name: Option, pub model_type: Option, + pub idempotency_token: Option, + + // Fields for moderators only + pub can_directly_have_models: Option, + pub can_have_subcategories: Option, } // =============== Success Response =============== @@ -116,6 +123,10 @@ pub async fn create_category_handler( Some(ModelType::W2l) => "w2l", }; + let idempotency_token = request.idempotency_token + .clone() + .ok_or(CreateCategoryError::BadInput("no idempotency token provided".to_string()))?; + let name = request.name .clone() .ok_or(CreateCategoryError::BadInput("no name provided".to_string()))?; @@ -141,11 +152,13 @@ pub async fn create_category_handler( INSERT INTO model_categories SET token = ?, + uuid_idempotency_token = ?, model_type = ?, name = ?, creator_user_token = ?, creator_ip_address_creation = ?, + creator_ip_address_last_update = ?, is_mod_approved = ?, maybe_mod_user_token = ?, @@ -154,10 +167,12 @@ SET "#, category_token, + idempotency_token, model_type, name, &user_session.user_token, - creator_ip_address, + &creator_ip_address, + &creator_ip_address, is_mod_approved, maybe_mod_user_token, ) diff --git a/debug/curl/create_category.py b/debug/curl/create_category.py new file mode 100755 index 0000000000..e8504b9a5b --- /dev/null +++ b/debug/curl/create_category.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +import re +import requests +import uuid + +login_url = 'http://localhost:12345/login' +category_create_url = 'http://localhost:12345/category/create' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +# NB: May not be set due to cookie domain: +#r.cookies['session'] + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + + +print("===== Create Category =====") + +cookies = { 'session': session_cookie } + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'name': 'First Category', + 'model_type': 'tts', # tts or w2l +} + +r = requests.post(category_create_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + diff --git a/migrations/2021-12-20-230000_model_categories/up.sql b/migrations/2021-12-20-230000_model_categories/up.sql index 3dbb313d4d..8f0f833e29 100644 --- a/migrations/2021-12-20-230000_model_categories/up.sql +++ b/migrations/2021-12-20-230000_model_categories/up.sql @@ -9,6 +9,11 @@ CREATE TABLE model_categories ( -- Visible "primary key" token VARCHAR(32) NOT NULL, + -- Idempotency token + -- This is so the frontend client doesn't submit duplicate requests. + -- Only used on create. + uuid_idempotency_token VARCHAR(36) NOT NULL, + -- ========== MODEL TYPE ========== model_type ENUM( @@ -77,6 +82,7 @@ CREATE TABLE model_categories ( -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (token), + UNIQUE KEY (uuid_idempotency_token), KEY fk_super_category_token (super_category_token), KEY index_model_type (model_type), KEY index_can_directly_have_models (can_directly_have_models), From 9668a3274e964fa183a657f46b87b78b77aac4c9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 22 Dec 2021 00:30:37 -0500 Subject: [PATCH 0699/1042] Add new params for mods in create category --- .../endpoints/categories/create_category.rs | 28 +++++++++++++------ debug/curl/create_category.py | 2 ++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs index 3a09e032be..b0e70f00c0 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs @@ -28,6 +28,9 @@ use std::fmt; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use crate::database::helpers::tokens::Tokens; +const DEFAULT_CAN_DIRECTLY_HAVE_MODELS : bool = true; +const DEFAULT_CAN_HAVE_SUBCATEGORIES : bool = false; + // =============== Request =============== #[derive(Deserialize)] @@ -139,13 +142,20 @@ pub async fn create_category_handler( let creator_ip_address = get_request_ip(&http_request); - let is_mod_approved = is_mod; + let mut is_mod_approved = false; + let mut maybe_mod_user_token = None; + let mut can_directly_have_models = DEFAULT_CAN_DIRECTLY_HAVE_MODELS; + let mut can_have_subcategories = DEFAULT_CAN_HAVE_SUBCATEGORIES; - let maybe_mod_user_token = if is_mod { - Some(user_session.user_token.clone()) - } else { - None - }; + if is_mod { + // Moderator fields and adjustments + is_mod_approved = true; + maybe_mod_user_token = Some(user_session.user_token.clone()); + can_directly_have_models = request.can_directly_have_models + .unwrap_or(DEFAULT_CAN_DIRECTLY_HAVE_MODELS); + can_have_subcategories= request.can_have_subcategories + .unwrap_or(DEFAULT_CAN_HAVE_SUBCATEGORIES); + } let query_result = sqlx::query!( r#" @@ -162,8 +172,8 @@ SET is_mod_approved = ?, maybe_mod_user_token = ?, - can_directly_have_models = true, - can_have_subcategories = false + can_directly_have_models = ?, + can_have_subcategories = ? "#, category_token, @@ -175,6 +185,8 @@ SET &creator_ip_address, is_mod_approved, maybe_mod_user_token, + can_directly_have_models, + can_have_subcategories ) .execute(&server_state.mysql_pool) .await; diff --git a/debug/curl/create_category.py b/debug/curl/create_category.py index e8504b9a5b..3af9ffe4ae 100755 --- a/debug/curl/create_category.py +++ b/debug/curl/create_category.py @@ -36,6 +36,8 @@ payload = { 'idempotency_token': str(uuid.uuid4()), 'name': 'First Category', 'model_type': 'tts', # tts or w2l + 'can_directly_have_models': False, + 'can_have_subcategories': True, } r = requests.post(category_create_url, cookies=cookies, json=payload) From eb11bb5118b22836ea865f998b8c2ab4e4184fba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 22 Dec 2021 01:02:11 -0500 Subject: [PATCH 0700/1042] Delete and undelete categories --- .../endpoints/categories/delete_category.rs | 156 ++++++++++++++++++ crates/app/storyteller_web/src/routes.rs | 6 + .../{create_category.py => category_tests.py} | 37 +++++ 3 files changed, 199 insertions(+) rename debug/curl/{create_category.py => category_tests.py} (58%) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/delete_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/delete_category.rs index e69de29bb2..db6be435a3 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/delete_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/delete_category.rs @@ -0,0 +1,156 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use std::fmt; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use crate::database::helpers::tokens::Tokens; + +// =============== Request =============== + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct DeleteCategoryPathInfo { + token: String, +} + +#[derive(Deserialize)] +pub struct DeleteCategoryRequest { + set_delete: bool, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct DeleteCategoryResponse { + pub success: bool, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum DeleteCategoryError { + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for DeleteCategoryError { + fn status_code(&self) -> StatusCode { + match *self { + DeleteCategoryError::NotFound => StatusCode::NOT_FOUND, + DeleteCategoryError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteCategoryError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for DeleteCategoryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn delete_category_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + DeleteCategoryError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(DeleteCategoryError::NotAuthorized); + } + }; + + // TODO: We don't have a permission for categories, so we use this as a proxy. + if !user_session.can_ban_users { + warn!("no permission to delete categories"); + return Err(DeleteCategoryError::NotAuthorized); + } + + let query_builder = if request.set_delete { + sqlx::query!(r#" + UPDATE model_categories + SET + deleted_at = CURRENT_TIMESTAMP + WHERE + token = ? + LIMIT 1 + "#, + path.token) + + } else { + sqlx::query!(r#" + UPDATE model_categories + SET + deleted_at = NULL + WHERE + token = ? + LIMIT 1 + "#, + path.token) + }; + + // NB: We're soft deleting so we don't delete the associations. + let query_result = query_builder.execute(&server_state.mysql_pool).await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Delete/undelete category edit DB error: {:?}", err); + return Err(DeleteCategoryError::ServerError); + } + }; + + let response = DeleteCategoryResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| DeleteCategoryError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/app/storyteller_web/src/routes.rs index b4820a580d..8cd91ddc32 100644 --- a/crates/app/storyteller_web/src/routes.rs +++ b/crates/app/storyteller_web/src/routes.rs @@ -60,6 +60,7 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; +use crate::http_server::endpoints::categories::delete_category::delete_category_handler; pub fn add_routes (app: App) -> App where @@ -414,6 +415,11 @@ fn add_category_routes (app: App) -> App .route(web::post().to(create_category_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/{token}/delete") + .route(web::post().to(delete_category_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) } diff --git a/debug/curl/create_category.py b/debug/curl/category_tests.py similarity index 58% rename from debug/curl/create_category.py rename to debug/curl/category_tests.py index 3af9ffe4ae..c817819532 100755 --- a/debug/curl/create_category.py +++ b/debug/curl/category_tests.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 +import json import re import requests import uuid login_url = 'http://localhost:12345/login' category_create_url = 'http://localhost:12345/category/create' +category_delete_url_format = 'http://localhost:12345/category/{}/delete' payload = { 'username_or_email': 'echelon', @@ -42,6 +44,41 @@ payload = { r = requests.post(category_create_url, cookies=cookies, json=payload) +print('Status: {}'.format(r.status_code)) + +d = json.loads(r.content) +category_token = d['token'] + +print(f'created category token: #{category_token}') + + +print("===== (Soft) Delete Category =====") + +category_delete_url = category_delete_url_format.format(category_token) + +cookies = { 'session': session_cookie } + +payload = { + 'set_delete': True, +} + +r = requests.post(category_delete_url, cookies=cookies, json=payload) + print('Status: {}'.format(r.status_code)) print(r.content) + +print("===== Undelete Category =====") + +cookies = { 'session': session_cookie } + +payload = { + 'set_delete': False, +} + +r = requests.post(category_delete_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + + From 1e0847ecddb6f8da04541e2e5978320c8797ce3b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 22 Dec 2021 03:16:53 -0500 Subject: [PATCH 0701/1042] hypothetical rustfmt rules (not yet applied to codebase) --- rustfmt.toml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 rustfmt.toml diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000000..00be236ad8 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,39 @@ +# Storyteller rustfmt rules +# (Not yet happy with these.) + +# Four spaces is excessive in any language. +# Conserve horizontal space for small laptop screens. +tab_spaces = 2 + +# Only Unix +newline_style = "unix" + +# We won't change any vendor code. +# Note: This appears not to work in stable, so it's necessary to run `cargo +nightly fmt`. +ignore = [ + "vendor" +] + +# Item-level imports makes for better diffs and fewer merge conflicts. +# Another suitable option is "Module", but that's not the default either. +# (This is commented out for now to reduce the first round of rustfmt diffs, but will be enabled.) +#imports_granularity = "Item" + +# It's better to keep stdlib, crate (self), and external crates separate. +# (This is commented out for now to reduce the first round of rustfmt diffs, but will be enabled.) +#group_imports = "StdExternalCrate" + +# The default is wrong +match_block_trailing_comma = true + +# 'try!' shouldn't exist in our codebase. +use_try_shorthand = true + +# MAYBE +struct_lit_single_line = true +use_small_heuristics = "Off" # Fix for method chaining? + +# TEMPORARY +reorder_imports = false +#max_width = 180 # default 100 +#chain_width = 10 From d23859fb75af34fc9410d4d9da3d99caf924b5b4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 22 Dec 2021 03:36:26 -0500 Subject: [PATCH 0702/1042] better rustfmt rules? still not ready to apply these --- rustfmt.toml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/rustfmt.toml b/rustfmt.toml index 00be236ad8..e4a54388f3 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -17,11 +17,11 @@ ignore = [ # Item-level imports makes for better diffs and fewer merge conflicts. # Another suitable option is "Module", but that's not the default either. # (This is commented out for now to reduce the first round of rustfmt diffs, but will be enabled.) -#imports_granularity = "Item" +imports_granularity = "Item" # It's better to keep stdlib, crate (self), and external crates separate. # (This is commented out for now to reduce the first round of rustfmt diffs, but will be enabled.) -#group_imports = "StdExternalCrate" +group_imports = "StdExternalCrate" # The default is wrong match_block_trailing_comma = true @@ -30,10 +30,13 @@ match_block_trailing_comma = true use_try_shorthand = true # MAYBE -struct_lit_single_line = true -use_small_heuristics = "Off" # Fix for method chaining? +#struct_lit_single_line = true + +# This fixes method chaining. +# (rustfmt ordinarily puts everything on one line.) +use_small_heuristics = "Off" # TEMPORARY -reorder_imports = false +#reorder_imports = false #max_width = 180 # default 100 #chain_width = 10 From c34de2b8d7611488141231b67cabd6b017574ec5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 22 Dec 2021 04:42:27 -0500 Subject: [PATCH 0703/1042] Minor cleanup --- crates/lib/config/src/lib.rs | 3 +-- .../http_server_common/src/endpoints/default_route_404.rs | 3 +-- crates/lib/http_server_common/src/endpoints/root_index.rs | 2 +- .../src/response/response_error_helpers.rs | 5 ++--- .../src/response/serialize_as_json_error.rs | 3 +-- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/crates/lib/config/src/lib.rs b/crates/lib/config/src/lib.rs index 6faad52aea..f78455f59a 100644 --- a/crates/lib/config/src/lib.rs +++ b/crates/lib/config/src/lib.rs @@ -1,2 +1 @@ - -pub mod shared_constants; \ No newline at end of file +pub mod shared_constants; diff --git a/crates/lib/http_server_common/src/endpoints/default_route_404.rs b/crates/lib/http_server_common/src/endpoints/default_route_404.rs index 15a0985f2a..117ed03336 100644 --- a/crates/lib/http_server_common/src/endpoints/default_route_404.rs +++ b/crates/lib/http_server_common/src/endpoints/default_route_404.rs @@ -1,5 +1,4 @@ - -use actix_web::{HttpResponse, HttpRequest, Responder, get}; +use actix_web::{HttpResponse, Responder}; use log::warn; pub async fn default_route_404() -> impl Responder { diff --git a/crates/lib/http_server_common/src/endpoints/root_index.rs b/crates/lib/http_server_common/src/endpoints/root_index.rs index 1d96a50fcf..ba745827f5 100644 --- a/crates/lib/http_server_common/src/endpoints/root_index.rs +++ b/crates/lib/http_server_common/src/endpoints/root_index.rs @@ -1,5 +1,5 @@ use actix_http::http::StatusCode; -use actix_web::{HttpResponse}; +use actix_web::HttpResponse; diff --git a/crates/lib/http_server_common/src/response/response_error_helpers.rs b/crates/lib/http_server_common/src/response/response_error_helpers.rs index 5247186d21..20e0010716 100644 --- a/crates/lib/http_server_common/src/response/response_error_helpers.rs +++ b/crates/lib/http_server_common/src/response/response_error_helpers.rs @@ -1,7 +1,6 @@ -use actix_http::http::header; use actix_web::HttpResponseBuilder; use actix_web::http::StatusCode; -use actix_web::{HttpResponse}; +use actix_web::HttpResponse; #[derive(Serialize)] @@ -26,6 +25,6 @@ pub fn to_simple_json_error( }; HttpResponseBuilder::new(status_code) - .set_header(header::CONTENT_TYPE, "application/json") + .content_type("application/json") .body(body) } \ No newline at end of file diff --git a/crates/lib/http_server_common/src/response/serialize_as_json_error.rs b/crates/lib/http_server_common/src/response/serialize_as_json_error.rs index 548488e2ff..9dec81d745 100644 --- a/crates/lib/http_server_common/src/response/serialize_as_json_error.rs +++ b/crates/lib/http_server_common/src/response/serialize_as_json_error.rs @@ -1,4 +1,3 @@ -use actix_http::http::{header}; use actix_web::HttpResponse; use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; @@ -16,6 +15,6 @@ pub fn serialize_as_json_error( }; HttpResponseBuilder::new(error_payload.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") + .content_type("application/json") .body(body) } From 732c90f1e60e38c30e790f51c0e71b512f9d90d6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 22 Dec 2021 05:10:29 -0500 Subject: [PATCH 0704/1042] edit category endpoint --- .../endpoints/categories/create_category.rs | 8 +- .../endpoints/categories/edit_category.rs | 177 ++++++++++++++++++ crates/app/storyteller_web/src/routes.rs | 6 + debug/curl/category_tests.py | 22 +++ 4 files changed, 209 insertions(+), 4 deletions(-) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs index b0e70f00c0..1abeaa7a35 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs @@ -1,12 +1,13 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::helpers::tokens::Tokens; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -17,16 +18,15 @@ use crate::util::markdown_to_html::markdown_to_html; use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::validations::check_for_slurs::contains_slurs; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; -use std::sync::Arc; use std::fmt; -use http_server_common::response::serialize_as_json_error::serialize_as_json_error; -use crate::database::helpers::tokens::Tokens; +use std::sync::Arc; const DEFAULT_CAN_DIRECTLY_HAVE_MODELS : bool = true; const DEFAULT_CAN_HAVE_SUBCATEGORIES : bool = false; diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs index e69de29bb2..3bcc4b925f 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs @@ -0,0 +1,177 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::helpers::tokens::Tokens; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Request =============== + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct EditCategoryPathInfo { + token: String, +} + +// NB: Only moderators can edit categories. +// These are not sparse updates! +#[derive(Deserialize)] +pub struct EditCategoryRequest { + pub name: String, + // If absent, null the field. + pub dropdown_name: Option, + + pub can_directly_have_models: bool, + pub can_have_subcategories: bool, + + // Moderation fields + pub is_mod_approved: bool, + // If absent, null the field. + pub maybe_mod_comments: Option, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct EditCategoryResponse { + pub success: bool, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum EditCategoryError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for EditCategoryError { + fn status_code(&self) -> StatusCode { + match *self { + EditCategoryError::BadInput(_) => StatusCode::BAD_REQUEST, + EditCategoryError::NotFound => StatusCode::NOT_FOUND, + EditCategoryError::NotAuthorized => StatusCode::UNAUTHORIZED, + EditCategoryError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for EditCategoryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn edit_category_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EditCategoryError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(EditCategoryError::NotAuthorized); + } + }; + + // TODO: We don't have a permission for categories, so we use this as a proxy. + if !user_session.can_ban_users { + warn!("no permission to edit categories"); + return Err(EditCategoryError::NotAuthorized); + } + + let query_result = + // We need to store the IP address details. + sqlx::query!( + r#" +UPDATE model_categories +SET + name = ?, + dropdown_name = ?, + + can_directly_have_models = ?, + can_have_subcategories = ?, + + is_mod_approved = ?, + maybe_mod_user_token = ?, + maybe_mod_comments = ?, + + version = version + 1 + +WHERE token = ? +LIMIT 1 + "#, + &request.name, + &request.dropdown_name, + &request.can_directly_have_models, + &request.can_have_subcategories, + &request.is_mod_approved, + &user_session.user_token, + &request.maybe_mod_comments, + &path.token, + ) + .execute(&server_state.mysql_pool) + .await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Edit category DB error: {:?}", err); + return Err(EditCategoryError::ServerError); + } + }; + + let response = EditCategoryResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| EditCategoryError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/app/storyteller_web/src/routes.rs index 8cd91ddc32..742690606e 100644 --- a/crates/app/storyteller_web/src/routes.rs +++ b/crates/app/storyteller_web/src/routes.rs @@ -61,6 +61,7 @@ use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_ use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; use crate::http_server::endpoints::categories::delete_category::delete_category_handler; +use crate::http_server::endpoints::categories::edit_category::edit_category_handler; pub fn add_routes (app: App) -> App where @@ -420,6 +421,11 @@ fn add_category_routes (app: App) -> App .route(web::post().to(delete_category_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/{token}/edit") + .route(web::post().to(edit_category_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) } diff --git a/debug/curl/category_tests.py b/debug/curl/category_tests.py index c817819532..6600c8a310 100755 --- a/debug/curl/category_tests.py +++ b/debug/curl/category_tests.py @@ -8,6 +8,7 @@ import uuid login_url = 'http://localhost:12345/login' category_create_url = 'http://localhost:12345/category/create' category_delete_url_format = 'http://localhost:12345/category/{}/delete' +category_edit_url_format = 'http://localhost:12345/category/{}/edit' payload = { 'username_or_email': 'echelon', @@ -82,3 +83,24 @@ print('Status: {}'.format(r.status_code)) print(r.content) +print("===== Edit Category =====") + +category_edit_url = category_edit_url_format.format(category_token) + +cookies = { 'session': session_cookie } + +payload = { + 'name': 'Edited Name', + 'dropdown_name': 'Edited Dropdown Name', + 'can_directly_have_models': True, + 'can_have_subcategories': True, + 'is_mod_approved': True, + 'maybe_mod_comments': 'mod comment', +} + +r = requests.post(category_edit_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + + From 48394fe5f8fdbb064cb85907b5618daddd4b14e3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 27 Dec 2021 04:25:40 -0500 Subject: [PATCH 0705/1042] modify category schema; create category list query builder --- .../list_categories_query_builder.rs | 302 ++++++++++++++++++ .../src/database/query_builders/mod.rs | 1 + .../endpoints/categories/edit_category.rs | 19 +- .../categories/list_my_categories.rs | 0 .../http_server/endpoints/categories/mod.rs | 3 +- .../2021-12-20-230000_model_categories/up.sql | 10 +- 6 files changed, 326 insertions(+), 9 deletions(-) create mode 100644 crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/list_my_categories.rs diff --git a/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs b/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs new file mode 100644 index 0000000000..3948c6d17c --- /dev/null +++ b/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs @@ -0,0 +1,302 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; +use crate::util::anyhow_result::AnyhowResult; +use log::{warn, info}; +use sqlx::MySqlPool; + +/// List of categories +/// This is *NOT* paginated, as that defeats the purpose of having a category system. +#[derive(Serialize)] +pub struct CategoryList { + pub categories: Vec, +} + +#[derive(Serialize)] +pub struct Category { + pub category_token: String, + + pub model_type: String, // TODO: ENUM + + pub maybe_super_category_token: Option, + + pub can_directly_have_models: bool, + pub can_have_subcategories: bool, + pub can_only_mods_apply: bool, + + pub name: String, + pub maybe_dropdown_name: Option, + + pub creator_user_token: Option, + pub creator_username: Option, + pub creator_display_name: Option, + + // Moderator fields + pub is_mod_approved: Option, + pub maybe_mod_comments: Option, + //pub maybe_mod_user_token: Option, + //pub maybe_mod_username: Option, + //pub maybe_mod_display_name: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub deleted_at: Option>, +} + +/// Query Builder for listing categories +/// These are very difficult queries, so the builder helps for +/// testability, construction, and correctness. +pub struct ListCategoriesQueryBuilder { + // The public lists won't include unapproved/deleted, but mod + // views will want to see both. + show_unapproved: bool, + + // Useful for showing categories a user themselves has suggested + // We don't want users spamming creation. + scope_creator_user_token: Option, + + // Typically we won't show both types together. + // Except in a mod view or user view, perhaps. + scope_model_type: Option, // todo: enum +} + +impl ListCategoriesQueryBuilder { + pub fn new() -> Self { + Self { + show_unapproved: true, + scope_creator_user_token: None, + scope_model_type: None, + } + } + + pub fn show_unapproved(mut self, show_unapproved: bool) -> Self { + self.show_unapproved = show_unapproved; + self + } + + pub fn scope_creator_user_token(mut self, scope_creator_user_token: Option<&str>) -> Self { + self.scope_creator_user_token = scope_creator_user_token.map(|u| u.to_string()); + self + } + + pub fn scope_model_type(mut self, scope_model_type: Option<&str>) -> Self { + self.scope_model_type = scope_model_type.map(|u| u.to_string()); + self + } + + /// Perform the query based on the set predicates. + pub async fn perform_query_for_page( + &self, + mysql_pool: &MySqlPool + ) -> AnyhowResult { + + let internal_results = self.perform_internal_query(mysql_pool).await?; + + let categories = internal_results + .iter() + .map(|c| { + Category { + category_token: c.category_token.clone(), + model_type: c.model_type.clone(), + maybe_super_category_token: c.maybe_super_category_token.clone(), + can_directly_have_models: i8_to_bool(c.can_directly_have_models), + can_have_subcategories: i8_to_bool(c.can_have_subcategories), + can_only_mods_apply: i8_to_bool(c.can_only_mods_apply), + name: c.name.clone(), + maybe_dropdown_name: c.maybe_dropdown_name.clone(), + creator_user_token: c.creator_user_token.clone(), + creator_username: c.creator_username.clone(), + creator_display_name: c.creator_display_name.clone(), + is_mod_approved: nullable_i8_to_optional_bool(c.is_mod_approved), + maybe_mod_comments: c.maybe_mod_comments.clone(), + created_at: c.created_at, + updated_at: c.updated_at, + deleted_at: c.deleted_at.clone(), + } + }) + .collect::>(); + + Ok(CategoryList { + categories, + }) + } + + /// Perform the query based on the set predicates. + pub async fn perform_internal_query( + &self, + mysql_pool: &MySqlPool + ) -> AnyhowResult> { + + let query = self.build_query_string(); + let mut query = sqlx::query_as::<_, RawInternalCategoryRecord>(&query); + + // NB: The following bindings must match the order of the query builder !! + + if let Some(user_token) = self.scope_creator_user_token.as_deref() { + query = query.bind(user_token); + } + + if let Some(model_type) = self.scope_model_type.as_deref() { + query = query.bind(model_type); + } + + let mut results = query.fetch_all(mysql_pool) + .await?; + + Ok(results) + } + + pub fn build_query_string(&self) -> String { + // TODO: I haven't figured out how to get field name disambiguation and type coercion working here. + // (1) tts_results.creator_set_visibility `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, + // Query error: no column found for name: creator_set_visibility + // (2) creator_set_visibility `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, + // Column 'creator_set_visibility' in field list is ambiguous + + // TODO/NB: Unfortunately SQLx can't statically typecheck this query + let mut query = r#" +SELECT + model_categories.token as model_token, + + model_categories.model_type, + + model_categories.maybe_super_category_token, + + model_categories.can_directly_have_models, + model_categories.can_have_subcategories, + model_categories.can_only_mods_apply, + + model_categories.name, + model_categories.maybe_dropdown_name, + + users.token as creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + + model_categories.is_mod_approved, + model_categories.maybe_mod_comments, + + model_categories.created_at, + model_categories.updated_at, + model_categories.deleted_at + +FROM model_categories +LEFT OUTER JOIN users + ON model_categories.creator_user_token = users.token + "#.to_string(); + + query.push_str(&self.build_predicates()); + query + } + + pub fn build_predicates(&self) -> String { + let mut first_predicate_added = false; + + let mut query = "".to_string(); + + if let Some(username) = self.scope_creator_user_token.as_deref() { + if !first_predicate_added { + query.push_str(" WHERE model_categories.creator_user_token = ?"); + first_predicate_added = true; + } else { + query.push_str(" AND model_categories.creator_user_token = ?"); + } + } + + if let Some(model_type) = self.scope_model_type.as_deref() { + if !first_predicate_added { + query.push_str(" WHERE model_categories.model_type = ?"); + first_predicate_added = true; + } else { + query.push_str(" AND model_categories.model_type = ?"); + } + } + + if !self.show_unapproved { + if !first_predicate_added { + query.push_str(" WHERE model_categories.is_mod_approved IS TRUE"); + query.push_str(" AND model_categories.deleted_at IS NOT NULL"); + first_predicate_added = true; + } else { + query.push_str(" AND model_categories.is_mod_approved IS TRUE"); + query.push_str(" AND model_categories.deleted_at IS NOT NULL"); + } + } + + query + } +} + +#[derive(sqlx::FromRow)] +pub struct RawInternalCategoryRecord { + pub category_token: String, + + pub model_type: String, // TODO: ENUM + + pub maybe_super_category_token: Option, + + pub can_directly_have_models: i8, + pub can_have_subcategories: i8, + pub can_only_mods_apply: i8, + + pub name: String, + pub maybe_dropdown_name: Option, + + pub creator_user_token: Option, + pub creator_username: Option, + pub creator_display_name: Option, + + // Moderator fields + pub is_mod_approved: Option, + pub maybe_mod_comments: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub deleted_at: Option>, +} + +#[cfg(test)] +mod tests { + use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; + + #[test] + fn predicates_without_scoping() { + let query_builder = ListCategoriesQueryBuilder::new(); + + assert_eq!(&query_builder.build_predicates(), ""); + } + + #[test] + fn predicates_scoped_to_user() { + let query_builder = ListCategoriesQueryBuilder::new() + .scope_creator_user_token(Some("U:ASDF")); + + assert_eq!(&query_builder.build_predicates(), + " WHERE model_categories.creator_user_token = ?"); + } + +// #[test] +// fn predicates_including_user_hidden() { +// let query_builder = ListCategoriesQueryBuilder::new() +// .include_user_hidden(true); +// +// assert_eq!(&query_builder.build_predicates(), +// " WHERE tts_results.user_deleted_at IS NULL \ +// AND tts_results.mod_deleted_at IS NULL \ +// ORDER BY tts_results.id DESC \ +// LIMIT ?"); +// } +// +// #[test] +// fn predicates_including_deleted() { +// let query_builder = ListCategoriesQueryBuilder::new() +// .include_mod_disabled_results(true); +// +// assert_eq!(&query_builder.build_predicates(), +// " WHERE tts_results.creator_set_visibility = 'public' \ +// ORDER BY tts_results.id DESC \ +// LIMIT ?"); +// } +} diff --git a/crates/app/storyteller_web/src/database/query_builders/mod.rs b/crates/app/storyteller_web/src/database/query_builders/mod.rs index 5ff3bb338b..072361672c 100644 --- a/crates/app/storyteller_web/src/database/query_builders/mod.rs +++ b/crates/app/storyteller_web/src/database/query_builders/mod.rs @@ -1,3 +1,4 @@ +pub mod list_categories_query_builder; pub mod list_tts_inference_results_query_builder; pub mod list_users_query_builder; pub mod list_w2l_inference_results_query_builder; diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs index 3bcc4b925f..91dfa56a8e 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs @@ -36,19 +36,23 @@ pub struct EditCategoryPathInfo { token: String, } -// NB: Only moderators can edit categories. +// NB: ONLY MODERATORS CAN EDIT CATEGORIES. // These are not sparse updates! #[derive(Deserialize)] pub struct EditCategoryRequest { pub name: String, - // If absent, null the field. - pub dropdown_name: Option, + + // If absent, null the fields. + pub maybe_dropdown_name: Option, + pub maybe_super_category_token: Option, pub can_directly_have_models: bool, pub can_have_subcategories: bool, + pub can_only_mods_apply: bool, // Moderation fields pub is_mod_approved: bool, + // If absent, null the field. pub maybe_mod_comments: Option, } @@ -130,10 +134,13 @@ pub async fn edit_category_handler( UPDATE model_categories SET name = ?, - dropdown_name = ?, + maybe_dropdown_name = ?, can_directly_have_models = ?, can_have_subcategories = ?, + can_only_mods_apply = ?, + + maybe_super_category_token = ?, is_mod_approved = ?, maybe_mod_user_token = ?, @@ -145,9 +152,11 @@ WHERE token = ? LIMIT 1 "#, &request.name, - &request.dropdown_name, + &request.maybe_dropdown_name, &request.can_directly_have_models, &request.can_have_subcategories, + &request.can_only_mods_apply, + &request.maybe_super_category_token, &request.is_mod_approved, &user_session.user_token, &request.maybe_mod_comments, diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_my_categories.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_my_categories.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs index ed0a0cfaed..621958b942 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs @@ -1,4 +1,5 @@ pub mod create_category; pub mod delete_category; pub mod edit_category; -pub mod list_categories; \ No newline at end of file +pub mod list_categories; +pub mod list_my_categories; \ No newline at end of file diff --git a/migrations/2021-12-20-230000_model_categories/up.sql b/migrations/2021-12-20-230000_model_categories/up.sql index 8f0f833e29..e7feeee433 100644 --- a/migrations/2021-12-20-230000_model_categories/up.sql +++ b/migrations/2021-12-20-230000_model_categories/up.sql @@ -26,7 +26,7 @@ CREATE TABLE model_categories ( -- Optional foreign key link to super category. -- If absent, this is a top-level category or top-level super category. - super_category_token VARCHAR(32) DEFAULT NULL, + maybe_super_category_token VARCHAR(32) DEFAULT NULL, -- If this category can be applied directly to models. can_directly_have_models BOOLEAN NOT NULL DEFAULT false, @@ -34,6 +34,9 @@ CREATE TABLE model_categories ( -- If this category can have subcategories. can_have_subcategories BOOLEAN NOT NULL DEFAULT false, + -- If only mods are allowed to tag (or untag) models with this. + can_only_mods_apply BOOLEAN NOT NULL DEFAULT false, + -- ========== CATEGORY NAME ========== -- The name of the category, which might be complicated. @@ -41,7 +44,7 @@ CREATE TABLE model_categories ( -- The name displayed in the dropdown (if it should be different), otherwise NULL. -- eg. "Gender" (name) and "By Gender" (dropdown name) - dropdown_name VARCHAR(20) DEFAULT NULL, + maybe_dropdown_name VARCHAR(20) DEFAULT NULL, -- ========== CREATOR DETAILS ========== @@ -83,7 +86,8 @@ CREATE TABLE model_categories ( PRIMARY KEY (id), UNIQUE KEY (token), UNIQUE KEY (uuid_idempotency_token), - KEY fk_super_category_token (super_category_token), + KEY fk_maybe_super_category_token (maybe_super_category_token), + KEY fk_creator_user_token (creator_user_token), KEY index_model_type (model_type), KEY index_can_directly_have_models (can_directly_have_models), KEY index_can_have_subcategories (can_have_subcategories), From d3435a1f4b28c148c4cdb94a4fdec79da03bb800 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 28 Dec 2021 02:09:19 -0500 Subject: [PATCH 0706/1042] fix tests on list categories, split filtering of deleted/unapproved --- .../list_categories_query_builder.rs | 99 +++++++++++++------ 1 file changed, 70 insertions(+), 29 deletions(-) diff --git a/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs b/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs index 3948c6d17c..ebf79ecc22 100644 --- a/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs +++ b/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs @@ -49,10 +49,14 @@ pub struct Category { /// These are very difficult queries, so the builder helps for /// testability, construction, and correctness. pub struct ListCategoriesQueryBuilder { - // The public lists won't include unapproved/deleted, but mod - // views will want to see both. + // The public lists won't include unapproved, but mod views + // will want to see them. show_unapproved: bool, + // The public lists won't include deleted models, but mod views + // will want to see them. + show_deleted: bool, + // Useful for showing categories a user themselves has suggested // We don't want users spamming creation. scope_creator_user_token: Option, @@ -65,7 +69,8 @@ pub struct ListCategoriesQueryBuilder { impl ListCategoriesQueryBuilder { pub fn new() -> Self { Self { - show_unapproved: true, + show_unapproved: false, + show_deleted: false, scope_creator_user_token: None, scope_model_type: None, } @@ -76,6 +81,11 @@ impl ListCategoriesQueryBuilder { self } + pub fn show_deleted(mut self, show_deleted: bool) -> Self { + self.show_deleted = show_deleted; + self + } + pub fn scope_creator_user_token(mut self, scope_creator_user_token: Option<&str>) -> Self { self.scope_creator_user_token = scope_creator_user_token.map(|u| u.to_string()); self @@ -217,10 +227,17 @@ LEFT OUTER JOIN users if !self.show_unapproved { if !first_predicate_added { query.push_str(" WHERE model_categories.is_mod_approved IS TRUE"); - query.push_str(" AND model_categories.deleted_at IS NOT NULL"); first_predicate_added = true; } else { query.push_str(" AND model_categories.is_mod_approved IS TRUE"); + } + } + + if !self.show_deleted { + if !first_predicate_added { + query.push_str(" WHERE model_categories.deleted_at IS NOT NULL"); + first_predicate_added = true; + } else { query.push_str(" AND model_categories.deleted_at IS NOT NULL"); } } @@ -262,10 +279,12 @@ mod tests { use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; #[test] - fn predicates_without_scoping() { + fn predicates_default_scoping() { let query_builder = ListCategoriesQueryBuilder::new(); - assert_eq!(&query_builder.build_predicates(), ""); + assert_eq!(&query_builder.build_predicates(), + " WHERE model_categories.is_mod_approved IS TRUE \ + AND model_categories.deleted_at IS NOT NULL"); } #[test] @@ -274,29 +293,51 @@ mod tests { .scope_creator_user_token(Some("U:ASDF")); assert_eq!(&query_builder.build_predicates(), - " WHERE model_categories.creator_user_token = ?"); + " WHERE model_categories.creator_user_token = ? \ + AND model_categories.is_mod_approved IS TRUE \ + AND model_categories.deleted_at IS NOT NULL"); } -// #[test] -// fn predicates_including_user_hidden() { -// let query_builder = ListCategoriesQueryBuilder::new() -// .include_user_hidden(true); -// -// assert_eq!(&query_builder.build_predicates(), -// " WHERE tts_results.user_deleted_at IS NULL \ -// AND tts_results.mod_deleted_at IS NULL \ -// ORDER BY tts_results.id DESC \ -// LIMIT ?"); -// } -// -// #[test] -// fn predicates_including_deleted() { -// let query_builder = ListCategoriesQueryBuilder::new() -// .include_mod_disabled_results(true); -// -// assert_eq!(&query_builder.build_predicates(), -// " WHERE tts_results.creator_set_visibility = 'public' \ -// ORDER BY tts_results.id DESC \ -// LIMIT ?"); -// } + #[test] + fn predicates_scoped_to_model_type() { + let query_builder = ListCategoriesQueryBuilder::new() + .scope_model_type(Some("foo")); + + assert_eq!(&query_builder.build_predicates(), + " WHERE model_categories.model_type = ? \ + AND model_categories.is_mod_approved IS TRUE \ + AND model_categories.deleted_at IS NOT NULL"); + } + + #[test] + fn predicates_scoped_to_unapproved() { + let query_builder = ListCategoriesQueryBuilder::new() + .show_unapproved(false); + + assert_eq!(&query_builder.build_predicates(), + " WHERE model_categories.is_mod_approved IS TRUE \ + AND model_categories.deleted_at IS NOT NULL"); + + let query_builder = ListCategoriesQueryBuilder::new() + .show_unapproved(true); + + assert_eq!(&query_builder.build_predicates(), + " WHERE model_categories.deleted_at IS NOT NULL"); + } + + #[test] + fn predicates_scoped_to_deleted() { + let query_builder = ListCategoriesQueryBuilder::new() + .show_deleted(false); + + assert_eq!(&query_builder.build_predicates(), + " WHERE model_categories.is_mod_approved IS TRUE \ + AND model_categories.deleted_at IS NOT NULL"); + + let query_builder = ListCategoriesQueryBuilder::new() + .show_deleted(true); + + assert_eq!(&query_builder.build_predicates(), + " WHERE model_categories.is_mod_approved IS TRUE"); + } } From 556fa360d3462fc8510be0de52c9c162a0306915 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 28 Dec 2021 03:14:37 -0500 Subject: [PATCH 0707/1042] add list tts category handler --- .../list_categories_query_builder.rs | 25 ++- .../endpoints/categories/list_categories.rs | 0 .../categories/list_tts_categories.rs | 143 ++++++++++++++++++ .../http_server/endpoints/categories/mod.rs | 2 +- crates/app/storyteller_web/src/routes.rs | 9 ++ 5 files changed, 165 insertions(+), 14 deletions(-) delete mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/list_categories.rs create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs diff --git a/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs b/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs index ebf79ecc22..784247c5c3 100644 --- a/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs +++ b/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs @@ -1,6 +1,5 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; use crate::util::anyhow_result::AnyhowResult; @@ -97,7 +96,7 @@ impl ListCategoriesQueryBuilder { } /// Perform the query based on the set predicates. - pub async fn perform_query_for_page( + pub async fn perform_query( &self, mysql_pool: &MySqlPool ) -> AnyhowResult { @@ -134,7 +133,7 @@ impl ListCategoriesQueryBuilder { } /// Perform the query based on the set predicates. - pub async fn perform_internal_query( + async fn perform_internal_query( &self, mysql_pool: &MySqlPool ) -> AnyhowResult> { @@ -158,7 +157,7 @@ impl ListCategoriesQueryBuilder { Ok(results) } - pub fn build_query_string(&self) -> String { + fn build_query_string(&self) -> String { // TODO: I haven't figured out how to get field name disambiguation and type coercion working here. // (1) tts_results.creator_set_visibility `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, // Query error: no column found for name: creator_set_visibility @@ -168,7 +167,7 @@ impl ListCategoriesQueryBuilder { // TODO/NB: Unfortunately SQLx can't statically typecheck this query let mut query = r#" SELECT - model_categories.token as model_token, + model_categories.token as category_token, model_categories.model_type, @@ -235,10 +234,10 @@ LEFT OUTER JOIN users if !self.show_deleted { if !first_predicate_added { - query.push_str(" WHERE model_categories.deleted_at IS NOT NULL"); + query.push_str(" WHERE model_categories.deleted_at IS NULL"); first_predicate_added = true; } else { - query.push_str(" AND model_categories.deleted_at IS NOT NULL"); + query.push_str(" AND model_categories.deleted_at IS NULL"); } } @@ -284,7 +283,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE model_categories.is_mod_approved IS TRUE \ - AND model_categories.deleted_at IS NOT NULL"); + AND model_categories.deleted_at IS NULL"); } #[test] @@ -295,7 +294,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE model_categories.creator_user_token = ? \ AND model_categories.is_mod_approved IS TRUE \ - AND model_categories.deleted_at IS NOT NULL"); + AND model_categories.deleted_at IS NULL"); } #[test] @@ -306,7 +305,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE model_categories.model_type = ? \ AND model_categories.is_mod_approved IS TRUE \ - AND model_categories.deleted_at IS NOT NULL"); + AND model_categories.deleted_at IS NULL"); } #[test] @@ -316,13 +315,13 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE model_categories.is_mod_approved IS TRUE \ - AND model_categories.deleted_at IS NOT NULL"); + AND model_categories.deleted_at IS NULL"); let query_builder = ListCategoriesQueryBuilder::new() .show_unapproved(true); assert_eq!(&query_builder.build_predicates(), - " WHERE model_categories.deleted_at IS NOT NULL"); + " WHERE model_categories.deleted_at IS NULL"); } #[test] @@ -332,7 +331,7 @@ mod tests { assert_eq!(&query_builder.build_predicates(), " WHERE model_categories.is_mod_approved IS TRUE \ - AND model_categories.deleted_at IS NOT NULL"); + AND model_categories.deleted_at IS NULL"); let query_builder = ListCategoriesQueryBuilder::new() .show_deleted(true); diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_categories.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_categories.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs new file mode 100644 index 0000000000..832fc12d9e --- /dev/null +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs @@ -0,0 +1,143 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct ListTtsCategoriesResponse { + pub success: bool, + pub categories: Vec, +} + +/// Public-facing and optimized (non-irrelevant fields) category list +/// Used for the main TTS dropdown as well as the TTS edit/tagging UI +#[derive(Serialize)] +pub struct DisplayCategory { + pub category_token: String, + + pub model_type: String, // TODO: ENUM + + pub maybe_super_category_token: Option, + + pub can_directly_have_models: bool, + pub can_have_subcategories: bool, + pub can_only_mods_apply: bool, + + pub name: String, + pub maybe_dropdown_name: Option, + + //pub creator_user_token: Option, + //pub creator_username: Option, + //pub creator_display_name: Option, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum ListTtsCategoriesError { + NotAuthorized, + ServerError, +} + +impl ResponseError for ListTtsCategoriesError { + fn status_code(&self) -> StatusCode { + match *self { + ListTtsCategoriesError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListTtsCategoriesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for ListTtsCategoriesError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn list_tts_categories_handler( + http_request: HttpRequest, + server_state: web::Data>) -> Result +{ + let query_builder = ListCategoriesQueryBuilder::new() + .show_deleted(false) + .show_unapproved(false) + .scope_model_type(Some("tts")); + + let query_result = + query_builder.perform_query(&server_state.mysql_pool).await; + + let results = match query_result { + Ok(results) => results, + Err(err) => { + warn!("DB error: {:?}", err); + return Err(ListTtsCategoriesError::ServerError); + } + }; + + warn!("Number of categories: {:?}", results.categories.len()); + + let mut categories = results.categories.iter() + .map(|c| { + DisplayCategory { + category_token: c.category_token.clone(), + model_type: c.model_type.clone(), + maybe_super_category_token: c.maybe_super_category_token.clone(), + can_directly_have_models: c.can_directly_have_models, + can_have_subcategories: c.can_have_subcategories, + can_only_mods_apply: c.can_only_mods_apply, + name: c.name.clone(), + maybe_dropdown_name:c.maybe_dropdown_name.clone(), + } + }) + .collect::>(); + + // TODO: Sort by dropdown name too. + categories.sort_by(|c1, c2| c1.name.cmp(&c2.name)); + + // TODO: Cache results in Redis w/ TTL. + let response = ListTtsCategoriesResponse { + success: true, + categories, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListTtsCategoriesError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs index 621958b942..48b65823da 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs @@ -1,5 +1,5 @@ pub mod create_category; pub mod delete_category; pub mod edit_category; -pub mod list_categories; +pub mod list_tts_categories; pub mod list_my_categories; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/app/storyteller_web/src/routes.rs index 742690606e..4a434604e5 100644 --- a/crates/app/storyteller_web/src/routes.rs +++ b/crates/app/storyteller_web/src/routes.rs @@ -62,6 +62,7 @@ use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_h use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; use crate::http_server::endpoints::categories::delete_category::delete_category_handler; use crate::http_server::endpoints::categories::edit_category::edit_category_handler; +use crate::http_server::endpoints::categories::list_tts_categories::list_tts_categories_handler; pub fn add_routes (app: App) -> App where @@ -426,6 +427,14 @@ fn add_category_routes (app: App) -> App .route(web::post().to(edit_category_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::scope("/list") + .service( + web::resource("/tts") + .route(web::get().to(list_tts_categories_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) ) } From 2f74df29ffbdf75db4e437dc216dbd66d6cda441 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 28 Dec 2021 22:53:13 -0500 Subject: [PATCH 0708/1042] shorter tokens, fix test --- crates/app/storyteller_web/src/database/helpers/tokens.rs | 2 +- debug/curl/category_tests.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/app/storyteller_web/src/database/helpers/tokens.rs b/crates/app/storyteller_web/src/database/helpers/tokens.rs index 9dd61b791b..8fc0c6bdbb 100644 --- a/crates/app/storyteller_web/src/database/helpers/tokens.rs +++ b/crates/app/storyteller_web/src/database/helpers/tokens.rs @@ -58,6 +58,6 @@ impl Tokens { } pub fn new_category() -> AnyhowResult { - random_prefix_crockford_token("CAT:", 32, false) + random_prefix_crockford_token("CAT:", 15, false) } } diff --git a/debug/curl/category_tests.py b/debug/curl/category_tests.py index 6600c8a310..d40762dc10 100755 --- a/debug/curl/category_tests.py +++ b/debug/curl/category_tests.py @@ -41,6 +41,7 @@ payload = { 'model_type': 'tts', # tts or w2l 'can_directly_have_models': False, 'can_have_subcategories': True, + 'is_mod_approved': True, } r = requests.post(category_create_url, cookies=cookies, json=payload) @@ -94,6 +95,7 @@ payload = { 'dropdown_name': 'Edited Dropdown Name', 'can_directly_have_models': True, 'can_have_subcategories': True, + 'can_only_mods_apply': False, 'is_mod_approved': True, 'maybe_mod_comments': 'mod comment', } From 10726dde2bdfc724bc5a2d300ab11de205981de1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Dec 2021 00:13:38 -0500 Subject: [PATCH 0709/1042] HTTP endpoint to assign / remove categories (untested) --- .../categories/get_category_by_token.rs | 154 ++++++++++ .../src/database/queries/categories/mod.rs | 1 + .../src/database/queries/mod.rs | 1 + .../categories/assign_tts_category.rs | 267 ++++++++++++++++++ .../http_server/endpoints/categories/mod.rs | 5 +- crates/app/storyteller_web/src/routes.rs | 17 +- .../2021-12-20-230000_model_categories/up.sql | 8 +- 7 files changed, 443 insertions(+), 10 deletions(-) create mode 100644 crates/app/storyteller_web/src/database/queries/categories/get_category_by_token.rs create mode 100644 crates/app/storyteller_web/src/database/queries/categories/mod.rs create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs diff --git a/crates/app/storyteller_web/src/database/queries/categories/get_category_by_token.rs b/crates/app/storyteller_web/src/database/queries/categories/get_category_by_token.rs new file mode 100644 index 0000000000..e0c0e19d9d --- /dev/null +++ b/crates/app/storyteller_web/src/database/queries/categories/get_category_by_token.rs @@ -0,0 +1,154 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; +use log::{info, warn, log}; +use sqlx::MySqlPool; + +#[derive(Serialize)] +pub struct ModelCategory { + // Non-included fields: + //pub id: i64, + //pub uuid_idempotency_token: String, + //pub version: i64, + + pub category_token: String, + pub model_type: String, // TODO: enum + pub maybe_super_category_token: Option, + + pub can_directly_have_models: bool, + pub can_have_subcategories: bool, + pub can_only_mods_apply: bool, + + pub name: String, + pub maybe_dropdown_name: Option, + + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + + pub creator_ip_address_creation: String, + pub creator_ip_address_last_update: String, + + pub is_mod_approved: Option, + + pub maybe_mod_comments: Option, + pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub deleted_at: Option>, +} + +pub async fn get_category_by_token( + category_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult> { + let maybe_category_record = sqlx::query_as!( + RawModelCategory, + r#" +SELECT + category.token as category_token, + category.model_type, + category.maybe_super_category_token, + category.can_directly_have_models, + category.can_have_subcategories, + category.can_only_mods_apply, + category.name, + category.maybe_dropdown_name, + category.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + category.creator_ip_address_creation, + category.creator_ip_address_last_update, + category.is_mod_approved, + category.maybe_mod_comments, + category.maybe_mod_user_token, + category.created_at, + category.updated_at, + category.deleted_at +FROM model_categories AS category +JOIN users +ON category.creator_user_token = users.token +WHERE + category.token = ? + "#, + category_token, + ) + .fetch_one(mysql_pool) + .await; + + let category : RawModelCategory = match maybe_category_record { + Ok(category) => category, + Err(err) => { + return match err { + sqlx::Error::RowNotFound => { + Ok(None) + }, + _ => { + warn!("User profile query error: {:?}", err); + Err(anyhow!("query error")) + } + } + } + }; + + let category = ModelCategory { + category_token: category.category_token.clone(), + model_type: category.model_type.clone(), + maybe_super_category_token: category.maybe_super_category_token.clone(), + can_directly_have_models: i8_to_bool(category.can_directly_have_models), + can_have_subcategories: i8_to_bool(category.can_have_subcategories), + can_only_mods_apply: i8_to_bool(category.can_only_mods_apply), + name: category.name.clone(), + maybe_dropdown_name: category.maybe_dropdown_name.clone(), + creator_user_token: category.creator_user_token.clone(), + creator_username: category.creator_username.clone(), + creator_display_name: category.creator_display_name.clone(), + creator_ip_address_creation: category.creator_ip_address_creation.clone(), + creator_ip_address_last_update: category.creator_ip_address_last_update.clone(), + is_mod_approved: nullable_i8_to_optional_bool(category.is_mod_approved), + maybe_mod_comments: category.maybe_mod_comments.clone(), + maybe_mod_user_token: category.maybe_mod_user_token.clone(), + created_at: category.created_at.clone(), + updated_at: category.updated_at.clone(), + deleted_at: category.deleted_at.clone(), + }; + + Ok(Some(category)) +} + +#[derive(Serialize)] +pub struct RawModelCategory { + // Non-included fields: + //pub id: i64, + //pub uuid_idempotency_token: String, + //pub version: i64, + + pub category_token: String, + pub model_type: String, // TODO: enum + pub maybe_super_category_token: Option, + + pub can_directly_have_models: i8, + pub can_have_subcategories: i8, + pub can_only_mods_apply: i8, + + pub name: String, + pub maybe_dropdown_name: Option, + + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + + pub creator_ip_address_creation: String, + pub creator_ip_address_last_update: String, + + pub is_mod_approved: Option, + + pub maybe_mod_comments: Option, + pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub deleted_at: Option>, +} diff --git a/crates/app/storyteller_web/src/database/queries/categories/mod.rs b/crates/app/storyteller_web/src/database/queries/categories/mod.rs new file mode 100644 index 0000000000..8012259411 --- /dev/null +++ b/crates/app/storyteller_web/src/database/queries/categories/mod.rs @@ -0,0 +1 @@ +pub mod get_category_by_token; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/database/queries/mod.rs b/crates/app/storyteller_web/src/database/queries/mod.rs index 602a7800f8..4045881345 100644 --- a/crates/app/storyteller_web/src/database/queries/mod.rs +++ b/crates/app/storyteller_web/src/database/queries/mod.rs @@ -1,6 +1,7 @@ pub mod badges; pub mod calculate_tts_model_leaderboard; pub mod calculate_w2l_template_leaderboard; +pub mod categories; pub mod create_session; pub mod delete_tts_model; pub mod delete_tts_result; diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs new file mode 100644 index 0000000000..368a2ebd40 --- /dev/null +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs @@ -0,0 +1,267 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use anyhow::anyhow; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::helpers::tokens::Tokens; +use crate::database::queries::categories::get_category_by_token::get_category_by_token; +use crate::database::queries::query_tts_model::select_tts_model_by_token; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Request =============== + +#[derive(Deserialize)] +pub struct AssignTtsCategoryRequest { + category_token: String, + tts_model_token: String, + + /// Whether to assign or delete the category. + assign: bool, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct AssignTtsCategoryResponse { + pub success: bool, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum AssignTtsCategoryError { + CategoryNotFound, + ModelNotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for AssignTtsCategoryError { + fn status_code(&self) -> StatusCode { + match *self { + AssignTtsCategoryError::CategoryNotFound => StatusCode::NOT_FOUND, + AssignTtsCategoryError::ModelNotFound => StatusCode::NOT_FOUND, + AssignTtsCategoryError::NotAuthorized => StatusCode::UNAUTHORIZED, + AssignTtsCategoryError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for AssignTtsCategoryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn assign_tts_category_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + AssignTtsCategoryError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(AssignTtsCategoryError::NotAuthorized); + } + }; + + let category_lookup_result + = get_category_by_token(&request.category_token, &server_state.mysql_pool).await; + + let category = match category_lookup_result { + Ok(Some(result)) => { + result + }, + Ok(None) => { + warn!("could not find category"); + return Err(AssignTtsCategoryError::CategoryNotFound); + }, + Err(err) => { + warn!("error looking up model: {:?}", err); + return Err(AssignTtsCategoryError::ServerError); + }, + }; + + // NB: First permission check. + // TODO: We don't have proper category permissions yet, so this is a proxy. + let is_category_mod= user_session.can_delete_other_users_tts_models; + + // If category is exclusively for mods + if !is_category_mod { + if category.can_only_mods_apply { + warn!("user is not allowed to assign this category: {}", user_session.user_token); + return Err(AssignTtsCategoryError::NotAuthorized); + } + + if category.deleted_at.is_some() || + !category.is_mod_approved.unwrap_or(false) { + warn!("user is not allowed to see this category: {}", user_session.user_token); + return Err(AssignTtsCategoryError::ModelNotFound); + } + } + + // NB: Second set of permission checks. + // Only mods should see deleted models (both user_* and mod_* deleted). + let is_mod_that_can_see_deleted = user_session.can_delete_other_users_tts_models; + + let model_lookup_result = select_tts_model_by_token( + &request.tts_model_token, + is_mod_that_can_see_deleted, + &server_state.mysql_pool).await; + + let model_record = match model_lookup_result { + Ok(Some(result)) => { + result + }, + Ok(None) => { + warn!("could not find model"); + return Err(AssignTtsCategoryError::ModelNotFound); + }, + Err(err) => { + warn!("error looking up model: {:?}", err); + return Err(AssignTtsCategoryError::ServerError); + }, + }; + + // NB: Third set of permission checks + let is_author = model_record.creator_user_token == user_session.user_token; + let is_mod = user_session.can_edit_other_users_tts_models ; + + if !is_author && !is_mod { + warn!("user is not allowed to add categories to model: {}", user_session.user_token); + return Err(AssignTtsCategoryError::NotAuthorized); + } + + if !is_mod { + if model_record.is_locked_from_user_modification || model_record.is_locked_from_use { + return Err(AssignTtsCategoryError::NotAuthorized); + } + } + + let ip_address = get_request_ip(&http_request); + + let query_builder = if request.assign { + // NB: deleted_at = NULL + sqlx::query!(r#" + INSERT INTO tts_category_assignments + SET + model_token = ?, + category_token = ?, + category_addition_user_token = ?, + ip_address_creation = ?, + ip_address_last_update = ?, + deleted_at = NULL + ON DUPLICATE KEY UPDATE + model_token = ?, + category_token = ?, + category_addition_user_token = ?, + ip_address_last_update = ?, + deleted_at = NULL + "#, + // Insert + &request.tts_model_token, + &request.category_token, + &user_session.user_token, + &ip_address, + &ip_address, + // Update + &request.tts_model_token, + &request.category_token, + &user_session.user_token, + &ip_address, + ) + + } else { + // NB: deleted_at = CURRENT_TIMESTAMP + sqlx::query!(r#" + INSERT INTO tts_category_assignments + SET + model_token = ?, + category_token = ?, + category_removal_user_token = ?, + ip_address_creation = ?, + ip_address_last_update = ?, + deleted_at = CURRENT_TIMESTAMP + ON DUPLICATE KEY UPDATE + model_token = ?, + category_token = ?, + category_removal_user_token = ?, + ip_address_last_update = ?, + deleted_at = CURRENT_TIMESTAMP + "#, + // Insert + &request.tts_model_token, + &request.category_token, + &user_session.user_token, + &ip_address, + &ip_address, + // Update + &request.tts_model_token, + &request.category_token, + &user_session.user_token, + &ip_address, + ) + }; + + // NB: We're soft deleting so we don't delete the associations. + let query_result = query_builder.execute(&server_state.mysql_pool).await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Assign category edit DB error: {:?}", err); + return Err(AssignTtsCategoryError::ServerError); + } + }; + + let response = AssignTtsCategoryResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| AssignTtsCategoryError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs index 48b65823da..261556a2f8 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs @@ -1,5 +1,6 @@ +pub mod assign_tts_category; pub mod create_category; pub mod delete_category; pub mod edit_category; -pub mod list_tts_categories; -pub mod list_my_categories; \ No newline at end of file +pub mod list_my_categories; +pub mod list_tts_categories; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/app/storyteller_web/src/routes.rs index 4a434604e5..145d618561 100644 --- a/crates/app/storyteller_web/src/routes.rs +++ b/crates/app/storyteller_web/src/routes.rs @@ -63,6 +63,7 @@ use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_t use crate::http_server::endpoints::categories::delete_category::delete_category_handler; use crate::http_server::endpoints::categories::edit_category::edit_category_handler; use crate::http_server::endpoints::categories::list_tts_categories::list_tts_categories_handler; +use crate::http_server::endpoints::categories::assign_tts_category::assign_tts_category_handler; pub fn add_routes (app: App) -> App where @@ -418,13 +419,13 @@ fn add_category_routes (app: App) -> App .route(web::head().to(|| HttpResponse::Ok())) ) .service( - web::resource("/{token}/delete") - .route(web::post().to(delete_category_handler)) + web::resource("/{token}/edit") + .route(web::post().to(edit_category_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) .service( - web::resource("/{token}/edit") - .route(web::post().to(edit_category_handler)) + web::resource("/{token}/delete") + .route(web::post().to(delete_category_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) .service( @@ -435,6 +436,14 @@ fn add_category_routes (app: App) -> App .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service( + web::scope("/assign") + .service( + web::resource("/tts") + .route(web::post().to(assign_tts_category_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) ) } diff --git a/migrations/2021-12-20-230000_model_categories/up.sql b/migrations/2021-12-20-230000_model_categories/up.sql index e7feeee433..8c8ab200ad 100644 --- a/migrations/2021-12-20-230000_model_categories/up.sql +++ b/migrations/2021-12-20-230000_model_categories/up.sql @@ -107,11 +107,11 @@ CREATE TABLE tts_category_assignments ( -- The person that created the assignment. -- This can be the model owner or a moderator. - category_addition_user_token VARCHAR(32) NOT NULL, + category_addition_user_token VARCHAR(32) DEFAULT NULL, -- The person that removed the assignment. -- This can be the model owner or a moderator. - category_removal_user_token VARCHAR(32) NOT NULL, + category_removal_user_token VARCHAR(32) DEFAULT NULL, -- For abuse tracking. -- THIS CAN BE A MODERATOR FOR SIMPLICITY. @@ -146,11 +146,11 @@ CREATE TABLE w2l_category_assignments ( -- The person that created the assignment. -- This can be the model owner or a moderator. - category_addition_user_token VARCHAR(32) NOT NULL, + category_addition_user_token VARCHAR(32) DEFAULT NULL, -- The person that removed the assignment. -- This can be the model owner or a moderator. - category_removal_user_token VARCHAR(32) NOT NULL, + category_removal_user_token VARCHAR(32) DEFAULT NULL, -- For abuse tracking. -- THIS CAN BE A MODERATOR FOR SIMPLICITY. From e78e1079e32464fd4c4627dceb795087a39a2952 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Dec 2021 01:12:03 -0500 Subject: [PATCH 0710/1042] test category assignments --- debug/curl/category_tests.py | 52 ++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/debug/curl/category_tests.py b/debug/curl/category_tests.py index d40762dc10..ad315ce1a6 100755 --- a/debug/curl/category_tests.py +++ b/debug/curl/category_tests.py @@ -9,6 +9,8 @@ login_url = 'http://localhost:12345/login' category_create_url = 'http://localhost:12345/category/create' category_delete_url_format = 'http://localhost:12345/category/{}/delete' category_edit_url_format = 'http://localhost:12345/category/{}/edit' +tts_model_list_url = 'http://localhost:12345/tts/list' +tts_category_assignment_url = 'http://localhost:12345/category/assign/tts' payload = { 'username_or_email': 'echelon', @@ -17,7 +19,7 @@ payload = { r = requests.post(login_url, json=payload) -print("===== Login =====") +print("===== [1] Login =====") print('Status: {}'.format(r.status_code)) print(r.content) for k, v in r.headers.items(): @@ -31,7 +33,7 @@ m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) session_cookie = m.group(1) -print("===== Create Category =====") +print("===== [2] Create Category =====") cookies = { 'session': session_cookie } @@ -54,7 +56,7 @@ category_token = d['token'] print(f'created category token: #{category_token}') -print("===== (Soft) Delete Category =====") +print("===== [3] (Soft) Delete Category =====") category_delete_url = category_delete_url_format.format(category_token) @@ -70,7 +72,7 @@ print('Status: {}'.format(r.status_code)) print(r.content) -print("===== Undelete Category =====") +print("===== [4] Undelete Category =====") cookies = { 'session': session_cookie } @@ -84,7 +86,7 @@ print('Status: {}'.format(r.status_code)) print(r.content) -print("===== Edit Category =====") +print("===== [5] Edit Category =====") category_edit_url = category_edit_url_format.format(category_token) @@ -106,3 +108,43 @@ print('Status: {}'.format(r.status_code)) print(r.content) +print("===== [6] List TTS Models =====") + +r = requests.get(tts_model_list_url, cookies=cookies) + +print('Status: {}'.format(r.status_code)) + +d = json.loads(r.content) + +tts_model_token = d['models'][0]['model_token'] + +print('TTS model token: {}'.format(tts_model_token)) + + +print("===== [7] Create Category Assignment to TTS Model =====") + +payload = { + 'category_token': category_token, + 'tts_model_token': tts_model_token, + 'assign': True, +} + +r = requests.post(tts_category_assignment_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + + +print("===== [8] Delete Category Assignment to TTS Model =====") + +payload = { + 'category_token': category_token, + 'tts_model_token': tts_model_token, + 'assign': False, +} + +r = requests.post(tts_category_assignment_url, cookies=cookies, json=payload) + +print('Status: {}'.format(r.status_code)) +print(r.content) + From 654f826598f24d9bf92c7031767f03ec2b4bd3f4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Dec 2021 02:50:50 -0500 Subject: [PATCH 0711/1042] add tts categories to model list results --- .../list_tts_model_category_assignments.rs | 80 +++++++++++++++++++ .../src/database/queries/categories/mod.rs | 3 +- .../categories/list_tts_categories.rs | 2 +- .../endpoints/tts/list_tts_models.rs | 27 ++++++- 4 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 crates/app/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs diff --git a/crates/app/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs b/crates/app/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs new file mode 100644 index 0000000000..e1c4ee0077 --- /dev/null +++ b/crates/app/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs @@ -0,0 +1,80 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::util::anyhow_result::AnyhowResult; +use log::warn; +use sqlx::MySqlPool; +use std::collections::{HashSet, HashMap}; + + +/// Map of model_token => vec +#[derive(Serialize, Default)] +pub struct TtsModelCategoryMap { + pub model_to_category_tokens: HashMap>, +} + +pub async fn fetch_tts_model_category_map(mysql_pool: &MySqlPool) + -> AnyhowResult +{ + let assignments = list_tts_model_category_assignments(mysql_pool).await?; + + let mut map : HashMap> = HashMap::new(); + + for assignment in assignments.iter() { + if !map.contains_key(&assignment.tts_model_token) { + map.insert(assignment.tts_model_token.clone(), HashSet::new()); + } + + map.get_mut(&assignment.tts_model_token).map(|hashset| { + hashset.insert(assignment.category_token.clone()) + }); + } + + Ok(TtsModelCategoryMap { + model_to_category_tokens: map, + }) +} + +#[derive(Serialize)] +pub struct CategoryAssignment { + pub tts_model_token: String, + pub category_token: String, +} + +// NB: This shows for all models, including deleted and unapproved. +// However, it does not return deleted/unapproved categories. +pub async fn list_tts_model_category_assignments(mysql_pool: &MySqlPool) -> AnyhowResult> { + let maybe_results = sqlx::query_as!( + CategoryAssignment, + r#" +SELECT + assignments.model_token AS tts_model_token, + assignments.category_token +FROM + tts_category_assignments AS assignments +JOIN + tts_models AS tts + ON assignments.model_token = tts.token +WHERE + tts.is_locked_from_use IS FALSE + AND tts.user_deleted_at IS NULL + AND tts.mod_deleted_at IS NULL + "#, + ) + .fetch_all(mysql_pool) + .await; + + match maybe_results { + Err(err) => { + match err { + sqlx::Error::RowNotFound => { + Ok(Vec::new()) + }, + _ => { + warn!("list tts model category assignments db error: {:?}", err); + Err(anyhow!("error with query: {:?}", err)) + } + } + }, + Ok(results) => Ok(results), + } +} diff --git a/crates/app/storyteller_web/src/database/queries/categories/mod.rs b/crates/app/storyteller_web/src/database/queries/categories/mod.rs index 8012259411..7692b77bed 100644 --- a/crates/app/storyteller_web/src/database/queries/categories/mod.rs +++ b/crates/app/storyteller_web/src/database/queries/categories/mod.rs @@ -1 +1,2 @@ -pub mod get_category_by_token; \ No newline at end of file +pub mod get_category_by_token; +pub mod list_tts_model_category_assignments; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs index 832fc12d9e..f31a2b819d 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs @@ -79,7 +79,7 @@ impl ResponseError for ListTtsCategoriesError { } } -// NB: Not using DeriveMore since Clion doesn't understand it. +// NB: Not using derive_more::Display since Clion doesn't understand it. impl fmt::Display for ListTtsCategoriesError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs index eb1066b464..a6f3ce2915 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs @@ -1,23 +1,24 @@ -use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; +use crate::database::queries::categories::list_tts_model_category_assignments::fetch_tts_model_category_map; use crate::database::queries::list_tts_models::list_tts_models; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::anyhow_result::AnyhowResult; -use derive_more::{Display, Error}; use log::{info, warn, log, error}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::collections::HashSet; +use std::fmt; use std::sync::Arc; #[derive(Serialize, Clone)] @@ -29,6 +30,11 @@ pub struct TtsModelRecordForResponse { pub creator_display_name: String, pub creator_gravatar_hash: String, pub title: String, + + /// Category assignments + /// From non-deleted, mod-approved categories only + pub category_tokens: HashSet, + pub created_at: DateTime, pub updated_at: DateTime, } @@ -39,7 +45,7 @@ pub struct ListTtsModelsSuccessResponse { pub models: Vec, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum ListTtsModelsError { ServerError, } @@ -73,6 +79,13 @@ impl ResponseError for ListTtsModelsError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListTtsModelsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn list_tts_models_handler( http_request: HttpRequest, server_state: web::Data>) -> Result @@ -126,6 +139,9 @@ async fn get_all_models(mysql_pool: &MySqlPool) -> AnyhowResult AnyhowResult Date: Thu, 30 Dec 2021 01:39:36 -0500 Subject: [PATCH 0712/1042] list assigned category query builder --- ...t_assigned_tts_categories_query_builder.rs | 255 ++++++++++++++++++ .../src/database/query_builders/mod.rs | 3 +- .../endpoints/categories/edit_category.rs | 2 +- .../endpoints/tts/get_tts_model.rs | 23 +- 4 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs diff --git a/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs b/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs new file mode 100644 index 0000000000..a58fe8e79f --- /dev/null +++ b/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs @@ -0,0 +1,255 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; +use crate::util::anyhow_result::AnyhowResult; +use log::{warn, info}; +use sqlx::MySqlPool; + +/// List of assigned categories for a TTS model +/// This is *NOT* paginated, as that defeats the purpose of having a category system. +#[derive(Serialize)] +pub struct AssignedCategoryList { + pub categories: Vec, +} + +#[derive(Serialize)] +pub struct AssignedCategory { + pub category_token: String, + + pub model_type: String, // TODO: ENUM + + pub maybe_super_category_token: Option, + + pub can_directly_have_models: bool, + pub can_have_subcategories: bool, + pub can_only_mods_apply: bool, + + pub name: String, + pub maybe_dropdown_name: Option, + + pub category_creator_user_token: Option, + pub category_creator_username: Option, + pub category_creator_display_name: Option, + pub category_creator_gravatar_hash: Option, + + // Moderator fields + pub is_mod_approved: Option, + pub maybe_mod_comments: Option, + //pub maybe_mod_user_token: Option, + //pub maybe_mod_username: Option, + //pub maybe_mod_display_name: Option, + + pub category_created_at: DateTime, + pub category_updated_at: DateTime, + pub category_deleted_at: Option>, + + pub category_assignment_created_at: DateTime, + pub category_assignment_updated_at: DateTime, + pub category_assignment_deleted_at: Option>, +} + +/// Query Builder for listing categories +/// These are very difficult queries, so the builder helps for +/// testability, construction, and correctness. +pub struct ListAssignedTtsCategoriesQueryBuilder { + // REQUIRED. + scope_tts_model_token: String, + + // The public lists won't include unapproved, but mod views + // will want to see them. + show_unapproved: bool, + + // The public lists won't include deleted models, but mod views + // will want to see them. + show_deleted: bool, +} + +impl ListAssignedTtsCategoriesQueryBuilder { + pub fn for_model_token(model_token: &str) -> Self { + Self { + scope_tts_model_token: model_token.to_string(), + show_unapproved: false, + show_deleted: false, + } + } + + pub fn show_unapproved(mut self, show_unapproved: bool) -> Self { + self.show_unapproved = show_unapproved; + self + } + + pub fn show_deleted(mut self, show_deleted: bool) -> Self { + self.show_deleted = show_deleted; + self + } + + /// Perform the query based on the set predicates. + pub async fn perform_query( + &self, + mysql_pool: &MySqlPool + ) -> AnyhowResult { + + let internal_results = self.perform_internal_query(mysql_pool).await?; + + let categories = internal_results + .iter() + .map(|c| { + AssignedCategory { + category_token: c.category_token.clone(), + model_type: c.model_type.clone(), + maybe_super_category_token: c.maybe_super_category_token.clone(), + can_directly_have_models: i8_to_bool(c.can_directly_have_models), + can_have_subcategories: i8_to_bool(c.can_have_subcategories), + can_only_mods_apply: i8_to_bool(c.can_only_mods_apply), + name: c.name.clone(), + maybe_dropdown_name: c.maybe_dropdown_name.clone(), + category_creator_user_token: c.category_creator_user_token.clone(), + category_creator_username: c.category_creator_username.clone(), + category_creator_display_name: c.category_creator_display_name.clone(), + category_creator_gravatar_hash: c.category_creator_gravatar_hash.clone(), + is_mod_approved: nullable_i8_to_optional_bool(c.is_mod_approved), + maybe_mod_comments: c.maybe_mod_comments.clone(), + category_created_at: c.category_created_at, + category_updated_at: c.category_updated_at, + category_deleted_at: c.category_deleted_at.clone(), + category_assignment_created_at: c.category_assignment_created_at, + category_assignment_updated_at: c.category_assignment_updated_at, + category_assignment_deleted_at: c.category_assignment_deleted_at.clone(), + } + }) + .collect::>(); + + Ok(AssignedCategoryList { + categories, + }) + } + + /// Perform the query based on the set predicates. + async fn perform_internal_query( + &self, + mysql_pool: &MySqlPool + ) -> AnyhowResult> { + + let query = self.build_query_string(); + let mut query = sqlx::query_as::<_, RawInternalCategoryRecord>(&query); + + // NB: The following bindings must match the order of the query builder !! + + // NB: Model token is always used for scoping. + query = query.bind(&self.scope_tts_model_token); + + let mut results = query.fetch_all(mysql_pool) + .await?; + + Ok(results) + } + + fn build_query_string(&self) -> String { + // TODO: I haven't figured out how to get field name disambiguation and type coercion working here. + // (1) tts_results.creator_set_visibility `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, + // Query error: no column found for name: creator_set_visibility + // (2) creator_set_visibility `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, + // Column 'creator_set_visibility' in field list is ambiguous + + // TODO/NB: Unfortunately SQLx can't statically typecheck this query + let mut query = r#" +SELECT + model_categories.token as category_token, + + model_categories.model_type, + + model_categories.maybe_super_category_token, + + model_categories.can_directly_have_models, + model_categories.can_have_subcategories, + model_categories.can_only_mods_apply, + + model_categories.name, + model_categories.maybe_dropdown_name, + + users.token as category_creator_user_token, + users.username as category_creator_username, + users.display_name as category_creator_display_name, + users.email_gravatar_hash as category_creator_gravatar_hash, + + model_categories.is_mod_approved, + model_categories.maybe_mod_comments, + + model_categories.created_at as category_created_at, + model_categories.updated_at as category_updated_at, + model_categories.deleted_at as category_deleted_at, + + tts_category_assignments.created_at as category_assignment_created_at, + tts_category_assignments.updated_at as category_assignment_updated_at, + tts_category_assignments.deleted_at as category_assignment_deleted_at + +FROM model_categories +JOIN tts_category_assignments + ON tts_category_assignments.category_token = model_categories.token +LEFT OUTER JOIN users + ON model_categories.creator_user_token = users.token + "#.to_string(); + + query.push_str(&self.build_predicates()); + query + } + + pub fn build_predicates(&self) -> String { + let mut query = "".to_string(); + + // Scoping to tts model is required. + query.push_str(" WHERE model_categories.model_token = ?"); + query.push_str(" AND model_categories.model_type = 'tts'"); + + // We also don't care to ever show soft-deleted assignments + query.push_str(" AND tts_category_assignments.deleted_at IS NULL"); + + if !self.show_unapproved { + query.push_str(" AND model_categories.is_mod_approved IS TRUE"); + } + + if !self.show_deleted { + query.push_str(" AND model_categories.deleted_at IS NULL"); + } + + query + } +} + +#[derive(sqlx::FromRow)] +pub struct RawInternalCategoryRecord { + pub category_token: String, + + pub model_type: String, // TODO: ENUM + + pub maybe_super_category_token: Option, + + pub can_directly_have_models: i8, + pub can_have_subcategories: i8, + pub can_only_mods_apply: i8, + + pub name: String, + pub maybe_dropdown_name: Option, + + pub category_creator_user_token: Option, + pub category_creator_username: Option, + pub category_creator_display_name: Option, + pub category_creator_gravatar_hash: Option, + + // Moderator fields + pub is_mod_approved: Option, + pub maybe_mod_comments: Option, + + pub category_created_at: DateTime, + pub category_updated_at: DateTime, + pub category_deleted_at: Option>, + + pub category_assignment_created_at: DateTime, + pub category_assignment_updated_at: DateTime, + pub category_assignment_deleted_at: Option>, +} + +#[cfg(test)] +mod tests { +} diff --git a/crates/app/storyteller_web/src/database/query_builders/mod.rs b/crates/app/storyteller_web/src/database/query_builders/mod.rs index 072361672c..9cd5018348 100644 --- a/crates/app/storyteller_web/src/database/query_builders/mod.rs +++ b/crates/app/storyteller_web/src/database/query_builders/mod.rs @@ -1,4 +1,5 @@ +pub mod list_assigned_tts_categories_query_builder; pub mod list_categories_query_builder; pub mod list_tts_inference_results_query_builder; pub mod list_users_query_builder; -pub mod list_w2l_inference_results_query_builder; +pub mod list_w2l_inference_results_query_builder; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs index 91dfa56a8e..cb8bad069d 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs @@ -89,7 +89,7 @@ impl ResponseError for EditCategoryError { } } -// NB: Not using DeriveMore since Clion doesn't understand it. +// NB: Not using derive_more::Display since Clion doesn't understand it. impl fmt::Display for EditCategoryError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs index baf749499a..d062308624 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs @@ -1,39 +1,45 @@ -use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use anyhow::Error; use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::database::queries::query_tts_model::TtsModelRecordForResponse; use crate::database::queries::query_tts_model::select_tts_model_by_token; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; +// =============== Request =============== + /// For the URL PathInfo #[derive(Deserialize)] pub struct GetTtsModelPathInfo { token: String, } +// =============== Success Response =============== + #[derive(Serialize)] pub struct GetTtsModelSuccessResponse { pub success: bool, pub model: TtsModelRecordForResponse, } -#[derive(Debug, Display)] +// =============== Error Response =============== + +#[derive(Debug)] pub enum GetTtsModelError { ServerError, NotFound, @@ -57,6 +63,15 @@ impl ResponseError for GetTtsModelError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GetTtsModelError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + pub async fn get_tts_model_handler( http_request: HttpRequest, path: Path, From 8138766583b125af5d7c49dc81c237f64b20dab0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Dec 2021 02:22:26 -0500 Subject: [PATCH 0713/1042] Endpoint to list categories for model (not working?) --- ...t_assigned_tts_categories_query_builder.rs | 2 +- .../list_tts_model_assigned_categories.rs | 172 ++++++++++++++++++ .../http_server/endpoints/categories/mod.rs | 3 +- crates/app/storyteller_web/src/routes.rs | 9 + 4 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs diff --git a/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs b/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs index a58fe8e79f..fdefd6504b 100644 --- a/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs +++ b/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs @@ -199,7 +199,7 @@ LEFT OUTER JOIN users let mut query = "".to_string(); // Scoping to tts model is required. - query.push_str(" WHERE model_categories.model_token = ?"); + query.push_str(" WHERE tts_category_assignments.model_token = ?"); query.push_str(" AND model_categories.model_type = 'tts'"); // We also don't care to ever show soft-deleted assignments diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs new file mode 100644 index 0000000000..647442e030 --- /dev/null +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs @@ -0,0 +1,172 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::database::query_builders::list_assigned_tts_categories_query_builder::ListAssignedTtsCategoriesQueryBuilder; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Request =============== + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct ListTtsModelAssignedCategoriesPathInfo { + /// TTS model token + token: String, +} +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct ListTtsModelAssignedCategoriesResponse { + pub success: bool, + pub categories: Vec, +} + +/// Public-facing and optimized (non-irrelevant fields) category list +/// Used for the main TTS dropdown as well as the TTS edit/tagging UI +#[derive(Serialize)] +pub struct DisplayCategory { + pub category_token: String, + + pub model_type: String, // TODO: ENUM + + pub maybe_super_category_token: Option, + + pub can_directly_have_models: bool, + pub can_have_subcategories: bool, + pub can_only_mods_apply: bool, + + pub name: String, + pub maybe_dropdown_name: Option, + + // NB: It's okay for non-mods to see this. + pub is_mod_approved: Option, + + pub category_created_at: DateTime, + pub category_updated_at: DateTime, + pub category_deleted_at: Option>, + + //pub creator_user_token: Option, + //pub creator_username: Option, + //pub creator_display_name: Option, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum ListTtsModelAssignedCategoriesError { + ServerError, +} + +impl ResponseError for ListTtsModelAssignedCategoriesError { + fn status_code(&self) -> StatusCode { + match *self { + ListTtsModelAssignedCategoriesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListTtsModelAssignedCategoriesError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn list_tts_model_assigned_categories_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListTtsModelAssignedCategoriesError::ServerError + })?; + + // TODO: We don't have any permissions for categories. This is a proxy. + let is_mod = maybe_user_session + .map(|session| session.can_ban_users) + .unwrap_or(false); + + + let query_builder = ListAssignedTtsCategoriesQueryBuilder::for_model_token(&path.token) + .show_deleted(is_mod) + .show_unapproved(is_mod); + + let query_result = + query_builder.perform_query(&server_state.mysql_pool).await; + + let results = match query_result { + Ok(results) => results, + Err(err) => { + warn!("DB error: {:?}", err); + return Err(ListTtsModelAssignedCategoriesError::ServerError); + } + }; + + let mut categories = results.categories.iter() + .map(|c| { + DisplayCategory { + category_token: c.category_token.clone(), + model_type: c.model_type.clone(), + maybe_super_category_token: c.maybe_super_category_token.clone(), + can_directly_have_models: c.can_directly_have_models, + can_have_subcategories: c.can_have_subcategories, + can_only_mods_apply: c.can_only_mods_apply, + name: c.name.clone(), + maybe_dropdown_name:c.maybe_dropdown_name.clone(), + is_mod_approved: c.is_mod_approved.clone(), + category_created_at: c.category_created_at.clone(), + category_updated_at: c.category_updated_at.clone(), + category_deleted_at: c.category_deleted_at.clone(), + } + }) + .collect::>(); + + // TODO: Sort by dropdown name too. + categories.sort_by(|c1, c2| c1.name.cmp(&c2.name)); + + let response = ListTtsModelAssignedCategoriesResponse { + success: true, + categories, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListTtsModelAssignedCategoriesError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs index 261556a2f8..521f4cf0e0 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs @@ -3,4 +3,5 @@ pub mod create_category; pub mod delete_category; pub mod edit_category; pub mod list_my_categories; -pub mod list_tts_categories; \ No newline at end of file +pub mod list_tts_categories; +pub mod list_tts_model_assigned_categories; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/app/storyteller_web/src/routes.rs index 145d618561..1fb83b5267 100644 --- a/crates/app/storyteller_web/src/routes.rs +++ b/crates/app/storyteller_web/src/routes.rs @@ -64,6 +64,7 @@ use crate::http_server::endpoints::categories::delete_category::delete_category_ use crate::http_server::endpoints::categories::edit_category::edit_category_handler; use crate::http_server::endpoints::categories::list_tts_categories::list_tts_categories_handler; use crate::http_server::endpoints::categories::assign_tts_category::assign_tts_category_handler; +use crate::http_server::endpoints::categories::list_tts_model_assigned_categories::list_tts_model_assigned_categories_handler; pub fn add_routes (app: App) -> App where @@ -444,6 +445,14 @@ fn add_category_routes (app: App) -> App .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service( + web::scope("/assignments") + .service( + web::resource("/tts/{token}") + .route(web::get().to(list_tts_model_assigned_categories_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) ) } From 4777ca25fa939362c6bdfa46c7197f4606cb3e94 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Dec 2021 02:49:42 -0500 Subject: [PATCH 0714/1042] Move edit category to moderation, add list handler --- .../categories/list_my_categories.rs | 0 .../http_server/endpoints/categories/mod.rs | 2 - .../categories/edit_category.rs | 0 .../list_tts_categories_for_moderation.rs | 180 ++++++++++++++++++ .../endpoints/moderation/categories/mod.rs | 2 + .../http_server/endpoints/moderation/mod.rs | 1 + crates/app/storyteller_web/src/routes.rs | 31 +-- 7 files changed, 203 insertions(+), 13 deletions(-) delete mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/list_my_categories.rs rename crates/app/storyteller_web/src/http_server/endpoints/{ => moderation}/categories/edit_category.rs (100%) create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_my_categories.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_my_categories.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs index 521f4cf0e0..6005bf61fb 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs @@ -1,7 +1,5 @@ pub mod assign_tts_category; pub mod create_category; pub mod delete_category; -pub mod edit_category; -pub mod list_my_categories; pub mod list_tts_categories; pub mod list_tts_model_assigned_categories; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/categories/edit_category.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs new file mode 100644 index 0000000000..63f4ca8cdd --- /dev/null +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs @@ -0,0 +1,180 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct ListTtsCategoriesForModerationResponse { + pub success: bool, + pub categories: Vec, +} + +#[derive(Serialize)] +pub struct CategoryForModeration { + pub category_token: String, + + pub model_type: String, // TODO: ENUM + + pub maybe_super_category_token: Option, + + pub can_directly_have_models: bool, + pub can_have_subcategories: bool, + pub can_only_mods_apply: bool, + + pub name: String, + pub maybe_dropdown_name: Option, + + pub creator_user_token: Option, + pub creator_username: Option, + pub creator_display_name: Option, + + // Moderator fields + pub is_mod_approved: Option, + pub maybe_mod_comments: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub deleted_at: Option>, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum ListTtsCategoriesForModerationError { + NotAuthorized, + ServerError, +} + +impl ResponseError for ListTtsCategoriesForModerationError { + fn status_code(&self) -> StatusCode { + match *self { + ListTtsCategoriesForModerationError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListTtsCategoriesForModerationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListTtsCategoriesForModerationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn list_tts_categories_for_moderation_handler( + http_request: HttpRequest, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListTtsCategoriesForModerationError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(ListTtsCategoriesForModerationError::NotAuthorized); + } + }; + + // TODO: We don't have a permission for categories, so we use this as a proxy. + if !user_session.can_ban_users { + warn!("no permission to edit categories"); + return Err(ListTtsCategoriesForModerationError::NotAuthorized); + } + + let query_builder = ListCategoriesQueryBuilder::new() + .show_deleted(true) + .show_unapproved(true) + .scope_model_type(Some("tts")); + + let query_result = + query_builder.perform_query(&server_state.mysql_pool).await; + + let results = match query_result { + Ok(results) => results, + Err(err) => { + warn!("DB error: {:?}", err); + return Err(ListTtsCategoriesForModerationError::ServerError); + } + }; + + warn!("Number of categories: {:?}", results.categories.len()); + + let mut categories = results.categories.iter() + .map(|c| { + CategoryForModeration { + category_token: c.category_token.clone(), + model_type: c.model_type.clone(), + maybe_super_category_token: c.maybe_super_category_token.clone(), + can_directly_have_models: c.can_directly_have_models, + can_have_subcategories: c.can_have_subcategories, + can_only_mods_apply: c.can_only_mods_apply, + name: c.name.clone(), + maybe_dropdown_name:c.maybe_dropdown_name.clone(), + creator_user_token: c.creator_user_token.clone(), + creator_username: c.creator_username.clone(), + creator_display_name: c.creator_display_name.clone(), + is_mod_approved: c.is_mod_approved.clone(), + maybe_mod_comments: c.maybe_mod_comments.clone(), + created_at: c.created_at.clone(), + updated_at: c.updated_at.clone(), + deleted_at: c.deleted_at.clone() + } + }) + .collect::>(); + + // TODO: Sort by dropdown name too. + categories.sort_by(|c1, c2| c1.name.cmp(&c2.name)); + + let response = ListTtsCategoriesForModerationResponse { + success: true, + categories, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListTtsCategoriesForModerationError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs new file mode 100644 index 0000000000..cf77590623 --- /dev/null +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs @@ -0,0 +1,2 @@ +pub mod edit_category; +pub mod list_tts_categories_for_moderation; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/mod.rs index afd7a69415..95f5febebb 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/mod.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/mod.rs @@ -1,4 +1,5 @@ pub mod approval; +pub mod categories; pub mod ip_bans; pub mod jobs; pub mod stats; diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/app/storyteller_web/src/routes.rs index 1fb83b5267..6a24de5fdd 100644 --- a/crates/app/storyteller_web/src/routes.rs +++ b/crates/app/storyteller_web/src/routes.rs @@ -1,9 +1,13 @@ -use actix_web::{App, web, HttpResponse}; use actix_http::body::MessageBody; use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; use actix_web::error::Error; +use actix_web::{App, web, HttpResponse}; +use crate::http_server::endpoints::categories::assign_tts_category::assign_tts_category_handler; use crate::http_server::endpoints::categories::create_category::create_category_handler; +use crate::http_server::endpoints::categories::delete_category::delete_category_handler; +use crate::http_server::endpoints::categories::list_tts_categories::list_tts_categories_handler; +use crate::http_server::endpoints::categories::list_tts_model_assigned_categories::list_tts_model_assigned_categories_handler; use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::misc::default_route_404::default_route_404; @@ -11,6 +15,7 @@ use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha use crate::http_server::endpoints::misc::enable_alpha_handler::enable_alpha_handler; use crate::http_server::endpoints::misc::root_index::get_root_index; use crate::http_server::endpoints::moderation::approval::pending_w2l_templates::get_pending_w2l_templates_handler; +use crate::http_server::endpoints::moderation::categories::edit_category::edit_category_handler; use crate::http_server::endpoints::moderation::ip_bans::add_ip_ban::add_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::delete_ip_ban::delete_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_handler; @@ -60,11 +65,7 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; -use crate::http_server::endpoints::categories::delete_category::delete_category_handler; -use crate::http_server::endpoints::categories::edit_category::edit_category_handler; -use crate::http_server::endpoints::categories::list_tts_categories::list_tts_categories_handler; -use crate::http_server::endpoints::categories::assign_tts_category::assign_tts_category_handler; -use crate::http_server::endpoints::categories::list_tts_model_assigned_categories::list_tts_model_assigned_categories_handler; +use crate::http_server::endpoints::moderation::categories::list_tts_categories_for_moderation::list_tts_categories_for_moderation_handler; pub fn add_routes (app: App) -> App where @@ -235,6 +236,19 @@ fn add_moderator_routes (app: App) -> App .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service( + web::scope("/categories") + .service( + web::resource("/{token}/edit") + .route(web::post().to(edit_category_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/tts/list") + .route(web::get().to(list_tts_categories_for_moderation_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) ) } @@ -419,11 +433,6 @@ fn add_category_routes (app: App) -> App .route(web::post().to(create_category_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - .service( - web::resource("/{token}/edit") - .route(web::post().to(edit_category_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) .service( web::resource("/{token}/delete") .route(web::post().to(delete_category_handler)) From db7f5d20fee85b0355cee25e3e07d50736a2de25 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 30 Dec 2021 19:37:50 -0500 Subject: [PATCH 0715/1042] additional create category bool --- .../endpoints/categories/create_category.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs index 1abeaa7a35..b32c36ee8c 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs @@ -30,13 +30,16 @@ use std::sync::Arc; const DEFAULT_CAN_DIRECTLY_HAVE_MODELS : bool = true; const DEFAULT_CAN_HAVE_SUBCATEGORIES : bool = false; +const DEFAULT_CAN_ONLY_MODS_APPLY : bool = false; // =============== Request =============== #[derive(Deserialize)] #[serde(rename_all = "snake_case")] pub enum ModelType { + /// 'tts' in database Tts, + /// 'w2l' in database W2l, } @@ -50,6 +53,7 @@ pub struct CreateCategoryRequest { // Fields for moderators only pub can_directly_have_models: Option, pub can_have_subcategories: Option, + pub can_only_mods_apply: Option, } // =============== Success Response =============== @@ -146,6 +150,7 @@ pub async fn create_category_handler( let mut maybe_mod_user_token = None; let mut can_directly_have_models = DEFAULT_CAN_DIRECTLY_HAVE_MODELS; let mut can_have_subcategories = DEFAULT_CAN_HAVE_SUBCATEGORIES; + let mut can_only_mods_apply = DEFAULT_CAN_ONLY_MODS_APPLY; if is_mod { // Moderator fields and adjustments @@ -153,8 +158,10 @@ pub async fn create_category_handler( maybe_mod_user_token = Some(user_session.user_token.clone()); can_directly_have_models = request.can_directly_have_models .unwrap_or(DEFAULT_CAN_DIRECTLY_HAVE_MODELS); - can_have_subcategories= request.can_have_subcategories + can_have_subcategories = request.can_have_subcategories .unwrap_or(DEFAULT_CAN_HAVE_SUBCATEGORIES); + can_only_mods_apply = request.can_only_mods_apply + .unwrap_or(DEFAULT_CAN_ONLY_MODS_APPLY); } let query_result = sqlx::query!( @@ -173,7 +180,8 @@ SET is_mod_approved = ?, maybe_mod_user_token = ?, can_directly_have_models = ?, - can_have_subcategories = ? + can_have_subcategories = ?, + can_only_mods_apply = ? "#, category_token, @@ -186,7 +194,8 @@ SET is_mod_approved, maybe_mod_user_token, can_directly_have_models, - can_have_subcategories + can_have_subcategories, + can_only_mods_apply ) .execute(&server_state.mysql_pool) .await; From 9f508aedbe756fae9968aec614e96aa01684567e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 31 Dec 2021 02:05:33 -0500 Subject: [PATCH 0716/1042] Category assignment flag check --- .../endpoints/categories/assign_tts_category.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs index 368a2ebd40..b12df1575c 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs @@ -54,6 +54,7 @@ pub struct AssignTtsCategoryResponse { #[derive(Debug, Serialize)] pub enum AssignTtsCategoryError { CategoryNotFound, + CategoryNotApplicable, ModelNotFound, NotAuthorized, ServerError, @@ -63,6 +64,7 @@ impl ResponseError for AssignTtsCategoryError { fn status_code(&self) -> StatusCode { match *self { AssignTtsCategoryError::CategoryNotFound => StatusCode::NOT_FOUND, + AssignTtsCategoryError::CategoryNotApplicable => StatusCode::BAD_REQUEST, AssignTtsCategoryError::ModelNotFound => StatusCode::NOT_FOUND, AssignTtsCategoryError::NotAuthorized => StatusCode::UNAUTHORIZED, AssignTtsCategoryError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, @@ -140,6 +142,11 @@ pub async fn assign_tts_category_handler( } } + if !category.can_directly_have_models { + warn!("category cannot have models: {}", category.category_token); + return Err(AssignTtsCategoryError::CategoryNotApplicable); + } + // NB: Second set of permission checks. // Only mods should see deleted models (both user_* and mod_* deleted). let is_mod_that_can_see_deleted = user_session.can_delete_other_users_tts_models; From c15f694a1d8f53c6ab39b113864027c51610f196 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 31 Dec 2021 02:48:42 -0500 Subject: [PATCH 0717/1042] forgot a GET endpoint for single categories --- .../categories/get_category_by_token.rs | 4 + .../list_categories_query_builder.rs | 4 + .../categories/assign_tts_category.rs | 2 +- .../endpoints/categories/get_category.rs | 180 ++++++++++++++++++ .../list_tts_model_assigned_categories.rs | 2 +- .../http_server/endpoints/categories/mod.rs | 1 + .../list_tts_categories_for_moderation.rs | 2 + crates/app/storyteller_web/src/routes.rs | 6 + 8 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 crates/app/storyteller_web/src/http_server/endpoints/categories/get_category.rs diff --git a/crates/app/storyteller_web/src/database/queries/categories/get_category_by_token.rs b/crates/app/storyteller_web/src/database/queries/categories/get_category_by_token.rs index e0c0e19d9d..80c8a2bf3a 100644 --- a/crates/app/storyteller_web/src/database/queries/categories/get_category_by_token.rs +++ b/crates/app/storyteller_web/src/database/queries/categories/get_category_by_token.rs @@ -26,6 +26,7 @@ pub struct ModelCategory { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, + pub creator_gravatar_hash: String, pub creator_ip_address_creation: String, pub creator_ip_address_last_update: String, @@ -59,6 +60,7 @@ SELECT category.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, + users.email_gravatar_hash AS creator_gravatar_hash, category.creator_ip_address_creation, category.creator_ip_address_last_update, category.is_mod_approved, @@ -105,6 +107,7 @@ WHERE creator_user_token: category.creator_user_token.clone(), creator_username: category.creator_username.clone(), creator_display_name: category.creator_display_name.clone(), + creator_gravatar_hash: category.creator_gravatar_hash.clone(), creator_ip_address_creation: category.creator_ip_address_creation.clone(), creator_ip_address_last_update: category.creator_ip_address_last_update.clone(), is_mod_approved: nullable_i8_to_optional_bool(category.is_mod_approved), @@ -139,6 +142,7 @@ pub struct RawModelCategory { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, + pub creator_gravatar_hash: String, pub creator_ip_address_creation: String, pub creator_ip_address_last_update: String, diff --git a/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs b/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs index 784247c5c3..4fe812c300 100644 --- a/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs +++ b/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs @@ -31,6 +31,7 @@ pub struct Category { pub creator_user_token: Option, pub creator_username: Option, pub creator_display_name: Option, + pub creator_gravatar_hash: Option, // Moderator fields pub is_mod_approved: Option, @@ -118,6 +119,7 @@ impl ListCategoriesQueryBuilder { creator_user_token: c.creator_user_token.clone(), creator_username: c.creator_username.clone(), creator_display_name: c.creator_display_name.clone(), + creator_gravatar_hash: c.creator_gravatar_hash.clone(), is_mod_approved: nullable_i8_to_optional_bool(c.is_mod_approved), maybe_mod_comments: c.maybe_mod_comments.clone(), created_at: c.created_at, @@ -183,6 +185,7 @@ SELECT users.token as creator_user_token, users.username as creator_username, users.display_name as creator_display_name, + users.email_gravatar_hash AS creator_gravatar_hash, model_categories.is_mod_approved, model_categories.maybe_mod_comments, @@ -263,6 +266,7 @@ pub struct RawInternalCategoryRecord { pub creator_user_token: Option, pub creator_username: Option, pub creator_display_name: Option, + pub creator_gravatar_hash: Option, // Moderator fields pub is_mod_approved: Option, diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs index b12df1575c..ba9e0cd0f9 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs @@ -119,7 +119,7 @@ pub async fn assign_tts_category_handler( return Err(AssignTtsCategoryError::CategoryNotFound); }, Err(err) => { - warn!("error looking up model: {:?}", err); + warn!("error looking up category: {:?}", err); return Err(AssignTtsCategoryError::ServerError); }, }; diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/get_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/get_category.rs new file mode 100644 index 0000000000..cff1802220 --- /dev/null +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/get_category.rs @@ -0,0 +1,180 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::queries::categories::get_category_by_token::get_category_by_token; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Request =============== + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetCategoryPathInfo { + /// TTS model token + token: String, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct GetCategoryResponse { + pub success: bool, + pub category: DisplayCategory, +} + +/// Public-facing and optimized (non-irrelevant fields) category list +/// Used for the main TTS dropdown as well as the TTS edit/tagging UI +#[derive(Serialize)] +pub struct DisplayCategory { + pub category_token: String, + + pub model_type: String, // TODO: ENUM + + pub maybe_super_category_token: Option, + + pub can_directly_have_models: bool, + pub can_have_subcategories: bool, + pub can_only_mods_apply: bool, + + pub name: String, + pub maybe_dropdown_name: Option, + + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub creator_gravatar_hash: String, + + // Moderator fields + pub is_mod_approved: Option, + pub maybe_mod_comments: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub deleted_at: Option>, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum GetCategoryError { + NotFound, + ServerError, +} + +impl ResponseError for GetCategoryError { + fn status_code(&self) -> StatusCode { + match *self { + GetCategoryError::NotFound => StatusCode::NOT_FOUND, + GetCategoryError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GetCategoryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn get_category_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetCategoryError::ServerError + })?; + + // TODO: We don't have any permissions for categories. This is a proxy. + let is_mod = maybe_user_session + .map(|session| session.can_ban_users) + .unwrap_or(false); + + let category_lookup_result + = get_category_by_token(&path.token, &server_state.mysql_pool).await; + + let category = match category_lookup_result { + Ok(Some(result)) => { + result + }, + Ok(None) => { + warn!("could not find category"); + return Err(GetCategoryError::NotFound); + }, + Err(err) => { + warn!("error looking up category: {:?}", err); + return Err(GetCategoryError::ServerError); + }, + }; + + let category_for_response = DisplayCategory { + category_token: category.category_token.clone(), + model_type: category.model_type.clone(), + maybe_super_category_token: category.maybe_super_category_token.clone(), + can_directly_have_models: category.can_directly_have_models, + can_have_subcategories: category.can_have_subcategories, + can_only_mods_apply: category.can_only_mods_apply, + name: category.name.clone(), + maybe_dropdown_name: category.maybe_dropdown_name.clone(), + creator_user_token: category.creator_user_token.clone(), + creator_username: category.creator_username.clone(), + creator_display_name: category.creator_display_name.clone(), + creator_gravatar_hash: category.creator_gravatar_hash.clone(), + is_mod_approved: category.is_mod_approved.clone(), + created_at: category.created_at.clone(), + updated_at: category.updated_at.clone(), + deleted_at: category.deleted_at.clone(), + // Moderator fields + // Clear out fields for non-mods + maybe_mod_comments: if is_mod { category.maybe_mod_comments.clone() } else { None }, + }; + + let response = GetCategoryResponse { + success: true, + category: category_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetCategoryError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs index 647442e030..327bb1bac2 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs @@ -36,6 +36,7 @@ pub struct ListTtsModelAssignedCategoriesPathInfo { /// TTS model token token: String, } + // =============== Success Response =============== #[derive(Serialize)] @@ -120,7 +121,6 @@ pub async fn list_tts_model_assigned_categories_handler( .map(|session| session.can_ban_users) .unwrap_or(false); - let query_builder = ListAssignedTtsCategoriesQueryBuilder::for_model_token(&path.token) .show_deleted(is_mod) .show_unapproved(is_mod); diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs index 6005bf61fb..94c7522b4a 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs @@ -1,5 +1,6 @@ pub mod assign_tts_category; pub mod create_category; pub mod delete_category; +pub mod get_category; pub mod list_tts_categories; pub mod list_tts_model_assigned_categories; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs index 63f4ca8cdd..6b97bbf76c 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs @@ -55,6 +55,7 @@ pub struct CategoryForModeration { pub creator_user_token: Option, pub creator_username: Option, pub creator_display_name: Option, + pub creator_gravatar_hash: Option, // Moderator fields pub is_mod_approved: Option, @@ -154,6 +155,7 @@ pub async fn list_tts_categories_for_moderation_handler( creator_user_token: c.creator_user_token.clone(), creator_username: c.creator_username.clone(), creator_display_name: c.creator_display_name.clone(), + creator_gravatar_hash: c.creator_gravatar_hash.clone(), is_mod_approved: c.is_mod_approved.clone(), maybe_mod_comments: c.maybe_mod_comments.clone(), created_at: c.created_at.clone(), diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/app/storyteller_web/src/routes.rs index 6a24de5fdd..60071fa96a 100644 --- a/crates/app/storyteller_web/src/routes.rs +++ b/crates/app/storyteller_web/src/routes.rs @@ -66,6 +66,7 @@ use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_ use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; use crate::http_server::endpoints::moderation::categories::list_tts_categories_for_moderation::list_tts_categories_for_moderation_handler; +use crate::http_server::endpoints::categories::get_category::get_category_handler; pub fn add_routes (app: App) -> App where @@ -438,6 +439,11 @@ fn add_category_routes (app: App) -> App .route(web::post().to(delete_category_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/view/{token}") + .route(web::get().to(get_category_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::scope("/list") .service( From 60cb078cd32ed1bed7ce1b49fd346c8d15291529 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 31 Dec 2021 03:55:53 -0500 Subject: [PATCH 0718/1042] comment --- .../src/http_server/endpoints/categories/get_category.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/get_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/get_category.rs index cff1802220..98fe959f51 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/get_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/get_category.rs @@ -69,8 +69,9 @@ pub struct DisplayCategory { pub creator_display_name: String, pub creator_gravatar_hash: String, - // Moderator fields pub is_mod_approved: Option, + + /// Absent for non-mods pub maybe_mod_comments: Option, pub created_at: DateTime, From f97cdc1ae53afe0487e0b6a45cd2887f4e64f9b3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 31 Dec 2021 05:11:07 -0500 Subject: [PATCH 0719/1042] tree integrity checks --- .../moderation/categories/edit_category.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs index cb8bad069d..cb80ae1927 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs @@ -27,6 +27,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; +use crate::database::queries::categories::get_category_by_token::get_category_by_token; // =============== Request =============== @@ -127,6 +128,31 @@ pub async fn edit_category_handler( return Err(EditCategoryError::NotAuthorized); } + // Category tree integrity + if let Some(parent_category_token) = request.maybe_super_category_token.as_deref() { + if parent_category_token == &path.token { + return Err(EditCategoryError::BadInput( + "category cannot have itself as a parent".to_string())); + } + + let parent_category_lookup + = get_category_by_token(parent_category_token, &server_state.mysql_pool).await; + + match parent_category_lookup { + Ok(Some(parent_category)) => { + if !parent_category.can_have_subcategories { + return Err(EditCategoryError::BadInput( + "parent category cannot have children".to_string())); + } + }, + Ok(None) => return Err(EditCategoryError::NotFound), + Err(err) => { + warn!("Category lookup DB error: {:?}", err); + return Err(EditCategoryError::ServerError) + }, + } + } + let query_result = // We need to store the IP address details. sqlx::query!( From 7a83a51ca3312a2067903b3fa99c54de0b8cceaa Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 31 Dec 2021 05:21:53 -0500 Subject: [PATCH 0720/1042] Fix bug with removing categories --- .../src/http_server/endpoints/categories/assign_tts_category.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs index ba9e0cd0f9..ba2dcd244c 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs @@ -142,7 +142,7 @@ pub async fn assign_tts_category_handler( } } - if !category.can_directly_have_models { + if request.assign && !category.can_directly_have_models { warn!("category cannot have models: {}", category.category_token); return Err(AssignTtsCategoryError::CategoryNotApplicable); } From a050c4d2defed4163702827a5f6d4ecfe589ee10 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 31 Dec 2021 06:25:26 -0500 Subject: [PATCH 0721/1042] additional filtering on models --- .../list_assigned_tts_categories_query_builder.rs | 14 ++++++++++++++ .../list_tts_model_assigned_categories.rs | 1 + 2 files changed, 15 insertions(+) diff --git a/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs b/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs index fdefd6504b..b7f618c40a 100644 --- a/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs +++ b/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs @@ -56,6 +56,10 @@ pub struct ListAssignedTtsCategoriesQueryBuilder { // REQUIRED. scope_tts_model_token: String, + // Sometimes "can_directly_have_models" categories change state. + // Mods will want to see invalid uses. + show_invalid_model_not_allowed_categories: bool, + // The public lists won't include unapproved, but mod views // will want to see them. show_unapproved: bool, @@ -69,11 +73,17 @@ impl ListAssignedTtsCategoriesQueryBuilder { pub fn for_model_token(model_token: &str) -> Self { Self { scope_tts_model_token: model_token.to_string(), + show_invalid_model_not_allowed_categories: false, show_unapproved: false, show_deleted: false, } } + pub fn show_invalid_model_not_allowed_categories(mut self, show_invalid_model_not_allowed_categories: bool) -> Self { + self.show_invalid_model_not_allowed_categories = show_invalid_model_not_allowed_categories; + self + } + pub fn show_unapproved(mut self, show_unapproved: bool) -> Self { self.show_unapproved = show_unapproved; self @@ -205,6 +215,10 @@ LEFT OUTER JOIN users // We also don't care to ever show soft-deleted assignments query.push_str(" AND tts_category_assignments.deleted_at IS NULL"); + if !self.show_invalid_model_not_allowed_categories { + query.push_str(" AND model_categories.can_directly_have_models IS TRUE"); + } + if !self.show_unapproved { query.push_str(" AND model_categories.is_mod_approved IS TRUE"); } diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs index 327bb1bac2..b46ff67b55 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs @@ -122,6 +122,7 @@ pub async fn list_tts_model_assigned_categories_handler( .unwrap_or(false); let query_builder = ListAssignedTtsCategoriesQueryBuilder::for_model_token(&path.token) + .show_invalid_model_not_allowed_categories(is_mod) .show_deleted(is_mod) .show_unapproved(is_mod); From d2f6fb5c3d04c78374a5b48078b91e53d92f190d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 1 Jan 2022 23:47:12 -0500 Subject: [PATCH 0722/1042] make "delete category" a mod handler --- .../src/http_server/endpoints/categories/mod.rs | 1 - .../categories/delete_category.rs | 0 .../endpoints/moderation/categories/mod.rs | 1 + crates/app/storyteller_web/src/routes.rs | 16 ++++++++-------- 4 files changed, 9 insertions(+), 9 deletions(-) rename crates/app/storyteller_web/src/http_server/endpoints/{ => moderation}/categories/delete_category.rs (100%) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs index 94c7522b4a..a8e6e001ae 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs @@ -1,6 +1,5 @@ pub mod assign_tts_category; pub mod create_category; -pub mod delete_category; pub mod get_category; pub mod list_tts_categories; pub mod list_tts_model_assigned_categories; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/delete_category.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/categories/delete_category.rs rename to crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs index cf77590623..0f457bdcc0 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs @@ -1,2 +1,3 @@ +pub mod delete_category; pub mod edit_category; pub mod list_tts_categories_for_moderation; \ No newline at end of file diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/app/storyteller_web/src/routes.rs index 60071fa96a..cc4c219583 100644 --- a/crates/app/storyteller_web/src/routes.rs +++ b/crates/app/storyteller_web/src/routes.rs @@ -5,7 +5,7 @@ use actix_web::error::Error; use actix_web::{App, web, HttpResponse}; use crate::http_server::endpoints::categories::assign_tts_category::assign_tts_category_handler; use crate::http_server::endpoints::categories::create_category::create_category_handler; -use crate::http_server::endpoints::categories::delete_category::delete_category_handler; +use crate::http_server::endpoints::categories::get_category::get_category_handler; use crate::http_server::endpoints::categories::list_tts_categories::list_tts_categories_handler; use crate::http_server::endpoints::categories::list_tts_model_assigned_categories::list_tts_model_assigned_categories_handler; use crate::http_server::endpoints::events::list_events::list_events_handler; @@ -15,7 +15,9 @@ use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha use crate::http_server::endpoints::misc::enable_alpha_handler::enable_alpha_handler; use crate::http_server::endpoints::misc::root_index::get_root_index; use crate::http_server::endpoints::moderation::approval::pending_w2l_templates::get_pending_w2l_templates_handler; +use crate::http_server::endpoints::moderation::categories::delete_category::delete_category_handler; use crate::http_server::endpoints::moderation::categories::edit_category::edit_category_handler; +use crate::http_server::endpoints::moderation::categories::list_tts_categories_for_moderation::list_tts_categories_for_moderation_handler; use crate::http_server::endpoints::moderation::ip_bans::add_ip_ban::add_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::delete_ip_ban::delete_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_handler; @@ -65,8 +67,6 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; -use crate::http_server::endpoints::moderation::categories::list_tts_categories_for_moderation::list_tts_categories_for_moderation_handler; -use crate::http_server::endpoints::categories::get_category::get_category_handler; pub fn add_routes (app: App) -> App where @@ -244,6 +244,11 @@ fn add_moderator_routes (app: App) -> App .route(web::post().to(edit_category_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/{token}/delete") + .route(web::post().to(delete_category_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/tts/list") .route(web::get().to(list_tts_categories_for_moderation_handler)) @@ -434,11 +439,6 @@ fn add_category_routes (app: App) -> App .route(web::post().to(create_category_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - .service( - web::resource("/{token}/delete") - .route(web::post().to(delete_category_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) .service( web::resource("/view/{token}") .route(web::get().to(get_category_handler)) From ad722ed505a7c404cb89c624d1f098da6c00a11f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 2 Jan 2022 01:00:42 -0500 Subject: [PATCH 0723/1042] show/hide categories with tri-state query params --- .../list_categories_query_builder.rs | 40 +++++++++++++++++ .../list_tts_categories_for_moderation.rs | 43 +++++++++++++++++-- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs b/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs index 4fe812c300..45fb053720 100644 --- a/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs +++ b/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs @@ -57,6 +57,16 @@ pub struct ListCategoriesQueryBuilder { // will want to see them. show_deleted: bool, + // Filter out approved records. + // Useful as a moderation filter. + // Doesn't make sense to use with `show_unapproved`, but don't want to refactor atm. + hide_approved: bool, + + // Filter out records with a non-deleted timestamp. + // Useful as a moderation filter. + // Doesn't make sense to use with `show_deleted`, but don't want to refactor atm. + hide_non_deleted: bool, + // Useful for showing categories a user themselves has suggested // We don't want users spamming creation. scope_creator_user_token: Option, @@ -71,6 +81,8 @@ impl ListCategoriesQueryBuilder { Self { show_unapproved: false, show_deleted: false, + hide_approved: false, + hide_non_deleted: false, scope_creator_user_token: None, scope_model_type: None, } @@ -86,6 +98,16 @@ impl ListCategoriesQueryBuilder { self } + pub fn hide_approved(mut self, hide_approved: bool) -> Self { + self.hide_approved = hide_approved; + self + } + + pub fn hide_non_deleted(mut self, hide_non_deleted: bool) -> Self { + self.hide_non_deleted = hide_non_deleted; + self + } + pub fn scope_creator_user_token(mut self, scope_creator_user_token: Option<&str>) -> Self { self.scope_creator_user_token = scope_creator_user_token.map(|u| u.to_string()); self @@ -244,6 +266,24 @@ LEFT OUTER JOIN users } } + if self.hide_approved { + if !first_predicate_added { + query.push_str(" WHERE (model_categories.is_mod_approved IS NULL OR model_categories.is_mod_approved IS FALSE) "); + first_predicate_added = true; + } else { + query.push_str(" AND (model_categories.is_mod_approved IS NULL OR model_categories.is_mod_approved IS FALSE) "); + } + } + + if self.hide_non_deleted { + if !first_predicate_added { + query.push_str(" WHERE model_categories.deleted_at IS NOT NULL"); + first_predicate_added = true; + } else { + query.push_str(" AND model_categories.deleted_at IS NOT NULL"); + } + } + query } } diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs index 6b97bbf76c..d6f9476232 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs @@ -29,6 +29,26 @@ use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; +// =============== Request =============== + +#[derive(Deserialize, Copy, Clone, Debug)] +#[serde(rename_all = "snake_case")] +pub enum ViewState { + /// 'exclude'; keep the objects out + Exclude, + /// 'include'; include the objects + Include, + /// 'only'; limit to just the objects + Only, +} + +// Only show approved&non-deleted , only show un-approved, only show deleted +#[derive(Deserialize)] +pub struct QueryParams { + deleted: Option, + unapproved: Option, +} + // =============== Success Response =============== #[derive(Serialize)] @@ -98,6 +118,7 @@ impl fmt::Display for ListTtsCategoriesForModerationError { pub async fn list_tts_categories_for_moderation_handler( http_request: HttpRequest, + query: web::Query, server_state: web::Data>) -> Result { let maybe_user_session = server_state @@ -123,11 +144,25 @@ pub async fn list_tts_categories_for_moderation_handler( return Err(ListTtsCategoriesForModerationError::NotAuthorized); } - let query_builder = ListCategoriesQueryBuilder::new() - .show_deleted(true) - .show_unapproved(true) + // The objects we'll show or hide + let delete_state = query.deleted.unwrap_or(ViewState::Exclude); + let unapproved_state = query.unapproved.unwrap_or(ViewState::Include); + + let mut query_builder = ListCategoriesQueryBuilder::new() .scope_model_type(Some("tts")); - + + query_builder = match delete_state { + ViewState::Exclude => query_builder.show_deleted(false), + ViewState::Include => query_builder.show_deleted(true), + ViewState::Only => query_builder.show_deleted(true).hide_non_deleted(true), + }; + + query_builder = match unapproved_state { + ViewState::Exclude => query_builder.show_unapproved(false), + ViewState::Include => query_builder.show_unapproved(true), + ViewState::Only => query_builder.show_unapproved(true).hide_approved(true), + }; + let query_result = query_builder.perform_query(&server_state.mysql_pool).await; From 684603eebdd73ef3e2bc2fbc579bd40794d97bda Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 3 Jan 2022 01:56:52 -0500 Subject: [PATCH 0724/1042] Fix OBS builds --- .../src/endpoints_ws/obs_twitch_thread.rs | 56 ++++++++++--------- .../app/obs_gateway/src/util/random_nonce.rs | 8 +-- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs b/crates/app/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs index 215cfa2c20..bafbe68ef6 100644 --- a/crates/app/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs +++ b/crates/app/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs @@ -5,8 +5,8 @@ use std::sync::atomic::AtomicBool; use std::sync::{Arc, RwLock, PoisonError}; use std::thread::sleep; use std::time::Duration; +use std::time::Instant; use tokio::runtime::Handle; -use time::Instant; pub struct ObsTwitchThread { twitch_client: Arc>, @@ -23,11 +23,16 @@ impl ObsTwitchThread { } pub fn keep_alive_thread(&self) { - let now = Instant::now(); - let lock = self.last_requested - .clone() - .get_mut(); - *lock = Some(now); + // TODO(bt, 2022-01-03): builds are failing, not sure when I worked on this last + //let now = Instant::now(); + //let lock = self.last_requested + // .clone() + // .get_mut(); + ////*lock = Some(now); + //match lock { + // Ok(l) => *l = Some(now), + // Err(_) => {} + //} } // TODO: Investigate use of a thread pool instead. @@ -44,29 +49,30 @@ impl ObsTwitchThread { fn thread_main_loop( mut last_requested: Arc>, ) { - loop { - let c = last_requested.get_mut(); + // TODO(bt, 2022-01-03): builds are failing, not sure when I worked on this last + //loop { + // let c = last_requested.get_mut(); - match c { - Err(e) => { - error!("Mutex failure: {:?}, ending job.", e); - break; - } - Ok(t) => { - let now = Instant::now(); - let delta : Duration = now.duration_since(t); + // match c { + // Err(e) => { + // error!("Mutex failure: {:?}, ending job.", e); + // break; + // } + // Ok(t) => { + // let now = Instant::now(); + // let delta : Duration = now.duration_since(t.clone()); - if delta.gt(&Duration::from_secs(60)) { - info!("Timeout elapsed, ending job."); - break; - } - } - } + // if delta.gt(&Duration::from_secs(60)) { + // info!("Timeout elapsed, ending job."); + // break; + // } + // } + // } - Self::thread_single_loop(); + // Self::thread_single_loop(); - sleep(Duration::from_millis(1000)); - } + // sleep(Duration::from_millis(1000)); + //} } fn thread_single_loop() { diff --git a/crates/app/obs_gateway/src/util/random_nonce.rs b/crates/app/obs_gateway/src/util/random_nonce.rs index 5f9034d3aa..f87ab1765e 100644 --- a/crates/app/obs_gateway/src/util/random_nonce.rs +++ b/crates/app/obs_gateway/src/util/random_nonce.rs @@ -16,9 +16,9 @@ mod tests { #[test] fn random_nonce_length() { - assert_eq(1, random_nonce(1).len()); - assert_eq(10, random_nonce(10).len()); - assert_eq(15, random_nonce(15).len()); - assert_eq(64, random_nonce(64).len()); + assert_eq!(1, random_nonce(1).len()); + assert_eq!(10, random_nonce(10).len()); + assert_eq!(15, random_nonce(15).len()); + assert_eq!(64, random_nonce(64).len()); } } \ No newline at end of file From 85a21efaf648f13b08dc423a1f38f94d1730a85a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 3 Jan 2022 02:27:24 -0500 Subject: [PATCH 0725/1042] Fix sqlx issue preventing builds Move to Rust workspace broke how OFFLINE mode db schema/query cache is read and stored. --- codegen_database.sh | 9 ++ sqlx-data.json | 370 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 379 insertions(+) diff --git a/codegen_database.sh b/codegen_database.sh index cd78945620..f1fc85b502 100755 --- a/codegen_database.sh +++ b/codegen_database.sh @@ -1,2 +1,11 @@ #!/bin/bash + +# Necessary since we're now in a more complex Rust workspace +pushd crates/app/storyteller_web + SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web + +popd + +mv crates/app/storyteller_web/sqlx-data.json . + diff --git a/sqlx-data.json b/sqlx-data.json index 5d4668089e..bfde92b74c 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -337,6 +337,44 @@ ] } }, + "013e26fbe298b3de8403f9541807190fe556470c55b7ca342a50aaafea1bbb87": { + "query": "\nSELECT\n assignments.model_token AS tts_model_token,\n assignments.category_token\nFROM\n tts_category_assignments AS assignments\nJOIN\n tts_models AS tts\n ON assignments.model_token = tts.token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "category_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false + ] + } + }, "02060ab9b8ca472a2bd4d4fa3120743e90148ccd6a4cb13bc491d00f36ed7a8f": { "query": "\nSELECT\n id,\n status\nFROM w2l_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", "describe": { @@ -1899,6 +1937,16 @@ ] } }, + "218b3fe667b34bfa158bb7d4f600a3b91a22460adf566dcdf6ce540006e6c684": { + "query": "\nUPDATE model_categories\nSET\n name = ?,\n maybe_dropdown_name = ?,\n\n can_directly_have_models = ?,\n can_have_subcategories = ?,\n can_only_mods_apply = ?,\n\n maybe_super_category_token = ?,\n\n is_mod_approved = ?,\n maybe_mod_user_token = ?,\n maybe_mod_comments = ?,\n\n version = version + 1\n\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, "25e7858d370ac1359deb42ef5292e85ccd1fe7debf3a1ba73303e5882523a4d0": { "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n download_url = ?,\n status = \"pending\"\n ", "describe": { @@ -3513,6 +3561,16 @@ ] } }, + "5473d3857478d001688998056377fefe4bcc792477f303b41fa3cf64f79170f3": { + "query": "\n INSERT INTO tts_category_assignments\n SET\n model_token = ?,\n category_token = ?,\n category_removal_user_token = ?,\n ip_address_creation = ?,\n ip_address_last_update = ?,\n deleted_at = CURRENT_TIMESTAMP\n ON DUPLICATE KEY UPDATE\n model_token = ?,\n category_token = ?,\n category_removal_user_token = ?,\n ip_address_last_update = ?,\n deleted_at = CURRENT_TIMESTAMP\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, "551749bc153f9c8683a3a1ca5bb50744322aac046db34327501cf991485c4dee": { "query": "\nUPDATE tts_models\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -4614,6 +4672,16 @@ "nullable": [] } }, + "720d7a26fb8ce75a8a1b3a62ef509d8ca99d419172e9cc5d8e2c803e0eebcb7f": { + "query": "\n UPDATE model_categories\n SET\n deleted_at = NULL\n WHERE\n token = ?\n LIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, "73558104936aab6add2d9c789b368b5f155f1fe01d32e7314f2f120422866f54": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { @@ -5461,6 +5529,278 @@ "nullable": [] } }, + "87d7261dd2a589220c5539f51dde42ef2407677f7ece7968761426c6fd0e0360": { + "query": "\nSELECT\n category.token as category_token,\n category.model_type,\n category.maybe_super_category_token,\n category.can_directly_have_models,\n category.can_have_subcategories,\n category.can_only_mods_apply,\n category.name,\n category.maybe_dropdown_name,\n category.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash AS creator_gravatar_hash,\n category.creator_ip_address_creation,\n category.creator_ip_address_last_update,\n category.is_mod_approved,\n category.maybe_mod_comments,\n category.maybe_mod_user_token,\n category.created_at,\n category.updated_at,\n category.deleted_at\nFROM model_categories AS category\nJOIN users\nON category.creator_user_token = users.token\nWHERE\n category.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "category_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "maybe_super_category_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "can_directly_have_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 4, + "name": "can_have_subcategories", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 5, + "name": "can_only_mods_apply", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 6, + "name": "name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "maybe_dropdown_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 10, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 11, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 12, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 13, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 14, + "name": "is_mod_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "maybe_mod_comments", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 16, + "name": "maybe_mod_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 17, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + true, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false, + true + ] + } + }, "88328e04ce947d3f541951aada9cee25135aa950ab478cf0dc6de453bb079e17": { "query": "\nSELECT\n events.token as event_token,\n events.event_type,\n events.maybe_target_user_token,\n users.username as maybe_target_username,\n users.display_name as maybe_target_display_name,\n users.email_gravatar_hash as maybe_target_user_gravatar_hash,\n events.maybe_target_entity_token,\n events.created_at,\n events.updated_at\nFROM firehose_entries as events\nLEFT OUTER JOIN users\nON events.maybe_target_user_token = users.token\nORDER BY events.id DESC\nLIMIT 25\n ", "describe": { @@ -5600,6 +5940,16 @@ "nullable": [] } }, + "8b1b9cf11b80da18d79237f2579a96b4a85fa7201118e01fffc01c1eba64cbaf": { + "query": "\n UPDATE model_categories\n SET\n deleted_at = CURRENT_TIMESTAMP\n WHERE\n token = ?\n LIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, "8ff27fc5583db97f157f1b808f593a72c77dc5dcf5a633cd9c1162af6b97c011": { "query": "\nUPDATE w2l_templates\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -7928,6 +8278,16 @@ "nullable": [] } }, + "ce629cea6c387590759cc89a060c471ecdcb5d5f46a53576897a963154ed4d74": { + "query": "\n INSERT INTO tts_category_assignments\n SET\n model_token = ?,\n category_token = ?,\n category_addition_user_token = ?,\n ip_address_creation = ?,\n ip_address_last_update = ?,\n deleted_at = NULL\n ON DUPLICATE KEY UPDATE\n model_token = ?,\n category_token = ?,\n category_addition_user_token = ?,\n ip_address_last_update = ?,\n deleted_at = NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, "d12757abef811cd6951e0e1427fdb496a8931b547519d320f5ade56007d3eb5d": { "query": "\nSELECT\n id,\n status\nFROM w2l_template_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", "describe": { @@ -8985,6 +9345,16 @@ ] } }, + "fd31f7abe386147c07f862d668c9e56eb3d809c4f0a61b7debf9b2e3785a4c83": { + "query": "\nINSERT INTO model_categories\nSET\n token = ?,\n uuid_idempotency_token = ?,\n model_type = ?,\n name = ?,\n\n creator_user_token = ?,\n creator_ip_address_creation = ?,\n creator_ip_address_last_update = ?,\n\n is_mod_approved = ?,\n maybe_mod_user_token = ?,\n can_directly_have_models = ?,\n can_have_subcategories = ?,\n can_only_mods_apply = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 12 + }, + "nullable": [] + } + }, "feb0dae816a5701707a76e15011ea6cb262b10c663abf2cf1fb46bd9b1cebe09": { "query": "\nSELECT token, username, email_address, password_hash, is_banned\nFROM users\nWHERE username = ?\nLIMIT 1\n ", "describe": { From 62d894b82655efdd884d25028ef68dc4982f90a2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 3 Jan 2022 05:54:47 -0500 Subject: [PATCH 0726/1042] Add deleted_at and other timestamps so it resembles other endpoints It's not too bloated (# categories is tractable), and we need these for endpoint uniformity --- .../endpoints/categories/list_tts_categories.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs index f31a2b819d..1493826ad0 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs @@ -6,6 +6,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; @@ -53,9 +54,18 @@ pub struct DisplayCategory { pub name: String, pub maybe_dropdown_name: Option, + // Moderator fields + // It's okay to leak this since we do for assigned categories and for mods to see + // which assigned categories might be invalid. + pub is_mod_approved: Option, + //pub creator_user_token: Option, //pub creator_username: Option, //pub creator_display_name: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + pub deleted_at: Option>, } // =============== Error Response =============== @@ -119,8 +129,12 @@ pub async fn list_tts_categories_handler( can_directly_have_models: c.can_directly_have_models, can_have_subcategories: c.can_have_subcategories, can_only_mods_apply: c.can_only_mods_apply, + is_mod_approved: c.is_mod_approved, name: c.name.clone(), - maybe_dropdown_name:c.maybe_dropdown_name.clone(), + maybe_dropdown_name: c.maybe_dropdown_name.clone(), + created_at: c.created_at.clone(), + updated_at: c.updated_at.clone(), + deleted_at: c.deleted_at.clone(), } }) .collect::>(); From 963c5f7c326f1daa9286a47d009fc74348132931 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 5 Jan 2022 02:04:55 -0500 Subject: [PATCH 0727/1042] natural+lexographic sort of category dropdown names --- Cargo.lock | 16 ++++++++++++++++ crates/app/storyteller_web/Cargo.toml | 1 + .../endpoints/categories/list_tts_categories.rs | 12 ++++++++---- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 44e91d03ea..d8f138e447 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -657,6 +657,12 @@ dependencies = [ "syn", ] +[[package]] +name = "any_ascii" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70033777eb8b5124a81a1889416543dddef2de240019b674c81285a2635a7e1e" + [[package]] name = "anyhow" version = "1.0.40" @@ -2143,6 +2149,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "lexical-sort" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c09e4591611e231daf4d4c685a66cb0410cc1e502027a20ae55f2bb9e997207a" +dependencies = [ + "any_ascii", +] + [[package]] name = "lfu" version = "0.2.5" @@ -3977,6 +3992,7 @@ dependencies = [ "infer", "jwt", "lazy_static", + "lexical-sort", "lfu", "limitation", "log", diff --git a/crates/app/storyteller_web/Cargo.toml b/crates/app/storyteller_web/Cargo.toml index 46deec8d45..180afb2edd 100644 --- a/crates/app/storyteller_web/Cargo.toml +++ b/crates/app/storyteller_web/Cargo.toml @@ -51,6 +51,7 @@ easyenv = "0.0.6" hex = "0.4.3" hostname = "0.3.1" lazy_static = "1.4.0" +lexical-sort = "0.3.1" lfu = "0.2.5" log = "0.4.14" newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs index 1493826ad0..2d9e08a6f8 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs @@ -20,6 +20,7 @@ use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::validations::check_for_slurs::contains_slurs; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use lexical_sort::natural_lexical_cmp; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; @@ -52,7 +53,8 @@ pub struct DisplayCategory { pub can_only_mods_apply: bool, pub name: String, - pub maybe_dropdown_name: Option, + /// Instead of making the frontend deal with "maybe_dropdown_name", always use this: + pub name_for_dropdown: String, // Moderator fields // It's okay to leak this since we do for assigned categories and for mods to see @@ -131,7 +133,7 @@ pub async fn list_tts_categories_handler( can_only_mods_apply: c.can_only_mods_apply, is_mod_approved: c.is_mod_approved, name: c.name.clone(), - maybe_dropdown_name: c.maybe_dropdown_name.clone(), + name_for_dropdown: c.maybe_dropdown_name.clone().unwrap_or(c.name.clone()), created_at: c.created_at.clone(), updated_at: c.updated_at.clone(), deleted_at: c.deleted_at.clone(), @@ -139,8 +141,10 @@ pub async fn list_tts_categories_handler( }) .collect::>(); - // TODO: Sort by dropdown name too. - categories.sort_by(|c1, c2| c1.name.cmp(&c2.name)); + // NB: This might produce weird sorting resorts relative to the "name" field, + // but the typical way this should be consumed is via dropdowns. + categories.sort_by(|c1, c2| + natural_lexical_cmp(&c1.name_for_dropdown, &c2.name_for_dropdown)); // TODO: Cache results in Redis w/ TTL. let response = ListTtsCategoriesResponse { From 14159a26a6edd3d82d4a2a23c0d26a8c5372d2df Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 5 Jan 2022 02:08:00 -0500 Subject: [PATCH 0728/1042] also sort the tts models --- .../src/http_server/endpoints/tts/list_tts_models.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs b/crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs index a6f3ce2915..7361749af1 100644 --- a/crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs +++ b/crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs @@ -20,6 +20,7 @@ use sqlx::mysql::MySqlDatabaseError; use std::collections::HashSet; use std::fmt; use std::sync::Arc; +use lexical_sort::natural_lexical_cmp; #[derive(Serialize, Clone)] pub struct TtsModelRecordForResponse { @@ -142,7 +143,9 @@ async fn get_all_models(mysql_pool: &MySqlPool) -> AnyhowResult Date: Thu, 6 Jan 2022 03:09:07 -0500 Subject: [PATCH 0729/1042] note about migrations in prod --- .../2021-08-25-010101_alter_tables_add_vocoder_type/up.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/migrations/2021-08-25-010101_alter_tables_add_vocoder_type/up.sql b/migrations/2021-08-25-010101_alter_tables_add_vocoder_type/up.sql index 7a16e2c67b..3ce73ad8f4 100644 --- a/migrations/2021-08-25-010101_alter_tables_add_vocoder_type/up.sql +++ b/migrations/2021-08-25-010101_alter_tables_add_vocoder_type/up.sql @@ -6,6 +6,7 @@ -- I'll have to repair the migrations table. -- version: 20210925010101 | run_on: 2021-08-25 03:01:15 (development) -- UPDATE __diesel_schema_migrations SET version=20210825010101 WHERE version=20210925010101 LIMIT 1; +-- NB: Fixed in production on 2022-01-06 ALTER TABLE tts_models ADD COLUMN maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL From ac95040be22a3dbfcaac4083b2410051ec4740b8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 6 Jan 2022 19:13:28 -0500 Subject: [PATCH 0730/1042] Add nginx config files (from mac) --- localdev/nginx-http-config/nginx.conf | 18 ++++++ .../sites-available/storyteller | 60 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 localdev/nginx-http-config/nginx.conf create mode 100644 localdev/nginx-http-config/sites-available/storyteller diff --git a/localdev/nginx-http-config/nginx.conf b/localdev/nginx-http-config/nginx.conf new file mode 100644 index 0000000000..19e69ff647 --- /dev/null +++ b/localdev/nginx-http-config/nginx.conf @@ -0,0 +1,18 @@ +server { + listen 80; + server_name api.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:12345; + proxy_redirect off; + } +} +server { + listen 80; + server_name jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:7000; + proxy_redirect off; + } +} diff --git a/localdev/nginx-http-config/sites-available/storyteller b/localdev/nginx-http-config/sites-available/storyteller new file mode 100644 index 0000000000..ff1c9e89ca --- /dev/null +++ b/localdev/nginx-http-config/sites-available/storyteller @@ -0,0 +1,60 @@ +server { + listen 80; + server_name api.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:12345; + proxy_redirect off; + } +} +server { + listen 80; + server_name jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:7000; + proxy_redirect off; + } +} + +#events {} +#http { + #server_names_hash_bucket_size 64; + + # upstream backend { + # server 127.0.0.1:7000 + # } + # server { + # server_name jungle.horse; + # rewrite ^(.*) https://jungle.horse$1 permanent; + # } + #server { + # listen 443 ssl; + # # ssl on; + # ssl_certificate /Users/bt/dev/storyteller/storyteller-web/localhost.crt; + # ssl_certificate_key /Users/bt/dev/storyteller/storyteller-web/localhost.key; + # ssl_ciphers HIGH:!aNULL:!MD5; + # server_name jungle.horse; + # location / { + # proxy_pass http://127.0.0.1:7000; + # } + #} +# server { +# listen 80; +# server_name api.jungle.horse; +# location / { +# proxy_set_header Host $host; +# proxy_pass http://127.0.0.1:12345; +# proxy_redirect off; +# } +# } +# server { +# listen 80; +# server_name jungle.horse; +# location / { +# proxy_set_header Host $host; +# proxy_pass http://127.0.0.1:7000; +# proxy_redirect off; +# } +# } +#} From 20676f595a46779dac87c61012ef3eff5c8d0c2c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 7 Jan 2022 01:52:56 -0500 Subject: [PATCH 0731/1042] only need one nginx file --- README.md | 76 ++++++++++++++++++- .../sites-available/storyteller | 60 --------------- .../{nginx.conf => storyteller} | 0 3 files changed, 75 insertions(+), 61 deletions(-) delete mode 100644 localdev/nginx-http-config/sites-available/storyteller rename localdev/nginx-http-config/{nginx.conf => storyteller} (100%) diff --git a/README.md b/README.md index d1b20f5ca4..590cc5bb06 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,10 @@ SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web Set up a local nginx to proxy to the frontend and backend so cookie issues aren't annoying -Edit, `/etc/nginx/sites-enabled/storyteller` +Edit the nginx config file, + +- `/etc/nginx/sites-enabled/storyteller` (linux) +- `/usr/local/etc/nginx/sites-enabled/storyteller` (mac) ``` server { @@ -174,6 +177,7 @@ server { } } + ``` And in /etc/hosts, @@ -184,6 +188,76 @@ And in /etc/hosts, ``` +### Setting up local Nginx with SSL self-signed certs + +From https://imagineer.in/blog/https-on-localhost-with-nginx/ , + +``` +openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt +``` + +Most fields can be blank, but the "Common Name" should be `*.jungle.horse` + + +Edit the appropriate Nginx config file, + +- `/etc/nginx/sites-enabled/storyteller` (linux) +- `/usr/local/etc/nginx/sites-enabled/storyteller` (mac typical) +- `/opt/homebrew/etc/nginx/nginx.conf (mac alternate, w/ homebrew) +- `/opt/homebrew/etc/nginx/servers/storyteller` (mac alternate - this is the one I'm using) + +``` +events {} +http { + # upstream backend { + # server 127.0.0.1:7000 + # } + # server { + # server_name jungle.horse; + # rewrite ^(.*) https://jungle.horse$1 permanent; + # } + server { + listen 443; + ssl on; + ssl_certificate /Users/bt/dev/storyteller/storyteller-web/localhost.crt; + ssl_certificate_key /Users/bt/dev/storyteller/storyteller-web/localhost.key; + ssl_ciphers HIGH:!aNULL:!MD5; + server_name: jungle.horse; + location / { + proxy_pass http://127.0.0.1:7000 + } + } +} +server { + listen 80; + server_name api.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:12345; + proxy_redirect off; + } +} +server { + listen 80; + server_name jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:7000; + proxy_redirect off; + } +} + +``` + +Nginx Mac version: + +``` +nginx version: nginx/1.21.4 +built by clang 13.0.0 (clang-1300.0.29.3) +built with OpenSSL 1.1.1l 24 Aug 2021 +TLS SNI support enabled +``` + ### Python 3.6 on Apple M1 Mac Python3.6 isn't supported on Apple silicon, and it's not in homebrew. It can be installed with diff --git a/localdev/nginx-http-config/sites-available/storyteller b/localdev/nginx-http-config/sites-available/storyteller deleted file mode 100644 index ff1c9e89ca..0000000000 --- a/localdev/nginx-http-config/sites-available/storyteller +++ /dev/null @@ -1,60 +0,0 @@ -server { - listen 80; - server_name api.jungle.horse; - location / { - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:12345; - proxy_redirect off; - } -} -server { - listen 80; - server_name jungle.horse; - location / { - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:7000; - proxy_redirect off; - } -} - -#events {} -#http { - #server_names_hash_bucket_size 64; - - # upstream backend { - # server 127.0.0.1:7000 - # } - # server { - # server_name jungle.horse; - # rewrite ^(.*) https://jungle.horse$1 permanent; - # } - #server { - # listen 443 ssl; - # # ssl on; - # ssl_certificate /Users/bt/dev/storyteller/storyteller-web/localhost.crt; - # ssl_certificate_key /Users/bt/dev/storyteller/storyteller-web/localhost.key; - # ssl_ciphers HIGH:!aNULL:!MD5; - # server_name jungle.horse; - # location / { - # proxy_pass http://127.0.0.1:7000; - # } - #} -# server { -# listen 80; -# server_name api.jungle.horse; -# location / { -# proxy_set_header Host $host; -# proxy_pass http://127.0.0.1:12345; -# proxy_redirect off; -# } -# } -# server { -# listen 80; -# server_name jungle.horse; -# location / { -# proxy_set_header Host $host; -# proxy_pass http://127.0.0.1:7000; -# proxy_redirect off; -# } -# } -#} diff --git a/localdev/nginx-http-config/nginx.conf b/localdev/nginx-http-config/storyteller similarity index 100% rename from localdev/nginx-http-config/nginx.conf rename to localdev/nginx-http-config/storyteller From b691750df42f37ab786684edcbaf0d40730c5e18 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 7 Jan 2022 01:57:45 -0500 Subject: [PATCH 0732/1042] docs --- localdev/nginx-http-config/storyteller | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/localdev/nginx-http-config/storyteller b/localdev/nginx-http-config/storyteller index 19e69ff647..451cab6d48 100644 --- a/localdev/nginx-http-config/storyteller +++ b/localdev/nginx-http-config/storyteller @@ -1,3 +1,15 @@ +# Nginx configs for "jungle.horse" and "api.jungle.horse" (http) +# +# On Ubuntu Linux, this config is located at the following location: +# +# - /etc/nginx/sites-enabled/{name-of-site} +# +# On Mac, this is located at the following locations: +# +# - /opt/homebrew/etc/nginx/servers/{name-of-site} (for nginx installed with homebrew) +# - /usr/local/etc/nginx/sites-enabled/{name-of-site} (for default nginx install) +# + server { listen 80; server_name api.jungle.horse; From 1f414016608a24a2279baed9e0d258e39bfeb7e5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 7 Jan 2022 02:30:29 -0500 Subject: [PATCH 0733/1042] add SSL version of nginx configs --- localdev/nginx-http-config/storyteller-ssl | 56 ++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 localdev/nginx-http-config/storyteller-ssl diff --git a/localdev/nginx-http-config/storyteller-ssl b/localdev/nginx-http-config/storyteller-ssl new file mode 100644 index 0000000000..614996eaf1 --- /dev/null +++ b/localdev/nginx-http-config/storyteller-ssl @@ -0,0 +1,56 @@ +# Nginx configs for "jungle.horse" and "api.jungle.horse" (http) +# +# On Ubuntu Linux, this config is located at the following location: +# +# - /etc/nginx/sites-enabled/{name-of-site} +# +# On Mac, this is located at the following locations: +# +# - /opt/homebrew/etc/nginx/servers/{name-of-site} (for nginx installed with homebrew) +# - /usr/local/etc/nginx/sites-enabled/{name-of-site} (for default nginx install) +# +# +# ================ +# SSL Instructions +# ================ +# +# To create self-signed certs on Mac, +# +# openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt +# +# ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse` +# +# On Mac, load the self signed cert into the keychain with the following command: +# +# sudo security add-trusted-cert \ +# -d -r trustRoot \ +# -k /Library/Keychains/System.keychain /etc/ssl/certs/localhost.crt +# +# Self signed certs might be considered insecure by the browser. In Firefox, +# set about:config `security.tls.insecure_fallback_hosts` to include a list of +# permitted domains. You may also need to configure other settings. +# + +server { + listen 80; + server_name api.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:12345; + proxy_redirect off; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/ssl/certs/localhost.crt; + ssl_certificate_key /etc/ssl/private/localhost.key; + ssl_protocols TLSv1.2 TLSv1.1 TLSv1; + server_name jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:7000; + proxy_redirect off; + } +} From df7283a407b18f664d4e7c51fecf6b247600d4c9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 7 Jan 2022 02:35:37 -0500 Subject: [PATCH 0734/1042] storyteller ssl nginx configs + plain http --- localdev/nginx-http-config/storyteller-ssl | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/localdev/nginx-http-config/storyteller-ssl b/localdev/nginx-http-config/storyteller-ssl index 614996eaf1..62584740ae 100644 --- a/localdev/nginx-http-config/storyteller-ssl +++ b/localdev/nginx-http-config/storyteller-ssl @@ -10,6 +10,11 @@ # - /usr/local/etc/nginx/sites-enabled/{name-of-site} (for default nginx install) # # +# On Mac, verify configs and restart with, +# +# sudo nginx -t +# sudo nginx -s reload +# # ================ # SSL Instructions # ================ @@ -30,7 +35,9 @@ # set about:config `security.tls.insecure_fallback_hosts` to include a list of # permitted domains. You may also need to configure other settings. # +# +# API backend over plain HTTP server { listen 80; server_name api.jungle.horse; @@ -41,6 +48,22 @@ server { } } +# Javascript frontend over plain HTTP +server { + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/ssl/certs/localhost.crt; + ssl_certificate_key /etc/ssl/private/localhost.key; + ssl_protocols TLSv1.2 TLSv1.1 TLSv1; + server_name www.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:7000; + proxy_redirect off; + } +} + +# Javascript frontend over SSL server { listen 443 ssl; listen [::]:443 ssl; @@ -54,3 +77,4 @@ server { proxy_redirect off; } } + From dc1fc75c2d2a2392be75d9d9bd2fc27e26c35169 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 7 Jan 2022 02:37:25 -0500 Subject: [PATCH 0735/1042] simplify readme --- README.md | 96 +------------------------------------------------------ 1 file changed, 1 insertion(+), 95 deletions(-) diff --git a/README.md b/README.md index 590cc5bb06..aaa2ca749e 100644 --- a/README.md +++ b/README.md @@ -152,31 +152,7 @@ SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web Set up a local nginx to proxy to the frontend and backend so cookie issues aren't annoying -Edit the nginx config file, - -- `/etc/nginx/sites-enabled/storyteller` (linux) -- `/usr/local/etc/nginx/sites-enabled/storyteller` (mac) - -``` -server { - listen 80; - server_name api.jungle.horse; - location / { - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:12345; - proxy_redirect off; - } -} -server { - listen 80; - server_name jungle.horse; - location / { - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:7000; - proxy_redirect off; - } -} - +Configure Nginx per the checked in Nginx configs (and instructions) in `localdev/nginx-http-config`. ``` @@ -188,76 +164,6 @@ And in /etc/hosts, ``` -### Setting up local Nginx with SSL self-signed certs - -From https://imagineer.in/blog/https-on-localhost-with-nginx/ , - -``` -openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt -``` - -Most fields can be blank, but the "Common Name" should be `*.jungle.horse` - - -Edit the appropriate Nginx config file, - -- `/etc/nginx/sites-enabled/storyteller` (linux) -- `/usr/local/etc/nginx/sites-enabled/storyteller` (mac typical) -- `/opt/homebrew/etc/nginx/nginx.conf (mac alternate, w/ homebrew) -- `/opt/homebrew/etc/nginx/servers/storyteller` (mac alternate - this is the one I'm using) - -``` -events {} -http { - # upstream backend { - # server 127.0.0.1:7000 - # } - # server { - # server_name jungle.horse; - # rewrite ^(.*) https://jungle.horse$1 permanent; - # } - server { - listen 443; - ssl on; - ssl_certificate /Users/bt/dev/storyteller/storyteller-web/localhost.crt; - ssl_certificate_key /Users/bt/dev/storyteller/storyteller-web/localhost.key; - ssl_ciphers HIGH:!aNULL:!MD5; - server_name: jungle.horse; - location / { - proxy_pass http://127.0.0.1:7000 - } - } -} -server { - listen 80; - server_name api.jungle.horse; - location / { - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:12345; - proxy_redirect off; - } -} -server { - listen 80; - server_name jungle.horse; - location / { - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:7000; - proxy_redirect off; - } -} - -``` - -Nginx Mac version: - -``` -nginx version: nginx/1.21.4 -built by clang 13.0.0 (clang-1300.0.29.3) -built with OpenSSL 1.1.1l 24 Aug 2021 -TLS SNI support enabled -``` - ### Python 3.6 on Apple M1 Mac Python3.6 isn't supported on Apple silicon, and it's not in homebrew. It can be installed with From eb028fe5610a429a9e4f66115cec83b2ea2bae54 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 7 Jan 2022 02:59:29 -0500 Subject: [PATCH 0736/1042] Filter out deleted category assignments --- .../list_tts_model_category_assignments.rs | 6 +- sqlx-data.json | 76 +++++++++---------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/crates/app/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs b/crates/app/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs index e1c4ee0077..f4aabcffb9 100644 --- a/crates/app/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs +++ b/crates/app/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs @@ -40,9 +40,7 @@ pub struct CategoryAssignment { pub category_token: String, } -// NB: This shows for all models, including deleted and unapproved. -// However, it does not return deleted/unapproved categories. -pub async fn list_tts_model_category_assignments(mysql_pool: &MySqlPool) -> AnyhowResult> { +async fn list_tts_model_category_assignments(mysql_pool: &MySqlPool) -> AnyhowResult> { let maybe_results = sqlx::query_as!( CategoryAssignment, r#" @@ -58,6 +56,8 @@ WHERE tts.is_locked_from_use IS FALSE AND tts.user_deleted_at IS NULL AND tts.mod_deleted_at IS NULL + AND assignments.deleted_at IS NULL + "#, ) .fetch_all(mysql_pool) diff --git a/sqlx-data.json b/sqlx-data.json index bfde92b74c..3034afff0e 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -337,44 +337,6 @@ ] } }, - "013e26fbe298b3de8403f9541807190fe556470c55b7ca342a50aaafea1bbb87": { - "query": "\nSELECT\n assignments.model_token AS tts_model_token,\n assignments.category_token\nFROM\n tts_category_assignments AS assignments\nJOIN\n tts_models AS tts\n ON assignments.model_token = tts.token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "category_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false - ] - } - }, "02060ab9b8ca472a2bd4d4fa3120743e90148ccd6a4cb13bc491d00f36ed7a8f": { "query": "\nSELECT\n id,\n status\nFROM w2l_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", "describe": { @@ -1854,6 +1816,44 @@ ] } }, + "12a176252df63d1e3a958362d754c828991d64ade427ab11a2025098d1581976": { + "query": "\nSELECT\n assignments.model_token AS tts_model_token,\n assignments.category_token\nFROM\n tts_category_assignments AS assignments\nJOIN\n tts_models AS tts\n ON assignments.model_token = tts.token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n AND assignments.deleted_at IS NULL\n\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "category_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false + ] + } + }, "150c98c00f4cd6954428736583eae913be578c4e69dac317d4445bc0e97743c9": { "query": "\nSELECT\n next_id\nFROM\n tts_result_synthetic_ids\nWHERE\n user_token = ?\nLIMIT 1\n ", "describe": { From 00036b0b5d21f1eeefab4a4c8a7cd6dc8db26109 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 7 Jan 2022 16:56:49 -0500 Subject: [PATCH 0737/1042] proper setup for local dev + optional ssl --- localdev/nginx-http-config/storyteller-ssl | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/localdev/nginx-http-config/storyteller-ssl b/localdev/nginx-http-config/storyteller-ssl index 62584740ae..65bef40313 100644 --- a/localdev/nginx-http-config/storyteller-ssl +++ b/localdev/nginx-http-config/storyteller-ssl @@ -50,12 +50,9 @@ server { # Javascript frontend over plain HTTP server { - listen 443 ssl; - listen [::]:443 ssl; - ssl_certificate /etc/ssl/certs/localhost.crt; - ssl_certificate_key /etc/ssl/private/localhost.key; - ssl_protocols TLSv1.2 TLSv1.1 TLSv1; - server_name www.jungle.horse; + listen 80; + # server_name www.jungle.horse; + server_name jungle.horse; location / { proxy_set_header Host $host; proxy_pass http://127.0.0.1:7000; From aad92d7295666c93f2acf84122b677a0152d0869 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 8 Jan 2022 02:43:32 -0500 Subject: [PATCH 0738/1042] CORS for storyteller.io --- crates/lib/http_server_common/src/cors.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/lib/http_server_common/src/cors.rs b/crates/lib/http_server_common/src/cors.rs index dffe077c62..ea7c6653f5 100644 --- a/crates/lib/http_server_common/src/cors.rs +++ b/crates/lib/http_server_common/src/cors.rs @@ -20,6 +20,7 @@ pub fn build_common_cors_config() -> Cors { .allowed_origin("http://localhost:8000") .allowed_origin("http://localhost:8080") .allowed_origin("http://mumble.stream") + .allowed_origin("http://storyteller.io") .allowed_origin("http://trumped.com") .allowed_origin("http://vo.codes") .allowed_origin("http://vocodes.com") @@ -29,6 +30,7 @@ pub fn build_common_cors_config() -> Cors { .allowed_origin("https://fakeyou.com") .allowed_origin("https://jungle.horse") .allowed_origin("https://mumble.stream") + .allowed_origin("https://storyteller.io") .allowed_origin("https://trumped.com") .allowed_origin("https://vo.codes") .allowed_origin("https://vocodes.com") From e9fd96cd4bcc10ead9d34e91947d4c7de1e75eed Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 9 Jan 2022 01:32:26 -0500 Subject: [PATCH 0739/1042] OBS --- .../obs_gateway/src/endpoints/oauth_begin_redirect.rs | 7 ------- crates/app/obs_gateway/src/endpoints/oauth_end.rs | 9 ++++++++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs b/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs index 9001d43f83..55cc60187b 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs @@ -66,13 +66,6 @@ pub async fn oauth_begin_enroll_redirect( let (url, _csrf_token) = builder.generate_url(); - let response = OauthBeginEnrollResult { - redirect_url: url.to_string(), - }; - - let response = to_json_success_response(&response) - .map_err(|_| OauthBeginEnrollError::ServerError)?; - Ok(HttpResponse::Found() .append_header((header::LOCATION, url.to_string())) .finish()) diff --git a/crates/app/obs_gateway/src/endpoints/oauth_end.rs b/crates/app/obs_gateway/src/endpoints/oauth_end.rs index 93167956a8..0e19146801 100644 --- a/crates/app/obs_gateway/src/endpoints/oauth_end.rs +++ b/crates/app/obs_gateway/src/endpoints/oauth_end.rs @@ -9,7 +9,7 @@ use log::warn; use std::fmt; use std::sync::Arc; use twitch_oauth2::tokens::BearerTokenType::UserToken; -use twitch_oauth2::CsrfToken; +use twitch_oauth2::{CsrfToken, TwitchToken}; #[derive(Deserialize)] pub struct QueryParams { @@ -127,9 +127,16 @@ pub async fn oauth_end_enroll_from_redirect( .as_ref() .map(|token| token.secret().to_string()); + let twitch_username = user_token.login.to_string(); + let never_expiring = user_token.never_expiring; + let expires_in = user_token.expires_in(); + info!("User id: {:?}", user_id); info!("Auth token: {:?}", auth_token); info!("Refresh token: {:?}", refresh_token); + info!("Twitch username: {:?}", twitch_username); + info!("Never expiring: {:?}", never_expiring); + info!("Expires in (don't store): {:?}", expires_in); // Should be ~4 hours Ok(HttpResponse::Ok() .body("TODO")) From cb5a135f68dbaca01e2c84cc8b1cfd86cb27e6ba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 9 Jan 2022 02:57:20 -0500 Subject: [PATCH 0740/1042] trying to figure out websocket stuff again --- .../obs_gateway_websocket_handler.rs | 24 ++++++++++--------- static/websocket_test.js | 15 ++++++++++-- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index a735ab214a..d89580dc81 100644 --- a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -20,25 +20,25 @@ pub async fn obs_gateway_websocket_handler( server_state: web::Data>, stream: web::Payload, ) -> Result { - error!(">>>>>> obs_ws_index"); + info!(">>>>>> obs_ws_index"); let mut client = PollingTwitchWebsocketClient::new().unwrap(); - error!("Connecting to Twitch PubSub..."); + info!("Connecting to Twitch PubSub..."); client.connect().await.unwrap(); - error!("Connected to Twitch PubSub"); + info!("Connected to Twitch PubSub"); //println!("Starting polling thread..."); //client.start_ping_thread().await; - error!("Sending Twitch PubSub PING..."); + info!("Sending Twitch PubSub PING..."); client.send_ping().await.unwrap(); - error!("Try read next from Twitch PubSub..."); + info!("Try read next from Twitch PubSub..."); let r = client.try_next().await.unwrap(); - error!("Twitch PubSub Result: {:?}", r); + info!("Twitch PubSub Result: {:?}", r); // User: vocodes //let user_id = 652567283; @@ -65,7 +65,7 @@ pub async fn obs_gateway_websocket_handler( let auth_token = server_state.twitch_oauth_temp.temp_oauth_access_token.clone(); - error!("Begin TwitchPubSub LISTEN on authenticated OAuth topics..."); + info!("Begin TwitchPubSub LISTEN on authenticated OAuth topics..."); client.listen(&auth_token, &topics).await.unwrap(); @@ -76,7 +76,7 @@ pub async fn obs_gateway_websocket_handler( //let res = rx.recv().unwrap(); - error!("Begin Javascript WebSocket..."); + info!("Begin Javascript WebSocket..."); let websocket = ObsGatewayWebSocket::new(client); ws::start(websocket, &request, stream) @@ -102,7 +102,7 @@ impl Actor for ObsGatewayWebSocket { type Context = ws::WebsocketContext; fn started(&mut self, _ctx: &mut Self::Context) { - error!(">>>>>> obs actor started"); + info!(">>>>>> obs actor started"); } } @@ -113,7 +113,7 @@ impl StreamHandler> for ObsGatewayWebSock ctx: &mut Self::Context, ) { if let Ok(msg) = msg { - error!(">>>>>> obs streamhandler::handle (msg = {:?})", msg); + info!(">>>>>> obs streamhandler::handle (msg = {:?})", msg); //let (tx, rx) = crossbeam::channel::bounded(1); //let handle = Handle::current(); @@ -148,11 +148,13 @@ impl StreamHandler> for ObsGatewayWebSock // } //} loop { - info!("Thread loop"); + info!("thread loop"); sleep(Duration::from_millis(1000)); } }); + info!("after thread spawn"); + match msg { ws::Message::Text(text) => ctx.text(text), diff --git a/static/websocket_test.js b/static/websocket_test.js index 98bb5940fd..c646feb927 100644 --- a/static/websocket_test.js +++ b/static/websocket_test.js @@ -8,16 +8,27 @@ sock.onopen = function (event) { console.log('on open event', event); - sock.send('on open message from client'); + sock.send('on open message from browser'); }; sock.onmessage = function (event) { console.log('on message event', event.data); } + sock.onerror = function(event) { + console.log('on error event', event.data); + } + setInterval(() => { - console.log('sending message to server on interval'); + console.log('sending browser message to server on interval'); sock.send('interval trigger'); }, 2000); + //setTimeout(() => { + // console.log('disconnect') + // const normal_close = 1000; + // sock.close(normal_close, 'close'); + // + //}, 10000) + })(); \ No newline at end of file From 6b03fb84c5ee8f1062450778d3370f4fae170cf0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 9 Jan 2022 21:14:15 -0500 Subject: [PATCH 0741/1042] OBS WIP --- .../obs_gateway_websocket_handler.rs | 38 ++++++++++--------- static/websocket_test.js | 2 +- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index d89580dc81..4839f1bb38 100644 --- a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -135,29 +135,33 @@ impl StreamHandler> for ObsGatewayWebSock //let res = rx.recv().unwrap(); - let handle = Handle::current(); + //let handle = Handle::current(); - handle.spawn(async move { - //error!("Twitch PubSub Try read next..."); - //match client2.try_next().await { - // Ok(r) => { - // error!("Twitch PubSub Result: {:?}", r); - // }, - // Err(e) => { - // warn!("pubsub error: {:?}", e); - // } - //} - loop { - info!("thread loop"); - sleep(Duration::from_millis(1000)); - } - }); + //handle.spawn(async move { + // //error!("Twitch PubSub Try read next..."); + // //match client2.try_next().await { + // // Ok(r) => { + // // error!("Twitch PubSub Result: {:?}", r); + // // }, + // // Err(e) => { + // // warn!("pubsub error: {:?}", e); + // // } + // //} + // loop { + // info!("thread loop"); + // sleep(Duration::from_millis(1000)); + // } + //}); info!("after thread spawn"); + info!("process message: {:?}", &msg); match msg { - ws::Message::Text(text) => ctx.text(text), + ws::Message::Text(text) => { + info!("sending text response"); + ctx.text("hello from Rust") + }, ws::Message::Binary(bin) => ctx.binary(bin), ws::Message::Ping(bytes) => ctx.pong(&bytes), ws::Message::Close(reason) => { diff --git a/static/websocket_test.js b/static/websocket_test.js index c646feb927..fac4d5a6e3 100644 --- a/static/websocket_test.js +++ b/static/websocket_test.js @@ -21,7 +21,7 @@ setInterval(() => { console.log('sending browser message to server on interval'); - sock.send('interval trigger'); + sock.send('interval trigger from browser'); }, 2000); //setTimeout(() => { From 8500c2d888e7685fd6c706a7f2ed2b6e22210d0d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 9 Jan 2022 21:32:03 -0500 Subject: [PATCH 0742/1042] this seems to work ? --- .../obs_gateway_websocket_handler.rs | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 4839f1bb38..66c7a2f10b 100644 --- a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -13,6 +13,7 @@ use tokio::runtime::Handle; use crate::endpoints_ws::obs_twitch_thread::ObsTwitchThread; use std::thread::sleep; use std::time::Duration; +use actix_rt::Runtime; /// Endpoint pub async fn obs_gateway_websocket_handler( @@ -103,6 +104,31 @@ impl Actor for ObsGatewayWebSocket { fn started(&mut self, _ctx: &mut Self::Context) { info!(">>>>>> obs actor started"); + + + info!("before spawn thread"); + + let handle = Handle::current(); + //let rt = Runtime::new().unwrap(); + + handle.spawn_blocking(|| { + //error!("Twitch PubSub Try read next..."); + //match client2.try_next().await { + // Ok(r) => { + // error!("Twitch PubSub Result: {:?}", r); + // }, + // Err(e) => { + // warn!("pubsub error: {:?}", e); + // } + //} + loop { + info!("thread loop"); + sleep(Duration::from_millis(1000)); + } + }); + + info!("after thread spawn"); + } } @@ -135,28 +161,9 @@ impl StreamHandler> for ObsGatewayWebSock //let res = rx.recv().unwrap(); - //let handle = Handle::current(); - - //handle.spawn(async move { - // //error!("Twitch PubSub Try read next..."); - // //match client2.try_next().await { - // // Ok(r) => { - // // error!("Twitch PubSub Result: {:?}", r); - // // }, - // // Err(e) => { - // // warn!("pubsub error: {:?}", e); - // // } - // //} - // loop { - // info!("thread loop"); - // sleep(Duration::from_millis(1000)); - // } - //}); - - info!("after thread spawn"); - info!("process message: {:?}", &msg); + match msg { ws::Message::Text(text) => { info!("sending text response"); From 380b5d62a311c752dbd8933cd0dd42f4221617ff Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 Jan 2022 01:52:44 -0500 Subject: [PATCH 0743/1042] sort shared library directory tree before app code --- Cargo.toml | 7 +++++-- crates/{app => service}/obs_gateway/Cargo.toml | 0 crates/{app => service}/obs_gateway/src/endpoints/mod.rs | 0 .../obs_gateway/src/endpoints/oauth_begin.rs | 0 .../obs_gateway/src/endpoints/oauth_begin_redirect.rs | 0 .../obs_gateway/src/endpoints/oauth_end.rs | 0 .../{app => service}/obs_gateway/src/endpoints_ws/mod.rs | 0 .../src/endpoints_ws/obs_gateway_websocket_handler.rs | 2 ++ .../obs_gateway/src/endpoints_ws/obs_twitch_thread.rs | 0 .../obs_gateway/src/endpoints_ws/pubsub_gateway.rs | 0 crates/{app => service}/obs_gateway/src/main.rs | 0 crates/{app => service}/obs_gateway/src/server_state.rs | 0 crates/{app => service}/obs_gateway/src/twitch/mod.rs | 0 .../obs_gateway/src/twitch/oauth_token_builder.rs | 0 .../obs_gateway/src/twitch/twitch_client_wrapper.rs | 0 .../obs_gateway/src/twitch/twitch_secrets.rs | 0 .../obs_gateway/src/twitch/websocket_client.rs | 0 crates/{app => service}/obs_gateway/src/util/mod.rs | 0 .../{app => service}/obs_gateway/src/util/random_nonce.rs | 0 crates/{app => service}/storyteller_web/Cargo.toml | 0 crates/{app => service}/storyteller_web/src/common_env.rs | 0 .../storyteller_web/src/database/enums/mod.rs | 0 .../src/database/enums/record_visibility.rs | 0 .../storyteller_web/src/database/enums/tts_model_type.rs | 0 .../storyteller_web/src/database/enums/vocoder_type.rs | 0 .../src/database/helpers/boolean_converters.rs | 0 .../storyteller_web/src/database/helpers/enums.rs | 0 .../storyteller_web/src/database/helpers/mod.rs | 0 .../storyteller_web/src/database/helpers/tokens.rs | 0 .../src/database/mediators/badge_granter.rs | 0 .../src/database/mediators/firehose_publisher.rs | 0 .../storyteller_web/src/database/mediators/mod.rs | 0 .../{app => service}/storyteller_web/src/database/mod.rs | 0 .../storyteller_web/src/database/queries/badges.rs | 0 .../database/queries/calculate_tts_model_leaderboard.rs | 0 .../database/queries/calculate_w2l_template_leaderboard.rs | 0 .../database/queries/categories/get_category_by_token.rs | 0 .../categories/list_tts_model_category_assignments.rs | 0 .../storyteller_web/src/database/queries/categories/mod.rs | 0 .../storyteller_web/src/database/queries/create_session.rs | 0 .../src/database/queries/delete_tts_model.rs | 0 .../src/database/queries/delete_tts_result.rs | 0 .../src/database/queries/delete_w2l_result.rs | 0 .../src/database/queries/delete_w2l_template.rs | 0 .../src/database/queries/get_user_profile_by_username.rs | 0 .../storyteller_web/src/database/queries/list_ip_bans.rs | 0 .../src/database/queries/list_tts_models.rs | 0 .../src/database/queries/list_user_badges.rs | 0 .../src/database/queries/list_user_roles.rs | 0 .../src/database/queries/list_w2l_templates.rs | 0 .../storyteller_web/src/database/queries/mod.rs | 0 .../src/database/queries/query_tts_model.rs | 0 .../src/database/queries/query_tts_result.rs | 0 .../src/database/queries/query_w2l_result.rs | 0 .../src/database/queries/query_w2l_template.rs | 0 .../list_assigned_tts_categories_query_builder.rs | 0 .../query_builders/list_categories_query_builder.rs | 0 .../list_tts_inference_results_query_builder.rs | 0 .../database/query_builders/list_users_query_builder.rs | 0 .../list_w2l_inference_results_query_builder.rs | 0 .../storyteller_web/src/database/query_builders/mod.rs | 0 .../storyteller_web/src/http_clients/mod.rs | 0 .../src/http_clients/tts_inference_sidecar_client.rs | 0 .../endpoints/categories/assign_tts_category.rs | 0 .../http_server/endpoints/categories/create_category.rs | 0 .../src/http_server/endpoints/categories/get_category.rs | 0 .../endpoints/categories/list_tts_categories.rs | 0 .../categories/list_tts_model_assigned_categories.rs | 0 .../src/http_server/endpoints/categories/mod.rs | 0 .../src/http_server/endpoints/events/list_events.rs | 0 .../src/http_server/endpoints/events/mod.rs | 0 .../http_server/endpoints/leaderboard/get_leaderboard.rs | 0 .../src/http_server/endpoints/leaderboard/mod.rs | 0 .../src/http_server/endpoints/misc/alpha_cookie.rs | 0 .../src/http_server/endpoints/misc/default_route_404.rs | 0 .../endpoints/misc/enable_alpha_easy_handler.rs | 0 .../src/http_server/endpoints/misc/enable_alpha_handler.rs | 0 .../storyteller_web/src/http_server/endpoints/misc/mod.rs | 0 .../src/http_server/endpoints/misc/root_index.rs | 0 .../storyteller_web/src/http_server/endpoints/mod.rs | 0 .../src/http_server/endpoints/moderation/approval/mod.rs | 0 .../endpoints/moderation/approval/pending_w2l_templates.rs | 0 .../endpoints/moderation/categories/delete_category.rs | 0 .../endpoints/moderation/categories/edit_category.rs | 0 .../categories/list_tts_categories_for_moderation.rs | 0 .../src/http_server/endpoints/moderation/categories/mod.rs | 0 .../http_server/endpoints/moderation/ip_bans/add_ip_ban.rs | 0 .../endpoints/moderation/ip_bans/delete_ip_ban.rs | 0 .../http_server/endpoints/moderation/ip_bans/get_ip_ban.rs | 0 .../endpoints/moderation/ip_bans/list_ip_bans.rs | 0 .../src/http_server/endpoints/moderation/ip_bans/mod.rs | 0 .../moderation/jobs/get_tts_inference_queue_count.rs | 0 .../moderation/jobs/get_w2l_inference_queue_count.rs | 0 .../src/http_server/endpoints/moderation/jobs/mod.rs | 0 .../src/http_server/endpoints/moderation/mod.rs | 0 .../endpoints/moderation/stats/get_voice_count_stats.rs | 0 .../src/http_server/endpoints/moderation/stats/mod.rs | 0 .../http_server/endpoints/moderation/user_bans/ban_user.rs | 0 .../endpoints/moderation/user_bans/list_banned_users.rs | 0 .../src/http_server/endpoints/moderation/user_bans/mod.rs | 0 .../endpoints/moderation/user_roles/list_roles.rs | 0 .../endpoints/moderation/user_roles/list_staff.rs | 0 .../src/http_server/endpoints/moderation/user_roles/mod.rs | 0 .../endpoints/moderation/user_roles/set_user_role.rs | 0 .../http_server/endpoints/moderation/users/list_users.rs | 0 .../src/http_server/endpoints/moderation/users/mod.rs | 0 .../src/http_server/endpoints/tts/delete_tts_model.rs | 0 .../src/http_server/endpoints/tts/delete_tts_result.rs | 0 .../src/http_server/endpoints/tts/edit_tts_model.rs | 0 .../src/http_server/endpoints/tts/edit_tts_result.rs | 0 .../src/http_server/endpoints/tts/enqueue_infer_tts.rs | 0 .../http_server/endpoints/tts/enqueue_upload_tts_model.rs | 0 .../endpoints/tts/get_tts_inference_job_status.rs | 0 .../src/http_server/endpoints/tts/get_tts_model.rs | 0 .../http_server/endpoints/tts/get_tts_model_use_count.rs | 0 .../src/http_server/endpoints/tts/get_tts_result.rs | 0 .../endpoints/tts/get_tts_upload_model_job_status.rs | 0 .../src/http_server/endpoints/tts/list_tts_models.rs | 0 .../storyteller_web/src/http_server/endpoints/tts/mod.rs | 0 .../src/http_server/endpoints/users/create_account.rs | 0 .../src/http_server/endpoints/users/edit_profile.rs | 0 .../src/http_server/endpoints/users/get_profile.rs | 0 .../endpoints/users/list_user_tts_inference_results.rs | 0 .../http_server/endpoints/users/list_user_tts_models.rs | 0 .../endpoints/users/list_user_w2l_inference_results.rs | 0 .../http_server/endpoints/users/list_user_w2l_templates.rs | 0 .../src/http_server/endpoints/users/login.rs | 0 .../src/http_server/endpoints/users/logout.rs | 0 .../storyteller_web/src/http_server/endpoints/users/mod.rs | 0 .../src/http_server/endpoints/users/session_info.rs | 0 .../src/http_server/endpoints/w2l/delete_w2l_result.rs | 0 .../src/http_server/endpoints/w2l/delete_w2l_template.rs | 0 .../src/http_server/endpoints/w2l/edit_w2l_result.rs | 0 .../src/http_server/endpoints/w2l/edit_w2l_template.rs | 0 .../src/http_server/endpoints/w2l/enqueue_infer_w2l.rs | 0 .../endpoints/w2l/enqueue_infer_w2l_with_uploads.rs | 0 .../endpoints/w2l/enqueue_upload_w2l_template.rs | 0 .../endpoints/w2l/get_w2l_inference_job_status.rs | 0 .../src/http_server/endpoints/w2l/get_w2l_result.rs | 0 .../src/http_server/endpoints/w2l/get_w2l_template.rs | 0 .../endpoints/w2l/get_w2l_template_use_count.rs | 0 .../endpoints/w2l/get_w2l_upload_template_job_status.rs | 0 .../src/http_server/endpoints/w2l/list_w2l_templates.rs | 0 .../storyteller_web/src/http_server/endpoints/w2l/mod.rs | 0 .../endpoints/w2l/set_w2l_template_mod_approval.rs | 0 .../src/http_server/middleware/ip_filter_middleware.rs | 0 .../storyteller_web/src/http_server/middleware/mod.rs | 0 .../storyteller_web/src/http_server/mod.rs | 0 .../src/http_server/web_utils/cookie_manager.rs | 0 .../src/http_server/web_utils/get_host_header.rs | 0 .../src/http_server/web_utils/ip_address.rs | 0 .../storyteller_web/src/http_server/web_utils/mod.rs | 0 .../http_server/web_utils/read_multipart_field_bytes.rs | 0 .../src/http_server/web_utils/redis_rate_limiter.rs | 0 .../src/http_server/web_utils/response_error_helpers.rs | 0 .../src/http_server/web_utils/response_success_helpers.rs | 0 .../src/http_server/web_utils/serialize_as_json_error.rs | 0 .../src/http_server/web_utils/session_checker.rs | 0 .../storyteller_web/src/job_queries/mod.rs | 0 .../src/job_queries/tts_download_job_queries.rs | 0 .../src/job_queries/tts_inference_job_queries.rs | 0 .../src/job_queries/w2l_download_job_queries.rs | 0 .../src/job_queries/w2l_inference_job_queries.rs | 0 crates/{app => service}/storyteller_web/src/main.rs | 0 crates/{app => service}/storyteller_web/src/routes.rs | 0 .../ffmpeg_generate_preview_image_command.rs | 0 .../ffmpeg_generate_preview_video_command.rs | 0 .../src/script_execution/google_drive_download_command.rs | 0 .../imagemagick_generate_preview_image_command.rs | 0 .../storyteller_web/src/script_execution/mod.rs | 0 .../src/script_execution/tacotron_inference_command.rs | 0 .../src/script_execution/tacotron_model_check_command.rs | 0 .../src/script_execution/talknet_model_check_command.rs | 0 .../src/script_execution/wav2lip_inference_command.rs | 0 .../src/script_execution/wav2lip_process_upload_command.rs | 0 .../{app => service}/storyteller_web/src/server_state.rs | 0 .../storyteller_web/src/threads/ip_banlist_set.rs | 0 crates/{app => service}/storyteller_web/src/threads/mod.rs | 0 .../storyteller_web/src/threads/poll_ip_banlist_thread.rs | 0 .../storyteller_web/src/tts_download_job.rs | 0 .../storyteller_web/src/tts_inference_job.rs | 0 .../storyteller_web/src/util/anyhow_result.rs | 0 .../storyteller_web/src/util/buckets/bucket_client.rs | 0 .../src/util/buckets/bucket_path_unifier.rs | 0 .../storyteller_web/src/util/buckets/bucket_paths.rs | 0 .../storyteller_web/src/util/buckets/mod.rs | 0 .../storyteller_web/src/util/caching/mod.rs | 0 .../src/util/caching/single_item_ttl_cache.rs | 0 .../storyteller_web/src/util/delete_role_disambiguation.rs | 0 .../storyteller_web/src/util/email_to_gravatar.rs | 0 .../storyteller_web/src/util/encrypted_sort_id.rs | 0 .../storyteller_web/src/util/filesystem.rs | 0 .../storyteller_web/src/util/hashing/hash_file_sha2.rs | 0 .../storyteller_web/src/util/hashing/hash_string_sha2.rs | 0 .../storyteller_web/src/util/hashing/mod.rs | 0 .../storyteller_web/src/util/hashing/sha256_digest.rs | 0 .../src/util/jobs/cache_miss_strategizer.rs | 0 .../src/util/jobs/cache_miss_strategizer_multi.rs | 0 .../{app => service}/storyteller_web/src/util/jobs/mod.rs | 0 .../storyteller_web/src/util/jobs/virtual_lfu_cache.rs | 0 .../storyteller_web/src/util/markdown_to_html.rs | 0 crates/{app => service}/storyteller_web/src/util/mod.rs | 0 .../storyteller_web/src/util/noop_logger.rs | 0 .../storyteller_web/src/util/random_crockford_token.rs | 0 .../src/util/random_prefix_crockford_token.rs | 0 .../storyteller_web/src/util/random_uuid.rs | 0 .../{app => service}/storyteller_web/src/util/redis/mod.rs | 0 .../src/util/redis/redis_job_status_logger.rs | 0 .../storyteller_web/src/util/redis/redis_keys.rs | 0 .../storyteller_web/src/util/semi_persistent_cache_dir.rs | 0 .../storyteller_web/src/validations/cashapp_username.rs | 0 .../storyteller_web/src/validations/check_for_slurs.rs | 0 .../storyteller_web/src/validations/discord_username.rs | 0 .../storyteller_web/src/validations/github_username.rs | 0 .../storyteller_web/src/validations/ip_addresses.rs | 0 .../storyteller_web/src/validations/mod.rs | 0 .../storyteller_web/src/validations/model_uploads.rs | 0 .../storyteller_web/src/validations/passwords.rs | 0 .../storyteller_web/src/validations/twitch_username.rs | 0 .../storyteller_web/src/validations/twitter_username.rs | 0 .../storyteller_web/src/validations/username.rs | 0 .../src/validations/username_reservations.rs | 0 .../storyteller_web/src/validations/website_url.rs | 0 .../storyteller_web/src/w2l_download_job.rs | 0 .../storyteller_web/src/w2l_inference_job.rs | 0 225 files changed, 7 insertions(+), 2 deletions(-) rename crates/{app => service}/obs_gateway/Cargo.toml (100%) rename crates/{app => service}/obs_gateway/src/endpoints/mod.rs (100%) rename crates/{app => service}/obs_gateway/src/endpoints/oauth_begin.rs (100%) rename crates/{app => service}/obs_gateway/src/endpoints/oauth_begin_redirect.rs (100%) rename crates/{app => service}/obs_gateway/src/endpoints/oauth_end.rs (100%) rename crates/{app => service}/obs_gateway/src/endpoints_ws/mod.rs (100%) rename crates/{app => service}/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs (99%) rename crates/{app => service}/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs (100%) rename crates/{app => service}/obs_gateway/src/endpoints_ws/pubsub_gateway.rs (100%) rename crates/{app => service}/obs_gateway/src/main.rs (100%) rename crates/{app => service}/obs_gateway/src/server_state.rs (100%) rename crates/{app => service}/obs_gateway/src/twitch/mod.rs (100%) rename crates/{app => service}/obs_gateway/src/twitch/oauth_token_builder.rs (100%) rename crates/{app => service}/obs_gateway/src/twitch/twitch_client_wrapper.rs (100%) rename crates/{app => service}/obs_gateway/src/twitch/twitch_secrets.rs (100%) rename crates/{app => service}/obs_gateway/src/twitch/websocket_client.rs (100%) rename crates/{app => service}/obs_gateway/src/util/mod.rs (100%) rename crates/{app => service}/obs_gateway/src/util/random_nonce.rs (100%) rename crates/{app => service}/storyteller_web/Cargo.toml (100%) rename crates/{app => service}/storyteller_web/src/common_env.rs (100%) rename crates/{app => service}/storyteller_web/src/database/enums/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/database/enums/record_visibility.rs (100%) rename crates/{app => service}/storyteller_web/src/database/enums/tts_model_type.rs (100%) rename crates/{app => service}/storyteller_web/src/database/enums/vocoder_type.rs (100%) rename crates/{app => service}/storyteller_web/src/database/helpers/boolean_converters.rs (100%) rename crates/{app => service}/storyteller_web/src/database/helpers/enums.rs (100%) rename crates/{app => service}/storyteller_web/src/database/helpers/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/database/helpers/tokens.rs (100%) rename crates/{app => service}/storyteller_web/src/database/mediators/badge_granter.rs (100%) rename crates/{app => service}/storyteller_web/src/database/mediators/firehose_publisher.rs (100%) rename crates/{app => service}/storyteller_web/src/database/mediators/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/database/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/badges.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/categories/get_category_by_token.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/categories/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/create_session.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/delete_tts_model.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/delete_tts_result.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/delete_w2l_result.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/delete_w2l_template.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/get_user_profile_by_username.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/list_ip_bans.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/list_tts_models.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/list_user_badges.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/list_user_roles.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/list_w2l_templates.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/query_tts_model.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/query_tts_result.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/query_w2l_result.rs (100%) rename crates/{app => service}/storyteller_web/src/database/queries/query_w2l_template.rs (100%) rename crates/{app => service}/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs (100%) rename crates/{app => service}/storyteller_web/src/database/query_builders/list_categories_query_builder.rs (100%) rename crates/{app => service}/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs (100%) rename crates/{app => service}/storyteller_web/src/database/query_builders/list_users_query_builder.rs (100%) rename crates/{app => service}/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs (100%) rename crates/{app => service}/storyteller_web/src/database/query_builders/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_clients/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/categories/create_category.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/categories/get_category.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/categories/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/events/list_events.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/events/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/misc/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/misc/root_index.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/tts/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/users/create_account.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/users/edit_profile.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/users/get_profile.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/users/login.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/users/logout.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/users/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/users/session_info.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/middleware/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/web_utils/cookie_manager.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/web_utils/get_host_header.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/web_utils/ip_address.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/web_utils/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/web_utils/response_error_helpers.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/web_utils/response_success_helpers.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs (100%) rename crates/{app => service}/storyteller_web/src/http_server/web_utils/session_checker.rs (100%) rename crates/{app => service}/storyteller_web/src/job_queries/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/job_queries/tts_download_job_queries.rs (100%) rename crates/{app => service}/storyteller_web/src/job_queries/tts_inference_job_queries.rs (100%) rename crates/{app => service}/storyteller_web/src/job_queries/w2l_download_job_queries.rs (100%) rename crates/{app => service}/storyteller_web/src/job_queries/w2l_inference_job_queries.rs (100%) rename crates/{app => service}/storyteller_web/src/main.rs (100%) rename crates/{app => service}/storyteller_web/src/routes.rs (100%) rename crates/{app => service}/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs (100%) rename crates/{app => service}/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs (100%) rename crates/{app => service}/storyteller_web/src/script_execution/google_drive_download_command.rs (100%) rename crates/{app => service}/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs (100%) rename crates/{app => service}/storyteller_web/src/script_execution/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/script_execution/tacotron_inference_command.rs (100%) rename crates/{app => service}/storyteller_web/src/script_execution/tacotron_model_check_command.rs (100%) rename crates/{app => service}/storyteller_web/src/script_execution/talknet_model_check_command.rs (100%) rename crates/{app => service}/storyteller_web/src/script_execution/wav2lip_inference_command.rs (100%) rename crates/{app => service}/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs (100%) rename crates/{app => service}/storyteller_web/src/server_state.rs (100%) rename crates/{app => service}/storyteller_web/src/threads/ip_banlist_set.rs (100%) rename crates/{app => service}/storyteller_web/src/threads/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/threads/poll_ip_banlist_thread.rs (100%) rename crates/{app => service}/storyteller_web/src/tts_download_job.rs (100%) rename crates/{app => service}/storyteller_web/src/tts_inference_job.rs (100%) rename crates/{app => service}/storyteller_web/src/util/anyhow_result.rs (100%) rename crates/{app => service}/storyteller_web/src/util/buckets/bucket_client.rs (100%) rename crates/{app => service}/storyteller_web/src/util/buckets/bucket_path_unifier.rs (100%) rename crates/{app => service}/storyteller_web/src/util/buckets/bucket_paths.rs (100%) rename crates/{app => service}/storyteller_web/src/util/buckets/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/util/caching/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/util/caching/single_item_ttl_cache.rs (100%) rename crates/{app => service}/storyteller_web/src/util/delete_role_disambiguation.rs (100%) rename crates/{app => service}/storyteller_web/src/util/email_to_gravatar.rs (100%) rename crates/{app => service}/storyteller_web/src/util/encrypted_sort_id.rs (100%) rename crates/{app => service}/storyteller_web/src/util/filesystem.rs (100%) rename crates/{app => service}/storyteller_web/src/util/hashing/hash_file_sha2.rs (100%) rename crates/{app => service}/storyteller_web/src/util/hashing/hash_string_sha2.rs (100%) rename crates/{app => service}/storyteller_web/src/util/hashing/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/util/hashing/sha256_digest.rs (100%) rename crates/{app => service}/storyteller_web/src/util/jobs/cache_miss_strategizer.rs (100%) rename crates/{app => service}/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs (100%) rename crates/{app => service}/storyteller_web/src/util/jobs/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/util/jobs/virtual_lfu_cache.rs (100%) rename crates/{app => service}/storyteller_web/src/util/markdown_to_html.rs (100%) rename crates/{app => service}/storyteller_web/src/util/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/util/noop_logger.rs (100%) rename crates/{app => service}/storyteller_web/src/util/random_crockford_token.rs (100%) rename crates/{app => service}/storyteller_web/src/util/random_prefix_crockford_token.rs (100%) rename crates/{app => service}/storyteller_web/src/util/random_uuid.rs (100%) rename crates/{app => service}/storyteller_web/src/util/redis/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/util/redis/redis_job_status_logger.rs (100%) rename crates/{app => service}/storyteller_web/src/util/redis/redis_keys.rs (100%) rename crates/{app => service}/storyteller_web/src/util/semi_persistent_cache_dir.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/cashapp_username.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/check_for_slurs.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/discord_username.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/github_username.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/ip_addresses.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/mod.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/model_uploads.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/passwords.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/twitch_username.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/twitter_username.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/username.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/username_reservations.rs (100%) rename crates/{app => service}/storyteller_web/src/validations/website_url.rs (100%) rename crates/{app => service}/storyteller_web/src/w2l_download_job.rs (100%) rename crates/{app => service}/storyteller_web/src/w2l_inference_job.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index fef48a845c..16baf41ea6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,12 @@ [workspace] members = [ - "crates/app/obs_gateway", - "crates/app/storyteller_web", + # Shared libraries "crates/lib/config", "crates/lib/container_common", "crates/lib/http_server_common", + # Application + "crates/service/obs_gateway", + "crates/service/storyteller_web", ] + diff --git a/crates/app/obs_gateway/Cargo.toml b/crates/service/obs_gateway/Cargo.toml similarity index 100% rename from crates/app/obs_gateway/Cargo.toml rename to crates/service/obs_gateway/Cargo.toml diff --git a/crates/app/obs_gateway/src/endpoints/mod.rs b/crates/service/obs_gateway/src/endpoints/mod.rs similarity index 100% rename from crates/app/obs_gateway/src/endpoints/mod.rs rename to crates/service/obs_gateway/src/endpoints/mod.rs diff --git a/crates/app/obs_gateway/src/endpoints/oauth_begin.rs b/crates/service/obs_gateway/src/endpoints/oauth_begin.rs similarity index 100% rename from crates/app/obs_gateway/src/endpoints/oauth_begin.rs rename to crates/service/obs_gateway/src/endpoints/oauth_begin.rs diff --git a/crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs b/crates/service/obs_gateway/src/endpoints/oauth_begin_redirect.rs similarity index 100% rename from crates/app/obs_gateway/src/endpoints/oauth_begin_redirect.rs rename to crates/service/obs_gateway/src/endpoints/oauth_begin_redirect.rs diff --git a/crates/app/obs_gateway/src/endpoints/oauth_end.rs b/crates/service/obs_gateway/src/endpoints/oauth_end.rs similarity index 100% rename from crates/app/obs_gateway/src/endpoints/oauth_end.rs rename to crates/service/obs_gateway/src/endpoints/oauth_end.rs diff --git a/crates/app/obs_gateway/src/endpoints_ws/mod.rs b/crates/service/obs_gateway/src/endpoints_ws/mod.rs similarity index 100% rename from crates/app/obs_gateway/src/endpoints_ws/mod.rs rename to crates/service/obs_gateway/src/endpoints_ws/mod.rs diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs similarity index 99% rename from crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs rename to crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 66c7a2f10b..f76cf66d35 100644 --- a/crates/app/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -106,6 +106,8 @@ impl Actor for ObsGatewayWebSocket { info!(">>>>>> obs actor started"); + + info!("before spawn thread"); let handle = Handle::current(); diff --git a/crates/app/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs similarity index 100% rename from crates/app/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs rename to crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs diff --git a/crates/app/obs_gateway/src/endpoints_ws/pubsub_gateway.rs b/crates/service/obs_gateway/src/endpoints_ws/pubsub_gateway.rs similarity index 100% rename from crates/app/obs_gateway/src/endpoints_ws/pubsub_gateway.rs rename to crates/service/obs_gateway/src/endpoints_ws/pubsub_gateway.rs diff --git a/crates/app/obs_gateway/src/main.rs b/crates/service/obs_gateway/src/main.rs similarity index 100% rename from crates/app/obs_gateway/src/main.rs rename to crates/service/obs_gateway/src/main.rs diff --git a/crates/app/obs_gateway/src/server_state.rs b/crates/service/obs_gateway/src/server_state.rs similarity index 100% rename from crates/app/obs_gateway/src/server_state.rs rename to crates/service/obs_gateway/src/server_state.rs diff --git a/crates/app/obs_gateway/src/twitch/mod.rs b/crates/service/obs_gateway/src/twitch/mod.rs similarity index 100% rename from crates/app/obs_gateway/src/twitch/mod.rs rename to crates/service/obs_gateway/src/twitch/mod.rs diff --git a/crates/app/obs_gateway/src/twitch/oauth_token_builder.rs b/crates/service/obs_gateway/src/twitch/oauth_token_builder.rs similarity index 100% rename from crates/app/obs_gateway/src/twitch/oauth_token_builder.rs rename to crates/service/obs_gateway/src/twitch/oauth_token_builder.rs diff --git a/crates/app/obs_gateway/src/twitch/twitch_client_wrapper.rs b/crates/service/obs_gateway/src/twitch/twitch_client_wrapper.rs similarity index 100% rename from crates/app/obs_gateway/src/twitch/twitch_client_wrapper.rs rename to crates/service/obs_gateway/src/twitch/twitch_client_wrapper.rs diff --git a/crates/app/obs_gateway/src/twitch/twitch_secrets.rs b/crates/service/obs_gateway/src/twitch/twitch_secrets.rs similarity index 100% rename from crates/app/obs_gateway/src/twitch/twitch_secrets.rs rename to crates/service/obs_gateway/src/twitch/twitch_secrets.rs diff --git a/crates/app/obs_gateway/src/twitch/websocket_client.rs b/crates/service/obs_gateway/src/twitch/websocket_client.rs similarity index 100% rename from crates/app/obs_gateway/src/twitch/websocket_client.rs rename to crates/service/obs_gateway/src/twitch/websocket_client.rs diff --git a/crates/app/obs_gateway/src/util/mod.rs b/crates/service/obs_gateway/src/util/mod.rs similarity index 100% rename from crates/app/obs_gateway/src/util/mod.rs rename to crates/service/obs_gateway/src/util/mod.rs diff --git a/crates/app/obs_gateway/src/util/random_nonce.rs b/crates/service/obs_gateway/src/util/random_nonce.rs similarity index 100% rename from crates/app/obs_gateway/src/util/random_nonce.rs rename to crates/service/obs_gateway/src/util/random_nonce.rs diff --git a/crates/app/storyteller_web/Cargo.toml b/crates/service/storyteller_web/Cargo.toml similarity index 100% rename from crates/app/storyteller_web/Cargo.toml rename to crates/service/storyteller_web/Cargo.toml diff --git a/crates/app/storyteller_web/src/common_env.rs b/crates/service/storyteller_web/src/common_env.rs similarity index 100% rename from crates/app/storyteller_web/src/common_env.rs rename to crates/service/storyteller_web/src/common_env.rs diff --git a/crates/app/storyteller_web/src/database/enums/mod.rs b/crates/service/storyteller_web/src/database/enums/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/database/enums/mod.rs rename to crates/service/storyteller_web/src/database/enums/mod.rs diff --git a/crates/app/storyteller_web/src/database/enums/record_visibility.rs b/crates/service/storyteller_web/src/database/enums/record_visibility.rs similarity index 100% rename from crates/app/storyteller_web/src/database/enums/record_visibility.rs rename to crates/service/storyteller_web/src/database/enums/record_visibility.rs diff --git a/crates/app/storyteller_web/src/database/enums/tts_model_type.rs b/crates/service/storyteller_web/src/database/enums/tts_model_type.rs similarity index 100% rename from crates/app/storyteller_web/src/database/enums/tts_model_type.rs rename to crates/service/storyteller_web/src/database/enums/tts_model_type.rs diff --git a/crates/app/storyteller_web/src/database/enums/vocoder_type.rs b/crates/service/storyteller_web/src/database/enums/vocoder_type.rs similarity index 100% rename from crates/app/storyteller_web/src/database/enums/vocoder_type.rs rename to crates/service/storyteller_web/src/database/enums/vocoder_type.rs diff --git a/crates/app/storyteller_web/src/database/helpers/boolean_converters.rs b/crates/service/storyteller_web/src/database/helpers/boolean_converters.rs similarity index 100% rename from crates/app/storyteller_web/src/database/helpers/boolean_converters.rs rename to crates/service/storyteller_web/src/database/helpers/boolean_converters.rs diff --git a/crates/app/storyteller_web/src/database/helpers/enums.rs b/crates/service/storyteller_web/src/database/helpers/enums.rs similarity index 100% rename from crates/app/storyteller_web/src/database/helpers/enums.rs rename to crates/service/storyteller_web/src/database/helpers/enums.rs diff --git a/crates/app/storyteller_web/src/database/helpers/mod.rs b/crates/service/storyteller_web/src/database/helpers/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/database/helpers/mod.rs rename to crates/service/storyteller_web/src/database/helpers/mod.rs diff --git a/crates/app/storyteller_web/src/database/helpers/tokens.rs b/crates/service/storyteller_web/src/database/helpers/tokens.rs similarity index 100% rename from crates/app/storyteller_web/src/database/helpers/tokens.rs rename to crates/service/storyteller_web/src/database/helpers/tokens.rs diff --git a/crates/app/storyteller_web/src/database/mediators/badge_granter.rs b/crates/service/storyteller_web/src/database/mediators/badge_granter.rs similarity index 100% rename from crates/app/storyteller_web/src/database/mediators/badge_granter.rs rename to crates/service/storyteller_web/src/database/mediators/badge_granter.rs diff --git a/crates/app/storyteller_web/src/database/mediators/firehose_publisher.rs b/crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs similarity index 100% rename from crates/app/storyteller_web/src/database/mediators/firehose_publisher.rs rename to crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs diff --git a/crates/app/storyteller_web/src/database/mediators/mod.rs b/crates/service/storyteller_web/src/database/mediators/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/database/mediators/mod.rs rename to crates/service/storyteller_web/src/database/mediators/mod.rs diff --git a/crates/app/storyteller_web/src/database/mod.rs b/crates/service/storyteller_web/src/database/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/database/mod.rs rename to crates/service/storyteller_web/src/database/mod.rs diff --git a/crates/app/storyteller_web/src/database/queries/badges.rs b/crates/service/storyteller_web/src/database/queries/badges.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/badges.rs rename to crates/service/storyteller_web/src/database/queries/badges.rs diff --git a/crates/app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs b/crates/service/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs rename to crates/service/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs diff --git a/crates/app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs b/crates/service/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs rename to crates/service/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs diff --git a/crates/app/storyteller_web/src/database/queries/categories/get_category_by_token.rs b/crates/service/storyteller_web/src/database/queries/categories/get_category_by_token.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/categories/get_category_by_token.rs rename to crates/service/storyteller_web/src/database/queries/categories/get_category_by_token.rs diff --git a/crates/app/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs b/crates/service/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs rename to crates/service/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs diff --git a/crates/app/storyteller_web/src/database/queries/categories/mod.rs b/crates/service/storyteller_web/src/database/queries/categories/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/categories/mod.rs rename to crates/service/storyteller_web/src/database/queries/categories/mod.rs diff --git a/crates/app/storyteller_web/src/database/queries/create_session.rs b/crates/service/storyteller_web/src/database/queries/create_session.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/create_session.rs rename to crates/service/storyteller_web/src/database/queries/create_session.rs diff --git a/crates/app/storyteller_web/src/database/queries/delete_tts_model.rs b/crates/service/storyteller_web/src/database/queries/delete_tts_model.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/delete_tts_model.rs rename to crates/service/storyteller_web/src/database/queries/delete_tts_model.rs diff --git a/crates/app/storyteller_web/src/database/queries/delete_tts_result.rs b/crates/service/storyteller_web/src/database/queries/delete_tts_result.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/delete_tts_result.rs rename to crates/service/storyteller_web/src/database/queries/delete_tts_result.rs diff --git a/crates/app/storyteller_web/src/database/queries/delete_w2l_result.rs b/crates/service/storyteller_web/src/database/queries/delete_w2l_result.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/delete_w2l_result.rs rename to crates/service/storyteller_web/src/database/queries/delete_w2l_result.rs diff --git a/crates/app/storyteller_web/src/database/queries/delete_w2l_template.rs b/crates/service/storyteller_web/src/database/queries/delete_w2l_template.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/delete_w2l_template.rs rename to crates/service/storyteller_web/src/database/queries/delete_w2l_template.rs diff --git a/crates/app/storyteller_web/src/database/queries/get_user_profile_by_username.rs b/crates/service/storyteller_web/src/database/queries/get_user_profile_by_username.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/get_user_profile_by_username.rs rename to crates/service/storyteller_web/src/database/queries/get_user_profile_by_username.rs diff --git a/crates/app/storyteller_web/src/database/queries/list_ip_bans.rs b/crates/service/storyteller_web/src/database/queries/list_ip_bans.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/list_ip_bans.rs rename to crates/service/storyteller_web/src/database/queries/list_ip_bans.rs diff --git a/crates/app/storyteller_web/src/database/queries/list_tts_models.rs b/crates/service/storyteller_web/src/database/queries/list_tts_models.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/list_tts_models.rs rename to crates/service/storyteller_web/src/database/queries/list_tts_models.rs diff --git a/crates/app/storyteller_web/src/database/queries/list_user_badges.rs b/crates/service/storyteller_web/src/database/queries/list_user_badges.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/list_user_badges.rs rename to crates/service/storyteller_web/src/database/queries/list_user_badges.rs diff --git a/crates/app/storyteller_web/src/database/queries/list_user_roles.rs b/crates/service/storyteller_web/src/database/queries/list_user_roles.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/list_user_roles.rs rename to crates/service/storyteller_web/src/database/queries/list_user_roles.rs diff --git a/crates/app/storyteller_web/src/database/queries/list_w2l_templates.rs b/crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/list_w2l_templates.rs rename to crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs diff --git a/crates/app/storyteller_web/src/database/queries/mod.rs b/crates/service/storyteller_web/src/database/queries/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/mod.rs rename to crates/service/storyteller_web/src/database/queries/mod.rs diff --git a/crates/app/storyteller_web/src/database/queries/query_tts_model.rs b/crates/service/storyteller_web/src/database/queries/query_tts_model.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/query_tts_model.rs rename to crates/service/storyteller_web/src/database/queries/query_tts_model.rs diff --git a/crates/app/storyteller_web/src/database/queries/query_tts_result.rs b/crates/service/storyteller_web/src/database/queries/query_tts_result.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/query_tts_result.rs rename to crates/service/storyteller_web/src/database/queries/query_tts_result.rs diff --git a/crates/app/storyteller_web/src/database/queries/query_w2l_result.rs b/crates/service/storyteller_web/src/database/queries/query_w2l_result.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/query_w2l_result.rs rename to crates/service/storyteller_web/src/database/queries/query_w2l_result.rs diff --git a/crates/app/storyteller_web/src/database/queries/query_w2l_template.rs b/crates/service/storyteller_web/src/database/queries/query_w2l_template.rs similarity index 100% rename from crates/app/storyteller_web/src/database/queries/query_w2l_template.rs rename to crates/service/storyteller_web/src/database/queries/query_w2l_template.rs diff --git a/crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs similarity index 100% rename from crates/app/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs rename to crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs diff --git a/crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs similarity index 100% rename from crates/app/storyteller_web/src/database/query_builders/list_categories_query_builder.rs rename to crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs diff --git a/crates/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs similarity index 100% rename from crates/app/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs rename to crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs diff --git a/crates/app/storyteller_web/src/database/query_builders/list_users_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs similarity index 100% rename from crates/app/storyteller_web/src/database/query_builders/list_users_query_builder.rs rename to crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs diff --git a/crates/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs similarity index 100% rename from crates/app/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs rename to crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs diff --git a/crates/app/storyteller_web/src/database/query_builders/mod.rs b/crates/service/storyteller_web/src/database/query_builders/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/database/query_builders/mod.rs rename to crates/service/storyteller_web/src/database/query_builders/mod.rs diff --git a/crates/app/storyteller_web/src/http_clients/mod.rs b/crates/service/storyteller_web/src/http_clients/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_clients/mod.rs rename to crates/service/storyteller_web/src/http_clients/mod.rs diff --git a/crates/app/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs b/crates/service/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs similarity index 100% rename from crates/app/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs rename to crates/service/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs rename to crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/categories/create_category.rs rename to crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/get_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/categories/get_category.rs rename to crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs rename to crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs rename to crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/categories/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/categories/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/events/list_events.rs b/crates/service/storyteller_web/src/http_server/endpoints/events/list_events.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/events/list_events.rs rename to crates/service/storyteller_web/src/http_server/endpoints/events/list_events.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/events/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/events/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/events/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/events/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs b/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs rename to crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/leaderboard/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs rename to crates/service/storyteller_web/src/http_server/endpoints/misc/alpha_cookie.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs rename to crates/service/storyteller_web/src/http_server/endpoints/misc/default_route_404.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs rename to crates/service/storyteller_web/src/http_server/endpoints/misc/enable_alpha_easy_handler.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs rename to crates/service/storyteller_web/src/http_server/endpoints/misc/enable_alpha_handler.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/misc/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/misc/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/misc/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/misc/root_index.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/root_index.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/misc/root_index.rs rename to crates/service/storyteller_web/src/http_server/endpoints/misc/root_index.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/approval/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/approval/pending_w2l_templates.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/get_w2l_inference_queue_count.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/stats/get_voice_count_stats.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/stats/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/moderation/users/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model_use_count.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/tts/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/tts/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/tts/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/create_account.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/users/create_account.rs rename to crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/users/edit_profile.rs rename to crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/users/get_profile.rs rename to crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs rename to crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs rename to crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs rename to crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs rename to crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/login.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/users/login.rs rename to crates/service/storyteller_web/src/http_server/endpoints/users/login.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/logout.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/logout.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/users/logout.rs rename to crates/service/storyteller_web/src/http_server/endpoints/users/logout.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/users/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/users/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/users/session_info.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/session_info.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/users/session_info.rs rename to crates/service/storyteller_web/src/http_server/endpoints/users/session_info.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template_use_count.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs rename to crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs diff --git a/crates/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs b/crates/service/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs rename to crates/service/storyteller_web/src/http_server/middleware/ip_filter_middleware.rs diff --git a/crates/app/storyteller_web/src/http_server/middleware/mod.rs b/crates/service/storyteller_web/src/http_server/middleware/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/middleware/mod.rs rename to crates/service/storyteller_web/src/http_server/middleware/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/mod.rs b/crates/service/storyteller_web/src/http_server/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/mod.rs rename to crates/service/storyteller_web/src/http_server/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/web_utils/cookie_manager.rs b/crates/service/storyteller_web/src/http_server/web_utils/cookie_manager.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/web_utils/cookie_manager.rs rename to crates/service/storyteller_web/src/http_server/web_utils/cookie_manager.rs diff --git a/crates/app/storyteller_web/src/http_server/web_utils/get_host_header.rs b/crates/service/storyteller_web/src/http_server/web_utils/get_host_header.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/web_utils/get_host_header.rs rename to crates/service/storyteller_web/src/http_server/web_utils/get_host_header.rs diff --git a/crates/app/storyteller_web/src/http_server/web_utils/ip_address.rs b/crates/service/storyteller_web/src/http_server/web_utils/ip_address.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/web_utils/ip_address.rs rename to crates/service/storyteller_web/src/http_server/web_utils/ip_address.rs diff --git a/crates/app/storyteller_web/src/http_server/web_utils/mod.rs b/crates/service/storyteller_web/src/http_server/web_utils/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/web_utils/mod.rs rename to crates/service/storyteller_web/src/http_server/web_utils/mod.rs diff --git a/crates/app/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs b/crates/service/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs rename to crates/service/storyteller_web/src/http_server/web_utils/read_multipart_field_bytes.rs diff --git a/crates/app/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs b/crates/service/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs rename to crates/service/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs diff --git a/crates/app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs b/crates/service/storyteller_web/src/http_server/web_utils/response_error_helpers.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/web_utils/response_error_helpers.rs rename to crates/service/storyteller_web/src/http_server/web_utils/response_error_helpers.rs diff --git a/crates/app/storyteller_web/src/http_server/web_utils/response_success_helpers.rs b/crates/service/storyteller_web/src/http_server/web_utils/response_success_helpers.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/web_utils/response_success_helpers.rs rename to crates/service/storyteller_web/src/http_server/web_utils/response_success_helpers.rs diff --git a/crates/app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs b/crates/service/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs rename to crates/service/storyteller_web/src/http_server/web_utils/serialize_as_json_error.rs diff --git a/crates/app/storyteller_web/src/http_server/web_utils/session_checker.rs b/crates/service/storyteller_web/src/http_server/web_utils/session_checker.rs similarity index 100% rename from crates/app/storyteller_web/src/http_server/web_utils/session_checker.rs rename to crates/service/storyteller_web/src/http_server/web_utils/session_checker.rs diff --git a/crates/app/storyteller_web/src/job_queries/mod.rs b/crates/service/storyteller_web/src/job_queries/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/job_queries/mod.rs rename to crates/service/storyteller_web/src/job_queries/mod.rs diff --git a/crates/app/storyteller_web/src/job_queries/tts_download_job_queries.rs b/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs similarity index 100% rename from crates/app/storyteller_web/src/job_queries/tts_download_job_queries.rs rename to crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs diff --git a/crates/app/storyteller_web/src/job_queries/tts_inference_job_queries.rs b/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs similarity index 100% rename from crates/app/storyteller_web/src/job_queries/tts_inference_job_queries.rs rename to crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs diff --git a/crates/app/storyteller_web/src/job_queries/w2l_download_job_queries.rs b/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs similarity index 100% rename from crates/app/storyteller_web/src/job_queries/w2l_download_job_queries.rs rename to crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs diff --git a/crates/app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs b/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs similarity index 100% rename from crates/app/storyteller_web/src/job_queries/w2l_inference_job_queries.rs rename to crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs diff --git a/crates/app/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs similarity index 100% rename from crates/app/storyteller_web/src/main.rs rename to crates/service/storyteller_web/src/main.rs diff --git a/crates/app/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs similarity index 100% rename from crates/app/storyteller_web/src/routes.rs rename to crates/service/storyteller_web/src/routes.rs diff --git a/crates/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs b/crates/service/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs similarity index 100% rename from crates/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs rename to crates/service/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs diff --git a/crates/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs b/crates/service/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs similarity index 100% rename from crates/app/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs rename to crates/service/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs diff --git a/crates/app/storyteller_web/src/script_execution/google_drive_download_command.rs b/crates/service/storyteller_web/src/script_execution/google_drive_download_command.rs similarity index 100% rename from crates/app/storyteller_web/src/script_execution/google_drive_download_command.rs rename to crates/service/storyteller_web/src/script_execution/google_drive_download_command.rs diff --git a/crates/app/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs b/crates/service/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs similarity index 100% rename from crates/app/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs rename to crates/service/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs diff --git a/crates/app/storyteller_web/src/script_execution/mod.rs b/crates/service/storyteller_web/src/script_execution/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/script_execution/mod.rs rename to crates/service/storyteller_web/src/script_execution/mod.rs diff --git a/crates/app/storyteller_web/src/script_execution/tacotron_inference_command.rs b/crates/service/storyteller_web/src/script_execution/tacotron_inference_command.rs similarity index 100% rename from crates/app/storyteller_web/src/script_execution/tacotron_inference_command.rs rename to crates/service/storyteller_web/src/script_execution/tacotron_inference_command.rs diff --git a/crates/app/storyteller_web/src/script_execution/tacotron_model_check_command.rs b/crates/service/storyteller_web/src/script_execution/tacotron_model_check_command.rs similarity index 100% rename from crates/app/storyteller_web/src/script_execution/tacotron_model_check_command.rs rename to crates/service/storyteller_web/src/script_execution/tacotron_model_check_command.rs diff --git a/crates/app/storyteller_web/src/script_execution/talknet_model_check_command.rs b/crates/service/storyteller_web/src/script_execution/talknet_model_check_command.rs similarity index 100% rename from crates/app/storyteller_web/src/script_execution/talknet_model_check_command.rs rename to crates/service/storyteller_web/src/script_execution/talknet_model_check_command.rs diff --git a/crates/app/storyteller_web/src/script_execution/wav2lip_inference_command.rs b/crates/service/storyteller_web/src/script_execution/wav2lip_inference_command.rs similarity index 100% rename from crates/app/storyteller_web/src/script_execution/wav2lip_inference_command.rs rename to crates/service/storyteller_web/src/script_execution/wav2lip_inference_command.rs diff --git a/crates/app/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs b/crates/service/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs similarity index 100% rename from crates/app/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs rename to crates/service/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs diff --git a/crates/app/storyteller_web/src/server_state.rs b/crates/service/storyteller_web/src/server_state.rs similarity index 100% rename from crates/app/storyteller_web/src/server_state.rs rename to crates/service/storyteller_web/src/server_state.rs diff --git a/crates/app/storyteller_web/src/threads/ip_banlist_set.rs b/crates/service/storyteller_web/src/threads/ip_banlist_set.rs similarity index 100% rename from crates/app/storyteller_web/src/threads/ip_banlist_set.rs rename to crates/service/storyteller_web/src/threads/ip_banlist_set.rs diff --git a/crates/app/storyteller_web/src/threads/mod.rs b/crates/service/storyteller_web/src/threads/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/threads/mod.rs rename to crates/service/storyteller_web/src/threads/mod.rs diff --git a/crates/app/storyteller_web/src/threads/poll_ip_banlist_thread.rs b/crates/service/storyteller_web/src/threads/poll_ip_banlist_thread.rs similarity index 100% rename from crates/app/storyteller_web/src/threads/poll_ip_banlist_thread.rs rename to crates/service/storyteller_web/src/threads/poll_ip_banlist_thread.rs diff --git a/crates/app/storyteller_web/src/tts_download_job.rs b/crates/service/storyteller_web/src/tts_download_job.rs similarity index 100% rename from crates/app/storyteller_web/src/tts_download_job.rs rename to crates/service/storyteller_web/src/tts_download_job.rs diff --git a/crates/app/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs similarity index 100% rename from crates/app/storyteller_web/src/tts_inference_job.rs rename to crates/service/storyteller_web/src/tts_inference_job.rs diff --git a/crates/app/storyteller_web/src/util/anyhow_result.rs b/crates/service/storyteller_web/src/util/anyhow_result.rs similarity index 100% rename from crates/app/storyteller_web/src/util/anyhow_result.rs rename to crates/service/storyteller_web/src/util/anyhow_result.rs diff --git a/crates/app/storyteller_web/src/util/buckets/bucket_client.rs b/crates/service/storyteller_web/src/util/buckets/bucket_client.rs similarity index 100% rename from crates/app/storyteller_web/src/util/buckets/bucket_client.rs rename to crates/service/storyteller_web/src/util/buckets/bucket_client.rs diff --git a/crates/app/storyteller_web/src/util/buckets/bucket_path_unifier.rs b/crates/service/storyteller_web/src/util/buckets/bucket_path_unifier.rs similarity index 100% rename from crates/app/storyteller_web/src/util/buckets/bucket_path_unifier.rs rename to crates/service/storyteller_web/src/util/buckets/bucket_path_unifier.rs diff --git a/crates/app/storyteller_web/src/util/buckets/bucket_paths.rs b/crates/service/storyteller_web/src/util/buckets/bucket_paths.rs similarity index 100% rename from crates/app/storyteller_web/src/util/buckets/bucket_paths.rs rename to crates/service/storyteller_web/src/util/buckets/bucket_paths.rs diff --git a/crates/app/storyteller_web/src/util/buckets/mod.rs b/crates/service/storyteller_web/src/util/buckets/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/util/buckets/mod.rs rename to crates/service/storyteller_web/src/util/buckets/mod.rs diff --git a/crates/app/storyteller_web/src/util/caching/mod.rs b/crates/service/storyteller_web/src/util/caching/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/util/caching/mod.rs rename to crates/service/storyteller_web/src/util/caching/mod.rs diff --git a/crates/app/storyteller_web/src/util/caching/single_item_ttl_cache.rs b/crates/service/storyteller_web/src/util/caching/single_item_ttl_cache.rs similarity index 100% rename from crates/app/storyteller_web/src/util/caching/single_item_ttl_cache.rs rename to crates/service/storyteller_web/src/util/caching/single_item_ttl_cache.rs diff --git a/crates/app/storyteller_web/src/util/delete_role_disambiguation.rs b/crates/service/storyteller_web/src/util/delete_role_disambiguation.rs similarity index 100% rename from crates/app/storyteller_web/src/util/delete_role_disambiguation.rs rename to crates/service/storyteller_web/src/util/delete_role_disambiguation.rs diff --git a/crates/app/storyteller_web/src/util/email_to_gravatar.rs b/crates/service/storyteller_web/src/util/email_to_gravatar.rs similarity index 100% rename from crates/app/storyteller_web/src/util/email_to_gravatar.rs rename to crates/service/storyteller_web/src/util/email_to_gravatar.rs diff --git a/crates/app/storyteller_web/src/util/encrypted_sort_id.rs b/crates/service/storyteller_web/src/util/encrypted_sort_id.rs similarity index 100% rename from crates/app/storyteller_web/src/util/encrypted_sort_id.rs rename to crates/service/storyteller_web/src/util/encrypted_sort_id.rs diff --git a/crates/app/storyteller_web/src/util/filesystem.rs b/crates/service/storyteller_web/src/util/filesystem.rs similarity index 100% rename from crates/app/storyteller_web/src/util/filesystem.rs rename to crates/service/storyteller_web/src/util/filesystem.rs diff --git a/crates/app/storyteller_web/src/util/hashing/hash_file_sha2.rs b/crates/service/storyteller_web/src/util/hashing/hash_file_sha2.rs similarity index 100% rename from crates/app/storyteller_web/src/util/hashing/hash_file_sha2.rs rename to crates/service/storyteller_web/src/util/hashing/hash_file_sha2.rs diff --git a/crates/app/storyteller_web/src/util/hashing/hash_string_sha2.rs b/crates/service/storyteller_web/src/util/hashing/hash_string_sha2.rs similarity index 100% rename from crates/app/storyteller_web/src/util/hashing/hash_string_sha2.rs rename to crates/service/storyteller_web/src/util/hashing/hash_string_sha2.rs diff --git a/crates/app/storyteller_web/src/util/hashing/mod.rs b/crates/service/storyteller_web/src/util/hashing/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/util/hashing/mod.rs rename to crates/service/storyteller_web/src/util/hashing/mod.rs diff --git a/crates/app/storyteller_web/src/util/hashing/sha256_digest.rs b/crates/service/storyteller_web/src/util/hashing/sha256_digest.rs similarity index 100% rename from crates/app/storyteller_web/src/util/hashing/sha256_digest.rs rename to crates/service/storyteller_web/src/util/hashing/sha256_digest.rs diff --git a/crates/app/storyteller_web/src/util/jobs/cache_miss_strategizer.rs b/crates/service/storyteller_web/src/util/jobs/cache_miss_strategizer.rs similarity index 100% rename from crates/app/storyteller_web/src/util/jobs/cache_miss_strategizer.rs rename to crates/service/storyteller_web/src/util/jobs/cache_miss_strategizer.rs diff --git a/crates/app/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs b/crates/service/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs similarity index 100% rename from crates/app/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs rename to crates/service/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs diff --git a/crates/app/storyteller_web/src/util/jobs/mod.rs b/crates/service/storyteller_web/src/util/jobs/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/util/jobs/mod.rs rename to crates/service/storyteller_web/src/util/jobs/mod.rs diff --git a/crates/app/storyteller_web/src/util/jobs/virtual_lfu_cache.rs b/crates/service/storyteller_web/src/util/jobs/virtual_lfu_cache.rs similarity index 100% rename from crates/app/storyteller_web/src/util/jobs/virtual_lfu_cache.rs rename to crates/service/storyteller_web/src/util/jobs/virtual_lfu_cache.rs diff --git a/crates/app/storyteller_web/src/util/markdown_to_html.rs b/crates/service/storyteller_web/src/util/markdown_to_html.rs similarity index 100% rename from crates/app/storyteller_web/src/util/markdown_to_html.rs rename to crates/service/storyteller_web/src/util/markdown_to_html.rs diff --git a/crates/app/storyteller_web/src/util/mod.rs b/crates/service/storyteller_web/src/util/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/util/mod.rs rename to crates/service/storyteller_web/src/util/mod.rs diff --git a/crates/app/storyteller_web/src/util/noop_logger.rs b/crates/service/storyteller_web/src/util/noop_logger.rs similarity index 100% rename from crates/app/storyteller_web/src/util/noop_logger.rs rename to crates/service/storyteller_web/src/util/noop_logger.rs diff --git a/crates/app/storyteller_web/src/util/random_crockford_token.rs b/crates/service/storyteller_web/src/util/random_crockford_token.rs similarity index 100% rename from crates/app/storyteller_web/src/util/random_crockford_token.rs rename to crates/service/storyteller_web/src/util/random_crockford_token.rs diff --git a/crates/app/storyteller_web/src/util/random_prefix_crockford_token.rs b/crates/service/storyteller_web/src/util/random_prefix_crockford_token.rs similarity index 100% rename from crates/app/storyteller_web/src/util/random_prefix_crockford_token.rs rename to crates/service/storyteller_web/src/util/random_prefix_crockford_token.rs diff --git a/crates/app/storyteller_web/src/util/random_uuid.rs b/crates/service/storyteller_web/src/util/random_uuid.rs similarity index 100% rename from crates/app/storyteller_web/src/util/random_uuid.rs rename to crates/service/storyteller_web/src/util/random_uuid.rs diff --git a/crates/app/storyteller_web/src/util/redis/mod.rs b/crates/service/storyteller_web/src/util/redis/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/util/redis/mod.rs rename to crates/service/storyteller_web/src/util/redis/mod.rs diff --git a/crates/app/storyteller_web/src/util/redis/redis_job_status_logger.rs b/crates/service/storyteller_web/src/util/redis/redis_job_status_logger.rs similarity index 100% rename from crates/app/storyteller_web/src/util/redis/redis_job_status_logger.rs rename to crates/service/storyteller_web/src/util/redis/redis_job_status_logger.rs diff --git a/crates/app/storyteller_web/src/util/redis/redis_keys.rs b/crates/service/storyteller_web/src/util/redis/redis_keys.rs similarity index 100% rename from crates/app/storyteller_web/src/util/redis/redis_keys.rs rename to crates/service/storyteller_web/src/util/redis/redis_keys.rs diff --git a/crates/app/storyteller_web/src/util/semi_persistent_cache_dir.rs b/crates/service/storyteller_web/src/util/semi_persistent_cache_dir.rs similarity index 100% rename from crates/app/storyteller_web/src/util/semi_persistent_cache_dir.rs rename to crates/service/storyteller_web/src/util/semi_persistent_cache_dir.rs diff --git a/crates/app/storyteller_web/src/validations/cashapp_username.rs b/crates/service/storyteller_web/src/validations/cashapp_username.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/cashapp_username.rs rename to crates/service/storyteller_web/src/validations/cashapp_username.rs diff --git a/crates/app/storyteller_web/src/validations/check_for_slurs.rs b/crates/service/storyteller_web/src/validations/check_for_slurs.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/check_for_slurs.rs rename to crates/service/storyteller_web/src/validations/check_for_slurs.rs diff --git a/crates/app/storyteller_web/src/validations/discord_username.rs b/crates/service/storyteller_web/src/validations/discord_username.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/discord_username.rs rename to crates/service/storyteller_web/src/validations/discord_username.rs diff --git a/crates/app/storyteller_web/src/validations/github_username.rs b/crates/service/storyteller_web/src/validations/github_username.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/github_username.rs rename to crates/service/storyteller_web/src/validations/github_username.rs diff --git a/crates/app/storyteller_web/src/validations/ip_addresses.rs b/crates/service/storyteller_web/src/validations/ip_addresses.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/ip_addresses.rs rename to crates/service/storyteller_web/src/validations/ip_addresses.rs diff --git a/crates/app/storyteller_web/src/validations/mod.rs b/crates/service/storyteller_web/src/validations/mod.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/mod.rs rename to crates/service/storyteller_web/src/validations/mod.rs diff --git a/crates/app/storyteller_web/src/validations/model_uploads.rs b/crates/service/storyteller_web/src/validations/model_uploads.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/model_uploads.rs rename to crates/service/storyteller_web/src/validations/model_uploads.rs diff --git a/crates/app/storyteller_web/src/validations/passwords.rs b/crates/service/storyteller_web/src/validations/passwords.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/passwords.rs rename to crates/service/storyteller_web/src/validations/passwords.rs diff --git a/crates/app/storyteller_web/src/validations/twitch_username.rs b/crates/service/storyteller_web/src/validations/twitch_username.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/twitch_username.rs rename to crates/service/storyteller_web/src/validations/twitch_username.rs diff --git a/crates/app/storyteller_web/src/validations/twitter_username.rs b/crates/service/storyteller_web/src/validations/twitter_username.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/twitter_username.rs rename to crates/service/storyteller_web/src/validations/twitter_username.rs diff --git a/crates/app/storyteller_web/src/validations/username.rs b/crates/service/storyteller_web/src/validations/username.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/username.rs rename to crates/service/storyteller_web/src/validations/username.rs diff --git a/crates/app/storyteller_web/src/validations/username_reservations.rs b/crates/service/storyteller_web/src/validations/username_reservations.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/username_reservations.rs rename to crates/service/storyteller_web/src/validations/username_reservations.rs diff --git a/crates/app/storyteller_web/src/validations/website_url.rs b/crates/service/storyteller_web/src/validations/website_url.rs similarity index 100% rename from crates/app/storyteller_web/src/validations/website_url.rs rename to crates/service/storyteller_web/src/validations/website_url.rs diff --git a/crates/app/storyteller_web/src/w2l_download_job.rs b/crates/service/storyteller_web/src/w2l_download_job.rs similarity index 100% rename from crates/app/storyteller_web/src/w2l_download_job.rs rename to crates/service/storyteller_web/src/w2l_download_job.rs diff --git a/crates/app/storyteller_web/src/w2l_inference_job.rs b/crates/service/storyteller_web/src/w2l_inference_job.rs similarity index 100% rename from crates/app/storyteller_web/src/w2l_inference_job.rs rename to crates/service/storyteller_web/src/w2l_inference_job.rs From a6855f5b804fd0a06463373554ac2444cd6eac3f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 Jan 2022 02:12:38 -0500 Subject: [PATCH 0744/1042] start using container shared lib --- Cargo.lock | 1 + crates/service/storyteller_web/Cargo.toml | 1 + crates/service/storyteller_web/src/common_env.rs | 2 +- .../src/database/enums/record_visibility.rs | 2 +- .../src/database/enums/tts_model_type.rs | 2 +- .../src/database/enums/vocoder_type.rs | 2 +- .../src/database/helpers/tokens.rs | 2 +- .../src/database/mediators/badge_granter.rs | 2 +- .../src/database/mediators/firehose_publisher.rs | 2 +- .../queries/calculate_tts_model_leaderboard.rs | 2 +- .../queries/calculate_w2l_template_leaderboard.rs | 2 +- .../list_tts_model_category_assignments.rs | 2 +- .../src/database/queries/delete_tts_model.rs | 2 +- .../src/database/queries/delete_tts_result.rs | 2 +- .../src/database/queries/delete_w2l_result.rs | 2 +- .../src/database/queries/delete_w2l_template.rs | 2 +- .../src/database/queries/list_ip_bans.rs | 2 +- .../src/database/queries/list_tts_models.rs | 2 +- .../src/database/queries/list_user_badges.rs | 2 +- .../src/database/queries/list_user_roles.rs | 2 +- .../src/database/queries/list_w2l_templates.rs | 2 +- .../list_assigned_tts_categories_query_builder.rs | 2 +- .../list_categories_query_builder.rs | 2 +- .../list_tts_inference_results_query_builder.rs | 2 +- .../query_builders/list_users_query_builder.rs | 2 +- .../list_w2l_inference_results_query_builder.rs | 2 +- .../http_clients/tts_inference_sidecar_client.rs | 2 +- .../http_server/endpoints/tts/list_tts_models.rs | 2 +- .../w2l/enqueue_infer_w2l_with_uploads.rs | 2 +- .../src/http_server/web_utils/get_host_header.rs | 2 +- .../src/job_queries/tts_download_job_queries.rs | 2 +- .../src/job_queries/tts_inference_job_queries.rs | 2 +- .../src/job_queries/w2l_download_job_queries.rs | 2 +- .../src/job_queries/w2l_inference_job_queries.rs | 2 +- crates/service/storyteller_web/src/main.rs | 3 +-- .../google_drive_download_command.rs | 2 +- .../storyteller_web/src/tts_download_job.rs | 2 +- .../storyteller_web/src/tts_inference_job.rs | 2 +- .../storyteller_web/src/util/anyhow_result.rs | 2 -- .../src/util/buckets/bucket_path_unifier.rs | 2 +- .../src/util/buckets/bucket_paths.rs | 2 +- .../src/util/caching/single_item_ttl_cache.rs | 2 +- .../storyteller_web/src/util/encrypted_sort_id.rs | 15 +++++++-------- .../storyteller_web/src/util/filesystem.rs | 2 +- .../src/util/hashing/hash_file_sha2.rs | 2 +- .../src/util/hashing/hash_string_sha2.rs | 2 +- .../src/util/hashing/sha256_digest.rs | 2 +- .../src/util/jobs/cache_miss_strategizer_multi.rs | 2 +- .../src/util/jobs/virtual_lfu_cache.rs | 6 +++--- .../storyteller_web/src/util/markdown_to_html.rs | 2 +- crates/service/storyteller_web/src/util/mod.rs | 1 - .../src/util/random_prefix_crockford_token.rs | 4 ++-- .../src/util/redis/redis_job_status_logger.rs | 2 +- .../src/util/semi_persistent_cache_dir.rs | 4 ++-- .../storyteller_web/src/w2l_download_job.rs | 2 +- .../storyteller_web/src/w2l_inference_job.rs | 2 +- 56 files changed, 64 insertions(+), 67 deletions(-) delete mode 100644 crates/service/storyteller_web/src/util/anyhow_result.rs diff --git a/Cargo.lock b/Cargo.lock index d8f138e447..e7ccec78f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3976,6 +3976,7 @@ dependencies = [ "bcrypt", "chrono", "config", + "container_common", "data-encoding", "derive_more", "diesel", diff --git a/crates/service/storyteller_web/Cargo.toml b/crates/service/storyteller_web/Cargo.toml index 180afb2edd..ed2c2e565e 100644 --- a/crates/service/storyteller_web/Cargo.toml +++ b/crates/service/storyteller_web/Cargo.toml @@ -39,6 +39,7 @@ path = "src/w2l_inference_job.rs" # Internal config = { path = "../../lib/config" } +container_common = { path = "../../lib/container_common" } http_server_common = { path = "../../lib/http_server_common" } # External diff --git a/crates/service/storyteller_web/src/common_env.rs b/crates/service/storyteller_web/src/common_env.rs index 9013327abc..77db528953 100644 --- a/crates/service/storyteller_web/src/common_env.rs +++ b/crates/service/storyteller_web/src/common_env.rs @@ -1,5 +1,5 @@ use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; // TODO: Move more shared configs here. diff --git a/crates/service/storyteller_web/src/database/enums/record_visibility.rs b/crates/service/storyteller_web/src/database/enums/record_visibility.rs index 54cfe94006..863d80154f 100644 --- a/crates/service/storyteller_web/src/database/enums/record_visibility.rs +++ b/crates/service/storyteller_web/src/database/enums/record_visibility.rs @@ -1,7 +1,7 @@ //! These are columns where users can control the visibility of their data. use anyhow::anyhow; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; /// To use this in a query, the query must have type annotations. /// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375 diff --git a/crates/service/storyteller_web/src/database/enums/tts_model_type.rs b/crates/service/storyteller_web/src/database/enums/tts_model_type.rs index 8a5d2b8ae0..3492d744b3 100644 --- a/crates/service/storyteller_web/src/database/enums/tts_model_type.rs +++ b/crates/service/storyteller_web/src/database/enums/tts_model_type.rs @@ -1,7 +1,7 @@ //! These denote the TTS synthesizer model types use anyhow::anyhow; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; #[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] #[serde(rename_all = "lowercase")] diff --git a/crates/service/storyteller_web/src/database/enums/vocoder_type.rs b/crates/service/storyteller_web/src/database/enums/vocoder_type.rs index 8ebcb54278..ec63f25a61 100644 --- a/crates/service/storyteller_web/src/database/enums/vocoder_type.rs +++ b/crates/service/storyteller_web/src/database/enums/vocoder_type.rs @@ -1,7 +1,7 @@ //! These denote the pretrained vocoders use anyhow::anyhow; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; #[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] #[serde(rename_all = "lowercase")] diff --git a/crates/service/storyteller_web/src/database/helpers/tokens.rs b/crates/service/storyteller_web/src/database/helpers/tokens.rs index 8fc0c6bdbb..892c932312 100644 --- a/crates/service/storyteller_web/src/database/helpers/tokens.rs +++ b/crates/service/storyteller_web/src/database/helpers/tokens.rs @@ -1,4 +1,4 @@ -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; /// Central place to define all the token types. diff --git a/crates/service/storyteller_web/src/database/mediators/badge_granter.rs b/crates/service/storyteller_web/src/database/mediators/badge_granter.rs index 714d4affb2..d1b957b369 100644 --- a/crates/service/storyteller_web/src/database/mediators/badge_granter.rs +++ b/crates/service/storyteller_web/src/database/mediators/badge_granter.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; use crate::database::helpers::tokens::Tokens; use crate::database::mediators::firehose_publisher::FirehosePublisher; -use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn,info}; use sqlx::error::Error::Database; diff --git a/crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs b/crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs index d9a1860fb0..a7379bdf84 100644 --- a/crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs +++ b/crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; use crate::database::helpers::tokens::Tokens; -use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn,info}; use sqlx::error::Error::Database; diff --git a/crates/service/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs b/crates/service/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs index 1aeb5cc3cb..18364cbd18 100644 --- a/crates/service/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs +++ b/crates/service/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs @@ -1,5 +1,5 @@ use anyhow::anyhow; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use log::warn; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs b/crates/service/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs index 0f830bad3a..94562e359d 100644 --- a/crates/service/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs +++ b/crates/service/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs @@ -1,5 +1,5 @@ use anyhow::anyhow; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use log::warn; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs b/crates/service/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs index f4aabcffb9..bfdd0ccd3d 100644 --- a/crates/service/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs +++ b/crates/service/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use log::warn; use sqlx::MySqlPool; use std::collections::{HashSet, HashMap}; diff --git a/crates/service/storyteller_web/src/database/queries/delete_tts_model.rs b/crates/service/storyteller_web/src/database/queries/delete_tts_model.rs index 7ec6622aed..f01091e14b 100644 --- a/crates/service/storyteller_web/src/database/queries/delete_tts_model.rs +++ b/crates/service/storyteller_web/src/database/queries/delete_tts_model.rs @@ -1,4 +1,4 @@ -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use sqlx::MySqlPool; pub async fn delete_tts_model_as_user( diff --git a/crates/service/storyteller_web/src/database/queries/delete_tts_result.rs b/crates/service/storyteller_web/src/database/queries/delete_tts_result.rs index bc9812c095..c5ca6f2f3f 100644 --- a/crates/service/storyteller_web/src/database/queries/delete_tts_result.rs +++ b/crates/service/storyteller_web/src/database/queries/delete_tts_result.rs @@ -1,4 +1,4 @@ -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use sqlx::MySqlPool; pub async fn delete_tts_inference_result_as_user( diff --git a/crates/service/storyteller_web/src/database/queries/delete_w2l_result.rs b/crates/service/storyteller_web/src/database/queries/delete_w2l_result.rs index 5218d3a3f2..adaced9ac7 100644 --- a/crates/service/storyteller_web/src/database/queries/delete_w2l_result.rs +++ b/crates/service/storyteller_web/src/database/queries/delete_w2l_result.rs @@ -1,4 +1,4 @@ -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use sqlx::MySqlPool; pub async fn delete_w2l_inference_result_as_user( diff --git a/crates/service/storyteller_web/src/database/queries/delete_w2l_template.rs b/crates/service/storyteller_web/src/database/queries/delete_w2l_template.rs index 8c29c56aa1..308684434d 100644 --- a/crates/service/storyteller_web/src/database/queries/delete_w2l_template.rs +++ b/crates/service/storyteller_web/src/database/queries/delete_w2l_template.rs @@ -1,4 +1,4 @@ -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use sqlx::MySqlPool; pub async fn delete_w2l_template_as_user( diff --git a/crates/service/storyteller_web/src/database/queries/list_ip_bans.rs b/crates/service/storyteller_web/src/database/queries/list_ip_bans.rs index 998a0d9cd4..4bff642506 100644 --- a/crates/service/storyteller_web/src/database/queries/list_ip_bans.rs +++ b/crates/service/storyteller_web/src/database/queries/list_ip_bans.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use log::warn; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/queries/list_tts_models.rs b/crates/service/storyteller_web/src/database/queries/list_tts_models.rs index e0f727a8c3..1988c1cc93 100644 --- a/crates/service/storyteller_web/src/database/queries/list_tts_models.rs +++ b/crates/service/storyteller_web/src/database/queries/list_tts_models.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use container_common::anyhow_result::AnyhowResult; use crate::database::helpers::boolean_converters::i8_to_bool; -use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/queries/list_user_badges.rs b/crates/service/storyteller_web/src/database/queries/list_user_badges.rs index 5e597bb9c4..7863bbe3c2 100644 --- a/crates/service/storyteller_web/src/database/queries/list_user_badges.rs +++ b/crates/service/storyteller_web/src/database/queries/list_user_badges.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/queries/list_user_roles.rs b/crates/service/storyteller_web/src/database/queries/list_user_roles.rs index 129b581fa0..d909889798 100644 --- a/crates/service/storyteller_web/src/database/queries/list_user_roles.rs +++ b/crates/service/storyteller_web/src/database/queries/list_user_roles.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use container_common::anyhow_result::AnyhowResult; use crate::database::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; -use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs b/crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs index c403b8c244..2f32f2160c 100644 --- a/crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs +++ b/crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use container_common::anyhow_result::AnyhowResult; use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; -use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs index b7f618c40a..64279387ad 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs @@ -1,8 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use container_common::anyhow_result::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; -use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs index 45fb053720..e8f64a5158 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs @@ -1,8 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use container_common::anyhow_result::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; -use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs index 011629933a..66e38cc1bd 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs @@ -1,8 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; +use container_common::anyhow_result::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; -use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs index d8691d61f3..83752a7af6 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs @@ -1,6 +1,6 @@ use chrono::{DateTime, Utc}; +use container_common::anyhow_result::AnyhowResult; use crate::database::helpers::boolean_converters::i8_to_bool; -use crate::util::anyhow_result::AnyhowResult; use sqlx::MySqlPool; /// NB: This is not to be shared externally. diff --git a/crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs index 0553bf0552..b9fc2bcf11 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs @@ -1,8 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; +use container_common::anyhow_result::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; -use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs b/crates/service/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs index 354cdfd1b0..149f956239 100644 --- a/crates/service/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs +++ b/crates/service/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs @@ -1,7 +1,7 @@ use actix_web::http::header; use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; use crate::database::enums::vocoder_type::VocoderType; -use crate::util::anyhow_result::AnyhowResult; use hyper::client::Client; use hyper::{Body, Request}; use log::info; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs index 7361749af1..56112f3254 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs @@ -5,12 +5,12 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; +use container_common::anyhow_result::AnyhowResult; use crate::database::queries::categories::list_tts_model_category_assignments::fetch_tts_model_category_map; use crate::database::queries::list_tts_models::list_tts_models; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use crate::util::anyhow_result::AnyhowResult; use log::{info, warn, log, error}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index a4b3925b81..d550055c12 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -7,6 +7,7 @@ use actix_web::http::header::ContentDisposition; use actix_web::web::{Data, Json, BytesMut}; use actix_web::{middleware, web, App, Error, HttpResponse, HttpServer, HttpRequest, Either, ResponseError}; use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; @@ -16,7 +17,6 @@ use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_fi use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_bytes; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::random_crockford_token::random_crockford_token; diff --git a/crates/service/storyteller_web/src/http_server/web_utils/get_host_header.rs b/crates/service/storyteller_web/src/http_server/web_utils/get_host_header.rs index fd47e41dac..57b4ec37e8 100644 --- a/crates/service/storyteller_web/src/http_server/web_utils/get_host_header.rs +++ b/crates/service/storyteller_web/src/http_server/web_utils/get_host_header.rs @@ -2,7 +2,7 @@ use actix_http::http::HeaderMap; use actix_http::http::HeaderName; use actix_web::HttpRequest; use actix_web::http::HeaderValue; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; // NB: This is extracted because CLion is having incredible difficulty with these types. pub fn get_host_header(http_request: &HttpRequest) -> AnyhowResult> { diff --git a/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs b/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs index 1ff2073a5d..7f84cd2b85 100644 --- a/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs @@ -4,8 +4,8 @@ use anyhow::anyhow; use chrono::Utc; +use container_common::anyhow_result::AnyhowResult; use crate::database::helpers::tokens::Tokens; -use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs b/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs index 8ebfb02157..1b8533eca9 100644 --- a/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs @@ -4,10 +4,10 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; +use container_common::anyhow_result::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; use crate::database::enums::vocoder_type::VocoderType; use crate::database::helpers::tokens::Tokens; -use crate::util::anyhow_result::AnyhowResult; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn, info}; use sqlx::{MySqlPool, Transaction, MySql}; diff --git a/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs b/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs index 77998b0327..1d7dc92b3d 100644 --- a/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs @@ -4,8 +4,8 @@ use anyhow::anyhow; use chrono::Utc; +use container_common::anyhow_result::AnyhowResult; use crate::database::helpers::tokens::Tokens; -use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs b/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs index de013eda66..19e670ebb1 100644 --- a/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs @@ -4,9 +4,9 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; +use container_common::anyhow_result::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::tokens::Tokens; -use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn, info}; diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 534f17254e..e618b7e806 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -35,6 +35,7 @@ use actix_web::{HttpServer, web, HttpResponse, App}; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; +use container_common::anyhow_result::AnyhowResult; use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::events::list_events::list_events_handler; @@ -146,8 +147,6 @@ const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; // Various bucket roots const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &'static str = "AUDIO_UPLOADS_BUCKET_ROOT"; -pub type AnyhowResult = anyhow::Result; - #[actix_web::main] async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); diff --git a/crates/service/storyteller_web/src/script_execution/google_drive_download_command.rs b/crates/service/storyteller_web/src/script_execution/google_drive_download_command.rs index 4b4e6ac059..1f8f7b7069 100644 --- a/crates/service/storyteller_web/src/script_execution/google_drive_download_command.rs +++ b/crates/service/storyteller_web/src/script_execution/google_drive_download_command.rs @@ -1,5 +1,5 @@ use anyhow::anyhow; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; use log::{info, warn}; use std::io::{BufReader, Read}; diff --git a/crates/service/storyteller_web/src/tts_download_job.rs b/crates/service/storyteller_web/src/tts_download_job.rs index 7718e356b5..2c4e6ddf01 100644 --- a/crates/service/storyteller_web/src/tts_download_job.rs +++ b/crates/service/storyteller_web/src/tts_download_job.rs @@ -20,6 +20,7 @@ use anyhow::anyhow; use chrono::Utc; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; +use container_common::anyhow_result::AnyhowResult; use crate::common_env::CommonEnv; use crate::database::enums::tts_model_type::TtsModelType; use crate::database::mediators::badge_granter::BadgeGranter; @@ -33,7 +34,6 @@ use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::tacotron_model_check_command::TacotronModelCheckCommand; use crate::script_execution::talknet_model_check_command::TalknetModelCheckCommand; -use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use crate::util::buckets::bucket_paths::hash_to_bucket_path; diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index 89aa39a78a..c7d9c2f0b0 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -21,6 +21,7 @@ use anyhow::{anyhow, Error}; use chrono::{Utc, DateTime, TimeZone}; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; +use container_common::anyhow_result::AnyhowResult; use crate::common_env::CommonEnv; use crate::database::enums::vocoder_type::VocoderType; use crate::database::mediators::firehose_publisher::FirehosePublisher; @@ -36,7 +37,6 @@ use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_failur use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_permanently_dead; use crate::job_queries::tts_inference_job_queries::query_tts_inference_job_records; use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; -use crate::util::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use crate::util::buckets::bucket_paths::hash_to_bucket_path; diff --git a/crates/service/storyteller_web/src/util/anyhow_result.rs b/crates/service/storyteller_web/src/util/anyhow_result.rs deleted file mode 100644 index 7784374637..0000000000 --- a/crates/service/storyteller_web/src/util/anyhow_result.rs +++ /dev/null @@ -1,2 +0,0 @@ -/// Easier to import than anyhow::Result. -pub type AnyhowResult = anyhow::Result; diff --git a/crates/service/storyteller_web/src/util/buckets/bucket_path_unifier.rs b/crates/service/storyteller_web/src/util/buckets/bucket_path_unifier.rs index 3e24289dab..a5e935f1d9 100644 --- a/crates/service/storyteller_web/src/util/buckets/bucket_path_unifier.rs +++ b/crates/service/storyteller_web/src/util/buckets/bucket_path_unifier.rs @@ -1,4 +1,4 @@ -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use crate::util::buckets::bucket_paths::hash_to_bucket_path; use std::path::PathBuf; diff --git a/crates/service/storyteller_web/src/util/buckets/bucket_paths.rs b/crates/service/storyteller_web/src/util/buckets/bucket_paths.rs index 670a0a98e5..eadedbf56c 100644 --- a/crates/service/storyteller_web/src/util/buckets/bucket_paths.rs +++ b/crates/service/storyteller_web/src/util/buckets/bucket_paths.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use anyhow::bail; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use std::path::PathBuf; pub fn hash_to_bucket_path(file_hash: &str, diff --git a/crates/service/storyteller_web/src/util/caching/single_item_ttl_cache.rs b/crates/service/storyteller_web/src/util/caching/single_item_ttl_cache.rs index c40c261164..32cb756213 100644 --- a/crates/service/storyteller_web/src/util/caching/single_item_ttl_cache.rs +++ b/crates/service/storyteller_web/src/util/caching/single_item_ttl_cache.rs @@ -1,5 +1,5 @@ use anyhow::bail; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use lru_time_cache::LruCache; use std::sync::{Arc, Mutex, PoisonError, MutexGuard}; use std::time::Duration; diff --git a/crates/service/storyteller_web/src/util/encrypted_sort_id.rs b/crates/service/storyteller_web/src/util/encrypted_sort_id.rs index aba0ef84eb..56ce681e2b 100644 --- a/crates/service/storyteller_web/src/util/encrypted_sort_id.rs +++ b/crates/service/storyteller_web/src/util/encrypted_sort_id.rs @@ -1,12 +1,11 @@ - -use base64; -use magic_crypt::{MagicCryptTrait, MagicCrypt256}; -use magic_crypt::generic_array::typenum::U256; -use std::io::Cursor; -use magic_crypt::new_magic_crypt; -use crate::util::anyhow_result::AnyhowResult; -use rand::RngCore; use base64::{Config, CharacterSet}; +use base64; +use container_common::anyhow_result::AnyhowResult; +use magic_crypt::generic_array::typenum::U256; +use magic_crypt::new_magic_crypt; +use magic_crypt::{MagicCryptTrait, MagicCrypt256}; +use rand::RngCore; +use std::io::Cursor; // TODO: A protobuf would be more compact! /// This gets encrypted and sent to the frontend as an opaque handle. diff --git a/crates/service/storyteller_web/src/util/filesystem.rs b/crates/service/storyteller_web/src/util/filesystem.rs index d3c84000ac..b4553ed95a 100644 --- a/crates/service/storyteller_web/src/util/filesystem.rs +++ b/crates/service/storyteller_web/src/util/filesystem.rs @@ -1,5 +1,5 @@ use anyhow::bail; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use log::info; use log::warn; use std::fs; diff --git a/crates/service/storyteller_web/src/util/hashing/hash_file_sha2.rs b/crates/service/storyteller_web/src/util/hashing/hash_file_sha2.rs index dd762538f6..b0a31ba68c 100644 --- a/crates/service/storyteller_web/src/util/hashing/hash_file_sha2.rs +++ b/crates/service/storyteller_web/src/util/hashing/hash_file_sha2.rs @@ -1,4 +1,4 @@ -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use crate::util::hashing::sha256_digest::sha256_digest; use data_encoding::HEXLOWER_PERMISSIVE; use ring::digest::{Context, Digest, SHA256}; diff --git a/crates/service/storyteller_web/src/util/hashing/hash_string_sha2.rs b/crates/service/storyteller_web/src/util/hashing/hash_string_sha2.rs index b44a4d0f24..a6fe989c72 100644 --- a/crates/service/storyteller_web/src/util/hashing/hash_string_sha2.rs +++ b/crates/service/storyteller_web/src/util/hashing/hash_string_sha2.rs @@ -1,4 +1,4 @@ -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use crate::util::hashing::sha256_digest::sha256_digest; use data_encoding::HEXLOWER_PERMISSIVE; use ring::digest::{Context, Digest, SHA256}; diff --git a/crates/service/storyteller_web/src/util/hashing/sha256_digest.rs b/crates/service/storyteller_web/src/util/hashing/sha256_digest.rs index 12474badcc..d66480ee82 100644 --- a/crates/service/storyteller_web/src/util/hashing/sha256_digest.rs +++ b/crates/service/storyteller_web/src/util/hashing/sha256_digest.rs @@ -1,4 +1,4 @@ -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use data_encoding::HEXLOWER_PERMISSIVE; use ring::digest::{Context, Digest, SHA256}; use std::fs::File; diff --git a/crates/service/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs b/crates/service/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs index 554be524cd..34dbf66997 100644 --- a/crates/service/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs +++ b/crates/service/storyteller_web/src/util/jobs/cache_miss_strategizer_multi.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use chrono::{DateTime, Utc, Duration}; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use crate::util::jobs::cache_miss_strategizer::{CacheMissStrategizer, CacheMissStrategy}; use std::sync::{RwLock, Arc}; diff --git a/crates/service/storyteller_web/src/util/jobs/virtual_lfu_cache.rs b/crates/service/storyteller_web/src/util/jobs/virtual_lfu_cache.rs index 0a25f23fd0..2b88dbd751 100644 --- a/crates/service/storyteller_web/src/util/jobs/virtual_lfu_cache.rs +++ b/crates/service/storyteller_web/src/util/jobs/virtual_lfu_cache.rs @@ -1,9 +1,9 @@ use anyhow::anyhow; -use crate::util::anyhow_result::AnyhowResult; +use container_common::anyhow_result::AnyhowResult; use lfu::LFUCache; -use std::sync::{Arc, RwLock}; -use std::collections::HashSet; use std::borrow::Borrow; +use std::collections::HashSet; +use std::sync::{Arc, RwLock}; /// This stands in front of the python HTTP sidecar and controls which /// models get to remain in memory. The Python sidecar keeps multiple diff --git a/crates/service/storyteller_web/src/util/markdown_to_html.rs b/crates/service/storyteller_web/src/util/markdown_to_html.rs index 9b89017945..2dccc6cafb 100644 --- a/crates/service/storyteller_web/src/util/markdown_to_html.rs +++ b/crates/service/storyteller_web/src/util/markdown_to_html.rs @@ -1,5 +1,5 @@ +use container_common::anyhow_result::AnyhowResult; use pulldown_cmark::{Options, Parser, html}; -use crate::util::anyhow_result::AnyhowResult; pub fn markdown_to_html(markdown_input: &str) -> String { // NB: CommonMark allows for HTML, including diff --git a/static/websocket_test.js b/static/websocket_test.js index 5e4ecd7051..674881d3c6 100644 --- a/static/websocket_test.js +++ b/static/websocket_test.js @@ -1,10 +1,18 @@ const WS_URL = 'ws://localhost:54321/obs'; +const DEFAULT_USERNAME = 'testytest512'; + (function() { console.log('installing script'); - const username = 'testytest512'; + const maybeUsername = window.location.hash.replace('#', '').trim(); + + const username = (!!maybeUsername) ? maybeUsername : DEFAULT_USERNAME; + + console.log('username', username); + document.getElementById('username').innerHTML = username; + const url = `${WS_URL}/${username}`; let sock = new WebSocket(url); From ab90af892f532775f4606c5bb0e613e2f2853d61 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 Jan 2022 21:47:00 -0500 Subject: [PATCH 0787/1042] misc --- .../twitch_pubsub_user_subscriber_thread.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index f452825eac..5529b52d2c 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -74,15 +74,17 @@ pub async fn twitch_pubsub_user_subscriber_thread( loop { warn!("client.next()"); - let event = client.next().await.unwrap(); + let maybe_event = client.try_next().await.unwrap(); - warn!("event: {:?}", event); + if let Some(event) = maybe_event.as_ref() { + warn!("event: {:?}", event); - match event { - Response::Response(_) => {} - Response::Message { .. } => {} - Response::Pong => {} - Response::Reconnect => {} + match event { + Response::Response(_) => {} + Response::Message { .. } => {} + Response::Pong => {} + Response::Reconnect => {} + } } sleep(Duration::from_secs(5)); From 0cb8cbd80205ed4e4f96f97181df0021d3f1e15f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 Jan 2022 22:42:05 -0500 Subject: [PATCH 0788/1042] store lowercase twitch username --- .../up.sql | 8 ++++++-- .../database_queries/src/twitch_oauth/find.rs | 20 ++++++++++++------- .../src/twitch_oauth/insert.rs | 4 ++++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql b/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql index d9f3497fb5..ac837ff3a3 100644 --- a/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql +++ b/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql @@ -26,10 +26,13 @@ CREATE TABLE twitch_oauth_tokens( -- https://discuss.dev.twitch.tv/t/bug-v5-api-returns--id-as-string-for-featured-channels/10310 twitch_user_id VARCHAR(64) NOT NULL, - -- Usernames are between 4 and 25 characters. + -- Twitch usernames are between 4 and 25 characters. -- This is returned in the oauth flow twitch_username VARCHAR(32) NOT NULL, + -- Twitch username, but lowercase for lookup. + twitch_username_lowercase VARCHAR(32) NOT NULL, + -- ========== OAUTH TOKEN DETAILS ========== -- The ever important OAuth access token. @@ -103,7 +106,8 @@ CREATE TABLE twitch_oauth_tokens( PRIMARY KEY (id), KEY fk_maybe_user_token (maybe_user_token), KEY index_twitch_user_id (twitch_user_id), - KEY index_twitch_username (twitch_username), + -- KEY index_twitch_username (twitch_username), + KEY index_twitch_username_lowercase (twitch_username_lowercase), KEY index_expires_at (expires_at), KEY index_user_deleted_at (user_deleted_at), KEY index_mod_deleted_at (mod_deleted_at) diff --git a/crates/lib/database_queries/src/twitch_oauth/find.rs b/crates/lib/database_queries/src/twitch_oauth/find.rs index df2c45e88b..dce52a05e6 100644 --- a/crates/lib/database_queries/src/twitch_oauth/find.rs +++ b/crates/lib/database_queries/src/twitch_oauth/find.rs @@ -18,6 +18,7 @@ pub struct TwitchOauthTokenRecord { /// Twitch user pub twitch_user_id: String, pub twitch_username: String, + pub twitch_username_lowercase: String, /// Token details pub access_token: String, @@ -43,7 +44,7 @@ pub struct TwitchOauthTokenFinder { scope_user_token: Option, scope_twitch_user_id: Option, - scope_twitch_username: Option, + scope_twitch_username_lowercase: Option, } impl TwitchOauthTokenFinder { @@ -51,7 +52,7 @@ impl TwitchOauthTokenFinder { Self { scope_user_token: None, scope_twitch_user_id: None, - scope_twitch_username: None + scope_twitch_username_lowercase: None } } @@ -65,8 +66,10 @@ impl TwitchOauthTokenFinder { self } + /// This will automatically look up against a lowercased username pub fn scope_twitch_username(mut self, twitch_username: Option<&str>) -> Self { - self.scope_twitch_username = twitch_username.map(|t| t.to_string()); + self.scope_twitch_username_lowercase = twitch_username + .map(|t| t.to_string().to_lowercase()); self } @@ -82,6 +85,7 @@ impl TwitchOauthTokenFinder { maybe_user_gravatar_hash: record.maybe_user_gravatar_hash.clone(), twitch_user_id: record.twitch_user_id.clone(), twitch_username: record.twitch_username.clone(), + twitch_username_lowercase: record.twitch_username_lowercase.clone(), access_token: record.access_token.clone(), maybe_refresh_token: record.maybe_refresh_token.clone(), token_type: record.token_type.clone(), @@ -117,7 +121,7 @@ impl TwitchOauthTokenFinder { query = query.bind(user_id); } - if let Some(twitch_username) = self.scope_twitch_username.as_deref() { + if let Some(twitch_username) = self.scope_twitch_username_lowercase.as_deref() { query = query.bind(twitch_username); } @@ -148,6 +152,7 @@ SELECT twitch_oauth_tokens.twitch_user_id, twitch_oauth_tokens.twitch_username, + twitch_oauth_tokens.twitch_username_lowercase, twitch_oauth_tokens.access_token, twitch_oauth_tokens.maybe_refresh_token, @@ -197,12 +202,12 @@ LEFT OUTER JOIN users } } - if let Some(username) = self.scope_twitch_username.as_deref() { + if let Some(username) = self.scope_twitch_username_lowercase.as_deref() { if !first_predicate_added { - query.push_str(" WHERE twitch_oauth_tokens.twitch_username = ?"); + query.push_str(" WHERE twitch_oauth_tokens.twitch_username_lowercase = ?"); first_predicate_added = true; } else { - query.push_str(" AND twitch_oauth_tokens.twitch_username = ?"); + query.push_str(" AND twitch_oauth_tokens.twitch_username_lowercase = ?"); } } @@ -234,6 +239,7 @@ pub struct TwitchOauthTokenRecordInternal { /// Twitch user pub twitch_user_id: String, pub twitch_username: String, + pub twitch_username_lowercase: String, /// Token details pub access_token: String, diff --git a/crates/lib/database_queries/src/twitch_oauth/insert.rs b/crates/lib/database_queries/src/twitch_oauth/insert.rs index 70ca12d97a..85340cc875 100644 --- a/crates/lib/database_queries/src/twitch_oauth/insert.rs +++ b/crates/lib/database_queries/src/twitch_oauth/insert.rs @@ -115,6 +115,7 @@ SET maybe_user_token = ?, twitch_user_id = ?, twitch_username = ?, + twitch_username_lowercase = ?, access_token = ?, maybe_refresh_token = ?, token_type = ?, @@ -129,6 +130,7 @@ SET self.maybe_user_token.clone(), self.twitch_user_id.clone(), self.twitch_username.clone(), + self.twitch_username.clone().to_lowercase(), self.access_token.clone(), self.maybe_refresh_token.clone(), self.token_type.clone(), @@ -148,6 +150,7 @@ SET maybe_user_token = ?, twitch_user_id = ?, twitch_username = ?, + twitch_username_lowercase = ?, access_token = ?, maybe_refresh_token = ?, token_type = ?, @@ -162,6 +165,7 @@ SET self.maybe_user_token.clone(), self.twitch_user_id.clone(), self.twitch_username.clone(), + self.twitch_username.clone().to_lowercase(), self.access_token.clone(), self.maybe_refresh_token.clone(), self.token_type.clone(), From 035acf62c57b64406cd90fbf79fd92f1886a3953 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 Jan 2022 22:57:55 -0500 Subject: [PATCH 0789/1042] less type juggling --- .../obs_gateway_websocket_handler.rs | 21 ++++++----- .../src/endpoints_ws/obs_twitch_thread.rs | 11 +++--- .../twitch_pubsub_user_subscriber_thread.rs | 1 + crates/service/obs_gateway/src/twitch/mod.rs | 1 + .../obs_gateway/src/twitch/twitch_user_id.rs | 35 +++++++++++++++++++ 5 files changed, 55 insertions(+), 14 deletions(-) create mode 100644 crates/service/obs_gateway/src/twitch/twitch_user_id.rs diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 230c13469c..22de958c0b 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -10,6 +10,7 @@ use crate::server_state::ObsGatewayServerState; use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use crate::twitch::polling_websocket_client::PollingTwitchWebsocketClient; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; +use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; use database_queries::twitch_oauth::find::{TwitchOauthTokenFinder, TwitchOauthTokenRecord}; use futures_timer::Delay; @@ -64,7 +65,7 @@ pub async fn obs_gateway_websocket_handler( }, }; - let user_id = token_record.twitch_user_id.parse::() + let twitch_user_id = TwitchUserId::from_string(&token_record.twitch_user_id) .map_err(|e| { error!("Error converting twitch user id: {}, id= {}", e, &token_record.twitch_user_id); CommonServerError::ServerError @@ -95,7 +96,7 @@ pub async fn obs_gateway_websocket_handler( let mut client = TwitchWebsocketClient::new().unwrap(); let token_refresher = OauthTokenRefresher::new( - user_id, + twitch_user_id.get_numeric(), &token_record.access_token, token_record.maybe_refresh_token.as_deref()); @@ -125,7 +126,7 @@ pub async fn obs_gateway_websocket_handler( let server_state_arc = server_state.get_ref().clone(); let websocket = ObsGatewayWebSocket::new( - user_id, + twitch_user_id.clone(), client, token_refresher, server_state_arc @@ -140,20 +141,22 @@ pub async fn obs_gateway_websocket_handler( /// Websocket behavior struct ObsGatewayWebSocket { - //twitch_client: PollingTwitchWebsocketClient, - twitch_user_id: u32, + twitch_user_id: TwitchUserId, twitch_thread: Arc, server_state: Arc, } impl ObsGatewayWebSocket { fn new( - twitch_user_id: u32, + twitch_user_id: TwitchUserId, twitch_client: TwitchWebsocketClient, oauth_token_refresher: OauthTokenRefresher, server_state: Arc, ) -> Self { - let twitch_thread = Arc::new(ObsTwitchThread::new(twitch_user_id, oauth_token_refresher, twitch_client)); + let twitch_thread = Arc::new(ObsTwitchThread::new( + twitch_user_id.clone(), + oauth_token_refresher, + twitch_client)); Self { twitch_user_id, twitch_thread, @@ -238,8 +241,8 @@ impl StreamHandler> for ObsGatewayWebSock let channel = RedisKeys::obs_active_session_topic(); - let twitch_user_id = self.twitch_user_id.to_string(); - let payload = ObsActivePayload::new(&twitch_user_id); + let payload = ObsActivePayload::new(self.twitch_user_id.get_str()); + let json_payload = payload.serialize() .map_err(|e| { error!("Could not serialize JSON: {:?}", e); diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs index 1d23df24f5..73fa9101c5 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs @@ -1,15 +1,16 @@ +use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use crate::twitch::polling_websocket_client::PollingTwitchWebsocketClient; +use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; +use futures::lock::Mutex; use log::error; use log::info; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock, PoisonError, RwLockWriteGuard}; use std::thread::sleep; -use futures::lock::Mutex; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; use std::time::Instant; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::runtime::Handle; -use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; // TODO: Let's make this an Arc>> // so that multiple browser sessions for the same user can @@ -27,7 +28,7 @@ pub struct ObsTwitchThread { } struct InnerData { - twitch_user_id: u32, + twitch_user_id: TwitchUserId, token_refresher: OauthTokenRefresher, is_connected: bool, twitch_client: TwitchWebsocketClient, @@ -36,7 +37,7 @@ struct InnerData { impl ObsTwitchThread { pub fn new( - twitch_user_id: u32, + twitch_user_id: TwitchUserId, token_refresher: OauthTokenRefresher, twitch_client: TwitchWebsocketClient ) -> Self { diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 5529b52d2c..a525187c9e 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -15,6 +15,7 @@ use std::thread::sleep; use std::time::Duration; use twitch_api2::pubsub::Response; +/// Receive both forms of twitch userid pub async fn twitch_pubsub_user_subscriber_thread( twitch_user_id: String, mysql_pool: Arc>, diff --git a/crates/service/obs_gateway/src/twitch/mod.rs b/crates/service/obs_gateway/src/twitch/mod.rs index dadd275d32..76a8eebc7e 100644 --- a/crates/service/obs_gateway/src/twitch/mod.rs +++ b/crates/service/obs_gateway/src/twitch/mod.rs @@ -3,4 +3,5 @@ pub mod polling_websocket_client; pub mod pubsub; pub mod twitch_client_wrapper; pub mod twitch_secrets; +pub mod twitch_user_id; pub mod websocket_client; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/twitch/twitch_user_id.rs b/crates/service/obs_gateway/src/twitch/twitch_user_id.rs new file mode 100644 index 0000000000..d3d75f23ef --- /dev/null +++ b/crates/service/obs_gateway/src/twitch/twitch_user_id.rs @@ -0,0 +1,35 @@ +use container_common::anyhow_result::AnyhowResult; + +/// Twitch says to encode these as strings, but the Twitch libraries +/// still deal in integers. This centralizes type juggling. +#[derive(Clone)] +pub struct TwitchUserId { + string_user_id: String, + numeric_user_id: u32, +} + +impl TwitchUserId { + + pub fn from_string(twitch_user_id: &str) -> AnyhowResult { + let numeric_id = twitch_user_id.parse::()?; + Ok(Self { + string_user_id: twitch_user_id.to_string(), + numeric_user_id: numeric_id, + }) + } + + pub fn from_numeric(twitch_user_id: u32) -> Self { + Self { + string_user_id: twitch_user_id.to_string(), + numeric_user_id: twitch_user_id, + } + } + + pub fn get_numeric(&self) -> u32 { + self.numeric_user_id + } + + pub fn get_str(&self) -> &str { + &self.string_user_id + } +} From 7bc698482c1442cfd2c0c6737346cc2df63660d4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 Jan 2022 23:02:04 -0500 Subject: [PATCH 0790/1042] more cleanup --- .../obs_gateway_websocket_handler.rs | 2 +- .../listen_for_active_obs_sessions_thread.rs | 76 +------------------ .../twitch_pubsub_user_subscriber_thread.rs | 10 +-- .../obs_gateway/src/twitch/twitch_user_id.rs | 2 +- 4 files changed, 10 insertions(+), 80 deletions(-) diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 22de958c0b..d8ddd361c8 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -65,7 +65,7 @@ pub async fn obs_gateway_websocket_handler( }, }; - let twitch_user_id = TwitchUserId::from_string(&token_record.twitch_user_id) + let twitch_user_id = TwitchUserId::from_str(&token_record.twitch_user_id) .map_err(|e| { error!("Error converting twitch user id: {}, id= {}", e, &token_record.twitch_user_id); CommonServerError::ServerError diff --git a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs index 8193a7e86d..1b000ab504 100644 --- a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs +++ b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs @@ -14,6 +14,7 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use tokio::runtime::Runtime; +use crate::twitch::twitch_user_id::TwitchUserId; pub async fn listen_for_active_obs_session_thread( mysql_pool: Arc>, @@ -27,26 +28,13 @@ pub async fn listen_for_active_obs_session_thread( let channel = RedisKeys::obs_active_session_topic(); pubsub.subscribe(channel).unwrap(); - let mut count = 0; - - //loop { - // let payload : String = try!(msg.get_payload()); - // println!("channel '{}': {}", msg.get_channel_name(), payload); - //} - loop { - //info!("[PubSub]"); - // TODO: ERROR HANDLING let message = pubsub.get_message().unwrap(); let payload : String = message.get_payload().unwrap(); - //info!("Message: {}", payload); - let payload = ObsActivePayload::from_json_str(&payload).unwrap(); - //info!("Message decoded: {:?}", payload); - let lease_key = RedisKeys::twitch_pubsub_lease(&payload.twitch_user_id); let mut redis = redis_pool.get().unwrap(); @@ -54,11 +42,7 @@ pub async fn listen_for_active_obs_session_thread( let lease_value : Option = redis.get(&lease_key).unwrap(); if let Some(value) = lease_value.as_deref() { - //info!("Already has lease."); let lease = LeasePayload::deserialize(value).unwrap(); - - //info!("Lease value: {:?}", &lease); - continue; } @@ -73,65 +57,11 @@ pub async fn listen_for_active_obs_session_thread( LEASE_TIMEOUT_SECONDS ).unwrap(); - let twitch_user_id = payload.twitch_user_id.clone(); + let twitch_user_id = TwitchUserId::from_str(&payload.twitch_user_id).unwrap(); + let redis_pool2 = redis_pool.clone(); let mysql_pool2 = mysql_pool.clone(); runtime.spawn(twitch_pubsub_user_subscriber_thread(twitch_user_id, mysql_pool2, redis_pool2)); - - - // Publish: (ActiveSession, user_id) - // - On: First connect - // - Every 30 seconds - // - 1,000 users x 10 servers = 10,000 = 333 QPS - // - 100 users x 5 servers = 500 = 16 QPS - // Redis Benchmarks - // SET: 552028.75 requests per second (high spec machine) - // GET: 707463.75 requests per second (high spec machine) - // SET: 100007 requests completed in 0.88 seconds (no pipeline) - // GET: 100000 requests completed in 1.23 seconds (no pipeline) - - // Lease Key w/ 60s TTL: "ActiveSessionLease:{user_id}" - // Lease Value: "{serverNonce}:{threadNonce}" (to kill same-server dupes) - - - - - // Data structures: - // - HashSet of user ids (maybe not needed?) - // - Random threads (don't need to be tracked) - // - Redis leases that expire every 30 seconds - 1 minute - // --- But polling these leases in a big cluster is high QPS - - // > New UserId: - // If not present in HashSet, try to create a Redis lease - // If lease already taken, ignore. - // If lease created, create a new thread. - // Thread continually renews lease (every 10 seconds) - // Puts in a threadlocal random nonce into lease. If differs at - // any point, we know some other process somehow got a lease. Abort. - - - // frontend --> lease key - // -- no, frontend won't know about disconnects, just drought of events - -/* - THIS WORKS - if count < 3 { - info!("Spawning...."); - - let count2 = count; - - runtime.spawn(async move { - let count3 = count2; - loop { - info!(".....spawned..... {}", count3); - sleep(Duration::from_millis(1_000)); - } - }); - } - - count += 1; - */ } } diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index a525187c9e..031af0d6e4 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -1,6 +1,7 @@ use crate::redis::lease_payload::LeasePayload; use crate::redis::lease_timeout::LEASE_TIMEOUT_SECONDS; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; +use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; use database_queries::twitch_oauth::find::{TwitchOauthTokenRecord, TwitchOauthTokenFinder}; use log::info; @@ -17,20 +18,19 @@ use twitch_api2::pubsub::Response; /// Receive both forms of twitch userid pub async fn twitch_pubsub_user_subscriber_thread( - twitch_user_id: String, + twitch_user_id: TwitchUserId, mysql_pool: Arc>, redis_pool: Arc>, ) { // TODO: Error handling - let numeric_twitch_user_id = twitch_user_id.parse::().unwrap(); loop { info!("Twitch subscriber thread"); let mut redis = redis_pool.get().unwrap(); - let lease_key = RedisKeys::twitch_pubsub_lease(&twitch_user_id); + let lease_key = RedisKeys::twitch_pubsub_lease(twitch_user_id.get_str()); let lease = LeasePayload::new("foo", "bar"); let serialized = lease.serialize(); @@ -41,7 +41,7 @@ pub async fn twitch_pubsub_user_subscriber_thread( ).unwrap(); let result = TwitchOauthTokenFinder::new() - .scope_twitch_user_id(Some(numeric_twitch_user_id)) + .scope_twitch_user_id(Some(twitch_user_id.get_numeric())) .perform_query(&mysql_pool) .await .unwrap(); @@ -69,7 +69,7 @@ pub async fn twitch_pubsub_user_subscriber_thread( client.send_ping().await.unwrap(); warn!("Listen to user..."); - let topics = build_pubsub_topics_for_user(numeric_twitch_user_id); + let topics = build_pubsub_topics_for_user(twitch_user_id.get_numeric()); client.listen(&record.access_token, &topics).await.unwrap(); diff --git a/crates/service/obs_gateway/src/twitch/twitch_user_id.rs b/crates/service/obs_gateway/src/twitch/twitch_user_id.rs index d3d75f23ef..a6a5022899 100644 --- a/crates/service/obs_gateway/src/twitch/twitch_user_id.rs +++ b/crates/service/obs_gateway/src/twitch/twitch_user_id.rs @@ -10,7 +10,7 @@ pub struct TwitchUserId { impl TwitchUserId { - pub fn from_string(twitch_user_id: &str) -> AnyhowResult { + pub fn from_str(twitch_user_id: &str) -> AnyhowResult { let numeric_id = twitch_user_id.parse::()?; Ok(Self { string_user_id: twitch_user_id.to_string(), From 1ec18e5daa15a649b9da74a9b7ca537a336c3928 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 Jan 2022 23:11:22 -0500 Subject: [PATCH 0791/1042] cleanup --- .../listen_for_active_obs_sessions_thread.rs | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs index 1b000ab504..e70580bd2f 100644 --- a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs +++ b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs @@ -2,6 +2,7 @@ use crate::redis::lease_payload::LeasePayload; use crate::redis::lease_timeout::LEASE_TIMEOUT_SECONDS; use crate::redis::obs_active_payload::ObsActivePayload; use crate::threads::twitch_pubsub_user_subscriber_thread::twitch_pubsub_user_subscriber_thread; +use crate::twitch::twitch_user_id::TwitchUserId; use log::error; use log::info; use log::warn; @@ -14,7 +15,8 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use tokio::runtime::Runtime; -use crate::twitch::twitch_user_id::TwitchUserId; + +// TODO: Apart from error handling, this looks mostly good. pub async fn listen_for_active_obs_session_thread( mysql_pool: Arc>, @@ -23,22 +25,27 @@ pub async fn listen_for_active_obs_session_thread( runtime: Arc, ) { // TODO: ERROR HANDLING + // TODO: ERROR HANDLING + // TODO: ERROR HANDLING + + // Setup PubSub. + let mut pubsub_pool = redis_pubsub_pool.get().unwrap(); let mut pubsub = pubsub_pool.as_pubsub(); - let channel = RedisKeys::obs_active_session_topic(); - pubsub.subscribe(channel).unwrap(); + + pubsub.subscribe(RedisKeys::obs_active_session_topic()).unwrap(); loop { - // TODO: ERROR HANDLING let message = pubsub.get_message().unwrap(); let payload : String = message.get_payload().unwrap(); - let payload = ObsActivePayload::from_json_str(&payload).unwrap(); - let lease_key = RedisKeys::twitch_pubsub_lease(&payload.twitch_user_id); + let twitch_user_id = TwitchUserId::from_str(&payload.twitch_user_id).unwrap(); + + // See if we have a lease already... let mut redis = redis_pool.get().unwrap(); - + let lease_key = RedisKeys::twitch_pubsub_lease(&payload.twitch_user_id); let lease_value : Option = redis.get(&lease_key).unwrap(); if let Some(value) = lease_value.as_deref() { @@ -48,6 +55,7 @@ pub async fn listen_for_active_obs_session_thread( info!("No existing lease for {:?}...", &lease_key); + // TODO: Thread and server ids. let lease = LeasePayload::new("foo", "bar"); let serialized = lease.serialize(); @@ -57,7 +65,7 @@ pub async fn listen_for_active_obs_session_thread( LEASE_TIMEOUT_SECONDS ).unwrap(); - let twitch_user_id = TwitchUserId::from_str(&payload.twitch_user_id).unwrap(); + // Then launch the thread... let redis_pool2 = redis_pool.clone(); let mysql_pool2 = mysql_pool.clone(); From 5647cacfa95ececb932d95e8bf10663901654b74 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 19 Jan 2022 23:51:57 -0500 Subject: [PATCH 0792/1042] wip --- .../obs_gateway/src/redis/lease_timeout.rs | 5 ++++ .../twitch_pubsub_user_subscriber_thread.rs | 29 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/crates/service/obs_gateway/src/redis/lease_timeout.rs b/crates/service/obs_gateway/src/redis/lease_timeout.rs index c4e1e2d4f8..ed89179035 100644 --- a/crates/service/obs_gateway/src/redis/lease_timeout.rs +++ b/crates/service/obs_gateway/src/redis/lease_timeout.rs @@ -1,7 +1,12 @@ use time::Duration; +// TODO: Think about these timings more. + /// Our Twitch PubSub subscriber threads must renew leases in /// order to be granted an exclusive "lock" on a single user. /// If the underlying Redis key goes away, another thread can /// begin leasing. pub const LEASE_TIMEOUT_SECONDS : usize = 60; + +/// The amount of time to wait before renewing our lease. +pub const LEASE_RENEW_PERIOD: Duration = Duration::seconds(30); diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 031af0d6e4..13c4477bba 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -1,5 +1,5 @@ use crate::redis::lease_payload::LeasePayload; -use crate::redis::lease_timeout::LEASE_TIMEOUT_SECONDS; +use crate::redis::lease_timeout::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD}; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; @@ -15,19 +15,42 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use twitch_api2::pubsub::Response; +use time::Instant; + +// TODO: ERROR HANDLING +// TODO: ERROR HANDLING +// TODO: ERROR HANDLING +// TODO: ERROR HANDLING + +// TODO: Timer to ping(). +// TODO: Timer to renew lease. +// TODO: Publish events back to OBS thread. +// TODO: Disconnect when OBS is done. +// TODO: Refresh oauth token. +// TODO: Handle disconnects. +// TODO: Server+thread IDs -/// Receive both forms of twitch userid pub async fn twitch_pubsub_user_subscriber_thread( twitch_user_id: TwitchUserId, mysql_pool: Arc>, redis_pool: Arc>, ) { - // TODO: Error handling + let mut last_renewed_lease = Instant::now(); loop { info!("Twitch subscriber thread"); + let now = Instant::now(); + let mut should_renew = false; + + match now.checked_sub(LEASE_RENEW_PERIOD) { + None => { should_renew = true } + Some(before) => { + + } + } + let mut redis = redis_pool.get().unwrap(); let lease_key = RedisKeys::twitch_pubsub_lease(twitch_user_id.get_str()); From ebda0d9317e564fffae97c28c43611df21c42b21 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 Jan 2022 21:24:31 -0500 Subject: [PATCH 0793/1042] threads as structs --- .../listen_for_active_obs_sessions_thread.rs | 5 +- .../twitch_pubsub_user_subscriber_thread.rs | 111 ++++++++++++++++++ 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs index e70580bd2f..37aaf0474d 100644 --- a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs +++ b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs @@ -1,7 +1,7 @@ use crate::redis::lease_payload::LeasePayload; use crate::redis::lease_timeout::LEASE_TIMEOUT_SECONDS; use crate::redis::obs_active_payload::ObsActivePayload; -use crate::threads::twitch_pubsub_user_subscriber_thread::twitch_pubsub_user_subscriber_thread; +use crate::threads::twitch_pubsub_user_subscriber_thread::{twitch_pubsub_user_subscriber_thread, TwitchPubsubUserSubscriberThread}; use crate::twitch::twitch_user_id::TwitchUserId; use log::error; use log::info; @@ -70,6 +70,7 @@ pub async fn listen_for_active_obs_session_thread( let redis_pool2 = redis_pool.clone(); let mysql_pool2 = mysql_pool.clone(); - runtime.spawn(twitch_pubsub_user_subscriber_thread(twitch_user_id, mysql_pool2, redis_pool2)); + let thread = TwitchPubsubUserSubscriberThread::new(twitch_user_id, mysql_pool2, redis_pool2); + runtime.spawn(thread.start_thread()); } } diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 13c4477bba..d73cc4a84a 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -30,6 +30,117 @@ use time::Instant; // TODO: Handle disconnects. // TODO: Server+thread IDs + +pub struct TwitchPubsubUserSubscriberThread { + twitch_user_id: TwitchUserId, + mysql_pool: Arc>, + redis_pool: Arc>, + + // Thread state + lease_last_renewed_at: Option, +} + +impl TwitchPubsubUserSubscriberThread { + + pub fn new( + twitch_user_id: TwitchUserId, + mysql_pool: Arc>, + redis_pool: Arc>, + ) -> Self { + Self { + twitch_user_id, + mysql_pool, + redis_pool, + lease_last_renewed_at: None, + } + } + + pub async fn start_thread(mut self) { + let mut last_renewed_lease = Instant::now(); + + self.foo = Some("asdf".to_string()); + + loop { + info!("Twitch subscriber thread"); + + let now = Instant::now(); + let mut should_renew = false; + + match now.checked_sub(LEASE_RENEW_PERIOD) { + None => { should_renew = true } + Some(before) => { + + } + } + + let mut redis = self.redis_pool.get().unwrap(); + + let lease_key = RedisKeys::twitch_pubsub_lease(self.twitch_user_id.get_str()); + let lease = LeasePayload::new("foo", "bar"); + + let serialized = lease.serialize(); + let _v : Option = redis.set_ex( + &lease_key, + &serialized, + LEASE_TIMEOUT_SECONDS + ).unwrap(); + + let result = TwitchOauthTokenFinder::new() + .scope_twitch_user_id(Some(self.twitch_user_id.get_numeric())) + .perform_query(&self.mysql_pool) + .await + .unwrap(); + + let record = match result { + None => { + info!("Twitch user oauth token does not exit. Auth please"); + sleep(Duration::from_secs(5)); + continue; + } + Some(record) => record + }; + + let mut client = TwitchWebsocketClient::new().unwrap(); + //let token_refresher = OauthTokenRefresher::new( + // user_id, + // &token_record.access_token, + // token_record.maybe_refresh_token.as_deref()); + + warn!("Connecting to Twitch PubSub for user {}...", &record.twitch_username); + client.connect().await.unwrap(); + + // TODO: BACKGROUND PINGS UGH! + warn!("Sending ping..."); + client.send_ping().await.unwrap(); + + warn!("Listen to user..."); + let topics = build_pubsub_topics_for_user(self.twitch_user_id.get_numeric()); + + client.listen(&record.access_token, &topics).await.unwrap(); + + loop { + warn!("client.next()"); + let maybe_event = client.try_next().await.unwrap(); + + if let Some(event) = maybe_event.as_ref() { + warn!("event: {:?}", event); + + match event { + Response::Response(_) => {} + Response::Message { .. } => {} + Response::Pong => {} + Response::Reconnect => {} + } + } + + sleep(Duration::from_secs(5)); + } + } + } + +} + + pub async fn twitch_pubsub_user_subscriber_thread( twitch_user_id: TwitchUserId, mysql_pool: Arc>, From 7e875f6b37432afd5aa2c5c6e3fb721ddcf0b8bb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 Jan 2022 22:03:06 -0500 Subject: [PATCH 0794/1042] much cleaner thread, but redis lease doesn't get renewed --- .../listen_for_active_obs_sessions_thread.rs | 2 +- .../twitch_pubsub_user_subscriber_thread.rs | 231 +++++++----------- .../obs_gateway/src/twitch/constants.rs | 11 + crates/service/obs_gateway/src/twitch/mod.rs | 1 + 4 files changed, 101 insertions(+), 144 deletions(-) create mode 100644 crates/service/obs_gateway/src/twitch/constants.rs diff --git a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs index 37aaf0474d..623afec364 100644 --- a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs +++ b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs @@ -1,7 +1,7 @@ use crate::redis::lease_payload::LeasePayload; use crate::redis::lease_timeout::LEASE_TIMEOUT_SECONDS; use crate::redis::obs_active_payload::ObsActivePayload; -use crate::threads::twitch_pubsub_user_subscriber_thread::{twitch_pubsub_user_subscriber_thread, TwitchPubsubUserSubscriberThread}; +use crate::threads::twitch_pubsub_user_subscriber_thread::TwitchPubsubUserSubscriberThread; use crate::twitch::twitch_user_id::TwitchUserId; use log::error; use log::info; diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index d73cc4a84a..1e35cb50d6 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -16,6 +16,8 @@ use std::thread::sleep; use std::time::Duration; use twitch_api2::pubsub::Response; use time::Instant; +use container_common::anyhow_result::AnyhowResult; +use crate::twitch::constants::TWITCH_PING_CADENCE; // TODO: ERROR HANDLING // TODO: ERROR HANDLING @@ -35,194 +37,137 @@ pub struct TwitchPubsubUserSubscriberThread { twitch_user_id: TwitchUserId, mysql_pool: Arc>, redis_pool: Arc>, + twitch_websocket_client: TwitchWebsocketClient, - // Thread state - lease_last_renewed_at: Option, + // ========== Thread state ========== + + // The thread must renew the lease, or another worker will pick it up. + // If the lease gets taken by another, we abandon our own workload. + redis_lease_last_renewed_at: Option, + + /// Twitch PubSub requires PINGs at regular intervals, + /// "To keep the server from closing the connection, clients must send a PING + /// command at least once every 5 minutes. If a client does not receive a PONG + /// message within 10 seconds of issuing a PING command, it should reconnect + /// to the server. See details in Handling Connection Failures." + twitch_last_pinged_at: Option, } impl TwitchPubsubUserSubscriberThread { - pub fn new( twitch_user_id: TwitchUserId, mysql_pool: Arc>, redis_pool: Arc>, ) -> Self { + let twitch_websocket_client = TwitchWebsocketClient::new().unwrap(); Self { twitch_user_id, mysql_pool, redis_pool, - lease_last_renewed_at: None, + twitch_websocket_client, + redis_lease_last_renewed_at: None, + twitch_last_pinged_at: None, } } pub async fn start_thread(mut self) { - let mut last_renewed_lease = Instant::now(); + // We lease from outside before starting the thread. + self.redis_lease_last_renewed_at = Some(Instant::now()); - self.foo = Some("asdf".to_string()); + // Lookup the OAuth credentials + let oauth_lookup_result = self.lookup_oauth_record().await; + let oauth_token_record = match oauth_lookup_result { + Err(e) => { + info!("Error looking up oauth creds: {:?}", e); + return; // TODO: Better flow. + } + Ok(None) => { + info!("Twitch user oauth token does not exit. Auth please."); + return; // TODO: Better flow. + } + Ok(Some(record)) => record, + }; + + warn!("Connecting to Twitch PubSub for user {}...", &oauth_token_record.twitch_username); + self.twitch_websocket_client.connect().await.unwrap(); // TODO: Error handling loop { - info!("Twitch subscriber thread"); - - let now = Instant::now(); - let mut should_renew = false; - - match now.checked_sub(LEASE_RENEW_PERIOD) { - None => { should_renew = true } - Some(before) => { - - } - } - - let mut redis = self.redis_pool.get().unwrap(); - - let lease_key = RedisKeys::twitch_pubsub_lease(self.twitch_user_id.get_str()); - let lease = LeasePayload::new("foo", "bar"); - - let serialized = lease.serialize(); - let _v : Option = redis.set_ex( - &lease_key, - &serialized, - LEASE_TIMEOUT_SECONDS - ).unwrap(); - - let result = TwitchOauthTokenFinder::new() - .scope_twitch_user_id(Some(self.twitch_user_id.get_numeric())) - .perform_query(&self.mysql_pool) - .await - .unwrap(); - - let record = match result { - None => { - info!("Twitch user oauth token does not exit. Auth please"); - sleep(Duration::from_secs(5)); - continue; - } - Some(record) => record - }; - - let mut client = TwitchWebsocketClient::new().unwrap(); - //let token_refresher = OauthTokenRefresher::new( - // user_id, - // &token_record.access_token, - // token_record.maybe_refresh_token.as_deref()); - - warn!("Connecting to Twitch PubSub for user {}...", &record.twitch_username); - client.connect().await.unwrap(); - - // TODO: BACKGROUND PINGS UGH! - warn!("Sending ping..."); - client.send_ping().await.unwrap(); + self.maybe_renew_redis_lease().unwrap(); + self.maybe_send_twitch_ping().await.unwrap(); warn!("Listen to user..."); let topics = build_pubsub_topics_for_user(self.twitch_user_id.get_numeric()); - - client.listen(&record.access_token, &topics).await.unwrap(); + self.twitch_websocket_client.listen(&oauth_token_record.access_token, &topics).await.unwrap(); loop { - warn!("client.next()"); - let maybe_event = client.try_next().await.unwrap(); + let maybe_event = self.twitch_websocket_client.try_next() + .await + .unwrap(); - if let Some(event) = maybe_event.as_ref() { - warn!("event: {:?}", event); + match maybe_event { + None => { + sleep(Duration::from_secs(5)); + } + Some(ref event) => { + warn!("event: {:?}", event); - match event { - Response::Response(_) => {} - Response::Message { .. } => {} - Response::Pong => {} - Response::Reconnect => {} + match event { + Response::Response(_) => {} + Response::Message { .. } => {} + Response::Pong => {} + Response::Reconnect => {} + } } } - - sleep(Duration::from_secs(5)); } } } -} + fn maybe_renew_redis_lease(&mut self) -> AnyhowResult<()> { + let mut should_renew_lease = self.redis_lease_last_renewed_at + .map(|last_write| last_write.elapsed().gt(&LEASE_RENEW_PERIOD)) + .unwrap_or(true); - -pub async fn twitch_pubsub_user_subscriber_thread( - twitch_user_id: TwitchUserId, - mysql_pool: Arc>, - redis_pool: Arc>, -) { - - let mut last_renewed_lease = Instant::now(); - - loop { - info!("Twitch subscriber thread"); - - let now = Instant::now(); - let mut should_renew = false; - - match now.checked_sub(LEASE_RENEW_PERIOD) { - None => { should_renew = true } - Some(before) => { - - } + if should_renew_lease { + info!("Renewing Redis Lease for user {}", self.twitch_user_id.get_numeric()); + self.renew_redis_lease()?; + self.redis_lease_last_renewed_at = Some(Instant::now()); } - let mut redis = redis_pool.get().unwrap(); + Ok(()) + } - let lease_key = RedisKeys::twitch_pubsub_lease(twitch_user_id.get_str()); - let lease = LeasePayload::new("foo", "bar"); - - let serialized = lease.serialize(); + fn renew_redis_lease(&mut self) -> AnyhowResult<()> { + // TODO: Error handling + let mut redis = self.redis_pool.get().unwrap(); + let lease_key = RedisKeys::twitch_pubsub_lease(self.twitch_user_id.get_str()); + let lease_value = LeasePayload::new("foo", "bar").serialize(); let _v : Option = redis.set_ex( &lease_key, - &serialized, + &lease_value, LEASE_TIMEOUT_SECONDS ).unwrap(); + Ok(()) + } - let result = TwitchOauthTokenFinder::new() - .scope_twitch_user_id(Some(twitch_user_id.get_numeric())) - .perform_query(&mysql_pool) + async fn lookup_oauth_record(&mut self) -> AnyhowResult> { + TwitchOauthTokenFinder::new() + .scope_twitch_user_id(Some(self.twitch_user_id.get_numeric())) + .perform_query(&self.mysql_pool) .await - .unwrap(); + } - let record = match result { - None => { - info!("Twitch user oauth token does not exit. Auth please"); - sleep(Duration::from_secs(5)); - continue; - } - Some(record) => record - }; + async fn maybe_send_twitch_ping(&mut self) -> AnyhowResult<()> { + let mut should_send_ping = self.twitch_last_pinged_at + .map(|last_ping| last_ping.elapsed().gt(&TWITCH_PING_CADENCE)) + .unwrap_or(true); - let mut client = TwitchWebsocketClient::new().unwrap(); - //let token_refresher = OauthTokenRefresher::new( - // user_id, - // &token_record.access_token, - // token_record.maybe_refresh_token.as_deref()); - - warn!("Connecting to Twitch PubSub for user {}...", &record.twitch_username); - client.connect().await.unwrap(); - - // TODO: BACKGROUND PINGS UGH! - warn!("Sending ping..."); - client.send_ping().await.unwrap(); - - warn!("Listen to user..."); - let topics = build_pubsub_topics_for_user(twitch_user_id.get_numeric()); - - client.listen(&record.access_token, &topics).await.unwrap(); - - loop { - warn!("client.next()"); - let maybe_event = client.try_next().await.unwrap(); - - if let Some(event) = maybe_event.as_ref() { - warn!("event: {:?}", event); - - match event { - Response::Response(_) => {} - Response::Message { .. } => {} - Response::Pong => {} - Response::Reconnect => {} - } - } - - sleep(Duration::from_secs(5)); + if should_send_ping { + info!("Sending Twitch ping for user {}", self.twitch_user_id.get_numeric()); + self.twitch_websocket_client.send_ping().await?; + self.twitch_last_pinged_at = Some(Instant::now()); } + + Ok(()) } } diff --git a/crates/service/obs_gateway/src/twitch/constants.rs b/crates/service/obs_gateway/src/twitch/constants.rs new file mode 100644 index 0000000000..02b790c58b --- /dev/null +++ b/crates/service/obs_gateway/src/twitch/constants.rs @@ -0,0 +1,11 @@ +use time::Duration; + +/// Twitch PubSub requires PINGs at regular intervals, +/// +/// "To keep the server from closing the connection, clients must send a PING +/// command at least once every 5 minutes. If a client does not receive a PONG +/// message within 10 seconds of issuing a PING command, it should reconnect +/// to the server. See details in Handling Connection Failures." +/// +/// We'll keep this intentionally short so we don't elapse. +pub const TWITCH_PING_CADENCE: Duration = Duration::minutes(3); diff --git a/crates/service/obs_gateway/src/twitch/mod.rs b/crates/service/obs_gateway/src/twitch/mod.rs index 76a8eebc7e..879380886c 100644 --- a/crates/service/obs_gateway/src/twitch/mod.rs +++ b/crates/service/obs_gateway/src/twitch/mod.rs @@ -1,3 +1,4 @@ +pub mod constants; pub mod oauth; pub mod polling_websocket_client; pub mod pubsub; From 6950a86821c1458fc8795d5a1eec39e251d23a53 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 Jan 2022 22:07:52 -0500 Subject: [PATCH 0795/1042] better, but gets stuck / blocking --- .../src/threads/twitch_pubsub_user_subscriber_thread.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 1e35cb50d6..f547f459f0 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -100,13 +100,13 @@ impl TwitchPubsubUserSubscriberThread { self.twitch_websocket_client.listen(&oauth_token_record.access_token, &topics).await.unwrap(); loop { + info!("maybe event..."); let maybe_event = self.twitch_websocket_client.try_next() .await .unwrap(); match maybe_event { None => { - sleep(Duration::from_secs(5)); } Some(ref event) => { warn!("event: {:?}", event); @@ -119,6 +119,11 @@ impl TwitchPubsubUserSubscriberThread { } } } + + self.maybe_renew_redis_lease().unwrap(); + self.maybe_send_twitch_ping().await.unwrap(); + + sleep(Duration::from_secs(5)); } } } From 72b69244229fb21cbae1f5f8c094bb415e2d3f83 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 Jan 2022 23:48:33 -0500 Subject: [PATCH 0796/1042] fix blocking behavior!! --- .../twitch_pubsub_user_subscriber_thread.rs | 59 ++++++++++++++----- models/tts/nemo/__init__.py | 28 --------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index f547f459f0..6bd9029094 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -100,30 +100,61 @@ impl TwitchPubsubUserSubscriberThread { self.twitch_websocket_client.listen(&oauth_token_record.access_token, &topics).await.unwrap(); loop { + //let maybe_event = self.twitch_websocket_client.try_next() + // .await + // .unwrap(); + info!("maybe event..."); - let maybe_event = self.twitch_websocket_client.try_next() - .await - .unwrap(); + let mut interval = tokio::time::interval(Duration::from_millis(1000)); - match maybe_event { - None => { - } - Some(ref event) => { - warn!("event: {:?}", event); - match event { - Response::Response(_) => {} - Response::Message { .. } => {} - Response::Pong => {} - Response::Reconnect => {} + // NB: We can't block forever. + // Adapted from https://github.com/snapview/tokio-tungstenite/blob/master/examples/interval-server.rs + tokio::select! { + maybe_event = self.twitch_websocket_client.try_next() => { + match maybe_event { + Err(e) => { + warn!("socket recv error: {:?}", e); + } + Ok(None) => {}, + Ok(Some(ref event)) => { + warn!("event: {:?}", event); + + match event { + Response::Response(_) => {} + Response::Message { .. } => {} + Response::Pong => {} + Response::Reconnect => {} + } + } } } + _ = interval.tick() => { + //ws_sender.send(Message::Text("tick".to_owned())).await?; + sleep(Duration::from_secs(5)); + } } + sleep(Duration::from_secs(1)); + +// match maybe_event { +// None => { +// } +// Some(ref event) => { +// warn!("event: {:?}", event); +// +// match event { +// Response::Response(_) => {} +// Response::Message { .. } => {} +// Response::Pong => {} +// Response::Reconnect => {} +// } +// } +// } + self.maybe_renew_redis_lease().unwrap(); self.maybe_send_twitch_ping().await.unwrap(); - sleep(Duration::from_secs(5)); } } } diff --git a/models/tts/nemo/__init__.py b/models/tts/nemo/__init__.py index 5b9fedbd1c..e69de29bb2 100644 --- a/models/tts/nemo/__init__.py +++ b/models/tts/nemo/__init__.py @@ -1,28 +0,0 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from nemo.package_info import ( - __contact_emails__, - __contact_names__, - __description__, - __download_url__, - __homepage__, - __keywords__, - __license__, - __package_name__, - __repository_url__, - __shortversion__, - __version__, -) From 516eb12ecd65cd1de6ed73eaafa8b15439415fed Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 Jan 2022 23:49:36 -0500 Subject: [PATCH 0797/1042] HOW DID THIS GET COMMITTED --- models/tts/nemo/__init__.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/models/tts/nemo/__init__.py b/models/tts/nemo/__init__.py index e69de29bb2..5b9fedbd1c 100644 --- a/models/tts/nemo/__init__.py +++ b/models/tts/nemo/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.package_info import ( + __contact_emails__, + __contact_names__, + __description__, + __download_url__, + __homepage__, + __keywords__, + __license__, + __package_name__, + __repository_url__, + __shortversion__, + __version__, +) From 795f1a7b45b8caabc04f2b9a864995f026a809a4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 21 Jan 2022 00:19:29 -0500 Subject: [PATCH 0798/1042] cleanup --- crates/service/obs_gateway/Cargo.toml | 5 +- .../obs_gateway_websocket_handler.rs | 9 -- .../src/endpoints_ws/obs_twitch_thread.rs | 5 - crates/service/obs_gateway/src/main.rs | 120 -------------- .../twitch_pubsub_user_subscriber_thread.rs | 20 +-- crates/service/obs_gateway/src/twitch/mod.rs | 2 - .../obs_gateway/src/twitch/oauth/mod.rs | 3 +- .../src/twitch/oauth/oauth_token_refresher.rs | 24 --- .../src/twitch/polling_websocket_client.rs | 146 ------------------ .../src/twitch/twitch_client_wrapper.rs | 65 -------- .../obs_gateway/src/twitch_pubsub_job.rs | 20 ++- 11 files changed, 13 insertions(+), 406 deletions(-) delete mode 100644 crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs delete mode 100644 crates/service/obs_gateway/src/twitch/polling_websocket_client.rs delete mode 100644 crates/service/obs_gateway/src/twitch/twitch_client_wrapper.rs diff --git a/crates/service/obs_gateway/Cargo.toml b/crates/service/obs_gateway/Cargo.toml index b1b7e94f6d..4c5ecb3fa0 100644 --- a/crates/service/obs_gateway/Cargo.toml +++ b/crates/service/obs_gateway/Cargo.toml @@ -9,7 +9,7 @@ authors = [ publish = false [[bin]] -name = "obs-gateway-server" +name = "obs-gateway" path = "src/main.rs" [[bin]] @@ -26,19 +26,18 @@ http_server_common = { path = "../../lib/http_server_common" } redis_common = { path = "../../lib/redis_common" } # External -futures-timer = "3.0.2" anyhow = "1.0.40" base64 = "0.13.0" chrono = { version = "0.4.19", features = ["serde"] } derive_more = "0.99.14" dotenv = "0.15.0" easyenv = "0.0.6" +futures-timer = "3.0.2" hex = "0.4.3" hostname = "0.3.1" lazy_static = "1.4.0" lfu = "0.2.5" log = "0.4.14" -#newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } rand = "0.8.3" regex = "1.5.4" time = "0.2.26" diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index d8ddd361c8..e706062ea7 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -7,8 +7,6 @@ use container_common::token::random_crockford_token::random_crockford_token; use crate::endpoints_ws::obs_twitch_thread::ObsTwitchThread; use crate::redis::obs_active_payload::ObsActivePayload; use crate::server_state::ObsGatewayServerState; -use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; -use crate::twitch::polling_websocket_client::PollingTwitchWebsocketClient; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; @@ -95,10 +93,6 @@ pub async fn obs_gateway_websocket_handler( })?; let mut client = TwitchWebsocketClient::new().unwrap(); - let token_refresher = OauthTokenRefresher::new( - twitch_user_id.get_numeric(), - &token_record.access_token, - token_record.maybe_refresh_token.as_deref()); info!("Connecting to Twitch PubSub..."); client.connect().await.unwrap(); @@ -128,7 +122,6 @@ pub async fn obs_gateway_websocket_handler( let websocket = ObsGatewayWebSocket::new( twitch_user_id.clone(), client, - token_refresher, server_state_arc ); @@ -150,12 +143,10 @@ impl ObsGatewayWebSocket { fn new( twitch_user_id: TwitchUserId, twitch_client: TwitchWebsocketClient, - oauth_token_refresher: OauthTokenRefresher, server_state: Arc, ) -> Self { let twitch_thread = Arc::new(ObsTwitchThread::new( twitch_user_id.clone(), - oauth_token_refresher, twitch_client)); Self { twitch_user_id, diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs index 73fa9101c5..140b2ca7ae 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs @@ -1,5 +1,3 @@ -use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; -use crate::twitch::polling_websocket_client::PollingTwitchWebsocketClient; use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; use futures::lock::Mutex; @@ -29,7 +27,6 @@ pub struct ObsTwitchThread { struct InnerData { twitch_user_id: TwitchUserId, - token_refresher: OauthTokenRefresher, is_connected: bool, twitch_client: TwitchWebsocketClient, last_ping: SystemTime, @@ -38,14 +35,12 @@ struct InnerData { impl ObsTwitchThread { pub fn new( twitch_user_id: TwitchUserId, - token_refresher: OauthTokenRefresher, twitch_client: TwitchWebsocketClient ) -> Self { let now = Instant::now(); Self { inner_data: Arc::new(Mutex::new(InnerData { twitch_user_id, - token_refresher, is_connected: false, twitch_client, last_ping: UNIX_EPOCH, diff --git a/crates/service/obs_gateway/src/main.rs b/crates/service/obs_gateway/src/main.rs index 0641a5239e..e53c8e74c8 100644 --- a/crates/service/obs_gateway/src/main.rs +++ b/crates/service/obs_gateway/src/main.rs @@ -32,8 +32,6 @@ use crate::endpoints::oauth_begin_redirect::oauth_begin_enroll_redirect; use crate::endpoints::oauth_end::oauth_end_enroll_from_redirect; use crate::endpoints_ws::obs_gateway_websocket_handler::obs_gateway_websocket_handler; use crate::server_state::{ObsGatewayServerState, EnvConfig, TwitchOauthSecrets, TwitchOauthTemp, BackendsConfig}; -use crate::twitch::polling_websocket_client::PollingTwitchWebsocketClient; -use crate::twitch::twitch_client_wrapper::TwitchClientWrapper; use crate::twitch::twitch_secrets::TwitchSecrets; use futures::Future; use futures::executor::ThreadPool; @@ -234,121 +232,3 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> Ok(()) } - - - -// ======================================================== - -// TODO: Temp comment out to debug actix-web vs. runtime -// -//info!("Getting app access token..."); - -////let scopes = Scope::all(); -//let scopes = vec![ -// Scope::BitsRead, -// Scope::UserReadEmail, -//]; - -//let mut twitch_client = TwitchClientWrapper::new(client_id.clone(), client_secret.clone()); -//twitch_client.request_access_token(scopes).await?; - -//info!("Getting user id ..."); - -////let user_id = twitch_client.get_user_id_from_username("testytest512").await?; -//let user_id = twitch_client.get_user_id_from_username("vocodes").await?; - -//info!("User ID: {}", user_id); - -//std::thread::sleep(Duration::from_secs(5000)); - - -// // ==================== OAUTH FLOW ==================== - - - -// -// let auth_token = user_token.access_token.as_str(); -// -// // ==================== PUBSUB SUBSCRIPTION AND MAIN LOOP ==================== -// -// let mut client = PollingTwitchWebsocketClient::new()?; -// -// println!("Connecting PubSub..."); -// client.connect().await?; -// -// println!("Connected"); -// -// //println!("Starting polling thread..."); -// //client.start_ping_thread().await; -// -// println!("Sending PING..."); -// -// client.send_ping().await?; -// -// println!("Try read next..."); -// let r = client.try_next().await?; -// println!("Result: {:?}", r); -// -// -// println!("Begin LISTEN..."); -// -// let bit_topic = pubsub::channel_bits::ChannelBitsEventsV2 { -// channel_id: user_id, -// }.into_topic(); -// -// let topics = [bit_topic]; -// -// client.listen(&auth_token, &topics).await?; -// -// println!("Try read next..."); -// let r = client.try_next().await?; -// println!("Result: {:?}", r); -// -// /* -// Result: Some(Message { data: ChannelBitsEventsV2 { topic: ChannelBitsEventsV2 { -// channel_id: 652567283 }, reply: BitsEvent { data: BitsEventData { badge_entitlement: -// Some(BadgeEntitlement { new_version: 100, previous_version: 1 }), bits_used: 100, -// channel_id: "652567283", channel_name: "vocodes", chat_message: "Cheer100 testing the cheer", -// context: Cheer, is_anonymous: false, time: "2021-09-27T04:30:53.627717085Z", -// total_bits_used: 101, user_id: "650154491", user_name: "testytest512" }, -// message_id: "793f745e-8f3e-5f71-bc41-4808c4b49a53", version: "1.0", is_anonymous: false } } }) -// -// Result: Some(Message { data: ChannelBitsEventsV2 { topic: ChannelBitsEventsV2 { -// channel_id: 652567283 }, reply: BitsEvent { data: BitsEventData { badge_entitlement: None, -// bits_used: 1, channel_id: "652567283", channel_name: "vocodes", -// chat_message: "mario: hello everybody Cheer1", context: Cheer, is_anonymous: false, -// time: "2021-09-27T04:47:32.943872952Z", total_bits_used: 102, user_id: "650154491", -// user_name: "testytest512" }, message_id: "5844328d-495b-5585-9d6c-3c99949f9a0a", version: "1.0", -// is_anonymous: false } } }) -// */ -// -// -// -// println!("Try read next..."); -// let r = client.try_next().await?; -// println!("Result: {:?}", r); -// -// println!("Sleep..."); -// std::thread::sleep(Duration::from_millis(30000)); -// -// //let text = msg.into_text()?; -// //println!("Text: {}", text); -// -// // ======================================================== - -// info!("Connecting to database..."); -// -// let db_connection_string = -// easyenv::get_env_string_or_default( -// "MYSQL_URL", -// DEFAULT_MYSQL_CONNECTION_STRING); -// -// let pool = MySqlPoolOptions::new() -// .max_connections(5) -// .connect(&db_connection_string) -// .await?; -// -// let redis_manager = RedisConnectionManager::new(redis_connection_string.clone())?; -// -// let redis_pool = r2d2::Pool::builder() -// .build(redis_manager)?; diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 6bd9029094..742600bf62 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -24,9 +24,7 @@ use crate::twitch::constants::TWITCH_PING_CADENCE; // TODO: ERROR HANDLING // TODO: ERROR HANDLING -// TODO: Timer to ping(). -// TODO: Timer to renew lease. -// TODO: Publish events back to OBS thread. +// TODO: Publish events back to OBS thread (keepalive redis key from websocket). // TODO: Disconnect when OBS is done. // TODO: Refresh oauth token. // TODO: Handle disconnects. @@ -137,24 +135,8 @@ impl TwitchPubsubUserSubscriberThread { sleep(Duration::from_secs(1)); -// match maybe_event { -// None => { -// } -// Some(ref event) => { -// warn!("event: {:?}", event); -// -// match event { -// Response::Response(_) => {} -// Response::Message { .. } => {} -// Response::Pong => {} -// Response::Reconnect => {} -// } -// } -// } - self.maybe_renew_redis_lease().unwrap(); self.maybe_send_twitch_ping().await.unwrap(); - } } } diff --git a/crates/service/obs_gateway/src/twitch/mod.rs b/crates/service/obs_gateway/src/twitch/mod.rs index 879380886c..98cfdf49c5 100644 --- a/crates/service/obs_gateway/src/twitch/mod.rs +++ b/crates/service/obs_gateway/src/twitch/mod.rs @@ -1,8 +1,6 @@ pub mod constants; pub mod oauth; -pub mod polling_websocket_client; pub mod pubsub; -pub mod twitch_client_wrapper; pub mod twitch_secrets; pub mod twitch_user_id; pub mod websocket_client; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/twitch/oauth/mod.rs b/crates/service/obs_gateway/src/twitch/oauth/mod.rs index 46b2d4c4f9..c406e2678c 100644 --- a/crates/service/obs_gateway/src/twitch/oauth/mod.rs +++ b/crates/service/obs_gateway/src/twitch/oauth/mod.rs @@ -1,2 +1 @@ -pub mod oauth_token_builder; -pub mod oauth_token_refresher; \ No newline at end of file +pub mod oauth_token_builder; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs b/crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs deleted file mode 100644 index 35d5c86149..0000000000 --- a/crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs +++ /dev/null @@ -1,24 +0,0 @@ -// TODO: I know this is something we'll need, but I haven't built it yet. -// We'll probably want to write back to the database. - -#[derive(Clone)] -pub struct OauthTokenRefresher { - user_id: u32, - auth_token: String, - refresh_token: Option, -} - -impl OauthTokenRefresher { - - pub fn new(user_id: u32, auth_token: &str, refresh_token: Option<&str>) -> Self { - Self { - user_id, - auth_token: auth_token.to_string(), - refresh_token: refresh_token.map(|str| str.to_string()), - } - } - - pub fn get_auth_token(&self) -> String { - self.auth_token.clone() - } -} diff --git a/crates/service/obs_gateway/src/twitch/polling_websocket_client.rs b/crates/service/obs_gateway/src/twitch/polling_websocket_client.rs deleted file mode 100644 index 4a4cfaf2d7..0000000000 --- a/crates/service/obs_gateway/src/twitch/polling_websocket_client.rs +++ /dev/null @@ -1,146 +0,0 @@ -use anyhow::anyhow; -use container_common::anyhow_result::AnyhowResult; -use crate::twitch::websocket_client::TwitchWebsocketClient; -use std::sync::atomic::AtomicBool; -use std::sync::{Arc, Mutex}; -use std::time::Duration; -use twitch_api2::pubsub::{Response, TopicData, Topic, Topics}; - -const PING_SECONDS : u64 = 1 * 60; // Twitch calls for every 5 minutes, we'll do every 1 minute. - -pub struct PollingTwitchWebsocketClient { - client: Arc>, - is_polling: AtomicBool, -} - -impl PollingTwitchWebsocketClient { - pub fn new() -> AnyhowResult { - Ok(Self { - client: Arc::new(Mutex::new(TwitchWebsocketClient::new()?)), - is_polling: AtomicBool::new(false), - }) - } - - pub async fn connect(&mut self) -> AnyhowResult<()> { - match self.client.lock() { - Err(e) => Err(anyhow!("unlock client err: {:?}", e)), - Ok(mut client) => { - client.connect().await - } - } - } - - pub async fn listen<'t>(&mut self, auth_token: &str, topics: &'t [Topics]) -> AnyhowResult<()> { - match self.client.lock() { - Err(e) => Err(anyhow!("unlock client err: {:?}", e)), - Ok(mut client) => { - client.listen(auth_token, topics).await - } - } - } - - pub async fn send_ping(&mut self) -> AnyhowResult<()> { - match self.client.lock() { - Err(e) => Err(anyhow!("unlock client err: {:?}", e)), - Ok(mut client) => { - client.send_ping().await - } - } - } - - /// Start a pinging background thread. - /// Per Twitch PubSub docs, - /// To keep the server from closing the connection, clients must send a PING command - /// at least once every 5 minutes. If a client does not receive a PONG message within - /// 10 seconds of issuing a PING command, it should reconnect to the server. - pub async fn start_ping_thread(&self) { - let client = self.client.clone(); - tokio::spawn(async move { - loop { - match client.lock() { - Ok(mut c) => { - // TODO: Future isn't Send - //let _r = c.send_ping().await; - // TODO: Need to read from all call sites due to ordering. - //c.read_response(); - } - Err(_) => { - // TODO - } - } - tokio::time::sleep(Duration::from_secs(PING_SECONDS)).await; - } - }); - } - - pub async fn next(&mut self) -> AnyhowResult { - match self.client.lock() { - Err(e) => Err(anyhow!("unlock client err: {:?}", e)), - Ok(mut client) => { - client.next().await - } - } - } - - pub async fn try_next(&mut self) -> AnyhowResult> { - match self.client.lock() { - Err(e) => Err(anyhow!("unlock client err: {:?}", e)), - Ok(mut client) => { - client.try_next().await - } - } - } - - /*async fn start_polling(&self) { - if self.is_polling.load(Ordering::Relaxed) { - return; - } - - tokio::spawn(async move { - self.is_polling.store(true, Ordering::Relaxed); - - let command = pubsub::channel_sub_gifts( - &topics, - "authtoken", - "random nonce string", - )?; - - }); - / - }*/ - - - /*let message = Message::Text(PING_COMMAND.to_string()); - socket.send(message).await?; - match self.read_response().await { - Err(e) => println!("Error getting PONG: {:?}", e), - Ok(typed_response) => { - match typed_response { - Response::Pong => { println!("Pong") } - Response::Reconnect => { println!("Reconnect") } - Response::Response(_0) => { println!("Sub/Unsub (pubsub?)") } - Response::Message { data } => { - match data { - TopicData::ChannelCheerEventsPublicV1 { topic, reply } => {} - TopicData::ChannelSubGiftsV1 { topic, reply } => {} - TopicData::ChannelSubscribeEventsV1 { topic, reply } => {} - TopicData::CommunityPointsChannelV1 { topic, reply } => {} - TopicData::Following { topic, reply} => {} - // Not interested in these events - TopicData::AutoModQueue { .. } => {} - TopicData::ChannelBitsBadgeUnlocks { .. } => {} - TopicData::ChannelBitsEventsV2 { .. } => {} - TopicData::ChannelPointsChannelV1 { .. } => {} - TopicData::ChatModeratorActions { .. } => {} - TopicData::HypeTrainEventsV1 { .. } => {} - TopicData::HypeTrainEventsV1Rewards { .. } => {} - TopicData::Raid { .. } => {} - TopicData::UserModerationNotifications { .. } => {} - TopicData::VideoPlayback { .. } => {} - TopicData::VideoPlaybackById { .. } => {} - } - } - } - } - }*/ -} diff --git a/crates/service/obs_gateway/src/twitch/twitch_client_wrapper.rs b/crates/service/obs_gateway/src/twitch/twitch_client_wrapper.rs deleted file mode 100644 index 881584e87b..0000000000 --- a/crates/service/obs_gateway/src/twitch/twitch_client_wrapper.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! The Helix client is independent of PubSub -//! This just wraps `twitch_api2` for more convenience. - -use anyhow::anyhow; -use container_common::anyhow_result::AnyhowResult; -use twitch_api2::TwitchClient; -use twitch_api2::helix::users::GetUsersRequest; -use twitch_api2::types::{Nickname, UserId}; -use twitch_oauth2::{ClientSecret, ClientId, AccessToken, AppAccessToken, Scope}; - -pub struct TwitchClientWrapper<'a> { - client_id: ClientId, - client_secret: ClientSecret, - twitch_client: TwitchClient<'a, reqwest::Client>, - access_token: Option, -} - -impl <'a> TwitchClientWrapper <'a> { - pub fn new(client_id: ClientId, client_secret: ClientSecret) -> Self { - Self { - client_id, - client_secret, - twitch_client: TwitchClient::default(), - access_token: None, - } - } - - pub async fn request_access_token(&mut self, scopes: Vec) -> AnyhowResult<()> { - self.get_access_token(scopes).await?; - Ok(()) - } - - async fn get_access_token(&mut self, scopes: Vec) -> AnyhowResult<()> { - let mut http_client = reqwest::Client::new(); - let access_token = twitch_oauth2::AppAccessToken::get_app_access_token( - &http_client, - self.client_id.clone(), - self.client_secret.clone(), - scopes, - ).await?; - - self.access_token = Some(access_token); - Ok(()) - } - - /// We frequently need to look up IDs from usernames. - pub async fn get_user_id_from_username(&self, username: &str) -> AnyhowResult { - let access_token = match self.access_token { - None => return Err(anyhow!("no access token has been granted")), - Some(ref token) => token, - }; - - let req = GetUsersRequest::builder() - .login(vec![Nickname::new(username)]) - .build(); - - let response = &self.twitch_client.helix.req_get(req, access_token) - .await?; - - match response.data.get(0) { - None => Err(anyhow!("No such user: `{}`.", username)), - Some(user) => Ok(user.id.as_str().parse::()?), - } - } -} diff --git a/crates/service/obs_gateway/src/twitch_pubsub_job.rs b/crates/service/obs_gateway/src/twitch_pubsub_job.rs index 58920e2a59..d280de4c47 100644 --- a/crates/service/obs_gateway/src/twitch_pubsub_job.rs +++ b/crates/service/obs_gateway/src/twitch_pubsub_job.rs @@ -14,22 +14,20 @@ use config::shared_constants::{DEFAULT_RUST_LOG, DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING, DEFAULT_MYSQL_CONNECTION_STRING}; use container_common::anyhow_result::AnyhowResult; +use crate::threads::listen_for_active_obs_sessions_thread::listen_for_active_obs_session_thread; +use crate::twitch::websocket_client::TwitchWebsocketClient; use futures::executor::{ThreadPool, ThreadPoolBuilder}; -use r2d2_redis::r2d2; +use futures::task::SpawnExt; use log::info; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2::Pool; +use r2d2_redis::r2d2; +use redis_common::redis_keys::RedisKeys; +use sqlx::mysql::MySqlPoolOptions; +use std::sync::Arc; use std::thread::sleep; use std::time::Duration; -use crate::twitch::twitch_client_wrapper::TwitchClientWrapper; -use crate::twitch::websocket_client::TwitchWebsocketClient; -use futures::task::SpawnExt; use tokio::runtime::{Builder, Runtime}; -use r2d2_redis::RedisConnectionManager; -use std::sync::Arc; -use r2d2_redis::r2d2::Pool; -use redis_common::redis_keys::RedisKeys; -use crate::threads::listen_for_active_obs_sessions_thread::listen_for_active_obs_session_thread; -use sqlx::mysql::MySqlPoolOptions; - pub mod redis; pub mod threads; pub mod twitch; From 51f6e8229ff2de32478f30d1bc095180f026c86c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 21 Jan 2022 09:50:52 -0500 Subject: [PATCH 0799/1042] thread id stuff --- crates/lib/container_common/src/lib.rs | 1 + crates/lib/container_common/src/thread/mod.rs | 1 + .../container_common/src/thread/thread_id.rs | 59 +++++++++++++++++++ .../obs_gateway/src/redis/lease_payload.rs | 20 +++++-- .../listen_for_active_obs_sessions_thread.rs | 14 ++++- .../twitch_pubsub_user_subscriber_thread.rs | 12 +++- .../obs_gateway/src/twitch_pubsub_job.rs | 19 +++--- 7 files changed, 111 insertions(+), 15 deletions(-) create mode 100644 crates/lib/container_common/src/thread/mod.rs create mode 100644 crates/lib/container_common/src/thread/thread_id.rs diff --git a/crates/lib/container_common/src/lib.rs b/crates/lib/container_common/src/lib.rs index 9faddc2bb5..ed39b1dd25 100644 --- a/crates/lib/container_common/src/lib.rs +++ b/crates/lib/container_common/src/lib.rs @@ -1,2 +1,3 @@ pub mod anyhow_result; +pub mod thread; pub mod token; diff --git a/crates/lib/container_common/src/thread/mod.rs b/crates/lib/container_common/src/thread/mod.rs new file mode 100644 index 0000000000..4a3d1cc9ff --- /dev/null +++ b/crates/lib/container_common/src/thread/mod.rs @@ -0,0 +1 @@ +pub mod thread_id; \ No newline at end of file diff --git a/crates/lib/container_common/src/thread/thread_id.rs b/crates/lib/container_common/src/thread/thread_id.rs new file mode 100644 index 0000000000..8b33d508e6 --- /dev/null +++ b/crates/lib/container_common/src/thread/thread_id.rs @@ -0,0 +1,59 @@ +use crate::token::random_crockford_token::random_crockford_token; + +/// Thread string identifiers +/// Used for leases, logging, etc. +/// This has nothing to do with actual OS thread names or IDs. +/// The `std::thread::ThreadId` type might be better for some cases. +#[derive(Eq, PartialEq, Debug, Clone)] +pub struct ThreadId { + id: String, +} + +impl ThreadId { + pub fn random_id() -> Self { + Self { + id: random_crockford_token(10), + } + } + + pub fn with_id(id: &str) -> Self { + Self { + id: id.to_string(), + } + } + + pub fn get_id(&self) -> &str { + &self.id + } +} + +#[cfg(test)] +mod tests { + use crate::thread::thread_id::ThreadId; + + #[test] + fn equals() { + let a = ThreadId::with_id("foo"); + let b = ThreadId::with_id("foo"); + assert_eq!(a, b); + } + + #[test] + fn not_equals() { + let a = ThreadId::with_id("foo"); + let b = ThreadId::with_id("bar"); + assert_ne!(a, b); + } + + #[test] + fn manual_id() { + let t = ThreadId::with_id("foo"); + assert_eq!("foo", t.get_id()); + } + + #[test] + fn random_id() { + let t = ThreadId::random_id(); + assert_eq!(t.get_id().len(), 10); + } +} \ No newline at end of file diff --git a/crates/service/obs_gateway/src/redis/lease_payload.rs b/crates/service/obs_gateway/src/redis/lease_payload.rs index f8aaa5396e..fd06f639fa 100644 --- a/crates/service/obs_gateway/src/redis/lease_payload.rs +++ b/crates/service/obs_gateway/src/redis/lease_payload.rs @@ -1,15 +1,27 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; +use container_common::thread::thread_id::ThreadId; +use container_common::token::random_crockford_token::random_crockford_token; +// TODO: This should be JSON for future compat. /// Lease values stored in Redis are in the form "{server_id}:{thread_id}" #[derive(Debug)] pub struct LeasePayload { - server_id: String, - thread_id: String, + pub server_id: String, + pub thread_id: String, } impl LeasePayload { - pub fn new(server_id: &str, thread_id: &str) -> Self { + pub fn with_random_thread_id(server_id: &str) -> Self { + let thread_id = random_crockford_token(10); + Self::from_string_id(server_id, &thread_id) + } + + pub fn from_thread_id(server_id: &str, thread_id: &ThreadId) -> Self { + Self::from_string_id(server_id, thread_id.get_id()) + } + + pub fn from_string_id(server_id: &str, thread_id: &str) -> Self { Self { server_id: server_id.to_string(), thread_id: thread_id.to_string(), @@ -29,7 +41,7 @@ impl LeasePayload { let pair = (pieces.get(0), pieces.get(1)); if let (Some(k), Some(v)) = pair { - return Ok(Self::new(k, v)) + return Ok(Self::from_string_id(k, v)) } else { return Err(anyhow!("Invalid payload: {}", payload)); } diff --git a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs index 623afec364..a8242df1d5 100644 --- a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs +++ b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use tokio::runtime::Runtime; +use container_common::thread::thread_id::ThreadId; // TODO: Apart from error handling, this looks mostly good. @@ -23,6 +24,7 @@ pub async fn listen_for_active_obs_session_thread( redis_pool: Arc>, redis_pubsub_pool: Arc>, runtime: Arc, + hostname: String, ) { // TODO: ERROR HANDLING // TODO: ERROR HANDLING @@ -55,8 +57,8 @@ pub async fn listen_for_active_obs_session_thread( info!("No existing lease for {:?}...", &lease_key); - // TODO: Thread and server ids. - let lease = LeasePayload::new("foo", "bar"); + let thread_id = ThreadId::random_id(); + let lease = LeasePayload::from_thread_id(&hostname, &thread_id); let serialized = lease.serialize(); let _v : Option = redis.set_ex( @@ -70,7 +72,13 @@ pub async fn listen_for_active_obs_session_thread( let redis_pool2 = redis_pool.clone(); let mysql_pool2 = mysql_pool.clone(); - let thread = TwitchPubsubUserSubscriberThread::new(twitch_user_id, mysql_pool2, redis_pool2); + let thread = TwitchPubsubUserSubscriberThread::new( + twitch_user_id, + mysql_pool2, + redis_pool2, + &hostname, + thread_id.clone()); + runtime.spawn(thread.start_thread()); } } diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 742600bf62..d1548a987f 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -18,6 +18,7 @@ use twitch_api2::pubsub::Response; use time::Instant; use container_common::anyhow_result::AnyhowResult; use crate::twitch::constants::TWITCH_PING_CADENCE; +use container_common::thread::thread_id::ThreadId; // TODO: ERROR HANDLING // TODO: ERROR HANDLING @@ -32,6 +33,8 @@ use crate::twitch::constants::TWITCH_PING_CADENCE; pub struct TwitchPubsubUserSubscriberThread { + thread_id: ThreadId, + server_hostname: String, twitch_user_id: TwitchUserId, mysql_pool: Arc>, redis_pool: Arc>, @@ -56,9 +59,13 @@ impl TwitchPubsubUserSubscriberThread { twitch_user_id: TwitchUserId, mysql_pool: Arc>, redis_pool: Arc>, + server_hostname: &str, + thread_id: ThreadId, ) -> Self { let twitch_websocket_client = TwitchWebsocketClient::new().unwrap(); Self { + thread_id, + server_hostname: server_hostname.to_string(), twitch_user_id, mysql_pool, redis_pool, @@ -158,8 +165,11 @@ impl TwitchPubsubUserSubscriberThread { fn renew_redis_lease(&mut self) -> AnyhowResult<()> { // TODO: Error handling let mut redis = self.redis_pool.get().unwrap(); + let lease_key = RedisKeys::twitch_pubsub_lease(self.twitch_user_id.get_str()); - let lease_value = LeasePayload::new("foo", "bar").serialize(); + let lease_value = LeasePayload::from_thread_id(&self.server_hostname, &self.thread_id) + .serialize(); + let _v : Option = redis.set_ex( &lease_key, &lease_value, diff --git a/crates/service/obs_gateway/src/twitch_pubsub_job.rs b/crates/service/obs_gateway/src/twitch_pubsub_job.rs index d280de4c47..31a20bdab0 100644 --- a/crates/service/obs_gateway/src/twitch_pubsub_job.rs +++ b/crates/service/obs_gateway/src/twitch_pubsub_job.rs @@ -12,6 +12,11 @@ #[macro_use] extern crate magic_crypt; #[macro_use] extern crate serde_derive; +pub mod redis; +pub mod threads; +pub mod twitch; +pub mod util; + use config::shared_constants::{DEFAULT_RUST_LOG, DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING, DEFAULT_MYSQL_CONNECTION_STRING}; use container_common::anyhow_result::AnyhowResult; use crate::threads::listen_for_active_obs_sessions_thread::listen_for_active_obs_session_thread; @@ -28,15 +33,16 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use tokio::runtime::{Builder, Runtime}; -pub mod redis; -pub mod threads; -pub mod twitch; -pub mod util; #[tokio::main] pub async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + let server_hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or("twitch-pubsub-job".to_string()); + let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", @@ -106,10 +112,9 @@ pub async fn main() -> AnyhowResult<()> { info!("Thread pool created"); - //runtime.spawn(watch_user_thread(10)); - //runtime.spawn(watch_user_thread(9999)); runtime.spawn(listen_for_active_obs_session_thread( - mysql_pool, redis_pool, redis_pubsub_pool, runtime_2)); + mysql_pool, redis_pool, redis_pubsub_pool, runtime_2, + server_hostname.to_string())); loop { sleep(Duration::from_millis(10_000)); From dc3d3cc24ab807dfb4e2c4930729d5c6b0a459c0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 21 Jan 2022 10:00:15 -0500 Subject: [PATCH 0800/1042] update main thread to struct --- .../listen_for_active_obs_sessions_thread.rs | 117 +++++++++++------- .../twitch_pubsub_user_subscriber_thread.rs | 1 + .../obs_gateway/src/twitch_pubsub_job.rs | 14 ++- 3 files changed, 80 insertions(+), 52 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs index a8242df1d5..28b0b9945f 100644 --- a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs +++ b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs @@ -19,66 +19,87 @@ use container_common::thread::thread_id::ThreadId; // TODO: Apart from error handling, this looks mostly good. -pub async fn listen_for_active_obs_session_thread( + +pub struct ListenForActiveObsSessionThread { + server_hostname: String, mysql_pool: Arc>, redis_pool: Arc>, redis_pubsub_pool: Arc>, runtime: Arc, - hostname: String, -) { - // TODO: ERROR HANDLING - // TODO: ERROR HANDLING - // TODO: ERROR HANDLING +} - // Setup PubSub. - - let mut pubsub_pool = redis_pubsub_pool.get().unwrap(); - let mut pubsub = pubsub_pool.as_pubsub(); - - pubsub.subscribe(RedisKeys::obs_active_session_topic()).unwrap(); - - loop { - let message = pubsub.get_message().unwrap(); - let payload : String = message.get_payload().unwrap(); - let payload = ObsActivePayload::from_json_str(&payload).unwrap(); - - let twitch_user_id = TwitchUserId::from_str(&payload.twitch_user_id).unwrap(); - - // See if we have a lease already... - - let mut redis = redis_pool.get().unwrap(); - let lease_key = RedisKeys::twitch_pubsub_lease(&payload.twitch_user_id); - let lease_value : Option = redis.get(&lease_key).unwrap(); - - if let Some(value) = lease_value.as_deref() { - let lease = LeasePayload::deserialize(value).unwrap(); - continue; +impl ListenForActiveObsSessionThread { + pub fn new( + server_hostname: String, + mysql_pool: Arc>, + redis_pool: Arc>, + redis_pubsub_pool: Arc>, + runtime: Arc, + ) -> Self { + Self { + server_hostname, + mysql_pool, + redis_pool, + redis_pubsub_pool, + runtime, } + } - info!("No existing lease for {:?}...", &lease_key); + pub async fn start_thread(mut self) { + // TODO: ERROR HANDLING + // TODO: ERROR HANDLING + // TODO: ERROR HANDLING - let thread_id = ThreadId::random_id(); - let lease = LeasePayload::from_thread_id(&hostname, &thread_id); + // Setup PubSub. - let serialized = lease.serialize(); - let _v : Option = redis.set_ex( - &lease_key, - &serialized, - LEASE_TIMEOUT_SECONDS - ).unwrap(); + let mut pubsub_pool = self.redis_pubsub_pool.get().unwrap(); + let mut pubsub = pubsub_pool.as_pubsub(); - // Then launch the thread... + pubsub.subscribe(RedisKeys::obs_active_session_topic()).unwrap(); - let redis_pool2 = redis_pool.clone(); - let mysql_pool2 = mysql_pool.clone(); + loop { + let message = pubsub.get_message().unwrap(); + let payload : String = message.get_payload().unwrap(); + let payload = ObsActivePayload::from_json_str(&payload).unwrap(); - let thread = TwitchPubsubUserSubscriberThread::new( - twitch_user_id, - mysql_pool2, - redis_pool2, - &hostname, - thread_id.clone()); + let twitch_user_id = TwitchUserId::from_str(&payload.twitch_user_id).unwrap(); - runtime.spawn(thread.start_thread()); + // See if we have a lease already... + + let mut redis = self.redis_pool.get().unwrap(); + let lease_key = RedisKeys::twitch_pubsub_lease(&payload.twitch_user_id); + let lease_value : Option = redis.get(&lease_key).unwrap(); + + if let Some(value) = lease_value.as_deref() { + let lease = LeasePayload::deserialize(value).unwrap(); + continue; + } + + info!("No existing lease for {:?}...", &lease_key); + + let thread_id = ThreadId::random_id(); + let lease = LeasePayload::from_thread_id(&self.server_hostname, &thread_id); + + let serialized = lease.serialize(); + let _v : Option = redis.set_ex( + &lease_key, + &serialized, + LEASE_TIMEOUT_SECONDS + ).unwrap(); + + // Then launch the thread... + + let redis_pool2 = self.redis_pool.clone(); + let mysql_pool2 = self.mysql_pool.clone(); + + let thread = TwitchPubsubUserSubscriberThread::new( + twitch_user_id, + mysql_pool2, + redis_pool2, + &self.server_hostname, + thread_id.clone()); + + self.runtime.spawn(thread.start_thread()); + } } } diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index d1548a987f..63ef01c019 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -36,6 +36,7 @@ pub struct TwitchPubsubUserSubscriberThread { thread_id: ThreadId, server_hostname: String, twitch_user_id: TwitchUserId, + mysql_pool: Arc>, redis_pool: Arc>, twitch_websocket_client: TwitchWebsocketClient, diff --git a/crates/service/obs_gateway/src/twitch_pubsub_job.rs b/crates/service/obs_gateway/src/twitch_pubsub_job.rs index 31a20bdab0..dab36bd0f9 100644 --- a/crates/service/obs_gateway/src/twitch_pubsub_job.rs +++ b/crates/service/obs_gateway/src/twitch_pubsub_job.rs @@ -19,7 +19,7 @@ pub mod util; use config::shared_constants::{DEFAULT_RUST_LOG, DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING, DEFAULT_MYSQL_CONNECTION_STRING}; use container_common::anyhow_result::AnyhowResult; -use crate::threads::listen_for_active_obs_sessions_thread::listen_for_active_obs_session_thread; +use crate::threads::listen_for_active_obs_sessions_thread::ListenForActiveObsSessionThread; use crate::twitch::websocket_client::TwitchWebsocketClient; use futures::executor::{ThreadPool, ThreadPoolBuilder}; use futures::task::SpawnExt; @@ -112,9 +112,15 @@ pub async fn main() -> AnyhowResult<()> { info!("Thread pool created"); - runtime.spawn(listen_for_active_obs_session_thread( - mysql_pool, redis_pool, redis_pubsub_pool, runtime_2, - server_hostname.to_string())); + let thread = ListenForActiveObsSessionThread::new( + server_hostname.to_string(), + mysql_pool, + redis_pool, + redis_pubsub_pool, + runtime_2, + ); + + runtime.spawn(thread.start_thread()); loop { sleep(Duration::from_millis(10_000)); From d4e90b9ec81dc464793666007621080f3b6bacf7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 21 Jan 2022 10:10:08 -0500 Subject: [PATCH 0801/1042] docs, logging --- .../twitch_pubsub_user_subscriber_thread.rs | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 63ef01c019..20d6bdb253 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -1,9 +1,14 @@ +use container_common::anyhow_result::AnyhowResult; +use container_common::thread::thread_id::ThreadId; use crate::redis::lease_payload::LeasePayload; use crate::redis::lease_timeout::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD}; +use crate::twitch::constants::TWITCH_PING_CADENCE; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; use database_queries::twitch_oauth::find::{TwitchOauthTokenRecord, TwitchOauthTokenFinder}; +use log::debug; +use log::error; use log::info; use log::warn; use r2d2_redis::RedisConnectionManager; @@ -14,11 +19,8 @@ use sqlx::MySql; use std::sync::Arc; use std::thread::sleep; use std::time::Duration; -use twitch_api2::pubsub::Response; use time::Instant; -use container_common::anyhow_result::AnyhowResult; -use crate::twitch::constants::TWITCH_PING_CADENCE; -use container_common::thread::thread_id::ThreadId; +use twitch_api2::pubsub::Response; // TODO: ERROR HANDLING // TODO: ERROR HANDLING @@ -84,47 +86,45 @@ impl TwitchPubsubUserSubscriberThread { let oauth_lookup_result = self.lookup_oauth_record().await; let oauth_token_record = match oauth_lookup_result { Err(e) => { - info!("Error looking up oauth creds: {:?}", e); + error!("Error looking up oauth creds: {:?}", e); return; // TODO: Better flow. } Ok(None) => { - info!("Twitch user oauth token does not exit. Auth please."); + error!("Twitch user oauth token does not exit. Auth please."); return; // TODO: Better flow. } Ok(Some(record)) => record, }; - warn!("Connecting to Twitch PubSub for user {}...", &oauth_token_record.twitch_username); + info!("Connecting to Twitch PubSub for user {}...", &oauth_token_record.twitch_username); self.twitch_websocket_client.connect().await.unwrap(); // TODO: Error handling loop { self.maybe_renew_redis_lease().unwrap(); self.maybe_send_twitch_ping().await.unwrap(); - warn!("Listen to user..."); + info!("Listen to user..."); let topics = build_pubsub_topics_for_user(self.twitch_user_id.get_numeric()); self.twitch_websocket_client.listen(&oauth_token_record.access_token, &topics).await.unwrap(); loop { - //let maybe_event = self.twitch_websocket_client.try_next() - // .await - // .unwrap(); + //debug!("[{}] maybe event for user {}...", self.thread_id.get_id(), self.twitch_user_id.get_str()); - info!("maybe event..."); let mut interval = tokio::time::interval(Duration::from_millis(1000)); - // NB: We can't block forever. - // Adapted from https://github.com/snapview/tokio-tungstenite/blob/master/examples/interval-server.rs + // Adapted from the very good tokio-tungstenite example here, which also splits sockets + // into bidirectional streams: + // https://github.com/snapview/tokio-tungstenite/blob/master/examples/interval-server.rs tokio::select! { maybe_event = self.twitch_websocket_client.try_next() => { match maybe_event { Err(e) => { - warn!("socket recv error: {:?}", e); + error!("socket recv error: {:?}", e); } Ok(None) => {}, Ok(Some(ref event)) => { - warn!("event: {:?}", event); + info!("event: {:?}", event); match event { Response::Response(_) => {} @@ -136,7 +136,6 @@ impl TwitchPubsubUserSubscriberThread { } } _ = interval.tick() => { - //ws_sender.send(Message::Text("tick".to_owned())).await?; sleep(Duration::from_secs(5)); } } From 2011f31e709dd35ffff51e0045dec03848aa0b05 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 21 Jan 2022 10:31:38 -0500 Subject: [PATCH 0802/1042] Check lease for expiry --- .../obs_gateway/src/redis/lease_payload.rs | 32 ++++++++- .../obs_gateway/src/redis/lease_timeout.rs | 5 +- .../twitch_pubsub_user_subscriber_thread.rs | 65 +++++++++++++++++-- 3 files changed, 94 insertions(+), 8 deletions(-) diff --git a/crates/service/obs_gateway/src/redis/lease_payload.rs b/crates/service/obs_gateway/src/redis/lease_payload.rs index fd06f639fa..f1305f7354 100644 --- a/crates/service/obs_gateway/src/redis/lease_payload.rs +++ b/crates/service/obs_gateway/src/redis/lease_payload.rs @@ -5,7 +5,7 @@ use container_common::token::random_crockford_token::random_crockford_token; // TODO: This should be JSON for future compat. /// Lease values stored in Redis are in the form "{server_id}:{thread_id}" -#[derive(Debug)] +#[derive(Debug, Eq, PartialEq, Clone)] pub struct LeasePayload { pub server_id: String, pub thread_id: String, @@ -47,3 +47,33 @@ impl LeasePayload { } } } + +#[cfg(test)] +mod tests { + use container_common::thread::thread_id::ThreadId; + use crate::redis::lease_payload::LeasePayload; + + #[test] + fn equals() { + let a = LeasePayload::from_string_id("foo", "bar"); + let b = LeasePayload::from_string_id("foo", "bar"); + assert_eq!(a, b); + let t = ThreadId::with_id("bar"); + let b = LeasePayload::from_thread_id("foo", &t); + assert_eq!(a, b); + } + + #[test] + fn not_equals() { + let a = LeasePayload::from_string_id("foo", "bar"); + let b = LeasePayload::from_string_id("asdf", "qwerty"); + assert_ne!(a, b); + let b = LeasePayload::from_string_id("foo", "ASDF"); + assert_ne!(a, b); + let b = LeasePayload::from_string_id("ASDF", "bar"); + assert_ne!(a, b); + let t = ThreadId::with_id("ASDF"); + let b = LeasePayload::from_thread_id("foo", &t); + assert_ne!(a, b); + } +} diff --git a/crates/service/obs_gateway/src/redis/lease_timeout.rs b/crates/service/obs_gateway/src/redis/lease_timeout.rs index ed89179035..c8bef35d1e 100644 --- a/crates/service/obs_gateway/src/redis/lease_timeout.rs +++ b/crates/service/obs_gateway/src/redis/lease_timeout.rs @@ -9,4 +9,7 @@ use time::Duration; pub const LEASE_TIMEOUT_SECONDS : usize = 60; /// The amount of time to wait before renewing our lease. -pub const LEASE_RENEW_PERIOD: Duration = Duration::seconds(30); +pub const LEASE_RENEW_PERIOD : Duration = Duration::seconds(30); + +/// The amount of time to wait before checking or lease is valid. +pub const LEASE_CHECK_PERIOD : Duration = Duration::seconds(15); diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 20d6bdb253..c8a5ea7383 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -1,7 +1,7 @@ use container_common::anyhow_result::AnyhowResult; use container_common::thread::thread_id::ThreadId; use crate::redis::lease_payload::LeasePayload; -use crate::redis::lease_timeout::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD}; +use crate::redis::lease_timeout::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD, LEASE_CHECK_PERIOD}; use crate::twitch::constants::TWITCH_PING_CADENCE; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; @@ -27,17 +27,17 @@ use twitch_api2::pubsub::Response; // TODO: ERROR HANDLING // TODO: ERROR HANDLING -// TODO: Publish events back to OBS thread (keepalive redis key from websocket). -// TODO: Disconnect when OBS is done. +// TODO: Publish events back to OBS thread +// TODO: Disconnect when OBS is done. (keepalive redis key from websocket). // TODO: Refresh oauth token. // TODO: Handle disconnects. // TODO: Server+thread IDs - pub struct TwitchPubsubUserSubscriberThread { thread_id: ThreadId, server_hostname: String, twitch_user_id: TwitchUserId, + expected_lease_payload: LeasePayload, mysql_pool: Arc>, redis_pool: Arc>, @@ -49,6 +49,10 @@ pub struct TwitchPubsubUserSubscriberThread { // If the lease gets taken by another, we abandon our own workload. redis_lease_last_renewed_at: Option, + // If the lease gets taken by another thread, we abandon our own workload. + // This controls when we periodically check. + redis_lease_last_checked_at: Option, + /// Twitch PubSub requires PINGs at regular intervals, /// "To keep the server from closing the connection, clients must send a PING /// command at least once every 5 minutes. If a client does not receive a PONG @@ -66,14 +70,17 @@ impl TwitchPubsubUserSubscriberThread { thread_id: ThreadId, ) -> Self { let twitch_websocket_client = TwitchWebsocketClient::new().unwrap(); + let expected_lease_payload = LeasePayload::from_thread_id(&server_hostname, &thread_id); Self { thread_id, + expected_lease_payload, server_hostname: server_hostname.to_string(), twitch_user_id, mysql_pool, redis_pool, twitch_websocket_client, redis_lease_last_renewed_at: None, + redis_lease_last_checked_at: None, twitch_last_pinged_at: None, } } @@ -81,6 +88,7 @@ impl TwitchPubsubUserSubscriberThread { pub async fn start_thread(mut self) { // We lease from outside before starting the thread. self.redis_lease_last_renewed_at = Some(Instant::now()); + self.redis_lease_last_checked_at = Some(Instant::now()); // Lookup the OAuth credentials let oauth_lookup_result = self.lookup_oauth_record().await; @@ -142,12 +150,58 @@ impl TwitchPubsubUserSubscriberThread { sleep(Duration::from_secs(1)); + let is_valid = self.maybe_check_redis_lease_is_valid().unwrap(); + if !is_valid { + warn!("Exiting thread."); + return; // Exit thread + } + self.maybe_renew_redis_lease().unwrap(); self.maybe_send_twitch_ping().await.unwrap(); } } } + fn maybe_check_redis_lease_is_valid(&mut self) -> AnyhowResult { + let mut should_check_lease = self.redis_lease_last_checked_at + .map(|last_read| last_read.elapsed().gt(&LEASE_CHECK_PERIOD)) + .unwrap_or(true); + + if should_check_lease { + info!("Checking Redis Lease for user {}", self.twitch_user_id.get_numeric()); + let is_valid = self.check_redis_lease_is_valid()?; + + if !is_valid { + warn!("Lease got taken by another thread"); + return Ok(false); + } + + self.redis_lease_last_checked_at = Some(Instant::now()); + } + + Ok(true) + } + + fn check_redis_lease_is_valid(&mut self) -> AnyhowResult { + // TODO: Error handling + + let mut redis = self.redis_pool.get().unwrap(); + let lease_key = RedisKeys::twitch_pubsub_lease(self.twitch_user_id.get_str()); + + let payload : Option = redis.get(&lease_key).unwrap(); + match payload { + None => { + warn!("Redis lease payload absent. Another thread could be started."); + Ok(true) + } + Some(payload) => { + let actual_payload = LeasePayload::deserialize(&payload).unwrap(); + let equals_expected = self.expected_lease_payload.eq(&actual_payload); + Ok(equals_expected) + } + } + } + fn maybe_renew_redis_lease(&mut self) -> AnyhowResult<()> { let mut should_renew_lease = self.redis_lease_last_renewed_at .map(|last_write| last_write.elapsed().gt(&LEASE_RENEW_PERIOD)) @@ -167,8 +221,7 @@ impl TwitchPubsubUserSubscriberThread { let mut redis = self.redis_pool.get().unwrap(); let lease_key = RedisKeys::twitch_pubsub_lease(self.twitch_user_id.get_str()); - let lease_value = LeasePayload::from_thread_id(&self.server_hostname, &self.thread_id) - .serialize(); + let lease_value = self.expected_lease_payload.serialize(); let _v : Option = redis.set_ex( &lease_key, From a9dbc7303c4e546705f66a2eecae6168fbe98bf8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 21 Jan 2022 23:43:51 -0500 Subject: [PATCH 0803/1042] cleanup --- crates/lib/redis_common/src/redis_keys.rs | 15 ++- .../obs_gateway_websocket_handler.rs | 115 +----------------- .../redis/{lease_timeout.rs => constants.rs} | 4 + crates/service/obs_gateway/src/redis/mod.rs | 2 +- .../listen_for_active_obs_sessions_thread.rs | 6 +- .../twitch_pubsub_user_subscriber_thread.rs | 2 +- 6 files changed, 22 insertions(+), 122 deletions(-) rename crates/service/obs_gateway/src/redis/{lease_timeout.rs => constants.rs} (80%) diff --git a/crates/lib/redis_common/src/redis_keys.rs b/crates/lib/redis_common/src/redis_keys.rs index dbd4bd59af..42332d168f 100644 --- a/crates/lib/redis_common/src/redis_keys.rs +++ b/crates/lib/redis_common/src/redis_keys.rs @@ -47,8 +47,15 @@ impl RedisKeys { /// This is a PubSub topic. /// We publish when new viewers arrive and as a "keep-alive ping". - pub fn obs_active_session_topic() -> &'static str { - "obsActiveSessionTopic" + pub fn obs_active_sessions_topic() -> &'static str { + "obsActiveSessionsTopic" + } + + /// This is a key that denotes a user is actively on the OBS gateway. + /// These keys have a short expiry and OBS gateway must continually bump them. + /// If a key goes away, the thread monitoring that user should exit. + pub fn obs_active_session(twitch_user_id: &str) -> String { + format!("obsActiveSession:{}", twitch_user_id) } /// This is a key that denotes a PubSub lease subscriber. @@ -61,7 +68,7 @@ impl RedisKeys { /// This is a PubSub topic. /// These are "unenriched" Twitch events from the PubSub (and eventually IRC) /// subscribers. Downstream listeners will enrich these for user-facing functionality. - pub fn twitchEventTopic(twitch_user_id: &str) -> String { - format!("twitchEventTopic:{}", twitch_user_id) + pub fn twitch_events_topic(twitch_user_id: &str) -> String { + format!("twitchEventsTopic:{}", twitch_user_id) } } diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index e706062ea7..1b8d35cd9b 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -77,7 +77,7 @@ pub async fn obs_gateway_websocket_handler( CommonServerError::ServerError })?; - let channel = RedisKeys::obs_active_session_topic(); + let channel = RedisKeys::obs_active_sessions_topic(); let payload = ObsActivePayload::new(&token_record.twitch_user_id); let json_payload = payload.serialize() @@ -92,36 +92,12 @@ pub async fn obs_gateway_websocket_handler( CommonServerError::ServerError })?; - let mut client = TwitchWebsocketClient::new().unwrap(); - - info!("Connecting to Twitch PubSub..."); - client.connect().await.unwrap(); - -// info!("Connected to Twitch PubSub"); -// -// info!("Sending Twitch PubSub PING..."); -// client.send_ping().await.unwrap(); -// -// info!("Try read next from Twitch PubSub..."); -// let r = client.try_next().await.unwrap(); -// info!("Twitch PubSub Result: {:?}", r); -// -// info!("Begin TwitchPubSub LISTEN on authenticated OAuth topics..."); -// let topics = build_pubsub_topics_for_user(user_id); -// client.listen(&auth_token, &topics).await.unwrap(); - - //let (tx, rx) = crossbeam::channel::bounded(1); - //let client = Arc::new(&self.twitch_client); - //let client2 = client.clone(); - //let res = rx.recv().unwrap(); - info!("Begin Javascript WebSocket..."); let server_state_arc = server_state.get_ref().clone(); let websocket = ObsGatewayWebSocket::new( twitch_user_id.clone(), - client, server_state_arc ); @@ -135,22 +111,16 @@ pub async fn obs_gateway_websocket_handler( /// Websocket behavior struct ObsGatewayWebSocket { twitch_user_id: TwitchUserId, - twitch_thread: Arc, server_state: Arc, } impl ObsGatewayWebSocket { fn new( twitch_user_id: TwitchUserId, - twitch_client: TwitchWebsocketClient, server_state: Arc, ) -> Self { - let twitch_thread = Arc::new(ObsTwitchThread::new( - twitch_user_id.clone(), - twitch_client)); Self { twitch_user_id, - twitch_thread, server_state, } } @@ -160,29 +130,6 @@ impl Actor for ObsGatewayWebSocket { type Context = ws::WebsocketContext; fn started(&mut self, _ctx: &mut Self::Context) { - let handle = Handle::current(); - let twitch_thread = self.twitch_thread.clone(); - - - //let future = self.twitch_thread.run_until_exit(); - ////let now_future = Delay::new(Duration::from_secs(5)); - //warn!("Starting thread..."); - ////actix_rt::spawn - ////actix_rt::spawn(twitch_thread.run_until_exit()); - - //actix_rt::spawn(future.map(|x| { - // println!("waited for 5 secs"); - //})); - - //handle.spawn_blocking(result); -// handle.spawn_blocking(async { -// warn!("inside thread 1"); -// let twitch_thread2 = twitch_thread.clone(); -// //async move { -// // warn!("inside thread 2"); -// // twitch_thread2.run_until_exit().await; -// //}.await; -// }); warn!("Thread started"); } } @@ -194,29 +141,6 @@ impl StreamHandler> for ObsGatewayWebSock ctx: &mut Self::Context, ) { if let Ok(msg) = msg { - //info!(">>>>>> obs streamhandler::handle (msg = {:?})", msg); - - //let (tx, rx) = crossbeam::channel::bounded(1); - //let handle = Handle::current(); - - //let client = Arc::new(&self.twitch_client); - //let client2 = client.clone(); - - //handle.spawn(async { - // error!("Twitch PubSub Try read next..."); - // match client2.try_next().await { - // Ok(r) => { - // error!("Twitch PubSub Result: {:?}", r); - // }, - // Err(e) => { - // warn!("pubsub error: {:?}", e); - // } - // } - //}); - - //let res = rx.recv().unwrap(); - - //info!("process message: {:?}", &msg); // TODO: Only send this every 60 seconds. // TODO: Error handling. @@ -230,7 +154,7 @@ impl StreamHandler> for ObsGatewayWebSock CommonServerError::ServerError }).unwrap(); // TODO: FIXME - let channel = RedisKeys::obs_active_session_topic(); + let channel = RedisKeys::obs_active_sessions_topic(); let payload = ObsActivePayload::new(self.twitch_user_id.get_str()); @@ -266,38 +190,3 @@ impl StreamHandler> for ObsGatewayWebSock } } } - -// From the Actix homepage: - -//use actix::{Actor, StreamHandler}; -//use actix_web::{web, App, Error, HttpRequest, HttpResponse, HttpServer}; -//use actix_web_actors::ws; -// -///// Define HTTP actor -//struct MyWs; -// -//impl Actor for MyWs { -// type Context = ws::WebsocketContext; -//} -// -///// Handler for ws::Message message -//impl StreamHandler> for MyWs { -// fn handle( -// &mut self, -// msg: Result, -// ctx: &mut Self::Context, -// ) { -// match msg { -// Ok(ws::Message::Ping(msg)) => ctx.pong(&msg), -// Ok(ws::Message::Text(text)) => ctx.text(text), -// Ok(ws::Message::Binary(bin)) => ctx.binary(bin), -// _ => (), -// } -// } -//} -// -//pub async fn twitch_pubsub_gateway(req: HttpRequest, stream: web::Payload) -> Result { -// let resp = ws::start(MyWs {}, &req, stream); -// println!("{:?}", resp); -// resp -//} diff --git a/crates/service/obs_gateway/src/redis/lease_timeout.rs b/crates/service/obs_gateway/src/redis/constants.rs similarity index 80% rename from crates/service/obs_gateway/src/redis/lease_timeout.rs rename to crates/service/obs_gateway/src/redis/constants.rs index c8bef35d1e..e4410d9ca6 100644 --- a/crates/service/obs_gateway/src/redis/lease_timeout.rs +++ b/crates/service/obs_gateway/src/redis/constants.rs @@ -13,3 +13,7 @@ pub const LEASE_RENEW_PERIOD : Duration = Duration::seconds(30); /// The amount of time to wait before checking or lease is valid. pub const LEASE_CHECK_PERIOD : Duration = Duration::seconds(15); + +/// The OBS websocket polls a key to inform the backend that the +/// user is still active. +pub const OBS_ACTIVE_TTL_SECONDS : usize = 120; diff --git a/crates/service/obs_gateway/src/redis/mod.rs b/crates/service/obs_gateway/src/redis/mod.rs index 9bc863f1de..e8a29762c6 100644 --- a/crates/service/obs_gateway/src/redis/mod.rs +++ b/crates/service/obs_gateway/src/redis/mod.rs @@ -1,3 +1,3 @@ pub mod lease_payload; -pub mod lease_timeout; +pub mod constants; pub mod obs_active_payload; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs index 28b0b9945f..7d0555ce6d 100644 --- a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs +++ b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs @@ -1,5 +1,6 @@ +use container_common::thread::thread_id::ThreadId; +use crate::redis::constants::LEASE_TIMEOUT_SECONDS; use crate::redis::lease_payload::LeasePayload; -use crate::redis::lease_timeout::LEASE_TIMEOUT_SECONDS; use crate::redis::obs_active_payload::ObsActivePayload; use crate::threads::twitch_pubsub_user_subscriber_thread::TwitchPubsubUserSubscriberThread; use crate::twitch::twitch_user_id::TwitchUserId; @@ -15,7 +16,6 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use tokio::runtime::Runtime; -use container_common::thread::thread_id::ThreadId; // TODO: Apart from error handling, this looks mostly good. @@ -55,7 +55,7 @@ impl ListenForActiveObsSessionThread { let mut pubsub_pool = self.redis_pubsub_pool.get().unwrap(); let mut pubsub = pubsub_pool.as_pubsub(); - pubsub.subscribe(RedisKeys::obs_active_session_topic()).unwrap(); + pubsub.subscribe(RedisKeys::obs_active_sessions_topic()).unwrap(); loop { let message = pubsub.get_message().unwrap(); diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index c8a5ea7383..2a798a1a91 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -1,7 +1,7 @@ use container_common::anyhow_result::AnyhowResult; use container_common::thread::thread_id::ThreadId; use crate::redis::lease_payload::LeasePayload; -use crate::redis::lease_timeout::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD, LEASE_CHECK_PERIOD}; +use crate::redis::constants::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD, LEASE_CHECK_PERIOD}; use crate::twitch::constants::TWITCH_PING_CADENCE; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; From 1bff8de1a5595740a095d1a250d806fa5b8ae8ba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 Jan 2022 17:14:23 -0500 Subject: [PATCH 0804/1042] kill thread when OBS session goes away --- crates/lib/redis_common/src/redis_keys.rs | 4 +- .../obs_gateway_websocket_handler.rs | 60 +++++++++++++++---- crates/service/obs_gateway/src/main.rs | 3 +- .../obs_gateway/src/redis/constants.rs | 3 + .../twitch_pubsub_user_subscriber_thread.rs | 52 +++++++++++++++- 5 files changed, 103 insertions(+), 19 deletions(-) diff --git a/crates/lib/redis_common/src/redis_keys.rs b/crates/lib/redis_common/src/redis_keys.rs index 42332d168f..7fd1829978 100644 --- a/crates/lib/redis_common/src/redis_keys.rs +++ b/crates/lib/redis_common/src/redis_keys.rs @@ -54,8 +54,8 @@ impl RedisKeys { /// This is a key that denotes a user is actively on the OBS gateway. /// These keys have a short expiry and OBS gateway must continually bump them. /// If a key goes away, the thread monitoring that user should exit. - pub fn obs_active_session(twitch_user_id: &str) -> String { - format!("obsActiveSession:{}", twitch_user_id) + pub fn obs_active_session_keepalive(twitch_user_id: &str) -> String { + format!("obsActiveSessionKeepalive:{}", twitch_user_id) } /// This is a key that denotes a PubSub lease subscriber. diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 1b8d35cd9b..15a8c3189c 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -3,8 +3,10 @@ use actix_rt::Runtime; use actix_web::web::Path; use actix_web::{middleware, web, App, Error, HttpRequest, HttpResponse, HttpServer}; use actix_web_actors::ws; +use container_common::anyhow_result::AnyhowResult; use container_common::token::random_crockford_token::random_crockford_token; use crate::endpoints_ws::obs_twitch_thread::ObsTwitchThread; +use crate::redis::constants::OBS_ACTIVE_TTL_SECONDS; use crate::redis::obs_active_payload::ObsActivePayload; use crate::server_state::ObsGatewayServerState; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; @@ -17,6 +19,8 @@ use http_server_common::error::common_server_error::CommonServerError; use log::error; use log::info; use log::warn; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2::PooledConnection; use r2d2_redis::redis::Commands; use redis_common::redis_keys::RedisKeys; use std::sync::Arc; @@ -26,6 +30,8 @@ use tokio::runtime::Handle; use twitch_api2::pubsub::Topic; use twitch_api2::pubsub; +// TODO: Redis calls are synchronous (but fast), but is there any way to make them async? + #[derive(Deserialize)] pub struct PathInfo { twitch_username: String, @@ -114,23 +120,11 @@ struct ObsGatewayWebSocket { server_state: Arc, } -impl ObsGatewayWebSocket { - fn new( - twitch_user_id: TwitchUserId, - server_state: Arc, - ) -> Self { - Self { - twitch_user_id, - server_state, - } - } -} - impl Actor for ObsGatewayWebSocket { type Context = ws::WebsocketContext; fn started(&mut self, _ctx: &mut Self::Context) { - warn!("Thread started"); + info!("ObsGatewayWebSocket Actor::started()"); } } @@ -170,6 +164,7 @@ impl StreamHandler> for ObsGatewayWebSock CommonServerError::ServerError }).unwrap(); // TODO: Fixme + self.write_obs_active().unwrap(); match msg { ws::Message::Text(text) => { @@ -190,3 +185,42 @@ impl StreamHandler> for ObsGatewayWebSock } } } + +impl ObsGatewayWebSocket { + fn new( + twitch_user_id: TwitchUserId, + server_state: Arc, + ) -> Self { + Self { + twitch_user_id, + server_state, + } + } + + fn get_redis(&self) + -> AnyhowResult> + { + let redis = self.server_state.backends + .redis_pool + .get() + .map_err(|err| { + error!("Could not get Redis: {:?}", err); + CommonServerError::ServerError + }).unwrap(); // TODO: FIXME + + Ok(redis) + } + + fn write_obs_active(&self) -> AnyhowResult<()> { + let mut redis = self.get_redis().unwrap(); + let redis_key = RedisKeys::obs_active_session_keepalive(self.twitch_user_id.get_str()); + let _r : bool = redis.set_ex(redis_key, "1", OBS_ACTIVE_TTL_SECONDS) + .map_err(|e| { + warn!("redis error: {:?}", e); + CommonServerError::ServerError + }).unwrap(); // TODO: Fixme + + Ok(()) + } +} + diff --git a/crates/service/obs_gateway/src/main.rs b/crates/service/obs_gateway/src/main.rs index e53c8e74c8..69e3c2209f 100644 --- a/crates/service/obs_gateway/src/main.rs +++ b/crates/service/obs_gateway/src/main.rs @@ -27,6 +27,7 @@ use actix_web::{HttpServer, web, HttpResponse, App}; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; +use container_common::anyhow_result::AnyhowResult; use crate::endpoints::oauth_begin::oauth_begin_enroll; use crate::endpoints::oauth_begin_redirect::oauth_begin_enroll_redirect; use crate::endpoints::oauth_end::oauth_end_enroll_from_redirect; @@ -56,8 +57,6 @@ use twitch_oauth2::{AppAccessToken, Scope, TwitchToken, tokens::errors::AppAcces const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:54321"; -pub type AnyhowResult = anyhow::Result; - #[actix_web::main] //#[tokio::main] async fn main() -> AnyhowResult<()> { diff --git a/crates/service/obs_gateway/src/redis/constants.rs b/crates/service/obs_gateway/src/redis/constants.rs index e4410d9ca6..a5117ad61e 100644 --- a/crates/service/obs_gateway/src/redis/constants.rs +++ b/crates/service/obs_gateway/src/redis/constants.rs @@ -17,3 +17,6 @@ pub const LEASE_CHECK_PERIOD : Duration = Duration::seconds(15); /// The OBS websocket polls a key to inform the backend that the /// user is still active. pub const OBS_ACTIVE_TTL_SECONDS : usize = 120; + +/// How often to check the OBS session activity. +pub const OBS_ACTIVE_CHECK_PERIOD : Duration = Duration::seconds(30); diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 2a798a1a91..306bc28b84 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -1,7 +1,7 @@ use container_common::anyhow_result::AnyhowResult; use container_common::thread::thread_id::ThreadId; use crate::redis::lease_payload::LeasePayload; -use crate::redis::constants::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD, LEASE_CHECK_PERIOD}; +use crate::redis::constants::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD, LEASE_CHECK_PERIOD, OBS_ACTIVE_CHECK_PERIOD}; use crate::twitch::constants::TWITCH_PING_CADENCE; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; @@ -53,6 +53,10 @@ pub struct TwitchPubsubUserSubscriberThread { // This controls when we periodically check. redis_lease_last_checked_at: Option, + // Check if the OBS session is still active. + // If the underlying Redis key dies, we abandon our thread. + obs_session_active_last_checked_at: Option, + /// Twitch PubSub requires PINGs at regular intervals, /// "To keep the server from closing the connection, clients must send a PING /// command at least once every 5 minutes. If a client does not receive a PONG @@ -82,6 +86,7 @@ impl TwitchPubsubUserSubscriberThread { redis_lease_last_renewed_at: None, redis_lease_last_checked_at: None, twitch_last_pinged_at: None, + obs_session_active_last_checked_at: None, } } @@ -90,6 +95,10 @@ impl TwitchPubsubUserSubscriberThread { self.redis_lease_last_renewed_at = Some(Instant::now()); self.redis_lease_last_checked_at = Some(Instant::now()); + // We know at the start that the session should be active. + // TODO: Ensure we write this before the thread even starts. + self.obs_session_active_last_checked_at = Some(Instant::now()); + // Lookup the OAuth credentials let oauth_lookup_result = self.lookup_oauth_record().await; let oauth_token_record = match oauth_lookup_result { @@ -152,12 +161,18 @@ impl TwitchPubsubUserSubscriberThread { let is_valid = self.maybe_check_redis_lease_is_valid().unwrap(); if !is_valid { - warn!("Exiting thread."); + warn!("Thread lease is invalid; exiting thread."); return; // Exit thread } self.maybe_renew_redis_lease().unwrap(); self.maybe_send_twitch_ping().await.unwrap(); + + let is_active = self.maybe_check_obs_session_active().unwrap(); + if !is_active { + warn!("OBS session ended; exiting thread."); + return; // Exit thread + } } } } @@ -231,6 +246,39 @@ impl TwitchPubsubUserSubscriberThread { Ok(()) } + fn maybe_check_obs_session_active(&mut self) -> AnyhowResult { + let mut should_check_active = self.obs_session_active_last_checked_at + .map(|last_check| last_check.elapsed().gt(&OBS_ACTIVE_CHECK_PERIOD)) + .unwrap_or(true); + + if should_check_active { + info!("Checking OBS active for user {}", self.twitch_user_id.get_numeric()); + let is_active = self.check_obs_session_active()?; + + if !is_active { + warn!("OBS session is no longer active"); + return Ok(false); + } + + self.obs_session_active_last_checked_at = Some(Instant::now()); + } + + Ok(true) + } + + fn check_obs_session_active(&mut self) -> AnyhowResult { + // TODO: Error handling + let mut redis = self.redis_pool.get().unwrap(); + let key = RedisKeys::obs_active_session_keepalive(self.twitch_user_id.get_str()); + + // The value doesn't matter, just the presence of the key. + let payload : Option = redis.get(&key).unwrap(); + match payload { + None => Ok(false), + Some(_payload) => Ok(true), + } + } + async fn lookup_oauth_record(&mut self) -> AnyhowResult> { TwitchOauthTokenFinder::new() .scope_twitch_user_id(Some(self.twitch_user_id.get_numeric())) From 07b480ef2604d6b4cdf562d9d895e1a57aa78a67 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 Jan 2022 00:59:37 -0500 Subject: [PATCH 0805/1042] begin work on token renewal --- .../database_queries/src/twitch_oauth/find.rs | 22 ++++++++++++++----- .../obs_gateway_websocket_handler.rs | 2 ++ .../twitch_pubsub_user_subscriber_thread.rs | 5 ++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/crates/lib/database_queries/src/twitch_oauth/find.rs b/crates/lib/database_queries/src/twitch_oauth/find.rs index dce52a05e6..1980d74851 100644 --- a/crates/lib/database_queries/src/twitch_oauth/find.rs +++ b/crates/lib/database_queries/src/twitch_oauth/find.rs @@ -45,6 +45,9 @@ pub struct TwitchOauthTokenFinder { scope_twitch_user_id: Option, scope_twitch_username_lowercase: Option, + + /// Allow return of expired tokens? Expired tokens can be renewed. + allow_expired_tokens: bool, } impl TwitchOauthTokenFinder { @@ -52,7 +55,8 @@ impl TwitchOauthTokenFinder { Self { scope_user_token: None, scope_twitch_user_id: None, - scope_twitch_username_lowercase: None + scope_twitch_username_lowercase: None, + allow_expired_tokens: false, } } @@ -73,6 +77,11 @@ impl TwitchOauthTokenFinder { self } + pub fn allow_expired_tokens(mut self, allow_expired: bool) -> Self { + self.allow_expired_tokens = allow_expired; + self + } + pub async fn perform_query( &self, mysql_pool: &MySqlPool @@ -169,7 +178,6 @@ SELECT twitch_oauth_tokens.user_deleted_at, twitch_oauth_tokens.mod_deleted_at - FROM twitch_oauth_tokens LEFT OUTER JOIN users ON twitch_oauth_tokens.maybe_user_token = users.token @@ -211,10 +219,12 @@ LEFT OUTER JOIN users } } - if !first_predicate_added { - query.push_str(" WHERE ( twitch_oauth_tokens.expires_at IS NULL OR twitch_oauth_tokens.expires_at > NOW() ) "); - } else { - query.push_str(" AND ( twitch_oauth_tokens.expires_at IS NULL OR twitch_oauth_tokens.expires_at > NOW() ) "); + if !self.allow_expired_tokens { + if !first_predicate_added { + query.push_str(" WHERE ( twitch_oauth_tokens.expires_at IS NULL OR twitch_oauth_tokens.expires_at > NOW() ) "); + } else { + query.push_str(" AND ( twitch_oauth_tokens.expires_at IS NULL OR twitch_oauth_tokens.expires_at > NOW() ) "); + } } // NB: Return the most recent. diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 15a8c3189c..5cba0fb13f 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -53,6 +53,7 @@ pub async fn obs_gateway_websocket_handler( ) -> Result { let mut finder = TwitchOauthTokenFinder::new() + .allow_expired_tokens(true) .scope_twitch_username(Some(&path.twitch_username)); let lookup_result = finder.perform_query(&server_state.backends.mysql_pool).await; @@ -164,6 +165,7 @@ impl StreamHandler> for ObsGatewayWebSock CommonServerError::ServerError }).unwrap(); // TODO: Fixme + // TODO: This should be done *BEFORE* PubSub self.write_obs_active().unwrap(); match msg { diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 306bc28b84..7558527eb1 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -28,10 +28,8 @@ use twitch_api2::pubsub::Response; // TODO: ERROR HANDLING // TODO: Publish events back to OBS thread -// TODO: Disconnect when OBS is done. (keepalive redis key from websocket). // TODO: Refresh oauth token. -// TODO: Handle disconnects. -// TODO: Server+thread IDs +// TODO: Error handling, handle disconnects. pub struct TwitchPubsubUserSubscriberThread { thread_id: ThreadId, @@ -282,6 +280,7 @@ impl TwitchPubsubUserSubscriberThread { async fn lookup_oauth_record(&mut self) -> AnyhowResult> { TwitchOauthTokenFinder::new() .scope_twitch_user_id(Some(self.twitch_user_id.get_numeric())) + .allow_expired_tokens(true) .perform_query(&self.mysql_pool) .await } From ca38a5d69de39f455c59564d568def3845225003 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 Jan 2022 01:57:53 -0500 Subject: [PATCH 0806/1042] add refresh token tool --- .../twitch_pubsub_user_subscriber_thread.rs | 9 ++- .../obs_gateway/src/twitch/oauth/mod.rs | 3 +- .../src/twitch/oauth/oauth_token_refresher.rs | 64 +++++++++++++++++++ 3 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 7558527eb1..a382b9c7ae 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -142,7 +142,14 @@ impl TwitchPubsubUserSubscriberThread { info!("event: {:?}", event); match event { - Response::Response(_) => {} + Response::Response(response) => { + match response.error.as_deref() { + Some("ERR_BADAUTH") => { + warn!("Invalid token. Bad auth. Need to refresh"); + } + _ => {}, + } + } Response::Message { .. } => {} Response::Pong => {} Response::Reconnect => {} diff --git a/crates/service/obs_gateway/src/twitch/oauth/mod.rs b/crates/service/obs_gateway/src/twitch/oauth/mod.rs index c406e2678c..46b2d4c4f9 100644 --- a/crates/service/obs_gateway/src/twitch/oauth/mod.rs +++ b/crates/service/obs_gateway/src/twitch/oauth/mod.rs @@ -1 +1,2 @@ -pub mod oauth_token_builder; \ No newline at end of file +pub mod oauth_token_builder; +pub mod oauth_token_refresher; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs b/crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs new file mode 100644 index 0000000000..b44328242b --- /dev/null +++ b/crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs @@ -0,0 +1,64 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use reqwest::Client; +use reqwest::redirect::Policy; +use std::time::Duration; +use twitch_oauth2::{ClientId, ClientSecret, RefreshToken, AccessToken}; + +/// Refresh OAuth tokens when they expire. +/// NB: This can be cloned and passed around. +#[derive(Clone)] +pub struct OauthTokenRefresher { + http_client: Client, + client_id: ClientId, + client_secret: ClientSecret, +} + +pub struct RefreshTokenResult { + pub access_token: AccessToken, + pub duration: Duration, + pub maybe_refresh_token: Option, +} + +impl OauthTokenRefresher { + + pub fn new(client_id: &str, client_secret: &str) -> AnyhowResult { + let http_client = Client::builder() + .redirect(Policy::none()) + .build()?; + + Ok(Self { + http_client, + client_id: ClientId::new(client_id), + client_secret: ClientSecret::new(client_secret), + }) + } + + /// Perform a token refresh + pub async fn refresh_token(&self, refresh_token: &str) -> AnyhowResult { + let refresh_token = RefreshToken::new(refresh_token); + + let result = twitch_oauth2::refresh_token( + &self.http_client, + &refresh_token, + &self.client_id, + &self.client_secret + ).await; + + match result { + Err(e) => { + Err(anyhow!("error refreshing token: {:?}", e)) + }, + Ok((access_token, + duration, + maybe_refresh_token + )) => { + Ok(RefreshTokenResult { + access_token, + duration, + maybe_refresh_token, + }) + } + } + } +} \ No newline at end of file From 26f42889bf9033b609cc517b6d1a068c15a3515b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 Jan 2022 02:30:07 -0500 Subject: [PATCH 0807/1042] plumb token refresher --- .../listen_for_active_obs_sessions_thread.rs | 5 +++ .../twitch_pubsub_user_subscriber_thread.rs | 5 +++ .../src/twitch/oauth/oauth_token_refresher.rs | 42 +++++++++---------- .../obs_gateway/src/twitch_pubsub_job.rs | 12 ++++++ 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs index 7d0555ce6d..ba53a3c5d7 100644 --- a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs +++ b/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs @@ -16,12 +16,14 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use tokio::runtime::Runtime; +use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; // TODO: Apart from error handling, this looks mostly good. pub struct ListenForActiveObsSessionThread { server_hostname: String, + oauth_token_refresher: OauthTokenRefresher, mysql_pool: Arc>, redis_pool: Arc>, redis_pubsub_pool: Arc>, @@ -31,6 +33,7 @@ pub struct ListenForActiveObsSessionThread { impl ListenForActiveObsSessionThread { pub fn new( server_hostname: String, + oauth_token_refresher: OauthTokenRefresher, mysql_pool: Arc>, redis_pool: Arc>, redis_pubsub_pool: Arc>, @@ -38,6 +41,7 @@ impl ListenForActiveObsSessionThread { ) -> Self { Self { server_hostname, + oauth_token_refresher, mysql_pool, redis_pool, redis_pubsub_pool, @@ -94,6 +98,7 @@ impl ListenForActiveObsSessionThread { let thread = TwitchPubsubUserSubscriberThread::new( twitch_user_id, + self.oauth_token_refresher.clone(), mysql_pool2, redis_pool2, &self.server_hostname, diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index a382b9c7ae..76a0c2f7e4 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -21,6 +21,7 @@ use std::thread::sleep; use std::time::Duration; use time::Instant; use twitch_api2::pubsub::Response; +use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; // TODO: ERROR HANDLING // TODO: ERROR HANDLING @@ -37,6 +38,8 @@ pub struct TwitchPubsubUserSubscriberThread { twitch_user_id: TwitchUserId, expected_lease_payload: LeasePayload, + oauth_token_refresher: OauthTokenRefresher, + mysql_pool: Arc>, redis_pool: Arc>, twitch_websocket_client: TwitchWebsocketClient, @@ -66,6 +69,7 @@ pub struct TwitchPubsubUserSubscriberThread { impl TwitchPubsubUserSubscriberThread { pub fn new( twitch_user_id: TwitchUserId, + oauth_token_refresher: OauthTokenRefresher, mysql_pool: Arc>, redis_pool: Arc>, server_hostname: &str, @@ -75,6 +79,7 @@ impl TwitchPubsubUserSubscriberThread { let expected_lease_payload = LeasePayload::from_thread_id(&server_hostname, &thread_id); Self { thread_id, + oauth_token_refresher, expected_lease_payload, server_hostname: server_hostname.to_string(), twitch_user_id, diff --git a/crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs b/crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs index b44328242b..ca1330286c 100644 --- a/crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs +++ b/crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs @@ -22,43 +22,39 @@ pub struct RefreshTokenResult { impl OauthTokenRefresher { - pub fn new(client_id: &str, client_secret: &str) -> AnyhowResult { + pub fn from_secrets(client_id: ClientId, client_secret: ClientSecret) -> AnyhowResult { let http_client = Client::builder() .redirect(Policy::none()) .build()?; - Ok(Self { http_client, - client_id: ClientId::new(client_id), - client_secret: ClientSecret::new(client_secret), + client_id, + client_secret, }) } + pub fn from_secrets_str(client_id: &str, client_secret: &str) -> AnyhowResult { + let client_id = ClientId::new(client_id); + let client_secret = ClientSecret::new(client_secret); + Self::from_secrets(client_id, client_secret) + } + /// Perform a token refresh pub async fn refresh_token(&self, refresh_token: &str) -> AnyhowResult { let refresh_token = RefreshToken::new(refresh_token); - let result = twitch_oauth2::refresh_token( &self.http_client, &refresh_token, &self.client_id, - &self.client_secret - ).await; - - match result { - Err(e) => { - Err(anyhow!("error refreshing token: {:?}", e)) - }, - Ok((access_token, - duration, - maybe_refresh_token - )) => { - Ok(RefreshTokenResult { - access_token, - duration, - maybe_refresh_token, - }) - } - } + &self.client_secret) + .await + .map(|(access_token, duration, maybe_refresh_token)| { + RefreshTokenResult { + access_token, + duration, + maybe_refresh_token, + } + })?; + Ok(result) } } \ No newline at end of file diff --git a/crates/service/obs_gateway/src/twitch_pubsub_job.rs b/crates/service/obs_gateway/src/twitch_pubsub_job.rs index dab36bd0f9..325ec9f36f 100644 --- a/crates/service/obs_gateway/src/twitch_pubsub_job.rs +++ b/crates/service/obs_gateway/src/twitch_pubsub_job.rs @@ -20,6 +20,8 @@ pub mod util; use config::shared_constants::{DEFAULT_RUST_LOG, DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING, DEFAULT_MYSQL_CONNECTION_STRING}; use container_common::anyhow_result::AnyhowResult; use crate::threads::listen_for_active_obs_sessions_thread::ListenForActiveObsSessionThread; +use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; +use crate::twitch::twitch_secrets::TwitchSecrets; use crate::twitch::websocket_client::TwitchWebsocketClient; use futures::executor::{ThreadPool, ThreadPoolBuilder}; use futures::task::SpawnExt; @@ -33,6 +35,7 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use tokio::runtime::{Builder, Runtime}; +use twitch_oauth2::{ClientId, ClientSecret, RefreshToken, AccessToken}; #[tokio::main] pub async fn main() -> AnyhowResult<()> { @@ -43,6 +46,14 @@ pub async fn main() -> AnyhowResult<()> { .and_then(|h| h.into_string().ok()) .unwrap_or("twitch-pubsub-job".to_string()); + info!("Reading Twitch secrets..."); + + let secrets = TwitchSecrets::from_file("twitch_secrets.toml")?; + let client_id = ClientId::new(&secrets.app_client_id); + let client_secret = ClientSecret::new(&secrets.app_client_secret); + + let oauth_token_refresher = OauthTokenRefresher::from_secrets(client_id, client_secret)?; + let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", @@ -114,6 +125,7 @@ pub async fn main() -> AnyhowResult<()> { let thread = ListenForActiveObsSessionThread::new( server_hostname.to_string(), + oauth_token_refresher, mysql_pool, redis_pool, redis_pubsub_pool, From 95ef47459f883bdf7841aa9e97b7450ba08e63d3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 Jan 2022 17:25:21 -0500 Subject: [PATCH 0808/1042] Token renewal, but broken. Need to reinit pubsub --- .../database_queries/src/twitch_oauth/find.rs | 2 +- .../twitch_pubsub_user_subscriber_thread.rs | 209 +++++++++++++----- 2 files changed, 153 insertions(+), 58 deletions(-) diff --git a/crates/lib/database_queries/src/twitch_oauth/find.rs b/crates/lib/database_queries/src/twitch_oauth/find.rs index 1980d74851..4785a33d98 100644 --- a/crates/lib/database_queries/src/twitch_oauth/find.rs +++ b/crates/lib/database_queries/src/twitch_oauth/find.rs @@ -8,7 +8,7 @@ use log::error; use log::warn; use sqlx::MySqlPool; -#[derive(Serialize)] +#[derive(Serialize, Clone)] pub struct TwitchOauthTokenRecord { /// NB: Vocodes/FakeYou/Storyteller user pub maybe_user_token: Option, diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 76a0c2f7e4..20fa017c89 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -1,3 +1,4 @@ +use anyhow::{anyhow, Error}; use container_common::anyhow_result::AnyhowResult; use container_common::thread::thread_id::ThreadId; use crate::redis::lease_payload::LeasePayload; @@ -22,6 +23,7 @@ use std::time::Duration; use time::Instant; use twitch_api2::pubsub::Response; use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; +use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; // TODO: ERROR HANDLING // TODO: ERROR HANDLING @@ -36,15 +38,90 @@ pub struct TwitchPubsubUserSubscriberThread { thread_id: ThreadId, server_hostname: String, twitch_user_id: TwitchUserId, - expected_lease_payload: LeasePayload, - oauth_token_refresher: OauthTokenRefresher, - mysql_pool: Arc>, redis_pool: Arc>, - twitch_websocket_client: TwitchWebsocketClient, +} - // ========== Thread state ========== +impl TwitchPubsubUserSubscriberThread { + pub fn new( + twitch_user_id: TwitchUserId, + oauth_token_refresher: OauthTokenRefresher, + mysql_pool: Arc>, + redis_pool: Arc>, + server_hostname: &str, + thread_id: ThreadId, + ) -> Self { + Self { + thread_id, + oauth_token_refresher, + server_hostname: server_hostname.to_string(), + twitch_user_id, + mysql_pool, + redis_pool, + } + } + + pub async fn start_thread(mut self) { + let twitch_websocket_client = TwitchWebsocketClient::new().unwrap(); + let expected_lease_payload + = LeasePayload::from_thread_id(&self.server_hostname, &self.thread_id); + + // By failing to look this up, the thread will fail fast. + // When the user auths, the thread will be picked back up again. + let lookup_result + = lookup_oauth_record(&self.twitch_user_id, &self.mysql_pool).await; + + let record = match lookup_result { + Ok(Some(record)) => record, + Ok(None) => { + error!("No twitch oauth token record"); + return; + } + Err(e) => { + error!("Error looking up twitch oauth token record: {:?}", e); + return; + } + }; + + let thread = TwitchPubsubUserSubscriberThreadStageTwo { + thread_id: self.thread_id, + server_hostname: self.server_hostname, + twitch_user_id: self.twitch_user_id, + oauth_token_refresher: self.oauth_token_refresher, + mysql_pool: self.mysql_pool, + redis_pool: self.redis_pool, + twitch_websocket_client, + expected_lease_payload, + twitch_oauth_token_record: record, + redis_lease_last_renewed_at: None, + redis_lease_last_checked_at: None, + obs_session_active_last_checked_at: None, + twitch_last_pinged_at: None + }; + + thread.continue_thread().await + } +} + +/// The thread is somewhat of a state machine. +/// The first stage of thread startup can end prematurely, which is why +/// this is modeled as two different structs. +struct TwitchPubsubUserSubscriberThreadStageTwo { + thread_id: ThreadId, + server_hostname: String, + twitch_user_id: TwitchUserId, + oauth_token_refresher: OauthTokenRefresher, + mysql_pool: Arc>, + redis_pool: Arc>, + + // ========== Stage Two Thread State ========== + + twitch_websocket_client: TwitchWebsocketClient, + expected_lease_payload: LeasePayload, + + // The user's oauth access and refresh tokens. + twitch_oauth_token_record: TwitchOauthTokenRecord, // The thread must renew the lease, or another worker will pick it up. // If the lease gets taken by another, we abandon our own workload. @@ -66,34 +143,8 @@ pub struct TwitchPubsubUserSubscriberThread { twitch_last_pinged_at: Option, } -impl TwitchPubsubUserSubscriberThread { - pub fn new( - twitch_user_id: TwitchUserId, - oauth_token_refresher: OauthTokenRefresher, - mysql_pool: Arc>, - redis_pool: Arc>, - server_hostname: &str, - thread_id: ThreadId, - ) -> Self { - let twitch_websocket_client = TwitchWebsocketClient::new().unwrap(); - let expected_lease_payload = LeasePayload::from_thread_id(&server_hostname, &thread_id); - Self { - thread_id, - oauth_token_refresher, - expected_lease_payload, - server_hostname: server_hostname.to_string(), - twitch_user_id, - mysql_pool, - redis_pool, - twitch_websocket_client, - redis_lease_last_renewed_at: None, - redis_lease_last_checked_at: None, - twitch_last_pinged_at: None, - obs_session_active_last_checked_at: None, - } - } - - pub async fn start_thread(mut self) { +impl TwitchPubsubUserSubscriberThreadStageTwo { + pub async fn continue_thread(mut self) { // We lease from outside before starting the thread. self.redis_lease_last_renewed_at = Some(Instant::now()); self.redis_lease_last_checked_at = Some(Instant::now()); @@ -102,21 +153,7 @@ impl TwitchPubsubUserSubscriberThread { // TODO: Ensure we write this before the thread even starts. self.obs_session_active_last_checked_at = Some(Instant::now()); - // Lookup the OAuth credentials - let oauth_lookup_result = self.lookup_oauth_record().await; - let oauth_token_record = match oauth_lookup_result { - Err(e) => { - error!("Error looking up oauth creds: {:?}", e); - return; // TODO: Better flow. - } - Ok(None) => { - error!("Twitch user oauth token does not exit. Auth please."); - return; // TODO: Better flow. - } - Ok(Some(record)) => record, - }; - - info!("Connecting to Twitch PubSub for user {}...", &oauth_token_record.twitch_username); + info!("Connecting to Twitch PubSub for user {}...", &self.twitch_oauth_token_record.twitch_username); self.twitch_websocket_client.connect().await.unwrap(); // TODO: Error handling loop { @@ -125,7 +162,7 @@ impl TwitchPubsubUserSubscriberThread { info!("Listen to user..."); let topics = build_pubsub_topics_for_user(self.twitch_user_id.get_numeric()); - self.twitch_websocket_client.listen(&oauth_token_record.access_token, &topics).await.unwrap(); + self.twitch_websocket_client.listen(&self.twitch_oauth_token_record.access_token, &topics).await.unwrap(); loop { //debug!("[{}] maybe event for user {}...", self.thread_id.get_id(), self.twitch_user_id.get_str()); @@ -151,6 +188,7 @@ impl TwitchPubsubUserSubscriberThread { match response.error.as_deref() { Some("ERR_BADAUTH") => { warn!("Invalid token. Bad auth. Need to refresh"); + self.refresh_twitch_oauth_token().await.unwrap(); } _ => {}, } @@ -289,14 +327,6 @@ impl TwitchPubsubUserSubscriberThread { } } - async fn lookup_oauth_record(&mut self) -> AnyhowResult> { - TwitchOauthTokenFinder::new() - .scope_twitch_user_id(Some(self.twitch_user_id.get_numeric())) - .allow_expired_tokens(true) - .perform_query(&self.mysql_pool) - .await - } - async fn maybe_send_twitch_ping(&mut self) -> AnyhowResult<()> { let mut should_send_ping = self.twitch_last_pinged_at .map(|last_ping| last_ping.elapsed().gt(&TWITCH_PING_CADENCE)) @@ -310,4 +340,69 @@ impl TwitchPubsubUserSubscriberThread { Ok(()) } + + // =============== OAUTH TOKEN LOOKUP AND RENEWAL =============== + + async fn refresh_twitch_oauth_token(&mut self) -> AnyhowResult<()> { + let refresh_token = match self.twitch_oauth_token_record.maybe_refresh_token.as_deref() { + Some(token) => token, + None => { + error!("No refresh token present. Cannot refresh"); + return Err(anyhow!("No refresh token present. Cannot refresh!")); + }, + }; + + // TODO: Move this somewhere common + let refresh_result = self.oauth_token_refresher.refresh_token(refresh_token) + .await?; + + let access_token = refresh_result.access_token.secret().to_string(); + let refresh_token : Option = refresh_result.maybe_refresh_token + .map(|t| t.secret().to_string()); + let expires_seconds = refresh_result.duration.as_secs() as u32; + + // TODO: Move this somewhere common + let mut query_builder = TwitchOauthTokenInsertBuilder::new( + &self.twitch_oauth_token_record.twitch_user_id, + &self.twitch_oauth_token_record.twitch_username, + &access_token) + .set_refresh_token(refresh_token.as_deref()) + .set_user_token(self.twitch_oauth_token_record.maybe_user_token.as_deref()) + .set_expires_in_seconds(Some(expires_seconds)) + // NB: We don't get these back from the refresh, but it seems like they would stay the same. + .set_token_type(self.twitch_oauth_token_record.token_type.as_deref()) + .set_has_bits_read(self.twitch_oauth_token_record.has_bits_read) + .has_channel_read_subscriptions(self.twitch_oauth_token_record.has_channel_read_subscriptions) + .has_channel_read_redemptions(self.twitch_oauth_token_record.has_channel_read_redemptions) + .has_user_read_follows(self.twitch_oauth_token_record.has_user_read_follows); + + query_builder.insert(&self.mysql_pool).await?; + + let maybe_inserted + = lookup_oauth_record(&self.twitch_user_id, &self.mysql_pool).await?; + + let new_record = match maybe_inserted { + Some(record) => record, + None => { + error!("Did not find oauth token record in database upon refresh"); + return Err(anyhow!("Did not find oauth token record in database upon refresh")); + } + }; + + // TODO: Compare a "refresh_count" + self.twitch_oauth_token_record = new_record; + + Ok(()) + } +} + +async fn lookup_oauth_record( + twitch_user_id: &TwitchUserId, + mysql_pool: &sqlx::Pool +) -> AnyhowResult> { + TwitchOauthTokenFinder::new() + .scope_twitch_user_id(Some(twitch_user_id.get_numeric())) + .allow_expired_tokens(true) + .perform_query(mysql_pool) + .await } From c86d8f81d33946b58969b63cab536285de61c531 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 Jan 2022 20:33:03 -0500 Subject: [PATCH 0809/1042] cleanup (this might be a regression) --- .../twitch_pubsub_user_subscriber_thread.rs | 251 +++++++++++------- 1 file changed, 151 insertions(+), 100 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 20fa017c89..a0fdf61fc5 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -31,8 +31,9 @@ use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; // TODO: ERROR HANDLING // TODO: Publish events back to OBS thread -// TODO: Refresh oauth token. +// TODO: Refresh oauth token and restart client. // TODO: Error handling, handle disconnects. +// TODO: (cleanup) make the logic clearer to follow. pub struct TwitchPubsubUserSubscriberThread { thread_id: ThreadId, @@ -63,10 +64,6 @@ impl TwitchPubsubUserSubscriberThread { } pub async fn start_thread(mut self) { - let twitch_websocket_client = TwitchWebsocketClient::new().unwrap(); - let expected_lease_payload - = LeasePayload::from_thread_id(&self.server_hostname, &self.thread_id); - // By failing to look this up, the thread will fail fast. // When the user auths, the thread will be picked back up again. let lookup_result @@ -84,23 +81,65 @@ impl TwitchPubsubUserSubscriberThread { } }; - let thread = TwitchPubsubUserSubscriberThreadStageTwo { - thread_id: self.thread_id, - server_hostname: self.server_hostname, - twitch_user_id: self.twitch_user_id, - oauth_token_refresher: self.oauth_token_refresher, - mysql_pool: self.mysql_pool, - redis_pool: self.redis_pool, - twitch_websocket_client, - expected_lease_payload, - twitch_oauth_token_record: record, - redis_lease_last_renewed_at: None, - redis_lease_last_checked_at: None, - obs_session_active_last_checked_at: None, - twitch_last_pinged_at: None - }; + let expected_lease_payload + = LeasePayload::from_thread_id(&self.server_hostname, &self.thread_id); - thread.continue_thread().await + loop { + // TODO: Error handling + info!("Connecting to Twitch PubSub for user {}...", &record.twitch_username); + let mut twitch_websocket_client = + self.create_subscribed_twitch_client(&record.access_token).await.unwrap(); + + let thread = TwitchPubsubUserSubscriberThreadStageTwo { + thread_id: self.thread_id.clone(), + server_hostname: self.server_hostname.clone(), + twitch_user_id: self.twitch_user_id.clone(), + oauth_token_refresher: self.oauth_token_refresher.clone(), + mysql_pool: self.mysql_pool.clone(), + redis_pool: self.redis_pool.clone(), + twitch_websocket_client, + expected_lease_payload: expected_lease_payload.clone(), + twitch_oauth_token_record: record.clone(), + redis_lease_last_renewed_at: None, + redis_lease_last_checked_at: None, + obs_session_active_last_checked_at: None, + twitch_last_pinged_at: None + }; + + // NB: The following call will run its main loop until/unless the Twitch client + // fails to auth or disconnects. If this happens, we'll try again. + match thread.continue_thread().await { + Ok(LoopEndedReason::TwitchNeedsReset) => { + sleep(Duration::from_secs(15)); + continue; + } + Ok(LoopEndedReason::ExitThread { reason}) => { + warn!("Thread has ended with reason: {}", reason); + return; + } + Err(e) => { + error!("There was an error, restarting thread shortly: {:?}", e); + sleep(Duration::from_secs(15)); + continue; + } + } + } + } + + async fn create_subscribed_twitch_client( + &self, + access_token: &str + ) -> AnyhowResult { + let mut twitch_websocket_client = TwitchWebsocketClient::new()?; + + twitch_websocket_client.connect().await?; + twitch_websocket_client.send_ping().await?; + + // NB: Failure to auth won't be immediate. + let topics = build_pubsub_topics_for_user(self.twitch_user_id.get_numeric()); + twitch_websocket_client.listen(access_token, &topics).await?; + + Ok(twitch_websocket_client) } } @@ -144,7 +183,10 @@ struct TwitchPubsubUserSubscriberThreadStageTwo { } impl TwitchPubsubUserSubscriberThreadStageTwo { - pub async fn continue_thread(mut self) { + + /// This function will loop until it either errors or hits a `LoopEndedReason` condition. + /// The caller will need to handle these cases. + pub async fn continue_thread(mut self) -> AnyhowResult { // We lease from outside before starting the thread. self.redis_lease_last_renewed_at = Some(Instant::now()); self.redis_lease_last_checked_at = Some(Instant::now()); @@ -153,78 +195,91 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { // TODO: Ensure we write this before the thread even starts. self.obs_session_active_last_checked_at = Some(Instant::now()); - info!("Connecting to Twitch PubSub for user {}...", &self.twitch_oauth_token_record.twitch_username); - self.twitch_websocket_client.connect().await.unwrap(); // TODO: Error handling - loop { + let mut interval = tokio::time::interval(Duration::from_millis(1000)); + + // NB: We can't block forever. + // Adapted from the very good tokio-tungstenite example here, which also splits sockets + // into bidirectional streams: + // https://github.com/snapview/tokio-tungstenite/blob/master/examples/interval-server.rs + tokio::select! { + maybe_event = self.twitch_websocket_client.try_next() => { + if let Some(r) = self.handle_event(maybe_event).await? { + return Ok(r); + } + } + _ = interval.tick() => { + sleep(Duration::from_secs(5)); + } + } + + sleep(Duration::from_secs(1)); + + let is_valid = self.maybe_check_redis_lease_is_valid()?; + if !is_valid { + return Ok(LoopEndedReason::ExitThread { reason: "Thread lease taken".to_string() }); + } + self.maybe_renew_redis_lease().unwrap(); self.maybe_send_twitch_ping().await.unwrap(); - info!("Listen to user..."); - let topics = build_pubsub_topics_for_user(self.twitch_user_id.get_numeric()); - self.twitch_websocket_client.listen(&self.twitch_oauth_token_record.access_token, &topics).await.unwrap(); - - loop { - //debug!("[{}] maybe event for user {}...", self.thread_id.get_id(), self.twitch_user_id.get_str()); - - let mut interval = tokio::time::interval(Duration::from_millis(1000)); - - // NB: We can't block forever. - // Adapted from the very good tokio-tungstenite example here, which also splits sockets - // into bidirectional streams: - // https://github.com/snapview/tokio-tungstenite/blob/master/examples/interval-server.rs - tokio::select! { - maybe_event = self.twitch_websocket_client.try_next() => { - match maybe_event { - Err(e) => { - error!("socket recv error: {:?}", e); - } - Ok(None) => {}, - Ok(Some(ref event)) => { - info!("event: {:?}", event); - - match event { - Response::Response(response) => { - match response.error.as_deref() { - Some("ERR_BADAUTH") => { - warn!("Invalid token. Bad auth. Need to refresh"); - self.refresh_twitch_oauth_token().await.unwrap(); - } - _ => {}, - } - } - Response::Message { .. } => {} - Response::Pong => {} - Response::Reconnect => {} - } - } - } - } - _ = interval.tick() => { - sleep(Duration::from_secs(5)); - } - } - - sleep(Duration::from_secs(1)); - - let is_valid = self.maybe_check_redis_lease_is_valid().unwrap(); - if !is_valid { - warn!("Thread lease is invalid; exiting thread."); - return; // Exit thread - } - - self.maybe_renew_redis_lease().unwrap(); - self.maybe_send_twitch_ping().await.unwrap(); - - let is_active = self.maybe_check_obs_session_active().unwrap(); - if !is_active { - warn!("OBS session ended; exiting thread."); - return; // Exit thread - } + let is_active = self.maybe_check_obs_session_active().unwrap(); + if !is_active { + return Ok(LoopEndedReason::ExitThread { reason: "OBS session ended".to_string() }); } } } + // =============== TWITCH PUBSUB EVENTS AND KEEPALIVE =============== + + async fn handle_event( + &mut self, + event: AnyhowResult> + ) -> AnyhowResult> { + + info!("event: {:?}", event); + + /*match maybe_event { + Err(e) => error!("socket recv error: {:?}", e), + Ok(None) => {}, + Ok(Some(ref event)) => { + } + } + match event { + Response::Response(response) => { + match response.error.as_deref() { + Some("ERR_BADAUTH") => { + warn!("Invalid token. Bad auth. Need to refresh"); + self.refresh_twitch_oauth_token().await.unwrap(); + return Ok(Some(LoopEndedReason::TwitchNeedsReset)); + } + _ => {}, + } + } + Response::Message { .. } => {} + Response::Pong => {} + Response::Reconnect => {} + }*/ + + Ok(None) + } + + async fn maybe_send_twitch_ping(&mut self) -> AnyhowResult<()> { + let mut should_send_ping = self.twitch_last_pinged_at + .map(|last_ping| last_ping.elapsed().gt(&TWITCH_PING_CADENCE)) + .unwrap_or(true); + + if should_send_ping { + info!("Sending Twitch ping for user {}", self.twitch_user_id.get_numeric()); + self.twitch_websocket_client.send_ping().await?; + self.twitch_last_pinged_at = Some(Instant::now()); + } + + Ok(()) + } + + // =============== REDIS THREAD LEASE =============== + fn maybe_check_redis_lease_is_valid(&mut self) -> AnyhowResult { let mut should_check_lease = self.redis_lease_last_checked_at .map(|last_read| last_read.elapsed().gt(&LEASE_CHECK_PERIOD)) @@ -294,6 +349,8 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { Ok(()) } + // =============== OBS SESSION ACTIVITY =============== + fn maybe_check_obs_session_active(&mut self) -> AnyhowResult { let mut should_check_active = self.obs_session_active_last_checked_at .map(|last_check| last_check.elapsed().gt(&OBS_ACTIVE_CHECK_PERIOD)) @@ -327,20 +384,6 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { } } - async fn maybe_send_twitch_ping(&mut self) -> AnyhowResult<()> { - let mut should_send_ping = self.twitch_last_pinged_at - .map(|last_ping| last_ping.elapsed().gt(&TWITCH_PING_CADENCE)) - .unwrap_or(true); - - if should_send_ping { - info!("Sending Twitch ping for user {}", self.twitch_user_id.get_numeric()); - self.twitch_websocket_client.send_ping().await?; - self.twitch_last_pinged_at = Some(Instant::now()); - } - - Ok(()) - } - // =============== OAUTH TOKEN LOOKUP AND RENEWAL =============== async fn refresh_twitch_oauth_token(&mut self) -> AnyhowResult<()> { @@ -406,3 +449,11 @@ async fn lookup_oauth_record( .perform_query(mysql_pool) .await } + +enum LoopEndedReason { + /// Reset the Twitch client + TwitchNeedsReset, + /// Terminate the thread + ExitThread { reason: String }, +} + From 245b3397a81c8895b2ce230568e4ffce21133478 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 Jan 2022 23:55:17 -0500 Subject: [PATCH 0810/1042] change order --- .../twitch_pubsub_user_subscriber_thread.rs | 58 ++++++++++--------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index a0fdf61fc5..59b2cab9ae 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -189,32 +189,12 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { pub async fn continue_thread(mut self) -> AnyhowResult { // We lease from outside before starting the thread. self.redis_lease_last_renewed_at = Some(Instant::now()); - self.redis_lease_last_checked_at = Some(Instant::now()); // We know at the start that the session should be active. // TODO: Ensure we write this before the thread even starts. self.obs_session_active_last_checked_at = Some(Instant::now()); loop { - let mut interval = tokio::time::interval(Duration::from_millis(1000)); - - // NB: We can't block forever. - // Adapted from the very good tokio-tungstenite example here, which also splits sockets - // into bidirectional streams: - // https://github.com/snapview/tokio-tungstenite/blob/master/examples/interval-server.rs - tokio::select! { - maybe_event = self.twitch_websocket_client.try_next() => { - if let Some(r) = self.handle_event(maybe_event).await? { - return Ok(r); - } - } - _ = interval.tick() => { - sleep(Duration::from_secs(5)); - } - } - - sleep(Duration::from_secs(1)); - let is_valid = self.maybe_check_redis_lease_is_valid()?; if !is_valid { return Ok(LoopEndedReason::ExitThread { reason: "Thread lease taken".to_string() }); @@ -227,6 +207,25 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { if !is_active { return Ok(LoopEndedReason::ExitThread { reason: "OBS session ended".to_string() }); } + + // NB: We can't have calls to read the Twitch websocket client block forever, and they + // would do exactly that if not for this code. This is adapted from the very good example + // in the `tokio-tungstenite` repo, which also contains good recipes for splitting sockets + // into two unidirectional streams: + // https://github.com/snapview/tokio-tungstenite/blob/master/examples/interval-server.rs + let mut interval = tokio::time::interval(Duration::from_secs(1)); + tokio::select! { + maybe_event = self.twitch_websocket_client.try_next() => { + if let Some(loop_end_reason) = self.handle_event(maybe_event).await? { + return Ok(loop_end_reason); + } + } + _ = interval.tick() => { + sleep(Duration::from_secs(1)); + } + } + + sleep(Duration::from_secs(1)); } } @@ -234,17 +233,20 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { async fn handle_event( &mut self, - event: AnyhowResult> + maybe_event: AnyhowResult> ) -> AnyhowResult> { + let event = match maybe_event { + Err(e) => { + error!("socket recv error: {:?}", e); + return Ok(None); + }, + Ok(Some(event)) => event, + Ok(None) => return Ok(None), + }; + info!("event: {:?}", event); - /*match maybe_event { - Err(e) => error!("socket recv error: {:?}", e), - Ok(None) => {}, - Ok(Some(ref event)) => { - } - } match event { Response::Response(response) => { match response.error.as_deref() { @@ -259,7 +261,7 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { Response::Message { .. } => {} Response::Pong => {} Response::Reconnect => {} - }*/ + } Ok(None) } From 92b31bc2c5aba96255dc5e760f20acd88826a4ee Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 Jan 2022 23:57:26 -0500 Subject: [PATCH 0811/1042] this nonsense helps my poor ADHD --- .../threads/twitch_pubsub_user_subscriber_thread.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 59b2cab9ae..b5ed3cc1ad 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -35,6 +35,10 @@ use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; // TODO: Error handling, handle disconnects. // TODO: (cleanup) make the logic clearer to follow. +// ========================================= +// =============== STAGE ONE =============== +// ========================================= + pub struct TwitchPubsubUserSubscriberThread { thread_id: ThreadId, server_hostname: String, @@ -143,6 +147,10 @@ impl TwitchPubsubUserSubscriberThread { } } +// ========================================= +// =============== STAGE TWO =============== +// ========================================= + /// The thread is somewhat of a state machine. /// The first stage of thread startup can end prematurely, which is why /// this is modeled as two different structs. @@ -441,6 +449,10 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { } } +// ==================================== +// =============== MISC =============== +// ==================================== + async fn lookup_oauth_record( twitch_user_id: &TwitchUserId, mysql_pool: &sqlx::Pool From aa8f483da08d808c6387569ddb455ab3de2ca740 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 Jan 2022 00:17:43 -0500 Subject: [PATCH 0812/1042] error handling --- .../twitch_pubsub_user_subscriber_thread.rs | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index b5ed3cc1ad..9103b2cab6 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -89,10 +89,17 @@ impl TwitchPubsubUserSubscriberThread { = LeasePayload::from_thread_id(&self.server_hostname, &self.thread_id); loop { - // TODO: Error handling info!("Connecting to Twitch PubSub for user {}...", &record.twitch_username); - let mut twitch_websocket_client = - self.create_subscribed_twitch_client(&record.access_token).await.unwrap(); + let maybe_client = + self.create_subscribed_twitch_client(&record.access_token).await; + + let twitch_websocket_client = match maybe_client { + Ok(client) => client, + Err(e) => { + error!("Error building Twitch client (exiting thread): {:?}", e); + return; + } + }; let thread = TwitchPubsubUserSubscriberThreadStageTwo { thread_id: self.thread_id.clone(), @@ -165,6 +172,8 @@ struct TwitchPubsubUserSubscriberThreadStageTwo { // ========== Stage Two Thread State ========== twitch_websocket_client: TwitchWebsocketClient, + + // We'll use this again and again, so precompute it. expected_lease_payload: LeasePayload, // The user's oauth access and refresh tokens. @@ -208,10 +217,10 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { return Ok(LoopEndedReason::ExitThread { reason: "Thread lease taken".to_string() }); } - self.maybe_renew_redis_lease().unwrap(); - self.maybe_send_twitch_ping().await.unwrap(); + self.maybe_renew_redis_lease()?; + self.maybe_send_twitch_ping().await?; - let is_active = self.maybe_check_obs_session_active().unwrap(); + let is_active = self.maybe_check_obs_session_active()?; if !is_active { return Ok(LoopEndedReason::ExitThread { reason: "OBS session ended".to_string() }); } @@ -244,13 +253,14 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { maybe_event: AnyhowResult> ) -> AnyhowResult> { + let maybe_event = maybe_event.map_err(|error| { + error!("socket error: {:?}", error); + error + })?; + let event = match maybe_event { - Err(e) => { - error!("socket recv error: {:?}", e); - return Ok(None); - }, - Ok(Some(event)) => event, - Ok(None) => return Ok(None), + None => return Ok(None), + Some(event) => event, }; info!("event: {:?}", event); @@ -260,7 +270,7 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { match response.error.as_deref() { Some("ERR_BADAUTH") => { warn!("Invalid token. Bad auth. Need to refresh"); - self.refresh_twitch_oauth_token().await.unwrap(); + self.refresh_twitch_oauth_token().await?; return Ok(Some(LoopEndedReason::TwitchNeedsReset)); } _ => {}, @@ -313,17 +323,17 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { fn check_redis_lease_is_valid(&mut self) -> AnyhowResult { // TODO: Error handling - let mut redis = self.redis_pool.get().unwrap(); + let mut redis = self.redis_pool.get()?; let lease_key = RedisKeys::twitch_pubsub_lease(self.twitch_user_id.get_str()); - let payload : Option = redis.get(&lease_key).unwrap(); + let payload : Option = redis.get(&lease_key)?; match payload { None => { warn!("Redis lease payload absent. Another thread could be started."); Ok(true) } Some(payload) => { - let actual_payload = LeasePayload::deserialize(&payload).unwrap(); + let actual_payload = LeasePayload::deserialize(&payload)?; let equals_expected = self.expected_lease_payload.eq(&actual_payload); Ok(equals_expected) } @@ -346,7 +356,7 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { fn renew_redis_lease(&mut self) -> AnyhowResult<()> { // TODO: Error handling - let mut redis = self.redis_pool.get().unwrap(); + let mut redis = self.redis_pool.get()?; let lease_key = RedisKeys::twitch_pubsub_lease(self.twitch_user_id.get_str()); let lease_value = self.expected_lease_payload.serialize(); @@ -355,7 +365,7 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { &lease_key, &lease_value, LEASE_TIMEOUT_SECONDS - ).unwrap(); + )?; Ok(()) } @@ -383,11 +393,11 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { fn check_obs_session_active(&mut self) -> AnyhowResult { // TODO: Error handling - let mut redis = self.redis_pool.get().unwrap(); + let mut redis = self.redis_pool.get()?; let key = RedisKeys::obs_active_session_keepalive(self.twitch_user_id.get_str()); // The value doesn't matter, just the presence of the key. - let payload : Option = redis.get(&key).unwrap(); + let payload : Option = redis.get(&key)?; match payload { None => Ok(false), Some(_payload) => Ok(true), From 7a23cb69fa20923ea3c890ae639b784a8d961cf4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 Jan 2022 00:23:21 -0500 Subject: [PATCH 0813/1042] simpler without optionals --- .../twitch_pubsub_user_subscriber_thread.rs | 60 ++++++++----------- 1 file changed, 24 insertions(+), 36 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 9103b2cab6..34de103c2e 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -25,14 +25,7 @@ use twitch_api2::pubsub::Response; use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; -// TODO: ERROR HANDLING -// TODO: ERROR HANDLING -// TODO: ERROR HANDLING -// TODO: ERROR HANDLING - // TODO: Publish events back to OBS thread -// TODO: Refresh oauth token and restart client. -// TODO: Error handling, handle disconnects. // TODO: (cleanup) make the logic clearer to follow. // ========================================= @@ -101,6 +94,10 @@ impl TwitchPubsubUserSubscriberThread { } }; + // NB: All of the timers (thus far) can wait to run. + // We set their first run time to now so we don't have to deal with Option<>. + let now = Instant::now(); + let thread = TwitchPubsubUserSubscriberThreadStageTwo { thread_id: self.thread_id.clone(), server_hostname: self.server_hostname.clone(), @@ -111,10 +108,10 @@ impl TwitchPubsubUserSubscriberThread { twitch_websocket_client, expected_lease_payload: expected_lease_payload.clone(), twitch_oauth_token_record: record.clone(), - redis_lease_last_renewed_at: None, - redis_lease_last_checked_at: None, - obs_session_active_last_checked_at: None, - twitch_last_pinged_at: None + redis_lease_last_renewed_at: now, + redis_lease_last_checked_at: now, + obs_session_active_last_checked_at: now, + twitch_last_pinged_at: now, }; // NB: The following call will run its main loop until/unless the Twitch client @@ -181,22 +178,22 @@ struct TwitchPubsubUserSubscriberThreadStageTwo { // The thread must renew the lease, or another worker will pick it up. // If the lease gets taken by another, we abandon our own workload. - redis_lease_last_renewed_at: Option, + redis_lease_last_renewed_at: Instant, // If the lease gets taken by another thread, we abandon our own workload. // This controls when we periodically check. - redis_lease_last_checked_at: Option, + redis_lease_last_checked_at: Instant, // Check if the OBS session is still active. // If the underlying Redis key dies, we abandon our thread. - obs_session_active_last_checked_at: Option, + obs_session_active_last_checked_at: Instant, /// Twitch PubSub requires PINGs at regular intervals, /// "To keep the server from closing the connection, clients must send a PING /// command at least once every 5 minutes. If a client does not receive a PONG /// message within 10 seconds of issuing a PING command, it should reconnect /// to the server. See details in Handling Connection Failures." - twitch_last_pinged_at: Option, + twitch_last_pinged_at: Instant, } impl TwitchPubsubUserSubscriberThreadStageTwo { @@ -204,13 +201,6 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { /// This function will loop until it either errors or hits a `LoopEndedReason` condition. /// The caller will need to handle these cases. pub async fn continue_thread(mut self) -> AnyhowResult { - // We lease from outside before starting the thread. - self.redis_lease_last_renewed_at = Some(Instant::now()); - - // We know at the start that the session should be active. - // TODO: Ensure we write this before the thread even starts. - self.obs_session_active_last_checked_at = Some(Instant::now()); - loop { let is_valid = self.maybe_check_redis_lease_is_valid()?; if !is_valid { @@ -286,13 +276,13 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { async fn maybe_send_twitch_ping(&mut self) -> AnyhowResult<()> { let mut should_send_ping = self.twitch_last_pinged_at - .map(|last_ping| last_ping.elapsed().gt(&TWITCH_PING_CADENCE)) - .unwrap_or(true); + .elapsed() + .gt(&TWITCH_PING_CADENCE); if should_send_ping { info!("Sending Twitch ping for user {}", self.twitch_user_id.get_numeric()); self.twitch_websocket_client.send_ping().await?; - self.twitch_last_pinged_at = Some(Instant::now()); + self.twitch_last_pinged_at = Instant::now(); } Ok(()) @@ -302,8 +292,8 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { fn maybe_check_redis_lease_is_valid(&mut self) -> AnyhowResult { let mut should_check_lease = self.redis_lease_last_checked_at - .map(|last_read| last_read.elapsed().gt(&LEASE_CHECK_PERIOD)) - .unwrap_or(true); + .elapsed() + .gt(&LEASE_CHECK_PERIOD); if should_check_lease { info!("Checking Redis Lease for user {}", self.twitch_user_id.get_numeric()); @@ -314,15 +304,13 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { return Ok(false); } - self.redis_lease_last_checked_at = Some(Instant::now()); + self.redis_lease_last_checked_at = Instant::now(); } Ok(true) } fn check_redis_lease_is_valid(&mut self) -> AnyhowResult { - // TODO: Error handling - let mut redis = self.redis_pool.get()?; let lease_key = RedisKeys::twitch_pubsub_lease(self.twitch_user_id.get_str()); @@ -342,13 +330,13 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { fn maybe_renew_redis_lease(&mut self) -> AnyhowResult<()> { let mut should_renew_lease = self.redis_lease_last_renewed_at - .map(|last_write| last_write.elapsed().gt(&LEASE_RENEW_PERIOD)) - .unwrap_or(true); + .elapsed() + .gt(&LEASE_RENEW_PERIOD); if should_renew_lease { info!("Renewing Redis Lease for user {}", self.twitch_user_id.get_numeric()); self.renew_redis_lease()?; - self.redis_lease_last_renewed_at = Some(Instant::now()); + self.redis_lease_last_renewed_at = Instant::now(); } Ok(()) @@ -373,8 +361,8 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { fn maybe_check_obs_session_active(&mut self) -> AnyhowResult { let mut should_check_active = self.obs_session_active_last_checked_at - .map(|last_check| last_check.elapsed().gt(&OBS_ACTIVE_CHECK_PERIOD)) - .unwrap_or(true); + .elapsed() + .gt(&OBS_ACTIVE_CHECK_PERIOD); if should_check_active { info!("Checking OBS active for user {}", self.twitch_user_id.get_numeric()); @@ -385,7 +373,7 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { return Ok(false); } - self.obs_session_active_last_checked_at = Some(Instant::now()); + self.obs_session_active_last_checked_at = Instant::now(); } Ok(true) From 2d9fa3bd2f922d096e59d255b912118a5a7a7c2c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 Jan 2022 00:34:02 -0500 Subject: [PATCH 0814/1042] oauth renewal works --- .../twitch_pubsub_user_subscriber_thread.rs | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 34de103c2e..ada9b5aaa0 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -66,7 +66,7 @@ impl TwitchPubsubUserSubscriberThread { let lookup_result = lookup_oauth_record(&self.twitch_user_id, &self.mysql_pool).await; - let record = match lookup_result { + let mut record = match lookup_result { Ok(Some(record)) => record, Ok(None) => { error!("No twitch oauth token record"); @@ -125,6 +125,11 @@ impl TwitchPubsubUserSubscriberThread { warn!("Thread has ended with reason: {}", reason); return; } + Ok(LoopEndedReason::RefreshedOauthToken { token }) => { + warn!("OAuth token was refreshed."); + record = token; + sleep(Duration::from_secs(3)); + } Err(e) => { error!("There was an error, restarting thread shortly: {:?}", e); sleep(Duration::from_secs(15)); @@ -260,8 +265,8 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { match response.error.as_deref() { Some("ERR_BADAUTH") => { warn!("Invalid token. Bad auth. Need to refresh"); - self.refresh_twitch_oauth_token().await?; - return Ok(Some(LoopEndedReason::TwitchNeedsReset)); + let token_record = self.refresh_twitch_oauth_token().await?; + return Ok(Some(LoopEndedReason::RefreshedOauthToken { token: token_record })); } _ => {}, } @@ -394,7 +399,7 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { // =============== OAUTH TOKEN LOOKUP AND RENEWAL =============== - async fn refresh_twitch_oauth_token(&mut self) -> AnyhowResult<()> { + async fn refresh_twitch_oauth_token(&mut self) -> AnyhowResult { let refresh_token = match self.twitch_oauth_token_record.maybe_refresh_token.as_deref() { Some(token) => token, None => { @@ -440,10 +445,11 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { } }; + // NB: Instead of resetting the local loop state, we'll create a fresh new Twitch client. // TODO: Compare a "refresh_count" - self.twitch_oauth_token_record = new_record; + self.twitch_oauth_token_record = new_record.clone(); - Ok(()) + Ok(new_record) } } @@ -467,5 +473,7 @@ enum LoopEndedReason { TwitchNeedsReset, /// Terminate the thread ExitThread { reason: String }, + /// A new OAuth token was minted. + RefreshedOauthToken { token: TwitchOauthTokenRecord } } From 2ed75e5b0fd872be1adaab583bb4553b5a8c137d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 Jan 2022 00:34:47 -0500 Subject: [PATCH 0815/1042] cleanup --- .../src/threads/twitch_pubsub_user_subscriber_thread.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index ada9b5aaa0..7ca88cb515 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -117,10 +117,6 @@ impl TwitchPubsubUserSubscriberThread { // NB: The following call will run its main loop until/unless the Twitch client // fails to auth or disconnects. If this happens, we'll try again. match thread.continue_thread().await { - Ok(LoopEndedReason::TwitchNeedsReset) => { - sleep(Duration::from_secs(15)); - continue; - } Ok(LoopEndedReason::ExitThread { reason}) => { warn!("Thread has ended with reason: {}", reason); return; @@ -469,8 +465,6 @@ async fn lookup_oauth_record( } enum LoopEndedReason { - /// Reset the Twitch client - TwitchNeedsReset, /// Terminate the thread ExitThread { reason: String }, /// A new OAuth token was minted. From 228e31294939b1fe850d0a2237a3deb0423f4e70 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 Jan 2022 00:50:22 -0500 Subject: [PATCH 0816/1042] Add refresh_count to model --- .../2022-01-11-042235_twitch_oauth_tokens/up.sql | 6 ++++++ crates/lib/database_queries/src/twitch_oauth/find.rs | 4 ++++ .../lib/database_queries/src/twitch_oauth/insert.rs | 11 ++++++++++- .../threads/twitch_pubsub_user_subscriber_thread.rs | 12 +++++------- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql b/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql index ac837ff3a3..0bf8aa6f48 100644 --- a/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql +++ b/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql @@ -5,6 +5,7 @@ -- **NOTE ABOUT DESIGN**: -- This table contains _many_ OAuth tokens for any single user. (many users : many tokens) -- An entire history of tokens is kept in the table. +-- Apart from "deleted_at", etc., these records should largely be IMMUTABLE. CREATE TABLE twitch_oauth_tokens( -- Not used for anything except replication. @@ -49,6 +50,11 @@ CREATE TABLE twitch_oauth_tokens( -- Null if it does not expire or we were not informed expires_in_seconds INT(10) UNSIGNED DEFAULT NULL, + -- Number of times this token has been refreshed + -- The original token creation will be "0", and each + -- subsequent time will bump by one. Mostly for debugging. + refresh_count INT(10) UNSIGNED DEFAULT 0, + -- ========== OAUTH SCOPES ========== -- "bits:read" diff --git a/crates/lib/database_queries/src/twitch_oauth/find.rs b/crates/lib/database_queries/src/twitch_oauth/find.rs index 4785a33d98..1f544a8243 100644 --- a/crates/lib/database_queries/src/twitch_oauth/find.rs +++ b/crates/lib/database_queries/src/twitch_oauth/find.rs @@ -25,6 +25,7 @@ pub struct TwitchOauthTokenRecord { pub maybe_refresh_token: Option, pub token_type: Option, pub expires_in_seconds: Option, + pub refresh_count: u32, pub has_bits_read: bool, pub has_channel_read_subscriptions: bool, pub has_channel_read_redemptions: bool, @@ -99,6 +100,7 @@ impl TwitchOauthTokenFinder { maybe_refresh_token: record.maybe_refresh_token.clone(), token_type: record.token_type.clone(), expires_in_seconds: record.expires_in_seconds.clone(), + refresh_count: record.refresh_count, has_bits_read: i8_to_bool(record.has_bits_read), has_channel_read_subscriptions: i8_to_bool(record.has_channel_read_subscriptions), has_channel_read_redemptions: i8_to_bool(record.has_channel_read_redemptions), @@ -167,6 +169,7 @@ SELECT twitch_oauth_tokens.maybe_refresh_token, twitch_oauth_tokens.token_type, twitch_oauth_tokens.expires_in_seconds, + twitch_oauth_tokens.refresh_count, twitch_oauth_tokens.has_bits_read, twitch_oauth_tokens.has_channel_read_subscriptions, twitch_oauth_tokens.has_channel_read_redemptions, @@ -256,6 +259,7 @@ pub struct TwitchOauthTokenRecordInternal { pub maybe_refresh_token: Option, pub token_type: Option, pub expires_in_seconds: Option, + pub refresh_count: u32, pub has_bits_read: i8, pub has_channel_read_subscriptions: i8, pub has_channel_read_redemptions: i8, diff --git a/crates/lib/database_queries/src/twitch_oauth/insert.rs b/crates/lib/database_queries/src/twitch_oauth/insert.rs index 85340cc875..e40458a6eb 100644 --- a/crates/lib/database_queries/src/twitch_oauth/insert.rs +++ b/crates/lib/database_queries/src/twitch_oauth/insert.rs @@ -35,6 +35,9 @@ pub struct TwitchOauthTokenInsertBuilder { /// TTL seconds from issue expires_in_seconds: Option, + /// Number of times we've refreshed the token + refresh_count: u32, + has_bits_read: bool, has_channel_read_subscriptions: bool, has_channel_read_redemptions: bool, @@ -55,7 +58,8 @@ impl TwitchOauthTokenInsertBuilder { has_bits_read: false, has_channel_read_subscriptions: false, has_channel_read_redemptions: false, - has_user_read_follows: false + has_user_read_follows: false, + refresh_count: 0, } } @@ -84,6 +88,11 @@ impl TwitchOauthTokenInsertBuilder { self } + pub fn set_refresh_count(mut self, refresh_count: u32) -> Self { + self.refresh_count = refresh_count; + self + } + pub fn set_has_bits_read(mut self, has_bits_read: bool) -> Self { self.has_bits_read = has_bits_read; self diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 7ca88cb515..5fd13eb7b4 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -1,13 +1,15 @@ use anyhow::{anyhow, Error}; use container_common::anyhow_result::AnyhowResult; use container_common::thread::thread_id::ThreadId; -use crate::redis::lease_payload::LeasePayload; use crate::redis::constants::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD, LEASE_CHECK_PERIOD, OBS_ACTIVE_CHECK_PERIOD}; +use crate::redis::lease_payload::LeasePayload; use crate::twitch::constants::TWITCH_PING_CADENCE; +use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; use database_queries::twitch_oauth::find::{TwitchOauthTokenRecord, TwitchOauthTokenFinder}; +use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; use log::debug; use log::error; use log::info; @@ -22,8 +24,6 @@ use std::thread::sleep; use std::time::Duration; use time::Instant; use twitch_api2::pubsub::Response; -use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; -use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; // TODO: Publish events back to OBS thread // TODO: (cleanup) make the logic clearer to follow. @@ -344,7 +344,6 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { } fn renew_redis_lease(&mut self) -> AnyhowResult<()> { - // TODO: Error handling let mut redis = self.redis_pool.get()?; let lease_key = RedisKeys::twitch_pubsub_lease(self.twitch_user_id.get_str()); @@ -381,7 +380,6 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { } fn check_obs_session_active(&mut self) -> AnyhowResult { - // TODO: Error handling let mut redis = self.redis_pool.get()?; let key = RedisKeys::obs_active_session_keepalive(self.twitch_user_id.get_str()); @@ -404,7 +402,6 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { }, }; - // TODO: Move this somewhere common let refresh_result = self.oauth_token_refresher.refresh_token(refresh_token) .await?; @@ -413,7 +410,7 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { .map(|t| t.secret().to_string()); let expires_seconds = refresh_result.duration.as_secs() as u32; - // TODO: Move this somewhere common + // TODO: Move saving a refreshed record somewhere common let mut query_builder = TwitchOauthTokenInsertBuilder::new( &self.twitch_oauth_token_record.twitch_user_id, &self.twitch_oauth_token_record.twitch_username, @@ -421,6 +418,7 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { .set_refresh_token(refresh_token.as_deref()) .set_user_token(self.twitch_oauth_token_record.maybe_user_token.as_deref()) .set_expires_in_seconds(Some(expires_seconds)) + .set_refresh_count(self.twitch_oauth_token_record.refresh_count.saturating_add(1)) // NB: We don't get these back from the refresh, but it seems like they would stay the same. .set_token_type(self.twitch_oauth_token_record.token_type.as_deref()) .set_has_bits_read(self.twitch_oauth_token_record.has_bits_read) From 0edcf2b9c1791d18308097dbc3299b2dee60317f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 24 Jan 2022 18:32:46 -0500 Subject: [PATCH 0817/1042] Fix refresh count --- crates/lib/database_queries/src/twitch_oauth/insert.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/lib/database_queries/src/twitch_oauth/insert.rs b/crates/lib/database_queries/src/twitch_oauth/insert.rs index e40458a6eb..68bd187b36 100644 --- a/crates/lib/database_queries/src/twitch_oauth/insert.rs +++ b/crates/lib/database_queries/src/twitch_oauth/insert.rs @@ -127,6 +127,7 @@ SET twitch_username_lowercase = ?, access_token = ?, maybe_refresh_token = ?, + refresh_count = ?, token_type = ?, expires_in_seconds = ?, has_bits_read = ?, @@ -142,6 +143,7 @@ SET self.twitch_username.clone().to_lowercase(), self.access_token.clone(), self.maybe_refresh_token.clone(), + self.refresh_count, self.token_type.clone(), expires_in_seconds, self.has_bits_read.clone(), @@ -162,6 +164,7 @@ SET twitch_username_lowercase = ?, access_token = ?, maybe_refresh_token = ?, + refresh_count = ?, token_type = ?, expires_in_seconds = NULL, has_bits_read = ?, @@ -177,6 +180,7 @@ SET self.twitch_username.clone().to_lowercase(), self.access_token.clone(), self.maybe_refresh_token.clone(), + self.refresh_count, self.token_type.clone(), self.has_bits_read.clone(), self.has_channel_read_subscriptions.clone(), From 427c1d2761293c72900870a77b3f78b63597b33b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 Jan 2022 03:43:36 -0500 Subject: [PATCH 0818/1042] message with bits --- .../obs_gateway_websocket_handler.rs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 5cba0fb13f..111d39224f 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -129,6 +129,37 @@ impl Actor for ObsGatewayWebSocket { } } +/* + +Message { + data: ChannelBitsEventsV2 { + topic: ChannelBitsEventsV2 { + channel_id: 652567283 + }, + reply: BitsEvent { + data: BitsEventData { + badge_entitlement: None, + bits_used: 1, + channel_id: "652567283", + channel_name: "vocodes", + chat_message: "test Cheer1", + context: Cheer, + is_anonymous: false, + time: "2022-01-25T08:40:04.760906991Z", + total_bits_used: 116, + user_id: "650154491", + user_name: "testytest512" + }, + message_id: "7703927a-78a5-56d1-aa28-4e6c12aa79a1", + version: "1.0", + is_anonymous: false + } + } +} + +*/ + + impl StreamHandler> for ObsGatewayWebSocket { fn handle( &mut self, From 80da30937e2028bffe526b1a46f625cd73f1db01 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 Jan 2022 03:48:05 -0500 Subject: [PATCH 0819/1042] channel points message --- .../obs_gateway_websocket_handler.rs | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 111d39224f..290a5da1b4 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -157,6 +157,66 @@ Message { } } +Message { + data: ChannelPointsChannelV1 { + topic: ChannelPointsChannelV1 { + channel_id: 652567283 + }, + reply: RewardRedeemed { + timestamp: "2022-01-25T08:44:09.266674947Z", + redemption: Redemption { + channel_id: "652567283", + id: "e90823c7-934c-497c-ba2c-34c93dcf7163", + redeemed_at: "2022-01-25T08:44:09.266674947Z", + reward: Reward { + background_color: "#BD0078", + channel_id: "652567283", + cooldown_expires_at: None, + cost: 50, + default_image: Some(Image { + url_1x: "https://static-cdn.jtvnw.net/custom-reward-images/default-1.png", + url_2x: "https://static-cdn.jtvnw.net/custom-reward-images/default-2.png", + url_4x: "https://static-cdn.jtvnw.net/custom-reward-images/default-4.png" + }), + global_cooldown: GlobalCooldown { + is_enabled: false, + global_cooldown_seconds: 0 + }, + id: "3e0eaf15-f454-482d-b48c-5be6ede61901", + image: None, + is_enabled: true, + is_in_stock: true, + is_paused: false, + is_sub_only: false, + is_user_input_required: true, + max_per_stream: MaxPerStream { + is_enabled: false, + max_per_stream: 0 + }, + max_per_user_per_stream: MaxPerUserPerStream { + is_enabled: false, + max_per_user_per_stream: 0 + }, + prompt: "This is a reward", + redemptions_redeemed_current_stream: None, + should_redemptions_skip_request_queue: false, + template_id: None, + title: "Reward #1", + updated_for_indicator_at: Some("2021-09-28T08:42:31.949564296Z") + }, + status: Unfulfilled, + user: User { + id: "650154491", + login: "testytest512", + display_name: "testytest512", + profile_image_url: None + }, + user_input: Some("highlight reward thing"), + cursor: None + } + } + } +} */ From 9a05b91788176b9252ca3987378a29df68073af2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 Jan 2022 04:16:14 -0500 Subject: [PATCH 0820/1042] more notes --- .../src/endpoints_ws/obs_gateway_websocket_handler.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 290a5da1b4..174c031f4c 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -157,6 +157,14 @@ Message { } } +// These cheer emotes always combine with a number +// This list seems pretty comprehensive +https://github.com/nossebro/TwitchPubSubMirror/blob/master/TwitchPubSubMirror_StreamlabsSystem.py +CheerMotes = [ "Cheer", "DoodleCheer", "BibleThump", "cheerwhal", "Corgo", "uni", "ShowLove", "Party", "SeemsGood", "Pride", "Kappa", "FrankerZ", "HeyGuys", "DansGame", "EleGiggle", "TriHard", "Kreygasm", "4Head", "SwiftRage", "NotLikeThis", "FailFish", "VoHiYo", "PJSalt", "MrDestructoid", "bday", "RIPCheer", "Shamrock", "BitBoss", "Streamlabs", "Muxy", "HolidayCheer" ] + + +// Map "Reward Name" ("title") --> FakeYou voice id. + Message { data: ChannelPointsChannelV1 { topic: ChannelPointsChannelV1 { From ec67c123d251c3be231978f47fc6547e6d35b690 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 Jan 2022 23:57:38 -0500 Subject: [PATCH 0821/1042] empty migrations for bits/channel points --- _migrations/2022-01-26-045721_twitch_bits/down.sql | 1 + _migrations/2022-01-26-045721_twitch_bits/up.sql | 1 + _migrations/2022-01-26-045730_twitch_channel_points/down.sql | 1 + _migrations/2022-01-26-045730_twitch_channel_points/up.sql | 1 + 4 files changed, 4 insertions(+) create mode 100644 _migrations/2022-01-26-045721_twitch_bits/down.sql create mode 100644 _migrations/2022-01-26-045721_twitch_bits/up.sql create mode 100644 _migrations/2022-01-26-045730_twitch_channel_points/down.sql create mode 100644 _migrations/2022-01-26-045730_twitch_channel_points/up.sql diff --git a/_migrations/2022-01-26-045721_twitch_bits/down.sql b/_migrations/2022-01-26-045721_twitch_bits/down.sql new file mode 100644 index 0000000000..291a97c5ce --- /dev/null +++ b/_migrations/2022-01-26-045721_twitch_bits/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` \ No newline at end of file diff --git a/_migrations/2022-01-26-045721_twitch_bits/up.sql b/_migrations/2022-01-26-045721_twitch_bits/up.sql new file mode 100644 index 0000000000..33cf74b578 --- /dev/null +++ b/_migrations/2022-01-26-045721_twitch_bits/up.sql @@ -0,0 +1 @@ +-- Your SQL goes here \ No newline at end of file diff --git a/_migrations/2022-01-26-045730_twitch_channel_points/down.sql b/_migrations/2022-01-26-045730_twitch_channel_points/down.sql new file mode 100644 index 0000000000..291a97c5ce --- /dev/null +++ b/_migrations/2022-01-26-045730_twitch_channel_points/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` \ No newline at end of file diff --git a/_migrations/2022-01-26-045730_twitch_channel_points/up.sql b/_migrations/2022-01-26-045730_twitch_channel_points/up.sql new file mode 100644 index 0000000000..33cf74b578 --- /dev/null +++ b/_migrations/2022-01-26-045730_twitch_channel_points/up.sql @@ -0,0 +1 @@ +-- Your SQL goes here \ No newline at end of file From 7f4cb37aeac9413e9456a340b5838fecd9bf7487 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 26 Jan 2022 23:32:32 -0500 Subject: [PATCH 0822/1042] schema wip --- .../2022-01-26-045721_twitch_bits/down.sql | 6 +++- .../2022-01-26-045721_twitch_bits/up.sql | 32 ++++++++++++++++++- .../down.sql | 6 +++- .../up.sql | 32 ++++++++++++++++++- 4 files changed, 72 insertions(+), 4 deletions(-) diff --git a/_migrations/2022-01-26-045721_twitch_bits/down.sql b/_migrations/2022-01-26-045721_twitch_bits/down.sql index 291a97c5ce..fb1e062e72 100644 --- a/_migrations/2022-01-26-045721_twitch_bits/down.sql +++ b/_migrations/2022-01-26-045721_twitch_bits/down.sql @@ -1 +1,5 @@ --- This file should undo anything in `up.sql` \ No newline at end of file +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE twitch_bits_events; diff --git a/_migrations/2022-01-26-045721_twitch_bits/up.sql b/_migrations/2022-01-26-045721_twitch_bits/up.sql index 33cf74b578..e4dcc4d47e 100644 --- a/_migrations/2022-01-26-045721_twitch_bits/up.sql +++ b/_migrations/2022-01-26-045721_twitch_bits/up.sql @@ -1 +1,31 @@ --- Your SQL goes here \ No newline at end of file +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- This is primarily meant to be for analysis. +CREATE TABLE twitch_bits_events( + -- Not used for anything except replication. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- ========== STORYTELLER/FAKEYOU USER ========== + + -- The user ID / channel ID are the same + -- https://discuss.dev.twitch.tv/t/what-is-the-difference-between-the-stream--id-and-channel--id/4423 + -- Several suggest this should be a string and not an integer + -- https://discuss.dev.twitch.tv/t/type-of-user-id-in-api-responses/10205 + -- Yep, strings + -- https://discuss.dev.twitch.tv/t/bug-v5-api-returns--id-as-string-for-featured-channels/10310 + sender_twitch_user_id VARCHAR(64) NOT NULL, + + -- Twitch usernames are between 4 and 25 characters. + -- This is returned in the oauth flow + sender_twitch_username VARCHAR(32) NOT NULL, + + -- Twitch username, but lowercase for lookup. + sender_twitch_username_lowercase VARCHAR(32) NOT NULL, + + -- INDICES -- + PRIMARY KEY (id) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + diff --git a/_migrations/2022-01-26-045730_twitch_channel_points/down.sql b/_migrations/2022-01-26-045730_twitch_channel_points/down.sql index 291a97c5ce..9cc5937547 100644 --- a/_migrations/2022-01-26-045730_twitch_channel_points/down.sql +++ b/_migrations/2022-01-26-045730_twitch_channel_points/down.sql @@ -1 +1,5 @@ --- This file should undo anything in `up.sql` \ No newline at end of file +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE twitch_channel_point_events; diff --git a/_migrations/2022-01-26-045730_twitch_channel_points/up.sql b/_migrations/2022-01-26-045730_twitch_channel_points/up.sql index 33cf74b578..3156c2002e 100644 --- a/_migrations/2022-01-26-045730_twitch_channel_points/up.sql +++ b/_migrations/2022-01-26-045730_twitch_channel_points/up.sql @@ -1 +1,31 @@ --- Your SQL goes here \ No newline at end of file +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- This is primarily meant to be for analysis. +CREATE TABLE twitch_channel_point_events ( + -- Not used for anything except replication. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- ========== STORYTELLER/FAKEYOU USER ========== + + -- The user ID / channel ID are the same + -- https://discuss.dev.twitch.tv/t/what-is-the-difference-between-the-stream--id-and-channel--id/4423 + -- Several suggest this should be a string and not an integer + -- https://discuss.dev.twitch.tv/t/type-of-user-id-in-api-responses/10205 + -- Yep, strings + -- https://discuss.dev.twitch.tv/t/bug-v5-api-returns--id-as-string-for-featured-channels/10310 + sender_twitch_user_id VARCHAR(64) NOT NULL, + + -- Twitch usernames are between 4 and 25 characters. + -- This is returned in the oauth flow + sender_twitch_username VARCHAR(32) NOT NULL, + + -- Twitch username, but lowercase for lookup. + sender_twitch_username_lowercase VARCHAR(32) NOT NULL, + + -- INDICES -- + PRIMARY KEY (id) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + From 115fc2fe704e6748fea026ca362c5dc4efaada93 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 27 Jan 2022 23:58:06 -0500 Subject: [PATCH 0823/1042] I'm cheating here... --- _migrations/2022-01-26-045721_twitch_bits/up.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/_migrations/2022-01-26-045721_twitch_bits/up.sql b/_migrations/2022-01-26-045721_twitch_bits/up.sql index e4dcc4d47e..9e0e23a590 100644 --- a/_migrations/2022-01-26-045721_twitch_bits/up.sql +++ b/_migrations/2022-01-26-045721_twitch_bits/up.sql @@ -28,4 +28,3 @@ CREATE TABLE twitch_bits_events( PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; - From 103d9108b162157c5b921b073bb3e171041b6e15 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 28 Jan 2022 23:42:18 -0500 Subject: [PATCH 0824/1042] cheating on git commits again, hopefully the last time --- _migrations/2022-01-26-045730_twitch_channel_points/up.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/_migrations/2022-01-26-045730_twitch_channel_points/up.sql b/_migrations/2022-01-26-045730_twitch_channel_points/up.sql index 3156c2002e..0955748861 100644 --- a/_migrations/2022-01-26-045730_twitch_channel_points/up.sql +++ b/_migrations/2022-01-26-045730_twitch_channel_points/up.sql @@ -28,4 +28,3 @@ CREATE TABLE twitch_channel_point_events ( PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; - From 30d96a94c5629e9949032aa872b983e80b96efe2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 Jan 2022 23:49:03 -0500 Subject: [PATCH 0825/1042] start adding docs --- _docs/database_schema.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 _docs/database_schema.md diff --git a/_docs/database_schema.md b/_docs/database_schema.md new file mode 100644 index 0000000000..a3fe5efc98 --- /dev/null +++ b/_docs/database_schema.md @@ -0,0 +1,17 @@ +database schema +================ + +Twitch Model +------------ + +`twitch_user_id` + +- Though numeric, this should always be a string per + [https://discuss.dev.twitch.tv/t/bug-v5-api-returns--id-as-string-for-featured-channels/10310] + and + [https://discuss.dev.twitch.tv/t/type-of-user-id-in-api-responses/10205] + +`twitch_channel_id` + +- This is the same as `twitch_user_id`. They're exactly 1:1. + From e49f6f66df8e56a57805c71d9dedd219efb6b883 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 Jan 2022 00:11:17 -0500 Subject: [PATCH 0826/1042] migrations wip --- _docs/database_schema.md | 12 +++--- .../2022-01-26-045721_twitch_bits/up.sql | 38 ++++++++++++------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/_docs/database_schema.md b/_docs/database_schema.md index a3fe5efc98..ef0d3eaa9f 100644 --- a/_docs/database_schema.md +++ b/_docs/database_schema.md @@ -4,14 +4,16 @@ database schema Twitch Model ------------ -`twitch_user_id` - -- Though numeric, this should always be a string per +- `twitch_user_id` + Though numeric, this should always be a string per [https://discuss.dev.twitch.tv/t/bug-v5-api-returns--id-as-string-for-featured-channels/10310] and [https://discuss.dev.twitch.tv/t/type-of-user-id-in-api-responses/10205] -`twitch_channel_id` +- `twitch_channel_id` + This is the same as `twitch_user_id`. They're exactly 1:1. Prefer to + use 'user_id' terminology. -- This is the same as `twitch_user_id`. They're exactly 1:1. +- `twitch_username` + Between 4 and 25 characters. diff --git a/_migrations/2022-01-26-045721_twitch_bits/up.sql b/_migrations/2022-01-26-045721_twitch_bits/up.sql index 9e0e23a590..6284bfef31 100644 --- a/_migrations/2022-01-26-045721_twitch_bits/up.sql +++ b/_migrations/2022-01-26-045721_twitch_bits/up.sql @@ -3,28 +3,38 @@ -- noinspection SqlResolveForFile -- This is primarily meant to be for analysis. +-- NB: See [_docs/database_schema.md] for details on Twitch-specifics. + CREATE TABLE twitch_bits_events( -- Not used for anything except replication. id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - -- ========== STORYTELLER/FAKEYOU USER ========== - - -- The user ID / channel ID are the same - -- https://discuss.dev.twitch.tv/t/what-is-the-difference-between-the-stream--id-and-channel--id/4423 - -- Several suggest this should be a string and not an integer - -- https://discuss.dev.twitch.tv/t/type-of-user-id-in-api-responses/10205 - -- Yep, strings - -- https://discuss.dev.twitch.tv/t/bug-v5-api-returns--id-as-string-for-featured-channels/10310 + -- The user sending the bits. + -- We'll need to keep the name and lowercase for lookup. sender_twitch_user_id VARCHAR(64) NOT NULL, - - -- Twitch usernames are between 4 and 25 characters. - -- This is returned in the oauth flow sender_twitch_username VARCHAR(32) NOT NULL, - - -- Twitch username, but lowercase for lookup. sender_twitch_username_lowercase VARCHAR(32) NOT NULL, + -- The channel the bits were being sent to. + -- We technically only need the ID since we can JOIN, but we'll keep + -- the name for convenience. + destination_channel_id VARCHAR(64) NOT NULL, + destination_channel_name VARCHAR(32) NOT NULL, + + is_anonymous BOOLEAN NOT NULL DEFAULT FALSE, + + -- The number of bits just spent + bits_used INT(10) UNSIGNED DEFAULT NULL, + + -- The lifetime(?) number of bits spent by the user in this channel. + total_bits_used INT(10) UNSIGNED DEFAULT NULL, + + chat_message TEXT NOT NULL, + -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + KEY index_sender_twitch_user_id (sender_twitch_user_id), + KEY index_sender_twitch_username_lowercase (sender_twitch_username_lowercase), + KEY index_destination_channel_id (destination_channel_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 6d6c7bc923f511fe81f517aa68f0411a9d31527f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 Jan 2022 02:11:27 -0500 Subject: [PATCH 0827/1042] more schema --- .../2022-01-26-045721_twitch_bits/up.sql | 4 ++ .../up.sql | 53 ++++++++++++++----- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/_migrations/2022-01-26-045721_twitch_bits/up.sql b/_migrations/2022-01-26-045721_twitch_bits/up.sql index 6284bfef31..83564535da 100644 --- a/_migrations/2022-01-26-045721_twitch_bits/up.sql +++ b/_migrations/2022-01-26-045721_twitch_bits/up.sql @@ -9,6 +9,8 @@ CREATE TABLE twitch_bits_events( -- Not used for anything except replication. id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + -- ========== USER / CHANNEL ========== + -- The user sending the bits. -- We'll need to keep the name and lowercase for lookup. sender_twitch_user_id VARCHAR(64) NOT NULL, @@ -21,6 +23,8 @@ CREATE TABLE twitch_bits_events( destination_channel_id VARCHAR(64) NOT NULL, destination_channel_name VARCHAR(32) NOT NULL, + -- ========== BITS DETAILS ========== + is_anonymous BOOLEAN NOT NULL DEFAULT FALSE, -- The number of bits just spent diff --git a/_migrations/2022-01-26-045730_twitch_channel_points/up.sql b/_migrations/2022-01-26-045730_twitch_channel_points/up.sql index 0955748861..5e5b02fdcc 100644 --- a/_migrations/2022-01-26-045730_twitch_channel_points/up.sql +++ b/_migrations/2022-01-26-045730_twitch_channel_points/up.sql @@ -3,28 +3,55 @@ -- noinspection SqlResolveForFile -- This is primarily meant to be for analysis. +-- NB: See [_docs/database_schema.md] for details on Twitch-specifics. + CREATE TABLE twitch_channel_point_events ( -- Not used for anything except replication. id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - -- ========== STORYTELLER/FAKEYOU USER ========== + -- ========== USER / CHANNEL ========== - -- The user ID / channel ID are the same - -- https://discuss.dev.twitch.tv/t/what-is-the-difference-between-the-stream--id-and-channel--id/4423 - -- Several suggest this should be a string and not an integer - -- https://discuss.dev.twitch.tv/t/type-of-user-id-in-api-responses/10205 - -- Yep, strings - -- https://discuss.dev.twitch.tv/t/bug-v5-api-returns--id-as-string-for-featured-channels/10310 + -- The user sending the bits. + -- We'll need to keep the name and lowercase for lookup. sender_twitch_user_id VARCHAR(64) NOT NULL, - - -- Twitch usernames are between 4 and 25 characters. - -- This is returned in the oauth flow sender_twitch_username VARCHAR(32) NOT NULL, - - -- Twitch username, but lowercase for lookup. sender_twitch_username_lowercase VARCHAR(32) NOT NULL, + -- The channel the bits were being sent to. + -- We technically only need the ID since we can JOIN, but we'll keep + -- the name for convenience. + destination_channel_id VARCHAR(64) NOT NULL, + destination_channel_name VARCHAR(32) NOT NULL, + + -- ========== CHANNEL POINTS DETAILS ========== + + -- This is the title used for the reward + -- We'll match on this to do TTS + title VARCHAR(256) NOT NULL, + + -- This is the description used for the reward + prompt VARCHAR(256) NOT NULL, + + -- This is the text that the user sends + -- This is optional, but we'll coerce null to empty + user_input TEXT NOT NULL, + + -- A UUID for this instance of reward redemption + redemption_id VARCHAR(36) NOT NULL, + + -- A UUID for the reward + reward_id VARCHAR(36) NOT NULL, + + reward_cost INT(10) UNSIGNED DEFAULT NULL, + + is_sub_only BOOLEAN NOT NULL DEFAULT FALSE, + max_per_stream INT(10) UNSIGNED DEFAULT NULL, + max_per_user_per_stream INT(10) UNSIGNED DEFAULT NULL, + -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + KEY index_sender_twitch_user_id (sender_twitch_user_id), + KEY index_sender_twitch_username_lowercase (sender_twitch_username_lowercase), + KEY index_destination_channel_id (destination_channel_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 4448a81ee586734df5c974c8f33a22d716150805 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 Jan 2022 03:51:53 -0500 Subject: [PATCH 0828/1042] insert bits (untested) --- crates/lib/database_queries/src/lib.rs | 1 + .../src/twitch_pubsub/insert_bits.rs | 131 ++++++++++++++++++ .../database_queries/src/twitch_pubsub/mod.rs | 2 + .../obs_gateway_websocket_handler.rs | 61 ++++---- 4 files changed, 168 insertions(+), 27 deletions(-) create mode 100644 crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs create mode 100644 crates/lib/database_queries/src/twitch_pubsub/mod.rs diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index f6f043ef9f..070f938779 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -1,4 +1,5 @@ #[macro_use] extern crate serde_derive; pub mod helpers; +pub mod twitch_pubsub; pub mod twitch_oauth; \ No newline at end of file diff --git a/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs b/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs new file mode 100644 index 0000000000..4e9804d67a --- /dev/null +++ b/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs @@ -0,0 +1,131 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use container_common::anyhow_result::AnyhowResult; +use log::info; +use log::warn; +use sqlx::MySqlPool; + +pub struct TwitchPubsubBitsInsertBuilder { + // ===== Required Fields ===== + + sender_twitch_user_id: Option, + sender_twitch_username: Option, + sender_twitch_username_lowercase: Option, + + destination_channel_id: Option, + destination_channel_name: Option, + + // ===== Optional / Default Fields ===== + + is_anonymous: bool, + + bits_used: u64, + total_bits_used: u64, + + chat_message: String, +} + +impl TwitchPubsubBitsInsertBuilder { + pub fn new() -> Self { + Self { + sender_twitch_user_id: None, + sender_twitch_username: None, + sender_twitch_username_lowercase: None, + destination_channel_id: None, + destination_channel_name: None, + is_anonymous: false, + bits_used: 0, + total_bits_used: 0, + chat_message: "".to_string(), + } + } + + pub fn set_sender_twitch_user_id(mut self, value: Option<&str>) -> Self { + self.sender_twitch_user_id = value.map(|s| s.to_string()); + self + } + + pub fn set_sender_twitch_username(mut self, value: Option<&str>) -> Self { + if let Some(username) = value { + self.sender_twitch_username = Some(username.to_string()); + self.sender_twitch_username_lowercase = Some(username.to_uppercase()); + } else { + self.sender_twitch_username = None; + self.sender_twitch_username_lowercase = None; + } + self + } + + pub fn set_destination_channel_id(mut self, value: Option<&str>) -> Self { + self.destination_channel_id = value.map(|s| s.to_string()); + self + } + + pub fn set_destination_channel_name(mut self, value: Option<&str>) -> Self { + self.destination_channel_name = value.map(|s| s.to_string()); + self + } + + pub fn set_is_anonymous(mut self, value: bool) -> Self { + self.is_anonymous = value; + self + } + + pub fn set_bits_used(mut self, value: u64) -> Self { + self.bits_used = value; + self + } + + pub fn set_total_bits_used(mut self, value: u64) -> Self { + self.total_bits_used = value; + self + } + + pub fn set_chat_message(mut self, value: Option<&str>) -> Self { + self.chat_message = value + .map(|s| s.to_string()) + .unwrap_or("".to_string()); + self + } + + pub async fn insert(&mut self, mysql_pool: &MySqlPool) -> AnyhowResult<()> { + let query = sqlx::query!( + r#" +INSERT INTO twitch_bits_events +SET + sender_twitch_user_id = ?, + sender_twitch_username = ?, + sender_twitch_username_lowercase = ?, + destination_channel_id = ?, + destination_channel_name = ?, + is_anonymous= ?, + bits_used = ?, + total_bits_used = ?, + chat_message = ? + "#, + self.sender_twitch_user_id.clone(), + self.sender_twitch_username.clone(), + self.sender_twitch_username_lowercase.clone(), + self.destination_channel_id.clone(), + self.destination_channel_name.clone(), + self.is_anonymous, + self.bits_used, + self.total_bits_used, + self.chat_message.clone(), + ); + + let query_result = query.execute(mysql_pool) + .await; + + let _record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + return Err(anyhow!("Twitch pubsub bits insert DB error: {:?}", err)); + } + }; + + Ok(()) + } +} diff --git a/crates/lib/database_queries/src/twitch_pubsub/mod.rs b/crates/lib/database_queries/src/twitch_pubsub/mod.rs new file mode 100644 index 0000000000..feb82f60b1 --- /dev/null +++ b/crates/lib/database_queries/src/twitch_pubsub/mod.rs @@ -0,0 +1,2 @@ + +pub mod insert_bits; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 174c031f4c..3454e55480 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -29,6 +29,8 @@ use std::time::Duration; use tokio::runtime::Handle; use twitch_api2::pubsub::Topic; use twitch_api2::pubsub; +use twitch_api2::pubsub::channel_bits::ChannelBitsEventsV2Reply::BitsEvent; +use twitch_api2::pubsub::TopicData::ChannelPointsChannelV1; // TODO: Redis calls are synchronous (but fast), but is there any way to make them async? @@ -134,25 +136,25 @@ impl Actor for ObsGatewayWebSocket { Message { data: ChannelBitsEventsV2 { topic: ChannelBitsEventsV2 { - channel_id: 652567283 + +channel_id: 652567283 }, reply: BitsEvent { data: BitsEventData { - badge_entitlement: None, - bits_used: 1, - channel_id: "652567283", - channel_name: "vocodes", - chat_message: "test Cheer1", - context: Cheer, - is_anonymous: false, - time: "2022-01-25T08:40:04.760906991Z", - total_bits_used: 116, - user_id: "650154491", - user_name: "testytest512" + -badge_entitlement: None, + +bits_used: 1, + +channel_id: "652567283", + -channel_name: "vocodes", + +chat_message: "test Cheer1", + -context: Cheer, + +is_anonymous: false, + -time: "2022-01-25T08:40:04.760906991Z", + +total_bits_used: 116, + +user_id: "650154491", + +user_name: "testytest512" }, - message_id: "7703927a-78a5-56d1-aa28-4e6c12aa79a1", - version: "1.0", - is_anonymous: false + -message_id: "7703927a-78a5-56d1-aa28-4e6c12aa79a1", + -version: "1.0", + +is_anonymous: false } } } @@ -177,25 +179,25 @@ Message { id: "e90823c7-934c-497c-ba2c-34c93dcf7163", redeemed_at: "2022-01-25T08:44:09.266674947Z", reward: Reward { - background_color: "#BD0078", - channel_id: "652567283", - cooldown_expires_at: None, - cost: 50, + -background_color: "#BD0078", + -channel_id: "652567283", + -cooldown_expires_at: None, + -cost: 50, default_image: Some(Image { url_1x: "https://static-cdn.jtvnw.net/custom-reward-images/default-1.png", url_2x: "https://static-cdn.jtvnw.net/custom-reward-images/default-2.png", url_4x: "https://static-cdn.jtvnw.net/custom-reward-images/default-4.png" }), - global_cooldown: GlobalCooldown { + -global_cooldown: GlobalCooldown { is_enabled: false, global_cooldown_seconds: 0 }, id: "3e0eaf15-f454-482d-b48c-5be6ede61901", image: None, - is_enabled: true, - is_in_stock: true, - is_paused: false, - is_sub_only: false, + -is_enabled: true, + -is_in_stock: true, + -is_paused: false, + +is_sub_only: false, is_user_input_required: true, max_per_stream: MaxPerStream { is_enabled: false, @@ -214,12 +216,12 @@ Message { }, status: Unfulfilled, user: User { - id: "650154491", - login: "testytest512", + +id: "650154491", + +login: "testytest512", display_name: "testytest512", profile_image_url: None }, - user_input: Some("highlight reward thing"), + +user_input: Some("highlight reward thing"), cursor: None } } @@ -267,6 +269,11 @@ impl StreamHandler> for ObsGatewayWebSock // TODO: This should be done *BEFORE* PubSub self.write_obs_active().unwrap(); + + //BitsEvent {} + //ChannelPointsChannelV1 {} + + match msg { ws::Message::Text(text) => { //info!("sending text response"); From 3f889c6aad43a12b9a94454279497379c8e43a54 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 30 Jan 2022 14:14:31 -0500 Subject: [PATCH 0829/1042] adjust inserts --- .../src/twitch_pubsub/insert_bits.rs | 30 +++++++++++++++---- .../database_queries/src/twitch_pubsub/mod.rs | 1 - 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs b/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs index 4e9804d67a..16bb404bdc 100644 --- a/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs +++ b/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs @@ -89,6 +89,26 @@ impl TwitchPubsubBitsInsertBuilder { } pub async fn insert(&mut self, mysql_pool: &MySqlPool) -> AnyhowResult<()> { + let sender_twitch_user_id = self.sender_twitch_user_id + .clone() + .ok_or(anyhow!("no sender_twitch_user_id"))?; + + let sender_twitch_username = self.sender_twitch_username + .clone() + .ok_or(anyhow!("no sender_twitch_username"))?; + + let sender_twitch_username_lowercase = self.sender_twitch_username_lowercase + .clone() + .ok_or(anyhow!("no sender_twitch_username_lowercase"))?; + + let destination_channel_id = self.destination_channel_id + .clone() + .ok_or(anyhow!("no destination_channel_id"))?; + + let destination_channel_name = self.destination_channel_name + .clone() + .ok_or(anyhow!("no destination_channel_name"))?; + let query = sqlx::query!( r#" INSERT INTO twitch_bits_events @@ -103,11 +123,11 @@ SET total_bits_used = ?, chat_message = ? "#, - self.sender_twitch_user_id.clone(), - self.sender_twitch_username.clone(), - self.sender_twitch_username_lowercase.clone(), - self.destination_channel_id.clone(), - self.destination_channel_name.clone(), + sender_twitch_user_id, + sender_twitch_username, + sender_twitch_username_lowercase, + destination_channel_id, + destination_channel_name, self.is_anonymous, self.bits_used, self.total_bits_used, diff --git a/crates/lib/database_queries/src/twitch_pubsub/mod.rs b/crates/lib/database_queries/src/twitch_pubsub/mod.rs index feb82f60b1..0930276ccd 100644 --- a/crates/lib/database_queries/src/twitch_pubsub/mod.rs +++ b/crates/lib/database_queries/src/twitch_pubsub/mod.rs @@ -1,2 +1 @@ - pub mod insert_bits; \ No newline at end of file From 37851f8f53787095dfda105baaeab062a3664df8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 00:46:44 -0500 Subject: [PATCH 0830/1042] channel points inserts --- .../up.sql | 2 +- .../twitch_pubsub/insert_channel_points.rs | 191 ++++++++++++++++++ .../database_queries/src/twitch_pubsub/mod.rs | 3 +- 3 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs diff --git a/_migrations/2022-01-26-045730_twitch_channel_points/up.sql b/_migrations/2022-01-26-045730_twitch_channel_points/up.sql index 5e5b02fdcc..a8a23e89ab 100644 --- a/_migrations/2022-01-26-045730_twitch_channel_points/up.sql +++ b/_migrations/2022-01-26-045730_twitch_channel_points/up.sql @@ -34,7 +34,7 @@ CREATE TABLE twitch_channel_point_events ( -- This is the text that the user sends -- This is optional, but we'll coerce null to empty - user_input TEXT NOT NULL, + user_text_input TEXT NOT NULL, -- A UUID for this instance of reward redemption redemption_id VARCHAR(36) NOT NULL, diff --git a/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs b/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs new file mode 100644 index 0000000000..5bae2f0d12 --- /dev/null +++ b/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs @@ -0,0 +1,191 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use container_common::anyhow_result::AnyhowResult; +use log::info; +use log::warn; +use sqlx::MySqlPool; + +pub struct TwitchPubsubBitsInsertBuilder { + // ===== Required Fields ===== + + sender_twitch_user_id: Option, + sender_twitch_username: Option, + sender_twitch_username_lowercase: Option, + + destination_channel_id: Option, + destination_channel_name: Option, + + // ===== Optional / Default Fields ===== + + title: String, + prompt: String, + + user_text_input: String, + + redemption_id: String, + reward_id: String, + + reward_cost: u64, + + is_sub_only: bool, + max_per_stream: u64, + max_per_user_per_stream: u64, +} + +impl TwitchPubsubBitsInsertBuilder { + pub fn new() -> Self { + Self { + sender_twitch_user_id: None, + sender_twitch_username: None, + sender_twitch_username_lowercase: None, + destination_channel_id: None, + destination_channel_name: None, + title: "".to_string(), + prompt: "".to_string(), + user_text_input: "".to_string(), + redemption_id: "".to_string(), + reward_id: "".to_string(), + reward_cost: 0, + is_sub_only: false, + max_per_stream: 0, + max_per_user_per_stream: 0, + } + } + + pub fn set_sender_twitch_user_id(mut self, value: Option<&str>) -> Self { + self.sender_twitch_user_id = value.map(|s| s.to_string()); + self + } + + pub fn set_sender_twitch_username(mut self, value: Option<&str>) -> Self { + if let Some(username) = value { + self.sender_twitch_username = Some(username.to_string()); + self.sender_twitch_username_lowercase = Some(username.to_uppercase()); + } else { + self.sender_twitch_username = None; + self.sender_twitch_username_lowercase = None; + } + self + } + + pub fn set_destination_channel_id(mut self, value: Option<&str>) -> Self { + self.destination_channel_id = value.map(|s| s.to_string()); + self + } + + pub fn set_destination_channel_name(mut self, value: Option<&str>) -> Self { + self.destination_channel_name = value.map(|s| s.to_string()); + self + } + + pub fn set_title(mut self, value: &str) -> Self { + self.title = value.to_string(); + self + } + + pub fn set_prompt(mut self, value: &str) -> Self { + self.prompt = value.to_string(); + self + } + + pub fn set_user_tet_input(mut self, value: &str) -> Self { + self.user_text_input = value.to_string(); + self + } + + pub fn set_redemption_id(mut self, value: &str) -> Self { + self.redemption_id = value.to_string(); + self + } + + pub fn set_reward_id(mut self, value: &str) -> Self { + self.reward_id = value.to_string(); + self + } + + pub fn set_is_sub_only(mut self, value: bool) -> Self { + self.is_sub_only = value; + self + } + + pub fn set_max_per_stream(mut self, value: u64) -> Self { + self.max_per_stream = value; + self + } + + pub fn set_max_per_user_per_stream(mut self, value: u64) -> Self { + self.max_per_user_per_stream = value; + self + } + + pub async fn insert(&mut self, mysql_pool: &MySqlPool) -> AnyhowResult<()> { + let sender_twitch_user_id = self.sender_twitch_user_id + .clone() + .ok_or(anyhow!("no sender_twitch_user_id"))?; + + let sender_twitch_username = self.sender_twitch_username + .clone() + .ok_or(anyhow!("no sender_twitch_username"))?; + + let sender_twitch_username_lowercase = self.sender_twitch_username_lowercase + .clone() + .ok_or(anyhow!("no sender_twitch_username_lowercase"))?; + + let destination_channel_id = self.destination_channel_id + .clone() + .ok_or(anyhow!("no destination_channel_id"))?; + + let destination_channel_name = self.destination_channel_name + .clone() + .ok_or(anyhow!("no destination_channel_name"))?; + + let query = sqlx::query!( + r#" +INSERT INTO twitch_channel_point_events +SET + sender_twitch_user_id = ?, + sender_twitch_username = ?, + sender_twitch_username_lowercase = ?, + destination_channel_id = ?, + destination_channel_name = ?, + title = ?, + prompt = ?, + user_text_input = ?, + redemption_id = ?, + reward_id = ?, + reward_cost = ?, + is_sub_only = ?, + max_per_stream = ?, + max_per_user_per_stream = ? + "#, + sender_twitch_user_id, + sender_twitch_username, + sender_twitch_username_lowercase, + destination_channel_id, + destination_channel_name, + self.title.clone(), + self.prompt.clone(), + self.user_text_input.clone(), + self.redemption_id.clone(), + self.reward_id.clone(), + self.reward_cost, + self.is_sub_only, + self.max_per_stream, + self.max_per_user_per_stream, + ); + + let query_result = query.execute(mysql_pool) + .await; + + let _record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + return Err(anyhow!("Twitch pubsub bits insert DB error: {:?}", err)); + } + }; + + Ok(()) + } +} diff --git a/crates/lib/database_queries/src/twitch_pubsub/mod.rs b/crates/lib/database_queries/src/twitch_pubsub/mod.rs index 0930276ccd..1f8758da1b 100644 --- a/crates/lib/database_queries/src/twitch_pubsub/mod.rs +++ b/crates/lib/database_queries/src/twitch_pubsub/mod.rs @@ -1 +1,2 @@ -pub mod insert_bits; \ No newline at end of file +pub mod insert_bits; +pub mod insert_channel_points; \ No newline at end of file From da5c1893b318ba2ebd212f2c7c29a2b0ce30dc2f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 01:39:16 -0500 Subject: [PATCH 0831/1042] record channel bits event --- .../src/twitch_pubsub/insert_bits.rs | 29 +++---- .../twitch_pubsub/insert_channel_points.rs | 4 +- .../twitch_pubsub_user_subscriber_thread.rs | 81 ++++++++++++++++--- 3 files changed, 84 insertions(+), 30 deletions(-) diff --git a/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs b/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs index 16bb404bdc..73d1239816 100644 --- a/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs +++ b/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs @@ -40,29 +40,24 @@ impl TwitchPubsubBitsInsertBuilder { } } - pub fn set_sender_twitch_user_id(mut self, value: Option<&str>) -> Self { - self.sender_twitch_user_id = value.map(|s| s.to_string()); + pub fn set_sender_twitch_user_id(mut self, value: &str) -> Self { + self.sender_twitch_user_id = Some(value.to_string()); self } - pub fn set_sender_twitch_username(mut self, value: Option<&str>) -> Self { - if let Some(username) = value { - self.sender_twitch_username = Some(username.to_string()); - self.sender_twitch_username_lowercase = Some(username.to_uppercase()); - } else { - self.sender_twitch_username = None; - self.sender_twitch_username_lowercase = None; - } + pub fn set_sender_twitch_username(mut self, value: &str) -> Self { + self.sender_twitch_username = Some(value.to_string()); + self.sender_twitch_username_lowercase = Some(value.to_uppercase()); self } - pub fn set_destination_channel_id(mut self, value: Option<&str>) -> Self { - self.destination_channel_id = value.map(|s| s.to_string()); + pub fn set_destination_channel_id(mut self, value: &str) -> Self { + self.destination_channel_id = Some(value.to_string()); self } - pub fn set_destination_channel_name(mut self, value: Option<&str>) -> Self { - self.destination_channel_name = value.map(|s| s.to_string()); + pub fn set_destination_channel_name(mut self, value: &str) -> Self { + self.destination_channel_name = Some(value.to_string()); self } @@ -81,10 +76,8 @@ impl TwitchPubsubBitsInsertBuilder { self } - pub fn set_chat_message(mut self, value: Option<&str>) -> Self { - self.chat_message = value - .map(|s| s.to_string()) - .unwrap_or("".to_string()); + pub fn set_chat_message(mut self, value: &str) -> Self { + self.chat_message = value.to_string(); self } diff --git a/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs b/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs index 5bae2f0d12..0a701affda 100644 --- a/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs +++ b/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs @@ -5,7 +5,7 @@ use log::info; use log::warn; use sqlx::MySqlPool; -pub struct TwitchPubsubBitsInsertBuilder { +pub struct TwitchPubsubChannelPointsInsertBuilder { // ===== Required Fields ===== sender_twitch_user_id: Option, @@ -32,7 +32,7 @@ pub struct TwitchPubsubBitsInsertBuilder { max_per_user_per_stream: u64, } -impl TwitchPubsubBitsInsertBuilder { +impl TwitchPubsubChannelPointsInsertBuilder { pub fn new() -> Self { Self { sender_twitch_user_id: None, diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 5fd13eb7b4..8037bcc8b1 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -10,6 +10,8 @@ use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; use database_queries::twitch_oauth::find::{TwitchOauthTokenRecord, TwitchOauthTokenFinder}; use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; +use database_queries::twitch_pubsub::insert_bits::TwitchPubsubBitsInsertBuilder; +use database_queries::twitch_pubsub::insert_channel_points::TwitchPubsubChannelPointsInsertBuilder; use log::debug; use log::error; use log::info; @@ -23,7 +25,8 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use time::Instant; -use twitch_api2::pubsub::Response; +use twitch_api2::pubsub::{Response, TwitchResponse, TopicData}; +use twitch_api2::pubsub::channel_bits::ChannelBitsEventsV2Reply; // TODO: Publish events back to OBS thread // TODO: (cleanup) make the logic clearer to follow. @@ -258,23 +261,81 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { match event { Response::Response(response) => { - match response.error.as_deref() { - Some("ERR_BADAUTH") => { - warn!("Invalid token. Bad auth. Need to refresh"); - let token_record = self.refresh_twitch_oauth_token().await?; - return Ok(Some(LoopEndedReason::RefreshedOauthToken { token: token_record })); - } - _ => {}, - } + // NB: Auth failure might cause the loop to end. + return self.handle_maybe_auth_error_event(response).await; + } + Response::Message { data } => { + self.handle_pubsub_topic_event(data).await?; } - Response::Message { .. } => {} Response::Pong => {} Response::Reconnect => {} } + Ok(None) // Don't end loop + } + + async fn handle_maybe_auth_error_event(&mut self, response: TwitchResponse) + -> AnyhowResult> + { + let error = match response.error.as_deref() { + None => return Ok(None), + Some(e) => e, + }; + match error { + "ERR_BADAUTH" => { + warn!("Invalid token. Bad auth. Need to refresh"); + let token_record = self.refresh_twitch_oauth_token().await?; + return Ok(Some(LoopEndedReason::RefreshedOauthToken { token: token_record })); + } + _ => warn!("Unknown Twitch PubSub error: {:?}", error), + } Ok(None) } + async fn handle_pubsub_topic_event(&mut self, topic_data: TopicData) -> AnyhowResult<()> { + match topic_data { + // Unimplemented + TopicData::AutoModQueue { .. } => {} + TopicData::ChannelBitsBadgeUnlocks { .. } => {} + TopicData::ChatModeratorActions { .. } => {} + TopicData::ChannelSubscribeEventsV1 { .. } => {} + TopicData::CommunityPointsChannelV1 { .. } => {} + TopicData::ChannelCheerEventsPublicV1 { .. } => {} + TopicData::ChannelSubGiftsV1 { .. } => {} + TopicData::VideoPlayback { .. } => {} + TopicData::VideoPlaybackById { .. } => {} + TopicData::HypeTrainEventsV1 { .. } => {} + TopicData::HypeTrainEventsV1Rewards { .. } => {} + TopicData::Following { .. } => {} + TopicData::Raid { .. } => {} + TopicData::UserModerationNotifications { .. } => {} + // Implemented + TopicData::ChannelBitsEventsV2 { topic, reply } => { + let mut event_builder = TwitchPubsubBitsInsertBuilder::new(); + match *reply { + ChannelBitsEventsV2Reply::BitsEvent { data, message_id, version, is_anonymous } => { + let user_id = data.user_id.to_string(); + let user_name = data.user_name.to_string(); + let mut event_builder = event_builder.set_sender_twitch_user_id(&user_id) + .set_sender_twitch_username(&user_name) + .set_destination_channel_id(&data.channel_id.to_string()) + .set_destination_channel_name(&data.channel_name.to_string()) + .set_bits_used(data.bits_used as u64) + .set_total_bits_used(data.total_bits_used as u64) + .set_is_anonymous(is_anonymous) + .set_chat_message(&data.chat_message); + event_builder.insert(&self.mysql_pool).await?; + } + _ => {} + } + } + TopicData::ChannelPointsChannelV1 { topic, reply } => { + } + } + + Ok(()) + } + async fn maybe_send_twitch_ping(&mut self) -> AnyhowResult<()> { let mut should_send_ping = self.twitch_last_pinged_at .elapsed() From 6445244957d4c8bbb310bcafa533b0f5fa428cd7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 01:53:27 -0500 Subject: [PATCH 0832/1042] save both event types to db --- .../twitch_pubsub/insert_channel_points.rs | 29 ++++++++---------- .../twitch_pubsub_user_subscriber_thread.rs | 30 +++++++++++++++++++ 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs b/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs index 0a701affda..4973fb2b3b 100644 --- a/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs +++ b/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs @@ -52,29 +52,24 @@ impl TwitchPubsubChannelPointsInsertBuilder { } } - pub fn set_sender_twitch_user_id(mut self, value: Option<&str>) -> Self { - self.sender_twitch_user_id = value.map(|s| s.to_string()); + pub fn set_sender_twitch_user_id(mut self, value: &str) -> Self { + self.sender_twitch_user_id = Some(value.to_string()); self } - pub fn set_sender_twitch_username(mut self, value: Option<&str>) -> Self { - if let Some(username) = value { - self.sender_twitch_username = Some(username.to_string()); - self.sender_twitch_username_lowercase = Some(username.to_uppercase()); - } else { - self.sender_twitch_username = None; - self.sender_twitch_username_lowercase = None; - } + pub fn set_sender_twitch_username(mut self, value: &str) -> Self { + self.sender_twitch_username = Some(value.to_string()); + self.sender_twitch_username_lowercase = Some(value.to_uppercase()); self } - pub fn set_destination_channel_id(mut self, value: Option<&str>) -> Self { - self.destination_channel_id = value.map(|s| s.to_string()); + pub fn set_destination_channel_id(mut self, value: &str) -> Self { + self.destination_channel_id = Some(value.to_string()); self } - pub fn set_destination_channel_name(mut self, value: Option<&str>) -> Self { - self.destination_channel_name = value.map(|s| s.to_string()); + pub fn set_destination_channel_name(mut self, value: &str) -> Self { + self.destination_channel_name = Some(value.to_string()); self } @@ -88,8 +83,10 @@ impl TwitchPubsubChannelPointsInsertBuilder { self } - pub fn set_user_tet_input(mut self, value: &str) -> Self { - self.user_text_input = value.to_string(); + pub fn set_user_text_input(mut self, value: Option<&str>) -> Self { + self.user_text_input = value + .map(|s| s.to_string()) + .unwrap_or("".to_string()); self } diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 8037bcc8b1..d782e9ff69 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -27,6 +27,7 @@ use std::time::Duration; use time::Instant; use twitch_api2::pubsub::{Response, TwitchResponse, TopicData}; use twitch_api2::pubsub::channel_bits::ChannelBitsEventsV2Reply; +use twitch_api2::pubsub::channel_points::ChannelPointsChannelV1Reply; // TODO: Publish events back to OBS thread // TODO: (cleanup) make the logic clearer to follow. @@ -330,6 +331,35 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { } } TopicData::ChannelPointsChannelV1 { topic, reply } => { + let mut event_builder = TwitchPubsubChannelPointsInsertBuilder::new(); + match *reply { + // Unimplemented + ChannelPointsChannelV1Reply::CustomRewardUpdated { .. } => {} + ChannelPointsChannelV1Reply::RedemptionStatusUpdate { .. } => {} + ChannelPointsChannelV1Reply::UpdateRedemptionStatusesFinished { .. } => {} + ChannelPointsChannelV1Reply::UpdateRedemptionStatusProgress { .. } => {} + // Implemented + ChannelPointsChannelV1Reply::RewardRedeemed { timestamp, redemption } => { + let user_id = redemption.user.id.to_string(); + let user_name = redemption.user.login.to_string(); + let mut event_builder = event_builder.set_sender_twitch_user_id(&user_id) + .set_sender_twitch_username(&user_name) + .set_destination_channel_id(&redemption.channel_id.to_string()) + // TODO: + .set_destination_channel_name("todo: not available") + .set_title(&redemption.reward.title) + .set_prompt(&redemption.reward.prompt) + .set_user_text_input(redemption.user_input.as_deref()) + .set_redemption_id(&redemption.id.to_string()) + .set_reward_id(&redemption.reward.id.to_string()) + .set_is_sub_only(redemption.reward.is_sub_only); + // TODO: + // .set_max_per_stream(redemption.reward.max_per_stream as u64) + // .set_max_per_user_per_stream(redemption.reward.max_per_user_per_stream as u64); + event_builder.insert(&self.mysql_pool).await?; + } + _ => {}, + } } } From 3dbc1890da697af20f5cfe09689b1da6cc590d09 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 02:01:54 -0500 Subject: [PATCH 0833/1042] fixup bugs --- .../lib/database_queries/src/twitch_pubsub/insert_bits.rs | 2 +- .../src/twitch_pubsub/insert_channel_points.rs | 7 ++++++- .../src/threads/twitch_pubsub_user_subscriber_thread.rs | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs b/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs index 73d1239816..889d25dd4e 100644 --- a/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs +++ b/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs @@ -47,7 +47,7 @@ impl TwitchPubsubBitsInsertBuilder { pub fn set_sender_twitch_username(mut self, value: &str) -> Self { self.sender_twitch_username = Some(value.to_string()); - self.sender_twitch_username_lowercase = Some(value.to_uppercase()); + self.sender_twitch_username_lowercase = Some(value.to_lowercase()); self } diff --git a/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs b/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs index 4973fb2b3b..001055e6ba 100644 --- a/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs +++ b/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs @@ -59,7 +59,7 @@ impl TwitchPubsubChannelPointsInsertBuilder { pub fn set_sender_twitch_username(mut self, value: &str) -> Self { self.sender_twitch_username = Some(value.to_string()); - self.sender_twitch_username_lowercase = Some(value.to_uppercase()); + self.sender_twitch_username_lowercase = Some(value.to_lowercase()); self } @@ -105,6 +105,11 @@ impl TwitchPubsubChannelPointsInsertBuilder { self } + pub fn set_reward_cost(mut self, value: u64) -> Self { + self.reward_cost = value; + self + } + pub fn set_max_per_stream(mut self, value: u64) -> Self { self.max_per_stream = value; self diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index d782e9ff69..11812af32a 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -352,7 +352,8 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { .set_user_text_input(redemption.user_input.as_deref()) .set_redemption_id(&redemption.id.to_string()) .set_reward_id(&redemption.reward.id.to_string()) - .set_is_sub_only(redemption.reward.is_sub_only); + .set_is_sub_only(redemption.reward.is_sub_only) + .set_reward_cost(redemption.reward.cost as u64); // TODO: // .set_max_per_stream(redemption.reward.max_per_stream as u64) // .set_max_per_user_per_stream(redemption.reward.max_per_user_per_stream as u64); From 6cbcd7354c14c4e5703441e037d476106cf5162b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 02:02:28 -0500 Subject: [PATCH 0834/1042] codegen db --- sqlx-data.json | 60 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 5b53a085f0..dfbde1d2eb 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -375,16 +375,6 @@ ] } }, - "05030f800d4e16abedc75fd0697711e9396a0eae783ad86a86dd7c0540a4d489": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n maybe_twitch_username = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_subscriptions = ?,\n has_channel_read_redemptions= ?,\n has_user_read_follows = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 13 - }, - "nullable": [] - } - }, "05612e384bdce558e4834c720d1eb7ca9a09e3cca64b72f1497929d860345ec7": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -2306,6 +2296,16 @@ "nullable": [] } }, + "3a4a4edd2b1a1957c5eb42ed426ef4c11468549b9b8c3d62f1d59c07f4495c0e": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_subscriptions = ?,\n has_channel_read_redemptions= ?,\n has_user_read_follows = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 13 + }, + "nullable": [] + } + }, "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c": { "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -3965,6 +3965,16 @@ ] } }, + "5e4a00c339ce4d7fac19aff728449156b1e02b9e1d9fd534ac37d4ac79f44820": { + "query": "\nINSERT INTO twitch_bits_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n is_anonymous= ?,\n bits_used = ?,\n total_bits_used = ?,\n chat_message = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, "5e54fd8569b2d6d3a0d11acee6bd03b75400b6325952eb16c8dac68a7933b13a": { "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n is_banned,\n disable_gravatar,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", "describe": { @@ -4172,6 +4182,16 @@ ] } }, + "5eacdf9c65dc13820077be2930cca7047aed07804ed2e7b2ff38e12a617b4858": { + "query": "\nINSERT INTO twitch_channel_point_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n title = ?,\n prompt = ?,\n user_text_input = ?,\n redemption_id = ?,\n reward_id = ?,\n reward_cost = ?,\n is_sub_only = ?,\n max_per_stream = ?,\n max_per_user_per_stream = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 14 + }, + "nullable": [] + } + }, "60c06a507349919addb5d26b1c4a283a04f66fce86e84ed0a51db6001c9fc2b5": { "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -5970,16 +5990,6 @@ "nullable": [] } }, - "91beefeb04c0baafe4c3983c2760df2e5d08ea5e2433225221d20da8d2dca114": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n maybe_twitch_username = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_subscriptions = ?,\n has_channel_read_redemptions= ?,\n has_user_read_follows = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 11 - }, - "nullable": [] - } - }, "961f792c24bee6009e3e8a215561db932871a53c48839b8cb80292de00e7b2e1": { "query": "\nUPDATE tts_results\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -8288,6 +8298,16 @@ "nullable": [] } }, + "cad1a53adb21efd78fa3a6f3c8c0f4d5f55164c4f397b50426bf987ea68af3d0": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_subscriptions = ?,\n has_channel_read_redemptions= ?,\n has_user_read_follows = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 15 + }, + "nullable": [] + } + }, "cc4372cdd1a65999cbaf008f200683eedc93ab83e17256e47e011667ba8384d8": { "query": "\nUPDATE tts_results\nSET\n creator_set_visibility = ?,\n creator_ip_address = ?\nWHERE token = ?\nLIMIT 1\n ", "describe": { From e363f90eba072df90bf5201646c6bfbb6fd694a8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 02:20:31 -0500 Subject: [PATCH 0835/1042] Add new binaries to Dockerfile --- Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Dockerfile b/Dockerfile index 443fa23df4..88df556a05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -191,6 +191,18 @@ RUN SQLX_OFFLINE=true \ --release \ --bin w2l-inference-job +RUN SQLX_OFFLINE=true \ + LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ + $HOME/.cargo/bin/cargo build \ + --release \ + --bin obs-gateway + +RUN SQLX_OFFLINE=true \ + LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ + $HOME/.cargo/bin/cargo build \ + --release \ + --bin twitch-pubsub-job + # Final image # FROM ubuntu:xenial FROM pybuild-requirements as final @@ -233,6 +245,8 @@ COPY --from=rust-build /tmp/target/release/tts-download-job / COPY --from=rust-build /tmp/target/release/tts-inference-job / COPY --from=rust-build /tmp/target/release/w2l-download-job / COPY --from=rust-build /tmp/target/release/w2l-inference-job / +COPY --from=rust-build /tmp/target/release/obs-gateway / +COPY --from=rust-build /tmp/target/release/twitch-pubsub-job / # SSL certs are required for crypto COPY --from=rust-build /etc/ssl /etc/ssl From db582ca7e71e563ba766956b3ee7e9f0c241e549 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 03:04:12 -0500 Subject: [PATCH 0836/1042] Add twitch_common crate --- crates/lib/twitch_common/Cargo.toml | 22 ++++++++++++ crates/lib/twitch_common/src/cheers.rs | 47 ++++++++++++++++++++++++++ crates/lib/twitch_common/src/lib.rs | 1 + 3 files changed, 70 insertions(+) create mode 100644 crates/lib/twitch_common/Cargo.toml create mode 100644 crates/lib/twitch_common/src/cheers.rs create mode 100644 crates/lib/twitch_common/src/lib.rs diff --git a/crates/lib/twitch_common/Cargo.toml b/crates/lib/twitch_common/Cargo.toml new file mode 100644 index 0000000000..e82dc8e63b --- /dev/null +++ b/crates/lib/twitch_common/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "twitch_common" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "twitch_common" +path = "src/lib.rs" + +[dependencies] + +# Internal +container_common = { path = "../container_common" } + +# External +anyhow = "1.0.40" +once_cell = "1.9.0" diff --git a/crates/lib/twitch_common/src/cheers.rs b/crates/lib/twitch_common/src/cheers.rs new file mode 100644 index 0000000000..fda0b5de45 --- /dev/null +++ b/crates/lib/twitch_common/src/cheers.rs @@ -0,0 +1,47 @@ +use once_cell::sync::Lazy; +use std::collections::HashSet; + +/// These cheer emotes always combine with a number +/// This list seems pretty comprehensive +/// https://github.com/nossebro/TwitchPubSubMirror/blob/master/TwitchPubSubMirror_StreamlabsSystem.py + +static TWITCH_CHEERS : Lazy> = Lazy::new(|| { + let cheers = vec![ + "Cheer", + "DoodleCheer", + "BibleThump", + "cheerwhal", + "Corgo", + "uni", + "ShowLove", + "Party", + "SeemsGood", + "Pride", + "Kappa", + "FrankerZ", + "HeyGuys", + "DansGame", + "EleGiggle", + "TriHard", + "Kreygasm", + "4Head", + "SwiftRage", + "NotLikeThis", + "FailFish", + "VoHiYo", + "PJSalt", + "MrDestructoid", + "bday", + "RIPCheer", + "Shamrock", + "BitBoss", + "Streamlabs", + "Muxy", + "HolidayCheer" + ]; + let mut s = HashSet::new(); + for cheer in cheers.into_iter() { + s.insert(cheer.to_string()); + } + s +}); diff --git a/crates/lib/twitch_common/src/lib.rs b/crates/lib/twitch_common/src/lib.rs new file mode 100644 index 0000000000..3d395e4749 --- /dev/null +++ b/crates/lib/twitch_common/src/lib.rs @@ -0,0 +1 @@ +pub mod cheers; \ No newline at end of file From 313950f7b3a5b9ef6b862cbfb6150beab31968f6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 03:25:23 -0500 Subject: [PATCH 0837/1042] remove cheers from text --- Cargo.lock | 14 +++++++-- Cargo.toml | 1 + crates/lib/twitch_common/Cargo.toml | 1 + crates/lib/twitch_common/src/cheers.rs | 39 ++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5d0483912..f41c3968e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2643,9 +2643,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" [[package]] name = "opaque-debug" @@ -4399,6 +4399,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "twitch_common" +version = "0.0.1" +dependencies = [ + "anyhow", + "container_common", + "once_cell", + "regex", +] + [[package]] name = "twitch_oauth2" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index 30adcef836..d25059b54f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/lib/database_queries", "crates/lib/http_server_common", "crates/lib/redis_common", + "crates/lib/twitch_common", # Application "crates/service/obs_gateway", diff --git a/crates/lib/twitch_common/Cargo.toml b/crates/lib/twitch_common/Cargo.toml index e82dc8e63b..e6dc378397 100644 --- a/crates/lib/twitch_common/Cargo.toml +++ b/crates/lib/twitch_common/Cargo.toml @@ -20,3 +20,4 @@ container_common = { path = "../container_common" } # External anyhow = "1.0.40" once_cell = "1.9.0" +regex = "1.5.4" diff --git a/crates/lib/twitch_common/src/cheers.rs b/crates/lib/twitch_common/src/cheers.rs index fda0b5de45..e9cceed601 100644 --- a/crates/lib/twitch_common/src/cheers.rs +++ b/crates/lib/twitch_common/src/cheers.rs @@ -1,5 +1,32 @@ use once_cell::sync::Lazy; use std::collections::HashSet; +use regex::Regex; + +macro_rules! regex { + ($re:literal $(,)?) => {{ + static RE: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); + RE.get_or_init(|| regex::Regex::new($re).unwrap()) + }}; +} + + +pub fn remove_cheers(text: &str) -> String { + return TWITCH_CHEER_REGEX.replace_all(text, "").to_string() +} + +static TWITCH_CHEER_REGEX : Lazy = Lazy::new(|| { + let mut regex_pieces = Vec::new(); + + for cheer in TWITCH_CHEERS.iter() { + regex_pieces.push(format!("({}\\d+)", cheer)); + } + + let regex = regex_pieces.join("|"); + let regex = format!("({})", regex); + + Regex::new(®ex).unwrap() +}); + /// These cheer emotes always combine with a number /// This list seems pretty comprehensive @@ -45,3 +72,15 @@ static TWITCH_CHEERS : Lazy> = Lazy::new(|| { } s }); + +#[cfg(test)] +mod tests { + use crate::cheers::remove_cheers; + + // TODO: REMOVE SPACES + #[test] + fn test_remove_cheers() { + let r = remove_cheers("testing Kappa1 SeemsGood100 removal"); + assert_eq!(&r, "testing removal"); // TODO: REMOVE SPACES + } +} \ No newline at end of file From 1b5c3f6b3f46e3f8270490a47fe24112b92ca767 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 03:43:52 -0500 Subject: [PATCH 0838/1042] trim spaces --- crates/lib/twitch_common/src/cheers.rs | 58 +++++++++++++++----------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/crates/lib/twitch_common/src/cheers.rs b/crates/lib/twitch_common/src/cheers.rs index e9cceed601..d3d211a7df 100644 --- a/crates/lib/twitch_common/src/cheers.rs +++ b/crates/lib/twitch_common/src/cheers.rs @@ -2,32 +2,21 @@ use once_cell::sync::Lazy; use std::collections::HashSet; use regex::Regex; -macro_rules! regex { - ($re:literal $(,)?) => {{ - static RE: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); - RE.get_or_init(|| regex::Regex::new($re).unwrap()) - }}; -} - +// NB: This can be used as a nice, succinct macro with once_cell: +//macro_rules! regex { +// ($re:literal $(,)?) => {{ +// static RE: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); +// RE.get_or_init(|| regex::Regex::new($re).unwrap()) +// }}; +//} +/// Remove cheermotes from text, eg "this is great Cheer1" -> "this is great" pub fn remove_cheers(text: &str) -> String { - return TWITCH_CHEER_REGEX.replace_all(text, "").to_string() + let result = TWITCH_CHEER_REGEX.replace_all(text, ""); + let result = MULTI_SPACE_REGEX.replace_all(&result, " ").to_string(); + result.trim().to_string() } -static TWITCH_CHEER_REGEX : Lazy = Lazy::new(|| { - let mut regex_pieces = Vec::new(); - - for cheer in TWITCH_CHEERS.iter() { - regex_pieces.push(format!("({}\\d+)", cheer)); - } - - let regex = regex_pieces.join("|"); - let regex = format!("({})", regex); - - Regex::new(®ex).unwrap() -}); - - /// These cheer emotes always combine with a number /// This list seems pretty comprehensive /// https://github.com/nossebro/TwitchPubSubMirror/blob/master/TwitchPubSubMirror_StreamlabsSystem.py @@ -73,14 +62,33 @@ static TWITCH_CHEERS : Lazy> = Lazy::new(|| { s }); +static MULTI_SPACE_REGEX : Lazy = Lazy::new(|| { + Regex::new("(\\s{2,})").unwrap() +}); + +static TWITCH_CHEER_REGEX : Lazy = Lazy::new(|| { + let mut regex_pieces = Vec::new(); + + for cheer in TWITCH_CHEERS.iter() { + regex_pieces.push(format!("({}\\d+)", cheer)); + } + + let regex = regex_pieces.join("|"); + let regex = format!("({})", regex); + + Regex::new(®ex).unwrap() +}); + #[cfg(test)] mod tests { use crate::cheers::remove_cheers; - // TODO: REMOVE SPACES #[test] fn test_remove_cheers() { let r = remove_cheers("testing Kappa1 SeemsGood100 removal"); - assert_eq!(&r, "testing removal"); // TODO: REMOVE SPACES + assert_eq!(&r, "testing removal"); + + let r = remove_cheers("Kappa1 SeemsGood100 no more Cheer1 cheers"); + assert_eq!(&r, "no more cheers"); } -} \ No newline at end of file +} From 99b642f7acf9d9ec0e20db34d9c8990249959497 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 04:24:17 -0500 Subject: [PATCH 0839/1042] insert tts job records (wip) --- Cargo.lock | 1 + crates/lib/database_queries/src/builders.rs | 5 + crates/lib/database_queries/src/lib.rs | 6 +- .../src/tts/insert_tts_inference_job.rs | 150 ++++++++++++++++++ crates/lib/database_queries/src/tts/mod.rs | 1 + crates/service/obs_gateway/Cargo.toml | 1 + .../twitch_pubsub_user_subscriber_thread.rs | 18 ++- 7 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 crates/lib/database_queries/src/builders.rs create mode 100644 crates/lib/database_queries/src/tts/insert_tts_inference_job.rs create mode 100644 crates/lib/database_queries/src/tts/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f41c3968e1..c748dc5fe1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2637,6 +2637,7 @@ dependencies = [ "tokio-tungstenite", "toml", "twitch_api2", + "twitch_common", "twitch_oauth2", "uuid", ] diff --git a/crates/lib/database_queries/src/builders.rs b/crates/lib/database_queries/src/builders.rs new file mode 100644 index 0000000000..5738718cef --- /dev/null +++ b/crates/lib/database_queries/src/builders.rs @@ -0,0 +1,5 @@ + +/// This is simply a semantic / literate way of saying +/// an Optional should be required. There should be a +/// better way to actually enforce this, though. +pub (crate) type RequiredOption = Option; diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index 070f938779..42f79b2e27 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -1,5 +1,7 @@ #[macro_use] extern crate serde_derive; +mod builders; pub mod helpers; -pub mod twitch_pubsub; -pub mod twitch_oauth; \ No newline at end of file +pub mod tts; +pub mod twitch_oauth; +pub mod twitch_pubsub; \ No newline at end of file diff --git a/crates/lib/database_queries/src/tts/insert_tts_inference_job.rs b/crates/lib/database_queries/src/tts/insert_tts_inference_job.rs new file mode 100644 index 0000000000..3e3b20ba21 --- /dev/null +++ b/crates/lib/database_queries/src/tts/insert_tts_inference_job.rs @@ -0,0 +1,150 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use container_common::token::random_uuid::generate_random_uuid; +use crate::builders::RequiredOption; +use sqlx::MySqlPool; + +pub struct TtsInferenceJobInsertBuilder { + // ========== Required ========== + // TODO: The builder should generate this + // This requires moving token code into the shared db crate. + job_token: RequiredOption, + uuid_idempotency_token: RequiredOption, + model_token: RequiredOption, + raw_inference_text: RequiredOption, + + // ========== Conditionally optional ========== + creator_ip_address: RequiredOption, + + // ========== Optional ========== + maybe_creator_user_token: Option, + creator_set_visibility: Option, +} + +impl TtsInferenceJobInsertBuilder { + + /// Everything needs to be manually specified + pub fn new_for_fakeyou_request() -> Self { + Self { + job_token: None, + uuid_idempotency_token: None, + model_token: None, + raw_inference_text: None, + creator_ip_address: None, + maybe_creator_user_token: None, + creator_set_visibility: None, + } + } + + /// We can default a lot of the fields to "empty"-ish values + pub fn new_for_internal_tts() -> Self { + let idempotency_token = generate_random_uuid(); + Self { + job_token: None, + model_token: None, + raw_inference_text: None, + // Defaults + uuid_idempotency_token: Some(idempotency_token), + maybe_creator_user_token: None, + creator_ip_address: Some("127.0.0.1".to_string()), + creator_set_visibility: Some("public".to_string()), + } + } + + pub fn set_job_token(mut self, value: &str) -> Self { + self.job_token = Some(value.to_string()); + self + } + + pub fn set_model_token(mut self, value: &str) -> Self { + self.job_token = Some(value.to_string()); + self + } + + pub fn set_raw_inference_text(mut self, value: &str) -> Self { + self.raw_inference_text = Some(value.to_string()); + self + } + + pub fn set_uuid_idempotency_token(mut self, value: &str) -> Self { + self.uuid_idempotency_token = Some(value.to_string()); + self + } + + pub fn set_creator_ip_address(mut self, value: &str) -> Self { + self.creator_ip_address = Some(value.to_string()); + self + } + + pub fn set_maybe_creator_user_token(mut self, value: Option<&str>) -> Self { + self.maybe_creator_user_token = value.map(|s| s.to_string()); + self + } + + pub fn set_creator_set_visibility(mut self, value: &str) -> Self { + self.creator_set_visibility = Some(value.to_string()); + self + } + + pub async fn insert(&mut self, mysql_pool: &MySqlPool) -> AnyhowResult<()> { + let job_token = self.job_token + .clone() + .ok_or(anyhow!("no job_token"))?; + + let uuid_idempotency_token = self.uuid_idempotency_token + .clone() + .ok_or(anyhow!("no uuid_idempotency_token"))?; + + let model_token = self.model_token + .clone() + .ok_or(anyhow!("no model_token"))?; + + let raw_inference_text = self.raw_inference_text + .clone() + .ok_or(anyhow!("no raw_inference_text"))?; + + let creator_ip_address = self.creator_ip_address + .clone() + .ok_or(anyhow!("no creator_ip_address"))?; + + let creator_set_visibility = self.creator_set_visibility + .clone() + .ok_or(anyhow!("no creator_set_visibility"))?; + + let query = sqlx::query!( + r#" +INSERT INTO tts_inference_jobs +SET + token = ?, + uuid_idempotency_token = ?, + model_token = ?, + raw_inference_text = ?, + maybe_creator_user_token = ?, + creator_ip_address = ?, + creator_set_visibility = ?, + status = "pending" + "#, + job_token, + uuid_idempotency_token, + model_token, + raw_inference_text, + self.maybe_creator_user_token.clone(), + creator_ip_address, + creator_set_visibility, + ); + + let query_result = query.execute(mysql_pool) + .await; + + let _record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + return Err(anyhow!("Twitch pubsub bits insert DB error: {:?}", err)); + } + }; + + Ok(()) + } +} diff --git a/crates/lib/database_queries/src/tts/mod.rs b/crates/lib/database_queries/src/tts/mod.rs new file mode 100644 index 0000000000..f7b53931ee --- /dev/null +++ b/crates/lib/database_queries/src/tts/mod.rs @@ -0,0 +1 @@ +pub mod insert_tts_inference_job; \ No newline at end of file diff --git a/crates/service/obs_gateway/Cargo.toml b/crates/service/obs_gateway/Cargo.toml index 4c5ecb3fa0..a050251f02 100644 --- a/crates/service/obs_gateway/Cargo.toml +++ b/crates/service/obs_gateway/Cargo.toml @@ -24,6 +24,7 @@ container_common = { path = "../../lib/container_common" } database_queries = { path = "../../lib/database_queries" } http_server_common = { path = "../../lib/http_server_common" } redis_common = { path = "../../lib/redis_common" } +twitch_common = { path = "../../lib/twitch_common" } # External anyhow = "1.0.40" diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 11812af32a..96f38c725e 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -28,6 +28,7 @@ use time::Instant; use twitch_api2::pubsub::{Response, TwitchResponse, TopicData}; use twitch_api2::pubsub::channel_bits::ChannelBitsEventsV2Reply; use twitch_api2::pubsub::channel_points::ChannelPointsChannelV1Reply; +use twitch_common::cheers::remove_cheers; // TODO: Publish events back to OBS thread // TODO: (cleanup) make the logic clearer to follow. @@ -241,7 +242,7 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { } } - // =============== TWITCH PUBSUB EVENTS AND KEEPALIVE =============== + // =============== TWITCH PUBSUB EVENTS =============== async fn handle_event( &mut self, @@ -326,6 +327,8 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { .set_is_anonymous(is_anonymous) .set_chat_message(&data.chat_message); event_builder.insert(&self.mysql_pool).await?; + + self.write_tts_inference_event(&data.chat_message).await?; } _ => {} } @@ -358,6 +361,10 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { // .set_max_per_stream(redemption.reward.max_per_stream as u64) // .set_max_per_user_per_stream(redemption.reward.max_per_user_per_stream as u64); event_builder.insert(&self.mysql_pool).await?; + + if let Some(user_text) = redemption.user_input.as_deref() { + self.write_tts_inference_event(user_text).await?; + } } _ => {}, } @@ -367,6 +374,8 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { Ok(()) } + // =============== TWITCH PUBSUB KEEPALIVE =============== + async fn maybe_send_twitch_ping(&mut self) -> AnyhowResult<()> { let mut should_send_ping = self.twitch_last_pinged_at .elapsed() @@ -537,6 +546,13 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { Ok(new_record) } + + // =============== TTS EVENTS =============== + + async fn write_tts_inference_event(&mut self, tts_text: &str) -> AnyhowResult<()> { + let sanitized_text = remove_cheers(tts_text); + Ok(()) + } } // ==================================== From 69186282a0a2a3eebd11d172ad5ce35f031ace25 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 04:35:22 -0500 Subject: [PATCH 0840/1042] inserts pending job --- .../src/tts/insert_tts_inference_job.rs | 6 ++++-- .../threads/twitch_pubsub_user_subscriber_thread.rs | 10 ++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/lib/database_queries/src/tts/insert_tts_inference_job.rs b/crates/lib/database_queries/src/tts/insert_tts_inference_job.rs index 3e3b20ba21..c54900b0a0 100644 --- a/crates/lib/database_queries/src/tts/insert_tts_inference_job.rs +++ b/crates/lib/database_queries/src/tts/insert_tts_inference_job.rs @@ -47,7 +47,8 @@ impl TtsInferenceJobInsertBuilder { uuid_idempotency_token: Some(idempotency_token), maybe_creator_user_token: None, creator_ip_address: Some("127.0.0.1".to_string()), - creator_set_visibility: Some("public".to_string()), + // hidden | public | ... + creator_set_visibility: Some("hidden".to_string()), } } @@ -57,7 +58,7 @@ impl TtsInferenceJobInsertBuilder { } pub fn set_model_token(mut self, value: &str) -> Self { - self.job_token = Some(value.to_string()); + self.model_token = Some(value.to_string()); self } @@ -87,6 +88,7 @@ impl TtsInferenceJobInsertBuilder { } pub async fn insert(&mut self, mysql_pool: &MySqlPool) -> AnyhowResult<()> { + // TODO: These should be custom error types with a custom macro to make this easy. let job_token = self.job_token .clone() .ok_or(anyhow!("no job_token"))?; diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 96f38c725e..c316dbfa55 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -8,6 +8,7 @@ use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; +use database_queries::tts::insert_tts_inference_job::TtsInferenceJobInsertBuilder; use database_queries::twitch_oauth::find::{TwitchOauthTokenRecord, TwitchOauthTokenFinder}; use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; use database_queries::twitch_pubsub::insert_bits::TwitchPubsubBitsInsertBuilder; @@ -284,6 +285,7 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { Some(e) => e, }; match error { + "" => {}, // No-op "ERR_BADAUTH" => { warn!("Invalid token. Bad auth. Need to refresh"); let token_record = self.refresh_twitch_oauth_token().await?; @@ -551,6 +553,14 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { async fn write_tts_inference_event(&mut self, tts_text: &str) -> AnyhowResult<()> { let sanitized_text = remove_cheers(tts_text); + let model_token = "TM:7wbtjphx8h8v"; // "Mario *" voice. + + let mut builder = TtsInferenceJobInsertBuilder::new_for_internal_tts() + .set_job_token("todo") + .set_model_token(model_token) + .set_raw_inference_text(&sanitized_text); + + let _r = builder.insert(&self.mysql_pool).await?; Ok(()) } } From 39ab109f194df1ab9d96f797402f25738c84b1b7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 04:45:25 -0500 Subject: [PATCH 0841/1042] move tokens::Tokens util for MySql to common lib crate --- crates/lib/database_queries/src/lib.rs | 4 +++- .../database/helpers => lib/database_queries/src}/tokens.rs | 0 crates/service/storyteller_web/src/database/helpers/mod.rs | 3 +-- .../storyteller_web/src/database/mediators/badge_granter.rs | 1 - .../src/database/mediators/firehose_publisher.rs | 2 +- .../storyteller_web/src/database/queries/create_session.rs | 2 +- .../http_server/endpoints/categories/assign_tts_category.rs | 1 - .../src/http_server/endpoints/categories/create_category.rs | 2 +- .../endpoints/moderation/categories/delete_category.rs | 1 - .../endpoints/moderation/categories/edit_category.rs | 1 - .../src/http_server/endpoints/tts/enqueue_infer_tts.rs | 4 ++-- .../src/http_server/endpoints/tts/enqueue_upload_tts_model.rs | 2 +- .../src/http_server/endpoints/users/create_account.rs | 2 +- .../endpoints/w2l/enqueue_infer_w2l_with_uploads.rs | 2 +- .../http_server/endpoints/w2l/enqueue_upload_w2l_template.rs | 2 +- .../src/job_queries/tts_download_job_queries.rs | 2 +- .../src/job_queries/tts_inference_job_queries.rs | 2 +- .../src/job_queries/w2l_download_job_queries.rs | 2 +- .../src/job_queries/w2l_inference_job_queries.rs | 2 +- 19 files changed, 17 insertions(+), 20 deletions(-) rename crates/{service/storyteller_web/src/database/helpers => lib/database_queries/src}/tokens.rs (100%) diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index 42f79b2e27..39645e7c12 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -1,7 +1,9 @@ #[macro_use] extern crate serde_derive; mod builders; + pub mod helpers; +pub mod tokens; pub mod tts; pub mod twitch_oauth; -pub mod twitch_pubsub; \ No newline at end of file +pub mod twitch_pubsub; diff --git a/crates/service/storyteller_web/src/database/helpers/tokens.rs b/crates/lib/database_queries/src/tokens.rs similarity index 100% rename from crates/service/storyteller_web/src/database/helpers/tokens.rs rename to crates/lib/database_queries/src/tokens.rs diff --git a/crates/service/storyteller_web/src/database/helpers/mod.rs b/crates/service/storyteller_web/src/database/helpers/mod.rs index 81606ff216..b5f78fd29f 100644 --- a/crates/service/storyteller_web/src/database/helpers/mod.rs +++ b/crates/service/storyteller_web/src/database/helpers/mod.rs @@ -1,3 +1,2 @@ pub mod boolean_converters; -pub mod enums; -pub mod tokens; \ No newline at end of file +pub mod enums; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/mediators/badge_granter.rs b/crates/service/storyteller_web/src/database/mediators/badge_granter.rs index 791e8d42f5..cb2ff74ca9 100644 --- a/crates/service/storyteller_web/src/database/mediators/badge_granter.rs +++ b/crates/service/storyteller_web/src/database/mediators/badge_granter.rs @@ -1,6 +1,5 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use crate::database::helpers::tokens::Tokens; use crate::database::mediators::firehose_publisher::FirehosePublisher; use log::{warn,info}; use sqlx::error::Error::Database; diff --git a/crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs b/crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs index 3b9242259e..0dbb212dce 100644 --- a/crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs +++ b/crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use crate::database::helpers::tokens::Tokens; +use database_queries::tokens::Tokens; use log::{warn,info}; use sqlx::error::Error::Database; use sqlx::mysql::MySqlQueryResult; diff --git a/crates/service/storyteller_web/src/database/queries/create_session.rs b/crates/service/storyteller_web/src/database/queries/create_session.rs index e9310dd38e..aa9654f5e8 100644 --- a/crates/service/storyteller_web/src/database/queries/create_session.rs +++ b/crates/service/storyteller_web/src/database/queries/create_session.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use crate::AnyhowResult; -use crate::database::helpers::tokens::Tokens; +use database_queries::tokens::Tokens; use log::{info, warn}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs index b4d19725d2..b7b4974f6a 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs @@ -8,7 +8,6 @@ use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use anyhow::anyhow; use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::helpers::tokens::Tokens; use crate::database::queries::categories::get_category_by_token::get_category_by_token; use crate::database::queries::query_tts_model::select_tts_model_by_token; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs index df0d5359b3..b04633a3d4 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::helpers::tokens::Tokens; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -16,6 +15,7 @@ use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; use crate::validations::check_for_slurs::contains_slurs; +use database_queries::tokens::Tokens; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs index 922f14b4e8..5e1824495a 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs @@ -24,7 +24,6 @@ use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use std::fmt; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; -use crate::database::helpers::tokens::Tokens; // =============== Request =============== diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs index eaa4d59d27..918615d946 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::helpers::tokens::Tokens; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 66d469e0e8..04648edc48 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -1,13 +1,12 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::enums::{DownloadUrlType, W2lTemplateType}; -use crate::database::helpers::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -15,6 +14,7 @@ use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; +use database_queries::tokens::Tokens; use derive_more::{Display, Error}; use log::{info, warn, log}; use r2d2_redis::redis::Commands; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index d14470cb0e..efed50ea53 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -6,13 +6,13 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database::helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; -use crate::database::helpers::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; +use database_queries::tokens::Tokens; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs index df89e15bf4..16bd4e627a 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs @@ -5,7 +5,6 @@ use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::tokens::Tokens; use crate::database::queries::create_session::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountErrorType::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::web_utils::ip_address::get_request_ip; @@ -17,6 +16,7 @@ use crate::validations::check_for_slurs::contains_slurs; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use crate::validations::username_reservations::is_reserved_username; +use database_queries::tokens::Tokens; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index ffe40a2a87..5f1302a203 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -10,7 +10,6 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; use container_common::token::random_uuid::generate_random_uuid; use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::helpers::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::read_multipart_field_bytes::checked_read_multipart_bytes; use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_boolean; @@ -20,6 +19,7 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_paths::hash_to_bucket_path; +use database_queries::tokens::Tokens; use derive_more::{Display, Error}; use futures::{StreamExt, TryStreamExt}; use log::{warn, info}; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index 21d279c930..81d6cbb0b5 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; -use crate::database::helpers::tokens::Tokens; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -14,6 +13,7 @@ use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; +use database_queries::tokens::Tokens; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs b/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs index 604722b377..b3a4a85560 100644 --- a/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs @@ -5,7 +5,7 @@ use anyhow::anyhow; use chrono::Utc; use container_common::anyhow_result::AnyhowResult; -use crate::database::helpers::tokens::Tokens; +use database_queries::tokens::Tokens; use sqlx::MySqlPool; use std::path::Path; diff --git a/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs b/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs index 5140f513dc..b0f551d3ce 100644 --- a/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs @@ -7,7 +7,7 @@ use chrono::{Utc, DateTime}; use container_common::anyhow_result::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; use crate::database::enums::vocoder_type::VocoderType; -use crate::database::helpers::tokens::Tokens; +use database_queries::tokens::Tokens; use log::{warn, info}; use sqlx::{MySqlPool, Transaction, MySql}; use std::error; diff --git a/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs b/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs index 64099c7cb0..c49a5128d1 100644 --- a/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs @@ -5,7 +5,7 @@ use anyhow::anyhow; use chrono::Utc; use container_common::anyhow_result::AnyhowResult; -use crate::database::helpers::tokens::Tokens; +use database_queries::tokens::Tokens; use sqlx::MySqlPool; /// table: w2l_template_upload_jobs diff --git a/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs b/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs index 6ce1022c14..11f698fa1c 100644 --- a/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs @@ -6,7 +6,7 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; use container_common::anyhow_result::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::helpers::tokens::Tokens; +use database_queries::tokens::Tokens; use log::{warn, info}; use sqlx::MySqlPool; use std::path::Path; From 7563f48f057cf8b8bc9fdc2de00fec77a3860ec2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 04:51:03 -0500 Subject: [PATCH 0842/1042] job token --- .../src/threads/twitch_pubsub_user_subscriber_thread.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index c316dbfa55..25099e55a4 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -8,6 +8,7 @@ use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; +use database_queries::tokens::Tokens; use database_queries::tts::insert_tts_inference_job::TtsInferenceJobInsertBuilder; use database_queries::twitch_oauth::find::{TwitchOauthTokenRecord, TwitchOauthTokenFinder}; use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; @@ -26,9 +27,9 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use time::Instant; -use twitch_api2::pubsub::{Response, TwitchResponse, TopicData}; use twitch_api2::pubsub::channel_bits::ChannelBitsEventsV2Reply; use twitch_api2::pubsub::channel_points::ChannelPointsChannelV1Reply; +use twitch_api2::pubsub::{Response, TwitchResponse, TopicData}; use twitch_common::cheers::remove_cheers; // TODO: Publish events back to OBS thread @@ -553,14 +554,17 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { async fn write_tts_inference_event(&mut self, tts_text: &str) -> AnyhowResult<()> { let sanitized_text = remove_cheers(tts_text); + let job_token = Tokens::new_tts_inference_job()?; let model_token = "TM:7wbtjphx8h8v"; // "Mario *" voice. let mut builder = TtsInferenceJobInsertBuilder::new_for_internal_tts() - .set_job_token("todo") + .set_job_token(&job_token) .set_model_token(model_token) .set_raw_inference_text(&sanitized_text); let _r = builder.insert(&self.mysql_pool).await?; + + // TODO: Report job token to frontend Ok(()) } } @@ -586,4 +590,3 @@ enum LoopEndedReason { /// A new OAuth token was minted. RefreshedOauthToken { token: TwitchOauthTokenRecord } } - From 1ed43fb5103b1e965dc2dcf6ed192d1463debf79 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 06:15:12 -0500 Subject: [PATCH 0843/1042] codegen db --- sqlx-data.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sqlx-data.json b/sqlx-data.json index dfbde1d2eb..2841026a80 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -3823,6 +3823,16 @@ ] } }, + "591bda610a0fe952cafc5d97808757da8652133082e94ad15ab063db9d4e2f44": { + "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + } + }, "5d1f983fd2555c0fcb2ba065e9b3c6174346ee0fdaecacc2a557715a68cf23e2": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { From 4b1aad995d5c08c58dc131229819e3326ffddfc2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 06:20:06 -0500 Subject: [PATCH 0844/1042] CORS: api.fakeyou.com --- crates/lib/http_server_common/src/cors.rs | 28 +++++++++++++++-------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/crates/lib/http_server_common/src/cors.rs b/crates/lib/http_server_common/src/cors.rs index ea7c6653f5..9d66ff0259 100644 --- a/crates/lib/http_server_common/src/cors.rs +++ b/crates/lib/http_server_common/src/cors.rs @@ -5,10 +5,8 @@ use actix_http::http; /// Return cors config for FakeYou / Vocodes / OBS / local development pub fn build_common_cors_config() -> Cors { Cors::default() - .allowed_origin("http://api.fakeyou.com") + // Local Development .allowed_origin("http://api.jungle.horse") - .allowed_origin("http://api.vo.codes") - .allowed_origin("http://fakeyou.com") .allowed_origin("http://jungle.horse") .allowed_origin("http://jungle.horse:12345") .allowed_origin("http://jungle.horse:7000") @@ -19,21 +17,31 @@ pub fn build_common_cors_config() -> Cors { .allowed_origin("http://localhost:7000") .allowed_origin("http://localhost:8000") .allowed_origin("http://localhost:8080") - .allowed_origin("http://mumble.stream") + .allowed_origin("https://api.jungle.horse") + .allowed_origin("https://jungle.horse") + // FakeYou + .allowed_origin("http://api.fakeyou.com") + .allowed_origin("http://fakeyou.com") + .allowed_origin("https://api.fakeyou.com") + .allowed_origin("https://fakeyou.com") + // Storyteller + .allowed_origin("http://api.storyteller.io") .allowed_origin("http://storyteller.io") + .allowed_origin("https://api.storyteller.io") + .allowed_origin("https://storyteller.io") + // Trumped (legacy) .allowed_origin("http://trumped.com") + .allowed_origin("https://trumped.com") + // Vocodes (legacy) + .allowed_origin("http://api.vo.codes") + .allowed_origin("http://mumble.stream") .allowed_origin("http://vo.codes") .allowed_origin("http://vocodes.com") - .allowed_origin("https://api.fakeyou.com") - .allowed_origin("https://api.jungle.horse") .allowed_origin("https://api.vo.codes") - .allowed_origin("https://fakeyou.com") - .allowed_origin("https://jungle.horse") .allowed_origin("https://mumble.stream") - .allowed_origin("https://storyteller.io") - .allowed_origin("https://trumped.com") .allowed_origin("https://vo.codes") .allowed_origin("https://vocodes.com") + // Remaining setup .allowed_methods(vec!["GET", "POST", "OPTIONS"]) .supports_credentials() .allowed_headers(vec![ From 2308d5f5d2f05f0adb4974bf5d36303a135b7e37 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 06:29:55 -0500 Subject: [PATCH 0845/1042] twitch secrets from env --- crates/service/obs_gateway/src/main.rs | 2 +- .../service/obs_gateway/src/twitch/twitch_secrets.rs | 10 ---------- .../service/obs_gateway/src/twitch/twitch_user_id.rs | 4 ++-- crates/service/obs_gateway/src/twitch_pubsub_job.rs | 6 +++++- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/crates/service/obs_gateway/src/main.rs b/crates/service/obs_gateway/src/main.rs index 69e3c2209f..22bac536f7 100644 --- a/crates/service/obs_gateway/src/main.rs +++ b/crates/service/obs_gateway/src/main.rs @@ -77,7 +77,7 @@ async fn main() -> AnyhowResult<()> { info!("Reading Twitch secrets..."); - let secrets = TwitchSecrets::from_file("twitch_secrets.toml")?; + let secrets = TwitchSecrets::from_env()?; let client_id = ClientId::new(&secrets.app_client_id); let client_secret = ClientSecret::new(&secrets.app_client_secret); diff --git a/crates/service/obs_gateway/src/twitch/twitch_secrets.rs b/crates/service/obs_gateway/src/twitch/twitch_secrets.rs index 2cb6c72e55..ac828ff86d 100644 --- a/crates/service/obs_gateway/src/twitch/twitch_secrets.rs +++ b/crates/service/obs_gateway/src/twitch/twitch_secrets.rs @@ -1,25 +1,17 @@ use container_common::anyhow_result::AnyhowResult; -// TODO: Not sure I need all of these. The less I need to keep, the better. -// Oauth delegation in particular seems not necessary. #[derive(Clone, serde::Deserialize)] pub struct TwitchSecrets { - pub username: String, - pub stream_key: String, pub app_client_id: String, pub app_client_secret: String, } impl TwitchSecrets { pub fn new( - username: &str, - stream_key: &str, app_client_id: &str, app_client_secret: &str, ) -> Self { Self { - username: username.to_string(), - stream_key: stream_key.to_string(), app_client_id: app_client_id.to_string(), app_client_secret: app_client_secret.to_string(), } @@ -27,8 +19,6 @@ impl TwitchSecrets { pub fn from_env() -> AnyhowResult { Ok(Self::new( - &easyenv::get_env_string_required("TWITCH_USERNAME")?, - &easyenv::get_env_string_required("TWITCH_STREAM_KEY")?, &easyenv::get_env_string_required("TWITCH_APP_CLIENT_ID")?, &easyenv::get_env_string_required("TWITCH_APP_CLIENT_SECRET")?, )) diff --git a/crates/service/obs_gateway/src/twitch/twitch_user_id.rs b/crates/service/obs_gateway/src/twitch/twitch_user_id.rs index a6a5022899..ae1b83aa41 100644 --- a/crates/service/obs_gateway/src/twitch/twitch_user_id.rs +++ b/crates/service/obs_gateway/src/twitch/twitch_user_id.rs @@ -1,7 +1,7 @@ use container_common::anyhow_result::AnyhowResult; -/// Twitch says to encode these as strings, but the Twitch libraries -/// still deal in integers. This centralizes type juggling. +/// Twitch says to encode these as strings, but the Twitch Rust libraries +/// still deal in integers (uh oh). This centralizes type juggling. #[derive(Clone)] pub struct TwitchUserId { string_user_id: String, diff --git a/crates/service/obs_gateway/src/twitch_pubsub_job.rs b/crates/service/obs_gateway/src/twitch_pubsub_job.rs index 325ec9f36f..cd4bf47274 100644 --- a/crates/service/obs_gateway/src/twitch_pubsub_job.rs +++ b/crates/service/obs_gateway/src/twitch_pubsub_job.rs @@ -41,6 +41,10 @@ use twitch_oauth2::{ClientId, ClientSecret, RefreshToken, AccessToken}; pub async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + // NB: Do not check this secrets-containing dotenv file into VCS. + // This file should only contain *development* secrets, never production. + let _ = dotenv::from_filename(".env-secrets").ok(); + let server_hostname = hostname::get() .ok() .and_then(|h| h.into_string().ok()) @@ -48,7 +52,7 @@ pub async fn main() -> AnyhowResult<()> { info!("Reading Twitch secrets..."); - let secrets = TwitchSecrets::from_file("twitch_secrets.toml")?; + let secrets = TwitchSecrets::from_env()?; let client_id = ClientId::new(&secrets.app_client_id); let client_secret = ClientSecret::new(&secrets.app_client_secret); From c6656aa417a1eab61c092315d8d7a201210d6069 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 16:53:37 -0500 Subject: [PATCH 0846/1042] Send to frontend --- crates/lib/redis_common/src/redis_keys.rs | 7 ++++ .../obs_gateway_websocket_handler.rs | 37 +++++++++++++++---- crates/service/obs_gateway/src/main.rs | 15 -------- .../obs_gateway/src/redis/constants.rs | 3 ++ .../twitch_pubsub_user_subscriber_thread.rs | 8 +++- 5 files changed, 47 insertions(+), 23 deletions(-) diff --git a/crates/lib/redis_common/src/redis_keys.rs b/crates/lib/redis_common/src/redis_keys.rs index 7fd1829978..e38f826b30 100644 --- a/crates/lib/redis_common/src/redis_keys.rs +++ b/crates/lib/redis_common/src/redis_keys.rs @@ -71,4 +71,11 @@ impl RedisKeys { pub fn twitch_events_topic(twitch_user_id: &str) -> String { format!("twitchEventsTopic:{}", twitch_user_id) } + + /// This is a lazy first approximation of the worker -> OBS messaging. + /// We push new TTS job tokens into a Redis list (with long expiry), and OBS + /// will dequeue the events in order. + pub fn twitch_tts_job_queue(twitch_user_id: &str) -> String { + format!("twitchTtsJobs:{}", twitch_user_id) + } } diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 3454e55480..c78460db93 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -46,6 +46,12 @@ pub struct QueryData { // Other preferences... } +/// Sent back to the frontend websocket. +#[derive(Serialize)] +pub struct FrontendEventPayload { + pub tts_job_tokens: Vec, +} + /// Endpoint pub async fn obs_gateway_websocket_handler( path: Path, @@ -270,17 +276,34 @@ impl StreamHandler> for ObsGatewayWebSock self.write_obs_active().unwrap(); - //BitsEvent {} - //ChannelPointsChannelV1 {} + let redis_key = RedisKeys::twitch_tts_job_queue(&self.twitch_user_id.get_str()); + let values : Vec = redis.lpop((redis_key, 5)).unwrap(); // TODO: Error handling + if !values.is_empty() { + info!("Got {} TTS values", values.len()); + + let payload = FrontendEventPayload { + tts_job_tokens: values, + }; + + match serde_json::to_string(&payload) { + Ok(json) => { + ctx.text(json); + }, + Err(e) => { + error!("Error with JSON payload: {:?}", e); + } + } + } match msg { - ws::Message::Text(text) => { - //info!("sending text response"); - ctx.text("hello from Rust") - }, - ws::Message::Binary(bin) => ctx.binary(bin), ws::Message::Ping(bytes) => ctx.pong(&bytes), + ws::Message::Text(text) => { + //ctx.text("response") + }, + ws::Message::Binary(bin) => { + //ctx.binary("response".as_bytes()) + }, ws::Message::Close(reason) => { ctx.close(reason); ctx.stop(); diff --git a/crates/service/obs_gateway/src/main.rs b/crates/service/obs_gateway/src/main.rs index 22bac536f7..968b397429 100644 --- a/crates/service/obs_gateway/src/main.rs +++ b/crates/service/obs_gateway/src/main.rs @@ -151,21 +151,6 @@ async fn main() -> AnyhowResult<()> { } }; - // Works, but can't Ctrl-C - //let handle = Handle::current(); - //handle.spawn_blocking(|| { - // loop { - // info!("...thread..."); - // sleep(Duration::from_millis(2000)); - // } - //}); - - //let tokio_runtime = Runtime::new()?; - //tokio_runtime.spawn(async { - // poll_ip_bans(ip_banlist2, mysql_pool3).await; - //}); - - info!("Starting server..."); serve(server_state).await?; diff --git a/crates/service/obs_gateway/src/redis/constants.rs b/crates/service/obs_gateway/src/redis/constants.rs index a5117ad61e..19dd683298 100644 --- a/crates/service/obs_gateway/src/redis/constants.rs +++ b/crates/service/obs_gateway/src/redis/constants.rs @@ -20,3 +20,6 @@ pub const OBS_ACTIVE_TTL_SECONDS : usize = 120; /// How often to check the OBS session activity. pub const OBS_ACTIVE_CHECK_PERIOD : Duration = Duration::seconds(30); + +/// When to expire the streamer-specific TTS job queue notifications +pub const STREAMER_TTS_JOB_QUEUE_TTL_SECONDS : usize = 600; diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 25099e55a4..8f684f4d5f 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Error}; use container_common::anyhow_result::AnyhowResult; use container_common::thread::thread_id::ThreadId; -use crate::redis::constants::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD, LEASE_CHECK_PERIOD, OBS_ACTIVE_CHECK_PERIOD}; +use crate::redis::constants::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD, LEASE_CHECK_PERIOD, OBS_ACTIVE_CHECK_PERIOD, STREAMER_TTS_JOB_QUEUE_TTL_SECONDS}; use crate::redis::lease_payload::LeasePayload; use crate::twitch::constants::TWITCH_PING_CADENCE; use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; @@ -565,6 +565,12 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { let _r = builder.insert(&self.mysql_pool).await?; // TODO: Report job token to frontend + let mut redis = self.redis_pool.get()?; + let redis_key = RedisKeys::twitch_tts_job_queue(&self.twitch_user_id.get_str()); + + let _size : Option = redis.rpush(&redis_key, job_token)?; + let _size : Option = redis.expire(&redis_key, STREAMER_TTS_JOB_QUEUE_TTL_SECONDS)?; + Ok(()) } } From c30fbfbf82cfda98399c3fabc60b0dda8458ad03 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 Jan 2022 22:43:50 -0500 Subject: [PATCH 0847/1042] SQLX db codegen / cache failed (??), this should fix it --- codegen_database.sh | 45 ++++++++++++++++--- sqlx-data.json | 102 ++++++++++++++++++++++---------------------- 2 files changed, 91 insertions(+), 56 deletions(-) diff --git a/codegen_database.sh b/codegen_database.sh index 74ed8ec3fa..dc9d904b7f 100755 --- a/codegen_database.sh +++ b/codegen_database.sh @@ -1,11 +1,46 @@ #!/bin/bash -# Necessary since we're now in a more complex Rust workspace -pushd crates/service/storyteller_web +# This is complicated since we're now in a more complex Rust workspace with +# multiple projects and targets. -SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web +# NB(2022-01-31): For some reason, 'storyteller-web', despite importing the obs queries +# via the `database_queries` shared library, started to exclude these queries. It was +# working, but mysteriously stopped. I'm using this more complicated setup with jq-based +# json merge as a stop gap. -popd +# NB: This format can be used without changing directory: +# SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web --manifest-path crates/service/storyteller_web/Cargo.toml -mv crates/service/storyteller_web/sqlx-data.json . +build_shared_database_library() { + pushd crates/lib/database_queries + SQLX_OFFLINE=true cargo sqlx prepare + popd +} + +build_storyteller_web_app() { + # NB: This imports the inference/upload job queries + # It should also import the shared database lib queries, but something(???) broke. + pushd crates/service/storyteller_web + SQLX_OFFLINE=true cargo sqlx prepare -- --bin storyteller-web # NB: Because multiple binary targets + popd +} + +combine_sqlx_queries() { + # Merge multiple JSON files into a single dictionary. + # https://stackoverflow.com/a/24904276 + jq -s '.[0] * .[1]' \ + crates/lib/database_queries/sqlx-data.json \ + crates/service/storyteller_web/sqlx-data.json \ + > sqlx-data.json +} + +cleanup_temp_files() { + rm crates/lib/database_queries/sqlx-data.json \ + crates/service/storyteller_web/sqlx-data.json +} + +build_shared_database_library +build_storyteller_web_app +combine_sqlx_queries +cleanup_temp_files diff --git a/sqlx-data.json b/sqlx-data.json index 2841026a80..5701613287 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1,5 +1,55 @@ { "db": "MySQL", + "3a4a4edd2b1a1957c5eb42ed426ef4c11468549b9b8c3d62f1d59c07f4495c0e": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_subscriptions = ?,\n has_channel_read_redemptions= ?,\n has_user_read_follows = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 13 + }, + "nullable": [] + } + }, + "591bda610a0fe952cafc5d97808757da8652133082e94ad15ab063db9d4e2f44": { + "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + } + }, + "5e4a00c339ce4d7fac19aff728449156b1e02b9e1d9fd534ac37d4ac79f44820": { + "query": "\nINSERT INTO twitch_bits_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n is_anonymous= ?,\n bits_used = ?,\n total_bits_used = ?,\n chat_message = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, + "5eacdf9c65dc13820077be2930cca7047aed07804ed2e7b2ff38e12a617b4858": { + "query": "\nINSERT INTO twitch_channel_point_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n title = ?,\n prompt = ?,\n user_text_input = ?,\n redemption_id = ?,\n reward_id = ?,\n reward_cost = ?,\n is_sub_only = ?,\n max_per_stream = ?,\n max_per_user_per_stream = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 14 + }, + "nullable": [] + } + }, + "cad1a53adb21efd78fa3a6f3c8c0f4d5f55164c4f397b50426bf987ea68af3d0": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_subscriptions = ?,\n has_channel_read_redemptions= ?,\n has_user_read_follows = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 15 + }, + "nullable": [] + } + }, "0033c76be0ea1c99069f1a8df1ce400c7064526feaf5dc88930da0718a885b19": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.is_banned as creator_is_banned,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n w2l.title,\n w2l.description_markdown,\n w2l.description_rendered_html,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", "describe": { @@ -2296,16 +2346,6 @@ "nullable": [] } }, - "3a4a4edd2b1a1957c5eb42ed426ef4c11468549b9b8c3d62f1d59c07f4495c0e": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_subscriptions = ?,\n has_channel_read_redemptions= ?,\n has_user_read_follows = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 13 - }, - "nullable": [] - } - }, "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c": { "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -3823,16 +3863,6 @@ ] } }, - "591bda610a0fe952cafc5d97808757da8652133082e94ad15ab063db9d4e2f44": { - "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 7 - }, - "nullable": [] - } - }, "5d1f983fd2555c0fcb2ba065e9b3c6174346ee0fdaecacc2a557715a68cf23e2": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -3975,16 +4005,6 @@ ] } }, - "5e4a00c339ce4d7fac19aff728449156b1e02b9e1d9fd534ac37d4ac79f44820": { - "query": "\nINSERT INTO twitch_bits_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n is_anonymous= ?,\n bits_used = ?,\n total_bits_used = ?,\n chat_message = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 9 - }, - "nullable": [] - } - }, "5e54fd8569b2d6d3a0d11acee6bd03b75400b6325952eb16c8dac68a7933b13a": { "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n is_banned,\n disable_gravatar,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", "describe": { @@ -4192,16 +4212,6 @@ ] } }, - "5eacdf9c65dc13820077be2930cca7047aed07804ed2e7b2ff38e12a617b4858": { - "query": "\nINSERT INTO twitch_channel_point_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n title = ?,\n prompt = ?,\n user_text_input = ?,\n redemption_id = ?,\n reward_id = ?,\n reward_cost = ?,\n is_sub_only = ?,\n max_per_stream = ?,\n max_per_user_per_stream = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 14 - }, - "nullable": [] - } - }, "60c06a507349919addb5d26b1c4a283a04f66fce86e84ed0a51db6001c9fc2b5": { "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -8308,16 +8318,6 @@ "nullable": [] } }, - "cad1a53adb21efd78fa3a6f3c8c0f4d5f55164c4f397b50426bf987ea68af3d0": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_subscriptions = ?,\n has_channel_read_redemptions= ?,\n has_user_read_follows = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 15 - }, - "nullable": [] - } - }, "cc4372cdd1a65999cbaf008f200683eedc93ab83e17256e47e011667ba8384d8": { "query": "\nUPDATE tts_results\nSET\n creator_set_visibility = ?,\n creator_ip_address = ?\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -9492,4 +9492,4 @@ "nullable": [] } } -} \ No newline at end of file +} From 24701aa9a8fb8fe7f762b6706b24323b5a145875 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Feb 2022 00:38:40 -0500 Subject: [PATCH 0848/1042] Move Twitch OAuth onboarding to storyteller-web --- Cargo.lock | 9 ++ crates/lib/twitch_common/Cargo.toml | 10 ++ crates/lib/twitch_common/src/lib.rs | 4 +- .../twitch_common/src}/oauth_token_builder.rs | 0 .../twitch_common/src}/twitch_secrets.rs | 8 +- .../obs_gateway_websocket_handler.rs | 100 ------------------ crates/service/obs_gateway/src/main.rs | 45 ++------ .../service/obs_gateway/src/server_state.rs | 11 -- crates/service/obs_gateway/src/twitch/mod.rs | 1 - .../obs_gateway/src/twitch/oauth/mod.rs | 1 - .../obs_gateway/src/twitch_pubsub_job.rs | 2 +- crates/service/storyteller_web/Cargo.toml | 12 +++ .../src/http_server/endpoints/mod.rs | 1 + .../src/http_server/endpoints/twitch}/mod.rs | 2 +- .../endpoints/twitch}/oauth_begin.rs | 6 +- .../endpoints/twitch}/oauth_begin_redirect.rs | 6 +- .../endpoints/twitch}/oauth_end.rs | 14 +-- crates/service/storyteller_web/src/main.rs | 14 ++- crates/service/storyteller_web/src/routes.rs | 35 ++++++ .../storyteller_web/src/server_state.rs | 10 ++ 20 files changed, 116 insertions(+), 175 deletions(-) rename crates/{service/obs_gateway/src/twitch/oauth => lib/twitch_common/src}/oauth_token_builder.rs (100%) rename crates/{service/obs_gateway/src/twitch => lib/twitch_common/src}/twitch_secrets.rs (77%) rename crates/service/{obs_gateway/src/endpoints => storyteller_web/src/http_server/endpoints/twitch}/mod.rs (72%) rename crates/service/{obs_gateway/src/endpoints => storyteller_web/src/http_server/endpoints/twitch}/oauth_begin.rs (92%) rename crates/service/{obs_gateway/src/endpoints => storyteller_web/src/http_server/endpoints/twitch}/oauth_begin_redirect.rs (92%) rename crates/service/{obs_gateway/src/endpoints => storyteller_web/src/http_server/endpoints/twitch}/oauth_end.rs (91%) diff --git a/Cargo.lock b/Cargo.lock index c748dc5fe1..83b5ebb9d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3793,12 +3793,14 @@ dependencies = [ "lru_time_cache", "magic-crypt", "md-5", + "native-tls", "newrelic-telemetry", "pulldown-cmark", "r2d2_redis", "rand 0.8.4", "redis_common", "regex", + "reqwest", "ring 0.17.0-alpha.10", "rust-s3", "serde", @@ -3811,6 +3813,9 @@ dependencies = [ "time 0.2.26", "tokio 1.13.0", "toml", + "twitch_api2", + "twitch_common", + "twitch_oauth2", "uuid", ] @@ -4406,8 +4411,12 @@ version = "0.0.1" dependencies = [ "anyhow", "container_common", + "easyenv", "once_cell", "regex", + "serde", + "serde_derive", + "twitch_oauth2", ] [[package]] diff --git a/crates/lib/twitch_common/Cargo.toml b/crates/lib/twitch_common/Cargo.toml index e6dc378397..11fc427157 100644 --- a/crates/lib/twitch_common/Cargo.toml +++ b/crates/lib/twitch_common/Cargo.toml @@ -21,3 +21,13 @@ container_common = { path = "../container_common" } anyhow = "1.0.40" once_cell = "1.9.0" regex = "1.5.4" +easyenv = "0.0.6" + +# Twitch +# NB "unsupported" pubsub features may disappear from the Twitch API at any time. It's an unofficial API. +# twitch_api2 = { version = "0.6.0-rc.2", features = ["helix", "reqwest_client", "pubsub", "chrono", "unsupported"] } +twitch_oauth2 = "0.6.0-rc.2" + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" diff --git a/crates/lib/twitch_common/src/lib.rs b/crates/lib/twitch_common/src/lib.rs index 3d395e4749..5373c7dce1 100644 --- a/crates/lib/twitch_common/src/lib.rs +++ b/crates/lib/twitch_common/src/lib.rs @@ -1 +1,3 @@ -pub mod cheers; \ No newline at end of file +pub mod cheers; +pub mod oauth_token_builder; +pub mod twitch_secrets; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/twitch/oauth/oauth_token_builder.rs b/crates/lib/twitch_common/src/oauth_token_builder.rs similarity index 100% rename from crates/service/obs_gateway/src/twitch/oauth/oauth_token_builder.rs rename to crates/lib/twitch_common/src/oauth_token_builder.rs diff --git a/crates/service/obs_gateway/src/twitch/twitch_secrets.rs b/crates/lib/twitch_common/src/twitch_secrets.rs similarity index 77% rename from crates/service/obs_gateway/src/twitch/twitch_secrets.rs rename to crates/lib/twitch_common/src/twitch_secrets.rs index ac828ff86d..dad0d7eb20 100644 --- a/crates/service/obs_gateway/src/twitch/twitch_secrets.rs +++ b/crates/lib/twitch_common/src/twitch_secrets.rs @@ -1,5 +1,7 @@ use container_common::anyhow_result::AnyhowResult; +/// Standardized way to read Twitch secrets. +/// These are used in several different apps. #[derive(Clone, serde::Deserialize)] pub struct TwitchSecrets { pub app_client_id: String, @@ -23,11 +25,5 @@ impl TwitchSecrets { &easyenv::get_env_string_required("TWITCH_APP_CLIENT_SECRET")?, )) } - - pub fn from_file(filename: &str) -> AnyhowResult { - let contents = std::fs::read_to_string(filename)?; - let secrets = toml::from_str(&contents)?; - Ok(secrets) - } } diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index c78460db93..d8c7dc0f17 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -137,105 +137,6 @@ impl Actor for ObsGatewayWebSocket { } } -/* - -Message { - data: ChannelBitsEventsV2 { - topic: ChannelBitsEventsV2 { - +channel_id: 652567283 - }, - reply: BitsEvent { - data: BitsEventData { - -badge_entitlement: None, - +bits_used: 1, - +channel_id: "652567283", - -channel_name: "vocodes", - +chat_message: "test Cheer1", - -context: Cheer, - +is_anonymous: false, - -time: "2022-01-25T08:40:04.760906991Z", - +total_bits_used: 116, - +user_id: "650154491", - +user_name: "testytest512" - }, - -message_id: "7703927a-78a5-56d1-aa28-4e6c12aa79a1", - -version: "1.0", - +is_anonymous: false - } - } -} - -// These cheer emotes always combine with a number -// This list seems pretty comprehensive -https://github.com/nossebro/TwitchPubSubMirror/blob/master/TwitchPubSubMirror_StreamlabsSystem.py -CheerMotes = [ "Cheer", "DoodleCheer", "BibleThump", "cheerwhal", "Corgo", "uni", "ShowLove", "Party", "SeemsGood", "Pride", "Kappa", "FrankerZ", "HeyGuys", "DansGame", "EleGiggle", "TriHard", "Kreygasm", "4Head", "SwiftRage", "NotLikeThis", "FailFish", "VoHiYo", "PJSalt", "MrDestructoid", "bday", "RIPCheer", "Shamrock", "BitBoss", "Streamlabs", "Muxy", "HolidayCheer" ] - - -// Map "Reward Name" ("title") --> FakeYou voice id. - -Message { - data: ChannelPointsChannelV1 { - topic: ChannelPointsChannelV1 { - channel_id: 652567283 - }, - reply: RewardRedeemed { - timestamp: "2022-01-25T08:44:09.266674947Z", - redemption: Redemption { - channel_id: "652567283", - id: "e90823c7-934c-497c-ba2c-34c93dcf7163", - redeemed_at: "2022-01-25T08:44:09.266674947Z", - reward: Reward { - -background_color: "#BD0078", - -channel_id: "652567283", - -cooldown_expires_at: None, - -cost: 50, - default_image: Some(Image { - url_1x: "https://static-cdn.jtvnw.net/custom-reward-images/default-1.png", - url_2x: "https://static-cdn.jtvnw.net/custom-reward-images/default-2.png", - url_4x: "https://static-cdn.jtvnw.net/custom-reward-images/default-4.png" - }), - -global_cooldown: GlobalCooldown { - is_enabled: false, - global_cooldown_seconds: 0 - }, - id: "3e0eaf15-f454-482d-b48c-5be6ede61901", - image: None, - -is_enabled: true, - -is_in_stock: true, - -is_paused: false, - +is_sub_only: false, - is_user_input_required: true, - max_per_stream: MaxPerStream { - is_enabled: false, - max_per_stream: 0 - }, - max_per_user_per_stream: MaxPerUserPerStream { - is_enabled: false, - max_per_user_per_stream: 0 - }, - prompt: "This is a reward", - redemptions_redeemed_current_stream: None, - should_redemptions_skip_request_queue: false, - template_id: None, - title: "Reward #1", - updated_for_indicator_at: Some("2021-09-28T08:42:31.949564296Z") - }, - status: Unfulfilled, - user: User { - +id: "650154491", - +login: "testytest512", - display_name: "testytest512", - profile_image_url: None - }, - +user_input: Some("highlight reward thing"), - cursor: None - } - } - } -} -*/ - - impl StreamHandler> for ObsGatewayWebSocket { fn handle( &mut self, @@ -275,7 +176,6 @@ impl StreamHandler> for ObsGatewayWebSock // TODO: This should be done *BEFORE* PubSub self.write_obs_active().unwrap(); - let redis_key = RedisKeys::twitch_tts_job_queue(&self.twitch_user_id.get_str()); let values : Vec = redis.lpop((redis_key, 5)).unwrap(); // TODO: Error handling diff --git a/crates/service/obs_gateway/src/main.rs b/crates/service/obs_gateway/src/main.rs index 968b397429..0af7b216ff 100644 --- a/crates/service/obs_gateway/src/main.rs +++ b/crates/service/obs_gateway/src/main.rs @@ -12,7 +12,6 @@ #[macro_use] extern crate magic_crypt; #[macro_use] extern crate serde_derive; -pub mod endpoints; pub mod endpoints_ws; pub mod server_state; pub mod redis; @@ -28,12 +27,8 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; -use crate::endpoints::oauth_begin::oauth_begin_enroll; -use crate::endpoints::oauth_begin_redirect::oauth_begin_enroll_redirect; -use crate::endpoints::oauth_end::oauth_end_enroll_from_redirect; use crate::endpoints_ws::obs_gateway_websocket_handler::obs_gateway_websocket_handler; -use crate::server_state::{ObsGatewayServerState, EnvConfig, TwitchOauthSecrets, TwitchOauthTemp, BackendsConfig}; -use crate::twitch::twitch_secrets::TwitchSecrets; +use crate::server_state::{ObsGatewayServerState, EnvConfig, TwitchOauthSecrets, BackendsConfig}; use futures::Future; use futures::executor::ThreadPool; use http_server_common::cors::build_common_cors_config; @@ -52,6 +47,7 @@ use std::time::Duration; use tokio::runtime::{Handle, Runtime}; use twitch_api2::pubsub::Topic; use twitch_api2::pubsub; +use twitch_common::twitch_secrets::TwitchSecrets; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{AppAccessToken, Scope, TwitchToken, tokens::errors::AppAccessTokenError, ClientId, ClientSecret}; @@ -77,9 +73,7 @@ async fn main() -> AnyhowResult<()> { info!("Reading Twitch secrets..."); - let secrets = TwitchSecrets::from_env()?; - let client_id = ClientId::new(&secrets.app_client_id); - let client_secret = ClientSecret::new(&secrets.app_client_secret); + let twitch_secrets = TwitchSecrets::from_env()?; info!("Reading env vars and setting up utils..."); @@ -96,11 +90,6 @@ async fn main() -> AnyhowResult<()> { "TWITCH_OAUTH_REDIRECT_URL", "http://localhost:54321/twitch/oauth_redirect"); - // TODO: These are temporary. - let temp_oauth_user_id = easyenv::get_env_string_or_default("TEMP_TWITCH_OAUTH_USER_ID", ""); - let temp_oauth_access_token = easyenv::get_env_string_or_default("TEMP_TWITCH_OAUTH_ACCESS", ""); - let temp_oauth_refresh_token = easyenv::get_env_string_or_default("TEMP_TWITCH_OAUTH_REFRESH", ""); - let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", @@ -135,15 +124,10 @@ async fn main() -> AnyhowResult<()> { website_homepage_redirect, }, twitch_oauth_secrets: TwitchOauthSecrets { - client_id: secrets.app_client_id.clone(), - client_secret: secrets.app_client_secret.clone(), + client_id: twitch_secrets.app_client_id.clone(), + client_secret: twitch_secrets.app_client_secret.clone(), redirect_url: oauth_redirect_url, }, - twitch_oauth_temp: TwitchOauthTemp { - temp_oauth_user_id, - temp_oauth_access_token, - temp_oauth_refresh_token, - }, hostname: server_hostname, backends: BackendsConfig { mysql_pool: pool, @@ -183,31 +167,18 @@ pub async fn serve(server_state: ObsGatewayServerState) -> AnyhowResult<()> .service(web::resource("/") .route(web::get().to(get_root_index)) ) - // Twitch - .service(web::scope("/twitch") - .service(web::resource("/oauth_enroll") - .route(web::get().to(oauth_begin_enroll)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service(web::resource("/oauth_enroll_redirect") - .route(web::get().to(oauth_begin_enroll_redirect)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service(web::resource("/oauth_redirect") - .route(web::get().to(oauth_end_enroll_from_redirect)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - ) + // Twitch websocket .service(web::resource("/obs/{twitch_username}") .route(web::get().to(obs_gateway_websocket_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + // Local development debugging .service( actix_files::Files::new("/static", "static") .show_files_listing() .use_last_modified(true), ) - //.default_service( web::route().to(default_route_404)) + .default_service( web::route().to(default_route_404)) }) .bind(bind_address)? .workers(num_workers) diff --git a/crates/service/obs_gateway/src/server_state.rs b/crates/service/obs_gateway/src/server_state.rs index 79bdb94610..3fd23142c3 100644 --- a/crates/service/obs_gateway/src/server_state.rs +++ b/crates/service/obs_gateway/src/server_state.rs @@ -11,7 +11,6 @@ pub struct ObsGatewayServerState { pub hostname: String, pub twitch_oauth_secrets: TwitchOauthSecrets, - pub twitch_oauth_temp: TwitchOauthTemp, pub backends: BackendsConfig, } @@ -41,13 +40,3 @@ pub struct TwitchOauthSecrets { pub client_secret: String, pub redirect_url: String, } - -/// TODO: THIS IS JUST FOR FEATURE DEVELOPMENT. KILL THIS. -/// Being lazy here until I have a decent DB model and know what I want to build. -#[deprecated] -#[derive(Clone)] -pub struct TwitchOauthTemp { - pub temp_oauth_user_id: String, - pub temp_oauth_access_token: String, - pub temp_oauth_refresh_token: String, -} diff --git a/crates/service/obs_gateway/src/twitch/mod.rs b/crates/service/obs_gateway/src/twitch/mod.rs index 98cfdf49c5..0c6e6e82b5 100644 --- a/crates/service/obs_gateway/src/twitch/mod.rs +++ b/crates/service/obs_gateway/src/twitch/mod.rs @@ -1,6 +1,5 @@ pub mod constants; pub mod oauth; pub mod pubsub; -pub mod twitch_secrets; pub mod twitch_user_id; pub mod websocket_client; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/twitch/oauth/mod.rs b/crates/service/obs_gateway/src/twitch/oauth/mod.rs index 46b2d4c4f9..02d8f8a8c2 100644 --- a/crates/service/obs_gateway/src/twitch/oauth/mod.rs +++ b/crates/service/obs_gateway/src/twitch/oauth/mod.rs @@ -1,2 +1 @@ -pub mod oauth_token_builder; pub mod oauth_token_refresher; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/twitch_pubsub_job.rs b/crates/service/obs_gateway/src/twitch_pubsub_job.rs index cd4bf47274..5861bf9973 100644 --- a/crates/service/obs_gateway/src/twitch_pubsub_job.rs +++ b/crates/service/obs_gateway/src/twitch_pubsub_job.rs @@ -21,7 +21,6 @@ use config::shared_constants::{DEFAULT_RUST_LOG, DEFAULT_REDIS_DATABASE_1_CONNEC use container_common::anyhow_result::AnyhowResult; use crate::threads::listen_for_active_obs_sessions_thread::ListenForActiveObsSessionThread; use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; -use crate::twitch::twitch_secrets::TwitchSecrets; use crate::twitch::websocket_client::TwitchWebsocketClient; use futures::executor::{ThreadPool, ThreadPoolBuilder}; use futures::task::SpawnExt; @@ -35,6 +34,7 @@ use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use tokio::runtime::{Builder, Runtime}; +use twitch_common::twitch_secrets::TwitchSecrets; use twitch_oauth2::{ClientId, ClientSecret, RefreshToken, AccessToken}; #[tokio::main] diff --git a/crates/service/storyteller_web/Cargo.toml b/crates/service/storyteller_web/Cargo.toml index dbbe2bed29..21e30c7593 100644 --- a/crates/service/storyteller_web/Cargo.toml +++ b/crates/service/storyteller_web/Cargo.toml @@ -43,6 +43,7 @@ container_common = { path = "../../lib/container_common" } database_queries = { path = "../../lib/database_queries" } http_server_common = { path = "../../lib/http_server_common" } redis_common = { path = "../../lib/redis_common" } +twitch_common = { path = "../../lib/twitch_common" } # External anyhow = "1.0.40" @@ -63,6 +64,17 @@ regex = "1.5.4" time = "0.2.26" uuid = { version = "0.8.2", features = ["v4"] } +# TODO/TEMPORARY: +# Twitch +# NB: "unsupported" pubsub features may disappear from the Twitch API at any time. It's an unofficial API. +twitch_api2 = { version = "0.6.0-rc.2", features = ["helix", "reqwest_client", "pubsub", "chrono", "unsupported"] } +twitch_oauth2 = "0.6.0-rc.2" + +# HTTP +# TODO/NB: This is just for Twitch, but we might need it later. +reqwest = { version = "0.11.4" } +native-tls = "0.2.8" + # Runtime # NB: We switched to Tokio from `async-std` for broader ecosystem support. tokio = { version = "1.6.0", features = ["macros"] } diff --git a/crates/service/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/mod.rs index f7f14dfbc9..4be42e802b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/mod.rs @@ -4,5 +4,6 @@ pub mod leaderboard; pub mod misc; pub mod moderation; pub mod tts; +pub mod twitch; pub mod users; pub mod w2l; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/endpoints/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs similarity index 72% rename from crates/service/obs_gateway/src/endpoints/mod.rs rename to crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs index bb1fc37db2..34e1b0b58b 100644 --- a/crates/service/obs_gateway/src/endpoints/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs @@ -1,3 +1,3 @@ pub mod oauth_begin; pub mod oauth_begin_redirect; -pub mod oauth_end; \ No newline at end of file +pub mod oauth_end; diff --git a/crates/service/obs_gateway/src/endpoints/oauth_begin.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin.rs similarity index 92% rename from crates/service/obs_gateway/src/endpoints/oauth_begin.rs rename to crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin.rs index 9483acf26c..61f4d1d0cc 100644 --- a/crates/service/obs_gateway/src/endpoints/oauth_begin.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin.rs @@ -1,7 +1,6 @@ use actix_http::{StatusCode, header}; use actix_web::{HttpRequest, web, HttpResponse, ResponseError, HttpResponseBuilder}; -use crate::ObsGatewayServerState; -use crate::twitch::oauth::oauth_token_builder::get_oauth_token_builder; +use crate::server_state::ServerState; use http_server_common::response::response_error_helpers::to_simple_json_error; use http_server_common::response::response_success_helpers::simple_json_success; use http_server_common::response::to_json_success_response::to_json_success_response; @@ -10,6 +9,7 @@ use log::warn; use std::fmt; use std::sync::Arc; use twitch_api2::twitch_oauth2::ClientId; +use twitch_common::oauth_token_builder::get_oauth_token_builder; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{Scope, ClientSecret}; @@ -48,7 +48,7 @@ impl ResponseError for OauthBeginEnrollError { pub async fn oauth_begin_enroll( http_request: HttpRequest, - server_state: web::Data> + server_state: web::Data> ) -> Result { let redirect_url = diff --git a/crates/service/obs_gateway/src/endpoints/oauth_begin_redirect.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_redirect.rs similarity index 92% rename from crates/service/obs_gateway/src/endpoints/oauth_begin_redirect.rs rename to crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_redirect.rs index 65f59d7e9b..83facbd0d9 100644 --- a/crates/service/obs_gateway/src/endpoints/oauth_begin_redirect.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_redirect.rs @@ -1,7 +1,6 @@ use actix_http::{StatusCode, header}; use actix_web::{HttpRequest, web, HttpResponse, ResponseError, HttpResponseBuilder}; -use crate::ObsGatewayServerState; -use crate::twitch::oauth::oauth_token_builder::get_oauth_token_builder; +use crate::server_state::ServerState; use http_server_common::response::response_error_helpers::to_simple_json_error; use http_server_common::response::response_success_helpers::simple_json_success; use http_server_common::response::to_json_success_response::to_json_success_response; @@ -10,6 +9,7 @@ use log::warn; use std::fmt; use std::sync::Arc; use twitch_api2::twitch_oauth2::ClientId; +use twitch_common::oauth_token_builder::get_oauth_token_builder; use twitch_oauth2::tokens::UserTokenBuilder; use twitch_oauth2::{Scope, ClientSecret}; @@ -48,7 +48,7 @@ impl ResponseError for OauthBeginEnrollError { pub async fn oauth_begin_enroll_redirect( http_request: HttpRequest, - server_state: web::Data> + server_state: web::Data> ) -> Result { let redirect_url = diff --git a/crates/service/obs_gateway/src/endpoints/oauth_end.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs similarity index 91% rename from crates/service/obs_gateway/src/endpoints/oauth_end.rs rename to crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs index c5cd76782c..e34b50c30d 100644 --- a/crates/service/obs_gateway/src/endpoints/oauth_end.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs @@ -1,17 +1,17 @@ use actix_http::StatusCode; use actix_web::{HttpResponse, HttpRequest, web, ResponseError}; -use crate::server_state::ObsGatewayServerState; -use crate::twitch::oauth::oauth_token_builder::get_oauth_token_builder; +use crate::server_state::ServerState; use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; +use http_server_common::request::get_request_ip::get_request_ip; use http_server_common::response::response_error_helpers::to_simple_json_error; use log::error; use log::info; use log::warn; use std::fmt; use std::sync::Arc; +use twitch_common::oauth_token_builder::get_oauth_token_builder; use twitch_oauth2::tokens::BearerTokenType::UserToken; use twitch_oauth2::{CsrfToken, TwitchToken}; -use http_server_common::request::get_request_ip::get_request_ip; #[derive(Deserialize)] pub struct QueryParams { @@ -62,7 +62,7 @@ impl ResponseError for OauthEndEnrollFromRedirectError { pub async fn oauth_end_enroll_from_redirect( http_request: HttpRequest, query: web::Query, - server_state: web::Data> + server_state: web::Data> ) -> Result { if let Some(error) = query.error.as_deref() { @@ -134,11 +134,7 @@ pub async fn oauth_end_enroll_from_redirect( let expires_in = user_token.expires_in(); info!("User id: {:?}", user_id); - info!("Auth token: {:?}", auth_token); - info!("Refresh token: {:?}", refresh_token); info!("Twitch username: {:?}", twitch_username); - info!("Never expiring: {:?}", never_expiring); - info!("Expires in (don't store): {:?}", expires_in); // Should be ~4 hours let ip_address = get_request_ip(&http_request); @@ -153,7 +149,7 @@ pub async fn oauth_end_enroll_from_redirect( //.set_token_type() //.set_has_bits_read(true) ... - let result = insert_builder.insert(&server_state.backends.mysql_pool) + let result = insert_builder.insert(&server_state.mysql_pool) .await .map_err(|e| { warn!("Error saving to db: {:?}", e); diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 1bd7ed1f4f..946842f524 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -107,7 +107,7 @@ use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::routes::add_routes; -use crate::server_state::{ServerState, EnvConfig}; +use crate::server_state::{ServerState, EnvConfig, TwitchOauthSecrets}; use crate::threads::ip_banlist_set::IpBanlistSet; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; @@ -125,6 +125,7 @@ use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; use tokio::runtime::Runtime; +use twitch_common::twitch_secrets::TwitchSecrets; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -267,6 +268,12 @@ async fn main() -> AnyhowResult<()> { let ip_banlist2 = ip_banlist.clone(); let mysql_pool3 = pool.clone(); + let twitch_oauth_redirect_url = easyenv::get_env_string_or_default( + "TWITCH_OAUTH_REDIRECT_URL", + "http://localhost:54321/twitch/oauth_redirect"); + + let twitch_secrets = TwitchSecrets::from_env()?; + // Background jobs. info!("Spawning IP ban polling thread."); @@ -299,6 +306,11 @@ async fn main() -> AnyhowResult<()> { sort_key_crypto, ip_banlist, voice_list_cache, + twitch_oauth_secrets : TwitchOauthSecrets { + client_id: twitch_secrets.app_client_id, + client_secret: twitch_secrets.app_client_secret, + redirect_url: twitch_oauth_redirect_url, + } }; serve(server_state) diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index cc4c219583..c4a5d8c3ce 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -43,6 +43,9 @@ use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_u use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; +use crate::http_server::endpoints::twitch::oauth_begin::oauth_begin_enroll; +use crate::http_server::endpoints::twitch::oauth_begin_redirect::oauth_begin_enroll_redirect; +use crate::http_server::endpoints::twitch::oauth_end::oauth_end_enroll_from_redirect; use crate::http_server::endpoints::users::create_account::create_account_handler; use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; use crate::http_server::endpoints::users::get_profile::get_profile_handler; @@ -84,6 +87,7 @@ pub fn add_routes (app: App) -> App app = add_w2l_routes(app); /* /w2l */ app = add_category_routes(app); /* /category */ app = add_user_profile_routes(app); /* /user */ + app = add_twitch_oauth_routes(app); /* /twitch */ // TODO: MAYBE TEMPORARY // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== app.service( @@ -518,3 +522,34 @@ fn add_user_profile_routes (app: App) -> App ) ) } +// ==================== TWITCH ROUTES ==================== + +// TODO: Maybe move these into an "oauth-gateway" type http service. +// We'll want the domain to have reputation and not confuse people. +// It'll also be nice to accrue all oauth things here. +fn add_twitch_oauth_routes (app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service(web::scope("/twitch") + .service(web::resource("/oauth_enroll") + .route(web::get().to(oauth_begin_enroll)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/oauth_enroll_redirect") + .route(web::get().to(oauth_begin_enroll_redirect)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/oauth_redirect") + .route(web::get().to(oauth_end_enroll_from_redirect)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) +} diff --git a/crates/service/storyteller_web/src/server_state.rs b/crates/service/storyteller_web/src/server_state.rs index 5ed28360cd..455dbf6cb7 100644 --- a/crates/service/storyteller_web/src/server_state.rs +++ b/crates/service/storyteller_web/src/server_state.rs @@ -44,6 +44,8 @@ pub struct ServerState { /// In-memory caches with TTL-based eviction. Contains a list of all voices. pub voice_list_cache: SingleItemTtlCache>, + + pub twitch_oauth_secrets: TwitchOauthSecrets, } #[derive(Clone)] @@ -56,3 +58,11 @@ pub struct EnvConfig { pub cookie_http_only: bool, pub website_homepage_redirect: String, } + +/// Necessary to run the OAuth flow. +#[derive(Clone)] +pub struct TwitchOauthSecrets { + pub client_id: String, + pub client_secret: String, + pub redirect_url: String, +} From 9693bf7704dbc4f27eece6ab189f5893dfd5f593 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Feb 2022 02:21:01 -0500 Subject: [PATCH 0849/1042] add all oauth scopes to mysql (untested, kind of dumb) --- .../up.sql | 145 +++++++++++++++++- .../src/twitch_oauth/insert.rs | 51 +++--- .../twitch_common/src/oauth_token_builder.rs | 46 +++++- crates/service/obs_gateway/src/main.rs | 5 - .../service/obs_gateway/src/server_state.rs | 1 - .../src/http_server/endpoints/twitch/mod.rs | 2 +- .../{oauth_begin.rs => oauth_begin_json.rs} | 2 +- .../http_server/endpoints/twitch/oauth_end.rs | 10 +- crates/service/storyteller_web/src/routes.rs | 6 +- 9 files changed, 225 insertions(+), 43 deletions(-) rename crates/service/storyteller_web/src/http_server/endpoints/twitch/{oauth_begin.rs => oauth_begin_json.rs} (98%) diff --git a/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql b/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql index 0bf8aa6f48..6c07fbb901 100644 --- a/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql +++ b/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql @@ -55,30 +55,163 @@ CREATE TABLE twitch_oauth_tokens( -- subsequent time will bump by one. Mostly for debugging. refresh_count INT(10) UNSIGNED DEFAULT 0, - -- ========== OAUTH SCOPES ========== + -- ====================================================== + -- ==================== OAUTH SCOPES ==================== + -- ====================================================== + + -- TODO: Probably could have stored this in a bitfield. Ugh. + + -- "analytics:read:extensions" + -- "View analytics data for the Twitch Extensions owned by the authenticated account." + has_analytics_read_extensions BOOLEAN NOT NULL DEFAULT FALSE, + + -- scope: "analytics:read:games" + -- "View analytics data for the games owned by the authenticated account." + has_analytics_read_games BOOLEAN NOT NULL DEFAULT FALSE, -- "bits:read" -- "View Bits information for a channel." has_bits_read BOOLEAN NOT NULL DEFAULT FALSE, + -- "channel:edit:commercial" + -- "Run commercials on a channel." + has_channel_edit_commercial BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:manage:broadcast" + -- "Manage a channel's broadcast configuration, including updating channel configuration and managing stream markers and stream tags." + has_channel_manage_broadcast BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:manage:extensions" + -- "Manage a channel's Extension configuration, including activating Extensions." + has_channel_manage_extensions BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:manage:polls" + -- "Manage a channel's polls." + has_channel_manage_polls BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:manage:predictions" + -- "Manage of channel's Channel Points Predictions" + has_channel_manage_predictions BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:manage:redemptions" + -- "Manage Channel Points custom rewards and their redemptions on a channel." + has_channel_manage_redemptions BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:manage:schedule" + -- "Manage a channel's stream schedule." + has_channel_manage_schedule BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:manage:videos" + -- "Manage a channel's videos, including deleting videos." + has_channel_manage_videos BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:moderate" + -- "Perform moderation actions in a channel. The user requesting the scope must be a moderator in the channel." + has_channel_moderate BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:read:editors" + -- "View a list of users with the editor role for a channel." + has_channel_read_editors BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:read:goals" + -- "View Creator Goals for a channel." + has_channel_read_goals BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:read:hype_train" + -- "View Hype Train information for a channel." + has_channel_read_hype_train BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:read:polls" + -- "View a channel's polls." + has_channel_read_polls BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:read:predictions" + -- "View a channel's Channel Points Predictions." + has_channel_read_predictions BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:read:redemptions" + -- "View Channel Points custom rewards and their redemptions on a channel." + -- eg. PubSub subscribe to `channel-points-channel-v1.` (channel points spends) + has_channel_read_redemptions BOOLEAN NOT NULL DEFAULT FALSE, + + -- "channel:read:stream_key" + -- "View an authorized user's stream key." + has_channel_read_stream_key BOOLEAN NOT NULL DEFAULT FALSE, + -- "channel:read:subscriptions" -- "View a list of all subscribers to a channel and check if a user is subscribed to a channel." -- eg. Enumerate list of subscribers for a channel -- eg. PubSub subscribe to `channel-subscribe-events-v1.` (subscribe, resubscribe, gift) has_channel_read_subscriptions BOOLEAN NOT NULL DEFAULT FALSE, - -- "channel:read:redemptions" - -- "View Channel Points custom rewards and their redemptions on a channel." - -- eg. PubSub subscribe to `channel-points-channel-v1.` (channel points spends) - has_channel_read_redemptions BOOLEAN NOT NULL DEFAULT FALSE, + -- "channel_subscriptions" + -- "\\[DEPRECATED\\] Read all subscribers to your channel." + has_channel_subscriptions BOOLEAN NOT NULL DEFAULT FALSE, + + -- "chat:edit" + -- "Send live stream chat and rooms messages." + has_chat_edit BOOLEAN NOT NULL DEFAULT FALSE, + + -- "chat:read" + -- "View live stream chat and rooms messages." + has_chat_read BOOLEAN NOT NULL DEFAULT FALSE, + + -- "clips:edit" + -- "Manage Clips for a channel." + has_clips_edit BOOLEAN NOT NULL DEFAULT FALSE, + + -- "moderation:read" + -- "View a channel's moderation data including Moderators, Bans, Timeouts, and Automod settings." + has_moderation_read BOOLEAN NOT NULL DEFAULT FALSE, + + -- "moderator:manage:automod" + -- "Manage messages held for review by AutoMod in channels where you are a moderator." + has_moderator_manage_auto_mod BOOLEAN NOT NULL DEFAULT FALSE, + + -- "user:edit" + -- "Manage a user object." + has_user_edit BOOLEAN NOT NULL DEFAULT FALSE, + + -- "user:edit:broadcast" + -- "Edit your channel's broadcast configuration, including extension configuration. (This scope implies user:read:broadcast capability.)" + has_user_edit_broadcast BOOLEAN NOT NULL DEFAULT FALSE, + + -- "user:edit:follows" + -- "Edit a user's follows." + has_user_edit_follows BOOLEAN NOT NULL DEFAULT FALSE, + + -- "user:manage:blocked_users" + -- "Manage the block list of a user." + has_user_manage_blocked_users BOOLEAN NOT NULL DEFAULT FALSE, + + -- "user:read:blocked_users" + -- "View the block list of a user." + has_user_read_blocked_users BOOLEAN NOT NULL DEFAULT FALSE, + + -- "user:read:broadcast" + -- "View a user's broadcasting configuration, including Extension configurations." + has_user_read_broadcast BOOLEAN NOT NULL DEFAULT FALSE, + + -- "user:read:email" + -- "Read an authorized user's email address." + has_user_read_email BOOLEAN NOT NULL DEFAULT FALSE, -- "user:read:follows" -- "View the list of channels a user follows." -- eg. We can use this to see if a Twitch user follows us. has_user_read_follows BOOLEAN NOT NULL DEFAULT FALSE, + -- "user:read:subscriptions" + -- "View if an authorized user is subscribed to specific channels." + has_user_read_subscriptions BOOLEAN NOT NULL DEFAULT FALSE, - -- THERE ARE MORE SCOPES... PERHAPS WE SHOULD ADD THEM? + -- "whispers:edit" + -- "Send whisper messages." + has_whispers_edit BOOLEAN NOT NULL DEFAULT FALSE, + + -- "whispers:read" + -- "View your whisper messages." + has_whispers_read BOOLEAN NOT NULL DEFAULT FALSE, -- ========== SECURITY ========== diff --git a/crates/lib/database_queries/src/twitch_oauth/insert.rs b/crates/lib/database_queries/src/twitch_oauth/insert.rs index 68bd187b36..914060b522 100644 --- a/crates/lib/database_queries/src/twitch_oauth/insert.rs +++ b/crates/lib/database_queries/src/twitch_oauth/insert.rs @@ -38,10 +38,13 @@ pub struct TwitchOauthTokenInsertBuilder { /// Number of times we've refreshed the token refresh_count: u32, + // ===== OAuth Scopes ===== + has_bits_read: bool, - has_channel_read_subscriptions: bool, has_channel_read_redemptions: bool, - has_user_read_follows: bool, + has_channel_read_subscriptions: bool, + has_chat_edit: bool, + has_chat_read: bool, } impl TwitchOauthTokenInsertBuilder { @@ -58,7 +61,8 @@ impl TwitchOauthTokenInsertBuilder { has_bits_read: false, has_channel_read_subscriptions: false, has_channel_read_redemptions: false, - has_user_read_follows: false, + has_chat_edit: false, + has_chat_read: false, refresh_count: 0, } } @@ -93,23 +97,28 @@ impl TwitchOauthTokenInsertBuilder { self } - pub fn set_has_bits_read(mut self, has_bits_read: bool) -> Self { - self.has_bits_read = has_bits_read; + pub fn has_bits_read(mut self, value: bool) -> Self { + self.has_bits_read = value; self } - pub fn has_channel_read_subscriptions(mut self, channel_read_subscriptions: bool) -> Self { - self.has_channel_read_subscriptions = channel_read_subscriptions; + pub fn has_channel_read_redemptions(mut self, value: bool) -> Self { + self.has_channel_read_redemptions = value; self } - pub fn has_channel_read_redemptions(mut self, channel_read_redemptions: bool) -> Self { - self.has_channel_read_redemptions = channel_read_redemptions; + pub fn has_channel_read_subscriptions(mut self, value: bool) -> Self { + self.has_channel_read_subscriptions = value; self } - pub fn has_user_read_follows(mut self, has_user_read_follows: bool) -> Self { - self.has_user_read_follows = has_user_read_follows; + pub fn has_chat_edit(mut self, value: bool) -> Self { + self.has_chat_edit = value; + self + } + + pub fn has_chat_read(mut self, value: bool) -> Self { + self.has_chat_read = value; self } @@ -131,9 +140,10 @@ SET token_type = ?, expires_in_seconds = ?, has_bits_read = ?, - has_channel_read_subscriptions = ?, has_channel_read_redemptions= ?, - has_user_read_follows = ?, + has_channel_read_subscriptions = ?, + has_chat_edit = ?, + has_chat_read = ?, ip_address_creation = ?, expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND) "#, @@ -147,9 +157,10 @@ SET self.token_type.clone(), expires_in_seconds, self.has_bits_read.clone(), - self.has_channel_read_subscriptions.clone(), self.has_channel_read_redemptions.clone(), - self.has_user_read_follows.clone(), + self.has_channel_read_subscriptions.clone(), + self.has_chat_edit.clone(), + self.has_chat_read.clone(), self.maybe_ip_address_creation.clone(), expires_in_seconds, ) @@ -168,9 +179,10 @@ SET token_type = ?, expires_in_seconds = NULL, has_bits_read = ?, - has_channel_read_subscriptions = ?, has_channel_read_redemptions= ?, - has_user_read_follows = ?, + has_channel_read_subscriptions = ?, + has_chat_edit = ?, + has_chat_read = ?, ip_address_creation = ?, expires_at = NULL "#, @@ -183,9 +195,10 @@ SET self.refresh_count, self.token_type.clone(), self.has_bits_read.clone(), - self.has_channel_read_subscriptions.clone(), self.has_channel_read_redemptions.clone(), - self.has_user_read_follows.clone(), + self.has_channel_read_subscriptions.clone(), + self.has_chat_edit.clone(), + self.has_chat_read.clone(), self.maybe_ip_address_creation.clone(), ) }; diff --git a/crates/lib/twitch_common/src/oauth_token_builder.rs b/crates/lib/twitch_common/src/oauth_token_builder.rs index 0f135aed74..49588c7ed2 100644 --- a/crates/lib/twitch_common/src/oauth_token_builder.rs +++ b/crates/lib/twitch_common/src/oauth_token_builder.rs @@ -14,13 +14,53 @@ pub fn get_oauth_token_builder( let client_id = ClientId::new(client_id); let client_secret = ClientSecret::new(client_secret); + /* + - AnalyticsReadExtensions, scope: "analytics:read:extensions", doc: "View analytics data for the Twitch Extensions owned by the authenticated account."; + - AnalyticsReadGames, scope: "analytics:read:games", doc: "View analytics data for the games owned by the authenticated account."; + [YES] BitsRead, scope: "bits:read", doc: "View Bits information for a channel."; + - ChannelEditCommercial, scope: "channel:edit:commercial", doc: "Run commercials on a channel."; + - ChannelManageBroadcast, scope: "channel:manage:broadcast", doc: "Manage a channel’s broadcast configuration, including updating channel configuration and managing stream markers and stream tags."; + - ChannelManageExtensions, scope: "channel:manage:extensions", doc: "Manage a channel’s Extension configuration, including activating Extensions."; + (?) ChannelManagePolls, scope: "channel:manage:polls", doc: "Manage a channel’s polls."; + (?) ChannelManagePredictions, scope: "channel:manage:predictions", doc: "Manage of channel’s Channel Points Predictions"; + - ChannelManageRedemptions, scope: "channel:manage:redemptions", doc: "Manage Channel Points custom rewards and their redemptions on a channel."; + - ChannelManageSchedule, scope: "channel:manage:schedule", doc: "Manage a channel’s stream schedule."; + - ChannelManageVideos, scope: "channel:manage:videos", doc: "Manage a channel’s videos, including deleting videos."; + - ChannelModerate, scope: "channel:moderate", doc: "Perform moderation actions in a channel. The user requesting the scope must be a moderator in the channel."; + - ChannelReadEditors, scope: "channel:read:editors", doc: "View a list of users with the editor role for a channel."; + - ChannelReadGoals, scope: "channel:read:goals", doc: "View Creator Goals for a channel."; + - ChannelReadHypeTrain, scope: "channel:read:hype_train", doc: "View Hype Train information for a channel."; + (?) ChannelReadPolls, scope: "channel:read:polls", doc: "View a channel’s polls."; + (?) ChannelReadPredictions, scope: "channel:read:predictions", doc: "View a channel’s Channel Points Predictions."; + + ChannelReadRedemptions, scope: "channel:read:redemptions", doc: "View Channel Points custom rewards and their redemptions on a channel."; + - ChannelReadStreamKey, scope: "channel:read:stream_key", doc: "View an authorized user’s stream key."; + + ChannelReadSubscriptions, scope: "channel:read:subscriptions", doc: "View a list of all subscribers to a channel and check if a user is subscribed to a channel."; + - ChannelSubscriptions, scope: "channel_subscriptions", doc: "\\[DEPRECATED\\] Read all subscribers to your channel."; + [YES] ChatEdit, scope: "chat:edit", doc: "Send live stream chat and rooms messages."; + [YES] ChatRead, scope: "chat:read", doc: "View live stream chat and rooms messages."; + (?) ClipsEdit, scope: "clips:edit", doc: "Manage Clips for a channel."; + (?) ModerationRead, scope: "moderation:read", doc: "View a channel’s moderation data including Moderators, Bans, Timeouts, and Automod settings."; + (?) ModeratorManageAutoMod, scope: "moderator:manage:automod", doc: "Manage messages held for review by AutoMod in channels where you are a moderator."; + - UserEdit, scope: "user:edit", doc: "Manage a user object."; + - UserEditBroadcast, scope: "user:edit:broadcast", doc: "Edit your channel's broadcast configuration, including extension configuration. (This scope implies user:read:broadcast capability.)"; + - UserEditFollows, scope: "user:edit:follows", doc: "Edit a user’s follows."; + - UserManageBlockedUsers, scope: "user:manage:blocked_users", doc: "Manage the block list of a user."; + - UserReadBlockedUsers, scope: "user:read:blocked_users", doc: "View the block list of a user."; + - UserReadBroadcast, scope: "user:read:broadcast", doc: "View a user’s broadcasting configuration, including Extension configurations."; + - UserReadEmail, scope: "user:read:email", doc: "Read an authorized user’s email address."; + - UserReadFollows, scope: "user:read:follows", doc: "View the list of channels a user follows."; + (?) UserReadSubscriptions, scope: "user:read:subscriptions", doc: "View if an authorized user is subscribed to specific channels."; + - WhispersEdit, scope: "whispers:edit", doc: "Send whisper messages."; + - WhispersRead, scope: "whispers:read", doc: "View your whisper messages."; + */ + UserTokenBuilder::new(client_id, client_secret, redirect_url.clone()) .set_scopes(vec![ Scope::BitsRead, // Using bits to cheer (cost money, larger streams) Scope::ChannelReadRedemptions, // Channel points redeemed (free points, smaller streams) - Scope::ChannelReadSubscriptions, - Scope::ChatEdit, - Scope::ChatRead, + Scope::ChannelReadSubscriptions, // List subscribing (paying) users + Scope::ChatEdit, // IRC send + Scope::ChatRead, // IRC read ]) .force_verify(force_verify) } diff --git a/crates/service/obs_gateway/src/main.rs b/crates/service/obs_gateway/src/main.rs index 0af7b216ff..06fb86b935 100644 --- a/crates/service/obs_gateway/src/main.rs +++ b/crates/service/obs_gateway/src/main.rs @@ -86,10 +86,6 @@ async fn main() -> AnyhowResult<()> { let website_homepage_redirect = easyenv::get_env_string_or_default("WEBSITE_HOMEPAGE_REDIRECT", "https://vo.codes/"); - let oauth_redirect_url = easyenv::get_env_string_or_default( - "TWITCH_OAUTH_REDIRECT_URL", - "http://localhost:54321/twitch/oauth_redirect"); - let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", @@ -126,7 +122,6 @@ async fn main() -> AnyhowResult<()> { twitch_oauth_secrets: TwitchOauthSecrets { client_id: twitch_secrets.app_client_id.clone(), client_secret: twitch_secrets.app_client_secret.clone(), - redirect_url: oauth_redirect_url, }, hostname: server_hostname, backends: BackendsConfig { diff --git a/crates/service/obs_gateway/src/server_state.rs b/crates/service/obs_gateway/src/server_state.rs index 3fd23142c3..230ebdcbec 100644 --- a/crates/service/obs_gateway/src/server_state.rs +++ b/crates/service/obs_gateway/src/server_state.rs @@ -38,5 +38,4 @@ pub struct EnvConfig { pub struct TwitchOauthSecrets { pub client_id: String, pub client_secret: String, - pub redirect_url: String, } diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs index 34e1b0b58b..0b47d3ca95 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs @@ -1,3 +1,3 @@ -pub mod oauth_begin; +pub mod oauth_begin_json; pub mod oauth_begin_redirect; pub mod oauth_end; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_json.rs similarity index 98% rename from crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin.rs rename to crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_json.rs index 61f4d1d0cc..cb4eefaa69 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_json.rs @@ -46,7 +46,7 @@ impl ResponseError for OauthBeginEnrollError { } } -pub async fn oauth_begin_enroll( +pub async fn oauth_begin_enroll_json( http_request: HttpRequest, server_state: web::Data> ) -> Result { diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs index e34b50c30d..d6d5ac2f36 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs @@ -144,10 +144,12 @@ pub async fn oauth_end_enroll_from_redirect( TwitchOauthTokenInsertBuilder::new(&user_id, &twitch_username, &auth_token) .set_expires_in_seconds(Some(expires_seconds)) .set_refresh_token(refresh_token.as_deref()) - .set_ip_address_creation(Some(&ip_address)); - //.set_user_token() - //.set_token_type() - //.set_has_bits_read(true) ... + .set_ip_address_creation(Some(&ip_address)) + .has_bits_read(true) + .has_channel_read_redemptions(true) + .has_channel_read_subscriptions(true) + .has_chat_edit(true) + .has_chat_read(true); let result = insert_builder.insert(&server_state.mysql_pool) .await diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index c4a5d8c3ce..30eafb239c 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -43,7 +43,7 @@ use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_u use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; -use crate::http_server::endpoints::twitch::oauth_begin::oauth_begin_enroll; +use crate::http_server::endpoints::twitch::oauth_begin_json::oauth_begin_enroll_json; use crate::http_server::endpoints::twitch::oauth_begin_redirect::oauth_begin_enroll_redirect; use crate::http_server::endpoints::twitch::oauth_end::oauth_end_enroll_from_redirect; use crate::http_server::endpoints::users::create_account::create_account_handler; @@ -539,8 +539,8 @@ fn add_twitch_oauth_routes (app: App) -> App >, { app.service(web::scope("/twitch") - .service(web::resource("/oauth_enroll") - .route(web::get().to(oauth_begin_enroll)) + .service(web::resource("/oauth_enroll_json") + .route(web::get().to(oauth_begin_enroll_json)) .route(web::head().to(|| HttpResponse::Ok())) ) .service(web::resource("/oauth_enroll_redirect") From 05db14bfbda824b56b986316ab152d5196122767 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Feb 2022 03:13:30 -0500 Subject: [PATCH 0850/1042] fixup query errors, add second redirect url --- .../database_queries/src/twitch_oauth/find.rs | 18 +++++++++------- .../twitch_pubsub_user_subscriber_thread.rs | 7 ++++--- .../endpoints/twitch/oauth_begin_json.rs | 6 +++--- .../endpoints/twitch/oauth_begin_redirect.rs | 6 +++--- .../http_server/endpoints/twitch/oauth_end.rs | 16 ++++++++------ crates/service/storyteller_web/src/main.rs | 21 ++++++++++++------- crates/service/storyteller_web/src/routes.rs | 2 +- .../storyteller_web/src/server_state.rs | 11 ++++++++-- 8 files changed, 55 insertions(+), 32 deletions(-) diff --git a/crates/lib/database_queries/src/twitch_oauth/find.rs b/crates/lib/database_queries/src/twitch_oauth/find.rs index 1f544a8243..af74a4186d 100644 --- a/crates/lib/database_queries/src/twitch_oauth/find.rs +++ b/crates/lib/database_queries/src/twitch_oauth/find.rs @@ -27,9 +27,10 @@ pub struct TwitchOauthTokenRecord { pub expires_in_seconds: Option, pub refresh_count: u32, pub has_bits_read: bool, - pub has_channel_read_subscriptions: bool, pub has_channel_read_redemptions: bool, - pub has_user_read_follows: bool, + pub has_channel_read_subscriptions: bool, + pub has_chat_edit: bool, + pub has_chat_read: bool, pub ip_address_creation: Option, /// Potentially when the token is expected to expire. @@ -102,9 +103,10 @@ impl TwitchOauthTokenFinder { expires_in_seconds: record.expires_in_seconds.clone(), refresh_count: record.refresh_count, has_bits_read: i8_to_bool(record.has_bits_read), - has_channel_read_subscriptions: i8_to_bool(record.has_channel_read_subscriptions), has_channel_read_redemptions: i8_to_bool(record.has_channel_read_redemptions), - has_user_read_follows: i8_to_bool(record.has_user_read_follows), + has_channel_read_subscriptions: i8_to_bool(record.has_channel_read_subscriptions), + has_chat_edit: i8_to_bool(record.has_chat_edit), + has_chat_read: i8_to_bool(record.has_chat_read), ip_address_creation: record.ip_address_creation.clone(), expires_at: record.expires_at.clone(), user_deleted_at: record.user_deleted_at.clone(), @@ -171,9 +173,10 @@ SELECT twitch_oauth_tokens.expires_in_seconds, twitch_oauth_tokens.refresh_count, twitch_oauth_tokens.has_bits_read, - twitch_oauth_tokens.has_channel_read_subscriptions, twitch_oauth_tokens.has_channel_read_redemptions, - twitch_oauth_tokens.has_user_read_follows, + twitch_oauth_tokens.has_channel_read_subscriptions, + twitch_oauth_tokens.has_chat_edit, + twitch_oauth_tokens.has_chat_read, twitch_oauth_tokens.ip_address_creation, twitch_oauth_tokens.created_at, twitch_oauth_tokens.updated_at, @@ -263,7 +266,8 @@ pub struct TwitchOauthTokenRecordInternal { pub has_bits_read: i8, pub has_channel_read_subscriptions: i8, pub has_channel_read_redemptions: i8, - pub has_user_read_follows: i8, + pub has_chat_edit: i8, + pub has_chat_read: i8, pub ip_address_creation: Option, /// Potentially when the token is expected to expire. diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 8f684f4d5f..94d0c184e2 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -525,10 +525,11 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { .set_refresh_count(self.twitch_oauth_token_record.refresh_count.saturating_add(1)) // NB: We don't get these back from the refresh, but it seems like they would stay the same. .set_token_type(self.twitch_oauth_token_record.token_type.as_deref()) - .set_has_bits_read(self.twitch_oauth_token_record.has_bits_read) - .has_channel_read_subscriptions(self.twitch_oauth_token_record.has_channel_read_subscriptions) + .has_bits_read(self.twitch_oauth_token_record.has_bits_read) .has_channel_read_redemptions(self.twitch_oauth_token_record.has_channel_read_redemptions) - .has_user_read_follows(self.twitch_oauth_token_record.has_user_read_follows); + .has_channel_read_subscriptions(self.twitch_oauth_token_record.has_channel_read_subscriptions) + .has_chat_edit(self.twitch_oauth_token_record.has_chat_edit) + .has_chat_read(self.twitch_oauth_token_record.has_chat_read); query_builder.insert(&self.mysql_pool).await?; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_json.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_json.rs index cb4eefaa69..5c1ef6cd82 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_json.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_json.rs @@ -52,15 +52,15 @@ pub async fn oauth_begin_enroll_json( ) -> Result { let redirect_url = - twitch_oauth2::url::Url::parse(&server_state.twitch_oauth_secrets.redirect_url) + twitch_oauth2::url::Url::parse(&server_state.twitch_oauth.redirect_landing_url) .map_err(|e| { warn!("Error parsing url: {:?}", e); OauthBeginEnrollError::ServerError })?; let mut builder = get_oauth_token_builder( - &server_state.twitch_oauth_secrets.client_id, - &server_state.twitch_oauth_secrets.client_secret, + &server_state.twitch_oauth.secrets.client_id, + &server_state.twitch_oauth.secrets.client_secret, &redirect_url, true); diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_redirect.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_redirect.rs index 83facbd0d9..a2e39bea79 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_redirect.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_redirect.rs @@ -52,15 +52,15 @@ pub async fn oauth_begin_enroll_redirect( ) -> Result { let redirect_url = - twitch_oauth2::url::Url::parse(&server_state.twitch_oauth_secrets.redirect_url) + twitch_oauth2::url::Url::parse(&server_state.twitch_oauth.redirect_landing_url) .map_err(|e| { warn!("Error parsing url: {:?}", e); OauthBeginEnrollError::ServerError })?; let mut builder = get_oauth_token_builder( - &server_state.twitch_oauth_secrets.client_id, - &server_state.twitch_oauth_secrets.client_secret, + &server_state.twitch_oauth.secrets.client_id, + &server_state.twitch_oauth.secrets.client_secret, &redirect_url, true); diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs index d6d5ac2f36..4b111e65bd 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs @@ -1,4 +1,4 @@ -use actix_http::StatusCode; +use actix_http::{StatusCode, header}; use actix_web::{HttpResponse, HttpRequest, web, ResponseError}; use crate::server_state::ServerState; use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; @@ -86,15 +86,15 @@ pub async fn oauth_end_enroll_from_redirect( let csrf_token = CsrfToken::new(&state); let redirect_url = - twitch_oauth2::url::Url::parse(&server_state.twitch_oauth_secrets.redirect_url) + twitch_oauth2::url::Url::parse(&server_state.twitch_oauth.redirect_landing_url) .map_err(|e| { warn!("Error parsing url: {:?}", e); OauthEndEnrollFromRedirectError::ServerError })?; let mut builder = get_oauth_token_builder( - &server_state.twitch_oauth_secrets.client_id, - &server_state.twitch_oauth_secrets.client_secret, + &server_state.twitch_oauth.secrets.client_id, + &server_state.twitch_oauth.secrets.client_secret, &redirect_url, true); @@ -158,6 +158,10 @@ pub async fn oauth_end_enroll_from_redirect( OauthEndEnrollFromRedirectError::ServerError })?; - Ok(HttpResponse::Ok() - .body("TODO")) + Ok(HttpResponse::Found() + .append_header(( + header::LOCATION, + server_state.twitch_oauth.redirect_landing_finished_url.as_str() + )) + .finish()) } diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 946842f524..ce64d83514 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -107,7 +107,7 @@ use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::routes::add_routes; -use crate::server_state::{ServerState, EnvConfig, TwitchOauthSecrets}; +use crate::server_state::{ServerState, EnvConfig, TwitchOauthSecrets, TwitchOauth}; use crate::threads::ip_banlist_set::IpBanlistSet; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; @@ -268,10 +268,14 @@ async fn main() -> AnyhowResult<()> { let ip_banlist2 = ip_banlist.clone(); let mysql_pool3 = pool.clone(); - let twitch_oauth_redirect_url = easyenv::get_env_string_or_default( - "TWITCH_OAUTH_REDIRECT_URL", + let twitch_oauth_redirect_landing_url = easyenv::get_env_string_or_default( + "TWITCH_OAUTH_REDIRECT_LANDING_URL", "http://localhost:54321/twitch/oauth_redirect"); + let twitch_oauth_redirect_landing_finished_url = easyenv::get_env_string_or_default( + "TWITCH_OAUTH_REDIRECT_LANDING_FINISHED_URL", + "http://localhost:54321/"); + let twitch_secrets = TwitchSecrets::from_env()?; // Background jobs. @@ -306,10 +310,13 @@ async fn main() -> AnyhowResult<()> { sort_key_crypto, ip_banlist, voice_list_cache, - twitch_oauth_secrets : TwitchOauthSecrets { - client_id: twitch_secrets.app_client_id, - client_secret: twitch_secrets.app_client_secret, - redirect_url: twitch_oauth_redirect_url, + twitch_oauth: TwitchOauth { + secrets: TwitchOauthSecrets { + client_id: twitch_secrets.app_client_id, + client_secret: twitch_secrets.app_client_secret, + }, + redirect_landing_url: twitch_oauth_redirect_landing_url, + redirect_landing_finished_url: twitch_oauth_redirect_landing_finished_url, } }; diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index 30eafb239c..51aea12e25 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -547,7 +547,7 @@ fn add_twitch_oauth_routes (app: App) -> App .route(web::get().to(oauth_begin_enroll_redirect)) .route(web::head().to(|| HttpResponse::Ok())) ) - .service(web::resource("/oauth_redirect") + .service(web::resource("/oauth_landing") .route(web::get().to(oauth_end_enroll_from_redirect)) .route(web::head().to(|| HttpResponse::Ok())) ) diff --git a/crates/service/storyteller_web/src/server_state.rs b/crates/service/storyteller_web/src/server_state.rs index 455dbf6cb7..2dde5dcb9a 100644 --- a/crates/service/storyteller_web/src/server_state.rs +++ b/crates/service/storyteller_web/src/server_state.rs @@ -45,7 +45,7 @@ pub struct ServerState { /// In-memory caches with TTL-based eviction. Contains a list of all voices. pub voice_list_cache: SingleItemTtlCache>, - pub twitch_oauth_secrets: TwitchOauthSecrets, + pub twitch_oauth: TwitchOauth, } #[derive(Clone)] @@ -59,10 +59,17 @@ pub struct EnvConfig { pub website_homepage_redirect: String, } +/// Necessary to run the OAuth flow. +#[derive(Clone)] +pub struct TwitchOauth { + pub secrets: TwitchOauthSecrets, + pub redirect_landing_url: String, + pub redirect_landing_finished_url: String, +} + /// Necessary to run the OAuth flow. #[derive(Clone)] pub struct TwitchOauthSecrets { pub client_id: String, pub client_secret: String, - pub redirect_url: String, } From e5cd8164c34dc4987c38ddfd5c587e65fdc41851 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Feb 2022 03:24:10 -0500 Subject: [PATCH 0851/1042] fix URLs --- crates/service/storyteller_web/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index ce64d83514..0fa707599d 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -270,11 +270,11 @@ async fn main() -> AnyhowResult<()> { let twitch_oauth_redirect_landing_url = easyenv::get_env_string_or_default( "TWITCH_OAUTH_REDIRECT_LANDING_URL", - "http://localhost:54321/twitch/oauth_redirect"); + "http://localhost:12345/twitch/oauth_landing"); let twitch_oauth_redirect_landing_finished_url = easyenv::get_env_string_or_default( "TWITCH_OAUTH_REDIRECT_LANDING_FINISHED_URL", - "http://localhost:54321/"); + "http://localhost:12345/"); let twitch_secrets = TwitchSecrets::from_env()?; From 89ee7f6606412b54a0f570d3629e5f780f8961e8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Feb 2022 03:35:04 -0500 Subject: [PATCH 0852/1042] cleanup --- .../obs_gateway/src/endpoints_ws/mod.rs | 3 +- .../obs_gateway_websocket_handler.rs | 1 - .../src/endpoints_ws/obs_twitch_thread.rs | 152 ------------------ .../obs_gateway/src/twitch_pubsub_job.rs | 28 +--- 4 files changed, 4 insertions(+), 180 deletions(-) delete mode 100644 crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs diff --git a/crates/service/obs_gateway/src/endpoints_ws/mod.rs b/crates/service/obs_gateway/src/endpoints_ws/mod.rs index 78235255e4..fa4404b876 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/mod.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/mod.rs @@ -1,2 +1 @@ -pub mod obs_gateway_websocket_handler; -pub mod obs_twitch_thread; \ No newline at end of file +pub mod obs_gateway_websocket_handler; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index d8c7dc0f17..a1869463d1 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -5,7 +5,6 @@ use actix_web::{middleware, web, App, Error, HttpRequest, HttpResponse, HttpServ use actix_web_actors::ws; use container_common::anyhow_result::AnyhowResult; use container_common::token::random_crockford_token::random_crockford_token; -use crate::endpoints_ws::obs_twitch_thread::ObsTwitchThread; use crate::redis::constants::OBS_ACTIVE_TTL_SECONDS; use crate::redis::obs_active_payload::ObsActivePayload; use crate::server_state::ObsGatewayServerState; diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs deleted file mode 100644 index 140b2ca7ae..0000000000 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_twitch_thread.rs +++ /dev/null @@ -1,152 +0,0 @@ -use crate::twitch::twitch_user_id::TwitchUserId; -use crate::twitch::websocket_client::TwitchWebsocketClient; -use futures::lock::Mutex; -use log::error; -use log::info; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, RwLock, PoisonError, RwLockWriteGuard}; -use std::thread::sleep; -use std::time::Instant; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use tokio::runtime::Handle; - -// TODO: Let's make this an Arc>> -// so that multiple browser sessions for the same user can -// share the same single backing instance. -// . -// however, this causes events (TTS) to get flushed to just -// a single browser instance. -// . -// maybe just kill other sessions ? browser+uuid -> LRU. - -pub struct ObsTwitchThread { - // NB: This is a `futures` Mutex, which is much more compatible with async code - // than the blocking std::sync mutexes. - inner_data: Arc>, -} - -struct InnerData { - twitch_user_id: TwitchUserId, - is_connected: bool, - twitch_client: TwitchWebsocketClient, - last_ping: SystemTime, -} - -impl ObsTwitchThread { - pub fn new( - twitch_user_id: TwitchUserId, - twitch_client: TwitchWebsocketClient - ) -> Self { - let now = Instant::now(); - Self { - inner_data: Arc::new(Mutex::new(InnerData { - twitch_user_id, - is_connected: false, - twitch_client, - last_ping: UNIX_EPOCH, - })) - } - } - - pub async fn run_until_exit(&self) { - loop { - //info!("TwitchThread::run_until_exit()"); - //let is_connected = self.is_connected.load(Ordering::Relaxed); - - //if !is_connected { - // info!("Connect to Twitch"); - // self.connect().await; - //} - - info!("run_until_exit()..."); - self.maybe_send_ping().await; - - sleep(Duration::from_millis(1000)); - } - } - - pub async fn connect(&self) { - //let mut write = self.twitch_client.write().unwrap(); - //let _r = write.connect().await.unwrap(); - //self.is_connected.store(true, Ordering::Relaxed); - } - - pub async fn maybe_send_ping(&self) { - let inner_clone = self.inner_data.clone(); - { - info!("maybe_send_ping() lock."); - let mut inner_data = inner_clone.lock().await; - let now = SystemTime::now(); - let duration = now.duration_since(inner_data.last_ping).unwrap(); - - let delta = Duration::from_secs(60); - if duration.lt(&delta) { - return; - } - - info!("Sending ping..."); - inner_data.twitch_client.send_ping().await.unwrap(); - info!("Sent ping."); - - inner_data.last_ping = now; - } - } - - pub fn keep_alive_thread(&self) { - // TODO(bt, 2022-01-03): builds are failing, not sure when I worked on this last - //let now = Instant::now(); - //let lock = self.last_requested - // .clone() - // .get_mut(); - ////*lock = Some(now); - //match lock { - // Ok(l) => *l = Some(now), - // Err(_) => {} - //} - } - - // TODO: Investigate use of a thread pool instead. - pub fn start_thread(&self) { -// let mut last_requested_lock = self.last_requested.clone(); -// -// let handle = Handle::current(); -// -// handle.spawn(async move { -// Self::thread_main_loop(last_requested_lock); -// }); - } - - fn thread_main_loop( - mut last_requested: Arc>, - ) { - // TODO(bt, 2022-01-03): builds are failing, not sure when I worked on this last - //loop { - // let c = last_requested.get_mut(); - - // match c { - // Err(e) => { - // error!("Mutex failure: {:?}, ending job.", e); - // break; - // } - // Ok(t) => { - // let now = Instant::now(); - // let delta : Duration = now.duration_since(t.clone()); - - // if delta.gt(&Duration::from_secs(60)) { - // info!("Timeout elapsed, ending job."); - // break; - // } - // } - // } - - // Self::thread_single_loop(); - - // sleep(Duration::from_millis(1000)); - //} - } - - fn thread_single_loop() { - info!("thread iteration") - } -} - diff --git a/crates/service/obs_gateway/src/twitch_pubsub_job.rs b/crates/service/obs_gateway/src/twitch_pubsub_job.rs index 5861bf9973..5cf9b99768 100644 --- a/crates/service/obs_gateway/src/twitch_pubsub_job.rs +++ b/crates/service/obs_gateway/src/twitch_pubsub_job.rs @@ -97,27 +97,19 @@ pub async fn main() -> AnyhowResult<()> { info!("Creating thread pool..."); - // "The thread pool multiplexes any number of tasks onto a fixed number - // of worker threads." - let thread_pool = ThreadPoolBuilder::new() - .pool_size(8) - .name_prefix("twitch-pubsub-") - .create()?; - // https://docs.rs/tokio/latest/tokio/runtime/struct.Builder.html let runtime = Builder::new_multi_thread() - .worker_threads(4) + .worker_threads(2) .thread_name("twitch-pubsub-") .thread_stack_size(3 * 1024 * 1024) .enable_all() .build()?; let runtime = Arc::new(runtime); - //let runtime_2 = runtime.clone(); // https://docs.rs/tokio/latest/tokio/runtime/struct.Builder.html let runtime_2 = Builder::new_multi_thread() - .worker_threads(4) + .worker_threads(8) .thread_name("twitch-pubsub-") .thread_stack_size(3 * 1024 * 1024) .enable_all() @@ -139,20 +131,6 @@ pub async fn main() -> AnyhowResult<()> { runtime.spawn(thread.start_thread()); loop { - sleep(Duration::from_millis(10_000)); + sleep(Duration::from_millis(60_000)); } } - -pub async fn watch_user_thread(user_id: u32) { - let mut client = TwitchWebsocketClient::new().unwrap(); - - info!("Connecting to Twitch PubSub... {}", user_id); - client.connect().await.unwrap(); - - loop { - info!("Sending ping... {}", user_id); - client.send_ping().await.unwrap(); - sleep(Duration::from_millis(10_000)); - } -} - From 7cbede9d3d823bd1716cd6a4df2bc6a710b83e01 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Feb 2022 03:36:22 -0500 Subject: [PATCH 0853/1042] re-codegen --- sqlx-data.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 5701613287..6456456c94 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1,15 +1,5 @@ { "db": "MySQL", - "3a4a4edd2b1a1957c5eb42ed426ef4c11468549b9b8c3d62f1d59c07f4495c0e": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_subscriptions = ?,\n has_channel_read_redemptions= ?,\n has_user_read_follows = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 13 - }, - "nullable": [] - } - }, "591bda610a0fe952cafc5d97808757da8652133082e94ad15ab063db9d4e2f44": { "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", "describe": { @@ -40,12 +30,22 @@ "nullable": [] } }, - "cad1a53adb21efd78fa3a6f3c8c0f4d5f55164c4f397b50426bf987ea68af3d0": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_subscriptions = ?,\n has_channel_read_redemptions= ?,\n has_user_read_follows = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", + "620465c06c1dbdeb6740d62b9240468bb35b16a2b5ce3322f2415e552bd644b2": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", "describe": { "columns": [], "parameters": { - "Right": 15 + "Right": 16 + }, + "nullable": [] + } + }, + "f661bddeb979daee9068d4757fbb88295c42f16e85f70ba9a44ec70bd0c9269c": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 14 }, "nullable": [] } From 6c04d35ef196ad278fe1ba546313b2fb1e8a2485 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Feb 2022 04:51:12 -0500 Subject: [PATCH 0854/1042] add oauth grouping token last minute --- .../up.sql | 6 +++++ crates/lib/database_queries/src/tokens.rs | 4 +++ .../database_queries/src/twitch_oauth/find.rs | 18 +++++++++---- .../src/twitch_oauth/insert.rs | 15 ++++++++++- .../twitch_pubsub_user_subscriber_thread.rs | 3 ++- .../http_server/endpoints/twitch/oauth_end.rs | 9 ++++++- sqlx-data.json | 26 +++++++++---------- 7 files changed, 60 insertions(+), 21 deletions(-) diff --git a/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql b/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql index 6c07fbb901..bcaba96033 100644 --- a/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql +++ b/_migrations/2022-01-11-042235_twitch_oauth_tokens/up.sql @@ -11,6 +11,11 @@ CREATE TABLE twitch_oauth_tokens( -- Not used for anything except replication. id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + -- Minted on first request and copied on each refresh. + -- This allows us to group access tokens all the way back up the + -- refresh chain. + oauth_refresh_grouping_token VARCHAR(32) NOT NULL, + -- ========== STORYTELLER/FAKEYOU USER ========== -- Foreign key to user (Storyteller/FakeYou user) @@ -243,6 +248,7 @@ CREATE TABLE twitch_oauth_tokens( -- INDICES -- PRIMARY KEY (id), + KEY index_oauth_refresh_grouping_token (oauth_refresh_grouping_token), KEY fk_maybe_user_token (maybe_user_token), KEY index_twitch_user_id (twitch_user_id), -- KEY index_twitch_username (twitch_username), diff --git a/crates/lib/database_queries/src/tokens.rs b/crates/lib/database_queries/src/tokens.rs index 90e80afd26..1144344c89 100644 --- a/crates/lib/database_queries/src/tokens.rs +++ b/crates/lib/database_queries/src/tokens.rs @@ -58,4 +58,8 @@ impl Tokens { pub fn new_category() -> AnyhowResult { random_prefix_crockford_token("CAT:", 15, false) } + + pub fn new_twitch_oauth_grouping_token() -> AnyhowResult { + random_prefix_crockford_token("OG:", 32, false) + } } diff --git a/crates/lib/database_queries/src/twitch_oauth/find.rs b/crates/lib/database_queries/src/twitch_oauth/find.rs index af74a4186d..debe662551 100644 --- a/crates/lib/database_queries/src/twitch_oauth/find.rs +++ b/crates/lib/database_queries/src/twitch_oauth/find.rs @@ -10,6 +10,11 @@ use sqlx::MySqlPool; #[derive(Serialize, Clone)] pub struct TwitchOauthTokenRecord { + /// Our internal token metadata / bookkeeping + pub oauth_refresh_grouping_token: String, + pub refresh_count: u32, + pub ip_address_creation: Option, + /// NB: Vocodes/FakeYou/Storyteller user pub maybe_user_token: Option, pub maybe_user_display_name: Option, @@ -20,18 +25,16 @@ pub struct TwitchOauthTokenRecord { pub twitch_username: String, pub twitch_username_lowercase: String, - /// Token details + /// Twitch token details pub access_token: String, pub maybe_refresh_token: Option, pub token_type: Option, pub expires_in_seconds: Option, - pub refresh_count: u32, pub has_bits_read: bool, pub has_channel_read_redemptions: bool, pub has_channel_read_subscriptions: bool, pub has_chat_edit: bool, pub has_chat_read: bool, - pub ip_address_creation: Option, /// Potentially when the token is expected to expire. /// Do not eagerly refresh. Lazily renew. @@ -91,6 +94,7 @@ impl TwitchOauthTokenFinder { Ok(self.perform_query_internal(mysql_pool).await? .map(|record| { TwitchOauthTokenRecord { + oauth_refresh_grouping_token: record.oauth_refresh_grouping_token.clone(), maybe_user_token: record.maybe_user_token.clone(), maybe_user_display_name: record.maybe_user_display_name.clone(), maybe_user_gravatar_hash: record.maybe_user_gravatar_hash.clone(), @@ -158,6 +162,7 @@ impl TwitchOauthTokenFinder { // TODO/NB: Unfortunately SQLx can't statically typecheck this query let mut query = r#" SELECT + twitch_oauth_tokens.oauth_refresh_grouping_token, users.username as maybe_username, users.display_name as maybe_user_display_name, users.email_gravatar_hash as maybe_user_gravatar_hash, @@ -247,6 +252,11 @@ LEFT OUTER JOIN users #[derive(sqlx::FromRow)] pub struct TwitchOauthTokenRecordInternal { + /// Our internal token metadata / bookkeeping + pub oauth_refresh_grouping_token: String, + pub refresh_count: u32, + pub ip_address_creation: Option, + /// NB: Vocodes/FakeYou/Storyteller user pub maybe_user_token: Option, pub maybe_user_display_name: Option, @@ -262,13 +272,11 @@ pub struct TwitchOauthTokenRecordInternal { pub maybe_refresh_token: Option, pub token_type: Option, pub expires_in_seconds: Option, - pub refresh_count: u32, pub has_bits_read: i8, pub has_channel_read_subscriptions: i8, pub has_channel_read_redemptions: i8, pub has_chat_edit: i8, pub has_chat_read: i8, - pub ip_address_creation: Option, /// Potentially when the token is expected to expire. /// Do not eagerly refresh. Lazily renew. diff --git a/crates/lib/database_queries/src/twitch_oauth/insert.rs b/crates/lib/database_queries/src/twitch_oauth/insert.rs index 914060b522..d7f6cf1755 100644 --- a/crates/lib/database_queries/src/twitch_oauth/insert.rs +++ b/crates/lib/database_queries/src/twitch_oauth/insert.rs @@ -18,6 +18,9 @@ pub struct TwitchOauthTokenInsertBuilder { /// The secret we use access_token: String, + /// Internal mechanism we use to group tokens across refresh events + oauth_grouping_token: String, + // ===== Optional / Default Fields ===== /// Other half of the access token, for renewal @@ -48,11 +51,17 @@ pub struct TwitchOauthTokenInsertBuilder { } impl TwitchOauthTokenInsertBuilder { - pub fn new(twitch_user_id: &str, twitch_username: &str, access_token: &str) -> Self { + pub fn new( + twitch_user_id: &str, + twitch_username: &str, + access_token: &str, + oauth_grouping_token: &str + ) -> Self { Self { twitch_user_id: twitch_user_id.to_string(), twitch_username: twitch_username.to_string(), access_token: access_token.to_string(), + oauth_grouping_token: oauth_grouping_token.to_string(), maybe_refresh_token: None, maybe_user_token: None, maybe_ip_address_creation: None, @@ -130,6 +139,7 @@ impl TwitchOauthTokenInsertBuilder { r#" INSERT INTO twitch_oauth_tokens SET + oauth_refresh_grouping_token = ?, maybe_user_token = ?, twitch_user_id = ?, twitch_username = ?, @@ -147,6 +157,7 @@ SET ip_address_creation = ?, expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND) "#, + self.oauth_grouping_token.clone(), self.maybe_user_token.clone(), self.twitch_user_id.clone(), self.twitch_username.clone(), @@ -169,6 +180,7 @@ SET r#" INSERT INTO twitch_oauth_tokens SET + oauth_refresh_grouping_token = ?, maybe_user_token = ?, twitch_user_id = ?, twitch_username = ?, @@ -186,6 +198,7 @@ SET ip_address_creation = ?, expires_at = NULL "#, + self.oauth_grouping_token.clone(), self.maybe_user_token.clone(), self.twitch_user_id.clone(), self.twitch_username.clone(), diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 94d0c184e2..2f18feac06 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -518,7 +518,8 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { let mut query_builder = TwitchOauthTokenInsertBuilder::new( &self.twitch_oauth_token_record.twitch_user_id, &self.twitch_oauth_token_record.twitch_username, - &access_token) + &access_token, + &self.twitch_oauth_token_record.oauth_refresh_grouping_token) .set_refresh_token(refresh_token.as_deref()) .set_user_token(self.twitch_oauth_token_record.maybe_user_token.as_deref()) .set_expires_in_seconds(Some(expires_seconds)) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs index 4b111e65bd..5b8df49835 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs @@ -12,6 +12,7 @@ use std::sync::Arc; use twitch_common::oauth_token_builder::get_oauth_token_builder; use twitch_oauth2::tokens::BearerTokenType::UserToken; use twitch_oauth2::{CsrfToken, TwitchToken}; +use database_queries::tokens::Tokens; #[derive(Deserialize)] pub struct QueryParams { @@ -140,8 +141,14 @@ pub async fn oauth_end_enroll_from_redirect( let expires_seconds = expires_in.as_secs() as u32; // NB: Silent overflow + let oauth_grouping_token = Tokens::new_twitch_oauth_grouping_token() + .map_err(|e| { + error!("token creation error: {:?}", e); + OauthEndEnrollFromRedirectError::ServerError + })?; + let mut insert_builder = - TwitchOauthTokenInsertBuilder::new(&user_id, &twitch_username, &auth_token) + TwitchOauthTokenInsertBuilder::new(&user_id, &twitch_username, &auth_token, &oauth_grouping_token) .set_expires_in_seconds(Some(expires_seconds)) .set_refresh_token(refresh_token.as_deref()) .set_ip_address_creation(Some(&ip_address)) diff --git a/sqlx-data.json b/sqlx-data.json index 6456456c94..d1376ffdef 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -10,6 +10,16 @@ "nullable": [] } }, + "5b8c906b09cc7068dcad9eb837b1146893b63c3ce7c8dbefea9129849448c3f6": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 15 + }, + "nullable": [] + } + }, "5e4a00c339ce4d7fac19aff728449156b1e02b9e1d9fd534ac37d4ac79f44820": { "query": "\nINSERT INTO twitch_bits_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n is_anonymous= ?,\n bits_used = ?,\n total_bits_used = ?,\n chat_message = ?\n ", "describe": { @@ -30,22 +40,12 @@ "nullable": [] } }, - "620465c06c1dbdeb6740d62b9240468bb35b16a2b5ce3322f2415e552bd644b2": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", + "d7ac9f84e92d6b2111162a674284c0b9fa50c718bedfb8ab7238ca0a7b0875e6": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", "describe": { "columns": [], "parameters": { - "Right": 16 - }, - "nullable": [] - } - }, - "f661bddeb979daee9068d4757fbb88295c42f16e85f70ba9a44ec70bd0c9269c": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 14 + "Right": 17 }, "nullable": [] } From f2e740133abbc00c330103497a3ed712009815ab Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Feb 2022 07:10:45 -0500 Subject: [PATCH 0855/1042] Fix redis panic (why is it okay in dev??) --- .../obs_gateway_websocket_handler.rs | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index a1869463d1..bd5be48429 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -176,11 +176,30 @@ impl StreamHandler> for ObsGatewayWebSock self.write_obs_active().unwrap(); let redis_key = RedisKeys::twitch_tts_job_queue(&self.twitch_user_id.get_str()); - let values : Vec = redis.lpop((redis_key, 5)).unwrap(); // TODO: Error handling +// let values : Vec = redis.lpop((&redis_key, 5)).unwrap(); // TODO: Error handling +// +// if !values.is_empty() { +// info!("Got {} TTS values", values.len()); +// +// let payload = FrontendEventPayload { +// tts_job_tokens: values, +// }; +// +// match serde_json::to_string(&payload) { +// Ok(json) => { +// ctx.text(json); +// }, +// Err(e) => { +// error!("Error with JSON payload: {:?}", e); +// } +// } +// } - if !values.is_empty() { - info!("Got {} TTS values", values.len()); + let lookup_result : Option = redis.lpop(&redis_key).unwrap(); // TODO: Error handling + let mut values = Vec::new(); + if let Some(v) = lookup_result.as_deref() { + values.push(v.to_string()); let payload = FrontendEventPayload { tts_job_tokens: values, }; From 8494a6c81646e8eda3714d1a3bd8bc675c4f028e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Feb 2022 22:58:09 -0500 Subject: [PATCH 0856/1042] more debug logging --- .../endpoints_ws/obs_gateway_websocket_handler.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index bd5be48429..761b9a68f3 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -15,6 +15,7 @@ use database_queries::twitch_oauth::find::{TwitchOauthTokenFinder, TwitchOauthTo use futures_timer::Delay; use futures_util::FutureExt; use http_server_common::error::common_server_error::CommonServerError; +use log::debug; use log::error; use log::info; use log::warn; @@ -142,7 +143,10 @@ impl StreamHandler> for ObsGatewayWebSock msg: Result, ctx: &mut Self::Context, ) { + debug!("Socket Handler::handle()"); + if let Ok(msg) = msg { + debug!("Socket Handler::handle(): msg = {:?}", msg); // TODO: Only send this every 60 seconds. // TODO: Error handling. @@ -215,14 +219,20 @@ impl StreamHandler> for ObsGatewayWebSock } match msg { - ws::Message::Ping(bytes) => ctx.pong(&bytes), + ws::Message::Ping(bytes) => { + debug!("Socket Handler::handle(): got ping"); + ctx.pong(&bytes) + }, ws::Message::Text(text) => { + debug!("Socket Handler::handle(): got text = {:?}", text); //ctx.text("response") }, ws::Message::Binary(bin) => { + debug!("Socket Handler::handle(): got binary..."); //ctx.binary("response".as_bytes()) }, ws::Message::Close(reason) => { + warn!("Socket Handler::handle(): got close, reason = {:?}", reason); ctx.close(reason); ctx.stop(); } From 916a3f8abfd12fc4ac1a1a9e33347601b9df9d73 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 4 Feb 2022 10:21:27 -0500 Subject: [PATCH 0857/1042] send pong --- .../obs_gateway_websocket_handler.rs | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 761b9a68f3..a038b5a6ee 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -31,6 +31,7 @@ use twitch_api2::pubsub::Topic; use twitch_api2::pubsub; use twitch_api2::pubsub::channel_bits::ChannelBitsEventsV2Reply::BitsEvent; use twitch_api2::pubsub::TopicData::ChannelPointsChannelV1; +use crate::endpoints_ws::obs_gateway_websocket_handler::ResponseType::TtsEvent; // TODO: Redis calls are synchronous (but fast), but is there any way to make them async? @@ -46,10 +47,18 @@ pub struct QueryData { // Other preferences... } +/// Sent back to the frontend websocket. +#[derive(Serialize, Copy, Clone)] +pub enum ResponseType { + Pong, + TtsEvent, +} + /// Sent back to the frontend websocket. #[derive(Serialize)] pub struct FrontendEventPayload { - pub tts_job_tokens: Vec, + pub response_type: ResponseType, + pub tts_job_tokens: Option>, } /// Endpoint @@ -205,7 +214,8 @@ impl StreamHandler> for ObsGatewayWebSock if let Some(v) = lookup_result.as_deref() { values.push(v.to_string()); let payload = FrontendEventPayload { - tts_job_tokens: values, + response_type: TtsEvent, + tts_job_tokens: Some(values), }; match serde_json::to_string(&payload) { @@ -225,6 +235,21 @@ impl StreamHandler> for ObsGatewayWebSock }, ws::Message::Text(text) => { debug!("Socket Handler::handle(): got text = {:?}", text); + + let payload = FrontendEventPayload { + response_type: ResponseType::Pong, + tts_job_tokens: None, + }; + + match serde_json::to_string(&payload) { + Ok(json) => { + ctx.text(json); + }, + Err(e) => { + error!("Error with JSON PONG payload: {:?}", e); + } + } + //ctx.text("response") }, ws::Message::Binary(bin) => { From aa59e05d5bcc563017c7ceec58a9196ed0dfc803 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Feb 2022 21:30:32 -0500 Subject: [PATCH 0858/1042] optional serialization --- .../src/endpoints_ws/obs_gateway_websocket_handler.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index a038b5a6ee..f3edd03409 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -58,6 +58,7 @@ pub enum ResponseType { #[derive(Serialize)] pub struct FrontendEventPayload { pub response_type: ResponseType, + #[serde(skip_serializing_if = "Option::is_none")] pub tts_job_tokens: Option>, } @@ -249,7 +250,7 @@ impl StreamHandler> for ObsGatewayWebSock error!("Error with JSON PONG payload: {:?}", e); } } - + //ctx.text("response") }, ws::Message::Binary(bin) => { From 75461a49dff5793ee84e6ece01571e9036affb93 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Feb 2022 22:40:04 -0500 Subject: [PATCH 0859/1042] new small crate to get rid of smart quotes, etc --- Cargo.lock | 12 +++++++ Cargo.toml | 1 + crates/lib/tts_common/Cargo.toml | 26 +++++++++++++++ crates/lib/tts_common/src/clean_symbols.rs | 39 ++++++++++++++++++++++ crates/lib/tts_common/src/lib.rs | 1 + 5 files changed, 79 insertions(+) create mode 100644 crates/lib/tts_common/Cargo.toml create mode 100644 crates/lib/tts_common/src/clean_symbols.rs create mode 100644 crates/lib/tts_common/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 83b5ebb9d5..5dc9b98b01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4357,6 +4357,18 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "tts_common" +version = "0.0.1" +dependencies = [ + "anyhow", + "container_common", + "log", + "once_cell", + "regex", + "unicode-segmentation", +] + [[package]] name = "tungstenite" version = "0.14.0" diff --git a/Cargo.toml b/Cargo.toml index d25059b54f..590406f5ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/lib/database_queries", "crates/lib/http_server_common", "crates/lib/redis_common", + "crates/lib/tts_common", "crates/lib/twitch_common", # Application diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml new file mode 100644 index 0000000000..4d71f79846 --- /dev/null +++ b/crates/lib/tts_common/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "tts_common" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "tts_common" +path = "src/lib.rs" + +[dependencies] + +# Internal +container_common = { path = "../container_common" } + +# External +anyhow = "1.0.40" +log = "0.4.14" +once_cell = "1.9.0" +regex = "1.5.4" +unicode-segmentation = "1.7.1" + diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs new file mode 100644 index 0000000000..bd261acee9 --- /dev/null +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -0,0 +1,39 @@ +use once_cell::sync::Lazy; +use std::collections::HashMap; +use log::info; +use unicode_segmentation::UnicodeSegmentation; + +static REPLACEMENTS : Lazy> = Lazy::new(|| { + let mut map = HashMap::new(); + map.insert("’".to_string(), "'".to_string()); + map.insert("‘".to_string(), "'".to_string()); + map.insert("“".to_string(), "\"".to_string()); + map.insert("”".to_string(), "\"".to_string()); + map +}); + +pub fn clean_symbols(input_text: &str) -> String { + let segmented= UnicodeSegmentation::graphemes(input_text, true) + .map(|segment| { + if let Some(replace) = REPLACEMENTS.get(segment) { + return replace.as_str(); + } + return segment; + }) + .collect::>(); + + segmented.join("") +} + +#[cfg(test)] +mod tests { + use crate::clean_symbols::clean_symbols; + + #[test] + fn filters_smart_quotes() { + assert_eq!(clean_symbols("That’s ok"), "That's ok".to_string()); + assert_eq!(clean_symbols("That’s it’s new home"), "That's it's new home".to_string()); + assert_eq!(clean_symbols("‘foo’"), "'foo'".to_string()); + assert_eq!(clean_symbols("“as ‘shown’ here.”"), "\"as 'shown' here.\"".to_string()); + } +} diff --git a/crates/lib/tts_common/src/lib.rs b/crates/lib/tts_common/src/lib.rs new file mode 100644 index 0000000000..82df584c77 --- /dev/null +++ b/crates/lib/tts_common/src/lib.rs @@ -0,0 +1 @@ +pub mod clean_symbols; \ No newline at end of file From fa5c88ea1e8e0d935a318fd987b781f8b4eb18a5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Feb 2022 22:56:21 -0500 Subject: [PATCH 0860/1042] en dash / em dash --- crates/lib/tts_common/src/clean_symbols.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index bd261acee9..c34a3c0fe9 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -9,6 +9,8 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { map.insert("‘".to_string(), "'".to_string()); map.insert("“".to_string(), "\"".to_string()); map.insert("”".to_string(), "\"".to_string()); + map.insert("–".to_string(), "-".to_string()); // endash + map.insert("—".to_string(), "-".to_string()); // emdash map }); @@ -36,4 +38,10 @@ mod tests { assert_eq!(clean_symbols("‘foo’"), "'foo'".to_string()); assert_eq!(clean_symbols("“as ‘shown’ here.”"), "\"as 'shown' here.\"".to_string()); } + + #[test] + fn filters_dashes() { + assert_eq!(clean_symbols("en – dash"), "en - dash".to_string()); + assert_eq!(clean_symbols("em — dash"), "em - dash".to_string()); + } } From 0c29c4133e83a288107fb0d1a5c73d61da18ceae Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Feb 2022 22:58:55 -0500 Subject: [PATCH 0861/1042] a few more tests --- crates/lib/tts_common/src/clean_symbols.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index c34a3c0fe9..cd467547f8 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -1,6 +1,5 @@ use once_cell::sync::Lazy; use std::collections::HashMap; -use log::info; use unicode_segmentation::UnicodeSegmentation; static REPLACEMENTS : Lazy> = Lazy::new(|| { @@ -31,6 +30,12 @@ pub fn clean_symbols(input_text: &str) -> String { mod tests { use crate::clean_symbols::clean_symbols; + #[test] + fn neutral_tests() { + assert_eq!(clean_symbols("this should be the same."), "this should be the same.".to_string()); + assert_eq!(clean_symbols("one\ntwo\r\nthree "), "one\ntwo\r\nthree ".to_string()); + } + #[test] fn filters_smart_quotes() { assert_eq!(clean_symbols("That’s ok"), "That's ok".to_string()); From b800ace75462d291fe9b7a49a45379c63651855b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Feb 2022 23:03:51 -0500 Subject: [PATCH 0862/1042] fix tts inference for smart quotes --- Cargo.lock | 1 + crates/service/storyteller_web/Cargo.toml | 1 + crates/service/storyteller_web/src/tts_inference_job.rs | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 5dc9b98b01..3b714f1036 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3813,6 +3813,7 @@ dependencies = [ "time 0.2.26", "tokio 1.13.0", "toml", + "tts_common", "twitch_api2", "twitch_common", "twitch_oauth2", diff --git a/crates/service/storyteller_web/Cargo.toml b/crates/service/storyteller_web/Cargo.toml index 21e30c7593..f90aae2e13 100644 --- a/crates/service/storyteller_web/Cargo.toml +++ b/crates/service/storyteller_web/Cargo.toml @@ -43,6 +43,7 @@ container_common = { path = "../../lib/container_common" } database_queries = { path = "../../lib/database_queries" } http_server_common = { path = "../../lib/http_server_common" } redis_common = { path = "../../lib/redis_common" } +tts_common = { path = "../../lib/tts_common" } twitch_common = { path = "../../lib/twitch_common" } # External diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index fc4f18c1a9..10d35cb5e0 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -75,6 +75,7 @@ use std::process::Command; use std::thread; use std::time::{Duration, SystemTime, UNIX_EPOCH, Instant}; use tempdir::TempDir; +use tts_common::clean_symbols::clean_symbols; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -773,7 +774,8 @@ async fn process_job( let text_input_fs_path = temp_dir.path().join("inference_input.txt"); - std::fs::write(&text_input_fs_path, &job.raw_inference_text)?; + let cleaned_inference_text = clean_symbols(&job.raw_inference_text); + std::fs::write(&text_input_fs_path, &cleaned_inference_text)?; // ==================== RUN INFERENCE ==================== // From 794010df9c047f408e8cdabf75760d473c945692 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Feb 2022 23:14:28 -0500 Subject: [PATCH 0863/1042] ellipsis --- crates/lib/tts_common/src/clean_symbols.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index cd467547f8..6a464f8406 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -10,6 +10,7 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { map.insert("”".to_string(), "\"".to_string()); map.insert("–".to_string(), "-".to_string()); // endash map.insert("—".to_string(), "-".to_string()); // emdash + map.insert("…".to_string(), "...".to_string()); map }); @@ -49,4 +50,9 @@ mod tests { assert_eq!(clean_symbols("en – dash"), "en - dash".to_string()); assert_eq!(clean_symbols("em — dash"), "em - dash".to_string()); } + + #[test] + fn filters_ellipsis() { + assert_eq!(clean_symbols("test…"), "test...".to_string()); + } } From ee7634b9d00262339c41415e6e25bcc6f19cf45c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Feb 2022 23:33:21 -0500 Subject: [PATCH 0864/1042] unicode literals --- crates/lib/tts_common/src/clean_symbols.rs | 30 +++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 6a464f8406..22dee66947 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -2,14 +2,31 @@ use once_cell::sync::Lazy; use std::collections::HashMap; use unicode_segmentation::UnicodeSegmentation; +// https://unicodelookup.com/#quo static REPLACEMENTS : Lazy> = Lazy::new(|| { let mut map = HashMap::new(); - map.insert("’".to_string(), "'".to_string()); - map.insert("‘".to_string(), "'".to_string()); - map.insert("“".to_string(), "\"".to_string()); - map.insert("”".to_string(), "\"".to_string()); - map.insert("–".to_string(), "-".to_string()); // endash - map.insert("—".to_string(), "-".to_string()); // emdash + // Quotes (single) + map.insert("\u{0060}".to_string(), "'".to_string()); // Grave Accent + map.insert("\u{00B4}".to_string(), "'".to_string()); // Acute Accent + map.insert("\u{2018}".to_string(), "'".to_string()); // Left Single Quotation Mark + map.insert("\u{2019}".to_string(), "'".to_string()); // Right Single Quotation Mark + map.insert("\u{201A}".to_string(), "'".to_string()); // Single Low-9 Quotation Mark + // Quotes (double) + map.insert("\u{201C}".to_string(), "\"".to_string()); // Left Double Quotation Mark + map.insert("\u{201D}".to_string(), "\"".to_string()); // Right Double Quotation Mark + map.insert("\u{201E}".to_string(), "\"".to_string()); // Double Low-9 Quotation Mark + // Dashes + map.insert("\u{2010}".to_string(), "-".to_string()); // Hyphen + map.insert("\u{2011}".to_string(), "-".to_string()); // Non-Breaking Hyphen + map.insert("\u{2013}".to_string(), "-".to_string()); // En Dash + map.insert("\u{2014}".to_string(), "-".to_string()); // Em Dash + map.insert("\u{2015}".to_string(), "-".to_string()); // Horizontal Bar + map.insert("\u{2E3A}".to_string(), "-".to_string()); // Two-Em Dash + map.insert("\u{2E3B}".to_string(), "-".to_string()); // Three-Em Dash + map.insert("\u{FE58}".to_string(), "-".to_string()); // Small Em Dash + map.insert("\u{FE63}".to_string(), "-".to_string()); // Small Hyphen-Minus + map.insert("\u{FF0D}".to_string(), "-".to_string()); // Fullwidth Hyphen-Minus + // Ellipsis map.insert("…".to_string(), "...".to_string()); map }); @@ -49,6 +66,7 @@ mod tests { fn filters_dashes() { assert_eq!(clean_symbols("en – dash"), "en - dash".to_string()); assert_eq!(clean_symbols("em — dash"), "em - dash".to_string()); + assert_eq!(clean_symbols("three em ⸻ dash"), "three em - dash".to_string()); } #[test] From 3df44a8d51cf70d6b4ec06344c352e4654cff873 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Feb 2022 23:36:54 -0500 Subject: [PATCH 0865/1042] more characters to convert --- crates/lib/tts_common/src/clean_symbols.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 22dee66947..3d9126297b 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -11,10 +11,16 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { map.insert("\u{2018}".to_string(), "'".to_string()); // Left Single Quotation Mark map.insert("\u{2019}".to_string(), "'".to_string()); // Right Single Quotation Mark map.insert("\u{201A}".to_string(), "'".to_string()); // Single Low-9 Quotation Mark + map.insert("\u{201B}".to_string(), "'".to_string()); // Single High-Revered-9 Quotation Mark // Quotes (double) map.insert("\u{201C}".to_string(), "\"".to_string()); // Left Double Quotation Mark map.insert("\u{201D}".to_string(), "\"".to_string()); // Right Double Quotation Mark map.insert("\u{201E}".to_string(), "\"".to_string()); // Double Low-9 Quotation Mark + map.insert("\u{201F}".to_string(), "\"".to_string()); // Double High-Reversed-9 Quotation Mark + map.insert("\u{301D}".to_string(), "\"".to_string()); // Reversed Double Prime Quotation Mark + map.insert("\u{301E}".to_string(), "\"".to_string()); // Double Prime Quotation Mark + map.insert("\u{301F}".to_string(), "\"".to_string()); // Low Double Prime Quotation Mark + map.insert("\u{FF02}".to_string(), "\"".to_string()); // Fullwidth Quotation Mark // Dashes map.insert("\u{2010}".to_string(), "-".to_string()); // Hyphen map.insert("\u{2011}".to_string(), "-".to_string()); // Non-Breaking Hyphen From ee6631f4327d118693cf9cd97c8d098993e8c2a0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Feb 2022 23:52:34 -0500 Subject: [PATCH 0866/1042] latin letters --- crates/lib/tts_common/src/clean_symbols.rs | 66 ++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 3d9126297b..0777b6a2ec 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -34,6 +34,72 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { map.insert("\u{FF0D}".to_string(), "-".to_string()); // Fullwidth Hyphen-Minus // Ellipsis map.insert("…".to_string(), "...".to_string()); + // Latin characters with accent (A) + map.insert("\u{C0}".to_string(), "A".to_string()); // Latin capital letter a with grave + map.insert("\u{C1}".to_string(), "A".to_string()); // Latin capital letter a with acute + map.insert("\u{C2}".to_string(), "A".to_string()); // Latin capital letter a with circumflex + map.insert("\u{C3}".to_string(), "A".to_string()); // Latin capital letter a with tilde + map.insert("\u{C4}".to_string(), "A".to_string()); // Latin capital letter a diaeresis + map.insert("\u{C5}".to_string(), "A".to_string()); // Latin capital letter a with ring above + // Latin characters with accent (AE) + map.insert("\u{C6}".to_string(), "AE".to_string()); // Latin capital letter ae + // Latin characters with accent (C) + map.insert("\u{C7}".to_string(), "C".to_string()); // Latin capital letter c with cedilla + // Latin characters with accent (E) + map.insert("\u{C8}".to_string(), "E".to_string()); // Latin capital letter e with grave + map.insert("\u{C9}".to_string(), "E".to_string()); // Latin capital letter e with acute + map.insert("\u{CA}".to_string(), "E".to_string()); // Latin capital letter e with circumflex + map.insert("\u{CB}".to_string(), "E".to_string()); // Latin capital letter e with diaeresis + // Latin characters with accent (I) + map.insert("\u{CC}".to_string(), "I".to_string()); // Latin capital letter i with grave + map.insert("\u{CD}".to_string(), "I".to_string()); // Latin capital letter i with acute + map.insert("\u{CE}".to_string(), "I".to_string()); // Latin capital letter i with circumflex + map.insert("\u{CF}".to_string(), "I".to_string()); // Latin capital letter i with diaeresis + // Latin characters with accent (ETH) + // map.insert("\u{D0}".to_string(), "D".to_string()); // Latin capital letter eth + // Latin characters with accent (N) + map.insert("\u{D1}".to_string(), "N".to_string()); // Latin capital letter n with tilde + // Latin characters with accent (O) + map.insert("\u{D2}".to_string(), "O".to_string()); // Latin capital letter o with grave + map.insert("\u{D3}".to_string(), "O".to_string()); // Latin capital letter o with acute + map.insert("\u{D4}".to_string(), "O".to_string()); // Latin capital letter o with circumflex + map.insert("\u{D5}".to_string(), "O".to_string()); // Latin capital letter o with tilde + map.insert("\u{D6}".to_string(), "O".to_string()); // Latin capital letter o with diaeresis + map.insert("\u{D7}".to_string(), "O".to_string()); // Latin capital letter o with stroke + // Latin characters with accent (U) + map.insert("\u{D9}".to_string(), "U".to_string()); // Latin capital letter u with grave + map.insert("\u{DA}".to_string(), "U".to_string()); // Latin capital letter u with acute + map.insert("\u{DB}".to_string(), "U".to_string()); // Latin capital letter u with circumflex + map.insert("\u{DC}".to_string(), "U".to_string()); // Latin capital letter u with diaeresis + // Latin characters with accent (Y) + map.insert("\u{DD}".to_string(), "Y".to_string()); // Latin capital letter y with acute + // Latin characters with accent (THORN) + // map.insert("\u{DE}".to_string(), "P".to_string()); // Latin capital letter thorn + // Latin characters with accent (SHARP S) + // map.insert("\u{DF}".to_string(), "B".to_string()); // Latin capital letter sharp s + // Latin characters with accent (a) + map.insert("\u{E0}".to_string(), "a".to_string()); // Latin small letter a with grave + map.insert("\u{E1}".to_string(), "a".to_string()); // Latin small letter a with acute + map.insert("\u{E2}".to_string(), "a".to_string()); // Latin small letter a with circumflex + map.insert("\u{E3}".to_string(), "a".to_string()); // Latin small letter a with tilde + map.insert("\u{E4}".to_string(), "a".to_string()); // Latin small letter a with diaeresis + map.insert("\u{E5}".to_string(), "a".to_string()); // Latin small letter a with ring above + // Latin characters with accent (ae) + map.insert("\u{E6}".to_string(), "ae".to_string()); // Latin small letter ae + // Latin characters with accent (c) + map.insert("\u{E7}".to_string(), "c".to_string()); // Latin small letter c with cedilla + // Latin characters with accent (e) + map.insert("\u{E8}".to_string(), "e".to_string()); // Latin small letter e with grave + map.insert("\u{E9}".to_string(), "e".to_string()); // Latin small letter e with acute + map.insert("\u{EA}".to_string(), "e".to_string()); // Latin small letter e with circumflex + map.insert("\u{EB}".to_string(), "e".to_string()); // Latin small letter e with diaeresis + // Latin characters with accent (i) + map.insert("\u{EC}".to_string(), "i".to_string()); // Latin small letter i with grave + map.insert("\u{ED}".to_string(), "i".to_string()); // Latin small letter i with acute + map.insert("\u{EE}".to_string(), "i".to_string()); // Latin small letter i with circumflex + map.insert("\u{EF}".to_string(), "i".to_string()); // Latin small letter i with diaeresis + // Latin characters with accent (eth) + // map.insert("\u{F0}".to_string(), "d".to_string()); // Latin small letter eith map }); From fd589f85e8af287ebcc4401306473791eaed681f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 00:04:31 -0500 Subject: [PATCH 0867/1042] latin tests --- crates/lib/tts_common/src/clean_symbols.rs | 24 +++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 0777b6a2ec..56dc874e04 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -65,7 +65,7 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { map.insert("\u{D4}".to_string(), "O".to_string()); // Latin capital letter o with circumflex map.insert("\u{D5}".to_string(), "O".to_string()); // Latin capital letter o with tilde map.insert("\u{D6}".to_string(), "O".to_string()); // Latin capital letter o with diaeresis - map.insert("\u{D7}".to_string(), "O".to_string()); // Latin capital letter o with stroke + map.insert("\u{D8}".to_string(), "O".to_string()); // Latin capital letter o with stroke (NB: Skips D7) // Latin characters with accent (U) map.insert("\u{D9}".to_string(), "U".to_string()); // Latin capital letter u with grave map.insert("\u{DA}".to_string(), "U".to_string()); // Latin capital letter u with acute @@ -145,4 +145,26 @@ mod tests { fn filters_ellipsis() { assert_eq!(clean_symbols("test…"), "test...".to_string()); } + + #[test] + fn filters_latin_characters() { + assert_eq!(clean_symbols("pokémon"), "pokemon".to_string()); + assert_eq!(clean_symbols("POKÉMON"), "POKEMON".to_string()); + assert_eq!(clean_symbols("Æther"), "AEther".to_string()); + assert_eq!(clean_symbols("æther"), "aether".to_string()); + // Almost exhaustive + assert_eq!(clean_symbols("ÀÁÂÃÄÅ"), "AAAAAA".to_string()); + assert_eq!(clean_symbols("Æ"), "AE".to_string()); + assert_eq!(clean_symbols("Ç"), "C".to_string()); + assert_eq!(clean_symbols("ÈÉÊË"), "EEEE".to_string()); + assert_eq!(clean_symbols("ÌÍÎÏ"), "IIII".to_string()); + assert_eq!(clean_symbols("ÒÓÔÕÖØ"), "OOOOOO".to_string()); + assert_eq!(clean_symbols("ÙÚÛÜ"), "UUUU".to_string()); + assert_eq!(clean_symbols("Ý"), "Y".to_string()); + assert_eq!(clean_symbols("àáâãäå"), "aaaaaa".to_string()); + assert_eq!(clean_symbols("æ"), "ae".to_string()); + assert_eq!(clean_symbols("ç"), "c".to_string()); + assert_eq!(clean_symbols("èéêë"), "eeee".to_string()); + assert_eq!(clean_symbols("ìíîï"), "iiii".to_string()); + } } From d6097b66c35fd3268d6b8ada68bd4fb61e5dcf68 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 00:22:41 -0500 Subject: [PATCH 0868/1042] more spanish characters --- crates/lib/tts_common/src/clean_symbols.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 56dc874e04..ad4604f66f 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use unicode_segmentation::UnicodeSegmentation; // https://unicodelookup.com/#quo +// http://www.geocities.ws/click2speak/unicode/chars_es.html static REPLACEMENTS : Lazy> = Lazy::new(|| { let mut map = HashMap::new(); // Quotes (single) @@ -100,6 +101,12 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { map.insert("\u{EF}".to_string(), "i".to_string()); // Latin small letter i with diaeresis // Latin characters with accent (eth) // map.insert("\u{F0}".to_string(), "d".to_string()); // Latin small letter eith + // Spanish special characters + map.insert("\u{F1}".to_string(), "n".to_string()); // Lower n tilde + map.insert("\u{A1}".to_string(), "!".to_string()); // Inverted exclamation mark + map.insert("\u{BF}".to_string(), "?".to_string()); // Inverted question mark + map.insert("\u{AA}".to_string(), "a".to_string()); // Feminine ordinal + map.insert("\u{BA}".to_string(), "o".to_string()); // Masculine ordinal map }); @@ -167,4 +174,19 @@ mod tests { assert_eq!(clean_symbols("èéêë"), "eeee".to_string()); assert_eq!(clean_symbols("ìíîï"), "iiii".to_string()); } + + #[test] + fn filters_spanish_characters() { + assert_eq!(clean_symbols("¡"), "!".to_string()); + assert_eq!(clean_symbols("¿"), "?".to_string()); + } + + #[test] + fn actual_database_failures() { + assert_eq!(clean_symbols( + "Sabías que?,tu papá es el tercer planeta del sistema solar"), + "Sabias que?,tu papa es el tercer planeta del sistema solar".to_string()); + assert_eq!(clean_symbols("señoras"), "senoras".to_string()); + + } } From 538568e96349264a61f8d735841381515d7b1641 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 02:05:34 -0500 Subject: [PATCH 0869/1042] more code points to convert --- .gitignore | 1 + crates/lib/tts_common/src/clean_symbols.rs | 74 ++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/.gitignore b/.gitignore index 49d8c6ee90..e32e3775cb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ _ignored_schema.rs models/Wav2Lip/python/ models/tacotron2/python/ production_database_secret_connect.sh +production_database_secret_query.sh secrets.toml secrets.yaml target/ diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index ad4604f66f..21ba3b6c55 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -6,6 +6,11 @@ use unicode_segmentation::UnicodeSegmentation; // http://www.geocities.ws/click2speak/unicode/chars_es.html static REPLACEMENTS : Lazy> = Lazy::new(|| { let mut map = HashMap::new(); + // Weird spacing characters + map.insert("\u{00a0}".to_string(), " ".to_string()); // Non-break space (aka  ) \xa0 + map.insert("\u{2005}".to_string(), " ".to_string()); // Four-Per-Em Space + map.insert("\u{2588}".to_string(), " ".to_string()); // Full Block + map.insert("\u{205F}".to_string(), " ".to_string()); // Medium Mathematical Space (MMSP) // Quotes (single) map.insert("\u{0060}".to_string(), "'".to_string()); // Grave Accent map.insert("\u{00B4}".to_string(), "'".to_string()); // Acute Accent @@ -35,6 +40,8 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { map.insert("\u{FF0D}".to_string(), "-".to_string()); // Fullwidth Hyphen-Minus // Ellipsis map.insert("…".to_string(), "...".to_string()); + // Commas + map.insert("\u{3001}".to_string(), ",".to_string()); // Latin characters with accent (A) map.insert("\u{C0}".to_string(), "A".to_string()); // Latin capital letter a with grave map.insert("\u{C1}".to_string(), "A".to_string()); // Latin capital letter a with acute @@ -107,6 +114,23 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { map.insert("\u{BF}".to_string(), "?".to_string()); // Inverted question mark map.insert("\u{AA}".to_string(), "a".to_string()); // Feminine ordinal map.insert("\u{BA}".to_string(), "o".to_string()); // Masculine ordinal + // Misc characters that frequently occur + map.insert("\u{00F3}".to_string(), "o".to_string()); // Latin Small Letter O with Acute + map.insert("\u{0131}".to_string(), "i".to_string()); // Latin Small Letter Dotless I + map.insert("\u{00FC}".to_string(), "u".to_string()); // Latin Small Letter U with Diaeresis + map.insert("\u{015F}".to_string(), "s".to_string()); // Latin Small Letter S with Cedilla + map.insert("\u{00F6}".to_string(), "o".to_string()); // Latin Small Letter O with Diaeresis + map.insert("\u{011F}".to_string(), "g".to_string()); // Latin Small Letter G with Breve + map.insert("\u{0130}".to_string(), "I".to_string()); // Latin Capital Letter I with Dot Above + map.insert("\u{0259}".to_string(), "e".to_string()); // Latin Small Letter Schwa (TODO: Inaccurate?) + map.insert("\u{012B}".to_string(), "i".to_string()); // Latin Small Letter I with Macron + map.insert("\u{1D3A}".to_string(), "n".to_string()); // Modifier Letter Capital N + map.insert("\u{01D0}".to_string(), "i".to_string()); // Latin Small Letter I with Caron + map.insert("\u{014D}".to_string(), "o".to_string()); // Latin Small Letter O with Macron + map.insert("\u{0101}".to_string(), "a".to_string()); // Latin Small Letter A with Macron + map.insert("\u{00F9}".to_string(), "u".to_string()); // Latin Small Letter U with Grave + map.insert("\u{01Ce}".to_string(), "a".to_string()); // Latin Small Letter A with Caron + map.insert("\u{203C}".to_string(), "!!".to_string()); // Double Exclamation Mark map }); @@ -129,6 +153,7 @@ mod tests { #[test] fn neutral_tests() { + assert_eq!(clean_symbols(""), "".to_string()); // Empty check assert_eq!(clean_symbols("this should be the same."), "this should be the same.".to_string()); assert_eq!(clean_symbols("one\ntwo\r\nthree "), "one\ntwo\r\nthree ".to_string()); } @@ -188,5 +213,54 @@ mod tests { "Sabias que?,tu papa es el tercer planeta del sistema solar".to_string()); assert_eq!(clean_symbols("señoras"), "senoras".to_string()); + // Most frequent according to partial histogram + assert_eq!(clean_symbols("á"), "a".to_string()); + assert_eq!(clean_symbols("í"), "i".to_string()); + assert_eq!(clean_symbols("ó"), "o".to_string()); + assert_eq!(clean_symbols("é"), "e".to_string()); + assert_eq!(clean_symbols("ñ"), "n".to_string()); + assert_eq!(clean_symbols("ú"), "ú".to_string()); + assert_eq!(clean_symbols("ı"), "i".to_string()); // Dotless i + assert_eq!(clean_symbols("ü"), "u".to_string()); + assert_eq!(clean_symbols("¿"), "?".to_string()); + assert_eq!(clean_symbols("…"), "...".to_string()); + assert_eq!(clean_symbols("¡"), "!".to_string()); + assert_eq!(clean_symbols("ş"), "s".to_string()); + assert_eq!(clean_symbols("ç"), "c".to_string()); + assert_eq!(clean_symbols("”"), "\"".to_string()); + assert_eq!(clean_symbols("“"), "\"".to_string()); + assert_eq!(clean_symbols("ö"), "o".to_string()); + assert_eq!(clean_symbols("ğ"), "g".to_string()); + assert_eq!(clean_symbols("\u{00a0}"), " ".to_string()); + assert_eq!(clean_symbols("ã"), "a".to_string()); + assert_eq!(clean_symbols("à"), "a".to_string()); + assert_eq!(clean_symbols("Á"), "A".to_string()); + assert_eq!(clean_symbols("İ"), "I".to_string()); + assert_eq!(clean_symbols(" "), " ".to_string()); + assert_eq!(clean_symbols(" "), " ".to_string()); + assert_eq!(clean_symbols("—"), "-".to_string()); + assert_eq!(clean_symbols("Ñ"), "N".to_string()); + assert_eq!(clean_symbols("´"), "'".to_string()); + assert_eq!(clean_symbols("ê"), "e".to_string()); + assert_eq!(clean_symbols("ə"), "e".to_string()); + assert_eq!(clean_symbols("è"), "e".to_string()); + assert_eq!(clean_symbols("ī"), "i".to_string()); + assert_eq!(clean_symbols("ᴺ"), "n".to_string()); + assert_eq!(clean_symbols(" "), " ".to_string()); + assert_eq!(clean_symbols("ä"), "a".to_string()); + assert_eq!(clean_symbols("ǐ"), "i".to_string()); + assert_eq!(clean_symbols("█"), " ".to_string()); + assert_eq!(clean_symbols("ō"), "o".to_string()); + assert_eq!(clean_symbols("‘"), "'".to_string()); + assert_eq!(clean_symbols("Í"), "I".to_string()); + assert_eq!(clean_symbols("É"), "E".to_string()); + assert_eq!(clean_symbols("ā"), "a".to_string()); + assert_eq!(clean_symbols("ù"), "u".to_string()); + assert_eq!(clean_symbols("、"), ",".to_string()); + assert_eq!(clean_symbols("ǎ"), "a".to_string()); + assert_eq!(clean_symbols("‼"), "!!".to_string()); + + // TODO: assert_eq!(clean_symbols("ß"), "B".to_string()); + // TODO: assert_eq!(clean_symbols("°"), "degrees".to_string()); } } From ec4816bd009cd91e19bfb0e7ea3b376d72553040 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 15:16:23 -0500 Subject: [PATCH 0870/1042] Upgrade vendored newrelic-telemetry-sdk tokio 0.2 -> 1.0 --- Cargo.lock | 46 ++----------------- vendor/newrelic-telemetry-sdk-rust/Cargo.toml | 8 ++-- 2 files changed, 9 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b714f1036..da8c41e035 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1931,19 +1931,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-tls" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" -dependencies = [ - "bytes 0.5.6", - "hyper 0.13.10", - "native-tls", - "tokio 0.2.25", - "tokio-tls", -] - [[package]] name = "hyper-tls" version = "0.5.0" @@ -2464,13 +2451,13 @@ dependencies = [ "env_logger", "flate2", "futures 0.3.17", - "hyper 0.13.10", - "hyper-tls 0.4.3", + "hyper 0.14.14", + "hyper-tls", "log", "rand 0.7.3", "serde", "serde_json", - "tokio 0.2.25", + "tokio 1.13.0", "uuid", ] @@ -3155,7 +3142,7 @@ dependencies = [ "http", "http-body 0.4.4", "hyper 0.14.14", - "hyper-tls 0.5.0", + "hyper-tls", "ipnet", "js-sys", "lazy_static", @@ -4037,11 +4024,9 @@ dependencies = [ "memchr", "mio 0.6.23", "mio-uds", - "num_cpus", "pin-project-lite 0.1.12", "signal-hook-registry", "slab", - "tokio-macros 0.2.6", "winapi 0.3.9", ] @@ -4061,7 +4046,7 @@ dependencies = [ "parking_lot 0.11.1", "pin-project-lite 0.2.6", "signal-hook-registry", - "tokio-macros 1.5.1", + "tokio-macros", "winapi 0.3.9", ] @@ -4097,17 +4082,6 @@ dependencies = [ "log", ] -[[package]] -name = "tokio-macros" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tokio-macros" version = "1.5.1" @@ -4194,16 +4168,6 @@ dependencies = [ "tokio-reactor", ] -[[package]] -name = "tokio-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" -dependencies = [ - "native-tls", - "tokio 0.2.25", -] - [[package]] name = "tokio-tungstenite" version = "0.15.0" diff --git a/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/vendor/newrelic-telemetry-sdk-rust/Cargo.toml index 0434c20be1..b11c42a815 100644 --- a/vendor/newrelic-telemetry-sdk-rust/Cargo.toml +++ b/vendor/newrelic-telemetry-sdk-rust/Cargo.toml @@ -15,15 +15,15 @@ log = "0.4" serde = { version = "1.0.57", features = ["derive"] } serde_json = "1.0.57" -hyper = { version = "0.13", features = [], optional = true } -hyper-tls = { version = "0.4.0", optional = true } +hyper = { version = "0.14", features = [], optional = true } +hyper-tls = { version = "0.5.0", optional = true } flate2 = { version = "1.0.16", optional = true } futures = { version = "0.3.1", optional = true } -tokio = { version = "0.2.6", features = ["rt-threaded", "macros"], optional = true } +tokio = { version = "1.1.0", features = ["macros"], optional = true } uuid = { version = "0.8.1", features = ["v4"] } [dev-dependencies] rand = "0.7.3" env_logger = "0.7.1" futures = { version = "0.3.1" } -tokio = { version = "0.2.6", features = ["rt-threaded", "macros"] } +tokio = { version = "1.1.0", features = ["macros"] } From 4e8c736814c4382f440c9568b3dc64d2bf66d1c8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 17:52:46 -0500 Subject: [PATCH 0871/1042] forgot a few references to raw text --- crates/service/storyteller_web/src/tts_inference_job.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index 10d35cb5e0..2875a043c0 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -767,6 +767,10 @@ async fn process_job( info!("Downloaded template media from bucket!"); } + // ==================== Preprocess text ==================== // + + let cleaned_inference_text = clean_symbols(&job.raw_inference_text); + // ==================== WRITE TEXT TO FILE ==================== // let temp_dir = format!("temp_tts_inference_{}", job.id); @@ -774,7 +778,6 @@ async fn process_job( let text_input_fs_path = temp_dir.path().join("inference_input.txt"); - let cleaned_inference_text = clean_symbols(&job.raw_inference_text); std::fs::write(&text_input_fs_path, &cleaned_inference_text)?; // ==================== RUN INFERENCE ==================== // @@ -818,7 +821,7 @@ async fn process_job( info!("With pretrained vocoder: {:?}", pretrained_vocoder); inferencer.tts_inference_sidecar_client.request_inference( - &job.raw_inference_text, + &cleaned_inference_text, &tts_synthesizer_fs_path, pretrained_vocoder, &hifigan_vocoder_model_fs_path, @@ -885,7 +888,7 @@ async fn process_job( // ==================== SAVE RECORDS ==================== // - let text_hash = hash_string_sha2(&job.raw_inference_text)?; + let text_hash = hash_string_sha2(&cleaned_inference_text)?; info!("Saving tts inference record..."); let (id, inference_result_token) = insert_tts_result( From c25639baa4eea83719ef646fbc89990b0cd7d01c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 19:04:08 -0500 Subject: [PATCH 0872/1042] Create user_input_common shared lib crate --- Cargo.lock | 12 ++++++++++ Cargo.toml | 1 + crates/lib/user_input_common/Cargo.toml | 24 +++++++++++++++++++ .../user_input_common/src}/check_for_slurs.rs | 15 ++++++------ crates/lib/user_input_common/src/lib.rs | 12 ++++++++++ crates/service/storyteller_web/Cargo.toml | 1 + .../categories/assign_tts_category.rs | 1 - .../endpoints/categories/create_category.rs | 1 - .../endpoints/categories/get_category.rs | 1 - .../categories/list_tts_categories.rs | 1 - .../list_tts_model_assigned_categories.rs | 1 - .../moderation/categories/delete_category.rs | 1 - .../moderation/categories/edit_category.rs | 1 - .../list_tts_categories_for_moderation.rs | 1 - .../moderation/ip_bans/add_ip_ban.rs | 1 - .../moderation/ip_bans/delete_ip_ban.rs | 1 - .../moderation/ip_bans/get_ip_ban.rs | 1 - .../moderation/ip_bans/list_ip_bans.rs | 1 - .../moderation/user_roles/list_roles.rs | 1 - .../moderation/user_roles/list_staff.rs | 1 - .../endpoints/tts/edit_tts_model.rs | 6 ++--- .../endpoints/tts/edit_tts_result.rs | 1 - .../endpoints/tts/enqueue_infer_tts.rs | 2 +- .../endpoints/users/create_account.rs | 4 ++-- .../endpoints/users/edit_profile.rs | 4 ++-- .../endpoints/w2l/edit_w2l_result.rs | 1 - .../endpoints/w2l/edit_w2l_template.rs | 4 ++-- .../w2l/enqueue_upload_w2l_template.rs | 4 ++-- crates/service/storyteller_web/src/main.rs | 1 - .../storyteller_web/src/validations/mod.rs | 1 - 30 files changed, 70 insertions(+), 37 deletions(-) create mode 100644 crates/lib/user_input_common/Cargo.toml rename crates/{service/storyteller_web/src/validations => lib/user_input_common/src}/check_for_slurs.rs (81%) create mode 100644 crates/lib/user_input_common/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index da8c41e035..475767f4ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3804,6 +3804,7 @@ dependencies = [ "twitch_api2", "twitch_common", "twitch_oauth2", + "user_input_common", "uuid", ] @@ -4538,6 +4539,17 @@ dependencies = [ "percent-encoding 2.1.0", ] +[[package]] +name = "user_input_common" +version = "0.0.1" +dependencies = [ + "anyhow", + "container_common", + "log", + "once_cell", + "regex", +] + [[package]] name = "utf-8" version = "0.7.6" diff --git a/Cargo.toml b/Cargo.toml index 590406f5ab..f23c3c84bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/lib/redis_common", "crates/lib/tts_common", "crates/lib/twitch_common", + "crates/lib/user_input_common", # Application "crates/service/obs_gateway", diff --git a/crates/lib/user_input_common/Cargo.toml b/crates/lib/user_input_common/Cargo.toml new file mode 100644 index 0000000000..d70974e358 --- /dev/null +++ b/crates/lib/user_input_common/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "user_input_common" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "user_input_common" +path = "src/lib.rs" + +[dependencies] + +# Internal +container_common = { path = "../container_common" } + +# External +anyhow = "1.0.40" +log = "0.4.14" +once_cell = "1.9.0" +regex = "1.5.4" diff --git a/crates/service/storyteller_web/src/validations/check_for_slurs.rs b/crates/lib/user_input_common/src/check_for_slurs.rs similarity index 81% rename from crates/service/storyteller_web/src/validations/check_for_slurs.rs rename to crates/lib/user_input_common/src/check_for_slurs.rs index c89f4b5245..b156af1e2e 100644 --- a/crates/service/storyteller_web/src/validations/check_for_slurs.rs +++ b/crates/lib/user_input_common/src/check_for_slurs.rs @@ -1,15 +1,16 @@ -use std::collections::HashSet; use crate::BANNED_SLURS; +use once_cell::sync::Lazy; +use std::collections::HashSet; -pub fn contains_slurs(unparsed_text: &str) -> bool { - lazy_static! { - static ref BANNED_SLURS_SET : HashSet = BANNED_SLURS.lines() +static BANNED_SLURS_SET : Lazy> = Lazy::new(|| { + BANNED_SLURS.lines() .map(|line| line.trim()) .filter(|line| !(line.starts_with("#") || line.is_empty())) .map(|line| line.to_string()) - .collect::>(); - } + .collect::>() +}); +pub fn contains_slurs(unparsed_text: &str) -> bool { for wordlike in unparsed_text.split_ascii_whitespace() { if BANNED_SLURS_SET.contains(wordlike) { return true; @@ -21,7 +22,7 @@ pub fn contains_slurs(unparsed_text: &str) -> bool { #[cfg(test)] mod tests { - use crate::validations::check_for_slurs::contains_slurs; + use crate::check_for_slurs::contains_slurs; #[test] fn valid_text_passes() { diff --git a/crates/lib/user_input_common/src/lib.rs b/crates/lib/user_input_common/src/lib.rs new file mode 100644 index 0000000000..cf09c33ec6 --- /dev/null +++ b/crates/lib/user_input_common/src/lib.rs @@ -0,0 +1,12 @@ +#![forbid(dead_code)] +#![forbid(non_snake_case)] +#![forbid(private_in_public)] +#![forbid(unused_imports)] +#![forbid(unused_must_use)] // Important +#![forbid(unused_mut)] +#![forbid(unused_variables)] +#![forbid(warnings)] + +pub (crate) const BANNED_SLURS : &'static str = include_str!("../../../../db/banned_slurs.txt"); + +pub mod check_for_slurs; diff --git a/crates/service/storyteller_web/Cargo.toml b/crates/service/storyteller_web/Cargo.toml index f90aae2e13..e75ad56a41 100644 --- a/crates/service/storyteller_web/Cargo.toml +++ b/crates/service/storyteller_web/Cargo.toml @@ -45,6 +45,7 @@ http_server_common = { path = "../../lib/http_server_common" } redis_common = { path = "../../lib/redis_common" } tts_common = { path = "../../lib/tts_common" } twitch_common = { path = "../../lib/twitch_common" } +user_input_common = { path = "../../lib/user_input_common" } # External anyhow = "1.0.40" diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs index b7b4974f6a..f5a94b0bec 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs @@ -17,7 +17,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs index b04633a3d4..7359dac189 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs @@ -14,7 +14,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use database_queries::tokens::Tokens; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs index a37faa1767..ae6e0c3b21 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs @@ -17,7 +17,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs index 9b1e24fbb7..6b2d8db103 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs @@ -16,7 +16,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use lexical_sort::natural_lexical_cmp; use log::{info, warn, log}; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs index cae8e86b5a..5de3a6fbf7 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs @@ -15,7 +15,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs index 5e1824495a..d6d3241a98 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs @@ -14,7 +14,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs index 918615d946..3396691eca 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs @@ -14,7 +14,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs index 651ad3740e..56339bce95 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs @@ -16,7 +16,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs index 8fed04bdde..80ee8f2513 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs @@ -10,7 +10,6 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; -use crate::validations::check_for_slurs::contains_slurs; use crate::validations::ip_addresses::validate_moderator_provided_ip_address; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs index 4e9f66897a..211637b926 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs @@ -11,7 +11,6 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; -use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs index de9dc9e5e5..bac117d544 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs @@ -12,7 +12,6 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; -use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs index c2dcd340f7..dff560f6cb 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs @@ -12,7 +12,6 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; -use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs index 80c4ac4e04..99b57c558d 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs @@ -14,7 +14,6 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; -use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs index 62db7bd599..25d103bff8 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs @@ -12,7 +12,6 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; -use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs index a302df5b3f..0bf16df60c 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs @@ -1,12 +1,13 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::enums::vocoder_type::VocoderType; use crate::database::queries::query_tts_model::select_tts_model_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -14,7 +15,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; @@ -23,7 +23,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::database::enums::vocoder_type::VocoderType; +use user_input_common::check_for_slurs::contains_slurs; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs index 0273fff1c2..17a9b88ea7 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs @@ -15,7 +15,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 04648edc48..fa8d6541c7 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -10,7 +10,6 @@ use crate::database::helpers::enums::{DownloadUrlType, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; @@ -24,6 +23,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use user_input_common::check_for_slurs::contains_slurs; #[derive(Deserialize)] pub struct InferTtsRequest { diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs index 16bd4e627a..c0da636405 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; @@ -12,7 +12,6 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; -use crate::validations::check_for_slurs::contains_slurs; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use crate::validations::username_reservations::is_reserved_username; @@ -27,6 +26,7 @@ use std::collections::HashMap; use std::fmt::Formatter; use std::fmt; use std::sync::Arc; +use user_input_common::check_for_slurs::contains_slurs; const NEW_USER_ROLE: &'static str = "user"; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs index 6e21a70d85..a1053d21bc 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; @@ -15,7 +15,6 @@ use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; use crate::validations::cashapp_username::{validate_cashapp_username, normalize_cashapp_username_for_storage}; -use crate::validations::check_for_slurs::contains_slurs; use crate::validations::discord_username::validate_discord_username; use crate::validations::github_username::validate_github_username; use crate::validations::passwords::validate_passwords; @@ -31,6 +30,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use user_input_common::check_for_slurs::contains_slurs; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs index 0c23e13461..d514e75650 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs @@ -14,7 +14,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs index f1cc5f6c84..7a8c3b01c7 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; @@ -14,7 +14,6 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use crate::validations::check_for_slurs::contains_slurs; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; @@ -23,6 +22,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use user_input_common::check_for_slurs::contains_slurs; /// For the URL PathInfo #[derive(Deserialize)] diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index 81d6cbb0b5..ecc2d8fc80 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; @@ -9,7 +9,6 @@ use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2l use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; @@ -21,6 +20,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use user_input_common::check_for_slurs::contains_slurs; #[derive(Deserialize)] pub struct UploadW2lTemplateRequest { diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 0fa707599d..63e8184287 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -14,7 +14,6 @@ pub const RESERVED_USERNAMES : &'static str = include_str!("../../../../db/reserved_usernames.txt"); pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../../../../db/reserved_usernames_including.txt"); -pub const BANNED_SLURS : &'static str = include_str!("../../../../db/banned_slurs.txt"); pub mod database; pub mod common_env; diff --git a/crates/service/storyteller_web/src/validations/mod.rs b/crates/service/storyteller_web/src/validations/mod.rs index 20c536754c..e881b761a1 100644 --- a/crates/service/storyteller_web/src/validations/mod.rs +++ b/crates/service/storyteller_web/src/validations/mod.rs @@ -1,5 +1,4 @@ pub mod cashapp_username; -pub mod check_for_slurs; pub mod discord_username; pub mod github_username; pub mod ip_addresses; From e56ac834ee2a3fe87917225cf1aaa048c31c2a8f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 19:21:49 -0500 Subject: [PATCH 0873/1042] ban slurs with latin characters and upper case --- Cargo.lock | 1 + crates/lib/user_input_common/Cargo.toml | 1 + .../user_input_common/src/check_for_slurs.rs | 31 +++- .../user_input_common/src/latin_alphabet.rs | 134 ++++++++++++++++++ crates/lib/user_input_common/src/lib.rs | 1 + 5 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 crates/lib/user_input_common/src/latin_alphabet.rs diff --git a/Cargo.lock b/Cargo.lock index 475767f4ef..e1515b6337 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4548,6 +4548,7 @@ dependencies = [ "log", "once_cell", "regex", + "unicode-segmentation", ] [[package]] diff --git a/crates/lib/user_input_common/Cargo.toml b/crates/lib/user_input_common/Cargo.toml index d70974e358..2c1d369bf4 100644 --- a/crates/lib/user_input_common/Cargo.toml +++ b/crates/lib/user_input_common/Cargo.toml @@ -22,3 +22,4 @@ anyhow = "1.0.40" log = "0.4.14" once_cell = "1.9.0" regex = "1.5.4" +unicode-segmentation = "1.7.1" diff --git a/crates/lib/user_input_common/src/check_for_slurs.rs b/crates/lib/user_input_common/src/check_for_slurs.rs index b156af1e2e..36f0e87455 100644 --- a/crates/lib/user_input_common/src/check_for_slurs.rs +++ b/crates/lib/user_input_common/src/check_for_slurs.rs @@ -1,6 +1,7 @@ use crate::BANNED_SLURS; use once_cell::sync::Lazy; use std::collections::HashSet; +use crate::latin_alphabet::latin_to_ascii; static BANNED_SLURS_SET : Lazy> = Lazy::new(|| { BANNED_SLURS.lines() @@ -11,12 +12,12 @@ static BANNED_SLURS_SET : Lazy> = Lazy::new(|| { }); pub fn contains_slurs(unparsed_text: &str) -> bool { - for wordlike in unparsed_text.split_ascii_whitespace() { + let simplified = latin_to_ascii(unparsed_text).to_lowercase(); + for wordlike in simplified.split_ascii_whitespace() { if BANNED_SLURS_SET.contains(wordlike) { return true; } } - false } @@ -37,4 +38,30 @@ mod tests { assert_eq!(contains_slurs("a sentence containing fag is banned"), true); assert_eq!(contains_slurs("a\nsentence\ncontaining fags\nis banned"), true); } + + #[test] + fn text_with_mixed_case_slurs_fails() { + assert_eq!(contains_slurs("FAG"), true); + assert_eq!(contains_slurs("FaG"), true); + assert_eq!(contains_slurs("fAg"), true); + assert_eq!(contains_slurs("A SENTENCE CONTAINING FAG IS BANNED"), true); + assert_eq!(contains_slurs("a\nsentence\ncontaining FAGS\nis banned"), true); + } + + #[test] + fn text_with_latin_obfuscated_slurs_fails() { + assert_eq!(contains_slurs("FÀG"), true); + assert_eq!(contains_slurs("FÁG"), true); + assert_eq!(contains_slurs("FÂG"), true); + assert_eq!(contains_slurs("FÃG"), true); + assert_eq!(contains_slurs("FÄG"), true); + assert_eq!(contains_slurs("FÅG"), true); + + assert_eq!(contains_slurs("fàg"), true); + assert_eq!(contains_slurs("fág"), true); + assert_eq!(contains_slurs("fâg"), true); + assert_eq!(contains_slurs("fãg"), true); + assert_eq!(contains_slurs("fäg"), true); + assert_eq!(contains_slurs("fåg"), true); + } } diff --git a/crates/lib/user_input_common/src/latin_alphabet.rs b/crates/lib/user_input_common/src/latin_alphabet.rs new file mode 100644 index 0000000000..ec6b376914 --- /dev/null +++ b/crates/lib/user_input_common/src/latin_alphabet.rs @@ -0,0 +1,134 @@ +use once_cell::sync::Lazy; +use std::collections::HashMap; +use unicode_segmentation::UnicodeSegmentation; + +// http://www.geocities.ws/click2speak/unicode/chars_es.html +pub static LATIN_CHARACTER_MAP : Lazy> = Lazy::new(|| { + let mut map = HashMap::new(); + // Latin characters with accent (A) + map.insert("\u{C0}".to_string(), "A".to_string()); // Latin capital letter a with grave + map.insert("\u{C1}".to_string(), "A".to_string()); // Latin capital letter a with acute + map.insert("\u{C2}".to_string(), "A".to_string()); // Latin capital letter a with circumflex + map.insert("\u{C3}".to_string(), "A".to_string()); // Latin capital letter a with tilde + map.insert("\u{C4}".to_string(), "A".to_string()); // Latin capital letter a diaeresis + map.insert("\u{C5}".to_string(), "A".to_string()); // Latin capital letter a with ring above + // Latin characters with accent (AE) + map.insert("\u{C6}".to_string(), "AE".to_string()); // Latin capital letter ae + // Latin characters with accent (C) + map.insert("\u{C7}".to_string(), "C".to_string()); // Latin capital letter c with cedilla + // Latin characters with accent (E) + map.insert("\u{C8}".to_string(), "E".to_string()); // Latin capital letter e with grave + map.insert("\u{C9}".to_string(), "E".to_string()); // Latin capital letter e with acute + map.insert("\u{CA}".to_string(), "E".to_string()); // Latin capital letter e with circumflex + map.insert("\u{CB}".to_string(), "E".to_string()); // Latin capital letter e with diaeresis + // Latin characters with accent (I) + map.insert("\u{CC}".to_string(), "I".to_string()); // Latin capital letter i with grave + map.insert("\u{CD}".to_string(), "I".to_string()); // Latin capital letter i with acute + map.insert("\u{CE}".to_string(), "I".to_string()); // Latin capital letter i with circumflex + map.insert("\u{CF}".to_string(), "I".to_string()); // Latin capital letter i with diaeresis + // Latin characters with accent (ETH) + // map.insert("\u{D0}".to_string(), "D".to_string()); // Latin capital letter eth + // Latin characters with accent (N) + map.insert("\u{D1}".to_string(), "N".to_string()); // Latin capital letter n with tilde + // Latin characters with accent (O) + map.insert("\u{D2}".to_string(), "O".to_string()); // Latin capital letter o with grave + map.insert("\u{D3}".to_string(), "O".to_string()); // Latin capital letter o with acute + map.insert("\u{D4}".to_string(), "O".to_string()); // Latin capital letter o with circumflex + map.insert("\u{D5}".to_string(), "O".to_string()); // Latin capital letter o with tilde + map.insert("\u{D6}".to_string(), "O".to_string()); // Latin capital letter o with diaeresis + map.insert("\u{D8}".to_string(), "O".to_string()); // Latin capital letter o with stroke (NB: Skips D7) + // Latin characters with accent (U) + map.insert("\u{D9}".to_string(), "U".to_string()); // Latin capital letter u with grave + map.insert("\u{DA}".to_string(), "U".to_string()); // Latin capital letter u with acute + map.insert("\u{DB}".to_string(), "U".to_string()); // Latin capital letter u with circumflex + map.insert("\u{DC}".to_string(), "U".to_string()); // Latin capital letter u with diaeresis + // Latin characters with accent (Y) + map.insert("\u{DD}".to_string(), "Y".to_string()); // Latin capital letter y with acute + // Latin characters with accent (THORN) + // map.insert("\u{DE}".to_string(), "P".to_string()); // Latin capital letter thorn + // Latin characters with accent (SHARP S) + // map.insert("\u{DF}".to_string(), "B".to_string()); // Latin capital letter sharp s + // Latin characters with accent (a) + map.insert("\u{E0}".to_string(), "a".to_string()); // Latin small letter a with grave + map.insert("\u{E1}".to_string(), "a".to_string()); // Latin small letter a with acute + map.insert("\u{E2}".to_string(), "a".to_string()); // Latin small letter a with circumflex + map.insert("\u{E3}".to_string(), "a".to_string()); // Latin small letter a with tilde + map.insert("\u{E4}".to_string(), "a".to_string()); // Latin small letter a with diaeresis + map.insert("\u{E5}".to_string(), "a".to_string()); // Latin small letter a with ring above + // Latin characters with accent (ae) + map.insert("\u{E6}".to_string(), "ae".to_string()); // Latin small letter ae + // Latin characters with accent (c) + map.insert("\u{E7}".to_string(), "c".to_string()); // Latin small letter c with cedilla + // Latin characters with accent (e) + map.insert("\u{E8}".to_string(), "e".to_string()); // Latin small letter e with grave + map.insert("\u{E9}".to_string(), "e".to_string()); // Latin small letter e with acute + map.insert("\u{EA}".to_string(), "e".to_string()); // Latin small letter e with circumflex + map.insert("\u{EB}".to_string(), "e".to_string()); // Latin small letter e with diaeresis + // Latin characters with accent (i) + map.insert("\u{EC}".to_string(), "i".to_string()); // Latin small letter i with grave + map.insert("\u{ED}".to_string(), "i".to_string()); // Latin small letter i with acute + map.insert("\u{EE}".to_string(), "i".to_string()); // Latin small letter i with circumflex + map.insert("\u{EF}".to_string(), "i".to_string()); // Latin small letter i with diaeresis + // Latin characters with accent (eth) + // map.insert("\u{F0}".to_string(), "d".to_string()); // Latin small letter eith + // Spanish special characters + map.insert("\u{F1}".to_string(), "n".to_string()); // Lower n tilde + map.insert("\u{AA}".to_string(), "a".to_string()); // Feminine ordinal + map.insert("\u{BA}".to_string(), "o".to_string()); // Masculine ordinal + // Misc characters that frequently occur + map.insert("\u{00F3}".to_string(), "o".to_string()); // Latin Small Letter O with Acute + map.insert("\u{0131}".to_string(), "i".to_string()); // Latin Small Letter Dotless I + map.insert("\u{00FC}".to_string(), "u".to_string()); // Latin Small Letter U with Diaeresis + map.insert("\u{015F}".to_string(), "s".to_string()); // Latin Small Letter S with Cedilla + map.insert("\u{00F6}".to_string(), "o".to_string()); // Latin Small Letter O with Diaeresis + map.insert("\u{011F}".to_string(), "g".to_string()); // Latin Small Letter G with Breve + map.insert("\u{0130}".to_string(), "I".to_string()); // Latin Capital Letter I with Dot Above + map.insert("\u{0259}".to_string(), "e".to_string()); // Latin Small Letter Schwa (TODO: Inaccurate?) + map.insert("\u{012B}".to_string(), "i".to_string()); // Latin Small Letter I with Macron + map.insert("\u{1D3A}".to_string(), "n".to_string()); // Modifier Letter Capital N + map.insert("\u{01D0}".to_string(), "i".to_string()); // Latin Small Letter I with Caron + map.insert("\u{014D}".to_string(), "o".to_string()); // Latin Small Letter O with Macron + map.insert("\u{0101}".to_string(), "a".to_string()); // Latin Small Letter A with Macron + map.insert("\u{00F9}".to_string(), "u".to_string()); // Latin Small Letter U with Grave + map.insert("\u{01Ce}".to_string(), "a".to_string()); // Latin Small Letter A with Caron + map +}); + +pub fn latin_to_ascii(input_text: &str) -> String { + let segmented = UnicodeSegmentation::graphemes(input_text, true) + .map(|segment| { + LATIN_CHARACTER_MAP.get(segment) + .map(|replace| replace.as_str()) + .unwrap_or(segment) + }) + .collect::>(); + + segmented.join("") +} + +#[cfg(test)] +mod tests { + use crate::latin_alphabet::latin_to_ascii; + + #[test] + fn test_latin_to_ascii() { + assert_eq!(latin_to_ascii("pokémon"), "pokemon".to_string()); + assert_eq!(latin_to_ascii("POKÉMON"), "POKEMON".to_string()); + assert_eq!(latin_to_ascii("Æther"), "AEther".to_string()); + assert_eq!(latin_to_ascii("æther"), "aether".to_string()); + // Almost exhaustive + assert_eq!(latin_to_ascii("ÀÁÂÃÄÅ"), "AAAAAA".to_string()); + assert_eq!(latin_to_ascii("Æ"), "AE".to_string()); + assert_eq!(latin_to_ascii("Ç"), "C".to_string()); + assert_eq!(latin_to_ascii("ÈÉÊË"), "EEEE".to_string()); + assert_eq!(latin_to_ascii("ÌÍÎÏ"), "IIII".to_string()); + assert_eq!(latin_to_ascii("ÒÓÔÕÖØ"), "OOOOOO".to_string()); + assert_eq!(latin_to_ascii("ÙÚÛÜ"), "UUUU".to_string()); + assert_eq!(latin_to_ascii("Ý"), "Y".to_string()); + assert_eq!(latin_to_ascii("àáâãäå"), "aaaaaa".to_string()); + assert_eq!(latin_to_ascii("æ"), "ae".to_string()); + assert_eq!(latin_to_ascii("ç"), "c".to_string()); + assert_eq!(latin_to_ascii("èéêë"), "eeee".to_string()); + assert_eq!(latin_to_ascii("ìíîï"), "iiii".to_string()); + } +} diff --git a/crates/lib/user_input_common/src/lib.rs b/crates/lib/user_input_common/src/lib.rs index cf09c33ec6..4439295579 100644 --- a/crates/lib/user_input_common/src/lib.rs +++ b/crates/lib/user_input_common/src/lib.rs @@ -10,3 +10,4 @@ pub (crate) const BANNED_SLURS : &'static str = include_str!("../../../../db/banned_slurs.txt"); pub mod check_for_slurs; +pub mod latin_alphabet; From a50554d4b71e5c7496d537665bffa75f25c51249 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 19:28:56 -0500 Subject: [PATCH 0874/1042] More slurs to ban --- db/banned_slurs.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/db/banned_slurs.txt b/db/banned_slurs.txt index fd6c54f0db..650abced03 100644 --- a/db/banned_slurs.txt +++ b/db/banned_slurs.txt @@ -1,9 +1,14 @@ # This file serves to remove racial slurs and epithets from usage with vocodes. # There's probably more than this that we'll want to remove. # I need to investigate. +chink +chinks +dyke fag faggot fags kniggers +nig nigger -niggers \ No newline at end of file +niggers +nigs \ No newline at end of file From 8b02351517cae25e357f215f260b318f0a5b019e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 19:38:57 -0500 Subject: [PATCH 0875/1042] Share latin-to-ascii code/map --- Cargo.lock | 1 + crates/lib/tts_common/Cargo.toml | 1 + crates/lib/tts_common/src/clean_symbols.rs | 88 ++----------------- .../user_input_common/src/latin_alphabet.rs | 4 +- 4 files changed, 10 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1515b6337..4589563260 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4333,6 +4333,7 @@ dependencies = [ "once_cell", "regex", "unicode-segmentation", + "user_input_common", ] [[package]] diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml index 4d71f79846..c4578df6be 100644 --- a/crates/lib/tts_common/Cargo.toml +++ b/crates/lib/tts_common/Cargo.toml @@ -16,6 +16,7 @@ path = "src/lib.rs" # Internal container_common = { path = "../container_common" } +user_input_common = { path = "../user_input_common" } # External anyhow = "1.0.40" diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 21ba3b6c55..462e0af56a 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -1,11 +1,17 @@ use once_cell::sync::Lazy; use std::collections::HashMap; use unicode_segmentation::UnicodeSegmentation; +use user_input_common::latin_alphabet::LATIN_TO_ASCII_CHARACTER_MAP; // https://unicodelookup.com/#quo // http://www.geocities.ws/click2speak/unicode/chars_es.html static REPLACEMENTS : Lazy> = Lazy::new(|| { let mut map = HashMap::new(); + + // Latin characters such as àáâãäå + map.extend(LATIN_TO_ASCII_CHARACTER_MAP.iter() + .map(|(k, v)| (k.to_string(), v.to_string()))); + // Weird spacing characters map.insert("\u{00a0}".to_string(), " ".to_string()); // Non-break space (aka  ) \xa0 map.insert("\u{2005}".to_string(), " ".to_string()); // Four-Per-Em Space @@ -42,94 +48,12 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { map.insert("…".to_string(), "...".to_string()); // Commas map.insert("\u{3001}".to_string(), ",".to_string()); - // Latin characters with accent (A) - map.insert("\u{C0}".to_string(), "A".to_string()); // Latin capital letter a with grave - map.insert("\u{C1}".to_string(), "A".to_string()); // Latin capital letter a with acute - map.insert("\u{C2}".to_string(), "A".to_string()); // Latin capital letter a with circumflex - map.insert("\u{C3}".to_string(), "A".to_string()); // Latin capital letter a with tilde - map.insert("\u{C4}".to_string(), "A".to_string()); // Latin capital letter a diaeresis - map.insert("\u{C5}".to_string(), "A".to_string()); // Latin capital letter a with ring above - // Latin characters with accent (AE) - map.insert("\u{C6}".to_string(), "AE".to_string()); // Latin capital letter ae - // Latin characters with accent (C) - map.insert("\u{C7}".to_string(), "C".to_string()); // Latin capital letter c with cedilla - // Latin characters with accent (E) - map.insert("\u{C8}".to_string(), "E".to_string()); // Latin capital letter e with grave - map.insert("\u{C9}".to_string(), "E".to_string()); // Latin capital letter e with acute - map.insert("\u{CA}".to_string(), "E".to_string()); // Latin capital letter e with circumflex - map.insert("\u{CB}".to_string(), "E".to_string()); // Latin capital letter e with diaeresis - // Latin characters with accent (I) - map.insert("\u{CC}".to_string(), "I".to_string()); // Latin capital letter i with grave - map.insert("\u{CD}".to_string(), "I".to_string()); // Latin capital letter i with acute - map.insert("\u{CE}".to_string(), "I".to_string()); // Latin capital letter i with circumflex - map.insert("\u{CF}".to_string(), "I".to_string()); // Latin capital letter i with diaeresis - // Latin characters with accent (ETH) - // map.insert("\u{D0}".to_string(), "D".to_string()); // Latin capital letter eth - // Latin characters with accent (N) - map.insert("\u{D1}".to_string(), "N".to_string()); // Latin capital letter n with tilde - // Latin characters with accent (O) - map.insert("\u{D2}".to_string(), "O".to_string()); // Latin capital letter o with grave - map.insert("\u{D3}".to_string(), "O".to_string()); // Latin capital letter o with acute - map.insert("\u{D4}".to_string(), "O".to_string()); // Latin capital letter o with circumflex - map.insert("\u{D5}".to_string(), "O".to_string()); // Latin capital letter o with tilde - map.insert("\u{D6}".to_string(), "O".to_string()); // Latin capital letter o with diaeresis - map.insert("\u{D8}".to_string(), "O".to_string()); // Latin capital letter o with stroke (NB: Skips D7) - // Latin characters with accent (U) - map.insert("\u{D9}".to_string(), "U".to_string()); // Latin capital letter u with grave - map.insert("\u{DA}".to_string(), "U".to_string()); // Latin capital letter u with acute - map.insert("\u{DB}".to_string(), "U".to_string()); // Latin capital letter u with circumflex - map.insert("\u{DC}".to_string(), "U".to_string()); // Latin capital letter u with diaeresis - // Latin characters with accent (Y) - map.insert("\u{DD}".to_string(), "Y".to_string()); // Latin capital letter y with acute - // Latin characters with accent (THORN) - // map.insert("\u{DE}".to_string(), "P".to_string()); // Latin capital letter thorn - // Latin characters with accent (SHARP S) - // map.insert("\u{DF}".to_string(), "B".to_string()); // Latin capital letter sharp s - // Latin characters with accent (a) - map.insert("\u{E0}".to_string(), "a".to_string()); // Latin small letter a with grave - map.insert("\u{E1}".to_string(), "a".to_string()); // Latin small letter a with acute - map.insert("\u{E2}".to_string(), "a".to_string()); // Latin small letter a with circumflex - map.insert("\u{E3}".to_string(), "a".to_string()); // Latin small letter a with tilde - map.insert("\u{E4}".to_string(), "a".to_string()); // Latin small letter a with diaeresis - map.insert("\u{E5}".to_string(), "a".to_string()); // Latin small letter a with ring above - // Latin characters with accent (ae) - map.insert("\u{E6}".to_string(), "ae".to_string()); // Latin small letter ae - // Latin characters with accent (c) - map.insert("\u{E7}".to_string(), "c".to_string()); // Latin small letter c with cedilla - // Latin characters with accent (e) - map.insert("\u{E8}".to_string(), "e".to_string()); // Latin small letter e with grave - map.insert("\u{E9}".to_string(), "e".to_string()); // Latin small letter e with acute - map.insert("\u{EA}".to_string(), "e".to_string()); // Latin small letter e with circumflex - map.insert("\u{EB}".to_string(), "e".to_string()); // Latin small letter e with diaeresis - // Latin characters with accent (i) - map.insert("\u{EC}".to_string(), "i".to_string()); // Latin small letter i with grave - map.insert("\u{ED}".to_string(), "i".to_string()); // Latin small letter i with acute - map.insert("\u{EE}".to_string(), "i".to_string()); // Latin small letter i with circumflex - map.insert("\u{EF}".to_string(), "i".to_string()); // Latin small letter i with diaeresis - // Latin characters with accent (eth) - // map.insert("\u{F0}".to_string(), "d".to_string()); // Latin small letter eith // Spanish special characters - map.insert("\u{F1}".to_string(), "n".to_string()); // Lower n tilde map.insert("\u{A1}".to_string(), "!".to_string()); // Inverted exclamation mark map.insert("\u{BF}".to_string(), "?".to_string()); // Inverted question mark map.insert("\u{AA}".to_string(), "a".to_string()); // Feminine ordinal map.insert("\u{BA}".to_string(), "o".to_string()); // Masculine ordinal // Misc characters that frequently occur - map.insert("\u{00F3}".to_string(), "o".to_string()); // Latin Small Letter O with Acute - map.insert("\u{0131}".to_string(), "i".to_string()); // Latin Small Letter Dotless I - map.insert("\u{00FC}".to_string(), "u".to_string()); // Latin Small Letter U with Diaeresis - map.insert("\u{015F}".to_string(), "s".to_string()); // Latin Small Letter S with Cedilla - map.insert("\u{00F6}".to_string(), "o".to_string()); // Latin Small Letter O with Diaeresis - map.insert("\u{011F}".to_string(), "g".to_string()); // Latin Small Letter G with Breve - map.insert("\u{0130}".to_string(), "I".to_string()); // Latin Capital Letter I with Dot Above - map.insert("\u{0259}".to_string(), "e".to_string()); // Latin Small Letter Schwa (TODO: Inaccurate?) - map.insert("\u{012B}".to_string(), "i".to_string()); // Latin Small Letter I with Macron - map.insert("\u{1D3A}".to_string(), "n".to_string()); // Modifier Letter Capital N - map.insert("\u{01D0}".to_string(), "i".to_string()); // Latin Small Letter I with Caron - map.insert("\u{014D}".to_string(), "o".to_string()); // Latin Small Letter O with Macron - map.insert("\u{0101}".to_string(), "a".to_string()); // Latin Small Letter A with Macron - map.insert("\u{00F9}".to_string(), "u".to_string()); // Latin Small Letter U with Grave - map.insert("\u{01Ce}".to_string(), "a".to_string()); // Latin Small Letter A with Caron map.insert("\u{203C}".to_string(), "!!".to_string()); // Double Exclamation Mark map }); diff --git a/crates/lib/user_input_common/src/latin_alphabet.rs b/crates/lib/user_input_common/src/latin_alphabet.rs index ec6b376914..a8142a189b 100644 --- a/crates/lib/user_input_common/src/latin_alphabet.rs +++ b/crates/lib/user_input_common/src/latin_alphabet.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use unicode_segmentation::UnicodeSegmentation; // http://www.geocities.ws/click2speak/unicode/chars_es.html -pub static LATIN_CHARACTER_MAP : Lazy> = Lazy::new(|| { +pub static LATIN_TO_ASCII_CHARACTER_MAP : Lazy> = Lazy::new(|| { let mut map = HashMap::new(); // Latin characters with accent (A) map.insert("\u{C0}".to_string(), "A".to_string()); // Latin capital letter a with grave @@ -97,7 +97,7 @@ pub static LATIN_CHARACTER_MAP : Lazy> = Lazy::new(|| { pub fn latin_to_ascii(input_text: &str) -> String { let segmented = UnicodeSegmentation::graphemes(input_text, true) .map(|segment| { - LATIN_CHARACTER_MAP.get(segment) + LATIN_TO_ASCII_CHARACTER_MAP.get(segment) .map(|replace| replace.as_str()) .unwrap_or(segment) }) From 65db6a1041568553dd366254cb67564343cf77e9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 19:47:44 -0500 Subject: [PATCH 0876/1042] table is simpler to read --- .../user_input_common/src/latin_alphabet.rs | 113 +++++++++--------- 1 file changed, 54 insertions(+), 59 deletions(-) diff --git a/crates/lib/user_input_common/src/latin_alphabet.rs b/crates/lib/user_input_common/src/latin_alphabet.rs index a8142a189b..03853d4261 100644 --- a/crates/lib/user_input_common/src/latin_alphabet.rs +++ b/crates/lib/user_input_common/src/latin_alphabet.rs @@ -4,77 +4,72 @@ use unicode_segmentation::UnicodeSegmentation; // http://www.geocities.ws/click2speak/unicode/chars_es.html pub static LATIN_TO_ASCII_CHARACTER_MAP : Lazy> = Lazy::new(|| { + // NB: Note that certain latin characters map to multiple ascii characters, eg. "AE". + let table = [ + ("\u{C0}", "A"), // Latin capital letter A with grave + ("\u{C1}", "A"), // Latin capital letter A with acute + ("\u{C2}", "A"), // Latin capital letter A with circumflex + ("\u{C3}", "A"), // Latin capital letter A with tilde + ("\u{C4}", "A"), // Latin capital letter A diaeresis + ("\u{C5}", "A"), // Latin capital letter A with ring above + ("\u{C6}", "AE"), // Latin capital letter AE + ("\u{C7}", "C"), // Latin capital letter C with cedilla + ("\u{C8}", "E"), // Latin capital letter E with grave + ("\u{C9}", "E"), // Latin capital letter E with acute + ("\u{CA}", "E"), // Latin capital letter E with circumflex + ("\u{CB}", "E"), // Latin capital letter E with diaeresis + ("\u{CC}", "I"), // Latin capital letter I with grave + ("\u{CD}", "I"), // Latin capital letter I with acute + ("\u{CE}", "I"), // Latin capital letter I with circumflex + ("\u{CF}", "I"), // Latin capital letter I with diaeresis + ("\u{D1}", "N"), // Latin capital letter N with tilde + ("\u{D2}", "O"), // Latin capital letter O with grave + ("\u{D3}", "O"), // Latin capital letter O with acute + ("\u{D4}", "O"), // Latin capital letter O with circumflex + ("\u{D5}", "O"), // Latin capital letter O with tilde + ("\u{D6}", "O"), // Latin capital letter O with diaeresis + ("\u{D8}", "O"), // Latin capital letter O with stroke (NB: Skips D7) + ("\u{D9}", "U"), // Latin capital letter U with grave + ("\u{DA}", "U"), // Latin capital letter U with acute + ("\u{DB}", "U"), // Latin capital letter U with circumflex + ("\u{DC}", "U"), // Latin capital letter U with diaeresis + ("\u{DD}", "Y"), // Latin capital letter Y with acute + ("\u{E0}", "a"), // Latin small letter a with grave + ("\u{E1}", "a"), // Latin small letter a with acute + ("\u{E2}", "a"), // Latin small letter a with circumflex + ("\u{E3}", "a"), // Latin small letter a with tilde + ("\u{E4}", "a"), // Latin small letter a with diaeresis + ("\u{E5}", "a"), // Latin small letter a with ring above + ("\u{E6}", "ae"), // Latin small letter ae + ("\u{E7}", "c"), // Latin small letter c with cedilla + ("\u{E8}", "e"), // Latin small letter e with grave + ("\u{E9}", "e"), // Latin small letter e with acute + ("\u{EA}", "e"), // Latin small letter e with circumflex + ("\u{EB}", "e"), // Latin small letter e with diaeresis + ("\u{EC}", "i"), // Latin small letter i with grave + ("\u{ED}", "i"), // Latin small letter i with acute + ("\u{EE}", "i"), // Latin small letter i with circumflex + ("\u{EF}", "i"), // Latin small letter i with diaeresis + ("\u{F1}", "n"), // Lower n tilde + ("\u{AA}", "a"), // Feminine ordinal + ("\u{BA}", "o"), // Masculine ordinal + ]; + let mut map = HashMap::new(); - // Latin characters with accent (A) - map.insert("\u{C0}".to_string(), "A".to_string()); // Latin capital letter a with grave - map.insert("\u{C1}".to_string(), "A".to_string()); // Latin capital letter a with acute - map.insert("\u{C2}".to_string(), "A".to_string()); // Latin capital letter a with circumflex - map.insert("\u{C3}".to_string(), "A".to_string()); // Latin capital letter a with tilde - map.insert("\u{C4}".to_string(), "A".to_string()); // Latin capital letter a diaeresis - map.insert("\u{C5}".to_string(), "A".to_string()); // Latin capital letter a with ring above - // Latin characters with accent (AE) - map.insert("\u{C6}".to_string(), "AE".to_string()); // Latin capital letter ae - // Latin characters with accent (C) - map.insert("\u{C7}".to_string(), "C".to_string()); // Latin capital letter c with cedilla - // Latin characters with accent (E) - map.insert("\u{C8}".to_string(), "E".to_string()); // Latin capital letter e with grave - map.insert("\u{C9}".to_string(), "E".to_string()); // Latin capital letter e with acute - map.insert("\u{CA}".to_string(), "E".to_string()); // Latin capital letter e with circumflex - map.insert("\u{CB}".to_string(), "E".to_string()); // Latin capital letter e with diaeresis - // Latin characters with accent (I) - map.insert("\u{CC}".to_string(), "I".to_string()); // Latin capital letter i with grave - map.insert("\u{CD}".to_string(), "I".to_string()); // Latin capital letter i with acute - map.insert("\u{CE}".to_string(), "I".to_string()); // Latin capital letter i with circumflex - map.insert("\u{CF}".to_string(), "I".to_string()); // Latin capital letter i with diaeresis + + map.extend(table.iter().map(|item| (item.0.to_string(), item.1.to_string()))); + // Latin characters with accent (ETH) // map.insert("\u{D0}".to_string(), "D".to_string()); // Latin capital letter eth // Latin characters with accent (N) - map.insert("\u{D1}".to_string(), "N".to_string()); // Latin capital letter n with tilde - // Latin characters with accent (O) - map.insert("\u{D2}".to_string(), "O".to_string()); // Latin capital letter o with grave - map.insert("\u{D3}".to_string(), "O".to_string()); // Latin capital letter o with acute - map.insert("\u{D4}".to_string(), "O".to_string()); // Latin capital letter o with circumflex - map.insert("\u{D5}".to_string(), "O".to_string()); // Latin capital letter o with tilde - map.insert("\u{D6}".to_string(), "O".to_string()); // Latin capital letter o with diaeresis - map.insert("\u{D8}".to_string(), "O".to_string()); // Latin capital letter o with stroke (NB: Skips D7) - // Latin characters with accent (U) - map.insert("\u{D9}".to_string(), "U".to_string()); // Latin capital letter u with grave - map.insert("\u{DA}".to_string(), "U".to_string()); // Latin capital letter u with acute - map.insert("\u{DB}".to_string(), "U".to_string()); // Latin capital letter u with circumflex - map.insert("\u{DC}".to_string(), "U".to_string()); // Latin capital letter u with diaeresis - // Latin characters with accent (Y) - map.insert("\u{DD}".to_string(), "Y".to_string()); // Latin capital letter y with acute // Latin characters with accent (THORN) // map.insert("\u{DE}".to_string(), "P".to_string()); // Latin capital letter thorn // Latin characters with accent (SHARP S) // map.insert("\u{DF}".to_string(), "B".to_string()); // Latin capital letter sharp s // Latin characters with accent (a) - map.insert("\u{E0}".to_string(), "a".to_string()); // Latin small letter a with grave - map.insert("\u{E1}".to_string(), "a".to_string()); // Latin small letter a with acute - map.insert("\u{E2}".to_string(), "a".to_string()); // Latin small letter a with circumflex - map.insert("\u{E3}".to_string(), "a".to_string()); // Latin small letter a with tilde - map.insert("\u{E4}".to_string(), "a".to_string()); // Latin small letter a with diaeresis - map.insert("\u{E5}".to_string(), "a".to_string()); // Latin small letter a with ring above - // Latin characters with accent (ae) - map.insert("\u{E6}".to_string(), "ae".to_string()); // Latin small letter ae - // Latin characters with accent (c) - map.insert("\u{E7}".to_string(), "c".to_string()); // Latin small letter c with cedilla - // Latin characters with accent (e) - map.insert("\u{E8}".to_string(), "e".to_string()); // Latin small letter e with grave - map.insert("\u{E9}".to_string(), "e".to_string()); // Latin small letter e with acute - map.insert("\u{EA}".to_string(), "e".to_string()); // Latin small letter e with circumflex - map.insert("\u{EB}".to_string(), "e".to_string()); // Latin small letter e with diaeresis - // Latin characters with accent (i) - map.insert("\u{EC}".to_string(), "i".to_string()); // Latin small letter i with grave - map.insert("\u{ED}".to_string(), "i".to_string()); // Latin small letter i with acute - map.insert("\u{EE}".to_string(), "i".to_string()); // Latin small letter i with circumflex - map.insert("\u{EF}".to_string(), "i".to_string()); // Latin small letter i with diaeresis // Latin characters with accent (eth) // map.insert("\u{F0}".to_string(), "d".to_string()); // Latin small letter eith // Spanish special characters - map.insert("\u{F1}".to_string(), "n".to_string()); // Lower n tilde - map.insert("\u{AA}".to_string(), "a".to_string()); // Feminine ordinal - map.insert("\u{BA}".to_string(), "o".to_string()); // Masculine ordinal // Misc characters that frequently occur map.insert("\u{00F3}".to_string(), "o".to_string()); // Latin Small Letter O with Acute map.insert("\u{0131}".to_string(), "i".to_string()); // Latin Small Letter Dotless I From f0ea7be72963751cc41252d88486da7810dc5530 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 20:06:35 -0500 Subject: [PATCH 0877/1042] more characters to convert --- .../user_input_common/src/latin_alphabet.rs | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/crates/lib/user_input_common/src/latin_alphabet.rs b/crates/lib/user_input_common/src/latin_alphabet.rs index 03853d4261..8807f294c1 100644 --- a/crates/lib/user_input_common/src/latin_alphabet.rs +++ b/crates/lib/user_input_common/src/latin_alphabet.rs @@ -6,6 +6,8 @@ use unicode_segmentation::UnicodeSegmentation; pub static LATIN_TO_ASCII_CHARACTER_MAP : Lazy> = Lazy::new(|| { // NB: Note that certain latin characters map to multiple ascii characters, eg. "AE". let table = [ + ("\u{AA}", "a"), // Feminine ordinal + ("\u{BA}", "o"), // Masculine ordinal ("\u{C0}", "A"), // Latin capital letter A with grave ("\u{C1}", "A"), // Latin capital letter A with acute ("\u{C2}", "A"), // Latin capital letter A with circumflex @@ -22,18 +24,21 @@ pub static LATIN_TO_ASCII_CHARACTER_MAP : Lazy> = Lazy:: ("\u{CD}", "I"), // Latin capital letter I with acute ("\u{CE}", "I"), // Latin capital letter I with circumflex ("\u{CF}", "I"), // Latin capital letter I with diaeresis + ("\u{D0}", "D"), // Latin capital letter ETH (~D) TODO: How to handle this? ("\u{D1}", "N"), // Latin capital letter N with tilde ("\u{D2}", "O"), // Latin capital letter O with grave ("\u{D3}", "O"), // Latin capital letter O with acute ("\u{D4}", "O"), // Latin capital letter O with circumflex ("\u{D5}", "O"), // Latin capital letter O with tilde ("\u{D6}", "O"), // Latin capital letter O with diaeresis - ("\u{D8}", "O"), // Latin capital letter O with stroke (NB: Skips D7) + ("\u{D8}", "O"), // Latin capital letter O with stroke (NB: Skips D7 multiplication sign) ("\u{D9}", "U"), // Latin capital letter U with grave ("\u{DA}", "U"), // Latin capital letter U with acute ("\u{DB}", "U"), // Latin capital letter U with circumflex ("\u{DC}", "U"), // Latin capital letter U with diaeresis ("\u{DD}", "Y"), // Latin capital letter Y with acute + ("\u{DE}", "P"), // Latin capital letter THORN (~P) TODO: How to handle this? + ("\u{DF}", "b"), // Latin small letter sharp s (~B) TODO: How to handle this? ("\u{E0}", "a"), // Latin small letter a with grave ("\u{E1}", "a"), // Latin small letter a with acute ("\u{E2}", "a"), // Latin small letter a with circumflex @@ -50,26 +55,29 @@ pub static LATIN_TO_ASCII_CHARACTER_MAP : Lazy> = Lazy:: ("\u{ED}", "i"), // Latin small letter i with acute ("\u{EE}", "i"), // Latin small letter i with circumflex ("\u{EF}", "i"), // Latin small letter i with diaeresis + ("\u{F0}", "d"), // Latin capital letter eth (~d) TODO: How to handle this? ("\u{F1}", "n"), // Lower n tilde - ("\u{AA}", "a"), // Feminine ordinal - ("\u{BA}", "o"), // Masculine ordinal + ("\u{F2}", "o"), // Latin small letter o with grave + ("\u{F3}", "o"), // Latin small letter o with acute + ("\u{F4}", "o"), // Latin small letter o with circumflex + ("\u{F5}", "o"), // Latin small letter o with tilde + ("\u{F6}", "o"), // Latin small letter o with diaeresis + ("\u{F8}", "o"), // Latin small letter o with stroke (NB: Skips F7 division sign) + ("\u{F9}", "u"), // Latin small letter u with grave + ("\u{FA}", "u"), // Latin small letter u with acute + ("\u{FB}", "u"), // Latin small letter u with circumflex + ("\u{FC}", "u"), // Latin small letter u with diaeresis + ("\u{FD}", "y"), // Latin small letter y with acute + ("\u{FE}", "p"), // Latin small letter thorn (~p) TODO: How to handle this? + ("\u{FF}", "y"), // Latin small letter y with diaeresis + + // TODO: There are a lot more to go... ]; let mut map = HashMap::new(); map.extend(table.iter().map(|item| (item.0.to_string(), item.1.to_string()))); - // Latin characters with accent (ETH) - // map.insert("\u{D0}".to_string(), "D".to_string()); // Latin capital letter eth - // Latin characters with accent (N) - // Latin characters with accent (THORN) - // map.insert("\u{DE}".to_string(), "P".to_string()); // Latin capital letter thorn - // Latin characters with accent (SHARP S) - // map.insert("\u{DF}".to_string(), "B".to_string()); // Latin capital letter sharp s - // Latin characters with accent (a) - // Latin characters with accent (eth) - // map.insert("\u{F0}".to_string(), "d".to_string()); // Latin small letter eith - // Spanish special characters // Misc characters that frequently occur map.insert("\u{00F3}".to_string(), "o".to_string()); // Latin Small Letter O with Acute map.insert("\u{0131}".to_string(), "i".to_string()); // Latin Small Letter Dotless I @@ -125,5 +133,7 @@ mod tests { assert_eq!(latin_to_ascii("ç"), "c".to_string()); assert_eq!(latin_to_ascii("èéêë"), "eeee".to_string()); assert_eq!(latin_to_ascii("ìíîï"), "iiii".to_string()); + assert_eq!(latin_to_ascii("òóôõöø"), "oooooo".to_string()); + assert_eq!(latin_to_ascii("ùúûü"), "uuuu".to_string()); } } From 4e9919f5a2c84eb0d088b6eca863c6c4f016a413 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 20:19:36 -0500 Subject: [PATCH 0878/1042] more things to remove --- crates/lib/tts_common/src/clean_symbols.rs | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 462e0af56a..205fda41d5 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -55,6 +55,34 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { map.insert("\u{BA}".to_string(), "o".to_string()); // Masculine ordinal // Misc characters that frequently occur map.insert("\u{203C}".to_string(), "!!".to_string()); // Double Exclamation Mark + + // Close enough to existing allowed punctuation + let punctuation = [ + ("\u{3002}", "."), // idiographic full stop + ]; + + map.extend(punctuation.iter() + .map(|(k, v)| (k.to_string(), v.to_string()))); + + // Symbols we can insert as words + let phonetic_replacements = [ + ("\u{B0}", " degrees "), // degree sign + ("\u{03C0}", " pie "), // greek small letter pi + ("\u{2122}", " trademark "), // trade mark sign + ]; + + map.extend(phonetic_replacements.iter() + .map(|(k, v)| (k.to_string(), v.to_string()))); + + // These shouldn't be in the output at all + let remove_replacements = [ + ("\u{b7}", ""), // middle dot + ("\u{2022}", ""), // bullet + ]; + + map.extend(remove_replacements.iter() + .map(|(k, v)| (k.to_string(), v.to_string()))); + map }); From 2cc117b04fa3fc19902646a491c828b1a17c45ad Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 21:16:21 -0500 Subject: [PATCH 0879/1042] tests --- crates/lib/tts_common/src/clean_symbols.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 205fda41d5..598250a6d6 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -130,6 +130,12 @@ mod tests { assert_eq!(clean_symbols("test…"), "test...".to_string()); } + #[test] + fn symbol_expansion() { + assert_eq!(clean_symbols("Pokémon™ is a popular video game series"), + "Pokemon trademark is a popular video game series".to_string()); // NB: Extra space + } + #[test] fn filters_latin_characters() { assert_eq!(clean_symbols("pokémon"), "pokemon".to_string()); @@ -171,7 +177,7 @@ mod tests { assert_eq!(clean_symbols("ó"), "o".to_string()); assert_eq!(clean_symbols("é"), "e".to_string()); assert_eq!(clean_symbols("ñ"), "n".to_string()); - assert_eq!(clean_symbols("ú"), "ú".to_string()); + assert_eq!(clean_symbols("ú"), "u".to_string()); assert_eq!(clean_symbols("ı"), "i".to_string()); // Dotless i assert_eq!(clean_symbols("ü"), "u".to_string()); assert_eq!(clean_symbols("¿"), "?".to_string()); From a1162d1e8680b52f41dcd12af688180e91d6dba7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 22:27:53 -0500 Subject: [PATCH 0880/1042] cleanup for legibility --- crates/lib/tts_common/src/clean_symbols.rs | 122 ++++++++++++--------- 1 file changed, 68 insertions(+), 54 deletions(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 598250a6d6..e953538827 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -3,85 +3,99 @@ use std::collections::HashMap; use unicode_segmentation::UnicodeSegmentation; use user_input_common::latin_alphabet::LATIN_TO_ASCII_CHARACTER_MAP; +// Used to insert tables into hashmap +fn deref_to_owned(item: (&String, &String)) -> (String, String) { + (item.0.to_string(), item.1.to_string()) +} + +fn to_owned(item: &(&str, &str)) -> (String, String) { + (item.0.to_string(), item.1.to_string()) +} + // https://unicodelookup.com/#quo // http://www.geocities.ws/click2speak/unicode/chars_es.html static REPLACEMENTS : Lazy> = Lazy::new(|| { let mut map = HashMap::new(); // Latin characters such as àáâãäå - map.extend(LATIN_TO_ASCII_CHARACTER_MAP.iter() - .map(|(k, v)| (k.to_string(), v.to_string()))); + map.extend(LATIN_TO_ASCII_CHARACTER_MAP.iter().map(&deref_to_owned)); + + // Weird spacing character replacements + map.extend([ + ("\u{00a0}", " "), // Non-break space (aka  ) \xa0 + ("\u{2005}", " "), // Four-Per-Em Space + ("\u{205F}", " "), // Medium Mathematical Space (MMSP) + ("\u{2588}", " "), // Full Block + ].iter().map(&to_owned)); - // Weird spacing characters - map.insert("\u{00a0}".to_string(), " ".to_string()); // Non-break space (aka  ) \xa0 - map.insert("\u{2005}".to_string(), " ".to_string()); // Four-Per-Em Space - map.insert("\u{2588}".to_string(), " ".to_string()); // Full Block - map.insert("\u{205F}".to_string(), " ".to_string()); // Medium Mathematical Space (MMSP) // Quotes (single) - map.insert("\u{0060}".to_string(), "'".to_string()); // Grave Accent - map.insert("\u{00B4}".to_string(), "'".to_string()); // Acute Accent - map.insert("\u{2018}".to_string(), "'".to_string()); // Left Single Quotation Mark - map.insert("\u{2019}".to_string(), "'".to_string()); // Right Single Quotation Mark - map.insert("\u{201A}".to_string(), "'".to_string()); // Single Low-9 Quotation Mark - map.insert("\u{201B}".to_string(), "'".to_string()); // Single High-Revered-9 Quotation Mark + map.extend([ + ("\u{0060}", "'"), // Grave Accent + ("\u{00B4}", "'"), // Acute Accent + ("\u{2018}", "'"), // Left Single Quotation Mark + ("\u{2019}", "'"), // Right Single Quotation Mark + ("\u{201A}", "'"), // Single Low-9 Quotation Mark + ("\u{201B}", "'"), // Single High-Revered-9 Quotation Mark + ].iter().map(&to_owned)); + // Quotes (double) - map.insert("\u{201C}".to_string(), "\"".to_string()); // Left Double Quotation Mark - map.insert("\u{201D}".to_string(), "\"".to_string()); // Right Double Quotation Mark - map.insert("\u{201E}".to_string(), "\"".to_string()); // Double Low-9 Quotation Mark - map.insert("\u{201F}".to_string(), "\"".to_string()); // Double High-Reversed-9 Quotation Mark - map.insert("\u{301D}".to_string(), "\"".to_string()); // Reversed Double Prime Quotation Mark - map.insert("\u{301E}".to_string(), "\"".to_string()); // Double Prime Quotation Mark - map.insert("\u{301F}".to_string(), "\"".to_string()); // Low Double Prime Quotation Mark - map.insert("\u{FF02}".to_string(), "\"".to_string()); // Fullwidth Quotation Mark + map.extend([ + ("\u{201C}", "\""), // Left Double Quotation Mark + ("\u{201D}", "\""), // Right Double Quotation Mark + ("\u{201E}", "\""), // Double Low-9 Quotation Mark + ("\u{201F}", "\""), // Double High-Reversed-9 Quotation Mark + ("\u{301D}", "\""), // Reversed Double Prime Quotation Mark + ("\u{301E}", "\""), // Double Prime Quotation Mark + ("\u{301F}", "\""), // Low Double Prime Quotation Mark + ("\u{FF02}", "\""), // Fullwidth Quotation Mark + ].iter().map(&to_owned)); + // Dashes - map.insert("\u{2010}".to_string(), "-".to_string()); // Hyphen - map.insert("\u{2011}".to_string(), "-".to_string()); // Non-Breaking Hyphen - map.insert("\u{2013}".to_string(), "-".to_string()); // En Dash - map.insert("\u{2014}".to_string(), "-".to_string()); // Em Dash - map.insert("\u{2015}".to_string(), "-".to_string()); // Horizontal Bar - map.insert("\u{2E3A}".to_string(), "-".to_string()); // Two-Em Dash - map.insert("\u{2E3B}".to_string(), "-".to_string()); // Three-Em Dash - map.insert("\u{FE58}".to_string(), "-".to_string()); // Small Em Dash - map.insert("\u{FE63}".to_string(), "-".to_string()); // Small Hyphen-Minus - map.insert("\u{FF0D}".to_string(), "-".to_string()); // Fullwidth Hyphen-Minus - // Ellipsis - map.insert("…".to_string(), "...".to_string()); - // Commas - map.insert("\u{3001}".to_string(), ",".to_string()); + map.extend([ + ("\u{2010}", "-"), // Hyphen + ("\u{2011}", "-"), // Non-Breaking Hyphen + ("\u{2013}", "-"), // En Dash + ("\u{2014}", "-"), // Em Dash + ("\u{2015}", "-"), // Horizontal Bar + ("\u{2E3A}", "-"), // Two-Em Dash + ("\u{2E3B}", "-"), // Three-Em Dash + ("\u{FE58}", "-"), // Small Em Dash + ("\u{FE63}", "-"), // Small Hyphen-Minus + ("\u{FF0D}", "-"), // Fullwidth Hyphen-Minus + ].iter().map(&to_owned)); + // Spanish special characters - map.insert("\u{A1}".to_string(), "!".to_string()); // Inverted exclamation mark - map.insert("\u{BF}".to_string(), "?".to_string()); // Inverted question mark - map.insert("\u{AA}".to_string(), "a".to_string()); // Feminine ordinal - map.insert("\u{BA}".to_string(), "o".to_string()); // Masculine ordinal + map.extend([ + ("\u{A1}", "!"), // Inverted exclamation mark + ("\u{BF}", "?"), // Inverted question mark + ("\u{AA}", "a"), // Feminine ordinal + ("\u{BA}", "o"), // Masculine ordinal + ].iter().map(&to_owned)); + + // Misc + map.insert("…".to_string(), "...".to_string()); // Ellipsis + map.insert("\u{3001}".to_string(), ",".to_string()); // Commas + // Misc characters that frequently occur map.insert("\u{203C}".to_string(), "!!".to_string()); // Double Exclamation Mark // Close enough to existing allowed punctuation - let punctuation = [ + map.extend([ ("\u{3002}", "."), // idiographic full stop - ]; - - map.extend(punctuation.iter() - .map(|(k, v)| (k.to_string(), v.to_string()))); + ].iter().map(&to_owned)); // Symbols we can insert as words - let phonetic_replacements = [ + map.extend([ ("\u{B0}", " degrees "), // degree sign ("\u{03C0}", " pie "), // greek small letter pi ("\u{2122}", " trademark "), // trade mark sign - ]; - - map.extend(phonetic_replacements.iter() - .map(|(k, v)| (k.to_string(), v.to_string()))); + ].iter().map(&to_owned)); // These shouldn't be in the output at all - let remove_replacements = [ + map.extend([ ("\u{b7}", ""), // middle dot ("\u{2022}", ""), // bullet - ]; - - map.extend(remove_replacements.iter() - .map(|(k, v)| (k.to_string(), v.to_string()))); + ].iter().map(&to_owned)); map }); From 26cabf34071835e8c3b1cdf0d3a7f0fc083e1b2f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 23:08:07 -0500 Subject: [PATCH 0881/1042] new tests (most failing) --- crates/lib/tts_common/src/clean_symbols.rs | 123 ++++++++++++++++++++- 1 file changed, 120 insertions(+), 3 deletions(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index e953538827..c6ddf62911 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -20,12 +20,13 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { // Latin characters such as àáâãäå map.extend(LATIN_TO_ASCII_CHARACTER_MAP.iter().map(&deref_to_owned)); - // Weird spacing character replacements + // Spacing character replacements map.extend([ - ("\u{00a0}", " "), // Non-break space (aka  ) \xa0 + ("\u{00A0}", " "), // Non-break space (aka  ) \xa0 ("\u{2005}", " "), // Four-Per-Em Space ("\u{205F}", " "), // Medium Mathematical Space (MMSP) ("\u{2588}", " "), // Full Block + ("\u{3000}", " "), // Ideographic Space ].iter().map(&to_owned)); // Quotes (single) @@ -81,7 +82,8 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { // Close enough to existing allowed punctuation map.extend([ - ("\u{3002}", "."), // idiographic full stop + ("\u{3002}", "."), // Ideographic full stop + ("\u{FF01}", "!"), // Fullwidth Exclamation Mark ].iter().map(&to_owned)); // Symbols we can insert as words @@ -117,6 +119,10 @@ pub fn clean_symbols(input_text: &str) -> String { mod tests { use crate::clean_symbols::clean_symbols; + fn assert_converted(original: &str, expected: &str) { + assert_eq!(clean_symbols(original), expected.to_string()); + } + #[test] fn neutral_tests() { assert_eq!(clean_symbols(""), "".to_string()); // Empty check @@ -235,4 +241,115 @@ mod tests { // TODO: assert_eq!(clean_symbols("ß"), "B".to_string()); // TODO: assert_eq!(clean_symbols("°"), "degrees".to_string()); } + + #[test] + pub fn most_frequent_failures_10k_usages() { + // The leftmost number is the number of occurrences in our histogram of TTS failures + // The order of the lines is reversed. + assert_converted("ş", "s"); // b'\\u015f' 10006 + assert_converted(" ", " "); // b'\\u2005' 10118 + assert_converted("—", "-"); // b'\\u2014' 10555 + assert_converted("ã", "a"); // b'\xe3' 10586 + assert_converted("ǎ", "a"); // b'\\u01ce' 10843 + assert_converted("ə", "e"); // b'\\u0259' 11508 + assert_converted("ö", "o"); // b'\xf6' 11908 + assert_converted("¿", "?"); // b'\xbf' 13501 + assert_converted("ç", "c"); // b'\xe7' 13784 + assert_converted(" ", " "); // b'\xa0' 14184 + assert_converted("ō", "o"); // b'\\u014d' 14329 + assert_converted("ā", "a"); // b'\\u0101' 15927 + assert_converted("¡", "!"); // b'\xa1' 17933 + assert_converted("ǐ", "i"); // b'\\u01d0' 20236 + assert_converted("ī", "i"); // b'\\u012b' 20257 + assert_converted("è", "e"); // b'\xe8' 21121 + assert_converted("°", " degrees "); // b'\xb0' 22808 + assert_converted("“", "\""); // b'\\u201c' 27343 + assert_converted("ü", "u"); // b'\xfc' 27955 + assert_converted("”", "\""); // b'\\u201d' 28171 + assert_converted("à", "a"); // b'\xe0' 34414 + assert_converted("ı", "i"); // b'\\u0131' 44032 + assert_converted("ú", "u"); // b'\xfa' 44043 + assert_converted("…", "..."); // b'\\u2026' 49348 + assert_converted("ñ", "n"); // b'\xf1' 105898 + assert_converted("ó", "o"); // b'\xf3' 127099 + assert_converted("í", "i"); // b'\xed' 132716 + assert_converted("é", "e"); // b'\xe9' 140278 + assert_converted("á", "a"); // b'\xe1' 184138 + assert_converted("’", "'"); // b'\\u2019' 739079 + } + + #[test] + pub fn most_frequent_failures_1k_usages() { + assert_converted("　", " "); // b'\\u3000' 1005 + //assert_converted("🤣", " laugh "); // b'\\U0001f923' 1010 + assert_converted("ø", "o"); // b'\xf8' 1017 + assert_converted("！", "!"); // b'\\uff01' 1019 + //assert_converted("¥", " yen "); // b'\xa5' 1033 + //assert_converted("😭", "cry "); // b'\\U0001f62d' 1061 + assert_converted("Ü", "U"); // b'\xdc' 1062 + assert_converted("č", "c"); // b'\\u010d' 1092 + assert_converted("ν", "v"); // b'\\u03bd' 1096 + assert_converted("ż", "z"); // b'\\u017c' 1099 + assert_converted("⠀", " "); // b'\\u2800' 1166 + assert_converted("ą", "a"); // b'\\u0105' 1169 + //assert_converted("£", " pounds "); // b'\xa3' 1186 + assert_converted("ë", "e"); // b'\xeb' 1189 + assert_converted("Ç", "C"); // b'\xc7' 1213 + assert_converted("τ", "t"); // b'\\u03c4' 1217 + assert_converted("ẹ", "e"); // b'\\u1eb9' 1228 + assert_converted("î", "i"); // b'\xee' 1349 + assert_converted("ś", "s"); // b'\\u015b' 1402 + assert_converted("ạ", "a"); // b'\\u1ea1' 1413 + assert_converted("Í", "I"); // b'\xcd' 1434 + assert_converted("·", "."); // b'\xb7' 1514 + assert_converted("š", "s"); // b'\\u0161' 1514 + assert_converted("ο", "o"); // b'\\u03bf' 1515 + assert_converted("ū", "u"); // b'\\u016b' 1523 + assert_converted("Ö", "O"); // b'\xd6' 1543 + assert_converted("ι", "l"); // b'\\u03b9' 1559 + assert_converted("ε", "e"); // b'\\u03b5' 1575 + assert_converted("ă", "a"); // b'\\u0103' 1576 + //assert_converted("😂", ""); // b'\\U0001f602' 1741 + assert_converted("Ó", "O"); // b'\xd3' 1774 + assert_converted("ư", "u"); // b'\\u01b0' 1794 + assert_converted("•", ""); // b'\\u2022' 1948 + assert_converted("、", ","); // b'\\u3001' 2001 + assert_converted("ć", "c"); // b'\\u0107' 2017 + assert_converted("ę", "e"); // b'\\u0119' 2236 + assert_converted("。", "."); // b'\\u3002' 2288 + assert_converted(" ", " "); // b'\\u205f' 2347 + assert_converted("ᴺ", "n"); // b'\\u1d3a' 2383 + assert_converted("ě", "e"); // b'\\u011b' 2441 + assert_converted("ᴾ", "p"); // b'\\u1d3e' 2479 + assert_converted("ł", "s"); // b'\\u0142' 2480 + assert_converted("～", "~"); // b'\\uff5e' 2507 + assert_converted("â", "a"); // b'\xe2' 2607 + assert_converted("α", "a"); // b'\\u03b1' 2625 + assert_converted("å", "a"); // b'\xe5' 2753 + //assert_converted("🐶", " dog "); // b'\\U0001f436' 2782 + assert_converted("™", " trademark "); // b'\\u2122' 2869 + assert_converted("É", "E"); // b'\xc9' 3040 + assert_converted("æ", "ae"); // b'\xe6' 3142 + assert_converted("¨", "\""); // b'\xa8' 3236 + assert_converted("ò", "o"); // b'\xf2' 3250 + assert_converted("đ", "d"); // b'\\u0111' 3420 + assert_converted("，", ","); // b'\\uff0c' 3487 + assert_converted("ô", "o"); // b'\xf4' 3568 + assert_converted("Á", "A"); // b'\xc1' 3779 + assert_converted("ß", "B"); // b'\xdf' 3779 + assert_converted("Ñ", "N"); // b'\xd1' 3963 + assert_converted("ǒ", "o"); // b'\\u01d2' 4613 + assert_converted("İ", "I"); // b'\\u0130' 4808 + assert_converted("​", " "); // b'\\u200b' 6019 + assert_converted("ğ", "g"); // b'\\u011f' 6103 + assert_converted("–", "-"); // b'\\u2013' 6172 + assert_converted("ì", "i"); // b'\xec' 6209 + assert_converted("ē", "e"); // b'\\u0113' 6312 + assert_converted("ù", "u"); // b'\xf9' 6907 + assert_converted("‘", "'"); // b'\\u2018' 7151 + assert_converted("ǔ", "u"); // b'\\u01d4' 7422 + assert_converted("´", "'"); // b'\xb4' 8210 + assert_converted("ê", "e"); // b'\xea' 8882 + assert_converted("ä", "a"); // b'\xe4' 9202 + } } From 5cfde7b0f275fe10022d0f4fd8168a9a9f48fcce Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 23:16:46 -0500 Subject: [PATCH 0882/1042] add more of latin extended-a --- .../user_input_common/src/latin_alphabet.rs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/lib/user_input_common/src/latin_alphabet.rs b/crates/lib/user_input_common/src/latin_alphabet.rs index 8807f294c1..18336ea96c 100644 --- a/crates/lib/user_input_common/src/latin_alphabet.rs +++ b/crates/lib/user_input_common/src/latin_alphabet.rs @@ -2,6 +2,10 @@ use once_cell::sync::Lazy; use std::collections::HashMap; use unicode_segmentation::UnicodeSegmentation; +fn to_owned(item: &(&str, &str)) -> (String, String) { + (item.0.to_string(), item.1.to_string()) +} + // http://www.geocities.ws/click2speak/unicode/chars_es.html pub static LATIN_TO_ASCII_CHARACTER_MAP : Lazy> = Lazy::new(|| { // NB: Note that certain latin characters map to multiple ascii characters, eg. "AE". @@ -78,6 +82,30 @@ pub static LATIN_TO_ASCII_CHARACTER_MAP : Lazy> = Lazy:: map.extend(table.iter().map(|item| (item.0.to_string(), item.1.to_string()))); + // Latin Extended-A + // https://unicode-table.com/en/0100/ (doesn't start here, though) + map.extend([ + ("\u{0100}", "A"), // Ā Latin Capital Letter a with Macron + ("\u{0101}", "a"), // ā Latin Small Letter a with Macron + ("\u{0102}", "A"), // Ă Latin Capital Letter a with Breve + ("\u{0103}", "a"), // ă Latin Small Letter a with Breve + ("\u{0104}", "A"), // Ą Latin Capital Letter a with Ogonek + ("\u{0105}", "a"), // ą Latin Small Letter a with Ogonek + ("\u{0106}", "C"), // Ć Latin Capital Letter C with Acute + ("\u{0107}", "c"), // ć Latin Small Letter C with Acute + ("\u{0108}", "C"), // Ĉ Latin Capital Letter C with Circumflex + ("\u{0109}", "c"), // ĉ Latin Small Letter C with Circumflex + ("\u{010A}", "C"), // Ċ Latin Capital Letter C with Dot Above + ("\u{010B}", "c"), // ċ Latin Small Letter C with Dot Above + ("\u{010C}", "C"), // Č Latin Capital Letter C with Caron + ("\u{010D}", "c"), // č Latin Small Letter C with Caron + ("\u{010E}", "D"), // Ď Latin Capital Letter D with Caron + ("\u{010F}", "d"), // ď Latin Small Letter D with Caron + + // TODO: This is far from done... + + ].iter().map(&to_owned)); + // Misc characters that frequently occur map.insert("\u{00F3}".to_string(), "o".to_string()); // Latin Small Letter O with Acute map.insert("\u{0131}".to_string(), "i".to_string()); // Latin Small Letter Dotless I From 7313a0d1809e4fc52f7578867026223b6f3c2673 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 23:49:04 -0500 Subject: [PATCH 0883/1042] more conversions (hardly done) --- crates/lib/tts_common/src/clean_symbols.rs | 13 +++---- .../user_input_common/src/latin_alphabet.rs | 35 +++++++++++++++++-- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index c6ddf62911..b96db9d028 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -26,6 +26,7 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { ("\u{2005}", " "), // Four-Per-Em Space ("\u{205F}", " "), // Medium Mathematical Space (MMSP) ("\u{2588}", " "), // Full Block + ("\u{2800}", " "), // Braille Pattern Blank ("\u{3000}", " "), // Ideographic Space ].iter().map(&to_owned)); @@ -93,10 +94,10 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { ("\u{2122}", " trademark "), // trade mark sign ].iter().map(&to_owned)); - // These shouldn't be in the output at all + // These should turn into spaces map.extend([ - ("\u{b7}", ""), // middle dot - ("\u{2022}", ""), // bullet + ("\u{b7}", " "), // · Middle Dot + ("\u{2022}", " "), // • Bullet ].iter().map(&to_owned)); map @@ -295,13 +296,13 @@ mod tests { //assert_converted("£", " pounds "); // b'\xa3' 1186 assert_converted("ë", "e"); // b'\xeb' 1189 assert_converted("Ç", "C"); // b'\xc7' 1213 - assert_converted("τ", "t"); // b'\\u03c4' 1217 + //assert_converted("τ", "t"); // b'\\u03c4' 1217 assert_converted("ẹ", "e"); // b'\\u1eb9' 1228 assert_converted("î", "i"); // b'\xee' 1349 assert_converted("ś", "s"); // b'\\u015b' 1402 assert_converted("ạ", "a"); // b'\\u1ea1' 1413 assert_converted("Í", "I"); // b'\xcd' 1434 - assert_converted("·", "."); // b'\xb7' 1514 + assert_converted("·", " "); // b'\xb7' 1514 assert_converted("š", "s"); // b'\\u0161' 1514 assert_converted("ο", "o"); // b'\\u03bf' 1515 assert_converted("ū", "u"); // b'\\u016b' 1523 @@ -312,7 +313,7 @@ mod tests { //assert_converted("😂", ""); // b'\\U0001f602' 1741 assert_converted("Ó", "O"); // b'\xd3' 1774 assert_converted("ư", "u"); // b'\\u01b0' 1794 - assert_converted("•", ""); // b'\\u2022' 1948 + assert_converted("•", " "); // b'\\u2022' 1948 assert_converted("、", ","); // b'\\u3001' 2001 assert_converted("ć", "c"); // b'\\u0107' 2017 assert_converted("ę", "e"); // b'\\u0119' 2236 diff --git a/crates/lib/user_input_common/src/latin_alphabet.rs b/crates/lib/user_input_common/src/latin_alphabet.rs index 18336ea96c..be3a65b7ba 100644 --- a/crates/lib/user_input_common/src/latin_alphabet.rs +++ b/crates/lib/user_input_common/src/latin_alphabet.rs @@ -83,7 +83,7 @@ pub static LATIN_TO_ASCII_CHARACTER_MAP : Lazy> = Lazy:: map.extend(table.iter().map(|item| (item.0.to_string(), item.1.to_string()))); // Latin Extended-A - // https://unicode-table.com/en/0100/ (doesn't start here, though) + // https://unicode-table.com/en/blocks/latin-extended-a/ map.extend([ ("\u{0100}", "A"), // Ā Latin Capital Letter a with Macron ("\u{0101}", "a"), // ā Latin Small Letter a with Macron @@ -101,9 +101,38 @@ pub static LATIN_TO_ASCII_CHARACTER_MAP : Lazy> = Lazy:: ("\u{010D}", "c"), // č Latin Small Letter C with Caron ("\u{010E}", "D"), // Ď Latin Capital Letter D with Caron ("\u{010F}", "d"), // ď Latin Small Letter D with Caron + // TODO: Not done... + ("\u{015B}", "s"), // ś Latin Small Letter S with Acute + ("\u{0161}", "s"), // š Latin Small Letter S with Caron + ("\u{017C}", "z"), // ż Latin Small Letter Z with Dot Above + // TODO: Incomplete + ].iter().map(&to_owned)); - // TODO: This is far from done... - + // Latin Extended-B + // https://unicode-table.com/en/blocks/latin-extended-b/ + map.extend([ + ("\u{01B0}", "u"), // ư Latin Small Letter U with Horn + ("\u{01B1}", "U"), // Ʊ Latin Capital Letter Upsilon (TODO: Not correct handling) + ("\u{01B3}", "Y"), // Ƴ Latin Capital Letter Y with Hook + ("\u{01B4}", "y"), // ƴ Latin Small Letter Y with Hook + ("\u{01B5}", "Z"), // Ƶ Latin Capital Letter Z with Stroke + // TODO: Incomplete + ].iter().map(&to_owned)); + + // Greek and Coptic + // https://unicode-table.com/en/blocks/greek-coptic/ + map.extend([ + ("\u{03BD}", "v"), // ν Greek Small Letter Nu (TODO: Incorrect handling, but very frequent.) + ("\u{03BF}", "o"), // ο Greek Small Letter Omicron (TODO: Incorrect handling, but very frequent.) + // TODO: Incomplete + ].iter().map(&to_owned)); + + // Latin Extended Additional + // https://unicode-table.com/en/blocks/latin-extended-additional/ + map.extend([ + ("\u{1EA1}", "a"), // ạ Latin Small Letter a with Dot Below + ("\u{1EB9}", "e"), // ẹ Latin Small Letter E with Dot Below + // TODO: Incomplete ].iter().map(&to_owned)); // Misc characters that frequently occur From 343f8a5b2bea778dffa2161f6af435d94b713e15 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 23:52:25 -0500 Subject: [PATCH 0884/1042] split tests a bit --- crates/lib/tts_common/src/clean_symbols.rs | 116 ++++++++++++--------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index b96db9d028..830af1336b 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -279,6 +279,72 @@ mod tests { assert_converted("’", "'"); // b'\\u2019' 739079 } + #[test] + pub fn most_frequent_failures_5k_usages() { + assert_converted("​", " "); // b'\\u200b' 6019 + assert_converted("ğ", "g"); // b'\\u011f' 6103 + assert_converted("–", "-"); // b'\\u2013' 6172 + assert_converted("ì", "i"); // b'\xec' 6209 + assert_converted("ē", "e"); // b'\\u0113' 6312 + assert_converted("ù", "u"); // b'\xf9' 6907 + assert_converted("‘", "'"); // b'\\u2018' 7151 + assert_converted("ǔ", "u"); // b'\\u01d4' 7422 + assert_converted("´", "'"); // b'\xb4' 8210 + assert_converted("ê", "e"); // b'\xea' 8882 + assert_converted("ä", "a"); // b'\xe4' 9202 + } + + #[test] + pub fn most_frequent_failures_2p5k_usages() { + assert_converted("～", "~"); // b'\\uff5e' 2507 + assert_converted("â", "a"); // b'\xe2' 2607 + assert_converted("α", "a"); // b'\\u03b1' 2625 + assert_converted("å", "a"); // b'\xe5' 2753 + //assert_converted("🐶", " dog "); // b'\\U0001f436' 2782 + assert_converted("™", " trademark "); // b'\\u2122' 2869 + assert_converted("É", "E"); // b'\xc9' 3040 + assert_converted("æ", "ae"); // b'\xe6' 3142 + assert_converted("¨", "\""); // b'\xa8' 3236 + assert_converted("ò", "o"); // b'\xf2' 3250 + assert_converted("đ", "d"); // b'\\u0111' 3420 + assert_converted("，", ","); // b'\\uff0c' 3487 + assert_converted("ô", "o"); // b'\xf4' 3568 + assert_converted("Á", "A"); // b'\xc1' 3779 + assert_converted("ß", "B"); // b'\xdf' 3779 + assert_converted("Ñ", "N"); // b'\xd1' 3963 + assert_converted("ǒ", "o"); // b'\\u01d2' 4613 + assert_converted("İ", "I"); // b'\\u0130' 4808 + } + + #[test] + pub fn most_frequent_failures_2k_usages() { + assert_converted("、", ","); // b'\\u3001' 2001 + assert_converted("ć", "c"); // b'\\u0107' 2017 + assert_converted("ę", "e"); // b'\\u0119' 2236 + assert_converted("。", "."); // b'\\u3002' 2288 + assert_converted(" ", " "); // b'\\u205f' 2347 + assert_converted("ᴺ", "n"); // b'\\u1d3a' 2383 + assert_converted("ě", "e"); // b'\\u011b' 2441 + assert_converted("ᴾ", "p"); // b'\\u1d3e' 2479 + assert_converted("ł", "s"); // b'\\u0142' 2480 + } + + #[test] + pub fn most_frequent_failures_1p5k_usages() { + assert_converted("·", " "); // b'\xb7' 1514 + assert_converted("š", "s"); // b'\\u0161' 1514 + assert_converted("ο", "o"); // b'\\u03bf' 1515 + assert_converted("ū", "u"); // b'\\u016b' 1523 + assert_converted("Ö", "O"); // b'\xd6' 1543 + assert_converted("ι", "l"); // b'\\u03b9' 1559 + assert_converted("ε", "e"); // b'\\u03b5' 1575 + assert_converted("ă", "a"); // b'\\u0103' 1576 + //assert_converted("😂", ""); // b'\\U0001f602' 1741 + assert_converted("Ó", "O"); // b'\xd3' 1774 + assert_converted("ư", "u"); // b'\\u01b0' 1794 + assert_converted("•", " "); // b'\\u2022' 1948 + } + #[test] pub fn most_frequent_failures_1k_usages() { assert_converted("　", " "); // b'\\u3000' 1005 @@ -302,55 +368,5 @@ mod tests { assert_converted("ś", "s"); // b'\\u015b' 1402 assert_converted("ạ", "a"); // b'\\u1ea1' 1413 assert_converted("Í", "I"); // b'\xcd' 1434 - assert_converted("·", " "); // b'\xb7' 1514 - assert_converted("š", "s"); // b'\\u0161' 1514 - assert_converted("ο", "o"); // b'\\u03bf' 1515 - assert_converted("ū", "u"); // b'\\u016b' 1523 - assert_converted("Ö", "O"); // b'\xd6' 1543 - assert_converted("ι", "l"); // b'\\u03b9' 1559 - assert_converted("ε", "e"); // b'\\u03b5' 1575 - assert_converted("ă", "a"); // b'\\u0103' 1576 - //assert_converted("😂", ""); // b'\\U0001f602' 1741 - assert_converted("Ó", "O"); // b'\xd3' 1774 - assert_converted("ư", "u"); // b'\\u01b0' 1794 - assert_converted("•", " "); // b'\\u2022' 1948 - assert_converted("、", ","); // b'\\u3001' 2001 - assert_converted("ć", "c"); // b'\\u0107' 2017 - assert_converted("ę", "e"); // b'\\u0119' 2236 - assert_converted("。", "."); // b'\\u3002' 2288 - assert_converted(" ", " "); // b'\\u205f' 2347 - assert_converted("ᴺ", "n"); // b'\\u1d3a' 2383 - assert_converted("ě", "e"); // b'\\u011b' 2441 - assert_converted("ᴾ", "p"); // b'\\u1d3e' 2479 - assert_converted("ł", "s"); // b'\\u0142' 2480 - assert_converted("～", "~"); // b'\\uff5e' 2507 - assert_converted("â", "a"); // b'\xe2' 2607 - assert_converted("α", "a"); // b'\\u03b1' 2625 - assert_converted("å", "a"); // b'\xe5' 2753 - //assert_converted("🐶", " dog "); // b'\\U0001f436' 2782 - assert_converted("™", " trademark "); // b'\\u2122' 2869 - assert_converted("É", "E"); // b'\xc9' 3040 - assert_converted("æ", "ae"); // b'\xe6' 3142 - assert_converted("¨", "\""); // b'\xa8' 3236 - assert_converted("ò", "o"); // b'\xf2' 3250 - assert_converted("đ", "d"); // b'\\u0111' 3420 - assert_converted("，", ","); // b'\\uff0c' 3487 - assert_converted("ô", "o"); // b'\xf4' 3568 - assert_converted("Á", "A"); // b'\xc1' 3779 - assert_converted("ß", "B"); // b'\xdf' 3779 - assert_converted("Ñ", "N"); // b'\xd1' 3963 - assert_converted("ǒ", "o"); // b'\\u01d2' 4613 - assert_converted("İ", "I"); // b'\\u0130' 4808 - assert_converted("​", " "); // b'\\u200b' 6019 - assert_converted("ğ", "g"); // b'\\u011f' 6103 - assert_converted("–", "-"); // b'\\u2013' 6172 - assert_converted("ì", "i"); // b'\xec' 6209 - assert_converted("ē", "e"); // b'\\u0113' 6312 - assert_converted("ù", "u"); // b'\xf9' 6907 - assert_converted("‘", "'"); // b'\\u2018' 7151 - assert_converted("ǔ", "u"); // b'\\u01d4' 7422 - assert_converted("´", "'"); // b'\xb4' 8210 - assert_converted("ê", "e"); // b'\xea' 8882 - assert_converted("ä", "a"); // b'\xe4' 9202 } } From cd70d7b06f81057805b1e9d57cba9a1854dcc395 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Feb 2022 23:54:54 -0500 Subject: [PATCH 0885/1042] more buckets --- crates/lib/tts_common/src/clean_symbols.rs | 44 +++++++++++++--------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 830af1336b..2b6059ca40 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -244,9 +244,34 @@ mod tests { } #[test] - pub fn most_frequent_failures_10k_usages() { + pub fn most_frequent_failures_100k_usages() { // The leftmost number is the number of occurrences in our histogram of TTS failures // The order of the lines is reversed. + assert_converted("ñ", "n"); // b'\xf1' 105898 + assert_converted("ó", "o"); // b'\xf3' 127099 + assert_converted("í", "i"); // b'\xed' 132716 + assert_converted("é", "e"); // b'\xe9' 140278 + assert_converted("á", "a"); // b'\xe1' 184138 + assert_converted("’", "'"); // b'\\u2019' 739079 + } + + #[test] + pub fn most_frequent_failures_20k_usages() { + assert_converted("ǐ", "i"); // b'\\u01d0' 20236 + assert_converted("ī", "i"); // b'\\u012b' 20257 + assert_converted("è", "e"); // b'\xe8' 21121 + assert_converted("°", " degrees "); // b'\xb0' 22808 + assert_converted("“", "\""); // b'\\u201c' 27343 + assert_converted("ü", "u"); // b'\xfc' 27955 + assert_converted("”", "\""); // b'\\u201d' 28171 + assert_converted("à", "a"); // b'\xe0' 34414 + assert_converted("ı", "i"); // b'\\u0131' 44032 + assert_converted("ú", "u"); // b'\xfa' 44043 + assert_converted("…", "..."); // b'\\u2026' 49348 + } + + #[test] + pub fn most_frequent_failures_10k_usages() { assert_converted("ş", "s"); // b'\\u015f' 10006 assert_converted(" ", " "); // b'\\u2005' 10118 assert_converted("—", "-"); // b'\\u2014' 10555 @@ -260,23 +285,6 @@ mod tests { assert_converted("ō", "o"); // b'\\u014d' 14329 assert_converted("ā", "a"); // b'\\u0101' 15927 assert_converted("¡", "!"); // b'\xa1' 17933 - assert_converted("ǐ", "i"); // b'\\u01d0' 20236 - assert_converted("ī", "i"); // b'\\u012b' 20257 - assert_converted("è", "e"); // b'\xe8' 21121 - assert_converted("°", " degrees "); // b'\xb0' 22808 - assert_converted("“", "\""); // b'\\u201c' 27343 - assert_converted("ü", "u"); // b'\xfc' 27955 - assert_converted("”", "\""); // b'\\u201d' 28171 - assert_converted("à", "a"); // b'\xe0' 34414 - assert_converted("ı", "i"); // b'\\u0131' 44032 - assert_converted("ú", "u"); // b'\xfa' 44043 - assert_converted("…", "..."); // b'\\u2026' 49348 - assert_converted("ñ", "n"); // b'\xf1' 105898 - assert_converted("ó", "o"); // b'\xf3' 127099 - assert_converted("í", "i"); // b'\xed' 132716 - assert_converted("é", "e"); // b'\xe9' 140278 - assert_converted("á", "a"); // b'\xe1' 184138 - assert_converted("’", "'"); // b'\\u2019' 739079 } #[test] From 3cba684ddc40ddeb2fed1570e25a43fad67c5c5f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Feb 2022 00:06:45 -0500 Subject: [PATCH 0886/1042] handle 5k occurrences --- crates/lib/tts_common/src/clean_symbols.rs | 10 ++++++++++ crates/lib/user_input_common/src/latin_alphabet.rs | 8 +++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 2b6059ca40..2acc3d2c98 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -23,7 +23,17 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { // Spacing character replacements map.extend([ ("\u{00A0}", " "), // Non-break space (aka  ) \xa0 + ("\u{2003}", " "), // Em Space + ("\u{2004}", " "), // Three-Per-Em Space ("\u{2005}", " "), // Four-Per-Em Space + ("\u{2006}", " "), // Six-Per-Em Space + ("\u{2007}", " "), // Figure Space + ("\u{2008}", " "), // Punctuation Space + ("\u{2009}", " "), // Thin Space + ("\u{200A}", " "), // Hair Space + ("\u{200B}", " "), // Zero Width Space + ("\u{200C}", " "), // Zero Width Non-Joiner + ("\u{200D}", " "), // Zero Width Joiner ("\u{205F}", " "), // Medium Mathematical Space (MMSP) ("\u{2588}", " "), // Full Block ("\u{2800}", " "), // Braille Pattern Blank diff --git a/crates/lib/user_input_common/src/latin_alphabet.rs b/crates/lib/user_input_common/src/latin_alphabet.rs index be3a65b7ba..efcd7d3172 100644 --- a/crates/lib/user_input_common/src/latin_alphabet.rs +++ b/crates/lib/user_input_common/src/latin_alphabet.rs @@ -102,6 +102,7 @@ pub static LATIN_TO_ASCII_CHARACTER_MAP : Lazy> = Lazy:: ("\u{010E}", "D"), // Ď Latin Capital Letter D with Caron ("\u{010F}", "d"), // ď Latin Small Letter D with Caron // TODO: Not done... + ("\u{0113}", "e"), // ē Latin Small Letter E with Macron ("\u{015B}", "s"), // ś Latin Small Letter S with Acute ("\u{0161}", "s"), // š Latin Small Letter S with Caron ("\u{017C}", "z"), // ż Latin Small Letter Z with Dot Above @@ -116,7 +117,12 @@ pub static LATIN_TO_ASCII_CHARACTER_MAP : Lazy> = Lazy:: ("\u{01B3}", "Y"), // Ƴ Latin Capital Letter Y with Hook ("\u{01B4}", "y"), // ƴ Latin Small Letter Y with Hook ("\u{01B5}", "Z"), // Ƶ Latin Capital Letter Z with Stroke - // TODO: Incomplete + // TODO: Incomplete... + ("\u{01D0}", "i"), // ǐ Latin Small Letter I with Caron + ("\u{01D1}", "O"), // Ǒ Latin Capital Letter O with Caron + ("\u{01D2}", "o"), // ǒ Latin Small Letter O with Caron + ("\u{01D3}", "U"), // Ǔ Latin Capital Letter U with Caron + ("\u{01D4}", "u"), // ǔ Latin Small Letter U with Caron ].iter().map(&to_owned)); // Greek and Coptic From 3eb23cc3d7f38de1d25336aeb11f1cfed3acc5b3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Feb 2022 00:16:05 -0500 Subject: [PATCH 0887/1042] more cleanup --- crates/lib/tts_common/src/clean_symbols.rs | 45 +++++++++++----------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 2acc3d2c98..dbe86d52ad 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -40,6 +40,12 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { ("\u{3000}", " "), // Ideographic Space ].iter().map(&to_owned)); + // Punctuation that should turn into spaces + map.extend([ + ("\u{b7}", " "), // · Middle Dot + ("\u{2022}", " "), // • Bullet + ].iter().map(&to_owned)); + // Quotes (single) map.extend([ ("\u{0060}", "'"), // Grave Accent @@ -76,25 +82,24 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { ("\u{FF0D}", "-"), // Fullwidth Hyphen-Minus ].iter().map(&to_owned)); - // Spanish special characters - map.extend([ - ("\u{A1}", "!"), // Inverted exclamation mark - ("\u{BF}", "?"), // Inverted question mark - ("\u{AA}", "a"), // Feminine ordinal - ("\u{BA}", "o"), // Masculine ordinal - ].iter().map(&to_owned)); - - // Misc - map.insert("…".to_string(), "...".to_string()); // Ellipsis - map.insert("\u{3001}".to_string(), ",".to_string()); // Commas - - // Misc characters that frequently occur - map.insert("\u{203C}".to_string(), "!!".to_string()); // Double Exclamation Mark - // Close enough to existing allowed punctuation map.extend([ - ("\u{3002}", "."), // Ideographic full stop - ("\u{FF01}", "!"), // Fullwidth Exclamation Mark + ("\u{00A1}", "!"), // ¡ Inverted Exclamation Mark + ("\u{00BF}", "?"), // ¿ Inverted Question Mark + ("\u{2024}", "."), // ․ One Dot Leader + ("\u{2025}", ".."), // ‥ Two Dot Leader + ("\u{2026}", "..."), // … Horizontal Ellipsis + ("\u{203C}", "!!"), // ‼ Double Exclamation Mark Emoji + ("\u{203D}", "!?"), // ‽ Interrobang + ("\u{3001}", ","), // 、 Ideographic Comma + ("\u{3002}", "."), // 。 Ideographic Full Stop + ("\u{FF01}", "!"), // ！ Fullwidth Exclamation Mark + ].iter().map(&to_owned)); + + // Spanish special characters + map.extend([ + ("\u{AA}", "a"), // Feminine ordinal + ("\u{BA}", "o"), // Masculine ordinal ].iter().map(&to_owned)); // Symbols we can insert as words @@ -104,12 +109,6 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { ("\u{2122}", " trademark "), // trade mark sign ].iter().map(&to_owned)); - // These should turn into spaces - map.extend([ - ("\u{b7}", " "), // · Middle Dot - ("\u{2022}", " "), // • Bullet - ].iter().map(&to_owned)); - map }); From 169739d6174ad40574d4a4d7233a8e822b6d56be Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Feb 2022 00:18:45 -0500 Subject: [PATCH 0888/1042] comments --- crates/lib/tts_common/src/clean_symbols.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index dbe86d52ad..0f155b807c 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -70,16 +70,16 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { // Dashes map.extend([ - ("\u{2010}", "-"), // Hyphen - ("\u{2011}", "-"), // Non-Breaking Hyphen - ("\u{2013}", "-"), // En Dash - ("\u{2014}", "-"), // Em Dash - ("\u{2015}", "-"), // Horizontal Bar - ("\u{2E3A}", "-"), // Two-Em Dash - ("\u{2E3B}", "-"), // Three-Em Dash - ("\u{FE58}", "-"), // Small Em Dash - ("\u{FE63}", "-"), // Small Hyphen-Minus - ("\u{FF0D}", "-"), // Fullwidth Hyphen-Minus + ("\u{2010}", "-"), // ‐ Hyphen + ("\u{2011}", "-"), // ‑ Non-Breaking Hyphen + ("\u{2013}", "-"), // – En Dash + ("\u{2014}", "-"), // — Em Dash + ("\u{2015}", "-"), // ― Horizontal Bar + ("\u{2E3A}", "-"), // ⸺ Two-Em Dash + ("\u{2E3B}", "-"), // ⸻ Three-Em Dash + ("\u{FE58}", "-"), // ﹘ Small Em Dash + ("\u{FE63}", "-"), // ﹣ Small Hyphen-Minus + ("\u{FF0D}", "-"), // － Fullwidth Hyphen-Minus ].iter().map(&to_owned)); // Close enough to existing allowed punctuation @@ -112,6 +112,8 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { map }); +/// Tacotron chokes on Unicode, latin characters, special punctuation, emoji, etc. +/// We attempt to turn as much as possible into ASCII and a reduced set of symbols. pub fn clean_symbols(input_text: &str) -> String { let segmented= UnicodeSegmentation::graphemes(input_text, true) .map(|segment| { From 820ac51190b9c79b09f0b5b9424d04176c5e6e30 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Feb 2022 00:51:10 -0500 Subject: [PATCH 0889/1042] add some emoji --- crates/lib/tts_common/src/clean_symbols.rs | 33 +++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 0f155b807c..51db2ac72c 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -34,10 +34,14 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { ("\u{200B}", " "), // Zero Width Space ("\u{200C}", " "), // Zero Width Non-Joiner ("\u{200D}", " "), // Zero Width Joiner + ("\u{2028}", " "), // Line Separator + ("\u{2029}", " "), // Paragraph Separator ("\u{205F}", " "), // Medium Mathematical Space (MMSP) ("\u{2588}", " "), // Full Block ("\u{2800}", " "), // Braille Pattern Blank ("\u{3000}", " "), // Ideographic Space + ("\u{3164}", " "), // Hangul Filler + ("\u{FEFF}", " "), // Zero Width No-Break Space ].iter().map(&to_owned)); // Punctuation that should turn into spaces @@ -70,6 +74,7 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { // Dashes map.extend([ + ("\u{00AD}", "-"), // ­ Soft Hyphen ("\u{2010}", "-"), // ‐ Hyphen ("\u{2011}", "-"), // ‑ Non-Breaking Hyphen ("\u{2013}", "-"), // – En Dash @@ -91,6 +96,7 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { ("\u{2026}", "..."), // … Horizontal Ellipsis ("\u{203C}", "!!"), // ‼ Double Exclamation Mark Emoji ("\u{203D}", "!?"), // ‽ Interrobang + ("\u{2588}", " "), // █ Full Block ("\u{3001}", ","), // 、 Ideographic Comma ("\u{3002}", "."), // 。 Ideographic Full Stop ("\u{FF01}", "!"), // ！ Fullwidth Exclamation Mark @@ -104,9 +110,28 @@ static REPLACEMENTS : Lazy> = Lazy::new(|| { // Symbols we can insert as words map.extend([ - ("\u{B0}", " degrees "), // degree sign - ("\u{03C0}", " pie "), // greek small letter pi - ("\u{2122}", " trademark "), // trade mark sign + ("\u{00A9}", " copyright "), // © Copyright Sign Emoji + ("\u{00B0}", " degrees "), // ° Degree Sign + ("\u{03C0}", " pie "), // greek small letter pi (TODO: Incorrect handling) + ("\u{2122}", " trademark "), // ™ Trade Mark Sign Emoji + ].iter().map(&to_owned)); + + // Emoji we can insert as words + // https://unicode-table.com/en/blocks/emoticons/ + map.extend([ + ("\u{1F33D}", " corn on the cobb "), // 🌽 Ear of Maize Emoji + ("\u{1F436}", " dog face "), // 🐶 Dog Face Emoji + ("\u{1F44C}", " okay "), // 👌 Ok Hand Sign Emoji + ("\u{1F4A6}", " splashing sweat "), // 💦 Splashing Sweat Symbol Emoji + ("\u{1F4A9}", " poop "), // 💩 Pile of Poo Emoji + ("\u{1F4AF}", " hundred points "), // 💯 Hundred Points Symbol Emoji + ("\u{1F525}", " fire "), // 🔥 Fire Emoji + ("\u{1F530}", " japanese symbol for beginner "), // 🔰 Japanese Symbol for Beginner Emoji + ("\u{1F602}", " face with tears of joy "), // 😂 Face with Tears of Joy Emoji + ("\u{1F60D}", " heart eyes "), // 😍 Smiling Face with Heart-Shaped Eyes Emoji + ("\u{1F62D}", " loudly crying face "), // 😭 Loudly Crying Face Emoji + ("\u{1F633}", " flushed face "), // 😳 Flushed Face Emoji + ("\u{1F923}", " rolling on the floor laughing "), // 🤣 Rolling On The Floor Laughing Emoji ].iter().map(&to_owned)); map @@ -319,7 +344,7 @@ mod tests { assert_converted("â", "a"); // b'\xe2' 2607 assert_converted("α", "a"); // b'\\u03b1' 2625 assert_converted("å", "a"); // b'\xe5' 2753 - //assert_converted("🐶", " dog "); // b'\\U0001f436' 2782 + assert_converted("🐶", " dog face "); // b'\\U0001f436' 2782 assert_converted("™", " trademark "); // b'\\u2122' 2869 assert_converted("É", "E"); // b'\xc9' 3040 assert_converted("æ", "ae"); // b'\xe6' 3142 From aa43554013312e7af1b26cc9ce3df181e7fd6c79 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Feb 2022 01:20:26 -0500 Subject: [PATCH 0890/1042] remove non-ASCII entirely --- crates/lib/tts_common/src/clean_symbols.rs | 35 +++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 51db2ac72c..b4f6cdd0ff 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -149,7 +149,17 @@ pub fn clean_symbols(input_text: &str) -> String { }) .collect::>(); - segmented.join("") + let before_clean : String = segmented.join(""); + + before_clean.chars() + .filter(|segment| { + match segment { + '\x00'..='\x7f' => true, // Full ASCII range + //'\x01'...'\x08' | '\u{10FFFE}'...'\u{10FFFF}' => true, + _ => false, + } + }) + .collect::() } #[cfg(test)] @@ -167,6 +177,29 @@ mod tests { assert_eq!(clean_symbols("one\ntwo\r\nthree "), "one\ntwo\r\nthree ".to_string()); } + #[test] + fn assert_ascii_retained() { + assert_converted("This, sentence. It\nhas\nnewlines.", + "This, sentence. It\nhas\nnewlines."); + assert_converted("12:34", "12:34"); + assert_converted("Punctuation!?.", "Punctuation!?."); + } + + #[test] + fn assert_non_handled_emoji_removed() { + assert_converted("👹", ""); + assert_converted("👹👍✅", ""); + assert_converted("✨what✨", "what"); + } + + #[test] + fn assert_non_handled_languages_removed() { + assert_converted("これはテスト", ""); + assert_converted("これはテストdesu", "desu"); + assert_converted("你好", ""); + assert_converted("hello你好hello", "hellohello"); + } + #[test] fn filters_smart_quotes() { assert_eq!(clean_symbols("That’s ok"), "That's ok".to_string()); From 8f25491e7859a36b13efa25d3684cb32dfb370a3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Feb 2022 05:12:25 -0500 Subject: [PATCH 0891/1042] Add language detection endpoint --- Cargo.lock | 1 + crates/lib/http_server_common/Cargo.toml | 4 + .../request/get_request_header_optional.rs | 15 +++ .../lib/http_server_common/src/request/mod.rs | 4 +- .../src/request/parse_accept_language.rs | 74 ++++++++++++ .../endpoints/misc/detect_locale_handler.rs | 108 ++++++++++++++++++ .../src/http_server/endpoints/misc/mod.rs | 1 + crates/service/storyteller_web/src/routes.rs | 6 + 8 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 crates/lib/http_server_common/src/request/get_request_header_optional.rs create mode 100644 crates/lib/http_server_common/src/request/parse_accept_language.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs diff --git a/Cargo.lock b/Cargo.lock index 4589563260..13a3879a06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1849,6 +1849,7 @@ dependencies = [ "chrono", "container_common", "dotenv", + "language-tags 0.3.2", "log", "serde", "serde_derive", diff --git a/crates/lib/http_server_common/Cargo.toml b/crates/lib/http_server_common/Cargo.toml index d90d35fc76..b351f38410 100644 --- a/crates/lib/http_server_common/Cargo.toml +++ b/crates/lib/http_server_common/Cargo.toml @@ -29,6 +29,10 @@ actix-cors = "0.6.0-beta.3" actix-http = "3.0.0-beta.11" actix-web = "4.0.0-beta.10" +# Language +language-tags = "0.3.2" # Parsing single tags richly +#accept-language = "2.0.0" # Parsing many tags poorly + # Serialization serde = "1.0.125" serde_derive = "1.0.125" diff --git a/crates/lib/http_server_common/src/request/get_request_header_optional.rs b/crates/lib/http_server_common/src/request/get_request_header_optional.rs new file mode 100644 index 0000000000..a3892150e3 --- /dev/null +++ b/crates/lib/http_server_common/src/request/get_request_header_optional.rs @@ -0,0 +1,15 @@ +use actix_web::HttpRequest; + +/// Pull out the named request header, if present. +pub fn get_request_header_optional(request: &HttpRequest, header_name: &str) -> Option { + // TODO: Clean up with transpose() once stable + let result = request.headers() + .get(header_name) + .map(|h| h.to_str()); + match result { + Some(Ok(header)) => Some(header.to_string()), + Some(Err(_)) => None, + None => None, + _ => None, + } +} \ No newline at end of file diff --git a/crates/lib/http_server_common/src/request/mod.rs b/crates/lib/http_server_common/src/request/mod.rs index e4ff0429af..c9c51cb571 100644 --- a/crates/lib/http_server_common/src/request/mod.rs +++ b/crates/lib/http_server_common/src/request/mod.rs @@ -1 +1,3 @@ -pub mod get_request_ip; \ No newline at end of file +pub mod get_request_header_optional; +pub mod get_request_ip; +pub mod parse_accept_language; \ No newline at end of file diff --git a/crates/lib/http_server_common/src/request/parse_accept_language.rs b/crates/lib/http_server_common/src/request/parse_accept_language.rs new file mode 100644 index 0000000000..982fb05e85 --- /dev/null +++ b/crates/lib/http_server_common/src/request/parse_accept_language.rs @@ -0,0 +1,74 @@ +use language_tags::{LanguageTag, ParseError}; +use log::warn; + +/// Parse out accept languages +/// Does not error so that the endpoint won't degrade +pub fn parse_accept_language(accept_languages_header: &str) -> Vec { + let unparsed_tags = accept_languages_header.split(",") + .into_iter() + .map(|tag| tag.trim().to_string()) + .collect::>(); + + let mut parsed_tags = Vec::new(); + + for tag in unparsed_tags.iter() { + match LanguageTag::parse(tag.as_ref()) { + Ok(t) => { + parsed_tags.push(t); + } + Err(e) => { + warn!("Error parsing language tag: {:?}", e); + } + } + } + + parsed_tags +} + +#[cfg(test)] +mod tests { + use crate::request::parse_accept_language::parse_accept_language; + + #[test] + fn test_empty() { + assert_eq!(parse_accept_language(""), Vec::new()); + } + + #[test] + fn test_garbage() { + assert_eq!(parse_accept_language("HJHKDJSHJKHSF"), Vec::new()); + assert_eq!(parse_accept_language("\n\n"), Vec::new()); + } + + #[test] + fn test_single_language() { + let list = parse_accept_language("en-US"); + let lang = list.get(0).unwrap(); + assert_eq!(lang.primary_language(), "en"); + + let list = parse_accept_language("en-GB"); + let lang = list.get(0).unwrap(); + assert_eq!(lang.primary_language(), "en"); + + let list = parse_accept_language("ja-JP"); + let lang = list.get(0).unwrap(); + assert_eq!(lang.primary_language(), "ja"); + } + + #[test] + fn test_multiple() { + let list = parse_accept_language("en-US, es-419"); + let first = list.get(0).unwrap(); + let second = list.get(1).unwrap(); + assert_eq!(first.primary_language(), "en"); + assert_eq!(second.primary_language(), "es"); + + let list = parse_accept_language("en, es, en-GB"); + let first = list.get(0).unwrap(); + let second = list.get(1).unwrap(); + let third= list.get(2).unwrap(); + assert_eq!(first.primary_language(), "en"); + assert_eq!(second.primary_language(), "es"); + assert_eq!(third.primary_language(), "en"); + } +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs new file mode 100644 index 0000000000..ff4ce7fad2 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs @@ -0,0 +1,108 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::queries::categories::get_category_by_token::get_category_by_token; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use http_server_common::request::get_request_header_optional::get_request_header_optional; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; +use http_server_common::request::parse_accept_language::parse_accept_language; + +// =============== Success Response =============== + +#[derive(Serialize, Default)] +pub struct DetectLocaleResponse { + pub success: bool, + /// Full BCP47 language tags + pub full_language_tags: Vec, + /// Parsed out languages + pub language_codes: Vec, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum DetectLocaleError { + ServerError, +} + +impl ResponseError for DetectLocaleError { + fn status_code(&self) -> StatusCode { + match *self { + DetectLocaleError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for DetectLocaleError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub const FORCE_LOCALE_COOKIE_HEADER_NAME : &'static str = "force-locale"; + +pub async fn detect_locale_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + let mut maybe_accept_language = + get_request_header_optional(&http_request, "accept-language"); + + if let Some(cookie) = http_request.cookie(FORCE_LOCALE_COOKIE_HEADER_NAME) { + warn!("Overriding default accept language with custom value (from cookie)"); + maybe_accept_language = Some(cookie.value().to_string()); + + } else if let Some(header) = get_request_header_optional(&http_request, FORCE_LOCALE_COOKIE_HEADER_NAME) { + warn!("Overriding default accept language with custom value (from header)"); + maybe_accept_language = Some(header); + } + + let accept_language = maybe_accept_language.unwrap_or("en".to_string()); + + let mut response = DetectLocaleResponse::default(); + response.success = true; + + let language_tags = parse_accept_language(&accept_language); + + for language_tag in language_tags.iter() { + response.full_language_tags.push(language_tag.to_string()); + response.language_codes.push(language_tag.primary_language().to_string()); + } + + let body = serde_json::to_string(&response) + .map_err(|e| DetectLocaleError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/misc/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/mod.rs index c4da5e82d0..417922d9aa 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/misc/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/misc/mod.rs @@ -1,5 +1,6 @@ pub mod alpha_cookie; pub mod default_route_404; +pub mod detect_locale_handler; pub mod enable_alpha_easy_handler; pub mod enable_alpha_handler; pub mod root_index; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index 51aea12e25..cee0183f37 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -70,6 +70,7 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; +use crate::http_server::endpoints::misc::detect_locale_handler::detect_locale_handler; pub fn add_routes (app: App) -> App where @@ -116,6 +117,11 @@ pub fn add_routes (app: App) -> App .route(web::get().to(list_events_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/detect_locale") + .route(web::get().to(detect_locale_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/leaderboard") .route(web::get().to(leaderboard_handler)) From 29f713c157260728acb11e4f69fa1aca9e91a8fd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Feb 2022 23:39:45 -0500 Subject: [PATCH 0892/1042] tts migrations with source metadata --- .../down.sql | 9 +++++++++ .../2022-02-09-041205_alter_tts_add_twitch/up.sql | 15 +++++++++++++++ .../tts_tables/tts_inference_jobs.sql | 5 +++++ _migrations_squashed/tts_tables/tts_results.sql | 5 +++++ 4 files changed, 34 insertions(+) create mode 100644 _migrations/2022-02-09-041205_alter_tts_add_twitch/down.sql create mode 100644 _migrations/2022-02-09-041205_alter_tts_add_twitch/up.sql diff --git a/_migrations/2022-02-09-041205_alter_tts_add_twitch/down.sql b/_migrations/2022-02-09-041205_alter_tts_add_twitch/down.sql new file mode 100644 index 0000000000..79756b537e --- /dev/null +++ b/_migrations/2022-02-09-041205_alter_tts_add_twitch/down.sql @@ -0,0 +1,9 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_inference_jobs DROP COLUMN is_from_api; +ALTER TABLE tts_inference_jobs DROP COLUMN is_for_twitch; + +ALTER TABLE tts_results DROP COLUMN is_from_api; +ALTER TABLE tts_results DROP COLUMN is_for_twitch; diff --git a/_migrations/2022-02-09-041205_alter_tts_add_twitch/up.sql b/_migrations/2022-02-09-041205_alter_tts_add_twitch/up.sql new file mode 100644 index 0000000000..9b11252a06 --- /dev/null +++ b/_migrations/2022-02-09-041205_alter_tts_add_twitch/up.sql @@ -0,0 +1,15 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_inference_jobs + ADD COLUMN is_from_api BOOLEAN NOT NULL DEFAULT FALSE + AFTER creator_set_visibility, + ADD COLUMN is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE + AFTER is_from_api; + +ALTER TABLE tts_results + ADD COLUMN is_from_api BOOLEAN NOT NULL DEFAULT FALSE + AFTER duration_millis, + ADD COLUMN is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE + AFTER is_from_api; diff --git a/_migrations_squashed/tts_tables/tts_inference_jobs.sql b/_migrations_squashed/tts_tables/tts_inference_jobs.sql index e25ae6680b..fa3658dac3 100644 --- a/_migrations_squashed/tts_tables/tts_inference_jobs.sql +++ b/_migrations_squashed/tts_tables/tts_inference_jobs.sql @@ -57,6 +57,11 @@ CREATE TABLE tts_inference_jobs ( 'private' ) NOT NULL DEFAULT 'public', + -- ========== SOURCE METADATA ========== + + is_from_api BOOLEAN NOT NULL DEFAULT FALSE, + is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE, + -- ========== JOB SYSTEM DETAILS ========== -- Jobs begin as "pending", then transition to other states. diff --git a/_migrations_squashed/tts_tables/tts_results.sql b/_migrations_squashed/tts_tables/tts_results.sql index 7b09574ac3..72756ab9d4 100644 --- a/_migrations_squashed/tts_tables/tts_results.sql +++ b/_migrations_squashed/tts_tables/tts_results.sql @@ -72,6 +72,11 @@ CREATE TABLE tts_results ( file_size_bytes INT(10) NOT NULL DEFAULT 0, duration_millis INT(10) NOT NULL DEFAULT 0, + -- ========== SOURCE METADATA ========== + + is_from_api BOOLEAN NOT NULL DEFAULT FALSE, + is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE, + -- ========== MODERATION DETAILS ========== -- The last moderator that made changes. From 67f93b84ac2001d6ad007cf7bfaffb8a4063a976 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Feb 2022 12:30:39 -0500 Subject: [PATCH 0893/1042] add priority level to tts --- .../2022-02-09-041205_alter_tts_add_twitch/down.sql | 3 ++- .../2022-02-09-041205_alter_tts_add_twitch/up.sql | 10 +++++++++- _migrations_squashed/tts_tables/tts_inference_jobs.sql | 8 +++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/_migrations/2022-02-09-041205_alter_tts_add_twitch/down.sql b/_migrations/2022-02-09-041205_alter_tts_add_twitch/down.sql index 79756b537e..6f51d20bd7 100644 --- a/_migrations/2022-02-09-041205_alter_tts_add_twitch/down.sql +++ b/_migrations/2022-02-09-041205_alter_tts_add_twitch/down.sql @@ -2,8 +2,9 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile -ALTER TABLE tts_inference_jobs DROP COLUMN is_from_api; ALTER TABLE tts_inference_jobs DROP COLUMN is_for_twitch; +ALTER TABLE tts_inference_jobs DROP COLUMN is_from_api; +ALTER TABLE tts_inference_jobs DROP COLUMN priority_level; ALTER TABLE tts_results DROP COLUMN is_from_api; ALTER TABLE tts_results DROP COLUMN is_for_twitch; diff --git a/_migrations/2022-02-09-041205_alter_tts_add_twitch/up.sql b/_migrations/2022-02-09-041205_alter_tts_add_twitch/up.sql index 9b11252a06..efda373105 100644 --- a/_migrations/2022-02-09-041205_alter_tts_add_twitch/up.sql +++ b/_migrations/2022-02-09-041205_alter_tts_add_twitch/up.sql @@ -5,8 +5,16 @@ ALTER TABLE tts_inference_jobs ADD COLUMN is_from_api BOOLEAN NOT NULL DEFAULT FALSE AFTER creator_set_visibility, + ADD COLUMN is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE - AFTER is_from_api; + AFTER is_from_api, + + -- Priority *increases*, so a level of 2 will be higher than 1. + -- By default, all jobs from FakeYou have level 0. + -- Twitch TTS has level 1. + -- Paid Twitch and Paid API have level 2. + ADD COLUMN priority_level TINYINT UNSIGNED NOT NULL DEFAULT 0 + AFTER status; ALTER TABLE tts_results ADD COLUMN is_from_api BOOLEAN NOT NULL DEFAULT FALSE diff --git a/_migrations_squashed/tts_tables/tts_inference_jobs.sql b/_migrations_squashed/tts_tables/tts_inference_jobs.sql index fa3658dac3..395790dc3c 100644 --- a/_migrations_squashed/tts_tables/tts_inference_jobs.sql +++ b/_migrations_squashed/tts_tables/tts_inference_jobs.sql @@ -84,7 +84,13 @@ CREATE TABLE tts_inference_jobs ( 'attempt_failed', 'dead') NOT NULL DEFAULT 'pending', - -- We can track this against a "max_attempt_count" + -- Priority *increases*, so a level of 2 will be higher than 1. + -- By default, all jobs from FakeYou have level 0. + -- Twitch TTS has level 1. + -- Paid Twitch and Paid API have level 2. + priority_level TINYINT UNSIGNED NOT NULL DEFAULT 0, + + -- We can track this against a "max_attempt_count" attempt_count INT(3) NOT NULL DEFAULT 0, -- If there is a failure, tell the user why. From 765ecfb1db124e0bf184fc738a6bfd470b9a3e6d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Feb 2022 12:42:46 -0500 Subject: [PATCH 0894/1042] a few more reservations --- db/reserved_usernames.txt | 2 ++ db/reserved_usernames_including.txt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index 36ca004326..10f6d1ff8e 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -8,6 +8,7 @@ # Stuff we care about fake_you fakeyou +hanashi monogatari storied stories @@ -19,6 +20,7 @@ vocode vocoder vocoders vocodes +voxelcam # Types of models first_order_model diff --git a/db/reserved_usernames_including.txt b/db/reserved_usernames_including.txt index b8b8c7a36c..cadc30d175 100644 --- a/db/reserved_usernames_including.txt +++ b/db/reserved_usernames_including.txt @@ -6,4 +6,5 @@ storytell storyteller test vocode -vocodes \ No newline at end of file +vocodes +voxelcam \ No newline at end of file From 7394667f4a9695ff13bc3b560f78ef3861fc49c0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Feb 2022 13:34:09 -0500 Subject: [PATCH 0895/1042] begin moving these to database_queries crate --- .../database_queries/src/column_types/mod.rs | 1 + .../src/column_types/record_visibility.rs | 35 +++++ crates/lib/database_queries/src/lib.rs | 1 + .../tts/list_available_tts_inference_jobs.rs | 137 ++++++++++++++++++ crates/lib/database_queries/src/tts/mod.rs | 3 +- 5 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 crates/lib/database_queries/src/column_types/mod.rs create mode 100644 crates/lib/database_queries/src/column_types/record_visibility.rs create mode 100644 crates/lib/database_queries/src/tts/list_available_tts_inference_jobs.rs diff --git a/crates/lib/database_queries/src/column_types/mod.rs b/crates/lib/database_queries/src/column_types/mod.rs new file mode 100644 index 0000000000..fa9a069908 --- /dev/null +++ b/crates/lib/database_queries/src/column_types/mod.rs @@ -0,0 +1 @@ +pub mod record_visibility; \ No newline at end of file diff --git a/crates/lib/database_queries/src/column_types/record_visibility.rs b/crates/lib/database_queries/src/column_types/record_visibility.rs new file mode 100644 index 0000000000..3f1ee15637 --- /dev/null +++ b/crates/lib/database_queries/src/column_types/record_visibility.rs @@ -0,0 +1,35 @@ +//! These are columns where users can control the visibility of their data. + +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; + +/// To use this in a query, the query must have type annotations. +/// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375 +/// eg. preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::column_types::record_visibility::RecordVisibility` +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] +#[serde(rename_all = "lowercase")] +#[sqlx(rename_all = "lowercase")] +pub enum RecordVisibility { + Public, + Hidden, + Private, +} + +impl RecordVisibility { + pub fn to_str(&self) -> &'static str { + match self { + RecordVisibility::Public => "public", + RecordVisibility::Hidden => "hidden", + RecordVisibility::Private => "private", + } + } + + pub fn from_str(record_visibility: &str) -> AnyhowResult { + match record_visibility { + "public" => Ok(RecordVisibility::Public), + "hidden" => Ok(RecordVisibility::Hidden), + "private" => Ok(RecordVisibility::Private), + _ => Err(anyhow!("invalid value: {:?}", record_visibility)), + } + } +} diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index 39645e7c12..a85062e2bc 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -2,6 +2,7 @@ mod builders; +pub mod column_types; pub mod helpers; pub mod tokens; pub mod tts; diff --git a/crates/lib/database_queries/src/tts/list_available_tts_inference_jobs.rs b/crates/lib/database_queries/src/tts/list_available_tts_inference_jobs.rs new file mode 100644 index 0000000000..3d94855ae4 --- /dev/null +++ b/crates/lib/database_queries/src/tts/list_available_tts_inference_jobs.rs @@ -0,0 +1,137 @@ +use anyhow::anyhow; +use chrono::{Utc, DateTime}; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::record_visibility::RecordVisibility; +use log::{warn, info}; +use sqlx::MySqlPool; +use crate::helpers::boolean_converters::i8_to_bool; + +/// table: tts_inference_jobs +#[derive(Debug)] +pub struct TtsInferenceJob { + pub id: i64, + pub inference_job_token: String, + pub uuid_idempotency_token: String, + + pub model_token: String, + pub raw_inference_text: String, + + pub creator_ip_address: String, + pub maybe_creator_user_token: Option, + pub creator_set_visibility: RecordVisibility, + + pub is_from_api: bool, + pub is_for_twitch: bool, + + pub status: String, // TODO: Enum + pub priority_level: u8, + pub attempt_count: i32, + pub failure_reason: Option, + + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub retry_at: Option>, +} + +pub async fn query_tts_inference_job_records( + pool: &MySqlPool, + num_records: u32 +) -> AnyhowResult> { + + let job_records : Vec = sqlx::query_as!( + TtsInferenceJobRawInternal, + r#" +SELECT + id, + token AS inference_job_token, + uuid_idempotency_token, + + model_token, + raw_inference_text, + + creator_ip_address, + maybe_creator_user_token, + creator_set_visibility as `creator_set_visibility: crate::column_types::record_visibility::RecordVisibility`, + + is_from_api, + is_for_twitch, + + status, + priority_level, + attempt_count, + failure_reason, + + created_at, + updated_at, + retry_at +FROM tts_inference_jobs +WHERE + ( + status IN ("pending", "attempt_failed") + ) + AND + ( + retry_at IS NULL + OR + retry_at < CURRENT_TIMESTAMP + ) + LIMIT ? + "#, + num_records, + ) + .fetch_all(pool) + .await?; + + let job_records = job_records.into_iter() + .map(|record : TtsInferenceJobRawInternal| { + TtsInferenceJob { + id: record.id, + inference_job_token: record.inference_job_token, + uuid_idempotency_token: record.uuid_idempotency_token, + model_token: record.model_token, + raw_inference_text: record.raw_inference_text, + creator_ip_address: record.creator_ip_address, + maybe_creator_user_token: record.maybe_creator_user_token, + creator_set_visibility: record.creator_set_visibility, + is_from_api: i8_to_bool(record.is_from_api), + is_for_twitch: i8_to_bool(record.is_for_twitch), + status: record.status, + priority_level: record.priority_level, + attempt_count: record.attempt_count, + failure_reason: record.failure_reason, + created_at: record.created_at, + updated_at: record.updated_at, + retry_at: record.retry_at, + } + }) + .collect::>(); + + Ok(job_records) +} + +#[derive(Debug)] +pub struct TtsInferenceJobRawInternal { + pub id: i64, + pub inference_job_token: String, + pub uuid_idempotency_token: String, + + pub model_token: String, + pub raw_inference_text: String, + + pub creator_ip_address: String, + pub maybe_creator_user_token: Option, + pub creator_set_visibility: RecordVisibility, + + pub is_from_api: i8, + pub is_for_twitch: i8, + + pub status: String, + pub priority_level: u8, + pub attempt_count: i32, + pub failure_reason: Option, + + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub retry_at: Option>, +} + diff --git a/crates/lib/database_queries/src/tts/mod.rs b/crates/lib/database_queries/src/tts/mod.rs index f7b53931ee..39427a96cc 100644 --- a/crates/lib/database_queries/src/tts/mod.rs +++ b/crates/lib/database_queries/src/tts/mod.rs @@ -1 +1,2 @@ -pub mod insert_tts_inference_job; \ No newline at end of file +pub mod insert_tts_inference_job; +pub mod list_available_tts_inference_jobs; \ No newline at end of file From 0fddc08868c1bcdf97b3130df58e80309e3c35d6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Feb 2022 13:41:58 -0500 Subject: [PATCH 0896/1042] Make Status an enum --- .../src/column_types/job_status.rs | 44 +++++++++++++++++++ .../database_queries/src/column_types/mod.rs | 1 + .../tts/list_available_tts_inference_jobs.rs | 10 +++-- .../job_queries/tts_inference_job_queries.rs | 9 ++++ 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 crates/lib/database_queries/src/column_types/job_status.rs diff --git a/crates/lib/database_queries/src/column_types/job_status.rs b/crates/lib/database_queries/src/column_types/job_status.rs new file mode 100644 index 0000000000..a9c03a9860 --- /dev/null +++ b/crates/lib/database_queries/src/column_types/job_status.rs @@ -0,0 +1,44 @@ +//! These are columns where users can control the visibility of their data. + +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; + +/// To use this in a query, the query must have type annotations. +/// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375 +/// eg. job_status as `job_status: crate::column_types::job_status::JobStatus` +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] +#[serde(rename_all = "lowercase")] +#[sqlx(rename_all = "lowercase")] +pub enum JobStatus { + Pending, + Started, + CompleteSuccess, + CompleteFailure, + AttemptFailed, + Dead, +} + +impl JobStatus { + pub fn to_str(&self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Started => "started", + Self::CompleteSuccess => "complete_success", + Self::CompleteFailure => "complete_failure", + Self::AttemptFailed => "attempt_failed", + Self::Dead => "dead", + } + } + + pub fn from_str(job_status: &str) -> AnyhowResult { + match job_status { + "pending" => Ok(Self::Pending), + "started" => Ok(Self::Started), + "complete_success" => Ok(Self::CompleteSuccess), + "complete_failure" => Ok(Self::CompleteFailure), + "attempt_failed" => Ok(Self::AttemptFailed), + "dead" => Ok(Self::Dead), + _ => Err(anyhow!("invalid job_status: {:?}", job_status)), + } + } +} diff --git a/crates/lib/database_queries/src/column_types/mod.rs b/crates/lib/database_queries/src/column_types/mod.rs index fa9a069908..4b0786fb98 100644 --- a/crates/lib/database_queries/src/column_types/mod.rs +++ b/crates/lib/database_queries/src/column_types/mod.rs @@ -1 +1,2 @@ +pub mod job_status; pub mod record_visibility; \ No newline at end of file diff --git a/crates/lib/database_queries/src/tts/list_available_tts_inference_jobs.rs b/crates/lib/database_queries/src/tts/list_available_tts_inference_jobs.rs index 3d94855ae4..1c077bdb62 100644 --- a/crates/lib/database_queries/src/tts/list_available_tts_inference_jobs.rs +++ b/crates/lib/database_queries/src/tts/list_available_tts_inference_jobs.rs @@ -1,10 +1,11 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; use container_common::anyhow_result::AnyhowResult; +use crate::column_types::job_status::JobStatus; use crate::column_types::record_visibility::RecordVisibility; +use crate::helpers::boolean_converters::i8_to_bool; use log::{warn, info}; use sqlx::MySqlPool; -use crate::helpers::boolean_converters::i8_to_bool; /// table: tts_inference_jobs #[derive(Debug)] @@ -23,7 +24,7 @@ pub struct TtsInferenceJob { pub is_from_api: bool, pub is_for_twitch: bool, - pub status: String, // TODO: Enum + pub status: JobStatus, pub priority_level: u8, pub attempt_count: i32, pub failure_reason: Option, @@ -33,6 +34,7 @@ pub struct TtsInferenceJob { pub retry_at: Option>, } +/// Query jobs that are ready to run pub async fn query_tts_inference_job_records( pool: &MySqlPool, num_records: u32 @@ -56,7 +58,7 @@ SELECT is_from_api, is_for_twitch, - status, + status as `status: crate::column_types::job_status::JobStatus`, priority_level, attempt_count, failure_reason, @@ -125,7 +127,7 @@ pub struct TtsInferenceJobRawInternal { pub is_from_api: i8, pub is_for_twitch: i8, - pub status: String, + pub status: JobStatus, pub priority_level: u8, pub attempt_count: i32, pub failure_reason: Option, diff --git a/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs b/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs index b0f551d3ce..31cb75f245 100644 --- a/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs @@ -27,9 +27,14 @@ pub struct TtsInferenceJobRecord { pub maybe_creator_user_token: Option, pub creator_set_visibility: RecordVisibility, + pub is_from_api: i8, + pub is_for_twitch: i8, + pub status: String, // TODO + pub priority_level: u8, pub attempt_count: i32, pub failure_reason: Option, + pub created_at: chrono::DateTime, pub updated_at: chrono::DateTime, pub retry_at: Option>, @@ -55,7 +60,11 @@ SELECT maybe_creator_user_token, creator_set_visibility as `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, + is_from_api, + is_for_twitch, + status, + priority_level, attempt_count, failure_reason, created_at, From bf4e0006446c6b402c56381d01b8207977aafd1a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Feb 2022 00:52:46 -0500 Subject: [PATCH 0897/1042] Huge refactor of TTS job queries into database_queries common lib --- .../src/column_types/job_status.rs | 4 +- .../database_queries/src/column_types/mod.rs | 3 +- .../src/column_types/vocoder_type.rs | 31 ++ crates/lib/database_queries/src/tts/mod.rs | 5 +- .../src/tts/tts_inference_jobs/_keys.rs | 5 + .../insert_tts_inference_job.rs | 0 .../list_available_tts_inference_jobs.rs | 23 +- .../mark_tts_inference_job_done.rs | 33 ++ .../mark_tts_inference_job_failure.rs | 41 ++ ...tts_inference_job_pending_and_grab_lock.rs | 85 +++ ...mark_tts_inference_job_permanently_dead.rs | 28 + .../src/tts/tts_inference_jobs/mod.rs | 7 + .../tts_models/get_tts_model_for_inference.rs | 103 ++++ .../src/tts/tts_models/mod.rs | 1 + .../src/tts/tts_results/insert_tts_result.rs | 165 ++++++ .../src/tts/tts_results/mod.rs | 1 + .../twitch_pubsub_user_subscriber_thread.rs | 2 +- .../tts_inference_sidecar_client.rs | 2 +- .../storyteller_web/src/job_queries/mod.rs | 4 +- .../job_queries/tts_download_job_queries.rs | 3 + .../job_queries/tts_inference_job_queries.rs | 504 ------------------ .../job_queries/w2l_download_job_queries.rs | 3 + .../job_queries/w2l_inference_job_queries.rs | 3 + .../storyteller_web/src/tts_inference_job.rs | 56 +- 24 files changed, 555 insertions(+), 557 deletions(-) create mode 100644 crates/lib/database_queries/src/column_types/vocoder_type.rs create mode 100644 crates/lib/database_queries/src/tts/tts_inference_jobs/_keys.rs rename crates/lib/database_queries/src/tts/{ => tts_inference_jobs}/insert_tts_inference_job.rs (100%) rename crates/lib/database_queries/src/tts/{ => tts_inference_jobs}/list_available_tts_inference_jobs.rs (84%) create mode 100644 crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_done.rs create mode 100644 crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs create mode 100644 crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_pending_and_grab_lock.rs create mode 100644 crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs create mode 100644 crates/lib/database_queries/src/tts/tts_inference_jobs/mod.rs create mode 100644 crates/lib/database_queries/src/tts/tts_models/get_tts_model_for_inference.rs create mode 100644 crates/lib/database_queries/src/tts/tts_models/mod.rs create mode 100644 crates/lib/database_queries/src/tts/tts_results/insert_tts_result.rs create mode 100644 crates/lib/database_queries/src/tts/tts_results/mod.rs delete mode 100644 crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs diff --git a/crates/lib/database_queries/src/column_types/job_status.rs b/crates/lib/database_queries/src/column_types/job_status.rs index a9c03a9860..e3606e50ca 100644 --- a/crates/lib/database_queries/src/column_types/job_status.rs +++ b/crates/lib/database_queries/src/column_types/job_status.rs @@ -7,8 +7,8 @@ use container_common::anyhow_result::AnyhowResult; /// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375 /// eg. job_status as `job_status: crate::column_types::job_status::JobStatus` #[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] -#[serde(rename_all = "lowercase")] -#[sqlx(rename_all = "lowercase")] +#[serde(rename_all = "snake_case")] +#[sqlx(rename_all = "snake_case")] pub enum JobStatus { Pending, Started, diff --git a/crates/lib/database_queries/src/column_types/mod.rs b/crates/lib/database_queries/src/column_types/mod.rs index 4b0786fb98..1c108ef9c1 100644 --- a/crates/lib/database_queries/src/column_types/mod.rs +++ b/crates/lib/database_queries/src/column_types/mod.rs @@ -1,2 +1,3 @@ pub mod job_status; -pub mod record_visibility; \ No newline at end of file +pub mod record_visibility; +pub mod vocoder_type; \ No newline at end of file diff --git a/crates/lib/database_queries/src/column_types/vocoder_type.rs b/crates/lib/database_queries/src/column_types/vocoder_type.rs new file mode 100644 index 0000000000..ad92b2687e --- /dev/null +++ b/crates/lib/database_queries/src/column_types/vocoder_type.rs @@ -0,0 +1,31 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] +#[serde(rename_all = "lowercase")] +#[sqlx(rename_all = "lowercase")] +pub enum VocoderType { + /// Current state of the art vocoder + #[serde(rename = "hifigan-superres")] + HifiGanSuperResolution, + + /// Older WaveGlow vocoder + WaveGlow, +} + +impl VocoderType { + pub fn to_str(&self) -> &'static str { + match self { + VocoderType::HifiGanSuperResolution => "hifigan-superres", + VocoderType::WaveGlow => "waveglow", + } + } + + pub fn from_str(vocoder_type: &str) -> AnyhowResult { + match vocoder_type { + "hifigan-superres" => Ok(VocoderType::HifiGanSuperResolution), + "waveglow" => Ok(VocoderType::WaveGlow), + _ => Err(anyhow!("invalid value: {:?}", vocoder_type)), + } + } +} diff --git a/crates/lib/database_queries/src/tts/mod.rs b/crates/lib/database_queries/src/tts/mod.rs index 39427a96cc..00c9ce9ee9 100644 --- a/crates/lib/database_queries/src/tts/mod.rs +++ b/crates/lib/database_queries/src/tts/mod.rs @@ -1,2 +1,3 @@ -pub mod insert_tts_inference_job; -pub mod list_available_tts_inference_jobs; \ No newline at end of file +pub mod tts_inference_jobs; +pub mod tts_models; +pub mod tts_results; \ No newline at end of file diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/_keys.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/_keys.rs new file mode 100644 index 0000000000..0a6b60dd8f --- /dev/null +++ b/crates/lib/database_queries/src/tts/tts_inference_jobs/_keys.rs @@ -0,0 +1,5 @@ +//! Primary and foreign keys for TTS-related models + +/// Since jobs are internal, we'll use IDs as primary keys. +#[derive(Clone, Copy, PartialOrd, PartialEq, Eq, Ord, Debug)] +pub struct TtsInferenceJobId(pub i64); diff --git a/crates/lib/database_queries/src/tts/insert_tts_inference_job.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/insert_tts_inference_job.rs similarity index 100% rename from crates/lib/database_queries/src/tts/insert_tts_inference_job.rs rename to crates/lib/database_queries/src/tts/tts_inference_jobs/insert_tts_inference_job.rs diff --git a/crates/lib/database_queries/src/tts/list_available_tts_inference_jobs.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs similarity index 84% rename from crates/lib/database_queries/src/tts/list_available_tts_inference_jobs.rs rename to crates/lib/database_queries/src/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs index 1c077bdb62..718414db09 100644 --- a/crates/lib/database_queries/src/tts/list_available_tts_inference_jobs.rs +++ b/crates/lib/database_queries/src/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs @@ -4,13 +4,14 @@ use container_common::anyhow_result::AnyhowResult; use crate::column_types::job_status::JobStatus; use crate::column_types::record_visibility::RecordVisibility; use crate::helpers::boolean_converters::i8_to_bool; +use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; use log::{warn, info}; use sqlx::MySqlPool; /// table: tts_inference_jobs #[derive(Debug)] -pub struct TtsInferenceJob { - pub id: i64, +pub struct AvailableTtsInferenceJob { + pub id: TtsInferenceJobId, pub inference_job_token: String, pub uuid_idempotency_token: String, @@ -35,13 +36,13 @@ pub struct TtsInferenceJob { } /// Query jobs that are ready to run -pub async fn query_tts_inference_job_records( +pub async fn list_available_tts_inference_jobs( pool: &MySqlPool, num_records: u32 -) -> AnyhowResult> { +) -> AnyhowResult> { - let job_records : Vec = sqlx::query_as!( - TtsInferenceJobRawInternal, + let job_records : Vec = sqlx::query_as!( + AvailableTtsInferenceJobRawInternal, r#" SELECT id, @@ -85,9 +86,9 @@ WHERE .await?; let job_records = job_records.into_iter() - .map(|record : TtsInferenceJobRawInternal| { - TtsInferenceJob { - id: record.id, + .map(|record : AvailableTtsInferenceJobRawInternal| { + AvailableTtsInferenceJob { + id: TtsInferenceJobId(record.id), inference_job_token: record.inference_job_token, uuid_idempotency_token: record.uuid_idempotency_token, model_token: record.model_token, @@ -106,13 +107,13 @@ WHERE retry_at: record.retry_at, } }) - .collect::>(); + .collect::>(); Ok(job_records) } #[derive(Debug)] -pub struct TtsInferenceJobRawInternal { +struct AvailableTtsInferenceJobRawInternal { pub id: i64, pub inference_job_token: String, pub uuid_idempotency_token: String, diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_done.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_done.rs new file mode 100644 index 0000000000..5acb43700d --- /dev/null +++ b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_done.rs @@ -0,0 +1,33 @@ +use container_common::anyhow_result::AnyhowResult; +use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; +use sqlx::MySqlPool; +use sqlx; + +pub async fn mark_tts_inference_job_done( + pool: &MySqlPool, + job_id: TtsInferenceJobId, + success: bool, + maybe_result_token: Option<&str> +) -> AnyhowResult<()> { + let status = if success { "complete_success" } else { "complete_failure" }; + + let query_result = sqlx::query!( + r#" +UPDATE tts_inference_jobs +SET + status = ?, + on_success_result_token = ?, + failure_reason = NULL, + retry_at = NULL +WHERE id = ? + "#, + status, + maybe_result_token, + job_id.0 + ) + .execute(pool) + .await?; + + Ok(()) +} + diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs new file mode 100644 index 0000000000..19824707be --- /dev/null +++ b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs @@ -0,0 +1,41 @@ +use container_common::anyhow_result::AnyhowResult; +use crate::tts::tts_inference_jobs::list_available_tts_inference_jobs::AvailableTtsInferenceJob; +use sqlx::MySqlPool; +use sqlx; + +// TODO: Rerun time shouldn't be 1-minute! + +/// Mark a single inference failure. The job may be re-run. +pub async fn mark_tts_inference_job_failure( + pool: &MySqlPool, + job: &AvailableTtsInferenceJob, + failure_reason: &str, + max_attempts: i32 +) -> AnyhowResult<()> { + + // statuses: "attempt_failed", "complete_failure", "dead" + let mut next_status = "attempt_failed"; + + if job.attempt_count >= max_attempts { + // NB: Job attempt count is incremented at start + next_status = "dead"; + } + + let query_result = sqlx::query!( + r#" +UPDATE tts_inference_jobs +SET + status = ?, + failure_reason = ?, + retry_at = NOW() + interval 1 minute +WHERE id = ? + "#, + next_status, + failure_reason.to_string(), + job.id.0, + ) + .execute(pool) + .await?; + + Ok(()) +} diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_pending_and_grab_lock.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_pending_and_grab_lock.rs new file mode 100644 index 0000000000..98708357bd --- /dev/null +++ b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_pending_and_grab_lock.rs @@ -0,0 +1,85 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; +use sqlx::MySqlPool; +use sqlx; + +// TODO(2022-02-09): It's unclear how stuck "pending" jobs don't become perma-dead +// I think this logic is flawed and the job system needs to pick up stuck "pending" jobs. + +pub struct TtsInferenceLockRecord { + id: i64, + status: String, +} + +pub async fn mark_tts_inference_job_pending_and_grab_lock( + pool: &MySqlPool, + job_id: TtsInferenceJobId +) -> AnyhowResult { + + // NB: We use transactions and "SELECT ... FOR UPDATE" to simulate mutexes. + let mut transaction = pool.begin().await?; + + let maybe_record = sqlx::query_as!( + TtsInferenceLockRecord, + r#" +SELECT + id, + status +FROM tts_inference_jobs +WHERE id = ? +FOR UPDATE + "#, + job_id.0, + ) + .fetch_one(&mut transaction) + .await; + + let record : TtsInferenceLockRecord = match maybe_record { + Ok(record) => record, + Err(err) => { + match err { + sqlx::Error::RowNotFound => { + return Err(anyhow!("could not job")); + }, + _ => { + return Err(anyhow!("query error")); + } + } + } + }; + + let can_transact = match record.status.as_ref() { + "pending" => true, // It's okay for us to take the lock. + "attempt_failed" => true, // We can retry. + "started" => false, // Job in progress (another job beat us, and we can't take the lock) + "complete_success" => false, // Job already complete + "complete_failure" => false, // Job already complete (permanently dead; no need to retry) + "dead" => false, // Job already complete (permanently dead; retries exhausted) + _ => false, // Future-proof + }; + + if !can_transact { + transaction.rollback().await?; + return Ok(false); + } + + let _acquire_lock = sqlx::query!( + r#" +UPDATE tts_inference_jobs +SET + status = 'started', + attempt_count = attempt_count + 1, + retry_at = NOW() + interval 2 minute +WHERE id = ? + "#, + job_id.0, + ) + .execute(&mut transaction) + .await?; + + transaction.commit().await?; + + Ok(true) +} + diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs new file mode 100644 index 0000000000..e1441afb38 --- /dev/null +++ b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs @@ -0,0 +1,28 @@ +use container_common::anyhow_result::AnyhowResult; +use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; +use sqlx::MySqlPool; +use sqlx; + +/// Mark a TTS job as dead ("do not ever retry"). +pub async fn mark_tts_inference_job_permanently_dead( + pool: &MySqlPool, + job_id: TtsInferenceJobId, + failure_reason: &str, +) -> AnyhowResult<()> { + let query_result = sqlx::query!( + r#" +UPDATE tts_inference_jobs +SET + status = "dead", + failure_reason = ?, + retry_at = NULL +WHERE id = ? + "#, + failure_reason.to_string(), + job_id.0, + ) + .execute(pool) + .await?; + + Ok(()) +} diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mod.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/mod.rs new file mode 100644 index 0000000000..7aaa57a6d6 --- /dev/null +++ b/crates/lib/database_queries/src/tts/tts_inference_jobs/mod.rs @@ -0,0 +1,7 @@ +pub mod _keys; +pub mod insert_tts_inference_job; +pub mod list_available_tts_inference_jobs; +pub mod mark_tts_inference_job_done; +pub mod mark_tts_inference_job_failure; +pub mod mark_tts_inference_job_pending_and_grab_lock; +pub mod mark_tts_inference_job_permanently_dead; diff --git a/crates/lib/database_queries/src/tts/tts_models/get_tts_model_for_inference.rs b/crates/lib/database_queries/src/tts/tts_models/get_tts_model_for_inference.rs new file mode 100644 index 0000000000..a5d131a7c4 --- /dev/null +++ b/crates/lib/database_queries/src/tts/tts_models/get_tts_model_for_inference.rs @@ -0,0 +1,103 @@ +use anyhow::anyhow; +use chrono::{Utc, DateTime}; +use log::warn; +use sqlx::MySqlPool; +use sqlx; + +// TODO: Can probably just reuse another query. + +pub struct TtsModelForInferenceRecord { + pub model_token: String, + pub tts_model_type: String, + + pub maybe_default_pretrained_vocoder: Option, + + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + + pub title: String, + pub private_bucket_hash: String, + pub created_at: DateTime, + pub updated_at: DateTime, + pub mod_deleted_at: Option>, + pub user_deleted_at: Option>, +} + + +#[derive(Clone, Debug)] +pub enum TtsModelForInferenceError { + ModelNotFound, + ModelDeleted, + DatabaseError { reason: String }, +} + +impl std::fmt::Display for TtsModelForInferenceError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TtsModelForInferenceError::ModelNotFound => write!(f, "ModelNotFound"), + TtsModelForInferenceError::ModelDeleted => write!(f, "ModelDeleted"), + TtsModelForInferenceError::DatabaseError { reason} => + write!(f, "Database error: {:?}", reason), + } + } +} + +impl std::error::Error for TtsModelForInferenceError {} + +pub async fn get_tts_model_for_inference( + pool: &MySqlPool, + model_token: &str +) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_model = sqlx::query_as!( + TtsModelForInferenceRecord, + r#" +SELECT + tts.token as model_token, + tts.tts_model_type, + tts.maybe_default_pretrained_vocoder, + tts.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + tts.title, + tts.private_bucket_hash, + tts.created_at, + tts.updated_at, + tts.user_deleted_at, + tts.mod_deleted_at +FROM tts_models as tts +JOIN users +ON users.token = tts.creator_user_token +WHERE tts.token = ? + "#, + &model_token + ) + .fetch_one(pool) + .await; // TODO: This will return error if it doesn't exist + + let model : TtsModelForInferenceRecord = match maybe_model { + Ok(model) => model, + Err(err) => { + match err { + sqlx::Error::RowNotFound => { + return Err(TtsModelForInferenceError::ModelNotFound); + }, + _ => { + warn!("tts model query error: {:?}", err); + return Err(TtsModelForInferenceError::DatabaseError { + reason: format!("Mysql error: {:?}", err) + }); + } + } + } + }; + + if model.mod_deleted_at.is_some() || model.user_deleted_at.is_some() { + return Err(TtsModelForInferenceError::ModelDeleted); + } + + Ok(model) +} diff --git a/crates/lib/database_queries/src/tts/tts_models/mod.rs b/crates/lib/database_queries/src/tts/tts_models/mod.rs new file mode 100644 index 0000000000..0bd84e0e68 --- /dev/null +++ b/crates/lib/database_queries/src/tts/tts_models/mod.rs @@ -0,0 +1 @@ +pub mod get_tts_model_for_inference; \ No newline at end of file diff --git a/crates/lib/database_queries/src/tts/tts_results/insert_tts_result.rs b/crates/lib/database_queries/src/tts/tts_results/insert_tts_result.rs new file mode 100644 index 0000000000..be881d67f3 --- /dev/null +++ b/crates/lib/database_queries/src/tts/tts_results/insert_tts_result.rs @@ -0,0 +1,165 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::vocoder_type::VocoderType; +use crate::tokens::Tokens; +use crate::tts::tts_inference_jobs::list_available_tts_inference_jobs::AvailableTtsInferenceJob; +use log::warn; +use sqlx::MySqlPool; +use sqlx; +use std::path::Path; + +/// Used to give user-facing order to logged in user inference requests +pub struct SyntheticIdRecord { + pub next_id: i64, +} + +pub async fn insert_tts_result>( + pool: &MySqlPool, + job: &AvailableTtsInferenceJob, + text_hash: &str, + pretrained_vocoder_used: VocoderType, + bucket_audio_results_path: P, + bucket_spectrogram_results_path: P, + file_size_bytes: u64, + duration_millis: u64 +) -> AnyhowResult<(u64, String)> +{ + let inference_result_token = Tokens::new_tts_result()?; + + let bucket_audio_result_path = &bucket_audio_results_path + .as_ref() + .display() + .to_string(); + + let bucket_spectrogram_result_path = &bucket_spectrogram_results_path + .as_ref() + .display() + .to_string(); + + let normalized_inference_text = job.raw_inference_text.clone(); // TODO + + let maybe_creator_user_token = job.maybe_creator_user_token.clone(); + let mut maybe_creator_synthetic_id : Option = None; + + let mut transaction = pool.begin().await?; + + if let Some(creator_user_token) = maybe_creator_user_token.as_deref() { + let query_result = sqlx::query!( + r#" +INSERT INTO tts_result_synthetic_ids +SET + user_token = ?, + next_id = 1 +ON DUPLICATE KEY UPDATE + user_token = ?, + next_id = next_id + 1 + "#, + creator_user_token, + creator_user_token + ) + .execute(&mut transaction) + .await; + + match query_result { + Ok(_) => {}, + Err(err) => { + //transaction.rollback().await?; + warn!("Transaction failure: {:?}", err); + } + } + + let query_result = sqlx::query_as!( + SyntheticIdRecord, + r#" +SELECT + next_id +FROM + tts_result_synthetic_ids +WHERE + user_token = ? +LIMIT 1 + "#, + creator_user_token, + ) + .fetch_one(&mut transaction) + .await; + + let record : SyntheticIdRecord = match query_result { + Ok(record) => record, + Err(err) => { + warn!("Transaction failure: {:?}", err); + transaction.rollback().await?; + return Err(anyhow!("Transaction failure: {:?}", err)); + } + }; + + let next_id = record.next_id as u64; + maybe_creator_synthetic_id = Some(next_id); + } + + let record_id = { + let query_result = sqlx::query!( + r#" +INSERT INTO tts_results +SET + token = ?, + + model_token = ?, + maybe_pretrained_vocoder_used = ?, + raw_inference_text = ?, + raw_inference_text_hash_sha2 = ?, + normalized_inference_text = ?, + + maybe_creator_user_token = ?, + maybe_creator_synthetic_id = ?, + + creator_ip_address = ?, + creator_set_visibility = ?, + + public_bucket_wav_audio_path = ?, + public_bucket_spectrogram_path = ?, + + file_size_bytes = ?, + duration_millis = ? + "#, + inference_result_token, + job.model_token.clone(), + pretrained_vocoder_used.to_str(), + job.raw_inference_text.clone(), + text_hash, + normalized_inference_text, + + maybe_creator_user_token, + maybe_creator_synthetic_id, + + job.creator_ip_address.clone(), + job.creator_set_visibility.to_str(), + + bucket_audio_result_path, + bucket_spectrogram_result_path, + + file_size_bytes, + duration_millis + ) + .execute(&mut transaction) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + //transaction.rollback().await?; + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + record_id + }; + + transaction.commit().await?; + + Ok((record_id, inference_result_token.clone())) +} + diff --git a/crates/lib/database_queries/src/tts/tts_results/mod.rs b/crates/lib/database_queries/src/tts/tts_results/mod.rs new file mode 100644 index 0000000000..933ac21575 --- /dev/null +++ b/crates/lib/database_queries/src/tts/tts_results/mod.rs @@ -0,0 +1 @@ +pub mod insert_tts_result; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 2f18feac06..81bd4cac4e 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -9,7 +9,7 @@ use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; use database_queries::tokens::Tokens; -use database_queries::tts::insert_tts_inference_job::TtsInferenceJobInsertBuilder; +use database_queries::tts::tts_inference_jobs::insert_tts_inference_job::TtsInferenceJobInsertBuilder; use database_queries::twitch_oauth::find::{TwitchOauthTokenRecord, TwitchOauthTokenFinder}; use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; use database_queries::twitch_pubsub::insert_bits::TwitchPubsubBitsInsertBuilder; diff --git a/crates/service/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs b/crates/service/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs index 149f956239..4e990e78b3 100644 --- a/crates/service/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs +++ b/crates/service/storyteller_web/src/http_clients/tts_inference_sidecar_client.rs @@ -1,7 +1,7 @@ use actix_web::http::header; use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use crate::database::enums::vocoder_type::VocoderType; +use database_queries::column_types::vocoder_type::VocoderType; use hyper::client::Client; use hyper::{Body, Request}; use log::info; diff --git a/crates/service/storyteller_web/src/job_queries/mod.rs b/crates/service/storyteller_web/src/job_queries/mod.rs index fe1fc8930b..c25ac9cc81 100644 --- a/crates/service/storyteller_web/src/job_queries/mod.rs +++ b/crates/service/storyteller_web/src/job_queries/mod.rs @@ -1,4 +1,6 @@ +// TODO(2022-02-10): Move these queries to the 'database_queries' crate +// I moved tts_inference_job queries out already. + pub mod tts_download_job_queries; -pub mod tts_inference_job_queries; pub mod w2l_download_job_queries; pub mod w2l_inference_job_queries; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs b/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs index b3a4a85560..3108000ded 100644 --- a/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs @@ -9,6 +9,9 @@ use database_queries::tokens::Tokens; use sqlx::MySqlPool; use std::path::Path; +// TODO(2022-02-10): Move these queries to the 'database_queries' crate +// I moved tts_inference_job queries out already. + /// table: tts_model_upload_jobs #[derive(Debug)] pub struct TtsUploadJobRecord { diff --git a/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs b/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs deleted file mode 100644 index 31cb75f245..0000000000 --- a/crates/service/storyteller_web/src/job_queries/tts_inference_job_queries.rs +++ /dev/null @@ -1,504 +0,0 @@ -//! NB: This seems required for sqlx to generate the cached queries. -//! Sqlx's prepare needs a *single* binary to work against, so we need to -//! include these in the main binary to generate all the queries. - -use anyhow::anyhow; -use chrono::{Utc, DateTime}; -use container_common::anyhow_result::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::enums::vocoder_type::VocoderType; -use database_queries::tokens::Tokens; -use log::{warn, info}; -use sqlx::{MySqlPool, Transaction, MySql}; -use std::error; -use std::path::Path; - -/// table: tts_inference_jobs -#[derive(Debug)] -pub struct TtsInferenceJobRecord { - pub id: i64, - pub inference_job_token: String, - pub uuid_idempotency_token: String, - - pub model_token: String, - pub raw_inference_text: String, - - pub creator_ip_address: String, - pub maybe_creator_user_token: Option, - pub creator_set_visibility: RecordVisibility, - - pub is_from_api: i8, - pub is_for_twitch: i8, - - pub status: String, // TODO - pub priority_level: u8, - pub attempt_count: i32, - pub failure_reason: Option, - - pub created_at: chrono::DateTime, - pub updated_at: chrono::DateTime, - pub retry_at: Option>, -} - -pub async fn query_tts_inference_job_records( - pool: &MySqlPool, - num_records: u32 -) -> AnyhowResult> { - - let job_records = sqlx::query_as!( - TtsInferenceJobRecord, - r#" -SELECT - id, - token AS inference_job_token, - uuid_idempotency_token, - - model_token, - raw_inference_text, - - creator_ip_address, - maybe_creator_user_token, - creator_set_visibility as `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, - - is_from_api, - is_for_twitch, - - status, - priority_level, - attempt_count, - failure_reason, - created_at, - updated_at, - retry_at -FROM tts_inference_jobs -WHERE - ( - status IN ("pending", "attempt_failed") - ) - AND - ( - retry_at IS NULL - OR - retry_at < CURRENT_TIMESTAMP - ) - LIMIT ? - "#, - num_records, - ) - .fetch_all(pool) - .await?; - - Ok(job_records) -} - - -pub struct TtsInferenceLockRecord { - id: i64, - status: String, -} - -pub async fn grab_job_lock_and_mark_pending( - pool: &MySqlPool, - job: &TtsInferenceJobRecord -) -> AnyhowResult { - - // NB: We use transactions and "SELECT ... FOR UPDATE" to simulate mutexes. - let mut transaction = pool.begin().await?; - - let maybe_record = sqlx::query_as!( - TtsInferenceLockRecord, - r#" -SELECT - id, - status -FROM tts_inference_jobs -WHERE id = ? -FOR UPDATE - "#, - job.id, - ) - .fetch_one(&mut transaction) - .await; - - let record : TtsInferenceLockRecord = match maybe_record { - Ok(record) => record, - Err(err) => { - match err { - sqlx::Error::RowNotFound => { - return Err(anyhow!("could not job")); - }, - _ => { - return Err(anyhow!("query error")); - } - } - } - }; - - let can_transact = match record.status.as_ref() { - "pending" => true, // It's okay for us to take the lock. - "attempt_failed" => true, // We can retry. - "started" => false, // Job in progress (another job beat us, and we can't take the lock) - "complete_success" => false, // Job already complete - "complete_failure" => false, // Job already complete (permanently dead; no need to retry) - "dead" => false, // Job already complete (permanently dead; retries exhausted) - _ => false, // Future-proof - }; - - if !can_transact { - transaction.rollback().await?; - return Ok(false); - } - - let _acquire_lock = sqlx::query!( - r#" -UPDATE tts_inference_jobs -SET - status = 'started', - attempt_count = attempt_count + 1, - retry_at = NOW() + interval 2 minute -WHERE id = ? - "#, - job.id, - ) - .execute(&mut transaction) - .await?; - - transaction.commit().await?; - - Ok(true) -} - -pub async fn mark_tts_inference_job_failure( - pool: &MySqlPool, - job: &TtsInferenceJobRecord, - failure_reason: &str, - max_attempts: i32 -) -> AnyhowResult<()> { - - // statuses: "attempt_failed", "complete_failure", "dead" - let mut next_status = "attempt_failed"; - - if job.attempt_count >= max_attempts { - // NB: Job attempt count is incremented at start - next_status = "dead"; - } - - let query_result = sqlx::query!( - r#" -UPDATE tts_inference_jobs -SET - status = ?, - failure_reason = ?, - retry_at = NOW() + interval 1 minute -WHERE id = ? - "#, - next_status, - failure_reason.to_string(), - job.id, - ) - .execute(pool) - .await?; - - Ok(()) -} - -pub async fn mark_tts_inference_job_permanently_dead( - pool: &MySqlPool, - job: &TtsInferenceJobRecord, - failure_reason: &str, -) -> AnyhowResult<()> { - let query_result = sqlx::query!( - r#" -UPDATE tts_inference_jobs -SET - status = "dead", - failure_reason = ?, - retry_at = NULL -WHERE id = ? - "#, - failure_reason.to_string(), - job.id, - ) - .execute(pool) - .await?; - - Ok(()) -} - -pub async fn mark_tts_inference_job_done( - pool: &MySqlPool, - job: &TtsInferenceJobRecord, - success: bool, - maybe_result_token: Option<&str> -) -> AnyhowResult<()> { - let status = if success { "complete_success" } else { "complete_failure" }; - - let query_result = sqlx::query!( - r#" -UPDATE tts_inference_jobs -SET - status = ?, - on_success_result_token = ?, - failure_reason = NULL, - retry_at = NULL -WHERE id = ? - "#, - status, - maybe_result_token, - job.id - ) - .execute(pool) - .await?; - - Ok(()) -} - -pub struct SyntheticIdRecord { - pub next_id: i64, -} - -pub async fn insert_tts_result>( - pool: &MySqlPool, - job: &TtsInferenceJobRecord, - text_hash: &str, - pretrained_vocoder_used: VocoderType, - bucket_audio_results_path: P, - bucket_spectrogram_results_path: P, - file_size_bytes: u64, - duration_millis: u64 -) -> AnyhowResult<(u64, String)> -{ - let inference_result_token = Tokens::new_tts_result()?; - - let bucket_audio_result_path = &bucket_audio_results_path - .as_ref() - .display() - .to_string(); - - let bucket_spectrogram_result_path = &bucket_spectrogram_results_path - .as_ref() - .display() - .to_string(); - - let normalized_inference_text = job.raw_inference_text.clone(); // TODO - - let maybe_creator_user_token = job.maybe_creator_user_token.clone(); - let mut maybe_creator_synthetic_id : Option = None; - - let mut transaction = pool.begin().await?; - - if let Some(creator_user_token) = maybe_creator_user_token.as_deref() { - let query_result = sqlx::query!( - r#" -INSERT INTO tts_result_synthetic_ids -SET - user_token = ?, - next_id = 1 -ON DUPLICATE KEY UPDATE - user_token = ?, - next_id = next_id + 1 - "#, - creator_user_token, - creator_user_token - ) - .execute(&mut transaction) - .await; - - match query_result { - Ok(_) => {}, - Err(err) => { - //transaction.rollback().await?; - warn!("Transaction failure: {:?}", err); - } - } - - let query_result = sqlx::query_as!( - SyntheticIdRecord, - r#" -SELECT - next_id -FROM - tts_result_synthetic_ids -WHERE - user_token = ? -LIMIT 1 - "#, - creator_user_token, - ) - .fetch_one(&mut transaction) - .await; - - let record : SyntheticIdRecord = match query_result { - Ok(record) => record, - Err(err) => { - warn!("Transaction failure: {:?}", err); - transaction.rollback().await?; - return Err(anyhow!("Transaction failure: {:?}", err)); - } - }; - - let next_id = record.next_id as u64; - maybe_creator_synthetic_id = Some(next_id); - } - - let record_id = { - let query_result = sqlx::query!( - r#" -INSERT INTO tts_results -SET - token = ?, - - model_token = ?, - maybe_pretrained_vocoder_used = ?, - raw_inference_text = ?, - raw_inference_text_hash_sha2 = ?, - normalized_inference_text = ?, - - maybe_creator_user_token = ?, - maybe_creator_synthetic_id = ?, - - creator_ip_address = ?, - creator_set_visibility = ?, - - public_bucket_wav_audio_path = ?, - public_bucket_spectrogram_path = ?, - - file_size_bytes = ?, - duration_millis = ? - "#, - inference_result_token, - job.model_token.clone(), - pretrained_vocoder_used.to_str(), - job.raw_inference_text.clone(), - text_hash, - normalized_inference_text, - - maybe_creator_user_token, - maybe_creator_synthetic_id, - - job.creator_ip_address.clone(), - job.creator_set_visibility.to_str(), - - bucket_audio_result_path, - bucket_spectrogram_result_path, - - file_size_bytes, - duration_millis - ) - .execute(&mut transaction) - .await; - - let record_id = match query_result { - Ok(res) => { - res.last_insert_id() - }, - Err(err) => { - // TODO: handle better - //transaction.rollback().await?; - return Err(anyhow!("Mysql error: {:?}", err)); - } - }; - - record_id - }; - - transaction.commit().await?; - - Ok((record_id, inference_result_token.clone())) -} - -pub struct TtsModelForInferenceRecord { - pub model_token: String, - pub tts_model_type: String, - - pub maybe_default_pretrained_vocoder: Option, - - pub creator_user_token: String, - pub creator_username: String, - pub creator_display_name: String, - - pub title: String, - pub private_bucket_hash: String, - pub created_at: DateTime, - pub updated_at: DateTime, - pub mod_deleted_at: Option>, - pub user_deleted_at: Option>, -} - - -#[derive(Clone, Debug)] -pub enum TtsModelForInferenceError { - ModelNotFound, - ModelDeleted, - DatabaseError { reason: String }, -} - -impl std::fmt::Display for TtsModelForInferenceError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - TtsModelForInferenceError::ModelNotFound => write!(f, "ModelNotFound"), - TtsModelForInferenceError::ModelDeleted => write!(f, "ModelDeleted"), - TtsModelForInferenceError::DatabaseError { reason} => - write!(f, "Database error: {:?}", reason), - } - } -} - -impl std::error::Error for TtsModelForInferenceError {} - -pub async fn get_tts_model_by_token( - pool: &MySqlPool, - model_token: &str -) -> Result -{ - // NB: Lookup failure is Err(RowNotFound). - // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_model = sqlx::query_as!( - TtsModelForInferenceRecord, - r#" -SELECT - tts.token as model_token, - tts.tts_model_type, - tts.maybe_default_pretrained_vocoder, - tts.creator_user_token, - users.username as creator_username, - users.display_name as creator_display_name, - tts.title, - tts.private_bucket_hash, - tts.created_at, - tts.updated_at, - tts.user_deleted_at, - tts.mod_deleted_at -FROM tts_models as tts -JOIN users -ON users.token = tts.creator_user_token -WHERE tts.token = ? - "#, - &model_token - ) - .fetch_one(pool) - .await; // TODO: This will return error if it doesn't exist - - let model : TtsModelForInferenceRecord = match maybe_model { - Ok(model) => model, - Err(err) => { - match err { - sqlx::Error::RowNotFound => { - return Err(TtsModelForInferenceError::ModelNotFound); - }, - _ => { - warn!("tts model query error: {:?}", err); - return Err(TtsModelForInferenceError::DatabaseError { - reason: format!("Mysql error: {:?}", err) - }); - } - } - } - }; - - if model.mod_deleted_at.is_some() || model.user_deleted_at.is_some() { - return Err(TtsModelForInferenceError::ModelDeleted); - } - - Ok(model) -} diff --git a/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs b/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs index c49a5128d1..fd8c8347ad 100644 --- a/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs @@ -8,6 +8,9 @@ use container_common::anyhow_result::AnyhowResult; use database_queries::tokens::Tokens; use sqlx::MySqlPool; +// TODO(2022-02-10): Move these queries to the 'database_queries' crate +// I moved tts_inference_job queries out already. + /// table: w2l_template_upload_jobs #[derive(Debug)] pub struct W2lTemplateUploadJobRecord { diff --git a/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs b/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs index 11f698fa1c..3e4b2878fb 100644 --- a/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs @@ -11,6 +11,9 @@ use log::{warn, info}; use sqlx::MySqlPool; use std::path::Path; +// TODO(2022-02-10): Move these queries to the 'database_queries' crate +// I moved tts_inference_job queries out already. + /// table: w2l_inference_jobs #[derive(Debug)] pub struct W2lInferenceJobRecord { diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index 2875a043c0..68d92f37c4 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -24,19 +24,8 @@ use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use container_common::token::random_uuid::generate_random_uuid; use crate::common_env::CommonEnv; -use crate::database::enums::vocoder_type::VocoderType; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; -use crate::job_queries::tts_inference_job_queries::TtsInferenceJobRecord; -use crate::job_queries::tts_inference_job_queries::TtsModelForInferenceError; -use crate::job_queries::tts_inference_job_queries::TtsModelForInferenceRecord; -use crate::job_queries::tts_inference_job_queries::get_tts_model_by_token; -use crate::job_queries::tts_inference_job_queries::grab_job_lock_and_mark_pending; -use crate::job_queries::tts_inference_job_queries::insert_tts_result; -use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_done; -use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_failure; -use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_permanently_dead; -use crate::job_queries::tts_inference_job_queries::query_tts_inference_job_records; use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; @@ -55,6 +44,14 @@ use crate::util::noop_logger::NoOpLogger; use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; +use database_queries::column_types::vocoder_type::VocoderType; +use database_queries::tts::tts_inference_jobs::list_available_tts_inference_jobs::{AvailableTtsInferenceJob, list_available_tts_inference_jobs}; +use database_queries::tts::tts_inference_jobs::mark_tts_inference_job_done::mark_tts_inference_job_done; +use database_queries::tts::tts_inference_jobs::mark_tts_inference_job_failure::mark_tts_inference_job_failure; +use database_queries::tts::tts_inference_jobs::mark_tts_inference_job_pending_and_grab_lock::mark_tts_inference_job_pending_and_grab_lock; +use database_queries::tts::tts_inference_jobs::mark_tts_inference_job_permanently_dead::mark_tts_inference_job_permanently_dead; +use database_queries::tts::tts_models::get_tts_model_for_inference::{get_tts_model_for_inference, TtsModelForInferenceError, TtsModelForInferenceRecord}; +use database_queries::tts::tts_results::insert_tts_result::insert_tts_result; use log::{warn, info, error}; use newrelic_telemetry::Client as NewRelicClient; use newrelic_telemetry::ClientBuilder; @@ -332,12 +329,10 @@ async fn main_loop(inferencer: Inferencer) { loop { let num_records = inferencer.job_batch_size; - let query_result = query_tts_inference_job_records( - &inferencer.mysql_pool, - num_records) - .await; + let maybe_available_jobs + = list_available_tts_inference_jobs(&inferencer.mysql_pool, num_records).await; - let jobs = match query_result { + let jobs = match maybe_available_jobs { Ok(jobs) => jobs, Err(e) => { warn!("Error querying jobs: {:?}", e); @@ -385,7 +380,7 @@ async fn main_loop(inferencer: Inferencer) { /// Process a batch of jobs, returning the count of cold-cache skipped jobs. async fn process_jobs( inferencer: &Inferencer, - jobs: Vec, + jobs: Vec, ) -> AnyhowResult> { let mut batch_spans = Vec::new(); @@ -423,7 +418,7 @@ async fn process_jobs( let _r = mark_tts_inference_job_permanently_dead( &inferencer.mysql_pool, - &job, + job.id, failure_reason ).await; } else { @@ -552,26 +547,18 @@ impl std::error::Error for ModelStateError {} impl ModelState { /// Query the model details and see if the model file is on the filesystem in one go. pub async fn query_model_and_check_filesystem( - job: &TtsInferenceJobRecord, + job: &AvailableTtsInferenceJob, mysql_pool: &MySqlPool, semi_persistent_cache: &SemiPersistentCacheDir, virtual_cache: &SyncVirtualLfuCache, ) -> Result { info!("Looking up TTS model by token: {}", &job.model_token); - let tts_model = get_tts_model_by_token( + let tts_model = get_tts_model_for_inference( &mysql_pool, &job.model_token ).await?; - /*let model_record = match query_result { - Ok(model) => model, - Err(e) => { - error!("TTS model lookup error: {}, reason: {:?}", &job.model_token, &e); - return Err(e) - }, - };*/ - let tts_synthesizer_fs_path = semi_persistent_cache.tts_synthesizer_model_path( &tts_model.model_token); @@ -595,7 +582,7 @@ impl ModelState { async fn process_job( inferencer: &Inferencer, - job: &TtsInferenceJobRecord, + job: &AvailableTtsInferenceJob, model_record: &TtsModelForInferenceRecord, ) -> AnyhowResult<(Span, Span)> { @@ -641,10 +628,11 @@ async fn process_job( // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // - let lock_acquired = grab_job_lock_and_mark_pending(&inferencer.mysql_pool, job).await?; + let lock_acquired = + mark_tts_inference_job_pending_and_grab_lock(&inferencer.mysql_pool, job.id).await?; if !lock_acquired { - warn!("Could not acquire job lock for: {}", &job.id); + warn!("Could not acquire job lock for: {:?}", &job.id); let duration = start.elapsed(); since_creation_span.set_attribute("status", "failure"); @@ -773,7 +761,7 @@ async fn process_job( // ==================== WRITE TEXT TO FILE ==================== // - let temp_dir = format!("temp_tts_inference_{}", job.id); + let temp_dir = format!("temp_tts_inference_{}", job.id.0); let temp_dir = TempDir::new(&temp_dir)?; // NB: Exists until it goes out of scope. let text_input_fs_path = temp_dir.path().join("inference_input.txt"); @@ -905,7 +893,7 @@ async fn process_job( info!("Marking job complete..."); mark_tts_inference_job_done( &inferencer.mysql_pool, - job, + job.id, true, Some(&inference_result_token) ).await?; @@ -922,7 +910,7 @@ async fn process_job( redis_logger.log_status("done")?; - info!("Job {} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}", + info!("Job {:?} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}", job.id, id); let duration = start.elapsed(); From c488773bc18d6665641c57bf36b613bd9c6e2e1b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Feb 2022 01:03:47 -0500 Subject: [PATCH 0898/1042] re-codegen db --- sqlx-data.json | 1009 +++++++++++++++++++++++++----------------------- 1 file changed, 524 insertions(+), 485 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index d1376ffdef..1704668eba 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1,5 +1,479 @@ { "db": "MySQL", + "150c98c00f4cd6954428736583eae913be578c4e69dac317d4445bc0e97743c9": { + "query": "\nSELECT\n next_id\nFROM\n tts_result_synthetic_ids\nWHERE\n user_token = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "next_id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 20 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + } + }, + "28cf5843d68d6cb0e67eb316ef9e7e5fb05a6be2433f86c081576b279167508d": { + "query": "\nINSERT INTO tts_result_synthetic_ids\nSET\n user_token = ?,\n next_id = 1\nON DUPLICATE KEY UPDATE\n user_token = ?,\n next_id = next_id + 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "4f61f621682ba255814f8a55455ea2b7fba9b6c85affaf87abfe2f81e11322f0": { + "query": "\nSELECT\n id,\n status\nFROM tts_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + }, + "53f0792791317fcd4038308a6630f541d5508ae913cbfa9bc3d4081fba9a048a": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.maybe_default_pretrained_vocoder,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at,\n tts.user_deleted_at,\n tts.mod_deleted_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "maybe_default_pretrained_vocoder", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 3, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "private_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 10, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 11, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + true, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, + "57cf798a0fe32a1d6d0349ee61baac6e536fe3e1a3aa7bd93b34c519812bf70a": { + "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n model_token,\n raw_inference_text,\n\n creator_ip_address,\n maybe_creator_user_token,\n creator_set_visibility as `creator_set_visibility: crate::column_types::record_visibility::RecordVisibility`,\n\n is_from_api,\n is_for_twitch,\n\n status as `status: crate::column_types::job_status::JobStatus`,\n priority_level,\n attempt_count,\n failure_reason,\n\n created_at,\n updated_at,\n retry_at\nFROM tts_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "inference_job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "creator_set_visibility: crate::column_types::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 8, + "name": "is_from_api", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_for_twitch", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "status: crate::column_types::job_status::JobStatus", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 11, + "name": "priority_level", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 33 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 12, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 11 + } + }, + { + "ordinal": 13, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 14, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 15, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 16, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + true, + false, + false, + true + ] + } + }, "591bda610a0fe952cafc5d97808757da8652133082e94ad15ab063db9d4e2f44": { "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", "describe": { @@ -40,6 +514,36 @@ "nullable": [] } }, + "65ce50a3aafe979e77a541d576228d7fcfe76e33400427d01382da36c1585edb": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "ab6f25eb0f6340d49aa705835ed428434d8b6ab14b0740f6a5ebe7841b9ece20": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "b8746ae4ff2d4dcd54563eff64877f924bb119681ad78f1300146e00b762ca4f": { + "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n maybe_pretrained_vocoder_used = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n\n file_size_bytes = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 14 + }, + "nullable": [] + } + }, "d7ac9f84e92d6b2111162a674284c0b9fa50c718bedfb8ab7238ca0a7b0875e6": { "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", "describe": { @@ -50,6 +554,26 @@ "nullable": [] } }, + "e0d442313b7861debaef984c5809647bd5df85106c194b66e864674b6af95330": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = \"dead\",\n failure_reason = ?,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "f19300f4e8434c2669dac062e5a79c5ca71767c7380aab0ab40b51be8ff449d8": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 1 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, "0033c76be0ea1c99069f1a8df1ce400c7064526feaf5dc88930da0718a885b19": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.is_banned as creator_is_banned,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n w2l.title,\n w2l.description_markdown,\n w2l.description_rendered_html,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", "describe": { @@ -1904,31 +2428,6 @@ ] } }, - "150c98c00f4cd6954428736583eae913be578c4e69dac317d4445bc0e97743c9": { - "query": "\nSELECT\n next_id\nFROM\n tts_result_synthetic_ids\nWHERE\n user_token = ?\nLIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "next_id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 20 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false - ] - } - }, "17c8d210d93ee9c0d8e8422eed605c9150f19f9bf72affcf111b15a67a5b9e70": { "query": "\nUPDATE w2l_templates\nSET\n is_public_listing_approved = ?,\n is_locked_from_user_modification = ?,\n is_locked_from_use = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -2007,16 +2506,6 @@ "nullable": [] } }, - "28cf5843d68d6cb0e67eb316ef9e7e5fb05a6be2433f86c081576b279167508d": { - "query": "\nINSERT INTO tts_result_synthetic_ids\nSET\n user_token = ?,\n next_id = 1\nON DUPLICATE KEY UPDATE\n user_token = ?,\n next_id = next_id + 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -2047,200 +2536,6 @@ "nullable": [] } }, - "2c7795f817e475290e32a2bf1061cdbbf9f08c9bcdf8b03d92b860a5ce15985e": { - "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n model_token,\n raw_inference_text,\n\n creator_ip_address,\n maybe_creator_user_token,\n creator_set_visibility as `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM tts_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "inference_job_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "uuid_idempotency_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 3, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "raw_inference_text", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 5, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 8, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 9, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 10, - "name": "failure_reason", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 11, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 12, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "retry_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - true, - false, - false, - false, - true, - false, - false, - true - ] - } - }, "314904656a333caa75a8644e73f4d338b7bd6f68c7ea9e37b5bc7f7b257bf34d": { "query": "\nSELECT 1 as does_exist\nFROM user_badges\nWHERE\n user_token = ?\nAND\n badge_slug = ?\nLIMIT 1\n ", "describe": { @@ -3055,44 +3350,6 @@ ] } }, - "4f61f621682ba255814f8a55455ea2b7fba9b6c85affaf87abfe2f81e11322f0": { - "query": "\nSELECT\n id,\n status\nFROM tts_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false - ] - } - }, "4ff1b1e0cc36f6c2f1a9b099f38751ae7f35666be939b4074bdfc98e031cd6ac": { "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n users.is_banned as maybe_creator_is_banned,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n template_users.is_banned as maybe_template_creator_is_banned,\n\n w2l_results.creator_set_visibility,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n\n w2l_results.creator_ip_address,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n AND w2l_results.user_deleted_at IS NULL\n AND w2l_results.mod_deleted_at IS NULL\n ", "describe": { @@ -3443,174 +3700,6 @@ ] } }, - "53f0792791317fcd4038308a6630f541d5508ae913cbfa9bc3d4081fba9a048a": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.maybe_default_pretrained_vocoder,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at,\n tts.user_deleted_at,\n tts.mod_deleted_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "maybe_default_pretrained_vocoder", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 3, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "private_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 10, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 11, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "5473d3857478d001688998056377fefe4bcc792477f303b41fa3cf64f79170f3": { "query": "\n INSERT INTO tts_category_assignments\n SET\n model_token = ?,\n category_token = ?,\n category_removal_user_token = ?,\n ip_address_creation = ?,\n ip_address_last_update = ?,\n deleted_at = CURRENT_TIMESTAMP\n ON DUPLICATE KEY UPDATE\n model_token = ?,\n category_token = ?,\n category_removal_user_token = ?,\n ip_address_last_update = ?,\n deleted_at = CURRENT_TIMESTAMP\n ", "describe": { @@ -4672,16 +4761,6 @@ "nullable": [] } }, - "65ce50a3aafe979e77a541d576228d7fcfe76e33400427d01382da36c1585edb": { - "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, "663506988f146de6f559e5df4f4a2b607e9421a69bb57db1f656774bf8b375db": { "query": "\nUPDATE w2l_results\nSET\n mod_deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -6205,16 +6284,6 @@ ] } }, - "ab6f25eb0f6340d49aa705835ed428434d8b6ab14b0740f6a5ebe7841b9ece20": { - "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, "ab981965f518a8986aaaba41ed821fcb438db4c6156cbb26f639921225a88d59": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = \"dead\",\n failure_reason = ?,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { @@ -6935,16 +7004,6 @@ ] } }, - "b8746ae4ff2d4dcd54563eff64877f924bb119681ad78f1300146e00b762ca4f": { - "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n maybe_pretrained_vocoder_used = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n\n file_size_bytes = ?,\n duration_millis = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 14 - }, - "nullable": [] - } - }, "ba31fefd3e96fd238fc2bcdfab64f390a17293f2c0996ae891ed8821e57512cd": { "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n users.is_banned as maybe_creator_is_banned,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n template_users.is_banned as maybe_template_creator_is_banned,\n\n w2l_results.creator_set_visibility,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n\n w2l_results.creator_ip_address,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n ", "describe": { @@ -8551,16 +8610,6 @@ "nullable": [] } }, - "e0d442313b7861debaef984c5809647bd5df85106c194b66e864674b6af95330": { - "query": "\nUPDATE tts_inference_jobs\nSET\n status = \"dead\",\n failure_reason = ?,\n retry_at = NULL\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, "e3a0bd8073c3c0d84b9ab0e2e1ad8371f14b10801b7e4658b73c42ce1b27fa0b": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.is_public_listing_approved IS TRUE\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -9233,16 +9282,6 @@ ] } }, - "f19300f4e8434c2669dac062e5a79c5ca71767c7380aab0ab40b51be8ff449d8": { - "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 1 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, "f75b713d166398ebea2bdbde4e416dc8eb2c48dba4aa9bd164bcf5bc77237ce1": { "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n preferred_tts_result_visibility = ?,\n preferred_w2l_result_visibility = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n ip_address_last_update = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", "describe": { From cd297c1b360dc0c7ec9ae4a9d59af43a74bca2ea Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Feb 2022 02:06:40 -0500 Subject: [PATCH 0899/1042] note on linux issue --- .../{development_notes.md => engineering_notes.md} | 0 _docs/local_development_setup.md | 13 +++++++++++++ 2 files changed, 13 insertions(+) rename _docs/{development_notes.md => engineering_notes.md} (100%) diff --git a/_docs/development_notes.md b/_docs/engineering_notes.md similarity index 100% rename from _docs/development_notes.md rename to _docs/engineering_notes.md diff --git a/_docs/local_development_setup.md b/_docs/local_development_setup.md index c895cdf462..57dce2a3cc 100644 --- a/_docs/local_development_setup.md +++ b/_docs/local_development_setup.md @@ -41,6 +41,19 @@ cargo install diesel_cli \ --features mysql ``` +#### Sqlx Error on Linux when Performing Schema Migrations + +You might get this error message during migration, + +``` +Encountered unknown type for Mysql: enum +thread 'main' panicked at 'internal error: entered unreachable code: Mysql only supports a closed set of types. + If you ever see this error message please open an issue at https://github.com/diesel-rs/diesel containing a dump of your schema definition.', diesel_cli/src/print_schema.rs:310:17 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +``` + +I haven't found the cause (it doesn't happen on Mac), but the migrations appear to work regardless of the error message. + #### Linux database notes If MySql in local dev can't be connected to, reset the accounts: From 6c131e2da185dd81a4e0a5a1f0332e56c0f449aa Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Feb 2022 03:14:20 -0500 Subject: [PATCH 0900/1042] additional cors for dev, fix twitch cheer case sensitivity --- crates/lib/http_server_common/src/cors.rs | 2 ++ crates/lib/twitch_common/src/cheers.rs | 14 +++++++++++++- .../twitch_pubsub_user_subscriber_thread.rs | 3 ++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/lib/http_server_common/src/cors.rs b/crates/lib/http_server_common/src/cors.rs index 9d66ff0259..e484753922 100644 --- a/crates/lib/http_server_common/src/cors.rs +++ b/crates/lib/http_server_common/src/cors.rs @@ -15,6 +15,8 @@ pub fn build_common_cors_config() -> Cors { .allowed_origin("http://localhost:54321") .allowed_origin("http://localhost:5555") .allowed_origin("http://localhost:7000") + .allowed_origin("http://localhost:7000") + .allowed_origin("http://localhost:7001") .allowed_origin("http://localhost:8000") .allowed_origin("http://localhost:8080") .allowed_origin("https://api.jungle.horse") diff --git a/crates/lib/twitch_common/src/cheers.rs b/crates/lib/twitch_common/src/cheers.rs index d3d211a7df..4d55ede5a0 100644 --- a/crates/lib/twitch_common/src/cheers.rs +++ b/crates/lib/twitch_common/src/cheers.rs @@ -74,7 +74,10 @@ static TWITCH_CHEER_REGEX : Lazy = Lazy::new(|| { } let regex = regex_pieces.join("|"); - let regex = format!("({})", regex); + + // NB: (?i) is case insensitive search + // https://docs.rs/regex/1.5.4/regex/#structs + let regex = format!("(?i)({})", regex); Regex::new(®ex).unwrap() }); @@ -91,4 +94,13 @@ mod tests { let r = remove_cheers("Kappa1 SeemsGood100 no more Cheer1 cheers"); assert_eq!(&r, "no more cheers"); } + + #[test] + fn test_remove_cheers_case_insensitive() { + let r = remove_cheers("testing cheer1"); + assert_eq!(&r, "testing"); + + let r = remove_cheers("TESTING CHEER1"); + assert_eq!(&r, "TESTING"); + } } diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 81bd4cac4e..7acef33ec4 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -557,7 +557,8 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { async fn write_tts_inference_event(&mut self, tts_text: &str) -> AnyhowResult<()> { let sanitized_text = remove_cheers(tts_text); let job_token = Tokens::new_tts_inference_job()?; - let model_token = "TM:7wbtjphx8h8v"; // "Mario *" voice. + //let model_token = "TM:7wbtjphx8h8v"; // "Mario *" voice (prod) + let model_token = "TM:40m3aqtt41y0"; // "Wakko" voice (dev) let mut builder = TtsInferenceJobInsertBuilder::new_for_internal_tts() .set_job_token(&job_token) From 93c85073861215b7cad4fca42a2825237402737e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Feb 2022 23:12:56 -0500 Subject: [PATCH 0901/1042] error reporting --- .../tts/get_tts_inference_job_status.rs | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index a0cf4eda44..141dbeaa79 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; @@ -10,8 +10,7 @@ use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; -use log::{info, warn, log}; +use log::{info, warn, log, error}; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2::{Pool, PooledConnection}; use r2d2_redis::redis::{Commands, RedisError, RedisResult}; @@ -22,6 +21,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::borrow::BorrowMut; +use std::fmt; use std::ops::Deref; use std::sync::Arc; @@ -61,7 +61,7 @@ pub struct GetTtsInferenceStatusSuccessResponse { pub state: TtsInferenceJobStatusForResponse, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum GetTtsInferenceStatusError { ServerError, } @@ -101,6 +101,14 @@ impl ResponseError for GetTtsInferenceStatusError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GetTtsInferenceStatusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + + pub async fn get_tts_inference_job_status_handler( http_request: HttpRequest, path: Path, @@ -144,10 +152,11 @@ WHERE jobs.token = ? Err(err) => { match err { sqlx::Error::RowNotFound => { + warn!("tts job record not found"); return Err(GetTtsInferenceStatusError::ServerError); }, _ => { - warn!("tts template query error: {:?}", err); + error!("tts job query error: {:?}", err); return Err(GetTtsInferenceStatusError::ServerError); } } @@ -157,7 +166,7 @@ WHERE jobs.token = ? let mut redis = server_state.redis_pool .get() .map_err(|e| { - warn!("redis error: {:?}", e); + error!("redis error: {:?}", e); GetTtsInferenceStatusError::ServerError })?; @@ -168,12 +177,12 @@ WHERE jobs.token = ? }, Ok(None) => None, Err(e) => { - warn!("redis error: {:?}", e); + error!("redis error: {:?}", e); None // Fail open }, }; - let model_for_response = TtsInferenceJobStatusForResponse { + let record_for_response = TtsInferenceJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), maybe_extra_status_description, @@ -189,11 +198,14 @@ WHERE jobs.token = ? let response = GetTtsInferenceStatusSuccessResponse { success: true, - state: model_for_response, + state: record_for_response, }; let body = serde_json::to_string(&response) - .map_err(|e| GetTtsInferenceStatusError::ServerError)?; + .map_err(|e| { + error!("error returning response: {:?}", e); + GetTtsInferenceStatusError::ServerError + })?; Ok(HttpResponse::Ok() .content_type("application/json") From 262024ec28df6a2814f76515649d329e56cd3841 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Feb 2022 23:25:55 -0500 Subject: [PATCH 0902/1042] Insert twitch and priority metadata --- .../insert_tts_inference_job.rs | 30 +++++++++++++++++++ .../twitch_pubsub_user_subscriber_thread.rs | 2 ++ 2 files changed, 32 insertions(+) diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/insert_tts_inference_job.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/insert_tts_inference_job.rs index c54900b0a0..ff95681774 100644 --- a/crates/lib/database_queries/src/tts/tts_inference_jobs/insert_tts_inference_job.rs +++ b/crates/lib/database_queries/src/tts/tts_inference_jobs/insert_tts_inference_job.rs @@ -19,6 +19,9 @@ pub struct TtsInferenceJobInsertBuilder { // ========== Optional ========== maybe_creator_user_token: Option, creator_set_visibility: Option, + is_from_api: bool, + is_for_twitch: bool, + priority_level: u8, // Priority is 0 by default and optionally increases } impl TtsInferenceJobInsertBuilder { @@ -33,6 +36,9 @@ impl TtsInferenceJobInsertBuilder { creator_ip_address: None, maybe_creator_user_token: None, creator_set_visibility: None, + is_from_api: false, + is_for_twitch: false, + priority_level: 0, } } @@ -49,6 +55,9 @@ impl TtsInferenceJobInsertBuilder { creator_ip_address: Some("127.0.0.1".to_string()), // hidden | public | ... creator_set_visibility: Some("hidden".to_string()), + is_from_api: false, + is_for_twitch: false, + priority_level: 0, } } @@ -87,6 +96,21 @@ impl TtsInferenceJobInsertBuilder { self } + pub fn set_is_from_api(mut self, value: bool) -> Self { + self.is_from_api = value; + self + } + + pub fn set_is_for_twitch(mut self, value: bool) -> Self { + self.is_for_twitch = value; + self + } + + pub fn set_priority_level(mut self, value: u8) -> Self { + self.priority_level = value; + self + } + pub async fn insert(&mut self, mysql_pool: &MySqlPool) -> AnyhowResult<()> { // TODO: These should be custom error types with a custom macro to make this easy. let job_token = self.job_token @@ -124,6 +148,9 @@ SET maybe_creator_user_token = ?, creator_ip_address = ?, creator_set_visibility = ?, + is_from_api = ?, + is_for_twitch = ?, + priority_level = ?, status = "pending" "#, job_token, @@ -133,6 +160,9 @@ SET self.maybe_creator_user_token.clone(), creator_ip_address, creator_set_visibility, + self.is_from_api, + self.is_for_twitch, + self.priority_level, ); let query_result = query.execute(mysql_pool) diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 7acef33ec4..80fcffa007 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -561,6 +561,8 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { let model_token = "TM:40m3aqtt41y0"; // "Wakko" voice (dev) let mut builder = TtsInferenceJobInsertBuilder::new_for_internal_tts() + .set_is_for_twitch(true) + .set_priority_level(1) .set_job_token(&job_token) .set_model_token(model_token) .set_raw_inference_text(&sanitized_text); From 0d52e963079353121c94fc0d9be986d09369d8dc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Feb 2022 23:40:01 -0500 Subject: [PATCH 0903/1042] re-codegen --- sqlx-data.json | 110 ++++++++++++++++++++++++------------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 1704668eba..b0f774d375 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -398,7 +398,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 3 } }, { @@ -474,16 +474,6 @@ ] } }, - "591bda610a0fe952cafc5d97808757da8652133082e94ad15ab063db9d4e2f44": { - "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 7 - }, - "nullable": [] - } - }, "5b8c906b09cc7068dcad9eb837b1146893b63c3ce7c8dbefea9129849448c3f6": { "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", "describe": { @@ -554,6 +544,16 @@ "nullable": [] } }, + "def4202f63874e7b04d8f8a5da028a9aacc4dfe19b8898d3d6820d393a633cf3": { + "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n is_from_api = ?,\n is_for_twitch = ?,\n priority_level = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, "e0d442313b7861debaef984c5809647bd5df85106c194b66e864674b6af95330": { "query": "\nUPDATE tts_inference_jobs\nSET\n status = \"dead\",\n failure_reason = ?,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { @@ -707,7 +707,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -719,7 +719,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -731,7 +731,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -1176,7 +1176,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -1188,7 +1188,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -1200,7 +1200,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -1502,7 +1502,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 3 } }, { @@ -1716,7 +1716,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -1728,7 +1728,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -1740,7 +1740,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -2549,7 +2549,7 @@ "bits": 129 }, "char_set": 63, - "max_size": 1 + "max_size": 2 } } ], @@ -2736,7 +2736,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -2748,7 +2748,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -2760,7 +2760,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -3567,7 +3567,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -3579,7 +3579,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -3591,7 +3591,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -3603,7 +3603,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -3757,7 +3757,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 3 } }, { @@ -3847,7 +3847,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 3 } }, { @@ -4360,7 +4360,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -4372,7 +4372,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -4384,7 +4384,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -5379,7 +5379,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 3 } }, { @@ -5539,7 +5539,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -5551,7 +5551,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -5563,7 +5563,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -6351,7 +6351,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 3 } }, { @@ -6666,7 +6666,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -6678,7 +6678,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -6929,7 +6929,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 3 } }, { @@ -7221,7 +7221,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -7233,7 +7233,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -7245,7 +7245,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -7257,7 +7257,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -7439,7 +7439,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -7451,7 +7451,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -7463,7 +7463,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -8259,7 +8259,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -8271,7 +8271,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -8695,7 +8695,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -8707,7 +8707,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -8719,7 +8719,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -9329,7 +9329,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 3 } }, { From cf11c115f94095ed9591297797539aeec8b206db Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Feb 2022 07:45:07 -0500 Subject: [PATCH 0904/1042] add raw inference text to inference job status --- .../tts/get_tts_inference_job_status.rs | 28 +- sqlx-data.json | 297 +++++++++--------- 2 files changed, 172 insertions(+), 153 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index 141dbeaa79..640076528a 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -49,7 +49,9 @@ pub struct TtsInferenceJobStatusForResponse { pub model_token: String, pub tts_model_type: String, - pub title: String, + pub title: String, // Name of the TTS model + + pub raw_inference_text: String, // User text pub created_at: DateTime, pub updated_at: DateTime, @@ -78,8 +80,9 @@ pub struct TtsInferenceJobStatusRecord { pub model_token: String, pub tts_model_type: String, + pub title: String, // Name of the TTS model - pub title: String, + pub raw_inference_text: String, // User text pub created_at: DateTime, pub updated_at: DateTime, @@ -131,6 +134,8 @@ SELECT tts.tts_model_type, tts.title, + jobs.raw_inference_text, + jobs.created_at, jobs.updated_at @@ -183,17 +188,18 @@ WHERE jobs.token = ? }; let record_for_response = TtsInferenceJobStatusForResponse { - job_token: record.job_token.clone(), - status: record.status.clone(), + job_token: record.job_token, + status: record.status, maybe_extra_status_description, attempt_count: record.attempt_count as u8, - maybe_result_token: record.maybe_result_token.clone(), - maybe_public_bucket_wav_audio_path: record.maybe_public_bucket_wav_audio_path.clone(), - model_token: record.model_token.clone(), - tts_model_type: record.tts_model_type.clone(), - title: record.title.clone(), - created_at: record.created_at.clone(), - updated_at: record.updated_at.clone(), + maybe_result_token: record.maybe_result_token, + maybe_public_bucket_wav_audio_path: record.maybe_public_bucket_wav_audio_path, + model_token: record.model_token, + tts_model_type: record.tts_model_type, + title: record.title, + raw_inference_text: record.raw_inference_text, + created_at: record.created_at, + updated_at: record.updated_at, }; let response = GetTtsInferenceStatusSuccessResponse { diff --git a/sqlx-data.json b/sqlx-data.json index b0f774d375..fa665e7061 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -6169,6 +6169,161 @@ "nullable": [] } }, + "a2b586dd2120f29dd26281626f7381e292c821f8e826b9bc584b92ffa45035b2": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,\n\n jobs.model_token,\n tts.tts_model_type,\n tts.title,\n\n jobs.raw_inference_text,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_inference_jobs as jobs\nJOIN tts_models as tts\n ON tts.token = jobs.model_token\nLEFT OUTER JOIN tts_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 3, + "name": "maybe_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 7, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 9, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 10, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + false, + false, + false, + false, + false, + false + ] + } + }, "a60b239dff945c0089ded216dc6b4ea1bd7acb2125671bebdd34e5e1312270ce": { "query": "\nSELECT\n users.username,\n users.display_name,\n users.email_gravatar_hash as gravatar_hash,\n creator_user_token,\n count(*) as uploaded_count\nFROM tts_models\nJOIN users\nON\n users.token = creator_user_token\nWHERE\n users.is_banned IS FALSE\n AND tts_models.user_deleted_at IS NULL\n AND tts_models.mod_deleted_at IS NULL\nGROUP BY creator_user_token\nORDER BY uploaded_count desc\nLIMIT 25;\n ", "describe": { @@ -9292,148 +9447,6 @@ "nullable": [] } }, - "fc0045b6fe338a903fa9f35307457d8dbb32c5838a4783fddf0c9beef7fdb4a5": { - "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,\n\n jobs.model_token,\n tts.tts_model_type,\n tts.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_inference_jobs as jobs\nJOIN tts_models as tts\n ON tts.token = jobs.model_token\nLEFT OUTER JOIN tts_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 2, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 3 - } - }, - { - "ordinal": 3, - "name": "maybe_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "maybe_public_bucket_wav_audio_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 5, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 7, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - true, - false, - false, - false, - false, - false - ] - } - }, "fd31f7abe386147c07f862d668c9e56eb3d809c4f0a61b7debf9b2e3785a4c83": { "query": "\nINSERT INTO model_categories\nSET\n token = ?,\n uuid_idempotency_token = ?,\n model_type = ?,\n name = ?,\n\n creator_user_token = ?,\n creator_ip_address_creation = ?,\n creator_ip_address_last_update = ?,\n\n is_mod_approved = ?,\n maybe_mod_user_token = ?,\n can_directly_have_models = ?,\n can_have_subcategories = ?,\n can_only_mods_apply = ?\n ", "describe": { From dd9733f9a8607e3e2c9ee1af94ab29d293b8344b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Feb 2022 10:18:24 -0500 Subject: [PATCH 0905/1042] Add featured TTS models (not yet done) + refactoring --- .../down.sql | 6 ++ .../up.sql | 10 +++ .../tts_tables/tts_models.sql | 4 + crates/lib/database_queries/src/tts/mod.rs | 1 + ...fetch_and_build_tts_model_category_map.rs} | 11 ++- .../src/tts/tts_category_assignments/mod.rs | 1 + .../src/tts/tts_models}/list_tts_models.rs | 89 ++++++++++++------- .../src/tts/tts_models/mod.rs | 3 +- .../src/database/queries/categories/mod.rs | 3 +- .../src/database/queries/mod.rs | 1 - .../endpoints/tts/list_tts_models.rs | 12 ++- .../endpoints/users/list_user_tts_models.rs | 16 ++-- 12 files changed, 104 insertions(+), 53 deletions(-) create mode 100644 _migrations/2022-02-11-145145_alter_tts_model_add_featured/down.sql create mode 100644 _migrations/2022-02-11-145145_alter_tts_model_add_featured/up.sql rename crates/{service/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs => lib/database_queries/src/tts/tts_category_assignments/fetch_and_build_tts_model_category_map.rs} (91%) create mode 100644 crates/lib/database_queries/src/tts/tts_category_assignments/mod.rs rename crates/{service/storyteller_web/src/database/queries => lib/database_queries/src/tts/tts_models}/list_tts_models.rs (75%) diff --git a/_migrations/2022-02-11-145145_alter_tts_model_add_featured/down.sql b/_migrations/2022-02-11-145145_alter_tts_model_add_featured/down.sql new file mode 100644 index 0000000000..3de7f7caf0 --- /dev/null +++ b/_migrations/2022-02-11-145145_alter_tts_model_add_featured/down.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_models DROP COLUMN is_front_page_featured; +ALTER TABLE tts_models DROP COLUMN is_twitch_featured; diff --git a/_migrations/2022-02-11-145145_alter_tts_model_add_featured/up.sql b/_migrations/2022-02-11-145145_alter_tts_model_add_featured/up.sql new file mode 100644 index 0000000000..9eb3c98d56 --- /dev/null +++ b/_migrations/2022-02-11-145145_alter_tts_model_add_featured/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_models + ADD COLUMN is_front_page_featured BOOLEAN NOT NULL DEFAULT FALSE + AFTER version_string, + + ADD COLUMN is_twitch_featured BOOLEAN NOT NULL DEFAULT FALSE + AFTER is_front_page_featured; diff --git a/_migrations_squashed/tts_tables/tts_models.sql b/_migrations_squashed/tts_tables/tts_models.sql index 41a03f70a4..71c6924e91 100644 --- a/_migrations_squashed/tts_tables/tts_models.sql +++ b/_migrations_squashed/tts_tables/tts_models.sql @@ -83,6 +83,10 @@ CREATE TABLE tts_models ( -- Optional; eg. if you train the model more than once. version_string VARCHAR(255) DEFAULT NULL, + -- Whether the voice features on the front of FakeYou or Storyteller Twitch TTS + is_front_page_featured BOOLEAN NOT NULL DEFAULT FALSE, + is_twitch_featured BOOLEAN NOT NULL DEFAULT FALSE, + -- ========== IMMUTABLE PROVENANCE AND METADATA ========== -- Where the file was originally downloaded (if it was downloaded) diff --git a/crates/lib/database_queries/src/tts/mod.rs b/crates/lib/database_queries/src/tts/mod.rs index 00c9ce9ee9..3002a2e0e7 100644 --- a/crates/lib/database_queries/src/tts/mod.rs +++ b/crates/lib/database_queries/src/tts/mod.rs @@ -1,3 +1,4 @@ +pub mod tts_category_assignments; pub mod tts_inference_jobs; pub mod tts_models; pub mod tts_results; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs b/crates/lib/database_queries/src/tts/tts_category_assignments/fetch_and_build_tts_model_category_map.rs similarity index 91% rename from crates/service/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs rename to crates/lib/database_queries/src/tts/tts_category_assignments/fetch_and_build_tts_model_category_map.rs index bfdd0ccd3d..e0c9066bb9 100644 --- a/crates/service/storyteller_web/src/database/queries/categories/list_tts_model_category_assignments.rs +++ b/crates/lib/database_queries/src/tts/tts_category_assignments/fetch_and_build_tts_model_category_map.rs @@ -1,20 +1,19 @@ use anyhow::anyhow; -use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; use log::warn; use sqlx::MySqlPool; use std::collections::{HashSet, HashMap}; - /// Map of model_token => vec #[derive(Serialize, Default)] pub struct TtsModelCategoryMap { pub model_to_category_tokens: HashMap>, } -pub async fn fetch_tts_model_category_map(mysql_pool: &MySqlPool) - -> AnyhowResult -{ +/// Fetch a map of every model to all of its categories. +pub async fn fetch_and_build_tts_model_category_map( + mysql_pool: &MySqlPool +) -> AnyhowResult { let assignments = list_tts_model_category_assignments(mysql_pool).await?; let mut map : HashMap> = HashMap::new(); @@ -23,7 +22,7 @@ pub async fn fetch_tts_model_category_map(mysql_pool: &MySqlPool) if !map.contains_key(&assignment.tts_model_token) { map.insert(assignment.tts_model_token.clone(), HashSet::new()); } - + map.get_mut(&assignment.tts_model_token).map(|hashset| { hashset.insert(assignment.category_token.clone()) }); diff --git a/crates/lib/database_queries/src/tts/tts_category_assignments/mod.rs b/crates/lib/database_queries/src/tts/tts_category_assignments/mod.rs new file mode 100644 index 0000000000..84f8c71e26 --- /dev/null +++ b/crates/lib/database_queries/src/tts/tts_category_assignments/mod.rs @@ -0,0 +1 @@ +pub mod fetch_and_build_tts_model_category_map; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/list_tts_models.rs b/crates/lib/database_queries/src/tts/tts_models/list_tts_models.rs similarity index 75% rename from crates/service/storyteller_web/src/database/queries/list_tts_models.rs rename to crates/lib/database_queries/src/tts/tts_models/list_tts_models.rs index 1988c1cc93..9d5071faf2 100644 --- a/crates/service/storyteller_web/src/database/queries/list_tts_models.rs +++ b/crates/lib/database_queries/src/tts/tts_models/list_tts_models.rs @@ -1,10 +1,12 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; -use crate::database::helpers::boolean_converters::i8_to_bool; +use crate::helpers::boolean_converters::i8_to_bool; use log::{warn, info}; use sqlx::MySqlPool; +// FIXME: This is the old style of query scoping and shouldn't be copied. + #[derive(Serialize)] pub struct TtsModelRecordForList { pub model_token: String, @@ -17,28 +19,16 @@ pub struct TtsModelRecordForList { pub creator_display_name: String, pub creator_gravatar_hash: String, - pub is_locked_from_use: bool, // converted + pub is_locked_from_use: bool, + + pub is_front_page_featured: bool, + pub is_twitch_featured: bool, pub created_at: DateTime, pub updated_at: DateTime, } -struct RawTtsModelRecordForList { - pub model_token: String, - pub tts_model_type: String, - - pub title: String, - - pub creator_user_token: String, - pub creator_username: String, - pub creator_display_name: String, - pub creator_gravatar_hash: String, - - pub is_locked_from_use: i8, // NB: needs conversion - - pub created_at: DateTime, - pub updated_at: DateTime, -} +// FIXME: This is the old style of query scoping and shouldn't be copied. pub async fn list_tts_models( mysql_pool: &MySqlPool, @@ -57,7 +47,7 @@ pub async fn list_tts_models( }, }; - let models : Vec = match maybe_models { + let models : Vec = match maybe_models { Ok(models) => models, Err(err) => { match err { @@ -75,16 +65,18 @@ pub async fn list_tts_models( Ok(models.into_iter() .map(|model| { TtsModelRecordForList { - model_token: model.model_token.clone(), - tts_model_type: model.tts_model_type.clone(), - creator_user_token: model.creator_user_token.clone(), - creator_username: model.creator_username.clone(), - creator_display_name: model.creator_display_name.clone(), - creator_gravatar_hash: model.creator_gravatar_hash.clone(), - title: model.title.clone(), + model_token: model.model_token, + tts_model_type: model.tts_model_type, + creator_user_token: model.creator_user_token, + creator_username: model.creator_username, + creator_display_name: model.creator_display_name, + creator_gravatar_hash: model.creator_gravatar_hash, + title: model.title, is_locked_from_use: i8_to_bool(model.is_locked_from_use), - created_at: model.created_at.clone(), - updated_at: model.updated_at.clone(), + is_front_page_featured: i8_to_bool(model.is_front_page_featured), + is_twitch_featured: i8_to_bool(model.is_twitch_featured), + created_at: model.created_at, + updated_at: model.updated_at, } }) .collect::>()) @@ -93,13 +85,13 @@ pub async fn list_tts_models( async fn list_tts_models_for_all_creators( mysql_pool: &MySqlPool, allow_mod_disabled: bool -) -> AnyhowResult> { +) -> AnyhowResult> { // TODO: There has to be a better way. // Sqlx doesn't like anything except string literals. let maybe_models = if !allow_mod_disabled { info!("listing tts models for everyone; mod-approved only"); sqlx::query_as!( - RawTtsModelRecordForList, + InternalRawTtsModelRecordForList, r#" SELECT tts.token as model_token, @@ -110,6 +102,8 @@ SELECT users.email_gravatar_hash as creator_gravatar_hash, tts.title, tts.is_locked_from_use, + tts.is_front_page_featured, + tts.is_twitch_featured, tts.created_at, tts.updated_at FROM tts_models as tts @@ -125,7 +119,7 @@ WHERE } else { info!("listing tts models for everyone; all"); sqlx::query_as!( - RawTtsModelRecordForList, + InternalRawTtsModelRecordForList, r#" SELECT tts.token as model_token, @@ -136,6 +130,8 @@ SELECT users.email_gravatar_hash as creator_gravatar_hash, tts.title, tts.is_locked_from_use, + tts.is_front_page_featured, + tts.is_twitch_featured, tts.created_at, tts.updated_at FROM tts_models as tts @@ -156,13 +152,13 @@ async fn list_tts_models_creator_scoped( mysql_pool: &MySqlPool, scope_creator_username: &str, allow_mod_disabled: bool -) -> AnyhowResult> { +) -> AnyhowResult> { // TODO: There has to be a better way. // Sqlx doesn't like anything except string literals. let maybe_models = if !allow_mod_disabled { info!("listing tts models for user; mod-approved only"); sqlx::query_as!( - RawTtsModelRecordForList, + InternalRawTtsModelRecordForList, r#" SELECT tts.token as model_token, @@ -173,6 +169,8 @@ SELECT users.email_gravatar_hash as creator_gravatar_hash, tts.title, tts.is_locked_from_use, + tts.is_front_page_featured, + tts.is_twitch_featured, tts.created_at, tts.updated_at FROM tts_models as tts @@ -191,7 +189,7 @@ WHERE } else { info!("listing tts models for user; all"); sqlx::query_as!( - RawTtsModelRecordForList, + InternalRawTtsModelRecordForList, r#" SELECT tts.token as model_token, @@ -202,6 +200,8 @@ SELECT users.email_gravatar_hash as creator_gravatar_hash, tts.title, tts.is_locked_from_use, + tts.is_front_page_featured, + tts.is_twitch_featured, tts.created_at, tts.updated_at FROM tts_models as tts @@ -220,3 +220,24 @@ WHERE Ok(maybe_models) } + +struct InternalRawTtsModelRecordForList { + pub model_token: String, + pub tts_model_type: String, + + pub title: String, + + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub creator_gravatar_hash: String, + + pub is_locked_from_use: i8, // bool + + pub is_front_page_featured: i8, // bool + pub is_twitch_featured: i8, // bool + + pub created_at: DateTime, + pub updated_at: DateTime, +} + diff --git a/crates/lib/database_queries/src/tts/tts_models/mod.rs b/crates/lib/database_queries/src/tts/tts_models/mod.rs index 0bd84e0e68..18d9bce34e 100644 --- a/crates/lib/database_queries/src/tts/tts_models/mod.rs +++ b/crates/lib/database_queries/src/tts/tts_models/mod.rs @@ -1 +1,2 @@ -pub mod get_tts_model_for_inference; \ No newline at end of file +pub mod get_tts_model_for_inference; +pub mod list_tts_models; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/categories/mod.rs b/crates/service/storyteller_web/src/database/queries/categories/mod.rs index 7692b77bed..8012259411 100644 --- a/crates/service/storyteller_web/src/database/queries/categories/mod.rs +++ b/crates/service/storyteller_web/src/database/queries/categories/mod.rs @@ -1,2 +1 @@ -pub mod get_category_by_token; -pub mod list_tts_model_category_assignments; \ No newline at end of file +pub mod get_category_by_token; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/mod.rs b/crates/service/storyteller_web/src/database/queries/mod.rs index 4045881345..7d34d54f97 100644 --- a/crates/service/storyteller_web/src/database/queries/mod.rs +++ b/crates/service/storyteller_web/src/database/queries/mod.rs @@ -9,7 +9,6 @@ pub mod delete_w2l_result; pub mod delete_w2l_template; pub mod get_user_profile_by_username; pub mod list_ip_bans; -pub mod list_tts_models; pub mod list_user_badges; pub mod list_user_roles; pub mod list_w2l_templates; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs index 56112f3254..443ec91587 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs @@ -6,11 +6,12 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; -use crate::database::queries::categories::list_tts_model_category_assignments::fetch_tts_model_category_map; -use crate::database::queries::list_tts_models::list_tts_models; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; +use database_queries::tts::tts_category_assignments::fetch_and_build_tts_model_category_map::fetch_and_build_tts_model_category_map; +use database_queries::tts::tts_models::list_tts_models::list_tts_models; +use lexical_sort::natural_lexical_cmp; use log::{info, warn, log, error}; use regex::Regex; use sqlx::MySqlPool; @@ -20,7 +21,6 @@ use sqlx::mysql::MySqlDatabaseError; use std::collections::HashSet; use std::fmt; use std::sync::Arc; -use lexical_sort::natural_lexical_cmp; #[derive(Serialize, Clone)] pub struct TtsModelRecordForResponse { @@ -31,6 +31,8 @@ pub struct TtsModelRecordForResponse { pub creator_display_name: String, pub creator_gravatar_hash: String, pub title: String, + pub is_front_page_featured: bool, + pub is_twitch_featured: bool, /// Category assignments /// From non-deleted, mod-approved categories only @@ -141,7 +143,7 @@ async fn get_all_models(mysql_pool: &MySqlPool) -> AnyhowResult AnyhowResult, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum ListTtsModelsForUserError { ServerError, } @@ -54,6 +53,13 @@ impl ResponseError for ListTtsModelsForUserError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListTtsModelsForUserError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn list_user_tts_models_handler( http_request: HttpRequest, path: Path, From 701c0139f9c9cdeed9c95731540fa3f0fd24a7c1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Feb 2022 10:41:00 -0500 Subject: [PATCH 0906/1042] move query to db lib --- .../src/tts/tts_models/get_tts_model.rs} | 143 ++++++++++-------- .../src/tts/tts_models/mod.rs | 1 + .../src/database/queries/mod.rs | 1 - .../categories/assign_tts_category.rs | 4 +- .../endpoints/tts/delete_tts_model.rs | 17 ++- .../endpoints/tts/edit_tts_model.rs | 19 ++- .../endpoints/tts/edit_tts_result.rs | 14 +- .../endpoints/tts/get_tts_model.rs | 5 +- 8 files changed, 118 insertions(+), 86 deletions(-) rename crates/{service/storyteller_web/src/database/queries/query_tts_model.rs => lib/database_queries/src/tts/tts_models/get_tts_model.rs} (76%) diff --git a/crates/service/storyteller_web/src/database/queries/query_tts_model.rs b/crates/lib/database_queries/src/tts/tts_models/get_tts_model.rs similarity index 76% rename from crates/service/storyteller_web/src/database/queries/query_tts_model.rs rename to crates/lib/database_queries/src/tts/tts_models/get_tts_model.rs index bbc3b4f41a..80f7e2beba 100644 --- a/crates/service/storyteller_web/src/database/queries/query_tts_model.rs +++ b/crates/lib/database_queries/src/tts/tts_models/get_tts_model.rs @@ -1,18 +1,14 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use crate::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::enums::vocoder_type::VocoderType; -use crate::database::helpers::boolean_converters::i8_to_bool; -use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; -use derive_more::{Display, Error}; -use log::{info, warn, log}; -use regex::Regex; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::record_visibility::RecordVisibility; +use crate::column_types::vocoder_type::VocoderType; +use crate::helpers::boolean_converters::i8_to_bool; +use log::warn; use sqlx::MySqlPool; -use sqlx::error::DatabaseError; -use sqlx::error::Error::Database; -use sqlx::mysql::MySqlDatabaseError; -use std::sync::Arc; + +// FIXME: This is the old style of query scoping and shouldn't be copied. +// The moderator-only fields are good practice, though. #[derive(Serialize)] pub struct TtsModelRecordForResponse { @@ -30,6 +26,9 @@ pub struct TtsModelRecordForResponse { pub description_markdown: String, pub description_rendered_html: String, + pub is_front_page_featured: bool, + pub is_twitch_featured: bool, + pub creator_set_visibility: RecordVisibility, pub is_locked_from_use: bool, @@ -52,39 +51,10 @@ pub struct TtsModelModeratorFields { pub mod_deleted_at: Option>, } -#[derive(Serialize)] -pub struct TtsModelRecordRaw { - pub model_token: String, - pub tts_model_type: String, - pub maybe_default_pretrained_vocoder: Option, - pub text_preprocessing_algorithm: String, +// FIXME: This is the old style of query scoping and shouldn't be copied. +// The moderator-only fields are good practice, though. - pub creator_user_token: String, - pub creator_username: String, - pub creator_display_name: String, - pub creator_gravatar_hash: String, - pub creator_is_banned: i8, - - pub title: String, - pub description_markdown: String, - pub description_rendered_html: String, - - pub creator_set_visibility: String, - - pub is_locked_from_use: i8, - pub is_locked_from_user_modification: i8, - - pub created_at: DateTime, - pub updated_at: DateTime, - - // Moderator fields - pub creator_ip_address_creation: String, - pub creator_ip_address_last_update: String, - pub user_deleted_at: Option>, - pub mod_deleted_at: Option>, -} - -pub async fn select_tts_model_by_token( +pub async fn get_tts_model_by_token( tts_model_token: &str, can_see_deleted: bool, mysql_pool: &MySqlPool @@ -96,7 +66,7 @@ pub async fn select_tts_model_by_token( select_without_deleted(tts_model_token, mysql_pool).await }; - let model : TtsModelRecordRaw = match maybe_record { + let model : InternalTtsModelRecordRaw = match maybe_record { Ok(model) => model, Err(ref err) => { match err { @@ -118,30 +88,32 @@ pub async fn select_tts_model_by_token( } let model_for_response = TtsModelRecordForResponse { - model_token: model.model_token.clone(), - tts_model_type: model.tts_model_type.clone(), + model_token: model.model_token, + tts_model_type: model.tts_model_type, maybe_default_pretrained_vocoder: maybe_vocoder, - text_preprocessing_algorithm: model.text_preprocessing_algorithm.clone(), - creator_user_token: model.creator_user_token.clone(), - creator_username: model.creator_username.clone(), - creator_display_name: model.creator_display_name.clone(), - creator_gravatar_hash: model.creator_gravatar_hash.clone(), - title: model.title.clone(), - description_markdown: model.description_markdown.clone(), - description_rendered_html: model.description_rendered_html.clone(), + text_preprocessing_algorithm: model.text_preprocessing_algorithm, + creator_user_token: model.creator_user_token, + creator_username: model.creator_username, + creator_display_name: model.creator_display_name, + creator_gravatar_hash: model.creator_gravatar_hash, + title: model.title, + description_markdown: model.description_markdown, + description_rendered_html: model.description_rendered_html, // NB: Fail open/public with creator_set_visibility since we're already looking at it + is_front_page_featured: i8_to_bool(model.is_front_page_featured), + is_twitch_featured: i8_to_bool(model.is_twitch_featured), creator_set_visibility: RecordVisibility::from_str(&model.creator_set_visibility) .unwrap_or(RecordVisibility::Public), is_locked_from_use: i8_to_bool(model.is_locked_from_use), is_locked_from_user_modification: i8_to_bool(model.is_locked_from_user_modification), - created_at: model.created_at.clone(), - updated_at: model.updated_at.clone(), + created_at: model.created_at, + updated_at: model.updated_at, maybe_moderator_fields: Some(TtsModelModeratorFields { creator_is_banned: i8_to_bool(model.creator_is_banned), - creator_ip_address_creation: model.creator_ip_address_creation.clone(), - creator_ip_address_last_update: model.creator_ip_address_last_update.clone(), - user_deleted_at: model.user_deleted_at.clone(), - mod_deleted_at: model.mod_deleted_at.clone(), + creator_ip_address_creation: model.creator_ip_address_creation, + creator_ip_address_last_update: model.creator_ip_address_last_update, + user_deleted_at: model.user_deleted_at, + mod_deleted_at: model.mod_deleted_at, }), }; @@ -151,9 +123,9 @@ pub async fn select_tts_model_by_token( async fn select_including_deleted( tts_model_token: &str, mysql_pool: &MySqlPool -) -> Result { +) -> Result { sqlx::query_as!( - TtsModelRecordRaw, + InternalTtsModelRecordRaw, r#" SELECT tts.token as model_token, @@ -171,6 +143,9 @@ SELECT tts.description_markdown, tts.description_rendered_html, + tts.is_front_page_featured, + tts.is_twitch_featured, + tts.creator_set_visibility, tts.is_locked_from_use, @@ -198,9 +173,9 @@ WHERE tts.token = ? async fn select_without_deleted( tts_model_token: &str, mysql_pool: &MySqlPool -) -> Result { +) -> Result { sqlx::query_as!( - TtsModelRecordRaw, + InternalTtsModelRecordRaw, r#" SELECT tts.token as model_token, @@ -218,6 +193,9 @@ SELECT tts.description_markdown, tts.description_rendered_html, + tts.is_front_page_featured, + tts.is_twitch_featured, + tts.creator_set_visibility, tts.is_locked_from_use, @@ -244,3 +222,38 @@ WHERE .fetch_one(mysql_pool) .await // TODO: This will return error if it doesn't exist } + +#[derive(Serialize)] +struct InternalTtsModelRecordRaw { + pub model_token: String, + pub tts_model_type: String, + pub maybe_default_pretrained_vocoder: Option, + pub text_preprocessing_algorithm: String, + + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub creator_gravatar_hash: String, + pub creator_is_banned: i8, + + pub title: String, + pub description_markdown: String, + pub description_rendered_html: String, + + pub is_front_page_featured: i8, + pub is_twitch_featured: i8, + + pub creator_set_visibility: String, + + pub is_locked_from_use: i8, + pub is_locked_from_user_modification: i8, + + pub created_at: DateTime, + pub updated_at: DateTime, + + // Moderator fields + pub creator_ip_address_creation: String, + pub creator_ip_address_last_update: String, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} diff --git a/crates/lib/database_queries/src/tts/tts_models/mod.rs b/crates/lib/database_queries/src/tts/tts_models/mod.rs index 18d9bce34e..d9c349ff7d 100644 --- a/crates/lib/database_queries/src/tts/tts_models/mod.rs +++ b/crates/lib/database_queries/src/tts/tts_models/mod.rs @@ -1,2 +1,3 @@ +pub mod get_tts_model; pub mod get_tts_model_for_inference; pub mod list_tts_models; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/mod.rs b/crates/service/storyteller_web/src/database/queries/mod.rs index 7d34d54f97..11ec43f601 100644 --- a/crates/service/storyteller_web/src/database/queries/mod.rs +++ b/crates/service/storyteller_web/src/database/queries/mod.rs @@ -12,7 +12,6 @@ pub mod list_ip_bans; pub mod list_user_badges; pub mod list_user_roles; pub mod list_w2l_templates; -pub mod query_tts_model; pub mod query_tts_result; pub mod query_w2l_result; pub mod query_w2l_template; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs index f5a94b0bec..e3b99e291b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs @@ -9,7 +9,6 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use anyhow::anyhow; use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::categories::get_category_by_token::get_category_by_token; -use crate::database::queries::query_tts_model::select_tts_model_by_token; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -17,6 +16,7 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; +use database_queries::tts::tts_models::get_tts_model::get_tts_model_by_token; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; @@ -147,7 +147,7 @@ pub async fn assign_tts_category_handler( // Only mods should see deleted models (both user_* and mod_* deleted). let is_mod_that_can_see_deleted = user_session.can_delete_other_users_tts_models; - let model_lookup_result = select_tts_model_by_token( + let model_lookup_result = get_tts_model_by_token( &request.tts_model_token, is_mod_that_can_see_deleted, &server_state.mysql_pool).await; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs index 9753a57a56..a99d8d954b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; @@ -10,7 +10,6 @@ use crate::database::queries::delete_tts_model::delete_tts_model_as_mod; use crate::database::queries::delete_tts_model::delete_tts_model_as_user; use crate::database::queries::delete_tts_model::undelete_tts_model_as_mod; use crate::database::queries::delete_tts_model::undelete_tts_model_as_user; -use crate::database::queries::query_tts_model::select_tts_model_by_token; use crate::database::queries::query_tts_result::select_tts_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -18,13 +17,14 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::delete_role_disambiguation::DeleteRole; use crate::util::delete_role_disambiguation::delete_role_disambiguation; -use derive_more::{Display, Error}; +use database_queries::tts::tts_models::get_tts_model::get_tts_model_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -40,7 +40,7 @@ pub struct DeleteTtsModelRequest { as_mod: Option, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum DeleteTtsModelError { BadInput(String), NotAuthorized, @@ -70,6 +70,13 @@ impl ResponseError for DeleteTtsModelError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for DeleteTtsModelError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn delete_tts_model_handler( http_request: HttpRequest, path: Path, @@ -97,7 +104,7 @@ pub async fn delete_tts_model_handler( // Only mods should see deleted models (both user_* and mod_* deleted). let is_mod = user_session.can_delete_other_users_tts_models; - let model_query_result = select_tts_model_by_token( + let model_query_result = get_tts_model_by_token( &path.token, is_mod, &server_state.mysql_pool, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs index 0bf16df60c..459eab8ff0 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs @@ -6,22 +6,22 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::enums::vocoder_type::VocoderType; -use crate::database::queries::query_tts_model::select_tts_model_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use derive_more::{Display, Error}; +use database_queries::column_types::record_visibility::RecordVisibility; +use database_queries::column_types::vocoder_type::VocoderType; +use database_queries::tts::tts_models::get_tts_model::get_tts_model_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; use user_input_common::check_for_slurs::contains_slurs; @@ -52,7 +52,7 @@ pub struct EditTtsModelRequest { pub maybe_mod_comments: Option, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum EditTtsModelError { BadInput(String), NotAuthorized, @@ -82,6 +82,13 @@ impl ResponseError for EditTtsModelError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for EditTtsModelError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn edit_tts_model_handler( http_request: HttpRequest, path: Path, @@ -109,7 +116,7 @@ pub async fn edit_tts_model_handler( // Only mods should see deleted models (both user_* and mod_* deleted). let is_mod_that_can_see_deleted = user_session.can_delete_other_users_tts_models; - let model_lookup_result = select_tts_model_by_token( + let model_lookup_result = get_tts_model_by_token( &path.model_token, is_mod_that_can_see_deleted, &server_state.mysql_pool).await; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs index 17a9b88ea7..f20d0799cd 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs @@ -1,13 +1,12 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::queries::query_tts_model::select_tts_model_by_token; use crate::database::queries::query_tts_result::select_tts_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -15,13 +14,13 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -36,7 +35,7 @@ pub struct EditTtsResultRequest { pub creator_set_visibility: Option, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum EditTtsResultError { BadInput(String), NotAuthorized, @@ -66,6 +65,13 @@ impl ResponseError for EditTtsResultError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for EditTtsResultError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn edit_tts_inference_result_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs index d062308624..5cc5c5c713 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs @@ -8,10 +8,9 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use anyhow::Error; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::queries::query_tts_model::TtsModelRecordForResponse; -use crate::database::queries::query_tts_model::select_tts_model_by_token; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; +use database_queries::tts::tts_models::get_tts_model::{TtsModelRecordForResponse, get_tts_model_by_token}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; @@ -98,7 +97,7 @@ pub async fn get_tts_model_handler( || user_session.can_edit_other_users_tts_models; } - let model_query_result = select_tts_model_by_token( + let model_query_result = get_tts_model_by_token( &path.token, show_deleted_models, &server_state.mysql_pool From 79e61b44af709645950b151d92779ef71de4ade6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Feb 2022 10:47:28 -0500 Subject: [PATCH 0907/1042] edit featured tts model status --- .../http_server/endpoints/tts/edit_tts_model.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs index 459eab8ff0..1545a654d8 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs @@ -44,12 +44,17 @@ pub struct EditTtsModelRequest { //pub text_preprocessing_algorithm: Option, //pub vocoder_token: Option, - // ========== Moderator options ========== + // ========== Moderator options (protection) ========== pub is_public_listing_approved: Option, pub is_locked_from_user_modification: Option, pub is_locked_from_use: Option, pub maybe_mod_comments: Option, + + // ========== Moderator options (front page) ========== + + pub is_front_page_featured: Option, + pub is_twitch_featured: Option, } #[derive(Debug)] @@ -281,6 +286,9 @@ async fn update_mod_details( let is_locked_from_user_modification = request.is_locked_from_user_modification.unwrap_or(false); let is_locked_from_use = request.is_locked_from_use.unwrap_or(false); + let is_front_page_featured = request.is_front_page_featured.unwrap_or(false); + let is_twitch_featured = request.is_twitch_featured.unwrap_or(false); + let query_result = sqlx::query!( r#" UPDATE tts_models @@ -290,6 +298,8 @@ SET is_locked_from_use = ?, maybe_mod_comments = ?, maybe_mod_user_token = ?, + is_front_page_featured = ?, + is_twitch_featured = ?, version = version + 1 WHERE token = ? LIMIT 1 @@ -299,6 +309,8 @@ LIMIT 1 is_locked_from_use, request.maybe_mod_comments, mod_user_token, + is_front_page_featured, + is_twitch_featured, model_token ) .execute(mysql_pool) From 5df24165247ce356c5d3cdef4767fbf5a6b506aa Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Feb 2022 10:48:40 -0500 Subject: [PATCH 0908/1042] re-codegen db after new cols and refactors --- sqlx-data.json | 2528 +++++++++++++++++++++++++----------------------- 1 file changed, 1342 insertions(+), 1186 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index fa665e7061..fb3a4c2c5d 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1,5 +1,43 @@ { "db": "MySQL", + "12a176252df63d1e3a958362d754c828991d64ade427ab11a2025098d1581976": { + "query": "\nSELECT\n assignments.model_token AS tts_model_token,\n assignments.category_token\nFROM\n tts_category_assignments AS assignments\nJOIN\n tts_models AS tts\n ON assignments.model_token = tts.token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n AND assignments.deleted_at IS NULL\n\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "category_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false + ] + } + }, "150c98c00f4cd6954428736583eae913be578c4e69dac317d4445bc0e97743c9": { "query": "\nSELECT\n next_id\nFROM\n tts_result_synthetic_ids\nWHERE\n user_token = ?\nLIMIT 1\n ", "describe": { @@ -35,6 +73,485 @@ "nullable": [] } }, + "443757e80423a52b09f8a4fa79088f6809dad4c1125b1555ce43a382ea745a04": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 11, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, + "4732b7d8c2beeec7e7821fd0139e32a12ae94cb81de1ba6cd2fff2f506e00336": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "maybe_default_pretrained_vocoder", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 4, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "creator_is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 11, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 12, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 15, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 20, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 21, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "4f61f621682ba255814f8a55455ea2b7fba9b6c85affaf87abfe2f81e11322f0": { "query": "\nSELECT\n id,\n status\nFROM tts_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", "describe": { @@ -474,6 +991,174 @@ ] } }, + "5a17891eec52d8304af8a54f6bb7629cb71d3603146f8b2c842f6e1047c31e91": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 11, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "5b8c906b09cc7068dcad9eb837b1146893b63c3ce7c8dbefea9129849448c3f6": { "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", "describe": { @@ -494,6 +1179,174 @@ "nullable": [] } }, + "5e65d510fdf83315c92cac1ac6acd8c878d7e3e5f050c022c64c246d0af260a2": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 11, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "5eacdf9c65dc13820077be2930cca7047aed07804ed2e7b2ff38e12a617b4858": { "query": "\nINSERT INTO twitch_channel_point_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n title = ?,\n prompt = ?,\n user_text_input = ?,\n redemption_id = ?,\n reward_id = ?,\n reward_cost = ?,\n is_sub_only = ?,\n max_per_stream = ?,\n max_per_user_per_stream = ?\n ", "describe": { @@ -514,6 +1367,317 @@ "nullable": [] } }, + "9e6aa0da19f7e428206c95498f748a30ab428892fb9b17547a7dc60470cc3c09": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "maybe_default_pretrained_vocoder", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 4, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "creator_is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 11, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 12, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 15, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 18, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 19, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 20, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 21, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "ab6f25eb0f6340d49aa705835ed428434d8b6ab14b0740f6a5ebe7841b9ece20": { "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -524,6 +1688,174 @@ "nullable": [] } }, + "b06e373275eb6d6f445abba38b8129d53196527bf953a50c85ccb93c23d2e4d6": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 11, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "b8746ae4ff2d4dcd54563eff64877f924bb119681ad78f1300146e00b762ca4f": { "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n maybe_pretrained_vocoder_used = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n\n file_size_bytes = ?,\n duration_millis = ?\n ", "describe": { @@ -949,148 +2281,6 @@ ] } }, - "05612e384bdce558e4834c720d1eb7ca9a09e3cca64b72f1497929d860345ec7": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "05c3ebf54cd8a5374d6f4bbe21b3c6027ce6314917af8acd0ed136d375d53342": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -1945,291 +3135,6 @@ ] } }, - "0beeed2c02db2008aac904e20de9eb3677e4b16a11d10975cec566e4ef84c6b6": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "text_preprocessing_algorithm", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 3, - "name": "maybe_default_pretrained_vocoder", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 4, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 5, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, - "name": "creator_is_banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 10, - "name": "description_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 11, - "name": "description_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 12, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 13, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 15, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 16, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 17, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 18, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 19, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "0caed749627ff22ae1cea52621a375156259a731e8070459593082b80d463491": { "query": "\nSELECT\n id,\n status\nFROM tts_model_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", "describe": { @@ -2390,44 +3295,6 @@ ] } }, - "12a176252df63d1e3a958362d754c828991d64ade427ab11a2025098d1581976": { - "query": "\nSELECT\n assignments.model_token AS tts_model_token,\n assignments.category_token\nFROM\n tts_category_assignments AS assignments\nJOIN\n tts_models AS tts\n ON assignments.model_token = tts.token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n AND assignments.deleted_at IS NULL\n\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "category_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false - ] - } - }, "17c8d210d93ee9c0d8e8422eed605c9150f19f9bf72affcf111b15a67a5b9e70": { "query": "\nUPDATE w2l_templates\nSET\n is_public_listing_approved = ?,\n is_locked_from_user_modification = ?,\n is_locked_from_use = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -3952,148 +4819,6 @@ ] } }, - "5d1f983fd2555c0fcb2ba065e9b3c6174346ee0fdaecacc2a557715a68cf23e2": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "5e54fd8569b2d6d3a0d11acee6bd03b75400b6325952eb16c8dac68a7933b13a": { "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n is_banned,\n disable_gravatar,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", "describe": { @@ -4466,291 +5191,6 @@ ] } }, - "63ac1d3f54c8258b7d20d3f689b2613809099feb4df925c680594a1e43e74bcd": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "text_preprocessing_algorithm", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 3, - "name": "maybe_default_pretrained_vocoder", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 4, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 5, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, - "name": "creator_is_banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 10, - "name": "description_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 11, - "name": "description_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 12, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 13, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 15, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 16, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 17, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 18, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 19, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, "641253d4c621f75037877651bba044a21033f50a5e3f8fed9b06ec967a508853": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -6929,16 +7369,6 @@ ] } }, - "b6a53361eda0f0f0517cc39b9bc5b7ab867cc8a1bcc7bc81c7d29c51193e3bbf": { - "query": "\nUPDATE tts_models\nSET\n is_public_listing_approved = ?,\n is_locked_from_user_modification = ?,\n is_locked_from_use = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - } - }, "b7535ecf2e63b7f02b5b96f833fd8cd28285b24473df6f89795ff360c1723215": { "query": "\nSELECT *\nFROM w2l_template_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { @@ -7723,148 +8153,6 @@ "nullable": [] } }, - "bf779826a10dd532f442d5b6a7ad3f35037276e8f822a3a633ca325c85c34234": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "c0f56e2656b566d3a8c869d39b0b56d1fbb75d7d9e817a24b10add44d348092d": { "query": "\nSELECT count(*) as count\nFROM w2l_templates\nWHERE\n creator_user_token = ?\nLIMIT 1\n ", "describe": { @@ -9166,148 +9454,6 @@ "nullable": [] } }, - "ef239806d026cd59106f79b3dc7c850da2bbc325a1adb831f319f87c0b07b56b": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "ef828549a221fe6fad3105fede0b38a439a685d6ea7f83a0657df9cdbb80c589": { "query": "\nSELECT\n ip_bans.ip_address,\n\n ip_bans.maybe_target_user_token,\n banned_users.username as maybe_target_username,\n\n ip_bans.mod_user_token,\n mod_users.username as mod_username,\n mod_users.display_name as mod_display_name,\n\n ip_bans.mod_notes,\n ip_bans.created_at,\n ip_bans.updated_at\nFROM\n ip_address_bans AS ip_bans\nLEFT OUTER JOIN users as banned_users\n ON ip_bans.maybe_target_user_token = banned_users.token\nJOIN users as mod_users\n ON ip_bans.mod_user_token = mod_users.token\nWHERE\n ip_bans.ip_address = ?\nLIMIT 1\n ", "describe": { @@ -9437,6 +9583,16 @@ ] } }, + "ef83a3b9cbc11a7e70637ada49b517e295a9a89ae273d3a77409de03fe483aac": { + "query": "\nUPDATE tts_models\nSET\n is_public_listing_approved = ?,\n is_locked_from_user_modification = ?,\n is_locked_from_use = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n is_front_page_featured = ?,\n is_twitch_featured = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 8 + }, + "nullable": [] + } + }, "f75b713d166398ebea2bdbde4e416dc8eb2c48dba4aa9bd164bcf5bc77237ce1": { "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n preferred_tts_result_visibility = ?,\n preferred_w2l_result_visibility = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n ip_address_last_update = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", "describe": { From 2a885c7758ebb04cfe869b5b48bcca3f133ba192 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Feb 2022 10:55:09 -0500 Subject: [PATCH 0909/1042] cleanup a lot of unused imports that warned --- .../src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs | 1 - .../http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs | 1 - .../src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs | 1 - .../http_server/endpoints/moderation/ip_bans/list_ip_bans.rs | 1 - .../http_server/endpoints/moderation/user_roles/list_roles.rs | 1 - .../http_server/endpoints/moderation/user_roles/list_staff.rs | 1 - .../endpoints/moderation/user_roles/set_user_role.rs | 1 - .../src/http_server/endpoints/tts/enqueue_upload_tts_model.rs | 2 +- .../src/http_server/endpoints/w2l/delete_w2l_template.rs | 1 - .../http_server/endpoints/w2l/enqueue_upload_w2l_template.rs | 2 +- .../http_server/endpoints/w2l/set_w2l_template_mod_approval.rs | 1 - 11 files changed, 2 insertions(+), 11 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs index 80ee8f2513..3be5a80fe9 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs @@ -5,7 +5,6 @@ use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs index 211637b926..eba7577ba4 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs index bac117d544..ea1336c298 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/get_ip_ban.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs index dff560f6cb..9c7a00a993 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs index 99b57c558d..19cda2400e 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::database::queries::list_user_roles::UserRoleForList; use crate::database::queries::list_user_roles::list_user_roles; use crate::http_server::web_utils::ip_address::get_request_ip; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs index 25d103bff8..37216b3ec2 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_staff.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs index 3eecd3ad12..3211988741 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::database::queries::list_user_roles::list_user_roles; use crate::database::queries::get_user_profile_by_username::get_user_profile_by_username; use crate::http_server::web_utils::ip_address::get_request_ip; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index efed50ea53..c775b463c2 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -5,7 +5,7 @@ use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; +use crate::database::helpers::enums::{TtsModelType, CreatorSetVisibility}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs index ddc9763024..99cd94fe27 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::database::queries::delete_w2l_template::delete_w2l_template_as_mod; use crate::database::queries::delete_w2l_template::delete_w2l_template_as_user; use crate::database::queries::delete_w2l_template::undelete_w2l_template_as_mod; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index ecc2d8fc80..d7fb8fca88 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -5,7 +5,7 @@ use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::database::helpers::enums::{CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs index 209f774b70..677f788432 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; From e2eb288fa8f6d54dd92867b81086c3b7879552cd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Feb 2022 11:36:19 -0500 Subject: [PATCH 0910/1042] no more badge --- .../endpoints/users/create_account.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs index c0da636405..a14e9f9cfe 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs @@ -237,14 +237,14 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) info!("new user id: {}", record_id); - // TODO: Remove me in the future. Too lazy to check the date, plus this will - // vary with adoption rate. - server_state.badge_granter.grant_early_user_badge(&user_token) - .await - .map_err(|e| { - warn!("error creating badge: {:?}", e); - CreateAccountErrorResponse::server_error() - })?; + //// TODO: Remove me in the future. Too lazy to check the date, plus this will + //// vary with adoption rate. + //server_state.badge_granter.grant_early_user_badge(&user_token) + // .await + // .map_err(|e| { + // warn!("error creating badge: {:?}", e); + // CreateAccountErrorResponse::server_error() + // })?; let create_session_result = create_session_for_user(&user_token, &ip_address, &server_state.mysql_pool).await; From 3528359d74cd2ab7749b8dd5ecb41fa5d0d0580d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Feb 2022 19:20:23 -0500 Subject: [PATCH 0911/1042] Twitch user configs (first iteration) --- .../down.sql | 5 +++ .../up.sql | 44 +++++++++++++++++++ .../down.sql | 5 +++ .../up.sql | 30 +++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 _migrations/2022-02-13-223514_twitch_event_action_configs/down.sql create mode 100644 _migrations/2022-02-13-223514_twitch_event_action_configs/up.sql create mode 100644 _migrations/2022-02-13-223523_twitch_general_settings/down.sql create mode 100644 _migrations/2022-02-13-223523_twitch_general_settings/up.sql diff --git a/_migrations/2022-02-13-223514_twitch_event_action_configs/down.sql b/_migrations/2022-02-13-223514_twitch_event_action_configs/down.sql new file mode 100644 index 0000000000..5cdec879c1 --- /dev/null +++ b/_migrations/2022-02-13-223514_twitch_event_action_configs/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE twitch_event_action_rules; diff --git a/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql b/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql new file mode 100644 index 0000000000..e90b326167 --- /dev/null +++ b/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql @@ -0,0 +1,44 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Multiple rows per user +CREATE TABLE twitch_event_action_rules( + -- Not used for anything except replication. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- Foreign key to user + -- This is a FakeYou/Storyteller account, *NOT* a twitch user id. + user_token VARCHAR(32) NOT NULL, + + -- The user can rearrange the rules in the UI. + -- This will be the order they apply in if matched. + -- NB: There is nothing in MySQL to guarantee unique ordering. + user_specified_rule_order INT(10) UNSIGNED NOT NULL DEFAULT 0, + + -- What type of event we'll be responding to. + event_category ENUM( + 'bits', + 'channel_points', + 'chat_command' + ) NOT NULL, + + -- A JSON payload containing any predicates we wish to apply to the match. + -- eg. bits_value > 100 + -- eg. channel_points_award_name == "Mario Voice" + event_match_predicate MEDIUMTEXT NOT NULL, + + -- A JSON payload containing how we wish to respond to the event. + -- eg. tts M:model + event_response MEDIUMTEXT NOT NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY fk_user_token (user_token), + KEY index_event_category (event_category) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_migrations/2022-02-13-223523_twitch_general_settings/down.sql b/_migrations/2022-02-13-223523_twitch_general_settings/down.sql new file mode 100644 index 0000000000..bc9ea84a15 --- /dev/null +++ b/_migrations/2022-02-13-223523_twitch_general_settings/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE twitch_general_settings; diff --git a/_migrations/2022-02-13-223523_twitch_general_settings/up.sql b/_migrations/2022-02-13-223523_twitch_general_settings/up.sql new file mode 100644 index 0000000000..76485f9448 --- /dev/null +++ b/_migrations/2022-02-13-223523_twitch_general_settings/up.sql @@ -0,0 +1,30 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- One row per user. Contains all of the general (non-repeated) configs. +CREATE TABLE twitch_general_settings ( + -- Not used for anything except replication. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- Foreign key to user + -- This is a FakeYou/Storyteller account, *NOT* a twitch user id. + user_token VARCHAR(32) NOT NULL, + + -- Kill switch for the entire service. + storyteller_is_enabled BOOLEAN NOT NULL DEFAULT FALSE, + + -- Kill switch for TTS. + tts_is_enabled BOOLEAN NOT NULL DEFAULT FALSE, + + -- Kill switch for W2L. + w2l_is_enabled BOOLEAN NOT NULL DEFAULT FALSE, + + -- Can set a fallback voice if a TTS voice goes missing. + maybe_fallback_tts_model_token VARCHAR(32) DEFAULT NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (user_token) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From fed4bac5de1043add31a9a6697af84fe977d8f2e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Feb 2022 19:20:38 -0500 Subject: [PATCH 0912/1042] database connect (local dev) --- dev_database_connect.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 dev_database_connect.sh diff --git a/dev_database_connect.sh b/dev_database_connect.sh new file mode 100755 index 0000000000..41963f4569 --- /dev/null +++ b/dev_database_connect.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +mysql -u "storyteller" \ + -p"password" \ + -D "storyteller" + From 70b6d1aa345d0378b6ff263a593e4c63e728b617 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Feb 2022 19:23:56 -0500 Subject: [PATCH 0913/1042] disable rule --- .../2022-02-13-223514_twitch_event_action_configs/up.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql b/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql index e90b326167..ab6b3280c5 100644 --- a/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql +++ b/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql @@ -26,6 +26,9 @@ CREATE TABLE twitch_event_action_rules( 'chat_command' ) NOT NULL, + -- Whether or not the rule is enabled. + rule_is_enabled BOOLEAN NOT NULL DEFAULT FALSE, + -- A JSON payload containing any predicates we wish to apply to the match. -- eg. bits_value > 100 -- eg. channel_points_award_name == "Mario Voice" From 64da009acba520cda06145ce4f1ff1ee5d136ab0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Feb 2022 21:24:52 -0500 Subject: [PATCH 0914/1042] Local dev configs for nginx (untested) --- localdev/nginx-http-config/storyteller | 27 ++++++++++++++++++++++ localdev/nginx-http-config/storyteller-ssl | 26 +++++++++++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/localdev/nginx-http-config/storyteller b/localdev/nginx-http-config/storyteller index 451cab6d48..9410d80522 100644 --- a/localdev/nginx-http-config/storyteller +++ b/localdev/nginx-http-config/storyteller @@ -10,6 +10,7 @@ # - /usr/local/etc/nginx/sites-enabled/{name-of-site} (for default nginx install) # +# API backend over plain HTTP server { listen 80; server_name api.jungle.horse; @@ -19,6 +20,20 @@ server { proxy_redirect off; } } + +# TODO(2022-02-13): This is new and untested +# WebSocket API backend over plain HTTP +server { + listen 80; + server_name ws.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:54321; + proxy_redirect off; + } +} + +# Javascript frontend over plain HTTP server { listen 80; server_name jungle.horse; @@ -28,3 +43,15 @@ server { proxy_redirect off; } } + +# TODO(2022-02-13): This is new and untested +# Javascript frontend #2 (OBS Layer) over plain HTTP +server { + listen 80; + server_name obs.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:7001; + proxy_redirect off; + } +} diff --git a/localdev/nginx-http-config/storyteller-ssl b/localdev/nginx-http-config/storyteller-ssl index 65bef40313..886b5fc264 100644 --- a/localdev/nginx-http-config/storyteller-ssl +++ b/localdev/nginx-http-config/storyteller-ssl @@ -48,10 +48,21 @@ server { } } +# TODO(2022-02-13): This is new and untested +# WebSocket API backend over plain HTTP +server { + listen 80; + server_name ws.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:54321; + proxy_redirect off; + } +} + # Javascript frontend over plain HTTP server { listen 80; - # server_name www.jungle.horse; server_name jungle.horse; location / { proxy_set_header Host $host; @@ -60,6 +71,18 @@ server { } } +# TODO(2022-02-13): This is new and untested +# Javascript frontend (OBS Layer) over plain HTTP +server { + listen 80; + server_name obs.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:7001; + proxy_redirect off; + } +} + # Javascript frontend over SSL server { listen 443 ssl; @@ -74,4 +97,3 @@ server { proxy_redirect off; } } - From 0507488ae10f3b6fa2d92611b30b52eb56c48e9c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Feb 2022 21:35:41 -0500 Subject: [PATCH 0915/1042] index page update with microservice name (for local dev) + discord link --- .../http_server/endpoints/misc/root_index.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/misc/root_index.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/root_index.rs index fe50b3a901..960d858d00 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/misc/root_index.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/misc/root_index.rs @@ -1,13 +1,22 @@ use actix_http::http::StatusCode; -use actix_web::{HttpResponse, HttpRequest, get}; +use actix_web::Error; use actix_web::Responder; use actix_web::Result; -use actix_web::Error; +use actix_web::web::Data; +use actix_web::{HttpResponse, HttpRequest, get}; +use crate::server_state::ServerState; use log::debug; +use std::sync::Arc; -pub async fn get_root_index() -> HttpResponse { +pub async fn get_root_index(server_state: Data>) -> HttpResponse { debug!("GET /"); // NB: Google load balancer hits this a lot, and it spams. + let body = format!(" \ +

hello!

\ + \ +

Maybe you want to work with us? We can pay! Get in touch!

\ +

storyteller-web instance: {}

\ + ", server_state.hostname); HttpResponse::build(StatusCode::OK) .content_type("text/html; charset=utf-8") - .body("

hello!

Are you looking for an API? Join our Discord!

Maybe you want to work with us? We can pay! Get in touch!

") + .body(body) } From 23e6f552e78bfd99607dcb0fd2da04369574e044 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 16 Feb 2022 02:41:02 -0500 Subject: [PATCH 0916/1042] migrations for api tokens --- .../2022-02-16-072149_api_tokens/down.sql | 5 ++++ .../2022-02-16-072149_api_tokens/up.sql | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 _migrations/2022-02-16-072149_api_tokens/down.sql create mode 100644 _migrations/2022-02-16-072149_api_tokens/up.sql diff --git a/_migrations/2022-02-16-072149_api_tokens/down.sql b/_migrations/2022-02-16-072149_api_tokens/down.sql new file mode 100644 index 0000000000..f8dce0d9c1 --- /dev/null +++ b/_migrations/2022-02-16-072149_api_tokens/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE api_tokens; diff --git a/_migrations/2022-02-16-072149_api_tokens/up.sql b/_migrations/2022-02-16-072149_api_tokens/up.sql new file mode 100644 index 0000000000..df5aaeb247 --- /dev/null +++ b/_migrations/2022-02-16-072149_api_tokens/up.sql @@ -0,0 +1,30 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- We'll use application logic to kill any API keys beyond the most recent 5 "active" a user creates. +CREATE TABLE api_tokens ( + -- Not used for anything except replication. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- This is the secret API token value + api_token VARCHAR(32) NOT NULL, + + -- Foreign key to user that owns the API token. + -- A user can have several. + user_token VARCHAR(32) NOT NULL, + + -- A user-defined short description (optional) + maybe_short_description VARCHAR(32) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (api_token), + KEY fk_user_token (user_token), + KEY index_deleted_at (deleted_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 653c282f0770be23fe80ebdb6a8e5ff438c96c1a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 16 Feb 2022 03:59:38 -0500 Subject: [PATCH 0917/1042] API tokens migration, queries, and handlers (list, create) --- .../2022-02-16-072149_api_tokens/up.sql | 6 + .../src/api_tokens/create_api_token.rs | 85 +++++++++++ .../list_available_api_tokens_for_user.rs | 70 +++++++++ .../database_queries/src/api_tokens/mod.rs | 2 + crates/lib/database_queries/src/lib.rs | 1 + crates/lib/database_queries/src/tokens.rs | 4 + .../endpoints/api_tokens/create_api_token.rs | 124 ++++++++++++++++ .../endpoints/api_tokens/list_api_tokens.rs | 133 ++++++++++++++++++ .../http_server/endpoints/api_tokens/mod.rs | 2 + .../src/http_server/endpoints/mod.rs | 1 + crates/service/storyteller_web/src/routes.rs | 29 ++++ 11 files changed, 457 insertions(+) create mode 100644 crates/lib/database_queries/src/api_tokens/create_api_token.rs create mode 100644 crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs create mode 100644 crates/lib/database_queries/src/api_tokens/mod.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/api_tokens/mod.rs diff --git a/_migrations/2022-02-16-072149_api_tokens/up.sql b/_migrations/2022-02-16-072149_api_tokens/up.sql index df5aaeb247..985d54072f 100644 --- a/_migrations/2022-02-16-072149_api_tokens/up.sql +++ b/_migrations/2022-02-16-072149_api_tokens/up.sql @@ -7,6 +7,10 @@ CREATE TABLE api_tokens ( -- Not used for anything except replication. id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + -- Idempotency token from client + -- This is so the frontend client doesn't submit duplicate jobs. + uuid_idempotency_token VARCHAR(36) NOT NULL, + -- This is the secret API token value api_token VARCHAR(32) NOT NULL, @@ -24,7 +28,9 @@ CREATE TABLE api_tokens ( -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (api_token), + UNIQUE KEY (uuid_idempotency_token), KEY fk_user_token (user_token), + KEY index_created_at (created_at), KEY index_deleted_at (deleted_at) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/crates/lib/database_queries/src/api_tokens/create_api_token.rs b/crates/lib/database_queries/src/api_tokens/create_api_token.rs new file mode 100644 index 0000000000..98b92f5c8c --- /dev/null +++ b/crates/lib/database_queries/src/api_tokens/create_api_token.rs @@ -0,0 +1,85 @@ +use container_common::anyhow_result::AnyhowResult; +use sqlx::MySqlPool; +use crate::tokens::Tokens; +use sqlx; +use anyhow::anyhow; +use crate::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; + +/// Create a new API token for the user. +/// If the user has more than five tokens, delete the least recent. +pub async fn create_api_token_for_user( + user_token: &str, + uuid_idempotency_token: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult { + + // TODO: Get last 5 API tokens for the user. + // TODO: Database transaction. + + let api_token = Tokens::new_api_token()?; + + let query = sqlx::query!( + r#" +INSERT INTO api_tokens +SET + api_token = ?, + user_token = ?, + uuid_idempotency_token = ? + "#, + api_token, + user_token, + uuid_idempotency_token + ); + + let query_result = query.execute(mysql_pool).await; + + let _record_id = match query_result { + Ok(res) => res.last_insert_id(), + Err(err) => return Err(anyhow!("Error creating API token: {:?}", err)), + }; + + // Delete older tokens. + let valid_api_tokens= list_available_api_tokens_for_user(user_token, mysql_pool).await?; + + let valid_api_tokens = valid_api_tokens.into_iter() + .map(|r| r.api_token) + .collect::>(); + + // TODO: THis is a massive hack since SQLx doesn't support binding Vec + // This array's elements will be replaced with any valid tokens. + let mut api_tokens = [ + api_token.as_str(), + api_token.as_str(), + api_token.as_str(), + api_token.as_str(), + api_token.as_str(), + api_token.as_str(), // One extra copy of the latest token. + ]; + + for i in 0..5 { + if let Some(token) = valid_api_tokens.get(i).map(|s| s.as_str()) { + api_tokens[i] = token; + } + } + + let query = sqlx::query!( + r#" +UPDATE api_tokens +SET + deleted_at = CURRENT_TIMESTAMP +WHERE + user_token = ? +AND api_token NOT IN (?, ?, ?, ?, ?) + "#, + user_token, + &api_tokens[0], + &api_tokens[1], + &api_tokens[2], + &api_tokens[3], + &api_tokens[4], + ); + + let _query_result = query.execute(mysql_pool).await?; + + Ok(api_token) +} \ No newline at end of file diff --git a/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs b/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs new file mode 100644 index 0000000000..83b7f38a74 --- /dev/null +++ b/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs @@ -0,0 +1,70 @@ +use anyhow::anyhow; +use chrono::{Utc, DateTime}; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::job_status::JobStatus; +use crate::column_types::record_visibility::RecordVisibility; +use crate::helpers::boolean_converters::i8_to_bool; +use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; +use log::{warn, info}; +use sqlx::MySqlPool; + +/// table: api_tokens +#[derive(Debug)] +pub struct AvailableApiToken { + pub api_token: String, + pub maybe_short_description: Option, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, +} + +/// Query available (non-deleted) API tokens for a user. +/// A user can only have five active tokens at a time, so we only return the five most recent. +pub async fn list_available_api_tokens_for_user( + user_token: &str, + pool: &MySqlPool, +) -> AnyhowResult> { + + let records : Vec = sqlx::query_as!( + AvailableApiTokenInternal, + r#" +SELECT + api_token, + maybe_short_description, + created_at, + updated_at +FROM api_tokens +WHERE + user_token = ? + AND deleted_at IS NULL +ORDER BY created_at DESC +LIMIT 5 + "#, + user_token, + ) + .fetch_all(pool) + .await?; + + let mut records = records.into_iter() + .map(|record : AvailableApiTokenInternal | { + AvailableApiToken { + api_token: record.api_token, + maybe_short_description: record.maybe_short_description, + created_at: record.created_at, + updated_at: record.updated_at, + } + }) + .collect::>(); + + // Queried in DESC order, but sort returned results ordered ASC. + records.sort_by_key(|r| r.created_at); + + Ok(records) +} + +#[derive(Debug)] +struct AvailableApiTokenInternal { + pub api_token: String, + pub maybe_short_description: Option, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, +} diff --git a/crates/lib/database_queries/src/api_tokens/mod.rs b/crates/lib/database_queries/src/api_tokens/mod.rs new file mode 100644 index 0000000000..30fa2f5281 --- /dev/null +++ b/crates/lib/database_queries/src/api_tokens/mod.rs @@ -0,0 +1,2 @@ +pub mod create_api_token; +pub mod list_available_api_tokens_for_user; \ No newline at end of file diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index a85062e2bc..57b7a39eda 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -2,6 +2,7 @@ mod builders; +pub mod api_tokens; pub mod column_types; pub mod helpers; pub mod tokens; diff --git a/crates/lib/database_queries/src/tokens.rs b/crates/lib/database_queries/src/tokens.rs index 1144344c89..8d1b9e9336 100644 --- a/crates/lib/database_queries/src/tokens.rs +++ b/crates/lib/database_queries/src/tokens.rs @@ -25,6 +25,10 @@ impl Tokens { random_prefix_crockford_token("WR:", 32, false) } + pub fn new_api_token() -> AnyhowResult { + random_prefix_crockford_token("API:", 32, true) + } + // ========== Tokens not typically visible to users ========== pub fn new_user() -> AnyhowResult { diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs new file mode 100644 index 0000000000..260c00034d --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs @@ -0,0 +1,124 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use database_queries::tokens::Tokens; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; +use database_queries::api_tokens::create_api_token::create_api_token_for_user; + +// =============== Request =============== + +#[derive(Deserialize)] +pub struct CreateApiTokenRequest { + pub idempotency_token: String, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct CreateApiTokenResponse { + pub success: bool, + pub api_token: String, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum CreateApiTokenError { + BadInput(String), + NotAuthorized, + ServerError, +} + +impl ResponseError for CreateApiTokenError { + fn status_code(&self) -> StatusCode { + match *self { + CreateApiTokenError::BadInput(_) => StatusCode::BAD_REQUEST, + CreateApiTokenError::NotAuthorized => StatusCode::UNAUTHORIZED, + CreateApiTokenError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for CreateApiTokenError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn create_api_token_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CreateApiTokenError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(CreateApiTokenError::NotAuthorized); + } + }; + + if user_session.is_banned { + warn!("banned users cannot create API tokens"); + return Err(CreateApiTokenError::NotAuthorized); + } + + let api_token = create_api_token_for_user( + &user_session.user_token, + &request.idempotency_token, + &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("API token creation errror: {:?}", e); + CreateApiTokenError::ServerError + })?; + + let response = CreateApiTokenResponse { + success: true, + api_token + }; + + let body = serde_json::to_string(&response) + .map_err(|e| CreateApiTokenError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs new file mode 100644 index 0000000000..80d99fc231 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs @@ -0,0 +1,133 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::database::enums::record_visibility::RecordVisibility; +use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use lexical_sort::natural_lexical_cmp; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; +use database_queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct ListApiTokensResponse { + pub success: bool, + pub api_tokens: Vec, +} + +/// Public-facing and optimized (non-irrelevant fields) +#[derive(Serialize)] +pub struct ApiToken { + pub api_token: String, + pub maybe_short_description: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum ListApiTokensError { + NotAuthorized, + ServerError, +} + +impl ResponseError for ListApiTokensError { + fn status_code(&self) -> StatusCode { + match *self { + ListApiTokensError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListApiTokensError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListApiTokensError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn list_api_tokens_handler( + http_request: HttpRequest, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListApiTokensError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(ListApiTokensError::NotAuthorized); + } + }; + + if user_session.is_banned { + warn!("banned users cannot use API tokens"); + return Err(ListApiTokensError::NotAuthorized); + } + + let api_tokens = list_available_api_tokens_for_user(&user_session.user_token, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("API token query error: {:?}", e); + ListApiTokensError::ServerError + })?; + + let mut api_tokens = api_tokens.into_iter() + .map(|r| { + ApiToken { + api_token: r.api_token, + maybe_short_description: r.maybe_short_description, + created_at: r.created_at, + updated_at: r.updated_at, + } + }) + .collect::>(); + + let response = ListApiTokensResponse { + success: true, + api_tokens, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListApiTokensError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/mod.rs new file mode 100644 index 0000000000..866920efd9 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/mod.rs @@ -0,0 +1,2 @@ +pub mod create_api_token; +pub mod list_api_tokens; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/mod.rs index 4be42e802b..e779339a76 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/mod.rs @@ -1,3 +1,4 @@ +pub mod api_tokens; pub mod categories; pub mod events; pub mod leaderboard; diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index cee0183f37..49b461c4b0 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -71,6 +71,8 @@ use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_ use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; use crate::http_server::endpoints::misc::detect_locale_handler::detect_locale_handler; +use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler; +use crate::http_server::endpoints::api_tokens::list_api_tokens::list_api_tokens_handler; pub fn add_routes (app: App) -> App where @@ -88,6 +90,7 @@ pub fn add_routes (app: App) -> App app = add_w2l_routes(app); /* /w2l */ app = add_category_routes(app); /* /category */ app = add_user_profile_routes(app); /* /user */ + app = add_api_token_routes(app); /* /api_tokens */ app = add_twitch_oauth_routes(app); /* /twitch */ // TODO: MAYBE TEMPORARY // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== @@ -528,6 +531,32 @@ fn add_user_profile_routes (app: App) -> App ) ) } + +// ==================== API TOKEN ROUTES ==================== + +fn add_api_token_routes (app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service(web::scope("/api_tokens") + .service(web::resource("/create") + .route(web::post().to(create_api_token_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/list") + .route(web::get().to(list_api_tokens_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) +} + // ==================== TWITCH ROUTES ==================== // TODO: Maybe move these into an "oauth-gateway" type http service. From 0a3b99fc70efa242ab9d81f03fea8f55c9ed216f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 16 Feb 2022 04:13:59 -0500 Subject: [PATCH 0918/1042] debug creation of API tokens --- debug/curl/create_api_token.py | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100755 debug/curl/create_api_token.py diff --git a/debug/curl/create_api_token.py b/debug/curl/create_api_token.py new file mode 100755 index 0000000000..4555dbb673 --- /dev/null +++ b/debug/curl/create_api_token.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import requests +import re +import uuid + +login_url = 'http://localhost:12345/login' +create_api_token_url = 'http://localhost:12345/api_tokens/create' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + +cookies = { 'session': session_cookie } + +payload = { + 'idempotency_token': str(uuid.uuid4()), +} + +r = requests.post(create_api_token_url, cookies=cookies, json=payload) + +print("===== Created API Token =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + From 84bbd3e2ca9dc9486ab293240bed50b63c3b120d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Feb 2022 04:46:34 -0500 Subject: [PATCH 0919/1042] Fix Gdown / Google Drive issue? --- scripts/download_internet_file.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/download_internet_file.py b/scripts/download_internet_file.py index ccce983600..7485701842 100755 --- a/scripts/download_internet_file.py +++ b/scripts/download_internet_file.py @@ -38,7 +38,12 @@ def _normalize_google_drive_url(url): return url document_token = match_groups[0] - return 'https://drive.google.com/uc?id={}'.format(document_token) + + # NB(2022-02-17): Google appears to have broken downloads + # Using this fix with URL structure: + # https://github.com/wkentaro/gdown/issues/146#issuecomment-1042382215 + # OLD FORMAT: return 'https://drive.google.com/uc?id={}'.format(document_token) + return 'https://drive.google.com/uc?export=download&confirm=pbef&id={}'.format(document_token) def download_url_or_gdrive(url, output_filename): From b642a9a03144164022edf1ec0a633727cda0ba68 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Feb 2022 05:10:42 -0500 Subject: [PATCH 0920/1042] re-codegen --- sqlx-data.json | 174 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 129 insertions(+), 45 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index fb3a4c2c5d..f96fddb2c2 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -915,7 +915,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -991,6 +991,16 @@ ] } }, + "5951014fbd8f71ea21a8d6f69c23e8b2a449595bf28216b9a9cc87979ea60b05": { + "query": "\nUPDATE api_tokens\nSET\n deleted_at = CURRENT_TIMESTAMP\nWHERE\n user_token = ?\nAND api_token NOT IN (?, ?, ?, ?, ?)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + } + }, "5a17891eec52d8304af8a54f6bb7629cb71d3603146f8b2c842f6e1047c31e91": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -1367,6 +1377,70 @@ "nullable": [] } }, + "9810f6a24480b44ec766e35a1c60daf36f9c8a048bb0192a6069214a695c8dd2": { + "query": "\nSELECT\n api_token,\n maybe_short_description,\n created_at,\n updated_at\nFROM api_tokens\nWHERE\n user_token = ?\n AND deleted_at IS NULL\nORDER BY created_at DESC\nLIMIT 5\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "api_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_short_description", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1161 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 3, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + false, + false + ] + } + }, "9e6aa0da19f7e428206c95498f748a30ab428892fb9b17547a7dc60470cc3c09": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -1866,6 +1940,16 @@ "nullable": [] } }, + "c638431e2bcdef1e14d5ef35d571df717a8d3782e15768dfbadb38ac7e94a674": { + "query": "\nINSERT INTO api_tokens\nSET\n api_token = ?,\n user_token = ?,\n uuid_idempotency_token = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, "d7ac9f84e92d6b2111162a674284c0b9fa50c718bedfb8ab7238ca0a7b0875e6": { "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", "describe": { @@ -2039,7 +2123,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -2051,7 +2135,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -2063,7 +2147,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -2366,7 +2450,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -2378,7 +2462,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -2390,7 +2474,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -2692,7 +2776,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -2906,7 +2990,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -2918,7 +3002,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -2930,7 +3014,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -3416,7 +3500,7 @@ "bits": 129 }, "char_set": 63, - "max_size": 2 + "max_size": 1 } } ], @@ -3603,7 +3687,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -3615,7 +3699,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -3627,7 +3711,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -4434,7 +4518,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -4446,7 +4530,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -4458,7 +4542,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -4470,7 +4554,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -4624,7 +4708,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -4714,7 +4798,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -5085,7 +5169,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -5097,7 +5181,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -5109,7 +5193,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -5819,7 +5903,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -5979,7 +6063,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -5991,7 +6075,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -6003,7 +6087,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -6646,7 +6730,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -6946,7 +7030,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -7261,7 +7345,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -7273,7 +7357,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -7514,7 +7598,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 3 + "max_size": 11 } }, { @@ -7806,7 +7890,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -7818,7 +7902,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -7830,7 +7914,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -7842,7 +7926,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -8024,7 +8108,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -8036,7 +8120,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -8048,7 +8132,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -8702,7 +8786,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -8714,7 +8798,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -9138,7 +9222,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -9150,7 +9234,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { @@ -9162,7 +9246,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 10 + "max_size": 11 } }, { From 56ac627c4a60d0d838ab35203c07827882fd526c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Feb 2022 21:54:54 -0500 Subject: [PATCH 0921/1042] move vendor/ to crates/vendor --- Dockerfile | 1 - crates/service/storyteller_web/Cargo.toml | 4 +++- .../.github/ISSUE_TEMPLATE/bug_report.md | 0 .../.github/ISSUE_TEMPLATE/enhancement.md | 0 .../newrelic-telemetry-sdk-rust/.github/dependabot.yml | 0 .../newrelic-telemetry-sdk-rust/.github/workflows/ci.yml | 0 .../vendor}/newrelic-telemetry-sdk-rust/.gitignore | 0 .../vendor}/newrelic-telemetry-sdk-rust/CODE_OF_CONDUCT.md | 0 .../vendor}/newrelic-telemetry-sdk-rust/CONTRIBUTING.md | 0 .../vendor}/newrelic-telemetry-sdk-rust/Cargo.toml | 0 {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/LICENSE | 0 .../vendor}/newrelic-telemetry-sdk-rust/README.md | 0 {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/cla.md | 0 .../vendor}/newrelic-telemetry-sdk-rust/src/attribute.rs | 0 .../vendor}/newrelic-telemetry-sdk-rust/src/client.rs | 0 .../vendor}/newrelic-telemetry-sdk-rust/src/lib.rs | 0 .../vendor}/newrelic-telemetry-sdk-rust/src/span.rs | 0 .../vendor}/newrelic-telemetry-sdk-rust/tests/blocking.rs | 0 .../vendor}/newrelic-telemetry-sdk-rust/tests/common/mod.rs | 0 .../vendor}/newrelic-telemetry-sdk-rust/tests/spans.rs | 0 20 files changed, 3 insertions(+), 2 deletions(-) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/bug_report.md (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/enhancement.md (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/.github/dependabot.yml (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/.github/workflows/ci.yml (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/.gitignore (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/CODE_OF_CONDUCT.md (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/CONTRIBUTING.md (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/Cargo.toml (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/LICENSE (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/README.md (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/cla.md (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/src/attribute.rs (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/src/client.rs (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/src/lib.rs (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/src/span.rs (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/tests/blocking.rs (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/tests/common/mod.rs (100%) rename {vendor => crates/vendor}/newrelic-telemetry-sdk-rust/tests/spans.rs (100%) diff --git a/Dockerfile b/Dockerfile index 88df556a05..f08c08a378 100644 --- a/Dockerfile +++ b/Dockerfile @@ -153,7 +153,6 @@ COPY Cargo.lock . COPY Cargo.toml . COPY sqlx-data.json . COPY crates/ ./crates -COPY vendor/ ./vendor COPY db/ ./db # TODO(2022-01-16): Might not be necessary: # COPY _migrations/ ./_migrations diff --git a/crates/service/storyteller_web/Cargo.toml b/crates/service/storyteller_web/Cargo.toml index e75ad56a41..a7c45e7975 100644 --- a/crates/service/storyteller_web/Cargo.toml +++ b/crates/service/storyteller_web/Cargo.toml @@ -47,6 +47,9 @@ tts_common = { path = "../../lib/tts_common" } twitch_common = { path = "../../lib/twitch_common" } user_input_common = { path = "../../lib/user_input_common" } +# Vendored code +newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" } + # External anyhow = "1.0.40" base64 = "0.13.0" @@ -60,7 +63,6 @@ lazy_static = "1.4.0" lexical-sort = "0.3.1" lfu = "0.2.5" log = "0.4.14" -newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" } rand = "0.8.3" regex = "1.5.4" time = "0.2.26" diff --git a/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/bug_report.md b/crates/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/bug_report.md similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/bug_report.md rename to crates/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/bug_report.md diff --git a/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/enhancement.md b/crates/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/enhancement.md similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/enhancement.md rename to crates/vendor/newrelic-telemetry-sdk-rust/.github/ISSUE_TEMPLATE/enhancement.md diff --git a/vendor/newrelic-telemetry-sdk-rust/.github/dependabot.yml b/crates/vendor/newrelic-telemetry-sdk-rust/.github/dependabot.yml similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/.github/dependabot.yml rename to crates/vendor/newrelic-telemetry-sdk-rust/.github/dependabot.yml diff --git a/vendor/newrelic-telemetry-sdk-rust/.github/workflows/ci.yml b/crates/vendor/newrelic-telemetry-sdk-rust/.github/workflows/ci.yml similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/.github/workflows/ci.yml rename to crates/vendor/newrelic-telemetry-sdk-rust/.github/workflows/ci.yml diff --git a/vendor/newrelic-telemetry-sdk-rust/.gitignore b/crates/vendor/newrelic-telemetry-sdk-rust/.gitignore similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/.gitignore rename to crates/vendor/newrelic-telemetry-sdk-rust/.gitignore diff --git a/vendor/newrelic-telemetry-sdk-rust/CODE_OF_CONDUCT.md b/crates/vendor/newrelic-telemetry-sdk-rust/CODE_OF_CONDUCT.md similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/CODE_OF_CONDUCT.md rename to crates/vendor/newrelic-telemetry-sdk-rust/CODE_OF_CONDUCT.md diff --git a/vendor/newrelic-telemetry-sdk-rust/CONTRIBUTING.md b/crates/vendor/newrelic-telemetry-sdk-rust/CONTRIBUTING.md similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/CONTRIBUTING.md rename to crates/vendor/newrelic-telemetry-sdk-rust/CONTRIBUTING.md diff --git a/vendor/newrelic-telemetry-sdk-rust/Cargo.toml b/crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/Cargo.toml rename to crates/vendor/newrelic-telemetry-sdk-rust/Cargo.toml diff --git a/vendor/newrelic-telemetry-sdk-rust/LICENSE b/crates/vendor/newrelic-telemetry-sdk-rust/LICENSE similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/LICENSE rename to crates/vendor/newrelic-telemetry-sdk-rust/LICENSE diff --git a/vendor/newrelic-telemetry-sdk-rust/README.md b/crates/vendor/newrelic-telemetry-sdk-rust/README.md similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/README.md rename to crates/vendor/newrelic-telemetry-sdk-rust/README.md diff --git a/vendor/newrelic-telemetry-sdk-rust/cla.md b/crates/vendor/newrelic-telemetry-sdk-rust/cla.md similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/cla.md rename to crates/vendor/newrelic-telemetry-sdk-rust/cla.md diff --git a/vendor/newrelic-telemetry-sdk-rust/src/attribute.rs b/crates/vendor/newrelic-telemetry-sdk-rust/src/attribute.rs similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/src/attribute.rs rename to crates/vendor/newrelic-telemetry-sdk-rust/src/attribute.rs diff --git a/vendor/newrelic-telemetry-sdk-rust/src/client.rs b/crates/vendor/newrelic-telemetry-sdk-rust/src/client.rs similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/src/client.rs rename to crates/vendor/newrelic-telemetry-sdk-rust/src/client.rs diff --git a/vendor/newrelic-telemetry-sdk-rust/src/lib.rs b/crates/vendor/newrelic-telemetry-sdk-rust/src/lib.rs similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/src/lib.rs rename to crates/vendor/newrelic-telemetry-sdk-rust/src/lib.rs diff --git a/vendor/newrelic-telemetry-sdk-rust/src/span.rs b/crates/vendor/newrelic-telemetry-sdk-rust/src/span.rs similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/src/span.rs rename to crates/vendor/newrelic-telemetry-sdk-rust/src/span.rs diff --git a/vendor/newrelic-telemetry-sdk-rust/tests/blocking.rs b/crates/vendor/newrelic-telemetry-sdk-rust/tests/blocking.rs similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/tests/blocking.rs rename to crates/vendor/newrelic-telemetry-sdk-rust/tests/blocking.rs diff --git a/vendor/newrelic-telemetry-sdk-rust/tests/common/mod.rs b/crates/vendor/newrelic-telemetry-sdk-rust/tests/common/mod.rs similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/tests/common/mod.rs rename to crates/vendor/newrelic-telemetry-sdk-rust/tests/common/mod.rs diff --git a/vendor/newrelic-telemetry-sdk-rust/tests/spans.rs b/crates/vendor/newrelic-telemetry-sdk-rust/tests/spans.rs similarity index 100% rename from vendor/newrelic-telemetry-sdk-rust/tests/spans.rs rename to crates/vendor/newrelic-telemetry-sdk-rust/tests/spans.rs From 54ca8d4c7a9eeb328512538850fff19bcb44333c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 17 Feb 2022 23:33:06 -0500 Subject: [PATCH 0922/1042] edit API token handler --- .../2022-02-16-072149_api_tokens/up.sql | 5 + .../src/api_tokens/create_api_token.rs | 13 +- .../src/api_tokens/edit_api_token.rs | 40 +++++ .../database_queries/src/api_tokens/mod.rs | 1 + .../endpoints/api_tokens/create_api_token.rs | 3 + .../endpoints/api_tokens/delete_api_token.rs | 0 .../endpoints/api_tokens/edit_api_token.rs | 146 ++++++++++++++++++ .../http_server/endpoints/api_tokens/mod.rs | 2 + crates/service/storyteller_web/src/routes.rs | 11 +- 9 files changed, 215 insertions(+), 6 deletions(-) create mode 100644 crates/lib/database_queries/src/api_tokens/edit_api_token.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs diff --git a/_migrations/2022-02-16-072149_api_tokens/up.sql b/_migrations/2022-02-16-072149_api_tokens/up.sql index 985d54072f..b3f8ccb059 100644 --- a/_migrations/2022-02-16-072149_api_tokens/up.sql +++ b/_migrations/2022-02-16-072149_api_tokens/up.sql @@ -21,6 +21,11 @@ CREATE TABLE api_tokens ( -- A user-defined short description (optional) maybe_short_description VARCHAR(32) DEFAULT NULL, + -- For abuse tracking. + -- Wide enough for IPv4/6 + ip_address_creation VARCHAR(40) NOT NULL, + ip_address_last_update VARCHAR(40) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, deleted_at TIMESTAMP, diff --git a/crates/lib/database_queries/src/api_tokens/create_api_token.rs b/crates/lib/database_queries/src/api_tokens/create_api_token.rs index 98b92f5c8c..80b0e3352c 100644 --- a/crates/lib/database_queries/src/api_tokens/create_api_token.rs +++ b/crates/lib/database_queries/src/api_tokens/create_api_token.rs @@ -10,6 +10,7 @@ use crate::api_tokens::list_available_api_tokens_for_user::list_available_api_to pub async fn create_api_token_for_user( user_token: &str, uuid_idempotency_token: &str, + creator_ip_address: &str, mysql_pool: &MySqlPool ) -> AnyhowResult { @@ -24,11 +25,15 @@ INSERT INTO api_tokens SET api_token = ?, user_token = ?, - uuid_idempotency_token = ? + uuid_idempotency_token = ?, + ip_address_creation = ?, + ip_address_last_update = ? "#, api_token, user_token, - uuid_idempotency_token + uuid_idempotency_token, + creator_ip_address, + creator_ip_address ); let query_result = query.execute(mysql_pool).await; @@ -66,11 +71,13 @@ SET r#" UPDATE api_tokens SET - deleted_at = CURRENT_TIMESTAMP + deleted_at = CURRENT_TIMESTAMP, + ip_address_last_update = ? WHERE user_token = ? AND api_token NOT IN (?, ?, ?, ?, ?) "#, + creator_ip_address, user_token, &api_tokens[0], &api_tokens[1], diff --git a/crates/lib/database_queries/src/api_tokens/edit_api_token.rs b/crates/lib/database_queries/src/api_tokens/edit_api_token.rs new file mode 100644 index 0000000000..9dd62164e8 --- /dev/null +++ b/crates/lib/database_queries/src/api_tokens/edit_api_token.rs @@ -0,0 +1,40 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use log::warn; +use sqlx::MySqlPool; +use std::collections::{HashSet, HashMap}; + +pub async fn edit_api_token( + user_token: &str, + api_token: &str, + maybe_short_description: Option<&str>, + ip_address_update: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult { + + let query = sqlx::query!( + r#" +UPDATE api_tokens +SET + maybe_short_description = ?, + ip_address_last_update = ? +WHERE + user_token = ? + AND api_token = ? +LIMIT 1 + "#, + maybe_short_description, + ip_address_update, + user_token, + api_token, + ); + + let result = query.execute(mysql_pool).await; + + match result { + Err(err) => { + Err(anyhow!("error with query: {:?}", err)) + }, + Ok(results) => Ok(true), + } +} diff --git a/crates/lib/database_queries/src/api_tokens/mod.rs b/crates/lib/database_queries/src/api_tokens/mod.rs index 30fa2f5281..35f0ab5d0c 100644 --- a/crates/lib/database_queries/src/api_tokens/mod.rs +++ b/crates/lib/database_queries/src/api_tokens/mod.rs @@ -1,2 +1,3 @@ pub mod create_api_token; +pub mod edit_api_token; pub mod list_available_api_tokens_for_user; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs index 260c00034d..abfc39bc04 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs @@ -100,9 +100,12 @@ pub async fn create_api_token_handler( return Err(CreateApiTokenError::NotAuthorized); } + let creator_ip_address = get_request_ip(&http_request); + let api_token = create_api_token_for_user( &user_session.user_token, &request.idempotency_token, + &creator_ip_address, &server_state.mysql_pool) .await .map_err(|e| { diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs new file mode 100644 index 0000000000..2aedf13e84 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs @@ -0,0 +1,146 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log, error}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; +use crate::database::queries::categories::get_category_by_token::get_category_by_token; +use database_queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; +use database_queries::api_tokens::edit_api_token::edit_api_token; +use crate::http_server::web_utils::ip_address::get_request_ip; + +// =============== Request =============== + +// NB: Ordinarily, `api_token` would be PathInfo, but these are secret values. + +// These are not sparse updates! +#[derive(Deserialize)] +pub struct EditApiTokenRequest { + pub api_token: String, + pub maybe_short_description: Option, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct EditApiTokenResponse { + pub success: bool, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum EditApiTokenError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for EditApiTokenError { + fn status_code(&self) -> StatusCode { + match *self { + EditApiTokenError::BadInput(_) => StatusCode::BAD_REQUEST, + EditApiTokenError::NotFound => StatusCode::NOT_FOUND, + EditApiTokenError::NotAuthorized => StatusCode::UNAUTHORIZED, + EditApiTokenError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for EditApiTokenError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn edit_api_token_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EditApiTokenError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(EditApiTokenError::NotAuthorized); + } + }; + + if user_session.is_banned { + warn!("banned users cannot edit API tokens"); + return Err(EditApiTokenError::NotAuthorized); + } + + let tokens = list_available_api_tokens_for_user(&user_session.user_token, + &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Error querying tokens: {:?}", e); + return EditApiTokenError::ServerError; + })?; + + let valid_token = tokens.iter() + .find(|t| t.api_token.eq(&request.api_token)) + .is_some(); + + if !valid_token { + warn!("Invalid API Token"); + return Err(EditApiTokenError::NotFound); + } + + let creator_ip_address = get_request_ip(&http_request); + + let _r = edit_api_token( + &user_session.user_token, + &request.api_token, + request.maybe_short_description.as_deref(), + &creator_ip_address, + &server_state.mysql_pool) + .await + .map_err(|e| { + error!("Error with query: {:?}", e); + return EditApiTokenError::ServerError; + }); + + let response = EditApiTokenResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| EditApiTokenError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/mod.rs index 866920efd9..e3c4f3427d 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/mod.rs @@ -1,2 +1,4 @@ pub mod create_api_token; +pub mod delete_api_token; +pub mod edit_api_token; pub mod list_api_tokens; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index 49b461c4b0..395782437d 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -3,6 +3,9 @@ use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; use actix_web::error::Error; use actix_web::{App, web, HttpResponse}; +use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler; +use crate::http_server::endpoints::api_tokens::edit_api_token::edit_api_token_handler; +use crate::http_server::endpoints::api_tokens::list_api_tokens::list_api_tokens_handler; use crate::http_server::endpoints::categories::assign_tts_category::assign_tts_category_handler; use crate::http_server::endpoints::categories::create_category::create_category_handler; use crate::http_server::endpoints::categories::get_category::get_category_handler; @@ -11,6 +14,7 @@ use crate::http_server::endpoints::categories::list_tts_model_assigned_categorie use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::misc::default_route_404::default_route_404; +use crate::http_server::endpoints::misc::detect_locale_handler::detect_locale_handler; use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha_easy_handler; use crate::http_server::endpoints::misc::enable_alpha_handler::enable_alpha_handler; use crate::http_server::endpoints::misc::root_index::get_root_index; @@ -70,9 +74,6 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; -use crate::http_server::endpoints::misc::detect_locale_handler::detect_locale_handler; -use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler; -use crate::http_server::endpoints::api_tokens::list_api_tokens::list_api_tokens_handler; pub fn add_routes (app: App) -> App where @@ -550,6 +551,10 @@ fn add_api_token_routes (app: App) -> App .route(web::post().to(create_api_token_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/edit") + .route(web::post().to(edit_api_token_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/list") .route(web::get().to(list_api_tokens_handler)) .route(web::head().to(|| HttpResponse::Ok())) From cb3c91f802fc0103a2c53634f34e89c9179ed557 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 00:04:01 -0500 Subject: [PATCH 0923/1042] delete api token endpoint --- .../src/api_tokens/delete_api_token.rs | 38 +++++ .../database_queries/src/api_tokens/mod.rs | 1 + .../endpoints/api_tokens/delete_api_token.rs | 143 ++++++++++++++++++ crates/service/storyteller_web/src/routes.rs | 5 + ...create_api_token.py => api_token_tests.py} | 25 ++- 5 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 crates/lib/database_queries/src/api_tokens/delete_api_token.rs rename debug/curl/{create_api_token.py => api_token_tests.py} (56%) diff --git a/crates/lib/database_queries/src/api_tokens/delete_api_token.rs b/crates/lib/database_queries/src/api_tokens/delete_api_token.rs new file mode 100644 index 0000000000..3c224b35b6 --- /dev/null +++ b/crates/lib/database_queries/src/api_tokens/delete_api_token.rs @@ -0,0 +1,38 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use log::warn; +use sqlx::MySqlPool; +use std::collections::{HashSet, HashMap}; + +pub async fn delete_api_token( + user_token: &str, + api_token: &str, + ip_address_update: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult { + + let query = sqlx::query!( + r#" +UPDATE api_tokens +SET + deleted_at = CURRENT_TIMESTAMP, + ip_address_last_update = ? +WHERE + user_token = ? + AND api_token = ? +LIMIT 1 + "#, + ip_address_update, + user_token, + api_token, + ); + + let result = query.execute(mysql_pool).await; + + match result { + Err(err) => { + Err(anyhow!("error with query: {:?}", err)) + }, + Ok(results) => Ok(true), + } +} diff --git a/crates/lib/database_queries/src/api_tokens/mod.rs b/crates/lib/database_queries/src/api_tokens/mod.rs index 35f0ab5d0c..2ce84cd568 100644 --- a/crates/lib/database_queries/src/api_tokens/mod.rs +++ b/crates/lib/database_queries/src/api_tokens/mod.rs @@ -1,3 +1,4 @@ pub mod create_api_token; +pub mod delete_api_token; pub mod edit_api_token; pub mod list_available_api_tokens_for_user; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs index e69de29bb2..3010c2744c 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs @@ -0,0 +1,143 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database::queries::categories::get_category_by_token::get_category_by_token; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use database_queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log, error}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; +use database_queries::api_tokens::delete_api_token::delete_api_token; + +// =============== Request =============== + +// NB: `api_token` is only in PathInfo because we're deleting it. + +#[derive(Deserialize)] +pub struct DeleteApiTokenPathInfo { + api_token: String, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct DeleteApiTokenResponse { + pub success: bool, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum DeleteApiTokenError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for DeleteApiTokenError { + fn status_code(&self) -> StatusCode { + match *self { + DeleteApiTokenError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteApiTokenError::NotFound => StatusCode::NOT_FOUND, + DeleteApiTokenError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteApiTokenError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for DeleteApiTokenError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn delete_api_token_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + DeleteApiTokenError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(DeleteApiTokenError::NotAuthorized); + } + }; + + if user_session.is_banned { + warn!("banned users cannot edit API tokens"); + return Err(DeleteApiTokenError::NotAuthorized); + } + + let tokens = list_available_api_tokens_for_user(&user_session.user_token, + &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Error querying tokens: {:?}", e); + return DeleteApiTokenError::ServerError; + })?; + + let valid_token = tokens.iter() + .find(|t| t.api_token.eq(&path.api_token)) + .is_some(); + + if !valid_token { + warn!("Invalid API Token"); + return Err(DeleteApiTokenError::NotFound); + } + + let creator_ip_address = get_request_ip(&http_request); + + let _r = delete_api_token( + &user_session.user_token, + &path.api_token, + &creator_ip_address, + &server_state.mysql_pool) + .await + .map_err(|e| { + error!("Error with query: {:?}", e); + return DeleteApiTokenError::ServerError; + }); + + let response = DeleteApiTokenResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| DeleteApiTokenError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index 395782437d..d1854b5e7b 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -4,6 +4,7 @@ use actix_web::dev::{ServiceRequest, ServiceResponse}; use actix_web::error::Error; use actix_web::{App, web, HttpResponse}; use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler; +use crate::http_server::endpoints::api_tokens::delete_api_token::delete_api_token_handler; use crate::http_server::endpoints::api_tokens::edit_api_token::edit_api_token_handler; use crate::http_server::endpoints::api_tokens::list_api_tokens::list_api_tokens_handler; use crate::http_server::endpoints::categories::assign_tts_category::assign_tts_category_handler; @@ -555,6 +556,10 @@ fn add_api_token_routes (app: App) -> App .route(web::post().to(edit_api_token_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/{api_token}/delete") + .route(web::post().to(delete_api_token_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/list") .route(web::get().to(list_api_tokens_handler)) .route(web::head().to(|| HttpResponse::Ok())) diff --git a/debug/curl/create_api_token.py b/debug/curl/api_token_tests.py similarity index 56% rename from debug/curl/create_api_token.py rename to debug/curl/api_token_tests.py index 4555dbb673..18be92ed8a 100755 --- a/debug/curl/create_api_token.py +++ b/debug/curl/api_token_tests.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 -import requests +import json import re +import requests import uuid login_url = 'http://localhost:12345/login' create_api_token_url = 'http://localhost:12345/api_tokens/create' +edit_api_token_url = 'http://localhost:12345/api_tokens/edit' +delete_api_token_url = 'http://localhost:12345/api_tokens/{}/delete' payload = { 'username_or_email': 'echelon', @@ -36,3 +39,23 @@ print("===== Created API Token =====") print('Status: {}'.format(r.status_code)) print(r.content) +api_token = json.loads(r.content) + + +payload = { + 'api_token': api_token['api_token'], + 'maybe_short_description': 'this describes my token', +} +r = requests.post(edit_api_token_url, cookies=cookies, json=payload) + +print("===== Edited API Token =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + +url = delete_api_token_url.format(api_token['api_token']) +r = requests.post(url, cookies=cookies) + +print("===== Deleted API Token =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + From 05edf16847e5d8cae65358c83342040c259560b5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 00:36:50 -0500 Subject: [PATCH 0924/1042] update twitch tables with timestamp columns --- .../2022-02-13-223514_twitch_event_action_configs/up.sql | 9 +++++++-- .../2022-02-13-223523_twitch_general_settings/up.sql | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql b/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql index ab6b3280c5..bd473f19bc 100644 --- a/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql +++ b/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql @@ -19,7 +19,7 @@ CREATE TABLE twitch_event_action_rules( -- NB: There is nothing in MySQL to guarantee unique ordering. user_specified_rule_order INT(10) UNSIGNED NOT NULL DEFAULT 0, - -- What type of event we'll be responding to. + -- What type of event we'll be responding to. event_category ENUM( 'bits', 'channel_points', @@ -38,10 +38,15 @@ CREATE TABLE twitch_event_action_rules( -- eg. tts M:model event_response MEDIUMTEXT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP, + -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (token), KEY fk_user_token (user_token), - KEY index_event_category (event_category) + KEY index_event_category (event_category), + KEY index_deleted_at (deleted_at) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_migrations/2022-02-13-223523_twitch_general_settings/up.sql b/_migrations/2022-02-13-223523_twitch_general_settings/up.sql index 76485f9448..7f734e6a2d 100644 --- a/_migrations/2022-02-13-223523_twitch_general_settings/up.sql +++ b/_migrations/2022-02-13-223523_twitch_general_settings/up.sql @@ -23,6 +23,9 @@ CREATE TABLE twitch_general_settings ( -- Can set a fallback voice if a TTS voice goes missing. maybe_fallback_tts_model_token VARCHAR(32) DEFAULT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (user_token) From 4b5c41e67d5888bd894ff9a4bd176e5f5091f023 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 00:38:40 -0500 Subject: [PATCH 0925/1042] add ip address columns --- .../2022-02-13-223514_twitch_event_action_configs/up.sql | 7 ++++++- .../2022-02-13-223523_twitch_general_settings/up.sql | 5 +++++ _migrations/2022-02-16-072149_api_tokens/up.sql | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql b/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql index bd473f19bc..151a108b1e 100644 --- a/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql +++ b/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql @@ -38,9 +38,14 @@ CREATE TABLE twitch_event_action_rules( -- eg. tts M:model event_response MEDIUMTEXT NOT NULL, + -- For abuse tracking. + -- Wide enough for IPv4/6 + ip_address_creation VARCHAR(40) NOT NULL, + ip_address_last_update VARCHAR(40) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP, + deleted_at TIMESTAMP DEFAULT NULL, -- INDICES -- PRIMARY KEY (id), diff --git a/_migrations/2022-02-13-223523_twitch_general_settings/up.sql b/_migrations/2022-02-13-223523_twitch_general_settings/up.sql index 7f734e6a2d..956dcbeb29 100644 --- a/_migrations/2022-02-13-223523_twitch_general_settings/up.sql +++ b/_migrations/2022-02-13-223523_twitch_general_settings/up.sql @@ -23,6 +23,11 @@ CREATE TABLE twitch_general_settings ( -- Can set a fallback voice if a TTS voice goes missing. maybe_fallback_tts_model_token VARCHAR(32) DEFAULT NULL, + -- For abuse tracking. + -- Wide enough for IPv4/6 + ip_address_creation VARCHAR(40) NOT NULL, + ip_address_last_update VARCHAR(40) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, diff --git a/_migrations/2022-02-16-072149_api_tokens/up.sql b/_migrations/2022-02-16-072149_api_tokens/up.sql index b3f8ccb059..08b7bb2698 100644 --- a/_migrations/2022-02-16-072149_api_tokens/up.sql +++ b/_migrations/2022-02-16-072149_api_tokens/up.sql @@ -28,9 +28,9 @@ CREATE TABLE api_tokens ( created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP, + deleted_at TIMESTAMP DEFAULT NULL, - -- INDICES -- + -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (api_token), UNIQUE KEY (uuid_idempotency_token), From 43160dd9e6721fd5c47c288f9cf33cc99b61021c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 02:43:03 -0500 Subject: [PATCH 0926/1042] rename migration --- .../down.sql | 0 .../up.sql | 1 + 2 files changed, 1 insertion(+) rename _migrations/{2022-02-13-223514_twitch_event_action_configs => 2022-02-13-223514_twitch_event_action_rules}/down.sql (100%) rename _migrations/{2022-02-13-223514_twitch_event_action_configs => 2022-02-13-223514_twitch_event_action_rules}/up.sql (96%) diff --git a/_migrations/2022-02-13-223514_twitch_event_action_configs/down.sql b/_migrations/2022-02-13-223514_twitch_event_action_rules/down.sql similarity index 100% rename from _migrations/2022-02-13-223514_twitch_event_action_configs/down.sql rename to _migrations/2022-02-13-223514_twitch_event_action_rules/down.sql diff --git a/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql b/_migrations/2022-02-13-223514_twitch_event_action_rules/up.sql similarity index 96% rename from _migrations/2022-02-13-223514_twitch_event_action_configs/up.sql rename to _migrations/2022-02-13-223514_twitch_event_action_rules/up.sql index 151a108b1e..24c14fb727 100644 --- a/_migrations/2022-02-13-223514_twitch_event_action_configs/up.sql +++ b/_migrations/2022-02-13-223514_twitch_event_action_rules/up.sql @@ -27,6 +27,7 @@ CREATE TABLE twitch_event_action_rules( ) NOT NULL, -- Whether or not the rule is enabled. + -- This is different than deleted. It still shows up in the UI. rule_is_enabled BOOLEAN NOT NULL DEFAULT FALSE, -- A JSON payload containing any predicates we wish to apply to the match. From ba1e61a78e5c0fe91ad78435efe985275695b921 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 02:44:14 -0500 Subject: [PATCH 0927/1042] rename *again*, sort columns --- .../down.sql | 2 +- .../up.sql | 25 ++++++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) rename _migrations/{2022-02-13-223514_twitch_event_action_rules => 2022-02-13-223514_twitch_event_rules}/down.sql (77%) rename _migrations/{2022-02-13-223514_twitch_event_action_rules => 2022-02-13-223514_twitch_event_rules}/up.sql (88%) diff --git a/_migrations/2022-02-13-223514_twitch_event_action_rules/down.sql b/_migrations/2022-02-13-223514_twitch_event_rules/down.sql similarity index 77% rename from _migrations/2022-02-13-223514_twitch_event_action_rules/down.sql rename to _migrations/2022-02-13-223514_twitch_event_rules/down.sql index 5cdec879c1..a2a3be1af1 100644 --- a/_migrations/2022-02-13-223514_twitch_event_action_rules/down.sql +++ b/_migrations/2022-02-13-223514_twitch_event_rules/down.sql @@ -2,4 +2,4 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile -DROP TABLE twitch_event_action_rules; +DROP TABLE twitch_event_rules; diff --git a/_migrations/2022-02-13-223514_twitch_event_action_rules/up.sql b/_migrations/2022-02-13-223514_twitch_event_rules/up.sql similarity index 88% rename from _migrations/2022-02-13-223514_twitch_event_action_rules/up.sql rename to _migrations/2022-02-13-223514_twitch_event_rules/up.sql index 24c14fb727..8860b62d1b 100644 --- a/_migrations/2022-02-13-223514_twitch_event_action_rules/up.sql +++ b/_migrations/2022-02-13-223514_twitch_event_rules/up.sql @@ -2,8 +2,9 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile +-- How to respond to different types of Twitch events. -- Multiple rows per user -CREATE TABLE twitch_event_action_rules( +CREATE TABLE twitch_event_rules( -- Not used for anything except replication. id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, @@ -14,10 +15,7 @@ CREATE TABLE twitch_event_action_rules( -- This is a FakeYou/Storyteller account, *NOT* a twitch user id. user_token VARCHAR(32) NOT NULL, - -- The user can rearrange the rules in the UI. - -- This will be the order they apply in if matched. - -- NB: There is nothing in MySQL to guarantee unique ordering. - user_specified_rule_order INT(10) UNSIGNED NOT NULL DEFAULT 0, + -- ========== EVENT MATCHING AND BEHAVIOR ========== -- What type of event we'll be responding to. event_category ENUM( @@ -26,10 +24,6 @@ CREATE TABLE twitch_event_action_rules( 'chat_command' ) NOT NULL, - -- Whether or not the rule is enabled. - -- This is different than deleted. It still shows up in the UI. - rule_is_enabled BOOLEAN NOT NULL DEFAULT FALSE, - -- A JSON payload containing any predicates we wish to apply to the match. -- eg. bits_value > 100 -- eg. channel_points_award_name == "Mario Voice" @@ -39,6 +33,19 @@ CREATE TABLE twitch_event_action_rules( -- eg. tts M:model event_response MEDIUMTEXT NOT NULL, + -- ========== RULE ORGANIZATION AND MANAGEMENT ========== + + -- The user can rearrange the rules in the UI. + -- This will be the order they apply in if matched. + -- NB: There is nothing in MySQL to guarantee unique ordering. + user_specified_rule_order INT(10) UNSIGNED NOT NULL DEFAULT 0, + + -- Whether or not the rule is enabled. + -- This is different than deleted. It still shows up in the UI. + rule_is_enabled BOOLEAN NOT NULL DEFAULT FALSE, + + -- ========== ABUSE TRACKING, ETC. ========== + -- For abuse tracking. -- Wide enough for IPv4/6 ip_address_creation VARCHAR(40) NOT NULL, From da6cbd5cfa2e76fbbbeefe8f8c2b8f2f78892ade Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 17:02:55 -0500 Subject: [PATCH 0928/1042] insert Twitch event rule --- .../up.sql | 5 ++ .../2022-02-16-072149_api_tokens/up.sql | 2 +- .../database_queries/src/column_types/mod.rs | 3 +- .../src/column_types/twitch_event_category.rs | 35 +++++++++++ crates/lib/database_queries/src/lib.rs | 1 + crates/lib/database_queries/src/tokens.rs | 6 ++ .../twitch_event_rules/create_event_rule.rs | 61 +++++++++++++++++++ .../src/twitch_event_rules/mod.rs | 1 + .../twitch/event_rules/create_event_rule.rs | 0 .../twitch/event_rules/delete_event_rule.rs | 0 .../twitch/event_rules/edit_event_rule.rs | 0 .../event_rules/list_event_rules_for_user.rs | 0 .../endpoints/twitch/event_rules/mod.rs | 4 ++ .../src/http_server/endpoints/twitch/mod.rs | 1 + 14 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 crates/lib/database_queries/src/column_types/twitch_event_category.rs create mode 100644 crates/lib/database_queries/src/twitch_event_rules/create_event_rule.rs create mode 100644 crates/lib/database_queries/src/twitch_event_rules/mod.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/delete_event_rule.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs diff --git a/_migrations/2022-02-13-223514_twitch_event_rules/up.sql b/_migrations/2022-02-13-223514_twitch_event_rules/up.sql index 8860b62d1b..fa49582ef0 100644 --- a/_migrations/2022-02-13-223514_twitch_event_rules/up.sql +++ b/_migrations/2022-02-13-223514_twitch_event_rules/up.sql @@ -8,6 +8,10 @@ CREATE TABLE twitch_event_rules( -- Not used for anything except replication. id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + -- Idempotency token from client + -- This is so the frontend client doesn't submit duplicate jobs. + uuid_idempotency_token VARCHAR(36) NOT NULL, + -- Effective "primary key" (PUBLIC) token VARCHAR(32) NOT NULL, @@ -57,6 +61,7 @@ CREATE TABLE twitch_event_rules( -- INDICES -- PRIMARY KEY (id), + UNIQUE KEY (uuid_idempotency_token), UNIQUE KEY (token), KEY fk_user_token (user_token), KEY index_event_category (event_category), diff --git a/_migrations/2022-02-16-072149_api_tokens/up.sql b/_migrations/2022-02-16-072149_api_tokens/up.sql index 08b7bb2698..900dc7239b 100644 --- a/_migrations/2022-02-16-072149_api_tokens/up.sql +++ b/_migrations/2022-02-16-072149_api_tokens/up.sql @@ -32,8 +32,8 @@ CREATE TABLE api_tokens ( -- INDICES -- PRIMARY KEY (id), - UNIQUE KEY (api_token), UNIQUE KEY (uuid_idempotency_token), + UNIQUE KEY (api_token), KEY fk_user_token (user_token), KEY index_created_at (created_at), KEY index_deleted_at (deleted_at) diff --git a/crates/lib/database_queries/src/column_types/mod.rs b/crates/lib/database_queries/src/column_types/mod.rs index 1c108ef9c1..6ecfe45966 100644 --- a/crates/lib/database_queries/src/column_types/mod.rs +++ b/crates/lib/database_queries/src/column_types/mod.rs @@ -1,3 +1,4 @@ pub mod job_status; pub mod record_visibility; -pub mod vocoder_type; \ No newline at end of file +pub mod twitch_event_category; +pub mod vocoder_type; diff --git a/crates/lib/database_queries/src/column_types/twitch_event_category.rs b/crates/lib/database_queries/src/column_types/twitch_event_category.rs new file mode 100644 index 0000000000..269b5d9ccf --- /dev/null +++ b/crates/lib/database_queries/src/column_types/twitch_event_category.rs @@ -0,0 +1,35 @@ +//! These are columns where users can control the visibility of their data. + +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; + +/// To use this in a query, the query must have type annotations. +/// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375 +/// eg. twitch_event_category as `twitch_event_category: crate::column_types::twitch_event_category::TwitchEventCategory` +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] +#[serde(rename_all = "snake_case")] +#[sqlx(rename_all = "snake_case")] +pub enum TwitchEventCategory { + Bits, + ChannelPoints, + ChatCommand, +} + +impl TwitchEventCategory { + pub fn to_str(&self) -> &'static str { + match self { + Self::Bits => "bits", + Self::ChannelPoints => "channel_points", + Self::ChatCommand => "chat_command", + } + } + + pub fn from_str(value: &str) -> AnyhowResult { + match value { + "bits" => Ok(Self::Bits), + "channel_points" => Ok(Self::ChannelPoints), + "chat_command" => Ok(Self::ChatCommand), + _ => Err(anyhow!("invalid value: {:?}", value)), + } + } +} diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index 57b7a39eda..90c1d15440 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -7,5 +7,6 @@ pub mod column_types; pub mod helpers; pub mod tokens; pub mod tts; +pub mod twitch_event_rules; pub mod twitch_oauth; pub mod twitch_pubsub; diff --git a/crates/lib/database_queries/src/tokens.rs b/crates/lib/database_queries/src/tokens.rs index 8d1b9e9336..eebb15f7fe 100644 --- a/crates/lib/database_queries/src/tokens.rs +++ b/crates/lib/database_queries/src/tokens.rs @@ -66,4 +66,10 @@ impl Tokens { pub fn new_twitch_oauth_grouping_token() -> AnyhowResult { random_prefix_crockford_token("OG:", 32, false) } + + // ========== [Twitch] Tokens not typically visible to users ========== + + pub fn new_twitch_event_rule() -> AnyhowResult { + random_prefix_crockford_token("TER:", 32, false) + } } diff --git a/crates/lib/database_queries/src/twitch_event_rules/create_event_rule.rs b/crates/lib/database_queries/src/twitch_event_rules/create_event_rule.rs new file mode 100644 index 0000000000..6eae6d8685 --- /dev/null +++ b/crates/lib/database_queries/src/twitch_event_rules/create_event_rule.rs @@ -0,0 +1,61 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::twitch_event_category::TwitchEventCategory; +use crate::tokens::Tokens; +use sqlx::MySqlPool; + +pub struct TwitchEventRuleInsertBuilder { + pub uuid_idempotency_token: String, + pub user_token: String, + pub event_category: TwitchEventCategory, + pub event_match_predicate: String, + pub event_response: String, + pub user_specified_rule_order: u32, + pub rule_is_enabled: bool, + pub ip_address_creation: String, +} + +impl TwitchEventRuleInsertBuilder { + + /// Returns the newly generated token. + pub async fn insert(&self, mysql_pool: &MySqlPool) -> AnyhowResult { + + let token = Tokens::new_twitch_event_rule()?; + + let query = sqlx::query!( + r#" +INSERT INTO twitch_event_rules +SET + uuid_idempotency_token = ?, + token = ?, + user_token = ?, + event_category = ?, + event_match_predicate = ?, + event_response = ?, + user_specified_rule_order = ?, + rule_is_enabled = ?, + ip_address_creation = ?, + ip_address_last_update = ? + "#, + &self.uuid_idempotency_token, + &token, + &self.user_token, + &self.event_category, + &self.event_match_predicate, + &self.event_response, + &self.user_specified_rule_order, + &self.rule_is_enabled, + &self.ip_address_creation, + &self.ip_address_creation, + ); + + let query_result = query.execute(mysql_pool).await; + + let _record_id = match query_result { + Ok(res) => res.last_insert_id(), + Err(err) => return Err(anyhow!("Error creating Twitch Event Rule: {:?}", err)), + }; + + Ok(token) + } +} diff --git a/crates/lib/database_queries/src/twitch_event_rules/mod.rs b/crates/lib/database_queries/src/twitch_event_rules/mod.rs new file mode 100644 index 0000000000..690e3110f8 --- /dev/null +++ b/crates/lib/database_queries/src/twitch_event_rules/mod.rs @@ -0,0 +1 @@ +pub mod create_event_rule; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/delete_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/delete_event_rule.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs new file mode 100644 index 0000000000..4f40936c19 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs @@ -0,0 +1,4 @@ +pub mod create_event_rule; +pub mod delete_event_rule; +pub mod edit_event_rule; +pub mod list_event_rules_for_user; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs index 0b47d3ca95..1d15747836 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs @@ -1,3 +1,4 @@ +pub mod event_rules; pub mod oauth_begin_json; pub mod oauth_begin_redirect; pub mod oauth_end; From dcde6edba425b2b0328144bbd8eb06e5250a54dd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 17:17:46 -0500 Subject: [PATCH 0929/1042] delete twitch event rule --- .../src/api_tokens/delete_api_token.rs | 2 -- ...nt_rule.rs => create_twitch_event_rule.rs} | 0 .../delete_twitch_event_rule.rs | 33 +++++++++++++++++++ .../src/twitch_event_rules/mod.rs | 3 +- 4 files changed, 35 insertions(+), 3 deletions(-) rename crates/lib/database_queries/src/twitch_event_rules/{create_event_rule.rs => create_twitch_event_rule.rs} (100%) create mode 100644 crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs diff --git a/crates/lib/database_queries/src/api_tokens/delete_api_token.rs b/crates/lib/database_queries/src/api_tokens/delete_api_token.rs index 3c224b35b6..dc1e2d00de 100644 --- a/crates/lib/database_queries/src/api_tokens/delete_api_token.rs +++ b/crates/lib/database_queries/src/api_tokens/delete_api_token.rs @@ -1,8 +1,6 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use log::warn; use sqlx::MySqlPool; -use std::collections::{HashSet, HashMap}; pub async fn delete_api_token( user_token: &str, diff --git a/crates/lib/database_queries/src/twitch_event_rules/create_event_rule.rs b/crates/lib/database_queries/src/twitch_event_rules/create_twitch_event_rule.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_event_rules/create_event_rule.rs rename to crates/lib/database_queries/src/twitch_event_rules/create_twitch_event_rule.rs diff --git a/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs b/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs new file mode 100644 index 0000000000..18e890424c --- /dev/null +++ b/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs @@ -0,0 +1,33 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use sqlx::MySqlPool; + +pub async fn delete_twitch_event_rule( + event_rule_token: &str, + ip_address_delete: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult { + + let query = sqlx::query!( + r#" +UPDATE twitch_event_rules +SET + deleted_at = CURRENT_TIMESTAMP, + ip_address_last_update = ? +WHERE + token = ? +LIMIT 1 + "#, + ip_address_delete, + event_rule_token, + ); + + let result = query.execute(mysql_pool).await; + + match result { + Err(err) => { + Err(anyhow!("error with query: {:?}", err)) + }, + Ok(results) => Ok(true), + } +} diff --git a/crates/lib/database_queries/src/twitch_event_rules/mod.rs b/crates/lib/database_queries/src/twitch_event_rules/mod.rs index 690e3110f8..e73b7171ad 100644 --- a/crates/lib/database_queries/src/twitch_event_rules/mod.rs +++ b/crates/lib/database_queries/src/twitch_event_rules/mod.rs @@ -1 +1,2 @@ -pub mod create_event_rule; \ No newline at end of file +pub mod create_twitch_event_rule; +pub mod delete_twitch_event_rule; \ No newline at end of file From d9039b9071533df185cb231e01a278fa5ce27706 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 18:26:34 -0500 Subject: [PATCH 0930/1042] cleanup some compile warnings --- .../src/api_tokens/delete_api_token.rs | 2 +- .../database_queries/src/api_tokens/edit_api_token.rs | 2 +- .../tts_inference_jobs/mark_tts_inference_job_done.rs | 10 ++++++++-- .../mark_tts_inference_job_failure.rs | 10 ++++++++-- .../mark_tts_inference_job_permanently_dead.rs | 10 ++++++++-- .../src/twitch_event_rules/delete_twitch_event_rule.rs | 2 +- crates/lib/database_queries/src/twitch_oauth/find.rs | 6 +++--- 7 files changed, 30 insertions(+), 12 deletions(-) diff --git a/crates/lib/database_queries/src/api_tokens/delete_api_token.rs b/crates/lib/database_queries/src/api_tokens/delete_api_token.rs index dc1e2d00de..7345fbcc0d 100644 --- a/crates/lib/database_queries/src/api_tokens/delete_api_token.rs +++ b/crates/lib/database_queries/src/api_tokens/delete_api_token.rs @@ -31,6 +31,6 @@ LIMIT 1 Err(err) => { Err(anyhow!("error with query: {:?}", err)) }, - Ok(results) => Ok(true), + Ok(_r) => Ok(true), } } diff --git a/crates/lib/database_queries/src/api_tokens/edit_api_token.rs b/crates/lib/database_queries/src/api_tokens/edit_api_token.rs index 9dd62164e8..0d4a668900 100644 --- a/crates/lib/database_queries/src/api_tokens/edit_api_token.rs +++ b/crates/lib/database_queries/src/api_tokens/edit_api_token.rs @@ -35,6 +35,6 @@ LIMIT 1 Err(err) => { Err(anyhow!("error with query: {:?}", err)) }, - Ok(results) => Ok(true), + Ok(_r) => Ok(true), } } diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_done.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_done.rs index 5acb43700d..6d94cf3ff6 100644 --- a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_done.rs +++ b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_done.rs @@ -1,3 +1,4 @@ +use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; use sqlx::MySqlPool; @@ -26,8 +27,13 @@ WHERE id = ? job_id.0 ) .execute(pool) - .await?; + .await; - Ok(()) + match query_result { + Err(err) => { + Err(anyhow!("error with query: {:?}", err)) + }, + Ok(_r) => Ok(()), + } } diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs index 19824707be..3bb8a93143 100644 --- a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs +++ b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs @@ -1,3 +1,4 @@ +use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; use crate::tts::tts_inference_jobs::list_available_tts_inference_jobs::AvailableTtsInferenceJob; use sqlx::MySqlPool; @@ -35,7 +36,12 @@ WHERE id = ? job.id.0, ) .execute(pool) - .await?; + .await; - Ok(()) + match query_result { + Err(err) => { + Err(anyhow!("error with query: {:?}", err)) + }, + Ok(_r) => Ok(()), + } } diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs index e1441afb38..58c605d9c5 100644 --- a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs +++ b/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs @@ -1,3 +1,4 @@ +use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; use sqlx::MySqlPool; @@ -22,7 +23,12 @@ WHERE id = ? job_id.0, ) .execute(pool) - .await?; + .await; - Ok(()) + match query_result { + Err(err) => { + Err(anyhow!("error with query: {:?}", err)) + }, + Ok(_r) => Ok(()), + } } diff --git a/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs b/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs index 18e890424c..966624552e 100644 --- a/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs +++ b/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs @@ -28,6 +28,6 @@ LIMIT 1 Err(err) => { Err(anyhow!("error with query: {:?}", err)) }, - Ok(results) => Ok(true), + Ok(_r) => Ok(true), } } diff --git a/crates/lib/database_queries/src/twitch_oauth/find.rs b/crates/lib/database_queries/src/twitch_oauth/find.rs index debe662551..453c674b23 100644 --- a/crates/lib/database_queries/src/twitch_oauth/find.rs +++ b/crates/lib/database_queries/src/twitch_oauth/find.rs @@ -203,7 +203,7 @@ LEFT OUTER JOIN users let mut query = "".to_string(); let mut first_predicate_added = false; - if let Some(user_token) = self.scope_user_token.as_deref() { + if let Some(_user_token) = self.scope_user_token.as_deref() { if !first_predicate_added { query.push_str(" WHERE users.token = ?"); first_predicate_added = true; @@ -212,7 +212,7 @@ LEFT OUTER JOIN users } } - if let Some(user_id) = self.scope_twitch_user_id { + if let Some(_user_id) = self.scope_twitch_user_id { if !first_predicate_added { query.push_str(" WHERE twitch_oauth_tokens.twitch_user_id = ?"); first_predicate_added = true; @@ -221,7 +221,7 @@ LEFT OUTER JOIN users } } - if let Some(username) = self.scope_twitch_username_lowercase.as_deref() { + if let Some(_username) = self.scope_twitch_username_lowercase.as_deref() { if !first_predicate_added { query.push_str(" WHERE twitch_oauth_tokens.twitch_username_lowercase = ?"); first_predicate_added = true; From 73de5d185c816ad61421825d16444464d1e1ebf9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 18:34:27 -0500 Subject: [PATCH 0931/1042] Add API token internal token as foreign key --- _migrations/2022-02-16-072149_api_tokens/up.sql | 8 +++++++- .../database_queries/src/api_tokens/create_api_token.rs | 3 +++ .../src/api_tokens/list_available_api_tokens_for_user.rs | 4 ++++ crates/lib/database_queries/src/tokens.rs | 8 ++++++-- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/_migrations/2022-02-16-072149_api_tokens/up.sql b/_migrations/2022-02-16-072149_api_tokens/up.sql index 900dc7239b..7873bb4889 100644 --- a/_migrations/2022-02-16-072149_api_tokens/up.sql +++ b/_migrations/2022-02-16-072149_api_tokens/up.sql @@ -11,7 +11,12 @@ CREATE TABLE api_tokens ( -- This is so the frontend client doesn't submit duplicate jobs. uuid_idempotency_token VARCHAR(36) NOT NULL, - -- This is the secret API token value + -- This is a private internal reference to the API token. + -- Not shared with consumers / users. + -- This should be used as the foreign key for the token. + internal_token VARCHAR(32) NOT NULL, + + -- This is the secret API token value given to consumers api_token VARCHAR(32) NOT NULL, -- Foreign key to user that owns the API token. @@ -33,6 +38,7 @@ CREATE TABLE api_tokens ( -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (uuid_idempotency_token), + UNIQUE KEY (internal_token), UNIQUE KEY (api_token), KEY fk_user_token (user_token), KEY index_created_at (created_at), diff --git a/crates/lib/database_queries/src/api_tokens/create_api_token.rs b/crates/lib/database_queries/src/api_tokens/create_api_token.rs index 80b0e3352c..4a9318cbb5 100644 --- a/crates/lib/database_queries/src/api_tokens/create_api_token.rs +++ b/crates/lib/database_queries/src/api_tokens/create_api_token.rs @@ -17,18 +17,21 @@ pub async fn create_api_token_for_user( // TODO: Get last 5 API tokens for the user. // TODO: Database transaction. + let internal_token = Tokens::new_api_token_internal_token()?; let api_token = Tokens::new_api_token()?; let query = sqlx::query!( r#" INSERT INTO api_tokens SET + internal_token = ?, api_token = ?, user_token = ?, uuid_idempotency_token = ?, ip_address_creation = ?, ip_address_last_update = ? "#, + internal_token, api_token, user_token, uuid_idempotency_token, diff --git a/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs b/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs index 83b7f38a74..0c4512f08d 100644 --- a/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs +++ b/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs @@ -11,6 +11,7 @@ use sqlx::MySqlPool; /// table: api_tokens #[derive(Debug)] pub struct AvailableApiToken { + pub internal_token: String, pub api_token: String, pub maybe_short_description: Option, pub created_at: chrono::DateTime, @@ -28,6 +29,7 @@ pub async fn list_available_api_tokens_for_user( AvailableApiTokenInternal, r#" SELECT + internal_token, api_token, maybe_short_description, created_at, @@ -47,6 +49,7 @@ LIMIT 5 let mut records = records.into_iter() .map(|record : AvailableApiTokenInternal | { AvailableApiToken { + internal_token: record.internal_token, api_token: record.api_token, maybe_short_description: record.maybe_short_description, created_at: record.created_at, @@ -63,6 +66,7 @@ LIMIT 5 #[derive(Debug)] struct AvailableApiTokenInternal { + pub internal_token: String, pub api_token: String, pub maybe_short_description: Option, pub created_at: chrono::DateTime, diff --git a/crates/lib/database_queries/src/tokens.rs b/crates/lib/database_queries/src/tokens.rs index eebb15f7fe..48ecf16b99 100644 --- a/crates/lib/database_queries/src/tokens.rs +++ b/crates/lib/database_queries/src/tokens.rs @@ -63,12 +63,16 @@ impl Tokens { random_prefix_crockford_token("CAT:", 15, false) } - pub fn new_twitch_oauth_grouping_token() -> AnyhowResult { - random_prefix_crockford_token("OG:", 32, false) + pub fn new_api_token_internal_token() -> AnyhowResult { + random_prefix_crockford_token("AIT:", 32, false) } // ========== [Twitch] Tokens not typically visible to users ========== + pub fn new_twitch_oauth_grouping_token() -> AnyhowResult { + random_prefix_crockford_token("OG:", 32, false) + } + pub fn new_twitch_event_rule() -> AnyhowResult { random_prefix_crockford_token("TER:", 32, false) } From 416c32d166a54ac4928c703c2b031902f308b110 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 18:48:41 -0500 Subject: [PATCH 0932/1042] update twitch event rule --- ...rs => insert_twitch_event_rule_builder.rs} | 4 +- .../src/twitch_event_rules/mod.rs | 5 +- .../update_twitch_event_rule_builder.rs | 50 +++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) rename crates/lib/database_queries/src/twitch_event_rules/{create_twitch_event_rule.rs => insert_twitch_event_rule_builder.rs} (95%) create mode 100644 crates/lib/database_queries/src/twitch_event_rules/update_twitch_event_rule_builder.rs diff --git a/crates/lib/database_queries/src/twitch_event_rules/create_twitch_event_rule.rs b/crates/lib/database_queries/src/twitch_event_rules/insert_twitch_event_rule_builder.rs similarity index 95% rename from crates/lib/database_queries/src/twitch_event_rules/create_twitch_event_rule.rs rename to crates/lib/database_queries/src/twitch_event_rules/insert_twitch_event_rule_builder.rs index 6eae6d8685..7ff73a4e9c 100644 --- a/crates/lib/database_queries/src/twitch_event_rules/create_twitch_event_rule.rs +++ b/crates/lib/database_queries/src/twitch_event_rules/insert_twitch_event_rule_builder.rs @@ -4,7 +4,7 @@ use crate::column_types::twitch_event_category::TwitchEventCategory; use crate::tokens::Tokens; use sqlx::MySqlPool; -pub struct TwitchEventRuleInsertBuilder { +pub struct InsertTwitchEventRuleBuilder { pub uuid_idempotency_token: String, pub user_token: String, pub event_category: TwitchEventCategory, @@ -15,7 +15,7 @@ pub struct TwitchEventRuleInsertBuilder { pub ip_address_creation: String, } -impl TwitchEventRuleInsertBuilder { +impl InsertTwitchEventRuleBuilder { /// Returns the newly generated token. pub async fn insert(&self, mysql_pool: &MySqlPool) -> AnyhowResult { diff --git a/crates/lib/database_queries/src/twitch_event_rules/mod.rs b/crates/lib/database_queries/src/twitch_event_rules/mod.rs index e73b7171ad..45a23b33db 100644 --- a/crates/lib/database_queries/src/twitch_event_rules/mod.rs +++ b/crates/lib/database_queries/src/twitch_event_rules/mod.rs @@ -1,2 +1,3 @@ -pub mod create_twitch_event_rule; -pub mod delete_twitch_event_rule; \ No newline at end of file +pub mod delete_twitch_event_rule; +pub mod insert_twitch_event_rule_builder; +pub mod update_twitch_event_rule_builder; \ No newline at end of file diff --git a/crates/lib/database_queries/src/twitch_event_rules/update_twitch_event_rule_builder.rs b/crates/lib/database_queries/src/twitch_event_rules/update_twitch_event_rule_builder.rs new file mode 100644 index 0000000000..8d07c5d311 --- /dev/null +++ b/crates/lib/database_queries/src/twitch_event_rules/update_twitch_event_rule_builder.rs @@ -0,0 +1,50 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use crate::tokens::Tokens; +use sqlx::MySqlPool; + +// TODO: This doesn't handle sparse updates. + +pub struct UpdateTwitchEventRuleBuilder { + pub token: String, + pub event_match_predicate: String, + pub event_response: String, + pub user_specified_rule_order: u32, + pub rule_is_enabled: bool, + pub ip_address_update: String, +} + +impl UpdateTwitchEventRuleBuilder { + + pub async fn update(&self, mysql_pool: &MySqlPool) -> AnyhowResult<()> { + + let query = sqlx::query!( + r#" +UPDATE twitch_event_rules +SET + event_match_predicate = ?, + event_response = ?, + user_specified_rule_order = ?, + rule_is_enabled = ?, + ip_address_last_update = ? +WHERE + token = ? + AND deleted_at IS NULL +LIMIT 1 + "#, + &self.event_match_predicate, + &self.event_response, + &self.user_specified_rule_order, + &self.rule_is_enabled, + &self.ip_address_update, + &self.token, + ); + + let query_result = query.execute(mysql_pool).await; + + match query_result { + Err(err) => return Err(anyhow!("Error updating Twitch Event Rule: {:?}", err)), + Ok(_r) => Ok(()), + } + } +} From 1c3dbbe7799555894f8dd208d35fd8f8615fc59f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 19:02:29 -0500 Subject: [PATCH 0933/1042] list twitch event categories --- .../list_available_api_tokens_for_user.rs | 5 -- .../list_twitch_event_rules.rs | 80 +++++++++++++++++++ .../src/twitch_event_rules/mod.rs | 3 +- 3 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules.rs diff --git a/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs b/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs index 0c4512f08d..034d24d131 100644 --- a/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs +++ b/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs @@ -1,11 +1,6 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; use container_common::anyhow_result::AnyhowResult; -use crate::column_types::job_status::JobStatus; -use crate::column_types::record_visibility::RecordVisibility; -use crate::helpers::boolean_converters::i8_to_bool; -use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; -use log::{warn, info}; use sqlx::MySqlPool; /// table: api_tokens diff --git a/crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules.rs b/crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules.rs new file mode 100644 index 0000000000..dcc9a44ff7 --- /dev/null +++ b/crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules.rs @@ -0,0 +1,80 @@ +use anyhow::anyhow; +use chrono::{Utc, DateTime}; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::twitch_event_category::TwitchEventCategory; +use crate::helpers::boolean_converters::i8_to_bool; +use sqlx::MySqlPool; + +/// table: api_tokens +#[derive(Debug)] +pub struct TwitchEventRule { + pub token: String, + pub event_category: TwitchEventCategory, + pub event_match_predicate: String, + pub event_response: String, + pub user_specified_rule_order: u32, + pub rule_is_enabled: bool, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, +} + +/// Query non-deleted Twitch event rules for a user +pub async fn list_twitch_event_rules_for_user( + user_token: &str, + pool: &MySqlPool, +) -> AnyhowResult> { + + let records : Vec = sqlx::query_as!( + TwitchEventRuleInternal, + r#" +SELECT + token, + event_category as `event_category: crate::column_types::twitch_event_category::TwitchEventCategory`, + event_match_predicate, + event_response, + user_specified_rule_order, + rule_is_enabled, + created_at, + updated_at +FROM twitch_event_rules +WHERE + user_token = ? + AND deleted_at IS NULL + "#, + user_token, + ) + .fetch_all(pool) + .await?; + + let mut records = records.into_iter() + .map(|record : TwitchEventRuleInternal | { + TwitchEventRule { + token: record.token, + event_category: record.event_category, + event_match_predicate: record.event_match_predicate, + event_response: record.event_response, + user_specified_rule_order: record.user_specified_rule_order, + rule_is_enabled: i8_to_bool(record.rule_is_enabled), + created_at: record.created_at, + updated_at: record.updated_at, + } + }) + .collect::>(); + + // Queried in DESC order, but sort returned results ordered ASC. + records.sort_by_key(|r| r.created_at); + + Ok(records) +} + +#[derive(Debug)] +struct TwitchEventRuleInternal { + pub token: String, + pub event_category: TwitchEventCategory, + pub event_match_predicate: String, + pub event_response: String, + pub user_specified_rule_order: u32, + pub rule_is_enabled: i8, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, +} diff --git a/crates/lib/database_queries/src/twitch_event_rules/mod.rs b/crates/lib/database_queries/src/twitch_event_rules/mod.rs index 45a23b33db..7dd7672371 100644 --- a/crates/lib/database_queries/src/twitch_event_rules/mod.rs +++ b/crates/lib/database_queries/src/twitch_event_rules/mod.rs @@ -1,3 +1,4 @@ pub mod delete_twitch_event_rule; pub mod insert_twitch_event_rule_builder; -pub mod update_twitch_event_rule_builder; \ No newline at end of file +pub mod update_twitch_event_rule_builder; +mod list_twitch_event_rules; \ No newline at end of file From cdc04b66d7a54e376a128f5f6b202644ad2c73b5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 19:13:52 -0500 Subject: [PATCH 0934/1042] get twitch event rule for user --- .../get_twitch_event_rule_for_user.rs | 82 +++++++++++++++++++ ...rs => list_twitch_event_rules_for_user.rs} | 0 .../src/twitch_event_rules/mod.rs | 5 +- 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 crates/lib/database_queries/src/twitch_event_rules/get_twitch_event_rule_for_user.rs rename crates/lib/database_queries/src/twitch_event_rules/{list_twitch_event_rules.rs => list_twitch_event_rules_for_user.rs} (100%) diff --git a/crates/lib/database_queries/src/twitch_event_rules/get_twitch_event_rule_for_user.rs b/crates/lib/database_queries/src/twitch_event_rules/get_twitch_event_rule_for_user.rs new file mode 100644 index 0000000000..850c3e3f6f --- /dev/null +++ b/crates/lib/database_queries/src/twitch_event_rules/get_twitch_event_rule_for_user.rs @@ -0,0 +1,82 @@ +use anyhow::anyhow; +use chrono::{Utc, DateTime}; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::twitch_event_category::TwitchEventCategory; +use crate::helpers::boolean_converters::i8_to_bool; +use sqlx::MySqlPool; + +#[derive(Debug)] +pub struct TwitchEventRule { + pub token: String, + pub event_category: TwitchEventCategory, + pub event_match_predicate: String, + pub event_response: String, + pub user_specified_rule_order: u32, + pub rule_is_enabled: bool, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub deleted_at: Option>, +} + +/// Query record by token and user token +pub async fn get_twitch_event_rules_for_user( + twitch_event_rule_token: &str, + user_token: &str, + pool: &MySqlPool, +) -> AnyhowResult> { + + let maybe_record = sqlx::query_as!( + TwitchEventRuleInternal, + r#" +SELECT + token, + event_category as `event_category: crate::column_types::twitch_event_category::TwitchEventCategory`, + event_match_predicate, + event_response, + user_specified_rule_order, + rule_is_enabled, + created_at, + updated_at, + deleted_at +FROM twitch_event_rules +WHERE + token = ? + AND user_token = ? + "#, + twitch_event_rule_token, + user_token, + ) + .fetch_one(pool) + .await; + + let record : TwitchEventRuleInternal = match maybe_record { + Ok(record) => record, + Err(sqlx::Error::RowNotFound) => return Ok(None), + Err(ref err) => return Err(anyhow!("database query error: {:?}", err)), + }; + + Ok(Some(TwitchEventRule { + token: record.token, + event_category: record.event_category, + event_match_predicate: record.event_match_predicate, + event_response: record.event_response, + user_specified_rule_order: record.user_specified_rule_order, + rule_is_enabled: i8_to_bool(record.rule_is_enabled), + created_at: record.created_at, + updated_at: record.updated_at, + deleted_at: record.deleted_at, + })) +} + +#[derive(Debug)] +struct TwitchEventRuleInternal { + pub token: String, + pub event_category: TwitchEventCategory, + pub event_match_predicate: String, + pub event_response: String, + pub user_specified_rule_order: u32, + pub rule_is_enabled: i8, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub deleted_at: Option>, +} diff --git a/crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules.rs b/crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules_for_user.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules.rs rename to crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules_for_user.rs diff --git a/crates/lib/database_queries/src/twitch_event_rules/mod.rs b/crates/lib/database_queries/src/twitch_event_rules/mod.rs index 7dd7672371..287e5bc6ef 100644 --- a/crates/lib/database_queries/src/twitch_event_rules/mod.rs +++ b/crates/lib/database_queries/src/twitch_event_rules/mod.rs @@ -1,4 +1,5 @@ pub mod delete_twitch_event_rule; +pub mod get_twitch_event_rule_for_user; pub mod insert_twitch_event_rule_builder; -pub mod update_twitch_event_rule_builder; -mod list_twitch_event_rules; \ No newline at end of file +pub mod list_twitch_event_rules_for_user; +pub mod update_twitch_event_rule_builder; \ No newline at end of file From 5b7d533dc3b3dcaca6b8812c5dba6456b4cacb14 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 19:17:21 -0500 Subject: [PATCH 0935/1042] cleanup --- .../src/api_tokens/create_api_token.rs | 11 +++++------ .../src/api_tokens/delete_api_token.rs | 4 +--- .../database_queries/src/api_tokens/edit_api_token.rs | 4 +--- .../twitch_event_rules/delete_twitch_event_rule.rs | 4 +--- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/crates/lib/database_queries/src/api_tokens/create_api_token.rs b/crates/lib/database_queries/src/api_tokens/create_api_token.rs index 4a9318cbb5..faa7403a54 100644 --- a/crates/lib/database_queries/src/api_tokens/create_api_token.rs +++ b/crates/lib/database_queries/src/api_tokens/create_api_token.rs @@ -1,9 +1,9 @@ -use container_common::anyhow_result::AnyhowResult; -use sqlx::MySqlPool; -use crate::tokens::Tokens; -use sqlx; use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; use crate::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; +use crate::tokens::Tokens; +use sqlx::MySqlPool; +use sqlx; /// Create a new API token for the user. /// If the user has more than five tokens, delete the least recent. @@ -14,7 +14,6 @@ pub async fn create_api_token_for_user( mysql_pool: &MySqlPool ) -> AnyhowResult { - // TODO: Get last 5 API tokens for the user. // TODO: Database transaction. let internal_token = Tokens::new_api_token_internal_token()?; @@ -53,7 +52,7 @@ SET .map(|r| r.api_token) .collect::>(); - // TODO: THis is a massive hack since SQLx doesn't support binding Vec + // TODO: This is a massive hack since SQLx doesn't support binding Vec // This array's elements will be replaced with any valid tokens. let mut api_tokens = [ api_token.as_str(), diff --git a/crates/lib/database_queries/src/api_tokens/delete_api_token.rs b/crates/lib/database_queries/src/api_tokens/delete_api_token.rs index 7345fbcc0d..7f8f310ba0 100644 --- a/crates/lib/database_queries/src/api_tokens/delete_api_token.rs +++ b/crates/lib/database_queries/src/api_tokens/delete_api_token.rs @@ -28,9 +28,7 @@ LIMIT 1 let result = query.execute(mysql_pool).await; match result { - Err(err) => { - Err(anyhow!("error with query: {:?}", err)) - }, + Err(err) => Err(anyhow!("error with query: {:?}", err)), Ok(_r) => Ok(true), } } diff --git a/crates/lib/database_queries/src/api_tokens/edit_api_token.rs b/crates/lib/database_queries/src/api_tokens/edit_api_token.rs index 0d4a668900..909075c105 100644 --- a/crates/lib/database_queries/src/api_tokens/edit_api_token.rs +++ b/crates/lib/database_queries/src/api_tokens/edit_api_token.rs @@ -32,9 +32,7 @@ LIMIT 1 let result = query.execute(mysql_pool).await; match result { - Err(err) => { - Err(anyhow!("error with query: {:?}", err)) - }, + Err(err) => Err(anyhow!("error with query: {:?}", err)), Ok(_r) => Ok(true), } } diff --git a/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs b/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs index 966624552e..914856bb9d 100644 --- a/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs +++ b/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs @@ -25,9 +25,7 @@ LIMIT 1 let result = query.execute(mysql_pool).await; match result { - Err(err) => { - Err(anyhow!("error with query: {:?}", err)) - }, + Err(err) => Err(anyhow!("error with query: {:?}", err)), Ok(_r) => Ok(true), } } From de0bd9f09d18ba06e9081df5c91c18817a8f5698 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 19:21:50 -0500 Subject: [PATCH 0936/1042] rule_is_enabled --> rule_is_disabled --- _migrations/2022-02-13-223514_twitch_event_rules/up.sql | 2 +- .../twitch_event_rules/get_twitch_event_rule_for_user.rs | 8 ++++---- .../insert_twitch_event_rule_builder.rs | 6 +++--- .../list_twitch_event_rules_for_user.rs | 8 ++++---- .../update_twitch_event_rule_builder.rs | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/_migrations/2022-02-13-223514_twitch_event_rules/up.sql b/_migrations/2022-02-13-223514_twitch_event_rules/up.sql index fa49582ef0..bd6f18513f 100644 --- a/_migrations/2022-02-13-223514_twitch_event_rules/up.sql +++ b/_migrations/2022-02-13-223514_twitch_event_rules/up.sql @@ -46,7 +46,7 @@ CREATE TABLE twitch_event_rules( -- Whether or not the rule is enabled. -- This is different than deleted. It still shows up in the UI. - rule_is_enabled BOOLEAN NOT NULL DEFAULT FALSE, + rule_is_disabled BOOLEAN NOT NULL DEFAULT FALSE, -- ========== ABUSE TRACKING, ETC. ========== diff --git a/crates/lib/database_queries/src/twitch_event_rules/get_twitch_event_rule_for_user.rs b/crates/lib/database_queries/src/twitch_event_rules/get_twitch_event_rule_for_user.rs index 850c3e3f6f..1e6a4ebf76 100644 --- a/crates/lib/database_queries/src/twitch_event_rules/get_twitch_event_rule_for_user.rs +++ b/crates/lib/database_queries/src/twitch_event_rules/get_twitch_event_rule_for_user.rs @@ -12,7 +12,7 @@ pub struct TwitchEventRule { pub event_match_predicate: String, pub event_response: String, pub user_specified_rule_order: u32, - pub rule_is_enabled: bool, + pub rule_is_disabled: bool, pub created_at: chrono::DateTime, pub updated_at: chrono::DateTime, pub deleted_at: Option>, @@ -34,7 +34,7 @@ SELECT event_match_predicate, event_response, user_specified_rule_order, - rule_is_enabled, + rule_is_disabled, created_at, updated_at, deleted_at @@ -61,7 +61,7 @@ WHERE event_match_predicate: record.event_match_predicate, event_response: record.event_response, user_specified_rule_order: record.user_specified_rule_order, - rule_is_enabled: i8_to_bool(record.rule_is_enabled), + rule_is_disabled: i8_to_bool(record.rule_is_disabled), created_at: record.created_at, updated_at: record.updated_at, deleted_at: record.deleted_at, @@ -75,7 +75,7 @@ struct TwitchEventRuleInternal { pub event_match_predicate: String, pub event_response: String, pub user_specified_rule_order: u32, - pub rule_is_enabled: i8, + pub rule_is_disabled: i8, pub created_at: chrono::DateTime, pub updated_at: chrono::DateTime, pub deleted_at: Option>, diff --git a/crates/lib/database_queries/src/twitch_event_rules/insert_twitch_event_rule_builder.rs b/crates/lib/database_queries/src/twitch_event_rules/insert_twitch_event_rule_builder.rs index 7ff73a4e9c..1c64aa1ed7 100644 --- a/crates/lib/database_queries/src/twitch_event_rules/insert_twitch_event_rule_builder.rs +++ b/crates/lib/database_queries/src/twitch_event_rules/insert_twitch_event_rule_builder.rs @@ -11,7 +11,7 @@ pub struct InsertTwitchEventRuleBuilder { pub event_match_predicate: String, pub event_response: String, pub user_specified_rule_order: u32, - pub rule_is_enabled: bool, + pub rule_is_disabled: bool, pub ip_address_creation: String, } @@ -33,7 +33,7 @@ SET event_match_predicate = ?, event_response = ?, user_specified_rule_order = ?, - rule_is_enabled = ?, + rule_is_disabled = ?, ip_address_creation = ?, ip_address_last_update = ? "#, @@ -44,7 +44,7 @@ SET &self.event_match_predicate, &self.event_response, &self.user_specified_rule_order, - &self.rule_is_enabled, + &self.rule_is_disabled, &self.ip_address_creation, &self.ip_address_creation, ); diff --git a/crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules_for_user.rs b/crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules_for_user.rs index dcc9a44ff7..a3c6b42a01 100644 --- a/crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules_for_user.rs +++ b/crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules_for_user.rs @@ -13,7 +13,7 @@ pub struct TwitchEventRule { pub event_match_predicate: String, pub event_response: String, pub user_specified_rule_order: u32, - pub rule_is_enabled: bool, + pub rule_is_disabled: bool, pub created_at: chrono::DateTime, pub updated_at: chrono::DateTime, } @@ -33,7 +33,7 @@ SELECT event_match_predicate, event_response, user_specified_rule_order, - rule_is_enabled, + rule_is_disabled, created_at, updated_at FROM twitch_event_rules @@ -54,7 +54,7 @@ WHERE event_match_predicate: record.event_match_predicate, event_response: record.event_response, user_specified_rule_order: record.user_specified_rule_order, - rule_is_enabled: i8_to_bool(record.rule_is_enabled), + rule_is_disabled: i8_to_bool(record.rule_is_disabled), created_at: record.created_at, updated_at: record.updated_at, } @@ -74,7 +74,7 @@ struct TwitchEventRuleInternal { pub event_match_predicate: String, pub event_response: String, pub user_specified_rule_order: u32, - pub rule_is_enabled: i8, + pub rule_is_disabled: i8, pub created_at: chrono::DateTime, pub updated_at: chrono::DateTime, } diff --git a/crates/lib/database_queries/src/twitch_event_rules/update_twitch_event_rule_builder.rs b/crates/lib/database_queries/src/twitch_event_rules/update_twitch_event_rule_builder.rs index 8d07c5d311..8a5c4b6292 100644 --- a/crates/lib/database_queries/src/twitch_event_rules/update_twitch_event_rule_builder.rs +++ b/crates/lib/database_queries/src/twitch_event_rules/update_twitch_event_rule_builder.rs @@ -10,7 +10,7 @@ pub struct UpdateTwitchEventRuleBuilder { pub event_match_predicate: String, pub event_response: String, pub user_specified_rule_order: u32, - pub rule_is_enabled: bool, + pub rule_is_disabled: bool, pub ip_address_update: String, } @@ -25,7 +25,7 @@ SET event_match_predicate = ?, event_response = ?, user_specified_rule_order = ?, - rule_is_enabled = ?, + rule_is_disabled = ?, ip_address_last_update = ? WHERE token = ? @@ -35,7 +35,7 @@ LIMIT 1 &self.event_match_predicate, &self.event_response, &self.user_specified_rule_order, - &self.rule_is_enabled, + &self.rule_is_disabled, &self.ip_address_update, &self.token, ); From 6a4e967a7074524b5185e1a83f01249e575ba89a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 19:58:47 -0500 Subject: [PATCH 0937/1042] Table for cloning voices --- .../down.sql | 5 ++ .../up.sql | 72 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 _migrations/2022-02-19-004032_voice_clone_requests/down.sql create mode 100644 _migrations/2022-02-19-004032_voice_clone_requests/up.sql diff --git a/_migrations/2022-02-19-004032_voice_clone_requests/down.sql b/_migrations/2022-02-19-004032_voice_clone_requests/down.sql new file mode 100644 index 0000000000..83299ec282 --- /dev/null +++ b/_migrations/2022-02-19-004032_voice_clone_requests/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE voice_clone_requests; diff --git a/_migrations/2022-02-19-004032_voice_clone_requests/up.sql b/_migrations/2022-02-19-004032_voice_clone_requests/up.sql new file mode 100644 index 0000000000..9db197dbd7 --- /dev/null +++ b/_migrations/2022-02-19-004032_voice_clone_requests/up.sql @@ -0,0 +1,72 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE TABLE voice_clone_requests( + -- Not used for anything except replication. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- Idempotency token from client + -- This is so the frontend client doesn't submit duplicate jobs. + uuid_idempotency_token VARCHAR(36) NOT NULL, + + -- Effective "primary key" (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- ========== CONTACT ========== + + -- Foreign key to user + -- This is a FakeYou/Storyteller account, *NOT* a twitch user id. + maybe_user_token VARCHAR(32) DEFAULT NULL, + + email_address VARCHAR(255) NOT NULL, + + discord_username VARCHAR(255) NOT NULL, + + -- ========== REASON ========== + + -- How they'll use + is_for_studio BOOLEAN NOT NULL DEFAULT FALSE, + is_for_twitch_tts BOOLEAN NOT NULL DEFAULT FALSE, + is_for_api_use BOOLEAN NOT NULL DEFAULT FALSE, + is_for_music BOOLEAN NOT NULL DEFAULT FALSE, + is_for_games BOOLEAN NOT NULL DEFAULT FALSE, + is_for_other BOOLEAN NOT NULL DEFAULT FALSE, + optional_notes_on_use MEDIUMTEXT DEFAULT NULL, + + -- Visibility + is_for_private_use BOOLEAN NOT NULL DEFAULT FALSE, + is_for_public_use BOOLEAN NOT NULL DEFAULT FALSE, + + -- ========== RECORDING QUALITY / EQUIPMENT ========== + + has_clean_audio_recordings BOOLEAN NOT NULL DEFAULT FALSE, + has_good_microphone BOOLEAN NOT NULL DEFAULT FALSE, + + -- ========== COMMENTS ========== + + optional_questions MEDIUMTEXT DEFAULT NULL, + optional_extra_comments MEDIUMTEXT DEFAULT NULL, + + -- ========== BUSINESS END / INTERNAL USE ========== + + internal_has_been_handled BOOLEAN NOT NULL DEFAULT FALSE, + + internal_notes MEDIUMTEXT DEFAULT NULL, + + -- ========== ABUSE TRACKING, ETC. ========== + + -- For abuse tracking. + -- Wide enough for IPv4/6 + ip_address_creation VARCHAR(40) NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (uuid_idempotency_token), + UNIQUE KEY (token), + KEY fk_maybe_user_token (maybe_user_token) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 71e8da44bfcf557d40033e5fd47f8bc4b9b8dcea Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 20:47:56 -0500 Subject: [PATCH 0938/1042] queries for voice clone request insert / fetch --- .../up.sql | 4 + crates/lib/database_queries/src/lib.rs | 1 + crates/lib/database_queries/src/tokens.rs | 4 + .../get_voice_clone_request.rs | 162 ++++++++++++++++++ .../insert_voice_clone_requests_builder.rs | 122 +++++++++++++ .../src/voice_clone_requests/mod.rs | 2 + 6 files changed, 295 insertions(+) create mode 100644 crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs create mode 100644 crates/lib/database_queries/src/voice_clone_requests/insert_voice_clone_requests_builder.rs create mode 100644 crates/lib/database_queries/src/voice_clone_requests/mod.rs diff --git a/_migrations/2022-02-19-004032_voice_clone_requests/up.sql b/_migrations/2022-02-19-004032_voice_clone_requests/up.sql index 9db197dbd7..a1435d7d44 100644 --- a/_migrations/2022-02-19-004032_voice_clone_requests/up.sql +++ b/_migrations/2022-02-19-004032_voice_clone_requests/up.sql @@ -38,6 +38,10 @@ CREATE TABLE voice_clone_requests( is_for_private_use BOOLEAN NOT NULL DEFAULT FALSE, is_for_public_use BOOLEAN NOT NULL DEFAULT FALSE, + -- Ownership + is_own_voice BOOLEAN NOT NULL DEFAULT FALSE, + is_third_party_voice BOOLEAN NOT NULL DEFAULT FALSE, + -- ========== RECORDING QUALITY / EQUIPMENT ========== has_clean_audio_recordings BOOLEAN NOT NULL DEFAULT FALSE, diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index 90c1d15440..a248fcebb1 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -10,3 +10,4 @@ pub mod tts; pub mod twitch_event_rules; pub mod twitch_oauth; pub mod twitch_pubsub; +pub mod voice_clone_requests; diff --git a/crates/lib/database_queries/src/tokens.rs b/crates/lib/database_queries/src/tokens.rs index 48ecf16b99..20fb8abef8 100644 --- a/crates/lib/database_queries/src/tokens.rs +++ b/crates/lib/database_queries/src/tokens.rs @@ -59,6 +59,10 @@ impl Tokens { random_prefix_crockford_token("EV:", 32, false) } + pub fn new_voice_clone_request_token() -> AnyhowResult { + random_prefix_crockford_token("VCR:", 32, false) + } + pub fn new_category() -> AnyhowResult { random_prefix_crockford_token("CAT:", 15, false) } diff --git a/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs b/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs new file mode 100644 index 0000000000..390532cc90 --- /dev/null +++ b/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs @@ -0,0 +1,162 @@ +use anyhow::anyhow; +use chrono::{Utc, DateTime}; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::twitch_event_category::TwitchEventCategory; +use crate::helpers::boolean_converters::i8_to_bool; +use sqlx::MySqlPool; + +/// Used for both *BY TOKEN* and for *BY USER* lookups. +/// The user field should be optional for one and not for the other, but code reuse is more +/// important and this is a quick / hasty feature. +#[derive(Debug)] +pub struct VoiceCloneRequest { + pub token: String, + + // Contact + pub maybe_user_token: Option, + pub email_address: String, + pub discord_username: String, + + // Use + pub is_for_studio: bool, + pub is_for_twitch_tts: bool, + pub is_for_api_use: bool, + pub is_for_music: bool, + pub is_for_games: bool, + pub is_for_other: bool, + pub optional_notes_on_use: Option, + + // Visibility + pub is_for_private_use: bool, + pub is_for_public_use: bool, + + // Ownership + pub is_own_voice: bool, + pub is_third_party_voice: bool, + + // Equipment + pub has_clean_audio_recordings: bool, + pub has_good_microphone: bool, + + // Comments + pub optional_questions: Option, + pub optional_extra_comments: Option, + + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, +} + +pub async fn get_voice_clone_request_by_token( + voice_clone_request_token: &str, + pool: &MySqlPool, +) -> AnyhowResult> { + + let maybe_record = sqlx::query_as!( + VoiceCloneRequestInternal, + r#" +SELECT + token, + + maybe_user_token, + email_address, + discord_username, + + is_for_studio, + is_for_twitch_tts, + is_for_api_use, + is_for_music, + is_for_games, + is_for_other, + optional_notes_on_use, + + is_for_private_use, + is_for_public_use, + + is_own_voice, + is_third_party_voice, + + has_clean_audio_recordings, + has_good_microphone, + + optional_questions, + optional_extra_comments, + + created_at, + updated_at +FROM voice_clone_requests +WHERE + token = ? + "#, + voice_clone_request_token, + ) + .fetch_one(pool) + .await; + + let record : VoiceCloneRequestInternal = match maybe_record { + Ok(record) => record, + Err(sqlx::Error::RowNotFound) => return Ok(None), + Err(ref err) => return Err(anyhow!("database query error: {:?}", err)), + }; + + Ok(Some(VoiceCloneRequest { + token: record.token, + maybe_user_token: record.maybe_user_token, + email_address: record.email_address, + discord_username: record.discord_username, + is_for_studio: i8_to_bool(record.is_for_studio), + is_for_twitch_tts: i8_to_bool(record.is_for_twitch_tts), + is_for_api_use: i8_to_bool(record.is_for_api_use), + is_for_music: i8_to_bool(record.is_for_music), + is_for_games: i8_to_bool(record.is_for_games), + is_for_other: i8_to_bool(record.is_for_other), + optional_notes_on_use: record.optional_notes_on_use, + is_for_private_use: i8_to_bool(record.is_for_private_use), + is_for_public_use: i8_to_bool(record.is_for_public_use), + is_own_voice: i8_to_bool(record.is_own_voice), + is_third_party_voice: i8_to_bool(record.is_third_party_voice), + has_clean_audio_recordings: i8_to_bool(record.has_clean_audio_recordings), + has_good_microphone: i8_to_bool(record.has_good_microphone), + optional_questions: record.optional_questions, + optional_extra_comments: record.optional_extra_comments, + created_at: record.created_at, + updated_at: record.updated_at, + })) +} + +#[derive(Debug)] +struct VoiceCloneRequestInternal { + pub token: String, + + // Contact + pub maybe_user_token: Option, + pub email_address: String, + pub discord_username: String, + + // Use + pub is_for_studio: i8, + pub is_for_twitch_tts: i8, + pub is_for_api_use: i8, + pub is_for_music: i8, + pub is_for_games: i8, + pub is_for_other: i8, + pub optional_notes_on_use: Option, + + // Visibility + pub is_for_private_use: i8, + pub is_for_public_use: i8, + + // Ownership + pub is_own_voice: i8, + pub is_third_party_voice: i8, + + // Equipment + pub has_clean_audio_recordings: i8, + pub has_good_microphone: i8, + + // Comments + pub optional_questions: Option, + pub optional_extra_comments: Option, + + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, +} diff --git a/crates/lib/database_queries/src/voice_clone_requests/insert_voice_clone_requests_builder.rs b/crates/lib/database_queries/src/voice_clone_requests/insert_voice_clone_requests_builder.rs new file mode 100644 index 0000000000..59c2052369 --- /dev/null +++ b/crates/lib/database_queries/src/voice_clone_requests/insert_voice_clone_requests_builder.rs @@ -0,0 +1,122 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::twitch_event_category::TwitchEventCategory; +use crate::tokens::Tokens; +use sqlx::MySqlPool; + +pub struct InsertVoiceCloneRequestBuilder { + pub uuid_idempotency_token: String, + + // Contact + pub maybe_user_token: Option, + pub email_address: String, + pub discord_username: String, + + // Use + pub is_for_studio: bool, + pub is_for_twitch_tts: bool, + pub is_for_api_use: bool, + pub is_for_music: bool, + pub is_for_games: bool, + pub is_for_other: bool, + pub optional_notes_on_use: Option, + + // Visibility + pub is_for_private_use: bool, + pub is_for_public_use: bool, + + // Ownership + pub is_own_voice: bool, + pub is_third_party_voice: bool, + + // Equipment + pub has_clean_audio_recordings: bool, + pub has_good_microphone: bool, + + // Comments + pub optional_questions: Option, + pub optional_extra_comments: Option, + + pub ip_address_creation: String, +} + +impl InsertVoiceCloneRequestBuilder { + + /// Returns the newly generated token. + pub async fn insert(&self, mysql_pool: &MySqlPool) -> AnyhowResult { + + let token = Tokens::new_voice_clone_request_token()?; + + let query = sqlx::query!( + r#" +INSERT INTO voice_clone_requests +SET + token = ?, + uuid_idempotency_token = ?, + + maybe_user_token = ?, + email_address = ?, + discord_username = ?, + + is_for_studio = ?, + is_for_twitch_tts = ?, + is_for_api_use = ?, + is_for_music = ?, + is_for_games = ?, + is_for_other = ?, + optional_notes_on_use = ?, + + is_for_private_use = ?, + is_for_public_use = ?, + + is_own_voice = ?, + is_third_party_voice = ?, + + has_clean_audio_recordings = ?, + has_good_microphone = ?, + + optional_questions = ?, + optional_extra_comments = ?, + + ip_address_creation = ? + "#, + &token, + &self.uuid_idempotency_token, + + &self.maybe_user_token, + &self.email_address, + &self.discord_username, + + &self.is_for_studio, + &self.is_for_twitch_tts, + &self.is_for_api_use, + &self.is_for_music, + &self.is_for_games, + &self.is_for_other, + &self.optional_notes_on_use, + + &self.is_for_private_use, + &self.is_for_public_use, + + &self.is_own_voice, + &self.is_third_party_voice, + + &self.has_clean_audio_recordings, + &self.has_good_microphone, + + &self.optional_questions, + &self.optional_extra_comments, + + &self.ip_address_creation, + ); + + let query_result = query.execute(mysql_pool).await; + + let _record_id = match query_result { + Ok(res) => res.last_insert_id(), + Err(err) => return Err(anyhow!("Error creating Voice Clone Request: {:?}", err)), + }; + + Ok(token) + } +} diff --git a/crates/lib/database_queries/src/voice_clone_requests/mod.rs b/crates/lib/database_queries/src/voice_clone_requests/mod.rs new file mode 100644 index 0000000000..cd31493d36 --- /dev/null +++ b/crates/lib/database_queries/src/voice_clone_requests/mod.rs @@ -0,0 +1,2 @@ +pub mod get_voice_clone_request; +pub mod insert_voice_clone_requests_builder; \ No newline at end of file From 62eff2ec8200f3e7eb61a042b6873222d5325360 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 20:51:40 -0500 Subject: [PATCH 0939/1042] query by user token --- .../get_voice_clone_request.rs | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs b/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs index 390532cc90..98c5379d90 100644 --- a/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs +++ b/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs @@ -50,7 +50,6 @@ pub async fn get_voice_clone_request_by_token( voice_clone_request_token: &str, pool: &MySqlPool, ) -> AnyhowResult> { - let maybe_record = sqlx::query_as!( VoiceCloneRequestInternal, r#" @@ -92,6 +91,61 @@ WHERE .fetch_one(pool) .await; + handle_result(maybe_record) +} + + +pub async fn get_voice_clone_request_by_user_token( + user_token: &str, + pool: &MySqlPool, +) -> AnyhowResult> { + let maybe_record = sqlx::query_as!( + VoiceCloneRequestInternal, + r#" +SELECT + token, + + maybe_user_token, + email_address, + discord_username, + + is_for_studio, + is_for_twitch_tts, + is_for_api_use, + is_for_music, + is_for_games, + is_for_other, + optional_notes_on_use, + + is_for_private_use, + is_for_public_use, + + is_own_voice, + is_third_party_voice, + + has_clean_audio_recordings, + has_good_microphone, + + optional_questions, + optional_extra_comments, + + created_at, + updated_at +FROM voice_clone_requests +WHERE + maybe_user_token = ? + "#, + user_token, + ) + .fetch_one(pool) + .await; + + handle_result(maybe_record) +} + +fn handle_result( + maybe_record: Result, +) -> AnyhowResult> { let record : VoiceCloneRequestInternal = match maybe_record { Ok(record) => record, Err(sqlx::Error::RowNotFound) => return Ok(None), From 0d3faf230d235e3eb0079d6870a701156d2228fa Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 20:54:11 -0500 Subject: [PATCH 0940/1042] make two queries a little more obvious --- .../get_voice_clone_request.rs | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs b/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs index 98c5379d90..20a3fee668 100644 --- a/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs +++ b/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs @@ -5,6 +5,22 @@ use crate::column_types::twitch_event_category::TwitchEventCategory; use crate::helpers::boolean_converters::i8_to_bool; use sqlx::MySqlPool; +/// Lookup by voice clone request token +pub async fn get_voice_clone_request_by_token( + voice_clone_request_token: &str, + pool: &MySqlPool, +) -> AnyhowResult> { + internal_voice_clone_request_by_token(voice_clone_request_token, pool).await +} + +/// Lookup by user +pub async fn get_voice_clone_request_by_user_token( + user_token: &str, + pool: &MySqlPool, +) -> AnyhowResult> { + internal_voice_clone_request_by_user_token(user_token, pool).await +} + /// Used for both *BY TOKEN* and for *BY USER* lookups. /// The user field should be optional for one and not for the other, but code reuse is more /// important and this is a quick / hasty feature. @@ -46,7 +62,7 @@ pub struct VoiceCloneRequest { pub updated_at: chrono::DateTime, } -pub async fn get_voice_clone_request_by_token( +async fn internal_voice_clone_request_by_token( voice_clone_request_token: &str, pool: &MySqlPool, ) -> AnyhowResult> { @@ -94,8 +110,7 @@ WHERE handle_result(maybe_record) } - -pub async fn get_voice_clone_request_by_user_token( +async fn internal_voice_clone_request_by_user_token( user_token: &str, pool: &MySqlPool, ) -> AnyhowResult> { From 30b484a92dc448ad9417e1cae23b7f39d470bf64 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Feb 2022 21:53:25 -0500 Subject: [PATCH 0941/1042] cleanup of in-app i8_to_bool, some derive_more, etc. --- .../database/helpers/boolean_converters.rs | 62 ------------------- .../src/database/helpers/mod.rs | 1 - .../categories/get_category_by_token.rs | 32 +++++----- .../queries/get_user_profile_by_username.rs | 2 +- .../src/database/queries/list_user_roles.rs | 10 +-- .../database/queries/list_w2l_templates.rs | 22 +++---- .../src/database/queries/query_tts_result.rs | 42 ++++++------- .../src/database/queries/query_w2l_result.rs | 44 +++++++------ .../database/queries/query_w2l_template.rs | 37 ++++++----- ...t_assigned_tts_categories_query_builder.rs | 28 ++++----- .../list_categories_query_builder.rs | 26 ++++---- .../list_users_query_builder.rs | 24 +++---- .../endpoints/users/get_profile.rs | 45 ++++++++------ .../src/http_server/endpoints/users/login.rs | 5 +- .../endpoints/w2l/get_w2l_template.rs | 14 +++-- .../http_server/web_utils/session_checker.rs | 14 ++--- 16 files changed, 176 insertions(+), 232 deletions(-) delete mode 100644 crates/service/storyteller_web/src/database/helpers/boolean_converters.rs diff --git a/crates/service/storyteller_web/src/database/helpers/boolean_converters.rs b/crates/service/storyteller_web/src/database/helpers/boolean_converters.rs deleted file mode 100644 index 33e6898f0c..0000000000 --- a/crates/service/storyteller_web/src/database/helpers/boolean_converters.rs +++ /dev/null @@ -1,62 +0,0 @@ - -/// MySQL stores non-nullable booleans as i8. -pub fn i8_to_bool(value: i8) -> bool { - if value == 0 { - false - } else { - true - } -} - -/// Bool conversion, but turn nulls to a default value. -pub fn nullable_i8_to_bool(value: Option, default_value: bool) -> bool { - value.map(|v| i8_to_bool(v)) - .unwrap_or(default_value) -} - -/// Bool conversion, but retain nulls. -pub fn nullable_i8_to_optional_bool(value: Option) -> Option { - value.map(|v| i8_to_bool(v)) -} - -#[cfg(test)] -mod tests { - use crate::database::helpers::boolean_converters::{ - i8_to_bool, - nullable_i8_to_bool, - nullable_i8_to_optional_bool - }; - - #[test] - fn test_i8_to_bool() { - assert_eq!(i8_to_bool(0), false); - assert_eq!(i8_to_bool(1), true); - assert_eq!(i8_to_bool(-1), true); - assert_eq!(i8_to_bool(120), true); - } - - #[test] - fn test_nullable_i8_to_bool() { - assert_eq!(nullable_i8_to_bool(None, false), false); - assert_eq!(nullable_i8_to_bool(None, true), true); - assert_eq!(nullable_i8_to_bool(Some(0), false), false); - assert_eq!(nullable_i8_to_bool(Some(0), true), false); - - assert_eq!(nullable_i8_to_bool(Some(1), false), true); - assert_eq!(nullable_i8_to_bool(Some(1), true), true); - - assert_eq!(nullable_i8_to_bool(Some(100), false), true); - assert_eq!(nullable_i8_to_bool(Some(100), true), true); - assert_eq!(nullable_i8_to_bool(Some(-100), false), true); - assert_eq!(nullable_i8_to_bool(Some(-100), true), true); - } - - #[test] - fn test_nullable_i8_to_optional_bool() { - assert_eq!(nullable_i8_to_optional_bool(None), None); - assert_eq!(nullable_i8_to_optional_bool(Some(0)), Some(false)); - assert_eq!(nullable_i8_to_optional_bool(Some(1)), Some(true)); - assert_eq!(nullable_i8_to_optional_bool(Some(100)), Some(true)); - assert_eq!(nullable_i8_to_optional_bool(Some(-100)), Some(true)); - } -} diff --git a/crates/service/storyteller_web/src/database/helpers/mod.rs b/crates/service/storyteller_web/src/database/helpers/mod.rs index b5f78fd29f..35b3598e87 100644 --- a/crates/service/storyteller_web/src/database/helpers/mod.rs +++ b/crates/service/storyteller_web/src/database/helpers/mod.rs @@ -1,2 +1 @@ -pub mod boolean_converters; pub mod enums; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/categories/get_category_by_token.rs b/crates/service/storyteller_web/src/database/queries/categories/get_category_by_token.rs index 80c8a2bf3a..fa7aeaa9a4 100644 --- a/crates/service/storyteller_web/src/database/queries/categories/get_category_by_token.rs +++ b/crates/service/storyteller_web/src/database/queries/categories/get_category_by_token.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; +use database_queries::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; use log::{info, warn, log}; use sqlx::MySqlPool; @@ -96,26 +96,26 @@ WHERE }; let category = ModelCategory { - category_token: category.category_token.clone(), - model_type: category.model_type.clone(), - maybe_super_category_token: category.maybe_super_category_token.clone(), + category_token: category.category_token, + model_type: category.model_type, + maybe_super_category_token: category.maybe_super_category_token, can_directly_have_models: i8_to_bool(category.can_directly_have_models), can_have_subcategories: i8_to_bool(category.can_have_subcategories), can_only_mods_apply: i8_to_bool(category.can_only_mods_apply), name: category.name.clone(), - maybe_dropdown_name: category.maybe_dropdown_name.clone(), - creator_user_token: category.creator_user_token.clone(), - creator_username: category.creator_username.clone(), - creator_display_name: category.creator_display_name.clone(), - creator_gravatar_hash: category.creator_gravatar_hash.clone(), - creator_ip_address_creation: category.creator_ip_address_creation.clone(), - creator_ip_address_last_update: category.creator_ip_address_last_update.clone(), + maybe_dropdown_name: category.maybe_dropdown_name, + creator_user_token: category.creator_user_token, + creator_username: category.creator_username, + creator_display_name: category.creator_display_name, + creator_gravatar_hash: category.creator_gravatar_hash, + creator_ip_address_creation: category.creator_ip_address_creation, + creator_ip_address_last_update: category.creator_ip_address_last_update, is_mod_approved: nullable_i8_to_optional_bool(category.is_mod_approved), - maybe_mod_comments: category.maybe_mod_comments.clone(), - maybe_mod_user_token: category.maybe_mod_user_token.clone(), - created_at: category.created_at.clone(), - updated_at: category.updated_at.clone(), - deleted_at: category.deleted_at.clone(), + maybe_mod_comments: category.maybe_mod_comments, + maybe_mod_user_token: category.maybe_mod_user_token, + created_at: category.created_at, + updated_at: category.updated_at, + deleted_at: category.deleted_at, }; Ok(Some(category)) diff --git a/crates/service/storyteller_web/src/database/queries/get_user_profile_by_username.rs b/crates/service/storyteller_web/src/database/queries/get_user_profile_by_username.rs index 939192e0a6..8fc2fd7cdd 100644 --- a/crates/service/storyteller_web/src/database/queries/get_user_profile_by_username.rs +++ b/crates/service/storyteller_web/src/database/queries/get_user_profile_by_username.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::helpers::boolean_converters::i8_to_bool; +use database_queries::helpers::boolean_converters::i8_to_bool; use derive_more::{Display, Error}; use log::{info, warn, log}; use md5::{Md5, Digest}; diff --git a/crates/service/storyteller_web/src/database/queries/list_user_roles.rs b/crates/service/storyteller_web/src/database/queries/list_user_roles.rs index d909889798..4528c3809c 100644 --- a/crates/service/storyteller_web/src/database/queries/list_user_roles.rs +++ b/crates/service/storyteller_web/src/database/queries/list_user_roles.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; -use crate::database::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; +use database_queries::helpers::boolean_converters::i8_to_bool; use log::{warn, info}; use sqlx::MySqlPool; @@ -125,8 +125,8 @@ FROM user_roles Ok(user_roles.into_iter() .map(|ur| { UserRoleForList { - slug: ur.slug.clone(), - name: ur.name.clone(), + slug: ur.slug, + name: ur.name, can_use_tts: i8_to_bool(ur.can_use_tts), can_use_w2l: i8_to_bool(ur.can_use_w2l), can_delete_own_tts_results: i8_to_bool(ur.can_delete_own_tts_results), @@ -146,8 +146,8 @@ FROM user_roles can_delete_other_users_w2l_results: i8_to_bool(ur.can_delete_other_users_w2l_results), can_ban_users: i8_to_bool(ur.can_ban_users), can_delete_users: i8_to_bool(ur.can_delete_users), - created_at: ur.created_at.clone(), - updated_at: ur.updated_at.clone(), + created_at: ur.created_at, + updated_at: ur.updated_at, } }) .collect::>()) diff --git a/crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs b/crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs index 2f32f2160c..cd0dd17349 100644 --- a/crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs +++ b/crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs @@ -1,9 +1,9 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; -use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use log::{warn, info}; use sqlx::MySqlPool; +use database_queries::helpers::boolean_converters::nullable_i8_to_optional_bool; #[derive(Serialize)] pub struct W2lTemplateRecordForList { @@ -80,20 +80,20 @@ pub async fn list_w2l_templates( Ok(templates.into_iter() .map(|template| { W2lTemplateRecordForList { - template_token: template.template_token.clone(), - template_type: template.template_type.clone(), - creator_user_token: template.creator_user_token.clone(), - creator_username: template.creator_username.clone(), - creator_display_name: template.creator_display_name.clone(), - title: template.title.clone(), + template_token: template.template_token, + template_type: template.template_type, + creator_user_token: template.creator_user_token, + creator_username: template.creator_username, + creator_display_name: template.creator_display_name, + title: template.title, frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, - maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), - maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), + maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name, + maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name, is_public_listing_approved: nullable_i8_to_optional_bool(template.is_public_listing_approved), - created_at: template.created_at.clone(), - updated_at: template.updated_at.clone(), + created_at: template.created_at, + updated_at: template.updated_at, } }) .collect::>()) diff --git a/crates/service/storyteller_web/src/database/queries/query_tts_result.rs b/crates/service/storyteller_web/src/database/queries/query_tts_result.rs index 6b93c0a30b..2dcc5f50d6 100644 --- a/crates/service/storyteller_web/src/database/queries/query_tts_result.rs +++ b/crates/service/storyteller_web/src/database/queries/query_tts_result.rs @@ -3,9 +3,7 @@ use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; use crate::database::enums::vocoder_type::VocoderType; -use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_bool}; -use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; -use derive_more::{Display, Error}; +use database_queries::helpers::boolean_converters::nullable_i8_to_bool; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; @@ -138,29 +136,29 @@ pub async fn select_tts_result_by_token( } let ir_for_response = TtsResultRecordForResponse { - tts_result_token: ir.tts_result_token.clone(), + tts_result_token: ir.tts_result_token, - raw_inference_text: ir.raw_inference_text.clone(), + raw_inference_text: ir.raw_inference_text, - tts_model_token: ir.tts_model_token.clone(), - tts_model_title: ir.tts_model_title.clone(), + tts_model_token: ir.tts_model_token, + tts_model_title: ir.tts_model_title, maybe_pretrained_vocoder_used: pretrained_vocoder, - maybe_creator_user_token: ir.maybe_creator_user_token.clone(), - maybe_creator_username: ir.maybe_creator_username.clone(), - maybe_creator_display_name: ir.maybe_creator_display_name.clone(), - maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash.clone(), + maybe_creator_user_token: ir.maybe_creator_user_token, + maybe_creator_username: ir.maybe_creator_username, + maybe_creator_display_name: ir.maybe_creator_display_name, + maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash, - maybe_model_creator_user_token: ir.maybe_model_creator_user_token.clone(), - maybe_model_creator_username: ir.maybe_model_creator_username.clone(), - maybe_model_creator_display_name: ir.maybe_model_creator_display_name.clone(), - maybe_model_creator_gravatar_hash: ir.maybe_model_creator_gravatar_hash.clone(), + maybe_model_creator_user_token: ir.maybe_model_creator_user_token, + maybe_model_creator_username: ir.maybe_model_creator_username, + maybe_model_creator_display_name: ir.maybe_model_creator_display_name, + maybe_model_creator_gravatar_hash: ir.maybe_model_creator_gravatar_hash, //model_is_mod_approved: if ir.model_is_mod_approved == 0 { false } else { true }, - public_bucket_wav_audio_path: ir.public_bucket_wav_audio_path.clone(), - public_bucket_spectrogram_path: ir.public_bucket_spectrogram_path.clone(), + public_bucket_wav_audio_path: ir.public_bucket_wav_audio_path, + public_bucket_spectrogram_path: ir.public_bucket_spectrogram_path, // NB: Fail open/public since we're already looking at it creator_set_visibility: RecordVisibility::from_str(&ir.creator_set_visibility) @@ -169,17 +167,17 @@ pub async fn select_tts_result_by_token( file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, - created_at: ir.created_at.clone(), - updated_at: ir.updated_at.clone(), + created_at: ir.created_at, + updated_at: ir.updated_at, maybe_moderator_fields: Some(TtsResultModeratorFields { model_creator_is_banned: nullable_i8_to_bool(ir.maybe_model_creator_is_banned, false), result_creator_is_banned_if_user: nullable_i8_to_bool(ir.maybe_creator_is_banned, false), - result_creator_ip_address: ir.creator_ip_address.clone(), - result_creator_deleted_at: ir.user_deleted_at.clone(), - mod_deleted_at: ir.mod_deleted_at.clone(), + result_creator_ip_address: ir.creator_ip_address, + result_creator_deleted_at: ir.user_deleted_at, + mod_deleted_at: ir.mod_deleted_at, }), }; diff --git a/crates/service/storyteller_web/src/database/queries/query_w2l_result.rs b/crates/service/storyteller_web/src/database/queries/query_w2l_result.rs index ce38cb86cb..3b2e73379d 100644 --- a/crates/service/storyteller_web/src/database/queries/query_w2l_result.rs +++ b/crates/service/storyteller_web/src/database/queries/query_w2l_result.rs @@ -1,9 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_bool}; -use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; -use derive_more::{Display, Error}; +use crate::database::enums::record_visibility::RecordVisibility; +use database_queries::helpers::boolean_converters::nullable_i8_to_bool; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; @@ -11,7 +10,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::database::enums::record_visibility::RecordVisibility; #[derive(Serialize)] pub struct W2lResultRecordForResponse { @@ -132,24 +130,24 @@ pub async fn select_w2l_result_by_token( }; let ir_for_response = W2lResultRecordForResponse { - w2l_result_token: ir.w2l_result_token.clone(), - maybe_w2l_template_token: ir.maybe_w2l_template_token.clone(), - maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), + w2l_result_token: ir.w2l_result_token, + maybe_w2l_template_token: ir.maybe_w2l_template_token, + maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token, - public_bucket_video_path: ir.public_bucket_video_path.clone(), + public_bucket_video_path: ir.public_bucket_video_path, - template_type: ir.template_type.clone(), - template_title: ir.template_title.clone(), + template_type: ir.template_type, + template_title: ir.template_title, - maybe_creator_user_token: ir.maybe_creator_user_token.clone(), - maybe_creator_username: ir.maybe_creator_username.clone(), - maybe_creator_display_name: ir.maybe_creator_display_name.clone(), - maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash.clone(), + maybe_creator_user_token: ir.maybe_creator_user_token, + maybe_creator_username: ir.maybe_creator_username, + maybe_creator_display_name: ir.maybe_creator_display_name, + maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash, - maybe_template_creator_user_token: ir.maybe_template_creator_user_token.clone(), - maybe_template_creator_username: ir.maybe_template_creator_username.clone(), - maybe_template_creator_display_name: ir.maybe_template_creator_display_name.clone(), - maybe_template_creator_gravatar_hash: ir.maybe_template_creator_gravatar_hash.clone(), + maybe_template_creator_user_token: ir.maybe_template_creator_user_token, + maybe_template_creator_username: ir.maybe_template_creator_username, + maybe_template_creator_display_name: ir.maybe_template_creator_display_name, + maybe_template_creator_gravatar_hash: ir.maybe_template_creator_gravatar_hash, // NB: Fail open/public since we're already looking at it creator_set_visibility: RecordVisibility::from_str(&ir.creator_set_visibility) @@ -162,17 +160,17 @@ pub async fn select_w2l_result_by_token( frame_height: if ir.frame_height > 0 { ir.frame_height as u32 } else { 0 }, duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, - created_at: ir.created_at.clone(), - updated_at: ir.updated_at.clone(), + created_at: ir.created_at, + updated_at: ir.updated_at, maybe_moderator_fields: Some(W2lResultModeratorFields { template_creator_is_banned: nullable_i8_to_bool(ir.maybe_template_creator_is_banned, false), result_creator_is_banned_if_user: nullable_i8_to_bool(ir.maybe_creator_is_banned, false), - result_creator_ip_address: ir.creator_ip_address.clone(), - result_creator_deleted_at: ir.user_deleted_at.clone(), - mod_deleted_at: ir.mod_deleted_at.clone(), + result_creator_ip_address: ir.creator_ip_address, + result_creator_deleted_at: ir.user_deleted_at, + mod_deleted_at: ir.mod_deleted_at, }), }; diff --git a/crates/service/storyteller_web/src/database/queries/query_w2l_template.rs b/crates/service/storyteller_web/src/database/queries/query_w2l_template.rs index 29ce15a925..96d3ca9495 100644 --- a/crates/service/storyteller_web/src/database/queries/query_w2l_template.rs +++ b/crates/service/storyteller_web/src/database/queries/query_w2l_template.rs @@ -2,8 +2,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::helpers::boolean_converters::i8_to_bool; -use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; +use database_queries::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; @@ -107,34 +106,34 @@ pub async fn select_w2l_template_by_token( }; let template_for_response = W2lTemplateRecordForResponse { - template_token: template.template_token.clone(), - template_type: template.template_type.clone(), - creator_user_token: template.creator_user_token.clone(), - creator_username: template.creator_username.clone(), - creator_display_name: template.creator_display_name.clone(), - creator_gravatar_hash: template.creator_gravatar_hash.clone(), - title: template.title.clone(), - description_markdown: template.description_markdown.clone(), - description_rendered_html: template.description_rendered_html.clone(), + template_token: template.template_token, + template_type: template.template_type, + creator_user_token: template.creator_user_token, + creator_username: template.creator_username, + creator_display_name: template.creator_display_name, + creator_gravatar_hash: template.creator_gravatar_hash, + title: template.title, + description_markdown: template.description_markdown, + description_rendered_html: template.description_rendered_html, frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, - maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), - maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), + maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name, + maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name, // NB: Fail open/public with creator_set_visibility since we're already looking at it creator_set_visibility: RecordVisibility::from_str(&template.creator_set_visibility) .unwrap_or(RecordVisibility::Public), is_public_listing_approved: nullable_i8_to_optional_bool(template.is_public_listing_approved), is_locked_from_use: i8_to_bool(template.is_locked_from_use), is_locked_from_user_modification: i8_to_bool(template.is_locked_from_user_modification), - created_at: template.created_at.clone(), - updated_at: template.updated_at.clone(), + created_at: template.created_at, + updated_at: template.updated_at, maybe_moderator_fields: Some(W2lTemplateModeratorFields { creator_is_banned: i8_to_bool(template.creator_is_banned), - creator_ip_address_creation: template.creator_ip_address_creation.clone(), - creator_ip_address_last_update: template.creator_ip_address_last_update.clone(), - user_deleted_at: template.user_deleted_at.clone(), - mod_deleted_at: template.mod_deleted_at.clone(), + creator_ip_address_creation: template.creator_ip_address_creation, + creator_ip_address_last_update: template.creator_ip_address_last_update, + user_deleted_at: template.user_deleted_at, + mod_deleted_at: template.mod_deleted_at, }), }; diff --git a/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs index 64279387ad..18c70ce50e 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs @@ -2,7 +2,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; +use database_queries::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; use log::{warn, info}; use sqlx::MySqlPool; @@ -103,29 +103,29 @@ impl ListAssignedTtsCategoriesQueryBuilder { let internal_results = self.perform_internal_query(mysql_pool).await?; let categories = internal_results - .iter() + .into_iter() .map(|c| { AssignedCategory { - category_token: c.category_token.clone(), - model_type: c.model_type.clone(), - maybe_super_category_token: c.maybe_super_category_token.clone(), + category_token: c.category_token, + model_type: c.model_type, + maybe_super_category_token: c.maybe_super_category_token, can_directly_have_models: i8_to_bool(c.can_directly_have_models), can_have_subcategories: i8_to_bool(c.can_have_subcategories), can_only_mods_apply: i8_to_bool(c.can_only_mods_apply), - name: c.name.clone(), - maybe_dropdown_name: c.maybe_dropdown_name.clone(), - category_creator_user_token: c.category_creator_user_token.clone(), - category_creator_username: c.category_creator_username.clone(), - category_creator_display_name: c.category_creator_display_name.clone(), - category_creator_gravatar_hash: c.category_creator_gravatar_hash.clone(), + name: c.name, + maybe_dropdown_name: c.maybe_dropdown_name, + category_creator_user_token: c.category_creator_user_token, + category_creator_username: c.category_creator_username, + category_creator_display_name: c.category_creator_display_name, + category_creator_gravatar_hash: c.category_creator_gravatar_hash, is_mod_approved: nullable_i8_to_optional_bool(c.is_mod_approved), - maybe_mod_comments: c.maybe_mod_comments.clone(), + maybe_mod_comments: c.maybe_mod_comments, category_created_at: c.category_created_at, category_updated_at: c.category_updated_at, - category_deleted_at: c.category_deleted_at.clone(), + category_deleted_at: c.category_deleted_at, category_assignment_created_at: c.category_assignment_created_at, category_assignment_updated_at: c.category_assignment_updated_at, - category_assignment_deleted_at: c.category_assignment_deleted_at.clone(), + category_assignment_deleted_at: c.category_assignment_deleted_at, } }) .collect::>(); diff --git a/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs index e8f64a5158..a72fb912d0 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs @@ -2,7 +2,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; +use database_queries::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; use log::{warn, info}; use sqlx::MySqlPool; @@ -127,26 +127,26 @@ impl ListCategoriesQueryBuilder { let internal_results = self.perform_internal_query(mysql_pool).await?; let categories = internal_results - .iter() + .into_iter() .map(|c| { Category { - category_token: c.category_token.clone(), - model_type: c.model_type.clone(), - maybe_super_category_token: c.maybe_super_category_token.clone(), + category_token: c.category_token, + model_type: c.model_type, + maybe_super_category_token: c.maybe_super_category_token, can_directly_have_models: i8_to_bool(c.can_directly_have_models), can_have_subcategories: i8_to_bool(c.can_have_subcategories), can_only_mods_apply: i8_to_bool(c.can_only_mods_apply), - name: c.name.clone(), - maybe_dropdown_name: c.maybe_dropdown_name.clone(), - creator_user_token: c.creator_user_token.clone(), - creator_username: c.creator_username.clone(), - creator_display_name: c.creator_display_name.clone(), - creator_gravatar_hash: c.creator_gravatar_hash.clone(), + name: c.name, + maybe_dropdown_name: c.maybe_dropdown_name, + creator_user_token: c.creator_user_token, + creator_username: c.creator_username, + creator_display_name: c.creator_display_name, + creator_gravatar_hash: c.creator_gravatar_hash, is_mod_approved: nullable_i8_to_optional_bool(c.is_mod_approved), - maybe_mod_comments: c.maybe_mod_comments.clone(), + maybe_mod_comments: c.maybe_mod_comments, created_at: c.created_at, updated_at: c.updated_at, - deleted_at: c.deleted_at.clone(), + deleted_at: c.deleted_at, } }) .collect::>(); diff --git a/crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs index 83752a7af6..4e55c62536 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs @@ -1,6 +1,6 @@ use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; -use crate::database::helpers::boolean_converters::i8_to_bool; +use database_queries::helpers::boolean_converters::i8_to_bool; use sqlx::MySqlPool; /// NB: This is not to be shared externally. @@ -70,21 +70,21 @@ impl ListUsersQueryBuilder { .map(|raw_result| raw_result.user_id); let users = internal_results - .iter() + .into_iter() .map(|r| { UserForList { user_id: r.user_id, - user_token: r.user_token.clone(), - username: r.username.clone(), - display_name: r.display_name.clone(), - gravatar_hash: r.gravatar_hash.clone(), + user_token: r.user_token, + username: r.username, + display_name: r.display_name, + gravatar_hash: r.gravatar_hash, is_banned: i8_to_bool(r.is_banned), - user_role_slug: r.user_role_slug.clone(), - ip_address_creation: r.ip_address_creation.clone(), - ip_address_last_login: r.ip_address_last_login.clone(), - ip_address_last_update: r.ip_address_last_update.clone(), - created_at: r.created_at.clone(), - updated_at: r.updated_at.clone(), + user_role_slug: r.user_role_slug, + ip_address_creation: r.ip_address_creation, + ip_address_last_login: r.ip_address_last_login, + ip_address_last_update: r.ip_address_last_update, + created_at: r.created_at, + updated_at: r.updated_at, } }) .collect::>(); diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs index 5556e2d745..45d41a5341 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; @@ -10,12 +10,11 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::helpers::boolean_converters::i8_to_bool; use crate::database::queries::list_user_badges::UserBadgeForList; use crate::database::queries::list_user_badges::list_user_badges; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; +use database_queries::helpers::boolean_converters::i8_to_bool; use log::{info, warn, log}; use md5::{Md5, Digest}; use regex::Regex; @@ -23,6 +22,7 @@ use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; // TODO: This is duplicated in query_user_profile @@ -89,7 +89,7 @@ pub struct RawUserProfileRecord { pub created_at: DateTime, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum ProfileError { ServerError, NotFound, @@ -119,6 +119,13 @@ impl ResponseError for ProfileError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ProfileError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn get_profile_handler( http_request: HttpRequest, path: Path, @@ -218,24 +225,24 @@ WHERE }; let profile_for_response = UserProfileRecordForResponse { - user_token: profile_record.user_token.clone(), - username: profile_record.username.clone(), - display_name: profile_record.display_name.clone(), - email_gravatar_hash: profile_record.email_gravatar_hash.clone(), - profile_markdown: profile_record.profile_markdown.clone(), - profile_rendered_html: profile_record.profile_rendered_html.clone(), - user_role_slug: profile_record.user_role_slug.clone(), + user_token: profile_record.user_token, + username: profile_record.username, + display_name: profile_record.display_name, + email_gravatar_hash: profile_record.email_gravatar_hash, + profile_markdown: profile_record.profile_markdown, + profile_rendered_html: profile_record.profile_rendered_html, + user_role_slug: profile_record.user_role_slug, disable_gravatar: i8_to_bool(profile_record.disable_gravatar), preferred_tts_result_visibility: profile_record.preferred_tts_result_visibility, preferred_w2l_result_visibility: profile_record.preferred_w2l_result_visibility, - discord_username: profile_record.discord_username.clone(), - twitch_username: profile_record.twitch_username.clone(), - twitter_username: profile_record.twitter_username.clone(), - patreon_username: profile_record.patreon_username.clone(), - github_username: profile_record.github_username.clone(), - cashapp_username: profile_record.cashapp_username.clone(), - website_url: profile_record.website_url.clone(), - created_at: profile_record.created_at.clone(), + discord_username: profile_record.discord_username, + twitch_username: profile_record.twitch_username, + twitter_username: profile_record.twitter_username, + patreon_username: profile_record.patreon_username, + github_username: profile_record.github_username, + cashapp_username: profile_record.cashapp_username, + website_url: profile_record.website_url, + created_at: profile_record.created_at, maybe_moderator_fields: maybe_mod_fields, badges, }; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs index a348294866..6a7dad39c0 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; @@ -11,7 +11,7 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; +use database_queries::helpers::boolean_converters::i8_to_bool; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; @@ -21,7 +21,6 @@ use sqlx::mysql::MySqlDatabaseError; use std::fmt::Formatter; use std::fmt; use std::sync::Arc; -use crate::database::helpers::boolean_converters::i8_to_bool; #[derive(Deserialize)] pub struct LoginRequest { diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs index 49a82e4475..29e416d29f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -1,14 +1,13 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::helpers::boolean_converters::nullable_i8_to_optional_bool; use crate::database::queries::query_w2l_template::W2lTemplateRecordForResponse; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; @@ -16,13 +15,13 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -37,7 +36,7 @@ pub struct GetW2lTemplateSuccessResponse { pub template: W2lTemplateRecordForResponse, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum GetW2lTemplateError { ServerError, NotFound, @@ -61,6 +60,13 @@ impl ResponseError for GetW2lTemplateError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GetW2lTemplateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn get_w2l_template_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/web_utils/session_checker.rs b/crates/service/storyteller_web/src/http_server/web_utils/session_checker.rs index a7691877f4..e922c1f629 100644 --- a/crates/service/storyteller_web/src/http_server/web_utils/session_checker.rs +++ b/crates/service/storyteller_web/src/http_server/web_utils/session_checker.rs @@ -2,8 +2,8 @@ use actix_web::HttpRequest; use anyhow::anyhow; use crate::AnyhowResult; use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; use crate::http_server::web_utils::cookie_manager::CookieManager; +use database_queries::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; use log::{info, warn}; use sqlx::MySqlPool; use sqlx::error::Error::RowNotFound; @@ -235,17 +235,17 @@ WHERE user_sessions.token = ? match maybe_user_record { Ok(raw_user_record) => { let result_user_record = SessionUserRecord { - user_token: raw_user_record.user_token.clone(), - username: raw_user_record.username.clone(), - display_name: raw_user_record.display_name.clone(), - email_address: raw_user_record.email_address.clone(), + user_token: raw_user_record.user_token, + username: raw_user_record.username, + display_name: raw_user_record.display_name, + email_address: raw_user_record.email_address, email_confirmed: i8_to_bool(raw_user_record.email_confirmed), - email_gravatar_hash: raw_user_record.email_gravatar_hash.clone(), + email_gravatar_hash: raw_user_record.email_gravatar_hash, // Preference disable_gravatar: i8_to_bool(raw_user_record.disable_gravatar), auto_play_audio_preference: nullable_i8_to_optional_bool(raw_user_record.auto_play_audio_preference), auto_play_video_preference: nullable_i8_to_optional_bool(raw_user_record.auto_play_video_preference), - user_role_slug: raw_user_record.user_role_slug.clone(), + user_role_slug: raw_user_record.user_role_slug, preferred_tts_result_visibility: raw_user_record.preferred_tts_result_visibility, preferred_w2l_result_visibility: raw_user_record.preferred_w2l_result_visibility, From e2c645a17ca5fbd8e3d4883e84277d234451c230 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 00:31:40 -0500 Subject: [PATCH 0942/1042] Refactor: move rest of TTS queries into tts library crate --- crates/lib/database_queries/src/tts/mod.rs | 1 + .../stats}/calculate_tts_model_leaderboard.rs | 0 .../lib/database_queries/src/tts/stats/mod.rs | 1 + .../delete_tts_model_various_scopes.rs} | 0 .../src/tts/tts_models/mod.rs | 1 + .../delete_tts_result_various_scopes.rs} | 0 .../src/tts/tts_results/mod.rs | 4 +++- .../src/tts/tts_results}/query_tts_result.rs | 9 ++++---- .../src/database/queries/mod.rs | 4 ---- .../endpoints/leaderboard/get_leaderboard.rs | 10 ++++---- .../endpoints/tts/delete_tts_model.rs | 9 ++++---- .../endpoints/tts/delete_tts_result.rs | 23 ++++++++++++------- .../endpoints/tts/edit_tts_result.rs | 4 ++-- .../endpoints/tts/get_tts_result.rs | 17 ++++++++++---- .../w2l/get_w2l_inference_job_status.rs | 13 ++++++++--- 15 files changed, 57 insertions(+), 39 deletions(-) rename crates/{service/storyteller_web/src/database/queries => lib/database_queries/src/tts/stats}/calculate_tts_model_leaderboard.rs (100%) create mode 100644 crates/lib/database_queries/src/tts/stats/mod.rs rename crates/{service/storyteller_web/src/database/queries/delete_tts_model.rs => lib/database_queries/src/tts/tts_models/delete_tts_model_various_scopes.rs} (100%) rename crates/{service/storyteller_web/src/database/queries/delete_tts_result.rs => lib/database_queries/src/tts/tts_results/delete_tts_result_various_scopes.rs} (100%) rename crates/{service/storyteller_web/src/database/queries => lib/database_queries/src/tts/tts_results}/query_tts_result.rs (97%) diff --git a/crates/lib/database_queries/src/tts/mod.rs b/crates/lib/database_queries/src/tts/mod.rs index 3002a2e0e7..9b179450a1 100644 --- a/crates/lib/database_queries/src/tts/mod.rs +++ b/crates/lib/database_queries/src/tts/mod.rs @@ -1,3 +1,4 @@ +pub mod stats; pub mod tts_category_assignments; pub mod tts_inference_jobs; pub mod tts_models; diff --git a/crates/service/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs b/crates/lib/database_queries/src/tts/stats/calculate_tts_model_leaderboard.rs similarity index 100% rename from crates/service/storyteller_web/src/database/queries/calculate_tts_model_leaderboard.rs rename to crates/lib/database_queries/src/tts/stats/calculate_tts_model_leaderboard.rs diff --git a/crates/lib/database_queries/src/tts/stats/mod.rs b/crates/lib/database_queries/src/tts/stats/mod.rs new file mode 100644 index 0000000000..c0e5981099 --- /dev/null +++ b/crates/lib/database_queries/src/tts/stats/mod.rs @@ -0,0 +1 @@ +pub mod calculate_tts_model_leaderboard; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/delete_tts_model.rs b/crates/lib/database_queries/src/tts/tts_models/delete_tts_model_various_scopes.rs similarity index 100% rename from crates/service/storyteller_web/src/database/queries/delete_tts_model.rs rename to crates/lib/database_queries/src/tts/tts_models/delete_tts_model_various_scopes.rs diff --git a/crates/lib/database_queries/src/tts/tts_models/mod.rs b/crates/lib/database_queries/src/tts/tts_models/mod.rs index d9c349ff7d..ecd8eec251 100644 --- a/crates/lib/database_queries/src/tts/tts_models/mod.rs +++ b/crates/lib/database_queries/src/tts/tts_models/mod.rs @@ -1,3 +1,4 @@ +pub mod delete_tts_model_various_scopes; pub mod get_tts_model; pub mod get_tts_model_for_inference; pub mod list_tts_models; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/delete_tts_result.rs b/crates/lib/database_queries/src/tts/tts_results/delete_tts_result_various_scopes.rs similarity index 100% rename from crates/service/storyteller_web/src/database/queries/delete_tts_result.rs rename to crates/lib/database_queries/src/tts/tts_results/delete_tts_result_various_scopes.rs diff --git a/crates/lib/database_queries/src/tts/tts_results/mod.rs b/crates/lib/database_queries/src/tts/tts_results/mod.rs index 933ac21575..5d84111c0d 100644 --- a/crates/lib/database_queries/src/tts/tts_results/mod.rs +++ b/crates/lib/database_queries/src/tts/tts_results/mod.rs @@ -1 +1,3 @@ -pub mod insert_tts_result; \ No newline at end of file +pub mod delete_tts_result_various_scopes; +pub mod insert_tts_result; +pub mod query_tts_result; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/query_tts_result.rs b/crates/lib/database_queries/src/tts/tts_results/query_tts_result.rs similarity index 97% rename from crates/service/storyteller_web/src/database/queries/query_tts_result.rs rename to crates/lib/database_queries/src/tts/tts_results/query_tts_result.rs index 2dcc5f50d6..958145448b 100644 --- a/crates/service/storyteller_web/src/database/queries/query_tts_result.rs +++ b/crates/lib/database_queries/src/tts/tts_results/query_tts_result.rs @@ -1,11 +1,10 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use crate::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::enums::vocoder_type::VocoderType; -use database_queries::helpers::boolean_converters::nullable_i8_to_bool; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::record_visibility::RecordVisibility; +use crate::column_types::vocoder_type::VocoderType; +use crate::helpers::boolean_converters::nullable_i8_to_bool; use log::{info, warn, log}; -use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; diff --git a/crates/service/storyteller_web/src/database/queries/mod.rs b/crates/service/storyteller_web/src/database/queries/mod.rs index 11ec43f601..622562cfae 100644 --- a/crates/service/storyteller_web/src/database/queries/mod.rs +++ b/crates/service/storyteller_web/src/database/queries/mod.rs @@ -1,10 +1,7 @@ pub mod badges; -pub mod calculate_tts_model_leaderboard; pub mod calculate_w2l_template_leaderboard; pub mod categories; pub mod create_session; -pub mod delete_tts_model; -pub mod delete_tts_result; pub mod delete_w2l_result; pub mod delete_w2l_template; pub mod get_user_profile_by_username; @@ -12,6 +9,5 @@ pub mod list_ip_bans; pub mod list_user_badges; pub mod list_user_roles; pub mod list_w2l_templates; -pub mod query_tts_result; pub mod query_w2l_result; pub mod query_w2l_template; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs b/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs index 0b2c49fcac..79e0ea3e13 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs @@ -1,20 +1,18 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; -use crate::AnyhowResult; -use crate::database::queries::calculate_tts_model_leaderboard::TtsLeaderboardRecordForList; -use crate::database::queries::calculate_tts_model_leaderboard::calculate_tts_model_leaderboard; use crate::database::queries::calculate_w2l_template_leaderboard::W2lLeaderboardRecordForList; use crate::database::queries::calculate_w2l_template_leaderboard::calculate_w2l_template_leaderboard; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; +use database_queries::tts::stats::calculate_tts_model_leaderboard::TtsLeaderboardRecordForList; +use database_queries::tts::stats::calculate_tts_model_leaderboard::calculate_tts_model_leaderboard; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; @@ -65,7 +63,7 @@ impl ResponseError for LeaderboardErrorResponse { } } -// NB: Not using DeriveMore since Clion doesn't understand it. +// NB: Not using derive_more::Display since Clion doesn't understand it. impl fmt::Display for LeaderboardErrorResponse { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self.error_type) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs index a99d8d954b..d018d6fee5 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs @@ -6,17 +6,16 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::delete_tts_model::delete_tts_model_as_mod; -use crate::database::queries::delete_tts_model::delete_tts_model_as_user; -use crate::database::queries::delete_tts_model::undelete_tts_model_as_mod; -use crate::database::queries::delete_tts_model::undelete_tts_model_as_user; -use crate::database::queries::query_tts_result::select_tts_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::delete_role_disambiguation::DeleteRole; use crate::util::delete_role_disambiguation::delete_role_disambiguation; +use database_queries::tts::tts_models::delete_tts_model_various_scopes::delete_tts_model_as_mod; +use database_queries::tts::tts_models::delete_tts_model_various_scopes::delete_tts_model_as_user; +use database_queries::tts::tts_models::delete_tts_model_various_scopes::undelete_tts_model_as_mod; +use database_queries::tts::tts_models::delete_tts_model_various_scopes::undelete_tts_model_as_user; use database_queries::tts::tts_models::get_tts_model::get_tts_model_by_token; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs index 6f3621d13d..5301abb70d 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs @@ -1,16 +1,11 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::delete_tts_result::delete_tts_inference_result_as_mod; -use crate::database::queries::delete_tts_result::delete_tts_inference_result_as_user; -use crate::database::queries::delete_tts_result::undelete_tts_inference_result_as_mod; -use crate::database::queries::delete_tts_result::undelete_tts_inference_result_as_user; -use crate::database::queries::query_tts_result::select_tts_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -20,13 +15,18 @@ use crate::util::delete_role_disambiguation::delete_role_disambiguation; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use derive_more::{Display, Error}; +use database_queries::tts::tts_results::delete_tts_result_various_scopes::delete_tts_inference_result_as_mod; +use database_queries::tts::tts_results::delete_tts_result_various_scopes::delete_tts_inference_result_as_user; +use database_queries::tts::tts_results::delete_tts_result_various_scopes::undelete_tts_inference_result_as_mod; +use database_queries::tts::tts_results::delete_tts_result_various_scopes::undelete_tts_inference_result_as_user; +use database_queries::tts::tts_results::query_tts_result::select_tts_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -42,7 +42,7 @@ pub struct DeleteTtsInferenceResultRequest { as_mod: Option, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum DeleteTtsInferenceResultError { BadInput(String), NotAuthorized, @@ -72,6 +72,13 @@ impl ResponseError for DeleteTtsInferenceResultError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for DeleteTtsInferenceResultError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn delete_tts_inference_result_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs index f20d0799cd..fa5081175f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs @@ -6,14 +6,14 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::queries::query_tts_result::select_tts_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; +use database_queries::column_types::record_visibility::RecordVisibility; +use database_queries::tts::tts_results::query_tts_result::select_tts_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs index 7a6d770efa..c87872956f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs @@ -1,28 +1,28 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::queries::query_tts_result::TtsResultRecordForResponse; -use crate::database::queries::query_tts_result::select_tts_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use derive_more::{Display, Error}; +use database_queries::tts::tts_results::query_tts_result::TtsResultRecordForResponse; +use database_queries::tts::tts_results::query_tts_result::select_tts_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -37,7 +37,7 @@ pub struct GetTtsResultSuccessResponse { pub result: TtsResultRecordForResponse, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum GetTtsResultError { ServerError, NotFound, @@ -61,6 +61,13 @@ impl ResponseError for GetTtsResultError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GetTtsResultError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn get_tts_inference_result_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs index 60b682347b..475d8f28b1 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; @@ -10,7 +10,6 @@ use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; use log::{info, warn, log}; use r2d2_redis::redis::Commands; use redis_common::redis_keys::RedisKeys; @@ -19,6 +18,7 @@ use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -56,7 +56,7 @@ pub struct GetW2lInferenceStatusSuccessResponse { pub state: W2lInferenceJobStatusForResponse, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum GetW2lInferenceStatusError { ServerError, } @@ -95,6 +95,13 @@ impl ResponseError for GetW2lInferenceStatusError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GetW2lInferenceStatusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn get_w2l_inference_job_status_handler( http_request: HttpRequest, path: Path, From 93c5ec0257d8afe5a07dcf1cee06ee34252f0dfb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 00:36:35 -0500 Subject: [PATCH 0943/1042] move w2l stats query --- crates/lib/database_queries/src/lib.rs | 1 + .../src/tts/stats/calculate_tts_model_leaderboard.rs | 2 +- crates/lib/database_queries/src/w2l/mod.rs | 1 + .../src/w2l/stats}/calculate_w2l_template_leaderboard.rs | 2 +- crates/lib/database_queries/src/w2l/stats/mod.rs | 1 + crates/service/storyteller_web/src/database/queries/mod.rs | 1 - .../src/http_server/endpoints/leaderboard/get_leaderboard.rs | 4 ++-- 7 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 crates/lib/database_queries/src/w2l/mod.rs rename crates/{service/storyteller_web/src/database/queries => lib/database_queries/src/w2l/stats}/calculate_w2l_template_leaderboard.rs (98%) create mode 100644 crates/lib/database_queries/src/w2l/stats/mod.rs diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index a248fcebb1..6f01cc9da8 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -11,3 +11,4 @@ pub mod twitch_event_rules; pub mod twitch_oauth; pub mod twitch_pubsub; pub mod voice_clone_requests; +pub mod w2l; diff --git a/crates/lib/database_queries/src/tts/stats/calculate_tts_model_leaderboard.rs b/crates/lib/database_queries/src/tts/stats/calculate_tts_model_leaderboard.rs index 18364cbd18..0f5b4a5e86 100644 --- a/crates/lib/database_queries/src/tts/stats/calculate_tts_model_leaderboard.rs +++ b/crates/lib/database_queries/src/tts/stats/calculate_tts_model_leaderboard.rs @@ -13,7 +13,7 @@ pub struct TtsLeaderboardRecordForList { } #[derive(Serialize)] -pub struct TtsLeaderboardRecordForListRaw { +struct TtsLeaderboardRecordForListRaw { pub username: String, pub display_name: String, pub gravatar_hash: String, diff --git a/crates/lib/database_queries/src/w2l/mod.rs b/crates/lib/database_queries/src/w2l/mod.rs new file mode 100644 index 0000000000..fedcafddb7 --- /dev/null +++ b/crates/lib/database_queries/src/w2l/mod.rs @@ -0,0 +1 @@ +pub mod stats; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs b/crates/lib/database_queries/src/w2l/stats/calculate_w2l_template_leaderboard.rs similarity index 98% rename from crates/service/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs rename to crates/lib/database_queries/src/w2l/stats/calculate_w2l_template_leaderboard.rs index 94562e359d..52ed9d5979 100644 --- a/crates/service/storyteller_web/src/database/queries/calculate_w2l_template_leaderboard.rs +++ b/crates/lib/database_queries/src/w2l/stats/calculate_w2l_template_leaderboard.rs @@ -13,7 +13,7 @@ pub struct W2lLeaderboardRecordForList { } #[derive(Serialize)] -pub struct W2lLeaderboardRecordForListRaw { +struct W2lLeaderboardRecordForListRaw { pub username: String, pub display_name: String, pub gravatar_hash: String, diff --git a/crates/lib/database_queries/src/w2l/stats/mod.rs b/crates/lib/database_queries/src/w2l/stats/mod.rs new file mode 100644 index 0000000000..0a326d2cd9 --- /dev/null +++ b/crates/lib/database_queries/src/w2l/stats/mod.rs @@ -0,0 +1 @@ +pub mod calculate_w2l_template_leaderboard; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/mod.rs b/crates/service/storyteller_web/src/database/queries/mod.rs index 622562cfae..528eebbe5d 100644 --- a/crates/service/storyteller_web/src/database/queries/mod.rs +++ b/crates/service/storyteller_web/src/database/queries/mod.rs @@ -1,5 +1,4 @@ pub mod badges; -pub mod calculate_w2l_template_leaderboard; pub mod categories; pub mod create_session; pub mod delete_w2l_result; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs b/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs index 79e0ea3e13..5b31ce7826 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs @@ -6,13 +6,13 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; -use crate::database::queries::calculate_w2l_template_leaderboard::W2lLeaderboardRecordForList; -use crate::database::queries::calculate_w2l_template_leaderboard::calculate_w2l_template_leaderboard; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; use database_queries::tts::stats::calculate_tts_model_leaderboard::TtsLeaderboardRecordForList; use database_queries::tts::stats::calculate_tts_model_leaderboard::calculate_tts_model_leaderboard; +use database_queries::w2l::stats::calculate_w2l_template_leaderboard::W2lLeaderboardRecordForList; +use database_queries::w2l::stats::calculate_w2l_template_leaderboard::calculate_w2l_template_leaderboard; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; From fd272a853a581dc1787d2c7be4b96f78637ab19d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 01:05:05 -0500 Subject: [PATCH 0944/1042] refactor: get rid of in-app database::enums module --- .../database_queries/src/column_types/mod.rs | 1 + .../src/column_types}/tts_model_type.rs | 0 .../src/column_types/vocoder_type.rs | 2 ++ .../storyteller_web/src/database/enums/mod.rs | 3 -- .../src/database/enums/record_visibility.rs | 35 ------------------- .../src/database/enums/vocoder_type.rs | 33 ----------------- .../storyteller_web/src/database/mod.rs | 1 - .../src/database/queries/query_w2l_result.rs | 2 +- .../database/queries/query_w2l_template.rs | 2 +- ...t_assigned_tts_categories_query_builder.rs | 1 - .../list_categories_query_builder.rs | 1 - ...ist_tts_inference_results_query_builder.rs | 2 +- ...ist_w2l_inference_results_query_builder.rs | 2 +- .../endpoints/api_tokens/create_api_token.rs | 1 - .../endpoints/api_tokens/list_api_tokens.rs | 4 +-- .../categories/assign_tts_category.rs | 1 - .../endpoints/categories/create_category.rs | 1 - .../endpoints/categories/get_category.rs | 1 - .../categories/list_tts_categories.rs | 2 -- .../endpoints/misc/detect_locale_handler.rs | 1 - .../moderation/categories/delete_category.rs | 2 -- .../moderation/categories/edit_category.rs | 4 +-- .../list_tts_categories_for_moderation.rs | 2 -- .../endpoints/tts/enqueue_infer_tts.rs | 13 +++++-- .../endpoints/users/edit_profile.rs | 13 +++++-- .../endpoints/users/get_profile.rs | 6 ++-- .../endpoints/w2l/edit_w2l_result.rs | 15 +++++--- .../endpoints/w2l/edit_w2l_template.rs | 13 +++++-- .../endpoints/w2l/enqueue_infer_w2l.rs | 15 +++++--- .../w2l/enqueue_infer_w2l_with_uploads.rs | 13 +++++-- .../http_server/web_utils/session_checker.rs | 6 ++-- .../job_queries/w2l_inference_job_queries.rs | 4 +-- .../storyteller_web/src/tts_download_job.rs | 2 +- 33 files changed, 80 insertions(+), 124 deletions(-) rename crates/{service/storyteller_web/src/database/enums => lib/database_queries/src/column_types}/tts_model_type.rs (100%) delete mode 100644 crates/service/storyteller_web/src/database/enums/mod.rs delete mode 100644 crates/service/storyteller_web/src/database/enums/record_visibility.rs delete mode 100644 crates/service/storyteller_web/src/database/enums/vocoder_type.rs diff --git a/crates/lib/database_queries/src/column_types/mod.rs b/crates/lib/database_queries/src/column_types/mod.rs index 6ecfe45966..9b54f0c4a4 100644 --- a/crates/lib/database_queries/src/column_types/mod.rs +++ b/crates/lib/database_queries/src/column_types/mod.rs @@ -1,4 +1,5 @@ pub mod job_status; pub mod record_visibility; +pub mod tts_model_type; pub mod twitch_event_category; pub mod vocoder_type; diff --git a/crates/service/storyteller_web/src/database/enums/tts_model_type.rs b/crates/lib/database_queries/src/column_types/tts_model_type.rs similarity index 100% rename from crates/service/storyteller_web/src/database/enums/tts_model_type.rs rename to crates/lib/database_queries/src/column_types/tts_model_type.rs diff --git a/crates/lib/database_queries/src/column_types/vocoder_type.rs b/crates/lib/database_queries/src/column_types/vocoder_type.rs index ad92b2687e..ec63f25a61 100644 --- a/crates/lib/database_queries/src/column_types/vocoder_type.rs +++ b/crates/lib/database_queries/src/column_types/vocoder_type.rs @@ -1,3 +1,5 @@ +//! These denote the pretrained vocoders + use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; diff --git a/crates/service/storyteller_web/src/database/enums/mod.rs b/crates/service/storyteller_web/src/database/enums/mod.rs deleted file mode 100644 index a30b2cbad3..0000000000 --- a/crates/service/storyteller_web/src/database/enums/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod record_visibility; -pub mod tts_model_type; -pub mod vocoder_type; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/enums/record_visibility.rs b/crates/service/storyteller_web/src/database/enums/record_visibility.rs deleted file mode 100644 index 863d80154f..0000000000 --- a/crates/service/storyteller_web/src/database/enums/record_visibility.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! These are columns where users can control the visibility of their data. - -use anyhow::anyhow; -use container_common::anyhow_result::AnyhowResult; - -/// To use this in a query, the query must have type annotations. -/// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375 -/// eg. preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility` -#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] -#[serde(rename_all = "lowercase")] -#[sqlx(rename_all = "lowercase")] -pub enum RecordVisibility { - Public, - Hidden, - Private, -} - -impl RecordVisibility { - pub fn to_str(&self) -> &'static str { - match self { - RecordVisibility::Public => "public", - RecordVisibility::Hidden => "hidden", - RecordVisibility::Private => "private", - } - } - - pub fn from_str(record_visibility: &str) -> AnyhowResult { - match record_visibility { - "public" => Ok(RecordVisibility::Public), - "hidden" => Ok(RecordVisibility::Hidden), - "private" => Ok(RecordVisibility::Private), - _ => Err(anyhow!("invalid value: {:?}", record_visibility)), - } - } -} diff --git a/crates/service/storyteller_web/src/database/enums/vocoder_type.rs b/crates/service/storyteller_web/src/database/enums/vocoder_type.rs deleted file mode 100644 index ec63f25a61..0000000000 --- a/crates/service/storyteller_web/src/database/enums/vocoder_type.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! These denote the pretrained vocoders - -use anyhow::anyhow; -use container_common::anyhow_result::AnyhowResult; - -#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] -#[serde(rename_all = "lowercase")] -#[sqlx(rename_all = "lowercase")] -pub enum VocoderType { - /// Current state of the art vocoder - #[serde(rename = "hifigan-superres")] - HifiGanSuperResolution, - - /// Older WaveGlow vocoder - WaveGlow, -} - -impl VocoderType { - pub fn to_str(&self) -> &'static str { - match self { - VocoderType::HifiGanSuperResolution => "hifigan-superres", - VocoderType::WaveGlow => "waveglow", - } - } - - pub fn from_str(vocoder_type: &str) -> AnyhowResult { - match vocoder_type { - "hifigan-superres" => Ok(VocoderType::HifiGanSuperResolution), - "waveglow" => Ok(VocoderType::WaveGlow), - _ => Err(anyhow!("invalid value: {:?}", vocoder_type)), - } - } -} diff --git a/crates/service/storyteller_web/src/database/mod.rs b/crates/service/storyteller_web/src/database/mod.rs index 909f503d5e..5cfe9b18f8 100644 --- a/crates/service/storyteller_web/src/database/mod.rs +++ b/crates/service/storyteller_web/src/database/mod.rs @@ -1,4 +1,3 @@ -pub mod enums; pub mod helpers; pub mod mediators; pub mod queries; diff --git a/crates/service/storyteller_web/src/database/queries/query_w2l_result.rs b/crates/service/storyteller_web/src/database/queries/query_w2l_result.rs index 3b2e73379d..e3e5cd19f3 100644 --- a/crates/service/storyteller_web/src/database/queries/query_w2l_result.rs +++ b/crates/service/storyteller_web/src/database/queries/query_w2l_result.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; +use database_queries::column_types::record_visibility::RecordVisibility; use database_queries::helpers::boolean_converters::nullable_i8_to_bool; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/database/queries/query_w2l_template.rs b/crates/service/storyteller_web/src/database/queries/query_w2l_template.rs index 96d3ca9495..96d095fd78 100644 --- a/crates/service/storyteller_web/src/database/queries/query_w2l_template.rs +++ b/crates/service/storyteller_web/src/database/queries/query_w2l_template.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; +use database_queries::column_types::record_visibility::RecordVisibility; use database_queries::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; use derive_more::{Display, Error}; use log::{info, warn, log}; diff --git a/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs index 18c70ce50e..b04355fc14 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs @@ -1,7 +1,6 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; use database_queries::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs index a72fb912d0..1b52af54c1 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs @@ -1,7 +1,6 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; use database_queries::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs index 66e38cc1bd..4b47af31b7 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs @@ -2,7 +2,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use container_common::anyhow_result::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; +use database_queries::column_types::record_visibility::RecordVisibility; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs b/crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs index b9fc2bcf11..2d3b31b66d 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs +++ b/crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs @@ -2,7 +2,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use container_common::anyhow_result::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; +use database_queries::column_types::record_visibility::RecordVisibility; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs index abfc39bc04..476fbcaf26 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs index 80d99fc231..2df591a159 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs @@ -7,15 +7,14 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; -use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; +use database_queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use lexical_sort::natural_lexical_cmp; use log::{info, warn, log}; @@ -26,7 +25,6 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; -use database_queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; // =============== Success Response =============== diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs index e3b99e291b..a439623ae5 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use anyhow::anyhow; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::categories::get_category_by_token::get_category_by_token; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs index 7359dac189..82c0ed8740 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs index ae6e0c3b21..225a10dfa6 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::categories::get_category_by_token::get_category_by_token; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs index 6b2d8db103..160171cb4c 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs @@ -7,8 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs index ff4ce7fad2..394dcb9f87 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::categories::get_category_by_token::get_category_by_token; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs index d6d3241a98..b8f8021d0f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/delete_category.rs @@ -6,8 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs index 3396691eca..87d61be31d 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs @@ -6,8 +6,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use crate::database::queries::categories::get_category_by_token::get_category_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -23,7 +22,6 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; -use crate::database::queries::categories::get_category_by_token::get_category_by_token; // =============== Request =============== diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs index 56339bce95..09cd85fad5 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs @@ -7,8 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::enums::record_visibility::RecordVisibility; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs index fa8d6541c7..4589f7be25 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -5,7 +5,6 @@ use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::enums::{DownloadUrlType, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -13,8 +12,8 @@ use crate::server_state::ServerState; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; +use database_queries::column_types::record_visibility::RecordVisibility; use database_queries::tokens::Tokens; -use derive_more::{Display, Error}; use log::{info, warn, log}; use r2d2_redis::redis::Commands; use redis_common::redis_keys::RedisKeys; @@ -22,6 +21,7 @@ use regex::Regex; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; use user_input_common::check_for_slurs::contains_slurs; @@ -39,7 +39,7 @@ pub struct InferTtsSuccessResponse { pub inference_job_token: String, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum InferTtsError { BadInput(String), ServerError, @@ -66,6 +66,13 @@ impl ResponseError for InferTtsError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for InferTtsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn infer_tts_handler( http_request: HttpRequest, request: web::Json, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs index a1053d21bc..dc9a8d40cd 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::create_session::create_session_for_user; use crate::database::queries::get_user_profile_by_username::get_user_profile_by_username; use crate::http_server::web_utils::ip_address::get_request_ip; @@ -23,12 +22,13 @@ use crate::validations::twitter_username::{validate_twitter_username, normalize_ use crate::validations::username::validate_username; use crate::validations::username_reservations::is_reserved_username; use crate::validations::website_url::validate_website_url; -use derive_more::{Display, Error}; +use database_queries::column_types::record_visibility::RecordVisibility; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; use user_input_common::check_for_slurs::contains_slurs; @@ -61,7 +61,7 @@ pub struct EditProfileSuccessResponse { pub success: bool, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum EditProfileError { BadInput(String), NotAuthorized, @@ -91,6 +91,13 @@ impl ResponseError for EditProfileError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for EditProfileError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn edit_profile_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs index 45d41a5341..3d101d0b71 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs @@ -9,11 +9,11 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use anyhow::anyhow; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::list_user_badges::UserBadgeForList; use crate::database::queries::list_user_badges::list_user_badges; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; +use database_queries::column_types::record_visibility::RecordVisibility; use database_queries::helpers::boolean_converters::i8_to_bool; use log::{info, warn, log}; use md5::{Md5, Digest}; @@ -160,8 +160,8 @@ SELECT profile_rendered_html, user_role_slug, disable_gravatar, - preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility`, - preferred_w2l_result_visibility as `preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility`, + preferred_tts_result_visibility as `preferred_tts_result_visibility: database_queries::column_types::record_visibility::RecordVisibility`, + preferred_w2l_result_visibility as `preferred_w2l_result_visibility: database_queries::column_types::record_visibility::RecordVisibility`, discord_username, twitch_username, twitter_username, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs index d514e75650..2a90ef0a93 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs @@ -1,12 +1,11 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::query_w2l_result::select_w2l_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -14,13 +13,14 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use derive_more::{Display, Error}; +use database_queries::column_types::record_visibility::RecordVisibility; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -35,7 +35,7 @@ pub struct EditW2lResultRequest { pub creator_set_visibility: Option, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum EditW2lResultError { BadInput(String), NotAuthorized, @@ -65,6 +65,13 @@ impl ResponseError for EditW2lResultError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for EditW2lResultError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn edit_w2l_inference_result_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs index 7a8c3b01c7..355e62faad 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -14,13 +13,14 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use derive_more::{Display, Error}; +use database_queries::column_types::record_visibility::RecordVisibility; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; use user_input_common::check_for_slurs::contains_slurs; @@ -46,7 +46,7 @@ pub struct EditW2lTemplateRequest { pub maybe_mod_comments: Option, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum EditW2lTemplateError { BadInput(String), NotAuthorized, @@ -76,6 +76,13 @@ impl ResponseError for EditW2lTemplateError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for EditW2lTemplateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn edit_w2l_template_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index 0a826daa04..559d7e90df 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -1,11 +1,10 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::enums::record_visibility::RecordVisibility; use crate::database::helpers::enums::{DownloadUrlType, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -13,7 +12,7 @@ use crate::server_state::ServerState; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use derive_more::{Display, Error}; +use database_queries::column_types::record_visibility::RecordVisibility; use log::{info, warn, log}; use r2d2_redis::redis::Commands; use redis_common::redis_keys::RedisKeys; @@ -21,6 +20,7 @@ use regex::Regex; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; #[derive(Deserialize)] @@ -35,7 +35,7 @@ pub struct InferW2lSuccessResponse { pub success: bool, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum InferW2lError { BadInput(String), NotAuthorized, @@ -65,6 +65,13 @@ impl ResponseError for InferW2lError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for InferW2lError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn infer_w2l_handler( http_request: HttpRequest, request: web::Json, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 5f1302a203..f2a1dbad48 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -9,7 +9,6 @@ use actix_web::{middleware, web, App, Error, HttpResponse, HttpServer, HttpReque use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; use container_common::token::random_uuid::generate_random_uuid; -use crate::database::enums::record_visibility::RecordVisibility; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::read_multipart_field_bytes::checked_read_multipart_bytes; use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_boolean; @@ -19,14 +18,15 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_paths::hash_to_bucket_path; +use database_queries::column_types::record_visibility::RecordVisibility; use database_queries::tokens::Tokens; -use derive_more::{Display, Error}; use futures::{StreamExt, TryStreamExt}; use log::{warn, info}; use r2d2_redis::redis::Commands; use redis_common::redis_keys::RedisKeys; use sqlx::MySqlPool; use sqlx::error::Error::Database; +use std::fmt; use std::io::Write; use std::sync::Arc; @@ -50,7 +50,7 @@ pub struct InferW2lWithUploadSuccessResponse { pub inference_job_token: String, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum InferW2lWithUploadError { BadInput(String), EmptyFileUploaded, @@ -80,6 +80,13 @@ impl ResponseError for InferW2lWithUploadError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for InferW2lWithUploadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + /// This handles audio uploads w/ W2L templates. pub async fn enqueue_infer_w2l_with_uploads( http_request: HttpRequest, diff --git a/crates/service/storyteller_web/src/http_server/web_utils/session_checker.rs b/crates/service/storyteller_web/src/http_server/web_utils/session_checker.rs index e922c1f629..19f64d8de1 100644 --- a/crates/service/storyteller_web/src/http_server/web_utils/session_checker.rs +++ b/crates/service/storyteller_web/src/http_server/web_utils/session_checker.rs @@ -1,8 +1,8 @@ use actix_web::HttpRequest; use anyhow::anyhow; use crate::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; use crate::http_server::web_utils::cookie_manager::CookieManager; +use database_queries::column_types::record_visibility::RecordVisibility; use database_queries::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; use log::{info, warn}; use sqlx::MySqlPool; @@ -189,8 +189,8 @@ SELECT users.disable_gravatar, users.auto_play_audio_preference, users.auto_play_video_preference, - users.preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility`, - users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility`, + users.preferred_tts_result_visibility as `preferred_tts_result_visibility: database_queries::column_types::record_visibility::RecordVisibility`, + users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: database_queries::column_types::record_visibility::RecordVisibility`, users.user_role_slug, users.is_banned, diff --git a/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs b/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs index 3e4b2878fb..0db4a6b258 100644 --- a/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs +++ b/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs @@ -5,7 +5,7 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; use container_common::anyhow_result::AnyhowResult; -use crate::database::enums::record_visibility::RecordVisibility; +use database_queries::column_types::record_visibility::RecordVisibility; use database_queries::tokens::Tokens; use log::{warn, info}; use sqlx::MySqlPool; @@ -75,7 +75,7 @@ SELECT creator_ip_address, maybe_creator_user_token, - creator_set_visibility as `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`, + creator_set_visibility as `creator_set_visibility: database_queries::column_types::record_visibility::RecordVisibility`, disable_end_bump, disable_watermark, diff --git a/crates/service/storyteller_web/src/tts_download_job.rs b/crates/service/storyteller_web/src/tts_download_job.rs index a2fcdf5172..89d91616ad 100644 --- a/crates/service/storyteller_web/src/tts_download_job.rs +++ b/crates/service/storyteller_web/src/tts_download_job.rs @@ -22,7 +22,6 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use crate::common_env::CommonEnv; -use crate::database::enums::tts_model_type::TtsModelType; use crate::database::mediators::badge_granter::BadgeGranter; use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; @@ -44,6 +43,7 @@ use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; +use database_queries::column_types::tts_model_type::TtsModelType; use log::{warn, info}; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2; From bea83760668806e2eb52a0cd26e399e0c21cd9f6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 01:23:55 -0500 Subject: [PATCH 0945/1042] more refactor --- Cargo.lock | 1 + crates/lib/database_queries/Cargo.toml | 1 + ...list_assigned_tts_categories_query_builder.rs | 2 +- .../src/tts/tts_category_assignments/mod.rs | 3 ++- .../list_tts_results_query_builder.rs} | 4 ++-- .../database_queries/src/tts/tts_results/mod.rs | 1 + .../src/database/query_builders/mod.rs | 2 -- .../list_tts_model_assigned_categories.rs | 6 +----- .../users/list_user_tts_inference_results.rs | 16 +++++++++++----- .../endpoints/w2l/enqueue_upload_w2l_template.rs | 11 +++++++++-- 10 files changed, 29 insertions(+), 18 deletions(-) rename crates/{service/storyteller_web/src/database/query_builders => lib/database_queries/src/tts/tts_category_assignments}/list_assigned_tts_categories_query_builder.rs (98%) rename crates/{service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs => lib/database_queries/src/tts/tts_results/list_tts_results_query_builder.rs} (98%) diff --git a/Cargo.lock b/Cargo.lock index 13a3879a06..fc78aff667 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1250,6 +1250,7 @@ dependencies = [ "anyhow", "base64 0.13.0", "chrono", + "config", "container_common", "dotenv", "log", diff --git a/crates/lib/database_queries/Cargo.toml b/crates/lib/database_queries/Cargo.toml index bd52e1e20c..b15ef8bd7f 100644 --- a/crates/lib/database_queries/Cargo.toml +++ b/crates/lib/database_queries/Cargo.toml @@ -15,6 +15,7 @@ path = "src/lib.rs" [dependencies] # Internal +config = { path = "../config" } container_common = { path = "../container_common" } # External diff --git a/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs b/crates/lib/database_queries/src/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs similarity index 98% rename from crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs rename to crates/lib/database_queries/src/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs index b04355fc14..eb3e225169 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_assigned_tts_categories_query_builder.rs +++ b/crates/lib/database_queries/src/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; -use database_queries::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; +use crate::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/lib/database_queries/src/tts/tts_category_assignments/mod.rs b/crates/lib/database_queries/src/tts/tts_category_assignments/mod.rs index 84f8c71e26..dc821db4fb 100644 --- a/crates/lib/database_queries/src/tts/tts_category_assignments/mod.rs +++ b/crates/lib/database_queries/src/tts/tts_category_assignments/mod.rs @@ -1 +1,2 @@ -pub mod fetch_and_build_tts_model_category_map; \ No newline at end of file +pub mod fetch_and_build_tts_model_category_map; +pub mod list_assigned_tts_categories_query_builder; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs b/crates/lib/database_queries/src/tts/tts_results/list_tts_results_query_builder.rs similarity index 98% rename from crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs rename to crates/lib/database_queries/src/tts/tts_results/list_tts_results_query_builder.rs index 4b47af31b7..8c40002ff0 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_tts_inference_results_query_builder.rs +++ b/crates/lib/database_queries/src/tts/tts_results/list_tts_results_query_builder.rs @@ -2,7 +2,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use container_common::anyhow_result::AnyhowResult; -use database_queries::column_types::record_visibility::RecordVisibility; +use crate::column_types::record_visibility::RecordVisibility; use log::{warn, info}; use sqlx::MySqlPool; @@ -326,7 +326,7 @@ pub struct RawInternalTtsRecord { #[cfg(test)] mod tests { - use crate::database::query_builders::list_tts_inference_results_query_builder::ListTtsResultsQueryBuilder; + use crate::tts::tts_results::list_tts_results_query_builder::ListTtsResultsQueryBuilder; #[test] fn predicates_without_scoping() { diff --git a/crates/lib/database_queries/src/tts/tts_results/mod.rs b/crates/lib/database_queries/src/tts/tts_results/mod.rs index 5d84111c0d..a4b88aa68b 100644 --- a/crates/lib/database_queries/src/tts/tts_results/mod.rs +++ b/crates/lib/database_queries/src/tts/tts_results/mod.rs @@ -1,3 +1,4 @@ pub mod delete_tts_result_various_scopes; pub mod insert_tts_result; +pub mod list_tts_results_query_builder; pub mod query_tts_result; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/query_builders/mod.rs b/crates/service/storyteller_web/src/database/query_builders/mod.rs index 9cd5018348..55cc079dcc 100644 --- a/crates/service/storyteller_web/src/database/query_builders/mod.rs +++ b/crates/service/storyteller_web/src/database/query_builders/mod.rs @@ -1,5 +1,3 @@ -pub mod list_assigned_tts_categories_query_builder; pub mod list_categories_query_builder; -pub mod list_tts_inference_results_query_builder; pub mod list_users_query_builder; pub mod list_w2l_inference_results_query_builder; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs index 5de3a6fbf7..c08faf969a 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs @@ -7,14 +7,10 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; -use crate::database::query_builders::list_assigned_tts_categories_query_builder::ListAssignedTtsCategoriesQueryBuilder; -use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; -use crate::util::email_to_gravatar::email_to_gravatar; -use crate::util::markdown_to_html::markdown_to_html; +use database_queries::tts::tts_category_assignments::list_assigned_tts_categories_query_builder::ListAssignedTtsCategoriesQueryBuilder; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs index 4b1a64dc5e..c01e2b2585 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -1,7 +1,7 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Query}; @@ -9,18 +9,17 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::database::queries::create_session::create_session_for_user; -use crate::database::query_builders::list_tts_inference_results_query_builder::ListTtsResultsQueryBuilder; -use crate::database::query_builders::list_tts_inference_results_query_builder::TtsInferenceRecordForList; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; +use database_queries::tts::tts_results::list_tts_results_query_builder::{ListTtsResultsQueryBuilder, TtsInferenceRecordForList}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -45,7 +44,7 @@ pub struct ListTtsInferenceResultsForUserSuccessResponse { pub cursor_previous: Option, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum ListTtsInferenceResultsForUserError { ServerError, } @@ -66,6 +65,13 @@ impl ResponseError for ListTtsInferenceResultsForUserError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListTtsInferenceResultsForUserError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn list_user_tts_inference_results_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index d7fb8fca88..cd9e4151f4 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -13,12 +13,12 @@ use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use database_queries::tokens::Tokens; -use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; use user_input_common::check_for_slurs::contains_slurs; @@ -38,7 +38,7 @@ pub struct UploadW2lTemplateSuccessResponse { pub job_token: String, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum UploadW2lTemplateError { BadInput(String), MustBeLoggedIn, @@ -68,6 +68,13 @@ impl ResponseError for UploadW2lTemplateError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for UploadW2lTemplateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn upload_w2l_template_handler( http_request: HttpRequest, request: web::Json, From a46879529402ab4dc68e43f108f00224f0c02483 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 01:32:39 -0500 Subject: [PATCH 0946/1042] more refactoring, getting rid of old enums --- .../src/database/helpers/enums.rs | 49 ------------------- .../src/database/helpers/mod.rs | 1 - .../storyteller_web/src/database/mod.rs | 1 - .../endpoints/tts/enqueue_infer_tts.rs | 1 - .../endpoints/tts/enqueue_upload_tts_model.rs | 37 ++++++++++++-- .../endpoints/w2l/enqueue_infer_w2l.rs | 1 - .../w2l/enqueue_upload_w2l_template.rs | 22 ++++++++- 7 files changed, 54 insertions(+), 58 deletions(-) delete mode 100644 crates/service/storyteller_web/src/database/helpers/enums.rs delete mode 100644 crates/service/storyteller_web/src/database/helpers/mod.rs diff --git a/crates/service/storyteller_web/src/database/helpers/enums.rs b/crates/service/storyteller_web/src/database/helpers/enums.rs deleted file mode 100644 index 71f915ab4b..0000000000 --- a/crates/service/storyteller_web/src/database/helpers/enums.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! Common enums in the database. -//! These may be used in one or more tables. -//! This defines the expected serializations. - -#![allow(non_camel_case_types)] - -#[deprecated(note = "Use `RecordVisibility` instead!")] -#[derive(Deserialize)] -pub enum CreatorSetVisibility { - /// public - #[deprecated(note = "Use `RecordVisibility` instead!")] - Public, - /// hidden - #[deprecated(note = "Use `RecordVisibility` instead!")] - Hidden, - /// private - #[deprecated(note = "Use `RecordVisibility` instead!")] - Private, -} - -#[derive(Deserialize)] -pub enum TtsModelType { - /// tacotron2 - Tacotron2, - /// glowtts - GlowTts, - /// glowtts-vocodes - GlowTts_Vocodes, - /// talknet - Talknet, -} - -#[derive(Deserialize)] -pub enum DownloadUrlType { - /// google-drive - Google_Drive, - /// web - Web, -} - -#[derive(Deserialize)] -pub enum W2lTemplateType { - /// unknown - Unknown, - /// video - Video, - /// image - Image, -} diff --git a/crates/service/storyteller_web/src/database/helpers/mod.rs b/crates/service/storyteller_web/src/database/helpers/mod.rs deleted file mode 100644 index 35b3598e87..0000000000 --- a/crates/service/storyteller_web/src/database/helpers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod enums; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/mod.rs b/crates/service/storyteller_web/src/database/mod.rs index 5cfe9b18f8..c101f589f6 100644 --- a/crates/service/storyteller_web/src/database/mod.rs +++ b/crates/service/storyteller_web/src/database/mod.rs @@ -1,4 +1,3 @@ -pub mod helpers; pub mod mediators; pub mod queries; pub mod query_builders; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 4589f7be25..8f295094c8 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -5,7 +5,6 @@ use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{DownloadUrlType, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index c775b463c2..877b9f9cd0 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -1,11 +1,10 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{TtsModelType, CreatorSetVisibility}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -13,14 +12,37 @@ use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use database_queries::tokens::Tokens; -use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; +#[derive(Deserialize)] +pub enum TtsModelType { + /// tacotron2 + Tacotron2, + /// glowtts + GlowTts, + /// glowtts-vocodes + GlowTts_Vocodes, + /// talknet + Talknet, +} + +#[deprecated(note = "Use `RecordVisibility` instead!")] +#[derive(Deserialize)] +pub enum CreatorSetVisibility { + /// public + Public, + /// hidden + Hidden, + /// private + Private, +} + #[derive(Deserialize)] pub struct UploadTtsModelRequest { idempotency_token: String, @@ -37,7 +59,7 @@ pub struct UploadTtsModelSuccessResponse { pub job_token: String, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum UploadTtsModelError { BadInput(String), MustBeLoggedIn, @@ -67,6 +89,13 @@ impl ResponseError for UploadTtsModelError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for UploadTtsModelError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn upload_tts_model_handler( http_request: HttpRequest, request: web::Json, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index 559d7e90df..978a5dc14f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -5,7 +5,6 @@ use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{DownloadUrlType, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index cd9e4151f4..a6b42301a0 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -5,7 +5,6 @@ use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::helpers::enums::{CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -22,6 +21,27 @@ use std::fmt; use std::sync::Arc; use user_input_common::check_for_slurs::contains_slurs; +#[derive(Deserialize)] +pub enum W2lTemplateType { + /// unknown + Unknown, + /// video + Video, + /// image + Image, +} + +#[deprecated(note = "Use `RecordVisibility` instead!")] +#[derive(Deserialize)] +pub enum CreatorSetVisibility { + /// public + Public, + /// hidden + Hidden, + /// private + Private, +} + #[derive(Deserialize)] pub struct UploadW2lTemplateRequest { idempotency_token: String, From 17ff619f79f73b3917f98ace12832b8fc616f51d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 01:52:55 -0500 Subject: [PATCH 0947/1042] continued refactor into database_queries lib --- crates/lib/database_queries/src/w2l/mod.rs | 5 +++- .../src/w2l/w2l_inference_jobs/mod.rs | 0 .../delete_w2l_result_various_scopes.rs} | 0 ...ist_w2l_inference_results_query_builder.rs | 4 ++-- .../src/w2l/w2l_results/mod.rs | 3 +++ .../src/w2l/w2l_results}/query_w2l_result.rs | 8 +++---- .../src/w2l/w2l_templates/mod.rs | 0 .../src/database/queries/mod.rs | 2 -- .../src/database/query_builders/mod.rs | 3 +-- .../users/list_user_w2l_inference_results.rs | 17 ++++++++++---- .../endpoints/w2l/delete_w2l_result.rs | 23 ++++++++++++------- .../endpoints/w2l/edit_w2l_result.rs | 2 +- .../endpoints/w2l/get_w2l_result.rs | 18 ++++++++++----- 13 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 crates/lib/database_queries/src/w2l/w2l_inference_jobs/mod.rs rename crates/{service/storyteller_web/src/database/queries/delete_w2l_result.rs => lib/database_queries/src/w2l/w2l_results/delete_w2l_result_various_scopes.rs} (100%) rename crates/{service/storyteller_web/src/database/query_builders => lib/database_queries/src/w2l/w2l_results}/list_w2l_inference_results_query_builder.rs (99%) create mode 100644 crates/lib/database_queries/src/w2l/w2l_results/mod.rs rename crates/{service/storyteller_web/src/database/queries => lib/database_queries/src/w2l/w2l_results}/query_w2l_result.rs (97%) create mode 100644 crates/lib/database_queries/src/w2l/w2l_templates/mod.rs diff --git a/crates/lib/database_queries/src/w2l/mod.rs b/crates/lib/database_queries/src/w2l/mod.rs index fedcafddb7..9cd08d628b 100644 --- a/crates/lib/database_queries/src/w2l/mod.rs +++ b/crates/lib/database_queries/src/w2l/mod.rs @@ -1 +1,4 @@ -pub mod stats; \ No newline at end of file +pub mod stats; +pub mod w2l_inference_jobs; +pub mod w2l_results; +pub mod w2l_templates; \ No newline at end of file diff --git a/crates/lib/database_queries/src/w2l/w2l_inference_jobs/mod.rs b/crates/lib/database_queries/src/w2l/w2l_inference_jobs/mod.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/storyteller_web/src/database/queries/delete_w2l_result.rs b/crates/lib/database_queries/src/w2l/w2l_results/delete_w2l_result_various_scopes.rs similarity index 100% rename from crates/service/storyteller_web/src/database/queries/delete_w2l_result.rs rename to crates/lib/database_queries/src/w2l/w2l_results/delete_w2l_result_various_scopes.rs diff --git a/crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs b/crates/lib/database_queries/src/w2l/w2l_results/list_w2l_inference_results_query_builder.rs similarity index 99% rename from crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs rename to crates/lib/database_queries/src/w2l/w2l_results/list_w2l_inference_results_query_builder.rs index 2d3b31b66d..252b213e86 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_w2l_inference_results_query_builder.rs +++ b/crates/lib/database_queries/src/w2l/w2l_results/list_w2l_inference_results_query_builder.rs @@ -2,7 +2,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use container_common::anyhow_result::AnyhowResult; -use database_queries::column_types::record_visibility::RecordVisibility; +use crate::column_types::record_visibility::RecordVisibility; use log::{warn, info}; use sqlx::MySqlPool; @@ -372,7 +372,7 @@ pub struct RawInternalTtsRecord { #[cfg(test)] mod tests { - use crate::database::query_builders::list_w2l_inference_results_query_builder::ListW2lResultsQueryBuilder; + use crate::w2l::w2l_results::list_w2l_inference_results_query_builder::ListW2lResultsQueryBuilder; #[test] fn predicates_without_scoping() { diff --git a/crates/lib/database_queries/src/w2l/w2l_results/mod.rs b/crates/lib/database_queries/src/w2l/w2l_results/mod.rs new file mode 100644 index 0000000000..8189a79f96 --- /dev/null +++ b/crates/lib/database_queries/src/w2l/w2l_results/mod.rs @@ -0,0 +1,3 @@ +pub mod delete_w2l_result_various_scopes; +pub mod list_w2l_inference_results_query_builder; +pub mod query_w2l_result; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/query_w2l_result.rs b/crates/lib/database_queries/src/w2l/w2l_results/query_w2l_result.rs similarity index 97% rename from crates/service/storyteller_web/src/database/queries/query_w2l_result.rs rename to crates/lib/database_queries/src/w2l/w2l_results/query_w2l_result.rs index e3e5cd19f3..2d8339d6ce 100644 --- a/crates/service/storyteller_web/src/database/queries/query_w2l_result.rs +++ b/crates/lib/database_queries/src/w2l/w2l_results/query_w2l_result.rs @@ -1,15 +1,13 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use crate::AnyhowResult; -use database_queries::column_types::record_visibility::RecordVisibility; -use database_queries::helpers::boolean_converters::nullable_i8_to_bool; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::record_visibility::RecordVisibility; +use crate::helpers::boolean_converters::nullable_i8_to_bool; use log::{info, warn, log}; -use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; -use std::sync::Arc; #[derive(Serialize)] pub struct W2lResultRecordForResponse { diff --git a/crates/lib/database_queries/src/w2l/w2l_templates/mod.rs b/crates/lib/database_queries/src/w2l/w2l_templates/mod.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/storyteller_web/src/database/queries/mod.rs b/crates/service/storyteller_web/src/database/queries/mod.rs index 528eebbe5d..5a0f2337f2 100644 --- a/crates/service/storyteller_web/src/database/queries/mod.rs +++ b/crates/service/storyteller_web/src/database/queries/mod.rs @@ -1,12 +1,10 @@ pub mod badges; pub mod categories; pub mod create_session; -pub mod delete_w2l_result; pub mod delete_w2l_template; pub mod get_user_profile_by_username; pub mod list_ip_bans; pub mod list_user_badges; pub mod list_user_roles; pub mod list_w2l_templates; -pub mod query_w2l_result; pub mod query_w2l_template; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/query_builders/mod.rs b/crates/service/storyteller_web/src/database/query_builders/mod.rs index 55cc079dcc..f3d5ea05aa 100644 --- a/crates/service/storyteller_web/src/database/query_builders/mod.rs +++ b/crates/service/storyteller_web/src/database/query_builders/mod.rs @@ -1,3 +1,2 @@ pub mod list_categories_query_builder; -pub mod list_users_query_builder; -pub mod list_w2l_inference_results_query_builder; \ No newline at end of file +pub mod list_users_query_builder; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs index 21e032853d..ae48e13259 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs @@ -1,24 +1,24 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Query}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::query_builders::list_w2l_inference_results_query_builder::ListW2lResultsQueryBuilder; -use crate::database::query_builders::list_w2l_inference_results_query_builder::W2lInferenceRecordForList; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; +use database_queries::w2l::w2l_results::list_w2l_inference_results_query_builder::ListW2lResultsQueryBuilder; +use database_queries::w2l::w2l_results::list_w2l_inference_results_query_builder::W2lInferenceRecordForList; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -43,7 +43,7 @@ pub struct ListW2lInferenceResultsForUserSuccessResponse { pub cursor_previous: Option, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum ListW2lInferenceResultsForUserError { ServerError, } @@ -64,6 +64,13 @@ impl ResponseError for ListW2lInferenceResultsForUserError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListW2lInferenceResultsForUserError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn list_user_w2l_inference_results_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs index 8eba2b589b..6cf259aa9c 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs @@ -1,16 +1,11 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::delete_w2l_result::delete_w2l_inference_result_as_mod; -use crate::database::queries::delete_w2l_result::delete_w2l_inference_result_as_user; -use crate::database::queries::delete_w2l_result::undelete_w2l_inference_result_as_mod; -use crate::database::queries::delete_w2l_result::undelete_w2l_inference_result_as_user; -use crate::database::queries::query_w2l_result::select_w2l_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -20,13 +15,18 @@ use crate::util::delete_role_disambiguation::delete_role_disambiguation; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use derive_more::{Display, Error}; +use database_queries::w2l::w2l_results::delete_w2l_result_various_scopes::delete_w2l_inference_result_as_mod; +use database_queries::w2l::w2l_results::delete_w2l_result_various_scopes::delete_w2l_inference_result_as_user; +use database_queries::w2l::w2l_results::delete_w2l_result_various_scopes::undelete_w2l_inference_result_as_mod; +use database_queries::w2l::w2l_results::delete_w2l_result_various_scopes::undelete_w2l_inference_result_as_user; +use database_queries::w2l::w2l_results::query_w2l_result::select_w2l_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -42,7 +42,7 @@ pub struct DeleteW2lInferenceResultRequest { as_mod: Option, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum DeleteW2lInferenceResultError { BadInput(String), NotAuthorized, @@ -72,6 +72,13 @@ impl ResponseError for DeleteW2lInferenceResultError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for DeleteW2lInferenceResultError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn delete_w2l_inference_result_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs index 2a90ef0a93..00b8713b5d 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::query_w2l_result::select_w2l_result_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -14,6 +13,7 @@ use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; use database_queries::column_types::record_visibility::RecordVisibility; +use database_queries::w2l::w2l_results::query_w2l_result::select_w2l_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs index f5b19d8eff..cd0bb254e4 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -1,24 +1,23 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; -use crate::AnyhowResult; -use crate::database::queries::query_w2l_result::W2lResultRecordForResponse; -use crate::database::queries::query_w2l_result::select_w2l_result_by_token; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; +use database_queries::w2l::w2l_results::query_w2l_result::W2lResultRecordForResponse; +use database_queries::w2l::w2l_results::query_w2l_result::select_w2l_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -33,7 +32,7 @@ pub struct GetW2lResultSuccessResponse { pub result: W2lResultRecordForResponse, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum GetW2lResultError { ServerError, NotFound, @@ -57,6 +56,13 @@ impl ResponseError for GetW2lResultError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GetW2lResultError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn get_w2l_inference_result_handler( http_request: HttpRequest, path: Path, From 8854366c1226eb6809d9d8080ac73860d2a4b2a9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 02:05:43 -0500 Subject: [PATCH 0948/1042] more refactor into common lib --- .../delete_w2l_template_various_scopes.rs} | 0 .../w2l/w2l_templates/get_w2l_template.rs} | 11 +++++----- .../src/w2l/w2l_templates/mod.rs | 2 ++ .../src/database/queries/mod.rs | 4 +--- .../endpoints/api_tokens/create_api_token.rs | 1 - .../endpoints/api_tokens/list_api_tokens.rs | 1 - .../categories/assign_tts_category.rs | 1 - .../endpoints/categories/create_category.rs | 1 - .../endpoints/categories/get_category.rs | 1 - .../endpoints/misc/detect_locale_handler.rs | 1 - .../endpoints/w2l/delete_w2l_template.rs | 21 ++++++++++++------- .../endpoints/w2l/edit_w2l_template.rs | 2 +- .../endpoints/w2l/get_w2l_template.rs | 5 ++--- .../w2l/set_w2l_template_mod_approval.rs | 15 +++++++++---- 14 files changed, 37 insertions(+), 29 deletions(-) rename crates/{service/storyteller_web/src/database/queries/delete_w2l_template.rs => lib/database_queries/src/w2l/w2l_templates/delete_w2l_template_various_scopes.rs} (100%) rename crates/{service/storyteller_web/src/database/queries/query_w2l_template.rs => lib/database_queries/src/w2l/w2l_templates/get_w2l_template.rs} (95%) diff --git a/crates/service/storyteller_web/src/database/queries/delete_w2l_template.rs b/crates/lib/database_queries/src/w2l/w2l_templates/delete_w2l_template_various_scopes.rs similarity index 100% rename from crates/service/storyteller_web/src/database/queries/delete_w2l_template.rs rename to crates/lib/database_queries/src/w2l/w2l_templates/delete_w2l_template_various_scopes.rs diff --git a/crates/service/storyteller_web/src/database/queries/query_w2l_template.rs b/crates/lib/database_queries/src/w2l/w2l_templates/get_w2l_template.rs similarity index 95% rename from crates/service/storyteller_web/src/database/queries/query_w2l_template.rs rename to crates/lib/database_queries/src/w2l/w2l_templates/get_w2l_template.rs index 96d095fd78..3c9df4abb3 100644 --- a/crates/service/storyteller_web/src/database/queries/query_w2l_template.rs +++ b/crates/lib/database_queries/src/w2l/w2l_templates/get_w2l_template.rs @@ -1,17 +1,18 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use crate::AnyhowResult; -use database_queries::column_types::record_visibility::RecordVisibility; -use database_queries::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; -use derive_more::{Display, Error}; +use container_common::anyhow_result::AnyhowResult; +use crate::column_types::record_visibility::RecordVisibility; +use crate::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; use log::{info, warn, log}; -use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +// FIXME: This is the old style of query scoping and shouldn't be copied. +// The moderator-only fields are good practice, though. + #[derive(Serialize)] pub struct W2lTemplateRecordForResponse { pub template_token: String, diff --git a/crates/lib/database_queries/src/w2l/w2l_templates/mod.rs b/crates/lib/database_queries/src/w2l/w2l_templates/mod.rs index e69de29bb2..53e519b560 100644 --- a/crates/lib/database_queries/src/w2l/w2l_templates/mod.rs +++ b/crates/lib/database_queries/src/w2l/w2l_templates/mod.rs @@ -0,0 +1,2 @@ +pub mod delete_w2l_template_various_scopes; +pub mod get_w2l_template; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/mod.rs b/crates/service/storyteller_web/src/database/queries/mod.rs index 5a0f2337f2..7cf0869e6f 100644 --- a/crates/service/storyteller_web/src/database/queries/mod.rs +++ b/crates/service/storyteller_web/src/database/queries/mod.rs @@ -1,10 +1,8 @@ pub mod badges; pub mod categories; pub mod create_session; -pub mod delete_w2l_template; pub mod get_user_profile_by_username; pub mod list_ip_bans; pub mod list_user_badges; pub mod list_user_roles; -pub mod list_w2l_templates; -pub mod query_w2l_template; \ No newline at end of file +pub mod list_w2l_templates; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs index 476fbcaf26..f5f1d82d21 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs index 2df591a159..d7e57c0c83 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs index a439623ae5..8a65e10444 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs @@ -8,7 +8,6 @@ use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use anyhow::anyhow; use crate::database::queries::categories::get_category_by_token::get_category_by_token; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs index 82c0ed8740..bb2f9f8d54 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/create_category.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs index 225a10dfa6..15b652e69a 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs @@ -8,7 +8,6 @@ use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; use crate::database::queries::categories::get_category_by_token::get_category_by_token; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs index 394dcb9f87..49322e718f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs @@ -8,7 +8,6 @@ use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; use crate::database::queries::categories::get_category_by_token::get_category_by_token; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs index 99cd94fe27..6ed18dd417 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -6,11 +6,11 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::delete_w2l_template::delete_w2l_template_as_mod; -use crate::database::queries::delete_w2l_template::delete_w2l_template_as_user; -use crate::database::queries::delete_w2l_template::undelete_w2l_template_as_mod; -use crate::database::queries::delete_w2l_template::undelete_w2l_template_as_user; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; +use database_queries::w2l::w2l_templates::delete_w2l_template_various_scopes::delete_w2l_template_as_mod; +use database_queries::w2l::w2l_templates::delete_w2l_template_various_scopes::delete_w2l_template_as_user; +use database_queries::w2l::w2l_templates::delete_w2l_template_various_scopes::undelete_w2l_template_as_mod; +use database_queries::w2l::w2l_templates::delete_w2l_template_various_scopes::undelete_w2l_template_as_user; +use database_queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -20,8 +20,8 @@ use crate::util::delete_role_disambiguation::delete_role_disambiguation; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use derive_more::{Display, Error}; use log::{info, warn, log}; +use std::fmt; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; @@ -42,7 +42,7 @@ pub struct DeleteW2lTemplateRequest { as_mod: Option, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum DeleteW2lTemplateError { BadInput(String), NotAuthorized, @@ -72,6 +72,13 @@ impl ResponseError for DeleteW2lTemplateError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for DeleteW2lTemplateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn delete_w2l_template_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs index 355e62faad..254b31979d 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -14,6 +13,7 @@ use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; use database_queries::column_types::record_visibility::RecordVisibility; +use database_queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs index 29e416d29f..41c042a24f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -7,14 +7,13 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; -use crate::AnyhowResult; -use crate::database::queries::query_w2l_template::W2lTemplateRecordForResponse; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; +use database_queries::w2l::w2l_templates::get_w2l_template::W2lTemplateRecordForResponse; +use database_queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs index 677f788432..88584e63aa 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs @@ -1,24 +1,24 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::query_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use derive_more::{Display, Error}; +use database_queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -37,7 +37,7 @@ pub struct SetW2lTemplateModApprovalSuccessResponse { pub success: bool, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum SetW2lTemplateModApprovalError { BadInput(String), NotAuthorized, @@ -67,6 +67,13 @@ impl ResponseError for SetW2lTemplateModApprovalError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for SetW2lTemplateModApprovalError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn set_w2l_template_mod_approval_handler( http_request: HttpRequest, path: Path, From c9b2d3ea10b52603b0e963958efa04726fc0c96e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 02:25:11 -0500 Subject: [PATCH 0949/1042] continued refactor of db queries into lib --- crates/lib/database_queries/src/lib.rs | 1 + .../model_categories}/get_category_by_token.rs | 4 ++-- .../list_categories_query_builder.rs | 4 ++-- .../database_queries/src/model_categories/mod.rs | 2 ++ .../src/w2l/w2l_inference_jobs/mod.rs | 1 + .../src/w2l/w2l_templates}/list_w2l_templates.rs | 4 +++- .../database_queries/src/w2l/w2l_templates/mod.rs | 3 ++- .../src/database/queries/categories/mod.rs | 1 - .../storyteller_web/src/database/queries/mod.rs | 4 +--- .../src/database/query_builders/mod.rs | 1 - .../endpoints/api_tokens/delete_api_token.rs | 1 - .../endpoints/api_tokens/edit_api_token.rs | 1 - .../endpoints/categories/assign_tts_category.rs | 2 +- .../endpoints/categories/get_category.rs | 3 +-- .../endpoints/categories/list_tts_categories.rs | 2 +- .../endpoints/misc/detect_locale_handler.rs | 2 -- .../moderation/categories/edit_category.rs | 2 +- .../list_tts_categories_for_moderation.rs | 2 +- .../endpoints/users/list_user_w2l_templates.rs | 15 +++++++++++---- .../endpoints/w2l/list_w2l_templates.rs | 15 +++++++++++---- 20 files changed, 41 insertions(+), 29 deletions(-) rename crates/{service/storyteller_web/src/database/queries/categories => lib/database_queries/src/model_categories}/get_category_by_token.rs (97%) rename crates/{service/storyteller_web/src/database/query_builders => lib/database_queries/src/model_categories}/list_categories_query_builder.rs (98%) create mode 100644 crates/lib/database_queries/src/model_categories/mod.rs rename crates/{service/storyteller_web/src/database/queries => lib/database_queries/src/w2l/w2l_templates}/list_w2l_templates.rs (98%) delete mode 100644 crates/service/storyteller_web/src/database/queries/categories/mod.rs diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index 6f01cc9da8..a393f3c05b 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -5,6 +5,7 @@ mod builders; pub mod api_tokens; pub mod column_types; pub mod helpers; +pub mod model_categories; pub mod tokens; pub mod tts; pub mod twitch_event_rules; diff --git a/crates/service/storyteller_web/src/database/queries/categories/get_category_by_token.rs b/crates/lib/database_queries/src/model_categories/get_category_by_token.rs similarity index 97% rename from crates/service/storyteller_web/src/database/queries/categories/get_category_by_token.rs rename to crates/lib/database_queries/src/model_categories/get_category_by_token.rs index fa7aeaa9a4..5d5b71b0d0 100644 --- a/crates/service/storyteller_web/src/database/queries/categories/get_category_by_token.rs +++ b/crates/lib/database_queries/src/model_categories/get_category_by_token.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use crate::AnyhowResult; -use database_queries::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; +use container_common::anyhow_result::AnyhowResult; +use crate::helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; use log::{info, warn, log}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs b/crates/lib/database_queries/src/model_categories/list_categories_query_builder.rs similarity index 98% rename from crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs rename to crates/lib/database_queries/src/model_categories/list_categories_query_builder.rs index 1b52af54c1..34c69f8b13 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_categories_query_builder.rs +++ b/crates/lib/database_queries/src/model_categories/list_categories_query_builder.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; -use database_queries::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; +use crate::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; use log::{warn, info}; use sqlx::MySqlPool; @@ -318,7 +318,7 @@ pub struct RawInternalCategoryRecord { #[cfg(test)] mod tests { - use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; + use crate::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder; #[test] fn predicates_default_scoping() { diff --git a/crates/lib/database_queries/src/model_categories/mod.rs b/crates/lib/database_queries/src/model_categories/mod.rs new file mode 100644 index 0000000000..f00a822919 --- /dev/null +++ b/crates/lib/database_queries/src/model_categories/mod.rs @@ -0,0 +1,2 @@ +pub mod get_category_by_token; +pub mod list_categories_query_builder; \ No newline at end of file diff --git a/crates/lib/database_queries/src/w2l/w2l_inference_jobs/mod.rs b/crates/lib/database_queries/src/w2l/w2l_inference_jobs/mod.rs index e69de29bb2..4e398c43ff 100644 --- a/crates/lib/database_queries/src/w2l/w2l_inference_jobs/mod.rs +++ b/crates/lib/database_queries/src/w2l/w2l_inference_jobs/mod.rs @@ -0,0 +1 @@ +// TODO: Move the "job queries" here and break them down into several files. \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs b/crates/lib/database_queries/src/w2l/w2l_templates/list_w2l_templates.rs similarity index 98% rename from crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs rename to crates/lib/database_queries/src/w2l/w2l_templates/list_w2l_templates.rs index cd0dd17349..0ce308eba7 100644 --- a/crates/service/storyteller_web/src/database/queries/list_w2l_templates.rs +++ b/crates/lib/database_queries/src/w2l/w2l_templates/list_w2l_templates.rs @@ -1,9 +1,11 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; +use crate::helpers::boolean_converters::nullable_i8_to_optional_bool; use log::{warn, info}; use sqlx::MySqlPool; -use database_queries::helpers::boolean_converters::nullable_i8_to_optional_bool; + +// FIXME: This is the old style of query scoping and shouldn't be copied. #[derive(Serialize)] pub struct W2lTemplateRecordForList { diff --git a/crates/lib/database_queries/src/w2l/w2l_templates/mod.rs b/crates/lib/database_queries/src/w2l/w2l_templates/mod.rs index 53e519b560..d6ae263916 100644 --- a/crates/lib/database_queries/src/w2l/w2l_templates/mod.rs +++ b/crates/lib/database_queries/src/w2l/w2l_templates/mod.rs @@ -1,2 +1,3 @@ pub mod delete_w2l_template_various_scopes; -pub mod get_w2l_template; \ No newline at end of file +pub mod get_w2l_template; +pub mod list_w2l_templates; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/categories/mod.rs b/crates/service/storyteller_web/src/database/queries/categories/mod.rs deleted file mode 100644 index 8012259411..0000000000 --- a/crates/service/storyteller_web/src/database/queries/categories/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod get_category_by_token; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/mod.rs b/crates/service/storyteller_web/src/database/queries/mod.rs index 7cf0869e6f..e7d90bdf2c 100644 --- a/crates/service/storyteller_web/src/database/queries/mod.rs +++ b/crates/service/storyteller_web/src/database/queries/mod.rs @@ -1,8 +1,6 @@ pub mod badges; -pub mod categories; pub mod create_session; pub mod get_user_profile_by_username; pub mod list_ip_bans; pub mod list_user_badges; -pub mod list_user_roles; -pub mod list_w2l_templates; \ No newline at end of file +pub mod list_user_roles; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/query_builders/mod.rs b/crates/service/storyteller_web/src/database/query_builders/mod.rs index f3d5ea05aa..32052e5d3d 100644 --- a/crates/service/storyteller_web/src/database/query_builders/mod.rs +++ b/crates/service/storyteller_web/src/database/query_builders/mod.rs @@ -1,2 +1 @@ -pub mod list_categories_query_builder; pub mod list_users_query_builder; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs index 3010c2744c..0ecb7713f4 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::categories::get_category_by_token::get_category_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs index 2aedf13e84..7454753195 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs @@ -18,7 +18,6 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; -use crate::database::queries::categories::get_category_by_token::get_category_by_token; use database_queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; use database_queries::api_tokens::edit_api_token::edit_api_token; use crate::http_server::web_utils::ip_address::get_request_ip; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs index 8a65e10444..155d9031d1 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs @@ -7,13 +7,13 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use anyhow::anyhow; -use crate::database::queries::categories::get_category_by_token::get_category_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; +use database_queries::model_categories::get_category_by_token::get_category_by_token; use database_queries::tts::tts_models::get_tts_model::get_tts_model_by_token; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs index 15b652e69a..e134e89919 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs @@ -7,14 +7,13 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::queries::categories::get_category_by_token::get_category_by_token; -use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; +use database_queries::model_categories::get_category_by_token::get_category_by_token; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs index 160171cb4c..10b969fef5 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs @@ -7,13 +7,13 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; +use database_queries::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use lexical_sort::natural_lexical_cmp; use log::{info, warn, log}; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs index 49322e718f..d1aecf561b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs @@ -7,8 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::queries::categories::get_category_by_token::get_category_by_token; -use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs index 87d61be31d..76cdd7eb76 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs @@ -6,13 +6,13 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::categories::get_category_by_token::get_category_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; +use database_queries::model_categories::get_category_by_token::get_category_by_token; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs index 09cd85fad5..fb4d21fe98 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs @@ -7,13 +7,13 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::query_builders::list_categories_query_builder::ListCategoriesQueryBuilder; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; +use database_queries::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs index 071e91af1b..399cae9737 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs @@ -1,23 +1,23 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::queries::list_w2l_templates::{W2lTemplateRecordForList, list_w2l_templates}; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; +use database_queries::w2l::w2l_templates::list_w2l_templates::{W2lTemplateRecordForList, list_w2l_templates}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -32,7 +32,7 @@ pub struct ListW2lTemplatesForUserSuccessResponse { pub templates: Vec, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum ListW2lTemplatesForUserError { ServerError, } @@ -53,6 +53,13 @@ impl ResponseError for ListW2lTemplatesForUserError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListW2lTemplatesForUserError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn list_user_w2l_templates_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs index 83b087acdf..2ff15bc617 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs @@ -1,22 +1,22 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::queries::list_w2l_templates::{list_w2l_templates, W2lTemplateRecordForList}; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; +use database_queries::w2l::w2l_templates::list_w2l_templates::{W2lTemplateRecordForList, list_w2l_templates}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; #[derive(Serialize)] @@ -25,7 +25,7 @@ pub struct ListW2lTemplatesSuccessResponse { pub templates: Vec, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum ListW2lTemplatesError { ServerError, } @@ -46,6 +46,13 @@ impl ResponseError for ListW2lTemplatesError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListW2lTemplatesError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn list_w2l_templates_handler( http_request: HttpRequest, server_state: web::Data>) -> Result From 20a6839eafdff7670be47ecf572755a636311e85 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 02:45:41 -0500 Subject: [PATCH 0950/1042] more refactor into mysql module --- crates/lib/database_queries/src/lib.rs | 1 + .../src/users}/get_user_profile_by_username.rs | 7 ++----- .../src/users}/list_users_query_builder.rs | 4 ++-- crates/lib/database_queries/src/users/mod.rs | 4 ++++ .../src/users/user_badges}/list_user_badges.rs | 12 ++++++------ .../src/users/user_badges/mod.rs | 1 + .../src/users/user_roles}/list_user_roles.rs | 4 ++-- .../src/users/user_roles/mod.rs | 1 + .../service/storyteller_web/src/database/mod.rs | 3 +-- .../storyteller_web/src/database/queries/mod.rs | 5 +---- .../src/database/query_builders/mod.rs | 1 - .../endpoints/misc/detect_locale_handler.rs | 2 +- .../endpoints/moderation/user_bans/ban_user.rs | 5 ++--- .../moderation/user_bans/list_banned_users.rs | 2 -- .../moderation/user_roles/list_roles.rs | 16 +++++++++++----- .../moderation/user_roles/set_user_role.rs | 17 ++++++++++++----- .../endpoints/moderation/users/list_users.rs | 6 ++---- .../http_server/endpoints/users/edit_profile.rs | 2 +- .../http_server/endpoints/users/get_profile.rs | 6 +++--- 19 files changed, 53 insertions(+), 46 deletions(-) rename crates/{service/storyteller_web/src/database/queries => lib/database_queries/src/users}/get_user_profile_by_username.rs (95%) rename crates/{service/storyteller_web/src/database/query_builders => lib/database_queries/src/users}/list_users_query_builder.rs (97%) create mode 100644 crates/lib/database_queries/src/users/mod.rs rename crates/{service/storyteller_web/src/database/queries => lib/database_queries/src/users/user_badges}/list_user_badges.rs (85%) create mode 100644 crates/lib/database_queries/src/users/user_badges/mod.rs rename crates/{service/storyteller_web/src/database/queries => lib/database_queries/src/users/user_roles}/list_user_roles.rs (98%) create mode 100644 crates/lib/database_queries/src/users/user_roles/mod.rs delete mode 100644 crates/service/storyteller_web/src/database/query_builders/mod.rs diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index a393f3c05b..891437aa3d 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -11,5 +11,6 @@ pub mod tts; pub mod twitch_event_rules; pub mod twitch_oauth; pub mod twitch_pubsub; +pub mod users; pub mod voice_clone_requests; pub mod w2l; diff --git a/crates/service/storyteller_web/src/database/queries/get_user_profile_by_username.rs b/crates/lib/database_queries/src/users/get_user_profile_by_username.rs similarity index 95% rename from crates/service/storyteller_web/src/database/queries/get_user_profile_by_username.rs rename to crates/lib/database_queries/src/users/get_user_profile_by_username.rs index 8fc2fd7cdd..f7c739d4a4 100644 --- a/crates/service/storyteller_web/src/database/queries/get_user_profile_by_username.rs +++ b/crates/lib/database_queries/src/users/get_user_profile_by_username.rs @@ -1,11 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use crate::AnyhowResult; -use database_queries::helpers::boolean_converters::i8_to_bool; -use derive_more::{Display, Error}; +use container_common::anyhow_result::AnyhowResult; +use crate::helpers::boolean_converters::i8_to_bool; use log::{info, warn, log}; -use md5::{Md5, Digest}; -use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; diff --git a/crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs b/crates/lib/database_queries/src/users/list_users_query_builder.rs similarity index 97% rename from crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs rename to crates/lib/database_queries/src/users/list_users_query_builder.rs index 4e55c62536..6c40f62bc0 100644 --- a/crates/service/storyteller_web/src/database/query_builders/list_users_query_builder.rs +++ b/crates/lib/database_queries/src/users/list_users_query_builder.rs @@ -1,6 +1,6 @@ use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; -use database_queries::helpers::boolean_converters::i8_to_bool; +use crate::helpers::boolean_converters::i8_to_bool; use sqlx::MySqlPool; /// NB: This is not to be shared externally. @@ -137,7 +137,7 @@ FROM users } #[derive(sqlx::FromRow)] -pub struct UserForListRaw { +struct UserForListRaw { pub user_id: i64, pub user_token: String, diff --git a/crates/lib/database_queries/src/users/mod.rs b/crates/lib/database_queries/src/users/mod.rs new file mode 100644 index 0000000000..2327971b74 --- /dev/null +++ b/crates/lib/database_queries/src/users/mod.rs @@ -0,0 +1,4 @@ +pub mod get_user_profile_by_username; +pub mod list_users_query_builder; +pub mod user_badges; +pub mod user_roles; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/list_user_badges.rs b/crates/lib/database_queries/src/users/user_badges/list_user_badges.rs similarity index 85% rename from crates/service/storyteller_web/src/database/queries/list_user_badges.rs rename to crates/lib/database_queries/src/users/user_badges/list_user_badges.rs index 7863bbe3c2..80769eb9a0 100644 --- a/crates/service/storyteller_web/src/database/queries/list_user_badges.rs +++ b/crates/lib/database_queries/src/users/user_badges/list_user_badges.rs @@ -13,7 +13,7 @@ pub struct UserBadgeForList { pub granted_at: DateTime, } -pub struct RawDbUserBadgeForList { +struct RawDbUserBadgeForList { slug: String, title: String, description: String, @@ -65,11 +65,11 @@ WHERE Ok(user_badges.into_iter() .map(|badge| { UserBadgeForList { - slug: badge.slug.clone(), - title: badge.title.clone(), - description: badge.description.clone(), - image_url: badge.image_url.clone(), - granted_at: badge.user_created_at.clone(), + slug: badge.slug, + title: badge.title, + description: badge.description, + image_url: badge.image_url, + granted_at: badge.user_created_at, } }) .collect::>()) diff --git a/crates/lib/database_queries/src/users/user_badges/mod.rs b/crates/lib/database_queries/src/users/user_badges/mod.rs new file mode 100644 index 0000000000..cd1a70c4b8 --- /dev/null +++ b/crates/lib/database_queries/src/users/user_badges/mod.rs @@ -0,0 +1 @@ +pub mod list_user_badges; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/list_user_roles.rs b/crates/lib/database_queries/src/users/user_roles/list_user_roles.rs similarity index 98% rename from crates/service/storyteller_web/src/database/queries/list_user_roles.rs rename to crates/lib/database_queries/src/users/user_roles/list_user_roles.rs index 4528c3809c..652c5402dc 100644 --- a/crates/service/storyteller_web/src/database/queries/list_user_roles.rs +++ b/crates/lib/database_queries/src/users/user_roles/list_user_roles.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use container_common::anyhow_result::AnyhowResult; -use database_queries::helpers::boolean_converters::i8_to_bool; +use crate::helpers::boolean_converters::i8_to_bool; use log::{warn, info}; use sqlx::MySqlPool; @@ -36,7 +36,7 @@ pub struct UserRoleForList { pub updated_at : DateTime, } -pub struct RawDbUserRoleForList { +struct RawDbUserRoleForList { slug: String, name: String, diff --git a/crates/lib/database_queries/src/users/user_roles/mod.rs b/crates/lib/database_queries/src/users/user_roles/mod.rs new file mode 100644 index 0000000000..e227b96ba9 --- /dev/null +++ b/crates/lib/database_queries/src/users/user_roles/mod.rs @@ -0,0 +1 @@ +pub mod list_user_roles; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/mod.rs b/crates/service/storyteller_web/src/database/mod.rs index c101f589f6..1c04fcc19f 100644 --- a/crates/service/storyteller_web/src/database/mod.rs +++ b/crates/service/storyteller_web/src/database/mod.rs @@ -1,3 +1,2 @@ pub mod mediators; -pub mod queries; -pub mod query_builders; \ No newline at end of file +pub mod queries; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/mod.rs b/crates/service/storyteller_web/src/database/queries/mod.rs index e7d90bdf2c..7b1b03af01 100644 --- a/crates/service/storyteller_web/src/database/queries/mod.rs +++ b/crates/service/storyteller_web/src/database/queries/mod.rs @@ -1,6 +1,3 @@ pub mod badges; pub mod create_session; -pub mod get_user_profile_by_username; -pub mod list_ip_bans; -pub mod list_user_badges; -pub mod list_user_roles; \ No newline at end of file +pub mod list_ip_bans; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/query_builders/mod.rs b/crates/service/storyteller_web/src/database/query_builders/mod.rs deleted file mode 100644 index 32052e5d3d..0000000000 --- a/crates/service/storyteller_web/src/database/query_builders/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod list_users_query_builder; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs index d1aecf561b..ce2515f879 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/misc/detect_locale_handler.rs @@ -14,6 +14,7 @@ use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; use http_server_common::request::get_request_header_optional::get_request_header_optional; +use http_server_common::request::parse_accept_language::parse_accept_language; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; @@ -23,7 +24,6 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; -use http_server_common::request::parse_accept_language::parse_accept_language; // =============== Success Response =============== diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs index 61fc2a2ce4..9c433ca30e 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs @@ -1,16 +1,15 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::get_user_profile_by_username::get_user_profile_by_username; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; +use database_queries::users::get_user_profile_by_username::get_user_profile_by_username; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs index 6c79346f14..310c64a9a9 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/list_banned_users.rs @@ -5,12 +5,10 @@ use actix_web::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::get_user_profile_by_username::get_user_profile_by_username; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs index 19cda2400e..32964ed038 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs @@ -1,14 +1,12 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::database::queries::list_user_roles::UserRoleForList; -use crate::database::queries::list_user_roles::list_user_roles; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -16,12 +14,13 @@ use crate::server_state::ServerState; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use derive_more::{Display, Error}; +use database_queries::users::user_roles::list_user_roles::{UserRoleForList, list_user_roles}; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; #[derive(Serialize)] @@ -30,7 +29,7 @@ pub struct ListUserRolesResponse { pub user_roles: Vec, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum ListUserRolesError { BadInput(String), ServerError, @@ -57,6 +56,13 @@ impl ResponseError for ListUserRolesError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListUserRolesError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn list_user_roles_handler( http_request: HttpRequest, server_state: web::Data> diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs index 3211988741..0ed4710c5b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs @@ -1,13 +1,11 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::list_user_roles::list_user_roles; -use crate::database::queries::get_user_profile_by_username::get_user_profile_by_username; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -15,12 +13,14 @@ use crate::server_state::ServerState; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use derive_more::{Display, Error}; +use database_queries::users::get_user_profile_by_username::get_user_profile_by_username; +use database_queries::users::user_roles::list_user_roles::list_user_roles; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::fmt; use std::sync::Arc; /// For the URL PathInfo @@ -34,7 +34,7 @@ pub struct SetUserRoleRequest { user_role_slug: String, } -#[derive(Debug, Display)] +#[derive(Debug)] pub enum SetUserRoleError { BadInput(String), NotAuthorized, @@ -64,6 +64,13 @@ impl ResponseError for SetUserRoleError { } } +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for SetUserRoleError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + pub async fn set_user_role_handler( http_request: HttpRequest, path: Path, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs index a30b6e1143..77fe21d928 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs @@ -1,17 +1,15 @@ use actix_http::Error; use actix_http::http::header; -use actix_web::cookie::Cookie; use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::get_user_profile_by_username::get_user_profile_by_username; -use crate::database::query_builders::list_users_query_builder::{ListUsersQueryBuilder, UserForList}; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; -use derive_more::{Display, Error}; +use database_queries::users::list_users_query_builder::{UserForList, ListUsersQueryBuilder}; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs index dc9a8d40cd..21d64e593c 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs @@ -7,7 +7,6 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database::queries::create_session::create_session_for_user; -use crate::database::queries::get_user_profile_by_username::get_user_profile_by_username; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; @@ -23,6 +22,7 @@ use crate::validations::username::validate_username; use crate::validations::username_reservations::is_reserved_username; use crate::validations::website_url::validate_website_url; use database_queries::column_types::record_visibility::RecordVisibility; +use database_queries::users::get_user_profile_by_username::get_user_profile_by_username; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs index 3d101d0b71..0f2acd33c6 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs @@ -8,13 +8,12 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use anyhow::anyhow; use chrono::{DateTime, Utc}; -use crate::AnyhowResult; -use crate::database::queries::list_user_badges::UserBadgeForList; -use crate::database::queries::list_user_badges::list_user_badges; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use database_queries::column_types::record_visibility::RecordVisibility; use database_queries::helpers::boolean_converters::i8_to_bool; +use database_queries::users::user_badges::list_user_badges::UserBadgeForList; +use database_queries::users::user_badges::list_user_badges::list_user_badges; use log::{info, warn, log}; use md5::{Md5, Digest}; use regex::Regex; @@ -26,6 +25,7 @@ use std::fmt; use std::sync::Arc; // TODO: This is duplicated in query_user_profile +// TODO: This handler has embedded queries. #[derive(Serialize)] pub struct UserProfileRecordForResponse { From 598e99d2dac8efa7e5f8504fa8ca86b2c1710770 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 03:15:10 -0500 Subject: [PATCH 0951/1042] refactor- almost done --- .../src/ip_bans}/list_ip_bans.rs | 0 .../lib/database_queries/src/ip_bans/mod.rs | 1 + crates/lib/database_queries/src/lib.rs | 1 + crates/lib/database_queries/src/users/mod.rs | 3 +- .../users/user_sessions}/create_session.rs | 4 +-- .../src/users/user_sessions/mod.rs | 1 + .../storyteller_web/src/database/mod.rs | 3 +- .../src/database/queries/badges.rs | 29 ------------------- .../src/database/queries/mod.rs | 3 -- .../endpoints/users/create_account.rs | 3 +- .../endpoints/users/edit_profile.rs | 1 - .../users/list_user_tts_inference_results.rs | 1 - .../src/http_server/endpoints/users/login.rs | 2 +- .../src/threads/poll_ip_banlist_thread.rs | 2 +- 14 files changed, 11 insertions(+), 43 deletions(-) rename crates/{service/storyteller_web/src/database/queries => lib/database_queries/src/ip_bans}/list_ip_bans.rs (100%) create mode 100644 crates/lib/database_queries/src/ip_bans/mod.rs rename crates/{service/storyteller_web/src/database/queries => lib/database_queries/src/users/user_sessions}/create_session.rs (91%) create mode 100644 crates/lib/database_queries/src/users/user_sessions/mod.rs delete mode 100644 crates/service/storyteller_web/src/database/queries/badges.rs delete mode 100644 crates/service/storyteller_web/src/database/queries/mod.rs diff --git a/crates/service/storyteller_web/src/database/queries/list_ip_bans.rs b/crates/lib/database_queries/src/ip_bans/list_ip_bans.rs similarity index 100% rename from crates/service/storyteller_web/src/database/queries/list_ip_bans.rs rename to crates/lib/database_queries/src/ip_bans/list_ip_bans.rs diff --git a/crates/lib/database_queries/src/ip_bans/mod.rs b/crates/lib/database_queries/src/ip_bans/mod.rs new file mode 100644 index 0000000000..2cebd7b6e9 --- /dev/null +++ b/crates/lib/database_queries/src/ip_bans/mod.rs @@ -0,0 +1 @@ +pub mod list_ip_bans; \ No newline at end of file diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index 891437aa3d..9fc2829f95 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -5,6 +5,7 @@ mod builders; pub mod api_tokens; pub mod column_types; pub mod helpers; +pub mod ip_bans; pub mod model_categories; pub mod tokens; pub mod tts; diff --git a/crates/lib/database_queries/src/users/mod.rs b/crates/lib/database_queries/src/users/mod.rs index 2327971b74..a4b95c51c7 100644 --- a/crates/lib/database_queries/src/users/mod.rs +++ b/crates/lib/database_queries/src/users/mod.rs @@ -1,4 +1,5 @@ pub mod get_user_profile_by_username; pub mod list_users_query_builder; pub mod user_badges; -pub mod user_roles; \ No newline at end of file +pub mod user_roles; +pub mod user_sessions; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/create_session.rs b/crates/lib/database_queries/src/users/user_sessions/create_session.rs similarity index 91% rename from crates/service/storyteller_web/src/database/queries/create_session.rs rename to crates/lib/database_queries/src/users/user_sessions/create_session.rs index aa9654f5e8..3748195e67 100644 --- a/crates/service/storyteller_web/src/database/queries/create_session.rs +++ b/crates/lib/database_queries/src/users/user_sessions/create_session.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; -use crate::AnyhowResult; -use database_queries::tokens::Tokens; +use container_common::anyhow_result::AnyhowResult; +use crate::tokens::Tokens; use log::{info, warn}; use sqlx::MySqlPool; diff --git a/crates/lib/database_queries/src/users/user_sessions/mod.rs b/crates/lib/database_queries/src/users/user_sessions/mod.rs new file mode 100644 index 0000000000..6a3f6dcb74 --- /dev/null +++ b/crates/lib/database_queries/src/users/user_sessions/mod.rs @@ -0,0 +1 @@ +pub mod create_session; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/mod.rs b/crates/service/storyteller_web/src/database/mod.rs index 1c04fcc19f..2fdbb114a4 100644 --- a/crates/service/storyteller_web/src/database/mod.rs +++ b/crates/service/storyteller_web/src/database/mod.rs @@ -1,2 +1 @@ -pub mod mediators; -pub mod queries; \ No newline at end of file +pub mod mediators; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/database/queries/badges.rs b/crates/service/storyteller_web/src/database/queries/badges.rs deleted file mode 100644 index 9425f0069e..0000000000 --- a/crates/service/storyteller_web/src/database/queries/badges.rs +++ /dev/null @@ -1,29 +0,0 @@ -use sqlx::MySqlPool; -use crate::AnyhowResult; - -pub struct NewBadge { - pub slug: String, - pub title: String, - pub description: String, - pub image_url: String, -} - -impl NewBadge { - pub async fn insert(&self, pool: &MySqlPool) -> AnyhowResult { - let record_id = sqlx::query!( - r#" -INSERT INTO badges ( slug, title, description, image_url ) -VALUES ( ?, ?, ?, ? ) - "#, - self.slug, - self.title, - self.description, - self.image_url, - ) - .execute(pool) - .await? - .last_insert_id(); - - Ok(record_id) - } -} diff --git a/crates/service/storyteller_web/src/database/queries/mod.rs b/crates/service/storyteller_web/src/database/queries/mod.rs deleted file mode 100644 index 7b1b03af01..0000000000 --- a/crates/service/storyteller_web/src/database/queries/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod badges; -pub mod create_session; -pub mod list_ip_bans; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs index a14e9f9cfe..f81565276b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs @@ -5,7 +5,6 @@ use actix_web::cookie::Cookie; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::create_session::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountErrorType::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -16,7 +15,7 @@ use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use crate::validations::username_reservations::is_reserved_username; use database_queries::tokens::Tokens; -use derive_more::{Display, Error}; +use database_queries::users::user_sessions::create_session::create_session_for_user; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs index 21d64e593c..9a6211d573 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs @@ -6,7 +6,6 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::database::queries::create_session::create_session_for_user; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs index c01e2b2585..69ff5c10f0 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -8,7 +8,6 @@ use actix_web::web::{Path, Query}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; -use crate::database::queries::create_session::create_session_for_user; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs index 6a7dad39c0..9bf940f700 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs @@ -6,12 +6,12 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::AnyhowResult; -use crate::database::queries::create_session::create_session_for_user; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; use database_queries::helpers::boolean_converters::i8_to_bool; +use database_queries::users::user_sessions::create_session::create_session_for_user; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/threads/poll_ip_banlist_thread.rs b/crates/service/storyteller_web/src/threads/poll_ip_banlist_thread.rs index 89b59aa32c..7706e84201 100644 --- a/crates/service/storyteller_web/src/threads/poll_ip_banlist_thread.rs +++ b/crates/service/storyteller_web/src/threads/poll_ip_banlist_thread.rs @@ -1,6 +1,6 @@ use anyhow::Error; -use crate::database::queries::list_ip_bans::list_ip_bans; use crate::threads::ip_banlist_set::IpBanlistSet; +use database_queries::ip_bans::list_ip_bans::list_ip_bans; use log::info; use log::warn; use sqlx::MySqlPool; From c4b8e0912a977817b7c381d83cbc7d2ea3f49892 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 03:42:39 -0500 Subject: [PATCH 0952/1042] finally done of in-app 'database' module --- crates/lib/database_queries/src/lib.rs | 1 + .../database_queries/src}/mediators/badge_granter.rs | 2 +- .../database_queries/src}/mediators/firehose_publisher.rs | 2 +- .../database => lib/database_queries/src}/mediators/mod.rs | 0 crates/service/storyteller_web/src/database/mod.rs | 1 - crates/service/storyteller_web/src/main.rs | 5 ++--- crates/service/storyteller_web/src/server_state.rs | 4 ++-- crates/service/storyteller_web/src/tts_download_job.rs | 5 ++--- crates/service/storyteller_web/src/tts_inference_job.rs | 3 +-- crates/service/storyteller_web/src/w2l_download_job.rs | 5 ++--- crates/service/storyteller_web/src/w2l_inference_job.rs | 3 +-- 11 files changed, 13 insertions(+), 18 deletions(-) rename crates/{service/storyteller_web/src/database => lib/database_queries/src}/mediators/badge_granter.rs (99%) rename crates/{service/storyteller_web/src/database => lib/database_queries/src}/mediators/firehose_publisher.rs (99%) rename crates/{service/storyteller_web/src/database => lib/database_queries/src}/mediators/mod.rs (100%) delete mode 100644 crates/service/storyteller_web/src/database/mod.rs diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index 9fc2829f95..eb91b80f33 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -6,6 +6,7 @@ pub mod api_tokens; pub mod column_types; pub mod helpers; pub mod ip_bans; +pub mod mediators; pub mod model_categories; pub mod tokens; pub mod tts; diff --git a/crates/service/storyteller_web/src/database/mediators/badge_granter.rs b/crates/lib/database_queries/src/mediators/badge_granter.rs similarity index 99% rename from crates/service/storyteller_web/src/database/mediators/badge_granter.rs rename to crates/lib/database_queries/src/mediators/badge_granter.rs index cb2ff74ca9..d771744beb 100644 --- a/crates/service/storyteller_web/src/database/mediators/badge_granter.rs +++ b/crates/lib/database_queries/src/mediators/badge_granter.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use crate::database::mediators::firehose_publisher::FirehosePublisher; +use crate::mediators::firehose_publisher::FirehosePublisher; use log::{warn,info}; use sqlx::error::Error::Database; use sqlx::mysql::MySqlQueryResult; diff --git a/crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs b/crates/lib/database_queries/src/mediators/firehose_publisher.rs similarity index 99% rename from crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs rename to crates/lib/database_queries/src/mediators/firehose_publisher.rs index 0dbb212dce..2499a2b144 100644 --- a/crates/service/storyteller_web/src/database/mediators/firehose_publisher.rs +++ b/crates/lib/database_queries/src/mediators/firehose_publisher.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use database_queries::tokens::Tokens; +use crate::tokens::Tokens; use log::{warn,info}; use sqlx::error::Error::Database; use sqlx::mysql::MySqlQueryResult; diff --git a/crates/service/storyteller_web/src/database/mediators/mod.rs b/crates/lib/database_queries/src/mediators/mod.rs similarity index 100% rename from crates/service/storyteller_web/src/database/mediators/mod.rs rename to crates/lib/database_queries/src/mediators/mod.rs diff --git a/crates/service/storyteller_web/src/database/mod.rs b/crates/service/storyteller_web/src/database/mod.rs deleted file mode 100644 index 2fdbb114a4..0000000000 --- a/crates/service/storyteller_web/src/database/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod mediators; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 63e8184287..53dee6cfd2 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -15,7 +15,6 @@ pub const RESERVED_USERNAMES : &'static str = include_str!("../../../../db/reserved_usernames.txt"); pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../../../../db/reserved_usernames_including.txt"); -pub mod database; pub mod common_env; pub mod http_clients; pub mod http_server; @@ -43,8 +42,6 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use crate::common_env::CommonEnv; -use crate::database::mediators::badge_granter::BadgeGranter; -use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::misc::default_route_404::default_route_404; @@ -112,6 +109,8 @@ use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; use crate::util::caching::single_item_ttl_cache::SingleItemTtlCache; use crate::util::encrypted_sort_id::SortKeyCrypto; +use database_queries::mediators::badge_granter::BadgeGranter; +use database_queries::mediators::firehose_publisher::FirehosePublisher; use futures::Future; use http_server_common::cors::build_common_cors_config; use limitation::Limiter; diff --git a/crates/service/storyteller_web/src/server_state.rs b/crates/service/storyteller_web/src/server_state.rs index 2dde5dcb9a..0dbaae0a7b 100644 --- a/crates/service/storyteller_web/src/server_state.rs +++ b/crates/service/storyteller_web/src/server_state.rs @@ -1,5 +1,3 @@ -use crate::database::mediators::badge_granter::BadgeGranter; -use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::tts::list_tts_models::TtsModelRecordForResponse; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; @@ -8,6 +6,8 @@ use crate::threads::ip_banlist_set::IpBanlistSet; use crate::util::buckets::bucket_client::BucketClient; use crate::util::caching::single_item_ttl_cache::SingleItemTtlCache; use crate::util::encrypted_sort_id::SortKeyCrypto; +use database_queries::mediators::badge_granter::BadgeGranter; +use database_queries::mediators::firehose_publisher::FirehosePublisher; use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/tts_download_job.rs b/crates/service/storyteller_web/src/tts_download_job.rs index 89d91616ad..d9ef14342a 100644 --- a/crates/service/storyteller_web/src/tts_download_job.rs +++ b/crates/service/storyteller_web/src/tts_download_job.rs @@ -11,7 +11,6 @@ #[macro_use] extern crate serde_derive; pub mod common_env; -pub mod database; pub mod job_queries; pub mod script_execution; pub mod util; @@ -22,8 +21,6 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use crate::common_env::CommonEnv; -use crate::database::mediators::badge_granter::BadgeGranter; -use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; use crate::job_queries::tts_download_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::tts_download_job_queries::insert_tts_model; @@ -44,6 +41,8 @@ use crate::util::noop_logger::NoOpLogger; use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use database_queries::column_types::tts_model_type::TtsModelType; +use database_queries::mediators::badge_granter::BadgeGranter; +use database_queries::mediators::firehose_publisher::FirehosePublisher; use log::{warn, info}; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2; diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index 68d92f37c4..879d1e09b0 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -11,7 +11,6 @@ #[macro_use] extern crate serde_derive; pub mod common_env; -pub mod database; pub mod http_clients; pub mod job_queries; pub mod script_execution; @@ -24,7 +23,6 @@ use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use container_common::token::random_uuid::generate_random_uuid; use crate::common_env::CommonEnv; -use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::http_clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; use crate::util::buckets::bucket_client::BucketClient; @@ -45,6 +43,7 @@ use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use database_queries::column_types::vocoder_type::VocoderType; +use database_queries::mediators::firehose_publisher::FirehosePublisher; use database_queries::tts::tts_inference_jobs::list_available_tts_inference_jobs::{AvailableTtsInferenceJob, list_available_tts_inference_jobs}; use database_queries::tts::tts_inference_jobs::mark_tts_inference_job_done::mark_tts_inference_job_done; use database_queries::tts::tts_inference_jobs::mark_tts_inference_job_failure::mark_tts_inference_job_failure; diff --git a/crates/service/storyteller_web/src/w2l_download_job.rs b/crates/service/storyteller_web/src/w2l_download_job.rs index 3504012dc1..370bbf8783 100644 --- a/crates/service/storyteller_web/src/w2l_download_job.rs +++ b/crates/service/storyteller_web/src/w2l_download_job.rs @@ -11,7 +11,6 @@ #[macro_use] extern crate serde_derive; pub mod common_env; -pub mod database; pub mod job_queries; pub mod script_execution; pub mod util; @@ -22,8 +21,6 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use crate::common_env::CommonEnv; -use crate::database::mediators::badge_granter::BadgeGranter; -use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; use crate::job_queries::w2l_download_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::w2l_download_job_queries::insert_w2l_template; @@ -46,6 +43,8 @@ use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; +use database_queries::mediators::badge_granter::BadgeGranter; +use database_queries::mediators::firehose_publisher::FirehosePublisher; use log::{warn, info}; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2; diff --git a/crates/service/storyteller_web/src/w2l_inference_job.rs b/crates/service/storyteller_web/src/w2l_inference_job.rs index c42302626f..6c2ce2178d 100644 --- a/crates/service/storyteller_web/src/w2l_inference_job.rs +++ b/crates/service/storyteller_web/src/w2l_inference_job.rs @@ -11,7 +11,6 @@ #[macro_use] extern crate serde_derive; pub mod common_env; -pub mod database; pub mod job_queries; pub mod script_execution; pub mod util; @@ -22,7 +21,6 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use crate::common_env::CommonEnv; -use crate::database::mediators::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; use crate::job_queries::w2l_inference_job_queries::grab_job_lock_and_mark_pending; @@ -47,6 +45,7 @@ use crate::util::noop_logger::NoOpLogger; use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; +use database_queries::mediators::firehose_publisher::FirehosePublisher; use log::{warn, info}; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2::PooledConnection; From 420a706f5d2d5b03e32bb6cc0689a54dd0694d3a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 04:03:45 -0500 Subject: [PATCH 0953/1042] massive in-library refactor --- crates/lib/database_queries/src/lib.rs | 11 +---------- .../{ => queries}/api_tokens/create_api_token.rs | 2 +- .../{ => queries}/api_tokens/delete_api_token.rs | 0 .../src/{ => queries}/api_tokens/edit_api_token.rs | 0 .../list_available_api_tokens_for_user.rs | 0 .../src/{ => queries}/api_tokens/mod.rs | 0 .../src/{ => queries}/ip_bans/list_ip_bans.rs | 0 .../src/{ => queries}/ip_bans/mod.rs | 0 crates/lib/database_queries/src/queries/mod.rs | 8 ++++++++ .../model_categories/get_category_by_token.rs | 0 .../list_categories_query_builder.rs | 2 +- .../src/{ => queries}/model_categories/mod.rs | 0 .../database_queries/src/{ => queries}/tts/mod.rs | 0 .../tts/stats/calculate_tts_model_leaderboard.rs | 0 .../src/{ => queries}/tts/stats/mod.rs | 0 .../fetch_and_build_tts_model_category_map.rs | 0 .../list_assigned_tts_categories_query_builder.rs | 0 .../tts/tts_category_assignments/mod.rs | 0 .../{ => queries}/tts/tts_inference_jobs/_keys.rs | 0 .../tts_inference_jobs/insert_tts_inference_job.rs | 0 .../list_available_tts_inference_jobs.rs | 2 +- .../mark_tts_inference_job_done.rs | 2 +- .../mark_tts_inference_job_failure.rs | 2 +- ...mark_tts_inference_job_pending_and_grab_lock.rs | 2 +- .../mark_tts_inference_job_permanently_dead.rs | 2 +- .../{ => queries}/tts/tts_inference_jobs/mod.rs | 0 .../tts_models/delete_tts_model_various_scopes.rs | 0 .../{ => queries}/tts/tts_models/get_tts_model.rs | 0 .../tts/tts_models/get_tts_model_for_inference.rs | 0 .../tts/tts_models/list_tts_models.rs | 0 .../src/{ => queries}/tts/tts_models/mod.rs | 0 .../delete_tts_result_various_scopes.rs | 0 .../tts/tts_results/insert_tts_result.rs | 2 +- .../tts_results/list_tts_results_query_builder.rs | 2 +- .../src/{ => queries}/tts/tts_results/mod.rs | 0 .../tts/tts_results/query_tts_result.rs | 0 .../lib/database_queries/src/queries/twitch/mod.rs | 3 +++ .../twitch_event_rules/delete_twitch_event_rule.rs | 0 .../get_twitch_event_rule_for_user.rs | 0 .../insert_twitch_event_rule_builder.rs | 0 .../list_twitch_event_rules_for_user.rs | 0 .../{ => queries/twitch}/twitch_event_rules/mod.rs | 0 .../update_twitch_event_rule_builder.rs | 0 .../src/{ => queries/twitch}/twitch_oauth/find.rs | 0 .../{ => queries/twitch}/twitch_oauth/insert.rs | 0 .../src/{ => queries/twitch}/twitch_oauth/mod.rs | 0 .../twitch}/twitch_pubsub/insert_bits.rs | 0 .../twitch}/twitch_pubsub/insert_channel_points.rs | 0 .../src/{ => queries/twitch}/twitch_pubsub/mod.rs | 0 .../users/get_user_profile_by_username.rs | 0 .../users/list_users_query_builder.rs | 0 .../src/{ => queries}/users/mod.rs | 0 .../users/user_badges/list_user_badges.rs | 0 .../src/{ => queries}/users/user_badges/mod.rs | 0 .../users/user_roles/list_user_roles.rs | 0 .../src/{ => queries}/users/user_roles/mod.rs | 0 .../users/user_sessions/create_session.rs | 0 .../src/{ => queries}/users/user_sessions/mod.rs | 0 .../get_voice_clone_request.rs | 0 .../insert_voice_clone_requests_builder.rs | 0 .../src/{ => queries}/voice_clone_requests/mod.rs | 0 .../database_queries/src/{ => queries}/w2l/mod.rs | 0 .../stats/calculate_w2l_template_leaderboard.rs | 0 .../src/{ => queries}/w2l/stats/mod.rs | 0 .../{ => queries}/w2l/w2l_inference_jobs/mod.rs | 0 .../delete_w2l_result_various_scopes.rs | 0 .../list_w2l_inference_results_query_builder.rs | 2 +- .../src/{ => queries}/w2l/w2l_results/mod.rs | 0 .../w2l/w2l_results/query_w2l_result.rs | 0 .../delete_w2l_template_various_scopes.rs | 0 .../w2l/w2l_templates/get_w2l_template.rs | 0 .../w2l/w2l_templates/list_w2l_templates.rs | 0 .../src/{ => queries}/w2l/w2l_templates/mod.rs | 0 .../endpoints_ws/obs_gateway_websocket_handler.rs | 8 ++++---- .../twitch_pubsub_user_subscriber_thread.rs | 10 +++++----- .../endpoints/api_tokens/create_api_token.rs | 2 +- .../endpoints/api_tokens/delete_api_token.rs | 4 ++-- .../endpoints/api_tokens/edit_api_token.rs | 4 ++-- .../endpoints/api_tokens/list_api_tokens.rs | 2 +- .../endpoints/categories/assign_tts_category.rs | 4 ++-- .../endpoints/categories/get_category.rs | 2 +- .../endpoints/categories/list_tts_categories.rs | 2 +- .../list_tts_model_assigned_categories.rs | 2 +- .../endpoints/leaderboard/get_leaderboard.rs | 8 ++++---- .../moderation/categories/edit_category.rs | 2 +- .../list_tts_categories_for_moderation.rs | 2 +- .../endpoints/moderation/user_bans/ban_user.rs | 2 +- .../endpoints/moderation/user_roles/list_roles.rs | 2 +- .../moderation/user_roles/set_user_role.rs | 4 ++-- .../endpoints/moderation/users/list_users.rs | 2 +- .../http_server/endpoints/tts/delete_tts_model.rs | 10 +++++----- .../http_server/endpoints/tts/delete_tts_result.rs | 10 +++++----- .../http_server/endpoints/tts/edit_tts_model.rs | 2 +- .../http_server/endpoints/tts/edit_tts_result.rs | 2 +- .../src/http_server/endpoints/tts/get_tts_model.rs | 2 +- .../http_server/endpoints/tts/get_tts_result.rs | 4 ++-- .../http_server/endpoints/tts/list_tts_models.rs | 4 ++-- .../src/http_server/endpoints/twitch/oauth_end.rs | 2 +- .../http_server/endpoints/users/create_account.rs | 2 +- .../http_server/endpoints/users/edit_profile.rs | 2 +- .../src/http_server/endpoints/users/get_profile.rs | 4 ++-- .../users/list_user_tts_inference_results.rs | 2 +- .../endpoints/users/list_user_tts_models.rs | 2 +- .../users/list_user_w2l_inference_results.rs | 4 ++-- .../endpoints/users/list_user_w2l_templates.rs | 2 +- .../src/http_server/endpoints/users/login.rs | 2 +- .../http_server/endpoints/w2l/delete_w2l_result.rs | 10 +++++----- .../endpoints/w2l/delete_w2l_template.rs | 10 +++++----- .../http_server/endpoints/w2l/edit_w2l_result.rs | 2 +- .../http_server/endpoints/w2l/edit_w2l_template.rs | 2 +- .../http_server/endpoints/w2l/get_w2l_result.rs | 4 ++-- .../http_server/endpoints/w2l/get_w2l_template.rs | 4 ++-- .../endpoints/w2l/list_w2l_templates.rs | 2 +- .../endpoints/w2l/set_w2l_template_mod_approval.rs | 2 +- crates/service/storyteller_web/src/main.rs | 4 ++-- .../src/threads/poll_ip_banlist_thread.rs | 2 +- .../storyteller_web/src/tts_inference_job.rs | 14 +++++++------- 117 files changed, 109 insertions(+), 107 deletions(-) rename crates/lib/database_queries/src/{ => queries}/api_tokens/create_api_token.rs (95%) rename crates/lib/database_queries/src/{ => queries}/api_tokens/delete_api_token.rs (100%) rename crates/lib/database_queries/src/{ => queries}/api_tokens/edit_api_token.rs (100%) rename crates/lib/database_queries/src/{ => queries}/api_tokens/list_available_api_tokens_for_user.rs (100%) rename crates/lib/database_queries/src/{ => queries}/api_tokens/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/ip_bans/list_ip_bans.rs (100%) rename crates/lib/database_queries/src/{ => queries}/ip_bans/mod.rs (100%) create mode 100644 crates/lib/database_queries/src/queries/mod.rs rename crates/lib/database_queries/src/{ => queries}/model_categories/get_category_by_token.rs (100%) rename crates/lib/database_queries/src/{ => queries}/model_categories/list_categories_query_builder.rs (99%) rename crates/lib/database_queries/src/{ => queries}/model_categories/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/stats/calculate_tts_model_leaderboard.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/stats/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_category_assignments/fetch_and_build_tts_model_category_map.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_category_assignments/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_inference_jobs/_keys.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_inference_jobs/insert_tts_inference_job.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs (98%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_inference_jobs/mark_tts_inference_job_done.rs (91%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs (90%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_inference_jobs/mark_tts_inference_job_pending_and_grab_lock.rs (96%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs (90%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_inference_jobs/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_models/delete_tts_model_various_scopes.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_models/get_tts_model.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_models/get_tts_model_for_inference.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_models/list_tts_models.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_models/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_results/delete_tts_result_various_scopes.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_results/insert_tts_result.rs (97%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_results/list_tts_results_query_builder.rs (99%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_results/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/tts/tts_results/query_tts_result.rs (100%) create mode 100644 crates/lib/database_queries/src/queries/twitch/mod.rs rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_event_rules/delete_twitch_event_rule.rs (100%) rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_event_rules/get_twitch_event_rule_for_user.rs (100%) rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_event_rules/insert_twitch_event_rule_builder.rs (100%) rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_event_rules/list_twitch_event_rules_for_user.rs (100%) rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_event_rules/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_event_rules/update_twitch_event_rule_builder.rs (100%) rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_oauth/find.rs (100%) rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_oauth/insert.rs (100%) rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_oauth/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_pubsub/insert_bits.rs (100%) rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_pubsub/insert_channel_points.rs (100%) rename crates/lib/database_queries/src/{ => queries/twitch}/twitch_pubsub/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/users/get_user_profile_by_username.rs (100%) rename crates/lib/database_queries/src/{ => queries}/users/list_users_query_builder.rs (100%) rename crates/lib/database_queries/src/{ => queries}/users/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/users/user_badges/list_user_badges.rs (100%) rename crates/lib/database_queries/src/{ => queries}/users/user_badges/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/users/user_roles/list_user_roles.rs (100%) rename crates/lib/database_queries/src/{ => queries}/users/user_roles/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/users/user_sessions/create_session.rs (100%) rename crates/lib/database_queries/src/{ => queries}/users/user_sessions/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/voice_clone_requests/get_voice_clone_request.rs (100%) rename crates/lib/database_queries/src/{ => queries}/voice_clone_requests/insert_voice_clone_requests_builder.rs (100%) rename crates/lib/database_queries/src/{ => queries}/voice_clone_requests/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/w2l/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/w2l/stats/calculate_w2l_template_leaderboard.rs (100%) rename crates/lib/database_queries/src/{ => queries}/w2l/stats/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/w2l/w2l_inference_jobs/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/w2l/w2l_results/delete_w2l_result_various_scopes.rs (100%) rename crates/lib/database_queries/src/{ => queries}/w2l/w2l_results/list_w2l_inference_results_query_builder.rs (99%) rename crates/lib/database_queries/src/{ => queries}/w2l/w2l_results/mod.rs (100%) rename crates/lib/database_queries/src/{ => queries}/w2l/w2l_results/query_w2l_result.rs (100%) rename crates/lib/database_queries/src/{ => queries}/w2l/w2l_templates/delete_w2l_template_various_scopes.rs (100%) rename crates/lib/database_queries/src/{ => queries}/w2l/w2l_templates/get_w2l_template.rs (100%) rename crates/lib/database_queries/src/{ => queries}/w2l/w2l_templates/list_w2l_templates.rs (100%) rename crates/lib/database_queries/src/{ => queries}/w2l/w2l_templates/mod.rs (100%) diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index eb91b80f33..defc1886a9 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -2,17 +2,8 @@ mod builders; -pub mod api_tokens; pub mod column_types; pub mod helpers; -pub mod ip_bans; pub mod mediators; -pub mod model_categories; +pub mod queries; pub mod tokens; -pub mod tts; -pub mod twitch_event_rules; -pub mod twitch_oauth; -pub mod twitch_pubsub; -pub mod users; -pub mod voice_clone_requests; -pub mod w2l; diff --git a/crates/lib/database_queries/src/api_tokens/create_api_token.rs b/crates/lib/database_queries/src/queries/api_tokens/create_api_token.rs similarity index 95% rename from crates/lib/database_queries/src/api_tokens/create_api_token.rs rename to crates/lib/database_queries/src/queries/api_tokens/create_api_token.rs index faa7403a54..50ee9b4ad7 100644 --- a/crates/lib/database_queries/src/api_tokens/create_api_token.rs +++ b/crates/lib/database_queries/src/queries/api_tokens/create_api_token.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use crate::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; +use crate::queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; use crate::tokens::Tokens; use sqlx::MySqlPool; use sqlx; diff --git a/crates/lib/database_queries/src/api_tokens/delete_api_token.rs b/crates/lib/database_queries/src/queries/api_tokens/delete_api_token.rs similarity index 100% rename from crates/lib/database_queries/src/api_tokens/delete_api_token.rs rename to crates/lib/database_queries/src/queries/api_tokens/delete_api_token.rs diff --git a/crates/lib/database_queries/src/api_tokens/edit_api_token.rs b/crates/lib/database_queries/src/queries/api_tokens/edit_api_token.rs similarity index 100% rename from crates/lib/database_queries/src/api_tokens/edit_api_token.rs rename to crates/lib/database_queries/src/queries/api_tokens/edit_api_token.rs diff --git a/crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs b/crates/lib/database_queries/src/queries/api_tokens/list_available_api_tokens_for_user.rs similarity index 100% rename from crates/lib/database_queries/src/api_tokens/list_available_api_tokens_for_user.rs rename to crates/lib/database_queries/src/queries/api_tokens/list_available_api_tokens_for_user.rs diff --git a/crates/lib/database_queries/src/api_tokens/mod.rs b/crates/lib/database_queries/src/queries/api_tokens/mod.rs similarity index 100% rename from crates/lib/database_queries/src/api_tokens/mod.rs rename to crates/lib/database_queries/src/queries/api_tokens/mod.rs diff --git a/crates/lib/database_queries/src/ip_bans/list_ip_bans.rs b/crates/lib/database_queries/src/queries/ip_bans/list_ip_bans.rs similarity index 100% rename from crates/lib/database_queries/src/ip_bans/list_ip_bans.rs rename to crates/lib/database_queries/src/queries/ip_bans/list_ip_bans.rs diff --git a/crates/lib/database_queries/src/ip_bans/mod.rs b/crates/lib/database_queries/src/queries/ip_bans/mod.rs similarity index 100% rename from crates/lib/database_queries/src/ip_bans/mod.rs rename to crates/lib/database_queries/src/queries/ip_bans/mod.rs diff --git a/crates/lib/database_queries/src/queries/mod.rs b/crates/lib/database_queries/src/queries/mod.rs new file mode 100644 index 0000000000..f7393c2273 --- /dev/null +++ b/crates/lib/database_queries/src/queries/mod.rs @@ -0,0 +1,8 @@ +pub mod api_tokens; +pub mod ip_bans; +pub mod model_categories; +pub mod tts; +pub mod twitch; +pub mod users; +pub mod voice_clone_requests; +pub mod w2l; diff --git a/crates/lib/database_queries/src/model_categories/get_category_by_token.rs b/crates/lib/database_queries/src/queries/model_categories/get_category_by_token.rs similarity index 100% rename from crates/lib/database_queries/src/model_categories/get_category_by_token.rs rename to crates/lib/database_queries/src/queries/model_categories/get_category_by_token.rs diff --git a/crates/lib/database_queries/src/model_categories/list_categories_query_builder.rs b/crates/lib/database_queries/src/queries/model_categories/list_categories_query_builder.rs similarity index 99% rename from crates/lib/database_queries/src/model_categories/list_categories_query_builder.rs rename to crates/lib/database_queries/src/queries/model_categories/list_categories_query_builder.rs index 34c69f8b13..9c2235d402 100644 --- a/crates/lib/database_queries/src/model_categories/list_categories_query_builder.rs +++ b/crates/lib/database_queries/src/queries/model_categories/list_categories_query_builder.rs @@ -318,7 +318,7 @@ pub struct RawInternalCategoryRecord { #[cfg(test)] mod tests { - use crate::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder; + use crate::queries::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder; #[test] fn predicates_default_scoping() { diff --git a/crates/lib/database_queries/src/model_categories/mod.rs b/crates/lib/database_queries/src/queries/model_categories/mod.rs similarity index 100% rename from crates/lib/database_queries/src/model_categories/mod.rs rename to crates/lib/database_queries/src/queries/model_categories/mod.rs diff --git a/crates/lib/database_queries/src/tts/mod.rs b/crates/lib/database_queries/src/queries/tts/mod.rs similarity index 100% rename from crates/lib/database_queries/src/tts/mod.rs rename to crates/lib/database_queries/src/queries/tts/mod.rs diff --git a/crates/lib/database_queries/src/tts/stats/calculate_tts_model_leaderboard.rs b/crates/lib/database_queries/src/queries/tts/stats/calculate_tts_model_leaderboard.rs similarity index 100% rename from crates/lib/database_queries/src/tts/stats/calculate_tts_model_leaderboard.rs rename to crates/lib/database_queries/src/queries/tts/stats/calculate_tts_model_leaderboard.rs diff --git a/crates/lib/database_queries/src/tts/stats/mod.rs b/crates/lib/database_queries/src/queries/tts/stats/mod.rs similarity index 100% rename from crates/lib/database_queries/src/tts/stats/mod.rs rename to crates/lib/database_queries/src/queries/tts/stats/mod.rs diff --git a/crates/lib/database_queries/src/tts/tts_category_assignments/fetch_and_build_tts_model_category_map.rs b/crates/lib/database_queries/src/queries/tts/tts_category_assignments/fetch_and_build_tts_model_category_map.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_category_assignments/fetch_and_build_tts_model_category_map.rs rename to crates/lib/database_queries/src/queries/tts/tts_category_assignments/fetch_and_build_tts_model_category_map.rs diff --git a/crates/lib/database_queries/src/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs b/crates/lib/database_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs rename to crates/lib/database_queries/src/queries/tts/tts_category_assignments/list_assigned_tts_categories_query_builder.rs diff --git a/crates/lib/database_queries/src/tts/tts_category_assignments/mod.rs b/crates/lib/database_queries/src/queries/tts/tts_category_assignments/mod.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_category_assignments/mod.rs rename to crates/lib/database_queries/src/queries/tts/tts_category_assignments/mod.rs diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/_keys.rs b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/_keys.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_inference_jobs/_keys.rs rename to crates/lib/database_queries/src/queries/tts/tts_inference_jobs/_keys.rs diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/insert_tts_inference_job.rs b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/insert_tts_inference_job.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_inference_jobs/insert_tts_inference_job.rs rename to crates/lib/database_queries/src/queries/tts/tts_inference_jobs/insert_tts_inference_job.rs diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs similarity index 98% rename from crates/lib/database_queries/src/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs rename to crates/lib/database_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs index 718414db09..20a54dc768 100644 --- a/crates/lib/database_queries/src/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs @@ -4,7 +4,7 @@ use container_common::anyhow_result::AnyhowResult; use crate::column_types::job_status::JobStatus; use crate::column_types::record_visibility::RecordVisibility; use crate::helpers::boolean_converters::i8_to_bool; -use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; +use crate::queries::tts::tts_inference_jobs::_keys::TtsInferenceJobId; use log::{warn, info}; use sqlx::MySqlPool; diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_done.rs b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_done.rs similarity index 91% rename from crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_done.rs rename to crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_done.rs index 6d94cf3ff6..abbd33a670 100644 --- a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_done.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_done.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; +use crate::queries::tts::tts_inference_jobs::_keys::TtsInferenceJobId; use sqlx::MySqlPool; use sqlx; diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs similarity index 90% rename from crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs rename to crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs index 3bb8a93143..a68476b592 100644 --- a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_failure.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use crate::tts::tts_inference_jobs::list_available_tts_inference_jobs::AvailableTtsInferenceJob; +use crate::queries::tts::tts_inference_jobs::list_available_tts_inference_jobs::AvailableTtsInferenceJob; use sqlx::MySqlPool; use sqlx; diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_pending_and_grab_lock.rs b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_pending_and_grab_lock.rs similarity index 96% rename from crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_pending_and_grab_lock.rs rename to crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_pending_and_grab_lock.rs index 98708357bd..2c8ae40f9c 100644 --- a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_pending_and_grab_lock.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_pending_and_grab_lock.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; +use crate::queries::tts::tts_inference_jobs::_keys::TtsInferenceJobId; use sqlx::MySqlPool; use sqlx; diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs similarity index 90% rename from crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs rename to crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs index 58c605d9c5..0b7c65e649 100644 --- a/crates/lib/database_queries/src/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mark_tts_inference_job_permanently_dead.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use crate::tts::tts_inference_jobs::_keys::TtsInferenceJobId; +use crate::queries::tts::tts_inference_jobs::_keys::TtsInferenceJobId; use sqlx::MySqlPool; use sqlx; diff --git a/crates/lib/database_queries/src/tts/tts_inference_jobs/mod.rs b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mod.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_inference_jobs/mod.rs rename to crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mod.rs diff --git a/crates/lib/database_queries/src/tts/tts_models/delete_tts_model_various_scopes.rs b/crates/lib/database_queries/src/queries/tts/tts_models/delete_tts_model_various_scopes.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_models/delete_tts_model_various_scopes.rs rename to crates/lib/database_queries/src/queries/tts/tts_models/delete_tts_model_various_scopes.rs diff --git a/crates/lib/database_queries/src/tts/tts_models/get_tts_model.rs b/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_models/get_tts_model.rs rename to crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model.rs diff --git a/crates/lib/database_queries/src/tts/tts_models/get_tts_model_for_inference.rs b/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model_for_inference.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_models/get_tts_model_for_inference.rs rename to crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model_for_inference.rs diff --git a/crates/lib/database_queries/src/tts/tts_models/list_tts_models.rs b/crates/lib/database_queries/src/queries/tts/tts_models/list_tts_models.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_models/list_tts_models.rs rename to crates/lib/database_queries/src/queries/tts/tts_models/list_tts_models.rs diff --git a/crates/lib/database_queries/src/tts/tts_models/mod.rs b/crates/lib/database_queries/src/queries/tts/tts_models/mod.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_models/mod.rs rename to crates/lib/database_queries/src/queries/tts/tts_models/mod.rs diff --git a/crates/lib/database_queries/src/tts/tts_results/delete_tts_result_various_scopes.rs b/crates/lib/database_queries/src/queries/tts/tts_results/delete_tts_result_various_scopes.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_results/delete_tts_result_various_scopes.rs rename to crates/lib/database_queries/src/queries/tts/tts_results/delete_tts_result_various_scopes.rs diff --git a/crates/lib/database_queries/src/tts/tts_results/insert_tts_result.rs b/crates/lib/database_queries/src/queries/tts/tts_results/insert_tts_result.rs similarity index 97% rename from crates/lib/database_queries/src/tts/tts_results/insert_tts_result.rs rename to crates/lib/database_queries/src/queries/tts/tts_results/insert_tts_result.rs index be881d67f3..00b75da4d1 100644 --- a/crates/lib/database_queries/src/tts/tts_results/insert_tts_result.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_results/insert_tts_result.rs @@ -1,8 +1,8 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; use crate::column_types::vocoder_type::VocoderType; +use crate::queries::tts::tts_inference_jobs::list_available_tts_inference_jobs::AvailableTtsInferenceJob; use crate::tokens::Tokens; -use crate::tts::tts_inference_jobs::list_available_tts_inference_jobs::AvailableTtsInferenceJob; use log::warn; use sqlx::MySqlPool; use sqlx; diff --git a/crates/lib/database_queries/src/tts/tts_results/list_tts_results_query_builder.rs b/crates/lib/database_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs similarity index 99% rename from crates/lib/database_queries/src/tts/tts_results/list_tts_results_query_builder.rs rename to crates/lib/database_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs index 8c40002ff0..a60b5bc44e 100644 --- a/crates/lib/database_queries/src/tts/tts_results/list_tts_results_query_builder.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_results/list_tts_results_query_builder.rs @@ -326,7 +326,7 @@ pub struct RawInternalTtsRecord { #[cfg(test)] mod tests { - use crate::tts::tts_results::list_tts_results_query_builder::ListTtsResultsQueryBuilder; + use crate::queries::tts::tts_results::list_tts_results_query_builder::ListTtsResultsQueryBuilder; #[test] fn predicates_without_scoping() { diff --git a/crates/lib/database_queries/src/tts/tts_results/mod.rs b/crates/lib/database_queries/src/queries/tts/tts_results/mod.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_results/mod.rs rename to crates/lib/database_queries/src/queries/tts/tts_results/mod.rs diff --git a/crates/lib/database_queries/src/tts/tts_results/query_tts_result.rs b/crates/lib/database_queries/src/queries/tts/tts_results/query_tts_result.rs similarity index 100% rename from crates/lib/database_queries/src/tts/tts_results/query_tts_result.rs rename to crates/lib/database_queries/src/queries/tts/tts_results/query_tts_result.rs diff --git a/crates/lib/database_queries/src/queries/twitch/mod.rs b/crates/lib/database_queries/src/queries/twitch/mod.rs new file mode 100644 index 0000000000..445b7129e7 --- /dev/null +++ b/crates/lib/database_queries/src/queries/twitch/mod.rs @@ -0,0 +1,3 @@ +pub mod twitch_event_rules; +pub mod twitch_oauth; +pub mod twitch_pubsub; diff --git a/crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/delete_twitch_event_rule.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_event_rules/delete_twitch_event_rule.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_event_rules/delete_twitch_event_rule.rs diff --git a/crates/lib/database_queries/src/twitch_event_rules/get_twitch_event_rule_for_user.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/get_twitch_event_rule_for_user.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_event_rules/get_twitch_event_rule_for_user.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_event_rules/get_twitch_event_rule_for_user.rs diff --git a/crates/lib/database_queries/src/twitch_event_rules/insert_twitch_event_rule_builder.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/insert_twitch_event_rule_builder.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_event_rules/insert_twitch_event_rule_builder.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_event_rules/insert_twitch_event_rule_builder.rs diff --git a/crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules_for_user.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_event_rules/list_twitch_event_rules_for_user.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs diff --git a/crates/lib/database_queries/src/twitch_event_rules/mod.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/mod.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_event_rules/mod.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_event_rules/mod.rs diff --git a/crates/lib/database_queries/src/twitch_event_rules/update_twitch_event_rule_builder.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/update_twitch_event_rule_builder.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_event_rules/update_twitch_event_rule_builder.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_event_rules/update_twitch_event_rule_builder.rs diff --git a/crates/lib/database_queries/src/twitch_oauth/find.rs b/crates/lib/database_queries/src/queries/twitch/twitch_oauth/find.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_oauth/find.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_oauth/find.rs diff --git a/crates/lib/database_queries/src/twitch_oauth/insert.rs b/crates/lib/database_queries/src/queries/twitch/twitch_oauth/insert.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_oauth/insert.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_oauth/insert.rs diff --git a/crates/lib/database_queries/src/twitch_oauth/mod.rs b/crates/lib/database_queries/src/queries/twitch/twitch_oauth/mod.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_oauth/mod.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_oauth/mod.rs diff --git a/crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs b/crates/lib/database_queries/src/queries/twitch/twitch_pubsub/insert_bits.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_pubsub/insert_bits.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_pubsub/insert_bits.rs diff --git a/crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs b/crates/lib/database_queries/src/queries/twitch/twitch_pubsub/insert_channel_points.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_pubsub/insert_channel_points.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_pubsub/insert_channel_points.rs diff --git a/crates/lib/database_queries/src/twitch_pubsub/mod.rs b/crates/lib/database_queries/src/queries/twitch/twitch_pubsub/mod.rs similarity index 100% rename from crates/lib/database_queries/src/twitch_pubsub/mod.rs rename to crates/lib/database_queries/src/queries/twitch/twitch_pubsub/mod.rs diff --git a/crates/lib/database_queries/src/users/get_user_profile_by_username.rs b/crates/lib/database_queries/src/queries/users/get_user_profile_by_username.rs similarity index 100% rename from crates/lib/database_queries/src/users/get_user_profile_by_username.rs rename to crates/lib/database_queries/src/queries/users/get_user_profile_by_username.rs diff --git a/crates/lib/database_queries/src/users/list_users_query_builder.rs b/crates/lib/database_queries/src/queries/users/list_users_query_builder.rs similarity index 100% rename from crates/lib/database_queries/src/users/list_users_query_builder.rs rename to crates/lib/database_queries/src/queries/users/list_users_query_builder.rs diff --git a/crates/lib/database_queries/src/users/mod.rs b/crates/lib/database_queries/src/queries/users/mod.rs similarity index 100% rename from crates/lib/database_queries/src/users/mod.rs rename to crates/lib/database_queries/src/queries/users/mod.rs diff --git a/crates/lib/database_queries/src/users/user_badges/list_user_badges.rs b/crates/lib/database_queries/src/queries/users/user_badges/list_user_badges.rs similarity index 100% rename from crates/lib/database_queries/src/users/user_badges/list_user_badges.rs rename to crates/lib/database_queries/src/queries/users/user_badges/list_user_badges.rs diff --git a/crates/lib/database_queries/src/users/user_badges/mod.rs b/crates/lib/database_queries/src/queries/users/user_badges/mod.rs similarity index 100% rename from crates/lib/database_queries/src/users/user_badges/mod.rs rename to crates/lib/database_queries/src/queries/users/user_badges/mod.rs diff --git a/crates/lib/database_queries/src/users/user_roles/list_user_roles.rs b/crates/lib/database_queries/src/queries/users/user_roles/list_user_roles.rs similarity index 100% rename from crates/lib/database_queries/src/users/user_roles/list_user_roles.rs rename to crates/lib/database_queries/src/queries/users/user_roles/list_user_roles.rs diff --git a/crates/lib/database_queries/src/users/user_roles/mod.rs b/crates/lib/database_queries/src/queries/users/user_roles/mod.rs similarity index 100% rename from crates/lib/database_queries/src/users/user_roles/mod.rs rename to crates/lib/database_queries/src/queries/users/user_roles/mod.rs diff --git a/crates/lib/database_queries/src/users/user_sessions/create_session.rs b/crates/lib/database_queries/src/queries/users/user_sessions/create_session.rs similarity index 100% rename from crates/lib/database_queries/src/users/user_sessions/create_session.rs rename to crates/lib/database_queries/src/queries/users/user_sessions/create_session.rs diff --git a/crates/lib/database_queries/src/users/user_sessions/mod.rs b/crates/lib/database_queries/src/queries/users/user_sessions/mod.rs similarity index 100% rename from crates/lib/database_queries/src/users/user_sessions/mod.rs rename to crates/lib/database_queries/src/queries/users/user_sessions/mod.rs diff --git a/crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs b/crates/lib/database_queries/src/queries/voice_clone_requests/get_voice_clone_request.rs similarity index 100% rename from crates/lib/database_queries/src/voice_clone_requests/get_voice_clone_request.rs rename to crates/lib/database_queries/src/queries/voice_clone_requests/get_voice_clone_request.rs diff --git a/crates/lib/database_queries/src/voice_clone_requests/insert_voice_clone_requests_builder.rs b/crates/lib/database_queries/src/queries/voice_clone_requests/insert_voice_clone_requests_builder.rs similarity index 100% rename from crates/lib/database_queries/src/voice_clone_requests/insert_voice_clone_requests_builder.rs rename to crates/lib/database_queries/src/queries/voice_clone_requests/insert_voice_clone_requests_builder.rs diff --git a/crates/lib/database_queries/src/voice_clone_requests/mod.rs b/crates/lib/database_queries/src/queries/voice_clone_requests/mod.rs similarity index 100% rename from crates/lib/database_queries/src/voice_clone_requests/mod.rs rename to crates/lib/database_queries/src/queries/voice_clone_requests/mod.rs diff --git a/crates/lib/database_queries/src/w2l/mod.rs b/crates/lib/database_queries/src/queries/w2l/mod.rs similarity index 100% rename from crates/lib/database_queries/src/w2l/mod.rs rename to crates/lib/database_queries/src/queries/w2l/mod.rs diff --git a/crates/lib/database_queries/src/w2l/stats/calculate_w2l_template_leaderboard.rs b/crates/lib/database_queries/src/queries/w2l/stats/calculate_w2l_template_leaderboard.rs similarity index 100% rename from crates/lib/database_queries/src/w2l/stats/calculate_w2l_template_leaderboard.rs rename to crates/lib/database_queries/src/queries/w2l/stats/calculate_w2l_template_leaderboard.rs diff --git a/crates/lib/database_queries/src/w2l/stats/mod.rs b/crates/lib/database_queries/src/queries/w2l/stats/mod.rs similarity index 100% rename from crates/lib/database_queries/src/w2l/stats/mod.rs rename to crates/lib/database_queries/src/queries/w2l/stats/mod.rs diff --git a/crates/lib/database_queries/src/w2l/w2l_inference_jobs/mod.rs b/crates/lib/database_queries/src/queries/w2l/w2l_inference_jobs/mod.rs similarity index 100% rename from crates/lib/database_queries/src/w2l/w2l_inference_jobs/mod.rs rename to crates/lib/database_queries/src/queries/w2l/w2l_inference_jobs/mod.rs diff --git a/crates/lib/database_queries/src/w2l/w2l_results/delete_w2l_result_various_scopes.rs b/crates/lib/database_queries/src/queries/w2l/w2l_results/delete_w2l_result_various_scopes.rs similarity index 100% rename from crates/lib/database_queries/src/w2l/w2l_results/delete_w2l_result_various_scopes.rs rename to crates/lib/database_queries/src/queries/w2l/w2l_results/delete_w2l_result_various_scopes.rs diff --git a/crates/lib/database_queries/src/w2l/w2l_results/list_w2l_inference_results_query_builder.rs b/crates/lib/database_queries/src/queries/w2l/w2l_results/list_w2l_inference_results_query_builder.rs similarity index 99% rename from crates/lib/database_queries/src/w2l/w2l_results/list_w2l_inference_results_query_builder.rs rename to crates/lib/database_queries/src/queries/w2l/w2l_results/list_w2l_inference_results_query_builder.rs index 252b213e86..6c4a6f1760 100644 --- a/crates/lib/database_queries/src/w2l/w2l_results/list_w2l_inference_results_query_builder.rs +++ b/crates/lib/database_queries/src/queries/w2l/w2l_results/list_w2l_inference_results_query_builder.rs @@ -372,7 +372,7 @@ pub struct RawInternalTtsRecord { #[cfg(test)] mod tests { - use crate::w2l::w2l_results::list_w2l_inference_results_query_builder::ListW2lResultsQueryBuilder; + use crate::queries::w2l::w2l_results::list_w2l_inference_results_query_builder::ListW2lResultsQueryBuilder; #[test] fn predicates_without_scoping() { diff --git a/crates/lib/database_queries/src/w2l/w2l_results/mod.rs b/crates/lib/database_queries/src/queries/w2l/w2l_results/mod.rs similarity index 100% rename from crates/lib/database_queries/src/w2l/w2l_results/mod.rs rename to crates/lib/database_queries/src/queries/w2l/w2l_results/mod.rs diff --git a/crates/lib/database_queries/src/w2l/w2l_results/query_w2l_result.rs b/crates/lib/database_queries/src/queries/w2l/w2l_results/query_w2l_result.rs similarity index 100% rename from crates/lib/database_queries/src/w2l/w2l_results/query_w2l_result.rs rename to crates/lib/database_queries/src/queries/w2l/w2l_results/query_w2l_result.rs diff --git a/crates/lib/database_queries/src/w2l/w2l_templates/delete_w2l_template_various_scopes.rs b/crates/lib/database_queries/src/queries/w2l/w2l_templates/delete_w2l_template_various_scopes.rs similarity index 100% rename from crates/lib/database_queries/src/w2l/w2l_templates/delete_w2l_template_various_scopes.rs rename to crates/lib/database_queries/src/queries/w2l/w2l_templates/delete_w2l_template_various_scopes.rs diff --git a/crates/lib/database_queries/src/w2l/w2l_templates/get_w2l_template.rs b/crates/lib/database_queries/src/queries/w2l/w2l_templates/get_w2l_template.rs similarity index 100% rename from crates/lib/database_queries/src/w2l/w2l_templates/get_w2l_template.rs rename to crates/lib/database_queries/src/queries/w2l/w2l_templates/get_w2l_template.rs diff --git a/crates/lib/database_queries/src/w2l/w2l_templates/list_w2l_templates.rs b/crates/lib/database_queries/src/queries/w2l/w2l_templates/list_w2l_templates.rs similarity index 100% rename from crates/lib/database_queries/src/w2l/w2l_templates/list_w2l_templates.rs rename to crates/lib/database_queries/src/queries/w2l/w2l_templates/list_w2l_templates.rs diff --git a/crates/lib/database_queries/src/w2l/w2l_templates/mod.rs b/crates/lib/database_queries/src/queries/w2l/w2l_templates/mod.rs similarity index 100% rename from crates/lib/database_queries/src/w2l/w2l_templates/mod.rs rename to crates/lib/database_queries/src/queries/w2l/w2l_templates/mod.rs diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index f3edd03409..bddc37ec98 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -5,13 +5,14 @@ use actix_web::{middleware, web, App, Error, HttpRequest, HttpResponse, HttpServ use actix_web_actors::ws; use container_common::anyhow_result::AnyhowResult; use container_common::token::random_crockford_token::random_crockford_token; +use crate::endpoints_ws::obs_gateway_websocket_handler::ResponseType::TtsEvent; use crate::redis::constants::OBS_ACTIVE_TTL_SECONDS; use crate::redis::obs_active_payload::ObsActivePayload; use crate::server_state::ObsGatewayServerState; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; -use database_queries::twitch_oauth::find::{TwitchOauthTokenFinder, TwitchOauthTokenRecord}; +use database_queries::queries::twitch::twitch_oauth::find::{TwitchOauthTokenFinder, TwitchOauthTokenRecord}; use futures_timer::Delay; use futures_util::FutureExt; use http_server_common::error::common_server_error::CommonServerError; @@ -28,10 +29,9 @@ use std::thread::sleep; use std::time::Duration; use tokio::runtime::Handle; use twitch_api2::pubsub::Topic; -use twitch_api2::pubsub; -use twitch_api2::pubsub::channel_bits::ChannelBitsEventsV2Reply::BitsEvent; use twitch_api2::pubsub::TopicData::ChannelPointsChannelV1; -use crate::endpoints_ws::obs_gateway_websocket_handler::ResponseType::TtsEvent; +use twitch_api2::pubsub::channel_bits::ChannelBitsEventsV2Reply::BitsEvent; +use twitch_api2::pubsub; // TODO: Redis calls are synchronous (but fast), but is there any way to make them async? diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 80fcffa007..9160e5910c 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -8,12 +8,12 @@ use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; +use database_queries::queries::tts::tts_inference_jobs::insert_tts_inference_job::TtsInferenceJobInsertBuilder; +use database_queries::queries::twitch::twitch_oauth::find::{TwitchOauthTokenRecord, TwitchOauthTokenFinder}; +use database_queries::queries::twitch::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; +use database_queries::queries::twitch::twitch_pubsub::insert_bits::TwitchPubsubBitsInsertBuilder; +use database_queries::queries::twitch::twitch_pubsub::insert_channel_points::TwitchPubsubChannelPointsInsertBuilder; use database_queries::tokens::Tokens; -use database_queries::tts::tts_inference_jobs::insert_tts_inference_job::TtsInferenceJobInsertBuilder; -use database_queries::twitch_oauth::find::{TwitchOauthTokenRecord, TwitchOauthTokenFinder}; -use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; -use database_queries::twitch_pubsub::insert_bits::TwitchPubsubBitsInsertBuilder; -use database_queries::twitch_pubsub::insert_channel_points::TwitchPubsubChannelPointsInsertBuilder; use log::debug; use log::error; use log::info; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs index f5f1d82d21..c45016a671 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs @@ -22,7 +22,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; -use database_queries::api_tokens::create_api_token::create_api_token_for_user; +use database_queries::queries::api_tokens::create_api_token::create_api_token_for_user; // =============== Request =============== diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs index 0ecb7713f4..c5a303d36f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/delete_api_token.rs @@ -10,7 +10,7 @@ use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; -use database_queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; +use database_queries::queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log, error}; use regex::Regex; @@ -20,7 +20,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; -use database_queries::api_tokens::delete_api_token::delete_api_token; +use database_queries::queries::api_tokens::delete_api_token::delete_api_token; // =============== Request =============== diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs index 7454753195..213cb0a584 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/edit_api_token.rs @@ -18,8 +18,8 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; -use database_queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; -use database_queries::api_tokens::edit_api_token::edit_api_token; +use database_queries::queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; +use database_queries::queries::api_tokens::edit_api_token::edit_api_token; use crate::http_server::web_utils::ip_address::get_request_ip; // =============== Request =============== diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs index d7e57c0c83..b9bb38519f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/list_api_tokens.rs @@ -13,7 +13,7 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use database_queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; +use database_queries::queries::api_tokens::list_available_api_tokens_for_user::list_available_api_tokens_for_user; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use lexical_sort::natural_lexical_cmp; use log::{info, warn, log}; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs index 155d9031d1..015ba9fa10 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/assign_tts_category.rs @@ -13,8 +13,8 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use database_queries::model_categories::get_category_by_token::get_category_by_token; -use database_queries::tts::tts_models::get_tts_model::get_tts_model_by_token; +use database_queries::queries::model_categories::get_category_by_token::get_category_by_token; +use database_queries::queries::tts::tts_models::get_tts_model::get_tts_model_by_token; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs index e134e89919..00fea85139 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/get_category.rs @@ -13,7 +13,7 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use database_queries::model_categories::get_category_by_token::get_category_by_token; +use database_queries::queries::model_categories::get_category_by_token::get_category_by_token; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs index 10b969fef5..0a7c312ae2 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs @@ -13,7 +13,7 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use database_queries::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder; +use database_queries::queries::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use lexical_sort::natural_lexical_cmp; use log::{info, warn, log}; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs index c08faf969a..3daff48f93 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_model_assigned_categories.rs @@ -10,7 +10,7 @@ use chrono::{DateTime, Utc}; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; -use database_queries::tts::tts_category_assignments::list_assigned_tts_categories_query_builder::ListAssignedTtsCategoriesQueryBuilder; +use database_queries::queries::tts::tts_category_assignments::list_assigned_tts_categories_query_builder::ListAssignedTtsCategoriesQueryBuilder; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs b/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs index 5b31ce7826..de65564b56 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/leaderboard/get_leaderboard.rs @@ -9,10 +9,10 @@ use chrono::{DateTime, Utc}; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; -use database_queries::tts::stats::calculate_tts_model_leaderboard::TtsLeaderboardRecordForList; -use database_queries::tts::stats::calculate_tts_model_leaderboard::calculate_tts_model_leaderboard; -use database_queries::w2l::stats::calculate_w2l_template_leaderboard::W2lLeaderboardRecordForList; -use database_queries::w2l::stats::calculate_w2l_template_leaderboard::calculate_w2l_template_leaderboard; +use database_queries::queries::tts::stats::calculate_tts_model_leaderboard::TtsLeaderboardRecordForList; +use database_queries::queries::tts::stats::calculate_tts_model_leaderboard::calculate_tts_model_leaderboard; +use database_queries::queries::w2l::stats::calculate_w2l_template_leaderboard::W2lLeaderboardRecordForList; +use database_queries::queries::w2l::stats::calculate_w2l_template_leaderboard::calculate_w2l_template_leaderboard; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs index 76cdd7eb76..5c19e5dee7 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/edit_category.rs @@ -12,7 +12,7 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use database_queries::model_categories::get_category_by_token::get_category_by_token; +use database_queries::queries::model_categories::get_category_by_token::get_category_by_token; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs index fb4d21fe98..e471a96184 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/categories/list_tts_categories_for_moderation.rs @@ -13,7 +13,7 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; -use database_queries::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder; +use database_queries::queries::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs index 9c433ca30e..9683d99b9b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_bans/ban_user.rs @@ -9,7 +9,7 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; -use database_queries::users::get_user_profile_by_username::get_user_profile_by_username; +use database_queries::queries::users::get_user_profile_by_username::get_user_profile_by_username; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs index 32964ed038..63bf3f01a7 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/list_roles.rs @@ -14,7 +14,7 @@ use crate::server_state::ServerState; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use database_queries::users::user_roles::list_user_roles::{UserRoleForList, list_user_roles}; +use database_queries::queries::users::user_roles::list_user_roles::{UserRoleForList, list_user_roles}; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs index 0ed4710c5b..acd6c34dfa 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/user_roles/set_user_role.rs @@ -13,8 +13,8 @@ use crate::server_state::ServerState; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use database_queries::users::get_user_profile_by_username::get_user_profile_by_username; -use database_queries::users::user_roles::list_user_roles::list_user_roles; +use database_queries::queries::users::get_user_profile_by_username::get_user_profile_by_username; +use database_queries::queries::users::user_roles::list_user_roles::list_user_roles; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs index 77fe21d928..07e1c22a68 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/users/list_users.rs @@ -9,7 +9,7 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; -use database_queries::users::list_users_query_builder::{UserForList, ListUsersQueryBuilder}; +use database_queries::queries::users::list_users_query_builder::{UserForList, ListUsersQueryBuilder}; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs index d018d6fee5..398392ad0b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_model.rs @@ -12,11 +12,11 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::delete_role_disambiguation::DeleteRole; use crate::util::delete_role_disambiguation::delete_role_disambiguation; -use database_queries::tts::tts_models::delete_tts_model_various_scopes::delete_tts_model_as_mod; -use database_queries::tts::tts_models::delete_tts_model_various_scopes::delete_tts_model_as_user; -use database_queries::tts::tts_models::delete_tts_model_various_scopes::undelete_tts_model_as_mod; -use database_queries::tts::tts_models::delete_tts_model_various_scopes::undelete_tts_model_as_user; -use database_queries::tts::tts_models::get_tts_model::get_tts_model_by_token; +use database_queries::queries::tts::tts_models::delete_tts_model_various_scopes::delete_tts_model_as_mod; +use database_queries::queries::tts::tts_models::delete_tts_model_various_scopes::delete_tts_model_as_user; +use database_queries::queries::tts::tts_models::delete_tts_model_various_scopes::undelete_tts_model_as_mod; +use database_queries::queries::tts::tts_models::delete_tts_model_various_scopes::undelete_tts_model_as_user; +use database_queries::queries::tts::tts_models::get_tts_model::get_tts_model_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs index 5301abb70d..2f4c5b45a0 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/delete_tts_result.rs @@ -15,11 +15,11 @@ use crate::util::delete_role_disambiguation::delete_role_disambiguation; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use database_queries::tts::tts_results::delete_tts_result_various_scopes::delete_tts_inference_result_as_mod; -use database_queries::tts::tts_results::delete_tts_result_various_scopes::delete_tts_inference_result_as_user; -use database_queries::tts::tts_results::delete_tts_result_various_scopes::undelete_tts_inference_result_as_mod; -use database_queries::tts::tts_results::delete_tts_result_various_scopes::undelete_tts_inference_result_as_user; -use database_queries::tts::tts_results::query_tts_result::select_tts_result_by_token; +use database_queries::queries::tts::tts_results::delete_tts_result_various_scopes::delete_tts_inference_result_as_mod; +use database_queries::queries::tts::tts_results::delete_tts_result_various_scopes::delete_tts_inference_result_as_user; +use database_queries::queries::tts::tts_results::delete_tts_result_various_scopes::undelete_tts_inference_result_as_mod; +use database_queries::queries::tts::tts_results::delete_tts_result_various_scopes::undelete_tts_inference_result_as_user; +use database_queries::queries::tts::tts_results::query_tts_result::select_tts_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs index 1545a654d8..3d96899698 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs @@ -14,7 +14,7 @@ use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; use database_queries::column_types::record_visibility::RecordVisibility; use database_queries::column_types::vocoder_type::VocoderType; -use database_queries::tts::tts_models::get_tts_model::get_tts_model_by_token; +use database_queries::queries::tts::tts_models::get_tts_model::get_tts_model_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs index fa5081175f..15aff691b5 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_result.rs @@ -13,7 +13,7 @@ use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; use database_queries::column_types::record_visibility::RecordVisibility; -use database_queries::tts::tts_results::query_tts_result::select_tts_result_by_token; +use database_queries::queries::tts::tts_results::query_tts_result::select_tts_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs index 5cc5c5c713..954e5e93cc 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs @@ -10,7 +10,7 @@ use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use database_queries::tts::tts_models::get_tts_model::{TtsModelRecordForResponse, get_tts_model_by_token}; +use database_queries::queries::tts::tts_models::get_tts_model::{TtsModelRecordForResponse, get_tts_model_by_token}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs index c87872956f..3da2574f26 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_result.rs @@ -14,8 +14,8 @@ use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use database_queries::tts::tts_results::query_tts_result::TtsResultRecordForResponse; -use database_queries::tts::tts_results::query_tts_result::select_tts_result_by_token; +use database_queries::queries::tts::tts_results::query_tts_result::TtsResultRecordForResponse; +use database_queries::queries::tts::tts_results::query_tts_result::select_tts_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs index 443ec91587..f56b113799 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs @@ -9,8 +9,8 @@ use container_common::anyhow_result::AnyhowResult; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use database_queries::tts::tts_category_assignments::fetch_and_build_tts_model_category_map::fetch_and_build_tts_model_category_map; -use database_queries::tts::tts_models::list_tts_models::list_tts_models; +use database_queries::queries::tts::tts_category_assignments::fetch_and_build_tts_model_category_map::fetch_and_build_tts_model_category_map; +use database_queries::queries::tts::tts_models::list_tts_models::list_tts_models; use lexical_sort::natural_lexical_cmp; use log::{info, warn, log, error}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs index 5b8df49835..612c75e1fc 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs @@ -1,7 +1,7 @@ use actix_http::{StatusCode, header}; use actix_web::{HttpResponse, HttpRequest, web, ResponseError}; use crate::server_state::ServerState; -use database_queries::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; +use database_queries::queries::twitch::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; use http_server_common::request::get_request_ip::get_request_ip; use http_server_common::response::response_error_helpers::to_simple_json_error; use log::error; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs index f81565276b..d61fb24fd9 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/create_account.rs @@ -15,7 +15,7 @@ use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use crate::validations::username_reservations::is_reserved_username; use database_queries::tokens::Tokens; -use database_queries::users::user_sessions::create_session::create_session_for_user; +use database_queries::queries::users::user_sessions::create_session::create_session_for_user; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs index 9a6211d573..83682fdc82 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/edit_profile.rs @@ -21,7 +21,7 @@ use crate::validations::username::validate_username; use crate::validations::username_reservations::is_reserved_username; use crate::validations::website_url::validate_website_url; use database_queries::column_types::record_visibility::RecordVisibility; -use database_queries::users::get_user_profile_by_username::get_user_profile_by_username; +use database_queries::queries::users::get_user_profile_by_username::get_user_profile_by_username; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs index 0f2acd33c6..b838e2ed10 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/get_profile.rs @@ -12,8 +12,8 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use database_queries::column_types::record_visibility::RecordVisibility; use database_queries::helpers::boolean_converters::i8_to_bool; -use database_queries::users::user_badges::list_user_badges::UserBadgeForList; -use database_queries::users::user_badges::list_user_badges::list_user_badges; +use database_queries::queries::users::user_badges::list_user_badges::UserBadgeForList; +use database_queries::queries::users::user_badges::list_user_badges::list_user_badges; use log::{info, warn, log}; use md5::{Md5, Digest}; use regex::Regex; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs index 69ff5c10f0..8197eceaaa 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -11,7 +11,7 @@ use crate::AnyhowResult; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use database_queries::tts::tts_results::list_tts_results_query_builder::{ListTtsResultsQueryBuilder, TtsInferenceRecordForList}; +use database_queries::queries::tts::tts_results::list_tts_results_query_builder::{ListTtsResultsQueryBuilder, TtsInferenceRecordForList}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs index c9141c8146..ace2c6a14a 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_tts_models.rs @@ -10,7 +10,7 @@ use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use database_queries::tts::tts_models::list_tts_models::{TtsModelRecordForList, list_tts_models}; +use database_queries::queries::tts::tts_models::list_tts_models::{TtsModelRecordForList, list_tts_models}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs index ae48e13259..5f8f585cdd 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_inference_results.rs @@ -10,8 +10,8 @@ use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use database_queries::w2l::w2l_results::list_w2l_inference_results_query_builder::ListW2lResultsQueryBuilder; -use database_queries::w2l::w2l_results::list_w2l_inference_results_query_builder::W2lInferenceRecordForList; +use database_queries::queries::w2l::w2l_results::list_w2l_inference_results_query_builder::ListW2lResultsQueryBuilder; +use database_queries::queries::w2l::w2l_results::list_w2l_inference_results_query_builder::W2lInferenceRecordForList; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs index 399cae9737..8c6f9f1f5d 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/list_user_w2l_templates.rs @@ -10,7 +10,7 @@ use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use database_queries::w2l::w2l_templates::list_w2l_templates::{W2lTemplateRecordForList, list_w2l_templates}; +use database_queries::queries::w2l::w2l_templates::list_w2l_templates::{W2lTemplateRecordForList, list_w2l_templates}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs b/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs index 9bf940f700..c1b9745be0 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/users/login.rs @@ -11,7 +11,7 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::serialize_as_json_error::serialize_as_json_error; use crate::server_state::ServerState; use database_queries::helpers::boolean_converters::i8_to_bool; -use database_queries::users::user_sessions::create_session::create_session_for_user; +use database_queries::queries::users::user_sessions::create_session::create_session_for_user; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs index 6cf259aa9c..4302111506 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_result.rs @@ -15,11 +15,11 @@ use crate::util::delete_role_disambiguation::delete_role_disambiguation; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use database_queries::w2l::w2l_results::delete_w2l_result_various_scopes::delete_w2l_inference_result_as_mod; -use database_queries::w2l::w2l_results::delete_w2l_result_various_scopes::delete_w2l_inference_result_as_user; -use database_queries::w2l::w2l_results::delete_w2l_result_various_scopes::undelete_w2l_inference_result_as_mod; -use database_queries::w2l::w2l_results::delete_w2l_result_various_scopes::undelete_w2l_inference_result_as_user; -use database_queries::w2l::w2l_results::query_w2l_result::select_w2l_result_by_token; +use database_queries::queries::w2l::w2l_results::delete_w2l_result_various_scopes::delete_w2l_inference_result_as_mod; +use database_queries::queries::w2l::w2l_results::delete_w2l_result_various_scopes::delete_w2l_inference_result_as_user; +use database_queries::queries::w2l::w2l_results::delete_w2l_result_various_scopes::undelete_w2l_inference_result_as_mod; +use database_queries::queries::w2l::w2l_results::delete_w2l_result_various_scopes::undelete_w2l_inference_result_as_user; +use database_queries::queries::w2l::w2l_results::query_w2l_result::select_w2l_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs index 6ed18dd417..a23d27f0b0 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -6,11 +6,11 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use database_queries::w2l::w2l_templates::delete_w2l_template_various_scopes::delete_w2l_template_as_mod; -use database_queries::w2l::w2l_templates::delete_w2l_template_various_scopes::delete_w2l_template_as_user; -use database_queries::w2l::w2l_templates::delete_w2l_template_various_scopes::undelete_w2l_template_as_mod; -use database_queries::w2l::w2l_templates::delete_w2l_template_various_scopes::undelete_w2l_template_as_user; -use database_queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; +use database_queries::queries::w2l::w2l_templates::delete_w2l_template_various_scopes::delete_w2l_template_as_mod; +use database_queries::queries::w2l::w2l_templates::delete_w2l_template_various_scopes::delete_w2l_template_as_user; +use database_queries::queries::w2l::w2l_templates::delete_w2l_template_various_scopes::undelete_w2l_template_as_mod; +use database_queries::queries::w2l::w2l_templates::delete_w2l_template_various_scopes::undelete_w2l_template_as_user; +use database_queries::queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs index 00b8713b5d..98c1301f5b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_result.rs @@ -13,7 +13,7 @@ use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; use database_queries::column_types::record_visibility::RecordVisibility; -use database_queries::w2l::w2l_results::query_w2l_result::select_w2l_result_by_token; +use database_queries::queries::w2l::w2l_results::query_w2l_result::select_w2l_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs index 254b31979d..20a4f93214 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/edit_w2l_template.rs @@ -13,7 +13,7 @@ use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; use database_queries::column_types::record_visibility::RecordVisibility; -use database_queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; +use database_queries::queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs index cd0bb254e4..0204780f6a 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -9,8 +9,8 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use database_queries::w2l::w2l_results::query_w2l_result::W2lResultRecordForResponse; -use database_queries::w2l::w2l_results::query_w2l_result::select_w2l_result_by_token; +use database_queries::queries::w2l::w2l_results::query_w2l_result::W2lResultRecordForResponse; +use database_queries::queries::w2l::w2l_results::query_w2l_result::select_w2l_result_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs index 41c042a24f..ba3f883c4f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -12,8 +12,8 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use database_queries::w2l::w2l_templates::get_w2l_template::W2lTemplateRecordForResponse; -use database_queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; +use database_queries::queries::w2l::w2l_templates::get_w2l_template::W2lTemplateRecordForResponse; +use database_queries::queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs index 2ff15bc617..4c7a0e0236 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/list_w2l_templates.rs @@ -9,7 +9,7 @@ use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; -use database_queries::w2l::w2l_templates::list_w2l_templates::{W2lTemplateRecordForList, list_w2l_templates}; +use database_queries::queries::w2l::w2l_templates::list_w2l_templates::{W2lTemplateRecordForList, list_w2l_templates}; use log::{info, warn, log}; use regex::Regex; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs index 88584e63aa..b6aab394a4 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs @@ -12,7 +12,7 @@ use crate::server_state::ServerState; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; -use database_queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; +use database_queries::queries::w2l::w2l_templates::get_w2l_template::select_w2l_template_by_token; use log::{info, warn, log}; use regex::Regex; use sqlx::error::DatabaseError; diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 53dee6cfd2..8df94e82ab 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -30,8 +30,8 @@ mod job_queries; // NB: Also so sqlx codegens everything. // Not sure if this is strictly necessary. mod shared_queries { - use database_queries::twitch_oauth::find; - use database_queries::twitch_oauth::insert; + use database_queries::queries::twitch::twitch_oauth::find; + use database_queries::queries::twitch::twitch_oauth::insert; } use actix_cors::Cors; diff --git a/crates/service/storyteller_web/src/threads/poll_ip_banlist_thread.rs b/crates/service/storyteller_web/src/threads/poll_ip_banlist_thread.rs index 7706e84201..babb3fe49a 100644 --- a/crates/service/storyteller_web/src/threads/poll_ip_banlist_thread.rs +++ b/crates/service/storyteller_web/src/threads/poll_ip_banlist_thread.rs @@ -1,6 +1,6 @@ use anyhow::Error; use crate::threads::ip_banlist_set::IpBanlistSet; -use database_queries::ip_bans::list_ip_bans::list_ip_bans; +use database_queries::queries::ip_bans::list_ip_bans::list_ip_bans; use log::info; use log::warn; use sqlx::MySqlPool; diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index 879d1e09b0..76982a56e2 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -44,13 +44,13 @@ use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use database_queries::column_types::vocoder_type::VocoderType; use database_queries::mediators::firehose_publisher::FirehosePublisher; -use database_queries::tts::tts_inference_jobs::list_available_tts_inference_jobs::{AvailableTtsInferenceJob, list_available_tts_inference_jobs}; -use database_queries::tts::tts_inference_jobs::mark_tts_inference_job_done::mark_tts_inference_job_done; -use database_queries::tts::tts_inference_jobs::mark_tts_inference_job_failure::mark_tts_inference_job_failure; -use database_queries::tts::tts_inference_jobs::mark_tts_inference_job_pending_and_grab_lock::mark_tts_inference_job_pending_and_grab_lock; -use database_queries::tts::tts_inference_jobs::mark_tts_inference_job_permanently_dead::mark_tts_inference_job_permanently_dead; -use database_queries::tts::tts_models::get_tts_model_for_inference::{get_tts_model_for_inference, TtsModelForInferenceError, TtsModelForInferenceRecord}; -use database_queries::tts::tts_results::insert_tts_result::insert_tts_result; +use database_queries::queries::tts::tts_inference_jobs::list_available_tts_inference_jobs::{AvailableTtsInferenceJob, list_available_tts_inference_jobs}; +use database_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_done::mark_tts_inference_job_done; +use database_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_failure::mark_tts_inference_job_failure; +use database_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_pending_and_grab_lock::mark_tts_inference_job_pending_and_grab_lock; +use database_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_permanently_dead::mark_tts_inference_job_permanently_dead; +use database_queries::queries::tts::tts_models::get_tts_model_for_inference::{get_tts_model_for_inference, TtsModelForInferenceError, TtsModelForInferenceRecord}; +use database_queries::queries::tts::tts_results::insert_tts_result::insert_tts_result; use log::{warn, info, error}; use newrelic_telemetry::Client as NewRelicClient; use newrelic_telemetry::ClientBuilder; From 867e1c0ed501c58482e2a99879704b219444fd09 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 04:34:23 -0500 Subject: [PATCH 0954/1042] huge re-codegen --- sqlx-data.json | 11788 +++++++++++++++++++++++++---------------------- 1 file changed, 6333 insertions(+), 5455 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index f96fddb2c2..a011038df4 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1,5 +1,985 @@ { "db": "MySQL", + "0033c76be0ea1c99069f1a8df1ce400c7064526feaf5dc88930da0718a885b19": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.is_banned as creator_is_banned,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n w2l.title,\n w2l.description_markdown,\n w2l.description_rendered_html,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 4, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 9, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 10, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 11, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 12, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 13, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 15, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 16, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 18, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 19, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 20, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 21, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 23, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 24, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + true, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, + "01f91b5f3c3357aa3397102f52524b4221124caecda2a62f4a4bb31855547b68": { + "query": "\nUPDATE api_tokens\nSET\n deleted_at = CURRENT_TIMESTAMP,\n ip_address_last_update = ?\nWHERE\n user_token = ?\n AND api_token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "05c3ebf54cd8a5374d6f4bbe21b3c6027ce6314917af8acd0ed136d375d53342": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] + } + }, + "089986414149d2d189f4262f8aa7488d35f6a162a4e56f06c1b897092d37ea20": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.is_banned as creator_is_banned,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n w2l.title,\n w2l.description_markdown,\n w2l.description_rendered_html,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.token = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 4, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 9, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 10, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 11, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 12, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 13, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 15, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 16, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 18, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 19, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 20, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 21, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 23, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 24, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + true, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, + "09e655ea4b3504cbb3bc18b92c6c25861c941ba073e3aead414e560b05bd7638": { + "query": "\nSELECT count(*) as count\nFROM tts_models\nWHERE\n creator_user_token = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + } + }, + "10d93ef1350115742d8dddb951f98dfebabf7df3e44577b97b52343f0650eba1": { + "query": "\nSELECT\n badges.slug,\n badges.title,\n badges.description,\n badges.image_url,\n user_badges.created_at as user_created_at\n\nFROM badges\nJOIN user_badges\nON\n badges.slug = user_badges.badge_slug\nWHERE\n user_badges.user_token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 2, + "name": "description", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 3, + "name": "image_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 4, + "name": "user_created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false + ] + } + }, "12a176252df63d1e3a958362d754c828991d64ade427ab11a2025098d1581976": { "query": "\nSELECT\n assignments.model_token AS tts_model_token,\n assignments.category_token\nFROM\n tts_category_assignments AS assignments\nJOIN\n tts_models AS tts\n ON assignments.model_token = tts.token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n AND assignments.deleted_at IS NULL\n\n ", "describe": { @@ -73,6 +1053,265 @@ "nullable": [] } }, + "2b80162b8957ac044f245af93d678aee976535d58fe0de0b39087323aa720792": { + "query": "\nUPDATE tts_models\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "314904656a333caa75a8644e73f4d338b7bd6f68c7ea9e37b5bc7f7b257bf34d": { + "query": "\nSELECT 1 as does_exist\nFROM user_badges\nWHERE\n user_token = ?\nAND\n badge_slug = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "does_exist", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 2 + } + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false + ] + } + }, + "345cbabbe69652c89ccd384ed7db44f70c53791488cf63721f9563291b45db3e": { + "query": "\nINSERT INTO user_sessions (\n token,\n user_token,\n ip_address_creation,\n expires_at\n)\nVALUES ( ?, ?, ?, NOW() + interval 1 year )\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "376e36b1357952599866262c7356ba35f15ccdfb338d8ccdbdc06ccd69331fc3": { + "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c": { + "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "417dd070beeaf6ccd86852b7e99bf7fda5c5b15da4a3bcc066a0e2bb90ca7df1": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.is_public_listing_approved IS TRUE\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "is_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 8 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] + } + }, "443757e80423a52b09f8a4fa79088f6809dad4c1125b1555ce43a382ea745a04": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -552,3678 +1791,6 @@ ] } }, - "4f61f621682ba255814f8a55455ea2b7fba9b6c85affaf87abfe2f81e11322f0": { - "query": "\nSELECT\n id,\n status\nFROM tts_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false - ] - } - }, - "53f0792791317fcd4038308a6630f541d5508ae913cbfa9bc3d4081fba9a048a": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.maybe_default_pretrained_vocoder,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at,\n tts.user_deleted_at,\n tts.mod_deleted_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "maybe_default_pretrained_vocoder", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 3, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "private_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 8, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 9, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 10, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 11, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, - "57cf798a0fe32a1d6d0349ee61baac6e536fe3e1a3aa7bd93b34c519812bf70a": { - "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n model_token,\n raw_inference_text,\n\n creator_ip_address,\n maybe_creator_user_token,\n creator_set_visibility as `creator_set_visibility: crate::column_types::record_visibility::RecordVisibility`,\n\n is_from_api,\n is_for_twitch,\n\n status as `status: crate::column_types::job_status::JobStatus`,\n priority_level,\n attempt_count,\n failure_reason,\n\n created_at,\n updated_at,\n retry_at\nFROM tts_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "inference_job_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "uuid_idempotency_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 3, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "raw_inference_text", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 5, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "creator_set_visibility: crate::column_types::record_visibility::RecordVisibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 8, - "name": "is_from_api", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_for_twitch", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "status: crate::column_types::job_status::JobStatus", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 11, - "name": "priority_level", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 33 - }, - "char_set": 63, - "max_size": 3 - } - }, - { - "ordinal": 12, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 13, - "name": "failure_reason", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 14, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 15, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 16, - "name": "retry_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - true, - false, - false, - true - ] - } - }, - "5951014fbd8f71ea21a8d6f69c23e8b2a449595bf28216b9a9cc87979ea60b05": { - "query": "\nUPDATE api_tokens\nSET\n deleted_at = CURRENT_TIMESTAMP\nWHERE\n user_token = ?\nAND api_token NOT IN (?, ?, ?, ?, ?)\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - } - }, - "5a17891eec52d8304af8a54f6bb7629cb71d3603146f8b2c842f6e1047c31e91": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "is_front_page_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_twitch_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 11, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, - "5b8c906b09cc7068dcad9eb837b1146893b63c3ce7c8dbefea9129849448c3f6": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 15 - }, - "nullable": [] - } - }, - "5e4a00c339ce4d7fac19aff728449156b1e02b9e1d9fd534ac37d4ac79f44820": { - "query": "\nINSERT INTO twitch_bits_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n is_anonymous= ?,\n bits_used = ?,\n total_bits_used = ?,\n chat_message = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 9 - }, - "nullable": [] - } - }, - "5e65d510fdf83315c92cac1ac6acd8c878d7e3e5f050c022c64c246d0af260a2": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "is_front_page_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_twitch_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 11, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, - "5eacdf9c65dc13820077be2930cca7047aed07804ed2e7b2ff38e12a617b4858": { - "query": "\nINSERT INTO twitch_channel_point_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n title = ?,\n prompt = ?,\n user_text_input = ?,\n redemption_id = ?,\n reward_id = ?,\n reward_cost = ?,\n is_sub_only = ?,\n max_per_stream = ?,\n max_per_user_per_stream = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 14 - }, - "nullable": [] - } - }, - "65ce50a3aafe979e77a541d576228d7fcfe76e33400427d01382da36c1585edb": { - "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "9810f6a24480b44ec766e35a1c60daf36f9c8a048bb0192a6069214a695c8dd2": { - "query": "\nSELECT\n api_token,\n maybe_short_description,\n created_at,\n updated_at\nFROM api_tokens\nWHERE\n user_token = ?\n AND deleted_at IS NULL\nORDER BY created_at DESC\nLIMIT 5\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "api_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "maybe_short_description", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1161 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 3, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - true, - false, - false - ] - } - }, - "9e6aa0da19f7e428206c95498f748a30ab428892fb9b17547a7dc60470cc3c09": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "text_preprocessing_algorithm", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 3, - "name": "maybe_default_pretrained_vocoder", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 4, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 5, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, - "name": "creator_is_banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 10, - "name": "description_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 11, - "name": "description_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 12, - "name": "is_front_page_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_twitch_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 15, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 16, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 17, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 18, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 19, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 20, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 21, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 22, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, - "ab6f25eb0f6340d49aa705835ed428434d8b6ab14b0740f6a5ebe7841b9ece20": { - "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, - "b06e373275eb6d6f445abba38b8129d53196527bf953a50c85ccb93c23d2e4d6": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "is_front_page_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_twitch_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 11, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, - "b8746ae4ff2d4dcd54563eff64877f924bb119681ad78f1300146e00b762ca4f": { - "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n maybe_pretrained_vocoder_used = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n\n file_size_bytes = ?,\n duration_millis = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 14 - }, - "nullable": [] - } - }, - "c638431e2bcdef1e14d5ef35d571df717a8d3782e15768dfbadb38ac7e94a674": { - "query": "\nINSERT INTO api_tokens\nSET\n api_token = ?,\n user_token = ?,\n uuid_idempotency_token = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "d7ac9f84e92d6b2111162a674284c0b9fa50c718bedfb8ab7238ca0a7b0875e6": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 17 - }, - "nullable": [] - } - }, - "def4202f63874e7b04d8f8a5da028a9aacc4dfe19b8898d3d6820d393a633cf3": { - "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n is_from_api = ?,\n is_for_twitch = ?,\n priority_level = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 10 - }, - "nullable": [] - } - }, - "e0d442313b7861debaef984c5809647bd5df85106c194b66e864674b6af95330": { - "query": "\nUPDATE tts_inference_jobs\nSET\n status = \"dead\",\n failure_reason = ?,\n retry_at = NULL\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "f19300f4e8434c2669dac062e5a79c5ca71767c7380aab0ab40b51be8ff449d8": { - "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 1 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "0033c76be0ea1c99069f1a8df1ce400c7064526feaf5dc88930da0718a885b19": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.is_banned as creator_is_banned,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n w2l.title,\n w2l.description_markdown,\n w2l.description_rendered_html,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_is_banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 4, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 8, - "name": "description_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 9, - "name": "description_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 10, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 11, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 12, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 13, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 14, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 15, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 16, - "name": "is_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 8 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 17, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 18, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 19, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 20, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 21, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 22, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 23, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 24, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, - "02060ab9b8ca472a2bd4d4fa3120743e90148ccd6a4cb13bc491d00f36ed7a8f": { - "query": "\nSELECT\n id,\n status\nFROM w2l_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false - ] - } - }, - "05c3ebf54cd8a5374d6f4bbe21b3c6027ce6314917af8acd0ed136d375d53342": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 7, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 8, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 9, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "is_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 8 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false - ] - } - }, - "05e0a883bcf6c0563bbb7cad4a5bb357b502b7134d6633a04c993d9f74c8923c": { - "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n maybe_w2l_template_token,\n maybe_public_image_bucket_location,\n maybe_tts_inference_result_token,\n maybe_public_audio_bucket_hash,\n maybe_public_audio_bucket_location,\n\n maybe_original_audio_filename,\n maybe_original_audio_download_url,\n maybe_audio_mime_type,\n\n creator_ip_address,\n maybe_creator_user_token,\n\n creator_set_visibility as `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n disable_end_bump,\n disable_watermark,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM w2l_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "inference_job_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "uuid_idempotency_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 3, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "maybe_public_image_bucket_location", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 5, - "name": "maybe_tts_inference_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "maybe_public_audio_bucket_hash", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 7, - "name": "maybe_public_audio_bucket_location", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 8, - "name": "maybe_original_audio_filename", - "type_info": { - "type": "String", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 9, - "name": "maybe_original_audio_download_url", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 10, - "name": "maybe_audio_mime_type", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 11, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 12, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 13, - "name": "creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 14, - "name": "disable_end_bump", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 15, - "name": "disable_watermark", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 16, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 17, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 18, - "name": "failure_reason", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 19, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 21, - "name": "retry_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - false, - true, - false, - false, - false, - false, - false, - true, - false, - false, - true - ] - } - }, - "089986414149d2d189f4262f8aa7488d35f6a162a4e56f06c1b897092d37ea20": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.is_banned as creator_is_banned,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n w2l.title,\n w2l.description_markdown,\n w2l.description_rendered_html,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.creator_set_visibility,\n w2l.is_public_listing_approved,\n w2l.is_locked_from_use,\n w2l.is_locked_from_user_modification,\n w2l.creator_ip_address_creation,\n w2l.creator_ip_address_last_update,\n w2l.created_at,\n w2l.updated_at,\n w2l.user_deleted_at,\n w2l.mod_deleted_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.token = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_is_banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 4, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 6, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 8, - "name": "description_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 9, - "name": "description_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 10, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 11, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 12, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 13, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 14, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 15, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 16, - "name": "is_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 8 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 17, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 18, - "name": "is_locked_from_user_modification", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 19, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 20, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 21, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 22, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 23, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 24, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - false, - false, - false, - false, - false, - true, - true - ] - } - }, - "09e655ea4b3504cbb3bc18b92c6c25861c941ba073e3aead414e560b05bd7638": { - "query": "\nSELECT count(*) as count\nFROM tts_models\nWHERE\n creator_user_token = ?\nLIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 129 - }, - "char_set": 63, - "max_size": 21 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false - ] - } - }, - "0caed749627ff22ae1cea52621a375156259a731e8070459593082b80d463491": { - "query": "\nSELECT\n id,\n status\nFROM tts_model_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false - ] - } - }, - "0ff13b84956c2625050211a5bb2352a489ef8435571c364c8b2d1d21d936cbd4": { - "query": "\nUPDATE tts_results\nSET\n creator_set_visibility = ?,\n maybe_mod_user_token = ?\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "10d93ef1350115742d8dddb951f98dfebabf7df3e44577b97b52343f0650eba1": { - "query": "\nSELECT\n badges.slug,\n badges.title,\n badges.description,\n badges.image_url,\n user_badges.created_at as user_created_at\n\nFROM badges\nJOIN user_badges\nON\n badges.slug = user_badges.badge_slug\nWHERE\n user_badges.user_token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 2, - "name": "description", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 3, - "name": "image_url", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 4, - "name": "user_created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false - ] - } - }, - "1168f39a5ec892d06e941b0dba0e955a27ef976567eeafd6e550bc42ea4c4331": { - "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 7 - }, - "nullable": [] - } - }, - "1236640a30d0d5177e13c31b61b791874f7c70026003a58ba6e07e7af46c30c1": { - "query": "\nSELECT\n next_id\nFROM\n w2l_result_synthetic_ids\nWHERE\n user_token = ?\nLIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "next_id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 20 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false - ] - } - }, - "17c8d210d93ee9c0d8e8422eed605c9150f19f9bf72affcf111b15a67a5b9e70": { - "query": "\nUPDATE w2l_templates\nSET\n is_public_listing_approved = ?,\n is_locked_from_user_modification = ?,\n is_locked_from_use = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - } - }, - "1cc62e5cdc8de18969d7e8018d91c38794e5a7d23f117e5b0a4126845f300e60": { - "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE\n token = ?\n AND deleted_at IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, - "1d0f302c0b7c4301000c024d6bbae4d3b7370ed7c944e4c557c64b0bd2f31c14": { - "query": "\nSELECT public_count, all_count FROM (\n (SELECT count(*) as public_count \n FROM tts_models \n WHERE user_deleted_at IS NULL \n AND mod_deleted_at IS NULL) as pc,\n (SELECT count(*) as all_count \n FROM tts_models) as ac\n)\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "public_count", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 21 - } - }, - { - "ordinal": 1, - "name": "all_count", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 21 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false - ] - } - }, - "218b3fe667b34bfa158bb7d4f600a3b91a22460adf566dcdf6ce540006e6c684": { - "query": "\nUPDATE model_categories\nSET\n name = ?,\n maybe_dropdown_name = ?,\n\n can_directly_have_models = ?,\n can_have_subcategories = ?,\n can_only_mods_apply = ?,\n\n maybe_super_category_token = ?,\n\n is_mod_approved = ?,\n maybe_mod_user_token = ?,\n maybe_mod_comments = ?,\n\n version = version + 1\n\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 10 - }, - "nullable": [] - } - }, - "25e7858d370ac1359deb42ef5292e85ccd1fe7debf3a1ba73303e5882523a4d0": { - "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n download_url = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 8 - }, - "nullable": [] - } - }, - "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { - "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "2b04dd7db52ef3a97a6f420a22a4dbee7416ae58697438df69e87ffd47bbf350": { - "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "2b80162b8957ac044f245af93d678aee976535d58fe0de0b39087323aa720792": { - "query": "\nUPDATE tts_models\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "314904656a333caa75a8644e73f4d338b7bd6f68c7ea9e37b5bc7f7b257bf34d": { - "query": "\nSELECT 1 as does_exist\nFROM user_badges\nWHERE\n user_token = ?\nAND\n badge_slug = ?\nLIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "does_exist", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 129 - }, - "char_set": 63, - "max_size": 1 - } - } - ], - "parameters": { - "Right": 2 - }, - "nullable": [ - false - ] - } - }, - "33cd0c0b8eaaf494ef97232ac89775b8eb6a22d310824b7aa7db310dd4fd41b1": { - "query": "\nUPDATE w2l_results\nSET\n creator_set_visibility = ?,\n maybe_mod_user_token = ?\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { - "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 12 - }, - "nullable": [] - } - }, - "345cbabbe69652c89ccd384ed7db44f70c53791488cf63721f9563291b45db3e": { - "query": "\nINSERT INTO user_sessions (\n token,\n user_token,\n ip_address_creation,\n expires_at\n)\nVALUES ( ?, ?, ?, NOW() + interval 1 year )\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "34642399c3e91960768cb659b6006db291574d9cd07a5bace3db573eff20c3fb": { - "query": "\nINSERT INTO w2l_result_synthetic_ids\nSET\n user_token = ?,\n next_id = 1\nON DUPLICATE KEY UPDATE\n user_token = ?,\n next_id = next_id + 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "346c02b215cb35efaa35fe96d866a8c9056ac46e987b4e01f64222259dccdcbf": { - "query": "\nUPDATE\n ip_address_bans\nSET\n mod_user_token = ?,\n deleted_at = NULL\nWHERE\n ip_address = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "369a7576edc128cf2b310648d6fc45a2191c1263d433fe6d1461d15dbf26690e": { - "query": "\nINSERT INTO w2l_results\nSET\n token = ?,\n maybe_w2l_template_token = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n maybe_creator_synthetic_id = ?,\n\n public_bucket_video_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n duration_millis = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 12 - }, - "nullable": [] - } - }, - "376e36b1357952599866262c7356ba35f15ccdfb338d8ccdbdc06ccd69331fc3": { - "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "3a2dc0a7e41dcbdf21f91d1a1e54e813f74b0bac7aea052302089cae195eeb53": { - "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 9 - }, - "nullable": [] - } - }, - "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c": { - "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "417dd070beeaf6ccd86852b7e99bf7fda5c5b15da4a3bcc066a0e2bb90ca7df1": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.is_public_listing_approved IS TRUE\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 7, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 8, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 9, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "is_public_listing_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 8 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false - ] - } - }, - "45c86917f422dc22026ba4ba31251cbd784ff1f13709e533048c8a4e5c438485": { - "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_confirmed,\n users.email_gravatar_hash,\n\n users.disable_gravatar,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n users.preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n\n users.user_role_slug,\n users.is_banned,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l,\n user_roles.can_delete_own_tts_results,\n user_roles.can_delete_own_w2l_results,\n user_roles.can_delete_own_account,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n user_roles.can_delete_own_tts_models,\n user_roles.can_delete_own_w2l_templates,\n\n user_roles.can_approve_w2l_templates,\n user_roles.can_edit_other_users_profiles,\n user_roles.can_edit_other_users_tts_models,\n user_roles.can_edit_other_users_w2l_templates,\n user_roles.can_delete_other_users_tts_models,\n user_roles.can_delete_other_users_tts_results,\n user_roles.can_delete_other_users_w2l_templates,\n user_roles.can_delete_other_users_w2l_results,\n user_roles.can_ban_users,\n user_roles.can_delete_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "email_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "email_confirmed", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 5, - "name": "email_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "disable_gravatar", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 7, - "name": "auto_play_audio_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "auto_play_video_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 10, - "name": "preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 11, - "name": "user_role_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 12, - "name": "is_banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "can_use_tts", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "can_use_w2l", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 15, - "name": "can_delete_own_tts_results", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 16, - "name": "can_delete_own_w2l_results", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 17, - "name": "can_delete_own_account", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 18, - "name": "can_upload_tts_models", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 19, - "name": "can_upload_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 20, - "name": "can_delete_own_tts_models", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 21, - "name": "can_delete_own_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 22, - "name": "can_approve_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 23, - "name": "can_edit_other_users_profiles", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 24, - "name": "can_edit_other_users_tts_models", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 25, - "name": "can_edit_other_users_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 26, - "name": "can_delete_other_users_tts_models", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 27, - "name": "can_delete_other_users_tts_results", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 28, - "name": "can_delete_other_users_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 29, - "name": "can_delete_other_users_w2l_results", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 30, - "name": "can_ban_users", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 31, - "name": "can_delete_users", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - true - ] - } - }, "4c7a92b6ac43e271096138f3594ca27b0ab6622b1e29a28f4a034c7732be9312": { "query": "\nSELECT\n users.username,\n users.display_name,\n users.email_gravatar_hash as gravatar_hash,\n creator_user_token,\n count(*) as uploaded_count\nFROM w2l_templates\nJOIN users\nON\n users.token = creator_user_token\nWHERE\n is_public_listing_approved IS TRUE\n AND users.is_banned IS FALSE\n AND w2l_templates.user_deleted_at IS NULL\n AND w2l_templates.mod_deleted_at IS NULL\nGROUP BY creator_user_token\nORDER BY uploaded_count desc\nLIMIT 25;\n ", "describe": { @@ -4301,6 +1868,121 @@ ] } }, + "4df4765816d8c431a241b5ea2e7e9b292bf8e603937b095fe256cbc8a7a165f6": { + "query": "\nSELECT\n internal_token,\n api_token,\n maybe_short_description,\n created_at,\n updated_at\nFROM api_tokens\nWHERE\n user_token = ?\n AND deleted_at IS NULL\nORDER BY created_at DESC\nLIMIT 5\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "internal_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "api_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "maybe_short_description", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1161 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + true, + false, + false + ] + } + }, + "4f61f621682ba255814f8a55455ea2b7fba9b6c85affaf87abfe2f81e11322f0": { + "query": "\nSELECT\n id,\n status\nFROM tts_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + }, "4ff1b1e0cc36f6c2f1a9b099f38751ae7f35666be939b4074bdfc98e031cd6ac": { "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n users.is_banned as maybe_creator_is_banned,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n template_users.is_banned as maybe_template_creator_is_banned,\n\n w2l_results.creator_set_visibility,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at,\n\n w2l_results.creator_ip_address,\n w2l_results.user_deleted_at,\n w2l_results.mod_deleted_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.token = ?\n AND w2l_results.user_deleted_at IS NULL\n AND w2l_results.mod_deleted_at IS NULL\n ", "describe": { @@ -4518,7 +2200,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -4530,7 +2212,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -4542,7 +2224,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -4554,7 +2236,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -4651,33 +2333,13 @@ ] } }, - "5473d3857478d001688998056377fefe4bcc792477f303b41fa3cf64f79170f3": { - "query": "\n INSERT INTO tts_category_assignments\n SET\n model_token = ?,\n category_token = ?,\n category_removal_user_token = ?,\n ip_address_creation = ?,\n ip_address_last_update = ?,\n deleted_at = CURRENT_TIMESTAMP\n ON DUPLICATE KEY UPDATE\n model_token = ?,\n category_token = ?,\n category_removal_user_token = ?,\n ip_address_last_update = ?,\n deleted_at = CURRENT_TIMESTAMP\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 9 - }, - "nullable": [] - } - }, - "551749bc153f9c8683a3a1ca5bb50744322aac046db34327501cf991485c4dee": { - "query": "\nUPDATE tts_models\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "56a0a2bfc3c73d6845ca9cf1d285b175a40b50a250b7effb9ce6474e58113d6e": { - "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_model_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_model_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", + "53f0792791317fcd4038308a6630f541d5508ae913cbfa9bc3d4081fba9a048a": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.maybe_default_pretrained_vocoder,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at,\n tts.user_deleted_at,\n tts.mod_deleted_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "job_token", + "name": "model_token", "type_info": { "type": "VarString", "flags": { @@ -4689,35 +2351,35 @@ }, { "ordinal": 1, - "name": "status", + "name": "tts_model_type", "type_info": { "type": "String", "flags": { - "bits": 393 + "bits": 385 }, "char_set": 224, - "max_size": 64 + "max_size": 60 } }, { "ordinal": 2, - "name": "attempt_count", + "name": "maybe_default_pretrained_vocoder", "type_info": { - "type": "Long", + "type": "VarString", "flags": { - "bits": 1 + "bits": 128 }, - "char_set": 63, - "max_size": 11 + "char_set": 224, + "max_size": 256 } }, { "ordinal": 3, - "name": "maybe_model_token", + "name": "creator_user_token", "type_info": { "type": "VarString", "flags": { - "bits": 136 + "bits": 4233 }, "char_set": 224, "max_size": 128 @@ -4725,135 +2387,33 @@ }, { "ordinal": 4, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 5, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - false, - false - ] - } - }, - "57928b1001a10a88ac5a07f43b1dd30f6dee6edbc0808be99fe1233f9e276c56": { - "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_video_path as maybe_public_bucket_video_path,\n\n jobs.maybe_w2l_template_token,\n w2l.template_type as w2l_template_type,\n w2l.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_inference_jobs as jobs\nJOIN w2l_templates as w2l\n ON w2l.token = jobs.maybe_w2l_template_token\nLEFT OUTER JOIN w2l_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_token", + "name": "creator_username", "type_info": { "type": "VarString", "flags": { "bits": 4229 }, "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 2, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 3, - "name": "maybe_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "maybe_public_bucket_video_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 + "max_size": 80 } }, { "ordinal": 5, - "name": "maybe_w2l_template_token", + "name": "creator_display_name", "type_info": { "type": "VarString", "flags": { - "bits": 136 + "bits": 4225 }, "char_set": 224, - "max_size": 128 + "max_size": 80 } }, { "ordinal": 6, - "name": "w2l_template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 7, "name": "title", "type_info": { - "type": "String", + "type": "VarString", "flags": { "bits": 4225 }, @@ -4861,6 +2421,18 @@ "max_size": 1020 } }, + { + "ordinal": 7, + "name": "private_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 256 + } + }, { "ordinal": 8, "name": "created_at", @@ -4884,18 +2456,455 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 10, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 11, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + true, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, + "551749bc153f9c8683a3a1ca5bb50744322aac046db34327501cf991485c4dee": { + "query": "\nUPDATE tts_models\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "57cf798a0fe32a1d6d0349ee61baac6e536fe3e1a3aa7bd93b34c519812bf70a": { + "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n model_token,\n raw_inference_text,\n\n creator_ip_address,\n maybe_creator_user_token,\n creator_set_visibility as `creator_set_visibility: crate::column_types::record_visibility::RecordVisibility`,\n\n is_from_api,\n is_for_twitch,\n\n status as `status: crate::column_types::job_status::JobStatus`,\n priority_level,\n attempt_count,\n failure_reason,\n\n created_at,\n updated_at,\n retry_at\nFROM tts_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "inference_job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "creator_set_visibility: crate::column_types::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 8, + "name": "is_from_api", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_for_twitch", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "status: crate::column_types::job_status::JobStatus", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 11, + "name": "priority_level", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 33 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 12, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 13, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 14, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 15, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 16, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { "Right": 1 }, "nullable": [ + false, + false, + false, false, false, false, true, + false, + false, + false, + false, + false, + false, true, - true, + false, + false, + true + ] + } + }, + "5a17891eec52d8304af8a54f6bb7629cb71d3603146f8b2c842f6e1047c31e91": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 11, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, false, false, false, @@ -4903,6 +2912,26 @@ ] } }, + "5b8c906b09cc7068dcad9eb837b1146893b63c3ce7c8dbefea9129849448c3f6": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 15 + }, + "nullable": [] + } + }, + "5e4a00c339ce4d7fac19aff728449156b1e02b9e1d9fd534ac37d4ac79f44820": { + "query": "\nINSERT INTO twitch_bits_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n is_anonymous= ?,\n bits_used = ?,\n total_bits_used = ?,\n chat_message = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, "5e54fd8569b2d6d3a0d11acee6bd03b75400b6325952eb16c8dac68a7933b13a": { "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n is_banned,\n disable_gravatar,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", "describe": { @@ -5110,23 +3139,13 @@ ] } }, - "60c06a507349919addb5d26b1c4a283a04f66fce86e84ed0a51db6001c9fc2b5": { - "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - } - }, - "619d18c87075a86c01f1e0ac26744774293ff8ca6b0011ce611b3f47d8a97a0e": { - "query": "\nSELECT\n w2l_templates.token as template_token,\n w2l_templates.title,\n w2l_templates.template_type,\n w2l_templates.duration_millis,\n w2l_templates.frame_width,\n w2l_templates.frame_height,\n w2l_templates.creator_user_token,\n users.username AS creator_username,\n users.display_name AS creator_display_name,\n users.email_gravatar_hash AS creator_gravatar_hash,\n w2l_templates.created_at\nFROM\n w2l_templates\nJOIN\n users\nON\n users.token = w2l_templates.creator_user_token\nWHERE\n w2l_templates.is_public_listing_approved IS NULL\n AND w2l_templates.user_deleted_at IS NULL\n AND w2l_templates.mod_deleted_at IS NULL\n AND w2l_templates.is_locked_from_use IS FALSE\n ", + "5e65d510fdf83315c92cac1ac6acd8c878d7e3e5f050c022c64c246d0af260a2": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "template_token", + "name": "model_token", "type_info": { "type": "VarString", "flags": { @@ -5138,66 +3157,18 @@ }, { "ordinal": 1, - "name": "title", + "name": "tts_model_type", "type_info": { "type": "String", "flags": { - "bits": 4225 + "bits": 385 }, "char_set": 224, - "max_size": 1020 + "max_size": 60 } }, { "ordinal": 2, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 3, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 4, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 5, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 6, "name": "creator_user_token", "type_info": { "type": "VarString", @@ -5209,7 +3180,7 @@ } }, { - "ordinal": 7, + "ordinal": 3, "name": "creator_username", "type_info": { "type": "VarString", @@ -5221,7 +3192,7 @@ } }, { - "ordinal": 8, + "ordinal": 4, "name": "creator_display_name", "type_info": { "type": "VarString", @@ -5233,7 +3204,7 @@ } }, { - "ordinal": 9, + "ordinal": 5, "name": "creator_gravatar_hash", "type_info": { "type": "String", @@ -5244,6 +3215,54 @@ "max_size": 128 } }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, { "ordinal": 10, "name": "created_at", @@ -5255,6 +3274,18 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 11, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { @@ -5271,16 +3302,27 @@ false, false, false, + false, false ] } }, - "641253d4c621f75037877651bba044a21033f50a5e3f8fed9b06ec967a508853": { - "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "5eacdf9c65dc13820077be2930cca7047aed07804ed2e7b2ff38e12a617b4858": { + "query": "\nINSERT INTO twitch_channel_point_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n title = ?,\n prompt = ?,\n user_text_input = ?,\n redemption_id = ?,\n reward_id = ?,\n reward_cost = ?,\n is_sub_only = ?,\n max_per_stream = ?,\n max_per_user_per_stream = ?\n ", "describe": { "columns": [], "parameters": { - "Right": 1 + "Right": 14 + }, + "nullable": [] + } + }, + "65ce50a3aafe979e77a541d576228d7fcfe76e33400427d01382da36c1585edb": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 }, "nullable": [] } @@ -5295,58 +3337,8 @@ "nullable": [] } }, - "68376f750b8421c10d319fa1045a2f3ce8fb1d35a5fe6ec92b2022babafbde80": { - "query": "\nUPDATE users\nSET\n user_role_slug = ?\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "6aaa7c04e6984434bd2feb6f42a9014cd45e995b94e4d5e29f2d502f09136691": { - "query": "\nUPDATE tts_models\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "6b66db376952ff4cfcea6ffe1f9ecd4462722843a3d19fcd4df16f0fb993decc": { - "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, - "720d7a26fb8ce75a8a1b3a62ef509d8ca99d419172e9cc5d8e2c803e0eebcb7f": { - "query": "\n UPDATE model_categories\n SET\n deleted_at = NULL\n WHERE\n token = ?\n LIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, - "73558104936aab6add2d9c789b368b5f155f1fe01d32e7314f2f120422866f54": { - "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "7490dcb8de10b1eda2c857e53ebc5c0465cdb589d3df4c5549859f7919acac77": { - "query": "\nSELECT token, username, email_address, password_hash, is_banned\nFROM users\nWHERE email_address = ?\nLIMIT 1\n ", + "6768d25558d29345eeb5aa43285485bf7b23e90834b27aacdc2da700a3440a03": { + "query": "\nSELECT\n token,\n\n maybe_user_token,\n email_address,\n discord_username,\n\n is_for_studio,\n is_for_twitch_tts,\n is_for_api_use,\n is_for_music,\n is_for_games,\n is_for_other,\n optional_notes_on_use,\n\n is_for_private_use,\n is_for_public_use,\n\n is_own_voice,\n is_third_party_voice,\n\n has_clean_audio_recordings,\n has_good_microphone,\n\n optional_questions,\n optional_extra_comments,\n\n created_at,\n updated_at\nFROM voice_clone_requests\nWHERE\n token = ?\n ", "describe": { "columns": [ { @@ -5363,14 +3355,14 @@ }, { "ordinal": 1, - "name": "username", + "name": "maybe_user_token", "type_info": { "type": "VarString", "flags": { - "bits": 4229 + "bits": 136 }, "char_set": 224, - "max_size": 80 + "max_size": 128 } }, { @@ -5379,7 +3371,7 @@ "type_info": { "type": "VarString", "flags": { - "bits": 4229 + "bits": 4225 }, "char_set": 224, "max_size": 1020 @@ -5387,19 +3379,19 @@ }, { "ordinal": 3, - "name": "password_hash", + "name": "discord_username", "type_info": { - "type": "String", + "type": "VarString", "flags": { "bits": 4225 }, - "char_set": 63, - "max_size": 60 + "char_set": 224, + "max_size": 1020 } }, { "ordinal": 4, - "name": "is_banned", + "name": "is_for_studio", "type_info": { "type": "Tiny", "flags": { @@ -5408,34 +3400,197 @@ "char_set": 63, "max_size": 1 } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false - ] - } - }, - "7622292c05fe1b5832136762042780a12a02048166bb1b4492dfd627c48c8968": { - "query": "\nSELECT\n token as template_token\nFROM w2l_templates\nWHERE\n token = ?\n AND user_deleted_at IS NULL\n AND mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ + }, { - "ordinal": 0, - "name": "template_token", + "ordinal": 5, + "name": "is_for_twitch_tts", "type_info": { - "type": "VarString", + "type": "Tiny", "flags": { - "bits": 4229 + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 6, + "name": "is_for_api_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 7, + "name": "is_for_music", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "is_for_games", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_for_other", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "optional_notes_on_use", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 }, "char_set": 224, - "max_size": 128 + "max_size": 67108860 + } + }, + { + "ordinal": 11, + "name": "is_for_private_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "is_for_public_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "is_own_voice", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "is_third_party_voice", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "has_clean_audio_recordings", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "has_good_microphone", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "optional_questions", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 18, + "name": "optional_extra_comments", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 19, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 } } ], @@ -5443,10 +3598,40 @@ "Right": 1 }, "nullable": [ + false, + true, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + true, + true, + false, false ] } }, + "6aaa7c04e6984434bd2feb6f42a9014cd45e995b94e4d5e29f2d502f09136691": { + "query": "\nUPDATE tts_models\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "78006a7a9388557d53c9a9cc8aeec84496e87355501c5b412f788a2291077473": { "query": "\nSELECT\n slug,\n name,\n\n can_use_tts,\n can_use_w2l,\n can_delete_own_tts_results,\n can_delete_own_w2l_results,\n can_delete_own_account,\n\n can_upload_tts_models,\n can_upload_w2l_templates,\n can_delete_own_tts_models,\n can_delete_own_w2l_templates,\n\n can_approve_w2l_templates,\n can_edit_other_users_profiles,\n can_edit_other_users_tts_models,\n can_edit_other_users_w2l_templates,\n can_delete_other_users_tts_models,\n can_delete_other_users_tts_results,\n can_delete_other_users_w2l_templates,\n can_delete_other_users_w2l_results,\n can_ban_users,\n can_delete_users,\n\n created_at,\n updated_at\n\nFROM user_roles\n ", "describe": { @@ -5758,226 +3943,6 @@ ] } }, - "7ab54466ca3021d90e0feb234acba56bf53d415dde69bc1c35120ed0c8dd1858": { - "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "uuid_idempotency_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 3, - "name": "on_success_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 5, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 6, - "name": "download_url", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 7, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 9, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 10, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 11, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 12, - "name": "failure_reason", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 13, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 14, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 15, - "name": "retry_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - true, - false, - false, - true, - false, - false, - false, - false, - false, - true, - false, - false, - true - ] - } - }, "7cd0ad5d7b9a2c393f799beadc1e0a6bc85d952e07a1b82a52115b99be40ff6d": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\n ON users.token = w2l.creator_user_token\nWHERE\n w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -6063,7 +4028,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -6075,7 +4040,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -6087,7 +4052,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -6454,13 +4419,33 @@ ] } }, - "88328e04ce947d3f541951aada9cee25135aa950ab478cf0dc6de453bb079e17": { - "query": "\nSELECT\n events.token as event_token,\n events.event_type,\n events.maybe_target_user_token,\n users.username as maybe_target_username,\n users.display_name as maybe_target_display_name,\n users.email_gravatar_hash as maybe_target_user_gravatar_hash,\n events.maybe_target_entity_token,\n events.created_at,\n events.updated_at\nFROM firehose_entries as events\nLEFT OUTER JOIN users\nON events.maybe_target_user_token = users.token\nORDER BY events.id DESC\nLIMIT 25\n ", + "8ff27fc5583db97f157f1b808f593a72c77dc5dcf5a633cd9c1162af6b97c011": { + "query": "\nUPDATE w2l_templates\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "9134347abc2b89b7336394894ed420db725be4efebd1c36e959652a10daf8551": { + "query": "\nUPDATE twitch_event_rules\nSET\n deleted_at = CURRENT_TIMESTAMP,\n ip_address_last_update = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "91e88859f2975447b7232264b6fbf98e35064531b7bc1ffe66552a8d7fee8bed": { + "query": "\nSELECT\n token,\n event_category as `event_category: crate::column_types::twitch_event_category::TwitchEventCategory`,\n event_match_predicate,\n event_response,\n user_specified_rule_order,\n rule_is_disabled,\n created_at,\n updated_at,\n deleted_at\nFROM twitch_event_rules\nWHERE\n token = ?\n AND user_token = ?\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "event_token", + "name": "token", "type_info": { "type": "VarString", "flags": { @@ -6472,78 +4457,66 @@ }, { "ordinal": 1, - "name": "event_type", + "name": "event_category: crate::column_types::twitch_event_category::TwitchEventCategory", "type_info": { - "type": "VarString", + "type": "String", "flags": { - "bits": 4233 + "bits": 4489 }, "char_set": 224, - "max_size": 128 + "max_size": 56 } }, { "ordinal": 2, - "name": "maybe_target_user_token", + "name": "event_match_predicate", "type_info": { - "type": "VarString", + "type": "Blob", "flags": { - "bits": 136 + "bits": 4241 }, "char_set": 224, - "max_size": 128 + "max_size": 67108860 } }, { "ordinal": 3, - "name": "maybe_target_username", + "name": "event_response", "type_info": { - "type": "VarString", + "type": "Blob", "flags": { - "bits": 4228 + "bits": 4241 }, "char_set": 224, - "max_size": 80 + "max_size": 67108860 } }, { "ordinal": 4, - "name": "maybe_target_display_name", + "name": "user_specified_rule_order", "type_info": { - "type": "VarString", + "type": "Long", "flags": { - "bits": 4224 + "bits": 33 }, - "char_set": 224, - "max_size": 80 + "char_set": 63, + "max_size": 10 } }, { "ordinal": 5, - "name": "maybe_target_user_gravatar_hash", + "name": "rule_is_disabled", "type_info": { - "type": "String", + "type": "Tiny", "flags": { - "bits": 4224 + "bits": 1 }, - "char_set": 224, - "max_size": 128 + "char_set": 63, + "max_size": 1 } }, { "ordinal": 6, - "name": "maybe_target_entity_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, "name": "created_at", "type_info": { "type": "Timestamp", @@ -6555,7 +4528,7 @@ } }, { - "ordinal": 8, + "ordinal": 7, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -6565,54 +4538,36 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 8, + "name": "deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 136 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { - "Right": 0 + "Right": 2 }, "nullable": [ false, false, - true, - true, - true, - true, - true, false, - false + false, + false, + false, + false, + false, + true ] } }, - "886861e91c7e8af53eb856c949389a172b8d9e079f1c2070d115423907c47c50": { - "query": "\nINSERT INTO tts_model_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n download_url = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 8 - }, - "nullable": [] - } - }, - "8b1b9cf11b80da18d79237f2579a96b4a85fa7201118e01fffc01c1eba64cbaf": { - "query": "\n UPDATE model_categories\n SET\n deleted_at = CURRENT_TIMESTAMP\n WHERE\n token = ?\n LIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } - }, - "8ff27fc5583db97f157f1b808f593a72c77dc5dcf5a633cd9c1162af6b97c011": { - "query": "\nUPDATE w2l_templates\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, "961f792c24bee6009e3e8a215561db932871a53c48839b8cb80292de00e7b2e1": { "query": "\nUPDATE tts_results\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -6623,83 +4578,13 @@ "nullable": [] } }, - "9a879f76345ef5682b995f99026fac8b2f4c64c8ec4bb078187d1e53effcfbaf": { - "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address_creation = ?,\n creator_ip_address_last_update = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n maybe_public_bucket_preview_image_object_name = ?,\n maybe_public_bucket_preview_video_object_name = ?,\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 19 - }, - "nullable": [] - } - }, - "9b607b814cd7c445c2c46ec5c13dc23f3a6a771a3469a786ed46ec099577d95e": { - "query": "\nUPDATE\n ip_address_bans\nSET\n mod_user_token = ?,\n deleted_at = CURRENT_TIMESTAMP\nWHERE\n ip_address = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, - "9c04e83ce78dafd7b84edf057f30a834d459f710d0836dcf7f211f7cd17e24e3": { - "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 7 - }, - "nullable": [] - } - }, - "9cc5f697350814199344d3d9a0ff026e0d1e09e9097648400d4b63cfcbd31178": { - "query": "\nUPDATE w2l_results\nSET\n creator_set_visibility = ?,\n creator_ip_address = ?\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "9cef4989671fb55d8e732f6f3d5c16aea7a9b3e01fc6a3bc7a56d0b066f07730": { - "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address_creation = ?,\n creator_ip_address_last_update = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n file_size_bytes = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 9 - }, - "nullable": [] - } - }, - "9d4932baa4023accc2d28af98b58a47149d37c4937cd0171e6b955c09855c5f8": { - "query": "\nINSERT INTO w2l_inference_jobs\nSET\n maybe_w2l_template_token = ?,\n maybe_tts_inference_result_token = ?,\n maybe_public_audio_bucket_location = NULL,\n maybe_public_image_bucket_location = NULL,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - } - }, - "a125f183f7ae5627fc2118e7302cea73d967200ba1cbb12550f5880187b7db5e": { - "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "a2b586dd2120f29dd26281626f7381e292c821f8e826b9bc584b92ffa45035b2": { - "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,\n\n jobs.model_token,\n tts.tts_model_type,\n tts.title,\n\n jobs.raw_inference_text,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_inference_jobs as jobs\nJOIN tts_models as tts\n ON tts.token = jobs.model_token\nLEFT OUTER JOIN tts_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", + "9e6aa0da19f7e428206c95498f748a30ab428892fb9b17547a7dc60470cc3c09": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "job_token", + "name": "model_token", "type_info": { "type": "VarString", "flags": { @@ -6711,66 +4596,6 @@ }, { "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 2, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 3, - "name": "maybe_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "maybe_public_bucket_wav_audio_path", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 5, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, "name": "tts_model_type", "type_info": { "type": "String", @@ -6781,8 +4606,92 @@ "max_size": 60 } }, + { + "ordinal": 2, + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "maybe_default_pretrained_vocoder", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 4, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, { "ordinal": 7, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "creator_is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, "name": "title", "type_info": { "type": "VarString", @@ -6794,8 +4703,8 @@ } }, { - "ordinal": 8, - "name": "raw_inference_text", + "ordinal": 10, + "name": "description_markdown", "type_info": { "type": "Blob", "flags": { @@ -6806,7 +4715,79 @@ } }, { - "ordinal": 9, + "ordinal": 11, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 12, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 15, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, "name": "created_at", "type_info": { "type": "Timestamp", @@ -6818,7 +4799,7 @@ } }, { - "ordinal": 10, + "ordinal": 18, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -6828,6 +4809,54 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 19, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 20, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 21, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } } ], "parameters": { @@ -6838,13 +4867,25 @@ false, false, true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, true, - false, - false, - false, - false, - false, - false + true ] } }, @@ -6925,60 +4966,22 @@ ] } }, - "a9173fd8148645bda75529aae0af63dcfb7cf9e114604ce4dcc962974e7268e1": { - "query": "\nSELECT\n NOW() - t2.created_at AS seconds_since_first,\n (\n SELECT\n count(t1.id) as pending_count\n FROM tts_inference_jobs AS t1\n WHERE t1.status = \"pending\"\n ) as pending_count\nFROM tts_inference_jobs AS t2\nWHERE t2.status = \"pending\"\nORDER BY t2.id ASC\nLIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "seconds_since_first", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 129 - }, - "char_set": 63, - "max_size": 16 - } - }, - { - "ordinal": 1, - "name": "pending_count", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 21 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - true - ] - } - }, - "ab981965f518a8986aaaba41ed821fcb438db4c6156cbb26f639921225a88d59": { - "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = \"dead\",\n failure_reason = ?,\n retry_at = NULL\nWHERE id = ?\n ", + "a6c74861387a61fbb1e0089f27a16bb01efe527c3a709ff0b2897cb4c0fcede8": { + "query": "\nUPDATE twitch_event_rules\nSET\n event_match_predicate = ?,\n event_response = ?,\n user_specified_rule_order = ?,\n rule_is_disabled = ?,\n ip_address_last_update = ?\nWHERE\n token = ?\n AND deleted_at IS NULL\nLIMIT 1\n ", "describe": { "columns": [], "parameters": { - "Right": 2 + "Right": 6 }, "nullable": [] } }, - "adc494ddafaa13aebe423984f32a0d8810da604b19f5d6e0081b5e891dfc0f52": { - "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "ab6f25eb0f6340d49aa705835ed428434d8b6ab14b0740f6a5ebe7841b9ece20": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { "columns": [], "parameters": { - "Right": 7 + "Right": 1 }, "nullable": [] } @@ -6993,13 +4996,23 @@ "nullable": [] } }, - "ae04054c4d32e581dd183fd7ebb8e48fc7df856d2ffd8036d9ce6006bb7a296a": { - "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_template_token,\n\n jobs.failure_reason as maybe_failure_reason,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_template_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", + "af032ec861cb91f15ec763d9452e03c23797af15b5714565aa330f6cc420e8d6": { + "query": "\nINSERT INTO api_tokens\nSET\n internal_token = ?,\n api_token = ?,\n user_token = ?,\n uuid_idempotency_token = ?,\n ip_address_creation = ?,\n ip_address_last_update = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + } + }, + "b06e373275eb6d6f445abba38b8129d53196527bf953a50c85ccb93c23d2e4d6": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "job_token", + "name": "model_token", "type_info": { "type": "VarString", "flags": { @@ -7011,54 +5024,114 @@ }, { "ordinal": 1, - "name": "status", + "name": "tts_model_type", "type_info": { "type": "String", "flags": { - "bits": 393 + "bits": 385 }, "char_set": 224, - "max_size": 64 + "max_size": 60 } }, { "ordinal": 2, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 3, - "name": "maybe_template_token", + "name": "creator_user_token", "type_info": { "type": "VarString", "flags": { - "bits": 136 + "bits": 4233 }, "char_set": 224, "max_size": 128 } }, { - "ordinal": 4, - "name": "maybe_failure_reason", + "ordinal": 3, + "name": "creator_username", "type_info": { "type": "VarString", "flags": { - "bits": 128 + "bits": 4229 }, "char_set": 224, - "max_size": 2048 + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 } }, { "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, "name": "created_at", "type_info": { "type": "Timestamp", @@ -7070,7 +5143,7 @@ } }, { - "ordinal": 6, + "ordinal": 11, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -7083,35 +5156,30 @@ } ], "parameters": { - "Right": 1 + "Right": 0 }, "nullable": [ false, false, false, - true, - true, + false, + false, + false, + false, + false, + false, + false, false, false ] } }, - "b1ad806aa93f66cab358d7cdb405d57c643e2ff131cfec9f1ced55c2e6ca9511": { - "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "b483867c3f767a01cc80e4c0e0f1950c65154e06c13fc2fcc6e3ec96e8e2c956": { + "query": "\nUPDATE api_tokens\nSET\n maybe_short_description = ?,\n ip_address_last_update = ?\nWHERE\n user_token = ?\n AND api_token = ?\nLIMIT 1\n ", "describe": { "columns": [], "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "b257c267dfe063995374786d3004f215e77e8a32843c024645da6fa435507dce": { - "query": "\nINSERT INTO\n ip_address_bans\nSET\n ip_address = ?,\n maybe_target_user_token = ?,\n mod_user_token = ?,\n mod_notes = ?\nON DUPLICATE KEY UPDATE\n expires_at = NULL,\n deleted_at = NULL,\n ip_address = ?,\n maybe_target_user_token = ?,\n mod_user_token = ?,\n mod_notes = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 8 + "Right": 4 }, "nullable": [] } @@ -7345,7 +5413,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -7357,7 +5425,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -7453,224 +5521,14 @@ ] } }, - "b7535ecf2e63b7f02b5b96f833fd8cd28285b24473df6f89795ff360c1723215": { - "query": "\nSELECT *\nFROM w2l_template_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "b8746ae4ff2d4dcd54563eff64877f924bb119681ad78f1300146e00b762ca4f": { + "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n maybe_pretrained_vocoder_used = ?,\n raw_inference_text = ?,\n raw_inference_text_hash_sha2 = ?,\n normalized_inference_text = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_synthetic_id = ?,\n\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n\n file_size_bytes = ?,\n duration_millis = ?\n ", "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "uuid_idempotency_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 3, - "name": "on_success_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 5, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 6, - "name": "download_url", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 7, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 8, - "name": "creator_ip_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 9, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 10, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 11, - "name": "attempt_count", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 12, - "name": "failure_reason", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 2048 - } - }, - { - "ordinal": 13, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 14, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 15, - "name": "retry_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - } - ], + "columns": [], "parameters": { - "Right": 1 + "Right": 14 }, - "nullable": [ - false, - false, - false, - true, - false, - false, - true, - false, - false, - false, - false, - false, - true, - false, - false, - true - ] + "nullable": [] } }, "ba31fefd3e96fd238fc2bcdfab64f390a17293f2c0996ae891ed8821e57512cd": { @@ -7890,7 +5748,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -7902,7 +5760,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -7914,7 +5772,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -7926,7 +5784,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -8023,220 +5881,6 @@ ] } }, - "bc9c6b57b2578b8163956d11a5f6a9a77ddab6098d2c4207e1a70502db30c9d1": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.private_bucket_hash,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\nAND w2l.user_deleted_at IS NULL\nAND w2l.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 6, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 7, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 8, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 11 - } - }, - { - "ordinal": 9, - "name": "private_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 12, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - false - ] - } - }, - "bee13160674414391587f56e8fb0aa3629e3dfcead0afa5759bff1e161f1bb6e": { - "query": "\nUPDATE w2l_templates\nSET\n is_public_listing_approved = ?,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "bf489cf419d342d144f072c310d6d70c9073b092b5721f489eaf2ffc8ad3915e": { - "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, "c0f56e2656b566d3a8c869d39b0b56d1fbb75d7d9e817a24b10add44d348092d": { "query": "\nSELECT count(*) as count\nFROM w2l_templates\nWHERE\n creator_user_token = ?\nLIMIT 1\n ", "describe": { @@ -8262,291 +5906,6 @@ ] } }, - "c15e7c97a254b2bf33e54e093e5f058f206602c9bde97ccd8fc0f4521e2ddfc2": { - "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n disable_gravatar,\n preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n preferred_w2l_result_visibility as `preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility`,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n cashapp_username,\n website_url,\n is_banned,\n maybe_mod_comments,\n maybe_mod_user_token,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "email_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 4, - "name": "profile_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 5, - "name": "profile_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 6, - "name": "user_role_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 7, - "name": "disable_gravatar", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "preferred_tts_result_visibility: crate::database::enums::record_visibility::RecordVisibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 9, - "name": "preferred_w2l_result_visibility: crate::database::enums::record_visibility::RecordVisibility", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 10, - "name": "discord_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 11, - "name": "twitch_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 12, - "name": "twitter_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 13, - "name": "patreon_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 14, - "name": "github_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 15, - "name": "cashapp_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 16, - "name": "website_url", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 17, - "name": "is_banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 18, - "name": "maybe_mod_comments", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 19, - "name": "maybe_mod_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 20, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - false, - true, - true, - false - ] - } - }, "c4640a618e1d179fa95adce67a329818f8b1626ac88fa76dbb871b8788136191": { "query": "\nUPDATE w2l_templates\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -8786,7 +6145,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -8798,7 +6157,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -8894,80 +6253,12 @@ ] } }, - "ca5efc3be0f09649df2add5270c06544133288a95ffcb9a0589efd4fe345efe7": { - "query": "\nINSERT INTO badges ( slug, title, description, image_url )\nVALUES ( ?, ?, ?, ? )\n ", + "d51e6e901cc8641b7b076f42b0fc507dad2168617c33c615ef97417f3a78ccc1": { + "query": "\nUPDATE api_tokens\nSET\n deleted_at = CURRENT_TIMESTAMP,\n ip_address_last_update = ?\nWHERE\n user_token = ?\nAND api_token NOT IN (?, ?, ?, ?, ?)\n ", "describe": { "columns": [], "parameters": { - "Right": 4 - }, - "nullable": [] - } - }, - "cc4372cdd1a65999cbaf008f200683eedc93ab83e17256e47e011667ba8384d8": { - "query": "\nUPDATE tts_results\nSET\n creator_set_visibility = ?,\n creator_ip_address = ?\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - } - }, - "ce629cea6c387590759cc89a060c471ecdcb5d5f46a53576897a963154ed4d74": { - "query": "\n INSERT INTO tts_category_assignments\n SET\n model_token = ?,\n category_token = ?,\n category_addition_user_token = ?,\n ip_address_creation = ?,\n ip_address_last_update = ?,\n deleted_at = NULL\n ON DUPLICATE KEY UPDATE\n model_token = ?,\n category_token = ?,\n category_addition_user_token = ?,\n ip_address_last_update = ?,\n deleted_at = NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 9 - }, - "nullable": [] - } - }, - "d12757abef811cd6951e0e1427fdb496a8931b547519d320f5ade56007d3eb5d": { - "query": "\nSELECT\n id,\n status\nFROM w2l_template_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 515 - }, - "char_set": 63, - "max_size": 20 - } - }, - { - "ordinal": 1, - "name": "status", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 64 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false - ] - } - }, - "d2ef33027abc15b6ec00260f7d30a8c66a3190229ea103ecb571d4957af63525": { - "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 + "Right": 7 }, "nullable": [] } @@ -8982,12 +6273,12 @@ "nullable": [] } }, - "d850784ca167e686b91fdd4d3cce20c7a00e4e38e4da629733114a1c55ba4ab3": { - "query": "\nINSERT INTO w2l_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n maybe_w2l_template_token = ?,\n maybe_public_audio_bucket_hash = ?,\n maybe_public_audio_bucket_location = ?,\n\n maybe_original_audio_filename = ?,\n maybe_audio_mime_type = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n disable_end_bump = false,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "d7ac9f84e92d6b2111162a674284c0b9fa50c718bedfb8ab7238ca0a7b0875e6": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", "describe": { "columns": [], "parameters": { - "Right": 10 + "Right": 17 }, "nullable": [] } @@ -9012,119 +6303,14 @@ "nullable": [] } }, - "de2d38b505ecff7b8b73996888ff6a904c5a0fab38e1c0b6156929c812b94302": { - "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n user_roles.slug as user_role_slug,\n user_roles.name as user_role_name\nFROM\n users\nJOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE\n user_roles.slug != 'user'\n ", + "def4202f63874e7b04d8f8a5da028a9aacc4dfe19b8898d3d6820d393a633cf3": { + "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n is_from_api = ?,\n is_for_twitch = ?,\n priority_level = ?,\n status = \"pending\"\n ", "describe": { - "columns": [ - { - "ordinal": 0, - "name": "user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "user_role_slug", - "type_info": { - "type": "String", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 4, - "name": "user_role_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - } - ], + "columns": [], "parameters": { - "Right": 0 + "Right": 10 }, - "nullable": [ - false, - false, - false, - false, - false - ] - } - }, - "dfe818ab849c5f6e2a5931142aab640d249aa96cb85dc06bfffae4d7feef541e": { - "query": "\nSELECT\n NOW() - t2.created_at AS seconds_since_first,\n (\n SELECT\n count(t1.id) as pending_count\n FROM w2l_inference_jobs AS t1\n WHERE t1.status = \"pending\"\n ) as pending_count\nFROM w2l_inference_jobs AS t2\nWHERE t2.status = \"pending\"\nORDER BY t2.id ASC\nLIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "seconds_since_first", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 129 - }, - "char_set": 63, - "max_size": 16 - } - }, - { - "ordinal": 1, - "name": "pending_count", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 21 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - true - ] + "nullable": [] } }, "e0ad2a7fe4ae795e4f097f604d5cfcf9605298b58cde5ca23bdf9c8ec7383b00": { @@ -9137,6 +6323,16 @@ "nullable": [] } }, + "e0d442313b7861debaef984c5809647bd5df85106c194b66e864674b6af95330": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = \"dead\",\n failure_reason = ?,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "e3a0bd8073c3c0d84b9ab0e2e1ad8371f14b10801b7e4658b73c42ce1b27fa0b": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.is_public_listing_approved IS TRUE\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -9222,7 +6418,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -9234,7 +6430,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -9246,7 +6442,7 @@ "bits": 1 }, "char_set": 63, - "max_size": 11 + "max_size": 10 } }, { @@ -9331,6 +6527,16 @@ ] } }, + "e3a4f23245c8cfd9eb288548174cb4c5d550ceeff3ee4ec8cd76a94c5afa3eda": { + "query": "\nINSERT INTO twitch_event_rules\nSET\n uuid_idempotency_token = ?,\n token = ?,\n user_token = ?,\n event_category = ?,\n event_match_predicate = ?,\n event_response = ?,\n user_specified_rule_order = ?,\n rule_is_disabled = ?,\n ip_address_creation = ?,\n ip_address_last_update = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, "e4e5be790f087b9aa6bee87e2315140bf621e4919362ebe72ab070163f18e9c8": { "query": "\nUPDATE w2l_templates\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -9341,42 +6547,14 @@ "nullable": [] } }, - "e73192a3e763a287d5737821443c5a81649f17e39978497b99c52e1e896a75f6": { - "query": "\nSELECT\n token as session_token,\n user_token\nFROM user_sessions\nWHERE token = ?\nAND deleted_at IS NULL\n ", + "e954932f95837c915c943aabdf55cf7535340036e32d4fba4c43a67d66bb21e0": { + "query": "\nINSERT INTO voice_clone_requests\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n maybe_user_token = ?,\n email_address = ?,\n discord_username = ?,\n\n is_for_studio = ?,\n is_for_twitch_tts = ?,\n is_for_api_use = ?,\n is_for_music = ?,\n is_for_games = ?,\n is_for_other = ?,\n optional_notes_on_use = ?,\n\n is_for_private_use = ?,\n is_for_public_use = ?,\n\n is_own_voice = ?,\n is_third_party_voice = ?,\n\n has_clean_audio_recordings = ?,\n has_good_microphone = ?,\n\n optional_questions = ?,\n optional_extra_comments = ?,\n\n ip_address_creation = ?\n ", "describe": { - "columns": [ - { - "ordinal": 0, - "name": "session_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - } - ], + "columns": [], "parameters": { - "Right": 1 + "Right": 21 }, - "nullable": [ - false, - false - ] + "nullable": [] } }, "e9ec861d6eb135370f9baf3a866282ec5686d4b299fc4ec10900ab32ea8577d8": { @@ -9389,14 +6567,120 @@ "nullable": [] } }, - "eab69a8d679b2dcfb06a9ccc79851f6c693d0c372142ff0e7ded89c95e4bcf44": { - "query": "\nUPDATE users\nSET\n is_banned = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", + "eacc0e022cd188eb930bbf4ffbf2fc8fc9081fa7a44119924b114422ceae7ad3": { + "query": "\nSELECT\n token,\n event_category as `event_category: crate::column_types::twitch_event_category::TwitchEventCategory`,\n event_match_predicate,\n event_response,\n user_specified_rule_order,\n rule_is_disabled,\n created_at,\n updated_at\nFROM twitch_event_rules\nWHERE\n user_token = ?\n AND deleted_at IS NULL\n ", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "event_category: crate::column_types::twitch_event_category::TwitchEventCategory", + "type_info": { + "type": "String", + "flags": { + "bits": 4489 + }, + "char_set": 224, + "max_size": 56 + } + }, + { + "ordinal": 2, + "name": "event_match_predicate", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 3, + "name": "event_response", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 4, + "name": "user_specified_rule_order", + "type_info": { + "type": "Long", + "flags": { + "bits": 33 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 5, + "name": "rule_is_disabled", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 7, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], "parameters": { - "Right": 4 + "Right": 1 }, - "nullable": [] + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false + ] } }, "eadbe83c69c09f3e049ec7c760d1af0313cec8f3d82c0993e526738aa4d0a6dc": { @@ -9528,6 +6812,2897 @@ ] } }, + "f19300f4e8434c2669dac062e5a79c5ca71767c7380aab0ab40b51be8ff449d8": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 1 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "fc421ad1b946460bf443271730b969c96850b3c5d467c8f785730bc8ec163780": { + "query": "\nSELECT\n token,\n\n maybe_user_token,\n email_address,\n discord_username,\n\n is_for_studio,\n is_for_twitch_tts,\n is_for_api_use,\n is_for_music,\n is_for_games,\n is_for_other,\n optional_notes_on_use,\n\n is_for_private_use,\n is_for_public_use,\n\n is_own_voice,\n is_third_party_voice,\n\n has_clean_audio_recordings,\n has_good_microphone,\n\n optional_questions,\n optional_extra_comments,\n\n created_at,\n updated_at\nFROM voice_clone_requests\nWHERE\n maybe_user_token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "discord_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "is_for_studio", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 5, + "name": "is_for_twitch_tts", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 6, + "name": "is_for_api_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 7, + "name": "is_for_music", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "is_for_games", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_for_other", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "optional_notes_on_use", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 11, + "name": "is_for_private_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "is_for_public_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "is_own_voice", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "is_third_party_voice", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "has_clean_audio_recordings", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "has_good_microphone", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "optional_questions", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 18, + "name": "optional_extra_comments", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 19, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + } + }, + "ffa85f7c65101a43653c724479ee080e0a74577d2ae37863b039f3a452493d1e": { + "query": "\nUPDATE w2l_results\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "02060ab9b8ca472a2bd4d4fa3120743e90148ccd6a4cb13bc491d00f36ed7a8f": { + "query": "\nSELECT\n id,\n status\nFROM w2l_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + }, + "0caed749627ff22ae1cea52621a375156259a731e8070459593082b80d463491": { + "query": "\nSELECT\n id,\n status\nFROM tts_model_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + }, + "0ff13b84956c2625050211a5bb2352a489ef8435571c364c8b2d1d21d936cbd4": { + "query": "\nUPDATE tts_results\nSET\n creator_set_visibility = ?,\n maybe_mod_user_token = ?\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "1168f39a5ec892d06e941b0dba0e955a27ef976567eeafd6e550bc42ea4c4331": { + "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + } + }, + "1236640a30d0d5177e13c31b61b791874f7c70026003a58ba6e07e7af46c30c1": { + "query": "\nSELECT\n next_id\nFROM\n w2l_result_synthetic_ids\nWHERE\n user_token = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "next_id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 20 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + } + }, + "17c8d210d93ee9c0d8e8422eed605c9150f19f9bf72affcf111b15a67a5b9e70": { + "query": "\nUPDATE w2l_templates\nSET\n is_public_listing_approved = ?,\n is_locked_from_user_modification = ?,\n is_locked_from_use = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + } + }, + "1cc62e5cdc8de18969d7e8018d91c38794e5a7d23f117e5b0a4126845f300e60": { + "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE\n token = ?\n AND deleted_at IS NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "1d0f302c0b7c4301000c024d6bbae4d3b7370ed7c944e4c557c64b0bd2f31c14": { + "query": "\nSELECT public_count, all_count FROM (\n (SELECT count(*) as public_count \n FROM tts_models \n WHERE user_deleted_at IS NULL \n AND mod_deleted_at IS NULL) as pc,\n (SELECT count(*) as all_count \n FROM tts_models) as ac\n)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "public_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 21 + } + }, + { + "ordinal": 1, + "name": "all_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false + ] + } + }, + "218b3fe667b34bfa158bb7d4f600a3b91a22460adf566dcdf6ce540006e6c684": { + "query": "\nUPDATE model_categories\nSET\n name = ?,\n maybe_dropdown_name = ?,\n\n can_directly_have_models = ?,\n can_have_subcategories = ?,\n can_only_mods_apply = ?,\n\n maybe_super_category_token = ?,\n\n is_mod_approved = ?,\n maybe_mod_user_token = ?,\n maybe_mod_comments = ?,\n\n version = version + 1\n\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, + "25e7858d370ac1359deb42ef5292e85ccd1fe7debf3a1ba73303e5882523a4d0": { + "query": "\nINSERT INTO w2l_template_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n template_type = ?,\n download_url = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 8 + }, + "nullable": [] + } + }, + "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "2b04dd7db52ef3a97a6f420a22a4dbee7416ae58697438df69e87ffd47bbf350": { + "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "33cd0c0b8eaaf494ef97232ac89775b8eb6a22d310824b7aa7db310dd4fd41b1": { + "query": "\nUPDATE w2l_results\nSET\n creator_set_visibility = ?,\n maybe_mod_user_token = ?\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { + "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 12 + }, + "nullable": [] + } + }, + "34642399c3e91960768cb659b6006db291574d9cd07a5bace3db573eff20c3fb": { + "query": "\nINSERT INTO w2l_result_synthetic_ids\nSET\n user_token = ?,\n next_id = 1\nON DUPLICATE KEY UPDATE\n user_token = ?,\n next_id = next_id + 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "346c02b215cb35efaa35fe96d866a8c9056ac46e987b4e01f64222259dccdcbf": { + "query": "\nUPDATE\n ip_address_bans\nSET\n mod_user_token = ?,\n deleted_at = NULL\nWHERE\n ip_address = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "369a7576edc128cf2b310648d6fc45a2191c1263d433fe6d1461d15dbf26690e": { + "query": "\nINSERT INTO w2l_results\nSET\n token = ?,\n maybe_w2l_template_token = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n maybe_creator_synthetic_id = ?,\n\n public_bucket_video_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 12 + }, + "nullable": [] + } + }, + "3a2dc0a7e41dcbdf21f91d1a1e54e813f74b0bac7aea052302089cae195eeb53": { + "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, + "4014c934d3705ce4293cef2ca3ebf14d36b6d569a2af6743a4e0bf94cdda07f9": { + "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n maybe_w2l_template_token,\n maybe_public_image_bucket_location,\n maybe_tts_inference_result_token,\n maybe_public_audio_bucket_hash,\n maybe_public_audio_bucket_location,\n\n maybe_original_audio_filename,\n maybe_original_audio_download_url,\n maybe_audio_mime_type,\n\n creator_ip_address,\n maybe_creator_user_token,\n\n creator_set_visibility as `creator_set_visibility: database_queries::column_types::record_visibility::RecordVisibility`,\n disable_end_bump,\n disable_watermark,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM w2l_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "inference_job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_public_image_bucket_location", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_public_audio_bucket_hash", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 7, + "name": "maybe_public_audio_bucket_location", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "maybe_original_audio_filename", + "type_info": { + "type": "String", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 9, + "name": "maybe_original_audio_download_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 10, + "name": "maybe_audio_mime_type", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 12, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 13, + "name": "creator_set_visibility: database_queries::column_types::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 14, + "name": "disable_end_bump", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "disable_watermark", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 17, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 18, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 19, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 20, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + false, + false, + false, + false, + false, + true, + false, + false, + true + ] + } + }, + "5473d3857478d001688998056377fefe4bcc792477f303b41fa3cf64f79170f3": { + "query": "\n INSERT INTO tts_category_assignments\n SET\n model_token = ?,\n category_token = ?,\n category_removal_user_token = ?,\n ip_address_creation = ?,\n ip_address_last_update = ?,\n deleted_at = CURRENT_TIMESTAMP\n ON DUPLICATE KEY UPDATE\n model_token = ?,\n category_token = ?,\n category_removal_user_token = ?,\n ip_address_last_update = ?,\n deleted_at = CURRENT_TIMESTAMP\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, + "56a0a2bfc3c73d6845ca9cf1d285b175a40b50a250b7effb9ce6474e58113d6e": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_model_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_model_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 3, + "name": "maybe_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 5, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + false, + false + ] + } + }, + "57928b1001a10a88ac5a07f43b1dd30f6dee6edbc0808be99fe1233f9e276c56": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_video_path as maybe_public_bucket_video_path,\n\n jobs.maybe_w2l_template_token,\n w2l.template_type as w2l_template_type,\n w2l.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_inference_jobs as jobs\nJOIN w2l_templates as w2l\n ON w2l.token = jobs.maybe_w2l_template_token\nLEFT OUTER JOIN w2l_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 3, + "name": "maybe_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_public_bucket_video_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "w2l_template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 7, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + false, + false, + false, + false + ] + } + }, + "60c06a507349919addb5d26b1c4a283a04f66fce86e84ed0a51db6001c9fc2b5": { + "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + } + }, + "619d18c87075a86c01f1e0ac26744774293ff8ca6b0011ce611b3f47d8a97a0e": { + "query": "\nSELECT\n w2l_templates.token as template_token,\n w2l_templates.title,\n w2l_templates.template_type,\n w2l_templates.duration_millis,\n w2l_templates.frame_width,\n w2l_templates.frame_height,\n w2l_templates.creator_user_token,\n users.username AS creator_username,\n users.display_name AS creator_display_name,\n users.email_gravatar_hash AS creator_gravatar_hash,\n w2l_templates.created_at\nFROM\n w2l_templates\nJOIN\n users\nON\n users.token = w2l_templates.creator_user_token\nWHERE\n w2l_templates.is_public_listing_approved IS NULL\n AND w2l_templates.user_deleted_at IS NULL\n AND w2l_templates.mod_deleted_at IS NULL\n AND w2l_templates.is_locked_from_use IS FALSE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 2, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 4, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 5, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 6, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 9, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 10, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, + "641253d4c621f75037877651bba044a21033f50a5e3f8fed9b06ec967a508853": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "68376f750b8421c10d319fa1045a2f3ce8fb1d35a5fe6ec92b2022babafbde80": { + "query": "\nUPDATE users\nSET\n user_role_slug = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "6b66db376952ff4cfcea6ffe1f9ecd4462722843a3d19fcd4df16f0fb993decc": { + "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "720d7a26fb8ce75a8a1b3a62ef509d8ca99d419172e9cc5d8e2c803e0eebcb7f": { + "query": "\n UPDATE model_categories\n SET\n deleted_at = NULL\n WHERE\n token = ?\n LIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "73558104936aab6add2d9c789b368b5f155f1fe01d32e7314f2f120422866f54": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "7490dcb8de10b1eda2c857e53ebc5c0465cdb589d3df4c5549859f7919acac77": { + "query": "\nSELECT token, username, email_address, password_hash, is_banned\nFROM users\nWHERE email_address = ?\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "password_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 63, + "max_size": 60 + } + }, + { + "ordinal": 4, + "name": "is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false + ] + } + }, + "7622292c05fe1b5832136762042780a12a02048166bb1b4492dfd627c48c8968": { + "query": "\nSELECT\n token as template_token\nFROM w2l_templates\nWHERE\n token = ?\n AND user_deleted_at IS NULL\n AND mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + } + }, + "7ab54466ca3021d90e0feb234acba56bf53d415dde69bc1c35120ed0c8dd1858": { + "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "on_success_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 6, + "name": "download_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 7, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 9, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 10, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 11, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 12, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 15, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + true, + false, + false, + false, + false, + false, + true, + false, + false, + true + ] + } + }, + "88328e04ce947d3f541951aada9cee25135aa950ab478cf0dc6de453bb079e17": { + "query": "\nSELECT\n events.token as event_token,\n events.event_type,\n events.maybe_target_user_token,\n users.username as maybe_target_username,\n users.display_name as maybe_target_display_name,\n users.email_gravatar_hash as maybe_target_user_gravatar_hash,\n events.maybe_target_entity_token,\n events.created_at,\n events.updated_at\nFROM firehose_entries as events\nLEFT OUTER JOIN users\nON events.maybe_target_user_token = users.token\nORDER BY events.id DESC\nLIMIT 25\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "event_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "event_type", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "maybe_target_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "maybe_target_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "maybe_target_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "maybe_target_user_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_target_entity_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 8, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + true, + true, + true, + true, + true, + false, + false + ] + } + }, + "886861e91c7e8af53eb856c949389a172b8d9e079f1c2070d115423907c47c50": { + "query": "\nINSERT INTO tts_model_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n download_url = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 8 + }, + "nullable": [] + } + }, + "8b1b9cf11b80da18d79237f2579a96b4a85fa7201118e01fffc01c1eba64cbaf": { + "query": "\n UPDATE model_categories\n SET\n deleted_at = CURRENT_TIMESTAMP\n WHERE\n token = ?\n LIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, + "9a879f76345ef5682b995f99026fac8b2f4c64c8ec4bb078187d1e53effcfbaf": { + "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address_creation = ?,\n creator_ip_address_last_update = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n maybe_public_bucket_preview_image_object_name = ?,\n maybe_public_bucket_preview_video_object_name = ?,\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 19 + }, + "nullable": [] + } + }, + "9b607b814cd7c445c2c46ec5c13dc23f3a6a771a3469a786ed46ec099577d95e": { + "query": "\nUPDATE\n ip_address_bans\nSET\n mod_user_token = ?,\n deleted_at = CURRENT_TIMESTAMP\nWHERE\n ip_address = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "9c04e83ce78dafd7b84edf057f30a834d459f710d0836dcf7f211f7cd17e24e3": { + "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + } + }, + "9cc5f697350814199344d3d9a0ff026e0d1e09e9097648400d4b63cfcbd31178": { + "query": "\nUPDATE w2l_results\nSET\n creator_set_visibility = ?,\n creator_ip_address = ?\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "9cef4989671fb55d8e732f6f3d5c16aea7a9b3e01fc6a3bc7a56d0b066f07730": { + "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address_creation = ?,\n creator_ip_address_last_update = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n file_size_bytes = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, + "9d4932baa4023accc2d28af98b58a47149d37c4937cd0171e6b955c09855c5f8": { + "query": "\nINSERT INTO w2l_inference_jobs\nSET\n maybe_w2l_template_token = ?,\n maybe_tts_inference_result_token = ?,\n maybe_public_audio_bucket_location = NULL,\n maybe_public_image_bucket_location = NULL,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + } + }, + "a125f183f7ae5627fc2118e7302cea73d967200ba1cbb12550f5880187b7db5e": { + "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "a2b586dd2120f29dd26281626f7381e292c821f8e826b9bc584b92ffa45035b2": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,\n\n jobs.model_token,\n tts.tts_model_type,\n tts.title,\n\n jobs.raw_inference_text,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_inference_jobs as jobs\nJOIN tts_models as tts\n ON tts.token = jobs.model_token\nLEFT OUTER JOIN tts_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 3, + "name": "maybe_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 7, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 9, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 10, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + false, + false, + false, + false, + false, + false + ] + } + }, + "a9173fd8148645bda75529aae0af63dcfb7cf9e114604ce4dcc962974e7268e1": { + "query": "\nSELECT\n NOW() - t2.created_at AS seconds_since_first,\n (\n SELECT\n count(t1.id) as pending_count\n FROM tts_inference_jobs AS t1\n WHERE t1.status = \"pending\"\n ) as pending_count\nFROM tts_inference_jobs AS t2\nWHERE t2.status = \"pending\"\nORDER BY t2.id ASC\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "seconds_since_first", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 16 + } + }, + { + "ordinal": 1, + "name": "pending_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + true + ] + } + }, + "ab981965f518a8986aaaba41ed821fcb438db4c6156cbb26f639921225a88d59": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = \"dead\",\n failure_reason = ?,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, + "adc494ddafaa13aebe423984f32a0d8810da604b19f5d6e0081b5e891dfc0f52": { + "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + } + }, + "ae04054c4d32e581dd183fd7ebb8e48fc7df856d2ffd8036d9ce6006bb7a296a": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_template_token,\n\n jobs.failure_reason as maybe_failure_reason,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_template_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 3, + "name": "maybe_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + false, + false + ] + } + }, + "b1ad806aa93f66cab358d7cdb405d57c643e2ff131cfec9f1ced55c2e6ca9511": { + "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "b257c267dfe063995374786d3004f215e77e8a32843c024645da6fa435507dce": { + "query": "\nINSERT INTO\n ip_address_bans\nSET\n ip_address = ?,\n maybe_target_user_token = ?,\n mod_user_token = ?,\n mod_notes = ?\nON DUPLICATE KEY UPDATE\n expires_at = NULL,\n deleted_at = NULL,\n ip_address = ?,\n maybe_target_user_token = ?,\n mod_user_token = ?,\n mod_notes = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 8 + }, + "nullable": [] + } + }, + "b7535ecf2e63b7f02b5b96f833fd8cd28285b24473df6f89795ff360c1723215": { + "query": "\nSELECT *\nFROM w2l_template_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "on_success_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 6, + "name": "download_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 7, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 9, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 10, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 11, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 12, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 15, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + true, + false, + false, + false, + false, + false, + true, + false, + false, + true + ] + } + }, + "bc9c6b57b2578b8163956d11a5f6a9a77ddab6098d2c4207e1a70502db30c9d1": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.private_bucket_hash,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\nAND w2l.user_deleted_at IS NULL\nAND w2l.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "private_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + } + }, + "bee13160674414391587f56e8fb0aa3629e3dfcead0afa5759bff1e161f1bb6e": { + "query": "\nUPDATE w2l_templates\nSET\n is_public_listing_approved = ?,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "bf489cf419d342d144f072c310d6d70c9073b092b5721f489eaf2ffc8ad3915e": { + "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "cc4372cdd1a65999cbaf008f200683eedc93ab83e17256e47e011667ba8384d8": { + "query": "\nUPDATE tts_results\nSET\n creator_set_visibility = ?,\n creator_ip_address = ?\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "ce629cea6c387590759cc89a060c471ecdcb5d5f46a53576897a963154ed4d74": { + "query": "\n INSERT INTO tts_category_assignments\n SET\n model_token = ?,\n category_token = ?,\n category_addition_user_token = ?,\n ip_address_creation = ?,\n ip_address_last_update = ?,\n deleted_at = NULL\n ON DUPLICATE KEY UPDATE\n model_token = ?,\n category_token = ?,\n category_addition_user_token = ?,\n ip_address_last_update = ?,\n deleted_at = NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, + "d12757abef811cd6951e0e1427fdb496a8931b547519d320f5ade56007d3eb5d": { + "query": "\nSELECT\n id,\n status\nFROM w2l_template_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + }, + "d2ef33027abc15b6ec00260f7d30a8c66a3190229ea103ecb571d4957af63525": { + "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + } + }, + "d850784ca167e686b91fdd4d3cce20c7a00e4e38e4da629733114a1c55ba4ab3": { + "query": "\nINSERT INTO w2l_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n maybe_w2l_template_token = ?,\n maybe_public_audio_bucket_hash = ?,\n maybe_public_audio_bucket_location = ?,\n\n maybe_original_audio_filename = ?,\n maybe_audio_mime_type = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n disable_end_bump = false,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, + "de2d38b505ecff7b8b73996888ff6a904c5a0fab38e1c0b6156929c812b94302": { + "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n user_roles.slug as user_role_slug,\n user_roles.name as user_role_name\nFROM\n users\nJOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE\n user_roles.slug != 'user'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "user_role_slug", + "type_info": { + "type": "String", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 4, + "name": "user_role_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false + ] + } + }, + "dfe818ab849c5f6e2a5931142aab640d249aa96cb85dc06bfffae4d7feef541e": { + "query": "\nSELECT\n NOW() - t2.created_at AS seconds_since_first,\n (\n SELECT\n count(t1.id) as pending_count\n FROM w2l_inference_jobs AS t1\n WHERE t1.status = \"pending\"\n ) as pending_count\nFROM w2l_inference_jobs AS t2\nWHERE t2.status = \"pending\"\nORDER BY t2.id ASC\nLIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "seconds_since_first", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 16 + } + }, + { + "ordinal": 1, + "name": "pending_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + true + ] + } + }, + "e73192a3e763a287d5737821443c5a81649f17e39978497b99c52e1e896a75f6": { + "query": "\nSELECT\n token as session_token,\n user_token\nFROM user_sessions\nWHERE token = ?\nAND deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "session_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + }, + "eab69a8d679b2dcfb06a9ccc79851f6c693d0c372142ff0e7ded89c95e4bcf44": { + "query": "\nUPDATE users\nSET\n is_banned = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 4 + }, + "nullable": [] + } + }, "ee05f95e45f7aaa5811d70b1f00d3f40a361536dfc6451affcad97586dc3338f": { "query": "\nUPDATE w2l_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -9677,6 +9852,719 @@ "nullable": [] } }, + "f70bd775718829cd10f508dfe09286020467cc34fb54daf0758dba2545b164f8": { + "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_confirmed,\n users.email_gravatar_hash,\n\n users.disable_gravatar,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n users.preferred_tts_result_visibility as `preferred_tts_result_visibility: database_queries::column_types::record_visibility::RecordVisibility`,\n users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: database_queries::column_types::record_visibility::RecordVisibility`,\n\n users.user_role_slug,\n users.is_banned,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l,\n user_roles.can_delete_own_tts_results,\n user_roles.can_delete_own_w2l_results,\n user_roles.can_delete_own_account,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n user_roles.can_delete_own_tts_models,\n user_roles.can_delete_own_w2l_templates,\n\n user_roles.can_approve_w2l_templates,\n user_roles.can_edit_other_users_profiles,\n user_roles.can_edit_other_users_tts_models,\n user_roles.can_edit_other_users_w2l_templates,\n user_roles.can_delete_other_users_tts_models,\n user_roles.can_delete_other_users_tts_results,\n user_roles.can_delete_other_users_w2l_templates,\n user_roles.can_delete_other_users_w2l_results,\n user_roles.can_ban_users,\n user_roles.can_delete_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "email_confirmed", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 5, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 7, + "name": "auto_play_audio_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "auto_play_video_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "preferred_tts_result_visibility: database_queries::column_types::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 10, + "name": "preferred_w2l_result_visibility: database_queries::column_types::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 11, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 12, + "name": "is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "can_use_tts", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "can_use_w2l", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "can_delete_own_tts_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "can_delete_own_w2l_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "can_delete_own_account", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 18, + "name": "can_upload_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 19, + "name": "can_upload_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 20, + "name": "can_delete_own_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 21, + "name": "can_delete_own_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 22, + "name": "can_approve_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 23, + "name": "can_edit_other_users_profiles", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 24, + "name": "can_edit_other_users_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 25, + "name": "can_edit_other_users_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 26, + "name": "can_delete_other_users_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 27, + "name": "can_delete_other_users_tts_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 28, + "name": "can_delete_other_users_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 29, + "name": "can_delete_other_users_w2l_results", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 30, + "name": "can_ban_users", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 31, + "name": "can_delete_users", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + } + }, + "f70dbc04f76d5ab9ea790482dffe06ebd2a6685b825b522f3bd16c3d14330453": { + "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n disable_gravatar,\n preferred_tts_result_visibility as `preferred_tts_result_visibility: database_queries::column_types::record_visibility::RecordVisibility`,\n preferred_w2l_result_visibility as `preferred_w2l_result_visibility: database_queries::column_types::record_visibility::RecordVisibility`,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n cashapp_username,\n website_url,\n is_banned,\n maybe_mod_comments,\n maybe_mod_user_token,\n created_at\nFROM users\nWHERE\n users.username = ?\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "profile_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "profile_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 6, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 7, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "preferred_tts_result_visibility: database_queries::column_types::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 9, + "name": "preferred_w2l_result_visibility: database_queries::column_types::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 10, + "name": "discord_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 11, + "name": "twitch_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 12, + "name": "twitter_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 13, + "name": "patreon_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 14, + "name": "github_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 15, + "name": "cashapp_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 16, + "name": "website_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 17, + "name": "is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 18, + "name": "maybe_mod_comments", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 19, + "name": "maybe_mod_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 20, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + false + ] + } + }, "f75b713d166398ebea2bdbde4e416dc8eb2c48dba4aa9bd164bcf5bc77237ce1": { "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n preferred_tts_result_visibility = ?,\n preferred_w2l_result_visibility = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n ip_address_last_update = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", "describe": { @@ -9773,15 +10661,5 @@ false ] } - }, - "ffa85f7c65101a43653c724479ee080e0a74577d2ae37863b039f3a452493d1e": { - "query": "\nUPDATE w2l_results\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - } } } From 8e0a21421bc491ccba2cbce04174e3f256da859d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 05:19:56 -0500 Subject: [PATCH 0955/1042] twitch event rule endpoints --- .../list_twitch_event_rules_for_user.rs | 4 +- crates/lib/database_queries/src/tokens.rs | 2 +- .../endpoints/api_tokens/create_api_token.rs | 2 +- .../twitch/event_rules/create_event_rule.rs | 135 ++++++++++++++++ .../twitch/event_rules/delete_event_rule.rs | 134 ++++++++++++++++ .../twitch/event_rules/edit_event_rule.rs | 150 ++++++++++++++++++ .../event_rules/list_event_rules_for_user.rs | 109 +++++++++++++ .../src/http_server/endpoints/twitch/mod.rs | 4 +- .../http_server/endpoints/twitch/oauth/mod.rs | 3 + .../twitch/{ => oauth}/oauth_begin_json.rs | 0 .../{ => oauth}/oauth_begin_redirect.rs | 0 .../endpoints/twitch/{ => oauth}/oauth_end.rs | 0 crates/service/storyteller_web/src/routes.rs | 29 +++- 13 files changed, 562 insertions(+), 10 deletions(-) create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/mod.rs rename crates/service/storyteller_web/src/http_server/endpoints/twitch/{ => oauth}/oauth_begin_json.rs (100%) rename crates/service/storyteller_web/src/http_server/endpoints/twitch/{ => oauth}/oauth_begin_redirect.rs (100%) rename crates/service/storyteller_web/src/http_server/endpoints/twitch/{ => oauth}/oauth_end.rs (100%) diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs index a3c6b42a01..f8290809d9 100644 --- a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs +++ b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs @@ -5,8 +5,8 @@ use crate::column_types::twitch_event_category::TwitchEventCategory; use crate::helpers::boolean_converters::i8_to_bool; use sqlx::MySqlPool; -/// table: api_tokens -#[derive(Debug)] +// TODO: This is exposed to the frontend. Write a different struct. +#[derive(Debug, Serialize)] pub struct TwitchEventRule { pub token: String, pub event_category: TwitchEventCategory, diff --git a/crates/lib/database_queries/src/tokens.rs b/crates/lib/database_queries/src/tokens.rs index 20fb8abef8..b99f06bae2 100644 --- a/crates/lib/database_queries/src/tokens.rs +++ b/crates/lib/database_queries/src/tokens.rs @@ -68,7 +68,7 @@ impl Tokens { } pub fn new_api_token_internal_token() -> AnyhowResult { - random_prefix_crockford_token("AIT:", 32, false) + random_prefix_crockford_token("INT_API:", 32, false) } // ========== [Twitch] Tokens not typically visible to users ========== diff --git a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs index c45016a671..7d8169f506 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/api_tokens/create_api_token.rs @@ -12,6 +12,7 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use crate::util::email_to_gravatar::email_to_gravatar; use crate::util::markdown_to_html::markdown_to_html; +use database_queries::queries::api_tokens::create_api_token::create_api_token_for_user; use database_queries::tokens::Tokens; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log}; @@ -22,7 +23,6 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; -use database_queries::queries::api_tokens::create_api_token::create_api_token_for_user; // =============== Request =============== diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs index e69de29bb2..b8315be056 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs @@ -0,0 +1,135 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use database_queries::column_types::twitch_event_category::TwitchEventCategory; +use database_queries::queries::twitch::twitch_event_rules::insert_twitch_event_rule_builder::InsertTwitchEventRuleBuilder; +use database_queries::tokens::Tokens; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log}; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Request =============== + +#[derive(Deserialize)] +pub struct CreateTwitchEventRuleRequest { + pub idempotency_token: String, + pub event_category: TwitchEventCategory, + pub event_match_predicate: String, + pub event_response: String, + pub user_specified_rule_order: u32, + pub rule_is_disabled: bool, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct CreateTwitchEventRuleResponse { + pub success: bool, + pub twitch_event_rule_token: String, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum CreateTwitchEventRuleError { + BadInput(String), + NotAuthorized, + ServerError, +} + +impl ResponseError for CreateTwitchEventRuleError { + fn status_code(&self) -> StatusCode { + match *self { + CreateTwitchEventRuleError::BadInput(_) => StatusCode::BAD_REQUEST, + CreateTwitchEventRuleError::NotAuthorized => StatusCode::UNAUTHORIZED, + CreateTwitchEventRuleError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for CreateTwitchEventRuleError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn create_twitch_event_rule_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CreateTwitchEventRuleError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(CreateTwitchEventRuleError::NotAuthorized); + } + }; + + if user_session.is_banned { + warn!("banned users cannot use"); + return Err(CreateTwitchEventRuleError::NotAuthorized); + } + + let creator_ip_address = get_request_ip(&http_request); + + let insert_builder = InsertTwitchEventRuleBuilder { + uuid_idempotency_token: request.idempotency_token.clone(), + user_token: user_session.user_token, + event_category: request.event_category, + event_match_predicate: request.event_match_predicate.clone(), + event_response: request.event_response.clone(), + user_specified_rule_order: request.user_specified_rule_order, + rule_is_disabled: request.rule_is_disabled, + ip_address_creation: creator_ip_address, + }; + + let twitch_event_rule_token = insert_builder.insert(&server_state.mysql_pool) + .await + .map_err(|e| { + warn!("creation error: {:?}", e); + CreateTwitchEventRuleError::ServerError + })?; + + let response = CreateTwitchEventRuleResponse { + success: true, + twitch_event_rule_token + }; + + let body = serde_json::to_string(&response) + .map_err(|e| CreateTwitchEventRuleError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/delete_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/delete_event_rule.rs index e69de29bb2..14e3048706 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/delete_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/delete_event_rule.rs @@ -0,0 +1,134 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use database_queries::queries::twitch::twitch_event_rules::delete_twitch_event_rule::delete_twitch_event_rule; +use database_queries::queries::twitch::twitch_event_rules::get_twitch_event_rule_for_user::get_twitch_event_rules_for_user; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log, error}; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Request =============== + +#[derive(Deserialize)] +pub struct DeleteTwitchEventRulePathInfo { + token: String, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct DeleteTwitchEventRuleResponse { + pub success: bool, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum DeleteTwitchEventRuleError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for DeleteTwitchEventRuleError { + fn status_code(&self) -> StatusCode { + match *self { + DeleteTwitchEventRuleError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteTwitchEventRuleError::NotFound => StatusCode::NOT_FOUND, + DeleteTwitchEventRuleError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteTwitchEventRuleError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for DeleteTwitchEventRuleError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn delete_twitch_event_rule_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + DeleteTwitchEventRuleError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(DeleteTwitchEventRuleError::NotAuthorized); + } + }; + + if user_session.is_banned { + warn!("banned users cannot delete"); + return Err(DeleteTwitchEventRuleError::NotAuthorized); + } + + let twitch_event_rule = get_twitch_event_rules_for_user( + &path.token, &user_session.user_token, &server_state.mysql_pool) + .await + .map_err(|e| { + error!("Error with query: {:?}", e); + return DeleteTwitchEventRuleError::ServerError; + })?; + + if twitch_event_rule.is_none() { + info!("event rule not found"); + return Err(DeleteTwitchEventRuleError::NotFound); + } + + let creator_ip_address = get_request_ip(&http_request); + + let _r = delete_twitch_event_rule( + &path.token, + &creator_ip_address, + &server_state.mysql_pool) + .await + .map_err(|e| { + error!("Error with query: {:?}", e); + return DeleteTwitchEventRuleError::ServerError; + }); + + let response = DeleteTwitchEventRuleResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| DeleteTwitchEventRuleError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs index e69de29bb2..aca1d71c87 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs @@ -0,0 +1,150 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use database_queries::queries::twitch::twitch_event_rules::get_twitch_event_rule_for_user::get_twitch_event_rules_for_user; +use database_queries::queries::twitch::twitch_event_rules::update_twitch_event_rule_builder::UpdateTwitchEventRuleBuilder; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log, error}; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Request =============== + +#[derive(Deserialize)] +pub struct EditTwitchEventRulePathInfo { + token: String, +} + +// These are not sparse updates! +#[derive(Deserialize)] +pub struct EditTwitchEventRuleRequest { + pub event_match_predicate: String, + pub event_response: String, + pub user_specified_rule_order: u32, + pub rule_is_disabled: bool, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct EditTwitchEventRuleResponse { + pub success: bool, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum EditTwitchEventRuleError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for EditTwitchEventRuleError { + fn status_code(&self) -> StatusCode { + match *self { + EditTwitchEventRuleError::BadInput(_) => StatusCode::BAD_REQUEST, + EditTwitchEventRuleError::NotFound => StatusCode::NOT_FOUND, + EditTwitchEventRuleError::NotAuthorized => StatusCode::UNAUTHORIZED, + EditTwitchEventRuleError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for EditTwitchEventRuleError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn edit_twitch_event_rule_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EditTwitchEventRuleError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(EditTwitchEventRuleError::NotAuthorized); + } + }; + + if user_session.is_banned { + warn!("banned users cannot edit"); + return Err(EditTwitchEventRuleError::NotAuthorized); + } + + let twitch_event_rule = get_twitch_event_rules_for_user( + &path.token, &user_session.user_token, &server_state.mysql_pool) + .await + .map_err(|e| { + error!("Error with query: {:?}", e); + return EditTwitchEventRuleError::ServerError; + })?; + + if twitch_event_rule.is_none() { + info!("event rule not found"); + return Err(EditTwitchEventRuleError::NotFound); + } + + let creator_ip_address = get_request_ip(&http_request); + + let update_builder = UpdateTwitchEventRuleBuilder { + token: path.token.clone(), + event_match_predicate: request.event_match_predicate.clone(), + event_response: request.event_response.clone(), + user_specified_rule_order: request.user_specified_rule_order, + rule_is_disabled: request.rule_is_disabled, + ip_address_update: creator_ip_address, + }; + + let _r = update_builder.update(&server_state.mysql_pool) + .await + .map_err(|e| { + error!("Error with query: {:?}", e); + return EditTwitchEventRuleError::ServerError; + }); + + let response = EditTwitchEventRuleResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| EditTwitchEventRuleError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs index e69de29bb2..aa42025c28 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs @@ -0,0 +1,109 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use database_queries::queries::twitch::twitch_event_rules::list_twitch_event_rules_for_user::{TwitchEventRule, list_twitch_event_rules_for_user}; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use lexical_sort::natural_lexical_cmp; +use log::{info, warn, log}; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct ListTwitchEventRulesResponse { + pub success: bool, + pub twitch_event_rules: Vec, +} + + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum ListTwitchEventRulesError { + NotAuthorized, + ServerError, +} + +impl ResponseError for ListTwitchEventRulesError { + fn status_code(&self) -> StatusCode { + match *self { + ListTwitchEventRulesError::NotAuthorized => StatusCode::UNAUTHORIZED, + ListTwitchEventRulesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for ListTwitchEventRulesError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn list_twitch_event_rules_for_user_handler( + http_request: HttpRequest, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListTwitchEventRulesError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(ListTwitchEventRulesError::NotAuthorized); + } + }; + + if user_session.is_banned { + warn!("banned users cannot use"); + return Err(ListTwitchEventRulesError::NotAuthorized); + } + + let twitch_event_rules = list_twitch_event_rules_for_user( + &user_session.user_token, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("query error: {:?}", e); + ListTwitchEventRulesError::ServerError + })?; + + let response = ListTwitchEventRulesResponse { + success: true, + twitch_event_rules, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListTwitchEventRulesError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs index 1d15747836..8ad19b93b8 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/mod.rs @@ -1,4 +1,2 @@ pub mod event_rules; -pub mod oauth_begin_json; -pub mod oauth_begin_redirect; -pub mod oauth_end; +pub mod oauth; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/mod.rs new file mode 100644 index 0000000000..0b47d3ca95 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/mod.rs @@ -0,0 +1,3 @@ +pub mod oauth_begin_json; +pub mod oauth_begin_redirect; +pub mod oauth_end; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_json.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/oauth_begin_json.rs similarity index 100% rename from crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_json.rs rename to crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/oauth_begin_json.rs diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_redirect.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/oauth_begin_redirect.rs similarity index 100% rename from crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_begin_redirect.rs rename to crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/oauth_begin_redirect.rs diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/oauth_end.rs similarity index 100% rename from crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth_end.rs rename to crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/oauth_end.rs diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index d1854b5e7b..ac29eebd4e 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -48,9 +48,9 @@ use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_u use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; -use crate::http_server::endpoints::twitch::oauth_begin_json::oauth_begin_enroll_json; -use crate::http_server::endpoints::twitch::oauth_begin_redirect::oauth_begin_enroll_redirect; -use crate::http_server::endpoints::twitch::oauth_end::oauth_end_enroll_from_redirect; +use crate::http_server::endpoints::twitch::oauth::oauth_begin_json::oauth_begin_enroll_json; +use crate::http_server::endpoints::twitch::oauth::oauth_begin_redirect::oauth_begin_enroll_redirect; +use crate::http_server::endpoints::twitch::oauth::oauth_end::oauth_end_enroll_from_redirect; use crate::http_server::endpoints::users::create_account::create_account_handler; use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; use crate::http_server::endpoints::users::get_profile::get_profile_handler; @@ -75,6 +75,10 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; +use crate::http_server::endpoints::twitch::event_rules::create_event_rule::create_twitch_event_rule_handler; +use crate::http_server::endpoints::twitch::event_rules::list_event_rules_for_user::list_twitch_event_rules_for_user_handler; +use crate::http_server::endpoints::twitch::event_rules::delete_event_rule::delete_twitch_event_rule_handler; +use crate::http_server::endpoints::twitch::event_rules::edit_event_rule::edit_twitch_event_rule_handler; pub fn add_routes (app: App) -> App where @@ -583,6 +587,7 @@ fn add_twitch_oauth_routes (app: App) -> App InitError = (), >, { + // TODO: Move oauth endpoints under /twitch/oauth (requires updating Twitch configs too.) app.service(web::scope("/twitch") .service(web::resource("/oauth_enroll_json") .route(web::get().to(oauth_begin_enroll_json)) @@ -596,5 +601,23 @@ fn add_twitch_oauth_routes (app: App) -> App .route(web::get().to(oauth_end_enroll_from_redirect)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::scope("/event_rule") + .service(web::resource("/create") + .route(web::post().to(create_twitch_event_rule_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/list") + .route(web::get().to(list_twitch_event_rules_for_user_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/update/{token}") + .route(web::post().to(edit_twitch_event_rule_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/delete/{token}") + .route(web::delete().to(delete_twitch_event_rule_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) ) } From 9280ec11f70b82010b69f3e0260819ed2b5839d9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 05:33:31 -0500 Subject: [PATCH 0956/1042] fix endpoint, debug script --- .../insert_twitch_event_rule_builder.rs | 2 +- debug/curl/twitch_event_rule_tests.py | 76 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100755 debug/curl/twitch_event_rule_tests.py diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/insert_twitch_event_rule_builder.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/insert_twitch_event_rule_builder.rs index 1c64aa1ed7..d186c9a060 100644 --- a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/insert_twitch_event_rule_builder.rs +++ b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/insert_twitch_event_rule_builder.rs @@ -40,7 +40,7 @@ SET &self.uuid_idempotency_token, &token, &self.user_token, - &self.event_category, + &self.event_category.to_str(), &self.event_match_predicate, &self.event_response, &self.user_specified_rule_order, diff --git a/debug/curl/twitch_event_rule_tests.py b/debug/curl/twitch_event_rule_tests.py new file mode 100755 index 0000000000..5fa7d32f7b --- /dev/null +++ b/debug/curl/twitch_event_rule_tests.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +import json +import re +import requests +import uuid + +login_url = 'http://localhost:12345/login' +create_url = 'http://localhost:12345/twitch/event_rule/create' +list_url = 'http://localhost:12345/twitch/event_rule/list' +update_url = 'http://localhost:12345/twitch/event_rule/update/{}' +delete_url = 'http://localhost:12345/twitch/event_rule/delete/{}' + +payload = { + 'username_or_email': 'echelon', + 'password': 'testing', +} + +r = requests.post(login_url, json=payload) + +print("===== Login =====") +print('Status: {}'.format(r.status_code)) +print(r.content) +for k, v in r.headers.items(): + print(' {}: {}'.format(k, v)) + +raw_set_cookie_header = r.headers['set-cookie'] +m = re.match(r'session=([^;\s]+);', raw_set_cookie_header) +session_cookie = m.group(1) + +cookies = { 'session': session_cookie } + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'channel_points', + 'event_match_predicate': 'TODO testing this', + 'event_response': 'TODO testing this', + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +r = requests.post(create_url, cookies=cookies, json=payload) + +print("===== Created Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + +response = json.loads(r.content) +token = response['twitch_event_rule_token'] + +payload = { + 'event_match_predicate': 'Updated', + 'event_response': 'Updated 2', + 'user_specified_rule_order': 123, + 'rule_is_disabled': True, +} +url = update_url.format(token) +r = requests.post(url, cookies=cookies, json=payload) + +print("===== Edited Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + +r = requests.get(list_url, cookies=cookies) + +print("===== List Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + +url = delete_url.format(token) +r = requests.delete(url, cookies=cookies) + +print("===== Deleted Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + From e6f0908a3db3bbac0e32397e36487d90cca98ddd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 13:48:36 -0500 Subject: [PATCH 0957/1042] voice clone requests endpoints --- .../up.sql | 11 +- .../get_voice_clone_request.rs | 41 +++-- .../insert_voice_clone_requests_builder.rs | 25 +-- .../src/http_server/endpoints/mod.rs | 1 + .../check_if_voice_clone_request_submitted.rs | 130 ++++++++++++++ .../create_voice_clone_request.rs | 158 ++++++++++++++++++ .../endpoints/voice_clone_requests/mod.rs | 2 + crates/service/storyteller_web/src/routes.rs | 29 ++++ 8 files changed, 363 insertions(+), 34 deletions(-) create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/check_if_voice_clone_request_submitted.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/create_voice_clone_request.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/mod.rs diff --git a/_migrations/2022-02-19-004032_voice_clone_requests/up.sql b/_migrations/2022-02-19-004032_voice_clone_requests/up.sql index a1435d7d44..e59d133c09 100644 --- a/_migrations/2022-02-19-004032_voice_clone_requests/up.sql +++ b/_migrations/2022-02-19-004032_voice_clone_requests/up.sql @@ -25,6 +25,10 @@ CREATE TABLE voice_clone_requests( -- ========== REASON ========== + -- Visibility + is_for_private_use BOOLEAN NOT NULL DEFAULT FALSE, + is_for_public_use BOOLEAN NOT NULL DEFAULT FALSE, + -- How they'll use is_for_studio BOOLEAN NOT NULL DEFAULT FALSE, is_for_twitch_tts BOOLEAN NOT NULL DEFAULT FALSE, @@ -34,13 +38,10 @@ CREATE TABLE voice_clone_requests( is_for_other BOOLEAN NOT NULL DEFAULT FALSE, optional_notes_on_use MEDIUMTEXT DEFAULT NULL, - -- Visibility - is_for_private_use BOOLEAN NOT NULL DEFAULT FALSE, - is_for_public_use BOOLEAN NOT NULL DEFAULT FALSE, - - -- Ownership + -- Subject/Ownership is_own_voice BOOLEAN NOT NULL DEFAULT FALSE, is_third_party_voice BOOLEAN NOT NULL DEFAULT FALSE, + optional_notes_on_subject VARCHAR(255) DEFAULT NULL, -- ========== RECORDING QUALITY / EQUIPMENT ========== diff --git a/crates/lib/database_queries/src/queries/voice_clone_requests/get_voice_clone_request.rs b/crates/lib/database_queries/src/queries/voice_clone_requests/get_voice_clone_request.rs index 20a3fee668..8da3ac7f51 100644 --- a/crates/lib/database_queries/src/queries/voice_clone_requests/get_voice_clone_request.rs +++ b/crates/lib/database_queries/src/queries/voice_clone_requests/get_voice_clone_request.rs @@ -33,6 +33,10 @@ pub struct VoiceCloneRequest { pub email_address: String, pub discord_username: String, + // Visibility + pub is_for_private_use: bool, + pub is_for_public_use: bool, + // Use pub is_for_studio: bool, pub is_for_twitch_tts: bool, @@ -42,13 +46,10 @@ pub struct VoiceCloneRequest { pub is_for_other: bool, pub optional_notes_on_use: Option, - // Visibility - pub is_for_private_use: bool, - pub is_for_public_use: bool, - - // Ownership + // Subject/Ownership pub is_own_voice: bool, pub is_third_party_voice: bool, + pub optional_notes_on_subject: Option, // Equipment pub has_clean_audio_recordings: bool, @@ -76,6 +77,9 @@ SELECT email_address, discord_username, + is_for_private_use, + is_for_public_use, + is_for_studio, is_for_twitch_tts, is_for_api_use, @@ -84,11 +88,9 @@ SELECT is_for_other, optional_notes_on_use, - is_for_private_use, - is_for_public_use, - is_own_voice, is_third_party_voice, + optional_notes_on_subject, has_clean_audio_recordings, has_good_microphone, @@ -124,6 +126,9 @@ SELECT email_address, discord_username, + is_for_private_use, + is_for_public_use, + is_for_studio, is_for_twitch_tts, is_for_api_use, @@ -132,11 +137,9 @@ SELECT is_for_other, optional_notes_on_use, - is_for_private_use, - is_for_public_use, - is_own_voice, is_third_party_voice, + optional_notes_on_subject, has_clean_audio_recordings, has_good_microphone, @@ -172,6 +175,8 @@ fn handle_result( maybe_user_token: record.maybe_user_token, email_address: record.email_address, discord_username: record.discord_username, + is_for_private_use: i8_to_bool(record.is_for_private_use), + is_for_public_use: i8_to_bool(record.is_for_public_use), is_for_studio: i8_to_bool(record.is_for_studio), is_for_twitch_tts: i8_to_bool(record.is_for_twitch_tts), is_for_api_use: i8_to_bool(record.is_for_api_use), @@ -179,10 +184,9 @@ fn handle_result( is_for_games: i8_to_bool(record.is_for_games), is_for_other: i8_to_bool(record.is_for_other), optional_notes_on_use: record.optional_notes_on_use, - is_for_private_use: i8_to_bool(record.is_for_private_use), - is_for_public_use: i8_to_bool(record.is_for_public_use), is_own_voice: i8_to_bool(record.is_own_voice), is_third_party_voice: i8_to_bool(record.is_third_party_voice), + optional_notes_on_subject: record.optional_notes_on_subject, has_clean_audio_recordings: i8_to_bool(record.has_clean_audio_recordings), has_good_microphone: i8_to_bool(record.has_good_microphone), optional_questions: record.optional_questions, @@ -201,6 +205,10 @@ struct VoiceCloneRequestInternal { pub email_address: String, pub discord_username: String, + // Visibility + pub is_for_private_use: i8, + pub is_for_public_use: i8, + // Use pub is_for_studio: i8, pub is_for_twitch_tts: i8, @@ -210,13 +218,10 @@ struct VoiceCloneRequestInternal { pub is_for_other: i8, pub optional_notes_on_use: Option, - // Visibility - pub is_for_private_use: i8, - pub is_for_public_use: i8, - - // Ownership + // Subject/Ownership pub is_own_voice: i8, pub is_third_party_voice: i8, + pub optional_notes_on_subject: Option, // Equipment pub has_clean_audio_recordings: i8, diff --git a/crates/lib/database_queries/src/queries/voice_clone_requests/insert_voice_clone_requests_builder.rs b/crates/lib/database_queries/src/queries/voice_clone_requests/insert_voice_clone_requests_builder.rs index 59c2052369..009d87c05a 100644 --- a/crates/lib/database_queries/src/queries/voice_clone_requests/insert_voice_clone_requests_builder.rs +++ b/crates/lib/database_queries/src/queries/voice_clone_requests/insert_voice_clone_requests_builder.rs @@ -12,6 +12,10 @@ pub struct InsertVoiceCloneRequestBuilder { pub email_address: String, pub discord_username: String, + // Visibility + pub is_for_private_use: bool, + pub is_for_public_use: bool, + // Use pub is_for_studio: bool, pub is_for_twitch_tts: bool, @@ -21,13 +25,10 @@ pub struct InsertVoiceCloneRequestBuilder { pub is_for_other: bool, pub optional_notes_on_use: Option, - // Visibility - pub is_for_private_use: bool, - pub is_for_public_use: bool, - - // Ownership + // Subject/Ownership pub is_own_voice: bool, pub is_third_party_voice: bool, + pub optional_notes_on_subject: Option, // Equipment pub has_clean_audio_recordings: bool, @@ -58,6 +59,9 @@ SET email_address = ?, discord_username = ?, + is_for_private_use = ?, + is_for_public_use = ?, + is_for_studio = ?, is_for_twitch_tts = ?, is_for_api_use = ?, @@ -66,11 +70,9 @@ SET is_for_other = ?, optional_notes_on_use = ?, - is_for_private_use = ?, - is_for_public_use = ?, - is_own_voice = ?, is_third_party_voice = ?, + optional_notes_on_subject = ?, has_clean_audio_recordings = ?, has_good_microphone = ?, @@ -87,6 +89,9 @@ SET &self.email_address, &self.discord_username, + &self.is_for_private_use, + &self.is_for_public_use, + &self.is_for_studio, &self.is_for_twitch_tts, &self.is_for_api_use, @@ -95,11 +100,9 @@ SET &self.is_for_other, &self.optional_notes_on_use, - &self.is_for_private_use, - &self.is_for_public_use, - &self.is_own_voice, &self.is_third_party_voice, + &self.optional_notes_on_subject, &self.has_clean_audio_recordings, &self.has_good_microphone, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/mod.rs index e779339a76..0c8b179f69 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/mod.rs @@ -7,4 +7,5 @@ pub mod moderation; pub mod tts; pub mod twitch; pub mod users; +pub mod voice_clone_requests; pub mod w2l; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/check_if_voice_clone_request_submitted.rs b/crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/check_if_voice_clone_request_submitted.rs new file mode 100644 index 0000000000..418cb41748 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/check_if_voice_clone_request_submitted.rs @@ -0,0 +1,130 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use database_queries::queries::voice_clone_requests::get_voice_clone_request::{get_voice_clone_request_by_user_token, get_voice_clone_request_by_token}; +use database_queries::queries::voice_clone_requests::insert_voice_clone_requests_builder::InsertVoiceCloneRequestBuilder; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Request =============== + +#[derive(Deserialize)] +pub struct CheckIfVoiceRequestSubmittedRequest { + /// Though we look up logged in users by user token, we can also look up non-logged-in users + /// with a token lookup. We can store this in frontend state or a cookie. + pub maybe_request_token: Option, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct CheckIfVoiceRequestSubmittedResponse { + pub success: bool, + pub has_submitted: bool, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum CheckIfVoiceRequestSubmittedError { + BadInput(String), + NotAuthorized, + ServerError, +} + +impl ResponseError for CheckIfVoiceRequestSubmittedError { + fn status_code(&self) -> StatusCode { + match *self { + CheckIfVoiceRequestSubmittedError::BadInput(_) => StatusCode::BAD_REQUEST, + CheckIfVoiceRequestSubmittedError::NotAuthorized => StatusCode::UNAUTHORIZED, + CheckIfVoiceRequestSubmittedError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for CheckIfVoiceRequestSubmittedError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn check_if_voice_clone_request_submitted_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CheckIfVoiceRequestSubmittedError::ServerError + })?; + + let mut submitted_by_token = false; + let mut submitted_by_user = false; + + // Non-logged in accounts (via cookie or frontend mechanism) + if let Some(token) = request.maybe_request_token.as_deref() { + let request = get_voice_clone_request_by_token( + token, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Database error: {:?}", e); + CheckIfVoiceRequestSubmittedError::ServerError + })?; + + submitted_by_token = request.is_some(); + } + + // Logged in users + if let Some(user) = maybe_user_session.as_ref() { + let request = get_voice_clone_request_by_user_token( + &user.user_token, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Database error: {:?}", e); + CheckIfVoiceRequestSubmittedError::ServerError + })?; + + submitted_by_user = request.is_some(); + } + + let has_submitted = submitted_by_token || submitted_by_user; + + let response = CheckIfVoiceRequestSubmittedResponse { + success: true, + has_submitted, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| CheckIfVoiceRequestSubmittedError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/create_voice_clone_request.rs b/crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/create_voice_clone_request.rs new file mode 100644 index 0000000000..024b3e1fc0 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/create_voice_clone_request.rs @@ -0,0 +1,158 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use database_queries::queries::voice_clone_requests::insert_voice_clone_requests_builder::InsertVoiceCloneRequestBuilder; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Request =============== + +#[derive(Deserialize)] +pub struct CreateVoiceRequestRequest { + pub idempotency_token: String, + + // Contact + pub email_address: String, + pub discord_username: String, + + // TODO: Make enum. + // Visibility + pub is_for_private_use: bool, + pub is_for_public_use: bool, + + // Use + pub is_for_studio: bool, + pub is_for_twitch_tts: bool, + pub is_for_api_use: bool, + pub is_for_music: bool, + pub is_for_games: bool, + pub is_for_other: bool, + pub optional_notes_on_use: Option, + + // Subject/Ownership + pub is_own_voice: bool, + pub is_third_party_voice: bool, + pub optional_notes_on_subject: Option, + + // Equipment + pub has_clean_audio_recordings: bool, + pub has_good_microphone: bool, + + // Comments + pub optional_questions: Option, + pub optional_extra_comments: Option, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct CreateVoiceRequestResponse { + pub success: bool, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum CreateVoiceRequestError { + BadInput(String), + NotAuthorized, + ServerError, +} + +impl ResponseError for CreateVoiceRequestError { + fn status_code(&self) -> StatusCode { + match *self { + CreateVoiceRequestError::BadInput(_) => StatusCode::BAD_REQUEST, + CreateVoiceRequestError::NotAuthorized => StatusCode::UNAUTHORIZED, + CreateVoiceRequestError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl fmt::Display for CreateVoiceRequestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn create_voice_clone_request_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CreateVoiceRequestError::ServerError + })?; + + let creator_ip_address = get_request_ip(&http_request); + + let builder = InsertVoiceCloneRequestBuilder { + uuid_idempotency_token: request.idempotency_token.clone(), + maybe_user_token: maybe_user_session.map(|user| user.user_token.clone()), + email_address: request.email_address.clone(), + discord_username: request.discord_username.clone(), + is_for_private_use: request.is_for_private_use, + is_for_public_use: request.is_for_public_use, + is_for_studio: request.is_for_studio, + is_for_twitch_tts: request.is_for_twitch_tts, + is_for_api_use: request.is_for_api_use, + is_for_music: request.is_for_music, + is_for_games: request.is_for_games, + is_for_other: request.is_for_other, + optional_notes_on_use: request.optional_notes_on_use.clone(), + is_own_voice: request.is_own_voice, + is_third_party_voice: request.is_third_party_voice, + optional_notes_on_subject: request.optional_notes_on_subject.clone(), + has_clean_audio_recordings: request.has_clean_audio_recordings, + has_good_microphone: request.has_good_microphone, + optional_questions: request.optional_questions.clone(), + optional_extra_comments: request.optional_extra_comments.clone(), + ip_address_creation: creator_ip_address, + }; + + builder.insert(&server_state.mysql_pool) + .await + .map_err(|e| { + warn!("creation error: {:?}", e); + CreateVoiceRequestError::ServerError + })?; + + let response = CreateVoiceRequestResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| CreateVoiceRequestError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/mod.rs new file mode 100644 index 0000000000..8cea07f4c1 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/voice_clone_requests/mod.rs @@ -0,0 +1,2 @@ +pub mod create_voice_clone_request; +pub mod check_if_voice_clone_request_submitted; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index ac29eebd4e..cd26f9b8f2 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -79,6 +79,8 @@ use crate::http_server::endpoints::twitch::event_rules::create_event_rule::creat use crate::http_server::endpoints::twitch::event_rules::list_event_rules_for_user::list_twitch_event_rules_for_user_handler; use crate::http_server::endpoints::twitch::event_rules::delete_event_rule::delete_twitch_event_rule_handler; use crate::http_server::endpoints::twitch::event_rules::edit_event_rule::edit_twitch_event_rule_handler; +use crate::http_server::endpoints::voice_clone_requests::create_voice_clone_request::create_voice_clone_request_handler; +use crate::http_server::endpoints::voice_clone_requests::check_if_voice_clone_request_submitted::check_if_voice_clone_request_submitted_handler; pub fn add_routes (app: App) -> App where @@ -97,6 +99,7 @@ pub fn add_routes (app: App) -> App app = add_category_routes(app); /* /category */ app = add_user_profile_routes(app); /* /user */ app = add_api_token_routes(app); /* /api_tokens */ + app = add_voice_clone_request_routes(app); /* /voice_clone_requests */ app = add_twitch_oauth_routes(app); /* /twitch */ // TODO: MAYBE TEMPORARY // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== @@ -571,6 +574,32 @@ fn add_api_token_routes (app: App) -> App ) } +// ==================== API TOKEN ROUTES ==================== + +fn add_voice_clone_request_routes (app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service(web::scope("/voice_clone_requests") + .service(web::resource("/create") + .route(web::post().to(create_voice_clone_request_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/check") + .route(web::post().to(check_if_voice_clone_request_submitted_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) +} + + // ==================== TWITCH ROUTES ==================== // TODO: Maybe move these into an "oauth-gateway" type http service. From 361bfed6d6dd6caa3c85b47637d79687c8895686 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 13:50:40 -0500 Subject: [PATCH 0958/1042] re-codegen db --- sqlx-data.json | 1176 +++++++++++++++++++++++++----------------------- 1 file changed, 601 insertions(+), 575 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index a011038df4..4497be793a 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1868,6 +1868,16 @@ ] } }, + "4cea10944b8869419ecdfea30c58f49a87c090b9083078043177c33737ecb949": { + "query": "\nINSERT INTO voice_clone_requests\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n maybe_user_token = ?,\n email_address = ?,\n discord_username = ?,\n\n is_for_private_use = ?,\n is_for_public_use = ?,\n\n is_for_studio = ?,\n is_for_twitch_tts = ?,\n is_for_api_use = ?,\n is_for_music = ?,\n is_for_games = ?,\n is_for_other = ?,\n optional_notes_on_use = ?,\n\n is_own_voice = ?,\n is_third_party_voice = ?,\n optional_notes_on_subject = ?,\n\n has_clean_audio_recordings = ?,\n has_good_microphone = ?,\n\n optional_questions = ?,\n optional_extra_comments = ?,\n\n ip_address_creation = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 22 + }, + "nullable": [] + } + }, "4df4765816d8c431a241b5ea2e7e9b292bf8e603937b095fe256cbc8a7a165f6": { "query": "\nSELECT\n internal_token,\n api_token,\n maybe_short_description,\n created_at,\n updated_at\nFROM api_tokens\nWHERE\n user_token = ?\n AND deleted_at IS NULL\nORDER BY created_at DESC\nLIMIT 5\n ", "describe": { @@ -3337,291 +3347,6 @@ "nullable": [] } }, - "6768d25558d29345eeb5aa43285485bf7b23e90834b27aacdc2da700a3440a03": { - "query": "\nSELECT\n token,\n\n maybe_user_token,\n email_address,\n discord_username,\n\n is_for_studio,\n is_for_twitch_tts,\n is_for_api_use,\n is_for_music,\n is_for_games,\n is_for_other,\n optional_notes_on_use,\n\n is_for_private_use,\n is_for_public_use,\n\n is_own_voice,\n is_third_party_voice,\n\n has_clean_audio_recordings,\n has_good_microphone,\n\n optional_questions,\n optional_extra_comments,\n\n created_at,\n updated_at\nFROM voice_clone_requests\nWHERE\n token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "maybe_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "email_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 3, - "name": "discord_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "is_for_studio", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 5, - "name": "is_for_twitch_tts", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 6, - "name": "is_for_api_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 7, - "name": "is_for_music", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "is_for_games", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_for_other", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "optional_notes_on_use", - "type_info": { - "type": "Blob", - "flags": { - "bits": 144 - }, - "char_set": 224, - "max_size": 67108860 - } - }, - { - "ordinal": 11, - "name": "is_for_private_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "is_for_public_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_own_voice", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "is_third_party_voice", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 15, - "name": "has_clean_audio_recordings", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 16, - "name": "has_good_microphone", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 17, - "name": "optional_questions", - "type_info": { - "type": "Blob", - "flags": { - "bits": 144 - }, - "char_set": 224, - "max_size": 67108860 - } - }, - { - "ordinal": 18, - "name": "optional_extra_comments", - "type_info": { - "type": "Blob", - "flags": { - "bits": 144 - }, - "char_set": 224, - "max_size": 67108860 - } - }, - { - "ordinal": 19, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - true, - false, - false, - false, - false, - false, - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - true, - true, - false, - false - ] - } - }, "6aaa7c04e6984434bd2feb6f42a9014cd45e995b94e4d5e29f2d502f09136691": { "query": "\nUPDATE tts_models\nSET\n maybe_mod_user_token = ?,\n mod_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -4568,6 +4293,304 @@ ] } }, + "922a7dbcfa24a2c5901d0453c229b066b3b4eea2c73fa533651bb27365f17be0": { + "query": "\nSELECT\n token,\n\n maybe_user_token,\n email_address,\n discord_username,\n\n is_for_private_use,\n is_for_public_use,\n\n is_for_studio,\n is_for_twitch_tts,\n is_for_api_use,\n is_for_music,\n is_for_games,\n is_for_other,\n optional_notes_on_use,\n\n is_own_voice,\n is_third_party_voice,\n optional_notes_on_subject,\n\n has_clean_audio_recordings,\n has_good_microphone,\n\n optional_questions,\n optional_extra_comments,\n\n created_at,\n updated_at\nFROM voice_clone_requests\nWHERE\n token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "discord_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "is_for_private_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 5, + "name": "is_for_public_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 6, + "name": "is_for_studio", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 7, + "name": "is_for_twitch_tts", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "is_for_api_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_for_music", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "is_for_games", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 11, + "name": "is_for_other", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "optional_notes_on_use", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 13, + "name": "is_own_voice", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "is_third_party_voice", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "optional_notes_on_subject", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 16, + "name": "has_clean_audio_recordings", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "has_good_microphone", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 18, + "name": "optional_questions", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 19, + "name": "optional_extra_comments", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 20, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false, + true, + false, + false, + true, + true, + false, + false + ] + } + }, "961f792c24bee6009e3e8a215561db932871a53c48839b8cb80292de00e7b2e1": { "query": "\nUPDATE tts_results\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -6547,14 +6570,302 @@ "nullable": [] } }, - "e954932f95837c915c943aabdf55cf7535340036e32d4fba4c43a67d66bb21e0": { - "query": "\nINSERT INTO voice_clone_requests\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n maybe_user_token = ?,\n email_address = ?,\n discord_username = ?,\n\n is_for_studio = ?,\n is_for_twitch_tts = ?,\n is_for_api_use = ?,\n is_for_music = ?,\n is_for_games = ?,\n is_for_other = ?,\n optional_notes_on_use = ?,\n\n is_for_private_use = ?,\n is_for_public_use = ?,\n\n is_own_voice = ?,\n is_third_party_voice = ?,\n\n has_clean_audio_recordings = ?,\n has_good_microphone = ?,\n\n optional_questions = ?,\n optional_extra_comments = ?,\n\n ip_address_creation = ?\n ", + "e5ef2bda28221a570da6dcfa73e76cae9910a0b4ee53005a87176ecf1be73dd7": { + "query": "\nSELECT\n token,\n\n maybe_user_token,\n email_address,\n discord_username,\n\n is_for_private_use,\n is_for_public_use,\n\n is_for_studio,\n is_for_twitch_tts,\n is_for_api_use,\n is_for_music,\n is_for_games,\n is_for_other,\n optional_notes_on_use,\n\n is_own_voice,\n is_third_party_voice,\n optional_notes_on_subject,\n\n has_clean_audio_recordings,\n has_good_microphone,\n\n optional_questions,\n optional_extra_comments,\n\n created_at,\n updated_at\nFROM voice_clone_requests\nWHERE\n maybe_user_token = ?\n ", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 3, + "name": "discord_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "is_for_private_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 5, + "name": "is_for_public_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 6, + "name": "is_for_studio", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 7, + "name": "is_for_twitch_tts", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "is_for_api_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_for_music", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "is_for_games", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 11, + "name": "is_for_other", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "optional_notes_on_use", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 13, + "name": "is_own_voice", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "is_third_party_voice", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "optional_notes_on_subject", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 16, + "name": "has_clean_audio_recordings", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "has_good_microphone", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 18, + "name": "optional_questions", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 19, + "name": "optional_extra_comments", + "type_info": { + "type": "Blob", + "flags": { + "bits": 144 + }, + "char_set": 224, + "max_size": 67108860 + } + }, + { + "ordinal": 20, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], "parameters": { - "Right": 21 + "Right": 1 }, - "nullable": [] + "nullable": [ + false, + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false, + true, + false, + false, + true, + true, + false, + false + ] } }, "e9ec861d6eb135370f9baf3a866282ec5686d4b299fc4ec10900ab32ea8577d8": { @@ -6822,291 +7133,6 @@ "nullable": [] } }, - "fc421ad1b946460bf443271730b969c96850b3c5d467c8f785730bc8ec163780": { - "query": "\nSELECT\n token,\n\n maybe_user_token,\n email_address,\n discord_username,\n\n is_for_studio,\n is_for_twitch_tts,\n is_for_api_use,\n is_for_music,\n is_for_games,\n is_for_other,\n optional_notes_on_use,\n\n is_for_private_use,\n is_for_public_use,\n\n is_own_voice,\n is_third_party_voice,\n\n has_clean_audio_recordings,\n has_good_microphone,\n\n optional_questions,\n optional_extra_comments,\n\n created_at,\n updated_at\nFROM voice_clone_requests\nWHERE\n maybe_user_token = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "maybe_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "email_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 3, - "name": "discord_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "is_for_studio", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 5, - "name": "is_for_twitch_tts", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 6, - "name": "is_for_api_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 7, - "name": "is_for_music", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "is_for_games", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_for_other", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "optional_notes_on_use", - "type_info": { - "type": "Blob", - "flags": { - "bits": 144 - }, - "char_set": 224, - "max_size": 67108860 - } - }, - { - "ordinal": 11, - "name": "is_for_private_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "is_for_public_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_own_voice", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "is_third_party_voice", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 15, - "name": "has_clean_audio_recordings", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 16, - "name": "has_good_microphone", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 17, - "name": "optional_questions", - "type_info": { - "type": "Blob", - "flags": { - "bits": 144 - }, - "char_set": 224, - "max_size": 67108860 - } - }, - { - "ordinal": 18, - "name": "optional_extra_comments", - "type_info": { - "type": "Blob", - "flags": { - "bits": 144 - }, - "char_set": 224, - "max_size": 67108860 - } - }, - { - "ordinal": 19, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 20, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - true, - false, - false, - false, - false, - false, - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - true, - true, - false, - false - ] - } - }, "ffa85f7c65101a43653c724479ee080e0a74577d2ae37863b039f3a452493d1e": { "query": "\nUPDATE w2l_results\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { From 8931c7175aeb42b024eb4d73772a4e85dc3dcc83 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 14:11:05 -0500 Subject: [PATCH 0959/1042] wip "complex models" --- .../complex_models/event_match_predicate.rs | 0 .../storyteller_web/src/complex_models/mod.rs | 34 +++++++++++++++++++ crates/service/storyteller_web/src/main.rs | 1 + 3 files changed, 35 insertions(+) create mode 100644 crates/service/storyteller_web/src/complex_models/event_match_predicate.rs create mode 100644 crates/service/storyteller_web/src/complex_models/mod.rs diff --git a/crates/service/storyteller_web/src/complex_models/event_match_predicate.rs b/crates/service/storyteller_web/src/complex_models/event_match_predicate.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/storyteller_web/src/complex_models/mod.rs b/crates/service/storyteller_web/src/complex_models/mod.rs new file mode 100644 index 0000000000..1a4969ac4f --- /dev/null +++ b/crates/service/storyteller_web/src/complex_models/mod.rs @@ -0,0 +1,34 @@ +pub mod event_match_predicate; + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum EventMatchPredicate { + BitsCheermoteNameExactMatch { + cheermote_name: String, + }, + BitsSpendThreshold { + minimum_bits_spent: u32, + }, + ChannelPointsRewardNameExactMatch { + reward_name: String, + }, +} + +#[cfg(test)] +mod tests { + use crate::complex_models::EventMatchPredicate; + + #[test] + fn bits_cheermote_name_exact_match() { + let rust_value = EventMatchPredicate::BitsCheermoteNameExactMatch { + cheermote_name: "Cheer1".to_string() + }; + let json = "{\"bits_cheermote_name_exact_match\":{\"cheermote_name\":\"Cheer1\"}}"; + + let converted_to_json= serde_json::to_string(&rust_value).unwrap(); + assert_eq!(&converted_to_json, json); + + let converted_from_json : EventMatchPredicate = serde_json::from_str(json).unwrap(); + assert_eq!(&converted_from_json, &rust_value); + } +} \ No newline at end of file diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 8df94e82ab..784eb37593 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -16,6 +16,7 @@ pub const RESERVED_USERNAMES : &'static str = include_str!("../../../../db/reser pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../../../../db/reserved_usernames_including.txt"); pub mod common_env; +pub mod complex_models; pub mod http_clients; pub mod http_server; pub mod routes; From f1a9def628d3109e5428d1c1a10e7dbe0ffc9251 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 22:19:51 -0500 Subject: [PATCH 0960/1042] event response and predicate --- .../complex_models/event_match_predicate.rs | 41 +++++++++++++++++++ .../src/complex_models/event_responses.rs | 32 +++++++++++++++ .../storyteller_web/src/complex_models/mod.rs | 34 +-------------- 3 files changed, 74 insertions(+), 33 deletions(-) create mode 100644 crates/service/storyteller_web/src/complex_models/event_responses.rs diff --git a/crates/service/storyteller_web/src/complex_models/event_match_predicate.rs b/crates/service/storyteller_web/src/complex_models/event_match_predicate.rs index e69de29bb2..b7a7a9fb90 100644 --- a/crates/service/storyteller_web/src/complex_models/event_match_predicate.rs +++ b/crates/service/storyteller_web/src/complex_models/event_match_predicate.rs @@ -0,0 +1,41 @@ + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum EventMatchPredicate { + /// A predicate for an exactly matched cheermote name, including the bit value, + /// eg "Cheer1". Case insensitive. + BitsCheermoteNameExactMatch { + cheermote_name: String, + }, + /// A predicate for a cheermote name without value, but with a minimum bits + /// threshold. eg "Cheer", minimum=150 bits. Case insensitive. + BitsCheermoteNameSpendThreshold { + cheermote_name_prefix: String, + minimum_bits_spent: u32, + }, + BitsSpendThreshold { + minimum_bits_spent: u32, + }, + ChannelPointsRewardNameExactMatch { + reward_name: String, + }, +} + +#[cfg(test)] +mod tests { + use crate::complex_models::event_match_predicate::EventMatchPredicate; + + #[test] + fn bits_cheermote_name_exact_match() { + let rust_value = EventMatchPredicate::BitsCheermoteNameExactMatch { + cheermote_name: "Cheer1".to_string() + }; + let json = "{\"bits_cheermote_name_exact_match\":{\"cheermote_name\":\"Cheer1\"}}"; + + let converted_to_json= serde_json::to_string(&rust_value).unwrap(); + assert_eq!(&converted_to_json, json); + + let converted_from_json : EventMatchPredicate = serde_json::from_str(json).unwrap(); + assert_eq!(&converted_from_json, &rust_value); + } +} diff --git a/crates/service/storyteller_web/src/complex_models/event_responses.rs b/crates/service/storyteller_web/src/complex_models/event_responses.rs new file mode 100644 index 0000000000..18c6835877 --- /dev/null +++ b/crates/service/storyteller_web/src/complex_models/event_responses.rs @@ -0,0 +1,32 @@ + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum EventResponse { + /// Respond with a single TTS voice. + TtsSingleVoice { + tts_model_token: String, + }, + /// Respond with a random TTS voice. + TtsRandomVoice { + tts_model_tokens: Vec, + }, +} + +#[cfg(test)] +mod tests { + use crate::complex_models::event_responses::EventResponse; + + #[test] + fn tts_single_voice() { + let rust_value = EventResponse::TtsSingleVoice { + tts_model_token: "foo".to_string() + }; + let json = "{\"tts_single_voice\":{\"tts_model_token\":\"foo\"}}"; + + let converted_to_json= serde_json::to_string(&rust_value).unwrap(); + assert_eq!(&converted_to_json, json); + + let converted_from_json : EventResponse = serde_json::from_str(json).unwrap(); + assert_eq!(&converted_from_json, &rust_value); + } +} diff --git a/crates/service/storyteller_web/src/complex_models/mod.rs b/crates/service/storyteller_web/src/complex_models/mod.rs index 1a4969ac4f..ed032a6422 100644 --- a/crates/service/storyteller_web/src/complex_models/mod.rs +++ b/crates/service/storyteller_web/src/complex_models/mod.rs @@ -1,34 +1,2 @@ pub mod event_match_predicate; - -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum EventMatchPredicate { - BitsCheermoteNameExactMatch { - cheermote_name: String, - }, - BitsSpendThreshold { - minimum_bits_spent: u32, - }, - ChannelPointsRewardNameExactMatch { - reward_name: String, - }, -} - -#[cfg(test)] -mod tests { - use crate::complex_models::EventMatchPredicate; - - #[test] - fn bits_cheermote_name_exact_match() { - let rust_value = EventMatchPredicate::BitsCheermoteNameExactMatch { - cheermote_name: "Cheer1".to_string() - }; - let json = "{\"bits_cheermote_name_exact_match\":{\"cheermote_name\":\"Cheer1\"}}"; - - let converted_to_json= serde_json::to_string(&rust_value).unwrap(); - assert_eq!(&converted_to_json, json); - - let converted_from_json : EventMatchPredicate = serde_json::from_str(json).unwrap(); - assert_eq!(&converted_from_json, &rust_value); - } -} \ No newline at end of file +pub mod event_responses; From 83aced21819022691109a46d3fc5c325b6e5f6c9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 23:35:59 -0500 Subject: [PATCH 0961/1042] use complex types --- .../complex_models/event_match_predicate.rs | 8 ++++- .../src/complex_models/event_responses.rs | 7 ++++- .../twitch/event_rules/create_event_rule.rs | 30 ++++++++++++++++--- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/crates/service/storyteller_web/src/complex_models/event_match_predicate.rs b/crates/service/storyteller_web/src/complex_models/event_match_predicate.rs index b7a7a9fb90..fe1b48be6a 100644 --- a/crates/service/storyteller_web/src/complex_models/event_match_predicate.rs +++ b/crates/service/storyteller_web/src/complex_models/event_match_predicate.rs @@ -1,21 +1,27 @@ -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] #[serde(rename_all = "snake_case")] pub enum EventMatchPredicate { + /// Default value + NotSet, + /// A predicate for an exactly matched cheermote name, including the bit value, /// eg "Cheer1". Case insensitive. BitsCheermoteNameExactMatch { cheermote_name: String, }, + /// A predicate for a cheermote name without value, but with a minimum bits /// threshold. eg "Cheer", minimum=150 bits. Case insensitive. BitsCheermoteNameSpendThreshold { cheermote_name_prefix: String, minimum_bits_spent: u32, }, + BitsSpendThreshold { minimum_bits_spent: u32, }, + ChannelPointsRewardNameExactMatch { reward_name: String, }, diff --git a/crates/service/storyteller_web/src/complex_models/event_responses.rs b/crates/service/storyteller_web/src/complex_models/event_responses.rs index 18c6835877..2e9b343b9e 100644 --- a/crates/service/storyteller_web/src/complex_models/event_responses.rs +++ b/crates/service/storyteller_web/src/complex_models/event_responses.rs @@ -1,11 +1,16 @@ +use container_common::anyhow_result::AnyhowResult; -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] #[serde(rename_all = "snake_case")] pub enum EventResponse { + /// Default value + NotSet, + /// Respond with a single TTS voice. TtsSingleVoice { tts_model_token: String, }, + /// Respond with a random TTS voice. TtsRandomVoice { tts_model_tokens: Vec, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs index b8315be056..d912c4f40f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs @@ -6,6 +6,8 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::complex_models::event_match_predicate::EventMatchPredicate; +use crate::complex_models::event_responses::EventResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -28,8 +30,8 @@ use std::sync::Arc; pub struct CreateTwitchEventRuleRequest { pub idempotency_token: String, pub event_category: TwitchEventCategory, - pub event_match_predicate: String, - pub event_response: String, + pub event_match_predicate: Option, + pub event_response: Option, pub user_specified_rule_order: u32, pub rule_is_disabled: bool, } @@ -103,12 +105,32 @@ pub async fn create_twitch_event_rule_handler( let creator_ip_address = get_request_ip(&http_request); + let event_match_predicate = request.event_match_predicate + .clone() + .unwrap_or(EventMatchPredicate::NotSet); + + let mut event_match_predicate = serde_json::to_string(&event_match_predicate) + .map_err(|e| { + return CreateTwitchEventRuleError::BadInput( + "improper EventMatchPredicate".to_string()); + })?; + + let event_response = request.event_response + .clone() + .unwrap_or(EventResponse::NotSet); + + let mut event_response = serde_json::to_string(&event_match_predicate) + .map_err(|e| { + return CreateTwitchEventRuleError::BadInput( + "improper EventResponse".to_string()); + })?; + let insert_builder = InsertTwitchEventRuleBuilder { uuid_idempotency_token: request.idempotency_token.clone(), user_token: user_session.user_token, event_category: request.event_category, - event_match_predicate: request.event_match_predicate.clone(), - event_response: request.event_response.clone(), + event_match_predicate, + event_response, user_specified_rule_order: request.user_specified_rule_order, rule_is_disabled: request.rule_is_disabled, ip_address_creation: creator_ip_address, From b7268981196ecc185c57e2b96f703c03891a980e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 23:43:24 -0500 Subject: [PATCH 0962/1042] comnplex types --- .../twitch/event_rules/edit_event_rule.rs | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs index aca1d71c87..3508cfce52 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs @@ -6,6 +6,8 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::complex_models::event_match_predicate::EventMatchPredicate; +use crate::complex_models::event_responses::EventResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -31,8 +33,8 @@ pub struct EditTwitchEventRulePathInfo { // These are not sparse updates! #[derive(Deserialize)] pub struct EditTwitchEventRuleRequest { - pub event_match_predicate: String, - pub event_response: String, + pub event_match_predicate: Option, + pub event_response: Option, pub user_specified_rule_order: u32, pub rule_is_disabled: bool, } @@ -121,10 +123,30 @@ pub async fn edit_twitch_event_rule_handler( let creator_ip_address = get_request_ip(&http_request); + let event_match_predicate = request.event_match_predicate + .clone() + .unwrap_or(EventMatchPredicate::NotSet); + + let mut event_match_predicate = serde_json::to_string(&event_match_predicate) + .map_err(|e| { + return EditTwitchEventRuleError::BadInput( + "improper EventMatchPredicate".to_string()); + })?; + + let event_response = request.event_response + .clone() + .unwrap_or(EventResponse::NotSet); + + let mut event_response = serde_json::to_string(&event_match_predicate) + .map_err(|e| { + return EditTwitchEventRuleError::BadInput( + "improper EventResponse".to_string()); + })?; + let update_builder = UpdateTwitchEventRuleBuilder { token: path.token.clone(), - event_match_predicate: request.event_match_predicate.clone(), - event_response: request.event_response.clone(), + event_match_predicate, + event_response, user_specified_rule_order: request.user_specified_rule_order, rule_is_disabled: request.rule_is_disabled, ip_address_update: creator_ip_address, From 5a51a0ca39e0ae640cf21ed1c1c95f7735f63b23 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 19 Feb 2022 23:54:07 -0500 Subject: [PATCH 0963/1042] hydrate on list too --- .../list_twitch_event_rules_for_user.rs | 1 - .../event_rules/list_event_rules_for_user.rs | 47 ++++++++++++++++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs index f8290809d9..c9a5581893 100644 --- a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs +++ b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs @@ -5,7 +5,6 @@ use crate::column_types::twitch_event_category::TwitchEventCategory; use crate::helpers::boolean_converters::i8_to_bool; use sqlx::MySqlPool; -// TODO: This is exposed to the frontend. Write a different struct. #[derive(Debug, Serialize)] pub struct TwitchEventRule { pub token: String, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs index aa42025c28..a8e3aacd9b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs @@ -7,14 +7,17 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; +use crate::complex_models::event_match_predicate::EventMatchPredicate; +use crate::complex_models::event_responses::EventResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; +use database_queries::column_types::twitch_event_category::TwitchEventCategory; use database_queries::queries::twitch::twitch_event_rules::list_twitch_event_rules_for_user::{TwitchEventRule, list_twitch_event_rules_for_user}; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use lexical_sort::natural_lexical_cmp; -use log::{info, warn, log}; +use log::{info, warn, log, error}; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; @@ -24,10 +27,22 @@ use std::sync::Arc; // =============== Success Response =============== +#[derive(Debug, Serialize)] +pub struct HydratedTwitchEventRule { + pub token: String, + pub event_category: TwitchEventCategory, + pub event_match_predicate: EventMatchPredicate, + pub event_response: EventResponse, + pub user_specified_rule_order: u32, + pub rule_is_disabled: bool, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, +} + #[derive(Serialize)] pub struct ListTwitchEventRulesResponse { pub success: bool, - pub twitch_event_rules: Vec, + pub twitch_event_rules: Vec, } @@ -95,6 +110,34 @@ pub async fn list_twitch_event_rules_for_user_handler( ListTwitchEventRulesError::ServerError })?; + let twitch_event_rules = twitch_event_rules.into_iter() + .map(|rule| { + + let event_match_predicate = serde_json::from_str(&rule.event_match_predicate) + .unwrap_or_else(|e| { + error!("Issue with deserializing: {}", e); + EventMatchPredicate::NotSet + }); + + let event_response = serde_json::from_str(&rule.event_response) + .unwrap_or_else(|e| { + error!("Issue with deserializing: {}", e); + EventResponse::NotSet + }); + + HydratedTwitchEventRule { + token: rule.token, + event_category: rule.event_category, + event_match_predicate, + event_response, + user_specified_rule_order: rule.user_specified_rule_order, + rule_is_disabled: rule.rule_is_disabled, + created_at: rule.created_at, + updated_at: rule.updated_at, + } + }) + .collect(); + let response = ListTwitchEventRulesResponse { success: true, twitch_event_rules, From 78623265df6651a4306e3b5ff03d1172b4b466f0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 20 Feb 2022 03:25:55 -0500 Subject: [PATCH 0964/1042] query website settings from twitch job --- .../complex_models/event_match_predicate.rs | 0 .../src/complex_models/event_responses.rs | 0 .../src/complex_models/mod.rs | 0 crates/lib/database_queries/src/lib.rs | 1 + crates/service/obs_gateway/src/threads/mod.rs | 1 + .../src/threads/thread_state/mod.rs | 1 + .../twitch_pubsub_subscriber_state.rs | 33 ++++++ .../twitch_pubsub_user_subscriber_thread.rs | 101 +++++++++++++++++- .../twitch/event_rules/create_event_rule.rs | 4 +- .../twitch/event_rules/edit_event_rule.rs | 4 +- .../event_rules/list_event_rules_for_user.rs | 4 +- crates/service/storyteller_web/src/main.rs | 1 - 12 files changed, 142 insertions(+), 8 deletions(-) rename crates/{service/storyteller_web => lib/database_queries}/src/complex_models/event_match_predicate.rs (100%) rename crates/{service/storyteller_web => lib/database_queries}/src/complex_models/event_responses.rs (100%) rename crates/{service/storyteller_web => lib/database_queries}/src/complex_models/mod.rs (100%) create mode 100644 crates/service/obs_gateway/src/threads/thread_state/mod.rs create mode 100644 crates/service/obs_gateway/src/threads/thread_state/twitch_pubsub_subscriber_state.rs diff --git a/crates/service/storyteller_web/src/complex_models/event_match_predicate.rs b/crates/lib/database_queries/src/complex_models/event_match_predicate.rs similarity index 100% rename from crates/service/storyteller_web/src/complex_models/event_match_predicate.rs rename to crates/lib/database_queries/src/complex_models/event_match_predicate.rs diff --git a/crates/service/storyteller_web/src/complex_models/event_responses.rs b/crates/lib/database_queries/src/complex_models/event_responses.rs similarity index 100% rename from crates/service/storyteller_web/src/complex_models/event_responses.rs rename to crates/lib/database_queries/src/complex_models/event_responses.rs diff --git a/crates/service/storyteller_web/src/complex_models/mod.rs b/crates/lib/database_queries/src/complex_models/mod.rs similarity index 100% rename from crates/service/storyteller_web/src/complex_models/mod.rs rename to crates/lib/database_queries/src/complex_models/mod.rs diff --git a/crates/lib/database_queries/src/lib.rs b/crates/lib/database_queries/src/lib.rs index defc1886a9..db1243c4eb 100644 --- a/crates/lib/database_queries/src/lib.rs +++ b/crates/lib/database_queries/src/lib.rs @@ -3,6 +3,7 @@ mod builders; pub mod column_types; +pub mod complex_models; pub mod helpers; pub mod mediators; pub mod queries; diff --git a/crates/service/obs_gateway/src/threads/mod.rs b/crates/service/obs_gateway/src/threads/mod.rs index 317b767cd4..43ebea277e 100644 --- a/crates/service/obs_gateway/src/threads/mod.rs +++ b/crates/service/obs_gateway/src/threads/mod.rs @@ -1,2 +1,3 @@ pub mod listen_for_active_obs_sessions_thread; +pub mod thread_state; pub mod twitch_pubsub_user_subscriber_thread; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/threads/thread_state/mod.rs b/crates/service/obs_gateway/src/threads/thread_state/mod.rs new file mode 100644 index 0000000000..173ba58045 --- /dev/null +++ b/crates/service/obs_gateway/src/threads/thread_state/mod.rs @@ -0,0 +1 @@ +pub mod twitch_pubsub_subscriber_state; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/threads/thread_state/twitch_pubsub_subscriber_state.rs b/crates/service/obs_gateway/src/threads/thread_state/twitch_pubsub_subscriber_state.rs new file mode 100644 index 0000000000..a6407a2917 --- /dev/null +++ b/crates/service/obs_gateway/src/threads/thread_state/twitch_pubsub_subscriber_state.rs @@ -0,0 +1,33 @@ +use database_queries::queries::twitch::twitch_event_rules::list_twitch_event_rules_for_user::TwitchEventRule; +use database_queries::column_types::twitch_event_category::TwitchEventCategory; +use database_queries::complex_models::event_match_predicate::EventMatchPredicate; +use database_queries::complex_models::event_responses::EventResponse; + +pub struct TwitchEventRuleLight { + pub token: String, + pub event_category: TwitchEventCategory, + pub event_match_predicate: EventMatchPredicate, + pub event_response: EventResponse, + pub user_specified_rule_order: u32, + pub rule_is_disabled: bool, +} + +/// The "Twitch Pubsub Subscriber" thread is complicated. +/// Let's maintain a cache we can pass around. +pub struct TwitchPubsubCachedState { + // TODO: Query global settings. + pub global_settings: Option, + + // TODO: Query event rules. + // TODO: Not in order. + pub event_rules: Vec, +} + +impl TwitchPubsubCachedState { + pub fn new() -> Self { + Self { + global_settings: None, + event_rules: Vec::new(), + } + } +} diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs index 9160e5910c..a5a87669ed 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -3,12 +3,16 @@ use container_common::anyhow_result::AnyhowResult; use container_common::thread::thread_id::ThreadId; use crate::redis::constants::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD, LEASE_CHECK_PERIOD, OBS_ACTIVE_CHECK_PERIOD, STREAMER_TTS_JOB_QUEUE_TTL_SECONDS}; use crate::redis::lease_payload::LeasePayload; +use crate::threads::thread_state::twitch_pubsub_subscriber_state::{TwitchEventRuleLight, TwitchPubsubCachedState}; use crate::twitch::constants::TWITCH_PING_CADENCE; use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; +use database_queries::complex_models::event_match_predicate::EventMatchPredicate; +use database_queries::complex_models::event_responses::EventResponse; use database_queries::queries::tts::tts_inference_jobs::insert_tts_inference_job::TtsInferenceJobInsertBuilder; +use database_queries::queries::twitch::twitch_event_rules::list_twitch_event_rules_for_user::list_twitch_event_rules_for_user; use database_queries::queries::twitch::twitch_oauth::find::{TwitchOauthTokenRecord, TwitchOauthTokenFinder}; use database_queries::queries::twitch::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; use database_queries::queries::twitch::twitch_pubsub::insert_bits::TwitchPubsubBitsInsertBuilder; @@ -20,10 +24,11 @@ use log::info; use log::warn; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2; +use time::Duration as TimeDuration; use r2d2_redis::redis::Commands; use redis_common::redis_keys::RedisKeys; use sqlx::MySql; -use std::sync::Arc; +use std::sync::{Arc, RwLock, PoisonError, RwLockWriteGuard}; use std::thread::sleep; use std::time::Duration; use time::Instant; @@ -31,10 +36,13 @@ use twitch_api2::pubsub::channel_bits::ChannelBitsEventsV2Reply; use twitch_api2::pubsub::channel_points::ChannelPointsChannelV1Reply; use twitch_api2::pubsub::{Response, TwitchResponse, TopicData}; use twitch_common::cheers::remove_cheers; +use std::ops::Sub; // TODO: Publish events back to OBS thread // TODO: (cleanup) make the logic clearer to follow. +pub const REFRESH_WEBSITE_SETTINGS_CADENCE : TimeDuration = TimeDuration::minutes(2); + // ========================================= // =============== STAGE ONE =============== // ========================================= @@ -43,9 +51,11 @@ pub struct TwitchPubsubUserSubscriberThread { thread_id: ThreadId, server_hostname: String, twitch_user_id: TwitchUserId, + maybe_user_token: Option, // Storyteller user oauth_token_refresher: OauthTokenRefresher, mysql_pool: Arc>, redis_pool: Arc>, + twitch_subscriber_state: Arc>, } impl TwitchPubsubUserSubscriberThread { @@ -62,8 +72,10 @@ impl TwitchPubsubUserSubscriberThread { oauth_token_refresher, server_hostname: server_hostname.to_string(), twitch_user_id, + maybe_user_token: None, mysql_pool, redis_pool, + twitch_subscriber_state: Arc::new(RwLock::new(TwitchPubsubCachedState::new())), } } @@ -85,6 +97,8 @@ impl TwitchPubsubUserSubscriberThread { } }; + self.maybe_user_token = record.maybe_user_token.clone(); + let expected_lease_payload = LeasePayload::from_thread_id(&self.server_hostname, &self.thread_id); @@ -105,10 +119,14 @@ impl TwitchPubsubUserSubscriberThread { // We set their first run time to now so we don't have to deal with Option<>. let now = Instant::now(); + // NB(2022-02-20): Actually, we can set these in the past with a bit of a hack. + let long_ago = Instant::now().sub(Duration::from_secs(60*60*24)); + let thread = TwitchPubsubUserSubscriberThreadStageTwo { thread_id: self.thread_id.clone(), server_hostname: self.server_hostname.clone(), twitch_user_id: self.twitch_user_id.clone(), + maybe_user_token: self.maybe_user_token.clone(), oauth_token_refresher: self.oauth_token_refresher.clone(), mysql_pool: self.mysql_pool.clone(), redis_pool: self.redis_pool.clone(), @@ -119,6 +137,8 @@ impl TwitchPubsubUserSubscriberThread { redis_lease_last_checked_at: now, obs_session_active_last_checked_at: now, twitch_last_pinged_at: now, + twitch_subscriber_state: self.twitch_subscriber_state.clone(), + website_settings_last_refreshed_at: long_ago, }; // NB: The following call will run its main loop until/unless the Twitch client @@ -170,10 +190,15 @@ struct TwitchPubsubUserSubscriberThreadStageTwo { thread_id: ThreadId, server_hostname: String, twitch_user_id: TwitchUserId, + maybe_user_token: Option, oauth_token_refresher: OauthTokenRefresher, mysql_pool: Arc>, redis_pool: Arc>, + // ========== CACHE ========== + + twitch_subscriber_state: Arc>, + // ========== Stage Two Thread State ========== twitch_websocket_client: TwitchWebsocketClient, @@ -202,6 +227,9 @@ struct TwitchPubsubUserSubscriberThreadStageTwo { /// message within 10 seconds of issuing a PING command, it should reconnect /// to the server. See details in Handling Connection Failures." twitch_last_pinged_at: Instant, + + // When we last reloaded the website settings + website_settings_last_refreshed_at: Instant, } impl TwitchPubsubUserSubscriberThreadStageTwo { @@ -223,6 +251,8 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { return Ok(LoopEndedReason::ExitThread { reason: "OBS session ended".to_string() }); } + self.maybe_refresh_website_user_settings().await?; + // NB: We can't have calls to read the Twitch websocket client block forever, and they // would do exactly that if not for this code. This is adapted from the very good example // in the `tokio-tungstenite` repo, which also contains good recipes for splitting sockets @@ -578,6 +608,44 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { Ok(()) } + + // =============== WEBSITE SETTINGS =============== + + async fn maybe_refresh_website_user_settings(&mut self) -> AnyhowResult<()> { + let user_token = match self.maybe_user_token.as_deref() { + Some(token) => token, + None => { + // Only Twitch users with a storyteller account get settings. + return Ok(()); + } + }; + + let mut should_refresh = self.website_settings_last_refreshed_at + .elapsed() + .gt(&REFRESH_WEBSITE_SETTINGS_CADENCE); + + if !should_refresh { + return Ok(()); + } + + info!("Reloading website settings for user {}", self.twitch_user_id.get_numeric()); + + let event_rules = + lookup_event_rules(user_token, &self.mysql_pool).await?; + + match self.twitch_subscriber_state.write() { + Err(e) => { + return Err(anyhow!("Lock error: {:?}", e)) + }, + Ok(mut lock) => { + lock.event_rules = event_rules; + } + } + + self.website_settings_last_refreshed_at = Instant::now(); + + Ok(()) + } } // ==================================== @@ -595,6 +663,37 @@ async fn lookup_oauth_record( .await } +async fn lookup_event_rules( + user_token: &str, + mysql_pool: &sqlx::Pool +) -> AnyhowResult> { + let rules = list_twitch_event_rules_for_user(user_token, mysql_pool) + .await?; + let rules = rules.into_iter() + .map(|rule| { + let event_match_predicate = serde_json::from_str(&rule.event_match_predicate) + .unwrap_or_else(|e| { + error!("Issue with deserializing: {}", e); + EventMatchPredicate::NotSet + }); + let event_response = serde_json::from_str(&rule.event_response) + .unwrap_or_else(|e| { + error!("Issue with deserializing: {}", e); + EventResponse::NotSet + }); + TwitchEventRuleLight { + token: rule.token, + event_category: rule.event_category, + event_match_predicate, + event_response, + user_specified_rule_order: 0, + rule_is_disabled: false + } + }) + .collect(); + Ok(rules) +} + enum LoopEndedReason { /// Terminate the thread ExitThread { reason: String }, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs index d912c4f40f..64e44fd93a 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs @@ -6,13 +6,13 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::complex_models::event_match_predicate::EventMatchPredicate; -use crate::complex_models::event_responses::EventResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use database_queries::column_types::twitch_event_category::TwitchEventCategory; +use database_queries::complex_models::event_match_predicate::EventMatchPredicate; +use database_queries::complex_models::event_responses::EventResponse; use database_queries::queries::twitch::twitch_event_rules::insert_twitch_event_rule_builder::InsertTwitchEventRuleBuilder; use database_queries::tokens::Tokens; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs index 3508cfce52..8caefa46bd 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs @@ -6,12 +6,12 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; -use crate::complex_models::event_match_predicate::EventMatchPredicate; -use crate::complex_models::event_responses::EventResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; +use database_queries::complex_models::event_match_predicate::EventMatchPredicate; +use database_queries::complex_models::event_responses::EventResponse; use database_queries::queries::twitch::twitch_event_rules::get_twitch_event_rule_for_user::get_twitch_event_rules_for_user; use database_queries::queries::twitch::twitch_event_rules::update_twitch_event_rule_builder::UpdateTwitchEventRuleBuilder; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs index a8e3aacd9b..11e2fb5ea8 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs @@ -7,13 +7,13 @@ use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use chrono::{DateTime, Utc}; -use crate::complex_models::event_match_predicate::EventMatchPredicate; -use crate::complex_models::event_responses::EventResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use database_queries::column_types::twitch_event_category::TwitchEventCategory; +use database_queries::complex_models::event_match_predicate::EventMatchPredicate; +use database_queries::complex_models::event_responses::EventResponse; use database_queries::queries::twitch::twitch_event_rules::list_twitch_event_rules_for_user::{TwitchEventRule, list_twitch_event_rules_for_user}; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use lexical_sort::natural_lexical_cmp; diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 784eb37593..8df94e82ab 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -16,7 +16,6 @@ pub const RESERVED_USERNAMES : &'static str = include_str!("../../../../db/reser pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../../../../db/reserved_usernames_including.txt"); pub mod common_env; -pub mod complex_models; pub mod http_clients; pub mod http_server; pub mod routes; From e47222ac29451e7c98bcafece68963ae863d8d55 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 20 Feb 2022 03:47:38 -0500 Subject: [PATCH 0965/1042] add user token to twitch oauth (untested, stupid ssl) --- .../endpoints/twitch/oauth/oauth_end.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/oauth_end.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/oauth_end.rs index 612c75e1fc..1bab1a8e9b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/oauth_end.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/oauth_end.rs @@ -64,7 +64,17 @@ pub async fn oauth_end_enroll_from_redirect( http_request: HttpRequest, query: web::Query, server_state: web::Data> -) -> Result { +) + -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + OauthEndEnrollFromRedirectError::ServerError + })?; if let Some(error) = query.error.as_deref() { return Err(OauthEndEnrollFromRedirectError::TwitchOauthError { reason: error.to_string() }) @@ -158,7 +168,11 @@ pub async fn oauth_end_enroll_from_redirect( .has_chat_edit(true) .has_chat_read(true); - let result = insert_builder.insert(&server_state.mysql_pool) + if let Some(user_session) = maybe_user_session { + insert_builder = insert_builder.set_user_token(Some(&user_session.user_token)); + } + + let _result = insert_builder.insert(&server_state.mysql_pool) .await .map_err(|e| { warn!("Error saving to db: {:?}", e); From bee4f25e2d2116068358fe3f1dd3a9899f32d02f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 20 Feb 2022 03:59:36 -0500 Subject: [PATCH 0966/1042] Local Dev SSL instructions for Linux (they work!) --- localdev/nginx-http-config/storyteller-ssl | 31 +++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/localdev/nginx-http-config/storyteller-ssl b/localdev/nginx-http-config/storyteller-ssl index 886b5fc264..305753808b 100644 --- a/localdev/nginx-http-config/storyteller-ssl +++ b/localdev/nginx-http-config/storyteller-ssl @@ -4,6 +4,10 @@ # # - /etc/nginx/sites-enabled/{name-of-site} # +# Move this file there, and then restart with, +# +# sudo service nginx restart +# # On Mac, this is located at the following locations: # # - /opt/homebrew/etc/nginx/servers/{name-of-site} (for nginx installed with homebrew) @@ -18,7 +22,17 @@ # ================ # SSL Instructions # ================ -# +# +# To create self-signed certs on Ubuntu Linux, +# +# sudo apt-get install openssl +# +# sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/localhost.key -out /etc/ssl/certs/localhost.crt +# +# ^ most fields can be blank, but the "Common Name" should be `*.jungle.horse` +# +# For more: https://www.cloudsavvyit.com/1306/how-to-create-and-use-self-signed-ssl-on-nginx/ +# # To create self-signed certs on Mac, # # openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt @@ -48,6 +62,21 @@ server { } } +# API backend over SSL +server { + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/ssl/certs/localhost.crt; + ssl_certificate_key /etc/ssl/private/localhost.key; + ssl_protocols TLSv1.2 TLSv1.1 TLSv1; + server_name api.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:12345; + proxy_redirect off; + } +} + # TODO(2022-02-13): This is new and untested # WebSocket API backend over plain HTTP server { From 8d2aad48aa5cb37cb77e3f2d1971b09a1f759d97 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 20 Feb 2022 20:05:41 -0500 Subject: [PATCH 0967/1042] necessary for local dev --- crates/service/storyteller_web/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 8df94e82ab..c56880a6d4 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -268,11 +268,11 @@ async fn main() -> AnyhowResult<()> { let twitch_oauth_redirect_landing_url = easyenv::get_env_string_or_default( "TWITCH_OAUTH_REDIRECT_LANDING_URL", - "http://localhost:12345/twitch/oauth_landing"); + "https://api.jungle.horse/twitch/oauth_landing"); let twitch_oauth_redirect_landing_finished_url = easyenv::get_env_string_or_default( "TWITCH_OAUTH_REDIRECT_LANDING_FINISHED_URL", - "http://localhost:12345/"); + "https://api.jungle.horse/"); let twitch_secrets = TwitchSecrets::from_env()?; From 844db3f6a86dd8d518239bd492730c25592cec63 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 20 Feb 2022 23:31:09 -0500 Subject: [PATCH 0968/1042] split obs-gateway into websocket-gateway and twitch-pubsub-... --- Cargo.lock | 203 ++++++++++++------ Cargo.toml | 3 +- Dockerfile | 8 +- .../src/queries/twitch/twitch_oauth/insert.rs | 1 + crates/lib/redis_common/Cargo.toml | 16 ++ crates/lib/redis_common/src/lib.rs | 4 + .../src/payloads}/lease_payload.rs | 4 +- .../redis_common/src/payloads}/mod.rs | 1 - .../src/payloads}/obs_active_payload.rs | 2 + .../redis_common/src/shared_constants.rs} | 0 crates/lib/twitch_common/src/lib.rs | 3 +- .../twitch_common/src}/twitch_user_id.rs | 0 .../twitch_pubsub_subscriber/Cargo.toml | 119 ++++++++++ .../src/main.rs} | 1 - .../listen_for_active_obs_sessions_thread.rs | 10 +- .../src/threads/mod.rs | 0 .../src/threads/thread_state/mod.rs | 0 .../twitch_pubsub_subscriber_state.rs | 0 .../twitch_pubsub_user_subscriber_thread.rs | 14 +- .../src/twitch/constants.rs | 0 .../src/twitch/mod.rs | 1 - .../src/twitch/oauth/mod.rs | 0 .../src/twitch/oauth/oauth_token_refresher.rs | 0 .../pubsub/build_pubsub_topics_for_user.rs | 0 .../src/twitch/pubsub/mod.rs | 0 .../src/twitch/websocket_client.rs | 0 .../src/util/mod.rs | 0 .../src/util/random_nonce.rs | 0 .../Cargo.toml | 4 +- .../src/endpoints_ws/mod.rs | 0 .../obs_gateway_websocket_handler.rs | 8 +- .../src/main.rs | 4 - .../src/server_state.rs | 0 33 files changed, 310 insertions(+), 96 deletions(-) rename crates/{service/obs_gateway/src/redis => lib/redis_common/src/payloads}/lease_payload.rs (96%) rename crates/{service/obs_gateway/src/redis => lib/redis_common/src/payloads}/mod.rs (72%) rename crates/{service/obs_gateway/src/redis => lib/redis_common/src/payloads}/obs_active_payload.rs (94%) rename crates/{service/obs_gateway/src/redis/constants.rs => lib/redis_common/src/shared_constants.rs} (100%) rename crates/{service/obs_gateway/src/twitch => lib/twitch_common/src}/twitch_user_id.rs (100%) create mode 100644 crates/service/twitch_pubsub_subscriber/Cargo.toml rename crates/service/{obs_gateway/src/twitch_pubsub_job.rs => twitch_pubsub_subscriber/src/main.rs} (99%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/threads/listen_for_active_obs_sessions_thread.rs (93%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/threads/mod.rs (100%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/threads/thread_state/mod.rs (100%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/threads/thread_state/twitch_pubsub_subscriber_state.rs (100%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/threads/twitch_pubsub_user_subscriber_thread.rs (98%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/twitch/constants.rs (100%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/twitch/mod.rs (75%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/twitch/oauth/mod.rs (100%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/twitch/oauth/oauth_token_refresher.rs (100%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/twitch/pubsub/build_pubsub_topics_for_user.rs (100%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/twitch/pubsub/mod.rs (100%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/twitch/websocket_client.rs (100%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/util/mod.rs (100%) rename crates/service/{obs_gateway => twitch_pubsub_subscriber}/src/util/random_nonce.rs (100%) rename crates/service/{obs_gateway => websocket_gateway}/Cargo.toml (98%) rename crates/service/{obs_gateway => websocket_gateway}/src/endpoints_ws/mod.rs (100%) rename crates/service/{obs_gateway => websocket_gateway}/src/endpoints_ws/obs_gateway_websocket_handler.rs (96%) rename crates/service/{obs_gateway => websocket_gateway}/src/main.rs (98%) rename crates/service/{obs_gateway => websocket_gateway}/src/server_state.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index fc78aff667..23c484d7ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2566,71 +2566,6 @@ dependencies = [ "libc", ] -[[package]] -name = "obs-gateway-server" -version = "0.0.1" -dependencies = [ - "actix", - "actix-cors 0.5.4", - "actix-files", - "actix-http 3.0.0-beta.11", - "actix-rt 2.4.0", - "actix-service 2.0.1", - "actix-utils 3.0.0", - "actix-web 4.0.0-beta.10", - "actix-web-actors", - "anyhow", - "base64 0.13.0", - "bcrypt", - "chrono", - "config", - "container_common", - "crossbeam", - "database_queries", - "derive_more", - "diesel", - "dotenv", - "easyenv", - "futures 0.1.31", - "futures 0.3.17", - "futures-timer", - "futures-util", - "hex", - "hmac 0.10.1", - "hostname", - "http_server_common", - "hyper 0.13.10", - "jwt", - "lazy_static", - "lfu", - "limitation", - "log", - "lru_time_cache", - "magic-crypt", - "md-5", - "native-tls", - "pulldown-cmark", - "r2d2_redis", - "rand 0.8.4", - "redis_common", - "regex", - "reqwest", - "rpassword", - "serde", - "serde_derive", - "serde_json", - "sha2", - "sqlx", - "time 0.2.26", - "tokio 1.13.0", - "tokio-tungstenite", - "toml", - "twitch_api2", - "twitch_common", - "twitch_oauth2", - "uuid", -] - [[package]] name = "once_cell" version = "1.9.0" @@ -3078,6 +3013,14 @@ dependencies = [ [[package]] name = "redis_common" version = "0.0.1" +dependencies = [ + "anyhow", + "container_common", + "serde", + "serde_derive", + "serde_json", + "time 0.2.26", +] [[package]] name = "redox_syscall" @@ -4360,6 +4303,71 @@ dependencies = [ "webpki", ] +[[package]] +name = "twitch-pubsub-subscriber" +version = "0.0.1" +dependencies = [ + "actix", + "actix-cors 0.5.4", + "actix-files", + "actix-http 3.0.0-beta.11", + "actix-rt 2.4.0", + "actix-service 2.0.1", + "actix-utils 3.0.0", + "actix-web 4.0.0-beta.10", + "actix-web-actors", + "anyhow", + "base64 0.13.0", + "bcrypt", + "chrono", + "config", + "container_common", + "crossbeam", + "database_queries", + "derive_more", + "diesel", + "dotenv", + "easyenv", + "futures 0.1.31", + "futures 0.3.17", + "futures-timer", + "futures-util", + "hex", + "hmac 0.10.1", + "hostname", + "http_server_common", + "hyper 0.13.10", + "jwt", + "lazy_static", + "lfu", + "limitation", + "log", + "lru_time_cache", + "magic-crypt", + "md-5", + "native-tls", + "pulldown-cmark", + "r2d2_redis", + "rand 0.8.4", + "redis_common", + "regex", + "reqwest", + "rpassword", + "serde", + "serde_derive", + "serde_json", + "sha2", + "sqlx", + "time 0.2.26", + "tokio 1.13.0", + "tokio-tungstenite", + "toml", + "twitch_api2", + "twitch_common", + "twitch_oauth2", + "uuid", +] + [[package]] name = "twitch_api2" version = "0.6.0-rc.3" @@ -4704,6 +4712,71 @@ dependencies = [ "webpki", ] +[[package]] +name = "websocket-gateway" +version = "0.0.1" +dependencies = [ + "actix", + "actix-cors 0.5.4", + "actix-files", + "actix-http 3.0.0-beta.11", + "actix-rt 2.4.0", + "actix-service 2.0.1", + "actix-utils 3.0.0", + "actix-web 4.0.0-beta.10", + "actix-web-actors", + "anyhow", + "base64 0.13.0", + "bcrypt", + "chrono", + "config", + "container_common", + "crossbeam", + "database_queries", + "derive_more", + "diesel", + "dotenv", + "easyenv", + "futures 0.1.31", + "futures 0.3.17", + "futures-timer", + "futures-util", + "hex", + "hmac 0.10.1", + "hostname", + "http_server_common", + "hyper 0.13.10", + "jwt", + "lazy_static", + "lfu", + "limitation", + "log", + "lru_time_cache", + "magic-crypt", + "md-5", + "native-tls", + "pulldown-cmark", + "r2d2_redis", + "rand 0.8.4", + "redis_common", + "regex", + "reqwest", + "rpassword", + "serde", + "serde_derive", + "serde_json", + "sha2", + "sqlx", + "time 0.2.26", + "tokio 1.13.0", + "tokio-tungstenite", + "toml", + "twitch_api2", + "twitch_common", + "twitch_oauth2", + "uuid", +] + [[package]] name = "whoami" version = "1.1.2" diff --git a/Cargo.toml b/Cargo.toml index f23c3c84bb..51af671f1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,8 @@ members = [ "crates/lib/user_input_common", # Application - "crates/service/obs_gateway", "crates/service/storyteller_web", + "crates/service/twitch_pubsub_subscriber", + "crates/service/websocket_gateway", ] diff --git a/Dockerfile b/Dockerfile index f08c08a378..91d0a45769 100644 --- a/Dockerfile +++ b/Dockerfile @@ -194,13 +194,13 @@ RUN SQLX_OFFLINE=true \ LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ $HOME/.cargo/bin/cargo build \ --release \ - --bin obs-gateway + --bin websocket-gateway RUN SQLX_OFFLINE=true \ LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} \ $HOME/.cargo/bin/cargo build \ --release \ - --bin twitch-pubsub-job + --bin twitch-pubsub-subscriber # Final image # FROM ubuntu:xenial @@ -244,8 +244,8 @@ COPY --from=rust-build /tmp/target/release/tts-download-job / COPY --from=rust-build /tmp/target/release/tts-inference-job / COPY --from=rust-build /tmp/target/release/w2l-download-job / COPY --from=rust-build /tmp/target/release/w2l-inference-job / -COPY --from=rust-build /tmp/target/release/obs-gateway / -COPY --from=rust-build /tmp/target/release/twitch-pubsub-job / +COPY --from=rust-build /tmp/target/release/websocket-gateway / +COPY --from=rust-build /tmp/target/release/twitch-pubsub-subscriber / # SSL certs are required for crypto COPY --from=rust-build /etc/ssl /etc/ssl diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_oauth/insert.rs b/crates/lib/database_queries/src/queries/twitch/twitch_oauth/insert.rs index d7f6cf1755..07d7bb3da7 100644 --- a/crates/lib/database_queries/src/queries/twitch/twitch_oauth/insert.rs +++ b/crates/lib/database_queries/src/queries/twitch/twitch_oauth/insert.rs @@ -81,6 +81,7 @@ impl TwitchOauthTokenInsertBuilder { self } + /// NB: This is the FakeYou/Storyteller user account, not Twitch! pub fn set_user_token(mut self, maybe_user_token: Option<&str>) -> Self { self.maybe_user_token = maybe_user_token.map(|t| t.to_string()); self diff --git a/crates/lib/redis_common/Cargo.toml b/crates/lib/redis_common/Cargo.toml index a81b3d4daf..79c7d5f2eb 100644 --- a/crates/lib/redis_common/Cargo.toml +++ b/crates/lib/redis_common/Cargo.toml @@ -11,3 +11,19 @@ publish = false [lib] name = "redis_common" path = "src/lib.rs" + +[dependencies] + +# Internal +container_common = { path = "../container_common" } + +# External +anyhow = "1.0.40" + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" + +# TODO: Get rid of in favor of std::time +time = "0.2.26" diff --git a/crates/lib/redis_common/src/lib.rs b/crates/lib/redis_common/src/lib.rs index 2f4e5ca934..ffd322f242 100644 --- a/crates/lib/redis_common/src/lib.rs +++ b/crates/lib/redis_common/src/lib.rs @@ -1 +1,5 @@ +#[macro_use] extern crate serde_derive; + +pub mod payloads; pub mod redis_keys; +pub mod shared_constants; diff --git a/crates/service/obs_gateway/src/redis/lease_payload.rs b/crates/lib/redis_common/src/payloads/lease_payload.rs similarity index 96% rename from crates/service/obs_gateway/src/redis/lease_payload.rs rename to crates/lib/redis_common/src/payloads/lease_payload.rs index f1305f7354..6d1b415e65 100644 --- a/crates/service/obs_gateway/src/redis/lease_payload.rs +++ b/crates/lib/redis_common/src/payloads/lease_payload.rs @@ -3,6 +3,8 @@ use container_common::anyhow_result::AnyhowResult; use container_common::thread::thread_id::ThreadId; use container_common::token::random_crockford_token::random_crockford_token; +// TODO: Rename, document + // TODO: This should be JSON for future compat. /// Lease values stored in Redis are in the form "{server_id}:{thread_id}" #[derive(Debug, Eq, PartialEq, Clone)] @@ -51,7 +53,7 @@ impl LeasePayload { #[cfg(test)] mod tests { use container_common::thread::thread_id::ThreadId; - use crate::redis::lease_payload::LeasePayload; + use crate::payloads::lease_payload::LeasePayload; #[test] fn equals() { diff --git a/crates/service/obs_gateway/src/redis/mod.rs b/crates/lib/redis_common/src/payloads/mod.rs similarity index 72% rename from crates/service/obs_gateway/src/redis/mod.rs rename to crates/lib/redis_common/src/payloads/mod.rs index e8a29762c6..6ed256ff9b 100644 --- a/crates/service/obs_gateway/src/redis/mod.rs +++ b/crates/lib/redis_common/src/payloads/mod.rs @@ -1,3 +1,2 @@ pub mod lease_payload; -pub mod constants; pub mod obs_active_payload; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/redis/obs_active_payload.rs b/crates/lib/redis_common/src/payloads/obs_active_payload.rs similarity index 94% rename from crates/service/obs_gateway/src/redis/obs_active_payload.rs rename to crates/lib/redis_common/src/payloads/obs_active_payload.rs index 949064a65d..cca1154553 100644 --- a/crates/service/obs_gateway/src/redis/obs_active_payload.rs +++ b/crates/lib/redis_common/src/payloads/obs_active_payload.rs @@ -1,5 +1,7 @@ use container_common::anyhow_result::AnyhowResult; +// TODO: Rename, document + #[derive(Serialize, Deserialize, Debug)] pub struct ObsActivePayload { pub twitch_user_id: String, diff --git a/crates/service/obs_gateway/src/redis/constants.rs b/crates/lib/redis_common/src/shared_constants.rs similarity index 100% rename from crates/service/obs_gateway/src/redis/constants.rs rename to crates/lib/redis_common/src/shared_constants.rs diff --git a/crates/lib/twitch_common/src/lib.rs b/crates/lib/twitch_common/src/lib.rs index 5373c7dce1..6ac11f6e4e 100644 --- a/crates/lib/twitch_common/src/lib.rs +++ b/crates/lib/twitch_common/src/lib.rs @@ -1,3 +1,4 @@ pub mod cheers; pub mod oauth_token_builder; -pub mod twitch_secrets; \ No newline at end of file +pub mod twitch_secrets; +pub mod twitch_user_id; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/twitch/twitch_user_id.rs b/crates/lib/twitch_common/src/twitch_user_id.rs similarity index 100% rename from crates/service/obs_gateway/src/twitch/twitch_user_id.rs rename to crates/lib/twitch_common/src/twitch_user_id.rs diff --git a/crates/service/twitch_pubsub_subscriber/Cargo.toml b/crates/service/twitch_pubsub_subscriber/Cargo.toml new file mode 100644 index 0000000000..c9c752bdfc --- /dev/null +++ b/crates/service/twitch_pubsub_subscriber/Cargo.toml @@ -0,0 +1,119 @@ +[package] +name = "twitch-pubsub-subscriber" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[[bin]] +name = "twitch-pubsub-subscriber" +path = "src/main.rs" + +[dependencies] + +# Internal +config = { path = "../../lib/config" } +container_common = { path = "../../lib/container_common" } +database_queries = { path = "../../lib/database_queries" } +http_server_common = { path = "../../lib/http_server_common" } +redis_common = { path = "../../lib/redis_common" } +twitch_common = { path = "../../lib/twitch_common" } + +# External +anyhow = "1.0.40" +base64 = "0.13.0" +chrono = { version = "0.4.19", features = ["serde"] } +derive_more = "0.99.14" +dotenv = "0.15.0" +easyenv = "0.0.6" +futures-timer = "3.0.2" +hex = "0.4.3" +hostname = "0.3.1" +lazy_static = "1.4.0" +lfu = "0.2.5" +log = "0.4.14" +rand = "0.8.3" +regex = "1.5.4" +time = "0.2.26" +uuid = { version = "0.8.2", features = ["v4"] } + +# Runtime +# NB: We switched to Tokio from `async-std` for broader ecosystem support. +tokio = { version = "1.10.0", features = ["rt-multi-thread", "macros", "time"] } + +# Futures +futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool +futures-util = "0.3.17" +futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate + +# Threading +crossbeam = "0.8.1" + +# MySql Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +#sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } +sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } +#sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] } + +# Redis Database +limitation = "0.1.1" # Redis rate limiter +r2d2_redis = "0.14.0" + +# Twitch +# NB "unsupported" pubsub features may disappear from the Twitch API at any time. It's an unofficial API. +twitch_api2 = { version = "0.6.0-rc.2", features = ["helix", "reqwest_client", "pubsub", "chrono", "unsupported"] } +twitch_oauth2 = "0.6.0-rc.2" + +# HTTP +reqwest = { version = "0.11.4" } +native-tls = "0.2.8" + +# In-memory caching +lru_time_cache = "0.11.11" + +# Web +actix = "0.12.0" +actix-cors = "0.5.4" +actix-files= "0.6.0-beta.8" +actix-http = "3.0.0-beta.11" +actix-rt = "2.4.0" +actix-service = "2.0.1" +actix-utils = "3.0.0" +actix-web = "4.0.0-beta.10" +actix-web-actors = "4.0.0-beta.7" +jwt = "0.13.0" + +# Websockets +tokio-tungstenite = { version = "0.15.0", features = ["rustls", "rustls-tls"] } + +# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime +# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. +hyper = { version = "0.13.10" } + +# Crypto +bcrypt = "0.8" # password hashes +hmac = "0.10.0" # cookies +magic-crypt = "3.1.8" # encrypting sort IDs for the frontend so we don't leak info +md-5 = "0.9.1" # gravatar +sha2 = "0.9.5" # file hashes, inference text hashes, etc. + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" +toml = "0.5.8" + +# Text processing +pulldown-cmark = "0.8.0" + +# Temporary (just to ask for oauth credentials during dev) +rpassword = "5.0.1" + +[build-dependencies] +# NB: Only using diesel for migrations support +diesel = { version = "1.4.6", features = ["mysql"] } diff --git a/crates/service/obs_gateway/src/twitch_pubsub_job.rs b/crates/service/twitch_pubsub_subscriber/src/main.rs similarity index 99% rename from crates/service/obs_gateway/src/twitch_pubsub_job.rs rename to crates/service/twitch_pubsub_subscriber/src/main.rs index 5cf9b99768..3362a2c49b 100644 --- a/crates/service/obs_gateway/src/twitch_pubsub_job.rs +++ b/crates/service/twitch_pubsub_subscriber/src/main.rs @@ -12,7 +12,6 @@ #[macro_use] extern crate magic_crypt; #[macro_use] extern crate serde_derive; -pub mod redis; pub mod threads; pub mod twitch; pub mod util; diff --git a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs b/crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions_thread.rs similarity index 93% rename from crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs rename to crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions_thread.rs index ba53a3c5d7..e3b33de4c2 100644 --- a/crates/service/obs_gateway/src/threads/listen_for_active_obs_sessions_thread.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions_thread.rs @@ -1,22 +1,22 @@ use container_common::thread::thread_id::ThreadId; -use crate::redis::constants::LEASE_TIMEOUT_SECONDS; -use crate::redis::lease_payload::LeasePayload; -use crate::redis::obs_active_payload::ObsActivePayload; use crate::threads::twitch_pubsub_user_subscriber_thread::TwitchPubsubUserSubscriberThread; -use crate::twitch::twitch_user_id::TwitchUserId; +use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use log::error; use log::info; use log::warn; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2; use r2d2_redis::redis::Commands; +use redis_common::payloads::lease_payload::LeasePayload; +use redis_common::payloads::obs_active_payload::ObsActivePayload; use redis_common::redis_keys::RedisKeys; +use redis_common::shared_constants::LEASE_TIMEOUT_SECONDS; use sqlx::MySql; use std::sync::Arc; use std::thread::sleep; use std::time::Duration; use tokio::runtime::Runtime; -use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; +use twitch_common::twitch_user_id::TwitchUserId; // TODO: Apart from error handling, this looks mostly good. diff --git a/crates/service/obs_gateway/src/threads/mod.rs b/crates/service/twitch_pubsub_subscriber/src/threads/mod.rs similarity index 100% rename from crates/service/obs_gateway/src/threads/mod.rs rename to crates/service/twitch_pubsub_subscriber/src/threads/mod.rs diff --git a/crates/service/obs_gateway/src/threads/thread_state/mod.rs b/crates/service/twitch_pubsub_subscriber/src/threads/thread_state/mod.rs similarity index 100% rename from crates/service/obs_gateway/src/threads/thread_state/mod.rs rename to crates/service/twitch_pubsub_subscriber/src/threads/thread_state/mod.rs diff --git a/crates/service/obs_gateway/src/threads/thread_state/twitch_pubsub_subscriber_state.rs b/crates/service/twitch_pubsub_subscriber/src/threads/thread_state/twitch_pubsub_subscriber_state.rs similarity index 100% rename from crates/service/obs_gateway/src/threads/thread_state/twitch_pubsub_subscriber_state.rs rename to crates/service/twitch_pubsub_subscriber/src/threads/thread_state/twitch_pubsub_subscriber_state.rs diff --git a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber_thread.rs similarity index 98% rename from crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs rename to crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber_thread.rs index a5a87669ed..7e7b086736 100644 --- a/crates/service/obs_gateway/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber_thread.rs @@ -1,13 +1,10 @@ use anyhow::{anyhow, Error}; use container_common::anyhow_result::AnyhowResult; use container_common::thread::thread_id::ThreadId; -use crate::redis::constants::{LEASE_TIMEOUT_SECONDS, LEASE_RENEW_PERIOD, LEASE_CHECK_PERIOD, OBS_ACTIVE_CHECK_PERIOD, STREAMER_TTS_JOB_QUEUE_TTL_SECONDS}; -use crate::redis::lease_payload::LeasePayload; use crate::threads::thread_state::twitch_pubsub_subscriber_state::{TwitchEventRuleLight, TwitchPubsubCachedState}; use crate::twitch::constants::TWITCH_PING_CADENCE; use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; -use crate::twitch::twitch_user_id::TwitchUserId; use crate::twitch::websocket_client::TwitchWebsocketClient; use database_queries::complex_models::event_match_predicate::EventMatchPredicate; use database_queries::complex_models::event_responses::EventResponse; @@ -24,19 +21,26 @@ use log::info; use log::warn; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2; -use time::Duration as TimeDuration; use r2d2_redis::redis::Commands; +use redis_common::payloads::lease_payload::LeasePayload; use redis_common::redis_keys::RedisKeys; +use redis_common::shared_constants::LEASE_CHECK_PERIOD; +use redis_common::shared_constants::LEASE_RENEW_PERIOD; +use redis_common::shared_constants::LEASE_TIMEOUT_SECONDS; +use redis_common::shared_constants::OBS_ACTIVE_CHECK_PERIOD; +use redis_common::shared_constants::STREAMER_TTS_JOB_QUEUE_TTL_SECONDS; use sqlx::MySql; +use std::ops::Sub; use std::sync::{Arc, RwLock, PoisonError, RwLockWriteGuard}; use std::thread::sleep; use std::time::Duration; +use time::Duration as TimeDuration; use time::Instant; use twitch_api2::pubsub::channel_bits::ChannelBitsEventsV2Reply; use twitch_api2::pubsub::channel_points::ChannelPointsChannelV1Reply; use twitch_api2::pubsub::{Response, TwitchResponse, TopicData}; use twitch_common::cheers::remove_cheers; -use std::ops::Sub; +use twitch_common::twitch_user_id::TwitchUserId; // TODO: Publish events back to OBS thread // TODO: (cleanup) make the logic clearer to follow. diff --git a/crates/service/obs_gateway/src/twitch/constants.rs b/crates/service/twitch_pubsub_subscriber/src/twitch/constants.rs similarity index 100% rename from crates/service/obs_gateway/src/twitch/constants.rs rename to crates/service/twitch_pubsub_subscriber/src/twitch/constants.rs diff --git a/crates/service/obs_gateway/src/twitch/mod.rs b/crates/service/twitch_pubsub_subscriber/src/twitch/mod.rs similarity index 75% rename from crates/service/obs_gateway/src/twitch/mod.rs rename to crates/service/twitch_pubsub_subscriber/src/twitch/mod.rs index 0c6e6e82b5..3aaaab19b6 100644 --- a/crates/service/obs_gateway/src/twitch/mod.rs +++ b/crates/service/twitch_pubsub_subscriber/src/twitch/mod.rs @@ -1,5 +1,4 @@ pub mod constants; pub mod oauth; pub mod pubsub; -pub mod twitch_user_id; pub mod websocket_client; \ No newline at end of file diff --git a/crates/service/obs_gateway/src/twitch/oauth/mod.rs b/crates/service/twitch_pubsub_subscriber/src/twitch/oauth/mod.rs similarity index 100% rename from crates/service/obs_gateway/src/twitch/oauth/mod.rs rename to crates/service/twitch_pubsub_subscriber/src/twitch/oauth/mod.rs diff --git a/crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs b/crates/service/twitch_pubsub_subscriber/src/twitch/oauth/oauth_token_refresher.rs similarity index 100% rename from crates/service/obs_gateway/src/twitch/oauth/oauth_token_refresher.rs rename to crates/service/twitch_pubsub_subscriber/src/twitch/oauth/oauth_token_refresher.rs diff --git a/crates/service/obs_gateway/src/twitch/pubsub/build_pubsub_topics_for_user.rs b/crates/service/twitch_pubsub_subscriber/src/twitch/pubsub/build_pubsub_topics_for_user.rs similarity index 100% rename from crates/service/obs_gateway/src/twitch/pubsub/build_pubsub_topics_for_user.rs rename to crates/service/twitch_pubsub_subscriber/src/twitch/pubsub/build_pubsub_topics_for_user.rs diff --git a/crates/service/obs_gateway/src/twitch/pubsub/mod.rs b/crates/service/twitch_pubsub_subscriber/src/twitch/pubsub/mod.rs similarity index 100% rename from crates/service/obs_gateway/src/twitch/pubsub/mod.rs rename to crates/service/twitch_pubsub_subscriber/src/twitch/pubsub/mod.rs diff --git a/crates/service/obs_gateway/src/twitch/websocket_client.rs b/crates/service/twitch_pubsub_subscriber/src/twitch/websocket_client.rs similarity index 100% rename from crates/service/obs_gateway/src/twitch/websocket_client.rs rename to crates/service/twitch_pubsub_subscriber/src/twitch/websocket_client.rs diff --git a/crates/service/obs_gateway/src/util/mod.rs b/crates/service/twitch_pubsub_subscriber/src/util/mod.rs similarity index 100% rename from crates/service/obs_gateway/src/util/mod.rs rename to crates/service/twitch_pubsub_subscriber/src/util/mod.rs diff --git a/crates/service/obs_gateway/src/util/random_nonce.rs b/crates/service/twitch_pubsub_subscriber/src/util/random_nonce.rs similarity index 100% rename from crates/service/obs_gateway/src/util/random_nonce.rs rename to crates/service/twitch_pubsub_subscriber/src/util/random_nonce.rs diff --git a/crates/service/obs_gateway/Cargo.toml b/crates/service/websocket_gateway/Cargo.toml similarity index 98% rename from crates/service/obs_gateway/Cargo.toml rename to crates/service/websocket_gateway/Cargo.toml index a050251f02..9b907e0e1b 100644 --- a/crates/service/obs_gateway/Cargo.toml +++ b/crates/service/websocket_gateway/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "obs-gateway-server" +name = "websocket-gateway" edition = "2018" version = "0.0.1" authors = [ @@ -9,7 +9,7 @@ authors = [ publish = false [[bin]] -name = "obs-gateway" +name = "websocket-gateway" path = "src/main.rs" [[bin]] diff --git a/crates/service/obs_gateway/src/endpoints_ws/mod.rs b/crates/service/websocket_gateway/src/endpoints_ws/mod.rs similarity index 100% rename from crates/service/obs_gateway/src/endpoints_ws/mod.rs rename to crates/service/websocket_gateway/src/endpoints_ws/mod.rs diff --git a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/websocket_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs similarity index 96% rename from crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs rename to crates/service/websocket_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index bddc37ec98..1187208e6b 100644 --- a/crates/service/obs_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/websocket_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -6,12 +6,7 @@ use actix_web_actors::ws; use container_common::anyhow_result::AnyhowResult; use container_common::token::random_crockford_token::random_crockford_token; use crate::endpoints_ws::obs_gateway_websocket_handler::ResponseType::TtsEvent; -use crate::redis::constants::OBS_ACTIVE_TTL_SECONDS; -use crate::redis::obs_active_payload::ObsActivePayload; use crate::server_state::ObsGatewayServerState; -use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; -use crate::twitch::twitch_user_id::TwitchUserId; -use crate::twitch::websocket_client::TwitchWebsocketClient; use database_queries::queries::twitch::twitch_oauth::find::{TwitchOauthTokenFinder, TwitchOauthTokenRecord}; use futures_timer::Delay; use futures_util::FutureExt; @@ -23,7 +18,9 @@ use log::warn; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2::PooledConnection; use r2d2_redis::redis::Commands; +use redis_common::payloads::obs_active_payload::ObsActivePayload; use redis_common::redis_keys::RedisKeys; +use redis_common::shared_constants::OBS_ACTIVE_TTL_SECONDS; use std::sync::Arc; use std::thread::sleep; use std::time::Duration; @@ -32,6 +29,7 @@ use twitch_api2::pubsub::Topic; use twitch_api2::pubsub::TopicData::ChannelPointsChannelV1; use twitch_api2::pubsub::channel_bits::ChannelBitsEventsV2Reply::BitsEvent; use twitch_api2::pubsub; +use twitch_common::twitch_user_id::TwitchUserId; // TODO: Redis calls are synchronous (but fast), but is there any way to make them async? diff --git a/crates/service/obs_gateway/src/main.rs b/crates/service/websocket_gateway/src/main.rs similarity index 98% rename from crates/service/obs_gateway/src/main.rs rename to crates/service/websocket_gateway/src/main.rs index 06fb86b935..3a2b3e4582 100644 --- a/crates/service/obs_gateway/src/main.rs +++ b/crates/service/websocket_gateway/src/main.rs @@ -14,10 +14,6 @@ pub mod endpoints_ws; pub mod server_state; -pub mod redis; -pub mod threads; -pub mod twitch; -pub mod util; use actix_cors::Cors; use actix_http::http; diff --git a/crates/service/obs_gateway/src/server_state.rs b/crates/service/websocket_gateway/src/server_state.rs similarity index 100% rename from crates/service/obs_gateway/src/server_state.rs rename to crates/service/websocket_gateway/src/server_state.rs From 480a2dc2163934831e201fc735e4e5bd5d998472 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 21 Feb 2022 00:00:01 -0500 Subject: [PATCH 0969/1042] thread refactor --- crates/service/twitch_pubsub_subscriber/src/main.rs | 2 +- .../listen_for_active_obs_sessions_thread.rs | 2 +- .../src/threads/listen_for_active_obs_sessions/mod.rs | 1 + crates/service/twitch_pubsub_subscriber/src/threads/mod.rs | 5 ++--- .../twitch_pubsub_subscriber/src/threads/thread_state/mod.rs | 1 - .../src/threads/twitch_pubsub_user_subscriber/mod.rs | 2 ++ .../subscriber_preferences_cache.rs} | 0 .../twitch_pubsub_user_subscriber_thread.rs | 3 ++- 8 files changed, 9 insertions(+), 7 deletions(-) rename crates/service/twitch_pubsub_subscriber/src/threads/{ => listen_for_active_obs_sessions}/listen_for_active_obs_sessions_thread.rs (96%) create mode 100644 crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions/mod.rs delete mode 100644 crates/service/twitch_pubsub_subscriber/src/threads/thread_state/mod.rs create mode 100644 crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/mod.rs rename crates/service/twitch_pubsub_subscriber/src/threads/{thread_state/twitch_pubsub_subscriber_state.rs => twitch_pubsub_user_subscriber/subscriber_preferences_cache.rs} (100%) rename crates/service/twitch_pubsub_subscriber/src/threads/{ => twitch_pubsub_user_subscriber}/twitch_pubsub_user_subscriber_thread.rs (99%) diff --git a/crates/service/twitch_pubsub_subscriber/src/main.rs b/crates/service/twitch_pubsub_subscriber/src/main.rs index 3362a2c49b..eca1f18f65 100644 --- a/crates/service/twitch_pubsub_subscriber/src/main.rs +++ b/crates/service/twitch_pubsub_subscriber/src/main.rs @@ -18,7 +18,7 @@ pub mod util; use config::shared_constants::{DEFAULT_RUST_LOG, DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING, DEFAULT_MYSQL_CONNECTION_STRING}; use container_common::anyhow_result::AnyhowResult; -use crate::threads::listen_for_active_obs_sessions_thread::ListenForActiveObsSessionThread; +use crate::threads::listen_for_active_obs_sessions::listen_for_active_obs_sessions_thread::ListenForActiveObsSessionThread; use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use crate::twitch::websocket_client::TwitchWebsocketClient; use futures::executor::{ThreadPool, ThreadPoolBuilder}; diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions_thread.rs b/crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions/listen_for_active_obs_sessions_thread.rs similarity index 96% rename from crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions_thread.rs rename to crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions/listen_for_active_obs_sessions_thread.rs index e3b33de4c2..ea25c146c2 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions_thread.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions/listen_for_active_obs_sessions_thread.rs @@ -1,5 +1,5 @@ use container_common::thread::thread_id::ThreadId; -use crate::threads::twitch_pubsub_user_subscriber_thread::TwitchPubsubUserSubscriberThread; +use crate::threads::twitch_pubsub_user_subscriber::twitch_pubsub_user_subscriber_thread::TwitchPubsubUserSubscriberThread; use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use log::error; use log::info; diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions/mod.rs b/crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions/mod.rs new file mode 100644 index 0000000000..dfaf27e527 --- /dev/null +++ b/crates/service/twitch_pubsub_subscriber/src/threads/listen_for_active_obs_sessions/mod.rs @@ -0,0 +1 @@ +pub mod listen_for_active_obs_sessions_thread; \ No newline at end of file diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/mod.rs b/crates/service/twitch_pubsub_subscriber/src/threads/mod.rs index 43ebea277e..4a9aa6ac56 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/mod.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/mod.rs @@ -1,3 +1,2 @@ -pub mod listen_for_active_obs_sessions_thread; -pub mod thread_state; -pub mod twitch_pubsub_user_subscriber_thread; \ No newline at end of file +pub mod listen_for_active_obs_sessions; +pub mod twitch_pubsub_user_subscriber; \ No newline at end of file diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/thread_state/mod.rs b/crates/service/twitch_pubsub_subscriber/src/threads/thread_state/mod.rs deleted file mode 100644 index 173ba58045..0000000000 --- a/crates/service/twitch_pubsub_subscriber/src/threads/thread_state/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod twitch_pubsub_subscriber_state; \ No newline at end of file diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/mod.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/mod.rs new file mode 100644 index 0000000000..abe7108e99 --- /dev/null +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/mod.rs @@ -0,0 +1,2 @@ +pub mod subscriber_preferences_cache; +pub mod twitch_pubsub_user_subscriber_thread; \ No newline at end of file diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/thread_state/twitch_pubsub_subscriber_state.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/subscriber_preferences_cache.rs similarity index 100% rename from crates/service/twitch_pubsub_subscriber/src/threads/thread_state/twitch_pubsub_subscriber_state.rs rename to crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/subscriber_preferences_cache.rs diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber_thread.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs similarity index 99% rename from crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber_thread.rs rename to crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs index 7e7b086736..d9f3f8f4e4 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs @@ -1,7 +1,8 @@ use anyhow::{anyhow, Error}; use container_common::anyhow_result::AnyhowResult; use container_common::thread::thread_id::ThreadId; -use crate::threads::thread_state::twitch_pubsub_subscriber_state::{TwitchEventRuleLight, TwitchPubsubCachedState}; +use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::TwitchEventRuleLight; +use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::TwitchPubsubCachedState; use crate::twitch::constants::TWITCH_PING_CADENCE; use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; From ee387ce3147ad6eb35b7b0673641ed802def5229 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 21 Feb 2022 04:00:48 -0500 Subject: [PATCH 0970/1042] wip handlers (broken) --- .../event_handlers/bits_event_handler.rs | 65 ++++++++++++++ .../channel_points_event_handler.rs | 83 ++++++++++++++++++ .../event_handlers/mod.rs | 2 + .../twitch_pubsub_user_subscriber/mod.rs | 4 +- .../tts_writer.rs | 59 +++++++++++++ .../twitch_pubsub_user_subscriber_thread.rs | 87 +++++++------------ 6 files changed, 244 insertions(+), 56 deletions(-) create mode 100644 crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs create mode 100644 crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs create mode 100644 crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/mod.rs create mode 100644 crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs new file mode 100644 index 0000000000..2cdb6ba37a --- /dev/null +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs @@ -0,0 +1,65 @@ +use container_common::anyhow_result::AnyhowResult; +use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::TwitchPubsubCachedState; +use crate::threads::twitch_pubsub_user_subscriber::tts_writer::TtsWriter; +use database_queries::queries::twitch::twitch_pubsub::insert_bits::TwitchPubsubBitsInsertBuilder; +use database_queries::queries::twitch::twitch_pubsub::insert_channel_points::TwitchPubsubChannelPointsInsertBuilder; +use r2d2_redis::r2d2; +use sqlx::MySql; +use std::sync::Arc; +use std::sync::RwLock; +use twitch_api2::pubsub::TopicData; +use twitch_api2::pubsub::channel_bits::{ChannelBitsEventsV2, ChannelBitsEventsV2Reply, BitsEventData}; + +pub struct BitsEventHandler { + twitch_subscriber_state: Arc>, + mysql_pool: Arc>, + tts_writer: Arc, +} + +impl BitsEventHandler { + pub fn new( + twitch_subscriber_state: Arc>, + mysql_pool: Arc>, + tts_writer: Arc, + ) -> Self { + Self { + twitch_subscriber_state, + mysql_pool, + tts_writer + } + } + + pub async fn handle(&mut self, topic: ChannelBitsEventsV2, reply: Box) -> AnyhowResult<()> { + match *reply { + ChannelBitsEventsV2Reply::BitsEvent { data, message_id, version, is_anonymous } => { + self.handle_bits_event(&data).await?; + } + _ => {} + } + Ok(()) + } + + // NB: &mut is for Redis pool in downstream write_tts. + async fn handle_bits_event(&mut self, data: &BitsEventData) -> AnyhowResult<()> { + self.tts_writer.write_tts(&data.chat_message).await?; + // Report event for analytics + self.report_event_for_analytics(&data).await?; + Ok(()) + } + + async fn report_event_for_analytics(&mut self, data: &BitsEventData) -> AnyhowResult<()> { + let user_id = data.user_id.to_string(); + let user_name = data.user_name.to_string(); + let mut event_builder = TwitchPubsubBitsInsertBuilder::new(); + let mut event_builder = event_builder.set_sender_twitch_user_id(&user_id) + .set_sender_twitch_username(&user_name) + .set_destination_channel_id(&data.channel_id.to_string()) + .set_destination_channel_name(&data.channel_name.to_string()) + .set_bits_used(data.bits_used as u64) + .set_total_bits_used(data.total_bits_used as u64) + .set_is_anonymous(data.is_anonymous) + .set_chat_message(&data.chat_message); + event_builder.insert(&self.mysql_pool).await?; + Ok(()) + } +} \ No newline at end of file diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs new file mode 100644 index 0000000000..c81c16500d --- /dev/null +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs @@ -0,0 +1,83 @@ +use container_common::anyhow_result::AnyhowResult; +use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::TwitchPubsubCachedState; +use crate::threads::twitch_pubsub_user_subscriber::tts_writer::TtsWriter; +use database_queries::queries::twitch::twitch_pubsub::insert_channel_points::TwitchPubsubChannelPointsInsertBuilder; +use r2d2_redis::r2d2; +use sqlx::MySql; +use std::sync::Arc; +use std::sync::RwLock; +use twitch_api2::pubsub::TopicData; +use twitch_api2::pubsub::channel_points::{ChannelPointsChannelV1, ChannelPointsChannelV1Reply, Redemption}; + +pub struct ChannelPointsEventHandler { + twitch_subscriber_state: Arc>, + mysql_pool: Arc>, + tts_writer: Arc, +} + +impl ChannelPointsEventHandler { + pub fn new( + twitch_subscriber_state: Arc>, + mysql_pool: Arc>, + tts_writer: Arc, + ) -> Self { + Self { + twitch_subscriber_state, + mysql_pool, + tts_writer + } + } + + // NB: &mut is for Redis pool in downstream write_tts. + pub async fn handle(&mut self, topic: ChannelPointsChannelV1, reply: Box) -> AnyhowResult<()> { + match *reply { + // Unimplemented + ChannelPointsChannelV1Reply::CustomRewardUpdated { .. } => {} + ChannelPointsChannelV1Reply::RedemptionStatusUpdate { .. } => {} + ChannelPointsChannelV1Reply::UpdateRedemptionStatusesFinished { .. } => {} + ChannelPointsChannelV1Reply::UpdateRedemptionStatusProgress { .. } => {} + // Implemented + ChannelPointsChannelV1Reply::RewardRedeemed { timestamp, redemption } => { + self.handle_reward_redeemed(redemption).await; + } + _ => {}, + } + Ok(()) + } + + // NB: &mut is for Redis pool in downstream write_tts. + async fn handle_reward_redeemed(&mut self, redemption: Redemption) -> AnyhowResult<()> { + if let Some(user_text) = redemption.user_input.as_deref() { + self.tts_writer.write_tts(user_text).await?; + } + // Report event for analytics + self.report_event_for_analytics(&redemption).await?; + Ok(()) + } + + async fn report_event_for_analytics(&self, redemption: &Redemption) -> AnyhowResult<()> { + let mut event_builder = TwitchPubsubChannelPointsInsertBuilder::new(); + + let user_id = redemption.user.id.to_string(); + let user_name = redemption.user.login.to_string(); + + let mut event_builder = event_builder.set_sender_twitch_user_id(&user_id) + .set_sender_twitch_username(&user_name) + .set_destination_channel_id(&redemption.channel_id.to_string()) + // TODO: + .set_destination_channel_name("todo: not available") + .set_title(&redemption.reward.title) + .set_prompt(&redemption.reward.prompt) + .set_user_text_input(redemption.user_input.as_deref()) + .set_redemption_id(&redemption.id.to_string()) + .set_reward_id(&redemption.reward.id.to_string()) + .set_is_sub_only(redemption.reward.is_sub_only) + .set_reward_cost(redemption.reward.cost as u64); + // TODO: + // .set_max_per_stream(redemption.reward.max_per_stream as u64) + // .set_max_per_user_per_stream(redemption.reward.max_per_user_per_stream as u64); + event_builder.insert(&self.mysql_pool).await?; + + Ok(()) + } +} \ No newline at end of file diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/mod.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/mod.rs new file mode 100644 index 0000000000..3f37ba9c3a --- /dev/null +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/mod.rs @@ -0,0 +1,2 @@ +pub mod bits_event_handler; +pub mod channel_points_event_handler; diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/mod.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/mod.rs index abe7108e99..d30da1b0c8 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/mod.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/mod.rs @@ -1,2 +1,4 @@ +pub mod event_handlers; pub mod subscriber_preferences_cache; -pub mod twitch_pubsub_user_subscriber_thread; \ No newline at end of file +pub mod tts_writer; +pub mod twitch_pubsub_user_subscriber_thread; diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs new file mode 100644 index 0000000000..aaaa302aa1 --- /dev/null +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs @@ -0,0 +1,59 @@ +use container_common::anyhow_result::AnyhowResult; +use database_queries::queries::tts::tts_inference_jobs::insert_tts_inference_job::TtsInferenceJobInsertBuilder; +use database_queries::tokens::Tokens; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2; +use r2d2_redis::redis::Commands; +use redis_common::redis_keys::RedisKeys; +use redis_common::shared_constants::STREAMER_TTS_JOB_QUEUE_TTL_SECONDS; +use sqlx::MySql; +use std::sync::Arc; +use twitch_common::cheers::remove_cheers; +use twitch_common::twitch_user_id::TwitchUserId; + +pub struct TtsWriter { + mysql_pool: Arc>, + redis_pool: Arc>, + twitch_user_id: TwitchUserId, +} + +impl TtsWriter { + + pub fn new( + mysql_pool: Arc>, + redis_pool: Arc>, + twitch_user_id: TwitchUserId, + ) -> Self { + Self { + mysql_pool, + redis_pool, + twitch_user_id, + } + } + + // NB: &mut is for Redis pool. + pub async fn write_tts(&mut self, message_text: &str) -> AnyhowResult<()> { + let sanitized_text = remove_cheers(message_text); + let job_token = Tokens::new_tts_inference_job()?; + //let model_token = "TM:7wbtjphx8h8v"; // "Mario *" voice (prod) + let model_token = "TM:40m3aqtt41y0"; // "Wakko" voice (dev) + + let mut builder = TtsInferenceJobInsertBuilder::new_for_internal_tts() + .set_is_for_twitch(true) + .set_priority_level(1) // TODO: This shouldn't be for everyone. + .set_job_token(&job_token) + .set_model_token(model_token) + .set_raw_inference_text(&sanitized_text); + + let _r = builder.insert(&self.mysql_pool).await?; + + // TODO: Report job token to frontend + let mut redis = self.redis_pool.get()?; + let redis_key = RedisKeys::twitch_tts_job_queue(&self.twitch_user_id.get_str()); + + let _size : Option = redis.rpush(&redis_key, job_token)?; + let _size : Option = redis.expire(&redis_key, STREAMER_TTS_JOB_QUEUE_TTL_SECONDS)?; + + Ok(()) + } +} \ No newline at end of file diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs index d9f3f8f4e4..521f5d2d16 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs @@ -1,8 +1,6 @@ use anyhow::{anyhow, Error}; use container_common::anyhow_result::AnyhowResult; use container_common::thread::thread_id::ThreadId; -use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::TwitchEventRuleLight; -use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::TwitchPubsubCachedState; use crate::twitch::constants::TWITCH_PING_CADENCE; use crate::twitch::oauth::oauth_token_refresher::OauthTokenRefresher; use crate::twitch::pubsub::build_pubsub_topics_for_user::build_pubsub_topics_for_user; @@ -42,6 +40,11 @@ use twitch_api2::pubsub::channel_points::ChannelPointsChannelV1Reply; use twitch_api2::pubsub::{Response, TwitchResponse, TopicData}; use twitch_common::cheers::remove_cheers; use twitch_common::twitch_user_id::TwitchUserId; +use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::TwitchPubsubCachedState; +use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::TwitchEventRuleLight; +use crate::threads::twitch_pubsub_user_subscriber::event_handlers::channel_points_event_handler::ChannelPointsEventHandler; +use crate::threads::twitch_pubsub_user_subscriber::tts_writer::TtsWriter; +use crate::threads::twitch_pubsub_user_subscriber::event_handlers::bits_event_handler::BitsEventHandler; // TODO: Publish events back to OBS thread // TODO: (cleanup) make the logic clearer to follow. @@ -120,6 +123,24 @@ impl TwitchPubsubUserSubscriberThread { } }; + let tts_writer = Arc::new(TtsWriter::new( + self.mysql_pool.clone(), + self.redis_pool.clone(), + self.twitch_user_id.clone(), + )); + + let bits_event_handler = BitsEventHandler::new( + self.twitch_subscriber_state.clone(), + self.mysql_pool.clone(), + tts_writer.clone(), + ); + + let channel_points_event_handler = ChannelPointsEventHandler::new( + self.twitch_subscriber_state.clone(), + self.mysql_pool.clone(), + tts_writer.clone(), + ); + // NB: All of the timers (thus far) can wait to run. // We set their first run time to now so we don't have to deal with Option<>. let now = Instant::now(); @@ -144,6 +165,8 @@ impl TwitchPubsubUserSubscriberThread { twitch_last_pinged_at: now, twitch_subscriber_state: self.twitch_subscriber_state.clone(), website_settings_last_refreshed_at: long_ago, + channel_points_event_handler, + bits_event_handler, }; // NB: The following call will run its main loop until/unless the Twitch client @@ -204,6 +227,11 @@ struct TwitchPubsubUserSubscriberThreadStageTwo { twitch_subscriber_state: Arc>, + // ========== Handlers ========== + + bits_event_handler: BitsEventHandler, + channel_points_event_handler: ChannelPointsEventHandler, + // ========== Stage Two Thread State ========== twitch_websocket_client: TwitchWebsocketClient, @@ -351,61 +379,10 @@ impl TwitchPubsubUserSubscriberThreadStageTwo { TopicData::UserModerationNotifications { .. } => {} // Implemented TopicData::ChannelBitsEventsV2 { topic, reply } => { - let mut event_builder = TwitchPubsubBitsInsertBuilder::new(); - match *reply { - ChannelBitsEventsV2Reply::BitsEvent { data, message_id, version, is_anonymous } => { - let user_id = data.user_id.to_string(); - let user_name = data.user_name.to_string(); - let mut event_builder = event_builder.set_sender_twitch_user_id(&user_id) - .set_sender_twitch_username(&user_name) - .set_destination_channel_id(&data.channel_id.to_string()) - .set_destination_channel_name(&data.channel_name.to_string()) - .set_bits_used(data.bits_used as u64) - .set_total_bits_used(data.total_bits_used as u64) - .set_is_anonymous(is_anonymous) - .set_chat_message(&data.chat_message); - event_builder.insert(&self.mysql_pool).await?; - - self.write_tts_inference_event(&data.chat_message).await?; - } - _ => {} - } + self.bits_event_handler.handle(topic, reply).await?; } TopicData::ChannelPointsChannelV1 { topic, reply } => { - let mut event_builder = TwitchPubsubChannelPointsInsertBuilder::new(); - match *reply { - // Unimplemented - ChannelPointsChannelV1Reply::CustomRewardUpdated { .. } => {} - ChannelPointsChannelV1Reply::RedemptionStatusUpdate { .. } => {} - ChannelPointsChannelV1Reply::UpdateRedemptionStatusesFinished { .. } => {} - ChannelPointsChannelV1Reply::UpdateRedemptionStatusProgress { .. } => {} - // Implemented - ChannelPointsChannelV1Reply::RewardRedeemed { timestamp, redemption } => { - let user_id = redemption.user.id.to_string(); - let user_name = redemption.user.login.to_string(); - let mut event_builder = event_builder.set_sender_twitch_user_id(&user_id) - .set_sender_twitch_username(&user_name) - .set_destination_channel_id(&redemption.channel_id.to_string()) - // TODO: - .set_destination_channel_name("todo: not available") - .set_title(&redemption.reward.title) - .set_prompt(&redemption.reward.prompt) - .set_user_text_input(redemption.user_input.as_deref()) - .set_redemption_id(&redemption.id.to_string()) - .set_reward_id(&redemption.reward.id.to_string()) - .set_is_sub_only(redemption.reward.is_sub_only) - .set_reward_cost(redemption.reward.cost as u64); - // TODO: - // .set_max_per_stream(redemption.reward.max_per_stream as u64) - // .set_max_per_user_per_stream(redemption.reward.max_per_user_per_stream as u64); - event_builder.insert(&self.mysql_pool).await?; - - if let Some(user_text) = redemption.user_input.as_deref() { - self.write_tts_inference_event(user_text).await?; - } - } - _ => {}, - } + self.channel_points_event_handler.handle(topic, reply).await?; } } From 4b0cd9c197dc1952ee681032fa718be2749fc5c5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 21 Feb 2022 15:27:40 -0500 Subject: [PATCH 0971/1042] mut fixup --- .../event_handlers/channel_points_event_handler.rs | 2 +- .../src/threads/twitch_pubsub_user_subscriber/tts_writer.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs index c81c16500d..b5b2ef02e5 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs @@ -38,7 +38,7 @@ impl ChannelPointsEventHandler { ChannelPointsChannelV1Reply::UpdateRedemptionStatusProgress { .. } => {} // Implemented ChannelPointsChannelV1Reply::RewardRedeemed { timestamp, redemption } => { - self.handle_reward_redeemed(redemption).await; + self.handle_reward_redeemed(redemption).await?; } _ => {}, } diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs index aaaa302aa1..42b0ddf03d 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs @@ -32,7 +32,7 @@ impl TtsWriter { } // NB: &mut is for Redis pool. - pub async fn write_tts(&mut self, message_text: &str) -> AnyhowResult<()> { + pub async fn write_tts(&self, message_text: &str) -> AnyhowResult<()> { let sanitized_text = remove_cheers(message_text); let job_token = Tokens::new_tts_inference_job()?; //let model_token = "TM:7wbtjphx8h8v"; // "Mario *" voice (prod) From 38deee044fe94ed13cb58afdb1d3eb0e91857372 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 21 Feb 2022 15:30:26 -0500 Subject: [PATCH 0972/1042] mut cleanup --- .../event_handlers/bits_event_handler.rs | 7 +++---- .../event_handlers/channel_points_event_handler.rs | 5 ++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs index 2cdb6ba37a..1bc7c7212f 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs @@ -29,7 +29,7 @@ impl BitsEventHandler { } } - pub async fn handle(&mut self, topic: ChannelBitsEventsV2, reply: Box) -> AnyhowResult<()> { + pub async fn handle(&self, topic: ChannelBitsEventsV2, reply: Box) -> AnyhowResult<()> { match *reply { ChannelBitsEventsV2Reply::BitsEvent { data, message_id, version, is_anonymous } => { self.handle_bits_event(&data).await?; @@ -39,15 +39,14 @@ impl BitsEventHandler { Ok(()) } - // NB: &mut is for Redis pool in downstream write_tts. - async fn handle_bits_event(&mut self, data: &BitsEventData) -> AnyhowResult<()> { + async fn handle_bits_event(&self, data: &BitsEventData) -> AnyhowResult<()> { self.tts_writer.write_tts(&data.chat_message).await?; // Report event for analytics self.report_event_for_analytics(&data).await?; Ok(()) } - async fn report_event_for_analytics(&mut self, data: &BitsEventData) -> AnyhowResult<()> { + async fn report_event_for_analytics(&self, data: &BitsEventData) -> AnyhowResult<()> { let user_id = data.user_id.to_string(); let user_name = data.user_name.to_string(); let mut event_builder = TwitchPubsubBitsInsertBuilder::new(); diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs index b5b2ef02e5..9281493a50 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs @@ -29,7 +29,7 @@ impl ChannelPointsEventHandler { } // NB: &mut is for Redis pool in downstream write_tts. - pub async fn handle(&mut self, topic: ChannelPointsChannelV1, reply: Box) -> AnyhowResult<()> { + pub async fn handle(&self, topic: ChannelPointsChannelV1, reply: Box) -> AnyhowResult<()> { match *reply { // Unimplemented ChannelPointsChannelV1Reply::CustomRewardUpdated { .. } => {} @@ -45,8 +45,7 @@ impl ChannelPointsEventHandler { Ok(()) } - // NB: &mut is for Redis pool in downstream write_tts. - async fn handle_reward_redeemed(&mut self, redemption: Redemption) -> AnyhowResult<()> { + async fn handle_reward_redeemed(&self, redemption: Redemption) -> AnyhowResult<()> { if let Some(user_text) = redemption.user_input.as_deref() { self.tts_writer.write_tts(user_text).await?; } From 937f9d86a4526f1d40dcea13252d9a4911879bd4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 21 Feb 2022 17:40:56 -0500 Subject: [PATCH 0973/1042] rule matching is much more robust --- .../container_common/src/collections/mod.rs | 1 + .../src/collections/random_from_vec.rs | 31 +++++ crates/lib/container_common/src/lib.rs | 4 +- .../lib/container_common/src/numerics/mod.rs | 1 + .../src/numerics/signed_to_unsigned.rs | 127 ++++++++++++++++++ .../complex_models/event_match_predicate.rs | 4 +- .../event_handlers/bits_event_handler.rs | 71 +++++++++- .../subscriber_preferences_cache.rs | 1 + .../tts_writer.rs | 9 +- 9 files changed, 237 insertions(+), 12 deletions(-) create mode 100644 crates/lib/container_common/src/collections/mod.rs create mode 100644 crates/lib/container_common/src/collections/random_from_vec.rs create mode 100644 crates/lib/container_common/src/numerics/mod.rs create mode 100644 crates/lib/container_common/src/numerics/signed_to_unsigned.rs diff --git a/crates/lib/container_common/src/collections/mod.rs b/crates/lib/container_common/src/collections/mod.rs new file mode 100644 index 0000000000..dc0324cd61 --- /dev/null +++ b/crates/lib/container_common/src/collections/mod.rs @@ -0,0 +1 @@ +pub mod random_from_vec; \ No newline at end of file diff --git a/crates/lib/container_common/src/collections/random_from_vec.rs b/crates/lib/container_common/src/collections/random_from_vec.rs new file mode 100644 index 0000000000..652da76e83 --- /dev/null +++ b/crates/lib/container_common/src/collections/random_from_vec.rs @@ -0,0 +1,31 @@ +use rand::seq::SliceRandom; + +/// Return a random item from a vec. +pub fn random_from_vec(vec: &Vec) -> Option<&T> { + vec.choose(&mut rand::thread_rng()) +} + +#[cfg(test)] +mod tests { + use crate::collections::random_from_vec::random_from_vec; + + #[test] + fn pulls_item_from_vec() { + let items = vec!["one", "two", "three"]; + let selected = random_from_vec(&items).unwrap(); + assert!(items.contains(selected)); + + let items = vec!["one", "two", "three"].into_iter() + .map(|s| s.to_string()) + .collect(); + let selected = random_from_vec(&items).unwrap(); + assert!(items.contains(selected)); + } + + #[test] + fn behavior_on_empty() { + let items : Vec = vec![]; + let selected = random_from_vec(&items); + assert!(selected.is_none()); + } +} \ No newline at end of file diff --git a/crates/lib/container_common/src/lib.rs b/crates/lib/container_common/src/lib.rs index ed39b1dd25..485bc177c4 100644 --- a/crates/lib/container_common/src/lib.rs +++ b/crates/lib/container_common/src/lib.rs @@ -1,3 +1,5 @@ pub mod anyhow_result; +pub mod collections; +pub mod numerics; pub mod thread; -pub mod token; +pub mod token; \ No newline at end of file diff --git a/crates/lib/container_common/src/numerics/mod.rs b/crates/lib/container_common/src/numerics/mod.rs new file mode 100644 index 0000000000..9a87330c06 --- /dev/null +++ b/crates/lib/container_common/src/numerics/mod.rs @@ -0,0 +1 @@ +pub mod signed_to_unsigned; \ No newline at end of file diff --git a/crates/lib/container_common/src/numerics/signed_to_unsigned.rs b/crates/lib/container_common/src/numerics/signed_to_unsigned.rs new file mode 100644 index 0000000000..205bb62b57 --- /dev/null +++ b/crates/lib/container_common/src/numerics/signed_to_unsigned.rs @@ -0,0 +1,127 @@ + +// TODO: Use macros to define these (or use a better numeric crate) + +/// Negative numbers coalesce to 0 +pub fn i8_to_unsigned_zeroing_negatives(num: i8) -> u8 { + if num.is_negative() { + 0 + } else { + num.unsigned_abs() + } +} + +/// Negative numbers coalesce to 0 +pub fn i16_to_unsigned_zeroing_negatives(num: i16) -> u16 { + if num.is_negative() { + 0 + } else { + num.unsigned_abs() + } +} + +/// Negative numbers coalesce to 0 +pub fn i32_to_unsigned_zeroing_negatives(num: i32) -> u32 { + if num.is_negative() { + 0 + } else { + num.unsigned_abs() + } +} + +/// Negative numbers coalesce to 0 +pub fn i64_to_unsigned_zeroing_negatives(num: i64) -> u64 { + if num.is_negative() { + 0 + } else { + num.unsigned_abs() + } +} + +/// Negative numbers coalesce to 0 +pub fn i128_to_unsigned_zeroing_negatives(num: i128) -> u128 { + if num.is_negative() { + 0 + } else { + num.unsigned_abs() + } +} + +/// Negative numbers coalesce to 0 +pub fn isize_to_unsigned_zeroing_negatives(num: isize) -> usize { + if num.is_negative() { + 0 + } else { + num.unsigned_abs() + } +} + +#[cfg(test)] +mod tests { + use crate::numerics::signed_to_unsigned::i8_to_unsigned_zeroing_negatives; + use crate::numerics::signed_to_unsigned::i16_to_unsigned_zeroing_negatives; + use crate::numerics::signed_to_unsigned::i32_to_unsigned_zeroing_negatives; + use crate::numerics::signed_to_unsigned::i64_to_unsigned_zeroing_negatives; + use crate::numerics::signed_to_unsigned::i128_to_unsigned_zeroing_negatives; + use crate::numerics::signed_to_unsigned::isize_to_unsigned_zeroing_negatives; + + #[test] + fn i8_to_unsigned() { + assert_eq!(i8_to_unsigned_zeroing_negatives(-127), 0); + assert_eq!(i8_to_unsigned_zeroing_negatives(-1), 0); + assert_eq!(i8_to_unsigned_zeroing_negatives(0), 0); + assert_eq!(i8_to_unsigned_zeroing_negatives(1), 1); + assert_eq!(i8_to_unsigned_zeroing_negatives(100), 100); + assert_eq!(i8_to_unsigned_zeroing_negatives(127), 127); + } + + #[test] + fn i16_to_unsigned() { + assert_eq!(i16_to_unsigned_zeroing_negatives(-127), 0); + assert_eq!(i16_to_unsigned_zeroing_negatives(-1), 0); + assert_eq!(i16_to_unsigned_zeroing_negatives(0), 0); + assert_eq!(i16_to_unsigned_zeroing_negatives(1), 1); + assert_eq!(i16_to_unsigned_zeroing_negatives(100), 100); + assert_eq!(i16_to_unsigned_zeroing_negatives(127), 127); + } + + #[test] + fn i32_to_unsigned() { + assert_eq!(i32_to_unsigned_zeroing_negatives(-127), 0); + assert_eq!(i32_to_unsigned_zeroing_negatives(-1), 0); + assert_eq!(i32_to_unsigned_zeroing_negatives(0), 0); + assert_eq!(i32_to_unsigned_zeroing_negatives(1), 1); + assert_eq!(i32_to_unsigned_zeroing_negatives(100), 100); + assert_eq!(i32_to_unsigned_zeroing_negatives(127), 127); + } + + #[test] + fn i64_to_unsigned() { + assert_eq!(i64_to_unsigned_zeroing_negatives(-127), 0); + assert_eq!(i64_to_unsigned_zeroing_negatives(-1), 0); + assert_eq!(i64_to_unsigned_zeroing_negatives(0), 0); + assert_eq!(i64_to_unsigned_zeroing_negatives(1), 1); + assert_eq!(i64_to_unsigned_zeroing_negatives(100), 100); + assert_eq!(i64_to_unsigned_zeroing_negatives(127), 127); + } + + + #[test] + fn i128_to_unsigned() { + assert_eq!(i128_to_unsigned_zeroing_negatives(-127), 0); + assert_eq!(i128_to_unsigned_zeroing_negatives(-1), 0); + assert_eq!(i128_to_unsigned_zeroing_negatives(0), 0); + assert_eq!(i128_to_unsigned_zeroing_negatives(1), 1); + assert_eq!(i128_to_unsigned_zeroing_negatives(100), 100); + assert_eq!(i128_to_unsigned_zeroing_negatives(127), 127); + } + + #[test] + fn isize_to_unsigned() { + assert_eq!(isize_to_unsigned_zeroing_negatives(-127), 0); + assert_eq!(isize_to_unsigned_zeroing_negatives(-1), 0); + assert_eq!(isize_to_unsigned_zeroing_negatives(0), 0); + assert_eq!(isize_to_unsigned_zeroing_negatives(1), 1); + assert_eq!(isize_to_unsigned_zeroing_negatives(100), 100); + assert_eq!(isize_to_unsigned_zeroing_negatives(127), 127); + } +} \ No newline at end of file diff --git a/crates/lib/database_queries/src/complex_models/event_match_predicate.rs b/crates/lib/database_queries/src/complex_models/event_match_predicate.rs index fe1b48be6a..35b13e031f 100644 --- a/crates/lib/database_queries/src/complex_models/event_match_predicate.rs +++ b/crates/lib/database_queries/src/complex_models/event_match_predicate.rs @@ -15,11 +15,11 @@ pub enum EventMatchPredicate { /// threshold. eg "Cheer", minimum=150 bits. Case insensitive. BitsCheermoteNameSpendThreshold { cheermote_name_prefix: String, - minimum_bits_spent: u32, + minimum_bits_spent: u64, }, BitsSpendThreshold { - minimum_bits_spent: u32, + minimum_bits_spent: u64, }, ChannelPointsRewardNameExactMatch { diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs index 1bc7c7212f..f1cc831982 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs @@ -1,12 +1,20 @@ +use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::TwitchPubsubCachedState; +use container_common::collections::random_from_vec::random_from_vec; +use container_common::numerics::signed_to_unsigned::i64_to_unsigned_zeroing_negatives; +use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::{TwitchPubsubCachedState, TwitchEventRuleLight}; use crate::threads::twitch_pubsub_user_subscriber::tts_writer::TtsWriter; +use database_queries::column_types::twitch_event_category::TwitchEventCategory; +use database_queries::complex_models::event_match_predicate::EventMatchPredicate; +use database_queries::complex_models::event_responses::EventResponse; use database_queries::queries::twitch::twitch_pubsub::insert_bits::TwitchPubsubBitsInsertBuilder; use database_queries::queries::twitch::twitch_pubsub::insert_channel_points::TwitchPubsubChannelPointsInsertBuilder; +use log::info; use r2d2_redis::r2d2; +use rand::seq::SliceRandom; use sqlx::MySql; -use std::sync::Arc; use std::sync::RwLock; +use std::sync::{Arc, PoisonError, RwLockReadGuard}; use twitch_api2::pubsub::TopicData; use twitch_api2::pubsub::channel_bits::{ChannelBitsEventsV2, ChannelBitsEventsV2Reply, BitsEventData}; @@ -34,15 +42,17 @@ impl BitsEventHandler { ChannelBitsEventsV2Reply::BitsEvent { data, message_id, version, is_anonymous } => { self.handle_bits_event(&data).await?; } - _ => {} + _ => {} // NB: There are no other enum variants as of 2022-02-21 } Ok(()) } async fn handle_bits_event(&self, data: &BitsEventData) -> AnyhowResult<()> { - self.tts_writer.write_tts(&data.chat_message).await?; - // Report event for analytics - self.report_event_for_analytics(&data).await?; + let maybe_rule = self.find_matching_rule(data)?; + if let Some(rule) = maybe_rule { + self.handle_matched_rule(&rule, data).await?; + self.report_event_for_analytics(&data).await?; // Report event for analytics + } Ok(()) } @@ -61,4 +71,53 @@ impl BitsEventHandler { event_builder.insert(&self.mysql_pool).await?; Ok(()) } + + fn find_matching_rule(&self, data: &BitsEventData) -> AnyhowResult> { + return match self.twitch_subscriber_state.read() { + Err(e) => { Err(anyhow!("Lock error: {:?}", e)) }, + Ok(state) => { + let maybe_rule = state.event_rules.iter() + .filter(|rule| rule.event_category.eq(&TwitchEventCategory::Bits)) + .find(|rule| { + match rule.event_match_predicate { + EventMatchPredicate::NotSet => false, // Not set + EventMatchPredicate::ChannelPointsRewardNameExactMatch { .. } => false, // Wrong type + EventMatchPredicate::BitsCheermoteNameExactMatch { ref cheermote_name } => { + false // TODO + }, + EventMatchPredicate::BitsCheermoteNameSpendThreshold { ref cheermote_name_prefix, minimum_bits_spent } => { + false // TODO + }, + EventMatchPredicate::BitsSpendThreshold { minimum_bits_spent } => { + let spent = i64_to_unsigned_zeroing_negatives(data.bits_used); + spent >= minimum_bits_spent + }, + } + }) + .map(|rule| rule.clone()); + Ok(maybe_rule) + } + }; + } + + async fn handle_matched_rule(&self, rule: &TwitchEventRuleLight, data: &BitsEventData) -> AnyhowResult<()> { + match rule.event_response { + EventResponse::NotSet => { + info!("Empty event response."); + return Ok(()) + }, + EventResponse::TtsSingleVoice { ref tts_model_token } => { + self.tts_writer.write_tts_with_model(&data.chat_message, tts_model_token).await?; + } + EventResponse::TtsRandomVoice { ref tts_model_tokens } => { + let maybe_token = random_from_vec(tts_model_tokens); + if let Some(token) = maybe_token { + self.tts_writer.write_tts_with_model(&data.chat_message, token.as_str()) + .await?; + } + } + } + + Ok(()) + } } \ No newline at end of file diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/subscriber_preferences_cache.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/subscriber_preferences_cache.rs index a6407a2917..f19f244212 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/subscriber_preferences_cache.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/subscriber_preferences_cache.rs @@ -3,6 +3,7 @@ use database_queries::column_types::twitch_event_category::TwitchEventCategory; use database_queries::complex_models::event_match_predicate::EventMatchPredicate; use database_queries::complex_models::event_responses::EventResponse; +#[derive(Clone)] pub struct TwitchEventRuleLight { pub token: String, pub event_category: TwitchEventCategory, diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs index 42b0ddf03d..0deb0f45b1 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs @@ -31,12 +31,15 @@ impl TtsWriter { } } - // NB: &mut is for Redis pool. pub async fn write_tts(&self, message_text: &str) -> AnyhowResult<()> { - let sanitized_text = remove_cheers(message_text); - let job_token = Tokens::new_tts_inference_job()?; //let model_token = "TM:7wbtjphx8h8v"; // "Mario *" voice (prod) let model_token = "TM:40m3aqtt41y0"; // "Wakko" voice (dev) + self.write_tts_with_model(message_text, model_token).await + } + + pub async fn write_tts_with_model(&self, message_text: &str, model_token: &str) -> AnyhowResult<()> { + let sanitized_text = remove_cheers(message_text); + let job_token = Tokens::new_tts_inference_job()?; let mut builder = TtsInferenceJobInsertBuilder::new_for_internal_tts() .set_is_for_twitch(true) From c6fb2716fab79ad8b205f1a5c92329729aa8ee33 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 21 Feb 2022 22:10:07 -0500 Subject: [PATCH 0974/1042] handle channel points events --- .../event_handlers/bits_event_handler.rs | 1 + .../channel_points_event_handler.rs | 69 +++++++++++++++++-- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs index f1cc831982..f38452b928 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs @@ -100,6 +100,7 @@ impl BitsEventHandler { }; } + //noinspection DuplicatedCode async fn handle_matched_rule(&self, rule: &TwitchEventRuleLight, data: &BitsEventData) -> AnyhowResult<()> { match rule.event_response { EventResponse::NotSet => { diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs index 9281493a50..eea7d2795c 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs @@ -1,7 +1,13 @@ +use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; -use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::TwitchPubsubCachedState; +use container_common::collections::random_from_vec::random_from_vec; +use crate::threads::twitch_pubsub_user_subscriber::subscriber_preferences_cache::{TwitchPubsubCachedState, TwitchEventRuleLight}; use crate::threads::twitch_pubsub_user_subscriber::tts_writer::TtsWriter; +use database_queries::column_types::twitch_event_category::TwitchEventCategory; +use database_queries::complex_models::event_match_predicate::EventMatchPredicate; +use database_queries::complex_models::event_responses::EventResponse; use database_queries::queries::twitch::twitch_pubsub::insert_channel_points::TwitchPubsubChannelPointsInsertBuilder; +use log::info; use r2d2_redis::r2d2; use sqlx::MySql; use std::sync::Arc; @@ -38,19 +44,19 @@ impl ChannelPointsEventHandler { ChannelPointsChannelV1Reply::UpdateRedemptionStatusProgress { .. } => {} // Implemented ChannelPointsChannelV1Reply::RewardRedeemed { timestamp, redemption } => { - self.handle_reward_redeemed(redemption).await?; + self.handle_reward_redeemed_event(&redemption).await?; } _ => {}, } Ok(()) } - async fn handle_reward_redeemed(&self, redemption: Redemption) -> AnyhowResult<()> { - if let Some(user_text) = redemption.user_input.as_deref() { - self.tts_writer.write_tts(user_text).await?; + async fn handle_reward_redeemed_event(&self, redemption: &Redemption) -> AnyhowResult<()> { + let maybe_rule = self.find_matching_rule(redemption)?; + if let Some(rule) = maybe_rule { + self.handle_matched_rule(&rule, redemption).await?; + self.report_event_for_analytics(redemption).await?; // Report event for analytics } - // Report event for analytics - self.report_event_for_analytics(&redemption).await?; Ok(()) } @@ -79,4 +85,53 @@ impl ChannelPointsEventHandler { Ok(()) } + + fn find_matching_rule(&self, redemption: &Redemption) -> AnyhowResult> { + return match self.twitch_subscriber_state.read() { + Err(e) => { Err(anyhow!("Lock error: {:?}", e)) }, + Ok(state) => { + let maybe_rule = state.event_rules.iter() + .filter(|rule| rule.event_category.eq(&TwitchEventCategory::ChannelPoints)) + .find(|rule| { + match rule.event_match_predicate { + EventMatchPredicate::NotSet => false, // Not set + EventMatchPredicate::BitsCheermoteNameExactMatch { ..} => false, // Wrong type + EventMatchPredicate::BitsCheermoteNameSpendThreshold { .. } => false, // Wrong type + EventMatchPredicate::BitsSpendThreshold { .. } => false, // Wrong type + EventMatchPredicate::ChannelPointsRewardNameExactMatch { ref reward_name } => { + reward_name.eq_ignore_ascii_case(&redemption.reward.title) + } + } + }) + .map(|rule| rule.clone()); + Ok(maybe_rule) + } + }; + } + + //noinspection DuplicatedCode + async fn handle_matched_rule(&self, rule: &TwitchEventRuleLight, redemption: &Redemption) -> AnyhowResult<()> { + let message = match redemption.user_input { + None => return Ok(()), // Nothing to do. + Some(ref m) => m.as_str(), + }; + match rule.event_response { + EventResponse::NotSet => { + info!("Empty event response."); + return Ok(()) + }, + EventResponse::TtsSingleVoice { ref tts_model_token } => { + self.tts_writer.write_tts_with_model(message, tts_model_token).await?; + } + EventResponse::TtsRandomVoice { ref tts_model_tokens } => { + let maybe_token = random_from_vec(tts_model_tokens); + if let Some(token) = maybe_token { + self.tts_writer.write_tts_with_model(message, token.as_str()) + .await?; + } + } + } + + Ok(()) + } } \ No newline at end of file From 2c390ca20ad5f1021dbbc9d1c1643898f006124c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 22 Feb 2022 00:16:26 -0500 Subject: [PATCH 0975/1042] had to adjust "NotSet" enum for serialization --- .../src/complex_models/event_match_predicate.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/lib/database_queries/src/complex_models/event_match_predicate.rs b/crates/lib/database_queries/src/complex_models/event_match_predicate.rs index 35b13e031f..3501854989 100644 --- a/crates/lib/database_queries/src/complex_models/event_match_predicate.rs +++ b/crates/lib/database_queries/src/complex_models/event_match_predicate.rs @@ -3,7 +3,8 @@ #[serde(rename_all = "snake_case")] pub enum EventMatchPredicate { /// Default value - NotSet, + /// NB: This has an empty struct so it serializes as JSON and not a String! + NotSet {}, /// A predicate for an exactly matched cheermote name, including the bit value, /// eg "Cheer1". Case insensitive. @@ -31,6 +32,19 @@ pub enum EventMatchPredicate { mod tests { use crate::complex_models::event_match_predicate::EventMatchPredicate; + #[test] + fn not_set() { + let rust_value = EventMatchPredicate::NotSet {}; + + let json = "{\"not_set\":{}}"; + + let converted_to_json= serde_json::to_string(&rust_value).unwrap(); + assert_eq!(&converted_to_json, json); + + let converted_from_json : EventMatchPredicate = serde_json::from_str(json).unwrap(); + assert_eq!(&converted_from_json, &rust_value); + } + #[test] fn bits_cheermote_name_exact_match() { let rust_value = EventMatchPredicate::BitsCheermoteNameExactMatch { From b59d7af915e856d9e90b0040e7a7f482b075e1fd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 22 Feb 2022 00:26:14 -0500 Subject: [PATCH 0976/1042] EventResponse::NotSet also needs to be empty enum for proper json --- .../src/complex_models/event_match_predicate.rs | 7 ++++--- .../src/complex_models/event_responses.rs | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/lib/database_queries/src/complex_models/event_match_predicate.rs b/crates/lib/database_queries/src/complex_models/event_match_predicate.rs index 3501854989..d989b4ca11 100644 --- a/crates/lib/database_queries/src/complex_models/event_match_predicate.rs +++ b/crates/lib/database_queries/src/complex_models/event_match_predicate.rs @@ -14,11 +14,13 @@ pub enum EventMatchPredicate { /// A predicate for a cheermote name without value, but with a minimum bits /// threshold. eg "Cheer", minimum=150 bits. Case insensitive. - BitsCheermoteNameSpendThreshold { - cheermote_name_prefix: String, + /// Twitch also calls the leading part "prefix" + BitsCheermotePrefixSpendThreshold { + cheermote_prefix: String, minimum_bits_spent: u64, }, + /// Spend a minimum amount of bits. BitsSpendThreshold { minimum_bits_spent: u64, }, @@ -35,7 +37,6 @@ mod tests { #[test] fn not_set() { let rust_value = EventMatchPredicate::NotSet {}; - let json = "{\"not_set\":{}}"; let converted_to_json= serde_json::to_string(&rust_value).unwrap(); diff --git a/crates/lib/database_queries/src/complex_models/event_responses.rs b/crates/lib/database_queries/src/complex_models/event_responses.rs index 2e9b343b9e..a49054696f 100644 --- a/crates/lib/database_queries/src/complex_models/event_responses.rs +++ b/crates/lib/database_queries/src/complex_models/event_responses.rs @@ -4,7 +4,8 @@ use container_common::anyhow_result::AnyhowResult; #[serde(rename_all = "snake_case")] pub enum EventResponse { /// Default value - NotSet, + /// NB: This has an empty struct so it serializes as JSON and not a String! + NotSet {}, /// Respond with a single TTS voice. TtsSingleVoice { @@ -21,6 +22,18 @@ pub enum EventResponse { mod tests { use crate::complex_models::event_responses::EventResponse; + #[test] + fn not_set() { + let rust_value = EventResponse::NotSet {}; + let json = "{\"not_set\":{}}"; + + let converted_to_json= serde_json::to_string(&rust_value).unwrap(); + assert_eq!(&converted_to_json, json); + + let converted_from_json : EventResponse = serde_json::from_str(json).unwrap(); + assert_eq!(&converted_from_json, &rust_value); + } + #[test] fn tts_single_voice() { let rust_value = EventResponse::TtsSingleVoice { From 40a83e1bad4bf3915b153e9dd5463ba47ed05325 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 22 Feb 2022 00:44:58 -0500 Subject: [PATCH 0977/1042] propagate type updates --- .../endpoints/twitch/event_rules/create_event_rule.rs | 4 ++-- .../endpoints/twitch/event_rules/edit_event_rule.rs | 4 ++-- .../twitch/event_rules/list_event_rules_for_user.rs | 4 ++-- .../event_handlers/bits_event_handler.rs | 6 +++--- .../event_handlers/channel_points_event_handler.rs | 6 +++--- .../twitch_pubsub_user_subscriber_thread.rs | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs index 64e44fd93a..32f517caac 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs @@ -107,7 +107,7 @@ pub async fn create_twitch_event_rule_handler( let event_match_predicate = request.event_match_predicate .clone() - .unwrap_or(EventMatchPredicate::NotSet); + .unwrap_or(EventMatchPredicate::NotSet {}); let mut event_match_predicate = serde_json::to_string(&event_match_predicate) .map_err(|e| { @@ -117,7 +117,7 @@ pub async fn create_twitch_event_rule_handler( let event_response = request.event_response .clone() - .unwrap_or(EventResponse::NotSet); + .unwrap_or(EventResponse::NotSet {}); let mut event_response = serde_json::to_string(&event_match_predicate) .map_err(|e| { diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs index 8caefa46bd..2684c86805 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs @@ -125,7 +125,7 @@ pub async fn edit_twitch_event_rule_handler( let event_match_predicate = request.event_match_predicate .clone() - .unwrap_or(EventMatchPredicate::NotSet); + .unwrap_or(EventMatchPredicate::NotSet {}); let mut event_match_predicate = serde_json::to_string(&event_match_predicate) .map_err(|e| { @@ -135,7 +135,7 @@ pub async fn edit_twitch_event_rule_handler( let event_response = request.event_response .clone() - .unwrap_or(EventResponse::NotSet); + .unwrap_or(EventResponse::NotSet {}); let mut event_response = serde_json::to_string(&event_match_predicate) .map_err(|e| { diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs index 11e2fb5ea8..015d580b59 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/list_event_rules_for_user.rs @@ -116,13 +116,13 @@ pub async fn list_twitch_event_rules_for_user_handler( let event_match_predicate = serde_json::from_str(&rule.event_match_predicate) .unwrap_or_else(|e| { error!("Issue with deserializing: {}", e); - EventMatchPredicate::NotSet + EventMatchPredicate::NotSet {} }); let event_response = serde_json::from_str(&rule.event_response) .unwrap_or_else(|e| { error!("Issue with deserializing: {}", e); - EventResponse::NotSet + EventResponse::NotSet {} }); HydratedTwitchEventRule { diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs index f38452b928..f0736ab791 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs @@ -80,12 +80,12 @@ impl BitsEventHandler { .filter(|rule| rule.event_category.eq(&TwitchEventCategory::Bits)) .find(|rule| { match rule.event_match_predicate { - EventMatchPredicate::NotSet => false, // Not set + EventMatchPredicate::NotSet {} => false, // Not set EventMatchPredicate::ChannelPointsRewardNameExactMatch { .. } => false, // Wrong type EventMatchPredicate::BitsCheermoteNameExactMatch { ref cheermote_name } => { false // TODO }, - EventMatchPredicate::BitsCheermoteNameSpendThreshold { ref cheermote_name_prefix, minimum_bits_spent } => { + EventMatchPredicate::BitsCheermotePrefixSpendThreshold { ref cheermote_prefix, minimum_bits_spent } => { false // TODO }, EventMatchPredicate::BitsSpendThreshold { minimum_bits_spent } => { @@ -103,7 +103,7 @@ impl BitsEventHandler { //noinspection DuplicatedCode async fn handle_matched_rule(&self, rule: &TwitchEventRuleLight, data: &BitsEventData) -> AnyhowResult<()> { match rule.event_response { - EventResponse::NotSet => { + EventResponse::NotSet {} => { info!("Empty event response."); return Ok(()) }, diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs index eea7d2795c..414ca3c70a 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs @@ -94,9 +94,9 @@ impl ChannelPointsEventHandler { .filter(|rule| rule.event_category.eq(&TwitchEventCategory::ChannelPoints)) .find(|rule| { match rule.event_match_predicate { - EventMatchPredicate::NotSet => false, // Not set + EventMatchPredicate::NotSet {} => false, // Not set EventMatchPredicate::BitsCheermoteNameExactMatch { ..} => false, // Wrong type - EventMatchPredicate::BitsCheermoteNameSpendThreshold { .. } => false, // Wrong type + EventMatchPredicate::BitsCheermotePrefixSpendThreshold { .. } => false, // Wrong type EventMatchPredicate::BitsSpendThreshold { .. } => false, // Wrong type EventMatchPredicate::ChannelPointsRewardNameExactMatch { ref reward_name } => { reward_name.eq_ignore_ascii_case(&redemption.reward.title) @@ -116,7 +116,7 @@ impl ChannelPointsEventHandler { Some(ref m) => m.as_str(), }; match rule.event_response { - EventResponse::NotSet => { + EventResponse::NotSet {} => { info!("Empty event response."); return Ok(()) }, diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs index 521f5d2d16..69c173a02b 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs @@ -656,12 +656,12 @@ async fn lookup_event_rules( let event_match_predicate = serde_json::from_str(&rule.event_match_predicate) .unwrap_or_else(|e| { error!("Issue with deserializing: {}", e); - EventMatchPredicate::NotSet + EventMatchPredicate::NotSet {} }); let event_response = serde_json::from_str(&rule.event_response) .unwrap_or_else(|e| { error!("Issue with deserializing: {}", e); - EventResponse::NotSet + EventResponse::NotSet {} }); TwitchEventRuleLight { token: rule.token, From b0df0a0e4f3211a46e59299880d0abdd4fb42af5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 22 Feb 2022 01:10:56 -0500 Subject: [PATCH 0978/1042] fix event response serialization, make use of test script --- .../twitch/event_rules/create_event_rule.rs | 4 +- .../twitch/event_rules/edit_event_rule.rs | 2 +- debug/curl/twitch_event_rule_tests.py | 93 ++++++++++++++++--- 3 files changed, 84 insertions(+), 15 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs index 32f517caac..509e49a1a9 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/create_event_rule.rs @@ -16,7 +16,7 @@ use database_queries::complex_models::event_responses::EventResponse; use database_queries::queries::twitch::twitch_event_rules::insert_twitch_event_rule_builder::InsertTwitchEventRuleBuilder; use database_queries::tokens::Tokens; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; -use log::{info, warn, log}; +use log::{info, warn, log, error}; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; @@ -119,7 +119,7 @@ pub async fn create_twitch_event_rule_handler( .clone() .unwrap_or(EventResponse::NotSet {}); - let mut event_response = serde_json::to_string(&event_match_predicate) + let mut event_response = serde_json::to_string(&event_response) .map_err(|e| { return CreateTwitchEventRuleError::BadInput( "improper EventResponse".to_string()); diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs index 2684c86805..2e8d439a55 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs @@ -137,7 +137,7 @@ pub async fn edit_twitch_event_rule_handler( .clone() .unwrap_or(EventResponse::NotSet {}); - let mut event_response = serde_json::to_string(&event_match_predicate) + let mut event_response = serde_json::to_string(&event_response) .map_err(|e| { return EditTwitchEventRuleError::BadInput( "improper EventResponse".to_string()); diff --git a/debug/curl/twitch_event_rule_tests.py b/debug/curl/twitch_event_rule_tests.py index 5fa7d32f7b..a30018bfb8 100755 --- a/debug/curl/twitch_event_rule_tests.py +++ b/debug/curl/twitch_event_rule_tests.py @@ -4,6 +4,7 @@ import json import re import requests import uuid +import pprint login_url = 'http://localhost:12345/login' create_url = 'http://localhost:12345/twitch/event_rule/create' @@ -11,6 +12,17 @@ list_url = 'http://localhost:12345/twitch/event_rule/list' update_url = 'http://localhost:12345/twitch/event_rule/update/{}' delete_url = 'http://localhost:12345/twitch/event_rule/delete/{}' + +def delete_rule(token, cookies): + url = delete_url.format(token) + r = requests.delete(url, cookies=cookies) + print("===== Deleted Event Rule Output =====") + print('Status: {}'.format(r.status_code)) + print(r.content) + + +# ========== Setup / Login ========== + payload = { 'username_or_email': 'echelon', 'password': 'testing', @@ -30,11 +42,33 @@ session_cookie = m.group(1) cookies = { 'session': session_cookie } +# ========== Delete All Existing ========== + +r = requests.get(list_url, cookies=cookies) + +print("===== List Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +response = json.loads(r.content) + +tokens = [rule['token'] for rule in response['twitch_event_rules']] +for token in tokens: + delete_rule(token, cookies) + +# ========== Create (1) ========== + payload = { 'idempotency_token': str(uuid.uuid4()), 'event_category': 'channel_points', - 'event_match_predicate': 'TODO testing this', - 'event_response': 'TODO testing this', + 'event_match_predicate': { + 'bits_spend_threshold': { + 'minimum_bits_spent': 2, + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:4c1hycjj3a3t', # "Zephyr" voice (dev) + } + }, 'user_specified_rule_order': 0, 'rule_is_disabled': False, } @@ -48,9 +82,48 @@ print(r.content) response = json.loads(r.content) token = response['twitch_event_rule_token'] +# ========== Create (2) ========== + payload = { - 'event_match_predicate': 'Updated', - 'event_response': 'Updated 2', + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'channel_points', + 'event_match_predicate': { + 'bits_spend_threshold': { + 'minimum_bits_spent': 1, + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +r = requests.post(create_url, cookies=cookies, json=payload) + +print("===== Created Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + +response = json.loads(r.content) +token = response['twitch_event_rule_token'] + +# ========== Update ========== + + +payload = { + 'event_match_predicate': { + 'bits_spend_threshold': { + 'minimum_bits_spent': 1, + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, 'user_specified_rule_order': 123, 'rule_is_disabled': True, } @@ -61,16 +134,12 @@ print("===== Edited Event Rule Output =====") print('Status: {}'.format(r.status_code)) print(r.content) +# ========== List ========== + r = requests.get(list_url, cookies=cookies) print("===== List Event Rule Output =====") print('Status: {}'.format(r.status_code)) -print(r.content) - -url = delete_url.format(token) -r = requests.delete(url, cookies=cookies) - -print("===== Deleted Event Rule Output =====") -print('Status: {}'.format(r.status_code)) -print(r.content) +response = json.loads(r.content) +pprint.pprint(response) From e517b393cfda5d74ff7d24986b45c1d118ca1b4b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 22 Feb 2022 03:26:18 -0500 Subject: [PATCH 0979/1042] test data --- debug/curl/twitch_event_rule_tests.py | 56 ++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/debug/curl/twitch_event_rule_tests.py b/debug/curl/twitch_event_rule_tests.py index a30018bfb8..9477d51abc 100755 --- a/debug/curl/twitch_event_rule_tests.py +++ b/debug/curl/twitch_event_rule_tests.py @@ -58,7 +58,7 @@ for token in tokens: payload = { 'idempotency_token': str(uuid.uuid4()), - 'event_category': 'channel_points', + 'event_category': 'bits', 'event_match_predicate': { 'bits_spend_threshold': { 'minimum_bits_spent': 2, @@ -86,7 +86,7 @@ token = response['twitch_event_rule_token'] payload = { 'idempotency_token': str(uuid.uuid4()), - 'event_category': 'channel_points', + 'event_category': 'bits', 'event_match_predicate': { 'bits_spend_threshold': { 'minimum_bits_spent': 1, @@ -134,6 +134,58 @@ print("===== Edited Event Rule Output =====") print('Status: {}'.format(r.status_code)) print(r.content) + +# ========== Create (3) ========== + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'channel_points', + 'event_match_predicate': { + 'channel_points_reward_name_exact_match': { + 'reward_name': 'My Reward', + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +r = requests.post(create_url, cookies=cookies, json=payload) + +print("===== Created Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + +# ========== Create (4) ========== + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'bits', + 'event_match_predicate': { + 'bits_cheermote_name_exact_match': { + 'cheermote_name': 'Cheer1', + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +r = requests.post(create_url, cookies=cookies, json=payload) + +print("===== Created Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + + # ========== List ========== r = requests.get(list_url, cookies=cookies) From 75c9c2a00d5678741846f59a0e88e675d88496ff Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 22 Feb 2022 04:43:02 -0500 Subject: [PATCH 0980/1042] new "get" endpoint, changed URLs --- .../get_twitch_event_rule_for_user.rs | 2 +- .../twitch/event_rules/delete_event_rule.rs | 4 +- .../twitch/event_rules/edit_event_rule.rs | 4 +- .../twitch/event_rules/get_event_rule.rs | 165 ++++++++++++++++++ .../endpoints/twitch/event_rules/mod.rs | 1 + crates/service/storyteller_web/src/routes.rs | 21 ++- debug/curl/twitch_event_rule_tests.py | 26 +++ 7 files changed, 210 insertions(+), 13 deletions(-) create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/get_event_rule.rs diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/get_twitch_event_rule_for_user.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/get_twitch_event_rule_for_user.rs index 1e6a4ebf76..fb370cd058 100644 --- a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/get_twitch_event_rule_for_user.rs +++ b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/get_twitch_event_rule_for_user.rs @@ -19,7 +19,7 @@ pub struct TwitchEventRule { } /// Query record by token and user token -pub async fn get_twitch_event_rules_for_user( +pub async fn get_twitch_event_rule_for_user( twitch_event_rule_token: &str, user_token: &str, pool: &MySqlPool, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/delete_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/delete_event_rule.rs index 14e3048706..070c4fddc6 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/delete_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/delete_event_rule.rs @@ -11,7 +11,7 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use database_queries::queries::twitch::twitch_event_rules::delete_twitch_event_rule::delete_twitch_event_rule; -use database_queries::queries::twitch::twitch_event_rules::get_twitch_event_rule_for_user::get_twitch_event_rules_for_user; +use database_queries::queries::twitch::twitch_event_rules::get_twitch_event_rule_for_user::get_twitch_event_rule_for_user; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log, error}; use sqlx::MySqlPool; @@ -96,7 +96,7 @@ pub async fn delete_twitch_event_rule_handler( return Err(DeleteTwitchEventRuleError::NotAuthorized); } - let twitch_event_rule = get_twitch_event_rules_for_user( + let twitch_event_rule = get_twitch_event_rule_for_user( &path.token, &user_session.user_token, &server_state.mysql_pool) .await .map_err(|e| { diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs index 2e8d439a55..2ced21b448 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs @@ -12,7 +12,7 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success use crate::server_state::ServerState; use database_queries::complex_models::event_match_predicate::EventMatchPredicate; use database_queries::complex_models::event_responses::EventResponse; -use database_queries::queries::twitch::twitch_event_rules::get_twitch_event_rule_for_user::get_twitch_event_rules_for_user; +use database_queries::queries::twitch::twitch_event_rules::get_twitch_event_rule_for_user::get_twitch_event_rule_for_user; use database_queries::queries::twitch::twitch_event_rules::update_twitch_event_rule_builder::UpdateTwitchEventRuleBuilder; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log, error}; @@ -108,7 +108,7 @@ pub async fn edit_twitch_event_rule_handler( return Err(EditTwitchEventRuleError::NotAuthorized); } - let twitch_event_rule = get_twitch_event_rules_for_user( + let twitch_event_rule = get_twitch_event_rule_for_user( &path.token, &user_session.user_token, &server_state.mysql_pool) .await .map_err(|e| { diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/get_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/get_event_rule.rs new file mode 100644 index 0000000000..e1664019b2 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/get_event_rule.rs @@ -0,0 +1,165 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use database_queries::column_types::twitch_event_category::TwitchEventCategory; +use database_queries::complex_models::event_match_predicate::EventMatchPredicate; +use database_queries::complex_models::event_responses::EventResponse; +use database_queries::queries::twitch::twitch_event_rules::list_twitch_event_rules_for_user::{TwitchEventRule, list_twitch_event_rules_for_user}; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use lexical_sort::natural_lexical_cmp; +use log::{info, warn, log, error}; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; +use database_queries::queries::twitch::twitch_event_rules::get_twitch_event_rule_for_user::get_twitch_event_rule_for_user; + +// =============== Request =============== + +#[derive(Deserialize)] +pub struct GetTwitchEventRulePathInfo { + token: String, +} + +// =============== Success Response =============== + +#[derive(Debug, Serialize)] +pub struct HydratedTwitchEventRule { + pub token: String, + pub event_category: TwitchEventCategory, + pub event_match_predicate: EventMatchPredicate, + pub event_response: EventResponse, + pub user_specified_rule_order: u32, + pub rule_is_disabled: bool, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, +} + +#[derive(Serialize)] +pub struct GetTwitchEventRuleResponse { + pub success: bool, + pub twitch_event_rule: HydratedTwitchEventRule, +} + + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum GetTwitchEventRuleError { + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for GetTwitchEventRuleError { + fn status_code(&self) -> StatusCode { + match *self { + GetTwitchEventRuleError::NotFound => StatusCode::NOT_FOUND, + GetTwitchEventRuleError::NotAuthorized => StatusCode::UNAUTHORIZED, + GetTwitchEventRuleError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GetTwitchEventRuleError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn get_twitch_event_rule_for_user_handler( + path: Path, + http_request: HttpRequest, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetTwitchEventRuleError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(GetTwitchEventRuleError::NotAuthorized); + } + }; + + if user_session.is_banned { + warn!("banned users cannot use"); + return Err(GetTwitchEventRuleError::NotAuthorized); + } + + let maybe_twitch_event_rule = get_twitch_event_rule_for_user( + &path.token, &user_session.user_token, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("query error: {:?}", e); + GetTwitchEventRuleError::ServerError + })?; + + let rule = match maybe_twitch_event_rule { + Some(rule) => rule, + None => { + return Err(GetTwitchEventRuleError::NotFound); + } + }; + + let event_match_predicate = serde_json::from_str(&rule.event_match_predicate) + .unwrap_or_else(|e| { + error!("Issue with deserializing: {}", e); + EventMatchPredicate::NotSet {} + }); + + let event_response = serde_json::from_str(&rule.event_response) + .unwrap_or_else(|e| { + error!("Issue with deserializing: {}", e); + EventResponse::NotSet {} + }); + + let twitch_event_rule = HydratedTwitchEventRule { + token: rule.token, + event_category: rule.event_category, + event_match_predicate, + event_response, + user_specified_rule_order: rule.user_specified_rule_order, + rule_is_disabled: rule.rule_is_disabled, + created_at: rule.created_at, + updated_at: rule.updated_at, + }; + + let response = GetTwitchEventRuleResponse { + success: true, + twitch_event_rule, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetTwitchEventRuleError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs index 4f40936c19..0349240c4f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs @@ -1,4 +1,5 @@ pub mod create_event_rule; pub mod delete_event_rule; pub mod edit_event_rule; +pub mod get_event_rule; pub mod list_event_rules_for_user; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index cd26f9b8f2..77c3938978 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -48,6 +48,11 @@ use crate::http_server::endpoints::tts::get_tts_model_use_count::get_tts_model_u use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; +use crate::http_server::endpoints::twitch::event_rules::create_event_rule::create_twitch_event_rule_handler; +use crate::http_server::endpoints::twitch::event_rules::delete_event_rule::delete_twitch_event_rule_handler; +use crate::http_server::endpoints::twitch::event_rules::edit_event_rule::edit_twitch_event_rule_handler; +use crate::http_server::endpoints::twitch::event_rules::get_event_rule::get_twitch_event_rule_for_user_handler; +use crate::http_server::endpoints::twitch::event_rules::list_event_rules_for_user::list_twitch_event_rules_for_user_handler; use crate::http_server::endpoints::twitch::oauth::oauth_begin_json::oauth_begin_enroll_json; use crate::http_server::endpoints::twitch::oauth::oauth_begin_redirect::oauth_begin_enroll_redirect; use crate::http_server::endpoints::twitch::oauth::oauth_end::oauth_end_enroll_from_redirect; @@ -61,6 +66,8 @@ use crate::http_server::endpoints::users::list_user_w2l_templates::list_user_w2l use crate::http_server::endpoints::users::login::login_handler; use crate::http_server::endpoints::users::logout::logout_handler; use crate::http_server::endpoints::users::session_info::session_info_handler; +use crate::http_server::endpoints::voice_clone_requests::check_if_voice_clone_request_submitted::check_if_voice_clone_request_submitted_handler; +use crate::http_server::endpoints::voice_clone_requests::create_voice_clone_request::create_voice_clone_request_handler; use crate::http_server::endpoints::w2l::delete_w2l_result::delete_w2l_inference_result_handler; use crate::http_server::endpoints::w2l::delete_w2l_template::delete_w2l_template_handler; use crate::http_server::endpoints::w2l::edit_w2l_result::edit_w2l_inference_result_handler; @@ -75,12 +82,6 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; -use crate::http_server::endpoints::twitch::event_rules::create_event_rule::create_twitch_event_rule_handler; -use crate::http_server::endpoints::twitch::event_rules::list_event_rules_for_user::list_twitch_event_rules_for_user_handler; -use crate::http_server::endpoints::twitch::event_rules::delete_event_rule::delete_twitch_event_rule_handler; -use crate::http_server::endpoints::twitch::event_rules::edit_event_rule::edit_twitch_event_rule_handler; -use crate::http_server::endpoints::voice_clone_requests::create_voice_clone_request::create_voice_clone_request_handler; -use crate::http_server::endpoints::voice_clone_requests::check_if_voice_clone_request_submitted::check_if_voice_clone_request_submitted_handler; pub fn add_routes (app: App) -> App where @@ -639,11 +640,15 @@ fn add_twitch_oauth_routes (app: App) -> App .route(web::get().to(list_twitch_event_rules_for_user_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - .service(web::resource("/update/{token}") + .service(web::resource("/{token}/info") + .route(web::get().to(get_twitch_event_rule_for_user_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/{token}/update") .route(web::post().to(edit_twitch_event_rule_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - .service(web::resource("/delete/{token}") + .service(web::resource("/{token}/delete") .route(web::delete().to(delete_twitch_event_rule_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) diff --git a/debug/curl/twitch_event_rule_tests.py b/debug/curl/twitch_event_rule_tests.py index 9477d51abc..938b270b97 100755 --- a/debug/curl/twitch_event_rule_tests.py +++ b/debug/curl/twitch_event_rule_tests.py @@ -185,6 +185,32 @@ print("===== Created Event Rule Output =====") print('Status: {}'.format(r.status_code)) print(r.content) +# ========== Create (5) ========== + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'bits', + 'event_match_predicate': { + 'bits_cheermote_prefix_spend_threshold': { + 'cheermote_prefix': 'Cheer', + 'minimum_bits_spent': 5, + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +r = requests.post(create_url, cookies=cookies, json=payload) + +print("===== Created Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + # ========== List ========== From b237fe0f48084ac5d08896afb4dbc9d6acd83376 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 22 Feb 2022 05:25:04 -0500 Subject: [PATCH 0981/1042] url fix, DELETE CORS --- crates/lib/http_server_common/src/cors.rs | 2 +- debug/curl/twitch_event_rule_tests.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/lib/http_server_common/src/cors.rs b/crates/lib/http_server_common/src/cors.rs index e484753922..b6aeb51c1b 100644 --- a/crates/lib/http_server_common/src/cors.rs +++ b/crates/lib/http_server_common/src/cors.rs @@ -44,7 +44,7 @@ pub fn build_common_cors_config() -> Cors { .allowed_origin("https://vo.codes") .allowed_origin("https://vocodes.com") // Remaining setup - .allowed_methods(vec!["GET", "POST", "OPTIONS"]) + .allowed_methods(vec!["GET", "POST", "OPTIONS", "DELETE"]) .supports_credentials() .allowed_headers(vec![ http::header::ACCEPT, diff --git a/debug/curl/twitch_event_rule_tests.py b/debug/curl/twitch_event_rule_tests.py index 938b270b97..a3f96be87a 100755 --- a/debug/curl/twitch_event_rule_tests.py +++ b/debug/curl/twitch_event_rule_tests.py @@ -9,8 +9,8 @@ import pprint login_url = 'http://localhost:12345/login' create_url = 'http://localhost:12345/twitch/event_rule/create' list_url = 'http://localhost:12345/twitch/event_rule/list' -update_url = 'http://localhost:12345/twitch/event_rule/update/{}' -delete_url = 'http://localhost:12345/twitch/event_rule/delete/{}' +update_url = 'http://localhost:12345/twitch/event_rule/{}/update' +delete_url = 'http://localhost:12345/twitch/event_rule/{}/delete' def delete_rule(token, cookies): From c4e1241966390856cce98959a2ce6f3a7465cd84 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 22 Feb 2022 14:58:12 -0500 Subject: [PATCH 0982/1042] better test data --- debug/curl/twitch_event_rule_tests.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/debug/curl/twitch_event_rule_tests.py b/debug/curl/twitch_event_rule_tests.py index a3f96be87a..55986b853c 100755 --- a/debug/curl/twitch_event_rule_tests.py +++ b/debug/curl/twitch_event_rule_tests.py @@ -211,6 +211,33 @@ print("===== Created Event Rule Output =====") print('Status: {}'.format(r.status_code)) print(r.content) +# ========== Create (6) ========== + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'channel_points', + 'event_match_predicate': { + 'channel_points_reward_name_exact_match': { + 'reward_name': 'My Reward', + }, + }, + 'event_response': { + 'tts_random_voice': { + 'tts_model_tokens': [ + 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + 'TM:4c1hycjj3a3t', # "Zephyr" voice (dev) + ], + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +r = requests.post(create_url, cookies=cookies, json=payload) + +print("===== Created Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +print(r.content) # ========== List ========== From eff693692bd63b7d91e2898ce4871d7107639599 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 24 Feb 2022 05:22:55 -0500 Subject: [PATCH 0983/1042] test data --- debug/curl/twitch_event_rule_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/curl/twitch_event_rule_tests.py b/debug/curl/twitch_event_rule_tests.py index 55986b853c..5b90498e6f 100755 --- a/debug/curl/twitch_event_rule_tests.py +++ b/debug/curl/twitch_event_rule_tests.py @@ -61,7 +61,7 @@ payload = { 'event_category': 'bits', 'event_match_predicate': { 'bits_spend_threshold': { - 'minimum_bits_spent': 2, + 'minimum_bits_spent': 5000, }, }, 'event_response': { From 8ca4264afd5e4041e05c1bffe593c00e6061a192 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 24 Feb 2022 23:32:28 -0500 Subject: [PATCH 0984/1042] testing --- debug/curl/twitch_event_rule_tests.py | 50 +++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/debug/curl/twitch_event_rule_tests.py b/debug/curl/twitch_event_rule_tests.py index 5b90498e6f..f9fce3f55d 100755 --- a/debug/curl/twitch_event_rule_tests.py +++ b/debug/curl/twitch_event_rule_tests.py @@ -239,6 +239,56 @@ print("===== Created Event Rule Output =====") print('Status: {}'.format(r.status_code)) print(r.content) +# ========== Create (7) ========== + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'bits', + 'event_match_predicate': { + 'bits_cheermote_name_exact_match': { + 'cheermote_name': 'Corgo5000', + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +r = requests.post(create_url, cookies=cookies, json=payload) + +print("===== Created Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + +# ========== Create (8) ========== + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'bits', + 'event_match_predicate': { + 'bits_cheermote_name_exact_match': { + 'cheermote_name': 'ZomboCom5000', + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +r = requests.post(create_url, cookies=cookies, json=payload) + +print("===== Created Event Rule Output =====") +print('Status: {}'.format(r.status_code)) +print(r.content) + # ========== List ========== r = requests.get(list_url, cookies=cookies) From bf97de31e07d0a0afcbe75bf0255d1443cc8c093 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 24 Feb 2022 23:42:18 -0500 Subject: [PATCH 0985/1042] more structured testing --- debug/curl/twitch_event_rule_tests.py | 230 +++++++++++++------------- 1 file changed, 118 insertions(+), 112 deletions(-) diff --git a/debug/curl/twitch_event_rule_tests.py b/debug/curl/twitch_event_rule_tests.py index f9fce3f55d..5000b554a0 100755 --- a/debug/curl/twitch_event_rule_tests.py +++ b/debug/curl/twitch_event_rule_tests.py @@ -20,6 +20,12 @@ def delete_rule(token, cookies): print('Status: {}'.format(r.status_code)) print(r.content) +def create_rule(payload, cookies): + r = requests.post(create_url, cookies=cookies, json=payload) + print("===== Created Event Rule Output =====") + print('Status: {}'.format(r.status_code)) + print(r.content) + # ========== Setup / Login ========== @@ -56,34 +62,6 @@ for token in tokens: # ========== Create (1) ========== -payload = { - 'idempotency_token': str(uuid.uuid4()), - 'event_category': 'bits', - 'event_match_predicate': { - 'bits_spend_threshold': { - 'minimum_bits_spent': 5000, - }, - }, - 'event_response': { - 'tts_single_voice': { - 'tts_model_token': 'TM:4c1hycjj3a3t', # "Zephyr" voice (dev) - } - }, - 'user_specified_rule_order': 0, - 'rule_is_disabled': False, -} - -r = requests.post(create_url, cookies=cookies, json=payload) - -print("===== Created Event Rule Output =====") -print('Status: {}'.format(r.status_code)) -print(r.content) - -response = json.loads(r.content) -token = response['twitch_event_rule_token'] - -# ========== Create (2) ========== - payload = { 'idempotency_token': str(uuid.uuid4()), 'event_category': 'bits', @@ -110,13 +88,12 @@ print(r.content) response = json.loads(r.content) token = response['twitch_event_rule_token'] -# ========== Update ========== - +# ========== Update (1) ========== payload = { 'event_match_predicate': { 'bits_spend_threshold': { - 'minimum_bits_spent': 1, + 'minimum_bits_spent': 12345, }, }, 'event_response': { @@ -135,14 +112,14 @@ print('Status: {}'.format(r.status_code)) print(r.content) -# ========== Create (3) ========== +# ========== Create: Bits Spend Threshold ========== payload = { 'idempotency_token': str(uuid.uuid4()), - 'event_category': 'channel_points', + 'event_category': 'bits', 'event_match_predicate': { - 'channel_points_reward_name_exact_match': { - 'reward_name': 'My Reward', + 'bits_spend_threshold': { + 'minimum_bits_spent': 1000, }, }, 'event_response': { @@ -151,16 +128,12 @@ payload = { } }, 'user_specified_rule_order': 0, - 'rule_is_disabled': False, + 'rule_is_disabled': True, } -r = requests.post(create_url, cookies=cookies, json=payload) +create_rule(payload, cookies) -print("===== Created Event Rule Output =====") -print('Status: {}'.format(r.status_code)) -print(r.content) - -# ========== Create (4) ========== +# ========== Create: Cheermote Exact Match ========== payload = { 'idempotency_token': str(uuid.uuid4()), @@ -179,21 +152,14 @@ payload = { 'rule_is_disabled': False, } -r = requests.post(create_url, cookies=cookies, json=payload) - -print("===== Created Event Rule Output =====") -print('Status: {}'.format(r.status_code)) -print(r.content) - -# ========== Create (5) ========== +create_rule(payload, cookies) payload = { 'idempotency_token': str(uuid.uuid4()), 'event_category': 'bits', 'event_match_predicate': { - 'bits_cheermote_prefix_spend_threshold': { - 'cheermote_prefix': 'Cheer', - 'minimum_bits_spent': 5, + 'bits_cheermote_name_exact_match': { + 'cheermote_name': 'Corgo5000', }, }, 'event_response': { @@ -205,13 +171,107 @@ payload = { 'rule_is_disabled': False, } -r = requests.post(create_url, cookies=cookies, json=payload) +create_rule(payload, cookies) -print("===== Created Event Rule Output =====") -print('Status: {}'.format(r.status_code)) -print(r.content) +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'bits', + 'event_match_predicate': { + 'bits_cheermote_name_exact_match': { + 'cheermote_name': 'ZomboCom5000', + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} -# ========== Create (6) ========== +create_rule(payload, cookies) + +# ========== Create: Cheermote Prefix And Spend Threshold ========== + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'bits', + 'event_match_predicate': { + 'bits_cheermote_prefix_spend_threshold': { + 'cheermote_prefix': 'Cheer', + 'minimum_bits_spent': 1000, + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +create_rule(payload, cookies) + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'bits', + 'event_match_predicate': { + 'bits_cheermote_prefix_spend_threshold': { + 'cheermote_prefix': 'Corgo', + 'minimum_bits_spent': 12345, + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'bits', + 'event_match_predicate': { + 'bits_cheermote_prefix_spend_threshold': { + 'cheermote_prefix': 'ZomboCom', + 'minimum_bits_spent': 1000, + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +create_rule(payload, cookies) + +# ========== Create: Channel Points ========== + +payload = { + 'idempotency_token': str(uuid.uuid4()), + 'event_category': 'channel_points', + 'event_match_predicate': { + 'channel_points_reward_name_exact_match': { + 'reward_name': 'My Reward', + }, + }, + 'event_response': { + 'tts_single_voice': { + 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) + } + }, + 'user_specified_rule_order': 0, + 'rule_is_disabled': False, +} + +create_rule(payload, cookies) payload = { 'idempotency_token': str(uuid.uuid4()), @@ -233,61 +293,7 @@ payload = { 'rule_is_disabled': False, } -r = requests.post(create_url, cookies=cookies, json=payload) - -print("===== Created Event Rule Output =====") -print('Status: {}'.format(r.status_code)) -print(r.content) - -# ========== Create (7) ========== - -payload = { - 'idempotency_token': str(uuid.uuid4()), - 'event_category': 'bits', - 'event_match_predicate': { - 'bits_cheermote_name_exact_match': { - 'cheermote_name': 'Corgo5000', - }, - }, - 'event_response': { - 'tts_single_voice': { - 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) - } - }, - 'user_specified_rule_order': 0, - 'rule_is_disabled': False, -} - -r = requests.post(create_url, cookies=cookies, json=payload) - -print("===== Created Event Rule Output =====") -print('Status: {}'.format(r.status_code)) -print(r.content) - -# ========== Create (8) ========== - -payload = { - 'idempotency_token': str(uuid.uuid4()), - 'event_category': 'bits', - 'event_match_predicate': { - 'bits_cheermote_name_exact_match': { - 'cheermote_name': 'ZomboCom5000', - }, - }, - 'event_response': { - 'tts_single_voice': { - 'tts_model_token': 'TM:40m3aqtt41y0', # "Wakko" voice (dev) - } - }, - 'user_specified_rule_order': 0, - 'rule_is_disabled': False, -} - -r = requests.post(create_url, cookies=cookies, json=payload) - -print("===== Created Event Rule Output =====") -print('Status: {}'.format(r.status_code)) -print(r.content) +create_rule(payload, cookies) # ========== List ========== From 65c821e3943366cc9592efcea95373f3b77a8ec0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 25 Feb 2022 04:26:35 -0500 Subject: [PATCH 0986/1042] bugfix --- debug/curl/twitch_event_rule_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/curl/twitch_event_rule_tests.py b/debug/curl/twitch_event_rule_tests.py index 5000b554a0..fd6583168e 100755 --- a/debug/curl/twitch_event_rule_tests.py +++ b/debug/curl/twitch_event_rule_tests.py @@ -199,7 +199,7 @@ payload = { 'event_category': 'bits', 'event_match_predicate': { 'bits_cheermote_prefix_spend_threshold': { - 'cheermote_prefix': 'Cheer', + 'cheermote_prefix': 'DoodleCheer', 'minimum_bits_spent': 1000, }, }, From 7e294d77d72dd9293eaaa7ee59b6f97bf282aa72 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 25 Feb 2022 20:29:17 -0500 Subject: [PATCH 0987/1042] remove rule order from edit event (since this is awkward in isolation) --- .../twitch_event_rules/update_twitch_event_rule_builder.rs | 3 --- .../endpoints/twitch/event_rules/edit_event_rule.rs | 2 -- 2 files changed, 5 deletions(-) diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/update_twitch_event_rule_builder.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/update_twitch_event_rule_builder.rs index 8a5c4b6292..08b07b384f 100644 --- a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/update_twitch_event_rule_builder.rs +++ b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/update_twitch_event_rule_builder.rs @@ -9,7 +9,6 @@ pub struct UpdateTwitchEventRuleBuilder { pub token: String, pub event_match_predicate: String, pub event_response: String, - pub user_specified_rule_order: u32, pub rule_is_disabled: bool, pub ip_address_update: String, } @@ -24,7 +23,6 @@ UPDATE twitch_event_rules SET event_match_predicate = ?, event_response = ?, - user_specified_rule_order = ?, rule_is_disabled = ?, ip_address_last_update = ? WHERE @@ -34,7 +32,6 @@ LIMIT 1 "#, &self.event_match_predicate, &self.event_response, - &self.user_specified_rule_order, &self.rule_is_disabled, &self.ip_address_update, &self.token, diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs index 2ced21b448..d15c7c7a89 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs @@ -35,7 +35,6 @@ pub struct EditTwitchEventRulePathInfo { pub struct EditTwitchEventRuleRequest { pub event_match_predicate: Option, pub event_response: Option, - pub user_specified_rule_order: u32, pub rule_is_disabled: bool, } @@ -147,7 +146,6 @@ pub async fn edit_twitch_event_rule_handler( token: path.token.clone(), event_match_predicate, event_response, - user_specified_rule_order: request.user_specified_rule_order, rule_is_disabled: request.rule_is_disabled, ip_address_update: creator_ip_address, }; From 9b54e5b4be8e532377545d8634674ebce8007a61 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 25 Feb 2022 20:43:34 -0500 Subject: [PATCH 0988/1042] validation of predicates and responses --- .../twitch/event_rules/edit_event_rule.rs | 8 ++++ .../endpoints/twitch/event_rules/mod.rs | 3 +- .../twitch/event_rules/validations/mod.rs | 2 + .../validate_event_match_predicate.rs | 37 +++++++++++++++++++ .../validations/validate_event_response.rs | 26 +++++++++++++ 5 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/mod.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_match_predicate.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_response.rs diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs index d15c7c7a89..cbfe0b9128 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/edit_event_rule.rs @@ -22,6 +22,8 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; +use crate::http_server::endpoints::twitch::event_rules::validations::validate_event_response::validate_event_response; +use crate::http_server::endpoints::twitch::event_rules::validations::validate_event_match_predicate::validate_event_match_predicate; // =============== Request =============== @@ -126,6 +128,9 @@ pub async fn edit_twitch_event_rule_handler( .clone() .unwrap_or(EventMatchPredicate::NotSet {}); + validate_event_match_predicate(&event_match_predicate) + .map_err(|reason| EditTwitchEventRuleError::BadInput(reason))?; + let mut event_match_predicate = serde_json::to_string(&event_match_predicate) .map_err(|e| { return EditTwitchEventRuleError::BadInput( @@ -136,6 +141,9 @@ pub async fn edit_twitch_event_rule_handler( .clone() .unwrap_or(EventResponse::NotSet {}); + validate_event_response(&event_response) + .map_err(|reason| EditTwitchEventRuleError::BadInput(reason))?; + let mut event_response = serde_json::to_string(&event_response) .map_err(|e| { return EditTwitchEventRuleError::BadInput( diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs index 0349240c4f..55446634a8 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs @@ -2,4 +2,5 @@ pub mod create_event_rule; pub mod delete_event_rule; pub mod edit_event_rule; pub mod get_event_rule; -pub mod list_event_rules_for_user; \ No newline at end of file +pub mod list_event_rules_for_user; +pub mod validations; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/mod.rs new file mode 100644 index 0000000000..9334525946 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/mod.rs @@ -0,0 +1,2 @@ +pub mod validate_event_match_predicate; +pub mod validate_event_response; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_match_predicate.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_match_predicate.rs new file mode 100644 index 0000000000..443cc02b2b --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_match_predicate.rs @@ -0,0 +1,37 @@ +use database_queries::complex_models::event_match_predicate::EventMatchPredicate; + +// TODO: Check that the tokens are valid models. + +/// Try to prevent saving garbage to the database. +pub fn validate_event_match_predicate(event_match_predicate: &EventMatchPredicate) -> Result<(), String> { + match event_match_predicate { + EventMatchPredicate::NotSet {} => { + // Implicitly valid + }, + EventMatchPredicate::BitsCheermoteNameExactMatch { cheermote_name } => { + if cheermote_name.trim().is_empty() { + return Err("cheermote_name is empty".to_string()); + } + } + EventMatchPredicate::BitsCheermotePrefixSpendThreshold { cheermote_prefix, minimum_bits_spent } => { + if cheermote_prefix.trim().is_empty() { + return Err("cheermote_prefix is empty".to_string()); + } + if *minimum_bits_spent < 1 { + return Err("minimum_bits_spent should be a positive integer".to_string()); + } + } + EventMatchPredicate::BitsSpendThreshold { minimum_bits_spent } => { + if *minimum_bits_spent < 1 { + return Err("minimum_bits_spent should be a positive integer".to_string()); + } + } + EventMatchPredicate::ChannelPointsRewardNameExactMatch { reward_name } => { + if reward_name.trim().is_empty() { + return Err("reward_name is empty".to_string()); + } + } + } + + Ok(()) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_response.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_response.rs new file mode 100644 index 0000000000..8d01fba03a --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_response.rs @@ -0,0 +1,26 @@ +use database_queries::complex_models::event_responses::EventResponse; + +// TODO: Check that the tokens are valid models. + +/// Try to prevent saving garbage to the database. +pub fn validate_event_response(event_response: &EventResponse) -> Result<(), String> { + match event_response { + EventResponse::NotSet {} => { + // Implicitly valid + }, + EventResponse::TtsSingleVoice { tts_model_token } => { + if tts_model_token.is_empty() { + return Err("tts_model_token is empty".to_string()); + } + }, + EventResponse::TtsRandomVoice { tts_model_tokens } => { + for model_token in tts_model_tokens { + if model_token.is_empty() { + return Err("one or more tts_model_tokens is invalid".to_string()); + } + } + }, + } + + Ok(()) +} From dfe902de4bcbb6ff2a70c66566e0f5d386222fac Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 26 Feb 2022 05:29:35 -0500 Subject: [PATCH 0989/1042] Begin work on "rule reorder", plus tests for unsafe query --- Cargo.lock | 2 + crates/lib/database_queries/Cargo.toml | 4 + .../queries/twitch/twitch_event_rules/mod.rs | 1 + .../reorder_twitch_event_rules.rs | 98 ++++++++++ .../endpoints/twitch/event_rules/mod.rs | 1 + .../event_rules/reorder_twitch_event_rules.rs | 178 ++++++++++++++++++ 6 files changed, 284 insertions(+) create mode 100644 crates/lib/database_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/reorder_twitch_event_rules.rs diff --git a/Cargo.lock b/Cargo.lock index 23c484d7ad..6b549d6abb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1254,6 +1254,8 @@ dependencies = [ "container_common", "dotenv", "log", + "once_cell", + "regex", "serde", "serde_derive", "serde_json", diff --git a/crates/lib/database_queries/Cargo.toml b/crates/lib/database_queries/Cargo.toml index b15ef8bd7f..ab092d51d2 100644 --- a/crates/lib/database_queries/Cargo.toml +++ b/crates/lib/database_queries/Cargo.toml @@ -25,6 +25,10 @@ chrono = { version = "0.4.19", features = ["serde"] } dotenv = "0.15.0" log = "0.4.14" +# Needed to safety check some queries we can't use escaping for +once_cell = "1.9.0" +regex = "1.5.4" + # Serialization serde = "1.0.125" serde_derive = "1.0.125" diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/mod.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/mod.rs index 287e5bc6ef..b0db364aac 100644 --- a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/mod.rs +++ b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/mod.rs @@ -2,4 +2,5 @@ pub mod delete_twitch_event_rule; pub mod get_twitch_event_rule_for_user; pub mod insert_twitch_event_rule_builder; pub mod list_twitch_event_rules_for_user; +pub mod reorder_twitch_event_rules; pub mod update_twitch_event_rule_builder; \ No newline at end of file diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs new file mode 100644 index 0000000000..3f62e126f1 --- /dev/null +++ b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs @@ -0,0 +1,98 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use once_cell::sync::Lazy; +use regex::Regex; +use sqlx::MySqlPool; +use std::collections::HashMap; + +/// Check that tokens are alphanumeric or can have colons. +static SAFE_TOKENS_REGEX : Lazy = Lazy::new(|| { + Regex::new("^[a-zA-Z0-9:]+$").unwrap() +}); + +/// Check that tokens are alphanumeric or can have colons. +#[must_use] +fn check_token_safety(token: &str) -> AnyhowResult<()> { + if SAFE_TOKENS_REGEX.is_match(token) { + Ok(()) + } else { + Err(anyhow!("Token contains invalid characters")) + } +} + +pub async fn reorder_twitch_event_rules( + rule_token_to_order_map: HashMap, + user_token: &str, + ip_address_update: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult { + + // THIS IS NOT SANITIZED, so we check for injection here. + for (rule_token, _position) in rule_token_to_order_map.iter() { + check_token_safety(rule_token.as_str())?; + } + + // THIS IS NOT SANITIZED, so we check for injection here. + check_token_safety(user_token)?; + + let query_inner = rule_token_to_order_map.iter() + .map(|(token, position)| { + format!("(\"{}\", {}, \"{}\")", token.as_str(), position, ip_address_update) + }) + .collect::>() + .join(", "); + + // Upsert to accomplish multiple updates at once. + // tl;dr https://stackoverflow.com/a/34866431 + // ALSO THIS IS NOT SANITIZED. + let query = format!(r#" +INSERT INTO twitch_event_rules (token, user_specified_rule_order, ip_address_update) +VALUES + {} +ON DUPLICATE KEY UPDATE + token=VALUES(token), + user_specified_rule_order=VALUES(user_specified_rule_order), + ip_address_update=VALUES(ip_address_update) +WHERE + user_token = {} + "#, query_inner, user_token); + + // NB: This, unfortunately, cannot be statically checked. + let mut query = sqlx::query(&query); + + let result = query.execute(mysql_pool).await; + + match result { + Err(err) => Err(anyhow!("error with query: {:?}", err)), + Ok(_r) => Ok(true), + } +} + +#[cfg(test)] +mod tests { + use crate::queries::twitch::twitch_event_rules::reorder_twitch_event_rules::check_token_safety; + + #[test] + fn safe_tokens() { + assert!(check_token_safety("foo").is_ok()); + assert!(check_token_safety("FOO").is_ok()); + assert!(check_token_safety("12345").is_ok()); + assert!(check_token_safety("abcdefABCDEF012345679").is_ok()); + assert!(check_token_safety("PREFIX:TOKEN1234").is_ok()); + } + + #[test] + fn unsafe_tokens() { + assert!(check_token_safety("").is_err()); + assert!(check_token_safety(",").is_err()); + assert!(check_token_safety("ABCD,").is_err()); + assert!(check_token_safety("foo;").is_err()); + assert!(check_token_safety("foo;foo").is_err()); + assert!(check_token_safety("foo;foo").is_err()); + assert!(check_token_safety("\"foo").is_err()); + assert!(check_token_safety("foo\"").is_err()); + assert!(check_token_safety("foo'").is_err()); + assert!(check_token_safety("foo'bar").is_err()); + assert!(check_token_safety("foo`foo").is_err()); + } +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs index 55446634a8..56d071b288 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/mod.rs @@ -3,4 +3,5 @@ pub mod delete_event_rule; pub mod edit_event_rule; pub mod get_event_rule; pub mod list_event_rules_for_user; +pub mod reorder_twitch_event_rules; pub mod validations; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/reorder_twitch_event_rules.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/reorder_twitch_event_rules.rs new file mode 100644 index 0000000000..cbfe0b9128 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/reorder_twitch_event_rules.rs @@ -0,0 +1,178 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use database_queries::complex_models::event_match_predicate::EventMatchPredicate; +use database_queries::complex_models::event_responses::EventResponse; +use database_queries::queries::twitch::twitch_event_rules::get_twitch_event_rule_for_user::get_twitch_event_rule_for_user; +use database_queries::queries::twitch::twitch_event_rules::update_twitch_event_rule_builder::UpdateTwitchEventRuleBuilder; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log, error}; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; +use crate::http_server::endpoints::twitch::event_rules::validations::validate_event_response::validate_event_response; +use crate::http_server::endpoints::twitch::event_rules::validations::validate_event_match_predicate::validate_event_match_predicate; + +// =============== Request =============== + +#[derive(Deserialize)] +pub struct EditTwitchEventRulePathInfo { + token: String, +} + +// These are not sparse updates! +#[derive(Deserialize)] +pub struct EditTwitchEventRuleRequest { + pub event_match_predicate: Option, + pub event_response: Option, + pub rule_is_disabled: bool, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct EditTwitchEventRuleResponse { + pub success: bool, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum EditTwitchEventRuleError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for EditTwitchEventRuleError { + fn status_code(&self) -> StatusCode { + match *self { + EditTwitchEventRuleError::BadInput(_) => StatusCode::BAD_REQUEST, + EditTwitchEventRuleError::NotFound => StatusCode::NOT_FOUND, + EditTwitchEventRuleError::NotAuthorized => StatusCode::UNAUTHORIZED, + EditTwitchEventRuleError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for EditTwitchEventRuleError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn edit_twitch_event_rule_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EditTwitchEventRuleError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(EditTwitchEventRuleError::NotAuthorized); + } + }; + + if user_session.is_banned { + warn!("banned users cannot edit"); + return Err(EditTwitchEventRuleError::NotAuthorized); + } + + let twitch_event_rule = get_twitch_event_rule_for_user( + &path.token, &user_session.user_token, &server_state.mysql_pool) + .await + .map_err(|e| { + error!("Error with query: {:?}", e); + return EditTwitchEventRuleError::ServerError; + })?; + + if twitch_event_rule.is_none() { + info!("event rule not found"); + return Err(EditTwitchEventRuleError::NotFound); + } + + let creator_ip_address = get_request_ip(&http_request); + + let event_match_predicate = request.event_match_predicate + .clone() + .unwrap_or(EventMatchPredicate::NotSet {}); + + validate_event_match_predicate(&event_match_predicate) + .map_err(|reason| EditTwitchEventRuleError::BadInput(reason))?; + + let mut event_match_predicate = serde_json::to_string(&event_match_predicate) + .map_err(|e| { + return EditTwitchEventRuleError::BadInput( + "improper EventMatchPredicate".to_string()); + })?; + + let event_response = request.event_response + .clone() + .unwrap_or(EventResponse::NotSet {}); + + validate_event_response(&event_response) + .map_err(|reason| EditTwitchEventRuleError::BadInput(reason))?; + + let mut event_response = serde_json::to_string(&event_response) + .map_err(|e| { + return EditTwitchEventRuleError::BadInput( + "improper EventResponse".to_string()); + })?; + + let update_builder = UpdateTwitchEventRuleBuilder { + token: path.token.clone(), + event_match_predicate, + event_response, + rule_is_disabled: request.rule_is_disabled, + ip_address_update: creator_ip_address, + }; + + let _r = update_builder.update(&server_state.mysql_pool) + .await + .map_err(|e| { + error!("Error with query: {:?}", e); + return EditTwitchEventRuleError::ServerError; + }); + + let response = EditTwitchEventRuleResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| EditTwitchEventRuleError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} From e738d147e47b8b163ac33a517e6c8b6b4fcde130 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 26 Feb 2022 05:50:50 -0500 Subject: [PATCH 0990/1042] handler for reorder rules (untested) --- .../event_rules/reorder_twitch_event_rules.rs | 132 ++++++++---------- crates/service/storyteller_web/src/routes.rs | 9 +- 2 files changed, 68 insertions(+), 73 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/reorder_twitch_event_rules.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/reorder_twitch_event_rules.rs index cbfe0b9128..75e1937573 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/reorder_twitch_event_rules.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/reorder_twitch_event_rules.rs @@ -6,6 +6,8 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::http_server::endpoints::twitch::event_rules::validations::validate_event_match_predicate::validate_event_match_predicate; +use crate::http_server::endpoints::twitch::event_rules::validations::validate_event_response::validate_event_response; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -13,6 +15,8 @@ use crate::server_state::ServerState; use database_queries::complex_models::event_match_predicate::EventMatchPredicate; use database_queries::complex_models::event_responses::EventResponse; use database_queries::queries::twitch::twitch_event_rules::get_twitch_event_rule_for_user::get_twitch_event_rule_for_user; +use database_queries::queries::twitch::twitch_event_rules::list_twitch_event_rules_for_user::list_twitch_event_rules_for_user; +use database_queries::queries::twitch::twitch_event_rules::reorder_twitch_event_rules::reorder_twitch_event_rules; use database_queries::queries::twitch::twitch_event_rules::update_twitch_event_rule_builder::UpdateTwitchEventRuleBuilder; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn, log, error}; @@ -20,50 +24,47 @@ use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; +use std::collections::{HashSet, HashMap}; use std::fmt; use std::sync::Arc; -use crate::http_server::endpoints::twitch::event_rules::validations::validate_event_response::validate_event_response; -use crate::http_server::endpoints::twitch::event_rules::validations::validate_event_match_predicate::validate_event_match_predicate; // =============== Request =============== #[derive(Deserialize)] -pub struct EditTwitchEventRulePathInfo { - token: String, +pub struct RuleTokenPositionPair { + pub rule_token: String, + pub position: u32, } -// These are not sparse updates! #[derive(Deserialize)] -pub struct EditTwitchEventRuleRequest { - pub event_match_predicate: Option, - pub event_response: Option, - pub rule_is_disabled: bool, +pub struct ReorderTwitchEventRulesRequest { + pub rule_token_position_pairs: Vec } // =============== Success Response =============== #[derive(Serialize)] -pub struct EditTwitchEventRuleResponse { +pub struct ReorderTwitchEventRulesResponse { pub success: bool, } // =============== Error Response =============== #[derive(Debug, Serialize)] -pub enum EditTwitchEventRuleError { +pub enum ReorderTwitchEventRulesError { BadInput(String), NotFound, NotAuthorized, ServerError, } -impl ResponseError for EditTwitchEventRuleError { +impl ResponseError for ReorderTwitchEventRulesError { fn status_code(&self) -> StatusCode { match *self { - EditTwitchEventRuleError::BadInput(_) => StatusCode::BAD_REQUEST, - EditTwitchEventRuleError::NotFound => StatusCode::NOT_FOUND, - EditTwitchEventRuleError::NotAuthorized => StatusCode::UNAUTHORIZED, - EditTwitchEventRuleError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + ReorderTwitchEventRulesError::BadInput(_) => StatusCode::BAD_REQUEST, + ReorderTwitchEventRulesError::NotFound => StatusCode::NOT_FOUND, + ReorderTwitchEventRulesError::NotAuthorized => StatusCode::UNAUTHORIZED, + ReorderTwitchEventRulesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, } } @@ -73,7 +74,7 @@ impl ResponseError for EditTwitchEventRuleError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for EditTwitchEventRuleError { +impl fmt::Display for ReorderTwitchEventRulesError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } @@ -81,11 +82,10 @@ impl fmt::Display for EditTwitchEventRuleError { // =============== Handler =============== -pub async fn edit_twitch_event_rule_handler( +pub async fn reorder_twitch_event_rules_handler( http_request: HttpRequest, - path: Path, - request: web::Json, - server_state: web::Data>) -> Result + request: web::Json, + server_state: web::Data>) -> Result { let maybe_user_session = server_state .session_checker @@ -93,84 +93,74 @@ pub async fn edit_twitch_event_rule_handler( .await .map_err(|e| { warn!("Session checker error: {:?}", e); - EditTwitchEventRuleError::ServerError + ReorderTwitchEventRulesError::ServerError })?; let user_session = match maybe_user_session { Some(session) => session, None => { warn!("not logged in"); - return Err(EditTwitchEventRuleError::NotAuthorized); + return Err(ReorderTwitchEventRulesError::NotAuthorized); } }; if user_session.is_banned { warn!("banned users cannot edit"); - return Err(EditTwitchEventRuleError::NotAuthorized); + return Err(ReorderTwitchEventRulesError::NotAuthorized); } - let twitch_event_rule = get_twitch_event_rule_for_user( - &path.token, &user_session.user_token, &server_state.mysql_pool) + let rules = list_twitch_event_rules_for_user( + &user_session.user_token, + &server_state.mysql_pool) .await .map_err(|e| { error!("Error with query: {:?}", e); - return EditTwitchEventRuleError::ServerError; + return ReorderTwitchEventRulesError::ServerError; })?; - if twitch_event_rule.is_none() { - info!("event rule not found"); - return Err(EditTwitchEventRuleError::NotFound); + // Here we check that the input set is entirely valid. + // The update query is pretty dangerous (though also uses input checks), so this + // should doubly guarantee nothing bad slips through. + // Also it serves as a permission check. + + let valid_rule_tokens = rules.iter() + .map(|rule| rule.token.to_string()) + .collect::>(); + + let check_tokens = request.rule_token_position_pairs.iter() + .map(|pair| pair.rule_token.to_string()) + .collect::>(); + + for check_token in check_tokens.iter() { + if !valid_rule_tokens.contains(check_token.as_str()) { + return Err(ReorderTwitchEventRulesError::BadInput("invalid token supplied".to_string())); + } } - let creator_ip_address = get_request_ip(&http_request); + let rule_token_to_order_map = request.rule_token_position_pairs.iter() + .map(|pair| (pair.rule_token.to_string(), pair.position)) + .collect::>(); - let event_match_predicate = request.event_match_predicate - .clone() - .unwrap_or(EventMatchPredicate::NotSet {}); + let ip_address = get_request_ip(&http_request); - validate_event_match_predicate(&event_match_predicate) - .map_err(|reason| EditTwitchEventRuleError::BadInput(reason))?; - - let mut event_match_predicate = serde_json::to_string(&event_match_predicate) - .map_err(|e| { - return EditTwitchEventRuleError::BadInput( - "improper EventMatchPredicate".to_string()); - })?; - - let event_response = request.event_response - .clone() - .unwrap_or(EventResponse::NotSet {}); - - validate_event_response(&event_response) - .map_err(|reason| EditTwitchEventRuleError::BadInput(reason))?; - - let mut event_response = serde_json::to_string(&event_response) - .map_err(|e| { - return EditTwitchEventRuleError::BadInput( - "improper EventResponse".to_string()); - })?; - - let update_builder = UpdateTwitchEventRuleBuilder { - token: path.token.clone(), - event_match_predicate, - event_response, - rule_is_disabled: request.rule_is_disabled, - ip_address_update: creator_ip_address, - }; - - let _r = update_builder.update(&server_state.mysql_pool) + reorder_twitch_event_rules( + rule_token_to_order_map, + &user_session.user_token, + &ip_address, + &server_state.mysql_pool) .await - .map_err(|e| { - error!("Error with query: {:?}", e); - return EditTwitchEventRuleError::ServerError; - }); + .map_err(|err| { + error!("Error with query: {:?}", err); + return ReorderTwitchEventRulesError::ServerError; + })?; - let response = EditTwitchEventRuleResponse { + + let response = ReorderTwitchEventRulesResponse { success: true, }; let body = serde_json::to_string(&response) - .map_err(|e| EditTwitchEventRuleError::ServerError)?; + .map_err(|e| ReorderTwitchEventRulesError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index 77c3938978..03bdf3fedb 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -53,6 +53,7 @@ use crate::http_server::endpoints::twitch::event_rules::delete_event_rule::delet use crate::http_server::endpoints::twitch::event_rules::edit_event_rule::edit_twitch_event_rule_handler; use crate::http_server::endpoints::twitch::event_rules::get_event_rule::get_twitch_event_rule_for_user_handler; use crate::http_server::endpoints::twitch::event_rules::list_event_rules_for_user::list_twitch_event_rules_for_user_handler; +use crate::http_server::endpoints::twitch::event_rules::reorder_twitch_event_rules::reorder_twitch_event_rules_handler; use crate::http_server::endpoints::twitch::oauth::oauth_begin_json::oauth_begin_enroll_json; use crate::http_server::endpoints::twitch::oauth::oauth_begin_redirect::oauth_begin_enroll_redirect; use crate::http_server::endpoints::twitch::oauth::oauth_end::oauth_end_enroll_from_redirect; @@ -101,7 +102,7 @@ pub fn add_routes (app: App) -> App app = add_user_profile_routes(app); /* /user */ app = add_api_token_routes(app); /* /api_tokens */ app = add_voice_clone_request_routes(app); /* /voice_clone_requests */ - app = add_twitch_oauth_routes(app); /* /twitch */ // TODO: MAYBE TEMPORARY + app = add_twitch_routes(app); /* /twitch */ // TODO: MAYBE TEMPORARY // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== app.service( @@ -606,7 +607,7 @@ fn add_voice_clone_request_routes (app: App) -> App // TODO: Maybe move these into an "oauth-gateway" type http service. // We'll want the domain to have reputation and not confuse people. // It'll also be nice to accrue all oauth things here. -fn add_twitch_oauth_routes (app: App) -> App +fn add_twitch_routes (app: App) -> App where B: MessageBody, T: ServiceFactory< @@ -636,6 +637,10 @@ fn add_twitch_oauth_routes (app: App) -> App .route(web::post().to(create_twitch_event_rule_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/reorder") + .route(web::post().to(reorder_twitch_event_rules_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/list") .route(web::get().to(list_twitch_event_rules_for_user_handler)) .route(web::head().to(|| HttpResponse::Ok())) From 9633e7515a0afe0abbd27404b7fc5e5e30d90ff2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 26 Feb 2022 06:38:06 -0500 Subject: [PATCH 0991/1042] fixed up, because upsert fails for rquired key --- .../reorder_twitch_event_rules.rs | 68 ++++++++++++++----- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs index 3f62e126f1..76a3f874b2 100644 --- a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs +++ b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/reorder_twitch_event_rules.rs @@ -1,5 +1,6 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; +use log::error; use once_cell::sync::Lazy; use regex::Regex; use sqlx::MySqlPool; @@ -32,30 +33,63 @@ pub async fn reorder_twitch_event_rules( check_token_safety(rule_token.as_str())?; } - // THIS IS NOT SANITIZED, so we check for injection here. check_token_safety(user_token)?; - let query_inner = rule_token_to_order_map.iter() - .map(|(token, position)| { - format!("(\"{}\", {}, \"{}\")", token.as_str(), position, ip_address_update) +// NB: Sadly upserts fail due to other required primary keys that we cannot supply here (eg. uuid). +// let query_inner = rule_token_to_order_map.iter() +// .map(|(token, position)| { +// format!("(\"{}\", {}, \"{}\")", token.as_str(), position, ip_address_update) +// }) +// .collect::>() +// .join(", "); +// +// // Upsert to accomplish multiple updates at once. +// // tl;dr https://stackoverflow.com/a/34866431 +// // ALSO THIS IS NOT SANITIZED. +// let query = format!(r#" +//INSERT INTO twitch_event_rules (token, user_specified_rule_order, ip_address_last_update) +//VALUES +// {} +//ON DUPLICATE KEY UPDATE +// token=VALUES(token), +// user_specified_rule_order=VALUES(user_specified_rule_order), +// ip_address_last_update=VALUES(ip_address_last_update) +// "#, query_inner); + + let rule_tokens = rule_token_to_order_map.iter() + .map(|(token, _position)| { + format!("'{}'", token) }) .collect::>() .join(", "); - // Upsert to accomplish multiple updates at once. - // tl;dr https://stackoverflow.com/a/34866431 - // ALSO THIS IS NOT SANITIZED. + let query_inner_positions = rule_token_to_order_map.iter() + .map(|(token, position)| { + format!("when token = '{}' then '{}'", token.as_str(), position) + }) + .collect::>() + .join("\n"); + + let query_inner_ip_addresses = rule_token_to_order_map.iter() + .map(|(token, position)| { + format!("when token = '{}' then '{}'", token.as_str(), ip_address_update) + }) + .collect::>() + .join("\n"); + let query = format!(r#" -INSERT INTO twitch_event_rules (token, user_specified_rule_order, ip_address_update) -VALUES - {} -ON DUPLICATE KEY UPDATE - token=VALUES(token), - user_specified_rule_order=VALUES(user_specified_rule_order), - ip_address_update=VALUES(ip_address_update) -WHERE - user_token = {} - "#, query_inner, user_token); +UPDATE twitch_event_rules +SET + user_specified_rule_order = (case + {} + end), + ip_address_last_update = (case + {} + end) + WHERE user_token = '{}' + AND deleted_at IS NULL + AND token IN ({}) + "#, query_inner_positions, query_inner_ip_addresses, user_token, rule_tokens); // NB: This, unfortunately, cannot be statically checked. let mut query = sqlx::query(&query); From 7a77207275e1128fe8e119d1052b32daa98a583a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 26 Feb 2022 20:46:40 -0500 Subject: [PATCH 0992/1042] sort by rule order --- .../twitch_event_rules/list_twitch_event_rules_for_user.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs index c9a5581893..6d7976dbde 100644 --- a/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs +++ b/crates/lib/database_queries/src/queries/twitch/twitch_event_rules/list_twitch_event_rules_for_user.rs @@ -61,7 +61,7 @@ WHERE .collect::>(); // Queried in DESC order, but sort returned results ordered ASC. - records.sort_by_key(|r| r.created_at); + records.sort_by_key(|r| r.user_specified_rule_order); Ok(records) } From 1ca99bdcafc8113850651f38a9c00a5f3649e3ba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 26 Feb 2022 20:48:23 -0500 Subject: [PATCH 0993/1042] codegen db --- sqlx-data.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 4497be793a..08ca688f44 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -4144,6 +4144,16 @@ ] } }, + "8c6a56bfa985324a0dd138c9ac9570d72195ee721d5e2d6c50c4266d1590ed20": { + "query": "\nUPDATE twitch_event_rules\nSET\n event_match_predicate = ?,\n event_response = ?,\n rule_is_disabled = ?,\n ip_address_last_update = ?\nWHERE\n token = ?\n AND deleted_at IS NULL\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + } + }, "8ff27fc5583db97f157f1b808f593a72c77dc5dcf5a633cd9c1162af6b97c011": { "query": "\nUPDATE w2l_templates\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -4989,16 +4999,6 @@ ] } }, - "a6c74861387a61fbb1e0089f27a16bb01efe527c3a709ff0b2897cb4c0fcede8": { - "query": "\nUPDATE twitch_event_rules\nSET\n event_match_predicate = ?,\n event_response = ?,\n user_specified_rule_order = ?,\n rule_is_disabled = ?,\n ip_address_last_update = ?\nWHERE\n token = ?\n AND deleted_at IS NULL\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - } - }, "ab6f25eb0f6340d49aa705835ed428434d8b6ab14b0740f6a5ebe7841b9ece20": { "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { From c44dc209b30a56e0301fab728b0b234b3a3ccabc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 27 Feb 2022 05:22:56 -0500 Subject: [PATCH 0994/1042] more cors --- crates/lib/http_server_common/src/cors.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/lib/http_server_common/src/cors.rs b/crates/lib/http_server_common/src/cors.rs index b6aeb51c1b..e0eada1edc 100644 --- a/crates/lib/http_server_common/src/cors.rs +++ b/crates/lib/http_server_common/src/cors.rs @@ -15,8 +15,15 @@ pub fn build_common_cors_config() -> Cors { .allowed_origin("http://localhost:54321") .allowed_origin("http://localhost:5555") .allowed_origin("http://localhost:7000") - .allowed_origin("http://localhost:7000") .allowed_origin("http://localhost:7001") + .allowed_origin("http://localhost:7002") + .allowed_origin("http://localhost:7003") + .allowed_origin("http://localhost:7004") + .allowed_origin("http://localhost:7005") + .allowed_origin("http://localhost:7006") + .allowed_origin("http://localhost:7007") + .allowed_origin("http://localhost:7008") + .allowed_origin("http://localhost:7009") .allowed_origin("http://localhost:8000") .allowed_origin("http://localhost:8080") .allowed_origin("https://api.jungle.horse") From 7b7417ae147facb57d633db1936d474ce1032071 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 27 Feb 2022 06:37:40 -0500 Subject: [PATCH 0995/1042] cors, nginx (still not working) --- crates/lib/http_server_common/src/cors.rs | 19 ++++++++--- .../obs_gateway_websocket_handler.rs | 6 ++-- localdev/nginx-http-config/storyteller-ssl | 32 ++++++++++++++++++- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/crates/lib/http_server_common/src/cors.rs b/crates/lib/http_server_common/src/cors.rs index e0eada1edc..19aafcef3c 100644 --- a/crates/lib/http_server_common/src/cors.rs +++ b/crates/lib/http_server_common/src/cors.rs @@ -5,11 +5,7 @@ use actix_http::http; /// Return cors config for FakeYou / Vocodes / OBS / local development pub fn build_common_cors_config() -> Cors { Cors::default() - // Local Development - .allowed_origin("http://api.jungle.horse") - .allowed_origin("http://jungle.horse") - .allowed_origin("http://jungle.horse:12345") - .allowed_origin("http://jungle.horse:7000") + // Local Development (Localhost) .allowed_origin("http://localhost:12345") .allowed_origin("http://localhost:3000") .allowed_origin("http://localhost:54321") @@ -26,8 +22,17 @@ pub fn build_common_cors_config() -> Cors { .allowed_origin("http://localhost:7009") .allowed_origin("http://localhost:8000") .allowed_origin("http://localhost:8080") + // Local Development (JungleHorse) + .allowed_origin("http://api.jungle.horse") + .allowed_origin("http://jungle.horse") + .allowed_origin("http://jungle.horse:12345") + .allowed_origin("http://jungle.horse:7000") + .allowed_origin("http://obs.jungle.horse") + .allowed_origin("http://ws.jungle.horse") .allowed_origin("https://api.jungle.horse") .allowed_origin("https://jungle.horse") + .allowed_origin("https://obs.jungle.horse") + .allowed_origin("https://ws.jungle.horse") // FakeYou .allowed_origin("http://api.fakeyou.com") .allowed_origin("http://fakeyou.com") @@ -35,9 +40,13 @@ pub fn build_common_cors_config() -> Cors { .allowed_origin("https://fakeyou.com") // Storyteller .allowed_origin("http://api.storyteller.io") + .allowed_origin("http://obs.storyteller.io") .allowed_origin("http://storyteller.io") + .allowed_origin("http://ws.storyteller.io") .allowed_origin("https://api.storyteller.io") + .allowed_origin("https://obs.storyteller.io") .allowed_origin("https://storyteller.io") + .allowed_origin("https://ws.storyteller.io") // Trumped (legacy) .allowed_origin("http://trumped.com") .allowed_origin("https://trumped.com") diff --git a/crates/service/websocket_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs b/crates/service/websocket_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs index 1187208e6b..7df0547faf 100644 --- a/crates/service/websocket_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs +++ b/crates/service/websocket_gateway/src/endpoints_ws/obs_gateway_websocket_handler.rs @@ -115,8 +115,6 @@ pub async fn obs_gateway_websocket_handler( CommonServerError::ServerError })?; - info!("Begin Javascript WebSocket..."); - let server_state_arc = server_state.get_ref().clone(); let websocket = ObsGatewayWebSocket::new( @@ -124,9 +122,11 @@ pub async fn obs_gateway_websocket_handler( server_state_arc ); + info!("Begin Javascript WebSocket..."); + ws::start(websocket, &http_request, stream) .map_err(|e| { - warn!("Websocket error: {}", e); + warn!("Websocket ws::start() error: {}", e); CommonServerError::ServerError }) } diff --git a/localdev/nginx-http-config/storyteller-ssl b/localdev/nginx-http-config/storyteller-ssl index 305753808b..68fb8d9b24 100644 --- a/localdev/nginx-http-config/storyteller-ssl +++ b/localdev/nginx-http-config/storyteller-ssl @@ -89,6 +89,21 @@ server { } } +# Websocket API backend over SSL +server { + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/ssl/certs/localhost.crt; + ssl_certificate_key /etc/ssl/private/localhost.key; + ssl_protocols TLSv1.2 TLSv1.1 TLSv1; + server_name ws.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:54321; + proxy_redirect off; + } +} + # Javascript frontend over plain HTTP server { listen 80; @@ -107,7 +122,7 @@ server { server_name obs.jungle.horse; location / { proxy_set_header Host $host; - proxy_pass http://127.0.0.1:7001; + proxy_pass http://127.0.0.1:7007; proxy_redirect off; } } @@ -126,3 +141,18 @@ server { proxy_redirect off; } } + +# Javascript frontend over SSL (OBS Layer) +server { + listen 443 ssl; + listen [::]:443 ssl; + ssl_certificate /etc/ssl/certs/localhost.crt; + ssl_certificate_key /etc/ssl/private/localhost.key; + ssl_protocols TLSv1.2 TLSv1.1 TLSv1; + server_name obs.jungle.horse; + location / { + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:7007; + proxy_redirect off; + } +} From 283a851a179d54b4b9fd1cfd1b571cbde717d8c8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 27 Feb 2022 08:06:39 -0500 Subject: [PATCH 0996/1042] COOOORRRRS --- crates/lib/http_server_common/src/cors.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/lib/http_server_common/src/cors.rs b/crates/lib/http_server_common/src/cors.rs index 19aafcef3c..f072f8857f 100644 --- a/crates/lib/http_server_common/src/cors.rs +++ b/crates/lib/http_server_common/src/cors.rs @@ -40,10 +40,12 @@ pub fn build_common_cors_config() -> Cors { .allowed_origin("https://fakeyou.com") // Storyteller .allowed_origin("http://api.storyteller.io") + .allowed_origin("http://create.storyteller.io") .allowed_origin("http://obs.storyteller.io") .allowed_origin("http://storyteller.io") .allowed_origin("http://ws.storyteller.io") .allowed_origin("https://api.storyteller.io") + .allowed_origin("https://create.storyteller.io") .allowed_origin("https://obs.storyteller.io") .allowed_origin("https://storyteller.io") .allowed_origin("https://ws.storyteller.io") From e571c36f787ee0714d3975443e6c95c637802752 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 27 Feb 2022 08:29:54 -0500 Subject: [PATCH 0997/1042] logging --- .../event_handlers/bits_event_handler.rs | 2 ++ .../event_handlers/channel_points_event_handler.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs index f0736ab791..45619e9dae 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs @@ -50,6 +50,7 @@ impl BitsEventHandler { async fn handle_bits_event(&self, data: &BitsEventData) -> AnyhowResult<()> { let maybe_rule = self.find_matching_rule(data)?; if let Some(rule) = maybe_rule { + info!("Bits Rule matched: {}", &rule.token); self.handle_matched_rule(&rule, data).await?; self.report_event_for_analytics(&data).await?; // Report event for analytics } @@ -76,6 +77,7 @@ impl BitsEventHandler { return match self.twitch_subscriber_state.read() { Err(e) => { Err(anyhow!("Lock error: {:?}", e)) }, Ok(state) => { + info!("Checking bits event against {} rules...", state.event_rules.len()); let maybe_rule = state.event_rules.iter() .filter(|rule| rule.event_category.eq(&TwitchEventCategory::Bits)) .find(|rule| { diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs index 414ca3c70a..a15c6d86c9 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs @@ -54,6 +54,7 @@ impl ChannelPointsEventHandler { async fn handle_reward_redeemed_event(&self, redemption: &Redemption) -> AnyhowResult<()> { let maybe_rule = self.find_matching_rule(redemption)?; if let Some(rule) = maybe_rule { + info!("Channel Points rule matched: {}", &rule.token); self.handle_matched_rule(&rule, redemption).await?; self.report_event_for_analytics(redemption).await?; // Report event for analytics } @@ -90,6 +91,7 @@ impl ChannelPointsEventHandler { return match self.twitch_subscriber_state.read() { Err(e) => { Err(anyhow!("Lock error: {:?}", e)) }, Ok(state) => { + info!("Checking channel points event against {} rules...", state.event_rules.len()); let maybe_rule = state.event_rules.iter() .filter(|rule| rule.event_category.eq(&TwitchEventCategory::ChannelPoints)) .find(|rule| { From 26bbbc412b53d659f80856533d985402bf730146 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Mar 2022 04:06:33 -0500 Subject: [PATCH 0998/1042] migrations for new columns: langauge, tts command --- .../down.sql | 9 +++++++++ .../up.sql | 13 +++++++++++++ .../down.sql | 6 ++++++ .../up.sql | 14 ++++++++++++++ _migrations_squashed/tts_tables/tts_models.sql | 11 +++++++++++ 5 files changed, 53 insertions(+) create mode 100644 _migrations/2022-03-01-085243_alter_table_tts_models_add_language/down.sql create mode 100644 _migrations/2022-03-01-085243_alter_table_tts_models_add_language/up.sql create mode 100644 _migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/down.sql create mode 100644 _migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/up.sql diff --git a/_migrations/2022-03-01-085243_alter_table_tts_models_add_language/down.sql b/_migrations/2022-03-01-085243_alter_table_tts_models_add_language/down.sql new file mode 100644 index 0000000000..7c3a1d738d --- /dev/null +++ b/_migrations/2022-03-01-085243_alter_table_tts_models_add_language/down.sql @@ -0,0 +1,9 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_models + DROP COLUMN ietf_language_tag; + +ALTER TABLE tts_models + DROP COLUMN ietf_primary_language_subtag; diff --git a/_migrations/2022-03-01-085243_alter_table_tts_models_add_language/up.sql b/_migrations/2022-03-01-085243_alter_table_tts_models_add_language/up.sql new file mode 100644 index 0000000000..7665828672 --- /dev/null +++ b/_migrations/2022-03-01-085243_alter_table_tts_models_add_language/up.sql @@ -0,0 +1,13 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- The IETF BCP47 language tag (eg. en-US, es-419, ja-JP, etc.) +ALTER TABLE tts_models + ADD COLUMN ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en-US' + AFTER maybe_default_pretrained_vocoder; + +-- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es") +ALTER TABLE tts_models + ADD COLUMN ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en' + AFTER ietf_language_tag; diff --git a/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/down.sql b/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/down.sql new file mode 100644 index 0000000000..30f0bd8a8e --- /dev/null +++ b/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/down.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE tts_models + DROP COLUMN maybe_unique_bot_command_prefix; diff --git a/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/up.sql b/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/up.sql new file mode 100644 index 0000000000..ad3a9af70b --- /dev/null +++ b/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/up.sql @@ -0,0 +1,14 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- A bot command prefix, eg. 'sonic' that could be interpreted as '/sonic' by a TTS system. +-- These are unique strings and can only be applied to one model at a time. +-- Only moderators can set this, and it's only for the best voices. +ALTER TABLE tts_models + ADD COLUMN maybe_unique_bot_command_prefix VARCHAR(255) DEFAULT NULL + AFTER is_twitch_featured; + +-- And the unique index. +ALTER TABLE tts_models + ADD UNIQUE INDEX unique_index_maybe_unique_bot_command_prefix (maybe_unique_bot_command_prefix); diff --git a/_migrations_squashed/tts_tables/tts_models.sql b/_migrations_squashed/tts_tables/tts_models.sql index 71c6924e91..e89f68defb 100644 --- a/_migrations_squashed/tts_tables/tts_models.sql +++ b/_migrations_squashed/tts_tables/tts_models.sql @@ -61,6 +61,12 @@ CREATE TABLE tts_models ( -- ========== UNSTRUCTURED METADATA ========== + -- The IETF BCP47 language tag (eg. en-US, es-419, ja-JP, etc.) + ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en-US', + + -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es") + ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en', + -- The "name" of the voice model, which might be complicated. -- If maybe_subject_token (etc.) is set, then it's authoritative instead. title VARCHAR(255) NOT NULL, @@ -87,6 +93,11 @@ CREATE TABLE tts_models ( is_front_page_featured BOOLEAN NOT NULL DEFAULT FALSE, is_twitch_featured BOOLEAN NOT NULL DEFAULT FALSE, + -- A bot command prefix, eg. 'sonic' that could be interpreted as '/sonic' by a TTS system. + -- These are unique strings and can only be applied to one model at a time. + -- Only moderators can set this, and it's only for the best voices. + maybe_unique_bot_command_prefix VARCHAR(255) DEFAULT NULL, + -- ========== IMMUTABLE PROVENANCE AND METADATA ========== -- Where the file was originally downloaded (if it was downloaded) From 41d5526c5057a21a24c269bc3c7b7c6636237807 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Mar 2022 04:26:57 -0500 Subject: [PATCH 0999/1042] return new fields in list --- .../down.sql | 2 +- .../up.sql | 4 +- .../tts_tables/tts_models.sql | 8 ++-- .../queries/tts/tts_models/list_tts_models.rs | 26 ++++++++++++ .../endpoints/tts/list_tts_models.rs | 40 ++++++++----------- 5 files changed, 50 insertions(+), 30 deletions(-) diff --git a/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/down.sql b/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/down.sql index 30f0bd8a8e..5b97b7b587 100644 --- a/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/down.sql +++ b/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/down.sql @@ -3,4 +3,4 @@ -- noinspection SqlResolveForFile ALTER TABLE tts_models - DROP COLUMN maybe_unique_bot_command_prefix; + DROP COLUMN maybe_suggested_unique_bot_command; diff --git a/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/up.sql b/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/up.sql index ad3a9af70b..e617b9b0ed 100644 --- a/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/up.sql +++ b/_migrations/2022-03-01-085254_alter_table_tts_models_add_bot_command/up.sql @@ -6,9 +6,9 @@ -- These are unique strings and can only be applied to one model at a time. -- Only moderators can set this, and it's only for the best voices. ALTER TABLE tts_models - ADD COLUMN maybe_unique_bot_command_prefix VARCHAR(255) DEFAULT NULL + ADD COLUMN maybe_suggested_unique_bot_command VARCHAR(255) DEFAULT NULL AFTER is_twitch_featured; -- And the unique index. ALTER TABLE tts_models - ADD UNIQUE INDEX unique_index_maybe_unique_bot_command_prefix (maybe_unique_bot_command_prefix); + ADD UNIQUE INDEX unique_index_maybe_suggested_unique_bot_command (maybe_suggested_unique_bot_command); diff --git a/_migrations_squashed/tts_tables/tts_models.sql b/_migrations_squashed/tts_tables/tts_models.sql index e89f68defb..b05d61f81c 100644 --- a/_migrations_squashed/tts_tables/tts_models.sql +++ b/_migrations_squashed/tts_tables/tts_models.sql @@ -93,10 +93,10 @@ CREATE TABLE tts_models ( is_front_page_featured BOOLEAN NOT NULL DEFAULT FALSE, is_twitch_featured BOOLEAN NOT NULL DEFAULT FALSE, - -- A bot command prefix, eg. 'sonic' that could be interpreted as '/sonic' by a TTS system. - -- These are unique strings and can only be applied to one model at a time. - -- Only moderators can set this, and it's only for the best voices. - maybe_unique_bot_command_prefix VARCHAR(255) DEFAULT NULL, + -- A suggested bot command prefix, eg. 'sonic' that could be interpreted as '/sonic' + -- by a TTS system. These are unique strings that can only be applied to one model at + -- a time at most. Only moderators can set this, and it's only for the best voices. + maybe_suggested_unique_bot_command VARCHAR(255) DEFAULT NULL, -- ========== IMMUTABLE PROVENANCE AND METADATA ========== diff --git a/crates/lib/database_queries/src/queries/tts/tts_models/list_tts_models.rs b/crates/lib/database_queries/src/queries/tts/tts_models/list_tts_models.rs index 9d5071faf2..9267cf25b0 100644 --- a/crates/lib/database_queries/src/queries/tts/tts_models/list_tts_models.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_models/list_tts_models.rs @@ -7,6 +7,7 @@ use sqlx::MySqlPool; // FIXME: This is the old style of query scoping and shouldn't be copied. +// TODO: This record type is publicly returned in some endpoints! #[derive(Serialize)] pub struct TtsModelRecordForList { pub model_token: String, @@ -14,6 +15,9 @@ pub struct TtsModelRecordForList { pub title: String, + pub ietf_language_tag: String, + pub ietf_primary_language_subtag: String, + pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, @@ -24,6 +28,8 @@ pub struct TtsModelRecordForList { pub is_front_page_featured: bool, pub is_twitch_featured: bool, + pub maybe_suggested_unique_bot_command: Option, + pub created_at: DateTime, pub updated_at: DateTime, } @@ -72,9 +78,12 @@ pub async fn list_tts_models( creator_display_name: model.creator_display_name, creator_gravatar_hash: model.creator_gravatar_hash, title: model.title, + ietf_language_tag: model.ietf_language_tag, + ietf_primary_language_subtag: model.ietf_primary_language_subtag, is_locked_from_use: i8_to_bool(model.is_locked_from_use), is_front_page_featured: i8_to_bool(model.is_front_page_featured), is_twitch_featured: i8_to_bool(model.is_twitch_featured), + maybe_suggested_unique_bot_command: model.maybe_suggested_unique_bot_command, created_at: model.created_at, updated_at: model.updated_at, } @@ -101,9 +110,12 @@ SELECT users.display_name as creator_display_name, users.email_gravatar_hash as creator_gravatar_hash, tts.title, + tts.ietf_language_tag, + tts.ietf_primary_language_subtag, tts.is_locked_from_use, tts.is_front_page_featured, tts.is_twitch_featured, + tts.maybe_suggested_unique_bot_command, tts.created_at, tts.updated_at FROM tts_models as tts @@ -129,9 +141,12 @@ SELECT users.display_name as creator_display_name, users.email_gravatar_hash as creator_gravatar_hash, tts.title, + tts.ietf_language_tag, + tts.ietf_primary_language_subtag, tts.is_locked_from_use, tts.is_front_page_featured, tts.is_twitch_featured, + tts.maybe_suggested_unique_bot_command, tts.created_at, tts.updated_at FROM tts_models as tts @@ -168,9 +183,12 @@ SELECT users.display_name as creator_display_name, users.email_gravatar_hash as creator_gravatar_hash, tts.title, + tts.ietf_language_tag, + tts.ietf_primary_language_subtag, tts.is_locked_from_use, tts.is_front_page_featured, tts.is_twitch_featured, + tts.maybe_suggested_unique_bot_command, tts.created_at, tts.updated_at FROM tts_models as tts @@ -199,9 +217,12 @@ SELECT users.display_name as creator_display_name, users.email_gravatar_hash as creator_gravatar_hash, tts.title, + tts.ietf_language_tag, + tts.ietf_primary_language_subtag, tts.is_locked_from_use, tts.is_front_page_featured, tts.is_twitch_featured, + tts.maybe_suggested_unique_bot_command, tts.created_at, tts.updated_at FROM tts_models as tts @@ -227,6 +248,9 @@ struct InternalRawTtsModelRecordForList { pub title: String, + pub ietf_language_tag: String, + pub ietf_primary_language_subtag: String, + pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, @@ -237,6 +261,8 @@ struct InternalRawTtsModelRecordForList { pub is_front_page_featured: i8, // bool pub is_twitch_featured: i8, // bool + pub maybe_suggested_unique_bot_command: Option, + pub created_at: DateTime, pub updated_at: DateTime, } diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs index f56b113799..67a8cdc3f5 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs @@ -31,8 +31,11 @@ pub struct TtsModelRecordForResponse { pub creator_display_name: String, pub creator_gravatar_hash: String, pub title: String, + pub ietf_language_tag: String, + pub ietf_primary_language_subtag: String, pub is_front_page_featured: bool, pub is_twitch_featured: bool, + pub maybe_suggested_unique_bot_command: Option, /// Category assignments /// From non-deleted, mod-approved categories only @@ -53,19 +56,6 @@ pub enum ListTtsModelsError { ServerError, } -#[derive(Serialize)] -pub struct TtsModelRecord { - pub model_token: String, - pub tts_model_type: String, - pub creator_user_token: String, - pub creator_username: String, - pub creator_display_name: String, - pub creator_gravatar_hash: String, - pub title: String, - pub created_at: DateTime, - pub updated_at: DateTime, -} - impl ResponseError for ListTtsModelsError { fn status_code(&self) -> StatusCode { match *self { @@ -151,21 +141,25 @@ async fn get_all_models(mysql_pool: &MySqlPool) -> AnyhowResult>(); From a976d3e84d9be0ce7eecaf1f9faf43eb926f3341 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Mar 2022 04:32:13 -0500 Subject: [PATCH 1000/1042] add to another endpoint (get) --- .../queries/tts/tts_models/get_tts_model.rs | 24 +++++++++++++++++++ .../queries/tts/tts_models/list_tts_models.rs | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model.rs b/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model.rs index 80f7e2beba..9af4051cf3 100644 --- a/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model.rs @@ -10,6 +10,7 @@ use sqlx::MySqlPool; // FIXME: This is the old style of query scoping and shouldn't be copied. // The moderator-only fields are good practice, though. +// TODO/FIXME : This struct is returned publicly in some endpoints! #[derive(Serialize)] pub struct TtsModelRecordForResponse { pub model_token: String, @@ -26,9 +27,14 @@ pub struct TtsModelRecordForResponse { pub description_markdown: String, pub description_rendered_html: String, + pub ietf_language_tag: String, + pub ietf_primary_language_subtag: String, + pub is_front_page_featured: bool, pub is_twitch_featured: bool, + pub maybe_suggested_unique_bot_command: Option, + pub creator_set_visibility: RecordVisibility, pub is_locked_from_use: bool, @@ -100,8 +106,11 @@ pub async fn get_tts_model_by_token( description_markdown: model.description_markdown, description_rendered_html: model.description_rendered_html, // NB: Fail open/public with creator_set_visibility since we're already looking at it + ietf_language_tag: model.ietf_language_tag, + ietf_primary_language_subtag: model.ietf_primary_language_subtag, is_front_page_featured: i8_to_bool(model.is_front_page_featured), is_twitch_featured: i8_to_bool(model.is_twitch_featured), + maybe_suggested_unique_bot_command: model.maybe_suggested_unique_bot_command, creator_set_visibility: RecordVisibility::from_str(&model.creator_set_visibility) .unwrap_or(RecordVisibility::Public), is_locked_from_use: i8_to_bool(model.is_locked_from_use), @@ -143,9 +152,14 @@ SELECT tts.description_markdown, tts.description_rendered_html, + tts.ietf_language_tag, + tts.ietf_primary_language_subtag, + tts.is_front_page_featured, tts.is_twitch_featured, + tts.maybe_suggested_unique_bot_command, + tts.creator_set_visibility, tts.is_locked_from_use, @@ -193,8 +207,13 @@ SELECT tts.description_markdown, tts.description_rendered_html, + tts.ietf_language_tag, + tts.ietf_primary_language_subtag, + tts.is_front_page_featured, tts.is_twitch_featured, + + tts.maybe_suggested_unique_bot_command, tts.creator_set_visibility, @@ -240,9 +259,14 @@ struct InternalTtsModelRecordRaw { pub description_markdown: String, pub description_rendered_html: String, + pub ietf_language_tag: String, + pub ietf_primary_language_subtag: String, + pub is_front_page_featured: i8, pub is_twitch_featured: i8, + pub maybe_suggested_unique_bot_command: Option, + pub creator_set_visibility: String, pub is_locked_from_use: i8, diff --git a/crates/lib/database_queries/src/queries/tts/tts_models/list_tts_models.rs b/crates/lib/database_queries/src/queries/tts/tts_models/list_tts_models.rs index 9267cf25b0..c56ddd4427 100644 --- a/crates/lib/database_queries/src/queries/tts/tts_models/list_tts_models.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_models/list_tts_models.rs @@ -7,7 +7,7 @@ use sqlx::MySqlPool; // FIXME: This is the old style of query scoping and shouldn't be copied. -// TODO: This record type is publicly returned in some endpoints! +// TODO/FIXME : This struct is returned publicly in some endpoints! #[derive(Serialize)] pub struct TtsModelRecordForList { pub model_token: String, From 252f0e1134399de937a7c322809108861665e3cc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Mar 2022 05:13:27 -0500 Subject: [PATCH 1001/1042] supported language tools --- Cargo.lock | 1 + crates/lib/container_common/Cargo.toml | 1 + crates/lib/container_common/src/i18n/mod.rs | 1 + .../src/i18n/supported_languages.rs | 106 ++++++++++++++++++ crates/lib/container_common/src/lib.rs | 1 + 5 files changed, 110 insertions(+) create mode 100644 crates/lib/container_common/src/i18n/mod.rs create mode 100644 crates/lib/container_common/src/i18n/supported_languages.rs diff --git a/Cargo.lock b/Cargo.lock index 6b549d6abb..a4b25c9c2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1042,6 +1042,7 @@ name = "container_common" version = "0.0.1" dependencies = [ "anyhow", + "once_cell", "rand 0.8.4", "uuid", ] diff --git a/crates/lib/container_common/Cargo.toml b/crates/lib/container_common/Cargo.toml index 4c3a55870e..6976629abf 100644 --- a/crates/lib/container_common/Cargo.toml +++ b/crates/lib/container_common/Cargo.toml @@ -16,5 +16,6 @@ path = "src/lib.rs" # External anyhow = "1.0.40" +once_cell = "1.9.0" rand = "0.8.3" uuid = { version = "0.8.2", features = ["v4"] } diff --git a/crates/lib/container_common/src/i18n/mod.rs b/crates/lib/container_common/src/i18n/mod.rs new file mode 100644 index 0000000000..51ee176450 --- /dev/null +++ b/crates/lib/container_common/src/i18n/mod.rs @@ -0,0 +1 @@ +pub mod supported_languages; \ No newline at end of file diff --git a/crates/lib/container_common/src/i18n/supported_languages.rs b/crates/lib/container_common/src/i18n/supported_languages.rs new file mode 100644 index 0000000000..c6aeb82c6b --- /dev/null +++ b/crates/lib/container_common/src/i18n/supported_languages.rs @@ -0,0 +1,106 @@ +use once_cell::sync::Lazy; +use std::collections::{HashSet, HashMap}; + +/// These are language tags we try to support. +static SUPPORTED_LANGUAGES_FOR_MODELS : Lazy> = Lazy::new(|| { + let language_tags = [ + // English + "en", + "en-AU", + "en-CA", + "en-GB", + "en-US", + // Spanish + "es", + "es-419", + "es-AR", + "es-CL", + "es-CO", + "es-ES", + "es-MX", + "es-PE", + "es-US", + // Portuguese + "pt", + "pt-BR", + // French + "fr", + "fr-FR", + // German + "de", + "de-DE", + // Japanese + "ja", + "ja-JP", + // Misc + "id", + "id-ID", + "it", + "it-IT", + "ru", + "ru-RU", + "th-TH", + "tr", + "tr-TR", + "zh-CN", + "zh-HK", + ]; + language_tags.iter() + .map(|tag| tag.to_string()) + .collect::>() +}); + +/// Convert lower case tags to canonical form +static SUPPORTED_LANGUAGES_FOR_MODELS_CANONICAL_MAP : Lazy> = Lazy::new(|| { + SUPPORTED_LANGUAGES_FOR_MODELS.iter() + .map(|tag| (tag.to_lowercase(), tag.clone())) + .collect::>() +}); + +/// Take a full IETF BCP 47 language tag and return the canonicalized form *IF* we support it +pub fn get_canonicalized_language_tag_for_model(language_tag: &str) -> Option<&'static str> { + let lowercase = language_tag.to_lowercase(); + SUPPORTED_LANGUAGES_FOR_MODELS_CANONICAL_MAP.get(&lowercase).map(|v| v.as_str()) +} + +/// Take a full IETF BCP 47 language tag and return true *IF* we support it +/// It must be the correct case. +pub fn is_valid_language_for_models(language_tag: &str) -> bool { + SUPPORTED_LANGUAGES_FOR_MODELS.contains(language_tag) +} + +#[cfg(test)] +mod tests { + use crate::i18n::supported_languages::{get_canonicalized_language_tag_for_model, is_valid_language_for_models}; + + #[test] + fn get_canonical_language_for_model_success() { + assert_eq!("en", get_canonicalized_language_tag_for_model("EN").unwrap()); + assert_eq!("es-419", get_canonicalized_language_tag_for_model("ES-419").unwrap()); + assert_eq!("en-US", get_canonicalized_language_tag_for_model("En-Us").unwrap()); + } + + #[test] + fn get_canonical_language_for_model_failure() { + assert!(get_canonicalized_language_tag_for_model("").is_none()); + assert!(get_canonicalized_language_tag_for_model("en-JP").is_none()); + assert!(get_canonicalized_language_tag_for_model("es-Lots-Of-Tags").is_none()); + } + + #[test] + fn test_valid_language_for_model() { + // Valid + assert!(is_valid_language_for_models("en")); + assert!(is_valid_language_for_models("es-419")); + assert!(is_valid_language_for_models("ja-JP")); + // Invalid due to case + assert!(!is_valid_language_for_models("EN")); + assert!(!is_valid_language_for_models("ES-419")); + assert!(!is_valid_language_for_models("JA-jp")); + // Invalid misc + assert!(!is_valid_language_for_models("")); + assert!(!is_valid_language_for_models("foo")); + assert!(!is_valid_language_for_models("en-FOO")); + } +} + diff --git a/crates/lib/container_common/src/lib.rs b/crates/lib/container_common/src/lib.rs index 485bc177c4..2e45d4f08b 100644 --- a/crates/lib/container_common/src/lib.rs +++ b/crates/lib/container_common/src/lib.rs @@ -1,5 +1,6 @@ pub mod anyhow_result; pub mod collections; +pub mod i18n; pub mod numerics; pub mod thread; pub mod token; \ No newline at end of file From 115aff8855b9f2bfc42eff94823c0f7cb21a424d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Mar 2022 05:47:03 -0500 Subject: [PATCH 1002/1042] edit language tags, tts command --- Cargo.lock | 1 + crates/lib/container_common/src/i18n/mod.rs | 2 +- ...s.rs => supported_languages_for_models.rs} | 8 +- .../queries/tts/tts_models/get_tts_model.rs | 2 +- crates/service/storyteller_web/Cargo.toml | 4 + .../endpoints/tts/edit_tts_model.rs | 78 ++++++++++++++++++- 6 files changed, 90 insertions(+), 5 deletions(-) rename crates/lib/container_common/src/i18n/{supported_languages.rs => supported_languages_for_models.rs} (91%) diff --git a/Cargo.lock b/Cargo.lock index a4b25c9c2c..cf28212e13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3720,6 +3720,7 @@ dependencies = [ "hyper 0.14.14", "infer", "jwt", + "language-tags 0.3.2", "lazy_static", "lexical-sort", "lfu", diff --git a/crates/lib/container_common/src/i18n/mod.rs b/crates/lib/container_common/src/i18n/mod.rs index 51ee176450..5530078424 100644 --- a/crates/lib/container_common/src/i18n/mod.rs +++ b/crates/lib/container_common/src/i18n/mod.rs @@ -1 +1 @@ -pub mod supported_languages; \ No newline at end of file +pub mod supported_languages_for_models; \ No newline at end of file diff --git a/crates/lib/container_common/src/i18n/supported_languages.rs b/crates/lib/container_common/src/i18n/supported_languages_for_models.rs similarity index 91% rename from crates/lib/container_common/src/i18n/supported_languages.rs rename to crates/lib/container_common/src/i18n/supported_languages_for_models.rs index c6aeb82c6b..6f7fd7a764 100644 --- a/crates/lib/container_common/src/i18n/supported_languages.rs +++ b/crates/lib/container_common/src/i18n/supported_languages_for_models.rs @@ -4,6 +4,9 @@ use std::collections::{HashSet, HashMap}; /// These are language tags we try to support. static SUPPORTED_LANGUAGES_FOR_MODELS : Lazy> = Lazy::new(|| { let language_tags = [ + + // ========== Technically, we only support these so far ========== + // English "en", "en-AU", @@ -23,6 +26,9 @@ static SUPPORTED_LANGUAGES_FOR_MODELS : Lazy> = Lazy::new(|| { // Portuguese "pt", "pt-BR", + + // ========== But these are on the horizon ========== + // French "fr", "fr-FR", @@ -71,7 +77,7 @@ pub fn is_valid_language_for_models(language_tag: &str) -> bool { #[cfg(test)] mod tests { - use crate::i18n::supported_languages::{get_canonicalized_language_tag_for_model, is_valid_language_for_models}; + use crate::i18n::supported_languages_for_models::{get_canonicalized_language_tag_for_model, is_valid_language_for_models}; #[test] fn get_canonical_language_for_model_success() { diff --git a/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model.rs b/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model.rs index 9af4051cf3..54b60c0033 100644 --- a/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model.rs @@ -212,7 +212,7 @@ SELECT tts.is_front_page_featured, tts.is_twitch_featured, - + tts.maybe_suggested_unique_bot_command, tts.creator_set_visibility, diff --git a/crates/service/storyteller_web/Cargo.toml b/crates/service/storyteller_web/Cargo.toml index a7c45e7975..5f5c91b8fa 100644 --- a/crates/service/storyteller_web/Cargo.toml +++ b/crates/service/storyteller_web/Cargo.toml @@ -74,6 +74,10 @@ uuid = { version = "0.8.2", features = ["v4"] } twitch_api2 = { version = "0.6.0-rc.2", features = ["helix", "reqwest_client", "pubsub", "chrono", "unsupported"] } twitch_oauth2 = "0.6.0-rc.2" +# Language tag parsing +# TODO: Move to language support crate +language-tags = "0.3.2" # Parsing single tags richly + # HTTP # TODO/NB: This is just for Twitch, but we might need it later. reqwest = { version = "0.11.4" } diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs index 3d96899698..28d2c215fa 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs @@ -6,6 +6,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use container_common::i18n::supported_languages_for_models::{is_valid_language_for_models, get_canonicalized_language_tag_for_model}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::response_success_helpers::simple_json_success; @@ -15,7 +16,8 @@ use crate::util::markdown_to_html::markdown_to_html; use database_queries::column_types::record_visibility::RecordVisibility; use database_queries::column_types::vocoder_type::VocoderType; use database_queries::queries::tts::tts_models::get_tts_model::get_tts_model_by_token; -use log::{info, warn, log}; +use language_tags::{LanguageTag, ParseError}; +use log::{info, warn, log, error}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; @@ -25,6 +27,9 @@ use std::fmt; use std::sync::Arc; use user_input_common::check_for_slurs::contains_slurs; +const DEFAULT_IETF_LANGUAGE_TAG : &'static str = "en-US"; +const DEFAULT_IETF_PRIMARY_LANGUAGE_SUBTAG : &'static str = "en"; + /// For the URL PathInfo #[derive(Deserialize)] pub struct EditTtsModelPathInfo { @@ -39,6 +44,10 @@ pub struct EditTtsModelRequest { pub description_markdown: Option, pub creator_set_visibility: Option, pub maybe_default_pretrained_vocoder: Option, + + // NB: We calculate 'ietf_primary_language_subtag' from this value. + pub ietf_language_tag: Option, + //pub updatable_slug: Option, //pub tts_model_type: Option, //pub text_preprocessing_algorithm: Option, @@ -51,10 +60,13 @@ pub struct EditTtsModelRequest { pub is_locked_from_use: Option, pub maybe_mod_comments: Option, - // ========== Moderator options (front page) ========== + // ========== Moderator options (front page, Discord, Twitch, etc.) ========== pub is_front_page_featured: Option, pub is_twitch_featured: Option, + + // NB: We take "empty string" to mean removal. + pub maybe_suggested_unique_bot_command: Option, } #[derive(Debug)] @@ -156,11 +168,17 @@ pub async fn edit_tts_model_handler( } } + // ============================================= // Author + Mod fields. // These fields must be present on all requests. + // ============================================= + let mut title = None; let mut description_markdown = None; let mut description_html = None; + let mut ietf_language_tag = None; + let mut ietf_primary_language_subtag = None; + let mut creator_set_visibility = RecordVisibility::Public; let mut maybe_default_pretrained_vocoder = model_record.maybe_default_pretrained_vocoder @@ -186,6 +204,35 @@ pub async fn edit_tts_model_handler( description_html = Some(html); } + if let Some(tag) = request.ietf_language_tag.as_deref() { + // eg. en, en-US, es-419, ja-JP, etc. + let maybe_full_canonical_tag = get_canonicalized_language_tag_for_model(tag); + + // eg. en, es, ja, etc. + let maybe_primary_language_subtag = maybe_full_canonical_tag + .map(|t| LanguageTag::parse(t) + .map(|language_tag| language_tag.primary_language().to_string()) + ) + .transpose() + .map_err(|e| { + error!("Error parsing language tag '{}': {:?}", tag, e); + EditTtsModelError::BadInput("bad locale string".to_string()) + })?; + + if let Some(full_tag) = maybe_full_canonical_tag { + if let Some(primary_subtag) = maybe_primary_language_subtag.as_deref() { + ietf_language_tag = Some(full_tag.to_string()); + ietf_primary_language_subtag = Some(primary_subtag.to_string()); + } + } + } + + let mut ietf_language_tag = ietf_language_tag + .unwrap_or(DEFAULT_IETF_LANGUAGE_TAG.to_string()); + + let mut ietf_primary_language_subtag = ietf_primary_language_subtag + .unwrap_or(DEFAULT_IETF_PRIMARY_LANGUAGE_SUBTAG.to_string()); + if let Some(visibility) = request.creator_set_visibility.as_deref() { creator_set_visibility = RecordVisibility::from_str(visibility) .map_err(|_| EditTtsModelError::BadInput("bad record visibility".to_string()))?; @@ -207,6 +254,8 @@ SET title = ?, description_markdown = ?, description_rendered_html = ?, + ietf_language_tag = ?, + ietf_primary_language_subtag = ?, creator_set_visibility = ?, creator_ip_address_last_update = ?, version = version + 1 @@ -217,6 +266,8 @@ LIMIT 1 &title, &description_markdown, &description_html, + &ietf_language_tag, + &ietf_primary_language_subtag, &creator_set_visibility.to_str(), &ip_address, &model_record.model_token, @@ -233,6 +284,8 @@ SET title = ?, description_markdown = ?, description_rendered_html = ?, + ietf_language_tag = ?, + ietf_primary_language_subtag = ?, creator_set_visibility = ?, maybe_mod_user_token = ?, version = version + 1 @@ -243,6 +296,8 @@ LIMIT 1 &title, &description_markdown, &description_html, + &ietf_language_tag, + &ietf_primary_language_subtag, &creator_set_visibility.to_str(), &user_session.user_token, &model_record.model_token, @@ -251,6 +306,10 @@ LIMIT 1 .await }; + // ============================================= + // Mod-only fields. + // ============================================= + // TODO: This is lazy and suboptimal af to UPDATE again. // The reason we're doing this is because `sqlx` only does static type checking of queries // with string literals. It does not support dynamic query building, thus the PREDICATES @@ -289,6 +348,19 @@ async fn update_mod_details( let is_front_page_featured = request.is_front_page_featured.unwrap_or(false); let is_twitch_featured = request.is_twitch_featured.unwrap_or(false); + // Commands must be non-empty, alphanumeric, and lowercase. + let mut maybe_suggested_unique_bot_command = None; + if let Some(command) = request.maybe_suggested_unique_bot_command.as_deref() { + // Clear empty commands to null. + let command = command.trim().to_lowercase(); + + let is_alphanumeric = command.chars().all(|c| c.is_ascii_alphanumeric()); + + if !command.is_empty() && is_alphanumeric { + maybe_suggested_unique_bot_command = Some(command); + } + } + let query_result = sqlx::query!( r#" UPDATE tts_models @@ -300,6 +372,7 @@ SET maybe_mod_user_token = ?, is_front_page_featured = ?, is_twitch_featured = ?, + maybe_suggested_unique_bot_command = ?, version = version + 1 WHERE token = ? LIMIT 1 @@ -311,6 +384,7 @@ LIMIT 1 mod_user_token, is_front_page_featured, is_twitch_featured, + maybe_suggested_unique_bot_command, model_token ) .execute(mysql_pool) From b049450d35fbaad81d6558321536fc8a87c9628f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Mar 2022 06:04:55 -0500 Subject: [PATCH 1003/1042] codegen --- sqlx-data.json | 1968 +++++++++++++++++++++++++++--------------------- 1 file changed, 1101 insertions(+), 867 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 08ca688f44..e34c04751f 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -347,6 +347,213 @@ "nullable": [] } }, + "047d52d9fe0d14843a5ade550e3fea114e1e369d330d2239f49194d34486c2de": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.ietf_language_tag,\n tts.ietf_primary_language_subtag,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.maybe_suggested_unique_bot_command,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "ietf_language_tag", + "type_info": { + "type": "VarString", + "flags": { + "bits": 129 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 8, + "name": "ietf_primary_language_subtag", + "type_info": { + "type": "VarString", + "flags": { + "bits": 129 + }, + "char_set": 224, + "max_size": 48 + } + }, + { + "ordinal": 9, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 11, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "maybe_suggested_unique_bot_command", + "type_info": { + "type": "VarString", + "flags": { + "bits": 132 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false + ] + } + }, "05c3ebf54cd8a5374d6f4bbe21b3c6027ce6314917af8acd0ed136d375d53342": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n users.username = ?\n AND w2l.user_deleted_at IS NULL\n AND w2l.mod_deleted_at IS NULL\n ", "describe": { @@ -1312,176 +1519,8 @@ ] } }, - "443757e80423a52b09f8a4fa79088f6809dad4c1125b1555ce43a382ea745a04": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "is_front_page_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_twitch_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 11, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, - "4732b7d8c2beeec7e7821fd0139e32a12ae94cb81de1ba6cd2fff2f506e00336": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", + "48dd7546f75e3c997d1b9c8f1ef88adcd5a2d37dae517aa7047f285c8c760c59": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.ietf_language_tag,\n tts.ietf_primary_language_subtag,\n\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n\n tts.maybe_suggested_unique_bot_command,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { @@ -1630,6 +1669,30 @@ }, { "ordinal": 12, + "name": "ietf_language_tag", + "type_info": { + "type": "VarString", + "flags": { + "bits": 129 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 13, + "name": "ietf_primary_language_subtag", + "type_info": { + "type": "VarString", + "flags": { + "bits": 129 + }, + "char_set": 224, + "max_size": 48 + } + }, + { + "ordinal": 14, "name": "is_front_page_featured", "type_info": { "type": "Tiny", @@ -1641,7 +1704,7 @@ } }, { - "ordinal": 13, + "ordinal": 15, "name": "is_twitch_featured", "type_info": { "type": "Tiny", @@ -1653,7 +1716,19 @@ } }, { - "ordinal": 14, + "ordinal": 16, + "name": "maybe_suggested_unique_bot_command", + "type_info": { + "type": "VarString", + "flags": { + "bits": 132 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 17, "name": "creator_set_visibility", "type_info": { "type": "String", @@ -1665,7 +1740,7 @@ } }, { - "ordinal": 15, + "ordinal": 18, "name": "is_locked_from_use", "type_info": { "type": "Tiny", @@ -1677,7 +1752,7 @@ } }, { - "ordinal": 16, + "ordinal": 19, "name": "is_locked_from_user_modification", "type_info": { "type": "Tiny", @@ -1689,7 +1764,7 @@ } }, { - "ordinal": 17, + "ordinal": 20, "name": "created_at", "type_info": { "type": "Timestamp", @@ -1701,7 +1776,7 @@ } }, { - "ordinal": 18, + "ordinal": 21, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -1713,7 +1788,7 @@ } }, { - "ordinal": 19, + "ordinal": 22, "name": "creator_ip_address_creation", "type_info": { "type": "VarString", @@ -1725,7 +1800,7 @@ } }, { - "ordinal": 20, + "ordinal": 23, "name": "creator_ip_address_last_update", "type_info": { "type": "VarString", @@ -1737,7 +1812,7 @@ } }, { - "ordinal": 21, + "ordinal": 24, "name": "mod_deleted_at", "type_info": { "type": "Timestamp", @@ -1749,7 +1824,7 @@ } }, { - "ordinal": 22, + "ordinal": 25, "name": "user_deleted_at", "type_info": { "type": "Timestamp", @@ -1781,6 +1856,9 @@ false, false, false, + true, + false, + false, false, false, false, @@ -2754,174 +2832,6 @@ ] } }, - "5a17891eec52d8304af8a54f6bb7629cb71d3603146f8b2c842f6e1047c31e91": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "is_front_page_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_twitch_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 11, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "5b8c906b09cc7068dcad9eb837b1146893b63c3ce7c8dbefea9129849448c3f6": { "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", "describe": { @@ -3149,174 +3059,6 @@ ] } }, - "5e65d510fdf83315c92cac1ac6acd8c878d7e3e5f050c022c64c246d0af260a2": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "is_front_page_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_twitch_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 11, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "5eacdf9c65dc13820077be2930cca7047aed07804ed2e7b2ff38e12a617b4858": { "query": "\nINSERT INTO twitch_channel_point_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n title = ?,\n prompt = ?,\n user_text_input = ?,\n redemption_id = ?,\n reward_id = ?,\n reward_cost = ?,\n is_sub_only = ?,\n max_per_stream = ?,\n max_per_user_per_stream = ?\n ", "describe": { @@ -4144,6 +3886,420 @@ ] } }, + "8ac132004408188cc55710185539ff6cd2011de3e0f80b4c91e2bf8b3212135e": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.ietf_language_tag,\n tts.ietf_primary_language_subtag,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.maybe_suggested_unique_bot_command,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "ietf_language_tag", + "type_info": { + "type": "VarString", + "flags": { + "bits": 129 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 8, + "name": "ietf_primary_language_subtag", + "type_info": { + "type": "VarString", + "flags": { + "bits": 129 + }, + "char_set": 224, + "max_size": 48 + } + }, + { + "ordinal": 9, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 11, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "maybe_suggested_unique_bot_command", + "type_info": { + "type": "VarString", + "flags": { + "bits": 132 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false + ] + } + }, + "8c49202950a8d6cf87bbc60517168fba4c9b0c52c71a25d766be49fedbd28827": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.ietf_language_tag,\n tts.ietf_primary_language_subtag,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.maybe_suggested_unique_bot_command,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "ietf_language_tag", + "type_info": { + "type": "VarString", + "flags": { + "bits": 129 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 8, + "name": "ietf_primary_language_subtag", + "type_info": { + "type": "VarString", + "flags": { + "bits": 129 + }, + "char_set": 224, + "max_size": 48 + } + }, + { + "ordinal": 9, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 11, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "maybe_suggested_unique_bot_command", + "type_info": { + "type": "VarString", + "flags": { + "bits": 132 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false + ] + } + }, "8c6a56bfa985324a0dd138c9ac9570d72195ee721d5e2d6c50c4266d1590ed20": { "query": "\nUPDATE twitch_event_rules\nSET\n event_match_predicate = ?,\n event_response = ?,\n rule_is_disabled = ?,\n ip_address_last_update = ?\nWHERE\n token = ?\n AND deleted_at IS NULL\nLIMIT 1\n ", "describe": { @@ -4154,6 +4310,356 @@ "nullable": [] } }, + "8d4fd8365f8f2843c8ade46b5bfd4add943f7dbdf02c582edf8d5f690ba608cd": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.ietf_language_tag,\n tts.ietf_primary_language_subtag,\n\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n\n tts.maybe_suggested_unique_bot_command,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE tts.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 3, + "name": "maybe_default_pretrained_vocoder", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 4, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "creator_is_banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 10, + "name": "description_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 11, + "name": "description_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 12, + "name": "ietf_language_tag", + "type_info": { + "type": "VarString", + "flags": { + "bits": 129 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 13, + "name": "ietf_primary_language_subtag", + "type_info": { + "type": "VarString", + "flags": { + "bits": 129 + }, + "char_set": 224, + "max_size": 48 + } + }, + { + "ordinal": 14, + "name": "is_front_page_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 15, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "maybe_suggested_unique_bot_command", + "type_info": { + "type": "VarString", + "flags": { + "bits": 132 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 17, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 18, + "name": "is_locked_from_use", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 19, + "name": "is_locked_from_user_modification", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 20, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 21, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 22, + "name": "creator_ip_address_creation", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 23, + "name": "creator_ip_address_last_update", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 24, + "name": "mod_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 25, + "name": "user_deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + false, + true, + true + ] + } + }, "8ff27fc5583db97f157f1b808f593a72c77dc5dcf5a633cd9c1162af6b97c011": { "query": "\nUPDATE w2l_templates\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -4611,8 +5117,8 @@ "nullable": [] } }, - "9e6aa0da19f7e428206c95498f748a30ab428892fb9b17547a7dc60470cc3c09": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n tts.maybe_default_pretrained_vocoder,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n users.is_banned as creator_is_banned,\n\n tts.title,\n tts.description_markdown,\n tts.description_rendered_html,\n\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n\n tts.creator_set_visibility,\n\n tts.is_locked_from_use,\n tts.is_locked_from_user_modification,\n\n tts.created_at,\n tts.updated_at,\n\n tts.creator_ip_address_creation,\n tts.creator_ip_address_last_update,\n tts.mod_deleted_at,\n tts.user_deleted_at\n\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.token = ?\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", + "9b63758edee6341cb417073775eea2782137c680c1827189d6f7a93c40d70e44": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.ietf_language_tag,\n tts.ietf_primary_language_subtag,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.maybe_suggested_unique_bot_command,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { "columns": [ { @@ -4641,30 +5147,6 @@ }, { "ordinal": 2, - "name": "text_preprocessing_algorithm", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 3, - "name": "maybe_default_pretrained_vocoder", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 4, "name": "creator_user_token", "type_info": { "type": "VarString", @@ -4676,7 +5158,7 @@ } }, { - "ordinal": 5, + "ordinal": 3, "name": "creator_username", "type_info": { "type": "VarString", @@ -4688,7 +5170,7 @@ } }, { - "ordinal": 6, + "ordinal": 4, "name": "creator_display_name", "type_info": { "type": "VarString", @@ -4700,7 +5182,7 @@ } }, { - "ordinal": 7, + "ordinal": 5, "name": "creator_gravatar_hash", "type_info": { "type": "String", @@ -4712,19 +5194,7 @@ } }, { - "ordinal": 8, - "name": "creator_is_banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, + "ordinal": 6, "name": "title", "type_info": { "type": "VarString", @@ -4736,67 +5206,31 @@ } }, { - "ordinal": 10, - "name": "description_markdown", + "ordinal": 7, + "name": "ietf_language_tag", "type_info": { - "type": "Blob", + "type": "VarString", "flags": { - "bits": 4241 + "bits": 129 }, "char_set": 224, - "max_size": 262140 + "max_size": 256 } }, { - "ordinal": 11, - "name": "description_rendered_html", + "ordinal": 8, + "name": "ietf_primary_language_subtag", "type_info": { - "type": "Blob", + "type": "VarString", "flags": { - "bits": 4241 + "bits": 129 }, "char_set": 224, - "max_size": 262140 + "max_size": 48 } }, { - "ordinal": 12, - "name": "is_front_page_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "is_twitch_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "creator_set_visibility", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 15, + "ordinal": 9, "name": "is_locked_from_use", "type_info": { "type": "Tiny", @@ -4808,19 +5242,43 @@ } }, { - "ordinal": 16, - "name": "is_locked_from_user_modification", + "ordinal": 10, + "name": "is_front_page_featured", "type_info": { "type": "Tiny", "flags": { - "bits": 9 + "bits": 1 }, "char_set": 63, "max_size": 1 } }, { - "ordinal": 17, + "ordinal": 11, + "name": "is_twitch_featured", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "maybe_suggested_unique_bot_command", + "type_info": { + "type": "VarString", + "flags": { + "bits": 132 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, "name": "created_at", "type_info": { "type": "Timestamp", @@ -4832,7 +5290,7 @@ } }, { - "ordinal": 18, + "ordinal": 14, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -4842,69 +5300,12 @@ "char_set": 63, "max_size": 19 } - }, - { - "ordinal": 19, - "name": "creator_ip_address_creation", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 20, - "name": "creator_ip_address_last_update", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 160 - } - }, - { - "ordinal": 21, - "name": "mod_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 22, - "name": "user_deleted_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 19 - } } ], "parameters": { - "Right": 1 + "Right": 0 }, "nullable": [ - false, - false, - false, - true, - false, - false, - false, - false, - false, false, false, false, @@ -4918,7 +5319,8 @@ false, false, true, - true + false, + false ] } }, @@ -5029,174 +5431,6 @@ "nullable": [] } }, - "b06e373275eb6d6f445abba38b8129d53196527bf953a50c85ccb93c23d2e4d6": { - "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "tts_model_type", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 60 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "creator_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "is_locked_from_use", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 9 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 8, - "name": "is_front_page_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "is_twitch_featured", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 11, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false - ] - } - }, "b483867c3f767a01cc80e4c0e0f1950c65154e06c13fc2fcc6e3ec96e8e2c956": { "query": "\nUPDATE api_tokens\nSET\n maybe_short_description = ?,\n ip_address_last_update = ?\nWHERE\n user_token = ?\n AND api_token = ?\nLIMIT 1\n ", "describe": { @@ -7352,6 +7586,16 @@ "nullable": [] } }, + "2a7837b626439198db1d9d63b22f6f6cac602d5e6b09de45f17b70272d1679d4": { + "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n ietf_language_tag = ?,\n ietf_primary_language_subtag = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, "2b04dd7db52ef3a97a6f420a22a4dbee7416ae58697438df69e87ffd47bbf350": { "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -7730,6 +7974,16 @@ "nullable": [] } }, + "553034e2300bb7cce54964b1cd93d6f4826eccef9b1aee3bc9abd69433ee0d5c": { + "query": "\nUPDATE tts_models\nSET\n is_public_listing_approved = ?,\n is_locked_from_user_modification = ?,\n is_locked_from_use = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n is_front_page_featured = ?,\n is_twitch_featured = ?,\n maybe_suggested_unique_bot_command = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, "56a0a2bfc3c73d6845ca9cf1d285b175a40b50a250b7effb9ce6474e58113d6e": { "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_model_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_model_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", "describe": { @@ -7962,6 +8216,16 @@ ] } }, + "5ccee09700cfc6d511451c9734d6dc1a8a0cc855ad45c3a73128fc3cf092c0a6": { + "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n ietf_language_tag = ?,\n ietf_primary_language_subtag = ?,\n creator_set_visibility = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, "60c06a507349919addb5d26b1c4a283a04f66fce86e84ed0a51db6001c9fc2b5": { "query": "\nUPDATE w2l_templates\nSET\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -8668,16 +8932,6 @@ "nullable": [] } }, - "9c04e83ce78dafd7b84edf057f30a834d459f710d0836dcf7f211f7cd17e24e3": { - "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n creator_ip_address_last_update = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 7 - }, - "nullable": [] - } - }, "9cc5f697350814199344d3d9a0ff026e0d1e09e9097648400d4b63cfcbd31178": { "query": "\nUPDATE w2l_results\nSET\n creator_set_visibility = ?,\n creator_ip_address = ?\nWHERE token = ?\nLIMIT 1\n ", "describe": { @@ -8921,16 +9175,6 @@ "nullable": [] } }, - "adc494ddafaa13aebe423984f32a0d8810da604b19f5d6e0081b5e891dfc0f52": { - "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 7 - }, - "nullable": [] - } - }, "ae04054c4d32e581dd183fd7ebb8e48fc7df856d2ffd8036d9ce6006bb7a296a": { "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_template_token,\n\n jobs.failure_reason as maybe_failure_reason,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_template_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", "describe": { @@ -9868,16 +10112,6 @@ ] } }, - "ef83a3b9cbc11a7e70637ada49b517e295a9a89ae273d3a77409de03fe483aac": { - "query": "\nUPDATE tts_models\nSET\n is_public_listing_approved = ?,\n is_locked_from_user_modification = ?,\n is_locked_from_use = ?,\n maybe_mod_comments = ?,\n maybe_mod_user_token = ?,\n is_front_page_featured = ?,\n is_twitch_featured = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 8 - }, - "nullable": [] - } - }, "f70bd775718829cd10f508dfe09286020467cc34fb54daf0758dba2545b164f8": { "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_confirmed,\n users.email_gravatar_hash,\n\n users.disable_gravatar,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n users.preferred_tts_result_visibility as `preferred_tts_result_visibility: database_queries::column_types::record_visibility::RecordVisibility`,\n users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: database_queries::column_types::record_visibility::RecordVisibility`,\n\n users.user_role_slug,\n users.is_banned,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l,\n user_roles.can_delete_own_tts_results,\n user_roles.can_delete_own_w2l_results,\n user_roles.can_delete_own_account,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n user_roles.can_delete_own_tts_models,\n user_roles.can_delete_own_w2l_templates,\n\n user_roles.can_approve_w2l_templates,\n user_roles.can_edit_other_users_profiles,\n user_roles.can_edit_other_users_tts_models,\n user_roles.can_edit_other_users_w2l_templates,\n user_roles.can_delete_other_users_tts_models,\n user_roles.can_delete_other_users_tts_results,\n user_roles.can_delete_other_users_w2l_templates,\n user_roles.can_delete_other_users_w2l_results,\n user_roles.can_ban_users,\n user_roles.can_delete_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\n ", "describe": { From e96b58afcab161ee344495743ec6dee6e797645f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Mar 2022 07:05:16 -0500 Subject: [PATCH 1004/1042] change default language from en-US to en --- .../up.sql | 4 ++-- _migrations_squashed/tts_tables/tts_models.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/_migrations/2022-03-01-085243_alter_table_tts_models_add_language/up.sql b/_migrations/2022-03-01-085243_alter_table_tts_models_add_language/up.sql index 7665828672..878a8a62c6 100644 --- a/_migrations/2022-03-01-085243_alter_table_tts_models_add_language/up.sql +++ b/_migrations/2022-03-01-085243_alter_table_tts_models_add_language/up.sql @@ -2,9 +2,9 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile --- The IETF BCP47 language tag (eg. en-US, es-419, ja-JP, etc.) +-- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.) ALTER TABLE tts_models - ADD COLUMN ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en-US' + ADD COLUMN ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en' AFTER maybe_default_pretrained_vocoder; -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es") diff --git a/_migrations_squashed/tts_tables/tts_models.sql b/_migrations_squashed/tts_tables/tts_models.sql index b05d61f81c..d31242ce0f 100644 --- a/_migrations_squashed/tts_tables/tts_models.sql +++ b/_migrations_squashed/tts_tables/tts_models.sql @@ -61,8 +61,8 @@ CREATE TABLE tts_models ( -- ========== UNSTRUCTURED METADATA ========== - -- The IETF BCP47 language tag (eg. en-US, es-419, ja-JP, etc.) - ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en-US', + -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.) + ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en', -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es") ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en', From 9dcc7665ff6dbcd597a12a9a7aca2a54e0399b25 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Mar 2022 05:17:06 -0500 Subject: [PATCH 1005/1042] Add internal token (dangerous, unsupported) --- .../down.sql | 6 ++++++ .../up.sql | 12 ++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 _migrations/2022-03-02-100137_alter_table_twitch_oauth_tokens_add_token_pk/down.sql create mode 100644 _migrations/2022-03-02-100137_alter_table_twitch_oauth_tokens_add_token_pk/up.sql diff --git a/_migrations/2022-03-02-100137_alter_table_twitch_oauth_tokens_add_token_pk/down.sql b/_migrations/2022-03-02-100137_alter_table_twitch_oauth_tokens_add_token_pk/down.sql new file mode 100644 index 0000000000..2768e2a632 --- /dev/null +++ b/_migrations/2022-03-02-100137_alter_table_twitch_oauth_tokens_add_token_pk/down.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE twitch_oauth_tokens + DROP COLUMN internal_token; diff --git a/_migrations/2022-03-02-100137_alter_table_twitch_oauth_tokens_add_token_pk/up.sql b/_migrations/2022-03-02-100137_alter_table_twitch_oauth_tokens_add_token_pk/up.sql new file mode 100644 index 0000000000..39a519c385 --- /dev/null +++ b/_migrations/2022-03-02-100137_alter_table_twitch_oauth_tokens_add_token_pk/up.sql @@ -0,0 +1,12 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- In case we ever want to query these via an admin interface, it'd be nice to not use IDs +ALTER TABLE twitch_oauth_tokens + ADD COLUMN internal_token VARCHAR(32) NOT NULL + AFTER id; + +-- And the unique index. +ALTER TABLE twitch_oauth_tokens + ADD UNIQUE INDEX unique_index_internal_token (internal_token); From 5dfe83fe0fa7ddd32a31b99fd513102ccba8b820 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Mar 2022 05:32:20 -0500 Subject: [PATCH 1006/1042] add oauth token internal pk --- .../src/queries/twitch/twitch_oauth/find.rs | 34 +++++++++++-------- .../src/queries/twitch/twitch_oauth/insert.rs | 8 +++++ crates/lib/database_queries/src/tokens.rs | 4 +++ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_oauth/find.rs b/crates/lib/database_queries/src/queries/twitch/twitch_oauth/find.rs index 453c674b23..09274990a9 100644 --- a/crates/lib/database_queries/src/queries/twitch/twitch_oauth/find.rs +++ b/crates/lib/database_queries/src/queries/twitch/twitch_oauth/find.rs @@ -11,6 +11,7 @@ use sqlx::MySqlPool; #[derive(Serialize, Clone)] pub struct TwitchOauthTokenRecord { /// Our internal token metadata / bookkeeping + pub internal_token: String, pub oauth_refresh_grouping_token: String, pub refresh_count: u32, pub ip_address_creation: Option, @@ -94,27 +95,28 @@ impl TwitchOauthTokenFinder { Ok(self.perform_query_internal(mysql_pool).await? .map(|record| { TwitchOauthTokenRecord { - oauth_refresh_grouping_token: record.oauth_refresh_grouping_token.clone(), - maybe_user_token: record.maybe_user_token.clone(), - maybe_user_display_name: record.maybe_user_display_name.clone(), - maybe_user_gravatar_hash: record.maybe_user_gravatar_hash.clone(), - twitch_user_id: record.twitch_user_id.clone(), - twitch_username: record.twitch_username.clone(), - twitch_username_lowercase: record.twitch_username_lowercase.clone(), - access_token: record.access_token.clone(), - maybe_refresh_token: record.maybe_refresh_token.clone(), - token_type: record.token_type.clone(), - expires_in_seconds: record.expires_in_seconds.clone(), + internal_token: record.internal_token, + oauth_refresh_grouping_token: record.oauth_refresh_grouping_token, + maybe_user_token: record.maybe_user_token, + maybe_user_display_name: record.maybe_user_display_name, + maybe_user_gravatar_hash: record.maybe_user_gravatar_hash, + twitch_user_id: record.twitch_user_id, + twitch_username: record.twitch_username, + twitch_username_lowercase: record.twitch_username_lowercase, + access_token: record.access_token, + maybe_refresh_token: record.maybe_refresh_token, + token_type: record.token_type, + expires_in_seconds: record.expires_in_seconds, refresh_count: record.refresh_count, has_bits_read: i8_to_bool(record.has_bits_read), has_channel_read_redemptions: i8_to_bool(record.has_channel_read_redemptions), has_channel_read_subscriptions: i8_to_bool(record.has_channel_read_subscriptions), has_chat_edit: i8_to_bool(record.has_chat_edit), has_chat_read: i8_to_bool(record.has_chat_read), - ip_address_creation: record.ip_address_creation.clone(), - expires_at: record.expires_at.clone(), - user_deleted_at: record.user_deleted_at.clone(), - mod_deleted_at: record.mod_deleted_at.clone(), + ip_address_creation: record.ip_address_creation, + expires_at: record.expires_at, + user_deleted_at: record.user_deleted_at, + mod_deleted_at: record.mod_deleted_at, } })) } @@ -162,6 +164,7 @@ impl TwitchOauthTokenFinder { // TODO/NB: Unfortunately SQLx can't statically typecheck this query let mut query = r#" SELECT + twitch_oauth_tokens.internal_token, twitch_oauth_tokens.oauth_refresh_grouping_token, users.username as maybe_username, users.display_name as maybe_user_display_name, @@ -253,6 +256,7 @@ LEFT OUTER JOIN users #[derive(sqlx::FromRow)] pub struct TwitchOauthTokenRecordInternal { /// Our internal token metadata / bookkeeping + pub internal_token: String, pub oauth_refresh_grouping_token: String, pub refresh_count: u32, pub ip_address_creation: Option, diff --git a/crates/lib/database_queries/src/queries/twitch/twitch_oauth/insert.rs b/crates/lib/database_queries/src/queries/twitch/twitch_oauth/insert.rs index 07d7bb3da7..3786075c5d 100644 --- a/crates/lib/database_queries/src/queries/twitch/twitch_oauth/insert.rs +++ b/crates/lib/database_queries/src/queries/twitch/twitch_oauth/insert.rs @@ -4,6 +4,7 @@ use container_common::anyhow_result::AnyhowResult; use log::info; use log::warn; use sqlx::MySqlPool; +use crate::tokens::Tokens; pub struct TwitchOauthTokenInsertBuilder { // ===== Required Fields ===== @@ -133,6 +134,9 @@ impl TwitchOauthTokenInsertBuilder { } pub async fn insert(&mut self, mysql_pool: &MySqlPool) -> AnyhowResult<()> { + // An internal token that we may use in the future. + let internal_token = Tokens::new_twitch_oauth_internal_token()?; + // NB: We have to duplicate these since the string literals must not // include concatenation. Boo. let query = if let Some(expires_in_seconds) = self.expires_in_seconds { @@ -140,6 +144,7 @@ impl TwitchOauthTokenInsertBuilder { r#" INSERT INTO twitch_oauth_tokens SET + internal_token = ?, oauth_refresh_grouping_token = ?, maybe_user_token = ?, twitch_user_id = ?, @@ -158,6 +163,7 @@ SET ip_address_creation = ?, expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND) "#, + internal_token.clone(), self.oauth_grouping_token.clone(), self.maybe_user_token.clone(), self.twitch_user_id.clone(), @@ -181,6 +187,7 @@ SET r#" INSERT INTO twitch_oauth_tokens SET + internal_token = ?, oauth_refresh_grouping_token = ?, maybe_user_token = ?, twitch_user_id = ?, @@ -199,6 +206,7 @@ SET ip_address_creation = ?, expires_at = NULL "#, + internal_token.clone(), self.oauth_grouping_token.clone(), self.maybe_user_token.clone(), self.twitch_user_id.clone(), diff --git a/crates/lib/database_queries/src/tokens.rs b/crates/lib/database_queries/src/tokens.rs index b99f06bae2..44df715317 100644 --- a/crates/lib/database_queries/src/tokens.rs +++ b/crates/lib/database_queries/src/tokens.rs @@ -73,6 +73,10 @@ impl Tokens { // ========== [Twitch] Tokens not typically visible to users ========== + pub fn new_twitch_oauth_internal_token() -> AnyhowResult { + random_prefix_crockford_token("TOI:", 32, false) + } + pub fn new_twitch_oauth_grouping_token() -> AnyhowResult { random_prefix_crockford_token("OG:", 32, false) } From 09cd602839cd4df1f05f26fbf81a889d3a28c6bc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Mar 2022 06:20:10 -0500 Subject: [PATCH 1007/1042] check oauth token existence --- .../twitch/oauth/check_oauth_status.rs | 98 +++++++++++++++++++ .../http_server/endpoints/twitch/oauth/mod.rs | 1 + crates/service/storyteller_web/src/routes.rs | 7 ++ 3 files changed, 106 insertions(+) create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/check_oauth_status.rs diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/check_oauth_status.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/check_oauth_status.rs new file mode 100644 index 0000000000..31dbf7451e --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/check_oauth_status.rs @@ -0,0 +1,98 @@ +use actix_http::{StatusCode, header}; +use actix_web::{HttpResponse, HttpRequest, web, ResponseError}; +use crate::server_state::ServerState; +use database_queries::queries::twitch::twitch_oauth::insert::TwitchOauthTokenInsertBuilder; +use http_server_common::request::get_request_ip::get_request_ip; +use http_server_common::response::response_error_helpers::to_simple_json_error; +use log::error; +use log::info; +use log::warn; +use std::fmt; +use std::sync::Arc; +use twitch_common::oauth_token_builder::get_oauth_token_builder; +use twitch_oauth2::tokens::BearerTokenType::UserToken; +use twitch_oauth2::{CsrfToken, TwitchToken}; +use database_queries::tokens::Tokens; +use database_queries::queries::twitch::twitch_oauth::find::{TwitchOauthTokenFinder, TwitchOauthTokenRecord}; + +#[derive(Serialize)] +pub struct CheckOauthResponse { + pub success: bool, + + /// This is false if the user isn't logged in, doesn't have oauth, or oauth is invalid + pub oauth_token_found: bool, +} + +#[derive(Debug)] +pub enum CheckOauthStatusError { + ServerError, +} + +impl ResponseError for CheckOauthStatusError { + fn status_code(&self) -> StatusCode { + match *self { + Self::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + Self::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for CheckOauthStatusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn check_oauth_status_handler( + http_request: HttpRequest, + server_state: web::Data> +) + -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CheckOauthStatusError::ServerError + })?; + + let oauth_token_found = match maybe_user_session { + None => false, // Not logged in - not found + Some(session) => { + let finder = TwitchOauthTokenFinder::new() + .allow_expired_tokens(true) + .scope_user_token(Some(&session.user_token)); + + let record = finder.perform_query(&server_state.mysql_pool).await + .map_err(|e| { + warn!("lookup error: {:?}", e); + CheckOauthStatusError::ServerError + })?; + + // Existence is sufficient. + record.is_some() + }, + }; + + let response = CheckOauthResponse { + success: true, + oauth_token_found, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| CheckOauthStatusError::ServerError)?; + + return Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)); +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/mod.rs index 0b47d3ca95..35e370f133 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/mod.rs @@ -1,3 +1,4 @@ +pub mod check_oauth_status; pub mod oauth_begin_json; pub mod oauth_begin_redirect; pub mod oauth_end; diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index 03bdf3fedb..fa1f9af86c 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -83,6 +83,7 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; +use crate::http_server::endpoints::twitch::oauth::check_oauth_status::check_oauth_status_handler; pub fn add_routes (app: App) -> App where @@ -632,6 +633,12 @@ fn add_twitch_routes (app: App) -> App .route(web::get().to(oauth_end_enroll_from_redirect)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::scope("/oauth") + .service(web::resource("/check") + .route(web::get().to(check_oauth_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) .service(web::scope("/event_rule") .service(web::resource("/create") .route(web::post().to(create_twitch_event_rule_handler)) From d2cc56940c5467d47e6bb97785c721cfdba8bc51 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Mar 2022 06:27:57 -0500 Subject: [PATCH 1008/1042] change oauth routes --- crates/service/storyteller_web/src/main.rs | 4 ++-- crates/service/storyteller_web/src/routes.rs | 24 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index c56880a6d4..d4257a5c06 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -268,11 +268,11 @@ async fn main() -> AnyhowResult<()> { let twitch_oauth_redirect_landing_url = easyenv::get_env_string_or_default( "TWITCH_OAUTH_REDIRECT_LANDING_URL", - "https://api.jungle.horse/twitch/oauth_landing"); + "https://api.jungle.horse/twitch/oauth/enroll_redirect_landing"); let twitch_oauth_redirect_landing_finished_url = easyenv::get_env_string_or_default( "TWITCH_OAUTH_REDIRECT_LANDING_FINISHED_URL", - "https://api.jungle.horse/"); + "https://jungle.horse/"); let twitch_secrets = TwitchSecrets::from_env()?; diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index fa1f9af86c..35c2721aa3 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -621,19 +621,19 @@ fn add_twitch_routes (app: App) -> App { // TODO: Move oauth endpoints under /twitch/oauth (requires updating Twitch configs too.) app.service(web::scope("/twitch") - .service(web::resource("/oauth_enroll_json") - .route(web::get().to(oauth_begin_enroll_json)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service(web::resource("/oauth_enroll_redirect") - .route(web::get().to(oauth_begin_enroll_redirect)) - .route(web::head().to(|| HttpResponse::Ok())) - ) - .service(web::resource("/oauth_landing") - .route(web::get().to(oauth_end_enroll_from_redirect)) - .route(web::head().to(|| HttpResponse::Ok())) - ) .service(web::scope("/oauth") + .service(web::resource("/enroll_json") + .route(web::get().to(oauth_begin_enroll_json)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/enroll_redirect_begin") + .route(web::get().to(oauth_begin_enroll_redirect)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/enroll_redirect_landing") + .route(web::get().to(oauth_end_enroll_from_redirect)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/check") .route(web::get().to(check_oauth_status_handler)) .route(web::head().to(|| HttpResponse::Ok())) From 9e55283b868a9e2a2967084eafa0e596768e5cae Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Mar 2022 06:29:39 -0500 Subject: [PATCH 1009/1042] codegendb --- sqlx-data.json | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index e34c04751f..82f28dfc58 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -2832,16 +2832,6 @@ ] } }, - "5b8c906b09cc7068dcad9eb837b1146893b63c3ce7c8dbefea9129849448c3f6": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 15 - }, - "nullable": [] - } - }, "5e4a00c339ce4d7fac19aff728449156b1e02b9e1d9fd534ac37d4ac79f44820": { "query": "\nINSERT INTO twitch_bits_events\nSET\n sender_twitch_user_id = ?,\n sender_twitch_username = ?,\n sender_twitch_username_lowercase = ?,\n destination_channel_id = ?,\n destination_channel_name = ?,\n is_anonymous= ?,\n bits_used = ?,\n total_bits_used = ?,\n chat_message = ?\n ", "describe": { @@ -3604,6 +3594,16 @@ ] } }, + "80e8ad515ab8fbfadc12463b87059a1eb6554be32527729d6075fc46c966630c": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n internal_token = ?,\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 18 + }, + "nullable": [] + } + }, "865ce236444a36d597939660b14eb4b0cee05c70fdb73f31c2fb8cddddfef42c": { "query": "\nINSERT INTO firehose_entries\nSET\n token = ?,\n event_type = ?,\n maybe_target_user_token = ?,\n maybe_target_entity_token = ?,\n maybe_created_entity_token = ?\n ", "describe": { @@ -5421,6 +5421,16 @@ "nullable": [] } }, + "aeeed0077e567e4b67ffa30e32f5475e168c6c68d2f206cc7b6c37168384089c": { + "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n internal_token = ?,\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = NULL,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 16 + }, + "nullable": [] + } + }, "af032ec861cb91f15ec763d9452e03c23797af15b5714565aa330f6cc420e8d6": { "query": "\nINSERT INTO api_tokens\nSET\n internal_token = ?,\n api_token = ?,\n user_token = ?,\n uuid_idempotency_token = ?,\n ip_address_creation = ?,\n ip_address_last_update = ?\n ", "describe": { @@ -6530,16 +6540,6 @@ "nullable": [] } }, - "d7ac9f84e92d6b2111162a674284c0b9fa50c718bedfb8ab7238ca0a7b0875e6": { - "query": "\nINSERT INTO twitch_oauth_tokens\nSET\n oauth_refresh_grouping_token = ?,\n maybe_user_token = ?,\n twitch_user_id = ?,\n twitch_username = ?,\n twitch_username_lowercase = ?,\n access_token = ?,\n maybe_refresh_token = ?,\n refresh_count = ?,\n token_type = ?,\n expires_in_seconds = ?,\n has_bits_read = ?,\n has_channel_read_redemptions= ?,\n has_channel_read_subscriptions = ?,\n has_chat_edit = ?,\n has_chat_read = ?,\n ip_address_creation = ?,\n expires_at = DATE_ADD(NOW(), INTERVAL ? SECOND)\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 17 - }, - "nullable": [] - } - }, "da3197d279b94817c6c8a6277aff7a0cf44d66e6777f46680b1b07de48875dd5": { "query": "\nINSERT INTO user_badges\nSET\n user_token = ?,\n badge_slug = ?\n ", "describe": { From 78efb11dbd44756b4d80e51d738ea8bb6d06dc72 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Mar 2022 06:54:43 -0500 Subject: [PATCH 1010/1042] return username in oauth check --- .../twitch/oauth/check_oauth_status.rs | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/check_oauth_status.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/check_oauth_status.rs index 31dbf7451e..60d1c24476 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/check_oauth_status.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/oauth/check_oauth_status.rs @@ -21,6 +21,10 @@ pub struct CheckOauthResponse { /// This is false if the user isn't logged in, doesn't have oauth, or oauth is invalid pub oauth_token_found: bool, + + /// Shouldn't be harmful to send this back since user must be logged in. + pub maybe_twitch_username: Option, + pub maybe_twitch_username_lowercase: Option, } #[derive(Debug)] @@ -66,27 +70,38 @@ pub async fn check_oauth_status_handler( CheckOauthStatusError::ServerError })?; - let oauth_token_found = match maybe_user_session { - None => false, // Not logged in - not found + let mut oauth_token_found = false; + let mut maybe_twitch_username = None; + let mut maybe_twitch_username_lowercase = None; + + match maybe_user_session { + None => {}, // Not logged in - not found Some(session) => { let finder = TwitchOauthTokenFinder::new() .allow_expired_tokens(true) .scope_user_token(Some(&session.user_token)); - let record = finder.perform_query(&server_state.mysql_pool).await + let maybe_record = finder.perform_query(&server_state.mysql_pool) + .await .map_err(|e| { warn!("lookup error: {:?}", e); CheckOauthStatusError::ServerError })?; // Existence is sufficient. - record.is_some() + if let Some(record) = maybe_record { + oauth_token_found = true; + maybe_twitch_username = Some(record.twitch_username.to_string()); + maybe_twitch_username_lowercase = Some(record.twitch_username_lowercase.to_string()); + } }, - }; + } let response = CheckOauthResponse { success: true, oauth_token_found, + maybe_twitch_username, + maybe_twitch_username_lowercase, }; let body = serde_json::to_string(&response) From f069d9e1f3a03004ac1bf13b40191ae3440f89e4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Mar 2022 07:29:02 -0500 Subject: [PATCH 1011/1042] fix rules not applying! --- .../event_handlers/bits_event_handler.rs | 9 +++++++-- .../event_handlers/channel_points_event_handler.rs | 1 + .../twitch_pubsub_user_subscriber_thread.rs | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs index 45619e9dae..a8610ce6fa 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs @@ -10,6 +10,7 @@ use database_queries::complex_models::event_responses::EventResponse; use database_queries::queries::twitch::twitch_pubsub::insert_bits::TwitchPubsubBitsInsertBuilder; use database_queries::queries::twitch::twitch_pubsub::insert_channel_points::TwitchPubsubChannelPointsInsertBuilder; use log::info; +use log::error; use r2d2_redis::r2d2; use rand::seq::SliceRandom; use sqlx::MySql; @@ -79,16 +80,20 @@ impl BitsEventHandler { Ok(state) => { info!("Checking bits event against {} rules...", state.event_rules.len()); let maybe_rule = state.event_rules.iter() + .filter(|rule| !rule.rule_is_disabled) .filter(|rule| rule.event_category.eq(&TwitchEventCategory::Bits)) .find(|rule| { match rule.event_match_predicate { EventMatchPredicate::NotSet {} => false, // Not set EventMatchPredicate::ChannelPointsRewardNameExactMatch { .. } => false, // Wrong type EventMatchPredicate::BitsCheermoteNameExactMatch { ref cheermote_name } => { - false // TODO + let contains_cheermote = data.chat_message.to_lowercase().contains(&cheermote_name.to_lowercase()); + contains_cheermote }, EventMatchPredicate::BitsCheermotePrefixSpendThreshold { ref cheermote_prefix, minimum_bits_spent } => { - false // TODO + let contains_cheermote = data.chat_message.to_lowercase().contains(&cheermote_prefix.to_lowercase()); + let spent = i64_to_unsigned_zeroing_negatives(data.bits_used); + contains_cheermote && spent >= minimum_bits_spent }, EventMatchPredicate::BitsSpendThreshold { minimum_bits_spent } => { let spent = i64_to_unsigned_zeroing_negatives(data.bits_used); diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs index a15c6d86c9..4abe29a971 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs @@ -93,6 +93,7 @@ impl ChannelPointsEventHandler { Ok(state) => { info!("Checking channel points event against {} rules...", state.event_rules.len()); let maybe_rule = state.event_rules.iter() + .filter(|rule| !rule.rule_is_disabled) .filter(|rule| rule.event_category.eq(&TwitchEventCategory::ChannelPoints)) .find(|rule| { match rule.event_match_predicate { diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs index 69c173a02b..a53ed2d963 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/twitch_pubsub_user_subscriber_thread.rs @@ -668,8 +668,8 @@ async fn lookup_event_rules( event_category: rule.event_category, event_match_predicate, event_response, - user_specified_rule_order: 0, - rule_is_disabled: false + user_specified_rule_order: rule.user_specified_rule_order, + rule_is_disabled: rule.rule_is_disabled, } }) .collect(); From 95617e9c0cdb0648f64b9bcbb34161455b5b5860 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Mar 2022 19:32:05 -0500 Subject: [PATCH 1012/1042] max lfu models not used! --- crates/service/storyteller_web/src/tts_inference_job.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index 76982a56e2..3516ba6d1f 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -256,7 +256,7 @@ async fn main() -> AnyhowResult<()> { mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe }; - let virtual_lfu_cache = SyncVirtualLfuCache::new(2)?; + let virtual_lfu_cache = SyncVirtualLfuCache::new(sidecar_max_synthesizer_models)?; let cache_miss_strategizers = { let in_memory_strategizer = CacheMissStrategizer::new( From 91eae38e3b33e72c8ae2ce53655aa0b6436e4579 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Mar 2022 19:45:55 -0500 Subject: [PATCH 1013/1042] aughhh usize --- crates/service/storyteller_web/src/tts_inference_job.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index 3516ba6d1f..aced4c1811 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -140,7 +140,7 @@ struct Inferencer { pub no_op_logger_millis: u64, // Maximum number of synthesizer models to hold in memory. - pub sidecar_max_synthesizer_models: u32, + pub sidecar_max_synthesizer_models: usize, } #[tokio::main] From 030b126a9c272b9bfe271cd818514238e641d227 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Mar 2022 20:32:09 -0500 Subject: [PATCH 1014/1042] tests around LFU --- .../src/util/jobs/virtual_lfu_cache.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/service/storyteller_web/src/util/jobs/virtual_lfu_cache.rs b/crates/service/storyteller_web/src/util/jobs/virtual_lfu_cache.rs index 2b88dbd751..1487630d4e 100644 --- a/crates/service/storyteller_web/src/util/jobs/virtual_lfu_cache.rs +++ b/crates/service/storyteller_web/src/util/jobs/virtual_lfu_cache.rs @@ -108,6 +108,23 @@ impl SyncVirtualLfuCache { pub mod tests { use crate::util::jobs::virtual_lfu_cache::VirtualLfuCache; + #[test] + fn large_capacity_does_not_shed() { + let mut cache = VirtualLfuCache::new(10).unwrap(); + cache.insert_returning_replaced("1"); + cache.insert_returning_replaced("2"); + cache.insert_returning_replaced("3"); + cache.insert_returning_replaced("4"); + cache.insert_returning_replaced("5"); + cache.insert_returning_replaced("6"); + cache.insert_returning_replaced("7"); + cache.insert_returning_replaced("8"); + cache.insert_returning_replaced("9"); + assert!(cache.insert_returning_replaced("10").is_none()); // Does not remove + assert!(cache.insert_returning_replaced("11").is_some()); // Removes + assert_eq!(cache.size(), 10); + } + #[test] fn insert_beyond_capacity() { let mut cache = VirtualLfuCache::new(3).unwrap(); From 95dc7617ff3744b66dadb72b3e4dd986e26dee66 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Mar 2022 00:29:58 -0500 Subject: [PATCH 1015/1042] minor stuff related to rate limit --- .env | 1 - crates/service/storyteller_web/src/tts_inference_job.rs | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.env b/.env index e6cbbe3434..7545992311 100644 --- a/.env +++ b/.env @@ -50,7 +50,6 @@ TALKNET_ROOT_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/tts TALKNET_MODEL_CHECK_SCRIPT_NAME="vocodes_model_check_talknet.py" TTS_INFERENCE_SIDECAR_HOSTNAME=localhost:8000 -SIDECAR_MAX_SYNTHESIZER_MODELS=2 #TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt # THIS SOUNDS HORRID #TTS_WAVEGLOW_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index aced4c1811..06106344b9 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -776,6 +776,10 @@ async fn process_job( .virtual_model_lfu .insert_returning_replaced(tts_synthesizer_fs_path.to_str().unwrap_or(""))?; + if let Some(model_path) = maybe_unload_model_path.as_deref() { + warn!("Remove model from LFU cache: {:?}", model_path); + } + let output_audio_fs_path = temp_dir.path().join("output.wav"); let output_metadata_fs_path = temp_dir.path().join("metadata.json"); let output_spectrogram_fs_path = temp_dir.path().join("spectrogram.json"); From 1730da9caa92536d2b9f6b1a4ada435c550c0b40 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Mar 2022 01:12:50 -0500 Subject: [PATCH 1016/1042] Multiple tiers of rate limiting --- .../endpoints/tts/enqueue_infer_tts.rs | 25 ++++++-- .../endpoints/tts/enqueue_upload_tts_model.rs | 2 +- .../endpoints/w2l/enqueue_infer_w2l.rs | 9 +-- .../w2l/enqueue_infer_w2l_with_uploads.rs | 27 +++++++-- .../w2l/enqueue_upload_w2l_template.rs | 2 +- .../web_utils/redis_rate_limiter.rs | 6 +- crates/service/storyteller_web/src/main.rs | 60 +++++++++++++++---- .../storyteller_web/src/server_state.rs | 15 ++++- 8 files changed, 117 insertions(+), 29 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 8f295094c8..eee447465b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -41,6 +41,7 @@ pub struct InferTtsSuccessResponse { #[derive(Debug)] pub enum InferTtsError { BadInput(String), + NotAuthorized, ServerError, RateLimited, } @@ -49,6 +50,7 @@ impl ResponseError for InferTtsError { fn status_code(&self) -> StatusCode { match *self { InferTtsError::BadInput(_) => StatusCode::BAD_REQUEST, + InferTtsError::NotAuthorized => StatusCode::UNAUTHORIZED, InferTtsError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, InferTtsError::RateLimited => StatusCode::TOO_MANY_REQUESTS, } @@ -57,6 +59,7 @@ impl ResponseError for InferTtsError { fn error_response(&self) -> HttpResponse { let error_reason = match self { InferTtsError::BadInput(reason) => reason.to_string(), + InferTtsError::NotAuthorized => "unauthorized".to_string(), InferTtsError::ServerError => "server error".to_string(), InferTtsError::RateLimited => "rate limited".to_string(), }; @@ -77,10 +80,6 @@ pub async fn infer_tts_handler( request: web::Json, server_state: web::Data>) -> Result { - if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { - return Err(InferTtsError::RateLimited); - } - let maybe_user_session = server_state .session_checker .maybe_get_user_session(&http_request, &server_state.mysql_pool) @@ -90,6 +89,24 @@ pub async fn infer_tts_handler( InferTtsError::ServerError })?; + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.is_banned { + return Err(InferTtsError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(InferTtsError::RateLimited); + } + + // ==================== CHECK AND PERFORM TTS ==================== // + let mut maybe_user_token : Option = maybe_user_session .as_ref() .map(|user_session| user_session.user_token.to_string()); diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index 877b9f9cd0..d4dddfe6c5 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -101,7 +101,7 @@ pub async fn upload_tts_model_handler( request: web::Json, server_state: web::Data>) -> Result { - if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { + if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { return Err(UploadTtsModelError::RateLimited); } diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index 978a5dc14f..4ec1a9413b 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -81,10 +81,6 @@ pub async fn infer_w2l_handler( unimplemented!("this isn't finished"); } - if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { - return Err(InferW2lError::RateLimited); - } - let maybe_session = server_state .session_checker .maybe_get_session(&http_request, &server_state.mysql_pool) @@ -94,6 +90,11 @@ pub async fn infer_w2l_handler( InferW2lError::ServerError })?; + // TODO + //if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { + // return Err(InferW2lError::RateLimited); + //} + let mut maybe_user_token : Option = maybe_session .as_ref() .map(|user_session| user_session.user_token.to_string()); diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index f2a1dbad48..97257db6e9 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -53,6 +53,7 @@ pub struct InferW2lWithUploadSuccessResponse { #[derive(Debug)] pub enum InferW2lWithUploadError { BadInput(String), + NotAuthorized, EmptyFileUploaded, ServerError, RateLimited, @@ -62,6 +63,7 @@ impl ResponseError for InferW2lWithUploadError { fn status_code(&self) -> StatusCode { match *self { InferW2lWithUploadError::BadInput(_) => StatusCode::BAD_REQUEST, + InferW2lWithUploadError::NotAuthorized => StatusCode::UNAUTHORIZED, InferW2lWithUploadError::EmptyFileUploaded => StatusCode::BAD_REQUEST, InferW2lWithUploadError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, InferW2lWithUploadError::RateLimited => StatusCode::TOO_MANY_REQUESTS, @@ -71,6 +73,7 @@ impl ResponseError for InferW2lWithUploadError { fn error_response(&self) -> HttpResponse { let error_reason = match self { InferW2lWithUploadError::BadInput(reason) => reason.to_string(), + InferW2lWithUploadError::NotAuthorized => "unauthorized".to_string(), InferW2lWithUploadError::EmptyFileUploaded => "empty file uploaded".to_string(), InferW2lWithUploadError::ServerError => "server error".to_string(), InferW2lWithUploadError::RateLimited => "rate limited".to_string(), @@ -94,12 +97,6 @@ pub async fn enqueue_infer_w2l_with_uploads( mut payload: Multipart ) -> Result { - // ==================== RATE LIMIT ==================== // - - if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { - return Err(InferW2lWithUploadError::RateLimited); - } - let ip_address = get_request_ip(&http_request); // ==================== READ SESSION ==================== // @@ -113,6 +110,24 @@ pub async fn enqueue_infer_w2l_with_uploads( InferW2lWithUploadError::ServerError })?; + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.is_banned { + return Err(InferW2lWithUploadError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(InferW2lWithUploadError::RateLimited); + } + + // ==================== SESSION DETAILS ==================== // + let mut maybe_user_token : Option = maybe_session .as_ref() .map(|user_session| user_session.user_token.to_string()); diff --git a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index a6b42301a0..0087eafedf 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -100,7 +100,7 @@ pub async fn upload_w2l_template_handler( request: web::Json, server_state: web::Data>) -> Result { - if let Err(_err) = server_state.redis_rate_limiter.rate_limit_request(&http_request) { + if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { return Err(UploadW2lTemplateError::RateLimited); } diff --git a/crates/service/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs b/crates/service/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs index 9bfef9f180..1a7004ec42 100644 --- a/crates/service/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs +++ b/crates/service/storyteller_web/src/http_server/web_utils/redis_rate_limiter.rs @@ -14,6 +14,7 @@ const RATE_LIMIT_BYPASS_HEADER : &'static str = "limitless"; #[derive(Clone)] // NB: Limiter is `Clone` pub struct RedisRateLimiter { limiter: Limiter, + limit_key_prefix: String, limiter_enabled: bool, rate_limit_bypass_header: HeaderName, } @@ -24,10 +25,11 @@ pub enum RateLimiterError { } impl RedisRateLimiter { - pub fn new(limiter: Limiter, limiter_enabled: bool) -> Self { + pub fn new(limiter: Limiter, limit_key_prefix: &str, limiter_enabled: bool) -> Self { let rate_limit_bypass_header = HeaderName::from_static(RATE_LIMIT_BYPASS_HEADER); RedisRateLimiter { limiter, + limit_key_prefix: limit_key_prefix.to_string(), limiter_enabled, rate_limit_bypass_header, } @@ -46,7 +48,7 @@ impl RedisRateLimiter { let ip_address = get_request_ip(&request); - let rate_limit_key = format!("rate_limit:{}", ip_address); + let rate_limit_key = format!("rate_limit:{}:{}", &self.limit_key_prefix, ip_address); self.rate_limit_key(&rate_limit_key) } diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index d4257a5c06..342a3059d0 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -103,7 +103,7 @@ use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::routes::add_routes; -use crate::server_state::{ServerState, EnvConfig, TwitchOauthSecrets, TwitchOauth}; +use crate::server_state::{ServerState, EnvConfig, TwitchOauthSecrets, TwitchOauth, RedisRateLimiters}; use crate::threads::ip_banlist_set::IpBanlistSet; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; @@ -200,16 +200,52 @@ async fn main() -> AnyhowResult<()> { let redis_pool = r2d2::Pool::builder() .build(redis_manager)?; - let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_ENABLED", true); - let limiter_max_requests = easyenv::get_env_num("LIMITER_MAX_REQUESTS", 3)?; - let limiter_window_seconds = easyenv::get_env_num("LIMITER_WINDOW_SECONDS", 10)?; + info!("Setting up Redis rate limiters..."); - let limiter = Limiter::build(&common_env.redis_0_connection_string) - .limit(limiter_max_requests) - .period(Duration::from_secs(limiter_window_seconds)) - .finish()?; + // Old env vars: + // + // "LIMITER_ENABLED" + // "LIMITER_MAX_REQUESTS" + // "LIMITER_WINDOW_SECONDS" + // + let logged_out_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_OUT_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_OUT_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_OUT_WINDOW_SECONDS", 10)?; - let redis_rate_limiter = RedisRateLimiter::new(limiter, limiter_enabled); + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "logged_out", limiter_enabled) + }; + + let logged_in_redis_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_LOGGED_IN_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_LOGGED_IN_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_LOGGED_IN_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "logged_in", limiter_enabled) + }; + + let model_upload_rate_limiter = { + let limiter_enabled = easyenv::get_env_bool_or_default("LIMITER_MODEL_UPLOAD_ENABLED", true); + let limiter_max_requests = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_MAX_REQUESTS", 3)?; + let limiter_window_seconds = easyenv::get_env_num("LIMITER_MODEL_UPLOAD_WINDOW_SECONDS", 10)?; + + let limiter = Limiter::build(&common_env.redis_0_connection_string) + .limit(limiter_max_requests) + .period(Duration::from_secs(limiter_window_seconds)) + .finish()?; + + RedisRateLimiter::new(limiter, "model_upload", limiter_enabled) + }; info!("Reading env vars and setting up utils..."); @@ -297,7 +333,11 @@ async fn main() -> AnyhowResult<()> { hostname: server_hostname, mysql_pool: pool, redis_pool, - redis_rate_limiter, + redis_rate_limiters: RedisRateLimiters { + logged_out: logged_out_redis_rate_limiter, + logged_in: logged_in_redis_rate_limiter, + model_upload: model_upload_rate_limiter, + }, firehose_publisher, badge_granter, cookie_manager, diff --git a/crates/service/storyteller_web/src/server_state.rs b/crates/service/storyteller_web/src/server_state.rs index 0dbaae0a7b..3dd035c58e 100644 --- a/crates/service/storyteller_web/src/server_state.rs +++ b/crates/service/storyteller_web/src/server_state.rs @@ -23,7 +23,7 @@ pub struct ServerState { pub redis_pool: r2d2::Pool, - pub redis_rate_limiter: RedisRateLimiter, + pub redis_rate_limiters: RedisRateLimiters, pub cookie_manager: CookieManager, @@ -73,3 +73,16 @@ pub struct TwitchOauthSecrets { pub client_id: String, pub client_secret: String, } + +/// Different rate limiters for different users +#[derive(Clone)] +pub struct RedisRateLimiters { + /// Logged out users have stricter limits + pub logged_out: RedisRateLimiter, + + /// Logged in users have a little more leeway + pub logged_in: RedisRateLimiter, + + /// A rate limiter for TTS and W2L uploads + pub model_upload: RedisRateLimiter, +} From 2f17465a844eea2496a3d509d41ca683b705f92a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Mar 2022 02:00:32 -0500 Subject: [PATCH 1017/1042] job sorts by priority during, but has starvation prevention + env config --- .../list_available_tts_inference_jobs.rs | 58 +++++++++++++++++-- .../storyteller_web/src/tts_inference_job.rs | 30 +++++++++- 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs index 20a54dc768..c70f536f30 100644 --- a/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs @@ -36,12 +36,63 @@ pub struct AvailableTtsInferenceJob { } /// Query jobs that are ready to run +/// If sorting by priority, be careful not to starve lower priority jobs +/// (ie. if there's an issue with higher priorities.) pub async fn list_available_tts_inference_jobs( pool: &MySqlPool, + sort_by_priority: bool, num_records: u32 ) -> AnyhowResult> { - let job_records : Vec = sqlx::query_as!( + // NB: This query is awkwardly written twice because this is the only way the + // macro can statically type check the query, result types, and parameter bindings. + // + // The only difference here is the 'ORDER BY' clause ! + let job_records : Vec = if sort_by_priority { + sqlx::query_as!( + AvailableTtsInferenceJobRawInternal, + r#" +SELECT + id, + token AS inference_job_token, + uuid_idempotency_token, + + model_token, + raw_inference_text, + + creator_ip_address, + maybe_creator_user_token, + creator_set_visibility as `creator_set_visibility: crate::column_types::record_visibility::RecordVisibility`, + + is_from_api, + is_for_twitch, + + status as `status: crate::column_types::job_status::JobStatus`, + priority_level, + attempt_count, + failure_reason, + + created_at, + updated_at, + retry_at +FROM tts_inference_jobs +WHERE + ( + status IN ("pending", "attempt_failed") + ) + AND + ( + retry_at IS NULL + OR + retry_at < CURRENT_TIMESTAMP + ) + ORDER BY priority_level DESC, id ASC + LIMIT ? + "#, + num_records, + ).fetch_all(pool).await? + } else { + sqlx::query_as!( AvailableTtsInferenceJobRawInternal, r#" SELECT @@ -81,9 +132,8 @@ WHERE LIMIT ? "#, num_records, - ) - .fetch_all(pool) - .await?; + ).fetch_all(pool).await? + }; let job_records = job_records.into_iter() .map(|record : AvailableTtsInferenceJobRawInternal| { diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index 06106344b9..3183782df9 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -141,6 +141,11 @@ struct Inferencer { // Maximum number of synthesizer models to hold in memory. pub sidecar_max_synthesizer_models: usize, + + // Typically we'll sort jobs by priority. Occasionally we introduce a chance for low + // priority jobs to run in the order they were enqueued. + // If this is set to "0", we no longer consider priority + pub low_priority_starvation_prevention_every_nth: usize, } #[tokio::main] @@ -252,6 +257,10 @@ async fn main() -> AnyhowResult<()> { let sidecar_max_synthesizer_models = easyenv::get_env_num( "SIDECAR_MAX_SYNTHESIZER_MODELS", 3)?; + // Set to "0" to always treat low priority the same as high priority + let low_priority_starvation_prevention_every_nth= easyenv::get_env_num( + "LOW_PRIORITY_STARVATION_PREVENTION_EVERY_NTH", 3)?; + let firehose_publisher = FirehosePublisher { mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe }; @@ -308,6 +317,7 @@ async fn main() -> AnyhowResult<()> { job_batch_size: common_env.job_batch_size, no_op_logger_millis: common_env.no_op_logger_millis, sidecar_max_synthesizer_models, + low_priority_starvation_prevention_every_nth, }; main_loop(inferencer).await; @@ -325,11 +335,27 @@ async fn main_loop(inferencer: Inferencer) { let mut span_batch = Vec::new(); + let mut sort_by_priority = true; + let mut sort_by_priority_count = 0; + loop { let num_records = inferencer.job_batch_size; - let maybe_available_jobs - = list_available_tts_inference_jobs(&inferencer.mysql_pool, num_records).await; + // Don't completely starve low-priority jobs + if sort_by_priority_count >= inferencer.low_priority_starvation_prevention_every_nth { + sort_by_priority_count = 0; + sort_by_priority = false; + } + + let maybe_available_jobs = + list_available_tts_inference_jobs( + &inferencer.mysql_pool, + sort_by_priority, + num_records + ).await; + + sort_by_priority = true; + sort_by_priority_count += 1; let jobs = match maybe_available_jobs { Ok(jobs) => jobs, From e9b1a4ca8ff333529da0b672b995b41c6e427820 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Mar 2022 02:49:13 -0500 Subject: [PATCH 1018/1042] priority levels --- _migrations_squashed/tts_tables/tts_inference_jobs.sql | 8 +++++--- .../src/http_server/endpoints/tts/enqueue_infer_tts.rs | 5 +++++ .../threads/twitch_pubsub_user_subscriber/tts_writer.rs | 6 +++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/_migrations_squashed/tts_tables/tts_inference_jobs.sql b/_migrations_squashed/tts_tables/tts_inference_jobs.sql index 395790dc3c..2d3dd610c5 100644 --- a/_migrations_squashed/tts_tables/tts_inference_jobs.sql +++ b/_migrations_squashed/tts_tables/tts_inference_jobs.sql @@ -85,9 +85,11 @@ CREATE TABLE tts_inference_jobs ( 'dead') NOT NULL DEFAULT 'pending', -- Priority *increases*, so a level of 2 will be higher than 1. - -- By default, all jobs from FakeYou have level 0. - -- Twitch TTS has level 1. - -- Paid Twitch and Paid API have level 2. + -- These are the level semantics currently: + -- - All jobs from anonymous FakeYou users have level 0. + -- - All jobs from logged in FakeYou users have level 1. + -- - All jobs from Twitch TTS (unpaid) have level 10 (ten). + -- - (There will be future levels for paid Twitch and social FakeYou rewards.) priority_level TINYINT UNSIGNED NOT NULL DEFAULT 0, -- We can track this against a "max_attempt_count" diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs index eee447465b..85b96da0d0 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -156,6 +156,9 @@ pub async fn infer_tts_handler( InferTtsError::ServerError })?; + // Give logged in users execution priority. + let priority_level = if maybe_user_session.is_some() { 1 } else { 0 }; + info!("Creating w2l inference job record..."); let query_result = sqlx::query!( @@ -170,6 +173,7 @@ SET maybe_creator_user_token = ?, creator_ip_address = ?, creator_set_visibility = ?, + priority_level = ?, status = "pending" "#, &job_token, @@ -179,6 +183,7 @@ SET maybe_user_token, ip_address, set_visibility.to_str(), + priority_level, ) .execute(&server_state.mysql_pool) .await; diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs index 0deb0f45b1..bfad6c1675 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/tts_writer.rs @@ -11,6 +11,10 @@ use std::sync::Arc; use twitch_common::cheers::remove_cheers; use twitch_common::twitch_user_id::TwitchUserId; +/// Twitch users get priority above all FakeYou users. +/// TODO: In the future, there should be higher priority for paid users. +const TWITCH_TTS_PRIORITY_LEVEL : u8 = 10; + pub struct TtsWriter { mysql_pool: Arc>, redis_pool: Arc>, @@ -43,7 +47,7 @@ impl TtsWriter { let mut builder = TtsInferenceJobInsertBuilder::new_for_internal_tts() .set_is_for_twitch(true) - .set_priority_level(1) // TODO: This shouldn't be for everyone. + .set_priority_level(TWITCH_TTS_PRIORITY_LEVEL) .set_job_token(&job_token) .set_model_token(model_token) .set_raw_inference_text(&sanitized_text); From e945772c6a0e95545ab17d7cd5b921a2957c5238 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Mar 2022 03:00:35 -0500 Subject: [PATCH 1019/1042] codegen db --- sqlx-data.json | 253 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 243 insertions(+), 10 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 82f28dfc58..2a9a6b9675 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1315,6 +1315,239 @@ "nullable": [] } }, + "38d9e9860cc87398a27053a90d46183bae6589555c7f4424929d9f900ae1809d": { + "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n model_token,\n raw_inference_text,\n\n creator_ip_address,\n maybe_creator_user_token,\n creator_set_visibility as `creator_set_visibility: crate::column_types::record_visibility::RecordVisibility`,\n\n is_from_api,\n is_for_twitch,\n\n status as `status: crate::column_types::job_status::JobStatus`,\n priority_level,\n attempt_count,\n failure_reason,\n\n created_at,\n updated_at,\n retry_at\nFROM tts_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n ORDER BY priority_level DESC, id ASC\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "inference_job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "raw_inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "creator_set_visibility: crate::column_types::record_visibility::RecordVisibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 8, + "name": "is_from_api", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "is_for_twitch", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "status: crate::column_types::job_status::JobStatus", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 11, + "name": "priority_level", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 33 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 12, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 13, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 14, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 15, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 16, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + true, + false, + false, + true + ] + } + }, "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c": { "query": "\nUPDATE tts_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { @@ -7463,16 +7696,6 @@ "nullable": [] } }, - "1168f39a5ec892d06e941b0dba0e955a27ef976567eeafd6e550bc42ea4c4331": { - "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 7 - }, - "nullable": [] - } - }, "1236640a30d0d5177e13c31b61b791874f7c70026003a58ba6e07e7af46c30c1": { "query": "\nSELECT\n next_id\nFROM\n w2l_result_synthetic_ids\nWHERE\n user_token = ?\nLIMIT 1\n ", "describe": { @@ -9925,6 +10148,16 @@ ] } }, + "e5d2e0881e3003256fbf54f494c356187c331c2311a6d8b53f1b4f0e72def332": { + "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n model_token = ?,\n raw_inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n priority_level = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 8 + }, + "nullable": [] + } + }, "e73192a3e763a287d5737821443c5a81649f17e39978497b99c52e1e896a75f6": { "query": "\nSELECT\n token as session_token,\n user_token\nFROM user_sessions\nWHERE token = ?\nAND deleted_at IS NULL\n ", "describe": { From 847805624e54f9400d1de793b3337d872cb16206 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Mar 2022 04:16:58 -0500 Subject: [PATCH 1020/1042] more stats about jobs --- .../jobs/get_tts_inference_queue_count.rs | 81 +++++++++--- sqlx-data.json | 115 ++++++++++++------ 2 files changed, 144 insertions(+), 52 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs index 0ccc33e3d1..081f061513 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/get_tts_inference_queue_count.rs @@ -22,8 +22,20 @@ use std::sync::Arc; #[derive(Serialize)] pub struct GetTtsInferenceQueueCountResponse { pub success: bool, - pub pending_count: i64, + // Seconds since job at head of queue was enqueued (proxy for queue wait time) pub seconds_since_first: i64, + + // All "pending" jobs + pub pending_count: i64, + + // All "pending" jobs with priority > 0 + pub pending_priority_nonzero_count: i64, + + // All "pending" jobs with priority > 1 + pub pending_priority_gt_one_count: i64, + + // Failed, but not permanently dead + pub attempt_failed_count: i64, } #[derive(Debug, Serialize)] @@ -80,22 +92,52 @@ pub async fn get_tts_inference_queue_count_handler( return Err(GetTtsInferenceQueueCountError::Unauthorized); } - // NB: Lookup failure is Err(RowNotFound). + // NB(old?): Lookup failure is Err(RowNotFound). + // NB(2022-03-05): The "seconds_since_first" result might return null if no pending, so IFNULL + // means we won't fail the full query. let maybe_result = sqlx::query_as!( PendingCountResult, r#" SELECT - NOW() - t2.created_at AS seconds_since_first, - ( + IFNULL(( SELECT - count(t1.id) as pending_count + NOW() - t1.created_at AS seconds_since_first FROM tts_inference_jobs AS t1 WHERE t1.status = "pending" - ) as pending_count -FROM tts_inference_jobs AS t2 -WHERE t2.status = "pending" -ORDER BY t2.id ASC -LIMIT 1 + ORDER BY t1.id ASC + LIMIT 1 + ), 0) as seconds_since_first, + sub2.pending_count, + sub3.pending_priority_nonzero_count, + sub4.pending_priority_gt_one_count, + sub5.attempt_failed_count +FROM + ( + SELECT + count(t2.id) as pending_count + FROM tts_inference_jobs AS t2 + WHERE t2.status = "pending" + ) as sub2, + ( + SELECT + count(t3.id) as pending_priority_nonzero_count + FROM tts_inference_jobs AS t3 + WHERE t3.status = "pending" + AND t3.priority_level > 0 + ) as sub3, + ( + SELECT + count(t4.id) as pending_priority_gt_one_count + FROM tts_inference_jobs AS t4 + WHERE t4.status = "pending" + AND t4.priority_level > 1 + ) as sub4, + ( + SELECT + count(t5.id) as attempt_failed_count + FROM tts_inference_jobs AS t5 + WHERE t5.status = "attempt_failed" + ) as sub5 "#, ) .fetch_one(&server_state.mysql_pool) @@ -108,8 +150,11 @@ LIMIT 1 sqlx::Error::RowNotFound => { // NB: Not Found for null results means nothing is pending in the queue PendingCountResult { - pending_count: None, seconds_since_first: 0, + pending_count: 0, + pending_priority_nonzero_count: 0, + pending_priority_gt_one_count: 0, + attempt_failed_count: 0, } }, _ => { @@ -120,10 +165,15 @@ LIMIT 1 }, }; + println!("Foo: {:?}", &result); + let response = GetTtsInferenceQueueCountResponse { success: true, - pending_count: result.pending_count.unwrap_or(0), seconds_since_first: result.seconds_since_first, + pending_count: result.pending_count, + pending_priority_nonzero_count: result.pending_priority_nonzero_count, + pending_priority_gt_one_count: result.pending_priority_gt_one_count, + attempt_failed_count: result.attempt_failed_count, }; let body = serde_json::to_string(&response) @@ -134,8 +184,11 @@ LIMIT 1 .body(body)) } -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub struct PendingCountResult { - pub pending_count: Option, pub seconds_since_first: i64, + pub pending_count: i64, + pub pending_priority_nonzero_count: i64, + pub pending_priority_gt_one_count: i64, + pub attempt_failed_count: i64, } diff --git a/sqlx-data.json b/sqlx-data.json index 2a9a6b9675..0bfc63ee51 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -9350,44 +9350,6 @@ ] } }, - "a9173fd8148645bda75529aae0af63dcfb7cf9e114604ce4dcc962974e7268e1": { - "query": "\nSELECT\n NOW() - t2.created_at AS seconds_since_first,\n (\n SELECT\n count(t1.id) as pending_count\n FROM tts_inference_jobs AS t1\n WHERE t1.status = \"pending\"\n ) as pending_count\nFROM tts_inference_jobs AS t2\nWHERE t2.status = \"pending\"\nORDER BY t2.id ASC\nLIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "seconds_since_first", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 129 - }, - "char_set": 63, - "max_size": 16 - } - }, - { - "ordinal": 1, - "name": "pending_count", - "type_info": { - "type": "LongLong", - "flags": { - "bits": 128 - }, - "char_set": 63, - "max_size": 21 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - true - ] - } - }, "ab981965f518a8986aaaba41ed821fcb438db4c6156cbb26f639921225a88d59": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = \"dead\",\n failure_reason = ?,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { @@ -10206,6 +10168,83 @@ "nullable": [] } }, + "ed8d32dd94719ad6479dd638cdc7a8cfbf725785f51678533988ddba94aac112": { + "query": "\nSELECT\n IFNULL((\n SELECT\n NOW() - t1.created_at AS seconds_since_first\n FROM tts_inference_jobs AS t1\n WHERE t1.status = \"pending\"\n ORDER BY t1.id ASC\n LIMIT 1\n ), 0) as seconds_since_first,\n sub2.pending_count,\n sub3.pending_priority_nonzero_count,\n sub4.pending_priority_gt_one_count,\n sub5.attempt_failed_count\nFROM\n (\n SELECT\n count(t2.id) as pending_count\n FROM tts_inference_jobs AS t2\n WHERE t2.status = \"pending\"\n ) as sub2,\n (\n SELECT\n count(t3.id) as pending_priority_nonzero_count\n FROM tts_inference_jobs AS t3\n WHERE t3.status = \"pending\"\n AND t3.priority_level > 0\n ) as sub3,\n (\n SELECT\n count(t4.id) as pending_priority_gt_one_count\n FROM tts_inference_jobs AS t4\n WHERE t4.status = \"pending\"\n AND t4.priority_level > 1\n ) as sub4,\n (\n SELECT\n count(t5.id) as attempt_failed_count\n FROM tts_inference_jobs AS t5\n WHERE t5.status = \"attempt_failed\"\n ) as sub5\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "seconds_since_first", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 16 + } + }, + { + "ordinal": 1, + "name": "pending_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 21 + } + }, + { + "ordinal": 2, + "name": "pending_priority_nonzero_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 21 + } + }, + { + "ordinal": 3, + "name": "pending_priority_gt_one_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 21 + } + }, + { + "ordinal": 4, + "name": "attempt_failed_count", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false + ] + } + }, "ee05f95e45f7aaa5811d70b1f00d3f40a361536dfc6451affcad97586dc3338f": { "query": "\nUPDATE w2l_inference_jobs\nSET\n status = 'started',\n attempt_count = attempt_count + 1,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { From 61ee75357224125a83ab24abff21d13d6b36a825 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Mar 2022 04:50:29 -0500 Subject: [PATCH 1021/1042] endpoint to kill running queries --- .../kill_tts_inference_jobs.rs | 67 +++++++++ .../src/queries/tts/tts_inference_jobs/mod.rs | 1 + .../jobs/kill_tts_inference_jobs.rs | 132 ++++++++++++++++++ .../endpoints/moderation/jobs/mod.rs | 3 +- crates/service/storyteller_web/src/routes.rs | 8 +- sqlx-data.json | 30 ++++ 6 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 crates/lib/database_queries/src/queries/tts/tts_inference_jobs/kill_tts_inference_jobs.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/kill_tts_inference_jobs.rs diff --git a/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/kill_tts_inference_jobs.rs b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/kill_tts_inference_jobs.rs new file mode 100644 index 0000000000..5e5f4e6747 --- /dev/null +++ b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/kill_tts_inference_jobs.rs @@ -0,0 +1,67 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; +use sqlx::mysql::MySqlArguments; +use sqlx::query::Query; +use sqlx::{MySqlPool, MySql}; + +#[derive(Copy, Clone)] +pub enum JobStatus { + /// Kill all "pending" jobs + AllPending, + /// Kill all "pending" and "attempt_failed" jobs + AllPendingAndFailed, + /// Kill "pending" jobs with priority_level = 0. + ZeroPriorityPending, +} + +pub async fn kill_tts_inference_jobs(job_status: JobStatus, mysql_pool: &MySqlPool) -> AnyhowResult<()> { + + let query = match job_status { + JobStatus::AllPending => kill_all_pending(), + JobStatus::AllPendingAndFailed => kill_all_pending_and_failed(), + JobStatus::ZeroPriorityPending => kill_all_zero_priority_pending(), + }; + + let result = query.execute(mysql_pool).await; + + match result { + Err(err) => { + Err(anyhow!("error with query: {:?}", err)) + }, + Ok(_r) => Ok(()), + } +} + +fn kill_all_pending() -> Query<'static, MySql, MySqlArguments> { + sqlx::query!( + r#" +UPDATE tts_inference_jobs +SET status='complete_failure' +WHERE + status IN ('pending'); + "# + ) +} + +fn kill_all_pending_and_failed() -> Query<'static, MySql, MySqlArguments> { + sqlx::query!( + r#" +UPDATE tts_inference_jobs +SET status='complete_failure' +WHERE + status IN ('pending', 'attempt_failed'); + "# + ) +} + +fn kill_all_zero_priority_pending() -> Query<'static, MySql, MySqlArguments> { + sqlx::query!( + r#" +UPDATE tts_inference_jobs +SET status='complete_failure' +WHERE + status IN ('pending') + AND priority_level = 0; + "# + ) +} diff --git a/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mod.rs b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mod.rs index 7aaa57a6d6..b15fce6350 100644 --- a/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mod.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_inference_jobs/mod.rs @@ -1,5 +1,6 @@ pub mod _keys; pub mod insert_tts_inference_job; +pub mod kill_tts_inference_jobs; pub mod list_available_tts_inference_jobs; pub mod mark_tts_inference_job_done; pub mod mark_tts_inference_job_failure; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/kill_tts_inference_jobs.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/kill_tts_inference_jobs.rs new file mode 100644 index 0000000000..c352c30bb9 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/kill_tts_inference_jobs.rs @@ -0,0 +1,132 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::server_state::ServerState; +use database_queries::queries::tts::tts_inference_jobs::kill_tts_inference_jobs::{kill_tts_inference_jobs, JobStatus}; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use log::{info, warn, log, error}; +use regex::Regex; +use sqlx::MySqlPool; +use std::fmt; +use std::sync::Arc; + +// =============== Request =============== + +#[derive(Copy, Clone, Deserialize)] +pub enum KillAction { + /// Kill all "pending" jobs + #[serde(rename_all = "snake_case")] + AllPending, + /// Kill all "pending" and "attempt_failed" jobs + #[serde(rename_all = "snake_case")] + AllPendingAndFailed, + /// Kill "pending" jobs with priority_level = 0. + #[serde(rename_all = "snake_case")] + ZeroPriorityPending, +} + +// NB: ONLY MODERATORS CAN EDIT CATEGORIES. +// These are not sparse updates! +#[derive(Deserialize)] +pub struct KillTtsInferenceJobsRequest { + kill_action: KillAction, +} + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct KillTtsInferenceJobsResponse { + pub success: bool, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum KillTtsInferenceJobsError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for KillTtsInferenceJobsError { + fn status_code(&self) -> StatusCode { + match *self { + KillTtsInferenceJobsError::BadInput(_) => StatusCode::BAD_REQUEST, + KillTtsInferenceJobsError::NotFound => StatusCode::NOT_FOUND, + KillTtsInferenceJobsError::NotAuthorized => StatusCode::UNAUTHORIZED, + KillTtsInferenceJobsError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for KillTtsInferenceJobsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn kill_tts_inference_jobs_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + KillTtsInferenceJobsError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(KillTtsInferenceJobsError::NotAuthorized); + } + }; + + // TODO: We don't have a permission for this, so use this as a proxy permission + if !user_session.can_ban_users { + warn!("no permission to edit categories"); + return Err(KillTtsInferenceJobsError::NotAuthorized); + } + + let job_status = match request.kill_action { + KillAction::AllPending => JobStatus::AllPending, + KillAction::AllPendingAndFailed => JobStatus::AllPendingAndFailed, + KillAction::ZeroPriorityPending => JobStatus::ZeroPriorityPending, + }; + + kill_tts_inference_jobs(job_status, &server_state.mysql_pool) + .await + .map_err(|e| { + error!("Error with query: {:?}", e); + return KillTtsInferenceJobsError::ServerError; + })?; + + let response = KillTtsInferenceJobsResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| KillTtsInferenceJobsError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs index 1b0e583d12..0b81519ead 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/mod.rs @@ -1,2 +1,3 @@ pub mod get_tts_inference_queue_count; -pub mod get_w2l_inference_queue_count; \ No newline at end of file +pub mod get_w2l_inference_queue_count; +pub mod kill_tts_inference_jobs; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index 35c2721aa3..861c2e2ad8 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -29,6 +29,7 @@ use crate::http_server::endpoints::moderation::ip_bans::get_ip_ban::get_ip_ban_h use crate::http_server::endpoints::moderation::ip_bans::list_ip_bans::list_ip_bans_handler; use crate::http_server::endpoints::moderation::jobs::get_tts_inference_queue_count::get_tts_inference_queue_count_handler; use crate::http_server::endpoints::moderation::jobs::get_w2l_inference_queue_count::get_w2l_inference_queue_count_handler; +use crate::http_server::endpoints::moderation::jobs::kill_tts_inference_jobs::kill_tts_inference_jobs_handler; use crate::http_server::endpoints::moderation::stats::get_voice_count_stats::get_voice_count_stats_handler; use crate::http_server::endpoints::moderation::user_bans::ban_user::ban_user_handler; use crate::http_server::endpoints::moderation::user_bans::list_banned_users::list_banned_users_handler; @@ -54,6 +55,7 @@ use crate::http_server::endpoints::twitch::event_rules::edit_event_rule::edit_tw use crate::http_server::endpoints::twitch::event_rules::get_event_rule::get_twitch_event_rule_for_user_handler; use crate::http_server::endpoints::twitch::event_rules::list_event_rules_for_user::list_twitch_event_rules_for_user_handler; use crate::http_server::endpoints::twitch::event_rules::reorder_twitch_event_rules::reorder_twitch_event_rules_handler; +use crate::http_server::endpoints::twitch::oauth::check_oauth_status::check_oauth_status_handler; use crate::http_server::endpoints::twitch::oauth::oauth_begin_json::oauth_begin_enroll_json; use crate::http_server::endpoints::twitch::oauth::oauth_begin_redirect::oauth_begin_enroll_redirect; use crate::http_server::endpoints::twitch::oauth::oauth_end::oauth_end_enroll_from_redirect; @@ -83,7 +85,6 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; -use crate::http_server::endpoints::twitch::oauth::check_oauth_status::check_oauth_status_handler; pub fn add_routes (app: App) -> App where @@ -235,6 +236,11 @@ fn add_moderator_routes (app: App) -> App .route(web::get().to(get_tts_inference_queue_count_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/kill_tts_inference_jobs") + .route(web::post().to(kill_tts_inference_jobs_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/w2l_inference_queue_stats") .route(web::get().to(get_w2l_inference_queue_count_handler)) diff --git a/sqlx-data.json b/sqlx-data.json index 0bfc63ee51..0a0ffd08ab 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -4119,6 +4119,16 @@ ] } }, + "89b66d91d1f2cd5c9871671b21a90e77d9041572a90cc6be6dd49845424677fa": { + "query": "\nUPDATE tts_inference_jobs\nSET status='complete_failure'\nWHERE\n status IN ('pending');\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 0 + }, + "nullable": [] + } + }, "8ac132004408188cc55710185539ff6cd2011de3e0f80b4c91e2bf8b3212135e": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.ietf_language_tag,\n tts.ietf_primary_language_subtag,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.maybe_suggested_unique_bot_command,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n users.username = ?\n AND tts.is_locked_from_use IS FALSE\n AND tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -4326,6 +4336,16 @@ ] } }, + "8c3f19f67e8871ada450832b5342efd870c87975f065f3d6b3789d537cab01ed": { + "query": "\nUPDATE tts_inference_jobs\nSET status='complete_failure'\nWHERE\n status IN ('pending', 'attempt_failed');\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 0 + }, + "nullable": [] + } + }, "8c49202950a8d6cf87bbc60517168fba4c9b0c52c71a25d766be49fedbd28827": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_gravatar_hash,\n tts.title,\n tts.ietf_language_tag,\n tts.ietf_primary_language_subtag,\n tts.is_locked_from_use,\n tts.is_front_page_featured,\n tts.is_twitch_featured,\n tts.maybe_suggested_unique_bot_command,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\n ON users.token = tts.creator_user_token\nWHERE\n tts.user_deleted_at IS NULL\n AND tts.mod_deleted_at IS NULL\n ", "describe": { @@ -6753,6 +6773,16 @@ ] } }, + "cb732b1732a36c826317f835b87c17d1b6d061c9363e9477f2603b435f9bf28f": { + "query": "\nUPDATE tts_inference_jobs\nSET status='complete_failure'\nWHERE\n status IN ('pending')\n AND priority_level = 0;\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 0 + }, + "nullable": [] + } + }, "d51e6e901cc8641b7b076f42b0fc507dad2168617c33c615ef97417f3a78ccc1": { "query": "\nUPDATE api_tokens\nSET\n deleted_at = CURRENT_TIMESTAMP,\n ip_address_last_update = ?\nWHERE\n user_token = ?\nAND api_token NOT IN (?, ?, ?, ?, ?)\n ", "describe": { From d7205ea3af5e4457b96612e64f8e35c6237c692d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Mar 2022 04:54:39 -0500 Subject: [PATCH 1022/1042] add more slurs. Thanks to bibby --- db/banned_slurs.txt | 101 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/db/banned_slurs.txt b/db/banned_slurs.txt index 650abced03..b85aab3faa 100644 --- a/db/banned_slurs.txt +++ b/db/banned_slurs.txt @@ -1,14 +1,113 @@ # This file serves to remove racial slurs and epithets from usage with vocodes. # There's probably more than this that we'll want to remove. # I need to investigate. +# Additional slurs from BibbyBoi 2022-03-04 +arabush +aravush +argie +argies +bakra +balija +bamboula +beaner +beaney +bluegum +bogtrotter +boonga +boonie +bootlip +bougnoule +bozgor +buddhahead +bule +bunga +burrhead +carcamano +chankoro chink chinks +choc-ice +coon +coonass +dago +dego +dink +dune-coon dyke fag faggot fags +greaser +groid +guidette +guido +haole +hebe +heeb +honkey +honkie +hymie +injun +jigaboo +jigarooni +jiggabo +jigger +jijjiboo +kike kniggers +kyke +maumau +mayomonkey +moolinyan +mulignan +mulignon +neger +negro +negro +negroid +nicc a +nicc er +nicca +niccer nig +nigar +nigette +niggah +niggar nigger niggers -nigs \ No newline at end of file +niggertoe +niggress +nigguh +nignog +nigs +nik-a +nika +nikk-a +nikk-er +ofay +poortagee +rastus +sambo +sheboon +sheeny +sooty +spic +spick +spig +spig +spigotty +spik +tarbaby +whigg-a +whigg-er +whigga +whigger +wigg-er +wigger +wikk-a +wikk-er +wikka +wikker +zigabo +zipperhead \ No newline at end of file From efe2bcd61d51a4464ef4b73f1425d81019e6e12f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Mar 2022 06:00:47 -0500 Subject: [PATCH 1023/1042] rename fields correctly --- .../endpoints/moderation/jobs/kill_tts_inference_jobs.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/kill_tts_inference_jobs.rs b/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/kill_tts_inference_jobs.rs index c352c30bb9..ec4df966b2 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/kill_tts_inference_jobs.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/moderation/jobs/kill_tts_inference_jobs.rs @@ -18,15 +18,13 @@ use std::sync::Arc; // =============== Request =============== #[derive(Copy, Clone, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum KillAction { /// Kill all "pending" jobs - #[serde(rename_all = "snake_case")] AllPending, /// Kill all "pending" and "attempt_failed" jobs - #[serde(rename_all = "snake_case")] AllPendingAndFailed, /// Kill "pending" jobs with priority_level = 0. - #[serde(rename_all = "snake_case")] ZeroPriorityPending, } From 30929b051e18b230c136227f64ac3f82b6b20701 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Mar 2022 05:18:15 -0500 Subject: [PATCH 1024/1042] add health checking --- crates/lib/config/src/shared_constants.rs | 2 +- .../health_check/health_check_query.rs | 30 ++++++ .../src/queries/health_check/mod.rs | 1 + .../lib/database_queries/src/queries/mod.rs | 1 + .../src/http_server/endpoints/mod.rs | 1 + .../endpoints/service/health_check_handler.rs | 97 +++++++++++++++++++ .../src/http_server/endpoints/service/mod.rs | 1 + crates/service/storyteller_web/src/main.rs | 25 +++-- crates/service/storyteller_web/src/routes.rs | 9 +- .../storyteller_web/src/server_state.rs | 3 + .../db_health_check_status.rs | 78 +++++++++++++++ .../db_health_checker_thread.rs | 39 ++++++++ .../threads/db_health_checker_thread/mod.rs | 2 + .../storyteller_web/src/threads/mod.rs | 1 + 14 files changed, 282 insertions(+), 8 deletions(-) create mode 100644 crates/lib/database_queries/src/queries/health_check/health_check_query.rs create mode 100644 crates/lib/database_queries/src/queries/health_check/mod.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/service/health_check_handler.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/service/mod.rs create mode 100644 crates/service/storyteller_web/src/threads/db_health_checker_thread/db_health_check_status.rs create mode 100644 crates/service/storyteller_web/src/threads/db_health_checker_thread/db_health_checker_thread.rs create mode 100644 crates/service/storyteller_web/src/threads/db_health_checker_thread/mod.rs diff --git a/crates/lib/config/src/shared_constants.rs b/crates/lib/config/src/shared_constants.rs index 51701db66d..b142962b30 100644 --- a/crates/lib/config/src/shared_constants.rs +++ b/crates/lib/config/src/shared_constants.rs @@ -3,7 +3,7 @@ /// NB: `sqlx::query` is spammy and dumps all queries as "info"-level log lines. /// NB: `hyper::proto::h1::io` is incredibly spammy and logs every chunk of bytes in very large files being downloaded. -pub const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn,hyper::proto::h1::io=warn"; +pub const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn,hyper::proto::h1::io=warn,storyteller_web::threads::db_health_checker_thread::db_health_checker_thread=warn"; //const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; // but sometimes we want to debug /// The default Redis connection string for use in development diff --git a/crates/lib/database_queries/src/queries/health_check/health_check_query.rs b/crates/lib/database_queries/src/queries/health_check/health_check_query.rs new file mode 100644 index 0000000000..847f77e20d --- /dev/null +++ b/crates/lib/database_queries/src/queries/health_check/health_check_query.rs @@ -0,0 +1,30 @@ +use anyhow::anyhow; +use chrono::{Utc, DateTime, NaiveDateTime}; +use container_common::anyhow_result::AnyhowResult; +use sqlx::MySqlPool; + +/// This just queries for current time. If it fails, we know the connection pool is dead. +pub struct HealthCheckResult { + pub present_time: NaiveDateTime, +} + +/// Query the DB for time as a proxy for DB health +pub async fn health_check_db( + pool: &MySqlPool, +) -> AnyhowResult { + + let maybe_record = sqlx::query_as!( + HealthCheckResult, + r#" +SELECT NOW() as present_time + "#, + ) + .fetch_one(pool) + .await; + + match maybe_record { + Ok(record) => Ok(record), + Err(sqlx::Error::RowNotFound) => Err(anyhow!("query didn't return anything")), + Err(ref err) => Err(anyhow!("database query error: {:?}", err)), + } +} diff --git a/crates/lib/database_queries/src/queries/health_check/mod.rs b/crates/lib/database_queries/src/queries/health_check/mod.rs new file mode 100644 index 0000000000..3712d08210 --- /dev/null +++ b/crates/lib/database_queries/src/queries/health_check/mod.rs @@ -0,0 +1 @@ +pub mod health_check_query; \ No newline at end of file diff --git a/crates/lib/database_queries/src/queries/mod.rs b/crates/lib/database_queries/src/queries/mod.rs index f7393c2273..eb9ea3bf70 100644 --- a/crates/lib/database_queries/src/queries/mod.rs +++ b/crates/lib/database_queries/src/queries/mod.rs @@ -1,4 +1,5 @@ pub mod api_tokens; +pub mod health_check; pub mod ip_bans; pub mod model_categories; pub mod tts; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/mod.rs index 0c8b179f69..aec49bb864 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/mod.rs @@ -4,6 +4,7 @@ pub mod events; pub mod leaderboard; pub mod misc; pub mod moderation; +pub mod service; pub mod tts; pub mod twitch; pub mod users; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/service/health_check_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/service/health_check_handler.rs new file mode 100644 index 0000000000..a6e8353236 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/service/health_check_handler.rs @@ -0,0 +1,97 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Path, Json}; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc, NaiveDateTime}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use database_queries::complex_models::event_responses::EventResponse; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use lexical_sort::natural_lexical_cmp; +use log::{info, warn, log, error}; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::fmt; +use std::sync::Arc; + +// =============== Success Response =============== + +#[derive(Serialize)] +pub struct HealthCheckResponse { + pub success: bool, + pub is_healthy: bool, + pub last_db_time: Option, + pub healthy_check_consecutive_count: Option, + pub unhealthy_check_consecutive_count: Option, +} + + +// =============== Error Response =============== + +#[derive(Debug, Serialize)] +pub enum HealthCheckError { + ServerError, +} + +impl ResponseError for HealthCheckError { + fn status_code(&self) -> StatusCode { + match *self { + HealthCheckError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for HealthCheckError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +pub async fn get_health_check_handler( + http_request: HttpRequest, + server_state: web::Data>) -> Result +{ + let health_check_status = server_state.health_check_status.get_health_check_status() + .map_err(|e| { + error!("Error serving health check status: {:?}", e); + HealthCheckError::ServerError + })?; + + let is_healthy = health_check_status.is_healthy; + + let response = HealthCheckResponse { + success: true, + is_healthy: health_check_status.is_healthy, + last_db_time: health_check_status.last_db_time, + healthy_check_consecutive_count: health_check_status.healthy_check_consecutive_count, + unhealthy_check_consecutive_count: health_check_status.unhealthy_check_consecutive_count, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| HealthCheckError::ServerError)?; + + if is_healthy { + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) + } else { + Ok(HttpResponse::InternalServerError() + .content_type("application/json") + .body(body)) + } +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/service/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/service/mod.rs new file mode 100644 index 0000000000..f6eeac1ec2 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/service/mod.rs @@ -0,0 +1 @@ +pub mod health_check_handler; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 342a3059d0..1d3ac5d92e 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -124,6 +124,8 @@ use std::sync::Arc; use std::time::Duration; use tokio::runtime::Runtime; use twitch_common::twitch_secrets::TwitchSecrets; +use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; +use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -298,10 +300,6 @@ async fn main() -> AnyhowResult<()> { easyenv::get_env_string_or_default("SORT_KEY_SECRET", "webscale"); let sort_key_crypto = SortKeyCrypto::new(&sort_key_crypto_secret); - let ip_banlist = IpBanlistSet::new(); - let ip_banlist2 = ip_banlist.clone(); - let mysql_pool3 = pool.clone(); - let twitch_oauth_redirect_landing_url = easyenv::get_env_string_or_default( "TWITCH_OAUTH_REDIRECT_LANDING_URL", "https://api.jungle.horse/twitch/oauth/enroll_redirect_landing"); @@ -313,12 +311,26 @@ async fn main() -> AnyhowResult<()> { let twitch_secrets = TwitchSecrets::from_env()?; // Background jobs. - info!("Spawning IP ban polling thread."); + + let health_check_status = HealthCheckStatus::new(); + let health_check_status2 = health_check_status.clone(); + let ip_banlist = IpBanlistSet::new(); + let ip_banlist2 = ip_banlist.clone(); + let mysql_pool3 = pool.clone(); + let mysql_pool4 = pool.clone(); let tokio_runtime = Runtime::new()?; + info!("Spawning DB health checker thread."); + tokio_runtime.spawn(async { - poll_ip_bans(ip_banlist2, mysql_pool3).await; + db_health_checker_thread(health_check_status2, mysql_pool3).await; + }); + + info!("Spawning IP ban polling thread."); + + tokio_runtime.spawn(async { + poll_ip_bans(ip_banlist2, mysql_pool4).await; }); let server_state = ServerState { @@ -331,6 +343,7 @@ async fn main() -> AnyhowResult<()> { website_homepage_redirect, }, hostname: server_hostname, + health_check_status, mysql_pool: pool, redis_pool, redis_rate_limiters: RedisRateLimiters { diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index 861c2e2ad8..5dda962444 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -85,6 +85,7 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; +use crate::http_server::endpoints::service::health_check_handler::get_health_check_handler; pub fn add_routes (app: App) -> App where @@ -106,8 +107,14 @@ pub fn add_routes (app: App) -> App app = add_voice_clone_request_routes(app); /* /voice_clone_requests */ app = add_twitch_routes(app); /* /twitch */ // TODO: MAYBE TEMPORARY - // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== + // ==================== SERVICE ==================== app.service( + web::resource("/_status") + .route(web::get().to(get_health_check_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== + .service( web::resource("/create_account") .route(web::post().to(create_account_handler)) .route(web::head().to(|| HttpResponse::Ok())) diff --git a/crates/service/storyteller_web/src/server_state.rs b/crates/service/storyteller_web/src/server_state.rs index 3dd035c58e..cc1194b4cf 100644 --- a/crates/service/storyteller_web/src/server_state.rs +++ b/crates/service/storyteller_web/src/server_state.rs @@ -10,6 +10,7 @@ use database_queries::mediators::badge_granter::BadgeGranter; use database_queries::mediators::firehose_publisher::FirehosePublisher; use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; +use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; /// State that is injected into every endpoint. #[derive(Clone)] @@ -19,6 +20,8 @@ pub struct ServerState { pub hostname: String, + pub health_check_status: HealthCheckStatus, + pub mysql_pool: MySqlPool, pub redis_pool: r2d2::Pool, diff --git a/crates/service/storyteller_web/src/threads/db_health_checker_thread/db_health_check_status.rs b/crates/service/storyteller_web/src/threads/db_health_checker_thread/db_health_check_status.rs new file mode 100644 index 0000000000..3c63842e11 --- /dev/null +++ b/crates/service/storyteller_web/src/threads/db_health_checker_thread/db_health_check_status.rs @@ -0,0 +1,78 @@ +use anyhow::anyhow; +use crate::AnyhowResult; +use std::collections::HashSet; +use std::collections::hash_map::RandomState; +use std::sync::{RwLock, PoisonError, RwLockWriteGuard, RwLockReadGuard, Arc}; +use chrono::{Utc, DateTime, NaiveDateTime}; + +#[derive(Clone)] +pub struct HealthCheckStatus { + internal_status: Arc>, +} + +#[derive(Clone)] +pub struct HealthCheckStatusData { + pub last_db_time: Option, + pub is_healthy: bool, + /// Number of consecutive tries the DB has been good. + /// Resets at the first sign of bad. + pub healthy_check_consecutive_count: Option, + /// Number of consecutive tries the DB has been bad. + /// Resets at the first sign of good. + pub unhealthy_check_consecutive_count: Option, +} + +impl HealthCheckStatus { + + pub fn new() -> Self { + Self { + internal_status: Arc::new(RwLock::new(HealthCheckStatusData { + last_db_time: None, + is_healthy: false, // Start false + healthy_check_consecutive_count: None, + unhealthy_check_consecutive_count: None, + })), + } + } + + pub fn record_ping_success(&self, db_time: NaiveDateTime) -> AnyhowResult<()> { + match self.internal_status.write() { + Err(_) => Err(anyhow!("Can't write lock")), + Ok(mut lock) => { + (*lock).last_db_time = Some(db_time); + (*lock).is_healthy = true; + (*lock).unhealthy_check_consecutive_count = None; + + let healthy_count = (*lock).healthy_check_consecutive_count.unwrap_or(0); + (*lock).healthy_check_consecutive_count = Some(healthy_count.saturating_add(1)); + + Ok(()) + }, + } + } + + pub fn record_ping_failure(&self) -> AnyhowResult<()> { + match self.internal_status.write() { + Err(_) => Err(anyhow!("Can't write lock")), + Ok(mut lock) => { + (*lock).is_healthy = false; + (*lock).healthy_check_consecutive_count = None; + + let unhealthy_count = (*lock).unhealthy_check_consecutive_count.unwrap_or(0); + (*lock).unhealthy_check_consecutive_count = Some(unhealthy_count.saturating_add(1)); + + Ok(()) + }, + } + } + + /// Read the status + pub fn get_health_check_status(&self) -> AnyhowResult { + match self.internal_status.read() { + Err(_) => Err(anyhow!("Can't read lock")), + Ok(lock) => { + Ok(lock.clone()) + }, + } + } +} diff --git a/crates/service/storyteller_web/src/threads/db_health_checker_thread/db_health_checker_thread.rs b/crates/service/storyteller_web/src/threads/db_health_checker_thread/db_health_checker_thread.rs new file mode 100644 index 0000000000..7698d3a4c5 --- /dev/null +++ b/crates/service/storyteller_web/src/threads/db_health_checker_thread/db_health_checker_thread.rs @@ -0,0 +1,39 @@ +use anyhow::Error; +use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; +use database_queries::queries::health_check::health_check_query::{health_check_db, HealthCheckResult}; +use log::debug; +use log::error; +use log::info; +use log::warn; +use sqlx::MySqlPool; +use std::thread; +use std::time::Duration; + +pub async fn db_health_checker_thread( + health_check_status: HealthCheckStatus, + mysql_pool: MySqlPool, +) { + loop { + debug!("Checking DB health..."); + + match health_check_db(&mysql_pool).await { + Ok(result) => { + match health_check_status.record_ping_success(result.present_time) { + Err(e) => error!("Problem updating application health checks!"), + Ok(_) => {}, + } + } + Err(e) => { + error!("Problem health checking database: {:?}", e); + match health_check_status.record_ping_failure() { + Err(e) => error!("Problem updating application health checks!"), + Ok(_) => {}, + } + } + } + + thread::sleep(Duration::from_millis(3_000)); + } + + warn!("Should never happen: Health Checker Exits"); +} diff --git a/crates/service/storyteller_web/src/threads/db_health_checker_thread/mod.rs b/crates/service/storyteller_web/src/threads/db_health_checker_thread/mod.rs new file mode 100644 index 0000000000..18698ccfad --- /dev/null +++ b/crates/service/storyteller_web/src/threads/db_health_checker_thread/mod.rs @@ -0,0 +1,2 @@ +pub mod db_health_check_status; +pub mod db_health_checker_thread; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/threads/mod.rs b/crates/service/storyteller_web/src/threads/mod.rs index 0fdb205acb..de34ef34d1 100644 --- a/crates/service/storyteller_web/src/threads/mod.rs +++ b/crates/service/storyteller_web/src/threads/mod.rs @@ -1,2 +1,3 @@ +pub mod db_health_checker_thread; pub mod ip_banlist_set; pub mod poll_ip_banlist_thread; \ No newline at end of file From 8d6a384ac75e02e355195900f8c2bf472d5fb6fe Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Mar 2022 05:21:12 -0500 Subject: [PATCH 1025/1042] health check query --- sqlx-data.json | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/sqlx-data.json b/sqlx-data.json index 0a0ffd08ab..4c2b18f872 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -6833,6 +6833,31 @@ "nullable": [] } }, + "dfe8dedd1838627a7bebbc2facabc3c51a7f3122e19ece6259629696643ae877": { + "query": "\nSELECT NOW() as present_time\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "present_time", + "type_info": { + "type": "Datetime", + "flags": { + "bits": 129 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false + ] + } + }, "e0ad2a7fe4ae795e4f097f604d5cfcf9605298b58cde5ca23bdf9c8ec7383b00": { "query": "\nUPDATE w2l_templates\nSET\n creator_ip_address_last_update = ?,\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ", "describe": { From 4734d20690543a4e484c56b3b7f60052981b7d55 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Mar 2022 05:42:28 -0500 Subject: [PATCH 1026/1042] category list caching --- crates/lib/container_common/src/cache/mod.rs | 0 .../categories/list_tts_categories.rs | 98 ++++++++++++------- .../endpoints/tts/list_tts_models.rs | 4 +- crates/service/storyteller_web/src/main.rs | 13 ++- .../storyteller_web/src/server_state.rs | 14 ++- .../src/util/caching/single_item_ttl_cache.rs | 3 + 6 files changed, 86 insertions(+), 46 deletions(-) create mode 100644 crates/lib/container_common/src/cache/mod.rs diff --git a/crates/lib/container_common/src/cache/mod.rs b/crates/lib/container_common/src/cache/mod.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs index 0a7c312ae2..5ccf74d83f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/categories/list_tts_categories.rs @@ -16,7 +16,7 @@ use crate::util::markdown_to_html::markdown_to_html; use database_queries::queries::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use lexical_sort::natural_lexical_cmp; -use log::{info, warn, log}; +use log::{info, warn, log, error}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; @@ -35,7 +35,7 @@ pub struct ListTtsCategoriesResponse { /// Public-facing and optimized (non-irrelevant fields) category list /// Used for the main TTS dropdown as well as the TTS edit/tagging UI -#[derive(Serialize)] +#[derive(Serialize, Clone)] pub struct DisplayCategory { pub category_token: String, @@ -99,47 +99,69 @@ pub async fn list_tts_categories_handler( http_request: HttpRequest, server_state: web::Data>) -> Result { - let query_builder = ListCategoriesQueryBuilder::new() - .show_deleted(false) - .show_unapproved(false) - .scope_model_type(Some("tts")); - - let query_result = - query_builder.perform_query(&server_state.mysql_pool).await; + let maybe_categories = server_state.caches.category_list.copy_without_bump_if_unexpired() + .map_err(|e| { + error!("Error consulting cache: {:?}", e); + ListTtsCategoriesError::ServerError + })?; - let results = match query_result { - Ok(results) => results, - Err(err) => { - warn!("DB error: {:?}", err); - return Err(ListTtsCategoriesError::ServerError); - } - }; + let categories = match maybe_categories { + Some(categories) => { + info!("Serving TTS categories from cache"); + categories + }, + None => { + let query_builder = ListCategoriesQueryBuilder::new() + .show_deleted(false) + .show_unapproved(false) + .scope_model_type(Some("tts")); - warn!("Number of categories: {:?}", results.categories.len()); + let query_result = + query_builder.perform_query(&server_state.mysql_pool).await; - let mut categories = results.categories.iter() - .map(|c| { - DisplayCategory { - category_token: c.category_token.clone(), - model_type: c.model_type.clone(), - maybe_super_category_token: c.maybe_super_category_token.clone(), - can_directly_have_models: c.can_directly_have_models, - can_have_subcategories: c.can_have_subcategories, - can_only_mods_apply: c.can_only_mods_apply, - is_mod_approved: c.is_mod_approved, - name: c.name.clone(), - name_for_dropdown: c.maybe_dropdown_name.clone().unwrap_or(c.name.clone()), - created_at: c.created_at.clone(), - updated_at: c.updated_at.clone(), - deleted_at: c.deleted_at.clone(), + let results = match query_result { + Ok(results) => results, + Err(err) => { + warn!("DB error: {:?}", err); + return Err(ListTtsCategoriesError::ServerError); } - }) - .collect::>(); + }; - // NB: This might produce weird sorting resorts relative to the "name" field, - // but the typical way this should be consumed is via dropdowns. - categories.sort_by(|c1, c2| - natural_lexical_cmp(&c1.name_for_dropdown, &c2.name_for_dropdown)); + warn!("Number of categories: {:?}", results.categories.len()); + + let mut categories = results.categories.iter() + .map(|c| { + DisplayCategory { + category_token: c.category_token.clone(), + model_type: c.model_type.clone(), + maybe_super_category_token: c.maybe_super_category_token.clone(), + can_directly_have_models: c.can_directly_have_models, + can_have_subcategories: c.can_have_subcategories, + can_only_mods_apply: c.can_only_mods_apply, + is_mod_approved: c.is_mod_approved, + name: c.name.clone(), + name_for_dropdown: c.maybe_dropdown_name.clone().unwrap_or(c.name.clone()), + created_at: c.created_at.clone(), + updated_at: c.updated_at.clone(), + deleted_at: c.deleted_at.clone(), + } + }) + .collect::>(); + + // NB: This might produce weird sorting resorts relative to the "name" field, + // but the typical way this should be consumed is via dropdowns. + categories.sort_by(|c1, c2| + natural_lexical_cmp(&c1.name_for_dropdown, &c2.name_for_dropdown)); + + server_state.caches.category_list.store_copy(&categories) + .map_err(|e| { + error!("Error storing cache: {:?}", e); + ListTtsCategoriesError::ServerError + })?; + + categories + }, + }; // TODO: Cache results in Redis w/ TTL. let response = ListTtsCategoriesResponse { diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs index 67a8cdc3f5..f094766144 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/list_tts_models.rs @@ -83,7 +83,7 @@ pub async fn list_tts_models_handler( http_request: HttpRequest, server_state: web::Data>) -> Result { - let maybe_models = server_state.voice_list_cache.copy_without_bump_if_unexpired() + let maybe_models = server_state.caches.voice_list.copy_without_bump_if_unexpired() .map_err(|e| { error!("Error consulting cache: {:?}", e); ListTtsModelsError::ServerError @@ -103,7 +103,7 @@ pub async fn list_tts_models_handler( ListTtsModelsError::ServerError })?; - server_state.voice_list_cache.store_copy(&models) + server_state.caches.voice_list.store_copy(&models) .map_err(|e| { error!("Error storing cache: {:?}", e); ListTtsModelsError::ServerError diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 1d3ac5d92e..7b7c0fd2e2 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -103,7 +103,7 @@ use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::routes::add_routes; -use crate::server_state::{ServerState, EnvConfig, TwitchOauthSecrets, TwitchOauth, RedisRateLimiters}; +use crate::server_state::{ServerState, EnvConfig, TwitchOauthSecrets, TwitchOauth, RedisRateLimiters, InMemoryCaches}; use crate::threads::ip_banlist_set::IpBanlistSet; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; @@ -291,8 +291,10 @@ async fn main() -> AnyhowResult<()> { )?; // In-Memory Cache - let cache_ttl = easyenv::get_env_duration_seconds_or_default("VOICE_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60)); - let voice_list_cache = SingleItemTtlCache::create_with_duration(cache_ttl); + let voice_list_cache_ttl = easyenv::get_env_duration_seconds_or_default("VOICE_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60)); + let category_list_cache_ttl = easyenv::get_env_duration_seconds_or_default("CATEGORY_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60)); + let voice_list_cache = SingleItemTtlCache::create_with_duration(voice_list_cache_ttl); + let category_list_cache = SingleItemTtlCache::create_with_duration(category_list_cache_ttl); // NB: This secret really isn't too important. // We can even rotate it without too much impact to users. @@ -360,7 +362,10 @@ async fn main() -> AnyhowResult<()> { audio_uploads_bucket_root, sort_key_crypto, ip_banlist, - voice_list_cache, + caches: InMemoryCaches { + voice_list: voice_list_cache, + category_list: category_list_cache, + }, twitch_oauth: TwitchOauth { secrets: TwitchOauthSecrets { client_id: twitch_secrets.app_client_id, diff --git a/crates/service/storyteller_web/src/server_state.rs b/crates/service/storyteller_web/src/server_state.rs index cc1194b4cf..3d62a80690 100644 --- a/crates/service/storyteller_web/src/server_state.rs +++ b/crates/service/storyteller_web/src/server_state.rs @@ -11,6 +11,7 @@ use database_queries::mediators::firehose_publisher::FirehosePublisher; use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; +use crate::http_server::endpoints::categories::list_tts_categories::DisplayCategory; /// State that is injected into every endpoint. #[derive(Clone)] @@ -45,8 +46,7 @@ pub struct ServerState { pub ip_banlist: IpBanlistSet, - /// In-memory caches with TTL-based eviction. Contains a list of all voices. - pub voice_list_cache: SingleItemTtlCache>, + pub caches: InMemoryCaches, pub twitch_oauth: TwitchOauth, } @@ -89,3 +89,13 @@ pub struct RedisRateLimiters { /// A rate limiter for TTS and W2L uploads pub model_upload: RedisRateLimiter, } + +/// In-memory caches +#[derive(Clone)] +pub struct InMemoryCaches { + /// In-memory caches with TTL-based eviction. Contains a list of all voices. + pub voice_list: SingleItemTtlCache>, + + /// In-memory caches with TTL-based eviction. Contains a list of all TTS categories. + pub category_list: SingleItemTtlCache>, +} diff --git a/crates/service/storyteller_web/src/util/caching/single_item_ttl_cache.rs b/crates/service/storyteller_web/src/util/caching/single_item_ttl_cache.rs index 32cb756213..a3ce6117c1 100644 --- a/crates/service/storyteller_web/src/util/caching/single_item_ttl_cache.rs +++ b/crates/service/storyteller_web/src/util/caching/single_item_ttl_cache.rs @@ -4,9 +4,12 @@ use lru_time_cache::LruCache; use std::sync::{Arc, Mutex, PoisonError, MutexGuard}; use std::time::Duration; +/// NB: There's only ONE ITEM of ONE TYPE in the cache. We can use a single key. const CACHE_KEY : &'static str = "ITEM"; /// Store a single payload in the cache +/// There's only ONE ITEM of ONE TYPE in the cache. +/// This is essentially a singleton cache with expiry. #[derive(Clone)] pub struct SingleItemTtlCache { cache: Arc>>, From c7fa4b65bcf1e99b5f8d0d00d385b26138d849af Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Mar 2022 07:02:18 -0500 Subject: [PATCH 1027/1042] document how to make CSV reports --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 549416ce0b..1dec2a4e4c 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,19 @@ Development notes [Development notes README](_docs/development_notes.md) +Dump a MySQL report +------------------- + +``` +mysql -u storyteller \ + -pPasswordHere \ + -h IP_HERE \ + -P 3306 \ + -D storyteller \ + -B \ + -e "select * from voice_clone_requests;" | sed "s/'/\'/;s/\t/\",\"/g;s/^/\"/;s/$/\"/;s/\n//g" > voice_clone_report.csv +``` + TODO ---- From 3d696b71eedc3dfa204fb472da7de1f8624b3668 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Mar 2022 23:37:36 -0500 Subject: [PATCH 1028/1042] Add two new EventResponse types: TtsCommandPreset, TtsCommandCustom --- .../src/complex_models/event_responses.rs | 26 +++++++++++++++++++ .../validations/validate_event_response.rs | 13 ++++++++++ .../storyteller_web/src/tts_inference_job.rs | 2 ++ 3 files changed, 41 insertions(+) diff --git a/crates/lib/database_queries/src/complex_models/event_responses.rs b/crates/lib/database_queries/src/complex_models/event_responses.rs index a49054696f..4c230e9545 100644 --- a/crates/lib/database_queries/src/complex_models/event_responses.rs +++ b/crates/lib/database_queries/src/complex_models/event_responses.rs @@ -1,4 +1,17 @@ use container_common::anyhow_result::AnyhowResult; +use std::collections::HashMap; + +/// How TTS commands should be prefixed: !voiceName, /voiceName, etc. +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Copy)] +#[serde(rename_all = "snake_case")] +pub enum CommandPrefixType { + /// Prefix voice commands with "!" + Exclamation, + /// Prefix voice commands with "!" or "/" + ExclamationOrSlash, + /// Prefix voice commands with "/" + Slash, +} #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] #[serde(rename_all = "snake_case")] @@ -16,6 +29,19 @@ pub enum EventResponse { TtsRandomVoice { tts_model_tokens: Vec, }, + + /// Use TTS commands that FakeYou recommends. + TtsCommandPresets { + command_prefix_type: CommandPrefixType, + }, + + /// Use your own TTS commands. + /// The map is from "!command" (without the "/" or "!" prefix) to model token. + /// Command is lower case, alphanum only, and compared case insensitive. + TtsCommandCustom { + command_prefix_type: CommandPrefixType, + command_map: HashMap, + }, } #[cfg(test)] diff --git a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_response.rs b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_response.rs index 8d01fba03a..3356246df4 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_response.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/twitch/event_rules/validations/validate_event_response.rs @@ -20,6 +20,19 @@ pub fn validate_event_response(event_response: &EventResponse) -> Result<(), Str } } }, + EventResponse::TtsCommandPresets { .. } => { + // Implicitly valid + } + EventResponse::TtsCommandCustom { command_map, .. } => { + for (command, model_token) in command_map.iter() { + if command.trim().is_empty() { + return Err("one or more commands is invalid".to_string()); + } + if model_token.is_empty() { + return Err("one or more model tokens is invalid".to_string()); + } + } + }, } Ok(()) diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index 3183782df9..1d952670b6 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -927,6 +927,8 @@ async fn process_job( Some(&inference_result_token) ).await?; + info!("TTS Done. Original text was: {}", &job.raw_inference_text); + inferencer.firehose_publisher.tts_inference_finished( job.maybe_creator_user_token.as_deref(), &model_record.model_token, From 86cd30e134ab62464ef8271a72dca782ba8b0a77 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Mar 2022 23:58:03 -0500 Subject: [PATCH 1029/1042] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 549416ce0b..b0044baef1 100644 --- a/README.md +++ b/README.md @@ -58,4 +58,5 @@ Notes / TODOs: * Jobs for analytics queries +* Pull jobs into own repo From db7e7705139e6fccaff9b514c6501a93e6b0b8cc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Mar 2022 23:58:04 -0400 Subject: [PATCH 1030/1042] begin work on logging schema for twitch --- _migrations/2022-03-14-035745_twitch_event_match_logs/down.sql | 1 + _migrations/2022-03-14-035745_twitch_event_match_logs/up.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 _migrations/2022-03-14-035745_twitch_event_match_logs/down.sql create mode 100644 _migrations/2022-03-14-035745_twitch_event_match_logs/up.sql diff --git a/_migrations/2022-03-14-035745_twitch_event_match_logs/down.sql b/_migrations/2022-03-14-035745_twitch_event_match_logs/down.sql new file mode 100644 index 0000000000..291a97c5ce --- /dev/null +++ b/_migrations/2022-03-14-035745_twitch_event_match_logs/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` \ No newline at end of file diff --git a/_migrations/2022-03-14-035745_twitch_event_match_logs/up.sql b/_migrations/2022-03-14-035745_twitch_event_match_logs/up.sql new file mode 100644 index 0000000000..33cf74b578 --- /dev/null +++ b/_migrations/2022-03-14-035745_twitch_event_match_logs/up.sql @@ -0,0 +1 @@ +-- Your SQL goes here \ No newline at end of file From c573a923b46e9b9a49e04dbce61674b3b934ed7c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 15 Mar 2022 23:56:04 -0400 Subject: [PATCH 1031/1042] Add analytics queries --- analytics/user_retention.sql | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 analytics/user_retention.sql diff --git a/analytics/user_retention.sql b/analytics/user_retention.sql new file mode 100644 index 0000000000..4b54794df4 --- /dev/null +++ b/analytics/user_retention.sql @@ -0,0 +1,60 @@ + +-- All logged in users with results in the last X days +select u.created_at from users as u +where u.token IN +(select distinct u.token from tts_inference_jobs as j join users u on u.token = j.maybe_creator_user_token where j.created_at > now() - interval 7 day) +order by u.created_at ASC; + + +-- All logged in users with results in the last X days that are older than Y +select u.created_at from users as u +where u.token IN +(select distinct u.token from tts_inference_jobs as j join users u on u.token = j.maybe_creator_user_token where j.created_at > now() - interval 30 day) +AND u.created_at < now() - interval 30 day +order by u.created_at ASC; + +-- ====================== FEBRUARY COHORT =================================== + +-- [1] All users created between X and Y +select count(*) from users +where created_at > now() - interval 60 day +and created_at < now() - interval 30 day; + +-- [2] Active users from that cohort ^ +select count(*) from users as u +where u.token IN +(select distinct u.token from tts_inference_jobs as j join users u on u.token = j.maybe_creator_user_token where j.created_at > now() - interval 30 day) +AND created_at > now() - interval 60 day +AND created_at < now() - interval 30 day; + + + +-- ====================== JANUARY COHORT =================================== + +-- [1] All users created between X and Y +select count(*) from users +where created_at > now() - interval 90 day +and created_at < now() - interval 60 day; + +-- [2] Active users from that cohort ^ +select count(*) from users as u +where u.token IN +(select distinct u.token from tts_inference_jobs as j join users u on u.token = j.maybe_creator_user_token where j.created_at > now() - interval 30 day) +AND created_at > now() - interval 90 day +AND created_at < now() - interval 60 day; + + + +(select distinct u.token from tts_inference_jobs as j join users u on u.token = j.maybe_creator_user_token where j.created_at > now() - interval 30 day) + + +select j.creator_ip_address, count(*) as number +from tts_inference_jobs as j +where j.created_at > now() - interval 1 day +group by j.creator_ip_address +order by number desc +limit 100; + + + + From 74d7f3c276365257f66a5940be0adcd14f367049 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Mar 2022 06:02:42 -0400 Subject: [PATCH 1032/1042] Investor mode: priority, cookie to enable --- .../investor_demo/default_redirect.rs | 18 +++++++ .../endpoints/investor_demo/demo_cookie.rs | 10 ++++ .../disable_demo_mode_handler.rs | 49 +++++++++++++++++++ .../investor_demo/enable_demo_mode_handler.rs | 48 ++++++++++++++++++ .../endpoints/investor_demo/mod.rs | 4 ++ .../src/http_server/endpoints/mod.rs | 1 + .../endpoints/tts/enqueue_infer_tts.rs | 42 ++++++++++++++-- crates/service/storyteller_web/src/routes.rs | 29 ++++++++++- 8 files changed, 196 insertions(+), 5 deletions(-) create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/investor_demo/default_redirect.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/investor_demo/demo_cookie.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs create mode 100644 crates/service/storyteller_web/src/http_server/endpoints/investor_demo/mod.rs diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/default_redirect.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/default_redirect.rs new file mode 100644 index 0000000000..535b61c44c --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/default_redirect.rs @@ -0,0 +1,18 @@ + +pub const DEFAULT_INVESTOR_REDIRECT : &'static str = "https://fakeyou.com"; + +pub fn redirect_is_allowed(redirect_url: &str) -> bool { + match redirect_url { + "http://jungle.horse" => true, + "http://jungle.horse/" => true, + "https://create.storyteller.io" => true, + "https://create.storyteller.io/" => true, + "https://fakeyou.com" => true, + "https://fakeyou.com/" => true, + "https://jungle.horse" => true, + "https://jungle.horse/" => true, + "https://storyteller.io" => true, + "https://storyteller.io/" => true, + _ => false, + } +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/demo_cookie.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/demo_cookie.rs new file mode 100644 index 0000000000..540ffe58a0 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/demo_cookie.rs @@ -0,0 +1,10 @@ +//! A temporary cookie for investor demo + +use actix_web::HttpRequest; + +pub const STORYTELLER_DEMO_COOKIE_NAME : &'static str = "storyteller_demo"; + +pub fn request_has_demo_cookie(http_request: &HttpRequest) -> bool { + // NB: Payload of the cookie does not matter. + http_request.cookie(STORYTELLER_DEMO_COOKIE_NAME).is_some() +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs new file mode 100644 index 0000000000..52fdabca6c --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs @@ -0,0 +1,49 @@ +use actix_http::header; +use actix_http::http::StatusCode; +use actix_web::cookie::Cookie; +use actix_web::web::Query; +use actix_web::{HttpResponse, HttpRequest, Responder, get, web, HttpMessage}; +use crate::http_server::endpoints::investor_demo::default_redirect::{DEFAULT_INVESTOR_REDIRECT, redirect_is_allowed}; +use crate::http_server::endpoints::investor_demo::demo_cookie::STORYTELLER_DEMO_COOKIE_NAME; +use crate::server_state::ServerState; +use hyper::header::LOCATION; +use log::info; +use std::ops::Deref; +use std::sync::Arc; +use time::OffsetDateTime; + +#[derive(Deserialize)] +pub struct QueryFields { + redirect_to: Option, +} + +pub async fn disable_demo_mode_handler( + http_request: HttpRequest, + query: Query, + server_state: web::Data> +) -> impl Responder +{ + let unsafe_redirect = query.redirect_to + .map(|r| r.clone()) + .unwrap_or(DEFAULT_INVESTOR_REDIRECT.to_string()); + + let redirect_allowed = redirect_is_allowed(&unsafe_redirect); + + let safe_redirect = if redirect_allowed { + unsafe_redirect + } else { + DEFAULT_INVESTOR_REDIRECT.to_string() + }; + + // Kill the cookie. + let cookie = Cookie::build(STORYTELLER_DEMO_COOKIE_NAME, "") + .secure(server_state.env_config.cookie_secure) // HTTPS-only + .expires(OffsetDateTime::unix_epoch()) + .http_only(false) // This is meant to be exposed to Javascript! + .permanent() + .finish(); + + HttpResponse::build(StatusCode::FOUND) + .append_header((header::LOCATION, url.to_string())) + .del_cookie(&cookie) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs new file mode 100644 index 0000000000..d08af71aa9 --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs @@ -0,0 +1,48 @@ +use actix_http::header; +use actix_http::http::StatusCode; +use actix_web::cookie::Cookie; +use actix_web::web::Query; +use actix_web::{HttpResponse, HttpRequest, Responder, get, web, HttpMessage}; +use crate::http_server::endpoints::investor_demo::default_redirect::{DEFAULT_INVESTOR_REDIRECT, redirect_is_allowed}; +use crate::http_server::endpoints::investor_demo::demo_cookie::STORYTELLER_DEMO_COOKIE_NAME; +use crate::server_state::ServerState; +use hyper::header::LOCATION; +use log::info; +use std::ops::Deref; +use std::sync::Arc; +use time::OffsetDateTime; + +#[derive(Deserialize)] +pub struct QueryFields { + redirect_to: Option, +} + +pub async fn enable_demo_mode_handler( + http_request: HttpRequest, + query: Query, + server_state: web::Data> +) -> impl Responder +{ + let unsafe_redirect = query.redirect_to + .map(|r| r.clone()) + .unwrap_or(DEFAULT_INVESTOR_REDIRECT.to_string()); + + let redirect_allowed = redirect_is_allowed(&unsafe_redirect); + + let safe_redirect = if redirect_allowed { + unsafe_redirect + } else { + DEFAULT_INVESTOR_REDIRECT.to_string() + }; + + // Add the cookie. + let cookie = Cookie::build(STORYTELLER_DEMO_COOKIE_NAME, "true") + .secure(server_state.env_config.cookie_secure) // HTTPS-only + .http_only(false) // This is meant to be exposed to Javascript! + .permanent() + .finish(); + + HttpResponse::build(StatusCode::FOUND) + .append_header((header::LOCATION, url.to_string())) + .cookie(cookie) +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/mod.rs new file mode 100644 index 0000000000..92c427c8cf --- /dev/null +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/mod.rs @@ -0,0 +1,4 @@ +pub mod demo_cookie; +pub mod disable_demo_mode_handler; +pub mod enable_demo_mode_handler; +pub mod default_redirect; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/storyteller_web/src/http_server/endpoints/mod.rs index aec49bb864..e7dc1acb4f 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/mod.rs @@ -1,6 +1,7 @@ pub mod api_tokens; pub mod categories; pub mod events; +pub mod investor_demo; pub mod leaderboard; pub mod misc; pub mod moderation; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 85b96da0d0..c68791dc24 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -23,6 +23,10 @@ use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; use user_input_common::check_for_slurs::contains_slurs; +use crate::http_server::endpoints::investor_demo::demo_cookie::request_has_demo_cookie; + +// TODO: Temporary for investor demo +const STORYTELLER_DEMO_COOKIE_NAME : &'static str = "storyteller_demo"; #[derive(Deserialize)] pub struct InferTtsRequest { @@ -30,6 +34,7 @@ pub struct InferTtsRequest { tts_model_token: String, inference_text: String, creator_set_visibility: Option, + is_storyteller_demo: Option, } #[derive(Serialize)] @@ -89,9 +94,36 @@ pub async fn infer_tts_handler( InferTtsError::ServerError })?; + // ==================== PRIORITY ==================== // + + // Give logged in users execution priority. + let mut priority_level = if maybe_user_session.is_some() { 1 } else { 0 }; + + // TODO/TEMP: Give investors even more priority + let mut is_investor = false; + + // TODO/TEMP: The storyteller.io website's AJAX calls will set this. + // This is just for the YCombinator demo. + match request.is_storyteller_demo { + Some(true) => { + is_investor = true; + }, + _ => {}, + }; + + // TODO/TEMP: The storyteller.io website will redirect and establish this cookie. + // This is just for the YCombinator demo. + if request_has_demo_cookie(&http_request) { + is_investor = true; + } + + if is_investor { + priority_level = 2; + } + // ==================== RATE LIMIT ==================== // - let rate_limiter = match maybe_user_session { + let mut rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, Some(ref user) => { if user.is_banned { @@ -101,6 +133,11 @@ pub async fn infer_tts_handler( }, }; + // TODO/TEMP + if is_investor { + rate_limiter = &server_state.redis_rate_limiters.logged_in; + } + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(InferTtsError::RateLimited); } @@ -156,9 +193,6 @@ pub async fn infer_tts_handler( InferTtsError::ServerError })?; - // Give logged in users execution priority. - let priority_level = if maybe_user_session.is_some() { 1 } else { 0 }; - info!("Creating w2l inference job record..."); let query_result = sqlx::query!( diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index 5dda962444..544b4ba020 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -13,6 +13,8 @@ use crate::http_server::endpoints::categories::get_category::get_category_handle use crate::http_server::endpoints::categories::list_tts_categories::list_tts_categories_handler; use crate::http_server::endpoints::categories::list_tts_model_assigned_categories::list_tts_model_assigned_categories_handler; use crate::http_server::endpoints::events::list_events::list_events_handler; +use crate::http_server::endpoints::investor_demo::disable_demo_mode_handler::disable_demo_mode_handler; +use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enable_demo_mode_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::misc::default_route_404::default_route_404; use crate::http_server::endpoints::misc::detect_locale_handler::detect_locale_handler; @@ -37,6 +39,7 @@ use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler; use crate::http_server::endpoints::moderation::user_roles::set_user_role::set_user_role_handler; use crate::http_server::endpoints::moderation::users::list_users::list_users_handler; +use crate::http_server::endpoints::service::health_check_handler::get_health_check_handler; use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handler; use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; @@ -85,7 +88,6 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; -use crate::http_server::endpoints::service::health_check_handler::get_health_check_handler; pub fn add_routes (app: App) -> App where @@ -106,6 +108,7 @@ pub fn add_routes (app: App) -> App app = add_api_token_routes(app); /* /api_tokens */ app = add_voice_clone_request_routes(app); /* /voice_clone_requests */ app = add_twitch_routes(app); /* /twitch */ // TODO: MAYBE TEMPORARY + app = add_investor_demo_routes(app); /* /demo_mode */ // TODO: DEFINITELY TEMPORARY // ==================== SERVICE ==================== app.service( @@ -615,6 +618,30 @@ fn add_voice_clone_request_routes (app: App) -> App ) } +// ==================== INVESTOR DEMO MODE ==================== + +fn add_investor_demo_routes (app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service(web::scope("/demo_mode") + .service(web::resource("/enable") + .route(web::post().to(enable_demo_mode_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/disable") + .route(web::post().to(disable_demo_mode_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) +} // ==================== TWITCH ROUTES ==================== From 838934880f715c20247b90f4d6deede7024f0583 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 18 Mar 2022 06:39:08 -0400 Subject: [PATCH 1033/1042] builder didn't match impl --- .../endpoints/investor_demo/disable_demo_mode_handler.rs | 1 + .../endpoints/investor_demo/enable_demo_mode_handler.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs index 52fdabca6c..7326bfedef 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs @@ -46,4 +46,5 @@ pub async fn disable_demo_mode_handler( HttpResponse::build(StatusCode::FOUND) .append_header((header::LOCATION, url.to_string())) .del_cookie(&cookie) + .finish() } diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs index d08af71aa9..a93acf6a1e 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs @@ -45,4 +45,5 @@ pub async fn enable_demo_mode_handler( HttpResponse::build(StatusCode::FOUND) .append_header((header::LOCATION, url.to_string())) .cookie(cookie) + .finish() } From 07923bb34f22adfd8fcaf75a6aea86df91ec3d4d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 21 Mar 2022 23:55:59 -0400 Subject: [PATCH 1034/1042] twitch event logs schema wip --- .../down.sql | 6 +++++- .../up.sql | 21 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/_migrations/2022-03-14-035745_twitch_event_match_logs/down.sql b/_migrations/2022-03-14-035745_twitch_event_match_logs/down.sql index 291a97c5ce..ff2b203e89 100644 --- a/_migrations/2022-03-14-035745_twitch_event_match_logs/down.sql +++ b/_migrations/2022-03-14-035745_twitch_event_match_logs/down.sql @@ -1 +1,5 @@ --- This file should undo anything in `up.sql` \ No newline at end of file +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE twitch_event_match_logs; diff --git a/_migrations/2022-03-14-035745_twitch_event_match_logs/up.sql b/_migrations/2022-03-14-035745_twitch_event_match_logs/up.sql index 33cf74b578..a6df161c47 100644 --- a/_migrations/2022-03-14-035745_twitch_event_match_logs/up.sql +++ b/_migrations/2022-03-14-035745_twitch_event_match_logs/up.sql @@ -1 +1,20 @@ --- Your SQL goes here \ No newline at end of file +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Log any event matching (bits, channel points, etc.) +CREATE TABLE twitch_event_match_logs( + -- Not used for anything except replication. + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + + -- Type of event, eg. + event_type VARCHAR(32) NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP DEFAULT NULL, + + -- INDICES -- + PRIMARY KEY (id) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 836536226ff6931df6b3229ada91b1abe01fe0ff Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 22 Mar 2022 15:07:59 -0400 Subject: [PATCH 1035/1042] fix build error with safe redirects (would have caught if mac not dumb) --- analytics/user_retention.sql | 15 +++++++++++++++ .../investor_demo/disable_demo_mode_handler.rs | 7 ++++--- .../investor_demo/enable_demo_mode_handler.rs | 7 ++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/analytics/user_retention.sql b/analytics/user_retention.sql index 4b54794df4..da1362f17c 100644 --- a/analytics/user_retention.sql +++ b/analytics/user_retention.sql @@ -56,5 +56,20 @@ order by number desc limit 100; +select count(*) from tts_inference_jobs +where created_at > now() - interval 1 day ; + +select count(distinct creator_ip_address) from tts_inference_jobs +where created_at > now() - interval 1 day ; + + + +select j.creator_ip_address, count(*) as number +from tts_inference_jobs as j +where j.created_at > now() - interval 1 day +group by j.creator_ip_address +order by number desc +limit 100; + diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs index 7326bfedef..7d2b61c503 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs @@ -24,12 +24,13 @@ pub async fn disable_demo_mode_handler( ) -> impl Responder { let unsafe_redirect = query.redirect_to - .map(|r| r.clone()) + .as_deref() + .map(|r| r.to_string()) .unwrap_or(DEFAULT_INVESTOR_REDIRECT.to_string()); let redirect_allowed = redirect_is_allowed(&unsafe_redirect); - let safe_redirect = if redirect_allowed { + let safe_redirect_url = if redirect_allowed { unsafe_redirect } else { DEFAULT_INVESTOR_REDIRECT.to_string() @@ -44,7 +45,7 @@ pub async fn disable_demo_mode_handler( .finish(); HttpResponse::build(StatusCode::FOUND) - .append_header((header::LOCATION, url.to_string())) + .append_header((header::LOCATION, safe_redirect_url.to_string())) .del_cookie(&cookie) .finish() } diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs index a93acf6a1e..8af17b3fd1 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs @@ -24,12 +24,13 @@ pub async fn enable_demo_mode_handler( ) -> impl Responder { let unsafe_redirect = query.redirect_to - .map(|r| r.clone()) + .as_deref() + .map(|r| r.to_string()) .unwrap_or(DEFAULT_INVESTOR_REDIRECT.to_string()); let redirect_allowed = redirect_is_allowed(&unsafe_redirect); - let safe_redirect = if redirect_allowed { + let safe_redirect_url = if redirect_allowed { unsafe_redirect } else { DEFAULT_INVESTOR_REDIRECT.to_string() @@ -43,7 +44,7 @@ pub async fn enable_demo_mode_handler( .finish(); HttpResponse::build(StatusCode::FOUND) - .append_header((header::LOCATION, url.to_string())) + .append_header((header::LOCATION, safe_redirect_url.to_string())) .cookie(cookie) .finish() } From da8df046fea590a9081eb34c83a87186392d8483 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 22 Mar 2022 15:57:29 -0400 Subject: [PATCH 1036/1042] more build errors, seriously wtf --- .../event_handlers/bits_event_handler.rs | 2 ++ .../event_handlers/channel_points_event_handler.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs index a8610ce6fa..ee95769659 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/bits_event_handler.rs @@ -124,6 +124,8 @@ impl BitsEventHandler { .await?; } } + EventResponse::TtsCommandPresets { .. } => {} // TODO + EventResponse::TtsCommandCustom { .. } => {} // TODO } Ok(()) diff --git a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs index 4abe29a971..aafa50bddf 100644 --- a/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs +++ b/crates/service/twitch_pubsub_subscriber/src/threads/twitch_pubsub_user_subscriber/event_handlers/channel_points_event_handler.rs @@ -133,6 +133,8 @@ impl ChannelPointsEventHandler { .await?; } } + EventResponse::TtsCommandPresets { .. } => {} // TODO + EventResponse::TtsCommandCustom { .. } => {} // TODO } Ok(()) From fa684466b262d35c8053a1d4d405fc3b6cea826e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 23 Mar 2022 10:27:14 -0400 Subject: [PATCH 1037/1042] new redirect url --- .../src/http_server/endpoints/investor_demo/default_redirect.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/default_redirect.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/default_redirect.rs index 535b61c44c..95ac8fbc41 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/default_redirect.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/default_redirect.rs @@ -8,7 +8,7 @@ pub fn redirect_is_allowed(redirect_url: &str) -> bool { "https://create.storyteller.io" => true, "https://create.storyteller.io/" => true, "https://fakeyou.com" => true, - "https://fakeyou.com/" => true, + "https://fakeyou.com/clone" => true, "https://jungle.horse" => true, "https://jungle.horse/" => true, "https://storyteller.io" => true, From 67df72d66996fd3b12769c4294bf554d1907e08e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 23 Mar 2022 10:29:28 -0400 Subject: [PATCH 1038/1042] http GET --- crates/service/storyteller_web/src/routes.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/service/storyteller_web/src/routes.rs b/crates/service/storyteller_web/src/routes.rs index 544b4ba020..432503075d 100644 --- a/crates/service/storyteller_web/src/routes.rs +++ b/crates/service/storyteller_web/src/routes.rs @@ -633,11 +633,11 @@ fn add_investor_demo_routes (app: App) -> App { app.service(web::scope("/demo_mode") .service(web::resource("/enable") - .route(web::post().to(enable_demo_mode_handler)) + .route(web::get().to(enable_demo_mode_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) .service(web::resource("/disable") - .route(web::post().to(disable_demo_mode_handler)) + .route(web::get().to(disable_demo_mode_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) ) From 732ade7540b517f7735df7aae542bb29be6c072e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 23 Mar 2022 19:02:43 -0400 Subject: [PATCH 1039/1042] Set cookie domain explicitly --- .../src/request/get_request_host.rs | 8 +++++++ .../lib/http_server_common/src/request/mod.rs | 1 + .../disable_demo_mode_handler.rs | 22 +++++++++++++++---- .../investor_demo/enable_demo_mode_handler.rs | 20 ++++++++++++++--- 4 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 crates/lib/http_server_common/src/request/get_request_host.rs diff --git a/crates/lib/http_server_common/src/request/get_request_host.rs b/crates/lib/http_server_common/src/request/get_request_host.rs new file mode 100644 index 0000000000..159d69facc --- /dev/null +++ b/crates/lib/http_server_common/src/request/get_request_host.rs @@ -0,0 +1,8 @@ +use actix_web::HttpRequest; +use actix_web::http::header::HOST; +use crate::request::get_request_header_optional::get_request_header_optional; + +/// Get the request host +pub fn get_request_host(request: &HttpRequest) -> Option { + get_request_header_optional(request, HOST.as_str()) +} diff --git a/crates/lib/http_server_common/src/request/mod.rs b/crates/lib/http_server_common/src/request/mod.rs index c9c51cb571..c4a13b3a69 100644 --- a/crates/lib/http_server_common/src/request/mod.rs +++ b/crates/lib/http_server_common/src/request/mod.rs @@ -1,3 +1,4 @@ pub mod get_request_header_optional; +pub mod get_request_host; pub mod get_request_ip; pub mod parse_accept_language; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs index 7d2b61c503..1d26ef0cfc 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs @@ -11,6 +11,7 @@ use log::info; use std::ops::Deref; use std::sync::Arc; use time::OffsetDateTime; +use http_server_common::request::get_request_host::get_request_host; #[derive(Deserialize)] pub struct QueryFields { @@ -36,13 +37,26 @@ pub async fn disable_demo_mode_handler( DEFAULT_INVESTOR_REDIRECT.to_string() }; - // Kill the cookie. - let cookie = Cookie::build(STORYTELLER_DEMO_COOKIE_NAME, "") + let maybe_host = get_request_host(&http_request); + let maybe_cookie_domain = match maybe_host.as_deref() { + Some("jungle.horse") | Some("api.jungle.horse") => Some(".jungle.horse"), + Some("fakeyou.com") | Some("api.fakeyou.com") => Some(".fakeyou.com"), + Some("storyteller.io") | Some("api.storyteller.io") => Some(".storyteller.io"), + _ => None, + }; + + // Kill the cookie + let mut cookie_builder = Cookie::build(STORYTELLER_DEMO_COOKIE_NAME, "") .secure(server_state.env_config.cookie_secure) // HTTPS-only .expires(OffsetDateTime::unix_epoch()) .http_only(false) // This is meant to be exposed to Javascript! - .permanent() - .finish(); + .permanent(); + + if let Some(cookie_domain) = maybe_cookie_domain { + cookie_builder = cookie_builder.domain("api.storyteller.io"); + } + + let cookie = cookie_builder.finish(); HttpResponse::build(StatusCode::FOUND) .append_header((header::LOCATION, safe_redirect_url.to_string())) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs index 8af17b3fd1..fc181f0a03 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs @@ -11,6 +11,7 @@ use log::info; use std::ops::Deref; use std::sync::Arc; use time::OffsetDateTime; +use http_server_common::request::get_request_host::get_request_host; #[derive(Deserialize)] pub struct QueryFields { @@ -36,12 +37,25 @@ pub async fn enable_demo_mode_handler( DEFAULT_INVESTOR_REDIRECT.to_string() }; + let maybe_host = get_request_host(&http_request); + let maybe_cookie_domain = match maybe_host.as_deref() { + Some("jungle.horse") | Some("api.jungle.horse") => Some(".jungle.horse"), + Some("fakeyou.com") | Some("api.fakeyou.com") => Some(".fakeyou.com"), + Some("storyteller.io") | Some("api.storyteller.io") => Some(".storyteller.io"), + _ => None, + }; + // Add the cookie. - let cookie = Cookie::build(STORYTELLER_DEMO_COOKIE_NAME, "true") + let mut cookie_builder = Cookie::build(STORYTELLER_DEMO_COOKIE_NAME, "true") .secure(server_state.env_config.cookie_secure) // HTTPS-only .http_only(false) // This is meant to be exposed to Javascript! - .permanent() - .finish(); + .permanent(); + + if let Some(cookie_domain) = maybe_cookie_domain { + cookie_builder = cookie_builder.domain("api.storyteller.io"); + } + + let cookie = cookie_builder.finish(); HttpResponse::build(StatusCode::FOUND) .append_header((header::LOCATION, safe_redirect_url.to_string())) From 340bedbcee472fb98350e06339163773dc67132f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 23 Mar 2022 19:06:23 -0400 Subject: [PATCH 1040/1042] bah, wrong cookie domain --- .../endpoints/investor_demo/disable_demo_mode_handler.rs | 2 +- .../endpoints/investor_demo/enable_demo_mode_handler.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs index 1d26ef0cfc..e21aa5ad19 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs @@ -53,7 +53,7 @@ pub async fn disable_demo_mode_handler( .permanent(); if let Some(cookie_domain) = maybe_cookie_domain { - cookie_builder = cookie_builder.domain("api.storyteller.io"); + cookie_builder = cookie_builder.domain(cookie_domain); } let cookie = cookie_builder.finish(); diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs index fc181f0a03..97affac701 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs @@ -52,7 +52,7 @@ pub async fn enable_demo_mode_handler( .permanent(); if let Some(cookie_domain) = maybe_cookie_domain { - cookie_builder = cookie_builder.domain("api.storyteller.io"); + cookie_builder = cookie_builder.domain(cookie_domain); } let cookie = cookie_builder.finish(); From 4a72295eee9e16b0b00469886b9a512c06a67a9c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 23 Mar 2022 22:54:04 -0400 Subject: [PATCH 1041/1042] Add path to cookie --- .../endpoints/investor_demo/disable_demo_mode_handler.rs | 4 ++++ .../endpoints/investor_demo/enable_demo_mode_handler.rs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs index e21aa5ad19..ddbd070863 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/disable_demo_mode_handler.rs @@ -50,6 +50,10 @@ pub async fn disable_demo_mode_handler( .secure(server_state.env_config.cookie_secure) // HTTPS-only .expires(OffsetDateTime::unix_epoch()) .http_only(false) // This is meant to be exposed to Javascript! + // NB: Since we're setting this from direct HTTP access rather than XHR, the browser + // implicitly sets the path used to access the cookie, ie. '/demo_mode', which is not what + // we want! + .path("/") .permanent(); if let Some(cookie_domain) = maybe_cookie_domain { diff --git a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs index 97affac701..2f75d9d206 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/investor_demo/enable_demo_mode_handler.rs @@ -49,6 +49,10 @@ pub async fn enable_demo_mode_handler( let mut cookie_builder = Cookie::build(STORYTELLER_DEMO_COOKIE_NAME, "true") .secure(server_state.env_config.cookie_secure) // HTTPS-only .http_only(false) // This is meant to be exposed to Javascript! + // NB: Since we're setting this from direct HTTP access rather than XHR, the browser + // implicitly sets the path used to access the cookie, ie. '/demo_mode', which is not what + // we want! + .path("/") .permanent(); if let Some(cookie_domain) = maybe_cookie_domain { From ef7227509eced57fbe2fb68d4cf3f0019b389835 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 25 Mar 2022 23:33:57 -0400 Subject: [PATCH 1042/1042] add token --- _migrations/2022-03-14-035745_twitch_event_match_logs/up.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/_migrations/2022-03-14-035745_twitch_event_match_logs/up.sql b/_migrations/2022-03-14-035745_twitch_event_match_logs/up.sql index a6df161c47..25e4fee799 100644 --- a/_migrations/2022-03-14-035745_twitch_event_match_logs/up.sql +++ b/_migrations/2022-03-14-035745_twitch_event_match_logs/up.sql @@ -7,6 +7,8 @@ CREATE TABLE twitch_event_match_logs( -- Not used for anything except replication. id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + token VARCHAR(32) NOT NULL, + -- Type of event, eg. event_type VARCHAR(32) NOT NULL, @@ -15,6 +17,7 @@ CREATE TABLE twitch_event_match_logs( deleted_at TIMESTAMP DEFAULT NULL, -- INDICES -- - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE KEY (token) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

Are you looking for an API? Join our Discord!

T%_ldfL3uyx0Sm36AQu-=q|E1o|+dWAm$8NsmFYNg36+hx85MTw)u;B2=D^}Zu@c$dEe7iV6UHQo2X(78qXnh&$0@kYCp zyrZ}bBy4v1_`l&M|Bk39#y7QZ)>n_!$w{kR9ZLtLN@{XnH!m-rWCE#(xo;?$m=fvv zLP)4__S8HkFEF!QDG8Dt&&dSkCU-Fx38CLpU|kICu;@BHLI%Fm_X*pLuK~dTxl@s0 zd6%Yb!c z8N^Nd$@2kS!|rFqAH>S!Cb0HnJ&rO}3ac+VaE>iFt2KugqeP~y58qH{4U`7%aIOqH zQTE;kD?4L)a=hZ3dJ~0m)dSA&zoac~-E%HS$um;$yb`UCOc`P>oVUyTcg~luTUSI# z&EkCn>fj1W9DU1MAgZa8K4NXEK4N~Tr*@Ep z_xdlMbjmim_-s#Hf~@n-BJQ4U+qmc|Rq~?cgW7@~;ROsE8K)=&b~$w^77u_MVZ^O* z%5%bzXE=a8A5=5_zRBYI6h#NaS|y=8aHE&HaH_Jj+uR!I5%-V3)!h2ELzH*+iO*k>R0w z5DAr^Wo@g|8B|nz_DILHNDj}BD3SMf6AV-=AaviRD>1ykD~vy-o?kpom{?18^3H0G zmvd8Yig)ya=p#KQhPv?9`*L@OaP|DN;!Athf@Xz1fg?9f+f=|?HG=mPP78U}|*9LW{Fb8%&8Fa4&r!!XW@HLWCGAQ9>JFynDHkCDox5kDC zZ)bZsJsUh3<2X6_`?;r0VOBhp!WI#~m3TDvsu~Dr*hkR!oobXLUarDvMM%YC_KRF| zt=iY!dMRUKYt@)fd@a)4t{7I4VXo3NF;k+2lQ-EyRGx@P7kTopeW5}{g z{oQ3|!#rIC2afQwkJtL^Bg_x?Y{ko7!-9h}nC8c&2UaHD=qk*!8mV~5(A=uF!Zm!~ zAp9PG_UvbIC-iyO&u?ph#Et--J&pam3H5-i5J^@a1f&(Chxhw{g^~!~^-W>-YGr=5 z5+CPnUn*A(hgepGOaGL;zpkZhV+I48`fSoP@D>5C3>zEc58>>3pmj5hnSM4)iumAuLQnY{$Zk4v@`qmZ#Ws^@%oAI2^=fEl#$66 z@$PwX6{}r zvg@L-h=BBiNVlYP51k6qDAF-B(%mqWfYQ?4Db3KGLk%56cQ-@#JHB7eKXCukiGk~9Bpld+}0M)|EK)p5^Qd< z$a&c%oU>g{^U=JFH%>;&*?sGh!B$~ll~IZ|Q|WK=?fi!ya|Dp1Qbb*Uis~{spv;zJ zagCHCzM|k%&DHNWdi`M4El5qdfbRG%Ye6h+9?LZOh>JKj3h%p}RPv_0e1}0HJDInr z09}S7nIpEQr^{H-R^2XHlP18u@_Fcjy$SUsw~7`Ks*K`N70}$tNPwbLqxH7&as<_d zE&N>ZL%uhXTtRfJI9uSH50 zBWBrigS<8IZ~1XNh5Y9_Hb&dmt^jp0mLZ~LVwKMxaMlV|7*sl`D`AU$6q(H+k$G%_ zh6Gp5ld#g6R#feucPM2F3QRV(Jn>*_a~Iii$RIG+R+2|!&;RouXi%WVcNcApIWI;7 zv1*+eGrD&yqJ3fv6RZeY@=c(f{O4`Gkb%tD-#@2yvvMXHVfpB%c~JA8QhM96o=kF- z*ynN>JO1&cFtKT8VDM;1<_Mxrf{=#0lsGEJb&-Pe_0#0T0X4XD7B_{yaSF}PP(U|d z^e4)~>dewbPyQW`eMUr*yrNHz`VmL?K`Y9Y>$s?@y^98kVypR0 zgc6AwccLluuQYO*^ADimE%&Aj&viXzKL1F9xg$l=@X_Q`sEy?>!WevRN;7i15mCcY zunT>I0Px*G2iip`8c1`St{@9S;}U&M_pu+L?vP&74PHe0bSR5pdB+N+yW46FiqG=j zSz%eHF2tEq`P!<)Rzk9AUlyaE8VHxa!z2^r6^?V_ju7DyyW{uf4(NFTBBO?RY}1@G zpfrSeW#`rRlw2|D$ujv5FB$|EY5cSm39cHJ3O-Jolg>Yna0Xcrs^g;Q^(Lo9^H$pCU2i;~E`JLd8^<{^R4oaf^7G9_Fo-(<4 z-Vc4TY_xWa;eFo9@O$SO)NR3q+YpK(f;p5FOzLY+tm@|S3HH^^t0&`DO9Z7_;PN=p zywJA;fkucEvu*Z$M5gMvITgD-4XnEhp}J~>y?nAITDE&f>Gq$4U3=FGd(d#9E^b2&s9zXPIir)5=1uDN4X=TmIzOlugeC%IxXz!hz9 z_e$t8W(WL$MfQAe^H%BZVT;meEAeDTYzRyP_@>>cxXa=vh~-~VkadK7PX5R< zkpo=M?s09M@61h7olA49Dq-#6He)qXhskkU{hZmbwY!g*rI z+9QMdG*FP{;V&ka7FgABJPM;u5$AT7JvfY^WvbNo5*(;-(jQp*iBb5tY>pIb|$T?v~M)?N|g4t}GIJ;*L3adUJ^)HmxHGpzSeswM#l8YH_)D^V`567LC8?0v4ua^jLpeUEP{cLH_;Jt+@?Wtu#ppKcK51f~2jA`xw} z-Qz1hD4gFcW*EYp$6m0hBm{ew^4*L<3l$d{9>WdH*HtQ7r3D-;B`Q*i`fYzsP7tuL zOmaE6XmYv1)AVbAA)qOl)*2ckBW4aCoo9kmY`%9bhy#&L^`+CUFT?%$ssUop5N{5j z_2A-*_8VvRdx4tBj~y=?HxFKH*5vL*3Kf(p@=Cp?T@zjTOXC!=fv8< zv`{M*c*+>0WA$}gg+j4@G6FKyuVW^^kkQcQpdbUd;+#Vd(p|7P+26e{i(52fKN|Cf zITqm#D5|gheMhcKvr+M_M5YXbBT+XY!L`FVpT%=e@Z7F!^i@GFHhJ635dulrwIGRE z;2HHz`eTm2-_=D2>5x|1oXDxB*j0kJ`0XfeuC#GiG$2f%=`J|hmqv*3R{Uwl?;z{I z`yN@P-%{yh%Dx~@rrOek1zxyKbmtY<>-rw>&m2ojr7QS4fuiGZvgQJ?^rS!`w5;%~ z<^~JvI(;MRr!l@L8M1%vBE7NwuEO^3f_LMQp}A4n)Oi_jctVwwP^xL$OA1;VBLHx4}J8$ zpq7^56|yIw_#*QArjjqqD5AJL97~j2ZYCT> zRa5PKx8;M~lN3fkOlvp45H6DwGG^F1Y47 zLV#ERQMr5jR}IgI$0(zjKVds$bf408hpsyy*7p_)jGtRe?CFp#BRG)-1zW{sOp0 z%M?T0^OIylxGj&Jk^2p#();+({5nry^KAEg_h780JKTXpC31Lqd*`HvW6rPiX$1t_ zNfVC)59NTFj$hm9Sr4>K@=IPw&Qd=P=T7tRQwWruycAxN{~K7nztXfneZ3Q6|Aa_e z+*0tg-m|ThI9bl3ril&nD`-T04qS?cpi5HmUpC_#|D`o(kf;>Lh+6@B&2L$?DJ5ow z#%W(4bX*8g4K-M0{aGRf9MVMx9C!`=0O-{wH8RXUDMoQxS@_-E0z>S^o4F3ZyrSq9 ziI&cVUez}Fv5zZ(l4l;&+cyG0rc+UTy!RS8ySh?4Ck&E}151NSHzkUD$j!CqWsNtq z32ovfzKX6K>%g1TD^PtQhrxx~s`{q(=G$S>yv8%Y#V^gLLwmQ66>d#@v(8N!F1f8b zP0%6cVLJ$dRFb12RnvkTVZQy}tE&jtkJrY;D)$|o7BB0%W4InX7;MF^d`WzJq zD`)2aJu%?AChasxh1=5SoFhPn?-9l3h2~Ud+UR;z8h?(ci0L&6V3NmXJ>~`L&GOro zDd^=NfsxEo#}|soTkO(j^F#`R-jCh(D7V6hjhU%yx`xKxm*2gvIIEYLbR4oQVykhc zln&+IvG+U4++zR{Bzs0rQs;-NLq0Ca2p{a@ z`!pYspLC3;cZdyC-zRH<%wV^GA8Z}lPC*w77Xx}vGEQ@|tm4a*yfI%jBBcnugK>(B z4JIvK-}9nAraI_j?CZKzGQ0wDhCctW-6(V`a6{F+;L>yL=CTF7XIR@0TY7n0+?xK8 zVUSYhd$$21F^GmNk5(^wfW1xU$~r&JUd^SY0A{nf>vAKbCrtSve+9pJ#nJtS zsK72(w4xO-Fk+`7L!@DFWwE|ALi;cZ=(aXfpC87FYBri&w)o;3qq7nAmlDsqLUk9* zdipUnSq`@Y^_~{OY2(28Qkmw#x;7cjS>J^7AJt1x)?MRDF50CWd90Bhii!D>?|&&X ze|GgWP)C_t*wwF@bxScv7i>u3KYIQyz&zNZ%r%;?v=M;((;XVzOvYz3=>;xX@@pH} zYB*n${rF(!_BT&v#fsnc5*h1$4{AUGhcZ`Mi0U-UK_wzw1^owRZ&$KBP;Ig?tGaOk z`cca|PS>_>BfhEu^*j2eb{kjm$z?Nr?DG8bXUiL*BUeo)Q=Cz^@#R+ABcHw$chS{J z5UX&iLMj7zF?bJqh9fl|n82;@9omg`_Rm+LB^>x~sV1x9Cii8kHPbKJWl)AjVzrCh z+{`u+L|6>`3_Pbt#$i8WwN}lPd6q(G?8B7_1xH=l^*l|G1A85+p01=(A)7F7o)*!3 zC~aZ~mu1tu=T<$$`es;vXRU7c+~SJU`kUTS-er*HQ0K5sFZ07-%<-3Nf~u3T4o&0L z;47wMWYvz6P@5AlZkNs*yQDxquBTxVNP$r8-=Npzuk%#QZCYL_)m1OpQZZJZ2RKwS z7{Y%jhn3E6PQ(y1T4#p}6OjOWV=Y5Pr;eWY-La^ybU*iH)PB(ZKb}(%Q6&)O80@`R zStFH6JX;Cw3c9*bWb(uKvd_&+8}Ilxw6%4K>kobE=Xugzq6Dvzx&6Jq9-tDGGz)&8 zP&Q_E*eqW9ZUFte6)G@wXbF%XrrWYGdPVV0p8mNggDp`X2Fcg^v5A&6nOUez(dHQe0$lp^oUw8ruTiQu+gM1jcnO<5(Sps%I{)*R9vCF92>pU zBwvUe&-+8>vPmFQw6u12E?mwND5p~k}}mMe=t4PcINCz zEdh*0n*NOX87qoYAvzs%LG=9WPTCfAJSy2#6ameY>c}RO&2D8POVCWJQ^b_bsPdXP zCM&}>0KhmZ^`K#=AZMl3tdeRFczAS3d`x!R+G*s4iN`c?raKC&pA(uLq~;pfz0R!E zIGvP^DNb{WM)7DErSi(lyV*(uX=$6N&?b}z>}WfMDJ(>N<^M1oLGF}Cva|qI`}M9c zfLn_N?LW}Rj}6kT8dSXo78*St%Vh}Q3d2}j5KFJ0=Qx`zL;sBf5TjYmAoc);a3L4k zA2t%HlX;8*s~dWp+aaE&0^2G7?KCm~?jrNSL%GAV5c<5bZ_Vw=Tt@Hhw%9v(S(a&b z!MHRW`Dmrrz3&%0P5L8E^6kgf*V7H`Zq6nxC+#PLDsW|w?s?{}EJ@UY7pbE%Dw8+{ zEoM+;FXX0v{#Y*&{&5iNFvaie+tZpiSP}ls?FCQwF;`mEW5q3B+yg<{H^yPB@>dY= z{y>-J0Yh1Fu%H~g{q=dh9W~~{CNseEy5qUWk7CnL(v#G4L><1eyuJW3EiAukil`G^ z^&sU=Q)9F zI3$m`mo}PfZmRTZG;16joKu50F=$>s)nPuhhehoEkt!u>qcjsGnWbN9W0$gw@=6nW z?FM$9ww+ zO%RkR@SMUcL%7AmrIDlCu5%-$B!T%xk!eAI$)LSr+~;+)O~EfO$bOFj0^`&k&jFc9 zjj{}(b&JB=zHO~FqQ%&kVqN!PK~F_XvyTlYEF#{SkQrOH^aK5EQjp6FXM1>cgj4dixd0OZ*glMz8+m)f$V~Po8K8+(}btq{rCR* zzYrs++%w;k^ijv|+UYx%lMk9)XHxc-$5sga_f$^o=F<6TO=;(ZIRi5qS%|Qj@eUWU zJxjOkL3mpdl+}`oOJd*(j1eVvvRu;9RK3z{IXryTBvz%xSRB#vIGbNRsk?d07Foac z3|iIi0!i$=@(Om!jWrc@Y|s{%kBly<{b01di)i0;ki)T=2vBD#pkGZvh*?uHk5q}} ze=4cu6^5@=7S2l6JdGHTnZ5`WsXUjeTq`k2G$RP;%BFB_o@AUbwE%(dcK%(+o^IEWMJ0*j@|nAseFP7F z7^(F53WrWO9%RTMC{!|O#Fp>u9RBdv+ew5Wx0xPsNks)Q@NreIb9=|6Ewh^k@DK^| zHN6X6ew1DM135{;p{$VaGAg;BVPz&9#s@!YuM zBz7-Pk``ONt`jV9;I7pF)#eltU zM>k}DWCx&Wc2om+H#JtrdfW@m!$@P7y@dE^mZsv%$0;BY{tw4pF)5pOY9}6zoR%3q z%`)rxJxlF_ggDE@nA1A>$_7icGNC)72wnBmho{Vmu$@gEUKk z^g{<)yDGw!HzI^5l|FJN=1^^n=t8($@lOYD#7NRS$dj9JI77EajCa;s(T`r<1@@WV z%Mb>565kP6!{KCB>i_YwC3UO@d2M=z0 zZVYo>)2E!+?R$`7-ar=z-1qAsbs|EHpXX^1+BpqzOVvqdYlK%WHWoxUBFDoKf!Q(p z`poynaPy~N?mtJ(f7TRb-nTl#1=$&t;iT8`-lrH|Bg|Je8aMTgW%k85rDA9nqQHU6 z#k)84kDn|x+=hj)jUMef%60NEAm4AQ`iYhKo+*^L?-*LZ{ z(B|hn5JP^s&~fR+ht1ipW4zg7DYO2Z*Ft7c|Gp2l@%qh{3iBoGKQ@(u(BS$n&Syf^ z*WTVBdap*8*ev+>brzUg5kG8XKHw6^ym$5vy5&#MFnNk4$B)SI_}dXPm1d`G68eBL zGx&8qS?{-PY{8%=_T~+{z?Ab^JqoHutvJPZqW~mM>Rd{0CeQ_b`N?+dZSG0$V z4%v8!i$rKZNj6+ZLb-2v`2F82nuDP~WsxgubsUu<%tsx4S>nf)?UNvT@S%H)-nJy@ zZ!2N?drh#W%L^~2Q*;oj;H?I*WRUb}p&-alLPT*v%k*~NUB4|r`sI;GjBDwZ^IrzGtZCX#&Zgd*$0Vb zE-W&3ueTvvkChzU^fvs?o@DJQLMG0yT+YBXv6tJlyf67*{&N zhkwu-ni-8Zb+S2&ju>2cFuGXoh;FKlU&5!D-YYE_t*9$_{R5kL($jw$W4~Mcmw-3j zh1Bw#gvI0PS^lEo3-i@D^}fvrE%Q0JQ@^MQsqFQQwamm2xchIKUH;*hn;A}j{h$4c z(kh?Grwj3yyJYViFz>Dj2m)cc;MwsVds(?y2cc{AY;ab_buw!`p(KuJ#Fsf5-`q4#aS}4sR=H{5!kaQDRQhzZ6i3SpY1Lljv(6S(4IOA(n<19 zBKplU%fFEP{fX;|!&ZLtRbAzZETNzk#0osIgK6E`wx+pAoL-k_P8nQ!&MII@0GEo& z6x`IlK+`=R=E&^M8gBafBx;v6_>W71ao18?NnovcCq;q?l@S--{s%OrvOrkLgN1(* zddcXj&Wm1TQ9X$cN_RBev#4wnUK8veS;za@dP$G((o{xoRCLK--LbUnnaBB^=C8|83j|>UX%1^=HI?&Bq67)` zLP%eZ(P>aBMW{{ydkL!3yH;-g90PsPx)fWPg_JrQ2wD-nPf?pp<;32!igT*di1q!doRT;a0naNl#U~r1%W`))JoA z3m(W8ev?P9FGb=8nvZL^ZOvy3Yj{ZAm_8oL_^wt_{lrcwyG^iDQQNv^NNY6ybWahK zyp(?wN5vqJdY1olar)zzYqm_K3e@X0c_-@{@wlV7DP|Ap02CC8?>WlIGbRj*thK<` z8X_yjJUIC zPB}-j5)=-OZle>(qyfr9-nRarqbe zl)#YKc0}B4%z{aU!V<_owE5$yz41Q(E6^pn&Bi$@`zA))f4NE`kp2kie?GOfeOwWW zP!a*Uyhzb?X+KWHc2vcvh=1jxKzxyYeXBuSMh>rc3Q8$!xy{4Qlr=bWFPu3jlBN9v z;#d9C5=!ED#d40lO;{OB(#N-Ppm`F_F%-jCqbr7zBofh@U}HCQGpa#orsg<=h;zV(PnALV&ntg@VqI4#wxiUl6bf|<0YwLLPld;+&3-yVc z&~=aJx64u0d;ZFnvw-hW2HU&8T8tfcfc3vOU7WWL@N9E^TRLiMoh(nvNpOt!qaDQ4 z$4Z^U8;Ni=b0M&v*u%1o$kj~FJ15|C!9M)^PcPZ=@d#yuO{(xbq{XEHhW0@#vtoEt4;Z7xcDNN zj5uSZyX|lDk<41T0TTsl&lZkttHb7(P9cb%vp+iOnULV zem-8TFJ-+64yAjC-|Z={i1q0$gFn)Qs_L|ZR=CtP95b;F;}EsAE6x6e?BVpx_%PC; znrNE^vF?XCq~Z8ka=w?~cVJcIe}X?HByebmsc&ai7gvg+irUKSS00Fcp@RMC`zjP; z9)~4$8oK57qP05K*1oL_zdPWDEHyR>NonJoHql-mdQ(kVT1q{YYIaVgLboXg@c2&W z7L$YkDc8xvHg;L7E=}1Kn1@$eIGfz_0W;}niUKwMd5wGk-e777yR zMZYV3+CDKoQHKoSPfO^fCszPcG^()6mj{2zR#KU)foz&Q$(bt^OiGFAeTUJT=ja%xytk zqbJZh<$+C9S3CWp#)K%6wEdfJG#p#h9>4B*l(NOo4B<$FC85lIjj2t1j^Dcv&z;aN zbed0I?-vN*paxH?9IR2iaMz*iQDwn=HZD|pi})wHd^P@JsLISD3Xvm*{A- zFt)&E2>u$B+C7*mV!Pi9rb{_rx%j^g?`Z0Xq32XTqMnSnmf zX(U>ixx}&Es%9`Y;6h1qjodyRg2TKY38XFl%l^dg?C3vkk^03x% z0$Mr2ST7s5xSWOJk@dFUfB}LlB5gOY%9NCF_F0gzG?N#$l~gEoEN@R03+6AXuGc&{ zb?Oo@>MrYkAB3q4oBps;3A)xa!RZ9pK`h{_PSz0?*S^<8#KYj^?lIQtr$RlmeM|3ZLB!DO`8_it)~j!2+f?xe-=BV*Xw6xCPN zbxkrod%dxx<-@yQ(^r3}uEaZ-@vh^QE0Mv?u~*iFptaFykK&!1Ky8h$qj^buu@i+7 z)aYBM>mlG`KrI=okCifd%UNH?tua(`!u&jr_hFRiRr^BM&eYiQOH=Yw4avJWgYx*Pp9i9%Os7$J+v%Fu{fT28Xg#3MgCrGe5$ODoJyqpQHJXpLsrGbo{BlO=w}di0~BCuM}*->cBd6k}_LWKgm0K4DIV~Xu%wscn38x1_-8ofQww|@KHQb_VNk` z?UGPBae7gceq~q&qZJc$*q*2_xDrCEFc|QPUlxi#_`=8Zr*MvW(1*IKM|;xv{}SE* ztS5|>&650{0v*EIG_$y@Ny18P?`4Fi>Nmglfh60#3}KMT)LMQb`w}>}+!h(;H@P|L z-^{2FSlX%d&XB{q@z1OravpQ%2&VzFs@Geu<C)XsaWD>`IC} zqcY_d?2yxJskWE4Uu0?W#xE!st=-qZqE{wuOiBEXe!K#1jsmlYaQ|wwO+zB%Sua(Y zXS7;Vt@?f0ecx6pVSH@)xYqwX&5j+C8auE@ZZ4E*bKBZ+4cA;cwqN z{?wyc)5d<1KK0uGi=Mj2@{^3F9Pyvldu(59HCxN)JIjyN1%U=GsE~IeFExgc+Xs%5 z=Wr~+Fl<&uG49_88$3iD_ktoeHm1GEd81tx2 z5>f0F-h6O1vxNC{7F@Pvvp#H!y}DKOnKwh2a8@T>DWWmbXg4{Y(MYsDtLJ3mRQxFK z(dor>4r4ehO20MfK84|iV}`+xZZ#i`R0=n zkgz=aWF-F}A0{h(!^qTYN-k9Kz~Vu9{8!3qNY6fXuv4)3hMOMbemd#bLwEx8a2~3Z zw@BaYM1jEvmlwNf%t+i4Ol;ldn=s+47zf|;4%SoW-`sO8M7#TL=%0fryjn8qLpY4j zQ26-k()%Hr*b4n#Rr(bNh3U@AjWG?1n=Kj2)@@a8g z9}*m)Z|z6^#B{}7T6jNWdwJ)d9m*ry>5=B(3=;sQLw40Z)KFMH8^O3ZN3^AO!+93# z&MIK~uDFd?o;^z=f;48)Q=+jf*?N@=e1mSI6RC8-3pyVd-ymN6Sc|a+gT>}BA?NHj3W74^ ze=E-^P57vb-!I=FrD?2AIls&{I$CnLbwzJRWJQj8s*^nXI?4|c7V;KRVYpw|ggURN zBS|xIP=ODD4@TeU#VoIZAaOB$NYT(5)zS@`q?^SkgXlmQBozo2fl!adG>5cEx?+w1 zBF0yhU!LK|F8stt4{k!+GAWK?RBtZ$i882`Ec+Sc3|5p^My?;e8~ofFTN+(yvY~ZV z%#dx)r~{vGn%mtqwS2WK*$DFA0R)F`=qmd#2=L{a(SQHEd zwVBsT_9MB-k*O1Fq1WP*D(W|GN)1ePkM$&(78Cg+v0}cd^+Vu8q|L?)hRWVY4uKPW zVM)_?Fcuic-57uQRtTOgHQqAC`E_>I&9^E8@QR5F(Vb*El&E?~bEJPX5f)za1>HjR zPvOf4k(@4Z6S4~{9PldDx-yS0m4W{L1rpml?Va~g( z5&V+2KdUpXyLjb$UL(YUxOR*K)A*wNiDAD_xU26~j)YIrr9=TpPJ=5V?&3{3N(9-+ZMJXX& zGCW|c97(^k4JO&3!H)S2?`P7Hw>&r1H%lEk&A0;Md&jEQNm75iER84Z2xi?XdHNO3 zcba^a+m)Rvn^OKe4S%yw`E4ETq@UtKeS2Da$Ty#s^ZVaYbsel)9$ws)lL%tlY{&OS z__Qey$vM-MpAuQUjcJuGoY%8pao>00?=6~sF;<<2C~>pCo7_F0Qc%=DR-yz`gI(hk zcdnppgTO>tHoy=!gsaYnO#Ou2f_z_Lf|jl3E;p61yOy}#!Q`f|hT_X3^HVFP`!~y@ zZ@bXT=snMz)&rFK+(pq3(3uZQA4a8@vmh^6V+HCD-hXI@cnxBuBMb*l93*N z#SCN7kSiU)DVs{&z7?3(#4`%@eSHn8BR_ol+uehBaEp}ix|BGZJ*F&MK4)sqWt3jg zAbO#OAx^r?Sd4yaIL;Zq%wwec4NfEiw2W&(yRQIsB-_E6UkV?Zwk>)T{E?4h;+j}UEXC2edaJPz>pXvhR zY?8Mvc4&&9yP&5TOgtFY?`I!bbbb)8IZr1!DaC@WKUv2zc5XaRF?YP}e_!HNlf6Kc zXON21vIB^OuhIz*Uwsgz)zPtV*gdL8bIdA0yzAzXa#;5i4kF%jv=(^KAJHxST@n(k zI{e9M6n^`gW9czC#^p<$Q^y9O*pgWGDK$hU+2L{x9}fvQge-#Wzx?KFn4t ztnlh^PqA6}9>Y@_ACq5b1y0MaXF*`Kd>J~Exjed&| z?og`4xvbw^3fw+w9_6J~J-xdlMpeI$#mbIclSE(ct3X+u3X}^X*3x%ZqDud&NgR{g zF-?QrOO(w-s?N{`ZN)-6x~I>jiX9xM;Sc;y=x0lMj!lng72}kRsASk=gxu2|S zKq})_+CA(-ONmuzZz=!stBR5=!`@NNt5MP++$19zCy8y(B#ls_nrfK0b4y2P zdF%y&S1RGfz+QT?z&OZ89bb;x!{Y-7ZQw$_=My#hqlg0Pi%kgGxe*2$_4&XQuC^c*# zT7=_XBc=a)6Q;XR;fiyfi)iuVkcb*3vp8%n%s3Xgme?}wj4bOpl6aTf&S>dyESNPj zCrf1b_~DAUQy|UxHJb*{sJT&0HU3y|z8Qgbot6IK3A0(MuojkI+!uhdup1MPNQNeT zGbPD5*&Rp3W8~z#lHT4+h!>J(w6KHBn8{e>K_U=4t4>%u3hbnG4QJo!&1M%=dAoqM zQ}<)I;$gxS{G8ApEy%Wz0lt?fbH1{P$g$m0&}w%K5M;NVrqO@AeE0Os5EF_ym<4wa zHX!U`9p-B)2JT^(~|ATdKNsE z-5q&~(-jUf^^~Eh3*9=9K4f+2la~B?6+;uwcG36+SKGD$4@rs1PF68Y0G%C{nm;xSgz^nmjHOr% zi9ZrKNsNexh#r1dy@(Z4vApygRjT2hrF{|_%q;w>Gji(Bg+pdazV~}y{;z5?LnHUc zGUi3n(#d9FQ{}`N51s>|kDK=+XAS5&Oa>3Egc<$q;Q(G!7b~p40$GHuDgJAztRmd6 znzaV>dV=g>pPEoiYt&=mfSPyxs0C6`?(TnC9lvxeLZK{r7}OE3bp4!#1#``&}1JV z05&|@i3-iQS2Y=ZmZ00jm5=nwKET!7#{QWi?nVh^_4(4F%u`DIrfofI<-gc~cw8&u z%kk&z7SdN4gz2GZ9HY`1s`_i|xMvby9*CIkM8SYdxPo7kg_+*RHlAh>c1oH=Z3+ER zQ$YTZFEN=qJJR9Rl4zV~M;Pq1V{1%D?5|zyDGhk_$n2{k;l`HLdfXJ9-#%`;(>e>d ztK(jta)(dLVyzwbQDpAw4DxdpGt)~_m$pa|ENWNX77ISmu4^=I#5& z&N066EQDLrHd&BmZ#SV4)i5r?;YNIg;&CnwHO+KjRA&2}b(Jg4)P$|!a}#OXQVXa| zHQWH)R|#o;OXRq$lwLdBhWRu%adqh0*XPNdxf%}M34TD~p-Sk;;NG{Jb&yR_iW>BL zmD!(sw8PTL`s+@GB?k~uN3xlnbbA~m)ZZt9W*D#oDYA?3BzUR$QE3r)pmiU-HMab* zhuYofcd7GFn?i+Wc@I>y$6FS3>t>{8kQwIrx$^mP5Sf3JwuGj0l#?~vQsmvqt zGNLCd;AXDvkS>N4d|$Z`@wia~LhZ3u8)l1bD-e$;lik%<{Y0^mWyve+5*0XZ%iXD> zKF`?VSi&xWXq#h(%b^I5_+86laqIk-ZTD~6c%Ml>3^OS`;#F|7tqpV8{4!Gfb<)1Q z-f$_13<2n(YHPNkA(eLRlhVxd8Ej5Tr})3f<|hEk5Br1g)j6R zl?Kh3Wk66({SoWf6ybd#2?n8cN{JEQql2eRA{RKJ481z^y>LczuhwNZlau$U-0 z{hWAmC-J*b{B@fwQo*xJW-07&OoDRm`ITSvi)aO3SFP8K{X?KOM|~Bt3p_C1ZZUWC!fc#tGt~=C3H)e z?FMOg-6uU<=3OyO9^Q?1Cd`;WIOqnp-(e;2zXKESK7 zP1}xYc@m?5S$JaD0+euxe&o3y)Li$uUG00GU*U9c2Q;r`baTe{iZ^8!TwZ!j`m+Mk zJFecToHe+Q%7GqBgP}AHvdKpR()n_Jlx3H7#D2v(b3CImJD#q~JL#dp{qa#4`(^gu zk@hsP{L`Mff484XCm$f~PUxa{qII_J)2cGifEBF-v6my64-Kx>sWg~;F|24BxqHmX ztp0G0z$w-OO&%gbis%VuH2@?9ry;kC;~>4WA-KHE~89e);lD`2aBtp4#k z2k~OZNgJu=amE@mn5jn*Lz@`~$dz0Fgc|?!MC`0rZ&>pVtxXZbTClR_Dzt9Qgz3P|H;1ruwVX6LgC|=$hNRBveQ8P@-z#JZw zTO#Lmll(ZysZ)kNBLZn0gAv)ywK3lX^6yGdTtBunq1>UL7z~*653-OJe=*rp6TrGH zT4`GTo%#1=Wj0z^^Dz9VPsLZ>l}hW&qM6;ck$W`U3t84S7JMkh6}V{8x!A<0wPs<0 zF=lXSQOmJbjYbu2A=cjA6}aB+jQToF$-84=fp67kXpCi|V%cx($u};d9j68r&S`2c zEU5p@Zau4W`WaRLV4!CD>W z1&4ZEgw;`FjV9UFh-z+;cYyo$R6Zj?)q){hNjtJBDiYCiE(On;7nl!$j)fqd-yGeJ zphtfQlFfpoRmGenkobv0pYp>pjiy%rq_5)02RE>FW;O@wFz|c}w&B{449k3%w9kcs z7yXt9Gd=Zic<;X16uVzbXzZuwXd`)SM|=i#Yt&|7PwLj2G5qlhwZbOhQtwYGE`UR* z_-JH@(fX)|wE(VltVLT>$*xjDO0+2MvhDZ99DeCzxvV5TqYPMl+mQN!ketd?jQPg% zH_CrfOhgyK4xIx%8M8?qi>>C?cRSowxu(cwyWmGDIBbI(z0*|UsB;*dyVC;dQ?xCX zZIn;@4ULgg!U9GG69=P^+i}_1a}#9BBHozyXj~^NL5o9Cbfczn;Y*GzaXjNvCQfm3 zy~*QEpnGgF35)#Y`gX20-Z|cwwFT0FbkeJKID*vRpb$$n$GGe0i`QntRF(maVZl5< zq1t7F(d)nhgwAi`Yd&EJEA-r*8V<^nW1cN)`y1H1pDPZxJ*Hof*E zIG?%MuQFA}tT^+2tD=hRu^nUM8bA17WH+S8U2x$c{n*THTgUF=PoByT9`dho(he_7 zDj4GXZTF=a^b@DL1?Gy;Tv5KS9|y*!Z~F7;ge8El{K){7`-q-uBeUAs2c7KhbGwE1 zWzFFI$9?kdR#8g(q$t!Etl51=ZN-I#wrvqbVh2iJk~i6v(RmzCDTd_SHs0C$B{_YLEB*5) zfdqTka6ZwDzB9nGsuPOPv%dnaJ;3zTHZrixvgF)bmoK})wo1j$K8InkMVN6b8thhvCn2 zfcevUop?l+j)_Tl7hJ&Q~_V7Zs2<3v7Rdv)VdEX9Ej53W3{#G5pHZ6j1#txsdvkcC-^+o7k@_Z;hfmpp4rviKw|@3$&;q^=-EQ zk#yB>P4;hFKtQEblr9B_bdOG@Q$c!kh?I0Q0Rffn?o_(FVRXmnhLNL24L06A-}f)L zkK_K~`kdEUmz<=$I6GYJpoC{Lzg*EFmtK?!3RoR-gHEwHG(+FipborP;~aL{hS~ph zvS;EmA$BN$5Cxsd7o9>8T>W3CUn9NAP!*l6l-}d&Y&?!RO?;4SU!IUAf_B$lIryK| ze%T~zz}?VIkAoq#a{uy&8pi}VNq;$I94l(~hz2%b23z^Ijo8x#ZvI8~+TUKZpe0MG zx%fXmvK5NX1Y;j}<(eaNT3>D_X0`M*-kDVz<`unBZR8Xgin+wA|!?Xu}3bnH-Df&ZaYqyawp}yLW3xJNgi>-X}MX^)WSr>!uA&Z zXqz?z*|%ul0*p4Pio%x?>GT^`yRMHOqNd!Tb(R{x%XXuxQR}@Osag?zQxrN3M+D@# z#6#3tvOtsc;TGo`^8WX-V%Nqwbj?;I#yHm_T&jlnpEiGMzP(8s*;aYL1NMD=enev4 zD%%M+lMrt?ftm}E923HvP*CPalpk-IYiU{I{<@}8CK#A=9KzjE#c)WufyB*Om&T>- zRl6_*IA}t$yBkZY*>B8VZ;Pniz1%b!S+lHbF{TM?p`~}5J2(y;x`=0`PFP~CvCBKs z6xpNqiVGG5*%}W9NpwUtpD;6F#aVf0YowJBk*Uy_R1nBzLk0h(LM73~N#|Ubbc3z*c!++~VI%;Ql0&MS3b!c#!y8{W1>Dx?HsL#t+Ju4w>3BiDzm{~2-W$jM^Qq&!Lh4i4(_gkW%A+|2W9u3I zy4qW$;*`@&)c+iD2W0fytG|ORyM6ZN`E(42NSuKH3-aNEE`#pj>i@14uatu6Jy}^h zY}@CYfAM}6E=<2A*B0NSxs0>U8 zz}8QV_l%LF$w(3BYnSaR>Lw1qXIzG}`>)5&Q^D~8OlTbLL`xS!^S6+CyP0#YKj)Ve zyAit2n=UyTLROD(Aef6%Hq<3Gr$vL_5QUMfY;$j~JQ+8uH)WCb`6G(UIDc)`5*1y& zUO&os+NPM`L>uQX`~B?NUFf|>K`1B=byI9aVeMAlmo+<#BdB?tSZADfnz_arWf$7q z5h^(63>S$;i?!f|Wx6VKSSYR&Yi?y>r}=;TtqQ)MdCAqAK8vTdD37$=eY2;NBDqe5 zje38~h7v;^s2RuFSOEsfM*vL$ZOgG$Hf0WmqBz>9)cxgOWa|Abh>PaC-Wk8#9#!L( zw(a5fCsw6i4%KGc{WpJ{=*`)8ndjeY3ifk@|ImT%mTi0593R-Y)uq=;h=eYS5-7r| zy3s{#tjJOho3j@HY^)ZAjKvO z(16MCyRDVwILntU>)&+a(xvsHy$t~8|Qjv#K z|5*}E#3yu`XW+wD+WNBv50Fu16~z5%pC2Dl-djEZci%^6T;AJ|=_*!`tvL7`)l2-q zKbl8CzS4iEBYSa}OHUxgpF7_GM=m{v+k77mJeQ011Kjd8MT9xecj||T6Y@pNAYFZQ zqk^`MT}6I2FC{86i1@dEt5jdGVaU0-l?psJdux+b@h}1(wcHI@mf4Q^bh~iT;QD;E zX@G79yY&d?VS#5a2R-vO>oms8+XZ=rr@mO6ivfM_OgjIx=hznE9w&_TJnleb;g|hqwqSpyse@P- zkcsbmlvf>G9t2O!Y@{1SsQTMnkG^wYR!h*qw$;BA-wq*9GQAd4xMq)HBCz!zF0a;T zaxF}Glku~~wYb9KZ5KFG87X(ycw&X05vpp0NW%_7W2(`jHUmCkNZZiYZN$JA? ? zD`c8s$-{ZrQ-!d=wLg=+SkbH5C5Z7_5AUx;(Ga4BiBDUDMi#q6+j16YmT@1D z7E0xS=$C9pVP@1p^F4>ynC9l?`PJ_-NHAUg#<=wSw*3ZOQ8P&+fA(*Rx;Jd-8el=! zQp&1?Y3pcUQ{~i>2t)yxBA$BW4*7XE^zrU%An1%_c_h^_t&z+a4cgsIw}>|#YAnCi z5#K|ax{51(Nt3o$atwOb<`?Uyiia}n+s0*{?6&5`nMVwVxcP?!@Ki}yt7r9guvfOx8p@8RO;CFo>Ay+j_PEyz)-CGJ>gyyng6h z(Y;XD(yje+-r$^4CmS*An1i1%mBOEdZ7W&Vn~pVZi#!|G$=}#1bFK?nw>25<{HDf+MJadfVVpP zPRq3H`xFHW!p+yX;%mJ0j9Crw(oxeGf0D{+EV(%gBBm*z;oYvxf53b-py`c-gB4za9x}Sp`x)YsCyGpsb1WBM%pM{T{MUA?d_-XBH?32WD=Yq;grP*d%u?^aD6v=; zyeew~LY969ZKXkJ8qZc=^=CJv)R(j?^es$eTWb@^|$uphVBr>Vehe}R^z|_xg zs{psQ&Jo{y4LPY;NR;oQlZl*_W|$s#v%asvOeCzww35^$0~dY(AmxB_D+$vhWcQ=Q zfZmkWjB*1u-g685Io>^o1oe$?*5(_tjtU)CTkq`&3WZ;vn-)nv-U*6YUKtS2rmMFi z0Wojv-P^q7U|~!CVevBasTZCGwtYCBDmlFI`FU;j3U+#^DxUX^*?R9=fgpzmZ$gaG z#IjoUqKkj6mSFHW7(?GKOYKW1@FAO0H@Kpm$pp7d z-j*svlxzm9(NRrOi*FxaXyZ=wV!@g|SvO}}amWa6 z5}Ya*R?;k5ljp-Xyv;6)A=F%?t*ZA58{FqrnO~hg_Wi*$x5@Mi=56Uzo7N?bDbwjN-1}d?8~@IdWS% z^mHRvBKc>;j_6wsyLzg=LYv(?d9n7@aHOp4F?T#4yd?ph8FTv*8z z`aCA<(iO6biL7FGfHeN#>aX;?+UutyYTG)@y$h^*(LCadXUsC2PG*k0@mBQ#77*V+ zu3vbAWV}JEvc!$WExh}4b=%|)e9p%9dm>ChkvZ!31uMcOC0~f=4+EjEc~E(g0xeMK z!7+Vfl4X@0ozVuXtR1%yd9O}=qfxq@8%@=J7|-)c#V7ArB{%lI*R^@w>13BhiJTa- z%>43tn#EV;LqJONtrj&CtLuq42@@~ZhjZ)CrIr(adG$@X4qtphtKAlT%z6btJ!^3t zzW>%LYq+01b}V9pre1;cp(4-m>$%5=hcaQTIB_y=I7mI9!r6}0?%Vf1qXb{Y$iO+FMWv0gP^QDSH$9mRN=8TBnGbV? zLJgzlbEo!aLGEpzQnml4*?zynQlNM^I}B>&xHVEo~Ulidt(z zMq0+G(7%?zfTxb96O~`F2h)G zFh`E9dpU0G2io7dJ*>Qch;7px@a8+@rI54^^>u?>PGTY(c*P%dntLokt zJUpGGir(Q9itX_*?F}*geFGFm3?L2M;S8x z6!&Ip*QP4=WF%htu5#)N^HS73ojUgEL&Zr6e{x^_9%4~RJpKjZdN!GpLg!#U`>ClQ ziSat2Q^mzD)e9gvIRGX_d74*5Z8DtnEr#Mv3vFlKnyWG&XS73dDgqt>&3SI1AMYhh zq@_??VYq?7$BXZ3~gty4-BLBZ}oVohWf+NOj)-=7o5C$%KuBdOu@rTb^IP=?- zNQ^b0icNUkE!$jMQ@aQWK6)w?QKBESyg+#_Z2{32(SJ^_(=6ubZZ68F+`|32mOtDV zh+6i!lJRXre~4Gqtjet|F4s9|!nTI(wI6!cHiM0YjVG3ytyDvw!>xV}z80i8f`4?Q z|E^XwRjK(C{m$p|PU(4`wmUhE8-;{^ zruiD)+~!xob?Sy6Y8eW|U(RaV$7S%2mb-Z5yX`Ws&njdgU>*-%c7@x@)$PCf)uqKQ zdo@w29%Rd7FA)Yyz`^C~PCt2)c6gTpxkI~{jNfbT4AqVS5A>AddukuOog|CyW6OO@ zwmV|Oy>P@=CXWq$!+Yw<*$HU4pwhH^J<>Qzima1+@1|7uC)^O|L-x7dl zhiRLN$2DN~>1wPoi67m@r6xkwT}b|pR2`@w6QbJ{J3j80xawSfM6zE~<0Wg+~# zQ6yrbSFHs!+ePkWRSoiKjdNlw4c{!(gf+&J=t-8(h3e%3=Q_szHI991n+MaOSbjVRX21yG$(H)M#VJwpwtSz%&2dQ9G}*bvqDDlmo$rZ;I{ zcRTR{%};&+_H%PCc}UE4QDGYgw;Ufg^pU&J`XOJ{@?Iu}+=mZ@2N%(+ z4})bGpRhYIlBP?cyfPEY!mpF;>e@0)M_ieDQJb9M@(3h2#nyy_S}k#{*Tl0%W4N1+ zp3=ZbtSZ2~>%|{~I5DhEqbyDD5wAkND3aMwdsk`5c2Y1 zfieY4bTn_J=1;K=7+Lmr>xG}IjtGQ))ucj&Eo^XJBxkT4(q_2R=`L}W1#nJj5`w)L zyG{vgLzrFfu`sBn2%_?$Kx0qq;LCLdWHW5_dlO+fP0f(iCeq7rUeikBUW z_$~LZ9IRZPMGve{Sq2fIu|X41HmTVefNDk)vwY!Rt|kbkZL_aAP4DS$O;ZDYwZ(3g z#UNbVcI)P*_;FRt9*aE2d%n%+h(2S$dtb44bCND0Ur&Otd^}b}{QJClhq|G+&RTM< zlDW}Qai(34?xoHMjfVj^It zFrpwbt!_iK5Z<{d5ncX_P-T*1)qRrTvUXjJFkP7BI2><6;^~lFifVRvH{p=efE@{| z;Qr)$75@waJVpPb(;MZH%tUKP@Z=1l>1bcXUh?M_x2HZ(sV{3YiB=w@a*zHs z8MWph=aS6+ew+2wt(^!q8pf3!QHn@ViG5b97hge##wd9?v;`~EIu`upW`?1DC;s{k zLXFoizN~4HlV7?7+q!w{_Gf5!lo-Afd<~8hTW1Yiq^GpaOJ5xuO(va7ZJnbY_QOtp zdOn0WAw3dqf;*^IhoU-BfA86E7cW{Gt#@LolMEnfRfz&{NJ=J?wQ+2~=&7K2_fj_Z zzL9MYu-?YYm-Nuz*~MOrG<^6?*!xS&a8c2nH|hF`-YNAbmN5y}6dbHgsNMYEjMJfWc_EnecMTE1xhe&jOW!%z(P zlDgMxOdxGM0t_1xAAw_9-#}h*yyvAU(auG$UP%N3`zzmsRq;DO>VEKWVPvPqBv^y7 z=Mq=?AU5sR8wiXC{}OSA=MI6hYacFX5kMo2biS>0aTJ1+s1@31=T_`TTm*SMn5URQ zIxjva$l!ZqV^*ZKiNg=`f7uew9jmXY=Vy)Ab&x4I>WVvzUIyn&!=~CmxrD>q*0?W{ zz!#uHcdQa&b)4O=lE}m;IY~Jmn6Zs{`=8MtwLbUvy?R$=>`YS?QyvwVnPba~l%cli zar~LcW|NjJ77N~{w!$psl$X4vJ6}t8Rxi6Q&4XOqoHJZj7|V~|?9gRsx`NGBAHv;F z@~X_XKk>m6wTwdT02a;APM3!^u%Bb;N#zS!RQayyWSrjckQPav#TB+|iMVt60d7D+5)Ly=KfeSyTXeg{z<&gPKMf> z(S4jNJ-t5cfwB?2P~=%Db123t5xvL+&53n+(v1s!3ztKfzp79QBZs<~rKZ13$ZZVR zByio`rlk$UBh~+&$2iHbUOSQWLMRJ775-a^FstQ#RSAyeZDu*xDZS(lvbKxu+=GqRaT+Mv)|isi!B^_p9fC4qqwFo1@V|tSuo> z-av*_tsEXF``%%fJZWq#el5q|>o^<<1#u!F64;hl>SNqm4vDA#oELFFc^lOSv-%`u z6twF+1pBl2=X5uV<*XVxmltrMw%TuaKO%4ax)0#B7vAZd98vO?d|TOud99jP$oY(Z zWNriXsTntMt6#4_8HuCH5nsSnb&FAAyC!&0z{bS~oN}p>9jP?LiQqms|IxNea-Vx# zd6f%U8i*x+bN`eIU!CDFss@ZVR6xz8jNaBx4!_sg6RFLPbhLWnm1F`DieKQ8K!%;+ zQZ7rSWmAyu!#30ivV^k7$@bEkfQD(0R-Uq5zxm)6_xDKeQ&F;>aj6oVZxvpt!suv~;vsm2|zT^>~)$C5PX zUSX44S*=PgQn1oyV?WCuxo1z(&H=0bxZWx>s(5@I_Tcs#>@mR&P z9X7t|g6Iv)6hcmbZ!{X$P@OG#Xm=C)czRxyU6jhXwL{-w_2ls6@R!T>rIm`R!w)(4 zEvu#b3HQ+MKEevv++|4VzB)v_Fbm!Rla0Zscoz=U77{}%8Y|)Ah@IqqPB9?I)DA_W ztVny}9)ZoB1j#dB+}~V(cmdw`C&!08$9mD$nttNDq0oAs1t&M5az+;0mHo~I1wVuc z%k?*LP|ld(hh_spz52jLarEm)2sS1Rx3jOtvQ9?{-SSjpLBieg7B@=U`o;G&<^1uD zu&pSi6PAD>oSr@aGRDR>8Pl{)SB;%?A~7F~J=mIujCg@N&sqxk89B3b(uNx8n38K5 zN|LTsEcDNb@Z#~;j}ayU4xN5Uc|ck}2mlAQ-#`4U>^G*`DN{C|k!6cA(%wI@^mwV= zw%7c&?Qe4}gsLHvac9x3p?bAwc`%VJhD&qTsy$XmR;?@L`p?>Prx79w=1-Y+G8y)f zRsKm8wKgu{z%Mq-Ze%9JVP3@Tu^+SOBsmHtZmvS3{&RszMXjhBycYn}6Mm26#4>Pe(+;xQ@pnuEo~pZVbQT z3XDLR1Uzud<1vYZfgsMU@nzXaist$G_6Z8!Pw2R4zSE z1|}RSL_dKYd4kN7Ub-WK{e;`U;V>q1?p&S!wJK$ww<#zravl9nRjFq98Q#muH?_or z9noDMNpT(#VXG0~5$ktO-~JdPSeeB8IsNv8Vt5-sB8TA;*hqIS7}jZ+$aXsVc!o$->4}kqY4u3&d%(xDP}T=d`R3U8$Ou=B zcusBB5le9hiW5H}vJTPM(SJCzK7sf9QmE+8r*Y{{riqBE?OrZDK0wlieD4?!==Znc zu~_9KkauvMx>gNGk|gd~+o-*_3P`464Y42?ob|!23aVI90-@6$W#H6GzZA2< z$apc;p?5!vcizc~zcZ9Gtxj9ZU3His$Z6_6C3onH=_i)b(Ph9sD7wrZ!0Bj5SCjBN zNo7UQ?%6-1(^5xGBqzfgDTnj^2Zu^pFfY9`n;Uxjy(+=3U$37c(4{jW?5ph`LLZ<& z8%RH=!KI5A^K_jpFg3TZ@rePAD;q;Hf*V~ub~l#;iY!LQhr;Dk*69YPD-Y!-NudRb zC;;jpJLScT1wdJyv90~$+!e_*6EvmiB9)Vp8hn#UA`k8g#3f)2=X+avsmz|uO}1h0 zK_-k~2Ww}KPXSUwCAsYjt(~Hro}2h@t~^|<#HaxW4R6MR=lS!vqJ;(=L(l!S>5_L> zd*tDqnj|Z0TV7O`;pF~}_}3J2R~8qkI|la~^eH001-P}* z5$X?eb|23$Lh+e&nDZW~c~aU*ds7tX3U|vIqT?G8B5P(Zvzwwpz-{+z8fsol{|4VJ ze=Z}ga&A~wd_@kFHXJH+Mev|7Q}vMWXQJ#iW#OJ>0YVo^>s=u-F@G4cczRGi^8J$0 zl>SoiAe4AHLf|1b@1CwYK5BkgT<_AHRC5-4WqiVS%3Xw8eTas_-Wa7sjrt2Nh*0&j z#@!`GTwp3_a(#0m+>^PgtMmf=6Rwbc^j^k}S%Nvczt2@}6+J1=HJv;4nurcc4yuye zzRQw`*v=~%?r)D6J3%eXRj1iAeC=O0tfe`{lO%8q11M9x)3f6EQ1}{I!#a9Z%GC4f;f0lLYy)UnKyk53LUMH|?EHhr zl{Y1KjEC6D9iAMWkA#i~#+1{0?C-22xghG6fk8=p2S%yF>)Ze35PoP?xN@uz2M4qK zSx9h`hnL=SkJH;k+gTAky9l;W{ey)GUNTZ;`;nt4m?r<)^cQ{yw4e!_iy9wpls$$x z**mOeOp1Xra5{c9@#THAfd6M`gsbIZ^r**qzn%rI<(xJwvZO2!s%_qA2>%Z4$Z zN{#ELk3PVe{pJ0Yz(*o^7QgObb)XeZxG(F8OCp7wp>w01C_ME!Rrf{z_H1!nYrqvI zv3~_S3m^~cZV-1jAguMv#9 zkzAJq7ohd^v>V7F2&g@Y$NPy@1A?fiHNN8%IXEVCg^;K071=zvqG~>FKZTtCo4U}` z5?_@Sxf9mcO^WbaE6W3`%_1&+r|qA;m7u z`z|5vr-0F-oa&!`voV)QK|KVf=5iRkEpd8O>`PwKab#P~M}CH>^JuDRmED(`4Vx$e z_^dG({#E_e@FA`7>$tmR>b@CE>VpEW+`~Wxczf?Ms#OB{l;ExbfZ5Q>%H^a%01?V&&v0AlyItmYPWfpux&NqpM0B|CF;*rJAy>Kw2`FsC!(4+ za>%{6kak)fnonug=R;mPr;3xNwr{?6`P-vU{rHrzY5Fu0JhoY@(AI#k!l z8KD$ZF6jdGVY$}3kak$eIj@8j?3mtI8?{h-W+|^h#y^|E!hH0PuKmf|%*%gN)GN}D zVej$#aW+Gs8@eNNR4-8+ZUIi6Y3MW6Gt!xn7dW$TkYc5EY8*&~<=_Jy5T4(dNgO}Y zD}-I{NlQUTNa$>O5#;H3o-5&rC64@U6_2q=(-^DR`xBTFC~l-hO~<8TrKR=roToF} zS|2>Lyt!Sw;ZAU2yumrWyG5~KidXNm)&mx1=m@WQvCJ`@__}_(HY>jf2frnlxui4s)+*hs3@h1lJqG_Pw3xKURkOv+|ErPYksGdw*D)zp>7|gBDceT!h z3|bffCybsJ9jd8aB&AhiBVItD1gZXeSgI{`l+OQm1A#>BWKZ>)pLddMNS6s{IdMx6?~C#{j?H6lhA>GLM8oUr4=CfKpoIRnom=v zuvWplmE~r>9dGD^M$^ZI^|I_*Qo-;((HFy2qYIXXcr6woxEGC2h6iF<`REqEhI!uL zeXDrC<>!Xy7Xx>as4GEdd#9=ncv<&f>rpiPqWFnBt5&GBa&*;g3M#J^;UKO6{k{A3 zQ5_>5=)9;_{9P{}+hOP9Em|wH*pv-ui1W+6PfdSMuTpFq*JtM<>z4KqU_IQP{MQ$n zTa*+wC@UO3^ln&LVyL)cYU!B0aH6B~_l_Ax3qFe!#`Y%*l%*kaxuMkVYz*TXT)yuety0|W zf{IZXx0>b=nx;)C$tGe16q#gp!35BN_MB~X15sUsLc{9Yq_JNujUM8K#(4u{D$1%C zM@Fex!)oWu=qb9CcmFZMH7SacC!I;=6D5I?XLMB^&3^ekHIZ#`twJIK`xONs^)D;8{#@v zs|s?f0;Iev7;y_?+i@4O7IAL24DrSBhTJHaGeSPai(U{TT1NV+11z(I=bv46EUu5# z@i%mdJ#NO`3-Ve}k?f#HQS*vcaoQU1Icyb!6O2#5f&>0^_n517_OPHB_K(fh;XhdJ ztKY_&?kk0KEYIq;YF4EpNWZ7KR2LbXiJ|S{`o2^GD6fqzD7GVykgSE}}k}4{@X)vq~;02Xjd5!F*VLRTiiS}4L>ELysyXggpzwgj48erKZK4zUUMMA3vlYG)K+Ww9N&b6G%@#VA!CSsG zBKLf`SXL5G=g5J4@27WAJ*?b)EDr>3Jtk}fgvqhz9wM#()GdwQX|J*vCp4^xf6JdL z$K#q8$=wT=>!+~YQoQ9i7;vaQdVJ3s?;sW0X+d(UV7(pYNcH&iZt3nv^)wtbhOcGB z{*td)NO9rP$8c4@V=Z{Zr(7##u>ChH*G|WfswD-w*UiSzkTIg1D^8KPW6%ao3B(x} zjqql`MnB&sQ8JBI3|uDK>as&fCC1&9Y^Z^lEe$&;(R1160c1fbk4@I!W=m2n8~ra3 zE)&E))?@P9{aGc&$OlQ%>a)w_I$wH*p{IY(T~oAfTiT}Gib8(VS(95W>xpiao^idl z6=Ag%lXR!%@b7RWF%a!I8q{}y!vQOw;G;s@Tzdg!4(=1=#(PUwex5ipai7wLAv8#3 zgn_-o%y0mWsa8X%9c1Y#EVZMCR$`&*`%1hmMX>e038l1hB2y<*vC`u;PaQCS9BcV?9++l;rC0F+a6asYS9Gds|~LF@z2^OfLKmle7TBg zah@FeYt*9Rlg_r%>|jlWj-MOk<>~HDJ$oDs^CE7n)>ulHrDlq;|6)4|h&$G2j$5YQ zBi;GHd&a4K9YCJ+u0uz#kAD!4>Rxo3)+xMhqyV2$sex5~?|Al^kEm`VS;Io}zCy{p zRcwblyNPpLUxEkfvL&}-ox>vF>W(cF!M%c_ zi4FGK_ymtd3zYAwrrz*@@y*lMyw*L;48!4KUOjPPSsqeg#07Eup&sXyIQn#}D%yEz z$uUgU%9SiSv%|Z`n0DoI02CwF{_O%$r|!|_;kokU!%8f_!@tn)uQ)w^&fW%f|JhBe zCz^jVv?B-2!@i9`JVR{pDx4X@tZSS3a}P8tbZ-7S{+RDOz(B#jz7@AwaDD%hogoW?x%@$g_*hS;4IU zpB&yTW1H~dN;dV1Rku$;)Hx?BBF|-SS=i5$xzWpDDK2I%!367@62lU^c!x&UjC}{F zz}6yN&Ay;n)4D?zv)B2FouI?en>hRUzJokDq`=Iqb8 z_1H|uyt_84yt8TFW43>5jv_`P$?FN~p7c;n%8kY1K4mE&?z9xl_;36efpJ1i6<;)_ z^I#k^T}M5qA0_?O*~hk^22JJLo0(K7v^;z$Vq=G@*+KT^OlR|}>NzH<*18F0x@&u# z(y$mHY1bq|O&$`7fk0Nmh)V?n*t3)k+x{<~@l4N=nww1LbkX;go>9X9sB5$k6p$y=Ca9Q9Hbotv|yjg^76fY0bp@Ad#Iqp9TVpF3$M0eoJnpVQdqc?KB> z1Wo$0Fk6}XJ!U_1M@=dmu)X{d5^j5lERx;Es$Mo2FBvJaM~IR9GgZ3F<>4cyZpzQ{ zVN}*HN=t?pwGbq~x4`~G3=#zj{8$eN!|Id>b0o}nlK^Kw^RuC4Ksu*3OiM58tQsuS zH|%|Uo(Q;qPHBvA7UE&aVwkKz$STf|KtF z^tNbNmG`-H93_^xFUD&Y9`KCdc5zghy7MuM{qMyfzaJ9LbpHPogDhQRp86wV2JvU{ z6GDf%3yz+^E%&Oc62W_N2O2k3Se&S4(H`sCt!@d1eu%&~v8D`LGU`&E$K==}%;;7FP@~#GO^sd!FfC7{FCO_k#8}$U=D|ojKDe z|IvLJS!7OzED1Rt%Autl8L7!(uX$8~JcA0oOpiAot)BaB24gzWZc~mBs=SzXm_D*j zGitvvy&z0V{AlPh`H5xM&Tfn2*u*}}V5{g2)s4-QHXcm6{mF#V8++2`mar<(Arj3Y zO)VdWJC#C*$9${*eDLR7hT6RC%VXfU7nz%MX^O+!Sq{NEh#S1j+gc5Iy;B!Zc+}yz zs77Qf$e#z;|Mdgm#cR9u%`kavV><>~UzS^piMJWtCY?O{qNxud$;W@@iKr$|zS8$V>_# za>Yuf|FaR{Db2t=mfvi!qp=Y3)If3>wv<8*ql|8ks*eRauO$I!YHJWnzJEiwYWMXu zOpJn5G1fklI6~u_i6jNJzyWg$DodMjb2b9S1Y-{QArDn0=5KyvB|g1n{R(S;!bhtH zq!;TQBF}sr<-Ba08tJ>{dLQhuB+IN=3*q*XPhyQN4_Ks!0DT-N-r z9qQsqMpt^!Tl;X5{A=P%ON4zaqjwh7h*KpJy3pvE7T@*><}Unp(5`-PKz-kS*I5P{ z6|d`U+CXrgGDAR?uYm_slI)7RsNcgp5qP3c1Qt3UAcNj9* z#3cCHMv<`*^^T66>$E{uj$yuS|a)phhXGM z!_yu00$A1&U*f&7Z%p^Xrccw%1^)CYxyti7%lpwSw`(fMxO6{>652$-M63}r5e_7u z(Z?tQms~+{)dVldZP1Q4MS?#qOb*>r0|-Sv8f*roPXD+#4IzS=nrYdSl9+AtWc`|D zu1$a$aWbkDOotCG=(?G9&13}ARE2D~d+X*IKPY7v`D}NXw1KNaS`6GhbW%$9lX-@h zhQY5N3;6GJ>sXOE&Ig=)g^i~8WnUC`%#0J>Kxh{4lAq)p=+$Xu(7ZWlP?@_NT8#8u z;`t_-dNR?$r)nr^X(FB?`e6sUgV!vf)kKDg>r|v} z`Y~ka9Tqhef=5fbPppXqh@E5PgNL9Xy{IrWMaNMR`3CR)TEr26aZ~|)s^0Nv?n&wh z7VJ6oB4M3i;1MI`v4$K`0}r6fwBCcyBY|v{`LMz!Hf!AmPU#oaXWDhVIep%gq&!5Y zrpv~bux2w*H8zn`NY0tTXTg_O|Mg?tZ(C!c zF4x)mcw$`CuoQD$7Ho{}O)_8Zc`UZ&2nKu%y_{A$t0A6xz}s)VZ#L*$pm6&tbub?; z2L3hE(Z%zj@WrW%=lpA2Evvu4sei5l@t#w4|3$y%mo`h~|My3hTn`1U;ve*H0>~5M zzpOopah7Hq|3_|`N=W++z+t{WfHcnWHpua2*F38-M(K)$^~r^tNJ;_+pG1k@=&~bM z{7(=G_5W4>@f{o3I4*EYuMO4BzyDxRngx7ZyHLAQXS~Cl^j}fmHfX#fBz3%dg#H5s zco&Wdc#T(DducChecGjzJld7hkum<$f`o2ap#iAo%tSb+bHdK5PiDby$@{0NfjB=g zm_?G_kMlwjsrU{t2>$%L>5;v~Zc$w_xyC}4K=PegxLlSR{fHIK33!qb@w+Mzl2~%d zj536G&-&|L#n5QL{=q5=1BK!&H{Ti|vxM=WX`lwFGo<*fc(ZZpr*^^%>)_*8HHTxB znIja+=efrT^?-#FA6x;FcrK2lA`E?x`n&&iVB}$90jq9z9?Iwe3ROYY1+uo7#n08U z2N)x-1qS7p__}Pz_EYoWrm9r;XzN?XyB}@K&tC8H9>wE3BuFQQ_8)Ih4=JJp`w2W! ziA#6IOnzemtu0kDr_qx|3T(svuFj&ixN4qb!1Qma|CkcOGBP-g@5>F3f< zd)C-9!~2Z4Ao%tCfuS25l%^EH1qK?_FXN(#?6k0I>1Vv?(3JDs#S8f9p%+ecbYqVf zIafJxeY>oP^3FwH_(cb{7r?cyqsy&XtYHp_3)x0jDljc9{-MCEt-X<~VqweZmGvv= z-z_=2jhd@3XK4uDefJ&H#UL9k)lf7U*=IVUMU7{*3jys!Z8PvIza_)54?fh?T}MRt zcxb~o#gIC#z+!)Q^UH*T<|j!p;0?7n@t`fYHpTB7w{^D!@L&3By!Wav09T%(-*c!L zhw?Kzo{z4)Sf!MBbo}C&kz~!g1Us8Kz5-!QKSB2|YmP!0#SeuoR{i}(eU28{E=T|N zU!#;55zISqZk(;(0Qy8XgDCoomK8t^l<4>|oJ?^IYW~=k*m^F^?!*=zoE~2dJ;fQ1 z6ae>qTFxc|$~%+qdL7g?N4^?!9TX91xON|!UlO$?4gGvpUl)2wwLkWlz0+lze(j=v zQfJSa{KiI6O0o}t-~RZ#`MMRJ>kw;7*qcji?n`zdg|hk8@^A*9f8XP|hjZ=Eu*>VI znS$H4AV(PlliY)I0yI}@&S^A6UR>qkW3p|+FAmQ}jVfAVY;Mv%Hapdy#{F;#Zl~%= z3c1<*L3V&x#-HEr4dja3vcs5_oDYC_W22HktCNKsh>=z|-u-7-+yf#> z3`0O)x(|nb{ByLuEQcyXxW^Fmw3_?-zg&X{9+Rzikh#D6AR-*wQVC5e<;%NT68^Td zq7KKo&VyCCXg1!HZ`20d77#Cg!B3yuZKB6}(D)eNb6-!2}C zX-e<9%vjr@_#wT}(tok{rU#b$wioYunMR(I6z($+Xv%2TSz$z_pt z*eAE7D_MI&c;=cB;0vqTP{!Ag4xFq13i@ON*$y>QBFaQmM4#ZB@+G?Li+uH!MKM~DsFOR3TA zHw1jLO4vpLJJ_?}Cvb>2+JBV)UiM|$+IG4a*_n29l*+!o%mxaJjlO)YI8<-}GYNlME!PJWLOjUyOf) zC!2G=zf(}n=kDj!JQlwFqXvr<_O85Ny`g;nT**w;rGtEBX~>W3-om5fQ#xyeKk9|N z2-e#XOSvP_?Ta$Kfc6OYSkQU9(;>eGc+hJGjlGuKlhzz~0LM1+t8wtiLrSdYW|+K; z9e-A|R_~sl;a?4N+eCVn2QVm*Q76D`8N>)JQ>uRtwgt2tK*F2V7BNLanR5{ul7ETR zuto9&b@^9CIE3(Fn_}#o&-TJ0ZfTfy^Z5;yypR|XuFs($udXp1NY|~k#*{f|Lmz19j}+^;uWt6_m9wY z;cYQzd}$UG(>3pVnD4pz0hGvE!S;uU$%-n)kTVUG%@eZ7)w`H$i09FNZK3m}(%^nM zO5pc!fRfKG|7Rvv>TBahb;UZi*%PMH{w9MfKMF72# zf}&5~z{#IZsVJ$vbX?aq$L~;*Smz!Oz_aj7-5M& z-}N)wlp&*G!;6=EN;*?3mjxk7o?)l18xp=E@vr#y5cv@_~xXf z=6n&3?=@XRgu$-Acq3FH*Q=|rsIiyr!Xf^dTn#qFLG5I;)?%IM{rzjFXEv*OGjT>m z+sxK;Y$`U?P61vCf7;MQ+x$AL6-8@!WS{;fdq6=$cE0 z;CJTjnhvxjTj$L(^9f^>>nZS;>Q3DFrT?Q^e*P&qckAo1|3hiWM^~hg(8=~ea6j^J zXoV2GQFLRL`l20KV;nK|15cGe=v(i9#S{}`(f|bnS+YCQNmb+n(bbi7$F%pu;9?^p zq-bcnJR_sL5j0T(x4Y2fzw{ zTd}PscR#Y^0Ch_5L0PWF1&tI<R*sKf_~?yyRI*fyj>1K%hGRd-3Savf1xe`QDi){S(W+d3TN zY6bKdf5#Q$pOR}D`)VlrFf{*_J|m}WM=a_AwPgy;PlkL6uLABX2g8*M^OlRcBh3q| ze5tA$i1gI+92mcx#MLyW726hytv{MrqO%Q>`B%oPw*&+XmZYm@rnPWw=RbTkg>}~C zab@GynP_hnbPVh}2+`V>E2k*v5B$j^p{|`4hHd_qFRjuk(DJ2EG#@%WwKpcn1+;)y?iO z`sqR#4iXoHx`6wBP56(G>?NiVZD2GJZ!ecCV8lF6$APYDVy% z+(b}?ZM(78?yy3ku zbPrb*MaC*CUz4i3O5giGE2Us`C!4!C>wkjx%TQf)cABY-BQu*nX%uF@SM(iut$E(a zfMC+=Ec7|ok&g@~ApI#Pu~Iq-qwM?-ArsM-`{mPMp}E#mooZ7a08tl!EJh6Dsst;3 zHsPmSYs!GTbFZD++@p)z?J&qqM6KcFlJccA&-VbHUi)Y>r~VbCM~rFu!ryPCO;;#W zOu5|^mO#5H4l!n5_Sx(6AHl)Yq#EyD^fxubN5DDFdgs!s%qDo;i=1yCdwrV8&dGjx zl>;g%maC+b;0~VT-MmewBbO+I^@CvZ#uqC-4gXyz?@^iZ*EmqHiNDF$YNA&YJ>qnE zcdYU*IV}gXls0xJt_%Y)3j4P`+?KjY)kC7^ED0=ogs%KWy%7#Yxy7 z@}+vi#v&CFyJz*#2VfbCK9OQc@jvjpqe7+Ep7c(&l7=qNS|*}=@YTRJWlBRT-pTcI z0AZk%BG+!Lc+_?XNId!0qqN@{1YnDm!9(UA;lfI-ZrU)W89CjV=m2y_kuk|KKb$Fg zw z)HwyDde>$HQwk(u*#w}PcvAxEe$ZeU6WeAC2a5VQu+OXa#+Kx=NJaKKE4I4o`wihh zlkp>|`z_v63pPng6Z30x2GL6Mhc8>Hk?Diz!Iu^rOET>(tf>i)r3DQ>TiRn`dastaHe2Qjw*(tzCW5}=*_@P@!)5aQTK7hJ=#eAy)~Q?>*wi}J#E{~CFRGI{=EBNFkg563Hue{37q>gxRH7#8`+yoaTa$i zSBU-h5U%SHr4Br>8+?H4*EJRH8#GAmk(_?#~XJ zeF2D!`|;xc7Pq%5;X1z!GQWCPDM5_Pks#lZ8{w}~h@@4RW0>fG5u z0aeK|ysNpiuo9Zwnc2=Qt-fh2F1}r2Vf^)@zYUwbVvJL`4t7EhE~ShDyDC)n0eLC- z_I*}mOc~b72ma#BXnu%W09PAb^1E7PQS5c#E7|I6lPZ)1$Jlx`yKW%TJwWZ;--PtA zfGlzU$@zC$`Ffn|@OVOi)(`mZakEMQXY>ids?q20B_5Y)IgPEEx$9?I7W_rz>F^N0 zZe`z=CMEKjQ%Fu-$uVZl{`I!?GL}B%YkS?*G3NOAV{8!>i0r)l5l89O^Bgg6_iXiC zw=KX#Vq&*>O`znFd$~OezaA60Jk%oRzAXOHzH_k%mMQ-UAFA(%GEA^4APdP5zCV`uy8x}*tay)P4*w2XIW+MA+i|7HjI;Ahi2IjOs;vJ)j zdoh&aFRdQXBFUBMITE+5IJw$mbm!vAfmYdp=6_6P#NT1Lvb_=s)=P|1ELT(M=$a_X zqv-FY(x$N5%gkH3Kss*=Lp!(LOb$z8^uKU*Os_VGuV)Z*+~4vp6L zu)lhKIdAbJL+i@xjFpQt(1Gl-17qM?!6;c{(l^=Sz;iC3^Z6u}5~Q5lS52#O3)#FT zRAiUDW`e4>XgPmXZ6&AA|6$kQ<{qDQZK`bVr9-y7#q79ZY6mkUCm45eklES!VxcX$VfoMGi zXtI=R38Hi2aNKMzfz=Lb8cxQUSxNgzd-7<6#kTPe9K6WMdF|`^Ss}J&>!eSV3vCkJ zKiEFdMkn1q;n~sCP0>KECwiss91Rv z$LRC}XSQ;E$oO4l-N+7?zs$saiA?KDBM;tKRy~B*cmA!j-q)Wkq^;j3%F&7o9fDbt zU|o|cgY)rv%ze=ke);gBd%L^RVcg9GYBSYbhrh~t$Du(h&%!bfN{z&MwgcZJZ(L#; zsJ6aX$mmt)R@3s|*W9aex#ab>U??{l_Wm$h3ZG9U%3LU-hJ5k4YsiWJ%uD5wMcs8X zm~pAr5o#w-inqbu2FFc+xm2Qb1>`gqOIy}{+rJ01r@^Eh^G|3O?^($!f-?DaUNt{( zv%JZYuA}{BZ-WqTxF8?Y9Xev}GzF~rTY%da4Cs_w-@i0l_}<(heA7F_1YWnZAJf?EvBY7{p z?vXh%q1}aso#%Dt&1j6RJ^+Uw{-C|@xeR!q`N3XFZ^J!LPQNevg36TU-2?ezDiDrcD-7; z@dd8gE9EM$S}-9m_Y!GxtpB`0COg4KCo@@hz)KY=*RH#zC@@i0gH{;^EQXs%qFrI9 z_7gdzNTd8t_5NCrEe?k_9=7)}U4AqiCxQ>^X-yKIp77j~K18q$yCm0Xu#sbAAcbeX z>fox-4L|PFycxFc%+L`@WtVERs2(Mi70F)f!Z;0u_D!88m%`oSAP@i-spu^vi&nm9 zZls~}y-B&U5(!!q=3XXSUdX92lb;H$ai4w<^=V>%t%gWNlhRs}36ro3iGHx(9h_YG zpdsv?l4hHT+EsmqlAycBz;14c^EJcSU&C$UPCJxCOk^8cgtKQMdLl22BT{g~$KXg{ zcB}jmboZW?2jGM?8x&j;4|m`9NwSn1*glVIq?!9bcschP`~myV@W%1#fOUg;gLaYJ z00QF6=i48xINThr8u+7kJ*NpNZ8{-TaYh$Rc}D?Wn|XW{PQi>X^=SfCYNZpOvllXL zM7Dp8&QXYnjcitk(toYW6uWrn6&1(iadDsm;RM#1E?ml49sVmjQTqHFXvZ6{2bnIG z9L@UBZ+p6R&FL#3zFJ-Rm^sDNy~m%rP{f7#DUeR}Idjr%holt@;1 z3q>)jFo}SgMP9c@?Yi6cPByPTG62@{Szz=OntfkT{DEv?`=`7KD2BH=3K=dbL;U`C zn|ZWOj9w%Uwov?npO(RFIZ8}T?`2F)V$3Ad0 zroZ!L(3Op2na%q)`lJ0$dDC5S&vm`@clDsFOyG6egLMXm8}UONfQ#O1=j3Sw55qW9 z`&oDqFDUx{+rIk^Jo8u#WoklecyYls22vl54WPA$SgMglVpqIr zFWtp}C%zj)mKcl+{dJOm(&leXwW<&>vNvq&G}-0cQJuR|3WsHItsVLKku6iXnu`o4 z^H_`wWT9tsA0W9G*%ChwH;~a$iXXAz@txeh~by;65`vXOf2mI|? zmlQqV1Ggx3dDBl5yYKO~XE?T8&?1=8p8tD(x6Re_A1tCIvn%Ee8+2*Y4 z$60w)-oggPgHW$1x$Y~1$~wkHeSPRGT(&4Y?)ZW;cIeC}zADSTYw^P2%@jIbuNyYX zb9{58PM-ayJDF|&JHkGixC3lg;eWg#fE5q8X8mdjvv}$?SzX%URT*HA8y$UP6t`*$ zQzk4y0dX$h@|_0qZmD-Iz z54%6Hs-l&(tIcEX#-ZL>Y3kd?E@jCr7BfnmUCd_Zi`MNGPSl2e=uu|b*-oDM9BuffOLk5)!ugyt|b%}hRg*!!6Bz-R7n>$aOvev zdzUU2wBflH=}LbgbbZi{-3b>ruzr120D^@|G8Tf#=h7G4q~TZ~Y55&fRc+>BSO((1 z{3CIijrm5FSbvc}!I)TdjA~*vE65c0wOkgvAKYb5Q2s7uVmsL<2gHx~Ykmb3i>{~| z&Iu&k3;>;pCc)fKImTBS*}IlOU5`{46G}zQ3P|zM5Z>M!b#1AVHXw4e9If-o)x*=Y zy7EXkJASIb1%bKN&AvRdfP3yavJfNbr?T529N`?{Ap?nyrezsjekpk^pKG$B z$m_TFHD%U`Qt}*tIQJesC|#I;D=Vn&O^7f_X!;AFB&1}a#Bgm65V)x&iJ;Op-!alr z%6Ww~I>0E@B6TaQMd8NjU8|qt7zRa|Arn-XZ1$c=;V6VovC^$dUle~_*8orNL)SPP z?!V=*fX$76Dt*7!V7|Wg>0}hOY|bKK-8|8~1@p5A*1TStKwcBScsLk+lxfhf_jTe#%GXw!X?KPbx>l(Qx<3VY3&C!a+>XC5=p(i`&zc}<1N9@^ z4=71D!8VD#73G3WQ$M+vN0SRSh9_e9GwRtEpc$O&Smo@S?S%I zh{H1Ry&*_L2gZ!1jlHz|2?4^u9@|c zZMX{i%Xha}trhyw;FD%tXZ)V+^p|+wxA`oDVs_Y{u1fgx{i~ne^}C-fZ|?Ofcms2G zK`haT6s@$j!mj-Iw`k9D&C1hhkJ;}K5GNG8U=TM{=a*`3oLLP18H)TI5Z$qQH((@! zWzFxc2jk+X(X6LxM=_m4->*-K9h0`rb~l7MnfbVGJzOaXx;^q}?_M{L4OrU5hjZ0j&vb88c%#E-!9V-{kF!2)X$u{%oi25e-h>GAJ~ZmARbWI>4N%%AD;5 zyd0xETqHo|NTviWDCl=BCdZGV8I?No^)iSe?XTC`P!{}c`GFxjU=#TH)2eFj2F*mK ztBK6L3&>YOa2EV=eY#FM6xNNLrGV7?A!aoD68zFuXEVaxR)}j|)SSm5a~2%|8(vvs z>p!lC>O88{$$_N!n)*;E${=c?$8mT; z<{QR+mf^~C)ZEyys5#czga@J>EMoU|4YItI_V-c{^e6u0hgoDV-Ei@NeOc*{o9=|d z*z2TU6G!50sJou|*t5*W^&BWH}V!c_R*y9Qw zx}TUE_K>gr8b1xDhUha|cp$eO6$7)gn>LeZ$&EIgR{F11<+mC*(w5Nf0$_4V9`9g#-7#CMOtZs*iw4{Du z@eN)VPx#%SOia&2T>QO|z4$^8yknCfp?hCXD?L0zpHgc8;cj~P;wR7Wj@r##brmU4Y#CsGC- zZIkP4I-AtKxt}U{>XCTx@3jiFC~2G(fM2N>qpu1qpj zobgmCF#?ljsqYcJXXz<`zlD&Fh~$vSKzVa$XYR&#CApy7#|it+HvWD}WI+Up2;ynO z@}TcPwSwbA7yf=^)KS9E)StBGtO9^zIVlqjFi~Fk-)UMNBf@Ydr z>+zGv*N6_(k|a^3V=?n+$(;RyNo{rv+pS4H%xt7wJQ z5Ic5+Bl-L;5YjXL9YNeILJ-psadJ~udqnjBeJ=>6$p8#sgQhI9&Hcg_!t00~~UrGT#?m~1)XJcAao&Shgnv?}3FzpPi*I)boB2cz)Jt%VbR{4kQSyTCmv~P0Xf1l;t znvkO$A*Yuciy_1`D{sDKxT@>y5F z1g;P>dl=c3u8<*#}XdJQqpW6P}@rXB(5w>c2}D z@jr6Dp!dZGm3^10wE-NWr0-I$H)cN%=V+)7mZ1hr$yJGQlt-XvRU-juwye|~wA=Qv zokgA8x&lJq@?|*}L?c>O_8ufI9{-(xkXS{BZz8_F@6f?|uyN9Uaf&~R3&9vSJR$A+ zA@BPA5=$u8FA62%|oGJ_ZR?R#g6Olv8B@KR7%)+METo5jS%%9C0mN zYPErUWHsvCWoyC?0%GTX&dpV%}-sAnsZwKer+=vmP{e+68 z^=;`l;FDmvmUAE5;r36^CC@;e2Qtd!iN15|FrwXuEAJ!@erUDdg+`Wg>6LAdQIGbO zVmG$^%pY;x$>otF7OsvS%Xt*PCC}P%<|r{LQe)NKe7aCZnHkZ(=T;9c2UWWSn@E0KPtGyR~g|C)Ak-w?3R7>~PuCVa5=6+QNJ zaS|C4cnrPyQR7a&v{}EM$gkylGbg$!^Bp+BoMf}dt7qY=Nn6tHw&cqY7M26*;w+5u zqLe*npJot2D?bk_47wQHsm@A+U#T&Ra<%>v?382~pM2q@o#)5#c3T29k0fm&d%m*C zwo9xRsFdojDSY-#KM&erwc>DiE7$uXL8SkTa#Cb_+dA*$+HyS=D%*z&K<8-z0@ekp zwY3#Lwl{aEywrpc)&cR79z4>i)#@1|0Q!lUK0X)w+pRi~KRH}>81H_4|I}*^GX*uN zz*3ZktR&P1iSzzSMbWX?Q&c&(Mp*Mh_2J_1oku2bpPe|lxxXDLHJeSn!r~) z(=WF6tsXgl@@D*a;R@B;3g0lV@4URfy%(~T0XSG|Tle;(%Hj3rb_k~r5!!(he*dGq zI=rKvu^{~2;#Lmam!Z5?f8h%3G!p-$K<8G*_FxD(^{8R#8a;Ddm9vT&~(zW z8U8(J1WjDdP0PyXZw6|;ZGm0Q6NoV2FNWJ_0VhR+h_|>QPy~|c5vqJ&rYSifUQBor zh)rv~{fBQnuWq>HiD$EG^Y6Oqt?W+}0N1Wf-iBmvR}VW%!C{NBmC6~EZ@rDB}5dSn!{>-(;21(w1*b_^@ysW=!0y&x!> z^&f?WoicB@J2Fvd`-tIT&}JD&~t zEs8k(4S8GjW=KJv2g*mbo8p6j3s$P4vORMK`QJ4p2|A zlJRsln8MvU>zgGwi;FK?D-9s$*vu;q%YPfV2|uc8b*vdu&Dn&VX0@I{9EWKu+7)O7 zFX*?Vc09TDyryxnDqL@*^2Ff$1-j1IYXDPWCf$%p&&ufenYy5q$`$I9uuOiRwtVj5 zZHCX(RdAI{1*LV#_geRAmqH`C+;ST|Fp=a-7$z!nIsQ47kCmW_C%M3ky0f8_76u_VsA!f6tSodMc$XZQRaRcT;>{Fx=AaTVWe6G zm%xfn_i1IXk@2ne)8=;9cH)8rf1eq(%=Dy zK87rOvua)ItP|&JIqkvt7$EssYuhMJ@#Go16sB7xUbc6C&49&LP`y`x(q#*sWTlHU z4fdj^WXl0dX~2Xf2I4;@?u*4~*fI=F!e{}T%T;(1HjUMJ3?+Vc{71Wu1d&|@T2Nvf zPM9)`yJM({VVU9k@TdId;woO^nVK_qz&)->P1jy4L|t~C_%o2Jcp@6hP(t%C?zczhI;zxf_Oa?4x~!V@q|>ARvo0n@|MY- zldQOh?1#~^lv z28+172mZSt`PJGhuJ_3|fTd&4bvekaz-aMHQaQoo2$Ail@m|b?EKNRFR%R)jx#~-8 z%{Z7tT?y@U3H6RrPiN+Vlg)Od_pcOZ9unRYW}O4K`&yJL?T*<+wu~922VjHco+jWc z_Yy*I^5iAhT9ieFjw3{kTgwX8nawx77I|&7;SFW-rFr)uEWXxp$JX%$st)>dCuVv+ zDoRRCmhC7;Y|}Ns5Lu1xwRU;I2_sY)EdK^zf!r07*($B@2iZzr8iCS})v`NGlqI`T zrU3Rz?(1T=f9ekJux&q&?YjL~s8yyX42?#UPsSUsQ8q)@Stkkb3Z;AgQVwH64@65m_ci~!3{;+5hCnf!{T(bi%gBsaVT z#o9S*4+`4QwK^TAUB{OfG)pim^!PT~|N$E?K4e*m*J3epHir z>E(IpRdzL46O#H;0dlJMnMy~Gx|;j2&HZ9`z1NE76VV$4`}b>#^3q@X@_G=?7u67H z&-~O-M=pX~62fJ}=C%p<1Dj^_%TxY8$MA-52i)JVfOzmuYHgKXqAvP%PsDwI*UfjG zn?+}YAyv@T!-wMh({)%!{M3Dx3%0#vlLGvY`_JpsKn+8gjPh*^lqx%Y{MLShFl8^) z95F;0i_4B5NtS^vLK<48i9W7`>HoJCB9AD0jlx5psMZ{BQOOoirl)|pv`nxXW7gS@ zKZ=4Na9Num#VXN(X_g}@mL6Kw8ETjy)M^umPM)_<9fT%Jsa#e>7I+5$Jz9?;k|=hH zEv*#BpDwqd02I90`{v3aIWBP6ko!QdLs}q5#A6Q!g$huS1a_dZY?JU0 zq%A8C%ebPMitgHK*}+TscK(j*?Abq~8~(CSAGqow25M>Og6kl^3VHfQ+4g%3+vWLj z1LD0l+Hy`;0RndV?TlWSB0o*osN=20H%?x9aq8v}Byzm>@oN*d#fYCR#=0r8;-qZFmx8&-5(@ z4Yl*Xh=1y(?|4&dwT|~fg7<rJT>=G6 zuYfu&9nx@u(XN)b%jx?ef`HyH5eJ4SP3Bz{`FB8 zp6YaO9S%vF>Aa3^PDR8WB`+{eT{yzNk!9B{lU4U_R41aQ5FWnzPWNn9eFZ(X1JpLY zJ=qBMkil}E!-$NLbsiNcJokmBk+&?WUt*q zw;KUV_+Dj(Vp(*R1~jcGj_eEU8y)AvZTey)8iv>`ETslSTAZ9^ z?6)0%m6_+KT!WFZ1h;R+6O>In(;!DuBkFJ`DS5 z#<>X@#zXC|DD#(#lX``$Ni>Dr=iY7zleHNWdnT<;{n{Q>wR!0?V;?D*TcCS-rcgv5 zd&cR@KIp_OosjVTMJ`r+MFv1qt;ad8Ady_$d4>5w{&Z;w)DTpPlUT>BENLkPH<35z zu|wwAcM1w?PHCgJHD}|B%MR)Mq-8iK;d#De-89WUugW-|q-GCJ6|=w{kL0jEN@bC7 z{w+tg)nQ~}s55+rfh;CidJTtPp6$peStN{^7gb*7e@ms<(=82KS{If&H;A|x0+Mxe zqeI1g@o_O2+Qm~f<M}q%t4ZD13Boe*4Du-~g7x#a^>aI}%$Cmy+i zS^YL2_!edT7f+T){_9wm*bE2w=dn;aR|{d$UYo*Fbq$-PSh+1HwY1*Zw#N~>_l=lJ zpi5k~yMS8w8N>2pxSd~=(*8Wnp|Yr8_SK-K(nlO|X)9tCZfHGVcE0QHlGcPubTT$4 zveZ6KgGVP5B;*iR!tlbCMtNuXs+!?OPC#vl_amgAFk8Z_DUrENC8$dosCA387zz=l=L61oEx?e8aEeZX=!BVh>ZB2vYw ziKO<_F2jka@mLVV6R)nPS{tzTN`!DJeS=l?MGig8XVW^PkS=lfyZ$1*yChy+EoBox4lG4>x#b`5OGBJord5%(baVi7`2p} z3)@QCyLW!ja-L}Ezr6tho;Ms=JUbsKmH_@*Vv~ioVrxO4HB6SzA8;`bxqe*AA`2ji ze3Yg?ySR2fb+D!DSVonaHcW!jaKP6BN;v!NqAHW`(*M4MaOYLUu4XT<`X7vvpHuE^ z(>BVJOy@!+Ek zLom*8*$Q&Lg5cV{I3Kz26Hl}P{;@xfk+z+vY=2F{Rv@^N0%m3{wpn^}!%;=xpkI1u zZ)&JKvdj%xF_J}lwN>;+_?_?b8+NHvp1PL6N8*CLCDQsMm>GMv<7+PNKXjzr+qPX& zN;xNsWh1%ot6xZzyG#s>O(mN*3s-XTfGPO-7I*d4@-@*H`5=ehNy+=3m~wkqk8Z&1 zG#{kVLI3t@$hB4H{$ZtvL&dJoN{3)A!5{|<&E3abZ$<(-N;?0iHL8zC%Fw@m)Qq|< z%bOP~I{fR`F^yjdaULs>HJzrEIlXJc@7$jmTuujmo2wLi6+n8;L@mYrbIYmPn>5Mz zDTo(3!BO(AxCV+%HP18bWthv+SmI_Q_oLMY`;n)*I_D`{Zdpv#9+x)!H&fPC zN?RQcgW!;T1!-KZ;*g;sbvXk%@8_7MT)kc@-zzxZ5v2>5U$1p6?bbi=nDD-UK}Ni> zKr7c6t6j4aPyaW>P~hUIV`>zmnbB?J%LP-#Vy!N1M6sPTAnRWK#L>^@k`->%;!$xk z%BD2&hrZy4-6aEa-SfV1P9_gMM(6JCc3ARn^-JN9(e7Vtu__iuq&ivb7(_TeXUxa5 zz!BOj3vE_{_Kx9}(eIz0VwJhxa4djW#FD&q+Yik?1+5?l?KYfj40@neuLUd5Vh@4E z16j_W&!HLl`b<41L2MZq)So{wlmu-pxp=*BqU`)G4%GssNK~rJnu7lW$ zV$u#;R-s6Cbi+c>XGF&JMoaFCCd!2CpiDT(lcR;jjM!fzVDKOG)(jt}sDh*@EfjL# zmpdwK8%{sq!+%(5qOZj*H$QoQHQ}067V%rGQ2keBe={RX8v=u=JP{`+|LLku)(Ky42+uvd8o3`y2c1yUAWcA_C zTR@lyqT*lD`W3E2=4SXB4~JQMw~6{0N(LV{jhuWq%o9~+<$jq-Fk~LhnodyG@I5_F z^uo-w?qG%c3Jd)g9Y(jM4*|TRgTf3DsP>7c41Jl^E1+fAJ>PrJnk`aj?3*W>@P>aY zL$)rOS0G#@+>Z<8`H$b)kJjZDIo;%e(5BZR-wFDaS{!QVmvXSmqtcLCy)CUKR)Y8( z)10?#V9h{8yyPZeN}xt()N_ zqgG6f^OkMrUVUI;%ZVVeN&NKg0D{F(VQx9no1x8rs6P}V9ikhs*hvIgjV zwmMsaVx!(LKYLG)`{5c_sB*Cx4vc4zPGv7Zua)B2jjEQkfF4Q%>uj`Sn+*ZlZPM1g z>nHLbOfRTb?)`zC`+Q##pP29BAL5JHg&2HfEPdG1YWYO*`mPNdQ(}jp1&$`|5dI;J z65=o2>KK?MC-b8O;S@T}m_RYULT4H_hM!$&ZjnYRS-!i?B?COGCHA(bv5O}> z`B1q$y}+KHKGeWJn~$vHA0Vt&x}*$h9X>9`;64f^x_#(18up8VT%VAn%~cuutG=6m zd+}1lepD#HMdCj$?IaynltRpp$ek)6G!~_Ab)M@Y5THMJw_zDMY7DRa6W~C2uproZ zu6o#hs`nqm-1!~^a$3E0cC;5l8t>)_miPTEJ*Fi3j+2Mb9lgw{_r`|STaBmX+)itc z#IqOQDHSgLBwiEr?A_~UdeFA}ue2r=GusQ|^YZ$Tt9!=H3#5AivG09d^JBfX<`@qR z&Xl3?Ldw&3Wyhb4ZkkNMmBU1Ygi zw~xf>E+#K=6`=ety_%KuIPE?s?{Boz?$RkC^lvD`UmM(UNHr3aOM_wp#94*E^IWX< zd7TaOX`CSwTz;P=9X0PJ)XA3PtlB7{<;7=-WF;)yZzLLcm6MKdPDa3B9U zKJ-!!jdcHpU(C}c?w&5=)7NFG8}K3U9I#MBY|;DF6sLFYQJj zecNJ(=*!0E+HZc{5(?y;@DBRD7?{)YE?U6=qr?81>vc%~z~_9`3Ec~L$G{Ya1&Z6A$+B`I zCIOnz`1|OAXqKu@$^6TO3_2k%n?gE$lRQdp`+1+mNpbDOh*&J7G(l!mc5dBADXslf z$v}Yl9{!w%z{Hcy{}T}87jdAx{|f{Aq~kePGTAXN!!Mh;eR+Hu5UADKbX0h8TDU)d z_cxpl-F_3h&hP+*nZ-%}u#sabn(RyF@!qM`iMtUWy6epn4-8ctX9GaQ%g9 z5AxY@%LS&Ti19MNZE%sko0U>TnQ58_i8VNoqOl%+9>Xxp<(=H4)Z zlYZNJ(eF3&w&+fYmu?492~ReuA9p?)LWYI>*2S;z%uYlidSJs_HcP!O=t060BfY`8pvok)1aQ@!FR z_yqD4a)ZkGuR=Px?YP0kTKy)6{Y=Ddx@Ob)Gz@J1YoyaQV5##6b_@ z)YjFy`CpA!SB>n@rCq3MdV9v1$a;O%*7pT#6Ht=N(Knqp=R+Eh_Y-}iDV@0j9)3r_ zv;cR*Sd@*FV8|WOjBlT*+H;~QE{c^ zvCAv=vKzYcpatId4NN zopl#XZspeyXqf2Yaa##^3%#*gx%I=)n zl|GE!D-DIP#A+r}@nSG;>CCBF1y$?#i=qc&_qaqyjb|I@UdU_`@8-jj zGwcPP287(U)g;H);vAJNc{T(%XU2B{OKRZBfd438p(^n4U$;x#zTT=VuCs0&O0KZ; zZmb-wSA*MEsYDB{p#{OjNj(U~6X{@BUQ>QsTDakHb5w!EtoqwiA>U7z2_cvt!c4ED zZart`FGtoYjzl!JF)Xe;4x+cE0VZ8H6BzTcGH1)N6c<=@8m>FSe_ds`zGxNj*OdEH z!=y(z9Xx05>SC02M=jDkdV?9W6#Mv}r&cSGwDb6^_itvy(y686;)}H>fuEk7`71>0 zEO;zjXQUCC!*><*F5F05S#$s0bncUF*7N9{7O8pSqd|$o?TPRBYi-?$2zsbW+z`Vy zArE9d*p?=YabHx?JJ!Cpo2r+)|K}zhyYjO$Cq8FhEan|=7N3Hs|MySe6!htO?xIu8 z(h~%x%>_1lbbI_!2m|~JqW|jx)SHXvG`b~A-thUyJdgWb33=v)!KUnG!C}}0i2?k) zCWy)Mk4Ik6lgrvB&E4gVY(y!GSFrds>EVsolkvdtNXb9#K(3So5AInrS8I#DFlx}R zaYK;0s9_o@F0;^;1f@R4OGaq0VeI#fBe_ zWeEzTFnXv&@j?EDk0CMj8qi>jmGzpss=2oLp({UOK>Fry^FC!*)mZXdZXbr%AsGlY zrPOp|dzZWXsKxveY{%Tp{NtFp-n#-DSlKAx8#&#W-HIdBN{J86)r`{=yfZ9d&1reW zrQND4e&K$kSFa&2Y1=LtSY7@# z8ROVf*fyoGc-kb_8{mvi3O2rESvtCXYfsQeipH`7D|OX46$x`+FPka$Qi+zT0^E zhOqYUJSQWd)p`B*v?4NL4uX)Ceds!I5Z^CU);#0zy(ndg71CF8uEiHq_gJrx+K)Km z5U~|__(*5E%>xA0Se}8A|H9l!rI|r1?64*{;;aM_6Z{2oi6|oO#=r9@wx_NTZsYY+6&9qk1lq|1iGuZ+Roa}|QkXvTcUrpNz}r}GSF`+fT`Rn*TaT6j6R&14Q}ka{>-j3|pPR6}+MKKGH!Z*}I{?35%psO`WV z)pU$uFb_sWLvNy2FaPbqEJp}75-pa|=KjnEg(3x(SA zhJssh{3xKBXtaXNZeq!5M$ye)?kM zd)=+|6MISRkXu5wp3&z-ZSqHsNs7&M{!xiSamIalr_=#6>Fn^in2N+7aeuq7H28dy zQyp(dk>Xq6i(@r+E!MwV#gxVSqeB@RlK946O6>^6DSqTu?$-3~gYLHaFGu#z=ui@l z*VZ~Q%y-or)cQLgdnGGn4#yE6*)}?>ZOxnJd-WH`k^A_WYQhaNx)LOBH~MJ4nIlfr z`jvIzQtlObMa7=AOsmRdyCp9a} zjxB^~IY-37);Ds@b>KR4lP27*Uqw6K^9so}Gs&GQUSTfV6ciW+b5H+yG_SJhTL}x! z0+H*M|H(Gua;^)^^93=kxK*f@D_A^c(6Q&KE7Ewv>+#3t@RVCmd5wntOg6+<*@}J% z&Hk__c#=Nehb)`Dg!|N)ukfmJf~a9P+;v`=H+<@wE#fXuwj}aqtqhdn%gM4X$0u9M zzJsYD6>JkqZ!KA}>z8o`P!o1sH{hf0isoZ}J3v2jma6-`U`1@{BXg)`q!Le?LPX0z zJEv1r^VmVbh16cy(Csn!psbJ)so4fL)ou4lv>P9KB0b44nQ7ve$v#z27mRL#LR7ZN z=FitKd?F`c@4^X6U#_@@JII3cyD#j!5XhfFzaV#oEX)g$3_Ws&e z;6ej{_if5@a4Ez*Lw7Gu-Q}55j_N^87}5jZx$qh9*Z7^Cp+DPnP*biueJi$zdPl1< zd7_#>TvaMD`gR#r&rT{0fq(PA5!T|*PSSjt4F}+*evN12vhjzQk{0WUFH$<`vtL>? zBMoy+NzLRHEyp=AALga$XA6Q2r8SHL@-K+}7Z{xN>ov6hn70WId6#RTOZG41e4sA# zgkB=IhEgnTl{D92o+^7b%VZ@si0J4uanfszLM>XIfdS_C0zJy?>$EvU>!to^ zivMPvx=e>&ooitH=QktUf9JO+*)w6@)3dp@ZuYe&M)_)XO@Kr=TMnLyG0+tzdYT8X zVqWpuXci^gpEs7w)GIGWxB)QlMmGtN?*%mGpZc86LM|1TSPh`Qyu-V?GfRh(7tQ-# z%`F&JbWA-~RxuMC_W6PRW^Wf@bx7gh)i+hU?LG3cwx}f_6$1Q{# zj(f-m@ApEezpr0ErGryO&C7W#W4+e*;db-9=MS0KB0}ja(oGTYqa^} zLHCm8ySM|g=~_k4U;_?D3E{o`mhK9f@3K_*D`OCq)3pweXZHQT<(g(+6trQF`3bml zAZ_`1OJj7ry&i$cG@XUx#cb1-L6c+6#KtXFHy zgE^-Sm%drY{znY(mHEQFrqqhuBD!wNMT2+8K8BZmO3tRuRrUP`F?`$-ea9zJ@#@xb zN@upSLCblO+ahjpTRGYAKH(aTB#Q|iCJ^OR_gNi}$$6KZ9lU7@`lVuKyO+l%Zn*2A zCFq=4bg)(^xlKTmq`a&(YjNot=^AzmDiDfY+*zQajU2Ws=bG4+f zul^DjK9D!5@SYsQM@aHX9Pct=riKSjXu?T9-K%MO?jlFBA4^4Pa&sp_Lid$xhZ#m> z`K#UIt&b*KcJpVgZf{?{eOntE0nQxA1)J8OQSQv@By|6CIz(ykFz(C}QuE5mW%u$T z#|UMfC?t-c)3`URUI4|EqpnCVZ`)Vc1a_-9c}`kC!wn_d2H zMeQSpQ6mZ3U6Oh5UqWSBSB|2Z$tiBHp1JB_9=GfUxl;cYIOyHK@`Rxb;o4-W+6`y* zcUf%l;i@r$^HplO#e))JOySuv!A&}K5DzwA>k9vN9L!&t5)K9%*v+30?#SqQ6}Ob; zG->nYWb;F8-n(HbeLy-@TsP!_LU z$uzY&a6et42wud#vQwEhYv<1VadlhZUv=m?uj|uaFvFHom{4t~raw1Za=X`(#^|mp z{$1Q0s^u@sQg#4v*;UfEMfoSf$0W>2)VYz)-r$EhNBr@F>FwesLw?27&oFum25f-7 zCe@alig>4i%n4h6u9Xily(%ox3OiKqPh;GsO?IMz;_gDDWWFD5Cp6&OdE;mD6calK zCez7PC2@@)fyHUadM0Fa)7`r*#Y&UEEm_sNObX0gaQY)f;g!w{q;aYeTyIiqf63g_ zkB6nD;_>7moO-fwg3r%F>hIXr#8!3gY12Y6B*<3Z3<89rS)?w%9pb5-ilmHUqKN}N zJivvJijQ>94uQAjz|q-$=f7pKd@&bK7Yc=vm%V>!We(W(oLp37^Nza0QCmZMkIKB@ zMY}%W@?`+l)NJ5ZS&cl}Lxc7Q+#ui)zaVEOnaLt;9Zeu*Nc)p^?#cQ4IZUlW%XlF0 zNoz+~EUs)8GRG57oA>s)?n!oxq(eK=y_zCgJZAJT2pOI~dy_Xv*ykKy-?Lz&@QnNrv^}k>f z`Fk+2m%KmKlwh*G@m9X);M{#n@|O*lc22ZR>E1mH*Y{I@%3OaZyQqDa&=!u*o`wyI zue@G(RB(~L?f?Ov3_-qLT#tb*Z@yAe#iHf{N5W4_pzjTy05Zm+u5?l(sM~dOlG<_bAEexDXroPAUSwd2OK*X%OOtiqR4mq2IHGTSIb=9 zzm;ikx(RIlQDNCHo@lY}z!UGFmDC_MRmY#~W6ZAZ{04A$01mizsKx{zx zuktBdld(!MYip}`gUNPRDK6Jc)>QiOdNaR)$Ycph=J$0-ZTBkgk5Ph81Tq=Pk;t%<8=4)t{XOIz*KGRB>YkYfJTR zx3)~dg$xY-ZZo0T<&5e>%wd#yN4#W;6alip@_33oAF^dIWk0R9S6i(qUW2@UQ-IiN z-uYPp9b$a;8aC+sw&Kehj`{)48-Kn^?pGXl;nc{UuWxY8hZ5P{+K?rarD=p+K}GeT zCU}BxAPw7k(?e@@bPG9(gI$17f*i5+@#5`g$wyf?+~EqLpq1JvRLc$@tPIs%=cc1@ zH1N^Bh_@0AhWBirK``kh^fIm^o+0&51wGQgUVMX2czZSVeJG}5WJKy<{GInH15orDyrgGU+EYrt*;)mkUgs72DdC~9wx&~f812P0X z1ga=<^y*NXxf~l*V_y&_Z^ksFb=Vz-i!cKFU$pdY5n$dNeyNXvykC2MsNS!H@toJ7 zA9@6o>Xh$$nQ1!j2UxrhM55L#eci6Q!L67lYJiqKyK$d0I(0mk+I>G(YKaludq(SU zwTl2rfFwgbZ#JTU-yqH_fdVPZr^U$U6&`%Sqj_QOPhzoHQuJS#?AR6#8hnZq0xhpv zx?sQsK5|!aS4^)#6GoPCZZCrPL^WrM@><6GIh1nNUw8m`s#(^b@B2AriB;x_$0#n{ zo7~k7z8zI)x_})5pxJ38!KCrV=UFqS#v>aw;}DfWlUNA;eND1}S%V9XcBf&}&n)n=rJIslIy8?ke_^s)>z zkC~hO+Ofq5BGKQ$wDV{zr8Thx;L>l3FH5t-ZyETSO;6sY2i5I3{MtuuTkvc@{I>k| z6~%l)6|)VVdmcVS$r`Iif061HE39>AKS^{F!Tqq0!!b#2$XwkqDNjczdbu+(XLMf` zd^0;L7yLdKni%zs51jdHv!=_ULWXDaj<2d7TNGZaodM}-Ob~G>Q{U1#viL2b?is07 zRO*;SgpzX(llxKTR3xgN3je@8l2o<;acWtB3V`TJ|v7m>pQpZ6DR zpU|yY!O^9j#Sr{h0`{;q1Kk}-N@8>x6uX4*Eg#h; zg;*BZ=Q?+onI+zR@~vB+uoE}ywG1z>YGC!GNB&jRY0o$1?+BX9S#lb3S1A4vs$BCQpVZv=#7(E%j&&)&93Ltyx_0 zCO^8Wi3d19k=!QM9dN^^zi3R#%%$}T`ZA{_>fwOxo$$}|X7n4aBSks;3tM;2)3c&T z3EBT6^gray4{I4& z83C8OT6N+ZB_%)OzHqzb0G_{A!SH1#)%MnVR4GMX8~}s&f&H}u5bO;8yB9JdI>iCG z3xCnTGeaHdUG?k4>Ev#wG2gbEVI%r3iPa0w#Q*$j9Vp@IHYkze!O*lO0V3YJEHwaK*>Mt=zV95^nHw> zV2Fqw%vR>I%R?lnL2?v7PwC<~V4>?8_Nq$gllK^9SUe3YV6pVJ!HH5F{8+;~mFhjXMA7iF|>p`OFK9GE2#zqad=yS?#E46Cad&)k+fq^Wv(i3mQ! z3-GQoIQkiAc6?@qJHfPuJ`omR&B(-*PWjw;wZ(FKIm?(*SfE?hR=g)=GM3B&&?Vkr z5LAn?$2g)}9{e&o%n|t)UD);wr+^8fmQbsg(DPg0NPjr*@F8pan&paJ!ZW2GUwS$%hp~70B`x;zE23r^V-}zS8vCN+|o3faB)A|H^pThX2iG z?d3QgFNc>Sp0x%3ipBJFiM*fW4B|WUM3slMhsQoeQC)`f!nul`d z=Oy{nx4XPhnIONSBG+optt_V(4~=!u*eD=}j0(mAHOQbkKTdewR92oVg{sD97+@(kb~ zj6%;M4Noc?jFdHelGP^CbT1`ev)Q!lGx-&npb?rO2CM1EZ*3NrX65EQ@BuG3l|=Yi zz`0W)@~~`1kJ4eYx6s$xh$X-FMs~u5DwPzX3!;pGzgzNoB#Mj@6tfI$-+5|T7^0Uk zFVY99&X{WH?Jv~Nx8go0pSPN^AmI?2H(!Ctl&zgA>s|KmZtQ7tFBisu!j!L z9^+?0*P5fG^HvZd&**l_qH;2tGUbzhoS|pAw znI~TC;G2voN_mtWhC6kX=I=Eg>uAI@z;5TT$rmY22 z2K8$;x2zIdb_-e+q*qXCrhbXvkY#YRZ+G8df^SQM*>7u4pBe^8^248ceDtSjet#$- zKVA(|Ko(RmIOsO(|Nnr1-%`m1PO9f`>NR*S;3;zMK81*C>-AWgcH=xV%k=zHtmBeH zI69;wpa(Qov^m-DcD*?s&yHVaLUwZK`lRiA2#X$|UnG^24OPgqoZt(Eklpl=LusOq zohZtH4TTq~C~qD8gX9KNtKco(ej8OR$wAiQO<h6&L(e^p{6n1FGwWm$(-gZADtNjm))n)%P5J!6EfZSKcA^l-<&1C2}2c|M3Cx1 zTicH-9*o%+_}rJhD+kA3MF9^F%@wazWs76QGaF>Qg8H;UYp)?XRGGO%!2pf?6`T*Rw#dv-oCr0R*Dg6a+tw)c6 zf#e}VN+Vo%w!)aUgN`sqy|w=x+<^&(!DV-EQBu>_$Bs4g+2xYdH~*WWhhw&O&Z&cM zhvjX~H5x`mQbr;?FKsD>VS`zV#SpPV>L9eK1-loxNdof-ovoHjTIhx{_=5U^3PpXX zBU>$wZ*}u0>7aUWIQFv4J|_ob-QjiIuLJMa0v26mMOEJel6fs(TS^v8@EFX@hoJtM zMBwOSI=l*ThwMN&31ab~Hb$ix@#GNa2O&z~{`wKrmsC*V$AQA>q-GVSNt#uc{yY94Qm>ZBys-sBFL} z1Wsmc>*~3(b#O-};)=383PB4+l3UZ($ZH1P-opAL$fy}h2#H8Nb6q~YU%4@Iy(%q0 zwBRnS2}Z+N&?G6rVm>x0**w*vm2za<&G@0KH14`Gpicb%bGOm23$ zzGs~ut7BHHx9I~sZP5e#I-3egZIT?yq6g*7zGf(f8JM+c<8RKBP8TPb}=%ZsCF z4jvyQV9zh?EwJklIpF!SCOYd+Q#o|%gXp{&d$aG=ypLEq&rgSzU%Ybn2)ws&(|b7n z-Dh5J4qO=FvIV=Uvs&Pm(UN7ru4=M{+M;DYQjT-7+;P`GF-{XwIMEdO2V1$xEwAS{ zSRv}uwsRMbha(oB9`G9(dnqTmHlV7^k}+Meus;Jk-NHPdIZ<6!tdCJ_FTMY->7 zI*J;5+VU9aY`hI*+uNS>G6Qy}8T#`^_cSk?z5W?U4DdmQKW+26abqKo^Ax@LjngbS z@CD*t={RCG6m(c@*BS~r+QhIqxSQ?%Gs1P6j>j^|otiNPM_KVQJufZ^VeCED>W%=U ztI1tu(jBhpLqf?zG2}r0!at?;qqIL_ee?FJrbX@g&fq@|UW<$q2U_l$b*?f}qEfMU zraRxV5ta9i_U+eVvI;9SzE4}ab!P*{d(dTCpMiz*5!$p5gZmF6G&+~&vAF=H)2vs) zC4R)63}&>g{Ct3Z(jpm)oOANTRXt($bv3xm`zbngx zUJ0=xt=JGDVfs~*INt7K$1&)y8TC78f(%-mu> zT$L&w*`E*WI!q0v-ZnLRv4UP@lQ+GeQT@Ey!Z3P6!n9SusBwH}5sxj9Y3D9z&#J%v z7xJ2SM}6I^@>Mohf>1TexYX!%3H5-AoD@|#W``%v^XLA2BWWT-73_Mvd}PLAF{X~9Y1@bv zZ@e^FZ!}!Fty2#{{k}P zKRZLpv^j|hD*Zb_1qk$HbzRZHu75}f71--2($RW z%WS~&kHhrbZTLL9OUK>E5N*4Ous-53^ca~#$rF6lHIF3LLzr>1lVvQ|0n7<6Q5q_K z+!e5Hg(4ATX10}7UKakQ@Xdrpip>c25Dt4scM);=Bk+gqsoPs2A`=F&wdb{^i`Z7q zrkqqJkZy;z{V31F(?rS|s5;SC>f&1VR)bS64@P#j$uvdJ@vd5=O!Elwn1b z0&m%!AJp3vdj?G@pmNx#wDD>OW&5|KJa8=Jo812Q>y_Or(T;Aj@bop(ZqMtDOqi@^ z@2-shtey$FA73o`QnO!Z&mU+JzM_mtMJGPBV~Dtpl4u zJVYRp+a2kfMFnReV@IF!R9^NMAH$1`CZGv@>7)5W^-4V4{zuIauRC~z)?EC+x@QiE ztu!-{s_0qKL<`v<4(ZAtCB#y&$Y}59$Ma@u?A*GTV?;>aWJhC{>>{D$N}t1JB}hCu zGtldvu+^`c%{_pobq@Uw$+i+MQ01WvCcg5jzfr+5LCI1c*K1+%)`edO%|XGkve3nq7*rkdOVpg6iv?i%WTmN;pTB z_mnL>UQObGPIHgC?+gUoqU5V{2Yu;V=%*OttE68hn}%#ZXgfhhkCK`#y;pevzcn23P*^dOn`^)4P$;YL$8JMj zdluWDL%(4Ic$N@@yc2!D)ZZ2GcCslP31^sOb1+TpI0#u67uj`f?FVybJYWRqLHMNR z@A1eW`$x}Ak4QDgSoWuYgYg(C%%_<4EVVWj-k;GL616Kcrw>s6rOngn zF4LwBMoiv_N8QzaF0+YmJuUXV26vHF!IBOA<8CzpH`F%pur5EuBNgaPJMN_2fh(9R z)5Z|@YyEbb$a5RNnrb_~m7%{txI4BguDOPsIYkIT4>}uEWjkvHb#ELcM0Oz>)3Efm zvAyaA*@IcFwCQf8wurXnE@s}PbF)+6qr>rZ+Eo~$GFUvP z$Tv;}xZ~k>VXrCB*gm9?)CMw^|K@Lt5A{V!Uh7K6Va&&&4<>I&`mfNoM-C!<^&8q3 z;*$tZ%SUU^Cy;KVVy)%`%ReIIZ+QY@LT(ug0 zOz1K>SCsp&8gR+A)My+47Q@NcFYrvy;WP3qyA1ggE7Y68o$1a99NW4XZ@GQC#*e@G zJ{!7h;OgC3MtnxGq%fS!zprc}{Gdntm*@YMQ$$>0IGY;w&+o>b7Cz~%vAP#7PFo)N ztM*>~D8F&n?=rJ9YO&KOreajH?W0~>`s9gOroc0n9)Y-8+4QAU>@C8flh@9n!&h?@vpPYss;Mh`N_{hss0zTkrIzj|dg1vG!_6$Kd zZU$m41fXc###v4b0s?oXI^{ca)@={_doX|7dbwE26CgwtnB`YCr<`O@Tc7 zTp_tWQ4)+O{-Dfstw7gS*CIomEQh!`^(j~?iaS2;dnq~pewkwadvVf^-40LiPL1ZS z1S>M&{7m?HHVBF5H%%&{4>!l2Slx|EyM}#SFMTie2KYON;^qcB(gRuSGx7g6rN|K* zcRW^%7$}Oh(|iHdY-v1VB$u}_>y<_yT(|~qv8ds0hoKxxc*vTv8rt?CIwx>^oU1|G z)zlDm%D)-ek||)>SWH*JvZU~wy$>xI`0Vc}p9YNBa2d}N-U!17TW>x^f?DH=6J{EQ z^rz`^MIf=!JkVK8$0CtkPzRbY zsPi9zTTdaz!xegYQSfh{RsH#LGd@^)m;HSW=HIGwFw6gPfdhBI7=FAN4ub~zODp@E z?7Cbn56jXHPp_iDj!&dD~U9j&vjA=(h^hq)uj@;i-mC7Kj>3dXs6c7I`sv$&}+ zQiFi-yDIJ9(BkZErZ1{N^FA2ItuKBg$`I&f>(RNrDF)wfwe_zB=KeO`{O@Oh)kl^Z z?I^{jdziz!uSzIB%6HtqxW}n|z4h zzcyI%(QxOg8dO8Rg_Z`W1JCh728?kWQon{(ty6y^`UZt9Zf=rf>fz7@xIolDMBL`Z zzgVhUFsgHkYyMNN!X*H41y*Ya-n=OAxdk3k1P8@_^Sh_(W#R%(LmnC9>@F4Dw~BFf zAj1m<#Jj@AEyRFdHPdrw!5{_}5{`0|GZE5X#aD3)&JxWAsX zW)Wj5IbM-h*967>wjKVP@d)Np<~m+szAG^xC3hNG?wDj=!ac4UgDk3rr>7_(|43$W zW{#sH=);3mo&vK+9JiovEmVSKg+}8Awllw^PX+R3nlS+^1|`({sNW@*vD{%x1e@9d zAbsy#7B;@R;d2R8jo3zaHDV)z!i|1@*U0~rKF<(c9;Lpx=rZTe)QqYy=oX-K%%!u2 zbHvUY1z(gRWp7?SC30kbB$E)-f#2XfqW^r@%!h5{iRF#OuaTurLGskfr|{F_uJs5$ z^uiL6ivXn0e6W%QEy09T;%~Cb9=_+KZAGATTg}L$3zB0!Fa@QsjiK)@bFYsciyp8RB2p@DeU}`WRws+~ocHg(?0LM)Qi{$ZE>4xN-{h+_V z3(1cqf9`Uv?1@ld{RmdHMzeM2lc&84dWdM^f$SmNbkQCwo|U@_PYkK1@mh71d+7$J z=g{4umv~JB9w4Qo=BxE`#gVbDwyrY9Cj?{Hr!vdLazHey!l7kau3);(UjZZ9Q|~wC z9Z&wg>IRc+bYU&+rhU@7H%+#J@qTj0yhuKp(i~q?S|2)w0kp<)@)4~P5!bJSxlnkY zDW+rlhY2=MzPG%g5D7WQx857b#w|`tZUwWF+lJ_8)~!B+Z|~RA6y?PKowe)h>>?!p zFMqLAfBUIE+?}b1qHtng|503cwY=rAl@kp1;h31FPXLOu1OeOGpY1q4bOIG1)@rw< z5J4V|9RI8sbUUbcQZt1FTnZ5|b|_zYs>ugI1M|2#2`mxKUj*W*8wF+Q48H%GPttdg zcM>WLv)z{d3ump`EAyQ7GIhdhz)0Lw8|-)d9tocWMl#KlTAPVx4**S-Tzb(JCDy00?-E>vy?PIFcB zz%n(j{4LbK15JEZh$;_s&h$>c{ZXI1oeK-x5v!FB!kD$Uzkm$EBYvu33s?X7mDl~I zdtPmKHHyCmw>UKen3^jw35IM;>TU+DOkN;ci{NJ?a-|&yl6~0?fuCCz4eG2@Tv=v^ z*iGU~U2M#B4#gcC-_3AeXTM{=TP1mMEN-BCSKy3du+h?{oNg+4E^s@}17gM?3Aa5S z5+BBju)qDxH^VI*HOhOHQ3xVi}B~I{xisH#f zMejT*_cx=@W$ZD%UQaUTh~*MKCKJ&y&U^JM#)LAFd&!L*5;2RjhdGP|R>zaMRZyDb z0?9m&*uodyunU}aQLnRNP1=!**&jYhm+P&*%ksbxi}0jU-M&>aUN;g7w0jHcGD+pw z&c%8vC^(bUt@KNz2P!!9SA7N8C;zv=Lu5SC6gejXCNv99i%dm5JCvLYHCUvXlHQfI z^TiWxijV4^>zEVoW0puKansx9e$ISJswZP?60^@FsnX0foGY_VTSwi(wkDOzY%)C)c=SP_iMVzvzu1i$#PXc%eV z^BW+x^~cj0OlV@jPhqa4<`S@9pVmYBk%fst2?G!6XnakbGx4yaafMtem~-EM$oE-& zitBW8ImTVF!%J1>56PN8_D$VWvFx7aZY!>R&$j3YytPWB&KVInE*0+rTH8X&e^(Du zcrr?Ik4NPr9rg2$PL|zwo|C*g#F$s}K+k9MP__*I`C7ULw`ZNd`+pya1*|$LbrzQ2RtTsMj3F$2e|A@L6%zLRroO{OJ0T>;oUx;;&fmurTtTOy8G$U_ce< zl6m{}iyXaYU;)JqJ!EL63%!Ixv|NX=Lsa^M5n6?$Srjq6e=nne$Dd=~JY}HYWsE;f zHWpY$ZnHgg1YLwn&HQr2rf`!tG|ma1bvv5Qeeu^Ee)PKd!W&#@C6cU zX@4@CJ9uJ3eeGu;G-L34Jc!#Cesj-#$jspj?%jdI1?_8-f0;U zJ|!>hjSmd_&}FH=Dq(Ss2<2}`PT>fC-M=9oOsz6l5IRD_57d&;$3K!q{90YnCyLL4 zjyvy$yUPzxh#XMvF-)mDr6iyK?yZ-bWtfttcj5z11A{732xCYRuS34M3Vw2c3;Q#N zr%5tyDj`Vq6q^NaAk%cyl=}`>*ev+ZpOfeVw7|^?)OG|agRZaG_GHL13`c3QHALb7zt$XFI zoyomF(M=qa9+UIZ`Y_L_<;G6JgQu3!ECSaqKPT{W=scKF%gEN7iY7*zgKxEm(G?#W zK`~=q{V2iKB(Wg8JHtl$&=T7UDknG-zZSO`Xe-9&~0r_?_8ADmf?|)epzZ z?o@)W|J9^@lP!;9iY=By+$4z%?;x-lJ<1PEih3nvJIWv2%iD4Dtiub z)7Oe*_XnhOR!JiqxkDK(6B>5lK&_fsn^ER-Q0VF7)+_>7SZxeTabp3MODPA-PJjKR z*I>CYQyz!SdoBGyT0I;%R z@~Mrr0hUC1&&A`9{F2kqC0R5!E~y$-l=~a-B5(J1a%`qd@O4Y2h^FD^^BcOsr{Pf? z^dlF7z&N<{?nW*9hf9mV^|wD70y|XiCQr3ikG!~xA+oq42gE2*5HSMwCJBO8&2}D9 zhcMqiA&5Ubl`VL;Q^|+~b^v3h8A{-QN_#!tn~`vRjvH1LNv=lo1jGXmcK8yEe(HMp z2kC!C7JyTr^fMp;M!|xk$8<&$H=0t)Peto;L z^Nm5l4Rt|^U}v#ol$vf>4h1ZZgHPyau&m^;`vUYQlIQI=n}i-?qe%#C1QD>PFjgc+ zCyl%d;-05P62phrwDFX%K$_hy@Je=|$jbwnkTCIA49chW?l0h4v2!0}GA6tNjQFmZ zqmWEfUxXbXE-tigmP|YiLx+g_ClhAK_ZFct>=&*&(F_8LC)hBHJe|66Qn`RURrRg( zil#$jhMfWjsYm{Ga+wyD4Ul%wqMy*{R1)uCLJ>C$9z znCK%BzsDzhY{>tz+VaLY(A*YG`STD8>)Ir2QGrR$6BtPK?ePAn6Cn|fD}vqh8)4VS zmmp2DAblhf0+PYARcl{QvFLjBEM*h1CA0sAc=+7EkwE%eAa7|nF2zO;E#sJiM2hv0 zCw>|4MXov7^GOt&xBR}mHgjGxli_$SGMb86x0JAe zFyCgLRMVS?{j0biSsI0XNPir(7p9uf)0pb#-;mGVZ@Fb~l_H0nIMr||TMA|&4gWeF z*Ut!k22XFAe379qV+cCu1wI9m5ZLR1vbfkkEQFWyNbL}u3rF9CJR_Zw-DPQBqodJ$ zhh518rqMIijONg5zAuo%Bjpl1G6gZ_?ufA`=HTz8>=EGIF22Thmn7$2ec}ezag+Bg zI0djF<;U2J&CE^v=Nh*@TsT7b{G8t0I_Ux*Z{faMvl{I<7XNtdgtitpcz<*q4>(Zc-ZHM`^$~0Nd9UhT1+cF>QmNM_LPCR$S^Rzwl5Ln88@1I$ip*mF6!jI zaV|(PHmdCWlNNsx9UXWtgI2a#C!{%SPICV?zK28BW!~%PZcK!Kc6zz+KuptT-CxA_O0ecszs_d%3fqcHlNK@tS{t--<7)lF@Mmn< zzeERngFC(MjupkYQ^0sCgJ6y}G~WjFK=~@bAy4BeSvGNtvByjJK8jL|xk;hTc*qHv zWl|pNS8_&;s^dXC>->RJOw(oMQS3Jd+$YV+qy*(@8hAEMDV30?_-4kaYcly3`ucnZ zyQz(@q)|sc=aLqdDef*?dA?&sn_FEbL(bqhZ=?hClJDMZQcxm^MZvu9JkbM()g4Q5aMk#`p3UlPjvCin4B0)OJ7 zhvq|Go3=R1Gmo=C_QBIWx*b=wu_`ZR0pV+y|CHDly1b?7! z%buWGaBm!5vc!32Es6$Z}2fsj-G3B!98_MmTfZhsD0mOm%&?+23%xR&4bK+5<~QQdq>fYOY0+yMDaxR=_e4 zj6pV4o;q7!{iu@^Q)HQsjATcE+U>r{kvUHAKrI>st7KWqQNa zm6K6-g3m)T@!f`)<$WvHPU#bHWMk;Wyjm-@U4m~}nAL-8=BCH(7lAhiPM2Cq+ZNV( zh3kjU*PI02^wC_Nxayz%nX`X#2<3@8E|lDIUlW+LlnxvMfHV%82wboy1f-5glS{L@ z@HUqGDA4ngip-Cc)T1rgE@GJRS1u|rqJb#IcsZBw^Ak}$DbTMGX2 zb?8BnA$CLxUUoS=lD!uHuv@m18SKe98%!S5?kDvdn zQ1`RsIw7SPQ~vH2G4)37dL0#XZyhIcq~ z8LdXAtcwT3VTIOp{X*9B5%AHU3+vFC&_j)DKtH2K$w&DWY6i|Z@NPUZMf30Eo`zZ~ zujgWB*e&MgpqyX2gts(1ZLBLp>chR8&{P9@XVE8sYB%U_`=qNF&C6N!knc2x?)eksosSJ)?|{U1ly9?$gu{oQIwnMmU!mQvEi zlHA#nii%V!a+|0m_vF5dYm;kHNiOS3av7C78@VjGrflwG?$>Q*7rXv^fB(GydH?g? zUXOEL=bY#BJfDPYq3!x#+UADvd#~BmU~ydcl{mvhAFs-U?xwUU?9pZPcTEe%>X|$n4JjGg+*5>29pJ#sDm?%V1J7Hd}Y|{{9R+2;~g-y-0MS z{4%Bg*ZXeWL1Rh1F9sU^#-;Q7S^jpk#oW9uL4z4W;f93%U54c^G>?%4g{OM6a@umJ zVKmS8C+^kE$%}e;X@wmoWpAXLqsa$55(7X;5W_tY2P&-XN1aa znD)8T)TNa{-d62Y8`Ue{a(cAU?YYJS2Ck1To{L^&0{+O4OHaxH?B!PP(4sgE^85p9 zl6$jP=|6rqTxy%VTtg}G$W;zCmhfPiG*$O<6{Sra)Hzq2Az^KIX zhO)3CEQ%)r(d8DTeT2@tI)I!n=(RvWoi>Gzq%@mG+59g6xzjDj=46?1@RrtWuL{TM zmhmabP)M(D`Gs{geG|-4Ql5^5yF*keeERdkez`wy@XwU;Xm7Z zm}Gb!sab?P61BTM;!av(z_Q$z+HxM~S|9%C$K@k)((sY~{|~z&fasb6jmC%km?G#3 z)-w<$6k3TNt6fESc@!uJgf&Bm9`^0o*BKW)8@EdEf3~halJfP3+aI-u{?-+>xZP$| zXD(m(fXd%_;}jh0&K*_xt8#WXgK-RIgF0M7{_J_z03saP3dYDA3ywK^TT7yjJToYa z&i3AH+=Y9lLp%q4&?q|p$_VzNaLHB{n&m>e0wLT6%icjASRT84r=}Hq=^V(N=$(1Y ztR#fEoX7{=b3agVE#S_Hua0V$1CE$6>;kGHBR79vq((Y8j#v^G?}DL%m~t_3H|S8~ zmNB*=c6Ms_#IE=1cvarUY==DPppZguFcO|?N-fO9z2^DhDoT#|_goXgfWE;Fq7z%s{kjhaB#}p-0`1Cs@=rBsNPNwS`Hj<) z9PCyv^m3OB65R(I3MnLtz;Ee?upt0T7LF$$%qx-Lb0!M;MfMb4BC}zEc|cB$RBAriDi&4 zrB%=*188hK4Q6F_W5`WZv+|Eb1F+9$B z_#xTt!W+dTCvOF^lee@Vh_HQ8qi{n(MBS!gx{wX|U`vQ;0RVZvZiCk2iFD(I;fW`B z^Xe4^+NhRIO9T1qV_Eq3O&N3gCRhv?Pj8&1!NkR4K_?N))k9t14ou!C*w-we;wl+r zT0{`0X&d2Ky!e`v(OAOHc6h~fiXJa-;S_OFL)y%6vGS?PzosV77nz1$MjM!l4Tnx9 zjaQY$puXsl-&nV_3Vw|`?mlY}4R|@NuSp5~BR`TbY0;w|J{h7ZoUJJ*b8UA7chpw* zVR}i4BxSNnCNpOU|F{~SgzNmOfmq9KQevGftt4-d|C)SuCk#uJm++*%qOW(Li+f@@ zAGZA4uBbT`amSp*Ld6M(30yIG3Z5K0$xDITP|-jiLjTicJv?HD^R>2s>P{5%&j~N9 zmACRvf9*3ktk_& zwkY9kAoO=`G6Ko?np18$ZDsj5S`!LZqMU{gL;KWW*eg1Cy3>>DFYvDLvAQ8m-3No4{HpMK7yG%lzlCG&1%C|(JQ%5=G5GC5W`kc zY;Y*3OaD@1!Of-6SA7TI^DZ&8lqIW{fA0z&ZmB0e)8Aqd#r|se@6n` z)jE}w3)juP0jNKcfiabJT)6S{!2~S)ALgXJw4ijK?s@J7S#gC6Ffc4vl{XSh@7(o3 z^|U*sgXCLO@XWBIu(KC!SGf5*<#~3)iLU2{P`1tu0QARCCn^DVN0?};dxU$9@xEE@ z4kD;fy!~AB1z@F=E1#YtypTQU|F(5<`I{D4-yOpIL;1UN4OYTA7vnG!TU|`jOk*ui_=)G37UL12u`nH&*f_Afg%lu(| zcjUIWj4r_X`TqZm(%*G_mo0gI`sC7+;hxRj3E95m>fm34_(CI zF_ZT*H$92V7eDBrKa7gb0%=P0vp~Y_II*b;;RrKa+(j^Js@YaSTxe0Bpm_H6XW!?= zTOjtxgci&~Kwz^9`HdKP;MB&pRh+0CHwQ2G@;h~eH_$z7Wx6uRjZs6lr1SS7I^GdH zKw%+EA-z95l``~3EZ7R+nH5D&C<$;|?_%8P^7c90KcV8Q)um5ZFQwv$FCg&JSv$tG zXu82&_`giBOzN(T@i5 z9^-riWo7~-JPBF8n*nrl#l*4a*WHQ#kH^<$~{oEkeAvFS)qkzT>8NI&c~J2S7L zWP98_iFojW-Xq@_{N7>UY}pScjyIHu)6Zn8ORHp94b2&_Y9G1J1`{#b(Ea8KFca3Yt)Y(0`g-LM@NH9O0fvoVyMQ6g|qC~+sCbL zl z$o9Z=BP;at0}c zU-mPkZ1mAfJ}HcLoCsYe#eJ-T*V;a5^$8;tZ9v(9;$ErqUan7JB^#+nEQD9nq(r7? z|N368xyF)0xq9!B9A7O;=q=K!T@|H&jjtP(y|JVWRD}T0OY7kCIxOV{>vYp^tnZcF zcFo4fz4Qm_KwwZl@~u-#=zq~C(}}rPn|#{pDkxOkAG&ybVkcX5{IzR~N@CaY3BPm> z&*<*oUa!_+-vQqTH+IG9Lw}z5g|=A>4^UkDlm7B}v(RX5TM#AbjbxBIu}Olu?gLHu ziAqGu={*BJ_Z9QLb%_$;NfPwKZjHd^I=+>`{tONC(HCg|0Pj5s>)hEO^nA3@l%(CKCqoG zu~U5bJh}n{f4%t=h|(nZVN9cHF-^0Yv}a(oy|f6AMc_gZI0n)>DXA`!`y5W-j0>g{ zvEl|8oUkw6OLC!Ed^4E$1hv~fBW7L7uo_;s z*zG{cC-Y{6rT9aaXw>@yS=(wlO$_fr;#=R?+zNft)A3(~GI>S)S~+GfH+9F^y!R&& zx|6i~YZ1}sf}a`-yR~)Uy*9ujQaC?mf~i$ z2y4Joqw;xLb(AL!95o2ivkCPoT&?{1IOskdzn5=@e>X+=x7`EWVleA1rof_k_f=g( zDDv>L+bWDaT{r6c@O$G;xd)G}83g6Ua7=hOIq_nR$e}}UlAzLR;&@uz;{MC9*Koj# zb07?INgkW-@@>56#$JjOrWWVcR-bYpKX2O7%z@Y|?r4R7Tim}}jhz2a@XMR21*_|! zJV|YH7*QHHpQBQl5gG@FAcp(2ikADNme5;8J{hufO`fHc1s*2utoK_ye%<+1&cpFn z`OI2HSnuJh-_O6iExzw%9*j=*&Ya9<*(&nA(SZ?4{VW6$VWIZU?^yODB5b$(>ICCw zj5u!@t+Lf)%h8fvULLqM!AQS1bZ2{nNfz_mL=@4*#U%>IOs}!U{9wxT#Mp6;9%_9R zDi&Ca&A}T}cDfS^;!dzo*61^iXOCa%3>5tgT!)wuX4LQr#IZOuCqty(7Yp=-GMLzi zbufwn>lZWO942B~h!w7&JFpa98bmd4fq&$>;b}u3tN&yPGTjk?E9pZBIER#`Hte2U!8r@Z?mBQgTP*IMPE=bgXE&B7GM>g{U%Z%cL~6YPc!^t$W-6be z*3{5zokoHsBf6aI5Cq1Z0zxAxjs#jKPS}PEc0;xR@G(R;Y0>~h*b0wYzW^6q5WL$~ zkJ`RbL|80})Xh_hJ~grHrg`<7n~gVW-aHN3{SbM8J&Jd!Cs~gOBZd_Uv=vpumix;3 zxjUiYU+Np$A$Zqrqt&dQiA9&ru+Rzl!58uZhMAb#2|#hA?ePpp!i#7s8M(4t3-k)u z1BSEb@u_@WRMdWUSu;|3;;3ktCi;$GiYw7j_8NS19X^F5aYRfSay<~gM4{Uqnj>ai zVD0=8}GwP zy>4ocn%ZogwRJqHBp$zgGhp^)RPcxqn7kAM{BZCQDjq=)wi9}Q7Mer)^a=K;c(5iU zquD){fAW;b1+?N)OOvVy_aJPxK@ikVnxKIaxeg#O0ino55BnB(HU_(CJ@2CYgt5C_2LAV%THbY|-B3LFg^PRcDcFoZ>u&8++NGhuIV1b$bNk1Dn z2w!;AXIV3sM4Fl!RSx?T`9wZn?E`czruG8j^&2;`yA9q55+GW%urQ_+E$r{Em^zp? zeMdv^6*p{#n3IFH0-TeuluasSimxfLN^ROj-2=bz7yM_8!Hy%UvX8aJQ+J{L*71Ff z8Sshb=+V9=mI*J`Dc^P=$G_>cs;?W!YkYbn&lgW3tjd;V!`EkmBg={+_%CF*$&fXR z4n^HLXQPK{@c4A5MN4qSb(q^wpB|~!=;!>7+Hz_J0>w!|W%Ffa?xXD55sPn=eUD6h z_rz`Lqo?|U+`Mp+#D)pROgMTr94p|X^~&w)3#4c~K{(?-R=p&y0Un}R!H5$5GZGco z&r!eMKweFUooX|la$4zNHfv&ke9Gj#cm4F>v}jun zXotUqM9sEQg|>e$#*1FgnnjXQ`ZIGY>|gxynU>b^dr)iAVfUZL%;a`eqxh{kv(d!X z19Pg@1S+lT@g=2a7w8~jG6DFg!i0Kp55;pUeIgDUb%Xi8GW;n|e64Z}-|nTn`2_b` zinKgt_WO0e`_m~^t1}9w9qC>ssd`U4h41}()CN8XY%enBuH=tDOeM4pwvJ%AOjvM) z1SUm6_hRiA#2>8a0}mEsb}F3PuL{nbJR)Nx;x0>!FNhc5g6q{ODSLD;C*}H1Iz>Bo z$LkuOFq{5T#B(>iq?MnY$7V{*9e8DF2z(3HTk1D#B88y-Bub4FxN*S#yj=LIO`{vk z8^qdDvl45OO#@D#7$s;%r!|Uk=tL`~EGesx|FdtJha@*p+5s%@vyGFxHATmQt4QKb zxIi0mh&wBu>cR>j;!g1mFyi0-)wv^Mm`H<`#^X=F6j=xj{DU~0>Aaj~&5v>^_|B?D zwz~eo^-o%ERE1++N|*E=;a@=R&u005;CjcEA#byHFH8T#R}Ogao}~Io2M7o(5TOcn zLlJDaXeR+RZs9c-qySX5nx1#MBze)uLM~pM%iS&?Z5RXiZ-~r@-AOt*(ZBkxc2&>f z+3H86`pTOR({JgBu!*8=9mcOdz5h4np<#Q?HFaioT)x=YCoBV7lF5o=-yeqSK@-}f zT@aV(Ai@%;4k9%a&Um`#Z0Tf6$4Q`35IPZakNbRGh-M1~nFsL;I|SSs0XGYX;7OYU zG!&-Orgy&4#}eK$#}2ubw_DGD=5A)ft^YM1lz4jj*Bm-?BT~J?)xB@IzJMbY+Ju2D zca}#@!P;===dHqE2ohk-i0Uo@{J^*-$9dlDjhfa^y);In@D9Vqe{~$IT`X2S^Ip z5f9^?TARL7!l~}eT-0JNm(B%?6VZ9rWw92|{P=%w2h_UPbDNIzy(m^V_S^BSMd=^u z*X6F{lAp@ueocc!wcOEjL~Tb{2v|`*jMUnCaK(=MT$yaYVn1XB-FOCMFlEO8Q<~87 z;vuh76q>*bA+wM3Aoyh4JSf5`arxL=xGECTluTbkC+aKudxf;{Q%SPmjcz)9x%hBcyz zzGu~&l^}a~zmTe|C7l<}OCMksO)23wxN_v}X+vPV=wQ zMK&dS-O6CeS&&16%cD1s`a}9F6$ioED)SfrD-3?EUJ;aUx$M&X7JDFs`0P>9!}0?z z@Ruz@akOaCV$5D3_a)+J+P6N0$h$yLY%eaZFZ9$1c4I)gCf_c=&$iEZeZllWt^t9D z!EQf;>t9P{EMED7a081iP{r&lF+0m1CDcX*bY! zxyXZdzw|yWO&2j-ZN?IN*=5XZf5?UT9@O(gQ_wB!J{XBs-9qqHH`j7Lb>KSmYx`om z?xIp1SvqeWwT3#);h#!s-~U$cd;0$<#t+H8wO|I8I&~H6uQF!qx(`42G?dx?@Xy4~ zugBG!;l|PtS#fH0_Nd7$7U)3K5Yec%GUu*C>jM%6v0SJE)8P`aLazy1Jg1AopP`88 zgtA;u^V$}>Ln3Ec;4^wQYQ(12bW9;Be4X3;Yv$av9p3yQ3QxgI#Q^@nt-}9r!3#&)ExqbbIP&rw_rn z$mX%;aKnr_M*Uy?th6gw#DfepQoz})_$Z=K$d!$EO!&B2es5QANT@3_Z<_7IX8^l(L^eJ?G zdlajS7+-$wYS*K+@@K?YKfJ2a#}i$%xZ29;h*D&&lqDgr3%lcZm05Xgu+hs;DHb0x zFI�-q&jXJ`A`dE;ABeM=m-JY~DBt-}75>D?x=O0CN;*q9YQ`yDiwag|lO8*$Ne} zML01<+-{Imm4H+wUvG+Sm4=mGcf~TAGk~X}R30?$YfT)wu=BwRj$}y*|GsvYVo+@Q z5Z2eijLm^8Y8t0j4gdSQ_pAA-o{t+xO5V_$sWRmhqgLh@z>BhU~Ed;DxY~Q9$Ku~@MAfW#0wA|@7&tr%+&j&@3Wt+EL$5gb< z9}LRN=%^?V(!bP|bzZ&U8k763_cspXGrCNuObF8m|B^NDjSd=^8Tc-LS_5&nFK%+H z%zp;CJvDWDt3JFgPqLT>p(f=vTPmbch}zO`!*6^);u%+;6Pvw2(=x|RdGgxrp47#0v zwUII=$=~yR^%4^NPjvAQN}GiD=EG$X6TMgaC0pscu%s1(h$feeJyF7?zVe(6!nD0} zqWMZt;}XY0$A8M>qkDZ2a0bZ2F97jLv_M9psjC&kLz%x*qkB63P7^VBawHkj$OS#1 zTbe-g+OzM|MJi+u6jy_GpDsZQA&5;-3<+9V=zbM2$rHv~h4cJH%Eud19i*1elt$C z9&T}lO)d&uZO~Z2>zfwpG$}Z55R++hCnKSPp~xo5wc z%7?Iz;fB4NP1|w5=yLvhqPZ6oB=q|Fc0f1US3o3rRPa9F<+X?q#yewvHf3`Ljv6PF z8mpC@zEOT%x5xFP&1dh((v_CVj?TJX-&F`(a_~>^-cKI9L3}Y0>xmvXCrmNZA>8>K~&za%u`$6L^20O^c;&wxE zqUw$%=G#|fW49~8b{(vAag|G(3_d9t%C=bciDmLY_QARc+3iR!*B;$i-n1b1J2*ga zcM?EYj2GZsc_7bgKy1o%=%i^nlyBe&PquXR63}D7eK!;+yhKB{5Jt{Xwykwf_<c z)QinKPfNK8VAC7CV}-e8272QJp7SMS=(ro>#=oo8IOY|~Gk0QJ#QSiCs%pNo{ju%u zis(XU#?6OBB?W+bw9X&L%z1%<-^Uv((`Z9?sk6)-r^{| z3gykuO=+yY$;j-q1g6JMm?)NOZsFb#X?&`8rojW1!oA$&#pIXMT_!e#esg7WS3Dek$m8ZoJ`c^z$^kXNvf6JgYbAth_LvwriN)FtJ(Q61dx) zBzHaf-lYNu|y>Bwe&b1Mn@oESMCSSy%c{QNtqs6^hun00&wPWP`V`h$%+8*Oil;_B` z45)R!z`9dT&9pro!>x0bNCSOOQ}3wB;0F{KW4;}(uq$NWh$!%LnGd}yKKSCV2!|FB z=7y+ZBVjpqv%LO*Bo{5FYpOwS^0|G;ypXwObI*xW=r?A9L;!R}T0sd-YYbME_8<2`=cQ;9dx=wUPs02#GIhYq%`uE$>tb%6fxXr7e+dHTdmU+z<=?f?$*{G zgmq1Le+0-!-H+UhOlq92ul#!(=!;JN=+`S2eGIxmM@kx(vcyyxJ?&jz zyc2WkM&*FXd1kswPyacK z;oX%JWJ`)SaJhYn8Sz&qShs?6HAUR^fE@YPpl}{Ji;9SqIF<5 z|A=!^sx~%hLPWW0>EV})kX*tPA0KNf@;&TkUv*K;+w!W_#zyrO6$tvnMg4m-JllPP zJ3`tc+wc)ph83tE^S{FiDQ%-G^V|5QiwOTFk5UKKd(-8S&$+If=Ilslo22I_T3rdh zR3Q*|4+I&8vB;ZD7-!R7#HNTx$J7`O@9;DpOBL9t0E{qxYH@q3#S#CM=bwT$wq}EF zVP_df_C%kAE(o^hKPGxt>eM`N~5jCN@e=Kg)WJk+r zrBl|_z6ImWjrde}cNgl_C-pOGG7h2F!>i+D@BUwkmP*i-M1>0U13SA$Qza4v5umfaLrwn3v#My&#XWq8^9~X_5F`VRc;`kAIQO$il(qr zwF|O?t3j~x;@aB}$CsQxi0h?h;hbf-HL2jc6v?1Gu)g}PcuxgDbNdtj_({1Flk#rI zBi(G>P?CC2-(<-jAl2Iw7e>~Et=U&^rSrG~2cZnDylw~-oG%;#z}Pb%;f(8R#M=30 zq%2e|F5Q&-&9LJId93l@0g~y0e$8lBxsx5hYu~Wv&}ZtXUosU&?#Q%ygg^hjnUc`gp;ryr>TdWQ`j=8wPxUy? z8^WnRc)x(jzm3ItnZ#>9ADGpFm&^?^bjz?-dpoXt0yodgKyQ_Wv%~5lQsk@(Z zms-{QpVYmGG<%7UeQy;Dz4{HA$KU2-WG|$im!GOk(+t(O%eug%#)?G12~u zuvj&Jz&8)ibzp;{*nj`5svDE6R6m)Vda$#cm2LFyt>dpVoIP@(t;2!l?PadSt?zNA z^$!AQO1zWmM#s1?qu}W{%#S4HmO$tpx!U(_YEF$)3BcJQK*dBQXj07n*JB4?PExn{ zwC#4tpTtT}#)mqV3CH-TTjfDGuc%h`ftsjY#ipQ$Bi$`>9!rw*`OvFweTu#7n}uPD zRg~cLM3sB0jx(SWiOrvoNtkq58t}1p3571y-5v%lXN@j3#6XUCVlRx*NQ!LN#y!o> z18PPqbx}$&tc1ragpp`|Zp^GS%%-D|dTXnM<+a?cpy37@u%Qn9=`|+#`uwvkQ6Bh> z@S`7Y{&%Xkc-JE@`Sd#nukbXEH_X+yKZ_Xgy&v7AQ1$&0e4dTJ)*m^SQ>pE#yXSaB ze=J5ps=SYvxmV-Bf!jZD2d^qz846)NpioXNnWk}@vy17m``4rqx|i*zTxT)2`<|ey zlCp|-XJl!7+yoi#*!EZgX?IxXm9HDd{yV_UL%d%lv?vKz(e2_lNF!geWTek zkx$%d4y+fEh@@Do3ceccLmNQC5J^|sw3b-Wk}Gf+S1hm;g&Uz4y+s3A1o1c;D<+2n z*ha6Wap|HCG0$kJHT*{jB>p|pJmNZd3+978Hl?3TG| z(&XMT(caWu%WkR`UqXoEd%_;mJ}w|S6(o@%57T&2i9vS!=$*kf;<#PE!}!wvX6%0V z$N7JISMiE8+_2@p_Khjlv+Ji`eyVf5re-!N+@*1OGu}cldvGG!11v?qM!SOMU)hA8 z+2m9853DH!MkI9rd}(RMsSz8HZ(HOX%#x<2L2tk!-D zl@E$ORU}f{he`Fa(JISwwyuK;q8g)Ve}prBMKu_vI_Wms(vE%ZcpQwF33S$YT;cJ^M$BY{*mbwu&80APr*APFa^1j>}x*x6~gp&t8SX9c-=a& zCRcj|`bLjsNPb6#al!J@BdN+bs00+|(Y;=&L%Zj^aK4R(c-Ud7AxQ z|5(i5ef1|x1tq@6$~A>@U%*?cvy73b-5N?IkoP->fEAFVw}aI8{&0M^15?Bz-=m62WBmLSM>2%|oES*Jr1-o}EoPdcZ-jhk7+xVmIt>){Jl z#ZG%a_Zat_dHgAL{Fr_NHJbJL<`8fF@<`Ri0i!b2@141(6B?g-B`x^pCN}bM`UAh> zyJ}+6BOC74bnu~?42wecyS?ZF17vT1Ei~5&;lBN@TIb%;txOCq^k7vP(_ax-uNrG2 zr~0%ALhAy=MjztkeTQ8KTjj5h9eZlgArxv{paEJ+`a0uE+_}U1()&1A*W>gHH?XE> z(xF&k(jS7U{%=00Xa#mUg$?>4j^a~1#15IDTHQ``19$p1lIB{xmYNMp2{uGHxf;NozO@}Tx(t;a*W3(^SvePD%Y zJNbzi(^KfE>7A2laZnqE5#X(vLd%3_u4>;44Tewy4)77%;H*K+=MMQ;B8{NhE9zf?( z5f32Px^XD<*kdT6l+j7v6%7d^9c~x|wbOptwLgm3mwGTjW(AZ4Lfh~iC<2H5AtIMezz+sUH(J2%cYpTGkA6jhTopYYRFlZ`B^iC&eL+Jnl~h5)@) z2iAP=Rne9rF6D2HQ#0L!&2N8;iYdJjN_6rTB!R2?F%nYwnr;_?JbUQj$*KBc!dICJ z4%IFOf4&8t`AoR~OGNij!g(3s?Gv=}T)gfL95Sp%9M_cgZ`n1{{jrPLlC}xCFY(6k zBU@*?LeJ9ll zxJSk<>-u&i>**eHbYqL-eyZszp`Un<`E$tlnbNq6h*b+(#e>EEQfuICD{%3`sQCIgXS&r>u)>uenhu&E@ZkbC4QUr( zxMaQ>;cEhLm=+N&S@{$L;^jnV7$Aa@eGj8nC4y6(TBEy1>{D~%>Hx7-kh>W>;j3@^`|D0UEDP1(7j#@VePKG!`joSLr`M_hYGo`EN~;9@7Mrk&cIt=kKf!?7#Lj$TJ< zKm*enhKEUjRgWR(<3H>WZpDc8_wHr^Q9}y%F{nv{w1B@y_TETxNV!$^%-QH$IC)kH zW-?mL{?>z^h=TnyYMPgDA2f)@&^C*=m=aTP(J9<)(ybq}SWPNVD8eCO{8_?@U}A3k zR?pX$U|&wQ^$F=&LMAUe&sU20?vv!DTXoCEeLo;Go4!pK4*H&a`VdK~e^p>eHXv0Z z7U3h4&nI->Wcm`WuynfaXB|p2_xms^tNwRNG?;yV~njQKr0X&6HFB6-pUM~FZs-5G-|I44_V;*8e7f-8DQ ziIBUV6#hy>2Xeco&$jlO=BUL1y<0><^7RqblJ%3T3hgz-=rYZhCrSTqx$vmp|HW${ z7Op&g68`c&!RHFmH$gftY>V|1W?8oQ;XSvYzn!B-$&!axI9059FX1o7JyxpDvi(yR z6oB}nQOrsa`U}b61NbEG%!gOV;3q*n+DRRMA)3wG(@dC`o^$f;4JV@aPK3R`C_JL`5SBaKco3APS6dn_TYiW;d$x| z)Z1htFFp@Y(P!7=6*TT(^Tbu7E6f7=SpM#JcnOLM-j{TvZn{o-Pb7F8BonaRN{72$ zv!i1&E-m3h+7TnQ3d6(xHz!-wmR|L}SsmkT%TZvfMX_&H#{lNJ&$m&&wxyjC!KY|k zN6L%O`WyX)(ncxVsy%*5wY5`Id>oQaE)cW^ssa0fxe3D~wm({ezF%VOer9M`E~+Sm zKYz@8n|`Hm=E&_h`NO9!-b({Pn}1b?c9`3Ou)FPF(w{N^hm&LO}iqRa}rWn?Yhu?$l+Iwm;@qKI?%d-q7sOPCDJUtwr=AhJ$gjWW;cu^UYhkVpgKnEQA5zJNC1{?`KL9>~19*cgu&8&{d?us=q%jm-KKX6xqqyyFQ3I z`1(y5(h?tF&x>0Ygsm0ZAMX@h*)W~Swo5;M)vRY7ydtwYY!C~q+&VJRHh(1R^+ zN8BMimxF&pzOjOCu%KU%;)5rwVqwAy*axupA8`v^i9P}zMnJ>_y`lsH=<4!j8s-t zyE)jjEP?0TXcTz60e=_o=W+C#PQ%-e~L7S?lGW=-s{CTI8 zL-TLXxcSzQPLpa#QES_TQOjiS*F_5{;#-dX5H_I9K*Be0W;(RO&xPnpF2=2t+poSm zfvi~0tFRk_EZ97NnA|vasaWFu)v$BKFc0-Ci&z6sre=PCU#_$ChX)od?&BZEwrAs$P0ME;zt#%+o=B zi#GQy+vqcy)y^%ix-L5LcjU#&l5&x|a7V*3>+O0dJ@ijb-nBdbOPMWU{s2o(lf>hp zKtI@ka0l)BfqM9hf@-MtUyI(G+M$74GlW+BJxYBAu8hK?{lc|(H0f>E|?Qpym_aIVC2Fq4hH`6mKAo-IS!-)1{D3rJxxlEPT`^vCkXwYqh8K9VavG3rQ+@ZL*ZcCc4nqyUi=LM#&5xA2 z(yr8tZ#}8{oK|v0D|OF1hNrxn*K)Us5A(a{xYI8-;ot)^q{bgq?;*;GDS=^zXWhOSchD`eF5Z*a11_G zO=k*k67nl?R0y>qgZeeZX7cZ9exmk-;g{sr^tA3;mBMKejfauf0TkQ ziqi4Zu4|)y<|%Zq{md3-o%EnwDD`k>#DGFFOspHn;s(_&B9btW1L9E_Gw=?P*dRB3 z&?~-rPAQMM8=1VjEoRZ7aq^9M`N$r~7V#Nk)3R_(RoH`qAAM#0^ex%u-qycKQ*6J` zZkyz}3!nV`!v5b-^B5){?Q1>X?@VWA-xGhONJ4*b8MJA5(R8eur((2%NbZv__}&4H z%D^c|Z!jb#ddIjK)k^0jT$<_#B+-N935L&@AIe-%KZWTvfW5x;wDWmW6U-I-cNv|C z5&n(r4U29LbS-=}>P{#^>`S};gEENs<}G}Bcoc6~b=wW$#$oB^=0&Z_sWm*QNfWvX z1(OawLK-e5?9&U(ILlJ4;XOHIizO{>wxSmWPL24Z3C_h0Pm)bTA6JAIP%)>Cre4>Z0#+&W%i!K)ZbP@QBMMAILdm8N1`JL zy0r39&p;yiZG^%D&P5m5xs+Z!=sv}d*Z3xFzo^Fu82Sv!V}eJ zsRqyX?>`<9RC=HBD4$=IeNc_;{gns>+Sn zZN=Zq`UeX)Os$t}2dst+m|X?NijNnqE?qwS-piC1EKzRlMBbVO)E2D2-H4@7K@SAZ zOYn2Ep(o3+9=}qy^Pb{u1S4{4vkPkMLC9u4bI6R)yqOxJ0ONvk&-rz+iO4 zN1V%$Z8gXS0b?}T@NZVGIcn8n>mI}P2Z49_ilgL0i<6&uKEizkvpHJO6k!p?RP>AP zQ{a1R|M6aBicWpO%Ndq8OEqU=OKEok&Nd9a%HF2eYwidBuJWhF5dR;?2l}q4$K+*T zyQYKLWFBOw-NaszmA3y`9yxG}GX6m5R1%PjMn zEn%-V4(JR+SXsiSl2`fZAi#Ml@+DmHwVKr9A=^oX0B6+Q9j_;MbAIRzZ=c1lK7D<< zN~a<4q0e_*%hE?b@5x-$Y%cdC%_IRFsoEY(i%0ivC&u zohdz1mvTWXBYw@ALkXt#x!H>WDUSZv5Yk-NnkyP5eev-mTj_uHbD-tn=S3zn|H$kPMtyvdnD&}Gl_Lwk;Gbw;}4{A z|83nAEkhycsr(C_^LDJO^LnEkrb8dT7}xvu!N15IkGesXlAwsUHTSU7)C;LMOV8GX z!lOn8lp!z8Gq4S9)Q=&AO&JA1ZKS2`5jOF^y1~5<>#Kj|A=HoVDrrPOQTLMp}ft+9<1QVf#l%kB^elpRj_d zuG;@gpoTY&#rw&-8i$*pf5iLt`r~zyBrgpmXU&G1eyZ74cmTRF^)ThtgyV1=92=aG zfZ0(mDxWxW@b<>ke$VavljVPnw^;gzyNf26I=eQ`kQ9gC>i690@v>>Z)Jr?2E6@1g zTMZ`H_L9N3EiYn^owu03Ois=fDKrS=MHkEj`&TCuf4 zf&?{Ugy@s+^ZO6-`s8)r_c`af&im4wm-GbI;q{WmI`U&PukwSgWv^Z-QjWOBDVZJ# zavdu=Ekr6|&GFsmH&@7h_t5SBHeAt?q4ui_Cp>cc4ZJvr0hUsx{Y}Nw%J~MU^$90Uq2f=a4KhvEHIIgYFkGJdd#)X?#Dmu7iT`LjF73uNzXZB&*H5?llOEfesY&Yu;}0wL)~Ja}qLoDt&51zD0#lu6A zLI0;PF4AqkxYmBbBG!)%jPF_d-Zz6sL>~fc4pZAH)M`yL%+PJRo&u3yz!|DFJ;nul zF+x;oy|v)|Eg+?zvP%M5c}I-nk`1y+);C$pb^E0tqvO__mb-FZcg?g8jUsBzZ|x%= zQ?&+~%e{$x!_dijecj5Ws8qfE{C>IY`QLYQlQ4hNpeCq_cSr5=K{ga2W6(&_11z4j znR2-SCS=E)tpo-Hv-n5RDqY0PH*f6yRE}v#=JP(JOCJZ{>aJ};v57<4oBbG5Drc7d zdhFwo&A^%dq&pWB$}I?pv}H?pkNO19k^>%HwFLEnCZF*Rv_gOVhztY z_aI3p0BP8T0W7N^!9rv0NPYUiGjinTF3CY;mam@f=3#mOzF{1P$)~U(( znZm>9$Bd51Dj~sPUtZaZU3A{5oN1j?LATX-%3BDV5^LGsS)~?1eY$`Py@Wo0VOdLa z{N5QEb4aU(mazlA_RfZ;+Fh%kz!6}IJn^h#h#qLso9=hxfsfl8>NTYc4)ZqB7bGIV zdW~)B;y)Kcy<8eO><|%~9eynKDSJJG7q0tta}m2#(o%Se4!qX8*J}ZoU}iBcQ~kOM z`G_0s!2|R!WEX-d>7T7V!U*08U7~aBOZ@EN#@I_z-TSO*Po-H~wu3be%r_QBf6V(}tkh4s8>l?C^P6acIh!SV$?r+6t7s7zc*4_KGMA3`L;fA0f zx#AGR;27GsF2XlrN24XfAxs{Q!=)aY?m#JreEb#8z3F>1_&XpEbev7-^;97FtoL8_ z`MUOo(i-Zk$0Jh@8}(tLO{`hsFXjBozWDbUHy(Qf^l!-&o<%J;9D>e)L0s@nx9`rI zvELf6m{Eh{@k~>!rS<_-4Wi6!N?#d!7lzKi#rZZZ{4zcBxp7H8-v8wIN8IV4%f(L` z$P}$r19Z3h`?7bgq8q2r6^}rSMr-&Iy@X0la5Iw?0xSXnkMUzj(09u;>I35)bKgk1 zZ2z0{cAeZJPK;>f%nhbS!vQtezxkzfr2b*!R<^OLkGtDcKA}|y2zM?|BR9nV2`&R5jLePsLpPk-=$0V6LD+rH*|b* z&6~KN5o|$zSrCW<)`xeqD+b^Ldt^f|8sOTASeBasOJ%JFi`O5%%1)fhUU*&pu4K&T znnZwmz;7HrNFnT8Yb8vcPcxy+DHU4;(=x9uEOSL`$P#UzTM9;eSB!`Hhw2|BW&$Vtvf z7c&qBG94m`@2Tt7Ins(c?P>zFEDKQT_7plK?c$}xQpkl$G8bXEp;QdO_z8R?^MgmW9n+b zZf3=M4y_*d5p9clzQ~F$;)+$taw?YUj#=9HwQ<|`5w@f*8T|_`>=pS3t_qME3k8;R zq3UO=7pvIcQ2`$p-V+Ch2#`Yw+v$z|Jecck=G`wJDEI#4KHAHYpgiTeap+1OoAItG z`g>7RjfaFt%@y$?tT-24MW~xm&Ey+C5kn02b~nEO zh6o4BeYrYVnAsI>!8*V;R?CgKq5axX&fvr9VAr@*Q0;9T*%Y1K$_s{N2GFoM;Q0aI zpxUypw6=BX>QBIM7c%dF|tIlgm62vz^w`V2UxbjtcD{ z_J(0emxi0;EO%%*&2nG4JQ|~CYWx!yy#D3}T?Qlke!QEA7)zS;6{(a82$nYkh)}GuV(8-x-M+!Ec;i;n6IrlNe zbJ%B~L-NtOAnrw_di?Ig^Kr5-;MmM5DFnUx4Gg0=KQ0a!=TtPvbo^!0OL?6BBv%RY zc|?+;0~PR?Y9o+JkwQoMuTG1gUa;?VBhR7|>E-jN+~%o0>4`r=_w=#vp8NJ&t1QXp z_jF)To1?f5FF~&5oZcrU9d%V&iCEf8qAO5x50-wVXntN-T9^OGlDm&y0^6rB@J)!y zFu57-;ie2CnOO2BwEH^oSC~DR=bgyILZ17+y3Btt;u1WlojFJ+={SD37{!Oy@*t^L z_!)4y5~Apr%O}~tT=YBFE9jl{<#%FbjqRY*bX|(P+fNER`Rel~c3V+#)v}O+$)=p) zUq(ZqsJx|AoRIs(UCFTeq(gaW^fbD)A6Y!0)UA1mYin6V-T}Dc<`zMHY~d))zmoGS zv@QPP0|{{I0So7L=0ricsYhDfhWD2@rj5`zz+LhQK+2yxWpNSCt)Tu0otBj|WXMZM z-IdH2k4sus28jbk7fK&23(U2d;~=`w&0>Rz>BQF;fMlI4;EM=A)-8`;*gEGVgF$i`i_t=`r%a8j?s;r^PN zTUYH7b<-awUB|uDcw}@3F}N~W0KDNg`f&Dqb@nqz}lA-I{uIHDgD@SWs1jK74?pm6Bl)o zZosvUY(SH>2i#*vq-LIS5o?L$R(&3gdQ=1{I7qQrdyy#~#A)RJl^Y)B5qc-Rc-+l& zjJ_WZJk~i*oklw~>>0Wk)C+f!MD;sDJcueRn)2=He;u*?A?Twlc%8j{7M0vCx6Rf< zZVxArx5Qgt&es;`q4U_MhXzlL8OkX?Q5_es4!$qVnUp*4l#_V-P4H@oJ+V$F)0OI` zLx|8meUez2U7zZrS)`9C(e$x2dE72>(tdLrBI7V7#GFkwo?#`DkoSvIa(UDDE11c> zK&jYD;~CiXynw_)-+oQ%>@Z~yiABP43UW|Fp?e&90Tjny`%?+Qe?=qNl4b(A@tJsr z)VvKLR+HmV-wdYXpjIQ5r0xuVdV3}GQEO72U?dspu6V$f3&eW0@y(5D@Jjjb!Mo!8 zT1{(alsyc|9!#RpTx27q+f=LE&oBD)zA}sJ=R2m)q$2_q!jF%P!;@1-7{v=Msy}l_ zYPgXmFGkz#bYN_D+B7K;Dze)1QmVh)KHZ-T5p6V7Qm?35-&EdgBr^|OJP3FuN0hKh zT^8!%Tho^WYe<QGwZt(p*ln6ipICo6BV1p;{CHA&s0TW)*a@+@ ziHyUxqkf`q7q#6UJ-x_7EpJ%(J#q{@yQrVrXVuicXA=}h>@1jhUuS%stVea8*Yq$I zM6)Dt^)CuAVE{M$voFA%p-wh3ex|!C0=t653)M-!`BXsm#yrU={@aE1jRWZ^l7DZo zg;SVLSlrqZfr_!Dg zP{nQaPbxo})+#~>F*!@O16&e^0DtV&yrh*D2hHRnadNW_mzXq-_|%baCu9-sJiIHq z=*J3&tnSzmFCk9<0d=w*04j#RqG`^^;SD8rI$t6bv(;bf7=J+7UXo!IGBoA-LAY@FWmx6>)jh^b?GwO_1^O$pGBc8sHAATz%V1U{msfoK%eu=_Ty1-HYX-Y}VCr7| zXZvhklkAre%>=fESXmSq^M+6TJx}rqOw1ib!7AF?PK%+}>xUCa536@NQz&Hdpw?~H z5ZyXj+IX>3kl<*kQLP1K22d;X|ad`2P;?ON) z;zD^5wr?G=q2 zt*(C_)X8VL_zLNE4N|WUeM7gI&7z}-rT zdJ#9?R|>XfqpC97(F0-Mow$A3L#j-;p;`s6i_Ttwq$$iHbxVD1_`4O8EXi(OukH5# zDHhiMY{j-F5UIHEmN;)nRu3AE=C{sPs_J$WpWGpQ`}S`vWD%{eef@gdl4lwHtrhZR z%dDp%!5bKvDMDa@~#UWqR~~GqHYF4)B0m&G$N9R3DcarU*OR z;z*n=)*q{s`7kSs}$J+IUiMP>vPA7EK? zRyk0-&d$Vd^LhG5Jg*Y-0-$OU0ZqQQFIU_r4C)`@`CU#2I65SfUfiuJP@6!UYWJ6` zJ+rc$*vu6kCw|6pQIuKp#zv|56^qowGETy+>um1V9{k>TUR}1{_LGE3owkRr za6PGrUv4EK#|$!{S9>2E!|R|Mv5Jdt%a6KAeNupwLa$Zm+E1nQYLF zS9@k$L65of10eq}L6JA>A^%MpI=(^``)lU;N|yJ!?nxqG0Zp`DF0VfY>_)6%x7ub7 zV99IV$A?X&vPWO3fa^NoXw*?}(~;7>ff(gh)WIRgauzsCaD~vQm|mY-B-|h;Gb-vj zXe_st(b^?>>bn11rGdB!B>jShE-1ONje`C0^J++}wK?1F6a7d5X)= z)z^%V1X8;yh`|5lsyp+;#|lkz*;vdJ!q)8pqGevVi4WNxw?I%#47xVl-rIM&}{??1)i*EXLu;w!_wX zLf~jXjSd#TkIU}iRv$vtAFm4`>uf^GxjZO`AU8}e$1MK+BB<`PM;NUlJT=YzeLFpy zCXbsTab^_hQ$%IBBe8S@S_prHwU)W(&&N9-qfv@c0fKh%Z2pJWZX1spdprtPWfJDY z&eY0<6ds_?w{Y)~WcjFmw2in6kZSe1Il&yswtWAx?TWFd+p7zIyr4TDj;-LByJQS5)8B$mbWA z_n_^-9{I*$cZcSDT;OqvMgk4`bWW&mW}>)KRyXkJ%-Jq)-<5a7C#)8#}=_ zO8Cs{)otW>`E1}!HY!chUz%fg>=w7TfZ@4tx_LxyQxh?CHl8d6&=%Jcc)(U=iVb8E zG}*8}Z11u^Gs^I0`u7{MVVAJ{4X8COirs}??eoh17J|=OxDrDv`n``mWZx_;No+k@ zi^-Nv6G!=4MO3ee>#ldgo(AO2os`B41J(BY{9>@Rh-6ogBQ)ezzMt#kcoiEXJlTgX zS{dC=XF`|dt$Bl&YSkk)K7Iw$n35XHQn5a!Rl{QL$zD=! z51P-TbMcbrwAj79iTZa{&Db2bKTEe*TC6wWgb%;6soW<_p5A2JNw!@r%o;TRb>EOi5+q)+AGkx+H{Rkx?PSu97 zLY=J7J>sPub7&8D(78SK@T_FQEz|D*(JV3|VZYntzxoXa|if6soU4ia1i_X_!9iar_oLja< zgYZFl*mFIO9|M0c7BXB0k_;(H;XfH_H!4Gp?p|*)X27Vrmhg3K4bPa#;td)GD^956 zJRplKkX(|20%tBQ=0wSyZyKI4dnU{yTyhM8H?l*Da$~Zo;@n=Lg&9RBKSg0rx9Emz z#0$+SzHqQVeVEGMTmKu?{Zw@D9c_-Ea0+VB>gnxV%Nmd2qKSSUy&M*Be)GOTud`x8 zXaGE>$L`_F>;8L#Qo8d7j04>*-~BA`<3J8nHCdh&Jo!UiNVG{)6`X_4Ur|MBki#=! z&l!=eTmYpMysbq^Fn+dp$LRyel&|alZ0>#Y`2Y?|@0X+txeg2oth)Hz67PRSOv~i- zFZcdD2iQK9p>AQaQLJ&}5&^YNrlH(8W&$p+Ow=8rlOGHAF51|HA}C7P-&aTi**@2X z0nUJcA~=@&`wJCN)JTff0SRlplWTsowGqIb2K&u|aUBDayn0!yffH#C!r@jAjX!m8 z6?cr}jLULnsrYNF?rHf1SsZXw*k&DAr_KQp5Y8Fj2Fh(Fr>GMCeBsm#hi`3OXlqd1 z2Mt1P-aJGUkz|RU8e7z?{Qx>{d0ZySz<)r`(-YEZ@rOGGWs6!RQ`Y1|og zXvqv*rl$bZr6CSK`-5SRa0Aw!UXLCDId{Hd)XL8vTuN3DEnE^F=?7EMmkx<$xp#>r z`OGTJh7t$2)$eEZTa8dVpKx?iU_odNM$Ok%%W#4(zjAV-6PL4k{PHf*tr? zg{w~_t1tUaq(9IN+0im--z}O{vmz+8c$a7V7Z?HnGN{Gou%k!O7TLG4=3S2Pak$rU z+miqp<|h(iW(=!r4eyP;I~`ZLUmQR9LLQb|)#bSEs;<=3FoP~~7t*subUEYSliW*- z@(Zubz@(8q&y6oL8Um~9G?z5jyS>=DcdKHY1DCuKTiWgFVr`OM+$EJUee+ND6~M6& z;bYrFcc?6R?JDgLS(?x=@Sh-YT=|NJ1vs^^Qr0wEboQ&;ATpQ8J>m#901B_8-J7OO z8ND9z4a~}4F>ghJ74t}nQ13!7?)C*^Jsf+9iR~YtEpCIgAslwnzdd9y5;x}F|I&UX zm{-=O_KU$~ajUwvpLp}{ZZo@YV+~Ge6QQOVMbf|<)CJL_4g7&-osQEv2E1&}-m`}@s{L&|_u>a~<_e+`UY z`#33aZ>&u*JYU$9HYfSCL7g=Sov3Yx=~Prn?g|O#9h|tR_|~k;bK0B~VtG20oQ)JQ z>JKq=IW24MXW3TEgW@opGT(NqLN|UZDC#gsc=O2V@}_)~+^zJ8Q&eCaqsmKHz z+)VM$5jO&eKTSM(!zbAU`H#m(s%&wMc_(EP^Q1Tfx~&WU17ACIorSzx(>}R;NUP8D z(KxNAtiePgWdf7C9|MXHHw#z^_4?fOA$8b7PuH#HQo}5$QOuMEQz{}SP6n}OGei?j zi;-c|@#$0+P(jrY0IV)U=5B_Su~>;qu!AZ_i8l@(ciVT$spPB<9p! zKsBR!S3+%i|3Gp`xp4i^=f5lf-Y;Sg9A8#EG@~amRCumB$MVj8Z^$5UJAazl{vvdG z%f8xOh^FP3V&#seE{p!6)#5NLSR8qKAarmJ&C_(pjEmb}b`#nTD628V-Ms zdBZF%tgp*H`)icZqM5op7VgIBB=&-*d~Q!74;kNW$%`i(bITPqCts@Jvg= zo&~69i-BYNQhM!iLr|UG1MTsE{BqyNi!w2iWjnO?YaiAfIZ*rS^-r*gq;_6%ySGI@ zzGDLqb*qJGTc=oaT{6hqYff6cD4zFU>S%{_VHlh{>Oh0abKXd4AOm$yw8|sg>lmso z0FsF`uvM8&$X_x>@U%ZkA1E`+#-(rIGk-1*kt>C7Qnw_@3=SLK9_EPtx8E^XdMRmI z2~z#5q);lO^!fTu!DJGfxqyl6o4~nen#W}iy*bZmu;Ah{srr$dh@0`h!b{e_K0mkv zMk!|(C7+tV|`Zf{_r>9e*~EB?(vkDwp(qkm^7ZfX%v5INtyELK$YOy zz4#8CGKbP<>b>4SDSMNE^0H z(umef^jk*}$jP!KPgdd_VmMw(>vpqHs@_NDmC`J9o+Ijn_m7XV+E9Fj>`Rq>Ti62i zVpgcL%ddG{`qIc!1t}Bz@G08z_zh3jp z+mf`7m~8?;c2m}GmqQIWKx}YjhT#jMV`)`-Hkk-(&PSk8x4dZsdo`-@x_0z%(hXti zh`Cc$Q<1Ndrw3Um;p;%oZ4mKl1~=A~zibU!vMQH>aI8Y#`Pw{7PFUgsRO~4$u?iy6 zjb94j3WMQqKIXupGC!yAoN9coXGsHD=XGiBu3fV)w%qRBvu2nK9ZW}Jq;w}p+vfBZ zhq6`@DjO7kv;O!r6)dj3E-iP5j=5neadl8j3NG$aWSw!o;%Vk(Tcx*0ujhHKT;_8= z+D)3xJkA{X!|r(dwA{i^P)V$|fl*DECF91%9Sq$oFU4mkeA*`2F%6J>R=^MopkS14 z+wFItAZYgZwBwt9p-_F$@S8K=XFrh#KF)8jQdS1)Tq2GtMgpgPJV)PEFS-Jk)JuPK zy1^#|O=Ai~uomRk(Pj%3ihk7&{Q7NBs>o&O^w2#gy;)i{sxhm@!d6M=w{iYD6MAC> zKG`Nes!RiUDZv#YqBhd1M%~0Bi~gKWIVjt$M@Awbuzj!hVhob$v4%FAMzXd69LXxg z#sHBew>RzIjkYJyMJKTKlhBbKBI{GXwx>*bra#LlpNQ6@d@dvPYO&lS9O1Nb;Bz~P z#fP7!zC}G2KeXhLtJo@Mw?0a!EItUXTZW0JacDVZ)p(@89noa^+3<30wrVREV~CMN z09K7i%DtZ0Vt`<2s={d+veg9E`@>h#{yikG#Ruyj?b>xH=0U=pyY(G4Ax%Uj)`uS; z5bQU;0}`5v7B8ZcYjVo_U6>0^v9-<%S;5xo9pUaDv5W%0s`2QWfHq+$b*Q@)^2wZ$ z8)uUeIFnYhJiiXYIC;CqE`FWS+w-%A)5dprxnqq|&3KeXYctLj=&$gA0O?NTyK)r7 z+9M`JR8HBXvr|>ySbso2D7DlO1ou=UoGSa|7UQzKAP)Om4y0*&Q9IesTnY7`mdiZl z8aoq^Jrj5cLyx*`z&q-eloJWiIgTT1I4F(-WxzZ%o?qOZuRioVChEtVl|nyn8G0W% z^S}NH4H~ZgEVW0~>lpQB;(O$Ro2|`+nLGbNTGNV`4JlTVC%m&7;3syrm(5~B0v!$e zl=(}Vr9Ka&rIm9k1EQY8)ftqgr?v`EKk4i?)px2w+ZO?n3H+_F-j&r+%=TUJH_)zx zcX9L!p~$5JdSV08@dKgKwS>6vyjC~CMr%^EAus-SqMX=!8|!l&SwCpOW8B%YmxO47 z{~W*dmY&c-w?8(ZCETi>l!@nE7EeiCa<#bqM2r5(xw`S{j)tCc-uFCj-Ylt)NECAP zmjPpyYLHX_Ftd6*O=I6l>|>XLml^cW&pj}cZu$>rcALe)RAtdYI{&&wj?9=eXI=QW z&bc!$BH$o@Y!b%Veg8b5R9NIw`y>yQ#Jj+Tj1tZ6MZJ-BDl5+K`5w%)%+QGVXBafg zTs-QA(J}Q3`Uh%7GBk+rP27*Hqdz|QGvPSbmx+DRY^wX$pF!_V*OSb&ak>vlA9jRH z;U$F~Sov~8iMmEtO^bL<@JmpytS{Oi;&>SU;O`|B7A@FL>9<$D{=%fV1A zPANY0Vw)&Dmrb^fH=S%nOl<6v=`26W8x12EUaY_HF-_#FeW1-ztXzpc40o)4y_m-q z7oIqyN&AL!Nqv4YRWjjc`f_ zA*X?7lEw2K^W6ted50VJE;CXqoKj0*PjHV_688bWKVi;K3?#XW5wyDjmHT2t_@*B` zGB;|~QD>pmeiaWD{PVJAfOVeVmcyFk>|9p<7+=u%?pib7@-tJwAy$alFeeQLmT6;F z43EVvG_Q#=_&_B#4un|R@FMWbCdhVCM5trli!I<0mt^r7Za(}CqW7qj+L2q|po}dB zn2`e^AA=|0e!g2h8E~7OfsYW5A4dB!v$v}>;R4RWOVd&cThVG4?=|V`v&1lA+^{?z zn3w4^yq)Y{KMO%7&^Zzx#tyj~*JUX!mlT~ui%V_z)G&ImT`T1&@D~x9p%29v_tI@I zC=b}*eg;nbDw?|^)UFfH}U;mH5jt8bj{IeCFc+$vpecciw0z?j6Y*OGhnY_JW$Qz(7K@ z$EwDGCxS7lxn*0Rbl*H1Ehnyw@}S}W6f~nLbj0z)_YIP16Mn@fQF^u$ep_e4#f@qP zlGzk_yRRblKHD&7lszkxD&^K2dmNC>J5G=OwH9Dk8>2|aWy5^O`pJ(ff()W(!t;x1 z9+ku?Q6L`J3kxr51)naY@8hznfrw#Mqwr#6q2}0Ed6G4AD$dc&TH4$%G6|8{mQ~x%XmO;3Ajl=h82S ze8n%<3&&AG>g9mdg{B=a3EqKvSCn2P?@*ysLTvq`7iRNg`fIuQ>N6!Qd%78eog> zL>)K~_ve!fFk>)`C@%lcx^Niq#XnX*IlT8W){2%UL{|w?`VpR`a9c9vq9j5}MEaF- zES_R9I>T%ww=kY(q}eZk)hp>Ik1^G(hm$rj5C$Y@a$e@qT9fMKl?bRj;&>hZ+?IlN z_)1wPJXHP2SNS93&IxY`f2tGUOQ5UZl*7t%% z3)y*Y7g3p!6S~ZyT;cI!QHJk8+`Ep$*?^HTr`M}T%4MWbE)koOA&C#D{+XpM`$5oy zdtg34UV7GUB^V&h{K#4G7(YvRl?k9xZ7^_b$d((gnxx&k=6I&Elr6r*0*+UnX89Et z#Z@Q*S|)ot9>Fb8bb+99|Hal+C!rGYYj)$L|(aY4^F zc_$VO<`mn6aMdJ4H|FMi>A8J?q0sl27Q_e}>GE6^H@h*ZIt{BB}sUDvZF24c-fChNyh}*3r@{Hher`79xwh3Ck`ff_8gcjd1JHS>*WDNVi^=kKdUtxqPafE z^nOYEgJ32&izI~x$0?`FRK;Gd$ucfApXJ-btu*41nuiY)u#0*pTf#^IKK&3?S%a9< zgWYQ=zD!}05(IxbGmL9Z?}ooFzW&_9vC%!Ryp(J#S4j)cVARFDN~k#w(^$QFnG*PS zr>WGnx<4sV4OA!?Qr9N7Nz&Ko@N%Szf=4&KYRt(USioB{-Z1zT-`SNw-8=74W0_2< z=Q1e55aMy>8C@p3A8!zzog?;cAHg7eVZ7%!jc7j&+eqwdS{zx)kJmB*>_|ToG$%Y~ z{GhZAMsvu{6)aIEjRPLS2i_Oxlla>Do9%r5&XsEY@^4*Z%(hI!M19>(U;Q2f@8v_p zNK0g&XMFsc%^dIZ=(99CTM{Ktexqq*jfE%IGpOtYC1Q?Ga2`y9n~QG}Zz^cVBMH>d z;*p~!gkcF~7HoFdyVG;F>+P?B(tI1Zr{Jv*xa2eaB9ZBkv77LXy9< zW4|!4qh`V+o3mjI>qrqwwNy)m4tqE-8a$lb*d-f8*)Da!p0t3NGMJK(!W+i?tOo5D ze#xrlPkSuWkNx6^gFe;z#0W0ft&2HaVr5PQ8z$Je(=+cdXz|YY(5j8qUSO<;2?W48 z*TvOV+fFEaWcfr*KuiT&~|4arB*84Q>JYX z)F3*M7o}gthjOtvPZEU-bTn38zMf9T>)C-Hvdriq4?dtj0(r7tMY?kOskbHAu|2X;w*8ML~h}k!$8at5Kf4; z{`Za6J<|W;7-EoC#&A`>p3vQMK*+^nnSnVYT*IO2D&HI}p3)d?y{SoJ704eIrHK^{ zKRTi9Ybxg|I4QIl@l%!ow0+xey4ED*gL=6Yft_9)HnGc3D~Pf2se7Tu_{y%e?uuHq zQbmHlAIDA|&DO!C_EIJ2VfUiIehBa@LE(wlddz0Q?uh34zDz`F;DE_X+QzM=LQ4uv zyF6}}A$XWeT>pU59W0#8Q%Bs5x>)6=3dY7GnH zVhzXiy;35*()}F^RQ^j%469)?^KV?EK(I=CH7K6njgg5cpi&_mBtkPZkTJPP!Mr~8 zkyPUDj4nXxJ*57Gqd^+$uYmV?$5vvg?ax1;T&R`2^Xend{()cAAjl)Bd2-SHx}Vd& zyC8u_5;XQ0Rpl2&6w){T8;A4W5oZi>bQXSTj1|}GDjdtzzF25p68c%(975m74Q~K& zTBXB#X^C!mylSJHg!f@Pgl$8^>b%viLdHoYjb71{4>!;T21a&sa-O}rr3rV{(f25l zc+}I3k7%k~cIJ#DC+`1VWD5OHU14UZ* zqFttlS0bmmALu}@Gn5-hffFuK@yYt}=zD`zF!CO#T{L_ju~7Tg1BvyR2ajX)4UV1t z5~x}}*d~84VUd{B+y$2g!mH#5f{Ne}vYlRYNa*nmwVT z?b?i~%N4r#SCs;M>{rp{&~hND<}Ha0`gN_^>LoKAaipvjV!Zs$^js@!E#g;Ij8^*< zwwsZ48HAX4Unz{T`K~#mmiVz*_lzT9h0Yqtgx8)CT;632usxc>Q$89nwiV2ndSA`y z-^p|(U=&DXuRl2}nDIOG@a1?~ehCriFmJ?uB`F zBgz2ky`u~bI4LPIztuiEu@9C}BcODKi3qFHal$atxi%;R@xBgb_7M<$O1{TfVF^td zs`cZXt6ePn`U!dKNeS!NC^!aId$);L@UIeq(MnY95~HP-)85=_U5@O>X+zM|YpF7c zWy6Lv=6F(pprw}wz3wIZ;{H4_$dPpjXJk?usOac6kw^@BW($V$m~5Ff1aQda%T83B zS&BE~U&>*R!dm&etVC=?U5qrP|HBp#yF%A<8Tm7g@%3PL&MLXZB+ie%OW$5Pbdtry za&oqmB;>|ss&#nwBgu;-46Y7}yFs(~UrWb0?sSTZv#2Lc4g^9JNq8ktGP3*?P*{jf71K4y50Q9K{db zV(iL9-XQvq%Eat^sEff_dzYlSADKcqlu?CPefHCb;vF>)JWGsD5+Wd^Rt;_2^*|jP zG92$&i~gJ7iKI$MoAdMjsS`6fx=~J9phAz_p6-0eNYvgRAVYOI)m1|HnYd6)fPuq7FrYXY8Eh^ zGu}egl>aCX%Cy&!pyaJe5aw<3zVr_!Yx6T$8-IoS8QSp9xYq=UF!+p|2qPp9kafGe z;}?yyV*N((?7CWud8L@|pc+p^Ty`~3?Rwk3fqwY(tYPp5a<-)GLtfa4+Rlt?x}!oJB=mG^4IxN+vCSK z8p^DiE!c~!eYBesD-qA!?#s%_1p`#l1qn3Ws{%pu(DRRGO>FB==^-gUTbh>XTpw;F zygvGrmNd$uS|X1-@To7qB-L5#nFi;URhrToHO|^fI5B~Qsq^#Q(YFw z^qBG;e4vR~Ikfoy0n}S3_WwLJ*#l91?;9()7}+PPE6Zq9<2h=&gSZ)%(eM*zX1vyL&q)W9;AHm6VKRw)$alr5)t zVbZbyLba`b+BFhe^c!Cl`(;>@xnYR~iyaSLsCGFbYkt#F<)X8 z8?e~#g9EB&EWeA?9o76iG$ePVkHfdlTPqFgetCN$b5H1`pZ05;!^*Ez_*;Z`c$kEpsDv-s)v( z)+cw2_|}QY%k*7PXps~j%ka(AotPVqwf-oB{xxZ&vU zx4L*;L3o|)wBjhP6MK>=FY~koQ+?I=)SvhAt(P05b?wxz$|KtaXS}WWKsx+;01wYh+ok_10lQ~@5oqsNs!gVgOoa;54S*4nFEL0^ab?ztr2`O6s=5z!6 zBCgXsh@kQ6qeawm*F+dM=29#FKjMko&*7mF>-XHU93su{+_E z&eFvrA}o4OPRPSpJC1L*X*PY5)~aiV^n6zb&^j^X8R&uoZtpVm#_{g$*@U}OImXK* zw5oC5ghka<-?m&vC-rG$!y&2f?QwVTvEZe=Zf`Y$XKgC8F1Ry9%1lLh`j`?u!O>NCd|TFX~SxSZ!^-om>99h&d4o2J*o9 zW{Zi4!FgrLwZ@#!wNl={I(3BwD4OBni(4Wc?8}BV9(X;ae;3b%S&o~EV*g`a0!d%S z#|Xnbh)H`>PDM?(^y$Co*Ae$rrQ#*dpVIJarXNmYtgd+GsZ)4$##o!Jp~Z-j^;RM- zb+jqOk4)ZQO;QEkcikcgF_|8<({fs7KQFef^Wm))Cx!=NiEqn%XNv2iT3DLMPtuW&e+=cMoU! z@!!W^rFcnEDdbooM5Hnd6Ds5!Iph>`NKQG795$7r97E28oX^bpG)c~f$$4hZ40FiF zW}CzB>Gl46|M>mux^`XL)AMnnxM4aHZ&LOqa-Js%LC_u z8LO*KlRp?G)Ei5#Hp39v$_0mPD1~IYtf-_Xo>hHv$&(tG()Ht%Gy0-*~Gs zF`a>*%`{-Ea}tEFL3LJdmCibz+PIuEYNn0!*XZ#MM!HF}NCt|p!|b05iuL`S`sy0^ z^j=|e=ZhY~ca03e8sD!jB>nGS4%LXri&y+wiVOLI67fPN!!6uE7UFWHqk%{`#pFy}1Hmd`%JmhwxSj z9J)(mLJ?W|!&n^k)_6x3FP-iFn{TA;*kXEN)LEyx<{xaI=nK6;F$>?hfyRvme2UO8 zN1z#%JG(5q^XlSvZ2|4L_GXJ@WtBY^@Zpc9LzxH@nOL=Qn+~8R&c6=q>*R$23wO$S z6TB;hw?R{#8nyz)p%yzBF2M!%OwPNSesFPpIyhY-70efyTE2!0JSxJ8<-6-uDIK z8MXOh@H z7}O2d6_N=}nTUwvXF%1mRWkJW$9*p$Jv^AcLNF|-K1&15y}c)3Mq2pcNAlYs-PxIx zEAho3T%5bm;womZLQyv-2-0&&3Re8Lw}3Mgn6yP0?koQY(uDBOGv}_d)Rv7mX*;D8 zI+Go%UQ=4861Q2~ltP3fmhC#4x{j+8dqLj=!wLCQ1pH~WN#)g50sz4cdS+qK=c3A0 z7R|qT88n^|t6z~^G29?cEiPPSzg(9atd>W%7fH;F?q@-rj+D{x%51QSS!1)m-F1sk4jM=DC8>RU>d%vK5A~EGeNsp zS6E#3J9@`QjRH8bUG`>2kYQXMXok=Ry}#2UlkvsnpYWq-1*^+G&aL#|OmRe7({Hy2 zru5iN69p?Pfn^dnOMtArzi}WuGy8Z}!RIJ!N5_TYS(od4D0jc%j^=T)UO-Q89;)61 zyrDL*4e2cs5D%K#1K*51_(!*ZFRRyq;(h*4Kf~1MP5xUsW=9T`Ne>6U6_!4|t>FVD z1n&DT$2L1YMc-1&w`IaCjF!mjUFq~Umkln;sq_=d@ij;xKkSBh=wMQB{cWYCixYm9 zR|(vK^;WXhe2a~h8O4NksC1K(3lo9c2kL!++cDia6aldzD0|CMqq;tw=dlh5UU~a0 z&|Jk{g;nhv5&1e>mFF6SLQ@{!I1Enh;S<-{8q?NL?A&NLjRd z33g~lSWpr#C~XdN`b4*GDg1W)T~1ff(#5_a3t?LQ;lP&KFXyzCR~1t-+az`St(csp zb3=KbP;ZwNgg{~5r$^CH`+7RVrt?)7{x^^O$L#kWgK5hRqG9idU$V;ylA zGgGd4NMkT=I&8j@Gm3HazMNM3(}U~j8ifCHq+?~L+2VJoHEGP1wCP>Lw&Q$BICf*5 z@FB4xvESn*DuV5?Pg5EFEvGCzr8N{1eO>@9zJma6#NX`AzVr8Owz%HEm3%*&O}#ZCNu=u0z#)mFNXB@z!{he zBbj&xFrkSzMjV7^7uxvu^VlR?q+&)XS?9smts6TtJ3G7jb2=wFuYn(X31xV+>hsAx z+AJLm-g?YdVED}UXPs)~*(B;O87q+6O94EDsW@D_r+0SmVs$Q=Fde(WKu-hR+cYvd z`k@cSgcw0*@b79hGg@(JTCCQe2{2k^+v^FmS`R{HRMzuE=94?wk$5INsDSrGY`JKqjC?s2p={)==-77`e?bc zk8tZZ{c9s2z4?Gxp}^(b-}%?P|0x&Awp66_y#+gBadva~`Rv}(nxa#Ha*_L*0PCMl z#Ng26xe?P~Y0k?+b!s$4&+jr5l0+Cp%(@1B#^=ieVlH~A+IJ{P*XzBV*E9cf_^OVc z-#3cs5|~ir5XNm4FhGhPre&dG1?88507Bt-UK!dc&UzI_^jT< zz&9LSyJAWtf8Yx50~gAg%!vwmi&_WmR%bxN#*2~8zI#g&siw9*vudn9TQXKp$n*kn zSKg%@`cD&shTQPy1lkabLiV_9qRfjGylkIuwYorLs~{n$Hs%6(6*F*-Mqq5RID zY8MpDki-e@ukdzh;dK+wA(zFyz6>Y1$HtDIjP(lSC0>(nm-SDS=Qob)x-=@keY-;0 z{<;zRzbAb*o&{UkcskO?J?PiMf$DbX-7#e+T9iq|*o$)jQFxiAlhE2%t!~j^6`Z*3 z*tFU8vEs1euvy!IW!Qk#Wy0udZ=lk>_lCVvl|)o6bdOD`EU_i9i2jb?RF^zGLml}} z69BGH@0%$BuL-9_QYny`^FKQcTf~Gu?Y1J;5bZj8?ZlrTwNbe^p_`ty_aPz*%<^k7}+6PT`L_U z?6$prD}as!=t>%b=;n`)aW-BLYjqfq({HM;mDUq(z-(T_iU-K2^uFu`!y|wQc+9lq zjF&r~yX!Rl>~&zAFK6B`u8tUN=-9OIxAJflBQ>ef)4h(D(O#V2fd2a>$75v@FDI~K zE3K|&EyXhLSJp2`Pb1%bl!)YHj@s70^{OO1reEaG!By;mC+fzK;UNF#_$}UhHK{(# z;KAk<;Rl`EK|Ic1uqg^e&1UUJx+<&nR|6i*_bK&j=y2dBUkMDN_^Cd30(3*pU5sPJ zM0928Q}*vD<#aU#TCm+{lIqs3-R*25i><5QYnqY^8`oEKn_QfPrkB)IER==v&is@Q z-96;6f0_5vhqHE9qfp3o10TUX@+h)~QC`DK#Y4$XCOXv57nd|av`nrmZJhH#XwPBIRhwHVv1Gj*bTOmuNV6alZkefEnn#MD^lhe^}vs9 zy%m})g;OX>zZ=NOx2Bsst&8BdDdmPBGZgxKgMa!i>cS!s{IdYMN@ks~ZAxsqb6A1G zp?Sr;!`6yMgnbgS%Qx@T;9prIiRq{-+t^NiYPaZwa}w9p5YImhw>cYe9;F9Uycr!I zZC`a1`2KBAJQ_BsWrmt9lTk&z&wK;$)>9&QhkL9kt`YCKKAi|9W`da{)=xty!5knj zp>Q6I(sx{$bNSA0MR%X2FRNL4J|J+=ZhRba^Xc+A+T<0`mZ~#q(0BP@41J+2es(9e z(o79L5YP9q5ryz8p<2?utB&IzA98O@!|K`!^<48UZ4WQasYxom`MiABcc*Mm^NEC` zC1!XnQA)`8&`xj$QJ#@6NPCHF8(8r``( zM)^>D>*Df{akD=i<2284ZtG3H-8H|2+47Hy`_xBwu{v0^Am6Vt)j9|V%d>#)Pc0Gn zmsH^z)P}#x+QgMN6KdbF_~MulX`0p7l=2#@*2qttJzanWXZbZ=I&VQ34ZJN-O?>8rADSKb=g3% z4^&L()uaQ%*_%C(V5@-0Ptxr5UpoC{IP3b0qjLG4bv;CT6UewQXHo*RGz$%O*vy$%lAlugen@YJomv@fE8fqFN^;%-RsKFu z(k-^cc0Kh*>;pAf5uEX@moc;M!Rn-rp%ZlVvtIkMhlS6QhGoMq7_W4#ae ztPssv$Lixo1nu3bDM*kSU&w*h%(no=EBR>ZT%tGjK#9@PA5*rq zG@B;=BZIetGiZF70Q!Cq^R=+aT~xF1U!W#OpMH`3q1@))J1v8~1zXwY#NW zotOLuje&NYZ_$j{e;KLdJykzyn&piZUmv+ZWLk%nKu`P8i^h;{=V2NTh`=xrO+9Y!CEE}BOh+}iP+gTM$sbs-fv`nUGo zl`p1BaEC2qdgsi!9#l^%R7BmMwF~Vo0|je&AN|b^uSmz}iK*WAHQXO+z@_6ITHll& zM45!OTYY`gN@iPcf(u_enhMnxkt*|=n*Hu{tQE6)n5|?{4IJSBa>DM9J9a_4(aSgy z9s2@AEDzQj z1Nva*jRGQNwe+c;BaKl30A!i{xH*&_oeYIRXd(Bq=zTNu;{ajb}{~O#+Il zC|``-)v9*fCAe6I+hkaj`@-fkrtS;m(5^ah{lHj#n!A#i)k}t9l3_`LZMp)4>#4ZD z8zWXjo92XuNolPhptqN@ik|G#yT1YV zb}3z})h-OJp=2f8GV@!^Mi((vDA>bch<_+F&sD+u@VT@=1!;Ci^loYC&uYdh4d=uK ze~nE6ALY~Ex6G%Mv~hi_RvMtF=jqjT6!{*N;Hr#53-91A>Ag_+W613M#<(Gs2ed^j(w}&l0JnT{Iq@jEdmtWK&-u*NQ$tO5pxhFogmbQ2Y zfsUAGn%rTX4~OJDw39}BG&75sFcONRh?s`fOmGlhPS zKDZJ_@M)Eq_d1@2MHj@-><231dXo1g&Wr zy5^G|nG4@8h1bh`=~muIHkm-*A&9OU1<08w)KT(j|LU5rlEXtfjSHk%7qnk(z^yX1 zkWpxYP1zk?W7uuCQQiG405w71*92OGKPDcd9&h@S@s(0RfU5Mj6ul~OT_eR+qa3s}>-H_c74y?c*?P8*n)@0QmRb#0_5g;~pEV{Je&Y#LwX>cA=J=I4 zpL@V%0z;ECYW`wwYyr{nQOCY!LDitGj?0AKMQg`FA3b17wRRP zx_)=b`d;rPCJlbh0geDn+iY)w4~ep$X6LbiM6K7*Y!b?87?-TTn6HVn(=(lX(VR62 zVJWW8Y4+V)TJqDgInlMf{SN7fpzOsel#65U4}lG5+Sw__j-mzM4hxpAqYY)W0Hvmx zl8w|dWh;&nQbR>~x?p;iRaJMV1Gki2m2;YmNfOik!FNM@dFxPcIDI6gPZ2E?^8_*_ zKI$yhTls-O9##?USI0)W^(SrmZji7agqa=#*^5c@x$$4a{9d#v`bbda4y~44DgEyV zW6VaXCeWh1qvbtv;xUWwfnd7r5?7mJzE)+o=iSCfOmY3}=OkZv{39~|ZI!$Z`L$pb zbh_(|uFdT4m?!)ez|nPi>#8kfr&wBMX9vvZkL2%}*~?R|l;SL#74Pc) z7G&diC8fW}>YzI)Z+n9L*4RFD=B_RE8e1c)Ek1^SwxdLvU#geSuk2R?$uhi~A;TKU z7NuKn)&+S)^tXU-#n^?_U$nR)MvzOiYe3^|wmKZ8N27N9M@fp3^FLa-;Lbkk8No>D zt+jGLmO`ovsD9xQB0S@{B}KDw0P>|u%1P)XOk~feS*r~=8swBLHb)Aqx$1qG0TdLm zo93@Y=<^>SztE5i*&EC_*u8l8jPGPRUghW=4c_Ja?%&9E)ZMfyW!Ej4_WHYlYfKX} zHIsd`nsRbm$C1e%h;FU^S3Eo>eXtOVm&<7-Wqk>P*xuax&S$TRKJwb+hk1+ zxf&kldX&3)*@F^y94l)_9^=>l9dUGw(-aS9}+|QMijtw0H(KH&|~=uee1qnuUJ-_%36{&#u$&Nj%$hI4jOTu@gXlH1s4clWT{`rWRbkgRBO=d5_XxH z9}*MI-@Fw%@BlpRL43C-36RUXGw^x9Ccb^9t*dF}IzB|giTnUvSK>v%R+bq~`JEJS zMnGgvH3b-2Ju(-{Kh0mspVN^L*k~TVag<*(@7ysyl2w8cP9+uUoj!1Uc_1Avcc$Fd z?3r`B8KGw)+}a%52<8>&_p-khx358QnX@* z@RNR*g{n*{>xnwNh=e-}NUU}t(A;9mZTf3p-mXmQ%}<{lA)OhOvMG|X5C>*&o3L=r z7A}HVilk3IW?;6U)PeAWV-o!m0*x@LzNoD|e| zqd%#ONy^Fx)<_t010LB)n3Xt>-{f<*BinBcu84v08qI`Ts`F9a8jWFA1-JLZBD@L^d~LM2?_Zt zVU{FfS;;bz^vu*i`l@LkI)%w-5S)#BW2wLq5oWzh$G7*M92>kZj#6z*=II=3t?tGni5V$XYku|(aN|vZxV7~-Qt6P~tH75uE+>g( zwe7{o>eRTD)=1o5?)9@(&I61ueP3xye$!n1l(vY6Zh{)nsl6N&w&w{JvG`K2yvX0E ztP|40BLGn9efMt@`j=p^`{^hruc6MER7P1A08ZMW>x-$!qy(M=Cw-!m?bnh*rACcorMhj(V4Y+55lXkA85h(qhYm{ zUYOCpsB$w>5u2^n9u(pf!R@6eyWllXejVkHOOKAl+s_X3cjmgcYXkF=73l6t6!-Q` zUDY*+dv5BkUx+1&pHucXvrC$C39(0V5S4Dj?XJskd}CsIIj{kTPbNF0Q+T)&gUiz$ zy`6Upz278UeL>}mX1E%3=lr$uFKYcazsKAw6Hyos3B<}E&6Qb#dKLmtifl&a0(*a2 zNs=4+b;}j0vh3mIMD4u~1YQj@CNTvQFaccjl+^liw7zy8cKkr(4W{{zJJ|-o38^bo z7J~FE6F-!<;cd7$Yvsk2mb_?fdT@7<0LNh{TOorD&f*-#<4RO;Kh zZKV#l2wfCdJk1g1eU9)S!zenF<-Gv)gYxl06|2`Rtg`7AOxU=#+647UiP#~FX|Ip` z(;9h4Vl(QK^9P+h&zW}N)-KlT0UcrOxfhgNJM#mbC`7~ALEV!v5}3QHSgp*F`Us;d z&(Z@!W;#cH06`|a$P)3P=&~4858V3`0k^aaSR+i}JN8}#UR01p%O71!gL~1rA4s7; zc2Gy7qqeNcUcS?OQs?{jYP^%>O=-kEdl-Yd7#j)O=Ld4FXETebv=uFVo+m@eu z&TpE^RpQ^XQ3N&Sssa-8>(OF7aDD5q?OmwuPGuwmR_;>a|9(CIs!vt*bTv)0(3o1`}1_mCi zvTOl+$Pd#i&P7dBiOk>VL~txci;J@~IzI+0Zc&S`}>Ua%_5ss-B z!-U)_)F}xfZM(2LZx#!jNZ)eY_-0*bvvK(_w?5YQmE znsm{JM6ZgSK3|YKZpAOh(YjmopObmfK0CVRO%lTZi&svXdioKniBE7edL{7}cgr|K-P1UY(W9xk=0XgX)WFR9;@H9ymx4;5hELEGHPOhth>rrx2$w?oA zCBZbIC#U|Ck$tS<{5JH3>|;)InoZDZ+86o2D;SLxaX9q8RIeXD$)u&kaCrc-tg++} z{FJGYZ*@7pTg=2oZk4d?eZg{BoIFXWtrzNR;$OEQe)$k*od3>~8t{ilO0Ex4QBDWCA)c~>hD~u`dLrI>-**$Y zb58nGw=`?sp5nLR9w$0YiU;G;s@qiGy2~>&;iK?MGE|WM;EKU*)%)&|17Y~gTzvz2erJ|9sfqMJN;)oH20#kN?jiTWRh3HbWkZ%gN#FBHfmIS)(-on?mDRhdvE zmIFfYSB8_-9IaozOKoYtu@#w|mReG#ut$AOPlG+L6T+60@}kYaERPGlyIcHvWH@dL zC<{mMcqYUc4z4VDqf(BC5X2%qR?Okvgy{{cTprF|UgXfrgUSih@`fnLx~5s%w`)9o zd_t*MduWGqoR7%@z4H?g9nE`x?zs$m?^oAgv6ysDfzKe3W8zu(VK8yVnDpr>{pZBR zKlzdGBGP!yv4i4v@3da8szmn!U1KI&x(D%NARrF*33OgY67rf#^afoPksE%s2#Q zGSY*hM~WpakygC)tUj;5b=j4Z{~b9PoJQ_ufxKV&(Ab$)Chcz980_mje%LZd_{ z^g{s~DK>x@{xPay(c?p99)Yi_OGQV@Oi)I1(j|E4MY1_i+x3DfZ$@)jge_y$gNq#^Z2!y z%y~%<=pk1kn=cLe7JF#{2Vj5F|AY>=nT=PHTAhZ{e2pZQnzbBNuX%PdQy=-4DA@@> zpDGzmF#%#hV3p;RZQ+tvCVHC}MesLn!N_`KzVZu!+yCUr+58br3_`}#aNf|AT?N%N_JOq$B{oBw+{u#E$$a~88A1+e>f@-ZUju2-cF;-c(8MNO@b07zw|mbhh- zN(63^7`VFf?nt=4f9xGNBnVf#2Db4pkP3iYV zwvetGqT@$f%>qBk#>u|@xSZoRIGNcIi}GNHRFNlx_YR<|wQolDAIDoNrI%;Z^GO-| zzK6|=9`8IV{yj^*DiYUXQx;&i$=B9IxxVX|-s4CXDlOZ{48H*N_8Df%z4?GNez)}P z_g{frHE!%klj-Of_o8qYL}jm*k(>X})NIrp1zV1m?{4*sFxdpL@#e|`@#LX%NH#W{q zz|(T=YXB)lMv}8JLNC&pMw8pl2^#bRncP;W)?*@8jAdkC4^@S?%XHo<_A5dL zlACWUXw;(JlHBEx0)=ejg$(@ zOm%k(9sShZ%gUK)Jc9*HTfg;21g>%O+#|(E>)Na7HF?LS)IIm3--*UEc4T%* zUqjgambrY;ujaev+y(E0327>$9Z@lk4t!$FeDtC}T`wj9i#bhg0Z&qlFpgE8{^5K_ z6!V@f8+1icN`8~?0h5e-MY2?h-)P!!hJQy`ydHvcOcU6urwP=08*u?7P-egZFmnZ| z<3wrCV>9#kTB)il)~dN_XhtdAVj@n?H>QP7Z!kgGS@^)w1yByOeJ=&ado|Zg9 zK4fC{Nf@b%L812!-_)>uPgl3MyYlY88PFs$v`nThcbgn~6=)hgY2C5aY+a_*Zgul! zl7Jjq+JnTxq(J!cj%axbe|(YqDzB#6TIGWdE)uL$n#mmbSI_4jI(b>6A2*oOn4h*M zc_oY>ZXq6|QV*htZhNR@oNE+PnZTQ(D>y>Rd zpu7s08~$7@DBm90i{=FZMO@rzc?8xdkm>{Qf)0p|+`0w;4R-d;=LwyC{}R2fBuhda z=|>u5WocL5#1epprBFTmogGgarZh5o!vp-LCX%}D<)Z=R(MAvT3?9r#ph9*UhU-Yw zaf&aLSaP5EN#yv)%rI;6#?o?wktB99`dZj4G9hXB1P=kSf`XOvXv%A?;_+7;+Rinl z)}EXs^KjQ4+V4T{!!( zl9DRV`0T_3x;gFgpv(E})L+Yv>Ws`pf(-p_WT0JSdEYu*@zBxwYftTD8CI@UdiXBj z7Pks;N|i1?r(oXZ)Yo3KI8otoAYKsN3{DF2+j(CQH5Na>3vRj|;w6e7a`*#^?E%e> zLv|+e66{r~((yN{{{A{K$gpU411^y~cvptnAZ19cMpx8z&jCkL-xF@?E5X^Yb~*0* zZ4Q80QVw>2bG=e`3XDDhFhQ9})f6lI^AAwSH)h-{{$Gnmv}V~y+RFbDwbn~9#R#H?j}pwowLS`;cfz_MgX zZJ;Lj2sw~0BuWpal65K?c253oD1Zr+R8W3o-vdM{>5nFC^?&x9Gm=BoWWP;B-BqTS zw>sDt7@zk`I_+7wp#-=(|1K)t{duAS-!O!4OjRwPz>2vH3i8t83uvhh+i&f<-_+Nd zj3TOD-l`j@t3pGr6P|54G`O6Q1TPene9fEii?6SJVhrr+0_v>OqzO0kXZofg#j?at zY)2(cOU7GW|19fyoBxG=w%WXSuCAl=-r!GMYf%@kSRjy_u+d@)1l|Ll)IA&%(vS0tPXU_47id>YfrHMHcHW*%V@IT6@3^#?yE9gLNkVy*ZCL}5febvIVfc{zTOkPA zd0}1ht%((4B}=fOrKE}I?lO)xLY-wQhe49wD+e~rn7@rDR=DqXlaz(_52&$^kllB281c=20m2>J?yS2@g!l~0%nX|e zWl$4``))?MpjJ4wd^5K`lBc9Y6vp=x+{kJ(`1@y>EM2&}gZ&1>fyiXQHZB_c^9Pm@ zsm|(lUM+d(a7rTZ{vQWJ`wGLx$lb3}-5-I*?Un3i>cvZwvR5iVsXc)G&N6mVQFwCd zjsVx$xoUu@>^zrr2N)*Y&hs7LxbM{bT-Nl+R#RwYs2r}+iRH%BtCe;1;gh7duPyoj z$|iO9YTXeS8~#9z?rK8=KftlN$N)N<_Yg6_qfOBvB{i-aH&88&*?uQAwb)i?kj3O= zKr6^N_|fz&t;WqC$R*%dL2TJqfcFH*Jvnky%ADNrz#Lv}zl68#sM;;mjy*D*{l!oH zwZ+-k^5|o>ZM;O2M@L|VOAnFdUMVY9HfSh!@NU@iM2XN$;#N?{oGF{VDy6S85H=nN zoBb^cMNP)J)z-D8Evr&@^wfPZZz$9kI}ax07;D4VT{ghC4w3C?@HglG3(%m{%$3>! zM~X?ao$cUcAu=K(eqhO`+YrZ=iF06&d;R34e4WRim+OL{jrruUb)MBMgmFz3` zOQwU(j+@Wp?l_Tm+suRj@7Azuvs`A;ZcK)ix2EvsCtE;r7;Nf`@)D(7MQ z*Ub6E%NqBYAax2-y`QY~h8n>BEpFsO>rU1B#-Ok};u#x!b^ce|+w8NWux2LBC2@c{ z=MyV>VH5iLF$WmMM?va)1tcxZ7bJ;>eKg&D+Ycr_r}na%!Bs-cV0MC^l(+@R`sxkW zgSvE}ub=xL7UFPJU77*hU2fOcax;oEQs=FOdAo!oXoed1=Sfc|01SKyW?T%!WoX(1 zuGc`A$o{Z>%ih%FNrk&=I9?-J;3M*(u}vA|l}Wegz+VN<J!f zP_s$^!xChgCWYa}Qr&lbvDff!Ae-0o7m4!#X137cvg)KTyZbo2+7D7um$96ER};2; zvFHU!c9g5cDk!s4qSH(ChsN57>BtkGrg**Kz~aWNgaJ>$9heh2b83eu!)S-$Df1Yt zWSrfFD1X9BD-#@^Vf2BDV$`C*f8nAf#msN1KxeK(nfI_*=aVL4QY@+|7#(Oc#<0$Hg%uE=E^gYFWK}-f(?AVoU3NheBsTQphTg3KBmY&X z?6`Pqpp2eK|F?b@R8ibBp9Fw4WK%Bg-E`eIQxuoclhD}iPV`HAb=E*s|-r7@3 z0KS^40VAGLIL4kyqi1QaBBsiM?$AjIK z^{AhUe4IlW`ClAI<+qQN!8WB-YXo0xD)&qk15>er{Hp1#)(yMw?p=UQ&n+!JO)pNjhH-WlF7!txzIT>Q1VXtdc$`l}vJ)=LCGLS+)R z%hbO8hN@Z~kj+CN9DG(N9@$8xE>8K)n9x=(n74MFU}Ty-($K`x21$|&?bIvS`JHrE zz--Y1S_bWR_%Iu-S=d;_00;@hL4XdPFQ9Aj)DHqgyqx*<{UplA4NiPvFLMALv!yjjMc=6fcwX5r|BuI3E_~aK z=l0CipXZk^;lrGPqy28ARcLMe8rQmU{=u7bW)LH8{O1ZIm)NW29J53UOn-V;zt4fk zSTev?oJ9tBxi0-+XTf2HT_trA8@AhL4LCkZ{_7RZp9U7BdXXUd{T9i{IAOIg~gZ~ z;GOX)v{Obkq2BBBK>Y(Nde3}9IT|VO>-D!wV|rd?pX9zOZPb-oJPdH30+xV%b?>75 zaPJQ1&%+}d?^wF>uJ_LNx=+Q?D#Jm>=Xs}|ny$wO>{Oalhl4hS?2g7XzmRdykAEu>xsvJc@OyFhq zfCQ{mNFSlEbFD|^;yM%ZG4lYajX&mD!#u>Fdc<+k0$l2c>$9>TxLakfM4Fv!0{w|4nT*Q zzDXvw9N_BzpaX3(!i63AfOdNT83%pVu*y6ppUH~(rhIuc@WC$i4ItipviMHDSNA*` zbpFk_R@^H2GCZpPR-dlbfPC)QTJ5@|JR#v^A%8NmPh1XLWTgb~-tWCiM=^eNy-+Y? zCpgcwH1nib%X&(+F7b0knysM^vNuoW9dExK0<*y{`QA%d}qZ32gv z?{+wmUAq$bFI$4%^N`K1=N8F9MRhjo1z0E+n&$V0`m?m`E}z0Tq2fAX=}=htc#!7b zdXLd*PD__H{$g#njQE}>2?C7uV_LG6)tqvC*nB=R&aZq=#G(l?vs3@G-t4zJrO!Cc zk7_e3?a;U~J8I@Bv~)5mhgFc21#j1uf7~y!qHJ!U)ZZ2OK8cyIr1~}&%GO9n$Yf7r zV*xd%DQ)inrh&S<1IPLR2N`I@6s!w{5vksh;`?`3I=|`Q-1PqMGeH06tH^!5 zcdLq&k_8GK;5Xoj8Y_-m>0KG!JfUnw2S4I$+WlR9%?0#%b7ip~xSjBbmk093uTg^p z*sw&9`dGl=kg4xR=LmtW_$8SGA&s@Mnz+=PvG?c`h!;LR*oo*fJH8vF&?&9e3nuK4Fn2a{CMfpeo71F7;Zq;LIWBJ9)s4HMXvL!4Hy`K4h^}+cAPBCpD zaktpZ%?k}pl8W%en0ZXa%QUY07u`ZoXjNOXgIZd2g{7QEuUZ5w5W0-mSS(qq0W z|2Jj~7&tpumH9W6cc4OdmGV}5`Ij~;BB-Xd#O=YeMrJQxi)49i&ULvH%;eHKpIq7V zoLVbFI%IU*^`!o6aSM0FOhmC8G&#u$rd=+sd~_`; z2gBGqQNc@hOplt0j(W4*d+T`RuY%Pj2}vL4gd=0`k(pnCHlr9SfvaMJFMCFxkQ96e zz?1(Cm8tkbDU!Vti4h5y;t9F;s>b)7EaKwcplG8{l-2ePSJ~86GIJp66lNq1uvkhJ zkc$-idutqQa&7;qIhIyYb>4JJwJ7dnfR9o*U9&PZj4g94unL)`yn6{$&%k-2V|vzS zNuM>@%!QVYV;ddtef4wcdjnl-K#|} z5ILO<7VL}ozZiQO9M!V8b2`Q+BX{Eb)Wb)7sQ0f%F8QRgB^6Q)ED=JDFZifZ)wOIq z9S#f_SBPV?B4hJTu!R7?>xqKT6iBz4`>%CWx&k5d4U(MCo&R9?r@Cz{Q1+y*p={4xTpPCq(u&=em8q(B`O zBac_!GT*i=_`!4N+k8o%*VnssTxD$z#Q}EYrhYV-h|>Ij@8rndw?J2%#GzU#n8|#S zYR$f}GM_+=-A;IWBQ*oEGT$_m83H7r-~kE%ptv7wo~~(b?9^oW2Y}%^$ApVKuV2~_nPl0tdulQY0ya~(AWYm@5Jz>0dl4B;R8JFQ{ALBDnZo!-+*_}N8yJI$x@l%a0k9jg6 zo%u)eDJ!8pF38)bvwQ-+QqR9$jOt=}GzCwvGJM5w3C0UbC^6-U*(uX@)r0-)?^LZj zm27e|l82czM|@B3tez@Jikz?0c~zwFEk}zlRDgcd0E*LJG<`5ZYIv4R4~Z2zF0*?1 zT6q)!NfA-|Q7OgO%;QBhrw%KRLd`$)y6?5-{cU)Wg_bT@3E;ZlKD#jY)2uw|(tr0J z`9B|Y6;AUCM7~!u;}6Kv!d-1HRhUtHt6e_SXHTg)wK3k;)>_DPn{SnT;qM+pJ)F=e zHV9?w%5D9>5@GGTZy{esm6^EPRNmFrRt{w-iOb8V+GX{X%F9Gc-yI<7Gvc@eej>N@ z0eX=LuwH?F==yn%U*V@@=esS1zblhgtm``kV00M2I+GEno5HLb9p>ER+srlHD7A03 zw2G+hQmhZ0LxsWyp|?U>JRiH5dWH}t0@X^r$)_-hb8yK*G%gfh@}h}<3?00g zHx`BenpDzQ@~%$lFSW!5OF5gRyFLZ|4@Qq(&vY8cxKsXGCx^r&eWW1hW^2F^rOB3B z;{!3j{#TP|XHNA{UF|NBz^Sk%AiH%15ENHI+??F`D2nWD(|@l1!8u@+sQ=-&l1EMl z{5Ke?`JFJZ5%TC!^O4T9MdLOXlZj_I((OD?+qpz#x1Ix*Vmm63T&Mn|L|K{Xu*6)U z-CftRsgQn2w)|OnU`ek|h;wyE9Ys5jG?`szk{-Sd7%zNf)}#H#i8dkmIAAhj>d7=$ zA3SoVGyZO#RDRM8xdg8gixsDz#kZ8d%Tjg}RN530oJ*9X+OB0M=b-+aT=u^XHHmH` z-e#(2LGTXt0PI}u%M8fnhM3lQGB7ml1)|wu0S&Y-k8wSAzsNs+9h_$jE_^5;8QYMHvx>8(pGWB~KsGgKrA=yRO2y)*vEeg(2KF50qMq(5VxcMC!|B zuivxS&1*u{tyFnbmGTU32wZy-0Y)#Q@sk-IU*IKd+SLcH#YD&1P{e&~SRZeR|1XDkn1FPp^ga=HI$Bc~lkoS!=Mj)v?~AB_-;v8`F%NfzI)t^b?7Ey=p~fh-r4R zfH>b>eQ7v9n#*>Ff86WcqKip|$rXwg*hjT+U3w1EPL3ZCH!#A}{5%#e0Z!338hXF1 zVbcNC7olZy!D0E*SKZx42J5>AmCF=siNdKZ@Q>?4-`xn!F}s=8%+2TOohs%XK+A@u@69Mr*Dd9s4Mf;xW7zxbES3Dnadu@pj`@>QUzHgJR zI#TCa!7L8gc$nGIBL>NPt$p!IFhtSlG)LNJn}Nd3I@*(1S9;t;Hkf}b^WRUWvM&%W zS*sWePiz@Y1S*eB3)Mqfj==~(X=r)V(3I|uJ7H7lM>uA4_vVO4=*{bJ#>o+P4*3B{ z36!#|jrbUKhmSu&!0BWeF3i=(3*KagU?FluVW#scX}TvDH0b&S^u&9$+FfoKliH~C z8#J+IAn?0st^<$Tj#Wvmrc6>95|xoO4_eR6s#6Y))c6!cKALwnh%+B+5-!cG@l?)M z8z<&}d0X#@NzIq1!_zeyC2L;4YdO8{ZpSfE`E1sBu39gn`Rf9?;-It*7UsJ;+q2a> zXXSKjRm~B9UT*IkKSOcirHy4s%{!&TxrYviQB(<;Xd~XT87`Rcy0q?V!pXhZypu{v zp4iibJZ!3sO0Q^pSu|^1$PNfrtlm{6G>13+He$fypW|=<4VJcqUfm@WR%Wi8p#WKP zBs{LMrn9VP{*rixGG7r!@k3(KArQrJ!QNM~3K)zzF+EV_++6?azG$_z)q-Qs)^QI@ zcupBnb2LA;+LN_;hLPB4Dv*XLoz?@y;y!Y9h-UiqcL~0J2*8s0ojQW(MP-@O zc2Y9g_Y^mcx#4q9n>sK0t~x^t=ua8$IGE>xGR|E>vv zXQSzOfoZ{$0#x=k-2V!fezzU6l4;!oe_^~O(=c$tSpWj4+#7O`A_j+uWZmv@dj+)z zXgQzG`*O~dd>R%|?xD?&I98P103pw~$-W5L{vv^uhVh0gWuAnY+~}#XL-hHO%S0-H zgR@&%WzGTt_ZSl3V!8o7MJdGM#ZqfJ0t8Vi(K1t02(`PDX3Xy#!@AN4Q@C zWRTE=oTWEwZR=c^&z;8rKC`W(c`2b94H?9SRD))Z;J`I%1Ooq+6~RZF>E%972d^)bDC>UT1{^+anEmwN^ z1tg@R-pWR?RjU5Gw#aIoDgb0j6R!A#pKLbvLk4_0L&dTO5qVycCQX{OId zhWaV$LR{ou`_Ebt%TFSvq^$AuH1@*auw*?a)aM|dDwZrp!a^*?F{kxCV>#4bo zdDi|WYeBxc8_I^u1=P(0unn$J@bc9ndBd)o6#pxDoE@|W_b}=5u~@!G=MZNn+`8J6 zD=;)vJG+|Tep}zR`?)b2q7&8Rf`_DP+&RIYaL8YoBv(7%5KJiIvrUS@o)XdcnbIREC^ z`2{MVLQD^ZD01lg$?}1To{- zP3a?5R~U4mLLM`v68!*n#qzSvwp|X~oE@c42?$W0h(@k6;JH-fg`*OcQBLeyLAaN#n->s3vt(yc&OK|MLrSvhm z1?1@37?sw7FV)D+X=LY!uVbLC)A+xXPtj(YjIkz<;`??KHG!7nSZU#qH*@s3icX?_ zO_$!hn>bGH^<~@1B#)?xw)}-B!)k zEgvj0e)5!XV25;0er=Si^LO()(E?ww)_06sYs5E z+o%DWeS6@Dd*9%Gr%6>p@bI1|<^><=UWA^#k@>)*AOzP?Fv(Br`_+KX2T``XhHLj5 zqo?U(G0hFRCA*JJ>*N<-TwVfO8|qOPmy_#1rIMic&2w_!ys+?j4f9mEA^v}e6ou+Y z4rPoow30GIqF&I;yCJsLx5 z@P=+qHhvkjN4({##e!%gC1#^KGjti~!cS3)U}Df`%d`PFBPH=Xicjoc<@ki*4T2g7 z`m_E9!UHAWH%c`psHAkYCKo#j89(A)GVDzrEtX+i05@Ms9rt*V4Uh@W1f20!s(BWk zmB8S4gXnRPTwT7rn_XM^+=z*^V`h`9^%^^5=@kw)XgEu932eT)T{rgdF7^41w;>H3 zWi$LXS$<^wso@{s3~VZ}`P0{Tb8yz(ZYYV`UhPZKJH&cwXD0xo$H^g{nfPWrF#y8w z;}*)AknqdFYJOz$YLqpxC8Qv;t-ZfhqKAbVHMh;~K)uB>6J!_VsGmhA#NIc&+0E0V zfFQQ@=^KR};eQm4m`D_T5;Z=G5s(y#_bD2yG9MoCwgksp}G{oO(EhL(*|E?n zqd+2cHN-bYCmOnTAvPD0#@$q&*sl4iiDyrQT4DZ~zF4N^lF>KbDY)sbzEpcR zt%YSEpCl&EelhV_A(BAjD_-%J8X3@xF6Cv2GL{~&i%0O?{X2-U`Gavy^{CCIT9Ga# z_g;PK}zi6SnSr(=BXQzdqv zp}h+QhPHh34kPPNQNQKQ?UKIb#Ziv06~eX!N_>zK&`_TkO11d%rJ}U{;q2!(+P(WX zWsCD0wAv?kvPKGDa_EKX-2b~mzu%@z7R7Fv_hvGJ^qhWGcRQzW7CW2``DeE7E_z4I zk*pi-st~hT^O>u%-~W;2cQi)7p!w33|7th8_)Pys=!5&A^1sU7m!<$+{*nGW+FmeZ mpM3WJ;q{--^DfUeGgxl0o&ddM`Hnxqd<^cH=%Ma9g#8bdarh+w literal 0 HcmV?d00001 From 5f77848304a9602f5ccec0f96534f8d40c5619c4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 17:58:21 -0400 Subject: [PATCH 0240/1042] wav inference works e2e! It's got staticy output, works! --- models/tacotron2/inference.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/models/tacotron2/inference.py b/models/tacotron2/inference.py index abe6f3d5b8..b42fc246ca 100755 --- a/models/tacotron2/inference.py +++ b/models/tacotron2/inference.py @@ -22,7 +22,9 @@ from text import text_to_sequence # NB(bt, 2021-05-31): Trying to get everything on the same device torch.device("cuda" if torch.cuda.is_available() else "cpu") -from vocodes_wav_images import render_histogram +# NB(bt, 2021-05-31): This is just a utility for testing mel output +# The outputs look good so far... +#from vocodes_wav_images import render_histogram # 2 def plot_data(data, figsize=(16, 4)): @@ -37,7 +39,9 @@ hparams.sampling_rate = 22050 # Fix bug running on CPU? #torch.set_default_dtype(torch.float16) -torch.set_default_tensor_type('torch.DoubleTensor') +# NB(bt, 2021-05-31): Perhaps this is blocking waveglow? It complains of the wrong tensor type. +#torch.set_default_tensor_type('torch.DoubleTensor') # 2021-05-31 commented out +torch.set_default_tensor_type('torch.HalfTensor') # 2021-05-31 added # 4 checkpoint_path = "/home/bt/models/tacotron2-nvidia/tacotron2_statedict.pt" @@ -60,9 +64,11 @@ model = model.to(device) _ = model.cuda().eval() # 5 -#waveglow_path = 'waveglow_256channels.pt' -#waveglow = torch.load(waveglow_path)['model'] -#waveglow.cuda().eval().half() +#waveglow_path = '/home/bt/models/waveglow_256channels.pt' +waveglow_path = '/home/bt/models/waveglow/waveglow_256channels_v4_uberduck.pt' +waveglow_path = '/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt' +waveglow = torch.load(waveglow_path)['model'] +waveglow.cuda().eval().half() #for k in waveglow.convinv: # k.float() #denoiser = Denoiser(waveglow) @@ -118,12 +124,20 @@ torch.save(mel_outputs, 'mel_outputs.mel') torch.save(mel_outputs_postnet, 'mel_outputs_postnet.mel') #print('Rendering histograms') -render_histogram(mel_outputs, 'mel_outputs.png') -render_histogram(mel_outputs_postnet, 'mel_outputs_postnet.png') +#render_histogram(mel_outputs, 'mel_outputs.png') +#render_histogram(mel_outputs_postnet, 'mel_outputs_postnet.png') # 8 with torch.no_grad(): - #audio = waveglow.infer(mel_outputs_postnet, sigma=0.666) + audio = waveglow.infer(mel_outputs_postnet, sigma=0.666) pass + + #ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate) +# https://pytorch.org/hub/nvidia_deeplearningexamples_waveglow/ +audio_numpy = audio[0].data.cpu().numpy() +rate = 22050 +from scipy.io.wavfile import write +write("audio.wav", rate, audio_numpy) + From e31a5557b6ca2aa7c3ae9b56e35c79667cd2e7ce Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 18:29:03 -0400 Subject: [PATCH 0241/1042] add working requirements --- models/tacotron2/inference.py | 3 +- .../requirements-working-2021-05-31.txt | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 models/tacotron2/requirements-working-2021-05-31.txt diff --git a/models/tacotron2/inference.py b/models/tacotron2/inference.py index b42fc246ca..f4423f31e4 100755 --- a/models/tacotron2/inference.py +++ b/models/tacotron2/inference.py @@ -48,6 +48,7 @@ checkpoint_path = "/home/bt/models/tacotron2-nvidia/tacotron2_statedict.pt" #checkpoint_path = "/home/bt/models/tigger_tt_model.pt" #checkpoint_path = "/home/bt/models/uber_test_2.pt" checkpoint_path = "/home/bt/models/tacotron2/tacotron2_uberduck_JorgenVonStrangle" +checkpoint_path = "/home/bt/models/tacotron2/tacotron2_uberduck_Noire.pt" model = load_model(hparams) #model.load_state_dict(torch.load(checkpoint_path, map_location=torch.device('cpu'))['state_dict']) model.load_state_dict(torch.load(checkpoint_path)['state_dict']) @@ -65,8 +66,8 @@ _ = model.cuda().eval() # 5 #waveglow_path = '/home/bt/models/waveglow_256channels.pt' -waveglow_path = '/home/bt/models/waveglow/waveglow_256channels_v4_uberduck.pt' waveglow_path = '/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt' +waveglow_path = '/home/bt/models/waveglow/waveglow_256channels_v4_uberduck.pt' waveglow = torch.load(waveglow_path)['model'] waveglow.cuda().eval().half() #for k in waveglow.convinv: diff --git a/models/tacotron2/requirements-working-2021-05-31.txt b/models/tacotron2/requirements-working-2021-05-31.txt new file mode 100644 index 0000000000..091352bc0e --- /dev/null +++ b/models/tacotron2/requirements-working-2021-05-31.txt @@ -0,0 +1,44 @@ +absl-py==0.12.0 +astor==0.8.1 +audioread==2.1.9 +cached-property==1.5.2 +cycler==0.10.0 +dataclasses==0.8 +decorator==5.0.9 +future==0.18.2 +gast==0.2.2 +google-pasta==0.2.0 +grpcio==1.38.0 +h5py==3.1.0 +importlib-metadata==4.4.0 +inflect==0.2.5 +joblib==1.0.1 +Keras-Applications==1.0.8 +Keras-Preprocessing==1.1.2 +librosa==0.6.0 +llvmlite==0.31.0 +Markdown==3.3.4 +matplotlib==2.1.0 +numba==0.48.0 +numpy==1.16.0 +opt-einsum==3.3.0 +Pillow==8.2.0 +protobuf==3.17.1 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +resampy==0.2.2 +scikit-learn==0.24.2 +scipy==1.0.0 +six==1.16.0 +tensorboard==1.15.0 +tensorflow==1.15.2 +tensorflow-estimator==1.15.1 +termcolor==1.1.0 +threadpoolctl==2.1.0 +torch==1.8.1 +typing-extensions==3.10.0.0 +Unidecode==1.0.22 +Werkzeug==2.0.1 +wrapt==1.12.1 +zipp==3.4.1 From 8574ed0796bb6e34c5ebd2eaa9fc913df21ce39a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 18:32:48 -0400 Subject: [PATCH 0242/1042] fix dockerfile --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b01f555508..9bba6a2ecc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -126,10 +126,13 @@ WORKDIR / COPY models/tacotron2 ./models/tacotron2 WORKDIR models/tacotron2 -RUN python3.8 -m venv python +# NB: Not sure if we need Python3.6 for Tacotron2, but Python3.8 gave me trouble. +# NB: Setuputils fix: https://github.com/tensorflow/tensorflow/issues/34302#issuecomment-554450289 +RUN python3.6 -m venv python RUN . python/bin/activate \ && pip install --upgrade pip \ - && pip install -r requirements.txt \ + && python -m pip install --upgrade pip setuptools + && pip install -r requirements-working-2021-05-31.txt \ && deactivate # ==================== Rust Build Base ==================== From 236b1210e64f65819d7ba2a8cf7941a50dc818d7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 18:34:38 -0400 Subject: [PATCH 0243/1042] simplify python install --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9bba6a2ecc..9d05deb6ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -127,11 +127,10 @@ COPY models/tacotron2 ./models/tacotron2 WORKDIR models/tacotron2 # NB: Not sure if we need Python3.6 for Tacotron2, but Python3.8 gave me trouble. -# NB: Setuputils fix: https://github.com/tensorflow/tensorflow/issues/34302#issuecomment-554450289 +# NB: Setuptools fix: https://github.com/tensorflow/tensorflow/issues/34302#issuecomment-554450289 RUN python3.6 -m venv python RUN . python/bin/activate \ - && pip install --upgrade pip \ - && python -m pip install --upgrade pip setuptools + && pip install --upgrade pip setuptools \ && pip install -r requirements-working-2021-05-31.txt \ && deactivate From e224eee89fc9ec20fd64bd4b1f3a14d87f2ed678 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 20:24:50 -0400 Subject: [PATCH 0244/1042] FREAKING HUGE MYSQL IN DEV YAK SHAVE --- .env | 4 ++- README.md | 86 ++++++++++++++++++++++++++++++++++++++++++++++------- src/main.rs | 2 +- 3 files changed, 79 insertions(+), 13 deletions(-) diff --git a/.env b/.env index e7f1c54d72..e0c2972a6a 100644 --- a/.env +++ b/.env @@ -1,4 +1,6 @@ -DATABASE_URL=mysql://root:root@localhost/storyteller +DATABASE_URL=mysql://storyteller:password@localhost/storyteller +#DATABASE_URL=mysql://root:root@localhost/storyteller +#DATABASE_URL=mysql://root@localhost/storyteller # SQLX normally requires a live database to reflect upon at build time. # This setting turns that on and requires a `sql-data.json` cache file instead. diff --git a/README.md b/README.md index e8ccd9d054..a60ae3382b 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,19 @@ This is our main user account monolith that we'll also bake other pieces into. Local development ----------------- +### Setup + +Install diesel, + +``` +sudo apt-get install libmysqlclient-dev +cargo install diesel_cli --no-default-features --features mysql +``` + +If MySql in local dev can't be connected to, reset the accounts: + +https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux + ### Database migrations To reset the entire database (drop, migrate), run: @@ -13,13 +26,13 @@ To reset the entire database (drop, migrate), run: diesel database reset ``` -To migrate at the current step and beyond: +To migrate at the current step and beyond: ``` diesel migration run ``` -To undo migrations at the current step: +To undo migrations at the current step: ``` diesel migration redo @@ -77,9 +90,9 @@ TODO Notes / TODOs: -* Examples for good Actix+Sqlx Tests: +* Examples for good Actix+Sqlx Tests: https://stackoverflow.com/questions/65370752/how-do-i-create-an-actix-web-server-that-accepts-both-sqlx-database-pools-and-tr - + * Actix/sqlx runtime compat: https://github.com/launchbadge/sqlx/issues/1117#issuecomment-801237734 @@ -91,7 +104,7 @@ The repository needs to be given read access to the base docker image: https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings -Local Nginx Proxy +Local Nginx Proxy ----------------- Set up a local nginx to proxy to the frontend and backend so cookie issues aren't annoying @@ -99,23 +112,74 @@ Edit, `/etc/nginx/sites-enabled/storyteller` ``` server { - listen 80; + listen 80; server_name api.jungle.horse; - location / { + location / { proxy_set_header Host $host; proxy_pass http://127.0.0.1:12345; proxy_redirect off; - } + } } server { - listen 80; + listen 80; server_name jungle.horse; - location / { + location / { proxy_set_header Host $host; proxy_pass http://127.0.0.1:7000; proxy_redirect off; - } + } } ``` +Fixing dev MySql on Ubuntu 20.04 +-------------------------------- + +For some reason, the MySql default install on 20.04 gave me a bunch of trouble. + +In retrospect, I _think_ this is because 'root@localhost' requires sudo to access, but if this +gives any trouble in the future, here's how I got around it (two full hours of distraction!) + + +``` +sudo apt-get install mysql-server + +# But for some reason the default password doesn't work and diesel/sqlx can't connect? + +# Kill everything + +sudo apt purge mysql-server mysql-client mysql-common +sudo apt autoremove +sudo mv -iv /var/lib/mysql /var/tmp/mysql-backup +sudo rm -rf /var/lib/mysql* + +sudo /usr/bin/mysql_secure_installation + +# still no work... + +sudo systemctl stop mysql.service + +# ugh, it wasn't chowned or created (both states observed in different installs)… +sudo mkdir -p /var/run/mysqld +sudo chown mysql:mysql /var/run/mysqld + +sudo mysqld_safe --skip-grant-tables --skip-networking & + +mysql -u root + +flush privileges; +USE mysql; +ALTER USER 'root'@'localhost' IDENTIFIED BY 'root'; + +sudo killall -u mysql +sudo systemctl restart mysql.service + +# now it works +sudo mysql -u root -p + +# but now I have to use "sudo" !?!? + +CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost'; +``` + diff --git a/src/main.rs b/src/main.rs index 3aa3640461..94c1947892 100644 --- a/src/main.rs +++ b/src/main.rs @@ -109,7 +109,7 @@ async fn main() -> AnyhowResult<()> { let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", - "mysql://root:root@localhost/storyteller"); + "mysql://storyteller:password@localhost/storyteller"); let pool = MySqlPoolOptions::new() .max_connections(5) From 16f6e6ebef6a38e796303cb1381622393be86395 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 20:44:59 -0400 Subject: [PATCH 0245/1042] use a shared common default log-level and db password --- README.md | 8 ++++++++ src/main.rs | 28 +++++++++++++--------------- src/shared_constants.rs | 9 +++++++++ src/tts_download_job.rs | 7 ++++--- src/w2l_download_job.rs | 11 ++++++----- src/w2l_inference_job.rs | 8 ++++---- 6 files changed, 44 insertions(+), 27 deletions(-) create mode 100644 src/shared_constants.rs diff --git a/README.md b/README.md index a60ae3382b..094eea69a5 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,14 @@ server { ``` +And in /etc/hosts, + +``` +127.0.0.1 jungle.horse +127.0.0.1 api.jungle.horse + +``` + Fixing dev MySql on Ubuntu 20.04 -------------------------------- diff --git a/src/main.rs b/src/main.rs index 94c1947892..5a3386036a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,7 @@ pub mod common_queries; pub mod database_helpers; pub mod http_server; pub mod server_state; +pub mod shared_constants; pub mod util; pub mod validations; @@ -26,35 +27,36 @@ use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; +use crate::buckets::bucket_client::BucketClient; +use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::default_route_404::default_route_404; use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::misc::enable_alpha::enable_alpha; use crate::http_server::endpoints::root_index::get_root_index; use crate::http_server::endpoints::tts::enqueue_infer_tts::infer_tts_handler; use crate::http_server::endpoints::tts::enqueue_upload_tts_model::upload_tts_model_handler; +use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; use crate::http_server::endpoints::users::create_account::create_account_handler; +use crate::http_server::endpoints::users::get_profile::get_profile_handler; +use crate::http_server::endpoints::users::list_user_w2l_inference_results::list_user_w2l_inference_results_handler; +use crate::http_server::endpoints::users::list_user_w2l_templates::list_user_w2l_templates_handler; use crate::http_server::endpoints::users::login::login_handler; use crate::http_server::endpoints::users::logout::logout_handler; use crate::http_server::endpoints::users::session_info::session_info_handler; use crate::http_server::endpoints::w2l::enqueue_infer_w2l::infer_w2l_handler; +use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; use crate::http_server::endpoints::w2l::enqueue_upload_w2l_template::upload_w2l_template_handler; +use crate::http_server::endpoints::w2l::get_w2l_result::get_w2l_inference_result_handler; +use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handler; +use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::server_state::{ServerState, EnvConfig}; +use crate::shared_constants::{DEFAULT_RUST_LOG, DEFAULT_MYSQL_PASSWORD}; use log::{info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; -use crate::http_server::endpoints::users::get_profile::get_profile_handler; -use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; -use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handler; -use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; -use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; -use crate::buckets::bucket_client::BucketClient; -use crate::http_server::endpoints::users::list_user_w2l_templates::list_user_w2l_templates_handler; -use crate::http_server::endpoints::users::list_user_w2l_inference_results::list_user_w2l_inference_results_handler; -use crate::http_server::endpoints::w2l::get_w2l_result::get_w2l_inference_result_handler; -use crate::common_queries::firehose_publisher::FirehosePublisher; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -68,10 +70,6 @@ use crate::common_queries::firehose_publisher::FirehosePublisher; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; -// NB: sqlx::query is spammy and logs all queries as "info"-level -const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; -//const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; - // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; @@ -109,7 +107,7 @@ async fn main() -> AnyhowResult<()> { let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", - "mysql://storyteller:password@localhost/storyteller"); + DEFAULT_MYSQL_PASSWORD); let pool = MySqlPoolOptions::new() .max_connections(5) diff --git a/src/shared_constants.rs b/src/shared_constants.rs new file mode 100644 index 0000000000..0e5581111c --- /dev/null +++ b/src/shared_constants.rs @@ -0,0 +1,9 @@ +//! Shared constants that all binaries use. + +/// NB: `sqlx::query` is spammy and dumps all queries as "info"-level log lines. +/// NB: `hyper::proto::h1::io` is incredibly spammy and logs every chunk of bytes in very large files being downloaded. +pub const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn,hyper::proto::h1::io=warn"; +//const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; // but sometimes we want to debug + +/// The default MySql password for use in development +pub const DEFAULT_MYSQL_PASSWORD: &'static str = "mysql://storyteller:password@localhost/storyteller"; diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 92f7256aa8..e1a41b8e74 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -9,6 +9,7 @@ pub mod buckets; pub mod job_queries; pub mod script_execution; +pub mod shared_constants; pub mod util; use anyhow::anyhow; @@ -28,6 +29,8 @@ use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; +use shared_constants::DEFAULT_MYSQL_PASSWORD; +use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::File; @@ -44,8 +47,6 @@ const ENV_REGION_NAME : &'static str = "REGION_NAME"; const ENV_BUCKET_NAME : &'static str = "TTS_DOWNLOAD_BUCKET_NAME"; const ENV_BUCKET_ROOT : &'static str = "TTS_DOWNLOAD_BUCKET_ROOT"; -// NB: sqlx::query is spammy and logs all queries as "info"-level -const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Downloader { @@ -106,7 +107,7 @@ async fn main() -> AnyhowResult<()> { let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", - "mysql://root:root@localhost/storyteller"); + DEFAULT_MYSQL_PASSWORD); info!("Connecting to database..."); diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index a66588505a..c0eef2fb27 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -8,9 +8,10 @@ #[macro_use] extern crate serde_derive; pub mod buckets; +pub mod common_queries; pub mod job_queries; pub mod script_execution; -pub mod common_queries; +pub mod shared_constants; pub mod util; use anyhow::anyhow; @@ -36,15 +37,17 @@ use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; +use shared_constants::DEFAULT_MYSQL_PASSWORD; +use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; use std::io::{BufReader, Read}; use std::path::{PathBuf, Path}; use std::process::Command; +use std::thread; use std::time::Duration; use tempdir::TempDir; -use std::thread; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -64,8 +67,6 @@ const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; const ENV_DOWNLOAD_SCRIPT_NAME : &'static str = "W2L_DOWNLOAD_SCRIPT_NAME"; -// NB: sqlx::query is spammy and logs all queries as "info"-level -const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Downloader { @@ -168,7 +169,7 @@ async fn main() -> AnyhowResult<()> { let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", - "mysql://root:root@localhost/storyteller"); + DEFAULT_MYSQL_PASSWORD); info!("Connecting to database..."); diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 3e91204e89..3fc88c61a7 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -11,6 +11,7 @@ pub mod buckets; pub mod common_queries; pub mod job_queries; pub mod script_execution; +pub mod shared_constants; pub mod util; use anyhow::anyhow; @@ -39,6 +40,8 @@ use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; +use shared_constants::DEFAULT_MYSQL_PASSWORD; +use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; @@ -67,9 +70,6 @@ const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; const ENV_INFERENCE_SCRIPT_NAME : &'static str = "W2L_INFERENCE_SCRIPT_NAME"; -/// NB: `sqlx::query` is spammy and logs all queries as "info"-level -/// NB: `hyper::proto::h1::io` is incredibly spammy and logs every chunk of bytes in very large files being downloaded -const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn,hyper::proto::h1::io=warn"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; struct Inferencer { @@ -165,7 +165,7 @@ async fn main() -> AnyhowResult<()> { let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", - "mysql://root:root@localhost/storyteller"); + DEFAULT_MYSQL_PASSWORD); info!("Connecting to database..."); From 335d355d643f52f8a9f2c53889573e772722485f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 21:45:45 -0400 Subject: [PATCH 0246/1042] add w2l deps, update readme --- README.md | 17 ++++++++++++++--- models/Wav2Lip/requirements.txt | 6 ++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 094eea69a5..b2639835b7 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,23 @@ This is our main user account monolith that we'll also bake other pieces into. Local development ----------------- -### Setup +### Setup development -Install diesel, +Install the following libraries, and see the notes further below about MySQL on Ubuntu 20.04. + +``` +mysql-server +imagemagick +``` + +To manage the database and perform migrations, install the Rust tools diesel and sqlx. + +We'll be using diesel to manage the migrations, but sqlx within the app to actually perform queries. +Diesel is an ORM, which is dumb, so we use sqlx as at-compile-time typesafe SQL. ``` sudo apt-get install libmysqlclient-dev +cargo install sqlx-cli --no-default-features --features mysql cargo install diesel_cli --no-default-features --features mysql ``` @@ -57,7 +68,7 @@ Production ### Database migrations -1. Set `DATABASE_URL` in `.env` +1. Set `DATABASE_URL` in `.env` to the production secrets (DO NOT COMMIT!) 2. Run `diesel migration run` Actix notes diff --git a/models/Wav2Lip/requirements.txt b/models/Wav2Lip/requirements.txt index de89d8f04b..e0422532b0 100644 --- a/models/Wav2Lip/requirements.txt +++ b/models/Wav2Lip/requirements.txt @@ -8,3 +8,9 @@ tensorflow==1.12.0 torch==1.1.0 torchvision==0.3.0 tqdm==4.45.0 + +# To download (w2l-download-job) +# Note: these will need frequent updates! +gdown==3.13.0 +youtube-dl==2021.5.16 + From 14a918a9c63a1424a1a663a8c3fd39911f813485 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 22:22:48 -0400 Subject: [PATCH 0247/1042] update endpoint to set token and require idempotency token --- debug/mysql/last_tts_inference_job.sh | 4 ++-- .../up.sql | 5 +++-- .../endpoints/tts/enqueue_infer_tts.rs | 19 +++++++++++++++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/debug/mysql/last_tts_inference_job.sh b/debug/mysql/last_tts_inference_job.sh index 4278bdc435..f72fc914bc 100755 --- a/debug/mysql/last_tts_inference_job.sh +++ b/debug/mysql/last_tts_inference_job.sh @@ -1,7 +1,7 @@ #!/bin/bash # ONLY FOR LOCAL DEV -mysql -u root \ - -proot \ +mysql -u storyteller \ + -ppassword \ -h localhost \ -D storyteller \ -e "select * from tts_inference_jobs\G" diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index c777210654..1889565dbc 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -1,5 +1,6 @@ -# noinspection SqlNoDataSourceInspectionForFile -# noinspection SqlResolveForFile +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile -- TTS MODELS CREATE TABLE tts_model_upload_jobs ( diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 010f1e0820..052ba872b7 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -8,7 +8,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::server_state::ServerState; -use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; @@ -24,6 +24,7 @@ const NEW_USER_ROLE: &'static str = "new-user"; #[derive(Deserialize)] pub struct InferTtsRequest { + uuid_idempotency_token: String, tts_model_token: String, inference_text: String, creator_set_visibility: Option, @@ -100,16 +101,28 @@ pub async fn infer_tts_handler( return Err(InferTtsError::BadInput(reason)); } - // TODO: CHECK DATABASE! + // TODO(bt): CHECK DATABASE! let model_token = request.tts_model_token.to_string(); let ip_address = get_request_ip(&http_request); let creator_set_visibility = "public".to_string(); + // This token is returned to the client. + let job_token = random_prefix_crockford_token("TTS_INF:", 32) + .map_err(|e| { + warn!("Error creating token"); + InferTtsError::ServerError + })?; + + info!("Creating w2l inference job record..."); + let query_result = sqlx::query!( r#" INSERT INTO tts_inference_jobs SET + token = ?, + uuid_idempotency_token = ?, + model_token = ?, inference_text = ?, maybe_creator_user_token = ?, @@ -117,6 +130,8 @@ SET creator_set_visibility = ?, status = "pending" "#, + job_token, + request.uuid_idempotency_token.clone(), model_token, inference_text, maybe_user_token, From 2e619782583cb0bd40fe341ade7d939964679f56 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 22:50:03 -0400 Subject: [PATCH 0248/1042] more firehose events and add new column! --- .../2021-05-15-212913_inference_jobs/up.sql | 6 +- migrations/2021-05-30-010200_firehose/up.sql | 33 ++++++----- src/common_queries/firehose_publisher.rs | 57 ++++++++++++++++--- .../endpoints/tts/enqueue_infer_tts.rs | 9 ++- .../endpoints/tts/enqueue_upload_tts_model.rs | 7 +++ .../w2l/enqueue_infer_w2l_with_uploads.rs | 2 +- src/job_queries/tts_download_job_queries.rs | 6 +- src/tts_download_job.rs | 19 ++++++- src/w2l_inference_job.rs | 1 + 9 files changed, 111 insertions(+), 29 deletions(-) diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index 37b9759ed7..ad9f95a2ea 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -1,5 +1,7 @@ -# noinspection SqlResolveForFile -# noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + CREATE TABLE tts_inference_jobs ( id BIGINT(20) NOT NULL AUTO_INCREMENT, diff --git a/migrations/2021-05-30-010200_firehose/up.sql b/migrations/2021-05-30-010200_firehose/up.sql index 374c8a34ee..bfc5e3c6cc 100644 --- a/migrations/2021-05-30-010200_firehose/up.sql +++ b/migrations/2021-05-30-010200_firehose/up.sql @@ -1,5 +1,6 @@ -# noinspection SqlResolveForFile -# noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile CREATE TABLE firehose_entries ( -- Not used for anything except replication. @@ -9,18 +10,6 @@ CREATE TABLE firehose_entries ( token VARCHAR(32) NOT NULL, -- The type of the event - event_type VARCHAR(32) NOT NULL, - - -- The target user - maybe_target_user_token VARCHAR(32) DEFAULT NULL, - - -- The target "entity", which varies by event type. - maybe_target_entity_token VARCHAR(32) DEFAULT NULL, - - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - - -- Settings -- DO NOT REORDER. -- event_type ENUM( -- 'not-set', @@ -40,12 +29,28 @@ CREATE TABLE firehose_entries ( -- 'twitch_subscribe', -- 'twitch_follow' -- ) NOT NULL DEFAULT 'not-set', + event_type VARCHAR(32) NOT NULL, + + -- The target user + maybe_target_user_token VARCHAR(32) DEFAULT NULL, + + -- The target "entity", which varies by event type. + -- eg. a model token or template token + maybe_target_entity_token VARCHAR(32) DEFAULT NULL, + + -- If something was created, this might be the token (if we know it) + -- eg. an inference job's token + maybe_created_entity_token VARCHAR(32) DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (token), KEY fk_maybe_target_user_token (maybe_target_user_token), KEY fk_maybe_target_entity_token (maybe_target_entity_token), + KEY fk_maybe_created_entity_token (maybe_created_entity_token), KEY index_event_type (event_type) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/src/common_queries/firehose_publisher.rs b/src/common_queries/firehose_publisher.rs index 235f2462d6..c4de611150 100644 --- a/src/common_queries/firehose_publisher.rs +++ b/src/common_queries/firehose_publisher.rs @@ -62,7 +62,43 @@ impl FirehosePublisher { let _record_id = self.insert( FirehoseEvent::UserSignUp, Some(user_token), - Some(user_token) + Some(user_token), + Some(user_token) + ).await?; + Ok(()) + } + + pub async fn enqueue_tts_model_upload(&self, user_token: &str, job_token: &str) -> AnyhowResult<()> { + let _record_id = self.insert( + FirehoseEvent::TtsModelUploadStarted, + Some(user_token), + None, + Some(job_token) + ).await?; + Ok(()) + } + + pub async fn publish_tts_model_upload_finished(&self, user_token: &str, model_token: &str) -> AnyhowResult<()> { + let _record_id = self.insert( + FirehoseEvent::TtsModelUploadCompleted, + Some(user_token), + Some(model_token), + Some(model_token) + ).await?; + Ok(()) + } + + pub async fn enqueue_tts_inference( + &self, + maybe_user_token: Option<&str>, + job_token: &str, + model_token: &str + ) -> AnyhowResult<()> { + let _record_id = self.insert( + FirehoseEvent::TtsInferenceStarted, + maybe_user_token, + Some(model_token), + Some(job_token) ).await?; Ok(()) } @@ -71,15 +107,17 @@ impl FirehosePublisher { let _record_id = self.insert( FirehoseEvent::W2lTemplateUploadStarted, Some(user_token), + None, Some(job_token) ).await?; Ok(()) } - pub async fn enqueue_w2l_inference(&self, maybe_user_token: Option<&str>, job_token: &str) -> AnyhowResult<()> { + pub async fn enqueue_w2l_inference(&self, maybe_user_token: Option<&str>, job_token: &str, template_token: &str) -> AnyhowResult<()> { let _record_id = self.insert( FirehoseEvent::W2lInferenceStarted, maybe_user_token, + Some(template_token), Some(job_token) ).await?; Ok(()) @@ -89,16 +127,18 @@ impl FirehosePublisher { let _record_id = self.insert( FirehoseEvent::W2lTemplateUploadCompleted, Some(user_token), - Some(template_token) + Some(template_token), + Some(template_token) ).await?; Ok(()) } - pub async fn w2l_inference_finished(&self, maybe_user_token: Option<&str>, job_token: &str) -> AnyhowResult<()> { + pub async fn w2l_inference_finished(&self, maybe_user_token: Option<&str>, job_token: &str, result_token: &str) -> AnyhowResult<()> { let _record_id = self.insert( FirehoseEvent::W2lInferenceCompleted, maybe_user_token, - Some(job_token) + Some(job_token), // TODO: This could be template_token + Some(result_token) ).await?; Ok(()) } @@ -109,7 +149,8 @@ impl FirehosePublisher { &self, event_type: FirehoseEvent, user_token: Option<&str>, - entity_token: Option<&str> + entity_token: Option<&str>, + created_entity_token: Option<&str> ) -> AnyhowResult { let token = random_prefix_crockford_token("EV", 32)?; @@ -120,12 +161,14 @@ SET token = ?, event_type = ?, maybe_target_user_token = ?, - maybe_target_entity_token = ? + maybe_target_entity_token = ?, + maybe_created_entity_token = ? "#, token, event_type.to_db_value(), user_token, entity_token, + created_entity_token ) .execute(&self.mysql_pool) .await; diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 052ba872b7..3af94f3831 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -130,7 +130,7 @@ SET creator_set_visibility = ?, status = "pending" "#, - job_token, + &job_token, request.uuid_idempotency_token.clone(), model_token, inference_text, @@ -172,6 +172,13 @@ SET info!("new w2l inference job id: {}", record_id); + server_state.firehose_publisher.enqueue_tts_inference(maybe_user_token.as_deref(), &job_token, &model_token) + .await + .map_err(|e| { + warn!("error publishing event: {:?}", e); + InferTtsError::ServerError + })?; + let response = InferTtsSuccessResponse { success: true, }; diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index eaa69e8ee0..c9dbe57804 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -200,6 +200,13 @@ SET info!("new model upload job id: {}", record_id); + server_state.firehose_publisher.enqueue_tts_model_upload(&user_session.user_token, &job_token) + .await + .map_err(|e| { + warn!("error publishing event: {:?}", e); + UploadTtsModelError::ServerError + })?; + let response = UploadTtsModelSuccessResponse { success: true, job_token: job_token.to_string(), diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index e196ae6035..5e66b444fb 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -341,7 +341,7 @@ SET // .map_err(|err| HttpResponse::InternalServerError() // .body(format!("Redis Err: {:?}", err)))?; - server_state.firehose_publisher.enqueue_w2l_inference(maybe_user_token.as_deref(), &job_token) + server_state.firehose_publisher.enqueue_w2l_inference(maybe_user_token.as_deref(), &job_token, &template_token) .await .map_err(|e| { warn!("error publishing event: {:?}", e); diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index 49b2a649f7..b7562d2730 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -115,7 +115,7 @@ pub async fn insert_tts_model(pool: &MySqlPool, job: &TtsUploadJobRecord, private_bucket_hash: &str, private_bucket_object_name: &str) - -> AnyhowResult + -> AnyhowResult<(u64, String)> { let model_token = random_prefix_crockford_token("TTS_MDL:", 32)?; let updatable_slug = model_token.clone(); @@ -136,7 +136,7 @@ SET private_bucket_hash = ?, private_bucket_object_name = ? "#, - model_token, + &model_token, updatable_slug, job.title.to_string(), job.creator_user_token.clone(), @@ -158,5 +158,5 @@ SET } }; - Ok(record_id) + Ok((record_id, model_token)) } \ No newline at end of file diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index e1a41b8e74..1436b02f41 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -6,7 +6,10 @@ #![warn(unused_must_use)] //#![allow(warnings)] +#[macro_use] extern crate serde_derive; + pub mod buckets; +pub mod common_queries; pub mod job_queries; pub mod script_execution; pub mod shared_constants; @@ -17,6 +20,7 @@ use chrono::Utc; use crate::buckets::bucket_client::BucketClient; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::buckets::file_hashing::get_file_hash; +use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; use crate::job_queries::tts_download_job_queries::insert_tts_model; use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_done; @@ -53,6 +57,7 @@ struct Downloader { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, pub bucket_client: BucketClient, + pub firehose_publisher: FirehosePublisher, pub google_drive_downloader: GoogleDriveDownloadCommand, // Command to run pub download_script: String, @@ -116,6 +121,10 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; + let firehose_publisher = FirehosePublisher { + mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe + }; + let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, @@ -123,6 +132,7 @@ async fn main() -> AnyhowResult<()> { download_script, google_drive_downloader, bucket_root_tts_model_uploads: bucket_root.to_string(), + firehose_publisher, }; main_loop(downloader).await; @@ -223,7 +233,7 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho downloader.bucket_client.upload_filename(&object_name, &file_path).await?; info!("Saving model record..."); - let id = insert_tts_model( + let (id, model_token) = insert_tts_model( &downloader.mysql_pool, job, &private_bucket_hash, @@ -232,6 +242,13 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("Saved model record: {}", id); + downloader.firehose_publisher.publish_tts_model_upload_finished(&job.creator_user_token, &model_token) + .await + .map_err(|e| { + warn!("error publishing event: {:?}", e); + anyhow!("error publishing event") + })?; + info!("Job done: {}", job.id); mark_tts_upload_job_done(&downloader.mysql_pool, job, true).await?; diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 3fc88c61a7..e2475a7d84 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -535,6 +535,7 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An inferencer.firehose_publisher.w2l_inference_finished( job.maybe_creator_user_token.as_deref(), + &job.inference_job_token, &inference_result_token) .await .map_err(|e| { From ff9c8e73e43ed86216cf0414edf0ee9d8c977de2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 23:20:23 -0400 Subject: [PATCH 0249/1042] seed tts models, youtube videos --- debug/curl/seed.py | 27 ++++++++++++++++++++++++++- debug/mysql/last_w2l_template.sh | 4 ++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/debug/curl/seed.py b/debug/curl/seed.py index 7cd89469ee..820f9e3181 100755 --- a/debug/curl/seed.py +++ b/debug/curl/seed.py @@ -3,6 +3,10 @@ import requests import re import uuid +import socket + +# There are some things here that will cause my laptop to freeze! +hostname = socket.gethostname() def print_response(response): print('\tStatus: {}'.format(response.status_code)) @@ -50,7 +54,7 @@ session_echelon = login('echelon', 'testing') session_nanashi = login('nanashi', 'nanashi') session_brandon = login('brandon', 'testing') -# ==================== UPLOAD DATA ==================== +# ==================== UPLOAD W2L TEMPLATES ==================== def upload_w2l_template(content_url, title, session_cookie): print('Upload W2L template') @@ -74,3 +78,24 @@ upload_w2l_template('https://i.imgur.com/uoGuTJo.jpeg', 'what do you mean vern?' upload_w2l_template('https://i.imgur.com/v8k9yau.jpeg', 'Miyamoto-san', session_brandon) +if hostname == 'halide': + upload_w2l_template('https://www.youtube.com/watch?v=a7mS9ZdU6k4', 'Steve Jobs talks', session_echelon) + +# ==================== UPLOAD TTS MODELS ==================== + +def upload_tts_model(content_url, title, session_cookie): + print('Upload TTS model') + upload_url = 'http://localhost:12345/tts/upload' + cookies = { 'session': session_cookie } + payload = { + 'idempotency_token': str(uuid.uuid4()), + 'title': title, + 'download_url': content_url, + } + r = requests.post(upload_url, cookies=cookies, json=payload) + print_response(r) + + +upload_tts_model('https://drive.google.com/file/d/1grwDGbAsPwCMVfye071TZ6PLUub1KGyR/view?usp=sharing', 'Jorgen', session_nanashi) +upload_tts_model('https://drive.google.com/file/d/1w3oBk5vzyHurqYwP95UEQv-_NeMNu6pt/view?usp=sharing', 'Noire', session_echelon) + diff --git a/debug/mysql/last_w2l_template.sh b/debug/mysql/last_w2l_template.sh index 957943c746..b71893244e 100755 --- a/debug/mysql/last_w2l_template.sh +++ b/debug/mysql/last_w2l_template.sh @@ -1,7 +1,7 @@ #!/bin/bash # ONLY FOR LOCAL DEV -mysql -u root \ - -proot \ +mysql -u storyteller \ + -ppassword \ -h localhost \ -D storyteller \ -e "select * from w2l_templates\G" From 446983a477b9f4568f0cc6d8761d6cc216607884 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 23:40:59 -0400 Subject: [PATCH 0250/1042] another video --- debug/curl/seed.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/curl/seed.py b/debug/curl/seed.py index 820f9e3181..21832d10bd 100755 --- a/debug/curl/seed.py +++ b/debug/curl/seed.py @@ -80,6 +80,7 @@ upload_w2l_template('https://i.imgur.com/v8k9yau.jpeg', 'Miyamoto-san', session_ if hostname == 'halide': upload_w2l_template('https://www.youtube.com/watch?v=a7mS9ZdU6k4', 'Steve Jobs talks', session_echelon) + upload_w2l_template('https://www.youtube.com/watch?v=6fg8SxgEUYI', 'Paul Graham lectures', session_echelon) # ==================== UPLOAD TTS MODELS ==================== From 164cb46fffe0118617deea05d8dcdb0e5e1b425d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 31 May 2021 23:42:29 -0400 Subject: [PATCH 0251/1042] add debug print to tts inference script --- models/tacotron2/inference.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/models/tacotron2/inference.py b/models/tacotron2/inference.py index f4423f31e4..4fc646da1b 100755 --- a/models/tacotron2/inference.py +++ b/models/tacotron2/inference.py @@ -19,6 +19,13 @@ from train import load_model from text import text_to_sequence #from denoiser import Denoiser +print('========================================') +print('Python interpreter', sys.executable) +print('PyTorch version', torch.__version__) +print('CUDA Available?', torch.cuda.is_available()) +print('CUDA Device count', torch.cuda.device_count()) +print('========================================', flush=True) + # NB(bt, 2021-05-31): Trying to get everything on the same device torch.device("cuda" if torch.cuda.is_available() else "cpu") From 05c9eff1c5708a7a84e572bb6b65c0612e00c9cd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Jun 2021 02:03:52 -0400 Subject: [PATCH 0252/1042] TTS inference queries+codegen, job/script WIP --- .gitignore | 2 + migrations/2021-05-15-071601_assets/up.sql | 5 +- .../up.sql | 14 +- models/tacotron2/vocodes_inference.py | 151 +++++ sqlx-data.json | 406 +++++++++++++- src/job_queries/mod.rs | 1 + src/job_queries/tts_inference_job_queries.rs | 256 +++++++++ src/job_queries/w2l_inference_job_queries.rs | 2 +- src/script_execution/mod.rs | 4 +- ...d_OLD.rs => tacotron_inference_command.rs} | 71 ++- src/tts_inference_job.rs | 526 +++++++++++++++++- 11 files changed, 1373 insertions(+), 65 deletions(-) create mode 100755 models/tacotron2/vocodes_inference.py create mode 100644 src/job_queries/tts_inference_job_queries.rs rename src/script_execution/{wav2lip_inference_command_OLD.rs => tacotron_inference_command.rs} (56%) diff --git a/.gitignore b/.gitignore index a81f2dfe7b..9f00f24867 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.swo +*.swp *secret*sh *~ .env diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 3f5a2db84d..cd33366697 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -1,5 +1,6 @@ -# noinspection SqlNoDataSourceInspectionForFile -# noinspection SqlResolveForFile +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile CREATE TABLE tts_models ( -- Not used for anything except replication. diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 3c40113267..94f93d8300 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -1,5 +1,6 @@ -# noinspection SqlResolveForFile -# noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile CREATE TABLE tts_results ( -- Not used for anything except replication. @@ -38,6 +39,15 @@ CREATE TABLE tts_results ( -- Where the wav, spectrogram, and etc. are located. public_bucket_hash CHAR(64) NOT NULL, + -- Where the wav, spectrogram, and etc. are located. + public_bucket_audio_path VARCHAR(255) NOT NULL, + + -- ========== METADATA ========== + + file_size_bytes INT(10) NOT NULL DEFAULT 0, + mime_type VARCHAR(32) NOT NULL DEFAULT '', + duration_millis INT(10) NOT NULL DEFAULT 0, + -- ========== MODERATION DETAILS ========== -- In this case, a moderator hides it from public. diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py new file mode 100755 index 0000000000..4fc646da1b --- /dev/null +++ b/models/tacotron2/vocodes_inference.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python + +# 1 +#import matplotlib +#import matplotlib.pylab as plt + +#import IPython.display as ipd + +import sys +sys.path.append('waveglow/') +import numpy as np +import torch + +from hparams import create_hparams +from model import Tacotron2 +from layers import TacotronSTFT, STFT +from audio_processing import griffin_lim +from train import load_model +from text import text_to_sequence +#from denoiser import Denoiser + +print('========================================') +print('Python interpreter', sys.executable) +print('PyTorch version', torch.__version__) +print('CUDA Available?', torch.cuda.is_available()) +print('CUDA Device count', torch.cuda.device_count()) +print('========================================', flush=True) + +# NB(bt, 2021-05-31): Trying to get everything on the same device +torch.device("cuda" if torch.cuda.is_available() else "cpu") + +# NB(bt, 2021-05-31): This is just a utility for testing mel output +# The outputs look good so far... +#from vocodes_wav_images import render_histogram + +# 2 +def plot_data(data, figsize=(16, 4)): + fig, axes = plt.subplots(1, len(data), figsize=figsize) + for i in range(len(data)): + axes[i].imshow(data[i], aspect='auto', origin='bottom', + interpolation='none') + +# 3 +hparams = create_hparams() +hparams.sampling_rate = 22050 + +# Fix bug running on CPU? +#torch.set_default_dtype(torch.float16) +# NB(bt, 2021-05-31): Perhaps this is blocking waveglow? It complains of the wrong tensor type. +#torch.set_default_tensor_type('torch.DoubleTensor') # 2021-05-31 commented out +torch.set_default_tensor_type('torch.HalfTensor') # 2021-05-31 added + +# 4 +checkpoint_path = "/home/bt/models/tacotron2-nvidia/tacotron2_statedict.pt" +#checkpoint_path = "/home/bt/models/tigger_tt_model.pt" +#checkpoint_path = "/home/bt/models/uber_test_2.pt" +checkpoint_path = "/home/bt/models/tacotron2/tacotron2_uberduck_JorgenVonStrangle" +checkpoint_path = "/home/bt/models/tacotron2/tacotron2_uberduck_Noire.pt" +model = load_model(hparams) +#model.load_state_dict(torch.load(checkpoint_path, map_location=torch.device('cpu'))['state_dict']) +model.load_state_dict(torch.load(checkpoint_path)['state_dict']) +#model.load_state_dict(torch.jit.load(checkpoint_path, map_location=torch.device('cpu'))['state_dict']) +#_ = model.cuda().eval().half() +#_ = model.eval().half() + + +# NB(bt, 2021-05-31): Trying to get everything on the same device +device = "cuda:0" +model = model.to(device) + + +_ = model.cuda().eval() + +# 5 +#waveglow_path = '/home/bt/models/waveglow_256channels.pt' +waveglow_path = '/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt' +waveglow_path = '/home/bt/models/waveglow/waveglow_256channels_v4_uberduck.pt' +waveglow = torch.load(waveglow_path)['model'] +waveglow.cuda().eval().half() +#for k in waveglow.convinv: +# k.float() +#denoiser = Denoiser(waveglow) + +# 6 +print("\n\n===TEXT===\n") +text = "This is the song that never ends. It goes on and on my friend." +#text = "hi you" +print("Text: {}".format(text)) +print("Text len: {}".format(len(text))) +seq = text_to_sequence(text, ['english_cleaners']) +print("Seq: {}".format(type(seq))) +print("Seq: {}".format(seq)) +sequence = np.array(seq)[None, :] +print("Sequence np.array: {}".format(type(sequence))) +print("Sequence np.array: {}".format(sequence)) +sequence = torch.from_numpy(sequence) +print("Sequence from_numpy: {}".format(type(sequence))) +print("Sequence from_numpy: {}".format(sequence)) +print("Sequence from_numpy dtype: {}".format(sequence.dtype)) +sequence = torch.autograd.Variable(sequence).long() + #torch.from_numpy(sequence)).cuda().long() + #torch.from_numpy(sequence)) + +print("Sequence torch autograd: {}".format(sequence)) + +# NB(bt, 2021-05-31): This works. Don't need to do it. +#print('Saving text sequence tensor') +#torch.save(sequence, 'text_sequence.pt') + +print("\n\n===END TEXT===\n") + + +# NB(bt, 2021-05-31): Trying to get everything to the same device +sequence = sequence.to(device) + +# 7 +mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence) +#plot_data((mel_outputs.float().data.cpu().numpy()[0], +# mel_outputs_postnet.float().data.cpu().numpy()[0], +# alignments.float().data.cpu().numpy()[0].T)) + +# Per melgan preprocess.py, need to output: +# mel_output: torch.FloatTensor of shape (B, n_mel_channels, T) +print("Mel outputs:") +print(mel_outputs) +print(mel_outputs.shape) +print(mel_outputs_postnet) +print(mel_outputs_postnet.shape) + +print('Saving mels') +torch.save(mel_outputs, 'mel_outputs.mel') +torch.save(mel_outputs_postnet, 'mel_outputs_postnet.mel') + +#print('Rendering histograms') +#render_histogram(mel_outputs, 'mel_outputs.png') +#render_histogram(mel_outputs_postnet, 'mel_outputs_postnet.png') + +# 8 +with torch.no_grad(): + audio = waveglow.infer(mel_outputs_postnet, sigma=0.666) + pass + + +#ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate) + +# https://pytorch.org/hub/nvidia_deeplearningexamples_waveglow/ +audio_numpy = audio[0].data.cpu().numpy() +rate = 22050 +from scipy.io.wavfile import write +write("audio.wav", rate, audio_numpy) + diff --git a/sqlx-data.json b/sqlx-data.json index d3a9e49cc4..3ba5b0b550 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1,5 +1,147 @@ { "db": "MySQL", + "12250420b0a3ff3bf2dd7be76daa2920777e349642245211706f32add3df442c": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.updatable_slug,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\nAND tts.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 6, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 7, + "name": "private_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "1eeb7a88f5f8c2961694622b739e0b0689d89dff8a5f6b79d5a57c5b807a23f8": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { @@ -20,6 +162,16 @@ "nullable": [] } }, + "289a4770fab1dd323fe18fd74efa243dec55f6df541bd193e2585971772497e5": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -471,6 +623,200 @@ ] } }, + "372e5a9d04fd56166ef38596b52b1707a5e6a78784f72395b24e91a15a523b2f": { + "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n model_token,\n inference_text,\n\n creator_ip_address,\n maybe_creator_user_token,\n creator_set_visibility,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM tts_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "inference_job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "uuid_idempotency_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 3, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "creator_ip_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 160 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_user_token", + "type_info": { + "type": "String", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 7, + "name": "creator_set_visibility", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 8, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 9, + "name": "attempt_count", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 3 + } + }, + { + "ordinal": 10, + "name": "failure_reason", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 2048 + } + }, + { + "ordinal": 11, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 12, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "retry_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + true, + false, + false, + true + ] + } + }, "3f40b37731f6049ef9167a8912e411123053ccdfebd1564e0a8f257c3a2db0b9": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND users.username = ?\n ", "describe": { @@ -1140,16 +1486,6 @@ ] } }, - "6df889b77bffbd3f0678ad7c0db88982fa79295b569072ed6bd424be9218a595": { - "query": "\nINSERT INTO firehose_entries\nSET\n token = ?,\n event_type = ?,\n maybe_target_user_token = ?,\n maybe_target_entity_token = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 4 - }, - "nullable": [] - } - }, "746526e2e150150ef804d717f743de85010972d98cbb137e44d8348cfdb4dddc": { "query": "\nINSERT INTO w2l_results\nSET\n token = ?,\n maybe_w2l_template_token = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_video_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n duration_millis = ?\n ", "describe": { @@ -1160,6 +1496,16 @@ "nullable": [] } }, + "7529fc02fff368b36f2dc630ec3c041384fb3d0ce8f87a8fbbd7ce06dc7f43bb": { + "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n model_token = ?,\n inference_text = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_hash = 'TODO',\n public_bucket_audio_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, "7ab54466ca3021d90e0feb234acba56bf53d415dde69bc1c35120ed0c8dd1858": { "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { @@ -1416,6 +1762,16 @@ "nullable": [] } }, + "865ce236444a36d597939660b14eb4b0cee05c70fdb73f31c2fb8cddddfef42c": { + "query": "\nINSERT INTO firehose_entries\nSET\n token = ?,\n event_type = ?,\n maybe_target_user_token = ?,\n maybe_target_entity_token = ?,\n maybe_created_entity_token = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + } + }, "86f9a76cc97b471a23fff5b9ea78f7fc7ecf7603215f5db5a1ca4e063568b054": { "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE token = ?\nAND deleted_at IS NULL\n ", "describe": { @@ -1426,16 +1782,6 @@ "nullable": [] } }, - "87882bf91555f47924b49656e5f4fdf8741d00c0543c56a06fc23cdd5b745303": { - "query": "\nINSERT INTO tts_inference_jobs\nSET\n model_token = ?,\n inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - } - }, "88328e04ce947d3f541951aada9cee25135aa950ab478cf0dc6de453bb079e17": { "query": "\nSELECT\n events.token as event_token,\n events.event_type,\n events.maybe_target_user_token,\n users.username as maybe_target_username,\n users.display_name as maybe_target_display_name,\n users.email_gravatar_hash as maybe_target_user_gravatar_hash,\n events.maybe_target_entity_token,\n events.created_at,\n events.updated_at\nFROM firehose_entries as events\nLEFT OUTER JOIN users\nON events.maybe_target_user_token = users.token\nORDER BY events.id DESC\nLIMIT 25\n ", "describe": { @@ -1629,6 +1975,16 @@ ] } }, + "8f19eee8d83541363f3a1797ddfa9647d2e01e8bb6c6d640e85886ae2b9567fa": { + "query": "\nINSERT INTO tts_inference_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n model_token = ?,\n inference_text = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + } + }, "94847f448294888db6c800cc9558738b0c0de023225961d070b84b3906a2e493": { "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND users.username = ?\n ", "describe": { @@ -4238,6 +4594,16 @@ ] } }, + "f19300f4e8434c2669dac062e5a79c5ca71767c7380aab0ab40b51be8ff449d8": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 1 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, "f6e625dde66da14091591f321eb65b0137fceb7168305bf17806f1876a644954": { "query": "\nSELECT\n token as template_token\nFROM w2l_templates\nWHERE token = ?\nAND deleted_at IS NULL\n ", "describe": { diff --git a/src/job_queries/mod.rs b/src/job_queries/mod.rs index e2b567b9df..fe1fc8930b 100644 --- a/src/job_queries/mod.rs +++ b/src/job_queries/mod.rs @@ -1,3 +1,4 @@ pub mod tts_download_job_queries; +pub mod tts_inference_job_queries; pub mod w2l_download_job_queries; pub mod w2l_inference_job_queries; \ No newline at end of file diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs new file mode 100644 index 0000000000..225fb4178f --- /dev/null +++ b/src/job_queries/tts_inference_job_queries.rs @@ -0,0 +1,256 @@ +//! NB: This seems required for sqlx to generate the cached queries. +//! Sqlx's prepare needs a *single* binary to work against, so we need to +//! include these in the main binary to generate all the queries. + +use anyhow::anyhow; +use chrono::{Utc, DateTime}; +use crate::util::anyhow_result::AnyhowResult; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use log::{warn, info}; +use sqlx::MySqlPool; +use std::path::Path; + +/// table: tts_inference_jobs +#[derive(Debug)] +pub struct TtsInferenceJobRecord { + pub id: i64, + pub inference_job_token: String, + pub uuid_idempotency_token: String, + + pub model_token: String, + pub inference_text: String, + + pub creator_ip_address: String, + pub maybe_creator_user_token: Option, + pub creator_set_visibility: String, // TODO + + pub status: String, // TODO + pub attempt_count: i32, + pub failure_reason: Option, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub retry_at: Option>, +} + +pub async fn query_tts_inference_job_records(pool: &MySqlPool, num_records: u32) + -> AnyhowResult> +{ + let job_records = sqlx::query_as!( + TtsInferenceJobRecord, + r#" +SELECT + id, + token AS inference_job_token, + uuid_idempotency_token, + + model_token, + inference_text, + + creator_ip_address, + maybe_creator_user_token, + creator_set_visibility, + + status, + attempt_count, + failure_reason, + created_at, + updated_at, + retry_at +FROM tts_inference_jobs +WHERE + ( + status IN ("pending", "attempt_failed") + ) + AND + ( + retry_at IS NULL + OR + retry_at < CURRENT_TIMESTAMP + ) + LIMIT ? + "#, + num_records, + ) + .fetch_all(pool) + .await?; + + Ok(job_records) +} + +pub async fn mark_tts_inference_job_failure(pool: &MySqlPool, + job: &TtsInferenceJobRecord, + failure_reason: &str) + -> AnyhowResult<()> +{ + // statuses: "attempt_failed", "complete_failure", "dead" + let status = "attempt_failed"; + + let query_result = sqlx::query!( + r#" +UPDATE tts_inference_jobs +SET + status = ?, + failure_reason = ?, + retry_at = NOW() + interval 1 minute +WHERE id = ? + "#, + status, + failure_reason.to_string(), + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} +pub async fn mark_tts_inference_job_done(pool: &MySqlPool, + job: &TtsInferenceJobRecord, + success: bool) + -> AnyhowResult<()> +{ + let status = if success { "complete_success" } else { "complete_failure" }; + + let query_result = sqlx::query!( + r#" +UPDATE tts_inference_jobs +SET + status = ?, + failure_reason = NULL, + retry_at = NULL +WHERE id = ? + "#, + status, + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} + +pub async fn insert_tts_result>( + pool: &MySqlPool, + job: &TtsInferenceJobRecord, + bucket_audio_results_path: P, + file_size_bytes: u64, + maybe_mime_type: Option<&str>, + duration_millis: u64 +) -> AnyhowResult<(u64, String)> +{ + let inference_result_token = random_prefix_crockford_token("TTS_RES:", 32)?; + + let bucket_audio_result_path = &bucket_audio_results_path + .as_ref() + .display() + .to_string(); + + let query_result = sqlx::query!( + r#" +INSERT INTO tts_results +SET + token = ?, + model_token = ?, + inference_text = ?, + + maybe_creator_user_token = ?, + creator_ip_address = ?, + creator_set_visibility = 'public', + + public_bucket_hash = 'TODO', + public_bucket_audio_path = ?, + + file_size_bytes = ?, + mime_type = ?, + duration_millis = ? + "#, + inference_result_token, + job.model_token.clone(), + job.inference_text.clone(), + + job.maybe_creator_user_token.clone(), + job.creator_ip_address.clone(), + + bucket_audio_result_path, + + file_size_bytes, + maybe_mime_type.unwrap_or(""), + duration_millis + ) + .execute(pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + Ok((record_id, inference_result_token.clone())) +} + +pub struct TtsModelRecord2 { + pub model_token: String, + pub tts_model_type: String, + + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + + pub title: String, + pub updatable_slug: String, + pub private_bucket_hash: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +pub async fn get_tts_model_by_token(pool: &MySqlPool, model_token: &str) + -> AnyhowResult> +{ + + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_model = sqlx::query_as!( + TtsModelRecord2, + r#" +SELECT + tts.token as model_token, + tts.tts_model_type, + tts.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + tts.title, + tts.updatable_slug, + tts.private_bucket_hash, + tts.created_at, + tts.updated_at +FROM tts_models as tts +JOIN users +ON users.token = tts.creator_user_token +WHERE tts.token = ? +AND tts.deleted_at IS NULL + "#, + &model_token + ) + .fetch_one(pool) + .await; // TODO: This will return error if it doesn't exist + + match maybe_model { + Ok(model) => Ok(Some(model)), + Err(err) => { + match err { + RowNotFound => { + Ok(None) + }, + _ => { + warn!("tts model query error: {:?}", err); + Err(anyhow!("Mysql error: {:?}", err)) + } + } + } + } +} diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index e10a573686..21b488765d 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -11,7 +11,7 @@ use log::{warn, info}; use sqlx::MySqlPool; use std::path::Path; -/// table: w2l_template_upload_jobs +/// table: w2l_inference_jobs #[derive(Debug)] pub struct W2lInferenceJobRecord { pub id: i64, diff --git a/src/script_execution/mod.rs b/src/script_execution/mod.rs index 36879f0f3b..24de37d06f 100644 --- a/src/script_execution/mod.rs +++ b/src/script_execution/mod.rs @@ -2,6 +2,6 @@ pub mod ffmpeg_generate_preview_image_command; pub mod ffmpeg_generate_preview_video_command; pub mod google_drive_download_command; pub mod imagemagick_generate_preview_image_command; +pub mod tacotron_inference_command; pub mod wav2lip_inference_command; -pub mod wav2lip_inference_command_OLD; -pub mod wav2lip_process_upload_command; +pub mod wav2lip_process_upload_command; \ No newline at end of file diff --git a/src/script_execution/wav2lip_inference_command_OLD.rs b/src/script_execution/tacotron_inference_command.rs similarity index 56% rename from src/script_execution/wav2lip_inference_command_OLD.rs rename to src/script_execution/tacotron_inference_command.rs index 5e91b2300b..fc22274dc1 100644 --- a/src/script_execution/wav2lip_inference_command_OLD.rs +++ b/src/script_execution/tacotron_inference_command.rs @@ -3,59 +3,56 @@ use log::{info,warn}; use std::process::{Command, Stdio}; use subprocess::{Popen, PopenConfig, Redirection}; use std::fs::OpenOptions; +use std::path::Path; -/// This command is used to preprocess the face detection frames from user-submitted video. -/// This should only ever need to run once. The frames can then be uploaded to Buckets and saved. +/// This command is used to run inference. +/// It uses preprocessed face files so that it's much faster. #[derive(Clone)] -pub struct Wav2LipInferenceCommandOLD { - w2l_directory: String, - script_name: String, - checkpoint_path: String, +pub struct TacotronInferenceCommand { + tacotron_directory: String, + inference_script_name: String, } -impl Wav2LipInferenceCommandOLD { +impl TacotronInferenceCommand { pub fn new( - w2l_directory: &str, - script_name: &str, - checkpoint_path: &str, + tacotron_directory: &str, + inference_script_name: &str, ) -> Self { Self { - w2l_directory: w2l_directory.to_string(), - script_name: script_name.to_string(), - checkpoint_path: checkpoint_path.to_string(), + tacotron_directory: tacotron_directory.to_string(), + inference_script_name: inference_script_name.to_string(), } } - pub fn execute(&self, - audio_filename: &str, - image_or_video_filename: &str, - output_video_filename: &str, - is_image: bool, - disable_end_bump: bool, - spawn_process: bool) -> AnyhowResult<()> - { + pub fn execute>( + &self, + synthesizer_checkpoint_path: P, + vocoder_checkpoint_path: P, + input_text_filename: P, + output_metadata_filename: P, + output_audio_filename: P, + spawn_process: bool + ) -> AnyhowResult<()> { let mut command = String::new(); command.push_str("echo 'test'"); command.push_str(" && "); - command.push_str(&format!("cd {}", self.w2l_directory)); + command.push_str(&format!("cd {}", self.tacotron_directory)); command.push_str(" && "); command.push_str("source python/bin/activate"); command.push_str(" && "); command.push_str("python "); - command.push_str(&self.script_name); - command.push_str(" --checkpoint_path "); - command.push_str(&self.checkpoint_path); - command.push_str(" --face "); - command.push_str(image_or_video_filename); - command.push_str(" --audio "); - command.push_str(audio_filename); - command.push_str(" --outfile "); - command.push_str(output_video_filename); - - if is_image { - command.push_str(" --is_image "); - } + command.push_str(&self.inference_script_name); + command.push_str(" --synthesizer_checkpoint_path "); + command.push_str(&synthesizer_checkpoint_path.as_ref().display().to_string()); + command.push_str(" --vocoder_checkpoint_path "); + command.push_str(&vocoder_checkpoint_path.as_ref().display().to_string()); + command.push_str(" --input_text_filename "); + command.push_str(&input_text_filename.as_ref().display().to_string()); + command.push_str(" --output_audio_filename "); + command.push_str(&output_audio_filename.as_ref().display().to_string()); + command.push_str(" --output_metadata_filename "); + command.push_str(&output_metadata_filename.as_ref().display().to_string()); info!("Command: {:?}", command); @@ -74,14 +71,14 @@ impl Wav2LipInferenceCommandOLD { .write(true) .create(true) .truncate(true) - .open("/tmp/stdout.txt")?; + .open("/tmp/tacotron_upload_stdout.txt")?; let stderr_file = OpenOptions::new() .read(true) .write(true) .create(true) .truncate(true) - .open("/tmp/stderr.txt")?; + .open("/tmp/tacotron_upload_stderr.txt")?; let mut p = Popen::create(&command_parts, PopenConfig { //stdout: Redirection::Pipe, diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 2cec8a7b50..fd5b9f160c 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -5,10 +5,534 @@ #![allow(unused_variables)] //#![allow(warnings)] +#[macro_use] extern crate serde_derive; + +pub mod buckets; +pub mod common_queries; +pub mod job_queries; +pub mod script_execution; +pub mod shared_constants; pub mod util; +use anyhow::anyhow; +use chrono::Utc; +use crate::buckets::bucket_client::BucketClient; +use crate::buckets::bucket_path_unifier::BucketPathUnifier; +use crate::buckets::bucket_paths::hash_to_bucket_path; +use crate::buckets::file_hashing::get_file_hash; +use crate::common_queries::firehose_publisher::FirehosePublisher; +use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; +use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; +use crate::job_queries::w2l_inference_job_queries::insert_w2l_result; +use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_done; +use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failure; +use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records; +use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; +use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; +use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; +use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; use crate::util::anyhow_result::AnyhowResult; +use crate::util::filesystem::check_directory_exists; +use crate::util::filesystem::check_file_exists; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; +use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; +use log::{warn, info}; +use ring::digest::{Context, Digest, SHA256}; +use shared_constants::DEFAULT_MYSQL_PASSWORD; +use shared_constants::DEFAULT_RUST_LOG; +use sqlx::MySqlPool; +use sqlx::mysql::MySqlPoolOptions; +use std::fs::{File, metadata}; +use std::io::{BufReader, Read}; +use std::path::{PathBuf, Path}; +use std::process::Command; +use std::thread; +use std::time::Duration; +use tempdir::TempDir; + +// Buckets (shared config) +const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; +const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; +const ENV_REGION_NAME : &'static str = "REGION_NAME"; + +// Buckets (private data) +const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; +// Buckets (public data) +const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; + +// Where models and other assets get downloaded to. +const ENV_SEMIPERSISTENT_CACHE_DIR : &'static str = "SEMIPERSISTENT_CACHE_DIR"; + +// Python code +const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; +const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; +const ENV_INFERENCE_SCRIPT_NAME : &'static str = "W2L_INFERENCE_SCRIPT_NAME"; + +const DEFAULT_TEMP_DIR: &'static str = "/tmp"; + +struct Inferencer { + pub download_temp_directory: PathBuf, + pub mysql_pool: MySqlPool, + + pub private_bucket_client: BucketClient, + pub public_bucket_client: BucketClient, + + pub firehose_publisher: FirehosePublisher, + + pub bucket_path_unifier: BucketPathUnifier, + pub semi_persistent_cache: SemiPersistentCacheDir, + + pub w2l_inference: Wav2LipInferenceCommand, + //pub ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand, + //pub ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand, + //pub imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand, + + // Command to run + pub inference_script: String, + pub w2l_model_filename: String, + pub w2l_end_bump_filename: String, +} + +#[tokio::main] +async fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + // NB: Do not check this secrets-containing dotenv file into VCS. + // This file should only contain *development* secrets, never production. + let _ = dotenv::from_filename(".env-secrets").ok(); + + info!("Obtaining hostname..."); + + let server_hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or("tts-inference-job".to_string()); + + info!("Hostname: {}", &server_hostname); + + // Bucket stuff (shared) + let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; + let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; + let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; + + // Private and Public Buckets + let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?; + let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?; + + let private_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &private_bucket_name, + None, + )?; + + let public_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &public_bucket_name, + None, + )?; + + let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?; + let py_script_name = easyenv::get_env_string_required(ENV_INFERENCE_SCRIPT_NAME)?; + //let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?; + + let w2l_inference_command = TacotronInferenceCommand::new( + &py_code_directory, + &py_script_name, + ); + + let temp_directory = easyenv::get_env_string_or_default( + "DOWNLOAD_TEMP_DIR", + DEFAULT_TEMP_DIR); + + let temp_directory = PathBuf::from(temp_directory); + + check_directory_exists(&temp_directory)?; + + let db_connection_string = + easyenv::get_env_string_or_default( + "MYSQL_URL", + DEFAULT_MYSQL_PASSWORD); + + info!("Connecting to database..."); + + let mysql_pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(&db_connection_string) + .await?; + + let inference_script = "TODO".to_string(); + + let persistent_cache_path = easyenv::get_env_string_or_default( + ENV_SEMIPERSISTENT_CACHE_DIR, + "/tmp"); + + let semi_persistent_cache = SemiPersistentCacheDir::configured_root(&persistent_cache_path); + + info!("Creating pod semi-persistent cache dirs..."); + semi_persistent_cache.create_tts_synthesizer_model_path()?; + semi_persistent_cache.create_tts_vocoder_model_path()?; + + let w2l_model_filename = easyenv::get_env_string_or_default( + "W2L_MODEL_FILENAME", "wav2lip_gan.pth"); + + let w2l_end_bump_filename = easyenv::get_env_string_or_default( + "W2L_END_BUMP_FILENAME", "vocodes-short-end-bump.mp4"); + + let firehose_publisher = FirehosePublisher { + mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe + }; + + let inferencer = Inferencer { + download_temp_directory: temp_directory, + mysql_pool, + public_bucket_client, + private_bucket_client, + inference_script, + //ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, + //ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, + //imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, + w2l_inference: w2l_inference_command, + bucket_path_unifier: BucketPathUnifier::default_paths(), + semi_persistent_cache, + firehose_publisher, + w2l_model_filename, + w2l_end_bump_filename, + }; + + main_loop(inferencer).await; + + Ok(()) +} + +const START_TIMEOUT_MILLIS : u64 = 500; +const INCREASE_TIMEOUT_MILLIS : u64 = 1000; + +async fn main_loop(inferencer: Inferencer) { + let mut timeout_millis = START_TIMEOUT_MILLIS; + + loop { + let num_records = 1; + + let query_result = query_tts_inference_job_records( + &inferencer.mysql_pool, + num_records) + .await; + + let jobs = match query_result { + Ok(jobs) => jobs, + Err(e) => { + warn!("Error querying jobs: {:?}", e); + std::thread::sleep(Duration::from_millis(timeout_millis)); + timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; + } + }; + + if jobs.is_empty() { + info!("No jobs!"); + std::thread::sleep(Duration::from_millis(1500)); + continue; + } + + let result = process_jobs(&inferencer, jobs).await; + + match result { + Ok(_) => {}, + Err(e) => { + warn!("Error querying jobs: {:?}", e); + std::thread::sleep(Duration::from_millis(timeout_millis)); + timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; + } + } + + timeout_millis = START_TIMEOUT_MILLIS; // reset + + std::thread::sleep(Duration::from_millis(500)); + } +} + +async fn process_jobs(inferencer: &Inferencer, jobs: Vec) -> AnyhowResult<()> { + for job in jobs.into_iter() { + let result = process_job(inferencer, &job).await; + match result { + Ok(_) => {}, + Err(e) => { + warn!("Failure to process job: {:?}", e); + let failure_reason = ""; + let _r = mark_w2l_inference_job_failure( + &inferencer.mysql_pool, + &job, + failure_reason) + .await; + } + } + } + + Ok(()) +} + +#[derive(Deserialize)] +struct FileMetadata { + pub is_video: bool, + pub width: u32, + pub height: u32, + //pub num_frames: u64, + //pub fps: Option, + pub duration_millis: Option, + pub mimetype: Option, + pub file_size_bytes: u64, +} + +fn read_metadata_file(filename: &PathBuf) -> AnyhowResult { + let mut file = File::open(filename)?; + let mut buffer = String::new(); + file.read_to_string(&mut buffer)?; + Ok(serde_json::from_str(&buffer)?) +} + +async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> AnyhowResult<()> { + + // TODO 1. Mark Processing + // + // TODO: 2. Check if w2l model is downloaded / download it to a stable cache location (DONE) + // TODO: 3. Check if w2l template faces are downloaded and download it (done) + // TODO: 4. Download user audio (done) + + // TODO: 5. Process Inference + + // TODO 6. Upload result + // TODO 7. Save record + // TODO 8. Mark job done + + // ==================== CONFIRM OR DOWNLOAD W2L MODEL ==================== // + + let model_filename = inferencer.w2l_model_filename.clone(); + let model_fs_path = inferencer.semi_persistent_cache.w2l_model_path(&model_filename); + + if !model_fs_path.exists() { + warn!("Model file does not exist: {:?}", &model_fs_path); + + let model_object_path = inferencer.bucket_path_unifier + .w2l_pretrained_models_path(&model_filename); + + info!("Download from bucket path: {:?}", &model_object_path); + + inferencer.private_bucket_client.download_file_to_disk( + &model_object_path, + &model_fs_path + ).await?; + + info!("Downloaded model from bucket!"); + } + + // ==================== CONFIRM OR DOWNLOAD W2L END BUMP ==================== // + + let end_bump_filename = inferencer.w2l_end_bump_filename.clone(); + let end_bump_fs_path = inferencer.semi_persistent_cache.w2l_end_bump_path(&end_bump_filename); + + if !end_bump_fs_path.exists() { + warn!("End bump file does not exist: {:?}", &end_bump_fs_path); + + let end_bump_object_path = inferencer.bucket_path_unifier + .end_bump_video_for_w2l_path(&end_bump_filename); + + info!("Download from bucket path: {:?}", &end_bump_object_path); + + inferencer.private_bucket_client.download_file_to_disk( + &end_bump_object_path, + &end_bump_fs_path + ).await?; + + info!("Downloaded end bump from bucket!"); + } + + // ==================== LOOK UP TEMPLATE RECORD ==================== // + + let template_token = match &job.maybe_w2l_template_token { + Some(token) => token.to_string(), + None => { + warn!("non-template token based inference not yet supported"); + return Err(anyhow!("non-template token based inference not yet supported")) + }, + }; + + info!("Looking up w2l template by token: {}", &template_token); + + let query_result = get_w2l_template_by_token(&inferencer.mysql_pool, &template_token).await?; + + let w2l_template = match query_result { + Some(template) => template, + None => { + warn!("W2L Template not found: {}", &template_token); + return Err(anyhow!("Template not found!")) + }, + }; + + // ==================== CONFIRM OR DOWNLOAD W2L TEMPLATE AUDIO OR VIDEO ==================== // + + // Template is based on the `private_bucket_hash`: + // - private_bucket_hash: 1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 + // - private_bucket_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 + // - private_bucket_cached_faces_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_detected_faces.pickle + // - maybe_public_bucket_preview_image_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_preview.webp + + let template_media_fs_path = inferencer.semi_persistent_cache.w2l_template_media_path( + &w2l_template.private_bucket_hash); + + if !template_media_fs_path.exists() { + info!("W2L template media file does not exist: {:?}", &template_media_fs_path); + + let template_media_object_path = inferencer.bucket_path_unifier + .media_templates_for_w2l_path(&w2l_template.private_bucket_hash); + + info!("Download from template media path: {:?}", &template_media_object_path); + + inferencer.private_bucket_client.download_file_to_disk( + &template_media_object_path, + &template_media_fs_path + ).await?; + + info!("Downloaded template media from bucket!"); + } + + // ==================== CONFIRM OR DOWNLOAD W2L TEMPLATE FACE ==================== // + + // Template is based on the `private_bucket_hash`: + // - private_bucket_hash: 1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 + // - private_bucket_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 + // - private_bucket_cached_faces_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_detected_faces.pickle + // - maybe_public_bucket_preview_image_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_preview.webp + + let face_template_fs_path = inferencer.semi_persistent_cache.w2l_face_template_path( + &w2l_template.private_bucket_hash); + + if !face_template_fs_path.exists() { + info!("W2L face template file does not exist: {:?}", &face_template_fs_path); + + let face_template_object_path = inferencer.bucket_path_unifier + .precomputed_faces_for_w2l_path(&w2l_template.private_bucket_hash); + + info!("Download from face template path: {:?}", &face_template_object_path); + + inferencer.private_bucket_client.download_file_to_disk( + &face_template_object_path, + &face_template_fs_path + ).await?; + + info!("Downloaded face template from bucket!"); + } + + // ==================== DOWNLOAD USER AUDIO ==================== // + + let temp_dir = format!("temp_{}", job.id); + let temp_dir = TempDir::new(&temp_dir)?; // NB: Exists until it goes out of scope. + + let audio_bucket_hash = match &job.maybe_public_audio_bucket_hash { + Some(l) => l.clone(), + None => { + warn!("Only W2L jobs with user-uploaded audio are supported right now"); + return Err(anyhow!("Only W2L jobs with user-uploaded audio are supported right now")) + }, + }; + + let audio_fs_path = temp_dir.path().join(&audio_bucket_hash); + + let audio_object_path = inferencer.bucket_path_unifier + .user_audio_for_w2l_inference_path(&audio_bucket_hash); + + inferencer.private_bucket_client.download_file_to_disk( + &audio_object_path, + &audio_fs_path + ).await?; + + + // ==================== RUN INFERENCE ==================== // + + let output_video_fs_path = temp_dir.path().join("output.mp4"); + let output_metadata_fs_path = temp_dir.path().join("metadata.json"); + + let is_image = w2l_template.template_type.contains("image"); + + info!("Is image? {}", is_image); + info!("Running W2L inference..."); + + inferencer.w2l_inference.execute( + &model_fs_path, + &audio_fs_path, + &end_bump_fs_path, + &template_media_fs_path, + &face_template_fs_path, + &output_metadata_fs_path, + &output_video_fs_path, + false, + false + )?; + + info!("Output filename: {:?}", &output_video_fs_path); + + // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // + + info!("Checking that output files exist..."); + + check_file_exists(&output_video_fs_path)?; + check_file_exists(&output_metadata_fs_path)?; + + let file_metadata = read_metadata_file(&output_metadata_fs_path)?; + + // ==================== UPLOAD TO BUCKETS ==================== // + + let result_object_path = inferencer.bucket_path_unifier.w2l_inference_video_output_path( + &job.inference_job_token); + + info!("Image/video destination bucket path: {:?}", &result_object_path); + + info!("Uploading image/video..."); + + let original_mime_type = file_metadata.mimetype + .as_deref() + .unwrap_or("application/octet-stream"); + + inferencer.public_bucket_client.upload_filename_with_content_type( + &result_object_path, + &output_video_fs_path, + original_mime_type) + .await?; + + // ==================== SAVE RECORDS ==================== // + + info!("Saving w2l inference record..."); + let (id, inference_result_token) = insert_w2l_result( + &inferencer.mysql_pool, + job, + &result_object_path, + file_metadata.file_size_bytes, + file_metadata.mimetype.as_deref(), + file_metadata.width, + file_metadata.height, + file_metadata.duration_millis.unwrap_or(0)) + .await?; + + info!("Marking job complete..."); + mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?; + + inferencer.firehose_publisher.w2l_inference_finished( + job.maybe_creator_user_token.as_deref(), + &job.inference_job_token, + &inference_result_token) + .await + .map_err(|e| { + warn!("error publishing event: {:?}", e); + anyhow!("error publishing event") + })?; + + info!("Job {} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}", + job.id, id); -fn main() -> AnyhowResult<()> { Ok(()) } From 5a1dafdc214762ee719b00dd056c47841b91fd61 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Jun 2021 02:17:54 -0400 Subject: [PATCH 0253/1042] local filesystem paths --- src/job_queries/tts_inference_job_queries.rs | 39 +++++++++-------- src/tts_inference_job.rs | 46 ++++++++------------ src/util/semi_persistent_cache_dir.rs | 43 +++++++++++++++--- 3 files changed, 77 insertions(+), 51 deletions(-) diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 225fb4178f..6f12a5649d 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -5,7 +5,6 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; use crate::util::anyhow_result::AnyhowResult; -use crate::util::random_crockford_token::random_crockford_token; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use log::{warn, info}; use sqlx::MySqlPool; @@ -33,9 +32,11 @@ pub struct TtsInferenceJobRecord { pub retry_at: Option>, } -pub async fn query_tts_inference_job_records(pool: &MySqlPool, num_records: u32) - -> AnyhowResult> -{ +pub async fn query_tts_inference_job_records( + pool: &MySqlPool, + num_records: u32 +) -> AnyhowResult> { + let job_records = sqlx::query_as!( TtsInferenceJobRecord, r#" @@ -78,11 +79,12 @@ WHERE Ok(job_records) } -pub async fn mark_tts_inference_job_failure(pool: &MySqlPool, - job: &TtsInferenceJobRecord, - failure_reason: &str) - -> AnyhowResult<()> -{ +pub async fn mark_tts_inference_job_failure( + pool: &MySqlPool, + job: &TtsInferenceJobRecord, + failure_reason: &str +) -> AnyhowResult<()> { + // statuses: "attempt_failed", "complete_failure", "dead" let status = "attempt_failed"; @@ -104,11 +106,11 @@ WHERE id = ? Ok(()) } -pub async fn mark_tts_inference_job_done(pool: &MySqlPool, - job: &TtsInferenceJobRecord, - success: bool) - -> AnyhowResult<()> -{ +pub async fn mark_tts_inference_job_done( + pool: &MySqlPool, + job: &TtsInferenceJobRecord, + success: bool +) -> AnyhowResult<()> { let status = if success { "complete_success" } else { "complete_failure" }; let query_result = sqlx::query!( @@ -167,7 +169,7 @@ SET inference_result_token, job.model_token.clone(), job.inference_text.clone(), - + job.maybe_creator_user_token.clone(), job.creator_ip_address.clone(), @@ -208,10 +210,11 @@ pub struct TtsModelRecord2 { pub updated_at: DateTime, } -pub async fn get_tts_model_by_token(pool: &MySqlPool, model_token: &str) - -> AnyhowResult> +pub async fn get_tts_model_by_token( + pool: &MySqlPool, + model_token: &str +) -> AnyhowResult> { - // NB: Lookup failure is Err(RowNotFound). // NB: Since this is publicly exposed, we don't query sensitive data. let maybe_model = sqlx::query_as!( diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index fd5b9f160c..9869257090 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -21,15 +21,12 @@ use crate::buckets::bucket_path_unifier::BucketPathUnifier; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::buckets::file_hashing::get_file_hash; use crate::common_queries::firehose_publisher::FirehosePublisher; -use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; -use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; -use crate::job_queries::w2l_inference_job_queries::insert_w2l_result; -use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_done; -use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failure; -use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records; -use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; -use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; -use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; +use crate::job_queries::tts_inference_job_queries::TtsInferenceJobRecord; +use crate::job_queries::tts_inference_job_queries::get_tts_model_by_token; +use crate::job_queries::tts_inference_job_queries::insert_tts_result; +use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_done; +use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_failure; +use crate::job_queries::tts_inference_job_queries::query_tts_inference_job_records; use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; @@ -83,13 +80,9 @@ struct Inferencer { pub bucket_path_unifier: BucketPathUnifier, pub semi_persistent_cache: SemiPersistentCacheDir, - pub w2l_inference: Wav2LipInferenceCommand, - //pub ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand, - //pub ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand, - //pub imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand, + pub tts_inference: TacotronInferenceCommand, // Command to run - pub inference_script: String, pub w2l_model_filename: String, pub w2l_end_bump_filename: String, } @@ -165,8 +158,6 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; - let inference_script = "TODO".to_string(); - let persistent_cache_path = easyenv::get_env_string_or_default( ENV_SEMIPERSISTENT_CACHE_DIR, "/tmp"); @@ -192,7 +183,6 @@ async fn main() -> AnyhowResult<()> { mysql_pool, public_bucket_client, private_bucket_client, - inference_script, //ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, //ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, //imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, @@ -298,22 +288,24 @@ fn read_metadata_file(filename: &PathBuf) -> AnyhowResult { async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> AnyhowResult<()> { - // TODO 1. Mark Processing - // - // TODO: 2. Check if w2l model is downloaded / download it to a stable cache location (DONE) - // TODO: 3. Check if w2l template faces are downloaded and download it (done) - // TODO: 4. Download user audio (done) + // TODO 1. Mark processing - // TODO: 5. Process Inference + // TODO 2. Check if vocoder model is downloaded / download to stable location - // TODO 6. Upload result + // TODO 3. Query model by token. + // TODO 4. Check if model is downloaded, otherwise download to stable location + + // TODO 5. Process Inference + + // TODO 6. Upload Result // TODO 7. Save record // TODO 8. Mark job done - // ==================== CONFIRM OR DOWNLOAD W2L MODEL ==================== // - let model_filename = inferencer.w2l_model_filename.clone(); - let model_fs_path = inferencer.semi_persistent_cache.w2l_model_path(&model_filename); + // ==================== CONFIRM OR DOWNLOAD TTS VOCODER MODEL ==================== // + + let tts_vocoder_model_filename = inferencer.tts_vocoder_model_filename.clone(); + let tts_vocoder_model_fs_path = inferencer.semi_persistent_cache.tts_vocoder_model_path(&tts_vocoder_model_filename); if !model_fs_path.exists() { warn!("Model file does not exist: {:?}", &model_fs_path); diff --git a/src/util/semi_persistent_cache_dir.rs b/src/util/semi_persistent_cache_dir.rs index 30f3211858..5cd05d51f5 100644 --- a/src/util/semi_persistent_cache_dir.rs +++ b/src/util/semi_persistent_cache_dir.rs @@ -6,7 +6,10 @@ use tempdir::TempDir; /// These are for files on the worker filesystems pub struct SemiPersistentCacheDir { cache_root: PathBuf, - tts_model_root: PathBuf, + + tts_synthesizer_model_root: PathBuf, + tts_vocoder_model_root: PathBuf, + w2l_model_root: PathBuf, w2l_end_bump_root: PathBuf, w2l_face_templates_root: PathBuf, @@ -27,7 +30,10 @@ impl SemiPersistentCacheDir { let cache_root = PathBuf::from(root_path); Self { cache_root: cache_root.clone(), - tts_model_root: cache_root.join("tts/models/"), + + tts_synthesizer_model_root: cache_root.join("tts/synthesizer_models/"), + tts_vocoder_model_root: cache_root.join("tts/vocoder_models/"), + w2l_model_root: cache_root.join("w2l/models/"), w2l_end_bump_root: cache_root.join("w2l/end_bumps/"), w2l_face_templates_root: cache_root.join("w2l/face_templates/"), @@ -36,12 +42,37 @@ impl SemiPersistentCacheDir { } } - // ==================== TTS MODELS ==================== + // ==================== TTS SYNTHESIZER MODELS ==================== - /// We cache TTS models here. + /// We cache TTS synthesizer models here. /// We'll likely need to LRU cache them. - pub fn tts_model_path(&self) -> &Path { - &self.tts_model_root + pub fn tts_synthesizer_model_path(&self, model_filename: &str) -> PathBuf { + self.tts_synthesizer_model_root.join(model_filename) + } + + pub fn tts_synthesizer_model_directory(&self) -> &Path { + &self.tts_synthesizer_model_root + } + + pub fn create_tts_synthesizer_model_path(&self) -> AnyhowResult<()> { + let _ = fs::create_dir_all(self.tts_synthesizer_model_directory())?; + Ok(()) + } + + // ==================== TTS VOCODER MODELS ==================== + + /// We'll start with just a few vocoders, but this may grow. + pub fn tts_vocoder_model_path(&self, model_filename: &str) -> PathBuf { + self.tts_vocoder_model_root.join(model_filename) + } + + pub fn tts_vocoder_model_directory(&self) -> &Path { + &self.tts_vocoder_model_root + } + + pub fn create_tts_vocoder_model_path(&self) -> AnyhowResult<()> { + let _ = fs::create_dir_all(self.tts_vocoder_model_directory())?; + Ok(()) } // ==================== W2L MODELS (there are only two of them) ==================== From 79f8b0ad7f982d94fe6623e74ecaea32f3616ac5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Jun 2021 03:02:05 -0400 Subject: [PATCH 0254/1042] add bucket paths for TTS (untested) --- .../up.sql | 2 +- src/buckets/bucket_path_unifier.rs | 90 ++++++++++++++++++- src/tts_inference_job.rs | 30 +++---- 3 files changed, 98 insertions(+), 24 deletions(-) diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 94f93d8300..b8b405c918 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -40,7 +40,7 @@ CREATE TABLE tts_results ( public_bucket_hash CHAR(64) NOT NULL, -- Where the wav, spectrogram, and etc. are located. - public_bucket_audio_path VARCHAR(255) NOT NULL, + inference_public_bucket_uuid VARCHAR(36) NOT NULL, -- ========== METADATA ========== diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs index 88dec45496..927d8b887a 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/buckets/bucket_path_unifier.rs @@ -5,9 +5,13 @@ use crate::util::anyhow_result::AnyhowResult; /// This is designed to make it centrally configurable where /// different types of objects are stored. pub struct BucketPathUnifier { + // TTS + pub tts_pretrained_vocoder_model_root: PathBuf, + pub tts_synthesizer_model_root: PathBuf, + pub tts_inference_output_root: PathBuf, + // W2L pub user_uploaded_w2l_templates_root: PathBuf, pub user_uploaded_audio_for_w2l_root: PathBuf, - pub tts_inference_output_root: PathBuf, pub w2l_inference_output_root: PathBuf, pub w2l_model_root: PathBuf, pub w2l_end_bump_root: PathBuf, @@ -25,15 +29,58 @@ impl BucketPathUnifier { pub fn default_paths() -> Self { Self { + // TTS + tts_pretrained_vocoder_model_root: PathBuf::from("/tts_pretrained_vocoders"), + tts_synthesizer_model_root: PathBuf::from("/user_uploaded_tts_synthesizers"), + tts_inference_output_root: PathBuf::from("/tts_inference_output"), + // W2L user_uploaded_audio_for_w2l_root: PathBuf::from("/user_uploaded_w2l_audio"), user_uploaded_w2l_templates_root: PathBuf::from("/user_uploaded_w2l_templates"), - tts_inference_output_root: PathBuf::from("/tts_inference_output"), w2l_inference_output_root: PathBuf::from("/w2l_inference_output"), w2l_model_root: PathBuf::from("/w2l_pretrained_models"), w2l_end_bump_root: PathBuf::from("/w2l_end_bumps"), } } + // ==================== TTS MODELS (SYNTHESIZER + VOCODER) ==================== // + + // TTS pretrained vocoder models. + // For now this will be limited, but once they're user-uploadable, we'll add more. + pub fn tts_pretrained_vocoders_path(&self, tts_vocoder_model_name: &str) -> PathBuf { + self.tts_pretrained_vocoder_model_root.join(tts_vocoder_model_name) + } + + pub fn tts_synthesizer_path(&self, tts_synthesizer_file_hash: &str) -> PathBuf { + let hashed_path = Self::hashed_directory_path(tts_synthesizer_file_hash); + let model_filename = format!("{}.pt", &tts_synthesizer_file_hash); + + self.tts_synthesizer_model_root + .join(hashed_path) + .join(model_filename) + } + + // ==================== TTS INFERENCE OUTPUT ==================== // + + pub fn tts_inference_wav_audio_output_path(&self, tts_inference_output_uuid: &str) -> PathBuf { + let hashed_path = Self::hashed_directory_path(tts_inference_output_uuid); + let audio_filename = format!("{}.wav", &tts_inference_output_uuid); + + self.tts_inference_output_root + .join(hashed_path) + .join(audio_filename) + } + + pub fn tts_inference_spectrogram_output_path(&self, tts_inference_output_uuid: &str) -> PathBuf { + let hashed_path = Self::hashed_directory_path(tts_inference_output_uuid); + let json_filename = format!("{}.json", &tts_inference_output_uuid); + + self.tts_inference_output_root + .join(hashed_path) + .join(json_filename) + } + + // ==================== W2L STATIC RESOURCES ==================== // + // W2L pretrained models. There are only two. pub fn w2l_pretrained_models_path(&self, w2l_model_name: &str) -> PathBuf { self.w2l_model_root.join(w2l_model_name) @@ -44,6 +91,8 @@ impl BucketPathUnifier { self.w2l_end_bump_root.join(end_bump_filename) } + // ==================== W2L USER-UPLOADED RESOURCES ==================== // + // The video or images uploaded as templates // eg. /user_uploaded_w2l_templates/1/5/1/151a[...60]... pub fn media_templates_for_w2l_path(&self, template_file_hash: &str) -> PathBuf { @@ -73,6 +122,8 @@ impl BucketPathUnifier { .join(audio_uuid) } + // ==================== W2L INFERENCE OUTPUT ==================== // + // W2L inference output videos pub fn w2l_inference_video_output_path(&self, w2l_inference_job_token: &str) -> PathBuf { // NB: We don't want colons from the token in the filename. @@ -90,7 +141,6 @@ impl BucketPathUnifier { } } - let video_filename = w2l_inference_job_token.replace(":", ""); let video_filename = format!("vocodes_video_{}.mp4", video_filename); @@ -118,15 +168,47 @@ mod tests { fn get_instance() -> BucketPathUnifier { BucketPathUnifier { + // TTS + tts_pretrained_vocoder_model_root: PathBuf::from("/test_path_tts_vocoders"), + tts_synthesizer_model_root: PathBuf::from("/test_path_synthesizers"), + tts_inference_output_root: PathBuf::from("/test_path_tts_output"), + // W2L user_uploaded_w2l_templates_root: PathBuf::from("/test_path_w2l_templates"), user_uploaded_audio_for_w2l_root: PathBuf::from("/test_path_w2l_audio"), - tts_inference_output_root: PathBuf::from("/test_path_tts_output"), w2l_inference_output_root: PathBuf::from("/test_path_w2l_output"), w2l_model_root: PathBuf::from("/test_path_w2l_pretrained_models"), w2l_end_bump_root: PathBuf::from("/test_path_w2l_end_bumps"), } } + #[test] + fn test_tts_pretrained_vocoders_path() { + let paths = get_instance(); + assert_eq!(paths.tts_pretrained_vocoders_path("melgan.pth").to_str().unwrap(), + "/test_path_tts_vocoders/melgan.pth"); + } + + #[test] + fn test_tts_synthesizer_path() { + let paths = get_instance(); + assert_eq!(paths.tts_synthesizer_path("foobar").to_str().unwrap(), + "/test_path_synthesizers/f/o/o/foobar.pth"); + } + + #[test] + fn test_tts_inference_wav_audio_output_path() { + let paths = get_instance(); + assert_eq!(paths.tts_inference_wav_audio_output_path("foobar").to_str().unwrap(), + "/test_path_tts_output/f/o/o/foobar.wav"); + } + + #[test] + fn test_tts_inference_spectrogram_output_path() { + let paths = get_instance(); + assert_eq!(paths.tts_inference_spectrogram_output_path("foobar").to_str().unwrap(), + "/test_path_tts_output/f/o/o/foobar.json"); + } + #[test] fn test_w2l_pretrained_models_path() { let paths = get_instance(); diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 9869257090..6d56196b8d 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -83,8 +83,7 @@ struct Inferencer { pub tts_inference: TacotronInferenceCommand, // Command to run - pub w2l_model_filename: String, - pub w2l_end_bump_filename: String, + pub tts_vocoder_model_filename: String, } #[tokio::main] @@ -133,7 +132,7 @@ async fn main() -> AnyhowResult<()> { let py_script_name = easyenv::get_env_string_required(ENV_INFERENCE_SCRIPT_NAME)?; //let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?; - let w2l_inference_command = TacotronInferenceCommand::new( + let tts_inference_command = TacotronInferenceCommand::new( &py_code_directory, &py_script_name, ); @@ -154,7 +153,7 @@ async fn main() -> AnyhowResult<()> { info!("Connecting to database..."); let mysql_pool = MySqlPoolOptions::new() - .max_connections(5) + .max_connections(2) .connect(&db_connection_string) .await?; @@ -162,17 +161,16 @@ async fn main() -> AnyhowResult<()> { ENV_SEMIPERSISTENT_CACHE_DIR, "/tmp"); - let semi_persistent_cache = SemiPersistentCacheDir::configured_root(&persistent_cache_path); + let semi_persistent_cache = + SemiPersistentCacheDir::configured_root(&persistent_cache_path); info!("Creating pod semi-persistent cache dirs..."); semi_persistent_cache.create_tts_synthesizer_model_path()?; semi_persistent_cache.create_tts_vocoder_model_path()?; - let w2l_model_filename = easyenv::get_env_string_or_default( - "W2L_MODEL_FILENAME", "wav2lip_gan.pth"); + let tts_vocoder_model_filename = easyenv::get_env_string_or_default( + "TTS_VOCODER_MODEL_FILENAME", "waveglow.pth"); - let w2l_end_bump_filename = easyenv::get_env_string_or_default( - "W2L_END_BUMP_FILENAME", "vocodes-short-end-bump.mp4"); let firehose_publisher = FirehosePublisher { mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe @@ -186,12 +184,11 @@ async fn main() -> AnyhowResult<()> { //ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, //ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, //imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, - w2l_inference: w2l_inference_command, + tts_inference: tts_inference_command, bucket_path_unifier: BucketPathUnifier::default_paths(), semi_persistent_cache, firehose_publisher, - w2l_model_filename, - w2l_end_bump_filename, + tts_vocoder_model_filename, }; main_loop(inferencer).await; @@ -269,11 +266,6 @@ async fn process_jobs(inferencer: &Inferencer, jobs: Vec) #[derive(Deserialize)] struct FileMetadata { - pub is_video: bool, - pub width: u32, - pub height: u32, - //pub num_frames: u64, - //pub fps: Option, pub duration_millis: Option, pub mimetype: Option, pub file_size_bytes: u64, @@ -307,8 +299,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An let tts_vocoder_model_filename = inferencer.tts_vocoder_model_filename.clone(); let tts_vocoder_model_fs_path = inferencer.semi_persistent_cache.tts_vocoder_model_path(&tts_vocoder_model_filename); - if !model_fs_path.exists() { - warn!("Model file does not exist: {:?}", &model_fs_path); + if !tts_vocoder_model_fs_path.exists() { + warn!("Vocoder model file does not exist: {:?}", &tts_vocoder_model_fs_path); let model_object_path = inferencer.bucket_path_unifier .w2l_pretrained_models_path(&model_filename); From 0f2da6f6e59bb39966d8b2f82bfd869bc21fdb2f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Jun 2021 13:32:53 -0400 Subject: [PATCH 0255/1042] fix column name --- README.md | 2 ++ src/job_queries/tts_inference_job_queries.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b2639835b7..f5488157ca 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,8 @@ sudo mysql -u root -p # but now I have to use "sudo" !?!? +use mysql; + CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password'; GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost'; ``` diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 6f12a5649d..7ea70921fb 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -160,7 +160,7 @@ SET creator_set_visibility = 'public', public_bucket_hash = 'TODO', - public_bucket_audio_path = ?, + inference_public_bucket_uuid = ?, file_size_bytes = ?, mime_type = ?, From a0919fffee3c59c613c214ec7f1e84613bf19a36 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Jun 2021 14:23:16 -0400 Subject: [PATCH 0256/1042] overhaul tts_job --- .../up.sql | 4 +- src/common_queries/firehose_publisher.rs | 15 ++ src/job_queries/tts_inference_job_queries.rs | 13 +- .../tacotron_inference_command.rs | 5 +- src/tts_inference_job.rs | 233 +++++++----------- 5 files changed, 116 insertions(+), 154 deletions(-) diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index b8b405c918..609d35718e 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -38,6 +38,9 @@ CREATE TABLE tts_results ( -- Where the wav, spectrogram, and etc. are located. public_bucket_hash CHAR(64) NOT NULL, + -- Where the wav, spectrogram, and etc. are located. + public_bucket_wav_audio_path VARCHAR(255) NOT NULL, + public_bucket_spectrogram_path VARCHAR(255) NOT NULL, -- Where the wav, spectrogram, and etc. are located. inference_public_bucket_uuid VARCHAR(36) NOT NULL, @@ -45,7 +48,6 @@ CREATE TABLE tts_results ( -- ========== METADATA ========== file_size_bytes INT(10) NOT NULL DEFAULT 0, - mime_type VARCHAR(32) NOT NULL DEFAULT '', duration_millis INT(10) NOT NULL DEFAULT 0, -- ========== MODERATION DETAILS ========== diff --git a/src/common_queries/firehose_publisher.rs b/src/common_queries/firehose_publisher.rs index c4de611150..98e4e00885 100644 --- a/src/common_queries/firehose_publisher.rs +++ b/src/common_queries/firehose_publisher.rs @@ -103,6 +103,21 @@ impl FirehosePublisher { Ok(()) } + pub async fn tts_inference_finished( + &self, + maybe_user_token: Option<&str>, + model_token: &str, + result_token: &str + ) -> AnyhowResult<()> { + let _record_id = self.insert( + FirehoseEvent::TtsInferenceCompleted, + maybe_user_token, + Some(model_token), + Some(result_token) + ).await?; + Ok(()) + } + pub async fn enqueue_w2l_template_upload(&self, user_token: &str, job_token: &str) -> AnyhowResult<()> { let _record_id = self.insert( FirehoseEvent::W2lTemplateUploadStarted, diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 6f12a5649d..1f201a78e9 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -135,8 +135,8 @@ pub async fn insert_tts_result>( pool: &MySqlPool, job: &TtsInferenceJobRecord, bucket_audio_results_path: P, + bucket_spectrogram_results_path: P, file_size_bytes: u64, - maybe_mime_type: Option<&str>, duration_millis: u64 ) -> AnyhowResult<(u64, String)> { @@ -147,6 +147,11 @@ pub async fn insert_tts_result>( .display() .to_string(); + let bucket_spectrogram_result_path = &bucket_audio_results_path + .as_ref() + .display() + .to_string(); + let query_result = sqlx::query!( r#" INSERT INTO tts_results @@ -160,10 +165,10 @@ SET creator_set_visibility = 'public', public_bucket_hash = 'TODO', - public_bucket_audio_path = ?, + public_bucket_wav_audio_path = ?, + public_bucket_spectrogram_path = ?, file_size_bytes = ?, - mime_type = ?, duration_millis = ? "#, inference_result_token, @@ -174,9 +179,9 @@ SET job.creator_ip_address.clone(), bucket_audio_result_path, + bucket_spectrogram_result_path, file_size_bytes, - maybe_mime_type.unwrap_or(""), duration_millis ) .execute(pool) diff --git a/src/script_execution/tacotron_inference_command.rs b/src/script_execution/tacotron_inference_command.rs index fc22274dc1..7a9b3c0344 100644 --- a/src/script_execution/tacotron_inference_command.rs +++ b/src/script_execution/tacotron_inference_command.rs @@ -29,8 +29,9 @@ impl TacotronInferenceCommand { synthesizer_checkpoint_path: P, vocoder_checkpoint_path: P, input_text_filename: P, - output_metadata_filename: P, output_audio_filename: P, + output_spectrogram_filename: P, + output_metadata_filename: P, spawn_process: bool ) -> AnyhowResult<()> { let mut command = String::new(); @@ -51,6 +52,8 @@ impl TacotronInferenceCommand { command.push_str(&input_text_filename.as_ref().display().to_string()); command.push_str(" --output_audio_filename "); command.push_str(&output_audio_filename.as_ref().display().to_string()); + command.push_str(" --output_spectrogram_filename "); + command.push_str(&output_spectrogram_filename.as_ref().display().to_string()); command.push_str(" --output_metadata_filename "); command.push_str(&output_metadata_filename.as_ref().display().to_string()); diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 6d56196b8d..9ed80a1941 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -244,7 +244,7 @@ async fn main_loop(inferencer: Inferencer) { } } -async fn process_jobs(inferencer: &Inferencer, jobs: Vec) -> AnyhowResult<()> { +async fn process_jobs(inferencer: &Inferencer, jobs: Vec) -> AnyhowResult<()> { for job in jobs.into_iter() { let result = process_job(inferencer, &job).await; match result { @@ -252,7 +252,7 @@ async fn process_jobs(inferencer: &Inferencer, jobs: Vec) Err(e) => { warn!("Failure to process job: {:?}", e); let failure_reason = ""; - let _r = mark_w2l_inference_job_failure( + let _r = mark_tts_inference_job_failure( &inferencer.mysql_pool, &job, failure_reason) @@ -278,20 +278,21 @@ fn read_metadata_file(filename: &PathBuf) -> AnyhowResult { Ok(serde_json::from_str(&buffer)?) } -async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> AnyhowResult<()> { +async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> AnyhowResult<()> { // TODO 1. Mark processing - // TODO 2. Check if vocoder model is downloaded / download to stable location + // TODO 2. Check if vocoder model is downloaded / download to stable location (DONE) - // TODO 3. Query model by token. - // TODO 4. Check if model is downloaded, otherwise download to stable location + // TODO 3. Query model by token. (DONE) + // TODO 4. Check if model is downloaded, otherwise download to stable location (DONE) - // TODO 5. Process Inference + // TODO 5. Write text to file + // TODO 6. Process Inference - // TODO 6. Upload Result - // TODO 7. Save record - // TODO 8. Mark job done + // TODO 7. Upload Result + // TODO 8. Save record + // TODO 9. Mark job done // ==================== CONFIRM OR DOWNLOAD TTS VOCODER MODEL ==================== // @@ -302,212 +303,148 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An if !tts_vocoder_model_fs_path.exists() { warn!("Vocoder model file does not exist: {:?}", &tts_vocoder_model_fs_path); - let model_object_path = inferencer.bucket_path_unifier - .w2l_pretrained_models_path(&model_filename); + let tts_vocoder_model_object_path = inferencer.bucket_path_unifier + .tts_pretrained_vocoders_path(&tts_vocoder_model_filename); - info!("Download from bucket path: {:?}", &model_object_path); + info!("Download vocoder from bucket path: {:?}", &tts_vocoder_model_object_path); inferencer.private_bucket_client.download_file_to_disk( - &model_object_path, - &model_fs_path + &tts_vocoder_model_object_path, + &tts_vocoder_model_fs_path ).await?; - info!("Downloaded model from bucket!"); + info!("Downloaded tts vocoder model from bucket!"); } - // ==================== CONFIRM OR DOWNLOAD W2L END BUMP ==================== // + // ==================== LOOK UP TTS SYNTHESIZER RECORD (WHICH CONTAINS ITS BUCKET PATH) ==================== // - let end_bump_filename = inferencer.w2l_end_bump_filename.clone(); - let end_bump_fs_path = inferencer.semi_persistent_cache.w2l_end_bump_path(&end_bump_filename); + info!("Looking up TTS model by token: {}", &job.model_token); - if !end_bump_fs_path.exists() { - warn!("End bump file does not exist: {:?}", &end_bump_fs_path); + let query_result = get_tts_model_by_token( + &inferencer.mysql_pool, + &job.model_token).await?; - let end_bump_object_path = inferencer.bucket_path_unifier - .end_bump_video_for_w2l_path(&end_bump_filename); - - info!("Download from bucket path: {:?}", &end_bump_object_path); - - inferencer.private_bucket_client.download_file_to_disk( - &end_bump_object_path, - &end_bump_fs_path - ).await?; - - info!("Downloaded end bump from bucket!"); - } - - // ==================== LOOK UP TEMPLATE RECORD ==================== // - - let template_token = match &job.maybe_w2l_template_token { - Some(token) => token.to_string(), + let tts_model = match query_result { + Some(model) => model, None => { - warn!("non-template token based inference not yet supported"); - return Err(anyhow!("non-template token based inference not yet supported")) + warn!("TTS model not found: {}", &job.model_token); + return Err(anyhow!("Model not found!")) }, }; - info!("Looking up w2l template by token: {}", &template_token); + // ==================== CONFIRM OR DOWNLOAD TTS SYNTHESIZER MODEL ==================== // - let query_result = get_w2l_template_by_token(&inferencer.mysql_pool, &template_token).await?; + // TODO: Let's just put paths in the db + // TODO: We'll probably need to LRU cache these. - let w2l_template = match query_result { - Some(template) => template, - None => { - warn!("W2L Template not found: {}", &template_token); - return Err(anyhow!("Template not found!")) - }, - }; + let tts_synthesizer_fs_path = inferencer.semi_persistent_cache.tts_synthesizer_model_path( + &tts_model.model_token); - // ==================== CONFIRM OR DOWNLOAD W2L TEMPLATE AUDIO OR VIDEO ==================== // + if !tts_synthesizer_fs_path.exists() { + info!("TTS synthesizer model file does not exist: {:?}", &tts_synthesizer_fs_path); - // Template is based on the `private_bucket_hash`: - // - private_bucket_hash: 1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 - // - private_bucket_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 - // - private_bucket_cached_faces_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_detected_faces.pickle - // - maybe_public_bucket_preview_image_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_preview.webp + let tts_synthesizer_object_path = inferencer.bucket_path_unifier + .tts_synthesizer_path(&tts_model.private_bucket_hash); - let template_media_fs_path = inferencer.semi_persistent_cache.w2l_template_media_path( - &w2l_template.private_bucket_hash); - - if !template_media_fs_path.exists() { - info!("W2L template media file does not exist: {:?}", &template_media_fs_path); - - let template_media_object_path = inferencer.bucket_path_unifier - .media_templates_for_w2l_path(&w2l_template.private_bucket_hash); - - info!("Download from template media path: {:?}", &template_media_object_path); + info!("Download from template media path: {:?}", &tts_synthesizer_object_path); inferencer.private_bucket_client.download_file_to_disk( - &template_media_object_path, - &template_media_fs_path + &tts_synthesizer_object_path, + &tts_synthesizer_fs_path ).await?; info!("Downloaded template media from bucket!"); } - // ==================== CONFIRM OR DOWNLOAD W2L TEMPLATE FACE ==================== // + // ==================== WRITE TEXT TO FILE ==================== // - // Template is based on the `private_bucket_hash`: - // - private_bucket_hash: 1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 - // - private_bucket_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2 - // - private_bucket_cached_faces_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_detected_faces.pickle - // - maybe_public_bucket_preview_image_object_name: /user_uploaded_w2l_templates/1/5/1/1519edf86e6975fdcd0a56a5953d84948db79f2b9ce588818d7fa544d5cb38b2_preview.webp - - let face_template_fs_path = inferencer.semi_persistent_cache.w2l_face_template_path( - &w2l_template.private_bucket_hash); - - if !face_template_fs_path.exists() { - info!("W2L face template file does not exist: {:?}", &face_template_fs_path); - - let face_template_object_path = inferencer.bucket_path_unifier - .precomputed_faces_for_w2l_path(&w2l_template.private_bucket_hash); - - info!("Download from face template path: {:?}", &face_template_object_path); - - inferencer.private_bucket_client.download_file_to_disk( - &face_template_object_path, - &face_template_fs_path - ).await?; - - info!("Downloaded face template from bucket!"); - } - - // ==================== DOWNLOAD USER AUDIO ==================== // - - let temp_dir = format!("temp_{}", job.id); + let temp_dir = format!("temp_tts_inference_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; // NB: Exists until it goes out of scope. - let audio_bucket_hash = match &job.maybe_public_audio_bucket_hash { - Some(l) => l.clone(), - None => { - warn!("Only W2L jobs with user-uploaded audio are supported right now"); - return Err(anyhow!("Only W2L jobs with user-uploaded audio are supported right now")) - }, - }; - - let audio_fs_path = temp_dir.path().join(&audio_bucket_hash); - - let audio_object_path = inferencer.bucket_path_unifier - .user_audio_for_w2l_inference_path(&audio_bucket_hash); - - inferencer.private_bucket_client.download_file_to_disk( - &audio_object_path, - &audio_fs_path - ).await?; - + let text_input_fs_path = None; // ==================== RUN INFERENCE ==================== // - let output_video_fs_path = temp_dir.path().join("output.mp4"); + let output_audio_fs_path = temp_dir.path().join("output.mp4"); let output_metadata_fs_path = temp_dir.path().join("metadata.json"); + let output_spectrogram_fs_path = temp_dir.path().join("spectrogram.json"); let is_image = w2l_template.template_type.contains("image"); - info!("Is image? {}", is_image); - info!("Running W2L inference..."); + info!("Running TTS inference..."); - inferencer.w2l_inference.execute( - &model_fs_path, - &audio_fs_path, - &end_bump_fs_path, - &template_media_fs_path, - &face_template_fs_path, + info!("Expected output audio filename: {:?}", &output_audio_fs_path); + info!("Expected output spectrogram filename: {:?}", &output_spectrogram_fs_path); + info!("Expected output metadata filename: {:?}", &output_metadata_fs_path); + + inferencer.tts_inference.execute( + &tts_synthesizer_fs_path, + &tts_vocoder_model_fs_path, + &text_input_fs_path, + &output_audio_fs_path, + &output_spectrogram_fs_path, &output_metadata_fs_path, - &output_video_fs_path, false, - false )?; - info!("Output filename: {:?}", &output_video_fs_path); - // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // info!("Checking that output files exist..."); - check_file_exists(&output_video_fs_path)?; + check_file_exists(&output_audio_fs_path)?; + check_file_exists(&output_spectrogram_fs_path)?; check_file_exists(&output_metadata_fs_path)?; let file_metadata = read_metadata_file(&output_metadata_fs_path)?; - // ==================== UPLOAD TO BUCKETS ==================== // + // ==================== UPLOAD AUDIO TO BUCKET ==================== // - let result_object_path = inferencer.bucket_path_unifier.w2l_inference_video_output_path( + let audio_result_object_path = inferencer.bucket_path_unifier.tts_inference_wav_audio_output_path( &job.inference_job_token); - info!("Image/video destination bucket path: {:?}", &result_object_path); + info!("Audio destination bucket path: {:?}", &audio_result_object_path); - info!("Uploading image/video..."); - - let original_mime_type = file_metadata.mimetype - .as_deref() - .unwrap_or("application/octet-stream"); + info!("Uploading audio..."); inferencer.public_bucket_client.upload_filename_with_content_type( - &result_object_path, - &output_video_fs_path, - original_mime_type) + &audio_result_object_path, + &output_audio_fs_path, + "audio/wav") + .await?; + + // ==================== UPLOAD SPECTROGRAM TO BUCKETS ==================== // + + let spectrogram_result_object_path = inferencer.bucket_path_unifier.tts_inference_spectrogram_output_path( + &job.inference_job_token); + + info!("Spectrogram destination bucket path: {:?}", &spectrogram_result_object_path); + + info!("Uploading spectrogram..."); + + inferencer.public_bucket_client.upload_filename_with_content_type( + &spectrogram_result_object_path, + &output_spectrogram_fs_path, + "application/json") .await?; // ==================== SAVE RECORDS ==================== // - info!("Saving w2l inference record..."); - let (id, inference_result_token) = insert_w2l_result( + info!("Saving tts inference record..."); + let (id, inference_result_token) = insert_tts_result( &inferencer.mysql_pool, job, - &result_object_path, + &audio_result_object_path, + &spectrogram_result_object_path, file_metadata.file_size_bytes, - file_metadata.mimetype.as_deref(), - file_metadata.width, - file_metadata.height, file_metadata.duration_millis.unwrap_or(0)) .await?; info!("Marking job complete..."); - mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?; + mark_tts_inference_job_done(&inferencer.mysql_pool, job, true).await?; - inferencer.firehose_publisher.w2l_inference_finished( + inferencer.firehose_publisher.tts_inference_finished( job.maybe_creator_user_token.as_deref(), - &job.inference_job_token, + &tts_model.model_token, &inference_result_token) .await .map_err(|e| { From 86b9d2eccacfa32b30e9af03f11670de686a80a4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Jun 2021 14:29:40 -0400 Subject: [PATCH 0257/1042] fixup tts_inference_job (python still needs writing) --- src/tts_inference_job.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 9ed80a1941..6d649b0943 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -361,7 +361,9 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An let temp_dir = format!("temp_tts_inference_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; // NB: Exists until it goes out of scope. - let text_input_fs_path = None; + let text_input_fs_path = temp_dir.path().join("inference_input.txt"); + + std::fs::write(&text_input_fs_path, &job.inference_text)?; // ==================== RUN INFERENCE ==================== // @@ -369,8 +371,6 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An let output_metadata_fs_path = temp_dir.path().join("metadata.json"); let output_spectrogram_fs_path = temp_dir.path().join("spectrogram.json"); - let is_image = w2l_template.template_type.contains("image"); - info!("Running TTS inference..."); info!("Expected output audio filename: {:?}", &output_audio_fs_path); From 46f878beaa72ab40608bd709e448a1d467a35803 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Jun 2021 16:21:03 -0400 Subject: [PATCH 0258/1042] tts inference script --- models/tacotron2/vocodes_inference.py | 109 +++++++++++--------------- src/tts_inference_job.rs | 13 ++- 2 files changed, 49 insertions(+), 73 deletions(-) diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py index 4fc646da1b..8c54e8fac6 100755 --- a/models/tacotron2/vocodes_inference.py +++ b/models/tacotron2/vocodes_inference.py @@ -1,23 +1,32 @@ #!/usr/bin/env python -# 1 -#import matplotlib -#import matplotlib.pylab as plt +""" +Vocodes Tacotron Inference Engine +This is a work in progress and is likely going to be slow for now. +""" -#import IPython.display as ipd +import argparse + +parser = argparse.ArgumentParser(description='Run TTS inference') + +parser.add_argument('--synthesizer_checkpoint_path', type=str, help='path the TTS synthesizer model', required=True) +parser.add_argument('--vocoder_checkpoint_path', type=str, help='path the TTS vocoder model', required=True) +parser.add_argument('--input_text_filename', type=str, help='path the file containing text to run', required=True) +parser.add_argument('--output_audio_filename', type=str, help='where to save result audio', required=True) +parser.add_argument('--output_spectrogram_filename', type=str, help='where to save result spectrogram', required=True) +parser.add_argument('--output_metadata_filename', type=str, help='where to save extra metadata', required=True) + +args = parser.parse_args() import sys sys.path.append('waveglow/') import numpy as np import torch +import json from hparams import create_hparams -from model import Tacotron2 -from layers import TacotronSTFT, STFT -from audio_processing import griffin_lim from train import load_model from text import text_to_sequence -#from denoiser import Denoiser print('========================================') print('Python interpreter', sys.executable) @@ -29,18 +38,6 @@ print('========================================', flush=True) # NB(bt, 2021-05-31): Trying to get everything on the same device torch.device("cuda" if torch.cuda.is_available() else "cpu") -# NB(bt, 2021-05-31): This is just a utility for testing mel output -# The outputs look good so far... -#from vocodes_wav_images import render_histogram - -# 2 -def plot_data(data, figsize=(16, 4)): - fig, axes = plt.subplots(1, len(data), figsize=figsize) - for i in range(len(data)): - axes[i].imshow(data[i], aspect='auto', origin='bottom', - interpolation='none') - -# 3 hparams = create_hparams() hparams.sampling_rate = 22050 @@ -50,77 +47,50 @@ hparams.sampling_rate = 22050 #torch.set_default_tensor_type('torch.DoubleTensor') # 2021-05-31 commented out torch.set_default_tensor_type('torch.HalfTensor') # 2021-05-31 added -# 4 -checkpoint_path = "/home/bt/models/tacotron2-nvidia/tacotron2_statedict.pt" -#checkpoint_path = "/home/bt/models/tigger_tt_model.pt" -#checkpoint_path = "/home/bt/models/uber_test_2.pt" -checkpoint_path = "/home/bt/models/tacotron2/tacotron2_uberduck_JorgenVonStrangle" -checkpoint_path = "/home/bt/models/tacotron2/tacotron2_uberduck_Noire.pt" -model = load_model(hparams) -#model.load_state_dict(torch.load(checkpoint_path, map_location=torch.device('cpu'))['state_dict']) -model.load_state_dict(torch.load(checkpoint_path)['state_dict']) -#model.load_state_dict(torch.jit.load(checkpoint_path, map_location=torch.device('cpu'))['state_dict']) -#_ = model.cuda().eval().half() -#_ = model.eval().half() +# Load synthesizer +checkpoint_path = args.synthesizer_checkpoint_path +model = load_model(hparams) +model.load_state_dict(torch.load(checkpoint_path)['state_dict']) # NB(bt, 2021-05-31): Trying to get everything on the same device device = "cuda:0" model = model.to(device) - _ = model.cuda().eval() -# 5 -#waveglow_path = '/home/bt/models/waveglow_256channels.pt' -waveglow_path = '/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt' -waveglow_path = '/home/bt/models/waveglow/waveglow_256channels_v4_uberduck.pt' +# Load vocoder +waveglow_path = args.vocoder_checkpoint_path + waveglow = torch.load(waveglow_path)['model'] waveglow.cuda().eval().half() #for k in waveglow.convinv: # k.float() #denoiser = Denoiser(waveglow) -# 6 -print("\n\n===TEXT===\n") -text = "This is the song that never ends. It goes on and on my friend." -#text = "hi you" -print("Text: {}".format(text)) -print("Text len: {}".format(len(text))) +text = open(args.input_text_filename, 'r').read() seq = text_to_sequence(text, ['english_cleaners']) -print("Seq: {}".format(type(seq))) -print("Seq: {}".format(seq)) + sequence = np.array(seq)[None, :] + print("Sequence np.array: {}".format(type(sequence))) print("Sequence np.array: {}".format(sequence)) + sequence = torch.from_numpy(sequence) + print("Sequence from_numpy: {}".format(type(sequence))) print("Sequence from_numpy: {}".format(sequence)) print("Sequence from_numpy dtype: {}".format(sequence.dtype)) + sequence = torch.autograd.Variable(sequence).long() - #torch.from_numpy(sequence)).cuda().long() - #torch.from_numpy(sequence)) print("Sequence torch autograd: {}".format(sequence)) -# NB(bt, 2021-05-31): This works. Don't need to do it. -#print('Saving text sequence tensor') -#torch.save(sequence, 'text_sequence.pt') - -print("\n\n===END TEXT===\n") - - # NB(bt, 2021-05-31): Trying to get everything to the same device sequence = sequence.to(device) -# 7 mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence) -#plot_data((mel_outputs.float().data.cpu().numpy()[0], -# mel_outputs_postnet.float().data.cpu().numpy()[0], -# alignments.float().data.cpu().numpy()[0].T)) -# Per melgan preprocess.py, need to output: -# mel_output: torch.FloatTensor of shape (B, n_mel_channels, T) print("Mel outputs:") print(mel_outputs) print(mel_outputs.shape) @@ -128,24 +98,33 @@ print(mel_outputs_postnet) print(mel_outputs_postnet.shape) print('Saving mels') -torch.save(mel_outputs, 'mel_outputs.mel') -torch.save(mel_outputs_postnet, 'mel_outputs_postnet.mel') + +class NumpyEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, np.ndarray): + return obj.tolist() + return json.JSONEncoder.default(self, obj) + +#json_dump = json.dumps({'mel': mel_outputs, 'mel_postnet': mel_outputs_postnet}, cls=NumpyEncoder) +#with open(args.output_spectrogram_filename, 'w') as outfile: +# json.dump(data, outfile) + +#torch.save(mel_outputs, 'mel_outputs.mel') +#torch.save(mel_outputs_postnet, 'mel_outputs_postnet.mel') #print('Rendering histograms') #render_histogram(mel_outputs, 'mel_outputs.png') #render_histogram(mel_outputs_postnet, 'mel_outputs_postnet.png') -# 8 with torch.no_grad(): audio = waveglow.infer(mel_outputs_postnet, sigma=0.666) pass - #ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate) # https://pytorch.org/hub/nvidia_deeplearningexamples_waveglow/ audio_numpy = audio[0].data.cpu().numpy() rate = 22050 from scipy.io.wavfile import write -write("audio.wav", rate, audio_numpy) +write(args.output_audio_filename, rate, audio_numpy) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 6d649b0943..b9c06a2f74 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -53,18 +53,16 @@ const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; const ENV_REGION_NAME : &'static str = "REGION_NAME"; -// Buckets (private data) -const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; -// Buckets (public data) -const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; +// Bucket names +const ENV_PRIVATE_BUCKET_NAME : &'static str = "PRIVATE_BUCKET_NAME"; +const ENV_PUBLIC_BUCKET_NAME : &'static str = "PUBLIC_BUCKET_NAME"; // Where models and other assets get downloaded to. const ENV_SEMIPERSISTENT_CACHE_DIR : &'static str = "SEMIPERSISTENT_CACHE_DIR"; // Python code -const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; -const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; -const ENV_INFERENCE_SCRIPT_NAME : &'static str = "W2L_INFERENCE_SCRIPT_NAME"; +const ENV_CODE_DIRECTORY : &'static str = "TTS_CODE_DIRECTORY"; +const ENV_INFERENCE_SCRIPT_NAME : &'static str = "TTS_INFERENCE_SCRIPT_NAME"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; @@ -130,7 +128,6 @@ async fn main() -> AnyhowResult<()> { let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?; let py_script_name = easyenv::get_env_string_required(ENV_INFERENCE_SCRIPT_NAME)?; - //let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?; let tts_inference_command = TacotronInferenceCommand::new( &py_code_directory, From 22de06afe06ae6eb725258e34d7d1b156ab0b14b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 1 Jun 2021 16:23:06 -0400 Subject: [PATCH 0259/1042] fixup local env vars --- .env | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.env b/.env index e0c2972a6a..10d9479644 100644 --- a/.env +++ b/.env @@ -20,17 +20,23 @@ TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models W2L_PRIVATE_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public +PRIVATE_BUCKET_NAME=dev-vocodes-private-uploads +PUBLIC_BUCKET_NAME=dev-vocodes-public + # Bucket roots AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates W2L_INFERENCE_OUTPUT_BUCKET_ROOT=/w2l_output W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip +W2L_INFERENCE_SCRIPT_NAME=vocodes_process_inference.py +W2L_MODEL_FILENAME=wav2lip_gan.pth + #W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/s3fd-619a316812.pth W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/wav2lip_gan.pth -W2L_MODEL_FILENAME=wav2lip_gan.pth W2L_END_BUMP_FILENAME=vocodes-short-end-bump.mp4 W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py -W2L_INFERENCE_SCRIPT_NAME=vocodes_process_inference.py +TTS_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/tacotron +TTS_INFERENCE_SCRIPT_NAME=vocodes_inference.py From dadb26c5f24a9fdad6ee08f51a82d0ac11d0cab2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 00:28:54 -0400 Subject: [PATCH 0260/1042] path bugfixing (still not working) --- .env | 4 +++- models/tacotron2/vocodes_inference.py | 4 +++- src/buckets/bucket_client.rs | 23 ++++++++++------------- src/tts_download_job.rs | 22 +++++++++++++++------- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/.env b/.env index 10d9479644..10e5978074 100644 --- a/.env +++ b/.env @@ -38,5 +38,7 @@ W2L_END_BUMP_FILENAME=vocodes-short-end-bump.mp4 W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py -TTS_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/tacotron +TTS_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/tacotron2 TTS_INFERENCE_SCRIPT_NAME=vocodes_inference.py + +TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt \ No newline at end of file diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py index 8c54e8fac6..f9fcdfc9fe 100755 --- a/models/tacotron2/vocodes_inference.py +++ b/models/tacotron2/vocodes_inference.py @@ -48,9 +48,11 @@ hparams.sampling_rate = 22050 torch.set_default_tensor_type('torch.HalfTensor') # 2021-05-31 added # Load synthesizer -checkpoint_path = args.synthesizer_checkpoint_path +checkpoint_path = args.synthesizer_checkpoint_path model = load_model(hparams) + +print('Loading synthesizer at path: {}'.format(checkpoint_path)) model.load_state_dict(torch.load(checkpoint_path)['state_dict']) # NB(bt, 2021-05-31): Trying to get everything on the same device diff --git a/src/buckets/bucket_client.rs b/src/buckets/bucket_client.rs index 912d0fcc60..2b5ac5950c 100644 --- a/src/buckets/bucket_client.rs +++ b/src/buckets/bucket_client.rs @@ -108,18 +108,15 @@ impl BucketClient { // Ok(()) // } - pub async fn upload_filename(&self, object_name: &str, filename: &Path) -> anyhow::Result<()> { - /*let mut file = File::open(filename).await?; - let mut reader = BufReader::new(file); - - info!("Uploading..."); - - let code = self.bucket.put_object_stream(&mut reader, object_name).await?; - - info!("upload code: {}", code); - - Ok(()) - */ + pub async fn upload_filename>( + &self, + object_path: P, + filename: P + ) -> anyhow::Result<()> { + let object_path_str = object_path.as_ref() + .to_str() + .map(|s| s.to_string()) + .ok_or(anyhow!("could not convert object path to string"))?; // TODO: does a newer version of this crate handle streaming/buffering file contents? let mut file = File::open(filename).await?; @@ -128,7 +125,7 @@ impl BucketClient { info!("Uploading..."); - self.upload_file(object_name, &buffer).await + self.upload_file(&object_path_str, &buffer).await } pub async fn upload_filename_with_content_type>( diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 1436b02f41..0e3ecfa0fb 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -18,6 +18,7 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; use crate::buckets::bucket_client::BucketClient; +use crate::buckets::bucket_path_unifier::BucketPathUnifier; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::buckets::file_hashing::get_file_hash; use crate::common_queries::firehose_publisher::FirehosePublisher; @@ -59,6 +60,9 @@ struct Downloader { pub bucket_client: BucketClient, pub firehose_publisher: FirehosePublisher, pub google_drive_downloader: GoogleDriveDownloadCommand, + + pub bucket_path_unifier: BucketPathUnifier, + // Command to run pub download_script: String, // Root to store TTS results @@ -131,6 +135,7 @@ async fn main() -> AnyhowResult<()> { bucket_client, download_script, google_drive_downloader, + bucket_path_unifier: BucketPathUnifier::default_paths(), bucket_root_tts_model_uploads: bucket_root.to_string(), firehose_publisher, }; @@ -222,22 +227,25 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("File hash: {}", private_bucket_hash); - // NB: /.../a/b/c/d/abcdefg.bin - let object_name = hash_to_bucket_path( - &private_bucket_hash, - Some(&downloader.bucket_root_tts_model_uploads))?; + //// NB: /.../a/b/c/d/abcdefg.bin + //let object_name = hash_to_bucket_path( + // &private_bucket_hash, + // Some(&downloader.bucket_root_tts_model_uploads))?; - info!("Destination bucket path: {}", object_name); + let synthesizer_model_bucket_path = downloader.bucket_path_unifier.tts_synthesizer_path( + &private_bucket_hash); + + info!("Destination bucket path: {:?}", &synthesizer_model_bucket_path); let file_path = PathBuf::from(download_filename); - downloader.bucket_client.upload_filename(&object_name, &file_path).await?; + downloader.bucket_client.upload_filename(&synthesizer_model_bucket_path, &file_path).await?; info!("Saving model record..."); let (id, model_token) = insert_tts_model( &downloader.mysql_pool, job, &private_bucket_hash, - &object_name) + synthesizer_model_bucket_path.as_path().to_str().unwrap_or("")) .await?; info!("Saved model record: {}", id); From 7003e5431e5abd0d927e43c53c33307ec6c53186 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 00:48:42 -0400 Subject: [PATCH 0261/1042] better, but still not working --- src/job_queries/tts_inference_job_queries.rs | 1 + src/tts_download_job.rs | 12 ++++++++++++ src/tts_inference_job.rs | 11 +++++++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 1f201a78e9..465956de57 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -167,6 +167,7 @@ SET public_bucket_hash = 'TODO', public_bucket_wav_audio_path = ?, public_bucket_spectrogram_path = ?, + inference_public_bucket_uuid = 'TODO', file_size_bytes = ?, duration_millis = ? diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 0e3ecfa0fb..994881303a 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -232,6 +232,18 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho // &private_bucket_hash, // Some(&downloader.bucket_root_tts_model_uploads))?; + // [2021-06-02T04:38:28Z INFO tts_download_job::buckets::bucket_client] Uploading... + // [2021-06-02T04:38:28Z INFO tts_download_job::buckets::bucket_client] + // Filename for bucket: /user_uploaded_tts_synthesizers/9/5/7/957dfe3f2fcf36ebcf13a6e3bc52fb2905ccd7b1834076bd6c3c41ace272640a.pt + // [2021-06-02T04:38:28Z INFO tts_download_job::buckets::bucket_client] + // Rooted filename for bucket: /user_uploaded_tts_synthesizers/9/5/7/957dfe3f2fcf36ebcf13a6e3bc52fb2905ccd7b1834076bd6c3c41ace272640a.pt + + // 2021-06-02T04:40:24Z INFO tts_inference_job] TTS synthesizer model file does not exist: + // "/tmp/tts/synthesizer_models/TTS_MDL:BSBQ48NWNFJFH5PSD3MCSCF2" + // [2021-06-02T04:40:24Z INFO tts_inference_job] + // Download from template media path: "/user_uploaded_tts_synthesizers/9/5/7/957dfe3f2fcf36ebcf13a6e3bc52fb2905ccd7b1834076bd6c3c41ace272640a.pt" + // [2021-06-02T04:40:24Z INFO tts_inference_job::buckets::bucket_client] downloading from bucket: + // "/user_uploaded_tts_synthesizers/9/5/7/957dfe3f2fcf36ebcf13a6e3bc52fb2905ccd7b1834076bd6c3c41ace272640a.pt" let synthesizer_model_bucket_path = downloader.bucket_path_unifier.tts_synthesizer_path( &private_bucket_hash); diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index b9c06a2f74..aee334f265 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -261,7 +261,7 @@ async fn process_jobs(inferencer: &Inferencer, jobs: Vec) Ok(()) } -#[derive(Deserialize)] +#[derive(Deserialize, Default)] struct FileMetadata { pub duration_millis: Option, pub mimetype: Option, @@ -389,10 +389,11 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An info!("Checking that output files exist..."); check_file_exists(&output_audio_fs_path)?; - check_file_exists(&output_spectrogram_fs_path)?; - check_file_exists(&output_metadata_fs_path)?; + //check_file_exists(&output_spectrogram_fs_path)?; + //check_file_exists(&output_metadata_fs_path)?; - let file_metadata = read_metadata_file(&output_metadata_fs_path)?; + let file_metadata = FileMetadata::default(); + //let file_metadata = read_metadata_file(&output_metadata_fs_path)?; // ==================== UPLOAD AUDIO TO BUCKET ==================== // @@ -414,6 +415,7 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An let spectrogram_result_object_path = inferencer.bucket_path_unifier.tts_inference_spectrogram_output_path( &job.inference_job_token); + /* info!("Spectrogram destination bucket path: {:?}", &spectrogram_result_object_path); info!("Uploading spectrogram..."); @@ -423,6 +425,7 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An &output_spectrogram_fs_path, "application/json") .await?; + */ // ==================== SAVE RECORDS ==================== // From 3f1a5fa7297a66fea3b6d31015b060b13dd6bcc7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 00:55:28 -0400 Subject: [PATCH 0262/1042] hack, kind of works. Wav doesn't play in firefox, vlc. Only asound. It sounds awful. Need to fix paths and so many other things --- src/tts_inference_job.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index aee334f265..7e8895340c 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -398,7 +398,7 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An // ==================== UPLOAD AUDIO TO BUCKET ==================== // let audio_result_object_path = inferencer.bucket_path_unifier.tts_inference_wav_audio_output_path( - &job.inference_job_token); + &job.uuid_idempotency_token); // TODO: Don't use this! info!("Audio destination bucket path: {:?}", &audio_result_object_path); From ef994670f201261698d63090267dbb824be1afd5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 20:59:40 -0400 Subject: [PATCH 0263/1042] fixed bugs introduced --- src/buckets/bucket_client.rs | 12 ++++++------ src/tts_download_job.rs | 4 ++-- src/w2l_download_job.rs | 11 ++++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/buckets/bucket_client.rs b/src/buckets/bucket_client.rs index 2b5ac5950c..a137184d5a 100644 --- a/src/buckets/bucket_client.rs +++ b/src/buckets/bucket_client.rs @@ -108,10 +108,10 @@ impl BucketClient { // Ok(()) // } - pub async fn upload_filename>( + pub async fn upload_filename, Q: AsRef>( &self, object_path: P, - filename: P + filename: Q ) -> anyhow::Result<()> { let object_path_str = object_path.as_ref() .to_str() @@ -128,10 +128,10 @@ impl BucketClient { self.upload_file(&object_path_str, &buffer).await } - pub async fn upload_filename_with_content_type>( + pub async fn upload_filename_with_content_type, Q: AsRef>( &self, object_path: P, - filename: &Path, + filename: Q, content_type: &str ) -> anyhow::Result<()> { let object_path_str = object_path.as_ref() @@ -172,10 +172,10 @@ impl BucketClient { Ok(bytes) } - pub async fn download_file_to_disk>( + pub async fn download_file_to_disk, Q: AsRef>( &self, object_path: P, - filesystem_path: P, + filesystem_path: Q, ) -> anyhow::Result<()> { let object_path_str = object_path.as_ref() .to_str() diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index f0458614a3..13e03a1e0c 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -20,7 +20,6 @@ use chrono::Utc; use crate::buckets::bucket_client::BucketClient; use crate::buckets::bucket_path_unifier::BucketPathUnifier; use crate::buckets::bucket_paths::hash_to_bucket_path; -use w2l_inference_job::util::hashing::file_hashing::get_file_hash; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; use crate::job_queries::tts_download_job_queries::insert_tts_model; @@ -30,6 +29,7 @@ use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; +use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; @@ -223,7 +223,7 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho let download_filename = downloader.google_drive_downloader .download_file(&download_url, &temp_dir).await?; - let private_bucket_hash = get_file_hash(&download_filename)?; + let private_bucket_hash = hash_file_sha2(&download_filename)?; info!("File hash: {}", private_bucket_hash); diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index d9fa0dff2b..93f14e6b5a 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -18,7 +18,6 @@ use anyhow::anyhow; use chrono::Utc; use crate::buckets::bucket_client::BucketClient; use crate::buckets::bucket_paths::hash_to_bucket_path; -use w2l_inference_job::util::hashing::file_hashing::get_file_hash; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; use crate::job_queries::w2l_download_job_queries::insert_w2l_template; @@ -33,6 +32,7 @@ use crate::script_execution::wav2lip_process_upload_command::Wav2LipPreprocessCl use crate::util::anyhow_result::AnyhowResult; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; +use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; @@ -347,8 +347,8 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // let video_or_image_path = PathBuf::from(&download_filename); - let cached_faces_path = &PathBuf::from(&cached_faces_filename); - let output_metadata_path = &PathBuf::from(&output_metadata_filename); + let cached_faces_path = PathBuf::from(&cached_faces_filename); + let output_metadata_path = PathBuf::from(&output_metadata_filename); info!("Checking that both files exist (original source + cached faces) ..."); @@ -360,7 +360,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) // ==================== BASE OBJECT NAMES BASED ON HASH ==================== // - let private_bucket_hash = get_file_hash(&download_filename)?; + let private_bucket_hash = hash_file_sha2(&download_filename)?; info!("File hash: {}", private_bucket_hash); @@ -441,9 +441,10 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) original_mime_type).await?; info!("Uploading cached faces..."); + let path_copy: PathBuf = cached_faces_path.clone(); downloader.private_bucket_client.upload_filename( &full_object_path_cached_faces, - &cached_faces_path).await?; + &path_copy).await?; // TODO: Fix this ugh. if let Some(image_preview_filename) = maybe_image_preview_filename.as_deref() { From 9d46818a564e6c19f8173729e3b62ea8f49d4b32 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 21:06:30 -0400 Subject: [PATCH 0264/1042] fixup --- debug/curl/seed.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/curl/seed.py b/debug/curl/seed.py index 3f629bf8d5..074759f9bf 100755 --- a/debug/curl/seed.py +++ b/debug/curl/seed.py @@ -28,11 +28,12 @@ def create_user_account(username, password, email): r = requests.post(signup_url, json=payload) print_response(r) -create_user_account('brandon', 'testing', 'brandon@storyteller.company') create_user_account('echelon', 'testing', 'echelon@gmail.com') +create_user_account('brandon', 'testing', 'brandon@storyteller.company') create_user_account('nanashi', 'nanashi', 'somebody@somewhere.com') -create_user_account('ryu', 'testing', 'ryu@storyteller.company') +create_user_account('other', 'testing', 'ryu@storyteller.company') create_user_account('raptor9', 'testing', 'raptor9@storyteller.company') +create_user_account('ryu', 'testing', 'ryu@storyteller.company') # ==================== LOGIN ==================== From ae91e19b9f9a5dda3f138316f309157888ad665b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 21:15:33 -0400 Subject: [PATCH 0265/1042] fixup filename, debugging --- models/tacotron2/vocodes_inference.py | 11 +++++++++++ src/tts_inference_job.rs | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py index f9fcdfc9fe..b9d340c9cb 100755 --- a/models/tacotron2/vocodes_inference.py +++ b/models/tacotron2/vocodes_inference.py @@ -22,6 +22,7 @@ import sys sys.path.append('waveglow/') import numpy as np import torch +import shutil import json from hparams import create_hparams @@ -130,3 +131,13 @@ rate = 22050 from scipy.io.wavfile import write write(args.output_audio_filename, rate, audio_numpy) +try: + debug_location = "/home/bt/dev/storyteller/storyteller-web/test" + print('Writing to debug location: {}'.format(debug_location)) + shutil.copy(args.output_audio_filename, debug_location) + +except Exception as e: + print("Error with saving file for local debugging.") + print(e) + pass + diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 5342ac426a..f29682095c 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -365,7 +365,7 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An // ==================== RUN INFERENCE ==================== // - let output_audio_fs_path = temp_dir.path().join("output.mp4"); + let output_audio_fs_path = temp_dir.path().join("output.wav"); let output_metadata_fs_path = temp_dir.path().join("metadata.json"); let output_spectrogram_fs_path = temp_dir.path().join("spectrogram.json"); From 077a4239d8877c8c8acf374288d9bceaa436070a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 21:53:30 -0400 Subject: [PATCH 0266/1042] sounds better, but still bad --- models/tacotron2/vocodes_inference.py | 97 +++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 12 deletions(-) diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py index b9d340c9cb..3e9e451847 100755 --- a/models/tacotron2/vocodes_inference.py +++ b/models/tacotron2/vocodes_inference.py @@ -25,9 +25,11 @@ import torch import shutil import json +from model import Tacotron2 from hparams import create_hparams from train import load_model from text import text_to_sequence +from denoiser import Denoiser print('========================================') print('Python interpreter', sys.executable) @@ -37,41 +39,54 @@ print('CUDA Device count', torch.cuda.device_count()) print('========================================', flush=True) # NB(bt, 2021-05-31): Trying to get everything on the same device -torch.device("cuda" if torch.cuda.is_available() else "cpu") +#torch.device("cuda" if torch.cuda.is_available() else "cpu") hparams = create_hparams() hparams.sampling_rate = 22050 +hparams.sampling_rate = 22050 # Don't change this +hparams.max_decoder_steps = 1000 # How long the audio will be before it cuts off (1000 is about 11 seconds) +hparams.gate_threshold = 0.1 # Model must be 90% sure the clip is over before ending generation (the higher this number is, the more likely that the AI will keep generating until it reaches the Max Decoder Steps) # Fix bug running on CPU? #torch.set_default_dtype(torch.float16) # NB(bt, 2021-05-31): Perhaps this is blocking waveglow? It complains of the wrong tensor type. #torch.set_default_tensor_type('torch.DoubleTensor') # 2021-05-31 commented out -torch.set_default_tensor_type('torch.HalfTensor') # 2021-05-31 added +#torch.set_default_tensor_type('torch.HalfTensor') # 2021-05-31 added # Load synthesizer checkpoint_path = args.synthesizer_checkpoint_path -model = load_model(hparams) + +#model = load_model(hparams) +model = Tacotron2(hparams) print('Loading synthesizer at path: {}'.format(checkpoint_path)) model.load_state_dict(torch.load(checkpoint_path)['state_dict']) +_ = model.cuda().eval().half() + # NB(bt, 2021-05-31): Trying to get everything on the same device device = "cuda:0" model = model.to(device) - -_ = model.cuda().eval() +#_ = model.cuda().eval() # Load vocoder waveglow_path = args.vocoder_checkpoint_path -waveglow = torch.load(waveglow_path)['model'] -waveglow.cuda().eval().half() #for k in waveglow.convinv: # k.float() #denoiser = Denoiser(waveglow) + +waveglow = torch.load(waveglow_path)['model'] +waveglow.cuda().eval().half() +for k in waveglow.convinv: + k.float() +denoiser = Denoiser(waveglow) + text = open(args.input_text_filename, 'r').read() + +""" seq = text_to_sequence(text, ['english_cleaners']) sequence = np.array(seq)[None, :] @@ -91,8 +106,32 @@ print("Sequence torch autograd: {}".format(sequence)) # NB(bt, 2021-05-31): Trying to get everything to the same device sequence = sequence.to(device) +""" -mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence) + + +raw_input = True +sequence = None + +for i in text.split("\n"): + if len(i) < 1: + continue + print(i) + if raw_input: + if i[-1] != ";": + i = i+";" + else: + #i = ARPA(i) + pass + print(i) + with torch.no_grad(): + # save VRAM by not including gradients + sequence = np.array(text_to_sequence(i, ['english_cleaners']))[None, :] + sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long() + break # TODO: Handle multiple lines. + +with torch.no_grad(): + mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence) print("Mel outputs:") print(mel_outputs) @@ -120,16 +159,22 @@ class NumpyEncoder(json.JSONEncoder): #render_histogram(mel_outputs_postnet, 'mel_outputs_postnet.png') with torch.no_grad(): - audio = waveglow.infer(mel_outputs_postnet, sigma=0.666) - pass + # This was from Tacotron 2's repo: + #audio = waveglow.infer(mel_outputs_postnet, sigma=0.666) + + # This is from the colab: + sigma = 0.8 + audio = waveglow.infer(mel_outputs_postnet, sigma=sigma) + #print("") + #ipd.display(ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate)) #ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate) # https://pytorch.org/hub/nvidia_deeplearningexamples_waveglow/ audio_numpy = audio[0].data.cpu().numpy() -rate = 22050 +#rate = 22050 from scipy.io.wavfile import write -write(args.output_audio_filename, rate, audio_numpy) +write(args.output_audio_filename, hparams.sampling_rate, audio_numpy) try: debug_location = "/home/bt/dev/storyteller/storyteller-web/test" @@ -141,3 +186,31 @@ except Exception as e: print(e) pass +""" +#@markdown # **Synthesis** +#@markdown # Enter your desired text here, the graph and audio will show up. + +text = "And so he ran across the street to measure the absence of being." #@param {type:"string"} +sigma = 0.8 +denoise_strength = 0.324 + +#@markdown #### Optional if you want automatic ARPA convertion. + +raw_input = True #@param {type:"boolean"} + +for i in text.split("\n"): + if len(i) < 1: + sys.exit() + print(i) + if raw_input: + if i[-1] != ";": i=i+";" + else: + i = ARPA(i) + print(i) + with torch.no_grad(): # save VRAM by not including gradients + sequence = np.array(text_to_sequence(i, ['english_cleaners']))[None, :] + sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long() + mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence) + plot_data((mel_outputs_postnet.float().data.cpu().numpy()[0], + alignments.float().data.cpu().numpy()[0].T)) +""" \ No newline at end of file From 7744f0e3240dce2e1b2e7d16cfaf2556830d49fc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 22:17:27 -0400 Subject: [PATCH 0267/1042] working on adapting the waveglow/inference to sound non-noisy --- .env | 4 +++- debug/curl/seed.py | 2 ++ models/tacotron2/vocodes_inference.py | 22 +++++++++++----------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.env b/.env index 10e5978074..1a588dc7a5 100644 --- a/.env +++ b/.env @@ -41,4 +41,6 @@ W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py TTS_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/tacotron2 TTS_INFERENCE_SCRIPT_NAME=vocodes_inference.py -TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt \ No newline at end of file +#TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt # THIS SOUNDS HORRID +#TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! +TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_v4.pt diff --git a/debug/curl/seed.py b/debug/curl/seed.py index 074759f9bf..a04896e040 100755 --- a/debug/curl/seed.py +++ b/debug/curl/seed.py @@ -114,6 +114,8 @@ upload_tts_model('https://drive.google.com/file/d/1atFn4vm7mjkufXNK9TZS6J8WzISrA upload_tts_model('https://drive.google.com/file/d/1-rlJpi2jAsagn-UCStcCc_ITNycXgn2F/view?usp=sharing', 'Jorgen', session_other) upload_tts_model('https://drive.google.com/file/d/1togTNOEKnaarqWWrbZLSwIbZIZ8IT_qw/view?usp=sharing', 'edsheeransinging', session_nanashi) +upload_tts_model('https://drive.google.com/file/d/1OAiu0cZDyUJMemyLc0Btqw7zEQgiN81D/view?usp=sharing', 'Wakko Warner v2 ST21', session_nanashi) + #upload_tts_model('', '', session_other) #upload_tts_model('', '', session_other) #upload_tts_model('', '', session_other) diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py index 3e9e451847..8e4487fd7a 100755 --- a/models/tacotron2/vocodes_inference.py +++ b/models/tacotron2/vocodes_inference.py @@ -20,14 +20,16 @@ args = parser.parse_args() import sys sys.path.append('waveglow/') -import numpy as np -import torch import shutil import json -from model import Tacotron2 +import numpy as np +import torch + from hparams import create_hparams -from train import load_model +from model import Tacotron2 +from layers import TacotronSTFT +from audio_processing import griffin_lim from text import text_to_sequence from denoiser import Denoiser @@ -41,20 +43,18 @@ print('========================================', flush=True) # NB(bt, 2021-05-31): Trying to get everything on the same device #torch.device("cuda" if torch.cuda.is_available() else "cpu") -hparams = create_hparams() -hparams.sampling_rate = 22050 -hparams.sampling_rate = 22050 # Don't change this -hparams.max_decoder_steps = 1000 # How long the audio will be before it cuts off (1000 is about 11 seconds) -hparams.gate_threshold = 0.1 # Model must be 90% sure the clip is over before ending generation (the higher this number is, the more likely that the AI will keep generating until it reaches the Max Decoder Steps) - # Fix bug running on CPU? #torch.set_default_dtype(torch.float16) # NB(bt, 2021-05-31): Perhaps this is blocking waveglow? It complains of the wrong tensor type. #torch.set_default_tensor_type('torch.DoubleTensor') # 2021-05-31 commented out #torch.set_default_tensor_type('torch.HalfTensor') # 2021-05-31 added -# Load synthesizer +hparams = create_hparams() +hparams.sampling_rate = 22050 # Don't change this +hparams.max_decoder_steps = 1000 # How long the audio will be before it cuts off (1000 is about 11 seconds) +hparams.gate_threshold = 0.1 # Model must be 90% sure the clip is over before ending generation (the higher this number is, the more likely that the AI will keep generating until it reaches the Max Decoder Steps) +# Load synthesizer checkpoint_path = args.synthesizer_checkpoint_path #model = load_model(hparams) From a1c77fb5279c2abdbac3a01bb61b423036df85c5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 22:26:55 -0400 Subject: [PATCH 0268/1042] Copying waveglow files @ 2fd4e63 (per collab synthesis.ipynb) --- models/tacotron2/waveglow/README.md | 4 ++-- models/tacotron2/waveglow/convert_model.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/models/tacotron2/waveglow/README.md b/models/tacotron2/waveglow/README.md index b94648cf8e..89886f00a6 100755 --- a/models/tacotron2/waveglow/README.md +++ b/models/tacotron2/waveglow/README.md @@ -12,7 +12,7 @@ using only a single network, trained using only a single cost function: maximizing the likelihood of the training data, which makes the training procedure simple and stable. -Our [PyTorch] implementation produces audio samples at a rate of 1200 +Our [PyTorch] implementation produces audio samples at a rate of 4850 kHz on an NVIDIA V100 GPU. Mean Opinion Scores show that it delivers audio quality as good as the best publicly available WaveNet implementation. @@ -85,7 +85,7 @@ with fused residual and skip connections. [Glow]: https://blog.openai.com/glow/ [WaveNet]: https://deepmind.com/blog/wavenet-generative-model-raw-audio/ [PyTorch]: http://pytorch.org -[published model]: https://drive.google.com/open?id=1rpK8CzAAirq9sWZhe9nlfvxMF1dRgFbF +[published model]: https://ngc.nvidia.com/catalog/models/nvidia:waveglow_ljs_256channels [mel-spectrograms]: https://drive.google.com/file/d/1g_VXK2lpP9J25dQFhQwx7doWl_p20fXA/view?usp=sharing [LJ Speech Data]: https://keithito.com/LJ-Speech-Dataset [Apex]: https://github.com/nvidia/apex diff --git a/models/tacotron2/waveglow/convert_model.py b/models/tacotron2/waveglow/convert_model.py index 6fdc41e871..32e77e2b20 100644 --- a/models/tacotron2/waveglow/convert_model.py +++ b/models/tacotron2/waveglow/convert_model.py @@ -61,15 +61,12 @@ def update_model(old_model): _update_model_res_skip(old_model, new_model) if hasattr(old_model.WN[0], 'cond_layers'): _update_model_cond(old_model, new_model) - for m in new_model.modules(): - if 'Conv' in str(type(m)) and not hasattr(m, 'padding_mode'): - setattr(m, 'padding_mode', 'zeros') return new_model if __name__ == '__main__': old_model_path = sys.argv[1] new_model_path = sys.argv[2] - model = torch.load(old_model_path, map_location='cpu') + model = torch.load(old_model_path) model['model'] = update_model(model['model']) torch.save(model, new_model_path) From 716a75c4f4603956d5ba811c05f180e074a2abd2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 22:35:08 -0400 Subject: [PATCH 0269/1042] THAT DID THE TRICK! I was using the wrong wav encoding! --- models/tacotron2/vocodes_inference.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py index 8e4487fd7a..e4e8e50ea2 100755 --- a/models/tacotron2/vocodes_inference.py +++ b/models/tacotron2/vocodes_inference.py @@ -171,10 +171,19 @@ with torch.no_grad(): #ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate) # https://pytorch.org/hub/nvidia_deeplearningexamples_waveglow/ -audio_numpy = audio[0].data.cpu().numpy() -#rate = 22050 + + +#audio_numpy = audio[0].data.cpu().numpy() +#audio = audio[0].data.cpu().numpy() +#audio_numpy = audio.astype('int16') + from scipy.io.wavfile import write -write(args.output_audio_filename, hparams.sampling_rate, audio_numpy) + +output_audio = audio[0].data.cpu().numpy().astype(np.float32) + +#audio.export('output.mp3', format="mp3", bitrate="64k") +#sf.write('bira.wav', audio[0].data.cpu().numpy().astype(np.float32), hparams.sampling_rate) +write(args.output_audio_filename, hparams.sampling_rate, output_audio) try: debug_location = "/home/bt/dev/storyteller/storyteller-web/test" From 71a19e4794905f9d7b9e7d2378703caeaf22f308 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 23:22:04 -0400 Subject: [PATCH 0270/1042] re-codegen db --- sqlx-data.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 3ba5b0b550..520018d8a6 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1496,16 +1496,6 @@ "nullable": [] } }, - "7529fc02fff368b36f2dc630ec3c041384fb3d0ce8f87a8fbbd7ce06dc7f43bb": { - "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n model_token = ?,\n inference_text = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_hash = 'TODO',\n public_bucket_audio_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n duration_millis = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 9 - }, - "nullable": [] - } - }, "7ab54466ca3021d90e0feb234acba56bf53d415dde69bc1c35120ed0c8dd1858": { "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { @@ -2189,6 +2179,16 @@ "nullable": [] } }, + "9e2a2897058a40d0b27b3afddf515fd9e458a7bec824de902a673362617e8987": { + "query": "\nINSERT INTO tts_results\nSET\n token = ?,\n\n model_token = ?,\n inference_text = ?,\n inference_text_hash_sha2 = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_hash = 'TODO',\n public_bucket_wav_audio_path = ?,\n public_bucket_spectrogram_path = ?,\n inference_public_bucket_uuid = 'TODO',\n\n file_size_bytes = ?,\n duration_millis = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, "ab78a81f85310750668e2c7de920c067290f2b004561ab706a5edfd893fd06fe": { "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n maybe_public_bucket_preview_image_object_name = ?,\n maybe_public_bucket_preview_video_object_name = ?,\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", "describe": { From 57ae2343dd385ff4343cbc3206ddc3cc41c66d6c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Jun 2021 23:36:10 -0400 Subject: [PATCH 0271/1042] stupid, but makes me smile --- loc.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 loc.sh diff --git a/loc.sh b/loc.sh new file mode 100755 index 0000000000..dcc9b46613 --- /dev/null +++ b/loc.sh @@ -0,0 +1,10 @@ +#!/bin/bash +wc -l \ + Cargo.toml \ + Dockerfile \ + migrations/*/*.sql \ + src/*.rs \ + src/*/*.rs \ + src/*/*/*.rs \ + src/*/*/*/*.rs \ + src/*/*/*/*/*.rs From 7b833d7c66384d6772ae911a5285d65aaf19cc15 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Jun 2021 01:03:34 -0400 Subject: [PATCH 0272/1042] more reserved words --- db/reserved_usernames.txt | 176 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index 24860afab0..5d8d5357f8 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -337,12 +337,188 @@ diary dict dictionary die +item +thread +post +posts +threads +forum +forums +title +topic +key +value +tag +tags +category +categories +listen +view +watch +watches +playlist +playlists +video +videos +upload +download +video +about +story +staff +who +learnmore +pending +data +datas +datum +learn_more +help +mobile +topics +posts +reddit +blog +explore +firehose +ingestion +ingest +fire_hose +explorer +notifications +bookmarks +notification +blogs +career +careers +join +joinus +subscribe +privacy +policies +privacypolicy +gdpr +login +logout +log_in +log_out +sign_in +sign_out +signin +signout +log_off +logoff +sign_off +signoff +subreddit +subreddits +reddits +threads +tweets +learn +investor +investors +aboutus +images +image +photo +photos +browser +arpabet +livestream +stream +streaming +view +live +plain +plaintext +edit +pfp +avatar +avatars +pfps +pvp +match +matches +matchmaking +versus +promote +ad +ads +analytics +dms +pms +inbox +outbox diff dir direct_messages directory discover +view +edit +share +ban +subscribe +list +delist +trash +recycle +send +receive +download +borrow +messages +edit +api +redo +document +video +audio +template +profile +sheet +music +ref +reference +picture +photo +id +browse +table +subscribe +ref +topic +topics +url +urls +share +shared +like +liked +follow +unfollow +followers +followed +favorites +save +saved +delete +saving +key +uuid +guid +href +src +source +unsubscribe +campaign +news +article +blog discussions +market +marketing display display_name displayname From 7904f6e56bbb6df55d9c9737f0db0453443c5b72 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Jun 2021 01:04:27 -0400 Subject: [PATCH 0273/1042] sorted, dupes removed --- db/reserved_usernames.txt | 255 ++++++++++++-------------------------- 1 file changed, 79 insertions(+), 176 deletions(-) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index 5d8d5357f8..dc507449f5 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -184,6 +184,7 @@ sindhi 5xx 7xx about +aboutus abuse access account @@ -218,12 +219,14 @@ app apps archive archives +arpabet article asct asset assets atom attributes +audio auth authentication autoconfig @@ -231,6 +234,7 @@ avatar avatars backup balancer-manager +ban bank banner banners @@ -244,11 +248,14 @@ board book bookmark bookmarks +borrow bot bots bounty branches broadcasthost +browse +browser bug bugs business @@ -320,9 +327,12 @@ customers dashboard dashboards data +datas +datum db default delete +delist demo design designer @@ -337,196 +347,22 @@ diary dict dictionary die -item -thread -post -posts -threads -forum -forums -title -topic -key -value -tag -tags -category -categories -listen -view -watch -watches -playlist -playlists -video -videos -upload -download -video -about -story -staff -who -learnmore -pending -data -datas -datum -learn_more -help -mobile -topics -posts -reddit -blog -explore -firehose -ingestion -ingest -fire_hose -explorer -notifications -bookmarks -notification -blogs -career -careers -join -joinus -subscribe -privacy -policies -privacypolicy -gdpr -login -logout -log_in -log_out -sign_in -sign_out -signin -signout -log_off -logoff -sign_off -signoff -subreddit -subreddits -reddits -threads -tweets -learn -investor -investors -aboutus -images -image -photo -photos -browser -arpabet -livestream -stream -streaming -view -live -plain -plaintext -edit -pfp -avatar -avatars -pfps -pvp -match -matches -matchmaking -versus -promote -ad -ads -analytics -dms -pms -inbox -outbox diff dir direct_messages directory discover -view -edit -share -ban -subscribe -list -delist -trash -recycle -send -receive -download -borrow -messages -edit -api -redo -document -video -audio -template -profile -sheet -music -ref -reference -picture -photo -id -browse -table -subscribe -ref -topic -topics -url -urls -share -shared -like -liked -follow -unfollow -followers -followed -favorites -save -saved -delete -saving -key -uuid -guid -href -src -source -unsubscribe -campaign -news -article -blog discussions -market -marketing display display_name displayname dist +dms dns doc docker docs +document documentation domain download @@ -552,6 +388,7 @@ event events exit explore +explorer export facebook false @@ -567,6 +404,8 @@ feedback feeds file files +fire_hose +firehose firewall first fixtures @@ -575,6 +414,7 @@ fleet fleets flog follow +followed followers following forgot @@ -596,6 +436,7 @@ gadgets game games garage +gdpr get ghost gift @@ -610,6 +451,7 @@ group groups guest guests +guid guide guides help @@ -624,6 +466,7 @@ hostname hovercards howto hpg +href html http httpd @@ -647,6 +490,8 @@ indice individual info information +ingest +ingestion inquiry instagram integration @@ -657,6 +502,7 @@ interfaces intranet introduction invalid-email-address +investor investors invitations invite @@ -676,11 +522,13 @@ javascript job jobs join +joinus journal journals js json jump +key keys keyserver knowledgebase @@ -694,15 +542,22 @@ launch layouts ldap-status learn +learn_more +learnmore legal library license +like +liked link links linux list +listen listings lists +live +livestream local localdomain localhost @@ -710,8 +565,10 @@ log log-in log-out log_in +log_off log_out login +logoff logos logout logs @@ -733,9 +590,13 @@ manager manual map maps +market marketing marketplace master +match +matches +matchmaking me media member @@ -824,6 +685,7 @@ organisations organization organizations orgs +outbox overview owner owners @@ -838,8 +700,11 @@ password patch payment payments +pending perl personal +pfp +pfps phone photo photoalbum @@ -850,11 +715,18 @@ phppgadmin phpredisadmin pic pics +picture ping +plain +plaintext plan plans +playlist +playlists plugin plugins +pms +policies policy pop pop3 @@ -882,11 +754,13 @@ profile project projects promo +promote pub public pulls purpose put +pvp pw python query @@ -895,11 +769,18 @@ ranking raw read readme +receive recent recommendations recruit recruitment +recycle +reddit +reddits redeem +redo +ref +reference register registration registry @@ -941,6 +822,7 @@ samples save save-net-neutrality saved +saving school scraping script @@ -965,14 +847,18 @@ sftp share shared shareholders +sheet shop show showcases sign-in sign-up sign_in +sign_off +sign_out sign_up signin +signoff signout signup site @@ -1012,12 +898,17 @@ storage store stores stories +story +stream +streaming style styleguide styles stylesheet stylesheets subdomain +subreddit +subreddits subscribe subscriptions suggest @@ -1032,6 +923,7 @@ sys sysadmin sysadministrator system +table tablet tablets tag @@ -1048,6 +940,7 @@ team teams tech telnet +template ten term terms @@ -1072,6 +965,7 @@ themes thread threads timeline +title tls tmp todo @@ -1087,6 +981,7 @@ tour train training translations +trash tree trending trends @@ -1095,6 +990,7 @@ tutorial tux tv tweet +tweets twitter type typescript @@ -1107,6 +1003,7 @@ upload uploads uptime url +urls usage usenet user @@ -1116,12 +1013,15 @@ usr usuario util uucp +uuid +value vanity vanity_url vanityurl vendas ver version +versus video videos view @@ -1131,6 +1031,7 @@ visualization vpn w watch +watches watching weather web @@ -1141,6 +1042,7 @@ webmaster website websites welcome +who widget widgets wiki @@ -1180,3 +1082,4 @@ yourdomain yourname yoursite yourusername + From ae32779c3567ceea57dbfc22de037e78171c2f30 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Jun 2021 01:06:22 -0400 Subject: [PATCH 0274/1042] reserve one-letters and a few more --- db/reserved_usernames.txt | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/db/reserved_usernames.txt b/db/reserved_usernames.txt index dc507449f5..7e4aa6687b 100644 --- a/db/reserved_usernames.txt +++ b/db/reserved_usernames.txt @@ -183,7 +183,9 @@ sindhi 511 5xx 7xx +a about +about_us aboutus abuse access @@ -232,6 +234,7 @@ authentication autoconfig avatar avatars +b backup balancer-manager ban @@ -324,6 +327,7 @@ css customer customer-stories customers +d dashboard dashboards data @@ -364,10 +368,12 @@ docker docs document documentation +documents domain download downloads downtime +e ecommerce edit editor @@ -390,6 +396,7 @@ exit explore explorer export +f facebook false faq @@ -431,6 +438,7 @@ friend friends ftp ftps +g gadget gadgets game @@ -454,6 +462,7 @@ guests guid guide guides +h help help-wanted home @@ -517,6 +526,7 @@ issues it item items +j java javascript job @@ -528,10 +538,12 @@ journals js json jump +k key keys keyserver knowledgebase +l lab labels labs @@ -627,6 +639,7 @@ musicas mx my mysql +n name named names @@ -668,6 +681,7 @@ ns7 ns8 ns9 null +o oauth oauth_clients offer @@ -689,6 +703,7 @@ outbox overview owner owners +p package page pager @@ -763,7 +778,9 @@ put pvp pw python +q query +r random ranking raw @@ -772,6 +789,9 @@ readme receive recent recommendations +recover +recover_email +recovery recruit recruitment recycle @@ -799,8 +819,10 @@ req request requests res +resend reset reset-password +reset_email reset_password resetpassword resource @@ -814,6 +836,7 @@ ruby rule rules rust +s sag sale sales @@ -923,6 +946,7 @@ sys sysadmin sysadministrator system +t table tablet tablets @@ -994,6 +1018,7 @@ tweets twitter type typescript +u undef unfollow unsubscribe @@ -1014,6 +1039,7 @@ usuario util uucp uuid +v value vanity vanity_url @@ -1069,17 +1095,21 @@ www8 www9 wwws wwww +x xfn xml xmpp xpg xxx +y yaml year yml you +your yourdomain yourname +yours yoursite yourusername - +z From a9aa0a5b50efddacc79b69f9ad54aa7bfa72af9e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Jun 2021 02:14:19 -0400 Subject: [PATCH 0275/1042] (BROKEN): WIP new endpoints for TTS models, results, etc. --- .../list_tts_inference_results.rs | 289 ++++++++++++++++++ src/common_queries/list_tts_models.rs | 249 +++++++++++++++ src/common_queries/mod.rs | 2 + .../endpoints/tts/get_tts_model.rs | 195 ++++++++++++ .../endpoints/tts/get_tts_result.rs | 259 ++++++++++++++++ src/http_server/endpoints/tts/mod.rs | 2 + .../users/list_user_tts_inference_results.rs | 115 +++++++ .../endpoints/users/list_user_tts_models.rs | 114 +++++++ src/http_server/endpoints/users/mod.rs | 2 + src/main.rs | 32 +- 10 files changed, 1255 insertions(+), 4 deletions(-) create mode 100644 src/common_queries/list_tts_inference_results.rs create mode 100644 src/common_queries/list_tts_models.rs create mode 100644 src/http_server/endpoints/tts/get_tts_model.rs create mode 100644 src/http_server/endpoints/tts/get_tts_result.rs create mode 100644 src/http_server/endpoints/users/list_user_tts_inference_results.rs create mode 100644 src/http_server/endpoints/users/list_user_tts_models.rs diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs new file mode 100644 index 0000000000..d0b42e314c --- /dev/null +++ b/src/common_queries/list_tts_inference_results.rs @@ -0,0 +1,289 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::util::anyhow_result::AnyhowResult; +use log::{warn, info}; +use sqlx::MySqlPool; + +#[derive(Serialize)] +pub struct TtsInferenceRecordForList { + pub tts_result_token: String, + pub maybe_tts_model_token: Option, + pub maybe_tts_inference_result_token: Option, + + pub model_type: Option, + pub model_title: Option, + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + + pub file_size_bytes: u32, + pub frame_width: u32, + pub frame_height: u32, + pub duration_millis: u32, + + //pub is_mod_hidden_from_public: bool, // converted + //pub model_is_mod_approved: bool, // converted + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +struct RawTtsInferenceRecordForList { + pub tts_result_token: String, // from field `tts_results.token` + + pub maybe_tts_model_token: Option, + pub maybe_tts_inference_result_token: Option, + + pub model_type: Option, + pub model_title: Option, // from field `tts_models.title` + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + + pub file_size_bytes: i32, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + + //pub is_mod_hidden_from_public: i8, // needs convert + //pub model_is_mod_approved: i8, // needs convert + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +pub async fn list_tts_inference_results( + mysql_pool: &MySqlPool, + scope_creator_username: Option<&str>, + require_mod_approved_models: bool +) -> AnyhowResult> { + + let maybe_results = match scope_creator_username { + Some(username) => { + list_tts_inference_results_creator_scoped(mysql_pool, username, require_mod_approved_models) + .await + }, + None => { + list_tts_inference_results_for_all_creators(mysql_pool, require_mod_approved_models) + .await + }, + }; + + let results : Vec = match maybe_results { + Ok(results) => { + info!("Results length: {}", results.len()); + results + }, + Err(err) => { + warn!("Error: {:?}", err); + + match err { + RowNotFound => { + return Ok(Vec::new()); + }, + _ => { + warn!("tts inference result list query error: {:?}", err); + return Err(anyhow!("tts inference result list query error")); + } + } + } + }; + + Ok(results.into_iter() + .map(|ir| { + TtsInferenceRecordForList { + tts_result_token: ir.tts_result_token.clone(), + maybe_tts_model_token: ir.maybe_tts_model_token.clone(), + maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), + + model_type: ir.model_type.clone(), + model_title: ir.model_title.clone(), + + maybe_creator_user_token: ir.maybe_creator_user_token.clone(), + maybe_creator_username: ir.maybe_creator_username.clone(), + maybe_creator_display_name: ir.maybe_creator_display_name.clone(), + //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, + //model_is_mod_approved: if ir.model_is_mod_approved == 0 { false } else { true }, + + file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, + frame_width: if ir.frame_width > 0 { ir.frame_width as u32 } else { 0 }, + frame_height: if ir.frame_height > 0 { ir.frame_height as u32 } else { 0 }, + duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, + + created_at: ir.created_at.clone(), + updated_at: ir.updated_at.clone(), + } + }) + .collect::>()) +} + +async fn list_tts_inference_results_for_all_creators( + mysql_pool: &MySqlPool, + require_mod_approved_models: bool +) -> AnyhowResult> { + // TODO: There has to be a better way. + // Sqlx doesn't like anything except string literals. + let maybe_results = if require_mod_approved_models { + info!("listing tts inference results for everyone; mod-approved only"); + sqlx::query_as!( + RawTtsInferenceRecordForList, + r#" +SELECT + tts_results.token as tts_result_token, + tts_results.maybe_tts_inference_result_token, + + tts_models.token as maybe_tts_model_token, + tts_models.model_type, + tts_models.title as model_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + tts_results.file_size_bytes, + tts_results.frame_width, + tts_results.frame_height, + tts_results.duration_millis, + tts_results.created_at, + tts_results.updated_at + +FROM tts_results +LEFT OUTER JOIN tts_models + ON tts_results.maybe_tts_model_token = tts_models.token +LEFT OUTER JOIN users + ON tts_results.maybe_creator_user_token = users.token +WHERE + tts_results.deleted_at IS NULL + AND tts_results.is_mod_hidden_from_public IS FALSE + AND tts_models.is_mod_approved IS TRUE + "#) + .fetch_all(mysql_pool) + .await? + } else { + info!("listing tts inference results for everyone; all"); + sqlx::query_as!( + RawTtsInferenceRecordForList, + r#" +SELECT + tts_results.token as tts_result_token, + tts_results.maybe_tts_inference_result_token, + + tts_models.token as maybe_tts_model_token, + tts_models.model_type, + tts_models.title as model_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + tts_results.file_size_bytes, + tts_results.frame_width, + tts_results.frame_height, + tts_results.duration_millis, + tts_results.created_at, + tts_results.updated_at + +FROM tts_results +LEFT OUTER JOIN tts_models + ON tts_results.maybe_tts_model_token = tts_models.token +LEFT OUTER JOIN users + ON tts_results.maybe_creator_user_token = users.token +WHERE + tts_results.deleted_at IS NULL + "#) + .fetch_all(mysql_pool) + .await? + }; + + Ok(maybe_results) +} + +async fn list_tts_inference_results_creator_scoped( + mysql_pool: &MySqlPool, + scope_creator_username: &str, + require_mod_approved_models: bool +) -> AnyhowResult> { + // TODO: There has to be a better way. + // Sqlx doesn't like anything except string literals. + let maybe_results = if require_mod_approved_models { + info!("listing tts inference results for user `{}`; mod-approved only", scope_creator_username); + sqlx::query_as!( + RawTtsInferenceRecordForList, + r#" +SELECT + tts_results.token as tts_result_token, + tts_results.maybe_tts_inference_result_token, + + tts_models.token as maybe_tts_model_token, + tts_models.model_type, + tts_models.title as model_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + tts_results.file_size_bytes, + tts_results.frame_width, + tts_results.frame_height, + tts_results.duration_millis, + tts_results.created_at, + tts_results.updated_at + +FROM tts_results +LEFT OUTER JOIN tts_models + ON tts_results.maybe_tts_model_token = tts_models.token +LEFT OUTER JOIN users + ON tts_results.maybe_creator_user_token = users.token +WHERE + tts_results.deleted_at IS NULL + AND tts_results.is_mod_hidden_from_public IS FALSE + AND tts_models.is_mod_approved IS TRUE + AND users.username = ? + "#, + scope_creator_username) + .fetch_all(mysql_pool) + .await? + } else { + info!("listing tts inference results for user `{}`; all", scope_creator_username); + sqlx::query_as!( + RawTtsInferenceRecordForList, + r#" +SELECT + tts_results.token as tts_result_token, + tts_results.maybe_tts_inference_result_token, + + tts_models.token as maybe_tts_model_token, + tts_models.model_type, + tts_models.title as model_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + tts_results.file_size_bytes, + tts_results.frame_width, + tts_results.frame_height, + tts_results.duration_millis, + tts_results.created_at, + tts_results.updated_at + +FROM tts_results +LEFT OUTER JOIN tts_models + ON tts_results.maybe_tts_model_token = tts_models.token +LEFT OUTER JOIN users + ON tts_results.maybe_creator_user_token = users.token +WHERE + tts_results.deleted_at IS NULL + AND users.username = ? + "#, + scope_creator_username) + .fetch_all(mysql_pool) + .await? + }; + + Ok(maybe_results) +} diff --git a/src/common_queries/list_tts_models.rs b/src/common_queries/list_tts_models.rs new file mode 100644 index 0000000000..5313d690aa --- /dev/null +++ b/src/common_queries/list_tts_models.rs @@ -0,0 +1,249 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::util::anyhow_result::AnyhowResult; +use log::{warn, info}; +use sqlx::MySqlPool; + +#[derive(Serialize)] +pub struct TtsModelRecordForList { + pub model_token: String, + pub model_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: u32, + pub frame_height: u32, + pub duration_millis: u32, + pub maybe_image_object_name: Option, + pub maybe_video_object_name: Option, + pub is_mod_approved: bool, // converted + pub created_at: DateTime, + pub updated_at: DateTime, +} + +struct RawTtsModelRecordForList { + pub model_token: String, + pub model_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + pub maybe_public_bucket_preview_image_object_name: Option, + pub maybe_public_bucket_preview_video_object_name: Option, + pub is_mod_approved: i8, // NB: needs conversion + pub created_at: DateTime, + pub updated_at: DateTime, +} + +pub async fn list_tts_models( + mysql_pool: &MySqlPool, + scope_creator_username: Option<&str>, + require_mod_approved: bool +) -> AnyhowResult> { + + let maybe_models = match scope_creator_username { + Some(username) => { + list_tts_models_creator_scoped(mysql_pool, username, require_mod_approved) + .await + }, + None => { + list_tts_models_for_all_creators(mysql_pool, require_mod_approved) + .await + }, + }; + + let models : Vec = match maybe_models { + Ok(models) => { + info!("Model length: {}", models.len()); + models + }, + Err(err) => { + warn!("Error: {:?}", err); + + match err { + RowNotFound => { + return Ok(Vec::new()); + }, + _ => { + warn!("tts model list query error: {:?}", err); + return Err(anyhow!("tts model list query error")); + } + } + } + }; + + Ok(models.into_iter() + .map(|model| { + TtsModelRecordForList { + model_token: model.model_token.clone(), + model_type: model.model_type.clone(), + creator_user_token: model.creator_user_token.clone(), + creator_username: model.creator_username.clone(), + creator_display_name: model.creator_display_name.clone(), + updatable_slug: model.updatable_slug.clone(), + title: model.title.clone(), + frame_width: if model.frame_width > 0 { model.frame_width as u32 } else { 0 }, + frame_height: if model.frame_height > 0 { model.frame_height as u32 } else { 0 }, + duration_millis: if model.duration_millis > 0 { model.duration_millis as u32 } else { 0 }, + maybe_image_object_name: model.maybe_public_bucket_preview_image_object_name.clone(), + maybe_video_object_name: model.maybe_public_bucket_preview_video_object_name.clone(), + is_mod_approved: if model.is_mod_approved == 0 { false } else { true }, + created_at: model.created_at.clone(), + updated_at: model.updated_at.clone(), + } + }) + .collect::>()) +} + +async fn list_tts_models_for_all_creators( + mysql_pool: &MySqlPool, + require_mod_approved: bool +) -> AnyhowResult> { + // TODO: There has to be a better way. + // Sqlx doesn't like anything except string literals. + let maybe_models = if require_mod_approved { + info!("listing tts models for everyone; mod-approved only"); + sqlx::query_as!( + RawTtsModelRecordForList, + r#" +SELECT + tts.token as model_token, + tts.model_type, + tts.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + tts.updatable_slug, + tts.title, + tts.frame_width, + tts.frame_height, + tts.duration_millis, + tts.maybe_public_bucket_preview_image_object_name, + tts.maybe_public_bucket_preview_video_object_name, + tts.is_mod_approved, + tts.created_at, + tts.updated_at +FROM tts_models as tts +JOIN users +ON users.token = tts.creator_user_token +WHERE + tts.deleted_at IS NULL + AND tts.is_mod_approved IS TRUE + "#) + .fetch_all(mysql_pool) + .await? + } else { + info!("listing tts models for everyone; all"); + sqlx::query_as!( + RawTtsModelRecordForList, + r#" +SELECT + tts.token as model_token, + tts.model_type, + tts.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + tts.updatable_slug, + tts.title, + tts.frame_width, + tts.frame_height, + tts.duration_millis, + tts.maybe_public_bucket_preview_image_object_name, + tts.maybe_public_bucket_preview_video_object_name, + tts.is_mod_approved, + tts.created_at, + tts.updated_at +FROM tts_models as tts +JOIN users +ON users.token = tts.creator_user_token +WHERE + tts.deleted_at IS NULL + "#) + .fetch_all(mysql_pool) + .await? + }; + + Ok(maybe_models) +} + +async fn list_tts_models_creator_scoped( + mysql_pool: &MySqlPool, + scope_creator_username: &str, + require_mod_approved: bool +) -> AnyhowResult> { + // TODO: There has to be a better way. + // Sqlx doesn't like anything except string literals. + let maybe_models = if require_mod_approved { + info!("listing tts models for user; mod-approved only"); + sqlx::query_as!( + RawTtsModelRecordForList, + r#" +SELECT + tts.token as model_token, + tts.model_type, + tts.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + tts.updatable_slug, + tts.title, + tts.frame_width, + tts.frame_height, + tts.duration_millis, + tts.maybe_public_bucket_preview_image_object_name, + tts.maybe_public_bucket_preview_video_object_name, + tts.is_mod_approved, + tts.created_at, + tts.updated_at +FROM tts_models as tts +JOIN users +ON + users.token = tts.creator_user_token +WHERE + tts.deleted_at IS NULL + AND tts.is_mod_approved IS TRUE + AND users.username = ? + "#, + scope_creator_username) + .fetch_all(mysql_pool) + .await? + } else { + info!("listing tts models for user; all"); + sqlx::query_as!( + RawTtsModelRecordForList, + r#" +SELECT + tts.token as model_token, + tts.model_type, + tts.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + tts.updatable_slug, + tts.title, + tts.frame_width, + tts.frame_height, + tts.duration_millis, + tts.maybe_public_bucket_preview_image_object_name, + tts.maybe_public_bucket_preview_video_object_name, + tts.is_mod_approved, + tts.created_at, + tts.updated_at +FROM tts_models as tts +JOIN users +ON + users.token = tts.creator_user_token +WHERE + tts.deleted_at IS NULL + AND users.username = ? + "#, + scope_creator_username) + .fetch_all(mysql_pool) + .await? + }; + + Ok(maybe_models) +} diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index bdefa2998f..26dfb92570 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -1,5 +1,7 @@ pub mod badges; pub mod firehose_publisher; +pub mod list_tts_inference_results; +pub mod list_tts_models; pub mod list_w2l_inference_results; pub mod list_w2l_templates; pub mod sessions; diff --git a/src/http_server/endpoints/tts/get_tts_model.rs b/src/http_server/endpoints/tts/get_tts_model.rs new file mode 100644 index 0000000000..8d6decd2fa --- /dev/null +++ b/src/http_server/endpoints/tts/get_tts_model.rs @@ -0,0 +1,195 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetTtsModelPathInfo { + slug: String, +} + +#[derive(Serialize)] +pub struct TtsModelRecordForResponse { + pub model_token: String, + pub model_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: u32, + pub frame_height: u32, + pub duration_millis: u32, + pub maybe_image_object_name: Option, + pub maybe_video_object_name: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct GetTtsModelSuccessResponse { + pub success: bool, + pub template: TtsModelRecordForResponse, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum GetTtsModelError { + ServerError, +} + +#[derive(Serialize)] +pub struct TtsModelRecord { + pub model_token: String, + pub model_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + pub maybe_public_bucket_preview_image_object_name: Option, + pub maybe_public_bucket_preview_video_object_name: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl ResponseError for GetTtsModelError { + fn status_code(&self) -> StatusCode { + match *self { + GetTtsModelError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetTtsModelError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn get_tts_model_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_models = sqlx::query_as!( + TtsModelRecord, + r#" +SELECT + tts.token as model_token, + tts.model_type, + tts.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + tts.updatable_slug, + tts.title, + tts.frame_width, + tts.frame_height, + tts.duration_millis, + tts.maybe_public_bucket_preview_image_object_name, + tts.maybe_public_bucket_preview_video_object_name, + tts.created_at, + tts.updated_at +FROM tts_models as tts +JOIN users +ON users.token = tts.creator_user_token +WHERE tts.updatable_slug = ? +AND tts.deleted_at IS NULL + "#, + &path.slug + ) + .fetch_one(&server_state.mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let template : TtsModelRecord = match maybe_models { + Ok(models) => models, + Err(err) => { + match err { + RowNotFound => { + return Err(GetTtsModelError::ServerError); + }, + _ => { + warn!("tts template query error: {:?}", err); + return Err(GetTtsModelError::ServerError); + } + } + } + }; + + let model_for_response = TtsModelRecordForResponse { + model_token: template.model_token.clone(), + model_type: template.model_type.clone(), + creator_user_token: template.creator_user_token.clone(), + creator_username: template.creator_username.clone(), + creator_display_name: template.creator_display_name.clone(), + updatable_slug: template.updatable_slug.clone(), + title: template.title.clone(), + frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, + frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, + duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, + maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), + maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), + created_at: template.created_at.clone(), + updated_at: template.updated_at.clone(), + }; + + let response = GetTtsModelSuccessResponse { + success: true, + template: model_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetTtsModelError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/tts/get_tts_result.rs b/src/http_server/endpoints/tts/get_tts_result.rs new file mode 100644 index 0000000000..205ab05ced --- /dev/null +++ b/src/http_server/endpoints/tts/get_tts_result.rs @@ -0,0 +1,259 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetTtsResultPathInfo { + token: String, +} + +#[derive(Serialize)] +pub struct TtsResultRecordForResponse { + pub tts_result_token: String, + pub maybe_tts_model_token: Option, + pub maybe_tts_inference_result_token: Option, + + pub public_bucket_video_path: String, + + pub template_type: Option, + pub template_title: Option, + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub maybe_template_creator_user_token: Option, + pub maybe_template_creator_username: Option, + pub maybe_template_creator_display_name: Option, + pub maybe_template_creator_gravatar_hash: Option, + + pub file_size_bytes: u32, + pub frame_width: u32, + pub frame_height: u32, + pub duration_millis: u32, + + //pub is_mod_hidden_from_public: bool, // converted + //pub template_is_mod_approved: bool, // converted + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct GetTtsResultSuccessResponse { + pub success: bool, + pub result: TtsResultRecordForResponse, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum GetTtsResultError { + ServerError, +} + +pub struct RawTtsResultRecord { + pub tts_result_token: String, // from field `tts_results.token` + + pub maybe_tts_model_token: Option, + pub maybe_tts_inference_result_token: Option, + + pub public_bucket_video_path: String, + + pub template_type: Option, + pub template_title: Option, // from field `tts_models.title` + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub maybe_template_creator_user_token: Option, + pub maybe_template_creator_username: Option, + pub maybe_template_creator_display_name: Option, + pub maybe_template_creator_gravatar_hash: Option, + + pub file_size_bytes: i32, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + + //pub is_mod_hidden_from_public: i8, // needs convert + //pub template_is_mod_approved: i8, // needs convert + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl ResponseError for GetTtsResultError { + fn status_code(&self) -> StatusCode { + match *self { + GetTtsResultError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetTtsResultError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn get_tts_inference_result_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_result = sqlx::query_as!( + RawTtsResultRecord, + r#" +SELECT + tts_results.token as tts_result_token, + tts_results.maybe_tts_inference_result_token, + + tts_results.public_bucket_video_path, + + tts_models.token as maybe_tts_model_token, + tts_models.template_type, + tts_models.title as template_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_gravatar_hash, + + template_users.token as maybe_template_creator_user_token, + template_users.username as maybe_template_creator_username, + template_users.display_name as maybe_template_creator_display_name, + template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash, + + tts_results.file_size_bytes, + tts_results.frame_width, + tts_results.frame_height, + tts_results.duration_millis, + tts_results.created_at, + tts_results.updated_at + +FROM tts_results +LEFT OUTER JOIN tts_models + ON tts_results.maybe_tts_model_token = tts_models.token +LEFT OUTER JOIN users + ON tts_results.maybe_creator_user_token = users.token +LEFT OUTER JOIN users as template_users + ON tts_models.creator_user_token = template_users.token +WHERE + tts_results.deleted_at IS NULL + AND tts_results.token = ? + "#, + &path.token + ) + .fetch_one(&server_state.mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let ir : RawTtsResultRecord = match maybe_result { + Ok(result) => result, + Err(err) => { + match err { + RowNotFound => { + return Err(GetTtsResultError::ServerError); + }, + _ => { + warn!("w2l inference result query error: {:?}", err); + return Err(GetTtsResultError::ServerError); + } + } + } + }; + + let result_for_response = TtsResultRecordForResponse { + tts_result_token: ir.tts_result_token.clone(), + maybe_tts_model_token: ir.maybe_tts_model_token.clone(), + maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), + + public_bucket_video_path: ir.public_bucket_video_path.clone(), + + template_type: ir.template_type.clone(), + template_title: ir.template_title.clone(), + + maybe_creator_user_token: ir.maybe_creator_user_token.clone(), + maybe_creator_username: ir.maybe_creator_username.clone(), + maybe_creator_display_name: ir.maybe_creator_display_name.clone(), + maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash.clone(), + + maybe_template_creator_user_token: ir.maybe_template_creator_user_token.clone(), + maybe_template_creator_username: ir.maybe_template_creator_username.clone(), + maybe_template_creator_display_name: ir.maybe_template_creator_display_name.clone(), + maybe_template_creator_gravatar_hash: ir.maybe_template_creator_gravatar_hash.clone(), + + //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, + //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, + + file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, + frame_width: if ir.frame_width > 0 { ir.frame_width as u32 } else { 0 }, + frame_height: if ir.frame_height > 0 { ir.frame_height as u32 } else { 0 }, + duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, + + created_at: ir.created_at.clone(), + updated_at: ir.updated_at.clone(), + }; + + let response = GetTtsResultSuccessResponse { + success: true, + result: result_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetTtsResultError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/tts/mod.rs b/src/http_server/endpoints/tts/mod.rs index e0e4f48731..831211eae2 100644 --- a/src/http_server/endpoints/tts/mod.rs +++ b/src/http_server/endpoints/tts/mod.rs @@ -1,3 +1,5 @@ pub mod enqueue_infer_tts; pub mod enqueue_upload_tts_model; +pub mod get_tts_model; +pub mod get_tts_result; pub mod list_tts_models; \ No newline at end of file diff --git a/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/src/http_server/endpoints/users/list_user_tts_inference_results.rs new file mode 100644 index 0000000000..e60dd83b04 --- /dev/null +++ b/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -0,0 +1,115 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::list_tts_inference_results::TtsInferenceRecordForList; +use crate::common_queries::list_tts_inference_results::list_tts_inference_results; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetProfilePathInfo { + username: String, +} + +#[derive(Serialize)] +pub struct ListTtsInferenceResultsForUserSuccessResponse { + pub success: bool, + pub results: Vec, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum ListTtsInferenceResultsForUserError { + ServerError, +} + +impl ResponseError for ListTtsInferenceResultsForUserError { + fn status_code(&self) -> StatusCode { + match *self { + ListTtsInferenceResultsForUserError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListTtsInferenceResultsForUserError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn list_user_tts_inference_results_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result +{ + info!("Fetching inference results for user: {}", &path.username); + + let query_results = list_tts_inference_results( + &server_state.mysql_pool, + Some(path.username.as_ref()), + false + ).await; + + let results = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListTtsInferenceResultsForUserError::ServerError); + } + }; + + let response = ListTtsInferenceResultsForUserSuccessResponse { + success: true, + results, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListTtsInferenceResultsForUserError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/users/list_user_tts_models.rs b/src/http_server/endpoints/users/list_user_tts_models.rs new file mode 100644 index 0000000000..089c13da04 --- /dev/null +++ b/src/http_server/endpoints/users/list_user_tts_models.rs @@ -0,0 +1,114 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::list_tts_models::{TtsModelRecordForList, list_tts_models}; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetProfilePathInfo { + username: String, +} + +#[derive(Serialize)] +pub struct ListTtsModelsForUserSuccessResponse { + pub success: bool, + pub models: Vec, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum ListTtsModelsForUserError { + ServerError, +} + +impl ResponseError for ListTtsModelsForUserError { + fn status_code(&self) -> StatusCode { + match *self { + ListTtsModelsForUserError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListTtsModelsForUserError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn list_user_tts_models_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data> +) -> Result +{ + info!("Fetching templates for user: {}", &path.username); + + let query_results = list_tts_models( + &server_state.mysql_pool, + Some(path.username.as_ref()), + false + ).await; + + let models = match query_results { + Ok(results) => results, + Err(e) => { + warn!("Query error: {:?}", e); + return Err(ListTtsModelsForUserError::ServerError); + } + }; + + let response = ListTtsModelsForUserSuccessResponse { + success: true, + models, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListTtsModelsForUserError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/users/mod.rs b/src/http_server/endpoints/users/mod.rs index f16ee1b6cb..c07f287ddb 100644 --- a/src/http_server/endpoints/users/mod.rs +++ b/src/http_server/endpoints/users/mod.rs @@ -1,5 +1,7 @@ pub mod create_account; pub mod get_profile; +pub mod list_user_tts_inference_results; +pub mod list_user_tts_models; pub mod list_user_w2l_inference_results; pub mod list_user_w2l_templates; pub mod login; diff --git a/src/main.rs b/src/main.rs index 5a3386036a..ddff8ec933 100644 --- a/src/main.rs +++ b/src/main.rs @@ -57,6 +57,10 @@ use log::{info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; +use crate::http_server::endpoints::users::list_user_tts_inference_results::list_user_tts_inference_results_handler; +use crate::http_server::endpoints::users::list_user_tts_models::list_user_tts_models_handler; +use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; +use crate::http_server::endpoints::tts::get_tts_model::get_tts_model_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -270,6 +274,16 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(list_tts_models_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/model/{slug}") + .route(web::get().to(get_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/result/{token}") + .route(web::get().to(get_tts_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service( web::scope("/w2l") @@ -307,15 +321,25 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .service( web::scope("/user") .service( - web::resource("/{username}/w2l_templates") - .route(web::get().to(list_user_w2l_templates_handler)) + web::resource("/{username}/tts_models") + .route(web::get().to(list_user_tts_models_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) .service( - web::resource("/{username}/w2l_results") - .route(web::get().to(list_user_w2l_inference_results_handler)) + web::resource("/{username}/tts_results") + .route(web::get().to(list_user_tts_inference_results_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/{username}/w2l_templates") + .route(web::get().to(list_user_w2l_templates_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( + web::resource("/{username}/w2l_results") + .route(web::get().to(list_user_w2l_inference_results_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service( web::resource("/events") From 3022193ff7e03b3eb7ef42ffc4676f58d7bbddfc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Jun 2021 02:35:31 -0400 Subject: [PATCH 0276/1042] fixed errors on one endpoint --- .../endpoints/tts/get_tts_result.rs | 99 ++++++++----------- 1 file changed, 43 insertions(+), 56 deletions(-) diff --git a/src/http_server/endpoints/tts/get_tts_result.rs b/src/http_server/endpoints/tts/get_tts_result.rs index 205ab05ced..977ac26623 100644 --- a/src/http_server/endpoints/tts/get_tts_result.rs +++ b/src/http_server/endpoints/tts/get_tts_result.rs @@ -35,31 +35,28 @@ pub struct GetTtsResultPathInfo { #[derive(Serialize)] pub struct TtsResultRecordForResponse { pub tts_result_token: String, - pub maybe_tts_model_token: Option, - pub maybe_tts_inference_result_token: Option, - pub public_bucket_video_path: String, - - pub template_type: Option, - pub template_title: Option, + pub tts_model_token: String, + pub inference_text: String, pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, pub maybe_creator_gravatar_hash: Option, - pub maybe_template_creator_user_token: Option, - pub maybe_template_creator_username: Option, - pub maybe_template_creator_display_name: Option, - pub maybe_template_creator_gravatar_hash: Option, + pub maybe_model_creator_user_token: Option, + pub maybe_model_creator_username: Option, + pub maybe_model_creator_display_name: Option, + pub maybe_model_creator_gravatar_hash: Option, + + pub public_bucket_wav_audio_path: String, + pub public_bucket_spectrogram_path: String, pub file_size_bytes: u32, - pub frame_width: u32, - pub frame_height: u32, pub duration_millis: u32, //pub is_mod_hidden_from_public: bool, // converted - //pub template_is_mod_approved: bool, // converted + //pub model_is_mod_approved: bool, // converted //pub maybe_mod_user_token: Option, pub created_at: DateTime, @@ -86,31 +83,27 @@ pub enum GetTtsResultError { pub struct RawTtsResultRecord { pub tts_result_token: String, // from field `tts_results.token` - pub maybe_tts_model_token: Option, - pub maybe_tts_inference_result_token: Option, - - pub public_bucket_video_path: String, - - pub template_type: Option, - pub template_title: Option, // from field `tts_models.title` + pub tts_model_token: String, + pub inference_text: String, pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, pub maybe_creator_gravatar_hash: Option, - pub maybe_template_creator_user_token: Option, - pub maybe_template_creator_username: Option, - pub maybe_template_creator_display_name: Option, - pub maybe_template_creator_gravatar_hash: Option, + pub maybe_model_creator_user_token: Option, + pub maybe_model_creator_username: Option, + pub maybe_model_creator_display_name: Option, + pub maybe_model_creator_gravatar_hash: Option, + + pub public_bucket_wav_audio_path: String, + pub public_bucket_spectrogram_path: String, pub file_size_bytes: i32, - pub frame_width: i32, - pub frame_height: i32, pub duration_millis: i32, //pub is_mod_hidden_from_public: i8, // needs convert - //pub template_is_mod_approved: i8, // needs convert + //pub model_is_mod_approved: i8, // needs convert //pub maybe_mod_user_token: Option, pub created_at: DateTime, @@ -157,38 +150,35 @@ pub async fn get_tts_inference_result_handler( r#" SELECT tts_results.token as tts_result_token, - tts_results.maybe_tts_inference_result_token, - tts_results.public_bucket_video_path, - - tts_models.token as maybe_tts_model_token, - tts_models.template_type, - tts_models.title as template_title, + tts_results.model_token as tts_model_token, + tts_results.inference_text, users.token as maybe_creator_user_token, users.username as maybe_creator_username, users.display_name as maybe_creator_display_name, users.email_gravatar_hash as maybe_creator_gravatar_hash, - template_users.token as maybe_template_creator_user_token, - template_users.username as maybe_template_creator_username, - template_users.display_name as maybe_template_creator_display_name, - template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash, + model_users.token as maybe_model_creator_user_token, + model_users.username as maybe_model_creator_username, + model_users.display_name as maybe_model_creator_display_name, + model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash, + + tts_results.public_bucket_wav_audio_path, + tts_results.public_bucket_spectrogram_path, tts_results.file_size_bytes, - tts_results.frame_width, - tts_results.frame_height, tts_results.duration_millis, tts_results.created_at, tts_results.updated_at FROM tts_results LEFT OUTER JOIN tts_models - ON tts_results.maybe_tts_model_token = tts_models.token + ON tts_results.model_token = tts_models.token LEFT OUTER JOIN users ON tts_results.maybe_creator_user_token = users.token -LEFT OUTER JOIN users as template_users - ON tts_models.creator_user_token = template_users.token +LEFT OUTER JOIN users as model_users + ON tts_models.creator_user_token = model_users.token WHERE tts_results.deleted_at IS NULL AND tts_results.token = ? @@ -206,7 +196,7 @@ WHERE return Err(GetTtsResultError::ServerError); }, _ => { - warn!("w2l inference result query error: {:?}", err); + warn!("tts inference result query error: {:?}", err); return Err(GetTtsResultError::ServerError); } } @@ -215,30 +205,27 @@ WHERE let result_for_response = TtsResultRecordForResponse { tts_result_token: ir.tts_result_token.clone(), - maybe_tts_model_token: ir.maybe_tts_model_token.clone(), - maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), - public_bucket_video_path: ir.public_bucket_video_path.clone(), - - template_type: ir.template_type.clone(), - template_title: ir.template_title.clone(), + tts_model_token: ir.tts_model_token.clone(), + inference_text: ir.inference_text.clone(), maybe_creator_user_token: ir.maybe_creator_user_token.clone(), maybe_creator_username: ir.maybe_creator_username.clone(), maybe_creator_display_name: ir.maybe_creator_display_name.clone(), maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash.clone(), - maybe_template_creator_user_token: ir.maybe_template_creator_user_token.clone(), - maybe_template_creator_username: ir.maybe_template_creator_username.clone(), - maybe_template_creator_display_name: ir.maybe_template_creator_display_name.clone(), - maybe_template_creator_gravatar_hash: ir.maybe_template_creator_gravatar_hash.clone(), + maybe_model_creator_user_token: ir.maybe_model_creator_user_token.clone(), + maybe_model_creator_username: ir.maybe_model_creator_username.clone(), + maybe_model_creator_display_name: ir.maybe_model_creator_display_name.clone(), + maybe_model_creator_gravatar_hash: ir.maybe_model_creator_gravatar_hash.clone(), //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, - //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, + //model_is_mod_approved: if ir.model_is_mod_approved == 0 { false } else { true }, + + public_bucket_wav_audio_path: ir.public_bucket_wav_audio_path.clone(), + public_bucket_spectrogram_path: ir.public_bucket_spectrogram_path.clone(), file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, - frame_width: if ir.frame_width > 0 { ir.frame_width as u32 } else { 0 }, - frame_height: if ir.frame_height > 0 { ir.frame_height as u32 } else { 0 }, duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, created_at: ir.created_at.clone(), From e8dc191d8e389bdafd8345ec5e2b9d6bbe602a95 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Jun 2021 02:44:17 -0400 Subject: [PATCH 0277/1042] fix four more errors --- src/common_queries/list_tts_models.rs | 91 ++++++++++----------------- src/main.rs | 2 + 2 files changed, 34 insertions(+), 59 deletions(-) diff --git a/src/common_queries/list_tts_models.rs b/src/common_queries/list_tts_models.rs index 5313d690aa..31d073866b 100644 --- a/src/common_queries/list_tts_models.rs +++ b/src/common_queries/list_tts_models.rs @@ -7,36 +7,34 @@ use sqlx::MySqlPool; #[derive(Serialize)] pub struct TtsModelRecordForList { pub model_token: String, - pub model_type: String, + pub tts_model_type: String, + + pub title: String, + pub updatable_slug: String, + pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, - pub title: String, - pub frame_width: u32, - pub frame_height: u32, - pub duration_millis: u32, - pub maybe_image_object_name: Option, - pub maybe_video_object_name: Option, - pub is_mod_approved: bool, // converted + + pub is_mod_disabled: bool, // converted + pub created_at: DateTime, pub updated_at: DateTime, } struct RawTtsModelRecordForList { pub model_token: String, - pub model_type: String, + pub tts_model_type: String, + + pub title: String, + pub updatable_slug: String, + pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, - pub title: String, - pub frame_width: i32, - pub frame_height: i32, - pub duration_millis: i32, - pub maybe_public_bucket_preview_image_object_name: Option, - pub maybe_public_bucket_preview_video_object_name: Option, - pub is_mod_approved: i8, // NB: needs conversion + + pub is_mod_disabled: i8, // NB: needs conversion + pub created_at: DateTime, pub updated_at: DateTime, } @@ -82,18 +80,13 @@ pub async fn list_tts_models( .map(|model| { TtsModelRecordForList { model_token: model.model_token.clone(), - model_type: model.model_type.clone(), + tts_model_type: model.tts_model_type.clone(), creator_user_token: model.creator_user_token.clone(), creator_username: model.creator_username.clone(), creator_display_name: model.creator_display_name.clone(), updatable_slug: model.updatable_slug.clone(), title: model.title.clone(), - frame_width: if model.frame_width > 0 { model.frame_width as u32 } else { 0 }, - frame_height: if model.frame_height > 0 { model.frame_height as u32 } else { 0 }, - duration_millis: if model.duration_millis > 0 { model.duration_millis as u32 } else { 0 }, - maybe_image_object_name: model.maybe_public_bucket_preview_image_object_name.clone(), - maybe_video_object_name: model.maybe_public_bucket_preview_video_object_name.clone(), - is_mod_approved: if model.is_mod_approved == 0 { false } else { true }, + is_mod_disabled: if model.is_mod_disabled == 0 { false } else { true }, created_at: model.created_at.clone(), updated_at: model.updated_at.clone(), } @@ -103,29 +96,24 @@ pub async fn list_tts_models( async fn list_tts_models_for_all_creators( mysql_pool: &MySqlPool, - require_mod_approved: bool + allow_mod_disabled: bool ) -> AnyhowResult> { // TODO: There has to be a better way. // Sqlx doesn't like anything except string literals. - let maybe_models = if require_mod_approved { + let maybe_models = if !allow_mod_disabled { info!("listing tts models for everyone; mod-approved only"); sqlx::query_as!( RawTtsModelRecordForList, r#" SELECT tts.token as model_token, - tts.model_type, + tts.tts_model_type, tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, tts.updatable_slug, tts.title, - tts.frame_width, - tts.frame_height, - tts.duration_millis, - tts.maybe_public_bucket_preview_image_object_name, - tts.maybe_public_bucket_preview_video_object_name, - tts.is_mod_approved, + tts.is_mod_disabled, tts.created_at, tts.updated_at FROM tts_models as tts @@ -133,7 +121,7 @@ JOIN users ON users.token = tts.creator_user_token WHERE tts.deleted_at IS NULL - AND tts.is_mod_approved IS TRUE + AND tts.is_mod_disabled IS FALSE "#) .fetch_all(mysql_pool) .await? @@ -144,18 +132,13 @@ WHERE r#" SELECT tts.token as model_token, - tts.model_type, + tts.tts_model_type, tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, tts.updatable_slug, tts.title, - tts.frame_width, - tts.frame_height, - tts.duration_millis, - tts.maybe_public_bucket_preview_image_object_name, - tts.maybe_public_bucket_preview_video_object_name, - tts.is_mod_approved, + tts.is_mod_disabled, tts.created_at, tts.updated_at FROM tts_models as tts @@ -174,29 +157,24 @@ WHERE async fn list_tts_models_creator_scoped( mysql_pool: &MySqlPool, scope_creator_username: &str, - require_mod_approved: bool + allow_mod_disabled: bool ) -> AnyhowResult> { // TODO: There has to be a better way. // Sqlx doesn't like anything except string literals. - let maybe_models = if require_mod_approved { + let maybe_models = if !allow_mod_disabled { info!("listing tts models for user; mod-approved only"); sqlx::query_as!( RawTtsModelRecordForList, r#" SELECT tts.token as model_token, - tts.model_type, + tts.tts_model_type, tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, tts.updatable_slug, tts.title, - tts.frame_width, - tts.frame_height, - tts.duration_millis, - tts.maybe_public_bucket_preview_image_object_name, - tts.maybe_public_bucket_preview_video_object_name, - tts.is_mod_approved, + tts.is_mod_disabled, tts.created_at, tts.updated_at FROM tts_models as tts @@ -205,7 +183,7 @@ ON users.token = tts.creator_user_token WHERE tts.deleted_at IS NULL - AND tts.is_mod_approved IS TRUE + AND tts.is_mod_disabled IS FALSE AND users.username = ? "#, scope_creator_username) @@ -218,18 +196,13 @@ WHERE r#" SELECT tts.token as model_token, - tts.model_type, + tts.tts_model_type, tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, tts.updatable_slug, tts.title, - tts.frame_width, - tts.frame_height, - tts.duration_millis, - tts.maybe_public_bucket_preview_image_object_name, - tts.maybe_public_bucket_preview_video_object_name, - tts.is_mod_approved, + tts.is_mod_disabled, tts.created_at, tts.updated_at FROM tts_models as tts diff --git a/src/main.rs b/src/main.rs index ddff8ec933..a8ace9af4a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -70,6 +70,8 @@ use crate::http_server::endpoints::tts::get_tts_model::get_tts_model_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO +// TODO TODO TODO -- ON signup, add an "early adopter" badge. And a tool for making it easy to add badges. +// TODO - badge for uploading template, badge for uploading model, etc. const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; From 6702b42808c4e6256ce531bbe64cd946eb9f1cec Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Jun 2021 02:50:04 -0400 Subject: [PATCH 0278/1042] more fixed --- .../endpoints/tts/get_tts_model.rs | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/src/http_server/endpoints/tts/get_tts_model.rs b/src/http_server/endpoints/tts/get_tts_model.rs index 8d6decd2fa..079c9bd7cd 100644 --- a/src/http_server/endpoints/tts/get_tts_model.rs +++ b/src/http_server/endpoints/tts/get_tts_model.rs @@ -35,17 +35,15 @@ pub struct GetTtsModelPathInfo { #[derive(Serialize)] pub struct TtsModelRecordForResponse { pub model_token: String, - pub model_type: String, + pub tts_model_type: String, + pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, + pub updatable_slug: String, pub title: String, - pub frame_width: u32, - pub frame_height: u32, - pub duration_millis: u32, - pub maybe_image_object_name: Option, - pub maybe_video_object_name: Option, + pub created_at: DateTime, pub updated_at: DateTime, } @@ -70,17 +68,15 @@ pub enum GetTtsModelError { #[derive(Serialize)] pub struct TtsModelRecord { pub model_token: String, - pub model_type: String, + pub tts_model_type: String, + pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, + pub updatable_slug: String, pub title: String, - pub frame_width: i32, - pub frame_height: i32, - pub duration_millis: i32, - pub maybe_public_bucket_preview_image_object_name: Option, - pub maybe_public_bucket_preview_video_object_name: Option, + pub created_at: DateTime, pub updated_at: DateTime, } @@ -125,17 +121,15 @@ pub async fn get_tts_model_handler( r#" SELECT tts.token as model_token, - tts.model_type, + tts.tts_model_type, + tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, + tts.updatable_slug, tts.title, - tts.frame_width, - tts.frame_height, - tts.duration_millis, - tts.maybe_public_bucket_preview_image_object_name, - tts.maybe_public_bucket_preview_video_object_name, + tts.created_at, tts.updated_at FROM tts_models as tts @@ -166,17 +160,12 @@ AND tts.deleted_at IS NULL let model_for_response = TtsModelRecordForResponse { model_token: template.model_token.clone(), - model_type: template.model_type.clone(), + tts_model_type: template.tts_model_type.clone(), creator_user_token: template.creator_user_token.clone(), creator_username: template.creator_username.clone(), creator_display_name: template.creator_display_name.clone(), updatable_slug: template.updatable_slug.clone(), title: template.title.clone(), - frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, - frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, - duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, - maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), - maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), created_at: template.created_at.clone(), updated_at: template.updated_at.clone(), }; From 9a4a8fd23a563a78069b5efde43f6bee71c6b509 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Jun 2021 03:07:14 -0400 Subject: [PATCH 0279/1042] fixed? still errors... --- .../list_tts_inference_results.rs | 83 ++++++------------- 1 file changed, 27 insertions(+), 56 deletions(-) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index d0b42e314c..516af149a9 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -7,19 +7,15 @@ use sqlx::MySqlPool; #[derive(Serialize)] pub struct TtsInferenceRecordForList { pub tts_result_token: String, - pub maybe_tts_model_token: Option, - pub maybe_tts_inference_result_token: Option, - pub model_type: Option, - pub model_title: Option, + pub tts_model_token: String, + pub inference_text: String, pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, pub file_size_bytes: u32, - pub frame_width: u32, - pub frame_height: u32, pub duration_millis: u32, //pub is_mod_hidden_from_public: bool, // converted @@ -33,19 +29,14 @@ pub struct TtsInferenceRecordForList { struct RawTtsInferenceRecordForList { pub tts_result_token: String, // from field `tts_results.token` - pub maybe_tts_model_token: Option, - pub maybe_tts_inference_result_token: Option, - - pub model_type: Option, - pub model_title: Option, // from field `tts_models.title` + pub tts_model_token: String, + pub inference_text: String, pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, pub maybe_creator_display_name: Option, pub file_size_bytes: i32, - pub frame_width: i32, - pub frame_height: i32, pub duration_millis: i32, //pub is_mod_hidden_from_public: i8, // needs convert @@ -59,16 +50,16 @@ struct RawTtsInferenceRecordForList { pub async fn list_tts_inference_results( mysql_pool: &MySqlPool, scope_creator_username: Option<&str>, - require_mod_approved_models: bool + block_mod_disabled_models: bool ) -> AnyhowResult> { let maybe_results = match scope_creator_username { Some(username) => { - list_tts_inference_results_creator_scoped(mysql_pool, username, require_mod_approved_models) + list_tts_inference_results_creator_scoped(mysql_pool, username, block_mod_disabled_models) .await }, None => { - list_tts_inference_results_for_all_creators(mysql_pool, require_mod_approved_models) + list_tts_inference_results_for_all_creators(mysql_pool, block_mod_disabled_models) .await }, }; @@ -97,11 +88,9 @@ pub async fn list_tts_inference_results( .map(|ir| { TtsInferenceRecordForList { tts_result_token: ir.tts_result_token.clone(), - maybe_tts_model_token: ir.maybe_tts_model_token.clone(), - maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), - model_type: ir.model_type.clone(), - model_title: ir.model_title.clone(), + tts_model_token: ir.tts_model_token.clone(), + inference_text: ir.inference_text.clone(), maybe_creator_user_token: ir.maybe_creator_user_token.clone(), maybe_creator_username: ir.maybe_creator_username.clone(), @@ -110,8 +99,6 @@ pub async fn list_tts_inference_results( //model_is_mod_approved: if ir.model_is_mod_approved == 0 { false } else { true }, file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, - frame_width: if ir.frame_width > 0 { ir.frame_width as u32 } else { 0 }, - frame_height: if ir.frame_height > 0 { ir.frame_height as u32 } else { 0 }, duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, created_at: ir.created_at.clone(), @@ -123,43 +110,39 @@ pub async fn list_tts_inference_results( async fn list_tts_inference_results_for_all_creators( mysql_pool: &MySqlPool, - require_mod_approved_models: bool + block_mod_disabled_models : bool ) -> AnyhowResult> { // TODO: There has to be a better way. // Sqlx doesn't like anything except string literals. - let maybe_results = if require_mod_approved_models { + let maybe_results = if block_mod_disabled_models { info!("listing tts inference results for everyone; mod-approved only"); sqlx::query_as!( RawTtsInferenceRecordForList, r#" SELECT tts_results.token as tts_result_token, - tts_results.maybe_tts_inference_result_token, - tts_models.token as maybe_tts_model_token, - tts_models.model_type, - tts_models.title as model_title, + tts_models.token as tts_model_token, + tts_results.inference_text as inference_text, users.token as maybe_creator_user_token, users.username as maybe_creator_username, users.display_name as maybe_creator_display_name, tts_results.file_size_bytes, - tts_results.frame_width, - tts_results.frame_height, tts_results.duration_millis, tts_results.created_at, tts_results.updated_at FROM tts_results LEFT OUTER JOIN tts_models - ON tts_results.maybe_tts_model_token = tts_models.token + ON tts_results.model_token = tts_models.token LEFT OUTER JOIN users ON tts_results.maybe_creator_user_token = users.token WHERE tts_results.deleted_at IS NULL AND tts_results.is_mod_hidden_from_public IS FALSE - AND tts_models.is_mod_approved IS TRUE + AND tts_models.is_mod_disabled IS FALSE "#) .fetch_all(mysql_pool) .await? @@ -170,26 +153,22 @@ WHERE r#" SELECT tts_results.token as tts_result_token, - tts_results.maybe_tts_inference_result_token, - tts_models.token as maybe_tts_model_token, - tts_models.model_type, - tts_models.title as model_title, + tts_models.token as tts_model_token, + tts_results.inference_text as inference_text, users.token as maybe_creator_user_token, users.username as maybe_creator_username, users.display_name as maybe_creator_display_name, tts_results.file_size_bytes, - tts_results.frame_width, - tts_results.frame_height, tts_results.duration_millis, tts_results.created_at, tts_results.updated_at FROM tts_results LEFT OUTER JOIN tts_models - ON tts_results.maybe_tts_model_token = tts_models.token + ON tts_results.model_token = tts_models.token LEFT OUTER JOIN users ON tts_results.maybe_creator_user_token = users.token WHERE @@ -205,43 +184,39 @@ WHERE async fn list_tts_inference_results_creator_scoped( mysql_pool: &MySqlPool, scope_creator_username: &str, - require_mod_approved_models: bool + block_mod_disabled : bool ) -> AnyhowResult> { // TODO: There has to be a better way. // Sqlx doesn't like anything except string literals. - let maybe_results = if require_mod_approved_models { + let maybe_results = if block_mod_disabled { info!("listing tts inference results for user `{}`; mod-approved only", scope_creator_username); sqlx::query_as!( RawTtsInferenceRecordForList, r#" SELECT tts_results.token as tts_result_token, - tts_results.maybe_tts_inference_result_token, - tts_models.token as maybe_tts_model_token, - tts_models.model_type, - tts_models.title as model_title, + tts_models.token as tts_model_token, + tts_results.inference_text as inference_text, users.token as maybe_creator_user_token, users.username as maybe_creator_username, users.display_name as maybe_creator_display_name, tts_results.file_size_bytes, - tts_results.frame_width, - tts_results.frame_height, tts_results.duration_millis, tts_results.created_at, tts_results.updated_at FROM tts_results LEFT OUTER JOIN tts_models - ON tts_results.maybe_tts_model_token = tts_models.token + ON tts_results.model_token = tts_models.token LEFT OUTER JOIN users ON tts_results.maybe_creator_user_token = users.token WHERE tts_results.deleted_at IS NULL AND tts_results.is_mod_hidden_from_public IS FALSE - AND tts_models.is_mod_approved IS TRUE + AND tts_models.is_mod_disabled IS FALSE AND users.username = ? "#, scope_creator_username) @@ -254,26 +229,22 @@ WHERE r#" SELECT tts_results.token as tts_result_token, - tts_results.maybe_tts_inference_result_token, - tts_models.token as maybe_tts_model_token, - tts_models.model_type, - tts_models.title as model_title, + tts_models.token as tts_model_token, + tts_results.inference_text as inference_text, users.token as maybe_creator_user_token, users.username as maybe_creator_username, users.display_name as maybe_creator_display_name, tts_results.file_size_bytes, - tts_results.frame_width, - tts_results.frame_height, tts_results.duration_millis, tts_results.created_at, tts_results.updated_at FROM tts_results LEFT OUTER JOIN tts_models - ON tts_results.maybe_tts_model_token = tts_models.token + ON tts_results.model_token = tts_models.token LEFT OUTER JOIN users ON tts_results.maybe_creator_user_token = users.token WHERE From e2e7950dd9018973815e835e08ab7b4daa2017f1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Jun 2021 03:11:07 -0400 Subject: [PATCH 0280/1042] build fixed --- src/common_queries/list_tts_inference_results.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index 516af149a9..786151b1e5 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -122,7 +122,7 @@ async fn list_tts_inference_results_for_all_creators( SELECT tts_results.token as tts_result_token, - tts_models.token as tts_model_token, + tts_results.model_token as tts_model_token, tts_results.inference_text as inference_text, users.token as maybe_creator_user_token, @@ -154,7 +154,7 @@ WHERE SELECT tts_results.token as tts_result_token, - tts_models.token as tts_model_token, + tts_results.model_token as tts_model_token, tts_results.inference_text as inference_text, users.token as maybe_creator_user_token, @@ -196,7 +196,7 @@ async fn list_tts_inference_results_creator_scoped( SELECT tts_results.token as tts_result_token, - tts_models.token as tts_model_token, + tts_results.model_token as tts_model_token, tts_results.inference_text as inference_text, users.token as maybe_creator_user_token, @@ -230,7 +230,7 @@ WHERE SELECT tts_results.token as tts_result_token, - tts_models.token as tts_model_token, + tts_results.model_token as tts_model_token, tts_results.inference_text as inference_text, users.token as maybe_creator_user_token, From 0202dd4d43a6257c467f9155c892a9d57135b645 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Jun 2021 17:25:13 -0400 Subject: [PATCH 0281/1042] fix response key --- src/http_server/endpoints/tts/get_tts_model.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http_server/endpoints/tts/get_tts_model.rs b/src/http_server/endpoints/tts/get_tts_model.rs index 079c9bd7cd..0e048d0b2f 100644 --- a/src/http_server/endpoints/tts/get_tts_model.rs +++ b/src/http_server/endpoints/tts/get_tts_model.rs @@ -51,7 +51,7 @@ pub struct TtsModelRecordForResponse { #[derive(Serialize)] pub struct GetTtsModelSuccessResponse { pub success: bool, - pub template: TtsModelRecordForResponse, + pub model: TtsModelRecordForResponse, } #[derive(Serialize)] @@ -172,7 +172,7 @@ AND tts.deleted_at IS NULL let response = GetTtsModelSuccessResponse { success: true, - template: model_for_response, + model: model_for_response, }; let body = serde_json::to_string(&response) From 19473726c69347addc99c13084304fc050c81cef Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 3 Jun 2021 20:06:43 -0400 Subject: [PATCH 0282/1042] endpoint to report status of tts inference --- migrations/2021-05-15-071534_users/up.sql | 5 +- .../2021-05-15-212913_inference_jobs/up.sql | 14 ++ migrations/2021-05-15-212923_misc/down.sql | 5 +- migrations/2021-05-15-212923_misc/up.sql | 5 +- .../endpoints/tts/enqueue_infer_tts.rs | 2 + .../tts/get_tts_inference_job_status.rs | 180 ++++++++++++++++++ src/http_server/endpoints/tts/mod.rs | 1 + src/job_queries/tts_inference_job_queries.rs | 7 +- src/main.rs | 6 + src/tts_inference_job.rs | 2 +- 10 files changed, 218 insertions(+), 9 deletions(-) create mode 100644 src/http_server/endpoints/tts/get_tts_inference_job_status.rs diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 1eae07bce6..3a22bccaf1 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -1,5 +1,6 @@ -# noinspection SqlNoDataSourceInspectionForFile -# noinspection SqlResolveForFile +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile CREATE TABLE users ( -- Not used for anything except replication. diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index ad9f95a2ea..d7fd4b163a 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -13,6 +13,12 @@ CREATE TABLE tts_inference_jobs ( -- This is so the frontend client doesn't submit duplicate jobs. uuid_idempotency_token VARCHAR(36) NOT NULL, + -- ========== SUCCESS CASE ========== + + -- If the job completes successfully, this is the result token. + -- This is only populated on a successful result. + on_success_result_token VARCHAR(32) DEFAULT NULL, + -- ========== INFERENCE DETAILS ========== -- The model to use. @@ -82,6 +88,7 @@ CREATE TABLE tts_inference_jobs ( PRIMARY KEY (id), UNIQUE KEY (token), UNIQUE KEY (uuid_idempotency_token), + KEY fk_on_success_result_token (on_success_result_token), KEY fk_model_token (model_token), KEY fk_maybe_creator_user_token (maybe_creator_user_token), KEY index_status (status), @@ -100,6 +107,12 @@ CREATE TABLE w2l_inference_jobs ( -- This is so the frontend client doesn't submit duplicate jobs. uuid_idempotency_token VARCHAR(36) NOT NULL, + -- ========== SUCCESS CASE ========== + + -- If the job completes successfully, this is the result token. + -- This is only populated on a successful result. + on_success_result_token VARCHAR(32) DEFAULT NULL, + -- ========== INFERENCE DETAILS : FACE TEMPLATE ========== -- The W2L template to use @@ -192,6 +205,7 @@ CREATE TABLE w2l_inference_jobs ( PRIMARY KEY (id), UNIQUE KEY (token), UNIQUE KEY (uuid_idempotency_token), + KEY fk_on_success_result_token (on_success_result_token), KEY fk_maybe_w2l_template_token (maybe_w2l_template_token), KEY fk_maybe_tts_inference_result_token (maybe_tts_inference_result_token), KEY fk_maybe_creator_user_token (maybe_creator_user_token), diff --git a/migrations/2021-05-15-212923_misc/down.sql b/migrations/2021-05-15-212923_misc/down.sql index 23db61a467..97075e079a 100644 --- a/migrations/2021-05-15-212923_misc/down.sql +++ b/migrations/2021-05-15-212923_misc/down.sql @@ -1,5 +1,6 @@ -# noinspection SqlResolveForFile -# noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile drop table badges; drop table user_badges; diff --git a/migrations/2021-05-15-212923_misc/up.sql b/migrations/2021-05-15-212923_misc/up.sql index 2bd06f5d62..31e2ccf155 100644 --- a/migrations/2021-05-15-212923_misc/up.sql +++ b/migrations/2021-05-15-212923_misc/up.sql @@ -1,5 +1,6 @@ -# noinspection SqlNoDataSourceInspectionForFile -# noinspection SqlResolveForFile +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile -- Badges are rewards for loyalty CREATE TABLE badges ( diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 3af94f3831..83592a1fc2 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -33,6 +33,7 @@ pub struct InferTtsRequest { #[derive(Serialize)] pub struct InferTtsSuccessResponse { pub success: bool, + pub inference_job_token: String, } #[derive(Serialize)] @@ -181,6 +182,7 @@ SET let response = InferTtsSuccessResponse { success: true, + inference_job_token: job_token.to_string(), }; let body = serde_json::to_string(&response) diff --git a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs new file mode 100644 index 0000000000..2edae26374 --- /dev/null +++ b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -0,0 +1,180 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetTtsInferenceStatusPathInfo { + token: String, +} + +#[derive(Serialize)] +pub struct TtsInferenceJobStatusForResponse { + pub job_token: String, + + pub status: String, + pub maybe_result_token: Option, + + pub model_token: String, + pub tts_model_type: String, + pub title: String, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct GetTtsInferenceStatusSuccessResponse { + pub success: bool, + pub state: TtsInferenceJobStatusForResponse, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum GetTtsInferenceStatusError { + ServerError, +} + +#[derive(Serialize)] +pub struct TtsInferenceJobStatusRecord { + pub job_token: String, + + pub status: String, + pub maybe_result_token: Option, + + pub model_token: String, + pub tts_model_type: String, + + pub title: String, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl ResponseError for GetTtsInferenceStatusError { + fn status_code(&self) -> StatusCode { + match *self { + GetTtsInferenceStatusError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetTtsInferenceStatusError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn get_tts_inference_job_status_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_status = sqlx::query_as!( + TtsInferenceJobStatusRecord, + r#" +SELECT + jobs.token as job_token, + + jobs.status, + jobs.on_success_result_token as maybe_result_token, + + jobs.model_token, + tts.tts_model_type, + tts.title, + + jobs.created_at, + jobs.updated_at +FROM tts_inference_jobs as jobs +JOIN tts_models as tts +ON tts.token = jobs.model_token +WHERE jobs.token = ? + "#, + &path.token + ) + .fetch_one(&server_state.mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let record : TtsInferenceJobStatusRecord = match maybe_status { + Ok(record) => record, + Err(err) => { + match err { + RowNotFound => { + return Err(GetTtsInferenceStatusError::ServerError); + }, + _ => { + warn!("tts template query error: {:?}", err); + return Err(GetTtsInferenceStatusError::ServerError); + } + } + } + }; + + let model_for_response = TtsInferenceJobStatusForResponse { + job_token: record.job_token.clone(), + status: record.status.clone(), + maybe_result_token: record.maybe_result_token.clone(), + model_token: record.model_token.clone(), + tts_model_type: record.tts_model_type.clone(), + title: record.title.clone(), + created_at: record.created_at.clone(), + updated_at: record.updated_at.clone(), + }; + + let response = GetTtsInferenceStatusSuccessResponse { + success: true, + state: model_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetTtsInferenceStatusError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/tts/mod.rs b/src/http_server/endpoints/tts/mod.rs index 831211eae2..0abb21694d 100644 --- a/src/http_server/endpoints/tts/mod.rs +++ b/src/http_server/endpoints/tts/mod.rs @@ -1,5 +1,6 @@ pub mod enqueue_infer_tts; pub mod enqueue_upload_tts_model; +pub mod get_tts_inference_job_status; pub mod get_tts_model; pub mod get_tts_result; pub mod list_tts_models; \ No newline at end of file diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index bb480430fe..1e7b1e03cd 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -109,7 +109,8 @@ WHERE id = ? pub async fn mark_tts_inference_job_done( pool: &MySqlPool, job: &TtsInferenceJobRecord, - success: bool + success: bool, + maybe_result_token: Option<&str> ) -> AnyhowResult<()> { let status = if success { "complete_success" } else { "complete_failure" }; @@ -118,12 +119,14 @@ pub async fn mark_tts_inference_job_done( UPDATE tts_inference_jobs SET status = ?, + on_success_result_token = ?, failure_reason = NULL, retry_at = NULL WHERE id = ? "#, status, - job.id, + maybe_result_token, + job.id ) .execute(pool) .await?; diff --git a/src/main.rs b/src/main.rs index a8ace9af4a..5b44cb9f7a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -61,6 +61,7 @@ use crate::http_server::endpoints::users::list_user_tts_inference_results::list_ use crate::http_server::endpoints::users::list_user_tts_models::list_user_tts_models_handler; use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; use crate::http_server::endpoints::tts::get_tts_model::get_tts_model_handler; +use crate::http_server::endpoints::tts::get_tts_inference_job_status::get_tts_inference_job_status_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -286,6 +287,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_tts_inference_result_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/job/{token}") + .route(web::get().to(get_tts_inference_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service( web::scope("/w2l") diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index f29682095c..9a37d0745a 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -444,7 +444,7 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An .await?; info!("Marking job complete..."); - mark_tts_inference_job_done(&inferencer.mysql_pool, job, true).await?; + mark_tts_inference_job_done(&inferencer.mysql_pool, job, true, Some(&inference_result_token)).await?; inferencer.firehose_publisher.tts_inference_finished( job.maybe_creator_user_token.as_deref(), From cda1bbda414ec95a43fb2054250f0782540fc44a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 4 Jun 2021 00:50:24 -0400 Subject: [PATCH 0283/1042] return bucket path --- .../endpoints/tts/get_tts_inference_job_status.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index 2edae26374..8e03547eed 100644 --- a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -38,6 +38,7 @@ pub struct TtsInferenceJobStatusForResponse { pub status: String, pub maybe_result_token: Option, + pub maybe_public_bucket_wav_audio_path: Option, pub model_token: String, pub tts_model_type: String, @@ -70,6 +71,7 @@ pub struct TtsInferenceJobStatusRecord { pub status: String, pub maybe_result_token: Option, + pub maybe_public_bucket_wav_audio_path: Option, pub model_token: String, pub tts_model_type: String, @@ -123,6 +125,7 @@ SELECT jobs.status, jobs.on_success_result_token as maybe_result_token, + results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path, jobs.model_token, tts.tts_model_type, @@ -130,9 +133,13 @@ SELECT jobs.created_at, jobs.updated_at + FROM tts_inference_jobs as jobs JOIN tts_models as tts -ON tts.token = jobs.model_token + ON tts.token = jobs.model_token +LEFT OUTER JOIN tts_results as results + ON jobs.on_success_result_token = results.token + WHERE jobs.token = ? "#, &path.token @@ -159,6 +166,7 @@ WHERE jobs.token = ? job_token: record.job_token.clone(), status: record.status.clone(), maybe_result_token: record.maybe_result_token.clone(), + maybe_public_bucket_wav_audio_path: record.maybe_public_bucket_wav_audio_path.clone(), model_token: record.model_token.clone(), tts_model_type: record.tts_model_type.clone(), title: record.title.clone(), From 870eba8a9c347e10af0c742fb8a4dbf42f3b440d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 4 Jun 2021 02:06:52 -0400 Subject: [PATCH 0284/1042] improvements --- migrations/2021-05-15-071601_assets/up.sql | 7 +++++++ src/http_server/endpoints/tts/get_tts_model.rs | 4 ++++ src/http_server/endpoints/tts/get_tts_result.rs | 14 ++++++++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index cd33366697..143d92895a 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -18,6 +18,13 @@ CREATE TABLE tts_models ( 'glowtts-vocodes' ) NOT NULL DEFAULT 'not-set', + -- NB: DO NOT CHANGE ORDER; APPEND ONLY! + -- How text should be handled. + text_preprocessing_algorithm ENUM( + 'basic-preprocess', + 'arpabet-preprocess' + ) NOT NULL DEFAULT 'basic-preprocess', + -- Optional Pointer to a newer version of the voice -- If there's a newer version, we can disable this one. maybe_updated_model_token VARCHAR(32) DEFAULT NULL, diff --git a/src/http_server/endpoints/tts/get_tts_model.rs b/src/http_server/endpoints/tts/get_tts_model.rs index 0e048d0b2f..9cffbab77f 100644 --- a/src/http_server/endpoints/tts/get_tts_model.rs +++ b/src/http_server/endpoints/tts/get_tts_model.rs @@ -36,6 +36,7 @@ pub struct GetTtsModelPathInfo { pub struct TtsModelRecordForResponse { pub model_token: String, pub tts_model_type: String, + pub text_preprocessing_algorithm: String, pub creator_user_token: String, pub creator_username: String, @@ -69,6 +70,7 @@ pub enum GetTtsModelError { pub struct TtsModelRecord { pub model_token: String, pub tts_model_type: String, + pub text_preprocessing_algorithm: String, pub creator_user_token: String, pub creator_username: String, @@ -122,6 +124,7 @@ pub async fn get_tts_model_handler( SELECT tts.token as model_token, tts.tts_model_type, + tts.text_preprocessing_algorithm, tts.creator_user_token, users.username as creator_username, @@ -161,6 +164,7 @@ AND tts.deleted_at IS NULL let model_for_response = TtsModelRecordForResponse { model_token: template.model_token.clone(), tts_model_type: template.tts_model_type.clone(), + text_preprocessing_algorithm: template.text_preprocessing_algorithm.clone(), creator_user_token: template.creator_user_token.clone(), creator_username: template.creator_username.clone(), creator_display_name: template.creator_display_name.clone(), diff --git a/src/http_server/endpoints/tts/get_tts_result.rs b/src/http_server/endpoints/tts/get_tts_result.rs index 977ac26623..c2ec9c531e 100644 --- a/src/http_server/endpoints/tts/get_tts_result.rs +++ b/src/http_server/endpoints/tts/get_tts_result.rs @@ -35,9 +35,10 @@ pub struct GetTtsResultPathInfo { #[derive(Serialize)] pub struct TtsResultRecordForResponse { pub tts_result_token: String, + pub inference_text: String, pub tts_model_token: String, - pub inference_text: String, + pub tts_model_title: Option, // TODO: Shouldn't be Option. pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, @@ -82,9 +83,10 @@ pub enum GetTtsResultError { pub struct RawTtsResultRecord { pub tts_result_token: String, // from field `tts_results.token` + pub inference_text: String, pub tts_model_token: String, - pub inference_text: String, + pub tts_model_title: Option, pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, @@ -151,9 +153,11 @@ pub async fn get_tts_inference_result_handler( SELECT tts_results.token as tts_result_token, - tts_results.model_token as tts_model_token, tts_results.inference_text, + tts_results.model_token as tts_model_token, + tts_models.title as tts_model_title, + users.token as maybe_creator_user_token, users.username as maybe_creator_username, users.display_name as maybe_creator_display_name, @@ -206,9 +210,11 @@ WHERE let result_for_response = TtsResultRecordForResponse { tts_result_token: ir.tts_result_token.clone(), - tts_model_token: ir.tts_model_token.clone(), inference_text: ir.inference_text.clone(), + tts_model_token: ir.tts_model_token.clone(), + tts_model_title: ir.tts_model_title.clone(), + maybe_creator_user_token: ir.maybe_creator_user_token.clone(), maybe_creator_username: ir.maybe_creator_username.clone(), maybe_creator_display_name: ir.maybe_creator_display_name.clone(), From f14a93501273c5a6dec9d42e080c1b1635c4eb7d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 4 Jun 2021 02:46:18 -0400 Subject: [PATCH 0285/1042] new endpoint to report w2l inference status --- .../w2l/get_w2l_inference_job_status.rs | 188 ++++++++++++++++++ src/http_server/endpoints/w2l/mod.rs | 1 + src/main.rs | 6 + 3 files changed, 195 insertions(+) create mode 100644 src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs diff --git a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs new file mode 100644 index 0000000000..621e6240a4 --- /dev/null +++ b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs @@ -0,0 +1,188 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetW2lInferenceStatusPathInfo { + token: String, +} + +#[derive(Serialize)] +pub struct W2lInferenceJobStatusForResponse { + pub job_token: String, + + pub status: String, + pub maybe_result_token: Option, + pub maybe_public_bucket_video_path: Option, + + pub maybe_w2l_template_token: Option, + pub w2l_template_type: String, + pub title: String, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct GetW2lInferenceStatusSuccessResponse { + pub success: bool, + pub state: W2lInferenceJobStatusForResponse, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum GetW2lInferenceStatusError { + ServerError, +} + +#[derive(Serialize)] +pub struct W2lInferenceJobStatusRecord { + pub job_token: String, + + pub status: String, + pub maybe_result_token: Option, + pub maybe_public_bucket_video_path: Option, + + pub maybe_w2l_template_token: Option, + pub w2l_template_type: String, + + pub title: String, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl ResponseError for GetW2lInferenceStatusError { + fn status_code(&self) -> StatusCode { + match *self { + GetW2lInferenceStatusError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetW2lInferenceStatusError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn get_w2l_inference_job_status_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_status = sqlx::query_as!( + W2lInferenceJobStatusRecord, + r#" +SELECT + jobs.token as job_token, + + jobs.status, + jobs.on_success_result_token as maybe_result_token, + results.public_bucket_video_path as maybe_public_bucket_video_path, + + jobs.maybe_w2l_template_token, + w2l.template_type as w2l_template_type, + w2l.title, + + jobs.created_at, + jobs.updated_at + +FROM w2l_inference_jobs as jobs +JOIN w2l_templates as w2l + ON w2l.token = jobs.maybe_w2l_template_token +LEFT OUTER JOIN w2l_results as results + ON jobs.on_success_result_token = results.token + +WHERE jobs.token = ? + "#, + &path.token + ) + .fetch_one(&server_state.mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let record : W2lInferenceJobStatusRecord = match maybe_status { + Ok(record) => record, + Err(err) => { + match err { + RowNotFound => { + return Err(GetW2lInferenceStatusError::ServerError); + }, + _ => { + warn!("w2l template query error: {:?}", err); + return Err(GetW2lInferenceStatusError::ServerError); + } + } + } + }; + + let template_for_response = W2lInferenceJobStatusForResponse { + job_token: record.job_token.clone(), + status: record.status.clone(), + maybe_result_token: record.maybe_result_token.clone(), + maybe_public_bucket_video_path: record.maybe_public_bucket_video_path.clone(), + maybe_w2l_template_token: record.maybe_w2l_template_token.clone(), + w2l_template_type: record.w2l_template_type.clone(), + title: record.title.clone(), + created_at: record.created_at.clone(), + updated_at: record.updated_at.clone(), + }; + + let response = GetW2lInferenceStatusSuccessResponse { + success: true, + state: template_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetW2lInferenceStatusError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index eded8523e8..7003d06b14 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -1,6 +1,7 @@ pub mod enqueue_infer_w2l; pub mod enqueue_infer_w2l_with_uploads; pub mod enqueue_upload_w2l_template; +pub mod get_w2l_inference_job_status; pub mod get_w2l_result; pub mod get_w2l_template; pub mod list_w2l_templates; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 5b44cb9f7a..2c13864646 100644 --- a/src/main.rs +++ b/src/main.rs @@ -62,6 +62,7 @@ use crate::http_server::endpoints::users::list_user_tts_models::list_user_tts_mo use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; use crate::http_server::endpoints::tts::get_tts_model::get_tts_model_handler; use crate::http_server::endpoints::tts::get_tts_inference_job_status::get_tts_inference_job_status_handler; +use crate::http_server::endpoints::w2l::get_w2l_inference_job_status::get_w2l_inference_job_status_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -320,6 +321,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_w2l_inference_result_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/job/{token}") + .route(web::get().to(get_w2l_inference_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service( web::resource("/profile/{username}") From 91f37486e29dbdc1a058b7b1df645b11047af57c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 4 Jun 2021 02:51:11 -0400 Subject: [PATCH 0286/1042] update job completion result token --- src/job_queries/w2l_inference_job_queries.rs | 13 ++++++++----- src/tts_inference_job.rs | 7 ++++++- src/w2l_inference_job.rs | 7 ++++++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index 21b488765d..35b9350010 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -129,11 +129,12 @@ WHERE id = ? Ok(()) } -pub async fn mark_w2l_inference_job_done(pool: &MySqlPool, - job: &W2lInferenceJobRecord, - success: bool) - -> AnyhowResult<()> -{ +pub async fn mark_w2l_inference_job_done( + pool: &MySqlPool, + job: &W2lInferenceJobRecord, + success: bool, + maybe_result_token: Option<&str> +) -> AnyhowResult<()> { let status = if success { "complete_success" } else { "complete_failure" }; let query_result = sqlx::query!( @@ -141,11 +142,13 @@ pub async fn mark_w2l_inference_job_done(pool: &MySqlPool, UPDATE w2l_inference_jobs SET status = ?, + on_success_result_token = ?, failure_reason = NULL, retry_at = NULL WHERE id = ? "#, status, + maybe_result_token, job.id, ) .execute(pool) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 9a37d0745a..d4b27cca80 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -444,7 +444,12 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An .await?; info!("Marking job complete..."); - mark_tts_inference_job_done(&inferencer.mysql_pool, job, true, Some(&inference_result_token)).await?; + mark_tts_inference_job_done( + &inferencer.mysql_pool, + job, + true, + Some(&inference_result_token) + ).await?; inferencer.firehose_publisher.tts_inference_finished( job.maybe_creator_user_token.as_deref(), diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 36bb4b2b60..425a263335 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -531,7 +531,12 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An .await?; info!("Marking job complete..."); - mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?; + mark_w2l_inference_job_done( + &inferencer.mysql_pool, + job, + true, + Some(&inference_result_token) + ).await?; inferencer.firehose_publisher.w2l_inference_finished( job.maybe_creator_user_token.as_deref(), From ce6c39b6c79127e2b1573b1b34de550f421c381f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 4 Jun 2021 03:20:23 -0400 Subject: [PATCH 0287/1042] respond with expected response --- .../endpoints/w2l/enqueue_infer_w2l_with_uploads.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 5e66b444fb..f657ecdd93 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -48,7 +48,7 @@ pub struct W2lTemplateExistenceRecord { pub struct InferW2lWithUploadSuccessResponse { pub success: bool, /// This is how frontend clients can request the job execution status. - pub job_token: String, + pub inference_job_token: String, } #[derive(Serialize)] @@ -351,7 +351,7 @@ SET let response = InferW2lWithUploadSuccessResponse { success: true, - job_token: job_token.to_string(), + inference_job_token: job_token.to_string(), }; let body = serde_json::to_string(&response) From 47c391571aa14b70f79b50cba503206ce4e31cba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 4 Jun 2021 03:39:31 -0400 Subject: [PATCH 0288/1042] update docs on denying public listing on public buckets --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index f5488157ca..b4e03839ee 100644 --- a/README.md +++ b/README.md @@ -204,3 +204,18 @@ CREATE USER 'storyteller'@'localhost' IDENTIFIED BY 'password'; GRANT ALL PRIVILEGES ON storyteller.* TO 'storyteller'@'localhost'; ``` +Production +---------- + +### Public buckets without list permission + +Public buckets that deny the `list` action should use the following Role: + +`roles/storage.legacyObjectReader` + +See: + +* https://stackoverflow.com/a/56354633 +* https://cloud.google.com/storage/docs/access-control/making-data-public#buckets + + From ec450230f8427f629db365c158178e4a1517cefe Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 4 Jun 2021 03:54:02 -0400 Subject: [PATCH 0289/1042] saves spectrogram json and uploads it! --- models/tacotron2/vocodes_inference.py | 12 +++++++++--- src/tts_inference_job.rs | 6 ++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py index e4e8e50ea2..6bda155373 100755 --- a/models/tacotron2/vocodes_inference.py +++ b/models/tacotron2/vocodes_inference.py @@ -147,9 +147,15 @@ class NumpyEncoder(json.JSONEncoder): return obj.tolist() return json.JSONEncoder.default(self, obj) -#json_dump = json.dumps({'mel': mel_outputs, 'mel_postnet': mel_outputs_postnet}, cls=NumpyEncoder) -#with open(args.output_spectrogram_filename, 'w') as outfile: -# json.dump(data, outfile) + +json_data = { + 'mel': mel_outputs.cpu().numpy(), + 'mel_postnet': mel_outputs_postnet.cpu().numpy() +} + +#json_dump = json.dumps(json_data, cls=NumpyEncoder) +with open(args.output_spectrogram_filename, 'w') as outfile: + json.dump(json_data, outfile, cls=NumpyEncoder) #torch.save(mel_outputs, 'mel_outputs.mel') #torch.save(mel_outputs_postnet, 'mel_outputs_postnet.mel') diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index d4b27cca80..8e8c3e5085 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -390,7 +390,7 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An info!("Checking that output files exist..."); check_file_exists(&output_audio_fs_path)?; - //check_file_exists(&output_spectrogram_fs_path)?; + check_file_exists(&output_spectrogram_fs_path)?; //check_file_exists(&output_metadata_fs_path)?; let file_metadata = FileMetadata::default(); @@ -414,9 +414,8 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An // ==================== UPLOAD SPECTROGRAM TO BUCKETS ==================== // let spectrogram_result_object_path = inferencer.bucket_path_unifier.tts_inference_spectrogram_output_path( - &job.inference_job_token); + &job.uuid_idempotency_token); // TODO: Don't use this! - /* info!("Spectrogram destination bucket path: {:?}", &spectrogram_result_object_path); info!("Uploading spectrogram..."); @@ -426,7 +425,6 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An &output_spectrogram_fs_path, "application/json") .await?; - */ // ==================== SAVE RECORDS ==================== // From 000f3b9ef559cc91a8e5dfc6ec007d00ace46951 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 4 Jun 2021 03:58:17 -0400 Subject: [PATCH 0290/1042] put spectrogram into correct dimensions --- models/tacotron2/vocodes_inference.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py index 6bda155373..a2ca27cb04 100755 --- a/models/tacotron2/vocodes_inference.py +++ b/models/tacotron2/vocodes_inference.py @@ -148,9 +148,11 @@ class NumpyEncoder(json.JSONEncoder): return json.JSONEncoder.default(self, obj) +# squeeze(0) -> 3D to 2D (by removing the singular 1-length "wrapper" dimension) +# transpose() -> originally 80x{N}, we turn to {N}x80 json_data = { - 'mel': mel_outputs.cpu().numpy(), - 'mel_postnet': mel_outputs_postnet.cpu().numpy() + 'mel': mel_outputs.cpu().numpy().squeeze(0).transpose(), + 'mel_postnet': mel_outputs_postnet.cpu().numpy().squeeze(0).transpose(), } #json_dump = json.dumps(json_data, cls=NumpyEncoder) From f804d0216b8f66335a3845273d0ad7b492cfe6f4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Jun 2021 14:13:31 -0400 Subject: [PATCH 0291/1042] some form of spectrogram --- models/tacotron2/vocodes_inference.py | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py index a2ca27cb04..8e96539f2b 100755 --- a/models/tacotron2/vocodes_inference.py +++ b/models/tacotron2/vocodes_inference.py @@ -148,11 +148,44 @@ class NumpyEncoder(json.JSONEncoder): return json.JSONEncoder.default(self, obj) +mel_for_scaling = mel_outputs_postnet.cpu().numpy().squeeze(0).transpose() + +""" +max_value = -10000.0 +min_value = 10000.0 +for i in range(len(mel_for_scaling)): + for j in range(len(mel_for_scaling[i])): + value = mel_for_scaling[i][j] + if value > max_value: + max_value = value + if value < min_value: + min_value = value +""" + +max_value = np.amax(mel_for_scaling) +min_value = np.amin(mel_for_scaling) +mel_range = max_value - min_value + + +print('max', max_value) +print('min', min_value) + +# https://stackoverflow.com/a/1735122 +#mel_for_scaling /= np.amax(np.abs(mel_for_scaling)) +#mel_for_scaling *= (255.0/np.amax(mel_for_scaling)) + +mel_for_scaling -= min_value +mel_for_scaling *= (255.0 / mel_range) +mel_for_scaling = mel_for_scaling.astype('int32') + # squeeze(0) -> 3D to 2D (by removing the singular 1-length "wrapper" dimension) # transpose() -> originally 80x{N}, we turn to {N}x80 json_data = { 'mel': mel_outputs.cpu().numpy().squeeze(0).transpose(), 'mel_postnet': mel_outputs_postnet.cpu().numpy().squeeze(0).transpose(), + 'mel_for_scaling': mel_for_scaling, + 'max_value': float(max_value), + 'min_value': float(min_value), } #json_dump = json.dumps(json_data, cls=NumpyEncoder) From 9c1a1e88f94a43ac3625725d7d53f549d849d907 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Jun 2021 15:36:30 -0400 Subject: [PATCH 0292/1042] simplify mel generation --- models/tacotron2/vocodes_inference.py | 100 ++++---------------------- 1 file changed, 14 insertions(+), 86 deletions(-) diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py index 8e96539f2b..54978a2b95 100755 --- a/models/tacotron2/vocodes_inference.py +++ b/models/tacotron2/vocodes_inference.py @@ -133,13 +133,13 @@ for i in text.split("\n"): with torch.no_grad(): mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence) -print("Mel outputs:") -print(mel_outputs) -print(mel_outputs.shape) -print(mel_outputs_postnet) -print(mel_outputs_postnet.shape) +#print("Mel outputs:") +#print(mel_outputs) +#print(mel_outputs.shape) +#print(mel_outputs_postnet) +#print(mel_outputs_postnet.shape) -print('Saving mels') +print('Saving mel as JSON...') class NumpyEncoder(json.JSONEncoder): def default(self, obj): @@ -148,84 +148,40 @@ class NumpyEncoder(json.JSONEncoder): return json.JSONEncoder.default(self, obj) -mel_for_scaling = mel_outputs_postnet.cpu().numpy().squeeze(0).transpose() - -""" -max_value = -10000.0 -min_value = 10000.0 -for i in range(len(mel_for_scaling)): - for j in range(len(mel_for_scaling[i])): - value = mel_for_scaling[i][j] - if value > max_value: - max_value = value - if value < min_value: - min_value = value -""" +# squeeze(0) -> 3D to 2D (by removing the singular 1-length "wrapper" dimension) +# transpose() -> originally 80x{N}, we turn to {N}x80 +mel_for_json = mel_outputs_postnet.cpu().numpy().squeeze(0).transpose() +mel_for_scaling = mel_for_json.copy() max_value = np.amax(mel_for_scaling) min_value = np.amin(mel_for_scaling) mel_range = max_value - min_value - -print('max', max_value) -print('min', min_value) - -# https://stackoverflow.com/a/1735122 -#mel_for_scaling /= np.amax(np.abs(mel_for_scaling)) -#mel_for_scaling *= (255.0/np.amax(mel_for_scaling)) - mel_for_scaling -= min_value mel_for_scaling *= (255.0 / mel_range) mel_for_scaling = mel_for_scaling.astype('int32') -# squeeze(0) -> 3D to 2D (by removing the singular 1-length "wrapper" dimension) -# transpose() -> originally 80x{N}, we turn to {N}x80 json_data = { - 'mel': mel_outputs.cpu().numpy().squeeze(0).transpose(), - 'mel_postnet': mel_outputs_postnet.cpu().numpy().squeeze(0).transpose(), - 'mel_for_scaling': mel_for_scaling, - 'max_value': float(max_value), - 'min_value': float(min_value), + 'mel': mel_for_json, + 'mel_scaled': mel_for_scaling, } -#json_dump = json.dumps(json_data, cls=NumpyEncoder) with open(args.output_spectrogram_filename, 'w') as outfile: json.dump(json_data, outfile, cls=NumpyEncoder) -#torch.save(mel_outputs, 'mel_outputs.mel') -#torch.save(mel_outputs_postnet, 'mel_outputs_postnet.mel') - -#print('Rendering histograms') -#render_histogram(mel_outputs, 'mel_outputs.png') -#render_histogram(mel_outputs_postnet, 'mel_outputs_postnet.png') +print('Encoding and saving audio...') with torch.no_grad(): - # This was from Tacotron 2's repo: - #audio = waveglow.infer(mel_outputs_postnet, sigma=0.666) - - # This is from the colab: sigma = 0.8 audio = waveglow.infer(mel_outputs_postnet, sigma=sigma) - #print("") - #ipd.display(ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate)) - -#ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate) - -# https://pytorch.org/hub/nvidia_deeplearningexamples_waveglow/ - - -#audio_numpy = audio[0].data.cpu().numpy() -#audio = audio[0].data.cpu().numpy() -#audio_numpy = audio.astype('int16') from scipy.io.wavfile import write output_audio = audio[0].data.cpu().numpy().astype(np.float32) -#audio.export('output.mp3', format="mp3", bitrate="64k") -#sf.write('bira.wav', audio[0].data.cpu().numpy().astype(np.float32), hparams.sampling_rate) write(args.output_audio_filename, hparams.sampling_rate, output_audio) +""" try: debug_location = "/home/bt/dev/storyteller/storyteller-web/test" print('Writing to debug location: {}'.format(debug_location)) @@ -235,32 +191,4 @@ except Exception as e: print("Error with saving file for local debugging.") print(e) pass - -""" -#@markdown # **Synthesis** -#@markdown # Enter your desired text here, the graph and audio will show up. - -text = "And so he ran across the street to measure the absence of being." #@param {type:"string"} -sigma = 0.8 -denoise_strength = 0.324 - -#@markdown #### Optional if you want automatic ARPA convertion. - -raw_input = True #@param {type:"boolean"} - -for i in text.split("\n"): - if len(i) < 1: - sys.exit() - print(i) - if raw_input: - if i[-1] != ";": i=i+";" - else: - i = ARPA(i) - print(i) - with torch.no_grad(): # save VRAM by not including gradients - sequence = np.array(text_to_sequence(i, ['english_cleaners']))[None, :] - sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long() - mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence) - plot_data((mel_outputs_postnet.float().data.cpu().numpy()[0], - alignments.float().data.cpu().numpy()[0].T)) """ \ No newline at end of file From dfc52681cdad992971707108e8b9347483214a5a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Jun 2021 15:47:21 -0400 Subject: [PATCH 0293/1042] calculate audio metadata --- models/Wav2Lip/vocodes_process_inference.py | 110 +----------------- .../requirements-working-2021-05-31.txt | 3 + models/tacotron2/vocodes_inference.py | 40 ++++++- src/tts_inference_job.rs | 5 +- 4 files changed, 47 insertions(+), 111 deletions(-) diff --git a/models/Wav2Lip/vocodes_process_inference.py b/models/Wav2Lip/vocodes_process_inference.py index 2348a4b976..86e891c059 100644 --- a/models/Wav2Lip/vocodes_process_inference.py +++ b/models/Wav2Lip/vocodes_process_inference.py @@ -102,94 +102,12 @@ parser.add_argument('--audio_end_pad_millis', default=0, type=int, parser.add_argument('--preserve_tempdir', default=False, action='store_true', help='Keep the tempdir arround for debugging') -#parser.add_argument('--frame_width', type=int, help='Width of the frame') -#parser.add_argument('--frame_height', type=int, help='Height of the frame') - args = parser.parse_args() args.img_size = 96 -#if os.path.isfile(args.image_or_video_filename) and os.path.splitext(args.image_or_video_filename)[1] in ['.jpg', '.png', '.jpeg']: -# args.static = True -#elif args.is_image: -# args.static = True - -#def get_smoothened_boxes(boxes, T): -# for i in range(len(boxes)): -# if i + T > len(boxes): -# window = boxes[len(boxes) - T:] -# else: -# window = boxes[i : i + T] -# boxes[i] = np.mean(window, axis=0) -# return boxes - -#def face_detect(images): -# detector = face_detection.FaceAlignment(face_detection.LandmarksType._2D, -# flip_input=False, device=device) -# -# batch_size = args.face_det_batch_size -# -# while 1: -# predictions = [] -# try: -# for i in tqdm(range(0, len(images), batch_size)): -# predictions.extend(detector.get_detections_for_batch(np.array(images[i:i + batch_size]))) -# except RuntimeError: -# if batch_size == 1: -# raise RuntimeError('Image too big to run face detection on GPU. Please use the --resize_factor argument') -# batch_size //= 2 -# print('Recovering from OOM error; New batch size: {}'.format(batch_size)) -# continue -# break -# -# results = [] -# pady1, pady2, padx1, padx2 = args.pads -# for rect, image in zip(predictions, images): -# if rect is None: -# cv2.imwrite('temp/faulty_frame.jpg', image) # check this frame where the face was not detected. -# raise ValueError('Face not detected! Ensure the video contains a face in all the frames.') -# -# y1 = max(0, rect[1] - pady1) -# y2 = min(image.shape[0], rect[3] + pady2) -# x1 = max(0, rect[0] - padx1) -# x2 = min(image.shape[1], rect[2] + padx2) -# -# results.append([x1, y1, x2, y2]) -# -# boxes = np.array(results) -# if not args.nosmooth: boxes = get_smoothened_boxes(boxes, T=5) -# results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2)] for image, (x1, y1, x2, y2) in zip(images, boxes)] -# -# del detector -# return results - -#def detect_faces_in_frames(frames, video_faces_pickle_file): -# t0 = datetime.datetime.now() -# if args.box[0] == -1: -# if not args.static: -# face_det_results = face_detect(frames) # BGR2RGB for CNN face detection -# else: -# face_det_results = face_detect([frames[0]]) -# else: -# print('Using the specified bounding box instead of face detection...') -# y1, y2, x1, x2 = args.box -# face_det_results = [[f[y1: y2, x1:x2], (y1, y2, x1, x2)] for f in frames] -# t1 = datetime.datetime.now() -# -# d = t1 - t0 -# print('Total seconds to detect faces in {} frames: {}'.format(len(frames), d.total_seconds())) -# -# with open(video_faces_pickle_file, 'wb') as f: -# print('Saving pickle file: {}'.format(video_faces_pickle_file)) -# pickle.dump(face_det_results, f) -# -# return face_det_results - -#def datagen(frames, mels): def datagen(frames, face_det_results, mels): img_batch, mel_batch, frame_batch, coords_batch = [], [], [], [] - #face_det_results = detect_faces_in_frames(frames) - for i, m in enumerate(mels): idx = 0 if args.static else i%len(frames) frame_to_save = frames[idx].copy() @@ -328,10 +246,6 @@ def maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h): def main(tempdir): - video_faces_pickle_file = args.cached_faces_filename - #video_faces_pickle_file = args.image_or_video_filename + '.faces' - #print('Video faces pickle file: {}'.format(video_faces_pickle_file), flush=True) - frame_w = 0 frame_h = 0 fps = 0 @@ -379,9 +293,6 @@ def main(tempdir): frame_h, frame_w = full_frames[0].shape[:-1] - #frame_w = args.frame_width - #frame_h = args.frame_height - print("Number of frames available for inference: "+str(len(full_frames)), flush=True) print("Frame dimensions: {}x{}".format(frame_w, frame_h), flush=True) @@ -424,31 +335,14 @@ def main(tempdir): full_frames = full_frames[:len(mel_chunks)] - #print('Face detection time...', flush=True) - #if not os.path.isfile(video_faces_pickle_file): - # print('Detecting faces...', flush=True) - # face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) - #else: - # with open(video_faces_pickle_file, 'rb') as f: - # face_det_results = pickle.load(f) - # if len(face_det_results) < len(full_frames): - # print("Face detect results have {} frames, but we need {}".format(len(face_det_results), len(full_frames))) - # print('Detecting faces again...', flush=True) - # face_det_results = detect_faces_in_frames(full_frames, video_faces_pickle_file) - # else: - # print("We don't need to find faces! Woo!", flush=True) - - # Load face detection results from pickle file video_faces_pickle_file = args.cached_faces_filename with open(video_faces_pickle_file, 'rb') as f: face_det_results = pickle.load(f) - #face_det_results = pickle.load(args.cached_faces_filename) face_det_results = face_det_results[:len(mel_chunks)] batch_size = args.wav2lip_batch_size - #gen = datagen(full_frames.copy(), mel_chunks) gen = datagen(full_frames.copy(), face_det_results.copy(), mel_chunks) output_video_filename = os.path.join(tempdir, 'result.avi') # previously 'temp/result.avi' @@ -486,6 +380,8 @@ def main(tempdir): maybe_concatenate_end_bump(tempdir, args, frame_w, frame_h) + # ==== METADATA (1) ==== + command = [ "ffprobe", "-loglevel", "quiet", @@ -499,7 +395,7 @@ def main(tempdir): out, err = pipe.communicate() ffmpeg_metadata = json.loads(out) - # ==== METADATA ==== + # ==== METADATA (2) ==== mime_type = magic.from_file(args.output_video_filename, mime=True) file_size_bytes = os.path.getsize(args.output_video_filename) diff --git a/models/tacotron2/requirements-working-2021-05-31.txt b/models/tacotron2/requirements-working-2021-05-31.txt index 091352bc0e..987c0f9960 100644 --- a/models/tacotron2/requirements-working-2021-05-31.txt +++ b/models/tacotron2/requirements-working-2021-05-31.txt @@ -42,3 +42,6 @@ Unidecode==1.0.22 Werkzeug==2.0.1 wrapt==1.12.1 zipp==3.4.1 + +# For metadata +python-magic==0.4.22 diff --git a/models/tacotron2/vocodes_inference.py b/models/tacotron2/vocodes_inference.py index 54978a2b95..cfa4d0cd91 100755 --- a/models/tacotron2/vocodes_inference.py +++ b/models/tacotron2/vocodes_inference.py @@ -33,6 +33,11 @@ from audio_processing import griffin_lim from text import text_to_sequence from denoiser import Denoiser +# For metadata +import subprocess +import magic +import os + print('========================================') print('Python interpreter', sys.executable) print('PyTorch version', torch.__version__) @@ -191,4 +196,37 @@ except Exception as e: print("Error with saving file for local debugging.") print(e) pass -""" \ No newline at end of file +""" + +# ==== METADATA (1) ==== + +command = [ + "ffprobe", + "-loglevel", "quiet", + "-print_format", "json", + "-show_format", + "-show_streams", + args.output_audio_filename +] + +pipe = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) +out, err = pipe.communicate() +ffmpeg_metadata = json.loads(out) + +# ==== METADATA (2) ==== +mime_type = magic.from_file(args.output_audio_filename, mime=True) +file_size_bytes = os.path.getsize(args.output_audio_filename) + +print(ffmpeg_metadata) + +duration_millis = int(float(ffmpeg_metadata['format']['duration']) * 1000) + +metadata = { + 'duration_millis': duration_millis, + 'mimetype': mime_type, + 'file_size_bytes': file_size_bytes, +} + +with open(args.output_metadata_filename, 'w') as json_file: + json.dump(metadata, json_file) + diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 8e8c3e5085..793ddc4123 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -391,10 +391,9 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An check_file_exists(&output_audio_fs_path)?; check_file_exists(&output_spectrogram_fs_path)?; - //check_file_exists(&output_metadata_fs_path)?; + check_file_exists(&output_metadata_fs_path)?; - let file_metadata = FileMetadata::default(); - //let file_metadata = read_metadata_file(&output_metadata_fs_path)?; + let file_metadata = read_metadata_file(&output_metadata_fs_path)?; // ==================== UPLOAD AUDIO TO BUCKET ==================== // From d5ed1d9c3b9bf105b4d6e4eb5324a86b8b004adc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Jun 2021 16:08:59 -0400 Subject: [PATCH 0294/1042] add result fields to upload jobs --- .../up.sql | 15 +- sqlx-data.json | 1924 ++++++++++++++++- src/job_queries/tts_download_job_queries.rs | 13 +- src/job_queries/w2l_download_job_queries.rs | 13 +- src/tts_download_job.rs | 26 +- src/w2l_download_job.rs | 11 +- 6 files changed, 1915 insertions(+), 87 deletions(-) diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index 1889565dbc..343326001a 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -14,6 +14,12 @@ CREATE TABLE tts_model_upload_jobs ( -- This is so the frontend client doesn't submit duplicate jobs. uuid_idempotency_token VARCHAR(36) NOT NULL, + -- ========== SUCCESS CASE ========== + + -- If the job completes successfully, this is the result token. + -- This is only populated on a successful result. + on_success_result_token VARCHAR(32) DEFAULT NULL, + -- ========== UPLOAD DETAILS ========== -- The name of the voice @@ -102,6 +108,7 @@ CREATE TABLE tts_model_upload_jobs ( PRIMARY KEY (id), UNIQUE KEY (token), UNIQUE KEY (uuid_idempotency_token), + KEY fk_on_success_result_token (on_success_result_token), KEY fk_creator_user_token (creator_user_token), KEY fk_maybe_subject_token (maybe_subject_token), KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), @@ -122,6 +129,12 @@ CREATE TABLE w2l_template_upload_jobs ( -- This is so the frontend client doesn't submit duplicate jobs. uuid_idempotency_token VARCHAR(36) NOT NULL, + -- ========== SUCCESS CASE ========== + + -- If the job completes successfully, this is the result token. + -- This is only populated on a successful result. + on_success_result_token VARCHAR(32) DEFAULT NULL, + -- ========== UPLOAD DETAILS ========== -- The title of the template @@ -207,6 +220,7 @@ CREATE TABLE w2l_template_upload_jobs ( PRIMARY KEY (id), UNIQUE KEY (token), UNIQUE KEY (uuid_idempotency_token), + KEY fk_on_success_result_token (on_success_result_token), KEY fk_creator_user_token (creator_user_token), KEY fk_maybe_subject_token (maybe_subject_token), KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), @@ -214,4 +228,3 @@ CREATE TABLE w2l_template_upload_jobs ( KEY index_creator_ip_address (creator_ip_address) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; - diff --git a/sqlx-data.json b/sqlx-data.json index 520018d8a6..18f2e3f85d 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1,5 +1,276 @@ { "db": "MySQL", + "0c32927e9292e0cf4aeb169ee3ed0ec122712a66a6850eb9e9da31d5a97e6e72": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_video_path as maybe_public_bucket_video_path,\n\n jobs.maybe_w2l_template_token,\n w2l.template_type as w2l_template_type,\n w2l.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_inference_jobs as jobs\nJOIN w2l_templates as w2l\n ON w2l.token = jobs.maybe_w2l_template_token\nLEFT OUTER JOIN w2l_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "maybe_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "maybe_public_bucket_video_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "w2l_template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 8, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + true, + true, + true, + false, + false, + false, + false + ] + } + }, + "10d2e4544e7ee74c3c394f1305e281cbc4003df207183ceaa18384c2e77baa61": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.is_mod_disabled,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n tts.deleted_at IS NULL\n AND users.username = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_mod_disabled", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, "12250420b0a3ff3bf2dd7be76daa2920777e349642245211706f32add3df442c": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.title,\n tts.updatable_slug,\n tts.private_bucket_hash,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.token = ?\nAND tts.deleted_at IS NULL\n ", "describe": { @@ -142,16 +413,6 @@ ] } }, - "1eeb7a88f5f8c2961694622b739e0b0689d89dff8a5f6b79d5a57c5b807a23f8": { - "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, "215f82f4034ccb47bfa000c3fead66c3c9f7d5caa905aa99a702d396aeeebbc6": { "query": "\nINSERT INTO tts_model_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", "describe": { @@ -162,16 +423,6 @@ "nullable": [] } }, - "289a4770fab1dd323fe18fd74efa243dec55f6df541bd193e2585971772497e5": { - "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - } - }, "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -182,6 +433,290 @@ "nullable": [] } }, + "2a00159789d38ecb28249096980d591606af8b55d60ce719d68309eb73d42bb0": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.is_mod_disabled,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n tts.deleted_at IS NULL\n AND tts.is_mod_disabled IS FALSE\n AND users.username = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_mod_disabled", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, + "2afb1d98fe7716b378778f706703dbaaa3ebb458e08ba7668a745484bce70895": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.model_token as tts_model_token,\n tts_results.inference_text as inference_text,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nWHERE\n tts_results.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 3, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + false, + false, + false, + false + ] + } + }, "2b04dd7db52ef3a97a6f420a22a4dbee7416ae58697438df69e87ffd47bbf350": { "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -1218,14 +1753,288 @@ ] } }, - "52b8b6c64114ec754d15f8931fa88c280a2a5e614a885c6be01915d195a795e2": { - "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "4b4b6c2b8306cd5b6b1fe28fd492de5e346f0a53c0efc4f0eec57c4238860764": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.model_token as tts_model_token,\n tts_results.inference_text as inference_text,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND tts_results.is_mod_hidden_from_public IS FALSE\n AND tts_models.is_mod_disabled IS FALSE\n ", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 3, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], "parameters": { - "Right": 2 + "Right": 0 }, - "nullable": [] + "nullable": [ + false, + false, + false, + true, + true, + true, + false, + false, + false, + false + ] + } + }, + "4fec6d4e2b33411835c679942860c328e20a4276fb39c3512f0608261c5467f4": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.is_mod_disabled,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE\n tts.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_mod_disabled", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] } }, "53d77ee58e9b1f7fb900914895c25bcccb9297378ff018c1e553b187d1956e92": { @@ -1292,6 +2101,16 @@ ] } }, + "65ce50a3aafe979e77a541d576228d7fcfe76e33400427d01382da36c1585edb": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, "685a1d5bb03668be301bab35d677e270c94661c53f370ecd6ce8f1b697c4a01d": { "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n ", "describe": { @@ -1486,6 +2305,158 @@ ] } }, + "68932d5d72489d06972510fb218ceeccc5c925bcd250a804c8b857502e373f18": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.is_mod_disabled,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE\n tts.deleted_at IS NULL\n AND tts.is_mod_disabled IS FALSE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "is_mod_disabled", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + } + }, + "73558104936aab6add2d9c789b368b5f155f1fe01d32e7314f2f120422866f54": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, "746526e2e150150ef804d717f743de85010972d98cbb137e44d8348cfdb4dddc": { "query": "\nINSERT INTO w2l_results\nSET\n token = ?,\n maybe_w2l_template_token = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_video_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n duration_millis = ?\n ", "describe": { @@ -1538,6 +2509,18 @@ }, { "ordinal": 3, + "name": "on_success_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, "name": "title", "type_info": { "type": "VarString", @@ -1549,7 +2532,7 @@ } }, { - "ordinal": 4, + "ordinal": 5, "name": "tts_model_type", "type_info": { "type": "String", @@ -1561,7 +2544,7 @@ } }, { - "ordinal": 5, + "ordinal": 6, "name": "maybe_subject_token", "type_info": { "type": "VarString", @@ -1573,7 +2556,7 @@ } }, { - "ordinal": 6, + "ordinal": 7, "name": "maybe_actor_subject_token", "type_info": { "type": "VarString", @@ -1585,7 +2568,7 @@ } }, { - "ordinal": 7, + "ordinal": 8, "name": "download_url", "type_info": { "type": "VarString", @@ -1597,7 +2580,7 @@ } }, { - "ordinal": 8, + "ordinal": 9, "name": "download_url_type", "type_info": { "type": "String", @@ -1609,7 +2592,7 @@ } }, { - "ordinal": 9, + "ordinal": 10, "name": "creator_user_token", "type_info": { "type": "VarString", @@ -1621,7 +2604,7 @@ } }, { - "ordinal": 10, + "ordinal": 11, "name": "creator_ip_address", "type_info": { "type": "VarString", @@ -1633,7 +2616,7 @@ } }, { - "ordinal": 11, + "ordinal": 12, "name": "creator_set_visibility", "type_info": { "type": "String", @@ -1645,7 +2628,7 @@ } }, { - "ordinal": 12, + "ordinal": 13, "name": "status", "type_info": { "type": "String", @@ -1657,7 +2640,7 @@ } }, { - "ordinal": 13, + "ordinal": 14, "name": "attempt_count", "type_info": { "type": "Long", @@ -1669,7 +2652,7 @@ } }, { - "ordinal": 14, + "ordinal": 15, "name": "failure_reason", "type_info": { "type": "VarString", @@ -1681,7 +2664,7 @@ } }, { - "ordinal": 15, + "ordinal": 16, "name": "created_at", "type_info": { "type": "Timestamp", @@ -1693,7 +2676,7 @@ } }, { - "ordinal": 16, + "ordinal": 17, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -1705,7 +2688,7 @@ } }, { - "ordinal": 17, + "ordinal": 18, "name": "retry_at", "type_info": { "type": "Timestamp", @@ -1724,6 +2707,7 @@ false, false, false, + true, false, false, true, @@ -2169,6 +3153,252 @@ ] } }, + "9c352e6fe6166ee18326908ef02dd61866d926ec63599aedf9f91a9d05d5c630": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND tts_results.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 2, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "tts_model_title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 8, + "name": "maybe_model_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "maybe_model_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 10, + "name": "maybe_model_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 11, + "name": "maybe_model_creator_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 12, + "name": "public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 13, + "name": "public_bucket_spectrogram_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 14, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 15, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 16, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 17, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ] + } + }, "9d4932baa4023accc2d28af98b58a47149d37c4937cd0171e6b955c09855c5f8": { "query": "\nINSERT INTO w2l_inference_jobs\nSET\n maybe_w2l_template_token = ?,\n maybe_tts_inference_result_token = ?,\n maybe_public_audio_bucket_location = NULL,\n maybe_public_image_bucket_location = NULL,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n status = \"pending\"\n ", "describe": { @@ -2189,6 +3419,145 @@ "nullable": [] } }, + "a125f183f7ae5627fc2118e7302cea73d967200ba1cbb12550f5880187b7db5e": { + "query": "\nUPDATE w2l_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, + "ab4913d3a45a2ee16110021823a0786a22aa561a55aafaee27851da1e0a67e12": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,\n\n jobs.model_token,\n tts.tts_model_type,\n tts.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_inference_jobs as jobs\nJOIN tts_models as tts\n ON tts.token = jobs.model_token\nLEFT OUTER JOIN tts_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "maybe_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "maybe_public_bucket_wav_audio_path", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 5, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 8, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + true, + true, + false, + false, + false, + false, + false + ] + } + }, "ab78a81f85310750668e2c7de920c067290f2b004561ab706a5edfd893fd06fe": { "query": "\nINSERT INTO w2l_templates\nSET\n token = ?,\n template_type = ?,\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n private_bucket_cached_faces_object_name = ?,\n maybe_public_bucket_preview_image_object_name = ?,\n maybe_public_bucket_preview_video_object_name = ?,\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n frame_count = ?,\n fps = ?,\n duration_millis = ?\n ", "describe": { @@ -2199,6 +3568,158 @@ "nullable": [] } }, + "ae2c639b0b9d4efd8734c23d5fd52178df985b4258ba7f3786fd609bfd647761": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.model_token as tts_model_token,\n tts_results.inference_text as inference_text,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND users.username = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 3, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + false, + false, + false, + false + ] + } + }, + "b1ad806aa93f66cab358d7cdb405d57c643e2ff131cfec9f1ced55c2e6ca9511": { + "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } + }, "b7535ecf2e63b7f02b5b96f833fd8cd28285b24473df6f89795ff360c1723215": { "query": "\nSELECT *\nFROM w2l_template_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { @@ -2241,6 +3762,18 @@ }, { "ordinal": 3, + "name": "on_success_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, "name": "title", "type_info": { "type": "VarString", @@ -2252,7 +3785,7 @@ } }, { - "ordinal": 4, + "ordinal": 5, "name": "template_type", "type_info": { "type": "String", @@ -2264,7 +3797,7 @@ } }, { - "ordinal": 5, + "ordinal": 6, "name": "maybe_subject_token", "type_info": { "type": "VarString", @@ -2276,7 +3809,7 @@ } }, { - "ordinal": 6, + "ordinal": 7, "name": "maybe_actor_subject_token", "type_info": { "type": "VarString", @@ -2288,7 +3821,7 @@ } }, { - "ordinal": 7, + "ordinal": 8, "name": "download_url", "type_info": { "type": "VarString", @@ -2300,7 +3833,7 @@ } }, { - "ordinal": 8, + "ordinal": 9, "name": "download_url_type", "type_info": { "type": "String", @@ -2312,7 +3845,7 @@ } }, { - "ordinal": 9, + "ordinal": 10, "name": "creator_user_token", "type_info": { "type": "VarString", @@ -2324,7 +3857,7 @@ } }, { - "ordinal": 10, + "ordinal": 11, "name": "creator_ip_address", "type_info": { "type": "VarString", @@ -2336,7 +3869,7 @@ } }, { - "ordinal": 11, + "ordinal": 12, "name": "creator_set_visibility", "type_info": { "type": "String", @@ -2348,7 +3881,7 @@ } }, { - "ordinal": 12, + "ordinal": 13, "name": "status", "type_info": { "type": "String", @@ -2360,7 +3893,7 @@ } }, { - "ordinal": 13, + "ordinal": 14, "name": "attempt_count", "type_info": { "type": "Long", @@ -2372,7 +3905,7 @@ } }, { - "ordinal": 14, + "ordinal": 15, "name": "failure_reason", "type_info": { "type": "VarString", @@ -2384,7 +3917,7 @@ } }, { - "ordinal": 15, + "ordinal": 16, "name": "created_at", "type_info": { "type": "Timestamp", @@ -2396,7 +3929,7 @@ } }, { - "ordinal": 16, + "ordinal": 17, "name": "updated_at", "type_info": { "type": "Timestamp", @@ -2408,7 +3941,7 @@ } }, { - "ordinal": 17, + "ordinal": 18, "name": "retry_at", "type_info": { "type": "Timestamp", @@ -2427,6 +3960,7 @@ false, false, false, + true, false, false, true, @@ -2465,14 +3999,146 @@ "nullable": [] } }, - "c11318a1a944cd74f1210b211e19dfaa12652bad2a5d79b010f6efa62f3fa8a9": { - "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", + "c1a4f3667de265a6f501b141cc35c44ce7b3703627c7d8336d0ef8445962ed34": { + "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.text_preprocessing_algorithm,\n\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n\n tts.updatable_slug,\n tts.title,\n\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE tts.updatable_slug = ?\nAND tts.deleted_at IS NULL\n ", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_type", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 60 + } + }, + { + "ordinal": 2, + "name": "text_preprocessing_algorithm", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 72 + } + }, + { + "ordinal": 3, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 7, + "name": "title", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], "parameters": { - "Right": 2 + "Right": 1 }, - "nullable": [] + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] } }, "c294456f67ddbc875cc5c09fc66451172c03a78d1b83ceb75563205aae36e2e0": { @@ -4322,6 +5988,148 @@ ] } }, + "ee3925235a4f3b0a5d6ec0053b15fb9a2d6af28ca1039e2563d7d53430078986": { + "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.model_token as tts_model_token,\n tts_results.inference_text as inference_text,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND tts_results.is_mod_hidden_from_public IS FALSE\n AND tts_models.is_mod_disabled IS FALSE\n AND users.username = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tts_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "inference_text", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 3, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 6, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 7, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + false, + false, + false, + false + ] + } + }, "f00402e757b7f5d8ded60e78ac47d93ccf4c71a2acba9dddcedfed0ac4d2e3a5": { "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_results.public_bucket_video_path,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n template_users.token as maybe_template_creator_user_token,\n template_users.username as maybe_template_creator_username,\n template_users.display_name as maybe_template_creator_display_name,\n template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as template_users\n ON w2l_templates.creator_user_token = template_users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND w2l_results.token = ?\n ", "describe": { diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index b7562d2730..ff9f86ddab 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -15,6 +15,7 @@ pub struct TtsUploadJobRecord { pub id: i64, pub token: String, pub uuid_idempotency_token: String, + pub on_success_result_token: Option, pub creator_user_token: String, pub creator_ip_address: String, pub creator_set_visibility: String, // TODO @@ -86,10 +87,12 @@ WHERE id = ? Ok(()) } -pub async fn mark_tts_upload_job_done(pool: &MySqlPool, - job: &TtsUploadJobRecord, - success: bool) - -> AnyhowResult<()> +pub async fn mark_tts_upload_job_done( + pool: &MySqlPool, + job: &TtsUploadJobRecord, + success: bool, + maybe_model_token: Option<&str>, +) -> AnyhowResult<()> { let status = if success { "complete_success" } else { "complete_failure" }; @@ -98,11 +101,13 @@ pub async fn mark_tts_upload_job_done(pool: &MySqlPool, UPDATE tts_model_upload_jobs SET status = ?, + on_success_result_token = ?, failure_reason = NULL, retry_at = NULL WHERE id = ? "#, status, + maybe_model_token, job.id, ) .execute(pool) diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index d3c782ea75..4d37117a30 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -15,6 +15,7 @@ pub struct W2lTemplateUploadJobRecord { pub id: i64, pub token: String, pub uuid_idempotency_token: String, + pub on_success_result_token: Option, pub creator_user_token: String, pub creator_ip_address: String, pub creator_set_visibility: String, // TODO @@ -86,10 +87,12 @@ WHERE id = ? Ok(()) } -pub async fn mark_w2l_template_upload_job_done(pool: &MySqlPool, - job: &W2lTemplateUploadJobRecord, - success: bool) - -> AnyhowResult<()> +pub async fn mark_w2l_template_upload_job_done( + pool: &MySqlPool, + job: &W2lTemplateUploadJobRecord, + success: bool, + maybe_model_token: Option<&str> +) -> AnyhowResult<()> { let status = if success { "complete_success" } else { "complete_failure" }; @@ -98,11 +101,13 @@ pub async fn mark_w2l_template_upload_job_done(pool: &MySqlPool, UPDATE w2l_template_upload_jobs SET status = ?, + on_success_result_token = ?, failure_reason = NULL, retry_at = NULL WHERE id = ? "#, status, + maybe_model_token, job.id, ) .execute(pool) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 13e03a1e0c..03da552996 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -227,23 +227,6 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("File hash: {}", private_bucket_hash); - //// NB: /.../a/b/c/d/abcdefg.bin - //let object_name = hash_to_bucket_path( - // &private_bucket_hash, - // Some(&downloader.bucket_root_tts_model_uploads))?; - - // [2021-06-02T04:38:28Z INFO tts_download_job::buckets::bucket_client] Uploading... - // [2021-06-02T04:38:28Z INFO tts_download_job::buckets::bucket_client] - // Filename for bucket: /user_uploaded_tts_synthesizers/9/5/7/957dfe3f2fcf36ebcf13a6e3bc52fb2905ccd7b1834076bd6c3c41ace272640a.pt - // [2021-06-02T04:38:28Z INFO tts_download_job::buckets::bucket_client] - // Rooted filename for bucket: /user_uploaded_tts_synthesizers/9/5/7/957dfe3f2fcf36ebcf13a6e3bc52fb2905ccd7b1834076bd6c3c41ace272640a.pt - - // 2021-06-02T04:40:24Z INFO tts_inference_job] TTS synthesizer model file does not exist: - // "/tmp/tts/synthesizer_models/TTS_MDL:BSBQ48NWNFJFH5PSD3MCSCF2" - // [2021-06-02T04:40:24Z INFO tts_inference_job] - // Download from template media path: "/user_uploaded_tts_synthesizers/9/5/7/957dfe3f2fcf36ebcf13a6e3bc52fb2905ccd7b1834076bd6c3c41ace272640a.pt" - // [2021-06-02T04:40:24Z INFO tts_inference_job::buckets::bucket_client] downloading from bucket: - // "/user_uploaded_tts_synthesizers/9/5/7/957dfe3f2fcf36ebcf13a6e3bc52fb2905ccd7b1834076bd6c3c41ace272640a.pt" let synthesizer_model_bucket_path = downloader.bucket_path_unifier.tts_synthesizer_path( &private_bucket_hash); @@ -260,6 +243,14 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho synthesizer_model_bucket_path.as_path().to_str().unwrap_or("")) .await?; + info!("Marking job complete..."); + mark_tts_upload_job_done( + &downloader.mysql_pool, + job, + true, + Some(&model_token) + ).await?; + info!("Saved model record: {}", id); downloader.firehose_publisher.publish_tts_model_upload_finished(&job.creator_user_token, &model_token) @@ -270,7 +261,6 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho })?; info!("Job done: {}", job.id); - mark_tts_upload_job_done(&downloader.mysql_pool, job, true).await?; Ok(()) } diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 93f14e6b5a..62b4bc27b0 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -503,6 +503,15 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) file_metadata.duration_millis.unwrap_or(0)) .await?; + info!("Marking job complete..."); + + mark_w2l_template_upload_job_done( + &downloader.mysql_pool, + job, + true, + Some(&model_token) + ).await?; + info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}", job.id, id); @@ -513,8 +522,6 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) anyhow!("error publishing event") })?; - mark_w2l_template_upload_job_done(&downloader.mysql_pool, job, true).await?; - if downloader.debug_job_end_sleep_millis != 0 { warn!("Debug sleep after job end: {} ms", downloader.debug_job_end_sleep_millis); thread::sleep(Duration::from_millis(downloader.debug_job_end_sleep_millis)); From 3aded66c09f2f26864e7d015f4acaa2c594a7a0c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Jun 2021 16:38:19 -0400 Subject: [PATCH 0295/1042] ban slurs from being said --- db/banned_slurs.txt | 8 ++++ src/buckets/bucket_path_unifier.rs | 2 +- .../endpoints/tts/enqueue_infer_tts.rs | 5 +++ src/main.rs | 1 + src/validations/check_for_slurs.rs | 39 +++++++++++++++++++ src/validations/mod.rs | 1 + 6 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 db/banned_slurs.txt create mode 100644 src/validations/check_for_slurs.rs diff --git a/db/banned_slurs.txt b/db/banned_slurs.txt new file mode 100644 index 0000000000..1bb81205b5 --- /dev/null +++ b/db/banned_slurs.txt @@ -0,0 +1,8 @@ +# This file serves to remove racial slurs and epithets from usage with vocodes. +# There's probably more than this that we'll want to remove. +# I need to investigate. +fag +faggot +fags +nigger +niggers \ No newline at end of file diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs index 927d8b887a..3fb2c9ec29 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/buckets/bucket_path_unifier.rs @@ -192,7 +192,7 @@ mod tests { fn test_tts_synthesizer_path() { let paths = get_instance(); assert_eq!(paths.tts_synthesizer_path("foobar").to_str().unwrap(), - "/test_path_synthesizers/f/o/o/foobar.pth"); + "/test_path_synthesizers/f/o/o/foobar.pt"); } #[test] diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 83592a1fc2..7bbbafc9c5 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -19,6 +19,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::validations::check_for_slurs::contains_slurs; const NEW_USER_ROLE: &'static str = "new-user"; @@ -102,6 +103,10 @@ pub async fn infer_tts_handler( return Err(InferTtsError::BadInput(reason)); } + if contains_slurs(&inference_text) { + return Err(InferTtsError::BadInput("text contains slurs".to_string())); + } + // TODO(bt): CHECK DATABASE! let model_token = request.tts_model_token.to_string(); diff --git a/src/main.rs b/src/main.rs index 2c13864646..4501a51f42 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ pub const RESERVED_USERNAMES : &'static str = include_str!("../db/reserved_usernames.txt"); pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../db/reserved_usernames_including.txt"); +pub const BANNED_SLURS : &'static str = include_str!("../db/banned_slurs.txt"); pub mod buckets; pub mod common_queries; diff --git a/src/validations/check_for_slurs.rs b/src/validations/check_for_slurs.rs new file mode 100644 index 0000000000..c89f4b5245 --- /dev/null +++ b/src/validations/check_for_slurs.rs @@ -0,0 +1,39 @@ +use std::collections::HashSet; +use crate::BANNED_SLURS; + +pub fn contains_slurs(unparsed_text: &str) -> bool { + lazy_static! { + static ref BANNED_SLURS_SET : HashSet = BANNED_SLURS.lines() + .map(|line| line.trim()) + .filter(|line| !(line.starts_with("#") || line.is_empty())) + .map(|line| line.to_string()) + .collect::>(); + } + + for wordlike in unparsed_text.split_ascii_whitespace() { + if BANNED_SLURS_SET.contains(wordlike) { + return true; + } + } + + false +} + +#[cfg(test)] +mod tests { + use crate::validations::check_for_slurs::contains_slurs; + + #[test] + fn valid_text_passes() { + assert_eq!(contains_slurs(""), false); + assert_eq!(contains_slurs("this is a test."), false); + assert_eq!(contains_slurs("this\nis\na\ntest\n\n"), false); + assert_eq!(contains_slurs(" this is a test"), false); + } + + #[test] + fn text_with_slurs_fails() { + assert_eq!(contains_slurs("a sentence containing fag is banned"), true); + assert_eq!(contains_slurs("a\nsentence\ncontaining fags\nis banned"), true); + } +} diff --git a/src/validations/mod.rs b/src/validations/mod.rs index fa53768548..2b24d755b8 100644 --- a/src/validations/mod.rs +++ b/src/validations/mod.rs @@ -1,3 +1,4 @@ +pub mod check_for_slurs; pub mod model_uploads; pub mod passwords; pub mod username; From 7638c640ff637a68bb07d324b78313405b94902d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Jun 2021 23:37:28 -0400 Subject: [PATCH 0296/1042] move profile URL --- src/main.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 4501a51f42..0953d141b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -242,6 +242,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .wrap(DefaultHeaders::new() .header("X-Backend-Hostname", &hostname) .header("X-Build-Sha", "")) + // ==================== ACCOUNT CREATION / SESSION MANAGEMENT ==================== .service( web::resource("/create_account") .route(web::post().to(create_account_handler)) @@ -262,6 +263,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(session_info_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + // ==================== TTS ==================== .service( web::scope("/tts") .service( @@ -295,6 +297,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) ) + // ==================== WAV2LIP ==================== .service( web::scope("/w2l") .service( @@ -328,13 +331,14 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) ) - .service( - web::resource("/profile/{username}") - .route(web::get().to(get_profile_handler)) - .route(web::head().to(|| HttpResponse::Ok())) - ) + // ==================== USER DATA ==================== .service( web::scope("/user") + .service( + web::resource("/{username}/profile") + .route(web::get().to(get_profile_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/{username}/tts_models") .route(web::get().to(list_user_tts_models_handler)) From 004bc71e96dadcc357b80c10f4dbf54010a3c54c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Jun 2021 23:49:16 -0400 Subject: [PATCH 0297/1042] TTS upload job status --- .../tts/get_tts_upload_model_job_status.rs | 164 ++++++++++++++++++ src/http_server/endpoints/tts/mod.rs | 1 + src/main.rs | 6 + 3 files changed, 171 insertions(+) create mode 100644 src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs diff --git a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs new file mode 100644 index 0000000000..24458b5098 --- /dev/null +++ b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs @@ -0,0 +1,164 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetTtsUploadModelStatusPathInfo { + token: String, +} + +#[derive(Serialize)] +pub struct TtsUploadModelJobStatusForResponse { + pub job_token: String, + + pub status: String, + pub maybe_model_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct GetTtsUploadModelStatusSuccessResponse { + pub success: bool, + pub state: TtsUploadModelJobStatusForResponse, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum GetTtsUploadModelStatusError { + ServerError, +} + +#[derive(Serialize)] +pub struct TtsUploadModelJobStatusRecord { + pub job_token: String, + + pub status: String, + pub maybe_model_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl ResponseError for GetTtsUploadModelStatusError { + fn status_code(&self) -> StatusCode { + match *self { + GetTtsUploadModelStatusError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetTtsUploadModelStatusError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn get_tts_upload_model_job_status_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_status = sqlx::query_as!( + TtsUploadModelJobStatusRecord, + r#" +SELECT + jobs.token as job_token, + + jobs.status, + jobs.on_success_result_token as maybe_model_token, + + jobs.created_at, + jobs.updated_at + +FROM tts_model_upload_jobs as jobs + +WHERE jobs.token = ? + "#, + &path.token + ) + .fetch_one(&server_state.mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let record : TtsUploadModelJobStatusRecord = match maybe_status { + Ok(record) => record, + Err(err) => { + match err { + RowNotFound => { + return Err(GetTtsUploadModelStatusError::ServerError); + }, + _ => { + warn!("tts template query error: {:?}", err); + return Err(GetTtsUploadModelStatusError::ServerError); + } + } + } + }; + + let model_for_response = TtsUploadModelJobStatusForResponse { + job_token: record.job_token.clone(), + status: record.status.clone(), + maybe_model_token: record.maybe_model_token.clone(), + created_at: record.created_at.clone(), + updated_at: record.updated_at.clone(), + }; + + let response = GetTtsUploadModelStatusSuccessResponse { + success: true, + state: model_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetTtsUploadModelStatusError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/tts/mod.rs b/src/http_server/endpoints/tts/mod.rs index 0abb21694d..bdff9c584d 100644 --- a/src/http_server/endpoints/tts/mod.rs +++ b/src/http_server/endpoints/tts/mod.rs @@ -3,4 +3,5 @@ pub mod enqueue_upload_tts_model; pub mod get_tts_inference_job_status; pub mod get_tts_model; pub mod get_tts_result; +pub mod get_tts_upload_model_job_status; pub mod list_tts_models; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 0953d141b3..0a92d3575d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -64,6 +64,7 @@ use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result use crate::http_server::endpoints::tts::get_tts_model::get_tts_model_handler; use crate::http_server::endpoints::tts::get_tts_inference_job_status::get_tts_inference_job_status_handler; use crate::http_server::endpoints::w2l::get_w2l_inference_job_status::get_w2l_inference_job_status_handler; +use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -296,6 +297,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_tts_inference_job_status_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/upload_model_job/{token}") + .route(web::get().to(get_tts_upload_model_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) // ==================== WAV2LIP ==================== .service( From d7a9f88493052a2165264222da5d6c89785cb7cb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 00:14:51 -0400 Subject: [PATCH 0298/1042] status check endpoint for w2l upload --- .../w2l/get_w2l_upload_template_job_status.rs | 164 ++++++++++++++++++ src/http_server/endpoints/w2l/mod.rs | 1 + src/main.rs | 6 + 3 files changed, 171 insertions(+) create mode 100644 src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs diff --git a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs new file mode 100644 index 0000000000..6a368428e3 --- /dev/null +++ b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs @@ -0,0 +1,164 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; +use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::session_checker::SessionRecord; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetW2lUploadTemplateStatusPathInfo { + token: String, +} + +#[derive(Serialize)] +pub struct W2lUploadTemplateJobStatusForResponse { + pub job_token: String, + + pub status: String, + pub maybe_template_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct GetW2lUploadTemplateStatusSuccessResponse { + pub success: bool, + pub state: W2lUploadTemplateJobStatusForResponse, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum GetW2lUploadTemplateStatusError { + ServerError, +} + +#[derive(Serialize)] +pub struct W2lUploadTemplateJobStatusRecord { + pub job_token: String, + + pub status: String, + pub maybe_template_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl ResponseError for GetW2lUploadTemplateStatusError { + fn status_code(&self) -> StatusCode { + match *self { + GetW2lUploadTemplateStatusError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetW2lUploadTemplateStatusError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn get_w2l_upload_template_job_status_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_status = sqlx::query_as!( + W2lUploadTemplateJobStatusRecord, + r#" +SELECT + jobs.token as job_token, + + jobs.status, + jobs.on_success_result_token as maybe_template_token, + + jobs.created_at, + jobs.updated_at + +FROM w2l_template_upload_jobs as jobs + +WHERE jobs.token = ? + "#, + &path.token + ) + .fetch_one(&server_state.mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let record : W2lUploadTemplateJobStatusRecord = match maybe_status { + Ok(record) => record, + Err(err) => { + match err { + RowNotFound => { + return Err(GetW2lUploadTemplateStatusError::ServerError); + }, + _ => { + warn!("w2l template query error: {:?}", err); + return Err(GetW2lUploadTemplateStatusError::ServerError); + } + } + } + }; + + let template_for_response = W2lUploadTemplateJobStatusForResponse { + job_token: record.job_token.clone(), + status: record.status.clone(), + maybe_template_token: record.maybe_template_token.clone(), + created_at: record.created_at.clone(), + updated_at: record.updated_at.clone(), + }; + + let response = GetW2lUploadTemplateStatusSuccessResponse { + success: true, + state: template_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetW2lUploadTemplateStatusError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index 7003d06b14..6d0065f8da 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -4,4 +4,5 @@ pub mod enqueue_upload_w2l_template; pub mod get_w2l_inference_job_status; pub mod get_w2l_result; pub mod get_w2l_template; +pub mod get_w2l_upload_template_job_status; pub mod list_w2l_templates; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 0a92d3575d..7a7d0e5329 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,6 +65,7 @@ use crate::http_server::endpoints::tts::get_tts_model::get_tts_model_handler; use crate::http_server::endpoints::tts::get_tts_inference_job_status::get_tts_inference_job_status_handler; use crate::http_server::endpoints::w2l::get_w2l_inference_job_status::get_w2l_inference_job_status_handler; use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; +use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -336,6 +337,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_w2l_inference_job_status_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/upload_template_job/{token}") + .route(web::get().to(get_w2l_upload_template_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) // ==================== USER DATA ==================== .service( From 802a348fb55d6db32154831443519833c44ceaac Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 00:56:43 -0400 Subject: [PATCH 0299/1042] re-codegen queries --- sqlx-data.json | 154 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/sqlx-data.json b/sqlx-data.json index 18f2e3f85d..21a328c199 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -129,6 +129,83 @@ ] } }, + "0fc26cb6ab34ee6cd9d0cd8fac8bae7bd6f25bbf7c785cd5dc8888d08099b88b": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_template_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_template_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "maybe_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + true, + false, + false + ] + } + }, "10d2e4544e7ee74c3c394f1305e281cbc4003df207183ceaa18384c2e77baa61": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.is_mod_disabled,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON\n users.token = tts.creator_user_token\nWHERE\n tts.deleted_at IS NULL\n AND users.username = ?\n ", "describe": { @@ -1352,6 +1429,83 @@ ] } }, + "3de0e9c442ae0d4ca2f1da19357ac7685084e2843216313d7836ea60bc3dcfe0": { + "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_model_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_model_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 2, + "name": "maybe_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 4, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + true, + false, + false + ] + } + }, "3f40b37731f6049ef9167a8912e411123053ccdfebd1564e0a8f257c3a2db0b9": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND users.username = ?\n ", "describe": { From 4bc948d0ac56225fa40d2d51f11ca5ec55d39416 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 17:23:39 -0400 Subject: [PATCH 0300/1042] A ton of new functionality to support editing profiles --- Cargo.lock | 28 ++ Cargo.toml | 3 + migrations/2021-05-15-071534_users/up.sql | 3 + src/common_queries/mod.rs | 4 +- src/common_queries/query_user_profile.rs | 138 ++++++++ src/common_queries/users.rs | 0 .../endpoints/users/edit_profile.rs | 299 ++++++++++++++++++ .../endpoints/users/get_profile.rs | 2 + src/http_server/endpoints/users/mod.rs | 1 + src/http_server/web_utils/session_checker.rs | 167 +++++++--- src/main.rs | 6 + src/util/markdown_to_html.rs | 14 + src/util/mod.rs | 3 +- src/validations/cashapp_username.rs | 42 +++ src/validations/discord_username.rs | 40 +++ src/validations/github_username.rs | 41 +++ src/validations/mod.rs | 8 +- src/validations/twitch_username.rs | 43 +++ src/validations/twitter_username.rs | 47 +++ src/validations/website_url.rs | 40 +++ 20 files changed, 886 insertions(+), 43 deletions(-) create mode 100644 src/common_queries/query_user_profile.rs delete mode 100644 src/common_queries/users.rs create mode 100644 src/http_server/endpoints/users/edit_profile.rs create mode 100644 src/util/markdown_to_html.rs create mode 100644 src/validations/cashapp_username.rs create mode 100644 src/validations/discord_username.rs create mode 100644 src/validations/github_username.rs create mode 100644 src/validations/twitch_username.rs create mode 100644 src/validations/twitter_username.rs create mode 100644 src/validations/website_url.rs diff --git a/Cargo.lock b/Cargo.lock index 751c664262..325243bfda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1351,6 +1351,15 @@ dependencies = [ "version_check 0.9.3", ] +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -2279,6 +2288,18 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "pulldown-cmark" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffade02495f22453cd593159ea2f59827aae7f53fa8323f756799b670881dcf8" +dependencies = [ + "bitflags", + "getopts", + "memchr", + "unicase", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -3026,6 +3047,7 @@ dependencies = [ "lazy_static", "log", "md-5", + "pulldown-cmark", "rand 0.8.3", "regex", "ring", @@ -3460,6 +3482,12 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + [[package]] name = "unicode-xid" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index e9dbfc8a47..e2b7c33fe9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,6 +99,9 @@ serde_derive = "1.0.125" serde_json = "1.0.64" toml = "0.5.8" +# Text processing +pulldown-cmark = "0.8.0" + [build-dependencies] # NB: Only using diesel for migrations support diesel = { version = "1.4.6", features = ["mysql"] } diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 3a22bccaf1..f6d367a8c2 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -87,6 +87,8 @@ CREATE TABLE users ( twitch_username VARCHAR(36) DEFAULT NULL, patreon_username VARCHAR(36) DEFAULT NULL, github_username VARCHAR(36) DEFAULT NULL, + cashapp_username VARCHAR(36) DEFAULT NULL, + website_url VARCHAR(255) DEFAULT NULL, -- Incremented with every update. version INT NOT NULL DEFAULT 0, @@ -118,6 +120,7 @@ CREATE TABLE user_roles ( can_ban_users BOOLEAN NOT NULL DEFAULT FALSE, can_edit_other_users_data BOOLEAN NOT NULL DEFAULT FALSE, + can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE, can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index 26dfb92570..30aab5e19b 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -4,5 +4,5 @@ pub mod list_tts_inference_results; pub mod list_tts_models; pub mod list_w2l_inference_results; pub mod list_w2l_templates; -pub mod sessions; -pub mod users; \ No newline at end of file +pub mod query_user_profile; +pub mod sessions; \ No newline at end of file diff --git a/src/common_queries/query_user_profile.rs b/src/common_queries/query_user_profile.rs new file mode 100644 index 0000000000..df5f8a3861 --- /dev/null +++ b/src/common_queries/query_user_profile.rs @@ -0,0 +1,138 @@ +use anyhow::anyhow; +use crate::AnyhowResult; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use chrono::{DateTime, Utc}; +use md5::{Md5, Digest}; + +// TODO: This duplicates the get_profile_handler. + +#[derive(Serialize)] +pub struct RawUserProfileRecord { + pub user_token: String, + pub username: String, + pub email_gravatar_hash: String, + pub display_name: String, + pub profile_markdown: String, + pub profile_rendered_html: String, + pub user_role_slug: String, + pub banned: i8, + pub avatar_public_bucket_hash: Option, + pub disable_gravatar: i8, + pub maybe_preferred_tts_model_token: Option, + pub maybe_preferred_w2l_template_token: Option, + pub discord_username: Option, + pub twitch_username: Option, + pub twitter_username: Option, + pub patreon_username: Option, + pub github_username: Option, + pub created_at: DateTime, +} + +/// This changes the record: +/// - changes banned to bool +/// - changes hide_results_preference to bool +/// - changes disable_gravatar to bool +#[derive(Serialize)] +pub struct UserProfileRecordForResponse { + pub user_token: String, + pub username: String, + pub display_name: String, + pub email_gravatar_hash: String, + pub profile_markdown: String, + pub profile_rendered_html: String, + pub user_role_slug: String, + pub banned: bool, + pub avatar_public_bucket_hash: Option, + pub disable_gravatar: bool, + pub maybe_preferred_tts_model_token: Option, + pub maybe_preferred_w2l_template_token: Option, + pub discord_username: Option, + pub twitch_username: Option, + pub twitter_username: Option, + pub patreon_username: Option, + pub github_username: Option, + pub created_at: DateTime, +} + +pub async fn select_user_profile_by_username( + username: &str, + mysql_pool: &MySqlPool +) -> AnyhowResult { + // NB: Lookup failure is Err(RowNotFound). + let maybe_profile_record = sqlx::query_as!( + RawUserProfileRecord, + r#" +SELECT + users.token as user_token, + username, + display_name, + email_gravatar_hash, + profile_markdown, + profile_rendered_html, + user_role_slug, + banned, + avatar_public_bucket_hash, + disable_gravatar, + maybe_preferred_tts_model_token, + maybe_preferred_w2l_template_token, + discord_username, + twitch_username, + twitter_username, + patreon_username, + github_username, + created_at +FROM users +WHERE users.username = ? +AND users.deleted_at IS NULL + "#, + username, + ) + .fetch_one(mysql_pool) + .await; // TODO: This will return error if it doesn't exist + + let profile_record : RawUserProfileRecord = match maybe_profile_record { + Ok(profile_record) => profile_record, + Err(err) => { + match err { + RowNotFound => { + warn!("Invalid user"); + return Err(anyhow!("could not find user")); + }, + _ => { + warn!("User profile query error: {:?}", err); + return Err(anyhow!("query error")); + } + } + } + }; + + let profile_for_response = UserProfileRecordForResponse { + user_token: profile_record.user_token.clone(), + username: profile_record.username.clone(), + display_name: profile_record.display_name.clone(), + email_gravatar_hash: profile_record.email_gravatar_hash.clone(), + profile_markdown: profile_record.profile_markdown.clone(), + profile_rendered_html: profile_record.profile_rendered_html.clone(), + user_role_slug: profile_record.user_role_slug.clone(), + banned: if profile_record.banned == 0 { false } else { true }, + avatar_public_bucket_hash: profile_record.avatar_public_bucket_hash.clone(), + disable_gravatar: if profile_record.disable_gravatar == 0 { false } else { true }, + maybe_preferred_tts_model_token: profile_record.maybe_preferred_tts_model_token.clone(), + maybe_preferred_w2l_template_token: profile_record.maybe_preferred_w2l_template_token.clone(), + discord_username: profile_record.discord_username.clone(), + twitch_username: profile_record.twitch_username.clone(), + twitter_username: profile_record.twitter_username.clone(), + patreon_username: profile_record.patreon_username.clone(), + github_username: profile_record.github_username.clone(), + created_at: profile_record.created_at.clone(), + }; + + Ok(profile_for_response) +} diff --git a/src/common_queries/users.rs b/src/common_queries/users.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs new file mode 100644 index 0000000000..090ef8d92c --- /dev/null +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -0,0 +1,299 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::cashapp_username::validate_cashapp_username; +use crate::validations::check_for_slurs::contains_slurs; +use crate::validations::discord_username::validate_discord_username; +use crate::validations::github_username::validate_github_username; +use crate::validations::passwords::validate_passwords; +use crate::validations::twitch_username::validate_twitch_username; +use crate::validations::twitter_username::validate_twitter_username; +use crate::validations::username::validate_username; +use crate::validations::username_reservations::is_reserved_username; +use crate::validations::website_url::validate_website_url; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use crate::common_queries::query_user_profile::select_user_profile_by_username; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct EditProfilePathInfo { + username: String, +} + +#[derive(Deserialize)] +pub struct EditProfileRequest { + pub display_name: Option, + + pub profile_markdown: Option, + + pub discord_username: Option, + pub twitter_username: Option, + pub twitch_username: Option, + pub patreon_username: Option, + pub github_username: Option, + pub cashapp_username: Option, + pub website_url: Option, +} + +#[derive(Serialize)] +pub struct EditProfileSuccessResponse { + pub success: bool, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum EditProfileError { + BadInput(String), + NotAuthorized, + UserNotFound, + ServerError, +} + +impl ResponseError for EditProfileError { + fn status_code(&self) -> StatusCode { + match *self { + EditProfileError::BadInput(_) => StatusCode::BAD_REQUEST, + EditProfileError::NotAuthorized => StatusCode::UNAUTHORIZED, + EditProfileError::UserNotFound => StatusCode::NOT_FOUND, + EditProfileError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + EditProfileError::BadInput(reason) => reason.to_string(), + EditProfileError::NotAuthorized=> "unauthorized".to_string(), + EditProfileError::UserNotFound => "user not found".to_string(), + EditProfileError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn edit_profile_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EditProfileError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(EditProfileError::NotAuthorized); + } + }; + + let user_lookup_result = + select_user_profile_by_username(&path.username, &server_state.mysql_pool) + .await; + + let user_record = match user_lookup_result { + Ok(result) => { + info!("Found user: {}", result.username); + result + } + Err(err) => { + warn!("could not find user"); + return Err(EditProfileError::UserNotFound); + } + }; + + let mut editor_is_original_user = false; + let mut editor_is_moderator = false; + + if path.username == user_session.username { + editor_is_original_user = true; + } + + if user_session.can_edit_other_users_data { + editor_is_moderator = true; + } + + if !editor_is_original_user && !editor_is_moderator { + return Err(EditProfileError::NotAuthorized); + } + + if let Some(twitter_username) = request.twitter_username.as_deref() { + if let Err(reason) = validate_twitter_username(twitter_username) { + return Err(EditProfileError::BadInput(reason)); + } + } + + if let Some(twitch_username) = request.twitch_username.as_deref() { + if let Err(reason) = validate_twitch_username(twitch_username) { + return Err(EditProfileError::BadInput(reason)); + } + } + + if let Some(discord_username) = request.discord_username.as_deref() { + if let Err(reason) = validate_discord_username(discord_username) { + return Err(EditProfileError::BadInput(reason)); + } + } + + if let Some(github_username) = request.github_username.as_deref() { + if let Err(reason) = validate_github_username(github_username) { + return Err(EditProfileError::BadInput(reason)); + } + } + + if let Some(cashapp_username) = request.cashapp_username.as_deref() { + if let Err(reason) = validate_cashapp_username(cashapp_username) { + return Err(EditProfileError::BadInput(reason)); + } + } + + if let Some(website_url) = request.website_url.as_deref() { + if let Err(reason) = validate_website_url(website_url) { + return Err(EditProfileError::BadInput(reason)); + } + } + + let mut profile_markdown = None; + let mut profile_html = None; + + if let Some(markdown) = request.profile_markdown.as_deref() { + if contains_slurs(markdown) { + return Err(EditProfileError::BadInput("profile contains slurs".to_string())); + } + + let markdown = markdown.trim().to_string(); + let html = markdown_to_html(&markdown); + + profile_markdown = Some(markdown); + profile_html = Some(html); + } + + let ip_address = get_request_ip(&http_request); + + let query_result = if editor_is_original_user { + // We need to store the IP address details. + sqlx::query!( + r#" +UPDATE users +SET + profile_markdown = ?, + profile_rendered_html = ?, + discord_username = ?, + twitter_username = ?, + twitch_username = ?, + github_username = ?, + cashapp_username = ?, + website_url = ?, + ip_address_last_update = ?, + version = version + 1 + +WHERE users.token = ? +LIMIT 1 + "#, + profile_markdown, + profile_html, + request.discord_username.clone(), + request.twitter_username.clone(), + request.twitch_username.clone(), + request.github_username.clone(), + request.cashapp_username.clone(), + request.website_url.clone(), + ip_address.clone(), + user_record.user_token.clone(), + ) + .execute(&server_state.mysql_pool) + .await + } else { + // We need to store the moderator details. + sqlx::query!( + r#" +UPDATE users +SET + profile_markdown = ?, + profile_rendered_html = ?, + discord_username = ?, + twitter_username = ?, + twitch_username = ?, + github_username = ?, + cashapp_username = ?, + website_url = ?, + version = version + 1 + +WHERE users.token = ? +LIMIT 1 + "#, + profile_markdown, + profile_html, + request.discord_username.clone(), + request.twitter_username.clone(), + request.twitch_username.clone(), + request.github_username.clone(), + request.cashapp_username.clone(), + request.website_url.clone(), + user_record.user_token.clone(), + ) + .execute(&server_state.mysql_pool) + .await + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Profile edit DB error: {:?}", err); + return Err(EditProfileError::ServerError); + } + }; + + let response = EditProfileSuccessResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| EditProfileError::BadInput("".to_string()))?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index 05bff72431..3a38529830 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -28,6 +28,8 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use md5::{Md5, Digest}; +// TODO: This is duplicated in query_user_profile + #[derive(Serialize)] pub struct UserProfileRecord { pub user_token: String, diff --git a/src/http_server/endpoints/users/mod.rs b/src/http_server/endpoints/users/mod.rs index c07f287ddb..8d5324ab12 100644 --- a/src/http_server/endpoints/users/mod.rs +++ b/src/http_server/endpoints/users/mod.rs @@ -1,4 +1,5 @@ pub mod create_account; +pub mod edit_profile; pub mod get_profile; pub mod list_user_tts_inference_results; pub mod list_user_tts_models; diff --git a/src/http_server/web_utils/session_checker.rs b/src/http_server/web_utils/session_checker.rs index 21d966400f..605b423972 100644 --- a/src/http_server/web_utils/session_checker.rs +++ b/src/http_server/web_utils/session_checker.rs @@ -20,26 +20,70 @@ pub struct SessionUserRecord { pub user_token: String, pub username: String, pub display_name: String, + + pub email_address: String, + pub email_confirmed: bool, + pub email_gravatar_hash: String, + pub avatar_public_bucket_hash: Option, + + // ===== PREFERENCES ===== // + + pub dark_mode_preference: String, + pub disable_gravatar: bool, + pub hide_results_preference: bool, + pub auto_play_audio_preference: bool, + pub auto_play_video_preference: bool, + pub maybe_preferred_tts_model_token: Option, + pub maybe_preferred_w2l_template_token: Option, + + // ===== ROLE ===== // + + pub user_role_slug: String, + pub banned: bool, + + // ===== PERMISSIONS FLAGS ===== // + + pub can_ban_users: bool, + pub can_edit_other_users_data: bool, + pub can_approve_w2l_templates: bool, + + pub can_upload_tts_models: bool, + pub can_upload_w2l_templates: bool, + + pub can_use_tts: bool, + pub can_use_w2l: bool, +} + +pub struct SessionUserRawDbRecord { + pub user_token: String, + pub username: String, + pub display_name: String, + pub email_address: String, pub email_confirmed: i8, pub email_gravatar_hash: String, - pub profile_markdown: String, - pub profile_rendered_html: String, - pub user_role_slug: String, - pub banned: i8, pub avatar_public_bucket_hash: Option, + + pub dark_mode_preference: String, pub disable_gravatar: i8, pub hide_results_preference: i8, pub auto_play_audio_preference: i8, pub auto_play_video_preference: i8, - pub dark_mode_preference: String, pub maybe_preferred_tts_model_token: Option, pub maybe_preferred_w2l_template_token: Option, - pub discord_username: Option, - pub twitch_username: Option, - pub twitter_username: Option, - pub patreon_username: Option, - pub github_username: Option, + + pub user_role_slug: String, + pub banned: i8, + + pub can_ban_users: Option, + pub can_edit_other_users_data: Option, + pub can_approve_w2l_templates: Option, + + pub can_upload_tts_models: Option, + pub can_upload_w2l_templates: Option, + + pub can_use_tts: Option, + pub can_use_w2l: Option, } impl SessionChecker { @@ -104,38 +148,47 @@ AND deleted_at IS NULL // NB: Lookup failure is Err(RowNotFound). let maybe_user_record = sqlx::query_as!( - SessionUserRecord, + SessionUserRawDbRecord, r#" SELECT users.token as user_token, - username, - display_name, - email_address, - email_confirmed, - email_gravatar_hash, - profile_markdown, - profile_rendered_html, - user_role_slug, - banned, - dark_mode_preference, - avatar_public_bucket_hash, - disable_gravatar, - hide_results_preference, - auto_play_audio_preference, - auto_play_video_preference, - maybe_preferred_tts_model_token, - maybe_preferred_w2l_template_token, - discord_username, - twitch_username, - twitter_username, - patreon_username, - github_username + users.username, + users.display_name, + + users.email_address, + users.email_confirmed, + users.email_gravatar_hash, + users.avatar_public_bucket_hash, + + users.dark_mode_preference, + users.disable_gravatar, + users.hide_results_preference, + users.auto_play_audio_preference, + users.auto_play_video_preference, + users.maybe_preferred_tts_model_token, + users.maybe_preferred_w2l_template_token, + + users.user_role_slug, + users.banned, + + user_roles.can_ban_users, + user_roles.can_edit_other_users_data, + user_roles.can_approve_w2l_templates, + + user_roles.can_upload_tts_models, + user_roles.can_upload_w2l_templates, + + user_roles.can_use_tts, + user_roles.can_use_w2l + FROM users LEFT OUTER JOIN user_sessions -ON users.token = user_sessions.user_token + ON users.token = user_sessions.user_token +LEFT OUTER JOIN user_roles + ON users.user_role_slug = user_roles.slug WHERE user_sessions.token = ? -AND user_sessions.deleted_at IS NULL -AND users.deleted_at IS NULL + AND user_sessions.deleted_at IS NULL + AND users.deleted_at IS NULL "#, session_token.to_string(), ) @@ -143,8 +196,34 @@ AND users.deleted_at IS NULL .await; // TODO: This will return error if it doesn't exist match maybe_user_record { - Ok(user_record) => { - Ok(Some(user_record)) + Ok(raw_user_record) => { + let result_user_record = SessionUserRecord { + user_token: raw_user_record.user_token.clone(), + username: raw_user_record.username.clone(), + display_name: raw_user_record.display_name.clone(), + email_address: raw_user_record.email_address.clone(), + email_confirmed: if raw_user_record.email_confirmed == 0 { false } else { true }, + email_gravatar_hash: raw_user_record.email_gravatar_hash.clone(), + avatar_public_bucket_hash: raw_user_record.avatar_public_bucket_hash.clone(), + dark_mode_preference: raw_user_record.dark_mode_preference.clone(), + disable_gravatar: if raw_user_record.disable_gravatar == 0 { false } else { true }, + hide_results_preference: if raw_user_record.hide_results_preference == 0 { false } else { true }, + auto_play_audio_preference: if raw_user_record.auto_play_audio_preference == 0 { false } else { true }, + auto_play_video_preference: if raw_user_record.auto_play_video_preference == 0 { false } else { true }, + maybe_preferred_tts_model_token: raw_user_record.maybe_preferred_tts_model_token.clone(), + maybe_preferred_w2l_template_token: raw_user_record.maybe_preferred_w2l_template_token.clone(), + user_role_slug: raw_user_record.user_role_slug.clone(), + banned: if raw_user_record.banned == 0 { false } else { true }, + can_ban_users: convert_optional_db_bool_default_false(raw_user_record.can_ban_users), + can_edit_other_users_data: convert_optional_db_bool_default_false(raw_user_record.can_edit_other_users_data), + can_approve_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_approve_w2l_templates), + can_upload_tts_models: convert_optional_db_bool_default_false(raw_user_record.can_upload_tts_models), + can_upload_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_upload_w2l_templates), + can_use_tts: convert_optional_db_bool_default_false(raw_user_record.can_use_tts), + can_use_w2l: convert_optional_db_bool_default_false(raw_user_record.can_use_w2l), + }; + + Ok(Some(result_user_record)) }, Err(err) => { match err { @@ -160,4 +239,14 @@ AND users.deleted_at IS NULL } } } -} \ No newline at end of file +} + +fn convert_optional_db_bool_default_false(value: Option) -> bool { + match value { + None => false, + Some(v) => match v { + 0 => false, + _ => true, + } + } +} diff --git a/src/main.rs b/src/main.rs index 7a7d0e5329..4c035f4edb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -66,6 +66,7 @@ use crate::http_server::endpoints::tts::get_tts_inference_job_status::get_tts_in use crate::http_server::endpoints::w2l::get_w2l_inference_job_status::get_w2l_inference_job_status_handler; use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; +use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -351,6 +352,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_profile_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/{username}/edit_profile") + .route(web::post().to(edit_profile_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/{username}/tts_models") .route(web::get().to(list_user_tts_models_handler)) diff --git a/src/util/markdown_to_html.rs b/src/util/markdown_to_html.rs new file mode 100644 index 0000000000..87bb66d58c --- /dev/null +++ b/src/util/markdown_to_html.rs @@ -0,0 +1,14 @@ +use pulldown_cmark::{Options, Parser, html}; + +pub fn markdown_to_html(markdown_input: &str) -> String { + let mut options = Options::empty(); + options.insert(Options::ENABLE_STRIKETHROUGH); + + let parser = Parser::new_ext(markdown_input, options); + + // TODO(bt): Is this buffer size sufficient? Just based on benchmarks? Will this panic? + let mut html_output: String = String::with_capacity(markdown_input.len() * 3 / 2); + html::push_html(&mut html_output, parser); + + html_output +} \ No newline at end of file diff --git a/src/util/mod.rs b/src/util/mod.rs index 289ae3aab9..3284b448c0 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -2,7 +2,8 @@ pub mod anyhow_result; pub mod email_to_gravatar; pub mod filesystem; pub mod hashing; +pub mod markdown_to_html; pub mod random_crockford_token; pub mod random_prefix_crockford_token; pub mod random_uuid; -pub mod semi_persistent_cache_dir; +pub mod semi_persistent_cache_dir; \ No newline at end of file diff --git a/src/validations/cashapp_username.rs b/src/validations/cashapp_username.rs new file mode 100644 index 0000000000..e51096a0cb --- /dev/null +++ b/src/validations/cashapp_username.rs @@ -0,0 +1,42 @@ +use crate::AnyhowResult; +use regex::Regex; + +pub fn validate_cashapp_username(username: &str) -> Result<(), String> { + lazy_static! { + static ref CASHAPP_USERNAME_REGEX: Regex = { + Regex::new(r"^.{1,20}$").expect("should be valid regex") + }; + } + + if username.len() < 1 { + return Err("cashapp username is too short".to_string()); + } + + if username.len() > 20 { + return Err("cashapp username is too long".to_string()); + } + + if !CASHAPP_USERNAME_REGEX.is_match(username) { + return Err("cashapp username invalid".to_string()); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::validations::cashapp_username::validate_cashapp_username; + + #[test] + fn valid_cases() { + assert!(validate_cashapp_username("echelon").is_ok()); + assert!(validate_cashapp_username("a").is_ok()); + assert!(validate_cashapp_username("12345678901234567890").is_ok()); + } + + #[test] + fn invalid_cases() { + assert!(validate_cashapp_username("").is_err()); + assert!(validate_cashapp_username("123456789012345678901").is_err()); + } +} \ No newline at end of file diff --git a/src/validations/discord_username.rs b/src/validations/discord_username.rs new file mode 100644 index 0000000000..1b49ce1fcd --- /dev/null +++ b/src/validations/discord_username.rs @@ -0,0 +1,40 @@ +use crate::AnyhowResult; +use regex::Regex; + +pub fn validate_discord_username(username: &str) -> Result<(), String> { + lazy_static! { + static ref DISCORD_USERNAME_REGEX: Regex = { + Regex::new(r"^@?[^#@:]{2,32}#[0-9]{4}$").expect("should be valid regex") + }; + } + + if username.len() < 2 { + return Err("discord username is too short".to_string()); + } + + if username.len() > 40 { + return Err("discord username is too long".to_string()); + } + + if !DISCORD_USERNAME_REGEX.is_match(username) { + return Err("discord username invalid".to_string()); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::validations::discord_username::validate_discord_username; + + #[test] + fn valid_cases() { + assert!(validate_discord_username("echelon#0001").is_ok()); + assert!(validate_discord_username("@echelon#0001").is_ok()); + } + + #[test] + fn invalid_cases() { + assert!(validate_discord_username("").is_err()); + } +} \ No newline at end of file diff --git a/src/validations/github_username.rs b/src/validations/github_username.rs new file mode 100644 index 0000000000..1654b887a1 --- /dev/null +++ b/src/validations/github_username.rs @@ -0,0 +1,41 @@ +use crate::AnyhowResult; +use regex::Regex; + +pub fn validate_github_username(username: &str) -> Result<(), String> { + lazy_static! { + // TODO: https://github.com/shinnn/github-username-regex + static ref GITHUB_USERNAME_REGEX: Regex = { + Regex::new(r"^[a-zA-Z0-9\-]{1,39}$").expect("should be valid regex") + }; + } + + if username.len() < 1 { + return Err("github username is too short".to_string()); + } + + if username.len() > 39 { + return Err("github username is too long".to_string()); + } + + if !GITHUB_USERNAME_REGEX.is_match(username) { + return Err("github username invalid".to_string()); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::validations::github_username::validate_github_username; + + #[test] + fn valid_cases() { + assert!(validate_github_username("echelon").is_ok()); + assert!(validate_github_username("a-b-c-D-E-F-0-1-2").is_ok()); + } + + #[test] + fn invalid_cases() { + assert!(validate_github_username("").is_err()); + } +} diff --git a/src/validations/mod.rs b/src/validations/mod.rs index 2b24d755b8..a4147f1a83 100644 --- a/src/validations/mod.rs +++ b/src/validations/mod.rs @@ -1,5 +1,11 @@ +pub mod cashapp_username; pub mod check_for_slurs; +pub mod discord_username; +pub mod github_username; pub mod model_uploads; pub mod passwords; +pub mod twitch_username; +pub mod twitter_username; pub mod username; -pub mod username_reservations; \ No newline at end of file +pub mod username_reservations; +pub mod website_url; \ No newline at end of file diff --git a/src/validations/twitch_username.rs b/src/validations/twitch_username.rs new file mode 100644 index 0000000000..e93509ffae --- /dev/null +++ b/src/validations/twitch_username.rs @@ -0,0 +1,43 @@ +use crate::AnyhowResult; +use regex::Regex; + +pub fn validate_twitch_username(username: &str) -> Result<(), String> { + lazy_static! { + static ref TWITCH_USERNAME_REGEX: Regex = { + Regex::new(r"^[a-zA-Z0-9]{4,25}$").expect("should be valid regex") + }; + } + + if username.len() < 4 { + return Err("twitch username is too short".to_string()); + } + + if username.len() > 25 { + return Err("twitch username is too long".to_string()); + } + + if !TWITCH_USERNAME_REGEX.is_match(username) { + return Err("twitch username invalid".to_string()); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::validations::twitch_username::validate_twitch_username; + + #[test] + fn valid_cases() { + assert!(validate_twitch_username("echelon").is_ok()); + assert!(validate_twitch_username("1234").is_ok()); + assert!(validate_twitch_username("1234567890123456789012345").is_ok()); + } + + #[test] + fn invalid_cases() { + assert!(validate_twitch_username("").is_err()); + assert!(validate_twitch_username("@=[]@34!").is_err()); + assert!(validate_twitch_username("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").is_err()); + } +} \ No newline at end of file diff --git a/src/validations/twitter_username.rs b/src/validations/twitter_username.rs new file mode 100644 index 0000000000..a12d96695a --- /dev/null +++ b/src/validations/twitter_username.rs @@ -0,0 +1,47 @@ +use crate::AnyhowResult; +use regex::Regex; + + +pub fn validate_twitter_username(username: &str) -> Result<(), String> { + lazy_static! { + static ref TWITTER_USERNAME_REGEX: Regex = { + Regex::new(r"^[A-Za-z0-9_]{4,15}$").expect("should be valid regex") + }; + } + + if username.len() < 4 { + return Err("twitter username is too short".to_string()); + } + + if username.len() > 15 { + return Err("twitter username is too long".to_string()); + } + + if !TWITTER_USERNAME_REGEX.is_match(username) { + return Err("twitter username invalid".to_string()); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::validations::twitter_username::validate_twitter_username; + + #[test] + fn valid_cases() { + assert!(validate_twitter_username("echelon").is_ok()); + assert!(validate_twitter_username("four").is_ok()); + assert!(validate_twitter_username("123456789012345").is_ok()); + assert!(validate_twitter_username("a_A_b_B_c_C_d_D").is_ok()); + } + + #[test] + fn invalid_cases() { + assert!(validate_twitter_username("").is_err()); + assert!(validate_twitter_username(" ").is_err()); + assert!(validate_twitter_username("!!!!!!!!!!!").is_err()); + assert!(validate_twitter_username("a").is_err()); + assert!(validate_twitter_username("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").is_err()); + } +} \ No newline at end of file diff --git a/src/validations/website_url.rs b/src/validations/website_url.rs new file mode 100644 index 0000000000..8f7e5b97c7 --- /dev/null +++ b/src/validations/website_url.rs @@ -0,0 +1,40 @@ +use crate::AnyhowResult; +use regex::Regex; + +pub fn validate_website_url(website_url: &str) -> Result<(), String> { + lazy_static! { + static ref WEBSITE_URL_REGEX: Regex = { + Regex::new(r"^(https?:\/\/)?.*$").expect("should be valid regex") + }; + } + + if website_url.len() < 3 { + return Err("website url is too short".to_string()); + } + + if website_url.len() > 200 { + return Err("website url is too long".to_string()); + } + + if !WEBSITE_URL_REGEX.is_match(website_url) { + return Err("website url invalid".to_string()); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::validations::website_url::validate_website_url; + + #[test] + fn valid_cases() { + assert!(validate_website_url("http://vo.codes").is_ok()); + assert!(validate_website_url("https://vocodes.com").is_ok()); + } + + #[test] + fn invalid_cases() { + assert!(validate_website_url("a").is_err()); + } +} From 6274250fbf5db8ec09ac8278a20ac16a81f9df15 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 17:25:08 -0400 Subject: [PATCH 0301/1042] fixup website validations --- src/validations/website_url.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/validations/website_url.rs b/src/validations/website_url.rs index 8f7e5b97c7..b3b5614b49 100644 --- a/src/validations/website_url.rs +++ b/src/validations/website_url.rs @@ -4,7 +4,7 @@ use regex::Regex; pub fn validate_website_url(website_url: &str) -> Result<(), String> { lazy_static! { static ref WEBSITE_URL_REGEX: Regex = { - Regex::new(r"^(https?:\/\/)?.*$").expect("should be valid regex") + Regex::new(r"^(https?:\\/\\/)?.*$").expect("should be valid regex") }; } @@ -31,6 +31,7 @@ mod tests { fn valid_cases() { assert!(validate_website_url("http://vo.codes").is_ok()); assert!(validate_website_url("https://vocodes.com").is_ok()); + assert!(validate_website_url("www.zombo.com").is_ok()); } #[test] From 5ec3c9cc3c03288dd153574f16444d7be279cc5d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 17:33:47 -0400 Subject: [PATCH 0302/1042] seed script to populate user_roles --- debug/mysql/last_firehose_events.sh | 4 ++-- debug/mysql/roles.sh | 7 +++++++ seed/bootstrap_inserts_roles_etc.sh | 12 ++++++++++++ seed/sql/system_roles.sql | 17 +++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100755 debug/mysql/roles.sh create mode 100755 seed/bootstrap_inserts_roles_etc.sh create mode 100644 seed/sql/system_roles.sql diff --git a/debug/mysql/last_firehose_events.sh b/debug/mysql/last_firehose_events.sh index 3e4077969b..59b82af149 100755 --- a/debug/mysql/last_firehose_events.sh +++ b/debug/mysql/last_firehose_events.sh @@ -1,7 +1,7 @@ #!/bin/bash # ONLY FOR LOCAL DEV -mysql -u root \ - -proot \ +mysql -u storyteller \ + -ppassword \ -h localhost \ -D storyteller \ -e "select * from firehose_entries\G" diff --git a/debug/mysql/roles.sh b/debug/mysql/roles.sh new file mode 100755 index 0000000000..5bd88a8db6 --- /dev/null +++ b/debug/mysql/roles.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u storyteller \ + -ppassword \ + -h localhost \ + -D storyteller \ + -e "select * from user_roles\G" diff --git a/seed/bootstrap_inserts_roles_etc.sh b/seed/bootstrap_inserts_roles_etc.sh new file mode 100755 index 0000000000..109a1213ad --- /dev/null +++ b/seed/bootstrap_inserts_roles_etc.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV + + +# --default-character-set=UTF8mb4 \ +mysql -u storyteller \ + -ppassword \ + -h localhost \ + -D storyteller \ + -e "source ./seed/sql/system_roles.sql" + + diff --git a/seed/sql/system_roles.sql b/seed/sql/system_roles.sql new file mode 100644 index 0000000000..6948a258e0 --- /dev/null +++ b/seed/sql/system_roles.sql @@ -0,0 +1,17 @@ + + +INSERT INTO user_roles +SET + slug='user', + name='User'; + +INSERT INTO user_roles +SET + slug='moderator', + name='Moderator'; + +INSERT INTO user_roles +SET + slug='admin', + name='Admin'; + From bc3fee54d6e414ed92dbea01c9a38013bec2f639 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 17:36:43 -0400 Subject: [PATCH 0303/1042] actual permissions --- seed/sql/system_roles.sql | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/seed/sql/system_roles.sql b/seed/sql/system_roles.sql index 6948a258e0..6ff1f72abe 100644 --- a/seed/sql/system_roles.sql +++ b/seed/sql/system_roles.sql @@ -2,16 +2,37 @@ INSERT INTO user_roles SET - slug='user', - name='User'; + slug = 'user', + name = 'User', + can_ban_users = FALSE, + can_edit_other_users_data = FALSE, + can_approve_w2l_templates = FALSE, + can_upload_tts_models = TRUE, + can_upload_w2l_templates = TRUE, + can_use_tts = TRUE, + can_use_w2l = TRUE; INSERT INTO user_roles SET - slug='moderator', - name='Moderator'; + slug = 'mod', + name = 'Moderator', + can_ban_users = TRUE, + can_edit_other_users_data = TRUE, + can_approve_w2l_templates = TRUE, + can_upload_tts_models = TRUE, + can_upload_w2l_templates = TRUE, + can_use_tts = TRUE, + can_use_w2l = TRUE; INSERT INTO user_roles SET - slug='admin', - name='Admin'; + slug = 'admin', + name = 'Admin', + can_ban_users = TRUE, + can_edit_other_users_data = TRUE, + can_approve_w2l_templates = TRUE, + can_upload_tts_models = TRUE, + can_upload_w2l_templates = TRUE, + can_use_tts = TRUE, + can_use_w2l = TRUE; From 8639d7b804d7c69daf3e2b49efaf6cb28bab91de Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 17:40:12 -0400 Subject: [PATCH 0304/1042] fixup initial role --- src/http_server/endpoints/tts/enqueue_infer_tts.rs | 2 -- src/http_server/endpoints/tts/enqueue_upload_tts_model.rs | 2 -- src/http_server/endpoints/users/create_account.rs | 2 +- src/http_server/endpoints/w2l/enqueue_infer_w2l.rs | 2 -- src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs | 2 -- 5 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 7bbbafc9c5..9cb0db89ea 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -21,8 +21,6 @@ use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::validations::check_for_slurs::contains_slurs; -const NEW_USER_ROLE: &'static str = "new-user"; - #[derive(Deserialize)] pub struct InferTtsRequest { uuid_idempotency_token: String, diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index c9dbe57804..b394795ddc 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -21,8 +21,6 @@ use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; -const NEW_USER_ROLE: &'static str = "new-user"; - #[derive(Deserialize)] pub struct UploadTtsModelRequest { idempotency_token: String, diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index 00d96e7a44..03677e7f79 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -23,7 +23,7 @@ use crate::util::email_to_gravatar::email_to_gravatar; use crate::validations::username_reservations::is_reserved_username; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; -const NEW_USER_ROLE: &'static str = "new-user"; +const NEW_USER_ROLE: &'static str = "user"; #[derive(Deserialize)] pub struct CreateAccountRequest { diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index 98e2680b34..7eac4fbba7 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -20,8 +20,6 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -const NEW_USER_ROLE: &'static str = "new-user"; - #[derive(Deserialize)] pub struct InferW2lRequest { w2l_template_token: Option, diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index 609badcf96..7d81bbaf3f 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -21,8 +21,6 @@ use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; -const NEW_USER_ROLE: &'static str = "new-user"; - #[derive(Deserialize)] pub struct UploadW2lTemplateRequest { idempotency_token: String, From b513dddbb66aaaa0bfa501b8a7874871e8b2143e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 17:41:53 -0400 Subject: [PATCH 0305/1042] re-codegen db --- sqlx-data.json | 642 +++++++++++++++++++++++++------------------------ 1 file changed, 331 insertions(+), 311 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 21a328c199..f17a7935d0 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1429,6 +1429,16 @@ ] } }, + "3a2dc0a7e41dcbdf21f91d1a1e54e813f74b0bac7aea052302089cae195eeb53": { + "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + } + }, "3de0e9c442ae0d4ca2f1da19357ac7685084e2843216313d7836ea60bc3dcfe0": { "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_model_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_model_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", "describe": { @@ -1713,6 +1723,317 @@ ] } }, + "420adb2abc58bde084116a82f1e96d4394a6faf51e591b5ab4d560df0d547ff9": { + "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_confirmed,\n users.email_gravatar_hash,\n users.avatar_public_bucket_hash,\n\n users.dark_mode_preference,\n users.disable_gravatar,\n users.hide_results_preference,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n users.maybe_preferred_tts_model_token,\n users.maybe_preferred_w2l_template_token,\n\n users.user_role_slug,\n users.banned,\n\n user_roles.can_ban_users,\n user_roles.can_edit_other_users_data,\n user_roles.can_approve_w2l_templates,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "email_confirmed", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 5, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "avatar_public_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "dark_mode_preference", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 40 + } + }, + { + "ordinal": 8, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "hide_results_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "auto_play_audio_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 11, + "name": "auto_play_video_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "maybe_preferred_tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 13, + "name": "maybe_preferred_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 14, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 15, + "name": "banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "can_ban_users", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "can_edit_other_users_data", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 18, + "name": "can_approve_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 19, + "name": "can_upload_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 20, + "name": "can_upload_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 21, + "name": "can_use_tts", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 22, + "name": "can_use_w2l", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + true + ] + } + }, "4995b1688dab1a25fa16ea59d6282b052be5c4ed6bf740a0840e24c9ad67e55a": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.updatable_slug = ?\nAND w2l.deleted_at IS NULL\n ", "describe": { @@ -2255,6 +2576,16 @@ ] } }, + "55a75db2d060bd49951db22f6b4ffd1545777b61c256030d59662543adc06ab2": { + "query": "\nUPDATE users\nSET\n profile_markdown = ?,\n profile_rendered_html = ?,\n discord_username = ?,\n twitter_username = ?,\n twitch_username = ?,\n github_username = ?,\n cashapp_username = ?,\n website_url = ?,\n ip_address_last_update = ?,\n version = version + 1\n\nWHERE users.token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + } + }, "65ce50a3aafe979e77a541d576228d7fcfe76e33400427d01382da36c1585edb": { "query": "\nUPDATE tts_inference_jobs\nSET\n status = ?,\n on_success_result_token = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n ", "describe": { @@ -5547,317 +5878,6 @@ ] } }, - "e568e1c13676e5cd06b2136808e71d10ddd87f6ef4a39848c03531ca65833a15": { - "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_address,\n email_confirmed,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n banned,\n dark_mode_preference,\n avatar_public_bucket_hash,\n disable_gravatar,\n hide_results_preference,\n auto_play_audio_preference,\n auto_play_video_preference,\n maybe_preferred_tts_model_token,\n maybe_preferred_w2l_template_token,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username\nFROM users\nLEFT OUTER JOIN user_sessions\nON users.token = user_sessions.user_token\nWHERE user_sessions.token = ?\nAND user_sessions.deleted_at IS NULL\nAND users.deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "email_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "email_confirmed", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 5, - "name": "email_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "profile_markdown", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 7, - "name": "profile_rendered_html", - "type_info": { - "type": "Blob", - "flags": { - "bits": 4241 - }, - "char_set": 224, - "max_size": 262140 - } - }, - { - "ordinal": 8, - "name": "user_role_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 9, - "name": "banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "dark_mode_preference", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 40 - } - }, - { - "ordinal": 11, - "name": "avatar_public_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 12, - "name": "disable_gravatar", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "hide_results_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 14, - "name": "auto_play_audio_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 15, - "name": "auto_play_video_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 16, - "name": "maybe_preferred_tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 17, - "name": "maybe_preferred_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 18, - "name": "discord_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 19, - "name": "twitch_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 20, - "name": "twitter_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 21, - "name": "patreon_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - }, - { - "ordinal": 22, - "name": "github_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 144 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true - ] - } - }, "e73192a3e763a287d5737821443c5a81649f17e39978497b99c52e1e896a75f6": { "query": "\nSELECT\n token as session_token,\n user_token\nFROM user_sessions\nWHERE token = ?\nAND deleted_at IS NULL\n ", "describe": { From 0cc67e3107a03e73f8708230b339273e9c741f8f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 17:44:32 -0400 Subject: [PATCH 0306/1042] no more annoying warnings --- src/main.rs | 1 + src/tts_download_job.rs | 2 +- src/tts_inference_job.rs | 1 + src/w2l_download_job.rs | 1 + src/w2l_inference_job.rs | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 4c035f4edb..24cc135f82 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ #![deny(unused_must_use)] #![allow(dead_code)] +#![allow(non_snake_case)] #![allow(unused_imports)] #![allow(unused_mut)] #![allow(unused_variables)] diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 03da552996..ff567ccd42 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -1,9 +1,9 @@ #![deny(unused_must_use)] #![allow(dead_code)] +#![allow(non_snake_case)] #![allow(unused_imports)] #![allow(unused_mut)] #![allow(unused_variables)] -#![warn(unused_must_use)] //#![allow(warnings)] #[macro_use] extern crate serde_derive; diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 793ddc4123..efbfa34f1d 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -1,5 +1,6 @@ #![deny(unused_must_use)] #![allow(dead_code)] +#![allow(non_snake_case)] #![allow(unused_imports)] #![allow(unused_mut)] #![allow(unused_variables)] diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 62b4bc27b0..af32c4a124 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -1,5 +1,6 @@ #![deny(unused_must_use)] #![allow(dead_code)] +#![allow(non_snake_case)] #![allow(unused_imports)] #![allow(unused_mut)] #![allow(unused_variables)] diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 425a263335..43be56ed5c 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -1,5 +1,6 @@ #![deny(unused_must_use)] #![allow(dead_code)] +#![allow(non_snake_case)] #![allow(unused_imports)] #![allow(unused_mut)] #![allow(unused_variables)] From abc82643044d424ad6835efc441853b67e08ace4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 17:52:07 -0400 Subject: [PATCH 0307/1042] more info from session endpoint --- .../endpoints/users/session_info.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/http_server/endpoints/users/session_info.rs b/src/http_server/endpoints/users/session_info.rs index 7dbb35bb08..22bee850fd 100644 --- a/src/http_server/endpoints/users/session_info.rs +++ b/src/http_server/endpoints/users/session_info.rs @@ -30,6 +30,19 @@ pub struct UserInfo { pub username: String, pub display_name: String, pub email_gravatar_hash: String, + + // Usage permissions: + pub can_use_tts: bool, + pub can_use_w2l: bool, + + // Contribution permissions: + pub can_upload_tts_models: bool, + pub can_upload_w2l_templates: bool, + + // Moderation permissions: + pub can_ban_users: bool, + pub can_edit_other_users_data: bool, + pub can_approve_w2l_templates: bool, } #[derive(Serialize)] @@ -103,6 +116,19 @@ pub async fn session_info_handler( username: session_data.username.to_string(), display_name: session_data.display_name.to_string(), email_gravatar_hash: session_data.email_gravatar_hash.to_string(), + + // Usage permissions: + can_use_tts: session_data.can_use_tts, + can_use_w2l: session_data.can_use_w2l, + + // Contribution permissions: + can_upload_tts_models: session_data.can_upload_tts_models, + can_upload_w2l_templates: session_data.can_upload_w2l_templates, + + // Moderation permissions: + can_ban_users: session_data.can_ban_users, + can_edit_other_users_data: session_data.can_edit_other_users_data, + can_approve_w2l_templates: session_data.can_approve_w2l_templates, }); } } From b6a8c40ddba9c2b505929c96bb5243b4ccb0318b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 19:18:40 -0400 Subject: [PATCH 0308/1042] add to get profile handler --- src/http_server/endpoints/users/get_profile.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index 3a38529830..f66fc92037 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -49,6 +49,8 @@ pub struct UserProfileRecord { pub twitter_username: Option, pub patreon_username: Option, pub github_username: Option, + pub cashapp_username: Option, + pub website_url: Option, pub created_at: DateTime, } @@ -75,6 +77,8 @@ pub struct UserProfileRecordForResponse { pub twitter_username: Option, pub patreon_username: Option, pub github_username: Option, + pub cashapp_username: Option, + pub website_url: Option, pub created_at: DateTime, } @@ -160,6 +164,8 @@ SELECT twitter_username, patreon_username, github_username, + cashapp_username, + website_url, created_at FROM users WHERE users.username = ? @@ -204,6 +210,8 @@ AND users.deleted_at IS NULL twitter_username: profile_record.twitter_username.clone(), patreon_username: profile_record.patreon_username.clone(), github_username: profile_record.github_username.clone(), + cashapp_username: profile_record.cashapp_username.clone(), + website_url: profile_record.website_url.clone(), created_at: profile_record.created_at.clone(), }; From 949579858364e53cf49c56d3e51d75549e73463e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 6 Jun 2021 23:13:19 -0400 Subject: [PATCH 0309/1042] template mod approval -> tri state, db helpers --- migrations/2021-05-15-071601_assets/up.sql | 8 +- sqlx-data.json | 288 +++++++++++++++++- src/common_queries/list_w2l_templates.rs | 6 +- src/database_helpers/boolean_converters.rs | 62 ++++ src/database_helpers/mod.rs | 1 + .../endpoints/w2l/get_w2l_template.rs | 5 + 6 files changed, 357 insertions(+), 13 deletions(-) create mode 100644 src/database_helpers/boolean_converters.rs diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 143d92895a..243f5677ef 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -231,8 +231,12 @@ CREATE TABLE w2l_templates ( -- ========== MODERATION DETAILS ========== - -- Mods have to approve of w2l templates. - is_mod_approved BOOLEAN NOT NULL DEFAULT FALSE, + -- Mods have to approve of w2l templates for them to show up in a public index. + -- Templates can be used by the author (or other parties that know the URL) before + -- they're approved, but unapproved templates won't show up in public indices. + -- This is independent of other visibility controls (is_mod_disabled, deleted_at, + -- and creator_set_visibility) + is_mod_approved BOOLEAN DEFAULT NULL, -- In this case, a moderator disables it. -- This also disables it for the creator. diff --git a/sqlx-data.json b/sqlx-data.json index f17a7935d0..25140e939d 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1670,7 +1670,7 @@ "type_info": { "type": "Tiny", "flags": { - "bits": 1 + "bits": 0 }, "char_set": 63, "max_size": 1 @@ -1717,7 +1717,7 @@ false, true, true, - false, + true, false, false ] @@ -3444,6 +3444,278 @@ "nullable": [] } }, + "916f732e506073fd54596a730906eaa1dae1699b59fac9ac87b1476e4524c877": { + "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n banned,\n avatar_public_bucket_hash,\n disable_gravatar,\n maybe_preferred_tts_model_token,\n maybe_preferred_w2l_template_token,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n cashapp_username,\n website_url,\n created_at\nFROM users\nWHERE users.username = ?\nAND users.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 4, + "name": "profile_markdown", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 5, + "name": "profile_rendered_html", + "type_info": { + "type": "Blob", + "flags": { + "bits": 4241 + }, + "char_set": 224, + "max_size": 262140 + } + }, + { + "ordinal": 6, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 7, + "name": "banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 8, + "name": "avatar_public_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 9, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "maybe_preferred_tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 11, + "name": "maybe_preferred_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 12, + "name": "discord_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 13, + "name": "twitch_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 14, + "name": "twitter_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 15, + "name": "patreon_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 16, + "name": "github_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 17, + "name": "cashapp_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 144 + } + }, + { + "ordinal": 18, + "name": "website_url", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 19, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false + ] + } + }, "94847f448294888db6c800cc9558738b0c0de023225961d070b84b3906a2e493": { "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND users.username = ?\n ", "describe": { @@ -4780,7 +5052,7 @@ "type_info": { "type": "Tiny", "flags": { - "bits": 1 + "bits": 0 }, "char_set": 63, "max_size": 1 @@ -4827,7 +5099,7 @@ false, true, true, - false, + true, false, false ] @@ -4987,7 +5259,7 @@ "type_info": { "type": "Tiny", "flags": { - "bits": 1 + "bits": 0 }, "char_set": 63, "max_size": 1 @@ -5034,7 +5306,7 @@ false, true, true, - false, + true, false, false ] @@ -5825,7 +6097,7 @@ "type_info": { "type": "Tiny", "flags": { - "bits": 1 + "bits": 0 }, "char_set": 63, "max_size": 1 @@ -5872,7 +6144,7 @@ false, true, true, - false, + true, false, false ] diff --git a/src/common_queries/list_w2l_templates.rs b/src/common_queries/list_w2l_templates.rs index d9a7994706..8377bb8ae3 100644 --- a/src/common_queries/list_w2l_templates.rs +++ b/src/common_queries/list_w2l_templates.rs @@ -18,7 +18,7 @@ pub struct W2lTemplateRecordForList { pub duration_millis: u32, pub maybe_image_object_name: Option, pub maybe_video_object_name: Option, - pub is_mod_approved: bool, // converted + pub is_mod_approved: Option, // converted pub created_at: DateTime, pub updated_at: DateTime, } @@ -36,7 +36,7 @@ struct RawW2lTemplateRecordForList { pub duration_millis: i32, pub maybe_public_bucket_preview_image_object_name: Option, pub maybe_public_bucket_preview_video_object_name: Option, - pub is_mod_approved: i8, // NB: needs conversion + pub is_mod_approved: Option, // NB: needs conversion pub created_at: DateTime, pub updated_at: DateTime, } @@ -93,7 +93,7 @@ pub async fn list_w2l_templates( duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), - is_mod_approved: if template.is_mod_approved == 0 { false } else { true }, + is_mod_approved: template.is_mod_approved.map(|v| if v == 0 { false } else { true }), created_at: template.created_at.clone(), updated_at: template.updated_at.clone(), } diff --git a/src/database_helpers/boolean_converters.rs b/src/database_helpers/boolean_converters.rs new file mode 100644 index 0000000000..43d395a1e8 --- /dev/null +++ b/src/database_helpers/boolean_converters.rs @@ -0,0 +1,62 @@ + +/// MySQL stores non-nullable booleans as i8. +pub fn i8_to_bool(value: i8) -> bool { + if value == 0 { + false + } else { + true + } +} + +/// Bool conversion, but turn nulls to a default value. +pub fn nullable_i8_to_bool(value: Option, default_value: bool) -> bool { + value.map(|v| i8_to_bool(v)) + .unwrap_or(default_value) +} + +/// Bool conversion, but retain nulls. +pub fn nullable_i8_to_optional_bool(value: Option) -> Option { + value.map(|v| i8_to_bool(v)) +} + +#[cfg(test)] +mod tests { + use crate::database_helpers::boolean_converters::{ + i8_to_bool, + nullable_i8_to_bool, + nullable_i8_to_optional_bool + }; + + #[test] + fn test_i8_to_bool() { + assert_eq!(i8_to_bool(0), false); + assert_eq!(i8_to_bool(1), true); + assert_eq!(i8_to_bool(-1), true); + assert_eq!(i8_to_bool(120), true); + } + + #[test] + fn test_nullable_i8_to_bool() { + assert_eq!(nullable_i8_to_bool(None, false), false); + assert_eq!(nullable_i8_to_bool(None, true), true); + assert_eq!(nullable_i8_to_bool(Some(0), false), false); + assert_eq!(nullable_i8_to_bool(Some(0), true), false); + + assert_eq!(nullable_i8_to_bool(Some(1), false), true); + assert_eq!(nullable_i8_to_bool(Some(1), true), true); + + assert_eq!(nullable_i8_to_bool(Some(100), false), true); + assert_eq!(nullable_i8_to_bool(Some(100), true), true); + assert_eq!(nullable_i8_to_bool(Some(-100), false), true); + assert_eq!(nullable_i8_to_bool(Some(-100), true), true); + } + + #[test] + fn test_nullable_i8_to_optional_bool() { + assert_eq!(nullable_i8_to_optional_bool(None), None); + assert_eq!(nullable_i8_to_optional_bool(Some(0)), Some(false)); + assert_eq!(nullable_i8_to_optional_bool(Some(1)), Some(true)); + assert_eq!(nullable_i8_to_optional_bool(Some(100)), Some(true)); + assert_eq!(nullable_i8_to_optional_bool(Some(-100)), Some(true)); + } +} diff --git a/src/database_helpers/mod.rs b/src/database_helpers/mod.rs index 35b3598e87..b5f78fd29f 100644 --- a/src/database_helpers/mod.rs +++ b/src/database_helpers/mod.rs @@ -1 +1,2 @@ +pub mod boolean_converters; pub mod enums; \ No newline at end of file diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index 8a076cd62b..6cf51fb76a 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -25,6 +25,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; /// For the URL PathInfo #[derive(Deserialize)] @@ -46,6 +47,7 @@ pub struct W2lTemplateRecordForResponse { pub duration_millis: u32, pub maybe_image_object_name: Option, pub maybe_video_object_name: Option, + pub is_mod_approved: Option, pub created_at: DateTime, pub updated_at: DateTime, } @@ -81,6 +83,7 @@ pub struct W2lTemplateRecord { pub duration_millis: i32, pub maybe_public_bucket_preview_image_object_name: Option, pub maybe_public_bucket_preview_video_object_name: Option, + pub is_mod_approved: Option, pub created_at: DateTime, pub updated_at: DateTime, } @@ -136,6 +139,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, + w2l.is_mod_approved, w2l.created_at, w2l.updated_at FROM w2l_templates as w2l @@ -177,6 +181,7 @@ AND w2l.deleted_at IS NULL duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), + is_mod_approved: nullable_i8_to_optional_bool(template.is_mod_approved), created_at: template.created_at.clone(), updated_at: template.updated_at.clone(), }; From 13e2e32ca521196ef2e1e2da99b265b5f98bb159 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 00:32:15 -0400 Subject: [PATCH 0310/1042] wip on moderation of templates --- sqlx-data.json | 401 +++++++++--------- src/common_queries/mod.rs | 1 + src/common_queries/query_w2l_template.rs | 182 ++++++++ .../endpoints/w2l/get_w2l_template.rs | 2 + src/http_server/endpoints/w2l/mod.rs | 3 +- .../w2l/set_w2l_template_mod_approval.rs | 169 ++++++++ src/tts_download_job.rs | 1 + src/tts_inference_job.rs | 1 + src/w2l_download_job.rs | 1 + src/w2l_inference_job.rs | 1 + 10 files changed, 567 insertions(+), 195 deletions(-) create mode 100644 src/common_queries/query_w2l_template.rs create mode 100644 src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs diff --git a/sqlx-data.json b/sqlx-data.json index 25140e939d..7cb6849be6 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -490,6 +490,213 @@ ] } }, + "1c35b51bf807a3cd27b9deef2278c066626215e631c651cabb0fff56675f3624": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.updatable_slug = ?\nAND w2l.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "is_mod_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] + } + }, "215f82f4034ccb47bfa000c3fead66c3c9f7d5caa905aa99a702d396aeeebbc6": { "query": "\nINSERT INTO tts_model_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", "describe": { @@ -2034,200 +2241,6 @@ ] } }, - "4995b1688dab1a25fa16ea59d6282b052be5c4ed6bf740a0840e24c9ad67e55a": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.updatable_slug = ?\nAND w2l.deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 12, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - false - ] - } - }, "4b4b6c2b8306cd5b6b1fe28fd492de5e346f0a53c0efc4f0eec57c4238860764": { "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.model_token as tts_model_token,\n tts_results.inference_text as inference_text,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND tts_results.is_mod_hidden_from_public IS FALSE\n AND tts_models.is_mod_disabled IS FALSE\n ", "describe": { diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index 30aab5e19b..8feffdc37f 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -5,4 +5,5 @@ pub mod list_tts_models; pub mod list_w2l_inference_results; pub mod list_w2l_templates; pub mod query_user_profile; +pub mod query_w2l_template; pub mod sessions; \ No newline at end of file diff --git a/src/common_queries/query_w2l_template.rs b/src/common_queries/query_w2l_template.rs new file mode 100644 index 0000000000..186a04a000 --- /dev/null +++ b/src/common_queries/query_w2l_template.rs @@ -0,0 +1,182 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; +use crate::database_helpers::boolean_converters::i8_to_bool; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +// TODO: This duplicates the get_w2l_template_handler. + +#[derive(Serialize)] +pub struct W2lTemplateRecordForResponse { + pub template_token: String, + pub template_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: u32, + pub frame_height: u32, + pub duration_millis: u32, + pub maybe_image_object_name: Option, + pub maybe_video_object_name: Option, + pub is_mod_public_listing_approved: Option, + pub is_mod_disabled: bool, + pub created_at: DateTime, + pub updated_at: DateTime, + pub deleted_at: Option>, +} + +#[derive(Serialize)] +pub struct W2lTemplateRecordRaw { + pub template_token: String, + pub template_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + pub maybe_public_bucket_preview_image_object_name: Option, + pub maybe_public_bucket_preview_video_object_name: Option, + pub is_mod_public_listing_approved: Option, + pub is_mod_disabled: i8, + pub created_at: DateTime, + pub updated_at: DateTime, + pub deleted_at: Option>, +} + + +pub async fn select_w2l_template_by_token( + w2l_template_token: &str, + can_see_deleted: bool, + mysql_pool: &MySqlPool +) -> AnyhowResult> { + + let maybe_record = if can_see_deleted { + select_including_deleted(w2l_template_token, mysql_pool).await + } else { + select_without_deleted(w2l_template_token, mysql_pool).await + }; + + let template : W2lTemplateRecordRaw = match maybe_record { + Ok(template) => template, + Err(ref err) => { + match err { + RowNotFound => { + warn!("w2l template not found: {:?}", &err); + return Ok(None); + }, + _ => { + warn!("w2l template query error: {:?}", &err); + return Err(anyhow!("database error")); + } + } + } + }; + + let template_for_response = W2lTemplateRecordForResponse { + template_token: template.template_token.clone(), + template_type: template.template_type.clone(), + creator_user_token: template.creator_user_token.clone(), + creator_username: template.creator_username.clone(), + creator_display_name: template.creator_display_name.clone(), + updatable_slug: template.updatable_slug.clone(), + title: template.title.clone(), + frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, + frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, + duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, + maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), + maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), + is_mod_public_listing_approved: nullable_i8_to_optional_bool(template.is_mod_public_listing_approved), + is_mod_disabled: i8_to_bool(template.is_mod_disabled), + created_at: template.created_at.clone(), + updated_at: template.updated_at.clone(), + deleted_at: template.deleted_at.clone(), + }; + + Ok(Some(template_for_response)) +} + +async fn select_including_deleted( + w2l_template_token: &str, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + W2lTemplateRecordRaw, + r#" +SELECT + w2l.token as template_token, + w2l.template_type, + w2l.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + w2l.updatable_slug, + w2l.title, + w2l.frame_width, + w2l.frame_height, + w2l.duration_millis, + w2l.maybe_public_bucket_preview_image_object_name, + w2l.maybe_public_bucket_preview_video_object_name, + w2l.is_mod_approved as is_mod_public_listing_approved, + w2l.is_mod_disabled, + w2l.created_at, + w2l.updated_at, + w2l.deleted_at +FROM w2l_templates as w2l +JOIN users +ON users.token = w2l.creator_user_token +WHERE w2l.token = ? + "#, + w2l_template_token + ) + .fetch_one(mysql_pool) + .await // TODO: This will return error if it doesn't exist +} + +async fn select_without_deleted( + w2l_template_token: &str, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + W2lTemplateRecordRaw, + r#" +SELECT + w2l.token as template_token, + w2l.template_type, + w2l.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + w2l.updatable_slug, + w2l.title, + w2l.frame_width, + w2l.frame_height, + w2l.duration_millis, + w2l.maybe_public_bucket_preview_image_object_name, + w2l.maybe_public_bucket_preview_video_object_name, + w2l.is_mod_approved as is_mod_public_listing_approved, + w2l.is_mod_disabled, + w2l.created_at, + w2l.updated_at, + w2l.deleted_at +FROM w2l_templates as w2l +JOIN users +ON users.token = w2l.creator_user_token +WHERE w2l.token = ? +AND w2l.deleted_at IS NULL + "#, + w2l_template_token + ) + .fetch_one(mysql_pool) + .await // TODO: This will return error if it doesn't exist +} diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index 6cf51fb76a..740ef51ea9 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -27,6 +27,8 @@ use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; +// TODO: This is duplicated in query_w2l_template + /// For the URL PathInfo #[derive(Deserialize)] pub struct GetW2lTemplatePathInfo { diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index 6d0065f8da..56b37f8cfb 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -5,4 +5,5 @@ pub mod get_w2l_inference_job_status; pub mod get_w2l_result; pub mod get_w2l_template; pub mod get_w2l_upload_template_job_status; -pub mod list_w2l_templates; \ No newline at end of file +pub mod list_w2l_templates; +pub mod set_w2l_template_mod_approval; \ No newline at end of file diff --git a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs new file mode 100644 index 0000000000..7489bc35f7 --- /dev/null +++ b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs @@ -0,0 +1,169 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::query_w2l_template::select_w2l_template_by_token; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct SetW2lTemplateModApprovalPathInfo { + w2l_template_token: String, +} + +#[derive(Deserialize)] +pub struct SetW2lTemplateModApprovalRequest { + is_approved: bool, +} + +#[derive(Serialize)] +pub struct SetW2lTemplateModApprovalSuccessResponse { + pub success: bool, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum SetW2lTemplateModApprovalError { + BadInput(String), + NotAuthorized, + NotFound, + ServerError, +} + +impl ResponseError for SetW2lTemplateModApprovalError { + fn status_code(&self) -> StatusCode { + match *self { + SetW2lTemplateModApprovalError::BadInput(_) => StatusCode::BAD_REQUEST, + SetW2lTemplateModApprovalError::NotAuthorized => StatusCode::UNAUTHORIZED, + SetW2lTemplateModApprovalError::NotFound => StatusCode::NOT_FOUND, + SetW2lTemplateModApprovalError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + SetW2lTemplateModApprovalError::BadInput(reason) => reason.to_string(), + SetW2lTemplateModApprovalError::NotAuthorized => "unauthorized".to_string(), + SetW2lTemplateModApprovalError::NotFound => "not found".to_string(), + SetW2lTemplateModApprovalError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn set_w2l_template_mod_approval_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + SetW2lTemplateModApprovalError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(SetW2lTemplateModApprovalError::NotAuthorized); + } + }; + + if !user_session.can_approve_w2l_templates { + warn!("user is not allowed to approve templates: {}", user_session.user_token); + return Err(SetW2lTemplateModApprovalError::NotAuthorized); + } + + let template_query_result = select_w2l_template_by_token( + &path.w2l_template_token, + true, // Only mods can perform this action + &server_state.mysql_pool, + ).await; + + let w2l_template = match template_query_result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(SetW2lTemplateModApprovalError::ServerError); + } + Ok(None) => return Err(SetW2lTemplateModApprovalError::NotFound), + Ok(Some(template)) => template, + }; + + let ip_address = get_request_ip(&http_request); + + let query_result = sqlx::query!( + r#" +UPDATE w2l_templates +SET + is_mod_approved = ?, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + + &request.is_approved, + &user_session.user_token, + &path.w2l_template_token, + ) + .execute(&server_state.mysql_pool) + .await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update w2l mod approval status DB error: {:?}", err); + return Err(SetW2lTemplateModApprovalError::ServerError); + } + }; + let response = SetW2lTemplateModApprovalSuccessResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| SetW2lTemplateModApprovalError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index ff567ccd42..1bb77723b8 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -10,6 +10,7 @@ pub mod buckets; pub mod common_queries; +pub mod database_helpers; pub mod job_queries; pub mod script_execution; pub mod shared_constants; diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index efbfa34f1d..f56931d6f6 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -10,6 +10,7 @@ pub mod buckets; pub mod common_queries; +pub mod database_helpers; pub mod job_queries; pub mod script_execution; pub mod shared_constants; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index af32c4a124..9a909812d3 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -10,6 +10,7 @@ pub mod buckets; pub mod common_queries; +pub mod database_helpers; pub mod job_queries; pub mod script_execution; pub mod shared_constants; diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 43be56ed5c..fa90615c46 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -10,6 +10,7 @@ pub mod buckets; pub mod common_queries; +pub mod database_helpers; pub mod job_queries; pub mod script_execution; pub mod shared_constants; From 94f8ab15bc03b4c5a838e17acf72e3214637eb51 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 00:38:51 -0400 Subject: [PATCH 0311/1042] reuse select query --- src/common_queries/query_w2l_template.rs | 3 - .../endpoints/w2l/get_w2l_template.rs | 140 +++++------------- 2 files changed, 34 insertions(+), 109 deletions(-) diff --git a/src/common_queries/query_w2l_template.rs b/src/common_queries/query_w2l_template.rs index 186a04a000..687ff8a1a9 100644 --- a/src/common_queries/query_w2l_template.rs +++ b/src/common_queries/query_w2l_template.rs @@ -12,8 +12,6 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -// TODO: This duplicates the get_w2l_template_handler. - #[derive(Serialize)] pub struct W2lTemplateRecordForResponse { pub template_token: String, @@ -56,7 +54,6 @@ pub struct W2lTemplateRecordRaw { pub deleted_at: Option>, } - pub async fn select_w2l_template_by_token( w2l_template_token: &str, can_see_deleted: bool, diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index 740ef51ea9..13e8ee32f0 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -8,11 +8,11 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::common_queries::query_w2l_template::W2lTemplateRecordForResponse; +use crate::common_queries::query_w2l_template::select_w2l_template_by_token; use crate::common_queries::sessions::create_session_for_user; -use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; -use crate::http_server::endpoints::users::login::LoginSuccessResponse; +use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; use crate::http_server::web_utils::ip_address::get_request_ip; -use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; use crate::validations::passwords::validate_passwords; @@ -25,9 +25,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; - -// TODO: This is duplicated in query_w2l_template /// For the URL PathInfo #[derive(Deserialize)] @@ -35,25 +32,6 @@ pub struct GetW2lTemplatePathInfo { slug: String, } -#[derive(Serialize)] -pub struct W2lTemplateRecordForResponse { - pub template_token: String, - pub template_type: String, - pub creator_user_token: String, - pub creator_username: String, - pub creator_display_name: String, - pub updatable_slug: String, - pub title: String, - pub frame_width: u32, - pub frame_height: u32, - pub duration_millis: u32, - pub maybe_image_object_name: Option, - pub maybe_video_object_name: Option, - pub is_mod_approved: Option, - pub created_at: DateTime, - pub updated_at: DateTime, -} - #[derive(Serialize)] pub struct GetW2lTemplateSuccessResponse { pub success: bool, @@ -69,37 +47,21 @@ pub struct ErrorResponse { #[derive(Debug, Display)] pub enum GetW2lTemplateError { ServerError, -} - -#[derive(Serialize)] -pub struct W2lTemplateRecord { - pub template_token: String, - pub template_type: String, - pub creator_user_token: String, - pub creator_username: String, - pub creator_display_name: String, - pub updatable_slug: String, - pub title: String, - pub frame_width: i32, - pub frame_height: i32, - pub duration_millis: i32, - pub maybe_public_bucket_preview_image_object_name: Option, - pub maybe_public_bucket_preview_video_object_name: Option, - pub is_mod_approved: Option, - pub created_at: DateTime, - pub updated_at: DateTime, + NotFound, } impl ResponseError for GetW2lTemplateError { fn status_code(&self) -> StatusCode { match *self { GetW2lTemplateError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + GetW2lTemplateError::NotFound => StatusCode::NOT_FOUND, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { GetW2lTemplateError::ServerError => "server error".to_string(), + GetW2lTemplateError::NotFound=> "not found".to_string(), }; let response = ErrorResponse { @@ -123,74 +85,40 @@ pub async fn get_w2l_template_handler( path: Path, server_state: web::Data>) -> Result { - // NB: Lookup failure is Err(RowNotFound). - // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_templates = sqlx::query_as!( - W2lTemplateRecord, - r#" -SELECT - w2l.token as template_token, - w2l.template_type, - w2l.creator_user_token, - users.username as creator_username, - users.display_name as creator_display_name, - w2l.updatable_slug, - w2l.title, - w2l.frame_width, - w2l.frame_height, - w2l.duration_millis, - w2l.maybe_public_bucket_preview_image_object_name, - w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_approved, - w2l.created_at, - w2l.updated_at -FROM w2l_templates as w2l -JOIN users -ON users.token = w2l.creator_user_token -WHERE w2l.updatable_slug = ? -AND w2l.deleted_at IS NULL - "#, - &path.slug - ) - .fetch_one(&server_state.mysql_pool) - .await; // TODO: This will return error if it doesn't exist + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetW2lTemplateError::ServerError + })?; - let template : W2lTemplateRecord = match maybe_templates { - Ok(templates) => templates, - Err(err) => { - match err { - RowNotFound => { - return Err(GetW2lTemplateError::ServerError); - }, - _ => { - warn!("w2l template query error: {:?}", err); - return Err(GetW2lTemplateError::ServerError); - } - } + let mut show_deleted_templates = false; + + if let Some(user_session) = maybe_user_session { + // NB: Moderators can see deleted templates. + show_deleted_templates = user_session.can_approve_w2l_templates; + } + + let template_query_result = select_w2l_template_by_token( + &path.slug, + show_deleted_templates, + &server_state.mysql_pool + ).await; + + let template = match template_query_result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(GetW2lTemplateError::ServerError); } - }; - - let template_for_response = W2lTemplateRecordForResponse { - template_token: template.template_token.clone(), - template_type: template.template_type.clone(), - creator_user_token: template.creator_user_token.clone(), - creator_username: template.creator_username.clone(), - creator_display_name: template.creator_display_name.clone(), - updatable_slug: template.updatable_slug.clone(), - title: template.title.clone(), - frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, - frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, - duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, - maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), - maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), - is_mod_approved: nullable_i8_to_optional_bool(template.is_mod_approved), - created_at: template.created_at.clone(), - updated_at: template.updated_at.clone(), + Ok(None) => return Err(GetW2lTemplateError::NotFound), + Ok(Some(template)) => template, }; let response = GetW2lTemplateSuccessResponse { success: true, - template: template_for_response, + template, }; let body = serde_json::to_string(&response) From 7d02dc5e009ca48398c11a28e1d64e82f0d8f1c2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 00:47:42 -0400 Subject: [PATCH 0312/1042] rename column --- migrations/2021-05-15-071601_assets/up.sql | 2 +- .../list_w2l_inference_results.rs | 4 ++-- src/common_queries/list_w2l_templates.rs | 19 ++++++++++--------- src/common_queries/query_w2l_template.rs | 4 ++-- .../w2l/set_w2l_template_mod_approval.rs | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 243f5677ef..85a645256c 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -236,7 +236,7 @@ CREATE TABLE w2l_templates ( -- they're approved, but unapproved templates won't show up in public indices. -- This is independent of other visibility controls (is_mod_disabled, deleted_at, -- and creator_set_visibility) - is_mod_approved BOOLEAN DEFAULT NULL, + is_mod_public_listing_approved BOOLEAN DEFAULT NULL, -- In this case, a moderator disables it. -- This also disables it for the creator. diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/common_queries/list_w2l_inference_results.rs index 8b21e82454..e81b91258c 100644 --- a/src/common_queries/list_w2l_inference_results.rs +++ b/src/common_queries/list_w2l_inference_results.rs @@ -159,7 +159,7 @@ LEFT OUTER JOIN users WHERE w2l_results.deleted_at IS NULL AND w2l_results.is_mod_hidden_from_public IS FALSE - AND w2l_templates.is_mod_approved IS TRUE + AND w2l_templates.is_mod_public_listing_approved IS TRUE "#) .fetch_all(mysql_pool) .await? @@ -241,7 +241,7 @@ LEFT OUTER JOIN users WHERE w2l_results.deleted_at IS NULL AND w2l_results.is_mod_hidden_from_public IS FALSE - AND w2l_templates.is_mod_approved IS TRUE + AND w2l_templates.is_mod_public_listing_approved IS TRUE AND users.username = ? "#, scope_creator_username) diff --git a/src/common_queries/list_w2l_templates.rs b/src/common_queries/list_w2l_templates.rs index 8377bb8ae3..4fd6e4cc8e 100644 --- a/src/common_queries/list_w2l_templates.rs +++ b/src/common_queries/list_w2l_templates.rs @@ -3,6 +3,7 @@ use chrono::{DateTime, Utc}; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; +use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; #[derive(Serialize)] pub struct W2lTemplateRecordForList { @@ -18,7 +19,7 @@ pub struct W2lTemplateRecordForList { pub duration_millis: u32, pub maybe_image_object_name: Option, pub maybe_video_object_name: Option, - pub is_mod_approved: Option, // converted + pub is_mod_public_listing_approved: Option, // converted pub created_at: DateTime, pub updated_at: DateTime, } @@ -36,7 +37,7 @@ struct RawW2lTemplateRecordForList { pub duration_millis: i32, pub maybe_public_bucket_preview_image_object_name: Option, pub maybe_public_bucket_preview_video_object_name: Option, - pub is_mod_approved: Option, // NB: needs conversion + pub is_mod_public_listing_approved: Option, // NB: needs conversion pub created_at: DateTime, pub updated_at: DateTime, } @@ -93,7 +94,7 @@ pub async fn list_w2l_templates( duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), - is_mod_approved: template.is_mod_approved.map(|v| if v == 0 { false } else { true }), + is_mod_public_listing_approved: nullable_i8_to_optional_bool(template.is_mod_public_listing_approved), created_at: template.created_at.clone(), updated_at: template.updated_at.clone(), } @@ -125,7 +126,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_approved, + w2l.is_mod_public_listing_approved, w2l.created_at, w2l.updated_at FROM w2l_templates as w2l @@ -133,7 +134,7 @@ JOIN users ON users.token = w2l.creator_user_token WHERE w2l.deleted_at IS NULL - AND w2l.is_mod_approved IS TRUE + AND w2l.is_mod_public_listing_approved IS TRUE "#) .fetch_all(mysql_pool) .await? @@ -155,7 +156,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_approved, + w2l.is_mod_public_listing_approved, w2l.created_at, w2l.updated_at FROM w2l_templates as w2l @@ -196,7 +197,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_approved, + w2l.is_mod_public_listing_approved, w2l.created_at, w2l.updated_at FROM w2l_templates as w2l @@ -205,7 +206,7 @@ ON users.token = w2l.creator_user_token WHERE w2l.deleted_at IS NULL - AND w2l.is_mod_approved IS TRUE + AND w2l.is_mod_public_listing_approved IS TRUE AND users.username = ? "#, scope_creator_username) @@ -229,7 +230,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_approved, + w2l.is_mod_public_listing_approved, w2l.created_at, w2l.updated_at FROM w2l_templates as w2l diff --git a/src/common_queries/query_w2l_template.rs b/src/common_queries/query_w2l_template.rs index 687ff8a1a9..c6fa76ca4c 100644 --- a/src/common_queries/query_w2l_template.rs +++ b/src/common_queries/query_w2l_template.rs @@ -125,7 +125,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_approved as is_mod_public_listing_approved, + w2l.is_mod_public_listing_approved, w2l.is_mod_disabled, w2l.created_at, w2l.updated_at, @@ -161,7 +161,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_approved as is_mod_public_listing_approved, + w2l.is_mod_public_listing_approved, w2l.is_mod_disabled, w2l.created_at, w2l.updated_at, diff --git a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs index 7489bc35f7..3d3c85b9a8 100644 --- a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs +++ b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs @@ -135,7 +135,7 @@ pub async fn set_w2l_template_mod_approval_handler( r#" UPDATE w2l_templates SET - is_mod_approved = ?, + is_mod_public_listing_approved = ?, maybe_mod_user_token = ? WHERE token = ? From 8642674481d9fc678d6d8d6a9af441b732fa905b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 00:48:10 -0400 Subject: [PATCH 0313/1042] codegen --- sqlx-data.json | 2345 +++++++++++++++++++++++++++--------------------- 1 file changed, 1307 insertions(+), 1038 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 7cb6849be6..3ed12e8a1a 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -490,8 +490,18 @@ ] } }, - "1c35b51bf807a3cd27b9deef2278c066626215e631c651cabb0fff56675f3624": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.updatable_slug = ?\nAND w2l.deleted_at IS NULL\n ", + "215f82f4034ccb47bfa000c3fead66c3c9f7d5caa905aa99a702d396aeeebbc6": { + "query": "\nINSERT INTO tts_model_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 11 + }, + "nullable": [] + } + }, + "2537264be5f06db65bb9faa4b1a3e3fe97a18e8247d85cb8b10ea6f3b72b942b": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND users.username = ?\n ", "describe": { "columns": [ { @@ -640,7 +650,7 @@ }, { "ordinal": 12, - "name": "is_mod_approved", + "name": "is_mod_public_listing_approved", "type_info": { "type": "Tiny", "flags": { @@ -697,16 +707,6 @@ ] } }, - "215f82f4034ccb47bfa000c3fead66c3c9f7d5caa905aa99a702d396aeeebbc6": { - "query": "\nINSERT INTO tts_model_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n maybe_subject_token = ?,\n maybe_actor_subject_token = ?,\n download_url = ?,\n download_url_type = ?,\n status = \"pending\"\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 11 - }, - "nullable": [] - } - }, "294c089b17fdd87d7e579c05a9d90e9bfa5b7465cafa9a95af08d58709212440": { "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = ?,\n failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", "describe": { @@ -1021,200 +1021,6 @@ "nullable": [] } }, - "32a54bab859ecca83cf42280a38ba9965d6c029624acd976eb024bc28090fd3b": { - "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND w2l_results.is_mod_hidden_from_public IS FALSE\n AND w2l_templates.is_mod_approved IS TRUE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "w2l_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "maybe_tts_inference_result_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 136 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 2, - "name": "maybe_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 392 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 4, - "name": "template_title", - "type_info": { - "type": "String", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 5, - "name": "maybe_creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "maybe_creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4228 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 7, - "name": "maybe_creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4224 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 8, - "name": "file_size_bytes", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 11, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 12, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 13, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - true, - true, - true, - true, - true, - true, - true, - false, - false, - false, - false, - false, - false - ] - } - }, "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", "describe": { @@ -1723,213 +1529,6 @@ ] } }, - "3f40b37731f6049ef9167a8912e411123053ccdfebd1564e0a8f257c3a2db0b9": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND users.username = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 12, - "name": "is_mod_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 14, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false - ] - } - }, "420adb2abc58bde084116a82f1e96d4394a6faf51e591b5ab4d560df0d547ff9": { "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_confirmed,\n users.email_gravatar_hash,\n users.avatar_public_bucket_hash,\n\n users.dark_mode_preference,\n users.disable_gravatar,\n users.hide_results_preference,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n users.maybe_preferred_tts_model_token,\n users.maybe_preferred_w2l_template_token,\n\n users.user_role_slug,\n users.banned,\n\n user_roles.can_ban_users,\n user_roles.can_edit_other_users_data,\n user_roles.can_approve_w2l_templates,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.deleted_at IS NULL\n ", "describe": { @@ -2955,6 +2554,213 @@ "nullable": [] } }, + "7408744db4c97b12e50585e2cdbc421a6c467316dea6589e3a317686cfe0deb4": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "is_mod_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] + } + }, "746526e2e150150ef804d717f743de85010972d98cbb137e44d8348cfdb4dddc": { "query": "\nINSERT INTO w2l_results\nSET\n token = ?,\n maybe_w2l_template_token = ?,\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = 'public',\n\n public_bucket_video_path = ?,\n\n file_size_bytes = ?,\n mime_type = ?,\n frame_width = ?,\n frame_height = ?,\n duration_millis = ?\n ", "describe": { @@ -3457,6 +3263,239 @@ "nullable": [] } }, + "9009e90e61f6672b072a1d28a18bffb6adca4ff804378a21b2611c8b62c747f0": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.is_mod_disabled,\n w2l.created_at,\n w2l.updated_at,\n w2l.deleted_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\nAND w2l.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "is_mod_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "is_mod_disabled", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 15, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 16, + "name": "deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false, + false, + true + ] + } + }, "916f732e506073fd54596a730906eaa1dae1699b59fac9ac87b1476e4524c877": { "query": "\nSELECT\n users.token as user_token,\n username,\n display_name,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n banned,\n avatar_public_bucket_hash,\n disable_gravatar,\n maybe_preferred_tts_model_token,\n maybe_preferred_w2l_template_token,\n discord_username,\n twitch_username,\n twitter_username,\n patreon_username,\n github_username,\n cashapp_username,\n website_url,\n created_at\nFROM users\nWHERE users.username = ?\nAND users.deleted_at IS NULL\n ", "describe": { @@ -3923,6 +3962,239 @@ ] } }, + "95db7007b9e184d64d7b8983822cb635e5b142deb8e2bea59b04a95430b62d4d": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.is_mod_disabled,\n w2l.created_at,\n w2l.updated_at,\n w2l.deleted_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE w2l.token = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "is_mod_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "is_mod_disabled", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 9 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 14, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 15, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 16, + "name": "deleted_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 128 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false, + false, + true + ] + } + }, "9c352e6fe6166ee18326908ef02dd61866d926ec63599aedf9f91a9d05d5c630": { "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND tts_results.token = ?\n ", "describe": { @@ -4199,6 +4471,200 @@ "nullable": [] } }, + "a9ff4b16669e8fb8efa9660c951ec545451fa2515ebd5d18308d088dec987121": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND w2l_results.is_mod_hidden_from_public IS FALSE\n AND w2l_templates.is_mod_public_listing_approved IS TRUE\n AND users.username = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "w2l_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "maybe_tts_inference_result_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 136 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 2, + "name": "maybe_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 392 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 4, + "name": "template_title", + "type_info": { + "type": "String", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 5, + "name": "maybe_creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "maybe_creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4228 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 7, + "name": "maybe_creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4224 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 8, + "name": "file_size_bytes", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 11, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 12, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 13, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ] + } + }, "ab4913d3a45a2ee16110021823a0786a22aa561a55aafaee27851da1e0a67e12": { "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,\n\n jobs.model_token,\n tts.tts_model_type,\n tts.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_inference_jobs as jobs\nJOIN tts_models as tts\n ON tts.token = jobs.model_token\nLEFT OUTER JOIN tts_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", "describe": { @@ -4490,6 +4956,420 @@ "nullable": [] } }, + "b3f987707efca036012cb6cbdc3a2e104ccf8d41f2e1608bcca6e05b308b5696": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND w2l.is_mod_public_listing_approved IS TRUE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "is_mod_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] + } + }, + "b62d4ca915206b8f956a2664c72e02832d38ab34fb3be50d9a77cf8145297689": { + "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND w2l.is_mod_public_listing_approved IS TRUE\n AND users.username = ?\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "template_type", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 28 + } + }, + { + "ordinal": 2, + "name": "creator_user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 3, + "name": "creator_username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 4, + "name": "creator_display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 5, + "name": "updatable_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 256 + } + }, + { + "ordinal": 6, + "name": "title", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 7, + "name": "frame_width", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 8, + "name": "frame_height", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 9, + "name": "duration_millis", + "type_info": { + "type": "Long", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 10 + } + }, + { + "ordinal": 10, + "name": "maybe_public_bucket_preview_image_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 11, + "name": "maybe_public_bucket_preview_video_object_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 12, + "name": "is_mod_public_listing_approved", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 13, + "name": "created_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 1153 + }, + "char_set": 63, + "max_size": 19 + } + }, + { + "ordinal": 14, + "name": "updated_at", + "type_info": { + "type": "Timestamp", + "flags": { + "bits": 9345 + }, + "char_set": 63, + "max_size": 19 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] + } + }, "b7535ecf2e63b7f02b5b96f833fd8cd28285b24473df6f89795ff360c1723215": { "query": "\nSELECT *\nFROM w2l_template_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { @@ -4911,420 +5791,6 @@ ] } }, - "c294456f67ddbc875cc5c09fc66451172c03a78d1b83ceb75563205aae36e2e0": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND w2l.is_mod_approved IS TRUE\n AND users.username = ?\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 12, - "name": "is_mod_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 14, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false - ] - } - }, - "c5aa964dead590345796e5c587c6abe47a6f3fcc201efe67b6134f4997f0ac7b": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 12, - "name": "is_mod_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 14, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false - ] - } - }, "ca5efc3be0f09649df2add5270c06544133288a95ffcb9a0589efd4fe345efe7": { "query": "\nINSERT INTO badges ( slug, title, description, image_url )\nVALUES ( ?, ?, ?, ? )\n ", "describe": { @@ -5335,8 +5801,8 @@ "nullable": [] } }, - "cfea55416f30c179080ae1fd9385a4201e5accfd9ace79d1a7f08cd34dcb0f01": { - "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND w2l_results.is_mod_hidden_from_public IS FALSE\n AND w2l_templates.is_mod_approved IS TRUE\n AND users.username = ?\n ", + "caa1b771bc360614445df502055125051fed57e758ea2bd81cf303bfb03c2c9f": { + "query": "\nSELECT\n w2l_results.token as w2l_result_token,\n w2l_results.maybe_tts_inference_result_token,\n\n w2l_templates.token as maybe_w2l_template_token,\n w2l_templates.template_type,\n w2l_templates.title as template_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n w2l_results.file_size_bytes,\n w2l_results.frame_width,\n w2l_results.frame_height,\n w2l_results.duration_millis,\n w2l_results.created_at,\n w2l_results.updated_at\n\nFROM w2l_results\nLEFT OUTER JOIN w2l_templates\n ON w2l_results.maybe_w2l_template_token = w2l_templates.token\nLEFT OUTER JOIN users\n ON w2l_results.maybe_creator_user_token = users.token\nWHERE\n w2l_results.deleted_at IS NULL\n AND w2l_results.is_mod_hidden_from_public IS FALSE\n AND w2l_templates.is_mod_public_listing_approved IS TRUE\n ", "describe": { "columns": [ { @@ -5509,7 +5975,7 @@ } ], "parameters": { - "Right": 1 + "Right": 0 }, "nullable": [ false, @@ -5956,213 +6422,6 @@ ] } }, - "de3d708b18ad5acc2769cf8783d76c1b37840aabb87d141b6eba5ddd2aca593b": { - "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND w2l.is_mod_approved IS TRUE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "template_type", - "type_info": { - "type": "String", - "flags": { - "bits": 393 - }, - "char_set": 224, - "max_size": 28 - } - }, - { - "ordinal": 2, - "name": "creator_user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 3, - "name": "creator_username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 4, - "name": "creator_display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 5, - "name": "updatable_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 256 - } - }, - { - "ordinal": 6, - "name": "title", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 7, - "name": "frame_width", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 8, - "name": "frame_height", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 9, - "name": "duration_millis", - "type_info": { - "type": "Long", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 10 - } - }, - { - "ordinal": 10, - "name": "maybe_public_bucket_preview_image_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 11, - "name": "maybe_public_bucket_preview_video_object_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 12, - "name": "is_mod_approved", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 13, - "name": "created_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 1153 - }, - "char_set": 63, - "max_size": 19 - } - }, - { - "ordinal": 14, - "name": "updated_at", - "type_info": { - "type": "Timestamp", - "flags": { - "bits": 9345 - }, - "char_set": 63, - "max_size": 19 - } - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - false, - false - ] - } - }, "e73192a3e763a287d5737821443c5a81649f17e39978497b99c52e1e896a75f6": { "query": "\nSELECT\n token as session_token,\n user_token\nFROM user_sessions\nWHERE token = ?\nAND deleted_at IS NULL\n ", "describe": { @@ -6895,5 +7154,15 @@ false ] } + }, + "fc8641968fc13589daceebeded8a8d0446309f8c2961066450f913c87156bbc6": { + "query": "\nUPDATE w2l_templates\nSET\n is_mod_public_listing_approved = ?,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + } } } \ No newline at end of file From 8143409ada4ad802b191f67423b84fbeace63856 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 01:19:57 -0400 Subject: [PATCH 0314/1042] moderate endpoint exposed --- src/http_server/endpoints/w2l/delete_w2l_template.rs | 0 src/http_server/endpoints/w2l/mod.rs | 1 + .../endpoints/w2l/set_w2l_template_mod_approval.rs | 6 +++--- src/main.rs | 6 ++++++ 4 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 src/http_server/endpoints/w2l/delete_w2l_template.rs diff --git a/src/http_server/endpoints/w2l/delete_w2l_template.rs b/src/http_server/endpoints/w2l/delete_w2l_template.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index 56b37f8cfb..9eb955a1d0 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -1,3 +1,4 @@ +pub mod delete_w2l_template; pub mod enqueue_infer_w2l; pub mod enqueue_infer_w2l_with_uploads; pub mod enqueue_upload_w2l_template; diff --git a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs index 3d3c85b9a8..08aa71c0f6 100644 --- a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs +++ b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs @@ -25,7 +25,7 @@ use std::sync::Arc; /// For the URL PathInfo #[derive(Deserialize)] pub struct SetW2lTemplateModApprovalPathInfo { - w2l_template_token: String, + slug: String, } #[derive(Deserialize)] @@ -115,7 +115,7 @@ pub async fn set_w2l_template_mod_approval_handler( } let template_query_result = select_w2l_template_by_token( - &path.w2l_template_token, + &path.slug, true, // Only mods can perform this action &server_state.mysql_pool, ).await; @@ -144,7 +144,7 @@ LIMIT 1 &request.is_approved, &user_session.user_token, - &path.w2l_template_token, + &path.slug, ) .execute(&server_state.mysql_pool) .await; diff --git a/src/main.rs b/src/main.rs index 24cc135f82..b7f18bacd5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -68,6 +68,7 @@ use crate::http_server::endpoints::w2l::get_w2l_inference_job_status::get_w2l_in use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; +use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -329,6 +330,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_w2l_template_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/template/{slug}/moderate") + .route(web::post().to(set_w2l_template_mod_approval_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/result/{token}") .route(web::get().to(get_w2l_inference_result_handler)) From e62fffe11f2627ef8be6e0fe55db4feb883c8e3f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 01:58:15 -0400 Subject: [PATCH 0315/1042] delete endpoint; revised permissions --- migrations/2021-05-15-071534_users/up.sql | 23 +- .../endpoints/users/edit_profile.rs | 2 +- .../endpoints/users/session_info.rs | 20 +- .../endpoints/w2l/delete_w2l_template.rs | 214 ++++++++++++++++++ src/http_server/web_utils/session_checker.rs | 83 +++++-- 5 files changed, 306 insertions(+), 36 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index f6d367a8c2..a36f09223b 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -118,16 +118,25 @@ CREATE TABLE user_roles ( name VARCHAR(255) NOT NULL, - can_ban_users BOOLEAN NOT NULL DEFAULT FALSE, - can_edit_other_users_data BOOLEAN NOT NULL DEFAULT FALSE, - can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, - - can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE, - can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, - + -- Usage can_use_tts BOOLEAN NOT NULL DEFAULT FALSE, can_use_w2l BOOLEAN NOT NULL DEFAULT FALSE, + -- Contribution + can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE, + can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_own_tts_models BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_own_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, + + -- Moderation + can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, + can_edit_other_users_profiles BOOLEAN NOT NULL DEFAULT FALSE, + can_edit_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE, + can_edit_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, + can_ban_users BOOLEAN NOT NULL DEFAULT FALSE, + -- Incremented with every update. version INT NOT NULL DEFAULT 0, diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs index 090ef8d92c..243993126d 100644 --- a/src/http_server/endpoints/users/edit_profile.rs +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -151,7 +151,7 @@ pub async fn edit_profile_handler( editor_is_original_user = true; } - if user_session.can_edit_other_users_data { + if user_session.can_edit_other_users_profiles { editor_is_moderator = true; } diff --git a/src/http_server/endpoints/users/session_info.rs b/src/http_server/endpoints/users/session_info.rs index 22bee850fd..bd09472eb6 100644 --- a/src/http_server/endpoints/users/session_info.rs +++ b/src/http_server/endpoints/users/session_info.rs @@ -38,11 +38,17 @@ pub struct UserInfo { // Contribution permissions: pub can_upload_tts_models: bool, pub can_upload_w2l_templates: bool, + pub can_delete_own_tts_models: bool, + pub can_delete_own_w2l_templates: bool, // Moderation permissions: - pub can_ban_users: bool, - pub can_edit_other_users_data: bool, pub can_approve_w2l_templates: bool, + pub can_edit_other_users_profiles: bool, + pub can_edit_other_users_tts_models: bool, + pub can_edit_other_users_w2l_templates: bool, + pub can_delete_other_users_tts_models: bool, + pub can_delete_other_users_w2l_templates: bool, + pub can_ban_users: bool, } #[derive(Serialize)] @@ -124,11 +130,17 @@ pub async fn session_info_handler( // Contribution permissions: can_upload_tts_models: session_data.can_upload_tts_models, can_upload_w2l_templates: session_data.can_upload_w2l_templates, + can_delete_own_tts_models: session_data.can_delete_own_tts_models, + can_delete_own_w2l_templates: session_data.can_delete_own_w2l_templates, // Moderation permissions: - can_ban_users: session_data.can_ban_users, - can_edit_other_users_data: session_data.can_edit_other_users_data, can_approve_w2l_templates: session_data.can_approve_w2l_templates, + can_edit_other_users_profiles: session_data.can_edit_other_users_profiles, + can_edit_other_users_tts_models: session_data.can_edit_other_users_tts_models, + can_edit_other_users_w2l_templates: session_data.can_edit_other_users_w2l_templates, + can_delete_other_users_tts_models: session_data.can_delete_other_users_tts_models, + can_delete_other_users_w2l_templates: session_data.can_delete_other_users_w2l_templates, + can_ban_users: session_data.can_ban_users, }); } } diff --git a/src/http_server/endpoints/w2l/delete_w2l_template.rs b/src/http_server/endpoints/w2l/delete_w2l_template.rs index e69de29bb2..b46a81af0c 100644 --- a/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -0,0 +1,214 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::query_w2l_template::select_w2l_template_by_token; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use sqlx::MySqlPool; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct DeleteW2lTemplatePathInfo { + slug: String, +} + +#[derive(Deserialize)] +pub struct DeleteW2lTemplateRequest { + set_delete: bool, +} + +#[derive(Serialize)] +pub struct DeleteW2lTemplateSuccessResponse { + pub success: bool, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum DeleteW2lTemplateError { + BadInput(String), + NotAuthorized, + NotFound, + ServerError, +} + +impl ResponseError for DeleteW2lTemplateError { + fn status_code(&self) -> StatusCode { + match *self { + DeleteW2lTemplateError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteW2lTemplateError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteW2lTemplateError::NotFound => StatusCode::NOT_FOUND, + DeleteW2lTemplateError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + DeleteW2lTemplateError::BadInput(reason) => reason.to_string(), + DeleteW2lTemplateError::NotAuthorized => "unauthorized".to_string(), + DeleteW2lTemplateError::NotFound => "not found".to_string(), + DeleteW2lTemplateError::ServerError => "server error".to_string(), + }; + + let response = ErrorResponse { + success: false, + error_reason, + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(self.status_code()) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) + } +} + +pub async fn delete_w2l_template_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data> +) -> Result { + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + DeleteW2lTemplateError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(DeleteW2lTemplateError::NotAuthorized); + } + }; + + if !user_session.can_delete_other_users_w2l_templates { + warn!("user is not allowed to delete templates: {}", user_session.user_token); + return Err(DeleteW2lTemplateError::NotAuthorized); + } + + let template_query_result = select_w2l_template_by_token( + &path.slug, + true, // Only mods can perform this action + &server_state.mysql_pool, + ).await; + + let w2l_template = match template_query_result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(DeleteW2lTemplateError::ServerError); + } + Ok(None) => return Err(DeleteW2lTemplateError::NotFound), + Ok(Some(template)) => template, + }; + + let ip_address = get_request_ip(&http_request); + + let query_result = if request.set_delete { + delete_template( + &path.slug, + &user_session.user_token, + &server_state.mysql_pool + ).await + } else { + undelete_template( + &path.slug, + &user_session.user_token, + &server_state.mysql_pool + ).await + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update w2l mod approval status DB error: {:?}", err); + return Err(DeleteW2lTemplateError::ServerError); + } + }; + + let response = DeleteW2lTemplateSuccessResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| DeleteW2lTemplateError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} + +async fn delete_template( + template_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + let _r = sqlx::query!( + r#" +UPDATE w2l_templates +SET + deleted_at = CURRENT_TIMESTAMP, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + template_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +async fn undelete_template( + template_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + + let _r = sqlx::query!( + r#" +UPDATE w2l_templates +SET + deleted_at = NULL, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + template_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} diff --git a/src/http_server/web_utils/session_checker.rs b/src/http_server/web_utils/session_checker.rs index 605b423972..d9ff4c4b6a 100644 --- a/src/http_server/web_utils/session_checker.rs +++ b/src/http_server/web_utils/session_checker.rs @@ -43,15 +43,24 @@ pub struct SessionUserRecord { // ===== PERMISSIONS FLAGS ===== // - pub can_ban_users: bool, - pub can_edit_other_users_data: bool, - pub can_approve_w2l_templates: bool, - - pub can_upload_tts_models: bool, - pub can_upload_w2l_templates: bool, - + // Usage pub can_use_tts: bool, pub can_use_w2l: bool, + + // Contribution + pub can_upload_tts_models: bool, + pub can_upload_w2l_templates: bool, + pub can_delete_own_tts_models: bool, + pub can_delete_own_w2l_templates: bool, + + // Moderation + pub can_approve_w2l_templates: bool, + pub can_edit_other_users_profiles: bool, + pub can_edit_other_users_tts_models: bool, + pub can_edit_other_users_w2l_templates: bool, + pub can_delete_other_users_tts_models: bool, + pub can_delete_other_users_w2l_templates: bool, + pub can_ban_users: bool, } pub struct SessionUserRawDbRecord { @@ -75,15 +84,25 @@ pub struct SessionUserRawDbRecord { pub user_role_slug: String, pub banned: i8, - pub can_ban_users: Option, - pub can_edit_other_users_data: Option, - pub can_approve_w2l_templates: Option, - - pub can_upload_tts_models: Option, - pub can_upload_w2l_templates: Option, - + // NB: These are `Option` due to the JOIN not being compile-time assured. + // Usage pub can_use_tts: Option, pub can_use_w2l: Option, + + // Contribution + pub can_upload_tts_models: Option, + pub can_upload_w2l_templates: Option, + pub can_delete_own_tts_models: Option, + pub can_delete_own_w2l_templates: Option, + + // Moderation + pub can_approve_w2l_templates: Option, + pub can_edit_other_users_profiles: Option, + pub can_edit_other_users_tts_models: Option, + pub can_edit_other_users_w2l_templates: Option, + pub can_delete_other_users_tts_models: Option, + pub can_delete_other_users_w2l_templates: Option, + pub can_ban_users: Option, } impl SessionChecker { @@ -171,15 +190,21 @@ SELECT users.user_role_slug, users.banned, - user_roles.can_ban_users, - user_roles.can_edit_other_users_data, - user_roles.can_approve_w2l_templates, + user_roles.can_use_tts, + user_roles.can_use_w2l, user_roles.can_upload_tts_models, user_roles.can_upload_w2l_templates, + user_roles.can_delete_own_tts_models, + user_roles.can_delete_own_w2l_templates, - user_roles.can_use_tts, - user_roles.can_use_w2l + user_roles.can_approve_w2l_templates, + user_roles.can_edit_other_users_profiles, + user_roles.can_edit_other_users_tts_models, + user_roles.can_edit_other_users_w2l_templates, + user_roles.can_delete_other_users_tts_models, + user_roles.can_delete_other_users_w2l_templates, + user_roles.can_ban_users FROM users LEFT OUTER JOIN user_sessions @@ -214,13 +239,23 @@ WHERE user_sessions.token = ? maybe_preferred_w2l_template_token: raw_user_record.maybe_preferred_w2l_template_token.clone(), user_role_slug: raw_user_record.user_role_slug.clone(), banned: if raw_user_record.banned == 0 { false } else { true }, - can_ban_users: convert_optional_db_bool_default_false(raw_user_record.can_ban_users), - can_edit_other_users_data: convert_optional_db_bool_default_false(raw_user_record.can_edit_other_users_data), - can_approve_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_approve_w2l_templates), - can_upload_tts_models: convert_optional_db_bool_default_false(raw_user_record.can_upload_tts_models), - can_upload_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_upload_w2l_templates), + + // Usage can_use_tts: convert_optional_db_bool_default_false(raw_user_record.can_use_tts), can_use_w2l: convert_optional_db_bool_default_false(raw_user_record.can_use_w2l), + // Contribution + can_upload_tts_models: convert_optional_db_bool_default_false(raw_user_record.can_upload_tts_models), + can_upload_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_upload_w2l_templates), + can_delete_own_tts_models: convert_optional_db_bool_default_false(raw_user_record.can_delete_own_tts_models), + can_delete_own_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_delete_own_w2l_templates), + // Moderation + can_approve_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_approve_w2l_templates), + can_edit_other_users_profiles: convert_optional_db_bool_default_false(raw_user_record.can_edit_other_users_profiles), + can_edit_other_users_tts_models: convert_optional_db_bool_default_false(raw_user_record.can_edit_other_users_tts_models), + can_edit_other_users_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_edit_other_users_w2l_templates), + can_delete_other_users_tts_models: convert_optional_db_bool_default_false(raw_user_record.can_delete_other_users_tts_models), + can_delete_other_users_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_delete_other_users_w2l_templates), + can_ban_users: convert_optional_db_bool_default_false(raw_user_record.can_ban_users), }; Ok(Some(result_user_record)) From 76b1e9ffed289e0e5a1b194226569d85967e0d82 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 01:58:45 -0400 Subject: [PATCH 0316/1042] codegen --- sqlx-data.json | 720 ++++++++++++++++++++++++++++--------------------- 1 file changed, 409 insertions(+), 311 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 3ed12e8a1a..be39c464ca 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -500,6 +500,16 @@ "nullable": [] } }, + "22a52ac04d4a53d48d5134fc929d39d8b13617bf62a2267be7bde0ac54ec38e2": { + "query": "\nUPDATE w2l_templates\nSET\n deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "2537264be5f06db65bb9faa4b1a3e3fe97a18e8247d85cb8b10ea6f3b72b942b": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND users.username = ?\n ", "describe": { @@ -1021,6 +1031,395 @@ "nullable": [] } }, + "2cebfec1341bb1066357b77440f36b3b7b7d292c64354912f21f7af6975de6fa": { + "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_confirmed,\n users.email_gravatar_hash,\n users.avatar_public_bucket_hash,\n\n users.dark_mode_preference,\n users.disable_gravatar,\n users.hide_results_preference,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n users.maybe_preferred_tts_model_token,\n users.maybe_preferred_w2l_template_token,\n\n users.user_role_slug,\n users.banned,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n user_roles.can_delete_own_tts_models,\n user_roles.can_delete_own_w2l_templates,\n\n user_roles.can_approve_w2l_templates,\n user_roles.can_edit_other_users_profiles,\n user_roles.can_edit_other_users_tts_models,\n user_roles.can_edit_other_users_w2l_templates,\n user_roles.can_delete_other_users_tts_models,\n user_roles.can_delete_other_users_w2l_templates,\n user_roles.can_ban_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.deleted_at IS NULL\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 1, + "name": "username", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 2, + "name": "display_name", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 80 + } + }, + { + "ordinal": 3, + "name": "email_address", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4229 + }, + "char_set": 224, + "max_size": 1020 + } + }, + { + "ordinal": 4, + "name": "email_confirmed", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 5, + "name": "email_gravatar_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 4225 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 6, + "name": "avatar_public_bucket_hash", + "type_info": { + "type": "String", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 7, + "name": "dark_mode_preference", + "type_info": { + "type": "String", + "flags": { + "bits": 385 + }, + "char_set": 224, + "max_size": 40 + } + }, + { + "ordinal": 8, + "name": "disable_gravatar", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 9, + "name": "hide_results_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 10, + "name": "auto_play_audio_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 11, + "name": "auto_play_video_preference", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 12, + "name": "maybe_preferred_tts_model_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 13, + "name": "maybe_preferred_w2l_template_token", + "type_info": { + "type": "VarString", + "flags": { + "bits": 128 + }, + "char_set": 224, + "max_size": 128 + } + }, + { + "ordinal": 14, + "name": "user_role_slug", + "type_info": { + "type": "VarString", + "flags": { + "bits": 4233 + }, + "char_set": 224, + "max_size": 64 + } + }, + { + "ordinal": 15, + "name": "banned", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 1 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 16, + "name": "can_use_tts", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 17, + "name": "can_use_w2l", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 18, + "name": "can_upload_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 19, + "name": "can_upload_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 20, + "name": "can_delete_own_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 21, + "name": "can_delete_own_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 22, + "name": "can_approve_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 23, + "name": "can_edit_other_users_profiles", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 24, + "name": "can_edit_other_users_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 25, + "name": "can_edit_other_users_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 26, + "name": "can_delete_other_users_tts_models", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 27, + "name": "can_delete_other_users_w2l_templates", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + }, + { + "ordinal": 28, + "name": "can_ban_users", + "type_info": { + "type": "Tiny", + "flags": { + "bits": 0 + }, + "char_set": 63, + "max_size": 1 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + } + }, "34364f9bc851971f8291f5f9942dd23d10718ccb210a93a236d96d0591f65f35": { "query": "\nINSERT INTO users (\n token,\n username,\n display_name,\n email_address,\n email_gravatar_hash,\n profile_markdown,\n profile_rendered_html,\n user_role_slug,\n password_hash,\n ip_address_creation,\n ip_address_last_login,\n ip_address_last_update\n)\nVALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )\n ", "describe": { @@ -1529,317 +1928,6 @@ ] } }, - "420adb2abc58bde084116a82f1e96d4394a6faf51e591b5ab4d560df0d547ff9": { - "query": "\nSELECT\n users.token as user_token,\n users.username,\n users.display_name,\n\n users.email_address,\n users.email_confirmed,\n users.email_gravatar_hash,\n users.avatar_public_bucket_hash,\n\n users.dark_mode_preference,\n users.disable_gravatar,\n users.hide_results_preference,\n users.auto_play_audio_preference,\n users.auto_play_video_preference,\n users.maybe_preferred_tts_model_token,\n users.maybe_preferred_w2l_template_token,\n\n users.user_role_slug,\n users.banned,\n\n user_roles.can_ban_users,\n user_roles.can_edit_other_users_data,\n user_roles.can_approve_w2l_templates,\n\n user_roles.can_upload_tts_models,\n user_roles.can_upload_w2l_templates,\n\n user_roles.can_use_tts,\n user_roles.can_use_w2l\n\nFROM users\nLEFT OUTER JOIN user_sessions\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.deleted_at IS NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "user_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 1, - "name": "username", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 80 - } - }, - { - "ordinal": 3, - "name": "email_address", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4229 - }, - "char_set": 224, - "max_size": 1020 - } - }, - { - "ordinal": 4, - "name": "email_confirmed", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 5, - "name": "email_gravatar_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 4225 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 6, - "name": "avatar_public_bucket_hash", - "type_info": { - "type": "String", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 7, - "name": "dark_mode_preference", - "type_info": { - "type": "String", - "flags": { - "bits": 385 - }, - "char_set": 224, - "max_size": 40 - } - }, - { - "ordinal": 8, - "name": "disable_gravatar", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 9, - "name": "hide_results_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 10, - "name": "auto_play_audio_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 11, - "name": "auto_play_video_preference", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 12, - "name": "maybe_preferred_tts_model_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 13, - "name": "maybe_preferred_w2l_template_token", - "type_info": { - "type": "VarString", - "flags": { - "bits": 128 - }, - "char_set": 224, - "max_size": 128 - } - }, - { - "ordinal": 14, - "name": "user_role_slug", - "type_info": { - "type": "VarString", - "flags": { - "bits": 4233 - }, - "char_set": 224, - "max_size": 64 - } - }, - { - "ordinal": 15, - "name": "banned", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 1 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 16, - "name": "can_ban_users", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 17, - "name": "can_edit_other_users_data", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 18, - "name": "can_approve_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 19, - "name": "can_upload_tts_models", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 20, - "name": "can_upload_w2l_templates", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 21, - "name": "can_use_tts", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - }, - { - "ordinal": 22, - "name": "can_use_w2l", - "type_info": { - "type": "Tiny", - "flags": { - "bits": 0 - }, - "char_set": 63, - "max_size": 1 - } - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - true, - false, - false, - false, - false, - false, - true, - true, - false, - false, - true, - true, - true, - true, - true, - true, - true - ] - } - }, "4b4b6c2b8306cd5b6b1fe28fd492de5e346f0a53c0efc4f0eec57c4238860764": { "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.model_token as tts_model_token,\n tts_results.inference_text as inference_text,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND tts_results.is_mod_hidden_from_public IS FALSE\n AND tts_models.is_mod_disabled IS FALSE\n ", "describe": { @@ -4195,6 +4283,16 @@ ] } }, + "9710f22f2579903571f71726585bb7de66b8a7fa9540b851c6ace30c51d29c69": { + "query": "\nUPDATE w2l_templates\nSET\n deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + } + }, "9c352e6fe6166ee18326908ef02dd61866d926ec63599aedf9f91a9d05d5c630": { "query": "\nSELECT\n tts_results.token as tts_result_token,\n\n tts_results.inference_text,\n\n tts_results.model_token as tts_model_token,\n tts_models.title as tts_model_title,\n\n users.token as maybe_creator_user_token,\n users.username as maybe_creator_username,\n users.display_name as maybe_creator_display_name,\n users.email_gravatar_hash as maybe_creator_gravatar_hash,\n\n model_users.token as maybe_model_creator_user_token,\n model_users.username as maybe_model_creator_username,\n model_users.display_name as maybe_model_creator_display_name,\n model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash,\n\n tts_results.public_bucket_wav_audio_path,\n tts_results.public_bucket_spectrogram_path,\n\n tts_results.file_size_bytes,\n tts_results.duration_millis,\n tts_results.created_at,\n tts_results.updated_at\n\nFROM tts_results\nLEFT OUTER JOIN tts_models\n ON tts_results.model_token = tts_models.token\nLEFT OUTER JOIN users\n ON tts_results.maybe_creator_user_token = users.token\nLEFT OUTER JOIN users as model_users\n ON tts_models.creator_user_token = model_users.token\nWHERE\n tts_results.deleted_at IS NULL\n AND tts_results.token = ?\n ", "describe": { From 3d23461863ef1e84616bf57c4f8d8cb66ae819d1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 02:06:12 -0400 Subject: [PATCH 0317/1042] fix seed script --- seed/sql/system_roles.sql | 57 ++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/seed/sql/system_roles.sql b/seed/sql/system_roles.sql index 6ff1f72abe..26c7860e31 100644 --- a/seed/sql/system_roles.sql +++ b/seed/sql/system_roles.sql @@ -4,35 +4,62 @@ INSERT INTO user_roles SET slug = 'user', name = 'User', - can_ban_users = FALSE, - can_edit_other_users_data = FALSE, - can_approve_w2l_templates = FALSE, + -- Usage + can_use_tts = TRUE, + can_use_w2l = TRUE, + -- Contribution can_upload_tts_models = TRUE, can_upload_w2l_templates = TRUE, - can_use_tts = TRUE, - can_use_w2l = TRUE; + can_delete_own_tts_models = TRUE, + can_delete_own_w2l_templates = TRUE, + -- Moderation + can_approve_w2l_templates = FALSE, + can_edit_other_users_profiles = FALSE, + can_edit_other_users_tts_models = FALSE, + can_edit_other_users_w2l_templates = FALSE, + can_delete_other_users_tts_models = FALSE, + can_delete_other_users_w2l_templates = FALSE, + can_ban_users = FALSE; INSERT INTO user_roles SET slug = 'mod', name = 'Moderator', - can_ban_users = TRUE, - can_edit_other_users_data = TRUE, - can_approve_w2l_templates = TRUE, + -- Usage + can_use_tts = TRUE, + can_use_w2l = TRUE, + -- Contribution can_upload_tts_models = TRUE, can_upload_w2l_templates = TRUE, - can_use_tts = TRUE, - can_use_w2l = TRUE; + can_delete_own_tts_models = TRUE, + can_delete_own_w2l_templates = TRUE, + -- Moderation + can_approve_w2l_templates = TRUE, + can_edit_other_users_profiles = TRUE, + can_edit_other_users_tts_models = TRUE, + can_edit_other_users_w2l_templates = TRUE, + can_delete_other_users_tts_models = TRUE, + can_delete_other_users_w2l_templates = TRUE, + can_ban_users = TRUE; INSERT INTO user_roles SET slug = 'admin', name = 'Admin', - can_ban_users = TRUE, - can_edit_other_users_data = TRUE, - can_approve_w2l_templates = TRUE, + -- Usage + can_use_tts = TRUE, + can_use_w2l = TRUE, + -- Contribution can_upload_tts_models = TRUE, can_upload_w2l_templates = TRUE, - can_use_tts = TRUE, - can_use_w2l = TRUE; + can_delete_own_tts_models = TRUE, + can_delete_own_w2l_templates = TRUE, + -- Moderation + can_approve_w2l_templates = TRUE, + can_edit_other_users_profiles = TRUE, + can_edit_other_users_tts_models = TRUE, + can_edit_other_users_w2l_templates = TRUE, + can_delete_other_users_tts_models = TRUE, + can_delete_other_users_w2l_templates = TRUE, + can_ban_users = TRUE; From 71e5eb34923d3796e6e6870eefd343a88119bde3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 02:06:30 -0400 Subject: [PATCH 0318/1042] scripts to toggle userroles --- debug/mysql/set_admin.sh | 7 +++++++ debug/mysql/set_regular_user.sh | 7 +++++++ 2 files changed, 14 insertions(+) create mode 100755 debug/mysql/set_admin.sh create mode 100755 debug/mysql/set_regular_user.sh diff --git a/debug/mysql/set_admin.sh b/debug/mysql/set_admin.sh new file mode 100755 index 0000000000..72d93980b2 --- /dev/null +++ b/debug/mysql/set_admin.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u storyteller \ + -ppassword \ + -h localhost \ + -D storyteller \ + -e "update users set user_role_slug='admin' where username='echelon' limit 1\G" diff --git a/debug/mysql/set_regular_user.sh b/debug/mysql/set_regular_user.sh new file mode 100755 index 0000000000..aff8414f96 --- /dev/null +++ b/debug/mysql/set_regular_user.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ONLY FOR LOCAL DEV +mysql -u storyteller \ + -ppassword \ + -h localhost \ + -D storyteller \ + -e "update users set user_role_slug='user' where username='echelon' limit 1\G" From 93ac3c6c498605aed65ee23580bdeaa2b6c476a5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 02:22:01 -0400 Subject: [PATCH 0319/1042] fix edit profile bug --- .../endpoints/users/edit_profile.rs | 96 ++++++++++++++----- 1 file changed, 70 insertions(+), 26 deletions(-) diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs index 243993126d..bbfaa975c8 100644 --- a/src/http_server/endpoints/users/edit_profile.rs +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -155,48 +155,92 @@ pub async fn edit_profile_handler( editor_is_moderator = true; } + // Fields to set + let mut twitter_username = None; + let mut twitch_username = None; + let mut discord_username = None; + let mut cashapp_username = None; + let mut github_username = None; + let mut website_url = None; + let mut profile_markdown = None; + let mut profile_html = None; + if !editor_is_original_user && !editor_is_moderator { return Err(EditProfileError::NotAuthorized); } - if let Some(twitter_username) = request.twitter_username.as_deref() { - if let Err(reason) = validate_twitter_username(twitter_username) { - return Err(EditProfileError::BadInput(reason)); + if let Some(twitter) = request.twitter_username.as_deref() { + let trimmed = twitter.trim(); + if trimmed.is_empty() { + twitter_username = None; + } else { + if let Err(reason) = validate_twitter_username(trimmed) { + return Err(EditProfileError::BadInput(reason)); + } + twitter_username = Some(trimmed); } } - if let Some(twitch_username) = request.twitch_username.as_deref() { - if let Err(reason) = validate_twitch_username(twitch_username) { - return Err(EditProfileError::BadInput(reason)); + if let Some(twitch) = request.twitch_username.as_deref() { + let trimmed = twitch.trim(); + if trimmed.is_empty() { + twitch_username = None; + } else { + if let Err(reason) = validate_twitch_username(trimmed) { + return Err(EditProfileError::BadInput(reason)); + } + twitch_username = Some(trimmed); } } - if let Some(discord_username) = request.discord_username.as_deref() { - if let Err(reason) = validate_discord_username(discord_username) { - return Err(EditProfileError::BadInput(reason)); + if let Some(discord) = request.discord_username.as_deref() { + let trimmed = discord.trim(); + if trimmed.is_empty() { + discord_username = None; + } else { + if let Err(reason) = validate_discord_username(trimmed) { + return Err(EditProfileError::BadInput(reason)); + } + discord_username = Some(trimmed); } } - if let Some(github_username) = request.github_username.as_deref() { - if let Err(reason) = validate_github_username(github_username) { - return Err(EditProfileError::BadInput(reason)); + if let Some(github) = request.github_username.as_deref() { + let trimmed = github.trim(); + if trimmed.is_empty() { + github_username = None; + } else { + if let Err(reason) = validate_github_username(trimmed) { + return Err(EditProfileError::BadInput(reason)); + } + github_username = Some(trimmed); } } - if let Some(cashapp_username) = request.cashapp_username.as_deref() { - if let Err(reason) = validate_cashapp_username(cashapp_username) { - return Err(EditProfileError::BadInput(reason)); + if let Some(cashapp) = request.cashapp_username.as_deref() { + let trimmed = cashapp.trim(); + if trimmed.is_empty() { + cashapp_username = None; + } else { + if let Err(reason) = validate_cashapp_username(trimmed) { + return Err(EditProfileError::BadInput(reason)); + } + cashapp_username = Some(trimmed); } } - if let Some(website_url) = request.website_url.as_deref() { - if let Err(reason) = validate_website_url(website_url) { - return Err(EditProfileError::BadInput(reason)); + if let Some(website) = request.website_url.as_deref() { + let trimmed = website.trim(); + if trimmed.is_empty() { + website_url = None; + } else { + if let Err(reason) = validate_website_url(trimmed) { + return Err(EditProfileError::BadInput(reason)); + } + website_url = Some(trimmed); } } - let mut profile_markdown = None; - let mut profile_html = None; if let Some(markdown) = request.profile_markdown.as_deref() { if contains_slurs(markdown) { @@ -234,12 +278,12 @@ LIMIT 1 "#, profile_markdown, profile_html, - request.discord_username.clone(), - request.twitter_username.clone(), - request.twitch_username.clone(), - request.github_username.clone(), - request.cashapp_username.clone(), - request.website_url.clone(), + discord_username, + twitter_username, + twitch_username, + github_username, + cashapp_username, + website_url, ip_address.clone(), user_record.user_token.clone(), ) From 7bec75e7551f2151b366ea8d60c749978c299fc1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 02:23:30 -0400 Subject: [PATCH 0320/1042] fix for mods --- src/http_server/endpoints/users/edit_profile.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs index bbfaa975c8..5a1d3b24c0 100644 --- a/src/http_server/endpoints/users/edit_profile.rs +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -310,12 +310,12 @@ LIMIT 1 "#, profile_markdown, profile_html, - request.discord_username.clone(), - request.twitter_username.clone(), - request.twitch_username.clone(), - request.github_username.clone(), - request.cashapp_username.clone(), - request.website_url.clone(), + discord_username, + twitter_username, + twitch_username, + github_username, + cashapp_username, + website_url, user_record.user_token.clone(), ) .execute(&server_state.mysql_pool) From 824324f2047d1d1a360c4ad2c9945d4be72cca1b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 02:53:08 -0400 Subject: [PATCH 0321/1042] expose delete handler --- src/main.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main.rs b/src/main.rs index b7f18bacd5..506b4faca5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -69,6 +69,7 @@ use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; +use crate::http_server::endpoints::w2l::delete_w2l_template::delete_w2l_template_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -335,6 +336,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::post().to(set_w2l_template_mod_approval_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/template/{slug}/delete") + .route(web::post().to(delete_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/result/{token}") .route(web::get().to(get_w2l_inference_result_handler)) From df765663eb6f99d27394dea84506cb470ac696a5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 03:11:11 -0400 Subject: [PATCH 0322/1042] good permissions settings for w2l templates! --- .../users/list_user_w2l_templates.rs | 26 ++++++++++++++++++- .../endpoints/w2l/get_w2l_template.rs | 3 ++- .../endpoints/w2l/list_w2l_templates.rs | 9 ++++--- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/http_server/endpoints/users/list_user_w2l_templates.rs b/src/http_server/endpoints/users/list_user_w2l_templates.rs index 6bd5f5a995..c1eef4c609 100644 --- a/src/http_server/endpoints/users/list_user_w2l_templates.rs +++ b/src/http_server/endpoints/users/list_user_w2l_templates.rs @@ -86,10 +86,34 @@ pub async fn list_user_w2l_templates_handler( { info!("Fetching templates for user: {}", &path.username); + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListW2lTemplatesForUserError::ServerError + })?; + + let mut viewer_is_original_user = false; + let mut viewer_is_moderator = false; + + match maybe_user_session { + None => {}, + Some(session) => { + viewer_is_moderator = session.can_delete_other_users_w2l_templates; + viewer_is_original_user = &session.username == &path.username; + }, + }; + + // The original user can see all their uploads, as can mods. + // Once they've been approved, everyone can see them. + let require_mod_approved = !(viewer_is_original_user || viewer_is_moderator); + let query_results = list_w2l_templates( &server_state.mysql_pool, Some(path.username.as_ref()), - false + require_mod_approved ).await; let templates = match query_results { diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index 13e8ee32f0..8cff2298fe 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -98,7 +98,8 @@ pub async fn get_w2l_template_handler( if let Some(user_session) = maybe_user_session { // NB: Moderators can see deleted templates. - show_deleted_templates = user_session.can_approve_w2l_templates; + // Original creators cannot see them (unless they're moderators!) + show_deleted_templates = user_session.can_delete_other_users_w2l_templates; } let template_query_result = select_w2l_template_by_token( diff --git a/src/http_server/endpoints/w2l/list_w2l_templates.rs b/src/http_server/endpoints/w2l/list_w2l_templates.rs index 21e9b67cbd..c055b01454 100644 --- a/src/http_server/endpoints/w2l/list_w2l_templates.rs +++ b/src/http_server/endpoints/w2l/list_w2l_templates.rs @@ -75,11 +75,14 @@ pub async fn list_w2l_templates_handler( http_request: HttpRequest, server_state: web::Data>) -> Result { - let scope_creator = None; + let no_scope_creator = None; + + // TODO(bt): Ideally we show users their own W2L templates here before they're mod approved. + let query_results = list_w2l_templates( &server_state.mysql_pool, - scope_creator, - false, + no_scope_creator, + true, ).await; let templates = match query_results { From af80527c47d6de944132bd090512bb1387ba6e07 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 22:56:56 -0400 Subject: [PATCH 0323/1042] Check TT2 model for validity and store file size --- .env | 1 + debug/curl/seed.py | 24 ++-- models/tacotron2/vocodes_model_check.py | 62 ++++++++++ src/job_queries/tts_download_job_queries.rs | 26 ++-- src/script_execution/mod.rs | 1 + .../tacotron_inference_command.rs | 1 - .../tacotron_model_check_command.rs | 112 ++++++++++++++++++ src/tts_download_job.rs | 63 +++++++++- 8 files changed, 267 insertions(+), 23 deletions(-) create mode 100755 models/tacotron2/vocodes_model_check.py create mode 100644 src/script_execution/tacotron_model_check_command.rs diff --git a/.env b/.env index 1a588dc7a5..a8b1b87739 100644 --- a/.env +++ b/.env @@ -40,6 +40,7 @@ W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py TTS_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/tacotron2 TTS_INFERENCE_SCRIPT_NAME=vocodes_inference.py +TTS_MODEL_CHECK_SCRIPT_NAME=vocodes_model_check.py #TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_universal_v5.pt # THIS SOUNDS HORRID #TTS_VOCODER_MODEL_FILENAME=waveglow_256channels_v4_uberduck.pt # THIS IS TOO SMALL! 300MB vs 1.2 GB! diff --git a/debug/curl/seed.py b/debug/curl/seed.py index a04896e040..877c52638e 100755 --- a/debug/curl/seed.py +++ b/debug/curl/seed.py @@ -104,19 +104,19 @@ def upload_tts_model(content_url, title, session_cookie): #upload_tts_model('https://drive.google.com/file/d/1w3oBk5vzyHurqYwP95UEQv-_NeMNu6pt/view?usp=sharing', 'Noire', session_echelon) # Copied into Storyteller Google Drive -upload_tts_model('https://drive.google.com/file/d/1d8KPwYqbRsGiYx1c4TCW4xCtcRLdIU1s/view?usp=sharing', 'Zephyr', session_echelon) -upload_tts_model('https://drive.google.com/file/d/1yhoamPz1NOMy403gu2v8sIx5hPj3LQWs/view?usp=sharing', 'Wakko Warner', session_other) -upload_tts_model('https://drive.google.com/file/d/1I6KGQZFMMB2ZvCzWEqy9xTgGl0L5cmfP/view?usp=sharing', 'Turret-192', session_echelon) -upload_tts_model('https://drive.google.com/file/d/1T_hvgGcNOfofhNmDWUqpVtYDdSXQrWdE/view?usp=sharing', 'technoblade-324', session_brandon) -upload_tts_model('https://drive.google.com/file/d/1NKyRiex3K1J0QyIv2np17D_H0rE98HyK/view?usp=sharing', 'Noire', session_other) -upload_tts_model('https://drive.google.com/file/d/1VDHDOpsUR8C-MD07vEsnUf1Lpcf8hXg4/view?usp=sharing', 'MatPat80', session_other) -upload_tts_model('https://drive.google.com/file/d/1atFn4vm7mjkufXNK9TZS6J8WzISrAn3-/view?usp=sharing', 'King Julien-70', session_other) -upload_tts_model('https://drive.google.com/file/d/1-rlJpi2jAsagn-UCStcCc_ITNycXgn2F/view?usp=sharing', 'Jorgen', session_other) -upload_tts_model('https://drive.google.com/file/d/1togTNOEKnaarqWWrbZLSwIbZIZ8IT_qw/view?usp=sharing', 'edsheeransinging', session_nanashi) +# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1d8KPwYqbRsGiYx1c4TCW4xCtcRLdIU1s/view?usp=sharing', 'Zephyr', session_echelon) +# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1yhoamPz1NOMy403gu2v8sIx5hPj3LQWs/view?usp=sharing', 'Wakko Warner', session_other) +# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1I6KGQZFMMB2ZvCzWEqy9xTgGl0L5cmfP/view?usp=sharing', 'Turret-192', session_echelon) +# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1T_hvgGcNOfofhNmDWUqpVtYDdSXQrWdE/view?usp=sharing', 'technoblade-324', session_brandon) +# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1NKyRiex3K1J0QyIv2np17D_H0rE98HyK/view?usp=sharing', 'Noire', session_other) +# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1VDHDOpsUR8C-MD07vEsnUf1Lpcf8hXg4/view?usp=sharing', 'MatPat80', session_other) +# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1atFn4vm7mjkufXNK9TZS6J8WzISrAn3-/view?usp=sharing', 'King Julien-70', session_other) +# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1-rlJpi2jAsagn-UCStcCc_ITNycXgn2F/view?usp=sharing', 'Jorgen', session_other) +# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1togTNOEKnaarqWWrbZLSwIbZIZ8IT_qw/view?usp=sharing', 'edsheeransinging', session_nanashi) +# TEMP COMMENT DUE TO RATE LIMIT +# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1OAiu0cZDyUJMemyLc0Btqw7zEQgiN81D/view?usp=sharing', 'Wakko Warner v2 ST21', session_nanashi) -upload_tts_model('https://drive.google.com/file/d/1OAiu0cZDyUJMemyLc0Btqw7zEQgiN81D/view?usp=sharing', 'Wakko Warner v2 ST21', session_nanashi) +upload_tts_model('https://drive.google.com/file/d/1vRVI2gQ5tAs8SLQPrR_M1J77L05gGBOp/view?usp=sharing', 'Testing voice', session_nanashi) #upload_tts_model('', '', session_other) -#upload_tts_model('', '', session_other) -#upload_tts_model('', '', session_other) diff --git a/models/tacotron2/vocodes_model_check.py b/models/tacotron2/vocodes_model_check.py new file mode 100755 index 0000000000..8e13c73430 --- /dev/null +++ b/models/tacotron2/vocodes_model_check.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +""" +Vocodes Tacotron Model Check +""" + +import argparse + +parser = argparse.ArgumentParser(description='Check that the model is valid') + +parser.add_argument('--synthesizer_checkpoint_path', type=str, help='path the TTS synthesizer model', required=True) +parser.add_argument('--output_metadata_filename', type=str, help='where to save extra metadata', required=True) + +args = parser.parse_args() + +import sys +sys.path.append('waveglow/') +import json + +import torch + +from hparams import create_hparams +from model import Tacotron2 + +# For metadata +import os + +print('========================================') +print('Python interpreter', sys.executable) +print('PyTorch version', torch.__version__) +print('CUDA Available?', torch.cuda.is_available()) +print('CUDA Device count', torch.cuda.device_count()) +print('========================================', flush=True) + +# NB(bt, 2021-05-31): Trying to get everything on the same device +#torch.device("cuda" if torch.cuda.is_available() else "cpu") + +hparams = create_hparams() +hparams.sampling_rate = 22050 # Don't change this +hparams.max_decoder_steps = 1000 # How long the audio will be before it cuts off (1000 is about 11 seconds) +hparams.gate_threshold = 0.1 # Model must be 90% sure the clip is over before ending generation (the higher this number is, the more likely that the AI will keep generating until it reaches the Max Decoder Steps) + +# Load synthesizer +checkpoint_path = args.synthesizer_checkpoint_path + +#model = load_model(hparams) +model = Tacotron2(hparams) + +model.load_state_dict(torch.load(checkpoint_path)['state_dict'], strict=True) + +_ = model.cuda().eval().half() + +print('Model is valid') + +file_size_bytes = os.path.getsize(args.synthesizer_checkpoint_path) + +metadata = { + 'file_size_bytes': file_size_bytes, +} + +with open(args.output_metadata_filename, 'w') as json_file: + json.dump(metadata, json_file) diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index ff9f86ddab..68ff48dae7 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -8,6 +8,7 @@ use crate::util::anyhow_result::AnyhowResult; use crate::util::random_crockford_token::random_crockford_token; use sqlx::MySqlPool; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use std::path::Path; /// table: tts_model_upload_jobs #[derive(Debug)] @@ -116,15 +117,22 @@ WHERE id = ? Ok(()) } -pub async fn insert_tts_model(pool: &MySqlPool, - job: &TtsUploadJobRecord, - private_bucket_hash: &str, - private_bucket_object_name: &str) - -> AnyhowResult<(u64, String)> -{ +pub async fn insert_tts_model>( + pool: &MySqlPool, + job: &TtsUploadJobRecord, + private_bucket_hash: &str, + private_bucket_object_name: P, + file_size_bytes: u64 +) -> AnyhowResult<(u64, String)> { + let model_token = random_prefix_crockford_token("TTS_MDL:", 32)?; let updatable_slug = model_token.clone(); + let private_bucket_object_name = &private_bucket_object_name + .as_ref() + .display() + .to_string(); + let query_result = sqlx::query!( r#" INSERT INTO tts_models @@ -139,7 +147,8 @@ SET creator_ip_address = ?, original_download_url = ?, private_bucket_hash = ?, - private_bucket_object_name = ? + private_bucket_object_name = ?, + file_size_bytes = ? "#, &model_token, updatable_slug, @@ -148,7 +157,8 @@ SET job.creator_ip_address.clone(), job.download_url.clone(), private_bucket_hash.to_string(), - private_bucket_object_name.to_string() + private_bucket_object_name.to_string(), + file_size_bytes ) .execute(pool) .await; diff --git a/src/script_execution/mod.rs b/src/script_execution/mod.rs index 24de37d06f..2b606badaf 100644 --- a/src/script_execution/mod.rs +++ b/src/script_execution/mod.rs @@ -3,5 +3,6 @@ pub mod ffmpeg_generate_preview_video_command; pub mod google_drive_download_command; pub mod imagemagick_generate_preview_image_command; pub mod tacotron_inference_command; +pub mod tacotron_model_check_command; pub mod wav2lip_inference_command; pub mod wav2lip_process_upload_command; \ No newline at end of file diff --git a/src/script_execution/tacotron_inference_command.rs b/src/script_execution/tacotron_inference_command.rs index 7a9b3c0344..7249c1cabd 100644 --- a/src/script_execution/tacotron_inference_command.rs +++ b/src/script_execution/tacotron_inference_command.rs @@ -6,7 +6,6 @@ use std::fs::OpenOptions; use std::path::Path; /// This command is used to run inference. -/// It uses preprocessed face files so that it's much faster. #[derive(Clone)] pub struct TacotronInferenceCommand { tacotron_directory: String, diff --git a/src/script_execution/tacotron_model_check_command.rs b/src/script_execution/tacotron_model_check_command.rs new file mode 100644 index 0000000000..a66fbb9be3 --- /dev/null +++ b/src/script_execution/tacotron_model_check_command.rs @@ -0,0 +1,112 @@ +use crate::AnyhowResult; +use log::{info,warn}; +use std::process::{Command, Stdio}; +use subprocess::{Popen, PopenConfig, Redirection}; +use std::fs::OpenOptions; +use std::path::Path; + +/// This command is used to check tacotron for being a real model +#[derive(Clone)] +pub struct TacotronModelCheckCommand { + tacotron_directory: String, + check_model_script_name: String, +} + +impl TacotronModelCheckCommand { + pub fn new( + tacotron_directory: &str, + check_model_script_name: &str, + ) -> Self { + Self { + tacotron_directory: tacotron_directory.to_string(), + check_model_script_name: check_model_script_name.to_string(), + } + } + + pub fn execute>( + &self, + synthesizer_checkpoint_path: P, + output_metadata_filename: P, + spawn_process: bool + ) -> AnyhowResult<()> { + let mut command = String::new(); + + command.push_str("echo 'test'"); + command.push_str(" && "); + command.push_str(&format!("cd {}", self.tacotron_directory)); + command.push_str(" && "); + command.push_str("source python/bin/activate"); + command.push_str(" && "); + command.push_str("python "); + command.push_str(&self.check_model_script_name); + command.push_str(" --synthesizer_checkpoint_path "); + command.push_str(&synthesizer_checkpoint_path.as_ref().display().to_string()); + command.push_str(" --output_metadata_filename "); + command.push_str(&output_metadata_filename.as_ref().display().to_string()); + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + if spawn_process { + // NB: This forks and returns immediately. + //let _child_pid = command_builder.spawn()?; + + let stdout_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/tacotron_upload_stdout.txt")?; + + let stderr_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/tacotron_upload_stderr.txt")?; + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + stdout: Redirection::File(stdout_file), + stderr: Redirection::File(stderr_file), + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + p.detach(); + + } else { + // NB: This is a blocking call. + /*let output = command_builder.output()?; + + info!("Output status: {}", output.status); + info!("Stdout: {:?}", String::from_utf8(output.stdout)); + error!("Stderr: {:?}", String::from_utf8(output.stderr)); + + if !output.status.success() { + bail!("Bad error code: {:?}", output.status); + }*/ + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + let exit_status = p.wait()?; + + info!("Exit status: {:?}", exit_status); + } + + Ok(()) + } +} diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 1bb77723b8..04cfe1a97e 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -28,8 +28,9 @@ use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_done; use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_failure; use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; +use crate::script_execution::tacotron_model_check_command::TacotronModelCheckCommand; use crate::util::anyhow_result::AnyhowResult; -use crate::util::filesystem::check_directory_exists; +use crate::util::filesystem::{check_directory_exists, check_file_exists}; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; @@ -55,6 +56,10 @@ const ENV_BUCKET_ROOT : &'static str = "TTS_DOWNLOAD_BUCKET_ROOT"; const DEFAULT_TEMP_DIR: &'static str = "/tmp"; +// Python code +const ENV_CODE_DIRECTORY : &'static str = "TTS_CODE_DIRECTORY"; +const ENV_MODEL_CHECK_SCRIPT_NAME : &'static str = "TTS_MODEL_CHECK_SCRIPT_NAME"; + struct Downloader { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, @@ -64,6 +69,8 @@ struct Downloader { pub bucket_path_unifier: BucketPathUnifier, + pub tts_check: TacotronModelCheckCommand, + // Command to run pub download_script: String, // Root to store TTS results @@ -130,6 +137,14 @@ async fn main() -> AnyhowResult<()> { mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe }; + let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?; + let py_script_name = easyenv::get_env_string_required(ENV_MODEL_CHECK_SCRIPT_NAME)?; + + let tts_model_check_command= TacotronModelCheckCommand::new( + &py_code_directory, + &py_script_name, + ); + let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, @@ -139,6 +154,7 @@ async fn main() -> AnyhowResult<()> { bucket_path_unifier: BucketPathUnifier::default_paths(), bucket_root_tts_model_uploads: bucket_root.to_string(), firehose_publisher, + tts_check: tts_model_check_command, }; main_loop(downloader).await; @@ -206,6 +222,18 @@ async fn process_jobs(downloader: &Downloader, jobs: Vec) -> Ok(()) } +#[derive(Deserialize)] +struct FileMetadata { + pub file_size_bytes: u64, +} + +fn read_metadata_file(filename: &PathBuf) -> AnyhowResult { + let mut file = File::open(filename)?; + let mut buffer = String::new(); + file.read_to_string(&mut buffer)?; + Ok(serde_json::from_str(&buffer)?) +} + async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. (DONE) @@ -216,6 +244,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; + // ==================== DOWNLOAD MODEL FILE ==================== // + let download_url = job.download_url.as_ref() .map(|c| c.to_string()) .unwrap_or("".to_string()); @@ -233,15 +263,44 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("Destination bucket path: {:?}", &synthesizer_model_bucket_path); + + // ==================== RUN MODEL CHECK ==================== // + + info!("Checking that model is valid..."); + let file_path = PathBuf::from(download_filename); + + let output_metadata_fs_path = temp_dir.path().join("metadata.json"); + + downloader.tts_check.execute( + &file_path, + &output_metadata_fs_path, + false, + )?; + + // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // + + info!("Checking that metadata output file exists..."); + + check_file_exists(&output_metadata_fs_path)?; + + let file_metadata = read_metadata_file(&output_metadata_fs_path)?; + + // ==================== UPLOAD MODEL FILE ==================== // + + info!("Uploading model to GCS..."); + downloader.bucket_client.upload_filename(&synthesizer_model_bucket_path, &file_path).await?; + // ==================== SAVE RECORDS ==================== // + info!("Saving model record..."); let (id, model_token) = insert_tts_model( &downloader.mysql_pool, job, &private_bucket_hash, - synthesizer_model_bucket_path.as_path().to_str().unwrap_or("")) + synthesizer_model_bucket_path, + file_metadata.file_size_bytes) .await?; info!("Marking job complete..."); From b6bd4ab68c2e252fda08efcc2bd837e33974b008 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 7 Jun 2021 23:57:21 -0400 Subject: [PATCH 0324/1042] fix debugging script --- debug/mysql/last_tts_model.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/mysql/last_tts_model.sh b/debug/mysql/last_tts_model.sh index 566dcfa713..aacd3fc0da 100755 --- a/debug/mysql/last_tts_model.sh +++ b/debug/mysql/last_tts_model.sh @@ -1,7 +1,7 @@ #!/bin/bash # ONLY FOR LOCAL DEV -mysql -u root \ - -proot \ +mysql -u storyteller \ + -ppassword \ -h localhost \ -D storyteller \ -e "select * from tts_models\G" From 373726b6859e87188212f42afd4fa386c6304c12 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 00:36:23 -0400 Subject: [PATCH 0325/1042] this locking appears to work (testing) --- src/job_queries/tts_download_job_queries.rs | 85 +++++++++++++++++++- src/job_queries/tts_inference_job_queries.rs | 81 +++++++++++++++++++ src/tts_inference_job.rs | 9 +++ 3 files changed, 171 insertions(+), 4 deletions(-) diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index 68ff48dae7..0ec209d05b 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -62,10 +62,86 @@ WHERE Ok(job_records) } -pub async fn mark_tts_upload_job_failure(pool: &MySqlPool, - job: &TtsUploadJobRecord, - failure_reason: &str) - -> AnyhowResult<()> +/*pub struct TtsLockRecord { + id: i64, + status: String, +} + +pub async fn can_grab_job_lock( + pool: &MySqlPool, + job: &TtsUploadJobRecord +) -> AnyhowResult { + + // NB: We use transactions and "SELECT ... FOR UPDATE" to simulate mutexes. + let mut transaction = pool.begin().await?; + + let maybe_record = sqlx::query_as!( + TtsLockRecord, + r#" +SELECT + id, + status +FROM tts_model_upload_jobs +FOR UPDATE +WHERE id = ? + "#, + status, + failure_reason.to_string(), + job.id, + ) + .fetch_one(&mut transaction) + .await?; + + let record : TtsLockRecord = match maybe_record { + Ok(record) => record, + Err(err) => { + match err { + RowNotFound => { + return Err(anyhow!("could not job")); + }, + _ => { + return Err(anyhow!("query error")); + } + } + } + }; + + let can_transact = match record.status.as_ref() { + "pending" => true, // It's okay for us to take the lock. + "attempt_failed" => true, // We can retry. + "started" => false, // Job in progress (another job beat us, and we can't take the lock) + "complete_success" => false, // Job already complete + "complete_failure" => false, // Job already complete (permanently dead; no need to retry) + "dead" => false, // Job already complete (permanently dead; retries exhausted) + _ => false, // Future-proof + }; + + if !can_transact { + return Ok(false); + } + + let _acquire_lock = sqlx::query!( + r#" +UPDATE tts_model_upload_jobs +SET + status = 'started', + retry_at = NOW() + interval 2 minute +WHERE id = ? + "#, + job.id, + ) + .execute(&mut transaction) + .await?; + + + Ok(true) +}*/ + +pub async fn mark_tts_upload_job_failure( + pool: &MySqlPool, + job: &TtsUploadJobRecord, + failure_reason: &str +) -> AnyhowResult<()> { // statuses: "attempt_failed", "complete_failure", "dead" let status = "attempt_failed"; @@ -88,6 +164,7 @@ WHERE id = ? Ok(()) } + pub async fn mark_tts_upload_job_done( pool: &MySqlPool, job: &TtsUploadJobRecord, diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 1e7b1e03cd..9d1450facd 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -79,6 +79,87 @@ WHERE Ok(job_records) } + +pub struct TtsInferenceLockRecord { + id: i64, + status: String, +} + +pub async fn can_grab_job_lock( + pool: &MySqlPool, + job: &TtsInferenceJobRecord +) -> AnyhowResult { + + // NB: We use transactions and "SELECT ... FOR UPDATE" to simulate mutexes. + let mut transaction = pool.begin().await?; + + let maybe_record = sqlx::query_as!( + TtsInferenceLockRecord, + r#" +SELECT + id, + status +FROM tts_inference_jobs +WHERE id = ? +FOR UPDATE + "#, + job.id, + ) + .fetch_one(&mut transaction) + .await; + + let record : TtsInferenceLockRecord = match maybe_record { + Ok(record) => record, + Err(err) => { + match err { + RowNotFound => { + return Err(anyhow!("could not job")); + }, + _ => { + return Err(anyhow!("query error")); + } + } + } + }; + + let can_transact = match record.status.as_ref() { + "pending" => true, // It's okay for us to take the lock. + "attempt_failed" => true, // We can retry. + "started" => false, // Job in progress (another job beat us, and we can't take the lock) + "complete_success" => false, // Job already complete + "complete_failure" => false, // Job already complete (permanently dead; no need to retry) + "dead" => false, // Job already complete (permanently dead; retries exhausted) + _ => false, // Future-proof + }; + + info!("Can acquire lock??? {}", can_transact); + info!("Can acquire lock??? {}", can_transact); + info!("Can acquire lock??? {}", can_transact); + info!("Can acquire lock??? {}", can_transact); + + if !can_transact { + return Ok(false); + } + + let _acquire_lock = sqlx::query!( + r#" +UPDATE tts_inference_jobs +SET + status = 'started', + retry_at = NOW() + interval 2 minute +WHERE id = ? + "#, + job.id, + ) + .execute(&mut transaction) + .await?; + + transaction.commit().await?; + + Ok(true) +} + + pub async fn mark_tts_inference_job_failure( pool: &MySqlPool, job: &TtsInferenceJobRecord, diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index f56931d6f6..f538d09ef1 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -23,6 +23,7 @@ use crate::buckets::bucket_path_unifier::BucketPathUnifier; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_inference_job_queries::TtsInferenceJobRecord; +use crate::job_queries::tts_inference_job_queries::can_grab_job_lock; use crate::job_queries::tts_inference_job_queries::get_tts_model_by_token; use crate::job_queries::tts_inference_job_queries::insert_tts_result; use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_done; @@ -294,6 +295,14 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An // TODO 8. Save record // TODO 9. Mark job done + // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // + + let lock_acquired = can_grab_job_lock(&inferencer.mysql_pool, job).await?; + + if !lock_acquired { + warn!("Could not acquire job lock for: {}", &job.id); + return Ok(()) + } // ==================== CONFIRM OR DOWNLOAD TTS VOCODER MODEL ==================== // From 9ce83025b449c500b237f6fbb7f3db8e57fb92b7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 00:47:41 -0400 Subject: [PATCH 0326/1042] mutual exclusion seems to work well! --- src/job_queries/tts_inference_job_queries.rs | 2 ++ src/tts_inference_job.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 9d1450facd..7025efc721 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -138,6 +138,8 @@ FOR UPDATE info!("Can acquire lock??? {}", can_transact); if !can_transact { + info!("returning"); + transaction.rollback().await?; return Ok(false); } diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index f538d09ef1..961dbb7215 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -223,7 +223,7 @@ async fn main_loop(inferencer: Inferencer) { if jobs.is_empty() { info!("No jobs!"); - std::thread::sleep(Duration::from_millis(1500)); + std::thread::sleep(Duration::from_millis(20)); continue; } @@ -241,7 +241,7 @@ async fn main_loop(inferencer: Inferencer) { timeout_millis = START_TIMEOUT_MILLIS; // reset - std::thread::sleep(Duration::from_millis(500)); + std::thread::sleep(Duration::from_millis(20)); } } From 23801d90f65319d571ccde4308500fe27fcfc39c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 01:08:39 -0400 Subject: [PATCH 0327/1042] remove debug log --- src/job_queries/tts_inference_job_queries.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 7025efc721..cea35efe5b 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -132,13 +132,7 @@ FOR UPDATE _ => false, // Future-proof }; - info!("Can acquire lock??? {}", can_transact); - info!("Can acquire lock??? {}", can_transact); - info!("Can acquire lock??? {}", can_transact); - info!("Can acquire lock??? {}", can_transact); - if !can_transact { - info!("returning"); transaction.rollback().await?; return Ok(false); } From c08000d11aa9b852e4fb1e10825508c3e62350a6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 01:42:49 -0400 Subject: [PATCH 0328/1042] lock on tts download job (not tested) --- src/job_queries/tts_download_job_queries.rs | 22 ++++++++++---------- src/job_queries/tts_inference_job_queries.rs | 3 +-- src/tts_download_job.rs | 14 +++++++++++-- src/tts_inference_job.rs | 6 +++--- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index 0ec209d05b..86d0e8f771 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -62,12 +62,12 @@ WHERE Ok(job_records) } -/*pub struct TtsLockRecord { +pub struct TtsUploadLockRecord { id: i64, status: String, } -pub async fn can_grab_job_lock( +pub async fn grab_job_lock_and_mark_pending( pool: &MySqlPool, job: &TtsUploadJobRecord ) -> AnyhowResult { @@ -76,23 +76,21 @@ pub async fn can_grab_job_lock( let mut transaction = pool.begin().await?; let maybe_record = sqlx::query_as!( - TtsLockRecord, + TtsUploadLockRecord, r#" SELECT id, status FROM tts_model_upload_jobs -FOR UPDATE WHERE id = ? +FOR UPDATE "#, - status, - failure_reason.to_string(), job.id, ) .fetch_one(&mut transaction) - .await?; + .await; - let record : TtsLockRecord = match maybe_record { + let record : TtsUploadLockRecord = match maybe_record { Ok(record) => record, Err(err) => { match err { @@ -117,6 +115,7 @@ WHERE id = ? }; if !can_transact { + transaction.rollback().await?; return Ok(false); } @@ -130,12 +129,13 @@ WHERE id = ? "#, job.id, ) - .execute(&mut transaction) - .await?; + .execute(&mut transaction) + .await?; + transaction.commit().await?; Ok(true) -}*/ +} pub async fn mark_tts_upload_job_failure( pool: &MySqlPool, diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index cea35efe5b..ef3ed7b7ea 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -85,7 +85,7 @@ pub struct TtsInferenceLockRecord { status: String, } -pub async fn can_grab_job_lock( +pub async fn grab_job_lock_and_mark_pending( pool: &MySqlPool, job: &TtsInferenceJobRecord ) -> AnyhowResult { @@ -155,7 +155,6 @@ WHERE id = ? Ok(true) } - pub async fn mark_tts_inference_job_failure( pool: &MySqlPool, job: &TtsInferenceJobRecord, diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 04cfe1a97e..e917eb1955 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -23,6 +23,7 @@ use crate::buckets::bucket_path_unifier::BucketPathUnifier; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; +use crate::job_queries::tts_download_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::tts_download_job_queries::insert_tts_model; use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_done; use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_failure; @@ -241,11 +242,20 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho // TODO: 4. Save record. (DONE) // TODO: 5. Mark job done. (DONE) - let temp_dir = format!("temp_{}", job.id); - let temp_dir = TempDir::new(&temp_dir)?; + // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // + + let lock_acquired = grab_job_lock_and_mark_pending(&downloader.mysql_pool, job).await?; + + if !lock_acquired { + warn!("Could not acquire job lock for: {}", &job.id); + return Ok(()) + } // ==================== DOWNLOAD MODEL FILE ==================== // + let temp_dir = format!("temp_{}", job.id); + let temp_dir = TempDir::new(&temp_dir)?; + let download_url = job.download_url.as_ref() .map(|c| c.to_string()) .unwrap_or("".to_string()); diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 961dbb7215..e0733963ee 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -23,8 +23,8 @@ use crate::buckets::bucket_path_unifier::BucketPathUnifier; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_inference_job_queries::TtsInferenceJobRecord; -use crate::job_queries::tts_inference_job_queries::can_grab_job_lock; use crate::job_queries::tts_inference_job_queries::get_tts_model_by_token; +use crate::job_queries::tts_inference_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::tts_inference_job_queries::insert_tts_result; use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_done; use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_failure; @@ -281,7 +281,7 @@ fn read_metadata_file(filename: &PathBuf) -> AnyhowResult { async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> AnyhowResult<()> { - // TODO 1. Mark processing + // TODO 1. Mark processing (DONE) // TODO 2. Check if vocoder model is downloaded / download to stable location (DONE) @@ -297,7 +297,7 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // - let lock_acquired = can_grab_job_lock(&inferencer.mysql_pool, job).await?; + let lock_acquired = grab_job_lock_and_mark_pending(&inferencer.mysql_pool, job).await?; if !lock_acquired { warn!("Could not acquire job lock for: {}", &job.id); From f088116dfa878e17ad5c8fe9b331285e0c6ffb53 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 01:51:57 -0400 Subject: [PATCH 0329/1042] job lock for w2l downloader (not tested) --- src/job_queries/w2l_download_job_queries.rs | 75 +++++++++++++++++++++ src/tts_download_job.rs | 6 +- src/w2l_download_job.rs | 20 ++++-- 3 files changed, 95 insertions(+), 6 deletions(-) diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 4d37117a30..8df5dc5bba 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -61,6 +61,81 @@ WHERE Ok(job_records) } +pub struct W2lDownloadLockRecord { + id: i64, + status: String, +} + +pub async fn grab_job_lock_and_mark_pending( + pool: &MySqlPool, + job: &W2lTemplateUploadJobRecord +) -> AnyhowResult { + + // NB: We use transactions and "SELECT ... FOR UPDATE" to simulate mutexes. + let mut transaction = pool.begin().await?; + + let maybe_record = sqlx::query_as!( + W2lDownloadLockRecord, + r#" +SELECT + id, + status +FROM w2l_template_upload_jobs +WHERE id = ? +FOR UPDATE + "#, + job.id, + ) + .fetch_one(&mut transaction) + .await; + + let record : W2lDownloadLockRecord = match maybe_record { + Ok(record) => record, + Err(err) => { + match err { + RowNotFound => { + return Err(anyhow!("could not job")); + }, + _ => { + return Err(anyhow!("query error")); + } + } + } + }; + + let can_transact = match record.status.as_ref() { + "pending" => true, // It's okay for us to take the lock. + "attempt_failed" => true, // We can retry. + "started" => false, // Job in progress (another job beat us, and we can't take the lock) + "complete_success" => false, // Job already complete + "complete_failure" => false, // Job already complete (permanently dead; no need to retry) + "dead" => false, // Job already complete (permanently dead; retries exhausted) + _ => false, // Future-proof + }; + + if !can_transact { + transaction.rollback().await?; + return Ok(false); + } + + let _acquire_lock = sqlx::query!( + r#" +UPDATE w2l_template_upload_jobs +SET + status = 'started', + retry_at = NOW() + interval 2 minute +WHERE id = ? + "#, + job.id, + ) + .execute(&mut transaction) + .await?; + + transaction.commit().await?; + + Ok(true) +} + pub async fn mark_w2l_template_upload_job_failure(pool: &MySqlPool, job: &W2lTemplateUploadJobRecord, failure_reason: &str) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index e917eb1955..16049fb5f4 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -236,7 +236,7 @@ fn read_metadata_file(filename: &PathBuf) -> AnyhowResult { } async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { - // TODO: 1. Mark processing. + // TODO: 1. Mark processing. (DONE) // TODO: 2. Download. (DONE) // TODO: 3. Upload. (DONE) // TODO: 4. Save record. (DONE) @@ -251,11 +251,13 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho return Ok(()) } - // ==================== DOWNLOAD MODEL FILE ==================== // + // ==================== SETUP TEMP DIRS ==================== // let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; + // ==================== DOWNLOAD MODEL FILE ==================== // + let download_url = job.download_url.as_ref() .map(|c| c.to_string()) .unwrap_or("".to_string()); diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 9a909812d3..ef726e6087 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -22,6 +22,7 @@ use crate::buckets::bucket_client::BucketClient; use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; +use crate::job_queries::w2l_download_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::w2l_download_job_queries::insert_w2l_template; use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_done; use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_failure; @@ -304,15 +305,26 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) // TODO: 7. Save record. (DONE) // TODO: 8. Mark job done. (DONE) + // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // + + let lock_acquired = grab_job_lock_and_mark_pending(&downloader.mysql_pool, job).await?; + + if !lock_acquired { + warn!("Could not acquire job lock for: {}", &job.id); + return Ok(()) + } + + // ==================== SETUP TEMP DIRS ==================== // + let temp_dir = format!("temp_{}", job.id); let temp_dir = TempDir::new(&temp_dir)?; - let download_url = job.download_url.as_ref() - .map(|c| c.to_string()) - .unwrap_or("".to_string()); - // ==================== DOWNLOAD FILE ==================== // + let download_url = job.download_url.as_ref() + .map(|c| c.to_string()) + .unwrap_or("".to_string()); + info!("Calling downloader..."); let download_filename = downloader.google_drive_downloader .download_file(&download_url, &temp_dir).await?; From e7018c085bc252b8aff943a7630a9a371914e6f5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 01:55:57 -0400 Subject: [PATCH 0330/1042] w2l inference job locks (not tested) --- src/job_queries/w2l_inference_job_queries.rs | 76 ++++++++++++++++++++ src/w2l_inference_job.rs | 10 +++ 2 files changed, 86 insertions(+) diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index 35b9350010..66bad12dc2 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -103,6 +103,82 @@ WHERE Ok(job_records) } +pub struct W2lInferenceLockRecord { + id: i64, + status: String, +} + +pub async fn grab_job_lock_and_mark_pending( + pool: &MySqlPool, + job: &W2lInferenceJobRecord +) -> AnyhowResult { + + // NB: We use transactions and "SELECT ... FOR UPDATE" to simulate mutexes. + let mut transaction = pool.begin().await?; + + let maybe_record = sqlx::query_as!( + W2lInferenceLockRecord, + r#" +SELECT + id, + status +FROM w2l_inference_jobs +WHERE id = ? +FOR UPDATE + "#, + job.id, + ) + .fetch_one(&mut transaction) + .await; + + let record : W2lInferenceLockRecord = match maybe_record { + Ok(record) => record, + Err(err) => { + match err { + RowNotFound => { + return Err(anyhow!("could not job")); + }, + _ => { + return Err(anyhow!("query error")); + } + } + } + }; + + let can_transact = match record.status.as_ref() { + "pending" => true, // It's okay for us to take the lock. + "attempt_failed" => true, // We can retry. + "started" => false, // Job in progress (another job beat us, and we can't take the lock) + "complete_success" => false, // Job already complete + "complete_failure" => false, // Job already complete (permanently dead; no need to retry) + "dead" => false, // Job already complete (permanently dead; retries exhausted) + _ => false, // Future-proof + }; + + if !can_transact { + transaction.rollback().await?; + return Ok(false); + } + + let _acquire_lock = sqlx::query!( + r#" +UPDATE w2l_inference_jobs +SET + status = 'started', + retry_at = NOW() + interval 2 minute +WHERE id = ? + "#, + job.id, + ) + .execute(&mut transaction) + .await?; + + transaction.commit().await?; + + Ok(true) +} + + pub async fn mark_w2l_inference_job_failure(pool: &MySqlPool, job: &W2lInferenceJobRecord, failure_reason: &str) diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index fa90615c46..7b3f6b6c29 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -25,6 +25,7 @@ use util::hashing::hash_file_sha2::hash_file_sha2; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; +use crate::job_queries::w2l_inference_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::w2l_inference_job_queries::insert_w2l_result; use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_done; use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failure; @@ -324,6 +325,15 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An // TODO 7. Save record // TODO 8. Mark job done + // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // + + let lock_acquired = grab_job_lock_and_mark_pending(&inferencer.mysql_pool, job).await?; + + if !lock_acquired { + warn!("Could not acquire job lock for: {}", &job.id); + return Ok(()) + } + // ==================== CONFIRM OR DOWNLOAD W2L MODEL ==================== // let model_filename = inferencer.w2l_model_filename.clone(); From 40b029274e3a321730f90f26dda45e967748948a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 01:56:37 -0400 Subject: [PATCH 0331/1042] re-codegen db --- sqlx-data.json | 198 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 195 insertions(+), 3 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index be39c464ca..92209881f3 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1,5 +1,43 @@ { "db": "MySQL", + "02060ab9b8ca472a2bd4d4fa3120743e90148ccd6a4cb13bc491d00f36ed7a8f": { + "query": "\nSELECT\n id,\n status\nFROM w2l_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + }, "0c32927e9292e0cf4aeb169ee3ed0ec122712a66a6850eb9e9da31d5a97e6e72": { "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_video_path as maybe_public_bucket_video_path,\n\n jobs.maybe_w2l_template_token,\n w2l.template_type as w2l_template_type,\n w2l.title,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_inference_jobs as jobs\nJOIN w2l_templates as w2l\n ON w2l.token = jobs.maybe_w2l_template_token\nLEFT OUTER JOIN w2l_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ", "describe": { @@ -129,6 +167,44 @@ ] } }, + "0caed749627ff22ae1cea52621a375156259a731e8070459593082b80d463491": { + "query": "\nSELECT\n id,\n status\nFROM tts_model_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + }, "0fc26cb6ab34ee6cd9d0cd8fac8bae7bd6f25bbf7c785cd5dc8888d08099b88b": { "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_template_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM w2l_template_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", "describe": { @@ -1851,6 +1927,16 @@ "nullable": [] } }, + "3bc397a1157c4564562138dbef80cf89d2185943f5f925228936f0b5c67e1b95": { + "query": "\nUPDATE tts_inference_jobs\nSET\n status = 'started',\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, "3de0e9c442ae0d4ca2f1da19357ac7685084e2843216313d7836ea60bc3dcfe0": { "query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.on_success_result_token as maybe_model_token,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_model_upload_jobs as jobs\n\nWHERE jobs.token = ?\n ", "describe": { @@ -2070,6 +2156,44 @@ ] } }, + "4f61f621682ba255814f8a55455ea2b7fba9b6c85affaf87abfe2f81e11322f0": { + "query": "\nSELECT\n id,\n status\nFROM tts_inference_jobs\nWHERE id = ?\nFOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + }, "4fec6d4e2b33411835c679942860c328e20a4276fb39c3512f0608261c5467f4": { "query": "\nSELECT\n tts.token as model_token,\n tts.tts_model_type,\n tts.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n tts.updatable_slug,\n tts.title,\n tts.is_mod_disabled,\n tts.created_at,\n tts.updated_at\nFROM tts_models as tts\nJOIN users\nON users.token = tts.creator_user_token\nWHERE\n tts.deleted_at IS NULL\n ", "describe": { @@ -2859,6 +2983,16 @@ "nullable": [] } }, + "79e6cc7829521ba8b26b5a97d3c8ab4e371945fa581e1632440c05658a2731ef": { + "query": "\nUPDATE w2l_inference_jobs\nSET\n status = 'started',\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, "7ab54466ca3021d90e0feb234acba56bf53d415dde69bc1c35120ed0c8dd1858": { "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { @@ -5261,6 +5395,16 @@ ] } }, + "b4582bc4d8865cc82bf1e0fe15209db07d1922a29c274b872cd5f5eaee4b261d": { + "query": "\nUPDATE w2l_template_upload_jobs\nSET\n status = 'started',\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, "b62d4ca915206b8f956a2664c72e02832d38ab34fb3be50d9a77cf8145297689": { "query": "\nSELECT\n w2l.token as template_token,\n w2l.template_type,\n w2l.creator_user_token,\n users.username as creator_username,\n users.display_name as creator_display_name,\n w2l.updatable_slug,\n w2l.title,\n w2l.frame_width,\n w2l.frame_height,\n w2l.duration_millis,\n w2l.maybe_public_bucket_preview_image_object_name,\n w2l.maybe_public_bucket_preview_video_object_name,\n w2l.is_mod_public_listing_approved,\n w2l.created_at,\n w2l.updated_at\nFROM w2l_templates as w2l\nJOIN users\nON\n users.token = w2l.creator_user_token\nWHERE\n w2l.deleted_at IS NULL\n AND w2l.is_mod_public_listing_approved IS TRUE\n AND users.username = ?\n ", "describe": { @@ -5727,12 +5871,12 @@ ] } }, - "b8e0f7961f7b339e4a81b7fd47f99d0cd07567b33f0d093b449586154dc16235": { - "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?\n ", + "bd151e17d1dd65c25ebc689a1d321ac14b8d5d8f1c9e8426c0c92ed278595de2": { + "query": "\nINSERT INTO tts_models\nSET\n token = ?,\n tts_model_type = \"tacotron2\",\n updatable_slug = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n creator_ip_address = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n file_size_bytes = ?\n ", "describe": { "columns": [], "parameters": { - "Right": 8 + "Right": 9 }, "nullable": [] } @@ -6093,6 +6237,44 @@ ] } }, + "d12757abef811cd6951e0e1427fdb496a8931b547519d320f5ade56007d3eb5d": { + "query": "\nSELECT\n id,\n status\nFROM w2l_template_upload_jobs\nWHERE id = ?\nFOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": { + "type": "LongLong", + "flags": { + "bits": 515 + }, + "char_set": 63, + "max_size": 20 + } + }, + { + "ordinal": 1, + "name": "status", + "type_info": { + "type": "String", + "flags": { + "bits": 393 + }, + "char_set": 224, + "max_size": 64 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + } + }, "d14d0e03acc71081726ef0e31bade27110f1ac8eff9ccfe155df5dbd47d06560": { "query": "\nSELECT\n id,\n token AS inference_job_token,\n uuid_idempotency_token,\n\n maybe_w2l_template_token,\n maybe_public_image_bucket_location,\n maybe_tts_inference_result_token,\n maybe_public_audio_bucket_hash,\n maybe_public_audio_bucket_location,\n\n maybe_original_audio_filename,\n maybe_original_audio_download_url,\n maybe_audio_mime_type,\n\n creator_ip_address,\n maybe_creator_user_token,\n\n creator_set_visibility,\n disable_end_bump,\n disable_watermark,\n\n status,\n attempt_count,\n failure_reason,\n created_at,\n updated_at,\n retry_at\nFROM w2l_inference_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n ", "describe": { @@ -6520,6 +6702,16 @@ ] } }, + "d5d3e1bd935d32639ad7398936f4c8ee9f3caff6c0789db29b45a81555d1bdce": { + "query": "\nUPDATE tts_model_upload_jobs\nSET\n status = 'started',\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + } + }, "e73192a3e763a287d5737821443c5a81649f17e39978497b99c52e1e896a75f6": { "query": "\nSELECT\n token as session_token,\n user_token\nFROM user_sessions\nWHERE token = ?\nAND deleted_at IS NULL\n ", "describe": { From 1bc18487f007326f1df81d7f19f46cfaae5990fe Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 01:58:17 -0400 Subject: [PATCH 0332/1042] sort results --- loc.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/loc.sh b/loc.sh index dcc9b46613..7f422b148f 100755 --- a/loc.sh +++ b/loc.sh @@ -7,4 +7,5 @@ wc -l \ src/*/*.rs \ src/*/*/*.rs \ src/*/*/*/*.rs \ - src/*/*/*/*/*.rs + src/*/*/*/*/*.rs \ + | sort -n From e83c003320a814251746676af2cef39bae4bb21b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 02:49:11 -0400 Subject: [PATCH 0333/1042] central config for batch query timeout --- src/common_env.rs | 22 ++++++++++++++++++++++ src/tts_download_job.rs | 24 ++++++++++++++++-------- src/tts_inference_job.rs | 24 ++++++++++++++++-------- src/w2l_download_job.rs | 24 ++++++++++++++++-------- src/w2l_inference_job.rs | 24 ++++++++++++++++-------- 5 files changed, 86 insertions(+), 32 deletions(-) create mode 100644 src/common_env.rs diff --git a/src/common_env.rs b/src/common_env.rs new file mode 100644 index 0000000000..e04938a702 --- /dev/null +++ b/src/common_env.rs @@ -0,0 +1,22 @@ +use crate::util::anyhow_result::AnyhowResult; + +// TODO: Move more shared configs here. + +/// Environment variables shared between one or more jobs or the main server. +/// These are +pub struct CommonEnv { + + /// The amount of time to wait between job batches (not individual jobs). + /// This prevents the outer loop of querying batches from flooding the DB. + /// (In theory, there's work within jobs that prevents rapidly pegging the DB.) + pub job_batch_wait_millis: u64, +} + +impl CommonEnv { + + pub fn read_from_env() -> AnyhowResult { + Ok(Self { + job_batch_wait_millis: easyenv::get_env_num("JOB_BATCH_WAIT_MILLIS", 100)?, + }) + } +} \ No newline at end of file diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 16049fb5f4..b220c29c3b 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -9,6 +9,7 @@ #[macro_use] extern crate serde_derive; pub mod buckets; +pub mod common_env; pub mod common_queries; pub mod database_helpers; pub mod job_queries; @@ -21,6 +22,7 @@ use chrono::Utc; use crate::buckets::bucket_client::BucketClient; use crate::buckets::bucket_path_unifier::BucketPathUnifier; use crate::buckets::bucket_paths::hash_to_bucket_path; +use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; use crate::job_queries::tts_download_job_queries::grab_job_lock_and_mark_pending; @@ -76,6 +78,9 @@ struct Downloader { pub download_script: String, // Root to store TTS results pub bucket_root_tts_model_uploads: String, + + // Sleep between batches + pub job_batch_wait_millis: u64, } #[tokio::main] @@ -146,6 +151,8 @@ async fn main() -> AnyhowResult<()> { &py_script_name, ); + let common_env = CommonEnv::read_from_env()?; + let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, @@ -156,6 +163,7 @@ async fn main() -> AnyhowResult<()> { bucket_root_tts_model_uploads: bucket_root.to_string(), firehose_publisher, tts_check: tts_model_check_command, + job_batch_wait_millis: common_env.job_batch_wait_millis, }; main_loop(downloader).await; @@ -167,7 +175,7 @@ const START_TIMEOUT_MILLIS : u64 = 500; const INCREASE_TIMEOUT_MILLIS : u64 = 1000; async fn main_loop(downloader: Downloader) { - let mut timeout_millis = START_TIMEOUT_MILLIS; + let mut error_timeout_millis = START_TIMEOUT_MILLIS; loop { let num_records = 1; @@ -177,15 +185,15 @@ async fn main_loop(downloader: Downloader) { Ok(jobs) => jobs, Err(e) => { warn!("Error querying jobs: {:?}", e); - std::thread::sleep(Duration::from_millis(timeout_millis)); - timeout_millis += INCREASE_TIMEOUT_MILLIS; + std::thread::sleep(Duration::from_millis(error_timeout_millis)); + error_timeout_millis += INCREASE_TIMEOUT_MILLIS; continue; } }; if jobs.is_empty() { info!("No jobs!"); - std::thread::sleep(Duration::from_millis(1500)); + std::thread::sleep(Duration::from_millis(downloader.job_batch_wait_millis)); continue; } @@ -195,15 +203,15 @@ async fn main_loop(downloader: Downloader) { Ok(_) => {}, Err(e) => { warn!("Error querying jobs: {:?}", e); - std::thread::sleep(Duration::from_millis(timeout_millis)); - timeout_millis += INCREASE_TIMEOUT_MILLIS; + std::thread::sleep(Duration::from_millis(error_timeout_millis)); + error_timeout_millis += INCREASE_TIMEOUT_MILLIS; continue; } } - timeout_millis = START_TIMEOUT_MILLIS; // reset + error_timeout_millis = START_TIMEOUT_MILLIS; // reset - std::thread::sleep(Duration::from_millis(500)); + std::thread::sleep(Duration::from_millis(downloader.job_batch_wait_millis)); } } diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index e0733963ee..a76e6760b3 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -9,6 +9,7 @@ #[macro_use] extern crate serde_derive; pub mod buckets; +pub mod common_env; pub mod common_queries; pub mod database_helpers; pub mod job_queries; @@ -21,6 +22,7 @@ use chrono::Utc; use crate::buckets::bucket_client::BucketClient; use crate::buckets::bucket_path_unifier::BucketPathUnifier; use crate::buckets::bucket_paths::hash_to_bucket_path; +use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_inference_job_queries::TtsInferenceJobRecord; use crate::job_queries::tts_inference_job_queries::get_tts_model_by_token; @@ -86,6 +88,9 @@ struct Inferencer { // Command to run pub tts_vocoder_model_filename: String, + + // Sleep between batches + pub job_batch_wait_millis: u64, } #[tokio::main] @@ -177,6 +182,8 @@ async fn main() -> AnyhowResult<()> { mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe }; + let common_env = CommonEnv::read_from_env()?; + let inferencer = Inferencer { download_temp_directory: temp_directory, mysql_pool, @@ -190,6 +197,7 @@ async fn main() -> AnyhowResult<()> { semi_persistent_cache, firehose_publisher, tts_vocoder_model_filename, + job_batch_wait_millis: common_env.job_batch_wait_millis, }; main_loop(inferencer).await; @@ -201,7 +209,7 @@ const START_TIMEOUT_MILLIS : u64 = 500; const INCREASE_TIMEOUT_MILLIS : u64 = 1000; async fn main_loop(inferencer: Inferencer) { - let mut timeout_millis = START_TIMEOUT_MILLIS; + let mut error_timeout_millis = START_TIMEOUT_MILLIS; loop { let num_records = 1; @@ -215,15 +223,15 @@ async fn main_loop(inferencer: Inferencer) { Ok(jobs) => jobs, Err(e) => { warn!("Error querying jobs: {:?}", e); - std::thread::sleep(Duration::from_millis(timeout_millis)); - timeout_millis += INCREASE_TIMEOUT_MILLIS; + std::thread::sleep(Duration::from_millis(error_timeout_millis)); + error_timeout_millis += INCREASE_TIMEOUT_MILLIS; continue; } }; if jobs.is_empty() { info!("No jobs!"); - std::thread::sleep(Duration::from_millis(20)); + std::thread::sleep(Duration::from_millis(inferencer.job_batch_wait_millis)); continue; } @@ -233,15 +241,15 @@ async fn main_loop(inferencer: Inferencer) { Ok(_) => {}, Err(e) => { warn!("Error querying jobs: {:?}", e); - std::thread::sleep(Duration::from_millis(timeout_millis)); - timeout_millis += INCREASE_TIMEOUT_MILLIS; + std::thread::sleep(Duration::from_millis(error_timeout_millis)); + error_timeout_millis += INCREASE_TIMEOUT_MILLIS; continue; } } - timeout_millis = START_TIMEOUT_MILLIS; // reset + error_timeout_millis = START_TIMEOUT_MILLIS; // reset - std::thread::sleep(Duration::from_millis(20)); + std::thread::sleep(Duration::from_millis(inferencer.job_batch_wait_millis)); } } diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index ef726e6087..1307714918 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -9,6 +9,7 @@ #[macro_use] extern crate serde_derive; pub mod buckets; +pub mod common_env; pub mod common_queries; pub mod database_helpers; pub mod job_queries; @@ -20,6 +21,7 @@ use anyhow::anyhow; use chrono::Utc; use crate::buckets::bucket_client::BucketClient; use crate::buckets::bucket_paths::hash_to_bucket_path; +use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; use crate::job_queries::w2l_download_job_queries::grab_job_lock_and_mark_pending; @@ -98,6 +100,9 @@ struct Downloader { pub debug_download_sleep_millis: u64, pub debug_face_detect_sleep_millis: u64, pub debug_job_end_sleep_millis: u64, + + // Sleep between batches + pub job_batch_wait_millis: u64, } #[tokio::main] @@ -185,6 +190,8 @@ async fn main() -> AnyhowResult<()> { mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe }; + let common_env = CommonEnv::read_from_env()?; + let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, @@ -201,6 +208,7 @@ async fn main() -> AnyhowResult<()> { debug_download_sleep_millis: easyenv::get_env_num("DEBUG_DOWNLOAD_SLEEP_MILLIS", 0)?, debug_face_detect_sleep_millis: easyenv::get_env_num("DEBUG_FACE_DETECT_SLEEP_MILLIS", 0)?, debug_job_end_sleep_millis: easyenv::get_env_num("DEBUG_JOB_END_SLEEP_MILLIS", 0)?, + job_batch_wait_millis: common_env.job_batch_wait_millis, }; main_loop(downloader).await; @@ -212,7 +220,7 @@ const START_TIMEOUT_MILLIS : u64 = 500; const INCREASE_TIMEOUT_MILLIS : u64 = 1000; async fn main_loop(downloader: Downloader) { - let mut timeout_millis = START_TIMEOUT_MILLIS; + let mut error_timeout_millis = START_TIMEOUT_MILLIS; loop { let num_records = 1; @@ -226,15 +234,15 @@ async fn main_loop(downloader: Downloader) { Ok(jobs) => jobs, Err(e) => { warn!("Error querying jobs: {:?}", e); - std::thread::sleep(Duration::from_millis(timeout_millis)); - timeout_millis += INCREASE_TIMEOUT_MILLIS; + std::thread::sleep(Duration::from_millis(error_timeout_millis)); + error_timeout_millis += INCREASE_TIMEOUT_MILLIS; continue; } }; if jobs.is_empty() { info!("No jobs!"); - std::thread::sleep(Duration::from_millis(1500)); + std::thread::sleep(Duration::from_millis(downloader.job_batch_wait_millis)); continue; } @@ -244,15 +252,15 @@ async fn main_loop(downloader: Downloader) { Ok(_) => {}, Err(e) => { warn!("Error querying jobs: {:?}", e); - std::thread::sleep(Duration::from_millis(timeout_millis)); - timeout_millis += INCREASE_TIMEOUT_MILLIS; + std::thread::sleep(Duration::from_millis(error_timeout_millis)); + error_timeout_millis += INCREASE_TIMEOUT_MILLIS; continue; } } - timeout_millis = START_TIMEOUT_MILLIS; // reset + error_timeout_millis = START_TIMEOUT_MILLIS; // reset - std::thread::sleep(Duration::from_millis(500)); + std::thread::sleep(Duration::from_millis(downloader.job_batch_wait_millis)); } } diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 7b3f6b6c29..89a71ecd7b 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -9,6 +9,7 @@ #[macro_use] extern crate serde_derive; pub mod buckets; +pub mod common_env; pub mod common_queries; pub mod database_helpers; pub mod job_queries; @@ -54,6 +55,7 @@ use std::process::Command; use std::thread; use std::time::Duration; use tempdir::TempDir; +use crate::common_env::CommonEnv; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -97,6 +99,9 @@ struct Inferencer { pub inference_script: String, pub w2l_model_filename: String, pub w2l_end_bump_filename: String, + + // Sleep between batches + pub job_batch_wait_millis: u64, } #[tokio::main] @@ -200,6 +205,8 @@ async fn main() -> AnyhowResult<()> { let firehose_publisher = FirehosePublisher { mysql_pool: mysql_pool.clone(), // NB: MySqlPool is clone/send/sync safe }; + + let common_env = CommonEnv::read_from_env()?; let inferencer = Inferencer { download_temp_directory: temp_directory, @@ -217,6 +224,7 @@ async fn main() -> AnyhowResult<()> { firehose_publisher, w2l_model_filename, w2l_end_bump_filename, + job_batch_wait_millis: common_env.job_batch_wait_millis, }; main_loop(inferencer).await; @@ -228,7 +236,7 @@ const START_TIMEOUT_MILLIS : u64 = 500; const INCREASE_TIMEOUT_MILLIS : u64 = 1000; async fn main_loop(inferencer: Inferencer) { - let mut timeout_millis = START_TIMEOUT_MILLIS; + let mut error_timeout_millis = START_TIMEOUT_MILLIS; loop { let num_records = 1; @@ -242,15 +250,15 @@ async fn main_loop(inferencer: Inferencer) { Ok(jobs) => jobs, Err(e) => { warn!("Error querying jobs: {:?}", e); - std::thread::sleep(Duration::from_millis(timeout_millis)); - timeout_millis += INCREASE_TIMEOUT_MILLIS; + std::thread::sleep(Duration::from_millis(error_timeout_millis)); + error_timeout_millis += INCREASE_TIMEOUT_MILLIS; continue; } }; if jobs.is_empty() { info!("No jobs!"); - std::thread::sleep(Duration::from_millis(1500)); + std::thread::sleep(Duration::from_millis(inferencer.job_batch_wait_millis)); continue; } @@ -260,15 +268,15 @@ async fn main_loop(inferencer: Inferencer) { Ok(_) => {}, Err(e) => { warn!("Error querying jobs: {:?}", e); - std::thread::sleep(Duration::from_millis(timeout_millis)); - timeout_millis += INCREASE_TIMEOUT_MILLIS; + std::thread::sleep(Duration::from_millis(error_timeout_millis)); + error_timeout_millis += INCREASE_TIMEOUT_MILLIS; continue; } } - timeout_millis = START_TIMEOUT_MILLIS; // reset + error_timeout_millis = START_TIMEOUT_MILLIS; // reset - std::thread::sleep(Duration::from_millis(500)); + std::thread::sleep(Duration::from_millis(inferencer.job_batch_wait_millis)); } } From b01b83cccf620b83798d746e377fe5b64e098b73 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 03:05:31 -0400 Subject: [PATCH 0334/1042] job max attempts --- src/common_env.rs | 8 +++++++- src/job_queries/tts_inference_job_queries.rs | 13 ++++++++++--- src/tts_inference_job.rs | 8 +++++++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/common_env.rs b/src/common_env.rs index e04938a702..36499cf53a 100644 --- a/src/common_env.rs +++ b/src/common_env.rs @@ -3,13 +3,18 @@ use crate::util::anyhow_result::AnyhowResult; // TODO: Move more shared configs here. /// Environment variables shared between one or more jobs or the main server. -/// These are +/// These do not have to have the same value for each instance, but they behave +/// similarly across all usages. pub struct CommonEnv { /// The amount of time to wait between job batches (not individual jobs). /// This prevents the outer loop of querying batches from flooding the DB. /// (In theory, there's work within jobs that prevents rapidly pegging the DB.) pub job_batch_wait_millis: u64, + + /// The maximum number of retries a job will get. + /// After a job exhausts these attempts, it will become "dead". + pub job_max_attempts: u8, } impl CommonEnv { @@ -17,6 +22,7 @@ impl CommonEnv { pub fn read_from_env() -> AnyhowResult { Ok(Self { job_batch_wait_millis: easyenv::get_env_num("JOB_BATCH_WAIT_MILLIS", 100)?, + job_max_attempts: easyenv::get_env_num("JOB_MAX_ATTEMPTS", 3)?, }) } } \ No newline at end of file diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index ef3ed7b7ea..38f0106ecc 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -142,6 +142,7 @@ FOR UPDATE UPDATE tts_inference_jobs SET status = 'started', + attempt_count = attempt_count + 1, retry_at = NOW() + interval 2 minute WHERE id = ? "#, @@ -158,11 +159,17 @@ WHERE id = ? pub async fn mark_tts_inference_job_failure( pool: &MySqlPool, job: &TtsInferenceJobRecord, - failure_reason: &str + failure_reason: &str, + max_attempts: i32 ) -> AnyhowResult<()> { // statuses: "attempt_failed", "complete_failure", "dead" - let status = "attempt_failed"; + let mut next_status = "attempt_failed"; + + if job.attempt_count >= max_attempts { + // NB: Job attempt count is incremented at start + next_status = "dead"; + } let query_result = sqlx::query!( r#" @@ -173,7 +180,7 @@ SET retry_at = NOW() + interval 1 minute WHERE id = ? "#, - status, + next_status, failure_reason.to_string(), job.id, ) diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index a76e6760b3..3d4778999c 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -91,6 +91,10 @@ struct Inferencer { // Sleep between batches pub job_batch_wait_millis: u64, + + // Max job attempts before failure. + // NB: This is an i32 so we don't need to convert to db column type. + pub job_max_attempts: i32, } #[tokio::main] @@ -198,6 +202,7 @@ async fn main() -> AnyhowResult<()> { firehose_publisher, tts_vocoder_model_filename, job_batch_wait_millis: common_env.job_batch_wait_millis, + job_max_attempts: common_env.job_max_attempts as i32, }; main_loop(inferencer).await; @@ -264,7 +269,8 @@ async fn process_jobs(inferencer: &Inferencer, jobs: Vec) let _r = mark_tts_inference_job_failure( &inferencer.mysql_pool, &job, - failure_reason) + failure_reason, + inferencer.job_max_attempts) .await; } } From 8393095be267e1d7fb1687feb42d011a84cd8713 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 03:16:04 -0400 Subject: [PATCH 0335/1042] job dead letter queueing (attempt count + config) --- src/job_queries/tts_download_job_queries.rs | 17 ++++++++++----- src/job_queries/w2l_download_job_queries.rs | 22 +++++++++++++------- src/job_queries/w2l_inference_job_queries.rs | 22 +++++++++++++------- src/tts_download_job.rs | 12 ++++++++++- src/tts_inference_job.rs | 4 ++-- src/w2l_download_job.rs | 10 +++++++-- src/w2l_inference_job.rs | 10 +++++++-- 7 files changed, 71 insertions(+), 26 deletions(-) diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index 86d0e8f771..aebab0804f 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -124,6 +124,7 @@ FOR UPDATE UPDATE tts_model_upload_jobs SET status = 'started', + attempt_count = attempt_count + 1, retry_at = NOW() + interval 2 minute WHERE id = ? "#, @@ -140,11 +141,17 @@ WHERE id = ? pub async fn mark_tts_upload_job_failure( pool: &MySqlPool, job: &TtsUploadJobRecord, - failure_reason: &str -) -> AnyhowResult<()> -{ + failure_reason: &str, + max_attempts: i32 +) -> AnyhowResult<()> { + // statuses: "attempt_failed", "complete_failure", "dead" - let status = "attempt_failed"; + let mut next_status = "attempt_failed"; + + if job.attempt_count >= max_attempts { + // NB: Job attempt count is incremented at start + next_status = "dead"; + } let query_result = sqlx::query!( r#" @@ -155,7 +162,7 @@ SET retry_at = NOW() + interval 2 minute WHERE id = ? "#, - status, + next_status, failure_reason.to_string(), job.id, ) diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 8df5dc5bba..93c5577d96 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -123,6 +123,7 @@ FOR UPDATE UPDATE w2l_template_upload_jobs SET status = 'started', + attempt_count = attempt_count + 1, retry_at = NOW() + interval 2 minute WHERE id = ? "#, @@ -136,13 +137,20 @@ WHERE id = ? Ok(true) } -pub async fn mark_w2l_template_upload_job_failure(pool: &MySqlPool, - job: &W2lTemplateUploadJobRecord, - failure_reason: &str) - -> AnyhowResult<()> -{ +pub async fn mark_w2l_template_upload_job_failure( + pool: &MySqlPool, + job: &W2lTemplateUploadJobRecord, + failure_reason: &str, + max_attempts: i32 +) -> AnyhowResult<()> { + // statuses: "attempt_failed", "complete_failure", "dead" - let status = "attempt_failed"; + let mut next_status = "attempt_failed"; + + if job.attempt_count >= max_attempts { + // NB: Job attempt count is incremented at start + next_status = "dead"; + } let query_result = sqlx::query!( r#" @@ -153,7 +161,7 @@ SET retry_at = NOW() + interval 2 minute WHERE id = ? "#, - status, + next_status, failure_reason.to_string(), job.id, ) diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index 66bad12dc2..d17c198c2a 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -165,6 +165,7 @@ FOR UPDATE UPDATE w2l_inference_jobs SET status = 'started', + attempt_count = attempt_count + 1, retry_at = NOW() + interval 2 minute WHERE id = ? "#, @@ -179,13 +180,20 @@ WHERE id = ? } -pub async fn mark_w2l_inference_job_failure(pool: &MySqlPool, - job: &W2lInferenceJobRecord, - failure_reason: &str) - -> AnyhowResult<()> -{ +pub async fn mark_w2l_inference_job_failure( + pool: &MySqlPool, + job: &W2lInferenceJobRecord, + failure_reason: &str, + max_attempts: i32 +) -> AnyhowResult<()> { + // statuses: "attempt_failed", "complete_failure", "dead" - let status = "attempt_failed"; + let mut next_status = "attempt_failed"; + + if job.attempt_count >= max_attempts { + // NB: Job attempt count is incremented at start + next_status = "dead"; + } let query_result = sqlx::query!( r#" @@ -196,7 +204,7 @@ SET retry_at = NOW() + interval 2 minute WHERE id = ? "#, - status, + next_status, failure_reason.to_string(), job.id, ) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index b220c29c3b..cc5eaf6d2a 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -81,6 +81,10 @@ struct Downloader { // Sleep between batches pub job_batch_wait_millis: u64, + + // Max job attempts before failure. + // NB: This is an i32 so we don't need to convert to db column type. + pub job_max_attempts: i32, } #[tokio::main] @@ -164,6 +168,7 @@ async fn main() -> AnyhowResult<()> { firehose_publisher, tts_check: tts_model_check_command, job_batch_wait_millis: common_env.job_batch_wait_millis, + job_max_attempts: common_env.job_max_attempts as i32, }; main_loop(downloader).await; @@ -223,7 +228,12 @@ async fn process_jobs(downloader: &Downloader, jobs: Vec) -> Err(e) => { warn!("Failure to process job: {:?}", e); let failure_reason = ""; - let _r = mark_tts_upload_job_failure(&downloader.mysql_pool, &job, failure_reason).await; + let _r = mark_tts_upload_job_failure( + &downloader.mysql_pool, + &job, + failure_reason, + downloader.job_max_attempts + ).await; } } } diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 3d4778999c..5a955a831d 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -270,8 +270,8 @@ async fn process_jobs(inferencer: &Inferencer, jobs: Vec) &inferencer.mysql_pool, &job, failure_reason, - inferencer.job_max_attempts) - .await; + inferencer.job_max_attempts + ).await; } } } diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 1307714918..eb8f82c001 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -103,6 +103,10 @@ struct Downloader { // Sleep between batches pub job_batch_wait_millis: u64, + + // Max job attempts before failure. + // NB: This is an i32 so we don't need to convert to db column type. + pub job_max_attempts: i32, } #[tokio::main] @@ -209,6 +213,7 @@ async fn main() -> AnyhowResult<()> { debug_face_detect_sleep_millis: easyenv::get_env_num("DEBUG_FACE_DETECT_SLEEP_MILLIS", 0)?, debug_job_end_sleep_millis: easyenv::get_env_num("DEBUG_JOB_END_SLEEP_MILLIS", 0)?, job_batch_wait_millis: common_env.job_batch_wait_millis, + job_max_attempts: common_env.job_max_attempts as i32, }; main_loop(downloader).await; @@ -275,8 +280,9 @@ async fn process_jobs(downloader: &Downloader, jobs: Vec AnyhowResult<()> { w2l_model_filename, w2l_end_bump_filename, job_batch_wait_millis: common_env.job_batch_wait_millis, + job_max_attempts: common_env.job_max_attempts as i32, }; main_loop(inferencer).await; @@ -291,8 +296,9 @@ async fn process_jobs(inferencer: &Inferencer, jobs: Vec) let _r = mark_w2l_inference_job_failure( &inferencer.mysql_pool, &job, - failure_reason) - .await; + failure_reason, + inferencer.job_max_attempts + ).await; } } } From c9a82d46eba3a1bf49b06b339e40d8be6ac83859 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 12:02:29 -0400 Subject: [PATCH 0336/1042] prevent slurs in username and w2l titles --- src/http_server/endpoints/users/create_account.rs | 5 +++++ src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index 03677e7f79..71cc55c857 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -22,6 +22,7 @@ use std::sync::Arc; use crate::util::email_to_gravatar::email_to_gravatar; use crate::validations::username_reservations::is_reserved_username; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; const NEW_USER_ROLE: &'static str = "user"; @@ -106,6 +107,10 @@ pub async fn create_account_handler( return Err(CreateAccountError::ReservedUsername); } + if contains_slurs(&request.username) { + return Err(CreateAccountError::BadInput("username contains slurs".to_string())); + } + if !request.email_address.contains("@") { return Err(CreateAccountError::BadInput("invalid email address".to_string())); } diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index 7d81bbaf3f..c3cf0cd4ea 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -20,6 +20,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; #[derive(Deserialize)] pub struct UploadW2lTemplateRequest { @@ -115,6 +116,10 @@ pub async fn upload_w2l_template_handler( return Err(UploadW2lTemplateError::BadInput(reason)); } + if contains_slurs(&request.title) { + return Err(UploadW2lTemplateError::BadInput("title contains slurs".to_string())); + } + let ip_address = get_request_ip(&http_request); let uuid = request.idempotency_token.to_string(); From 4485cf34d47dd48fd926c87bc4be7b3e51197f72 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 8 Jun 2021 12:12:42 -0400 Subject: [PATCH 0337/1042] add attempt_count to all job status endpoints --- .../endpoints/tts/get_tts_inference_job_status.rs | 6 ++++++ .../endpoints/tts/get_tts_upload_model_job_status.rs | 4 ++++ src/http_server/endpoints/users/edit_profile.rs | 1 - .../endpoints/w2l/get_w2l_inference_job_status.rs | 4 ++++ .../endpoints/w2l/get_w2l_upload_template_job_status.rs | 4 ++++ 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index 8e03547eed..1797a57d6e 100644 --- a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -37,6 +37,8 @@ pub struct TtsInferenceJobStatusForResponse { pub job_token: String, pub status: String, + pub attempt_count: u8, + pub maybe_result_token: Option, pub maybe_public_bucket_wav_audio_path: Option, @@ -70,6 +72,8 @@ pub struct TtsInferenceJobStatusRecord { pub job_token: String, pub status: String, + pub attempt_count: i32, + pub maybe_result_token: Option, pub maybe_public_bucket_wav_audio_path: Option, @@ -124,6 +128,7 @@ SELECT jobs.token as job_token, jobs.status, + jobs.attempt_count, jobs.on_success_result_token as maybe_result_token, results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path, @@ -165,6 +170,7 @@ WHERE jobs.token = ? let model_for_response = TtsInferenceJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), + attempt_count: record.attempt_count as u8, maybe_result_token: record.maybe_result_token.clone(), maybe_public_bucket_wav_audio_path: record.maybe_public_bucket_wav_audio_path.clone(), model_token: record.model_token.clone(), diff --git a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs index 24458b5098..3d62c931fb 100644 --- a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs @@ -37,6 +37,7 @@ pub struct TtsUploadModelJobStatusForResponse { pub job_token: String, pub status: String, + pub attempt_count: u8, pub maybe_model_token: Option, pub created_at: DateTime, @@ -65,6 +66,7 @@ pub struct TtsUploadModelJobStatusRecord { pub job_token: String, pub status: String, + pub attempt_count: i32, pub maybe_model_token: Option, pub created_at: DateTime, @@ -113,6 +115,7 @@ SELECT jobs.token as job_token, jobs.status, + jobs.attempt_count, jobs.on_success_result_token as maybe_model_token, jobs.created_at, @@ -145,6 +148,7 @@ WHERE jobs.token = ? let model_for_response = TtsUploadModelJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), + attempt_count: record.attempt_count as u8, maybe_model_token: record.maybe_model_token.clone(), created_at: record.created_at.clone(), updated_at: record.updated_at.clone(), diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs index 5a1d3b24c0..220727c0cd 100644 --- a/src/http_server/endpoints/users/edit_profile.rs +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -241,7 +241,6 @@ pub async fn edit_profile_handler( } } - if let Some(markdown) = request.profile_markdown.as_deref() { if contains_slurs(markdown) { return Err(EditProfileError::BadInput("profile contains slurs".to_string())); diff --git a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs index 621e6240a4..01a1a41f65 100644 --- a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs +++ b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs @@ -37,6 +37,7 @@ pub struct W2lInferenceJobStatusForResponse { pub job_token: String, pub status: String, + pub attempt_count: u8, pub maybe_result_token: Option, pub maybe_public_bucket_video_path: Option, @@ -70,6 +71,7 @@ pub struct W2lInferenceJobStatusRecord { pub job_token: String, pub status: String, + pub attempt_count: i32, pub maybe_result_token: Option, pub maybe_public_bucket_video_path: Option, @@ -124,6 +126,7 @@ SELECT jobs.token as job_token, jobs.status, + jobs.attempt_count, jobs.on_success_result_token as maybe_result_token, results.public_bucket_video_path as maybe_public_bucket_video_path, @@ -165,6 +168,7 @@ WHERE jobs.token = ? let template_for_response = W2lInferenceJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), + attempt_count: record.attempt_count as u8, maybe_result_token: record.maybe_result_token.clone(), maybe_public_bucket_video_path: record.maybe_public_bucket_video_path.clone(), maybe_w2l_template_token: record.maybe_w2l_template_token.clone(), diff --git a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs index 6a368428e3..2a498b9912 100644 --- a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs +++ b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs @@ -37,6 +37,7 @@ pub struct W2lUploadTemplateJobStatusForResponse { pub job_token: String, pub status: String, + pub attempt_count: u8, pub maybe_template_token: Option, pub created_at: DateTime, @@ -65,6 +66,7 @@ pub struct W2lUploadTemplateJobStatusRecord { pub job_token: String, pub status: String, + pub attempt_count: i32, pub maybe_template_token: Option, pub created_at: DateTime, @@ -113,6 +115,7 @@ SELECT jobs.token as job_token, jobs.status, + jobs.attempt_count, jobs.on_success_result_token as maybe_template_token, jobs.created_at, @@ -145,6 +148,7 @@ WHERE jobs.token = ? let template_for_response = W2lUploadTemplateJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), + attempt_count: record.attempt_count as u8, maybe_template_token: record.maybe_template_token.clone(), created_at: record.created_at.clone(), updated_at: record.updated_at.clone(), From 342f9cd10e8a6fd7313585cbee10b82b40400a59 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Jun 2021 04:15:11 -0400 Subject: [PATCH 0338/1042] new tables --- .../down.sql | 0 .../up.sql | 0 migrations/2021-05-30-010200_firehose/up.sql | 19 ---------- migrations/2021-06-09-010200_ip_bans/down.sql | 5 +++ migrations/2021-06-09-010200_ip_bans/up.sql | 37 ++++++++++++++++++ .../2021-06-09-010300_mod_audit_log/down.sql | 5 +++ .../2021-06-09-010300_mod_audit_log/up.sql | 38 +++++++++++++++++++ 7 files changed, 85 insertions(+), 19 deletions(-) rename migrations/{2021-05-15-212923_misc => 2021-05-15-212923_badges}/down.sql (100%) rename migrations/{2021-05-15-212923_misc => 2021-05-15-212923_badges}/up.sql (100%) create mode 100644 migrations/2021-06-09-010200_ip_bans/down.sql create mode 100644 migrations/2021-06-09-010200_ip_bans/up.sql create mode 100644 migrations/2021-06-09-010300_mod_audit_log/down.sql create mode 100644 migrations/2021-06-09-010300_mod_audit_log/up.sql diff --git a/migrations/2021-05-15-212923_misc/down.sql b/migrations/2021-05-15-212923_badges/down.sql similarity index 100% rename from migrations/2021-05-15-212923_misc/down.sql rename to migrations/2021-05-15-212923_badges/down.sql diff --git a/migrations/2021-05-15-212923_misc/up.sql b/migrations/2021-05-15-212923_badges/up.sql similarity index 100% rename from migrations/2021-05-15-212923_misc/up.sql rename to migrations/2021-05-15-212923_badges/up.sql diff --git a/migrations/2021-05-30-010200_firehose/up.sql b/migrations/2021-05-30-010200_firehose/up.sql index bfc5e3c6cc..40302a1e21 100644 --- a/migrations/2021-05-30-010200_firehose/up.sql +++ b/migrations/2021-05-30-010200_firehose/up.sql @@ -10,25 +10,6 @@ CREATE TABLE firehose_entries ( token VARCHAR(32) NOT NULL, -- The type of the event - -- DO NOT REORDER. - -- event_type ENUM( - -- 'not-set', - -- 'user_sign_up', - -- 'tts_model_upload_started', - -- 'tts_model_upload_completed', - -- 'tts_inference_started', - -- 'tts_inference_completed', - -- 'w2l_template_upload_started', - -- 'w2l_template_upload_completed', - -- 'w2l_inference_started', - -- 'w2l_inference_completed', - -- 'twitter_mention', - -- 'twitter_retweet', - -- 'discord_join', - -- 'discord_message', - -- 'twitch_subscribe', - -- 'twitch_follow' - -- ) NOT NULL DEFAULT 'not-set', event_type VARCHAR(32) NOT NULL, -- The target user diff --git a/migrations/2021-06-09-010200_ip_bans/down.sql b/migrations/2021-06-09-010200_ip_bans/down.sql new file mode 100644 index 0000000000..05e576cc5d --- /dev/null +++ b/migrations/2021-06-09-010200_ip_bans/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +drop table ip_address_bans; diff --git a/migrations/2021-06-09-010200_ip_bans/up.sql b/migrations/2021-06-09-010200_ip_bans/up.sql new file mode 100644 index 0000000000..8b285e9914 --- /dev/null +++ b/migrations/2021-06-09-010200_ip_bans/up.sql @@ -0,0 +1,37 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE TABLE ip_address_bans ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- The IP address to ban + ip_address VARCHAR(40) DEFAULT NULL, + + -- Mod who created or last edited the ban. + mod_user_token VARCHAR(32) NOT NULL, + + -- Maybe the target user. + maybe_target_user_token VARCHAR(32) DEFAULT NULL, + + -- Optional notes about the ban. + notes TEXT DEFAULT NULL, + + -- Moderator can set an expire time (optional). + -- If not set, the ban is indefinite + expires_at TIMESTAMP DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If deleted, the ban is removed. + deleted_at TIMESTAMP DEFAULT NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (ip_address), + KEY fk_mod_user_token (mod_user_token), + KEY fk_maybe_target_user_token (maybe_target_user_token) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-06-09-010300_mod_audit_log/down.sql b/migrations/2021-06-09-010300_mod_audit_log/down.sql new file mode 100644 index 0000000000..72523365cb --- /dev/null +++ b/migrations/2021-06-09-010300_mod_audit_log/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +drop table moderator_audit_log_entries; \ No newline at end of file diff --git a/migrations/2021-06-09-010300_mod_audit_log/up.sql b/migrations/2021-06-09-010300_mod_audit_log/up.sql new file mode 100644 index 0000000000..ca30b2650a --- /dev/null +++ b/migrations/2021-06-09-010300_mod_audit_log/up.sql @@ -0,0 +1,38 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +CREATE TABLE moderator_audit_logs_entries ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Visible "primary key" + token VARCHAR(32) NOT NULL, + + -- The type of the event (ban, delete, etc.) + event_type VARCHAR(32) NOT NULL, + + -- The target user + maybe_target_user_token VARCHAR(32) DEFAULT NULL, + + -- The target "entity", which varies by event type. + -- eg. a model token or template token + maybe_target_entity_token VARCHAR(32) DEFAULT NULL, + + -- If an IP was banned, we record it here. + maybe_target_ip_address VARCHAR(40) DEFAULT NULL, + + -- Optional mod-specified notes. + optional_notes TEXT DEFAULT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + KEY fk_maybe_target_user_token (maybe_target_user_token), + KEY fk_maybe_target_entity_token (maybe_target_entity_token), + KEY index_event_type (event_type) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 2d776517cccfb36f965c1acfc15f20a03a1610f2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Jun 2021 04:33:32 -0400 Subject: [PATCH 0339/1042] delete schemas, two types of deleted record (WIP - code broken) --- migrations/2021-05-15-071534_users/down.sql | 6 +- migrations/2021-05-15-071534_users/up.sql | 38 +---- migrations/2021-05-15-071601_assets/up.sql | 6 +- migrations/2021-05-15-071602_voices/down.sql | 6 - migrations/2021-05-15-071602_voices/up.sql | 140 ------------------ .../up.sql | 14 +- 6 files changed, 20 insertions(+), 190 deletions(-) delete mode 100644 migrations/2021-05-15-071602_voices/down.sql delete mode 100644 migrations/2021-05-15-071602_voices/up.sql diff --git a/migrations/2021-05-15-071534_users/down.sql b/migrations/2021-05-15-071534_users/down.sql index 7d31cdce5e..262c9cf4d3 100644 --- a/migrations/2021-05-15-071534_users/down.sql +++ b/migrations/2021-05-15-071534_users/down.sql @@ -1,7 +1,7 @@ -# noinspection SqlNoDataSourceInspectionForFile -# noinspection SqlResolveForFile +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile -drop table email_verifications; drop table user_roles; drop table user_sessions; drop table users; diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index a36f09223b..fbcd7426e2 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -98,7 +98,8 @@ CREATE TABLE users ( -- If the user is deleted, we set this. -- This is different than banned. These users won't show up at all. - deleted_at TIMESTAMP NULL, + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, -- INDICES -- PRIMARY KEY (id), @@ -171,6 +172,9 @@ CREATE TABLE user_sessions ( expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, -- deletion = session termination + -- Typically these are destroyed by users, but if in the future we allow mods to + -- delete them, it doesn't really matter who did the deletion: sessions are not + -- designed to be recoverable. deleted_at TIMESTAMP NULL, -- INDICES -- @@ -180,35 +184,3 @@ CREATE TABLE user_sessions ( KEY index_ip_address_creation (ip_address_creation) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; - --- We only allow the most recent record for any given user to be redeemed. -CREATE TABLE email_verifications ( - -- Not used for anything except replication. - id BIGINT(20) NOT NULL AUTO_INCREMENT, - - verification_type ENUM( - 'email-verification', - 'password-reset' - ) NOT NULL, - - -- The redemption secret - verification_code VARCHAR(32) NOT NULL, - - -- Whether the attempt worked - successful BOOLEAN NOT NULL DEFAULT false, - - -- Foreign key to user - user_token VARCHAR(32) NOT NULL, - - -- Cannot be redeemed after this date - expires_at TIMESTAMP NOT NULL, - - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - - -- INDICES -- - PRIMARY KEY (id), - KEY fk_user_token (user_token), - KEY index_verification_type (verification_type) - -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 85a645256c..83ca2bae8d 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -118,7 +118,8 @@ CREATE TABLE tts_models ( -- If this is removed by a mod. -- It completely disappears from the system. - deleted_at TIMESTAMP NULL, + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, -- INDICES -- PRIMARY KEY (id), @@ -255,7 +256,8 @@ CREATE TABLE w2l_templates ( -- If this is removed by a mod. -- It shows up nowhere if deleted. - deleted_at TIMESTAMP NULL, + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, -- INDICES -- PRIMARY KEY (id), diff --git a/migrations/2021-05-15-071602_voices/down.sql b/migrations/2021-05-15-071602_voices/down.sql deleted file mode 100644 index 96d002ce10..0000000000 --- a/migrations/2021-05-15-071602_voices/down.sql +++ /dev/null @@ -1,6 +0,0 @@ -# noinspection SqlNoDataSourceInspectionForFile -# noinspection SqlResolveForFile - -drop table subjects; ---drop table voices; - diff --git a/migrations/2021-05-15-071602_voices/up.sql b/migrations/2021-05-15-071602_voices/up.sql deleted file mode 100644 index 42b32bb82c..0000000000 --- a/migrations/2021-05-15-071602_voices/up.sql +++ /dev/null @@ -1,140 +0,0 @@ -# noinspection SqlNoDataSourceInspectionForFile -# noinspection SqlResolveForFile - --- These can be people, cartoon characters, fictional characters, etc. --- Subjects may be "played" by different entities. -CREATE TABLE subjects ( - -- Not used for anything except replication. - id BIGINT(20) NOT NULL AUTO_INCREMENT, - - -- Effective "primary key" (PUBLIC) - token VARCHAR(32) NOT NULL, - - -- The name of the subject - -- Might have parens to disambiguate, eg. "JFK (Clone High)" - subject_name VARCHAR(255) NOT NULL, - - -- The URL we access the voice at. - -- These should be stable, but could be changed if necessary. - -- As such, these should not be foreign keys. - updatable_slug VARCHAR(64) NOT NULL, - - -- The type of subject - -- For eg. "JFK (Clone High)", that would be a separate subject. - type ENUM( - 'not-set', - 'real-person', - 'fictional-character' - ) NOT NULL DEFAULT 'not-set', - - -- The description of the model in markdown. - description_markdown TEXT NOT NULL, - - -- Generated HTML (not user-editable). - description_rendered_html TEXT NOT NULL, - - -- The person that created the template. - creator_user_token VARCHAR(32) NOT NULL, - - -- For abuse tracking. - -- Wide enough for IPv4/6 - creator_ip_address VARCHAR(40) NOT NULL, - - -- The person that created the template. - -- More than just mods may update. - maybe_updater_user_token VARCHAR(32) NOT NULL, - - -- More than just mods may update. - -- For abuse tracking. - -- Wide enough for IPv4/6 - maybe_updater_ip_address VARCHAR(40) NOT NULL, - - -- We can assign an exemplary model to the subject - maybe_default_tts_model_token VARCHAR(32) DEFAULT NULL, - maybe_default_w2l_template_token VARCHAR(32) DEFAULT NULL, - - -- The 800x600 image - maybe_image_banner_public_bucket_hash CHAR(32) DEFAULT NULL, - - -- The square avatar image - maybe_image_square_public_bucket_hash CHAR(32) DEFAULT NULL, - - -- If a moderator author disables it. - -- This should prevent the voice from showing up in lists. - -- Eg. if someone creates "Hitler", we'd want to prevent it from showing up. - is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, - - -- If a moderator has comments. - maybe_mod_comments VARCHAR(255) DEFAULT NULL, - -- The last moderator that made changes. - maybe_mod_user_token VARCHAR(32) DEFAULT NULL, - - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - - -- If this is removed by a mod. - -- It completely disappears from the system. - deleted_at TIMESTAMP NULL, - - -- INDICES -- - PRIMARY KEY (id), - UNIQUE KEY (token), - UNIQUE KEY (updatable_slug), - KEY fk_creator_user_token (creator_user_token), - KEY fk_maybe_updater_user_token (maybe_updater_user_token), - KEY fk_maybe_default_tts_model_token (maybe_default_tts_model_token), - KEY fk_maybe_default_w2l_template_token (maybe_default_w2l_template_token), - KEY fk_maybe_mod_user_token (maybe_mod_user_token), - KEY index_type (type), - KEY index_creator_ip_address (creator_ip_address), - KEY index_maybe_updater_ip_address (maybe_updater_ip_address), - KEY index_is_mod_disabled (is_mod_disabled) - -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; - --- CREATE TABLE voices ( --- -- Not used for anything except replication. --- id BIGINT(20) NOT NULL AUTO_INCREMENT, --- --- -- Effective "primary key" (PUBLIC) --- token VARCHAR(32) NOT NULL, --- --- -- The URL we access the voice at. --- -- These should be stable, but could be changed if necessary. --- -- As such, these should not be foreign keys. --- updatable_slug VARCHAR(64) NOT NULL, --- --- -- We can assign an exemplary model to the voice --- default_model_token VARCHAR(32) DEFAULT NULL, --- --- -- If a moderator author disables it. --- -- This should prevent the voice from showing up in lists. --- mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, --- --- -- The name of the voice --- voice_name VARCHAR(255) NOT NULL, --- --- -- If the voice is "happy" or a singer "a-capella", etc. --- voice_characteristic VARCHAR(255) DEFAULT NULL, --- --- -- The speaker (in the case of cartoon characters) --- voice_actor_name VARCHAR(255) DEFAULT NULL, --- --- -- The 800x600 image --- image_banner_public_bucket_hash CHAR(32) DEFAULT NULL, --- --- -- The square avatar image --- image_square_public_bucket_hash CHAR(32) DEFAULT NULL, --- --- created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, --- updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, --- --- -- If this is removed by a mod. --- deleted_at TIMESTAMP NULL, --- --- -- INDICES -- --- PRIMARY KEY (id), --- UNIQUE KEY (token), --- UNIQUE KEY (updatable_slug) --- --- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 5e4ff29d3b..2cd4cdd764 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -70,7 +70,8 @@ CREATE TABLE tts_results ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- If this is removed by a mod or the creator. - deleted_at TIMESTAMP NULL, + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, -- INDICES -- PRIMARY KEY (id), @@ -80,8 +81,8 @@ CREATE TABLE tts_results ( KEY fk_maybe_mod_user_token (maybe_mod_user_token), KEY index_creator_ip_address (creator_ip_address), KEY index_is_mod_hidden_from_public(is_mod_hidden_from_public), - KEY index_deleted_at(deleted_at), - KEY index_deleted_at_and_is_mod_hidden_from_public(deleted_at, is_mod_hidden_from_public) + KEY index_user_deleted_at(user_deleted_at), + KEY index_mod_deleted_at(mod_deleted_at) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -145,7 +146,8 @@ CREATE TABLE w2l_results ( updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- If this is removed by a mod or the creator. - deleted_at TIMESTAMP NULL, + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, -- INDICES -- PRIMARY KEY (id), @@ -156,7 +158,7 @@ CREATE TABLE w2l_results ( KEY fk_maybe_mod_user_token (maybe_mod_user_token), KEY index_creator_ip_address (creator_ip_address), KEY index_is_mod_hidden_from_public(is_mod_hidden_from_public), - KEY index_deleted_at(deleted_at), - KEY index_deleted_at_and_is_mod_hidden_from_public(deleted_at, is_mod_hidden_from_public) + KEY index_user_deleted_at(user_deleted_at), + KEY index_mod_deleted_at(mod_deleted_at) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 318221549be40199d8d874b22c08604970ff8407 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Jun 2021 04:36:45 -0400 Subject: [PATCH 0340/1042] columns to prevent user modification --- migrations/2021-05-15-071601_assets/up.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 83ca2bae8d..90ff5bae3b 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -106,6 +106,9 @@ CREATE TABLE tts_models ( -- In this case, a moderator disables it. is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, + -- Extremely popular models may be locked from deletion or modification by users. + is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE, + -- If a moderator has comments. maybe_mod_comments VARCHAR(255) DEFAULT NULL, -- The last moderator that made changes. @@ -243,6 +246,9 @@ CREATE TABLE w2l_templates ( -- This also disables it for the creator. is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, + -- Extremely popular models may be locked from deletion or modification by users. + is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE, + -- If a moderator has comments. maybe_mod_comments VARCHAR(255) DEFAULT NULL, From 2fcae0a39c394a66f73c69ee320d2fe0d5351bed Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Jun 2021 22:05:42 -0400 Subject: [PATCH 0341/1042] fixup all queries for user_deleted_at / mod_deleted_at (untested) --- .../list_tts_inference_results.rs | 34 +++--- src/common_queries/list_tts_models.rs | 22 ++-- .../list_w2l_inference_results.rs | 34 +++--- src/common_queries/list_w2l_templates.rs | 22 ++-- src/common_queries/query_user_profile.rs | 6 +- src/common_queries/query_w2l_template.rs | 25 +++-- .../endpoints/tts/get_tts_model.rs | 6 +- .../endpoints/tts/get_tts_result.rs | 5 +- .../endpoints/tts/list_tts_models.rs | 6 +- .../endpoints/users/get_profile.rs | 6 +- src/http_server/endpoints/users/logout.rs | 5 +- .../endpoints/w2l/delete_w2l_template.rs | 104 ++++++++++++++---- .../w2l/enqueue_infer_w2l_with_uploads.rs | 6 +- .../endpoints/w2l/get_w2l_result.rs | 5 +- src/http_server/web_utils/session_checker.rs | 3 +- src/job_queries/tts_inference_job_queries.rs | 3 +- src/job_queries/w2l_inference_job_queries.rs | 3 +- 17 files changed, 200 insertions(+), 95 deletions(-) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index 786151b1e5..73391cc518 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -136,13 +136,14 @@ SELECT FROM tts_results LEFT OUTER JOIN tts_models - ON tts_results.model_token = tts_models.token + ON tts_results.model_token = tts_models.token LEFT OUTER JOIN users - ON tts_results.maybe_creator_user_token = users.token + ON tts_results.maybe_creator_user_token = users.token WHERE - tts_results.deleted_at IS NULL - AND tts_results.is_mod_hidden_from_public IS FALSE + tts_results.is_mod_hidden_from_public IS FALSE AND tts_models.is_mod_disabled IS FALSE + AND tts_results.user_deleted_at IS NULL + AND tts_results.mod_deleted_at IS NULL "#) .fetch_all(mysql_pool) .await? @@ -168,11 +169,12 @@ SELECT FROM tts_results LEFT OUTER JOIN tts_models - ON tts_results.model_token = tts_models.token + ON tts_results.model_token = tts_models.token LEFT OUTER JOIN users - ON tts_results.maybe_creator_user_token = users.token + ON tts_results.maybe_creator_user_token = users.token WHERE - tts_results.deleted_at IS NULL + tts_results.user_deleted_at IS NULL + AND tts_results.mod_deleted_at IS NULL "#) .fetch_all(mysql_pool) .await? @@ -210,14 +212,15 @@ SELECT FROM tts_results LEFT OUTER JOIN tts_models - ON tts_results.model_token = tts_models.token + ON tts_results.model_token = tts_models.token LEFT OUTER JOIN users - ON tts_results.maybe_creator_user_token = users.token + ON tts_results.maybe_creator_user_token = users.token WHERE - tts_results.deleted_at IS NULL + users.username = ? AND tts_results.is_mod_hidden_from_public IS FALSE AND tts_models.is_mod_disabled IS FALSE - AND users.username = ? + AND tts_results.user_deleted_at IS NULL + AND tts_results.mod_deleted_at IS NULL "#, scope_creator_username) .fetch_all(mysql_pool) @@ -244,12 +247,13 @@ SELECT FROM tts_results LEFT OUTER JOIN tts_models - ON tts_results.model_token = tts_models.token + ON tts_results.model_token = tts_models.token LEFT OUTER JOIN users - ON tts_results.maybe_creator_user_token = users.token + ON tts_results.maybe_creator_user_token = users.token WHERE - tts_results.deleted_at IS NULL - AND users.username = ? + users.username = ? + AND tts_results.user_deleted_at IS NULL + AND tts_results.mod_deleted_at IS NULL "#, scope_creator_username) .fetch_all(mysql_pool) diff --git a/src/common_queries/list_tts_models.rs b/src/common_queries/list_tts_models.rs index 31d073866b..29f38189ae 100644 --- a/src/common_queries/list_tts_models.rs +++ b/src/common_queries/list_tts_models.rs @@ -118,10 +118,11 @@ SELECT tts.updated_at FROM tts_models as tts JOIN users -ON users.token = tts.creator_user_token + ON users.token = tts.creator_user_token WHERE - tts.deleted_at IS NULL - AND tts.is_mod_disabled IS FALSE + tts.is_mod_disabled IS FALSE + AND tts.user_deleted_at IS NULL + AND tts.mod_deleted_at IS NULL "#) .fetch_all(mysql_pool) .await? @@ -143,9 +144,10 @@ SELECT tts.updated_at FROM tts_models as tts JOIN users -ON users.token = tts.creator_user_token + ON users.token = tts.creator_user_token WHERE - tts.deleted_at IS NULL + tts.user_deleted_at IS NULL + AND tts.mod_deleted_at IS NULL "#) .fetch_all(mysql_pool) .await? @@ -182,9 +184,10 @@ JOIN users ON users.token = tts.creator_user_token WHERE - tts.deleted_at IS NULL + users.username = ? AND tts.is_mod_disabled IS FALSE - AND users.username = ? + AND tts.user_deleted_at IS NULL + AND tts.mod_deleted_at IS NULL "#, scope_creator_username) .fetch_all(mysql_pool) @@ -210,8 +213,9 @@ JOIN users ON users.token = tts.creator_user_token WHERE - tts.deleted_at IS NULL - AND users.username = ? + users.username = ? + AND tts.user_deleted_at IS NULL + AND tts.mod_deleted_at IS NULL "#, scope_creator_username) .fetch_all(mysql_pool) diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/common_queries/list_w2l_inference_results.rs index e81b91258c..ce241d7a48 100644 --- a/src/common_queries/list_w2l_inference_results.rs +++ b/src/common_queries/list_w2l_inference_results.rs @@ -153,13 +153,14 @@ SELECT FROM w2l_results LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token + ON w2l_results.maybe_w2l_template_token = w2l_templates.token LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token + ON w2l_results.maybe_creator_user_token = users.token WHERE - w2l_results.deleted_at IS NULL - AND w2l_results.is_mod_hidden_from_public IS FALSE + w2l_results.is_mod_hidden_from_public IS FALSE AND w2l_templates.is_mod_public_listing_approved IS TRUE + AND w2l_results.user_deleted_at IS NULL + AND w2l_results.mod_deleted_at IS NULL "#) .fetch_all(mysql_pool) .await? @@ -189,11 +190,12 @@ SELECT FROM w2l_results LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token + ON w2l_results.maybe_w2l_template_token = w2l_templates.token LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token + ON w2l_results.maybe_creator_user_token = users.token WHERE - w2l_results.deleted_at IS NULL + w2l_results.user_deleted_at IS NULL + AND w2l_results.mod_deleted_at IS NULL "#) .fetch_all(mysql_pool) .await? @@ -235,14 +237,15 @@ SELECT FROM w2l_results LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token + ON w2l_results.maybe_w2l_template_token = w2l_templates.token LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token + ON w2l_results.maybe_creator_user_token = users.token WHERE - w2l_results.deleted_at IS NULL + users.username = ? AND w2l_results.is_mod_hidden_from_public IS FALSE AND w2l_templates.is_mod_public_listing_approved IS TRUE - AND users.username = ? + AND w2l_results.user_deleted_at IS NULL + AND w2l_results.mod_deleted_at IS NULL "#, scope_creator_username) .fetch_all(mysql_pool) @@ -273,12 +276,13 @@ SELECT FROM w2l_results LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token + ON w2l_results.maybe_w2l_template_token = w2l_templates.token LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token + ON w2l_results.maybe_creator_user_token = users.token WHERE - w2l_results.deleted_at IS NULL - AND users.username = ? + users.username = ? + AND w2l_results.user_deleted_at IS NULL + AND w2l_results.mod_deleted_at IS NULL "#, scope_creator_username) .fetch_all(mysql_pool) diff --git a/src/common_queries/list_w2l_templates.rs b/src/common_queries/list_w2l_templates.rs index 4fd6e4cc8e..0e20e95032 100644 --- a/src/common_queries/list_w2l_templates.rs +++ b/src/common_queries/list_w2l_templates.rs @@ -131,10 +131,11 @@ SELECT w2l.updated_at FROM w2l_templates as w2l JOIN users -ON users.token = w2l.creator_user_token + ON users.token = w2l.creator_user_token WHERE - w2l.deleted_at IS NULL - AND w2l.is_mod_public_listing_approved IS TRUE + w2l.is_mod_public_listing_approved IS TRUE + AND w2l.user_deleted_at IS NULL + AND w2l.mod_deleted_at IS NULL "#) .fetch_all(mysql_pool) .await? @@ -161,9 +162,10 @@ SELECT w2l.updated_at FROM w2l_templates as w2l JOIN users -ON users.token = w2l.creator_user_token + ON users.token = w2l.creator_user_token WHERE - w2l.deleted_at IS NULL + w2l.user_deleted_at IS NULL + AND w2l.mod_deleted_at IS NULL "#) .fetch_all(mysql_pool) .await? @@ -205,9 +207,10 @@ JOIN users ON users.token = w2l.creator_user_token WHERE - w2l.deleted_at IS NULL + users.username = ? AND w2l.is_mod_public_listing_approved IS TRUE - AND users.username = ? + AND w2l.user_deleted_at IS NULL + AND w2l.mod_deleted_at IS NULL "#, scope_creator_username) .fetch_all(mysql_pool) @@ -238,8 +241,9 @@ JOIN users ON users.token = w2l.creator_user_token WHERE - w2l.deleted_at IS NULL - AND users.username = ? + users.username = ? + AND w2l.user_deleted_at IS NULL + AND w2l.mod_deleted_at IS NULL "#, scope_creator_username) .fetch_all(mysql_pool) diff --git a/src/common_queries/query_user_profile.rs b/src/common_queries/query_user_profile.rs index df5f8a3861..3c17625e76 100644 --- a/src/common_queries/query_user_profile.rs +++ b/src/common_queries/query_user_profile.rs @@ -89,8 +89,10 @@ SELECT github_username, created_at FROM users -WHERE users.username = ? -AND users.deleted_at IS NULL +WHERE + users.username = ? + AND users.user_deleted_at IS NULL + AND users.mod_deleted_at IS NULL "#, username, ) diff --git a/src/common_queries/query_w2l_template.rs b/src/common_queries/query_w2l_template.rs index c6fa76ca4c..9c7bced44d 100644 --- a/src/common_queries/query_w2l_template.rs +++ b/src/common_queries/query_w2l_template.rs @@ -30,7 +30,8 @@ pub struct W2lTemplateRecordForResponse { pub is_mod_disabled: bool, pub created_at: DateTime, pub updated_at: DateTime, - pub deleted_at: Option>, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, } #[derive(Serialize)] @@ -51,7 +52,8 @@ pub struct W2lTemplateRecordRaw { pub is_mod_disabled: i8, pub created_at: DateTime, pub updated_at: DateTime, - pub deleted_at: Option>, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, } pub async fn select_w2l_template_by_token( @@ -99,7 +101,8 @@ pub async fn select_w2l_template_by_token( is_mod_disabled: i8_to_bool(template.is_mod_disabled), created_at: template.created_at.clone(), updated_at: template.updated_at.clone(), - deleted_at: template.deleted_at.clone(), + user_deleted_at: template.user_deleted_at.clone(), + mod_deleted_at: template.mod_deleted_at.clone(), }; Ok(Some(template_for_response)) @@ -129,10 +132,11 @@ SELECT w2l.is_mod_disabled, w2l.created_at, w2l.updated_at, - w2l.deleted_at + w2l.user_deleted_at, + w2l.mod_deleted_at FROM w2l_templates as w2l JOIN users -ON users.token = w2l.creator_user_token + ON users.token = w2l.creator_user_token WHERE w2l.token = ? "#, w2l_template_token @@ -165,12 +169,15 @@ SELECT w2l.is_mod_disabled, w2l.created_at, w2l.updated_at, - w2l.deleted_at + w2l.user_deleted_at, + w2l.mod_deleted_at FROM w2l_templates as w2l JOIN users -ON users.token = w2l.creator_user_token -WHERE w2l.token = ? -AND w2l.deleted_at IS NULL + ON users.token = w2l.creator_user_token +WHERE + w2l.token = ? + AND w2l.user_deleted_at IS NULL + AND w2l.mod_deleted_at IS NULL "#, w2l_template_token ) diff --git a/src/http_server/endpoints/tts/get_tts_model.rs b/src/http_server/endpoints/tts/get_tts_model.rs index 9cffbab77f..76fb2efa00 100644 --- a/src/http_server/endpoints/tts/get_tts_model.rs +++ b/src/http_server/endpoints/tts/get_tts_model.rs @@ -138,8 +138,10 @@ SELECT FROM tts_models as tts JOIN users ON users.token = tts.creator_user_token -WHERE tts.updatable_slug = ? -AND tts.deleted_at IS NULL +WHERE + tts.updatable_slug = ? + AND tts.user_deleted_at IS NULL + AND tts.mod_deleted_at IS NULL "#, &path.slug ) diff --git a/src/http_server/endpoints/tts/get_tts_result.rs b/src/http_server/endpoints/tts/get_tts_result.rs index c2ec9c531e..7727778d83 100644 --- a/src/http_server/endpoints/tts/get_tts_result.rs +++ b/src/http_server/endpoints/tts/get_tts_result.rs @@ -184,8 +184,9 @@ LEFT OUTER JOIN users LEFT OUTER JOIN users as model_users ON tts_models.creator_user_token = model_users.token WHERE - tts_results.deleted_at IS NULL - AND tts_results.token = ? + tts_results.token = ? + AND tts_results.user_deleted_at IS NULL + AND tts_results.mod_deleted_at IS NULL "#, &path.token ) diff --git a/src/http_server/endpoints/tts/list_tts_models.rs b/src/http_server/endpoints/tts/list_tts_models.rs index 4f58eb9874..f8155e41e2 100644 --- a/src/http_server/endpoints/tts/list_tts_models.rs +++ b/src/http_server/endpoints/tts/list_tts_models.rs @@ -117,8 +117,10 @@ SELECT tts.updated_at FROM tts_models as tts JOIN users -ON users.token = tts.creator_user_token -WHERE tts.deleted_at IS NULL + ON users.token = tts.creator_user_token +WHERE + tts.user_deleted_at IS NULL + AND tts.mod_deleted_at IS NULL "#, ) .fetch_all(&server_state.mysql_pool) diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index f66fc92037..1124cd02ab 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -168,8 +168,10 @@ SELECT website_url, created_at FROM users -WHERE users.username = ? -AND users.deleted_at IS NULL +WHERE + users.username = ? + AND users.user_deleted_at IS NULL + AND users.mod_deleted_at IS NULL "#, &path.username, ) diff --git a/src/http_server/endpoints/users/logout.rs b/src/http_server/endpoints/users/logout.rs index f66d445653..6a5f3e9960 100644 --- a/src/http_server/endpoints/users/logout.rs +++ b/src/http_server/endpoints/users/logout.rs @@ -107,8 +107,9 @@ async fn delete_session(session_token: &str, mysql_pool: &MySqlPool) -> AnyhowRe r#" UPDATE user_sessions SET deleted_at = CURRENT_TIMESTAMP() -WHERE token = ? -AND deleted_at IS NULL +WHERE + token = ? + AND deleted_at IS NULL "#, session_token.to_string(), ) diff --git a/src/http_server/endpoints/w2l/delete_w2l_template.rs b/src/http_server/endpoints/w2l/delete_w2l_template.rs index b46a81af0c..203dd96468 100644 --- a/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -110,11 +110,6 @@ pub async fn delete_w2l_template_handler( } }; - if !user_session.can_delete_other_users_w2l_templates { - warn!("user is not allowed to delete templates: {}", user_session.user_token); - return Err(DeleteW2lTemplateError::NotAuthorized); - } - let template_query_result = select_w2l_template_by_token( &path.slug, true, // Only mods can perform this action @@ -130,20 +125,44 @@ pub async fn delete_w2l_template_handler( Ok(Some(template)) => template, }; + let is_author = w2l_template.creator_user_token == user_session.user_token; + let is_mod = user_session.can_delete_other_users_w2l_templates; + + if !is_author && !is_mod { + warn!("user is not allowed to delete templates: {}", user_session.user_token); + return Err(DeleteW2lTemplateError::NotAuthorized); + } + let ip_address = get_request_ip(&http_request); let query_result = if request.set_delete { - delete_template( - &path.slug, - &user_session.user_token, - &server_state.mysql_pool - ).await + if is_author { + user_delete_template( + &path.slug, + &user_session.user_token, + &server_state.mysql_pool + ).await + } else { + mod_delete_template( + &path.slug, + &user_session.user_token, + &server_state.mysql_pool + ).await + } } else { - undelete_template( - &path.slug, - &user_session.user_token, - &server_state.mysql_pool - ).await + if is_author { + user_undelete_template( + &path.slug, + &user_session.user_token, + &server_state.mysql_pool + ).await + } else { + mod_undelete_template( + &path.slug, + &user_session.user_token, + &server_state.mysql_pool + ).await + } }; match query_result { @@ -166,7 +185,7 @@ pub async fn delete_w2l_template_handler( .body(body)) } -async fn delete_template( +async fn user_delete_template( template_token: &str, mod_user_token: &str, mysql_pool: &MySqlPool @@ -175,7 +194,30 @@ async fn delete_template( r#" UPDATE w2l_templates SET - deleted_at = CURRENT_TIMESTAMP, + user_deleted_at = CURRENT_TIMESTAMP, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + template_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +async fn mod_delete_template( + template_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + let _r = sqlx::query!( + r#" +UPDATE w2l_templates +SET + mod_deleted_at = CURRENT_TIMESTAMP, maybe_mod_user_token = ? WHERE token = ? @@ -189,7 +231,7 @@ LIMIT 1 Ok(()) } -async fn undelete_template( +async fn user_undelete_template( template_token: &str, mod_user_token: &str, mysql_pool: &MySqlPool @@ -199,7 +241,7 @@ async fn undelete_template( r#" UPDATE w2l_templates SET - deleted_at = NULL, + user_deleted_at = NULL, maybe_mod_user_token = ? WHERE token = ? @@ -212,3 +254,27 @@ LIMIT 1 .await?; Ok(()) } + +async fn mod_undelete_template( + template_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + + let _r = sqlx::query!( + r#" +UPDATE w2l_templates +SET + mod_deleted_at = NULL, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + template_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index f657ecdd93..83602a62bb 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -372,8 +372,10 @@ async fn check_template_exists(template_token: &str, mysql_pool: &MySqlPool) -> SELECT token as template_token FROM w2l_templates -WHERE token = ? -AND deleted_at IS NULL +WHERE + token = ? + AND user_deleted_at IS NULL + AND mod_deleted_at IS NULL "#, &template_token ) diff --git a/src/http_server/endpoints/w2l/get_w2l_result.rs b/src/http_server/endpoints/w2l/get_w2l_result.rs index 02dc49528c..a7a1f85b5a 100644 --- a/src/http_server/endpoints/w2l/get_w2l_result.rs +++ b/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -190,8 +190,9 @@ LEFT OUTER JOIN users LEFT OUTER JOIN users as template_users ON w2l_templates.creator_user_token = template_users.token WHERE - w2l_results.deleted_at IS NULL - AND w2l_results.token = ? + w2l_results.token = ? + AND w2l_results.user_deleted_at IS NULL + AND w2l_results.mod_deleted_at IS NULL "#, &path.token ) diff --git a/src/http_server/web_utils/session_checker.rs b/src/http_server/web_utils/session_checker.rs index d9ff4c4b6a..9c52114c26 100644 --- a/src/http_server/web_utils/session_checker.rs +++ b/src/http_server/web_utils/session_checker.rs @@ -213,7 +213,8 @@ LEFT OUTER JOIN user_roles ON users.user_role_slug = user_roles.slug WHERE user_sessions.token = ? AND user_sessions.deleted_at IS NULL - AND users.deleted_at IS NULL + AND users.user_deleted_at IS NULL + AND users.mod_deleted_at IS NULL "#, session_token.to_string(), ) diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 38f0106ecc..621c9d5aae 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -331,7 +331,8 @@ FROM tts_models as tts JOIN users ON users.token = tts.creator_user_token WHERE tts.token = ? -AND tts.deleted_at IS NULL +AND tts.user_deleted_at IS NULL +AND tts.mod_deleted_at IS NULL "#, &model_token ) diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index d17c198c2a..42eec12078 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -353,7 +353,8 @@ FROM w2l_templates as w2l JOIN users ON users.token = w2l.creator_user_token WHERE w2l.token = ? -AND w2l.deleted_at IS NULL +AND w2l.user_deleted_at IS NULL +AND w2l.mod_deleted_at IS NULL "#, &template_token ) From 7d1d1c1791a17a609a6cb5e1d13c2c1baf4b5c14 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Jun 2021 22:27:12 -0400 Subject: [PATCH 0342/1042] add new permissions (unused) --- debug/curl/seed.py | 2 +- migrations/2021-05-15-071534_users/up.sql | 6 ++++++ .../down.sql | 0 .../up.sql | 0 migrations/2021-06-09-010300_mod_audit_log/up.sql | 2 +- 5 files changed, 8 insertions(+), 2 deletions(-) rename migrations/{2021-05-15-212923_badges => 2021-05-15-212920_badges}/down.sql (100%) rename migrations/{2021-05-15-212923_badges => 2021-05-15-212920_badges}/up.sql (100%) diff --git a/debug/curl/seed.py b/debug/curl/seed.py index 877c52638e..95826c4460 100755 --- a/debug/curl/seed.py +++ b/debug/curl/seed.py @@ -104,7 +104,7 @@ def upload_tts_model(content_url, title, session_cookie): #upload_tts_model('https://drive.google.com/file/d/1w3oBk5vzyHurqYwP95UEQv-_NeMNu6pt/view?usp=sharing', 'Noire', session_echelon) # Copied into Storyteller Google Drive -# TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1d8KPwYqbRsGiYx1c4TCW4xCtcRLdIU1s/view?usp=sharing', 'Zephyr', session_echelon) +upload_tts_model('https://drive.google.com/file/d/1d8KPwYqbRsGiYx1c4TCW4xCtcRLdIU1s/view?usp=sharing', 'Zephyr', session_echelon) # TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1yhoamPz1NOMy403gu2v8sIx5hPj3LQWs/view?usp=sharing', 'Wakko Warner', session_other) # TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1I6KGQZFMMB2ZvCzWEqy9xTgGl0L5cmfP/view?usp=sharing', 'Turret-192', session_echelon) # TEMP COMMENT DUE TO RATE LIMIT upload_tts_model('https://drive.google.com/file/d/1T_hvgGcNOfofhNmDWUqpVtYDdSXQrWdE/view?usp=sharing', 'technoblade-324', session_brandon) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index fbcd7426e2..dacf161d03 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -127,7 +127,10 @@ CREATE TABLE user_roles ( can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE, can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, can_delete_own_tts_models BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_own_tts_results BOOLEAN NOT NULL DEFAULT FALSE, can_delete_own_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_own_w2l_results BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_own_account BOOLEAN NOT NULL DEFAULT FALSE, -- Moderation can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, @@ -135,8 +138,11 @@ CREATE TABLE user_roles ( can_edit_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE, can_edit_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, can_delete_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_other_users_tts_results BOOLEAN NOT NULL DEFAULT FALSE, can_delete_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_other_users_w2l_results BOOLEAN NOT NULL DEFAULT FALSE, can_ban_users BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_users BOOLEAN NOT NULL DEFAULT FALSE, -- Incremented with every update. version INT NOT NULL DEFAULT 0, diff --git a/migrations/2021-05-15-212923_badges/down.sql b/migrations/2021-05-15-212920_badges/down.sql similarity index 100% rename from migrations/2021-05-15-212923_badges/down.sql rename to migrations/2021-05-15-212920_badges/down.sql diff --git a/migrations/2021-05-15-212923_badges/up.sql b/migrations/2021-05-15-212920_badges/up.sql similarity index 100% rename from migrations/2021-05-15-212923_badges/up.sql rename to migrations/2021-05-15-212920_badges/up.sql diff --git a/migrations/2021-06-09-010300_mod_audit_log/up.sql b/migrations/2021-06-09-010300_mod_audit_log/up.sql index ca30b2650a..a131f54031 100644 --- a/migrations/2021-06-09-010300_mod_audit_log/up.sql +++ b/migrations/2021-06-09-010300_mod_audit_log/up.sql @@ -2,7 +2,7 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile -CREATE TABLE moderator_audit_logs_entries ( +CREATE TABLE moderator_audit_log_entries ( -- Not used for anything except replication. id BIGINT(20) NOT NULL AUTO_INCREMENT, From 5cee2f4c5e83f285b05fbed3b3673f8a739b2102 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Jun 2021 22:37:28 -0400 Subject: [PATCH 0343/1042] more permissions --- migrations/2021-05-15-071534_users/up.sql | 6 ++--- .../endpoints/users/session_info.rs | 12 +++++++++ src/http_server/web_utils/session_checker.rs | 26 ++++++++++++++++++- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index dacf161d03..36cdc55b65 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -122,15 +122,15 @@ CREATE TABLE user_roles ( -- Usage can_use_tts BOOLEAN NOT NULL DEFAULT FALSE, can_use_w2l BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_own_tts_results BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_own_w2l_results BOOLEAN NOT NULL DEFAULT FALSE, + can_delete_own_account BOOLEAN NOT NULL DEFAULT FALSE, -- Contribution can_upload_tts_models BOOLEAN NOT NULL DEFAULT FALSE, can_upload_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, can_delete_own_tts_models BOOLEAN NOT NULL DEFAULT FALSE, - can_delete_own_tts_results BOOLEAN NOT NULL DEFAULT FALSE, can_delete_own_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, - can_delete_own_w2l_results BOOLEAN NOT NULL DEFAULT FALSE, - can_delete_own_account BOOLEAN NOT NULL DEFAULT FALSE, -- Moderation can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, diff --git a/src/http_server/endpoints/users/session_info.rs b/src/http_server/endpoints/users/session_info.rs index bd09472eb6..ae2060c9d4 100644 --- a/src/http_server/endpoints/users/session_info.rs +++ b/src/http_server/endpoints/users/session_info.rs @@ -34,6 +34,9 @@ pub struct UserInfo { // Usage permissions: pub can_use_tts: bool, pub can_use_w2l: bool, + pub can_delete_own_tts_results: bool, + pub can_delete_own_w2l_results: bool, + pub can_delete_own_account: bool, // Contribution permissions: pub can_upload_tts_models: bool, @@ -47,8 +50,11 @@ pub struct UserInfo { pub can_edit_other_users_tts_models: bool, pub can_edit_other_users_w2l_templates: bool, pub can_delete_other_users_tts_models: bool, + pub can_delete_other_users_tts_results: bool, pub can_delete_other_users_w2l_templates: bool, + pub can_delete_other_users_w2l_results: bool, pub can_ban_users: bool, + pub can_delete_users: bool, } #[derive(Serialize)] @@ -126,6 +132,9 @@ pub async fn session_info_handler( // Usage permissions: can_use_tts: session_data.can_use_tts, can_use_w2l: session_data.can_use_w2l, + can_delete_own_tts_results: session_data.can_delete_own_tts_results, + can_delete_own_w2l_results: session_data.can_delete_own_w2l_results, + can_delete_own_account: session_data.can_delete_own_account, // Contribution permissions: can_upload_tts_models: session_data.can_upload_tts_models, @@ -139,8 +148,11 @@ pub async fn session_info_handler( can_edit_other_users_tts_models: session_data.can_edit_other_users_tts_models, can_edit_other_users_w2l_templates: session_data.can_edit_other_users_w2l_templates, can_delete_other_users_tts_models: session_data.can_delete_other_users_tts_models, + can_delete_other_users_tts_results: session_data.can_delete_other_users_tts_results, can_delete_other_users_w2l_templates: session_data.can_delete_other_users_w2l_templates, + can_delete_other_users_w2l_results: session_data.can_delete_other_users_w2l_results, can_ban_users: session_data.can_ban_users, + can_delete_users: session_data.can_delete_users, }); } } diff --git a/src/http_server/web_utils/session_checker.rs b/src/http_server/web_utils/session_checker.rs index 9c52114c26..1de8b5f8ca 100644 --- a/src/http_server/web_utils/session_checker.rs +++ b/src/http_server/web_utils/session_checker.rs @@ -46,6 +46,9 @@ pub struct SessionUserRecord { // Usage pub can_use_tts: bool, pub can_use_w2l: bool, + pub can_delete_own_tts_results: bool, + pub can_delete_own_w2l_results: bool, + pub can_delete_own_account: bool, // Contribution pub can_upload_tts_models: bool, @@ -59,8 +62,11 @@ pub struct SessionUserRecord { pub can_edit_other_users_tts_models: bool, pub can_edit_other_users_w2l_templates: bool, pub can_delete_other_users_tts_models: bool, + pub can_delete_other_users_tts_results: bool, pub can_delete_other_users_w2l_templates: bool, + pub can_delete_other_users_w2l_results: bool, pub can_ban_users: bool, + pub can_delete_users: bool, } pub struct SessionUserRawDbRecord { @@ -88,6 +94,9 @@ pub struct SessionUserRawDbRecord { // Usage pub can_use_tts: Option, pub can_use_w2l: Option, + pub can_delete_own_tts_results: Option, + pub can_delete_own_w2l_results: Option, + pub can_delete_own_account: Option, // Contribution pub can_upload_tts_models: Option, @@ -101,8 +110,11 @@ pub struct SessionUserRawDbRecord { pub can_edit_other_users_tts_models: Option, pub can_edit_other_users_w2l_templates: Option, pub can_delete_other_users_tts_models: Option, + pub can_delete_other_users_tts_results: Option, pub can_delete_other_users_w2l_templates: Option, + pub can_delete_other_users_w2l_results: Option, pub can_ban_users: Option, + pub can_delete_users: Option, } impl SessionChecker { @@ -192,6 +204,9 @@ SELECT user_roles.can_use_tts, user_roles.can_use_w2l, + user_roles.can_delete_own_tts_results, + user_roles.can_delete_own_w2l_results, + user_roles.can_delete_own_account, user_roles.can_upload_tts_models, user_roles.can_upload_w2l_templates, @@ -203,8 +218,11 @@ SELECT user_roles.can_edit_other_users_tts_models, user_roles.can_edit_other_users_w2l_templates, user_roles.can_delete_other_users_tts_models, + user_roles.can_delete_other_users_tts_results, user_roles.can_delete_other_users_w2l_templates, - user_roles.can_ban_users + user_roles.can_delete_other_users_w2l_results, + user_roles.can_ban_users, + user_roles.can_delete_users FROM users LEFT OUTER JOIN user_sessions @@ -244,6 +262,9 @@ WHERE user_sessions.token = ? // Usage can_use_tts: convert_optional_db_bool_default_false(raw_user_record.can_use_tts), can_use_w2l: convert_optional_db_bool_default_false(raw_user_record.can_use_w2l), + can_delete_own_tts_results: convert_optional_db_bool_default_false(raw_user_record.can_delete_own_tts_results), + can_delete_own_w2l_results: convert_optional_db_bool_default_false(raw_user_record.can_delete_own_w2l_results), + can_delete_own_account: convert_optional_db_bool_default_false(raw_user_record.can_delete_own_account), // Contribution can_upload_tts_models: convert_optional_db_bool_default_false(raw_user_record.can_upload_tts_models), can_upload_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_upload_w2l_templates), @@ -255,8 +276,11 @@ WHERE user_sessions.token = ? can_edit_other_users_tts_models: convert_optional_db_bool_default_false(raw_user_record.can_edit_other_users_tts_models), can_edit_other_users_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_edit_other_users_w2l_templates), can_delete_other_users_tts_models: convert_optional_db_bool_default_false(raw_user_record.can_delete_other_users_tts_models), + can_delete_other_users_tts_results: convert_optional_db_bool_default_false(raw_user_record.can_delete_other_users_tts_results), can_delete_other_users_w2l_templates: convert_optional_db_bool_default_false(raw_user_record.can_delete_other_users_w2l_templates), + can_delete_other_users_w2l_results: convert_optional_db_bool_default_false(raw_user_record.can_delete_other_users_w2l_results), can_ban_users: convert_optional_db_bool_default_false(raw_user_record.can_ban_users), + can_delete_users: convert_optional_db_bool_default_false(raw_user_record.can_delete_users) , }; Ok(Some(result_user_record)) From 72a0ca01d986255d8fca0c37e3ec04c2d58fc22f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Jun 2021 22:41:22 -0400 Subject: [PATCH 0344/1042] seed with new permissions --- seed/sql/system_roles.sql | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/seed/sql/system_roles.sql b/seed/sql/system_roles.sql index 26c7860e31..d9e1c42c11 100644 --- a/seed/sql/system_roles.sql +++ b/seed/sql/system_roles.sql @@ -7,6 +7,9 @@ SET -- Usage can_use_tts = TRUE, can_use_w2l = TRUE, + can_delete_own_tts_results = TRUE, + can_delete_own_w2l_results = TRUE, + can_delete_own_account = TRUE, -- Contribution can_upload_tts_models = TRUE, can_upload_w2l_templates = TRUE, @@ -18,8 +21,11 @@ SET can_edit_other_users_tts_models = FALSE, can_edit_other_users_w2l_templates = FALSE, can_delete_other_users_tts_models = FALSE, + can_delete_other_users_tts_results = FALSE, can_delete_other_users_w2l_templates = FALSE, - can_ban_users = FALSE; + can_delete_other_users_w2l_results = FALSE, + can_ban_users = FALSE, + can_delete_users = FALSE; INSERT INTO user_roles SET @@ -28,6 +34,9 @@ SET -- Usage can_use_tts = TRUE, can_use_w2l = TRUE, + can_delete_own_tts_results = TRUE, + can_delete_own_w2l_results = TRUE, + can_delete_own_account = TRUE, -- Contribution can_upload_tts_models = TRUE, can_upload_w2l_templates = TRUE, @@ -39,8 +48,11 @@ SET can_edit_other_users_tts_models = TRUE, can_edit_other_users_w2l_templates = TRUE, can_delete_other_users_tts_models = TRUE, + can_delete_other_users_tts_results = TRUE, can_delete_other_users_w2l_templates = TRUE, - can_ban_users = TRUE; + can_delete_other_users_w2l_results = TRUE, + can_ban_users = TRUE, + can_delete_users = TRUE; INSERT INTO user_roles SET @@ -49,6 +61,9 @@ SET -- Usage can_use_tts = TRUE, can_use_w2l = TRUE, + can_delete_own_tts_results = TRUE, + can_delete_own_w2l_results = TRUE, + can_delete_own_account = TRUE, -- Contribution can_upload_tts_models = TRUE, can_upload_w2l_templates = TRUE, @@ -60,6 +75,9 @@ SET can_edit_other_users_tts_models = TRUE, can_edit_other_users_w2l_templates = TRUE, can_delete_other_users_tts_models = TRUE, + can_delete_other_users_tts_results = TRUE, can_delete_other_users_w2l_templates = TRUE, - can_ban_users = TRUE; + can_delete_other_users_w2l_results = TRUE, + can_ban_users = TRUE, + can_delete_users = TRUE; From ec1e280eda06484d5719343c3250500564a2693e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 00:11:40 -0400 Subject: [PATCH 0345/1042] new edit w2l endpoint (untested) + http helper functions --- migrations/2021-05-15-071601_assets/up.sql | 24 +- src/common_queries/query_w2l_template.rs | 5 + .../endpoints/w2l/edit_w2l_template.rs | 230 ++++++++++++++++++ src/http_server/endpoints/w2l/mod.rs | 1 + src/http_server/web_utils/mod.rs | 2 + .../web_utils/response_error_helpers.rs | 30 +++ .../web_utils/response_success_helpers.rs | 19 ++ src/job_queries/tts_download_job_queries.rs | 2 +- src/job_queries/w2l_download_job_queries.rs | 37 +-- src/main.rs | 6 + 10 files changed, 332 insertions(+), 24 deletions(-) create mode 100644 src/http_server/endpoints/w2l/edit_w2l_template.rs create mode 100644 src/http_server/web_utils/response_error_helpers.rs create mode 100644 src/http_server/web_utils/response_success_helpers.rs diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 90ff5bae3b..7dd0a03369 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -66,7 +66,8 @@ CREATE TABLE tts_models ( -- For abuse tracking. -- Wide enough for IPv4/6 - creator_ip_address VARCHAR(40) NOT NULL, + creator_ip_address_creation VARCHAR(40) NOT NULL, + creator_ip_address_last_update VARCHAR(40) NOT NULL, -- (THIS MIGHT NOT BE USED) -- NB: DO NOT SORT! @@ -114,6 +115,11 @@ CREATE TABLE tts_models ( -- The last moderator that made changes. maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + -- ========== TIMESTAMPS ========== created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -133,7 +139,8 @@ CREATE TABLE tts_models ( KEY fk_maybe_mod_user_token (maybe_mod_user_token), KEY fk_maybe_subject_token (maybe_subject_token), KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), - KEY index_creator_ip_address (creator_ip_address), + KEY index_creator_ip_address_creation (creator_ip_address_creation), + KEY index_creator_ip_address_last_update (creator_ip_address_last_update), KEY index_creator_set_visibility (creator_set_visibility), KEY index_private_bucket_hash (private_bucket_hash), KEY index_is_mod_disabled (is_mod_disabled) @@ -187,7 +194,8 @@ CREATE TABLE w2l_templates ( -- For abuse tracking. -- Wide enough for IPv4/6 - creator_ip_address VARCHAR(40) NOT NULL, + creator_ip_address_creation VARCHAR(40) NOT NULL, + creator_ip_address_last_update VARCHAR(40) NOT NULL, -- (THIS MIGHT NOT BE USED) -- NB: DO NOT SORT! @@ -244,6 +252,7 @@ CREATE TABLE w2l_templates ( -- In this case, a moderator disables it. -- This also disables it for the creator. + -- Unlike deletion, it remains "visible" to those that have access. is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, -- Extremely popular models may be locked from deletion or modification by users. @@ -255,6 +264,11 @@ CREATE TABLE w2l_templates ( -- The last moderator that made changes. maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + -- ========== TIMESTAMPS ========== created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -275,10 +289,10 @@ CREATE TABLE w2l_templates ( KEY fk_maybe_subject_token (maybe_subject_token), KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), KEY index_template_type (template_type), - KEY index_creator_ip_address (creator_ip_address), + KEY index_creator_ip_address_creation (creator_ip_address_creation), + KEY index_creator_ip_address_last_update (creator_ip_address_last_update), KEY index_creator_set_visibility (creator_set_visibility), KEY index_private_bucket_hash (private_bucket_hash), KEY index_is_mod_disabled (is_mod_disabled) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; - diff --git a/src/common_queries/query_w2l_template.rs b/src/common_queries/query_w2l_template.rs index 9c7bced44d..026548e5e1 100644 --- a/src/common_queries/query_w2l_template.rs +++ b/src/common_queries/query_w2l_template.rs @@ -28,6 +28,7 @@ pub struct W2lTemplateRecordForResponse { pub maybe_video_object_name: Option, pub is_mod_public_listing_approved: Option, pub is_mod_disabled: bool, + pub is_locked_from_user_modification: bool, pub created_at: DateTime, pub updated_at: DateTime, pub user_deleted_at: Option>, @@ -50,6 +51,7 @@ pub struct W2lTemplateRecordRaw { pub maybe_public_bucket_preview_video_object_name: Option, pub is_mod_public_listing_approved: Option, pub is_mod_disabled: i8, + pub is_locked_from_user_modification: i8, pub created_at: DateTime, pub updated_at: DateTime, pub user_deleted_at: Option>, @@ -99,6 +101,7 @@ pub async fn select_w2l_template_by_token( maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), is_mod_public_listing_approved: nullable_i8_to_optional_bool(template.is_mod_public_listing_approved), is_mod_disabled: i8_to_bool(template.is_mod_disabled), + is_locked_from_user_modification: i8_to_bool(template.is_locked_from_user_modification), created_at: template.created_at.clone(), updated_at: template.updated_at.clone(), user_deleted_at: template.user_deleted_at.clone(), @@ -130,6 +133,7 @@ SELECT w2l.maybe_public_bucket_preview_video_object_name, w2l.is_mod_public_listing_approved, w2l.is_mod_disabled, + w2l.is_locked_from_user_modification, w2l.created_at, w2l.updated_at, w2l.user_deleted_at, @@ -167,6 +171,7 @@ SELECT w2l.maybe_public_bucket_preview_video_object_name, w2l.is_mod_public_listing_approved, w2l.is_mod_disabled, + w2l.is_locked_from_user_modification, w2l.created_at, w2l.updated_at, w2l.user_deleted_at, diff --git a/src/http_server/endpoints/w2l/edit_w2l_template.rs b/src/http_server/endpoints/w2l/edit_w2l_template.rs new file mode 100644 index 0000000000..8002f6ebc4 --- /dev/null +++ b/src/http_server/endpoints/w2l/edit_w2l_template.rs @@ -0,0 +1,230 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::query_w2l_template::select_w2l_template_by_token; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct EditW2lTemplatePathInfo { + template_token: String, +} + +#[derive(Deserialize)] +pub struct EditW2lTemplateRequest { + pub title: Option, + pub description_markdown: Option, + + //pub updatable_slug: Option, + //pub creator_set_visibility: Option, + + //pub is_mod_disabled: Option, + //pub is_locked_from_user_modification: Option, + //pub maybe_mod_comments: Option, +} + +#[derive(Debug, Display)] +pub enum EditW2lTemplateError { + BadInput(String), + NotAuthorized, + TemplateNotFound, + ServerError, +} + +impl ResponseError for EditW2lTemplateError { + fn status_code(&self) -> StatusCode { + match *self { + EditW2lTemplateError::BadInput(_) => StatusCode::BAD_REQUEST, + EditW2lTemplateError::NotAuthorized => StatusCode::UNAUTHORIZED, + EditW2lTemplateError::TemplateNotFound => StatusCode::NOT_FOUND, + EditW2lTemplateError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + EditW2lTemplateError::BadInput(reason) => reason.to_string(), + EditW2lTemplateError::NotAuthorized=> "unauthorized".to_string(), + EditW2lTemplateError::TemplateNotFound => "not found".to_string(), + EditW2lTemplateError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn edit_w2l_template_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EditW2lTemplateError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(EditW2lTemplateError::NotAuthorized); + } + }; + + // NB: First permission check. + // Only mods should see deleted templates (both user_* and mod_* deleted). + let is_mod_that_can_see_deleted = user_session.can_delete_other_users_w2l_templates; + + let template_lookup_result = select_w2l_template_by_token( + &path.template_token, + is_mod_that_can_see_deleted, + &server_state.mysql_pool).await; + + let template_record = match template_lookup_result { + Ok(Some(result)) => { + info!("Found template: {}", result.template_token); + result + }, + Ok(None) => { + warn!("could not find template"); + return Err(EditW2lTemplateError::TemplateNotFound); + }, + Err(err) => { + warn!("error looking up template: {:?}", err); + return Err(EditW2lTemplateError::TemplateNotFound); + }, + }; + + // NB: Second set of permission checks + let mut editor_is_original_user = false; + let mut editor_is_moderator = false; + + if template_record.creator_user_token == user_session.user_token { + editor_is_original_user = true; + } + + if user_session.can_edit_other_users_w2l_templates { + editor_is_moderator = true; + } + + if !editor_is_original_user && !editor_is_moderator { + return Err(EditW2lTemplateError::NotAuthorized); + } + + if !editor_is_moderator { + if template_record.is_locked_from_user_modification || template_record.is_mod_disabled { + return Err(EditW2lTemplateError::NotAuthorized); + } + } + + // Fields to set + let mut title = None; + let mut description_markdown = None; + let mut description_html = None; + + if let Some(payload) = request.title.as_deref() { + if contains_slurs(payload) { + return Err(EditW2lTemplateError::BadInput("title contains slurs".to_string())); + } + + title = Some(payload.to_string()); + } + + if let Some(markdown) = request.description_markdown.as_deref() { + if contains_slurs(markdown) { + return Err(EditW2lTemplateError::BadInput("description contains slurs".to_string())); + } + + let markdown = markdown.trim().to_string(); + let html = markdown_to_html(&markdown); + + description_markdown = Some(markdown); + description_html = Some(html); + } + + let ip_address = get_request_ip(&http_request); + + let query_result = if editor_is_original_user { + // We need to store the IP address details. + sqlx::query!( + r#" +UPDATE w2l_templates +SET + title = ?, + description_markdown = ?, + description_rendered_html = ?, + creator_ip_address_last_update = ?, + version = version + 1 + +WHERE w2l_templates.token = ? +LIMIT 1 + "#, + &title, + &description_markdown, + &description_html, + &ip_address, + &template_record.template_token, + ) + .execute(&server_state.mysql_pool) + .await + } else { + // We need to store the moderator details. + sqlx::query!( + r#" +UPDATE w2l_templates +SET + title = ?, + description_markdown = ?, + description_rendered_html = ?, + maybe_mod_user_token = ?, + version = version + 1 + +WHERE w2l_templates.token = ? +LIMIT 1 + "#, + &title, + &description_markdown, + &description_html, + &user_session.user_token, + &template_record.template_token, + ) + .execute(&server_state.mysql_pool) + .await + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update W2L template edit DB error: {:?}", err); + return Err(EditW2lTemplateError::ServerError); + } + }; + + Ok(simple_json_success()) +} diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index 9eb955a1d0..4f323aa6f4 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -1,4 +1,5 @@ pub mod delete_w2l_template; +pub mod edit_w2l_template; pub mod enqueue_infer_w2l; pub mod enqueue_infer_w2l_with_uploads; pub mod enqueue_upload_w2l_template; diff --git a/src/http_server/web_utils/mod.rs b/src/http_server/web_utils/mod.rs index 53c5f9f362..701d2bbdb3 100644 --- a/src/http_server/web_utils/mod.rs +++ b/src/http_server/web_utils/mod.rs @@ -1,4 +1,6 @@ pub mod cookie_manager; pub mod ip_address; pub mod read_multipart_field_bytes; +pub mod response_error_helpers; +pub mod response_success_helpers; pub mod session_checker; \ No newline at end of file diff --git a/src/http_server/web_utils/response_error_helpers.rs b/src/http_server/web_utils/response_error_helpers.rs new file mode 100644 index 0000000000..2977eb183d --- /dev/null +++ b/src/http_server/web_utils/response_error_helpers.rs @@ -0,0 +1,30 @@ +use actix_http::http::header; +use actix_web::dev::HttpResponseBuilder; +use actix_web::http::StatusCode; +use actix_web::{HttpResponse, ResponseError}; + +#[derive(Serialize)] +pub struct SimpleGenericJsonError { + pub success: bool, + pub error_reason: String, +} + +pub fn to_simple_json_error( + error_reason: &str, + status_code: StatusCode, +) -> HttpResponse { + + let response = SimpleGenericJsonError { + success: false, + error_reason : error_reason.to_string(), + }; + + let body = match serde_json::to_string(&response) { + Ok(json) => json, + Err(_) => "{}".to_string(), + }; + + HttpResponseBuilder::new(status_code) + .set_header(header::CONTENT_TYPE, "application/json") + .body(body) +} \ No newline at end of file diff --git a/src/http_server/web_utils/response_success_helpers.rs b/src/http_server/web_utils/response_success_helpers.rs new file mode 100644 index 0000000000..dae959059f --- /dev/null +++ b/src/http_server/web_utils/response_success_helpers.rs @@ -0,0 +1,19 @@ +use actix_web::HttpResponse; + +#[derive(Serialize)] +pub struct SimpleGenericJsonSuccess { + pub success: bool, +} + +pub fn simple_json_success() -> HttpResponse { + let response = SimpleGenericJsonSuccess { + success: true, + }; + + let body = serde_json::to_string(&response) + .unwrap_or("{ \"success\": true }".to_string()); + + HttpResponse::Ok() + .content_type("application/json") + .body(body) +} diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index aebab0804f..6a71e019bc 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -228,7 +228,7 @@ SET description_markdown = '', description_rendered_html = '', creator_user_token = ?, - creator_ip_address = ?, + creator_ip_address_creation = ?, original_download_url = ?, private_bucket_hash = ?, private_bucket_object_name = ?, diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 93c5577d96..99e421e3d3 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -199,23 +199,24 @@ WHERE id = ? Ok(()) } -pub async fn insert_w2l_template(pool: &MySqlPool, - template_type: &str, // TODO: ENUM! - job: &W2lTemplateUploadJobRecord, - private_bucket_hash: &str, - private_bucket_object_name: &str, - private_bucket_cached_faces_object_name: &str, - maybe_image_preview_object_name: Option<&str>, - maybe_video_preview_object_name: Option<&str>, - file_size_bytes: u64, - maybe_mime_type: Option<&str>, - frame_width: u32, - frame_height: u32, - frame_count: u64, - fps: f32, - duration_millis: u64) - -> AnyhowResult<(u64, String)> -{ +pub async fn insert_w2l_template( + pool: &MySqlPool, + template_type: &str, // TODO: ENUM! + job: &W2lTemplateUploadJobRecord, + private_bucket_hash: &str, + private_bucket_object_name: &str, + private_bucket_cached_faces_object_name: &str, + maybe_image_preview_object_name: Option<&str>, + maybe_video_preview_object_name: Option<&str>, + file_size_bytes: u64, + maybe_mime_type: Option<&str>, + frame_width: u32, + frame_height: u32, + frame_count: u64, + fps: f32, + duration_millis: u64 +) -> AnyhowResult<(u64, String)> { + let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?; let updatable_slug = model_token.clone(); @@ -230,7 +231,7 @@ SET description_markdown = '', description_rendered_html = '', creator_user_token = ?, - creator_ip_address = ?, + creator_ip_address_creation = ?, original_download_url = ?, private_bucket_hash = ?, private_bucket_object_name = ?, diff --git a/src/main.rs b/src/main.rs index 506b4faca5..9377b4dca1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -70,6 +70,7 @@ use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_ use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; use crate::http_server::endpoints::w2l::delete_w2l_template::delete_w2l_template_handler; +use crate::http_server::endpoints::w2l::edit_w2l_template::edit_w2l_template_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -332,6 +333,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) .service( + web::resource("/template/{template_token}/edit") + .route(web::post().to(edit_w2l_template_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service( web::resource("/template/{slug}/moderate") .route(web::post().to(set_w2l_template_mod_approval_handler)) .route(web::head().to(|| HttpResponse::Ok())) From 3065f20f9ff30811416cdcd1c79a166f1c92fea8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 00:29:38 -0400 Subject: [PATCH 0346/1042] add endpoint for editing tts models (untested) --- src/common_queries/mod.rs | 1 + src/common_queries/query_tts_model.rs | 171 +++++++++++++ .../endpoints/tts/edit_tts_model.rs | 234 ++++++++++++++++++ src/http_server/endpoints/tts/mod.rs | 1 + src/main.rs | 8 +- 5 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 src/common_queries/query_tts_model.rs create mode 100644 src/http_server/endpoints/tts/edit_tts_model.rs diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index 8feffdc37f..1caba8130d 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -4,6 +4,7 @@ pub mod list_tts_inference_results; pub mod list_tts_models; pub mod list_w2l_inference_results; pub mod list_w2l_templates; +pub mod query_tts_model; pub mod query_user_profile; pub mod query_w2l_template; pub mod sessions; \ No newline at end of file diff --git a/src/common_queries/query_tts_model.rs b/src/common_queries/query_tts_model.rs new file mode 100644 index 0000000000..8184d4be54 --- /dev/null +++ b/src/common_queries/query_tts_model.rs @@ -0,0 +1,171 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::database_helpers::boolean_converters::i8_to_bool; +use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +#[derive(Serialize)] +pub struct TtsModelRecordForResponse { + pub model_token: String, + pub tts_model_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub description_markdown: String, + pub description_rendered_html: String, + pub is_mod_disabled: bool, + pub is_locked_from_user_modification: bool, + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +#[derive(Serialize)] +pub struct TtsModelRecordRaw { + pub model_token: String, + pub tts_model_type: String, + pub creator_user_token: String, + pub creator_username: String, + pub creator_display_name: String, + pub updatable_slug: String, + pub title: String, + pub description_markdown: String, + pub description_rendered_html: String, + pub is_mod_disabled: i8, + pub is_locked_from_user_modification: i8, + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +pub async fn select_tts_model_by_token( + tts_model_token: &str, + can_see_deleted: bool, + mysql_pool: &MySqlPool +) -> AnyhowResult> { + + let maybe_record = if can_see_deleted { + select_including_deleted(tts_model_token, mysql_pool).await + } else { + select_without_deleted(tts_model_token, mysql_pool).await + }; + + let model : TtsModelRecordRaw = match maybe_record { + Ok(model) => model, + Err(ref err) => { + match err { + RowNotFound => { + warn!("tts model not found: {:?}", &err); + return Ok(None); + }, + _ => { + warn!("tts model query error: {:?}", &err); + return Err(anyhow!("database error")); + } + } + } + }; + + let model_for_response = TtsModelRecordForResponse { + model_token: model.model_token.clone(), + tts_model_type: model.tts_model_type.clone(), + creator_user_token: model.creator_user_token.clone(), + creator_username: model.creator_username.clone(), + creator_display_name: model.creator_display_name.clone(), + updatable_slug: model.updatable_slug.clone(), + title: model.title.clone(), + description_markdown: model.description_markdown.clone(), + description_rendered_html: model.description_rendered_html.clone(), + is_mod_disabled: i8_to_bool(model.is_mod_disabled), + is_locked_from_user_modification: i8_to_bool(model.is_locked_from_user_modification), + created_at: model.created_at.clone(), + updated_at: model.updated_at.clone(), + user_deleted_at: model.user_deleted_at.clone(), + mod_deleted_at: model.mod_deleted_at.clone(), + }; + + Ok(Some(model_for_response)) +} + +async fn select_including_deleted( + tts_model_token: &str, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + TtsModelRecordRaw, + r#" +SELECT + tts.token as model_token, + tts.tts_model_type, + tts.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + tts.updatable_slug, + tts.title, + tts.description_markdown, + tts.description_rendered_html, + tts.is_mod_disabled, + tts.is_locked_from_user_modification, + tts.created_at, + tts.updated_at, + tts.user_deleted_at, + tts.mod_deleted_at +FROM tts_models as tts +JOIN users + ON users.token = tts.creator_user_token +WHERE tts.token = ? + "#, + tts_model_token + ) + .fetch_one(mysql_pool) + .await // TODO: This will return error if it doesn't exist +} + +async fn select_without_deleted( + tts_model_token: &str, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + TtsModelRecordRaw, + r#" +SELECT + tts.token as model_token, + tts.tts_model_type, + tts.creator_user_token, + users.username as creator_username, + users.display_name as creator_display_name, + tts.updatable_slug, + tts.title, + tts.description_markdown, + tts.description_rendered_html, + tts.is_mod_disabled, + tts.is_locked_from_user_modification, + tts.created_at, + tts.updated_at, + tts.user_deleted_at, + tts.mod_deleted_at +FROM tts_models as tts +JOIN users + ON users.token = tts.creator_user_token +WHERE + tts.token = ? + AND tts.user_deleted_at IS NULL + AND tts.mod_deleted_at IS NULL + "#, + tts_model_token + ) + .fetch_one(mysql_pool) + .await // TODO: This will return error if it doesn't exist +} diff --git a/src/http_server/endpoints/tts/edit_tts_model.rs b/src/http_server/endpoints/tts/edit_tts_model.rs new file mode 100644 index 0000000000..2b4a4d2be8 --- /dev/null +++ b/src/http_server/endpoints/tts/edit_tts_model.rs @@ -0,0 +1,234 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::query_tts_model::select_tts_model_by_token; +use crate::common_queries::sessions::create_session_for_user; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::email_to_gravatar::email_to_gravatar; +use crate::util::markdown_to_html::markdown_to_html; +use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct EditTtsModelPathInfo { + model_token: String, +} + +#[derive(Deserialize)] +pub struct EditTtsModelRequest { + pub title: Option, + pub description_markdown: Option, + + //pub updatable_slug: Option, + //pub creator_set_visibility: Option, + + //pub tts_model_type: Option, + //pub text_preprocessing_algorithm: Option, + //pub vocoder_token: Option, + + //pub is_mod_disabled: Option, + //pub is_locked_from_user_modification: Option, + //pub maybe_mod_comments: Option, +} + +#[derive(Debug, Display)] +pub enum EditTtsModelError { + BadInput(String), + NotAuthorized, + TemplateNotFound, + ServerError, +} + +impl ResponseError for EditTtsModelError { + fn status_code(&self) -> StatusCode { + match *self { + EditTtsModelError::BadInput(_) => StatusCode::BAD_REQUEST, + EditTtsModelError::NotAuthorized => StatusCode::UNAUTHORIZED, + EditTtsModelError::TemplateNotFound => StatusCode::NOT_FOUND, + EditTtsModelError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + EditTtsModelError::BadInput(reason) => reason.to_string(), + EditTtsModelError::NotAuthorized=> "unauthorized".to_string(), + EditTtsModelError::TemplateNotFound => "not found".to_string(), + EditTtsModelError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn edit_tts_model_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EditTtsModelError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(EditTtsModelError::NotAuthorized); + } + }; + + // NB: First permission check. + // Only mods should see deleted models (both user_* and mod_* deleted). + let is_mod_that_can_see_deleted = user_session.can_delete_other_users_tts_models; + + let model_lookup_result = select_tts_model_by_token( + &path.model_token, + is_mod_that_can_see_deleted, + &server_state.mysql_pool).await; + + let model_record = match model_lookup_result { + Ok(Some(result)) => { + info!("Found model: {}", result.model_token); + result + }, + Ok(None) => { + warn!("could not find model"); + return Err(EditTtsModelError::TemplateNotFound); + }, + Err(err) => { + warn!("error looking up model: {:?}", err); + return Err(EditTtsModelError::TemplateNotFound); + }, + }; + + // NB: Second set of permission checks + let mut editor_is_original_user = false; + let mut editor_is_moderator = false; + + if model_record.creator_user_token == user_session.user_token { + editor_is_original_user = true; + } + + if user_session.can_edit_other_users_tts_models { + editor_is_moderator = true; + } + + if !editor_is_original_user && !editor_is_moderator { + return Err(EditTtsModelError::NotAuthorized); + } + + if !editor_is_moderator { + if model_record.is_locked_from_user_modification || model_record.is_mod_disabled { + return Err(EditTtsModelError::NotAuthorized); + } + } + + // Fields to set + let mut title = None; + let mut description_markdown = None; + let mut description_html = None; + + if let Some(payload) = request.title.as_deref() { + if contains_slurs(payload) { + return Err(EditTtsModelError::BadInput("title contains slurs".to_string())); + } + + title = Some(payload.to_string()); + } + + if let Some(markdown) = request.description_markdown.as_deref() { + if contains_slurs(markdown) { + return Err(EditTtsModelError::BadInput("description contains slurs".to_string())); + } + + let markdown = markdown.trim().to_string(); + let html = markdown_to_html(&markdown); + + description_markdown = Some(markdown); + description_html = Some(html); + } + + let ip_address = get_request_ip(&http_request); + + let query_result = if editor_is_original_user { + // We need to store the IP address details. + sqlx::query!( + r#" +UPDATE tts_models +SET + title = ?, + description_markdown = ?, + description_rendered_html = ?, + creator_ip_address_last_update = ?, + version = version + 1 + +WHERE tts_models.token = ? +LIMIT 1 + "#, + &title, + &description_markdown, + &description_html, + &ip_address, + &model_record.model_token, + ) + .execute(&server_state.mysql_pool) + .await + } else { + // We need to store the moderator details. + sqlx::query!( + r#" +UPDATE tts_models +SET + title = ?, + description_markdown = ?, + description_rendered_html = ?, + maybe_mod_user_token = ?, + version = version + 1 + +WHERE tts_models.token = ? +LIMIT 1 + "#, + &title, + &description_markdown, + &description_html, + &user_session.user_token, + &model_record.model_token, + ) + .execute(&server_state.mysql_pool) + .await + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update W2L model edit DB error: {:?}", err); + return Err(EditTtsModelError::ServerError); + } + }; + + Ok(simple_json_success()) +} diff --git a/src/http_server/endpoints/tts/mod.rs b/src/http_server/endpoints/tts/mod.rs index bdff9c584d..20d7957abf 100644 --- a/src/http_server/endpoints/tts/mod.rs +++ b/src/http_server/endpoints/tts/mod.rs @@ -1,3 +1,4 @@ +pub mod edit_tts_model; pub mod enqueue_infer_tts; pub mod enqueue_upload_tts_model; pub mod get_tts_inference_job_status; diff --git a/src/main.rs b/src/main.rs index 9377b4dca1..c2e467aafe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -71,6 +71,7 @@ use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; use crate::http_server::endpoints::w2l::delete_w2l_template::delete_w2l_template_handler; use crate::http_server::endpoints::w2l::edit_w2l_template::edit_w2l_template_handler; +use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -293,6 +294,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_tts_model_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/model/{model_token}/edit") + .route(web::post().to(edit_tts_model_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/result/{token}") .route(web::get().to(get_tts_inference_result_handler)) @@ -333,7 +339,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) .service( - web::resource("/template/{template_token}/edit") + web::resource("/template/{template_token}/edit") .route(web::post().to(edit_w2l_template_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) From dfb552abfc62d57221e9333d23dc4db33bc1a23d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 00:38:22 -0400 Subject: [PATCH 0347/1042] prevent locked templates/models from being edited by non-mods --- .../endpoints/tts/edit_tts_model.rs | 19 ++++++------------- .../endpoints/w2l/delete_w2l_template.rs | 14 +++++++++++++- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/http_server/endpoints/tts/edit_tts_model.rs b/src/http_server/endpoints/tts/edit_tts_model.rs index 2b4a4d2be8..d4e69aa756 100644 --- a/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/src/http_server/endpoints/tts/edit_tts_model.rs @@ -126,22 +126,15 @@ pub async fn edit_tts_model_handler( }; // NB: Second set of permission checks - let mut editor_is_original_user = false; - let mut editor_is_moderator = false; + let is_author = model_record.creator_user_token == user_session.user_token; + let is_mod = user_session.can_edit_other_users_tts_models ; - if model_record.creator_user_token == user_session.user_token { - editor_is_original_user = true; - } - - if user_session.can_edit_other_users_tts_models { - editor_is_moderator = true; - } - - if !editor_is_original_user && !editor_is_moderator { + if !is_author && !is_mod { + warn!("user is not allowed to edit model: {}", user_session.user_token); return Err(EditTtsModelError::NotAuthorized); } - if !editor_is_moderator { + if !is_mod { if model_record.is_locked_from_user_modification || model_record.is_mod_disabled { return Err(EditTtsModelError::NotAuthorized); } @@ -174,7 +167,7 @@ pub async fn edit_tts_model_handler( let ip_address = get_request_ip(&http_request); - let query_result = if editor_is_original_user { + let query_result = if is_author { // We need to store the IP address details. sqlx::query!( r#" diff --git a/src/http_server/endpoints/w2l/delete_w2l_template.rs b/src/http_server/endpoints/w2l/delete_w2l_template.rs index 203dd96468..5656bcb0e8 100644 --- a/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -110,9 +110,13 @@ pub async fn delete_w2l_template_handler( } }; + // NB: First permission check. + // Only mods should see deleted models (both user_* and mod_* deleted). + let is_mod_that_can_see_deleted = user_session.can_delete_other_users_w2l_templates; + let template_query_result = select_w2l_template_by_token( &path.slug, - true, // Only mods can perform this action + is_mod_that_can_see_deleted, &server_state.mysql_pool, ).await; @@ -125,6 +129,7 @@ pub async fn delete_w2l_template_handler( Ok(Some(template)) => template, }; + // NB: Second set of permission checks let is_author = w2l_template.creator_user_token == user_session.user_token; let is_mod = user_session.can_delete_other_users_w2l_templates; @@ -133,6 +138,12 @@ pub async fn delete_w2l_template_handler( return Err(DeleteW2lTemplateError::NotAuthorized); } + if !is_mod { + if w2l_template.is_locked_from_user_modification || w2l_template.is_mod_disabled { + return Err(DeleteW2lTemplateError::NotAuthorized); + } + } + let ip_address = get_request_ip(&http_request); let query_result = if request.set_delete { @@ -151,6 +162,7 @@ pub async fn delete_w2l_template_handler( } } else { if is_author { + // NB: Technically only mods can see their own templates here user_undelete_template( &path.slug, &user_session.user_token, From 727c57ddaa054ec479db7766a591ae3046c4fb35 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 01:44:02 -0400 Subject: [PATCH 0348/1042] delete w2l results endpoint (not tested) --- .../up.sql | 2 + src/common_queries/mod.rs | 1 + src/common_queries/query_w2l_result.rs | 259 ++++++++++++++++++ .../endpoints/w2l/delete_w2l_result.rs | 252 +++++++++++++++++ .../endpoints/w2l/delete_w2l_template.rs | 38 +-- .../endpoints/w2l/get_w2l_result.rs | 210 +++----------- src/http_server/endpoints/w2l/mod.rs | 1 + src/main.rs | 6 + 8 files changed, 561 insertions(+), 208 deletions(-) create mode 100644 src/common_queries/query_w2l_result.rs create mode 100644 src/http_server/endpoints/w2l/delete_w2l_result.rs diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 2cd4cdd764..d85aa5d0d2 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -58,6 +58,7 @@ CREATE TABLE tts_results ( -- ========== MODERATION DETAILS ========== -- In this case, a moderator hides it from public. + -- It won't appear in lists on profiles or otherwise. is_mod_hidden_from_public BOOLEAN NOT NULL DEFAULT FALSE, -- If a moderator has comments. @@ -134,6 +135,7 @@ CREATE TABLE w2l_results ( -- ========== MODERATION DETAILS ========== -- In this case, a moderator hides it from public. + -- It won't appear in lists on profiles or otherwise. is_mod_hidden_from_public BOOLEAN NOT NULL DEFAULT FALSE, -- If a moderator has comments. diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index 1caba8130d..7ec3096913 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -6,5 +6,6 @@ pub mod list_w2l_inference_results; pub mod list_w2l_templates; pub mod query_tts_model; pub mod query_user_profile; +pub mod query_w2l_result; pub mod query_w2l_template; pub mod sessions; \ No newline at end of file diff --git a/src/common_queries/query_w2l_result.rs b/src/common_queries/query_w2l_result.rs new file mode 100644 index 0000000000..412b4d62ab --- /dev/null +++ b/src/common_queries/query_w2l_result.rs @@ -0,0 +1,259 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::database_helpers::boolean_converters::i8_to_bool; +use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +#[derive(Serialize)] +pub struct W2lResultRecordForResponse { + pub w2l_result_token: String, + pub maybe_w2l_template_token: Option, + pub maybe_tts_inference_result_token: Option, + + pub public_bucket_video_path: String, + + pub template_type: Option, + pub template_title: Option, + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub maybe_template_creator_user_token: Option, + pub maybe_template_creator_username: Option, + pub maybe_template_creator_display_name: Option, + pub maybe_template_creator_gravatar_hash: Option, + + pub file_size_bytes: u32, + pub frame_width: u32, + pub frame_height: u32, + pub duration_millis: u32, + + //pub is_mod_hidden_from_public: bool, // converted + //pub template_is_mod_approved: bool, // converted + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +#[derive(Serialize)] +pub struct W2lResultRecordRaw { + pub w2l_result_token: String, // from field `w2l_results.token` + + pub maybe_w2l_template_token: Option, + pub maybe_tts_inference_result_token: Option, + + pub public_bucket_video_path: String, + + pub template_type: Option, + pub template_title: Option, // from field `w2l_templates.title` + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub maybe_template_creator_user_token: Option, + pub maybe_template_creator_username: Option, + pub maybe_template_creator_display_name: Option, + pub maybe_template_creator_gravatar_hash: Option, + + pub file_size_bytes: i32, + pub frame_width: i32, + pub frame_height: i32, + pub duration_millis: i32, + + //pub is_mod_hidden_from_public: i8, // needs convert + //pub template_is_mod_approved: i8, // needs convert + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +pub async fn select_w2l_result_by_token( + w2l_result_token: &str, + can_see_deleted: bool, + mysql_pool: &MySqlPool +) -> AnyhowResult> { + + let maybe_record = if can_see_deleted { + select_including_deleted(w2l_result_token, mysql_pool).await + } else { + select_without_deleted(w2l_result_token, mysql_pool).await + }; + + let ir : W2lResultRecordRaw = match maybe_record { + Ok(inference_result) => inference_result, + Err(ref err) => { + match err { + RowNotFound => { + warn!("w2l result not found: {:?}", &err); + return Ok(None); + }, + _ => { + warn!("w2l result query error: {:?}", &err); + return Err(anyhow!("database error")); + } + } + } + }; + + let ir_for_response = W2lResultRecordForResponse { + w2l_result_token: ir.w2l_result_token.clone(), + maybe_w2l_template_token: ir.maybe_w2l_template_token.clone(), + maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), + + public_bucket_video_path: ir.public_bucket_video_path.clone(), + + template_type: ir.template_type.clone(), + template_title: ir.template_title.clone(), + + maybe_creator_user_token: ir.maybe_creator_user_token.clone(), + maybe_creator_username: ir.maybe_creator_username.clone(), + maybe_creator_display_name: ir.maybe_creator_display_name.clone(), + maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash.clone(), + + maybe_template_creator_user_token: ir.maybe_template_creator_user_token.clone(), + maybe_template_creator_username: ir.maybe_template_creator_username.clone(), + maybe_template_creator_display_name: ir.maybe_template_creator_display_name.clone(), + maybe_template_creator_gravatar_hash: ir.maybe_template_creator_gravatar_hash.clone(), + + //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, + //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, + + file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, + frame_width: if ir.frame_width > 0 { ir.frame_width as u32 } else { 0 }, + frame_height: if ir.frame_height > 0 { ir.frame_height as u32 } else { 0 }, + duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, + + created_at: ir.created_at.clone(), + updated_at: ir.updated_at.clone(), + user_deleted_at: ir.user_deleted_at.clone(), + mod_deleted_at: ir.mod_deleted_at.clone(), + }; + + Ok(Some(ir_for_response)) +} + +async fn select_including_deleted( + w2l_result_token: &str, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + W2lResultRecordRaw, + r#" +SELECT + w2l_results.token as w2l_result_token, + w2l_results.maybe_tts_inference_result_token, + + w2l_results.public_bucket_video_path, + + w2l_templates.token as maybe_w2l_template_token, + w2l_templates.template_type, + w2l_templates.title as template_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_gravatar_hash, + + template_users.token as maybe_template_creator_user_token, + template_users.username as maybe_template_creator_username, + template_users.display_name as maybe_template_creator_display_name, + template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash, + + w2l_results.file_size_bytes, + w2l_results.frame_width, + w2l_results.frame_height, + w2l_results.duration_millis, + w2l_results.created_at, + w2l_results.updated_at, + w2l_results.user_deleted_at, + w2l_results.mod_deleted_at + +FROM w2l_results +LEFT OUTER JOIN w2l_templates + ON w2l_results.maybe_w2l_template_token = w2l_templates.token +LEFT OUTER JOIN users + ON w2l_results.maybe_creator_user_token = users.token +LEFT OUTER JOIN users as template_users + ON w2l_templates.creator_user_token = template_users.token +WHERE + w2l_results.token = ? + "#, + w2l_result_token + ) + .fetch_one(mysql_pool) + .await // TODO: This will return error if it doesn't exist +} + +async fn select_without_deleted( + w2l_result_token: &str, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + W2lResultRecordRaw, + r#" +SELECT + w2l_results.token as w2l_result_token, + w2l_results.maybe_tts_inference_result_token, + + w2l_results.public_bucket_video_path, + + w2l_templates.token as maybe_w2l_template_token, + w2l_templates.template_type, + w2l_templates.title as template_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_gravatar_hash, + + template_users.token as maybe_template_creator_user_token, + template_users.username as maybe_template_creator_username, + template_users.display_name as maybe_template_creator_display_name, + template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash, + + w2l_results.file_size_bytes, + w2l_results.frame_width, + w2l_results.frame_height, + w2l_results.duration_millis, + w2l_results.created_at, + w2l_results.updated_at, + w2l_results.user_deleted_at, + w2l_results.mod_deleted_at + +FROM w2l_results +LEFT OUTER JOIN w2l_templates + ON w2l_results.maybe_w2l_template_token = w2l_templates.token +LEFT OUTER JOIN users + ON w2l_results.maybe_creator_user_token = users.token +LEFT OUTER JOIN users as template_users + ON w2l_templates.creator_user_token = template_users.token +WHERE + w2l_results.token = ? + AND w2l_results.user_deleted_at IS NULL + AND w2l_results.mod_deleted_at IS NULL + "#, + w2l_result_token + ) + .fetch_one(mysql_pool) + .await // TODO: This will return error if it doesn't exist +} diff --git a/src/http_server/endpoints/w2l/delete_w2l_result.rs b/src/http_server/endpoints/w2l/delete_w2l_result.rs new file mode 100644 index 0000000000..7fa7b9216d --- /dev/null +++ b/src/http_server/endpoints/w2l/delete_w2l_result.rs @@ -0,0 +1,252 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::query_w2l_result::select_w2l_result_by_token; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct DeleteW2lInferenceResultPathInfo { + token: String, +} + +#[derive(Deserialize)] +pub struct DeleteW2lInferenceResultRequest { + set_delete: bool, +} + +#[derive(Debug, Display)] +pub enum DeleteW2lInferenceResultError { + BadInput(String), + NotAuthorized, + NotFound, + ServerError, +} + +impl ResponseError for DeleteW2lInferenceResultError { + fn status_code(&self) -> StatusCode { + match *self { + DeleteW2lInferenceResultError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteW2lInferenceResultError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteW2lInferenceResultError::NotFound => StatusCode::NOT_FOUND, + DeleteW2lInferenceResultError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + DeleteW2lInferenceResultError::BadInput(reason) => reason.to_string(), + DeleteW2lInferenceResultError::NotAuthorized => "unauthorized".to_string(), + DeleteW2lInferenceResultError::NotFound => "not found".to_string(), + DeleteW2lInferenceResultError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn delete_w2l_inference_result_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data> +) -> Result { + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + DeleteW2lInferenceResultError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(DeleteW2lInferenceResultError::NotAuthorized); + } + }; + + // NB: First permission check. + // Only mods should see deleted models (both user_* and mod_* deleted). + let is_mod_that_can_see_deleted = user_session.can_delete_other_users_w2l_results; + + let inference_result_query_result = select_w2l_result_by_token( + &path.token, + is_mod_that_can_see_deleted, + &server_state.mysql_pool, + ).await; + + let w2l_inference_result = match inference_result_query_result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(DeleteW2lInferenceResultError::ServerError); + } + Ok(None) => return Err(DeleteW2lInferenceResultError::NotFound), + Ok(Some(inference_result)) => inference_result, + }; + + // NB: Second set of permission checks + let is_author = w2l_inference_result.maybe_creator_user_token + .as_deref() + .map(|creator_token| creator_token == &user_session.user_token) + .unwrap_or(false); + + let is_mod = user_session.can_delete_other_users_w2l_results; + + if !is_author && !is_mod { + warn!("user is not allowed to delete inference results: {}", user_session.user_token); + return Err(DeleteW2lInferenceResultError::NotAuthorized); + } + + // NB: I can't imagine we need to store this. + // let ip_address = get_request_ip(&http_request); + + let query_result = if request.set_delete { + if is_author { + user_delete_inference_result( + &path.token, + &server_state.mysql_pool + ).await + } else { + mod_delete_inference_result( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } + } else { + if is_author { + // NB: Technically only mods can see their own inference_results here + user_undelete_inference_result( + &path.token, + &server_state.mysql_pool + ).await + } else { + mod_undelete_inference_result( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update w2l mod approval status DB error: {:?}", err); + return Err(DeleteW2lInferenceResultError::ServerError); + } + }; + + Ok(simple_json_success()) +} + +async fn user_delete_inference_result( + inference_result_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + let _r = sqlx::query!( + r#" +UPDATE w2l_results +SET + user_deleted_at = CURRENT_TIMESTAMP +WHERE + token = ? +LIMIT 1 + "#, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +async fn mod_delete_inference_result( + inference_result_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + let _r = sqlx::query!( + r#" +UPDATE w2l_results +SET + mod_deleted_at = CURRENT_TIMESTAMP, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +async fn user_undelete_inference_result( + inference_result_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + + let _r = sqlx::query!( + r#" +UPDATE w2l_results +SET + user_deleted_at = NULL +WHERE + token = ? +LIMIT 1 + "#, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +async fn mod_undelete_inference_result( + inference_result_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + + let _r = sqlx::query!( + r#" +UPDATE w2l_results +SET + mod_deleted_at = NULL, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} diff --git a/src/http_server/endpoints/w2l/delete_w2l_template.rs b/src/http_server/endpoints/w2l/delete_w2l_template.rs index 5656bcb0e8..59068e50f4 100644 --- a/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -22,6 +22,8 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use sqlx::MySqlPool; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -34,17 +36,6 @@ pub struct DeleteW2lTemplateRequest { set_delete: bool, } -#[derive(Serialize)] -pub struct DeleteW2lTemplateSuccessResponse { - pub success: bool, -} - -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum DeleteW2lTemplateError { BadInput(String), @@ -71,19 +62,7 @@ impl ResponseError for DeleteW2lTemplateError { DeleteW2lTemplateError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } @@ -185,16 +164,7 @@ pub async fn delete_w2l_template_handler( } }; - let response = DeleteW2lTemplateSuccessResponse { - success: true, - }; - - let body = serde_json::to_string(&response) - .map_err(|e| DeleteW2lTemplateError::ServerError)?; - - Ok(HttpResponse::Ok() - .content_type("application/json") - .body(body)) + Ok(simple_json_success()) } async fn user_delete_template( diff --git a/src/http_server/endpoints/w2l/get_w2l_result.rs b/src/http_server/endpoints/w2l/get_w2l_result.rs index a7a1f85b5a..8010c59990 100644 --- a/src/http_server/endpoints/w2l/get_w2l_result.rs +++ b/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -8,6 +8,8 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::common_queries::query_w2l_result::W2lResultRecordForResponse; +use crate::common_queries::query_w2l_result::select_w2l_result_by_token; use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; @@ -25,6 +27,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -32,40 +35,6 @@ pub struct GetW2lResultPathInfo { token: String, } -#[derive(Serialize)] -pub struct W2lResultRecordForResponse { - pub w2l_result_token: String, - pub maybe_w2l_template_token: Option, - pub maybe_tts_inference_result_token: Option, - - pub public_bucket_video_path: String, - - pub template_type: Option, - pub template_title: Option, - - pub maybe_creator_user_token: Option, - pub maybe_creator_username: Option, - pub maybe_creator_display_name: Option, - pub maybe_creator_gravatar_hash: Option, - - pub maybe_template_creator_user_token: Option, - pub maybe_template_creator_username: Option, - pub maybe_template_creator_display_name: Option, - pub maybe_template_creator_gravatar_hash: Option, - - pub file_size_bytes: u32, - pub frame_width: u32, - pub frame_height: u32, - pub duration_millis: u32, - - //pub is_mod_hidden_from_public: bool, // converted - //pub template_is_mod_approved: bool, // converted - //pub maybe_mod_user_token: Option, - - pub created_at: DateTime, - pub updated_at: DateTime, -} - #[derive(Serialize)] pub struct GetW2lResultSuccessResponse { pub success: bool, @@ -81,174 +50,67 @@ pub struct ErrorResponse { #[derive(Debug, Display)] pub enum GetW2lResultError { ServerError, -} - -pub struct RawW2lResultRecord { - pub w2l_result_token: String, // from field `w2l_results.token` - - pub maybe_w2l_template_token: Option, - pub maybe_tts_inference_result_token: Option, - - pub public_bucket_video_path: String, - - pub template_type: Option, - pub template_title: Option, // from field `w2l_templates.title` - - pub maybe_creator_user_token: Option, - pub maybe_creator_username: Option, - pub maybe_creator_display_name: Option, - pub maybe_creator_gravatar_hash: Option, - - pub maybe_template_creator_user_token: Option, - pub maybe_template_creator_username: Option, - pub maybe_template_creator_display_name: Option, - pub maybe_template_creator_gravatar_hash: Option, - - pub file_size_bytes: i32, - pub frame_width: i32, - pub frame_height: i32, - pub duration_millis: i32, - - //pub is_mod_hidden_from_public: i8, // needs convert - //pub template_is_mod_approved: i8, // needs convert - //pub maybe_mod_user_token: Option, - - pub created_at: DateTime, - pub updated_at: DateTime, + NotFound, } impl ResponseError for GetW2lResultError { fn status_code(&self) -> StatusCode { match *self { - GetW2lResultError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + GetW2lResultError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + GetW2lResultError::NotFound => StatusCode::NOT_FOUND, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { GetW2lResultError::ServerError => "server error".to_string(), + GetW2lResultError::NotFound => "not founnd".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } pub async fn get_w2l_inference_result_handler( http_request: HttpRequest, path: Path, - server_state: web::Data>) -> Result -{ - // NB: Lookup failure is Err(RowNotFound). - // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_result = sqlx::query_as!( - RawW2lResultRecord, - r#" -SELECT - w2l_results.token as w2l_result_token, - w2l_results.maybe_tts_inference_result_token, + server_state: web::Data> +) -> Result { + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetW2lResultError::ServerError + })?; - w2l_results.public_bucket_video_path, + let mut show_deleted_results = false; - w2l_templates.token as maybe_w2l_template_token, - w2l_templates.template_type, - w2l_templates.title as template_title, + if let Some(user_session) = maybe_user_session { + // NB: Moderators can see deleted templates. + // Original creators cannot see them (unless they're moderators!) + show_deleted_results = user_session.can_delete_other_users_w2l_results; + } - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - users.email_gravatar_hash as maybe_creator_gravatar_hash, + let inference_result_query_result = select_w2l_result_by_token( + &path.token, + show_deleted_results, + &server_state.mysql_pool + ).await; - template_users.token as maybe_template_creator_user_token, - template_users.username as maybe_template_creator_username, - template_users.display_name as maybe_template_creator_display_name, - template_users.email_gravatar_hash as maybe_template_creator_gravatar_hash, - - w2l_results.file_size_bytes, - w2l_results.frame_width, - w2l_results.frame_height, - w2l_results.duration_millis, - w2l_results.created_at, - w2l_results.updated_at - -FROM w2l_results -LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token -LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token -LEFT OUTER JOIN users as template_users - ON w2l_templates.creator_user_token = template_users.token -WHERE - w2l_results.token = ? - AND w2l_results.user_deleted_at IS NULL - AND w2l_results.mod_deleted_at IS NULL - "#, - &path.token - ) - .fetch_one(&server_state.mysql_pool) - .await; // TODO: This will return error if it doesn't exist - - let ir : RawW2lResultRecord = match maybe_result { - Ok(result) => result, - Err(err) => { - match err { - RowNotFound => { - return Err(GetW2lResultError::ServerError); - }, - _ => { - warn!("w2l inference result query error: {:?}", err); - return Err(GetW2lResultError::ServerError); - } - } + let inference_result = match inference_result_query_result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(GetW2lResultError::ServerError); } - }; - - let result_for_response = W2lResultRecordForResponse { - w2l_result_token: ir.w2l_result_token.clone(), - maybe_w2l_template_token: ir.maybe_w2l_template_token.clone(), - maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), - - public_bucket_video_path: ir.public_bucket_video_path.clone(), - - template_type: ir.template_type.clone(), - template_title: ir.template_title.clone(), - - maybe_creator_user_token: ir.maybe_creator_user_token.clone(), - maybe_creator_username: ir.maybe_creator_username.clone(), - maybe_creator_display_name: ir.maybe_creator_display_name.clone(), - maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash.clone(), - - maybe_template_creator_user_token: ir.maybe_template_creator_user_token.clone(), - maybe_template_creator_username: ir.maybe_template_creator_username.clone(), - maybe_template_creator_display_name: ir.maybe_template_creator_display_name.clone(), - maybe_template_creator_gravatar_hash: ir.maybe_template_creator_gravatar_hash.clone(), - - //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, - //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, - - file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, - frame_width: if ir.frame_width > 0 { ir.frame_width as u32 } else { 0 }, - frame_height: if ir.frame_height > 0 { ir.frame_height as u32 } else { 0 }, - duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, - - created_at: ir.created_at.clone(), - updated_at: ir.updated_at.clone(), + Ok(None) => return Err(GetW2lResultError::NotFound), + Ok(Some(inference_result)) => inference_result, }; let response = GetW2lResultSuccessResponse { success: true, - result: result_for_response, + result: inference_result, }; let body = serde_json::to_string(&response) diff --git a/src/http_server/endpoints/w2l/mod.rs b/src/http_server/endpoints/w2l/mod.rs index 4f323aa6f4..2d7ed379de 100644 --- a/src/http_server/endpoints/w2l/mod.rs +++ b/src/http_server/endpoints/w2l/mod.rs @@ -1,3 +1,4 @@ +pub mod delete_w2l_result; pub mod delete_w2l_template; pub mod edit_w2l_template; pub mod enqueue_infer_w2l; diff --git a/src/main.rs b/src/main.rs index c2e467aafe..89fb41668b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -72,6 +72,7 @@ use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_t use crate::http_server::endpoints::w2l::delete_w2l_template::delete_w2l_template_handler; use crate::http_server::endpoints::w2l::edit_w2l_template::edit_w2l_template_handler; use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; +use crate::http_server::endpoints::w2l::delete_w2l_result::delete_w2l_inference_result_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -358,6 +359,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_w2l_inference_result_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/result/{token}/delete") + .route(web::get().to(delete_w2l_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/job/{token}") .route(web::get().to(get_w2l_inference_job_status_handler)) From 0a1aa0aeab13ce34136e64a5e40927ec307cea07 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 01:49:18 -0400 Subject: [PATCH 0349/1042] forbid private types in public --- src/http_server/endpoints/misc/enable_alpha.rs | 2 +- src/main.rs | 4 +++- src/tts_download_job.rs | 4 +++- src/tts_inference_job.rs | 4 +++- src/w2l_download_job.rs | 4 +++- src/w2l_inference_job.rs | 4 +++- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/http_server/endpoints/misc/enable_alpha.rs b/src/http_server/endpoints/misc/enable_alpha.rs index c2bbe54e5c..9f48d5134e 100644 --- a/src/http_server/endpoints/misc/enable_alpha.rs +++ b/src/http_server/endpoints/misc/enable_alpha.rs @@ -15,7 +15,7 @@ const STATUS_LINK : &'static str = "status"; #[derive(Deserialize)] -struct QueryFields { +pub struct QueryFields { enable: Option, } diff --git a/src/main.rs b/src/main.rs index 89fb41668b..b51f8214b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,6 @@ -#![deny(unused_must_use)] +#![forbid(private_in_public)] +#![forbid(unused_must_use)] + #![allow(dead_code)] #![allow(non_snake_case)] #![allow(unused_imports)] diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index cc5eaf6d2a..34560cb7ff 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -1,4 +1,6 @@ -#![deny(unused_must_use)] +#![forbid(private_in_public)] +#![forbid(unused_must_use)] + #![allow(dead_code)] #![allow(non_snake_case)] #![allow(unused_imports)] diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 5a955a831d..941a34f1a5 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -1,4 +1,6 @@ -#![deny(unused_must_use)] +#![forbid(private_in_public)] +#![forbid(unused_must_use)] + #![allow(dead_code)] #![allow(non_snake_case)] #![allow(unused_imports)] diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index eb8f82c001..2f01070f4f 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -1,4 +1,6 @@ -#![deny(unused_must_use)] +#![forbid(private_in_public)] +#![forbid(unused_must_use)] + #![allow(dead_code)] #![allow(non_snake_case)] #![allow(unused_imports)] diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 3adae92e18..4625fabd8c 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -1,4 +1,6 @@ -#![deny(unused_must_use)] +#![forbid(private_in_public)] +#![forbid(unused_must_use)] + #![allow(dead_code)] #![allow(non_snake_case)] #![allow(unused_imports)] From 1496090142f3f349533465b6b7ec5d1b1ef4f847 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 01:59:56 -0400 Subject: [PATCH 0350/1042] tts inference result delete endpoint (untested) --- src/common_queries/mod.rs | 1 + src/common_queries/query_tts_result.rs | 251 +++++++++++++++++ .../endpoints/tts/delete_tts_result.rs | 252 ++++++++++++++++++ src/http_server/endpoints/tts/mod.rs | 1 + src/main.rs | 6 + 5 files changed, 511 insertions(+) create mode 100644 src/common_queries/query_tts_result.rs create mode 100644 src/http_server/endpoints/tts/delete_tts_result.rs diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index 7ec3096913..0223f21e0d 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -5,6 +5,7 @@ pub mod list_tts_models; pub mod list_w2l_inference_results; pub mod list_w2l_templates; pub mod query_tts_model; +pub mod query_tts_result; pub mod query_user_profile; pub mod query_w2l_result; pub mod query_w2l_template; diff --git a/src/common_queries/query_tts_result.rs b/src/common_queries/query_tts_result.rs new file mode 100644 index 0000000000..76e6fa699b --- /dev/null +++ b/src/common_queries/query_tts_result.rs @@ -0,0 +1,251 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::database_helpers::boolean_converters::i8_to_bool; +use crate::database_helpers::boolean_converters::nullable_i8_to_optional_bool; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +#[derive(Serialize)] +pub struct TtsResultRecordForResponse { + pub tts_result_token: String, + pub inference_text: String, + + pub tts_model_token: String, + pub tts_model_title: Option, // TODO: Shouldn't be Option. + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub maybe_model_creator_user_token: Option, + pub maybe_model_creator_username: Option, + pub maybe_model_creator_display_name: Option, + pub maybe_model_creator_gravatar_hash: Option, + + pub public_bucket_wav_audio_path: String, + pub public_bucket_spectrogram_path: String, + + pub file_size_bytes: u32, + pub duration_millis: u32, + + //pub is_mod_hidden_from_public: bool, // converted + //pub model_is_mod_approved: bool, // converted + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +#[derive(Serialize)] +pub struct TtsResultRecordRaw { + pub tts_result_token: String, // from field `tts_results.token` + pub inference_text: String, + + pub tts_model_token: String, + pub tts_model_title: Option, + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub maybe_model_creator_user_token: Option, + pub maybe_model_creator_username: Option, + pub maybe_model_creator_display_name: Option, + pub maybe_model_creator_gravatar_hash: Option, + + pub public_bucket_wav_audio_path: String, + pub public_bucket_spectrogram_path: String, + + pub file_size_bytes: i32, + pub duration_millis: i32, + + //pub is_mod_hidden_from_public: i8, // needs convert + //pub model_is_mod_approved: i8, // needs convert + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +pub async fn select_tts_result_by_token( + tts_result_token: &str, + can_see_deleted: bool, + mysql_pool: &MySqlPool +) -> AnyhowResult> { + + let maybe_record = if can_see_deleted { + select_including_deleted(tts_result_token, mysql_pool).await + } else { + select_without_deleted(tts_result_token, mysql_pool).await + }; + + let ir : TtsResultRecordRaw = match maybe_record { + Ok(inference_result) => inference_result, + Err(ref err) => { + match err { + RowNotFound => { + warn!("tts result not found: {:?}", &err); + return Ok(None); + }, + _ => { + warn!("tts result query error: {:?}", &err); + return Err(anyhow!("database error")); + } + } + } + }; + + let ir_for_response = TtsResultRecordForResponse { + tts_result_token: ir.tts_result_token.clone(), + + inference_text: ir.inference_text.clone(), + + tts_model_token: ir.tts_model_token.clone(), + tts_model_title: ir.tts_model_title.clone(), + + maybe_creator_user_token: ir.maybe_creator_user_token.clone(), + maybe_creator_username: ir.maybe_creator_username.clone(), + maybe_creator_display_name: ir.maybe_creator_display_name.clone(), + maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash.clone(), + + maybe_model_creator_user_token: ir.maybe_model_creator_user_token.clone(), + maybe_model_creator_username: ir.maybe_model_creator_username.clone(), + maybe_model_creator_display_name: ir.maybe_model_creator_display_name.clone(), + maybe_model_creator_gravatar_hash: ir.maybe_model_creator_gravatar_hash.clone(), + + //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, + //model_is_mod_approved: if ir.model_is_mod_approved == 0 { false } else { true }, + + public_bucket_wav_audio_path: ir.public_bucket_wav_audio_path.clone(), + public_bucket_spectrogram_path: ir.public_bucket_spectrogram_path.clone(), + + file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, + duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, + + created_at: ir.created_at.clone(), + updated_at: ir.updated_at.clone(), + user_deleted_at: ir.user_deleted_at.clone(), + mod_deleted_at: ir.mod_deleted_at.clone(), + }; + + Ok(Some(ir_for_response)) +} + +async fn select_including_deleted( + tts_result_token: &str, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + TtsResultRecordRaw, + r#" +SELECT + tts_results.token as tts_result_token, + + tts_results.inference_text, + + tts_results.model_token as tts_model_token, + tts_models.title as tts_model_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_gravatar_hash, + + model_users.token as maybe_model_creator_user_token, + model_users.username as maybe_model_creator_username, + model_users.display_name as maybe_model_creator_display_name, + model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash, + + tts_results.public_bucket_wav_audio_path, + tts_results.public_bucket_spectrogram_path, + + tts_results.file_size_bytes, + tts_results.duration_millis, + tts_results.created_at, + tts_results.updated_at, + tts_results.user_deleted_at, + tts_results.mod_deleted_at + +FROM tts_results +LEFT OUTER JOIN tts_models + ON tts_results.model_token = tts_models.token +LEFT OUTER JOIN users + ON tts_results.maybe_creator_user_token = users.token +LEFT OUTER JOIN users as model_users + ON tts_models.creator_user_token = model_users.token +WHERE + tts_results.token = ? + "#, + tts_result_token + ) + .fetch_one(mysql_pool) + .await // TODO: This will return error if it doesn't exist +} + +async fn select_without_deleted( + tts_result_token: &str, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + TtsResultRecordRaw, + r#" +SELECT + tts_results.token as tts_result_token, + + tts_results.inference_text, + + tts_results.model_token as tts_model_token, + tts_models.title as tts_model_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_gravatar_hash, + + model_users.token as maybe_model_creator_user_token, + model_users.username as maybe_model_creator_username, + model_users.display_name as maybe_model_creator_display_name, + model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash, + + tts_results.public_bucket_wav_audio_path, + tts_results.public_bucket_spectrogram_path, + + tts_results.file_size_bytes, + tts_results.duration_millis, + tts_results.created_at, + tts_results.updated_at, + tts_results.user_deleted_at, + tts_results.mod_deleted_at + +FROM tts_results +LEFT OUTER JOIN tts_models + ON tts_results.model_token = tts_models.token +LEFT OUTER JOIN users + ON tts_results.maybe_creator_user_token = users.token +LEFT OUTER JOIN users as model_users + ON tts_models.creator_user_token = model_users.token +WHERE + tts_results.token = ? + AND tts_results.user_deleted_at IS NULL + AND tts_results.mod_deleted_at IS NULL + "#, + tts_result_token + ) + .fetch_one(mysql_pool) + .await // TODO: This will return error if it doesn't exist +} diff --git a/src/http_server/endpoints/tts/delete_tts_result.rs b/src/http_server/endpoints/tts/delete_tts_result.rs new file mode 100644 index 0000000000..3dc15cf2e7 --- /dev/null +++ b/src/http_server/endpoints/tts/delete_tts_result.rs @@ -0,0 +1,252 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::query_tts_result::select_tts_result_by_token; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct DeleteTtsInferenceResultPathInfo { + token: String, +} + +#[derive(Deserialize)] +pub struct DeleteTtsInferenceResultRequest { + set_delete: bool, +} + +#[derive(Debug, Display)] +pub enum DeleteTtsInferenceResultError { + BadInput(String), + NotAuthorized, + NotFound, + ServerError, +} + +impl ResponseError for DeleteTtsInferenceResultError { + fn status_code(&self) -> StatusCode { + match *self { + DeleteTtsInferenceResultError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteTtsInferenceResultError::NotAuthorized => StatusCode::UNAUTHORIZED, + DeleteTtsInferenceResultError::NotFound => StatusCode::NOT_FOUND, + DeleteTtsInferenceResultError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + DeleteTtsInferenceResultError::BadInput(reason) => reason.to_string(), + DeleteTtsInferenceResultError::NotAuthorized => "unauthorized".to_string(), + DeleteTtsInferenceResultError::NotFound => "not found".to_string(), + DeleteTtsInferenceResultError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn delete_tts_inference_result_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data> +) -> Result { + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + DeleteTtsInferenceResultError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(DeleteTtsInferenceResultError::NotAuthorized); + } + }; + + // NB: First permission check. + // Only mods should see deleted models (both user_* and mod_* deleted). + let is_mod_that_can_see_deleted = user_session.can_delete_other_users_tts_results; + + let inference_result_query_result = select_tts_result_by_token( + &path.token, + is_mod_that_can_see_deleted, + &server_state.mysql_pool, + ).await; + + let tts_inference_result = match inference_result_query_result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(DeleteTtsInferenceResultError::ServerError); + } + Ok(None) => return Err(DeleteTtsInferenceResultError::NotFound), + Ok(Some(inference_result)) => inference_result, + }; + + // NB: Second set of permission checks + let is_author = tts_inference_result.maybe_creator_user_token + .as_deref() + .map(|creator_token| creator_token == &user_session.user_token) + .unwrap_or(false); + + let is_mod = user_session.can_delete_other_users_tts_results; + + if !is_author && !is_mod { + warn!("user is not allowed to delete inference results: {}", user_session.user_token); + return Err(DeleteTtsInferenceResultError::NotAuthorized); + } + + // NB: I can't imagine we need to store this. + // let ip_address = get_request_ip(&http_request); + + let query_result = if request.set_delete { + if is_author { + user_delete_inference_result( + &path.token, + &server_state.mysql_pool + ).await + } else { + mod_delete_inference_result( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } + } else { + if is_author { + // NB: Technically only mods can see their own inference_results here + user_undelete_inference_result( + &path.token, + &server_state.mysql_pool + ).await + } else { + mod_undelete_inference_result( + &path.token, + &user_session.user_token, + &server_state.mysql_pool + ).await + } + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Update tts mod approval status DB error: {:?}", err); + return Err(DeleteTtsInferenceResultError::ServerError); + } + }; + + Ok(simple_json_success()) +} + +async fn user_delete_inference_result( + inference_result_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + let _r = sqlx::query!( + r#" +UPDATE tts_results +SET + user_deleted_at = CURRENT_TIMESTAMP +WHERE + token = ? +LIMIT 1 + "#, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +async fn mod_delete_inference_result( + inference_result_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + let _r = sqlx::query!( + r#" +UPDATE tts_results +SET + mod_deleted_at = CURRENT_TIMESTAMP, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +async fn user_undelete_inference_result( + inference_result_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + + let _r = sqlx::query!( + r#" +UPDATE tts_results +SET + user_deleted_at = NULL +WHERE + token = ? +LIMIT 1 + "#, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} + +async fn mod_undelete_inference_result( + inference_result_token: &str, + mod_user_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), sqlx::Error> { + + let _r = sqlx::query!( + r#" +UPDATE tts_results +SET + mod_deleted_at = NULL, + maybe_mod_user_token = ? +WHERE + token = ? +LIMIT 1 + "#, + mod_user_token, + inference_result_token, + ) + .execute(mysql_pool) + .await?; + Ok(()) +} diff --git a/src/http_server/endpoints/tts/mod.rs b/src/http_server/endpoints/tts/mod.rs index 20d7957abf..32a08e8399 100644 --- a/src/http_server/endpoints/tts/mod.rs +++ b/src/http_server/endpoints/tts/mod.rs @@ -1,3 +1,4 @@ +pub mod delete_tts_result; pub mod edit_tts_model; pub mod enqueue_infer_tts; pub mod enqueue_upload_tts_model; diff --git a/src/main.rs b/src/main.rs index b51f8214b3..cce3c9c59d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -75,6 +75,7 @@ use crate::http_server::endpoints::w2l::delete_w2l_template::delete_w2l_template use crate::http_server::endpoints::w2l::edit_w2l_template::edit_w2l_template_handler; use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; use crate::http_server::endpoints::w2l::delete_w2l_result::delete_w2l_inference_result_handler; +use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -307,6 +308,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(get_tts_inference_result_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/result/{token}/delete") + .route(web::get().to(delete_tts_inference_result_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/job/{token}") .route(web::get().to(get_tts_inference_job_status_handler)) From bdc6e6d1e93d44be5c7aa272192babd49d2ddefb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 02:04:36 -0400 Subject: [PATCH 0351/1042] cleanup get_tts_result endpoint to use shared queries --- .../endpoints/tts/get_tts_result.rs | 205 +++--------------- .../endpoints/w2l/get_w2l_result.rs | 2 +- 2 files changed, 34 insertions(+), 173 deletions(-) diff --git a/src/http_server/endpoints/tts/get_tts_result.rs b/src/http_server/endpoints/tts/get_tts_result.rs index 7727778d83..b762ed20cb 100644 --- a/src/http_server/endpoints/tts/get_tts_result.rs +++ b/src/http_server/endpoints/tts/get_tts_result.rs @@ -8,9 +8,9 @@ use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::common_queries::query_tts_result::TtsResultRecordForResponse; +use crate::common_queries::query_tts_result::select_tts_result_by_token; use crate::common_queries::sessions::create_session_for_user; -use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; -use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; @@ -25,6 +25,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -32,111 +33,33 @@ pub struct GetTtsResultPathInfo { token: String, } -#[derive(Serialize)] -pub struct TtsResultRecordForResponse { - pub tts_result_token: String, - pub inference_text: String, - - pub tts_model_token: String, - pub tts_model_title: Option, // TODO: Shouldn't be Option. - - pub maybe_creator_user_token: Option, - pub maybe_creator_username: Option, - pub maybe_creator_display_name: Option, - pub maybe_creator_gravatar_hash: Option, - - pub maybe_model_creator_user_token: Option, - pub maybe_model_creator_username: Option, - pub maybe_model_creator_display_name: Option, - pub maybe_model_creator_gravatar_hash: Option, - - pub public_bucket_wav_audio_path: String, - pub public_bucket_spectrogram_path: String, - - pub file_size_bytes: u32, - pub duration_millis: u32, - - //pub is_mod_hidden_from_public: bool, // converted - //pub model_is_mod_approved: bool, // converted - //pub maybe_mod_user_token: Option, - - pub created_at: DateTime, - pub updated_at: DateTime, -} - #[derive(Serialize)] pub struct GetTtsResultSuccessResponse { pub success: bool, pub result: TtsResultRecordForResponse, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum GetTtsResultError { ServerError, -} - -pub struct RawTtsResultRecord { - pub tts_result_token: String, // from field `tts_results.token` - pub inference_text: String, - - pub tts_model_token: String, - pub tts_model_title: Option, - - pub maybe_creator_user_token: Option, - pub maybe_creator_username: Option, - pub maybe_creator_display_name: Option, - pub maybe_creator_gravatar_hash: Option, - - pub maybe_model_creator_user_token: Option, - pub maybe_model_creator_username: Option, - pub maybe_model_creator_display_name: Option, - pub maybe_model_creator_gravatar_hash: Option, - - pub public_bucket_wav_audio_path: String, - pub public_bucket_spectrogram_path: String, - - pub file_size_bytes: i32, - pub duration_millis: i32, - - //pub is_mod_hidden_from_public: i8, // needs convert - //pub model_is_mod_approved: i8, // needs convert - //pub maybe_mod_user_token: Option, - - pub created_at: DateTime, - pub updated_at: DateTime, + NotFound, } impl ResponseError for GetTtsResultError { fn status_code(&self) -> StatusCode { match *self { GetTtsResultError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + GetTtsResultError::NotFound=> StatusCode::NOT_FOUND, } } fn error_response(&self) -> HttpResponse { let error_reason = match self { GetTtsResultError::ServerError => "server error".to_string(), + GetTtsResultError::NotFound => "not found".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } @@ -145,103 +68,41 @@ pub async fn get_tts_inference_result_handler( path: Path, server_state: web::Data>) -> Result { - // NB: Lookup failure is Err(RowNotFound). - // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_result = sqlx::query_as!( - RawTtsResultRecord, - r#" -SELECT - tts_results.token as tts_result_token, + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetTtsResultError::ServerError + })?; - tts_results.inference_text, + let mut show_deleted_results = false; - tts_results.model_token as tts_model_token, - tts_models.title as tts_model_title, + if let Some(user_session) = maybe_user_session { + // NB: Moderators can see deleted results. + // Original creators cannot see them (unless they're moderators!) + show_deleted_results = user_session.can_delete_other_users_tts_results; + } - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - users.email_gravatar_hash as maybe_creator_gravatar_hash, + let inference_result_query_result = select_tts_result_by_token( + &path.token, + show_deleted_results, + &server_state.mysql_pool + ).await; - model_users.token as maybe_model_creator_user_token, - model_users.username as maybe_model_creator_username, - model_users.display_name as maybe_model_creator_display_name, - model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash, - - tts_results.public_bucket_wav_audio_path, - tts_results.public_bucket_spectrogram_path, - - tts_results.file_size_bytes, - tts_results.duration_millis, - tts_results.created_at, - tts_results.updated_at - -FROM tts_results -LEFT OUTER JOIN tts_models - ON tts_results.model_token = tts_models.token -LEFT OUTER JOIN users - ON tts_results.maybe_creator_user_token = users.token -LEFT OUTER JOIN users as model_users - ON tts_models.creator_user_token = model_users.token -WHERE - tts_results.token = ? - AND tts_results.user_deleted_at IS NULL - AND tts_results.mod_deleted_at IS NULL - "#, - &path.token - ) - .fetch_one(&server_state.mysql_pool) - .await; // TODO: This will return error if it doesn't exist - - let ir : RawTtsResultRecord = match maybe_result { - Ok(result) => result, - Err(err) => { - match err { - RowNotFound => { - return Err(GetTtsResultError::ServerError); - }, - _ => { - warn!("tts inference result query error: {:?}", err); - return Err(GetTtsResultError::ServerError); - } - } + let inference_result = match inference_result_query_result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(GetTtsResultError::ServerError); } - }; - - let result_for_response = TtsResultRecordForResponse { - tts_result_token: ir.tts_result_token.clone(), - - inference_text: ir.inference_text.clone(), - - tts_model_token: ir.tts_model_token.clone(), - tts_model_title: ir.tts_model_title.clone(), - - maybe_creator_user_token: ir.maybe_creator_user_token.clone(), - maybe_creator_username: ir.maybe_creator_username.clone(), - maybe_creator_display_name: ir.maybe_creator_display_name.clone(), - maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash.clone(), - - maybe_model_creator_user_token: ir.maybe_model_creator_user_token.clone(), - maybe_model_creator_username: ir.maybe_model_creator_username.clone(), - maybe_model_creator_display_name: ir.maybe_model_creator_display_name.clone(), - maybe_model_creator_gravatar_hash: ir.maybe_model_creator_gravatar_hash.clone(), - - //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, - //model_is_mod_approved: if ir.model_is_mod_approved == 0 { false } else { true }, - - public_bucket_wav_audio_path: ir.public_bucket_wav_audio_path.clone(), - public_bucket_spectrogram_path: ir.public_bucket_spectrogram_path.clone(), - - file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, - duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, - - created_at: ir.created_at.clone(), - updated_at: ir.updated_at.clone(), + Ok(None) => return Err(GetTtsResultError::NotFound), + Ok(Some(inference_result)) => inference_result, }; let response = GetTtsResultSuccessResponse { success: true, - result: result_for_response, + result: inference_result, }; let body = serde_json::to_string(&response) diff --git a/src/http_server/endpoints/w2l/get_w2l_result.rs b/src/http_server/endpoints/w2l/get_w2l_result.rs index 8010c59990..94a2b6d96f 100644 --- a/src/http_server/endpoints/w2l/get_w2l_result.rs +++ b/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -88,7 +88,7 @@ pub async fn get_w2l_inference_result_handler( let mut show_deleted_results = false; if let Some(user_session) = maybe_user_session { - // NB: Moderators can see deleted templates. + // NB: Moderators can see deleted results. // Original creators cannot see them (unless they're moderators!) show_deleted_results = user_session.can_delete_other_users_w2l_results; } From 9587d2a5485576c3846315d42556faa01da0e960 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 03:00:23 -0400 Subject: [PATCH 0352/1042] major tts/w2l schema changes (won't compile / broken code) --- migrations/2021-05-15-071601_assets/up.sql | 120 +++++++++++++-------- 1 file changed, 73 insertions(+), 47 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 7dd0a03369..7df9218e2b 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -21,23 +21,23 @@ CREATE TABLE tts_models ( -- NB: DO NOT CHANGE ORDER; APPEND ONLY! -- How text should be handled. text_preprocessing_algorithm ENUM( - 'basic-preprocess', - 'arpabet-preprocess' - ) NOT NULL DEFAULT 'basic-preprocess', + 'basic', + 'arpabet' + ) NOT NULL DEFAULT 'basic', -- Optional Pointer to a newer version of the voice -- If there's a newer version, we can disable this one. - maybe_updated_model_token VARCHAR(32) DEFAULT NULL, + -- maybe_updated_model_token VARCHAR(32) DEFAULT NULL, -- We can set as a combination of ['username' + 'voice-name'] -- There can be public aliases for voices, eg. a voice's default model. -- A user can change this. -- As such, these should not be foreign keys. - updatable_slug VARCHAR(64) NOT NULL, + -- updatable_slug VARCHAR(64) NOT NULL, -- Can be linked to a well-known voice - maybe_subject_token VARCHAR(32) DEFAULT NULL, - maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, + -- maybe_subject_token VARCHAR(32) DEFAULT NULL, + -- maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, -- ========== UNSTRUCTURED METADATA ========== @@ -45,20 +45,35 @@ CREATE TABLE tts_models ( -- If maybe_subject_token (etc.) is set, then it's authoritative instead. title VARCHAR(255) NOT NULL, - -- If the voice is "happy" or a singer "a-capella", etc. - -- This is used with "maybe_subject_token" etc. - maybe_voice_characteristic VARCHAR(255) DEFAULT NULL, - - -- Eg. ("poor quality", "trained longer", etc.) - -- This is used with "maybe_subject_token" etc. - maybe_extra_model_disambiguation VARCHAR(255) DEFAULT NULL, - -- The description of the model in markdown. description_markdown TEXT NOT NULL, -- Generated HTML (not user-editable). description_rendered_html TEXT NOT NULL, + -- eg. Bill Clinton, Batman, or Goku. + voice_name VARCHAR(255) DEFAULT NULL, + + -- Optional; eg. Sean Schemmel + actor_name VARCHAR(255) DEFAULT NULL, + + -- Optional; eg. if the voice is "happy" or a singer "a-capella", etc. + characteristic VARCHAR(255) DEFAULT NULL, + + -- Optional; eg. if you train the model more than once. + version_string VARCHAR(255) DEFAULT NULL, + + -- ========== IMMUTABLE PROVENANCE AND METADATA ========== + + -- Where the file was originally downloaded (if it was downloaded) + original_download_url VARCHAR(512) DEFAULT NULL, + + -- The filename that was used at upload time (if available) + original_filename VARCHAR(255) DEFAULT NULL, + + -- File characteristics + file_size_bytes INT(10) NOT NULL DEFAULT 0, + -- ========== CREATOR DETAILS ========== -- The person that created the template. @@ -78,14 +93,7 @@ CREATE TABLE tts_models ( 'private' ) NOT NULL DEFAULT 'public', - -- Where the file was originally downloaded (if it was downloaded) - original_download_url VARCHAR(512) DEFAULT NULL, - - -- The filename that was used at upload time (if available) - original_filename VARCHAR(255) DEFAULT NULL, - - -- File characteristics - file_size_bytes INT(10) NOT NULL DEFAULT 0, + -- ========== BUCKET STORAGE ========== -- The pytorch model -- For now, this will be a hash of the file contents. @@ -94,16 +102,25 @@ CREATE TABLE tts_models ( -- The "full url" version of the path private_bucket_object_name VARCHAR(255) NOT NULL, + -- ========== RATING AND RANKING ========== + -- Calculated average, on a scale of 0-100 -- Null with zero ratings. - calculated_average_score INT(3) DEFAULT NULL, + -- calculated_average_score INT(3) DEFAULT NULL, -- Other metrics - calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0, - calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0, + -- calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0, + -- calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0, -- ========== MODERATION DETAILS ========== + -- Mods have to approve of tts models for them to show up in a public index. + -- Models can be used by the author (or other parties that know the URL) before + -- they're approved, but unapproved models won't show up in public indices. + -- This is independent of other visibility controls (is_mod_disabled, deleted_at, + -- and creator_set_visibility) + is_mod_public_listing_approved BOOLEAN DEFAULT NULL, + -- In this case, a moderator disables it. is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, @@ -112,6 +129,7 @@ CREATE TABLE tts_models ( -- If a moderator has comments. maybe_mod_comments VARCHAR(255) DEFAULT NULL, + -- The last moderator that made changes. maybe_mod_user_token VARCHAR(32) DEFAULT NULL, @@ -143,6 +161,7 @@ CREATE TABLE tts_models ( KEY index_creator_ip_address_last_update (creator_ip_address_last_update), KEY index_creator_set_visibility (creator_set_visibility), KEY index_private_bucket_hash (private_bucket_hash), + KEY index_is_mod_public_listing_approved (is_mod_public_listing_approved), KEY index_is_mod_disabled (is_mod_disabled) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -164,17 +183,17 @@ CREATE TABLE w2l_templates ( -- Optional Pointer to a newer version of the template -- If there's a newer version, we can disable this one. - maybe_updated_template_token VARCHAR(32) DEFAULT NULL, + -- maybe_updated_template_token VARCHAR(32) DEFAULT NULL, -- A combination of ['username' + 'template-name'] -- There can be public aliases for voices, eg. a voice's default model. -- A user can change this. -- As such, these should not be foreign keys. - updatable_slug VARCHAR(64) NOT NULL, + -- updatable_slug VARCHAR(64) NOT NULL, -- Can be linked to a well-known voice - maybe_subject_token VARCHAR(32) DEFAULT NULL, - maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, + -- maybe_subject_token VARCHAR(32) DEFAULT NULL, + -- maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, -- ========== UNSTRUCTURED METADATA ========== @@ -187,6 +206,23 @@ CREATE TABLE w2l_templates ( -- Generated HTML (not user-editable). description_rendered_html TEXT NOT NULL, + -- ========== IMMUTABLE PROVENANCE AND METADATA ========== + + -- Where the file was originally downloaded (if it was downloaded) + original_download_url VARCHAR(512) DEFAULT NULL, + + -- The filename that was used at upload time (if available) + original_filename CHAR(255) DEFAULT NULL, + + -- Dimensions and other characteristics + file_size_bytes INT(10) NOT NULL DEFAULT 0, + mime_type VARCHAR(32) NOT NULL DEFAULT '', + frame_width INT(10) NOT NULL DEFAULT 0, + frame_height INT(10) NOT NULL DEFAULT 0, + frame_count INT(10) NOT NULL DEFAULT 0, + fps FLOAT NOT NULL DEFAULT 0.0, + duration_millis INT(10) NOT NULL DEFAULT 0, + -- ========== CREATOR DETAILS ========== -- The person that created the template. @@ -206,20 +242,7 @@ CREATE TABLE w2l_templates ( 'private' ) NOT NULL DEFAULT 'public', - -- Where the file was originally downloaded (if it was downloaded) - original_download_url VARCHAR(512) DEFAULT NULL, - - -- The filename that was used at upload time (if available) - original_filename CHAR(255) DEFAULT NULL, - - -- Dimensions and other characteristics - file_size_bytes INT(10) NOT NULL DEFAULT 0, - mime_type VARCHAR(32) NOT NULL DEFAULT '', - frame_width INT(10) NOT NULL DEFAULT 0, - frame_height INT(10) NOT NULL DEFAULT 0, - frame_count INT(10) NOT NULL DEFAULT 0, - fps FLOAT NOT NULL DEFAULT 0.0, - duration_millis INT(10) NOT NULL DEFAULT 0, + -- ========== BUCKET STORAGE ========== -- The hash of the original source image/video private_bucket_hash CHAR(64) NOT NULL, @@ -233,13 +256,15 @@ CREATE TABLE w2l_templates ( maybe_public_bucket_preview_image_object_name VARCHAR(255) DEFAULT NULL, maybe_public_bucket_preview_video_object_name VARCHAR(255) DEFAULT NULL, + -- ========== RATING AND RANKING ========== + -- Calculated average, on a scale of 0-100 -- Null with zero ratings. - calculated_average_score INT(3) DEFAULT NULL, + -- calculated_average_score INT(3) DEFAULT NULL, -- Other metrics - calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0, - calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0, + -- calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0, + -- calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0, -- ========== MODERATION DETAILS ========== @@ -293,6 +318,7 @@ CREATE TABLE w2l_templates ( KEY index_creator_ip_address_last_update (creator_ip_address_last_update), KEY index_creator_set_visibility (creator_set_visibility), KEY index_private_bucket_hash (private_bucket_hash), + KEY index_is_mod_public_listing_approved (is_mod_public_listing_approved), KEY index_is_mod_disabled (is_mod_disabled) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; From 12ebee4523263aa6223ca2910824600331eaa6d5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 03:12:22 -0400 Subject: [PATCH 0353/1042] changes to user (code breaking) --- migrations/2021-05-15-071534_users/up.sql | 56 ++++++++++++++++------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 36cdc55b65..6b67d6a15c 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -16,18 +16,11 @@ CREATE TABLE users ( email_address VARCHAR(255) NOT NULL, email_confirmed BOOLEAN NOT NULL DEFAULT false, - -- Gravatar image hashes are precomputed. - email_gravatar_hash CHAR(32) NOT NULL, - - -- The profile of the user in markdown (user editable). - profile_markdown TEXT NOT NULL, - - -- Generated HTML (not user-editable). - profile_rendered_html TEXT NOT NULL, - -- The role assigned to the user confers permissions. user_role_slug VARCHAR(16) NOT NULL, + -- ========== CREDENTIALS ========== + -- Bcrypt password hash. Granted, there are newer methods: -- https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html -- Passwords may be a max of 64 characters per BCrypt. @@ -36,9 +29,7 @@ CREATE TABLE users ( -- Incremented with every update to the password. password_version INT NOT NULL DEFAULT 0, - -- Different than deleted. - -- Users still show up, but can't do anything. - banned BOOLEAN NOT NULL DEFAULT false, + -- ========== ABUSE TRACKING ========== -- For abuse tracking. -- Wide enough for IPv4/6 @@ -46,14 +37,22 @@ CREATE TABLE users ( ip_address_last_login VARCHAR(40) NOT NULL, ip_address_last_update VARCHAR(40) NOT NULL, - -- For tracking stats. - -- The "cached" values are updated by a background job. - cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0, - cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0, + -- ========== DISPLAY / PROFILE ========== + + -- Gravatar image hashes are precomputed. + email_gravatar_hash CHAR(32) NOT NULL, + + -- The profile of the user in markdown (user editable). + profile_markdown TEXT NOT NULL, + + -- Generated HTML (not user-editable). + profile_rendered_html TEXT NOT NULL, -- An uploaded avatar. Public hash in our bucket. avatar_public_bucket_hash CHAR(32) DEFAULT NULL, + -- ========== USER PREFERENCES ========== + -- If the user doesn't want to use gravatar and doesn't have an uploaded avatar. disable_gravatar BOOLEAN NOT NULL DEFAULT false, @@ -80,6 +79,15 @@ CREATE TABLE users ( 'use-clock' ) NOT NULL DEFAULT 'light-mode', + -- ========== STATS ========== + + -- For tracking stats. + -- The "cached" values are updated by a background job. + -- cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0, + -- cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0, + + -- ========== SOCIAL MEDIA ========== + -- Social media usernames -- These are not confirmed. We'll need to build an OAuth system to handle that. discord_username VARCHAR(36) DEFAULT NULL, @@ -90,9 +98,25 @@ CREATE TABLE users ( cashapp_username VARCHAR(36) DEFAULT NULL, website_url VARCHAR(255) DEFAULT NULL, + -- ========== MODERATION DETAILS ========== + + -- Different than deleted. + -- Users still show up, but can't do anything. + is_banned BOOLEAN NOT NULL DEFAULT false, + + -- If a moderator has comments. + maybe_mod_comments VARCHAR(255) DEFAULT NULL, + + -- The last moderator that made changes. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + + -- ========== VECTOR CLOCK ========== + -- Incremented with every update. version INT NOT NULL DEFAULT 0, + -- ========== TIMESTAMPS ========== + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, From c2a6663e8703ab9c60288fae59718fc2d19c06c0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 03:20:46 -0400 Subject: [PATCH 0354/1042] more user model changes (code breaking changes) --- migrations/2021-05-15-071534_users/up.sql | 90 ++++++++++++++--------- 1 file changed, 54 insertions(+), 36 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 6b67d6a15c..557cc74dba 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -49,42 +49,7 @@ CREATE TABLE users ( profile_rendered_html TEXT NOT NULL, -- An uploaded avatar. Public hash in our bucket. - avatar_public_bucket_hash CHAR(32) DEFAULT NULL, - - -- ========== USER PREFERENCES ========== - - -- If the user doesn't want to use gravatar and doesn't have an uploaded avatar. - disable_gravatar BOOLEAN NOT NULL DEFAULT false, - - -- Hide results from others (ie. won't show up in the wall) - -- Moderators will still see them. - -- If the URLs are shared, they'll be visible. - hide_results_preference BOOLEAN NOT NULL DEFAULT false, - - -- Auto play preferences - auto_play_audio_preference BOOLEAN NOT NULL DEFAULT false, - auto_play_video_preference BOOLEAN NOT NULL DEFAULT false, - - -- Favorite TTS voice to use by default - maybe_preferred_tts_model_token VARCHAR(32) DEFAULT NULL, - - -- Favorite W2L model to use by default - maybe_preferred_w2l_template_token VARCHAR(32) DEFAULT NULL, - - -- Settings - -- DO NOT REORDER. - dark_mode_preference ENUM( - 'light-mode', - 'dark-mode', - 'use-clock' - ) NOT NULL DEFAULT 'light-mode', - - -- ========== STATS ========== - - -- For tracking stats. - -- The "cached" values are updated by a background job. - -- cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0, - -- cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0, + -- avatar_public_bucket_hash CHAR(32) DEFAULT NULL, -- ========== SOCIAL MEDIA ========== @@ -98,6 +63,59 @@ CREATE TABLE users ( cashapp_username VARCHAR(36) DEFAULT NULL, website_url VARCHAR(255) DEFAULT NULL, + -- ========== USER PREFERENCES ========== + + -- If the user doesn't want to use gravatar and doesn't have an uploaded avatar. + disable_gravatar BOOLEAN NOT NULL DEFAULT false, + + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + -- Visibility preference used at TTS inference time (which can be changed on the result itself). + -- Moderators will still see them. + preferred_tts_result_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- (THIS MIGHT NOT BE USED) + -- NB: DO NOT SORT! + -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE. + -- Visibility preference used at TTS inference time (which can be changed on the result itself). + -- Moderators will still see them. + preferred_w2l_result_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- Auto play preferences + -- Tri-state because we want to potentially change the default behavior. + auto_play_audio_preference BOOLEAN DEFAULT NULL, + auto_play_video_preference BOOLEAN DEFAULT NULL, + + -- Favorite TTS voice to use by default + -- maybe_preferred_tts_model_token VARCHAR(32) DEFAULT NULL, + + -- Favorite W2L model to use by default + -- maybe_preferred_w2l_template_token VARCHAR(32) DEFAULT NULL, + + -- Settings + -- DO NOT REORDER. + -- dark_mode_preference ENUM( + -- 'light-mode', + -- 'dark-mode', + -- 'use-clock' + -- ) NOT NULL DEFAULT 'light-mode', + + -- ========== STATS ========== + + -- For tracking stats. + -- The "cached" values are updated by a background job. + -- cached_tts_rendered_counter INT(10) NOT NULL DEFAULT 0, + -- cached_w2l_rendered_counter INT(10) NOT NULL DEFAULT 0, + -- ========== MODERATION DETAILS ========== -- Different than deleted. From 3b8c1e753145617b350b0ff34a5f80bf4ecacbd3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 03:46:46 -0400 Subject: [PATCH 0355/1042] more schema changes (code breaking) --- .../2021-05-15-212913_inference_jobs/up.sql | 12 +++++-- .../up.sql | 31 +++++++++++++++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/migrations/2021-05-15-212913_inference_jobs/up.sql b/migrations/2021-05-15-212913_inference_jobs/up.sql index d7fd4b163a..450595392a 100644 --- a/migrations/2021-05-15-212913_inference_jobs/up.sql +++ b/migrations/2021-05-15-212913_inference_jobs/up.sql @@ -26,13 +26,17 @@ CREATE TABLE tts_inference_jobs ( model_token VARCHAR(32) NOT NULL, -- The raw, unprocessed user input. - inference_text TEXT NOT NULL, + raw_inference_text TEXT NOT NULL, -- ========== CREATOR DETAILS ========== -- Foreign key to user -- If no user is logged in, this is null. - maybe_creator_user_token CHAR(16) DEFAULT NULL, + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, -- For abuse tracking. -- Wide enough for IPv4/6 @@ -147,6 +151,10 @@ CREATE TABLE w2l_inference_jobs ( -- If no user is logged in, this is null. maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, + -- For abuse tracking. -- Wide enough for IPv4/6 creator_ip_address VARCHAR(40) NOT NULL, diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index d85aa5d0d2..f5d6045263 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -15,10 +15,17 @@ CREATE TABLE tts_results ( model_token VARCHAR(32) NOT NULL, -- The original raw, unprocessed user input. - inference_text TEXT NOT NULL, + raw_inference_text TEXT NOT NULL, - -- SHA2 hash of the text [SHA2 = CHAR(64), SHA1 = CHAR(4)] - inference_text_hash_sha2 CHAR(64) NOT NULL, + -- SHA2 hash of the text [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)] + raw_inference_text_hash_sha2 CHAR(64) NOT NULL, + + -- First pass at text normalization. + -- eg. 14th -> fourteenth, etc. + normalized_inference_text TEXT NOT NULL, + + -- If the model uses arpabet, we'll save the arpabet we used. + maybe_arpabet_text TEXT DEFAULT NULL, -- ========== CREATOR DETAILS ========== @@ -26,6 +33,10 @@ CREATE TABLE tts_results ( -- If the user wasn't logged in, this is null maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, + -- For abuse tracking. -- Wide enough for IPv4/6 creator_ip_address VARCHAR(40) NOT NULL, @@ -39,6 +50,8 @@ CREATE TABLE tts_results ( 'private' ) NOT NULL DEFAULT 'public', + -- ========== BUCKET STORAGE ========== + -- SHA2. -- Where the wav, spectrogram, and etc. are located. public_bucket_hash CHAR(64) NOT NULL, @@ -67,6 +80,8 @@ CREATE TABLE tts_results ( -- The last moderator that made changes. maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + -- ========== TIMESTAMPS ========== + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -79,6 +94,7 @@ CREATE TABLE tts_results ( UNIQUE KEY (token), KEY fk_model_token (model_token), KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_maybe_creator_anonymous_visitor_token (maybe_creator_anonymous_visitor_token), KEY fk_maybe_mod_user_token (maybe_mod_user_token), KEY index_creator_ip_address (creator_ip_address), KEY index_is_mod_hidden_from_public(is_mod_hidden_from_public), @@ -108,6 +124,10 @@ CREATE TABLE w2l_results ( -- If the user wasn't logged in, this is null maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, + -- For abuse tracking. -- Wide enough for IPv4/6 creator_ip_address VARCHAR(40) NOT NULL, @@ -121,6 +141,8 @@ CREATE TABLE w2l_results ( 'private' ) NOT NULL DEFAULT 'public', + -- ========== BUCKET STORAGE ========== + -- Where the wav, spectrogram, and etc. are located. public_bucket_video_path VARCHAR(255) NOT NULL, @@ -144,6 +166,8 @@ CREATE TABLE w2l_results ( -- The last moderator that made changes. maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + -- ========== TIMESTAMPS ========== + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -155,6 +179,7 @@ CREATE TABLE w2l_results ( PRIMARY KEY (id), UNIQUE KEY (token), KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_maybe_creator_anonymous_visitor_token (maybe_creator_anonymous_visitor_token), KEY fk_maybe_w2l_template_token (maybe_w2l_template_token), KEY fk_maybe_tts_inference_result_token (maybe_tts_inference_result_token), KEY fk_maybe_mod_user_token (maybe_mod_user_token), From 2c6172eb88d2210d900eacab94011102f7848bf1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 03:54:00 -0400 Subject: [PATCH 0356/1042] don't need these --- migrations/2021-05-15-071545_asset_upload_jobs/up.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index 343326001a..4cc20e26c0 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -37,8 +37,8 @@ CREATE TABLE tts_model_upload_jobs ( ) NOT NULL DEFAULT 'not-set', -- Can be linked to a well-known voice - maybe_subject_token VARCHAR(32) DEFAULT NULL, - maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, + -- maybe_subject_token VARCHAR(32) DEFAULT NULL, + -- maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, -- If we need to download the file from Google Drive. download_url VARCHAR(512) DEFAULT NULL, @@ -149,8 +149,8 @@ CREATE TABLE w2l_template_upload_jobs ( ) NOT NULL DEFAULT 'unknown', -- Can be linked to a well-known subject - maybe_subject_token VARCHAR(32) DEFAULT NULL, - maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, + -- maybe_subject_token VARCHAR(32) DEFAULT NULL, + -- maybe_actor_subject_token VARCHAR(32) DEFAULT NULL, -- If we need to download the file from Google Drive. download_url VARCHAR(512) DEFAULT NULL, From 8eebffd7fe857da04f1fcdcaf41500219ade03ac Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 04:03:49 -0400 Subject: [PATCH 0357/1042] more schema changes --- .../2021-05-15-071545_asset_upload_jobs/up.sql | 16 ++++++++-------- migrations/2021-05-15-071601_assets/up.sql | 6 ++++-- migrations/2021-06-09-010200_ip_bans/up.sql | 8 ++++---- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index 4cc20e26c0..c4f5d98bd7 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -44,10 +44,10 @@ CREATE TABLE tts_model_upload_jobs ( download_url VARCHAR(512) DEFAULT NULL, -- NB: DO NOT SORT! - download_url_type ENUM( - 'google-drive', - 'web' - ) NOT NULL DEFAULT 'web', + -- download_url_type ENUM( + -- 'google-drive', + -- 'web' + -- ) NOT NULL DEFAULT 'web', -- ========== CREATOR DETAILS AND PREFERENCES ========== @@ -156,10 +156,10 @@ CREATE TABLE w2l_template_upload_jobs ( download_url VARCHAR(512) DEFAULT NULL, -- NB: DO NOT SORT! - download_url_type ENUM( - 'google-drive', - 'web' - ) NOT NULL DEFAULT 'web', + -- download_url_type ENUM( + -- 'google-drive', + -- 'web' + -- ) NOT NULL DEFAULT 'web', -- ========== CREATOR DETAILS AND PREFERENCES ========== diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index 7df9218e2b..c316a68662 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -121,7 +121,9 @@ CREATE TABLE tts_models ( -- and creator_set_visibility) is_mod_public_listing_approved BOOLEAN DEFAULT NULL, - -- In this case, a moderator disables it. + -- In this case, a moderator disables it from being used for inference. + -- This also disables it for the creator. + -- Unlike deletion, it remains "visible" to those that have access. is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, -- Extremely popular models may be locked from deletion or modification by users. @@ -275,7 +277,7 @@ CREATE TABLE w2l_templates ( -- and creator_set_visibility) is_mod_public_listing_approved BOOLEAN DEFAULT NULL, - -- In this case, a moderator disables it. + -- In this case, a moderator disables it from being used for inference. -- This also disables it for the creator. -- Unlike deletion, it remains "visible" to those that have access. is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, diff --git a/migrations/2021-06-09-010200_ip_bans/up.sql b/migrations/2021-06-09-010200_ip_bans/up.sql index 8b285e9914..ae200476df 100644 --- a/migrations/2021-06-09-010200_ip_bans/up.sql +++ b/migrations/2021-06-09-010200_ip_bans/up.sql @@ -9,14 +9,14 @@ CREATE TABLE ip_address_bans ( -- The IP address to ban ip_address VARCHAR(40) DEFAULT NULL, - -- Mod who created or last edited the ban. - mod_user_token VARCHAR(32) NOT NULL, - -- Maybe the target user. maybe_target_user_token VARCHAR(32) DEFAULT NULL, + -- Mod who created or last edited the ban. + mod_user_token VARCHAR(32) NOT NULL, + -- Optional notes about the ban. - notes TEXT DEFAULT NULL, + mod_notes TEXT DEFAULT NULL, -- Moderator can set an expire time (optional). -- If not set, the ban is indefinite From 728e7363f2501ea3beb25b38a543ec2eae28c92c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 04:06:52 -0400 Subject: [PATCH 0358/1042] get rid of indices against deleted columns --- migrations/2021-05-15-071545_asset_upload_jobs/up.sql | 4 ---- migrations/2021-05-15-071601_assets/up.sql | 8 -------- 2 files changed, 12 deletions(-) diff --git a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql index c4f5d98bd7..6807d24d83 100644 --- a/migrations/2021-05-15-071545_asset_upload_jobs/up.sql +++ b/migrations/2021-05-15-071545_asset_upload_jobs/up.sql @@ -110,8 +110,6 @@ CREATE TABLE tts_model_upload_jobs ( UNIQUE KEY (uuid_idempotency_token), KEY fk_on_success_result_token (on_success_result_token), KEY fk_creator_user_token (creator_user_token), - KEY fk_maybe_subject_token (maybe_subject_token), - KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), KEY index_status (status), KEY index_creator_ip_address (creator_ip_address) @@ -222,8 +220,6 @@ CREATE TABLE w2l_template_upload_jobs ( UNIQUE KEY (uuid_idempotency_token), KEY fk_on_success_result_token (on_success_result_token), KEY fk_creator_user_token (creator_user_token), - KEY fk_maybe_subject_token (maybe_subject_token), - KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), KEY index_status (status), KEY index_creator_ip_address (creator_ip_address) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index c316a68662..f414b95980 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -153,12 +153,8 @@ CREATE TABLE tts_models ( -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (token), - UNIQUE KEY (updatable_slug), - KEY fk_maybe_updated_model_token (maybe_updated_model_token), KEY fk_creator_user_token (creator_user_token), KEY fk_maybe_mod_user_token (maybe_mod_user_token), - KEY fk_maybe_subject_token (maybe_subject_token), - KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), KEY index_creator_ip_address_creation (creator_ip_address_creation), KEY index_creator_ip_address_last_update (creator_ip_address_last_update), KEY index_creator_set_visibility (creator_set_visibility), @@ -309,12 +305,8 @@ CREATE TABLE w2l_templates ( -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (token), - UNIQUE KEY (updatable_slug), - KEY fk_maybe_updated_template_token (maybe_updated_template_token), KEY fk_creator_user_token (creator_user_token), KEY fk_maybe_mod_user_token (maybe_mod_user_token), - KEY fk_maybe_subject_token (maybe_subject_token), - KEY fk_maybe_actor_subject_token (maybe_actor_subject_token), KEY index_template_type (template_type), KEY index_creator_ip_address_creation (creator_ip_address_creation), KEY index_creator_ip_address_last_update (creator_ip_address_last_update), From 8e633ee5c51d53381fbcf562329e8b54cb17364a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 04:59:42 -0400 Subject: [PATCH 0359/1042] propagate schema changes through to code (all binaries run) not tested --- .../list_tts_inference_results.rs | 14 +++--- src/common_queries/list_tts_models.rs | 7 --- src/common_queries/list_w2l_templates.rs | 7 --- src/common_queries/query_tts_model.rs | 5 --- src/common_queries/query_tts_result.rs | 10 ++--- src/common_queries/query_user_profile.rs | 21 +++------ src/common_queries/query_w2l_template.rs | 5 --- .../endpoints/tts/enqueue_infer_tts.rs | 2 +- .../endpoints/tts/enqueue_upload_tts_model.rs | 12 ----- .../endpoints/tts/get_tts_model.rs | 10 ++--- .../endpoints/tts/list_tts_models.rs | 4 -- .../endpoints/users/get_profile.rs | 27 ++++------- .../endpoints/w2l/delete_w2l_template.rs | 12 ++--- .../w2l/enqueue_upload_w2l_template.rs | 12 ----- .../endpoints/w2l/get_w2l_template.rs | 4 +- .../w2l/set_w2l_template_mod_approval.rs | 6 +-- src/http_server/web_utils/session_checker.rs | 45 ++++++------------- src/job_queries/tts_download_job_queries.rs | 6 --- src/job_queries/tts_inference_job_queries.rs | 16 ++++--- src/job_queries/w2l_download_job_queries.rs | 6 --- src/job_queries/w2l_inference_job_queries.rs | 2 - src/main.rs | 10 ++--- src/tts_download_job.rs | 2 +- src/tts_inference_job.rs | 6 +-- src/w2l_download_job.rs | 2 +- src/w2l_inference_job.rs | 2 +- 26 files changed, 73 insertions(+), 182 deletions(-) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index 73391cc518..cd96375b79 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -9,7 +9,7 @@ pub struct TtsInferenceRecordForList { pub tts_result_token: String, pub tts_model_token: String, - pub inference_text: String, + pub raw_inference_text: String, pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, @@ -30,7 +30,7 @@ struct RawTtsInferenceRecordForList { pub tts_result_token: String, // from field `tts_results.token` pub tts_model_token: String, - pub inference_text: String, + pub raw_inference_text: String, pub maybe_creator_user_token: Option, pub maybe_creator_username: Option, @@ -90,7 +90,7 @@ pub async fn list_tts_inference_results( tts_result_token: ir.tts_result_token.clone(), tts_model_token: ir.tts_model_token.clone(), - inference_text: ir.inference_text.clone(), + raw_inference_text: ir.raw_inference_text.clone(), maybe_creator_user_token: ir.maybe_creator_user_token.clone(), maybe_creator_username: ir.maybe_creator_username.clone(), @@ -123,7 +123,7 @@ SELECT tts_results.token as tts_result_token, tts_results.model_token as tts_model_token, - tts_results.inference_text as inference_text, + tts_results.raw_inference_text as raw_inference_text, users.token as maybe_creator_user_token, users.username as maybe_creator_username, @@ -156,7 +156,7 @@ SELECT tts_results.token as tts_result_token, tts_results.model_token as tts_model_token, - tts_results.inference_text as inference_text, + tts_results.raw_inference_text as raw_inference_text, users.token as maybe_creator_user_token, users.username as maybe_creator_username, @@ -199,7 +199,7 @@ SELECT tts_results.token as tts_result_token, tts_results.model_token as tts_model_token, - tts_results.inference_text as inference_text, + tts_results.raw_inference_text as raw_inference_text, users.token as maybe_creator_user_token, users.username as maybe_creator_username, @@ -234,7 +234,7 @@ SELECT tts_results.token as tts_result_token, tts_results.model_token as tts_model_token, - tts_results.inference_text as inference_text, + tts_results.raw_inference_text as raw_inference_text, users.token as maybe_creator_user_token, users.username as maybe_creator_username, diff --git a/src/common_queries/list_tts_models.rs b/src/common_queries/list_tts_models.rs index 29f38189ae..aae246302f 100644 --- a/src/common_queries/list_tts_models.rs +++ b/src/common_queries/list_tts_models.rs @@ -10,7 +10,6 @@ pub struct TtsModelRecordForList { pub tts_model_type: String, pub title: String, - pub updatable_slug: String, pub creator_user_token: String, pub creator_username: String, @@ -27,7 +26,6 @@ struct RawTtsModelRecordForList { pub tts_model_type: String, pub title: String, - pub updatable_slug: String, pub creator_user_token: String, pub creator_username: String, @@ -84,7 +82,6 @@ pub async fn list_tts_models( creator_user_token: model.creator_user_token.clone(), creator_username: model.creator_username.clone(), creator_display_name: model.creator_display_name.clone(), - updatable_slug: model.updatable_slug.clone(), title: model.title.clone(), is_mod_disabled: if model.is_mod_disabled == 0 { false } else { true }, created_at: model.created_at.clone(), @@ -111,7 +108,6 @@ SELECT tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - tts.updatable_slug, tts.title, tts.is_mod_disabled, tts.created_at, @@ -137,7 +133,6 @@ SELECT tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - tts.updatable_slug, tts.title, tts.is_mod_disabled, tts.created_at, @@ -174,7 +169,6 @@ SELECT tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - tts.updatable_slug, tts.title, tts.is_mod_disabled, tts.created_at, @@ -203,7 +197,6 @@ SELECT tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - tts.updatable_slug, tts.title, tts.is_mod_disabled, tts.created_at, diff --git a/src/common_queries/list_w2l_templates.rs b/src/common_queries/list_w2l_templates.rs index 0e20e95032..c3a2fdd781 100644 --- a/src/common_queries/list_w2l_templates.rs +++ b/src/common_queries/list_w2l_templates.rs @@ -12,7 +12,6 @@ pub struct W2lTemplateRecordForList { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, pub title: String, pub frame_width: u32, pub frame_height: u32, @@ -30,7 +29,6 @@ struct RawW2lTemplateRecordForList { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, pub title: String, pub frame_width: i32, pub frame_height: i32, @@ -87,7 +85,6 @@ pub async fn list_w2l_templates( creator_user_token: template.creator_user_token.clone(), creator_username: template.creator_username.clone(), creator_display_name: template.creator_display_name.clone(), - updatable_slug: template.updatable_slug.clone(), title: template.title.clone(), frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, @@ -119,7 +116,6 @@ SELECT w2l.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - w2l.updatable_slug, w2l.title, w2l.frame_width, w2l.frame_height, @@ -150,7 +146,6 @@ SELECT w2l.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - w2l.updatable_slug, w2l.title, w2l.frame_width, w2l.frame_height, @@ -192,7 +187,6 @@ SELECT w2l.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - w2l.updatable_slug, w2l.title, w2l.frame_width, w2l.frame_height, @@ -226,7 +220,6 @@ SELECT w2l.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - w2l.updatable_slug, w2l.title, w2l.frame_width, w2l.frame_height, diff --git a/src/common_queries/query_tts_model.rs b/src/common_queries/query_tts_model.rs index 8184d4be54..a37d5f348e 100644 --- a/src/common_queries/query_tts_model.rs +++ b/src/common_queries/query_tts_model.rs @@ -19,7 +19,6 @@ pub struct TtsModelRecordForResponse { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, pub title: String, pub description_markdown: String, pub description_rendered_html: String, @@ -38,7 +37,6 @@ pub struct TtsModelRecordRaw { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, pub title: String, pub description_markdown: String, pub description_rendered_html: String, @@ -84,7 +82,6 @@ pub async fn select_tts_model_by_token( creator_user_token: model.creator_user_token.clone(), creator_username: model.creator_username.clone(), creator_display_name: model.creator_display_name.clone(), - updatable_slug: model.updatable_slug.clone(), title: model.title.clone(), description_markdown: model.description_markdown.clone(), description_rendered_html: model.description_rendered_html.clone(), @@ -112,7 +109,6 @@ SELECT tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - tts.updatable_slug, tts.title, tts.description_markdown, tts.description_rendered_html, @@ -146,7 +142,6 @@ SELECT tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - tts.updatable_slug, tts.title, tts.description_markdown, tts.description_rendered_html, diff --git a/src/common_queries/query_tts_result.rs b/src/common_queries/query_tts_result.rs index 76e6fa699b..5279722b4a 100644 --- a/src/common_queries/query_tts_result.rs +++ b/src/common_queries/query_tts_result.rs @@ -15,7 +15,7 @@ use std::sync::Arc; #[derive(Serialize)] pub struct TtsResultRecordForResponse { pub tts_result_token: String, - pub inference_text: String, + pub raw_inference_text: String, pub tts_model_token: String, pub tts_model_title: Option, // TODO: Shouldn't be Option. @@ -50,7 +50,7 @@ pub struct TtsResultRecordForResponse { #[derive(Serialize)] pub struct TtsResultRecordRaw { pub tts_result_token: String, // from field `tts_results.token` - pub inference_text: String, + pub raw_inference_text: String, pub tts_model_token: String, pub tts_model_title: Option, @@ -113,7 +113,7 @@ pub async fn select_tts_result_by_token( let ir_for_response = TtsResultRecordForResponse { tts_result_token: ir.tts_result_token.clone(), - inference_text: ir.inference_text.clone(), + raw_inference_text: ir.raw_inference_text.clone(), tts_model_token: ir.tts_model_token.clone(), tts_model_title: ir.tts_model_title.clone(), @@ -156,7 +156,7 @@ async fn select_including_deleted( SELECT tts_results.token as tts_result_token, - tts_results.inference_text, + tts_results.raw_inference_text, tts_results.model_token as tts_model_token, tts_models.title as tts_model_title, @@ -207,7 +207,7 @@ async fn select_without_deleted( SELECT tts_results.token as tts_result_token, - tts_results.inference_text, + tts_results.raw_inference_text, tts_results.model_token as tts_model_token, tts_models.title as tts_model_title, diff --git a/src/common_queries/query_user_profile.rs b/src/common_queries/query_user_profile.rs index 3c17625e76..530b302158 100644 --- a/src/common_queries/query_user_profile.rs +++ b/src/common_queries/query_user_profile.rs @@ -10,6 +10,7 @@ use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use chrono::{DateTime, Utc}; use md5::{Md5, Digest}; +use crate::database_helpers::boolean_converters::i8_to_bool; // TODO: This duplicates the get_profile_handler. @@ -22,11 +23,8 @@ pub struct RawUserProfileRecord { pub profile_markdown: String, pub profile_rendered_html: String, pub user_role_slug: String, - pub banned: i8, - pub avatar_public_bucket_hash: Option, + pub is_banned: i8, pub disable_gravatar: i8, - pub maybe_preferred_tts_model_token: Option, - pub maybe_preferred_w2l_template_token: Option, pub discord_username: Option, pub twitch_username: Option, pub twitter_username: Option, @@ -48,11 +46,8 @@ pub struct UserProfileRecordForResponse { pub profile_markdown: String, pub profile_rendered_html: String, pub user_role_slug: String, - pub banned: bool, - pub avatar_public_bucket_hash: Option, + pub is_banned: bool, pub disable_gravatar: bool, - pub maybe_preferred_tts_model_token: Option, - pub maybe_preferred_w2l_template_token: Option, pub discord_username: Option, pub twitch_username: Option, pub twitter_username: Option, @@ -77,11 +72,8 @@ SELECT profile_markdown, profile_rendered_html, user_role_slug, - banned, - avatar_public_bucket_hash, + is_banned, disable_gravatar, - maybe_preferred_tts_model_token, - maybe_preferred_w2l_template_token, discord_username, twitch_username, twitter_username, @@ -123,11 +115,8 @@ WHERE profile_markdown: profile_record.profile_markdown.clone(), profile_rendered_html: profile_record.profile_rendered_html.clone(), user_role_slug: profile_record.user_role_slug.clone(), - banned: if profile_record.banned == 0 { false } else { true }, - avatar_public_bucket_hash: profile_record.avatar_public_bucket_hash.clone(), + is_banned: i8_to_bool(profile_record.is_banned), disable_gravatar: if profile_record.disable_gravatar == 0 { false } else { true }, - maybe_preferred_tts_model_token: profile_record.maybe_preferred_tts_model_token.clone(), - maybe_preferred_w2l_template_token: profile_record.maybe_preferred_w2l_template_token.clone(), discord_username: profile_record.discord_username.clone(), twitch_username: profile_record.twitch_username.clone(), twitter_username: profile_record.twitter_username.clone(), diff --git a/src/common_queries/query_w2l_template.rs b/src/common_queries/query_w2l_template.rs index 026548e5e1..faf4359f7a 100644 --- a/src/common_queries/query_w2l_template.rs +++ b/src/common_queries/query_w2l_template.rs @@ -19,7 +19,6 @@ pub struct W2lTemplateRecordForResponse { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, pub title: String, pub frame_width: u32, pub frame_height: u32, @@ -42,7 +41,6 @@ pub struct W2lTemplateRecordRaw { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, pub title: String, pub frame_width: i32, pub frame_height: i32, @@ -92,7 +90,6 @@ pub async fn select_w2l_template_by_token( creator_user_token: template.creator_user_token.clone(), creator_username: template.creator_username.clone(), creator_display_name: template.creator_display_name.clone(), - updatable_slug: template.updatable_slug.clone(), title: template.title.clone(), frame_width: if template.frame_width > 0 { template.frame_width as u32 } else { 0 }, frame_height: if template.frame_height > 0 { template.frame_height as u32 } else { 0 }, @@ -124,7 +121,6 @@ SELECT w2l.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - w2l.updatable_slug, w2l.title, w2l.frame_width, w2l.frame_height, @@ -162,7 +158,6 @@ SELECT w2l.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - w2l.updatable_slug, w2l.title, w2l.frame_width, w2l.frame_height, diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index 9cb0db89ea..d6133a91b8 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -128,7 +128,7 @@ SET uuid_idempotency_token = ?, model_token = ?, - inference_text = ?, + raw_inference_text = ?, maybe_creator_user_token = ?, creator_ip_address = ?, creator_set_visibility = ?, diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index b394795ddc..ec285851c3 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -26,11 +26,8 @@ pub struct UploadTtsModelRequest { idempotency_token: String, title: String, download_url: String, - download_url_type: Option, tts_model_type: Option, creator_set_visibility: Option, - maybe_subject_token: Option, - maybe_actor_subject_token: Option, } #[derive(Serialize)] @@ -122,10 +119,7 @@ pub async fn upload_tts_model_handler( let download_url = request.download_url.to_string(); let tts_model_type = "tacotron2".to_string(); - let download_url_type = "google-drive".to_string(); let creator_set_visibility = "public".to_string(); - let maybe_subject_token : Option = None; - let maybe_actor_subject_token : Option = None; // This token is returned to the client. let job_token = random_prefix_crockford_token("TTS_UP:", 32) @@ -146,10 +140,7 @@ SET creator_set_visibility = ?, title = ?, tts_model_type = ?, - maybe_subject_token = ?, - maybe_actor_subject_token = ?, download_url = ?, - download_url_type = ?, status = "pending" "#, job_token.to_string(), @@ -159,10 +150,7 @@ SET creator_set_visibility.to_string(), title.to_string(), tts_model_type, - maybe_subject_token, - maybe_actor_subject_token, download_url, - download_url_type ) .execute(&server_state.mysql_pool) .await; diff --git a/src/http_server/endpoints/tts/get_tts_model.rs b/src/http_server/endpoints/tts/get_tts_model.rs index 76fb2efa00..e79e23f5f1 100644 --- a/src/http_server/endpoints/tts/get_tts_model.rs +++ b/src/http_server/endpoints/tts/get_tts_model.rs @@ -29,7 +29,7 @@ use std::sync::Arc; /// For the URL PathInfo #[derive(Deserialize)] pub struct GetTtsModelPathInfo { - slug: String, + token: String, } #[derive(Serialize)] @@ -42,7 +42,6 @@ pub struct TtsModelRecordForResponse { pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, pub title: String, pub created_at: DateTime, @@ -76,7 +75,6 @@ pub struct TtsModelRecord { pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, pub title: String, pub created_at: DateTime, @@ -130,7 +128,6 @@ SELECT users.username as creator_username, users.display_name as creator_display_name, - tts.updatable_slug, tts.title, tts.created_at, @@ -139,11 +136,11 @@ FROM tts_models as tts JOIN users ON users.token = tts.creator_user_token WHERE - tts.updatable_slug = ? + tts.token = ? AND tts.user_deleted_at IS NULL AND tts.mod_deleted_at IS NULL "#, - &path.slug + &path.token ) .fetch_one(&server_state.mysql_pool) .await; // TODO: This will return error if it doesn't exist @@ -170,7 +167,6 @@ WHERE creator_user_token: template.creator_user_token.clone(), creator_username: template.creator_username.clone(), creator_display_name: template.creator_display_name.clone(), - updatable_slug: template.updatable_slug.clone(), title: template.title.clone(), created_at: template.created_at.clone(), updated_at: template.updated_at.clone(), diff --git a/src/http_server/endpoints/tts/list_tts_models.rs b/src/http_server/endpoints/tts/list_tts_models.rs index f8155e41e2..179840ca6b 100644 --- a/src/http_server/endpoints/tts/list_tts_models.rs +++ b/src/http_server/endpoints/tts/list_tts_models.rs @@ -32,7 +32,6 @@ pub struct TtsModelRecordForResponse { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, pub title: String, pub created_at: DateTime, pub updated_at: DateTime, @@ -62,7 +61,6 @@ pub struct TtsModelRecord { pub creator_user_token: String, pub creator_username: String, pub creator_display_name: String, - pub updatable_slug: String, pub title: String, pub created_at: DateTime, pub updated_at: DateTime, @@ -111,7 +109,6 @@ SELECT tts.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - tts.updatable_slug, tts.title, tts.created_at, tts.updated_at @@ -149,7 +146,6 @@ WHERE creator_user_token: model.creator_user_token.clone(), creator_username: model.creator_username.clone(), creator_display_name: model.creator_display_name.clone(), - updatable_slug: model.updatable_slug.clone(), title: model.title.clone(), created_at: model.created_at.clone(), updated_at: model.updated_at.clone(), diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index 1124cd02ab..5de60e7496 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -7,8 +7,10 @@ use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use anyhow::anyhow; +use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::common_queries::sessions::create_session_for_user; +use crate::database_helpers::boolean_converters::i8_to_bool; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; @@ -19,14 +21,13 @@ use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; +use md5::{Md5, Digest}; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use chrono::{DateTime, Utc}; -use md5::{Md5, Digest}; // TODO: This is duplicated in query_user_profile @@ -39,11 +40,8 @@ pub struct UserProfileRecord { pub profile_markdown: String, pub profile_rendered_html: String, pub user_role_slug: String, - pub banned: i8, - pub avatar_public_bucket_hash: Option, + pub is_banned: i8, pub disable_gravatar: i8, - pub maybe_preferred_tts_model_token: Option, - pub maybe_preferred_w2l_template_token: Option, pub discord_username: Option, pub twitch_username: Option, pub twitter_username: Option, @@ -67,11 +65,8 @@ pub struct UserProfileRecordForResponse { pub profile_markdown: String, pub profile_rendered_html: String, pub user_role_slug: String, - pub banned: bool, - pub avatar_public_bucket_hash: Option, + pub is_banned: bool, pub disable_gravatar: bool, - pub maybe_preferred_tts_model_token: Option, - pub maybe_preferred_w2l_template_token: Option, pub discord_username: Option, pub twitch_username: Option, pub twitter_username: Option, @@ -154,11 +149,8 @@ SELECT profile_markdown, profile_rendered_html, user_role_slug, - banned, - avatar_public_bucket_hash, + is_banned, disable_gravatar, - maybe_preferred_tts_model_token, - maybe_preferred_w2l_template_token, discord_username, twitch_username, twitter_username, @@ -202,11 +194,8 @@ WHERE profile_markdown: profile_record.profile_markdown.clone(), profile_rendered_html: profile_record.profile_rendered_html.clone(), user_role_slug: profile_record.user_role_slug.clone(), - banned: if profile_record.banned == 0 { false } else { true }, - avatar_public_bucket_hash: profile_record.avatar_public_bucket_hash.clone(), - disable_gravatar: if profile_record.disable_gravatar == 0 { false } else { true }, - maybe_preferred_tts_model_token: profile_record.maybe_preferred_tts_model_token.clone(), - maybe_preferred_w2l_template_token: profile_record.maybe_preferred_w2l_template_token.clone(), + is_banned: i8_to_bool(profile_record.is_banned), + disable_gravatar: i8_to_bool(profile_record.disable_gravatar), discord_username: profile_record.discord_username.clone(), twitch_username: profile_record.twitch_username.clone(), twitter_username: profile_record.twitter_username.clone(), diff --git a/src/http_server/endpoints/w2l/delete_w2l_template.rs b/src/http_server/endpoints/w2l/delete_w2l_template.rs index 59068e50f4..607f766351 100644 --- a/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -28,7 +28,7 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] pub struct DeleteW2lTemplatePathInfo { - slug: String, + token: String, } #[derive(Deserialize)] @@ -94,7 +94,7 @@ pub async fn delete_w2l_template_handler( let is_mod_that_can_see_deleted = user_session.can_delete_other_users_w2l_templates; let template_query_result = select_w2l_template_by_token( - &path.slug, + &path.token, is_mod_that_can_see_deleted, &server_state.mysql_pool, ).await; @@ -128,13 +128,13 @@ pub async fn delete_w2l_template_handler( let query_result = if request.set_delete { if is_author { user_delete_template( - &path.slug, + &path.token, &user_session.user_token, &server_state.mysql_pool ).await } else { mod_delete_template( - &path.slug, + &path.token, &user_session.user_token, &server_state.mysql_pool ).await @@ -143,13 +143,13 @@ pub async fn delete_w2l_template_handler( if is_author { // NB: Technically only mods can see their own templates here user_undelete_template( - &path.slug, + &path.token, &user_session.user_token, &server_state.mysql_pool ).await } else { mod_undelete_template( - &path.slug, + &path.token, &user_session.user_token, &server_state.mysql_pool ).await diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index c3cf0cd4ea..f81aa5b217 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -27,11 +27,8 @@ pub struct UploadW2lTemplateRequest { idempotency_token: String, title: String, download_url: String, - download_url_type: Option, template_type: Option, creator_set_visibility: Option, - maybe_subject_token: Option, - maybe_actor_subject_token: Option, } #[derive(Serialize)] @@ -127,10 +124,7 @@ pub async fn upload_w2l_template_handler( let download_url = request.download_url.to_string(); let template_type = "unknown".to_string(); - let download_url_type = "google-drive".to_string(); let creator_set_visibility = "public".to_string(); - let maybe_subject_token : Option = None; - let maybe_actor_subject_token : Option = None; // This token is returned to the client. let job_token = random_prefix_crockford_token("W2L_UP:", 32) @@ -151,10 +145,7 @@ SET creator_set_visibility = ?, title = ?, template_type = ?, - maybe_subject_token = ?, - maybe_actor_subject_token = ?, download_url = ?, - download_url_type = ?, status = "pending" "#, job_token.to_string(), @@ -164,10 +155,7 @@ SET creator_set_visibility.to_string(), title.to_string(), template_type, - maybe_subject_token, - maybe_actor_subject_token, download_url, - download_url_type ) .execute(&server_state.mysql_pool) .await; diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index 8cff2298fe..31d69441d1 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -29,7 +29,7 @@ use std::sync::Arc; /// For the URL PathInfo #[derive(Deserialize)] pub struct GetW2lTemplatePathInfo { - slug: String, + token: String, } #[derive(Serialize)] @@ -103,7 +103,7 @@ pub async fn get_w2l_template_handler( } let template_query_result = select_w2l_template_by_token( - &path.slug, + &path.token, show_deleted_templates, &server_state.mysql_pool ).await; diff --git a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs index 08aa71c0f6..b29acc1a90 100644 --- a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs +++ b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs @@ -25,7 +25,7 @@ use std::sync::Arc; /// For the URL PathInfo #[derive(Deserialize)] pub struct SetW2lTemplateModApprovalPathInfo { - slug: String, + token: String, } #[derive(Deserialize)] @@ -115,7 +115,7 @@ pub async fn set_w2l_template_mod_approval_handler( } let template_query_result = select_w2l_template_by_token( - &path.slug, + &path.token, true, // Only mods can perform this action &server_state.mysql_pool, ).await; @@ -144,7 +144,7 @@ LIMIT 1 &request.is_approved, &user_session.user_token, - &path.slug, + &path.token, ) .execute(&server_state.mysql_pool) .await; diff --git a/src/http_server/web_utils/session_checker.rs b/src/http_server/web_utils/session_checker.rs index 1de8b5f8ca..4d4f2e6a41 100644 --- a/src/http_server/web_utils/session_checker.rs +++ b/src/http_server/web_utils/session_checker.rs @@ -1,6 +1,7 @@ use actix_web::HttpRequest; use anyhow::anyhow; use crate::AnyhowResult; +use crate::database_helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool}; use crate::http_server::web_utils::cookie_manager::CookieManager; use log::{info, warn}; use sqlx::MySqlPool; @@ -24,22 +25,17 @@ pub struct SessionUserRecord { pub email_address: String, pub email_confirmed: bool, pub email_gravatar_hash: String, - pub avatar_public_bucket_hash: Option, // ===== PREFERENCES ===== // - pub dark_mode_preference: String, pub disable_gravatar: bool, - pub hide_results_preference: bool, - pub auto_play_audio_preference: bool, - pub auto_play_video_preference: bool, - pub maybe_preferred_tts_model_token: Option, - pub maybe_preferred_w2l_template_token: Option, + pub auto_play_audio_preference: Option, + pub auto_play_video_preference: Option, // ===== ROLE ===== // pub user_role_slug: String, - pub banned: bool, + pub is_banned: bool, // ===== PERMISSIONS FLAGS ===== // @@ -77,18 +73,13 @@ pub struct SessionUserRawDbRecord { pub email_address: String, pub email_confirmed: i8, pub email_gravatar_hash: String, - pub avatar_public_bucket_hash: Option, - pub dark_mode_preference: String, pub disable_gravatar: i8, - pub hide_results_preference: i8, - pub auto_play_audio_preference: i8, - pub auto_play_video_preference: i8, - pub maybe_preferred_tts_model_token: Option, - pub maybe_preferred_w2l_template_token: Option, + pub auto_play_audio_preference: Option, + pub auto_play_video_preference: Option, pub user_role_slug: String, - pub banned: i8, + pub is_banned: i8, // NB: These are `Option` due to the JOIN not being compile-time assured. // Usage @@ -189,18 +180,13 @@ SELECT users.email_address, users.email_confirmed, users.email_gravatar_hash, - users.avatar_public_bucket_hash, - users.dark_mode_preference, users.disable_gravatar, - users.hide_results_preference, users.auto_play_audio_preference, users.auto_play_video_preference, - users.maybe_preferred_tts_model_token, - users.maybe_preferred_w2l_template_token, users.user_role_slug, - users.banned, + users.is_banned, user_roles.can_use_tts, user_roles.can_use_w2l, @@ -246,18 +232,13 @@ WHERE user_sessions.token = ? username: raw_user_record.username.clone(), display_name: raw_user_record.display_name.clone(), email_address: raw_user_record.email_address.clone(), - email_confirmed: if raw_user_record.email_confirmed == 0 { false } else { true }, + email_confirmed: i8_to_bool(raw_user_record.email_confirmed), email_gravatar_hash: raw_user_record.email_gravatar_hash.clone(), - avatar_public_bucket_hash: raw_user_record.avatar_public_bucket_hash.clone(), - dark_mode_preference: raw_user_record.dark_mode_preference.clone(), - disable_gravatar: if raw_user_record.disable_gravatar == 0 { false } else { true }, - hide_results_preference: if raw_user_record.hide_results_preference == 0 { false } else { true }, - auto_play_audio_preference: if raw_user_record.auto_play_audio_preference == 0 { false } else { true }, - auto_play_video_preference: if raw_user_record.auto_play_video_preference == 0 { false } else { true }, - maybe_preferred_tts_model_token: raw_user_record.maybe_preferred_tts_model_token.clone(), - maybe_preferred_w2l_template_token: raw_user_record.maybe_preferred_w2l_template_token.clone(), + disable_gravatar: i8_to_bool(raw_user_record.disable_gravatar), + auto_play_audio_preference: nullable_i8_to_optional_bool(raw_user_record.auto_play_audio_preference), + auto_play_video_preference: nullable_i8_to_optional_bool(raw_user_record.auto_play_video_preference), user_role_slug: raw_user_record.user_role_slug.clone(), - banned: if raw_user_record.banned == 0 { false } else { true }, + is_banned: i8_to_bool(raw_user_record.is_banned), // Usage can_use_tts: convert_optional_db_bool_default_false(raw_user_record.can_use_tts), diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index 6a71e019bc..c36e63b8ea 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -22,10 +22,7 @@ pub struct TtsUploadJobRecord { pub creator_set_visibility: String, // TODO pub title: String, pub tts_model_type: String, // TODO - pub maybe_subject_token: Option, - pub maybe_actor_subject_token: Option, pub download_url: Option, - pub download_url_type: String, // TODO pub status: String, // TODO pub attempt_count: i32, pub failure_reason: Option, @@ -210,7 +207,6 @@ pub async fn insert_tts_model>( ) -> AnyhowResult<(u64, String)> { let model_token = random_prefix_crockford_token("TTS_MDL:", 32)?; - let updatable_slug = model_token.clone(); let private_bucket_object_name = &private_bucket_object_name .as_ref() @@ -223,7 +219,6 @@ INSERT INTO tts_models SET token = ?, tts_model_type = "tacotron2", - updatable_slug = ?, title = ?, description_markdown = '', description_rendered_html = '', @@ -235,7 +230,6 @@ SET file_size_bytes = ? "#, &model_token, - updatable_slug, job.title.to_string(), job.creator_user_token.clone(), job.creator_ip_address.clone(), diff --git a/src/job_queries/tts_inference_job_queries.rs b/src/job_queries/tts_inference_job_queries.rs index 621c9d5aae..b06bb8ca98 100644 --- a/src/job_queries/tts_inference_job_queries.rs +++ b/src/job_queries/tts_inference_job_queries.rs @@ -18,7 +18,7 @@ pub struct TtsInferenceJobRecord { pub uuid_idempotency_token: String, pub model_token: String, - pub inference_text: String, + pub raw_inference_text: String, pub creator_ip_address: String, pub maybe_creator_user_token: Option, @@ -46,7 +46,7 @@ SELECT uuid_idempotency_token, model_token, - inference_text, + raw_inference_text, creator_ip_address, maybe_creator_user_token, @@ -239,6 +239,8 @@ pub async fn insert_tts_result>( .display() .to_string(); + let normalized_inference_text = job.raw_inference_text.clone(); // TODO + let query_result = sqlx::query!( r#" INSERT INTO tts_results @@ -246,8 +248,9 @@ SET token = ?, model_token = ?, - inference_text = ?, - inference_text_hash_sha2 = ?, + raw_inference_text = ?, + raw_inference_text_hash_sha2 = ?, + normalized_inference_text = ?, maybe_creator_user_token = ?, creator_ip_address = ?, @@ -263,7 +266,8 @@ SET "#, inference_result_token, job.model_token.clone(), - job.inference_text.clone(), + job.raw_inference_text.clone(), + normalized_inference_text, text_hash, job.maybe_creator_user_token.clone(), @@ -300,7 +304,6 @@ pub struct TtsModelRecord2 { pub creator_display_name: String, pub title: String, - pub updatable_slug: String, pub private_bucket_hash: String, pub created_at: DateTime, pub updated_at: DateTime, @@ -323,7 +326,6 @@ SELECT users.username as creator_username, users.display_name as creator_display_name, tts.title, - tts.updatable_slug, tts.private_bucket_hash, tts.created_at, tts.updated_at diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 99e421e3d3..442ea0faba 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -21,10 +21,7 @@ pub struct W2lTemplateUploadJobRecord { pub creator_set_visibility: String, // TODO pub title: String, pub template_type: String, // TODO - pub maybe_subject_token: Option, - pub maybe_actor_subject_token: Option, pub download_url: Option, - pub download_url_type: String, // TODO pub status: String, // TODO pub attempt_count: i32, pub failure_reason: Option, @@ -218,7 +215,6 @@ pub async fn insert_w2l_template( ) -> AnyhowResult<(u64, String)> { let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?; - let updatable_slug = model_token.clone(); let query_result = sqlx::query!( r#" @@ -226,7 +222,6 @@ INSERT INTO w2l_templates SET token = ?, template_type = ?, - updatable_slug = ?, title = ?, description_markdown = '', description_rendered_html = '', @@ -248,7 +243,6 @@ SET "#, model_token, template_type, - updatable_slug, job.title.to_string(), job.creator_user_token.clone(), job.creator_ip_address.clone(), diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs index 42eec12078..d917bb270d 100644 --- a/src/job_queries/w2l_inference_job_queries.rs +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -313,7 +313,6 @@ pub struct W2lTemplateRecord2 { pub creator_username: String, pub private_bucket_hash: String, pub creator_display_name: String, - pub updatable_slug: String, pub title: String, pub frame_width: i32, pub frame_height: i32, @@ -339,7 +338,6 @@ SELECT w2l.creator_user_token, users.username as creator_username, users.display_name as creator_display_name, - w2l.updatable_slug, w2l.title, w2l.frame_width, w2l.frame_height, diff --git a/src/main.rs b/src/main.rs index cce3c9c59d..08a264c8a4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,12 +1,12 @@ #![forbid(private_in_public)] #![forbid(unused_must_use)] +//#![forbid(warnings)] #![allow(dead_code)] #![allow(non_snake_case)] #![allow(unused_imports)] #![allow(unused_mut)] #![allow(unused_variables)] -//#![allow(warnings)] #[macro_use] extern crate lazy_static; #[macro_use] extern crate serde_derive; @@ -294,7 +294,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) .service( - web::resource("/model/{slug}") + web::resource("/model/{token}") .route(web::get().to(get_tts_model_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) @@ -343,7 +343,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) .service( - web::resource("/template/{slug}") + web::resource("/template/{token}") .route(web::get().to(get_w2l_template_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) @@ -353,12 +353,12 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) .service( - web::resource("/template/{slug}/moderate") + web::resource("/template/{token}/moderate") .route(web::post().to(set_w2l_template_mod_approval_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) .service( - web::resource("/template/{slug}/delete") + web::resource("/template/{token}/delete") .route(web::post().to(delete_w2l_template_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 34560cb7ff..4776c6b04f 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -1,12 +1,12 @@ #![forbid(private_in_public)] #![forbid(unused_must_use)] +//#![forbid(warnings)] #![allow(dead_code)] #![allow(non_snake_case)] #![allow(unused_imports)] #![allow(unused_mut)] #![allow(unused_variables)] -//#![allow(warnings)] #[macro_use] extern crate serde_derive; diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 941a34f1a5..f0a3ab747f 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -1,12 +1,12 @@ #![forbid(private_in_public)] #![forbid(unused_must_use)] +//#![forbid(warnings)] #![allow(dead_code)] #![allow(non_snake_case)] #![allow(unused_imports)] #![allow(unused_mut)] #![allow(unused_variables)] -//#![allow(warnings)] #[macro_use] extern crate serde_derive; @@ -388,7 +388,7 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An let text_input_fs_path = temp_dir.path().join("inference_input.txt"); - std::fs::write(&text_input_fs_path, &job.inference_text)?; + std::fs::write(&text_input_fs_path, &job.raw_inference_text)?; // ==================== RUN INFERENCE ==================== // @@ -454,7 +454,7 @@ async fn process_job(inferencer: &Inferencer, job: &TtsInferenceJobRecord) -> An // ==================== SAVE RECORDS ==================== // - let text_hash = hash_string_sha2(&job.inference_text)?; + let text_hash = hash_string_sha2(&job.raw_inference_text)?; info!("Saving tts inference record..."); let (id, inference_result_token) = insert_tts_result( diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index 2f01070f4f..6d95c6c410 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -1,12 +1,12 @@ #![forbid(private_in_public)] #![forbid(unused_must_use)] +//#![forbid(warnings)] #![allow(dead_code)] #![allow(non_snake_case)] #![allow(unused_imports)] #![allow(unused_mut)] #![allow(unused_variables)] -//#![allow(warnings)] #[macro_use] extern crate serde_derive; diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 4625fabd8c..4bb423073b 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -1,12 +1,12 @@ #![forbid(private_in_public)] #![forbid(unused_must_use)] +//#![forbid(warnings)] #![allow(dead_code)] #![allow(non_snake_case)] #![allow(unused_imports)] #![allow(unused_mut)] #![allow(unused_variables)] -//#![allow(warnings)] #[macro_use] extern crate serde_derive; From 653fa93c8af100e3f12c1de471f5f1608afeb82b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 14:08:16 -0400 Subject: [PATCH 0360/1042] Don't need these colums / column renames --- migrations/2021-05-15-071601_assets/up.sql | 30 ++++++++++--------- .../up.sql | 10 ------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/migrations/2021-05-15-071601_assets/up.sql b/migrations/2021-05-15-071601_assets/up.sql index f414b95980..1fdaecb4c9 100644 --- a/migrations/2021-05-15-071601_assets/up.sql +++ b/migrations/2021-05-15-071601_assets/up.sql @@ -119,15 +119,15 @@ CREATE TABLE tts_models ( -- they're approved, but unapproved models won't show up in public indices. -- This is independent of other visibility controls (is_mod_disabled, deleted_at, -- and creator_set_visibility) - is_mod_public_listing_approved BOOLEAN DEFAULT NULL, + is_public_listing_approved BOOLEAN DEFAULT NULL, + + -- Extremely popular models may be locked from deletion or modification by users. + is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE, -- In this case, a moderator disables it from being used for inference. -- This also disables it for the creator. -- Unlike deletion, it remains "visible" to those that have access. - is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, - - -- Extremely popular models may be locked from deletion or modification by users. - is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE, + is_locked_from_use BOOLEAN NOT NULL DEFAULT FALSE, -- If a moderator has comments. maybe_mod_comments VARCHAR(255) DEFAULT NULL, @@ -159,8 +159,9 @@ CREATE TABLE tts_models ( KEY index_creator_ip_address_last_update (creator_ip_address_last_update), KEY index_creator_set_visibility (creator_set_visibility), KEY index_private_bucket_hash (private_bucket_hash), - KEY index_is_mod_public_listing_approved (is_mod_public_listing_approved), - KEY index_is_mod_disabled (is_mod_disabled) + KEY index_is_public_listing_approved (is_public_listing_approved), + KEY index_is_locked_from_user_modification (is_locked_from_user_modification), + KEY index_is_locked_from_use (is_locked_from_use) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -271,15 +272,15 @@ CREATE TABLE w2l_templates ( -- they're approved, but unapproved templates won't show up in public indices. -- This is independent of other visibility controls (is_mod_disabled, deleted_at, -- and creator_set_visibility) - is_mod_public_listing_approved BOOLEAN DEFAULT NULL, + is_public_listing_approved BOOLEAN DEFAULT NULL, + + -- Extremely popular models may be locked from deletion or modification by users. + is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE, -- In this case, a moderator disables it from being used for inference. -- This also disables it for the creator. -- Unlike deletion, it remains "visible" to those that have access. - is_mod_disabled BOOLEAN NOT NULL DEFAULT FALSE, - - -- Extremely popular models may be locked from deletion or modification by users. - is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE, + is_locked_from_use BOOLEAN NOT NULL DEFAULT FALSE, -- If a moderator has comments. maybe_mod_comments VARCHAR(255) DEFAULT NULL, @@ -312,7 +313,8 @@ CREATE TABLE w2l_templates ( KEY index_creator_ip_address_last_update (creator_ip_address_last_update), KEY index_creator_set_visibility (creator_set_visibility), KEY index_private_bucket_hash (private_bucket_hash), - KEY index_is_mod_public_listing_approved (is_mod_public_listing_approved), - KEY index_is_mod_disabled (is_mod_disabled) + KEY index_is_public_listing_approved (is_public_listing_approved), + KEY index_is_locked_from_user_modification (is_locked_from_user_modification), + KEY index_is_locked_from_use (is_locked_from_use) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index f5d6045263..8e3ed05ed8 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -70,10 +70,6 @@ CREATE TABLE tts_results ( -- ========== MODERATION DETAILS ========== - -- In this case, a moderator hides it from public. - -- It won't appear in lists on profiles or otherwise. - is_mod_hidden_from_public BOOLEAN NOT NULL DEFAULT FALSE, - -- If a moderator has comments. maybe_mod_comments VARCHAR(255) DEFAULT NULL, @@ -97,7 +93,6 @@ CREATE TABLE tts_results ( KEY fk_maybe_creator_anonymous_visitor_token (maybe_creator_anonymous_visitor_token), KEY fk_maybe_mod_user_token (maybe_mod_user_token), KEY index_creator_ip_address (creator_ip_address), - KEY index_is_mod_hidden_from_public(is_mod_hidden_from_public), KEY index_user_deleted_at(user_deleted_at), KEY index_mod_deleted_at(mod_deleted_at) @@ -156,10 +151,6 @@ CREATE TABLE w2l_results ( -- ========== MODERATION DETAILS ========== - -- In this case, a moderator hides it from public. - -- It won't appear in lists on profiles or otherwise. - is_mod_hidden_from_public BOOLEAN NOT NULL DEFAULT FALSE, - -- If a moderator has comments. maybe_mod_comments VARCHAR(255) DEFAULT NULL, @@ -184,7 +175,6 @@ CREATE TABLE w2l_results ( KEY fk_maybe_tts_inference_result_token (maybe_tts_inference_result_token), KEY fk_maybe_mod_user_token (maybe_mod_user_token), KEY index_creator_ip_address (creator_ip_address), - KEY index_is_mod_hidden_from_public(is_mod_hidden_from_public), KEY index_user_deleted_at(user_deleted_at), KEY index_mod_deleted_at(mod_deleted_at) From 812f43254cf70d445cc283295ad646faa9f1b80e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 14:18:14 -0400 Subject: [PATCH 0361/1042] fix code to deal with moderation field renames and deletes --- .../list_tts_inference_results.rs | 9 ++------- src/common_queries/list_tts_models.rs | 19 +++++++++--------- .../list_w2l_inference_results.rs | 9 ++------- src/common_queries/list_w2l_templates.rs | 18 ++++++++--------- src/common_queries/query_tts_model.rs | 10 +++++----- src/common_queries/query_tts_result.rs | 3 --- src/common_queries/query_w2l_result.rs | 3 --- src/common_queries/query_w2l_template.rs | 20 +++++++++---------- .../endpoints/tts/edit_tts_model.rs | 4 ++-- .../endpoints/w2l/delete_w2l_template.rs | 2 +- .../endpoints/w2l/edit_w2l_template.rs | 4 ++-- .../w2l/set_w2l_template_mod_approval.rs | 2 +- src/main.rs | 1 + 13 files changed, 45 insertions(+), 59 deletions(-) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index cd96375b79..7203058e95 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -18,7 +18,6 @@ pub struct TtsInferenceRecordForList { pub file_size_bytes: u32, pub duration_millis: u32, - //pub is_mod_hidden_from_public: bool, // converted //pub model_is_mod_approved: bool, // converted //pub maybe_mod_user_token: Option, @@ -39,7 +38,6 @@ struct RawTtsInferenceRecordForList { pub file_size_bytes: i32, pub duration_millis: i32, - //pub is_mod_hidden_from_public: i8, // needs convert //pub model_is_mod_approved: i8, // needs convert //pub maybe_mod_user_token: Option, @@ -95,7 +93,6 @@ pub async fn list_tts_inference_results( maybe_creator_user_token: ir.maybe_creator_user_token.clone(), maybe_creator_username: ir.maybe_creator_username.clone(), maybe_creator_display_name: ir.maybe_creator_display_name.clone(), - //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, //model_is_mod_approved: if ir.model_is_mod_approved == 0 { false } else { true }, file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, @@ -140,8 +137,7 @@ LEFT OUTER JOIN tts_models LEFT OUTER JOIN users ON tts_results.maybe_creator_user_token = users.token WHERE - tts_results.is_mod_hidden_from_public IS FALSE - AND tts_models.is_mod_disabled IS FALSE + tts_models.is_locked_from_use IS FALSE AND tts_results.user_deleted_at IS NULL AND tts_results.mod_deleted_at IS NULL "#) @@ -217,8 +213,7 @@ LEFT OUTER JOIN users ON tts_results.maybe_creator_user_token = users.token WHERE users.username = ? - AND tts_results.is_mod_hidden_from_public IS FALSE - AND tts_models.is_mod_disabled IS FALSE + AND tts_models.is_locked_from_use IS FALSE AND tts_results.user_deleted_at IS NULL AND tts_results.mod_deleted_at IS NULL "#, diff --git a/src/common_queries/list_tts_models.rs b/src/common_queries/list_tts_models.rs index aae246302f..a9468463d7 100644 --- a/src/common_queries/list_tts_models.rs +++ b/src/common_queries/list_tts_models.rs @@ -3,6 +3,7 @@ use chrono::{DateTime, Utc}; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; +use crate::database_helpers::boolean_converters::i8_to_bool; #[derive(Serialize)] pub struct TtsModelRecordForList { @@ -15,7 +16,7 @@ pub struct TtsModelRecordForList { pub creator_username: String, pub creator_display_name: String, - pub is_mod_disabled: bool, // converted + pub is_locked_from_use: bool, // converted pub created_at: DateTime, pub updated_at: DateTime, @@ -31,7 +32,7 @@ struct RawTtsModelRecordForList { pub creator_username: String, pub creator_display_name: String, - pub is_mod_disabled: i8, // NB: needs conversion + pub is_locked_from_use: i8, // NB: needs conversion pub created_at: DateTime, pub updated_at: DateTime, @@ -83,7 +84,7 @@ pub async fn list_tts_models( creator_username: model.creator_username.clone(), creator_display_name: model.creator_display_name.clone(), title: model.title.clone(), - is_mod_disabled: if model.is_mod_disabled == 0 { false } else { true }, + is_locked_from_use: i8_to_bool(model.is_locked_from_use), created_at: model.created_at.clone(), updated_at: model.updated_at.clone(), } @@ -109,14 +110,14 @@ SELECT users.username as creator_username, users.display_name as creator_display_name, tts.title, - tts.is_mod_disabled, + tts.is_locked_from_use, tts.created_at, tts.updated_at FROM tts_models as tts JOIN users ON users.token = tts.creator_user_token WHERE - tts.is_mod_disabled IS FALSE + tts.is_locked_from_use IS FALSE AND tts.user_deleted_at IS NULL AND tts.mod_deleted_at IS NULL "#) @@ -134,7 +135,7 @@ SELECT users.username as creator_username, users.display_name as creator_display_name, tts.title, - tts.is_mod_disabled, + tts.is_locked_from_use, tts.created_at, tts.updated_at FROM tts_models as tts @@ -170,7 +171,7 @@ SELECT users.username as creator_username, users.display_name as creator_display_name, tts.title, - tts.is_mod_disabled, + tts.is_locked_from_use, tts.created_at, tts.updated_at FROM tts_models as tts @@ -179,7 +180,7 @@ ON users.token = tts.creator_user_token WHERE users.username = ? - AND tts.is_mod_disabled IS FALSE + AND tts.is_locked_from_use IS FALSE AND tts.user_deleted_at IS NULL AND tts.mod_deleted_at IS NULL "#, @@ -198,7 +199,7 @@ SELECT users.username as creator_username, users.display_name as creator_display_name, tts.title, - tts.is_mod_disabled, + tts.is_locked_from_use, tts.created_at, tts.updated_at FROM tts_models as tts diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/common_queries/list_w2l_inference_results.rs index ce241d7a48..364a34b9d9 100644 --- a/src/common_queries/list_w2l_inference_results.rs +++ b/src/common_queries/list_w2l_inference_results.rs @@ -22,7 +22,6 @@ pub struct W2lInferenceRecordForList { pub frame_height: u32, pub duration_millis: u32, - //pub is_mod_hidden_from_public: bool, // converted //pub template_is_mod_approved: bool, // converted //pub maybe_mod_user_token: Option, @@ -48,7 +47,6 @@ struct RawW2lInferenceRecordForList { pub frame_height: i32, pub duration_millis: i32, - //pub is_mod_hidden_from_public: i8, // needs convert //pub template_is_mod_approved: i8, // needs convert //pub maybe_mod_user_token: Option, @@ -106,7 +104,6 @@ pub async fn list_w2l_inference_results( maybe_creator_user_token: ir.maybe_creator_user_token.clone(), maybe_creator_username: ir.maybe_creator_username.clone(), maybe_creator_display_name: ir.maybe_creator_display_name.clone(), - //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, @@ -157,8 +154,7 @@ LEFT OUTER JOIN w2l_templates LEFT OUTER JOIN users ON w2l_results.maybe_creator_user_token = users.token WHERE - w2l_results.is_mod_hidden_from_public IS FALSE - AND w2l_templates.is_mod_public_listing_approved IS TRUE + w2l_templates.is_public_listing_approved IS TRUE AND w2l_results.user_deleted_at IS NULL AND w2l_results.mod_deleted_at IS NULL "#) @@ -242,8 +238,7 @@ LEFT OUTER JOIN users ON w2l_results.maybe_creator_user_token = users.token WHERE users.username = ? - AND w2l_results.is_mod_hidden_from_public IS FALSE - AND w2l_templates.is_mod_public_listing_approved IS TRUE + AND w2l_templates.is_public_listing_approved IS TRUE AND w2l_results.user_deleted_at IS NULL AND w2l_results.mod_deleted_at IS NULL "#, diff --git a/src/common_queries/list_w2l_templates.rs b/src/common_queries/list_w2l_templates.rs index c3a2fdd781..e294aa8f47 100644 --- a/src/common_queries/list_w2l_templates.rs +++ b/src/common_queries/list_w2l_templates.rs @@ -18,7 +18,7 @@ pub struct W2lTemplateRecordForList { pub duration_millis: u32, pub maybe_image_object_name: Option, pub maybe_video_object_name: Option, - pub is_mod_public_listing_approved: Option, // converted + pub is_public_listing_approved: Option, // converted pub created_at: DateTime, pub updated_at: DateTime, } @@ -35,7 +35,7 @@ struct RawW2lTemplateRecordForList { pub duration_millis: i32, pub maybe_public_bucket_preview_image_object_name: Option, pub maybe_public_bucket_preview_video_object_name: Option, - pub is_mod_public_listing_approved: Option, // NB: needs conversion + pub is_public_listing_approved: Option, // NB: needs conversion pub created_at: DateTime, pub updated_at: DateTime, } @@ -91,7 +91,7 @@ pub async fn list_w2l_templates( duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), - is_mod_public_listing_approved: nullable_i8_to_optional_bool(template.is_mod_public_listing_approved), + is_public_listing_approved: nullable_i8_to_optional_bool(template.is_public_listing_approved), created_at: template.created_at.clone(), updated_at: template.updated_at.clone(), } @@ -122,14 +122,14 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_public_listing_approved, + w2l.is_public_listing_approved, w2l.created_at, w2l.updated_at FROM w2l_templates as w2l JOIN users ON users.token = w2l.creator_user_token WHERE - w2l.is_mod_public_listing_approved IS TRUE + w2l.is_public_listing_approved IS TRUE AND w2l.user_deleted_at IS NULL AND w2l.mod_deleted_at IS NULL "#) @@ -152,7 +152,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_public_listing_approved, + w2l.is_public_listing_approved, w2l.created_at, w2l.updated_at FROM w2l_templates as w2l @@ -193,7 +193,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_public_listing_approved, + w2l.is_public_listing_approved, w2l.created_at, w2l.updated_at FROM w2l_templates as w2l @@ -202,7 +202,7 @@ ON users.token = w2l.creator_user_token WHERE users.username = ? - AND w2l.is_mod_public_listing_approved IS TRUE + AND w2l.is_public_listing_approved IS TRUE AND w2l.user_deleted_at IS NULL AND w2l.mod_deleted_at IS NULL "#, @@ -226,7 +226,7 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_public_listing_approved, + w2l.is_public_listing_approved, w2l.created_at, w2l.updated_at FROM w2l_templates as w2l diff --git a/src/common_queries/query_tts_model.rs b/src/common_queries/query_tts_model.rs index a37d5f348e..1ba4db4a2b 100644 --- a/src/common_queries/query_tts_model.rs +++ b/src/common_queries/query_tts_model.rs @@ -22,7 +22,7 @@ pub struct TtsModelRecordForResponse { pub title: String, pub description_markdown: String, pub description_rendered_html: String, - pub is_mod_disabled: bool, + pub is_locked_from_use: bool, pub is_locked_from_user_modification: bool, pub created_at: DateTime, pub updated_at: DateTime, @@ -40,7 +40,7 @@ pub struct TtsModelRecordRaw { pub title: String, pub description_markdown: String, pub description_rendered_html: String, - pub is_mod_disabled: i8, + pub is_locked_from_use: i8, pub is_locked_from_user_modification: i8, pub created_at: DateTime, pub updated_at: DateTime, @@ -85,7 +85,7 @@ pub async fn select_tts_model_by_token( title: model.title.clone(), description_markdown: model.description_markdown.clone(), description_rendered_html: model.description_rendered_html.clone(), - is_mod_disabled: i8_to_bool(model.is_mod_disabled), + is_locked_from_use: i8_to_bool(model.is_locked_from_use), is_locked_from_user_modification: i8_to_bool(model.is_locked_from_user_modification), created_at: model.created_at.clone(), updated_at: model.updated_at.clone(), @@ -112,7 +112,7 @@ SELECT tts.title, tts.description_markdown, tts.description_rendered_html, - tts.is_mod_disabled, + tts.is_locked_from_use, tts.is_locked_from_user_modification, tts.created_at, tts.updated_at, @@ -145,7 +145,7 @@ SELECT tts.title, tts.description_markdown, tts.description_rendered_html, - tts.is_mod_disabled, + tts.is_locked_from_use, tts.is_locked_from_user_modification, tts.created_at, tts.updated_at, diff --git a/src/common_queries/query_tts_result.rs b/src/common_queries/query_tts_result.rs index 5279722b4a..632b2d6ac8 100644 --- a/src/common_queries/query_tts_result.rs +++ b/src/common_queries/query_tts_result.rs @@ -36,7 +36,6 @@ pub struct TtsResultRecordForResponse { pub file_size_bytes: u32, pub duration_millis: u32, - //pub is_mod_hidden_from_public: bool, // converted //pub model_is_mod_approved: bool, // converted //pub maybe_mod_user_token: Option, @@ -71,7 +70,6 @@ pub struct TtsResultRecordRaw { pub file_size_bytes: i32, pub duration_millis: i32, - //pub is_mod_hidden_from_public: i8, // needs convert //pub model_is_mod_approved: i8, // needs convert //pub maybe_mod_user_token: Option, @@ -128,7 +126,6 @@ pub async fn select_tts_result_by_token( maybe_model_creator_display_name: ir.maybe_model_creator_display_name.clone(), maybe_model_creator_gravatar_hash: ir.maybe_model_creator_gravatar_hash.clone(), - //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, //model_is_mod_approved: if ir.model_is_mod_approved == 0 { false } else { true }, public_bucket_wav_audio_path: ir.public_bucket_wav_audio_path.clone(), diff --git a/src/common_queries/query_w2l_result.rs b/src/common_queries/query_w2l_result.rs index 412b4d62ab..3d49bec025 100644 --- a/src/common_queries/query_w2l_result.rs +++ b/src/common_queries/query_w2l_result.rs @@ -38,7 +38,6 @@ pub struct W2lResultRecordForResponse { pub frame_height: u32, pub duration_millis: u32, - //pub is_mod_hidden_from_public: bool, // converted //pub template_is_mod_approved: bool, // converted //pub maybe_mod_user_token: Option, @@ -76,7 +75,6 @@ pub struct W2lResultRecordRaw { pub frame_height: i32, pub duration_millis: i32, - //pub is_mod_hidden_from_public: i8, // needs convert //pub template_is_mod_approved: i8, // needs convert //pub maybe_mod_user_token: Option, @@ -135,7 +133,6 @@ pub async fn select_w2l_result_by_token( maybe_template_creator_display_name: ir.maybe_template_creator_display_name.clone(), maybe_template_creator_gravatar_hash: ir.maybe_template_creator_gravatar_hash.clone(), - //is_mod_hidden_from_public: if ir.is_mod_hidden_from_public == 0 { false } else { true }, //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, diff --git a/src/common_queries/query_w2l_template.rs b/src/common_queries/query_w2l_template.rs index faf4359f7a..8a1ed4e0ce 100644 --- a/src/common_queries/query_w2l_template.rs +++ b/src/common_queries/query_w2l_template.rs @@ -25,8 +25,8 @@ pub struct W2lTemplateRecordForResponse { pub duration_millis: u32, pub maybe_image_object_name: Option, pub maybe_video_object_name: Option, - pub is_mod_public_listing_approved: Option, - pub is_mod_disabled: bool, + pub is_public_listing_approved: Option, + pub is_locked_from_use: bool, pub is_locked_from_user_modification: bool, pub created_at: DateTime, pub updated_at: DateTime, @@ -47,8 +47,8 @@ pub struct W2lTemplateRecordRaw { pub duration_millis: i32, pub maybe_public_bucket_preview_image_object_name: Option, pub maybe_public_bucket_preview_video_object_name: Option, - pub is_mod_public_listing_approved: Option, - pub is_mod_disabled: i8, + pub is_public_listing_approved: Option, + pub is_locked_from_use: i8, pub is_locked_from_user_modification: i8, pub created_at: DateTime, pub updated_at: DateTime, @@ -96,8 +96,8 @@ pub async fn select_w2l_template_by_token( duration_millis: if template.duration_millis > 0 { template.duration_millis as u32 } else { 0 }, maybe_image_object_name: template.maybe_public_bucket_preview_image_object_name.clone(), maybe_video_object_name: template.maybe_public_bucket_preview_video_object_name.clone(), - is_mod_public_listing_approved: nullable_i8_to_optional_bool(template.is_mod_public_listing_approved), - is_mod_disabled: i8_to_bool(template.is_mod_disabled), + is_public_listing_approved: nullable_i8_to_optional_bool(template.is_public_listing_approved), + is_locked_from_use: i8_to_bool(template.is_locked_from_use), is_locked_from_user_modification: i8_to_bool(template.is_locked_from_user_modification), created_at: template.created_at.clone(), updated_at: template.updated_at.clone(), @@ -127,8 +127,8 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_public_listing_approved, - w2l.is_mod_disabled, + w2l.is_public_listing_approved, + w2l.is_locked_from_use, w2l.is_locked_from_user_modification, w2l.created_at, w2l.updated_at, @@ -164,8 +164,8 @@ SELECT w2l.duration_millis, w2l.maybe_public_bucket_preview_image_object_name, w2l.maybe_public_bucket_preview_video_object_name, - w2l.is_mod_public_listing_approved, - w2l.is_mod_disabled, + w2l.is_public_listing_approved, + w2l.is_locked_from_use, w2l.is_locked_from_user_modification, w2l.created_at, w2l.updated_at, diff --git a/src/http_server/endpoints/tts/edit_tts_model.rs b/src/http_server/endpoints/tts/edit_tts_model.rs index d4e69aa756..364e7f1a0b 100644 --- a/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/src/http_server/endpoints/tts/edit_tts_model.rs @@ -43,7 +43,7 @@ pub struct EditTtsModelRequest { //pub text_preprocessing_algorithm: Option, //pub vocoder_token: Option, - //pub is_mod_disabled: Option, + //pub is_locked_from_use: Option, //pub is_locked_from_user_modification: Option, //pub maybe_mod_comments: Option, } @@ -135,7 +135,7 @@ pub async fn edit_tts_model_handler( } if !is_mod { - if model_record.is_locked_from_user_modification || model_record.is_mod_disabled { + if model_record.is_locked_from_user_modification || model_record.is_locked_from_use { return Err(EditTtsModelError::NotAuthorized); } } diff --git a/src/http_server/endpoints/w2l/delete_w2l_template.rs b/src/http_server/endpoints/w2l/delete_w2l_template.rs index 607f766351..5033aa2193 100644 --- a/src/http_server/endpoints/w2l/delete_w2l_template.rs +++ b/src/http_server/endpoints/w2l/delete_w2l_template.rs @@ -118,7 +118,7 @@ pub async fn delete_w2l_template_handler( } if !is_mod { - if w2l_template.is_locked_from_user_modification || w2l_template.is_mod_disabled { + if w2l_template.is_locked_from_user_modification || w2l_template.is_locked_from_use { return Err(DeleteW2lTemplateError::NotAuthorized); } } diff --git a/src/http_server/endpoints/w2l/edit_w2l_template.rs b/src/http_server/endpoints/w2l/edit_w2l_template.rs index 8002f6ebc4..1692af9572 100644 --- a/src/http_server/endpoints/w2l/edit_w2l_template.rs +++ b/src/http_server/endpoints/w2l/edit_w2l_template.rs @@ -39,7 +39,7 @@ pub struct EditW2lTemplateRequest { //pub updatable_slug: Option, //pub creator_set_visibility: Option, - //pub is_mod_disabled: Option, + //pub is_locked_from_use: Option, //pub is_locked_from_user_modification: Option, //pub maybe_mod_comments: Option, } @@ -138,7 +138,7 @@ pub async fn edit_w2l_template_handler( } if !editor_is_moderator { - if template_record.is_locked_from_user_modification || template_record.is_mod_disabled { + if template_record.is_locked_from_user_modification || template_record.is_locked_from_use { return Err(EditW2lTemplateError::NotAuthorized); } } diff --git a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs index b29acc1a90..206f527d29 100644 --- a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs +++ b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs @@ -135,7 +135,7 @@ pub async fn set_w2l_template_mod_approval_handler( r#" UPDATE w2l_templates SET - is_mod_public_listing_approved = ?, + is_public_listing_approved = ?, maybe_mod_user_token = ? WHERE token = ? diff --git a/src/main.rs b/src/main.rs index 08a264c8a4..b7814b621e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,6 +11,7 @@ #[macro_use] extern crate lazy_static; #[macro_use] extern crate serde_derive; + pub const RESERVED_USERNAMES : &'static str = include_str!("../db/reserved_usernames.txt"); pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../db/reserved_usernames_including.txt"); pub const BANNED_SLURS : &'static str = include_str!("../db/banned_slurs.txt"); From fe75726e04753d8d76933b1d01e8cfda05cd227a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 21:49:15 -0400 Subject: [PATCH 0362/1042] add moderator fields to edit_tts_model endpoint --- migrations/2021-05-15-071534_users/up.sql | 1 + .../endpoints/tts/edit_tts_model.rs | 74 +++++++++++++++++-- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/migrations/2021-05-15-071534_users/up.sql b/migrations/2021-05-15-071534_users/up.sql index 557cc74dba..fb511b3835 100644 --- a/migrations/2021-05-15-071534_users/up.sql +++ b/migrations/2021-05-15-071534_users/up.sql @@ -176,6 +176,7 @@ CREATE TABLE user_roles ( -- Moderation can_approve_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, + -- Edit includes "lock_from_use" and "lock_from_modification" can_edit_other_users_profiles BOOLEAN NOT NULL DEFAULT FALSE, can_edit_other_users_tts_models BOOLEAN NOT NULL DEFAULT FALSE, can_edit_other_users_w2l_templates BOOLEAN NOT NULL DEFAULT FALSE, diff --git a/src/http_server/endpoints/tts/edit_tts_model.rs b/src/http_server/endpoints/tts/edit_tts_model.rs index 364e7f1a0b..391ca648fe 100644 --- a/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/src/http_server/endpoints/tts/edit_tts_model.rs @@ -4,7 +4,7 @@ use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; -use actix_web::web::Path; +use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::common_queries::query_tts_model::select_tts_model_by_token; use crate::common_queries::sessions::create_session_for_user; @@ -24,6 +24,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use sqlx::MySqlPool; /// For the URL PathInfo #[derive(Deserialize)] @@ -33,19 +34,22 @@ pub struct EditTtsModelPathInfo { #[derive(Deserialize)] pub struct EditTtsModelRequest { + // ========== Author + Moderator options ========== + pub title: Option, pub description_markdown: Option, - //pub updatable_slug: Option, //pub creator_set_visibility: Option, - //pub tts_model_type: Option, //pub text_preprocessing_algorithm: Option, //pub vocoder_token: Option, - //pub is_locked_from_use: Option, - //pub is_locked_from_user_modification: Option, - //pub maybe_mod_comments: Option, + // ========== Moderator options ========== + + pub is_public_listing_approved: Option, + pub is_locked_from_user_modification: Option, + pub is_locked_from_use: Option, + pub maybe_mod_comments: Option, } #[derive(Debug, Display)] @@ -140,7 +144,8 @@ pub async fn edit_tts_model_handler( } } - // Fields to set + // Author + Mod fields. + // These fields must be present on all requests. let mut title = None; let mut description_markdown = None; let mut description_html = None; @@ -215,6 +220,19 @@ LIMIT 1 .await }; + // TODO: This is lazy and suboptimal af to query again + // The reason we're doing this is because `sqlx` only does static type checking of queries + // with string literals. It does not support dynamic query building, thus the predicates + // must be held constant. :( + if is_mod { + update_mod_details( + &request, + &user_session.user_token, + &model_record.model_token, + &server_state.mysql_pool + ).await?; + } + match query_result { Ok(_) => {}, Err(err) => { @@ -225,3 +243,45 @@ LIMIT 1 Ok(simple_json_success()) } + +async fn update_mod_details( + request: &Json, + mod_user_token: &str, + model_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), EditTtsModelError> { + + let is_public_listing_approved= request.is_public_listing_approved.unwrap_or(false); + let is_locked_from_user_modification = request.is_locked_from_user_modification.unwrap_or(false); + let is_locked_from_use = request.is_locked_from_use.unwrap_or(false); + + let query_result = sqlx::query!( + r#" +UPDATE tts_models +SET + is_locked_from_user_modification = ?, + is_locked_from_use = ?, + maybe_mod_comments = ?, + maybe_mod_user_token = ?, + version = version + 1 + +WHERE tts_models.token = ? +LIMIT 1 + "#, + &is_locked_from_user_modification, + &is_locked_from_use, + &request.maybe_mod_comments, + &mod_user_token, + &model_token + ) + .execute(mysql_pool) + .await; + + match query_result { + Ok(_) => Ok(()), + Err(err) => { + warn!("Update W2L model edit DB error: {:?}", err); + Err(EditTtsModelError::ServerError) + } + } +} From 544c61b0a87c5dd970628952c53409825a2c6028 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 21:53:27 -0400 Subject: [PATCH 0363/1042] add one more field --- src/http_server/endpoints/tts/edit_tts_model.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/http_server/endpoints/tts/edit_tts_model.rs b/src/http_server/endpoints/tts/edit_tts_model.rs index 391ca648fe..4a9eedc867 100644 --- a/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/src/http_server/endpoints/tts/edit_tts_model.rs @@ -259,6 +259,7 @@ async fn update_mod_details( r#" UPDATE tts_models SET + is_public_listing_approved = ?, is_locked_from_user_modification = ?, is_locked_from_use = ?, maybe_mod_comments = ?, @@ -268,11 +269,12 @@ SET WHERE tts_models.token = ? LIMIT 1 "#, - &is_locked_from_user_modification, - &is_locked_from_use, - &request.maybe_mod_comments, - &mod_user_token, - &model_token + is_public_listing_approved, + is_locked_from_user_modification, + is_locked_from_use, + request.maybe_mod_comments, + mod_user_token, + model_token ) .execute(mysql_pool) .await; From 848233b6fa6765b58990365bb483fe513443f95a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 22:01:02 -0400 Subject: [PATCH 0364/1042] add mod details editing to w2l templates --- .../endpoints/tts/edit_tts_model.rs | 11 +-- .../endpoints/w2l/edit_w2l_template.rs | 97 ++++++++++++++----- 2 files changed, 78 insertions(+), 30 deletions(-) diff --git a/src/http_server/endpoints/tts/edit_tts_model.rs b/src/http_server/endpoints/tts/edit_tts_model.rs index 4a9eedc867..47c733a147 100644 --- a/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/src/http_server/endpoints/tts/edit_tts_model.rs @@ -183,8 +183,7 @@ SET description_rendered_html = ?, creator_ip_address_last_update = ?, version = version + 1 - -WHERE tts_models.token = ? +WHERE token = ? LIMIT 1 "#, &title, @@ -206,8 +205,7 @@ SET description_rendered_html = ?, maybe_mod_user_token = ?, version = version + 1 - -WHERE tts_models.token = ? +WHERE token = ? LIMIT 1 "#, &title, @@ -265,8 +263,7 @@ SET maybe_mod_comments = ?, maybe_mod_user_token = ?, version = version + 1 - -WHERE tts_models.token = ? +WHERE token = ? LIMIT 1 "#, is_public_listing_approved, @@ -282,7 +279,7 @@ LIMIT 1 match query_result { Ok(_) => Ok(()), Err(err) => { - warn!("Update W2L model edit DB error: {:?}", err); + warn!("Update TTS model (mod details) DB error: {:?}", err); Err(EditTtsModelError::ServerError) } } diff --git a/src/http_server/endpoints/w2l/edit_w2l_template.rs b/src/http_server/endpoints/w2l/edit_w2l_template.rs index 1692af9572..15374c715c 100644 --- a/src/http_server/endpoints/w2l/edit_w2l_template.rs +++ b/src/http_server/endpoints/w2l/edit_w2l_template.rs @@ -4,7 +4,7 @@ use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; -use actix_web::web::Path; +use actix_web::web::{Path, Json}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::common_queries::query_w2l_template::select_w2l_template_by_token; use crate::common_queries::sessions::create_session_for_user; @@ -24,6 +24,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use sqlx::MySqlPool; /// For the URL PathInfo #[derive(Deserialize)] @@ -33,15 +34,18 @@ pub struct EditW2lTemplatePathInfo { #[derive(Deserialize)] pub struct EditW2lTemplateRequest { + // ========== Author + Moderator options ========== pub title: Option, pub description_markdown: Option, - //pub updatable_slug: Option, //pub creator_set_visibility: Option, - //pub is_locked_from_use: Option, - //pub is_locked_from_user_modification: Option, - //pub maybe_mod_comments: Option, + // ========== Moderator options ========== + + pub is_public_listing_approved: Option, + pub is_locked_from_user_modification: Option, + pub is_locked_from_use: Option, + pub maybe_mod_comments: Option, } #[derive(Debug, Display)] @@ -122,28 +126,21 @@ pub async fn edit_w2l_template_handler( }; // NB: Second set of permission checks - let mut editor_is_original_user = false; - let mut editor_is_moderator = false; + let is_author = template_record.creator_user_token == user_session.user_token; + let is_mod = user_session.can_edit_other_users_w2l_templates ; - if template_record.creator_user_token == user_session.user_token { - editor_is_original_user = true; - } - - if user_session.can_edit_other_users_w2l_templates { - editor_is_moderator = true; - } - - if !editor_is_original_user && !editor_is_moderator { + if !is_author && !is_mod { return Err(EditW2lTemplateError::NotAuthorized); } - if !editor_is_moderator { + if !is_mod { if template_record.is_locked_from_user_modification || template_record.is_locked_from_use { return Err(EditW2lTemplateError::NotAuthorized); } } - // Fields to set + // Author + Mod fields. + // These fields must be present on all requests. let mut title = None; let mut description_markdown = None; let mut description_html = None; @@ -170,7 +167,7 @@ pub async fn edit_w2l_template_handler( let ip_address = get_request_ip(&http_request); - let query_result = if editor_is_original_user { + let query_result = if is_author { // We need to store the IP address details. sqlx::query!( r#" @@ -181,8 +178,7 @@ SET description_rendered_html = ?, creator_ip_address_last_update = ?, version = version + 1 - -WHERE w2l_templates.token = ? +WHERE token = ? LIMIT 1 "#, &title, @@ -204,8 +200,7 @@ SET description_rendered_html = ?, maybe_mod_user_token = ?, version = version + 1 - -WHERE w2l_templates.token = ? +WHERE token = ? LIMIT 1 "#, &title, @@ -226,5 +221,61 @@ LIMIT 1 } }; + // TODO: This is lazy and suboptimal af to query again + // The reason we're doing this is because `sqlx` only does static type checking of queries + // with string literals. It does not support dynamic query building, thus the predicates + // must be held constant. :( + if is_mod { + update_mod_details( + &request, + &user_session.user_token, + &template_record.template_token, + &server_state.mysql_pool + ).await?; + } + Ok(simple_json_success()) } + +async fn update_mod_details( + request: &Json, + mod_user_token: &str, + template_token: &str, + mysql_pool: &MySqlPool +) -> Result<(), EditW2lTemplateError> { + + let is_public_listing_approved= request.is_public_listing_approved.unwrap_or(false); + let is_locked_from_user_modification = request.is_locked_from_user_modification.unwrap_or(false); + let is_locked_from_use = request.is_locked_from_use.unwrap_or(false); + + let query_result = sqlx::query!( + r#" +UPDATE w2l_templates +SET + is_public_listing_approved = ?, + is_locked_from_user_modification = ?, + is_locked_from_use = ?, + maybe_mod_comments = ?, + maybe_mod_user_token = ?, + version = version + 1 +WHERE token = ? +LIMIT 1 + "#, + is_public_listing_approved, + is_locked_from_user_modification, + is_locked_from_use, + request.maybe_mod_comments, + mod_user_token, + template_token + ) + .execute(mysql_pool) + .await; + + match query_result { + Ok(_) => Ok(()), + Err(err) => { + warn!("Update W2L template (mod details) DB error: {:?}", err); + Err(EditW2lTemplateError::ServerError) + } + } +} From 10a84b624e36ad49f76bed64e0d9001428fd8cb5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 22:13:03 -0400 Subject: [PATCH 0365/1042] no mod comments on inference results --- migrations/2021-05-15-212918_inference_results/up.sql | 6 ------ src/http_server/endpoints/w2l/edit_w2l_template.rs | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/migrations/2021-05-15-212918_inference_results/up.sql b/migrations/2021-05-15-212918_inference_results/up.sql index 8e3ed05ed8..6a32694575 100644 --- a/migrations/2021-05-15-212918_inference_results/up.sql +++ b/migrations/2021-05-15-212918_inference_results/up.sql @@ -70,9 +70,6 @@ CREATE TABLE tts_results ( -- ========== MODERATION DETAILS ========== - -- If a moderator has comments. - maybe_mod_comments VARCHAR(255) DEFAULT NULL, - -- The last moderator that made changes. maybe_mod_user_token VARCHAR(32) DEFAULT NULL, @@ -151,9 +148,6 @@ CREATE TABLE w2l_results ( -- ========== MODERATION DETAILS ========== - -- If a moderator has comments. - maybe_mod_comments VARCHAR(255) DEFAULT NULL, - -- The last moderator that made changes. maybe_mod_user_token VARCHAR(32) DEFAULT NULL, diff --git a/src/http_server/endpoints/w2l/edit_w2l_template.rs b/src/http_server/endpoints/w2l/edit_w2l_template.rs index 15374c715c..86ebc0fbb2 100644 --- a/src/http_server/endpoints/w2l/edit_w2l_template.rs +++ b/src/http_server/endpoints/w2l/edit_w2l_template.rs @@ -20,11 +20,11 @@ use crate::validations::check_for_slurs::contains_slurs; use derive_more::{Display, Error}; use log::{info, warn, log}; use regex::Regex; +use sqlx::MySqlPool; use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use sqlx::MySqlPool; /// For the URL PathInfo #[derive(Deserialize)] From 214f048967770339ebd36c2580aca16846b73744 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 23:06:46 -0400 Subject: [PATCH 0366/1042] add ip ban handler (untested); stubs for other endpoints --- src/http_server/endpoints/mod.rs | 1 + .../endpoints/moderation/add_ip_ban.rs | 136 ++++++++++++++++++ .../endpoints/moderation/delete_ip_ban.rs | 0 .../endpoints/moderation/list_ip_bans.rs | 0 .../endpoints/moderation/list_mods.rs | 0 .../endpoints/moderation/list_roles.rs | 0 src/http_server/endpoints/moderation/mod.rs | 6 + .../endpoints/moderation/set_user_role.rs | 0 src/main.rs | 13 ++ src/validations/ip_addresses.rs | 44 ++++++ src/validations/mod.rs | 1 + 11 files changed, 201 insertions(+) create mode 100644 src/http_server/endpoints/moderation/add_ip_ban.rs create mode 100644 src/http_server/endpoints/moderation/delete_ip_ban.rs create mode 100644 src/http_server/endpoints/moderation/list_ip_bans.rs create mode 100644 src/http_server/endpoints/moderation/list_mods.rs create mode 100644 src/http_server/endpoints/moderation/list_roles.rs create mode 100644 src/http_server/endpoints/moderation/mod.rs create mode 100644 src/http_server/endpoints/moderation/set_user_role.rs create mode 100644 src/validations/ip_addresses.rs diff --git a/src/http_server/endpoints/mod.rs b/src/http_server/endpoints/mod.rs index e4b320f98c..304cc633f3 100644 --- a/src/http_server/endpoints/mod.rs +++ b/src/http_server/endpoints/mod.rs @@ -1,6 +1,7 @@ pub mod default_route_404; pub mod events; pub mod misc; +pub mod moderation; pub mod root_index; pub mod tts; pub mod users; diff --git a/src/http_server/endpoints/moderation/add_ip_ban.rs b/src/http_server/endpoints/moderation/add_ip_ban.rs new file mode 100644 index 0000000000..d8b1d5b1cb --- /dev/null +++ b/src/http_server/endpoints/moderation/add_ip_ban.rs @@ -0,0 +1,136 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::server_state::ServerState; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use crate::validations::check_for_slurs::contains_slurs; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; + +#[derive(Deserialize)] +pub struct AddIpBanRequest { + ip_address: String, + mod_notes: String, + maybe_target_user_token: Option, +} + +#[derive(Serialize)] +pub struct ErrorResponse { + pub success: bool, + pub error_reason: String, +} + +#[derive(Debug, Display)] +pub enum AddIpBanError { + BadInput(String), + ServerError, + Unauthorized, +} + +impl ResponseError for AddIpBanError { + fn status_code(&self) -> StatusCode { + match *self { + AddIpBanError::BadInput(_) => StatusCode::BAD_REQUEST, + AddIpBanError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + AddIpBanError::Unauthorized => StatusCode::UNAUTHORIZED, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + AddIpBanError::BadInput(reason) => reason.to_string(), + AddIpBanError::ServerError => "server error".to_string(), + AddIpBanError::Unauthorized => "unauthorized".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn add_ip_ban_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + AddIpBanError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(AddIpBanError::Unauthorized); + } + }; + + if !user_session.can_ban_users { + warn!("user is not allowed to add bans: {}", user_session.user_token); + return Err(AddIpBanError::Unauthorized); + } + + info!("Creating ban..."); + + let query_result = sqlx::query!( + r#" +INSERT INTO + ip_address_bans +SET + ip_address = ?, + maybe_target_user_token = ?, + mod_user_token = ?, + mod_notes = ? +ON DUPLICATE KEY UPDATE + expires_at = NULL, + deleted_at = NULL, + ip_address = ?, + maybe_target_user_token = ?, + mod_user_token = ?, + mod_notes = ? + "#, + // Insert + &request.ip_address, + &request.maybe_target_user_token, + &user_session.user_token, + &request.mod_notes, + // Update + &request.ip_address, + &request.maybe_target_user_token, + &user_session.user_token, + &request.mod_notes, + ) + .execute(&server_state.mysql_pool) + .await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("Add IP ban DB error: {:?}", err); + return Err(AddIpBanError::ServerError); + } + }; + + Ok(simple_json_success()) +} diff --git a/src/http_server/endpoints/moderation/delete_ip_ban.rs b/src/http_server/endpoints/moderation/delete_ip_ban.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/http_server/endpoints/moderation/list_ip_bans.rs b/src/http_server/endpoints/moderation/list_ip_bans.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/http_server/endpoints/moderation/list_mods.rs b/src/http_server/endpoints/moderation/list_mods.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/http_server/endpoints/moderation/list_roles.rs b/src/http_server/endpoints/moderation/list_roles.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/http_server/endpoints/moderation/mod.rs b/src/http_server/endpoints/moderation/mod.rs new file mode 100644 index 0000000000..b4a75705bf --- /dev/null +++ b/src/http_server/endpoints/moderation/mod.rs @@ -0,0 +1,6 @@ +pub mod add_ip_ban; +pub mod delete_ip_ban; +pub mod list_ip_bans; +pub mod list_mods; +pub mod list_roles; +pub mod set_user_role; \ No newline at end of file diff --git a/src/http_server/endpoints/moderation/set_user_role.rs b/src/http_server/endpoints/moderation/set_user_role.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/main.rs b/src/main.rs index b7814b621e..b2d282b3c5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -77,6 +77,7 @@ use crate::http_server::endpoints::w2l::edit_w2l_template::edit_w2l_template_han use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; use crate::http_server::endpoints::w2l::delete_w2l_result::delete_w2l_inference_result_handler; use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; +use crate::http_server::endpoints::moderation::add_ip_ban::add_ip_ban_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -276,6 +277,18 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(session_info_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + // ==================== MODERATOR ==================== + .service( + web::scope("/moderation") + .service( + web::scope("/ip_bans") + .service( + web::resource("/add") + .route(web::post().to(add_ip_ban_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) + ) // ==================== TTS ==================== .service( web::scope("/tts") diff --git a/src/validations/ip_addresses.rs b/src/validations/ip_addresses.rs new file mode 100644 index 0000000000..20a22d4e45 --- /dev/null +++ b/src/validations/ip_addresses.rs @@ -0,0 +1,44 @@ +use crate::AnyhowResult; +use regex::Regex; + +/// Do not use this for IP addresses from requests. +/// Only for moderator-input IP addresses +pub fn validate_moderator_provided_ip_address(ip_address: &str) -> Result<(), String> { + lazy_static! { + static ref IP_ADDRESS_REGEX: Regex = { + // TODO: This doesn't require valid IP addresses. Just a number/dot format. + Regex::new(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$").expect("should be valid regex") + }; + } + + if !IP_ADDRESS_REGEX.is_match(ip_address) { + return Err("ip address is invalid".to_string()); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::validations::ip_addresses::validate_moderator_provided_ip_address; + + #[test] + fn valid_cases() { + assert!(validate_moderator_provided_ip_address("0.0.0.0").is_ok()); + assert!(validate_moderator_provided_ip_address("1.1.1.1").is_ok()); + assert!(validate_moderator_provided_ip_address("1.2.3.4").is_ok()); + assert!(validate_moderator_provided_ip_address("111.222.3.0").is_ok()); + assert!(validate_moderator_provided_ip_address("127.0.0.1").is_ok()); + assert!(validate_moderator_provided_ip_address("255.255.255.255").is_ok()); + } + + #[test] + fn invalid_cases() { + assert!(validate_moderator_provided_ip_address(" 255.255.255.255 ").is_err()); + assert!(validate_moderator_provided_ip_address("").is_err()); + assert!(validate_moderator_provided_ip_address(".1.1.1.1").is_err()); + assert!(validate_moderator_provided_ip_address("1.1.1").is_err()); + assert!(validate_moderator_provided_ip_address("127.0.0.1notvalid").is_err()); + assert!(validate_moderator_provided_ip_address("127.127.127.127.").is_err()); + } +} diff --git a/src/validations/mod.rs b/src/validations/mod.rs index a4147f1a83..20c536754c 100644 --- a/src/validations/mod.rs +++ b/src/validations/mod.rs @@ -2,6 +2,7 @@ pub mod cashapp_username; pub mod check_for_slurs; pub mod discord_username; pub mod github_username; +pub mod ip_addresses; pub mod model_uploads; pub mod passwords; pub mod twitch_username; From a3cb58e716a4502b5df05df115fde1164d4d5363 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 23:25:55 -0400 Subject: [PATCH 0367/1042] delete/undelete ip address ban handler --- .../endpoints/moderation/add_ip_ban.rs | 12 +- .../endpoints/moderation/delete_ip_ban.rs | 143 ++++++++++++++++++ src/main.rs | 6 + 3 files changed, 152 insertions(+), 9 deletions(-) diff --git a/src/http_server/endpoints/moderation/add_ip_ban.rs b/src/http_server/endpoints/moderation/add_ip_ban.rs index d8b1d5b1cb..da0480e9a3 100644 --- a/src/http_server/endpoints/moderation/add_ip_ban.rs +++ b/src/http_server/endpoints/moderation/add_ip_ban.rs @@ -7,8 +7,11 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; use crate::server_state::ServerState; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; @@ -19,9 +22,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::validations::check_for_slurs::contains_slurs; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; -use crate::http_server::web_utils::response_success_helpers::simple_json_success; #[derive(Deserialize)] pub struct AddIpBanRequest { @@ -30,12 +30,6 @@ pub struct AddIpBanRequest { maybe_target_user_token: Option, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum AddIpBanError { BadInput(String), diff --git a/src/http_server/endpoints/moderation/delete_ip_ban.rs b/src/http_server/endpoints/moderation/delete_ip_ban.rs index e69de29bb2..1aef770a33 100644 --- a/src/http_server/endpoints/moderation/delete_ip_ban.rs +++ b/src/http_server/endpoints/moderation/delete_ip_ban.rs @@ -0,0 +1,143 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use actix_web::web::Path; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct DeleteIpBanPathInfo { + ip_address: String, +} + +#[derive(Deserialize)] +pub struct DeleteIpBanRequest { + delete: bool, +} + +#[derive(Debug, Display)] +pub enum DeleteIpBanError { + BadInput(String), + ServerError, + Unauthorized, +} + +impl ResponseError for DeleteIpBanError { + fn status_code(&self) -> StatusCode { + match *self { + DeleteIpBanError::BadInput(_) => StatusCode::BAD_REQUEST, + DeleteIpBanError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + DeleteIpBanError::Unauthorized => StatusCode::UNAUTHORIZED, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + DeleteIpBanError::BadInput(reason) => reason.to_string(), + DeleteIpBanError::ServerError => "server error".to_string(), + DeleteIpBanError::Unauthorized => "unauthorized".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn delete_ip_ban_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + DeleteIpBanError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(DeleteIpBanError::Unauthorized); + } + }; + + if !user_session.can_ban_users { + warn!("user is not allowed to delete bans: {}", user_session.user_token); + return Err(DeleteIpBanError::Unauthorized); + } + + let query_result = if request.delete { + info!("Deleting IP ban: {}", &path.ip_address); + + sqlx::query!( + r#" +UPDATE + ip_address_bans +SET + mod_user_token = ?, + deleted_at = CURRENT_TIMESTAMP +WHERE + ip_address = ? +LIMIT 1 + "#, + &user_session.user_token, + &path.ip_address, + ) + .execute(&server_state.mysql_pool) + .await + } else { + info!("Restoring IP ban: {}", &path.ip_address); + + sqlx::query!( + r#" +UPDATE + ip_address_bans +SET + mod_user_token = ?, + deleted_at = NULL +WHERE + ip_address = ? +LIMIT 1 + "#, + &user_session.user_token, + &path.ip_address, + ) + .execute(&server_state.mysql_pool) + .await + }; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("(un)delete IP ban DB error: {:?}", err); + return Err(DeleteIpBanError::ServerError); + } + }; + + Ok(simple_json_success()) +} diff --git a/src/main.rs b/src/main.rs index b2d282b3c5..2ace0fcce6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -78,6 +78,7 @@ use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; use crate::http_server::endpoints::w2l::delete_w2l_result::delete_w2l_inference_result_handler; use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; use crate::http_server::endpoints::moderation::add_ip_ban::add_ip_ban_handler; +use crate::http_server::endpoints::moderation::delete_ip_ban::delete_ip_ban_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -287,6 +288,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::post().to(add_ip_ban_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/{ip_address}/delete") + .route(web::post().to(delete_ip_ban_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) ) // ==================== TTS ==================== From 1be0d3b0eb80ce298f82342da4b6bfe80a17c9c5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 10 Jun 2021 23:38:40 -0400 Subject: [PATCH 0368/1042] use `to_simple_json_error` to replace all old, verbose constructions --- .../endpoints/events/list_events.rs | 21 ++----------------- .../endpoints/tts/enqueue_infer_tts.rs | 21 ++----------------- .../endpoints/tts/enqueue_upload_tts_model.rs | 21 ++----------------- .../tts/get_tts_inference_job_status.rs | 21 ++----------------- .../endpoints/tts/get_tts_model.rs | 21 ++----------------- .../tts/get_tts_upload_model_job_status.rs | 21 ++----------------- .../endpoints/tts/list_tts_models.rs | 21 ++----------------- .../endpoints/users/create_account.rs | 21 ++----------------- .../endpoints/users/edit_profile.rs | 21 ++----------------- .../endpoints/users/get_profile.rs | 21 ++----------------- .../users/list_user_tts_inference_results.rs | 21 ++----------------- .../endpoints/users/list_user_tts_models.rs | 21 ++----------------- .../users/list_user_w2l_inference_results.rs | 21 ++----------------- .../users/list_user_w2l_templates.rs | 21 ++----------------- src/http_server/endpoints/users/login.rs | 21 ++----------------- src/http_server/endpoints/users/logout.rs | 21 ++----------------- .../endpoints/users/session_info.rs | 21 ++----------------- .../endpoints/w2l/enqueue_infer_w2l.rs | 21 ++----------------- .../w2l/enqueue_infer_w2l_with_uploads.rs | 21 ++----------------- .../w2l/enqueue_upload_w2l_template.rs | 21 ++----------------- .../w2l/get_w2l_inference_job_status.rs | 21 ++----------------- .../endpoints/w2l/get_w2l_result.rs | 6 ------ .../endpoints/w2l/get_w2l_template.rs | 21 ++----------------- .../w2l/get_w2l_upload_template_job_status.rs | 21 ++----------------- .../endpoints/w2l/list_w2l_templates.rs | 21 ++----------------- .../w2l/set_w2l_template_mod_approval.rs | 21 ++----------------- 26 files changed, 50 insertions(+), 481 deletions(-) diff --git a/src/http_server/endpoints/events/list_events.rs b/src/http_server/endpoints/events/list_events.rs index 8055feb660..3f1bdad007 100644 --- a/src/http_server/endpoints/events/list_events.rs +++ b/src/http_server/endpoints/events/list_events.rs @@ -25,6 +25,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Serialize)] pub struct EventRecord { @@ -45,12 +46,6 @@ pub struct ListEventsSuccessResponse { pub events: Vec, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum ListEventsError { ServerError, @@ -68,19 +63,7 @@ impl ResponseError for ListEventsError { ListEventsError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/tts/enqueue_infer_tts.rs b/src/http_server/endpoints/tts/enqueue_infer_tts.rs index d6133a91b8..0ae039be73 100644 --- a/src/http_server/endpoints/tts/enqueue_infer_tts.rs +++ b/src/http_server/endpoints/tts/enqueue_infer_tts.rs @@ -20,6 +20,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::validations::check_for_slurs::contains_slurs; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct InferTtsRequest { @@ -35,12 +36,6 @@ pub struct InferTtsSuccessResponse { pub inference_job_token: String, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum InferTtsError { BadInput(String), @@ -61,19 +56,7 @@ impl ResponseError for InferTtsError { InferTtsError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index ec285851c3..a1ee1d2265 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -20,6 +20,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct UploadTtsModelRequest { @@ -37,12 +38,6 @@ pub struct UploadTtsModelSuccessResponse { pub job_token: String, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum UploadTtsModelError { BadInput(String), @@ -66,19 +61,7 @@ impl ResponseError for UploadTtsModelError { UploadTtsModelError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs index 1797a57d6e..33de8929c4 100644 --- a/src/http_server/endpoints/tts/get_tts_inference_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_inference_job_status.rs @@ -25,6 +25,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -56,12 +57,6 @@ pub struct GetTtsInferenceStatusSuccessResponse { pub state: TtsInferenceJobStatusForResponse, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum GetTtsInferenceStatusError { ServerError, @@ -98,19 +93,7 @@ impl ResponseError for GetTtsInferenceStatusError { GetTtsInferenceStatusError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/tts/get_tts_model.rs b/src/http_server/endpoints/tts/get_tts_model.rs index e79e23f5f1..84de136aae 100644 --- a/src/http_server/endpoints/tts/get_tts_model.rs +++ b/src/http_server/endpoints/tts/get_tts_model.rs @@ -25,6 +25,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -54,12 +55,6 @@ pub struct GetTtsModelSuccessResponse { pub model: TtsModelRecordForResponse, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum GetTtsModelError { ServerError, @@ -93,19 +88,7 @@ impl ResponseError for GetTtsModelError { GetTtsModelError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs index 3d62c931fb..cbde835a1a 100644 --- a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs @@ -25,6 +25,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -50,12 +51,6 @@ pub struct GetTtsUploadModelStatusSuccessResponse { pub state: TtsUploadModelJobStatusForResponse, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum GetTtsUploadModelStatusError { ServerError, @@ -85,19 +80,7 @@ impl ResponseError for GetTtsUploadModelStatusError { GetTtsUploadModelStatusError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/tts/list_tts_models.rs b/src/http_server/endpoints/tts/list_tts_models.rs index 179840ca6b..276992520d 100644 --- a/src/http_server/endpoints/tts/list_tts_models.rs +++ b/src/http_server/endpoints/tts/list_tts_models.rs @@ -24,6 +24,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use chrono::{DateTime, Utc}; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Serialize)] pub struct TtsModelRecordForResponse { @@ -43,12 +44,6 @@ pub struct ListTtsModelsSuccessResponse { pub models: Vec, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum ListTtsModelsError { ServerError, @@ -78,19 +73,7 @@ impl ResponseError for ListTtsModelsError { ListTtsModelsError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/users/create_account.rs b/src/http_server/endpoints/users/create_account.rs index 71cc55c857..4f5a658d77 100644 --- a/src/http_server/endpoints/users/create_account.rs +++ b/src/http_server/endpoints/users/create_account.rs @@ -23,6 +23,7 @@ use crate::util::email_to_gravatar::email_to_gravatar; use crate::validations::username_reservations::is_reserved_username; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::validations::check_for_slurs::contains_slurs; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; const NEW_USER_ROLE: &'static str = "user"; @@ -39,12 +40,6 @@ pub struct CreateAccountSuccessResponse { pub success: bool, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum CreateAccountError { BadInput(String), @@ -74,19 +69,7 @@ impl ResponseError for CreateAccountError { CreateAccountError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/users/edit_profile.rs b/src/http_server/endpoints/users/edit_profile.rs index 220727c0cd..76f45f1c44 100644 --- a/src/http_server/endpoints/users/edit_profile.rs +++ b/src/http_server/endpoints/users/edit_profile.rs @@ -31,6 +31,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::common_queries::query_user_profile::select_user_profile_by_username; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -58,12 +59,6 @@ pub struct EditProfileSuccessResponse { pub success: bool, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum EditProfileError { BadInput(String), @@ -90,19 +85,7 @@ impl ResponseError for EditProfileError { EditProfileError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/users/get_profile.rs b/src/http_server/endpoints/users/get_profile.rs index 5de60e7496..bfd65a5506 100644 --- a/src/http_server/endpoints/users/get_profile.rs +++ b/src/http_server/endpoints/users/get_profile.rs @@ -28,6 +28,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; // TODO: This is duplicated in query_user_profile @@ -83,12 +84,6 @@ pub struct ProfileSuccessResponse { pub user: Option, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum ProfileError { ServerError, @@ -115,19 +110,7 @@ impl ResponseError for ProfileError { ProfileError::NotFound => "not found error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/src/http_server/endpoints/users/list_user_tts_inference_results.rs index e60dd83b04..0eceb177d1 100644 --- a/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -27,6 +27,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -40,12 +41,6 @@ pub struct ListTtsInferenceResultsForUserSuccessResponse { pub results: Vec, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum ListTtsInferenceResultsForUserError { ServerError, @@ -63,19 +58,7 @@ impl ResponseError for ListTtsInferenceResultsForUserError { ListTtsInferenceResultsForUserError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/users/list_user_tts_models.rs b/src/http_server/endpoints/users/list_user_tts_models.rs index 089c13da04..3024060668 100644 --- a/src/http_server/endpoints/users/list_user_tts_models.rs +++ b/src/http_server/endpoints/users/list_user_tts_models.rs @@ -26,6 +26,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -39,12 +40,6 @@ pub struct ListTtsModelsForUserSuccessResponse { pub models: Vec, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum ListTtsModelsForUserError { ServerError, @@ -62,19 +57,7 @@ impl ResponseError for ListTtsModelsForUserError { ListTtsModelsForUserError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs index ce5a17ada7..14a27a268d 100644 --- a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs @@ -27,6 +27,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use actix_web::web::Path; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -40,12 +41,6 @@ pub struct ListW2lInferenceResultsForUserSuccessResponse { pub results: Vec, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum ListW2lInferenceResultsForUserError { ServerError, @@ -63,19 +58,7 @@ impl ResponseError for ListW2lInferenceResultsForUserError { ListW2lInferenceResultsForUserError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/users/list_user_w2l_templates.rs b/src/http_server/endpoints/users/list_user_w2l_templates.rs index c1eef4c609..30b13dd5c0 100644 --- a/src/http_server/endpoints/users/list_user_w2l_templates.rs +++ b/src/http_server/endpoints/users/list_user_w2l_templates.rs @@ -26,6 +26,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use actix_web::web::Path; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -39,12 +40,6 @@ pub struct ListW2lTemplatesForUserSuccessResponse { pub templates: Vec, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum ListW2lTemplatesForUserError { ServerError, @@ -62,19 +57,7 @@ impl ResponseError for ListW2lTemplatesForUserError { ListW2lTemplatesForUserError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/users/login.rs b/src/http_server/endpoints/users/login.rs index 07200fa014..71d8258a93 100644 --- a/src/http_server/endpoints/users/login.rs +++ b/src/http_server/endpoints/users/login.rs @@ -21,6 +21,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct LoginRequest { @@ -33,12 +34,6 @@ pub struct LoginSuccessResponse { pub success: bool, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum LoginError { WrongCredentials, @@ -59,19 +54,7 @@ impl ResponseError for LoginError { LoginError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/users/logout.rs b/src/http_server/endpoints/users/logout.rs index 6a5f3e9960..32ee04d7de 100644 --- a/src/http_server/endpoints/users/logout.rs +++ b/src/http_server/endpoints/users/logout.rs @@ -22,18 +22,13 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Serialize)] pub struct LogoutSuccessResponse { pub success: bool, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum LogoutError { ServerError, @@ -51,19 +46,7 @@ impl ResponseError for LogoutError { LogoutError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/users/session_info.rs b/src/http_server/endpoints/users/session_info.rs index ae2060c9d4..76697f4678 100644 --- a/src/http_server/endpoints/users/session_info.rs +++ b/src/http_server/endpoints/users/session_info.rs @@ -23,6 +23,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Serialize)] pub struct UserInfo { @@ -64,12 +65,6 @@ pub struct SessionInfoSuccessResponse { pub user: Option, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum SessionInfoError { ServerError, @@ -87,19 +82,7 @@ impl ResponseError for SessionInfoError { SessionInfoError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs index 7eac4fbba7..310c0db496 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l.rs @@ -19,6 +19,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct InferW2lRequest { @@ -32,12 +33,6 @@ pub struct InferW2lSuccessResponse { pub success: bool, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum InferW2lError { BadInput(String), @@ -58,19 +53,7 @@ impl ResponseError for InferW2lError { InferW2lError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 83602a62bb..a396e216a3 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -28,6 +28,7 @@ use sqlx::error::Error::Database; use std::io::Write; use std::sync::Arc; use uuid::Uuid; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; const BUCKET_AUDIO_FILE_NAME : &'static str = "input_audio_file"; const BUCKET_IMAGE_FILE_NAME: &'static str = "input_image_file"; @@ -51,12 +52,6 @@ pub struct InferW2lWithUploadSuccessResponse { pub inference_job_token: String, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum InferW2lWithUploadError { BadInput(String), @@ -80,19 +75,7 @@ impl ResponseError for InferW2lWithUploadError { InferW2lWithUploadError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs index f81aa5b217..5bb5b29128 100644 --- a/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs +++ b/src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs @@ -21,6 +21,7 @@ use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::validations::check_for_slurs::contains_slurs; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct UploadW2lTemplateRequest { @@ -38,12 +39,6 @@ pub struct UploadW2lTemplateSuccessResponse { pub job_token: String, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum UploadW2lTemplateError { BadInput(String), @@ -67,19 +62,7 @@ impl ResponseError for UploadW2lTemplateError { UploadW2lTemplateError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs index 01a1a41f65..6e0ef10c9e 100644 --- a/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs +++ b/src/http_server/endpoints/w2l/get_w2l_inference_job_status.rs @@ -25,6 +25,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -55,12 +56,6 @@ pub struct GetW2lInferenceStatusSuccessResponse { pub state: W2lInferenceJobStatusForResponse, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum GetW2lInferenceStatusError { ServerError, @@ -96,19 +91,7 @@ impl ResponseError for GetW2lInferenceStatusError { GetW2lInferenceStatusError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/w2l/get_w2l_result.rs b/src/http_server/endpoints/w2l/get_w2l_result.rs index 94a2b6d96f..a66dc82f66 100644 --- a/src/http_server/endpoints/w2l/get_w2l_result.rs +++ b/src/http_server/endpoints/w2l/get_w2l_result.rs @@ -41,12 +41,6 @@ pub struct GetW2lResultSuccessResponse { pub result: W2lResultRecordForResponse, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum GetW2lResultError { ServerError, diff --git a/src/http_server/endpoints/w2l/get_w2l_template.rs b/src/http_server/endpoints/w2l/get_w2l_template.rs index 31d69441d1..4dafbfad82 100644 --- a/src/http_server/endpoints/w2l/get_w2l_template.rs +++ b/src/http_server/endpoints/w2l/get_w2l_template.rs @@ -25,6 +25,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -38,12 +39,6 @@ pub struct GetW2lTemplateSuccessResponse { pub template: W2lTemplateRecordForResponse, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum GetW2lTemplateError { ServerError, @@ -64,19 +59,7 @@ impl ResponseError for GetW2lTemplateError { GetW2lTemplateError::NotFound=> "not found".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs index 2a498b9912..cc3b996759 100644 --- a/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs +++ b/src/http_server/endpoints/w2l/get_w2l_upload_template_job_status.rs @@ -25,6 +25,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -50,12 +51,6 @@ pub struct GetW2lUploadTemplateStatusSuccessResponse { pub state: W2lUploadTemplateJobStatusForResponse, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum GetW2lUploadTemplateStatusError { ServerError, @@ -85,19 +80,7 @@ impl ResponseError for GetW2lUploadTemplateStatusError { GetW2lUploadTemplateStatusError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/w2l/list_w2l_templates.rs b/src/http_server/endpoints/w2l/list_w2l_templates.rs index c055b01454..197a8a12d4 100644 --- a/src/http_server/endpoints/w2l/list_w2l_templates.rs +++ b/src/http_server/endpoints/w2l/list_w2l_templates.rs @@ -25,6 +25,7 @@ use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; use chrono::{DateTime, Utc}; use crate::common_queries::list_w2l_templates::{list_w2l_templates, W2lTemplateRecordForList}; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Serialize)] pub struct ListW2lTemplatesSuccessResponse { @@ -32,12 +33,6 @@ pub struct ListW2lTemplatesSuccessResponse { pub templates: Vec, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum ListW2lTemplatesError { ServerError, @@ -55,19 +50,7 @@ impl ResponseError for ListW2lTemplatesError { ListW2lTemplatesError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } diff --git a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs index 206f527d29..46cf707786 100644 --- a/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs +++ b/src/http_server/endpoints/w2l/set_w2l_template_mod_approval.rs @@ -21,6 +21,7 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -38,12 +39,6 @@ pub struct SetW2lTemplateModApprovalSuccessResponse { pub success: bool, } -#[derive(Serialize)] -pub struct ErrorResponse { - pub success: bool, - pub error_reason: String, -} - #[derive(Debug, Display)] pub enum SetW2lTemplateModApprovalError { BadInput(String), @@ -70,19 +65,7 @@ impl ResponseError for SetW2lTemplateModApprovalError { SetW2lTemplateModApprovalError::ServerError => "server error".to_string(), }; - let response = ErrorResponse { - success: false, - error_reason, - }; - - let body = match serde_json::to_string(&response) { - Ok(json) => json, - Err(_) => "{}".to_string(), - }; - - HttpResponseBuilder::new(self.status_code()) - .set_header(header::CONTENT_TYPE, "application/json") - .body(body) + to_simple_json_error(&error_reason, self.status_code()) } } From c9852699535bcccc39ba1932001b95d2c19c7ab5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 00:11:21 -0400 Subject: [PATCH 0369/1042] list bans endpoint (not tested) --- migrations/2021-06-09-010200_ip_bans/up.sql | 4 +- .../endpoints/moderation/list_ip_bans.rs | 155 ++++++++++++++++++ src/main.rs | 6 + 3 files changed, 163 insertions(+), 2 deletions(-) diff --git a/migrations/2021-06-09-010200_ip_bans/up.sql b/migrations/2021-06-09-010200_ip_bans/up.sql index ae200476df..6c71635b98 100644 --- a/migrations/2021-06-09-010200_ip_bans/up.sql +++ b/migrations/2021-06-09-010200_ip_bans/up.sql @@ -7,7 +7,7 @@ CREATE TABLE ip_address_bans ( id BIGINT(20) NOT NULL AUTO_INCREMENT, -- The IP address to ban - ip_address VARCHAR(40) DEFAULT NULL, + ip_address VARCHAR(40) NOT NULL, -- Maybe the target user. maybe_target_user_token VARCHAR(32) DEFAULT NULL, @@ -16,7 +16,7 @@ CREATE TABLE ip_address_bans ( mod_user_token VARCHAR(32) NOT NULL, -- Optional notes about the ban. - mod_notes TEXT DEFAULT NULL, + mod_notes TEXT NOT NULL, -- Moderator can set an expire time (optional). -- If not set, the ban is indefinite diff --git a/src/http_server/endpoints/moderation/list_ip_bans.rs b/src/http_server/endpoints/moderation/list_ip_bans.rs index e69de29bb2..d34a68d096 100644 --- a/src/http_server/endpoints/moderation/list_ip_bans.rs +++ b/src/http_server/endpoints/moderation/list_ip_bans.rs @@ -0,0 +1,155 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +#[derive(Deserialize)] +pub struct ListIpBansRequest { + delete: bool, +} + +#[derive(Serialize)] +pub struct ListIpBansResponse { + pub success: bool, + pub ip_address_bans: Vec, +} + +#[derive(Serialize)] +pub struct IpBanRecordForList { + pub ip_address: String, + pub maybe_target_user_token: Option, + pub maybe_target_username: Option, + pub mod_user_token: String, + pub mod_notes: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug, Display)] +pub enum ListIpBansError { + BadInput(String), + ServerError, + Unauthorized, +} + +impl ResponseError for ListIpBansError { + fn status_code(&self) -> StatusCode { + match *self { + ListIpBansError::BadInput(_) => StatusCode::BAD_REQUEST, + ListIpBansError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + ListIpBansError::Unauthorized => StatusCode::UNAUTHORIZED, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListIpBansError::BadInput(reason) => reason.to_string(), + ListIpBansError::ServerError => "server error".to_string(), + ListIpBansError::Unauthorized => "unauthorized".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn list_ip_bans_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListIpBansError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(ListIpBansError::Unauthorized); + } + }; + + if !user_session.can_ban_users { + warn!("user is not allowed to delete bans: {}", user_session.user_token); + return Err(ListIpBansError::Unauthorized); + } + + // NB: Lookup failure is Err(RowNotFound). + let maybe_results = sqlx::query_as!( + IpBanRecordForList, + r#" +SELECT + ip_bans.ip_address, + ip_bans.maybe_target_user_token, + users.username as maybe_target_username, + ip_bans.mod_user_token, + ip_bans.mod_notes, + ip_bans.created_at, + ip_bans.updated_at +FROM + ip_address_bans AS ip_bans +LEFT OUTER JOIN users + ON ip_bans.maybe_target_user_token = users.token + "#, + ) + .fetch_all(&server_state.mysql_pool) + .await; + + let results : Vec = match maybe_results { + Ok(results) => { + info!("Results length: {}", results.len()); + results + }, + Err(err) => { + match err { + RowNotFound => { + Vec::new() + }, + _ => { + warn!("list ip bans db error: {:?}", err); + return Err(ListIpBansError::ServerError); + } + } + } + }; + + let response = ListIpBansResponse { + success: true, + ip_address_bans: results, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListIpBansError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/main.rs b/src/main.rs index 2ace0fcce6..dbf508e495 100644 --- a/src/main.rs +++ b/src/main.rs @@ -79,6 +79,7 @@ use crate::http_server::endpoints::w2l::delete_w2l_result::delete_w2l_inference_ use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; use crate::http_server::endpoints::moderation::add_ip_ban::add_ip_ban_handler; use crate::http_server::endpoints::moderation::delete_ip_ban::delete_ip_ban_handler; +use crate::http_server::endpoints::moderation::list_ip_bans::list_ip_bans_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -283,6 +284,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> web::scope("/moderation") .service( web::scope("/ip_bans") + .service( + web::resource("/list") + .route(web::get().to(list_ip_bans_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::resource("/add") .route(web::post().to(add_ip_ban_handler)) From 1a341c570577af0ea156ab6b043fc1f32f1a665f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 00:31:30 -0400 Subject: [PATCH 0370/1042] move moderation handler files --- .../moderation/{ => ip_bans}/add_ip_ban.rs | 0 .../moderation/{ => ip_bans}/delete_ip_ban.rs | 0 .../moderation/{ => ip_bans}/list_ip_bans.rs | 0 .../endpoints/moderation/ip_bans/mod.rs | 3 ++ src/http_server/endpoints/moderation/mod.rs | 8 ++--- .../moderation/{ => user_roles}/list_mods.rs | 0 .../moderation/{ => user_roles}/list_roles.rs | 0 .../endpoints/moderation/user_roles/mod.rs | 3 ++ .../{ => user_roles}/set_user_role.rs | 0 src/main.rs | 36 +++++++++---------- 10 files changed, 26 insertions(+), 24 deletions(-) rename src/http_server/endpoints/moderation/{ => ip_bans}/add_ip_ban.rs (100%) rename src/http_server/endpoints/moderation/{ => ip_bans}/delete_ip_ban.rs (100%) rename src/http_server/endpoints/moderation/{ => ip_bans}/list_ip_bans.rs (100%) create mode 100644 src/http_server/endpoints/moderation/ip_bans/mod.rs rename src/http_server/endpoints/moderation/{ => user_roles}/list_mods.rs (100%) rename src/http_server/endpoints/moderation/{ => user_roles}/list_roles.rs (100%) create mode 100644 src/http_server/endpoints/moderation/user_roles/mod.rs rename src/http_server/endpoints/moderation/{ => user_roles}/set_user_role.rs (100%) diff --git a/src/http_server/endpoints/moderation/add_ip_ban.rs b/src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs similarity index 100% rename from src/http_server/endpoints/moderation/add_ip_ban.rs rename to src/http_server/endpoints/moderation/ip_bans/add_ip_ban.rs diff --git a/src/http_server/endpoints/moderation/delete_ip_ban.rs b/src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs similarity index 100% rename from src/http_server/endpoints/moderation/delete_ip_ban.rs rename to src/http_server/endpoints/moderation/ip_bans/delete_ip_ban.rs diff --git a/src/http_server/endpoints/moderation/list_ip_bans.rs b/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs similarity index 100% rename from src/http_server/endpoints/moderation/list_ip_bans.rs rename to src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs diff --git a/src/http_server/endpoints/moderation/ip_bans/mod.rs b/src/http_server/endpoints/moderation/ip_bans/mod.rs new file mode 100644 index 0000000000..99c0855390 --- /dev/null +++ b/src/http_server/endpoints/moderation/ip_bans/mod.rs @@ -0,0 +1,3 @@ +pub mod add_ip_ban; +pub mod delete_ip_ban; +pub mod list_ip_bans; diff --git a/src/http_server/endpoints/moderation/mod.rs b/src/http_server/endpoints/moderation/mod.rs index b4a75705bf..d706c8568b 100644 --- a/src/http_server/endpoints/moderation/mod.rs +++ b/src/http_server/endpoints/moderation/mod.rs @@ -1,6 +1,2 @@ -pub mod add_ip_ban; -pub mod delete_ip_ban; -pub mod list_ip_bans; -pub mod list_mods; -pub mod list_roles; -pub mod set_user_role; \ No newline at end of file +pub mod ip_bans; +pub mod user_roles; \ No newline at end of file diff --git a/src/http_server/endpoints/moderation/list_mods.rs b/src/http_server/endpoints/moderation/user_roles/list_mods.rs similarity index 100% rename from src/http_server/endpoints/moderation/list_mods.rs rename to src/http_server/endpoints/moderation/user_roles/list_mods.rs diff --git a/src/http_server/endpoints/moderation/list_roles.rs b/src/http_server/endpoints/moderation/user_roles/list_roles.rs similarity index 100% rename from src/http_server/endpoints/moderation/list_roles.rs rename to src/http_server/endpoints/moderation/user_roles/list_roles.rs diff --git a/src/http_server/endpoints/moderation/user_roles/mod.rs b/src/http_server/endpoints/moderation/user_roles/mod.rs new file mode 100644 index 0000000000..9bb628e473 --- /dev/null +++ b/src/http_server/endpoints/moderation/user_roles/mod.rs @@ -0,0 +1,3 @@ +pub mod list_mods; +pub mod list_roles; +pub mod set_user_role; diff --git a/src/http_server/endpoints/moderation/set_user_role.rs b/src/http_server/endpoints/moderation/user_roles/set_user_role.rs similarity index 100% rename from src/http_server/endpoints/moderation/set_user_role.rs rename to src/http_server/endpoints/moderation/user_roles/set_user_role.rs diff --git a/src/main.rs b/src/main.rs index dbf508e495..aefdd0ec79 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,23 +37,41 @@ use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::default_route_404::default_route_404; use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::misc::enable_alpha::enable_alpha; +use crate::http_server::endpoints::moderation::ip_bans::add_ip_ban::add_ip_ban_handler; +use crate::http_server::endpoints::moderation::ip_bans::delete_ip_ban::delete_ip_ban_handler; +use crate::http_server::endpoints::moderation::ip_bans::list_ip_bans::list_ip_bans_handler; use crate::http_server::endpoints::root_index::get_root_index; +use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; +use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; use crate::http_server::endpoints::tts::enqueue_infer_tts::infer_tts_handler; use crate::http_server::endpoints::tts::enqueue_upload_tts_model::upload_tts_model_handler; +use crate::http_server::endpoints::tts::get_tts_inference_job_status::get_tts_inference_job_status_handler; +use crate::http_server::endpoints::tts::get_tts_model::get_tts_model_handler; +use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; +use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; use crate::http_server::endpoints::users::create_account::create_account_handler; +use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; use crate::http_server::endpoints::users::get_profile::get_profile_handler; +use crate::http_server::endpoints::users::list_user_tts_inference_results::list_user_tts_inference_results_handler; +use crate::http_server::endpoints::users::list_user_tts_models::list_user_tts_models_handler; use crate::http_server::endpoints::users::list_user_w2l_inference_results::list_user_w2l_inference_results_handler; use crate::http_server::endpoints::users::list_user_w2l_templates::list_user_w2l_templates_handler; use crate::http_server::endpoints::users::login::login_handler; use crate::http_server::endpoints::users::logout::logout_handler; use crate::http_server::endpoints::users::session_info::session_info_handler; +use crate::http_server::endpoints::w2l::delete_w2l_result::delete_w2l_inference_result_handler; +use crate::http_server::endpoints::w2l::delete_w2l_template::delete_w2l_template_handler; +use crate::http_server::endpoints::w2l::edit_w2l_template::edit_w2l_template_handler; use crate::http_server::endpoints::w2l::enqueue_infer_w2l::infer_w2l_handler; use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; use crate::http_server::endpoints::w2l::enqueue_upload_w2l_template::upload_w2l_template_handler; +use crate::http_server::endpoints::w2l::get_w2l_inference_job_status::get_w2l_inference_job_status_handler; use crate::http_server::endpoints::w2l::get_w2l_result::get_w2l_inference_result_handler; use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handler; +use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler; +use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::server_state::{ServerState, EnvConfig}; @@ -62,24 +80,6 @@ use log::{info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; -use crate::http_server::endpoints::users::list_user_tts_inference_results::list_user_tts_inference_results_handler; -use crate::http_server::endpoints::users::list_user_tts_models::list_user_tts_models_handler; -use crate::http_server::endpoints::tts::get_tts_result::get_tts_inference_result_handler; -use crate::http_server::endpoints::tts::get_tts_model::get_tts_model_handler; -use crate::http_server::endpoints::tts::get_tts_inference_job_status::get_tts_inference_job_status_handler; -use crate::http_server::endpoints::w2l::get_w2l_inference_job_status::get_w2l_inference_job_status_handler; -use crate::http_server::endpoints::tts::get_tts_upload_model_job_status::get_tts_upload_model_job_status_handler; -use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler; -use crate::http_server::endpoints::users::edit_profile::edit_profile_handler; -use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler; -use crate::http_server::endpoints::w2l::delete_w2l_template::delete_w2l_template_handler; -use crate::http_server::endpoints::w2l::edit_w2l_template::edit_w2l_template_handler; -use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; -use crate::http_server::endpoints::w2l::delete_w2l_result::delete_w2l_inference_result_handler; -use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; -use crate::http_server::endpoints::moderation::add_ip_ban::add_ip_ban_handler; -use crate::http_server::endpoints::moderation::delete_ip_ban::delete_ip_ban_handler; -use crate::http_server::endpoints::moderation::list_ip_bans::list_ip_bans_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO From 03d8b9ada2d627e3d863ecb5b0dd05b25ab44af2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 00:54:35 -0400 Subject: [PATCH 0371/1042] list user roles endpoint (untested) --- src/common_queries/list_user_roles.rs | 154 ++++++++++++++++++ src/common_queries/mod.rs | 1 + .../moderation/user_roles/list_roles.rs | 119 ++++++++++++++ src/main.rs | 9 + 4 files changed, 283 insertions(+) create mode 100644 src/common_queries/list_user_roles.rs diff --git a/src/common_queries/list_user_roles.rs b/src/common_queries/list_user_roles.rs new file mode 100644 index 0000000000..31261b605a --- /dev/null +++ b/src/common_queries/list_user_roles.rs @@ -0,0 +1,154 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::database_helpers::boolean_converters::{nullable_i8_to_optional_bool, i8_to_bool}; +use crate::util::anyhow_result::AnyhowResult; +use log::{warn, info}; +use sqlx::MySqlPool; + +#[derive(Serialize)] +pub struct UserRoleForList { + slug: String, + name: String, + + can_use_tts: bool, + can_use_w2l: bool, + can_delete_own_tts_results: bool, + can_delete_own_w2l_results: bool, + can_delete_own_account: bool, + + can_upload_tts_models: bool, + can_upload_w2l_templates: bool, + can_delete_own_tts_models: bool, + can_delete_own_w2l_templates: bool, + + can_approve_w2l_templates: bool, + can_edit_other_users_profiles: bool, + can_edit_other_users_tts_models: bool, + can_edit_other_users_w2l_templates: bool, + can_delete_other_users_tts_models: bool, + can_delete_other_users_tts_results: bool, + can_delete_other_users_w2l_templates: bool, + can_delete_other_users_w2l_results: bool, + can_ban_users: bool, + can_delete_users: bool, + + created_at : DateTime, + updated_at : DateTime, +} + +pub struct RawDbUserRoleForList { + slug: String, + name: String, + + can_use_tts: i8, + can_use_w2l: i8, + can_delete_own_tts_results: i8, + can_delete_own_w2l_results: i8, + can_delete_own_account: i8, + + can_upload_tts_models: i8, + can_upload_w2l_templates: i8, + can_delete_own_tts_models: i8, + can_delete_own_w2l_templates: i8, + + can_approve_w2l_templates: i8, + can_edit_other_users_profiles: i8, + can_edit_other_users_tts_models: i8, + can_edit_other_users_w2l_templates: i8, + can_delete_other_users_tts_models: i8, + can_delete_other_users_tts_results: i8, + can_delete_other_users_w2l_templates: i8, + can_delete_other_users_w2l_results: i8, + can_ban_users: i8, + can_delete_users: i8, + + created_at : DateTime, + updated_at : DateTime, +} + +pub async fn list_user_roles( + mysql_pool: &MySqlPool, +) -> AnyhowResult> { + info!("listing user roles"); + let maybe_user_roles = sqlx::query_as!( + RawDbUserRoleForList, + r#" +SELECT + slug, + name, + + can_use_tts, + can_use_w2l, + can_delete_own_tts_results, + can_delete_own_w2l_results, + can_delete_own_account, + + can_upload_tts_models, + can_upload_w2l_templates, + can_delete_own_tts_models, + can_delete_own_w2l_templates, + + can_approve_w2l_templates, + can_edit_other_users_profiles, + can_edit_other_users_tts_models, + can_edit_other_users_w2l_templates, + can_delete_other_users_tts_models, + can_delete_other_users_tts_results, + can_delete_other_users_w2l_templates, + can_delete_other_users_w2l_results, + can_ban_users, + can_delete_users, + + created_at, + updated_at + +FROM user_roles + "#, + ) + .fetch_all(mysql_pool) + .await; + + let user_roles : Vec = match maybe_user_roles { + Ok(roles) => roles, + Err(err) => { + warn!("Error: {:?}", err); + match err { + RowNotFound => Vec::new(), + _ => { + warn!("user role query error: {:?}", err); + return Err(anyhow!("error querying user roles")); + } + } + } + }; + + Ok(user_roles.into_iter() + .map(|ur| { + UserRoleForList { + slug: ur.slug.clone(), + name: ur.name.clone(), + can_use_tts: i8_to_bool(ur.can_use_tts), + can_use_w2l: i8_to_bool(ur.can_use_w2l), + can_delete_own_tts_results: i8_to_bool(ur.can_delete_own_tts_results), + can_delete_own_w2l_results: i8_to_bool(ur.can_delete_own_w2l_results), + can_delete_own_account: i8_to_bool(ur.can_delete_own_account), + can_upload_tts_models: i8_to_bool(ur.can_upload_tts_models), + can_upload_w2l_templates: i8_to_bool(ur.can_upload_w2l_templates), + can_delete_own_tts_models: i8_to_bool(ur.can_delete_own_tts_models), + can_delete_own_w2l_templates: i8_to_bool(ur.can_delete_own_w2l_templates), + can_approve_w2l_templates: i8_to_bool(ur.can_approve_w2l_templates), + can_edit_other_users_profiles: i8_to_bool(ur.can_edit_other_users_profiles), + can_edit_other_users_tts_models: i8_to_bool(ur.can_edit_other_users_tts_models), + can_edit_other_users_w2l_templates: i8_to_bool(ur.can_edit_other_users_w2l_templates), + can_delete_other_users_tts_models: i8_to_bool(ur.can_delete_other_users_tts_models), + can_delete_other_users_tts_results: i8_to_bool(ur.can_delete_other_users_tts_results), + can_delete_other_users_w2l_templates: i8_to_bool(ur.can_delete_other_users_w2l_templates), + can_delete_other_users_w2l_results: i8_to_bool(ur.can_delete_other_users_w2l_results), + can_ban_users: i8_to_bool(ur.can_ban_users), + can_delete_users: i8_to_bool(ur.can_delete_users), + created_at: ur.created_at.clone(), + updated_at: ur.updated_at.clone(), + } + }) + .collect::>()) +} diff --git a/src/common_queries/mod.rs b/src/common_queries/mod.rs index 0223f21e0d..794cb9bf6d 100644 --- a/src/common_queries/mod.rs +++ b/src/common_queries/mod.rs @@ -2,6 +2,7 @@ pub mod badges; pub mod firehose_publisher; pub mod list_tts_inference_results; pub mod list_tts_models; +pub mod list_user_roles; pub mod list_w2l_inference_results; pub mod list_w2l_templates; pub mod query_tts_model; diff --git a/src/http_server/endpoints/moderation/user_roles/list_roles.rs b/src/http_server/endpoints/moderation/user_roles/list_roles.rs index e69de29bb2..46946eff9e 100644 --- a/src/http_server/endpoints/moderation/user_roles/list_roles.rs +++ b/src/http_server/endpoints/moderation/user_roles/list_roles.rs @@ -0,0 +1,119 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::common_queries::list_user_roles::UserRoleForList; +use crate::common_queries::list_user_roles::list_user_roles; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + +#[derive(Deserialize)] +pub struct ListUserRolesRequest { + delete: bool, +} + +#[derive(Serialize)] +pub struct ListUserRolesResponse { + pub success: bool, + pub user_roles: Vec, +} + +#[derive(Debug, Display)] +pub enum ListUserRolesError { + BadInput(String), + ServerError, + Unauthorized, +} + +impl ResponseError for ListUserRolesError { + fn status_code(&self) -> StatusCode { + match *self { + ListUserRolesError::BadInput(_) => StatusCode::BAD_REQUEST, + ListUserRolesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + ListUserRolesError::Unauthorized => StatusCode::UNAUTHORIZED, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListUserRolesError::BadInput(reason) => reason.to_string(), + ListUserRolesError::ServerError => "server error".to_string(), + ListUserRolesError::Unauthorized => "unauthorized".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn list_user_roles_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListUserRolesError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(ListUserRolesError::Unauthorized); + } + }; + + // TODO: Add new permission for this. + if !user_session.can_ban_users { + warn!("user is not allowed to view user roles: {}", user_session.user_token); + return Err(ListUserRolesError::Unauthorized); + } + + let maybe_user_roles = + list_user_roles(&server_state.mysql_pool).await; + + let user_roles = match maybe_user_roles { + Ok(results) => results, + Err(e) => { + warn!("Error querying user roles: {:?}", e); + return Err(ListUserRolesError::ServerError); + } + }; + + let response = ListUserRolesResponse { + success: true, + user_roles, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListUserRolesError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/main.rs b/src/main.rs index aefdd0ec79..0f15de12f8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -80,6 +80,7 @@ use log::{info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; +use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user_roles_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -300,6 +301,14 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service( + web::scope("/roles") + .service( + web::resource("/list") + .route(web::get().to(list_user_roles_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) ) // ==================== TTS ==================== .service( From d603b832cf00bc7a32372fe60e55e2bc7276acd6 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 02:24:55 -0400 Subject: [PATCH 0372/1042] add staff list endpoint (untested) --- .../moderation/ip_bans/list_ip_bans.rs | 6 - .../moderation/user_roles/list_mods.rs | 0 .../moderation/user_roles/list_roles.rs | 6 - .../moderation/user_roles/list_staff.rs | 149 ++++++++++++++++++ .../endpoints/moderation/user_roles/mod.rs | 2 +- src/main.rs | 6 + 6 files changed, 156 insertions(+), 13 deletions(-) delete mode 100644 src/http_server/endpoints/moderation/user_roles/list_mods.rs create mode 100644 src/http_server/endpoints/moderation/user_roles/list_staff.rs diff --git a/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs b/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs index d34a68d096..36aaef6854 100644 --- a/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs +++ b/src/http_server/endpoints/moderation/ip_bans/list_ip_bans.rs @@ -25,11 +25,6 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -#[derive(Deserialize)] -pub struct ListIpBansRequest { - delete: bool, -} - #[derive(Serialize)] pub struct ListIpBansResponse { pub success: bool, @@ -76,7 +71,6 @@ impl ResponseError for ListIpBansError { pub async fn list_ip_bans_handler( http_request: HttpRequest, - request: web::Json, server_state: web::Data> ) -> Result { diff --git a/src/http_server/endpoints/moderation/user_roles/list_mods.rs b/src/http_server/endpoints/moderation/user_roles/list_mods.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/http_server/endpoints/moderation/user_roles/list_roles.rs b/src/http_server/endpoints/moderation/user_roles/list_roles.rs index 46946eff9e..836f0b91bc 100644 --- a/src/http_server/endpoints/moderation/user_roles/list_roles.rs +++ b/src/http_server/endpoints/moderation/user_roles/list_roles.rs @@ -27,11 +27,6 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -#[derive(Deserialize)] -pub struct ListUserRolesRequest { - delete: bool, -} - #[derive(Serialize)] pub struct ListUserRolesResponse { pub success: bool, @@ -67,7 +62,6 @@ impl ResponseError for ListUserRolesError { pub async fn list_user_roles_handler( http_request: HttpRequest, - request: web::Json, server_state: web::Data> ) -> Result { diff --git a/src/http_server/endpoints/moderation/user_roles/list_staff.rs b/src/http_server/endpoints/moderation/user_roles/list_staff.rs new file mode 100644 index 0000000000..3c21a63977 --- /dev/null +++ b/src/http_server/endpoints/moderation/user_roles/list_staff.rs @@ -0,0 +1,149 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use chrono::{DateTime, Utc}; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::server_state::ServerState; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::validations::check_for_slurs::contains_slurs; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; + + +#[derive(Serialize)] +pub struct ListStaffResponse { + pub success: bool, + pub staff: Vec, +} + +#[derive(Serialize)] +pub struct StaffRecordForList { + pub user_token: String, + pub username: String, + pub display_name: String, + pub user_role_slug: String, + pub user_role_name: String, +} + +#[derive(Debug, Display)] +pub enum ListStaffError { + BadInput(String), + ServerError, + Unauthorized, +} + +impl ResponseError for ListStaffError { + fn status_code(&self) -> StatusCode { + match *self { + ListStaffError::BadInput(_) => StatusCode::BAD_REQUEST, + ListStaffError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + ListStaffError::Unauthorized => StatusCode::UNAUTHORIZED, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + ListStaffError::BadInput(reason) => reason.to_string(), + ListStaffError::ServerError => "server error".to_string(), + ListStaffError::Unauthorized => "unauthorized".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn list_staff_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListStaffError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(ListStaffError::Unauthorized); + } + }; + + // TODO: This is not the correct permission. + if !user_session.can_ban_users { + warn!("user is not allowed to delete bans: {}", user_session.user_token); + return Err(ListStaffError::Unauthorized); + } + + // NB: Lookup failure is Err(RowNotFound). + let maybe_results = sqlx::query_as!( + StaffRecordForList, + r#" +SELECT + users.token as user_token, + users.username, + users.display_name, + user_roles.slug as user_role_slug, + user_roles.name as user_role_name +FROM + users +JOIN user_roles + ON users.user_role_slug = user_roles.slug +WHERE + user_roles.slug != 'user' + "#, + ) + .fetch_all(&server_state.mysql_pool) + .await; + + let results : Vec = match maybe_results { + Ok(results) => { + info!("Results length: {}", results.len()); + results + }, + Err(err) => { + match err { + RowNotFound => { + Vec::new() + }, + _ => { + warn!("list staff db error: {:?}", err); + return Err(ListStaffError::ServerError); + } + } + } + }; + + let response = ListStaffResponse { + success: true, + staff: results, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| ListStaffError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/src/http_server/endpoints/moderation/user_roles/mod.rs b/src/http_server/endpoints/moderation/user_roles/mod.rs index 9bb628e473..8e10abf233 100644 --- a/src/http_server/endpoints/moderation/user_roles/mod.rs +++ b/src/http_server/endpoints/moderation/user_roles/mod.rs @@ -1,3 +1,3 @@ -pub mod list_mods; +pub mod list_staff; pub mod list_roles; pub mod set_user_role; diff --git a/src/main.rs b/src/main.rs index 0f15de12f8..c3ea3e2c4a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -81,6 +81,7 @@ use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user_roles_handler; +use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -283,6 +284,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> // ==================== MODERATOR ==================== .service( web::scope("/moderation") + .service( + web::resource("/staff") + .route(web::get().to(list_staff_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service( web::scope("/ip_bans") .service( From 4fe93050997599271510bfe351af840bf3c82306 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 02:38:20 -0400 Subject: [PATCH 0373/1042] set user role handler (untested) --- src/common_queries/list_user_roles.rs | 46 +++--- .../moderation/user_roles/set_user_role.rs | 146 ++++++++++++++++++ src/main.rs | 6 + 3 files changed, 175 insertions(+), 23 deletions(-) diff --git a/src/common_queries/list_user_roles.rs b/src/common_queries/list_user_roles.rs index 31261b605a..a606f8b9fa 100644 --- a/src/common_queries/list_user_roles.rs +++ b/src/common_queries/list_user_roles.rs @@ -7,33 +7,33 @@ use sqlx::MySqlPool; #[derive(Serialize)] pub struct UserRoleForList { - slug: String, - name: String, + pub slug: String, + pub name: String, - can_use_tts: bool, - can_use_w2l: bool, - can_delete_own_tts_results: bool, - can_delete_own_w2l_results: bool, - can_delete_own_account: bool, + pub can_use_tts: bool, + pub can_use_w2l: bool, + pub can_delete_own_tts_results: bool, + pub can_delete_own_w2l_results: bool, + pub can_delete_own_account: bool, - can_upload_tts_models: bool, - can_upload_w2l_templates: bool, - can_delete_own_tts_models: bool, - can_delete_own_w2l_templates: bool, + pub can_upload_tts_models: bool, + pub can_upload_w2l_templates: bool, + pub can_delete_own_tts_models: bool, + pub can_delete_own_w2l_templates: bool, - can_approve_w2l_templates: bool, - can_edit_other_users_profiles: bool, - can_edit_other_users_tts_models: bool, - can_edit_other_users_w2l_templates: bool, - can_delete_other_users_tts_models: bool, - can_delete_other_users_tts_results: bool, - can_delete_other_users_w2l_templates: bool, - can_delete_other_users_w2l_results: bool, - can_ban_users: bool, - can_delete_users: bool, + pub can_approve_w2l_templates: bool, + pub can_edit_other_users_profiles: bool, + pub can_edit_other_users_tts_models: bool, + pub can_edit_other_users_w2l_templates: bool, + pub can_delete_other_users_tts_models: bool, + pub can_delete_other_users_tts_results: bool, + pub can_delete_other_users_w2l_templates: bool, + pub can_delete_other_users_w2l_results: bool, + pub can_ban_users: bool, + pub can_delete_users: bool, - created_at : DateTime, - updated_at : DateTime, + pub created_at : DateTime, + pub updated_at : DateTime, } pub struct RawDbUserRoleForList { diff --git a/src/http_server/endpoints/moderation/user_roles/set_user_role.rs b/src/http_server/endpoints/moderation/user_roles/set_user_role.rs index e69de29bb2..d07942f284 100644 --- a/src/http_server/endpoints/moderation/user_roles/set_user_role.rs +++ b/src/http_server/endpoints/moderation/user_roles/set_user_role.rs @@ -0,0 +1,146 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::cookie::Cookie; +use actix_web::dev::HttpResponseBuilder; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; +use crate::common_queries::query_user_profile::select_user_profile_by_username; +use crate::common_queries::list_user_roles::list_user_roles; +use crate::database_helpers::enums::{DownloadUrlType, CreatorSetVisibility, W2lTemplateType}; +use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::server_state::ServerState; +use crate::util::random_crockford_token::random_crockford_token; +use crate::validations::model_uploads::validate_model_title; +use crate::validations::passwords::validate_passwords; +use crate::validations::username::validate_username; +use derive_more::{Display, Error}; +use log::{info, warn, log}; +use regex::Regex; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::sync::Arc; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct SetUserRolePathInfo { + username: String, +} + +#[derive(Deserialize)] +pub struct SetUserRoleRequest { + user_role_slug: String, +} + +#[derive(Debug, Display)] +pub enum SetUserRoleError { + BadInput(String), + NotAuthorized, + NotFound, + ServerError, +} + +impl ResponseError for SetUserRoleError { + fn status_code(&self) -> StatusCode { + match *self { + SetUserRoleError::BadInput(_) => StatusCode::BAD_REQUEST, + SetUserRoleError::NotAuthorized => StatusCode::UNAUTHORIZED, + SetUserRoleError::NotFound => StatusCode::NOT_FOUND, + SetUserRoleError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + SetUserRoleError::BadInput(reason) => reason.to_string(), + SetUserRoleError::NotAuthorized => "unauthorized".to_string(), + SetUserRoleError::NotFound => "not found".to_string(), + SetUserRoleError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +pub async fn set_user_role_handler( + http_request: HttpRequest, + path: Path, + request: web::Json, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + SetUserRoleError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + warn!("not logged in"); + return Err(SetUserRoleError::NotAuthorized); + } + }; + + // TODO: This is not the correct permission + if !user_session.can_ban_users { + warn!("user is not allowed to change user roles: {}", user_session.user_token); + return Err(SetUserRoleError::NotAuthorized); + } + + // TODO: This is lazy and inefficient + let user_roles = list_user_roles(&server_state.mysql_pool) + .await + .map_err(|err| { + warn!("error listing roles: {:?}", err); + return SetUserRoleError::ServerError; + })?; + + let role_exists = user_roles.into_iter() + .find(|user_role| user_role.slug == request.user_role_slug) + .is_some(); + + if !role_exists { + return Err(SetUserRoleError::BadInput("invalid user role".to_string())); + } + + // TODO: Error *and* NotFound should be possible. + let target_user = select_user_profile_by_username(&path.username, &server_state.mysql_pool) + .await + .map_err(|err| { + warn!("error listing roles: {:?}", err); + return SetUserRoleError::ServerError; + })?; + + let query_result = sqlx::query!( + r#" +UPDATE users +SET + user_role_slug = ? +WHERE + token = ? +LIMIT 1 + "#, + &request.user_role_slug, + &target_user.user_token, + ) + .execute(&server_state.mysql_pool) + .await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("unable to update user role: {:?}", err); + return Err(SetUserRoleError::ServerError); + } + }; + + Ok(simple_json_success()) +} diff --git a/src/main.rs b/src/main.rs index c3ea3e2c4a..150d07d12e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -82,6 +82,7 @@ use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user_roles_handler; use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler; +use crate::http_server::endpoints::moderation::user_roles::set_user_role::set_user_role_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -314,6 +315,11 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()> .route(web::get().to(list_user_roles_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service( + web::resource("/{username}/edit") + .route(web::post().to(set_user_role_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) ) // ==================== TTS ==================== From af4c831aaa14df28fc7531e0384432b65c3f8817 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 03:23:56 -0400 Subject: [PATCH 0374/1042] sort key encrypter (so we don't leak how big our database is) --- Cargo.lock | 91 +++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/main.rs | 2 +- src/util/encrypted_sort_id.rs | 87 +++++++++++++++++++++++++++++++++ src/util/mod.rs | 1 + 5 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 src/util/encrypted_sort_id.rs diff --git a/Cargo.lock b/Cargo.lock index 325243bfda..9a72fa1aa2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -352,6 +352,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes-soft" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" +dependencies = [ + "cipher", + "opaque-debug", +] + [[package]] name = "ahash" version = "0.3.8" @@ -675,6 +685,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ + "block-padding", "generic-array", ] @@ -687,6 +698,22 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-modes" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a0e8073e8baa88212fb5823574c02ebccb395136ba9a164ab89379ec6072f0" +dependencies = [ + "block-padding", + "cipher", +] + +[[package]] +name = "block-padding" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" + [[package]] name = "blocking" version = "1.0.2" @@ -850,6 +877,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "cipher" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" +dependencies = [ + "generic-array", +] + [[package]] name = "concurrent-queue" version = "1.2.2" @@ -922,6 +958,15 @@ dependencies = [ "build_const", ] +[[package]] +name = "crc-any" +version = "2.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9950e91c5c444b0729f5f1b0aec76c523e01920ce828e37bccfa27803ff34e1" +dependencies = [ + "debug-helper", +] + [[package]] name = "crc32fast" version = "1.2.1" @@ -998,6 +1043,12 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" +[[package]] +name = "debug-helper" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4460596867846f73bddca51f7403b6a29f5315125be10a1640259b4db5b9494c" + [[package]] name = "derive_more" version = "0.99.14" @@ -1010,6 +1061,17 @@ dependencies = [ "syn", ] +[[package]] +name = "des" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b24e7c748888aa2fa8bce21d8c64a52efc810663285315ac7476f7197a982fae" +dependencies = [ + "byteorder", + "cipher", + "opaque-debug", +] + [[package]] name = "diesel" version = "1.4.6" @@ -1783,6 +1845,23 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "magic-crypt" +version = "3.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3c94f1281833c690f81e6a00c545063b4f034509d3af6d29b58d48e39aa64c9" +dependencies = [ + "aes-soft", + "base64 0.13.0", + "block-modes", + "crc-any", + "des", + "digest", + "md-5", + "sha2", + "tiger", +] + [[package]] name = "maplit" version = "1.0.2" @@ -3046,6 +3125,7 @@ dependencies = [ "jwt", "lazy_static", "log", + "magic-crypt", "md-5", "pulldown-cmark", "rand 0.8.3", @@ -3182,6 +3262,17 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "tiger" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443e531cbcf9de83258cfef70bcd56c91188de5819ebd4b19c85f589e0617005" +dependencies = [ + "block-buffer", + "byteorder", + "digest", +] + [[package]] name = "time" version = "0.1.44" diff --git a/Cargo.toml b/Cargo.toml index e2b7c33fe9..3b45184ee3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,6 +90,7 @@ jwt = "0.13.0" # Crypto bcrypt = "0.8" # password hashes hmac = "0.10.0" # cookies +magic-crypt = "3.1.8" # encrypting sort IDs for the frontend so we don't leak info md-5 = "0.9.1" # gravatar sha2 = "0.9.5" # file hashes, inference text hashes, etc. diff --git a/src/main.rs b/src/main.rs index 150d07d12e..2f2605eb33 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,9 +9,9 @@ #![allow(unused_variables)] #[macro_use] extern crate lazy_static; +#[macro_use] extern crate magic_crypt; #[macro_use] extern crate serde_derive; - pub const RESERVED_USERNAMES : &'static str = include_str!("../db/reserved_usernames.txt"); pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../db/reserved_usernames_including.txt"); pub const BANNED_SLURS : &'static str = include_str!("../db/banned_slurs.txt"); diff --git a/src/util/encrypted_sort_id.rs b/src/util/encrypted_sort_id.rs new file mode 100644 index 0000000000..81559ab8c0 --- /dev/null +++ b/src/util/encrypted_sort_id.rs @@ -0,0 +1,87 @@ + +use base64; +use magic_crypt::{MagicCryptTrait, MagicCrypt256}; +use magic_crypt::generic_array::typenum::U256; +use std::io::Cursor; +use magic_crypt::new_magic_crypt; +use crate::util::anyhow_result::AnyhowResult; + +// TODO: A protobuf would be more compact! +/// This gets encrypted and sent to the frontend as an opaque handle. +#[derive(Serialize, Deserialize)] +pub struct SortId { + pub before_entropy: u32, + pub column_id: u64, + pub after_entropy: u32, +} + +pub struct SortKeyCrypto { + crypt: MagicCrypt256, +} + +impl SortKeyCrypto { + pub fn new(secret: &str) -> Self { + Self { + crypt: new_magic_crypt!(secret, 256), + } + } + + pub fn encrypt_id(&self, id: u64) -> AnyhowResult { + let payload = SortId { + before_entropy: 1234, + column_id: id, + after_entropy: 1234, + }; + + let payload = serde_json::to_string(&payload)?; + + let mut reader = Cursor::new(payload); + let mut writer = Vec::new(); + + self.crypt.encrypt_reader_to_writer2::(&mut reader, &mut writer)?; + let encoded = base64::encode(&writer); + + Ok(encoded) + } + + pub fn decrypt_id(&self, base_64_payload: &str) -> AnyhowResult { + let payload = self.crypt.decrypt_base64_to_string(base_64_payload)?; + let payload = serde_json::from_str::(&payload)?; + Ok(payload.column_id) + } +} + +#[cfg(test)] +mod tests { + use crate::util::encrypted_sort_id::SortKeyCrypto; + + #[test] + fn encrypt_roundtrip() { + let sorter = SortKeyCrypto::new("secret"); + let encrypted = sorter.encrypt_id(1234).unwrap(); + let decrypted = sorter.decrypt_id(&encrypted).unwrap(); + assert_eq!(decrypted, 1234); + } + + #[test] + fn encrypt_lots_roundtrip_low_numbers() { + let sorter = SortKeyCrypto::new("secret"); + + for i in 0 .. 1000 { + let encrypted = sorter.encrypt_id(i).unwrap(); + let decrypted = sorter.decrypt_id(&encrypted).unwrap(); + assert_eq!(decrypted, i); + } + } + + #[test] + fn encrypt_lots_roundtrip_high_numbers() { + let sorter = SortKeyCrypto::new("secret"); + + for i in 100000000 .. 100005000 { + let encrypted = sorter.encrypt_id(i).unwrap(); + let decrypted = sorter.decrypt_id(&encrypted).unwrap(); + assert_eq!(decrypted, i); + } + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 3284b448c0..6ac0370ea4 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,5 +1,6 @@ pub mod anyhow_result; pub mod email_to_gravatar; +pub mod encrypted_sort_id; pub mod filesystem; pub mod hashing; pub mod markdown_to_html; From 84548abf76bc97ff065129d451afb4180afb621a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 03:29:15 -0400 Subject: [PATCH 0375/1042] entropy + tests --- src/util/encrypted_sort_id.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/util/encrypted_sort_id.rs b/src/util/encrypted_sort_id.rs index 81559ab8c0..65d75f10af 100644 --- a/src/util/encrypted_sort_id.rs +++ b/src/util/encrypted_sort_id.rs @@ -5,14 +5,14 @@ use magic_crypt::generic_array::typenum::U256; use std::io::Cursor; use magic_crypt::new_magic_crypt; use crate::util::anyhow_result::AnyhowResult; +use rand::RngCore; // TODO: A protobuf would be more compact! /// This gets encrypted and sent to the frontend as an opaque handle. #[derive(Serialize, Deserialize)] pub struct SortId { - pub before_entropy: u32, + pub entropy: u32, pub column_id: u64, - pub after_entropy: u32, } pub struct SortKeyCrypto { @@ -27,10 +27,11 @@ impl SortKeyCrypto { } pub fn encrypt_id(&self, id: u64) -> AnyhowResult { + let mut rng = rand::thread_rng(); + let payload = SortId { - before_entropy: 1234, + entropy: rng.next_u32(), column_id: id, - after_entropy: 1234, }; let payload = serde_json::to_string(&payload)?; @@ -54,6 +55,21 @@ impl SortKeyCrypto { #[cfg(test)] mod tests { use crate::util::encrypted_sort_id::SortKeyCrypto; + use std::collections::HashSet; + + #[test] + fn encrypt_entropy_means_no_duplicate_values() { + let sorter = SortKeyCrypto::new("secret"); + + let mut encrypted_tokens = HashSet::new(); + + for i in 0 .. 1000 { + let encrypted = sorter.encrypt_id(1234).unwrap(); + encrypted_tokens.insert(encrypted); + } + + assert_eq!(encrypted_tokens.len(), 1000); + } #[test] fn encrypt_roundtrip() { From c015f431b257c694380aec666154460adf45483f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 03:34:56 -0400 Subject: [PATCH 0376/1042] sort key crypto tool added to server --- src/main.rs | 14 +++++++++++--- src/server_state.rs | 9 ++++++--- src/util/encrypted_sort_id.rs | 4 ++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2f2605eb33..397b3c5ae6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -40,6 +40,9 @@ use crate::http_server::endpoints::misc::enable_alpha::enable_alpha; use crate::http_server::endpoints::moderation::ip_bans::add_ip_ban::add_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::delete_ip_ban::delete_ip_ban_handler; use crate::http_server::endpoints::moderation::ip_bans::list_ip_bans::list_ip_bans_handler; +use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user_roles_handler; +use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler; +use crate::http_server::endpoints::moderation::user_roles::set_user_role::set_user_role_handler; use crate::http_server::endpoints::root_index::get_root_index; use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; @@ -76,13 +79,11 @@ use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::server_state::{ServerState, EnvConfig}; use crate::shared_constants::{DEFAULT_RUST_LOG, DEFAULT_MYSQL_PASSWORD}; +use crate::util::encrypted_sort_id::SortKeyCrypto; use log::{info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; -use crate::http_server::endpoints::moderation::user_roles::list_roles::list_user_roles_handler; -use crate::http_server::endpoints::moderation::user_roles::list_staff::list_staff_handler; -use crate::http_server::endpoints::moderation::user_roles::set_user_role::set_user_role_handler; // TODO TODO TODO TODO // TODO TODO TODO TODO @@ -185,6 +186,12 @@ async fn main() -> AnyhowResult<()> { None, )?; + // NB: This secret really isn't too important. + // We can even rotate it without too much impact to users. + let sort_key_crypto_secret = + easyenv::get_env_string_or_default("SORT_KEY_SECRET", "webscale"); + let sort_key_crypto = SortKeyCrypto::new(&sort_key_crypto_secret); + let server_state = ServerState { env_config: EnvConfig { num_workers, @@ -201,6 +208,7 @@ async fn main() -> AnyhowResult<()> { private_bucket_client, public_bucket_client, audio_uploads_bucket_root, + sort_key_crypto, }; serve(server_state) diff --git a/src/server_state.rs b/src/server_state.rs index 97d0be1178..1968307d7a 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,8 +1,9 @@ -use crate::http_server::web_utils::cookie_manager::CookieManager; -use crate::http_server::web_utils::session_checker::SessionChecker; -use sqlx::MySqlPool; use crate::buckets::bucket_client::BucketClient; use crate::common_queries::firehose_publisher::FirehosePublisher; +use crate::http_server::web_utils::cookie_manager::CookieManager; +use crate::http_server::web_utils::session_checker::SessionChecker; +use crate::util::encrypted_sort_id::SortKeyCrypto; +use sqlx::MySqlPool; /// State that is injected into every endpoint. #[derive(Clone)] @@ -25,6 +26,8 @@ pub struct ServerState { /// Where to store audio uploads for w2l pub audio_uploads_bucket_root: String, + + pub sort_key_crypto: SortKeyCrypto, } #[derive(Clone)] diff --git a/src/util/encrypted_sort_id.rs b/src/util/encrypted_sort_id.rs index 65d75f10af..85f93bee76 100644 --- a/src/util/encrypted_sort_id.rs +++ b/src/util/encrypted_sort_id.rs @@ -15,6 +15,10 @@ pub struct SortId { pub column_id: u64, } +/// This exists so that we don't leak our database IDs. +/// If competitors see our IDs, they'll know how big our database is. +/// We encrypt our IDs so we don't leak "business secret information". +#[derive(Clone)] pub struct SortKeyCrypto { crypt: MagicCrypt256, } From adf3741d855a6ea0e342fdbd3951c325a781b374 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 04:52:03 -0400 Subject: [PATCH 0377/1042] wip untested dynamic query builder for tts results --- .../list_tts_inference_results.rs | 108 ++++++++++++++++++ .../users/list_user_tts_inference_results.rs | 5 + 2 files changed, 113 insertions(+) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index 7203058e95..27f78788da 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -257,3 +257,111 @@ WHERE Ok(maybe_results) } + +#[derive(sqlx::FromRow)] +pub struct RawInternalTtsRecord { + pub tts_result_id: i64, + pub tts_result_token: String, + + pub tts_model_token: Option, + pub raw_inference_text: String, + + pub maybe_creator_user_token : Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + + pub file_size_bytes : i64, + pub duration_millis : i64, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +async fn list_tts_inference_results_fixed ( + mysql_pool: &MySqlPool, + scope_creator_username: Option<&str>, + sort_ascending: bool, + block_mod_disabled : bool, + limit: u16, + offset: Option, +) -> AnyhowResult> { + info!("listing tts inference results"); + + // TODO/NB: Unfortunately SQLx can't statically typecheck this query + let mut query = r#" +SELECT + tts_results.id as tts_result_id, + tts_results.token as tts_result_token, + + tts_results.model_token as tts_model_token, + tts_results.raw_inference_text as raw_inference_text, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + tts_results.file_size_bytes, + tts_results.duration_millis, + tts_results.created_at, + tts_results.updated_at + +FROM tts_results +LEFT OUTER JOIN tts_models + ON tts_results.model_token = tts_models.token +LEFT OUTER JOIN users + ON tts_results.maybe_creator_user_token = users.token + "#.to_string(); + + let mut first_predicate_added = false; + + if let Some(offset) = offset { + if !first_predicate_added { + query.push_str(r#" WHERE tts_results.id < ? "#); + first_predicate_added = true; + } else { + query.push_str(r#" AND tts_results.id < ? "#); + } + } + + if let Some(username) = scope_creator_username { + if !first_predicate_added { + query.push_str(r#" WHERE users.username = ? "#); + first_predicate_added = true; + } else { + query.push_str(r#" AND users.username = ? "#); + } + } + + if block_mod_disabled { + if !first_predicate_added { + query.push_str(r#" + WHERE tts_results.user_deleted_at IS NULL + AND tts_results.mod_deleted_at IS NULL + "#); + first_predicate_added = true; + } else { + query.push_str(r#" + AND tts_results.user_deleted_at IS NULL + AND tts_results.mod_deleted_at IS NULL + "#); + } + } + + query.push_str(" LIMIT ? "); + + let mut query = sqlx::query_as::<_, RawInternalTtsRecord>(&query); + + if let Some(offset) = offset { + query = query.bind(offset); + } + + if let Some(username) = scope_creator_username { + query = query.bind(username); + } + + query = query.bind(limit); + + let results = query.fetch_all(mysql_pool) + .await?; + + Ok(results) +} diff --git a/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/src/http_server/endpoints/users/list_user_tts_inference_results.rs index 0eceb177d1..f2f0371133 100644 --- a/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -33,12 +33,16 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct GetProfilePathInfo { username: String, + sort_ascending: Option, + limit: Option, + cursor: Option, } #[derive(Serialize)] pub struct ListTtsInferenceResultsForUserSuccessResponse { pub success: bool, pub results: Vec, + pub next_cursor: Option, } #[derive(Debug, Display)] @@ -87,6 +91,7 @@ pub async fn list_user_tts_inference_results_handler( let response = ListTtsInferenceResultsForUserSuccessResponse { success: true, results, + next_cursor: Some("".to_string()), }; let body = serde_json::to_string(&response) From be40729ee80bdeefdd38716e964c8cda232a425b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 18:45:00 -0400 Subject: [PATCH 0378/1042] order by clause (not tested) --- src/common_queries/list_tts_inference_results.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index 27f78788da..d6cfcd0edb 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -346,6 +346,12 @@ LEFT OUTER JOIN users } } + if sort_ascending { + query.push_str(" ORDER BY tts_results.id ASC "); + } else { + query.push_str(" ORDER BY tts_results.id DESC "); + } + query.push_str(" LIMIT ? "); let mut query = sqlx::query_as::<_, RawInternalTtsRecord>(&query); From 50b694f8e6302a07ad591572fea744c1621a322f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 18:58:37 -0400 Subject: [PATCH 0379/1042] paginated queries --- .../list_tts_inference_results.rs | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index d6cfcd0edb..23d45851dc 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -4,6 +4,14 @@ use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; +#[derive(Serialize)] +pub struct TtsInferenceListPage { + pub inference_records: Vec, + pub sort_ascending: bool, + pub first_id: Option, + pub last_id: Option, +} + #[derive(Serialize)] pub struct TtsInferenceRecordForList { pub tts_result_token: String, @@ -263,7 +271,7 @@ pub struct RawInternalTtsRecord { pub tts_result_id: i64, pub tts_result_token: String, - pub tts_model_token: Option, + pub tts_model_token: String, pub raw_inference_text: String, pub maybe_creator_user_token : Option, @@ -276,7 +284,58 @@ pub struct RawInternalTtsRecord { pub updated_at: DateTime, } -async fn list_tts_inference_results_fixed ( +/// Paginated queries. +pub async fn list_tts_inference_page( + mysql_pool: &MySqlPool, + scope_creator_username: Option<&str>, + sort_ascending: bool, + block_mod_disabled : bool, + limit: u16, + offset: Option, +) -> AnyhowResult { + + let inference_results = list_tts_inference_results_query( + mysql_pool, + scope_creator_username, + sort_ascending, + block_mod_disabled, + limit, + offset + ).await?; + + let mut first_id = inference_results.first() + .map(|raw_result| raw_result.tts_result_id); + + let mut last_id = inference_results.last() + .map(|raw_result| raw_result.tts_result_id); + + let inference_results = inference_results + .iter() + .map(|r| { + TtsInferenceRecordForList { + tts_result_token: r.tts_result_token.clone(), + tts_model_token: r.tts_model_token.clone(), + raw_inference_text: r.raw_inference_text.clone(), + maybe_creator_user_token: r.maybe_creator_user_token.clone(), + maybe_creator_username: r.maybe_creator_username.clone(), + maybe_creator_display_name: r.maybe_creator_display_name.clone(), + file_size_bytes: if r.file_size_bytes > 0 { r.file_size_bytes as u32 } else { 0 }, + duration_millis: if r.duration_millis > 0 { r.duration_millis as u32 } else { 0 }, + created_at: r.created_at, + updated_at: r.updated_at, + } + }) + .collect::>(); + + Ok(TtsInferenceListPage { + inference_records: inference_results, + sort_ascending, + first_id, + last_id, + }) +} + +async fn list_tts_inference_results_query ( mysql_pool: &MySqlPool, scope_creator_username: Option<&str>, sort_ascending: bool, From 0713de86585e58f69c8685c45c839cd9dde56731 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 19:18:30 -0400 Subject: [PATCH 0380/1042] pagination from query string (untested) --- .../list_tts_inference_results.rs | 4 +- .../users/list_user_tts_inference_results.rs | 76 +++++++++++++++---- 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index 23d45851dc..ec5f29208f 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -291,7 +291,7 @@ pub async fn list_tts_inference_page( sort_ascending: bool, block_mod_disabled : bool, limit: u16, - offset: Option, + offset: Option, ) -> AnyhowResult { let inference_results = list_tts_inference_results_query( @@ -341,7 +341,7 @@ async fn list_tts_inference_results_query ( sort_ascending: bool, block_mod_disabled : bool, limit: u16, - offset: Option, + offset: Option, ) -> AnyhowResult> { info!("listing tts inference results"); diff --git a/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/src/http_server/endpoints/users/list_user_tts_inference_results.rs index f2f0371133..73d4ddc6e3 100644 --- a/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -4,12 +4,12 @@ use actix_web::cookie::Cookie; use actix_web::dev::HttpResponseBuilder; use actix_web::error::ResponseError; use actix_web::http::StatusCode; -use actix_web::web::Path; +use actix_web::web::{Path, Query}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::common_queries::list_tts_inference_results::TtsInferenceRecordForList; -use crate::common_queries::list_tts_inference_results::list_tts_inference_results; +use crate::common_queries::list_tts_inference_results::list_tts_inference_page; use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; @@ -31,18 +31,23 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] -pub struct GetProfilePathInfo { - username: String, - sort_ascending: Option, - limit: Option, - cursor: Option, +pub struct ListTtsInferenceResultsForUserPathInfo { + pub username: String, +} + +#[derive(Deserialize)] +pub struct ListTtsInferenceResultsForUserQuery { + pub sort_ascending: Option, + pub limit: Option, + pub cursor: Option, } #[derive(Serialize)] pub struct ListTtsInferenceResultsForUserSuccessResponse { pub success: bool, pub results: Vec, - pub next_cursor: Option, + pub cursor_next: Option, + pub cursor_previous: Option, } #[derive(Debug, Display)] @@ -68,19 +73,39 @@ impl ResponseError for ListTtsInferenceResultsForUserError { pub async fn list_user_tts_inference_results_handler( http_request: HttpRequest, - path: Path, + path: Path, + query: Query, server_state: web::Data> ) -> Result { info!("Fetching inference results for user: {}", &path.username); - let query_results = list_tts_inference_results( + let mod_disabled = false; + + let limit = query.limit.unwrap_or(25); + let limit = std::cmp::max(limit, 100); + + let cursor = if let Some(cursor) = query.cursor.as_deref() { + let cursor = server_state.sort_key_crypto.decrypt_id(cursor) + .map_err(|e| { + warn!("crypto error: {:?}", e); + return ListTtsInferenceResultsForUserError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let query_results = list_tts_inference_page( &server_state.mysql_pool, Some(path.username.as_ref()), - false + false, + mod_disabled, + limit, + cursor, ).await; - let results = match query_results { + let results_page = match query_results { Ok(results) => results, Err(e) => { warn!("Query error: {:?}", e); @@ -88,10 +113,33 @@ pub async fn list_user_tts_inference_results_handler( } }; + let cursor_next = if let Some(id) = results_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + return ListTtsInferenceResultsForUserError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let cursor_previous = if let Some(id) = results_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + return ListTtsInferenceResultsForUserError::ServerError + })?; + Some(cursor) + } else { + None + }; + let response = ListTtsInferenceResultsForUserSuccessResponse { success: true, - results, - next_cursor: Some("".to_string()), + results: results_page.inference_records, + cursor_next, + cursor_previous, }; let body = serde_json::to_string(&response) From 76e3f96a86334abe1154d688edc4f89189e477d1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 19:58:23 -0400 Subject: [PATCH 0381/1042] fix tts dl job --- src/job_queries/tts_download_job_queries.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/job_queries/tts_download_job_queries.rs b/src/job_queries/tts_download_job_queries.rs index c36e63b8ea..478726e345 100644 --- a/src/job_queries/tts_download_job_queries.rs +++ b/src/job_queries/tts_download_job_queries.rs @@ -224,6 +224,7 @@ SET description_rendered_html = '', creator_user_token = ?, creator_ip_address_creation = ?, + creator_ip_address_last_update = ?, original_download_url = ?, private_bucket_hash = ?, private_bucket_object_name = ?, @@ -233,6 +234,7 @@ SET job.title.to_string(), job.creator_user_token.clone(), job.creator_ip_address.clone(), + job.creator_ip_address.clone(), job.download_url.clone(), private_bucket_hash.to_string(), private_bucket_object_name.to_string(), From ea64e212b30dcf6062433af29cd48c997ec4cef1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 20:49:32 -0400 Subject: [PATCH 0382/1042] sorting both ways seems to work --- .../list_tts_inference_results.rs | 38 +++++++++++++++++-- .../users/list_user_tts_inference_results.rs | 10 ++++- src/util/encrypted_sort_id.rs | 15 +++++++- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index ec5f29208f..89b2601873 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -289,6 +289,7 @@ pub async fn list_tts_inference_page( mysql_pool: &MySqlPool, scope_creator_username: Option<&str>, sort_ascending: bool, + cursor_is_previous: bool, block_mod_disabled : bool, limit: u16, offset: Option, @@ -298,6 +299,7 @@ pub async fn list_tts_inference_page( mysql_pool, scope_creator_username, sort_ascending, + cursor_is_previous, block_mod_disabled, limit, offset @@ -339,6 +341,7 @@ async fn list_tts_inference_results_query ( mysql_pool: &MySqlPool, scope_creator_username: Option<&str>, sort_ascending: bool, + cursor_is_previous: bool, block_mod_disabled : bool, limit: u16, offset: Option, @@ -370,14 +373,39 @@ LEFT OUTER JOIN users ON tts_results.maybe_creator_user_token = users.token "#.to_string(); + let mut sort_ascending = sort_ascending; // NB: We may need to change the direction. + let mut reverse_results = false; + let mut first_predicate_added = false; if let Some(offset) = offset { if !first_predicate_added { - query.push_str(r#" WHERE tts_results.id < ? "#); + query.push_str(" WHERE "); first_predicate_added = true; } else { - query.push_str(r#" AND tts_results.id < ? "#); + query.push_str(" AND "); + } + + // TODO: Cleanup, builder class, tests. + + if sort_ascending { + if cursor_is_previous { + // NB: We're searching backwards. + query.push_str(" tts_results.id < ? "); + sort_ascending = !sort_ascending; + reverse_results = true; + } else { + query.push_str(" tts_results.id > ? "); + } + } else { + if cursor_is_previous { + // NB: We're searching backwards. + query.push_str(" tts_results.id > ? "); + sort_ascending = !sort_ascending; + reverse_results = true; + } else { + query.push_str(" tts_results.id < ? "); + } } } @@ -425,8 +453,12 @@ LEFT OUTER JOIN users query = query.bind(limit); - let results = query.fetch_all(mysql_pool) + let mut results = query.fetch_all(mysql_pool) .await?; + if reverse_results { + results.reverse() + } + Ok(results) } diff --git a/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/src/http_server/endpoints/users/list_user_tts_inference_results.rs index 73d4ddc6e3..f8d4549f7a 100644 --- a/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -40,6 +40,7 @@ pub struct ListTtsInferenceResultsForUserQuery { pub sort_ascending: Option, pub limit: Option, pub cursor: Option, + pub cursor_is_previous: Option, } #[derive(Serialize)] @@ -83,7 +84,11 @@ pub async fn list_user_tts_inference_results_handler( let mod_disabled = false; let limit = query.limit.unwrap_or(25); - let limit = std::cmp::max(limit, 100); + //let limit = std::cmp::max(limit, 100); + + let sort_ascending = query.sort_ascending.unwrap_or(false); + + let cursor_is_previous = query.cursor_is_previous.unwrap_or(false); let cursor = if let Some(cursor) = query.cursor.as_deref() { let cursor = server_state.sort_key_crypto.decrypt_id(cursor) @@ -99,7 +104,8 @@ pub async fn list_user_tts_inference_results_handler( let query_results = list_tts_inference_page( &server_state.mysql_pool, Some(path.username.as_ref()), - false, + sort_ascending, + cursor_is_previous, mod_disabled, limit, cursor, diff --git a/src/util/encrypted_sort_id.rs b/src/util/encrypted_sort_id.rs index 85f93bee76..6c10953054 100644 --- a/src/util/encrypted_sort_id.rs +++ b/src/util/encrypted_sort_id.rs @@ -6,6 +6,7 @@ use std::io::Cursor; use magic_crypt::new_magic_crypt; use crate::util::anyhow_result::AnyhowResult; use rand::RngCore; +use base64::{Config, CharacterSet}; // TODO: A protobuf would be more compact! /// This gets encrypted and sent to the frontend as an opaque handle. @@ -21,12 +22,16 @@ pub struct SortId { #[derive(Clone)] pub struct SortKeyCrypto { crypt: MagicCrypt256, + base64_config: Config, } impl SortKeyCrypto { pub fn new(secret: &str) -> Self { + let base64_config = Config::new(CharacterSet::UrlSafe, false) + .decode_allow_trailing_bits(true); Self { crypt: new_magic_crypt!(secret, 256), + base64_config, } } @@ -44,14 +49,20 @@ impl SortKeyCrypto { let mut writer = Vec::new(); self.crypt.encrypt_reader_to_writer2::(&mut reader, &mut writer)?; - let encoded = base64::encode(&writer); + let encoded = base64::encode_config(&writer, self.base64_config.clone()); Ok(encoded) } pub fn decrypt_id(&self, base_64_payload: &str) -> AnyhowResult { - let payload = self.crypt.decrypt_base64_to_string(base_64_payload)?; + //let payload = self.crypt.decrypt_base64_to_string(base_64_payload)?; + + let decoded_bytes= base64::decode_config(base_64_payload, self.base64_config)?; + let decrypted_bytes = self.crypt.decrypt_bytes_to_bytes(&decoded_bytes)?; + + let payload = String::from_utf8(decrypted_bytes)?; let payload = serde_json::from_str::(&payload)?; + Ok(payload.column_id) } } From e7c3db608d0a7c8d4d5582cfb27414ba57c318d4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 21:10:17 -0400 Subject: [PATCH 0383/1042] wip query builder --- .../list_tts_inference_results.rs | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index 89b2601873..ad47ea3744 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -462,3 +462,134 @@ LEFT OUTER JOIN users Ok(results) } + +/// These are very difficult queries, so this helps. +pub struct QueryBuilder { + scope_creator_username: Option, + include_mod_disabled_results: bool, + sort_ascending: bool, + offset: Option, + limit: u16, + cursor_is_reversed: bool, + result_set_requires_reverse_sort: bool, +} + +impl QueryBuilder { + pub fn new() -> Self { + Self { + scope_creator_username: None, + include_mod_disabled_results: false, + sort_ascending: false, + offset: None, + limit: 5, + cursor_is_reversed: false, + result_set_requires_reverse_sort: false, + } + } + + pub fn scope_creator_username(mut self, scope_creator_username: Option<&str>) -> Self { + self.scope_creator_username = scope_creator_username.map(|u| u.to_string()); + self + } + + pub fn include_mod_disabled_results(mut self, include_mod_disabled_results: bool) -> Self { + self.include_mod_disabled_results = include_mod_disabled_results; + self + } + + pub fn build_predicates(&self) -> String { + // NB: Reverse cursors require us to invert the sort direction. + let mut sort_ascending = self.sort_ascending; + // NB: If the sort direction is artificially reversed, we'll restore the result order. + let mut reverse_results = false; + + let mut first_predicate_added = false; + + let mut query = "".to_string(); + + if let Some(offset) = self.offset { + if !first_predicate_added { + query.push_str(" WHERE"); + first_predicate_added = true; + } else { + query.push_str(" AND"); + } + + if sort_ascending { + if self.cursor_is_reversed { + // NB: We're searching backwards. + query.push_str(" tts_results.id < ?"); + sort_ascending = !sort_ascending; + reverse_results = true; + } else { + query.push_str(" tts_results.id > ?"); + } + } else { + if self.cursor_is_reversed { + // NB: We're searching backwards. + query.push_str(" tts_results.id > ?"); + sort_ascending = !sort_ascending; + reverse_results = true; + } else { + query.push_str(" tts_results.id < ?"); + } + } + } + + if let Some(username) = self.scope_creator_username.as_deref() { + if !first_predicate_added { + query.push_str(" WHERE users.username = ?"); + first_predicate_added = true; + } else { + query.push_str(" AND users.username = ?"); + } + } + + if !self.include_mod_disabled_results { + if !first_predicate_added { + query.push_str(" WHERE tts_results.user_deleted_at IS NULL"); + query.push_str(" AND tts_results.mod_deleted_at IS NULL"); + first_predicate_added = true; + } else { + query.push_str(" AND tts_results.user_deleted_at IS NULL"); + query.push_str(" AND tts_results.mod_deleted_at IS NULL"); + } + } + + if sort_ascending { + query.push_str(" ORDER BY tts_results.id ASC"); + } else { + query.push_str(" ORDER BY tts_results.id DESC"); + } + + query.push_str(" LIMIT ?"); + + query + } +} + +#[cfg(test)] +mod tests { + use crate::common_queries::list_tts_inference_results::QueryBuilder; + + #[test] + fn predicates_without_scoping() { + let query_builder = QueryBuilder::new(); + + assert_eq!(&query_builder.build_predicates(), + " WHERE tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_including_deleted() { + let query_builder = QueryBuilder::new() + .include_mod_disabled_results(true); + + assert_eq!(&query_builder.build_predicates(), + " ORDER BY tts_results.id DESC \ + LIMIT ?"); + } +} \ No newline at end of file From 981b2f746a46618522a80c3371375062120a00bc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 21:23:49 -0400 Subject: [PATCH 0384/1042] another predicate --- .../list_tts_inference_results.rs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index ad47ea3744..533a92f572 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -497,6 +497,11 @@ impl QueryBuilder { self } + pub fn sort_ascending(mut self, sort_ascending: bool) -> Self { + self.sort_ascending = sort_ascending; + self + } + pub fn build_predicates(&self) -> String { // NB: Reverse cursors require us to invert the sort direction. let mut sort_ascending = self.sort_ascending; @@ -583,6 +588,19 @@ mod tests { LIMIT ?"); } + #[test] + fn predicates_scoped_to_user() { + let query_builder = QueryBuilder::new() + .scope_creator_username(Some("echelon")); + + assert_eq!(&query_builder.build_predicates(), + " WHERE users.username = ? \ + AND tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id DESC \ + LIMIT ?"); + } + #[test] fn predicates_including_deleted() { let query_builder = QueryBuilder::new() @@ -592,4 +610,16 @@ mod tests { " ORDER BY tts_results.id DESC \ LIMIT ?"); } + + #[test] + fn predicates_sort_ascending() { + let query_builder = QueryBuilder::new() + .sort_ascending(true); + + assert_eq!(&query_builder.build_predicates(), + " WHERE tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id ASC \ + LIMIT ?"); + } } \ No newline at end of file From e7dc15d8328ca724cb5040ea70eb2a71642c9bf4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 21:26:14 -0400 Subject: [PATCH 0385/1042] offset predicate --- .../list_tts_inference_results.rs | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index 533a92f572..10291017b5 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -502,6 +502,11 @@ impl QueryBuilder { self } + pub fn offset(mut self, offset: Option) -> Self { + self.offset = offset; + self + } + pub fn build_predicates(&self) -> String { // NB: Reverse cursors require us to invert the sort direction. let mut sort_ascending = self.sort_ascending; @@ -622,4 +627,31 @@ mod tests { ORDER BY tts_results.id ASC \ LIMIT ?"); } -} \ No newline at end of file + + #[test] + fn predicates_offset() { + let query_builder = QueryBuilder::new() + .offset(Some(100)); + + assert_eq!(&query_builder.build_predicates(), + " WHERE tts_results.id < ? \ + AND tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_offset_and_sort_ascending() { + let query_builder = QueryBuilder::new() + .sort_ascending(true) + .offset(Some(100)); + + assert_eq!(&query_builder.build_predicates(), + " WHERE tts_results.id > ? \ + AND tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id ASC \ + LIMIT ?"); + } +} From 806339d29503d45e5a947a9d9ceab5dc59aa6b90 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 21:33:15 -0400 Subject: [PATCH 0386/1042] tests for all predicates --- .../list_tts_inference_results.rs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index 10291017b5..b3041f837a 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -507,6 +507,16 @@ impl QueryBuilder { self } + pub fn limit(mut self, limit: u16) -> Self { + self.limit = limit; + self + } + + pub fn cursor_is_reversed(mut self, cursor_is_reversed: bool) -> Self { + self.cursor_is_reversed = cursor_is_reversed; + self + } + pub fn build_predicates(&self) -> String { // NB: Reverse cursors require us to invert the sort direction. let mut sort_ascending = self.sort_ascending; @@ -654,4 +664,62 @@ mod tests { ORDER BY tts_results.id ASC \ LIMIT ?"); } + + #[test] + fn predicates_limit() { + let query_builder = QueryBuilder::new() + .limit(15); + + // NB: Does not change the query itself! Just the downstream binding. + assert_eq!(&query_builder.build_predicates(), + " WHERE tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_cursor_is_reversed_without_cursor() { + let query_builder = QueryBuilder::new() + .cursor_is_reversed(true); + + // NB: Without a cursor, nothing happens. + assert_eq!(&query_builder.build_predicates(), + " WHERE tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_offset_cursor_is_reversed() { + let query_builder = QueryBuilder::new() + .offset(Some(100)) + .cursor_is_reversed(true); + + // NB: This will change the sort order and greater/less than direction! + assert_eq!(&query_builder.build_predicates(), + " WHERE tts_results.id > ? \ + AND tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id ASC \ + LIMIT ?"); + } + + + #[test] + fn predicates_offset_cursor_is_reversed_sort_ascending() { + let query_builder = QueryBuilder::new() + .offset(Some(100)) + .cursor_is_reversed(true) + .sort_ascending(true); + + // NB: This will change the sort order and greater/less than direction! + assert_eq!(&query_builder.build_predicates(), + " WHERE tts_results.id < ? \ + AND tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id DESC \ + LIMIT ?"); + } } From 9aac3beb9c635b552def90e58e0afa458f74d445 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 21:47:51 -0400 Subject: [PATCH 0387/1042] query mechanics in the builder --- .../list_tts_inference_results.rs | 90 +++++++++++++++++-- src/shared_constants.rs | 3 + 2 files changed, 85 insertions(+), 8 deletions(-) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index b3041f837a..5c4933f9da 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -3,6 +3,7 @@ use chrono::{DateTime, Utc}; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; +use crate::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; #[derive(Serialize)] pub struct TtsInferenceListPage { @@ -471,7 +472,6 @@ pub struct QueryBuilder { offset: Option, limit: u16, cursor_is_reversed: bool, - result_set_requires_reverse_sort: bool, } impl QueryBuilder { @@ -481,9 +481,8 @@ impl QueryBuilder { include_mod_disabled_results: false, sort_ascending: false, offset: None, - limit: 5, + limit: DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE, cursor_is_reversed: false, - result_set_requires_reverse_sort: false, } } @@ -517,11 +516,70 @@ impl QueryBuilder { self } + /// Perform the query based on the set predicates. + pub async fn perform_query( + &self, + mysql_pool: &MySqlPool + ) -> AnyhowResult> { + + let query = self.build_query_string(); + let mut query = sqlx::query_as::<_, RawInternalTtsRecord>(&query); + + // NB: The following bindings must match the order of the query builder !! + + if let Some(offset) = self.offset { + query = query.bind(offset); + } + + if let Some(username) = self.scope_creator_username.as_deref() { + query = query.bind(username); + } + + query = query.bind(self.limit); + + let mut results = query.fetch_all(mysql_pool) + .await?; + + if self.cursor_is_reversed { + results.reverse() + } + + Ok(results) + } + + pub fn build_query_string(&self) -> String { + // TODO/NB: Unfortunately SQLx can't statically typecheck this query + let mut query = r#" +SELECT + tts_results.id as tts_result_id, + tts_results.token as tts_result_token, + + tts_results.model_token as tts_model_token, + tts_results.raw_inference_text as raw_inference_text, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + tts_results.file_size_bytes, + tts_results.duration_millis, + tts_results.created_at, + tts_results.updated_at + +FROM tts_results +LEFT OUTER JOIN tts_models + ON tts_results.model_token = tts_models.token +LEFT OUTER JOIN users + ON tts_results.maybe_creator_user_token = users.token + "#.to_string(); + + query.push_str(&self.build_predicates()); + query + } + pub fn build_predicates(&self) -> String { // NB: Reverse cursors require us to invert the sort direction. let mut sort_ascending = self.sort_ascending; - // NB: If the sort direction is artificially reversed, we'll restore the result order. - let mut reverse_results = false; let mut first_predicate_added = false; @@ -540,7 +598,6 @@ impl QueryBuilder { // NB: We're searching backwards. query.push_str(" tts_results.id < ?"); sort_ascending = !sort_ascending; - reverse_results = true; } else { query.push_str(" tts_results.id > ?"); } @@ -549,7 +606,6 @@ impl QueryBuilder { // NB: We're searching backwards. query.push_str(" tts_results.id > ?"); sort_ascending = !sort_ascending; - reverse_results = true; } else { query.push_str(" tts_results.id < ?"); } @@ -706,7 +762,6 @@ mod tests { LIMIT ?"); } - #[test] fn predicates_offset_cursor_is_reversed_sort_ascending() { let query_builder = QueryBuilder::new() @@ -722,4 +777,23 @@ mod tests { ORDER BY tts_results.id DESC \ LIMIT ?"); } + + #[test] + fn predicates_limit_scope_user_offset_cursor_is_reversed_sort_ascending() { + let query_builder = QueryBuilder::new() + .limit(1000) + .scope_creator_username(Some("pikachu")) + .offset(Some(100)) + .cursor_is_reversed(true) + .sort_ascending(true); + + // NB: This will change the sort order and greater/less than direction! + assert_eq!(&query_builder.build_predicates(), + " WHERE tts_results.id < ? \ + AND users.username = ? \ + AND tts_results.user_deleted_at IS NULL \ + AND tts_results.mod_deleted_at IS NULL \ + ORDER BY tts_results.id DESC \ + LIMIT ?"); + } } diff --git a/src/shared_constants.rs b/src/shared_constants.rs index 0e5581111c..fdf33c4087 100644 --- a/src/shared_constants.rs +++ b/src/shared_constants.rs @@ -7,3 +7,6 @@ pub const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=war /// The default MySql password for use in development pub const DEFAULT_MYSQL_PASSWORD: &'static str = "mysql://storyteller:password@localhost/storyteller"; + +/// The number of results we return by default for paginated queries. +pub const DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE: u16 = 25; From 75df0cda4a294cc3ecbe0e7e5cd97148a0f0e196 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 22:02:21 -0400 Subject: [PATCH 0388/1042] cleanup (let's test if it works...) --- .../list_tts_inference_results.rs | 531 +++--------------- .../users/list_user_tts_inference_results.rs | 32 +- 2 files changed, 100 insertions(+), 463 deletions(-) diff --git a/src/common_queries/list_tts_inference_results.rs b/src/common_queries/list_tts_inference_results.rs index 5c4933f9da..f2bf3440f6 100644 --- a/src/common_queries/list_tts_inference_results.rs +++ b/src/common_queries/list_tts_inference_results.rs @@ -1,15 +1,19 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use crate::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; -use crate::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; #[derive(Serialize)] pub struct TtsInferenceListPage { pub inference_records: Vec, pub sort_ascending: bool, + + /// ID of the first record in `inference_records`. pub first_id: Option, + + /// ID of the last record in `inference_records`. pub last_id: Option, } @@ -34,438 +38,10 @@ pub struct TtsInferenceRecordForList { pub updated_at: DateTime, } -struct RawTtsInferenceRecordForList { - pub tts_result_token: String, // from field `tts_results.token` - - pub tts_model_token: String, - pub raw_inference_text: String, - - pub maybe_creator_user_token: Option, - pub maybe_creator_username: Option, - pub maybe_creator_display_name: Option, - - pub file_size_bytes: i32, - pub duration_millis: i32, - - //pub model_is_mod_approved: i8, // needs convert - //pub maybe_mod_user_token: Option, - - pub created_at: DateTime, - pub updated_at: DateTime, -} - -pub async fn list_tts_inference_results( - mysql_pool: &MySqlPool, - scope_creator_username: Option<&str>, - block_mod_disabled_models: bool -) -> AnyhowResult> { - - let maybe_results = match scope_creator_username { - Some(username) => { - list_tts_inference_results_creator_scoped(mysql_pool, username, block_mod_disabled_models) - .await - }, - None => { - list_tts_inference_results_for_all_creators(mysql_pool, block_mod_disabled_models) - .await - }, - }; - - let results : Vec = match maybe_results { - Ok(results) => { - info!("Results length: {}", results.len()); - results - }, - Err(err) => { - warn!("Error: {:?}", err); - - match err { - RowNotFound => { - return Ok(Vec::new()); - }, - _ => { - warn!("tts inference result list query error: {:?}", err); - return Err(anyhow!("tts inference result list query error")); - } - } - } - }; - - Ok(results.into_iter() - .map(|ir| { - TtsInferenceRecordForList { - tts_result_token: ir.tts_result_token.clone(), - - tts_model_token: ir.tts_model_token.clone(), - raw_inference_text: ir.raw_inference_text.clone(), - - maybe_creator_user_token: ir.maybe_creator_user_token.clone(), - maybe_creator_username: ir.maybe_creator_username.clone(), - maybe_creator_display_name: ir.maybe_creator_display_name.clone(), - //model_is_mod_approved: if ir.model_is_mod_approved == 0 { false } else { true }, - - file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, - duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, - - created_at: ir.created_at.clone(), - updated_at: ir.updated_at.clone(), - } - }) - .collect::>()) -} - -async fn list_tts_inference_results_for_all_creators( - mysql_pool: &MySqlPool, - block_mod_disabled_models : bool -) -> AnyhowResult> { - // TODO: There has to be a better way. - // Sqlx doesn't like anything except string literals. - let maybe_results = if block_mod_disabled_models { - info!("listing tts inference results for everyone; mod-approved only"); - sqlx::query_as!( - RawTtsInferenceRecordForList, - r#" -SELECT - tts_results.token as tts_result_token, - - tts_results.model_token as tts_model_token, - tts_results.raw_inference_text as raw_inference_text, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - tts_results.file_size_bytes, - tts_results.duration_millis, - tts_results.created_at, - tts_results.updated_at - -FROM tts_results -LEFT OUTER JOIN tts_models - ON tts_results.model_token = tts_models.token -LEFT OUTER JOIN users - ON tts_results.maybe_creator_user_token = users.token -WHERE - tts_models.is_locked_from_use IS FALSE - AND tts_results.user_deleted_at IS NULL - AND tts_results.mod_deleted_at IS NULL - "#) - .fetch_all(mysql_pool) - .await? - } else { - info!("listing tts inference results for everyone; all"); - sqlx::query_as!( - RawTtsInferenceRecordForList, - r#" -SELECT - tts_results.token as tts_result_token, - - tts_results.model_token as tts_model_token, - tts_results.raw_inference_text as raw_inference_text, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - tts_results.file_size_bytes, - tts_results.duration_millis, - tts_results.created_at, - tts_results.updated_at - -FROM tts_results -LEFT OUTER JOIN tts_models - ON tts_results.model_token = tts_models.token -LEFT OUTER JOIN users - ON tts_results.maybe_creator_user_token = users.token -WHERE - tts_results.user_deleted_at IS NULL - AND tts_results.mod_deleted_at IS NULL - "#) - .fetch_all(mysql_pool) - .await? - }; - - Ok(maybe_results) -} - -async fn list_tts_inference_results_creator_scoped( - mysql_pool: &MySqlPool, - scope_creator_username: &str, - block_mod_disabled : bool -) -> AnyhowResult> { - // TODO: There has to be a better way. - // Sqlx doesn't like anything except string literals. - let maybe_results = if block_mod_disabled { - info!("listing tts inference results for user `{}`; mod-approved only", scope_creator_username); - sqlx::query_as!( - RawTtsInferenceRecordForList, - r#" -SELECT - tts_results.token as tts_result_token, - - tts_results.model_token as tts_model_token, - tts_results.raw_inference_text as raw_inference_text, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - tts_results.file_size_bytes, - tts_results.duration_millis, - tts_results.created_at, - tts_results.updated_at - -FROM tts_results -LEFT OUTER JOIN tts_models - ON tts_results.model_token = tts_models.token -LEFT OUTER JOIN users - ON tts_results.maybe_creator_user_token = users.token -WHERE - users.username = ? - AND tts_models.is_locked_from_use IS FALSE - AND tts_results.user_deleted_at IS NULL - AND tts_results.mod_deleted_at IS NULL - "#, - scope_creator_username) - .fetch_all(mysql_pool) - .await? - } else { - info!("listing tts inference results for user `{}`; all", scope_creator_username); - sqlx::query_as!( - RawTtsInferenceRecordForList, - r#" -SELECT - tts_results.token as tts_result_token, - - tts_results.model_token as tts_model_token, - tts_results.raw_inference_text as raw_inference_text, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - tts_results.file_size_bytes, - tts_results.duration_millis, - tts_results.created_at, - tts_results.updated_at - -FROM tts_results -LEFT OUTER JOIN tts_models - ON tts_results.model_token = tts_models.token -LEFT OUTER JOIN users - ON tts_results.maybe_creator_user_token = users.token -WHERE - users.username = ? - AND tts_results.user_deleted_at IS NULL - AND tts_results.mod_deleted_at IS NULL - "#, - scope_creator_username) - .fetch_all(mysql_pool) - .await? - }; - - Ok(maybe_results) -} - -#[derive(sqlx::FromRow)] -pub struct RawInternalTtsRecord { - pub tts_result_id: i64, - pub tts_result_token: String, - - pub tts_model_token: String, - pub raw_inference_text: String, - - pub maybe_creator_user_token : Option, - pub maybe_creator_username: Option, - pub maybe_creator_display_name: Option, - - pub file_size_bytes : i64, - pub duration_millis : i64, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -/// Paginated queries. -pub async fn list_tts_inference_page( - mysql_pool: &MySqlPool, - scope_creator_username: Option<&str>, - sort_ascending: bool, - cursor_is_previous: bool, - block_mod_disabled : bool, - limit: u16, - offset: Option, -) -> AnyhowResult { - - let inference_results = list_tts_inference_results_query( - mysql_pool, - scope_creator_username, - sort_ascending, - cursor_is_previous, - block_mod_disabled, - limit, - offset - ).await?; - - let mut first_id = inference_results.first() - .map(|raw_result| raw_result.tts_result_id); - - let mut last_id = inference_results.last() - .map(|raw_result| raw_result.tts_result_id); - - let inference_results = inference_results - .iter() - .map(|r| { - TtsInferenceRecordForList { - tts_result_token: r.tts_result_token.clone(), - tts_model_token: r.tts_model_token.clone(), - raw_inference_text: r.raw_inference_text.clone(), - maybe_creator_user_token: r.maybe_creator_user_token.clone(), - maybe_creator_username: r.maybe_creator_username.clone(), - maybe_creator_display_name: r.maybe_creator_display_name.clone(), - file_size_bytes: if r.file_size_bytes > 0 { r.file_size_bytes as u32 } else { 0 }, - duration_millis: if r.duration_millis > 0 { r.duration_millis as u32 } else { 0 }, - created_at: r.created_at, - updated_at: r.updated_at, - } - }) - .collect::>(); - - Ok(TtsInferenceListPage { - inference_records: inference_results, - sort_ascending, - first_id, - last_id, - }) -} - -async fn list_tts_inference_results_query ( - mysql_pool: &MySqlPool, - scope_creator_username: Option<&str>, - sort_ascending: bool, - cursor_is_previous: bool, - block_mod_disabled : bool, - limit: u16, - offset: Option, -) -> AnyhowResult> { - info!("listing tts inference results"); - - // TODO/NB: Unfortunately SQLx can't statically typecheck this query - let mut query = r#" -SELECT - tts_results.id as tts_result_id, - tts_results.token as tts_result_token, - - tts_results.model_token as tts_model_token, - tts_results.raw_inference_text as raw_inference_text, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - tts_results.file_size_bytes, - tts_results.duration_millis, - tts_results.created_at, - tts_results.updated_at - -FROM tts_results -LEFT OUTER JOIN tts_models - ON tts_results.model_token = tts_models.token -LEFT OUTER JOIN users - ON tts_results.maybe_creator_user_token = users.token - "#.to_string(); - - let mut sort_ascending = sort_ascending; // NB: We may need to change the direction. - let mut reverse_results = false; - - let mut first_predicate_added = false; - - if let Some(offset) = offset { - if !first_predicate_added { - query.push_str(" WHERE "); - first_predicate_added = true; - } else { - query.push_str(" AND "); - } - - // TODO: Cleanup, builder class, tests. - - if sort_ascending { - if cursor_is_previous { - // NB: We're searching backwards. - query.push_str(" tts_results.id < ? "); - sort_ascending = !sort_ascending; - reverse_results = true; - } else { - query.push_str(" tts_results.id > ? "); - } - } else { - if cursor_is_previous { - // NB: We're searching backwards. - query.push_str(" tts_results.id > ? "); - sort_ascending = !sort_ascending; - reverse_results = true; - } else { - query.push_str(" tts_results.id < ? "); - } - } - } - - if let Some(username) = scope_creator_username { - if !first_predicate_added { - query.push_str(r#" WHERE users.username = ? "#); - first_predicate_added = true; - } else { - query.push_str(r#" AND users.username = ? "#); - } - } - - if block_mod_disabled { - if !first_predicate_added { - query.push_str(r#" - WHERE tts_results.user_deleted_at IS NULL - AND tts_results.mod_deleted_at IS NULL - "#); - first_predicate_added = true; - } else { - query.push_str(r#" - AND tts_results.user_deleted_at IS NULL - AND tts_results.mod_deleted_at IS NULL - "#); - } - } - - if sort_ascending { - query.push_str(" ORDER BY tts_results.id ASC "); - } else { - query.push_str(" ORDER BY tts_results.id DESC "); - } - - query.push_str(" LIMIT ? "); - - let mut query = sqlx::query_as::<_, RawInternalTtsRecord>(&query); - - if let Some(offset) = offset { - query = query.bind(offset); - } - - if let Some(username) = scope_creator_username { - query = query.bind(username); - } - - query = query.bind(limit); - - let mut results = query.fetch_all(mysql_pool) - .await?; - - if reverse_results { - results.reverse() - } - - Ok(results) -} - -/// These are very difficult queries, so this helps. -pub struct QueryBuilder { +/// Query Builder for listing TTS results. +/// These are very difficult queries, so the builder helps for +/// testability, construction, and correctness. +pub struct ListTtsResultsQueryBuilder { scope_creator_username: Option, include_mod_disabled_results: bool, sort_ascending: bool, @@ -474,7 +50,7 @@ pub struct QueryBuilder { cursor_is_reversed: bool, } -impl QueryBuilder { +impl ListTtsResultsQueryBuilder { pub fn new() -> Self { Self { scope_creator_username: None, @@ -517,7 +93,50 @@ impl QueryBuilder { } /// Perform the query based on the set predicates. - pub async fn perform_query( + /// Wrap the results in a "Page" we can return to the frontend. + /// This will remove any IDs from the records, but the ids themselves also need scrubbing. + pub async fn perform_query_for_page( + &self, + mysql_pool: &MySqlPool + ) -> AnyhowResult { + + let internal_results = self.perform_internal_query(mysql_pool).await?; + + let mut first_id = internal_results.first() + .map(|raw_result| raw_result.tts_result_id); + + let mut last_id = internal_results.last() + .map(|raw_result| raw_result.tts_result_id); + + let inference_results = internal_results + .iter() + .map(|r| { + TtsInferenceRecordForList { + tts_result_token: r.tts_result_token.clone(), + tts_model_token: r.tts_model_token.clone(), + raw_inference_text: r.raw_inference_text.clone(), + maybe_creator_user_token: r.maybe_creator_user_token.clone(), + maybe_creator_username: r.maybe_creator_username.clone(), + maybe_creator_display_name: r.maybe_creator_display_name.clone(), + file_size_bytes: if r.file_size_bytes > 0 { r.file_size_bytes as u32 } else { 0 }, + duration_millis: if r.duration_millis > 0 { r.duration_millis as u32 } else { 0 }, + created_at: r.created_at, + updated_at: r.updated_at, + } + }) + .collect::>(); + + Ok(TtsInferenceListPage { + inference_records: inference_results, + sort_ascending: self.sort_ascending, + first_id, + last_id, + }) + } + + /// Perform the query based on the set predicates. + /// The records contain IDs, which we should not expose to the frontend. + pub async fn perform_internal_query( &self, mysql_pool: &MySqlPool ) -> AnyhowResult> { @@ -644,13 +263,31 @@ LEFT OUTER JOIN users } } +#[derive(sqlx::FromRow)] +pub struct RawInternalTtsRecord { + pub tts_result_id: i64, + pub tts_result_token: String, + + pub tts_model_token: String, + pub raw_inference_text: String, + + pub maybe_creator_user_token : Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + + pub file_size_bytes : i64, + pub duration_millis : i64, + pub created_at: DateTime, + pub updated_at: DateTime, +} + #[cfg(test)] mod tests { - use crate::common_queries::list_tts_inference_results::QueryBuilder; + use crate::common_queries::list_tts_inference_results::ListTtsResultsQueryBuilder; #[test] fn predicates_without_scoping() { - let query_builder = QueryBuilder::new(); + let query_builder = ListTtsResultsQueryBuilder::new(); assert_eq!(&query_builder.build_predicates(), " WHERE tts_results.user_deleted_at IS NULL \ @@ -661,7 +298,7 @@ mod tests { #[test] fn predicates_scoped_to_user() { - let query_builder = QueryBuilder::new() + let query_builder = ListTtsResultsQueryBuilder::new() .scope_creator_username(Some("echelon")); assert_eq!(&query_builder.build_predicates(), @@ -674,7 +311,7 @@ mod tests { #[test] fn predicates_including_deleted() { - let query_builder = QueryBuilder::new() + let query_builder = ListTtsResultsQueryBuilder::new() .include_mod_disabled_results(true); assert_eq!(&query_builder.build_predicates(), @@ -684,7 +321,7 @@ mod tests { #[test] fn predicates_sort_ascending() { - let query_builder = QueryBuilder::new() + let query_builder = ListTtsResultsQueryBuilder::new() .sort_ascending(true); assert_eq!(&query_builder.build_predicates(), @@ -696,7 +333,7 @@ mod tests { #[test] fn predicates_offset() { - let query_builder = QueryBuilder::new() + let query_builder = ListTtsResultsQueryBuilder::new() .offset(Some(100)); assert_eq!(&query_builder.build_predicates(), @@ -709,7 +346,7 @@ mod tests { #[test] fn predicates_offset_and_sort_ascending() { - let query_builder = QueryBuilder::new() + let query_builder = ListTtsResultsQueryBuilder::new() .sort_ascending(true) .offset(Some(100)); @@ -723,7 +360,7 @@ mod tests { #[test] fn predicates_limit() { - let query_builder = QueryBuilder::new() + let query_builder = ListTtsResultsQueryBuilder::new() .limit(15); // NB: Does not change the query itself! Just the downstream binding. @@ -736,7 +373,7 @@ mod tests { #[test] fn predicates_cursor_is_reversed_without_cursor() { - let query_builder = QueryBuilder::new() + let query_builder = ListTtsResultsQueryBuilder::new() .cursor_is_reversed(true); // NB: Without a cursor, nothing happens. @@ -749,7 +386,7 @@ mod tests { #[test] fn predicates_offset_cursor_is_reversed() { - let query_builder = QueryBuilder::new() + let query_builder = ListTtsResultsQueryBuilder::new() .offset(Some(100)) .cursor_is_reversed(true); @@ -764,7 +401,7 @@ mod tests { #[test] fn predicates_offset_cursor_is_reversed_sort_ascending() { - let query_builder = QueryBuilder::new() + let query_builder = ListTtsResultsQueryBuilder::new() .offset(Some(100)) .cursor_is_reversed(true) .sort_ascending(true); @@ -780,7 +417,7 @@ mod tests { #[test] fn predicates_limit_scope_user_offset_cursor_is_reversed_sort_ascending() { - let query_builder = QueryBuilder::new() + let query_builder = ListTtsResultsQueryBuilder::new() .limit(1000) .scope_creator_username(Some("pikachu")) .offset(Some(100)) diff --git a/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/src/http_server/endpoints/users/list_user_tts_inference_results.rs index f8d4549f7a..07383589cd 100644 --- a/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -8,12 +8,13 @@ use actix_web::web::{Path, Query}; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; +use crate::common_queries::list_tts_inference_results::ListTtsResultsQueryBuilder; use crate::common_queries::list_tts_inference_results::TtsInferenceRecordForList; -use crate::common_queries::list_tts_inference_results::list_tts_inference_page; use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; @@ -27,7 +28,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] @@ -40,7 +40,7 @@ pub struct ListTtsInferenceResultsForUserQuery { pub sort_ascending: Option, pub limit: Option, pub cursor: Option, - pub cursor_is_previous: Option, + pub cursor_is_reversed: Option, } #[derive(Serialize)] @@ -81,14 +81,11 @@ pub async fn list_user_tts_inference_results_handler( { info!("Fetching inference results for user: {}", &path.username); - let mod_disabled = false; - let limit = query.limit.unwrap_or(25); //let limit = std::cmp::max(limit, 100); let sort_ascending = query.sort_ascending.unwrap_or(false); - - let cursor_is_previous = query.cursor_is_previous.unwrap_or(false); + let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); let cursor = if let Some(cursor) = query.cursor.as_deref() { let cursor = server_state.sort_key_crypto.decrypt_id(cursor) @@ -101,15 +98,18 @@ pub async fn list_user_tts_inference_results_handler( None }; - let query_results = list_tts_inference_page( - &server_state.mysql_pool, - Some(path.username.as_ref()), - sort_ascending, - cursor_is_previous, - mod_disabled, - limit, - cursor, - ).await; + let include_mod_disabled = false; // TODO: Permission check + + let mut query_builder = ListTtsResultsQueryBuilder::new() + .sort_ascending(sort_ascending) + .scope_creator_username(Some(path.username.as_ref())) + .include_mod_disabled_results(include_mod_disabled) + .limit(limit) + .cursor_is_reversed(cursor_is_reversed) + .offset(cursor); + + let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool) + .await; let results_page = match query_results { Ok(results) => results, From a2e4098fd6ed7fb7f82e556864463b94e76704b4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 22:12:17 -0400 Subject: [PATCH 0389/1042] mod check (manually tested the cursors - they work!) --- .../users/list_user_tts_inference_results.rs | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/src/http_server/endpoints/users/list_user_tts_inference_results.rs index 07383589cd..a50add629f 100644 --- a/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -81,6 +81,24 @@ pub async fn list_user_tts_inference_results_handler( { info!("Fetching inference results for user: {}", &path.username); + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListTtsInferenceResultsForUserError::ServerError + })?; + + let mut is_mod_that_can_see_deleted = false; + + match maybe_user_session { + None => {}, + Some(session) => { + is_mod_that_can_see_deleted = session.can_delete_other_users_tts_results; + }, + }; + let limit = query.limit.unwrap_or(25); //let limit = std::cmp::max(limit, 100); @@ -98,12 +116,10 @@ pub async fn list_user_tts_inference_results_handler( None }; - let include_mod_disabled = false; // TODO: Permission check - let mut query_builder = ListTtsResultsQueryBuilder::new() .sort_ascending(sort_ascending) .scope_creator_username(Some(path.username.as_ref())) - .include_mod_disabled_results(include_mod_disabled) + .include_mod_disabled_results(is_mod_that_can_see_deleted) .limit(limit) .cursor_is_reversed(cursor_is_reversed) .offset(cursor); From 255f54c39a4ff246c1f98307e50a981ddebbae50 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 23:27:03 -0400 Subject: [PATCH 0390/1042] all predicates for W2L pagination (endpoint is no-op temporarily) --- .../list_w2l_inference_results.rs | 662 ++++++++++++------ .../users/list_user_w2l_inference_results.rs | 8 +- 2 files changed, 442 insertions(+), 228 deletions(-) diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/common_queries/list_w2l_inference_results.rs index 364a34b9d9..334334722e 100644 --- a/src/common_queries/list_w2l_inference_results.rs +++ b/src/common_queries/list_w2l_inference_results.rs @@ -1,9 +1,22 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use crate::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; +#[derive(Serialize)] +pub struct W2lInferenceListPage { + pub inference_records: Vec, + pub sort_ascending: bool, + + /// ID of the first record in `inference_records`. + pub first_id: Option, + + /// ID of the last record in `inference_records`. + pub last_id: Option, +} + #[derive(Serialize)] pub struct W2lInferenceRecordForList { pub w2l_result_token: String, @@ -29,8 +42,273 @@ pub struct W2lInferenceRecordForList { pub updated_at: DateTime, } -struct RawW2lInferenceRecordForList { - pub w2l_result_token: String, // from field `w2l_results.token` +/// Query Builder for listing W2L results. +/// These are very difficult queries, so the builder helps for +/// testability, construction, and correctness. +pub struct ListW2lResultsQueryBuilder { + scope_creator_username: Option, + include_mod_deleted_results: bool, + include_user_deleted_results: bool, + include_templates_not_approved_for_public_listing: bool, + sort_ascending: bool, + offset: Option, + limit: u16, + cursor_is_reversed: bool, +} + +impl ListW2lResultsQueryBuilder { + pub fn new() -> Self { + Self { + scope_creator_username: None, + include_mod_deleted_results: false, + include_user_deleted_results: false, + include_templates_not_approved_for_public_listing: false, + sort_ascending: false, + offset: None, + limit: DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE, + cursor_is_reversed: false, + } + } + + pub fn scope_creator_username(mut self, scope_creator_username: Option<&str>) -> Self { + self.scope_creator_username = scope_creator_username.map(|u| u.to_string()); + self + } + + pub fn include_mod_deleted_results(mut self, include_mod_deleted_results: bool) -> Self { + self.include_mod_deleted_results = include_mod_deleted_results; + self + } + + pub fn include_user_deleted_results(mut self, include_user_deleted_results: bool) -> Self { + self.include_user_deleted_results = include_user_deleted_results; + self + } + + pub fn include_templates_not_approved_for_public_listing(mut self, include_templates_not_approved_for_public_listing: bool) -> Self { + self.include_templates_not_approved_for_public_listing = include_templates_not_approved_for_public_listing; + self + } + + pub fn sort_ascending(mut self, sort_ascending: bool) -> Self { + self.sort_ascending = sort_ascending; + self + } + + pub fn offset(mut self, offset: Option) -> Self { + self.offset = offset; + self + } + + pub fn limit(mut self, limit: u16) -> Self { + self.limit = limit; + self + } + + pub fn cursor_is_reversed(mut self, cursor_is_reversed: bool) -> Self { + self.cursor_is_reversed = cursor_is_reversed; + self + } + + /// Perform the query based on the set predicates. + /// Wrap the results in a "Page" we can return to the frontend. + /// This will remove any IDs from the records, but the ids themselves also need scrubbing. + pub async fn perform_query_for_page( + &self, + mysql_pool: &MySqlPool + ) -> AnyhowResult { + + let internal_results = self.perform_internal_query(mysql_pool).await?; + + let mut first_id = internal_results.first() + .map(|raw_result| raw_result.w2l_result_id); + + let mut last_id = internal_results.last() + .map(|raw_result| raw_result.w2l_result_id); + + let inference_results = internal_results + .iter() + .map(|r| { + W2lInferenceRecordForList { + w2l_result_token: r.w2l_result_token.clone(), + maybe_w2l_template_token: r.maybe_w2l_template_token.clone(), + maybe_tts_inference_result_token: r.maybe_tts_inference_result_token.clone(), + template_type: r.template_type.clone(), + template_title: r.template_title.clone(), + maybe_creator_user_token: r.maybe_creator_user_token.clone(), + maybe_creator_username: r.maybe_creator_username.clone(), + maybe_creator_display_name: r.maybe_creator_display_name.clone(), + file_size_bytes: if r.file_size_bytes > 0 { r.file_size_bytes as u32 } else { 0 }, + frame_width: if r.frame_width > 0 { r.frame_width as u32 } else { 0 }, + frame_height: if r.frame_height > 0 { r.frame_height as u32 } else { 0 }, + duration_millis: if r.duration_millis > 0 { r.duration_millis as u32 } else { 0 }, + created_at: r.created_at, + updated_at: r.updated_at, + } + }) + .collect::>(); + + Ok(W2lInferenceListPage { + inference_records: inference_results, + sort_ascending: self.sort_ascending, + first_id, + last_id, + }) + } + + /// Perform the query based on the set predicates. + /// The records contain IDs, which we should not expose to the frontend. + pub async fn perform_internal_query( + &self, + mysql_pool: &MySqlPool + ) -> AnyhowResult> { + + let query = self.build_query_string(); + let mut query = sqlx::query_as::<_, RawInternalTtsRecord>(&query); + + // NB: The following bindings must match the order of the query builder !! + + if let Some(offset) = self.offset { + query = query.bind(offset); + } + + if let Some(username) = self.scope_creator_username.as_deref() { + query = query.bind(username); + } + + query = query.bind(self.limit); + + let mut results = query.fetch_all(mysql_pool) + .await?; + + if self.cursor_is_reversed { + results.reverse() + } + + Ok(results) + } + + pub fn build_query_string(&self) -> String { + // TODO/NB: Unfortunately SQLx can't statically typecheck this query + let mut query = r#" +SELECT + w2l_results.id as w2l_result_id, + w2l_results.token as w2l_result_token, + w2l_results.maybe_tts_inference_result_token, + + w2l_templates.token as maybe_w2l_template_token, + w2l_templates.template_type, + w2l_templates.title as template_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + w2l_results.file_size_bytes, + w2l_results.frame_width, + w2l_results.frame_height, + w2l_results.duration_millis, + w2l_results.created_at, + w2l_results.updated_at + +FROM w2l_results +LEFT OUTER JOIN w2l_templates + ON w2l_results.maybe_w2l_template_token = w2l_templates.token +LEFT OUTER JOIN users + ON w2l_results.maybe_creator_user_token = users.token + "#.to_string(); + + query.push_str(&self.build_predicates()); + query + } + + pub fn build_predicates(&self) -> String { + // NB: Reverse cursors require us to invert the sort direction. + let mut sort_ascending = self.sort_ascending; + + let mut first_predicate_added = false; + + let mut query = "".to_string(); + + if let Some(offset) = self.offset { + if !first_predicate_added { + query.push_str(" WHERE"); + first_predicate_added = true; + } else { + query.push_str(" AND"); + } + + if sort_ascending { + if self.cursor_is_reversed { + // NB: We're searching backwards. + query.push_str(" w2l_results.id < ?"); + sort_ascending = !sort_ascending; + } else { + query.push_str(" w2l_results.id > ?"); + } + } else { + if self.cursor_is_reversed { + // NB: We're searching backwards. + query.push_str(" w2l_results.id > ?"); + sort_ascending = !sort_ascending; + } else { + query.push_str(" w2l_results.id < ?"); + } + } + } + + if let Some(username) = self.scope_creator_username.as_deref() { + if !first_predicate_added { + query.push_str(" WHERE users.username = ?"); + first_predicate_added = true; + } else { + query.push_str(" AND users.username = ?"); + } + } + + if !self.include_templates_not_approved_for_public_listing { + if !first_predicate_added { + query.push_str(" WHERE w2l_templates.is_public_listing_approved IS TRUE"); + first_predicate_added = true; + } else { + query.push_str(" AND w2l_templates.is_public_listing_approved IS TRUE"); + } + } + + if !self.include_mod_deleted_results { + if !first_predicate_added { + query.push_str(" AND w2l_results.mod_deleted_at IS NULL"); + first_predicate_added = true; + } else { + query.push_str(" AND w2l_results.mod_deleted_at IS NULL"); + } + } + + if !self.include_user_deleted_results { + if !first_predicate_added { + query.push_str(" WHERE w2l_results.user_deleted_at IS NULL"); + first_predicate_added = true; + } else { + query.push_str(" AND w2l_results.user_deleted_at IS NULL"); + } + } + + if sort_ascending { + query.push_str(" ORDER BY w2l_results.id ASC"); + } else { + query.push_str(" ORDER BY w2l_results.id DESC"); + } + + query.push_str(" LIMIT ?"); + + query + } +} + +#[derive(sqlx::FromRow)] +pub struct RawInternalTtsRecord { + pub w2l_result_id: i64, + pub w2l_result_token: String, pub maybe_w2l_template_token: Option, pub maybe_tts_inference_result_token: Option, @@ -54,235 +332,169 @@ struct RawW2lInferenceRecordForList { pub updated_at: DateTime, } -pub async fn list_w2l_inference_results( - mysql_pool: &MySqlPool, - scope_creator_username: Option<&str>, - require_mod_approved_templates: bool -) -> AnyhowResult> { +#[cfg(test)] +mod tests { + use crate::common_queries::list_w2l_inference_results::ListW2lResultsQueryBuilder; - let maybe_results = match scope_creator_username { - Some(username) => { - list_w2l_inference_results_creator_scoped(mysql_pool, username, require_mod_approved_templates) - .await - }, - None => { - list_w2l_inference_results_for_all_creators(mysql_pool, require_mod_approved_templates) - .await - }, - }; + #[test] + fn predicates_without_scoping() { + let query_builder = ListW2lResultsQueryBuilder::new(); - let results : Vec = match maybe_results { - Ok(results) => { - info!("Results length: {}", results.len()); - results - }, - Err(err) => { - warn!("Error: {:?}", err); + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } - match err { - RowNotFound => { - return Ok(Vec::new()); - }, - _ => { - warn!("w2l inference result list query error: {:?}", err); - return Err(anyhow!("w2l inference result list query error")); - } - } - } - }; + #[test] + fn predicates_scoped_to_user() { + let query_builder = ListW2lResultsQueryBuilder::new() + .scope_creator_username(Some("echelon")); - Ok(results.into_iter() - .map(|ir| { - W2lInferenceRecordForList { - w2l_result_token: ir.w2l_result_token.clone(), - maybe_w2l_template_token: ir.maybe_w2l_template_token.clone(), - maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), + assert_eq!(&query_builder.build_predicates(), + " WHERE users.username = ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } - template_type: ir.template_type.clone(), - template_title: ir.template_title.clone(), + // TODO: Insufficient testing here. + #[test] + fn predicates_including_deleted_and_unapproved() { + let query_builder = ListW2lResultsQueryBuilder::new() + .include_mod_deleted_results(true) + .include_user_deleted_results(true) + .include_templates_not_approved_for_public_listing(true); - maybe_creator_user_token: ir.maybe_creator_user_token.clone(), - maybe_creator_username: ir.maybe_creator_username.clone(), - maybe_creator_display_name: ir.maybe_creator_display_name.clone(), - //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, + assert_eq!(&query_builder.build_predicates(), + " ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } - file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, - frame_width: if ir.frame_width > 0 { ir.frame_width as u32 } else { 0 }, - frame_height: if ir.frame_height > 0 { ir.frame_height as u32 } else { 0 }, - duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, + #[test] + fn predicates_sort_ascending() { + let query_builder = ListW2lResultsQueryBuilder::new() + .sort_ascending(true); - created_at: ir.created_at.clone(), - updated_at: ir.updated_at.clone(), - } - }) - .collect::>()) -} - -async fn list_w2l_inference_results_for_all_creators( - mysql_pool: &MySqlPool, - require_mod_approved_templates: bool -) -> AnyhowResult> { - // TODO: There has to be a better way. - // Sqlx doesn't like anything except string literals. - let maybe_results = if require_mod_approved_templates { - info!("listing w2l inference results for everyone; mod-approved only"); - sqlx::query_as!( - RawW2lInferenceRecordForList, - r#" -SELECT - w2l_results.token as w2l_result_token, - w2l_results.maybe_tts_inference_result_token, - - w2l_templates.token as maybe_w2l_template_token, - w2l_templates.template_type, - w2l_templates.title as template_title, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - w2l_results.file_size_bytes, - w2l_results.frame_width, - w2l_results.frame_height, - w2l_results.duration_millis, - w2l_results.created_at, - w2l_results.updated_at - -FROM w2l_results -LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token -LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token -WHERE - w2l_templates.is_public_listing_approved IS TRUE - AND w2l_results.user_deleted_at IS NULL - AND w2l_results.mod_deleted_at IS NULL - "#) - .fetch_all(mysql_pool) - .await? - } else { - info!("listing w2l inference results for everyone; all"); - sqlx::query_as!( - RawW2lInferenceRecordForList, - r#" -SELECT - w2l_results.token as w2l_result_token, - w2l_results.maybe_tts_inference_result_token, - - w2l_templates.token as maybe_w2l_template_token, - w2l_templates.template_type, - w2l_templates.title as template_title, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - w2l_results.file_size_bytes, - w2l_results.frame_width, - w2l_results.frame_height, - w2l_results.duration_millis, - w2l_results.created_at, - w2l_results.updated_at - -FROM w2l_results -LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token -LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token -WHERE - w2l_results.user_deleted_at IS NULL - AND w2l_results.mod_deleted_at IS NULL - "#) - .fetch_all(mysql_pool) - .await? - }; - - Ok(maybe_results) -} - -async fn list_w2l_inference_results_creator_scoped( - mysql_pool: &MySqlPool, - scope_creator_username: &str, - require_mod_approved_templates: bool -) -> AnyhowResult> { - // TODO: There has to be a better way. - // Sqlx doesn't like anything except string literals. - let maybe_results = if require_mod_approved_templates { - info!("listing w2l inference results for user `{}`; mod-approved only", scope_creator_username); - sqlx::query_as!( - RawW2lInferenceRecordForList, - r#" -SELECT - w2l_results.token as w2l_result_token, - w2l_results.maybe_tts_inference_result_token, - - w2l_templates.token as maybe_w2l_template_token, - w2l_templates.template_type, - w2l_templates.title as template_title, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - w2l_results.file_size_bytes, - w2l_results.frame_width, - w2l_results.frame_height, - w2l_results.duration_millis, - w2l_results.created_at, - w2l_results.updated_at - -FROM w2l_results -LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token -LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token -WHERE - users.username = ? - AND w2l_templates.is_public_listing_approved IS TRUE - AND w2l_results.user_deleted_at IS NULL - AND w2l_results.mod_deleted_at IS NULL - "#, - scope_creator_username) - .fetch_all(mysql_pool) - .await? - } else { - info!("listing w2l inference results for user `{}`; all", scope_creator_username); - sqlx::query_as!( - RawW2lInferenceRecordForList, - r#" -SELECT - w2l_results.token as w2l_result_token, - w2l_results.maybe_tts_inference_result_token, - - w2l_templates.token as maybe_w2l_template_token, - w2l_templates.template_type, - w2l_templates.title as template_title, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - w2l_results.file_size_bytes, - w2l_results.frame_width, - w2l_results.frame_height, - w2l_results.duration_millis, - w2l_results.created_at, - w2l_results.updated_at - -FROM w2l_results -LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token -LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token -WHERE - users.username = ? - AND w2l_results.user_deleted_at IS NULL - AND w2l_results.mod_deleted_at IS NULL - "#, - scope_creator_username) - .fetch_all(mysql_pool) - .await? - }; - - Ok(maybe_results) + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id ASC \ + LIMIT ?"); + } + + #[test] + fn predicates_offset() { + let query_builder = ListW2lResultsQueryBuilder::new() + .offset(Some(100)); + + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.id < ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_offset_and_sort_ascending() { + let query_builder = ListW2lResultsQueryBuilder::new() + .sort_ascending(true) + .offset(Some(100)); + + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.id > ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id ASC \ + LIMIT ?"); + } + + #[test] + fn predicates_limit() { + let query_builder = ListW2lResultsQueryBuilder::new() + .limit(15); + + // NB: Does not change the query itself! Just the downstream binding. + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_cursor_is_reversed_without_cursor() { + let query_builder = ListW2lResultsQueryBuilder::new() + .cursor_is_reversed(true); + + // NB: Without a cursor, nothing happens. + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_offset_cursor_is_reversed() { + let query_builder = ListW2lResultsQueryBuilder::new() + .offset(Some(100)) + .cursor_is_reversed(true); + + // NB: This will change the sort order and greater/less than direction! + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.id > ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id ASC \ + LIMIT ?"); + } + + #[test] + fn predicates_offset_cursor_is_reversed_sort_ascending() { + let query_builder = ListW2lResultsQueryBuilder::new() + .offset(Some(100)) + .cursor_is_reversed(true) + .sort_ascending(true); + + // NB: This will change the sort order and greater/less than direction! + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.id < ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_limit_scope_user_offset_cursor_is_reversed_sort_ascending() { + let query_builder = ListW2lResultsQueryBuilder::new() + .limit(1000) + .scope_creator_username(Some("pikachu")) + .offset(Some(100)) + .cursor_is_reversed(true) + .sort_ascending(true); + + // NB: This will change the sort order and greater/less than direction! + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.id < ? \ + AND users.username = ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } } diff --git a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs index 14a27a268d..d8cd35f2f8 100644 --- a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs @@ -8,7 +8,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::common_queries::list_w2l_inference_results::W2lInferenceRecordForList; -use crate::common_queries::list_w2l_inference_results::list_w2l_inference_results; +use crate::common_queries::list_w2l_inference_results::ListW2lResultsQueryBuilder; use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; @@ -70,7 +70,7 @@ pub async fn list_user_w2l_inference_results_handler( { info!("Fetching inference results for user: {}", &path.username); - let query_results = list_w2l_inference_results( + /*let query_results = list_w2l_inference_results( &server_state.mysql_pool, Some(path.username.as_ref()), false @@ -82,7 +82,9 @@ pub async fn list_user_w2l_inference_results_handler( warn!("Query error: {:?}", e); return Err(ListW2lInferenceResultsForUserError::ServerError); } - }; + };*/ + + let results = Vec::new(); let response = ListW2lInferenceResultsForUserSuccessResponse { success: true, From 6e4f1225a6686b33d96f96b3f4855927d4855c43 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 23:41:35 -0400 Subject: [PATCH 0391/1042] fix endpoint, fix bug, add tests (not tested in browser) --- .../list_w2l_inference_results.rs | 77 +++++++++++- .../users/list_user_tts_inference_results.rs | 1 + .../users/list_user_w2l_inference_results.rs | 112 +++++++++++++++--- 3 files changed, 172 insertions(+), 18 deletions(-) diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/common_queries/list_w2l_inference_results.rs index 334334722e..910a43df11 100644 --- a/src/common_queries/list_w2l_inference_results.rs +++ b/src/common_queries/list_w2l_inference_results.rs @@ -277,7 +277,7 @@ LEFT OUTER JOIN users if !self.include_mod_deleted_results { if !first_predicate_added { - query.push_str(" AND w2l_results.mod_deleted_at IS NULL"); + query.push_str(" WHERE w2l_results.mod_deleted_at IS NULL"); first_predicate_added = true; } else { query.push_str(" AND w2l_results.mod_deleted_at IS NULL"); @@ -362,9 +362,80 @@ mod tests { LIMIT ?"); } - // TODO: Insufficient testing here. #[test] - fn predicates_including_deleted_and_unapproved() { + fn predicates_including_mod_deleted() { + let query_builder = ListW2lResultsQueryBuilder::new() + .include_mod_deleted_results(true); + + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_including_user_deleted() { + let query_builder = ListW2lResultsQueryBuilder::new() + .include_user_deleted_results(true); + + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_including_mod_deleted_and_user_deleted() { + let query_builder = ListW2lResultsQueryBuilder::new() + .include_mod_deleted_results(true) + .include_user_deleted_results(true); + + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_including_unapproved() { + let query_builder = ListW2lResultsQueryBuilder::new() + .include_templates_not_approved_for_public_listing(true); + + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_including_unapproved_and_mod_deleted() { + let query_builder = ListW2lResultsQueryBuilder::new() + .include_templates_not_approved_for_public_listing(true) + .include_mod_deleted_results(true); + + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_including_unapproved_and_user_deleted() { + let query_builder = ListW2lResultsQueryBuilder::new() + .include_templates_not_approved_for_public_listing(true) + .include_user_deleted_results(true); + + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.mod_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_including_mod_deleted_user_deleted_and_unapproved() { let query_builder = ListW2lResultsQueryBuilder::new() .include_mod_deleted_results(true) .include_user_deleted_results(true) diff --git a/src/http_server/endpoints/users/list_user_tts_inference_results.rs b/src/http_server/endpoints/users/list_user_tts_inference_results.rs index a50add629f..cfc3de9c07 100644 --- a/src/http_server/endpoints/users/list_user_tts_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_tts_inference_results.rs @@ -99,6 +99,7 @@ pub async fn list_user_tts_inference_results_handler( }, }; + // TODO: Enforce real maximums and defaults let limit = query.limit.unwrap_or(25); //let limit = std::cmp::max(limit, 100); diff --git a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs index d8cd35f2f8..d8adb03969 100644 --- a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs @@ -26,19 +26,29 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use actix_web::web::Path; +use actix_web::web::{Path, Query}; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; /// For the URL PathInfo #[derive(Deserialize)] -pub struct GetProfilePathInfo { +pub struct ListW2lInferenceResultsForUserPathInfo { username: String, } +#[derive(Deserialize)] +pub struct ListW2lInferenceResultsForUserQuery { + pub sort_ascending: Option, + pub limit: Option, + pub cursor: Option, + pub cursor_is_reversed: Option, +} + #[derive(Serialize)] pub struct ListW2lInferenceResultsForUserSuccessResponse { pub success: bool, pub results: Vec, + pub cursor_next: Option, + pub cursor_previous: Option, } #[derive(Debug, Display)] @@ -64,37 +74,109 @@ impl ResponseError for ListW2lInferenceResultsForUserError { pub async fn list_user_w2l_inference_results_handler( http_request: HttpRequest, - path: Path, + path: Path, + query: Query, server_state: web::Data> ) -> Result { info!("Fetching inference results for user: {}", &path.username); - /*let query_results = list_w2l_inference_results( - &server_state.mysql_pool, - Some(path.username.as_ref()), - false - ).await; + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + ListW2lInferenceResultsForUserError::ServerError + })?; - let results = match query_results { + let mut is_author = false; + let mut is_mod_that_can_see_deleted = false; + let mut is_mod_that_can_approve_w2l_templates = false; + + match maybe_user_session { + None => {}, + Some(session) => { + is_author = session.username == path.username; + is_mod_that_can_see_deleted = session.can_delete_other_users_w2l_results; + is_mod_that_can_approve_w2l_templates = session.can_approve_w2l_templates; + }, + }; + + // TODO: Enforce real maximums and defaults + let limit = query.limit.unwrap_or(25); + //let limit = std::cmp::max(limit, 100); + + let sort_ascending = query.sort_ascending.unwrap_or(false); + let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); + + let cursor = if let Some(cursor) = query.cursor.as_deref() { + let cursor = server_state.sort_key_crypto.decrypt_id(cursor) + .map_err(|e| { + warn!("crypto error: {:?}", e); + return ListW2lInferenceResultsForUserError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let include_unapproved = is_author || is_mod_that_can_approve_w2l_templates; + + let mut query_builder = ListW2lResultsQueryBuilder::new() + .sort_ascending(sort_ascending) + .scope_creator_username(Some(path.username.as_ref())) + .include_user_deleted_results(is_mod_that_can_see_deleted) + .include_mod_deleted_results(is_mod_that_can_see_deleted) + .include_templates_not_approved_for_public_listing(include_unapproved) + .limit(limit) + .cursor_is_reversed(cursor_is_reversed) + .offset(cursor); + + let query_results = query_builder.perform_query_for_page(&server_state.mysql_pool) + .await; + + let results_page = match query_results { Ok(results) => results, Err(e) => { warn!("Query error: {:?}", e); return Err(ListW2lInferenceResultsForUserError::ServerError); } - };*/ + }; - let results = Vec::new(); + let cursor_next = if let Some(id) = results_page.last_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + return ListW2lInferenceResultsForUserError::ServerError + })?; + Some(cursor) + } else { + None + }; + + let cursor_previous = if let Some(id) = results_page.first_id { + let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) + .map_err(|e| { + warn!("crypto error: {:?}", e); + return ListW2lInferenceResultsForUserError::ServerError + })?; + Some(cursor) + } else { + None + }; let response = ListW2lInferenceResultsForUserSuccessResponse { success: true, - results, + results: results_page.inference_records, + cursor_next, + cursor_previous, }; let body = serde_json::to_string(&response) - .map_err(|e| ListW2lInferenceResultsForUserError::ServerError)?; + .map_err(|e| ListW2lInferenceResultsForUserError::ServerError)?; Ok(HttpResponse::Ok() - .content_type("application/json") - .body(body)) + .content_type("application/json") + .body(body)) } From 1a98e10a4f63d667176c49242dfdb945b367cc19 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 12 Jun 2021 19:13:40 -0400 Subject: [PATCH 0392/1042] fix --- src/job_queries/w2l_download_job_queries.rs | 2 ++ src/main.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/job_queries/w2l_download_job_queries.rs b/src/job_queries/w2l_download_job_queries.rs index 442ea0faba..7aff182411 100644 --- a/src/job_queries/w2l_download_job_queries.rs +++ b/src/job_queries/w2l_download_job_queries.rs @@ -227,6 +227,7 @@ SET description_rendered_html = '', creator_user_token = ?, creator_ip_address_creation = ?, + creator_ip_address_last_update = ?, original_download_url = ?, private_bucket_hash = ?, private_bucket_object_name = ?, @@ -246,6 +247,7 @@ SET job.title.to_string(), job.creator_user_token.clone(), job.creator_ip_address.clone(), + job.creator_ip_address.clone(), job.download_url.clone(), private_bucket_hash.to_string(), private_bucket_object_name.to_string(), diff --git a/src/main.rs b/src/main.rs index 397b3c5ae6..f1a7a1225a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -96,6 +96,8 @@ use std::sync::Arc; // TODO TODO TODO -- ON signup, add an "early adopter" badge. And a tool for making it easy to add badges. // TODO - badge for uploading template, badge for uploading model, etc. +// TODO TODO -- also this: https://material-ui.com + const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; From b380a3197c6da7f7198ccd5125fa2900014a5b83 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 01:17:31 -0400 Subject: [PATCH 0393/1042] fix markdown from containing HTML !!!! also: temporary no-entropy cursors --- src/util/encrypted_sort_id.rs | 31 +++++++++++---------- src/util/markdown_to_html.rs | 52 +++++++++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/src/util/encrypted_sort_id.rs b/src/util/encrypted_sort_id.rs index 6c10953054..aba0ef84eb 100644 --- a/src/util/encrypted_sort_id.rs +++ b/src/util/encrypted_sort_id.rs @@ -12,7 +12,8 @@ use base64::{Config, CharacterSet}; /// This gets encrypted and sent to the frontend as an opaque handle. #[derive(Serialize, Deserialize)] pub struct SortId { - pub entropy: u32, + // NB: Entropy is causing React to panic on re-renders + //pub entropy: u32, pub column_id: u64, } @@ -39,7 +40,7 @@ impl SortKeyCrypto { let mut rng = rand::thread_rng(); let payload = SortId { - entropy: rng.next_u32(), + //entropy: rng.next_u32(), column_id: id, }; @@ -72,19 +73,19 @@ mod tests { use crate::util::encrypted_sort_id::SortKeyCrypto; use std::collections::HashSet; - #[test] - fn encrypt_entropy_means_no_duplicate_values() { - let sorter = SortKeyCrypto::new("secret"); - - let mut encrypted_tokens = HashSet::new(); - - for i in 0 .. 1000 { - let encrypted = sorter.encrypt_id(1234).unwrap(); - encrypted_tokens.insert(encrypted); - } - - assert_eq!(encrypted_tokens.len(), 1000); - } +// #[test] +// fn encrypt_entropy_means_no_duplicate_values() { +// let sorter = SortKeyCrypto::new("secret"); +// +// let mut encrypted_tokens = HashSet::new(); +// +// for i in 0 .. 1000 { +// let encrypted = sorter.encrypt_id(1234).unwrap(); +// encrypted_tokens.insert(encrypted); +// } +// +// assert_eq!(encrypted_tokens.len(), 1000); +// } #[test] fn encrypt_roundtrip() { diff --git a/src/util/markdown_to_html.rs b/src/util/markdown_to_html.rs index 87bb66d58c..9b89017945 100644 --- a/src/util/markdown_to_html.rs +++ b/src/util/markdown_to_html.rs @@ -1,14 +1,60 @@ use pulldown_cmark::{Options, Parser, html}; +use crate::util::anyhow_result::AnyhowResult; pub fn markdown_to_html(markdown_input: &str) -> String { + // NB: CommonMark allows for HTML, including "), + "